diff --git a/.clippy.toml b/.clippy.toml
index 137f41d203de3..a51de9a463803 100644
--- a/.clippy.toml
+++ b/.clippy.toml
@@ -9,3 +9,13 @@ disallowed-macros = [
     # it here, see: https://github.com/rust-lang/rust-clippy/issues/11303.
     { path = "kernel::dbg", reason = "the `dbg!` macro is intended as a debugging tool", allow-invalid = true },
 ]
+
+[[disallowed-methods]]
+path = "core::ffi::CStr::as_ptr"
+replacement = "kernel::prelude::CStrExt::as_char_ptr"
+reason = "kernel's `char` is always unsigned, use `as_char_ptr` instead"
+
+[[disallowed-methods]]
+path = "core::ffi::CStr::from_ptr"
+replacement = "kernel::prelude::CStrExt::from_char_ptr"
+reason = "kernel's `char` is always unsigned, use `from_char_ptr` instead"
diff --git a/.mailmap b/.mailmap
index 63c11ea7e35d7..4009a587d32ab 100644
--- a/.mailmap
+++ b/.mailmap
@@ -305,7 +305,10 @@ Gokul Sriram Palanisamy <quic_gokulsri@quicinc.com> <gokulsri@codeaurora.org>
 Govindaraj Saminathan <quic_gsamin@quicinc.com> <gsamin@codeaurora.org>
 Guo Ren <guoren@kernel.org> <guoren@linux.alibaba.com>
 Guo Ren <guoren@kernel.org> <ren_guo@c-sky.com>
-Guru Das Srinagesh <quic_gurus@quicinc.com> <gurus@codeaurora.org>
+Guru Das Srinagesh <linux@gurudas.dev>
+Guru Das Srinagesh <linux@gurudas.dev> <quic_gurus@quicinc.com>
+Guru Das Srinagesh <linux@gurudas.dev> <gurus@codeaurora.org>
+Guru Das Srinagesh <linux@gurudas.dev> <gurooodas@gmail.com>
 Gustavo Padovan <gustavo@las.ic.unicamp.br>
 Gustavo Padovan <padovan@profusion.mobi>
 Hamza Mahfooz <hamzamahfooz@linux.microsoft.com> <hamza.mahfooz@amd.com>
@@ -314,6 +317,7 @@ Hans de Goede <hansg@kernel.org> <hdegoede@redhat.com>
 Hans Verkuil <hverkuil@kernel.org> <hverkuil@xs4all.nl>
 Hans Verkuil <hverkuil@kernel.org> <hverkuil-cisco@xs4all.nl>
 Hans Verkuil <hverkuil@kernel.org> <hansverk@cisco.com>
+Hans Verkuil <hverkuil@kernel.org> <hans.verkuil@cisco.com>
 Hao Ge <hao.ge@linux.dev> <gehao@kylinos.cn>
 Harry Yoo <harry.yoo@oracle.com> <42.hyeyoo@gmail.com>
 Heiko Carstens <hca@linux.ibm.com> <h.carstens@de.ibm.com>
@@ -327,6 +331,7 @@ Henrik Rydberg <rydberg@bitmath.org>
 Herbert Xu <herbert@gondor.apana.org.au>
 Huacai Chen <chenhuacai@kernel.org> <chenhc@lemote.com>
 Huacai Chen <chenhuacai@kernel.org> <chenhuacai@loongson.cn>
+Ignat Korchagin <ignat@linux.win> <ignat@cloudflare.com>
 Ike Panhc <ikepanhc@gmail.com> <ike.pan@canonical.com>
 J. Bruce Fields <bfields@fieldses.org> <bfields@redhat.com>
 J. Bruce Fields <bfields@fieldses.org> <bfields@citi.umich.edu>
@@ -574,6 +579,7 @@ Michel Lespinasse <michel@lespinasse.org> <walken@google.com>
 Michel Lespinasse <michel@lespinasse.org> <walken@zoy.org>
 Mickaël Salaün <mic@digikod.net> <mic@linux.microsoft.com>
 Miguel Ojeda <ojeda@kernel.org> <miguel.ojeda.sandonis@gmail.com>
+Mike Leach <mike.leach@arm.com> <mike.leach@linaro.org>
 Mike Rapoport <rppt@kernel.org> <mike@compulab.co.il>
 Mike Rapoport <rppt@kernel.org> <mike.rapoport@gmail.com>
 Mike Rapoport <rppt@kernel.org> <rppt@linux.ibm.com>
@@ -586,6 +592,7 @@ Morten Welinder <terra@gnome.org>
 Morten Welinder <welinder@anemone.rentec.com>
 Morten Welinder <welinder@darter.rentec.com>
 Morten Welinder <welinder@troll.com>
+Muhammad Usama Anjum <usama.anjum@arm.com> <usama.anjum@collabora.com>
 Mukesh Ojha <quic_mojha@quicinc.com> <mojha@codeaurora.org>
 Muna Sinada <quic_msinada@quicinc.com> <msinada@codeaurora.org>
 Murali Nalajala <quic_mnalajal@quicinc.com> <mnalajal@codeaurora.org>
diff --git a/CREDITS b/CREDITS
index 9091bac3d2da4..885fb05d88169 100644
--- a/CREDITS
+++ b/CREDITS
@@ -71,11 +71,6 @@ D: dosfs, LILO, some fd features, ATM, various other hacks here and there
 S: Buenos Aires
 S: Argentina
 
-NTFS FILESYSTEM
-N: Anton Altaparmakov
-E: anton@tuxera.com
-D: NTFS filesystem
-
 N: Tim Alpaerts
 E: tim_alpaerts@toyota-motor-europe.com
 D: 802.2 class II logical link control layer,
@@ -85,8 +80,8 @@ S: B-2610 Wilrijk-Antwerpen
 S: Belgium
 
 N: Anton Altaparmakov
-E: aia21@cantab.net
-W: http://www-stu.christs.cam.ac.uk/~aia21/
+E: anton@tuxera.com
+W: http://www.tuxera.com/
 D: Author of new NTFS driver, various other kernel hacks.
 S: Christ's College
 S: Cambridge CB2 3BU
@@ -4560,8 +4555,5 @@ D: MD driver
 D: EISA/sysfs subsystem
 S: France
 
-# Don't add your name here, unless you really _are_ after Marc
-# alphabetically. Leonard used to be very proud of being the
-# last entry, and he'll get positively pissed if he can't even
-# be second-to-last.  (and this file really _is_ supposed to be
-# in alphabetic order)
+# Don't add your name here unless you really are last alphabetically.
+# (This file is supposed to be kept in alphabetical order by last name.)
diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block
index 09a9d4aca0fd5..900b3fc4c72d0 100644
--- a/Documentation/ABI/stable/sysfs-block
+++ b/Documentation/ABI/stable/sysfs-block
@@ -886,6 +886,21 @@ Description:
 		zone commands, they will be treated as regular block devices and
 		zoned will report "none".
 
+What:		/sys/block/<disk>/queue/zoned_qd1_writes
+Date:		January 2026
+Contact:	Damien Le Moal <dlemoal@kernel.org>
+Description:
+		[RW] zoned_qd1_writes indicates if write operations to a zoned
+		block device are being handled using a single issuer context (a
+		kernel thread) operating at a maximum queue depth of 1. This
+		attribute is visible only for zoned block devices. The default
+		value for zoned block devices that are not rotational devices
+		(e.g. ZNS SSDs or zoned UFS devices) is 0. For rotational zoned
+		block devices (e.g. SMR HDDs) the default value is 1. Since
+		this default may not be appropriate for some devices, e.g.
+		remotely connected devices over high latency networks, the user
+		can disable this feature by setting this attribute to 0.
+
 
 What:		/sys/block/<disk>/hidden
 Date:		March 2023
diff --git a/Documentation/ABI/testing/sysfs-bus-i3c b/Documentation/ABI/testing/sysfs-bus-i3c
index c1e048957a010..19f5cf8b1b114 100644
--- a/Documentation/ABI/testing/sysfs-bus-i3c
+++ b/Documentation/ABI/testing/sysfs-bus-i3c
@@ -172,3 +172,23 @@ Description:
 		the automatic retries. Exist only when I3C constroller supports
 		this retry on nack feature.
 
+What:		/sys/bus/i3c/devices/i3c-<bus-id>/do_daa
+KernelVersion:  7.0
+Contact:	linux-i3c@vger.kernel.org
+Description:
+		Write-only attribute that triggers a Dynamic Address Assignment
+		(DAA) procedure which discovers new I3C devices on the bus.
+		Writing a boolean true value (1, y, yes, true, on) to this
+		attribute causes the master controller to perform DAA, which
+		includes broadcasting an ENTDAA (Enter Dynamic Address Assignment)
+		Common Command Code (CCC) on the bus. Writing a false value
+		returns -EINVAL.
+
+		This is useful for discovering I3C devices that were not present
+		during initial bus initialization and are unable to issue
+		Hot-Join. Only devices without a currently assigned dynamic address
+		will respond to the ENTDAA broadcast and be assigned addresses.
+
+		Note that this mechanism is distinct from Hot-Join, since this is
+		controller-initiated discovery, while Hot-Join is device-initiated
+		method to provoke controller discovery procedure.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index 5f87dcee78f76..4fc9f6bd42812 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -1428,7 +1428,7 @@ KernelVersion:	2.6.35
 Contact:	linux-iio@vger.kernel.org
 Description:
 		The name of the trigger source being used, as per string given
-		in /sys/class/iio/triggerY/name.
+		in /sys/bus/iio/devices/triggerY/name.
 
 What:		/sys/bus/iio/devices/iio:deviceX/bufferY/length
 KernelVersion:	5.11
diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power
index 4b21d5d232513..32697b926cc8c 100644
--- a/Documentation/ABI/testing/sysfs-class-power
+++ b/Documentation/ABI/testing/sysfs-class-power
@@ -675,7 +675,8 @@ Description:
 
 		Valid values:
 			      "Unknown", "SDP", "DCP", "CDP", "ACA", "C", "PD",
-			      "PD_DRP", "PD_PPS", "BrickID"
+			      "PD_DRP", "PD_PPS", "BrickID", "PD_SPR_AVS",
+			      "PD_PPS_SPR_AVS"
 
 **Device Specific Properties**
 
diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
new file mode 100644
index 0000000000000..a16c54ab841bb
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
@@ -0,0 +1,36 @@
+What:		/sys/class/reboot-mode/<driver>/reboot_modes
+Date:		March 2026(TBD)
+KernelVersion:	TBD
+Contact:	linux-pm@vger.kernel.org
+		Description:
+		This interface exposes the reboot-mode arguments
+		registered with the reboot-mode framework. It is
+		a read-only interface and provides a space
+		separated list of reboot-mode arguments supported
+		on the current platform.
+		Example:
+		 recovery fastboot bootloader
+
+		The exact sysfs path may vary depending on the
+		name of the driver that registers the arguments.
+		Example:
+		 /sys/class/reboot-mode/nvmem-reboot-mode/reboot_modes
+		 /sys/class/reboot-mode/syscon-reboot-mode/reboot_modes
+		 /sys/class/reboot-mode/qcom-pon/reboot_modes
+
+		The supported arguments can be used by userspace to
+		invoke device reset using the standard reboot() system
+		call interface, with the "argument" as string to "*arg"
+		parameter along with LINUX_REBOOT_CMD_RESTART2.
+
+		A driver can expose the supported arguments by
+		registering them with the reboot-mode framework
+		using the property names that follow the
+		mode-<argument> format.
+		Example:
+		 mode-bootloader, mode-recovery.
+
+		This attribute is useful for scripts or initramfs
+		logic that need to programmatically determine
+		which reboot-mode arguments are valid before
+		triggering a reboot.
diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 3a05604c21bf8..82d10d556cc89 100644
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -327,6 +327,24 @@ Description:	Energy performance preference
 
 		This file is only present if the cppc-cpufreq driver is in use.
 
+What:		/sys/devices/system/cpu/cpuX/cpufreq/perf_limited
+Date:		February 2026
+Contact:	linux-pm@vger.kernel.org
+Description:	Performance Limited
+
+		Read to check if platform throttling (thermal/power/current
+		limits) caused delivered performance to fall below the
+		requested level. A non-zero value indicates throttling occurred.
+
+		Write the bitmask of bits to clear:
+
+		  - 0x1 = clear bit 0 (desired performance excursion)
+		  - 0x2 = clear bit 1 (minimum performance excursion)
+		  - 0x3 = clear both bits
+
+		The platform sets these bits; OSPM can only clear them.
+
+		This file is only present if the cppc-cpufreq driver is in use.
 
 What:		/sys/devices/system/cpu/cpu*/cache/index3/cache_disable_{0,1}
 Date:		August 2008
diff --git a/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go b/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go
new file mode 100644
index 0000000000000..c8221373ef76a
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go
@@ -0,0 +1,724 @@
+What:		/sys/class/leds/go:rgb:joystick_rings/effect
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the display effect of the RGB interface.
+
+		Values are monocolor, breathe, chroma, or rainbow.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/effect_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the effect attribute.
+
+		Values are monocolor, breathe, chroma, or rainbow.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the RGB interface.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the enabled attribute.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the operating mode of the RGB interface.
+
+		Values are dynamic or custom. Custom allows setting the RGB effect and color.
+    Dynamic is a Windows mode for syncing Lenovo RGB interfaces not currently
+    supported under Linux.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the mode attribute.
+
+		Values are dynamic or custom.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/profile
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls selecting the configured RGB profile.
+
+		Values are 1-3.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/profile_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the profile attribute.
+
+		Values are 1-3.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/speed
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the change rate for the breathe, chroma, and rainbow effects.
+
+		Values are 0-100.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/class/leds/go:rgb:joystick_rings/speed_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the speed attribute.
+
+		Values are 0-100.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/firmware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the firmware version of the internal MCU.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/fps_mode_dpi
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the DPI of the right handle when the FPS mode switch is on.
+
+		Values are 500, 800, 1200, and 1800.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/fps_mode_dpi_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the fps_mode_dpi attribute.
+
+		Values are 500, 800, 1200, and 1800.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/hardware_generation
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware generation of the internal MCU.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/hardware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware version of the internal MCU.
+
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/auto_sleep_time
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the sleep timer due to inactivity for the left removable controller.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/auto_sleep_time_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/auto_sleep_time attribute.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_gyro
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the left removable controller's IMU.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_gyro_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/calibrate_gyro attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_gyro_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the left removable controller's IMU.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_joystick
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the left removable controller's joystick.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_joystick_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/calibrate_jotstick attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_joystick_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the left removable controller's joystick.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_tirgger
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the left removable controller's trigger.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_gyro_trigger
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/calibrate_trigger attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/calibrate_trigger_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the left removable controller's trigger.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/firmware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the left removable controller's firmware version.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/hardware_generation
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware generation of the left removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/hardware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware version of the left removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/imu_bypass_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU bypass function of the left removable controller.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/imu_bypass_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/imu_bypass_enabled attribute.
+
+		Values are true or false.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/imu_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU of the left removable controller.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/imu_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/imu_enabled attribute.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/product_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the product version of the left removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/protocol_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the protocol version of the left removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/reset
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	Resets the left removable controller to factory defaults.
+
+		Writing 1 to this path initiates.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/rumble_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls setting the response behavior for rumble events for the left removable controller.
+
+		Values are fps, racing, standarg, spg, rpg.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/rumble_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/rumble_mode attribute.
+
+		Values are fps, racing, standarg, spg, rpg.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/rumble_notification
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling haptic rumble events for the left removable controller.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/rumble_notification_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the left_handle/rumble_notification attribute.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the operating mode of the built-in controller.
+
+		Values are xinput or dinput.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/left_handle/mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the mode attribute.
+
+		Values are xinput or dinput.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/os_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the behavior of built in chord combinations.
+
+		Values are windows or linux.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/os_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the os_mode attribute.
+
+		Values are windows or linux.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/product_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the product version of the internal MCU.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/protocol_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the protocol version of the internal MCU.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/reset_mcu
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	Resets the internal MCU to factory defaults.
+
+		Writing 1 to this path initiates.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/auto_sleep_time
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the sleep timer due to inactivity for the right removable controller.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/auto_sleep_time_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/auto_sleep_time attribute.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_gyro
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the right removable controller's IMU.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_gyro_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/calibrate_gyro attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_gyro_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the right removable controller's IMU.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_joystick
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the right removable controller's joystick.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_joystick_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/calibrate_jotstick attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_joystick_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the right removable controller's joystick.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_tirgger
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This initiates or halts calibration of the right removable controller's trigger.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_gyro_trigger
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/calibrate_trigger attribute.
+
+		Values are start, stop.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/calibrate_trigger_status
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the result of the last attempted calibration of the right removable controller's trigger.
+
+		Values are unknown, success, failure.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/firmware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the right removable controller's firmware version.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/hardware_generation
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware generation of the right removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/hardware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware version of the right removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/imu_bypass_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU bypass function of the right removable controller.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/imu_bypass_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/imu_bypass_enabled attribute.
+
+		Values are true or false.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/imu_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU of the right removable controller.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/imu_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/imu_enabled attribute.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/product_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the product version of the right removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/protocol_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the protocol version of the right removable controller.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/reset
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	Resets the right removable controller to factory defaults.
+
+		Writing 1 to this path initiates.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/rumble_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls setting the response behavior for rumble events for the right removable controller.
+
+		Values are fps, racing, standarg, spg, rpg.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/rumble_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/rumble_mode attribute.
+
+		Values are fps, racing, standarg, spg, rpg.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/rumble_notification
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling haptic rumble events for the right removable controller.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/right_handle/rumble_notification_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the right_handle/rumble_notification attribute.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/rumble_intensity
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls setting the rumble intensity for both removable controllers.
+
+		Values are off, low, medium, high.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/rumble_intensity_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the rumble_intensity attribute.
+
+		Values are off, low, medium, high.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the touchpad.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/enabled attribute.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/vibration_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling haptic rumble events for the touchpad.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/vibration_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/vibration_enabled attribute.
+
+		Values are true, false.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/vibration_intensity
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls setting the intensity of the touchpad haptics.
+
+		Values are off, low, medium, high.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/vibration_intensity_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/vibration_intensity attribute.
+
+		Values are off, low, medium, high.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/tx_dongle/firmware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the firmware version of the internal wireless transmission dongle.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/tx_dongle/hardware_generation
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware generation of the internal wireless transmission dongle.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/tx_dongle/hardware_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the hardware version of the internal wireless transmission dongle.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/tx_dongle/product_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the product version of the internal wireless transmission dongle.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/tx_dongle/protocol_version
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the protocol version of the internal wireless transmission dongle.
+
+		Applies to Lenovo Legion Go and Go 2 line of handheld devices.
+
diff --git a/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go-s b/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go-s
new file mode 100644
index 0000000000000..4d317074bb7e6
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-hid-lenovo-go-s
@@ -0,0 +1,304 @@
+What:		/sys/class/leds/go_s:rgb:joystick_rings/effect
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the display effect of the RGB interface.
+
+		Values are monocolor, breathe, chroma, or rainbow.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/effect_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the effect attribute.
+
+		Values are monocolor, breathe, chroma, or rainbow.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the RGB interface.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the enabled attribute.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the operating mode of the RGB interface.
+
+		Values are dynamic or custom. Custom allows setting the RGB effect and color.
+    Dynamic is a Windows mode for syncing Lenovo RGB interfaces not currently
+    supported under Linux.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the mode attribute.
+
+		Values are dynamic or custom.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/profile
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls selecting the configured RGB profile.
+
+		Values are 1-3.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/profile_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the profile attribute.
+
+		Values are 1-3.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/speed
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the change rate for the breathe, chroma, and rainbow effects.
+
+		Values are 0-100.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/class/leds/go_s:rgb:joystick_rings/speed_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the speed attribute.
+
+		Values are 0-100.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/auto_sleep_time
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the sleep timer due to inactivity for the built-in controller.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/auto_sleep_time_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the gamepad/auto_sleep_time attribute.
+
+		Values are 0-255.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/dpad_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the operating mode of the built-in controllers D-pad.
+
+		Values are 4-way or 8-way.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/dpad_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the gamepad/dpad_mode attribute.
+
+		Values are 4-way or 8-way.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the operating mode of the built-in controller.
+
+		Values are xinput or dinput.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the gamepad/mode attribute.
+
+		Values are xinput or dinput.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/poll_rate
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls the poll rate in Hz of the built-in controller.
+
+		Values are 125, 250, 500, or 1000.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/gamepad/poll_rate_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the gamepad/poll_rate attribute.
+
+		Values are 125, 250, 500, or 1000.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/imu/bypass_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU bypass function. When enabled the IMU data is directly reported to the OS through
+an HIDRAW interface.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/imu/bypass_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the imu/bypass_enabled attribute.
+
+		Values are true or false.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/imu/manufacturer
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the manufacturer of the intertial measurment unit.
+
+		Values are Bosch or ST.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/imu/sensor_enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the IMU.
+
+		Values are true, false, or wake-2s.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/imu/sensor_enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the imu/sensor_enabled attribute.
+
+		Values are true, false, or wake-2s.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/mcu_id
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the MCU Identification Number
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/mouse/step
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls which value is used for the mouse sensitivity.
+
+		Values are 1-127.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/mouse/step_range
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the mouse/step attribute.
+
+		Values are 1-127.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/os_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls which value is used for the touchpads operating mode.
+
+		Values are windows or linux.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/os_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the os_mode attribute.
+
+		Values are windows or linux.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/enabled
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls enabling or disabling the built-in touchpad.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/enabled_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/enabled attribute.
+
+		Values are true or false.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/linux_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls behavior of the touchpad events when os_mode is set to linux.
+
+		Values are absolute or relative.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/linux_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/linux_mode attribute.
+
+		Values are absolute or relative.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/windows_mode
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This controls behavior of the touchpad events when os_mode is set to windows.
+
+		Values are absolute or relative.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
+
+What:		/sys/bus/usb/devices/<busnum>-<devnum>:<config num>.<interface num>/<hid-bus>:<vendor-id>:<product-id>.<num>/touchpad/windows_mode_index
+Date:		April 2026
+Contact:	linux-input@vger.kernel.org
+Description:	This displays the available options for the touchpad/windows_mode attribute.
+
+		Values are absolute or relative.
+
+		Applies to Lenovo Legion Go S line of handheld devices.
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
index 7f5ef9eada531..1d6eaff6882f0 100644
--- a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
+++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
@@ -129,6 +129,37 @@ Description:
 			-EIO if FW refuses to change the provisioning.
 
 
+What:		/sys/bus/pci/drivers/xe/.../sriov_admin/.bulk_profile/vram_quota
+What:		/sys/bus/pci/drivers/xe/.../sriov_admin/vf<n>/profile/vram_quota
+Date:		February 2026
+KernelVersion:	7.0
+Contact:	intel-xe@lists.freedesktop.org
+Description:
+		These files allow to perform initial VFs VRAM provisioning prior to VFs
+		enabling or to change VFs VRAM provisioning once the VFs are enabled.
+		Any non-zero initial VRAM provisioning will block VFs auto-provisioning.
+		Without initial VRAM provisioning those files will show result of the
+		VRAM auto-provisioning performed by the PF once the VFs are enabled.
+		Once the VFs are disabled, all VRAM provisioning will be released.
+		These files are visible only on discrete Intel Xe platforms with VRAM
+		and are writeable only if dynamic VFs VRAM provisioning is supported.
+
+		.bulk_profile/vram_quota: (WO) unsigned integer
+			The amount of the provisioned VRAM in [bytes] for each VF.
+			Actual quota value might be aligned per HW/FW requirements.
+
+		profile/vram_quota: (RW) unsigned integer
+			The amount of the provisioned VRAM in [bytes] for this VF.
+			Actual quota value might be aligned per HW/FW requirements.
+
+			Default is 0 (unprovisioned).
+
+		Writes to these attributes may fail with errors like:
+			-EINVAL if provided input is malformed or not recognized,
+			-EPERM if change is not applicable on given HW/FW,
+			-EIO if FW refuses to change the provisioning.
+
+
 What:		/sys/bus/pci/drivers/xe/.../sriov_admin/vf<n>/stop
 Date:		October 2025
 KernelVersion:	6.19
diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs
index a90612ab57800..3c422aac778bf 100644
--- a/Documentation/ABI/testing/sysfs-driver-ufs
+++ b/Documentation/ABI/testing/sysfs-driver-ufs
@@ -1768,3 +1768,26 @@ Description:
 		====================   ===========================
 
 		The attribute is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/dme_qos_notification
+What:		/sys/bus/platform/devices/*.ufs/dme_qos_notification
+Date:		March 2026
+Contact:	Can Guo <can.guo@oss.qualcomm.com>
+Description:
+		This attribute reports and clears pending DME (Device Management
+		Entity) Quality of Service (QoS) notifications. This attribute
+		is a bitfield with the following bit assignments:
+
+		Bit	Description
+		===	======================================
+		0	DME QoS Monitor has been reset by host
+		1	QoS from TX is detected
+		2	QoS from RX is detected
+		3	QoS from PA_INIT is detected
+
+		Reading this attribute returns the pending DME QoS notification
+		bits. Writing '0' to this attribute clears pending DME QoS
+		notification bits. Writing any non-zero value is invalid and
+		will be rejected.
+
+		The attribute is read/write.
diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
index c1d2b3fd9c65d..423ec40e2e4e2 100644
--- a/Documentation/ABI/testing/sysfs-fs-f2fs
+++ b/Documentation/ABI/testing/sysfs-fs-f2fs
@@ -407,6 +407,12 @@ Contact:	"Hridya Valsaraju" <hridya@google.com>
 Description:	Average number of valid blocks.
 		Available when CONFIG_F2FS_STAT_FS=y.
 
+What:		/sys/fs/f2fs/<disk>/defrag_blocks
+Date:		February 2026
+Contact:	"Jinbao Liu" <liujinbao1@xiaomi.com>
+Description:	Number of blocks moved by defragment.
+		Available when CONFIG_F2FS_STAT_FS=y.
+
 What:		/sys/fs/f2fs/<disk>/mounted_time_sec
 Date:		February 2020
 Contact:	"Jaegeuk Kim" <jaegeuk@kernel.org>
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon
index f2af2ddedd323..2424237ebb105 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-damon
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon
@@ -316,6 +316,12 @@ Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the path
 		parameter of the goal.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goal_tuner
+Date:		Mar 2026
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the
+		goal-based effective quota auto-tuning algorithm to use.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/weights/sz_permil
 Date:		Mar 2022
 Contact:	SeongJae Park <sj@kernel.org>
diff --git a/Documentation/ABI/testing/sysfs-module b/Documentation/ABI/testing/sysfs-module
index 6bc9af6229f00..d5b7d19bd3103 100644
--- a/Documentation/ABI/testing/sysfs-module
+++ b/Documentation/ABI/testing/sysfs-module
@@ -48,6 +48,15 @@ Contact:	Kay Sievers <kay.sievers@vrfy.org>
 Description:	Show the initialization state(live, coming, going) of
 		the module.
 
+What:		/sys/module/*/import_ns
+Date:		January 2026
+KernelVersion:	7.1
+Contact:	linux-modules@vger.kernel.org
+Description:	List of symbol namespaces imported by this module via
+		MODULE_IMPORT_NS(). Each namespace appears on a separate line.
+		This file only exists for modules that import at least one
+		namespace.
+
 What:		/sys/module/*/taint
 Date:		Jan 2012
 KernelVersion:	3.3
diff --git a/Documentation/PCI/msi-howto.rst b/Documentation/PCI/msi-howto.rst
index 667ebe2156b47..844c1d3c395dd 100644
--- a/Documentation/PCI/msi-howto.rst
+++ b/Documentation/PCI/msi-howto.rst
@@ -113,8 +113,11 @@ vectors, use the following function::
 
   int pci_irq_vector(struct pci_dev *dev, unsigned int nr);
 
-Any allocated resources should be freed before removing the device using
-the following function::
+If the driver enables the device using pcim_enable_device(), the driver
+shouldn't call pci_free_irq_vectors() because pcim_enable_device()
+activates automatic management for IRQ vectors. Otherwise, the driver should
+free any allocated IRQ vectors before removing the device using the following
+function::
 
   void pci_free_irq_vectors(struct pci_dev *dev);
 
diff --git a/Documentation/admin-guide/blockdev/zram.rst b/Documentation/admin-guide/blockdev/zram.rst
index 451fa00d3004b..60b07a7e30cd0 100644
--- a/Documentation/admin-guide/blockdev/zram.rst
+++ b/Documentation/admin-guide/blockdev/zram.rst
@@ -462,7 +462,7 @@ know it via /sys/block/zram0/bd_stat's 3rd column.
 recompression
 -------------
 
-With CONFIG_ZRAM_MULTI_COMP, zram can recompress pages using alternative
+With `CONFIG_ZRAM_MULTI_COMP`, zram can recompress pages using alternative
 (secondary) compression algorithms. The basic idea is that alternative
 compression algorithm can provide better compression ratio at a price of
 (potentially) slower compression/decompression speeds. Alternative compression
@@ -471,7 +471,7 @@ that default algorithm failed to compress). Another application is idle pages
 recompression - pages that are cold and sit in the memory can be recompressed
 using more effective algorithm and, hence, reduce zsmalloc memory usage.
 
-With CONFIG_ZRAM_MULTI_COMP, zram supports up to 4 compression algorithms:
+With `CONFIG_ZRAM_MULTI_COMP`, zram supports up to 4 compression algorithms:
 one primary and up to 3 secondary ones. Primary zram compressor is explained
 in "3) Select compression algorithm", secondary algorithms are configured
 using recomp_algorithm device attribute.
@@ -495,56 +495,43 @@ configuration:::
 	#select deflate recompression algorithm, priority 2
 	echo "algo=deflate priority=2" > /sys/block/zramX/recomp_algorithm
 
-Another device attribute that CONFIG_ZRAM_MULTI_COMP enables is recompress,
+Another device attribute that `CONFIG_ZRAM_MULTI_COMP` enables is `recompress`,
 which controls recompression.
 
 Examples:::
 
 	#IDLE pages recompression is activated by `idle` mode
-	echo "type=idle" > /sys/block/zramX/recompress
+	echo "type=idle priority=1" > /sys/block/zramX/recompress
 
 	#HUGE pages recompression is activated by `huge` mode
-	echo "type=huge" > /sys/block/zram0/recompress
+	echo "type=huge priority=2" > /sys/block/zram0/recompress
 
 	#HUGE_IDLE pages recompression is activated by `huge_idle` mode
-	echo "type=huge_idle" > /sys/block/zramX/recompress
+	echo "type=huge_idle priority=1" > /sys/block/zramX/recompress
 
 The number of idle pages can be significant, so user-space can pass a size
 threshold (in bytes) to the recompress knob: zram will recompress only pages
 of equal or greater size:::
 
 	#recompress all pages larger than 3000 bytes
-	echo "threshold=3000" > /sys/block/zramX/recompress
+	echo "threshold=3000 priority=1" > /sys/block/zramX/recompress
 
 	#recompress idle pages larger than 2000 bytes
-	echo "type=idle threshold=2000" > /sys/block/zramX/recompress
+	echo "type=idle threshold=2000 priority=1" > \
+		/sys/block/zramX/recompress
 
 It is also possible to limit the number of pages zram re-compression will
 attempt to recompress:::
 
-	echo "type=huge_idle max_pages=42" > /sys/block/zramX/recompress
-
-During re-compression for every page, that matches re-compression criteria,
-ZRAM iterates the list of registered alternative compression algorithms in
-order of their priorities. ZRAM stops either when re-compression was
-successful (re-compressed object is smaller in size than the original one)
-and matches re-compression criteria (e.g. size threshold) or when there are
-no secondary algorithms left to try. If none of the secondary algorithms can
-successfully re-compressed the page such a page is marked as incompressible,
-so ZRAM will not attempt to re-compress it in the future.
-
-This re-compression behaviour, when it iterates through the list of
-registered compression algorithms, increases our chances of finding the
-algorithm that successfully compresses a particular page. Sometimes, however,
-it is convenient (and sometimes even necessary) to limit recompression to
-only one particular algorithm so that it will not try any other algorithms.
-This can be achieved by providing a `algo` or `priority` parameter:::
-
-	#use zstd algorithm only (if registered)
-	echo "type=huge algo=zstd" > /sys/block/zramX/recompress
-
-	#use zstd algorithm only (if zstd was registered under priority 1)
-	echo "type=huge priority=1" > /sys/block/zramX/recompress
+	echo "type=huge_idle priority=1 max_pages=42" > \
+		/sys/block/zramX/recompress
+
+It is advised to always specify `priority` parameter.  While it is also
+possible to specify `algo` parameter, so that `zram` will use algorithm's
+name to determine the priority, it is not recommended, since it can lead to
+unexpected results when the same algorithm is configured with different
+priorities (e.g. different parameters).  `priority` is the only way to
+guarantee that the expected algorithm will be used.
 
 memory tracking
 ===============
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 91beaa6798ce0..8ad0b27813175 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1734,6 +1734,11 @@ The following nested keys are defined.
 	  zswpwb
 		Number of pages written from zswap to swap.
 
+	  zswap_incomp
+		Number of incompressible pages currently stored in zswap
+		without compression. These pages could not be compressed to
+		a size smaller than PAGE_SIZE, so they are stored as-is.
+
 	  thp_fault_alloc (npn)
 		Number of transparent hugepages which were allocated to satisfy
 		a page fault. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE
diff --git a/Documentation/admin-guide/device-mapper/verity.rst b/Documentation/admin-guide/device-mapper/verity.rst
index 3ecab1cff9c64..eb9475d7e1965 100644
--- a/Documentation/admin-guide/device-mapper/verity.rst
+++ b/Documentation/admin-guide/device-mapper/verity.rst
@@ -102,29 +102,42 @@ ignore_zero_blocks
     that are not guaranteed to contain zeroes.
 
 use_fec_from_device <fec_dev>
-    Use forward error correction (FEC) to recover from corruption if hash
-    verification fails. Use encoding data from the specified device. This
-    may be the same device where data and hash blocks reside, in which case
-    fec_start must be outside data and hash areas.
+    Use forward error correction (FEC) parity data from the specified device to
+    try to automatically recover from corruption and I/O errors.
 
-    If the encoding data covers additional metadata, it must be accessible
-    on the hash device after the hash blocks.
+    If this option is given, then <fec_roots> and <fec_blocks> must also be
+    given.  <hash_block_size> must also be equal to <data_block_size>.
 
-    Note: block sizes for data and hash devices must match. Also, if the
-    verity <dev> is encrypted the <fec_dev> should be too.
+    <fec_dev> can be the same as <dev>, in which case <fec_start> must be
+    outside the data area.  It can also be the same as <hash_dev>, in which case
+    <fec_start> must be outside the hash and optional additional metadata areas.
+
+    If the data <dev> is encrypted, the <fec_dev> should be too.
+
+    For more information, see `Forward error correction`_.
 
 fec_roots <num>
-    Number of generator roots. This equals to the number of parity bytes in
-    the encoding data. For example, in RS(M, N) encoding, the number of roots
-    is M-N.
+    The number of parity bytes in each 255-byte Reed-Solomon codeword.  The
+    Reed-Solomon code used will be an RS(255, k) code where k = 255 - fec_roots.
+
+    The supported values are 2 through 24 inclusive.  Higher values provide
+    stronger error correction.  However, the minimum value of 2 already provides
+    strong error correction due to the use of interleaving, so 2 is the
+    recommended value for most users.  fec_roots=2 corresponds to an
+    RS(255, 253) code, which has a space overhead of about 0.8%.
 
 fec_blocks <num>
-    The number of encoding data blocks on the FEC device. The block size for
-    the FEC device is <data_block_size>.
+    The total number of <data_block_size> blocks that are error-checked using
+    FEC.  This must be at least the sum of <num_data_blocks> and the number of
+    blocks needed by the hash tree.  It can include additional metadata blocks,
+    which are assumed to be accessible on <hash_dev> following the hash blocks.
+
+    Note that this is *not* the number of parity blocks.  The number of parity
+    blocks is inferred from <fec_blocks>, <fec_roots>, and <data_block_size>.
 
 fec_start <offset>
-    This is the offset, in <data_block_size> blocks, from the start of the
-    FEC device to the beginning of the encoding data.
+    This is the offset, in <data_block_size> blocks, from the start of <fec_dev>
+    to the beginning of the parity data.
 
 check_at_most_once
     Verify data blocks only the first time they are read from the data device,
@@ -180,11 +193,6 @@ per-block basis. This allows for a lightweight hash computation on first read
 into the page cache. Block hashes are stored linearly, aligned to the nearest
 block size.
 
-If forward error correction (FEC) support is enabled any recovery of
-corrupted data will be verified using the cryptographic hash of the
-corresponding data. This is why combining error correction with
-integrity checking is essential.
-
 Hash Tree
 ---------
 
@@ -212,6 +220,80 @@ The tree looks something like:
            / ... \             /   . . .  \             /           \
      blk_0 ... blk_127  blk_16256   blk_16383      blk_32640 . . . blk_32767
 
+Forward error correction
+------------------------
+
+dm-verity's optional forward error correction (FEC) support adds strong error
+correction capabilities to dm-verity.  It allows systems that would be rendered
+inoperable by errors to continue operating, albeit with reduced performance.
+
+FEC uses Reed-Solomon (RS) codes that are interleaved across the entire
+device(s), allowing long bursts of corrupt or unreadable blocks to be recovered.
+
+dm-verity validates any FEC-corrected block against the wanted hash before using
+it.  Therefore, FEC doesn't affect the security properties of dm-verity.
+
+The integration of FEC with dm-verity provides significant benefits over a
+separate error correction layer:
+
+- dm-verity invokes FEC only when a block's hash doesn't match the wanted hash
+  or the block cannot be read at all.  As a result, FEC doesn't add overhead to
+  the common case where no error occurs.
+
+- dm-verity hashes are also used to identify erasure locations for RS decoding.
+  This allows correcting twice as many errors.
+
+FEC uses an RS(255, k) code where k = 255 - fec_roots.  fec_roots is usually 2.
+This means that each k (usually 253) message bytes have fec_roots (usually 2)
+bytes of parity data added to get a 255-byte codeword.  (Many external sources
+call RS codewords "blocks".  Since dm-verity already uses the term "block" to
+mean something else, we'll use the clearer term "RS codeword".)
+
+FEC checks fec_blocks blocks of message data in total, consisting of:
+
+1. The data blocks from the data device
+2. The hash blocks from the hash device
+3. Optional additional metadata that follows the hash blocks on the hash device
+
+dm-verity assumes that the FEC parity data was computed as if the following
+procedure were followed:
+
+1. Concatenate the message data from the above sources.
+2. Zero-pad to the next multiple of k blocks.  Let msg be the resulting byte
+   array, and msglen its length in bytes.
+3. For 0 <= i < msglen / k (for each RS codeword):
+     a. Select msg[i + j * msglen / k] for 0 <= j < k.
+        Consider these to be the 'k' message bytes of an RS codeword.
+     b. Compute the corresponding 'fec_roots' parity bytes of the RS codeword,
+        and concatenate them to the FEC parity data.
+
+Step 3a interleaves the RS codewords across the entire device using an
+interleaving degree of data_block_size * ceil(fec_blocks / k).  This is the
+maximal interleaving, such that the message data consists of a region containing
+byte 0 of all the RS codewords, then a region containing byte 1 of all the RS
+codewords, and so on up to the region for byte 'k - 1'.  Note that the number of
+codewords is set to a multiple of data_block_size; thus, the regions are
+block-aligned, and there is an implicit zero padding of up to 'k - 1' blocks.
+
+This interleaving allows long bursts of errors to be corrected.  It provides
+much stronger error correction than storage devices typically provide, while
+keeping the space overhead low.
+
+The cost is slow decoding: correcting a single block usually requires reading
+254 extra blocks spread evenly across the device(s).  However, that is
+acceptable because dm-verity uses FEC only when there is actually an error.
+
+The list below contains additional details about the RS codes used by
+dm-verity's FEC.  Userspace programs that generate the parity data need to use
+these parameters for the parity data to match exactly:
+
+- Field used is GF(256)
+- Bytes are mapped to/from GF(256) elements in the natural way, where bits 0
+  through 7 (low-order to high-order) map to the coefficients of x^0 through x^7
+- Field generator polynomial is x^8 + x^4 + x^3 + x^2 + 1
+- The codes used are systematic, BCH-view codes
+- Primitive element alpha is 'x'
+- First consecutive root of code generator polynomial is 'x^0'
 
 On-disk format
 ==============
diff --git a/Documentation/admin-guide/index.rst b/Documentation/admin-guide/index.rst
index b734f8a2a2c48..1801b9880aeb7 100644
--- a/Documentation/admin-guide/index.rst
+++ b/Documentation/admin-guide/index.rst
@@ -73,6 +73,7 @@ problems and bugs in particular.
    ramoops
    dynamic-debug-howto
    init
+   kallsyms-lineinfo
    kdump/index
    perf/index
    pstore-blk
diff --git a/Documentation/admin-guide/kallsyms-lineinfo.rst b/Documentation/admin-guide/kallsyms-lineinfo.rst
new file mode 100644
index 0000000000000..dd264830c8d5b
--- /dev/null
+++ b/Documentation/admin-guide/kallsyms-lineinfo.rst
@@ -0,0 +1,97 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+====================================
+Kallsyms Source Line Info (LINEINFO)
+====================================
+
+Overview
+========
+
+``CONFIG_KALLSYMS_LINEINFO`` embeds DWARF-derived source file and line number
+mappings into the kernel image so that stack traces include
+``(file.c:123)`` annotations next to each symbol.  This makes it significantly
+easier to pinpoint the exact source location during debugging, without needing
+to manually cross-reference addresses with ``addr2line``.
+
+Enabling the Feature
+====================
+
+Enable the following kernel configuration options::
+
+    CONFIG_KALLSYMS=y
+    CONFIG_DEBUG_INFO=y
+    CONFIG_KALLSYMS_LINEINFO=y
+
+Build dependency: the host tool ``scripts/gen_lineinfo`` requires ``libdw``
+from elfutils.  Install the development package:
+
+- Debian/Ubuntu: ``apt install libdw-dev``
+- Fedora/RHEL: ``dnf install elfutils-devel``
+- Arch Linux: ``pacman -S elfutils``
+
+Example Output
+==============
+
+Without ``CONFIG_KALLSYMS_LINEINFO``::
+
+    Call Trace:
+     <TASK>
+     dump_stack_lvl+0x5d/0x80
+     do_syscall_64+0x82/0x190
+     entry_SYSCALL_64_after_hwframe+0x76/0x7e
+
+With ``CONFIG_KALLSYMS_LINEINFO``::
+
+    Call Trace:
+     <TASK>
+     dump_stack_lvl+0x5d/0x80 (lib/dump_stack.c:123)
+     do_syscall_64+0x82/0x190 (arch/x86/entry/common.c:52)
+     entry_SYSCALL_64_after_hwframe+0x76/0x7e
+
+Note that assembly routines (such as ``entry_SYSCALL_64_after_hwframe``) are
+not annotated because they lack DWARF debug information.
+
+Module Support
+==============
+
+``CONFIG_KALLSYMS_LINEINFO_MODULES`` extends the feature to loadable kernel
+modules.  When enabled, each ``.ko`` is post-processed at build time to embed
+a ``.mod_lineinfo`` section containing the same kind of address-to-source
+mapping.
+
+Enable in addition to the base options::
+
+    CONFIG_MODULES=y
+    CONFIG_KALLSYMS_LINEINFO_MODULES=y
+
+Stack traces from module code will then include annotations::
+
+    my_driver_func+0x30/0x100 [my_driver] (drivers/foo/bar.c:123)
+
+The ``.mod_lineinfo`` section is loaded into read-only module memory alongside
+the module text.  No additional runtime memory allocation is required; the data
+is freed when the module is unloaded.
+
+Memory Overhead
+===============
+
+The vmlinux lineinfo tables are stored in ``.rodata`` and typically add
+approximately 10-15 MiB to the kernel image for a standard configuration
+(~4.6 million DWARF line entries, ~2-3 bytes per entry after delta
+compression).
+
+Per-module lineinfo adds approximately 2-3 bytes per DWARF line entry to each
+``.ko`` file.
+
+Known Limitations
+=================
+
+- **4 GiB offset limit**: Address offsets from ``_text`` (vmlinux) or
+  ``.text`` base (modules) are stored as 32-bit values.  Entries beyond
+  4 GiB are skipped at build time with a warning.
+- **65535 file limit**: Source file IDs are stored as 16-bit values.  Builds
+  with more than 65535 unique source files will fail with an error.
+- **No assembly annotations**: Functions implemented in assembly that lack
+  DWARF ``.debug_line`` data are not annotated.
+- **No init text**: For modules, functions in ``.init.text`` are not annotated
+  because that memory is freed after module initialization.
diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst
index 404a15f6782c3..7663c610fe901 100644
--- a/Documentation/admin-guide/kdump/vmcoreinfo.rst
+++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst
@@ -141,7 +141,7 @@ nodemask_t
 The size of a nodemask_t type. Used to compute the number of online
 nodes.
 
-(page, flags|_refcount|mapping|lru|_mapcount|private|compound_order|compound_head)
+(page, flags|_refcount|mapping|lru|_mapcount|private|compound_order|compound_info)
 ----------------------------------------------------------------------------------
 
 User-space tools compute their values based on the offset of these
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 03a550630644f..7ecc1f0dfd2cd 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -190,6 +190,14 @@ Kernel parameters
 			unusable.  The "log_buf_len" parameter may be useful
 			if you need to capture more output.
 
+	acpi.poweroff_on_fatal=	[ACPI]
+			{0 | 1}
+			Causes the system to poweroff when the ACPI bytecode signals
+			a fatal error. The default value of this setting is 1.
+			Overriding this value should only be done for diagnosing
+			ACPI firmware problems, as the system might behave erratically
+			after having encountered a fatal ACPI error.
+
 	acpi_enforce_resources=	[ACPI]
 			{ strict | lax | no }
 			Check for resource conflicts between native drivers
@@ -2395,23 +2403,6 @@ Kernel parameters
 			[IMA] Define a custom template format.
 			Format: { "field1|...|fieldN" }
 
-	ima.ahash_minsize= [IMA] Minimum file size for asynchronous hash usage
-			Format: <min_file_size>
-			Set the minimal file size for using asynchronous hash.
-			If left unspecified, ahash usage is disabled.
-
-			ahash performance varies for different data sizes on
-			different crypto accelerators. This option can be used
-			to achieve the best performance for a particular HW.
-
-	ima.ahash_bufsize= [IMA] Asynchronous hash buffer size
-			Format: <bufsize>
-			Set hashing buffer size. Default: 4k.
-
-			ahash performance varies for different chunk sizes on
-			different crypto accelerators. This option can be used
-			to achieve best performance for particular HW.
-
 	ima=		[IMA] Enable or disable IMA
 			Format: { "off" | "on" }
 			Default: "on"
@@ -2959,6 +2950,12 @@ Kernel parameters
 			Format: <bool>
 			Default: CONFIG_KFENCE_DEFERRABLE
 
+	kfence.fault=	[MM,KFENCE] Controls the behavior when a KFENCE
+			error is detected.
+			report - print the error report and continue (default).
+			oops   - print the error report and oops.
+			panic  - print the error report and panic.
+
 	kfence.sample_interval=
 			[MM,KFENCE] KFENCE's sample interval in milliseconds.
 			Format: <unsigned integer>
@@ -6746,7 +6743,7 @@ Kernel parameters
 			Default is 'on'.
 
 	initramfs_options= [KNL]
-                        Specify mount options for for the initramfs mount.
+			Specify mount options for the initramfs mount.
 
 	rootfstype=	[KNL] Set root filesystem type
 
@@ -8392,7 +8389,9 @@ Kernel parameters
 
 			emulate     Vsyscalls turn into traps and are emulated
 			            reasonably safely.  The vsyscall page is
-				    readable.
+				    readable.  This disables the Linear
+				    Address Space Separation (LASS) security
+				    feature and makes the system less secure.
 
 			xonly       [default] Vsyscalls turn into traps and are
 			            emulated reasonably safely.  The vsyscall
diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
index 03951ed6b628f..f874db31801dc 100644
--- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst
+++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
@@ -1522,6 +1522,27 @@ Currently 2 antenna types are supported as mentioned below:
 The property is read-only. If the platform doesn't have support the sysfs
 class is not created.
 
+doubletap_enable
+----------------
+
+sysfs: doubletap_enable
+
+Controls whether TrackPoint doubletap events are filtered out. Doubletap is a
+feature where quickly tapping the TrackPoint twice triggers a special function key event.
+
+The available commands are::
+
+                cat /sys/devices/platform/thinkpad_acpi/doubletap_enable
+                echo 1 | sudo tee /sys/devices/platform/thinkpad_acpi/doubletap_enable
+                echo 0 | sudo tee /sys/devices/platform/thinkpad_acpi/doubletap_enable
+
+Values:
+
+	* 1 - doubletap events are processed (default)
+	* 0 - doubletap events are filtered out (ignored)
+
+	This setting can also be toggled via the Fn+doubletap hotkey.
+
 Auxmac
 ------
 
diff --git a/Documentation/admin-guide/lockup-watchdogs.rst b/Documentation/admin-guide/lockup-watchdogs.rst
index 3e09284a8b9be..7ae7ce3abd2c8 100644
--- a/Documentation/admin-guide/lockup-watchdogs.rst
+++ b/Documentation/admin-guide/lockup-watchdogs.rst
@@ -16,7 +16,7 @@ details), and a compile option, "BOOTPARAM_SOFTLOCKUP_PANIC", are
 provided for this.
 
 A 'hardlockup' is defined as a bug that causes the CPU to loop in
-kernel mode for more than 10 seconds (see "Implementation" below for
+kernel mode for several seconds (see "Implementation" below for
 details), without letting other interrupts have a chance to run.
 Similarly to the softlockup case, the current stack trace is displayed
 upon detection and the system will stay locked up unless the default
@@ -30,22 +30,23 @@ timeout is set through the confusingly named "kernel.panic" sysctl),
 to cause the system to reboot automatically after a specified amount
 of time.
 
+Configuration
+=============
+
+A kernel knob is provided that allows administrators to configure
+this period. The "watchdog_thresh" parameter (default 10 seconds)
+controls the threshold. The right value for a particular environment
+is a trade-off between fast response to lockups and detection overhead.
+
 Implementation
 ==============
 
-The soft and hard lockup detectors are built on top of the hrtimer and
-perf subsystems, respectively. A direct consequence of this is that,
-in principle, they should work in any architecture where these
-subsystems are present.
+The soft lockup detector is built on top of the hrtimer subsystem.
+The hard lockup detector is built on top of the perf subsystem
+(on architectures that support it) or uses an SMP "buddy" system.
 
-A periodic hrtimer runs to generate interrupts and kick the watchdog
-job. An NMI perf event is generated every "watchdog_thresh"
-(compile-time initialized to 10 and configurable through sysctl of the
-same name) seconds to check for hardlockups. If any CPU in the system
-does not receive any hrtimer interrupt during that time the
-'hardlockup detector' (the handler for the NMI perf event) will
-generate a kernel warning or call panic, depending on the
-configuration.
+Softlockup Detector
+-------------------
 
 The watchdog job runs in a stop scheduling thread that updates a
 timestamp every time it is scheduled. If that timestamp is not updated
@@ -55,14 +56,105 @@ will dump useful debug information to the system log, after which it
 will call panic if it was instructed to do so or resume execution of
 other kernel code.
 
-The period of the hrtimer is 2*watchdog_thresh/5, which means it has
-two or three chances to generate an interrupt before the hardlockup
-detector kicks in.
+Frequency and Heartbeats
+------------------------
+
+The hrtimer used by the softlockup detector serves a dual purpose:
+it detects softlockups, and it also generates the interrupts
+(heartbeats) that the hardlockup detectors use to verify CPU liveness.
+
+The period of this hrtimer is 2*watchdog_thresh/5. This means the
+hrtimer has two or three chances to generate an interrupt before the
+NMI hardlockup detector kicks in.
+
+Hardlockup Detector (NMI/Perf)
+------------------------------
+
+On architectures that support NMI (Non-Maskable Interrupt) perf events,
+a periodic NMI is generated every "watchdog_thresh" seconds.
+
+If any CPU in the system does not receive any hrtimer interrupt
+(heartbeat) during the "watchdog_thresh" window, the 'hardlockup
+detector' (the handler for the NMI perf event) will generate a kernel
+warning or call panic.
+
+**Detection Overhead (NMI):**
+
+The time to detect a lockup can vary depending on when the lockup
+occurs relative to the NMI check window. Examples below assume a watchdog_thresh of 10.
+
+* **Best Case:** The lockup occurs just before the first heartbeat is
+  due. The detector will notice the missing hrtimer interrupt almost
+  immediately during the next check.
+
+  ::
+
+    Time 100.0: cpu 1 heartbeat
+    Time 100.1: hardlockup_check, cpu1 stores its state
+    Time 103.9: Hard Lockup on cpu1
+    Time 104.0: cpu 1 heartbeat never comes
+    Time 110.1: hardlockup_check, cpu1 checks the state again, should be the same, declares lockup
+
+    Time to detection: ~6 seconds
+
+* **Worst Case:** The lockup occurs shortly after a valid interrupt
+  (heartbeat) which itself happened just after the NMI check. The next
+  NMI check sees that the interrupt count has changed (due to that one
+  heartbeat), assumes the CPU is healthy, and resets the baseline. The
+  lockup is only detected at the subsequent check.
+
+  ::
+
+    Time 100.0: hardlockup_check, cpu1 stores its state
+    Time 100.1: cpu 1 heartbeat
+    Time 100.2: Hard Lockup on cpu1
+    Time 110.0: hardlockup_check, cpu1 stores its state (misses lockup as state changed)
+    Time 120.0: hardlockup_check, cpu1 checks the state again, should be the same, declares lockup
+
+    Time to detection: ~20 seconds
+
+Hardlockup Detector (Buddy)
+---------------------------
+
+On architectures or configurations where NMI perf events are not
+available (or disabled), the kernel may use the "buddy" hardlockup
+detector. This mechanism requires SMP (Symmetric Multi-Processing).
+
+In this mode, each CPU is assigned a "buddy" CPU to monitor. The
+monitoring CPU runs its own hrtimer (the same one used for softlockup
+detection) and checks if the buddy CPU's hrtimer interrupt count has
+increased.
+
+To ensure timeliness and avoid false positives, the buddy system performs
+checks at every hrtimer interval (2*watchdog_thresh/5, which is 4 seconds
+by default). It uses a missed-interrupt threshold of 3. If the buddy's
+interrupt count has not changed for 3 consecutive checks, it is assumed
+that the buddy CPU is hardlocked (interrupts disabled). The monitoring
+CPU will then trigger the hardlockup response (warning or panic).
+
+**Detection Overhead (Buddy):**
+
+With a default check interval of 4 seconds (watchdog_thresh = 10):
+
+* **Best case:** Lockup occurs just before a check.
+    Detected in ~8s (0s till 1st check + 4s till 2nd + 4s till 3rd).
+* **Worst case:** Lockup occurs just after a check.
+    Detected in ~12s (4s till 1st check + 4s till 2nd + 4s till 3rd).
+
+**Limitations of the Buddy Detector:**
+
+1.  **All-CPU Lockup:** If all CPUs lock up simultaneously, the buddy
+    detector cannot detect the condition because the monitoring CPUs
+    are also frozen.
+2.  **Stack Traces:** Unlike the NMI detector, the buddy detector
+    cannot directly interrupt the locked CPU to grab a stack trace.
+    It relies on architecture-specific mechanisms (like NMI backtrace
+    support) to try and retrieve the status of the locked CPU. If
+    such support is missing, the log may only show that a lockup
+    occurred without providing the locked CPU's stack.
 
-As explained above, a kernel knob is provided that allows
-administrators to configure the period of the hrtimer and the perf
-event. The right value for a particular environment is a trade-off
-between fast response to lockups and detection overhead.
+Watchdog Core Exclusion
+=======================
 
 By default, the watchdog runs on all online cores.  However, on a
 kernel configured with NO_HZ_FULL, by default the watchdog runs only
diff --git a/Documentation/admin-guide/media/mgb4.rst b/Documentation/admin-guide/media/mgb4.rst
index 0a8a56e837f77..8e429fd777124 100644
--- a/Documentation/admin-guide/media/mgb4.rst
+++ b/Documentation/admin-guide/media/mgb4.rst
@@ -74,6 +74,7 @@ Common FPDL3/GMSL input parameters
 
     | 0 - OLDI/JEIDA
     | 1 - SPWG/VESA (default)
+    | 2 - ZDML
 
 **link_status** (R):
     Video link status. If the link is locked, chips are properly connected and
@@ -240,6 +241,13 @@ Common FPDL3/GMSL output parameters
     *Note: This parameter can not be changed while the output v4l2 device is
     open.*
 
+**color_mapping** (RW):
+    Mapping of the outgoing bits in the signal to the colour bits of the pixels.
+
+    | 0 - OLDI/JEIDA
+    | 1 - SPWG/VESA (default)
+    | 2 - ZDML
+
 **frame_rate** (RW):
     Output video signal frame rate limit in frames per second. Due to
     the limited output pixel clock steps, the card can not always generate
diff --git a/Documentation/admin-guide/media/starfive_camss.rst b/Documentation/admin-guide/media/starfive_camss.rst
deleted file mode 100644
index ca42e9447c47b..0000000000000
--- a/Documentation/admin-guide/media/starfive_camss.rst
+++ /dev/null
@@ -1,72 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-.. include:: <isonum.txt>
-
-================================
-Starfive Camera Subsystem driver
-================================
-
-Introduction
-------------
-
-This file documents the driver for the Starfive Camera Subsystem found on
-Starfive JH7110 SoC. The driver is located under drivers/staging/media/starfive/
-camss.
-
-The driver implements V4L2, Media controller and v4l2_subdev interfaces. Camera
-sensor using V4L2 subdev interface in the kernel is supported.
-
-The driver has been successfully used on the Gstreamer 1.18.5 with v4l2src
-plugin.
-
-
-Starfive Camera Subsystem hardware
-----------------------------------
-
-The Starfive Camera Subsystem hardware consists of::
-
-                    |\         +---------------+      +-----------+
-  +----------+      |  \       |               |      |           |
-  |          |      |   |      |               |      |           |
-  |   MIPI   |----->|   |----->|      ISP      |----->|           |
-  |          |      |   |      |               |      |           |
-  +----------+      |   |      |               |      |  Memory   |
-                    |MUX|      +---------------+      | Interface |
-  +----------+      |   |                             |           |
-  |          |      |   |---------------------------->|           |
-  | Parallel |----->|   |                             |           |
-  |          |      |   |                             |           |
-  +----------+      |  /                              |           |
-                    |/                                +-----------+
-
-- MIPI: The MIPI interface, receiving data from a MIPI CSI-2 camera sensor.
-
-- Parallel: The parallel interface,  receiving data from a parallel sensor.
-
-- ISP: The ISP, processing raw Bayer data from an image sensor and producing
-  YUV frames.
-
-
-Topology
---------
-
-The media controller pipeline graph is as follows:
-
-.. _starfive_camss_graph:
-
-.. kernel-figure:: starfive_camss_graph.dot
-    :alt:   starfive_camss_graph.dot
-    :align: center
-
-The driver has 2 video devices:
-
-- capture_raw: The capture device, capturing image data directly from a sensor.
-- capture_yuv: The capture device, capturing YUV frame data processed by the
-  ISP module
-
-The driver has 3 subdevices:
-
-- stf_isp: is responsible for all the isp operations, outputs YUV frames.
-- cdns_csi2rx: a CSI-2 bridge supporting up to 4 CSI lanes in input, and 4
-  different pixel streams in output.
-- imx219: an image sensor, image data is sent through MIPI CSI-2.
diff --git a/Documentation/admin-guide/media/starfive_camss_graph.dot b/Documentation/admin-guide/media/starfive_camss_graph.dot
deleted file mode 100644
index 8eff1f161ac7f..0000000000000
--- a/Documentation/admin-guide/media/starfive_camss_graph.dot
+++ /dev/null
@@ -1,12 +0,0 @@
-digraph board {
-	rankdir=TB
-	n00000001 [label="{{<port0> 0} | stf_isp\n/dev/v4l-subdev0 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green]
-	n00000001:port1 -> n00000008 [style=dashed]
-	n00000004 [label="capture_raw\n/dev/video0", shape=box, style=filled, fillcolor=yellow]
-	n00000008 [label="capture_yuv\n/dev/video1", shape=box, style=filled, fillcolor=yellow]
-	n0000000e [label="{{<port0> 0} | cdns_csi2rx.19800000.csi-bridge\n | {<port1> 1 | <port2> 2 | <port3> 3 | <port4> 4}}", shape=Mrecord, style=filled, fillcolor=green]
-	n0000000e:port1 -> n00000001:port0 [style=dashed]
-	n0000000e:port1 -> n00000004 [style=dashed]
-	n00000018 [label="{{} | imx219 6-0010\n/dev/v4l-subdev1 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green]
-	n00000018:port0 -> n0000000e:port0 [style=bold]
-}
diff --git a/Documentation/admin-guide/media/v4l-drivers.rst b/Documentation/admin-guide/media/v4l-drivers.rst
index 393f83e8dc4dd..d31da8e0a54f2 100644
--- a/Documentation/admin-guide/media/v4l-drivers.rst
+++ b/Documentation/admin-guide/media/v4l-drivers.rst
@@ -33,7 +33,6 @@ Video4Linux (V4L) driver-specific documentation
 	si470x
 	si4713
 	si476x
-	starfive_camss
 	vimc
 	visl
 	vivid
diff --git a/Documentation/admin-guide/mm/damon/lru_sort.rst b/Documentation/admin-guide/mm/damon/lru_sort.rst
index 20a8378d5a946..56690646cf398 100644
--- a/Documentation/admin-guide/mm/damon/lru_sort.rst
+++ b/Documentation/admin-guide/mm/damon/lru_sort.rst
@@ -91,8 +91,8 @@ increases and decreases the effective level of the quota aiming the LRU
 
 Disabled by default.
 
-Auto-tune monitoring intervals
-------------------------------
+autotune_monitoring_intervals
+-----------------------------
 
 If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes DAMON's
 sampling and aggregation intervals.  The auto-tuning aims to capture meaningful
@@ -351,3 +351,8 @@ the LRU-list based page granularity reclamation. ::
     # echo 400 > wmarks_mid
     # echo 200 > wmarks_low
     # echo Y > enabled
+
+Note that this module (damon_lru_sort) cannot run simultaneously with other
+DAMON-based special-purpose modules.  Refer to :ref:`DAMON design special
+purpose modules exclusivity <damon_design_special_purpose_modules_exclusivity>`
+for more details.
diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst
index 8eba3da8dcee0..442ac5c647951 100644
--- a/Documentation/admin-guide/mm/damon/reclaim.rst
+++ b/Documentation/admin-guide/mm/damon/reclaim.rst
@@ -318,6 +318,11 @@ granularity reclamation. ::
     # echo 200 > wmarks_low
     # echo Y > enabled
 
+Note that this module (damon_reclaim) cannot run simultaneously with other
+DAMON-based special-purpose modules.  Refer to :ref:`DAMON design special
+purpose modules exclusivity <damon_design_special_purpose_modules_exclusivity>`
+for more details.
+
 .. [1] https://research.google/pubs/pub48551/
 .. [2] https://lwn.net/Articles/787611/
 .. [3] https://www.kernel.org/doc/html/latest/mm/free_page_reporting.html
diff --git a/Documentation/admin-guide/mm/damon/stat.rst b/Documentation/admin-guide/mm/damon/stat.rst
index e5a5a2c4f803a..c4b14daeb2dd6 100644
--- a/Documentation/admin-guide/mm/damon/stat.rst
+++ b/Documentation/admin-guide/mm/damon/stat.rst
@@ -45,6 +45,11 @@ You can enable DAMON_STAT by setting the value of this parameter as ``Y``.
 Setting it as ``N`` disables DAMON_STAT.  The default value is set by
 ``CONFIG_DAMON_STAT_ENABLED_DEFAULT`` build config option.
 
+Note that this module (damon_stat) cannot run simultaneously with other
+DAMON-based special-purpose modules.  Refer to :ref:`DAMON design special
+purpose modules exclusivity <damon_design_special_purpose_modules_exclusivity>`
+for more details.
+
 .. _damon_stat_aggr_interval_us:
 
 aggr_interval_us
diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst
index b0f3969b6b3b1..534e1199cf091 100644
--- a/Documentation/admin-guide/mm/damon/usage.rst
+++ b/Documentation/admin-guide/mm/damon/usage.rst
@@ -83,7 +83,7 @@ comma (",").
     │ │ │ │ │ │ │ │ sz/min,max
     │ │ │ │ │ │ │ │ nr_accesses/min,max
     │ │ │ │ │ │ │ │ age/min,max
-    │ │ │ │ │ │ │ :ref:`quotas <sysfs_quotas>`/ms,bytes,reset_interval_ms,effective_bytes
+    │ │ │ │ │ │ │ :ref:`quotas <sysfs_quotas>`/ms,bytes,reset_interval_ms,effective_bytes,goal_tuner
     │ │ │ │ │ │ │ │ weights/sz_permil,nr_accesses_permil,age_permil
     │ │ │ │ │ │ │ │ :ref:`goals <sysfs_schemes_quota_goals>`/nr_goals
     │ │ │ │ │ │ │ │ │ 0/target_metric,target_value,current_value,nid,path
@@ -377,9 +377,9 @@ schemes/<N>/quotas/
 The directory for the :ref:`quotas <damon_design_damos_quotas>` of the given
 DAMON-based operation scheme.
 
-Under ``quotas`` directory, four files (``ms``, ``bytes``,
-``reset_interval_ms``, ``effective_bytes``) and two directories (``weights`` and
-``goals``) exist.
+Under ``quotas`` directory, five files (``ms``, ``bytes``,
+``reset_interval_ms``, ``effective_bytes`` and ``goal_tuner``) and two
+directories (``weights`` and ``goals``) exist.
 
 You can set the ``time quota`` in milliseconds, ``size quota`` in bytes, and
 ``reset interval`` in milliseconds by writing the values to the three files,
@@ -390,6 +390,14 @@ apply the action to only up to ``bytes`` bytes of memory regions within the
 quota limits unless at least one :ref:`goal <sysfs_schemes_quota_goals>` is
 set.
 
+You can set the goal-based effective quota auto-tuning algorithm to use, by
+writing the algorithm name to ``goal_tuner`` file.  Reading the file returns
+the currently selected tuner algorithm.  Refer to the design documentation of
+:ref:`automatic quota tuning goals <damon_design_damos_quotas_auto_tuning>` for
+the background design of the feature and the name of the selectable algorithms.
+Refer to :ref:`goals directory <sysfs_schemes_quota_goals>` for the goals
+setup.
+
 The time quota is internally transformed to a size quota.  Between the
 transformed size quota and user-specified size quota, smaller one is applied.
 Based on the user-specified :ref:`goal <sysfs_schemes_quota_goals>`, the
diff --git a/Documentation/admin-guide/mm/kho.rst b/Documentation/admin-guide/mm/kho.rst
index 6dc18ed4b8861..cb9a20f649206 100644
--- a/Documentation/admin-guide/mm/kho.rst
+++ b/Documentation/admin-guide/mm/kho.rst
@@ -28,20 +28,10 @@ per NUMA node scratch regions on boot.
 Perform a KHO kexec
 ===================
 
-First, before you perform a KHO kexec, you need to move the system into
-the :ref:`KHO finalization phase <kho-finalization-phase>` ::
-
-  $ echo 1 > /sys/kernel/debug/kho/out/finalize
-
-After this command, the KHO FDT is available in
-``/sys/kernel/debug/kho/out/fdt``. Other subsystems may also register
-their own preserved sub FDTs under
-``/sys/kernel/debug/kho/out/sub_fdts/``.
-
-Next, load the target payload and kexec into it. It is important that you
-use the ``-s`` parameter to use the in-kernel kexec file loader, as user
-space kexec tooling currently has no support for KHO with the user space
-based file loader ::
+To perform a KHO kexec, load the target payload and kexec into it. It
+is important that you use the ``-s`` parameter to use the in-kernel
+kexec file loader, as user space kexec tooling currently has no
+support for KHO with the user space based file loader ::
 
   # kexec -l /path/to/bzImage --initrd /path/to/initrd -s
   # kexec -e
@@ -52,40 +42,19 @@ For example, if you used ``reserve_mem`` command line parameter to create
 an early memory reservation, the new kernel will have that memory at the
 same physical address as the old kernel.
 
-Abort a KHO exec
-================
-
-You can move the system out of KHO finalization phase again by calling ::
-
-  $ echo 0 > /sys/kernel/debug/kho/out/active
-
-After this command, the KHO FDT is no longer available in
-``/sys/kernel/debug/kho/out/fdt``.
-
 debugfs Interfaces
 ==================
 
+These debugfs interfaces are available when the kernel is compiled with
+``CONFIG_KEXEC_HANDOVER_DEBUGFS`` enabled.
+
 Currently KHO creates the following debugfs interfaces. Notice that these
 interfaces may change in the future. They will be moved to sysfs once KHO is
 stabilized.
 
-``/sys/kernel/debug/kho/out/finalize``
-    Kexec HandOver (KHO) allows Linux to transition the state of
-    compatible drivers into the next kexec'ed kernel. To do so,
-    device drivers will instruct KHO to preserve memory regions,
-    which could contain serialized kernel state.
-    While the state is serialized, they are unable to perform
-    any modifications to state that was serialized, such as
-    handed over memory allocations.
-
-    When this file contains "1", the system is in the transition
-    state. When contains "0", it is not. To switch between the
-    two states, echo the respective number into this file.
-
 ``/sys/kernel/debug/kho/out/fdt``
-    When KHO state tree is finalized, the kernel exposes the
-    flattened device tree blob that carries its current KHO
-    state in this file. Kexec user space tooling can use this
+    The kernel exposes the flattened device tree blob that carries its
+    current KHO state in this file. Kexec user space tooling can use this
     as input file for the KHO payload image.
 
 ``/sys/kernel/debug/kho/out/scratch_len``
@@ -100,8 +69,8 @@ stabilized.
     it should place its payload images.
 
 ``/sys/kernel/debug/kho/out/sub_fdts/``
-    In the KHO finalization phase, KHO producers register their own
-    FDT blob under this directory.
+    KHO producers can register their own FDT or another binary blob under
+    this directory.
 
 ``/sys/kernel/debug/kho/in/fdt``
     When the kernel was booted with Kexec HandOver (KHO),
diff --git a/Documentation/admin-guide/mm/numa_memory_policy.rst b/Documentation/admin-guide/mm/numa_memory_policy.rst
index a70f20ce1ffb4..90ab26e805a9a 100644
--- a/Documentation/admin-guide/mm/numa_memory_policy.rst
+++ b/Documentation/admin-guide/mm/numa_memory_policy.rst
@@ -217,7 +217,7 @@ MPOL_PREFERRED
 	the MPOL_F_STATIC_NODES or MPOL_F_RELATIVE_NODES flags
 	described below.
 
-MPOL_INTERLEAVED
+MPOL_INTERLEAVE
 	This mode specifies that page allocations be interleaved, on a
 	page granularity, across the nodes specified in the policy.
 	This mode also behaves slightly differently, based on the
diff --git a/Documentation/admin-guide/nfs/pnfs-block-server.rst b/Documentation/admin-guide/nfs/pnfs-block-server.rst
index 20fe9f5117fe0..b4f5997009af7 100644
--- a/Documentation/admin-guide/nfs/pnfs-block-server.rst
+++ b/Documentation/admin-guide/nfs/pnfs-block-server.rst
@@ -40,3 +40,33 @@ how to translate the device into a serial number from SCSI EVPD 0x80::
 
 	echo "fencing client ${CLIENT} serial ${EVPD}" >> /var/log/pnfsd-fence.log
 	EOF
+
+If the nfsd server needs to fence a non-responding client and the
+fencing operation fails, the server logs a warning message in the
+system log with the following format:
+
+    FENCE failed client[IP_address] clid[#n] device[dev_name]
+
+    Where:
+
+    IP_address: refers to the IP address of the affected client.
+    #n: indicates the unique client identifier.
+    dev_name: specifies the name of the block device related
+              to the fencing attempt.
+
+The server will repeatedly retry the operation indefinitely. During
+this time, access to the affected file is restricted for all other
+clients. This is to prevent potential data corruption if multiple
+clients access the same file simultaneously.
+
+To restore access to the affected file for other clients, the admin
+needs to take the following actions:
+
+    . shutdown or power off the client being fenced.
+    . manually expire the client to release all its state on the server:
+
+      echo 'expire' > /proc/fs/nfsd/clients/clid/ctl'.
+
+      Where:
+
+      clid: is the unique client identifier displayed in the system log.
diff --git a/Documentation/admin-guide/nfs/pnfs-scsi-server.rst b/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
index b2eec22883291..db34afbf67a96 100644
--- a/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
+++ b/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
@@ -22,3 +22,34 @@ option and the underlying SCSI device support persistent reservations.
 On the client make sure the kernel has the CONFIG_PNFS_BLOCK option
 enabled, and the file system is mounted using the NFSv4.1 protocol
 version (mount -o vers=4.1).
+
+If the nfsd server needs to fence a non-responding client and the
+fencing operation fails, the server logs a warning message in the
+system log with the following format:
+
+    FENCE failed client[IP_address] clid[#n] device[dev_name]
+
+    Where:
+
+    IP_address: refers to the IP address of the affected client.
+    #n: indicates the unique client identifier.
+    dev_name: specifies the name of the block device related
+              to the fencing attempt.
+
+The server will repeatedly retry the operation indefinitely. During
+this time, access to the affected file is restricted for all other
+clients. This is to prevent potential data corruption if multiple
+clients access the same file simultaneously.
+
+To restore access to the affected file for other clients, the admin
+needs to take the following actions:
+
+    . shutdown or power off the client being fenced.
+    . manually expire the client to release all its state on the server:
+
+      echo 'expire' > /proc/fs/nfsd/clients/clid/ctl'.
+
+      Where:
+
+      clid: is the unique client identifier displayed in the system log.
+
diff --git a/Documentation/admin-guide/sysctl/crypto.rst b/Documentation/admin-guide/sysctl/crypto.rst
new file mode 100644
index 0000000000000..b707bd314a64b
--- /dev/null
+++ b/Documentation/admin-guide/sysctl/crypto.rst
@@ -0,0 +1,47 @@
+=================
+/proc/sys/crypto/
+=================
+
+These files show up in ``/proc/sys/crypto/``, depending on the
+kernel configuration:
+
+.. contents:: :local:
+
+fips_enabled
+============
+
+Read-only flag that indicates whether FIPS mode is enabled.
+
+- ``0``: FIPS mode is disabled (default).
+- ``1``: FIPS mode is enabled.
+
+This value is set at boot time via the ``fips=1`` kernel command line
+parameter. When enabled, the cryptographic API will restrict the use
+of certain algorithms and perform self-tests to ensure compliance with
+FIPS (Federal Information Processing Standards) requirements, such as
+FIPS 140-2 and the newer FIPS 140-3, depending on the kernel
+configuration and the module in use.
+
+fips_name
+=========
+
+Read-only file that contains the name of the FIPS module currently in use.
+The value is typically configured via the ``CONFIG_CRYPTO_FIPS_NAME``
+kernel configuration option.
+
+fips_version
+============
+
+Read-only file that contains the version string of the FIPS module.
+If ``CONFIG_CRYPTO_FIPS_CUSTOM_VERSION`` is set, it uses the value from
+``CONFIG_CRYPTO_FIPS_VERSION``. Otherwise, it defaults to the kernel
+release version (``UTS_RELEASE``).
+
+Copyright (c) 2026, Shubham Chakraborty <chakrabortyshubham66@gmail.com>
+
+For general info and legal blurb, please look in
+Documentation/admin-guide/sysctl/index.rst.
+
+.. See scripts/check-sysctl-docs to keep this up to date:
+.. scripts/check-sysctl-docs -vtable="crypto" \
+..         $(git grep -l register_sysctl_)
diff --git a/Documentation/admin-guide/sysctl/debug.rst b/Documentation/admin-guide/sysctl/debug.rst
new file mode 100644
index 0000000000000..506bd5e48594a
--- /dev/null
+++ b/Documentation/admin-guide/sysctl/debug.rst
@@ -0,0 +1,52 @@
+================
+/proc/sys/debug/
+================
+
+These files show up in ``/proc/sys/debug/``, depending on the
+kernel configuration:
+
+.. contents:: :local:
+
+exception-trace
+===============
+
+This flag controls whether the kernel prints information about unhandled
+signals (like segmentation faults) to the kernel log (``dmesg``).
+
+- ``0``: Unhandled signals are not traced.
+- ``1``: Information about unhandled signals is printed.
+
+The default value is ``1`` on most architectures (like x86, MIPS, RISC-V),
+but it is ``0`` on **arm64**.
+
+The actual information printed and the context provided varies
+significantly depending on the CPU architecture. For example:
+
+- On **x86**, it typically prints the instruction pointer (IP), error
+  code, and address that caused a page fault.
+- On **PowerPC**, it may print the next instruction pointer (NIP),
+  link register (LR), and other relevant registers.
+
+When enabled, this feature is often rate-limited to prevent the kernel
+log from being flooded during a crash loop.
+
+kprobes-optimization
+====================
+
+This flag enables or disables the optimization of Kprobes on certain
+architectures (like x86).
+
+- ``0``: Kprobes optimization is turned off.
+- ``1``: Kprobes optimization is turned on (default).
+
+For more details on Kprobes and its optimization, please refer to
+Documentation/trace/kprobes.rst.
+
+Copyright (c) 2026, Shubham Chakraborty <chakrabortyshubham66@gmail.com>
+
+For general info and legal blurb, please look in
+Documentation/admin-guide/sysctl/index.rst.
+
+.. See scripts/check-sysctl-docs to keep this up to date:
+.. scripts/check-sysctl-docs -vtable="debug" \
+..         $(git grep -l register_sysctl_)
diff --git a/Documentation/admin-guide/sysctl/index.rst b/Documentation/admin-guide/sysctl/index.rst
index 4dd2c9b5d7525..e153c96119df2 100644
--- a/Documentation/admin-guide/sysctl/index.rst
+++ b/Documentation/admin-guide/sysctl/index.rst
@@ -67,8 +67,8 @@ This documentation is about:
 =============== ===============================================================
 abi/		execution domains & personalities
 <$ARCH>		tuning controls for various CPU architecture (e.g. csky, s390)
-crypto/		<undocumented>
-debug/		<undocumented>
+crypto/		cryptographic subsystem
+debug/		debugging features
 dev/		device specific information (e.g. dev/cdrom/info)
 fs/		specific filesystems
 		filehandle, inode, dentry and quota tuning
@@ -96,6 +96,8 @@ it :-)
    :maxdepth: 1
 
    abi
+   crypto
+   debug
    fs
    kernel
    net
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index 9aed74e65cf4e..c6994e55d1411 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -418,7 +418,8 @@ hung_task_detect_count
 ======================
 
 Indicates the total number of tasks that have been detected as hung since
-the system boot.
+the system boot or since the counter was reset. The counter is zeroed when
+a value of 0 is written.
 
 This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
 
diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst
index 3b2ad61995d4c..0724a793798f3 100644
--- a/Documentation/admin-guide/sysctl/net.rst
+++ b/Documentation/admin-guide/sysctl/net.rst
@@ -602,3 +602,31 @@ it does not modify the current namespace or any existing children.
 
 A namespace with ``ns_mode`` set to ``local`` cannot change
 ``child_ns_mode`` to ``global`` (returns ``-EPERM``).
+
+g2h_fallback
+------------
+
+Controls whether connections to CIDs not owned by the host-to-guest (H2G)
+transport automatically fall back to the guest-to-host (G2H) transport.
+
+When enabled, if a connect targets a CID that the H2G transport (e.g.
+vhost-vsock) does not serve, or if no H2G transport is loaded at all, the
+connection is routed via the G2H transport (e.g. virtio-vsock) instead. This
+allows a host running both nested VMs (via vhost-vsock) and sibling VMs
+reachable through the hypervisor (e.g. Nitro Enclaves) to address both using
+a single CID space, without requiring applications to set
+``VMADDR_FLAG_TO_HOST``.
+
+When the fallback is taken, ``VMADDR_FLAG_TO_HOST`` is automatically set on
+the remote address so that userspace can determine the path via
+``getpeername()``.
+
+Note: With this sysctl enabled, user space that attempts to talk to a guest
+CID which is not implemented by the H2G transport will create host vsock
+traffic. Environments that rely on H2G-only isolation should set it to 0.
+
+Values:
+
+	- 0 - Connections to CIDs <= 2 or with VMADDR_FLAG_TO_HOST use G2H;
+	  all others use H2G (or fail with ENODEV if H2G is not loaded).
+	- 1 - Connections to CIDs not owned by H2G fall back to G2H. (default)
diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst
index ed1f8f1e86c58..9ead927a37c0f 100644
--- a/Documentation/admin-guide/tainted-kernels.rst
+++ b/Documentation/admin-guide/tainted-kernels.rst
@@ -74,7 +74,7 @@ a particular type of taint. It's best to leave that to the aforementioned
 script, but if you need something quick you can use this shell command to check
 which bits are set::
 
-	$ for i in $(seq 18); do echo $(($i-1)) $(($(cat /proc/sys/kernel/tainted)>>($i-1)&1));done
+	$ for i in $(seq 20); do echo $(($i-1)) $(($(cat /proc/sys/kernel/tainted)>>($i-1)&1));done
 
 Table for decoding tainted state
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst
index 61670e7df2f7c..ff6b110291bc6 100644
--- a/Documentation/arch/x86/tdx.rst
+++ b/Documentation/arch/x86/tdx.rst
@@ -60,44 +60,18 @@ Besides initializing the TDX module, a per-cpu initialization SEAMCALL
 must be done on one cpu before any other SEAMCALLs can be made on that
 cpu.
 
-The kernel provides two functions, tdx_enable() and tdx_cpu_enable() to
-allow the user of TDX to enable the TDX module and enable TDX on local
-cpu respectively.
-
-Making SEAMCALL requires VMXON has been done on that CPU.  Currently only
-KVM implements VMXON.  For now both tdx_enable() and tdx_cpu_enable()
-don't do VMXON internally (not trivial), but depends on the caller to
-guarantee that.
-
-To enable TDX, the caller of TDX should: 1) temporarily disable CPU
-hotplug; 2) do VMXON and tdx_enable_cpu() on all online cpus; 3) call
-tdx_enable().  For example::
-
-        cpus_read_lock();
-        on_each_cpu(vmxon_and_tdx_cpu_enable());
-        ret = tdx_enable();
-        cpus_read_unlock();
-        if (ret)
-                goto no_tdx;
-        // TDX is ready to use
-
-And the caller of TDX must guarantee the tdx_cpu_enable() has been
-successfully done on any cpu before it wants to run any other SEAMCALL.
-A typical usage is do both VMXON and tdx_cpu_enable() in CPU hotplug
-online callback, and refuse to online if tdx_cpu_enable() fails.
-
 User can consult dmesg to see whether the TDX module has been initialized.
 
 If the TDX module is initialized successfully, dmesg shows something
 like below::
 
   [..] virt/tdx: 262668 KBs allocated for PAMT
-  [..] virt/tdx: module initialized
+  [..] virt/tdx: TDX-Module initialized
 
 If the TDX module failed to initialize, dmesg also shows it failed to
 initialize::
 
-  [..] virt/tdx: module initialization failed ...
+  [..] virt/tdx: TDX-Module initialization failed ...
 
 TDX Interaction to Other Kernel Components
 ------------------------------------------
@@ -129,9 +103,9 @@ CPU Hotplug
 ~~~~~~~~~~~
 
 TDX module requires the per-cpu initialization SEAMCALL must be done on
-one cpu before any other SEAMCALLs can be made on that cpu.  The kernel
-provides tdx_cpu_enable() to let the user of TDX to do it when the user
-wants to use a new cpu for TDX task.
+one cpu before any other SEAMCALLs can be made on that cpu.  The kernel,
+via the CPU hotplug framework, performs the necessary initialization when
+a CPU is first brought online.
 
 TDX doesn't support physical (ACPI) CPU hotplug.  During machine boot,
 TDX verifies all boot-time present logical CPUs are TDX compatible before
diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst
index 11126ed6f40f7..b9dc0c9dbee44 100644
--- a/Documentation/block/biovecs.rst
+++ b/Documentation/block/biovecs.rst
@@ -135,6 +135,7 @@ Usage of helpers:
 	bio_first_bvec_all()
 	bio_first_page_all()
 	bio_first_folio_all()
+	bio_last_bvec_all()
 
 * The following helpers iterate over single-page segment. The passed 'struct
   bio_vec' will contain a single-page IO vector during the iteration::
diff --git a/Documentation/block/drbd-app-resync-io.rst b/Documentation/block/drbd-app-resync-io.rst
new file mode 100644
index 0000000000000..8953124532299
--- /dev/null
+++ b/Documentation/block/drbd-app-resync-io.rst
@@ -0,0 +1,208 @@
+=========================================
+Application and Resync IO Synchronization
+=========================================
+
+This summary describes one aspect of the Distributed Replicated Block Device
+(DRBD) protocol. For a full definition of the protocol see the DRBD code.
+
+IO operations to the DRBD backing device can originate from 2 sources. Normal
+operations from a filesystem or other user of DRBD are called application IO.
+Resync operations in DRBD also perform reads and writes. These operations must
+be synchronized to ensure that the data on the backing device is correct.
+
+For instance, the following must be prevented:
+
+* Resync read obtains block version v1 on some node
+* Application writes block version v2 on all nodes
+* Resync write overwrites block version v2 with version v1
+
+In addition, care must be taken to ensure that bitmap bits are only cleared
+when this block is genuinely in sync between these nodes.
+
+Synchronization with resync extents
+===================================
+
+Older versions of DRBD without the feature flag ``DRBD_FF_RESYNC_DAGTAG``
+perform synchronization using "resync extents". These are also known as "bitmap
+extents". They are stored in an LRU cache. These extents are exclusive with the
+activity log extents.
+
+Synchronization with data generation tags
+=========================================
+
+When the feature flag ``DRBD_FF_RESYNC_DAGTAG`` is present, DRBD synchronizes
+resync and application IO using the concept of the "data generation tag"
+(dagtag). This is coupled with fine-grained locking of the request intervals,
+implemented internally using an interval tree.
+
+For a given node, the dagtag is the number of sectors written by the
+application on that node. The dagtag is used to determine which node has newer
+data in certain scenarios. DRBD keeps track of the dagtags of its peers.
+
+Whenever a resync request is made, the ``L_SYNC_TARGET`` node making the
+request sends the dagtag from the current Primary node, if any. The
+``L_SYNC_SOURCE`` node must wait until it has received the data corresponding
+to this dagtag before responding to the resync request. This is important for
+preventing the resync from writing older data over newer data.
+
+In addition, the request intervals are locked according to the following
+scheme:
+
+* Primary: The application IO interval is locked while the data is being
+  written to the backing disk. In addition, conflicting application IO is
+  prevented until the epoch containing the request is complete. However, this
+  is effectively a separate lock. Resync IO is not blocked while this lock is
+  held.
+* Secondary: The application IO interval is locked while the data is being
+  written to the backing disk.
+* Sync target: The ``P_RS_DATA_REQUEST`` interval is locked. The lock is taken
+  in two phases. Before sending the request, the interval is locked for
+  conflicts with other peers. Then the dagtag is recorded and the request is
+  sent. When the reply is received, the interval is additionally locked for
+  conflicts with ongoing IO, in particular writes from the same peer. The lock
+  is released when the reply has been received and the data written to the
+  backing disk.
+* Sync source: The ``P_RS_DATA_REQUEST`` interval is locked. The lock is taken
+  when the dagtag for the request is reached. It is released when the
+  ``P_RS_WRITE_ACK`` is received. This lock is a read lock; it is exclusive
+  with write locks, but not with other read locks.
+* Verify source: The online verify interval is marked but does not block any
+  other requests. The mark is set then the dagtag is recorded and the request
+  is sent. The mark is removed when the ``P_OV_REPLY`` has been received, the
+  dagtag from the reply has been reached and the data read. If any conflicting
+  writes occur while the mark is set, the sectors are skipped.
+* Verify target: The online verify interval is marked but does not block any
+  other requests. The mark is set when the dagtag for the request has been
+  reached. It is removed after reading the data. The latest dagtag received by
+  this node is sent with the ``P_OV_REPLY``. If any conflicting writes occur
+  while the mark is set, the sectors are skipped.
+* Sending ``P_PEERS_IN_SYNC``: Intervals are briefly locked while sending
+  ``P_PEERS_IN_SYNC`` to ensure that the bits remain in sync until the packet
+  has been sent.
+
+If a conflict occurs when an interval should be locked, the request is delayed
+until the conflict resolves. Internally this is implemented by storing the
+interval in the tree in an unlocked form. When an interval is removed from the
+tree, the tree is searched for any intervals which can now be released.
+
+Application IO defers to resync IO. That is, application IO is blocked by
+resync IO even when that resync IO has not yet obtained the lock for its
+interval. This is important for ensuring progress. In the normal case, resyncs
+only make one pass through the data. Hence they will eventually terminate.
+Application IO, on the other hand, can keep a given region busy for an
+arbitrary length of time. So resync IO must not wait indefinitely for
+application IO.
+
+Correctness of data
+-------------------
+
+We only consider the synchronization between application and resync IO here.
+
+The locking scheme prevents any writes from other peers to the resync request
+interval from when the request is initiated until the received data is written.
+After the lock is taken on the target, the dagtag is recorded and the request
+is sent to the source. The source then waits until it has reached this dagtag
+before reading. This ensures that the resync data is at least as new as the
+data on the target when the request was made.
+
+Conflicting application writes that reach the target while the resync request
+is in progress are held until the resync data has been written. Hence they
+overwrite the resync data. In the case where the source had already received
+this application write when it performed the resync read, the application write
+will overwrite the resync write on the target with identical data. This is
+harmless.
+
+Resync requests sent from the target are not exclusive with application writes
+from the same peer. However, since the resync and application data originate
+from the same node, they are transmitted in the correct order in the data
+stream. Application IO defers to received resync IO, ensuring that a resync
+write received before an application write is also submitted first.
+
+Correctness of bitmap bits
+--------------------------
+
+DRBD guarantees that bitmap bits are set, or the corresponding activity log
+extent is active, on at least one peer whenever 2 nodes are out of sync with
+each other. A resync is called "stable" when the target is a neighbor of the
+Primary node, if there is one. After a stable resync, all bitmap bits should be
+clear. In other situations, DRBD makes a best effort attempt to clear bits when
+appropriate.
+
+Hence we need to ensure that:
+
+1. Bits are set when out of sync
+2. Bits are only cleared when in sync
+3. Bits are cleared in a stable resync
+
+We are only considering the synchronization of application and resync IO here,
+so we only need to consider interactions between the 2 types. Requirement (1)
+holds due to the general design for how writes work in DRBD. Requirement (3)
+holds because there is no operation that sets bits in a stable resync. The
+potential issues with these interactions arise with requirement (2). We need to
+ensure that bits are never cleared that have become out of sync during the
+operation.
+
+On a Primary node, writes cause bits to be set and cleared when the
+corresponding ``P_BARRIER_ACK`` packets are received. On a Secondary node,
+writes cause bits to be set and cleared when the corresponding ``P_PEER_ACK``
+is received. On a sync target, bits are cleared when the resync data has been
+written. On a sync source, bits are cleared when ``P_RS_WRITE_ACK`` is
+received.
+
+The bits cleared by writes must always be in sync because the corresponding
+nodes have received the write. As demonstrated in the section "correctness of
+data", they cannot lose this data due to a resync.
+
+For a stable resync, bits will not become out of sync for the peer device on
+either side during the resync operation because both peers receive the
+application writes.
+
+On a sync target for an unstable resync, no application writes are received, so
+there will be no bits set that could be incorrectly cleared.
+
+On a sync source for an unstable resync, the interval is locked until
+``P_RS_WRITE_ACK`` is received. Hence, when the bit is cleared, the target has
+the same data for the interval as the source. That is, they are still in sync.
+
+For ``P_PEERS_IN_SYNC`` we consider only the 3 node case. There is only one
+configuration with an unstable resync with 3 nodes. That is a chain
+A - B - C with A being Primary and a sync from B to C. The only
+``P_PEERS_IN_SYNC`` packets that have an effect in this configuration are those
+from B to A indicating that C is in sync for some interval. B only sends this
+packet when no bitmap bits are set towards C for the interval. In addition,
+B must ensure that no application write causes bits to be incorrectly cleared
+on A towards C. This could occur when B has sent ``P_BARRIER_ACK`` for a write
+which is not yet represented in its bitmap towards C. So B must not send
+``P_PEERS_IN_SYNC`` for an interval where this may be the case. To do this, it
+checks that there is no activity in the activity log that overlaps with this
+interval. To ensure that no writes occur between this check and sending
+``P_PEERS_IN_SYNC``, it locks the interval temporarily.
+
+Deadlock safety
+---------------
+
+We can ignore the locking of application IO until the containing epoch is
+complete. No other lock acquisition depends on it. To put it another way, it
+operates on a level above the rest of the locking.
+
+The locking on Primary and Secondary while application IO is being written to
+the backing disk does not depend on any other lock acquisition. So it is
+guaranteed that a locked interval of this type will eventually be unlocked.
+
+Online verify does not block any other operations, so cannot be involved in
+causing a deadlock.
+
+Sending ``P_PEERS_IN_SYNC`` also cannot be involved in causing a deadlock
+because it does not depend on any other lock acquisition.
+
+Resync requests depend on the corresponding peer. If the connection is lost,
+the operation is aborted, so no deadlock will occur as a result of
+non-responsive peers.
+
+A node cannot be both sync source and sync target simultaneously. Hence there
+are no locks in the scheme which can block sync source reads indefinitely. So
+a resync request from a sync target will always eventually receive a reply,
+which allows it to perform the write and unlock its interval. This in turn
+guarantees that the sync source will receive an ack and unlock its interval.
+
+Hence the locking scheme itself is free from distributed deadlocks.
diff --git a/Documentation/block/index.rst b/Documentation/block/index.rst
index 9fea696f9daa0..4812f2b56354b 100644
--- a/Documentation/block/index.rst
+++ b/Documentation/block/index.rst
@@ -13,6 +13,7 @@ Block
    cmdline-partition
    data-integrity
    deadline-iosched
+   drbd-app-resync-io
    inline-encryption
    ioprio
    kyber-iosched
diff --git a/Documentation/core-api/dma-attributes.rst b/Documentation/core-api/dma-attributes.rst
index 1d7bfad73b1c7..123c8468d58f2 100644
--- a/Documentation/core-api/dma-attributes.rst
+++ b/Documentation/core-api/dma-attributes.rst
@@ -149,11 +149,33 @@ For architectures that require cache flushing for DMA coherence
 DMA_ATTR_MMIO will not perform any cache flushing. The address
 provided must never be mapped cacheable into the CPU.
 
-DMA_ATTR_CPU_CACHE_CLEAN
-------------------------
-
-This attribute indicates the CPU will not dirty any cacheline overlapping this
-DMA_FROM_DEVICE/DMA_BIDIRECTIONAL buffer while it is mapped. This allows
-multiple small buffers to safely share a cacheline without risk of data
-corruption, suppressing DMA debug warnings about overlapping mappings.
-All mappings sharing a cacheline should have this attribute.
+DMA_ATTR_DEBUGGING_IGNORE_CACHELINES
+------------------------------------
+
+This attribute indicates that CPU cache lines may overlap for buffers mapped
+with DMA_FROM_DEVICE or DMA_BIDIRECTIONAL.
+
+Such overlap may occur when callers map multiple small buffers that reside
+within the same cache line. In this case, callers must guarantee that the CPU
+will not dirty these cache lines after the mappings are established. When this
+condition is met, multiple buffers can safely share a cache line without risking
+data corruption.
+
+All mappings that share a cache line must set this attribute to suppress DMA
+debug warnings about overlapping mappings.
+
+DMA_ATTR_REQUIRE_COHERENT
+-------------------------
+
+DMA mapping requests with the DMA_ATTR_REQUIRE_COHERENT fail on any
+system where SWIOTLB or cache management is required. This should only
+be used to support uAPI designs that require continuous HW DMA
+coherence with userspace processes, for example RDMA and DRM. At a
+minimum the memory being mapped must be userspace memory from
+pin_user_pages() or similar.
+
+Drivers should consider using dma_mmap_pages() instead of this
+interface when building their uAPIs, when possible.
+
+It must never be used in an in-kernel driver that only works with
+kernel memory.
diff --git a/Documentation/core-api/housekeeping.rst b/Documentation/core-api/housekeeping.rst
index e5417302774c7..92c6e53cea755 100644
--- a/Documentation/core-api/housekeeping.rst
+++ b/Documentation/core-api/housekeeping.rst
@@ -15,7 +15,7 @@ various deferrals etc...
 Sometimes housekeeping is just some unbound work (unbound workqueues,
 unbound timers, ...) that gets easily assigned to non-isolated CPUs.
 But sometimes housekeeping is tied to a specific CPU and requires
-elaborated tricks to be offloaded to non-isolated CPUs (RCU_NOCB, remote
+elaborate tricks to be offloaded to non-isolated CPUs (RCU_NOCB, remote
 scheduler tick, etc...).
 
 Thus, a housekeeping CPU can be considered as the reverse of an isolated
diff --git a/Documentation/core-api/kho/abi.rst b/Documentation/core-api/kho/abi.rst
index 2e63be3486cf4..799d743105a67 100644
--- a/Documentation/core-api/kho/abi.rst
+++ b/Documentation/core-api/kho/abi.rst
@@ -22,6 +22,12 @@ memblock preservation ABI
 .. kernel-doc:: include/linux/kho/abi/memblock.h
    :doc: memblock kexec handover ABI
 
+KHO persistent memory tracker ABI
+=================================
+
+.. kernel-doc:: include/linux/kho/abi/kexec_handover.h
+  :doc: KHO persistent memory tracker
+
 See Also
 ========
 
diff --git a/Documentation/core-api/kho/index.rst b/Documentation/core-api/kho/index.rst
index dcc6a36cc134c..0a2dee4f8e7d0 100644
--- a/Documentation/core-api/kho/index.rst
+++ b/Documentation/core-api/kho/index.rst
@@ -71,17 +71,17 @@ for boot memory allocations and as target memory for kexec blobs, some parts
 of that memory region may be reserved. These reservations are irrelevant for
 the next KHO, because kexec can overwrite even the original kernel.
 
-.. _kho-finalization-phase:
+Kexec Handover Radix Tree
+=========================
 
-KHO finalization phase
-======================
+.. kernel-doc:: include/linux/kho_radix_tree.h
+  :doc: Kexec Handover Radix Tree
 
-To enable user space based kexec file loader, the kernel needs to be able to
-provide the FDT that describes the current kernel's state before
-performing the actual kexec. The process of generating that FDT is
-called serialization. When the FDT is generated, some properties
-of the system may become immutable because they are already written down
-in the FDT. That state is called the KHO finalization phase.
+Public API
+==========
+
+.. kernel-doc:: kernel/liveupdate/kexec_handover.c
+  :export:
 
 See Also
 ========
diff --git a/Documentation/core-api/memory-hotplug.rst b/Documentation/core-api/memory-hotplug.rst
index 8fc97c2379de8..46b0490f5319f 100644
--- a/Documentation/core-api/memory-hotplug.rst
+++ b/Documentation/core-api/memory-hotplug.rst
@@ -96,7 +96,7 @@ NODE_CANCEL_ADDING_FIRST_MEMORY
  Generated if NODE_ADDING_FIRST_MEMORY fails.
 
 NODE_ADDED_FIRST_MEMORY
- Generated when memory has become available fo this node for the first time.
+ Generated when memory has become available for this node for the first time.
 
 NODE_REMOVING_LAST_MEMORY
  Generated when the last memory available to this node is about to be offlined.
diff --git a/Documentation/core-api/percpu-counter-tree.rst b/Documentation/core-api/percpu-counter-tree.rst
new file mode 100644
index 0000000000000..196da056e7b42
--- /dev/null
+++ b/Documentation/core-api/percpu-counter-tree.rst
@@ -0,0 +1,75 @@
+========================================
+The Hierarchical Per-CPU Counters (HPCC)
+========================================
+
+:Author: Mathieu Desnoyers
+
+Introduction
+============
+
+Counters come in many varieties, each with their own trade offs:
+
+ * A global atomic counter provides a fast read access to the current
+   sum, at the expense of cache-line bouncing on updates. This leads to
+   poor performance of frequent updates from various cores on large SMP
+   systems.
+
+ * A per-cpu split counter provides fast updates to per-cpu counters,
+   at the expense of a slower aggregation (sum). The sum operation needs
+   to iterate over all per-cpu counters to calculate the current total.
+
+The hierarchical per-cpu counters attempt to provide the best of both
+worlds (fast updates, and fast sum) by relaxing requirements on the sum
+accuracy. It allows quickly querying an approximated sum value, along
+with the possible min/max ranges of the associated precise sum. The
+exact precise sum can still be calculated with an iteration on all
+per-cpu counter, but the availability of an approximated sum value with
+possible precise sum min/max ranges allows eliminating candidates which
+are certainly outside of a known target range without the overhead of
+precise sums.
+
+Overview
+========
+
+The herarchical per-cpu counters are organized as a tree with the tree
+root at the bottom (last level) and the first level of the tree
+consisting of per-cpu counters.
+
+The intermediate tree levels contain carry propagation counters. When
+reaching a threshold (batch size), the carry is propagated down the
+tree.
+
+This allows reading an approximated value at the root, which has a
+bounded accuracy (minimum/maximum possible precise sum range) determined
+by the tree topology.
+
+Use Cases
+=========
+
+Use cases HPCC is meant to handle invove tracking resources which are
+used across many CPUs to quickly sum as feedback for decision making to
+apply throttling, quota limits, sort tasks, and perform memory or task
+migration decisions. When considering approximated sums within the
+accuracy range of the decision threshold, the user can either:
+
+ * Be conservative and fast: Consider that the sum has reached the
+   limit as soon as the given limit is within the approximation range.
+
+ * Be aggressive and fast: Consider that the sum is over the
+   limit only when the approximation range is over the given limit.
+
+ * Be precise and slow: Do a precise comparison with the limit, which
+   requires a precise sum when the limit is within the approximated
+   range.
+
+One use-case for these hierarchical counters is to implement a two-pass
+algorithm to speed up sorting picking a maximum/minimunm sum value from
+a set. A first pass compares the approximated values, and then a second
+pass only needs the precise sum for counter trees which are within the
+possible precise sum range of the counter tree chosen by the first pass.
+
+Functions and structures
+========================
+
+.. kernel-doc:: include/linux/percpu_counter_tree.h
+.. kernel-doc:: lib/percpu_counter_tree.c
diff --git a/Documentation/core-api/printk-basics.rst b/Documentation/core-api/printk-basics.rst
index 2dde24ca7d9f6..48eaff0ce44cb 100644
--- a/Documentation/core-api/printk-basics.rst
+++ b/Documentation/core-api/printk-basics.rst
@@ -103,6 +103,42 @@ For debugging purposes there are also two conditionally-compiled macros:
 pr_debug() and pr_devel(), which are compiled-out unless ``DEBUG`` (or
 also ``CONFIG_DYNAMIC_DEBUG`` in the case of pr_debug()) is defined.
 
+Avoiding lockups from excessive printk() use
+============================================
+
+.. note::
+
+   This section is relevant only for legacy console drivers (those not
+   using the nbcon API) and !PREEMPT_RT kernels. Once all console drivers
+   are updated to nbcon, this documentation can be removed.
+
+Using ``printk()`` in hot paths (such as interrupt handlers, timer
+callbacks, or high-frequency network receive routines) with legacy
+consoles (e.g., ``console=ttyS0``) may cause lockups. Legacy consoles
+synchronously acquire ``console_sem`` and block while flushing messages,
+potentially disabling interrupts long enough to trigger hard or soft
+lockup detectors.
+
+To avoid this:
+
+- Use rate-limited variants (e.g., ``pr_*_ratelimited()``) or one-time
+  macros (e.g., ``pr_*_once()``) to reduce message frequency.
+- Assign lower log levels (e.g., ``KERN_DEBUG``) to non-essential messages
+  and filter console output via ``console_loglevel``.
+- Use ``printk_deferred()`` to log messages immediately to the ringbuffer
+  and defer console printing. This is a workaround for legacy consoles.
+- Port legacy console drivers to the non-blocking ``nbcon`` API (indicated
+  by ``CON_NBCON``). This is the preferred solution, as nbcon consoles
+  offload message printing to a dedicated kernel thread.
+
+For temporary debugging, ``trace_printk()`` can be used, but it must not
+appear in mainline code. See ``Documentation/trace/debugging.rst`` for
+more information.
+
+If more permanent output is needed in a hot path, trace events can be used.
+See ``Documentation/trace/events.rst`` and
+``samples/trace_events/trace-events-sample.[ch]``.
+
 
 Function reference
 ==================
diff --git a/Documentation/core-api/symbol-namespaces.rst b/Documentation/core-api/symbol-namespaces.rst
index 034898e81ba20..2304d5bffccea 100644
--- a/Documentation/core-api/symbol-namespaces.rst
+++ b/Documentation/core-api/symbol-namespaces.rst
@@ -114,6 +114,11 @@ inspected with modinfo::
 	import_ns:      USB_STORAGE
 	[...]
 
+For modules that are currently loaded, imported namespaces are also available
+via sysfs::
+
+	$ cat /sys/module/ums_karma/import_ns
+	USB_STORAGE
 
 It is advisable to add the MODULE_IMPORT_NS() statement close to other module
 metadata definitions like MODULE_AUTHOR() or MODULE_LICENSE().
diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst
index 8158b363cd98f..021759198fe77 100644
--- a/Documentation/crypto/userspace-if.rst
+++ b/Documentation/crypto/userspace-if.rst
@@ -23,7 +23,7 @@ user space, however. This includes the difference between synchronous
 and asynchronous invocations. The user space API call is fully
 synchronous.
 
-[1] https://www.chronox.de/libkcapi.html
+[1] https://www.chronox.de/libkcapi/index.html
 
 User Space API General Remarks
 ------------------------------
@@ -406,4 +406,4 @@ Please see [1] for libkcapi which provides an easy-to-use wrapper around
 the aforementioned Netlink kernel interface. [1] also contains a test
 application that invokes all libkcapi API calls.
 
-[1] https://www.chronox.de/libkcapi.html
+[1] https://www.chronox.de/libkcapi/index.html
diff --git a/Documentation/dev-tools/coccinelle.rst b/Documentation/dev-tools/coccinelle.rst
index c714780d458a0..f73ccf5397f3f 100644
--- a/Documentation/dev-tools/coccinelle.rst
+++ b/Documentation/dev-tools/coccinelle.rst
@@ -29,6 +29,7 @@ of many distributions, e.g. :
  - Ubuntu
  - OpenSUSE
  - Arch Linux
+ - Gentoo
  - NetBSD
  - FreeBSD
 
@@ -60,7 +61,7 @@ Supplemental documentation
 
 For supplemental documentation refer to the wiki:
 
-https://bottest.wiki.kernel.org/coccicheck
+https://bottest.wiki.kernel.org/coccicheck.html
 
 The wiki documentation always refers to the linux-next version of the script.
 
diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst
index a034700da7c46..4968b2aa60c80 100644
--- a/Documentation/dev-tools/kasan.rst
+++ b/Documentation/dev-tools/kasan.rst
@@ -75,9 +75,6 @@ Software Tag-Based KASAN supports slab, page_alloc, vmalloc, and stack memory.
 Hardware Tag-Based KASAN supports slab, page_alloc, and non-executable vmalloc
 memory.
 
-For slab, both software KASAN modes support SLUB and SLAB allocators, while
-Hardware Tag-Based KASAN only supports SLUB.
-
 Usage
 -----
 
diff --git a/Documentation/dev-tools/kfence.rst b/Documentation/dev-tools/kfence.rst
index 5418993538653..b03d1201ddaea 100644
--- a/Documentation/dev-tools/kfence.rst
+++ b/Documentation/dev-tools/kfence.rst
@@ -81,6 +81,13 @@ tables being allocated.
 Error reports
 ~~~~~~~~~~~~~
 
+The boot parameter ``kfence.fault`` can be used to control the behavior when a
+KFENCE error is detected:
+
+- ``kfence.fault=report``: Print the error report and continue (default).
+- ``kfence.fault=oops``: Print the error report and oops.
+- ``kfence.fault=panic``: Print the error report and panic.
+
 A typical out-of-bounds access looks like this::
 
     ==================================================================
diff --git a/Documentation/dev-tools/kunit/run_wrapper.rst b/Documentation/dev-tools/kunit/run_wrapper.rst
index 3c0b585dcfffb..770bb09a475ae 100644
--- a/Documentation/dev-tools/kunit/run_wrapper.rst
+++ b/Documentation/dev-tools/kunit/run_wrapper.rst
@@ -336,6 +336,8 @@ command line arguments:
 - ``--list_tests_attr``: If set, lists all tests that will be run and all of their
   attributes.
 
+- ``--list_suites``: If set, lists all suites that will be run.
+
 Command-line completion
 ==============================
 
diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml
index 13a3a96968214..206686f3eebce 100644
--- a/Documentation/devicetree/bindings/arm/altera.yaml
+++ b/Documentation/devicetree/bindings/arm/altera.yaml
@@ -84,6 +84,12 @@ properties:
               - altr,socfpga-stratix10-swvp
           - const: altr,socfpga-stratix10
 
+      - description: Stratix 10 SoCDK eMMC variant
+        items:
+          - const: altr,socfpga-stratix10-socdk-emmc
+          - const: altr,socfpga-stratix10-socdk
+          - const: altr,socfpga-stratix10
+
       - description: AgileX boards
         items:
           - enum:
@@ -105,6 +111,7 @@ properties:
           - enum:
               - intel,socfpga-agilex5-socdk
               - intel,socfpga-agilex5-socdk-013b
+              - intel,socfpga-agilex5-socdk-modular
               - intel,socfpga-agilex5-socdk-nand
           - const: intel,socfpga-agilex5
 
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
index 2a91670ccb8cd..949444aba1f8b 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
@@ -128,6 +128,9 @@ properties:
   "#address-cells":
     const: 1
 
+  access-controllers:
+    maxItems: 1
+
 patternProperties:
   '^trig-conns@([0-9]+)$':
     type: object
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml
index b74db15e5f8af..b0693cd46d27a 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml
@@ -78,6 +78,9 @@ properties:
         description: Output connection to CoreSight Trace bus
         $ref: /schemas/graph.yaml#/properties/port
 
+  access-controllers:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml
index 71f2e1ed27e5b..10ebbbeadf939 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml
@@ -118,6 +118,9 @@ properties:
         description: Output connection from the ETM to CoreSight Trace bus.
         $ref: /schemas/graph.yaml#/properties/port
 
+  access-controllers:
+    maxItems: 1
+
 required:
   - compatible
   - clocks
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml
index 378380c3f5aab..f243e76f597f8 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml
@@ -73,6 +73,9 @@ properties:
         description: Output connection to the CoreSight Trace bus.
         $ref: /schemas/graph.yaml#/properties/port
 
+  access-controllers:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml
index 96dd5b5f771a3..9dc096698c657 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml
@@ -128,6 +128,9 @@ properties:
       - const: tracedata
       - const: metadata
 
+  access-controllers:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml
index a207f6899e675..29bbc3961fdf4 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml
@@ -70,6 +70,9 @@ properties:
         description: Input connection from the CoreSight Trace bus.
         $ref: /schemas/graph.yaml#/properties/port
 
+  access-controllers:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
index ba04576f0ad63..95d4baa85506f 100644
--- a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
@@ -119,6 +119,16 @@ properties:
         items:
           - const: arm,foundation-aarch64
           - const: arm,vexpress
+      - description: Arm Zena Compute Subsystem Platforms
+          Arm Zena Compute Subsystem (CSS) is a compute platform targeting
+          the automotive sector. Arm Zena CSS is a high-performance Arm
+          Cortex-A720AE Application Processor system augmented with an Arm
+          Cortex-R82AE based Safety Island and real-time domain.
+        items:
+          - enum:
+              - arm,zena-css-fvp
+          - const: arm,zena-css
+          - const: arm,vexpress
 
   arm,vexpress,position:
     description: When daughterboards are stacked on one site, their position
diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
index f9925a14680ea..8ec7a3e74a21d 100644
--- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
+++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
@@ -35,6 +35,7 @@ properties:
               - ampere,mtjade-bmc
               - aspeed,ast2500-evb
               - asrock,altrad8-bmc
+              - asrock,ast2500-paul-ipmi-card
               - asrock,e3c246d4i-bmc
               - asrock,e3c256d4i-bmc
               - asrock,romed8hm3-bmc
@@ -80,6 +81,7 @@ properties:
               - ampere,mtmitchell-bmc
               - aspeed,ast2600-evb
               - aspeed,ast2600-evb-a1
+              - asus,ast2600-kommando-ipmi-card
               - asus,x4tf-bmc
               - facebook,anacapa-bmc
               - facebook,bletchley-bmc
diff --git a/Documentation/devicetree/bindings/arm/atmel,at91rm9200-sdramc.yaml b/Documentation/devicetree/bindings/arm/atmel,at91rm9200-sdramc.yaml
new file mode 100644
index 0000000000000..ac7e0f454a34a
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/atmel,at91rm9200-sdramc.yaml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/atmel,at91rm9200-sdramc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip (Atmel) SDRAM / DDR Controller (RAMC / DDRAMC / UDDRC)
+
+maintainers:
+  - Nicolas Ferre <nicolas.ferre@microchip.com>
+  - Claudiu Beznea <claudiu.beznea@tuxon.dev>
+
+description:
+  The SDRAM/DDR Controller (often called RAMC or DDRAMC) in various
+  Atmel/Microchip ARM9 and Cortex-A5/A7 SoCs  manages external
+  SDRAM / DDR memory. It is typically exposed as a syscon node for
+  register access from other drivers (e.g. for initialization or mode
+  configuration). No interrupts or clocks are usually required in the
+  binding.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: atmel,at91rm9200-sdramc
+          - const: syscon
+      - items:
+          - const: microchip,sama7d65-uddrc
+          - const: microchip,sama7g5-uddrc
+      - enum:
+          - atmel,at91sam9260-sdramc
+          - atmel,at91sam9g45-ddramc
+          - atmel,sama5d3-ddramc
+          - microchip,sam9x60-ddramc
+          - microchip,sam9x7-ddramc
+          - microchip,sama7g5-uddrc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 2
+
+  clock-names:
+    minItems: 1
+    items:
+      - const: ddrck
+      - const: mpddr
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/at91.h>
+    ramc@ffffe400 {
+        compatible = "atmel,at91sam9g45-ddramc";
+        reg = <0xffffe400 0x200>;
+        clocks = <&pmc PMC_TYPE_SYSTEM 2>;
+        clock-names = "ddrck";
+    };
+...
diff --git a/Documentation/devicetree/bindings/arm/atmel,at91rm9200-st.yaml b/Documentation/devicetree/bindings/arm/atmel,at91rm9200-st.yaml
new file mode 100644
index 0000000000000..3f6a934a2a690
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/atmel,at91rm9200-st.yaml
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/atmel,at91rm9200-st.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Atmel System Timer
+
+maintainers:
+  - Nicolas Ferre <nicolas.ferre@microchip.com>
+  - Claudiu Beznea <claudiu.beznea@tuxon.dev>
+
+description:
+  The System Timer (ST) module in AT91RM9200 provides periodic tick and
+  alarm capabilities. It is exposed as a simple multi-function device
+  (simple-mfd + syscon) because it shares its register space and interrupt
+  with other System Controller blocks.
+
+properties:
+  compatible:
+    items:
+      - const: atmel,at91rm9200-st
+      - const: syscon
+      - const: simple-mfd
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+patternProperties:
+  "^watchdog@[0-9a-f]+$":
+    $ref: /schemas/watchdog/atmel,at91rm9200-wdt.yaml#
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    timer@fffffd00 {
+        compatible = "atmel,at91rm9200-st", "syscon", "simple-mfd";
+        reg = <0xfffffd00 0x100>;
+        interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
+        clocks = <&slow_xtal>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+
+        watchdog@fffffd40 {
+            compatible = "atmel,at91rm9200-wdt";
+            reg = <0xfffffd40 0x40>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/arm/atmel,at91sam9260-pit.yaml b/Documentation/devicetree/bindings/arm/atmel,at91sam9260-pit.yaml
new file mode 100644
index 0000000000000..d1bdc4a4f9e02
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/atmel,at91sam9260-pit.yaml
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/atmel,at91sam9260-pit.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Atmel AT91SAM9260 Periodic Interval Timer (PIT)
+
+maintainers:
+  - Nicolas Ferre <nicolas.ferre@microchip.com>
+  - Claudiu Beznea <claudiu.beznea@tuxon.dev>
+
+description:
+  The Periodic Interval Timer (PIT) is part of the System Controller of
+  various Microchip 32-bit ARM-based SoCs (formerly Atmel AT91 series).
+  It is a simple down-counter timer used mainly as the kernel tick source.
+  The PIT is clocked from the slow clock and shares a single IRQ line with
+  other System Controller peripherals.
+
+properties:
+  compatible:
+    const: atmel,at91sam9260-pit
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    timer@fffffd30 {
+        compatible = "atmel,at91sam9260-pit";
+        reg = <0xfffffd30 0x10>;
+        interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk32k>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/arm/atmel-at91.yaml b/Documentation/devicetree/bindings/arm/atmel-at91.yaml
index 68d306d17c2a1..bf161e0950ea2 100644
--- a/Documentation/devicetree/bindings/arm/atmel-at91.yaml
+++ b/Documentation/devicetree/bindings/arm/atmel-at91.yaml
@@ -243,6 +243,12 @@ properties:
           - const: microchip,lan9668
           - const: microchip,lan966
 
+      - description: Microchip LAN9696 EV23X71A Evaluation Board
+        items:
+          - const: microchip,ev23x71a
+          - const: microchip,lan9696
+          - const: microchip,lan9691
+
       - description: Kontron KSwitch D10 MMT series
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt b/Documentation/devicetree/bindings/arm/atmel-sysregs.txt
deleted file mode 100644
index 5ce54f9befe62..0000000000000
--- a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-Atmel system registers
-
-Chipid required properties:
-- compatible: Should be "atmel,sama5d2-chipid" or "microchip,sama7g5-chipid"
-			"microchip,sama7d65-chipid"
-- reg : Should contain registers location and length
-
-PIT Timer required properties:
-- compatible: Should be "atmel,at91sam9260-pit"
-- reg: Should contain registers location and length
-- interrupts: Should contain interrupt for the PIT which is the IRQ line
-  shared across all System Controller members.
-
-PIT64B Timer required properties:
-- compatible: Should be "microchip,sam9x60-pit64b" or
-			"microchip,sam9x7-pit64b", "microchip,sam9x60-pit64b"
-			"microchip,sama7d65-pit64b", "microchip,sam9x60-pit64b"
-- reg: Should contain registers location and length
-- interrupts: Should contain interrupt for PIT64B timer
-- clocks: Should contain the available clock sources for PIT64B timer.
-
-System Timer (ST) required properties:
-- compatible: Should be "atmel,at91rm9200-st", "syscon", "simple-mfd"
-- reg: Should contain registers location and length
-- interrupts: Should contain interrupt for the ST which is the IRQ line
-  shared across all System Controller members.
-- clocks: phandle to input clock.
-Its subnodes can be:
-- watchdog: compatible should be "atmel,at91rm9200-wdt"
-
-RAMC SDRAM/DDR Controller required properties:
-- compatible: Should be "atmel,at91rm9200-sdramc", "syscon" or
-			"atmel,at91sam9260-sdramc" or
-			"atmel,at91sam9g45-ddramc" or
-			"atmel,sama5d3-ddramc" or
-			"microchip,sam9x60-ddramc" or
-			"microchip,sama7g5-uddrc" or
-			"microchip,sama7d65-uddrc", "microchip,sama7g5-uddrc" or
-			"microchip,sam9x7-ddramc", "atmel,sama5d3-ddramc".
-- reg: Should contain registers location and length
-
-Examples:
-
-	ramc0: ramc@ffffe800 {
-		compatible = "atmel,at91sam9g45-ddramc";
-		reg = <0xffffe800 0x200>;
-	};
-
diff --git a/Documentation/devicetree/bindings/arm/axis.yaml b/Documentation/devicetree/bindings/arm/axis.yaml
index 63e9aca85db7a..3062901196a69 100644
--- a/Documentation/devicetree/bindings/arm/axis.yaml
+++ b/Documentation/devicetree/bindings/arm/axis.yaml
@@ -31,6 +31,12 @@ properties:
               - axis,artpec8-grizzly
           - const: axis,artpec8
 
+      - description: Axis ARTPEC-9 SoC board
+        items:
+          - enum:
+              - axis,artpec9-alfred
+          - const: axis,artpec9
+
 additionalProperties: true
 
 ...
diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml
index 736b7ab1bd0a0..67d65c7f1df60 100644
--- a/Documentation/devicetree/bindings/arm/cpus.yaml
+++ b/Documentation/devicetree/bindings/arm/cpus.yaml
@@ -79,149 +79,160 @@ properties:
       All other bits in the reg cells must be set to 0.
 
   compatible:
-    enum:
-      - apm,potenza
-      - apm,strega
-      - apple,avalanche
-      - apple,blizzard
-      - apple,cyclone
-      - apple,firestorm
-      - apple,hurricane-zephyr
-      - apple,icestorm
-      - apple,mistral
-      - apple,monsoon
-      - apple,twister
-      - apple,typhoon
-      - arm,arm710t
-      - arm,arm720t
-      - arm,arm740t
-      - arm,arm7ej-s
-      - arm,arm7tdmi
-      - arm,arm7tdmi-s
-      - arm,arm9es
-      - arm,arm9ej-s
-      - arm,arm920t
-      - arm,arm922t
-      - arm,arm925
-      - arm,arm926e-s
-      - arm,arm926ej-s
-      - arm,arm940t
-      - arm,arm946e-s
-      - arm,arm966e-s
-      - arm,arm968e-s
-      - arm,arm9tdmi
-      - arm,arm1020e
-      - arm,arm1020t
-      - arm,arm1022e
-      - arm,arm1026ej-s
-      - arm,arm1136j-s
-      - arm,arm1136jf-s
-      - arm,arm1156t2-s
-      - arm,arm1156t2f-s
-      - arm,arm1176jzf
-      - arm,arm1176jz-s
-      - arm,arm1176jzf-s
-      - arm,arm11mpcore
-      - arm,armv8 # Only for s/w models
-      - arm,c1-nano
-      - arm,c1-premium
-      - arm,c1-pro
-      - arm,c1-ultra
-      - arm,cortex-a5
-      - arm,cortex-a7
-      - arm,cortex-a8
-      - arm,cortex-a9
-      - arm,cortex-a12
-      - arm,cortex-a15
-      - arm,cortex-a17
-      - arm,cortex-a32
-      - arm,cortex-a34
-      - arm,cortex-a35
-      - arm,cortex-a53
-      - arm,cortex-a55
-      - arm,cortex-a57
-      - arm,cortex-a65
-      - arm,cortex-a72
-      - arm,cortex-a73
-      - arm,cortex-a75
-      - arm,cortex-a76
-      - arm,cortex-a77
-      - arm,cortex-a78
-      - arm,cortex-a78ae
-      - arm,cortex-a78c
-      - arm,cortex-a320
-      - arm,cortex-a510
-      - arm,cortex-a520
-      - arm,cortex-a520ae
-      - arm,cortex-a710
-      - arm,cortex-a715
-      - arm,cortex-a720
-      - arm,cortex-a720ae
-      - arm,cortex-a725
-      - arm,cortex-m0
-      - arm,cortex-m0+
-      - arm,cortex-m1
-      - arm,cortex-m3
-      - arm,cortex-m4
-      - arm,cortex-r4
-      - arm,cortex-r5
-      - arm,cortex-r7
-      - arm,cortex-r52
-      - arm,cortex-x1
-      - arm,cortex-x1c
-      - arm,cortex-x2
-      - arm,cortex-x3
-      - arm,cortex-x4
-      - arm,cortex-x925
-      - arm,neoverse-e1
-      - arm,neoverse-n1
-      - arm,neoverse-n2
-      - arm,neoverse-n3
-      - arm,neoverse-v1
-      - arm,neoverse-v2
-      - arm,neoverse-v3
-      - arm,neoverse-v3ae
-      - arm,rainier
-      - brcm,brahma-b15
-      - brcm,brahma-b53
-      - brcm,vulcan
-      - cavium,thunder
-      - cavium,thunder2
-      - faraday,fa526
-      - intel,sa110
-      - intel,sa1100
-      - marvell,feroceon
-      - marvell,mohawk
-      - marvell,pj4a
-      - marvell,pj4b
-      - marvell,sheeva-v5
-      - marvell,sheeva-v7
-      - nvidia,tegra132-denver
-      - nvidia,tegra186-denver
-      - nvidia,tegra194-carmel
-      - qcom,krait
-      - qcom,kryo
-      - qcom,kryo240
-      - qcom,kryo250
-      - qcom,kryo260
-      - qcom,kryo280
-      - qcom,kryo360
-      - qcom,kryo385
-      - qcom,kryo465
-      - qcom,kryo468
-      - qcom,kryo470
-      - qcom,kryo485
-      - qcom,kryo560
-      - qcom,kryo570
-      - qcom,kryo660
-      - qcom,kryo670
-      - qcom,kryo685
-      - qcom,kryo780
-      - qcom,oryon
-      - qcom,scorpion
-      - samsung,mongoose-m2
-      - samsung,mongoose-m3
-      - samsung,mongoose-m5
+    oneOf:
+      - enum:
+          - apm,potenza
+          - apm,strega
+          - apple,avalanche
+          - apple,blizzard
+          - apple,cyclone
+          - apple,firestorm
+          - apple,hurricane-zephyr
+          - apple,icestorm
+          - apple,mistral
+          - apple,monsoon
+          - apple,twister
+          - apple,typhoon
+          - arm,arm710t
+          - arm,arm720t
+          - arm,arm740t
+          - arm,arm7ej-s
+          - arm,arm7tdmi
+          - arm,arm7tdmi-s
+          - arm,arm9es
+          - arm,arm9ej-s
+          - arm,arm920t
+          - arm,arm922t
+          - arm,arm925
+          - arm,arm926e-s
+          - arm,arm926ej-s
+          - arm,arm940t
+          - arm,arm946e-s
+          - arm,arm966e-s
+          - arm,arm968e-s
+          - arm,arm9tdmi
+          - arm,arm1020e
+          - arm,arm1020t
+          - arm,arm1022e
+          - arm,arm1026ej-s
+          - arm,arm1136j-s
+          - arm,arm1136jf-s
+          - arm,arm1156t2-s
+          - arm,arm1156t2f-s
+          - arm,arm1176jzf
+          - arm,arm1176jz-s
+          - arm,arm1176jzf-s
+          - arm,arm11mpcore
+          - arm,armv8 # Only for s/w models
+          - arm,c1-nano
+          - arm,c1-premium
+          - arm,c1-pro
+          - arm,c1-ultra
+          - arm,cortex-a5
+          - arm,cortex-a7
+          - arm,cortex-a8
+          - arm,cortex-a9
+          - arm,cortex-a12
+          - arm,cortex-a15
+          - arm,cortex-a17
+          - arm,cortex-a32
+          - arm,cortex-a34
+          - arm,cortex-a35
+          - arm,cortex-a53
+          - arm,cortex-a55
+          - arm,cortex-a57
+          - arm,cortex-a65
+          - arm,cortex-a72
+          - arm,cortex-a73
+          - arm,cortex-a75
+          - arm,cortex-a76
+          - arm,cortex-a77
+          - arm,cortex-a78
+          - arm,cortex-a78ae
+          - arm,cortex-a78c
+          - arm,cortex-a320
+          - arm,cortex-a510
+          - arm,cortex-a520
+          - arm,cortex-a520ae
+          - arm,cortex-a710
+          - arm,cortex-a715
+          - arm,cortex-a720
+          - arm,cortex-a720ae
+          - arm,cortex-a725
+          - arm,cortex-m0
+          - arm,cortex-m0+
+          - arm,cortex-m1
+          - arm,cortex-m3
+          - arm,cortex-m4
+          - arm,cortex-r4
+          - arm,cortex-r5
+          - arm,cortex-r7
+          - arm,cortex-r52
+          - arm,cortex-x1
+          - arm,cortex-x1c
+          - arm,cortex-x2
+          - arm,cortex-x3
+          - arm,cortex-x4
+          - arm,cortex-x925
+          - arm,neoverse-e1
+          - arm,neoverse-n1
+          - arm,neoverse-n2
+          - arm,neoverse-n3
+          - arm,neoverse-v1
+          - arm,neoverse-v2
+          - arm,neoverse-v3
+          - arm,neoverse-v3ae
+          - arm,rainier
+          - brcm,brahma-b15
+          - brcm,brahma-b53
+          - brcm,vulcan
+          - cavium,thunder
+          - cavium,thunder2
+          - faraday,fa526
+          - intel,sa110
+          - intel,sa1100
+          - marvell,feroceon
+          - marvell,mohawk
+          - marvell,pj4a
+          - marvell,pj4b
+          - marvell,sheeva-v5
+          - marvell,sheeva-v7
+          - nvidia,tegra132-denver
+          - nvidia,tegra186-denver
+          - nvidia,tegra194-carmel
+          - qcom,krait
+          - qcom,kryo240
+          - qcom,kryo250
+          - qcom,kryo260
+          - qcom,kryo280
+          - qcom,kryo360
+          - qcom,kryo385
+          - qcom,kryo465
+          - qcom,kryo468
+          - qcom,kryo470
+          - qcom,kryo485
+          - qcom,kryo560
+          - qcom,kryo570
+          - qcom,kryo660
+          - qcom,kryo670
+          - qcom,kryo685
+          - qcom,kryo780
+          - qcom,oryon-1-1
+          - qcom,oryon-1-2
+          - qcom,oryon-1-3
+          - qcom,oryon-1-4
+          - qcom,oryon-2-1
+          - qcom,oryon-2-2
+          - qcom,oryon-2-3
+          - qcom,scorpion
+          - samsung,mongoose-m2
+          - samsung,mongoose-m3
+          - samsung,mongoose-m5
+      - enum:
+          - qcom,kryo
+          - qcom,oryon
+        # Too generic, do not use in new code
+        deprecated: true
 
   enable-method:
     $ref: /schemas/types.yaml#/definitions/string
diff --git a/Documentation/devicetree/bindings/arm/freescale/fsl,imx51-m4if.yaml b/Documentation/devicetree/bindings/arm/freescale/fsl,imx51-m4if.yaml
index 1f515bea39599..6130b048de7b1 100644
--- a/Documentation/devicetree/bindings/arm/freescale/fsl,imx51-m4if.yaml
+++ b/Documentation/devicetree/bindings/arm/freescale/fsl,imx51-m4if.yaml
@@ -15,6 +15,7 @@ properties:
   compatible:
     oneOf:
       - enum:
+          - fsl,imx25-aips
           - fsl,imx51-m4if
           - fsl,imx51-tigerp
           - fsl,imx51-aipstz
diff --git a/Documentation/devicetree/bindings/arm/freescale/fsl,imx7ulp-pm.yaml b/Documentation/devicetree/bindings/arm/freescale/fsl,imx7ulp-pm.yaml
index 9d377e193c123..7ad470260c0d0 100644
--- a/Documentation/devicetree/bindings/arm/freescale/fsl,imx7ulp-pm.yaml
+++ b/Documentation/devicetree/bindings/arm/freescale/fsl,imx7ulp-pm.yaml
@@ -28,6 +28,9 @@ properties:
   reg:
     maxItems: 1
 
+  '#clock-cells':
+    const: 1
+
   clocks:
     maxItems: 2
 
@@ -39,6 +42,7 @@ properties:
 required:
   - compatible
   - reg
+  - '#clock-cells'
 
 additionalProperties: false
 
@@ -47,4 +51,5 @@ examples:
     smc1@40410000 {
         compatible = "fsl,imx7ulp-smc1";
         reg = <0x40410000 0x1000>;
+        #clock-cells = <1>;
     };
diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml
index 5716d701292cf..da2be7114f641 100644
--- a/Documentation/devicetree/bindings/arm/fsl.yaml
+++ b/Documentation/devicetree/bindings/arm/fsl.yaml
@@ -996,6 +996,14 @@ properties:
           - const: engicam,icore-mx8mm             # i.MX8MM Engicam i.Core MX8M Mini SoM
           - const: fsl,imx8mm
 
+      - description: Ka-Ro Electronics TX8M-1610 based boards
+        items:
+          - enum:
+              - gocontroll,moduline-iv-306-d
+              - gocontroll,moduline-mini-111
+          - const: karo,tx8m-1610
+          - const: fsl,imx8mm
+
       - description: Kontron BL i.MX8MM (N801X S) Board
         items:
           - const: kontron,imx8mm-bl
@@ -1041,6 +1049,13 @@ properties:
           - const: phytec,imx8mm-phycore-som        # phyCORE-i.MX8MM SoM
           - const: fsl,imx8mm
 
+      - description: SolidRun i.MX8MM SoM based boards
+        items:
+          - enum:
+              - solidrun,imx8mm-hummingboard-ripple # SolidRun i.MX8MM SoM on HummingBoard Ripple
+          - const: solidrun,imx8mm-sr-som
+          - const: fsl,imx8mm
+
       - description: Variscite VAR-SOM-MX8MM based boards
         items:
           - const: variscite,var-som-mx8mm-symphony
@@ -1069,6 +1084,7 @@ properties:
               - fsl,imx8mn-ddr4-evk       # i.MX8MN DDR4 EVK Board
               - fsl,imx8mn-evk            # i.MX8MN LPDDR4 EVK Board
               - gw,imx8mn-gw7902          # i.MX8MM Gateworks Board
+              - solidrun,solidsense-n8-compact # SolidRun SolidSense N8 Compact
           - const: fsl,imx8mn
 
       - description: ifm i.MX8MN VHIP4 based boards
@@ -1106,6 +1122,7 @@ properties:
               - beacon,imx8mp-beacon-kit  # i.MX8MP Beacon Development Kit
               - dmo,imx8mp-data-modul-edm-sbc # i.MX8MP eDM SBC
               - emcraft,imx8mp-navqp      # i.MX8MP Emcraft Systems NavQ+ Kit
+              - fsl,imx8mp-ab2            # i.MX8MP Audio Board V2
               - fsl,imx8mp-evk            # i.MX8MP EVK Board
               - fsl,imx8mp-evk-revb4      # i.MX8MP EVK Rev B4 Board
               - fsl,imx8mp-frdm           # i.MX8MP Freedom Board
@@ -1225,6 +1242,7 @@ properties:
         items:
           - enum:
               - solidrun,imx8mp-cubox-m             # SolidRun i.MX8MP SoM on CuBox-M
+              - solidrun,imx8mp-hummingboard-iiot   # SolidRun i.MX8MP SoM on HummingBoard IIoT
               - solidrun,imx8mp-hummingboard-mate   # SolidRun i.MX8MP SoM on HummingBoard Mate
               - solidrun,imx8mp-hummingboard-pro    # SolidRun i.MX8MP SoM on HummingBoard Pro
               - solidrun,imx8mp-hummingboard-pulse  # SolidRun i.MX8MP SoM on HummingBoard Pulse
@@ -1420,6 +1438,16 @@ properties:
               - const: tq,imx8dxp-tqma8xdps         # TQ-Systems GmbH TQMa8XDPS SOM
               - const: fsl,imx8dxp
 
+      - description:
+          TQMa8x is a series of SOM featuring NXP i.MX8 system-on-chip
+          variants. It is designed to be clicked on different carrier boards
+          MBa8x is the starterkit
+        items:
+          - enum:
+              - tq,imx8qm-tqma8qm-mba8x   # TQ-Systems GmbH TQMa8QM SOM on MBa8x
+          - const: tq,imx8qm-tqma8qm      # TQ-Systems GmbH TQMa8QM SOM
+          - const: fsl,imx8qm
+
       - description: i.MX8ULP based Boards
         items:
           - enum:
@@ -1432,6 +1460,7 @@ properties:
           - enum:
               - fsl,imx91-11x11-evk       # i.MX91 11x11 EVK Board
               - fsl,imx91-11x11-frdm      # FRDM i.MX91 Development Board
+              - fsl,imx91-11x11-frdm-s    # FRDM i.MX91S Development Board
           - const: fsl,imx91
 
       - description: i.MX93 based Boards
@@ -1441,6 +1470,7 @@ properties:
               - fsl,imx93-11x11-evk       # i.MX93 11x11 EVK Board
               - fsl,imx93-11x11-frdm      # i.MX93 11x11 FRDM Board
               - fsl,imx93-14x14-evk       # i.MX93 14x14 EVK Board
+              - fsl,imx93-wireless-evk    # i.MX93 and IW610G WLCSP (Wi-Fi + BLE + 802.15.4) SiP EVK Board
           - const: fsl,imx93
 
       - description: i.MX94 based Boards
@@ -1477,6 +1507,12 @@ properties:
           - const: toradex,smarc-imx95     # Toradex SMARC iMX95 Module
           - const: fsl,imx95
 
+      - description: Variscite DART-MX95 based Boards
+        items:
+          - const: variscite,var-dart-mx95-sonata # Variscite DART-MX95 SOM on Sonata Development Board
+          - const: variscite,var-dart-mx95 # Variscite DART-MX95 SOM
+          - const: fsl,imx95
+
       - description: i.MXRT1050 based Boards
         items:
           - enum:
@@ -1545,6 +1581,12 @@ properties:
           - const: phytec,imx93-phycore-som # phyCORE-i.MX93 SoM
           - const: fsl,imx93
 
+      - description: Variscite DART-MX91 based boards
+        items:
+          - const: variscite,var-dart-mx91-sonata # Variscite DART-MX91 on Sonata Development Board
+          - const: variscite,var-dart-mx91 # Variscite DART-MX91 SOM
+          - const: fsl,imx91
+
       - description: Variscite VAR-SOM-MX93 based boards
         items:
           - const: variscite,var-som-mx93-symphony
@@ -1827,6 +1869,12 @@ properties:
               - fsl,s32v234-evb           # S32V234-EVB2 Customer Evaluation Board
           - const: fsl,s32v234
 
+      - description: S32N79 based Boards
+        items:
+          - enum:
+              - nxp,s32n79-rdb
+          - const: nxp,s32n79
+
       - description: Traverse LS1088A based Boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-7k-8k.yaml b/Documentation/devicetree/bindings/arm/marvell/armada-7k-8k.yaml
index 4bc7454a5d3ac..7e77310da626f 100644
--- a/Documentation/devicetree/bindings/arm/marvell/armada-7k-8k.yaml
+++ b/Documentation/devicetree/bindings/arm/marvell/armada-7k-8k.yaml
@@ -21,6 +21,17 @@ properties:
           - const: marvell,armada-ap806-dual
           - const: marvell,armada-ap806
 
+      - description:
+          Falcon (DB-98CX85x0) Development board COM Express Carrier plus
+          Armada 7020 SoC COM Express CPU module
+        items:
+          - const: marvell,armada7020-falcon-carrier
+          - const: marvell,db-falcon-carrier
+          - const: marvell,armada7020-cpu-module
+          - const: marvell,armada7020
+          - const: marvell,armada-ap806-dual
+          - const: marvell,armada-ap806
+
       - description: Armada 7040 SoC
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
index 09a6c16e7e829..9aa39b002361d 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
@@ -49,38 +49,37 @@ required:
   - '#clock-cells'
 
 allOf:
- - if:
-     properties:
-       compatible:
-         contains:
-           enum:
-             - mediatek,mt2701-audsys
-             - mediatek,mt7622-audsys
-   then:
-     properties:
-       audio-controller:
-         $ref: /schemas/sound/mediatek,mt2701-audio.yaml#
-
- - if:
-     properties:
-       compatible:
-         contains:
-           const: mediatek,mt8183-audiosys
-   then:
-     properties:
-       audio-controller:
-         $ref: /schemas/sound/mediatek,mt8183-audio.yaml#
-
- - if:
-     properties:
-       compatible:
-         contains:
-           const: mediatek,mt8192-audsys
-   then:
-     properties:
-       audio-controller:
-         $ref: /schemas/sound/mt8192-afe-pcm.yaml#
-
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - mediatek,mt2701-audsys
+              - mediatek,mt7622-audsys
+    then:
+      properties:
+        audio-controller:
+          $ref: /schemas/sound/mediatek,mt2701-audio.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: mediatek,mt8183-audiosys
+    then:
+      properties:
+        audio-controller:
+          $ref: /schemas/sound/mediatek,mt8183-audio.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: mediatek,mt8192-audsys
+    then:
+      properties:
+        audio-controller:
+          $ref: /schemas/sound/mt8192-afe-pcm.yaml#
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/arm/microchip,sam9x60-pit64b.yaml b/Documentation/devicetree/bindings/arm/microchip,sam9x60-pit64b.yaml
new file mode 100644
index 0000000000000..802cf2424c422
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/microchip,sam9x60-pit64b.yaml
@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/microchip,sam9x60-pit64b.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip PIT64B 64-bit Periodic Interval Timer
+
+maintainers:
+  - Nicolas Ferre <nicolas.ferre@microchip.com>
+  - Claudiu Beznea <claudiu.beznea@tuxon.dev>
+
+description:
+  The Microchip PIT64B is a 64-bit periodic interval timer used in
+  several modern Microchip ARM SoCs including SAM9X60, SAM9X7 and
+  SAMA7D65 families. It provides extended timing range, flexible
+  clock selection and supports both periodic and one-shot interrupt
+  generation modes.
+
+properties:
+  compatible:
+    oneOf:
+      - const: microchip,sam9x60-pit64b
+      - items:
+          - enum:
+              - microchip,sam9x7-pit64b
+              - microchip,sama7d65-pit64b
+              - microchip,sama7g5-pit64b
+          - const: microchip,sam9x60-pit64b
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 2
+
+  clock-names:
+    minItems: 1
+    maxItems: 2
+    items:
+      enum:
+        - pclk
+        - gclk
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/clock/at91.h>
+    timer@f0028000 {
+        compatible = "microchip,sama7g5-pit64b", "microchip,sam9x60-pit64b";
+        reg = <0xf0028000 0x100>;
+        interrupts = <37 IRQ_TYPE_LEVEL_HIGH 7>;
+        clocks = <&pmc PMC_TYPE_PERIPHERAL 37>, <&pmc PMC_TYPE_GCK 37>;
+        clock-names = "pclk", "gclk";
+    };
+...
diff --git a/Documentation/devicetree/bindings/arm/microchip,sama7g5-chipid.yaml b/Documentation/devicetree/bindings/arm/microchip,sama7g5-chipid.yaml
new file mode 100644
index 0000000000000..4d6442ba5ac91
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/microchip,sama7g5-chipid.yaml
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/microchip,sama7g5-chipid.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Atmel/Microchip RAMC SDRAM/DDR Controller
+
+maintainers:
+  - Nicolas Ferre <nicolas.ferre@microchip.com>
+  - Claudiu Beznea <claudiu.beznea@tuxon.dev>
+
+description:
+  This binding describes the Atmel/Microchip Chip ID register block used
+  for SoC identification and revision information. It requires compatible
+  strings matching specific SoC families and a reg property defining the
+  register address and size.
+
+properties:
+  compatible:
+    enum:
+      - atmel,sama5d2-chipid
+      - microchip,sama7d65-chipid
+      - microchip,sama7g5-chipid
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    chipid@fc069000 {
+        compatible = "atmel,sama5d2-chipid";
+        reg = <0xfc069000 0x8>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index d48c625d3fc42..b4c79f89b2854 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -61,6 +61,21 @@ properties:
               - qcom,apq8084-sbc
           - const: qcom,apq8084
 
+      - items:
+          - enum:
+              - qcom,eliza-mtp
+          - const: qcom,eliza
+
+      - items:
+          - enum:
+              - qcom,glymur-crd
+          - const: qcom,glymur
+
+      - items:
+          - enum:
+              - qcom,mahua-crd
+          - const: qcom,mahua
+
       - items:
           - enum:
               - fairphone,fp6
@@ -186,6 +201,7 @@ properties:
               - samsung,serranove
               - thwc,uf896
               - thwc,ufi001c
+              - wiko,chuppito
               - wingtech,wt86518
               - wingtech,wt86528
               - wingtech,wt88047
@@ -243,6 +259,13 @@ properties:
           - const: qcom,apq8096-sbc
           - const: qcom,apq8096
 
+      - items:
+          - const: arrow,apq8096sg-db820c
+          - const: arrow,apq8096-db820c
+          - const: qcom,apq8096-sbc
+          - const: qcom,apq8096sg
+          - const: qcom,apq8096
+
       - items:
           - enum:
               - oneplus,oneplus3
@@ -326,8 +349,10 @@ properties:
       - items:
           - enum:
               - qcom,ipq9574-ap-al02-c2
+              - qcom,ipq9574-ap-al02-c2-emmc
               - qcom,ipq9574-ap-al02-c6
               - qcom,ipq9574-ap-al02-c7
+              - qcom,ipq9574-ap-al02-c7-emmc
               - qcom,ipq9574-ap-al02-c8
               - qcom,ipq9574-ap-al02-c9
           - const: qcom,ipq9574
@@ -360,6 +385,7 @@ properties:
               - qcom,qcs6490-rb3gen2
               - radxa,dragon-q6a
               - shift,otter
+              - thundercomm,minipc-g1iot
               - thundercomm,rubikpi3
           - const: qcom,qcm6490
 
@@ -385,6 +411,7 @@ properties:
       - items:
           - enum:
               - acer,aspire1
+              - ecs,liva-qc710
               - qcom,sc7180-idp
           - const: qcom,sc7180
 
@@ -844,6 +871,12 @@ properties:
               - google,sargo
           - const: qcom,sdm670
 
+      - items:
+          - enum:
+              - google,bonito-tianma
+          - const: google,bonito
+          - const: qcom,sdm670
+
       - items:
           - enum:
               - qcom,sdx55-mtp
@@ -876,6 +909,7 @@ properties:
 
       - items:
           - enum:
+              - arduino,monza
               - qcom,monaco-evk
               - qcom,qcs8300-ride
           - const: qcom,qcs8300
@@ -966,6 +1000,7 @@ properties:
               - sony,pdx201
               - xiaomi,ginkgo
               - xiaomi,laurel-sprout
+              - xiaomi,willow
           - const: qcom,sm6125
 
       - items:
@@ -1122,6 +1157,12 @@ properties:
           - const: microsoft,denali
           - const: qcom,x1e80100
 
+      - items:
+          - enum:
+              - qcom,purwa-iot-evk
+          - const: qcom,purwa-iot-som
+          - const: qcom,x1p42100
+
       - items:
           - enum:
               - asus,zenbook-a14-ux3407qa-lcd
@@ -1131,6 +1172,7 @@ properties:
 
       - items:
           - enum:
+              - asus,vivobook-s15-x1p4
               - hp,omnibook-x14-fe1
               - lenovo,thinkbook-16
               - qcom,x1p42100-crd
diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml
index ae77ded9fe47a..91a387621bcbf 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip.yaml
@@ -813,6 +813,11 @@ properties:
           - const: openailab,eaidk-610
           - const: rockchip,rk3399
 
+      - description: OneThing Edge Cube series
+        items:
+          - const: onething,edge-cube
+          - const: rockchip,rk3566
+
       - description: Xunlong Orange Pi RK3399 board
         items:
           - const: xunlong,rk3399-orangepi
@@ -1187,7 +1192,9 @@ properties:
 
       - description: Rockchip RK3576 Evaluation board
         items:
-          - const: rockchip,rk3576-evb1-v10
+          - enum:
+              - rockchip,rk3576-evb1-v10
+              - rockchip,rk3576-evb2-v10
           - const: rockchip,rk3576
 
       - description: Rockchip RK3588 Evaluation board
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
index f8e20e602c205..753b3ba1b607d 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
@@ -117,6 +117,7 @@ properties:
       - description: Exynos5250 based boards
         items:
           - enum:
+              - google,manta                    # Google Manta (Nexus 10)
               - google,snow-rev5                # Google Snow Rev 5+
               - google,spring                   # Google Spring
               - insignal,arndale                # Insignal Arndale
@@ -216,7 +217,9 @@ properties:
         items:
           - enum:
               - samsung,a2corelte               # Samsung Galaxy A2 Core
+              - samsung,j5y17lte                # Samsung Galaxy J5 (2017)
               - samsung,j6lte                   # Samsung Galaxy J6
+              - samsung,j7xelte                 # Samsung Galaxy J7 (2016)
               - samsung,on7xelte                # Samsung Galaxy J7 Prime
           - const: samsung,exynos7870
 
diff --git a/Documentation/devicetree/bindings/arm/stm32/stm32.yaml b/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
index ad144c02eb7ed..c6af3a46364fc 100644
--- a/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
+++ b/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
@@ -183,10 +183,12 @@ properties:
           - const: seeed,stm32mp157c-odyssey-som
           - const: st,stm32mp157
 
-      - description: Phytec STM32MP1 SoM based Boards
+      - description: Phytec STM32MP157 SoM based Boards
         items:
-          - const: phytec,phycore-stm32mp1-3
-          - const: phytec,phycore-stm32mp157c-som
+          - enum:
+              - phytec,phycore-stm32mp1-3 # phyBOARD-Sargas with phyCORE-STM32MP157C SoM
+          - enum:
+              - phytec,phycore-stm32mp157c-som # phyCORE-STM32MP157C SoM
           - const: st,stm32mp157
 
       - description: Ultratronik STM32MP1 SBC based Boards
diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml
index 9e4627f97d7e3..e6443c266fa1e 100644
--- a/Documentation/devicetree/bindings/arm/sunxi.yaml
+++ b/Documentation/devicetree/bindings/arm/sunxi.yaml
@@ -901,6 +901,11 @@ properties:
           - const: allwinner,sl631
           - const: allwinner,sun8i-v3
 
+      - description: TaiqiCat A01
+        items:
+          - const: ultrapower,taiqicat-a01
+          - const: allwinner,sun50i-h6
+
       - description: Tanix TX1
         items:
           - const: oranth,tanix-tx1
diff --git a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra-ccplex-cluster.yaml b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra-ccplex-cluster.yaml
index 36dbd0838f2de..fe9c8791f2271 100644
--- a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra-ccplex-cluster.yaml
+++ b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra-ccplex-cluster.yaml
@@ -24,6 +24,7 @@ properties:
     enum:
       - nvidia,tegra186-ccplex-cluster
       - nvidia,tegra234-ccplex-cluster
+      - nvidia,tegra238-ccplex-cluster
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/arm/ti/k3.yaml b/Documentation/devicetree/bindings/arm/ti/k3.yaml
index 85deda6d42920..adda61e06173c 100644
--- a/Documentation/devicetree/bindings/arm/ti/k3.yaml
+++ b/Documentation/devicetree/bindings/arm/ti/k3.yaml
@@ -208,7 +208,6 @@ properties:
         items:
           - enum:
               - beagle,am67a-beagley-ai
-              - kontron,sa67             # Kontron SMARC-sAM67 board
               - ti,j722s-evm
           - const: ti,j722s
 
diff --git a/Documentation/devicetree/bindings/arm/ti/omap.yaml b/Documentation/devicetree/bindings/arm/ti/omap.yaml
index 14f1b9d8f59d9..f694dcbf23486 100644
--- a/Documentation/devicetree/bindings/arm/ti/omap.yaml
+++ b/Documentation/devicetree/bindings/arm/ti/omap.yaml
@@ -144,6 +144,8 @@ properties:
               - motorola,droid-bionic   # Motorola Droid Bionic XT875
               - motorola,xyboard-mz609
               - motorola,xyboard-mz617
+              - samsung,espresso7
+              - samsung,espresso10
               - ti,omap4-panda
               - ti,omap4-sdp
           - const: ti,omap4430
diff --git a/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml b/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml
deleted file mode 100644
index 9b7ca4759bd72..0000000000000
--- a/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml
+++ /dev/null
@@ -1,115 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/ata/baikal,bt1-ahci.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 SoC AHCI SATA controller
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description:
-  AHCI SATA controller embedded into the Baikal-T1 SoC is based on the
-  DWC AHCI SATA v4.10a IP-core.
-
-allOf:
-  - $ref: snps,dwc-ahci-common.yaml#
-
-properties:
-  compatible:
-    const: baikal,bt1-ahci
-
-  clocks:
-    items:
-      - description: Peripheral APB bus clock
-      - description: Application AXI BIU clock
-      - description: SATA Ports reference clock
-
-  clock-names:
-    items:
-      - const: pclk
-      - const: aclk
-      - const: ref
-
-  resets:
-    items:
-      - description: Application AXI BIU domain reset
-      - description: SATA Ports clock domain reset
-
-  reset-names:
-    items:
-      - const: arst
-      - const: ref
-
-  ports-implemented:
-    maximum: 0x3
-
-patternProperties:
-  "^sata-port@[0-1]$":
-    $ref: /schemas/ata/snps,dwc-ahci-common.yaml#/$defs/dwc-ahci-port
-
-    properties:
-      reg:
-        minimum: 0
-        maximum: 1
-
-      snps,tx-ts-max:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description:
-          Due to having AXI3 bus interface utilized the maximum Tx DMA
-          transaction size can't exceed 16 beats (AxLEN[3:0]).
-        enum: [ 1, 2, 4, 8, 16 ]
-
-      snps,rx-ts-max:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description:
-          Due to having AXI3 bus interface utilized the maximum Rx DMA
-          transaction size can't exceed 16 beats (AxLEN[3:0]).
-        enum: [ 1, 2, 4, 8, 16 ]
-
-    unevaluatedProperties: false
-
-required:
-  - compatible
-  - reg
-  - interrupts
-  - clocks
-  - clock-names
-  - resets
-
-unevaluatedProperties: false
-
-examples:
-  - |
-    sata@1f050000 {
-      compatible = "baikal,bt1-ahci";
-      reg = <0x1f050000 0x2000>;
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      interrupts = <0 64 4>;
-
-      clocks = <&ccu_sys 1>, <&ccu_axi 2>, <&sata_ref_clk>;
-      clock-names = "pclk", "aclk", "ref";
-
-      resets = <&ccu_axi 2>, <&ccu_sys 0>;
-      reset-names = "arst", "ref";
-
-      ports-implemented = <0x3>;
-
-      sata-port@0 {
-        reg = <0>;
-
-        snps,tx-ts-max = <4>;
-        snps,rx-ts-max = <4>;
-      };
-
-      sata-port@1 {
-        reg = <1>;
-
-        snps,tx-ts-max = <4>;
-        snps,rx-ts-max = <4>;
-      };
-    };
-...
diff --git a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml
index b90eec2077b4b..fe1272e86467e 100644
--- a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml
+++ b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml
@@ -66,7 +66,7 @@ then:
   required:
     - refresh-rate-hz
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml b/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml
deleted file mode 100644
index 37ba3337f944a..0000000000000
--- a/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/bus/baikal,bt1-apb.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 APB-bus
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description: |
-  Baikal-T1 CPU or DMAC MMIO requests are handled by the AMBA 3 AXI Interconnect
-  which routes them to the AXI-APB bridge. This interface is a single master
-  multiple slaves bus in turn serializing IO accesses and routing them to the
-  addressed APB slave devices. In case of any APB protocol collisions, slave
-  device not responding on timeout an IRQ is raised with an erroneous address
-  reported to the APB terminator (APB Errors Handler Block).
-
-allOf:
-  - $ref: /schemas/simple-bus.yaml#
-
-properties:
-  compatible:
-    contains:
-      const: baikal,bt1-apb
-
-  reg:
-    items:
-      - description: APB EHB MMIO registers
-      - description: APB MMIO region with no any device mapped
-
-  reg-names:
-    items:
-      - const: ehb
-      - const: nodev
-
-  interrupts:
-    maxItems: 1
-
-  clocks:
-    items:
-      - description: APB reference clock
-
-  clock-names:
-    items:
-      - const: pclk
-
-  resets:
-    items:
-      - description: APB domain reset line
-
-  reset-names:
-    items:
-      - const: prst
-
-unevaluatedProperties: false
-
-required:
-  - compatible
-  - reg
-  - reg-names
-  - interrupts
-  - clocks
-  - clock-names
-
-examples:
-  - |
-    #include <dt-bindings/interrupt-controller/mips-gic.h>
-
-    bus@1f059000 {
-      compatible = "baikal,bt1-apb", "simple-bus";
-      reg = <0x1f059000 0x1000>,
-            <0x1d000000 0x2040000>;
-      reg-names = "ehb", "nodev";
-      #address-cells = <1>;
-      #size-cells = <1>;
-
-      ranges;
-
-      interrupts = <GIC_SHARED 16 IRQ_TYPE_LEVEL_HIGH>;
-
-      clocks = <&ccu_sys 1>;
-      clock-names = "pclk";
-
-      resets = <&ccu_sys 1>;
-      reset-names = "prst";
-    };
-...
diff --git a/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml b/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml
deleted file mode 100644
index 4ac78b44e45eb..0000000000000
--- a/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/bus/baikal,bt1-axi.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 AXI-bus
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description: |
-  AXI3-bus is the main communication bus of Baikal-T1 SoC connecting all
-  high-speed peripheral IP-cores with RAM controller and with MIPS P5600
-  cores. Traffic arbitration is done by means of DW AXI Interconnect (so
-  called AXI Main Interconnect) routing IO requests from one block to
-  another: from CPU to SoC peripherals and between some SoC peripherals
-  (mostly between peripheral devices and RAM, but also between DMA and
-  some peripherals). In case of any protocol error, device not responding
-  an IRQ is raised and a faulty situation is reported to the AXI EHB
-  (Errors Handler Block) embedded on top of the DW AXI Interconnect and
-  accessible by means of the Baikal-T1 System Controller.
-
-allOf:
-  - $ref: /schemas/simple-bus.yaml#
-
-properties:
-  compatible:
-    contains:
-      const: baikal,bt1-axi
-
-  reg:
-    minItems: 1
-    items:
-      - description: Synopsys DesignWare AXI Interconnect QoS registers
-      - description: AXI EHB MMIO system controller registers
-
-  reg-names:
-    minItems: 1
-    items:
-      - const: qos
-      - const: ehb
-
-  '#interconnect-cells':
-    const: 1
-
-  syscon:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description: Phandle to the Baikal-T1 System Controller DT node
-
-  interrupts:
-    maxItems: 1
-
-  clocks:
-    items:
-      - description: Main Interconnect uplink reference clock
-
-  clock-names:
-    items:
-      - const: aclk
-
-  resets:
-    items:
-      - description: Main Interconnect reset line
-
-  reset-names:
-    items:
-      - const: arst
-
-unevaluatedProperties: false
-
-required:
-  - compatible
-  - reg
-  - reg-names
-  - syscon
-  - interrupts
-  - clocks
-  - clock-names
-
-examples:
-  - |
-    #include <dt-bindings/interrupt-controller/mips-gic.h>
-
-    bus@1f05a000 {
-      compatible = "baikal,bt1-axi", "simple-bus";
-      reg = <0x1f05a000 0x1000>,
-            <0x1f04d110 0x8>;
-      reg-names = "qos", "ehb";
-      #address-cells = <1>;
-      #size-cells = <1>;
-      #interconnect-cells = <1>;
-
-      syscon = <&syscon>;
-
-      ranges;
-
-      interrupts = <GIC_SHARED 127 IRQ_TYPE_LEVEL_HIGH>;
-
-      clocks = <&ccu_axi 0>;
-      clock-names = "aclk";
-
-      resets = <&ccu_axi 0>;
-      reset-names = "arst";
-    };
-...
diff --git a/Documentation/devicetree/bindings/bus/st,stm32mp131-dbg-bus.yaml b/Documentation/devicetree/bindings/bus/st,stm32mp131-dbg-bus.yaml
new file mode 100644
index 0000000000000..6c74433efbe34
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/st,stm32mp131-dbg-bus.yaml
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bus/st,stm32mp131-dbg-bus.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: STM32 Coresight bus
+
+maintainers:
+  - Gatien Chevallier <gatien.chevallier@foss.st.com>
+
+description:
+  The STM32 debug bus is in charge of checking the debug configuration
+  of the platform before probing the peripheral drivers that rely on the debug
+  domain.
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - st,stm32mp131-dbg-bus
+          - st,stm32mp151-dbg-bus
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ranges:
+    minItems: 1
+    maxItems: 2
+
+  "#access-controller-cells":
+    const: 1
+    description:
+      Contains the debug profile necessary to access the peripheral.
+
+patternProperties:
+  "@[0-9a-f]+$":
+    description: Debug related peripherals
+    type: object
+
+    additionalProperties: true
+
+    required:
+      - access-controllers
+
+required:
+  - "#access-controller-cells"
+  - "#address-cells"
+  - "#size-cells"
+  - compatible
+  - ranges
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/stm32mp1-clks.h>
+
+    dbg_bus: bus@50080000 {
+      compatible = "st,stm32mp131-dbg-bus";
+      #address-cells = <1>;
+      #size-cells = <1>;
+      #access-controller-cells = <1>;
+      ranges = <0x50080000 0x50080000 0x3f80000>;
+
+      cti@50094000 {
+        compatible = "arm,coresight-cti", "arm,primecell";
+        reg = <0x50094000 0x1000>;
+        clocks = <&rcc CK_DBG>;
+        clock-names = "apb_pclk";
+        access-controllers = <&dbg_bus 0>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/cache/baikal,bt1-l2-ctl.yaml b/Documentation/devicetree/bindings/cache/baikal,bt1-l2-ctl.yaml
deleted file mode 100644
index ec4f367bc0b41..0000000000000
--- a/Documentation/devicetree/bindings/cache/baikal,bt1-l2-ctl.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/cache/baikal,bt1-l2-ctl.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 L2-cache Control Block
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description: |
-  By means of the System Controller Baikal-T1 SoC exposes a few settings to
-  tune the MIPS P5600 CM2 L2 cache performance up. In particular it's possible
-  to change the Tag, Data and Way-select RAM access latencies. Baikal-T1
-  L2-cache controller block is responsible for the tuning. Its DT node is
-  supposed to be a child of the system controller.
-
-properties:
-  compatible:
-    const: baikal,bt1-l2-ctl
-
-  reg:
-    maxItems: 1
-
-  baikal,l2-ws-latency:
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description: Cycles of latency for Way-select RAM accesses
-    default: 0
-    minimum: 0
-    maximum: 3
-
-  baikal,l2-tag-latency:
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description: Cycles of latency for Tag RAM accesses
-    default: 0
-    minimum: 0
-    maximum: 3
-
-  baikal,l2-data-latency:
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description: Cycles of latency for Data RAM accesses
-    default: 1
-    minimum: 0
-    maximum: 3
-
-additionalProperties: false
-
-required:
-  - compatible
-
-examples:
-  - |
-    l2@1f04d028 {
-      compatible = "baikal,bt1-l2-ctl";
-      reg = <0x1f04d028 0x004>;
-
-      baikal,l2-ws-latency = <1>;
-      baikal,l2-tag-latency = <1>;
-      baikal,l2-data-latency = <2>;
-    };
-...
diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
index 6671e461e34ab..995d578157810 100644
--- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
+++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
@@ -33,6 +33,7 @@ properties:
       - qcom,sc7280-llcc
       - qcom,sc8180x-llcc
       - qcom,sc8280xp-llcc
+      - qcom,sdm670-llcc
       - qcom,sdm845-llcc
       - qcom,sm6350-llcc
       - qcom,sm7150-llcc
@@ -204,6 +205,7 @@ allOf:
           contains:
             enum:
               - qcom,sc7280-llcc
+              - qcom,sdm670-llcc
     then:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/clock/axis,artpec9-clock.yaml b/Documentation/devicetree/bindings/clock/axis,artpec9-clock.yaml
new file mode 100644
index 0000000000000..63442b91e7ace
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/axis,artpec9-clock.yaml
@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/axis,artpec9-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axis ARTPEC-9 SoC clock controller
+
+maintainers:
+  - Jesper Nilsson <jesper.nilsson@axis.com>
+
+description: |
+  ARTPEC-9 clock controller is comprised of several CMU (Clock Management Unit)
+  units, generating clocks for different domains. Those CMU units are modeled
+  as separate device tree nodes, and might depend on each other.
+  The root clock in that root tree is an external clock: OSCCLK (25 MHz).
+  This external clock must be defined as a fixed-rate clock in dts.
+
+  CMU_CMU is a top-level CMU, where all base clocks are prepared using PLLs and
+  dividers, all other clocks of function blocks (other CMUs) are usually
+  derived from CMU_CMU.
+
+  Each clock is assigned an identifier and client nodes can use this identifier
+  to specify the clock which they consume. All clocks available for usage
+  in clock consumer nodes are defined as preprocessor macros in
+  'include/dt-bindings/clock/axis,artpec9-clk.h' header.
+
+properties:
+  compatible:
+    enum:
+      - axis,artpec9-cmu-cmu
+      - axis,artpec9-cmu-bus
+      - axis,artpec9-cmu-core
+      - axis,artpec9-cmu-cpucl
+      - axis,artpec9-cmu-fsys0
+      - axis,artpec9-cmu-fsys1
+      - axis,artpec9-cmu-imem
+      - axis,artpec9-cmu-peri
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 5
+
+  clock-names:
+    minItems: 1
+    maxItems: 5
+
+  "#clock-cells":
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - "#clock-cells"
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-cmu
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+
+        clock-names:
+          items:
+            - const: fin_pll
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-bus
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_BUS bus clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: bus
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-core
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_CORE main clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: main
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-cpucl
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_CPUCL switch clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: switch
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-fsys0
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_FSYS0 bus clock (from CMU_CMU)
+            - description: CMU_FSYS0 IP clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: bus
+            - const: ip
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-fsys1
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_FSYS1 scan0 clock (from CMU_CMU)
+            - description: CMU_FSYS1 scan1 clock (from CMU_CMU)
+            - description: CMU_FSYS1 bus clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: scan0
+            - const: scan1
+            - const: bus
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-imem
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_IMEM ACLK clock (from CMU_CMU)
+            - description: CMU_IMEM CA5 clock (from CMU_CMU)
+            - description: CMU_IMEM JPEG clock (from CMU_CMU)
+            - description: CMU_IMEM SSS clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: aclk
+            - const: ca5
+            - const: jpeg
+            - const: sss
+
+  - if:
+      properties:
+        compatible:
+          const: axis,artpec9-cmu-peri
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (25 MHz)
+            - description: CMU_PERI IP clock (from CMU_CMU)
+            - description: CMU_PERI DISP clock (from CMU_CMU)
+
+        clock-names:
+          items:
+            - const: fin_pll
+            - const: ip
+            - const: disp
+
+additionalProperties: false
+
+examples:
+  # Clock controller node for CMU_FSYS1
+  - |
+    #include <dt-bindings/clock/axis,artpec9-clk.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        cmu_fsys1: clock-controller@14c10000 {
+            compatible = "axis,artpec9-cmu-fsys1";
+            reg = <0x0 0x14c10000 0x0 0x4000>;
+            #clock-cells = <1>;
+            clocks = <&fin_pll>,
+                     <&cmu_cmu CLK_DOUT_CMU_FSYS1_SCAN0>,
+                     <&cmu_cmu CLK_DOUT_CMU_FSYS1_SCAN1>,
+                     <&cmu_cmu CLK_DOUT_CMU_FSYS1_BUS>;
+            clock-names = "fin_pll", "scan0", "scan1", "bus";
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/imx6q-clock.yaml b/Documentation/devicetree/bindings/clock/imx6q-clock.yaml
index cd3c04c883df4..0e6febe1c8750 100644
--- a/Documentation/devicetree/bindings/clock/imx6q-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/imx6q-clock.yaml
@@ -29,20 +29,24 @@ properties:
     const: 1
 
   clocks:
+    minItems: 5
     items:
       - description: 24m osc
       - description: 32k osc
       - description: ckih1 clock input
       - description: anaclk1 clock input
       - description: anaclk2 clock input
+      - description: clock input from enet ref pad
 
   clock-names:
+    minItems: 5
     items:
       - const: osc
       - const: ckil
       - const: ckih1
       - const: anaclk1
       - const: anaclk2
+      - const: enet_ref_pad
 
   fsl,pmic-stby-poweroff:
     $ref: /schemas/types.yaml#/definitions/flag
diff --git a/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml b/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml
index d57e18a210cc1..035002721a3b3 100644
--- a/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml
@@ -29,18 +29,22 @@ properties:
     const: 1
 
   clocks:
+    minItems: 4
     items:
       - description: 32k osc
       - description: 24m osc
       - description: ipp_di0 clock input
       - description: ipp_di1 clock input
+      - description: clock input from enet1 ref pad
 
   clock-names:
+    minItems: 4
     items:
       - const: ckil
       - const: osc
       - const: ipp_di0
       - const: ipp_di1
+      - const: enet1_ref_pad
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/clock/qcom,glymur-dispcc.yaml b/Documentation/devicetree/bindings/clock/qcom,glymur-dispcc.yaml
index 45f027c70e03f..9de4ba71f1d96 100644
--- a/Documentation/devicetree/bindings/clock/qcom,glymur-dispcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,glymur-dispcc.yaml
@@ -4,14 +4,14 @@
 $id: http://devicetree.org/schemas/clock/qcom,glymur-dispcc.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Qualcomm Display Clock & Reset Controller on GLYMUR
+title: Qualcomm Display Clock & Reset Controller on Glymur SoC
 
 maintainers:
   - Taniya Das <taniya.das@oss.qualcomm.com>
 
 description: |
   Qualcomm display clock control module which supports the clocks, resets and
-  power domains for the MDSS instances on GLYMUR SoC.
+  power domains for the MDSS instances on Glymur SoC.
 
   See also:
     include/dt-bindings/clock/qcom,dispcc-glymur.h
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq5210-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq5210-gcc.yaml
new file mode 100644
index 0000000000000..f1cc3fc190855
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq5210-gcc.yaml
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,ipq5210-gcc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Global Clock & Reset Controller on IPQ5210
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+  - Kathiravan Thirumoorthy <kathiravan.thirumoorthy@oss.qualcomm.com>
+
+description: |
+  Qualcomm global clock control module provides the clocks, resets and power
+  domains on IPQ5210
+
+  See also:
+    include/dt-bindings/clock/qcom,ipq5210-gcc.h
+    include/dt-bindings/reset/qcom,ipq5210-gcc.h
+
+properties:
+  compatible:
+    const: qcom,ipq5210-gcc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: Sleep clock source
+      - description: PCIE30 PHY0 pipe clock source
+      - description: PCIE30 PHY1 pipe clock source
+      - description: USB3 PHY pipe clock source
+      - description: NSS common clock source
+
+  '#power-domain-cells': false
+
+  '#interconnect-cells':
+    const: 1
+
+required:
+  - compatible
+  - clocks
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    clock-controller@1800000 {
+      compatible = "qcom,ipq5210-gcc";
+      reg = <0x01800000 0x40000>;
+      clocks = <&xo_board_clk>,
+               <&sleep_clk>,
+               <&pcie30_phy0_pipe_clk>,
+               <&pcie30_phy1_pipe_clk>,
+               <&usb3phy_0_cc_pipe_clk>,
+               <&nss_cmn_clk>;
+      #clock-cells = <1>;
+      #reset-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
index 5490a975f3db7..55bf3f8110171 100644
--- a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
@@ -20,6 +20,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,glymur-gxclkctl
       - qcom,kaanapali-gxclkctl
 
   power-domains:
diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-gcc.yaml
index cf244c155f9a6..60f1c8ca2c135 100644
--- a/Documentation/devicetree/bindings/clock/qcom,milos-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,milos-gcc.yaml
@@ -8,16 +8,21 @@ title: Qualcomm Global Clock & Reset Controller on Milos
 
 maintainers:
   - Luca Weiss <luca.weiss@fairphone.com>
+  - Taniya Das <taniya.das@oss.qualcomm.com>
 
 description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on Milos.
 
-  See also: include/dt-bindings/clock/qcom,milos-gcc.h
+  See also:
+   - include/dt-bindings/clock/qcom,eliza-gcc.h
+   - include/dt-bindings/clock/qcom,milos-gcc.h
 
 properties:
   compatible:
-    const: qcom,milos-gcc
+    enum:
+      - qcom,eliza-gcc
+      - qcom,milos-gcc
 
   clocks:
     items:
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
index 3f5f1336262ee..9690169baa469 100644
--- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
@@ -17,6 +17,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,eliza-rpmh-clk
       - qcom,glymur-rpmh-clk
       - qcom,kaanapali-rpmh-clk
       - qcom,milos-rpmh-clk
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
index 6feaa32569f9a..5993804c91fa5 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
@@ -13,7 +13,8 @@ description: |
   Qualcomm graphics clock control module provides the clocks, resets and power
   domains on Qualcomm SoCs.
 
-  See also::
+  See also:
+    include/dt-bindings/clock/qcom,glymur-gpucc.h
     include/dt-bindings/clock/qcom,kaanapali-gpucc.h
     include/dt-bindings/clock/qcom,milos-gpucc.h
     include/dt-bindings/clock/qcom,sar2130p-gpucc.h
@@ -27,6 +28,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,glymur-gpucc
       - qcom,kaanapali-gpucc
       - qcom,milos-gpucc
       - qcom,sar2130p-gpucc
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
index e6beebd6a36ee..7bbf120d928cc 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
@@ -15,6 +15,7 @@ description: |
   domains on SM8450.
 
   See also:
+    include/dt-bindings/clock/qcom,glymur-videocc.h
     include/dt-bindings/clock/qcom,kaanapali-videocc.h
     include/dt-bindings/clock/qcom,sm8450-videocc.h
     include/dt-bindings/clock/qcom,sm8650-videocc.h
@@ -23,6 +24,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,glymur-videocc
       - qcom,kaanapali-videocc
       - qcom,sm8450-videocc
       - qcom,sm8475-videocc
@@ -63,6 +65,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,glymur-videocc
               - qcom,kaanapali-videocc
               - qcom,sm8450-videocc
               - qcom,sm8550-videocc
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
index 784fef8306812..ae9aef0e54e8b 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
@@ -15,6 +15,7 @@ description: |
   power domains on SM8550
 
   See also:
+  - include/dt-bindings/clock/qcom,eliza-tcsr.h
   - include/dt-bindings/clock/qcom,glymur-tcsr.h
   - include/dt-bindings/clock/qcom,sm8550-tcsr.h
   - include/dt-bindings/clock/qcom,sm8650-tcsr.h
@@ -24,6 +25,7 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,eliza-tcsr
           - qcom,glymur-tcsr
           - qcom,kaanapali-tcsr
           - qcom,milos-tcsr
diff --git a/Documentation/devicetree/bindings/clock/rockchip,rv1126b-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rv1126b-cru.yaml
index 04b0a5c51e4e0..b6d3a04be8f16 100644
--- a/Documentation/devicetree/bindings/clock/rockchip,rv1126b-cru.yaml
+++ b/Documentation/devicetree/bindings/clock/rockchip,rv1126b-cru.yaml
@@ -17,6 +17,7 @@ description:
 properties:
   compatible:
     enum:
+      - rockchip,rv1103b-cru
       - rockchip,rv1126b-cru
 
   reg:
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
index 1318720193b3d..6b1fc61a2ff94 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
@@ -35,6 +35,7 @@ properties:
       - samsung,exynosautov920-cmu-cpucl0
       - samsung,exynosautov920-cmu-cpucl1
       - samsung,exynosautov920-cmu-cpucl2
+      - samsung,exynosautov920-cmu-g3d
       - samsung,exynosautov920-cmu-hsi0
       - samsung,exynosautov920-cmu-hsi1
       - samsung,exynosautov920-cmu-hsi2
@@ -287,6 +288,26 @@ allOf:
             - const: oscclk
             - const: noc
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: samsung,exynosautov920-cmu-g3d
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (38.4 MHz)
+            - description: CMU_G3D SWITCH clock (from CMU_TOP)
+            - description: CMU_G3D NOCP clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: switch
+            - const: nocp
+
 required:
   - compatible
   - "#clock-cells"
diff --git a/Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml b/Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml
new file mode 100644
index 0000000000000..7fa16526efcea
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/tenstorrent,atlantis-prcm-rcpu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Tenstorrent Atlantis PRCM (Power, Reset, Clock Management) Module
+
+maintainers:
+  - Anirudh Srinivasan <asrinivasan@oss.tenstorrent.com>
+
+description:
+  Multifunctional register block found in Tenstorrent Atlantis SoC whose main
+  function is to control clocks and resets. This block is instantiated multiple
+  times in the SoC, each block controls clock and resets for a different
+  subsystem. RCPU prcm serves low speed IO interfaces.
+
+properties:
+  compatible:
+    enum:
+      - tenstorrent,atlantis-prcm-rcpu
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  "#clock-cells":
+    const: 1
+    description:
+      See <dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h> for valid indices.
+
+  "#reset-cells":
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - "#clock-cells"
+  - "#reset-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    clock-controller@a8000000 {
+      compatible = "tenstorrent,atlantis-prcm-rcpu";
+      reg = <0xa8000000 0x10000>;
+      clocks = <&osc_24m>;
+      #clock-cells = <1>;
+      #reset-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/connector/usb-connector.yaml b/Documentation/devicetree/bindings/connector/usb-connector.yaml
index 11e40d225b9f3..a00b239960a32 100644
--- a/Documentation/devicetree/bindings/connector/usb-connector.yaml
+++ b/Documentation/devicetree/bindings/connector/usb-connector.yaml
@@ -300,6 +300,40 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint8-array
     maxItems: 4
 
+  sink-load-step:
+    description: Indicates the preferred load step slew rate in mA/usec for
+      the port (in sink mode). This property is defined in "6.5.13.7" of
+      "USB Power Delivery Specification Revision 3.1 Version 1.8".
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [150, 500]
+    default: 150
+
+  sink-load-characteristics:
+    description: Indicates the port's (in sink mode) preferred load
+      characteristics. Users can leverage SINK_LOAD_CHAR() defined in
+      dt-bindings/usb/pd.h to populate this field. This property is defined in
+      "6.5.13.8" of "USB Power Delivery Specification Revision 3.1 Version 1.8".
+    $ref: /schemas/types.yaml#/definitions/uint16
+
+  sink-compliance:
+    description: Represents the types of sources the sink device has been tested
+      and certified with. This property is defined in "6.5.13.9" of
+      "USB Power Delivery Specification Revision 3.1 Version 1.8"
+      Bit 0 when set indicates it has been tested on LPS compliant source
+      Bit 1 when set indicates it has been tested on PS1 compliant source
+      Bit 2 when set indicates it has been tested on PS2 compliant source
+    $ref: /schemas/types.yaml#/definitions/uint8
+    maximum: 7
+
+  charging-adapter-pdp-milliwatt:
+    description: This corresponds to the Power Delivery Profile rating of the
+      charging adapter shipped or recommended for use with the connector port.
+      This property is a requirement to infer the USB PD property
+      "SPR Sink Operational PDP" given in "6.5.13.14" of
+      "USB Power Delivery Specification Revision 3.1 Version 1.8".
+    minimum: 0
+    maximum: 100000
+
 dependencies:
   sink-vdos-v1: [ sink-vdos ]
   sink-vdos: [ sink-vdos-v1 ]
@@ -330,8 +364,9 @@ $defs:
           "Universal Serial Bus Power Delivery Specification" chapter 6.4.1.3
           Sink Capabilities Message, the order of each entry(PDO) should follow the
           PD spec chapter 6.4.1. Required for power sink and power dual role. User
-          can specify the sink PDO array via PDO_FIXED/BATT/VAR/PPS_APDO() defined
-          in dt-bindings/usb/pd.h.
+          can specify the sink PDO array via
+          PDO_FIXED/BATT/VAR/PPS_APDO/SPR_AVS_SNK_APDO() defined in
+          dt-bindings/usb/pd.h.
         minItems: 1
         maxItems: 7
         $ref: /schemas/types.yaml#/definitions/uint32-array
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
index 22eeaef14f557..98eb36bff1727 100644
--- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
@@ -35,6 +35,7 @@ properties:
       - description: v2 of CPUFREQ HW (EPSS)
         items:
           - enum:
+              - qcom,eliza-cpufreq-epss
               - qcom,milos-cpufreq-epss
               - qcom,qcs8300-cpufreq-epss
               - qcom,qdu1000-cpufreq-epss
diff --git a/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml b/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml
index 3dc6c5f89d327..a34d13e92c590 100644
--- a/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml
+++ b/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml
@@ -18,6 +18,7 @@ properties:
       - items:
           - enum:
               - marvell,armada-3700-crypto
+              - mediatek,mt7981-crypto
               - mediatek,mt7986-crypto
           - const: inside-secure,safexcel-eip97ies
       - const: inside-secure,safexcel-eip197b
@@ -80,7 +81,9 @@ allOf:
         compatible:
           not:
             contains:
-              const: mediatek,mt7986-crypto
+              enum:
+                - mediatek,mt7981-crypto
+                - mediatek,mt7986-crypto
     then:
       properties:
         interrupts:
diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
index 061ff718b23d6..876bf90ed96ef 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
@@ -13,6 +13,7 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,eliza-inline-crypto-engine
           - qcom,kaanapali-inline-crypto-engine
           - qcom,milos-inline-crypto-engine
           - qcom,qcs8300-inline-crypto-engine
@@ -31,6 +32,11 @@ properties:
   clocks:
     maxItems: 1
 
+  operating-points-v2: true
+
+  opp-table:
+    type: object
+
 required:
   - compatible
   - reg
@@ -47,5 +53,26 @@ examples:
                    "qcom,inline-crypto-engine";
       reg = <0x01d88000 0x8000>;
       clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+
+      operating-points-v2 = <&ice_opp_table>;
+
+      ice_opp_table: opp-table {
+        compatible = "operating-points-v2";
+
+        opp-100000000 {
+          opp-hz = /bits/ 64 <100000000>;
+          required-opps = <&rpmhpd_opp_low_svs>;
+        };
+
+        opp-201500000 {
+          opp-hz = /bits/ 64 <201500000>;
+          required-opps = <&rpmhpd_opp_svs_l1>;
+        };
+
+        opp-403000000 {
+          opp-hz = /bits/ 64 <403000000>;
+          required-opps = <&rpmhpd_opp_nom>;
+        };
+      };
     };
 ...
diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml
index a1ed1004651b9..6ad466952c02d 100644
--- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml
@@ -85,6 +85,11 @@ properties:
   aux-bus:
     $ref: /schemas/display/dp-aux-bus.yaml#
 
+  connector:
+    type: object
+    $ref: /schemas/connector/usb-connector.yaml#
+    unevaluatedProperties: false
+
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
 
@@ -117,7 +122,6 @@ properties:
 
     required:
       - port@0
-      - port@1
 
 required:
   - compatible
@@ -127,6 +131,28 @@ required:
   - vdd33-supply
   - ports
 
+allOf:
+  - if:
+      required:
+        - aux-bus
+        - connector
+    then:
+      false
+
+  - if:
+      required:
+        - connector
+    then:
+      properties:
+        ports:
+          properties:
+            port@1: false
+    else:
+      properties:
+        ports:
+          required:
+            - port@1
+
 additionalProperties: false
 
 examples:
@@ -185,3 +211,73 @@ examples:
             };
         };
     };
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        encoder@58 {
+            compatible = "analogix,anx7625";
+            reg = <0x58>;
+            enable-gpios = <&pio 45 GPIO_ACTIVE_HIGH>;
+            reset-gpios = <&pio 73 GPIO_ACTIVE_HIGH>;
+            vdd10-supply = <&pp1000_mipibrdg>;
+            vdd18-supply = <&pp1800_mipibrdg>;
+            vdd33-supply = <&pp3300_mipibrdg>;
+            analogix,audio-enable;
+            analogix,lane0-swing = /bits/ 8 <0x14 0x54 0x64 0x74>;
+            analogix,lane1-swing = /bits/ 8 <0x14 0x54 0x64 0x74>;
+
+            connector {
+                compatible = "usb-c-connector";
+                power-role = "dual";
+                data-role = "dual";
+                vbus-supply = <&vbus_reg>;
+
+                ports {
+                    #address-cells = <1>;
+                    #size-cells = <0>;
+
+                    port@0 {
+                        reg = <0>;
+
+                        endpoint {
+                            remote-endpoint = <&usb_hs>;
+                        };
+                    };
+
+                    port@1 {
+                        reg = <1>;
+
+                        endpoint {
+                            remote-endpoint = <&usb_ss>;
+                        };
+                    };
+
+                    port@2 {
+                        reg = <2>;
+
+                        endpoint {
+                            remote-endpoint = <&usb_sbu>;
+                        };
+                    };
+                };
+            };
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    endpoint {
+                        remote-endpoint = <&mipi_dsi>;
+                        bus-type = <7>;
+                        data-lanes = <0 1 2 3>;
+                    };
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
index 49664101a3537..7f380879fffdf 100644
--- a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
@@ -35,6 +35,15 @@ properties:
       - const: ldb
       - const: lvds
 
+  nxp,enable-termination-resistor:
+    type: boolean
+    description:
+      Indicates that the built-in 100 Ohm termination resistor on the LVDS
+      output is enabled. This property is optional and controlled via the
+      HS_EN bit in the LVDS_CTRL register. Enabling it can improve signal
+      quality and prevent visual artifacts on some boards, but increases
+      power consumption.
+
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
 
@@ -84,6 +93,15 @@ allOf:
       required:
         - reg-names
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,imx6sx-ldb
+    then:
+      properties:
+        nxp,enable-termination-resistor: false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml
new file mode 100644
index 0000000000000..a5ba4db11a7c2
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/bridge/lontium,lt8713sx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Lontium LT8713SX Type-C/DP1.4 to Type-C/DP1.4/HDMI2.0/DP++ bridge-hub
+
+maintainers:
+  - Vishnu Saini <vishnu.saini@oss.qualcomm.com>
+
+description:
+  The Lontium LT8713SX is a Type-C/DP1.4 to Type-C/DP1.4/HDMI2.0 converter
+  that integrates one DP input and up to three configurable output interfaces
+  (DP1.4 / HDMI2.0 / DP++), with SST/MST functionality and audio support.
+
+properties:
+  compatible:
+    enum:
+      - lontium,lt8713sx
+
+  reg:
+    maxItems: 1
+
+  vcc-supply:
+    description: Regulator for 3.3V vcc.
+
+  vdd-supply:
+    description: Regulator for 1.1V vdd.
+
+  reset-gpios:
+    description: GPIO connected to active low RESET pin.
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          DP port for DP input from soc to bridge chip
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          DP port for DP output from bridge
+
+      port@2:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          Additional DP port for DP output from bridge
+
+      port@3:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          Additional DP port for DP output from bridge
+
+    required:
+      - port@0
+
+required:
+  - compatible
+  - reg
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        bridge@4f {
+            compatible = "lontium,lt8713sx";
+            reg = <0x4f>;
+            reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    lt8713sx_dp_in: endpoint {
+                        remote-endpoint = <&mdss_dp0_out>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+                    lt8713sx_dp0_out: endpoint {
+                        remote-endpoint = <&dp0_connector_in>;
+                    };
+                };
+
+                port@2 {
+                    reg = <2>;
+                    lt8713sx_dp1_out: endpoint {
+                        remote-endpoint = <&dp1_connector_in>;
+                    };
+                };
+
+                port@3 {
+                    reg = <3>;
+                    lt8713sx_dp2_out: endpoint {
+                        remote-endpoint = <&dp2_connector_in>;
+                    };
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml
index 655db8cfdc25f..429a06057ae85 100644
--- a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml
@@ -44,21 +44,28 @@ properties:
       port@0:
         $ref: /schemas/graph.yaml#/properties/port
         description:
-          Primary MIPI port-1 for MIPI input
+          DSI Port A input. directly drives the display, or works in
+          combination with Port B for higher resolution displays.
 
       port@1:
         $ref: /schemas/graph.yaml#/properties/port
         description:
-          Additional MIPI port-2 for MIPI input, used in combination
-          with primary MIPI port-1 to drive higher resolution displays
+          DSI Port B input. Can be used alone if DSI is physically
+          connected to Port B, or in combination with Port A for higher
+          resolution displays.
 
       port@2:
         $ref: /schemas/graph.yaml#/properties/port
         description:
           HDMI port for HDMI output
 
+    anyOf:
+      - required:
+          - port@0
+      - required:
+          - port@1
+
     required:
-      - port@0
       - port@2
 
 required:
diff --git a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
index 4f52e35d02537..7586d681bcc6b 100644
--- a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
@@ -33,6 +33,7 @@ properties:
     oneOf:
       - items:
           - enum:
+              - doestek,dtc34lm85am # For the Doestek DTC34LM85AM Flat Panel Display (FPD) Transmitter
               - onnn,fin3385  # OnSemi FIN3385
               - ti,ds90c185   # For the TI DS90C185 FPD-Link Serializer
               - ti,ds90c187   # For the TI DS90C187 FPD-Link Serializer
diff --git a/Documentation/devicetree/bindings/display/bridge/thead,th1520-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/bridge/thead,th1520-dw-hdmi.yaml
new file mode 100644
index 0000000000000..68fff885ce15b
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/bridge/thead,th1520-dw-hdmi.yaml
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/bridge/thead,th1520-dw-hdmi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: T-Head TH1520 DesignWare HDMI TX Encoder
+
+maintainers:
+  - Icenowy Zheng <uwu@icenowy.me>
+
+description:
+  The HDMI transmitter is a Synopsys DesignWare HDMI TX controller
+  paired with a DesignWare HDMI Gen2 TX PHY.
+
+allOf:
+  - $ref: /schemas/display/bridge/synopsys,dw-hdmi.yaml#
+
+properties:
+  compatible:
+    enum:
+      - thead,th1520-dw-hdmi
+
+  reg-io-width:
+    const: 4
+
+  clocks:
+    maxItems: 4
+
+  clock-names:
+    items:
+      - const: iahb
+      - const: isfr
+      - const: cec
+      - const: pix
+
+  resets:
+    items:
+      - description: Main reset
+      - description: Configuration APB reset
+
+  reset-names:
+    items:
+      - const: main
+      - const: apb
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: Input port connected to DC8200 DPU "DP" output
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: HDMI output port
+
+    required:
+      - port@0
+      - port@1
+
+required:
+  - compatible
+  - reg
+  - reg-io-width
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+  - interrupts
+  - ports
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/clock/thead,th1520-clk-ap.h>
+    #include <dt-bindings/reset/thead,th1520-reset.h>
+
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      hdmi@ffef540000 {
+        compatible = "thead,th1520-dw-hdmi";
+        reg = <0xff 0xef540000 0x0 0x40000>;
+        reg-io-width = <4>;
+        interrupts = <111 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk_vo CLK_HDMI_PCLK>,
+                 <&clk_vo CLK_HDMI_SFR>,
+                 <&clk_vo CLK_HDMI_CEC>,
+                 <&clk_vo CLK_HDMI_PIXCLK>;
+        clock-names = "iahb", "isfr", "cec", "pix";
+        resets = <&rst_vo TH1520_RESET_ID_HDMI>,
+                 <&rst_vo TH1520_RESET_ID_HDMI_APB>;
+        reset-names = "main", "apb";
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+          port@0 {
+            reg = <0>;
+
+            hdmi_in: endpoint {
+              remote-endpoint = <&dpu_out_dp1>;
+            };
+          };
+
+          port@1 {
+            reg = <1>;
+
+            hdmi_out_conn: endpoint {
+              remote-endpoint = <&hdmi_conn_in>;
+            };
+          };
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml b/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
index 5e8498c8303dd..3820dd7e11af1 100644
--- a/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
@@ -40,9 +40,12 @@ properties:
             properties:
               data-lanes:
                 description: array of physical DSI data lane indexes.
+                minItems: 1
                 items:
                   - const: 1
                   - const: 2
+                  - const: 3
+                  - const: 4
 
             required:
               - data-lanes
diff --git a/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml b/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml
index 0aa2d3fbadaa0..72cbb9ee5eae3 100644
--- a/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml
@@ -20,11 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  backlight: true
-  port: true
-  power-supply: true
-  reset-gpios: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/advantech,idk-1110wr.yaml b/Documentation/devicetree/bindings/display/panel/advantech,idk-1110wr.yaml
index f6fea9085aabf..76b48836ddf62 100644
--- a/Documentation/devicetree/bindings/display/panel/advantech,idk-1110wr.yaml
+++ b/Documentation/devicetree/bindings/display/panel/advantech,idk-1110wr.yaml
@@ -41,8 +41,6 @@ properties:
   panel-timing: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - data-mapping
@@ -51,6 +49,8 @@ required:
   - panel-timing
   - port
 
+additionalProperties: false
+
 examples:
   - |+
     panel {
diff --git a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml
index 05ca3b2385f83..c9b066e69e2f1 100644
--- a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml
+++ b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml
@@ -56,8 +56,6 @@ properties:
       - port@0
       - port@1
 
-additionalProperties: false
-
 required:
   - compatible
   - width-mm
@@ -65,6 +63,8 @@ required:
   - data-mapping
   - panel-timing
 
+additionalProperties: false
+
 examples:
   - |+
     panel-lvds {
diff --git a/Documentation/devicetree/bindings/display/panel/bananapi,s070wv20-ct16.yaml b/Documentation/devicetree/bindings/display/panel/bananapi,s070wv20-ct16.yaml
index bbf127fb28f7a..46e7cff5b2fad 100644
--- a/Documentation/devicetree/bindings/display/panel/bananapi,s070wv20-ct16.yaml
+++ b/Documentation/devicetree/bindings/display/panel/bananapi,s070wv20-ct16.yaml
@@ -22,10 +22,10 @@ properties:
   enable-gpios: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - power-supply
 
+additionalProperties: false
+
 ...
diff --git a/Documentation/devicetree/bindings/display/panel/dlc,dlc0700yzg-1.yaml b/Documentation/devicetree/bindings/display/panel/dlc,dlc0700yzg-1.yaml
index 287e2feb65336..9a2c532dbc92a 100644
--- a/Documentation/devicetree/bindings/display/panel/dlc,dlc0700yzg-1.yaml
+++ b/Documentation/devicetree/bindings/display/panel/dlc,dlc0700yzg-1.yaml
@@ -22,10 +22,10 @@ properties:
   backlight: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - power-supply
 
+additionalProperties: false
+
 ...
diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml
index 56bcd152f43cb..2c60d0cd704e2 100644
--- a/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml
@@ -33,8 +33,6 @@ properties:
   vsp-supply:
     description: Negative source voltage rail
 
-  port: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
index 5725a587e35c9..182a2b825e1ca 100644
--- a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
+++ b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
@@ -54,8 +54,6 @@ required:
   - vcc-supply
   - iovcc-supply
 
-additionalProperties: false
-
 allOf:
   - $ref: panel-common.yaml#
   - if:
@@ -68,6 +66,8 @@ allOf:
       required:
         - reset-gpios
 
+additionalProperties: false
+
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml
index ef5a2240b684d..cc80d0e90f1a1 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml
@@ -34,10 +34,6 @@ properties:
     maxItems: 1
     description: Display data/command selection (D/CX)
 
-  backlight: true
-  reset-gpios: true
-  rotation: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
index 4bdc33d12306b..c97bfd0f2ebcd 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
@@ -29,9 +29,6 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-  port: true
-
   vcc-supply:
     description: Core voltage supply
 
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml
index 5f41758c96d5c..aeb7cb26c0588 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml
@@ -40,8 +40,6 @@ properties:
   spi-max-frequency:
     const: 10000000
 
-  port: true
-
   vci-supply:
     description: Analog voltage supply (2.5 .. 3.3V)
 
@@ -51,8 +49,6 @@ properties:
   vddi-led-supply:
     description: Voltage supply for the LED driver (1.65 .. 3.3 V)
 
-unevaluatedProperties: false
-
 required:
   - compatible
   - reg
@@ -68,6 +64,8 @@ then:
   required:
     - port
 
+unevaluatedProperties: false
+
 examples:
   - |+
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml b/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml
index c7df9a7f6589e..59cc7edb22bbd 100644
--- a/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml
+++ b/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml
@@ -20,11 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  backlight: true
-  port: true
-  power-supply: true
-  reset-gpios: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml
index 4164e3f7061d8..7c75e01797f6c 100644
--- a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml
+++ b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Lin Huang <hl@rock-chips.com>
 
 allOf:
-  - $ref: panel-common.yaml#
+  - $ref: panel-common-dual.yaml#
 
 properties:
   compatible:
@@ -28,6 +28,9 @@ properties:
   avee-supply:
     description: The regulator that provides negative voltage
 
+  port: true
+  ports: true
+
 required:
   - compatible
   - reg
@@ -52,6 +55,27 @@ examples:
             avee-supply = <&avee>;
             backlight = <&backlight>;
             enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    mipi_in_panel: endpoint {
+                        remote-endpoint = <&mipi_out_panel>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+
+                    mipi1_in_panel: endpoint {
+                        remote-endpoint = <&mipi1_out_panel>;
+                    };
+                };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
index 5802fb3c9ffe2..e39efb44ed42c 100644
--- a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
+++ b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
@@ -23,6 +23,7 @@ properties:
           - melfas,lmfbx101117480
           - radxa,display-10hd-ad001
           - radxa,display-8hd-ad002
+          - taiguanck,xti05101-01a
       - const: jadard,jd9365da-h3
 
   reg:
@@ -35,9 +36,8 @@ properties:
     description: supply regulator for VCCIO, usually 1.8V
 
   reset-gpios: true
-
   backlight: true
-
+  rotation: true
   port: true
 
 required:
diff --git a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
index d86c916f7b554..fe7ad266e1b0b 100644
--- a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
+++ b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
@@ -20,11 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  backlight: true
-  port: true
-  power-supply: true
-  reset-gpios: true
-
   spi-3wire: true
 
 required:
diff --git a/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml b/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml
index 3de17fd8513bf..3c8c65c6a8692 100644
--- a/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml
+++ b/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml
@@ -20,10 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  label: true
-  enable-gpios: true
-  port: true
-
   spi-cpha: true
   spi-cpol: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/lxd,m9189a.yaml b/Documentation/devicetree/bindings/display/panel/lxd,m9189a.yaml
new file mode 100644
index 0000000000000..226974a4077fb
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/lxd,m9189a.yaml
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/lxd,m9189a.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: LXD M9189A DSI Display Panel
+
+maintainers:
+  - Michael Tretter <m.tretter@pengutronix.de>
+
+allOf:
+  - $ref: panel-common.yaml
+
+properties:
+  compatible:
+    const: lxd,m9189a
+
+  reg:
+    maxItems: 1
+
+  standby-gpios:
+    description: GPIO used for the standby pin
+    maxItems: 1
+
+  reset-gpios: true
+  power-supply: true
+  backlight: true
+  port: true
+
+required:
+  - compatible
+  - reg
+  - standby-gpios
+  - reset-gpios
+  - power-supply
+  - backlight
+  - port
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      panel@0 {
+        compatible = "lxd,m9189a";
+        reg = <0>;
+        backlight = <&backlight>;
+        reset-gpios = <&gpio3 25 GPIO_ACTIVE_LOW>;
+        standby-gpios = <&gpio5 22 GPIO_ACTIVE_LOW>;
+        power-supply = <&reg_display_3v3>;
+
+        port {
+          mipi_panel_in: endpoint {
+            remote-endpoint = <&mipi_dsi_out>;
+          };
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/display/panel/mitsubishi,aa104xd12.yaml b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa104xd12.yaml
index 96621b89ae9e6..43e98bb07c38a 100644
--- a/Documentation/devicetree/bindings/display/panel/mitsubishi,aa104xd12.yaml
+++ b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa104xd12.yaml
@@ -47,8 +47,6 @@ properties:
   panel-timing: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - data-mapping
@@ -57,6 +55,8 @@ required:
   - panel-timing
   - port
 
+additionalProperties: false
+
 examples:
   - |+
 
diff --git a/Documentation/devicetree/bindings/display/panel/mitsubishi,aa121td01.yaml b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa121td01.yaml
index 37f01d847aac5..2af993d736199 100644
--- a/Documentation/devicetree/bindings/display/panel/mitsubishi,aa121td01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa121td01.yaml
@@ -44,8 +44,6 @@ properties:
   panel-timing: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - vcc-supply
@@ -55,6 +53,8 @@ required:
   - panel-timing
   - port
 
+additionalProperties: false
+
 examples:
   - |+
     panel {
diff --git a/Documentation/devicetree/bindings/display/panel/motorola,mot-panel.yaml b/Documentation/devicetree/bindings/display/panel/motorola,mot-panel.yaml
new file mode 100644
index 0000000000000..99fa1b3ed426b
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/motorola,mot-panel.yaml
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/motorola,mot-panel.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Atrix 4G and Droid X2 DSI Display Panel
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+description:
+  Atrix 4G and Droid X2 use the same 540x960 DSI video mode panel. Exact
+  panel vendor and model are unknown hence generic compatible based on the
+  board name "Mot" is used.
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - const: motorola,mot-panel
+
+  reg:
+    maxItems: 1
+
+  vdd-supply:
+    description: Regulator for main power supply.
+
+  vddio-supply:
+    description: Regulator for 1.8V IO power supply.
+
+  backlight: true
+  reset-gpios: true
+  port: true
+
+required:
+  - compatible
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "motorola,mot-panel";
+            reg = <0>;
+
+            reset-gpios = <&gpio 35 GPIO_ACTIVE_LOW>;
+
+            vdd-supply = <&vdd_5v0_panel>;
+            vddio-supply = <&vdd_1v8_vio>;
+
+            backlight = <&backlight>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&dsi_out>;
+                };
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml b/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml
index 1cffe4d6d4984..eb9eeba92359c 100644
--- a/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml
+++ b/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml
@@ -24,10 +24,6 @@ properties:
   reg:
     maxItems: 1
 
-  label: true
-  port: true
-  reset-gpios: true
-
   spi-max-frequency:
     maximum: 10000000
 
diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml
index c4bae4f77085b..b9300a1f26465 100644
--- a/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml
+++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml
@@ -37,9 +37,6 @@ properties:
   vddio-supply:
     description: regulator that supplies the I/O voltage
 
-  rotation: true
-  backlight: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml
index 800a2f0a4dad9..5d16d85117254 100644
--- a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml
@@ -47,9 +47,6 @@ properties:
   vddneg-supply:
     description: phandle of the negative boost supply regulator
 
-  port: true
-  backlight: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/orisetech,otm8009a.yaml b/Documentation/devicetree/bindings/display/panel/orisetech,otm8009a.yaml
index 1e4f140f48b87..1f697dab832bc 100644
--- a/Documentation/devicetree/bindings/display/panel/orisetech,otm8009a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/orisetech,otm8009a.yaml
@@ -31,12 +31,12 @@ properties:
   reset-gpios:
     maxItems: 1
 
-additionalProperties: false
-
 required:
   - compatible
   - reg
 
+additionalProperties: false
+
 examples:
   - |
     dsi {
diff --git a/Documentation/devicetree/bindings/display/panel/panel-edp-legacy.yaml b/Documentation/devicetree/bindings/display/panel/panel-edp-legacy.yaml
index b308047c1edf8..afe7dc54ebf44 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-edp-legacy.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-edp-legacy.yaml
@@ -44,6 +44,8 @@ properties:
       - boe,nv133fhm-n62
         # BOE NV140FHM-N49 14.0" FHD a-Si FT panel
       - boe,nv140fhmn49
+        # FriendlyELEC HD702E 800x1280 LCD panel
+      - friendlyarm,hd702e
         # Innolux Corporation 11.6" WXGA (1366x768) TFT LCD panel
       - innolux,n116bca-ea1
         # Innolux Corporation 11.6" WXGA (1366x768) TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/panel-lvds.yaml b/Documentation/devicetree/bindings/display/panel/panel-lvds.yaml
index dbc01e6408958..b31c67babaa86 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-lvds.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-lvds.yaml
@@ -58,6 +58,10 @@ properties:
           - hydis,hv070wx2-1e0
           # Jenson Display BL-JT60050-01A 7" WSVGA (1024x600) color TFT LCD LVDS panel
           - jenson,bl-jt60050-01a
+          # Samsung LTN070NL01 7.0" WSVGA (1024x600) TFT LCD LVDS panel
+          - samsung,ltn070nl01
+          # Samsung LTN101AL03 10.1" WXGA (800x1280) TFT LCD LVDS panel
+          - samsung,ltn101al03
           - tbs,a711-panel
           # Winstar WF70A8SYJHLNGA 7" WSVGA (1024x600) color TFT LCD LVDS panel
           - winstar,wf70a8syjhlnga
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
index 2f90c887b7b89..cc8d795df732c 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
@@ -49,6 +49,8 @@ properties:
       - lg,lh500wx1-sd03
         # Lincoln LCD197 5" 1080x1920 LCD panel
       - lincolntech,lcd197
+        # Novatek NT37700F 1080x2160 AMOLED panel
+      - novatek,nt37700f
         # One Stop Displays OSD101T2587-53TS 10.1" 1920x1200 panel
       - osddisplays,osd101t2587-53ts
         # Panasonic 10" WUXGA TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
index 868edb04989a5..e98ca672ba496 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
@@ -103,6 +103,8 @@ properties:
       - dlc,dlc1010gig
         # Emerging Display Technology Corp. 3.5" QVGA TFT LCD panel
       - edt,et035012dm6
+        # Emerging Display Technology Corp. 5.7" 24-bit VGA TFT LCD panel
+      - edt,et057023udba
         # Emerging Display Technology Corp. 5.7" VGA TFT LCD panel
       - edt,et057090dhu
       - edt,et070080dh6
@@ -144,8 +146,6 @@ properties:
       - foxlink,fl500wvr00-a0t
         # Frida FRD350H54004 3.5" QVGA TFT LCD panel
       - frida,frd350h54004
-        # FriendlyELEC HD702E 800x1280 LCD panel
-      - friendlyarm,hd702e
         # GiantPlus GPG48273QS5 4.3" (480x272) WQVGA TFT LCD panel
       - giantplus,gpg48273qs5
         # GiantPlus GPM940B0 3.0" QVGA TFT LCD panel
@@ -268,6 +268,8 @@ properties:
       - powertip,ph128800t006-zhc01
         # POWERTIP PH800480T013-IDF2 7.0" WVGA TFT LCD panel
       - powertip,ph800480t013-idf02
+        # POWERTIP PH800480T032-ZHC19 7.0" WVGA TFT LCD panel
+      - powertip,ph800480t032-zhc19
         # PrimeView PM070WL4 7.0" 800x480 TFT LCD panel
       - primeview,pm070wl4
         # QiaoDian XianShi Corporation 4"3 TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/pda,91-00156-a0.yaml b/Documentation/devicetree/bindings/display/panel/pda,91-00156-a0.yaml
index ccd3623b49557..871e4c2d9824b 100644
--- a/Documentation/devicetree/bindings/display/panel/pda,91-00156-a0.yaml
+++ b/Documentation/devicetree/bindings/display/panel/pda,91-00156-a0.yaml
@@ -21,11 +21,11 @@ properties:
   backlight: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - power-supply
   - backlight
 
+additionalProperties: false
+
 ...
diff --git a/Documentation/devicetree/bindings/display/panel/raydium,rm68200.yaml b/Documentation/devicetree/bindings/display/panel/raydium,rm68200.yaml
index 46fe1014ebc49..8fb7c013dfb8e 100644
--- a/Documentation/devicetree/bindings/display/panel/raydium,rm68200.yaml
+++ b/Documentation/devicetree/bindings/display/panel/raydium,rm68200.yaml
@@ -33,13 +33,13 @@ properties:
   reset-gpios:
     maxItems: 1
 
-additionalProperties: false
-
 required:
   - compatible
   - power-supply
   - reg
 
+additionalProperties: false
+
 examples:
   - |
     dsi {
diff --git a/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml b/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml
index 7ad223f98253a..616a5f3ec9fc0 100644
--- a/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml
+++ b/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml
@@ -34,8 +34,6 @@ properties:
   vddio-supply:
     description: I/O voltage rail
 
-  port: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
index f1723e9102525..1bbe0da3997c3 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
@@ -43,13 +43,13 @@ properties:
   no-hpd: true
   hpd-gpios: true
 
-additionalProperties: false
-
 required:
   - compatible
   - enable-gpios
   - power-supply
 
+additionalProperties: false
+
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml b/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml
index bc92b16c95b9e..2e64fba472cc9 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml
@@ -20,10 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  display-timings: true
-  port: true
-  reset-gpios: true
-
   vdd3-supply:
     description: core voltage supply
 
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
index 74c2a617c2ff3..828b7d7ba17f8 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
@@ -31,8 +31,6 @@ properties:
       configuration.
     maxItems: 1
 
-  reset-gpios: true
-
   vci-supply:
     description: regulator that supplies the VCI analog voltage
       usually around 3.0 V
@@ -41,8 +39,6 @@ properties:
     description: regulator that supplies the VCCIO voltage usually
       around 1.8 V
 
-  backlight: true
-
   spi-cpha: true
 
   spi-cpol: true
@@ -50,8 +46,6 @@ properties:
   spi-max-frequency:
     maximum: 1200000
 
-  port: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
index 4cecf502a1506..c04d47e59f24e 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
@@ -23,8 +23,6 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-
   vci-supply:
     description: regulator that supplies the VCI analog voltage
       usually around 3.0 V
@@ -33,8 +31,6 @@ properties:
     description: regulator that supplies the VCCIO voltage usually
       around 1.8 V
 
-  backlight: true
-
   spi-cpha: true
 
   spi-cpol: true
@@ -44,8 +40,6 @@ properties:
       maximum 300 ns minimum cycle which gives around 3 MHz max frequency
     maximum: 3000000
 
-  port: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml
index d74904164719d..0d57f97e8a76e 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml
@@ -30,8 +30,6 @@ properties:
       configuration.
     maxItems: 1
 
-  reset-gpios: true
-
   vci-supply:
     description: regulator that supplies the VCI analog voltage
       usually around 3.0 V
@@ -40,8 +38,6 @@ properties:
     description: regulator that supplies the VCCIO voltage usually
       around 1.8 V
 
-  backlight: true
-
   spi-cpha: true
 
   spi-cpol: true
@@ -49,8 +45,6 @@ properties:
   spi-max-frequency:
     maximum: 1200000
 
-  port: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml
index 939da65114bfc..1f753b7069113 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml
@@ -44,6 +44,8 @@ properties:
   vmipi-supply:
     description: VMIPI supply, usually 1.8v.
 
+  port: true
+
 required:
   - compatible
   - reg
@@ -65,6 +67,12 @@ examples:
             power-supply = <&display_3v3_supply>;
             reset-gpios = <&gpf0 4 GPIO_ACTIVE_LOW>;
             backlight = <&backlight>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&mdss_dsi0_out>;
+                };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e3ha8.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e3ha8.yaml
index 05a78429aaead..00ce5a4e1c6be 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6e3ha8.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e3ha8.yaml
@@ -22,10 +22,6 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-
-  port: true
-
   vdd3-supply:
     description: VDD regulator
 
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
index c47e2a1a30e5b..b65f0688bdf0b 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
@@ -21,8 +21,6 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-  port: true
   default-brightness: true
   max-brightness: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml
index e32d9188a3e09..1beb4ba92248b 100644
--- a/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml
@@ -41,8 +41,6 @@ properties:
   panel-timing: true
   port: true
 
-additionalProperties: false
-
 required:
   - compatible
   - port
@@ -51,6 +49,8 @@ required:
   - height-mm
   - panel-timing
 
+additionalProperties: false
+
 examples:
   - |+
     panel {
diff --git a/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml b/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
index 1e434240ea3f6..044b84d8638d6 100644
--- a/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
@@ -49,12 +49,6 @@ properties:
       If not set, the controller is in 3-line SPI mode.
       Disallowed for DSI.
 
-  port: true
-  reset-gpios: true
-  rotation: true
-
-  backlight: true
-
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml b/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml
index c35d4f2ab9a44..e4fa05163d2d9 100644
--- a/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml
@@ -24,12 +24,6 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-  power-supply: true
-  backlight: true
-  port: true
-  rotation: true
-
   spi-cpha: true
   spi-cpol: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml b/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml
index 5a8260224b747..12e5ad5040018 100644
--- a/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml
@@ -20,10 +20,6 @@ properties:
   reg:
     maxItems: 1
 
-  label: true
-  reset-gpios: true
-  port: true
-
 required:
   - compatible
   - port
diff --git a/Documentation/devicetree/bindings/display/panel/startek,kd070fhfid015.yaml b/Documentation/devicetree/bindings/display/panel/startek,kd070fhfid015.yaml
index d817f998cddcc..0819f38a9d2c8 100644
--- a/Documentation/devicetree/bindings/display/panel/startek,kd070fhfid015.yaml
+++ b/Documentation/devicetree/bindings/display/panel/startek,kd070fhfid015.yaml
@@ -31,8 +31,6 @@ properties:
 
   power-supply: true
 
-additionalProperties: false
-
 required:
   - compatible
   - enable-gpios
@@ -42,6 +40,8 @@ required:
   - port
   - power-supply
 
+additionalProperties: false
+
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/display/panel/tpo,td.yaml b/Documentation/devicetree/bindings/display/panel/tpo,td.yaml
index 7edd29df4bbb5..855911588d731 100644
--- a/Documentation/devicetree/bindings/display/panel/tpo,td.yaml
+++ b/Documentation/devicetree/bindings/display/panel/tpo,td.yaml
@@ -25,11 +25,6 @@ properties:
   reg:
     maxItems: 1
 
-  label: true
-  reset-gpios: true
-  backlight: true
-  port: true
-
   spi-cpha: true
   spi-cpol: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml b/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml
index 187840bb76c7a..49ef45c035931 100644
--- a/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml
+++ b/Documentation/devicetree/bindings/display/panel/visionox,r66451.yaml
@@ -25,8 +25,6 @@ properties:
   port: true
   reset-gpios: true
 
-additionalProperties: false
-
 required:
   - compatible
   - reg
@@ -35,6 +33,8 @@ required:
   - reset-gpios
   - port
 
+additionalProperties: false
+
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml b/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml
index f0a82f0ff7909..f61a528c04131 100644
--- a/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml
+++ b/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml
@@ -36,8 +36,6 @@ properties:
   port: true
   reset-gpios: true
 
-additionalProperties: false
-
 required:
   - compatible
   - reg
@@ -46,6 +44,8 @@ required:
   - reset-gpios
   - port
 
+additionalProperties: false
+
 examples:
   - |
     dsi {
diff --git a/Documentation/devicetree/bindings/display/panel/visionox,vtdr6130.yaml b/Documentation/devicetree/bindings/display/panel/visionox,vtdr6130.yaml
index d5a8295106c1c..c99f4146f1bb0 100644
--- a/Documentation/devicetree/bindings/display/panel/visionox,vtdr6130.yaml
+++ b/Documentation/devicetree/bindings/display/panel/visionox,vtdr6130.yaml
@@ -26,8 +26,6 @@ properties:
   port: true
   reset-gpios: true
 
-additionalProperties: false
-
 required:
   - compatible
   - reg
@@ -37,6 +35,8 @@ required:
   - reset-gpios
   - port
 
+additionalProperties: false
+
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-dp.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-dp.yaml
index 6345f0132d43d..2b0d9e23e9432 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-dp.yaml
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-dp.yaml
@@ -27,12 +27,10 @@ description: |
   * Pixel clock up to 594MHz
   * I2S, SPDIF audio interface
 
-allOf:
-  - $ref: /schemas/sound/dai-common.yaml#
-
 properties:
   compatible:
     enum:
+      - rockchip,rk3576-dp
       - rockchip,rk3588-dp
 
   reg:
@@ -42,6 +40,7 @@ properties:
     maxItems: 1
 
   clocks:
+    minItems: 3
     items:
       - description: Peripheral/APB bus clock
       - description: DisplayPort AUX clock
@@ -50,6 +49,7 @@ properties:
       - description: SPDIF interfce clock
 
   clock-names:
+    minItems: 3
     items:
       - const: apb
       - const: aux
@@ -95,6 +95,27 @@ required:
   - ports
   - resets
 
+allOf:
+  - $ref: /schemas/sound/dai-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3588-dp
+    then:
+      properties:
+        clocks:
+          minItems: 5
+        clock-names:
+          minItems: 5
+    else:
+      properties:
+        clocks:
+          maxItems: 3
+        clock-names:
+          maxItems: 3
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/display/tilcdc/panel.txt b/Documentation/devicetree/bindings/display/tilcdc/panel.txt
index 808216310ea27..b973174d704ed 100644
--- a/Documentation/devicetree/bindings/display/tilcdc/panel.txt
+++ b/Documentation/devicetree/bindings/display/tilcdc/panel.txt
@@ -1,4 +1,5 @@
 Device-Tree bindings for tilcdc DRM generic panel output driver
+This binding is deprecated and should not be used.
 
 Required properties:
  - compatible: value should be "ti,tilcdc,panel".
diff --git a/Documentation/devicetree/bindings/display/tilcdc/ti,am33xx-tilcdc.yaml b/Documentation/devicetree/bindings/display/tilcdc/ti,am33xx-tilcdc.yaml
new file mode 100644
index 0000000000000..eb0ebb678fa87
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/tilcdc/ti,am33xx-tilcdc.yaml
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright 2025 Bootlin
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/tilcdc/ti,am33xx-tilcdc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI LCD Controller, found on AM335x, DA850, AM18x and OMAP-L138
+
+maintainers:
+  - Kory Maincent <kory.maincent@bootlin.com>
+
+properties:
+  compatible:
+    enum:
+      - ti,am33xx-tilcdc
+      - ti,da850-tilcdc
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  port:
+    $ref: /schemas/graph.yaml#/properties/port
+
+  ti,hwmods:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      Name of the hwmod associated to the LCDC
+
+  max-bandwidth:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      The maximum pixels per second that the memory interface / lcd
+      controller combination can sustain
+    # maximum: 2048*2048*60
+    maximum: 251658240
+
+  max-width:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      The maximum horizontal pixel width supported by the lcd controller.
+    maximum: 2048
+
+  max-pixelclock:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      The maximum pixel clock that can be supported by the lcd controller
+      in KHz.
+
+  blue-and-red-wiring:
+    enum: [straight, crossed]
+    description:
+      This property deals with the LCDC revision 2 (found on AM335x)
+      color errata [1].
+       - "straight" indicates normal wiring that supports RGB565,
+         BGR888, and XBGR8888 color formats.
+       - "crossed" indicates wiring that has blue and red wires
+         crossed. This setup supports BGR565, RGB888 and XRGB8888
+         formats.
+       - If the property is not present or its value is not recognized
+         the legacy mode is assumed. This configuration supports RGB565,
+         RGB888 and XRGB8888 formats. However, depending on wiring, the red
+         and blue colors are swapped in either 16 or 24-bit color modes.
+
+       [1] There is an errata about AM335x color wiring. For 16-bit color
+       mode the wires work as they should (LCD_DATA[0:4] is for Blue[3:7]),
+       but for 24 bit color modes the wiring of blue and red components is
+       crossed and LCD_DATA[0:4] is for Red[3:7] and LCD_DATA[11:15] is
+       for Blue[3-7]. For more details see section 3.1.1 in AM335x
+       Silicon Errata
+       https://www.ti.com/general/docs/lit/getliterature.tsp?baseLiteratureNumber=sprz360
+
+required:
+  - compatible
+  - interrupts
+  - reg
+  - port
+
+additionalProperties: false
+
+examples:
+  - |
+    display-controller@4830e000 {
+        compatible = "ti,am33xx-tilcdc";
+        reg = <0x4830e000 0x1000>;
+        interrupt-parent = <&intc>;
+        interrupts = <36>;
+        ti,hwmods = "lcdc";
+
+        blue-and-red-wiring = "crossed";
+
+        port {
+            endpoint {
+                remote-endpoint = <&hdmi_0>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt b/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
deleted file mode 100644
index 3b3d0bbfcfff4..0000000000000
--- a/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-Device-Tree bindings for tilcdc DRM driver
-
-Required properties:
- - compatible: value should be one of the following:
-    - "ti,am33xx-tilcdc" for AM335x based boards
-    - "ti,da850-tilcdc" for DA850/AM18x/OMAP-L138 based boards
- - interrupts: the interrupt number
- - reg: base address and size of the LCDC device
-
-Recommended properties:
- - ti,hwmods: Name of the hwmod associated to the LCDC
-
-Optional properties:
- - max-bandwidth: The maximum pixels per second that the memory
-   interface / lcd controller combination can sustain
- - max-width: The maximum horizontal pixel width supported by
-   the lcd controller.
- - max-pixelclock: The maximum pixel clock that can be supported
-   by the lcd controller in KHz.
- - blue-and-red-wiring: Recognized values "straight" or "crossed".
-   This property deals with the LCDC revision 2 (found on AM335x)
-   color errata [1].
-    - "straight" indicates normal wiring that supports RGB565,
-      BGR888, and XBGR8888 color formats.
-    - "crossed" indicates wiring that has blue and red wires
-      crossed. This setup supports BGR565, RGB888 and XRGB8888
-      formats.
-    - If the property is not present or its value is not recognized
-      the legacy mode is assumed. This configuration supports RGB565,
-      RGB888 and XRGB8888 formats. However, depending on wiring, the red
-      and blue colors are swapped in either 16 or 24-bit color modes.
-
-Optional nodes:
-
- - port/ports: to describe a connection to an external encoder. The
-   binding follows Documentation/devicetree/bindings/graph.txt and
-   supports a single port with a single endpoint.
-
- - See also Documentation/devicetree/bindings/display/tilcdc/panel.txt and
-   Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml for connecting
-   tfp410 DVI encoder or lcd panel to lcdc
-
-[1] There is an errata about AM335x color wiring. For 16-bit color mode
-    the wires work as they should (LCD_DATA[0:4] is for Blue[3:7]),
-    but for 24 bit color modes the wiring of blue and red components is
-    crossed and LCD_DATA[0:4] is for Red[3:7] and LCD_DATA[11:15] is
-    for Blue[3-7]. For more details see section 3.1.1 in AM335x
-    Silicon Errata:
-    https://www.ti.com/general/docs/lit/getliterature.tsp?baseLiteratureNumber=sprz360
-
-Example:
-
-	fb: fb@4830e000 {
-		compatible = "ti,am33xx-tilcdc", "ti,da850-tilcdc";
-		reg = <0x4830e000 0x1000>;
-		interrupt-parent = <&intc>;
-		interrupts = <36>;
-		ti,hwmods = "lcdc";
-
-		blue-and-red-wiring = "crossed";
-
-		port {
-			lcdc_0: endpoint {
-				remote-endpoint = <&hdmi_0>;
-			};
-		};
-	};
-
-	tda19988: tda19988 {
-		compatible = "nxp,tda998x";
-		reg = <0x70>;
-
-		pinctrl-names = "default", "off";
-		pinctrl-0 = <&nxp_hdmi_bonelt_pins>;
-		pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>;
-
-		port {
-			hdmi_0: endpoint {
-				remote-endpoint = <&lcdc_0>;
-			};
-		};
-	};
diff --git a/Documentation/devicetree/bindings/display/verisilicon,dc.yaml b/Documentation/devicetree/bindings/display/verisilicon,dc.yaml
new file mode 100644
index 0000000000000..9dc35ab973f20
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/verisilicon,dc.yaml
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/verisilicon,dc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Verisilicon DC-series display controllers
+
+maintainers:
+  - Icenowy Zheng <uwu@icenowy.me>
+
+properties:
+  $nodename:
+    pattern: "^display@[0-9a-f]+$"
+
+  compatible:
+    items:
+      - enum:
+          - thead,th1520-dc8200
+      - const: verisilicon,dc # DC IPs have discoverable ID/revision registers
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: DC Core clock
+      - description: DMA AXI bus clock
+      - description: Configuration AHB bus clock
+      - description: Pixel clock of output 0
+      - description: Pixel clock of output 1
+
+  clock-names:
+    items:
+      - const: core
+      - const: axi
+      - const: ahb
+      - const: pix0
+      - const: pix1
+
+  resets:
+    items:
+      - description: DC Core reset
+      - description: DMA AXI bus reset
+      - description: Configuration AHB bus reset
+
+  reset-names:
+    items:
+      - const: core
+      - const: axi
+      - const: ahb
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: The first output channel , endpoint 0 should be
+          used for DPI format output and endpoint 1 should be used
+          for DP format output.
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: The second output channel if the DC variant
+          supports. Follow the same endpoint addressing rule with
+          the first port.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/clock/thead,th1520-clk-ap.h>
+    #include <dt-bindings/reset/thead,th1520-reset.h>
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      display@ffef600000 {
+        compatible = "thead,th1520-dc8200", "verisilicon,dc";
+        reg = <0xff 0xef600000 0x0 0x100000>;
+        interrupts = <93 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk_vo CLK_DPU_CCLK>,
+                 <&clk_vo CLK_DPU_ACLK>,
+                 <&clk_vo CLK_DPU_HCLK>,
+                 <&clk_vo CLK_DPU_PIXELCLK0>,
+                 <&clk_vo CLK_DPU_PIXELCLK1>;
+        clock-names = "core", "axi", "ahb", "pix0", "pix1";
+        resets = <&rst TH1520_RESET_ID_DPU_CORE>,
+                 <&rst TH1520_RESET_ID_DPU_AXI>,
+                 <&rst TH1520_RESET_ID_DPU_AHB>;
+        reset-names = "core", "axi", "ahb";
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+
+          port@1 {
+            reg = <1>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            dpu_out_dp1: endpoint@1 {
+              reg = <1>;
+              remote-endpoint = <&hdmi_in>;
+            };
+          };
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/dma/loongson,ls2k0300-dma.yaml b/Documentation/devicetree/bindings/dma/loongson,ls2k0300-dma.yaml
new file mode 100644
index 0000000000000..8095214ccaf72
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/loongson,ls2k0300-dma.yaml
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/dma/loongson,ls2k0300-dma.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Loongson-2 Multi-Channel DMA controller
+
+description:
+  The Loongson-2 Multi-Channel DMA controller is used for transferring data
+  between system memory and the peripherals on the APB bus.
+
+maintainers:
+  - Binbin Zhou <zhoubinbin@loongson.cn>
+
+allOf:
+  - $ref: dma-controller.yaml#
+
+properties:
+  compatible:
+    enum:
+      - loongson,ls2k0300-dma
+      - loongson,ls2k3000-dma
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    description:
+      Should contain all of the per-channel DMA interrupts in ascending order
+      with respect to the DMA channel index.
+    minItems: 4
+    maxItems: 8
+
+  clocks:
+    maxItems: 1
+
+  '#dma-cells':
+    const: 2
+    description: |
+      DMA request from clients consists of 2 cells:
+        1. Channel index
+        2. Transfer request factor number, If no transfer factor, use 0.
+           The number is SoC-specific, and this should be specified with
+           relation to the device to use the DMA controller.
+
+  dma-channels:
+    enum: [4, 8]
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - '#dma-cells'
+  - dma-channels
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/clock/loongson,ls2k-clk.h>
+
+    dma-controller@1612c000 {
+        compatible = "loongson,ls2k0300-dma";
+        reg = <0x1612c000 0xff>;
+        interrupt-parent = <&liointc0>;
+        interrupts = <23 IRQ_TYPE_LEVEL_HIGH>,
+                     <24 IRQ_TYPE_LEVEL_HIGH>,
+                     <25 IRQ_TYPE_LEVEL_HIGH>,
+                     <26 IRQ_TYPE_LEVEL_HIGH>,
+                     <27 IRQ_TYPE_LEVEL_HIGH>,
+                     <28 IRQ_TYPE_LEVEL_HIGH>,
+                     <29 IRQ_TYPE_LEVEL_HIGH>,
+                     <30 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk LS2K0300_CLK_APB_GATE>;
+        #dma-cells = <2>;
+        dma-channels = <8>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
index d137b9cbaee91..0155a15e200b8 100644
--- a/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
+++ b/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
@@ -19,6 +19,7 @@ properties:
               - renesas,r9a07g044-dmac # RZ/G2{L,LC}
               - renesas,r9a07g054-dmac # RZ/V2L
               - renesas,r9a08g045-dmac # RZ/G3S
+              - renesas,r9a08g046-dmac # RZ/G3L
           - const: renesas,rz-dmac
 
       - items:
@@ -29,6 +30,13 @@ properties:
 
       - const: renesas,r9a09g057-dmac # RZ/V2H(P)
 
+      - const: renesas,r9a09g077-dmac # RZ/T2H
+
+      - items:
+          - enum:
+              - renesas,r9a09g087-dmac # RZ/N2H
+          - const: renesas,r9a09g077-dmac
+
   reg:
     items:
       - description: Control and channel register block
@@ -36,27 +44,12 @@ properties:
     minItems: 1
 
   interrupts:
+    minItems: 16
     maxItems: 17
 
   interrupt-names:
-    items:
-      - const: error
-      - const: ch0
-      - const: ch1
-      - const: ch2
-      - const: ch3
-      - const: ch4
-      - const: ch5
-      - const: ch6
-      - const: ch7
-      - const: ch8
-      - const: ch9
-      - const: ch10
-      - const: ch11
-      - const: ch12
-      - const: ch13
-      - const: ch14
-      - const: ch15
+    minItems: 16
+    maxItems: 17
 
   clocks:
     items:
@@ -122,6 +115,35 @@ required:
 allOf:
   - $ref: dma-controller.yaml#
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - renesas,rz-dmac
+              - renesas,r9a09g057-dmac
+    then:
+      properties:
+        interrupt-names:
+          items:
+            - const: error
+            - const: ch0
+            - const: ch1
+            - const: ch2
+            - const: ch3
+            - const: ch4
+            - const: ch5
+            - const: ch6
+            - const: ch7
+            - const: ch8
+            - const: ch9
+            - const: ch10
+            - const: ch11
+            - const: ch12
+            - const: ch13
+            - const: ch14
+            - const: ch15
+
   - if:
       properties:
         compatible:
@@ -131,6 +153,7 @@ allOf:
               - renesas,r9a07g044-dmac
               - renesas,r9a07g054-dmac
               - renesas,r9a08g045-dmac
+              - renesas,r9a08g046-dmac
     then:
       properties:
         reg:
@@ -189,6 +212,49 @@ allOf:
         - renesas,icu
         - resets
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g077-dmac
+    then:
+      properties:
+        reg:
+          maxItems: 1
+        clocks:
+          maxItems: 1
+
+        clock-names: false
+        resets: false
+        reset-names: false
+
+        interrupts:
+          maxItems: 16
+
+        interrupt-names:
+          items:
+            - const: ch0
+            - const: ch1
+            - const: ch2
+            - const: ch3
+            - const: ch4
+            - const: ch5
+            - const: ch6
+            - const: ch7
+            - const: ch8
+            - const: ch9
+            - const: ch10
+            - const: ch11
+            - const: ch12
+            - const: ch13
+            - const: ch14
+            - const: ch15
+
+      required:
+        - clocks
+        - power-domains
+        - renesas,icu
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
index 216cda21c538b..804514732dbe6 100644
--- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
+++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
@@ -21,6 +21,7 @@ properties:
       - enum:
           - snps,axi-dma-1.01a
           - intel,kmb-axi-dma
+          - sophgo,cv1800b-axi-dma
           - starfive,jh7110-axi-dma
           - starfive,jh8100-axi-dma
       - items:
@@ -68,6 +69,8 @@ properties:
 
   dma-noncoherent: true
 
+  dma-coherent: true
+
   resets:
     minItems: 1
     maxItems: 2
diff --git a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt b/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
deleted file mode 100644
index b567107270cbe..0000000000000
--- a/Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
+++ /dev/null
@@ -1,111 +0,0 @@
-Xilinx AXI VDMA engine, it does transfers between memory and video devices.
-It can be configured to have one channel or two channels. If configured
-as two channels, one is to transmit to the video device and another is
-to receive from the video device.
-
-Xilinx AXI DMA engine, it does transfers between memory and AXI4 stream
-target devices. It can be configured to have one channel or two channels.
-If configured as two channels, one is to transmit to the device and another
-is to receive from the device.
-
-Xilinx AXI CDMA engine, it does transfers between memory-mapped source
-address and a memory-mapped destination address.
-
-Xilinx AXI MCDMA engine, it does transfer between memory and AXI4 stream
-target devices. It can be configured to have up to 16 independent transmit
-and receive channels.
-
-Required properties:
-- compatible: Should be one of-
-		"xlnx,axi-vdma-1.00.a"
-		"xlnx,axi-dma-1.00.a"
-		"xlnx,axi-cdma-1.00.a"
-		"xlnx,axi-mcdma-1.00.a"
-- #dma-cells: Should be <1>, see "dmas" property below
-- reg: Should contain VDMA registers location and length.
-- xlnx,addrwidth: Should be the vdma addressing size in bits(ex: 32 bits).
-- dma-ranges: Should be as the following <dma_addr cpu_addr max_len>.
-- dma-channel child node: Should have at least one channel and can have up to
-	two channels per device. This node specifies the properties of each
-	DMA channel (see child node properties below).
-- clocks: Input clock specifier. Refer to common clock bindings.
-- clock-names: List of input clocks
-	For VDMA:
-	Required elements: "s_axi_lite_aclk"
-	Optional elements: "m_axi_mm2s_aclk" "m_axi_s2mm_aclk",
-			   "m_axis_mm2s_aclk", "s_axis_s2mm_aclk"
-	For CDMA:
-	Required elements: "s_axi_lite_aclk", "m_axi_aclk"
-	For AXIDMA and MCDMA:
-	Required elements: "s_axi_lite_aclk"
-	Optional elements: "m_axi_mm2s_aclk", "m_axi_s2mm_aclk",
-			   "m_axi_sg_aclk"
-
-Required properties for VDMA:
-- xlnx,num-fstores: Should be the number of framebuffers as configured in h/w.
-
-Optional properties for AXI DMA and MCDMA:
-- xlnx,sg-length-width: Should be set to the width in bits of the length
-	register as configured in h/w. Takes values {8...26}. If the property
-	is missing or invalid then the default value 23 is used. This is the
-	maximum value that is supported by all IP versions.
-
-Optional properties for AXI DMA:
-- xlnx,axistream-connected: Tells whether DMA is connected to AXI stream IP.
-- xlnx,irq-delay: Tells the interrupt delay timeout value. Valid range is from
-	0-255. Setting this value to zero disables the delay timer interrupt.
-	1 timeout interval = 125 * clock period of SG clock.
-Optional properties for VDMA:
-- xlnx,flush-fsync: Tells which channel to Flush on Frame sync.
-	It takes following values:
-	{1}, flush both channels
-	{2}, flush mm2s channel
-	{3}, flush s2mm channel
-
-Required child node properties:
-- compatible:
-	For VDMA: It should be either "xlnx,axi-vdma-mm2s-channel" or
-	"xlnx,axi-vdma-s2mm-channel".
-	For CDMA: It should be "xlnx,axi-cdma-channel".
-	For AXIDMA and MCDMA: It should be either "xlnx,axi-dma-mm2s-channel"
-	or "xlnx,axi-dma-s2mm-channel".
-- interrupts: Should contain per channel VDMA interrupts.
-- xlnx,datawidth: Should contain the stream data width, take values
-	{32,64...1024}.
-
-Optional child node properties:
-- xlnx,include-dre: Tells hardware is configured for Data
-	Realignment Engine.
-Optional child node properties for VDMA:
-- xlnx,genlock-mode: Tells Genlock synchronization is
-	enabled/disabled in hardware.
-- xlnx,enable-vert-flip: Tells vertical flip is
-	enabled/disabled in hardware(S2MM path).
-Optional child node properties for MCDMA:
-- dma-channels: Number of dma channels in child node.
-
-Example:
-++++++++
-
-axi_vdma_0: axivdma@40030000 {
-	compatible = "xlnx,axi-vdma-1.00.a";
-	#dma_cells = <1>;
-	reg = < 0x40030000 0x10000 >;
-	dma-ranges = <0x00000000 0x00000000 0x40000000>;
-	xlnx,num-fstores = <0x8>;
-	xlnx,flush-fsync = <0x1>;
-	xlnx,addrwidth = <0x20>;
-	clocks = <&clk 0>, <&clk 1>, <&clk 2>, <&clk 3>, <&clk 4>;
-	clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk",
-		      "m_axis_mm2s_aclk", "s_axis_s2mm_aclk";
-	dma-channel@40030000 {
-		compatible = "xlnx,axi-vdma-mm2s-channel";
-		interrupts = < 0 54 4 >;
-		xlnx,datawidth = <0x40>;
-	} ;
-	dma-channel@40030030 {
-		compatible = "xlnx,axi-vdma-s2mm-channel";
-		interrupts = < 0 53 4 >;
-		xlnx,datawidth = <0x40>;
-	} ;
-} ;
diff --git a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
new file mode 100644
index 0000000000000..340ae9e91cb01
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/dma/xilinx/xlnx,axi-dma.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Xilinx AXI VDMA, DMA, CDMA and MCDMA IP
+
+maintainers:
+  - Radhey Shyam Pandey <radhey.shyam.pandey@amd.com>
+  - Abin Joseph <abin.joseph@amd.com>
+
+description: >
+  Xilinx AXI VDMA engine, it does transfers between memory and video devices.
+  It can be configured to have one channel or two channels. If configured
+  as two channels, one is to transmit to the video device and another is
+  to receive from the video device.
+
+  Xilinx AXI DMA engine, it does transfers between memory and AXI4 stream
+  target devices. It can be configured to have one channel or two channels.
+  If configured as two channels, one is to transmit to the device and another
+  is to receive from the device.
+
+  Xilinx AXI CDMA engine, it does transfers between memory-mapped source
+  address and a memory-mapped destination address.
+
+  Xilinx AXI MCDMA engine, it does transfer between memory and AXI4 stream
+  target devices. It can be configured to have up to 16 independent transmit
+  and receive channels.
+
+properties:
+  compatible:
+    enum:
+      - xlnx,axi-cdma-1.00.a
+      - xlnx,axi-dma-1.00.a
+      - xlnx,axi-mcdma-1.00.a
+      - xlnx,axi-vdma-1.00.a
+
+  reg:
+    maxItems: 1
+
+  "#dma-cells":
+    const: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  interrupts:
+    items:
+      - description: Interrupt for single channel (MM2S or S2MM)
+      - description: Interrupt for dual channel configuration
+    minItems: 1
+    description:
+      Interrupt lines for the DMA controller. Only used when
+      xlnx,axistream-connected is present (DMA connected to AXI Stream
+      IP). When child dma-channel nodes are present, interrupts are
+      specified in the child nodes instead.
+
+  clocks:
+    minItems: 1
+    maxItems: 5
+
+  clock-names:
+    minItems: 1
+    maxItems: 5
+
+  dma-ranges: true
+
+  xlnx,addrwidth:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [32, 64]
+    description: The DMA addressing size in bits.
+
+  xlnx,num-fstores:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 32
+    description: Should be the number of framebuffers as configured in h/w.
+
+  xlnx,flush-fsync:
+    type: boolean
+    description: Tells which channel to Flush on Frame sync.
+
+  xlnx,sg-length-width:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 8
+    maximum: 26
+    default: 23
+    description:
+      Width in bits of the length register as configured in hardware.
+
+  xlnx,irq-delay:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 0
+    maximum: 255
+    description:
+      Tells the interrupt delay timeout value. Valid range is from 0-255.
+      Setting this value to zero disables the delay timer interrupt.
+      1 timeout interval = 125 * clock period of SG clock.
+
+  xlnx,axistream-connected:
+    type: boolean
+    description: Tells whether DMA is connected to AXI stream IP.
+
+patternProperties:
+  "^dma-channel(-mm2s|-s2mm)?$":
+    type: object
+    description:
+      Should have at least one channel and can have up to two channels per
+      device. This node specifies the properties of each DMA channel.
+
+    properties:
+      compatible:
+        enum:
+          - xlnx,axi-vdma-mm2s-channel
+          - xlnx,axi-vdma-s2mm-channel
+          - xlnx,axi-cdma-channel
+          - xlnx,axi-dma-mm2s-channel
+          - xlnx,axi-dma-s2mm-channel
+
+      interrupts:
+        maxItems: 1
+
+      xlnx,datawidth:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        enum: [32, 64, 128, 256, 512, 1024]
+        description: Should contain the stream data width, take values {32,64...1024}.
+
+      xlnx,include-dre:
+        type: boolean
+        description: Tells hardware is configured for Data Realignment Engine.
+
+      xlnx,genlock-mode:
+        type: boolean
+        description: Tells Genlock synchronization is enabled/disabled in hardware.
+
+      xlnx,enable-vert-flip:
+        type: boolean
+        description:
+          Tells vertical flip is enabled/disabled in hardware(S2MM path).
+
+      dma-channels:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: Number of dma channels in child node.
+
+    required:
+      - compatible
+      - interrupts
+      - xlnx,datawidth
+
+    additionalProperties: false
+
+allOf:
+  - $ref: ../dma-controller.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: xlnx,axi-vdma-1.00.a
+    then:
+      properties:
+        clock-names:
+          items:
+            - const: s_axi_lite_aclk
+            - const: m_axi_mm2s_aclk
+            - const: m_axi_s2mm_aclk
+            - const: m_axis_mm2s_aclk
+            - const: s_axis_s2mm_aclk
+          minItems: 1
+        interrupts: false
+      patternProperties:
+        "^dma-channel(-mm2s|-s2mm)?$":
+          properties:
+            compatible:
+              enum:
+                - xlnx,axi-vdma-mm2s-channel
+                - xlnx,axi-vdma-s2mm-channel
+      required:
+        - xlnx,num-fstores
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: xlnx,axi-cdma-1.00.a
+    then:
+      properties:
+        clock-names:
+          items:
+            - const: s_axi_lite_aclk
+            - const: m_axi_aclk
+        interrupts: false
+      patternProperties:
+        "^dma-channel(-mm2s|-s2mm)?$":
+          properties:
+            compatible:
+              enum:
+                - xlnx,axi-cdma-channel
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - xlnx,axi-dma-1.00.a
+              - xlnx,axi-mcdma-1.00.a
+    then:
+      properties:
+        clock-names:
+          items:
+            - const: s_axi_lite_aclk
+            - const: m_axi_mm2s_aclk
+            - const: m_axi_s2mm_aclk
+            - const: m_axi_sg_aclk
+          minItems: 1
+      patternProperties:
+        "^dma-channel(-mm2s|-s2mm)?(@[0-9a-f]+)?$":
+          properties:
+            compatible:
+              enum:
+                - xlnx,axi-dma-mm2s-channel
+                - xlnx,axi-dma-s2mm-channel
+
+required:
+  - "#dma-cells"
+  - reg
+  - xlnx,addrwidth
+  - dma-ranges
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    dma-controller@40030000 {
+        compatible = "xlnx,axi-vdma-1.00.a";
+        reg = <0x40030000 0x10000>;
+        #dma-cells = <1>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        dma-ranges = <0x0 0x0 0x40000000>;
+        clocks = <&clk 0>, <&clk 1>, <&clk 2>, <&clk 3>, <&clk 4>;
+        clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk",
+                      "m_axi_s2mm_aclk", "m_axis_mm2s_aclk",
+                      "s_axis_s2mm_aclk";
+        xlnx,num-fstores = <8>;
+        xlnx,flush-fsync;
+        xlnx,addrwidth = <32>;
+
+        dma-channel-mm2s {
+            compatible = "xlnx,axi-vdma-mm2s-channel";
+            interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+            xlnx,datawidth = <64>;
+        };
+
+        dma-channel-s2mm {
+            compatible = "xlnx,axi-vdma-s2mm-channel";
+            interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
+            xlnx,datawidth = <64>;
+        };
+    };
+
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    dma-controller@a4030000 {
+        compatible = "xlnx,axi-dma-1.00.a";
+        reg = <0xa4030000 0x10000>;
+        #dma-cells = <1>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        dma-ranges = <0x0 0x0 0x40000000>;
+        clocks = <&clk 0>, <&clk 1>, <&clk 2>, <&clk 3>;
+        clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk",
+                      "m_axi_s2mm_aclk", "m_axi_sg_aclk";
+        xlnx,addrwidth = <32>;
+        xlnx,sg-length-width = <14>;
+
+        dma-channel-mm2s {
+            compatible = "xlnx,axi-dma-mm2s-channel";
+            interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+            xlnx,datawidth = <64>;
+            xlnx,include-dre;
+        };
+
+        dma-channel-s2mm {
+            compatible = "xlnx,axi-dma-s2mm-channel";
+            interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+            xlnx,datawidth = <64>;
+            xlnx,include-dre;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/embedded-controller/kontron,sl28cpld.yaml b/Documentation/devicetree/bindings/embedded-controller/kontron,sl28cpld.yaml
index a77e67f6cb82f..0b752f3baaa94 100644
--- a/Documentation/devicetree/bindings/embedded-controller/kontron,sl28cpld.yaml
+++ b/Documentation/devicetree/bindings/embedded-controller/kontron,sl28cpld.yaml
@@ -16,12 +16,7 @@ description: |
 
 properties:
   compatible:
-    oneOf:
-      - items:
-          - enum:
-              - kontron,sa67mcu
-          - const: kontron,sl28cpld
-      - const: kontron,sl28cpld
+    const: kontron,sl28cpld
 
   reg:
     description:
diff --git a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
index 072b3c0c5fd03..79f88b5f4e5cc 100644
--- a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
+++ b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
@@ -42,6 +42,9 @@ properties:
     description:
       A port node to link the usb controller for the dual role switch.
 
+  connector:
+    $ref: /schemas/connector/usb-connector.yaml#
+
 required:
   - compatible
   - interrupts
diff --git a/Documentation/devicetree/bindings/firmware/arm,scmi.yaml b/Documentation/devicetree/bindings/firmware/arm,scmi.yaml
index be817fd9cc34b..d06cca9273c48 100644
--- a/Documentation/devicetree/bindings/firmware/arm,scmi.yaml
+++ b/Documentation/devicetree/bindings/firmware/arm,scmi.yaml
@@ -146,6 +146,13 @@ properties:
       this platform. If set, the value should be non-zero.
     minimum: 1
 
+  arm,no-completion-irq:
+    type: boolean
+    description:
+      This optional property is intended for hardware that does not generate
+      completion interrupts and can be used to unconditionally enable forced
+      polling mode of operation.
+
   arm,smc-id:
     $ref: /schemas/types.yaml#/definitions/uint32
     description:
@@ -379,6 +386,9 @@ then:
     - shmem
 
 else:
+  properties:
+    arm,no-completion-irq: false
+
   if:
     properties:
       compatible:
diff --git a/Documentation/devicetree/bindings/firmware/google,gs101-acpm-ipc.yaml b/Documentation/devicetree/bindings/firmware/google,gs101-acpm-ipc.yaml
index 4a1e3e3c0505a..e68f9c3ca5e26 100644
--- a/Documentation/devicetree/bindings/firmware/google,gs101-acpm-ipc.yaml
+++ b/Documentation/devicetree/bindings/firmware/google,gs101-acpm-ipc.yaml
@@ -37,6 +37,7 @@ properties:
     maxItems: 1
 
   pmic:
+    deprecated: true
     description: Child node describing the main PMIC.
     type: object
     additionalProperties: true
@@ -45,6 +46,24 @@ properties:
       compatible:
         const: samsung,s2mpg10-pmic
 
+  pmic-1:
+    description: Child node describing the main PMIC.
+    type: object
+    additionalProperties: true
+
+    properties:
+      compatible:
+        const: samsung,s2mpg10-pmic
+
+  pmic-2:
+    description: Child node describing the sub PMIC.
+    type: object
+    additionalProperties: true
+
+    properties:
+      compatible:
+        const: samsung,s2mpg11-pmic
+
   shmem:
     description:
       List of phandle pointing to the shared memory (SHM) area. The memory
@@ -62,7 +81,9 @@ additionalProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/regulator/samsung,s2mpg10-regulator.h>
 
     power-management {
         compatible = "google,gs101-acpm-ipc";
@@ -70,10 +91,12 @@ examples:
         mboxes = <&ap2apm_mailbox>;
         shmem = <&apm_sram>;
 
-        pmic {
+        pmic-1 {
             compatible = "samsung,s2mpg10-pmic";
             interrupts-extended = <&gpa0 6 IRQ_TYPE_LEVEL_LOW>;
 
+            vinl3m-supply = <&buck8m>;
+
             regulators {
                 ldo1m {
                     regulator-name = "vdd_ldo1";
@@ -82,7 +105,13 @@ examples:
                     regulator-always-on;
                 };
 
-                // ...
+                ldo20m {
+                    regulator-name = "vdd_dmics";
+                    regulator-min-microvolt = <700000>;
+                    regulator-max-microvolt = <1300000>;
+                    regulator-always-on;
+                    samsung,ext-control = <S2MPG10_EXTCTRL_LDO20M_EN2>;
+                };
 
                 buck8m {
                     regulator-name = "vdd_mif";
@@ -93,4 +122,21 @@ examples:
                 };
             };
         };
+
+        pmic-2 {
+            compatible = "samsung,s2mpg11-pmic";
+            interrupts-extended = <&gpa0 7 IRQ_TYPE_LEVEL_LOW>;
+
+            vinl1s-supply = <&buck8m>;
+            vinl2s-supply = <&buck6s>;
+
+            regulators {
+                buckd {
+                    regulator-name = "vcc_ufs";
+                    regulator-ramp-delay = <6250>;
+                    enable-gpios = <&gpp0 1 GPIO_ACTIVE_HIGH>;
+                    samsung,ext-control = <S2MPG11_EXTCTRL_UFS_EN>;
+                };
+            };
+        };
     };
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
index d66459f1d84e4..6d9b71a9b9b3f 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
@@ -26,6 +26,7 @@ properties:
           - qcom,scm-glymur
           - qcom,scm-ipq4019
           - qcom,scm-ipq5018
+          - qcom,scm-ipq5210
           - qcom,scm-ipq5332
           - qcom,scm-ipq5424
           - qcom,scm-ipq6018
diff --git a/Documentation/devicetree/bindings/gpio/gpio-delay.yaml b/Documentation/devicetree/bindings/gpio/gpio-delay.yaml
index 1cebc4058e272..b99ceff6c5f69 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-delay.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-delay.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: GPIO delay controller
 
 maintainers:
-  - Alexander Stein <linux@ew.tq-group.com>
+  - Alexander Stein <alexander.stein@ew.tq-group.com>
 
 description: |
   This binding describes an electrical setup where setting an GPIO output
diff --git a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml
index 184432d24ea18..eaa254a468063 100644
--- a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml
@@ -33,6 +33,9 @@ properties:
   clocks:
     maxItems: 1
 
+  resets:
+    maxItems: 1
+
   "#gpio-cells":
     const: 2
 
diff --git a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
index 728099c658246..b18f8f0ca0ae4 100644
--- a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
@@ -30,6 +30,7 @@ properties:
           - realtek,rtl8390-gpio
           - realtek,rtl9300-gpio
           - realtek,rtl9310-gpio
+          - realtek,rtl9607-gpio
       - const: realtek,otto-gpio
 
   reg: true
diff --git a/Documentation/devicetree/bindings/hwmon/baikal,bt1-pvt.yaml b/Documentation/devicetree/bindings/hwmon/baikal,bt1-pvt.yaml
deleted file mode 100644
index 5d3ce641fcdeb..0000000000000
--- a/Documentation/devicetree/bindings/hwmon/baikal,bt1-pvt.yaml
+++ /dev/null
@@ -1,105 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/hwmon/baikal,bt1-pvt.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 PVT Sensor
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description: |
-  Baikal-T1 SoC provides an embedded process, voltage and temperature
-  sensor to monitor an internal SoC environment (chip temperature, supply
-  voltage and process monitor) and on time detect critical situations,
-  which may cause the system instability and even damages. The IP-block
-  is based on the Analog Bits PVT sensor, but is equipped with a dedicated
-  control wrapper, which provides a MMIO registers-based access to the
-  sensor core functionality (APB3-bus based) and exposes an additional
-  functions like thresholds/data ready interrupts, its status and masks,
-  measurements timeout. Its internal structure is depicted on the next
-  diagram:
-
-     Analog Bits core                     Bakal-T1 PVT control block
-  +--------------------+                  +------------------------+
-  | Temperature sensor |-+         +------| Sensors control        |
-  |--------------------| |<---En---|      |------------------------|
-  | Voltage sensor     |-|<--Mode--| +--->| Sampled data           |
-  |--------------------| |<--Trim--+ |    |------------------------|
-  | Low-Vt sensor      |-|           | +--| Thresholds comparator  |
-  |--------------------| |---Data----| |  |------------------------|
-  | High-Vt sensor     |-|           | +->| Interrupts status      |
-  |--------------------| |--Valid--+-+ |  |------------------------|
-  | Standard-Vt sensor |-+         +---+--| Interrupts mask        |
-  +--------------------+                  |------------------------|
-           ^                              | Interrupts timeout     |
-           |                              +------------------------+
-           |                                        ^  ^
-  Rclk-----+----------------------------------------+  |
-  APB3-------------------------------------------------+
-
-  This bindings describes the external Baikal-T1 PVT control interfaces
-  like MMIO registers space, interrupt request number and clocks source.
-  These are then used by the corresponding hwmon device driver to
-  implement the sysfs files-based access to the sensors functionality.
-
-properties:
-  compatible:
-    const: baikal,bt1-pvt
-
-  reg:
-    maxItems: 1
-
-  interrupts:
-    maxItems: 1
-
-  clocks:
-    items:
-      - description: PVT reference clock
-      - description: APB3 interface clock
-
-  clock-names:
-    items:
-      - const: ref
-      - const: pclk
-
-  "#thermal-sensor-cells":
-    description: Baikal-T1 can be referenced as the CPU thermal-sensor
-    const: 0
-
-  baikal,pvt-temp-offset-millicelsius:
-    description: |
-      Temperature sensor trimming factor. It can be used to manually adjust the
-      temperature measurements within 7.130 degrees Celsius.
-    default: 0
-    minimum: 0
-    maximum: 7130
-
-additionalProperties: false
-
-required:
-  - compatible
-  - reg
-  - interrupts
-  - clocks
-  - clock-names
-
-examples:
-  - |
-    #include <dt-bindings/interrupt-controller/mips-gic.h>
-
-    pvt@1f200000 {
-      compatible = "baikal,bt1-pvt";
-      reg = <0x1f200000 0x1000>;
-      #thermal-sensor-cells = <0>;
-
-      interrupts = <GIC_SHARED 31 IRQ_TYPE_LEVEL_HIGH>;
-
-      baikal,pvt-temp-offset-millicelsius = <1000>;
-
-      clocks = <&ccu_sys>, <&ccu_sys>;
-      clock-names = "ref", "pclk";
-    };
-...
diff --git a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
index 56db2292f062d..7d57c2934a8a1 100644
--- a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
+++ b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
@@ -105,7 +105,7 @@ properties:
       G coefficient for temperature equation.
       Default for series 5 = 60000
       Default for series 6 = 57400
-    multipleOf: 100
+    multipleOf: 10
     minimum: 1000
     $ref: /schemas/types.yaml#/definitions/uint32
 
@@ -131,7 +131,7 @@ properties:
       J coefficient for temperature equation.
       Default for series 5 = -100
       Default for series 6 = 0
-    multipleOf: 100
+    multipleOf: 10
     maximum: 0
     $ref: /schemas/types.yaml#/definitions/int32
 
diff --git a/Documentation/devicetree/bindings/hwmon/npcm750-pwm-fan.txt b/Documentation/devicetree/bindings/hwmon/npcm750-pwm-fan.txt
deleted file mode 100644
index 18095ba87a5a9..0000000000000
--- a/Documentation/devicetree/bindings/hwmon/npcm750-pwm-fan.txt
+++ /dev/null
@@ -1,88 +0,0 @@
-Nuvoton NPCM PWM and Fan Tacho controller device
-
-The Nuvoton BMC NPCM7XX supports 8 Pulse-width modulation (PWM)
-controller outputs and 16 Fan tachometer controller inputs.
-
-The Nuvoton BMC NPCM8XX supports 12 Pulse-width modulation (PWM)
-controller outputs and 16 Fan tachometer controller inputs.
-
-Required properties for pwm-fan node
-- #address-cells : should be 1.
-- #size-cells	: should be 0.
-- compatible	: "nuvoton,npcm750-pwm-fan" for Poleg NPCM7XX.
-				: "nuvoton,npcm845-pwm-fan" for Arbel NPCM8XX.
-- reg			: specifies physical base address and size of the registers.
-- reg-names	: must contain:
-					* "pwm" for the PWM registers.
-					* "fan" for the Fan registers.
-- clocks		: phandle of reference clocks.
-- clock-names	: must contain
-					* "pwm" for PWM controller operating clock.
-					* "fan" for Fan controller operating clock.
-- interrupts	: contain the Fan interrupts with flags for falling edge.
-- pinctrl-names	: a pinctrl state named "default" must be defined.
-- pinctrl-0	: phandle referencing pin configuration of the PWM and Fan
-					controller ports.
-
-fan subnode format:
-===================
-Under fan subnode can be upto 8 child nodes, each child node representing a fan.
-Each fan subnode must have one PWM channel and at least one Fan tach channel.
-
-For PWM channel can be configured cooling-levels to create cooling device.
-Cooling device could be bound to a thermal zone for the thermal control.
-
-Required properties for each child node:
-- reg : specify the PWM output channel.
-	integer value in the range 0 through 7, that represent
-	the PWM channel number that used.
-
-- fan-tach-ch : specify the Fan tach input channel.
-		integer value in the range 0 through 15, that represent
-		the fan tach channel number that used.
-
-		At least one Fan tach input channel is required
-
-Optional property for each child node:
-- cooling-levels: PWM duty cycle values in a range from 0 to 255
-                  which correspond to thermal cooling states.
-
-Examples:
-
-pwm_fan:pwm-fan-controller@103000 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-	compatible = "nuvoton,npcm750-pwm-fan";
-	reg = <0x103000 0x2000>,
-		<0x180000 0x8000>;
-	reg-names = "pwm", "fan";
-	clocks = <&clk NPCM7XX_CLK_APB3>,
-		<&clk NPCM7XX_CLK_APB4>;
-	clock-names = "pwm","fan";
-	interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pwm0_pins &pwm1_pins &pwm2_pins
-			&fanin0_pins &fanin1_pins &fanin2_pins
-			&fanin3_pins &fanin4_pins>;
-	fan@0 {
-		reg = <0x00>;
-		fan-tach-ch = /bits/ 8 <0x00 0x01>;
-		cooling-levels = <127 255>;
-	};
-	fan@1 {
-		reg = <0x01>;
-		fan-tach-ch = /bits/ 8 <0x02 0x03>;
-	};
-	fan@2 {
-		reg = <0x02>;
-		fan-tach-ch = /bits/ 8 <0x04>;
-	};
-
-};
diff --git a/Documentation/devicetree/bindings/hwmon/nuvoton,npcm750-pwm-fan.yaml b/Documentation/devicetree/bindings/hwmon/nuvoton,npcm750-pwm-fan.yaml
new file mode 100644
index 0000000000000..73464af3078ec
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/nuvoton,npcm750-pwm-fan.yaml
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/nuvoton,npcm750-pwm-fan.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Nuvoton NPCM7xx/NPCM8xx PWM and Fan Tach Controller
+
+maintainers:
+  - Tomer Maimon <tmaimon77@gmail.com>
+
+description:
+  The NPCM7xx/NPCM8xx family includes a PWM and Fan Tachometer controller.
+  The controller provides up to 8 (NPCM7xx) or 12 (NPCM8xx) PWM channels and up
+  to 16 tachometer inputs. It is used for fan speed control and monitoring.
+
+properties:
+  compatible:
+    enum:
+      - nuvoton,npcm750-pwm-fan
+      - nuvoton,npcm845-pwm-fan
+
+  reg:
+    maxItems: 2
+    description: Register addresses for PWM and Fan Tach units.
+
+  reg-names:
+    items:
+      - const: pwm
+      - const: fan
+
+  clocks:
+    maxItems: 2
+    description: Clocks for the PWM and Fan Tach modules.
+
+  clock-names:
+    items:
+      - const: pwm
+      - const: fan
+
+  interrupts:
+    description:
+      Contains the Fan interrupts with flags for falling edge.
+      For NPCM7XX, 8 interrupt lines are expected (one per PWM channel).
+      For NPCM8XX, 12 interrupt lines are expected (one per PWM channel).
+
+    minItems: 8
+    maxItems: 12
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  "^fan@[0-9a-f]+$":
+    type: object
+    $ref: fan-common.yaml#
+    unevaluatedProperties: false
+
+    properties:
+      reg:
+        description:
+          Specify the PWM output channel. Integer value in the range 0-7 for
+          NPCM7XX or 0-11 for NPCM8XX, representing the PWM channel number.
+
+        maximum: 11
+
+      fan-tach-ch:
+        $ref: /schemas/types.yaml#/definitions/uint8-array
+        description:
+          The tach channel(s) used for the fan.
+          Integer values in the range 0-15.
+
+        items:
+          maximum: 15
+
+      cooling-levels:
+        description:
+          PWM duty cycle values in a range from 0 to 255 which
+          correspond to thermal cooling states. This property enables
+          thermal zone integration for automatic fan speed control
+          based on temperature.
+
+        items:
+          maximum: 255
+
+    required:
+      - reg
+      - fan-tach-ch
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - clocks
+  - clock-names
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/nuvoton,npcm7xx-clock.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    pwm_fan: pwm-fan@103000 {
+        compatible = "nuvoton,npcm750-pwm-fan";
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        reg = <0x103000 0x2000>, <0x180000 0x8000>;
+        reg-names = "pwm", "fan";
+
+        clocks = <&clk NPCM7XX_CLK_APB3>, <&clk NPCM7XX_CLK_APB4>;
+        clock-names = "pwm", "fan";
+
+        interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
+        pinctrl-names = "default";
+        pinctrl-0 = <&pwm0_pins &fanin0_pins>;
+
+        fan@0 {
+            reg = <0>;
+            fan-tach-ch = <0 1>;
+            cooling-levels = <64 128 192 255>;
+        };
+
+        fan@1 {
+            reg = <1>;
+            fan-tach-ch = <2>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
index d3cde89366866..009d78b308596 100644
--- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
+++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
@@ -29,6 +29,7 @@ properties:
       - ti,ina230
       - ti,ina231
       - ti,ina233
+      - ti,ina234
       - ti,ina237
       - ti,ina238
       - ti,ina260
@@ -113,6 +114,7 @@ allOf:
               - ti,ina228
               - ti,ina230
               - ti,ina231
+              - ti,ina234
               - ti,ina237
               - ti,ina238
               - ti,ina260
@@ -134,6 +136,7 @@ allOf:
               - ti,ina226
               - ti,ina230
               - ti,ina231
+              - ti,ina234
               - ti,ina260
               - ti,ina700
               - ti,ina780
diff --git a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
index c1387e02eb826..7f9c5eec35ddd 100644
--- a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
+++ b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
@@ -16,25 +16,27 @@ description:
 
 properties:
   compatible:
-    enum:
-      # bmc150-accel driver in Linux
-      - bosch,bma222
-      - bosch,bma222e
-      - bosch,bma250e
-      - bosch,bma253
-      - bosch,bma254
-      - bosch,bma255
-      - bosch,bma280
-      - bosch,bmc150_accel
-      - bosch,bmc156_accel
-      - bosch,bmi055_accel
-
-      # bma180 driver in Linux
-      - bosch,bma023
-      - bosch,bma150
-      - bosch,bma180
-      - bosch,bma250
-      - bosch,smb380
+    oneOf:
+      - enum:
+          - bosch,bma222
+          - bosch,bma222e
+          - bosch,bma250e
+          - bosch,bma253
+          - bosch,bma254
+          - bosch,bma255
+          - bosch,bma280
+          - bosch,bmc150_accel
+          - bosch,bmc156_accel
+          - bosch,bmi055_accel
+
+          - bosch,bma023
+          - bosch,bma150
+          - bosch,bma180
+          - bosch,bma250
+          - bosch,smb380
+      - items:
+          - const: bosch,bmx055-accel
+          - const: bosch,bmc150_accel
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4030.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4030.yaml
index e22d518135f26..08b1f9d75f894 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4030.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4030.yaml
@@ -19,6 +19,10 @@ description: |
   * https://www.analog.com/media/en/technical-documentation/data-sheets/ad4030-24-4032-24.pdf
   * https://www.analog.com/media/en/technical-documentation/data-sheets/ad4630-24_ad4632-24.pdf
   * https://www.analog.com/media/en/technical-documentation/data-sheets/ad4630-16-4632-16.pdf
+  * https://www.analog.com/media/en/technical-documentation/data-sheets/adaq4216.pdf
+  * https://www.analog.com/media/en/technical-documentation/data-sheets/adaq4224.pdf
+
+$ref: /schemas/spi/spi-peripheral-props.yaml#
 
 properties:
   compatible:
@@ -29,6 +33,8 @@ properties:
       - adi,ad4630-24
       - adi,ad4632-16
       - adi,ad4632-24
+      - adi,adaq4216
+      - adi,adaq4224
 
   reg:
     maxItems: 1
@@ -60,6 +66,14 @@ properties:
     description:
       Internal buffered Reference. Used when ref-supply is not connected.
 
+  vddh-supply:
+    description:
+      PGIA Positive Power Supply.
+
+  vdd-fda-supply:
+    description:
+      FDA Positive Power Supply.
+
   cnv-gpios:
     description:
       The Convert Input (CNV). It initiates the sampling conversions.
@@ -70,6 +84,17 @@ properties:
       The Reset Input (/RST). Used for asynchronous device reset.
     maxItems: 1
 
+  pga-gpios:
+    description:
+      A0 and A1 pins for gain selection. For devices that have PGA configuration
+      input pins, pga-gpios should be defined.
+    minItems: 2
+    maxItems: 2
+
+  pwms:
+    description: PWM signal connected to the CNV pin.
+    maxItems: 1
+
   interrupts:
     description:
       The BUSY pin is used to signal that the conversions results are available
@@ -107,6 +132,22 @@ allOf:
       properties:
         spi-rx-bus-width:
           maxItems: 1
+  # ADAQ devices require a gain property to indicate how hardware PGA is set
+  - if:
+      properties:
+        compatible:
+          contains:
+            pattern: ^adi,adaq
+    then:
+      required:
+        - vddh-supply
+        - vdd-fda-supply
+        - pga-gpios
+      properties:
+        ref-supply: false
+    else:
+      properties:
+        pga-gpios: false
 
 examples:
   - |
@@ -148,3 +189,26 @@ examples:
             reset-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
         };
     };
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+            compatible = "adi,adaq4216";
+            reg = <0>;
+            spi-max-frequency = <80000000>;
+            vdd-5v-supply = <&supply_5V>;
+            vdd-1v8-supply = <&supply_1_8V>;
+            vio-supply = <&supply_1_8V>;
+            refin-supply = <&refin_sup>;
+            vddh-supply = <&vddh>;
+            vdd-fda-supply = <&vdd_fda>;
+            cnv-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>;
+            reset-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
+            pga-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>,
+                        <&gpio0 3 GPIO_ACTIVE_HIGH>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
index ccd6a0ac1539e..79df2696ef249 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
@@ -27,10 +27,13 @@ properties:
     enum:
       - adi,ad4080
       - adi,ad4081
+      - adi,ad4082
       - adi,ad4083
       - adi,ad4084
+      - adi,ad4085
       - adi,ad4086
       - adi,ad4087
+      - adi,ad4088
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7380.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7380.yaml
index b91bfb16ed6bc..396e1a1aa8052 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad7380.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7380.yaml
@@ -62,6 +62,11 @@ properties:
   spi-cpol: true
   spi-cpha: true
 
+  spi-rx-bus-width:
+    maxItems: 4
+    items:
+      maximum: 1
+
   vcc-supply:
     description: A 3V to 3.6V supply that powers the chip.
 
@@ -160,6 +165,23 @@ patternProperties:
 unevaluatedProperties: false
 
 allOf:
+  # 2-channel chips only have two SDO lines
+  - if:
+      properties:
+        compatible:
+          enum:
+            - adi,ad7380
+            - adi,ad7381
+            - adi,ad7383
+            - adi,ad7384
+            - adi,ad7386
+            - adi,ad7387
+            - adi,ad7388
+    then:
+      properties:
+        spi-rx-bus-width:
+          maxItems: 2
+
   # pseudo-differential chips require common mode voltage supplies,
   # true differential chips don't use them
   - if:
@@ -284,6 +306,7 @@ examples:
             spi-cpol;
             spi-cpha;
             spi-max-frequency = <80000000>;
+            spi-rx-bus-width = <1>, <1>, <1>, <1>;
 
             interrupts = <27 IRQ_TYPE_EDGE_FALLING>;
             interrupt-parent = <&gpio0>;
diff --git a/Documentation/devicetree/bindings/iio/adc/motorola,cpcap-adc.yaml b/Documentation/devicetree/bindings/iio/adc/motorola,cpcap-adc.yaml
index 9ceb6f18c854f..1f77da7f8e064 100644
--- a/Documentation/devicetree/bindings/iio/adc/motorola,cpcap-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/motorola,cpcap-adc.yaml
@@ -19,6 +19,7 @@ properties:
     enum:
       - motorola,cpcap-adc
       - motorola,mapphone-cpcap-adc
+      - motorola,mot-cpcap-adc
 
   interrupts:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-adc5-gen3.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-adc5-gen3.yaml
new file mode 100644
index 0000000000000..149f4af8f4b8b
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-adc5-gen3.yaml
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/qcom,spmi-adc5-gen3.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm's SPMI PMIC ADC5 Gen3
+
+maintainers:
+  - Jishnu Prakash <jishnu.prakash@oss.qualcomm.com>
+
+description: |
+  SPMI PMIC5 Gen3 voltage ADC (ADC) provides interface to clients to read
+  voltage. It is a 16-bit sigma-delta ADC. It also performs the same thermal
+  monitoring function as the existing ADC_TM devices.
+
+  The interface is implemented on SDAM (Shared Direct Access Memory) peripherals
+  on the master PMIC rather than a dedicated ADC peripheral. The number of PMIC
+  SDAM peripherals allocated for ADC is not correlated with the PMIC used, it is
+  programmed in FW (PBS) and is fixed per SOC, based on the SOC requirements.
+  All boards using a particular (SOC + master PMIC) combination will have the
+  same number of ADC SDAMs supported on that PMIC.
+
+properties:
+  compatible:
+    const: qcom,spmi-adc5-gen3
+
+  reg:
+    items:
+      - description: SDAM0 base address in the SPMI PMIC register map
+      - description: SDAM1 base address
+    minItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+  "#io-channel-cells":
+    const: 1
+
+  "#thermal-sensor-cells":
+    const: 1
+
+  interrupts:
+    items:
+      - description: SDAM0 end of conversion (EOC) interrupt
+      - description: SDAM1 EOC interrupt
+    minItems: 1
+
+patternProperties:
+  "^channel@[0-9a-f]+$":
+    type: object
+    unevaluatedProperties: false
+    $ref: /schemas/iio/adc/qcom,spmi-vadc-common.yaml
+    description:
+      Represents the external channels which are connected to the ADC.
+
+    properties:
+      qcom,decimation:
+        enum: [ 85, 340, 1360 ]
+        default: 1360
+
+      qcom,hw-settle-time:
+        enum: [ 15, 100, 200, 300, 400, 500, 600, 700,
+                1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000 ]
+        default: 15
+
+      qcom,avg-samples:
+        enum: [ 1, 2, 4, 8, 16 ]
+        default: 1
+
+      qcom,adc-tm:
+        description:
+          ADC_TM is a threshold monitoring feature in HW which can be enabled
+          on any ADC channel, to trigger an IRQ for threshold violation. In
+          earlier ADC generations, it was implemented in a separate device
+          (documented in Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml.)
+          In Gen3, this feature can be enabled in the same ADC device for any
+          channel and threshold monitoring and IRQ triggering are handled in FW
+          (PBS) instead of another dedicated HW block.
+          This property indicates ADC_TM monitoring is done on this channel.
+        type: boolean
+
+required:
+  - compatible
+  - reg
+  - "#address-cells"
+  - "#size-cells"
+  - "#io-channel-cells"
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    pmic {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      adc@9000 {
+        compatible = "qcom,spmi-adc5-gen3";
+        reg = <0x9000>, <0x9100>;
+        interrupts = <0x0 0x90 0x1 IRQ_TYPE_EDGE_RISING>,
+                      <0x0 0x91 0x1 IRQ_TYPE_EDGE_RISING>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+        #io-channel-cells = <1>;
+        #thermal-sensor-cells = <1>;
+
+        /* PMK8550 Channel nodes */
+        channel@3 {
+          reg = <0x3>;
+          label = "pmk8550_die_temp";
+          qcom,pre-scaling = <1 1>;
+        };
+
+        channel@44 {
+          reg = <0x44>;
+          label = "pmk8550_xo_therm";
+          qcom,pre-scaling = <1 1>;
+          qcom,ratiometric;
+          qcom,hw-settle-time = <200>;
+          qcom,adc-tm;
+        };
+
+        /* PM8550 Channel nodes */
+        channel@103 {
+          reg = <0x103>;
+          label = "pm8550_die_temp";
+          qcom,pre-scaling = <1 1>;
+        };
+
+        /* PM8550B Channel nodes */
+        channel@78f {
+          reg = <0x78f>;
+          label = "pm8550b_vbat_sns_qbg";
+          qcom,pre-scaling = <1 3>;
+        };
+
+        /* PM8550VS_C Channel nodes */
+        channel@203 {
+          reg = <0x203>;
+          label = "pm8550vs_c_die_temp";
+          qcom,pre-scaling = <1 1>;
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc-common.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc-common.yaml
new file mode 100644
index 0000000000000..3ae252c17b91a
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc-common.yaml
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/qcom,spmi-vadc-common.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. SPMI PMIC ADC channels
+
+maintainers:
+  - Jishnu Prakash <jishnu.prakash@oss.qualcomm.com>
+
+description:
+  This defines the common properties used to define Qualcomm VADC channels.
+
+properties:
+  reg:
+    description:
+      ADC channel number (PMIC-specific for versions after PMIC5 ADC).
+    maxItems: 1
+
+  label:
+    description:
+      ADC input of the platform as seen in the schematics.
+      For thermistor inputs connected to generic AMUX or GPIO inputs
+      these can vary across platform for the same pins. Hence select
+      the platform schematics name for this channel.
+
+  qcom,decimation:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      This parameter is used to decrease ADC sampling rate.
+      Quicker measurements can be made by reducing decimation ratio.
+
+  qcom,pre-scaling:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      Used for scaling the channel input signal before the signal is
+      fed to VADC. The configuration for this node is to know the
+      pre-determined ratio and use it for post scaling. It is a pair of
+      integers, denoting the numerator and denominator of the fraction by which
+      input signal is multiplied. For example, <1 3> indicates the signal is scaled
+      down to 1/3 of its value before ADC measurement.
+      If property is not found default value depending on chip will be used.
+    oneOf:
+      - items:
+          - const: 1
+          - enum: [ 1, 3, 4, 6, 20, 8, 10, 16 ]
+      - items:
+          - const: 10
+          - const: 81
+
+  qcom,ratiometric:
+    type: boolean
+    description: |
+      Channel calibration type.
+      - For compatible property "qcom,spmi-vadc", if this property is
+        specified VADC will use the VDD reference (1.8V) and GND for
+        channel calibration. If property is not found, channel will be
+        calibrated with 0.625V and 1.25V reference channels, also
+        known as absolute calibration.
+      - For other compatible properties, if this property is specified
+        VADC will use the VDD reference (1.875V) and GND for channel
+        calibration. If property is not found, channel will be calibrated
+        with 0V and 1.25V reference channels, also known as absolute calibration.
+
+  qcom,hw-settle-time:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Time between AMUX getting configured and the ADC starting
+      conversion. The 'hw_settle_time' is an index used from valid values
+      and programmed in hardware to achieve the hardware settling delay.
+
+  qcom,avg-samples:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Number of samples to be used for measurement.
+      Averaging provides the option to obtain a single measurement
+      from the ADC that is an average of multiple samples. The value
+      selected is 2^(value).
+
+required:
+  - reg
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml
index b9dc04b0d307c..72188041e8b53 100644
--- a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml
@@ -15,6 +15,8 @@ description: |
   voltage. The VADC is a 15-bit sigma-delta ADC.
   SPMI PMIC5/PMIC7 voltage ADC (ADC) provides interface to clients to read
   voltage. The VADC is a 16-bit sigma-delta ADC.
+  Note that PMIC7 ADC is the generation between PMIC5 and PMIC5 Gen3 ADC,
+  it can be considered like PMIC5 Gen2.
 
 properties:
   compatible:
@@ -56,7 +58,7 @@ required:
 patternProperties:
   "^channel@[0-9a-f]+$":
     type: object
-    additionalProperties: false
+    unevaluatedProperties: false
     description: |
       Represents the external channels which are connected to the ADC.
       For compatible property "qcom,spmi-vadc" following channels, also known as
@@ -64,79 +66,7 @@ patternProperties:
       configuration nodes should be defined:
       VADC_REF_625MV and/or VADC_SPARE1(based on PMIC version) VADC_REF_1250MV,
       VADC_GND_REF and VADC_VDD_VADC.
-
-    properties:
-      reg:
-        maxItems: 1
-        description: |
-          ADC channel number.
-          See include/dt-bindings/iio/qcom,spmi-vadc.h
-          For PMIC7 ADC, the channel numbers are specified separately per PMIC
-          in the PMIC-specific files in include/dt-bindings/iio/.
-
-      label:
-        description: |
-            ADC input of the platform as seen in the schematics.
-            For thermistor inputs connected to generic AMUX or GPIO inputs
-            these can vary across platform for the same pins. Hence select
-            the platform schematics name for this channel.
-
-      qcom,decimation:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description: |
-            This parameter is used to decrease ADC sampling rate.
-            Quicker measurements can be made by reducing decimation ratio.
-
-      qcom,pre-scaling:
-        description: |
-            Used for scaling the channel input signal before the signal is
-            fed to VADC. The configuration for this node is to know the
-            pre-determined ratio and use it for post scaling. It is a pair of
-            integers, denoting the numerator and denominator of the fraction by which
-            input signal is multiplied. For example, <1 3> indicates the signal is scaled
-            down to 1/3 of its value before ADC measurement.
-            If property is not found default value depending on chip will be used.
-        $ref: /schemas/types.yaml#/definitions/uint32-array
-        oneOf:
-          - items:
-              - const: 1
-              - enum: [ 1, 3, 4, 6, 20, 8, 10, 16 ]
-          - items:
-              - const: 10
-              - const: 81
-
-      qcom,ratiometric:
-        description: |
-            Channel calibration type.
-            - For compatible property "qcom,spmi-vadc", if this property is
-              specified VADC will use the VDD reference (1.8V) and GND for
-              channel calibration. If property is not found, channel will be
-              calibrated with 0.625V and 1.25V reference channels, also
-              known as absolute calibration.
-            - For compatible property "qcom,spmi-adc5", "qcom,spmi-adc7" and
-              "qcom,spmi-adc-rev2", if this property is specified VADC will use
-              the VDD reference (1.875V) and GND for channel calibration. If
-              property is not found, channel will be calibrated with 0V and 1.25V
-              reference channels, also known as absolute calibration.
-        type: boolean
-
-      qcom,hw-settle-time:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description: |
-            Time between AMUX getting configured and the ADC starting
-            conversion. The 'hw_settle_time' is an index used from valid values
-            and programmed in hardware to achieve the hardware settling delay.
-
-      qcom,avg-samples:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description: |
-            Number of samples to be used for measurement.
-            Averaging provides the option to obtain a single measurement
-            from the ADC that is an average of multiple samples. The value
-            selected is 2^(value).
-
-    required:
-      - reg
+    $ref: /schemas/iio/adc/qcom,spmi-vadc-common.yaml
 
 allOf:
   - if:
diff --git a/Documentation/devicetree/bindings/iio/amplifiers/adi,ad8366.yaml b/Documentation/devicetree/bindings/iio/amplifiers/adi,ad8366.yaml
new file mode 100644
index 0000000000000..2719de1166a17
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/amplifiers/adi,ad8366.yaml
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/amplifiers/adi,ad8366.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: AD8366 and similar Gain Amplifiers and Digital Attenuators
+
+maintainers:
+  - Michael Hennerich <michael.hennerich@analog.com>
+  - Rodrigo Alencar <rodrigo.alencar@analog.com>
+
+description:
+  Digital Variable Gain Amplifiers (VGAs) and Digital Attenuators with
+  SPI interface.
+
+properties:
+  compatible:
+    enum:
+      - adi,ad8366
+      - adi,ada4961
+      - adi,adl5240
+      - adi,adrf5720
+      - adi,adrf5730
+      - adi,adrf5731
+      - adi,hmc271a
+      - adi,hmc792a
+      - adi,hmc1018a
+      - adi,hmc1019a
+      - adi,hmc1119
+
+  reg:
+    maxItems: 1
+
+  vcc-supply:
+    description: Regulator that provides power to the device.
+
+  reset-gpios:
+    maxItems: 1
+
+  enable-gpios:
+    maxItems: 1
+    description: Power-up or Serial Mode Enable GPIO.
+
+required:
+  - compatible
+  - reg
+  - vcc-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: adi,hmc271a
+    then:
+      properties:
+        reset-gpios: false
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              anyOf:
+                - const: adi,ad8366
+                - const: adi,ada4961
+                - const: adi,adrf5720
+                - const: adi,adrf5730
+                - const: adi,adrf5731
+                - const: adi,hmc792a
+                - const: adi,hmc1018a
+                - const: adi,hmc1019a
+                - const: adi,hmc1119
+    then:
+      properties:
+        enable-gpios: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      amplifier@0 {
+        compatible = "adi,ad8366";
+        reg = <0>;
+        spi-max-frequency = <1000000>;
+        vcc-supply = <&vcc_3v3>;
+        enable-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>;
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml b/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml
index 20dd1370660d5..624c640be4c84 100644
--- a/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml
@@ -9,7 +9,7 @@ title: Texas Instruments DAC7612 family of DACs
 description:
   The DAC7612 is a dual, 12-bit digital-to-analog converter (DAC) with
   guaranteed 12-bit monotonicity performance over the industrial temperature
-  range. Is is programmable through an SPI interface.
+  range. It is programmable through an SPI interface.
 
 maintainers:
   - Ricardo Ribalda Delgado <ricardo@ribalda.com>
diff --git a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
index 3c6fe74af0b83..fcbd4b430e48e 100644
--- a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
+++ b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
@@ -11,10 +11,14 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - bosch,bmg160
-      - bosch,bmi055_gyro
-      - bosch,bmi088_gyro
+    oneOf:
+      - enum:
+          - bosch,bmg160
+          - bosch,bmi055_gyro
+          - bosch,bmi088_gyro
+      - items:
+          - const: bosch,bmx055-gyro
+          - const: bosch,bmg160
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/light/vishay,vcnl4000.yaml b/Documentation/devicetree/bindings/iio/light/vishay,vcnl4000.yaml
index 4d1a225e8868d..2ba4d5de4ec40 100644
--- a/Documentation/devicetree/bindings/iio/light/vishay,vcnl4000.yaml
+++ b/Documentation/devicetree/bindings/iio/light/vishay,vcnl4000.yaml
@@ -18,12 +18,17 @@ allOf:
 
 properties:
   compatible:
-    enum:
-      - vishay,vcnl4000
-      - vishay,vcnl4010
-      - vishay,vcnl4020
-      - vishay,vcnl4040
-      - vishay,vcnl4200
+    oneOf:
+      - enum:
+          - capella,cm36672p
+          - vishay,vcnl4000
+          - vishay,vcnl4010
+          - vishay,vcnl4020
+          - vishay,vcnl4040
+          - vishay,vcnl4200
+      - items:
+          - const: capella,cm36686
+          - const: vishay,vcnl4040
 
   interrupts:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/magnetometer/bosch,bmc150_magn.yaml b/Documentation/devicetree/bindings/iio/magnetometer/bosch,bmc150_magn.yaml
index a3838ab0c524a..c1a6892b0194c 100644
--- a/Documentation/devicetree/bindings/iio/magnetometer/bosch,bmc150_magn.yaml
+++ b/Documentation/devicetree/bindings/iio/magnetometer/bosch,bmc150_magn.yaml
@@ -21,11 +21,15 @@ properties:
     description:
       Note the bmm150_magn is a deprecated compatible as this part contains only
       a magnetometer.
-    enum:
-      - bosch,bmc150_magn
-      - bosch,bmc156_magn
-      - bosch,bmm150
-      - bosch,bmm150_magn
+    oneOf:
+      - enum:
+          - bosch,bmc150_magn
+          - bosch,bmc156_magn
+          - bosch,bmm150
+          - bosch,bmm150_magn
+      - items:
+          - const: bosch,bmx055-magn
+          - const: bosch,bmc150_magn
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/proximity/tyhx,hx9023s.yaml b/Documentation/devicetree/bindings/iio/proximity/tyhx,hx9023s.yaml
index 64ce8bc8bd365..cc5b5284c2672 100644
--- a/Documentation/devicetree/bindings/iio/proximity/tyhx,hx9023s.yaml
+++ b/Documentation/devicetree/bindings/iio/proximity/tyhx,hx9023s.yaml
@@ -28,6 +28,9 @@ properties:
 
   vdd-supply: true
 
+  firmware-name:
+    maxItems: 1
+
   "#address-cells":
     const: 1
 
@@ -65,6 +68,7 @@ examples:
         interrupt-parent = <&pio>;
         interrupts = <16 IRQ_TYPE_EDGE_FALLING>;
         vdd-supply = <&pp1800_prox>;
+        firmware-name = "hx9023s.bin";
 
         #address-cells = <1>;
         #size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/input/parade,tc3408.yaml b/Documentation/devicetree/bindings/input/parade,tc3408.yaml
new file mode 100644
index 0000000000000..30ffefb96c68a
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/parade,tc3408.yaml
@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/parade,tc3408.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Parade TC3408 touchscreen controller
+
+maintainers:
+  - Langyan Ye <yelangyan@huaqin.corp-partner.google.com>
+
+description: |
+  Parade TC3408 is a touchscreen controller supporting the I2C-HID protocol.
+  It requires a reset GPIO and two power supplies (3.3V and 1.8V).
+
+allOf:
+  - $ref: /schemas/input/touchscreen/touchscreen.yaml#
+
+properties:
+  compatible:
+    const: parade,tc3408
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  reset-gpios:
+    maxItems: 1
+
+  vcc33-supply:
+    description: The 3.3V supply to the touchscreen.
+
+  vccio-supply:
+    description: The 1.8V supply to the touchscreen.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - reset-gpios
+  - vcc33-supply
+  - vccio-supply
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      touchscreen: touchscreen@24 {
+        compatible = "parade,tc3408";
+        reg = <0x24>;
+
+        interrupt-parent = <&pio>;
+        interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+
+        reset-gpios = <&pio 126 GPIO_ACTIVE_LOW>;
+        vcc33-supply = <&pp3300_tchscr_x>;
+        vccio-supply = <&pp1800_tchscr_report_disable>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/input/touchscreen/fsl,imx25-tcq.yaml b/Documentation/devicetree/bindings/input/touchscreen/fsl,imx25-tcq.yaml
new file mode 100644
index 0000000000000..94452ac423d04
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/touchscreen/fsl,imx25-tcq.yaml
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/touchscreen/fsl,imx25-tcq.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Freescale mx25 TS conversion queue module
+
+maintainers:
+  - Frank Li <Frank.Li@nxp.com>
+
+description:
+  mx25 touchscreen conversion queue module which controls the ADC unit of the
+  mx25 for attached touchscreens.
+
+properties:
+  compatible:
+    const: fsl,imx25-tcq
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  fsl,wires:
+    description: touch wires number.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [4, 5]
+
+  fsl,pen-debounce-ns:
+    description:
+      Pen debounce time in nanoseconds.
+
+  fsl,pen-threshold:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Pen-down threshold for the touchscreen. This is a value
+      between 1 and 4096. It is the ratio between the internal reference voltage
+      and the measured voltage after the plate was precharged. Resistance between
+      plates and therefore the voltage decreases with pressure so that a smaller
+      value is equivalent to a higher pressure.
+
+  fsl,settling-time-ns:
+    description:
+      Settling time in nanoseconds. The settling time is before
+      the actual touch detection to wait for an even charge distribution in the
+      plate.
+
+allOf:
+  - $ref: touchscreen.yaml
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - fsl,wires
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    touchscreen@50030400 {
+        compatible = "fsl,imx25-tcq";
+        reg = <0x50030400 0x60>;
+        interrupt-parent = <&tscadc>;
+        interrupts = <0>;
+        fsl,wires = <4>;
+    };
diff --git a/Documentation/devicetree/bindings/input/touchscreen/fsl-mx25-tcq.txt b/Documentation/devicetree/bindings/input/touchscreen/fsl-mx25-tcq.txt
deleted file mode 100644
index 99d6f9d253357..0000000000000
--- a/Documentation/devicetree/bindings/input/touchscreen/fsl-mx25-tcq.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-Freescale mx25 TS conversion queue module
-
-mx25 touchscreen conversion queue module which controls the ADC unit of the
-mx25 for attached touchscreens.
-
-Required properties:
- - compatible: Should be "fsl,imx25-tcq".
- - reg: Memory range of the device.
- - interrupts: Should be the interrupt number associated with this module within
-   the tscadc unit (<0>).
- - fsl,wires: Should be '<4>' or '<5>'
-
-Optional properties:
- - fsl,pen-debounce-ns: Pen debounce time in nanoseconds.
- - fsl,pen-threshold: Pen-down threshold for the touchscreen. This is a value
-   between 1 and 4096. It is the ratio between the internal reference voltage
-   and the measured voltage after the plate was precharged. Resistance between
-   plates and therefore the voltage decreases with pressure so that a smaller
-   value is equivalent to a higher pressure.
- - fsl,settling-time-ns: Settling time in nanoseconds. The settling time is before
-   the actual touch detection to wait for an even charge distribution in the
-   plate.
-
-This device includes two conversion queues which can be added as subnodes.
-The first queue is for the touchscreen, the second for general purpose ADC.
-
-Example:
-	tsc: tcq@50030400 {
-		compatible = "fsl,imx25-tcq";
-		reg = <0x50030400 0x60>;
-		interrupt-parent = <&tscadc>;
-		interrupts = <0>;
-		fsl,wires = <4>;
-	};
diff --git a/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml b/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
index 978afaa4fcef3..fe1fa217d8426 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
@@ -32,6 +32,9 @@ properties:
     description: A phandle to the reset GPIO
     maxItems: 1
 
+  wakeup-source:
+    type: boolean
+
 required:
   - compatible
   - reg
@@ -51,6 +54,7 @@ examples:
                     reg = <0x55>;
                     interrupts = <2 0>;
                     gpios = <&gpio1 166 0>;
+                    wakeup-source;
 
                     touch-overlay {
                             segment-0 {
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,eliza-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,eliza-rpmh.yaml
new file mode 100644
index 0000000000000..9a926a97e7bfa
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,eliza-rpmh.yaml
@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,eliza-rpmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm RPMh Network-On-Chip Interconnect on Eliza SoC
+
+maintainers:
+  - Odelu Kukatla <odelu.kukatla@oss.qualcomm.com>
+
+description: |
+  RPMh interconnect providers support system bandwidth requirements through
+  RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is
+  able to communicate with the BCM through the Resource State Coordinator (RSC)
+  associated with each execution environment. Provider nodes must point to at
+  least one RPMh device child node pertaining to their RSC and each provider
+  can map to multiple RPMh resources.
+
+  See also: include/dt-bindings/interconnect/qcom,eliza-rpmh.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,eliza-aggre1-noc
+      - qcom,eliza-aggre2-noc
+      - qcom,eliza-clk-virt
+      - qcom,eliza-cnoc-cfg
+      - qcom,eliza-cnoc-main
+      - qcom,eliza-gem-noc
+      - qcom,eliza-lpass-ag-noc
+      - qcom,eliza-lpass-lpiaon-noc
+      - qcom,eliza-lpass-lpicx-noc
+      - qcom,eliza-mc-virt
+      - qcom,eliza-mmss-noc
+      - qcom,eliza-nsp-noc
+      - qcom,eliza-pcie-anoc
+      - qcom,eliza-system-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 2
+
+required:
+  - compatible
+
+allOf:
+  - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-clk-virt
+              - qcom,eliza-mc-virt
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre USB3 PRIM AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-aggre2-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-pcie-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre-NOC PCIe AXI clock
+            - description: cfg-NOC PCIe a-NOC AHB clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-aggre1-noc
+              - qcom,eliza-aggre2-noc
+              - qcom,eliza-pcie-anoc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    gem_noc: interconnect@24100000 {
+        compatible = "qcom,eliza-gem-noc";
+        reg = <0x24100000 0x163080>;
+        #interconnect-cells = <2>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    mc_virt: interconnect-2 {
+        compatible = "qcom,eliza-mc-virt";
+        #interconnect-cells = <2>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    aggre1_noc: interconnect@16e0000 {
+        compatible = "qcom,eliza-aggre1-noc";
+        reg = <0x16e0000 0x16400>;
+        #interconnect-cells = <2>;
+        clocks = <&gcc_phy_axi_clk>, <&gcc_prim_axi_clk>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,glymur-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,glymur-rpmh.yaml
index d55a7bcf5591e..f69b2facb658e 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,glymur-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,glymur-rpmh.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/interconnect/qcom,glymur-rpmh.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Qualcomm RPMh Network-On-Chip Interconnect on GLYMUR
+title: Qualcomm RPMh Network-On-Chip Interconnect on Glymur and Mahua SoCs
 
 maintainers:
   - Raviteja Laggyshetty <raviteja.laggyshetty@oss.qualcomm.com>
@@ -21,28 +21,98 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - qcom,glymur-aggre1-noc
-      - qcom,glymur-aggre2-noc
-      - qcom,glymur-aggre3-noc
-      - qcom,glymur-aggre4-noc
-      - qcom,glymur-clk-virt
-      - qcom,glymur-cnoc-cfg
-      - qcom,glymur-cnoc-main
-      - qcom,glymur-hscnoc
-      - qcom,glymur-lpass-ag-noc
-      - qcom,glymur-lpass-lpiaon-noc
-      - qcom,glymur-lpass-lpicx-noc
-      - qcom,glymur-mc-virt
-      - qcom,glymur-mmss-noc
-      - qcom,glymur-nsinoc
-      - qcom,glymur-nsp-noc
-      - qcom,glymur-oobm-ss-noc
-      - qcom,glymur-pcie-east-anoc
-      - qcom,glymur-pcie-east-slv-noc
-      - qcom,glymur-pcie-west-anoc
-      - qcom,glymur-pcie-west-slv-noc
-      - qcom,glymur-system-noc
+    oneOf:
+      - items:
+          - enum:
+              - qcom,mahua-aggre1-noc
+          - const: qcom,glymur-aggre1-noc
+      - items:
+          - enum:
+              - qcom,mahua-aggre2-noc
+          - const: qcom,glymur-aggre2-noc
+      - items:
+          - enum:
+              - qcom,mahua-aggre3-noc
+          - const: qcom,glymur-aggre3-noc
+      - items:
+          - enum:
+              - qcom,mahua-aggre4-noc
+          - const: qcom,glymur-aggre4-noc
+      - items:
+          - enum:
+              - qcom,mahua-clk-virt
+          - const: qcom,glymur-clk-virt
+      - items:
+          - enum:
+              - qcom,mahua-cnoc-main
+          - const: qcom,glymur-cnoc-main
+      - items:
+          - enum:
+              - qcom,mahua-lpass-ag-noc
+          - const: qcom,glymur-lpass-ag-noc
+      - items:
+          - enum:
+              - qcom,mahua-lpass-lpiaon-noc
+          - const: qcom,glymur-lpass-lpiaon-noc
+      - items:
+          - enum:
+              - qcom,mahua-lpass-lpicx-noc
+          - const: qcom,glymur-lpass-lpicx-noc
+      - items:
+          - enum:
+              - qcom,mahua-mmss-noc
+          - const: qcom,glymur-mmss-noc
+      - items:
+          - enum:
+              - qcom,mahua-nsinoc
+          - const: qcom,glymur-nsinoc
+      - items:
+          - enum:
+              - qcom,mahua-nsp-noc
+          - const: qcom,glymur-nsp-noc
+      - items:
+          - enum:
+              - qcom,mahua-oobm-ss-noc
+          - const: qcom,glymur-oobm-ss-noc
+      - items:
+          - enum:
+              - qcom,mahua-pcie-east-anoc
+          - const: qcom,glymur-pcie-east-anoc
+      - items:
+          - enum:
+              - qcom,mahua-pcie-east-slv-noc
+          - const: qcom,glymur-pcie-east-slv-noc
+      - items:
+          - enum:
+              - qcom,mahua-system-noc
+          - const: qcom,glymur-system-noc
+      - enum:
+          - qcom,glymur-aggre1-noc
+          - qcom,glymur-aggre2-noc
+          - qcom,glymur-aggre3-noc
+          - qcom,glymur-aggre4-noc
+          - qcom,glymur-clk-virt
+          - qcom,glymur-cnoc-cfg
+          - qcom,glymur-cnoc-main
+          - qcom,glymur-hscnoc
+          - qcom,glymur-lpass-ag-noc
+          - qcom,glymur-lpass-lpiaon-noc
+          - qcom,glymur-lpass-lpicx-noc
+          - qcom,glymur-mc-virt
+          - qcom,glymur-mmss-noc
+          - qcom,glymur-nsinoc
+          - qcom,glymur-nsp-noc
+          - qcom,glymur-oobm-ss-noc
+          - qcom,glymur-pcie-east-anoc
+          - qcom,glymur-pcie-east-slv-noc
+          - qcom,glymur-pcie-west-anoc
+          - qcom,glymur-pcie-west-slv-noc
+          - qcom,glymur-system-noc
+          - qcom,mahua-cnoc-cfg
+          - qcom,mahua-hscnoc
+          - qcom,mahua-mc-virt
+          - qcom,mahua-pcie-west-anoc
+          - qcom,mahua-pcie-west-slv-noc
 
   reg:
     maxItems: 1
@@ -63,6 +133,7 @@ allOf:
             enum:
               - qcom,glymur-clk-virt
               - qcom,glymur-mc-virt
+              - qcom,mahua-mc-virt
     then:
       properties:
         reg: false
@@ -85,6 +156,20 @@ allOf:
             - description: aggre PCIE_4 WEST AXI clock
             - description: aggre PCIE_6 WEST AXI clock
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,mahua-pcie-west-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre PCIE_3B WEST AXI clock
+            - description: aggre PCIE_4 WEST AXI clock
+            - description: aggre PCIE_6 WEST AXI clock
+
   - if:
       properties:
         compatible:
@@ -131,10 +216,11 @@ allOf:
         compatible:
           contains:
             enum:
-              - qcom,glymur-pcie-west-anoc
-              - qcom,glymur-pcie-east-anoc
               - qcom,glymur-aggre2-noc
               - qcom,glymur-aggre4-noc
+              - qcom,glymur-pcie-east-anoc
+              - qcom,glymur-pcie-west-anoc
+              - qcom,mahua-pcie-west-anoc
     then:
       required:
         - clocks
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
index 4b9b98fbe8f22..41b9f758bf8b8 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
@@ -28,12 +28,14 @@ properties:
           - const: qcom,osm-l3
       - items:
           - enum:
+              - qcom,eliza-epss-l3
               - qcom,sa8775p-epss-l3
               - qcom,sc7280-epss-l3
               - qcom,sc8280xp-epss-l3
               - qcom,sm6375-cpucp-l3
               - qcom,sm8250-epss-l3
               - qcom,sm8350-epss-l3
+              - qcom,sm8550-epss-l3
               - qcom,sm8650-epss-l3
           - const: qcom,epss-l3
       - items:
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,qcs615-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,qcs615-rpmh.yaml
index e064048288248..a9cd49bbe247e 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,qcs615-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,qcs615-rpmh.yaml
@@ -34,6 +34,13 @@ properties:
   reg:
     maxItems: 1
 
+  clocks:
+    items:
+      - description: aggre UFS PHY AXI clock
+      - description: aggre USB2 SEC AXI clock
+      - description: aggre USB3 PRIM AXI clock
+      - description: RPMH CC IPA clock
+
 required:
   - compatible
 
@@ -53,6 +60,22 @@ allOf:
       required:
         - reg
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,qcs615-camnoc-virt
+              - qcom,qcs615-config-noc
+              - qcom,qcs615-dc-noc
+              - qcom,qcs615-gem-noc
+              - qcom,qcs615-mc-virt
+              - qcom,qcs615-mmss-noc
+              - qcom,qcs615-system-noc
+    then:
+      properties:
+        clocks: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,qcs8300-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,qcs8300-rpmh.yaml
index e9f528d6d9a8c..88fe172771102 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,qcs8300-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,qcs8300-rpmh.yaml
@@ -35,6 +35,10 @@ properties:
   reg:
     maxItems: 1
 
+  clocks:
+    minItems: 1
+    maxItems: 4
+
 required:
   - compatible
 
@@ -54,6 +58,64 @@ allOf:
       required:
         - reg
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,qcs8300-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre QUP PRIM AXI clock
+            - description: aggre USB2 PRIM AXI clock
+            - description: aggre USB3 PRIM AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,qcs8300-aggre2-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,qcs8300-gem-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: GCC DDRSS GPU AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,qcs8300-clk-virt
+              - qcom,qcs8300-config-noc
+              - qcom,qcs8300-dc-noc
+              - qcom,qcs8300-gpdsp-anoc
+              - qcom,qcs8300-lpass-ag-noc
+              - qcom,qcs8300-mc-virt
+              - qcom,qcs8300-mmss-noc
+              - qcom,qcs8300-nspa-noc
+              - qcom,qcs8300-pcie-anoc
+              - qcom,qcs8300-system-noc
+    then:
+      properties:
+        clocks: false
+
 unevaluatedProperties: false
 
 examples:
@@ -63,6 +125,7 @@ examples:
         reg = <0x9100000 0xf7080>;
         #interconnect-cells = <2>;
         qcom,bcm-voters = <&apps_bcm_voter>;
+        clocks = <&gcc_ddrss_gpu_axi_clk>;
     };
 
     clk_virt: interconnect-0 {
diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
index ee5a0dfff4378..d0d9a90e96e7b 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
@@ -4,10 +4,10 @@
 $id: http://devicetree.org/schemas/interrupt-controller/apple,aic2.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Apple Interrupt Controller 2
+title: Apple Interrupt Controller 2 and 3
 
 maintainers:
-  - Hector Martin <marcan@marcan.st>
+  - Janne Grunau <j@jannau.net>
 
 description: |
   The Apple Interrupt Controller 2 is a simple interrupt controller present on
@@ -28,14 +28,24 @@ description: |
   which do not go through a discrete interrupt controller. It also handles
   FIQ-based Fast IPIs.
 
+  The Apple Interrupt Controller 3 is in its base functionality very similar to
+  the Apple Interrupt Controller 2 and uses the same device tree bindings. It is
+  found on Apple ARM SoCs platforms starting with t8122 (M3).
+
 properties:
   compatible:
-    items:
-      - enum:
-          - apple,t8112-aic
-          - apple,t6000-aic
-          - apple,t6020-aic
-      - const: apple,aic2
+    oneOf:
+      - items:
+          - enum:
+              - apple,t6000-aic
+              - apple,t6020-aic
+              - apple,t8112-aic
+          - const: apple,aic2
+      - items:
+          - enum:
+              - apple,t6030-aic3
+          - const: apple,t8122-aic3
+      - const: apple,t8122-aic3
 
   interrupt-controller: true
 
@@ -117,7 +127,9 @@ allOf:
       properties:
         compatible:
           contains:
-            const: apple,t8112-aic
+            enum:
+              - apple,t8112-aic
+              - apple,t8122-aic3
     then:
       properties:
         '#interrupt-cells':
diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
index bfd30aae682bf..360a0643a0b56 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
@@ -50,7 +50,7 @@ properties:
       The 2nd cell contains the interrupt number for the interrupt type.
       SPI interrupts are in the range [0-987]. PPI interrupts are in the
       range [0-15]. Extended SPI interrupts are in the range [0-1023].
-      Extended PPI interrupts are in the range [0-127].
+      Extended PPI interrupts are in the range [0-63].
 
       The 3rd cell is the flags, encoded as follows:
       bits[3:0] trigger type and level flags.
diff --git a/Documentation/devicetree/bindings/interrupt-controller/fsl,irqsteer.yaml b/Documentation/devicetree/bindings/interrupt-controller/fsl,irqsteer.yaml
index 5c768c1e159c1..13cd37bf48e45 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/fsl,irqsteer.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/fsl,irqsteer.yaml
@@ -12,7 +12,9 @@ maintainers:
 properties:
   compatible:
     oneOf:
-      - const: fsl,imx-irqsteer
+      - enum:
+          - fsl,imx-irqsteer
+          - nxp,s32n79-irqsteer
       - items:
           - enum:
               - fsl,imx8m-irqsteer
diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
index f9321366cae45..5ad68b2c6fc63 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
@@ -26,6 +26,7 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,eliza-pdc
           - qcom,glymur-pdc
           - qcom,kaanapali-pdc
           - qcom,milos-pdc
diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
index e0267223887ec..639bbeb1f6bdf 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
@@ -71,6 +71,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-plic
+              - canaan,k230-plic
               - sophgo,cv1800b-plic
               - sophgo,cv1812h-plic
               - sophgo,sg2002-plic
diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml
index cdbd23b5c08c1..27d25bc98cbe2 100644
--- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml
+++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml
@@ -35,6 +35,7 @@ properties:
       - description: Qcom SoCs implementing "qcom,smmu-500" and "arm,mmu-500"
         items:
           - enum:
+              - qcom,eliza-smmu-500
               - qcom,glymur-smmu-500
               - qcom,kaanapali-smmu-500
               - qcom,milos-smmu-500
diff --git a/Documentation/devicetree/bindings/leds/allwinner,sun50i-a100-ledc.yaml b/Documentation/devicetree/bindings/leds/allwinner,sun50i-a100-ledc.yaml
index 760cb336dccb5..0b73fe5b662fd 100644
--- a/Documentation/devicetree/bindings/leds/allwinner,sun50i-a100-ledc.yaml
+++ b/Documentation/devicetree/bindings/leds/allwinner,sun50i-a100-ledc.yaml
@@ -21,6 +21,7 @@ properties:
           - enum:
               - allwinner,sun20i-d1-ledc
               - allwinner,sun50i-r329-ledc
+              - allwinner,sun55i-a523-ledc
           - const: allwinner,sun50i-a100-ledc
 
   reg:
diff --git a/Documentation/devicetree/bindings/leds/sprd,sc2731-bltc.yaml b/Documentation/devicetree/bindings/leds/sprd,sc2731-bltc.yaml
index 97535d6dc47a7..2ae5cc31e623e 100644
--- a/Documentation/devicetree/bindings/leds/sprd,sc2731-bltc.yaml
+++ b/Documentation/devicetree/bindings/leds/sprd,sc2731-bltc.yaml
@@ -18,7 +18,12 @@ description: |
 
 properties:
   compatible:
-    const: sprd,sc2731-bltc
+    oneOf:
+      - items:
+          - enum:
+              - sprd,sc2730-bltc
+          - const: sprd,sc2731-bltc
+      - const: sprd,sc2731-bltc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/media/i2c/alliedvision,alvium-csi2.yaml b/Documentation/devicetree/bindings/media/i2c/alliedvision,alvium-csi2.yaml
index d3329e991d165..ae48dd4ab5897 100644
--- a/Documentation/devicetree/bindings/media/i2c/alliedvision,alvium-csi2.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/alliedvision,alvium-csi2.yaml
@@ -8,7 +8,7 @@ title: Allied Vision Alvium Camera
 
 maintainers:
   - Tommaso Merciai <tomm.merciai@gmail.com>
-  - Martin Hecht <martin.hecht@avnet.eu>
+  - Martin Hecht <mhecht73@gmail.com>
 
 allOf:
   - $ref: /schemas/media/video-interface-devices.yaml#
diff --git a/Documentation/devicetree/bindings/media/i2c/onnn,mt9m114.yaml b/Documentation/devicetree/bindings/media/i2c/onnn,mt9m114.yaml
index dffd23ca4839c..e896f4db24214 100644
--- a/Documentation/devicetree/bindings/media/i2c/onnn,mt9m114.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/onnn,mt9m114.yaml
@@ -17,7 +17,9 @@ description: |-
 
 properties:
   compatible:
-    const: onnn,mt9m114
+    enum:
+      - onnn,mt9m114
+      - aptina,mi1040
 
   reg:
     description: I2C device address
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
new file mode 100644
index 0000000000000..6050d7e7dcfea
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/sony,imx355.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sony IMX355 Sensor
+
+maintainers:
+  - Richard Acayan <mailingradian@gmail.com>
+
+description:
+  The IMX355 sensor is a 3280x2464 image sensor, commonly found as the front
+  camera in smartphones.
+
+allOf:
+  - $ref: /schemas/media/video-interface-devices.yaml#
+
+properties:
+  compatible:
+    const: sony,imx355
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  avdd-supply:
+    description: Analog power supply.
+
+  dvdd-supply:
+    description: Digital power supply.
+
+  dovdd-supply:
+    description: Interface power supply.
+
+  reset-gpios:
+    description: Reset GPIO (active low).
+    maxItems: 1
+
+  port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    additionalProperties: false
+
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml
+        unevaluatedProperties: false
+
+        properties:
+          data-lanes:
+            minItems: 4
+            maxItems: 4
+
+        required:
+          - link-frequencies
+
+    required:
+      - endpoint
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - avdd-supply
+  - dvdd-supply
+  - dovdd-supply
+  - port
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,camcc-sdm845.h>
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        camera@1a {
+            compatible = "sony,imx355";
+            reg = <0x1a>;
+
+            clocks = <&camcc CAM_CC_MCLK2_CLK>;
+
+            assigned-clocks = <&camcc CAM_CC_MCLK2_CLK>;
+            assigned-clock-rates = <24000000>;
+
+            reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>;
+
+            avdd-supply = <&cam_front_ldo>;
+            dvdd-supply = <&cam_front_ldo>;
+            dovdd-supply = <&cam_vio_ldo>;
+
+            pinctrl-names = "default";
+            pinctrl-0 = <&cam_front_default>;
+
+            rotation = <270>;
+            orientation = <0>;
+
+            port {
+                cam_front_endpoint: endpoint {
+                    data-lanes = <1 2 3 4>;
+                    link-frequencies = /bits/ 64 <360000000>;
+                    remote-endpoint = <&camss_endpoint1>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/i2c/ti,ds90ub960.yaml b/Documentation/devicetree/bindings/media/i2c/ti,ds90ub960.yaml
index 0539d52de4223..8e2b82d6dc816 100644
--- a/Documentation/devicetree/bindings/media/i2c/ti,ds90ub960.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ti,ds90ub960.yaml
@@ -13,12 +13,10 @@ description:
   The TI DS90UB9XX devices are FPD-Link video deserializers with I2C and GPIO
   forwarding.
 
-allOf:
-  - $ref: /schemas/i2c/i2c-atr.yaml#
-
 properties:
   compatible:
     enum:
+      - ti,ds90ub954-q1
       - ti,ds90ub960-q1
       - ti,ds90ub9702-q1
 
@@ -125,116 +123,127 @@ properties:
 
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
+    description:
+      Ports represent FPD-Link inputs to the deserializer and CSI TX outputs
+      from the deserializer. The number of ports is model-dependent.
 
-    properties:
-      port@0:
-        $ref: /schemas/graph.yaml#/$defs/port-base
-        unevaluatedProperties: false
-        description: FPD-Link input 0
-
-        properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-            description:
-              Endpoint for FPD-Link port. If the RX mode for this port is RAW,
-              hsync-active and vsync-active must be defined.
-
-      port@1:
-        $ref: /schemas/graph.yaml#/$defs/port-base
-        unevaluatedProperties: false
-        description: FPD-Link input 1
-
-        properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-            description:
-              Endpoint for FPD-Link port. If the RX mode for this port is RAW,
-              hsync-active and vsync-active must be defined.
-
-      port@2:
-        $ref: /schemas/graph.yaml#/$defs/port-base
-        unevaluatedProperties: false
-        description: FPD-Link input 2
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - ports
 
-        properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-            description:
-              Endpoint for FPD-Link port. If the RX mode for this port is RAW,
-              hsync-active and vsync-active must be defined.
+$defs:
+  FPDLink-input-port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    unevaluatedProperties: false
+    description: FPD-Link input
 
-      port@3:
-        $ref: /schemas/graph.yaml#/$defs/port-base
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
         unevaluatedProperties: false
-        description: FPD-Link input 3
+        description:
+          Endpoint for FPD-Link port. If the RX mode for this port is RAW,
+          hsync-active and vsync-active must be defined.
 
-        properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-            description:
-              Endpoint for FPD-Link port. If the RX mode for this port is RAW,
-              hsync-active and vsync-active must be defined.
+  CSI2-output-port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    unevaluatedProperties: false
+    description: CSI-2 Output
 
-      port@4:
-        $ref: /schemas/graph.yaml#/$defs/port-base
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
         unevaluatedProperties: false
-        description: CSI-2 Output 0
 
         properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-
-            properties:
-              data-lanes:
-                minItems: 1
-                maxItems: 4
-              link-frequencies:
-                maxItems: 1
-
-            required:
-              - data-lanes
-              - link-frequencies
-
-      port@5:
-        $ref: /schemas/graph.yaml#/$defs/port-base
-        unevaluatedProperties: false
-        description: CSI-2 Output 1
+          data-lanes:
+            minItems: 1
+            maxItems: 4
+          link-frequencies:
+            maxItems: 1
 
-        properties:
-          endpoint:
-            $ref: /schemas/media/video-interfaces.yaml#
-            unevaluatedProperties: false
-
-            properties:
-              data-lanes:
-                minItems: 1
-                maxItems: 4
-              link-frequencies:
-                maxItems: 1
-
-            required:
-              - data-lanes
-              - link-frequencies
-
-    required:
-      - port@0
-      - port@1
-      - port@2
-      - port@3
-      - port@4
-      - port@5
+        required:
+          - data-lanes
+          - link-frequencies
 
-required:
-  - compatible
-  - reg
-  - clocks
-  - clock-names
-  - ports
+allOf:
+  - $ref: /schemas/i2c/i2c-atr.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - ti,ds90ub960-q1
+              - ti,ds90ub9702-q1
+    then:
+      properties:
+        ports:
+          properties:
+            port@0:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 0
+
+            port@1:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 1
+
+            port@2:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 2
+
+            port@3:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 3
+
+            port@4:
+              $ref: '#/$defs/CSI2-output-port'
+              description: CSI-2 Output 0
+
+            port@5:
+              $ref: '#/$defs/CSI2-output-port'
+              description: CSI-2 Output 1
+
+          required:
+            - port@0
+            - port@1
+            - port@2
+            - port@3
+            - port@4
+            - port@5
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,ds90ub954-q1
+    then:
+      properties:
+        ports:
+          properties:
+            port@0:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 0
+
+            port@1:
+              $ref: '#/$defs/FPDLink-input-port'
+              description: FPD-Link input 1
+
+            port@2:
+              $ref: '#/$defs/CSI2-output-port'
+              description: CSI-2 Output 0
+
+          required:
+            - port@0
+            - port@1
+            - port@2
+
+        links:
+          properties:
+            link@2: false
+            link@3: false
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/media/qcom,sdm670-camss.yaml b/Documentation/devicetree/bindings/media/qcom,sdm670-camss.yaml
index 46cc7fff15992..084b65740d534 100644
--- a/Documentation/devicetree/bindings/media/qcom,sdm670-camss.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,sdm670-camss.yaml
@@ -124,7 +124,6 @@ properties:
                 maxItems: 4
 
             required:
-              - clock-lanes
               - data-lanes
 
       port@1:
@@ -147,7 +146,6 @@ properties:
                 maxItems: 4
 
             required:
-              - clock-lanes
               - data-lanes
 
       port@2:
@@ -170,7 +168,6 @@ properties:
                 maxItems: 4
 
             required:
-              - clock-lanes
               - data-lanes
 
 required:
diff --git a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
index 34147127192fd..979360e9099e7 100644
--- a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
+++ b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
@@ -27,11 +27,14 @@ properties:
       - const: mclk
 
   dmas:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
 
   dma-names:
     items:
       - const: tx
+      - const: mdma_tx
+    minItems: 1
 
   resets:
     maxItems: 1
@@ -40,6 +43,12 @@ properties:
     minItems: 1
     maxItems: 2
 
+  sram:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      phandle to a reserved SRAM region which is used as temporary
+      storage memory between DMA and MDMA engines.
+
   port:
     $ref: /schemas/graph.yaml#/$defs/port-base
     unevaluatedProperties: false
diff --git a/Documentation/devicetree/bindings/media/starfive,jh7110-camss.yaml b/Documentation/devicetree/bindings/media/starfive,jh7110-camss.yaml
deleted file mode 100644
index c66586d90fa2b..0000000000000
--- a/Documentation/devicetree/bindings/media/starfive,jh7110-camss.yaml
+++ /dev/null
@@ -1,180 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/media/starfive,jh7110-camss.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Starfive SoC CAMSS ISP
-
-maintainers:
-  - Jack Zhu <jack.zhu@starfivetech.com>
-  - Changhuang Liang <changhuang.liang@starfivetech.com>
-
-description:
-  The Starfive CAMSS ISP is a Camera interface for Starfive JH7110 SoC. It
-  consists of a VIN controller (Video In Controller, a top-level control unit)
-  and an ISP.
-
-properties:
-  compatible:
-    const: starfive,jh7110-camss
-
-  reg:
-    maxItems: 2
-
-  reg-names:
-    items:
-      - const: syscon
-      - const: isp
-
-  clocks:
-    maxItems: 7
-
-  clock-names:
-    items:
-      - const: apb_func
-      - const: wrapper_clk_c
-      - const: dvp_inv
-      - const: axiwr
-      - const: mipi_rx0_pxl
-      - const: ispcore_2x
-      - const: isp_axi
-
-  resets:
-    maxItems: 6
-
-  reset-names:
-    items:
-      - const: wrapper_p
-      - const: wrapper_c
-      - const: axird
-      - const: axiwr
-      - const: isp_top_n
-      - const: isp_top_axi
-
-  power-domains:
-    items:
-      - description: JH7110 ISP Power Domain Switch Controller.
-
-  interrupts:
-    maxItems: 4
-
-  ports:
-    $ref: /schemas/graph.yaml#/properties/ports
-
-    properties:
-      port@0:
-        $ref: /schemas/graph.yaml#/$defs/port-base
-        unevaluatedProperties: false
-        description: Input port for receiving DVP data.
-
-        properties:
-          endpoint:
-            $ref: video-interfaces.yaml#
-            unevaluatedProperties: false
-
-            properties:
-              bus-type:
-                enum: [5, 6]
-
-              bus-width:
-                enum: [8, 10, 12]
-
-              data-shift:
-                enum: [0, 2]
-                default: 0
-
-              hsync-active:
-                enum: [0, 1]
-                default: 1
-
-              vsync-active:
-                enum: [0, 1]
-                default: 1
-
-            required:
-              - bus-type
-              - bus-width
-
-      port@1:
-        $ref: /schemas/graph.yaml#/properties/port
-        description: Input port for receiving CSI data.
-
-    required:
-      - port@0
-      - port@1
-
-required:
-  - compatible
-  - reg
-  - reg-names
-  - clocks
-  - clock-names
-  - resets
-  - reset-names
-  - power-domains
-  - interrupts
-  - ports
-
-additionalProperties: false
-
-examples:
-  - |
-    isp@19840000 {
-        compatible = "starfive,jh7110-camss";
-        reg = <0x19840000 0x10000>,
-              <0x19870000 0x30000>;
-        reg-names = "syscon", "isp";
-        clocks = <&ispcrg 0>,
-                 <&ispcrg 13>,
-                 <&ispcrg 2>,
-                 <&ispcrg 12>,
-                 <&ispcrg 1>,
-                 <&syscrg 51>,
-                 <&syscrg 52>;
-        clock-names = "apb_func",
-                      "wrapper_clk_c",
-                      "dvp_inv",
-                      "axiwr",
-                      "mipi_rx0_pxl",
-                      "ispcore_2x",
-                      "isp_axi";
-        resets = <&ispcrg 0>,
-                 <&ispcrg 1>,
-                 <&ispcrg 10>,
-                 <&ispcrg 11>,
-                 <&syscrg 41>,
-                 <&syscrg 42>;
-        reset-names = "wrapper_p",
-                      "wrapper_c",
-                      "axird",
-                      "axiwr",
-                      "isp_top_n",
-                      "isp_top_axi";
-        power-domains = <&pwrc 5>;
-        interrupts = <92>, <87>, <88>, <90>;
-
-        ports {
-            #address-cells = <1>;
-            #size-cells = <0>;
-            port@0 {
-                reg = <0>;
-                vin_from_sc2235: endpoint {
-                    remote-endpoint = <&sc2235_to_vin>;
-                    bus-type = <5>;
-                    bus-width = <8>;
-                    data-shift = <2>;
-                    hsync-active = <1>;
-                    vsync-active = <0>;
-                    pclk-sample = <1>;
-                };
-            };
-
-            port@1 {
-                reg = <1>;
-                vin_from_csi2rx: endpoint {
-                    remote-endpoint = <&csi2rx_to_vin>;
-                };
-            };
-        };
-    };
diff --git a/Documentation/devicetree/bindings/mfd/fsl,imx25-tsadc.yaml b/Documentation/devicetree/bindings/mfd/fsl,imx25-tsadc.yaml
new file mode 100644
index 0000000000000..b5c6a2d475011
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/fsl,imx25-tsadc.yaml
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/fsl,imx25-tsadc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Freescale MX25 ADC/TSC MultiFunction Device (MFD)
+
+maintainers:
+  - Frank Li <Frank.Li@nxp.com>
+
+description:
+  This device combines two general purpose conversion queues one used for general
+  ADC and the other used for touchscreens.
+
+properties:
+  compatible:
+    const: fsl,imx25-tsadc
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    items:
+      - const: ipg
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    const: 1
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
+  ranges: true
+
+patternProperties:
+  '^touchscreen@[0-9a-f]+$':
+    type: object
+    $ref: /schemas/input/touchscreen/fsl,imx25-tcq.yaml
+    unevaluatedProperties: false
+
+  '^adc@[0-9a-f]+$':
+    type: object
+    $ref: /schemas/iio/adc/fsl,imx25-gcq.yaml
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - '#interrupt-cells'
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    tscadc@50030000 {
+        compatible = "fsl,imx25-tsadc";
+        reg = <0x50030000 0xc>;
+        interrupts = <46>;
+        clocks = <&clks 119>;
+        clock-names = "ipg";
+        interrupt-controller;
+        #interrupt-cells = <1>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges;
+
+        touchscreen@50030400 {
+            compatible = "fsl,imx25-tcq";
+            reg = <0x50030400 0x60>;
+            interrupts = <0>;
+            fsl,wires = <4>;
+        };
+
+        adc@50030800 {
+            compatible = "fsl,imx25-gcq";
+            reg = <0x50030800 0x60>;
+            interrupts = <1>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/fsl-imx25-tsadc.txt b/Documentation/devicetree/bindings/mfd/fsl-imx25-tsadc.txt
deleted file mode 100644
index b035052869976..0000000000000
--- a/Documentation/devicetree/bindings/mfd/fsl-imx25-tsadc.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-Freescale MX25 ADC/TSC MultiFunction Device (MFD)
-
-This device combines two general purpose conversion queues one used for general
-ADC and the other used for touchscreens.
-
-Required properties:
- - compatible:			Should be "fsl,imx25-tsadc".
- - reg:				Start address and size of the memory area of
- 					the device
- - interrupts:			Interrupt for this device
-					(See: ../interrupt-controller/interrupts.txt)
- - clocks:			An 'ipg' clock (See: ../clock/clock-bindings.txt)
- - interrupt-controller:	This device is an interrupt controller. It
-   					controls the interrupts of both
-					conversion queues.
- - #interrupt-cells:		Should be '<1>'.
- - #address-cells:		Should be '<1>'.
- - #size-cells:			Should be '<1>'.
-
-This device includes two conversion queues which can be added as subnodes.
-The first queue is for the touchscreen, the second for general purpose ADC.
-
-Example:
-	tscadc: tscadc@50030000 {
-		compatible = "fsl,imx25-tsadc";
-		reg = <0x50030000 0xc>;
-		interrupts = <46>;
-		clocks = <&clks 119>;
-		clock-names = "ipg";
-		interrupt-controller;
-		#interrupt-cells = <1>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		tsc: tcq@50030400 {
-			compatible = "fsl,imx25-tcq";
-			reg = <0x50030400 0x60>;
-			...
-		};
-
-		adc: gcq@50030800 {
-			compatible = "fsl,imx25-gcq";
-			reg = <0x50030800 0x60>;
-			...
-		};
-	};
diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
index e5931d18d9984..644c42b5e2e57 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
@@ -135,6 +135,7 @@ patternProperties:
   "^adc@[0-9a-f]+$":
     type: object
     oneOf:
+      - $ref: /schemas/iio/adc/qcom,spmi-adc5-gen3.yaml#
       - $ref: /schemas/iio/adc/qcom,spmi-iadc.yaml#
       - $ref: /schemas/iio/adc/qcom,spmi-rradc.yaml#
       - $ref: /schemas/iio/adc/qcom,spmi-vadc.yaml#
diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
index 9f42097dfbace..b094542339e83 100644
--- a/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
@@ -4,19 +4,19 @@
 $id: http://devicetree.org/schemas/mfd/rohm,bd72720-pmic.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: ROHM BD72720 Power Management Integrated Circuit
+title: ROHM BD72720 and BD73900 Power Management Integrated Circuits
 
 maintainers:
   - Matti Vaittinen <mazziesaccount@gmail.com>
 
 description:
-  BD72720 is a single-chip power management IC for battery-powered portable
-  devices. The BD72720 integrates 10 bucks and 11 LDOs, and a 3000 mA
-  switching charger. The IC also includes a Coulomb counter, a real-time
-  clock (RTC), GPIOs and a 32.768 kHz clock gate.
+  BD72720 and BD73900 are single-chip power management ICs for
+  battery-powered portable devices. They integrate 10 bucks and 11 LDOs,
+  and a 3000 mA switching charger. ICs also include a Coulomb counter,
+  a real-time clock (RTC), GPIOs and a 32.768 kHz clock gate.
 
-# In addition to the properties found from the charger node, the ROHM BD72720
-# uses properties from a static battery node. Please see the:
+# In addition to the properties found from the charger node, PMICs
+# use properties from a static battery node. Please see the:
 # Documentation/devicetree/bindings/power/supply/battery.yaml
 #
 # Following properties are used
@@ -48,7 +48,12 @@ description:
 
 properties:
   compatible:
-    const: rohm,bd72720
+    oneOf:
+      - const: rohm,bd72720
+
+      - items:
+          - const: rohm,bd73900
+          - const: rohm,bd72720
 
   reg:
     description:
@@ -84,7 +89,7 @@ properties:
     minimum: 10000
     maximum: 50000
     description:
-      BD72720 has a SAR ADC for measuring charging currents. External sense
+      PMIC has a SAR ADC for measuring charging currents. External sense
       resistor (RSENSE in data sheet) should be used. If some other but
       30 mOhm resistor is used the resistance value should be given here in
       micro Ohms.
@@ -100,7 +105,7 @@ properties:
   rohm,pin-fault_b:
     $ref: /schemas/types.yaml#/definitions/string
     description:
-      BD72720 has an OTP option to use fault_b-pin for different
+      PMIC has an OTP option to use fault_b-pin for different
       purposes. Set this property accordingly. OTP options are
       OTP0 - bi-directional FAULT_B or READY indicator depending on a
       'sub option'
@@ -116,7 +121,7 @@ patternProperties:
   "^rohm,pin-dvs[0-1]$":
     $ref: /schemas/types.yaml#/definitions/string
     description:
-      BD72720 has 4 different OTP options to determine the use of dvs<X>-pins.
+      PMIC has 4 different OTP options to determine the use of dvs<X>-pins.
       OTP0 - regulator RUN state control.
       OTP1 - GPI.
       OTP2 - GPO.
@@ -130,7 +135,7 @@ patternProperties:
 
   "^rohm,pin-exten[0-1]$":
     $ref: /schemas/types.yaml#/definitions/string
-    description: BD72720 has an OTP option to use exten0-pin for different
+    description: PMIC has an OTP option to use exten0-pin for different
       purposes. Set this property accordingly.
       OTP0 - GPO
       OTP1 - Power sequencer output.
diff --git a/Documentation/devicetree/bindings/mfd/spacemit,p1.yaml b/Documentation/devicetree/bindings/mfd/spacemit,p1.yaml
index c6593ac6ef6ad..c67b1c6e4e4f9 100644
--- a/Documentation/devicetree/bindings/mfd/spacemit,p1.yaml
+++ b/Documentation/devicetree/bindings/mfd/spacemit,p1.yaml
@@ -27,8 +27,41 @@ properties:
   interrupts:
     maxItems: 1
 
-  vin-supply:
-    description: Input supply phandle.
+  vin1-supply:
+    description:
+      Power supply for BUCK1. Required if BUCK1 is defined.
+
+  vin2-supply:
+    description:
+      Power supply for BUCK2. Required if BUCK2 is defined.
+
+  vin3-supply:
+    description:
+      Power supply for BUCK3. Required if BUCK3 is defined.
+
+  vin4-supply:
+    description:
+      Power supply for BUCK4. Required if BUCK4 is defined.
+
+  vin5-supply:
+    description:
+      Power supply for BUCK5. Required if BUCK5 is defined.
+
+  vin6-supply:
+    description:
+      Power supply for BUCK6. Required if BUCK6 is defined.
+
+  aldoin-supply:
+    description:
+      Power supply for ALDO1-4. Required if any are defined.
+
+  dldoin1-supply:
+    description:
+      Power supply for DLDO1-4. Required if any are defined.
+
+  dldoin2-supply:
+    description:
+      Power supply for DLDO5-7. Required if any are defined.
 
   regulators:
     type: object
@@ -58,6 +91,10 @@ examples:
             compatible = "spacemit,p1";
             reg = <0x41>;
             interrupts = <64>;
+            vin1-supply = <&reg_vcc_5v>;
+            vin5-supply = <&reg_vcc_5v>;
+            aldoin-supply = <&reg_vcc_5v>;
+            dldoin1-supply = <&buck5>;
 
             regulators {
                 buck1 {
@@ -68,6 +105,14 @@ examples:
                     regulator-always-on;
                 };
 
+                buck5: buck5 {
+                    regulator-name = "buck5";
+                    regulator-min-microvolt = <500000>;
+                    regulator-max-microvolt = <3450000>;
+                    regulator-ramp-delay = <5000>;
+                    regulator-always-on;
+                };
+
                 aldo1 {
                     regulator-name = "aldo1";
                     regulator-min-microvolt = <500000>;
diff --git a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
index 8e79de97b242a..f343fb78e114c 100644
--- a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
@@ -106,6 +106,9 @@ properties:
         description:
           For this device it is strongly suggested to include
           arasan,soc-ctl-syscon.
+      - items:
+          - const: axiado,ax3000-sdhci-5.1-emmc  # Axiado AX3000 eMMC controller
+          - const: arasan,sdhci-5.1
 
   reg:
     maxItems: 1
@@ -121,6 +124,8 @@ properties:
       - const: clk_ahb
       - const: gate
 
+  dma-coherent: true
+
   interrupts:
     minItems: 1
     maxItems: 2
diff --git a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
index f90fd73904a24..8d62be4355a07 100644
--- a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
+++ b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
@@ -11,7 +11,7 @@ maintainers:
   - Ulf Hansson <ulf.hansson@linaro.org>
 
 description:
-  The ARM PrimeCells MMCI PL180 and PL181 provides an interface for
+  The ARM PrimeCell MMCI PL180 and PL181 provides an interface for
   reading and writing to MultiMedia and SD cards alike. Over the years
   vendors have use the VHDL code from ARM to create derivative MMC/SD/SDIO
   host controllers with very similar characteristics.
diff --git a/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
index 2f63f2cdeb71e..65bb2f66f8cfb 100644
--- a/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
@@ -26,9 +26,14 @@ properties:
   reg:
     minItems: 1
 
+  dma-coherent: true
+
   interrupts:
     maxItems: 1
 
+  iommus:
+    maxItems: 1
+
   clocks:
     maxItems: 1
     description:
diff --git a/Documentation/devicetree/bindings/mmc/bst,c1200-sdhci.yaml b/Documentation/devicetree/bindings/mmc/bst,c1200-sdhci.yaml
new file mode 100644
index 0000000000000..8358bb70c3335
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/bst,c1200-sdhci.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mmc/bst,c1200-sdhci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Black Sesame Technologies DWCMSHC SDHCI Controller
+
+maintainers:
+  - Ge Gordon <gordon.ge@bst.ai>
+
+allOf:
+  - $ref: sdhci-common.yaml#
+
+properties:
+  compatible:
+    const: bst,c1200-sdhci
+
+  reg:
+    items:
+      - description: Core SDHCI registers
+      - description: CRM registers
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    items:
+      - const: core
+
+  memory-region:
+    maxItems: 1
+
+  dma-coherent: true
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    bus {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        mmc@22200000 {
+            compatible = "bst,c1200-sdhci";
+            reg = <0x0 0x22200000 0x0 0x1000>,
+                  <0x0 0x23006000 0x0 0x1000>;
+            interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+            clocks = <&clk_mmc>;
+            clock-names = "core";
+            memory-region = <&mmc0_reserved>;
+            max-frequency = <200000000>;
+            bus-width = <8>;
+            non-removable;
+            dma-coherent;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
index ac75d694611ac..6c7317d13aa6c 100644
--- a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
@@ -134,8 +134,6 @@ allOf:
           items:
             - description: Host controller registers
             - description: Elba byte-lane enable register for writes
-      required:
-        - resets
     else:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml
index b98a84f932772..014b049baeb64 100644
--- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml
+++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml
@@ -35,6 +35,7 @@ properties:
           - fsl,imx8mm-usdhc
           - fsl,imxrt1050-usdhc
           - nxp,s32g2-usdhc
+          - nxp,s32n79-usdhc
       - items:
           - const: fsl,imx50-esdhc
           - const: fsl,imx53-esdhc
diff --git a/Documentation/devicetree/bindings/mmc/loongson,ls2k0500-mmc.yaml b/Documentation/devicetree/bindings/mmc/loongson,ls2k0500-mmc.yaml
index c142421bc7238..b3e8d3f135920 100644
--- a/Documentation/devicetree/bindings/mmc/loongson,ls2k0500-mmc.yaml
+++ b/Documentation/devicetree/bindings/mmc/loongson,ls2k0500-mmc.yaml
@@ -22,6 +22,7 @@ allOf:
 properties:
   compatible:
     enum:
+      - loongson,ls2k0300-mmc
       - loongson,ls2k0500-mmc
       - loongson,ls2k1000-mmc
       - loongson,ls2k2000-mmc
diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
index 6dd26ad314916..eb3755bdfdf7e 100644
--- a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
+++ b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
@@ -25,6 +25,7 @@ properties:
           - mediatek,mt8135-mmc
           - mediatek,mt8173-mmc
           - mediatek,mt8183-mmc
+          - mediatek,mt8189-mmc
           - mediatek,mt8196-mmc
           - mediatek,mt8516-mmc
       - items:
@@ -192,6 +193,7 @@ allOf:
             - mediatek,mt8183-mmc
             - mediatek,mt8186-mmc
             - mediatek,mt8188-mmc
+            - mediatek,mt8189-mmc
             - mediatek,mt8195-mmc
             - mediatek,mt8196-mmc
             - mediatek,mt8516-mmc
@@ -240,6 +242,7 @@ allOf:
               - mediatek,mt7986-mmc
               - mediatek,mt7988-mmc
               - mediatek,mt8183-mmc
+              - mediatek,mt8189-mmc
               - mediatek,mt8196-mmc
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
index c754ea71f51f7..64fac0d11329a 100644
--- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
+++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
@@ -106,6 +106,11 @@ properties:
   iommus:
     maxItems: 1
 
+  mux-states:
+    description:
+      mux controller node to route the SD/SDIO/eMMC signals from SoC to cards.
+    maxItems: 1
+
   power-domains:
     maxItems: 1
 
@@ -275,6 +280,7 @@ examples:
         max-frequency = <195000000>;
         power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
         resets = <&cpg 314>;
+        mux-states = <&mux 0>;
     };
 
     sdhi1: mmc@ee120000 {
diff --git a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
index acb9fb9a92cd7..a75209bd2710c 100644
--- a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
@@ -43,6 +43,7 @@ properties:
               - rockchip,rk3562-dw-mshc
               - rockchip,rk3568-dw-mshc
               - rockchip,rk3588-dw-mshc
+              - rockchip,rv1103b-dw-mshc
               - rockchip,rv1108-dw-mshc
               - rockchip,rv1126-dw-mshc
           - const: rockchip,rk3288-dw-mshc
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
index 938be8228d668..fd1d5b04e7556 100644
--- a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
+++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
@@ -38,6 +38,7 @@ properties:
       - items:
           - enum:
               - qcom,ipq5018-sdhci
+              - qcom,ipq5210-sdhci
               - qcom,ipq5332-sdhci
               - qcom,ipq5424-sdhci
               - qcom,ipq6018-sdhci
diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
index 7e7c55dc24403..5cebe5eb1efb8 100644
--- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
@@ -50,9 +50,11 @@ properties:
     maxItems: 1
 
   resets:
+    minItems: 4
     maxItems: 5
 
   reset-names:
+    minItems: 4
     maxItems: 5
 
   rockchip,txclk-tapnum:
@@ -146,6 +148,7 @@ allOf:
     else:
       properties:
         resets:
+          minItems: 5
           maxItems: 5
         reset-names:
           items:
diff --git a/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml b/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
index 13d9382058fbc..9a055d963a7f0 100644
--- a/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
@@ -14,7 +14,9 @@ allOf:
 
 properties:
   compatible:
-    const: spacemit,k1-sdhci
+    enum:
+      - spacemit,k1-sdhci
+      - spacemit,k3-sdhci
 
   reg:
     maxItems: 1
@@ -32,6 +34,16 @@ properties:
       - const: core
       - const: io
 
+  resets:
+    items:
+      - description: axi reset, connect to AXI bus, shared by all controllers
+      - description: sdh reset, connect to individual controller separately
+
+  reset-names:
+    items:
+      - const: axi
+      - const: sdh
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/mtd/mxc-nand.yaml b/Documentation/devicetree/bindings/mtd/mxc-nand.yaml
index bd8f7b6839535..433ae5727ad85 100644
--- a/Documentation/devicetree/bindings/mtd/mxc-nand.yaml
+++ b/Documentation/devicetree/bindings/mtd/mxc-nand.yaml
@@ -15,7 +15,9 @@ allOf:
 properties:
   compatible:
     oneOf:
-      - const: fsl,imx27-nand
+      - enum:
+          - fsl,imx25-nand
+          - fsl,imx27-nand
       - items:
           - enum:
               - fsl,imx31-nand
@@ -26,6 +28,9 @@ properties:
   interrupts:
     maxItems: 1
 
+  clocks:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/mtd/partitions/partition.yaml b/Documentation/devicetree/bindings/mtd/partitions/partition.yaml
index 2397d97ecac53..eaeac2f2ea947 100644
--- a/Documentation/devicetree/bindings/mtd/partitions/partition.yaml
+++ b/Documentation/devicetree/bindings/mtd/partitions/partition.yaml
@@ -57,6 +57,15 @@ properties:
       user space from
     type: boolean
 
+  part-concat-next:
+    description: List of phandles to MTD partitions that need be concatenated
+      with the current partition.
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    minItems: 1
+    maxItems: 16
+    items:
+      maxItems: 1
+
   align:
     $ref: /schemas/types.yaml#/definitions/uint32
     minimum: 2
@@ -180,4 +189,15 @@ examples:
             reg = <0x200000 0x100000>;
             align = <0x4000>;
         };
+
+        part0: partition@400000 {
+            part-concat-next = <&part1>;
+            label = "part0_0";
+            reg = <0x400000 0x100000>;
+        };
+
+        part1: partition@800000 {
+            label = "part0_1";
+            reg = <0x800000 0x800000>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/mtd/st,spear600-smi.yaml b/Documentation/devicetree/bindings/mtd/st,spear600-smi.yaml
index 8fe27aae75279..e7385d906591b 100644
--- a/Documentation/devicetree/bindings/mtd/st,spear600-smi.yaml
+++ b/Documentation/devicetree/bindings/mtd/st,spear600-smi.yaml
@@ -19,9 +19,6 @@ description:
   Flash sub nodes describe the memory range and optional per-flash
   properties.
 
-allOf:
-  - $ref: mtd.yaml#
-
 properties:
   compatible:
     const: st,spear600-smi
@@ -42,14 +39,29 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: Functional clock rate of the SMI controller in Hz.
 
-  st,smi-fast-mode:
-    type: boolean
-    description: Indicates that the attached flash supports fast read mode.
+patternProperties:
+  "^flash@.*$":
+    $ref: /schemas/mtd/mtd.yaml#
+
+    properties:
+      reg:
+        maxItems: 1
+
+      st,smi-fast-mode:
+        type: boolean
+        description: Indicates that the attached flash supports fast read mode.
+
+    unevaluatedProperties: false
+
+    required:
+      - reg
 
 required:
   - compatible
   - reg
   - clock-rate
+  - "#address-cells"
+  - "#size-cells"
 
 unevaluatedProperties: false
 
@@ -64,7 +76,7 @@ examples:
         interrupts = <12>;
         clock-rate = <50000000>;  /* 50 MHz */
 
-        flash@f8000000 {
+        flash@fc000000 {
             reg = <0xfc000000 0x1000>;
             st,smi-fast-mode;
         };
diff --git a/Documentation/devicetree/bindings/mux/mux-controller.yaml b/Documentation/devicetree/bindings/mux/mux-controller.yaml
index 78340bbe4df62..6defb9da10f7e 100644
--- a/Documentation/devicetree/bindings/mux/mux-controller.yaml
+++ b/Documentation/devicetree/bindings/mux/mux-controller.yaml
@@ -63,18 +63,12 @@ description: |
 
 select:
   anyOf:
-    - properties:
-        $nodename:
-          pattern: '^mux-controller'
     - required:
         - '#mux-control-cells'
     - required:
         - '#mux-state-cells'
 
 properties:
-  $nodename:
-    pattern: '^mux-controller(@.*|-([0-9]|[1-9][0-9]+))?$'
-
   '#mux-control-cells':
     enum: [ 0, 1 ]
 
diff --git a/Documentation/devicetree/bindings/net/cdns,macb.yaml b/Documentation/devicetree/bindings/net/cdns,macb.yaml
index cb14c35ba9969..feb1683858374 100644
--- a/Documentation/devicetree/bindings/net/cdns,macb.yaml
+++ b/Documentation/devicetree/bindings/net/cdns,macb.yaml
@@ -70,6 +70,14 @@ properties:
               - microchip,sama7d65-gem   # Microchip SAMA7D65 gigabit ethernet interface
           - const: microchip,sama7g5-gem # Microchip SAMA7G5 gigabit ethernet interface
 
+      - items:
+          - const: microchip,pic64hpsc-gem # Microchip PIC64-HPSC
+          - const: cdns,gem
+      - items:
+          - const: microchip,pic64hx-gem   # Microchip PIC64HX
+          - const: microchip,pic64hpsc-gem # Microchip PIC64-HPSC
+          - const: cdns,gem
+
   reg:
     minItems: 1
     items:
@@ -196,6 +204,17 @@ allOf:
       required:
         - phys
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: microchip,pic64hpsc-gem
+    then:
+      patternProperties:
+        "^ethernet-phy@[0-9a-f]$": false
+      properties:
+        mdio: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/net/dsa/maxlinear,mxl862xx.yaml b/Documentation/devicetree/bindings/net/dsa/maxlinear,mxl862xx.yaml
index f1d667f7a055d..2f19c19c60f3b 100644
--- a/Documentation/devicetree/bindings/net/dsa/maxlinear,mxl862xx.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/maxlinear,mxl862xx.yaml
@@ -110,7 +110,6 @@ examples:
 
                 port@9 {
                     reg = <9>;
-                    label = "cpu";
                     ethernet = <&gmac0>;
                     phy-mode = "usxgmii";
 
diff --git a/Documentation/devicetree/bindings/net/micrel.yaml b/Documentation/devicetree/bindings/net/micrel.yaml
index ecc00169ef805..6fa568057b927 100644
--- a/Documentation/devicetree/bindings/net/micrel.yaml
+++ b/Documentation/devicetree/bindings/net/micrel.yaml
@@ -51,9 +51,10 @@ properties:
       bits that are currently supported:
 
       KSZ8001: register 0x1e, bits 15..14
-      KSZ8041: register 0x1e, bits 15..14
       KSZ8021: register 0x1f, bits 5..4
       KSZ8031: register 0x1f, bits 5..4
+      KSZ8041: register 0x1e, bits 15..14
+      KSZ8041RNLI: register 0x1e, bits 15..14
       KSZ8051: register 0x1f, bits 5..4
       KSZ8081: register 0x1f, bits 5..4
       KSZ8091: register 0x1f, bits 5..4
@@ -80,9 +81,10 @@ allOf:
             contains:
               enum:
                 - ethernet-phy-id0022.1510
+                - ethernet-phy-id0022.1537
+                - ethernet-phy-id0022.1550
                 - ethernet-phy-id0022.1555
                 - ethernet-phy-id0022.1556
-                - ethernet-phy-id0022.1550
                 - ethernet-phy-id0022.1560
                 - ethernet-phy-id0022.161a
     then:
diff --git a/Documentation/devicetree/bindings/net/nxp,s32-dwmac.yaml b/Documentation/devicetree/bindings/net/nxp,s32-dwmac.yaml
index 1b2934f3c87ca..753a04941659b 100644
--- a/Documentation/devicetree/bindings/net/nxp,s32-dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/nxp,s32-dwmac.yaml
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright 2021-2024 NXP
+# Copyright 2021-2026 NXP
 %YAML 1.2
 ---
 $id: http://devicetree.org/schemas/net/nxp,s32-dwmac.yaml#
@@ -16,6 +16,8 @@ description:
   the SoC S32R45 has two instances. The devices can use RGMII/RMII/MII
   interface over Pinctrl device or the output can be routed
   to the embedded SerDes for SGMII connectivity.
+  The DWMAC instances have connected all RX/TX queues interrupts,
+  enabling load balancing of data traffic across all CPU cores.
 
 properties:
   compatible:
@@ -45,10 +47,25 @@ properties:
       FlexTimer Modules connect to GMAC_0.
 
   interrupts:
-    maxItems: 1
+    minItems: 1
+    maxItems: 11
 
   interrupt-names:
-    const: macirq
+    oneOf:
+      - items:
+          - const: macirq
+      - items:
+          - const: macirq
+          - const: tx-queue-0
+          - const: rx-queue-0
+          - const: tx-queue-1
+          - const: rx-queue-1
+          - const: tx-queue-2
+          - const: rx-queue-2
+          - const: tx-queue-3
+          - const: rx-queue-3
+          - const: tx-queue-4
+          - const: rx-queue-4
 
   clocks:
     items:
@@ -88,8 +105,28 @@ examples:
               <0x0 0x4007c004 0x0 0x4>;    /* GMAC_0_CTRL_STS */
         nxp,phy-sel = <&gpr 0x4>;
         interrupt-parent = <&gic>;
-        interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
-        interrupt-names = "macirq";
+        interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+                     /* CHN 0: tx, rx */
+                     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
+                     /* CHN 1: tx, rx */
+                     <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>,
+                     /* CHN 2: tx, rx */
+                     <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>,
+                     /* CHN 3: tx, rx */
+                     <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>,
+                     /* CHN 4: tx, rx */
+                     <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "macirq",
+                          "tx-queue-0", "rx-queue-0",
+                          "tx-queue-1", "rx-queue-1",
+                          "tx-queue-2", "rx-queue-2",
+                          "tx-queue-3", "rx-queue-3",
+                          "tx-queue-4", "rx-queue-4";
         snps,mtl-rx-config = <&mtl_rx_setup>;
         snps,mtl-tx-config = <&mtl_tx_setup>;
         clocks = <&clks 24>, <&clks 17>, <&clks 16>, <&clks 15>;
diff --git a/Documentation/devicetree/bindings/net/qcom,ipa.yaml b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
index c7f5f2ef74521..e4bb627e17570 100644
--- a/Documentation/devicetree/bindings/net/qcom,ipa.yaml
+++ b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
@@ -53,6 +53,10 @@ properties:
           - qcom,sm6350-ipa
           - qcom,sm8350-ipa
           - qcom,sm8550-ipa
+      - items:
+          - enum:
+              - qcom,qcm2290-ipa
+          - const: qcom,sc7180-ipa
       - items:
           - enum:
               - qcom,sm8650-ipa
@@ -165,6 +169,13 @@ properties:
       initializing IPA hardware.  Optional, and only used when
       Trust Zone performs early initialization.
 
+  sram:
+    maxItems: 1
+    description:
+      A reference to an additional region residing in IMEM (special
+      on-chip SRAM), which is accessed by the IPA firmware and needs
+      to be IOMMU-mapped from the OS.
+
 required:
   - compatible
   - iommus
diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
index 38bc34dc4f09b..98ebb6276bc6b 100644
--- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
@@ -109,6 +109,7 @@ properties:
         - snps,dwmac-5.10a
         - snps,dwmac-5.20
         - snps,dwmac-5.30a
+        - snps,dwmac-5.40a
         - snps,dwxgmac
         - snps,dwxgmac-2.10
         - sophgo,sg2042-dwmac
@@ -656,6 +657,7 @@ allOf:
                 - snps,dwmac-5.10a
                 - snps,dwmac-5.20
                 - snps,dwmac-5.30a
+                - snps,dwmac-5.40a
                 - snps,dwxgmac
                 - snps,dwxgmac-2.10
                 - st,spear600-gmac
diff --git a/Documentation/devicetree/bindings/net/spacemit,k3-dwmac.yaml b/Documentation/devicetree/bindings/net/spacemit,k3-dwmac.yaml
new file mode 100644
index 0000000000000..678eccf044f97
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/spacemit,k3-dwmac.yaml
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/spacemit,k3-dwmac.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Spacemit K3 DWMAC glue layer
+
+maintainers:
+  - Inochi Amaoto <inochiama@gmail.com>
+
+select:
+  properties:
+    compatible:
+      contains:
+        const: spacemit,k3-dwmac
+  required:
+    - compatible
+
+properties:
+  compatible:
+    items:
+      - const: spacemit,k3-dwmac
+      - const: snps,dwmac-5.40a
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: GMAC application clock
+      - description: PTP clock
+      - description: TX clock
+
+  clock-names:
+    items:
+      - const: stmmaceth
+      - const: ptp_ref
+      - const: tx
+
+  interrupts:
+    minItems: 1
+    items:
+      - description: MAC interrupt
+      - description: MAC wake interrupt
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - const: macirq
+      - const: eth_wake_irq
+
+  resets:
+    maxItems: 1
+
+  reset-names:
+    const: stmmaceth
+
+  spacemit,apmu:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: phandle to the syscon node which control the glue register
+          - description: offset of the control register
+          - description: offset of the dline register
+    description:
+      A phandle to syscon with offset to control registers for this MAC
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - interrupts
+  - interrupt-names
+  - resets
+  - reset-names
+  - spacemit,apmu
+
+allOf:
+  - $ref: snps,dwmac.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    ethernet@cac80000 {
+      compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a";
+      reg = <0xcac80000 0x2000>;
+      clocks = <&syscon_apmu 66>, <&syscon_apmu 68>,
+               <&syscon_apmu 69>;
+      clock-names = "stmmaceth", "ptp_ref", "tx";
+      interrupts = <131 IRQ_TYPE_LEVEL_HIGH>, <276 IRQ_TYPE_LEVEL_HIGH>;
+      interrupt-names = "macirq", "eth_wake_irq";
+      phy-mode = "rgmii-id";
+      phy-handle = <&phy0>;
+      resets = <&syscon_apmu 67>;
+      reset-names = "stmmaceth";
+      spacemit,apmu = <&syscon_apmu 0x384 0x38c>;
+    };
diff --git a/Documentation/devicetree/bindings/nvmem/layouts/kontron,sl28-vpd.yaml b/Documentation/devicetree/bindings/nvmem/layouts/kontron,sl28-vpd.yaml
index afd1919c6b1c5..c713e23819f11 100644
--- a/Documentation/devicetree/bindings/nvmem/layouts/kontron,sl28-vpd.yaml
+++ b/Documentation/devicetree/bindings/nvmem/layouts/kontron,sl28-vpd.yaml
@@ -19,12 +19,7 @@ select: false
 
 properties:
   compatible:
-    oneOf:
-      - items:
-          - enum:
-              - kontron,sa67-vpd
-          - const: kontron,sl28-vpd
-      - const: kontron,sl28-vpd
+    const: kontron,sl28-vpd
 
   serial-number:
     type: object
diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
index 839513d4b499d..2ab047f2bb69d 100644
--- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
+++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
@@ -26,6 +26,7 @@ properties:
           - qcom,ipq8064-qfprom
           - qcom,ipq8074-qfprom
           - qcom,ipq9574-qfprom
+          - qcom,kaanapali-qfprom
           - qcom,msm8226-qfprom
           - qcom,msm8916-qfprom
           - qcom,msm8917-qfprom
diff --git a/Documentation/devicetree/bindings/nvmem/rockchip,otp.yaml b/Documentation/devicetree/bindings/nvmem/rockchip,otp.yaml
index dc89020b09506..7e4d5e1c4cedc 100644
--- a/Documentation/devicetree/bindings/nvmem/rockchip,otp.yaml
+++ b/Documentation/devicetree/bindings/nvmem/rockchip,otp.yaml
@@ -14,6 +14,9 @@ properties:
     enum:
       - rockchip,px30-otp
       - rockchip,rk3308-otp
+      - rockchip,rk3528-otp
+      - rockchip,rk3562-otp
+      - rockchip,rk3568-otp
       - rockchip,rk3576-otp
       - rockchip,rk3588-otp
 
@@ -26,19 +29,15 @@ properties:
 
   clock-names:
     minItems: 3
-    items:
-      - const: otp
-      - const: apb_pclk
-      - const: phy
-      - const: arb
+    maxItems: 4
 
   resets:
     minItems: 1
-    maxItems: 3
+    maxItems: 4
 
   reset-names:
     minItems: 1
-    maxItems: 3
+    maxItems: 4
 
 required:
   - compatible
@@ -64,7 +63,10 @@ allOf:
         clocks:
           maxItems: 3
         clock-names:
-          maxItems: 3
+          items:
+            - const: otp
+            - const: apb_pclk
+            - const: phy
         resets:
           maxItems: 1
         reset-names:
@@ -76,13 +78,68 @@ allOf:
         compatible:
           contains:
             enum:
-              - rockchip,rk3576-otp
+              - rockchip,rk3528-otp
     then:
       properties:
         clocks:
           maxItems: 3
         clock-names:
+          items:
+            - const: otp
+            - const: apb_pclk
+            - const: sbpi
+        resets:
+          minItems: 3
+          maxItems: 3
+        reset-names:
+          items:
+            - const: otp
+            - const: apb
+            - const: sbpi
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3562-otp
+              - rockchip,rk3568-otp
+    then:
+      properties:
+        clocks:
+          minItems: 4
+          maxItems: 4
+        clock-names:
+          items:
+            - const: otp
+            - const: apb_pclk
+            - const: phy
+            - const: sbpi
+        resets:
+          minItems: 4
+          maxItems: 4
+        reset-names:
+          items:
+            - const: otp
+            - const: apb
+            - const: phy
+            - const: sbpi
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3576-otp
+    then:
+      properties:
+        clocks:
           maxItems: 3
+        clock-names:
+          items:
+            - const: otp
+            - const: apb_pclk
+            - const: phy
         resets:
           minItems: 2
           maxItems: 2
@@ -101,10 +158,16 @@ allOf:
       properties:
         clocks:
           minItems: 4
+          maxItems: 4
         clock-names:
-          minItems: 4
+          items:
+            - const: otp
+            - const: apb_pclk
+            - const: phy
+            - const: arb
         resets:
           minItems: 3
+          maxItems: 3
         reset-names:
           items:
             - const: otp
diff --git a/Documentation/devicetree/bindings/pci/andestech,qilai-pcie.yaml b/Documentation/devicetree/bindings/pci/andestech,qilai-pcie.yaml
new file mode 100644
index 0000000000000..97ba97fdc5a9d
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/andestech,qilai-pcie.yaml
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pci/andestech,qilai-pcie.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Andes QiLai PCIe host controller
+
+description:
+  Andes QiLai PCIe host controller is based on the Synopsys DesignWare
+  PCI core.
+
+maintainers:
+  - Randolph Lin <randolph@andestech.com>
+
+allOf:
+  - $ref: /schemas/pci/snps,dw-pcie.yaml#
+
+properties:
+  compatible:
+    const: andestech,qilai-pcie
+
+  reg:
+    items:
+      - description: Data Bus Interface (DBI) registers.
+      - description: APB registers.
+      - description: PCIe configuration space region.
+
+  reg-names:
+    items:
+      - const: dbi
+      - const: apb
+      - const: config
+
+  dma-coherent: true
+
+  ranges:
+    maxItems: 2
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-names:
+    items:
+      - const: msi
+
+required:
+  - reg
+  - reg-names
+  - interrupts
+  - interrupt-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      pcie@80000000 {
+        compatible = "andestech,qilai-pcie";
+        device_type = "pci";
+        reg = <0x0 0x80000000 0x0 0x20000000>,
+              <0x0 0x04000000 0x0 0x00001000>,
+              <0x0 0x00000000 0x0 0x00010000>;
+        reg-names = "dbi", "apb", "config";
+        dma-coherent;
+
+        linux,pci-domain = <0>;
+        #address-cells = <3>;
+        #size-cells = <2>;
+        ranges = <0x02000000 0x00 0x10000000 0x00 0x10000000 0x00 0xf0000000>,
+                 <0x43000000 0x01 0x00000000 0x01 0x00000000 0x02 0x00000000>;
+
+        #interrupt-cells = <1>;
+        interrupts = <0xf>;
+        interrupt-names = "msi";
+        interrupt-parent = <&plic0>;
+        interrupt-map-mask = <0 0 0 0>;
+        interrupt-map = <0 0 0 1 &plic0 0xf IRQ_TYPE_LEVEL_HIGH>,
+                        <0 0 0 2 &plic0 0xf IRQ_TYPE_LEVEL_HIGH>,
+                        <0 0 0 3 &plic0 0xf IRQ_TYPE_LEVEL_HIGH>,
+                        <0 0 0 4 &plic0 0xf IRQ_TYPE_LEVEL_HIGH>;
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pci/baikal,bt1-pcie.yaml b/Documentation/devicetree/bindings/pci/baikal,bt1-pcie.yaml
deleted file mode 100644
index 8eaa07ae97743..0000000000000
--- a/Documentation/devicetree/bindings/pci/baikal,bt1-pcie.yaml
+++ /dev/null
@@ -1,168 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/pci/baikal,bt1-pcie.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Baikal-T1 PCIe Root Port Controller
-
-maintainers:
-  - Serge Semin <fancer.lancer@gmail.com>
-
-description:
-  Embedded into Baikal-T1 SoC Root Complex controller with a single port
-  activated. It's based on the DWC RC PCIe v4.60a IP-core, which is configured
-  to have just a single Root Port function and is capable of establishing the
-  link up to Gen.3 speed on x4 lanes. It doesn't have embedded clock and reset
-  control module, so the proper interface initialization is supposed to be
-  performed by software. There four in- and four outbound iATU regions
-  which can be used to emit all required TLP types on the PCIe bus.
-
-allOf:
-  - $ref: /schemas/pci/snps,dw-pcie.yaml#
-
-properties:
-  compatible:
-    const: baikal,bt1-pcie
-
-  reg:
-    description:
-      DBI, DBI2 and at least 4KB outbound iATU-capable region for the
-      peripheral devices CFG-space access.
-    maxItems: 3
-
-  reg-names:
-    items:
-      - const: dbi
-      - const: dbi2
-      - const: config
-
-  interrupts:
-    description:
-      MSI, AER, PME, Hot-plug, Link Bandwidth Management, Link Equalization
-      request and eight Read/Write eDMA IRQ lines are available.
-    maxItems: 14
-
-  interrupt-names:
-    items:
-      - const: dma0
-      - const: dma1
-      - const: dma2
-      - const: dma3
-      - const: dma4
-      - const: dma5
-      - const: dma6
-      - const: dma7
-      - const: msi
-      - const: aer
-      - const: pme
-      - const: hp
-      - const: bw_mg
-      - const: l_eq
-
-  clocks:
-    description:
-      DBI (attached to the APB bus), AXI-bus master and slave interfaces
-      are fed up by the dedicated application clocks. A common reference
-      clock signal is supposed to be attached to the corresponding Ref-pad
-      of the SoC. It will be redistributed amongst the controller core
-      sub-modules (pipe, core, aux, etc).
-    maxItems: 4
-
-  clock-names:
-    items:
-      - const: dbi
-      - const: mstr
-      - const: slv
-      - const: ref
-
-  resets:
-    description:
-      A comprehensive controller reset logic is supposed to be implemented
-      by software, so almost all the possible application and core reset
-      signals are exposed via the system CCU module.
-    maxItems: 9
-
-  reset-names:
-    items:
-      - const: mstr
-      - const: slv
-      - const: pwr
-      - const: hot
-      - const: phy
-      - const: core
-      - const: pipe
-      - const: sticky
-      - const: non-sticky
-
-  baikal,bt1-syscon:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description:
-      Phandle to the Baikal-T1 System Controller DT node. It's required to
-      access some additional PM, Reset-related and LTSSM signals.
-
-  num-lanes:
-    maximum: 4
-
-  max-link-speed:
-    maximum: 3
-
-required:
-  - compatible
-  - reg
-  - reg-names
-  - interrupts
-  - interrupt-names
-
-unevaluatedProperties: false
-
-examples:
-  - |
-    #include <dt-bindings/interrupt-controller/mips-gic.h>
-    #include <dt-bindings/gpio/gpio.h>
-
-    pcie@1f052000 {
-      compatible = "baikal,bt1-pcie";
-      device_type = "pci";
-      reg = <0x1f052000 0x1000>, <0x1f053000 0x1000>, <0x1bdbf000 0x1000>;
-      reg-names = "dbi", "dbi2", "config";
-      #address-cells = <3>;
-      #size-cells = <2>;
-      ranges = <0x81000000 0 0x00000000 0x1bdb0000 0 0x00008000>,
-               <0x82000000 0 0x20000000 0x08000000 0 0x13db0000>;
-      bus-range = <0x0 0xff>;
-
-      interrupts = <GIC_SHARED 80 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 81 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 82 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 83 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 84 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 85 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 86 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 87 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 88 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 89 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 90 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 91 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 92 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SHARED 93 IRQ_TYPE_LEVEL_HIGH>;
-      interrupt-names = "dma0", "dma1", "dma2", "dma3",
-                        "dma4", "dma5", "dma6", "dma7",
-                        "msi", "aer", "pme", "hp", "bw_mg",
-                        "l_eq";
-
-      clocks = <&ccu_sys 1>, <&ccu_axi 6>, <&ccu_axi 7>, <&clk_pcie>;
-      clock-names = "dbi", "mstr", "slv", "ref";
-
-      resets = <&ccu_axi 6>, <&ccu_axi 7>, <&ccu_sys 7>, <&ccu_sys 10>,
-               <&ccu_sys 4>, <&ccu_sys 6>, <&ccu_sys 5>, <&ccu_sys 8>,
-               <&ccu_sys 9>;
-      reset-names = "mstr", "slv", "pwr", "hot", "phy", "core", "pipe",
-                    "sticky", "non-sticky";
-
-      reset-gpios = <&port0 0 GPIO_ACTIVE_LOW>;
-
-      num-lanes = <4>;
-      max-link-speed = <3>;
-    };
-...
diff --git a/Documentation/devicetree/bindings/pci/eswin,pcie.yaml b/Documentation/devicetree/bindings/pci/eswin,pcie.yaml
new file mode 100644
index 0000000000000..057e1f363dde8
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/eswin,pcie.yaml
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pci/eswin,pcie.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ESWIN PCIe Root Complex
+
+maintainers:
+  - Yu Ning <ningyu@eswincomputing.com>
+  - Senchuan Zhang <zhangsenchuan@eswincomputing.com>
+  - Yanghui Ou <ouyanghui@eswincomputing.com>
+
+description:
+  ESWIN SoCs PCIe Root Complex is based on the Synopsys DesignWare PCIe IP.
+
+properties:
+  compatible:
+    const: eswin,eic7700-pcie
+
+  reg:
+    maxItems: 3
+
+  reg-names:
+    items:
+      - const: dbi
+      - const: config
+      - const: elbi
+
+  ranges:
+    maxItems: 3
+
+  '#interrupt-cells':
+    const: 1
+
+  interrupt-names:
+    items:
+      - const: msi
+      - const: inta
+      - const: intb
+      - const: intc
+      - const: intd
+
+  interrupt-map:
+    maxItems: 4
+
+  interrupt-map-mask:
+    items:
+      - const: 0
+      - const: 0
+      - const: 0
+      - const: 7
+
+  clocks:
+    maxItems: 4
+
+  clock-names:
+    items:
+      - const: mstr
+      - const: dbi
+      - const: phy_reg
+      - const: aux
+
+  resets:
+    maxItems: 2
+
+  reset-names:
+    items:
+      - const: dbi
+      - const: pwr
+
+patternProperties:
+  "^pcie@":
+    type: object
+    $ref: /schemas/pci/pci-pci-bridge.yaml#
+
+    properties:
+      reg:
+        maxItems: 1
+
+      num-lanes:
+        maximum: 4
+
+      resets:
+        maxItems: 1
+
+      reset-names:
+        items:
+          - const: perst
+
+    required:
+      - reg
+      - ranges
+      - num-lanes
+      - resets
+      - reset-names
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - ranges
+  - interrupts
+  - interrupt-names
+  - interrupt-map-mask
+  - interrupt-map
+  - '#interrupt-cells'
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+
+allOf:
+  - $ref: /schemas/pci/snps,dw-pcie.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        pcie@54000000 {
+            compatible = "eswin,eic7700-pcie";
+            reg = <0x0 0x54000000 0x0 0x4000000>,
+                  <0x0 0x40000000 0x0 0x800000>,
+                  <0x0 0x50000000 0x0 0x100000>;
+            reg-names = "dbi", "config", "elbi";
+            #address-cells = <3>;
+            #size-cells = <2>;
+            #interrupt-cells = <1>;
+            ranges = <0x01000000 0x0 0x40800000 0x0 0x40800000 0x0 0x800000>,
+                     <0x02000000 0x0 0x41000000 0x0 0x41000000 0x0 0xf000000>,
+                     <0x43000000 0x80 0x00000000 0x80 0x00000000 0x2 0x00000000>;
+            bus-range = <0x00 0xff>;
+            clocks = <&clock 144>,
+                     <&clock 145>,
+                     <&clock 146>,
+                     <&clock 147>;
+            clock-names = "mstr", "dbi", "phy_reg", "aux";
+            resets = <&reset 97>,
+                     <&reset 98>;
+            reset-names = "dbi", "pwr";
+            interrupts = <220>, <179>, <180>, <181>, <182>, <183>, <184>, <185>, <186>;
+            interrupt-names = "msi", "inta", "intb", "intc", "intd";
+            interrupt-parent = <&plic>;
+            interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+            interrupt-map = <0x0 0x0 0x0 0x1 &plic 179>,
+                            <0x0 0x0 0x0 0x2 &plic 180>,
+                            <0x0 0x0 0x0 0x3 &plic 181>,
+                            <0x0 0x0 0x0 0x4 &plic 182>;
+            device_type = "pci";
+            pcie@0 {
+                reg = <0x0 0x0 0x0 0x0 0x0>;
+                #address-cells = <3>;
+                #size-cells = <2>;
+                ranges;
+                device_type = "pci";
+                num-lanes = <4>;
+                resets = <&reset 99>;
+                reset-names = "perst";
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
index d668782546a23..a67108c48feb8 100644
--- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
@@ -10,17 +10,21 @@ maintainers:
   - Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>
 
 description:
-  Renesas RZ/G3S PCIe host controller complies with PCIe Base Specification
-  4.0 and supports up to 5 GT/s (Gen2).
+  Renesas RZ/G3{E,S} PCIe host controllers comply with PCIe
+  Base Specification 4.0 and support up to 5 GT/s (Gen2) for RZ/G3S and
+  up to 8 GT/s (Gen3) for RZ/G3E.
 
 properties:
   compatible:
-    const: renesas,r9a08g045-pcie # RZ/G3S
+    enum:
+      - renesas,r9a08g045-pcie # RZ/G3S
+      - renesas,r9a09g047-pcie # RZ/G3E
 
   reg:
     maxItems: 1
 
   interrupts:
+    minItems: 16
     items:
       - description: System error interrupt
       - description: System error on correctable error interrupt
@@ -38,39 +42,55 @@ properties:
       - description: PCIe event interrupt
       - description: Message interrupt
       - description: All interrupts
+      - description: Link equalization request interrupt
+      - description: Turn off event interrupt
+      - description: PMU power off interrupt
+      - description: D3 event function 0 interrupt
+      - description: D3 event function 1 interrupt
+      - description: Configuration PMCSR write clear function 0 interrupt
+      - description: Configuration PMCSR write clear function 1 interrupt
 
   interrupt-names:
+    minItems: 16
     items:
-      - description: serr
-      - description: ser_cor
-      - description: serr_nonfatal
-      - description: serr_fatal
-      - description: axi_err
-      - description: inta
-      - description: intb
-      - description: intc
-      - description: intd
-      - description: msi
-      - description: link_bandwidth
-      - description: pm_pme
-      - description: dma
-      - description: pcie_evt
-      - description: msg
-      - description: all
+      - const: serr
+      - const: serr_cor
+      - const: serr_nonfatal
+      - const: serr_fatal
+      - const: axi_err
+      - const: inta
+      - const: intb
+      - const: intc
+      - const: intd
+      - const: msi
+      - const: link_bandwidth
+      - const: pm_pme
+      - const: dma
+      - const: pcie_evt
+      - const: msg
+      - const: all
+      - const: link_equalization_request
+      - const: turn_off_event
+      - const: pmu_poweroff
+      - const: d3_event_f0
+      - const: d3_event_f1
+      - const: cfg_pmcsr_writeclear_f0
+      - const: cfg_pmcsr_writeclear_f1
 
   interrupt-controller: true
 
   clocks:
     items:
       - description: System clock
-      - description: PM control clock
+      - description: PM control clock or clock for L1 substate handling
 
   clock-names:
     items:
-      - description: aclk
-      - description: pm
+      - const: aclk
+      - enum: [pm, pmu]
 
   resets:
+    minItems: 1
     items:
       - description: AXI2PCIe Bridge reset
       - description: Data link layer/transaction layer reset
@@ -81,14 +101,15 @@ properties:
       - description: Configuration register reset
 
   reset-names:
+    minItems: 1
     items:
-      - description: aresetn
-      - description: rst_b
-      - description: rst_gp_b
-      - description: rst_ps_b
-      - description: rst_rsm_b
-      - description: rst_cfg_b
-      - description: rst_load_b
+      - const: aresetn
+      - const: rst_b
+      - const: rst_gp_b
+      - const: rst_ps_b
+      - const: rst_rsm_b
+      - const: rst_cfg_b
+      - const: rst_load_b
 
   power-domains:
     maxItems: 1
@@ -128,7 +149,9 @@ patternProperties:
         const: 0x1912
 
       device-id:
-        const: 0x0033
+        enum:
+          - 0x0033
+          - 0x0039
 
       clocks:
         items:
@@ -167,6 +190,44 @@ required:
 
 allOf:
   - $ref: /schemas/pci/pci-host-bridge.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a08g045-pcie
+    then:
+      properties:
+        interrupts:
+          maxItems: 16
+        interrupt-names:
+          maxItems: 16
+        clock-names:
+          items:
+            - const: aclk
+            - const: pm
+        resets:
+          minItems: 7
+        reset-names:
+          minItems: 7
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g047-pcie
+    then:
+      properties:
+        interrupts:
+          minItems: 23
+        interrupt-names:
+          minItems: 23
+        clock-names:
+          items:
+            - const: aclk
+            - const: pmu
+        resets:
+          maxItems: 1
+        reset-names:
+          maxItems: 1
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/phy/canaan,k230-usb-phy.yaml b/Documentation/devicetree/bindings/phy/canaan,k230-usb-phy.yaml
new file mode 100644
index 0000000000000..b959b381c44c2
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/canaan,k230-usb-phy.yaml
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/canaan,k230-usb-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Canaan K230 USB2.0 PHY
+
+maintainers:
+  - Jiayu Du <jiayu.riscv@isrc.iscas.ac.cn>
+
+properties:
+  compatible:
+    const: canaan,k230-usb-phy
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    usbphy: usb-phy@91585000 {
+        compatible = "canaan,k230-usb-phy";
+        reg = <0x91585000 0x400>;
+        #phy-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/phy/eswin,eic7700-sata-phy.yaml b/Documentation/devicetree/bindings/phy/eswin,eic7700-sata-phy.yaml
new file mode 100644
index 0000000000000..fc7dbac77acf7
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/eswin,eic7700-sata-phy.yaml
@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/eswin,eic7700-sata-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Eswin EIC7700 SoC SATA PHY
+
+maintainers:
+  - Yulin Lu <luyulin@eswincomputing.com>
+  - Huan He <hehuan1@eswincomputing.com>
+
+properties:
+  compatible:
+    const: eswin,eic7700-sata-phy
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: phy
+
+  resets:
+    maxItems: 2
+
+  reset-names:
+    items:
+      - const: port
+      - const: phy
+
+  eswin,tx-amplitude-tuning:
+    description: This adjusts the transmitter amplitude signal, and its value
+      is derived from eye diagram tuning. The three values correspond to Gen1,
+      Gen2, and Gen3 parameters respectively.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    items:
+      - description: Gen1 parameter.
+        minimum: 0
+        maximum: 0x7f
+      - description: Gen2 parameter.
+        minimum: 0
+        maximum: 0x7f
+      - description: Gen3 parameter.
+        minimum: 0
+        maximum: 0x7f
+    default: [0, 0, 0]
+
+  eswin,tx-preemph-tuning:
+    description: This adjusts the transmitter de-emphasis signal, and its value
+      is derived from eye diagram tuning. The three values correspond to Gen1,
+      Gen2, and Gen3 parameters respectively.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    items:
+      - description: Gen1 parameter.
+        minimum: 0
+        maximum: 0x3f
+      - description: Gen2 parameter.
+        minimum: 0
+        maximum: 0x3f
+      - description: Gen3 parameter.
+        minimum: 0
+        maximum: 0x3f
+    default: [0, 0, 0]
+
+  "#phy-cells":
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    sata-phy@50440300 {
+        compatible = "eswin,eic7700-sata-phy";
+        reg = <0x50440300 0x40>;
+        clocks = <&hspcrg 17>;
+        clock-names = "phy";
+        resets = <&hspcrg 0>, <&hspcrg 1>;
+        reset-names = "port", "phy";
+        #phy-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml
index acdbce937b0a4..c6d0bbdbe0e22 100644
--- a/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml
@@ -23,6 +23,7 @@ properties:
       - items:
           - enum:
               - mediatek,mt7623-mipi-tx
+              - mediatek,mt8167-mipi-tx
           - const: mediatek,mt2701-mipi-tx
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
index a1731b08c9d14..9616c736b6d46 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
@@ -16,6 +16,10 @@ description:
 properties:
   compatible:
     oneOf:
+      - items:
+          - enum:
+              - qcom,qcs8300-qmp-ufs-phy
+          - const: qcom,sa8775p-qmp-ufs-phy
       - items:
           - enum:
               - qcom,qcs615-qmp-ufs-phy
@@ -26,8 +30,8 @@ properties:
           - const: qcom,sm8550-qmp-ufs-phy
       - items:
           - enum:
-              - qcom,qcs8300-qmp-ufs-phy
-          - const: qcom,sa8775p-qmp-ufs-phy
+              - qcom,eliza-qmp-ufs-phy
+          - const: qcom,sm8650-qmp-ufs-phy
       - items:
           - enum:
               - qcom,kaanapali-qmp-ufs-phy
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx27-iomuxc.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx27-iomuxc.yaml
new file mode 100644
index 0000000000000..1254bfcaa7cb6
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx27-iomuxc.yaml
@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/fsl,imx27-iomuxc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Freescale i.MX1/i.MX25/i.MX27 IOMUX Controller
+
+maintainers:
+  - Frank Li <Frank.Li@nxp.com>
+
+description:
+  Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory
+  for common binding part and usage.
+
+properties:
+  compatible:
+    enum:
+      - fsl,imx1-iomuxc
+      - fsl,imx27-iomuxc
+
+  reg:
+    maxItems: 1
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
+  ranges: true
+
+patternProperties:
+  '^gpio@[0-9a-f]+$':
+    type: object
+    $ref: /schemas/gpio/fsl-imx-gpio.yaml
+    unevaluatedProperties: false
+
+  'grp$':
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+
+    properties:
+      fsl,pins:
+        description:
+          three integers array, represents a group of pins mux and config
+          setting. The format is fsl,pins = <PIN MUX_ID CONFIG>.
+        $ref: /schemas/types.yaml#/definitions/uint32-matrix
+        items:
+          items:
+            - description:
+                PIN is an integer between 0 and 0xbf. imx27 has 6 ports with 32
+                configurable pins each. PIN is PORT * 32 + PORT_PIN, PORT_PIN
+                is the pin number on the specific port (between 0 and 31)
+            - description: |
+                MUX_ID is function + (direction << 2) + (gpio_oconf << 4)
+                + (gpio_iconfa << 8) + (gpio_iconfb << 10)
+
+                function value is used to select the pin function.
+                Possible values:
+                  0 - Primary function
+                  1 - Alternate function
+                  2 - GPIO
+                Registers: GIUS (GPIO In Use), GPR (General Purpose Register)
+
+                direction defines the data direction of the pin.
+                Possible values:
+                  0 - Input
+                  1 - Output
+                Register: DDIR
+
+                gpio_oconf configures the gpio submodule output signal.
+                This does not have any effect unless GPIO function is
+                selected. A/B/C_IN are output signals of function blocks
+                A,B and C. Specific function blocks are described in the
+                reference manual.
+                Possible values:
+                  0 - A_IN
+                  1 - B_IN
+                  2 - C_IN
+                  3 - Data Register
+                Registers: OCR1, OCR2
+
+                gpio_iconfa/b configures the gpio submodule input to
+                functionblocks A and B. GPIO function should be selected if
+                this is configured.
+                Possible values:
+                  0 - GPIO_IN
+                  1 - Interrupt Status Register
+                  2 - Pulldown
+                  3 - Pullup
+                Registers ICONFA1, ICONFA2, ICONFB1 and ICONFB2
+
+            - description:
+                CONFIG can be 0 or 1, meaning Pullup disable/enable.
+    required:
+      - fsl,pins
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    pinmux@10015000 {
+        compatible = "fsl,imx27-iomuxc";
+        reg = <0x10015000 0x600>;
+
+        uartgrp {
+            fsl,pins = <
+                0x8c 0x004 0x0 /* UART1_TXD__UART1_TXD */
+                0x8d 0x000 0x0 /* UART1_RXD__UART1_RXD */
+                0x8e 0x004 0x0 /* UART1_CTS__UART1_CTS */
+                0x8f 0x000 0x0 /* UART1_RTS__UART1_RTS */
+            >;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx27-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx27-pinctrl.txt
deleted file mode 100644
index d1706ea825723..0000000000000
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx27-pinctrl.txt
+++ /dev/null
@@ -1,121 +0,0 @@
-* Freescale IMX27 IOMUX Controller
-
-Required properties:
-- compatible: "fsl,imx27-iomuxc"
-
-The iomuxc driver node should define subnodes containing of pinctrl configuration subnodes.
-
-Required properties for pin configuration node:
-- fsl,pins: three integers array, represents a group of pins mux and config
-  setting. The format is fsl,pins = <PIN MUX_ID CONFIG>.
-
-  PIN is an integer between 0 and 0xbf. imx27 has 6 ports with 32 configurable
-  configurable pins each. PIN is PORT * 32 + PORT_PIN, PORT_PIN is the pin
-  number on the specific port (between 0 and 31).
-
-  MUX_ID is
-    function + (direction << 2) + (gpio_oconf << 4) + (gpio_iconfa << 8) + (gpio_iconfb << 10)
-
-      function value is used to select the pin function.
-      Possible values:
-          0 - Primary function
-          1 - Alternate function
-          2 - GPIO
-      Registers: GIUS (GPIO In Use), GPR (General Purpose Register)
-
-      direction defines the data direction of the pin.
-      Possible values:
-          0 - Input
-          1 - Output
-      Register: DDIR
-
-      gpio_oconf configures the gpio submodule output signal. This does not
-      have any effect unless GPIO function is selected. A/B/C_IN are output
-      signals of function blocks A,B and C. Specific function blocks are
-      described in the reference manual.
-      Possible values:
-          0 - A_IN
-          1 - B_IN
-          2 - C_IN
-          3 - Data Register
-      Registers: OCR1, OCR2
-
-      gpio_iconfa/b configures the gpio submodule input to functionblocks A and
-      B. GPIO function should be selected if this is configured.
-      Possible values:
-          0 - GPIO_IN
-          1 - Interrupt Status Register
-          2 - Pulldown
-          3 - Pullup
-      Registers ICONFA1, ICONFA2, ICONFB1 and ICONFB2
-
-  CONFIG can be 0 or 1, meaning Pullup disable/enable.
-
-
-The iomux controller has gpio child nodes which are embedded in the iomux
-control registers. They have to be defined as child nodes of the iomux device
-node. If gpio subnodes are defined "#address-cells", "#size-cells" and "ranges"
-properties for the iomux device node are required.
-
-Example:
-
-iomuxc: iomuxc@10015000 {
-	compatible = "fsl,imx27-iomuxc";
-	reg = <0x10015000 0x600>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges;
-
-	gpio1: gpio@10015000 {
-		...
-	};
-
-	...
-
-	uart {
-		pinctrl_uart1: uart-1 {
-			fsl,pins = <
-				0x8c 0x004 0x0 /* UART1_TXD__UART1_TXD */
-				0x8d 0x000 0x0 /* UART1_RXD__UART1_RXD */
-				0x8e 0x004 0x0 /* UART1_CTS__UART1_CTS */
-				0x8f 0x000 0x0 /* UART1_RTS__UART1_RTS */
-			>;
-		};
-
-		...
-	};
-};
-
-
-For convenience there are macros defined in imx27-pinfunc.h which provide PIN
-and MUX_ID. They are structured as MX27_PAD_<Pad name>__<Signal name>. The names
-are defined in the i.MX27 reference manual.
-
-The above example using macros:
-
-iomuxc: iomuxc@10015000 {
-	compatible = "fsl,imx27-iomuxc";
-	reg = <0x10015000 0x600>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges;
-
-	gpio1: gpio@10015000 {
-		...
-	};
-
-	...
-
-	uart {
-		pinctrl_uart1: uart-1 {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD 0x0
-				MX27_PAD_UART1_RXD__UART1_RXD 0x0
-				MX27_PAD_UART1_CTS__UART1_CTS 0x0
-				MX27_PAD_UART1_RTS__UART1_RTS 0x0
-			>;
-		};
-
-		...
-	};
-};
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
index 265c43ab76f4c..846e110062b2a 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
@@ -20,6 +20,7 @@ properties:
   compatible:
     oneOf:
       - enum:
+          - fsl,imx25-iomuxc
           - fsl,imx35-iomuxc
           - fsl,imx51-iomuxc
           - fsl,imx53-iomuxc
diff --git a/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml
index 4f9013d368749..727da7fb490ce 100644
--- a/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml
@@ -84,11 +84,12 @@ patternProperties:
 
     properties:
       groups:
-        enum: [ emmc_nb, i2c1, i2c2, jtag, mii_col, onewire, pcie1,
-                pcie1_clkreq, pcie1_wakeup, pmic0, pmic1, ptp, ptp_clk,
-                ptp_trig, pwm0, pwm1, pwm2, pwm3, rgmii, sdio0, sdio_sb, smi,
-                spi_cs1, spi_cs2, spi_cs3, spi_quad, uart1, uart2,
-                usb2_drvvbus1, usb32_drvvbus0 ]
+        items:
+          enum: [ emmc_nb, i2c1, i2c2, jtag, mii_col, onewire, pcie1,
+                  pcie1_clkreq, pcie1_wakeup, pmic0, pmic1, ptp, ptp_clk,
+                  ptp_trig, pwm0, pwm1, pwm2, pwm3, rgmii, sdio0, sdio_sb,
+                  smi, spi_cs1, spi_cs2, spi_cs3, spi_quad, uart1, uart2,
+                  usb2_drvvbus1, usb32_drvvbus0 ]
 
       function:
         enum: [ drvbus, emmc, gpio, i2c, jtag, led, mii, mii_err, onewire,
diff --git a/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml b/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
index a916d0fc79a99..981f45c2f56b0 100644
--- a/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
@@ -162,12 +162,99 @@ properties:
       this affects the expected delay in ps before latching a value to
       an output pin.
 
-if:
-  required:
-    - skew-delay
-then:
-  properties:
-    skew-delay-input-ps: false
-    skew-delay-output-ps: false
+allOf:
+  - if:
+      required:
+        - skew-delay
+    then:
+      properties:
+        skew-delay-input-ps: false
+        skew-delay-output-ps: false
+
+  - if:
+      required:
+        - input-disable
+    then:
+      properties:
+        input-enable: false
+
+  - if:
+      required:
+        - output-disable
+    then:
+      properties:
+        output-enable: false
+        output-impedance-ohms: false
+
+  - if:
+      required:
+        - output-low
+    then:
+      properties:
+        output-high: false
+
+  - if:
+      required:
+        - low-power-enable
+    then:
+      properties:
+        low-power-disable: false
+
+  - if:
+      required:
+        - input-schmitt-disable
+    then:
+      properties:
+        input-schmitt-enable: false
+        input-schmitt-microvolt: false
+
+  - if:
+      required:
+        - drive-strength
+    then:
+      properties:
+        drive-strength-microamp: false
+
+  - if:
+      anyOf:
+        - required:
+            - drive-open-source
+        - required:
+            - drive-open-drain
+        - required:
+            - drive-push-pull
+    then:
+      oneOf:
+        - required:
+            - drive-open-source
+        - required:
+            - drive-open-drain
+        - required:
+            - drive-push-pull
+
+  - if:
+      anyOf:
+        - required:
+            - bias-disable
+        - required:
+            - bias-bus-hold
+        - required:
+            - bias-pull-up
+        - required:
+            - bias-pull-down
+        - required:
+            - bias-pull-pin-default
+    then:
+      oneOf:
+        - required:
+            - bias-disable
+        - required:
+            - bias-bus-hold
+        - required:
+            - bias-pull-up
+        - required:
+            - bias-pull-down
+        - required:
+            - bias-pull-pin-default
 
 additionalProperties: true
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
new file mode 100644
index 0000000000000..2826504264879
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,eliza-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. Eliza TLMM block
+
+maintainers:
+  - Abel Vesa <abel.vesa@oss.qualcomm.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm Eliza SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,eliza-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 93
+
+  gpio-line-names:
+    maxItems: 185
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-eliza-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-eliza-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-eliza-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          oneOf:
+            - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|18[0-4])$"
+            - enum: [ ufs_reset ]
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+        enum: [ gpio, aoss_cti, atest_char, atest_usb, audio_ext_mclk0,
+                audio_ref_clk, cam_mclk, cci_async_in, cci_i2c_scl,
+                cci_i2c_sda, cci_timer, coex_uart1_rx, coex_uart1_tx,
+                coex_uart2_rx, coex_uart2_tx, dbg_out_clk,
+                ddr_bist_complete, ddr_bist_fail, ddr_bist_start,
+                ddr_bist_stop, ddr_pxi0, ddr_pxi1, dp0_hot, egpio,
+                gcc_gp1, gcc_gp2, gcc_gp3, gnss_adc0, gnss_adc1,
+                hdmi_ddc_scl, hdmi_ddc_sda, hdmi_dtest0, hdmi_dtest1,
+                hdmi_hot_plug, hdmi_pixel_clk, hdmi_rcv_det, hdmi_tx_cec,
+                host2wlan_sol, i2s0_data0, i2s0_data1, i2s0_sck, i2s0_ws,
+                ibi_i3c, jitter_bist, mdp_esync0_out, mdp_esync1_out,
+                mdp_vsync, mdp_vsync0_out, mdp_vsync11_out,
+                mdp_vsync1_out, mdp_vsync2_out, mdp_vsync3_out,
+                mdp_vsync_e, nav_gpio0, nav_gpio1, nav_gpio2, nav_gpio3,
+                pcie0_clk_req_n, pcie1_clk_req_n, phase_flag,
+                pll_bist_sync, pll_clk_aux, prng_rosc0, prng_rosc1,
+                prng_rosc2, prng_rosc3, qdss_cti, qdss_gpio_traceclk,
+                qdss_gpio_tracectl, qdss_gpio_tracedata, qlink_big_enable,
+                qlink_big_request, qlink_little_enable,
+                qlink_little_request, qlink_wmss, qspi0, qspi_clk,
+                qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se3, qup1_se4,
+                qup1_se5, qup1_se6, qup1_se7, qup2_se0, qup2_se1,
+                qup2_se2, qup2_se3, qup2_se4, qup2_se5, qup2_se6,
+                qup2_se7, resout_gpio, sd_write_protect, sdc1, sdc2,
+                sdc2_fb_clk, tb_trig_sdc1, tb_trig_sdc2, tmess_prng0,
+                tmess_prng1, tmess_prng2, tmess_prng3, tsense_pwm1,
+                tsense_pwm2, tsense_pwm3, tsense_pwm4, uim0_clk,
+                uim0_data, uim0_present, uim0_reset, uim1_clk, uim1_data,
+                uim1_present, uim1_reset, usb0_hs, usb_phy, vfr_0, vfr_1,
+                vsense_trigger_mirnat, wcn_sw_ctrl ]
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    tlmm: pinctrl@f100000 {
+        compatible = "qcom,eliza-tlmm";
+        reg = <0x0f100000 0x300000>;
+
+        interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+
+        gpio-controller;
+        #gpio-cells = <2>;
+
+        interrupt-controller;
+        #interrupt-cells = <2>;
+
+        gpio-ranges = <&tlmm 0 0 186>;
+
+        gpio-wo-state {
+            pins = "gpio1";
+            function = "gpio";
+        };
+
+        qup-uart14-default-state {
+            pins = "gpio18", "gpio19";
+            function = "qup2_se5";
+            drive-strength = <2>;
+            bias-disable;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
new file mode 100644
index 0000000000000..73e84f1885911
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Milos SoC LPASS LPI TLMM
+
+maintainers:
+  - Luca Weiss <luca.weiss@fairphone.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in the Low Power Audio SubSystem
+  (LPASS) Low Power Island (LPI) of Qualcomm Milos SoC.
+
+properties:
+  compatible:
+    const: qcom,milos-lpass-lpi-pinctrl
+
+  reg:
+    items:
+      - description: LPASS LPI TLMM Control and Status registers
+      - description: LPASS LPI MCC registers
+
+  clocks:
+    items:
+      - description: LPASS Core voting clock
+      - description: LPASS Audio voting clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: audio
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-milos-lpass-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-milos-lpass-state"
+        additionalProperties: false
+
+$defs:
+  qcom-milos-lpass-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,lpass-lpi-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|1[0-9]|2[0-2])$"
+
+      function:
+        enum: [ dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic3_clk,
+                dmic3_data, dmic4_clk, dmic4_data, ext_mclk1_a, ext_mclk1_b,
+                ext_mclk1_c, ext_mclk1_d, ext_mclk1_e, gpio, i2s0_clk,
+                i2s0_data, i2s0_ws, i2s1_clk, i2s1_data, i2s1_ws, i2s2_clk,
+                i2s2_data, i2s2_ws, i2s3_clk, i2s3_data, i2s3_ws, qca_swr_clk,
+                qca_swr_data, slimbus_clk, slimbus_data, swr_rx_clk,
+                swr_rx_data, swr_tx_clk, swr_tx_data, wsa_swr_clk,
+                wsa_swr_data ]
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+allOf:
+  - $ref: qcom,lpass-lpi-common.yaml#
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+    pinctrl@3440000 {
+        compatible = "qcom,milos-lpass-lpi-pinctrl";
+        reg = <0x03440000 0x20000>,
+              <0x034d0000 0x10000>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&lpass_tlmm 0 0 23>;
+
+        clocks = <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+                 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+        clock-names = "core",
+                      "audio";
+
+        tx-swr-active-clk-state {
+            pins = "gpio0";
+            function = "swr_tx_clk";
+            drive-strength = <4>;
+            slew-rate = <1>;
+            bias-disable;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml
index e7565592da862..541c1c54ddb0f 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml
@@ -15,7 +15,13 @@ description:
 
 properties:
   compatible:
-    const: qcom,sm8450-lpass-lpi-pinctrl
+    oneOf:
+      - const: qcom,sm8450-lpass-lpi-pinctrl
+      - items:
+          - enum:
+              - qcom,qcs8300-lpass-lpi-pinctrl
+              - qcom,sa8775p-lpass-lpi-pinctrl
+          - const: qcom,sm8450-lpass-lpi-pinctrl
 
   reg:
     items:
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
index 74df912e60ade..1bf08860a4baf 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
@@ -19,7 +19,9 @@ properties:
     oneOf:
       - const: qcom,sm8650-lpass-lpi-pinctrl
       - items:
-          - const: qcom,sm8750-lpass-lpi-pinctrl
+          - enum:
+              - qcom,glymur-lpass-lpi-pinctrl
+              - qcom,sm8750-lpass-lpi-pinctrl
           - const: qcom,sm8650-lpass-lpi-pinctrl
 
   reg:
diff --git a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml
index 76e6072817160..9b3cbeb54fed4 100644
--- a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml
@@ -50,6 +50,7 @@ properties:
       - rockchip,rk3568-pinctrl
       - rockchip,rk3576-pinctrl
       - rockchip,rk3588-pinctrl
+      - rockchip,rv1103b-pinctrl
       - rockchip,rv1108-pinctrl
       - rockchip,rv1126-pinctrl
 
diff --git a/Documentation/devicetree/bindings/pinctrl/st,stm32-hdp.yaml b/Documentation/devicetree/bindings/pinctrl/st,stm32-hdp.yaml
index 845b6b7b7552f..8f8b4b68aaa31 100644
--- a/Documentation/devicetree/bindings/pinctrl/st,stm32-hdp.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/st,stm32-hdp.yaml
@@ -27,6 +27,12 @@ properties:
   clocks:
     maxItems: 1
 
+  access-controllers:
+    minItems: 1
+    items:
+      - description: debug configuration access controller
+      - description: access controller that manages the HDP as a peripheral
+
 patternProperties:
   "^hdp[0-7]-pins$":
     type: object
diff --git a/Documentation/devicetree/bindings/power/allwinner,sun20i-d1-ppu.yaml b/Documentation/devicetree/bindings/power/allwinner,sun20i-d1-ppu.yaml
index a28e75a9cb6a5..b97361ce2a000 100644
--- a/Documentation/devicetree/bindings/power/allwinner,sun20i-d1-ppu.yaml
+++ b/Documentation/devicetree/bindings/power/allwinner,sun20i-d1-ppu.yaml
@@ -20,6 +20,7 @@ properties:
       - allwinner,sun20i-d1-ppu
       - allwinner,sun55i-a523-pck-600
       - allwinner,sun55i-a523-ppu
+      - allwinner,sun60i-a733-pck-600
 
   reg:
     maxItems: 1
@@ -38,9 +39,23 @@ required:
   - compatible
   - reg
   - clocks
-  - resets
   - '#power-domain-cells'
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - allwinner,sun8i-v853-ppu
+              - allwinner,sun20i-d1-ppu
+              - allwinner,sun55i-a523-pck-600
+              - allwinner,sun55i-a523-ppu
+
+    then:
+      required:
+        - resets
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/power/mediatek,mt8196-gpufreq.yaml b/Documentation/devicetree/bindings/power/mediatek,mt8196-gpufreq.yaml
index b9e43abaf8a42..66fc59b3c8b47 100644
--- a/Documentation/devicetree/bindings/power/mediatek,mt8196-gpufreq.yaml
+++ b/Documentation/devicetree/bindings/power/mediatek,mt8196-gpufreq.yaml
@@ -74,9 +74,18 @@ properties:
   "#clock-cells":
     const: 1
 
+  "#nvmem-cell-cells":
+    const: 0
+
   "#power-domain-cells":
     const: 0
 
+  shader-present:
+    type: object
+
+dependencies:
+  shader-present: [ "#nvmem-cell-cells" ]
+
 required:
   - compatible
   - reg
@@ -113,5 +122,9 @@ examples:
                      "ccf", "fast-dvfs";
         memory-region = <&gpueb_shared_memory>;
         #clock-cells = <1>;
+        #nvmem-cell-cells = <0>;
         #power-domain-cells = <0>;
+
+        shader-present {
+        };
     };
diff --git a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
index 9507b342a7ee6..07f046277f8a6 100644
--- a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
+++ b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
@@ -31,6 +31,7 @@ properties:
       - mediatek,mt8183-power-controller
       - mediatek,mt8186-power-controller
       - mediatek,mt8188-power-controller
+      - mediatek,mt8189-power-controller
       - mediatek,mt8192-power-controller
       - mediatek,mt8195-power-controller
       - mediatek,mt8196-hwv-hfrp-power-controller
diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
index 27af5b8aa1340..8174ceeab5723 100644
--- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
+++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
@@ -17,6 +17,7 @@ properties:
   compatible:
     oneOf:
       - enum:
+          - qcom,eliza-rpmhpd
           - qcom,glymur-rpmhpd
           - qcom,kaanapali-rpmhpd
           - qcom,mdm9607-rpmpd
diff --git a/Documentation/devicetree/bindings/power/supply/cpcap-battery.yaml b/Documentation/devicetree/bindings/power/supply/cpcap-battery.yaml
index 694bfdb5815ca..6dcca55d6d90a 100644
--- a/Documentation/devicetree/bindings/power/supply/cpcap-battery.yaml
+++ b/Documentation/devicetree/bindings/power/supply/cpcap-battery.yaml
@@ -55,6 +55,7 @@ properties:
       - const: chg_isense
       - const: batti
 
+  monitored-battery: true
   power-supplies: true
 
 required:
diff --git a/Documentation/devicetree/bindings/power/supply/maxim,max17042.yaml b/Documentation/devicetree/bindings/power/supply/maxim,max17042.yaml
index 14242de7fc081..242b33f2bcba5 100644
--- a/Documentation/devicetree/bindings/power/supply/maxim,max17042.yaml
+++ b/Documentation/devicetree/bindings/power/supply/maxim,max17042.yaml
@@ -20,6 +20,7 @@ properties:
       - maxim,max17050
       - maxim,max17055
       - maxim,max77705-battery
+      - maxim,max77759-fg
       - maxim,max77849-battery
 
   reg:
@@ -27,36 +28,42 @@ properties:
 
   interrupts:
     maxItems: 1
-    description: |
-      The ALRT pin, an open-drain interrupt.
+    description:
+      The ALRT pin (or FG_INTB pin on MAX77759), an open-drain interrupt.
+
+  shunt-resistor-micro-ohms:
+    description:
+      Resistance of rsns resistor in micro Ohms (datasheet-recommended value is 10000).
+      Defining this property enables current-sense functionality.
 
   maxim,rsns-microohm:
+    deprecated: true
     $ref: /schemas/types.yaml#/definitions/uint32
-    description: |
+    description:
       Resistance of rsns resistor in micro Ohms (datasheet-recommended value is 10000).
       Defining this property enables current-sense functionality.
 
   maxim,cold-temp:
     $ref: /schemas/types.yaml#/definitions/uint32
-    description: |
+    description:
       Temperature threshold to report battery as cold (in tenths of degree Celsius).
       Default is not to report cold events.
 
   maxim,over-heat-temp:
     $ref: /schemas/types.yaml#/definitions/uint32
-    description: |
+    description:
       Temperature threshold to report battery as over heated (in tenths of degree Celsius).
       Default is not to report over heating events.
 
   maxim,dead-volt:
     $ref: /schemas/types.yaml#/definitions/uint32
-    description: |
+    description:
       Voltage threshold to report battery as dead (in mV).
       Default is not to report dead battery events.
 
   maxim,over-volt:
     $ref: /schemas/types.yaml#/definitions/uint32
-    description: |
+    description:
       Voltage threshold to report battery as over voltage (in mV).
       Default is not to report over-voltage events.
 
diff --git a/Documentation/devicetree/bindings/power/supply/samsung,s2mu005-fuel-gauge.yaml b/Documentation/devicetree/bindings/power/supply/samsung,s2mu005-fuel-gauge.yaml
new file mode 100644
index 0000000000000..05e420316a26b
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/supply/samsung,s2mu005-fuel-gauge.yaml
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/power/supply/samsung,s2mu005-fuel-gauge.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Battery Fuel Gauge for Samsung S2M series PMICs
+
+maintainers:
+  - Kaustabh Chakraborty <kauschluss@disroot.org>
+
+allOf:
+  - $ref: power-supply.yaml#
+
+properties:
+  compatible:
+    enum:
+      - samsung,s2mu005-fuel-gauge
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      fuel-gauge@3b {
+        compatible = "samsung,s2mu005-fuel-gauge";
+        reg = <0x3b>;
+
+        interrupt-parent = <&gpa0>;
+        interrupts = <3 IRQ_TYPE_EDGE_BOTH>;
+
+        monitored-battery = <&battery>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml
index cc3ebd4deeb69..c337d85da40f1 100644
--- a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml
+++ b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml
@@ -39,7 +39,10 @@ properties:
           - amlogic,meson-s4-pwm
       - items:
           - enum:
+              - amlogic,a4-pwm
+              - amlogic,a5-pwm
               - amlogic,c3-pwm
+              - amlogic,t7-pwm
               - amlogic,meson-a1-pwm
           - const: amlogic,meson-s4-pwm
       - items:
diff --git a/Documentation/devicetree/bindings/regulator/cpcap-regulator.txt b/Documentation/devicetree/bindings/regulator/cpcap-regulator.txt
deleted file mode 100644
index 36f5e2f5cc0fb..0000000000000
--- a/Documentation/devicetree/bindings/regulator/cpcap-regulator.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-Motorola CPCAP PMIC voltage regulators
-------------------------------------
-
-Requires node properties:
-- "compatible" value one of:
-    "motorola,cpcap-regulator"
-    "motorola,mapphone-cpcap-regulator"
-    "motorola,xoom-cpcap-regulator"
-
-Required regulator properties:
-- "regulator-name"
-- "regulator-enable-ramp-delay"
-- "regulator-min-microvolt"
-- "regulator-max-microvolt"
-
-Optional regulator properties:
-- "regulator-boot-on"
-
-See Documentation/devicetree/bindings/regulator/regulator.txt
-for more details about the regulator properties.
-
-Example:
-
-cpcap_regulator: regulator {
-	compatible = "motorola,cpcap-regulator";
-
-	cpcap_regulators: regulators {
-		sw5: SW5 {
-			regulator-min-microvolt = <5050000>;
-			regulator-max-microvolt = <5050000>;
-			regulator-enable-ramp-delay = <50000>;
-			regulator-boot-on;
-		};
-	};
-};
diff --git a/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml b/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml
index c6585e3bacbec..00d66b9230475 100644
--- a/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml
+++ b/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml
@@ -66,6 +66,7 @@ properties:
 required:
   - compatible
   - reg
+  - vin-supply
   - pg-gpios
   - enable-gpios
 
diff --git a/Documentation/devicetree/bindings/regulator/motorola,cpcap-regulator.yaml b/Documentation/devicetree/bindings/regulator/motorola,cpcap-regulator.yaml
new file mode 100644
index 0000000000000..1a44c8e61243f
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/motorola,cpcap-regulator.yaml
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/motorola,cpcap-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Motorola CPCAP PMIC regulators
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+description:
+  This module is part of the Motorola CPCAP MFD device. For more details
+  see Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml. The
+  regulator controller is represented as a sub-node of the PMIC node
+  on the device tree.
+
+properties:
+  compatible:
+    enum:
+      - motorola,cpcap-regulator
+      - motorola,mapphone-cpcap-regulator
+      - motorola,mot-cpcap-regulator
+      - motorola,xoom-cpcap-regulator
+
+  regulators:
+    type: object
+
+    patternProperties:
+      "^(SW[1-6]|V(CAM|CSI|DAC|DIG|FUSE|HVIO|SDIO|PLL|RF[12]|RFREF|WLAN[12]|SIM|SIMCARD|VIB|USB|AUDIO))$":
+        $ref: /schemas/regulator/regulator.yaml#
+        type: object
+
+        required:
+          - regulator-name
+          - regulator-enable-ramp-delay
+          - regulator-min-microvolt
+          - regulator-max-microvolt
+
+        unevaluatedProperties: false
+
+required:
+  - compatible
+
+additionalProperties: false
+
+...
diff --git a/Documentation/devicetree/bindings/regulator/regulator.yaml b/Documentation/devicetree/bindings/regulator/regulator.yaml
index 042e56396399f..019aeb664caec 100644
--- a/Documentation/devicetree/bindings/regulator/regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/regulator.yaml
@@ -168,7 +168,7 @@ properties:
       offset from voltage set to regulator.
 
   regulator-uv-protection-microvolt:
-    description: Set over under voltage protection limit. This is a limit where
+    description: Set under voltage protection limit. This is a limit where
       hardware performs emergency shutdown. Zero can be passed to disable
       protection and value '1' indicates that protection should be enabled but
       limit setting can be omitted. Limit is given as microvolt offset from
@@ -182,7 +182,7 @@ properties:
       is given as microvolt offset from voltage set to regulator.
 
   regulator-uv-warn-microvolt:
-    description: Set over under voltage warning limit. This is a limit where
+    description: Set under voltage warning limit. This is a limit where
       hardware is assumed still to be functional but approaching limit where
       it gets damaged. Recovery actions should be initiated. Zero can be passed
       to disable detection and value '1' indicates that detection should
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
index c179b560572b2..8c0ff4dfad100 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
@@ -17,10 +17,14 @@ properties:
   compatible:
     oneOf:
       - enum:
+          - qcom,mdm9607-mss-pil
           - qcom,msm8226-mss-pil
           - qcom,msm8909-mss-pil
           - qcom,msm8916-mss-pil
+          - qcom,msm8917-mss-pil
           - qcom,msm8926-mss-pil
+          - qcom,msm8937-mss-pil
+          - qcom,msm8940-mss-pil
           - qcom,msm8953-mss-pil
           - qcom,msm8974-mss-pil
 
@@ -89,7 +93,7 @@ properties:
     description: PLL proxy supply (control handed over after startup)
 
   mss-supply:
-    description: MSS power domain supply (only valid for qcom,msm8974-mss-pil)
+    description: MSS power domain supply
 
   resets:
     items:
@@ -226,8 +230,12 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,mdm9607-mss-pil
               - qcom,msm8909-mss-pil
               - qcom,msm8916-mss-pil
+              - qcom,msm8917-mss-pil
+              - qcom,msm8937-mss-pil
+              - qcom,msm8940-mss-pil
     then:
       properties:
         power-domains:
@@ -271,6 +279,9 @@ allOf:
           contains:
             enum:
               - qcom,msm8926-mss-pil
+              - qcom,msm8917-mss-pil
+              - qcom,msm8937-mss-pil
+              - qcom,msm8940-mss-pil
               - qcom,msm8974-mss-pil
     then:
       required:
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
index 11b056d6a4808..6a29d239ef413 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
@@ -28,8 +28,16 @@ properties:
           - qcom,x1e80100-adsp-pas
           - qcom,x1e80100-cdsp-pas
       - items:
-          - const: qcom,sm8750-adsp-pas
+          - enum:
+              - qcom,glymur-adsp-pas
+              - qcom,kaanapali-adsp-pas
+              - qcom,sm8750-adsp-pas
           - const: qcom,sm8550-adsp-pas
+      - items:
+          - enum:
+              - qcom,glymur-cdsp-pas
+              - qcom,kaanapali-cdsp-pas
+          - const: qcom,sm8550-cdsp-pas
       - items:
           - const: qcom,sm8750-cdsp-pas
           - const: qcom,sm8650-cdsp-pas
@@ -95,6 +103,10 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,glymur-adsp-pas
+              - qcom,glymur-cdsp-pas
+              - qcom,kaanapali-adsp-pas
+              - qcom,kaanapali-cdsp-pas
               - qcom,sm8750-adsp-pas
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml
index 41fd11f70a49b..f9854ff43ac68 100644
--- a/Documentation/devicetree/bindings/riscv/canaan.yaml
+++ b/Documentation/devicetree/bindings/riscv/canaan.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Damien Le Moal <dlemoal@kernel.org>
 
 description:
-  Canaan Kendryte K210 SoC-based boards
+  Canaan Kendryte SoC-based boards
 
 properties:
   $nodename:
@@ -42,6 +42,12 @@ properties:
       - items:
           - const: canaan,kendryte-k210
 
+      - items:
+          - enum:
+              - canaan,canmv-k230
+              - canaan,k230-usip-lp3-evb
+          - const: canaan,kendryte-k230
+
 additionalProperties: true
 
 ...
diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml
index c6ec9290fe07f..2b0a8a93bb214 100644
--- a/Documentation/devicetree/bindings/riscv/extensions.yaml
+++ b/Documentation/devicetree/bindings/riscv/extensions.yaml
@@ -262,6 +262,23 @@ properties:
             ratified in RISC-V Profiles Version 1.0, with commit b1d806605f87
             ("Updated to ratified state.")
 
+        - const: supm
+          description: |
+            The standard Supm extension for pointer masking support in user
+            mode (U-mode) as ratified at commit d70011dde6c2 ("Update to
+            ratified state") of riscv-j-extension.
+
+            Supm represents a combination of underlying hardware capability
+            (Smnpm or Ssnpm), U-mode consumer privilege level, and M/S-mode
+            software configuration that enables pointer masking for U-mode.
+
+            DO NOT include this property in device trees targeting privileged
+            system software (S-mode or M-mode).
+
+            This property is only appropriate in device trees provided to
+            U-mode software where the next-higher-privilege-mode supports
+            Smnpm or Ssnpm and enables it for U-mode.
+
         - const: svade
           description: |
             The standard Svade supervisor-level extension for SW-managed PTE A/D
@@ -907,6 +924,16 @@ properties:
         then:
           contains:
             const: b
+      # Supm depends on Smnpm or Ssnpm
+      - if:
+          contains:
+            const: supm
+        then:
+          oneOf:
+            - contains:
+                const: smnpm
+            - contains:
+                const: ssnpm
       # Za64rs and Ziccrse depend on Zalrsc or A
       - if:
           contains:
diff --git a/Documentation/devicetree/bindings/riscv/microchip.yaml b/Documentation/devicetree/bindings/riscv/microchip.yaml
index 381d6eb6672e5..137a6f4134308 100644
--- a/Documentation/devicetree/bindings/riscv/microchip.yaml
+++ b/Documentation/devicetree/bindings/riscv/microchip.yaml
@@ -4,14 +4,14 @@
 $id: http://devicetree.org/schemas/riscv/microchip.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Microchip PolarFire SoC-based boards
+title: Microchip SoC-based boards
 
 maintainers:
   - Conor Dooley <conor.dooley@microchip.com>
   - Daire McNamara <daire.mcnamara@microchip.com>
 
 description:
-  Microchip PolarFire SoC-based boards
+  Microchip SoC-based boards
 
 properties:
   $nodename:
@@ -46,6 +46,9 @@ properties:
               - microchip,mpfs-sev-kit
               - sundance,polarberry
           - const: microchip,mpfs
+      - items:
+          - const: microchip,pic64gx-curiosity-kit
+          - const: microchip,pic64gx
 
 additionalProperties: true
 
diff --git a/Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml b/Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml
index f78614100ea8b..3628251b8c516 100644
--- a/Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml
+++ b/Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml
@@ -19,6 +19,7 @@ properties:
           - microchip,sam9x60-trng
       - items:
           - enum:
+              - microchip,lan9691-trng
               - microchip,sama7g5-trng
           - const: atmel,at91sam9g45-trng
       - items:
diff --git a/Documentation/devicetree/bindings/rtc/isil,isl12026.txt b/Documentation/devicetree/bindings/rtc/isil,isl12026.txt
deleted file mode 100644
index 2e0be45193bb9..0000000000000
--- a/Documentation/devicetree/bindings/rtc/isil,isl12026.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-ISL12026 I2C RTC/EEPROM
-
-ISL12026 is an I2C RTC/EEPROM combination device.  The RTC and control
-registers respond at bus address 0x6f, and the EEPROM array responds
-at bus address 0x57.  The canonical "reg" value will be for the RTC portion.
-
-Required properties supported by the device:
-
- - "compatible": must be "isil,isl12026"
- - "reg": I2C bus address of the device (always 0x6f)
-
-Optional properties:
-
- - "isil,pwr-bsw": If present PWR.BSW bit must be set to the specified
-                   value for proper operation.
-
- - "isil,pwr-sbib": If present PWR.SBIB bit must be set to the specified
-                    value for proper operation.
-
-
-Example:
-
-	rtc@6f {
-		compatible = "isil,isl12026";
-		reg = <0x6f>;
-		isil,pwr-bsw = <0>;
-		isil,pwr-sbib = <1>;
-	}
diff --git a/Documentation/devicetree/bindings/rtc/isil,isl12026.yaml b/Documentation/devicetree/bindings/rtc/isil,isl12026.yaml
new file mode 100644
index 0000000000000..152edce2ab416
--- /dev/null
+++ b/Documentation/devicetree/bindings/rtc/isil,isl12026.yaml
@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/rtc/isil,isl12026.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Intersil ISL12026 I2C RTC/EEPROM
+
+maintainers:
+  - Piyush Patle <piyushpatle228@gmail.com>
+
+description:
+  The ISL12026 is a combination RTC and EEPROM device connected via I2C.
+  The RTC and control registers respond at address 0x6f, while the EEPROM
+  array responds at address 0x57. The "reg" property refers to the RTC
+  portion of the device.
+
+allOf:
+  - $ref: rtc.yaml#
+
+properties:
+  compatible:
+    const: isil,isl12026
+
+  reg:
+    maxItems: 1
+    description: I2C address of the RTC portion (must be 0x6f)
+
+  isil,pwr-bsw:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [ 0, 1 ]
+    description:
+      Value written to the PWR.BSW bit for proper device operation.
+
+  isil,pwr-sbib:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [ 0, 1 ]
+    description:
+      Value written to the PWR.SBIB bit for proper device operation.
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        rtc@6f {
+            compatible = "isil,isl12026";
+            reg = <0x6f>;
+            isil,pwr-bsw = <0>;
+            isil,pwr-sbib = <1>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/rtc/microchip,mpfs-rtc.yaml b/Documentation/devicetree/bindings/rtc/microchip,mpfs-rtc.yaml
index a3e60d9f83993..e26e92b1af03f 100644
--- a/Documentation/devicetree/bindings/rtc/microchip,mpfs-rtc.yaml
+++ b/Documentation/devicetree/bindings/rtc/microchip,mpfs-rtc.yaml
@@ -47,6 +47,9 @@ properties:
       - const: rtc
       - const: rtcref
 
+  resets:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/rtc/microcrystal,rv3028.yaml b/Documentation/devicetree/bindings/rtc/microcrystal,rv3028.yaml
index cda8ad7c12037..2ea3b40419530 100644
--- a/Documentation/devicetree/bindings/rtc/microcrystal,rv3028.yaml
+++ b/Documentation/devicetree/bindings/rtc/microcrystal,rv3028.yaml
@@ -32,6 +32,8 @@ properties:
       - 9000
       - 15000
 
+  vdd-supply: true
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/serial/8250.yaml b/Documentation/devicetree/bindings/serial/8250.yaml
index 73851f19330d7..e9e07c2356bc9 100644
--- a/Documentation/devicetree/bindings/serial/8250.yaml
+++ b/Documentation/devicetree/bindings/serial/8250.yaml
@@ -63,7 +63,9 @@ allOf:
       properties:
         compatible:
           contains:
-            const: spacemit,k1-uart
+            enum:
+              - spacemit,k1-uart
+              - spacemit,k3-uart
     then:
       properties:
         clock-names:
@@ -76,6 +78,7 @@ allOf:
           contains:
             enum:
               - spacemit,k1-uart
+              - spacemit,k3-uart
               - nxp,lpc1850-uart
     then:
       required:
diff --git a/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml b/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml
index d8ad1bb6172da..a2702319685d5 100644
--- a/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml
+++ b/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml
@@ -56,6 +56,7 @@ properties:
         items:
           - enum:
               - amlogic,a4-uart
+              - amlogic,a9-uart
               - amlogic,s6-uart
               - amlogic,s7-uart
               - amlogic,s7d-uart
diff --git a/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml b/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml
index 087a8926f8b4e..375cd50bc5cca 100644
--- a/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml
+++ b/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml
@@ -24,6 +24,7 @@ properties:
           - const: atmel,at91sam9260-usart
       - items:
           - enum:
+              - microchip,lan9691-usart
               - microchip,sam9x60-usart
               - microchip,sam9x7-usart
               - microchip,sama7d65-usart
diff --git a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
index 6efe43089a74d..685c1eceb7825 100644
--- a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
+++ b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
@@ -71,6 +71,7 @@ properties:
               - rockchip,rk3568-uart
               - rockchip,rk3576-uart
               - rockchip,rk3588-uart
+              - rockchip,rv1103b-uart
               - rockchip,rv1108-uart
               - rockchip,rv1126-uart
               - sophgo,sg2044-uart
diff --git a/Documentation/devicetree/bindings/soc/cix/cix,sky1-system-control.yaml b/Documentation/devicetree/bindings/soc/cix/cix,sky1-system-control.yaml
new file mode 100644
index 0000000000000..a01a515222c6d
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/cix/cix,sky1-system-control.yaml
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/cix/cix,sky1-system-control.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Cix Sky1 SoC system control register region
+
+maintainers:
+  - Gary Yang <gary.yang@cixtech.com>
+
+description:
+  An wide assortment of registers of the system controller on Sky1 SoC,
+  including resets, usb, wakeup sources and so on.
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - cix,sky1-system-control
+          - cix,sky1-s5-system-control
+      - const: syscon
+
+  reg:
+    maxItems: 1
+
+  '#reset-cells':
+    const: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    syscon@4160000 {
+      compatible = "cix,sky1-system-control", "syscon";
+      reg = <0x4160000 0x100>;
+      #reset-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
index 34aea58094e55..d828c2e82965c 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
@@ -40,6 +40,58 @@ properties:
     minItems: 8
     maxItems: 10
 
+  dpi-bridge:
+    type: object
+    additionalProperties: false
+
+    properties:
+      compatible:
+        enum:
+          - nxp,imx91-pdfc
+          - nxp,imx93-pdfc
+
+      ports:
+        $ref: /schemas/graph.yaml#/properties/ports
+
+        properties:
+          port@0:
+            $ref: /schemas/graph.yaml#/properties/port
+            description: Input port node to receive pixel data.
+
+          port@1:
+            $ref: /schemas/graph.yaml#/$defs/port-base
+            unevaluatedProperties: false
+            description: Output port node to downstream pixel data receivers.
+
+            properties:
+              endpoint:
+                $ref: /schemas/media/video-interfaces.yaml#
+                unevaluatedProperties: false
+
+                properties:
+                  bus-width:
+                    enum: [ 16, 18, 24 ]
+                    description:
+                      Specify the physical parallel bus width.
+
+                      This property is optional if the display bus-width
+                      matches the SoC bus-width, e.g. a 18-bit RGB666 (display)
+                      is connected and all 18-bit data lines are muxed to the
+                      parallel-output pads.
+
+                      This property must be set to 18 to cut only the LSBs
+                      instead of the MSBs in case a 24-bit RGB888 display is
+                      connected and only the lower 18-bit data lanes are muxed
+                      to the parallel-output pads.
+
+        required:
+          - port@0
+          - port@1
+
+    required:
+      - compatible
+      - ports
+
 allOf:
   - if:
       properties:
@@ -112,4 +164,30 @@ examples:
                clock-names = "apb", "axi", "nic", "disp", "cam",
                              "pxp", "lcdif", "isi", "csi", "dsi";
       #power-domain-cells = <1>;
+
+      dpi-bridge {
+        compatible = "nxp,imx93-pdfc";
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+
+          port@0 {
+            reg = <0>;
+
+            pdfc_from_lcdif: endpoint {
+              remote-endpoint = <&lcdif_to_pdfc>;
+            };
+          };
+
+          port@1 {
+            reg = <1>;
+
+            pdfc_to_panel: endpoint {
+              remote-endpoint = <&panel_from_pdfc>;
+              bus-width = <18>;
+            };
+          };
+        };
+      };
     };
diff --git a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-mss-top-sysreg.yaml b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-mss-top-sysreg.yaml
index 44e4a50c31554..2c1efcdf59b75 100644
--- a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-mss-top-sysreg.yaml
+++ b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-mss-top-sysreg.yaml
@@ -15,10 +15,16 @@ description:
 
 properties:
   compatible:
-    items:
-      - const: microchip,mpfs-mss-top-sysreg
-      - const: syscon
-      - const: simple-mfd
+    oneOf:
+      - items:
+          - const: microchip,mpfs-mss-top-sysreg
+          - const: syscon
+          - const: simple-mfd
+      - items:
+          - const: microchip,pic64gx-mss-top-sysreg
+          - const: microchip,mpfs-mss-top-sysreg
+          - const: syscon
+          - const: simple-mfd
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml
index a3fa04f3a1bd9..6cebc19db4f5d 100644
--- a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml
+++ b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml
@@ -24,7 +24,9 @@ properties:
     maxItems: 1
 
   compatible:
-    const: microchip,mpfs-sys-controller
+    enum:
+      - microchip,mpfs-sys-controller
+      - microchip,pic64gx-sys-controller
 
   microchip,bitstream-flash:
     $ref: /schemas/types.yaml#/definitions/phandle
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
index 7085bf88afaba..ff01d2f3ee5be 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
@@ -23,6 +23,8 @@ properties:
     oneOf:
       - items:
           - enum:
+              - qcom,glymur-pmic-glink
+              - qcom,kaanapali-pmic-glink
               - qcom,qcm6490-pmic-glink
               - qcom,sc8180x-pmic-glink
               - qcom,sc8280xp-pmic-glink
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
index 4386b2c3fa4de..94ae72eb8fb65 100644
--- a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
@@ -24,6 +24,7 @@ properties:
       - renesas,r9a07g044-sysc # RZ/G2{L,LC}
       - renesas,r9a07g054-sysc # RZ/V2L
       - renesas,r9a08g045-sysc # RZ/G3S
+      - renesas,r9a08g046-sysc # RZ/G3L
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
index f4947ac65460b..5c22c51b1533d 100644
--- a/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
@@ -548,6 +548,19 @@ properties:
           - const: renesas,r9a08g045s33 # PCIe support
           - const: renesas,r9a08g045
 
+      - description: RZ/G3L (R9A08G046)
+        items:
+          - enum:
+              - renesas,smarc2-evk # RZ SMARC Carrier-II EVK
+          - enum:
+              - renesas,rzg3l-smarcm # RZ/G3L SMARC Module (SoM)
+          - enum:
+              - renesas,r9a08g046l26 # Dual Cortex-A55 + Cortex-M33 + GE3D/VCP (14mm LFBGA)
+              - renesas,r9a08g046l28 # Dual Cortex-A55 + Cortex-M33 + GE3D/VCP (17mm LFBGA)
+              - renesas,r9a08g046l46 # Quad Cortex-A55 + Cortex-M33 + GE3D/VCP (14mm LFBGA)
+              - renesas,r9a08g046l48 # Quad Cortex-A55 + Cortex-M33 + GE3D/VCP (17mm LFBGA)
+          - const: renesas,r9a08g046
+
       - description: RZ/V2M (R9A09G011)
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/sound/adi,ssm2305.txt b/Documentation/devicetree/bindings/sound/adi,ssm2305.txt
deleted file mode 100644
index a9c9d83c8a30a..0000000000000
--- a/Documentation/devicetree/bindings/sound/adi,ssm2305.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-Analog Devices SSM2305 Speaker Amplifier
-========================================
-
-Required properties:
-  - compatible : "adi,ssm2305"
-  - shutdown-gpios : The gpio connected to the shutdown pin.
-                     The gpio signal is ACTIVE_LOW.
-
-Example:
-
-ssm2305: analog-amplifier {
-	compatible = "adi,ssm2305";
-	shutdown-gpios = <&gpio3 20 GPIO_ACTIVE_LOW>;
-};
diff --git a/Documentation/devicetree/bindings/sound/adi,ssm2305.yaml b/Documentation/devicetree/bindings/sound/adi,ssm2305.yaml
new file mode 100644
index 0000000000000..b841da2dc284d
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/adi,ssm2305.yaml
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/adi,ssm2305.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices SSM2305 Class-D Speaker Amplifier
+
+maintainers:
+  - Lars-Peter Clausen <lars@metafoo.de>
+
+description:
+  The SSM2305 is a filterless, high efficiency, mono 2.8 W Class-D
+  audio amplifier with a micropower shutdown mode controlled via a
+  dedicated active-low GPIO pin.
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    const: adi,ssm2305
+
+  shutdown-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the shutdown pin (SD) of the SSM2305.
+      The pin is active-low; asserting it puts the device into
+      micropower shutdown mode.
+
+required:
+  - compatible
+  - shutdown-gpios
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    analog-amplifier {
+        compatible = "adi,ssm2305";
+        shutdown-gpios = <&gpio3 20 GPIO_ACTIVE_LOW>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/awinic,aw88395.yaml b/Documentation/devicetree/bindings/sound/awinic,aw88395.yaml
index 994d68c074a9f..b9abb10942bad 100644
--- a/Documentation/devicetree/bindings/sound/awinic,aw88395.yaml
+++ b/Documentation/devicetree/bindings/sound/awinic,aw88395.yaml
@@ -35,6 +35,10 @@ properties:
 
   dvdd-supply: true
 
+  firmware-name:
+    maxItems: 1
+    description: Name of the *_acf.bin file used for amplifier initialization
+
   awinic,audio-channel:
     description:
       It is used to distinguish multiple PA devices, so that different
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l43.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l43.yaml
index 99a536601cc7e..376928d1f64bf 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs42l43.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l43.yaml
@@ -16,6 +16,8 @@ description: |
   DAC for headphone output, two integrated Class D amplifiers for
   loudspeakers, and two ADCs for wired headset microphone input or
   stereo line input. PDM inputs are provided for digital microphones.
+  CS42L43B variant adds dedicated PDM interface, SoundWire Clock Gearing
+  support and more decimators to ISRCs.
 
 allOf:
   - $ref: dai-common.yaml#
@@ -24,6 +26,7 @@ properties:
   compatible:
     enum:
       - cirrus,cs42l43
+      - cirrus,cs42l43b
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
index 3c75c8c789870..5424d4f16f528 100644
--- a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
+++ b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
@@ -24,6 +24,7 @@ patternProperties:
       cpu/codec dais.
 
     type: object
+    $ref: tdm-slot.yaml#
 
     properties:
       link-name:
@@ -38,13 +39,9 @@ patternProperties:
             - i2s
             - dsp_b
 
-      dai-tdm-slot-num:
-        description: see tdm-slot.txt.
-        $ref: /schemas/types.yaml#/definitions/uint32
+      dai-tdm-slot-num: true
 
-      dai-tdm-slot-width:
-        description: see tdm-slot.txt.
-        $ref: /schemas/types.yaml#/definitions/uint32
+      dai-tdm-slot-width: true
 
       playback-only:
         description: link is used only for playback
diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-cpcap.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-cpcap.yaml
new file mode 100644
index 0000000000000..69af2022d0fab
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-cpcap.yaml
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/nvidia,tegra-audio-cpcap.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NVIDIA Tegra audio complex with CPCAP CODEC
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+allOf:
+  - $ref: nvidia,tegra-audio-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - pattern: '^motorola,tegra-audio-cpcap(-[a-z0-9]+)+$'
+      - const: nvidia,tegra-audio-cpcap
+
+  nvidia,audio-routing:
+    $ref: /schemas/types.yaml#/definitions/non-unique-string-array
+    description:
+      A list of the connections between audio components. Each entry is a
+      pair of strings, the first being the connection's sink, the second
+      being the connection's source. Valid names for sources and sinks are
+      the pins (documented in the binding document), and the jacks on the
+      board.
+    minItems: 2
+    items:
+      enum:
+        # Board Connectors
+        - Speakers
+        - Int Spk
+        - Earpiece
+        - Int Mic
+        - Headset Mic
+        - Internal Mic 1
+        - Internal Mic 2
+        - Headphone
+        - Headphones
+        - Headphone Jack
+        - Mic Jack
+
+        # CODEC Pins
+        - MICR
+        - HSMIC
+        - EMUMIC
+        - MICL
+        - EXTR
+        - EXTL
+        - EP
+        - SPKR
+        - SPKL
+        - LINER
+        - LINEL
+        - HSR
+        - HSL
+        - EMUR
+        - EMUL
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/tegra20-car.h>
+    #include <dt-bindings/soc/tegra-pmc.h>
+    sound {
+        compatible = "motorola,tegra-audio-cpcap-olympus",
+                     "nvidia,tegra-audio-cpcap";
+        nvidia,model = "Motorola Atrix 4G (MB860) CPCAP";
+
+        nvidia,audio-routing =
+                "Headphones", "HSR",
+                "Headphones", "HSL",
+                "Int Spk", "SPKR",
+                "Int Spk", "SPKL",
+                "Earpiece", "EP",
+                "HSMIC", "Mic Jack",
+                "MICR", "Internal Mic 1",
+                "MICL", "Internal Mic 2";
+
+        nvidia,i2s-controller = <&tegra_i2s1>;
+        nvidia,audio-codec = <&cpcap_audio>;
+
+        clocks = <&tegra_car TEGRA20_CLK_PLL_A>,
+                 <&tegra_car TEGRA20_CLK_PLL_A_OUT0>,
+                 <&tegra_car TEGRA20_CLK_CDEV1>;
+        clock-names = "pll_a", "pll_a_out0", "mclk";
+    };
diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-max9808x.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-max9808x.yaml
index 241d20f3aad08..4957645a8e035 100644
--- a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-max9808x.yaml
+++ b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-max9808x.yaml
@@ -35,10 +35,15 @@ properties:
     items:
       enum:
         # Board Connectors
+        - Speakers
         - Int Spk
+        - Headphone
+        - Headphones
         - Headphone Jack
         - Earpiece
         - Headset Mic
+        - Mic Jack
+        - Int Mic
         - Internal Mic 1
         - Internal Mic 2
 
diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8962.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8962.yaml
new file mode 100644
index 0000000000000..2c3bf5a02a34f
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8962.yaml
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/nvidia,tegra-audio-wm8962.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NVIDIA Tegra audio complex with WM8962 CODEC
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+allOf:
+  - $ref: nvidia,tegra-audio-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - pattern: '^[a-z0-9]+,tegra-audio-wm8962(-[a-z0-9]+)+$'
+      - const: nvidia,tegra-audio-wm8962
+
+  nvidia,audio-routing:
+    $ref: /schemas/types.yaml#/definitions/non-unique-string-array
+    description:
+      A list of the connections between audio components. Each entry is a
+      pair of strings, the first being the connection's sink, the second
+      being the connection's source. Valid names for sources and sinks are
+      the pins (documented in the binding document), and the jacks on the
+      board.
+    minItems: 2
+    items:
+      enum:
+        # Board Connectors
+        - Speakers
+        - Int Spk
+        - Earpiece
+        - Int Mic
+        - Headset Mic
+        - Internal Mic 1
+        - Internal Mic 2
+        - Headphone
+        - Headphones
+        - Headphone Jack
+        - Mic Jack
+
+        # CODEC Pins
+        - IN1L
+        - IN1R
+        - IN2L
+        - IN2R
+        - IN3L
+        - IN3R
+        - IN4L
+        - IN4R
+        - DMICDAT
+        - HPOUTL
+        - HPOUTR
+        - SPKOUT
+        - SPKOUTL
+        - SPKOUTR
+
+required:
+  - nvidia,i2s-controller
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/tegra30-car.h>
+    #include <dt-bindings/soc/tegra-pmc.h>
+    sound {
+        compatible = "microsoft,tegra-audio-wm8962-surface-rt",
+                     "nvidia,tegra-audio-wm8962";
+        nvidia,model = "Microsoft Surface RT WM8962";
+
+        nvidia,audio-routing =
+                "Headphone Jack", "HPOUTR",
+                "Headphone Jack", "HPOUTL",
+                "Int Spk", "SPKOUTR",
+                "Int Spk", "SPKOUTL";
+
+        nvidia,i2s-controller = <&tegra_i2s1>;
+        nvidia,audio-codec = <&wm8962>;
+
+        clocks = <&tegra_car TEGRA30_CLK_PLL_A>,
+                 <&tegra_car TEGRA30_CLK_PLL_A_OUT0>,
+                 <&tegra_pmc TEGRA_PMC_CLK_OUT_1>;
+        clock-names = "pll_a", "pll_a_out0", "mclk";
+    };
diff --git a/Documentation/devicetree/bindings/sound/rockchip-spdif.yaml b/Documentation/devicetree/bindings/sound/rockchip-spdif.yaml
index 56c755c22945f..502907dd28b36 100644
--- a/Documentation/devicetree/bindings/sound/rockchip-spdif.yaml
+++ b/Documentation/devicetree/bindings/sound/rockchip-spdif.yaml
@@ -33,6 +33,7 @@ properties:
           - const: rockchip,rk3066-spdif
       - items:
           - enum:
+              - rockchip,rk3576-spdif
               - rockchip,rk3588-spdif
           - const: rockchip,rk3568-spdif
 
diff --git a/Documentation/devicetree/bindings/sound/simple-card.yaml b/Documentation/devicetree/bindings/sound/simple-card.yaml
index 533d0a1da56e3..a14716b2732f0 100644
--- a/Documentation/devicetree/bindings/sound/simple-card.yaml
+++ b/Documentation/devicetree/bindings/sound/simple-card.yaml
@@ -27,14 +27,6 @@ definitions:
     description: dai-link uses bit clock inversion
     $ref: /schemas/types.yaml#/definitions/flag
 
-  dai-tdm-slot-num:
-    description: see tdm-slot.txt.
-    $ref: /schemas/types.yaml#/definitions/uint32
-
-  dai-tdm-slot-width:
-    description: see tdm-slot.txt.
-    $ref: /schemas/types.yaml#/definitions/uint32
-
   system-clock-frequency:
     description: |
       If a clock is specified and a multiplication factor is given with
@@ -115,6 +107,8 @@ definitions:
 
   dai:
     type: object
+    $ref: tdm-slot.yaml#
+
     properties:
       sound-dai:
         maxItems: 1
@@ -133,10 +127,6 @@ definitions:
       bitclock-master:
         $ref: /schemas/types.yaml#/definitions/flag
 
-      dai-tdm-slot-num:
-        $ref: "#/definitions/dai-tdm-slot-num"
-      dai-tdm-slot-width:
-        $ref: "#/definitions/dai-tdm-slot-width"
       clocks:
         maxItems: 1
       system-clock-frequency:
diff --git a/Documentation/devicetree/bindings/sound/tdm-slot.txt b/Documentation/devicetree/bindings/sound/tdm-slot.txt
deleted file mode 100644
index 4bb513ae62fc6..0000000000000
--- a/Documentation/devicetree/bindings/sound/tdm-slot.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-TDM slot:
-
-This specifies audio DAI's TDM slot.
-
-TDM slot properties:
-dai-tdm-slot-num : Number of slots in use.
-dai-tdm-slot-width : Width in bits for each slot.
-dai-tdm-slot-tx-mask : Transmit direction slot mask, optional
-dai-tdm-slot-rx-mask : Receive direction slot mask, optional
-
-For instance:
-	dai-tdm-slot-num = <2>;
-	dai-tdm-slot-width = <8>;
-	dai-tdm-slot-tx-mask = <0 1>;
-	dai-tdm-slot-rx-mask = <1 0>;
-
-And for each specified driver, there could be one .of_xlate_tdm_slot_mask()
-to specify an explicit mapping of the channels and the slots. If it's absent
-the default snd_soc_of_xlate_tdm_slot_mask() will be used to generating the
-tx and rx masks.
-
-For snd_soc_of_xlate_tdm_slot_mask(), the tx and rx masks will use a 1 bit
-for an active slot as default, and the default active bits are at the LSB of
-the masks.
-
-The explicit masks are given as array of integers, where the first
-number presents bit-0 (LSB), second presents bit-1, etc. Any non zero
-number is considered 1 and 0 is 0. snd_soc_of_xlate_tdm_slot_mask()
-does not do anything, if either mask is set non zero value.
diff --git a/Documentation/devicetree/bindings/sound/tdm-slot.yaml b/Documentation/devicetree/bindings/sound/tdm-slot.yaml
new file mode 100644
index 0000000000000..457a899e8872e
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/tdm-slot.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/tdm-slot.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Time Division Multiplexing (TDM) Slot Parameters
+
+maintainers:
+  - Liam Girdwood <lgirdwood@gmail.com>
+
+select: false
+
+properties:
+  dai-tdm-slot-num:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Number of slots in use
+
+  dai-tdm-slot-width:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Width, in bits, of each slot
+
+  dai-tdm-idle-mode:
+    $ref: /schemas/types.yaml#/definitions/string
+    enum:
+      - none
+      - off
+      - zero
+      - pulldown
+      - hiz
+      - pullup
+      - drivehigh
+    description: Drive mode for inactive/idle TDM slots. For hardware that
+      implements .set_tdm_idle(). Optional. "None" represents undefined
+      behaviour and is the same as not setting this property.
+
+patternProperties:
+  '^dai-tdm-slot-[rt]x-mask$':
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description: Slot mask for active TDM slots. Optional. Drivers may
+      specify .xlate_tdm_slot_mask() to generate a slot mask dynamically. If
+      neither this property nor a driver-specific function are specified, the
+      default snd_soc_xlate_tdm_slot_mask() function will be used to generate
+      a mask. The first element of the array is slot 0 (LSB). Any nonzero
+      value will be treated as 1.
+
+  '^dai-tdm-slot-[rt]x-idle-mask$':
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Idle slot mask. Optional. A bit being set to 1 indicates
+      that the corresponding TDM slot is inactive/idle.
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/sound/ti,tas2770.yaml b/Documentation/devicetree/bindings/sound/ti,tas2770.yaml
index 8eab98a0f7a25..8d49fbcf0b9b7 100644
--- a/Documentation/devicetree/bindings/sound/ti,tas2770.yaml
+++ b/Documentation/devicetree/bindings/sound/ti,tas2770.yaml
@@ -30,7 +30,7 @@ properties:
     description: |
        I2C address of the device can be between 0x41 to 0x48.
 
-  reset-gpio:
+  reset-gpios:
     maxItems: 1
     description: GPIO used to reset the device.
 
@@ -82,7 +82,7 @@ examples:
             #sound-dai-cells = <0>;
             interrupt-parent = <&gpio1>;
             interrupts = <14>;
-            reset-gpio = <&gpio1 15 GPIO_ACTIVE_HIGH>;
+            reset-gpios = <&gpio1 15 GPIO_ACTIVE_HIGH>;
             shutdown-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>;
             ti,imon-slot-no = <0>;
             ti,vmon-slot-no = <2>;
diff --git a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
index 636338d24bdfb..8ff50dfcf5855 100644
--- a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
@@ -35,10 +35,10 @@ properties:
   interrupts:
     maxItems: 1
 
-  clock-names:
+  clocks:
     maxItems: 1
 
-  clocks:
+  resets:
     maxItems: 1
 
   microchip,apb-datawidth:
diff --git a/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml b/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
index a588b112e11e2..2c9045fd51dec 100644
--- a/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
+++ b/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
@@ -16,7 +16,9 @@ properties:
           - renesas,r9a09g057-rspi # RZ/V2H(P)
           - renesas,r9a09g077-rspi # RZ/T2H
       - items:
-          - const: renesas,r9a09g056-rspi # RZ/V2N
+          - enum:
+              - renesas,r9a09g047-rspi # RZ/G3E
+              - renesas,r9a09g056-rspi # RZ/V2N
           - const: renesas,r9a09g057-rspi
       - items:
           - const: renesas,r9a09g087-rspi # RZ/N2H
@@ -58,12 +60,19 @@ properties:
       - const: tresetn
 
   dmas:
-    maxItems: 2
+    minItems: 2
+    maxItems: 10
+    description:
+      Must contain a list of pairs of references to DMA specifiers, one for
+      transmission, and one for reception.
 
   dma-names:
+    minItems: 2
+    maxItems: 10
     items:
-      - const: rx
-      - const: tx
+      enum:
+        - rx
+        - tx
 
   power-domains:
     maxItems: 1
@@ -121,6 +130,12 @@ allOf:
         resets: false
         reset-names: false
 
+        dmas:
+          maxItems: 6
+
+        dma-names:
+          maxItems: 6
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/sram/qcom,imem.yaml b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
index 6a627c57ae2fe..c63026904061d 100644
--- a/Documentation/devicetree/bindings/sram/qcom,imem.yaml
+++ b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
@@ -67,6 +67,20 @@ properties:
     $ref: /schemas/power/reset/syscon-reboot-mode.yaml#
 
 patternProperties:
+  "^modem-tables@[0-9a-f]+$":
+    type: object
+    description:
+      Region containing packet processing configuration for the IP Accelerator.
+
+    properties:
+      reg:
+        maxItems: 1
+
+    required:
+      - reg
+
+    additionalProperties: false
+
   "^pil-reloc@[0-9a-f]+$":
     $ref: /schemas/remoteproc/qcom,pil-info.yaml#
     description: Peripheral image loader relocation region
diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
index 3bab40500df9b..7936aebe17654 100644
--- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml
+++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
@@ -31,6 +31,7 @@ properties:
           - enum:
               - canaan,k210-clint       # Canaan Kendryte K210
               - eswin,eic7700-clint     # ESWIN EIC7700
+              - microchip,pic64gx-clint # Microchip PIC64GX
               - sifive,fu540-c000-clint # SiFive FU540
               - spacemit,k1-clint       # SpacemiT K1
               - spacemit,k3-clint       # SpacemiT K3
@@ -48,6 +49,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-clint
+              - canaan,k230-clint
               - sophgo,cv1800b-clint
               - sophgo,cv1812h-clint
               - sophgo,sg2002-clint
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index a482aeadcd444..03a2749432236 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -59,6 +59,10 @@ properties:
           - adi,lt7182s
             # AMS iAQ-Core VOC Sensor
           - ams,iaq-core
+            # Aosong temperature & humidity sensors with I2C interface
+          - aosong,aht10
+          - aosong,aht20
+          - aosong,dht20
             # Arduino microcontroller interface over SPI on UnoQ board
           - arduino,unoq-mcu
             # Temperature monitoring of Astera Labs PT5161L PCIe retimer
@@ -97,6 +101,10 @@ properties:
           - delta,dps920ab
             # 1/4 Brick DC/DC Regulated Power Module
           - delta,q54sj108a2
+            # 1300W 1/4 Brick DC/DC Regulated Power Module
+          - delta,q54sn120a1
+            # 2000W 1/4 Brick DC/DC Regulated Power Module
+          - delta,q54sw120a7
             # Devantech SRF02 ultrasonic ranger in I2C mode
           - devantech,srf02
             # Devantech SRF08 ultrasonic ranger
@@ -157,6 +165,9 @@ properties:
           - infineon,xdpe15284
             # Infineon Multi-phase Digital VR Controller xdpe152c4
           - infineon,xdpe152c4
+            # Infineon Multi-phase Digital VR Controller xdpe1a2g7b
+          - infineon,xdpe1a2g5b
+          - infineon,xdpe1a2g7b
             # Injoinic IP5108 2.0A Power Bank IC with I2C
           - injoinic,ip5108
             # Injoinic IP5109 2.1A Power Bank IC with I2C
diff --git a/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml
index d94ef4e6b85a4..3c407426d6972 100644
--- a/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml
+++ b/Documentation/devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml
@@ -15,6 +15,7 @@ select:
     compatible:
       contains:
         enum:
+          - qcom,milos-ufshc
           - qcom,msm8998-ufshc
           - qcom,qcs8300-ufshc
           - qcom,sa8775p-ufshc
@@ -31,21 +32,28 @@ select:
 
 properties:
   compatible:
-    items:
-      - enum:
-          - qcom,msm8998-ufshc
-          - qcom,qcs8300-ufshc
-          - qcom,sa8775p-ufshc
-          - qcom,sc7180-ufshc
-          - qcom,sc7280-ufshc
-          - qcom,sc8180x-ufshc
-          - qcom,sc8280xp-ufshc
-          - qcom,sm8250-ufshc
-          - qcom,sm8350-ufshc
-          - qcom,sm8450-ufshc
-          - qcom,sm8550-ufshc
-      - const: qcom,ufshc
-      - const: jedec,ufs-2.0
+    oneOf:
+      - items:
+          - enum:
+              - qcom,x1e80100-ufshc
+          - const: qcom,sm8550-ufshc
+          - const: qcom,ufshc
+      - items:
+          - enum:
+              - qcom,milos-ufshc
+              - qcom,msm8998-ufshc
+              - qcom,qcs8300-ufshc
+              - qcom,sa8775p-ufshc
+              - qcom,sc7180-ufshc
+              - qcom,sc7280-ufshc
+              - qcom,sc8180x-ufshc
+              - qcom,sc8280xp-ufshc
+              - qcom,sm8250-ufshc
+              - qcom,sm8350-ufshc
+              - qcom,sm8450-ufshc
+              - qcom,sm8550-ufshc
+          - const: qcom,ufshc
+          - const: jedec,ufs-2.0
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
index cea84ab2204f4..f28641c6e68fe 100644
--- a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
+++ b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
@@ -15,6 +15,7 @@ select:
     compatible:
       contains:
         enum:
+          - qcom,eliza-ufshc
           - qcom,kaanapali-ufshc
           - qcom,sm8650-ufshc
           - qcom,sm8750-ufshc
@@ -25,6 +26,7 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,eliza-ufshc
           - qcom,kaanapali-ufshc
           - qcom,sm8650-ufshc
           - qcom,sm8750-ufshc
@@ -66,6 +68,18 @@ required:
 
 allOf:
   - $ref: qcom,ufs-common.yaml
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-ufshc
+    then:
+      properties:
+        reg:
+          minItems: 2
+        reg-names:
+          minItems: 2
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml b/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml
index c7d17cf4dc42b..e738153a309c8 100644
--- a/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml
+++ b/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml
@@ -41,7 +41,7 @@ properties:
     maxItems: 1
 
   resets:
-    maxItems: 4
+    maxItems: 5
 
   reset-names:
     items:
@@ -49,6 +49,7 @@ properties:
       - const: sys
       - const: ufs
       - const: grf
+      - const: mphy
 
   reset-gpios:
     maxItems: 1
@@ -98,8 +99,8 @@ examples:
             interrupts = <GIC_SPI 361 IRQ_TYPE_LEVEL_HIGH>;
             power-domains = <&power RK3576_PD_USB>;
             resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, <&cru SRST_A_UFS>,
-                     <&cru SRST_P_UFS_GRF>;
-            reset-names = "biu", "sys", "ufs", "grf";
+                     <&cru SRST_P_UFS_GRF>, <&cru SRST_MPHY_INIT>;
+            reset-names = "biu", "sys", "ufs", "grf", "mphy";
             reset-gpios = <&gpio4 RK_PD0 GPIO_ACTIVE_LOW>;
         };
     };
diff --git a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
index f454ddd9bbaa6..a199e5ba64161 100644
--- a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
+++ b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
@@ -85,6 +85,7 @@ required:
 
 allOf:
   - $ref: usb-drd.yaml#
+  - $ref: usb-xhci.yaml#
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/usb/corechips,sl6341.yaml b/Documentation/devicetree/bindings/usb/corechips,sl6341.yaml
new file mode 100644
index 0000000000000..82996791aaf17
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/corechips,sl6341.yaml
@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/corechips,sl6341.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Corechips SL6341 USB 2.0/3.0 Hub Controller
+
+maintainers:
+  - Alexey Charkov <alchark@flipper.net>
+
+allOf:
+  - $ref: usb-hub.yaml#
+
+properties:
+  compatible:
+    enum:
+      - usb3431,6241
+      - usb3431,6341
+
+  reg: true
+
+  peer-hub: true
+
+  reset-gpios:
+    description: GPIO controlling the RSTN pin.
+
+  vdd1v1-supply:
+    description:
+      The regulator that provides 1.1V core power to the hub.
+
+  vdd3v3-supply:
+    description:
+      The regulator that provides 3.3V IO power to the hub.
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    patternProperties:
+      '^port@':
+        $ref: /schemas/graph.yaml#/properties/port
+
+        properties:
+          reg:
+            minimum: 1
+            maximum: 4
+
+required:
+  - compatible
+  - reg
+  - vdd1v1-supply
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    usb {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        /* 2.0 hub */
+        hub_2_0: hub@1 {
+          compatible = "usb3431,6241";
+          reg = <1>;
+          peer-hub = <&hub_3_0>;
+          reset-gpios = <&gpio0 20 GPIO_ACTIVE_LOW>;
+          vdd1v1-supply = <&vdd1v1_hub>;
+        };
+
+        /* 3.0 hub */
+        hub_3_0: hub@2 {
+          compatible = "usb3431,6341";
+          reg = <2>;
+          peer-hub = <&hub_2_0>;
+          reset-gpios = <&gpio0 20 GPIO_ACTIVE_LOW>;
+          vdd1v1-supply = <&vdd1v1_hub>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml b/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml
index 73e7a60a0060d..66d368e65c0a2 100644
--- a/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml
+++ b/Documentation/devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml
@@ -10,6 +10,8 @@ title: NXP iMX8MP Soc USB Controller
 maintainers:
   - Li Jun <jun.li@nxp.com>
 
+deprecated: true
+
 properties:
   compatible:
     oneOf:
diff --git a/Documentation/devicetree/bindings/usb/maxim,max33359.yaml b/Documentation/devicetree/bindings/usb/maxim,max33359.yaml
index 3de4dc40b7919..46a3748c8be4d 100644
--- a/Documentation/devicetree/bindings/usb/maxim,max33359.yaml
+++ b/Documentation/devicetree/bindings/usb/maxim,max33359.yaml
@@ -75,6 +75,10 @@ examples:
                                        PDO_FIXED(9000, 2000, 0)>;
                 sink-bc12-completion-time-ms = <500>;
                 pd-revision = /bits/ 8 <0x03 0x01 0x01 0x08>;
+                sink-load-step = <150>;
+                sink-load-characteristics = /bits/ 16 <SINK_LOAD_CHAR(0, 1, 1, 2)>;
+                sink-compliance = /bits/ 8 <(COMPLIANCE_LPS | COMPLIANCE_PS1)>;
+                charging-adapter-pdp-milliwatt = <18000>;
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/usb/maxim,max3421.txt b/Documentation/devicetree/bindings/usb/maxim,max3421.txt
deleted file mode 100644
index 90495b1aeec27..0000000000000
--- a/Documentation/devicetree/bindings/usb/maxim,max3421.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-Maxim Integrated SPI-based USB 2.0 host controller MAX3421E
-
-Required properties:
- - compatible: Should be "maxim,max3421"
- - spi-max-frequency: maximum frequency for this device must not exceed 26 MHz.
- - reg: chip select number to which this device is connected.
- - maxim,vbus-en-pin: <GPOUTx ACTIVE_LEVEL>
-   GPOUTx is the number (1-8) of the GPOUT pin of MAX3421E to drive Vbus.
-   ACTIVE_LEVEL is 0 or 1.
- - interrupts: the interrupt line description for the interrupt controller.
-   The driver configures MAX3421E for active low level triggered interrupts,
-   configure your interrupt line accordingly.
-
-Example:
-
-	usb@0 {
-		compatible = "maxim,max3421";
-		reg = <0>;
-		maxim,vbus-en-pin = <3 1>;
-		spi-max-frequency = <26000000>;
-		interrupt-parent = <&PIC>;
-		interrupts = <42>;
-	};
diff --git a/Documentation/devicetree/bindings/usb/maxim,max3421.yaml b/Documentation/devicetree/bindings/usb/maxim,max3421.yaml
new file mode 100644
index 0000000000000..4639be7ab059d
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/maxim,max3421.yaml
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/maxim,max3421.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MAXIM MAX3421e USB Peripheral/Host Controller
+
+maintainers:
+  - David Mosberger <davidm@egauge.net>
+
+description: |
+  The controller provides USB2.0 compliant with Full Speed or Low Speed when in
+  the host mode. At peripheral, it operates at Full Speed. At both cases, it
+  uses a SPI interface.
+  Datasheet at:
+    https://www.analog.com/media/en/technical-documentation/data-sheets/max3421e.pdf
+
+properties:
+  compatible:
+    const: maxim,max3421
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 26000000
+
+  maxim,vbus-en-pin:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      One of eight GPOUT pins to control external VBUS power and the polarity
+      of the active level. It's an array of GPIO number and the active level of it.
+    minItems: 2
+    maxItems: 2
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - maxim,vbus-en-pin
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        usb@0 {
+          compatible = "maxim,max3421";
+          reg = <0>;
+          maxim,vbus-en-pin = <3 1>;
+          spi-max-frequency = <26000000>;
+          interrupt-parent = <&gpio>;
+          interrupts = <42>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml b/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml
index a812317d80893..c4e1c2d73bdb3 100644
--- a/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml
+++ b/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml
@@ -37,6 +37,9 @@ properties:
   clocks:
     maxItems: 1
 
+  resets:
+    maxItems: 1
+
   microchip,ext-vbus-drv:
     description:
       Some ULPI USB PHYs do not support an internal VBUS supply and driving
diff --git a/Documentation/devicetree/bindings/usb/nxp,imx-dwc3.yaml b/Documentation/devicetree/bindings/usb/nxp,imx-dwc3.yaml
new file mode 100644
index 0000000000000..1911e71f01eb7
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/nxp,imx-dwc3.yaml
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright 2026 NXP
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/nxp,imx-dwc3.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP i.MX Soc USB Controller
+
+maintainers:
+  - Xu Yang <xu.yang_2@nxp.com>
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - nxp,imx94-dwc3
+              - nxp,imx95-dwc3
+          - const: nxp,imx8mp-dwc3
+      - const: nxp,imx8mp-dwc3
+
+  reg:
+    items:
+      - description: DWC3 core registers
+      - description: HSIO Block Control registers
+      - description: Wrapper registers of dwc3 core
+
+  reg-names:
+    items:
+      - const: core
+      - const: blkctl
+      - const: glue
+
+  interrupts:
+    items:
+      - description: DWC3 controller interrupt
+      - description: Wakeup interrupt from glue logic
+
+  interrupt-names:
+    items:
+      - const: dwc_usb3
+      - const: wakeup
+
+  iommus:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: System hsio root clock
+      - description: SoC Bus Clock for AHB/AXI/Native
+      - description: Reference clock for generating ITP when UTMI/ULPI PHY is suspended
+      - description: Suspend clock used for usb wakeup logic
+
+  clock-names:
+    items:
+      - const: hsio
+      - const: bus_early
+      - const: ref
+      - const: suspend
+
+  fsl,permanently-attached:
+    type: boolean
+    description:
+      Indicates if the device attached to a downstream port is
+      permanently attached
+
+  fsl,disable-port-power-control:
+    type: boolean
+    description:
+      Indicates whether the host controller implementation includes port
+      power control. Defines Bit 3 in capability register (HCCPARAMS)
+
+  fsl,over-current-active-low:
+    type: boolean
+    description:
+      Over current signal polarity is active low
+
+  fsl,power-active-low:
+    type: boolean
+    description:
+      Power pad (PWR) polarity is active low
+
+  power-domains:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - interrupts
+  - power-domains
+
+allOf:
+  - $ref: snps,dwc3-common.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    usb@4c100000 {
+      compatible = "nxp,imx94-dwc3", "nxp,imx8mp-dwc3";
+      reg = <0x4c100000 0x10000>,
+            <0x4c010010 0x04>,
+            <0x4c1f0000 0x20>;
+      reg-names = "core", "blkctl", "glue";
+      clocks = <&scmi_clk 74>,    //IMX94_CLK_HSIO
+               <&scmi_clk 74>,    //IMX94_CLK_HSIO
+               <&scmi_clk 2>,     //IMX94_CLK_24M
+               <&scmi_clk 1>;     //IMX94_CLK_32K
+      clock-names = "hsio", "bus_early", "ref", "suspend";
+      interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>,
+                   <GIC_SPI 386 IRQ_TYPE_LEVEL_HIGH>;
+      interrupt-names = "dwc_usb3", "wakeup";
+      power-domains = <&scmi_devpd 13>;   //IMX94_PD_HSIO_TOP
+      phys = <&usb3_phy>, <&usb3_phy>;
+      phy-names = "usb2-phy", "usb3-phy";
+      snps,gfladj-refclk-lpm-sel-quirk;
+      snps,parkmode-disable-ss-quirk;
+    };
diff --git a/Documentation/devicetree/bindings/usb/ohci-st.txt b/Documentation/devicetree/bindings/usb/ohci-st.txt
deleted file mode 100644
index 1c735573abc03..0000000000000
--- a/Documentation/devicetree/bindings/usb/ohci-st.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-ST USB OHCI controller
-
-Required properties:
-
- - compatible		: must be "st,st-ohci-300x"
- - reg			: physical base addresses of the controller and length of memory mapped
-			  region
- - interrupts		: one OHCI controller interrupt should be described here
- - clocks		: phandle list of usb clocks
- - clock-names		: should be "ic" for interconnect clock and "clk48"
-See: Documentation/devicetree/bindings/clock/clock-bindings.txt
-
- - phys			: phandle for the PHY device
- - phy-names		: should be "usb"
-
- - resets		: phandle to the powerdown and reset controller for the USB IP
- - reset-names		: should be "power" and "softreset".
-See: Documentation/devicetree/bindings/reset/st,stih407-powerdown.yaml
-See: Documentation/devicetree/bindings/reset/reset.txt
-
-Example:
-
-	ohci0: usb@fe1ffc00 {
-		compatible = "st,st-ohci-300x";
-		reg = <0xfe1ffc00 0x100>;
-		interrupts = <GIC_SPI 149 IRQ_TYPE_NONE>;
-		clocks = <&clk_s_a1_ls 0>,
-			 <&clockgen_b0 0>;
-		clock-names = "ic", "clk48";
-		phys = <&usb2_phy>;
-		phy-names = "usb";
-
-		resets = <&powerdown STIH416_USB0_POWERDOWN>,
-			 <&softreset STIH416_USB0_SOFTRESET>;
-		reset-names = "power", "softreset";
-	};
diff --git a/Documentation/devicetree/bindings/usb/omap-usb.txt b/Documentation/devicetree/bindings/usb/omap-usb.txt
deleted file mode 100644
index f0dbc5ae45ae8..0000000000000
--- a/Documentation/devicetree/bindings/usb/omap-usb.txt
+++ /dev/null
@@ -1,80 +0,0 @@
-OMAP GLUE AND OTHER OMAP SPECIFIC COMPONENTS
-
-OMAP MUSB GLUE
- - compatible : Should be "ti,omap4-musb" or "ti,omap3-musb"
- - ti,hwmods : must be "usb_otg_hs"
- - multipoint : Should be "1" indicating the musb controller supports
-   multipoint. This is a MUSB configuration-specific setting.
- - num-eps : Specifies the number of endpoints. This is also a
-   MUSB configuration-specific setting. Should be set to "16"
- - ram-bits : Specifies the ram address size. Should be set to "12"
- - interface-type : This is a board specific setting to describe the type of
-   interface between the controller and the phy. It should be "0" or "1"
-   specifying ULPI and UTMI respectively.
- - mode : Should be "3" to represent OTG. "1" signifies HOST and "2"
-   represents PERIPHERAL.
- - power : Should be "50". This signifies the controller can supply up to
-   100mA when operating in host mode.
- - usb-phy : the phandle for the PHY device
- - phys : the phandle for the PHY device (used by generic PHY framework)
- - phy-names : the names of the PHY corresponding to the PHYs present in the
-   *phy* phandle.
-
-Optional properties:
- - ctrl-module : phandle of the control module this glue uses to write to
-   mailbox
-
-SOC specific device node entry
-usb_otg_hs: usb_otg_hs@4a0ab000 {
-	compatible = "ti,omap4-musb";
-	ti,hwmods = "usb_otg_hs";
-	multipoint = <1>;
-	num-eps = <16>;
-	ram-bits = <12>;
-	ctrl-module = <&omap_control_usb>;
-	phys = <&usb2_phy>;
-	phy-names = "usb2-phy";
-};
-
-Board specific device node entry
-&usb_otg_hs {
-	interface-type = <1>;
-	mode = <3>;
-	power = <50>;
-};
-
-OMAP DWC3 GLUE
- - compatible : Should be
-	* "ti,dwc3" for OMAP5 and DRA7
-	* "ti,am437x-dwc3" for AM437x
- - ti,hwmods : Should be "usb_otg_ss"
- - reg : Address and length of the register set for the device.
- - interrupts : The irq number of this device that is used to interrupt the
-   MPU
- - #address-cells, #size-cells : Must be present if the device has sub-nodes
- - utmi-mode : controls the source of UTMI/PIPE status for VBUS and OTG ID.
-   It should be set to "1" for HW mode and "2" for SW mode.
- - ranges: the child address space are mapped 1:1 onto the parent address space
-
-Optional Properties:
- - extcon : phandle for the extcon device omap dwc3 uses to detect
-   connect/disconnect events.
- - vbus-supply : phandle to the regulator device tree node if needed.
-
-Sub-nodes:
-The dwc3 core should be added as subnode to omap dwc3 glue.
-- dwc3 :
-   The binding details of dwc3 can be found in:
-   Documentation/devicetree/bindings/usb/snps,dwc3.yaml
-
-omap_dwc3 {
-	compatible = "ti,dwc3";
-	ti,hwmods = "usb_otg_ss";
-	reg = <0x4a020000 0x1ff>;
-	interrupts = <0 93 4>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-	utmi-mode = <2>;
-	ranges;
-};
-
diff --git a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
index 7d784a648b7d9..f879e2e104c4c 100644
--- a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
+++ b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
@@ -500,7 +500,7 @@ allOf:
             - const: pwr_event
             - const: dp_hs_phy_irq
             - const: dm_hs_phy_irq
-            - const: ss_phy_irq
+            - enum: [hs_phy_irq, ss_phy_irq]
 
   - if:
       properties:
diff --git a/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
index ae611f7e57ca4..1eb611f35998b 100644
--- a/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
+++ b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
@@ -19,10 +19,11 @@ description: |
 properties:
   compatible:
     enum:
+      - etekmicro,et7304
       - richtek,rt1711h
       - richtek,rt1715
     description:
-      RT1711H support PD20, RT1715 support PD30 except Fast Role Swap.
+      RT1711H support PD20, ET7304 and RT1715 support PD30 except Fast Role Swap.
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/usb/st,st-ohci-300x.yaml b/Documentation/devicetree/bindings/usb/st,st-ohci-300x.yaml
new file mode 100644
index 0000000000000..a225bf5a2ee40
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/st,st-ohci-300x.yaml
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/st,st-ohci-300x.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: STMicroelectronics USB OHCI Controller
+
+maintainers:
+  - Peter Griffin <peter.griffin@linaro.org>
+
+description:
+  The STMicroelectronics USB Open Host Controller Interface (OHCI)
+  compliant USB host controller found in ST platforms. The controller
+  provides full- and low-speed USB host functionality and interfaces
+  with an external USB PHY. It requires dedicated clock, reset, and
+  interrupt resources for proper operation.
+
+allOf:
+  - $ref: /schemas/usb/usb-hcd.yaml#
+
+properties:
+  compatible:
+    const: st,st-ohci-300x
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 2
+
+  clock-names:
+    items:
+      - const: ic
+      - const: clk48
+
+  phys:
+    maxItems: 1
+
+  phy-names:
+    items:
+      - const: usb
+
+  resets:
+    maxItems: 2
+
+  reset-names:
+    items:
+      - const: power
+      - const: softreset
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - phys
+  - phy-names
+  - resets
+  - reset-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/reset/stih407-resets.h>
+    usb@fe1ffc00 {
+        compatible = "st,st-ohci-300x";
+        reg = <0xfe1ffc00 0x100>;
+        interrupts = <GIC_SPI 149 IRQ_TYPE_NONE>;
+        clocks = <&clk_s_a1_ls 0>,
+                 <&clockgen_b0 0>;
+        clock-names = "ic", "clk48";
+        phys = <&usb2_phy>;
+        phy-names = "usb";
+        resets = <&powerdown STIH407_USB2_PORT0_POWERDOWN>,
+                 <&softreset STIH407_USB2_PORT0_SOFTRESET>;
+        reset-names = "power", "softreset";
+    };
+...
diff --git a/Documentation/devicetree/bindings/usb/ti,dwc3.yaml b/Documentation/devicetree/bindings/usb/ti,dwc3.yaml
new file mode 100644
index 0000000000000..77ac11c3b2db2
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,dwc3.yaml
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,dwc3.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments OMAP DWC3 USB Glue Layer
+
+maintainers:
+  - Felipe Balbi <balbi@ti.com>
+
+description:
+  Texas Instruments glue layer for Synopsys DesignWare USB3 (DWC3)
+  controller on OMAP and AM43xx SoCs. Manages SoC-specific integration
+  including register mapping, interrupt routing, UTMI/PIPE interface mode
+  selection (HW/SW), and child DWC3 core instantiation via address space
+  translation. Supports both legacy single-instance and multi-instance
+  (numbered) configurations.
+
+properties:
+  compatible:
+    enum:
+      - ti,dwc3
+      - ti,am437x-dwc3
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  utmi-mode:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Controls the source of UTMI/PIPE status for VBUS and OTG ID.
+      1 for HW mode, 2 for SW mode.
+    enum: [1, 2]
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ranges: true
+
+  extcon:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Phandle for the extcon device used to detect connect/
+      disconnect events.
+
+  vbus-supply:
+    description: Phandle to the regulator device tree node if needed.
+
+patternProperties:
+  "^usb@[0-9a-f]+$":
+    type: object
+    $ref: snps,dwc3.yaml#
+    unevaluatedProperties: false
+
+required:
+  - reg
+  - compatible
+  - interrupts
+  - "#address-cells"
+  - "#size-cells"
+  - utmi-mode
+  - ranges
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    omap_dwc3_1@0 {
+        compatible = "ti,dwc3";
+        reg = <0x0 0x10000>;
+        interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        utmi-mode = <2>;
+        ranges = <0 0 0x20000>;
+
+        usb@10000 {
+            compatible = "snps,dwc3";
+            reg = <0x10000 0x17000>;
+            interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+            interrupt-names = "peripheral", "host", "otg";
+            phys = <&usb2_phy1>, <&usb3_phy1>;
+            phy-names = "usb2-phy", "usb3-phy";
+            maximum-speed = "super-speed";
+            dr_mode = "otg";
+            snps,dis_u3_susphy_quirk;
+            snps,dis_u2_susphy_quirk;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml b/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
new file mode 100644
index 0000000000000..a3d15f2176586
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,omap4-musb.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments OMAP MUSB USB OTG Controller
+
+maintainers:
+  - Felipe Balbi <balbi@ti.com>
+
+description:
+  Texas Instruments glue layer for the Mentor Graphics MUSB OTG controller.
+  Handles SoC-specific integration including PHY interface bridging(ULPI/
+  UTMI), interrupt aggregation, DMA engine coordination (internal/
+  external), VBUS/session control via control module mailbox, and
+  clock/reset management. Provides fixed hardware configuration parameters
+  to the generic MUSB core driver.
+
+properties:
+  compatible:
+    enum:
+      - ti,omap3-musb
+      - ti,omap4-musb
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    minItems: 1
+    maxItems: 2
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - const: mc
+      - const: dma
+
+  multipoint:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Indicates the MUSB controller supports multipoint. This is a MUSB
+      configuration-specific setting.
+    const: 1
+
+  num-eps:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Specifies the number of endpoints. This is a MUSB configuration
+      specific setting.
+    const: 16
+
+  ram-bits:
+    description: Specifies the RAM address size.
+    const: 12
+
+  interface-type:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Describes the type of interface between the controller and the PHY.
+      0 for ULPI, 1 for UTMI.
+    enum: [0, 1]
+
+  mode:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: 1 for HOST, 2 for PERIPHERAL, 3 for OTG.
+    enum: [1, 2, 3]
+
+  power:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Indicates the maximum current the controller can supply when
+      operating in host mode. A value of 50 corresponds to 100 mA, and a
+      value of 150 corresponds to 300 mA.
+    enum: [50, 150]
+
+  phys:
+    maxItems: 1
+
+  phy-names:
+    const: usb2-phy
+
+  usb-phy:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    description: Phandle for the PHY device.
+    deprecated: true
+
+  ctrl-module:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Phandle of the control module this glue uses to write to mailbox.
+
+required:
+  - reg
+  - compatible
+  - interrupts
+  - interrupt-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    usb@4a0ab000 {
+        compatible = "ti,omap4-musb";
+        reg = <0x4a0ab000 0x1000>;
+        interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "mc", "dma";
+        multipoint = <1>;
+        num-eps = <16>;
+        ram-bits = <12>;
+        ctrl-module = <&omap_control_usb>;
+        phys = <&usb2_phy>;
+        phy-names = "usb2-phy";
+        interface-type = <1>;
+        mode = <3>;
+        power = <50>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index ee7fd3cfe2039..306343407019f 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -361,6 +361,8 @@ patternProperties:
     description: CORERIVER Semiconductor Co.,Ltd.
   "^corpro,.*":
     description: Chengdu Corpro Technology Co., Ltd.
+  "^corechips,.*":
+    description: Shenzhen Corechips Microelectronics Co., Ltd.
   "^cortina,.*":
     description: Cortina Systems, Inc.
   "^cosmic,.*":
@@ -441,6 +443,8 @@ patternProperties:
     description: D-Link Corporation
   "^dmo,.*":
     description: Data Modul AG
+  "^doestek,.*":
+    description: Doestek Co., Ltd.
   "^domintech,.*":
     description: Domintech Co., Ltd.
   "^dongwoon,.*":
@@ -541,6 +545,8 @@ patternProperties:
     description: ESTeem Wireless Modems
   "^eswin,.*":
     description: Beijing ESWIN Technology Group Co. Ltd.
+  "^etekmicro,.*":
+    description: Wuxi ETEK Micro-Electronics Co.,Ltd.
   "^ettus,.*":
     description: NI Ettus Research
   "^eukrea,.*":
@@ -973,6 +979,8 @@ patternProperties:
     description: Liebherr-Werk Nenzing GmbH
   "^lxa,.*":
     description: Linux Automation GmbH
+  "^lxd,.*":
+    description: LXD Research & Display, LLC
   "^m5stack,.*":
     description: M5Stack
   "^macnica,.*":
@@ -1199,6 +1207,8 @@ patternProperties:
     description: One Laptop Per Child
   "^oneplus,.*":
     description: OnePlus Technology (Shenzhen) Co., Ltd.
+  "^onething,.*":
+    description: Shenzhen OneThing Technologies Co., Ltd.
   "^onie,.*":
     description: Open Network Install Environment group
   "^onion,.*":
@@ -1610,6 +1620,8 @@ patternProperties:
   "^synopsys,.*":
     description: Synopsys, Inc. (deprecated, use snps)
     deprecated: true
+  "^taiguanck,.*":
+    description: Shenzhen Top Group Technology Co., Ltd.
   "^taos,.*":
     description: Texas Advanced Optoelectronic Solutions Inc.
   "^tbs,.*":
@@ -1731,6 +1743,8 @@ patternProperties:
     description: Ufi Space Co., Ltd.
   "^ugoos,.*":
     description: Ugoos Industrial Co., Ltd.
+  "^ultrapower,.*":
+    description: Beijing Ultrapower Software Co., Ltd.
   "^uni-t,.*":
     description: Uni-Trend Technology (China) Co., Ltd.
   "^uniwest,.*":
@@ -1761,6 +1775,8 @@ patternProperties:
     description: Variscite Ltd.
   "^vdl,.*":
     description: Van der Laan b.v.
+  "^verisilicon,.*":
+    description: VeriSilicon Microelectronics (Shanghai) Co., Ltd.
   "^vertexcom,.*":
     description: Vertexcom Technologies, Inc.
   "^via,.*":
@@ -1821,6 +1837,8 @@ patternProperties:
     description: Wi2Wi, Inc.
   "^widora,.*":
     description: Beijing Widora Technology Co., Ltd.
+  "^wiko,.*":
+    description: Wiko SAS
   "^wiligear,.*":
     description: Wiligear, Ltd.
   "^willsemi,.*":
diff --git a/Documentation/devicetree/of_unittest.rst b/Documentation/devicetree/of_unittest.rst
index 8b557acd29d1d..6ed6e32919644 100644
--- a/Documentation/devicetree/of_unittest.rst
+++ b/Documentation/devicetree/of_unittest.rst
@@ -48,30 +48,30 @@ from 'scripts/dtc/of_unittest_expect --help'.
 3. Test-data
 ============
 
-The Device Tree Source file (drivers/of/unittest-data/testcases.dts) contains
+The Device Tree Source file (drivers/of/unittest-data/testcases.dtso) contains
 the test data required for executing the unit tests automated in
 drivers/of/unittest.c. See the content of the folder::
 
     drivers/of/unittest-data/tests-*.dtsi
 
-for the Device Tree Source Include files (.dtsi) included in testcases.dts.
+for the Device Tree Source Include files (.dtsi) included in testcases.dtso.
 
 When the kernel is built with CONFIG_OF_UNITTEST enabled, then the following make
 rule::
 
-    $(obj)/%.dtb: $(src)/%.dts FORCE
-	    $(call if_changed_dep, dtc)
+    $(obj)/%.dtbo: $(src)/%.dtso $(DTC) FORCE
+	    $(call if_changed_dep,dtc)
 
-is used to compile the DT source file (testcases.dts) into a binary blob
-(testcases.dtb), also referred as flattened DT.
+is used to compile the DT source file (testcases.dtso) into a binary blob
+(testcases.dtbo), also referred as flattened DT.
 
 After that, using the following rule the binary blob above is wrapped as an
-assembly file (testcases.dtb.S)::
+assembly file (testcases.dtbo.S)::
 
-    $(obj)/%.dtb.S: $(obj)/%.dtb
-	    $(call cmd, dt_S_dtb)
+    $(obj)/%.dtbo.S: $(obj)/%.dtbo FORCE
+	    $(call if_changed,wrap_S_dtb)
 
-The assembly file is compiled into an object file (testcases.dtb.o), and is
+The assembly file is compiled into an object file (testcases.dtbo.o), and is
 linked into the kernel image.
 
 
diff --git a/Documentation/driver-api/driver-model/binding.rst b/Documentation/driver-api/driver-model/binding.rst
index d1d311a4011fe..fa0888c2b3b92 100644
--- a/Documentation/driver-api/driver-model/binding.rst
+++ b/Documentation/driver-api/driver-model/binding.rst
@@ -99,3 +99,51 @@ of the driver is decremented. All symlinks between the two are removed.
 When a driver is removed, the list of devices that it supports is
 iterated over, and the driver's remove callback is called for each
 one. The device is removed from that list and the symlinks removed.
+
+
+Driver Override
+~~~~~~~~~~~~~~~
+
+Userspace may override the standard matching by writing a driver name to
+a device's ``driver_override`` sysfs attribute.  When set, only a driver
+whose name matches the override will be considered during binding.  This
+bypasses all bus-specific matching (OF, ACPI, ID tables, etc.).
+
+The override may be cleared by writing an empty string, which returns
+the device to standard matching rules.  Writing to ``driver_override``
+does not automatically unbind the device from its current driver or
+make any attempt to load the specified driver.
+
+Buses opt into this mechanism by setting the ``driver_override`` flag in
+their ``struct bus_type``::
+
+  const struct bus_type example_bus_type = {
+      ...
+      .driver_override = true,
+  };
+
+When the flag is set, the driver core automatically creates the
+``driver_override`` sysfs attribute for every device on that bus.
+
+The bus's ``match()`` callback should check the override before performing
+its own matching, using ``device_match_driver_override()``::
+
+  static int example_match(struct device *dev, const struct device_driver *drv)
+  {
+      int ret;
+
+      ret = device_match_driver_override(dev, drv);
+      if (ret >= 0)
+          return ret;
+
+      /* Fall through to bus-specific matching... */
+  }
+
+``device_match_driver_override()`` returns > 0 if the override matches
+the given driver, 0 if the override is set but does not match, or < 0 if
+no override is set at all.
+
+Additional helpers are available:
+
+- ``device_set_driver_override()`` - set or clear the override from kernel code.
+- ``device_has_driver_override()`` - check whether an override is set.
diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst
index 7d2b897d66fa9..017fb155a5bc2 100644
--- a/Documentation/driver-api/driver-model/devres.rst
+++ b/Documentation/driver-api/driver-model/devres.rst
@@ -464,3 +464,7 @@ SPI
 
 WATCHDOG
   devm_watchdog_register_device()
+
+WORKQUEUE
+  devm_alloc_workqueue()
+  devm_alloc_ordered_workqueue()
diff --git a/Documentation/driver-api/early-userspace/early_userspace_support.rst b/Documentation/driver-api/early-userspace/early_userspace_support.rst
index 61bdeac1bae54..60d1e1bc9413b 100644
--- a/Documentation/driver-api/early-userspace/early_userspace_support.rst
+++ b/Documentation/driver-api/early-userspace/early_userspace_support.rst
@@ -73,7 +73,7 @@ usr/gen_initramfs.sh.  This means that CONFIG_INITRAMFS_SOURCE
 can really be interpreted as any legal argument to
 gen_initramfs.sh.  If a directory is specified as an argument then
 the contents are scanned, uid/gid translation is performed, and
-usr/gen_init_cpio file directives are output.  If a directory is
+usr/gen_init_cpio file directives are output.  If a file is
 specified as an argument to usr/gen_initramfs.sh then the
 contents of the file are simply copied to the output.  All of the output
 directives from directory scanning and file contents copying are
diff --git a/Documentation/driver-api/gpio/board.rst b/Documentation/driver-api/gpio/board.rst
index 069b54d8591bd..0993cac891fb5 100644
--- a/Documentation/driver-api/gpio/board.rst
+++ b/Documentation/driver-api/gpio/board.rst
@@ -239,22 +239,6 @@ mapping and is thus transparent to GPIO consumers.
 A set of functions such as gpiod_set_value() is available to work with
 the new descriptor-oriented interface.
 
-Boards using platform data can also hog GPIO lines by defining GPIO hog tables.
-
-.. code-block:: c
-
-        struct gpiod_hog gpio_hog_table[] = {
-                GPIO_HOG("gpio.0", 10, "foo", GPIO_ACTIVE_LOW, GPIOD_OUT_HIGH),
-                { }
-        };
-
-And the table can be added to the board code as follows::
-
-        gpiod_add_hogs(gpio_hog_table);
-
-The line will be hogged as soon as the gpiochip is created or - in case the
-chip was created earlier - when the hog table is registered.
-
 Arrays of pins
 --------------
 In addition to requesting pins belonging to a function one by one, a device may
diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst
index 85d86f92c41ba..a4f160b950892 100644
--- a/Documentation/driver-api/gpio/driver.rst
+++ b/Documentation/driver-api/gpio/driver.rst
@@ -87,6 +87,33 @@ atomic context on realtime kernels (inside hard IRQ handlers and similar
 contexts). Normally this should not be required.
 
 
+GPIO level semantics
+--------------------
+
+The gpip_chip .get/set[_multiple]() line values are clamped to the boolean
+space [0, 1], low level or high level.
+
+Low and high values are defined as physical low on the line in/out to the
+connector such as a physical pad, pin or rail.
+
+The GPIO library has internal logic to handle lines that are active low, such
+as indicated by overstrike or #name in a schematic, and the driver should not
+try to second-guess the logic value of a line.
+
+The way GPIO values are handled by the consumers is that the library present
+the *logical* value to the consumer. A line is *asserted* if its *logical*
+value is 1, and *de-asserted* if its logical value is 0. If inversion is
+required, this is handled by gpiolib and configured using hardware descriptions
+such as device tree or ACPI that can clearly indicate if a line is active
+high or low.
+
+Since electronics commonly insert inverters as driving stages or protection
+buffers in front of a GPIO line it is necessary that this semantic is part
+of the hardware description, so that consumers such as kernel drivers need
+not worry about this, and can for example assert a RESET line tied to a GPIO
+pin by setting it to logic 1 even if it is physically active low.
+
+
 GPIO electrical configuration
 -----------------------------
 
diff --git a/Documentation/driver-api/infiniband.rst b/Documentation/driver-api/infiniband.rst
index 10d8be9e74fe1..d48f246774d2f 100644
--- a/Documentation/driver-api/infiniband.rst
+++ b/Documentation/driver-api/infiniband.rst
@@ -92,21 +92,6 @@ iSCSI Extensions for RDMA (iSER)
 .. kernel-doc:: drivers/infiniband/ulp/iser/iser_verbs.c
    :internal:
 
-Omni-Path (OPA) Virtual NIC support
------------------------------------
-
-.. kernel-doc:: drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
-   :internal:
-
-.. kernel-doc:: drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
-   :internal:
-
-.. kernel-doc:: drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c
-   :internal:
-
-.. kernel-doc:: drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
-   :internal:
-
 InfiniBand SCSI RDMA protocol target support
 --------------------------------------------
 
diff --git a/Documentation/driver-api/interconnect.rst b/Documentation/driver-api/interconnect.rst
index a92d0f277a1f0..cebb77b49d8d4 100644
--- a/Documentation/driver-api/interconnect.rst
+++ b/Documentation/driver-api/interconnect.rst
@@ -84,13 +84,25 @@ be registered with the interconnect provider core.
 
 .. kernel-doc:: include/linux/interconnect-provider.h
 
+.. kernel-doc:: drivers/interconnect/core.c
+   :functions: icc_provider_init icc_provider_register icc_provider_deregister
+               icc_node_create icc_node_create_dyn icc_node_destroy
+               icc_node_add icc_node_del icc_nodes_remove icc_node_set_name
+               icc_link_create icc_link_nodes
+
 Interconnect consumers
 ----------------------
 
 Interconnect consumers are the clients which use the interconnect APIs to
 get paths between endpoints and set their bandwidth/latency/QoS requirements
-for these interconnect paths.  These interfaces are not currently
-documented.
+for these interconnect paths.
+
+.. kernel-doc:: drivers/interconnect/core.c
+   :functions: devm_of_icc_get of_icc_get_by_index of_icc_get icc_get
+               icc_put icc_enable icc_disable icc_set_bw icc_set_tag
+               icc_get_name
+
+.. kernel-doc:: drivers/interconnect/bulk.c
 
 Interconnect debugfs interfaces
 -------------------------------
diff --git a/Documentation/driver-api/media/index.rst b/Documentation/driver-api/media/index.rst
index d5593182a3f96..08fc2cfc07a30 100644
--- a/Documentation/driver-api/media/index.rst
+++ b/Documentation/driver-api/media/index.rst
@@ -26,6 +26,7 @@ Documentation/userspace-api/media/index.rst
     :numbered:
 
     maintainer-entry-profile
+    media-committers
 
     v4l2-core
     dtv-core
diff --git a/Documentation/driver-api/media/maintainer-entry-profile.rst b/Documentation/driver-api/media/maintainer-entry-profile.rst
index 2127e5b15e8f6..c5c00c66d85c7 100644
--- a/Documentation/driver-api/media/maintainer-entry-profile.rst
+++ b/Documentation/driver-api/media/maintainer-entry-profile.rst
@@ -1,45 +1,328 @@
+.. SPDX-License-Identifier: GPL-2.0
+
 Media Subsystem Profile
 =======================
 
 Overview
 --------
 
-The media subsystem covers support for a variety of devices: stream
-capture, analog and digital TV streams, cameras, remote controllers, HDMI CEC
-and media pipeline control.
+The Linux Media Community (aka: the LinuxTV Community) is formed by
+developers working on Linux Kernel Media Subsystem, together with users
+who also play an important role in testing the code.
+
+The Media Subsystem has code to support a wide variety of media-related
+devices: stream capture, analog and digital TV streams, cameras,
+video codecs, video processing (resizers, etc.), radio, remote controllers,
+HDMI CEC and media pipeline control.
 
-It covers, mainly, the contents of those directories:
+The Media Subsystem consists of the following directories in the kernel
+tree:
 
   - drivers/media
   - drivers/staging/media
+  - include/media
+  - Documentation/devicetree/bindings/media/\ [1]_
   - Documentation/admin-guide/media
   - Documentation/driver-api/media
   - Documentation/userspace-api/media
-  - Documentation/devicetree/bindings/media/\ [1]_
-  - include/media
 
 .. [1] Device tree bindings are maintained by the
        OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS maintainers
        (see the MAINTAINERS file). So, changes there must be reviewed
-       by them before being merged via the media subsystem's development
+       by them before being merged into the media subsystem's development
        tree.
 
 Both media userspace and Kernel APIs are documented and the documentation
 must be kept in sync with the API changes. It means that all patches that
 add new features to the subsystem must also bring changes to the
-corresponding API files.
+corresponding API documentation.
+
+Media Maintainers
+-----------------
+
+Media Maintainers are not just people capable of writing code, but they
+are developers who have demonstrated their ability to collaborate with
+the team, get the most knowledgeable people to review code, contribute
+high-quality code, and follow through to fix issues (in code or tests).
+
+Due to the size and wide scope of the media subsystem, multiple layers of
+maintainers are required, each with their own areas of expertise:
+
+- **Media Driver Maintainer**:
+    Responsible for one or more drivers within the Media Subsystem. They
+    are listed in the MAINTAINERS file as maintainer for those drivers. Media
+    Driver Maintainers review patches for those drivers, provide feedback if
+    patches do not follow the subsystem rules, or are not using the
+    media kernel or userspace APIs correctly, or if they have poor code
+    quality.
+
+    If you are the patch author, you work with other Media
+    Maintainers to ensure your patches are reviewed.
+
+    Some Media Driver Maintainers have additional responsibilities. They have
+    been granted Patchwork access and keep
+    `Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_
+    up to date, decide when patches are ready for merging, and create Pull
+    Requests for the Media Subsystem Maintainers to merge.
+
+- **Media Core Maintainer**:
+    Media Driver Maintainers with Patchwork access who are also responsible for
+    one or more media core frameworks.
+
+    Core framework changes are done via consensus between the relevant Media
+    Core Maintainers. Media Maintainers may include core framework changes in
+    their Pull Requests if they are signed off by the relevant Media Core
+    Maintainers.
+
+- **Media Subsystem Maintainers**:
+    Media Core Maintainers who are also responsible for the subsystem as a
+    whole, with access to the entire subsystem. Responsible for merging Pull
+    Requests from other Media Maintainers.
+
+    Userspace API/ABI changes are made via consensus among Media Subsystem
+    Maintainers\ [2]_. Media Maintainers may include API/ABI changes in
+    their Pull Requests if they are signed off by all Media Subsystem
+    Maintainers.
+
+All Media Maintainers shall agree with the Kernel development process as
+described in Documentation/process/index.rst and with the Kernel development
+rules in the Kernel documentation, including its code of conduct.
+
+Media Maintainers are often reachable via the #linux-media IRC channel at OFTC.
+
+.. [2] Everything that would break backward compatibility with existing
+       non-kernel code are API/ABI changes. This includes ioctl and sysfs
+       interfaces, v4l2 controls, and their behaviors.
+
+Patchwork Access
+----------------
+
+All Media Maintainers who have been granted Patchwork access shall ensure that
+`Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_
+will reflect the current status, e.g. patches shall be delegated to the Media
+Maintainer who is handling them and the patch status shall be updated according
+to these rules:
+
+- ``Under Review``: Used if the patch requires a second opinion
+  or when it is part of a Pull Request;
+- ``Superseded``: There is a newer version of the patch posted to the
+  mailing list.
+- ``Duplicated``: There was another patch doing the same thing from someone
+  else that was accepted.
+- ``Not Applicable``: Use for patch series that are not merged at media.git
+  tree (e.g. drm, dmabuf, upstream merge, etc.) but were cross-posted to the
+  linux-media mailing list.
+- ``Accepted``: Once a patch is merged in the multi-committer tree. Only Media
+  Maintainers with commit rights are allowed to set this state.
+
+If Media Maintainers decide not to accept a patch, they should reply to the
+patch authors by e‑mail, explaining why it is not accepted, and
+update `Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_
+accordingly with one of the following statuses:
+
+- ``Changes Requested``: if a new revision was requested;
+- ``Rejected``: if the proposed change is not acceptable at all.
+
+.. Note::
+
+   Patchwork supports a couple of clients to help semi-automate
+   status updates via its REST interface:
+
+   https://patchwork.readthedocs.io/en/latest/usage/clients/
+
+For patches that fall within their area of responsibility a Media Maintainer
+also decides when those patches are ready for merging, and create Pull Requests
+for the Media Subsystem Maintainers to merge.
+
+The most important aspect of becoming a Media Maintainer with Patchwork access
+is that you have demonstrated an ability to give good code reviews. We value
+your ability to deliver thorough, constructive code reviews.
+
+As such, potential maintainers must earn enough credibility and trust from the
+Linux Media Community. To do that, developers shall be familiar with the open
+source model and have been active in the Linux Kernel community for some time,
+and, in particular, in the media subsystem.
+
+In addition to actually making the code changes, you are basically
+demonstrating your:
+
+- commitment to the project;
+- ability to collaborate with the team and communicate well;
+- understanding of how upstream and the Linux Media Community work
+  (policies, processes for testing, code review, ...)
+- reasonable knowledge about:
+
+  - the Kernel development process:
+    Documentation/process/index.rst
+
+  - the Media development profile:
+    Documentation/driver-api/media/maintainer-entry-profile.rst
+
+- understanding of the projects' code base and coding style;
+- ability to provide feedback to the patch authors;
+- ability to judge when a patch might be ready for review and to submit;
+- ability to write good code (last but certainly not least).
+
+Media Driver Maintainers that desire to get Patchwork access are encouraged
+to participate at the yearly Linux Media Summit, typically co-located with
+a Linux-related conference. These summits are announced on the linux-media
+mailing list.
+
+If you are doing such tasks and have become a valued developer, an
+existing Media Maintainer can nominate you to the Media Subsystem Maintainers.
+
+The ultimate responsibility for accepting a nominated maintainer is up to
+the subsystem's maintainers. The nominated maintainer must have earned a trust
+relationship with all Media Subsystem Maintainers, as, by being granted
+Patchwork access, you will take over part of their maintenance tasks.
+
+Media Committers
+----------------
+
+Experienced and trusted Media Maintainers may be granted commit rights
+which allow them to directly push patches to the media development tree instead
+of posting a Pull Request for the Media Subsystem Maintainers. This helps
+offloading some of the work of the Media Subsystem Maintainers.
+
+More details about Media Committers' roles and responsibilities can be
+found here: :ref:`Media Committers`.
+
+Media development sites
+-----------------------
+
+The `LinuxTV <https://linuxtv.org/>`_ web site hosts news about the subsystem,
+together with:
+
+- `Wiki pages <https://www.linuxtv.org/wiki/index.php/Main_Page>`_;
+- `Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_;
+- `Linux Media documentation <https://linuxtv.org/docs.php>`_;
+- and more.
+
+The main development trees used by the media subsystem are at:
+
+- Stable tree:
+  - https://git.linuxtv.org/media.git/
+
+- Media committers tree:
+  - https://gitlab.freedesktop.org/linux-media/media-committers.git
+
+    Please note that it can be rebased, although only as a last resort.
+
+- Media development trees, including apps and CI:
+
+  - https://git.linuxtv.org/
+  - https://gitlab.freedesktop.org/linux-media/
+
+
+.. _Media development workflow:
+
+Media development workflow
+++++++++++++++++++++++++++
+
+All changes for the media subsystem shall be sent first as e-mails to the
+media mailing list, following the process documented at
+Documentation/process/index.rst.
+
+It means that patches shall be submitted as plain text only via e-mail to
+linux-media@vger.kernel.org (aka: LMML). While subscription is not mandatory,
+you can find details about how to subscribe to it and to see its archives at:
+
+  https://subspace.kernel.org/vger.kernel.org.html
+
+Emails with HTML will be automatically rejected by the mail server.
+
+It could be wise to also copy the relevant Media Maintainer(s). You should use
+``scripts/get_maintainers.pl`` to identify whom else needs to be copied.
+Please always copy driver's authors and maintainers.
+
+To minimize the chance of merge conflicts for your patch series, and make it
+easier to backport patches to stable Kernels, we recommend that you use the
+following baseline for your patch series:
 
-Due to the size and wide scope of the media subsystem, media's
-maintainership model is to have sub-maintainers that have a broad
-knowledge of a specific aspect of the subsystem. It is the sub-maintainers'
-task to review the patches, providing feedback to users if the patches are
-following the subsystem rules and are properly using the media kernel and
-userspace APIs.
+1. Features for the next mainline release:
 
-Patches for the media subsystem must be sent to the media mailing list
-at linux-media@vger.kernel.org as plain text only e-mail. Emails with
-HTML will be automatically rejected by the mail server. It could be wise
-to also copy the sub-maintainer(s).
+   - baseline shall be the ``media-committers.git next`` branch;
+
+2. Bug fixes for the next mainline release:
+
+   - baseline shall be the ``media-committers.git next`` branch. If the
+     changes depend on a fix from the ``media-committers.git fixes``
+     branch, then you can use that as baseline.
+
+3. Bug fixes for the current mainline release (-rcX):
+
+   - baseline shall be the latest mainline -rcX release or the
+     ``media-committers.git fixes`` branch if changes depend on a mainline
+     fix that is not yet merged;
+
+.. Note::
+
+   See https://www.kernel.org/category/releases.html for an overview
+   about Kernel release types.
+
+Patches with fixes shall have:
+
+- a ``Fixes:`` tag pointing to the first commit that introduced the bug;
+- when applicable, a ``Cc: stable@vger.kernel.org``.
+
+Patches that were fixing bugs publicly reported by someone at the
+linux-media@vger.kernel.org mailing list shall have:
+
+- a ``Reported-by:`` tag immediately followed by a ``Closes:`` tag.
+
+Patches that change API shall update documentation accordingly at the
+same patch series.
+
+See Documentation/process/index.rst for more details about e-mail submission.
+
+Once a patch is submitted, it may follow either one of the following
+workflows:
+
+a. Media Maintainers' workflow: Media Maintainers post the Pull Requests,
+   which are handled by the Media Subsystem Maintainers::
+
+     +-------+   +------------+   +------+   +-------+   +---------------------+
+     |e-mail |-->|picked up by|-->|code  |-->|pull   |-->|Subsystem Maintainers|
+     |to LMML|   |Patchwork   |   |review|   |request|   |merge in             |
+     |       |   |            |   |      |   |       |   |media-committers.git |
+     +-------+   +------------+   +------+   +-------+   +---------------------+
+
+   For this workflow, Pull Requests are generated by Media Maintainers with
+   Patchwork access.  If you do not have Patchwork access, then please don't
+   submit Pull Requests, as they will not be processed.
+
+b. Media Committers' workflow: patches are handled by Media Maintainers with
+   commit rights::
+
+     +-------+   +------------+   +------+   +--------------------------+
+     |e-mail |-->|picked up by|-->|code  |-->|Media Committers merge in |
+     |to LMML|   |Patchwork   |   |review|   |media-committers.git      |
+     +-------+   +------------+   +------+   +--------------------------+
+
+When patches are picked up by
+`Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_
+and when merged at media-committers, Media CI bots will check for errors and
+may provide e-mail feedback about patch problems. When this happens, the patch
+submitter must fix them or explain why the errors are false positives.
+
+Patches will only be moved to the next stage in these two workflows if they
+pass on Media CI or if there are false-positives in the Media CI reports.
+
+For both workflows, all patches shall be properly reviewed at
+linux-media@vger.kernel.org (LMML) before being merged in
+``media-committers.git``. Media patches will be reviewed in a timely manner
+by the maintainers and reviewers as listed in the MAINTAINERS file.
+
+Media Maintainers shall request reviews from other Media Maintainers and
+developers where applicable, i.e. because those developers have more
+knowledge about some areas that are changed by a patch.
+
+There shall be no open issues or unresolved or conflicting feedback
+from anyone. Clear them up first. Defer to the Media Subsystem
+Maintainers if needed.
+
+Failures during e-mail submission
++++++++++++++++++++++++++++++++++
 
 Media's workflow is heavily based on Patchwork, meaning that, once a patch
 is submitted, the e-mail will first be accepted by the mailing list
@@ -47,51 +330,107 @@ server, and, after a while, it should appear at:
 
    - https://patchwork.linuxtv.org/project/linux-media/list/
 
-If it doesn't automatically appear there after a few minutes, then
+If it doesn't automatically appear there after some time [3]_, then
 probably something went wrong on your submission. Please check if the
-email is in plain text\ [2]_ only and if your emailer is not mangling
+email is in plain text\ [4]_ only and if your emailer is not mangling
 whitespaces before complaining or submitting them again.
 
-You can check if the mailing list server accepted your patch, by looking at:
+To troubleshoot problems, you should first check if the mailing list
+server has accepted your patch, by looking at:
 
    - https://lore.kernel.org/linux-media/
 
-.. [2] If your email contains HTML, the mailing list server will simply
+If the patch is there and not at
+`Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_,
+it is likely that your e-mailer mangled the patch. Patchwork internally
+has logic that checks if the received e-mail contains a valid patch.
+Any whitespace and new line breakages mangling the patch won't be recognized by
+`Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_,
+and such a patch will be rejected.
+
+.. [3] It usually takes a few minutes for the patch to arrive, but
+       the e-mail server may be busy, so it may take a longer time
+       for a patch to be picked by
+       `Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_.
+
+.. [4] If your email contains HTML, the mailing list server will simply
        drop it, without any further notice.
 
+.. _media-developers-gpg:
+
+Authentication for pull and merge requests
+++++++++++++++++++++++++++++++++++++++++++
+
+The authenticity of developers submitting Pull Requests and merge requests
+shall be validated by using the Linux Kernel Web of Trust, with PGP signing
+at some moment. See: :ref:`kernel_org_trust_repository`.
 
-Media maintainers
-+++++++++++++++++
+With the Pull Request workflow, Pull Requests shall use PGP-signed tags.
 
-At the media subsystem, we have a group of senior developers that
-are responsible for doing the code reviews at the drivers (also known as
-sub-maintainers), and another senior developer responsible for the
-subsystem as a whole. For core changes, whenever possible, multiple
-media maintainers do the review.
+With the committers' workflow, this is ensured at the time merge request
+rights will be granted to the gitlab instance used by the media-committers.git
+tree, after receiving the e-mail documented in
+:ref:`media-committer-agreement`.
+
+For more details about PGP signing, please read
+Documentation/process/maintainer-pgp-guide.rst.
+
+Maintaining media maintainer status
+-----------------------------------
+
+See :ref:`Maintain Media Status`.
+
+List of Media Maintainers
+-------------------------
 
-The media maintainers that work on specific areas of the subsystem are:
+The Media Maintainers listed here all have patchwork access and can
+make Pull Requests or have commit rights.
 
-- Remote Controllers (infrared):
-    Sean Young <sean@mess.org>
+The Media Subsystem Maintainers are:
+  - Mauro Carvalho Chehab <mchehab@kernel.org>
+  - Hans Verkuil <hverkuil@kernel.org>
 
-- HDMI CEC:
-    Hans Verkuil <hverkuil@kernel.org>
+The Media Core Maintainers are:
+  - Sakari Ailus <sakari.ailus@linux.intel.com>
 
-- Media controller drivers:
-    Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+    - Media controller drivers
+    - Core media controller framework
+    - ISP
+    - sensor drivers
+    - v4l2-async and v4l2-fwnode core frameworks
+    - v4l2-flash-led-class core framework
 
-- ISP, v4l2-async, v4l2-fwnode, v4l2-flash-led-class and Sensor drivers:
-    Sakari Ailus <sakari.ailus@linux.intel.com>
+  - Mauro Carvalho Chehab <mchehab@kernel.org>
 
-- V4L2 drivers and core V4L2 frameworks:
-    Hans Verkuil <hverkuil@kernel.org>
+    - DVB
 
-The subsystem maintainer is:
-  Mauro Carvalho Chehab <mchehab@kernel.org>
+  - Laurent Pinchart <laurent.pinchart@ideasonboard.com>
 
-Media maintainers may delegate a patch to other media maintainers as needed.
-On such case, checkpatch's ``delegate`` field indicates who's currently
-responsible for reviewing a patch.
+    - Media controller drivers
+    - Core media controller framework
+    - ISP
+
+  - Hans Verkuil <hverkuil@kernel.org>
+
+    - V4L2 drivers
+    - V4L2 and videobuf2 core frameworks
+    - HDMI CEC drivers
+    - HDMI CEC core framework
+
+  - Sean Young <sean@mess.org>
+
+    - Remote Controller (infrared) drivers
+    - Remote Controller (infrared) core framework
+
+The Media Driver Maintainers responsible for specific areas are:
+  - Nicolas Dufresne <nicolas.dufresne@collabora.com>
+
+    - Codec drivers
+    - M2M driver not otherwise delegated
+
+  - Bryan O'Donoghue <bryan.odonoghue@linaro.org>
+
+    - Qualcomm drivers
 
 Submit Checklist Addendum
 -------------------------
@@ -106,18 +445,15 @@ that should be used in order to check if the drivers are properly
 implementing the media APIs:
 
 ====================	=======================================================
-Type			Tool
+Type			Utility
 ====================	=======================================================
-V4L2 drivers\ [3]_	``v4l2-compliance``
+V4L2 drivers\ [5]_	``v4l2-compliance``
 V4L2 virtual drivers	``contrib/test/test-media``
 CEC drivers		``cec-compliance``
 ====================	=======================================================
 
-.. [3] The ``v4l2-compliance`` also covers the media controller usage inside
-       V4L2 drivers.
-
-Other compliance tools are under development to check other parts of the
-subsystem.
+.. [5] The ``v4l2-compliance`` utility also covers the media controller usage
+       inside V4L2 drivers.
 
 Those tests need to pass before the patches go upstream.
 
@@ -134,6 +470,8 @@ Where the check script is::
 Be sure to not introduce new warnings on your patches without a
 very good reason.
 
+Please see `Media development workflow`_ for e-mail submission rules.
+
 Style Cleanup Patches
 +++++++++++++++++++++
 
@@ -173,34 +511,35 @@ least, simply wrapping the lines.
 In particular, we accept lines with more than 80 columns:
 
     - on strings, as they shouldn't be broken due to line length limits;
-    - when a function or variable name need to have a big identifier name,
-      which keeps hard to honor the 80 columns limit;
+    - when a function or variable name needs to have a long identifier name,
+      which makes hard to honor the 80 columns limit;
     - on arithmetic expressions, when breaking lines makes them harder to
       read;
-    - when they avoid a line to end with an open parenthesis or an open
+    - when they avoid a line ending with an open parenthesis or an open
       bracket.
 
 Key Cycle Dates
 ---------------
 
-New submissions can be sent at any time, but if they intend to hit the
+New submissions can be sent at any time, but if they are intended to hit the
 next merge window they should be sent before -rc5, and ideally stabilized
 in the linux-media branch by -rc6.
 
 Review Cadence
 --------------
 
-Provided that your patch is at https://patchwork.linuxtv.org, it should
-be sooner or later handled, so you don't need to re-submit a patch.
+Provided that your patch has landed in
+`Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_, it
+should be sooner or later handled, so you don't need to re-submit a patch.
 
-Except for bug fixes, we don't usually add new patches to the development
-tree between -rc6 and the next -rc1.
+Except for important bug fixes, we don't usually add new patches to the
+development tree between -rc6 and the next -rc1.
 
 Please notice that the media subsystem is a high traffic one, so it
 could take a while for us to be able to review your patches. Feel free
 to ping if you don't get a feedback in a couple of weeks or to ask
-other developers to publicly add Reviewed-by and, more importantly,
+other developers to publicly add ``Reviewed-by:`` and, more importantly,
 ``Tested-by:`` tags.
 
 Please note that we expect a detailed description for ``Tested-by:``,
-identifying what boards were used at the test and what it was tested.
+identifying what boards were used during the test and what it was tested.
diff --git a/Documentation/driver-api/media/media-committers.rst b/Documentation/driver-api/media/media-committers.rst
new file mode 100644
index 0000000000000..a905856f6a619
--- /dev/null
+++ b/Documentation/driver-api/media/media-committers.rst
@@ -0,0 +1,203 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _Media Committers:
+
+Media Committers
+================
+
+Who is a Media Committer?
+-------------------------
+
+A Media Committer is a Media Maintainer with patchwork access who has been
+granted commit access to push patches from other developers and their own
+patches to the
+`media-committers <https://gitlab.freedesktop.org/linux-media/media-committers>`_
+tree.
+
+These commit rights are granted with expectation of responsibility:
+committers are people who care about the Linux Kernel as a whole and
+about the Linux media subsystem and want to advance its development. It
+is also based on a trust relationship among other committers, maintainers
+and the Linux Media community.
+
+As Media Committer you have the following additional responsibilities:
+
+1. Patches you authored must have a ``Signed-off-by``, ``Reviewed-by``
+   or ``Acked-by`` from another Media Maintainer;
+2. If a patch introduces a regression, then that must be corrected as soon
+   as possible. Typically the patch is either reverted, or an additional
+   patch is committed to fix the regression;
+3. If patches are fixing bugs against already released Kernels, including
+   the reverts mentioned above, the Media Committer shall add the needed
+   tags. Please see :ref:`Media development workflow` for more details.
+4. All Media Committers are responsible for maintaining
+   `Patchwork <https://patchwork.linuxtv.org/project/linux-media/list/>`_,
+   updating the state of the patches they review or merge.
+
+
+Becoming a Media Committer
+--------------------------
+
+Existing Media Committers can nominate a Media Maintainer to be granted
+commit rights. The Media Maintainer must have patchwork access,
+have been reviewing patches from third parties for some time, and has
+demonstrated a good understanding of the maintainer's duties and processes.
+
+The ultimate responsibility for accepting a nominated committer is up to
+the Media Subsystem Maintainers. The nominated committer must have earned a
+trust relationship with all Media Subsystem Maintainers, as, by granting you
+commit rights, part of their responsibilities are handed over to you.
+
+Due to that, to become a Media Committer, a consensus between all Media
+Subsystem Maintainers is required.
+
+.. Note::
+
+   In order to preserve/protect the developers that could have their commit
+   rights granted, denied or removed as well as the subsystem maintainers who
+   have the task to accept or deny commit rights, all communication related to
+   changing commit rights should happen in private as much as possible.
+
+.. _media-committer-agreement:
+
+Media Committer's agreement
+---------------------------
+
+Once a nominated committer is accepted by all Media Subsystem Maintainers,
+they will ask if the developer is interested in the nomination and discuss
+what area(s) of the media subsystem the committer will be responsible for.
+Those areas will typically be the same as the areas that the nominated
+committer is already maintaining.
+
+When the developer accepts being a committer, the new committer shall
+explicitly accept the Kernel development policies described under its
+Documentation/, and in particular to the rules in this document, by writing
+an e-mail to media-committers@linuxtv.org, with a declaration of intent
+following the model below::
+
+   I, John Doe, would like to change my status to: Committer
+
+   As Media Maintainer I accept commit rights for the following areas of
+   the media subsystem:
+
+   ...
+
+   For the purpose of committing patches to the media-committers tree,
+   I'll be using my user https://gitlab.freedesktop.org/users/<username>.
+
+Followed by a formal declaration of agreement with the Kernel development
+rules::
+
+   I agree to follow the Kernel development rules described at:
+
+   https://www.kernel.org/doc/html/latest/driver-api/media/media-committers.rst
+
+   and to the Linux Kernel development process rules.
+
+   I agree to abide by the Code of Conduct as documented in:
+   https://www.kernel.org/doc/html/latest/process/code-of-conduct.rst
+
+   I am aware that I can, at any point of time, retire. In that case, I will
+   send an e-mail to notify the Media Subsystem Maintainers for them to revoke
+   my commit rights.
+
+   I am aware that the Kernel development rules change over time.
+   By doing a new push to media-committers tree, I understand that I agree
+   to follow the rules in effect at the time of the commit.
+
+That e-mail shall be signed via the Kernel Web of trust with a PGP key cross
+signed by other Kernel and media developers. As described at
+:ref:`media-developers-gpg`, the PGP signature, together with the gitlab user
+security are fundamental components that ensure the authenticity of the merge
+requests that will happen at the media-committers.git tree.
+
+In case the kernel development process changes, by merging new commits to the
+`media-committers tree <https://gitlab.freedesktop.org/linux-media/media-committers>`_,
+the Media Committer implicitly declares their agreement with the latest
+version of the documented process including the contents of this file.
+
+If a Media Committer decides to retire, it is the committer's duty to
+notify the Media Subsystem Maintainers about that decision.
+
+.. note::
+
+   1. Changes to the kernel media development process shall be announced in
+      the media-committers mailing list with a reasonable review period. All
+      committers are automatically subscribed to that mailing list;
+   2. Due to the distributed nature of the Kernel development, it is
+      possible that kernel development process changes may end being
+      reviewed/merged at the Linux Docs and/or at the Linux Kernel mailing
+      lists, especially for the contents under Documentation/process and for
+      trivial typo fixes.
+
+Media Core Committers
+---------------------
+
+A Media Core Committer is a Media Core Maintainer with commit rights.
+
+As described in Documentation/driver-api/media/maintainer-entry-profile.rst,
+a Media Core Maintainer maintains media core frameworks as well, besides
+just drivers, and so is allowed to change core files and the media subsystem's
+Kernel API. The extent of the core committer's grants will be detailed by the
+Media Subsystem Maintainers when they nominate a Media Core Committer.
+
+Existing Media Committers may become Media Core Committers and vice versa.
+Such decisions will be taken in consensus among the Media Subsystem
+Maintainers.
+
+Media committers rules
+----------------------
+
+Media committers shall do their best efforts to avoid merging patches that
+would break any existing drivers. If it breaks, fixup or revert patches
+shall be merged as soon as possible, aiming to be merged at the same Kernel
+cycle the bug is reported.
+
+Media committers shall behave accordingly to the rights granted by
+the Media Subsystem Maintainers, especially with regards of the scope of changes
+they may apply directly at the media-committers tree. That scope can
+change over time on a mutual agreement between Media Committers and
+Media Subsystem Maintainers.
+
+The Media Committer workflow is described at :ref:`Media development workflow`.
+
+.. _Maintain Media Status:
+
+Maintaining Media Maintainer or Committer status
+------------------------------------------------
+
+A community of maintainers working together to move the Linux Kernel
+forward is essential to creating successful projects that are rewarding
+to work on. If there are problems or disagreements within the community,
+they can usually be solved through healthy discussion and debate.
+
+In the unhappy event that a Media Maintainer or Committer continues to
+disregard good citizenship (or actively disrupts the project), we may need
+to revoke that person's status. In such cases, if someone suggests the
+revocation with a good reason, then after discussing this among the Media
+Maintainers, the final decision is taken by the Media Subsystem Maintainers.
+
+As the decision to become a Media Maintainer or Committer comes from a
+consensus between Media Subsystem Maintainers, a single Media Subsystem
+Maintainer not trusting the Media Maintainer or Committer anymore is enough
+to revoke their maintenance, Patchwork grants and/or commit rights.
+
+Having commit rights revoked doesn't prevent Media Maintainers to keep
+contributing to the subsystem either via the pull request or via email workflow
+as documented at the :ref:`Media development workflow`.
+
+If a maintainer is inactive for more than a couple of Kernel cycles,
+maintainers will try to reach you via e-mail. If not possible, they may
+revoke their maintainer/patchwork and committer rights and update MAINTAINERS
+file entries accordingly. If you wish to resume contributing as maintainer
+later on, then contact the Media Subsystem Maintainers to ask if your
+maintenance, Patchwork grants and commit rights can be restored.
+
+References
+----------
+
+Much of this was inspired by/copied from the committer policies of:
+
+- `Chromium <https://chromium.googlesource.com/chromium/src/+/main/docs/contributing.md>`_;
+- `WebKit <https://webkit.org/commit-and-review-policy/>`_;
+- `Mozilla <https://www.mozilla.org/hacking/committer/>`_.
diff --git a/Documentation/driver-api/reset.rst b/Documentation/driver-api/reset.rst
index f773100daaa41..7a6571849664f 100644
--- a/Documentation/driver-api/reset.rst
+++ b/Documentation/driver-api/reset.rst
@@ -198,7 +198,6 @@ query the reset line status using reset_control_status().
                reset_control_rearm
                reset_control_put
                of_reset_control_get_count
-               of_reset_control_array_get
                devm_reset_control_array_get
                reset_control_get_count
 
diff --git a/Documentation/driver-api/vme.rst b/Documentation/driver-api/vme.rst
index c0b475369de0a..7111999abc14c 100644
--- a/Documentation/driver-api/vme.rst
+++ b/Documentation/driver-api/vme.rst
@@ -107,7 +107,7 @@ The function :c:func:`vme_master_read` can be used to read from and
 
 In addition to simple reads and writes, :c:func:`vme_master_rmw` is provided to
 do a read-modify-write transaction. Parts of a VME window can also be mapped
-into user space memory using :c:func:`vme_master_mmap`.
+into user space memory using :c:func:`vme_master_mmap_prepare`.
 
 
 Slave windows
diff --git a/Documentation/features/perf/perf-regs/arch-support.txt b/Documentation/features/perf/perf-regs/arch-support.txt
index 09431518b0e85..4c9a5a0120751 100644
--- a/Documentation/features/perf/perf-regs/arch-support.txt
+++ b/Documentation/features/perf/perf-regs/arch-support.txt
@@ -18,7 +18,7 @@
     |        mips: |  ok  |
     |       nios2: | TODO |
     |    openrisc: | TODO |
-    |      parisc: | TODO |
+    |      parisc: |  ok  |
     |     powerpc: |  ok  |
     |       riscv: |  ok  |
     |        s390: |  ok  |
diff --git a/Documentation/features/perf/perf-stackdump/arch-support.txt b/Documentation/features/perf/perf-stackdump/arch-support.txt
index f9db4dd8ef79a..dd362b5cb6387 100644
--- a/Documentation/features/perf/perf-stackdump/arch-support.txt
+++ b/Documentation/features/perf/perf-stackdump/arch-support.txt
@@ -18,7 +18,7 @@
     |        mips: |  ok  |
     |       nios2: | TODO |
     |    openrisc: | TODO |
-    |      parisc: | TODO |
+    |      parisc: |  ok  |
     |     powerpc: |  ok  |
     |       riscv: |  ok  |
     |        s390: |  ok  |
diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst
index f4873197587df..fc7254d01a2b2 100644
--- a/Documentation/filesystems/index.rst
+++ b/Documentation/filesystems/index.rst
@@ -29,6 +29,7 @@ algorithms work.
    fiemap
    files
    locks
+   mmap_prepare
    multigrain-ts
    mount_api
    quota
@@ -98,6 +99,7 @@ Documentation for filesystem implementations.
    isofs
    nilfs2
    nfs/index
+   ntfs
    ntfs3
    ocfs2
    ocfs2-online-filecheck
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 8025df6e64997..8421ea21bd35e 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -398,6 +398,7 @@ prototypes::
 	bool (*lm_breaker_owns_lease)(struct file_lock *);
         bool (*lm_lock_expirable)(struct file_lock *);
         void (*lm_expire_lock)(void);
+        bool (*lm_breaker_timedout)(struct file_lease *);
 
 locking rules:
 
@@ -412,6 +413,7 @@ lm_breaker_owns_lease:	yes     	no			no
 lm_lock_expirable	yes		no			no
 lm_expire_lock		no		no			yes
 lm_open_conflict	yes		no			no
+lm_breaker_timedout     yes             no                      no
 ======================	=============	=================	=========
 
 buffer_head
diff --git a/Documentation/filesystems/mmap_prepare.rst b/Documentation/filesystems/mmap_prepare.rst
new file mode 100644
index 0000000000000..e810aa4134eb1
--- /dev/null
+++ b/Documentation/filesystems/mmap_prepare.rst
@@ -0,0 +1,168 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===========================
+mmap_prepare callback HOWTO
+===========================
+
+Introduction
+============
+
+The ``struct file->f_op->mmap()`` callback has been deprecated as it is both a
+stability and security risk, and doesn't always permit the merging of adjacent
+mappings resulting in unnecessary memory fragmentation.
+
+It has been replaced with the ``file->f_op->mmap_prepare()`` callback which
+solves these problems.
+
+This hook is called right at the beginning of setting up the mapping, and
+importantly it is invoked *before* any merging of adjacent mappings has taken
+place.
+
+If an error arises upon mapping, it might arise after this callback has been
+invoked, therefore it should be treated as effectively stateless.
+
+That is - no resources should be allocated nor state updated to reflect that a
+mapping has been established, as the mapping may either be merged, or fail to be
+mapped after the callback is complete.
+
+Mapped callback
+---------------
+
+If resources need to be allocated per-mapping, or state such as a reference
+count needs to be manipulated, this should be done using the ``vm_ops->mapped``
+hook, which itself should be set by the >mmap_prepare hook.
+
+This callback is only invoked if a new mapping has been established and was not
+merged with any other, and is invoked at a point where no error may occur before
+the mapping is established.
+
+You may return an error to the callback itself, which will cause the mapping to
+become unmapped and an error returned to the mmap() caller. This is useful if
+resources need to be allocated, and that allocation might fail.
+
+How To Use
+==========
+
+In your driver's struct file_operations struct, specify an ``mmap_prepare``
+callback rather than an ``mmap`` one, e.g. for ext4:
+
+.. code-block:: C
+
+    const struct file_operations ext4_file_operations = {
+        ...
+        .mmap_prepare    = ext4_file_mmap_prepare,
+    };
+
+This has a signature of ``int (*mmap_prepare)(struct vm_area_desc *)``.
+
+Examining the struct vm_area_desc type:
+
+.. code-block:: C
+
+    struct vm_area_desc {
+        /* Immutable state. */
+        const struct mm_struct *const mm;
+        struct file *const file; /* May vary from vm_file in stacked callers. */
+        unsigned long start;
+        unsigned long end;
+
+        /* Mutable fields. Populated with initial state. */
+        pgoff_t pgoff;
+        struct file *vm_file;
+        vma_flags_t vma_flags;
+        pgprot_t page_prot;
+
+        /* Write-only fields. */
+        const struct vm_operations_struct *vm_ops;
+        void *private_data;
+
+        /* Take further action? */
+        struct mmap_action action;
+    };
+
+This is straightforward - you have all the fields you need to set up the
+mapping, and you can update the mutable and writable fields, for instance:
+
+.. code-block:: C
+
+    static int ext4_file_mmap_prepare(struct vm_area_desc *desc)
+    {
+        int ret;
+        struct file *file = desc->file;
+        struct inode *inode = file->f_mapping->host;
+
+        ...
+
+        file_accessed(file);
+        if (IS_DAX(file_inode(file))) {
+            desc->vm_ops = &ext4_dax_vm_ops;
+            vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT);
+        } else {
+            desc->vm_ops = &ext4_file_vm_ops;
+        }
+        return 0;
+    }
+
+Importantly, you no longer have to dance around with reference counts or locks
+when updating these fields - **you can simply go ahead and change them**.
+
+Everything is taken care of by the mapping code.
+
+VMA Flags
+---------
+
+Along with ``mmap_prepare``, VMA flags have undergone an overhaul. Where before
+you would invoke one of vm_flags_init(), vm_flags_reset(), vm_flags_set(),
+vm_flags_clear(), and vm_flags_mod() to modify flags (and to have the
+locking done correctly for you, this is no longer necessary.
+
+Also, the legacy approach of specifying VMA flags via ``VM_READ``, ``VM_WRITE``,
+etc. - i.e. using a ``-VM_xxx``- macro has changed too.
+
+When implementing mmap_prepare(), reference flags by their bit number, defined
+as a ``VMA_xxx_BIT`` macro, e.g. ``VMA_READ_BIT``, ``VMA_WRITE_BIT`` etc.,
+and use one of (where ``desc`` is a pointer to struct vm_area_desc):
+
+* ``vma_desc_test_flags(desc, ...)`` - Specify a comma-separated list of flags
+  you wish to test for (whether _any_ are set), e.g. - ``vma_desc_test_flags(
+  desc, VMA_WRITE_BIT, VMA_MAYWRITE_BIT)`` - returns ``true`` if either are set,
+  otherwise ``false``.
+* ``vma_desc_set_flags(desc, ...)`` - Update the VMA descriptor flags to set
+  additional flags specified by a comma-separated list,
+  e.g. - ``vma_desc_set_flags(desc, VMA_PFNMAP_BIT, VMA_IO_BIT)``.
+* ``vma_desc_clear_flags(desc, ...)`` - Update the VMA descriptor flags to clear
+  flags specified by a comma-separated list, e.g. - ``vma_desc_clear_flags(
+  desc, VMA_WRITE_BIT, VMA_MAYWRITE_BIT)``.
+
+Actions
+=======
+
+You can now very easily have actions be performed upon a mapping once set up by
+utilising simple helper functions invoked upon the struct vm_area_desc
+pointer. These are:
+
+* mmap_action_remap() - Remaps a range consisting only of PFNs for a specific
+  range starting a virtual address and PFN number of a set size.
+
+* mmap_action_remap_full() - Same as mmap_action_remap(), only remaps the
+  entire mapping from ``start_pfn`` onward.
+
+* mmap_action_ioremap() - Same as mmap_action_remap(), only performs an I/O
+  remap.
+
+* mmap_action_ioremap_full() - Same as mmap_action_ioremap(), only remaps
+  the entire mapping from ``start_pfn`` onward.
+
+* mmap_action_simple_ioremap() - Sets up an I/O remap from a specified
+  physical address and over a specified length.
+
+* mmap_action_map_kernel_pages() - Maps a specified array of `struct page`
+  pointers in the VMA from a specific offset.
+
+* mmap_action_map_kernel_pages_full() - Maps a specified array of `struct
+  page` pointers over the entire VMA. The caller must ensure there are
+  sufficient entries in the page array to cover the entire range of the
+  described VMA.
+
+**NOTE:** The ``action`` field should never normally be manipulated directly,
+rather you ought to use one of these helpers.
diff --git a/Documentation/filesystems/mount_api.rst b/Documentation/filesystems/mount_api.rst
index a064234fed5bb..e8b94357b4dff 100644
--- a/Documentation/filesystems/mount_api.rst
+++ b/Documentation/filesystems/mount_api.rst
@@ -647,9 +647,7 @@ The members are as follows:
 	fs_param_is_u64		64-bit unsigned int	result->uint_64
 	fs_param_is_enum	Enum value name 	result->uint_32
 	fs_param_is_string	Arbitrary string	param->string
-	fs_param_is_blob	Binary blob		param->blob
 	fs_param_is_blockdev	Blockdev path		* Needs lookup
-	fs_param_is_path	Path			* Needs lookup
 	fs_param_is_fd		File descriptor		result->int_32
 	fs_param_is_uid		User ID (u32)           result->uid
 	fs_param_is_gid		Group ID (u32)          result->gid
@@ -681,9 +679,7 @@ The members are as follows:
 	fsparam_u64()		fs_param_is_u64
 	fsparam_enum()		fs_param_is_enum
 	fsparam_string()	fs_param_is_string
-	fsparam_blob()		fs_param_is_blob
 	fsparam_bdev()		fs_param_is_blockdev
-	fsparam_path()		fs_param_is_path
 	fsparam_fd()		fs_param_is_fd
 	fsparam_uid()		fs_param_is_uid
 	fsparam_gid()		fs_param_is_gid
diff --git a/Documentation/filesystems/ntfs.rst b/Documentation/filesystems/ntfs.rst
new file mode 100644
index 0000000000000..5c96b04a4d7a8
--- /dev/null
+++ b/Documentation/filesystems/ntfs.rst
@@ -0,0 +1,159 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================================
+The Linux NTFS filesystem driver
+=================================
+
+
+.. Table of contents
+
+   - Overview
+   - Utilities support
+   - Supported mount options
+
+
+Overview
+========
+
+NTFS is a Linux kernel filesystem driver that provides full read and write
+support for NTFS volumes. It is designed for high performance, modern
+kernel infrastructure (iomap, folio), and stable long-term maintenance.
+
+
+Utilities support
+=================
+
+The NTFS utilities project, called ntfsprogs-plus, provides mkfs.ntfs,
+fsck.ntfs, and other related tools (e.g., ntfsinfo, ntfsclone, etc.) for
+creating, checking, and managing NTFS volumes. These utilities can be used
+for filesystem testing with xfstests as well as for recovering corrupted
+NTFS devices.
+
+The project is available at:
+
+  https://github.com/ntfsprogs-plus/ntfsprogs-plus
+
+
+Supported mount options
+=======================
+
+The NTFS driver supports the following mount options:
+
+======================= ====================================================
+iocharset=name          Character set to use for converting between
+                        the encoding is used for user visible filename and
+                        16 bit Unicode characters.
+
+nls=name                Deprecated option.  Still supported but please use
+                        iocharset=name in the future.
+
+uid=
+gid=
+umask=                  Provide default owner, group, and access mode mask.
+                        These options work as documented in mount(8).  By
+                        default, the files/directories are owned by root
+                        and he/she has read and write permissions, as well
+                        as browse permission for directories.  No one else
+                        has any access permissions.  I.e. the mode on all
+                        files is by default rw------- and
+                        for directories rwx------, a consequence of
+                        the default fmask=0177 and dmask=0077.
+                        Using a umask of zero will grant all permissions to
+                        everyone, i.e. all files and directories will have
+                        mode rwxrwxrwx.
+
+fmask=
+dmask=                  Instead of specifying umask which applies both to
+                        files and directories, fmask applies only to files
+                        and dmask only to directories.
+
+showmeta=<BOOL>
+show_sys_files=<BOOL>   If show_sys_files is specified, show the system
+                        files in directory listings.  Otherwise the default
+                        behaviour is to hide the system files.
+                        Note that even when show_sys_files is specified,
+                        "$MFT" will not be visible due to bugs/mis-features
+                        in glibc. Further, note that irrespective of
+                        show_sys_files, all files are accessible by name,
+                        i.e. you can always do "ls -l \$UpCase" for example
+                        to specifically show the system file containing
+                        the Unicode upcase table.
+
+case_sensitive=<BOOL>   If case_sensitive is specified, treat all filenames
+                        as case sensitive and create file names in
+                        the POSIX namespace (default behavior). Note,
+                        the Linux NTFS driver will never create short
+                        filenames and will remove them on rename/delete of
+                        the corresponding long file name. Note that files
+                        remain accessible via their short file name, if it
+                        exists.
+
+nocase=<BOOL>           If nocase is specified, treat filenames
+                        case-insensitively.
+
+disable_sparse=<BOOL>   If disable_sparse is specified, creation of sparse
+                        regions, i.e. holes, inside files is disabled for
+                        the volume (for the duration of this mount only).
+                        By default, creation of sparse regions is enabled,
+                        which is consistent with the behaviour of
+                        traditional Unix filesystems.
+
+errors=opt              Specify NTFS behavior on critical errors: panic,
+                        remount the partition in read-only mode or
+                        continue without doing anything (default behavior).
+
+mft_zone_multiplier=    Set the MFT zone multiplier for the volume (this
+                        setting is not persistent across mounts and can be
+                        changed from mount to mount but cannot be changed
+                        on remount).  Values of 1 to 4 are allowed, 1 being
+                        the default.  The MFT zone multiplier determines
+                        how much space is reserved for the MFT on the
+                        volume.  If all other space is used up, then the
+                        MFT zone will be shrunk dynamically, so this has no
+                        impact on the amount of free space.  However, it
+                        can have an impact on performance by affecting
+                        fragmentation of the MFT. In general use the
+                        default.  If you have a lot of small files then use
+                        a higher value.  The values have the following
+                        meaning:
+
+                        =====   =================================
+                        Value   MFT zone size (% of volume size)
+                        =====   =================================
+                          1             12.5%
+                          2             25%
+                          3             37.5%
+                          4             50%
+                        =====   =================================
+
+                        Note this option is irrelevant for read-only mount.
+
+preallocated_size=      Set preallocated size to optimize runlist merge
+                        overhead with small chunck size.(64KB size by
+                        default)
+
+acl=<BOOL>              Enable POSIX ACL support. When specified, POSIX
+                        ACLs stored in extended attributes are enforced.
+                        Default is off. Requires kernel config
+                        NTFS_FS_POSIX_ACL enabled.
+
+sys_immutable=<BOOL>    Make NTFS system files (e.g. $MFT, $LogFile,
+                        $Bitmap, $UpCase, etc.) immutable to user initiated
+                        modifications for extra safety. Default is off.
+
+nohidden=<BOOL>         Hide files and directories marked with the Windows
+                        "hidden" attribute. By default hidden items are
+                        shown.
+
+hide_dot_files=<BOOL>   Hide names beginning with a dot ("."). By default
+                        dot files are shown. When enabled, files and
+                        directories created with a leading '.' will be
+                        hidden from directory listings.
+
+windows_names=<BOOL>    Refuse creation/rename of files with characters or
+                        reserved device names disallowed on Windows (e.g.
+                        CON, NUL, AUX, COM1, LPT1, etc.). Default is off.
+discard=<BOOL>          Issue block device discard for clusters freed on
+                        file deletion/truncation to inform underlying
+                        storage.
+======================= ====================================================
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index 52ff1d19405be..d02aa57e44775 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -1361,3 +1361,17 @@ to match what strlen() would return if it was ran on the string.
 
 However, if the string is freely accessible for the duration of inode's
 lifetime, consider using inode_set_cached_link() instead.
+
+---
+
+**mandatory**
+
+lookup_one_qstr_excl() is no longer exported - use start_creating() or
+similar.
+---
+
+** mandatory**
+
+lock_rename(), lock_rename_child(), unlock_rename() are no
+longer available.  Use start_renaming() or similar.
+
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index b0c0d1b45b997..e2d22a424dcde 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -464,26 +464,37 @@ Memory Area, or VMA) there is a series of lines such as the following::
     KSM:                   0 kB
     LazyFree:              0 kB
     AnonHugePages:         0 kB
+    FilePmdMapped:         0 kB
     ShmemPmdMapped:        0 kB
     Shared_Hugetlb:        0 kB
     Private_Hugetlb:       0 kB
     Swap:                  0 kB
     SwapPss:               0 kB
-    KernelPageSize:        4 kB
-    MMUPageSize:           4 kB
     Locked:                0 kB
     THPeligible:           0
     VmFlags: rd ex mr mw me dw
 
 The first of these lines shows the same information as is displayed for
 the mapping in /proc/PID/maps.  Following lines show the size of the
-mapping (size); the size of each page allocated when backing a VMA
-(KernelPageSize), which is usually the same as the size in the page table
-entries; the page size used by the MMU when backing a VMA (in most cases,
-the same as KernelPageSize); the amount of the mapping that is currently
-resident in RAM (RSS); the process's proportional share of this mapping
-(PSS); and the number of clean and dirty shared and private pages in the
-mapping.
+mapping (size); the smallest possible page size allocated when backing a
+VMA (KernelPageSize), which is the granularity in which VMA modifications
+can be performed; the smallest possible page size that could be used by the
+MMU (MMUPageSize) when backing a VMA; the amount of the mapping that is
+currently resident in RAM (RSS); the process's proportional share of this
+mapping (PSS); and the number of clean and dirty shared and private pages
+in the mapping.
+
+"KernelPageSize" always corresponds to "MMUPageSize", except when a larger
+kernel page size is emulated on a system with a smaller page size used by the
+MMU, which is the case for some PPC64 setups with hugetlb.  Furthermore,
+"KernelPageSize" and "MMUPageSize" always correspond to the smallest
+possible granularity (fallback) that can be encountered in a VMA throughout
+its lifetime.  These values are not affected by Transparent Huge Pages
+being in effect, or any usage of larger MMU page sizes (either through
+architectural huge-page mappings or other explicit/implicit coalescing of
+virtual ranges performed by the MMU).  "AnonHugePages", "ShmemPmdMapped" and
+"FilePmdMapped" provide insight into the usage of PMD-level architectural
+huge-page mappings.
 
 The "proportional set size" (PSS) of a process is the count of pages it has
 in memory, where each page is divided by the number of processes sharing it.
@@ -528,10 +539,15 @@ pressure if the memory is clean. Please note that the printed value might
 be lower than the real value due to optimizations used in the current
 implementation. If this is not desirable please file a bug report.
 
-"AnonHugePages" shows the amount of memory backed by transparent hugepage.
+"AnonHugePages", "ShmemPmdMapped" and "FilePmdMapped" show the amount of
+memory backed by Transparent Huge Pages that are currently mapped by
+architectural huge-page mappings at the PMD level. "AnonHugePages"
+corresponds to memory that does not belong to a file, "ShmemPmdMapped" to
+shared memory (shmem/tmpfs) and "FilePmdMapped" to file-backed memory
+(excluding shmem/tmpfs).
 
-"ShmemPmdMapped" shows the amount of shared (shmem/tmpfs) memory backed by
-huge pages.
+There are no dedicated entries for Transparent Huge Pages (or similar concepts)
+that are not mapped by architectural huge-page mappings at the PMD level.
 
 "Shared_Hugetlb" and "Private_Hugetlb" show the amounts of memory backed by
 hugetlbfs page which is *not* counted in "RSS" or "PSS" field for historical
diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
index f22433470c761..32fb506db05b5 100644
--- a/Documentation/gpu/drm-mm.rst
+++ b/Documentation/gpu/drm-mm.rst
@@ -526,8 +526,14 @@ DRM GPUVM Function References
 DRM Buddy Allocator
 ===================
 
-DRM Buddy Function References
------------------------------
+Buddy Allocator Function References (GPU buddy)
+-----------------------------------------------
+
+.. kernel-doc:: drivers/gpu/buddy.c
+   :export:
+
+DRM Buddy Specific Logging Function References
+----------------------------------------------
 
 .. kernel-doc:: drivers/gpu/drm/drm_buddy.c
    :export:
diff --git a/Documentation/gpu/drm-ras.rst b/Documentation/gpu/drm-ras.rst
new file mode 100644
index 0000000000000..70b246a78fc8a
--- /dev/null
+++ b/Documentation/gpu/drm-ras.rst
@@ -0,0 +1,103 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+============================
+DRM RAS over Generic Netlink
+============================
+
+The DRM RAS (Reliability, Availability, Serviceability) interface provides a
+standardized way for GPU/accelerator drivers to expose error counters and
+other reliability nodes to user space via Generic Netlink. This allows
+diagnostic tools, monitoring daemons, or test infrastructure to query hardware
+health in a uniform way across different DRM drivers.
+
+Key Goals:
+
+* Provide a standardized RAS solution for GPU and accelerator drivers, enabling
+  data center monitoring and reliability operations.
+* Implement a single drm-ras Generic Netlink family to meet modern Netlink YAML
+  specifications and centralize all RAS-related communication in one namespace.
+* Support a basic error counter interface, addressing the immediate, essential
+  monitoring needs.
+* Offer a flexible, future-proof interface that can be extended to support
+  additional types of RAS data in the future.
+* Allow multiple nodes per driver, enabling drivers to register separate
+  nodes for different IP blocks, sub-blocks, or other logical subdivisions
+  as applicable.
+
+Nodes
+=====
+
+Nodes are logical abstractions representing an error type or error source within
+the device. Currently, only error counter nodes is supported.
+
+Drivers are responsible for registering and unregistering nodes via the
+`drm_ras_node_register()` and `drm_ras_node_unregister()` APIs.
+
+Node Management
+-------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_ras.c
+   :doc: DRM RAS Node Management
+.. kernel-doc:: drivers/gpu/drm/drm_ras.c
+   :internal:
+
+Generic Netlink Usage
+=====================
+
+The interface is implemented as a Generic Netlink family named ``drm-ras``.
+User space tools can:
+
+* List registered nodes with the ``list-nodes`` command.
+* List all error counters in an node with the ``get-error-counter`` command with ``node-id``
+  as a parameter.
+* Query specific error counter values with the ``get-error-counter`` command, using both
+  ``node-id`` and ``error-id`` as parameters.
+
+YAML-based Interface
+--------------------
+
+The interface is described in a YAML specification ``Documentation/netlink/specs/drm_ras.yaml``
+
+This YAML is used to auto-generate user space bindings via
+``tools/net/ynl/pyynl/ynl_gen_c.py``, and drives the structure of netlink
+attributes and operations.
+
+Usage Notes
+-----------
+
+* User space must first enumerate nodes to obtain their IDs.
+* Node IDs or Node names can be used for all further queries, such as error counters.
+* Error counters can be queried by either the Error ID or Error name.
+* Query Parameters should be defined as part of the uAPI to ensure user interface stability.
+* The interface supports future extension by adding new node types and
+  additional attributes.
+
+Example: List nodes using ynl
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --dump list-nodes
+    [{'device-name': '0000:03:00.0',
+    'node-id': 0,
+    'node-name': 'correctable-errors',
+    'node-type': 'error-counter'},
+    {'device-name': '0000:03:00.0',
+     'node-id': 1,
+     'node-name': 'uncorrectable-errors',
+     'node-type': 'error-counter'}]
+
+Example: List all error counters using ynl
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --dump get-error-counter --json '{"node-id":0}'
+    [{'error-id': 1, 'error-name': 'error_name1', 'error-value': 0},
+    {'error-id': 2, 'error-name': 'error_name2', 'error-value': 0}]
+
+Example: Query an error counter for a given node
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --do get-error-counter --json '{"node-id":0, "error-id":1}'
+    {'error-id': 1, 'error-name': 'error_name1', 'error-value': 0}
+
diff --git a/Documentation/gpu/index.rst b/Documentation/gpu/index.rst
index 2fafa1f35ef32..5d708a106b3fa 100644
--- a/Documentation/gpu/index.rst
+++ b/Documentation/gpu/index.rst
@@ -9,6 +9,7 @@ GPU Driver Developer's Guide
    drm-mm
    drm-kms
    drm-kms-helpers
+   drm-ras
    drm-uapi
    drm-usage-stats
    driver-uapi
diff --git a/Documentation/gpu/xe/xe_firmware.rst b/Documentation/gpu/xe/xe_firmware.rst
index 5d23e9f273914..9c15a300bc625 100644
--- a/Documentation/gpu/xe/xe_firmware.rst
+++ b/Documentation/gpu/xe/xe_firmware.rst
@@ -31,6 +31,9 @@ GuC Power Conservation (PC)
 .. kernel-doc:: drivers/gpu/drm/xe/xe_guc_pc.c
    :doc: GuC Power Conservation (PC)
 
+.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_rc.c
+   :doc: GuC Render C-states (GuC RC)
+
 PCIe Gen5 Limitations
 =====================
 
diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst
index 58986546c7233..9ad3f0a57f55f 100644
--- a/Documentation/hwmon/asus_ec_sensors.rst
+++ b/Documentation/hwmon/asus_ec_sensors.rst
@@ -22,6 +22,7 @@ Supported boards:
  * ROG CROSSHAIR VIII FORMULA
  * ROG CROSSHAIR VIII HERO
  * ROG CROSSHAIR VIII IMPACT
+ * ROG CROSSHAIR X670E EXTREME
  * ROG CROSSHAIR X670E HERO
  * ROG CROSSHAIR X670E GENE
  * ROG MAXIMUS X HERO
@@ -32,6 +33,7 @@ Supported boards:
  * ROG STRIX B550-I GAMING
  * ROG STRIX B650E-I GAMING WIFI
  * ROG STRIX B850-I GAMING WIFI
+ * ROG STRIX X470-F GAMING
  * ROG STRIX X470-I GAMING
  * ROG STRIX X570-E GAMING
  * ROG STRIX X570-E GAMING WIFI II
@@ -48,6 +50,7 @@ Supported boards:
  * ROG STRIX Z690-A GAMING WIFI D4
  * ROG STRIX Z690-E GAMING WIFI
  * ROG STRIX Z790-E GAMING WIFI II
+ * ROG STRIX Z790-H GAMING WIFI
  * ROG STRIX Z790-I GAMING WIFI
  * ROG ZENITH II EXTREME
  * ROG ZENITH II EXTREME ALPHA
diff --git a/Documentation/hwmon/bt1-pvt.rst b/Documentation/hwmon/bt1-pvt.rst
deleted file mode 100644
index cbb0c06131322..0000000000000
--- a/Documentation/hwmon/bt1-pvt.rst
+++ /dev/null
@@ -1,117 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0-only
-
-Kernel driver bt1-pvt
-=====================
-
-Supported chips:
-
-  * Baikal-T1 PVT sensor (in SoC)
-
-    Prefix: 'bt1-pvt'
-
-    Addresses scanned: -
-
-    Datasheet: Provided by BAIKAL ELECTRONICS upon request and under NDA
-
-Authors:
-    Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru>
-    Serge Semin <Sergey.Semin@baikalelectronics.ru>
-
-Description
------------
-
-This driver implements support for the hardware monitoring capabilities of the
-embedded into Baikal-T1 process, voltage and temperature sensors. PVT IP-core
-consists of one temperature and four voltage sensors, which can be used to
-monitor the chip internal environment like heating, supply voltage and
-transistors performance. The driver can optionally provide the hwmon alarms
-for each sensor the PVT controller supports. The alarms functionality is made
-compile-time configurable due to the hardware interface implementation
-peculiarity, which is connected with an ability to convert data from only one
-sensor at a time. Additional limitation is that the controller performs the
-thresholds checking synchronously with the data conversion procedure. Due to
-these in order to have the hwmon alarms automatically detected the driver code
-must switch from one sensor to another, read converted data and manually check
-the threshold status bits. Depending on the measurements timeout settings
-(update_interval sysfs node value) this design may cause additional burden on
-the system performance. So in case if alarms are unnecessary in your system
-design it's recommended to have them disabled to prevent the PVT IRQs being
-periodically raised to get the data cache/alarms status up to date. By default
-in alarm-less configuration the data conversion is performed by the driver
-on demand when read operation is requested via corresponding _input-file.
-
-Temperature Monitoring
-----------------------
-
-Temperature is measured with 10-bit resolution and reported in millidegree
-Celsius. The driver performs all the scaling by itself therefore reports true
-temperatures that don't need any user-space adjustments. While the data
-translation formulae isn't linear, which gives us non-linear discreteness,
-it's close to one, but giving a bit better accuracy for higher temperatures.
-The temperature input is mapped as follows (the last column indicates the input
-ranges)::
-
-	temp1: CPU embedded diode	-48.38C - +147.438C
-
-In case if the alarms kernel config is enabled in the driver the temperature input
-has associated min and max limits which trigger an alarm when crossed.
-
-Voltage Monitoring
-------------------
-
-The voltage inputs are also sampled with 10-bit resolution and reported in
-millivolts. But in this case the data translation formulae is linear, which
-provides a constant measurements discreteness. The data scaling is also
-performed by the driver, so returning true millivolts. The voltage inputs are
-mapped as follows (the last column indicates the input ranges)::
-
-	in0: VDD		(processor core)		0.62V - 1.168V
-	in1: Low-Vt		(low voltage threshold)		0.62V - 1.168V
-	in2: High-Vt		(high voltage threshold)	0.62V - 1.168V
-	in3: Standard-Vt	(standard voltage threshold)	0.62V - 1.168V
-
-In case if the alarms config is enabled in the driver the voltage inputs
-have associated min and max limits which trigger an alarm when crossed.
-
-Sysfs Attributes
-----------------
-
-Following is a list of all sysfs attributes that the driver provides, their
-permissions and a short description:
-
-=============================== ======= =======================================
-Name				Perm	Description
-=============================== ======= =======================================
-update_interval			RW	Measurements update interval per
-					sensor.
-temp1_type			RO	Sensor type (always 1 as CPU embedded
-					diode).
-temp1_label			RO	CPU Core Temperature sensor.
-temp1_input			RO	Measured temperature in millidegree
-					Celsius.
-temp1_min			RW	Low limit for temp input.
-temp1_max			RW	High limit for temp input.
-temp1_min_alarm			RO	Temperature input alarm. Returns 1 if
-					temperature input went below min limit,
-					0 otherwise.
-temp1_max_alarm			RO	Temperature input alarm. Returns 1 if
-					temperature input went above max limit,
-					0 otherwise.
-temp1_offset			RW	Temperature offset in millidegree
-					Celsius which is added to the
-					temperature reading by the chip. It can
-					be used to manually adjust the
-					temperature measurements within 7.130
-					degrees Celsius.
-in[0-3]_label			RO	CPU Voltage sensor (either core or
-					low/high/standard thresholds).
-in[0-3]_input			RO	Measured voltage in millivolts.
-in[0-3]_min			RW	Low limit for voltage input.
-in[0-3]_max			RW	High limit for voltage input.
-in[0-3]_min_alarm		RO	Voltage input alarm. Returns 1 if
-					voltage input went below min limit,
-					0 otherwise.
-in[0-3]_max_alarm		RO	Voltage input alarm. Returns 1 if
-					voltage input went above max limit,
-					0 otherwise.
-=============================== ======= =======================================
diff --git a/Documentation/hwmon/ina2xx.rst b/Documentation/hwmon/ina2xx.rst
index a3860aae444c0..d64e7af46a124 100644
--- a/Documentation/hwmon/ina2xx.rst
+++ b/Documentation/hwmon/ina2xx.rst
@@ -74,6 +74,16 @@ Supported chips:
 	       https://us1.silergy.com/
 
 
+  * Texas Instruments INA234
+
+    Prefix: 'ina234'
+
+    Addresses: I2C 0x40 - 0x43
+
+    Datasheet: Publicly available at the Texas Instruments website
+
+	       https://www.ti.com/
+
 Author: Lothar Felten <lothar.felten@gmail.com>
 
 Description
@@ -89,7 +99,7 @@ interface. The INA220 monitors both shunt drop and supply voltage.
 The INA226 is a current shunt and power monitor with an I2C interface.
 The INA226 monitors both a shunt voltage drop and bus supply voltage.
 
-INA230 and INA231 are high or low side current shunt and power monitors
+INA230, INA231, and INA234 are high or low side current shunt and power monitors
 with an I2C interface. The chips monitor both a shunt voltage drop and
 bus supply voltage.
 
@@ -124,8 +134,17 @@ power1_input		Power(uW) measurement channel
 shunt_resistor		Shunt resistance(uOhm) channel (not for ina260)
 ======================= ===============================================
 
-Additional sysfs entries for ina226, ina230, ina231, ina260, and sy24655
-------------------------------------------------------------------------
+Additional sysfs entries
+------------------------
+
+Additional entries are available for the following chips:
+
+  * ina226
+  * ina230
+  * ina231
+  * ina234
+  * ina260
+  * sy24655
 
 ======================= ====================================================
 curr1_lcrit		Critical low current
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index b2ca8513cfcd9..559c32344cd38 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -52,7 +52,6 @@ Hardware Monitoring Kernel Drivers
    bcm54140
    bel-pfe
    bpa-rs600
-   bt1-pvt
    cgbc-hwmon
    chipcap2
    coretemp
@@ -111,6 +110,7 @@ Hardware Monitoring Kernel Drivers
    kbatt
    kfan
    lan966x
+   lattepanda-sigma-ec
    lineage-pem
    lm25066
    lm63
diff --git a/Documentation/hwmon/lattepanda-sigma-ec.rst b/Documentation/hwmon/lattepanda-sigma-ec.rst
new file mode 100644
index 0000000000000..8a521ee1fef1e
--- /dev/null
+++ b/Documentation/hwmon/lattepanda-sigma-ec.rst
@@ -0,0 +1,61 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver lattepanda-sigma-ec
+=================================
+
+Supported systems:
+
+  * LattePanda Sigma (Intel 13th Gen i5-1340P)
+
+    DMI vendor: LattePanda
+
+    DMI product: LattePanda Sigma
+
+    BIOS version: 5.27 (verified)
+
+    Datasheet: Not available (EC registers discovered empirically)
+
+Author: Mariano Abad <weimaraner@gmail.com>
+
+Description
+-----------
+
+This driver provides hardware monitoring for the LattePanda Sigma
+single-board computer made by DFRobot. The board uses an ITE IT8613E
+Embedded Controller to manage a CPU cooling fan and thermal sensors.
+
+The BIOS declares the ACPI Embedded Controller (``PNP0C09``) with
+``_STA`` returning 0, preventing the kernel's ACPI EC subsystem from
+initializing. This driver reads the EC directly via the standard ACPI
+EC I/O ports (``0x62`` data, ``0x66`` command/status).
+
+Sysfs attributes
+----------------
+
+======================= ===============================================
+``fan1_input``          Fan speed in RPM (EC registers 0x2E:0x2F,
+                        16-bit big-endian)
+``fan1_label``          "CPU Fan"
+``temp1_input``         Board/ambient temperature in millidegrees
+                        Celsius (EC register 0x60, unsigned)
+``temp1_label``         "Board Temp"
+``temp2_input``         CPU proximity temperature in millidegrees
+                        Celsius (EC register 0x70, unsigned)
+``temp2_label``         "CPU Temp"
+======================= ===============================================
+
+Module parameters
+-----------------
+
+``force`` (bool, default false)
+    Force loading on BIOS versions other than 5.27. The driver still
+    requires DMI vendor and product name matching.
+
+Known limitations
+-----------------
+
+* Fan speed control is not supported. The fan is always under EC
+  automatic control.
+* The EC register map was verified only on BIOS version 5.27.
+  Other versions may use different register offsets; use the ``force``
+  parameter at your own risk.
diff --git a/Documentation/hwmon/ltc4282.rst b/Documentation/hwmon/ltc4282.rst
index a87ec3564998f..dd730207b1413 100644
--- a/Documentation/hwmon/ltc4282.rst
+++ b/Documentation/hwmon/ltc4282.rst
@@ -9,8 +9,7 @@ Supported chips:
 
     Prefix: 'ltc4282'
 
-    Addresses scanned: - I2C 0x40 - 0x5A (7-bit)
-    Addresses scanned: - I2C 0x80 - 0xB4 with a step of 2 (8-bit)
+    Addresses scanned: -
 
     Datasheet:
 
diff --git a/Documentation/hwmon/tmp102.rst b/Documentation/hwmon/tmp102.rst
index b1f585531a88d..3c2cb5bab1e98 100644
--- a/Documentation/hwmon/tmp102.rst
+++ b/Documentation/hwmon/tmp102.rst
@@ -11,6 +11,22 @@ Supported chips:
 
     Datasheet: http://focus.ti.com/docs/prod/folders/print/tmp102.html
 
+  * Texas Instruments TMP110
+
+    Prefix: 'tmp110'
+
+    Addresses scanned: none
+
+    Datasheet: http://focus.ti.com/docs/prod/folders/print/tmp110.html
+
+  * Texas Instruments TMP113
+
+    Prefix: 'tmp113'
+
+    Addresses scanned: none
+
+    Datasheet: http://focus.ti.com/docs/prod/folders/print/tmp113.html
+
 Author:
 
 	Steven King <sfking@fdwdc.com>
@@ -27,5 +43,10 @@ operating temperature has a minimum of -55 C and a maximum of +150 C.
 The TMP102 has a programmable update rate that can select between 8, 4, 1, and
 0.5 Hz. (Currently the driver only supports the default of 4 Hz).
 
+The TMP110 and TMP113 are software compatible with TMP102, but have different
+accuracy (maximum error) specifications. The TMP110 has an accuracy (maximum error)
+of 1.0 degree, TMP113 has an accuracy (maximum error) of 0.3 degree, while TMP102
+has an accuracy (maximum error) of 2.0 degree.
+
 The driver provides the common sysfs-interface for temperatures (see
 Documentation/hwmon/sysfs-interface.rst under Temperatures).
diff --git a/Documentation/iio/ad4030.rst b/Documentation/iio/ad4030.rst
index b57424b650a8a..9caafa4148b03 100644
--- a/Documentation/iio/ad4030.rst
+++ b/Documentation/iio/ad4030.rst
@@ -92,6 +92,45 @@ Interleaved mode
 In this mode, both channels conversion results are bit interleaved one SDO line.
 As such the wiring is the same as `One lane mode`_.
 
+SPI offload wiring
+^^^^^^^^^^^^^^^^^^
+
+.. code-block::
+
+    +-------------+         +-------------+
+    |         CNV |<-----+--| GPIO        |
+    |             |      +--| PWM0        |
+    |             |         |             |
+    |             |      +--| PWM1        |
+    |             |      |  +-------------+
+    |             |      +->| TRIGGER     |
+    |          CS |<--------| CS          |
+    |             |         |             |
+    |     ADC     |         |     SPI     |
+    |             |         |             |
+    |         SDI |<--------| SDO         |
+    |         SDO |-------->| SDI         |
+    |        SCLK |<--------| SCLK        |
+    +-------------+         +-------------+
+
+In this mode, both the ``cnv-gpios`` and a ``pwms`` properties are required.
+The ``pwms`` property specifies the PWM that is connected to the ADC CNV pin.
+The SPI offload will have a ``trigger-sources`` property to indicate the SPI
+offload (PWM) trigger source. For AD4030 and similar ADCs, there are two
+possible data transfer zones for sample N. One of them (zone 1) starts after the
+data conversion for sample N is complete while the other one (zone 2) starts 9.8
+nanoseconds after the rising edge of CNV for sample N + 1.
+
+The configuration depicted in the above diagram is intended to perform data
+transfer in zone 2. To achieve high sample rates while meeting ADC timing
+requirements, an offset is added between the rising edges of PWM0 and PWM1 to
+delay the SPI transfer until 9.8 nanoseconds after CNV rising edge. This
+requires a specialized PWM controller that can provide such an offset.
+The `AD4630-FMC HDL project`_, for example, can be configured to sample AD4030
+data during zone 2 data read window.
+
+.. _AD4630-FMC HDL project: https://analogdevicesinc.github.io/hdl/projects/ad4630_fmc/index.html
+
 SPI Clock mode
 --------------
 
diff --git a/Documentation/iio/adxl345.rst b/Documentation/iio/adxl345.rst
index bb19d64f67c3d..978f746a81985 100644
--- a/Documentation/iio/adxl345.rst
+++ b/Documentation/iio/adxl345.rst
@@ -12,16 +12,21 @@ This driver supports Analog Device's ADXL345/375 on SPI/I2C bus.
 * `ADXL345 <https://www.analog.com/ADXL345>`_
 * `ADXL375 <https://www.analog.com/ADXL375>`_
 
-The ADXL345 is a generic purpose low power, 3-axis accelerometer with selectable
-measurement ranges. The ADXL345 supports the ±2 g, ±4 g, ±8 g, and ±16 g ranges.
+The ADXL345 is a general-purpose, low-power, 3-axis accelerometer with selectable
+measurement ranges. The ADXL345 supports the following ranges:
+
+- ±2g  (approx. ±19.61 m/s^2)
+- ±4g  (approx. ±39.23 m/s^2)
+- ±8g  (approx. ±78.45 m/s^2)
+- ±16g (approx. ±156.91 m/s^2)
 
 2. Device Attributes
 ====================
 
-Each IIO device, has a device folder under ``/sys/bus/iio/devices/iio:deviceX``,
+Each IIO device has a device folder under ``/sys/bus/iio/devices/iio:deviceX``,
 where X is the IIO index of the device. Under these folders reside a set of
 device files, depending on the characteristics and features of the hardware
-device in questions. These files are consistently generalized and documented in
+device in question. These files are consistently generalized and documented in
 the IIO ABI documentation.
 
 The following table shows the ADXL345 related device files, found in the
@@ -42,7 +47,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``.
 +-------------------------------------------+----------------------------------------------------------+
 | in_accel_x_raw                            | Raw X-axis accelerometer channel value.                  |
 +-------------------------------------------+----------------------------------------------------------+
-| in_accel_y_calibbias                      | y-axis acceleration offset correction                    |
+| in_accel_y_calibbias                      | Y-axis acceleration offset correction                    |
 +-------------------------------------------+----------------------------------------------------------+
 | in_accel_y_raw                            | Raw Y-axis accelerometer channel value.                  |
 +-------------------------------------------+----------------------------------------------------------+
@@ -68,7 +73,7 @@ present, simply assume its value is 0.
 +-------------------------------------+---------------------------+
 | Channel type                        | Measurement unit          |
 +-------------------------------------+---------------------------+
-| Acceleration on X, Y, and Z axis    | Meters per second squared |
+| Acceleration on X, Y, and Z axes    | Meters per second squared |
 +-------------------------------------+---------------------------+
 
 Sensor Events
@@ -78,8 +83,8 @@ Specific IIO events are triggered by their corresponding interrupts. The sensor
 driver supports either none or a single active interrupt (INT) line, selectable
 from the two available options: INT1 or INT2. The active INT line should be
 specified in the device tree. If no INT line is configured, the sensor defaults
-to FIFO bypass mode, where event detection is disabled and only X, Y, and Z axis
-measurements are available.
+to FIFO bypass mode, where event detection is disabled and only individual
+X, Y, and Z axis measurements are available.
 
 The table below lists the ADXL345-related device files located in the
 device-specific path: ``/sys/bus/iio/devices/iio:deviceX/events``.
@@ -90,37 +95,51 @@ listed.
 +---------------------------------------------+---------------------------------------------+
 | Event handle                                | Description                                 |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_gesture_doubletap_en               | Enable double tap detection on all axis     |
+| in_accel_gesture_doubletap_en               | Enable double tap detection on all axes     |
 +---------------------------------------------+---------------------------------------------+
 | in_accel_gesture_doubletap_reset_timeout    | Double tap window in [us]                   |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_gesture_doubletap_tap2_min_delay   | Double tap latent in [us]                   |
+| in_accel_gesture_doubletap_scale            | Double tap gesture threshold scale.         |
++---------------------------------------------+---------------------------------------------+
+| in_accel_gesture_doubletap_tap2_min_delay   | Double tap latency in [us]                  |
++---------------------------------------------+---------------------------------------------+
+| in_accel_gesture_doubletap_value            | Double tap threshold value                  |
++---------------------------------------------+---------------------------------------------+
+| in_accel_gesture_singletap_scale            | Single tap gesture threshold scale.         |
 +---------------------------------------------+---------------------------------------------+
 | in_accel_gesture_singletap_timeout          | Single tap duration in [us]                 |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_gesture_singletap_value            | Single tap threshold value in 62.5/LSB      |
+| in_accel_gesture_singletap_value            | Single tap threshold value                  |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_falling_period                 | Inactivity time in seconds                  |
+| in_accel_mag_adaptive_falling_period        | AC coupled inactivity time in seconds       |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_falling_value                  | Inactivity threshold value in 62.5/LSB      |
+| in_accel_mag_adaptive_falling_scale         | AC coupled inactivity threshold scale.      |
++---------------------------------------------+---------------------------------------------+
+| in_accel_mag_adaptive_falling_value         | AC coupled inactivity threshold             |
 +---------------------------------------------+---------------------------------------------+
 | in_accel_mag_adaptive_rising_en             | Enable AC coupled activity on X axis        |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_adaptive_falling_period        | AC coupled inactivity time in seconds       |
+| in_accel_mag_adaptive_rising_scale          | AC coupled activity threshold scale.        |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_adaptive_falling_value         | AC coupled inactivity threshold in 62.5/LSB |
+| in_accel_mag_adaptive_rising_value          | AC coupled activity threshold               |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_adaptive_rising_value          | AC coupled activity threshold in 62.5/LSB   |
+| in_accel_mag_falling_period                 | Inactivity time in seconds                  |
++---------------------------------------------+---------------------------------------------+
+| in_accel_mag_falling_scale                  | DC coupled inactivity threshold scale.      |
++---------------------------------------------+---------------------------------------------+
+| in_accel_mag_falling_value                  | Inactivity threshold value                  |
 +---------------------------------------------+---------------------------------------------+
 | in_accel_mag_rising_en                      | Enable activity detection on X axis         |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_mag_rising_value                   | Activity threshold value in 62.5/LSB        |
+| in_accel_mag_rising_scale                   | DC coupled activity threshold scale.        |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_x_gesture_singletap_en             | Enable single tap detection on X axis       |
+| in_accel_mag_rising_value                   | Activity threshold value                    |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_x&y&z_mag_falling_en               | Enable inactivity detection on all axis     |
+| in_accel_x&y&z_mag_adaptive_falling_en      | Enable AC coupled inactivity on all axes    |
 +---------------------------------------------+---------------------------------------------+
-| in_accel_x&y&z_mag_adaptive_falling_en      | Enable AC coupled inactivity on all axis    |
+| in_accel_x&y&z_mag_falling_en               | Enable inactivity detection on all axes     |
++---------------------------------------------+---------------------------------------------+
+| in_accel_x_gesture_singletap_en             | Enable single tap detection on X axis       |
 +---------------------------------------------+---------------------------------------------+
 | in_accel_y_gesture_singletap_en             | Enable single tap detection on Y axis       |
 +---------------------------------------------+---------------------------------------------+
@@ -140,8 +159,8 @@ When changing the **g range** configuration, the driver attempts to estimate
 appropriate activity and inactivity thresholds by scaling the default values
 based on the ratio of the previous range to the new one. The resulting threshold
 will never be zero and will always fall between 1 and 255, corresponding to up
-to 62.5 g/LSB as specified in the datasheet. However, you can override these
-estimated thresholds by setting explicit values.
+to 62.5 mg/LSB (0.612915 m/s^2/LSB) as specified in the datasheet. However,
+you can override these estimated thresholds by setting explicit values.
 
 When **activity** and **inactivity** events are enabled, the driver
 automatically manages hysteresis behavior by setting the **link** and
@@ -270,13 +289,13 @@ Scale range configuration:
 .. code-block:: bash
 
         root:/sys/bus/iio/devices/iio:device0> cat ./in_accel_scale
-        0.478899
+        0.004789
         root:/sys/bus/iio/devices/iio:device0> cat ./in_accel_scale_available
-        0.478899 0.957798 1.915595 3.831190
+        0.004789 0.009578 0.019156 0.038312
 
-        root:/sys/bus/iio/devices/iio:device0> echo 1.915595 > ./in_accel_scale
+        root:/sys/bus/iio/devices/iio:device0> echo 0.019156 > ./in_accel_scale
         root:/sys/bus/iio/devices/iio:device0> cat ./in_accel_scale
-        1.915595
+        0.019156
 
 Set output data rate (ODR):
 
@@ -312,10 +331,14 @@ Configure one or several events:
 
         root:/sys/bus/iio/devices/iio:device0> echo 24 > ./buffer0/length
 
-        ## AC coupled activity, threshold [62.5/LSB]
+        ## Check the event scale factor (0.0625 * 9.80665)
+        root:/sys/bus/iio/devices/iio:device0> cat ./events/in_accel_gesture_doubletap_scale
+        0.612915
+
+        ## AC coupled activity, threshold [0.612915 m/s^2/LSB]
         root:/sys/bus/iio/devices/iio:device0> echo 6 > ./events/in_accel_mag_adaptive_rising_value
 
-        ## AC coupled inactivity, threshold, [62.5/LSB]
+        ## AC coupled inactivity, threshold, [0.612915 m/s^2/LSB]
         root:/sys/bus/iio/devices/iio:device0> echo 4 > ./events/in_accel_mag_adaptive_falling_value
 
         ## AC coupled inactivity, time [s]
@@ -330,7 +353,7 @@ Configure one or several events:
         ## doubletap, window [us]
         root:/sys/bus/iio/devices/iio:device0> echo 0.025 > ./events/in_accel_gesture_doubletap_reset_timeout
 
-        ## doubletap, latent [us]
+        ## doubletap, latency [us]
         root:/sys/bus/iio/devices/iio:device0> echo 0.025 > ./events/in_accel_gesture_doubletap_tap2_min_delay
 
         ## AC coupled activity, enable
diff --git a/Documentation/infiniband/index.rst b/Documentation/infiniband/index.rst
index c11049d25703c..f57387a923387 100644
--- a/Documentation/infiniband/index.rst
+++ b/Documentation/infiniband/index.rst
@@ -9,7 +9,6 @@ InfiniBand
 
    core_locking
    ipoib
-   opa_vnic
    sysfs
    tag_matching
    ucaps
diff --git a/Documentation/infiniband/opa_vnic.rst b/Documentation/infiniband/opa_vnic.rst
deleted file mode 100644
index 2f888d9ffec02..0000000000000
--- a/Documentation/infiniband/opa_vnic.rst
+++ /dev/null
@@ -1,159 +0,0 @@
-=================================================================
-Intel Omni-Path (OPA) Virtual Network Interface Controller (VNIC)
-=================================================================
-
-Intel Omni-Path (OPA) Virtual Network Interface Controller (VNIC) feature
-supports Ethernet functionality over Omni-Path fabric by encapsulating
-the Ethernet packets between HFI nodes.
-
-Architecture
-=============
-The patterns of exchanges of Omni-Path encapsulated Ethernet packets
-involves one or more virtual Ethernet switches overlaid on the Omni-Path
-fabric topology. A subset of HFI nodes on the Omni-Path fabric are
-permitted to exchange encapsulated Ethernet packets across a particular
-virtual Ethernet switch. The virtual Ethernet switches are logical
-abstractions achieved by configuring the HFI nodes on the fabric for
-header generation and processing. In the simplest configuration all HFI
-nodes across the fabric exchange encapsulated Ethernet packets over a
-single virtual Ethernet switch. A virtual Ethernet switch, is effectively
-an independent Ethernet network. The configuration is performed by an
-Ethernet Manager (EM) which is part of the trusted Fabric Manager (FM)
-application. HFI nodes can have multiple VNICs each connected to a
-different virtual Ethernet switch. The below diagram presents a case
-of two virtual Ethernet switches with two HFI nodes::
-
-                               +-------------------+
-                               |      Subnet/      |
-                               |     Ethernet      |
-                               |      Manager      |
-                               +-------------------+
-                                  /          /
-                                /           /
-                              /            /
-                            /             /
-  +-----------------------------+  +------------------------------+
-  |  Virtual Ethernet Switch    |  |  Virtual Ethernet Switch     |
-  |  +---------+    +---------+ |  | +---------+    +---------+   |
-  |  | VPORT   |    |  VPORT  | |  | |  VPORT  |    |  VPORT  |   |
-  +--+---------+----+---------+-+  +-+---------+----+---------+---+
-           |                 \        /                 |
-           |                   \    /                   |
-           |                     \/                     |
-           |                    /  \                    |
-           |                  /      \                  |
-       +-----------+------------+  +-----------+------------+
-       |   VNIC    |    VNIC    |  |    VNIC   |    VNIC    |
-       +-----------+------------+  +-----------+------------+
-       |          HFI           |  |          HFI           |
-       +------------------------+  +------------------------+
-
-
-The Omni-Path encapsulated Ethernet packet format is as described below.
-
-==================== ================================
-Bits                 Field
-==================== ================================
-Quad Word 0:
-0-19                 SLID (lower 20 bits)
-20-30                Length (in Quad Words)
-31                   BECN bit
-32-51                DLID (lower 20 bits)
-52-56                SC (Service Class)
-57-59                RC (Routing Control)
-60                   FECN bit
-61-62                L2 (=10, 16B format)
-63                   LT (=1, Link Transfer Head Flit)
-
-Quad Word 1:
-0-7                  L4 type (=0x78 ETHERNET)
-8-11                 SLID[23:20]
-12-15                DLID[23:20]
-16-31                PKEY
-32-47                Entropy
-48-63                Reserved
-
-Quad Word 2:
-0-15                 Reserved
-16-31                L4 header
-32-63                Ethernet Packet
-
-Quad Words 3 to N-1:
-0-63                 Ethernet packet (pad extended)
-
-Quad Word N (last):
-0-23                 Ethernet packet (pad extended)
-24-55                ICRC
-56-61                Tail
-62-63                LT (=01, Link Transfer Tail Flit)
-==================== ================================
-
-Ethernet packet is padded on the transmit side to ensure that the VNIC OPA
-packet is quad word aligned. The 'Tail' field contains the number of bytes
-padded. On the receive side the 'Tail' field is read and the padding is
-removed (along with ICRC, Tail and OPA header) before passing packet up
-the network stack.
-
-The L4 header field contains the virtual Ethernet switch id the VNIC port
-belongs to. On the receive side, this field is used to de-multiplex the
-received VNIC packets to different VNIC ports.
-
-Driver Design
-==============
-Intel OPA VNIC software design is presented in the below diagram.
-OPA VNIC functionality has a HW dependent component and a HW
-independent component.
-
-The support has been added for IB device to allocate and free the RDMA
-netdev devices. The RDMA netdev supports interfacing with the network
-stack thus creating standard network interfaces. OPA_VNIC is an RDMA
-netdev device type.
-
-The HW dependent VNIC functionality is part of the HFI1 driver. It
-implements the verbs to allocate and free the OPA_VNIC RDMA netdev.
-It involves HW resource allocation/management for VNIC functionality.
-It interfaces with the network stack and implements the required
-net_device_ops functions. It expects Omni-Path encapsulated Ethernet
-packets in the transmit path and provides HW access to them. It strips
-the Omni-Path header from the received packets before passing them up
-the network stack. It also implements the RDMA netdev control operations.
-
-The OPA VNIC module implements the HW independent VNIC functionality.
-It consists of two parts. The VNIC Ethernet Management Agent (VEMA)
-registers itself with IB core as an IB client and interfaces with the
-IB MAD stack. It exchanges the management information with the Ethernet
-Manager (EM) and the VNIC netdev. The VNIC netdev part allocates and frees
-the OPA_VNIC RDMA netdev devices. It overrides the net_device_ops functions
-set by HW dependent VNIC driver where required to accommodate any control
-operation. It also handles the encapsulation of Ethernet packets with an
-Omni-Path header in the transmit path. For each VNIC interface, the
-information required for encapsulation is configured by the EM via VEMA MAD
-interface. It also passes any control information to the HW dependent driver
-by invoking the RDMA netdev control operations::
-
-        +-------------------+ +----------------------+
-        |                   | |       Linux          |
-        |     IB MAD        | |      Network         |
-        |                   | |       Stack          |
-        +-------------------+ +----------------------+
-                 |               |          |
-                 |               |          |
-        +----------------------------+      |
-        |                            |      |
-        |      OPA VNIC Module       |      |
-        |  (OPA VNIC RDMA Netdev     |      |
-        |     & EMA functions)       |      |
-        |                            |      |
-        +----------------------------+      |
-                    |                       |
-                    |                       |
-           +------------------+             |
-           |     IB core      |             |
-           +------------------+             |
-                    |                       |
-                    |                       |
-        +--------------------------------------------+
-        |                                            |
-        |      HFI1 Driver with VNIC support         |
-        |                                            |
-        +--------------------------------------------+
diff --git a/Documentation/kbuild/llvm.rst b/Documentation/kbuild/llvm.rst
index bc8a283bc44bb..441d8786fcbce 100644
--- a/Documentation/kbuild/llvm.rst
+++ b/Documentation/kbuild/llvm.rst
@@ -71,10 +71,6 @@ recommend::
 
 	PATH=/path/to/llvm/:$PATH make LLVM=-14
 
-``LLVM=0`` is not the same as omitting ``LLVM`` altogether, it will behave like
-``LLVM=1``. If you only wish to use certain LLVM utilities, use their
-respective make variables.
-
 The same value used for ``LLVM=`` should be set for each invocation of ``make``
 if configuring and building via distinct commands. ``LLVM=`` should also be set
 as an environment variable when running scripts that will eventually run
diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst
index dd64f5d7f3193..dc37402c0fee9 100644
--- a/Documentation/mm/damon/design.rst
+++ b/Documentation/mm/damon/design.rst
@@ -150,6 +150,8 @@ address on the given address space.  Support of ``address unit`` parameter is
 up to each operations set implementation.  ``paddr`` is the only operations set
 implementation that supports the parameter.
 
+If the value is smaller than ``PAGE_SIZE``, only a power of two should be used.
+
 .. _damon_core_logic:
 
 Core Logics
@@ -564,6 +566,18 @@ aggressiveness (the quota) of the corresponding scheme.  For example, if DAMOS
 is under achieving the goal, DAMOS automatically increases the quota.  If DAMOS
 is over achieving the goal, it decreases the quota.
 
+There are two such tuning algorithms that users can select as they need.
+
+- ``consist``: A proportional feedback loop based algorithm.  Tries to find an
+  optimum quota that should be consistently kept, to keep achieving the goal.
+  Useful for kernel-only operation on dynamic and long-running environments.
+  This is the default selection.  If unsure, use this.
+- ``temporal``: More straightforward algorithm.  Tries to achieve the goal as
+  fast as possible, using maximum allowed quota, but only for a temporal short
+  time.  When the quota is under-achieved, this algorithm keeps tuning quota to
+  a maximum allowed one.  Once the quota is [over]-achieved, this sets the
+  quota zero.  Useful for deterministic control required environments.
+
 The goal can be specified with five parameters, namely ``target_metric``,
 ``target_value``, ``current_value``, ``nid`` and ``path``.  The auto-tuning
 mechanism tries to make ``current_value`` of ``target_metric`` be same to
@@ -839,6 +853,10 @@ more detail, please read the usage documents for those
 (:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` and
 :doc:`/admin-guide/mm/damon/lru_sort`).
 
+.. _damon_design_special_purpose_modules_exclusivity:
+
+Note that these modules currently run in an exclusive manner.  If one of those
+is already running, others will return ``-EBUSY`` upon start requests.
 
 Sample DAMON Modules
 --------------------
diff --git a/Documentation/mm/damon/index.rst b/Documentation/mm/damon/index.rst
index 82f6c5eea49a7..318f6a7bfea47 100644
--- a/Documentation/mm/damon/index.rst
+++ b/Documentation/mm/damon/index.rst
@@ -12,7 +12,7 @@ DAMON is a Linux kernel subsystem for efficient :ref:`data access monitoring
  - *light-weight* (for production online usages),
  - *scalable* (in terms of memory size),
  - *tunable* (for flexible usages), and
- - *autoamted* (for production operation without manual tunings).
+ - *automated* (for production operation without manual tunings).
 
 .. toctree::
    :maxdepth: 2
diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst
index 41b1d73b9bd7b..bcb9798a27a86 100644
--- a/Documentation/mm/damon/maintainer-profile.rst
+++ b/Documentation/mm/damon/maintainer-profile.rst
@@ -63,10 +63,10 @@ management subsystem maintainer.
 Review cadence
 --------------
 
-The DAMON maintainer does the work on the usual work hour (09:00 to 17:00,
-Mon-Fri) in PT (Pacific Time).  The response to patches will occasionally be
-slow.  Do not hesitate to send a ping if you have not heard back within a week
-of sending a patch.
+The DAMON maintainer usually work in a flexible way, except early morning in PT
+(Pacific Time).  The response to patches will occasionally be slow.  Do not
+hesitate to send a ping if you have not heard back within a week of sending a
+patch.
 
 Mailing tool
 ------------
diff --git a/Documentation/mm/hugetlbfs_reserv.rst b/Documentation/mm/hugetlbfs_reserv.rst
index 4914fbf07966c..a49115db18c76 100644
--- a/Documentation/mm/hugetlbfs_reserv.rst
+++ b/Documentation/mm/hugetlbfs_reserv.rst
@@ -155,7 +155,7 @@ are enough free huge pages to accommodate the reservation.  If there are,
 the global reservation count resv_huge_pages is adjusted something like the
 following::
 
-	if (resv_needed <= (resv_huge_pages - free_huge_pages))
+	if (resv_needed <= (free_huge_pages - resv_huge_pages)
 		resv_huge_pages += resv_needed;
 
 Note that the global lock hugetlb_lock is held when checking and adjusting
diff --git a/Documentation/mm/hwpoison.rst b/Documentation/mm/hwpoison.rst
index 483b72aa7c117..71b4b45c35055 100644
--- a/Documentation/mm/hwpoison.rst
+++ b/Documentation/mm/hwpoison.rst
@@ -38,7 +38,7 @@ To quote the overview comment::
 	for the mapping from a vma to a process. Since this case is expected
 	to be rare we hope we can get away with this.
 
-The code consists of a the high level handler in mm/memory-failure.c,
+The code consists of the high level handler in mm/memory-failure.c,
 a new page poison bit and various checks in the VM to handle poisoned
 pages.
 
diff --git a/Documentation/mm/numa.rst b/Documentation/mm/numa.rst
index 0f1b56809dcab..b765295c6e856 100644
--- a/Documentation/mm/numa.rst
+++ b/Documentation/mm/numa.rst
@@ -140,7 +140,7 @@ this.
 
 If the architecture supports--does not hide--memoryless nodes, then CPUs
 attached to memoryless nodes would always incur the fallback path overhead
-or some subsystems would fail to initialize if they attempted to allocated
+or some subsystems would fail to initialize if they attempted to allocate
 memory exclusively from a node without memory.  To support such
 architectures transparently, kernel subsystems can use the numa_mem_id()
 or cpu_to_mem() function to locate the "local memory node" for the calling or
diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst
index b76183545e5bf..2398d87ac1563 100644
--- a/Documentation/mm/physical_memory.rst
+++ b/Documentation/mm/physical_memory.rst
@@ -500,11 +500,11 @@ General
 ``nr_isolate_pageblock``
   Number of isolated pageblocks. It is used to solve incorrect freepage counting
   problem due to racy retrieving migratetype of pageblock. Protected by
-  ``zone->lock``. Defined only when ``CONFIG_MEMORY_ISOLATION`` is enabled.
+  zone lock. Defined only when ``CONFIG_MEMORY_ISOLATION`` is enabled.
 
 ``span_seqlock``
   The seqlock to protect ``zone_start_pfn`` and ``spanned_pages``. It is a
-  seqlock because it has to be read outside of ``zone->lock``, and it is done in
+  seqlock because it has to be read outside of zone lock, and it is done in
   the main allocator path. However, the seqlock is written quite infrequently.
   Defined only when ``CONFIG_MEMORY_HOTPLUG`` is enabled.
 
diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst
index b4a55b6569fa5..9fa8642ded483 100644
--- a/Documentation/mm/vmemmap_dedup.rst
+++ b/Documentation/mm/vmemmap_dedup.rst
@@ -24,7 +24,7 @@ For each base page, there is a corresponding ``struct page``.
 Within the HugeTLB subsystem, only the first 4 ``struct page`` are used to
 contain unique information about a HugeTLB page. ``__NR_USED_SUBPAGE`` provides
 this upper limit. The only 'useful' information in the remaining ``struct page``
-is the compound_head field, and this field is the same for all tail pages.
+is the compound_info field, and this field is the same for all tail pages.
 
 By removing redundant ``struct page`` for HugeTLB pages, memory can be returned
 to the buddy allocator for other uses.
@@ -124,33 +124,35 @@ Here is how things look before optimization::
  |           |
  +-----------+
 
-The value of page->compound_head is the same for all tail pages. The first
-page of ``struct page`` (page 0) associated with the HugeTLB page contains the 4
-``struct page`` necessary to describe the HugeTLB. The only use of the remaining
-pages of ``struct page`` (page 1 to page 7) is to point to page->compound_head.
-Therefore, we can remap pages 1 to 7 to page 0. Only 1 page of ``struct page``
-will be used for each HugeTLB page. This will allow us to free the remaining
-7 pages to the buddy allocator.
+The first page of ``struct page`` (page 0) associated with the HugeTLB page
+contains the 4 ``struct page`` necessary to describe the HugeTLB. The remaining
+pages of ``struct page`` (page 1 to page 7) are tail pages.
+
+The optimization is only applied when the size of the struct page is a power
+of 2. In this case, all tail pages of the same order are identical. See
+compound_head(). This allows us to remap the tail pages of the vmemmap to a
+shared, read-only page. The head page is also remapped to a new page. This
+allows the original vmemmap pages to be freed.
 
 Here is how things look after remapping::
 
-    HugeTLB                  struct pages(8 pages)         page frame(8 pages)
- +-----------+ ---virt_to_page---> +-----------+   mapping to   +-----------+
- |           |                     |     0     | -------------> |     0     |
- |           |                     +-----------+                +-----------+
- |           |                     |     1     | ---------------^ ^ ^ ^ ^ ^ ^
- |           |                     +-----------+                  | | | | | |
- |           |                     |     2     | -----------------+ | | | | |
- |           |                     +-----------+                    | | | | |
- |           |                     |     3     | -------------------+ | | | |
- |           |                     +-----------+                      | | | |
- |           |                     |     4     | ---------------------+ | | |
- |    PMD    |                     +-----------+                        | | |
- |   level   |                     |     5     | -----------------------+ | |
- |  mapping  |                     +-----------+                          | |
- |           |                     |     6     | -------------------------+ |
- |           |                     +-----------+                            |
- |           |                     |     7     | ---------------------------+
+    HugeTLB                  struct pages(8 pages)                 page frame (new)
+ +-----------+ ---virt_to_page---> +-----------+   mapping to   +----------------+
+ |           |                     |     0     | -------------> |       0        |
+ |           |                     +-----------+                +----------------+
+ |           |                     |     1     | ------┐
+ |           |                     +-----------+       |
+ |           |                     |     2     | ------┼        +----------------------------+
+ |           |                     +-----------+       |        | A single, per-zone page    |
+ |           |                     |     3     | ------┼------> | frame shared among all     |
+ |           |                     +-----------+       |        | hugepages of the same size |
+ |           |                     |     4     | ------┼        +----------------------------+
+ |           |                     +-----------+       |
+ |           |                     |     5     | ------┼
+ |    PMD    |                     +-----------+       |
+ |   level   |                     |     6     | ------┼
+ |  mapping  |                     +-----------+       |
+ |           |                     |     7     | ------┘
  |           |                     +-----------+
  |           |
  |           |
@@ -172,16 +174,6 @@ The contiguous bit is used to increase the mapping size at the pmd and pte
 (last) level. So this type of HugeTLB page can be optimized only when its
 size of the ``struct page`` structs is greater than **1** page.
 
-Notice: The head vmemmap page is not freed to the buddy allocator and all
-tail vmemmap pages are mapped to the head vmemmap page frame. So we can see
-more than one ``struct page`` struct with ``PG_head`` (e.g. 8 per 2 MB HugeTLB
-page) associated with each HugeTLB page. The ``compound_head()`` can handle
-this correctly. There is only **one** head ``struct page``, the tail
-``struct page`` with ``PG_head`` are fake head ``struct page``.  We need an
-approach to distinguish between those two different types of ``struct page`` so
-that ``compound_head()`` can return the real head ``struct page`` when the
-parameter is the tail ``struct page`` but with ``PG_head``.
-
 Device DAX
 ==========
 
diff --git a/Documentation/netlink/genetlink.yaml b/Documentation/netlink/genetlink.yaml
index b020a537d8acd..a1194d5d93fc3 100644
--- a/Documentation/netlink/genetlink.yaml
+++ b/Documentation/netlink/genetlink.yaml
@@ -262,7 +262,7 @@ properties:
               description: Command flags.
               type: array
               items:
-                enum: [ admin-perm ]
+                enum: [ admin-perm, uns-admin-perm ]
             dont-validate:
               description: Kernel attribute validation flags.
               type: array
diff --git a/Documentation/netlink/netlink-raw.yaml b/Documentation/netlink/netlink-raw.yaml
index 0166a7e4afbbf..dd98dda55bd0f 100644
--- a/Documentation/netlink/netlink-raw.yaml
+++ b/Documentation/netlink/netlink-raw.yaml
@@ -19,6 +19,12 @@ $defs:
     type: [ string, integer ]
     pattern: ^[0-9A-Za-z_-]+( - 1)?$
     minimum: 0
+  len-or-limit:
+    # literal int, const name, or limit based on fixed-width type
+    # e.g. u8-min, u16-max, etc.
+    type: [ string, integer ]
+    pattern: ^[0-9A-Za-z_-]+$
+    minimum: 0
 
 # Schema for specs
 title: Protocol
@@ -270,7 +276,10 @@ properties:
                     type: string
                   min:
                     description: Min value for an integer attribute.
-                    type: integer
+                    $ref: '#/$defs/len-or-limit'
+                  max:
+                    description: Max value for an integer attribute.
+                    $ref: '#/$defs/len-or-limit'
                   min-len:
                     description: Min length for a binary attribute.
                     $ref: '#/$defs/len-or-define'
diff --git a/Documentation/netlink/specs/devlink.yaml b/Documentation/netlink/specs/devlink.yaml
index 837112da67380..b495d56b91371 100644
--- a/Documentation/netlink/specs/devlink.yaml
+++ b/Documentation/netlink/specs/devlink.yaml
@@ -867,6 +867,12 @@ attribute-sets:
         type: flag
         doc: Request restoring parameter to its default value.
         value: 183
+      -
+        name: index
+        type: uint
+        doc: Unique devlink instance index.
+        checks:
+          max: u32-max
   -
     name: dl-dev-stats
     subset-of: devlink
@@ -1306,11 +1312,13 @@ operations:
           attributes: &dev-id-attrs
             - bus-name
             - dev-name
+            - index
         reply: &get-reply
           value: 3
           attributes:
             - bus-name
             - dev-name
+            - index
             - reload-failed
             - dev-stats
       dump:
@@ -1329,6 +1337,7 @@ operations:
           attributes: &port-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
         reply:
           value: 7
@@ -1353,6 +1362,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - port-type
             - port-function
@@ -1370,6 +1380,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - port-flavour
             - port-pci-pf-number
@@ -1404,6 +1415,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - port-split-count
 
@@ -1432,6 +1444,7 @@ operations:
           attributes: &sb-id-attrs
             - bus-name
             - dev-name
+            - index
             - sb-index
         reply: &sb-get-reply
           value: 13
@@ -1454,6 +1467,7 @@ operations:
           attributes: &sb-pool-id-attrs
             - bus-name
             - dev-name
+            - index
             - sb-index
             - sb-pool-index
         reply: &sb-pool-get-reply
@@ -1477,6 +1491,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - sb-index
             - sb-pool-index
             - sb-pool-threshold-type
@@ -1495,6 +1510,7 @@ operations:
           attributes: &sb-port-pool-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - sb-index
             - sb-pool-index
@@ -1519,6 +1535,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - sb-index
             - sb-pool-index
@@ -1537,6 +1554,7 @@ operations:
           attributes: &sb-tc-pool-bind-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - sb-index
             - sb-pool-type
@@ -1562,6 +1580,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - sb-index
             - sb-pool-index
@@ -1583,6 +1602,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - sb-index
 
     -
@@ -1598,6 +1618,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - sb-index
 
     -
@@ -1616,6 +1637,7 @@ operations:
           attributes: &eswitch-attrs
             - bus-name
             - dev-name
+            - index
             - eswitch-mode
             - eswitch-inline-mode
             - eswitch-encap-mode
@@ -1644,12 +1666,14 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-table-name
         reply:
           value: 31
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-tables
 
     -
@@ -1664,11 +1688,13 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-table-name
         reply:
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-entries
 
     -
@@ -1683,10 +1709,12 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
         reply:
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-headers
 
     -
@@ -1702,6 +1730,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - dpipe-table-name
             - dpipe-table-counters-enabled
 
@@ -1718,6 +1747,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - resource-id
             - resource-size
 
@@ -1733,11 +1763,13 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
         reply:
           value: 36
           attributes:
             - bus-name
             - dev-name
+            - index
             - resource-list
 
     -
@@ -1753,6 +1785,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - reload-action
             - reload-limits
             - netns-pid
@@ -1762,6 +1795,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - reload-actions-performed
 
     -
@@ -1776,6 +1810,7 @@ operations:
           attributes: &param-id-attrs
             - bus-name
             - dev-name
+            - index
             - param-name
         reply: &param-get-reply
           attributes: *param-id-attrs
@@ -1797,6 +1832,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - param-name
             - param-type
             # param-value-data is missing here as the type is variable
@@ -1816,6 +1852,7 @@ operations:
           attributes: &region-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - region-name
         reply: &region-get-reply
@@ -1840,6 +1877,7 @@ operations:
           attributes: &region-snapshot-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - region-name
             - region-snapshot-id
@@ -1870,6 +1908,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - region-name
             - region-snapshot-id
@@ -1881,6 +1920,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - region-name
 
@@ -1930,6 +1970,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - info-driver-name
             - info-serial-number
             - info-version-fixed
@@ -1951,6 +1992,7 @@ operations:
           attributes: &health-reporter-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - health-reporter-name
         reply: &health-reporter-get-reply
@@ -1973,6 +2015,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
             - health-reporter-name
             - health-reporter-graceful-period
@@ -2043,6 +2086,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - flash-update-file-name
             - flash-update-component
             - flash-update-overwrite-mask
@@ -2060,6 +2104,7 @@ operations:
           attributes: &trap-id-attrs
             - bus-name
             - dev-name
+            - index
             - trap-name
         reply: &trap-get-reply
           value: 63
@@ -2082,6 +2127,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - trap-name
             - trap-action
 
@@ -2098,6 +2144,7 @@ operations:
           attributes: &trap-group-id-attrs
             - bus-name
             - dev-name
+            - index
             - trap-group-name
         reply: &trap-group-get-reply
           value: 67
@@ -2120,6 +2167,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - trap-group-name
             - trap-action
             - trap-policer-id
@@ -2137,6 +2185,7 @@ operations:
           attributes: &trap-policer-id-attrs
             - bus-name
             - dev-name
+            - index
             - trap-policer-id
         reply: &trap-policer-get-reply
           value: 71
@@ -2159,6 +2208,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - trap-policer-id
             - trap-policer-rate
             - trap-policer-burst
@@ -2189,6 +2239,7 @@ operations:
           attributes: &rate-id-attrs
             - bus-name
             - dev-name
+            - index
             - port-index
             - rate-node-name
         reply: &rate-get-reply
@@ -2212,6 +2263,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - rate-node-name
             - rate-tx-share
             - rate-tx-max
@@ -2233,6 +2285,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - rate-node-name
             - rate-tx-share
             - rate-tx-max
@@ -2254,6 +2307,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - rate-node-name
 
     -
@@ -2269,6 +2323,7 @@ operations:
           attributes: &linecard-id-attrs
             - bus-name
             - dev-name
+            - index
             - linecard-index
         reply: &linecard-get-reply
           value: 80
@@ -2291,6 +2346,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - linecard-index
             - linecard-type
 
@@ -2324,6 +2380,7 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - selftests
 
     -
@@ -2335,4 +2392,5 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - index
             - port-index
diff --git a/Documentation/netlink/specs/drm_ras.yaml b/Documentation/netlink/specs/drm_ras.yaml
new file mode 100644
index 0000000000000..79af25dac3c57
--- /dev/null
+++ b/Documentation/netlink/specs/drm_ras.yaml
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+---
+name: drm-ras
+protocol: genetlink
+uapi-header: drm/drm_ras.h
+
+doc: >-
+  DRM RAS (Reliability, Availability, Serviceability) over Generic Netlink.
+  Provides a standardized mechanism for DRM drivers to register "nodes"
+  representing hardware/software components capable of reporting error counters.
+  Userspace tools can query the list of nodes or individual error counters
+  via the Generic Netlink interface.
+
+definitions:
+  -
+    type: enum
+    name: node-type
+    value-start: 1
+    entries: [error-counter]
+    doc: >-
+         Type of the node. Currently, only error-counter nodes are
+         supported, which expose reliability counters for a hardware/software
+         component.
+
+attribute-sets:
+  -
+    name: node-attrs
+    attributes:
+      -
+        name: node-id
+        type: u32
+        doc: >-
+             Unique identifier for the node.
+             Assigned dynamically by the DRM RAS core upon registration.
+      -
+        name: device-name
+        type: string
+        doc: >-
+             Device name chosen by the driver at registration.
+             Can be a PCI BDF, UUID, or module name if unique.
+      -
+        name: node-name
+        type: string
+        doc: >-
+             Node name chosen by the driver at registration.
+             Can be an IP block name, or any name that identifies the
+             RAS node inside the device.
+      -
+        name: node-type
+        type: u32
+        doc: Type of this node, identifying its function.
+        enum: node-type
+  -
+    name: error-counter-attrs
+    attributes:
+      -
+        name: node-id
+        type: u32
+        doc: Node ID targeted by this error counter operation.
+      -
+        name: error-id
+        type: u32
+        doc: Unique identifier for a specific error counter within an node.
+      -
+        name: error-name
+        type: string
+        doc: Name of the error.
+      -
+        name: error-value
+        type: u32
+        doc: Current value of the requested error counter.
+
+operations:
+  list:
+    -
+      name: list-nodes
+      doc: >-
+           Retrieve the full list of currently registered DRM RAS nodes.
+           Each node includes its dynamically assigned ID, name, and type.
+           **Important:** User space must call this operation first to obtain
+           the node IDs. These IDs are required for all subsequent
+           operations on nodes, such as querying error counters.
+      attribute-set: node-attrs
+      flags: [admin-perm]
+      dump:
+        reply:
+          attributes:
+            - node-id
+            - device-name
+            - node-name
+            - node-type
+    -
+      name: get-error-counter
+      doc: >-
+           Retrieve error counter for a given node.
+           The response includes the id, the name, and even the current
+           value of each counter.
+      attribute-set: error-counter-attrs
+      flags: [admin-perm]
+      do:
+        request:
+          attributes:
+            - node-id
+            - error-id
+        reply:
+          attributes: &errorinfo
+            - error-id
+            - error-name
+            - error-value
+      dump:
+        request:
+          attributes:
+            - node-id
+        reply:
+          attributes: *errorinfo
diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml
index 0a2d2343f79ac..5dd4d1b5d94bb 100644
--- a/Documentation/netlink/specs/ethtool.yaml
+++ b/Documentation/netlink/specs/ethtool.yaml
@@ -306,10 +306,6 @@ attribute-sets:
     name: strings
     attr-cnt-name: __ethtool-a-strings-cnt
     attributes:
-      -
-        name: unspec
-        type: unused
-        value: 0
       -
         name: unspec
         type: unused
@@ -861,6 +857,12 @@ attribute-sets:
         name: tx-profile
         type: nest
         nested-attributes: profile
+      -
+        name: rx-cqe-frames
+        type: u32
+      -
+        name: rx-cqe-nsecs
+        type: u32
 
   -
     name: pause-stat
@@ -879,6 +881,19 @@ attribute-sets:
       -
         name: rx-frames
         type: u64
+      -
+        name: tx-pause-storm-events
+        type: u64
+        doc: >-
+            TX pause storm event count. Increments each time device
+            detects that its pause assertion condition has been true
+            for too long for normal operation. As a result, the device
+            has temporarily disabled its own Pause TX function to
+            protect the network from itself.
+            This counter should never increment under normal overload
+            conditions; it indicates catastrophic failure like an OS
+            crash. The rate of incrementing is implementation specific.
+
   -
     name: pause
     attr-cnt-name: __ethtool-a-pause-cnt
@@ -2244,6 +2259,8 @@ operations:
             - tx-aggr-time-usecs
             - rx-profile
             - tx-profile
+            - rx-cqe-frames
+            - rx-cqe-nsecs
       dump: *coalesce-get-op
     -
       name: coalesce-set
diff --git a/Documentation/netlink/specs/net_shaper.yaml b/Documentation/netlink/specs/net_shaper.yaml
index 0b1b54be48f92..3f2ad772b64b1 100644
--- a/Documentation/netlink/specs/net_shaper.yaml
+++ b/Documentation/netlink/specs/net_shaper.yaml
@@ -247,8 +247,8 @@ operations:
       flags: [admin-perm]
 
       do:
-        pre: net-shaper-nl-pre-doit
-        post: net-shaper-nl-post-doit
+        pre: net-shaper-nl-pre-doit-write
+        post: net-shaper-nl-post-doit-write
         request:
           attributes:
             - ifindex
@@ -278,8 +278,8 @@ operations:
       flags: [admin-perm]
 
       do:
-        pre: net-shaper-nl-pre-doit
-        post: net-shaper-nl-post-doit
+        pre: net-shaper-nl-pre-doit-write
+        post: net-shaper-nl-post-doit-write
         request:
           attributes: *ns-binding
 
@@ -309,8 +309,8 @@ operations:
       flags: [admin-perm]
 
       do:
-        pre: net-shaper-nl-pre-doit
-        post: net-shaper-nl-post-doit
+        pre: net-shaper-nl-pre-doit-write
+        post: net-shaper-nl-post-doit-write
         request:
           attributes:
             - ifindex
diff --git a/Documentation/netlink/specs/nftables.yaml b/Documentation/netlink/specs/nftables.yaml
index 17ad707fa0d5d..21edf3d25f34b 100644
--- a/Documentation/netlink/specs/nftables.yaml
+++ b/Documentation/netlink/specs/nftables.yaml
@@ -66,9 +66,21 @@ definitions:
     name: bitwise-ops
     type: enum
     entries:
-      - bool
-      - lshift
-      - rshift
+      -
+        name: mask-xor  # aka bool (old name)
+        doc: >-
+          mask-and-xor operation used to implement NOT, AND, OR and XOR boolean
+          operations
+      -
+        name: lshift
+      -
+        name: rshift
+      -
+        name: and
+      -
+        name: or
+      -
+        name: xor
   -
     name: cmp-ops
     type: enum
@@ -132,6 +144,12 @@ definitions:
       - object
       - concat
       - expr
+  -
+    name: set-elem-flags
+    type: flags
+    entries:
+      - interval-end
+      - catchall
   -
     name: lookup-flags
     type: flags
@@ -225,19 +243,244 @@ definitions:
       - icmp-unreach
       - tcp-rst
       - icmpx-unreach
+  -
+    name: reject-inet-code
+    doc: These codes are mapped to real ICMP and ICMPv6 codes.
+    type: enum
+    entries:
+      - icmpx-no-route
+      - icmpx-port-unreach
+      - icmpx-host-unreach
+      - icmpx-admin-prohibited
+  -
+    name: payload-base
+    type: enum
+    entries:
+      - link-layer-header
+      - network-header
+      - transport-header
+      - inner-header
+      - tun-header
+  -
+    name: range-ops
+    doc: Range operator
+    type: enum
+    entries:
+      - eq
+      - neq
+  -
+    name: registers
+    doc: |
+      nf_tables registers.
+      nf_tables used to have five registers: a verdict register and four data
+      registers of size 16. The data registers have been changed to 16 registers
+      of size 4. For compatibility reasons, the NFT_REG_[1-4] registers still
+      map to areas of size 16, the 4 byte registers are addressed using
+      NFT_REG32_00 - NFT_REG32_15.
+    type: enum
+    entries:
+      -
+        name: reg-verdict
+      -
+        name: reg-1
+      -
+        name: reg-2
+      -
+        name: reg-3
+      -
+        name: reg-4
+      -
+        name: reg32-00
+        value: 8
+      -
+        name: reg32-01
+      -
+        name: reg32-02
+      -
+        name: reg32-03
+      -
+        name: reg32-04
+      -
+        name: reg32-05
+      -
+        name: reg32-06
+      -
+        name: reg32-07
+      -
+        name: reg32-08
+      -
+        name: reg32-09
+      -
+        name: reg32-10
+      -
+        name: reg32-11
+      -
+        name: reg32-12
+      -
+        name: reg32-13
+      -
+        name: reg32-14
+      -
+        name: reg32-15
+  -
+    name: numgen-types
+    type: enum
+    entries:
+      - incremental
+      - random
+  -
+    name: log-level
+    doc: nf_tables log levels
+    type: enum
+    entries:
+      -
+        name: emerg
+        doc: system is unusable
+      -
+        name: alert
+        doc: action must be taken immediately
+      -
+        name: crit
+        doc: critical conditions
+      -
+        name: err
+        doc: error conditions
+      -
+        name: warning
+        doc: warning conditions
+      -
+        name: notice
+        doc: normal but significant condition
+      -
+        name: info
+        doc: informational
+      -
+        name: debug
+        doc: debug-level messages
+      -
+        name: audit
+        doc: enabling audit logging
+  -
+    name: log-flags
+    doc: nf_tables log flags
+    header: linux/netfilter/nf_log.h
+    type: flags
+    entries:
+      -
+        name: tcpseq
+        doc: Log TCP sequence numbers
+      -
+        name: tcpopt
+        doc: Log TCP options
+      -
+        name: ipopt
+        doc: Log IP options
+      -
+        name: uid
+        doc: Log UID owning local socket
+      -
+        name: nflog
+        doc: Unsupported, don't reuse
+      -
+        name: macdecode
+        doc: Decode MAC header
 
 attribute-sets:
   -
-    name: empty-attrs
+    name: log-attrs
+    doc: log expression netlink attributes
     attributes:
+      # Mentioned in nft_log_init()
       -
-        name: name
+        name: group
+        doc: netlink group to send messages to
+        type: u16
+        byte-order: big-endian
+      -
+        name: prefix
+        doc: prefix to prepend to log messages
         type: string
+      -
+        name: snaplen
+        doc: length of payload to include in netlink message
+        type: u32
+        byte-order: big-endian
+      -
+        name: qthreshold
+        doc: queue threshold
+        type: u16
+        byte-order: big-endian
+      -
+        name: level
+        doc: log level
+        type: u32
+        enum: log-level
+        byte-order: big-endian
+      -
+        name: flags
+        doc: logging flags
+        type: u32
+        enum: log-flags
+        byte-order: big-endian
+  -
+    name: numgen-attrs
+    doc: nf_tables number generator expression netlink attributes
+    attributes:
+      -
+        name: dreg
+        doc: destination register
+        type: u32
+        enum: registers
+      -
+        name: modulus
+        doc: maximum counter value
+        type: u32
+        byte-order: big-endian
+      -
+        name: type
+        doc: operation type
+        type: u32
+        byte-order: big-endian
+        enum: numgen-types
+      -
+        name: offset
+        doc: offset to be added to the counter
+        type: u32
+        byte-order: big-endian
+  -
+    name: range-attrs
+    attributes:
+      # Mentioned in net/netfilter/nft_range.c
+      -
+        name: sreg
+        doc: source register of data to compare
+        type: u32
+        byte-order: big-endian
+        enum: registers
+      -
+        name: op
+        doc: cmp operation
+        type: u32
+        byte-order: big-endian
+        enum: range-ops
+        checks:
+          max: 255
+      -
+        name: from-data
+        doc: data range from
+        type: nest
+        nested-attributes: data-attrs
+      -
+        name: to-data
+        doc: data range to
+        type: nest
+        nested-attributes: data-attrs
   -
     name: batch-attrs
     attributes:
       -
         name: genid
+        doc: generation ID for this changeset
         type: u32
         byte-order: big-endian
   -
@@ -264,10 +507,18 @@ attribute-sets:
         type: u64
         byte-order: big-endian
         doc: numeric handle of the table
+      -
+        name: pad
+        type: pad
       -
         name: userdata
         type: binary
         doc: user data
+      -
+        name: owner
+        type: u32
+        byte-order: big-endian
+        doc: owner of this table through netlink portID
   -
     name: chain-attrs
     attributes:
@@ -371,9 +622,11 @@ attribute-sets:
       -
         name: bytes
         type: u64
+        byte-order: big-endian
       -
         name: packets
         type: u64
+        byte-order: big-endian
   -
     name: rule-attrs
     attributes:
@@ -443,15 +696,18 @@ attribute-sets:
         selector: name
         doc: type specific data
   -
+    # Mentioned in nft_parse_compat() in net/netfilter/nft_compat.c
     name: rule-compat-attrs
     attributes:
       -
         name: proto
-        type: binary
+        type: u32
+        byte-order: big-endian
         doc: numeric value of the handled protocol
       -
         name: flags
-        type: binary
+        type: u32
+        byte-order: big-endian
         doc: bitmask of flags
   -
     name: set-attrs
@@ -540,6 +796,15 @@ attribute-sets:
         type: nest
         nested-attributes: set-list-attrs
         doc: list of expressions
+      -
+        name: type
+        type: string
+        doc: set backend type
+      -
+        name: count
+        type: u32
+        byte-order: big-endian
+        doc: number of set elements
   -
     name: set-desc-attrs
     attributes:
@@ -767,6 +1032,22 @@ attribute-sets:
         nested-attributes: hook-dev-attrs
   -
     name: expr-bitwise-attrs
+    doc: |
+      The bitwise expression supports boolean and shift operations. It
+      implements the boolean operations by performing the following
+      operation::
+
+          dreg = (sreg & mask) ^ xor
+
+          with these mask and xor values:
+
+          op      mask    xor
+          ----    ----    ---
+          NOT:     1       1
+          OR:     ~x       x
+          XOR:     1       x
+          AND:     x       0
+
     attributes:
       -
         name: sreg
@@ -793,6 +1074,8 @@ attribute-sets:
         type: u32
         byte-order: big-endian
         enum: bitwise-ops
+        checks:
+          max: 255
       -
         name: data
         type: nest
@@ -829,25 +1112,31 @@ attribute-sets:
     attributes:
       -
         name: code
+        doc: nf_tables verdict
         type: u32
         byte-order: big-endian
         enum: verdict-code
       -
         name: chain
+        doc: jump target chain name
         type: string
       -
         name: chain-id
+        doc: jump target chain ID
         type: u32
+        byte-order: big-endian
   -
     name: expr-counter-attrs
     attributes:
       -
         name: bytes
         type: u64
+        byte-order: big-endian
         doc: Number of bytes
       -
         name: packets
         type: u64
+        byte-order: big-endian
         doc: Number of packets
       -
         name: pad
@@ -932,6 +1221,25 @@ attribute-sets:
         type: u32
         byte-order: big-endian
         enum: lookup-flags
+  -
+    name: expr-masq-attrs
+    attributes:
+      -
+        name: flags
+        type: u32
+        byte-order: big-endian
+        enum: nat-range-flags
+        enum-as-flags: true
+      -
+        name: reg-proto-min
+        type: u32
+        byte-order: big-endian
+        enum: registers
+      -
+        name: reg-proto-max
+        type: u32
+        byte-order: big-endian
+        enum: registers
   -
     name: expr-meta-attrs
     attributes:
@@ -983,37 +1291,49 @@ attribute-sets:
         enum-as-flags: true
   -
     name: expr-payload-attrs
+    doc: nf_tables payload expression netlink attributes
     attributes:
       -
         name: dreg
+        doc: destination register to load data into
         type: u32
         byte-order: big-endian
+        enum: registers
       -
         name: base
+        doc: payload base
         type: u32
+        enum: payload-base
         byte-order: big-endian
       -
         name: offset
+        doc: payload offset relative to base
         type: u32
         byte-order: big-endian
       -
         name: len
+        doc: payload length
         type: u32
         byte-order: big-endian
       -
         name: sreg
+        doc: source register to load data from
         type: u32
         byte-order: big-endian
+        enum: registers
       -
         name: csum-type
+        doc: checksum type
         type: u32
         byte-order: big-endian
       -
         name: csum-offset
+        doc: checksum offset relative to base
         type: u32
         byte-order: big-endian
       -
         name: csum-flags
+        doc: checksum flags
         type: u32
         byte-order: big-endian
   -
@@ -1079,6 +1399,61 @@ attribute-sets:
         type: u32
         byte-order: big-endian
         doc: id of object map
+  -
+    name: compat-target-attrs
+    header: linux/netfilter/nf_tables_compat.h
+    attributes:
+      -
+        name: name
+        type: string
+        checks:
+          max-len: 32
+      -
+        name: rev
+        type: u32
+        byte-order: big-endian
+        checks:
+          max: 255
+      -
+        name: info
+        type: binary
+  -
+    name: compat-match-attrs
+    header: linux/netfilter/nf_tables_compat.h
+    attributes:
+      -
+        name: name
+        type: string
+        checks:
+          max-len: 32
+      -
+        name: rev
+        type: u32
+        byte-order: big-endian
+        checks:
+          max: 255
+      -
+        name: info
+        type: binary
+  -
+    name: compat-attrs
+    header: linux/netfilter/nf_tables_compat.h
+    attributes:
+      -
+        name: name
+        type: string
+        checks:
+          max-len: 32
+      -
+        name: rev
+        type: u32
+        byte-order: big-endian
+        checks:
+          max: 255
+      -
+        name: type
+        type: u32
+        byte-order: big-endian
 
 sub-messages:
   -
@@ -1105,15 +1480,24 @@ sub-messages:
       -
         value: immediate
         attribute-set: expr-immediate-attrs
+      -
+        value: log
+        attribute-set: log-attrs
       -
         value: lookup
         attribute-set: expr-lookup-attrs
+      -
+        value: match
+        attribute-set: compat-match-attrs
       -
         value: meta
         attribute-set: expr-meta-attrs
       -
         value: nat
         attribute-set: expr-nat-attrs
+      -
+        value: numgen
+        attribute-set: numgen-attrs
       -
         value: objref
         attribute-set: expr-objref-attrs
@@ -1123,6 +1507,9 @@ sub-messages:
       -
         value: quota
         attribute-set: quota-attrs
+      -
+        value: range
+        attribute-set: range-attrs
       -
         value: reject
         attribute-set: expr-reject-attrs
@@ -1132,6 +1519,9 @@ sub-messages:
       -
         value: tproxy
         attribute-set: expr-tproxy-attrs
+        # There're more sub-messages to go:
+        #   grep -A10 nft_expr_type
+        # and look for .name\s*=\s*"..."
   -
     name: obj-data
     formats:
@@ -1178,7 +1568,10 @@ operations:
         request:
           value: 0xa00
           attributes:
+            # Mentioned in nf_tables_newtable()
             - name
+            - flags
+            - userdata
     -
       name: gettable
       doc: Get / dump tables.
@@ -1188,11 +1581,21 @@ operations:
         request:
           value: 0xa01
           attributes:
+            # Mentioned in nf_tables_gettable()
             - name
         reply:
           value: 0xa00
-          attributes:
+          attributes: &get-table
+            # Mentioned in nf_tables_fill_table_info()
             - name
+            - use
+            - handle
+            - flags
+            - owner
+            - userdata
+      dump:
+        reply:
+          attributes: *get-table
     -
       name: deltable
       doc: Delete an existing table.
@@ -1201,8 +1604,10 @@ operations:
       do:
         request:
           value: 0xa02
-          attributes:
+          attributes: &del-table
+            # Mentioned in nf_tables_deltable()
             - name
+            - handle
     -
       name: destroytable
       doc: |
@@ -1213,8 +1618,7 @@ operations:
       do:
         request:
           value: 0xa1a
-          attributes:
-            - name
+          attributes: *del-table
     -
       name: newchain
       doc: Create a new chain.
@@ -1224,7 +1628,19 @@ operations:
         request:
           value: 0xa03
           attributes:
+            # Mentioned in nf_tables_newchain()
+            - table
+            - handle
+            - policy
+            - flags
+            # Mentioned in nf_tables_updchain()
+            - hook
             - name
+            - counters
+            # Mentioned in nf_tables_addchain()
+            - userdata
+            # Mentioned in nft_chain_parse_hook()
+            - type
     -
       name: getchain
       doc: Get / dump chains.
@@ -1234,11 +1650,27 @@ operations:
         request:
           value: 0xa04
           attributes:
+            # Mentioned in nf_tables_getchain()
+            - table
             - name
         reply:
           value: 0xa03
-          attributes:
+          attributes: &get-chain
+            # Mentioned in nf_tables_fill_chain_info()
+            - table
             - name
+            - handle
+            - hook
+            - policy
+            - type
+            - flags
+            - counters
+            - id
+            - use
+            - userdata
+      dump:
+        reply:
+          attributes: *get-chain
     -
       name: delchain
       doc: Delete an existing chain.
@@ -1247,8 +1679,12 @@ operations:
       do:
         request:
           value: 0xa05
-          attributes:
+          attributes: &del-chain
+            # Mentioned in nf_tables_delchain()
+            - table
+            - handle
             - name
+            - hook
     -
       name: destroychain
       doc: |
@@ -1259,8 +1695,7 @@ operations:
       do:
         request:
           value: 0xa1b
-          attributes:
-            - name
+          attributes: *del-chain
     -
       name: newrule
       doc: Create a new rule.
@@ -1270,7 +1705,16 @@ operations:
         request:
           value: 0xa06
           attributes:
-            - name
+            # Mentioned in nf_tables_newrule()
+            - table
+            - chain
+            - chain-id
+            - handle
+            - position
+            - position-id
+            - expressions
+            - userdata
+            - compat
     -
       name: getrule
       doc: Get / dump rules.
@@ -1279,12 +1723,30 @@ operations:
       do:
         request:
           value: 0xa07
-          attributes:
-            - name
+          attributes: &get-rule-request
+            # Mentioned in nf_tables_getrule_single()
+            - table
+            - chain
+            - handle
         reply:
           value: 0xa06
+          attributes: &get-rule
+            # Mentioned in nf_tables_fill_rule_info()
+            - table
+            - chain
+            - handle
+            - position
+            - expressions
+            - userdata
+      dump:
+        request:
           attributes:
-            - name
+            # Mentioned in nf_tables_dump_rules_start()
+            - table
+            - chain
+        reply:
+          attributes: *get-rule
+
     -
       name: getrule-reset
       doc: Get / dump rules and reset stateful expressions.
@@ -1293,12 +1755,15 @@ operations:
       do:
         request:
           value: 0xa19
-          attributes:
-            - name
+          attributes: *get-rule-request
         reply:
           value: 0xa06
-          attributes:
-            - name
+          attributes: *get-rule
+      dump:
+        request:
+          attributes: *get-rule-request
+        reply:
+          attributes: *get-rule
     -
       name: delrule
       doc: Delete an existing rule.
@@ -1307,8 +1772,11 @@ operations:
       do:
         request:
           value: 0xa08
-          attributes:
-            - name
+          attributes: &del-rule
+            - table
+            - chain
+            - handle
+            - id
     -
       name: destroyrule
       doc: |
@@ -1318,8 +1786,7 @@ operations:
       do:
         request:
           value: 0xa1c
-          attributes:
-            - name
+          attributes: *del-rule
     -
       name: newset
       doc: Create a new set.
@@ -1329,7 +1796,21 @@ operations:
         request:
           value: 0xa09
           attributes:
+            # Mentioned in nf_tables_newset()
+            - table
             - name
+            - key-len
+            - id
+            - key-type
+            - flags
+            - data-type
+            - data-len
+            - obj-type
+            - timeout
+            - gc-interval
+            - policy
+            - desc
+            - userdata
     -
       name: getset
       doc: Get / dump sets.
@@ -1339,11 +1820,35 @@ operations:
         request:
           value: 0xa0a
           attributes:
+            # Mentioned in nf_tables_getset()
+            - table
             - name
         reply:
           value: 0xa09
-          attributes:
+          attributes: &get-set
+            # Mentioned in nf_tables_fill_set()
+            - table
             - name
+            - handle
+            - flags
+            - key-len
+            - key-type
+            - data-type
+            - data-len
+            - obj-type
+            - gc-interval
+            - policy
+            - userdata
+            - desc
+            - expr
+            - expressions
+      dump:
+        request:
+          attributes:
+            # Mentioned in nf_tables_getset()
+            - table
+        reply:
+          attributes: *get-set
     -
       name: delset
       doc: Delete an existing set.
@@ -1352,7 +1857,10 @@ operations:
       do:
         request:
           value: 0xa0b
-          attributes:
+          attributes: &del-set
+            # Mentioned in nf_tables_delset()
+            - table
+            - handle
             - name
     -
       name: destroyset
@@ -1363,8 +1871,7 @@ operations:
       do:
         request:
           value: 0xa1d
-          attributes:
-            - name
+          attributes: *del-set
     -
       name: newsetelem
       doc: Create a new set element.
@@ -1374,7 +1881,11 @@ operations:
         request:
           value: 0xa0c
           attributes:
-            - name
+            # Mentioned in nf_tables_newsetelem()
+            - table
+            - set
+            - set-id
+            - elements
     -
       name: getsetelem
       doc: Get / dump set elements.
@@ -1384,11 +1895,27 @@ operations:
         request:
           value: 0xa0d
           attributes:
-            - name
+            # Mentioned in nf_tables_getsetelem()
+            - table
+            - set
+            - elements
         reply:
           value: 0xa0c
           attributes:
-            - name
+            # Mentioned in nf_tables_fill_setelem_info()
+            - elements
+      dump:
+        request:
+          attributes: &dump-set-request
+            # Mentioned in nft_set_dump_ctx_init()
+            - table
+            - set
+        reply:
+          attributes: &dump-set
+            # Mentioned in nf_tables_dump_set()
+            - table
+            - set
+            - elements
     -
       name: getsetelem-reset
       doc: Get / dump set elements and reset stateful expressions.
@@ -1398,11 +1925,20 @@ operations:
         request:
           value: 0xa21
           attributes:
-            - name
+            # Mentioned in nf_tables_getsetelem_reset()
+            - elements
         reply:
           value: 0xa0c
           attributes:
-            - name
+            # Mentioned in nf_tables_dumpreset_set()
+            - table
+            - set
+            - elements
+      dump:
+        request:
+          attributes: *dump-set-request
+        reply:
+          attributes: *dump-set
     -
       name: delsetelem
       doc: Delete an existing set element.
@@ -1411,8 +1947,11 @@ operations:
       do:
         request:
           value: 0xa0e
-          attributes:
-            - name
+          attributes: &del-setelem
+            # Mentioned in nf_tables_delsetelem()
+            - table
+            - set
+            - elements
     -
       name: destroysetelem
       doc: Delete an existing set element with destroy semantics.
@@ -1421,8 +1960,7 @@ operations:
       do:
         request:
           value: 0xa1e
-          attributes:
-            - name
+          attributes: *del-setelem
     -
       name: getgen
       doc: Get / dump rule-set generation.
@@ -1431,12 +1969,16 @@ operations:
       do:
         request:
           value: 0xa10
-          attributes:
-            - name
         reply:
           value: 0xa0f
-          attributes:
-            - name
+          attributes: &get-gen
+            # Mentioned in nf_tables_fill_gen_info()
+            - id
+            - proc-pid
+            - proc-name
+      dump:
+        reply:
+          attributes: *get-gen
     -
       name: newobj
       doc: Create a new stateful object.
@@ -1446,7 +1988,12 @@ operations:
         request:
           value: 0xa12
           attributes:
+            # Mentioned in nf_tables_newobj()
+            - type
             - name
+            - data
+            - table
+            - userdata
     -
       name: getobj
       doc: Get / dump stateful objects.
@@ -1456,11 +2003,29 @@ operations:
         request:
           value: 0xa13
           attributes:
+            # Mentioned in nf_tables_getobj_single()
             - name
+            - type
+            - table
         reply:
           value: 0xa12
-          attributes:
+          attributes: &obj-info
+            # Mentioned in nf_tables_fill_obj_info()
+            - table
             - name
+            - type
+            - handle
+            - use
+            - data
+            - userdata
+      dump:
+        request:
+          attributes:
+            # Mentioned in nf_tables_dump_obj_start()
+            - table
+            - type
+        reply:
+          attributes: *obj-info
     -
       name: delobj
       doc: Delete an existing stateful object.
@@ -1470,7 +2035,11 @@ operations:
         request:
           value: 0xa14
           attributes:
+            # Mentioned in nf_tables_delobj()
+            - table
             - name
+            - type
+            - handle
     -
       name: destroyobj
       doc: Delete an existing stateful object with destroy semantics.
@@ -1480,7 +2049,11 @@ operations:
         request:
           value: 0xa1f
           attributes:
+            # Mentioned in nf_tables_delobj()
+            - table
             - name
+            - type
+            - handle
     -
       name: newflowtable
       doc: Create a new flow table.
@@ -1490,7 +2063,11 @@ operations:
         request:
           value: 0xa16
           attributes:
+            # Mentioned in nf_tables_newflowtable()
+            - table
             - name
+            - hook
+            - flags
     -
       name: getflowtable
       doc: Get / dump flow tables.
@@ -1500,11 +2077,22 @@ operations:
         request:
           value: 0xa17
           attributes:
+            # Mentioned in nf_tables_getflowtable()
             - name
+            - table
         reply:
           value: 0xa16
-          attributes:
+          attributes: &flowtable-info
+            # Mentioned in nf_tables_fill_flowtable_info()
+            - table
             - name
+            - handle
+            - use
+            - flags
+            - hook
+      dump:
+        reply:
+          attributes: *flowtable-info
     -
       name: delflowtable
       doc: Delete an existing flow table.
@@ -1513,8 +2101,12 @@ operations:
       do:
         request:
           value: 0xa18
-          attributes:
+          attributes: &del-flowtable
+            # Mentioned in nf_tables_delflowtable()
+            - table
             - name
+            - handle
+            - hook
     -
       name: destroyflowtable
       doc: Delete an existing flow table with destroy semantics.
@@ -1523,8 +2115,7 @@ operations:
       do:
         request:
           value: 0xa20
-          attributes:
-            - name
+          attributes: *del-flowtable
 
 mcast-groups:
   list:
diff --git a/Documentation/netlink/specs/ovpn.yaml b/Documentation/netlink/specs/ovpn.yaml
index 1b91045cee2e4..b0c782e59a32b 100644
--- a/Documentation/netlink/specs/ovpn.yaml
+++ b/Documentation/netlink/specs/ovpn.yaml
@@ -43,7 +43,8 @@ attribute-sets:
         type: u32
         doc: >-
           The unique ID of the peer in the device context. To be used to
-          identify peers during operations for a specific device
+          identify peers during operations for a specific device.
+          Also used to match packets received from this peer.
         checks:
           max: 0xFFFFFF
       -
@@ -160,6 +161,16 @@ attribute-sets:
         name: link-tx-packets
         type: uint
         doc: Number of packets transmitted at the transport level
+      -
+        name: tx-id
+        type: u32
+        doc: >-
+          The ID value used when transmitting packets to this peer. This
+          way outgoing packets can have a different ID than incoming ones.
+          Useful in multipeer-to-multipeer connections, where each peer
+          will advertise the tx-id to be used on the link.
+        checks:
+          max: 0xFFFFFF
   -
     name: peer-new-input
     subset-of: peer
@@ -188,6 +199,8 @@ attribute-sets:
         name: keepalive-interval
       -
         name: keepalive-timeout
+      -
+        name: tx-id
   -
     name: peer-set-input
     subset-of: peer
@@ -214,6 +227,8 @@ attribute-sets:
         name: keepalive-interval
       -
         name: keepalive-timeout
+      -
+        name: tx-id
   -
     name: peer-del-input
     subset-of: peer
@@ -502,6 +517,12 @@ operations:
             - ifindex
             - keyconf
 
+    -
+      name: peer-float-ntf
+      doc: Notification about a peer floating (changing its remote UDP endpoint)
+      notify: peer-get
+      mcgrp: peers
+
 mcast-groups:
   list:
     -
diff --git a/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst b/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst
index a52850602cd80..c31c6c197cdb2 100644
--- a/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst
+++ b/Documentation/networking/device_drivers/ethernet/marvell/octeontx2.rst
@@ -323,7 +323,7 @@ Setup HTB offload
 
         # ethtool -K <interface> hw-tc-offload on
 
-2. Crate htb root::
+2. Create htb root::
 
         # tc qdisc add dev <interface> clsact
         # tc qdisc replace dev <interface> root handle 1: htb offload
diff --git a/Documentation/networking/devlink/devlink-shared.rst b/Documentation/networking/devlink/devlink-shared.rst
new file mode 100644
index 0000000000000..16bf6a7d25d94
--- /dev/null
+++ b/Documentation/networking/devlink/devlink-shared.rst
@@ -0,0 +1,97 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========================
+Devlink Shared Instances
+========================
+
+Overview
+========
+
+Shared devlink instances allow multiple physical functions (PFs) on the same
+chip to share a devlink instance for chip-wide operations.
+
+Multiple PFs may reside on the same physical chip, running a single firmware.
+Some of the resources and configurations may be shared among these PFs. The
+shared devlink instance provides an object to pin configuration knobs on.
+
+There are two possible usage models:
+
+1. The shared devlink instance is used alongside individual PF devlink
+   instances, providing chip-wide configuration in addition to per-PF
+   configuration.
+2. The shared devlink instance is the only devlink instance, without
+   per-PF instances.
+
+It is up to the driver to decide which usage model to use.
+
+The shared devlink instance is not backed by any struct *device*.
+
+Implementation
+==============
+
+Architecture
+------------
+
+The implementation uses:
+
+* **Chip identification**: PFs are grouped by chip using a driver-specific identifier
+* **Shared instance management**: Global list of shared instances with reference counting
+
+API Functions
+-------------
+
+The following functions are provided for managing shared devlink instances:
+
+* ``devlink_shd_get()``: Get or create a shared devlink instance identified by a string ID
+* ``devlink_shd_put()``: Release a reference on a shared devlink instance
+* ``devlink_shd_get_priv()``: Get private data from shared devlink instance
+
+Initialization Flow
+-------------------
+
+1. **PF calls shared devlink init** during driver probe
+2. **Chip identification** using driver-specific method to determine device identity
+3. **Get or create shared instance** using ``devlink_shd_get()``:
+
+   * The function looks up existing instance by identifier
+   * If none exists, creates new instance:
+     - Allocates and registers devlink instance
+     - Adds to global shared instances list
+     - Increments reference count
+
+4. **Set nested devlink instance** for the PF devlink instance using
+   ``devl_nested_devlink_set()`` before registering the PF devlink instance
+
+Cleanup Flow
+------------
+
+1. **Cleanup** when PF is removed
+2. **Call** ``devlink_shd_put()`` to release reference (decrements reference count)
+3. **Shared instance is automatically destroyed** when the last PF removes (reference count reaches zero)
+
+Chip Identification
+-------------------
+
+PFs belonging to the same chip are identified using a driver-specific method.
+The driver is free to choose any identifier that is suitable for determining
+whether two PFs are part of the same device. Examples include:
+
+* **PCI VPD serial numbers**: Extract from PCI VPD
+* **Device tree properties**: Read chip identifier from device tree
+* **Other hardware-specific identifiers**: Any unique identifier that groups PFs by chip
+
+Locking
+-------
+
+A global mutex (``shd_mutex``) protects the shared instances list during registration/deregistration.
+
+Similarly to other nested devlink instance relationships, devlink lock of
+the shared instance should be always taken after the devlink lock of PF.
+
+Reference Counting
+------------------
+
+Each shared devlink instance maintains a reference count (``refcount_t refcount``).
+The reference count is incremented when ``devlink_shd_get()`` is called and decremented
+when ``devlink_shd_put()`` is called. When the reference count reaches zero, the shared
+instance is automatically destroyed.
diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst
index 35b12a2bfeba5..f7ba7dcf477d0 100644
--- a/Documentation/networking/devlink/index.rst
+++ b/Documentation/networking/devlink/index.rst
@@ -68,6 +68,7 @@ general.
    devlink-resource
    devlink-selftests
    devlink-trap
+   devlink-shared
 
 Driver-specific documentation
 -----------------------------
diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst
index af56c304cef43..e92abf45faf50 100644
--- a/Documentation/networking/ethtool-netlink.rst
+++ b/Documentation/networking/ethtool-netlink.rst
@@ -96,7 +96,7 @@ For short bitmaps of (reasonably) fixed length, standard ``NLA_BITFIELD32``
 type is used. For arbitrary length bitmaps, ethtool netlink uses a nested
 attribute with contents of one of two forms: compact (two binary bitmaps
 representing bit values and mask of affected bits) and bit-by-bit (list of
-bits identified by either index or name).
+bits identified by index or name).
 
 Verbose (bit-by-bit) bitsets allow sending symbolic names for bits together
 with their values which saves a round trip (when the bitset is passed in a
@@ -156,12 +156,16 @@ Bit-by-bit form: nested (bitset) attribute contents:
  | | | ``ETHTOOL_A_BITSET_BIT_VALUE`` | flag   | present if bit is set       |
  +-+-+--------------------------------+--------+-----------------------------+
 
-Bit size is optional for bit-by-bit form. ``ETHTOOL_A_BITSET_BITS`` nest can
+For bit-by-bit form, ``ETHTOOL_A_BITSET_SIZE`` is optional, and
+``ETHTOOL_A_BITSET_BITS`` is mandatory. ``ETHTOOL_A_BITSET_BITS`` nest can
 only contain ``ETHTOOL_A_BITSET_BITS_BIT`` attributes but there can be an
 arbitrary number of them.  A bit may be identified by its index or by its
 name. When used in requests, listed bits are set to 0 or 1 according to
-``ETHTOOL_A_BITSET_BIT_VALUE``, the rest is preserved. A request fails if
-index exceeds kernel bit length or if name is not recognized.
+``ETHTOOL_A_BITSET_BIT_VALUE``, the rest is preserved.
+
+A request fails if index exceeds kernel bit length or if name is not
+recognized. If both name and index are set, the request will fail if they
+point to different bits.
 
 When ``ETHTOOL_A_BITSET_NOMASK`` flag is present, bitset is interpreted as
 a simple bitmap. ``ETHTOOL_A_BITSET_BIT_VALUE`` attributes are not used in
@@ -1072,6 +1076,8 @@ Kernel response contents:
   ``ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS``    u32     time (us), aggr, Tx
   ``ETHTOOL_A_COALESCE_RX_PROFILE``            nested  profile of DIM, Rx
   ``ETHTOOL_A_COALESCE_TX_PROFILE``            nested  profile of DIM, Tx
+  ``ETHTOOL_A_COALESCE_RX_CQE_FRAMES``         u32     max packets, Rx CQE
+  ``ETHTOOL_A_COALESCE_RX_CQE_NSECS``          u32     delay (ns), Rx CQE
   ===========================================  ======  =======================
 
 Attributes are only included in reply if their value is not zero or the
@@ -1105,6 +1111,13 @@ well with frequent small-sized URBs transmissions.
 to DIM parameters, see `Generic Network Dynamic Interrupt Moderation (Net DIM)
 <https://www.kernel.org/doc/Documentation/networking/net_dim.rst>`_.
 
+Rx CQE coalescing allows multiple received packets to be coalesced into a
+single Completion Queue Entry (CQE) or descriptor writeback.
+``ETHTOOL_A_COALESCE_RX_CQE_FRAMES`` describes the maximum number of
+frames that can be coalesced into a CQE or writeback.
+``ETHTOOL_A_COALESCE_RX_CQE_NSECS`` describes max time in nanoseconds after
+the first packet arrival in a coalesced CQE or writeback to be sent.
+
 COALESCE_SET
 ============
 
@@ -1143,6 +1156,8 @@ Request contents:
   ``ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS``    u32     time (us), aggr, Tx
   ``ETHTOOL_A_COALESCE_RX_PROFILE``            nested  profile of DIM, Rx
   ``ETHTOOL_A_COALESCE_TX_PROFILE``            nested  profile of DIM, Tx
+  ``ETHTOOL_A_COALESCE_RX_CQE_FRAMES``         u32     max packets, Rx CQE
+  ``ETHTOOL_A_COALESCE_RX_CQE_NSECS``          u32     delay (ns), Rx CQE
   ===========================================  ======  =======================
 
 Request is rejected if it attributes declared as unsupported by driver (i.e.
diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst
index 6921d8594b849..2e3a746fcc6da 100644
--- a/Documentation/networking/ip-sysctl.rst
+++ b/Documentation/networking/ip-sysctl.rst
@@ -202,6 +202,24 @@ neigh/default/gc_thresh3 - INTEGER
 
 	Default: 1024
 
+neigh/default/gc_interval - INTEGER
+	Specifies how often the garbage collector for neighbor entries
+	should run. This value applies to the entire table, not
+	individual entries. Unused since kernel v2.6.8.
+
+	Default: 30 seconds
+
+neigh/default/gc_stale_time - INTEGER
+	Determines how long a neighbor entry can remain unused before it is
+	considered stale and eligible for garbage collection. Entries that have
+	not been used for longer than this time will be removed by the garbage
+	collector, unless they have active references, are marked as PERMANENT,
+	or carry the NTF_EXT_LEARNED or NTF_EXT_VALIDATED flag. Stale entries
+	are only removed by the periodic GC when there are at least gc_thresh1
+	neighbors in the table.
+
+	Default: 60 seconds
+
 neigh/default/unres_qlen_bytes - INTEGER
 	The maximum number of bytes which may be used by packets
 	queued for each	unresolved address by other network layers.
@@ -1612,6 +1630,22 @@ ip_local_reserved_ports - list of comma separated ranges
 
 	Default: Empty
 
+ip_local_port_step_width - INTEGER
+        Defines the numerical maximum increment between successive port
+        allocations within the ephemeral port range when an unavailable port is
+        reached. This can be used to mitigate accumulated nodes in port
+        distribution when reserved ports have been configured. Please note that
+        port collisions may be more frequent in a system with a very high load.
+
+        It is recommended to set this value strictly larger than the largest
+        contiguous block of ports configure in ip_local_reserved_ports. For
+        large reserved port ranges, setting this to 3x or 4x the size of the
+        largest block is advised. Using a value equal or greater than the local
+        port range size completely solves the uneven port distribution problem,
+        but it can degrade performance under port exhaustion situations.
+
+        Default: 0 (disabled)
+
 ip_unprivileged_port_start - INTEGER
 	This is a per-namespace sysctl.  It defines the first
 	unprivileged port in the network namespace.  Privileged ports
@@ -1747,14 +1781,14 @@ icmp_msgs_per_sec - INTEGER
 	controlled by this limit. For security reasons, the precise count
 	of messages per second is randomized.
 
-	Default: 1000
+	Default: 10000
 
 icmp_msgs_burst - INTEGER
 	icmp_msgs_per_sec controls number of ICMP packets sent per second,
-	while icmp_msgs_burst controls the burst size of these packets.
+	while icmp_msgs_burst controls the token bucket size.
 	For security reasons, the precise burst size is randomized.
 
-	Default: 50
+	Default: 10000
 
 icmp_ratemask - INTEGER
 	Mask made of ICMP types for which rates are being limited.
diff --git a/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst b/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
index beaf1880a19bf..6dbd97d435e9f 100644
--- a/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
+++ b/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
@@ -52,6 +52,7 @@ u8                              sysctl_ip_fwd_update_priority
 u8                              sysctl_ip_nonlocal_bind
 u8                              sysctl_ip_autobind_reuse
 u8                              sysctl_ip_dynaddr
+u32                             sysctl_ip_local_port_step_width
 u8                              sysctl_ip_early_demux                                            read_mostly         ip(6)_rcv_finish_core
 u8                              sysctl_raw_l3mdev_accept
 u8                              sysctl_tcp_early_demux                                           read_mostly         ip(6)_rcv_finish_core
@@ -104,6 +105,7 @@ u8                              sysctl_tcp_nometrics_save
 u8                              sysctl_tcp_no_ssthresh_metrics_save                                                  TCP_LAST_ACK/tcp_(update/init)_metrics
 u8                              sysctl_tcp_moderate_rcvbuf                                       read_mostly         tcp_rcvbuf_grow()
 u32                             sysctl_tcp_rcvbuf_low_rtt                                        read_mostly         tcp_rcvbuf_grow()
+u8                              sysctl_tcp_shrink_window                     read_mostly         read_mostly         __tcp_select_window()
 u8                              sysctl_tcp_tso_win_divisor                   read_mostly                             tcp_tso_should_defer(tcp_write_xmit)
 u8                              sysctl_tcp_workaround_signed_windows                                                 tcp_select_window
 int                             sysctl_tcp_limit_output_bytes                read_mostly                             tcp_small_queue_check(tcp_write_xmit)
diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst
index 563daea10d6c5..fecf61166a54e 100644
--- a/Documentation/networking/net_cachelines/tcp_sock.rst
+++ b/Documentation/networking/net_cachelines/tcp_sock.rst
@@ -121,6 +121,7 @@ u64                           delivered_mstamp        read_write
 u32                           rate_delivered                              read_mostly         tcp_rate_gen
 u32                           rate_interval_us                            read_mostly         rate_delivered,rate_app_limited
 u32                           rcv_wnd                 read_write          read_mostly         tcp_select_window,tcp_receive_window,tcp_fast_path_check
+u32                           rcv_mwnd_seq            read_write                              tcp_select_window
 u32                           write_seq               read_write                              tcp_rate_check_app_limited,tcp_write_queue_empty,tcp_skb_entail,forced_push,tcp_mark_push
 u32                           notsent_lowat           read_mostly                             tcp_stream_memory_free
 u32                           pushed_seq              read_write                              tcp_mark_push,forced_push
diff --git a/Documentation/networking/scaling.rst b/Documentation/networking/scaling.rst
index 0023afa530ec1..6c261eb48845a 100644
--- a/Documentation/networking/scaling.rst
+++ b/Documentation/networking/scaling.rst
@@ -403,16 +403,21 @@ Both of these need to be set before RFS is enabled for a receive queue.
 Values for both are rounded up to the nearest power of two. The
 suggested flow count depends on the expected number of active connections
 at any given time, which may be significantly less than the number of open
-connections. We have found that a value of 32768 for rps_sock_flow_entries
-works fairly well on a moderately loaded server.
+connections. We have found that a value of 65536 for rps_sock_flow_entries
+works fairly well on a moderately loaded server. Big servers might
+need 1048576 or even higher values.
+
+On a NUMA host it is advisable to spread rps_sock_flow_entries on all nodes.
+
+numactl --interleave=all bash -c "echo 1048576 >/proc/sys/net/core/rps_sock_flow_entries"
 
 For a single queue device, the rps_flow_cnt value for the single queue
 would normally be configured to the same value as rps_sock_flow_entries.
 For a multi-queue device, the rps_flow_cnt for each queue might be
 configured as rps_sock_flow_entries / N, where N is the number of
-queues. So for instance, if rps_sock_flow_entries is set to 32768 and there
+queues. So for instance, if rps_sock_flow_entries is set to 131072 and there
 are 16 configured receive queues, rps_flow_cnt for each queue might be
-configured as 2048.
+configured as 8192.
 
 
 Accelerated RFS
diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
index 904a910f198e4..a8b4f357174ef 100644
--- a/Documentation/networking/smc-sysctl.rst
+++ b/Documentation/networking/smc-sysctl.rst
@@ -23,17 +23,17 @@ autocorking_size - INTEGER
 	Default: 64K
 
 smcr_buf_type - INTEGER
-        Controls which type of sndbufs and RMBs to use in later newly created
-        SMC-R link group. Only for SMC-R.
+	Controls which type of sndbufs and RMBs to use in later newly created
+	SMC-R link group. Only for SMC-R.
 
-        Default: 0 (physically contiguous sndbufs and RMBs)
+	Default: 0 (physically contiguous sndbufs and RMBs)
 
-        Possible values:
+	Possible values:
 
-        - 0 - Use physically contiguous buffers
-        - 1 - Use virtually contiguous buffers
-        - 2 - Mixed use of the two types. Try physically contiguous buffers first.
-          If not available, use virtually contiguous buffers then.
+	- 0 - Use physically contiguous buffers
+	- 1 - Use virtually contiguous buffers
+	- 2 - Mixed use of the two types. Try physically contiguous buffers first.
+	  If not available, use virtually contiguous buffers then.
 
 smcr_testlink_time - INTEGER
 	How frequently SMC-R link sends out TEST_LINK LLC messages to confirm
@@ -111,3 +111,30 @@ smcr_max_recv_wr - INTEGER
 	like before having this control.
 
 	Default: 48
+
+limit_smc_hs - INTEGER
+	Whether to limit SMC handshake for newly created sockets.
+
+	When enabled, SMC listen path applies handshake limitation based on
+	handshake worker congestion and queued SMC handshake load.
+
+	Possible values:
+
+	- 0 - Disable handshake limitation
+	- 1 - Enable handshake limitation
+
+	Default: 0 (disable)
+
+hs_ctrl - STRING
+	Select the SMC handshake control profile by name.
+
+	This string refers to the name of a user-implemented
+	BPF struct_ops instance of type smc_hs_ctrl.
+
+	The selected profile controls whether SMC options are advertised
+	during TCP SYN/SYN-ACK handshake.
+
+	Only available when CONFIG_SMC_HS_CTRL_BPF is enabled.
+	Write an empty string to clear the current profile.
+
+	Default: empty string
diff --git a/Documentation/networking/tls-handshake.rst b/Documentation/networking/tls-handshake.rst
index 6f5ea1646a470..4f7bc1087df99 100644
--- a/Documentation/networking/tls-handshake.rst
+++ b/Documentation/networking/tls-handshake.rst
@@ -7,7 +7,7 @@ In-Kernel TLS Handshake
 Overview
 ========
 
-Transport Layer Security (TLS) is a Upper Layer Protocol (ULP) that runs
+Transport Layer Security (TLS) is an Upper Layer Protocol (ULP) that runs
 over TCP. TLS provides end-to-end data integrity and confidentiality in
 addition to peer authentication.
 
diff --git a/Documentation/process/2.Process.rst b/Documentation/process/2.Process.rst
index 57fa8cac58a6e..77f3f80e7cd7e 100644
--- a/Documentation/process/2.Process.rst
+++ b/Documentation/process/2.Process.rst
@@ -291,7 +291,7 @@ Use of the MMOTM tree is likely to be a frustrating experience, though;
 there is a definite chance that it will not even compile.
 
 The primary tree for next-cycle patch merging is linux-next, maintained by
-Stephen Rothwell.  The linux-next tree is, by design, a snapshot of what
+Mark Brown.  The linux-next tree is, by design, a snapshot of what
 the mainline is expected to look like after the next merge window closes.
 Linux-next trees are announced on the linux-kernel and linux-next mailing
 lists when they are assembled; they can be downloaded from:
diff --git a/Documentation/process/backporting.rst b/Documentation/process/backporting.rst
index c42779fbcd331..0de9eacd46a73 100644
--- a/Documentation/process/backporting.rst
+++ b/Documentation/process/backporting.rst
@@ -432,7 +432,7 @@ The same goes for added ``return``, ``break``, and ``continue``
 statements.
 
 Error handling is typically located at the bottom of the function, so it
-may not be part of the conflict even though could have been changed by
+may not be part of the conflict even though it could have been changed by
 other patches.
 
 A good way to ensure that you review the error paths is to always use
diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst
index 6b373e1935485..f9f6f5c478825 100644
--- a/Documentation/process/changes.rst
+++ b/Documentation/process/changes.rst
@@ -391,7 +391,7 @@ Kernel documentation
 Sphinx
 ------
 
-Please see :ref:`sphinx_install` in :ref:`Documentation/doc-guide/sphinx.rst <sphinxdoc>`
+Please see :ref:`sphinx_install` in Documentation/doc-guide/sphinx.rst
 for details about Sphinx requirements.
 
 rustdoc
diff --git a/Documentation/process/handling-regressions.rst b/Documentation/process/handling-regressions.rst
index 1f5ab49c48a48..c71b5d403f0c9 100644
--- a/Documentation/process/handling-regressions.rst
+++ b/Documentation/process/handling-regressions.rst
@@ -461,325 +461,556 @@ which both cover more details than the above section.
 Quotes from Linus about regression
 ----------------------------------
 
-Find below a few real life examples of how Linus Torvalds expects regressions to
-be handled:
+The following statements from Linus Torvalds provide some insight into Linux
+"no regressions" rule and how he expects regressions to be handled:
 
- * From `2017-10-26 (1/2)
-   <https://lore.kernel.org/lkml/CA+55aFwiiQYJ+YoLKCXjN_beDVfu38mg=Ggg5LFOcqHE8Qi7Zw@mail.gmail.com/>`_::
+On how quickly regressions should be fixed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       If you break existing user space setups THAT IS A REGRESSION.
+* From `2026-01-22 <https://lore.kernel.org/all/CAHk-=wheQNiW_WtHGO7bKkT7Uib-p+ai2JP9M+z+FYcZ6CAxYA@mail.gmail.com/>`_::
 
-       It's not ok to say "but we'll fix the user space setup".
+    But a user complaining should basically result in an immediate fix -
+    possibly a "revert and rethink".
 
-       Really. NOT OK.
+  With a later clarification on `2026-01-28 <https://lore.kernel.org/all/CAHk-%3Dwi86AosXs66-yi54%2BmpQjPu0upxB8ZAfG%2BLsMyJmcuMSA@mail.gmail.com/>`_::
 
-       [...]
+    It's also worth noting that "immediate" obviously doesn't mean "right
+    this *second* when the problem has been reported".
 
-       The first rule is:
+    But if it's a regression with a known commit that caused it, I think
+    the rule of thumb should generally be "within a week", preferably
+    before the next rc.
 
-        - we don't cause regressions
+* From `2023-04-21 <https://lore.kernel.org/all/CAHk-=wgD98pmSK3ZyHk_d9kZ2bhgN6DuNZMAJaV0WTtbkf=RDw@mail.gmail.com/>`_::
 
-       and the corollary is that when regressions *do* occur, we admit to
-       them and fix them, instead of blaming user space.
+    Known-broken commits either
+     (a) get a timely fix that doesn't have other questions
+    or
+     (b) get reverted
 
-       The fact that you have apparently been denying the regression now for
-       three weeks means that I will revert, and I will stop pulling apparmor
-       requests until the people involved understand how kernel development
-       is done.
+* From `2021-09-20(2) <https://lore.kernel.org/all/CAHk-=wgOvmtRw1TNbMC1rn5YqyTKyn0hz+sc4k0DGNn++u9aYw@mail.gmail.com/>`_::
 
- * From `2017-10-26 (2/2)
-   <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
+    [...] review shouldn't hold up reported regressions of existing code. That's
+    just basic _testing_ - either the fix should be applied, or - if the fix is
+    too invasive or too ugly - the problematic source of the regression should
+    be reverted.
 
-       People should basically always feel like they can update their kernel
-       and simply not have to worry about it.
+    Review should be about new code, it shouldn't be holding up "there's a
+    bug report, here's the obvious fix".
 
-       I refuse to introduce "you can only update the kernel if you also
-       update that other program" kind of limitations. If the kernel used to
-       work for you, the rule is that it continues to work for you.
+* From `2023-05-08 <https://lore.kernel.org/all/CAHk-=wgzU8_dGn0Yg+DyX7ammTkDUCyEJ4C=NvnHRhxKWC7Wpw@mail.gmail.com/>`_::
 
-       There have been exceptions, but they are few and far between, and they
-       generally have some major and fundamental reasons for having happened,
-       that were basically entirely unavoidable, and people _tried_hard_ to
-       avoid them. Maybe we can't practically support the hardware any more
-       after it is decades old and nobody uses it with modern kernels any
-       more. Maybe there's a serious security issue with how we did things,
-       and people actually depended on that fundamentally broken model. Maybe
-       there was some fundamental other breakage that just _had_ to have a
-       flag day for very core and fundamental reasons.
+    If something doesn't even build, it should damn well be fixed ASAP.
 
-       And notice that this is very much about *breaking* peoples environments.
+On how fixing regressions with reverts can help prevent maintainer burnout
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       Behavioral changes happen, and maybe we don't even support some
-       feature any more. There's a number of fields in /proc/<pid>/stat that
-       are printed out as zeroes, simply because they don't even *exist* in
-       the kernel any more, or because showing them was a mistake (typically
-       an information leak). But the numbers got replaced by zeroes, so that
-       the code that used to parse the fields still works. The user might not
-       see everything they used to see, and so behavior is clearly different,
-       but things still _work_, even if they might no longer show sensitive
-       (or no longer relevant) information.
+* From `2026-01-28 <https://lore.kernel.org/all/CAHk-%3Dwi86AosXs66-yi54%2BmpQjPu0upxB8ZAfG%2BLsMyJmcuMSA@mail.gmail.com/>`_::
 
-       But if something actually breaks, then the change must get fixed or
-       reverted. And it gets fixed in the *kernel*. Not by saying "well, fix
-       your user space then". It was a kernel change that exposed the
-       problem, it needs to be the kernel that corrects for it, because we
-       have a "upgrade in place" model. We don't have a "upgrade with new
-       user space".
+    > So how can I/we make "immediate fixes" happen more often without
+    > contributing to maintainer burnout?
 
-       And I seriously will refuse to take code from people who do not
-       understand and honor this very simple rule.
+    [...] the "revert and rethink" model [...] often a good idea in general [...]
 
-       This rule is also not going to change.
+    Exactly so that maintainers don't get stressed out over having a pending
+    problem report that people keep pestering them about.
 
-       And yes, I realize that the kernel is "special" in this respect. I'm
-       proud of it.
+    I think people are sometimes a bit too bought into whatever changes
+    they made, and reverting is seen as "too drastic", but I think it's
+    often the quick and easy solution for when there isn't some obvious
+    response to a regression report.
 
-       I have seen, and can point to, lots of projects that go "We need to
-       break that use case in order to make progress" or "you relied on
-       undocumented behavior, it sucks to be you" or "there's a better way to
-       do what you want to do, and you have to change to that new better
-       way", and I simply don't think that's acceptable outside of very early
-       alpha releases that have experimental users that know what they signed
-       up for. The kernel hasn't been in that situation for the last two
-       decades.
+On mainlining fixes when the last -rc or a new release is close
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       We do API breakage _inside_ the kernel all the time. We will fix
-       internal problems by saying "you now need to do XYZ", but then it's
-       about internal kernel API's, and the people who do that then also
-       obviously have to fix up all the in-kernel users of that API. Nobody
-       can say "I now broke the API you used, and now _you_ need to fix it
-       up". Whoever broke something gets to fix it too.
+* From `2026-02-01 <https://lore.kernel.org/all/CAHk-%3DwhXTw1oPsa%2BTLuY1Rc9D1OAiPVOdR_-R2xG45kwDObKdA@mail.gmail.com/>`_::
 
-       And we simply do not break user space.
+    So I think I'd rather see them hit rc8 (later today) and have a week
+    of testing in my tree and be reverted if they cause problems, than
+    have them go in after rc8 and then cause problems in the 6.19 release
+    instead.
 
- * From `2020-05-21
-   <https://lore.kernel.org/all/CAHk-=wiVi7mSrsMP=fLXQrXK_UimybW=ziLOwSzFTtoXUacWVQ@mail.gmail.com/>`_::
+* From `2023-04-20 <https://lore.kernel.org/all/CAHk-=wis_qQy4oDNynNKi5b7Qhosmxtoj1jxo5wmB6SRUwQUBQ@mail.gmail.com/>`_::
 
-       The rules about regressions have never been about any kind of
-       documented behavior, or where the code lives.
+    But something like this, where the regression was in the previous release
+    and it's just a clear fix with no semantic subtlety, I consider to be just a
+    regular regression that should be expedited - partly to make it into stable,
+    and partly to avoid having to put the fix into _another_ stable kernel.
 
-       The rules about regressions are always about "breaks user workflow".
+On sending merge requests with just one fix
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       Users are literally the _only_ thing that matters.
+* From `2024-04-24 <https://lore.kernel.org/all/CAHk-=wjy_ph9URuFt-pq+2AJ__p7gFDx=yzVSCsx16xAYvNw9g@mail.gmail.com/>`_::
 
-       No amount of "you shouldn't have used this" or "that behavior was
-       undefined, it's your own fault your app broke" or "that used to work
-       simply because of a kernel bug" is at all relevant.
+    If the issue is just that there's nothing else happening, I think people
+    should just point me to the patch and say "can you apply this single fix?"
 
-       Now, reality is never entirely black-and-white. So we've had things
-       like "serious security issue" etc that just forces us to make changes
-       that may break user space. But even then the rule is that we don't
-       really have other options that would allow things to continue.
+* From `2023-04-20 <https://lore.kernel.org/all/CAHk-=wis_qQy4oDNynNKi5b7Qhosmxtoj1jxo5wmB6SRUwQUBQ@mail.gmail.com/>`_::
 
-       And obviously, if users take years to even notice that something
-       broke, or if we have sane ways to work around the breakage that
-       doesn't make for too much trouble for users (ie "ok, there are a
-       handful of users, and they can use a kernel command line to work
-       around it" kind of things) we've also been a bit less strict.
+    I'm always open to direct fixes when there is no controversy about the fix.
+    No problem. I still happily deal with individual patches.
 
-       But no, "that was documented to be broken" (whether it's because the
-       code was in staging or because the man-page said something else) is
-       irrelevant. If staging code is so useful that people end up using it,
-       that means that it's basically regular kernel code with a flag saying
-       "please clean this up".
+On the importance of pointing to bug reports using Link:/Closes: tags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       The other side of the coin is that people who talk about "API
-       stability" are entirely wrong. API's don't matter either. You can make
-       any changes to an API you like - as long as nobody notices.
+* From `2025-07-29(1) <https://lore.kernel.org/all/CAHk-=wj2kJRPWx8B09AAtzj+_g+T6UBX11TP0ebs1WJdTtv=WQ@mail.gmail.com/>`_::
 
-       Again, the regression rule is not about documentation, not about
-       API's, and not about the phase of the moon.
+    [...] revert like this, it really would be good to link to the problems, so
+    that when people try to re-enable it, they have the history for why it
+    didn't work the first time.
 
-       It's entirely about "we caused problems for user space that used to work".
+* From `2022-05-08 <https://lore.kernel.org/all/CAHk-=wjMmSZzMJ3Xnskdg4+GGz=5p5p+GSYyFBTh0f-DgvdBWg@mail.gmail.com/>`_::
 
- * From `2017-11-05
-   <https://lore.kernel.org/all/CA+55aFzUvbGjD8nQ-+3oiMBx14c_6zOj2n7KLN3UsJ-qsd4Dcw@mail.gmail.com/>`_::
+    So I have to once more complain [...]
 
-       And our regression rule has never been "behavior doesn't change".
-       That would mean that we could never make any changes at all.
+    [...] There's no link to the actual problem the patch fixes.
 
-       For example, we do things like add new error handling etc all the
-       time, which we then sometimes even add tests for in our kselftest
-       directory.
+* From `2022-06-22 <https://lore.kernel.org/all/CAHk-=wjxzafG-=J8oT30s7upn4RhBs6TX-uVFZ5rME+L5_DoJA@mail.gmail.com/>`_::
 
-       So clearly behavior changes all the time and we don't consider that a
-       regression per se.
+    See, *that* link [to the report] would have been useful in the commit.
 
-       The rule for a regression for the kernel is that some real user
-       workflow breaks. Not some test. Not a "look, I used to be able to do
-       X, now I can't".
+On why the "no regressions" rule exists
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
- * From `2018-08-03
-   <https://lore.kernel.org/all/CA+55aFwWZX=CXmWDTkDGb36kf12XmTehmQjbiMPCqCRG2hi9kw@mail.gmail.com/>`_::
+* From `2026-01-22 <https://lore.kernel.org/all/CAHk-=wheQNiW_WtHGO7bKkT7Uib-p+ai2JP9M+z+FYcZ6CAxYA@mail.gmail.com/>`_::
 
-       YOU ARE MISSING THE #1 KERNEL RULE.
+    But the basic rule is: be so good about backwards compatibility that
+    users never have to worry about upgrading. They should absolutely feel
+    confident that any kernel-reported problem will either be solved, or
+    have an easy solution that is appropriate for *them* (ie a
+    non-technical user shouldn't be expected to be able to do a lot).
 
-       We do not regress, and we do not regress exactly because your are 100% wrong.
+    Because the last thing we want is people holding back from trying new
+    kernels.
 
-       And the reason you state for your opinion is in fact exactly *WHY* you
-       are wrong.
+* From `2024-05-28 <https://lore.kernel.org/all/CAHk-=wgtb7y-bEh7tPDvDWru7ZKQ8-KMjZ53Tsk37zsPPdwXbA@mail.gmail.com/>`_::
 
-       Your "good reasons" are pure and utter garbage.
+    I introduced that "no regressions" rule something like two decades
+    ago, because people need to be able to update their kernel without
+    fear of something they relied on suddenly stopping to work.
 
-       The whole point of "we do not regress" is so that people can upgrade
-       the kernel and never have to worry about it.
+* From `2018-08-03 <https://lore.kernel.org/all/CA+55aFwWZX=CXmWDTkDGb36kf12XmTehmQjbiMPCqCRG2hi9kw@mail.gmail.com/>`_::
 
-       > Kernel had a bug which has been fixed
+    The whole point of "we do not regress" is so that people can upgrade
+    the kernel and never have to worry about it.
 
-       That is *ENTIRELY* immaterial.
+    [...]
 
-       Guys, whether something was buggy or not DOES NOT MATTER.
+    Because the only thing that matters IS THE USER.
 
-       Why?
+* From `2017-10-26(1) <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
 
-       Bugs happen. That's a fact of life. Arguing that "we had to break
-       something because we were fixing a bug" is completely insane. We fix
-       tens of bugs every single day, thinking that "fixing a bug" means that
-       we can break something is simply NOT TRUE.
+    If the kernel used to work for you, the rule is that it continues to work
+    for you.
 
-       So bugs simply aren't even relevant to the discussion. They happen,
-       they get found, they get fixed, and it has nothing to do with "we
-       break users".
+    [...]
 
-       Because the only thing that matters IS THE USER.
+    People should basically always feel like they can update their kernel
+    and simply not have to worry about it.
 
-       How hard is that to understand?
+    I refuse to introduce "you can only update the kernel if you also
+    update that other program" kind of limitations. If the kernel used to
+    work for you, the rule is that it continues to work for you.
 
-       Anybody who uses "but it was buggy" as an argument is entirely missing
-       the point. As far as the USER was concerned, it wasn't buggy - it
-       worked for him/her.
+On exceptions to the "no regressions" rule
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       Maybe it worked *because* the user had taken the bug into account,
-       maybe it worked because the user didn't notice - again, it doesn't
-       matter. It worked for the user.
+* From `2026-01-22 <https://lore.kernel.org/all/CAHk-=wheQNiW_WtHGO7bKkT7Uib-p+ai2JP9M+z+FYcZ6CAxYA@mail.gmail.com/>`_::
 
-       Breaking a user workflow for a "bug" is absolutely the WORST reason
-       for breakage you can imagine.
+    There are _very_ few exceptions to that rule, the main one being "the
+    problem was a fundamental huge and gaping security issue and we *had* to
+    make that change, and we couldn't even make your limited use-case just
+    continue to work".
 
-       It's basically saying "I took something that worked, and I broke it,
-       but now it's better". Do you not see how f*cking insane that statement
-       is?
+    The other exception is "the problem was reported years after it was
+    introduced, and now most people rely on the new behavior".
 
-       And without users, your program is not a program, it's a pointless
-       piece of code that you might as well throw away.
+    [...]
 
-       Seriously. This is *why* the #1 rule for kernel development is "we
-       don't break users". Because "I fixed a bug" is absolutely NOT AN
-       ARGUMENT if that bug fix broke a user setup. You actually introduced a
-       MUCH BIGGER bug by "fixing" something that the user clearly didn't
-       even care about.
+    Now, if it's one or two users and you can just get them to recompile,
+    that's one thing. Niche hardware and odd use-cases can sometimes be
+    solved that way, and regressions can sometimes be fixed by handholding
+    every single reporter if the reporter is willing and able to change
+    his or her workflow.
 
-       And dammit, we upgrade the kernel ALL THE TIME without upgrading any
-       other programs at all. It is absolutely required, because flag-days
-       and dependencies are horribly bad.
+* From `2023-04-20 <https://lore.kernel.org/all/CAHk-=wis_qQy4oDNynNKi5b7Qhosmxtoj1jxo5wmB6SRUwQUBQ@mail.gmail.com/>`_::
 
-       And it is also required simply because I as a kernel developer do not
-       upgrade random other tools that I don't even care about as I develop
-       the kernel, and I want any of my users to feel safe doing the same
-       time.
+    And yes, I do consider "regression in an earlier release" to be a
+    regression that needs fixing.
 
-       So no. Your rule is COMPLETELY wrong. If you cannot upgrade a kernel
-       without upgrading some other random binary, then we have a problem.
+    There's obviously a time limit: if that "regression in an earlier
+    release" was a year or more ago, and just took forever for people to
+    notice, and it had semantic changes that now mean that fixing the
+    regression could cause a _new_ regression, then that can cause me to
+    go "Oh, now the new semantics are what we have to live with".
 
- * From `2021-06-05
-   <https://lore.kernel.org/all/CAHk-=wiUVqHN76YUwhkjZzwTdjMMJf_zN4+u7vEJjmEGh3recw@mail.gmail.com/>`_::
+* From `2017-10-26(2) <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
 
-       THERE ARE NO VALID ARGUMENTS FOR REGRESSIONS.
+    There have been exceptions, but they are few and far between, and they
+    generally have some major and fundamental reasons for having happened,
+    that were basically entirely unavoidable, and people _tried_hard_ to
+    avoid them. Maybe we can't practically support the hardware any more
+    after it is decades old and nobody uses it with modern kernels any
+    more. Maybe there's a serious security issue with how we did things,
+    and people actually depended on that fundamentally broken model. Maybe
+    there was some fundamental other breakage that just _had_ to have a
+    flag day for very core and fundamental reasons.
 
-       Honestly, security people need to understand that "not working" is not
-       a success case of security. It's a failure case.
+On situations where updating something in userspace can resolve regressions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       Yes, "not working" may be secure. But security in that case is *pointless*.
+* From `2018-08-03 <https://lore.kernel.org/all/CA+55aFwWZX=CXmWDTkDGb36kf12XmTehmQjbiMPCqCRG2hi9kw@mail.gmail.com/>`_::
 
- * From `2011-05-06 (1/3)
-   <https://lore.kernel.org/all/BANLkTim9YvResB+PwRp7QTK-a5VNg2PvmQ@mail.gmail.com/>`_::
+    And dammit, we upgrade the kernel ALL THE TIME without upgrading any
+    other programs at all. It is absolutely required, because flag-days
+    and dependencies are horribly bad.
 
-       Binary compatibility is more important.
+    And it is also required simply because I as a kernel developer do not
+    upgrade random other tools that I don't even care about as I develop the
+    kernel, and I want any of my users to feel safe doing the same time.
 
-       And if binaries don't use the interface to parse the format (or just
-       parse it wrongly - see the fairly recent example of adding uuid's to
-       /proc/self/mountinfo), then it's a regression.
+* From `2017-10-26(3) <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
 
-       And regressions get reverted, unless there are security issues or
-       similar that makes us go "Oh Gods, we really have to break things".
+    But if something actually breaks, then the change must get fixed or
+    reverted. And it gets fixed in the *kernel*. Not by saying "well, fix your
+    user space then". It was a kernel change that exposed the problem, it needs
+    to be the kernel that corrects for it, because we have a "upgrade in place"
+    model. We don't have a "upgrade with new user space".
 
-       I don't understand why this simple logic is so hard for some kernel
-       developers to understand. Reality matters. Your personal wishes matter
-       NOT AT ALL.
+    And I seriously will refuse to take code from people who do not understand
+    and honor this very simple rule.
 
-       If you made an interface that can be used without parsing the
-       interface description, then we're stuck with the interface. Theory
-       simply doesn't matter.
+    This rule is also not going to change.
 
-       You could help fix the tools, and try to avoid the compatibility
-       issues that way. There aren't that many of them.
+    And yes, I realize that the kernel is "special" in this respect. I'm proud
+    of it.
 
-   From `2011-05-06 (2/3)
-   <https://lore.kernel.org/all/BANLkTi=KVXjKR82sqsz4gwjr+E0vtqCmvA@mail.gmail.com/>`_::
+* From `2017-10-26(4) <https://lore.kernel.org/all/CA+55aFwiiQYJ+YoLKCXjN_beDVfu38mg=Ggg5LFOcqHE8Qi7Zw@mail.gmail.com/>`_::
 
-       it's clearly NOT an internal tracepoint. By definition. It's being
-       used by powertop.
+    If you break existing user space setups THAT IS A REGRESSION.
 
-   From `2011-05-06 (3/3)
-   <https://lore.kernel.org/all/BANLkTinazaXRdGovYL7rRVp+j6HbJ7pzhg@mail.gmail.com/>`_::
+    It's not ok to say "but we'll fix the user space setup".
 
-       We have programs that use that ABI and thus it's a regression if they break.
+    Really. NOT OK.
 
- * From `2012-07-06 <https://lore.kernel.org/all/CA+55aFwnLJ+0sjx92EGREGTWOx84wwKaraSzpTNJwPVV8edw8g@mail.gmail.com/>`_::
+On what qualifies as userspace interface, ABI, API, documented interfaces, etc.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-       > Now this got me wondering if Debian _unstable_ actually qualifies as a
-       > standard distro userspace.
+* From `2026-01-20 <https://lore.kernel.org/all/CAHk-=wga8Qu0-OSE9VZbviq9GuqwhPhLUXeAt-S7_9+fMCLkKg@mail.gmail.com/>`_::
 
-       Oh, if the kernel breaks some standard user space, that counts. Tons
-       of people run Debian unstable
+    So I absolutely detest the whole notion of "ABI changes". It's a
+    meaningless concept, and I hate it with a passion, [...]
 
- * From `2019-09-15
-   <https://lore.kernel.org/lkml/CAHk-=wiP4K8DRJWsCo=20hn_6054xBamGKF2kPgUzpB5aMaofA@mail.gmail.com/>`_::
+    The Linux rule for regressions is basically based on the philosophical
+    question of "If a tree falls in the forest, and nobody is around to
+    hear it, does it make a sound?".
 
-       One _particularly_ last-minute revert is the top-most commit (ignoring
-       the version change itself) done just before the release, and while
-       it's very annoying, it's perhaps also instructive.
+    So the only thing that matters is if something breaks user-*conscious*
+    behavior.
 
-       What's instructive about it is that I reverted a commit that wasn't
-       actually buggy. In fact, it was doing exactly what it set out to do,
-       and did it very well. In fact it did it _so_ well that the much
-       improved IO patterns it caused then ended up revealing a user-visible
-       regression due to a real bug in a completely unrelated area.
+    And when that happens, the distinction between "bug fix" and "new
+    feature" and "ABI change" matters not one whit, and the change needs
+    to be done differently.
 
-       The actual details of that regression are not the reason I point that
-       revert out as instructive, though. It's more that it's an instructive
-       example of what counts as a regression, and what the whole "no
-       regressions" kernel rule means. The reverted commit didn't change any
-       API's, and it didn't introduce any new bugs. But it ended up exposing
-       another problem, and as such caused a kernel upgrade to fail for a
-       user. So it got reverted.
+    [...]
 
-       The point here being that we revert based on user-reported _behavior_,
-       not based on some "it changes the ABI" or "it caused a bug" concept.
-       The problem was really pre-existing, and it just didn't happen to
-       trigger before. The better IO patterns introduced by the change just
-       happened to expose an old bug, and people had grown to depend on the
-       previously benign behavior of that old issue.
+    I just wanted to point out that the argument about whether it's an ABI
+    change or not is irrelevant. If it turns out that some program - not a test
+    script, but something with relevance to conscious user expectations ~
+    depended on the old broken behavior, then it needs to be done some other
+    way.
 
-       And never fear, we'll re-introduce the fix that improved on the IO
-       patterns once we've decided just how to handle the fact that we had a
-       bad interaction with an interface that people had then just happened
-       to rely on incidental behavior for before. It's just that we'll have
-       to hash through how to do that (there are no less than three different
-       patches by three different developers being discussed, and there might
-       be more coming...). In the meantime, I reverted the thing that exposed
-       the problem to users for this release, even if I hope it will be
-       re-introduced (perhaps even backported as a stable patch) once we have
-       consensus about the issue it exposed.
+* From `2026-02-13 <https://lore.kernel.org/all/CAHk-=whY-N8kjm8kiFUV5Ei-8AuYw--EPGD-AR3Pd+5GTx2sAQ@mail.gmail.com/>`_::
 
-       Take-away from the whole thing: it's not about whether you change the
-       kernel-userspace ABI, or fix a bug, or about whether the old code
-       "should never have worked in the first place". It's about whether
-       something breaks existing users' workflow.
+    > [...] this should not fall under the don't break user space rule [...]
 
-       Anyway, that was my little aside on the whole regression thing.  Since
-       it's that "first rule of kernel programming", I felt it is perhaps
-       worth just bringing it up every once in a while
+    Note that the rule is about breaking *users*, not breaking user space per
+    se. [...]
+
+    If some user setup breaks, things need fixing.
+
+    [...] but I want to make it very clear that there are no excuses about "user
+    space applications".
+
+* From `2021-09-20(4) <https://lore.kernel.org/all/CAHk-=wi7DB2SJ-wngVvsJ7Ak2cM556Q8437sOXo4EJt2BWPdEg@mail.gmail.com/>`_::
+
+    [...] a regression is a bit like Schrödinger's cat - if nobody is around
+    to notice it and it doesn't actually affect any real workload, then you
+    can treat the regression as if it doesn't exist.
+
+* From `2020-05-21 <https://lore.kernel.org/all/CAHk-=wiVi7mSrsMP=fLXQrXK_UimybW=ziLOwSzFTtoXUacWVQ@mail.gmail.com/>`_::
+
+    The rules about regressions have never been about any kind of documented
+    behavior, or where the code lives.
+
+    The rules about regressions are always about "breaks user workflow".
+
+    Users are literally the _only_ thing that matters.
+
+* From `2019-09-15 <https://lore.kernel.org/lkml/CAHk-=wiP4K8DRJWsCo=20hn_6054xBamGKF2kPgUzpB5aMaofA@mail.gmail.com/>`_::
+
+    One _particularly_ last-minute revert is the top-most commit (ignoring
+    the version change itself) done just before the release, and while
+    it's very annoying, it's perhaps also instructive.
+
+    What's instructive about it is that I reverted a commit that wasn't
+    actually buggy. In fact, it was doing exactly what it set out to do,
+    and did it very well. In fact it did it _so_ well that the much
+    improved IO patterns it caused then ended up revealing a user-visible
+    regression due to a real bug in a completely unrelated area.
+
+    The actual details of that regression are not the reason I point that
+    revert out as instructive, though. It's more that it's an instructive
+    example of what counts as a regression, and what the whole "no
+    regressions" kernel rule means.
+
+    [...] The reverted commit didn't change any API's, and it didn't introduce
+    any new bugs. But it ended up exposing another problem, and as such caused
+    a kernel upgrade to fail for a user. So it got reverted.
+
+    The point here being that we revert based on user-reported _behavior_, not
+    based on some "it changes the ABI" or "it caused a bug" concept. The problem
+    was really pre-existing, and it just didn't happen to trigger before. [...]
+
+    Take-away from the whole thing: it's not about whether you change the
+    kernel-userspace ABI, or fix a bug, or about whether the old code
+    "should never have worked in the first place". It's about whether
+    something breaks existing users' workflow.
+
+* From `2017-11-05 <https://lore.kernel.org/all/CA+55aFzUvbGjD8nQ-+3oiMBx14c_6zOj2n7KLN3UsJ-qsd4Dcw@mail.gmail.com/>`_::
+
+    And our regression rule has never been "behavior doesn't change".
+    That would mean that we could never make any changes at all.
+
+* From `2020-05-21 <https://lore.kernel.org/all/CAHk-=wiVi7mSrsMP=fLXQrXK_UimybW=ziLOwSzFTtoXUacWVQ@mail.gmail.com/>`_::
+
+    No amount of "you shouldn't have used this" or "that behavior was
+    undefined, it's your own fault your app broke" or "that used to work
+    simply because of a kernel bug" is at all relevant.
+
+* From `2021-05-21 <https://lore.kernel.org/all/CAHk-=wiVi7mSrsMP=fLXQrXK_UimybW=ziLOwSzFTtoXUacWVQ@mail.gmail.com/>`_::
+
+    But no, "that was documented to be broken" (whether it's because the code
+    was in staging or because the man-page said something else) is irrelevant.
+    If staging code is so useful that people end up using it, that means that
+    it's basically regular kernel code with a flag saying "please clean this
+    up".
+
+    [...]
+
+    The other side of the coin is that people who talk about "API stability" are
+    entirely wrong. API's don't matter either. You can make any changes to an
+    API you like - as long as nobody notices.
+
+    Again, the regression rule is not about documentation, not about API's, and
+    not about the phase of the moon.
+
+* From `2012-07-06 <https://lore.kernel.org/all/CA+55aFwnLJ+0sjx92EGREGTWOx84wwKaraSzpTNJwPVV8edw8g@mail.gmail.com/>`_::
+
+    > Now this got me wondering if Debian _unstable_ actually qualifies as a
+    > standard distro userspace.
+
+    Oh, if the kernel breaks some standard user space, that counts. Tons
+    of people run Debian unstable
+
+* From `2011-05-06 <https://lore.kernel.org/all/BANLkTi=KVXjKR82sqsz4gwjr+E0vtqCmvA@mail.gmail.com/>`_::
+
+    It's clearly NOT an internal tracepoint. By definition. It's being
+    used by powertop.
+
+On regressions noticed by users or test-suites/CIs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2026-01-22 <https://lore.kernel.org/all/CAHk-=wheQNiW_WtHGO7bKkT7Uib-p+ai2JP9M+z+FYcZ6CAxYA@mail.gmail.com/>`_::
+
+    Users complaining is the only real line in the end.
+
+    [...] a test-suite complaining is then often a *very* good indication that
+    maybe users will hit some problem, and test suite issues should be taken
+    very seriously [...]
+
+    But a test-suite error isn't necessarily where you have to draw the
+    line - it's a big red flag [...]
+
+* From `2024-29-01 <https://lore.kernel.org/all/CAHk-=wg8BrZEzjJ5kUyZzHPZmFqH6ooMN1gRBCofxxCfucgjaw@mail.gmail.com/>`_::
+
+    The "no regressions" rule is not about made-up "if I do this, behavior
+    changes".
+
+    The "no regressions" rule is about *users*.
+
+    If you have an actual user that has been doing insane things, and we
+    change something, and now the insane thing no longer works, at that
+    point it's a regression, and we'll sigh, and go "Users are insane" and
+    have to fix it.
+
+    But if you have some random test that now behaves differently, it's
+    not a regression. It's a *warning* sign, sure: tests are useful.
+
+On accepting when a regression occurred
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2026-01-22 <https://lore.kernel.org/all/CAHk-=wheQNiW_WtHGO7bKkT7Uib-p+ai2JP9M+z+FYcZ6CAxYA@mail.gmail.com/>`_::
+
+    But starting to argue about users reporting breaking changes is
+    basically the final line for me. I have a couple of people that I have
+    in my spam block-list and refuse to have anything to do with, and they
+    have generally been about exactly that.
+
+    Note how it's not about making mistakes and _causing_ the regression.
+    That's normal. That's development. But then arguing about it is a
+    no-no.
+
+* From `2024-06-23 <https://lore.kernel.org/all/CAHk-=wi_KMO_rJ6OCr8mAWBRg-irziM=T9wxGC+J1VVoQb39gw@mail.gmail.com/>`_::
+
+    We don't introduce regressions and then blame others.
+
+    There's a very clear rule in kernel development: things that break
+    other things ARE NOT FIXES.
+
+    EVER.
+
+    They get reverted, or the thing they broke gets fixed.
+
+* From `2021-06-05 <https://lore.kernel.org/all/CAHk-=wiUVqHN76YUwhkjZzwTdjMMJf_zN4+u7vEJjmEGh3recw@mail.gmail.com/>`_::
+
+    THERE ARE NO VALID ARGUMENTS FOR REGRESSIONS.
+
+    Honestly, security people need to understand that "not working" is not
+    a success case of security. It's a failure case.
+
+    Yes, "not working" may be secure. But security in that case is *pointless*.
+
+* From `2017-10-26(5) <https://lore.kernel.org/lkml/CA+55aFwiiQYJ+YoLKCXjN_beDVfu38mg=Ggg5LFOcqHE8Qi7Zw@mail.gmail.com/>`_::
+
+    [...] when regressions *do* occur, we admit to them and fix them, instead of
+    blaming user space.
+
+    The fact that you have apparently been denying the regression now for
+    three weeks means that I will revert, and I will stop pulling apparmor
+    requests until the people involved understand how kernel development
+    is done.
+
+On back-and-forth
+~~~~~~~~~~~~~~~~~
+
+* From `2024-05-28 <https://lore.kernel.org/all/CAHk-=wgtb7y-bEh7tPDvDWru7ZKQ8-KMjZ53Tsk37zsPPdwXbA@mail.gmail.com/>`_::
+
+    The "no regressions" rule is that we do not introduce NEW bugs.
+
+    It *literally* came about because we had an endless dance of "fix two
+    bugs, introduce one new one", and that then resulted in a system that
+    you cannot TRUST.
+
+* From `2021-09-20(1) <https://lore.kernel.org/all/CAHk-=wi7DB2SJ-wngVvsJ7Ak2cM556Q8437sOXo4EJt2BWPdEg@mail.gmail.com/>`_::
+
+    And the thing that makes regressions special is that back when I
+    wasn't so strict about these things, we'd end up in endless "seesaw
+    situations" where somebody would fix something, it would break
+    something else, then that something else would break, and it would
+    never actually converge on anything reliable at all.
+
+* From `2015-08-13 <https://lore.kernel.org/all/CA+55aFxk8-BsiKwr_S-c+4G6wihKPQVMLE34H9wOZpeua6W9+Q@mail.gmail.com/>`_::
+
+    The strict policy of no regressions actually originally started mainly wrt
+    suspend/resume issues, where the "fix one machine, break another" kind of
+    back-and-forth caused endless problems, and meant that we didn't actually
+    necessarily make any forward progress, just moving a problem around.
+
+On changes with a risk of causing regressions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2023-06-02 <https://lore.kernel.org/all/CAHk-=wgyAGUMHmQM-5Eb556z5xiHZB7cF05qjrtUH4F7P-1rSA@mail.gmail.com/>`_::
+
+    So what I think you should do is to fix the bug right, with a clean
+    patch, and no crazy hacks. That is something we can then apply and
+    test. All the while knowing full well that "uhhuh, this is a visible
+    change, we may have to revert it".
+
+    If then some *real* load ends up showing a regression, we may just be
+    screwed. Our current behavior may be buggy, but we have the rule that
+    once user space depends on kernel bugs, they become features pretty
+    much by definition, however much we might dislike it.
+
+On in-kernel workarounds to avoid regressions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2017-10-26(6) <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
+
+    Behavioral changes happen, and maybe we don't even support some
+    feature any more. There's a number of fields in /proc/<pid>/stat that
+    are printed out as zeroes, simply because they don't even *exist* in
+    the kernel any more, or because showing them was a mistake (typically
+    an information leak). But the numbers got replaced by zeroes, so that
+    the code that used to parse the fields still works. The user might not
+    see everything they used to see, and so behavior is clearly different,
+    but things still _work_, even if they might no longer show sensitive
+    (or no longer relevant) information.
+
+On regressions caused by bugfixes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2018-08-03 <https://lore.kernel.org/all/CA+55aFwWZX=CXmWDTkDGb36kf12XmTehmQjbiMPCqCRG2hi9kw@mail.gmail.com/>`_::
+
+    > Kernel had a bug which has been fixed
+
+    That is *ENTIRELY* immaterial.
+
+    Guys, whether something was buggy or not DOES NOT MATTER.
+
+    [...]
+
+    It's basically saying "I took something that worked, and I broke it,
+    but now it's better". Do you not see how f*cking insane that statement
+    is?
+
+On internal API changes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2017-10-26(7) <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
+
+    We do API breakage _inside_ the kernel all the time. We will fix
+    internal problems by saying "you now need to do XYZ", but then it's
+    about internal kernel API's, and the people who do that then also
+    obviously have to fix up all the in-kernel users of that API. Nobody
+    can say "I now broke the API you used, and now _you_ need to fix it
+    up". Whoever broke something gets to fix it too.
+
+On regressions only found after a long time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2024-03-28 <https://lore.kernel.org/all/CAHk-=wgFuoHpMk_Z_R3qMXVDgq0N1592+bABkyGjwwSL4zBtHA@mail.gmail.com/>`_::
+
+    I'm definitely not reverting a patch from almost a decade ago as a
+    regression.
+
+    If it took that long to find, it can't be that critical of a regression.
+
+    So yes, let's treat it as a regular bug.
+
+On testing regressions fixes in linux-next
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* On `maintainers summit 2024 <https://lwn.net/Articles/990599/>`_::
+
+   So running fixes though linux-next is just a waste of time.
+
+On a few other aspects related to regressions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* From `2025-07-29(2) <https://lore.kernel.org/all/CAHk-=wjj9DvOZtmTkoLtyfHmy5mNKy6q_96d9=4FUEDXre=cww@mail.gmail.com/>`_
+  [which `is not quite a regression, but a huge inconvenience <https://lore.kernel.org/all/CAHk-=wgO0Rx2LcYT4f75Xs46orbJ4JxO2jbAFQnVKDYAjV5HeQ@mail.gmail.com/>`_]::
+
+    I no longer have sound.
+
+    I also suspect that it's purely because "make oldconfig" doesn't work,
+    and probably turned off my old Intel HDA settings. Or something.
+
+    Renaming config parameters is *bad*. I've harped on the Kconfig phase
+    of the kernel build probably being our nastiest point, and a real pain
+    point to people getting involved with development simply because
+    building your own kernel can be so daunting with hundreds of fairly
+    esoteric questions.
 
 ..
    end-of-content
diff --git a/Documentation/process/license-rules.rst b/Documentation/process/license-rules.rst
index 59a7832df7d08..b0176bb8a4653 100644
--- a/Documentation/process/license-rules.rst
+++ b/Documentation/process/license-rules.rst
@@ -63,8 +63,11 @@ License identifier syntax
    The SPDX license identifier in kernel files shall be added at the first
    possible line in a file which can contain a comment.  For the majority
    of files this is the first line, except for scripts which require the
-   '#!PATH_TO_INTERPRETER' in the first line.  For those scripts the SPDX
-   identifier goes into the second line.
+   '#!PATH_TO_INTERPRETER' in the first line.  For those scripts, the SPDX
+   license identifier goes into the second line.
+
+   The license identifier line can then be followed by one or multiple
+   SPDX-FileCopyrightText lines if desired.
 
 |
 
diff --git a/Documentation/process/maintainer-handbooks.rst b/Documentation/process/maintainer-handbooks.rst
index 976391cec528e..3d72ad25fc6ac 100644
--- a/Documentation/process/maintainer-handbooks.rst
+++ b/Documentation/process/maintainer-handbooks.rst
@@ -1,7 +1,5 @@
 .. SPDX-License-Identifier: GPL-2.0
 
-.. _maintainer_handbooks_main:
-
 Subsystem and maintainer tree specific development process notes
 ================================================================
 
diff --git a/Documentation/process/maintainer-netdev.rst b/Documentation/process/maintainer-netdev.rst
index 6bce4507d5d31..3aa13bc2405d5 100644
--- a/Documentation/process/maintainer-netdev.rst
+++ b/Documentation/process/maintainer-netdev.rst
@@ -479,8 +479,14 @@ netdevsim
 
 ``netdevsim`` is a test driver which can be used to exercise driver
 configuration APIs without requiring capable hardware.
-Mock-ups and tests based on ``netdevsim`` are strongly encouraged when
-adding new APIs, but ``netdevsim`` in itself is **not** considered
+Mock-ups and tests based on ``netdevsim`` are encouraged when
+adding new APIs with complex logic in the stack. The tests should
+be written so that they can run both against ``netdevsim`` and a real
+device (see ``tools/testing/selftests/drivers/net/README.rst``).
+``netdevsim``-only tests should focus on testing corner cases
+and failure paths in the core which are hard to exercise with a real driver.
+
+``netdevsim`` in itself is **not** considered
 a use case/user. You must also implement the new APIs in a real driver.
 
 We give no guarantees that ``netdevsim`` won't change in the future
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
index bfe877a1a7e4a..652dfbe64102c 100644
--- a/Documentation/process/maintainer-pgp-guide.rst
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -897,6 +897,8 @@ the new default in GnuPG v2). To set it, add (or modify) the
 
     trust-model tofu+pgp
 
+.. _kernel_org_trust_repository:
+
 Using the kernel.org web of trust repository
 --------------------------------------------
 
diff --git a/Documentation/process/maintainer-tip.rst b/Documentation/process/maintainer-tip.rst
index 41d5855700cd4..b2b14439be227 100644
--- a/Documentation/process/maintainer-tip.rst
+++ b/Documentation/process/maintainer-tip.rst
@@ -352,7 +352,7 @@ following tag ordering scheme:
      Changelog text starts here....
 
    so the authorship is preserved. The 'From:' line has to be followed
-   by a empty newline. If that 'From:' line is missing, then the patch
+   by an empty newline. If that 'From:' line is missing, then the patch
    would be attributed to the person who sent (transported, handled) it.
    The 'From:' line is automatically removed when the patch is applied
    and does not show up in the final git changelog. It merely affects
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index e69d19ad658fa..c88e75bf05341 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -23,7 +23,7 @@ easier.
 
 Some subsystems and maintainer trees have additional information about
 their workflow and expectations, see
-:ref:`Documentation/process/maintainer-handbooks.rst <maintainer_handbooks_main>`.
+Documentation/process/maintainer-handbooks.rst.
 
 Obtain a current source tree
 ----------------------------
diff --git a/Documentation/scheduler/sched-deadline.rst b/Documentation/scheduler/sched-deadline.rst
index ec543a12f848e..3ad93cd7b59ad 100644
--- a/Documentation/scheduler/sched-deadline.rst
+++ b/Documentation/scheduler/sched-deadline.rst
@@ -628,10 +628,21 @@ Deadline Task Scheduling
   * the new scheduling related syscalls that manipulate it, i.e.,
     sched_setattr() and sched_getattr() are implemented.
 
- For debugging purposes, the leftover runtime and absolute deadline of a
- SCHED_DEADLINE task can be retrieved through /proc/<pid>/sched (entries
- dl.runtime and dl.deadline, both values in ns). A programmatic way to
- retrieve these values from production code is under discussion.
+ The leftover runtime and absolute deadline of a SCHED_DEADLINE task can be
+ read using the sched_getattr() syscall, setting the last syscall parameter
+ flags to the SCHED_GETATTR_FLAG_DL_DYNAMIC=1 value. This updates the
+ runtime left, converts the absolute deadline in CLOCK_MONOTONIC reference,
+ then returns these parameters to user-space. The absolute deadline is
+ returned as the number of nanoseconds since the CLOCK_MONOTONIC time
+ reference (boot instant), as a u64 in the sched_deadline field of sched_attr,
+ which can represent nearly 585 years since boot time (calling sched_getattr()
+ with flags=0 causes retrieval of the static parameters instead).
+
+ For debugging purposes, these parameters can also be retrieved through
+ /proc/<pid>/sched (entries dl.runtime and dl.deadline, both values in ns),
+ but: this is highly inefficient; the returned runtime left is not updated as
+ done by sched_getattr(); the deadline is provided in kernel rq_clock time
+ reference, that is not directly usable from user-space.
 
 
 4.3 Default behavior
@@ -700,7 +711,8 @@ Deadline Task Scheduling
 5.2 Using cgroup v2 cpuset controller
 -------------------------------------
 
- Assuming the cgroup v2 root is mounted at ``/sys/fs/cgroup``.
+ Assuming the cgroup v2 root is mounted at ``/sys/fs/cgroup``, an example of a
+ simple configuration (pin a -deadline task to CPU0) follows::
 
    cd /sys/fs/cgroup
    echo '+cpuset' > cgroup.subtree_control
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
index d74c2c2b9ef39..f4f7d8f4f9e42 100644
--- a/Documentation/scheduler/sched-ext.rst
+++ b/Documentation/scheduler/sched-ext.rst
@@ -228,16 +228,23 @@ The following briefly shows how a waking task is scheduled and executed.
    scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper,
    using ``ops.select_cpu()`` judiciously can be simpler and more efficient.
 
-   A task can be immediately inserted into a DSQ from ``ops.select_cpu()``
-   by calling ``scx_bpf_dsq_insert()``. If the task is inserted into
-   ``SCX_DSQ_LOCAL`` from ``ops.select_cpu()``, it will be inserted into the
-   local DSQ of whichever CPU is returned from ``ops.select_cpu()``.
-   Additionally, inserting directly from ``ops.select_cpu()`` will cause the
-   ``ops.enqueue()`` callback to be skipped.
-
    Note that the scheduler core will ignore an invalid CPU selection, for
    example, if it's outside the allowed cpumask of the task.
 
+   A task can be immediately inserted into a DSQ from ``ops.select_cpu()``
+   by calling ``scx_bpf_dsq_insert()`` or ``scx_bpf_dsq_insert_vtime()``.
+
+   If the task is inserted into ``SCX_DSQ_LOCAL`` from
+   ``ops.select_cpu()``, it will be added to the local DSQ of whichever CPU
+   is returned from ``ops.select_cpu()``. Additionally, inserting directly
+   from ``ops.select_cpu()`` will cause the ``ops.enqueue()`` callback to
+   be skipped.
+
+   Any other attempt to store a task in BPF-internal data structures from
+   ``ops.select_cpu()`` does not prevent ``ops.enqueue()`` from being
+   invoked. This is discouraged, as it can introduce racy behavior or
+   inconsistent state.
+
 2. Once the target CPU is selected, ``ops.enqueue()`` is invoked (unless the
    task was inserted directly from ``ops.select_cpu()``). ``ops.enqueue()``
    can make one of the following decisions:
@@ -251,6 +258,61 @@ The following briefly shows how a waking task is scheduled and executed.
 
    * Queue the task on the BPF side.
 
+   **Task State Tracking and ops.dequeue() Semantics**
+
+   A task is in the "BPF scheduler's custody" when the BPF scheduler is
+   responsible for managing its lifecycle. A task enters custody when it is
+   dispatched to a user DSQ or stored in the BPF scheduler's internal data
+   structures. Custody is entered only from ``ops.enqueue()`` for those
+   operations. The only exception is dispatching to a user DSQ from
+   ``ops.select_cpu()``: although the task is not yet technically in BPF
+   scheduler custody at that point, the dispatch has the same semantic
+   effect as dispatching from ``ops.enqueue()`` for custody-related
+   purposes.
+
+   Once ``ops.enqueue()`` is called, the task may or may not enter custody
+   depending on what the scheduler does:
+
+   * **Directly dispatched to terminal DSQs** (``SCX_DSQ_LOCAL``,
+     ``SCX_DSQ_LOCAL_ON | cpu``, or ``SCX_DSQ_GLOBAL``): the BPF scheduler
+     is done with the task - it either goes straight to a CPU's local run
+     queue or to the global DSQ as a fallback. The task never enters (or
+     exits) BPF custody, and ``ops.dequeue()`` will not be called.
+
+   * **Dispatch to user-created DSQs** (custom DSQs): the task enters the
+     BPF scheduler's custody. When the task later leaves BPF custody
+     (dispatched to a terminal DSQ, picked by core-sched, or dequeued for
+     sleep/property changes), ``ops.dequeue()`` will be called exactly
+     once.
+
+   * **Stored in BPF data structures** (e.g., internal BPF queues): the
+     task is in BPF custody. ``ops.dequeue()`` will be called when it
+     leaves (e.g., when ``ops.dispatch()`` moves it to a terminal DSQ, or
+     on property change / sleep).
+
+   When a task leaves BPF scheduler custody, ``ops.dequeue()`` is invoked.
+   The dequeue can happen for different reasons, distinguished by flags:
+
+   1. **Regular dispatch**: when a task in BPF custody is dispatched to a
+      terminal DSQ from ``ops.dispatch()`` (leaving BPF custody for
+      execution), ``ops.dequeue()`` is triggered without any special flags.
+
+   2. **Core scheduling pick**: when ``CONFIG_SCHED_CORE`` is enabled and
+      core scheduling picks a task for execution while it's still in BPF
+      custody, ``ops.dequeue()`` is called with the
+      ``SCX_DEQ_CORE_SCHED_EXEC`` flag.
+
+   3. **Scheduling property change**: when a task property changes (via
+      operations like ``sched_setaffinity()``, ``sched_setscheduler()``,
+      priority changes, CPU migrations, etc.) while the task is still in
+      BPF custody, ``ops.dequeue()`` is called with the
+      ``SCX_DEQ_SCHED_CHANGE`` flag set in ``deq_flags``.
+
+   **Important**: Once a task has left BPF custody (e.g., after being
+   dispatched to a terminal DSQ), property changes will not trigger
+   ``ops.dequeue()``, since the task is no longer managed by the BPF
+   scheduler.
+
 3. When a CPU is ready to schedule, it first looks at its local DSQ. If
    empty, it then looks at the global DSQ. If there still isn't a task to
    run, ``ops.dispatch()`` is invoked which can use the following two
@@ -318,6 +380,8 @@ by a sched_ext scheduler:
                 /* Any usable CPU becomes available */
 
                 ops.dispatch(); /* Task is moved to a local DSQ */
+
+                ops.dequeue(); /* Exiting BPF scheduler */
             }
             ops.running();      /* Task starts running on its assigned CPU */
             while (task->scx.slice > 0 && task is runnable)
diff --git a/Documentation/security/ipe.rst b/Documentation/security/ipe.rst
index 4a7d953abcdc3..5eb3e6265fbde 100644
--- a/Documentation/security/ipe.rst
+++ b/Documentation/security/ipe.rst
@@ -18,7 +18,7 @@ strong integrity guarantees over both the executable code, and specific
 *data files* on the system, that were critical to its function. These
 specific data files would not be readable unless they passed integrity
 policy. A mandatory access control system would be present, and
-as a result, xattrs would have to be protected. This lead to a selection
+as a result, xattrs would have to be protected. This led to a selection
 of what would provide the integrity claims. At the time, there were two
 main mechanisms considered that could guarantee integrity for the system
 with these requirements:
@@ -195,7 +195,7 @@ of the policy to apply the minute usermode starts. Generally, that storage
 can be handled in one of three ways:
 
   1. The policy file(s) live on disk and the kernel loads the policy prior
-     to an code path that would result in an enforcement decision.
+     to a code path that would result in an enforcement decision.
   2. The policy file(s) are passed by the bootloader to the kernel, who
      parses the policy.
   3. There is a policy file that is compiled into the kernel that is
@@ -235,8 +235,8 @@ Updatable, Rebootless Policy
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 As requirements change over time (vulnerabilities are found in previously
-trusted applications, keys roll, etcetera). Updating a kernel to change the
-meet those security goals is not always a suitable option, as updates are not
+trusted applications, keys roll, etcetera), updating a kernel to meet
+those security goals is not always a suitable option, as updates are not
 always risk-free, and blocking a security update leaves systems vulnerable.
 This means IPE requires a policy that can be completely updated (allowing
 revocations of existing policy) from a source external to the kernel (allowing
@@ -370,7 +370,7 @@ Simplified Policy:
 Finally, IPE's policy is designed for sysadmins, not kernel developers. Instead
 of covering individual LSM hooks (or syscalls), IPE covers operations. This means
 instead of sysadmins needing to know that the syscalls ``mmap``, ``mprotect``,
-``execve``, and ``uselib`` must have rules protecting them, they must simple know
+``execve``, and ``uselib`` must have rules protecting them, they must simply know
 that they want to restrict code execution. This limits the amount of bypasses that
 could occur due to a lack of knowledge of the underlying system; whereas the
 maintainers of IPE, being kernel developers can make the correct choice to determine
diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst
index 55b845d382368..f75f087639415 100644
--- a/Documentation/sound/alsa-configuration.rst
+++ b/Documentation/sound/alsa-configuration.rst
@@ -2376,6 +2376,13 @@ quirk_flags
           Skip the probe-time interface setup (usb_set_interface,
           init_pitch, init_sample_rate); redundant with
           snd_usb_endpoint_prepare() at stream-open time
+        * bit 27: ``mixer_playback_linear_vol``
+          Set linear volume mapping for devices where the playback volume
+          control value is mapped to voltage (instead of dB) level linearly.
+          In short: ``x(raw) = (raw - raw_min) / (raw_max - raw_min)``;
+          ``V(x) = k * x``; ``dB(x) = 20 * log10(x)``. Overrides bit 24
+        * bit 28: ``mixer_capture_linear_vol``
+          Similar to bit 27 but for capture streams. Overrides bit 25
 
 This module supports multiple devices, autoprobe and hotplugging.
 
diff --git a/Documentation/sphinx/translations.py b/Documentation/sphinx/translations.py
index 32c2b32b2b5ee..a03d4402b4f18 100644
--- a/Documentation/sphinx/translations.py
+++ b/Documentation/sphinx/translations.py
@@ -25,6 +25,7 @@ all_languages = {
     'it_IT': 'Italian',
     'ja_JP': 'Japanese',
     'ko_KR': 'Korean',
+    'pt_BR': 'Portuguese (Brazilian)',
     'sp_SP': 'Spanish',
 }
 
diff --git a/Documentation/sunrpc/xdr/nlm4.x b/Documentation/sunrpc/xdr/nlm4.x
new file mode 100644
index 0000000000000..0c44a80ef6748
--- /dev/null
+++ b/Documentation/sunrpc/xdr/nlm4.x
@@ -0,0 +1,211 @@
+/*
+ * This file was extracted by hand from
+ * https://www.rfc-editor.org/rfc/rfc1813.html .
+ *
+ * Note that RFC 1813 is Informational. Its official date of
+ * publication (June 1995) is before the IETF required its RFCs to
+ * carry an explicit copyright or other IP ownership notices.
+ *
+ * Note also that RFC 1813 does not specify the whole NLM4 protocol.
+ * In particular, the argument and result types are not present in
+ * that document, and had to be reverse-engineered.
+ */
+
+/*
+ * The NLMv4 protocol
+ */
+
+pragma header nlm4;
+
+/*
+ * The following definitions are missing in RFC 1813,
+ * but can be found in the OpenNetworking Network Lock
+ * Manager protocol:
+ *
+ * https://pubs.opengroup.org/onlinepubs/9629799/chap10.htm
+ */
+
+const LM_MAXSTRLEN = 1024;
+
+const LM_MAXNAMELEN = 1025;
+
+const MAXNETOBJ_SZ = 1024;
+
+typedef opaque netobj<MAXNETOBJ_SZ>;
+
+enum fsh4_mode {
+	fsm_DN = 0,		/* deny none */
+	fsm_DR = 1,		/* deny read */
+	fsm_DW = 2,		/* deny write */
+	fsm_DRW = 3		/* deny read/write */
+};
+
+enum fsh4_access {
+	fsa_NONE = 0,		/* for completeness */
+	fsa_R = 1,		/* read-only */
+	fsa_W = 2,		/* write-only */
+	fsa_RW = 3		/* read/write */
+};
+
+/*
+ * The following definitions come from the OpenNetworking
+ * Network Status Monitor protocol:
+ *
+ * https://pubs.opengroup.org/onlinepubs/9629799/chap11.htm
+ */
+
+const SM_MAXSTRLEN = 1024;
+
+/*
+ * The NLM protocol as extracted from:
+ * https://tools.ietf.org/html/rfc1813 Appendix II
+ */
+
+typedef unsigned hyper uint64;
+
+typedef hyper int64;
+
+typedef unsigned long uint32;
+
+typedef long int32;
+
+enum nlm4_stats {
+	NLM4_GRANTED = 0,
+	NLM4_DENIED = 1,
+	NLM4_DENIED_NOLOCKS = 2,
+	NLM4_BLOCKED = 3,
+	NLM4_DENIED_GRACE_PERIOD = 4,
+	NLM4_DEADLCK = 5,
+	NLM4_ROFS = 6,
+	NLM4_STALE_FH = 7,
+	NLM4_FBIG = 8,
+	NLM4_FAILED = 9
+};
+
+pragma big_endian nlm4_stats;
+
+struct nlm4_holder {
+	bool		exclusive;
+	int32		svid;
+	netobj		oh;
+	uint64		l_offset;
+	uint64		l_len;
+};
+
+union nlm4_testrply switch (nlm4_stats stat) {
+	case NLM4_DENIED:
+		nlm4_holder	holder;
+	default:
+		void;
+};
+
+struct nlm4_stat {
+	nlm4_stats	stat;
+};
+
+struct nlm4_res {
+	netobj		cookie;
+	nlm4_stat	stat;
+};
+
+struct nlm4_testres {
+	netobj		cookie;
+	nlm4_testrply	stat;
+};
+
+struct nlm4_lock {
+	string		caller_name<LM_MAXSTRLEN>;
+	netobj		fh;
+	netobj		oh;
+	int32		svid;
+	uint64		l_offset;
+	uint64		l_len;
+};
+
+struct nlm4_lockargs {
+	netobj		cookie;
+	bool		block;
+	bool		exclusive;
+	nlm4_lock	alock;
+	bool		reclaim;
+	int32		state;
+};
+
+struct nlm4_cancargs {
+	netobj		cookie;
+	bool		block;
+	bool		exclusive;
+	nlm4_lock	alock;
+};
+
+struct nlm4_testargs {
+	netobj		cookie;
+	bool		exclusive;
+	nlm4_lock	alock;
+};
+
+struct nlm4_unlockargs {
+	netobj		cookie;
+	nlm4_lock	alock;
+};
+
+struct nlm4_share {
+	string		caller_name<LM_MAXSTRLEN>;
+	netobj		fh;
+	netobj		oh;
+	fsh4_mode	mode;
+	fsh4_access	access;
+};
+
+struct nlm4_shareargs {
+	netobj		cookie;
+	nlm4_share	share;
+	bool		reclaim;
+};
+
+struct nlm4_shareres {
+	netobj		cookie;
+	nlm4_stats	stat;
+	int32		sequence;
+};
+
+struct nlm4_notify {
+	string		name<LM_MAXNAMELEN>;
+	int32		state;
+};
+
+/*
+ * Argument for the Linux-private SM_NOTIFY procedure
+ */
+const SM_PRIV_SIZE = 16;
+
+struct nlm4_notifyargs {
+	nlm4_notify	notify;
+	opaque		private[SM_PRIV_SIZE];
+};
+
+program NLM4_PROG {
+	version NLM4_VERS {
+		void		NLMPROC4_NULL(void)			= 0;
+		nlm4_testres	NLMPROC4_TEST(nlm4_testargs)		= 1;
+		nlm4_res	NLMPROC4_LOCK(nlm4_lockargs)		= 2;
+		nlm4_res	NLMPROC4_CANCEL(nlm4_cancargs)		= 3;
+		nlm4_res	NLMPROC4_UNLOCK(nlm4_unlockargs)	= 4;
+		nlm4_res	NLMPROC4_GRANTED(nlm4_testargs)		= 5;
+		void		NLMPROC4_TEST_MSG(nlm4_testargs)	= 6;
+		void		NLMPROC4_LOCK_MSG(nlm4_lockargs)	= 7;
+		void		NLMPROC4_CANCEL_MSG(nlm4_cancargs)	= 8;
+		void		NLMPROC4_UNLOCK_MSG(nlm4_unlockargs)	= 9;
+		void		NLMPROC4_GRANTED_MSG(nlm4_testargs)	= 10;
+		void		NLMPROC4_TEST_RES(nlm4_testres)		= 11;
+		void		NLMPROC4_LOCK_RES(nlm4_res)		= 12;
+		void		NLMPROC4_CANCEL_RES(nlm4_res)		= 13;
+		void		NLMPROC4_UNLOCK_RES(nlm4_res)		= 14;
+		void		NLMPROC4_GRANTED_RES(nlm4_res)		= 15;
+		void		NLMPROC4_SM_NOTIFY(nlm4_notifyargs)	= 16;
+		nlm4_shareres	NLMPROC4_SHARE(nlm4_shareargs)		= 20;
+		nlm4_shareres	NLMPROC4_UNSHARE(nlm4_shareargs)	= 21;
+		nlm4_res	NLMPROC4_NM_LOCK(nlm4_lockargs)		= 22;
+		void		NLMPROC4_FREE_ALL(nlm4_notify)		= 23;
+	} = 4;
+} = 100021;
diff --git a/Documentation/tools/kdoc_parser.rst b/Documentation/tools/kdoc_parser.rst
index 03ee54a1b1cc2..55b202173195d 100644
--- a/Documentation/tools/kdoc_parser.rst
+++ b/Documentation/tools/kdoc_parser.rst
@@ -4,6 +4,14 @@
 Kernel-doc parser stage
 =======================
 
+C replacement rules used by the parser
+======================================
+
+.. automodule:: lib.python.kdoc.xforms_lists
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 File handler classes
 ====================
 
diff --git a/Documentation/tools/rtla/rtla.rst b/Documentation/tools/rtla/rtla.rst
index 2a5fb7004ad44..6df1296b8cc14 100644
--- a/Documentation/tools/rtla/rtla.rst
+++ b/Documentation/tools/rtla/rtla.rst
@@ -21,6 +21,10 @@ results.
 
 COMMANDS
 ========
+**hwnoise**
+
+        Detect and quantify hardware-related noise.
+
 **osnoise**
 
         Gives information about the operating system noise (osnoise).
@@ -39,7 +43,7 @@ For other options, see the man page for the corresponding command.
 
 SEE ALSO
 ========
-**rtla-osnoise**\(1), **rtla-timerlat**\(1)
+**rtla-hwnoise**\(1), **rtla-osnoise**\(1), **rtla-timerlat**\(1)
 
 AUTHOR
 ======
diff --git a/Documentation/trace/events-kmem.rst b/Documentation/trace/events-kmem.rst
index 68fa75247488b..42f08f3b136c7 100644
--- a/Documentation/trace/events-kmem.rst
+++ b/Documentation/trace/events-kmem.rst
@@ -57,7 +57,7 @@ the per-CPU allocator (high performance) or the buddy allocator.
 
 If pages are allocated directly from the buddy allocator, the
 mm_page_alloc_zone_locked event is triggered. This event is important as high
-amounts of activity imply high activity on the zone->lock. Taking this lock
+amounts of activity imply high activity on the zone lock. Taking this lock
 impairs performance by disabling interrupts, dirtying cache lines between
 CPUs and serialising many CPUs.
 
@@ -79,11 +79,11 @@ contention on the lruvec->lru_lock.
   mm_page_pcpu_drain		page=%p pfn=%lu order=%d cpu=%d migratetype=%d
 
 In front of the page allocator is a per-cpu page allocator. It exists only
-for order-0 pages, reduces contention on the zone->lock and reduces the
+for order-0 pages, reduces contention on the zone lock and reduces the
 amount of writing on struct page.
 
 When a per-CPU list is empty or pages of the wrong type are allocated,
-the zone->lock will be taken once and the per-CPU list refilled. The event
+the zone lock will be taken once and the per-CPU list refilled. The event
 triggered is mm_page_alloc_zone_locked for each page allocated with the
 event indicating whether it is for a percpu_refill or not.
 
@@ -92,7 +92,7 @@ which triggers a mm_page_pcpu_drain event.
 
 The individual nature of the events is so that pages can be tracked
 between allocation and freeing. A number of drain or refill pages that occur
-consecutively imply the zone->lock being taken once. Large amounts of per-CPU
+consecutively imply the zone lock being taken once. Large amounts of per-CPU
 refills and drains could imply an imbalance between CPUs where too much work
 is being concentrated in one place. It could also indicate that the per-CPU
 lists should be a larger size. Finally, large amounts of refills on one CPU
diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst
index 338bc4d7cfab1..036db96864d23 100644
--- a/Documentation/trace/index.rst
+++ b/Documentation/trace/index.rst
@@ -91,6 +91,17 @@ interactions.
    user_events
    uprobetracer
 
+Remote Tracing
+--------------
+
+This section covers the framework to read compatible ring-buffers, written by
+entities outside of the kernel (most likely firmware or hypervisor)
+
+.. toctree::
+   :maxdepth: 1
+
+   remotes
+
 Additional Resources
 --------------------
 
diff --git a/Documentation/trace/remotes.rst b/Documentation/trace/remotes.rst
new file mode 100644
index 0000000000000..1f9d764f69aa6
--- /dev/null
+++ b/Documentation/trace/remotes.rst
@@ -0,0 +1,66 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===============
+Tracing Remotes
+===============
+
+:Author: Vincent Donnefort <vdonnefort@google.com>
+
+Overview
+========
+Firmware and hypervisors are black boxes to the kernel. Having a way to see what
+they are doing can be useful to debug both. This is where remote tracing buffers
+come in. A remote tracing buffer is a ring buffer executed by the firmware or
+hypervisor into memory that is memory mapped to the host kernel. This is similar
+to how user space memory maps the kernel ring buffer but in this case the kernel
+is acting like user space and the firmware or hypervisor is the "kernel" side.
+With a trace remote ring buffer, the firmware and hypervisor can record events
+for which the host kernel can see and expose to user space.
+
+Register a remote
+=================
+A remote must provide a set of callbacks `struct trace_remote_callbacks` whom
+description can be found below. Those callbacks allows Tracefs to enable and
+disable tracing and events, to load and unload a tracing buffer (a set of
+ring-buffers) and to swap a reader page with the head page, which enables
+consuming reading.
+
+.. kernel-doc:: include/linux/trace_remote.h
+
+Once registered, an instance will appear for this remote in the Tracefs
+directory **remotes/**. Buffers can then be read using the usual Tracefs files
+**trace_pipe** and **trace**.
+
+Declare a remote event
+======================
+Macros are provided to ease the declaration of remote events, in a similar
+fashion to in-kernel events. A declaration must provide an ID, a description of
+the event arguments and how to print the event:
+
+.. code-block:: c
+
+	REMOTE_EVENT(foo, EVENT_FOO_ID,
+		RE_STRUCT(
+			re_field(u64, bar)
+		),
+		RE_PRINTK("bar=%lld", __entry->bar)
+	);
+
+Then those events must be declared in a C file with the following:
+
+.. code-block:: c
+
+	#define REMOTE_EVENT_INCLUDE_FILE foo_events.h
+	#include <trace/define_remote_events.h>
+
+This will provide a `struct remote_event remote_event_foo` that can be given to
+`trace_remote_register`.
+
+Registered events appear in the remote directory under **events/**.
+
+Simple ring-buffer
+==================
+A simple implementation for a ring-buffer writer can be found in
+kernel/trace/simple_ring_buffer.c.
+
+.. kernel-doc:: include/linux/simple_ring_buffer.h
diff --git a/Documentation/translations/index.rst b/Documentation/translations/index.rst
index b826c34791c0b..b6d24f6f17d1a 100644
--- a/Documentation/translations/index.rst
+++ b/Documentation/translations/index.rst
@@ -10,11 +10,11 @@ Translations
    zh_CN/index
    zh_TW/index
    it_IT/index
-   ko_KR/index
    ja_JP/index
+   ko_KR/index
+   pt_BR/index
    sp_SP/index
 
-
 .. _translations_disclaimer:
 
 Disclaimer
diff --git a/Documentation/translations/ja_JP/process/submitting-patches.rst b/Documentation/translations/ja_JP/process/submitting-patches.rst
index d61583399ef46..b89e63a1f65c0 100644
--- a/Documentation/translations/ja_JP/process/submitting-patches.rst
+++ b/Documentation/translations/ja_JP/process/submitting-patches.rst
@@ -35,7 +35,7 @@ Documentation/devicetree/bindings/submitting-patches.rst を読んでくださ
 
 いくつかのサブシステムやメンテナツリーには、各々のワークフローや
 期待事項に関する追加情報があります。次を参照してください:
-:ref:`Documentation/process/maintainer-handbooks.rst <maintainer_handbooks_main>`.
+Documentation/process/maintainer-handbooks.rst.
 
 現在のソースツリーを入手する
 ----------------------------
@@ -52,5 +52,47 @@ Documentation/devicetree/bindings/submitting-patches.rst を読んでくださ
 ツリーは MAINTAINERS ファイル内の **T:** エントリを参照して見つけてください。
 そこに掲載されていない場合は、メンテナに問い合わせてください。
 
-変更内容を説明する
+変更内容を記述する
 ------------------
+
+問題を記述してください。あなたのパッチが 1 行のバグ修正であっても、
+5000 行の新機能であっても、それを行う動機となった根本的な問題が
+必ずあるはずです。修正すべき価値のある問題が存在し、レビューアが
+最初の段落以降を読む意味があることを納得させてください。
+
+ユーザーから見える影響を記述してください。クラッシュやロックアップは
+分かりやすいですが、すべてのバグがそこまで露骨とは限りません。
+たとえコードレビュー中に見つかった問題であっても、ユーザーに
+どのような影響があり得るかを記述してください。
+Linux の多くの環境は、上流から特定のパッチだけを取り込む二次的な
+安定版ツリーや、ベンダー／製品固有のツリーのカーネルで動いています。
+したがって、変更を下流へ適切に流す助けになる情報（発生条件、dmesg
+の抜粋、クラッシュ内容、性能劣化、レイテンシのスパイク、
+ロックアップ等）があれば記載してください。
+
+最適化とトレードオフを定量的に示してください。パフォーマンス、
+メモリ消費量、スタックフットプリント、バイナリサイズの改善を主張する
+場合は、それを裏付ける数値を記載してください。
+また、目に見えないコストについても記述してください。最適化は通常
+無料ではなく、CPU・メモリ・可読性の間でのトレードオフになります。
+ヒューリスティクスの場合は、異なるワークロード間でのトレードオフに
+なります。レビューアがコストとメリットを比較検討できるよう、
+最適化によって予想されるデメリットも記述してください。
+
+問題を明確にできたら、実際にどのような対策を講じているかを技術的に
+詳しく記述してください。レビューアがコードが意図したとおりに動作して
+いるかを確認できるよう、変更内容を平易な言葉で書き下すことが重要です。
+
+パッチ説明を Linux のソースコード管理システム ``git`` の
+「コミットログ」としてそのまま取り込める形で書けば、メンテナは
+助かります。詳細は原文の該当節を参照してください。
+
+.. TODO: Convert to file-local cross-reference when the destination is
+   translated.
+
+1 つのパッチでは 1 つの問題だけを解決してください。記述が長くなり
+始めたら、パッチを分割すべきサインです。詳細は原文の該当節を参照
+してください。
+
+.. TODO: Convert to file-local cross-reference when the destination is
+   translated.
diff --git a/Documentation/translations/pt_BR/disclaimer-pt_BR.rst b/Documentation/translations/pt_BR/disclaimer-pt_BR.rst
new file mode 100644
index 0000000000000..3cae22c13d3f8
--- /dev/null
+++ b/Documentation/translations/pt_BR/disclaimer-pt_BR.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+
+
+Aviso sobre traduções para português
+====================================
+
+Esta documentação foi traduzida para português brasileiro por voluntários.
+Em caso de qualquer divergência entre esta tradução e o documento original
+em inglês, a versão em inglês (encontrada no diretório Documentation/)
+deve ser considerada a única fonte de verdade.
diff --git a/Documentation/translations/pt_BR/index.rst b/Documentation/translations/pt_BR/index.rst
new file mode 100644
index 0000000000000..8822e21cf4637
--- /dev/null
+++ b/Documentation/translations/pt_BR/index.rst
@@ -0,0 +1,72 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+
+
+=========================================
+Documentação do Kernel Linux em Português
+=========================================
+
+.. raw:: latex
+
+	\kerneldocCJKoff
+
+:mantenedor: Daniel Pereira <danielmaraboo@gmail.com>
+
+Este é o nível principal da documentação do kernel em língua portuguesa (Brasil).
+A tradução ainda está em seu estágio inicial e incompleta; você notará avisos
+sinalizando a falta de traduções para grupos específicos de documentos.
+
+De maneira geral, a documentação, assim como o próprio kernel, está em constante
+desenvolvimento; isso é especialmente verdade agora, pois estamos trabalhando
+na reorganização da documentação de forma mais coerente. Melhorias na
+documentação são sempre bem-vindas; se você deseja ajudar, inscreva-se na lista
+de discussão linux-doc em vger.kernel.org.
+
+
+
+Avisos
+======
+
+.. include:: disclaimer-pt_BR.rst
+
+O objetivo desta tradução é facilitar a leitura e compreensão para aqueles que
+não dominam o inglês ou têm dúvidas sobre sua interpretação, ou simplesmente
+para quem prefere ler em sua língua nativa. No entanto, tenha em mente que a
+*única* documentação oficial é a em língua inglesa: :ref:`linux_doc`
+
+A propagação simultânea de uma alteração em :ref:`linux_doc` para todas as
+traduções é altamente improvável. Os mantenedores das traduções — e seus
+contribuidores — acompanham a evolução da documentação oficial e tentam manter
+as respectivas traduções alinhadas na medida do possível. Por este motivo, não
+há garantia de que uma tradução esteja atualizada com a última modificação.
+Se o que você ler em uma tradução não corresponder ao que ler no código,
+informe o mantenedor da tradução e — se puder — verifique também a
+documentação em inglês.
+
+Uma tradução não é um *fork* da documentação oficial; portanto, os usuários não
+encontrarão nela informações diferentes daquelas contidas na versão oficial.
+Qualquer adição, remoção ou modificação de conteúdo deve ser feita primeiro nos
+documentos em inglês. Posteriormente, quando possível, a mesma alteração deve
+ser aplicada às traduções. Os mantenedores das traduções aceitam contribuições
+que afetem puramente a atividade de tradução (por exemplo, novas traduções,
+atualizações, correções).
+
+As traduções buscam ser o mais precisas possível, mas não é possível mapear
+diretamente uma língua em outra. Cada língua possui sua própria gramática e
+cultura, portanto, a tradução de uma frase em inglês pode ser modificada para
+se adaptar ao português. Por esse motivo, ao ler esta tradução, você poderá
+encontrar algumas diferenças de forma, mas que transmitem a mensagem original.
+
+Trabalhando com a comunidade de desenvolvimento
+===============================================
+
+As guias fundamentais para a interação com a comunidade de desenvolvimento do
+kernel e sobre como ver seu trabalho integrado.
+
+.. toctree::
+   :maxdepth: 1
+
+   Como começar <process/howto>
+   Requisitos mínimos <process/changes>
+   Manuais dos mantenedores <process/maintainer-handbooks>
+   Processo do subsistema de rede (netdev) <process/maintainer-netdev>
diff --git a/Documentation/translations/pt_BR/process/changes.rst b/Documentation/translations/pt_BR/process/changes.rst
new file mode 100644
index 0000000000000..1964c1c93b345
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/changes.rst
@@ -0,0 +1,576 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+
+
+Requisitos mínimos para compilar o Kernel
+++++++++++++++++++++++++++++++++++++++++++
+
+Introdução
+===========
+
+Este documento foi projetado para fornecer uma lista das versões mínimas
+de software necessárias para executar a versão atual do kernel.
+
+Este documento é originalmente baseado no meu arquivo 'Changes' para os kernels
+2.0.x e portanto, deve créditos às mesmas pessoas que aquele arquivo (Jared
+Mauch, Axel Boldt, Alessandro Sigala e inúmeros outros usuários em toda a rede).
+
+Requisitos Mínimos Atuais
+****************************
+
+Atualize para pelo menos estas revisões de software antes de pensar que
+encontrou um bug! Se não tiver certeza de qual versão está executando atualmente
+, o comando sugerido deve lhe informar.
+
+Novamente, tenha em mente que esta lista pressupõe que você já possui um kernel
+Linux em execução funcional. Além disso, nem todas as ferramentas são
+necessárias em todos os sistemas; obviamente, se você não possui nenhum hardware
+PC Card por exemplo, provavelmente não precisará se preocupar com o pcmciautils.
+
+====================== ===============  ========================================
+        Programa        Versão mínima       Comando para verificar a versão
+====================== ===============  ========================================
+GNU C                  8.1              gcc --version
+Clang/LLVM (optional)  15.0.0           clang --version
+Rust (optional)        1.78.0           rustc --version
+bindgen (optional)     0.65.1           bindgen --version
+GNU make               4.0              make --version
+bash                   4.2              bash --version
+binutils               2.30             ld -v
+flex                   2.5.35           flex --version
+bison                  2.0              bison --version
+pahole                 1.16             pahole --version
+util-linux             2.10o            mount --version
+kmod                   13               depmod -V
+e2fsprogs              1.41.4           e2fsck -V
+jfsutils               1.1.3            fsck.jfs -V
+xfsprogs               2.6.0            xfs_db -V
+squashfs-tools         4.0              mksquashfs -version
+btrfs-progs            0.18             btrfs --version
+pcmciautils            004              pccardctl -V
+quota-tools            3.09             quota -V
+PPP                    2.4.0            pppd --version
+nfs-utils              1.0.5            showmount --version
+procps                 3.2.0            ps --version
+udev                   081              udevd --version
+grub                   0.93             grub --version || grub-install --version
+mcelog                 0.6              mcelog --version
+iptables               1.4.2            iptables -V
+openssl & libcrypto    1.0.0            openssl version
+bc                     1.06.95          bc --version
+Sphinx\ [#f1]_         3.4.3            sphinx-build --version
+GNU tar                1.28             tar --version
+gtags (opcional)       6.6.5            gtags --version
+mkimage (opcional)     2017.01          mkimage --version
+Python                 3.9.x            python3 --version
+GNU AWK (opcional)     5.1.0            gawk --version
+====================== ===============  ========================================
+
+.. [#f1] O Sphinx é necessário apenas para gerar a documentação do Kernel.
+
+Compilação do Kernel
+*********************
+
+GCC
+---
+
+Os requisitos da versão do gcc podem variar dependendo do tipo de CPU
+do seu computador.
+
+Clang/LLVM (opcional)
+---------------------
+
+A versão formal mais recente do clang e dos utilitários LLVM (de acordo com
+releases.llvm.org <https://releases.llvm.org>_) é suportada para a compilação
+de kernels. Versões anteriores não têm funcionamento garantido, e poderemos
+remover do kernel soluções de contorno (workarounds) que eram utilizadas para
+suportar versões mais antigas. Por favor, veja a documentação adicional em:
+ref:Building Linux with Clang/LLVM <kbuild_llvm>.
+
+Rust (opcional)
+---------------
+
+É necessária uma versão recente do compilador Rust.
+
+Por favor, consulte Documentation/rust/quick-start.rst para obter instruções
+sobre como atender aos requisitos de compilação do suporte a Rust. Em
+particular, o alvo (target) rustavailable do Makefile é útil para verificar por
+que a cadeia de ferramentas (toolchain) Rust pode não estar sendo detectada.
+
+bindgen (opcional)
+------------------
+
+O ``bindgen`` é utilizado para gerar os vínculos (bindings) Rust para o lado C
+do kernel. Ele depende da ``libclang``.
+
+Make
+----
+
+Você precisará do GNU make 4.0 ou superior para compilar o kernel.
+
+Bash
+----
+
+Alguns scripts bash são usados para a compilação do kernel.
+É necessário o Bash 4.2 ou mais recente.
+
+Binutils
+--------
+
+O binutils 2.30 ou mais recente é necessário para compilar o kernel.
+
+pkg-config
+----------
+
+O sistema de compilação, a partir da versão 4.18, requer o pkg-config para
+verificar as ferramentas kconfig instaladas e para determinar as configurações
+de flags para uso em make {g,x}config. Anteriormente, o pkg-config já era
+utilizado, mas não era verificado nem documentado.
+
+Flex
+----
+
+Desde o Linux 4.16, o sistema de compilação gera analisadores léxicos durante a
+compilação. Isso requer o flex 2.5.35 ou superior.
+
+
+Bison
+-----
+
+Desde o Linux 4.16, o sistema de compilação gera analisadores sintáticos durante
+a compilação. Isso requer o bison 2.0 ou superior
+
+pahole
+------
+
+Desde o Linux 5.2, se CONFIG_DEBUG_INFO_BTF estiver selecionado, o sistema de
+compilação gera BTF (BPF Type Format) a partir do DWARF no vmlinux, e um pouco
+depois para os módulos do kernel também. Isso requer o pahole v1.16 ou superior.
+
+Ele pode ser encontrado nos pacotes ``dwarves`` ou ``pahole`` das
+distribuições, ou em https://fedorapeople.org/~acme/dwarves/.
+
+Perl
+----
+
+Você precisará do perl 5 e dos seguintes módulos: Getopt::Long,
+Getopt::Std, File::Basename e File::Find para compilar o kernel.
+
+Python
+------
+
+Várias opções de configuração o exigem: ele é necessário para as configurações
+padrão (defconfigs) de arm/arm64, CONFIG_LTO_CLANG, algumas configurações
+opcionais de DRM, a ferramenta kernel-doc e a geração da documentação (Sphinx),
+entre outros.
+
+BC
+--
+
+Você precisará do bc para compilar kernels 3.10 ou superior.
+
+
+OpenSSL
+-------
+
+A assinatura de módulos e a manipulação de certificados externos utilizam o
+programa OpenSSL e a biblioteca de criptografia para realizar a criação de
+chaves e a geração de assinaturas.
+
+Você precisará do openssl para compilar kernels 3.7 e superiores se a assinatura
+de módulos estiver habilitada. Você também precisará dos pacotes de
+desenvolvimento do openssl para compilar kernels 4.3 e superiores.
+
+Tar
+---
+
+O GNU tar é necessário caso você deseje habilitar o acesso aos cabeçalhos do
+kernel via sysfs (CONFIG_IKHEADERS).
+
+gtags / GNU GLOBAL (optional)
+-----------------------------
+
+A compilação do kernel requer o GNU GLOBAL versão 6.6.5 ou superior para gerar
+arquivos de tags através de make gtags. Isso se deve ao uso da flag -C
+(--directory) pelo gtags.
+
+mkimage
+-------
+
+Esta ferramenta é utilizada ao gerar uma Flat Image Tree (FIT), comumente usada
+em plataformas ARM. A ferramenta está disponível através do pacote u-boot-tools
+ou pode ser compilada a partir do código-fonte do U-Boot. Veja as instruções em
+https://docs.u-boot.org/en/latest/build/tools.html#building-tools-for-linux
+
+GNU AWK
+-------
+
+O GNU AWK é necessário caso você deseje que a compilação do kernel gere dados de
+intervalo de endereços para
+módulos integrados (CONFIG_BUILTIN_MODULE_RANGES).
+
+Utilitários de sistema
+***********************
+
+Mudanças de arquitetura
+------------------------
+
+O DevFS tornou-se obsoleto em favor do udev
+(https://www.kernel.org/pub/linux/utils/kernel/hotplug/)
+
+O suporte a UIDs de 32 bits já está implementado. Divirta-se!
+
+A documentação das funções do Linux está migrando para a documentação embutida
+(inline), por meio de comentários com formatação especial próximos às suas
+definições no código-fonte. Esses comentários podem ser combinados com arquivos
+ReST no diretório Documentation/ para criar uma documentação enriquecida, que
+pode então ser convertida para arquivos PostScript, HTML, LaTeX, ePUB e PDF.
+Para converter do formato ReST para o formato de sua escolha,você precisará do
+Sphinx.
+
+Util-linux
+----------
+
+Novas versões do util-linux oferecem suporte no fdisk para discos maiores,
+suporte a novas opções para o mount, reconhecimento de mais tipos de partição e
+outras funcionalidades interessantes. Você provavelmente vai querer atualizar.
+
+Ksymoops
+--------
+
+Se o impensável acontecer e o seu kernel sofrer um oops, você pode precisar da
+ferramenta ksymoops para decodificá-lo, mas na maioria dos casos, não será
+necessário. É geralmente preferível compilar o kernel com CONFIG_KALLSYMS para
+que ele produza dumps legíveis que possam ser usados no estado em que se
+encontram (isso também gera uma saída melhor do que a do ksymoops).
+Se por algum motivo o seu kernel não for compilado com CONFIG_KALLSYMS e você
+não tiver como recompilar e reproduzir o oops com essa opção, você ainda poderá
+decodificá-lo com o ksymoops.
+
+Mkinitrd
+--------
+
+Estas mudanças no layout da árvore de arquivos /lib/modules também exigem que o
+mkinitrd seja atualizado.
+
+E2fsprogs
+---------
+
+A versão mais recente do e2fsprogs corrige diversos bugs no fsck e no debugfs.
+Obviamente, é uma boa ideia atualizar.
+
+JFSutils
+--------
+
+O pacote jfsutils contém os utilitários para o sistema de arquivos. Os seguintes
+utilitários estão disponíveis:
+
+- ``fsck.jfs`` - inicia a reprodução (replay) do log de transações, além de
+  verificar e reparar uma partição formatada em JFS.
+
+- ``mkfs.jfs`` - cria uma partição formatada em JFS.
+
+- Para o seu arquivo changes.rst, a tradução técnica adequada é:
+
+Outros utilitários de sistema de arquivos também estão disponíveis neste pacote.
+
+Xfsprogs
+--------
+
+A versão mais recente do ``xfsprogs`` contém os utilitários ``mkfs.xfs``,
+``xfs_db`` e ``xfs_repair``, entre outros, para o sistema de arquivos XFS. Ele é
+independente de arquitetura e qualquer versão a partir da 2.0.0 deve funcionar
+corretamente com esta versão do código do kernel XFS (recomenda-se a
+versão 2.6.0 ou posterior, devido a algumas melhorias significativas).
+
+PCMCIAutils
+-----------
+
+O PCMCIAutils substitui o pcmcia-cs. Ele configura corretamente os sockets
+PCMCIA na inicialização do sistema e carrega os módulos apropriados para
+dispositivos PCMCIA de 16 bits, caso o kernel esteja modularizado e o subsistema
+de hotplug seja utilizado.
+
+Quota-tools
+-----------
+
+O suporte a UIDs e GIDs de 32 bits é necessário caso você deseje utilizar o
+formato de cota versão 2 mais recente. O quota-tools versão 3.07 e superiores
+possuem esse suporte. Utilize a versão recomendada ou superior da tabela acima.
+
+Intel IA32 microcode
+--------------------
+
+Um driver foi adicionado para permitir a atualização do microcódigo Intel IA32,
+acessível como um dispositivo de caracteres comum (misc). Se você não estiver
+usando o udev, você poderá precisar de::
+
+  mkdir /dev/cpu
+  mknod /dev/cpu/microcode c 10 184
+  chmod 0644 /dev/cpu/microcode
+
+Se você não estiver usando o udev, você poderá precisar executar os comandos
+acima como root antes de poder usar isso. Você provavelmente também desejará
+obter o utilitário de espaço de usuário ``microcode_ctl`` para utilizar em
+conjunto com este driver.
+
+udev
+----
+
+O udev é uma aplicação de espaço de usuário para popular o diretório /dev
+dinamicamente, apenas com entradas para dispositivos de fat presentes no
+sistema. O udev substitui a funcionalidade básica do devfs, permitindo ao mesmo
+tempo a nomeação persistente de dispositivos.
+
+FUSE
+----
+
+Necessita do libfuse 2.4.0 ou posterior. O mínimo absoluto é a versão 2.3.0,
+mas as opções de montagem direct_io e kernel_cache não funcionarão.
+
+Redes
+******
+
+Mudanças gerais
+----------------
+
+Caso você tenha necessidades avançadas de configuração de rede, você deve
+provavelmente considerar o uso das ferramentas de rede do iproute2.
+
+Filtro de Pacotes / NAT
+------------------------
+
+O código de filtragem de pacotes e NAT utiliza as mesmas ferramentas da série
+anterior de kernels 2.4.x (iptables). Ele ainda inclui módulos de
+retrocompatibilidade para o ipchains (estilo 2.2.x) e o ipfwadm (estilo 2.0.x).
+
+PPP
+---
+
+O driver PPP foi reestruturado para suportar multilink e permitir que opere
+sobre diversas camadas de mídia. Se você utiliza PPP, atualize o pppd para, no
+mínimo, a versão 2.4.0.
+
+Se você não estiver usando o udev, você deve possuir o arquivo de dispositivo
+``/dev/ppp``, o qual pode ser criado por::
+
+  mknod /dev/ppp c 108 0
+
+como root.
+
+NFS-utils
+---------
+
+Em kernels antigos (2.4 e anteriores), o servidor NFS precisava conhecer
+qualquer cliente que pretendesse acessar arquivos via NFS. Essa informação era
+fornecida ao kernel pelo mountd quando o cliente montava o sistema de arquivos,
+ou pelo exportfs na inicialização do sistema. O exportfs obtinha informações
+sobre clientes ativos a partir de /var/lib/nfs/rmtab.
+
+Esta abordagem é bastante frágil, pois depende da integridade do rmtab, o que
+nem sempre é fácil, particularmente ao tentar implementar fail-over. Mesmo
+quando o sistema está funcionando bem, o rmtab sofre com o acúmulo de muitas
+entradas antigas que nunca são removidas.
+
+Com kernels modernos, temos a opção de fazer o kernel informar ao mountd quando
+recebe uma requisição de um host desconhecido, permitindo que o mountd forneça
+as informações de exportação apropriadas ao kernel. Isso remove a dependência do
+rmtab e significa que o kernel só precisa conhecer os clientes ativos no
+momento.
+
+Para habilitar esta nova funcionalidade, você precisa::
+
+  mount -t nfsd nfsd /proc/fs/nfsd
+
+antes de executar o exportfs ou o mountd. Recomenda-se que todos os serviços NFS
+sejam protegidos da internet em geral por um firewall, sempre que possível.
+
+mcelog
+------
+
+Em kernels x86, o utilitário mcelog é necessário para processar e registrar
+eventos de machine check quando opção CONFIG_X86_MCE está ativada. Eventos de
+machine check são erros relatados pela CPU. O processamento desses eventos é
+fortemente recomendado.
+
+Documentação do Kernel
+***********************
+
+Sphinx
+------
+
+Por favor, consulte Documentation/doc-guide/sphinx.rst para detalhes sobre os
+requisitos do Sphinx.
+
+rustdoc
+-------
+
+O rustdoc é utilizado para gerar a documentação para código Rust. Por favor,
+consulte Documentation/rust/general-information.rst para mais informações.
+
+Obtendo software atualizado
+============================
+
+Compilação do kernel
+**********************
+
+gcc
+---
+
+- <ftp://ftp.gnu.org/gnu/gcc/>
+
+Clang/LLVM
+----------
+
+- :ref:`Getting LLVM <getting_llvm>`.
+
+Rust
+----
+
+- Documentation/rust/quick-start.rst.
+
+bindgen
+-------
+
+- Documentation/rust/quick-start.rst.
+
+Make
+----
+
+- <ftp://ftp.gnu.org/gnu/make/>
+
+Bash
+----
+
+- <ftp://ftp.gnu.org/gnu/bash/>
+
+Binutils
+--------
+
+- <https://www.kernel.org/pub/linux/devel/binutils/>
+
+Flex
+----
+
+- <https://github.com/westes/flex/releases>
+
+Bison
+-----
+
+- <ftp://ftp.gnu.org/gnu/bison/>
+
+OpenSSL
+-------
+
+- <https://www.openssl.org/>
+
+System utilities
+****************
+
+Util-linux
+----------
+
+- <https://www.kernel.org/pub/linux/utils/util-linux/>
+
+Kmod
+----
+
+- <https://www.kernel.org/pub/linux/utils/kernel/kmod/>
+- <https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git>
+
+Ksymoops
+--------
+
+- <https://www.kernel.org/pub/linux/utils/kernel/ksymoops/v2.4/>
+
+Mkinitrd
+--------
+
+- <https://code.launchpad.net/initrd-tools/main>
+
+E2fsprogs
+---------
+
+- <https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/>
+- <https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/>
+
+JFSutils
+--------
+
+- <https://jfs.sourceforge.net/>
+
+Xfsprogs
+--------
+
+- <https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git>
+- <https://www.kernel.org/pub/linux/utils/fs/xfs/xfsprogs/>
+
+Pcmciautils
+-----------
+
+- <https://www.kernel.org/pub/linux/utils/kernel/pcmcia/>
+
+Quota-tools
+-----------
+
+- <https://sourceforge.net/projects/linuxquota/>
+
+
+Intel P6 microcode
+------------------
+
+- <https://downloadcenter.intel.com/>
+
+udev
+----
+
+- <https://www.freedesktop.org/software/systemd/man/udev.html>
+
+FUSE
+----
+
+- <https://github.com/libfuse/libfuse/releases>
+
+mcelog
+------
+
+- <https://www.mcelog.org/>
+
+Redes
+******
+
+PPP
+---
+
+- <https://download.samba.org/pub/ppp/>
+- <https://git.ozlabs.org/?p=ppp.git>
+- <https://github.com/paulusmack/ppp/>
+
+NFS-utils
+---------
+
+- <https://sourceforge.net/project/showfiles.php?group_id=14>
+- <https://nfs.sourceforge.net/>
+
+Iptables
+--------
+
+- <https://netfilter.org/projects/iptables/index.html>
+
+Ip-route2
+---------
+
+- <https://www.kernel.org/pub/linux/utils/net/iproute2/>
+
+OProfile
+--------
+
+- <https://oprofile.sf.net/download/>
+
+Kernel documentation
+********************
+
+Sphinx
+------
+
+- <https://www.sphinx-doc.org/>
diff --git a/Documentation/translations/pt_BR/process/howto.rst b/Documentation/translations/pt_BR/process/howto.rst
new file mode 100644
index 0000000000000..bcedee7273fd8
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/howto.rst
@@ -0,0 +1,637 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_process_howto:
+
+COMO FAZER o desenvolvimento do kernel Linux
+============================================
+
+Este é o documento definitivo sobre este tópico. Ele contém instruções
+sobre como se tornar um desenvolvedor do kernel Linux e como aprender a
+trabalhar com a comunidade de desenvolvimento do kernel Linux. Ele tenta
+não conter nada relacionado aos aspectos técnicos da programação do kernel,
+mas ajudará a apontar a direção certa para isso.
+
+Se algo neste documento ficar desatualizado, por favor, envie patches para
+o mantenedor deste arquivo, que está listado no final do documento.
+
+
+Introdução
+------------
+
+Então, você quer aprender como se tornar um desenvolvedor do kernel Linux?
+Ou o seu gerente lhe disse: "Vá escrever um driver Linux para este
+dispositivo". O objetivo deste documento é ensinar tudo o que você precisa
+saber para conseguir isso, descrevendo o processo pelo qual você deve passar
+e oferecendo dicas sobre como trabalhar com a comunidade. Ele também tentará
+explicar algumas das razões pelas quais a comunidade trabalha da forma que
+trabalha.
+
+O kernel é escrito principalmente em C, com algumas partes dependentes de
+arquitetura escritas em assembly. Um bom entendimento de C é necessário para
+o desenvolvimento do kernel. O conhecimento de Assembly (de qualquer
+arquitetura) não é obrigatório, a menos que você planeje fazer
+desenvolvimento de baixo nível para essa arquitetura específica. Embora não
+sejam um substituto para uma formação sólida em C e/ou anos de experiência,
+os seguintes livros são bons para, no mínimo, referência:
+
+  - "The C Programming Language" por Kernighan e Ritchie [Prentice Hall]
+
+  - "Practical C Programming" por Steve Oualline [O'Reilly]
+
+  - "C: A Reference Manual" por Harbison e Steele [Prentice Hall]
+
+O kernel é escrito usando o GNU C e a GNU toolchain. Embora ele siga o
+padrão ISO C11, ele utiliza uma série de extensões que não estão presentes
+no padrão. O kernel é um ambiente C independente (freestanding), sem
+dependência da biblioteca C padrão (libc), portanto, algumas partes do
+padrão C não são suportadas. Divisões arbitrárias de "long long" e ponto
+flutuante não são permitidas. Às vezes, pode ser difícil entender as
+suposições que o kernel faz sobre a toolchain e as extensões que ele utiliza
+e, infelizmente, não existe uma referência definitiva para elas. Por favor,
+verifique as páginas de informações do gcc (`info gcc`) para obter algumas
+informações sobre elas.
+
+Por favor, lembre-se de que você está tentando aprender como trabalhar com a
+comunidade de desenvolvimento existente. É um grupo diversificado de pessoas,
+com altos padrões de codificação, estilo e procedimento. Esses padrões foram
+criados ao longo do tempo com base no que se descobriu funcionar melhor para
+uma equipe tão grande e geograficamente dispersa. Tente aprender o máximo
+possível sobre esses padrões com antecedência, pois eles estão bem
+documentados; não espere que as pessoas se adaptem a você ou à forma de fazer
+as coisas da sua empresa.
+
+
+Questões Legais
+---------------
+
+O código-fonte do kernel Linux é lançado sob a GPL. Por favor, veja o arquivo
+COPYING no diretório principal da árvore de fontes. As regras de licenciamento
+do kernel Linux e como usar os identificadores `SPDX <https://spdx.org/>`_ no
+código-fonte estão descritas em :ref:`Documentation/process/license-rules.rst <kernel_licensing>`.
+Se você tiver mais perguntas sobre a licença, por favor, entre em contato com
+um advogado e não pergunte na lista de discussão do kernel Linux. As pessoas
+nas listas de discussão não são advogados e você não deve confiar em suas
+declarações sobre assuntos jurídicos.
+
+Para perguntas e respostas comuns sobre a GPL, por favor, veja:
+
+        https://www.gnu.org/licenses/gpl-faq.html
+
+
+Documentação
+------------
+
+A árvore de fontes do kernel Linux possui uma vasta gama de documentos que
+são inestimáveis para aprender como interagir com a comunidade do kernel.
+Quando novos recursos são adicionados ao kernel, recomenda-se que novos
+arquivos de documentação também sejam adicionados explicando como usar o
+recurso. Quando uma mudança no kernel faz com que a interface que o kernel
+expõe para o espaço do usuário (userspace) mude, recomenda-se que você envie
+a informação ou um patch para as páginas de manual explicando a mudança para
+o mantenedor das páginas de manual em alx@kernel.org, e coloque em cópia (CC)
+a lista linux-api@vger.kernel.org.
+
+Aqui está uma lista de arquivos que estão na árvore de fontes do kernel e
+que são de leitura obrigatória:
+
+  :ref:`Documentation/admin-guide/README.rst <readme>`
+    Este arquivo fornece um breve histórico sobre o kernel Linux e descreve
+    o que é necessário fazer para configurar e compilar o kernel. Pessoas
+    que são novas no kernel devem começar por aqui.
+
+  :doc:`changes`
+    Este arquivo fornece uma lista das versões mínimas de vários pacotes de
+    software que são necessários para compilar e executar o kernel com
+    sucesso.
+
+  :ref:`Documentation/process/coding-style.rst <codingstyle>`
+    Este documento descreve o estilo de codificação do kernel Linux e parte
+    da fundamentação por trás dele. Espera-se que todo código novo siga as
+    diretrizes deste documento. A maioria dos mantenedores apenas aceitará
+    patches se essas regras forem seguidas, e muitas pessoas apenas
+    revisarão o código se ele estiver no estilo adequado.
+
+  :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+    Este arquivo descreve em detalhes explícitos como criar e enviar
+    um patch com sucesso, incluindo (mas não limitado a):
+
+        - Conteúdo do e-mail
+        - Formato do e-mail
+        - Para quem enviá-lo
+
+    Seguir estas regras não garantirá o sucesso (já que todos os patches
+    estão sujeitos a um escrutínio de conteúdo e estilo), mas não segui-las
+    quase sempre o impedirá.
+
+Outras excelentes descrições de como criar patches adequadamente são:
+
+	"O Patch Perfeito"
+		https://www.ozlabs.org/~akpm/stuff/tpp.txt
+
+	"Formato de Submissão de Patch do Kernel Linux"
+		https://web.archive.org/web/20180829112450/http://linux.yyz.us/patch-format.html
+
+  :ref:`Documentation/process/stable-api-nonsense.rst <stable_api_nonsense>`
+    Este arquivo descreve a justificativa por trás da decisão consciente de
+    não ter uma API estável dentro do kernel, incluindo pontos como:
+
+      - Camadas de adaptação (shim-layers) de subsistemas (para compatibilidade?)
+      - Portabilidade de drivers entre sistemas operacionais.
+      - Mitigação de mudanças rápidas dentro da árvore de fontes do kernel
+        (ou impedimento de mudanças rápidas).
+
+    Este documento é crucial para compreender a filosofia de desenvolvimento
+    do Linux e é muito importante para pessoas que estão migrando para o
+    Linux vindas do desenvolvimento em outros Sistemas Operacionais.
+
+  :ref:`Documentation/process/security-bugs.rst <securitybugs>`
+    Se você acredita ter encontrado um problema de segurança no kernel Linux,
+    por favor, siga os passos descritos neste documento para ajudar a
+    notificar os desenvolvedores do kernel e auxiliar na resolução do problema.
+
+  :ref:`Documentation/process/management-style.rst <managementstyle>`
+    Este documento descreve como os mantenedores do kernel Linux operam e o
+    ethos compartilhado por trás de suas metodologias. Esta é uma leitura
+    importante para qualquer pessoa nova no desenvolvimento do kernel (ou
+    para qualquer pessoa simplesmente curiosa sobre isso), pois resolve muitos
+    equívocos comuns e confusões sobre o comportamento único dos mantenedores
+    do kernel.
+
+  :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+    Este arquivo descreve as regras sobre como ocorrem os lançamentos das
+    versões estáveis (stable) do kernel e o que fazer se você desejar que
+    uma alteração seja incluída em um desses lançamentos.
+
+  :ref:`Documentation/process/kernel-docs.rst <kernel_docs>`
+    Uma lista de documentação externa que pertence ao desenvolvimento do
+    kernel. Por favor, consulte esta lista caso não encontre o que está
+    procurando dentro da documentação interna do kernel.
+
+  :ref:`Documentation/process/applying-patches.rst <applying_patches>`
+    Uma boa introdução descrevendo exatamente o que é um patch e como
+    aplicá-lo aos diferentes ramos (branches) de desenvolvimento do kernel.
+
+O kernel também possui um grande número de documentos que podem ser
+gerados automaticamente a partir do próprio código-fonte ou de
+marcações ReStructuredText (ReST), como esta. Isso inclui uma
+descrição completa da API interna do kernel e regras sobre como
+manipular o bloqueio (locking) corretamente.
+
+Todos esses documentos podem ser gerados em formato PDF ou HTML ao
+executar::
+
+	make pdfdocs
+	make htmldocs
+
+respectivamente, a partir do diretório principal do código-fonte do kernel.
+
+Os documentos que utilizam a marcação ReST serão gerados em
+Documentation/output. Eles também podem ser gerados nos formatos
+LaTeX e ePub com::
+
+	make latexdocs
+	make epubdocs
+
+Como se tornar um desenvolvedor do kernel
+------------------------------------------
+
+Se você não sabe nada sobre o desenvolvimento do kernel Linux, você deve
+consultar o projeto Linux KernelNewbies:
+
+	https://kernelnewbies.org
+
+Ele consiste em uma lista de discussão útil onde você pode fazer quase
+qualquer tipo de pergunta básica sobre o desenvolvimento do kernel
+(certifique-se de pesquisar nos arquivos primeiro, antes de perguntar
+algo que já foi respondido no passado). Ele também possui um canal de
+IRC que você pode usar para fazer perguntas em tempo real, e muita
+documentação útil para aprender sobre o desenvolvimento do kernel Linux.
+
+O site possui informações básicas sobre a organização do código,
+subsistemas e projetos atuais (tanto in-tree quanto out-of-tree).
+Também descreve algumas informações logísticas básicas, como por exemplo,
+como compilar um kernel e aplicar um patch.
+
+Se você não sabe por onde começar, mas deseja procurar alguma tarefa
+para iniciar sua integração na comunidade de desenvolvimento do kernel,
+acesse o projeto Linux Kernel Janitor:
+
+	https://kernelnewbies.org/KernelJanitors
+
+É um ótimo lugar para começar. Ele descreve uma lista de problemas
+relativamente simples que precisam ser limpos e corrigidos dentro da
+árvore de códigos-fonte do kernel Linux. Ao trabalhar com os
+desenvolvedores responsáveis por este projeto, você aprenderá o básico
+sobre como incluir seu patch na árvore do kernel Linux e,
+possivelmente, será orientado sobre o que trabalhar em seguida, caso
+ainda não tenha uma ideia.
+
+Antes de fazer qualquer modificação real no código do kernel Linux, é
+imperativo entender como o código em questão funciona. Para esse
+propósito, nada é melhor do que lê-lo diretamente (a maioria das partes
+complexas está bem comentada), talvez até com a ajuda de ferramentas
+especializadas. Uma ferramenta particularmente recomendada é o projeto
+Linux Cross-Reference, que é capaz de apresentar o código-fonte em um
+formato de página web indexada e auto-referenciada. Um excelente
+repositório atualizado do código do kernel pode ser encontrado em:
+
+    https://elixir.bootlin.com/
+
+
+O processo de desenvolvimento
+-----------------------------
+
+O processo de desenvolvimento do kernel Linux consiste atualmente em algumas
+"branches" (ramos) principais diferentes e muitos ramos de subsistemas
+específicos. Esses diferentes ramos são:
+
+  - Árvore principal (mainline) do Linus
+  - Várias árvores estáveis com múltiplos números de versão principal
+  - Árvores específicas de subsistemas
+  - Árvore de testes de integração linux-next
+
+Árvore principal (Mainline tree)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A árvore principal é mantida por Linus Torvalds e pode ser encontrada em
+https://kernel.org ou no repositório. Seu processo de desenvolvimento é
+o seguinte:
+
+  - Assim que um novo kernel é lançado, uma janela de duas semanas é aberta;
+    durante esse período, os mantenedores podem enviar grandes diffs para
+    Linus, geralmente patches que já foram incluídos na linux-next por algumas
+    semanas. A forma preferida de enviar grandes mudanças é usando o git
+    (a ferramenta de gerenciamento de código-fonte do kernel, mais informações
+    podem ser encontradas em https://git-scm.com/), mas patches simples
+    também são aceitos.
+  - Após duas semanas, um kernel -rc1 é lançado e o foco passa a ser tornar
+    o novo kernel o mais sólido possível. A maioria dos patches neste estágio
+    deve corrigir uma regressão. Bugs que sempre existiram não são regressões,
+    portanto, envie esses tipos de correções apenas se forem importantes.
+    Observe que um driver (ou sistema de arquivos) totalmente novo pode ser
+    aceito após o -rc1 porque não há risco de causar regressões com tal
+    mudança, desde que a alteração seja autocontida e não afete áreas fora do
+    código que está sendo adicionado. O git pode ser usado para enviar
+    patches para Linus após o lançamento do -rc1, mas os patches também
+    precisam ser enviados para uma lista de discussão pública para revisão.
+  - Um novo -rc é lançado sempre que Linus considerar que a árvore git atual
+    está em um estado razoavelmente estável e adequado para testes. O objetivo
+    é lançar um novo kernel -rc a cada semana.
+  - O processo continua até que o kernel seja considerado "pronto"; o
+    processo deve durar cerca de 6 semanas.
+
+Vale a pena mencionar o que Andrew Morton escreveu na lista de discussão
+do kernel Linux sobre os lançamentos do kernel:
+
+    *"Ninguém sabe quando um kernel será lançado, porque ele é
+    lançado de acordo com o status percebido dos bugs, não de acordo
+    com um cronograma pré-concebido."*
+
+Várias árvores estáveis com múltiplos números de versão principal
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Kernels com versões de 3 partes são kernels -stable (estáveis). Eles
+contêm correções relativamente pequenas e críticas para problemas de
+segurança ou regressões significativas descobertas em um determinado
+lançamento principal da árvore mainline. Cada lançamento em uma série
+estável principal incrementa a terceira parte do número da versão,
+mantendo as duas primeiras partes iguais.
+
+Este é o ramo recomendado para usuários que desejam o kernel estável
+mais recente e não estão interessados em ajudar a testar versões de
+desenvolvimento ou experimentais.
+
+As árvores estáveis são mantidas pela equipe "stable"
+<stable@vger.kernel.org> e são lançadas conforme a necessidade exigir.
+O período normal de lançamento é de aproximadamente duas semanas, mas
+pode ser mais longo se não houver problemas urgentes. Por outro lado,
+um problema relacionado à segurança pode fazer com que um lançamento
+ocorra quase instantaneamente.
+
+O arquivo :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+na árvore do kernel documenta quais tipos de mudanças são aceitáveis para
+a árvore -stable e como o processo de lançamento funciona.
+
+Árvores específicas de subsistemas
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Os mantenedores dos vários subsistemas do kernel — e também muitos
+desenvolvedores de subsistemas do kernel — expõem seu estado atual de
+desenvolvimento em repositórios de código-fonte. Dessa forma, outros
+podem ver o que está acontecendo nas diferentes áreas do kernel. Em
+áreas onde o desenvolvimento é rápido, um desenvolvedor pode ser
+solicitado a basear suas submissões em tal árvore de subsistema do
+kernel para que conflitos entre a submissão e outros trabalhos já em
+andamento sejam evitados.
+
+A maioria desses repositórios são árvores git, mas também existem outros
+SCMs em uso, ou filas de patches sendo publicadas como séries quilt. Os
+endereços desses repositórios de subsistemas estão listados no arquivo
+MAINTAINERS. Muitos deles podem ser navegados em https://git.kernel.org/.
+
+Antes que um patch proposto seja incluído em tal árvore de subsistema,
+ele está sujeito a uma revisão que ocorre principalmente em listas de
+discussão (veja a seção respectiva abaixo). Para vários subsistemas do
+kernel, este processo de revisão é rastreado com a ferramenta patchwork.
+O Patchwork oferece uma interface web que mostra as postagens de patches,
+quaisquer comentários sobre um patch ou revisões feitas a ele, e os
+mantenedores podem marcar os patches como "sob revisão", "aceitos" ou
+"rejeitados". A maioria desses sites patchwork está listada em
+https://patchwork.kernel.org/.
+
+Árvore de testes de integração linux-next
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Antes que as atualizações das árvores de subsistemas sejam mescladas na
+árvore mainline, elas precisam ser testadas quanto à integração. Para
+este propósito, existe um repositório de testes especial no qual
+praticamente todas as árvores de subsistemas são integradas (pulled)
+quase diariamente:
+
+    https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
+
+Dessa forma, a linux-next oferece uma visão resumida do que se espera
+que entre no kernel mainline no próximo período de mesclagem (merge
+window). Testadores aventureiros são muito bem-vindos para testar a
+linux-next em tempo de execução.
+
+
+Relato de Bugs
+--------------
+
+O arquivo 'Documentation/admin-guide/reporting-issues.rst' no diretório
+principal de códigos-fonte do kernel descreve como relatar um possível
+bug no kernel e detalha que tipo de informação é necessária para os
+desenvolvedores do kernel ajudarem a rastrear o problema.
+
+Gerenciando relatos de bugs
+---------------------------
+
+Uma das melhores maneiras de colocar em prática suas habilidades de hacking
+é corrigindo bugs relatados por outras pessoas. Você não apenas ajudará a
+tornar o kernel mais estável, mas também aprenderá a resolver problemas do
+mundo real, melhorará suas habilidades e outros desenvolvedores passarão a
+notar sua presença. Corrigir bugs é uma das melhores formas de obter mérito
+entre outros desenvolvedores, pois poucas pessoas gostam de gastar tempo
+corrigindo bugs de terceiros.
+
+Para trabalhar em relatos de bugs já existentes, encontre um subsistema no
+qual você esteja interessado. Verifique no arquivo MAINTAINERS para onde
+os bugs daquele subsistema são relatados; geralmente será uma lista de
+discussão, raramente um rastreador de bugs (bugtracker). Pesquise nos
+arquivos de mensagens do local indicado por relatos recentes e ajude onde
+achar apropriado. Você também pode verificar o site
+https://bugzilla.kernel.org para relatos de bugs; apenas alguns
+subsistemas do kernel o utilizam ativamente para relato ou rastreamento,
+entretanto, bugs de todo o kernel acabam sendo registrados lá.
+
+
+Listas de discussão
+-------------------
+
+Como alguns dos documentos acima descrevem, a maioria dos desenvolvedores
+do núcleo (core) do kernel participa da Linux Kernel Mailing List (LKML).
+Detalhes sobre como se inscrever e cancelar a inscrição na lista podem
+ser encontrados em:
+
+    https://subspace.kernel.org/subscribing.html
+
+Existem arquivos de mensagens da lista na web em muitos lugares diferentes.
+Use um mecanismo de busca para encontrar esses arquivos. Por exemplo:
+
+    https://lore.kernel.org/linux-kernel/
+
+É altamente recomendável que você pesquise nos arquivos sobre o tópico que
+deseja abordar antes de postar na lista. Muitas coisas já discutidas em
+detalhes estão registradas apenas nos arquivos das listas de discussão.
+
+A maioria dos subsistemas individuais do kernel também possui sua própria
+lista de discussão separada, onde realizam seus esforços de desenvolvimento.
+Consulte o arquivo MAINTAINERS para obter uma lista de quais são essas
+listas para os diferentes grupos.
+
+Muitas das listas estão hospedadas no kernel.org. Informações sobre elas
+podem ser encontradas em:
+
+    https://subspace.kernel.org
+
+Por favor, lembre-se de seguir bons hábitos de comportamento ao usar as
+listas. Embora um pouco clichê, a URL a seguir possui algumas diretrizes
+simples para interagir com a lista (ou qualquer outra lista):
+
+    https://subspace.kernel.org/etiquette.html
+
+Se várias pessoas responderem ao seu e-mail, a lista de destinatários em
+CC: pode se tornar bem grande. Não remova ninguém da lista CC: sem um
+bom motivo, e não responda apenas para o endereço da lista. Acostume-se
+a receber o e-mail duas vezes (um do remetente e outro da lista) e não
+tente ajustar isso adicionando cabeçalhos de e-mail complexos; as pessoas
+não gostarão disso.
+
+Lembre-se de manter o contexto e a atribuição de suas respostas intactos;
+mantenha as linhas do tipo "John Kernelhacker escreveu...:" no topo da
+sua resposta e adicione seus comentários entre as seções citadas
+individualmente, em vez de escrever tudo no topo do e-mail.
+
+Se você adicionar patches ao seu e-mail, certifique-se de que sejam texto
+puro legível, conforme declarado em
+:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+Os desenvolvedores do kernel não querem lidar com anexos ou patches
+compactados; eles podem querer comentar linhas individuais do seu patch,
+o que só funciona dessa forma. Certifique-se de usar um programa de
+e-mail que não altere espaços e caracteres de tabulação (tabs). Um bom
+primeiro teste é enviar o e-mail para si mesmo e tentar aplicar o seu
+próprio patch. Se isso não funcionar, conserte seu programa de e-mail ou
+troque-o até que funcione.
+
+Acima de tudo, por favor, lembre-se de mostrar respeito aos outros
+inscritos.
+
+
+Trabalhando com a comunidade
+----------------------------
+
+O objetivo da comunidade do kernel é fornecer o melhor kernel possível.
+Quando você envia um patch para aceitação, ele será revisado por seus
+méritos técnicos e apenas por eles. Então, o que você deve esperar?
+
+  - críticas
+  - comentários
+  - solicitações de mudança
+  - solicitações de justificativa
+  - silêncio
+
+Lembre-se, isso faz parte do processo de incluir seu patch no kernel.
+Você deve ser capaz de aceitar críticas e comentários sobre seus patches,
+avaliá-los em nível técnico e retrabalhar seus patches ou fornecer
+raciocínios claros e concisos sobre o porquê de certas mudanças não
+deverem ser feitas. Se não houver respostas à sua postagem, aguarde
+alguns dias e tente novamente; às vezes, as coisas se perdem no enorme
+volume de mensagens.
+
+O que você não deve fazer?
+
+  - esperar que seu patch seja aceito sem questionamentos
+  - tornar-se defensivo
+  - ignorar comentários
+  - reenviar o patch sem fazer nenhuma das alterações solicitadas
+
+Em uma comunidade que busca a melhor solução técnica possível, sempre
+haverá opiniões divergentes sobre o quão benéfico é um patch. Você deve
+ser cooperativo e estar disposto a adaptar sua ideia para que ela se
+encaixe no kernel. Ou, pelo menos, estar disposto a provar que sua ideia
+vale a pena. Lembre-se: estar errado é aceitável, desde que você esteja
+disposto a trabalhar em direção a uma solução correta.
+
+É normal que as respostas ao seu primeiro patch sejam apenas uma lista
+de uma dúzia de coisas que você deve corrigir. Isso **não** implica que
+seu patch não será aceito e **não** é algo pessoal contra você. Simplesmente
+corrija todos os problemas apontados em seu patch e envie-o novamente.
+
+
+Diferenças entre a comunidade do kernel e estruturas corporativas
+-----------------------------------------------------------------
+
+A comunidade do kernel trabalha de forma diferente da maioria dos ambientes
+tradicionais de desenvolvimento corporativo. Aqui está uma lista de coisas
+que você pode tentar fazer para evitar problemas:
+
+  Boas coisas a dizer em relação às suas mudanças propostas:
+
+    - "Isto resolve múltiplos problemas."
+    - "Isto remove 2000 linhas de código."
+    - "Aqui está um patch que explica o que estou tentando descrever."
+    - "Eu testei isso em 5 arquiteturas diferentes..."
+    - "Aqui está uma série de pequenos patches que..."
+    - "Isto aumenta a performance em máquinas comuns..."
+
+  Coisas ruins que você deve evitar dizer:
+
+    - "Nós fizemos desta forma no AIX/ptx/Solaris, portanto deve ser bom..."
+    - "Eu faço isso há 20 anos, então..."
+    - "Isto é necessário para minha empresa ganhar dinheiro."
+    - "Isto é para nossa linha de produtos Enterprise."
+    - "Aqui está meu documento de design de 1000 páginas que descreve minha ideia."
+    - "Estou trabalhando nisso há 6 meses..."
+    - "Aqui está um patch de 5000 linhas que..."
+    - "Eu reescrevi toda a bagunça atual, e aqui está..."
+    - "Eu tenho um prazo (deadline), e este patch precisa ser aplicado agora."
+
+Outra forma em que a comunidade do kernel difere da maioria dos ambientes
+tradicionais de engenharia de software é a natureza anônima da interação.
+Um benefício de usar e-mail e IRC como as principais formas de comunicação
+é a ausência de discriminação baseada em gênero ou raça. O ambiente de
+trabalho do kernel Linux aceita mulheres e minorias porque tudo o que você
+é, é um endereço de e-mail. O aspecto internacional também ajuda a nivelar
+o campo de jogo porque você não pode adivinhar o gênero com base no nome
+de uma pessoa. Um homem pode se chamar Andrea e uma mulher pode se chamar
+Pat. A maioria das mulheres que trabalharam no kernel Linux e expressaram
+uma opinião tiveram experiências positivas.
+
+A barreira do idioma pode causar problemas para algumas pessoas que não
+se sentem confortáveis com o inglês. Um bom domínio do idioma pode ser
+necessário para transmitir ideias adequadamente nas listas de discussão,
+por isso recomenda-se que você verifique seus e-mails para garantir que
+façam sentido em inglês antes de enviá-los.
+
+
+Divida suas alterações
+----------------------
+
+A comunidade do kernel Linux não aceita de bom grado grandes blocos de
+código jogados de uma só vez. As mudanças precisam ser devidamente
+introduzidas, discutidas e divididas em porções minúsculas e individuais.
+Isso é quase o exato oposto do que as empresas costumam fazer. Sua proposta
+também deve ser introduzida muito cedo no processo de desenvolvimento, para
+que você possa receber feedback sobre o que está fazendo. Isso também permite
+que a comunidade sinta que você está trabalhando com eles, e não simplesmente
+usando-os como um depósito para sua funcionalidade. No entanto, não envie
+50 e-mails de uma só vez para uma lista de discussão; sua série de patches
+deve ser menor que isso quase sempre.
+
+As razões para dividir as coisas são as seguintes:
+
+1) Patches pequenos aumentam a probabilidade de serem aplicados, pois não
+   exigem muito tempo ou esforço para verificar sua correção. Um patch de
+   5 linhas pode ser aplicado por um mantenedor com apenas um olhar rápido.
+   No entanto, um patch de 500 linhas pode levar horas para ser revisado
+   (o tempo que leva é exponencialmente proporcional ao tamanho do patch,
+   ou algo assim).
+
+   Patches pequenos também tornam muito fácil a depuração (debug) quando
+   algo dá errado. É muito mais fácil reverter patches um por um do que
+   dissecar um patch muito grande após ele ter sido aplicado (e quebrado algo).
+
+2) É importante não apenas enviar patches pequenos, mas também reescrever
+   e simplificar (ou simplesmente reordenar) os patches antes de submetê-los.
+
+Aqui está uma analogia do desenvolvedor do kernel Al Viro:
+
+    *"Pense em um professor corrigindo o dever de casa de um aluno de
+    matemática. O professor não quer ver as tentativas e erros do aluno
+    antes de chegar à solução. Ele quer ver a resposta mais limpa e
+    elegante. Um bom aluno sabe disso e nunca enviaria seu trabalho
+    intermediário antes da solução final.*
+
+    *O mesmo vale para o desenvolvimento do kernel. Os mantenedores e
+    revisores não querem ver o processo de pensamento por trás da solução
+    do problema que se está resolvendo. Eles querem ver uma solução
+    simples e elegante."*
+
+Pode ser desafiador manter o equilíbrio entre apresentar uma solução
+elegante e trabalhar em conjunto com a comunidade discutindo seu trabalho
+inacabado. Portanto, é bom entrar no processo cedo para obter feedback e
+melhorar seu trabalho, mas também manter suas alterações em pequenos blocos
+que possam ser aceitos, mesmo quando sua tarefa completa ainda não esteja
+pronta para inclusão.
+
+Também entenda que não é aceitável enviar patches para inclusão que estejam
+inacabados e que serão "consertados mais tarde".
+
+
+Justifique sua alteração
+------------------------
+
+Além de dividir seus patches, é muito importante que você deixe a comunidade
+Linux saber por que eles deveriam adicionar essa mudança. Novas
+funcionalidades devem ser justificadas como necessárias e úteis.
+
+
+Documente sua alteração
+-----------------------
+
+Ao enviar seus patches, preste atenção especial ao que você diz no texto
+do seu e-mail. Essas informações se tornarão as informações do ChangeLog
+para o patch e serão preservadas para que todos vejam para sempre. Elas
+devem descrever o patch completamente, contendo:
+
+  - por que a mudança é necessária
+  - a abordagem geral de design no patch
+  - detalhes de implementação
+  - resultados de testes
+
+Para mais detalhes sobre como tudo isso deve ser, por favor, veja a seção
+ChangeLog do documento:
+
+  "O Patch Perfeito"
+      https://www.ozlabs.org/~akpm/stuff/tpp.txt
+
+Todas essas coisas às vezes são muito difíceis de fazer. Pode levar anos
+para aperfeiçoar essas práticas (se é que é possível). É um processo
+contínuo de melhoria que exige muita paciência e determinação. Mas não
+desista, é possível. Muitos fizeram isso antes, e cada um teve que começar
+exatamente onde você está agora.
+
+----------
+
+Agradecimentos a Paolo Ciarrocchi, que permitiu que a seção "Processo de
+Desenvolvimento" (https://lwn.net/Articles/94386/) fosse baseada em um
+texto que ele escreveu, e a Randy Dunlap e Gerrit Huizenga por parte da
+lista de coisas que você deve ou não dizer. Também agradecemos a Pat Mochel,
+Hanna Linder, Randy Dunlap, Kay Sievers, Vojtech Pavlik, Jan Kara,
+Josh Boyer, Kees Cook, Andrew Morton, Andi Kleen, Vadim Lobanov, Jesper Juhl,
+Adrian Bunk, Keri Harris, Frans Pop, David A. Wheeler, Junio Hamano,
+Michael Kerrisk e Alex Shepard por suas revisões, comentários e contribuições.
+Sem a ajuda deles, este documento não teria sido possível.
+
+Mantenedor: Greg Kroah-Hartman <greg@kroah.com>
diff --git a/Documentation/translations/pt_BR/process/maintainer-handbooks.rst b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
new file mode 100644
index 0000000000000..20bb32490303b
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
@@ -0,0 +1,17 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Notas sobre o processo de desenvolvimento de subsistemas e mantenedores
+=======================================================================
+
+O propósito deste documento é fornecer informações específicas de
+subsistemas que são suplementares ao manual geral do processo de
+desenvolvimento.
+
+Conteúdos:
+
+.. toctree::
+   :numbered:
+   :maxdepth: 2
+
+   maintainer-netdev
+
diff --git a/Documentation/translations/pt_BR/process/maintainer-netdev.rst b/Documentation/translations/pt_BR/process/maintainer-netdev.rst
new file mode 100644
index 0000000000000..5de2828041b98
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/maintainer-netdev.rst
@@ -0,0 +1,596 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+Subsistema de Rede do Linux (netdev)
+=====================================
+
+tl;dr
+-----
+
+- **Direcione seu patch para uma árvore** – use ``[PATCH net]``para correções
+  ou ``[PATCH net-next]`` para novas funcionalidades.
+- **Tag Fixes** – para correções, a tag ``Fixes:`` é obrigatória,
+  independentemente da árvore de destino.
+- **Tamanho da série** – não envie séries grandes (> 15 patches);divida-as em
+  partes menores.
+- **Intervalo de envio** – não reenvie seus patches dentro de um período de 24
+  horas.
+- **Reverse xmas tree** – organize as declarações de variáveis locais da mais
+  longa para a mais curta.
+
+netdev
+------
+A **netdev** é a lista de discussão para todos os assuntos do Linux relacionados
+a rede. Isso inclui qualquer item encontrado em ``net/`` (ex: código principal
+como IPv6) e  em ``drivers/net`` (ex: drivers específicos de hardware) na árvore
+de diretórios do Linux.
+
+Note que alguns subsistemas (ex: drivers de rede sem fio/wireless), que possuem
+um  alto volume de tráfego, possuem suas próprias listas de discussão e árvores
+específicas.
+
+Como muitas outras listas de discussão do Linux, a lista netdev é hospedada no
+`kernel.org <https://www.kernel.org/>`_, com arquivos disponíveis em
+https://lore.kernel.org/netdev/.
+
+À exceção dos subsistemas mencionados anteriormente, todo o desenvolvimento de
+rede  do Linux (ex: RFCs, revisões, comentários, etc.) ocorre na **netdev**.
+
+Ciclo de Desenvolvimento
+------------------------
+
+Aqui está um pouco de informação contextual sobre a cadência de desenvolvimento
+do Linux. Cada nova versão (release) inicia-se com uma "janela de mesclagem"
+(*merge window*) de duas semanas, onde os mantenedores principais enviam suas
+novas implementações para o Linus para incorporação na árvore principal
+(*mainline tree*).
+
+Após as duas semanas, a janela de mesclagem é fechada e a versão é
+nomeada/etiquetada  como ``-rc1``. Nenhuma funcionalidade nova é incorporada à
+árvore principal após  isso -- espera-se apenas correções (*fixes*) para o
+conteúdo da rc1.
+
+Após cerca de uma semana coletando correções para a rc1, a rc2 é lançada. Isso
+se  repete semanalmente até a rc7 (tipicamente; às vezes rc6 se o ritmo estiver
+calmo, ou rc8 se houver muita instabilidade); uma semana após a última vX.Y-rcN
+ser  concluída, a versão oficial vX.Y é lançada.
+
+Para descobrir em que ponto do ciclo estamos agora - carregue a página da
+mainline (Linus) aqui:
+
+  https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
+
+e observe o topo da seção de "tags". Se for rc1, estamos no início do ciclo
+de desenvolvimento. Se a rc7 foi marcada há uma semana, então um lançamento
+é provavelmente iminente. Se a tag mais recente for uma tag de lançamento
+final (sem o sufixo ``-rcN``) - muito provavelmente estamos em uma janela de
+mesclagem (*merge window*) e o ``net-next`` está fechado.
+
+Árvores git e fluxo de patches
+------------------------------
+
+Existem duas árvores de rede (repositórios git) em jogo. Ambas são coordenadas
+por David Miller, o mantenedor principal de rede. Há a árvore ``net``e a árvore
+``net-next``. Como você provavelmente pode adivinhar pelos nomes, a árvore
+``net`` é para correções de código existente já na árvore mainline de Linus, e a
+``net-next`` é para onde o novo código vai para o lançamento futuro.
+Você pode encontrar as árvores aqui:
+
+- https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git
+- https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git
+
+Relacionando isso ao desenvolvimento do kernel: no início da janela de mesclagem
+(*merge window*) de 2 semanas, a árvore ``net-next`` será fechada, sem novas
+mudanças ou funcionalidades. O conteúdo novo acumulado nas últimas 10 semanas
+será passado para a mainline/Linus via um *pull request* para a vX.Y ao mesmo
+tempo, a árvore ``net`` começará a acumular correções para este conteúdo enviado
+relacionado à vX.Y.
+
+Um anúncio indicando quando a ``net-next`` foi fechada é geralmente enviado para
+a netdev, mas sabendo o que foi dito acima, você pode prever isso com
+antecedência.
+
+.. warning::
+
+  Não envie novo conteúdo para a ``net-next`` para a netdev durante o período
+  em que a árvore ``net-next`` estiver fechada.
+
+Patches RFC enviados apenas para revisão são obviamente bem-vindos a qualquer
+momento (use ``--subject-prefix='RFC net-next'`` com ``git format-patch``).
+
+Pouco depois das duas semanas terem passado (e a vX.Y-rc1 ser lançada), a árvore
+para a ``net-next`` reabre para coletar conteúdo para o próximo lançamento
+(vX.Y+1).
+
+Se você não estiver inscrito na netdev e/ou simplesmente não tiver certeza se a
+``net-next`` já reabriu, basta verificar o link do repositório git da
+``net-next`` acima para quaisquer novos *commits* relacionados à rede. Você
+também pode verificar o seguinte site para o status atual:
+
+  https://netdev.bots.linux.dev/net-next.html
+
+A árvore ``net`` continua a coletar correções para o conteúdo da vX.Y e é
+enviada de volta para Linus em intervalos regulares (~semanais). Isso significa
+que o foco da ``net`` é a estabilização e correções de bugs.
+
+Finalmente, a vX.Y é lançada e todo o ciclo recomeça.
+
+Revisão de patches da netdev
+----------------------------
+
+Status do patch
+~~~~~~~~~~~~~~~
+
+O status de um patch pode ser verificado olhando a fila principal do patchwork
+para a netdev:
+
+  https://patchwork.kernel.org/project/netdevbpf/list/
+
+O campo "State" informará exatamente onde as coisas estão com o seu patch:
+
+=================  ============================================================
+Estado do patch    Descrição
+=================  ============================================================
+New, Under review  revisão pendente, o patch está na fila do mantenedor
+                   para revisão; os dois estados são usados alternadamente
+                   (dependendo do co-mantenedor exato que estiver lidando
+                   com o patchwork no momento)
+Accepted           o patch foi aplicado à árvore de rede apropriada,
+                   isso é geralmente definido de forma automática pelo pw-bot
+Needs ACK          aguardando um "ack" de um especialista da área
+                   ou testes
+Changes requested  o patch não passou na revisão, espera-se uma nova
+                   revisão com mudanças apropriadas no código e na mensagem
+                   de commit
+Rejected           o patch foi rejeitado e não se espera uma nova
+                   revisão
+Not applicable     espera-se que o patch seja aplicado fora do
+                   subsistema de rede
+Awaiting upstream  o patch deve ser revisado e tratado pelo sub-mantenedor
+                   apropriado, que o enviará para as árvores de rede;
+                   patches definidos como ``Awaiting upstream`` no patchwork
+                   da netdev geralmente permanecerão neste estado,
+                   independentemente de o sub-mantenedor ter solicitado
+                   mudanças, aceito ou rejeitado o patch
+Deferred           o patch precisa ser reenviado mais tarde, geralmente
+                   devido a alguma dependência ou porque foi enviado para
+                   uma árvore fechada
+Superseded         uma nova versão do patch foi enviada, geralmente
+                   definido pelo pw-bot
+RFC                não deve ser aplicado, geralmente não está na
+                   fila de revisão do mantenedor; o pw-bot pode definir
+                   patches para este estado automaticamente com base nas
+                   tags do assunto
+=================  ============================================================
+
+Os patches são indexados pelo cabeçalho ``Message-ID`` dos e-mails que os
+transportaram; portanto, se você tiver problemas para encontrar seu patch,
+anexe o valor do ``Message-ID`` à URL acima.
+
+Atualizando o status do patch
+-----------------------------
+
+Colaboradores e revisores não têm permissões para atualizar o estado do patch
+diretamente no patchwork. O Patchwork não expõe muitas informações sobre o
+histórico do estado dos patches; portanto, ter várias pessoas atualizando o
+estado leva a confusões.
+
+Em vez de delegar permissões do patchwork, a netdev usa um robô de e-mail
+simples (bot) que procura por comandos/linhas especiais dentro dos e-mails
+enviados para a lista de discussão. Por exemplo, para marcar uma série como
+Mudanças Solicitadas (*Changes Requested*), é necessário enviar a seguinte
+linha em qualquer lugar na thread do e-mail::
+
+  pw-bot: changes-requested
+
+Como resultado, o bot definirá toda a série como Mudanças Solicitadas. Isso
+pode ser útil quando o autor descobre um bug em sua própria série e deseja
+evitar que ela seja aplicada.
+
+O uso do bot é totalmente opcional; em caso de dúvida, ignore completamente a
+existência dele. Os mantenedores classificarão e atualizarão o estado dos
+patches por conta própria. Nenhum e-mail deve ser enviado à lista com o
+propósito principal de se comunicar com o bot; os comandos do bot devem ser
+vistos como metadados.
+
+O uso do bot é restrito aos autores dos patches (o cabeçalho ``From:`` no envio
+do patch e no comando deve coincidir!), mantenedores do código modificado de
+acordo com o arquivo MAINTAINERS (novamente, o ``From:`` deve coincidir
+com a entrada no MAINTAINERS) e alguns revisores seniores.
+
+O bot registra sua atividade aqui:
+
+  https://netdev.bots.linux.dev/pw-bot.html
+
+Prazos de revisão
+~~~~~~~~~~~~~~~~~
+
+De modo geral, os patches são triados rapidamente (em menos de 48h). Mas
+seja paciente; se o seu patch estiver ativo no patchwork (ou seja, listado
+na lista de patches do projeto), as chances de ele ter sido esquecido são
+próximas de zero.
+
+O alto volume de desenvolvimento na netdev faz com que os revisores encerrem
+discussões de forma relativamente rápida. É muito improvável que novos
+comentários e respostas cheguem após uma semana de silêncio. Se um
+patch não estiver mais ativo no patchwork e a thread ficar inativa por mais de
+uma semana - esclareça os próximos passos e/ou envie a próxima versão.
+
+Especificamente para envios de RFC, se ninguém responder em uma semana  ou os
+revisores perderam o envio ou não têm opiniões fortes a respeito. Se o código
+estiver pronto, reenvie como um PATCH.
+
+E-mails dizendo apenas "ping" ou "bump" são considerados rudes. Se você não
+conseguir identificar o status do patch pelo patchwork ou onde a discussão
+parou - descreva sua melhor suposição e pergunte se ela está correta. Por
+exemplo::
+
+  Não entendo quais são os próximos passos. A Pessoa X parece estar  descontente
+  com A; devo fazer B e enviar novamente os patches?
+
+.. _Solicitações de mudanças:
+
+Mudanças solicitadas
+~~~~~~~~~~~~~~~~~~~~
+
+Patches marcados como ``Changes Requested`` precisam ser revisados. A nova
+versão deve vir com um registro de alterações (changelog),
+preferencialmente incluindo links para as postagens anteriores, por exemplo::
+
+  [PATCH net-next v3] net: faz as vacas dizerem "muuu"
+
+  Mesmo os usuários que não bebem leite apreciam ouvir as vacas dizendo
+  "muuu".
+
+  A quantidade de mugidos dependerá da taxa de pacotes, portanto, deve
+  corresponder muito bem ao ciclo diurno.
+
+  Signed-off-by: Joe Defarmer <joe@barn.org>
+  ---
+  v3:
+    - adicionada uma nota sobre a flutuação do mugido conforme a hora
+      do dia na
+      mensagem de commit
+  v2: https://lore.kernel.org/netdev/123themessageid@barn.org/
+    - corrigido argumento ausente na kernel doc para netif_is_bovine()
+    - corrigido vazamento de memória (memory leak) em
+    netdev_register_cow()
+  v1: https://lore.kernel.org/netdev/456getstheclicks@barn.org/
+
+A mensagem de commit deve ser revisada para responder a quaisquer perguntas que
+os revisores tenham feito em discussões anteriores. Ocasionalmente, a
+atualização da mensagem de commit será a única mudança na nova versão.
+
+Reenvios parciais
+~~~~~~~~~~~~~~~~~
+
+Por favor, sempre reenvie a série completa de patches e certifique-se de
+numerar seus patches de forma que fique claro que este é o conjunto mais
+recente e completo de patches que pode ser aplicado. Não tente reenviar apenas
+os patches que foram alterados.
+
+Lidando com patches aplicados incorretamente
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ocasionalmente, uma série de patches é aplicada antes de receber feedback
+crítico, ou a versão errada de uma série é aplicada.
+
+Não é possível fazer o patch desaparecer uma vez que ele foi enviado (pushed);
+o histórico de commits nas árvores netdev é imutável. Por favor, envie versões
+incrementais sobre o que foi mesclado para corrigir os patches da maneira que
+eles ficariam se a sua série de patches mais recente fosse mesclada.
+
+Em casos onde uma reversão completa (revert) é necessária, a reversão deve ser
+enviada como um patch para a lista com uma mensagem de commit explicando os
+problemas técnicos com o commit revertido. Reversões devem ser usadas como
+último recurso, quando a mudança original está completamente errada; correções
+incrementais são preferidas.
+
+Árvore estável
+~~~~~~~~~~~~~~
+
+Embora antigamente as submissões para a netdev não devessem carregar tags
+explícitas ``CC: stable@vger.kernel.org``, esse não é mais o caso hoje em dia.
+Por favor, siga as regras padrão de estabilidade em
+``Documentation/process/stable-kernel-rules.rst``, e certifique-se de incluir as
+tags Fixes apropriadas!
+
+Correções de segurança
+~~~~~~~~~~~~~~~~~~~~~~
+
+Não envie e-mails diretamente para os mantenedores da netdev se você acha que
+descobriu um bug que possa ter possíveis implicações de segurança. O atual
+mantenedor da netdev tem solicitado consistentemente que as pessoas usem as
+listas de discussão e não entrem em contato diretamente. Se você não estiver
+de acordo com isso, considere enviar um e-mail para security@kernel.org ou
+ler sobre http://oss-security.openwall.org/wiki/mailing-lists/distros como
+possíveis mecanismos alternativos.
+
+Envio conjunto de mudanças em componentes de espaço do usuário
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+O código de espaço do usuário (*user space*) que exercita funcionalidades do
+kernel deve ser enviado juntamente com os patches do kernel. Isso dá aos
+revisores a chance de ver como qualquer nova interface é usada e quão
+bem ela funciona.
+
+Quando as ferramentas de espaço do usuário residem no próprio repositório do
+kernel, todas as alterações devem geralmente vir em uma única série. Se a série
+se tornar muito grande ou se o projeto de espaço do usuário não for revisado na
+netdev, inclua um link para um repositório público onde os patches de espaço do
+usuário possam ser vistos.
+
+No caso de ferramentas de espaço do usuário residirem em um repositório
+separado, mas serem revisadas na netdev (por exemplo, patches para ferramentas
+``iproute2``), os patches do kernel e do espaço do usuário devem formar séries
+(threads) separadas quando postados na lista de discussão, por exemplo::
+
+  [PATCH net-next 0/3] net: carta de apresentação de alguma funcionalidade
+   └─ [PATCH net-next 1/3] net: preparação para alguma funcionalidade
+   └─ [PATCH net-next 2/3] net: implementação de alguma funcionalidade
+   └─ [PATCH net-next 3/3] selftest: net: alguma funcionalidade
+
+  [PATCH iproute2-next] ip: adiciona suporte para alguma funcionalidade
+
+A postagem em uma única thread é desencorajada porque confunde o patchwork
+(a partir da versão 2.2.2 do patchwork).
+
+Envio conjunto de selftests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Os selftests devem fazer parte da mesma série que as mudanças de código.
+Especificamente para correções, tanto a mudança de código quanto o teste
+relacionado devem ir para a mesma árvore (os testes podem não ter uma tag
+Fixes, o que é esperado). Misturar mudanças de código e mudanças de teste em
+um único commit é desencorajado.
+
+Preparando as mudanças
+----------------------
+
+Atenção aos detalhes é importante. Releia seu próprio trabalho como se você
+fosse o revisor. Você pode começar usando o ``checkpatch.pl``, talvez até com
+a flag ``--strict``. Mas não seja robótico e irracional ao fazer isso. Se sua
+mudança for uma correção de bug, certifique-se de que seu log de commit indique
+o sintoma visível para o usuário final, a razão subjacente de por que isso
+acontece e, se necessário, explique por que a correção proposta é a melhor
+maneira de resolver as coisas. Não corrompa espaços em branco e, como é comum,
+não use recuos incorretos em argumentos de função que abrangem várias linhas.
+Se for o seu primeiro patch, envie-o para si mesmo por e-mail para que você
+possa testar a aplicação em uma árvore sem patches para confirmar que a
+infraestrutura não o danificou.
+
+Finalmente, volte e leia ``Documentation/process/submitting-patches.rst``
+para ter certeza de que não está repetindo algum erro comum documentado lá.
+
+Indicando a árvore de destino
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Para ajudar os mantenedores e os bots de CI, você deve marcar explicitamente
+qual árvore seu patch tem como alvo. Supondo que você use git, utilize a flag
+de prefixo::
+
+  git format-patch --subject-prefix='PATCH net-next' inicio..fim
+
+Use ``net`` em vez de ``net-next`` (sempre em letras minúsculas) no comando
+acima para conteúdos de correção de bugs da árvore ``net``.
+
+Dividindo o trabalho em patches
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Coloque-se no lugar do revisor. Cada patch é lido separadamente e, portanto,
+deve constituir um passo compreensível em direção ao seu objetivo declarado.
+
+Evite enviar séries com mais de 15 patches. Séries maiores levam mais tempo
+para serem revisadas, pois os revisores adiarão a análise até encontrarem um
+grande bloco de tempo disponível. Uma série pequena pode ser revisada em pouco
+tempo, então os mantenedores simplesmente a revisam de imediato. Como resultado,
+uma sequência de séries menores é mesclada mais rapidamente e com melhor
+cobertura de revisão. Reenviar séries grandes também aumenta o tráfego na lista
+de discussão.
+
+Limitar patches pendentes na lista de discussão
+-----------------------------------------------
+
+Evite ter mais de 15 patches, em todas as séries, pendentes de revisão na lista
+de discussão para uma única árvore. Em outras palavras, um máximo de 15 patches
+sob revisão na ``net`` e um máximo de 15 patches sob revisão na ``net-next``.
+
+Este limite tem o objetivo de focar o esforço do desenvolvedor nos testes dos
+patches antes da revisão upstream, auxiliando a qualidade das submissões
+upstream e aliviando a carga sobre os revisores.
+
+Ordenação de variáveis locais ("árvore invertida", "RCS")
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A netdev tem uma convenção para ordenar variáveis locais em funções. Ordene as
+linhas de declaração de variáveis da mais longa para a mais curta, por exemplo::
+
+  struct scatterlist *sg;
+  struct sk_buff *skb;
+  int err, i;
+
+Se houver dependências entre as variáveis que impeçam a ordenação, mova a
+inicialização para fora da linha de declaração.
+
+Precedência de formatação
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ao trabalhar em código existente que utiliza formatação não padrão, faça com
+que seu código siga as diretrizes mais recentes, para que, eventualmente,
+todo o código no domínio da netdev esteja no formato preferido.
+
+Uso de construções gerenciadas por dispositivo e cleanup.h
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Historicamente, a netdev permanece cética em relação às promessas de todas as
+APIs de "auto-limpeza" (auto-cleanup), incluindo até mesmo os auxiliares
+``devm_``. Eles não são o estilo preferido de implementação, apenas um estilo
+aceitável.
+
+O uso de ``guard()`` é desencorajado em qualquer função com mais de 20 linhas;
+``scoped_guard()`` é considerado mais legível. O uso de lock/unlock normal
+ainda é (levemente) preferido.
+
+Construções de limpeza de baixo nível (como ``__free()``) podem ser usadas ao
+construir APIs e auxiliares, especialmente iteradores com escopo. No entanto, o
+uso direto de ``__free()`` dentro do núcleo de rede (networking core) e drivers
+é desencorajado. Orientações semelhantes se aplicam à declaração de variáveis
+no meio da função.
+
+Patches de limpeza (Clean-up patches)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A netdev desencoraja patches que realizam limpezas simples que não estejam no
+contexto de outro trabalho. Por exemplo:
+
+* Tratar avisos do ``checkpatch.pl`` e outros avisos triviais de estilo de
+  codificação
+* Tratar problemas de Ordenação de variáveis locais
+* Conversões para APIs gerenciadas por dispositivo (auxiliares ``devm_``)
+
+Isso ocorre porque se considera que a agitação (*churn*) que tais mudanças
+produzem tem um custo maior do que o valor de tais limpezas.
+
+Por outro lado, correções de ortografia e gramática não são desencorajadas.
+
+Reenviando após a revisão
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Aguarde pelo menos 24 horas entre as postagens. Isso garantirá que revisores de
+todas as localizações geográficas tenham a chance de se manifestar. Não espere
+muito tempo (semanas) entre as postagens, pois isso tornará mais difícil para
+os revisores lembrarem de todo o contexto.
+
+Certifique-se de tratar todo o feedback em sua nova postagem. Não envie uma
+nova versão do código se a discussão sobre a versão anterior ainda estiver em
+andamento, a menos que seja instruído diretamente por um revisor.
+
+A nova versão dos patches deve ser postada como uma thread separada, não como
+uma resposta à postagem anterior. O registro de alterações (changelog) deve
+incluir um link para a postagem anterior (veja :ref:`Solicitações
+de mudanças`).
+
+Testes
+------
+
+Nível de teste esperado
+~~~~~~~~~~~~~~~~~~~~~~~
+
+No mínimo, suas alterações devem passar por uma compilação ``allyesconfig`` e
+uma ``allmodconfig`` com ``W=1`` definido, sem novos avisos ou falhas.
+
+O ideal é que você tenha feito testes em tempo de execução específicos para sua
+alteração, e que a série de patches contenha um conjunto de selftests do kernel
+para ``tools/testing/selftests/net`` ou usando o framework KUnit.
+
+Espera-se que você teste suas alterações no topo da árvore de rede relevante
+(``net`` ou ``net-next``) e não, por exemplo, em uma árvore estável ou na
+``linux-next``.
+
+Verificações do patchwork
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As verificações (*checks*) no patchwork são, em sua maioria, wrappers simples
+em torno de scripts existentes do kernel; as fontes estão disponíveis em:
+
+https://github.com/linux-netdev/nipa/tree/master/tests
+
+**Não** envie seus patches apenas para executá-los nas verificações. Você deve
+garantir que seus patches estejam prontos, testando-os localmente antes de
+postar na lista de discussão. A instância do bot de build do patchwork fica
+sobrecarregada com muita facilidade e a netdev@vger realmente não precisa de
+mais tráfego se pudermos evitar.
+
+netdevsim
+~~~~~~~~~
+
+O ``netdevsim`` é um driver de teste que pode ser usado para exercitar APIs de
+configuração de driver sem a necessidade de hardware compatível. Mock-ups e
+testes baseados no ``netdevsim`` são fortemente encorajados ao adicionar novas
+APIs, mas o ``netdevsim`` em si **não** é considerado um caso de uso/usuário.
+Você também deve implementar as novas APIs em um driver real.
+
+Não damos garantias de que o ``netdevsim`` mudará no futuro de uma forma que
+quebraria o que normalmente seria considerado uAPI.  O ``netdevsim`` é reservado
+apenas para uso por testes upstream, portanto, quaisquer novos recursos do
+``netdevsim`` devem ser acompanhados de selftests em ``tools/testing/selftests/``.
+
+Status de suporte para drivers
+------------------------------
+
+.. note:
+
+Os requisitos a seguir aplicam-se apenas a drivers de NIC Ethernet.
+
+A netdev define requisitos adicionais para drivers que desejam adquirir o status
+``Supported`` (Suportado) no arquivo MAINTAINERS. Drivers ``Supported`` devem
+executar todos os testes de driver upstream e relatar os resultados duas vezes
+por dia. Drivers que não cumprirem este requisito devem usar o status
+``Maintained`` (Mantido). Atualmente, não há diferença na forma como os drivers
+``Supported`` e ``Maintained`` são tratados no upstream.
+
+As regras exatas que um driver deve seguir para adquirir o status ``Supported``:
+
+1. Deve executar todos os testes sob os alvos ``drivers/net`` e
+   ``drivers/net/hw`` dos selftests do Linux. A execução e o relato
+   de testes privados / internos também são bem-vindos, mas os testes
+   upstream são obrigatórios.
+
+2. A frequência mínima de execução é uma vez a cada 12 horas. Deve
+   testar o branch designado a partir do feed de branches selecionado.
+   Observe que os branches são construídos automaticamente e estão
+   expostos à postagem intencional de patches maliciosos; portanto,
+   os sistemas de teste devem ser isolados.
+
+3. Drivers que suportam múltiplas gerações de dispositivos devem
+   testar pelo menos um dispositivo de cada geração. Um manifesto do
+   ambiente de teste (*testbed manifest* - formato exato a definir)
+   deve descrever os modelos de dispositivos testados.
+
+4. Os testes devem ser executados de forma confiável; se múltiplos
+   branches forem ignorados ou se os testes falharem devido a problemas
+   no ambiente de execução, o status ``Supported`` será retirado.
+
+5. Falhas nos testes devido a bugs no driver ou no próprio teste,
+   ou falta de suporte para a funcionalidade que o teste visa, *não*
+   são motivo para a perda do status ``Supported``.
+
+O CI da netdev manterá uma página oficial de dispositivos suportados, listando
+seus resultados de testes recentes.
+
+O mantenedor do driver pode providenciar para que outra pessoa execute o teste;
+não há exigência de que a pessoa listada como mantenedora (ou seu empregador)
+seja responsável pela execução dos testes. Colaborações entre
+fornecedores, hospedagem de CI no GitHub (GH CI), outros repositórios sob o
+linux-netdev, etc., são muito bem-vindas.
+
+Veja https://github.com/linux-netdev/nipa/wiki para mais informações sobre o CI
+da netdev. Sinta-se à vontade para entrar em contato com os mantenedores ou com
+a lista para quaisquer dúvidas.
+
+Orientações para revisores
+--------------------------
+
+Revisar patches de outras pessoas na lista é altamente incentivado,
+independentemente do nível de experiência. Para orientações gerais e dicas
+úteis, consulte `revisão de tópicos avançados de desenvolvimento`.
+
+É seguro assumir que os mantenedores da netdev conhecem a comunidade e o nível
+de experiência dos revisores. Os revisores não devem se preocupar com o fato de
+seus comentários impedirem ou desviarem o fluxo de patches. Revisores menos
+experientes são fortemente incentivados a fazer uma revisão mais aprofundada das
+submissões e não focar exclusivamente em questões triviais ou subjetivas, como
+formatação de código, tags, etc.
+
+Depoimentos / feedback
+----------------------
+
+Algumas empresas utilizam o feedback de colegas em revisões de desempenho de
+funcionários. Sinta-se à vontade para solicitar feedback dos mantenedores da
+netdev, especialmente se você dedica uma quantidade significativa de tempo
+revisando código e se esforça além do esperado para melhorar a infraestrutura
+compartilhada.
+
+O feedback deve ser solicitado por você, o colaborador, e será sempre
+compartilhado com você (mesmo que você solicite que ele seja enviado ao seu
+gerente).
\ No newline at end of file
diff --git a/Documentation/translations/sp_SP/process/submitting-patches.rst b/Documentation/translations/sp_SP/process/submitting-patches.rst
index ecb08b14c2c02..fc90c942f0e42 100644
--- a/Documentation/translations/sp_SP/process/submitting-patches.rst
+++ b/Documentation/translations/sp_SP/process/submitting-patches.rst
@@ -30,7 +30,7 @@ más sencilla.
 
 Algunos subsistemas y árboles de mantenimiento cuentan con información
 adicional sobre su flujo de trabajo y expectativas, consulte
-:ref:`Documentation/process/maintainer-handbooks.rst <maintainer_handbooks_main>`.
+Documentation/process/maintainer-handbooks.rst.
 
 Obtenga el código fuente actual
 --------------------------------
diff --git a/Documentation/translations/zh_CN/devicetree/of_unittest.rst b/Documentation/translations/zh_CN/devicetree/of_unittest.rst
index 5c1a8e0cfd160..cfd0b751ef273 100644
--- a/Documentation/translations/zh_CN/devicetree/of_unittest.rst
+++ b/Documentation/translations/zh_CN/devicetree/of_unittest.rst
@@ -32,27 +32,30 @@ OF Selftest被设计用来测试提供给设备驱动开发者的接口（includ
 2. 测试数据
 ===========
 
-设备树源文件（drivers/of/unittest-data/testcases.dts）包含执行drivers/of/unittest.c
-中自动化单元测试所需的测试数据。目前，以下设备树源包含文件（.dtsi）被包含在testcases.dt中::
+设备树源文件（drivers/of/unittest-data/testcases.dtso）包含执行drivers/of/unittest.c
+中自动化单元测试所需的测试数据。目前，以下设备树源包含文件（.dtsi）被包含在testcases.dtso中::
 
     drivers/of/unittest-data/tests-interrupts.dtsi
     drivers/of/unittest-data/tests-platform.dtsi
     drivers/of/unittest-data/tests-phandle.dtsi
     drivers/of/unittest-data/tests-match.dtsi
+    drivers/of/unittest-data/tests-address.dtsi
+    drivers/of/unittest-data/tests-overlay.dtsi
+    drivers/of/unittest-data/tests-lifecycle.dtsi
 
 当内核在启用CONFIG_OF_UNITTEST的情况下被构建时，那么下面的make规则::
 
-    $(obj)/%.dtb: $(src)/%.dts FORCE
-	    $(call if_changed_dep, dtc)
+    $(obj)/%.dtbo: $(src)/%.dtso $(DTC) FORCE
+	    $(call if_changed_dep,dtc)
 
-用于将DT源文件（testcases.dts）编译成二进制blob（testcases.dtb），也被称为扁平化的DT。
+用于将DT源文件（testcases.dtso）编译成二进制blob（testcases.dtbo），也被称为扁平化的DT。
 
-之后，使用以下规则将上述二进制blob包装成一个汇编文件（testcases.dtb.S）::
+之后，使用以下规则将上述二进制blob包装成一个汇编文件（testcases.dtbo.S）::
 
-    $(obj)/%.dtb.S: $(obj)/%.dtb
-	    $(call cmd, dt_S_dtb)
+    $(obj)/%.dtbo.S: $(obj)/%.dtbo FORCE
+	    $(call if_changed,wrap_S_dtb)
 
-汇编文件被编译成一个对象文件（testcases.dtb.o），并被链接到内核镜像中。
+汇编文件被编译成一个对象文件（testcases.dtbo.o），并被链接到内核镜像中。
 
 
 2.1. 添加测试数据
diff --git a/Documentation/translations/zh_CN/infiniband/index.rst b/Documentation/translations/zh_CN/infiniband/index.rst
index 5634cc48379f9..aeeea0b49939e 100644
--- a/Documentation/translations/zh_CN/infiniband/index.rst
+++ b/Documentation/translations/zh_CN/infiniband/index.rst
@@ -24,7 +24,6 @@ infiniband
 
    core_locking
    ipoib
-   opa_vnic
    sysfs
    tag_matching
    user_mad
diff --git a/Documentation/translations/zh_CN/infiniband/opa_vnic.rst b/Documentation/translations/zh_CN/infiniband/opa_vnic.rst
deleted file mode 100644
index 12b147fbf7926..0000000000000
--- a/Documentation/translations/zh_CN/infiniband/opa_vnic.rst
+++ /dev/null
@@ -1,156 +0,0 @@
-.. include:: ../disclaimer-zh_CN.rst
-
-:Original: Documentation/infiniband/opa_vnic.rst
-
-:翻译:
-
- 司延腾 Yanteng Si <siyanteng@loongson.cn>
-
-:校译:
-
- 王普宇 Puyu Wang <realpuyuwang@gmail.com>
- 时奎亮 Alex Shi <alexs@kernel.org>
-
-.. _cn_infiniband_opa_vnic:
-
-=============================================
-英特尔全路径（OPA）虚拟网络接口控制器（VNIC）
-=============================================
-
-英特尔全路径（OPA）虚拟网络接口控制器（VNIC）功能通过封装HFI节点之间的以
-太网数据包，支持Omni-Path结构上的以太网功能。
-
-体系结构
-========
-
-Omni-Path封装的以太网数据包的交换模式涉及Omni-Path结构拓扑上覆盖的一个或
-多个虚拟以太网交换机。Omni-Path结构上的HFI节点的一个子集被允许在特定的虚
-拟以太网交换机上交换封装的以太网数据包。虚拟以太网交换机是通过配置结构上的
-HFI节点实现的逻辑抽象，用于生成和处理报头。在最简单的配置中，整个结构的所有
-HFI节点通过一个虚拟以太网交换机交换封装的以太网数据包。一个虚拟以太网交换机，
-实际上是一个独立的以太网网络。该配置由以太网管理器（EM）执行，它是可信的结
-构管理器（FM）应用程序的一部分。HFI节点可以有多个VNIC，每个连接到不同的虚
-拟以太网交换机。下图介绍了两个虚拟以太网交换机与两个HFI节点的情况::
-
-                               +-------------------+
-                               |      子网/        |
-                               |     以太网        |
-                               |      管理         |
-                               +-------------------+
-                                  /          /
-                                /           /
-                              /            /
-                            /             /
-  +-----------------------------+  +------------------------------+
-  |     虚拟以太网切换          |  |      虚拟以太网切换          |
-  |  +---------+    +---------+ |  | +---------+    +---------+   |
-  |  | VPORT   |    |  VPORT  | |  | |  VPORT  |    |  VPORT  |   |
-  +--+---------+----+---------+-+  +-+---------+----+---------+---+
-           |                 \        /                 |
-           |                   \    /                   |
-           |                     \/                     |
-           |                    /  \                    |
-           |                  /      \                  |
-       +-----------+------------+  +-----------+------------+
-       |   VNIC    |    VNIC    |  |    VNIC   |    VNIC    |
-       +-----------+------------+  +-----------+------------+
-       |          HFI           |  |          HFI           |
-       +------------------------+  +------------------------+
-
-
-Omni-Path封装的以太网数据包格式如下所述。
-
-==================== ================================
-位                   域
-==================== ================================
-Quad Word 0:
-0-19                 SLID (低20位)
-20-30                长度 (以四字为单位)
-31                   BECN 位
-32-51                DLID (低20位)
-52-56                SC (服务级别)
-57-59                RC (路由控制)
-60                   FECN 位
-61-62                L2 (=10, 16B 格式)
-63                   LT (=1, 链路传输头 Flit)
-
-Quad Word 1:
-0-7                  L4 type (=0x78 ETHERNET)
-8-11                 SLID[23:20]
-12-15                DLID[23:20]
-16-31                PKEY
-32-47                熵
-48-63                保留
-
-Quad Word 2:
-0-15                 保留
-16-31                L4 头
-32-63                以太网数据包
-
-Quad Words 3 to N-1:
-0-63                 以太网数据包 (pad拓展)
-
-Quad Word N (last):
-0-23                 以太网数据包 (pad拓展)
-24-55                ICRC
-56-61                尾
-62-63                LT (=01, 链路传输尾 Flit)
-==================== ================================
-
-以太网数据包在传输端被填充，以确保VNIC OPA数据包是四字对齐的。“尾”字段
-包含填充的字节数。在接收端，“尾”字段被读取，在将数据包向上传递到网络堆
-栈之前，填充物被移除（与ICRC、尾和OPA头一起）。
-
-L4头字段包含VNIC端口所属的虚拟以太网交换机ID。在接收端，该字段用于将收
-到的VNIC数据包去多路复用到不同的VNIC端口。
-
-驱动设计
-========
-
-英特尔OPA VNIC的软件设计如下图所示。OPA VNIC功能有一个依赖于硬件的部分
-和一个独立于硬件的部分。
-
-对IB设备分配和释放RDMA netdev设备的支持已经被加入。RDMA netdev支持与
-网络堆栈的对接，从而创建标准的网络接口。OPA_VNIC是一个RDMA netdev设备
-类型。
-
-依赖于HW的VNIC功能是HFI1驱动的一部分。它实现了分配和释放OPA_VNIC RDMA
-netdev的动作。它涉及VNIC功能的HW资源分配/管理。它与网络堆栈接口并实现所
-需的net_device_ops功能。它在传输路径中期待Omni-Path封装的以太网数据包，
-并提供对它们的HW访问。在将数据包向上传递到网络堆栈之前，它把Omni-Path头
-从接收的数据包中剥离。它还实现了RDMA netdev控制操作。
-
-OPA VNIC模块实现了独立于硬件的VNIC功能。它由两部分组成。VNIC以太网管理
-代理（VEMA）作为一个IB客户端向IB核心注册，并与IB MAD栈接口。它与以太网
-管理器（EM）和VNIC netdev交换管理信息。VNIC netdev部分分配和释放OPA_VNIC
-RDMA netdev设备。它在需要时覆盖由依赖HW的VNIC驱动设置的net_device_ops函数，
-以适应任何控制操作。它还处理以太网数据包的封装，在传输路径中使用Omni-Path头。
-对于每个VNIC接口，封装所需的信息是由EM通过VEMA MAD接口配置的。它还通过调用
-RDMA netdev控制操作将任何控制信息传递给依赖于HW的驱动程序::
-
-        +-------------------+ +----------------------+
-        |                   | |       Linux          |
-        |     IB MAD        | |       网络           |
-        |                   | |       栈             |
-        +-------------------+ +----------------------+
-                 |               |          |
-                 |               |          |
-        +----------------------------+      |
-        |                            |      |
-        |      OPA VNIC 模块         |      |
-        |  (OPA VNIC RDMA Netdev     |      |
-        |     & EMA 函数)            |      |
-        |                            |      |
-        +----------------------------+      |
-                    |                       |
-                    |                       |
-           +------------------+             |
-           |     IB 核心      |             |
-           +------------------+             |
-                    |                       |
-                    |                       |
-        +--------------------------------------------+
-        |                                            |
-        |      HFI1 驱动和 VNIC 支持                 |
-        |                                            |
-        +--------------------------------------------+
diff --git a/Documentation/translations/zh_TW/admin-guide/README.rst b/Documentation/translations/zh_TW/admin-guide/README.rst
index c8b7ccfaa656d..9be4915ae420e 100644
--- a/Documentation/translations/zh_TW/admin-guide/README.rst
+++ b/Documentation/translations/zh_TW/admin-guide/README.rst
@@ -33,7 +33,7 @@ Linux內核6.x版本 <http://kernel.org/>
 
   雖然Linux最初是爲32位的x86 PC機（386或更高版本）開發的，但今天它也能運行在
   （至少）Compaq Alpha AXP、Sun SPARC與UltraSPARC、Motorola 68000、PowerPC、
-  PowerPC64、ARM、Hitachi SuperH、Cell、IBM S/390、MIPS、HP PA-RISC、Intel 
+  PowerPC64、ARM、Hitachi SuperH、Cell、IBM S/390、MIPS、HP PA-RISC、Intel
   IA-64、DEC VAX、AMD x86-64 Xtensa和ARC架構上。
 
   Linux很容易移植到大多數通用的32位或64位體系架構，只要它們有一個分頁內存管理
diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst
index 13134bccdd39d..7f86d7a37dc23 100644
--- a/Documentation/userspace-api/landlock.rst
+++ b/Documentation/userspace-api/landlock.rst
@@ -8,7 +8,7 @@ Landlock: unprivileged access control
 =====================================
 
 :Author: Mickaël Salaün
-:Date: January 2026
+:Date: March 2026
 
 The goal of Landlock is to enable restriction of ambient rights (e.g. global
 filesystem or network access) for a set of processes.  Because Landlock
@@ -197,12 +197,27 @@ similar backwards compatibility check is needed for the restrict flags
 
 .. code-block:: c
 
-    __u32 restrict_flags = LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON;
-    if (abi < 7) {
-        /* Clear logging flags unsupported before ABI 7. */
+    __u32 restrict_flags =
+        LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON |
+        LANDLOCK_RESTRICT_SELF_TSYNC;
+    switch (abi) {
+    case 1 ... 6:
+        /* Removes logging flags for ABI < 7 */
         restrict_flags &= ~(LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF |
                             LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON |
                             LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF);
+        __attribute__((fallthrough));
+    case 7:
+        /*
+         * Removes multithreaded enforcement flag for ABI < 8
+         *
+         * WARNING: Without this flag, calling landlock_restrict_self(2) is
+         * only equivalent if the calling process is single-threaded. Below
+         * ABI v8 (and as of ABI v8, when not using this flag), a Landlock
+         * policy would only be enforced for the calling thread and its
+         * children (and not for all threads, including parents and siblings).
+         */
+        restrict_flags &= ~LANDLOCK_RESTRICT_SELF_TSYNC;
     }
 
 The next step is to restrict the current thread from gaining more privileges
diff --git a/Documentation/userspace-api/media/dvb/legacy_dvb_audio.rst b/Documentation/userspace-api/media/dvb/legacy_dvb_audio.rst
index 81b762ef17c4c..99ffda355204d 100644
--- a/Documentation/userspace-api/media/dvb/legacy_dvb_audio.rst
+++ b/Documentation/userspace-api/media/dvb/legacy_dvb_audio.rst
@@ -444,7 +444,7 @@ Description
 ~~~~~~~~~~~
 
 A call to `AUDIO_GET_CAPABILITIES`_ returns an unsigned integer with the
-following bits set according to the hardwares capabilities.
+following bits set according to the hardware's capabilities.
 
 
 -----
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 032516783e962..03d87d9b97d94 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -907,10 +907,12 @@ The irq_type field has the following values:
 - KVM_ARM_IRQ_TYPE_CPU:
 	       out-of-kernel GIC: irq_id 0 is IRQ, irq_id 1 is FIQ
 - KVM_ARM_IRQ_TYPE_SPI:
-	       in-kernel GIC: SPI, irq_id between 32 and 1019 (incl.)
+	       in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.)
                (the vcpu_index field is ignored)
+	       in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.)
 - KVM_ARM_IRQ_TYPE_PPI:
-	       in-kernel GIC: PPI, irq_id between 16 and 31 (incl.)
+	       in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.)
+	       in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.)
 
 (The irq_id field thus corresponds nicely to the IRQ ID in the ARM GIC specs)
 
diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst
new file mode 100644
index 0000000000000..29335ea823fc5
--- /dev/null
+++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst
@@ -0,0 +1,50 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+====================================================
+ARM Virtual Generic Interrupt Controller v5 (VGICv5)
+====================================================
+
+
+Device types supported:
+  - KVM_DEV_TYPE_ARM_VGIC_V5     ARM Generic Interrupt Controller v5.0
+
+Only one VGIC instance may be instantiated through this API.  The created VGIC
+will act as the VM interrupt controller, requiring emulated user-space devices
+to inject interrupts to the VGIC instead of directly to CPUs.
+
+Creating a guest GICv5 device requires a host GICv5 host.  The current VGICv5
+device only supports PPI interrupts.  These can either be injected from emulated
+in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE
+ioctl.
+
+Groups:
+  KVM_DEV_ARM_VGIC_GRP_CTRL
+   Attributes:
+
+    KVM_DEV_ARM_VGIC_CTRL_INIT
+      request the initialization of the VGIC, no additional parameter in
+      kvm_device_attr.addr. Must be called after all VCPUs have been created.
+
+   KVM_DEV_ARM_VGIC_USERPSPACE_PPIs
+      request the mask of userspace-drivable PPIs. Only a subset of the PPIs can
+      be directly driven from userspace with GICv5, and the returned mask
+      informs userspace of which it is allowed to drive via KVM_IRQ_LINE.
+
+      Userspace must allocate and point to __u64[2] of data in
+      kvm_device_attr.addr. When this call returns, the provided memory will be
+      populated with the userspace PPI mask. The lower __u64 contains the mask
+      for the lower 64 PPIS, with the remaining 64 being in the second __u64.
+
+      This is a read-only attribute, and cannot be set. Attempts to set it are
+      rejected.
+
+  Errors:
+
+    =======  ========================================================
+    -ENXIO   VGIC not properly configured as required prior to calling
+             this attribute
+    -ENODEV  no online VCPU
+    -ENOMEM  memory shortage when allocating vgic internal data
+    -EFAULT  Invalid guest ram access
+    -EBUSY   One or more VCPUS are running
+    =======  ========================================================
diff --git a/Documentation/virt/kvm/devices/index.rst b/Documentation/virt/kvm/devices/index.rst
index 192cda7405c84..70845aba38f45 100644
--- a/Documentation/virt/kvm/devices/index.rst
+++ b/Documentation/virt/kvm/devices/index.rst
@@ -10,6 +10,7 @@ Devices
    arm-vgic-its
    arm-vgic
    arm-vgic-v3
+   arm-vgic-v5
    mpic
    s390_flic
    vcpu
diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
index 60bf205cb3730..5e38058200105 100644
--- a/Documentation/virt/kvm/devices/vcpu.rst
+++ b/Documentation/virt/kvm/devices/vcpu.rst
@@ -37,7 +37,8 @@ Returns:
 A value describing the PMUv3 (Performance Monitor Unit v3) overflow interrupt
 number for this vcpu. This interrupt could be a PPI or SPI, but the interrupt
 type must be same for each vcpu. As a PPI, the interrupt number is the same for
-all vcpus, while as an SPI it must be a separate number per vcpu.
+all vcpus, while as an SPI it must be a separate number per vcpu. For
+GICv5-based guests, the architected PPI (23) must be used.
 
 1.2 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_INIT
 ---------------------------------------
@@ -50,7 +51,7 @@ Returns:
 	 -EEXIST  Interrupt number already used
 	 -ENODEV  PMUv3 not supported or GIC not initialized
 	 -ENXIO   PMUv3 not supported, missing VCPU feature or interrupt
-		  number not set
+		  number not set (non-GICv5 guests, only)
 	 -EBUSY   PMUv3 already initialized
 	 =======  ======================================================
 
diff --git a/MAINTAINERS b/MAINTAINERS
index 96ea84948d76a..a626dee5c1064 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -35,6 +35,7 @@ Descriptions of section entries and preferred order
 	   F:	drivers/net/	all files in and below drivers/net
 	   F:	drivers/net/*	all files in drivers/net, but not below
 	   F:	*/net/*		all files in "any top level directory"/net
+	   F:	fs/**/*foo*.c	all *foo*.c files in any subdirectory of fs
 	   One pattern per line.  Multiple F: lines acceptable.
 	X: *Excluded* files and directories that are NOT maintained, same
 	   rules as F:. Files exclusions are tested before file matches.
@@ -1591,6 +1592,15 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/adc/adi,ad7780.yaml
 F:	drivers/iio/adc/ad7780.c
 
+ANALOG DEVICES INC AD8366 DRIVER
+M:	Michael Hennerich <Michael.Hennerich@analog.com>
+M:	Rodrigo Alencar <rodrigo.alencar@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/amplifiers/adi,ad8366.yaml
+F:	drivers/iio/amplifiers/ad8366.c
+
 ANALOG DEVICES INC AD9467 DRIVER
 M:	Michael Hennerich <Michael.Hennerich@analog.com>
 M:	Nuno Sa <nuno.sa@analog.com>
@@ -1699,6 +1709,14 @@ S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/imu/adi,adis16550.yaml
 
+ANALOG DEVICES INC ADL8113 DRIVER
+M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/amplifiers/adi,adl8113.yaml
+F:	drivers/iio/amplifiers/adl8113.c
+
 ANALOG DEVICES INC ADM1177 DRIVER
 M:	Michael Hennerich <Michael.Hennerich@analog.com>
 L:	linux-hwmon@vger.kernel.org
@@ -2549,6 +2567,7 @@ F:	drivers/nvmem/apple-spmi-nvmem.c
 F:	drivers/phy/apple/
 F:	drivers/pinctrl/pinctrl-apple-gpio.c
 F:	drivers/power/reset/macsmc-reboot.c
+F:	drivers/power/supply/macsmc-power.c
 F:	drivers/pwm/pwm-apple.c
 F:	drivers/rtc/rtc-macsmc.c
 F:	drivers/soc/apple/*
@@ -2643,7 +2662,9 @@ R:	BST Linux Kernel Upstream Group <bst-upstream@bstai.top>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
 F:	Documentation/devicetree/bindings/arm/bst.yaml
+F:	Documentation/devicetree/bindings/mmc/bst,c1200-sdhci.yaml
 F:	arch/arm64/boot/dts/bst/
+F:	drivers/mmc/host/sdhci-of-bst.c
 
 ARM/CALXEDA HIGHBANK ARCHITECTURE
 M:	Andre Przywara <andre.przywara@arm.com>
@@ -2705,8 +2726,9 @@ N:	digicolor
 
 ARM/CORESIGHT FRAMEWORK AND DRIVERS
 M:	Suzuki K Poulose <suzuki.poulose@arm.com>
-R:	Mike Leach <mike.leach@linaro.org>
+R:	Mike Leach <mike.leach@arm.com>
 R:	James Clark <james.clark@linaro.org>
+R:	Leo Yan <leo.yan@arm.com>
 L:	coresight@lists.linaro.org (moderated for non-subscribers)
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -2803,8 +2825,7 @@ S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git
 F:	Documentation/devicetree/bindings/firmware/fsl*
 F:	Documentation/devicetree/bindings/firmware/nxp*
-F:	arch/arm/boot/dts/nxp/imx/
-F:	arch/arm/boot/dts/nxp/mxs/
+F:	arch/arm/boot/dts/nxp/
 F:	arch/arm64/boot/dts/freescale/
 X:	Documentation/devicetree/bindings/media/i2c/
 X:	arch/arm64/boot/dts/freescale/fsl-*
@@ -2910,7 +2931,6 @@ F:	include/linux/soc/ixp4xx/qmgr.h
 
 ARM/INTEL KEEMBAY ARCHITECTURE
 M:	Paul J. Murphy <paul.j.murphy@intel.com>
-M:	Daniele Alessandrelli <daniele.alessandrelli@intel.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/arm/intel,keembay.yaml
 F:	arch/arm64/boot/dts/intel/keembay-evm.dts
@@ -2991,7 +3011,6 @@ S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git
 F:	Documentation/devicetree/bindings/arm/marvell/
 F:	arch/arm/boot/dts/marvell/armada*
-F:	arch/arm/boot/dts/marvell/db-falcon*
 F:	arch/arm/boot/dts/marvell/kirkwood*
 F:	arch/arm/configs/mvebu_*_defconfig
 F:	arch/arm/mach-mvebu/
@@ -3986,7 +4005,7 @@ F:	drivers/hwmon/asus-ec-sensors.c
 ASUS NOTEBOOKS AND EEEPC ACPI/WMI EXTRAS DRIVERS
 M:	Corentin Chary <corentin.chary@gmail.com>
 M:	Luke D. Jones <luke@ljones.dev>
-M:	Denis Benato <benato.denis96@gmail.com>
+M:	Denis Benato <denis.benato@linux.dev>
 L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
 W:	https://asus-linux.org/
@@ -4022,7 +4041,7 @@ F:	drivers/hwmon/asus_wmi_sensors.c
 ASYMMETRIC KEYS
 M:	David Howells <dhowells@redhat.com>
 M:	Lukas Wunner <lukas@wunner.de>
-M:	Ignat Korchagin <ignat@cloudflare.com>
+M:	Ignat Korchagin <ignat@linux.win>
 L:	keyrings@vger.kernel.org
 L:	linux-crypto@vger.kernel.org
 S:	Maintained
@@ -4035,7 +4054,7 @@ F:	include/linux/verification.h
 
 ASYMMETRIC KEYS - ECDSA
 M:	Lukas Wunner <lukas@wunner.de>
-M:	Ignat Korchagin <ignat@cloudflare.com>
+M:	Ignat Korchagin <ignat@linux.win>
 R:	Stefan Berger <stefanb@linux.ibm.com>
 L:	linux-crypto@vger.kernel.org
 S:	Maintained
@@ -4045,14 +4064,14 @@ F:	include/crypto/ecc*
 
 ASYMMETRIC KEYS - GOST
 M:	Lukas Wunner <lukas@wunner.de>
-M:	Ignat Korchagin <ignat@cloudflare.com>
+M:	Ignat Korchagin <ignat@linux.win>
 L:	linux-crypto@vger.kernel.org
 S:	Odd fixes
 F:	crypto/ecrdsa*
 
 ASYMMETRIC KEYS - RSA
 M:	Lukas Wunner <lukas@wunner.de>
-M:	Ignat Korchagin <ignat@cloudflare.com>
+M:	Ignat Korchagin <ignat@linux.win>
 L:	linux-crypto@vger.kernel.org
 S:	Maintained
 F:	crypto/rsa*
@@ -6280,6 +6299,7 @@ M:	Daniel Lezcano <daniel.lezcano@kernel.org>
 M:	Thomas Gleixner <tglx@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
 F:	Documentation/devicetree/bindings/timer/
 F:	drivers/clocksource/
@@ -6616,6 +6636,17 @@ M:	Bence Csókás <bence98@sch.bme.hu>
 S:	Maintained
 F:	drivers/i2c/busses/i2c-cp2615.c
 
+CPU FREQUENCY DRIVERS - CPPC CPUFREQ
+M:	"Rafael J. Wysocki" <rafael@kernel.org>
+M:	Viresh Kumar <viresh.kumar@linaro.org>
+R:	Jie Zhan <zhanjie9@hisilicon.com>
+R:	Lifeng Zheng <zhenglifeng1@huawei.com>
+R:	Pierre Gondois <pierre.gondois@arm.com>
+R:	Sumit Gupta <sumitg@nvidia.com>
+L:	linux-pm@vger.kernel.org
+S:	Maintained
+F:	drivers/cpufreq/cppc_cpufreq.c
+
 CPU FREQUENCY DRIVERS - VEXPRESS SPC ARM BIG LITTLE
 M:	Viresh Kumar <viresh.kumar@linaro.org>
 M:	Sudeep Holla <sudeep.holla@kernel.org>
@@ -6624,6 +6655,12 @@ S:	Maintained
 W:	http://www.arm.com/products/processors/technologies/biglittleprocessing.php
 F:	drivers/cpufreq/vexpress-spc-cpufreq.c
 
+CPU FREQUENCY DRIVERS - VIRTUAL MACHINE CPUFREQ
+M:	Saravana Kannan <saravanak@kernel.org>
+L:	linux-pm@vger.kernel.org
+S:	Maintained
+F:	drivers/cpufreq/virtual-cpufreq.c
+
 CPU FREQUENCY SCALING FRAMEWORK
 M:	"Rafael J. Wysocki" <rafael@kernel.org>
 M:	Viresh Kumar <viresh.kumar@linaro.org>
@@ -6643,17 +6680,12 @@ F:	kernel/sched/cpufreq*.c
 F:	rust/kernel/cpufreq.rs
 F:	tools/testing/selftests/cpufreq/
 
-CPU FREQUENCY DRIVERS - VIRTUAL MACHINE CPUFREQ
-M:	Saravana Kannan <saravanak@kernel.org>
-L:	linux-pm@vger.kernel.org
-S:	Maintained
-F:	drivers/cpufreq/virtual-cpufreq.c
-
 CPU HOTPLUG
 M:	Thomas Gleixner <tglx@kernel.org>
 M:	Peter Zijlstra <peterz@infradead.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git smp/core
 F:	include/linux/cpu.h
 F:	include/linux/cpuhotplug.h
@@ -6826,12 +6858,6 @@ L:	linux-crypto@vger.kernel.org
 S:	Maintained
 F:	tools/crypto/tcrypt/tcrypt_speed_compare.py
 
-CRYPTOGRAPHIC RANDOM NUMBER GENERATOR
-M:	Neil Horman <nhorman@tuxdriver.com>
-L:	linux-crypto@vger.kernel.org
-S:	Maintained
-F:	crypto/rng.c
-
 CS3308 MEDIA DRIVER
 M:	Hans Verkuil <hverkuil@kernel.org>
 L:	linux-media@vger.kernel.org
@@ -7082,6 +7108,7 @@ DEBUGOBJECTS:
 M:	Thomas Gleixner <tglx@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core/debugobjects
 F:	include/linux/debugobjects.h
 F:	lib/debugobjects.c
@@ -7998,7 +8025,9 @@ F:	Documentation/devicetree/bindings/display/himax,hx8357.yaml
 F:	drivers/gpu/drm/tiny/hx8357d.c
 
 DRM DRIVER FOR HYPERV SYNTHETIC VIDEO DEVICE
-M:	Deepak Rawat <drawat.floss@gmail.com>
+M:	Dexuan Cui <decui@microsoft.com>
+M:	Long Li <longli@microsoft.com>
+M:	Saurabh Sengar <ssengar@linux.microsoft.com>
 L:	linux-hyperv@vger.kernel.org
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
@@ -8059,6 +8088,12 @@ F:	Documentation/devicetree/bindings/display/lvds.yaml
 F:	Documentation/devicetree/bindings/display/panel/panel-lvds.yaml
 F:	drivers/gpu/drm/panel/panel-lvds.c
 
+DRM DRIVER FOR LXD M9189A PANELS
+M:	Michael Tretter <m.tretter@pengutronix.de>
+S:	Maintained
+F:	Documentation/devicetree/bindings/display/panel/lxd,m9189a.yaml
+F:	drivers/gpu/drm/panel/panel-lxd-m9189a.c
+
 DRM DRIVER FOR MANTIX MLAF057WE51 PANELS
 M:	Guido Günther <agx@sigxcpu.org>
 R:	Purism Kernel Team <kernel@puri.sm>
@@ -8788,6 +8823,14 @@ F:	Documentation/devicetree/bindings/display/brcm,bcm2835-*.yaml
 F:	drivers/gpu/drm/vc4/
 F:	include/uapi/drm/vc4_drm.h
 
+DRM DRIVERS FOR VERISILICON DISPLAY CONTROLLER IP
+M:	Icenowy Zheng <zhengxingda@iscas.ac.cn>
+L:	dri-devel@lists.freedesktop.org
+S:	Maintained
+T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	Documentation/devicetree/bindings/display/verisilicon,dc.yaml
+F:	drivers/gpu/drm/verisilicon/
+
 DRM DRIVERS FOR VIVANTE GPU IP
 M:	Lucas Stach <l.stach@pengutronix.de>
 R:	Russell King <linux+etnaviv@armlinux.org.uk>
@@ -8900,16 +8943,18 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	drivers/gpu/drm/ttm/
 F:	include/drm/ttm/
 
-DRM BUDDY ALLOCATOR
+GPU BUDDY ALLOCATOR
 M:	Matthew Auld <matthew.auld@intel.com>
 M:	Arun Pravin <arunpravin.paneerselvam@amd.com>
 R:	Christian Koenig <christian.koenig@amd.com>
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	drivers/gpu/buddy.c
 F:	drivers/gpu/drm/drm_buddy.c
-F:	drivers/gpu/drm/tests/drm_buddy_test.c
+F:	drivers/gpu/tests/gpu_buddy_test.c
 F:	include/drm/drm_buddy.h
+F:	include/linux/gpu_buddy.h
 
 DRM AUTOMATED TESTING
 M:	Helen Koike <helen.fornazier@gmail.com>
@@ -9546,7 +9591,6 @@ F:	include/linux/phy_fixed.h
 F:	include/linux/phy_link_topology.h
 F:	include/linux/phylib_stubs.h
 F:	include/linux/platform_data/mdio-bcm-unimac.h
-F:	include/linux/platform_data/mdio-gpio.h
 F:	include/net/phy/
 F:	include/trace/events/mdio.h
 F:	include/uapi/linux/mdio.h
@@ -10511,6 +10555,7 @@ R:	Davidlohr Bueso <dave@stgolabs.net>
 R:	André Almeida <andrealmeid@igalia.com>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core
 F:	Documentation/locking/*futex*
 F:	include/asm-generic/futex.h
@@ -10652,6 +10697,7 @@ M:	Peter Zijlstra <peterz@infradead.org>
 M:	Andy Lutomirski <luto@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core/entry
 F:	include/linux/entry-common.h
 F:	include/linux/entry-virt.h
@@ -10846,6 +10892,7 @@ L:	chrome-platform@lists.linux.dev
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git
 F:	drivers/firmware/google/
+F:	include/linux/coreboot.h
 
 GOOGLE TENSOR SoC SUPPORT
 M:	Peter Griffin <peter.griffin@linaro.org>
@@ -10954,7 +11001,6 @@ F:	drivers/gpio/
 F:	include/dt-bindings/gpio/
 F:	include/linux/gpio.h
 F:	include/linux/gpio/
-F:	include/linux/of_gpio.h
 K:	(devm_)?gpio_(request|free|direction|get|set)
 K:	GPIOD_FLAGS_BIT_NONEXCLUSIVE
 K:	devm_gpiod_unhinge
@@ -11351,14 +11397,6 @@ F:	drivers/hid/hid-sensor-*
 F:	drivers/iio/*/hid-*
 F:	include/linux/hid-sensor-*
 
-HID UNIVERSAL PIDFF DRIVER
-M:	Tomasz Pakuła <tomasz.pakula.oficjalny@gmail.com>
-M:	Oleg Makarenko <oleg@makarenk.ooo>
-L:	linux-input@vger.kernel.org
-S:	Maintained
-B:	https://github.com/JacKeTUs/universal-pidff/issues
-F:	drivers/hid/hid-universal-pidff.c
-
 HID VRC-2 CAR CONTROLLER DRIVER
 M:	Marcus Folkesson <marcus.folkesson@gmail.com>
 L:	linux-input@vger.kernel.org
@@ -12663,6 +12701,7 @@ R:	Eric Snowberg <eric.snowberg@oracle.com>
 L:	linux-integrity@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity.git
+F:	include/linux/secure_boot.h
 F:	security/integrity/
 F:	security/integrity/ima/
 
@@ -12870,7 +12909,6 @@ F:	drivers/dma/ioat*
 INTEL IAA CRYPTO DRIVER
 M:	Kristen Accardi <kristen.c.accardi@intel.com>
 M:	Vinicius Costa Gomes <vinicius.gomes@intel.com>
-M:	Kanchana P Sridhar <kanchana.p.sridhar@intel.com>
 L:	linux-crypto@vger.kernel.org
 S:	Supported
 F:	Documentation/driver-api/crypto/iaa/iaa-crypto.rst
@@ -12973,14 +13011,12 @@ F:	drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
 
 INTEL KEEM BAY DRM DRIVER
 M:	Anitha Chrisanthus <anitha.chrisanthus@intel.com>
-M:	Edmund Dea <edmund.j.dea@intel.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/display/intel,keembay-display.yaml
 F:	drivers/gpu/drm/kmb/
 
 INTEL KEEM BAY OCS AES/SM4 CRYPTO DRIVER
-M:	Daniele Alessandrelli <daniele.alessandrelli@intel.com>
-S:	Maintained
+S:	Orphan
 F:	Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml
 F:	drivers/crypto/intel/keembay/Kconfig
 F:	drivers/crypto/intel/keembay/Makefile
@@ -12989,7 +13025,6 @@ F:	drivers/crypto/intel/keembay/ocs-aes.c
 F:	drivers/crypto/intel/keembay/ocs-aes.h
 
 INTEL KEEM BAY OCS ECC CRYPTO DRIVER
-M:	Daniele Alessandrelli <daniele.alessandrelli@intel.com>
 M:	Prabhjot Khurana <prabhjot.khurana@intel.com>
 M:	Mark Gross <mgross@linux.intel.com>
 S:	Maintained
@@ -12999,7 +13034,6 @@ F:	drivers/crypto/intel/keembay/Makefile
 F:	drivers/crypto/intel/keembay/keembay-ocs-ecc.c
 
 INTEL KEEM BAY OCS HCU CRYPTO DRIVER
-M:	Daniele Alessandrelli <daniele.alessandrelli@intel.com>
 M:	Declan Murphy <declan.murphy@intel.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml
@@ -13489,6 +13523,7 @@ F:	sound/soc/codecs/sma*
 IRQ DOMAINS (IRQ NUMBER MAPPING LIBRARY)
 M:	Thomas Gleixner <tglx@kernel.org>
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq/core
 F:	Documentation/core-api/irq/irq-domain.rst
 F:	include/linux/irqdomain.h
@@ -13500,6 +13535,7 @@ IRQ SUBSYSTEM
 M:	Thomas Gleixner <tglx@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq/core
 F:	include/linux/group_cpus.h
 F:	include/linux/irq.h
@@ -13513,6 +13549,7 @@ IRQCHIP DRIVERS
 M:	Thomas Gleixner <tglx@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq/core
 F:	Documentation/devicetree/bindings/interrupt-controller/
 F:	drivers/irqchip/
@@ -13723,6 +13760,15 @@ S:	Maintained
 F:	Documentation/hwmon/k8temp.rst
 F:	drivers/hwmon/k8temp.c
 
+KALLSYMS LINEINFO
+M:	Sasha Levin <sashal@kernel.org>
+S:	Maintained
+F:	Documentation/admin-guide/kallsyms-lineinfo.rst
+F:	include/linux/mod_lineinfo.h
+F:	lib/tests/lineinfo_kunit.c
+F:	scripts/gen-mod-lineinfo.sh
+F:	scripts/gen_lineinfo.c
+
 KASAN
 M:	Andrey Ryabinin <ryabinin.a.a@gmail.com>
 R:	Alexander Potapenko <glider@google.com>
@@ -14410,6 +14456,13 @@ F:	drivers/net/wan/framer/
 F:	drivers/pinctrl/pinctrl-pef2256.c
 F:	include/linux/framer/
 
+LATTEPANDA SIGMA EC HARDWARE MONITOR DRIVER
+M:	Mariano Abad <weimaraner@gmail.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/lattepanda-sigma-ec.rst
+F:	drivers/hwmon/lattepanda-sigma-ec.c
+
 LASI 53c700 driver for PARISC
 M:	"James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
 L:	linux-scsi@vger.kernel.org
@@ -14466,6 +14519,17 @@ L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
 F:	drivers/platform/x86/lenovo/wmi-hotkey-utilities.c
 
+LENOVO HID drivers
+M:	Derek J. Clark <derekjohn.clark@gmail.com>
+M:	Mark Pearson <mpearson-lenovo@squebb.ca>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+F:	Documentation/ABI/testing/sysfs-driver-hid-lenovo-go
+F:	Documentation/ABI/testing/sysfs-driver-hid-lenovo-go-s
+F:	drivers/hid/hid-lenovo-go-s.c
+F:	drivers/hid/hid-lenovo-go.c
+F:	drivers/hid/hid-lenovo.c
+
 LETSKETCH HID TABLET DRIVER
 M:	Hans de Goede <hansg@kernel.org>
 L:	linux-input@vger.kernel.org
@@ -14742,7 +14806,6 @@ F:	Documentation/memory-barriers.txt
 F:	tools/memory-model/
 
 LINUX-NEXT TREE
-M:	Stephen Rothwell <sfr@canb.auug.org.au>
 M:	Mark Brown <broonie@kernel.org>
 L:	linux-next@vger.kernel.org
 S:	Supported
@@ -14883,6 +14946,7 @@ M:	Boqun Feng <boqun@kernel.org> (LOCKDEP & RUST)
 R:	Waiman Long <longman@redhat.com>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core
 F:	Documentation/locking/
 F:	arch/*/include/asm/spinlock*.h
@@ -14942,12 +15006,14 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/gpio/loongson,ls-gpio.yaml
 F:	drivers/gpio/gpio-loongson-64bit.c
 
-LOONGSON-2 APB DMA DRIVER
+LOONGSON-2 DMA DRIVER
 M:	Binbin Zhou <zhoubinbin@loongson.cn>
 L:	dmaengine@vger.kernel.org
 S:	Maintained
+F:	Documentation/devicetree/bindings/dma/loongson,ls2k0300-dma.yaml
 F:	Documentation/devicetree/bindings/dma/loongson,ls2x-apbdma.yaml
-F:	drivers/dma/loongson2-apb-dma.c
+F:	drivers/dma/loongson/loongson2-apb-cmc-dma.c
+F:	drivers/dma/loongson/loongson2-apb-dma.c
 
 LOONGSON LS2X I2C DRIVER
 M:	Binbin Zhou <zhoubinbin@loongson.cn>
@@ -16103,8 +16169,9 @@ MEDIA INPUT INFRASTRUCTURE (V4L/DVB)
 M:	Mauro Carvalho Chehab <mchehab@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Maintained
+P:	Documentation/driver-api/media/maintainer-entry-profile.rst
 W:	https://linuxtv.org
-Q:	http://patchwork.kernel.org/project/linux-media/list/
+Q:	https://patchwork.linuxtv.org/project/linux-media/list/
 T:	git git://linuxtv.org/media.git
 F:	Documentation/admin-guide/media/
 F:	Documentation/devicetree/bindings/media/
@@ -16651,6 +16718,7 @@ L:	linux-mm@kvack.org
 S:	Maintained
 W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+F:	include/linux/folio_batch.h
 F:	include/linux/gfp.h
 F:	include/linux/gfp_types.h
 F:	include/linux/highmem.h
@@ -16659,6 +16727,7 @@ F:	include/linux/memory.h
 F:	include/linux/mm.h
 F:	include/linux/mm_*.h
 F:	include/linux/mmzone.h
+F:	include/linux/mmzone_lock.h
 F:	include/linux/mmdebug.h
 F:	include/linux/mmu_notifier.h
 F:	include/linux/pagewalk.h
@@ -16667,6 +16736,8 @@ F:	include/linux/pgtable.h
 F:	include/linux/ptdump.h
 F:	include/linux/vmpressure.h
 F:	include/linux/vmstat.h
+F:	include/trace/events/zone_lock.h
+F:	fs/proc/meminfo.c
 F:	kernel/fork.c
 F:	mm/Kconfig
 F:	mm/debug.c
@@ -16900,6 +16971,7 @@ R:	Kemeng Shi <shikemeng@huaweicloud.com>
 R:	Nhat Pham <nphamcs@gmail.com>
 R:	Baoquan He <bhe@redhat.com>
 R:	Barry Song <baohua@kernel.org>
+R:	Youngjun Park <youngjun.park@lge.com>
 L:	linux-mm@kvack.org
 S:	Maintained
 F:	Documentation/mm/swap-table.rst
@@ -16981,6 +17053,8 @@ S:	Maintained
 W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 F:	include/trace/events/mmap.h
+F:	fs/proc/task_mmu.c
+F:	fs/proc/task_nommu.c
 F:	mm/interval_tree.c
 F:	mm/mincore.c
 F:	mm/mlock.c
@@ -17708,6 +17782,7 @@ F:	arch/mips/boot/dts/loongson/loongson1*
 F:	arch/mips/configs/loongson1_defconfig
 F:	arch/mips/loongson32/
 F:	drivers/*/*loongson1*
+F:	drivers/dma/loongson/loongson1-apb-dma.c
 F:	drivers/mtd/nand/raw/loongson-nand-controller.c
 F:	drivers/net/ethernet/stmicro/stmmac/dwmac-loongson1.c
 F:	sound/soc/loongson/loongson1_ac97.c
@@ -18855,6 +18930,15 @@ W:	https://github.com/davejiang/linux/wiki
 T:	git https://github.com/davejiang/linux.git
 F:	drivers/ntb/hw/intel/
 
+NTFS FILESYSTEM
+M:	Namjae Jeon <linkinjeon@kernel.org>
+M:	Hyunchul Lee <hyc.lee@gmail.com>
+L:	linux-fsdevel@vger.kernel.org
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git
+F:	Documentation/filesystems/ntfs.rst
+F:	fs/ntfs/
+
 NTFS3 FILESYSTEM
 M:	Konstantin Komarov <almaz.alexandrovich@paragon-software.com>
 L:	ntfs3@lists.linux.dev
@@ -19209,6 +19293,7 @@ OBJTOOL
 M:	Josh Poimboeuf <jpoimboe@kernel.org>
 M:	Peter Zijlstra <peterz@infradead.org>
 S:	Supported
+P:	Documentation/process/maintainer-tip.rst
 F:	include/linux/objtool*.h
 F:	tools/objtool/
 
@@ -19750,12 +19835,6 @@ L:	linux-rtc@vger.kernel.org
 S:	Maintained
 F:	drivers/rtc/rtc-optee.c
 
-OPA-VNIC DRIVER
-M:	Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
-L:	linux-rdma@vger.kernel.org
-S:	Supported
-F:	drivers/infiniband/ulp/opa_vnic
-
 OPEN ALLIANCE 10BASE-T1S MACPHY SERIAL INTERFACE FRAMEWORK
 M:	Parthiban Veerasooran <parthiban.veerasooran@microchip.com>
 L:	netdev@vger.kernel.org
@@ -20111,6 +20190,13 @@ S:	Orphan
 F:	Documentation/devicetree/bindings/pci/altr,pcie-root-port.yaml
 F:	drivers/pci/controller/pcie-altera.c
 
+PCI DRIVER FOR ANDES QILAI PCIE
+M:	Randolph Lin <randolph@andestech.com>
+L:	linux-pci@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/pci/andestech,qilai-pcie.yaml
+F:	drivers/pci/controller/dwc/pcie-andes-qilai.c
+
 PCI DRIVER FOR APPLIEDMICRO XGENE
 M:	Toan Le <toan@os.amperecomputing.com>
 L:	linux-pci@vger.kernel.org
@@ -20476,6 +20562,13 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Odd Fixes
 F:	drivers/pci/controller/pci-thunder-*
 
+PCIE DRIVER FOR ESWIN
+M:	Senchuan Zhang <zhangsenchuan@eswincomputing.com>
+L:	linux-pci@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/pci/eswin,pcie.yaml
+F:	drivers/pci/controller/dwc/pcie-eswin.c
+
 PCIE DRIVER FOR HISILICON
 M:	Zhou Wang <wangzhou1@hisilicon.com>
 L:	linux-pci@vger.kernel.org
@@ -20677,6 +20770,7 @@ L:	linux-perf-users@vger.kernel.org
 L:	linux-kernel@vger.kernel.org
 S:	Supported
 W:	https://perf.wiki.kernel.org/
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git perf/core
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-tools
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
@@ -20697,7 +20791,7 @@ PERFORMANCE EVENTS TOOLING ARM64
 R:	John Garry <john.g.garry@oracle.com>
 R:	Will Deacon <will@kernel.org>
 R:	James Clark <james.clark@linaro.org>
-R:	Mike Leach <mike.leach@linaro.org>
+R:	Mike Leach <mike.leach@arm.com>
 R:	Leo Yan <leo.yan@linux.dev>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
@@ -20976,6 +21070,12 @@ S:	Maintained
 F:	drivers/pnp/
 F:	include/linux/pnp.h
 
+PORTUGUESE (BRAZILIAN) TRANSLATION
+M:	Daniel Pereira <danielmaraboo@gmail.com>
+L:	linux-doc@vger.kernel.org
+S:	Maintained
+F:	Documentation/translations/pt_BR/
+
 PORTWELL EC DRIVER
 M:	Yen-Chi Huang <jesse.huang@portwell.com.tw>
 L:	platform-driver-x86@vger.kernel.org
@@ -20988,6 +21088,7 @@ M:	Frederic Weisbecker <frederic@kernel.org>
 M:	Thomas Gleixner <tglx@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
 F:	fs/timerfd.c
 F:	include/linux/time_namespace.h
@@ -21319,6 +21420,7 @@ M:	Uwe Kleine-König <ukleinek@kernel.org>
 L:	linux-pwm@vger.kernel.org
 S:	Maintained
 Q:	https://patchwork.ozlabs.org/project/linux-pwm/list/
+C:	irc://irc.libera.chat/linux-pwm
 T:	git https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git
 F:	Documentation/devicetree/bindings/pwm/
 F:	Documentation/driver-api/pwm.rst
@@ -22800,8 +22902,12 @@ M:	Joel Stanley <jms@oss.tenstorrent.com>
 L:	linux-riscv@lists.infradead.org
 S:	Maintained
 T:	git https://github.com/tenstorrent/linux.git
+F:	Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml
 F:	Documentation/devicetree/bindings/riscv/tenstorrent.yaml
 F:	arch/riscv/boot/dts/tenstorrent/
+F:	drivers/clk/tenstorrent/
+F:	drivers/reset/reset-tenstorrent-atlantis.c
+F:	include/dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h
 
 RISC-V THEAD SoC SUPPORT
 M:	Drew Fustini <fustini@kernel.org>
@@ -22821,6 +22927,7 @@ F:	Documentation/devicetree/bindings/reset/thead,th1520-reset.yaml
 F:	arch/riscv/boot/dts/thead/
 F:	drivers/clk/thead/clk-th1520-ap.c
 F:	drivers/firmware/thead,th1520-aon.c
+F:	drivers/gpu/drm/bridge/th1520-dw-hdmi.c
 F:	drivers/mailbox/mailbox-th1520.c
 F:	drivers/net/ethernet/stmicro/stmmac/dwmac-thead.c
 F:	drivers/pinctrl/pinctrl-th1520.c
@@ -23663,6 +23770,7 @@ R:	Mel Gorman <mgorman@suse.de> (CONFIG_NUMA_BALANCING)
 R:	Valentin Schneider <vschneid@redhat.com> (TOPOLOGY)
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core
 F:	include/linux/preempt.h
 F:	include/linux/sched.h
@@ -24504,7 +24612,6 @@ F:	include/uapi/rdma/rdma_user_rxe.h
 
 SOFTLOGIC 6x10 MPEG CODEC
 M:	Bluecherry Maintainers <maintainers@bluecherrydvr.com>
-M:	Andrey Utkin <andrey_utkin@fastmail.com>
 M:	Ismael Luceno <ismael@iodev.co.uk>
 L:	linux-media@vger.kernel.org
 S:	Supported
@@ -24827,6 +24934,12 @@ W:	https://linuxtv.org
 Q:	http://patchwork.linuxtv.org/project/linux-media/list/
 F:	drivers/media/dvb-frontends/sp2*
 
+SPACEMIT DWMAC GLUE LAYER
+M:	Inochi Amaoto <inochiama@gmail.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/net/spacemit,k3-dwmac.yaml
+F:	drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c
+
 SPACEMIT K1 I2C DRIVER
 M:	Troy Mitchell <troy.mitchell@linux.spacemit.com>
 S:	Maintained
@@ -24900,9 +25013,9 @@ F:	drivers/clk/spear/
 F:	drivers/pinctrl/spear/
 
 SPI NOR SUBSYSTEM
-M:	Tudor Ambarus <tudor.ambarus@linaro.org>
 M:	Pratyush Yadav <pratyush@kernel.org>
 M:	Michael Walle <mwalle@kernel.org>
+R:	Takahiro Kuwano <takahiro.kuwano@infineon.com>
 L:	linux-mtd@lists.infradead.org
 S:	Maintained
 W:	http://www.linux-mtd.infradead.org/
@@ -25000,6 +25113,7 @@ F:	drivers/hwmon/pmbus/stef48h28.c
 ST STM32 FIREWALL
 M:	Gatien Chevallier <gatien.chevallier@foss.st.com>
 S:	Maintained
+F:	drivers/bus/stm32_dbg_bus.c
 F:	drivers/bus/stm32_etzpc.c
 F:	drivers/bus/stm32_firewall.c
 F:	drivers/bus/stm32_rifsc.c
@@ -25149,15 +25263,6 @@ M:	Ion Badulescu <ionut@badula.org>
 S:	Odd Fixes
 F:	drivers/net/ethernet/adaptec/starfire*
 
-STARFIVE CAMERA SUBSYSTEM DRIVER
-M:	Jack Zhu <jack.zhu@starfivetech.com>
-M:	Changhuang Liang <changhuang.liang@starfivetech.com>
-L:	linux-media@vger.kernel.org
-S:	Maintained
-F:	Documentation/admin-guide/media/starfive_camss.rst
-F:	Documentation/devicetree/bindings/media/starfive,jh7110-camss.yaml
-F:	drivers/staging/media/starfive/camss
-
 STARFIVE CRYPTO DRIVER
 M:	Jia Jie Ho <jiajie.ho@starfivetech.com>
 M:	William Qiu <william.qiu@starfivetech.com>
@@ -25515,6 +25620,13 @@ S:	Supported
 F:	include/net/switchdev.h
 F:	net/switchdev/
 
+SWITCHTEC DMA DRIVER
+M:	Kelvin Cao <kelvin.cao@microchip.com>
+M:	Logan Gunthorpe <logang@deltatee.com>
+L:	dmaengine@vger.kernel.org
+S:	Maintained
+F:	drivers/dma/switchtec_dma.c
+
 SY8106A REGULATOR DRIVER
 M:	Icenowy Zheng <icenowy@aosc.io>
 S:	Maintained
@@ -26293,7 +26405,7 @@ F:	drivers/media/i2c/thp7312.c
 F:	include/uapi/linux/thp7312.h
 
 THUNDERBOLT DMA TRAFFIC TEST DRIVER
-M:	Isaac Hazan <isaac.hazan@intel.com>
+M:	Mika Westerberg <westeri@kernel.org>
 L:	linux-usb@vger.kernel.org
 S:	Maintained
 F:	drivers/thunderbolt/dma_test.c
@@ -26610,6 +26722,7 @@ F:	include/linux/timekeeping.h
 F:	include/linux/timex.h
 F:	include/uapi/linux/time.h
 F:	include/uapi/linux/timex.h
+F:	kernel/time/.kunitconfig
 F:	kernel/time/alarmtimer.c
 F:	kernel/time/clocksource*
 F:	kernel/time/ntp*
@@ -26947,8 +27060,6 @@ F:	drivers/platform/x86/tuxedo/
 
 TW5864 VIDEO4LINUX DRIVER
 M:	Bluecherry Maintainers <maintainers@bluecherrydvr.com>
-M:	Andrey Utkin <andrey.utkin@corp.bluecherry.net>
-M:	Andrey Utkin <andrey_utkin@fastmail.com>
 L:	linux-media@vger.kernel.org
 S:	Supported
 F:	drivers/media/pci/tw5864/
@@ -27268,6 +27379,15 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git
 F:	Documentation/hid/hiddev.rst
 F:	drivers/hid/usbhid/
 
+USB HID PID DRIVERS (USB WHEELBASES, JOYSTICKS, RUDDERS, ...)
+M:	Tomasz Pakuła <tomasz.pakula.oficjalny@gmail.com>
+M:	Oleg Makarenko <oleg@makarenk.ooo>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+B:	https://github.com/JacKeTUs/universal-pidff/issues
+F:	drivers/hid/usbhid/hid-pidff*
+F:	drivers/hid/hid-universal-pidff.c
+
 USB INTEL XHCI ROLE MUX DRIVER
 M:	Hans de Goede <hansg@kernel.org>
 L:	linux-usb@vger.kernel.org
@@ -28548,6 +28668,7 @@ M:	x86@kernel.org
 R:	"H. Peter Anvin" <hpa@zytor.com>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-tip.rst
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86/core
 F:	Documentation/arch/x86/
 F:	Documentation/devicetree/bindings/x86/
diff --git a/Makefile b/Makefile
index c9b7bee102e81..cc5c9a515329b 100644
--- a/Makefile
+++ b/Makefile
@@ -588,7 +588,6 @@ LINUXINCLUDE    := \
 KBUILD_AFLAGS   := -D__ASSEMBLY__ -fno-PIE
 
 KBUILD_CFLAGS :=
-KBUILD_CFLAGS += -std=gnu11
 KBUILD_CFLAGS += -fshort-wchar
 KBUILD_CFLAGS += -funsigned-char
 KBUILD_CFLAGS += -fno-common
@@ -677,14 +676,19 @@ print_env_for_makefile = \
 	echo "export KBUILD_OUTPUT = $(CURDIR)"
 endif
 
-quiet_cmd_makefile = GEN     Makefile
-      cmd_makefile = { \
+filechk_makefile = { \
 	echo "\# Automatically generated by $(abs_srctree)/Makefile: don't edit"; \
 	$(print_env_for_makefile); \
 	echo "include $(abs_srctree)/Makefile"; \
-	} > Makefile
+	}
 
-outputmakefile:
+$(objtree)/Makefile: FORCE
+	$(call filechk,makefile)
+
+# Prevent $(srcroot)/Makefile from inhibiting the rule to run.
+PHONY += $(objtree)/Makefile
+
+outputmakefile: $(objtree)/Makefile
 ifeq ($(KBUILD_EXTMOD),)
 	@if [ -f $(srctree)/.config -o \
 		 -d $(srctree)/include/config -o \
@@ -705,7 +709,6 @@ else
 	fi
 endif
 	$(Q)ln -fsn $(srcroot) source
-	$(call cmd,makefile)
 	$(Q)test -e .gitignore || \
 	{ echo "# this is build directory, ignore it"; echo "*"; } > .gitignore
 endif
@@ -791,6 +794,18 @@ ifdef need-config
 include $(objtree)/include/config/auto.conf
 endif
 
+CC_FLAGS_DIALECT := -std=gnu11
+# Allow including a tagged struct or union anonymously in another struct/union.
+CC_FLAGS_DIALECT += $(CONFIG_CC_MS_EXTENSIONS)
+# Clang enables warnings about GNU and Microsoft extensions by default, disable
+# them because this is expected with the above options.
+ifdef CONFIG_CC_IS_CLANG
+CC_FLAGS_DIALECT += -Wno-gnu
+CC_FLAGS_DIALECT += -Wno-microsoft-anon-tag
+endif
+export CC_FLAGS_DIALECT
+KBUILD_CFLAGS += $(CC_FLAGS_DIALECT)
+
 ifeq ($(KBUILD_EXTMOD),)
 # Objects we will link into vmlinux / subdirs we need to visit
 core-y		:=
@@ -1035,6 +1050,7 @@ endif
 ifdef CONFIG_LTO_CLANG
 ifdef CONFIG_LTO_CLANG_THIN
 CC_FLAGS_LTO	:= -flto=thin -fsplit-lto-unit
+KBUILD_LDFLAGS += $(call ld-option,--lto-whole-program-visibility -mllvm -always-rename-promoted-locals=false)
 else
 CC_FLAGS_LTO	:= -flto
 endif
@@ -1094,9 +1110,6 @@ NOSTDINC_FLAGS += -nostdinc
 # perform bounds checking.
 KBUILD_CFLAGS += $(call cc-option, -fstrict-flex-arrays=3)
 
-# Allow including a tagged struct or union anonymously in another struct/union.
-KBUILD_CFLAGS += -fms-extensions
-
 # disable invalid "can't wrap" optimizations for signed / pointers
 KBUILD_CFLAGS	+= -fno-strict-overflow
 
@@ -1654,7 +1667,7 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \
 	       modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \
 	       compile_commands.json rust/test \
 	       rust-project.json .vmlinux.objs .vmlinux.export.c \
-               .builtin-dtbs-list .builtin-dtb.S
+               .builtin-dtbs-list .builtin-dtbs.S
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_FILES += include/config include/generated          \
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 0000000000000..fa74d4ab0761d
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,413 @@
+Name		SHA1
+----		----
+origin		0e4f8f1a3d081e834be5fd0a62bdb2554fadd307
+ext4-fixes	d250bdf531d9cd4096fedbb9f172bb2ca660c868
+vfs-brauner-fixes	e9075e420a1eb3b52c60f3b95893a55e77419ce8
+fscrypt-current	1f318b96cc84d7c2ab792fcc0bfd42a7ca890681
+fsverity-current	1f318b96cc84d7c2ab792fcc0bfd42a7ca890681
+btrfs-fixes	1a30969e525d3b89b3d709e16d87518b8dda1f5c
+vfs-fixes	49c5d168a3a8f4eb27d44a2a22b7e8a856ca601f
+erofs-fixes	9bbb89d45a5df6103d6e59823a909a52b436712d
+nfsd-fixes	5133b61aaf437e5f25b1b396b14242a6bb0508e2
+v9fs-fixes	2014c95afecee3e76ca4a56956a936e23283f05b
+fscrypt		4377a22d84f726f0a650927edf75cdc0698baf06
+btrfs		41019679bc4f7ed6b072a83ba264908bc88752dd
+ceph		1ff957199bd11ece8280a29a13120dde5152aa7f
+cifs		23b5df09c27aec13962b30d32a4167ebdd043f8e
+configfs	6363844fdbbb76afe1d44d678fe0746390204a5f
+ecryptfs	2a9cb23307db71319e6b841e1c3d70066c0ff660
+dlm		2c3a0b730012ef87aaaf35243e1fbe9880666f7c
+erofs		11439c4635edd669ae435eec308f4ab8a0804808
+exfat		efbb1375be25dccb79944fc390d36a9820bd2e94
+ext3		b9755b4825901c571f5cd508f5150737028417f8
+ext4		4f5e8e6f012349a107531b02eed5b5ace6181449
+f2fs		5f04e90eedd052d43dab1edc4381d7c1a6826d70
+fsverity	1f318b96cc84d7c2ab792fcc0bfd42a7ca890681
+fuse		74b73faa4ab38c3a39daffe40ef212db1ecd7336
+gfs2		163a287dee723affbf19037fb9b4128b875be41f
+jfs		dad98c5b2a05ef744af4c884c97066a3c8cdad61
+ksmbd		c5f3a29cdc9f389946bd80de93502272ab17218a
+nfs		11439c4635edd669ae435eec308f4ab8a0804808
+nfs-anna	4529e0015432977af3ecc3b9f940fc2a1ef1b265
+nfsd		084a0939af2b80bbaaaacf399e36521d9acefdaf
+ntfs		77f58db7391e227f8ff6ef5d83966347d759daed
+ntfs3		e98266e823a1fa06fe6499df61aeaac2fd6f7a49
+orangefs	9e835108a9ae1c37aef52a6f8d53265f474904a1
+overlayfs	869056dbbd636f8f256b695f39c102eb3ce2edd0
+ubifs		11efa98bcc0d00271e8f66c7c940c284f973f39d
+v9fs		da2519ee01067fe9dee9bafac407ca1d8b276af3
+v9fs-ericvh	2014c95afecee3e76ca4a56956a936e23283f05b
+xfs		2c0ff6151c7bc51b20e7b25be2073e6f01f750ef
+zonefs		a42938e80357a13f8b8592111e63f2e33a919863
+vfs-brauner	a4e79b1233089509af213d59f7aeef3926c15599
+vfs		0787a93baa1aab9fd0cb8500105d11d3d3a58f7a
+mm-hotfixes	d2c7d0f86d9949af81a85a997d7e1150ef97b644
+fs-current	8b5f70b18f13d1b14dfcfc62fb0b1c23fbe6369b
+kbuild-current	775af5cbb22c1de2ad0f486959739c35cfc55ac8
+arc-current	d7b8f8e20813f0179d8ef519541a3527e7661d3a
+arm-current	23ea2a4c72323feb6e3e025e8a6f18336513d5ad
+arm64-fixes	8c6e9b60f5c7985a9fe41320556a92d7a33451df
+arm-soc-fixes	df3ef89d7ef8185fa719812e2d175b83112aa315
+davinci-current	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+drivers-memory-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+sophgo-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+sophgo-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+m68k-current	cb39cf99d88e8f2c68a41fb3bb01c4f40a8fcc30
+powerpc-fixes	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+s390-fixes	598bbefa8032cc58b564a81d1ad68bd815c8dc0f
+net		7c770dadfda5cbbde6aa3c4363ed513f1d212bf8
+bpf		146bd2a87a65aa407bb17fac70d8d583d19aba06
+ipsec		d849a2f7309fc0616e79d13b008b0a47e0458b6e
+netfilter	e7577a06ae28287ca415aec5c12277e3a80ee372
+ipvs		6cfc3bc02b977f2fba5f7268e6504d1931a774f7
+wireless	a1d9d8e833781c44ab688708804ce35f20f3cbbd
+ath		6dccbc9f3e1d38565dff7730d2b7d1e8b16c9b09
+iwlwifi		fb7f54aa2a99b07945911152c5d3d4a6eb39f797
+wpan		8ce4f287524c74a118b0af1eebd4b24a8efca57a
+rdma-fixes	0f2055db7b630559870afb40fc84490816ab8ec5
+sound-current	591721223be9e28f83489a59289579493b8e3d83
+sound-asoc-fixes	a437601a0a1383260222223440a95dd4322eb7fa
+regmap-fixes	09e70e4f119ff650d24c96161fd2f62ac7e424b0
+regulator-fixes	4221f30e3e0a2507641b3397d21aff9e71e749f8
+spi-fixes	8c89a077ca796a2fe248c584e9d7e66cff0388c8
+pci-current	21647677ba9af2cb6bc460e17d9f29a7132c40c3
+driver-core.current	2b38efc05bf7a8568ec74bfffea0f5cfa62bc01d
+tty.current	5eb608319bb56464674a71b4a66ea65c6c435d64
+usb.current	f50200dd44125e445a6164e88c217472fa79cdbc
+usb-serial-fixes	0e01c3416eb863ee7f156a9d7e7421ec0a9f68a0
+phy		81af9e40e2e4e1aa95f09fb34811760be6742c58
+staging.current	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+iio-fixes	dd154646d292cce7de952f216760c58c35cfecde
+counter-current	9448598b22c50c8a5bb77a9103e2d49f134c9578
+char-misc.current	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+soundwire-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+thunderbolt-fixes	73a505dc48144ec72e25874e2b2a72487b02d3bc
+input-current	7adaaee5edd35a423ae199c41b86bd1ed60ed483
+crypto-current	5c52607c43c397b79a9852ce33fc61de58c3645c
+libcrypto-fixes	d5b66179b0e27c14a9033c4356937506577485e3
+vfio-fixes	61ceaf236115f20f4fdd7cf60f883ada1063349a
+kselftest-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+dmaengine-fixes	a17ce4bc6f4f9acf77ba416c36791a15602e53aa
+backlight-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mtd-fixes	b9465b04de4b90228de03db9a1e0d56b00814366
+mfd-fixes	aa6e63373d91f82628b20f6fae7ffe9b97b568a5
+v4l-dvb-fixes	e8d97c270cb46a2a88739019d0f8547adc7d97da
+reset-fixes	c961cc86af01246a7ce706bbc29072d314e00880
+mips-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+at91-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+omap-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+kvm-fixes	d2ea4ff1ce50787a98a3900b3fb1636f3620b7cf
+kvms390-fixes	ab5119735e984f6b724ef1b699c01479949ed1de
+kvm-arm-fixes	e3d585ed3ff891a00c2284fef4be9cf8581735ab
+hwmon-fixes	88838ea00402c58ced630c29e36a02c862970477
+nvdimm-fixes	a8aec14230322ed8f1e8042b6d656c1631d41163
+cxl-fixes	be5c5280cf2b20e363dc8e2a424dd200a29b1c77
+dma-mapping-fixes	f5ebf241c407dbf629fcf515015e139fcea2c2f0
+drivers-x86-fixes	58a1bf172388b07a6fa2bd35c27d6cd68bcc3905
+samsung-krzk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pinctrl-samsung-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+devicetree-fixes	81122fba08fa3ccafab6ed272a5c6f2203923a7e
+dt-krzk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+scsi-fixes	01f784fc9d0ab2a6dac45ee443620e517cb2a19b
+drm-fixes	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+drm-intel-fixes	0162ab3220bac870e43e229e6e3024d1a21c3f26
+mmc-fixes	5e3486e64094c28a526543f1e8aa0d5964b7f02d
+rtc-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+gnss-fixes	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+hyperv-fixes	c0e296f257671ba10249630fe58026f29e4804d9
+risc-v-fixes	11212007bf29af0ff03f7fa2df4e0ab822af87bd
+riscv-dt-fixes	305f2865bd034146b2eebc77c27fc50d8d79778d
+riscv-soc-fixes	31c139acb6f54941cf90a765d1982953ff494fb0
+fpga-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+spdx		ebf1bafd090790704ba54c032de299fccd90a9da
+gpio-brgl-fixes	09c8ef6236004601b930965dfed432ac0e683b7e
+gpio-intel-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pinctrl-intel-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+auxdisplay-fixes	995a418a6ca33e466e5e1527663ae3d5eee18304
+kunit-fixes	8b8f1d5e350acdf972b6b02e225d9e14c600f7ad
+memblock-fixes	f043a93fff9e3e3e648b6525483f59104b0819fa
+renesas-fixes	418a9d4aa02d24c57139cf8cab7f3b13abd06e94
+perf-current	563d39928db602c58d24301769e25e33a48a65ab
+efi-fixes	a4b0bf6a40f3c107c67a24fbc614510ef5719980
+battery-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+iommufd-fixes	11439c4635edd669ae435eec308f4ab8a0804808
+rust-fixes	a989fde763f4f24209e4702f50a45be572340e68
+w1-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pmdomain-fixes	b826d2c0b0ecb844c84431ba6b502e744f5d919a
+i2c-host-fixes	78a6ee14f8b9e1c8f7c77612122444f3be8dc8cc
+sparc-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+clk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+thead-clk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+tenstorrent-clk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pwrseq-fixes	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+thead-dt-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+ftrace-fixes	1650a1b6cb1ae6cb99bb4fce21b30ebdf9fc238e
+ring-buffer-fixes	b042fdf18e89a347177a49e795d8e5184778b5b6
+trace-fixes	d008ba8be8984760e36d7dcd4adbd5a41a645708
+tracefs-fixes	8b55572e51805184353ee7d587c720a51818fb82
+spacemit-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+tip-fixes	5f6970056f68f92790bcd2632f1bc15dd266b3fa
+slab-fixes	464b1c115852fe025635ae2065e00caced184d92
+drm-msm-fixes	e4eb11b34d6c84f398d8f08d7cb4d6c38e739dd2
+uml-fixes	df447a3b4a4b961c9979b4b3ffb74317394b9b40
+fwctl-fixes	11439c4635edd669ae435eec308f4ab8a0804808
+devsec-tsm-fixes	c2012263047689e495e81c96d7d5b0586299578d
+drm-rust-fixes	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+tenstorrent-dt-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+drm-misc-fixes	fc3bbf34e643faa8678aabdc3810c60109f3435a
+rust		79e25710e7227228902d672417b552dd1d7e5d3b
+rust-interop	05f7e89ab9731565d8a62e3b5d1ec206485eeb0b
+rust-alloc	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+rust-io		86731a2a651e58953fc949573895f2fa6d456841
+rust-pin-init	aeb5ecad5316f6af160993915163367290825b6b
+rust-timekeeping	4521438fb076f8a6a52f45b0e508f6ef10ac0c49
+rust-xarray	c455f19bbe6104debd980bb15515faf716bd81b8
+rust-analyzer	5c8d16ac49405c5b77c955684849528f7d4d6b81
+mm-stable	f0caa1d49cc07b30a7e2f104d3853ec6dc1c3cad
+mm-nonmm-stable	f0caa1d49cc07b30a7e2f104d3853ec6dc1c3cad
+mm-unstable	a12ff2f1b54744e4f7e6ba30157f49d607437f8c
+mm-nonmm-unstable	ed597c460186e314ddf85c37634f260eec4a3879
+kbuild		c9bb03ac2c66bc5aa81b51ea0792477524c2763a
+clang-format	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+perf		616cd6047cbf736d93808f652086dd10a836005f
+compiler-attributes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+dma-mapping	661f8a193d48d123aedcbd401ace137333d02523
+asm-generic	adbbd9714f8058730f93c8df5c5bf1679456424b
+alpha		d58041d2c63e09a1c9083e0e9f4151e487c4e16a
+arm		b9cbb9e3bed3ffd40201fc28398f5f0a5f9a324a
+arm64		d118f32246fdabfb4f6a3fd2e511dc5e622bc553
+arm-perf	ab26d9c85554c4ff1d95ca8341522880ed9219d6
+arm-soc		2962cc9b98a52f68dd6225036424507e269b8142
+amlogic		e3194dfb772304a1b7ca3bcfccacefec3468b7bf
+asahi-soc	d31647c698f632e0c08080a760e004ba4d71f8dd
+at91		112090051aa2f77aae383b28e32a3b77b08888a2
+bmc		710dbb13377c80a6e39ef049a517665841e3221e
+broadcom	ac6f668a239487ceb7bcd60ebcc6f6338d0cd8fb
+cix		b6812b03e30e6d15e0d5ce5f130e31b9f6689d8d
+davinci		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+drivers-memory	82169065ffb07577075a5088b313d78673ded331
+fsl		cf8e9203cc653f0a82639f7ce8089fa92afe6739
+imx-mxs		aa31e3bb9b98a86e5dde5c0c637a11cc57470281
+mediatek	405c09548a695ca7be58b5b9d3ac8388630e907f
+mvebu		6b14ccadcd807bfaac1ebe054d4a749ff859b366
+omap		adaad0760ba7c3c09a7671f2a846a66a65104b96
+qcom		3b11871f8e473fe399fb348ea4d0c122f4c182b1
+renesas		d158b37b9b4890687f507a221cb68fb27ca1cff7
+reset		223af4a569d1f491e55d4de8ed40333c938622d0
+rockchip	74b04802e23d970099c522b711b0d94e9a29dd74
+samsung-krzk	dcba46f4b7a86b0b93f46ecd9e7a82d758c7ea78
+scmi		351139c6984d74f8138f34667de942624ff63213
+sophgo		9aa6068586a9b4cd34cf04f8dee72a7283ab4ae4
+sophgo-soc	c8754c7deab4cbfa947fa2d656cbaf83771828ef
+spacemit	28a7f755d7c9a4b9c41c12620fb4885f39b554ad
+stm32		1dd6f0f026268472a69a201abd4f761caf3d73ad
+sunxi		2657afa4d7d16ac07a6946201e5c8ad17b11dc21
+tee		68ab55bcc5f514cbde225f8628bbe43e40cdc2f3
+tegra		5d341fe395182434fd0849ebfced103db1b69d5c
+tenstorrent-dt	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+thead-dt	9c99a784d9117a192ebf779d4f72ebec435ada97
+ti		def2848b738945760303ff99afad650012340577
+xilinx		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+socfpga		9d29bcefcdbec4498fe0539e6d8a4e7962d92489
+clk		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+clk-imx		d16f57caa78776e6e8a88b96cb2597797b376138
+clk-renesas	c8d5972a25408b1daf73653ccd5207fdfc80c964
+thead-clk	baf4fc7c03bd0f68c768cfe27829674bd060c6b4
+tenstorrent-clk	23c8ebc952849b3ba47d04d0ec95daf5cc136061
+csky		817d21578d51e801df58ab012654486a71073074
+loongarch	c252c12d1f55bd5737e3b8e7839914ccdc7a701c
+m68k		cb39cf99d88e8f2c68a41fb3bb01c4f40a8fcc30
+m68knommu	f338e77383789c0cae23ca3d48adcc5e9e137e3c
+microblaze	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+mips		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+openrisc	0dfffa5479d6260d04d021f69203b1926f73d889
+parisc-hd	2c98a8fbd6aa647414c6248dacf254ebe91c79ad
+powerpc		f338e77383789c0cae23ca3d48adcc5e9e137e3c
+risc-v		18be4ca5cb4e5a86833de97d331f5bc14a6c5a6d
+riscv-dt	d95214e862ab5f585dcc3a027a23d7453e7c51f9
+riscv-soc	9a9d7b6a45f5f45ebf0e7c9352101b756956c4d2
+s390		efa9a04bf4945c32036aa4e5e815c5564f9582be
+sh		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+sparc		d844152d85cfcc7d2ef9430a25882604c12da279
+uml		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+xtensa		aa5903b47d4bc0e7d1fb76941731cbda32ef9cbc
+fs-next		40cdae4812a3f588fc7d051a70394603a2dfd427
+printk		9095f233c0258e9a05e958c7d822eb38681e7a5a
+pci		13292811c9d461d4b0478ce8fa2d55d3cba5a905
+pstore		bb5278a89dad28bddf3b880e68fa7847960f2fb4
+hid		2bbe306cc6f970474cb2ad7d5332220e81119bbd
+i2c		e9b13580b334040da0184a9f32106d808acd7fd5
+i2c-host	08e392059b7554e30435b477cd059117fcc165ec
+i3c		8ecd876fdaedac3d6459801b7d1d8d9638037b28
+dmi		1afafbaf749d8e8ec53f8e38efdc731131902b5b
+hwmon-staging	be8aad7a8a14151fd471aadf368e1582f91a7817
+jc_docs		98e7b5752898f74788098bef51f53205e365ab9d
+v4l-dvb		0e2c4117c3512cf6b8f54c2c3d37564bfa3ccd67
+v4l-dvb-next	adc218676eef25575469234709c2d87185ca223a
+pm		b9d1532ab4db35d9d35fc093b349f48896ca94ec
+cpufreq-arm	8655a4e35cda5534f93303af393eac4e71704701
+cpupower	3817b1d34432d3d83a61576719570d04c341b7cd
+devfreq		85eaa767662a69f1d36d29b0bcbd8efddb0955c9
+pmdomain	bc330699801d3b4f99110365512caed5adcfaca3
+opp		3d2398f44a2d48fb1c575a6e0bc6b38f3e689e22
+thermal		b3414148bbc1f9cd56217e58a558c6ac4fd1b4a6
+rdma		1487bad4ea518add12201ab37e6b672e308cfa81
+net-next	ca7e99335aea7c5977683624ba319157a4603f96
+bpf-next	4b21ea5024830f5db6826542048541fab757085b
+ipsec-next	be14d13625c9b070c33c423026b598ed65695225
+mlx5-next	4dd2115f43594da5271a1aa34fde6719b4259047
+netfilter-next	9ac76f3d0bb2940db3a9684d596b9c8f301ef315
+ipvs-next	4686679a14d269d4f02533228e82de56f432bae5
+bluetooth	eae2412ea6f80623e44075912f12037449ccc38d
+wireless-next	9ac76f3d0bb2940db3a9684d596b9c8f301ef315
+ath-next	7bbb578fc43e7dcb8690cfc98844bd67bc311e8a
+iwlwifi-next	cdba9ccc1fe4001bde67420a496439e27661233a
+wpan-next	1dd9291eb90378f8096669d9cb33761fd84599e0
+wpan-staging	1dd9291eb90378f8096669d9cb33761fd84599e0
+drbd		945807e66a140e5ee0f919d04e15eadd4cc294be
+mtd		b800359a4dfacae983cd01f8c3f1cbb6f4c9f816
+nand		ee78d466db8a001d137d6f9c97010b343aee456b
+spi-nor		94645aa41bf9ecb87c2ce78b1c3405bfb6074a37
+crypto		c708d3fad4217f23421b8496e231b0c5cee617a0
+libcrypto	6bc9effb4cbf9b6eba0f51aba1c8893dfd4c8100
+drm		d93f8ea0e5ad41d661496d205fac3e2fbd9358c0
+drm-exynos	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc	fbef867cf6614178bc6afd15ee15f85cddf19eab
+amdgpu		daf9f9dd509113d6f49ce7c00d92bc17db58f215
+drm-intel	a464bace0482aa9a83e9aa7beefbaf44cd58e6cf
+drm-msm		50c4a49f7292b33b454ea1a16c4f77d6965405dc
+drm-msm-lumag	50c4a49f7292b33b454ea1a16c4f77d6965405dc
+drm-xe		2c440f2fccf35d18f1b7eafc9015f0230c25395b
+etnaviv		6bde14ba5f7ef59e103ac317df6cc5ac4291ff4a
+drm-rust	a19457958c3018783881c4416f272cd594f13049
+drm-nova	93296e9d9528f0d87f2cf3fee494599060a0f14a
+fbdev		d3da7f00e21c2ff274b991167c69e96501746f28
+regmap		10ec8c44032962ebeefa2f6d0706be6cbdbc683e
+sound		df3eec203b940bad98a7c0b7ec0edaaaa8cd0247
+ieee1394	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+sound-asoc	5113dd03be757564d1cb6f8c7c4b7c67721e0ed3
+modules		f15dbe8a94b6e3768b10e10bf8ab95b28682db80
+input		b8303880b641fa12db4e752b19f1b5160f0fa965
+block		c4ae84167d02581871ad2d907a9134bf111aaff2
+device-mapper	23e6e57a93bcabe86d5f0eab1df0c44706ab18f3
+libata		d78c5bbf098fd93884a5dc05aec045f32cce7525
+pcmcia		4a81f78caa53e0633cf311ca1526377d9bff7479
+mmc		d75c6c7d8e9855e4dfa28d17c4ab9ad4f29a40a9
+mfd		405c1338219f0d0e272db85659ac8b9c2d19ac9c
+backlight	797cc011ae02bda26f93d25a4442d7a1a77d84df
+battery		aa2132799817fb052d95a87f0c23cc6af38541c0
+regulator	f33d6582eb20716e2451f8e3110e990f8ba2bde5
+security	306dc96b226af064174a6d631495bc459088fd10
+apparmor	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+integrity	7a60fe48af206d34571e446d685672f5730a6b90
+selinux		8dc51459ef702bcc0ef5fb26bb4d362b38aa56c2
+smack		33d589ed60ae433b483761987b85e0d24e54584e
+tomoyo		824a14076c358e77cfde68617a80b619b5753a14
+tpmdd		d79526b89571ae447c1a5cfd3d627efa07098348
+watchdog	450517fa5aae98b857c68743fd12ce93e4b8989d
+iommu		ca3bbc9287400c1274d87ee57a16e3126ba2969a
+audit		360160f75592bdc85edba8fe78fb20d90924c7e8
+devicetree	3ee3d8a44976ac7e39584637ee4c840d70ab2ad1
+dt-krzk		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+mailbox		57df858a46f0a4cc104716e0ec88864e5c386ca4
+spi		01ad82a9260b3d99d75f7166936a3a105370d4e0
+tip		00296b545b4aab827ea948ccced15d9d2bcf3cc2
+clockevents	a83f9c04be4e807d1c3961eec3fe3310c60ed9aa
+edac		b7699a32144419b676cfcc5f2fd0afe774f38169
+ftrace		60447f56379509b5d5d9cec8b8738a7cd931b877
+rcu		ed062c41dfda2de8d1712c91e089303dae013bb7
+paulmck		0075737499dc87cb1d125f4bdc088bd8b0618d2c
+kvm		d2ea4ff1ce50787a98a3900b3fb1636f3620b7cf
+kvm-arm		e3d585ed3ff891a00c2284fef4be9cf8581735ab
+kvms390		e3372ffb5f9e2dda3da259b768aab6271672b90d
+kvm-ppc		f338e77383789c0cae23ca3d48adcc5e9e137e3c
+kvm-riscv	c61ec3e8cc5d46fa269434a9ec16ca36d362e0dd
+kvm-x86		3d6cdcc8883b5726513d245eef0e91cabfc397f7
+xen-tip		e2dcf9065536ab4a1b00828ff0d19f7d282dfecc
+percpu		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+workqueues	5fb24c982349020aa18893eb18bd48103591c24b
+sched-ext	f6689792ffc4bc226636a513f8b0ac7bd45c5091
+drivers-x86	6da22b031a3cf995d6488500a5edae07cae3df2d
+chrome-platform	168e4b208ca8c2e04de20cc6cb7e2fb035dc1ec8
+chrome-platform-firmware	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+hsi		e61af3ca893363838f263f3528476f6e1faed9aa
+leds-lj		e68f95a51d1a8c1594b536c4d495cbea38d47561
+ipmi		122d16da1313f1746a4cdd31a620bbb141be7060
+driver-core	9aa64d2503c6f5a803ff2990608312e5bdc6b0de
+usb		84db3719d27337b952fe382413d341fb95351130
+thunderbolt	500e54d449f60e9692e2622ad2ba4f1e79590e87
+usb-serial	5a6a33b56402167ef019fd2520728e8c25614f74
+tty		eb3b0d92c9c39890592cca6647601fe5c631efea
+char-misc	88cf8a9ad32f85012eaf4bd1fe70ab39635ce89c
+coresight	a90863095f84f6d17c49716e4e2212d28a6b25b5
+fastrpc		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+fpga		26e5e1be2e640a562618c1aa03ed15b62bfbf2d6
+icc		89b6f071c4a07267be82159b5118054fd7771ea9
+iio		ff0843ceb1fb11a6b73e0e77b932ef7967aecd4b
+phy-next	caf08514bbee0736c31d8d4f406e3415cdf726bb
+soundwire	b8f2d65fec19f3866905ac6ae3deb5c0c9faf162
+extcon		1bf0ba46d9d2c784120fd9cb235c08add3a6e7be
+gnss		f338e77383789c0cae23ca3d48adcc5e9e137e3c
+vfio		96ca4caf9066f5ebd35b561a521af588a8eb0215
+w1		f2c1ae05fe82030537b36012551a5cac28d00254
+spmi		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+staging		82e1c68ac206efe42854296c462aa83f541ea22c
+counter-next	ad415677b7e3b733270adaf04e3a7a9c46f1e929
+mux		59b723cd2adbac2a34fc8e12c74ae26ae45bf230
+dmaengine	b7560798466a07d9c3fb011698e92c335ab28baf
+cgroup		4c7c17b16c98e56507d57335faa1c92c466294c8
+scsi		8387c81a4eb9e0716e66d15c60151c97815c2e51
+scsi-mkp	48b2de8505437805116a3dc51d5afa09308c6659
+vhost		ebcff9dacaf2c1418f8bc927388186d7d3674603
+rpmsg		dfc5c561d9fc709adc3c06e197f6d8cad44d2aa2
+gpio-brgl	a6e53d05ab849779d55ca985566a1da7f22435b9
+gpio-intel	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pinctrl		7c839b398814aa4a4ca07c1ac8b158095d2efe34
+pinctrl-intel	1ca468e78ea97c3365befdd408f71bda4b295134
+pinctrl-renesas	fb22bb9701d48c4b0e81fe204c2f96a37a520568
+pinctrl-samsung	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pwm		07bee454fbf45fa36fdd505d97840b5412fde2e8
+ktest		229517449879b3ca8ca5588593cbea6a67ba0ad2
+kselftest	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+kunit		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+kunit-next	fa2c3ed5f079d41db0433a8dc829b9f6fb408aaf
+livepatching	920e5001f4beb38685d5b8cac061cb1d2760eeab
+rtc		5ff89ef425d17a43b1a48173482f8bfe2ce4fcd1
+nvdimm		a9ba6733c7f1096c4506bf4e34a546e07242df74
+at24		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+ntb		7b3302c687cae50d9506c77ad9d40f8c242e9c79
+seccomp		7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+slimbus		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+nvmem		8d48419d141cb39679a3a01782db3241660704a5
+hyperv		e733a9e281805f0d066b4ade092dff44baf1ec09
+auxdisplay	995a418a6ca33e466e5e1527663ae3d5eee18304
+kgdb		fdbdd0ccb30af18d3b29e714ac8d5ab6163279e0
+hmm		19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+cfi		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+mhi		f2d1643ddc0f3d0b847a6877ec37f1fabacfbfed
+memblock	f56ccc32468ee7885d3a9175e7d2cb608d301521
+cxl		d5f9bfc37906bbb737790af11f1537593f8778a5
+zstd		65d1f5507ed2c78c64fce40e44e5574a9419eb09
+efi		56e2ef8f5f9161d08d45656d373da83d4a789be1
+unicode		bcfee135d584714c2130031c7e28aafa91057b9a
+slab		ac9004d2224a4d522f3ad6d5eded46f5cb6d1be7
+random		bb9ff576fdff48c242876f55098a3ee20a29df5d
+landlock	0955b24481490fee5663867ff7510d90a8b37d4c
+sysctl		6932256d3a3764f3a5e06e2cb8603be45b6a9fef
+execve		46a03ea50b5f380bdb99178b8f90b39c6ba1f528
+bitmap		32a55dedc06aee4482ce3827274e3af19454012c
+hte		92dfd92f747698352b256cd9ddd7497bb7ebe9c8
+kspp		54031989e49fdd3fadb80f917e8a47ab64dd4ad6
+nolibc		d59131de97ee69ab4e4b8926089a6e953d699b11
+iommufd		5e3387f809556f2d226c7dab9c1f91d44a36041e
+turbostat	a23c1298f1593a42dfc78bbbbd1ecc3aabee5c26
+pwrseq		6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+capabilities-next	071588136007482d70fd2667b827036bc60b1f8f
+ipe		f37070985ed913f2d73bf3368676b5f2f96e4c54
+kcsan		dcb6fa37fd7bc9c3d2b066329b0d27dedf8becaa
+crc		6e4d63e8993c681e1cec7d564b4e018e21e658d0
+keys-next	965e9a2cf23b066d8bdeb690dff9cd7089c5f667
+fwctl		11439c4635edd669ae435eec308f4ab8a0804808
+devsec-tsm	43185067c6fd55b548ecb648a69d9569fcf622b5
+hisilicon	6c177775dcc5e70a64ddf4ee842c66af498f2c7c
+kthread		fa39ec4f89f2637ed1cdbcde3656825951787668
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 0000000000000..d0e7cc99506ec
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,413 @@
+Trees included into this release:
+
+Name		Url
+----		---
+origin		https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+ext4-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#fixes
+vfs-brauner-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.fixes
+fscrypt-current	https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+btrfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+erofs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+nfsd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+v9fs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+fscrypt		https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+btrfs		https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		https://github.com/ceph/ceph-client.git#master
+cifs		git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs	https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git#configfs-next
+ecryptfs	https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+dlm		https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+erofs		https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		https://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		https://github.com/smfrench/smb3-kernel.git#ksmbd-for-next
+nfs		git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git#ntfs-next
+ntfs3		https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git#for-next
+overlayfs	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		https://github.com/martinetd/linux#9p-next
+v9fs-ericvh	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+vfs-brauner	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+mm-hotfixes	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-hotfixes-unstable
+kbuild-current	https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-fixes-for-next
+arc-current	https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#fixes
+arm64-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+drivers-memory-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+sophgo-fixes	https://github.com/sophgo/linux.git#fixes
+sophgo-soc-fixes	https://github.com/sophgo/linux.git#soc-fixes
+m68k-current	https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+net		https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/#master
+ipsec		https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+wireless	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+ath		https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-current
+iwlwifi		https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#fixes
+wpan		https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-linus
+tty.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+counter-current	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+libcrypto-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-fixes
+vfio-fixes	https://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+dmaengine-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git://linuxtv.org/media-ci/media-pending.git#fixes
+reset-fixes	https://git.pengutronix.de/git/pza/linux#reset/fixes
+mips-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#fixes
+kvm-fixes	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+kvm-arm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#fixes
+hwmon-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+dma-mapping-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-fixes
+drivers-x86-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+devicetree-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	https://gitlab.freedesktop.org/drm/kernel.git#drm-fixes
+drm-intel-fixes	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next-fixes
+mmc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+risc-v-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+auxdisplay-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#fixes
+kunit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+memblock-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#fixes
+renesas-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+perf-current	https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git#perf-tools
+efi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+battery-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+iommufd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+w1-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+i2c-host-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host-fixes
+sparc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-linus
+clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-fixes
+thead-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-fixes
+tenstorrent-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-fixes
+pwrseq-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-current
+thead-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-fixes
+ftrace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ftrace/fixes
+ring-buffer-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ring-buffer/fixes
+trace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#trace/fixes
+tracefs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#tracefs/fixes
+spacemit-fixes	https://github.com/spacemit-com/linux#fixes
+tip-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#tip/urgent
+slab-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next-fixes
+drm-msm-fixes	https://gitlab.freedesktop.org/drm/msm.git#msm-fixes
+uml-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+fwctl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-rc
+devsec-tsm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#fixes
+drm-rust-fixes	https://gitlab.freedesktop.org/drm/rust/kernel.git#drm-rust-fixes
+tenstorrent-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-fixes
+drm-misc-fixes	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next-fixes
+rust		https://github.com/Rust-for-Linux/linux.git#rust-next
+rust-interop	https://github.com/Rust-for-Linux/linux.git#interop-next
+rust-alloc	https://github.com/Rust-for-Linux/linux.git#alloc-next
+rust-io		https://github.com/Rust-for-Linux/linux.git#io-next
+rust-pin-init	https://github.com/Rust-for-Linux/linux.git#pin-init-next
+rust-timekeeping	https://github.com/Rust-for-Linux/linux.git#timekeeping-next
+rust-xarray	https://github.com/Rust-for-Linux/linux.git#xarray-next
+rust-analyzer	https://github.com/Rust-for-Linux/linux.git#rust-analyzer-next
+mm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-stable
+mm-nonmm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-unstable
+mm-nonmm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-unstable
+kbuild		https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-for-next
+clang-format	https://github.com/ojeda/linux.git#clang-format
+perf		https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-for-next
+asm-generic	https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic#master
+alpha		https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git#alpha-next
+arm		https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#for-next
+arm64		https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+at91		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+bmc		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git#for-next
+broadcom	https://github.com/Broadcom/stblinux.git#next
+cix		https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git#for-next
+davinci		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+fsl		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git#soc_fsl
+imx-mxs		https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git#for-next
+mediatek	https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git#for-next
+mvebu		https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#for-next
+qcom		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+renesas		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		https://git.pengutronix.de/git/pza/linux#reset/next
+rockchip	https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+sophgo		https://github.com/sophgo/linux.git#for-next
+sophgo-soc	https://github.com/sophgo/linux.git#soc-for-next
+spacemit	https://github.com/spacemit-com/linux#for-next
+stm32		https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git#next
+tegra		https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+tenstorrent-dt	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-for-next
+thead-dt	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-for-next
+ti		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		https://github.com/Xilinx/linux-xlnx.git#for-next
+socfpga		https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git#for-next
+clk		https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+thead-clk	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-for-next
+tenstorrent-clk	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-for-next
+csky		https://github.com/c-sky/csky-linux.git#linux-next
+loongarch	https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	https://github.com/openrisc/linux.git#for-next
+parisc-hd	https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+risc-v		https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+sparc		https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-next
+uml		https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		https://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+printk		https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i2c-host	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host
+i3c		https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git://linuxtv.org/media-ci/media-pending.git#next
+v4l-dvb-next	git://linuxtv.org/mchehab/media-next.git#master
+pm		https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+rdma		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+ath-next	https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-next
+iwlwifi-next	https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#next
+wpan-next	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+drbd		https://github.com/LINBIT/linux-drbd.git#drbd-next
+mtd		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+libcrypto	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-next
+drm		https://gitlab.freedesktop.org/drm/kernel.git#drm-next
+drm-exynos	https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git#for-linux-next
+drm-misc	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next
+amdgpu		https://gitlab.freedesktop.org/agd5f/linux.git#drm-next
+drm-intel	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next
+drm-msm		https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+drm-xe		https://gitlab.freedesktop.org/drm/xe/kernel.git#drm-xe-next
+etnaviv		https://git.pengutronix.de/git/lst/linux#etnaviv/next
+drm-rust	https://gitlab.freedesktop.org/drm/rust/kernel.git#drm-rust-next
+drm-nova	https://gitlab.freedesktop.org/drm/nova.git#nova-next
+fbdev		https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git#modules-next
+input		https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git#for-next
+device-mapper	https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux#for-next
+pcmcia		https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+selinux		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		https://github.com/cschaufler/smack-next#next
+tomoyo		git://git.code.sf.net/p/tomoyo/tomoyo.git#master
+tpmdd		https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#next
+watchdog	git://www.linux-watchdog.org/linux-watchdog-next.git#master
+iommu		https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git#next
+audit		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+clockevents	https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git#timers/drivers/next
+edac		https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+ftrace		https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux#next
+paulmck		https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#non-rcu/next
+kvm		git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		https://github.com/kvm-x86/linux.git#next
+xen-tip		https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+sched-ext	https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git#for-next
+drivers-x86	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds-lj		https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		https://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-next
+usb		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+coresight	https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+phy-next	https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		https://github.com/awilliam/linux-vfio.git#next
+w1		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+spmi		https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git#spmi-next
+staging		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio-brgl	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pwm		https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#pwm/for-next
+ktest		https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git#for-next
+rtc		https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		https://github.com/jonmason/ntb.git#ntb-next
+seccomp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+slimbus		https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+hyperv		https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#for-next
+kgdb		https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+memblock	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#for-next
+cxl		https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		https://github.com/terrelln/linux.git#zstd-next
+efi		https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+slab		https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next
+random		https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+sysctl		https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git#sysctl-next
+execve		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		https://github.com/norov/linux.git#bitmap-for-next
+hte		https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+kspp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+nolibc		https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git#for-next
+iommufd		https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
+turbostat	https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+pwrseq		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-next
+capabilities-next	https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git#caps-next
+ipe		https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git#next
+kcsan		https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git#next
+crc		https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#crc-next
+keys-next	https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#keys-next
+fwctl		https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-next
+devsec-tsm	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#next
+hisilicon	https://github.com/hisilicon/linux-hisi.git#for-next
+kthread		https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git#for-next
diff --git a/Next/kselftest-targets b/Next/kselftest-targets
new file mode 100644
index 0000000000000..932bb653513e2
--- /dev/null
+++ b/Next/kselftest-targets
@@ -0,0 +1,109 @@
+acct
+amd-pstate
+breakpoints
+cachestat
+cgroup
+clone3
+connector
+core
+coredump
+cpufreq
+cpu-hotplug
+damon
+devices/error_logs
+devices/probe
+dmabuf-heaps
+drivers/dma-buf
+drivers/net/bonding
+drivers/net/team
+drivers/net/virtio_net
+dt
+efivarfs
+exec
+filelock
+filesystems
+filesystems/binderfs
+filesystems/epoll
+filesystems/fat
+filesystems/mount-notify
+filesystems/overlayfs
+filesystems/statmount
+firmware
+fpu
+ftrace
+futex
+gpio
+intel_pstate
+iommu
+ipc
+ir
+kcmp
+kmod
+kselftest_harness
+kvm
+landlock
+lib
+livepatch
+lkdtm
+locking
+lsm
+media_tests
+membarrier
+memory-hotplug
+mincore
+module
+mount
+mount_setattr
+move_mount_set_group
+mseal_system_mappings
+namespaces
+nci
+net/can
+net/forwarding
+net/hsr
+net/openvswitch
+net/packetdrill
+net/tcp_ao
+pcie_bwctrl
+pci_endpoint
+perf_events
+pidfd
+pid_namespace
+powerpc
+power_supply
+proc
+pstore
+ptp
+ptrace
+resctrl
+ring-buffer
+riscv
+rlimits
+rseq
+rtc
+rust
+safesetid
+sched
+seccomp
+sgx
+signal
+size
+sparc64
+splice
+static_keys
+sync
+syscall_user_dispatch
+sysctl
+tc-testing
+tdx
+timens
+timers
+tmpfs
+tpm2
+tty
+user_events
+vDSO
+verification
+watchdog
+x86
+zram
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 0000000000000..24eca65f1864d
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,11796 @@
+$ date -R
+Fri, 20 Mar 2026 12:38:29 +0000
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+Updating files:  59% (4141/6925)
Updating files:  60% (4155/6925)
Updating files:  61% (4225/6925)
Updating files:  62% (4294/6925)
Updating files:  63% (4363/6925)
Updating files:  64% (4432/6925)
Updating files:  65% (4502/6925)
Updating files:  66% (4571/6925)
Updating files:  67% (4640/6925)
Updating files:  68% (4709/6925)
Updating files:  69% (4779/6925)
Updating files:  70% (4848/6925)
Updating files:  71% (4917/6925)
Updating files:  72% (4986/6925)
Updating files:  73% (5056/6925)
Updating files:  74% (5125/6925)
Updating files:  75% (5194/6925)
Updating files:  76% (5263/6925)
Updating files:  77% (5333/6925)
Updating files:  78% (5402/6925)
Updating files:  79% (5471/6925)
Updating files:  80% (5540/6925)
Updating files:  81% (5610/6925)
Updating files:  82% (5679/6925)
Updating files:  83% (5748/6925)
Updating files:  84% (5817/6925)
Updating files:  85% (5887/6925)
Updating files:  86% (5956/6925)
Updating files:  87% (6025/6925)
Updating files:  88% (6094/6925)
Updating files:  89% (6164/6925)
Updating files:  90% (6233/6925)
Updating files:  91% (6302/6925)
Updating files:  92% (6371/6925)
Updating files:  93% (6441/6925)
Updating files:  94% (6510/6925)
Updating files:  95% (6579/6925)
Updating files:  96% (6648/6925)
Updating files:  97% (6718/6925)
Updating files:  98% (6787/6925)
Updating files:  99% (6856/6925)
Updating files: 100% (6925/6925)
Updating files: 100% (6925/6925), done.
+HEAD is now at 8a30aeb0d1b4e Merge tag 'nfsd-7.0-2' of git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux
+Merging origin/master (0e4f8f1a3d081 Merge tag 'parisc-for-7.0-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating 8a30aeb0d1b4e..0e4f8f1a3d081
+Fast-forward (no commit created; -m option ignored)
+ Documentation/netlink/specs/net_shaper.yaml        |  12 +-
+ arch/arm64/crypto/aes-neonbs-glue.c                |  37 +++--
+ arch/parisc/kernel/cache.c                         |   4 +-
+ drivers/acpi/acpi_processor.c                      |  15 +-
+ drivers/acpi/acpica/acpredef.h                     |   2 +-
+ drivers/acpi/bus.c                                 |   3 -
+ drivers/base/power/runtime.c                       |   1 +
+ drivers/bluetooth/btqca.c                          |   2 +
+ drivers/crypto/padlock-sha.c                       |   7 +
+ drivers/firewire/net.c                             |   5 +-
+ drivers/net/bonding/bond_debugfs.c                 |  16 ++-
+ drivers/net/bonding/bond_main.c                    |   8 +-
+ drivers/net/dsa/bcm_sf2.c                          |   8 +-
+ drivers/net/ethernet/airoha/airoha_eth.c           |   1 -
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |   2 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt.h          |   2 +-
+ drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c |   2 +-
+ drivers/net/ethernet/broadcom/tg3.c                |  11 ++
+ drivers/net/ethernet/cadence/macb_main.c           |  26 +++-
+ drivers/net/ethernet/cadence/macb_ptp.c            |   4 +-
+ drivers/net/ethernet/intel/iavf/iavf_main.c        |   9 +-
+ drivers/net/ethernet/intel/igc/igc.h               |   2 +
+ drivers/net/ethernet/intel/igc/igc_main.c          |  14 +-
+ drivers/net/ethernet/intel/igc/igc_ptp.c           |  33 +++++
+ drivers/net/ethernet/intel/libie/fwlog.c           |  49 +++++--
+ drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c    |   4 +-
+ .../ethernet/mellanox/mlx5/core/en_accel/ipsec.h   |   1 +
+ .../mellanox/mlx5/core/en_accel/ipsec_offload.c    |  52 +++----
+ drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c  |  23 ++-
+ drivers/net/ethernet/microsoft/mana/hw_channel.c   |   6 +-
+ drivers/net/ethernet/ti/icssg/icssg_common.c       |   5 +
+ drivers/net/netdevsim/netdev.c                     |   5 +-
+ drivers/net/usb/aqc111.c                           |  12 +-
+ drivers/net/usb/cdc_ncm.c                          |  10 +-
+ drivers/net/wireless/ath/ath9k/channel.c           |   6 +-
+ drivers/net/wireless/mediatek/mt76/scan.c          |   4 +-
+ drivers/net/wireless/ti/wlcore/tx.c                |   2 +-
+ drivers/net/wireless/virtual/mac80211_hwsim.c      |   3 +-
+ drivers/nfc/nxp-nci/i2c.c                          |   4 +-
+ drivers/pci/endpoint/functions/pci-epf-test.c      |   5 +
+ drivers/pci/pwrctrl/core.c                         |  54 +++++--
+ include/linux/etherdevice.h                        |   3 +-
+ include/linux/if_ether.h                           |   3 +-
+ include/linux/netdevice.h                          |   9 +-
+ include/net/ip_tunnels.h                           |  28 +++-
+ include/net/mac80211.h                             |   4 +-
+ include/net/netfilter/nf_tables.h                  |   6 +-
+ include/net/sch_generic.h                          |  33 +++++
+ include/net/udp_tunnel.h                           |   2 +-
+ kernel/sched/idle.c                                |  30 ++--
+ lib/crypto/Makefile                                |   3 +
+ net/atm/lec.c                                      |  72 ++++++----
+ net/atm/lec.h                                      |   2 +-
+ net/batman-adv/bat_iv_ogm.c                        |   3 +
+ net/bluetooth/hci_conn.c                           |   4 +-
+ net/bluetooth/hci_sync.c                           |   2 +-
+ net/bluetooth/hidp/core.c                          |  16 ++-
+ net/bluetooth/l2cap_core.c                         |  51 ++++---
+ net/bluetooth/mgmt.c                               |   7 +-
+ net/bluetooth/smp.c                                |   2 +-
+ net/bridge/br_cfm.c                                |   4 +-
+ net/ethernet/eth.c                                 |   9 +-
+ net/ipv4/icmp.c                                    |   4 +-
+ net/ipv4/ip_gre.c                                  |   3 +-
+ net/ipv6/exthdrs.c                                 |   4 +
+ net/ipv6/seg6_hmac.c                               |   2 +
+ net/mac80211/cfg.c                                 |  12 +-
+ net/mac80211/chan.c                                |   6 +-
+ net/mac80211/debugfs.c                             |  14 +-
+ net/mac80211/mesh.c                                |   3 +
+ net/mac80211/sta_info.c                            |   7 +-
+ net/mac80211/tdls.c                                |   2 +-
+ net/mac80211/tx.c                                  |   4 +-
+ net/mac802154/iface.c                              |   4 +-
+ net/mpls/af_mpls.c                                 |   1 +
+ net/mptcp/pm_kernel.c                              |   2 +-
+ net/netfilter/nf_bpf_link.c                        |   2 +-
+ net/netfilter/nf_conntrack_h323_asn1.c             |   4 +
+ net/netfilter/nf_conntrack_netlink.c               |  28 +++-
+ net/netfilter/nf_conntrack_proto_sctp.c            |   3 +-
+ net/netfilter/nf_conntrack_sip.c                   |   6 +-
+ net/netfilter/nf_flow_table_ip.c                   |   1 +
+ net/netfilter/nf_tables_api.c                      |  26 +---
+ net/netfilter/nfnetlink_osf.c                      |  13 ++
+ net/netfilter/nft_ct.c                             |   4 +
+ net/netfilter/nft_dynset.c                         |  10 +-
+ net/netfilter/nft_set_rbtree.c                     |  71 ++-------
+ net/netfilter/xt_CT.c                              |   4 +
+ net/netfilter/xt_time.c                            |   4 +-
+ net/phonet/af_phonet.c                             |   5 +-
+ net/rose/af_rose.c                                 |   5 +
+ net/sched/sch_generic.c                            |  27 ----
+ net/sched/sch_ingress.c                            |  14 +-
+ net/sched/sch_teql.c                               |   7 +-
+ net/shaper/shaper.c                                | 160 ++++++++++++---------
+ net/shaper/shaper_nl_gen.c                         |  12 +-
+ net/shaper/shaper_nl_gen.h                         |   5 +
+ net/smc/af_smc.c                                   |  23 ++-
+ net/smc/smc.h                                      |   5 +
+ net/smc/smc_close.c                                |   2 +-
+ net/unix/af_unix.c                                 |   2 +
+ net/unix/af_unix.h                                 |   1 +
+ net/unix/garbage.c                                 |  79 ++++++----
+ net/wireless/pmsr.c                                |   1 +
+ 104 files changed, 847 insertions(+), 509 deletions(-)
+Merging ext4-fixes/fixes (d250bdf531d9c ext4: fix iloc.bh leak in ext4_xattr_inode_update_ref)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4-fixes/fixes
+Already up to date.
+Merging vfs-brauner-fixes/vfs.fixes (e9075e420a1eb netfs: Fix NULL pointer dereference in netfs_unbuffered_write() on retry)
+$ git merge -m Merge branch 'vfs.fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner-fixes/vfs.fixes
+Merge made by the 'ort' strategy.
+ fs/netfs/direct_write.c | 14 +++++++++++---
+ fs/netfs/iterator.c     | 43 +++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 54 insertions(+), 3 deletions(-)
+Merging fscrypt-current/for-current (1f318b96cc84d Linux 7.0-rc3)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (1f318b96cc84d Linux 7.0-rc3)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging btrfs-fixes/next-fixes (1a30969e525d3 Merge branch 'misc-7.0' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+ fs/btrfs/backref.c         | 28 +++++++++++++
+ fs/btrfs/block-group.c     | 36 +++++++++++++++++
+ fs/btrfs/compression.c     | 11 ++++--
+ fs/btrfs/disk-io.c         | 20 ++++++++--
+ fs/btrfs/extent-tree.c     | 98 +++++++++++++++++++++++++++++++++++++++++++---
+ fs/btrfs/file-item.c       |  7 ++++
+ fs/btrfs/free-space-tree.c |  9 ++++-
+ fs/btrfs/inode.c           | 25 +++++++++---
+ fs/btrfs/ioctl.c           | 12 ++++--
+ fs/btrfs/lzo.c             |  4 +-
+ fs/btrfs/qgroup.c          |  8 ++++
+ fs/btrfs/raid56.c          | 12 +++++-
+ fs/btrfs/relocation.c      | 39 ++++++++++++++----
+ fs/btrfs/tree-checker.c    | 17 ++++++++
+ fs/btrfs/tree-log.c        | 21 ++++++++++
+ fs/btrfs/volumes.c         | 25 ++++++++----
+ fs/btrfs/zoned.c           |  7 ++++
+ fs/btrfs/zstd.c            |  2 +-
+ 18 files changed, 341 insertions(+), 40 deletions(-)
+Merging vfs-fixes/fixes (49c5d168a3a8f udf: fix nls leak on udf_fill_super() failure)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Auto-merging fs/udf/super.c
+Merge made by the 'ort' strategy.
+ fs/udf/super.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging erofs-fixes/fixes (9bbb89d45a5df erofs: harden h_shared_count in erofs_init_inode_xattrs())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Merge made by the 'ort' strategy.
+ fs/erofs/fileio.c | 6 ++----
+ fs/erofs/xattr.c  | 8 ++++++++
+ fs/erofs/zdata.c  | 3 +++
+ 3 files changed, 13 insertions(+), 4 deletions(-)
+Merging nfsd-fixes/nfsd-fixes (5133b61aaf437 nfsd: fix heap overflow in NFSv4.0 LOCK replay cache)
+$ git merge -m Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Already up to date.
+Merging v9fs-fixes/fixes/next (2014c95afecee Linux 6.14-rc1)
+$ git merge -m Merge branch 'fixes/next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging fscrypt/for-next (4377a22d84f72 ext4: use a byte granularity cursor in ext4_mpage_readpages)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Merge made by the 'ort' strategy.
+ fs/buffer.c                 | 18 +++++++++-
+ fs/crypto/bio.c             | 38 +++++++++-----------
+ fs/crypto/fscrypt_private.h |  3 --
+ fs/crypto/inline_crypt.c    | 86 +++++----------------------------------------
+ fs/crypto/keysetup.c        |  2 --
+ fs/ext4/inode.c             |  5 ++-
+ fs/ext4/page-io.c           | 28 +++++++++++----
+ fs/ext4/readpage.c          | 10 +++---
+ fs/f2fs/data.c              |  7 ++--
+ fs/f2fs/file.c              |  4 ++-
+ fs/iomap/direct-io.c        |  6 ++--
+ include/linux/fscrypt.h     | 37 +++++--------------
+ 12 files changed, 91 insertions(+), 153 deletions(-)
+Merging btrfs/for-next (41019679bc4f7 Merge branch 'for-next-next-v7.0-20260316' into for-next-20260316)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Auto-merging fs/btrfs/backref.c
+Auto-merging fs/btrfs/block-group.c
+Auto-merging fs/btrfs/compression.c
+CONFLICT (content): Merge conflict in fs/btrfs/compression.c
+Auto-merging fs/btrfs/disk-io.c
+CONFLICT (content): Merge conflict in fs/btrfs/disk-io.c
+Auto-merging fs/btrfs/extent-tree.c
+Auto-merging fs/btrfs/file-item.c
+Auto-merging fs/btrfs/inode.c
+CONFLICT (content): Merge conflict in fs/btrfs/inode.c
+Auto-merging fs/btrfs/ioctl.c
+Auto-merging fs/btrfs/lzo.c
+Auto-merging fs/btrfs/qgroup.c
+Auto-merging fs/btrfs/raid56.c
+Auto-merging fs/btrfs/relocation.c
+Auto-merging fs/btrfs/tree-checker.c
+Auto-merging fs/btrfs/tree-log.c
+Auto-merging fs/btrfs/volumes.c
+Auto-merging fs/btrfs/zoned.c
+Auto-merging fs/btrfs/zstd.c
+Resolved 'fs/btrfs/compression.c' using previous resolution.
+Resolved 'fs/btrfs/disk-io.c' using previous resolution.
+Resolved 'fs/btrfs/inode.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[fs-next e1d8683336afd] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git
+$ git diff -M --stat --summary HEAD^..
+ fs/btrfs/Kconfig                |   1 +
+ fs/btrfs/Makefile               |   4 +
+ fs/btrfs/backref.c              |  10 -
+ fs/btrfs/bio.c                  |   3 +-
+ fs/btrfs/block-group.c          | 329 +++++++++++---------
+ fs/btrfs/block-group.h          |   1 +
+ fs/btrfs/block-rsv.c            |  25 ++
+ fs/btrfs/compression.c          |   5 +-
+ fs/btrfs/compression.h          |   6 +-
+ fs/btrfs/ctree.c                |  36 ++-
+ fs/btrfs/delayed-ref.c          |  28 ++
+ fs/btrfs/dev-replace.c          |   4 +-
+ fs/btrfs/dir-item.c             |   4 +-
+ fs/btrfs/direct-io.c            |  11 +-
+ fs/btrfs/disk-io.c              | 228 +++++++-------
+ fs/btrfs/disk-io.h              |   7 +-
+ fs/btrfs/extent-tree.c          |  10 +-
+ fs/btrfs/extent_io.c            | 161 ++++++----
+ fs/btrfs/extent_io.h            |   8 +-
+ fs/btrfs/file-item.c            |   6 +-
+ fs/btrfs/file-item.h            |   6 +-
+ fs/btrfs/file.c                 |   4 +-
+ fs/btrfs/fs.h                   |   9 +-
+ fs/btrfs/inode.c                | 298 +++++++++---------
+ fs/btrfs/ioctl.c                |   4 +-
+ fs/btrfs/lzo.c                  |   7 +-
+ fs/btrfs/messages.h             |   8 +-
+ fs/btrfs/misc.h                 |  18 +-
+ fs/btrfs/ordered-data.c         |  67 ++--
+ fs/btrfs/ordered-data.h         |  66 ++--
+ fs/btrfs/print-tree.c           |   4 -
+ fs/btrfs/qgroup.c               |  10 +-
+ fs/btrfs/raid-stripe-tree.c     |   2 +-
+ fs/btrfs/raid56.c               |  17 +-
+ fs/btrfs/reflink.c              |   4 +-
+ fs/btrfs/relocation.c           |  12 +-
+ fs/btrfs/scrub.c                |  22 +-
+ fs/btrfs/send.c                 |   6 +-
+ fs/btrfs/space-info.c           |  63 ++--
+ fs/btrfs/space-info.h           |  22 +-
+ fs/btrfs/super.h                |   2 +-
+ fs/btrfs/tests/btrfs-tests.c    |   3 +
+ fs/btrfs/tests/btrfs-tests.h    |  10 +
+ fs/btrfs/tests/zoned-tests.c    | 675 ++++++++++++++++++++++++++++++++++++++++
+ fs/btrfs/transaction.c          |  78 +++--
+ fs/btrfs/transaction.h          |   3 +
+ fs/btrfs/tree-checker.c         | 122 ++++++++
+ fs/btrfs/tree-log.c             | 158 ++++++----
+ fs/btrfs/tree-mod-log.c         |   8 +-
+ fs/btrfs/uuid-tree.c            |   7 +-
+ fs/btrfs/volumes.c              | 186 ++++++++++-
+ fs/btrfs/volumes.h              |  34 ++
+ fs/btrfs/zlib.c                 |   2 +-
+ fs/btrfs/zoned.c                |  15 +-
+ fs/btrfs/zstd.c                 |   2 +-
+ include/trace/events/btrfs.h    |  24 ++
+ include/uapi/linux/btrfs_tree.h |   3 +-
+ 57 files changed, 2044 insertions(+), 824 deletions(-)
+ create mode 100644 fs/btrfs/tests/zoned-tests.c
+Merging ceph/master (1ff957199bd11 libceph: Remove obsolete session key alignment logic)
+$ git merge -m Merge branch 'master' of https://github.com/ceph/ceph-client.git ceph/master
+Merge made by the 'ort' strategy.
+ fs/ceph/addr.c          |  4 ++++
+ net/ceph/auth.c         |  2 +-
+ net/ceph/messenger_v2.c | 13 +++++--------
+ 3 files changed, 10 insertions(+), 9 deletions(-)
+Merging cifs/for-next (23b5df09c27ae smb: client: fix generic/694 due to wrong ->i_blocks)
+$ git merge -m Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6.git cifs/for-next
+Merge made by the 'ort' strategy.
+ fs/smb/client/cifsglob.h      |  6 ++++++
+ fs/smb/client/connect.c       |  4 ++++
+ fs/smb/client/file.c          |  1 -
+ fs/smb/client/inode.c         | 21 ++++++---------------
+ fs/smb/client/smb1transport.c |  2 +-
+ fs/smb/client/smb2ops.c       | 20 ++++----------------
+ 6 files changed, 21 insertions(+), 33 deletions(-)
+Merging configfs/configfs-next (6363844fdbbb7 samples: configfs: Constify struct configfs_item_operations and configfs_group_operations)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git configfs/configfs-next
+Already up to date.
+Merging ecryptfs/next (2a9cb23307db7 ecryptfs: Annotate struct ecryptfs_message with __counted_by)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Auto-merging fs/ecryptfs/ecryptfs_kernel.h
+Merge made by the 'ort' strategy.
+Merging dlm/next (2c3a0b730012e dlm: Avoid -Wflex-array-member-not-at-end warning)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Already up to date.
+Merging erofs/dev (11439c4635edd Linux 7.0-rc2)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Already up to date.
+Merging exfat/dev (efbb1375be25d exfat: fix s_maxbytes)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Merge made by the 'ort' strategy.
+ fs/exfat/balloc.c    | 18 ++++------
+ fs/exfat/dir.c       | 54 +++++++++--------------------
+ fs/exfat/exfat_fs.h  | 13 +++++--
+ fs/exfat/exfat_raw.h |  1 +
+ fs/exfat/fatent.c    | 95 ++++++++++++++++++++++++++++++++++++++++++----------
+ fs/exfat/file.c      | 42 +++++++++++++++++++++++
+ fs/exfat/inode.c     |  8 ++---
+ fs/exfat/misc.c      |  8 +++--
+ fs/exfat/namei.c     |  3 +-
+ fs/exfat/super.c     | 11 ++++--
+ 10 files changed, 171 insertions(+), 82 deletions(-)
+Merging ext3/for_next (b9755b4825901 Pull udf partition parsing fix.)
+$ git merge -m Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Auto-merging fs/udf/super.c
+Merge made by the 'ort' strategy.
+ fs/ext2/balloc.c                   |  4 +--
+ fs/ext2/inode.c                    |  3 ++-
+ fs/ext2/namei.c                    |  7 +++---
+ fs/notify/fanotify/fanotify_user.c | 50 ++++++++++++++++++--------------------
+ fs/notify/inotify/inotify_user.c   |  3 ++-
+ fs/udf/inode.c                     | 11 ++++-----
+ fs/udf/super.c                     |  4 ++-
+ 7 files changed, 41 insertions(+), 41 deletions(-)
+Merging ext4/dev (4f5e8e6f01234 et4: allow zeroout when doing written to unwritten split)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Already up to date.
+$ git am -3 ../patches/0001-ext4-kunit-extents-test-Fix-percpu_counters-list-cor.patch
+Applying: ext4: kunit: extents-test: Fix percpu_counters list corruption
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/ext4/extents-test.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[fs-next 6b8eb8bd6f56e] Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git
+ Date: Fri Mar 20 12:48:26 2026 +0000
+Merging f2fs/dev (5f04e90eedd05 f2fs: drop unused sbi parameter from f2fs_in_warm_node_list())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Auto-merging fs/f2fs/data.c
+Auto-merging fs/f2fs/file.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-fs-f2fs |  6 ++++++
+ fs/f2fs/data.c                          | 18 ++++--------------
+ fs/f2fs/debug.c                         |  1 +
+ fs/f2fs/f2fs.h                          |  7 ++++++-
+ fs/f2fs/file.c                          |  4 +++-
+ fs/f2fs/gc.c                            | 10 +++++++---
+ fs/f2fs/inline.c                        | 13 +++++++++----
+ fs/f2fs/namei.c                         |  1 +
+ fs/f2fs/node.c                          |  4 ++--
+ fs/f2fs/segment.c                       |  2 +-
+ fs/f2fs/sysfs.c                         | 10 ++++++++++
+ 11 files changed, 50 insertions(+), 26 deletions(-)
+Merging fsverity/for-next (1f318b96cc84d Linux 7.0-rc3)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Already up to date.
+Merging fuse/for-next (74b73faa4ab38 fuse: abort on fatal signal during sync init)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Merge made by the 'ort' strategy.
+ fs/fuse/control.c   |  4 ++--
+ fs/fuse/dev.c       | 61 +++++++++++++++++++++++++++++------------------------
+ fs/fuse/dev_uring.c | 36 ++++++++++++++-----------------
+ fs/fuse/dir.c       | 14 +++++++-----
+ fs/fuse/file.c      | 18 ++++++++++------
+ fs/fuse/fuse_i.h    |  1 +
+ fs/fuse/inode.c     |  1 +
+ fs/fuse/readdir.c   |  8 +++----
+ fs/fuse/virtio_fs.c | 29 +++++++++++++++++++++----
+ 9 files changed, 103 insertions(+), 69 deletions(-)
+Merging gfs2/for-next (163a287dee723 gfs2: less aggressive low-memory log flushing)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Auto-merging fs/gfs2/super.c
+Merge made by the 'ort' strategy.
+ fs/gfs2/aops.c  |  5 +++--
+ fs/gfs2/inode.c |  3 +--
+ fs/gfs2/super.c | 54 +++++++++++++++++++++++++++++++++++-------------------
+ 3 files changed, 39 insertions(+), 23 deletions(-)
+Merging jfs/jfs-next (dad98c5b2a05e jfs: avoid -Wtautological-constant-out-of-range-compare warning again)
+$ git merge -m Merge branch 'jfs-next' of https://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Merge made by the 'ort' strategy.
+ fs/jfs/jfs_dmap.c     | 114 +++++++++++++++++++++++++++++-
+ fs/jfs/jfs_dtree.c    | 192 ++++++++++++++++++++++++++++++++++++++++++++++++--
+ fs/jfs/jfs_dtree.h    |   4 ++
+ fs/jfs/jfs_imap.c     |   4 ++
+ fs/jfs/jfs_logmgr.c   |  34 ++++-----
+ fs/jfs/jfs_logmgr.h   |   7 ++
+ fs/jfs/jfs_metapage.c |   3 +-
+ fs/jfs/jfs_mount.c    |   3 +-
+ fs/jfs/jfs_txnmgr.c   |   3 +-
+ fs/jfs/jfs_umount.c   |  10 +++
+ 10 files changed, 344 insertions(+), 30 deletions(-)
+Merging ksmbd/ksmbd-for-next (c5f3a29cdc9f3 smb: smbdirect: fix inverted comparison operator in negotiation log message)
+$ git merge -m Merge branch 'ksmbd-for-next' of https://github.com/smfrench/smb3-kernel.git ksmbd/ksmbd-for-next
+Merge made by the 'ort' strategy.
+ fs/smb/Kconfig                                 |    1 +
+ fs/smb/client/Kconfig                          |    4 +-
+ fs/smb/client/cifs_debug.c                     |   67 +-
+ fs/smb/client/smb2pdu.c                        |    9 +-
+ fs/smb/client/smbdirect.c                      | 3116 ++----------------------
+ fs/smb/client/smbdirect.h                      |   19 +-
+ fs/smb/common/Makefile                         |    1 +
+ fs/smb/common/smbdirect/Kconfig                |    9 +
+ fs/smb/common/smbdirect/Makefile               |   18 +
+ fs/smb/common/smbdirect/smbdirect.h            |    9 +
+ fs/smb/common/smbdirect/smbdirect_accept.c     |  857 +++++++
+ fs/smb/common/smbdirect/smbdirect_connect.c    |  925 +++++++
+ fs/smb/common/smbdirect/smbdirect_connection.c | 2175 +++++++++++++++++
+ fs/smb/common/smbdirect/smbdirect_debug.c      |   88 +
+ fs/smb/common/smbdirect/smbdirect_devices.c    |  277 +++
+ fs/smb/common/smbdirect/smbdirect_internal.h   |  141 ++
+ fs/smb/common/smbdirect/smbdirect_listen.c     |  308 +++
+ fs/smb/common/smbdirect/smbdirect_main.c       |  121 +
+ fs/smb/common/smbdirect/smbdirect_mr.c         |  485 ++++
+ fs/smb/common/smbdirect/smbdirect_pdu.h        |    4 +
+ fs/smb/common/smbdirect/smbdirect_public.h     |  148 ++
+ fs/smb/common/smbdirect/smbdirect_rw.c         |  255 ++
+ fs/smb/common/smbdirect/smbdirect_socket.c     |  724 ++++++
+ fs/smb/common/smbdirect/smbdirect_socket.h     |  303 ++-
+ fs/smb/server/Kconfig                          |    5 +-
+ fs/smb/server/connection.c                     |    5 -
+ fs/smb/server/connection.h                     |    1 -
+ fs/smb/server/mgmt/tree_connect.c              |    9 +-
+ fs/smb/server/oplock.c                         |   72 +-
+ fs/smb/server/smb2pdu.c                        |   55 +-
+ fs/smb/server/transport_rdma.c                 | 2958 ++--------------------
+ fs/smb/server/transport_rdma.h                 |    4 +-
+ 32 files changed, 7321 insertions(+), 5852 deletions(-)
+ create mode 100644 fs/smb/common/smbdirect/Kconfig
+ create mode 100644 fs/smb/common/smbdirect/Makefile
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_accept.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_connect.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_connection.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_debug.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_devices.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_internal.h
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_listen.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_main.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_mr.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_public.h
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_rw.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_socket.c
+Merging nfs/linux-next (11439c4635edd Linux 7.0-rc2)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (4529e00154329 NFS: Fix NFS KConfig typos)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Already up to date.
+Merging nfsd/nfsd-next (084a0939af2b8 sunrpc: Add XPT flags missing from SVC_XPRT_FLAG_LIST)
+$ git merge -m Merge branch 'nfsd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Auto-merging fs/nfsd/nfs4xdr.c
+Auto-merging fs/nfsd/nfsctl.c
+Auto-merging fs/nfsd/state.h
+Auto-merging net/sunrpc/cache.c
+Merge made by the 'ort' strategy.
+ .../admin-guide/nfs/pnfs-block-server.rst          |   30 +
+ Documentation/admin-guide/nfs/pnfs-scsi-server.rst |   31 +
+ Documentation/filesystems/locking.rst              |    2 +
+ Documentation/sunrpc/xdr/nlm4.x                    |  211 +++
+ fs/lockd/Makefile                                  |   30 +-
+ fs/lockd/clnt4xdr.c                                |    5 +-
+ fs/lockd/clntlock.c                                |    2 +-
+ fs/lockd/clntproc.c                                |    2 +-
+ fs/lockd/clntxdr.c                                 |    3 +-
+ fs/lockd/host.c                                    |   31 +-
+ {include/linux => fs}/lockd/lockd.h                |  101 +-
+ fs/lockd/mon.c                                     |    2 +-
+ {include/linux => fs}/lockd/nlm.h                  |    8 +-
+ fs/lockd/nlm4xdr_gen.c                             |  724 ++++++++
+ fs/lockd/nlm4xdr_gen.h                             |   32 +
+ {include/linux => fs}/lockd/share.h                |   19 +-
+ fs/lockd/svc.c                                     |   50 +-
+ fs/lockd/svc4proc.c                                | 1786 ++++++++++++++------
+ fs/lockd/svclock.c                                 |   21 +-
+ fs/lockd/svcproc.c                                 |  126 +-
+ fs/lockd/svcshare.c                                |   40 +-
+ fs/lockd/svcsubs.c                                 |   32 +-
+ fs/lockd/trace.h                                   |    3 +-
+ fs/lockd/xdr.c                                     |    6 +-
+ {include/linux => fs}/lockd/xdr.h                  |   15 +-
+ fs/lockd/xdr4.c                                    |  347 ----
+ fs/locks.c                                         |   26 +-
+ fs/nfs/blocklayout/blocklayout.c                   |    4 +-
+ fs/nfs/nfs3proc.c                                  |    1 +
+ fs/nfs/proc.c                                      |    1 +
+ fs/nfs/sysfs.c                                     |    4 +-
+ fs/nfsd/Kconfig                                    |   10 -
+ fs/nfsd/blocklayout.c                              |  102 +-
+ fs/nfsd/debugfs.c                                  |    4 +
+ fs/nfsd/lockd.c                                    |   50 +-
+ fs/nfsd/nfs4layouts.c                              |  152 +-
+ fs/nfsd/nfs4proc.c                                 |    3 +-
+ fs/nfsd/nfs4state.c                                |   35 +-
+ fs/nfsd/nfs4xdr.c                                  |    3 +-
+ fs/nfsd/nfscache.c                                 |    3 +-
+ fs/nfsd/nfsctl.c                                   |    2 +-
+ fs/nfsd/nfsd.h                                     |    6 +
+ fs/nfsd/nfsfh.c                                    |    9 +-
+ fs/nfsd/nfssvc.c                                   |   10 +-
+ fs/nfsd/pnfs.h                                     |    5 +-
+ fs/nfsd/state.h                                    |    9 +
+ include/linux/filelock.h                           |    1 +
+ include/linux/lockd/bind.h                         |   26 +-
+ include/linux/lockd/debug.h                        |   40 -
+ include/linux/lockd/xdr4.h                         |   43 -
+ include/linux/sunrpc/debug.h                       |   10 +-
+ include/linux/sunrpc/sched.h                       |    3 -
+ include/linux/sunrpc/svc.h                         |    6 +-
+ include/linux/sunrpc/xdrgen/nlm4.h                 |  233 +++
+ include/trace/events/sunrpc.h                      |    4 +-
+ net/sunrpc/auth_gss/gss_krb5_test.c                |   93 +-
+ net/sunrpc/cache.c                                 |   62 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c           |   27 +-
+ 58 files changed, 3316 insertions(+), 1330 deletions(-)
+ create mode 100644 Documentation/sunrpc/xdr/nlm4.x
+ rename {include/linux => fs}/lockd/lockd.h (83%)
+ rename {include/linux => fs}/lockd/nlm.h (91%)
+ create mode 100644 fs/lockd/nlm4xdr_gen.c
+ create mode 100644 fs/lockd/nlm4xdr_gen.h
+ rename {include/linux => fs}/lockd/share.h (58%)
+ rename {include/linux => fs}/lockd/xdr.h (91%)
+ delete mode 100644 fs/lockd/xdr4.c
+ delete mode 100644 include/linux/lockd/debug.h
+ delete mode 100644 include/linux/lockd/xdr4.h
+ create mode 100644 include/linux/sunrpc/xdrgen/nlm4.h
+Merging ntfs/ntfs-next (77f58db7391e2 ntfs: fix ignoring unreachable code warnings)
+$ git merge -m Merge branch 'ntfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git ntfs/ntfs-next
+Auto-merging CREDITS
+Auto-merging MAINTAINERS
+Auto-merging fs/ntfs3/file.c
+Auto-merging fs/ntfs3/ntfs_fs.h
+Auto-merging fs/ntfs3/super.c
+Merge made by the 'ort' strategy.
+ CREDITS                             |    9 +-
+ Documentation/filesystems/index.rst |    1 +
+ Documentation/filesystems/ntfs.rst  |  159 +
+ MAINTAINERS                         |    9 +
+ fs/Kconfig                          |    1 +
+ fs/Makefile                         |    1 +
+ fs/ntfs/Kconfig                     |   48 +
+ fs/ntfs/Makefile                    |   10 +
+ fs/ntfs/aops.c                      |  263 ++
+ fs/ntfs/attrib.c                    | 5424 +++++++++++++++++++++++++++++++++++
+ fs/ntfs/attrib.h                    |  164 ++
+ fs/ntfs/attrlist.c                  |  289 ++
+ fs/ntfs/attrlist.h                  |   20 +
+ fs/ntfs/bdev-io.c                   |  117 +
+ fs/ntfs/bitmap.c                    |  287 ++
+ fs/ntfs/bitmap.h                    |  100 +
+ fs/ntfs/collate.c                   |  146 +
+ fs/ntfs/collate.h                   |   36 +
+ fs/ntfs/compress.c                  | 1577 ++++++++++
+ fs/ntfs/debug.c                     |  171 ++
+ fs/ntfs/debug.h                     |   63 +
+ fs/ntfs/dir.c                       | 1238 ++++++++
+ fs/ntfs/dir.h                       |   32 +
+ fs/ntfs/ea.c                        |  954 ++++++
+ fs/ntfs/ea.h                        |   30 +
+ fs/ntfs/file.c                      | 1161 ++++++++
+ fs/ntfs/index.c                     | 2117 ++++++++++++++
+ fs/ntfs/index.h                     |  111 +
+ fs/ntfs/inode.c                     | 3815 ++++++++++++++++++++++++
+ fs/ntfs/inode.h                     |  358 +++
+ fs/ntfs/iomap.c                     |  870 ++++++
+ fs/ntfs/iomap.h                     |   23 +
+ fs/ntfs/layout.h                    | 2346 +++++++++++++++
+ fs/ntfs/lcnalloc.c                  | 1049 +++++++
+ fs/ntfs/lcnalloc.h                  |  134 +
+ fs/ntfs/logfile.c                   |  778 +++++
+ fs/ntfs/logfile.h                   |  245 ++
+ fs/ntfs/malloc.h                    |   77 +
+ fs/ntfs/mft.c                       | 2924 +++++++++++++++++++
+ fs/ntfs/mft.h                       |   91 +
+ fs/ntfs/mst.c                       |  194 ++
+ fs/ntfs/namei.c                     | 1692 +++++++++++
+ fs/ntfs/ntfs.h                      |  294 ++
+ fs/ntfs/object_id.c                 |  158 +
+ fs/ntfs/object_id.h                 |   14 +
+ fs/ntfs/quota.c                     |   95 +
+ fs/ntfs/quota.h                     |   15 +
+ fs/ntfs/reparse.c                   |  573 ++++
+ fs/ntfs/reparse.h                   |   20 +
+ fs/ntfs/runlist.c                   | 2066 +++++++++++++
+ fs/ntfs/runlist.h                   |   97 +
+ fs/ntfs/super.c                     | 2771 ++++++++++++++++++
+ fs/ntfs/sysctl.c                    |   54 +
+ fs/ntfs/sysctl.h                    |   26 +
+ fs/ntfs/time.h                      |   87 +
+ fs/ntfs/unistr.c                    |  477 +++
+ fs/ntfs/upcase.c                    |   70 +
+ fs/ntfs/volume.h                    |  296 ++
+ fs/ntfs3/Kconfig                    |   10 +-
+ fs/ntfs3/dir.c                      |   10 -
+ fs/ntfs3/file.c                     |   11 -
+ fs/ntfs3/inode.c                    |   16 +-
+ fs/ntfs3/ntfs_fs.h                  |   11 -
+ fs/ntfs3/super.c                    |   59 +-
+ include/uapi/linux/fs.h             |   12 +
+ 65 files changed, 36258 insertions(+), 118 deletions(-)
+ create mode 100644 Documentation/filesystems/ntfs.rst
+ create mode 100644 fs/ntfs/Kconfig
+ create mode 100644 fs/ntfs/Makefile
+ create mode 100644 fs/ntfs/aops.c
+ create mode 100644 fs/ntfs/attrib.c
+ create mode 100644 fs/ntfs/attrib.h
+ create mode 100644 fs/ntfs/attrlist.c
+ create mode 100644 fs/ntfs/attrlist.h
+ create mode 100644 fs/ntfs/bdev-io.c
+ create mode 100644 fs/ntfs/bitmap.c
+ create mode 100644 fs/ntfs/bitmap.h
+ create mode 100644 fs/ntfs/collate.c
+ create mode 100644 fs/ntfs/collate.h
+ create mode 100644 fs/ntfs/compress.c
+ create mode 100644 fs/ntfs/debug.c
+ create mode 100644 fs/ntfs/debug.h
+ create mode 100644 fs/ntfs/dir.c
+ create mode 100644 fs/ntfs/dir.h
+ create mode 100644 fs/ntfs/ea.c
+ create mode 100644 fs/ntfs/ea.h
+ create mode 100644 fs/ntfs/file.c
+ create mode 100644 fs/ntfs/index.c
+ create mode 100644 fs/ntfs/index.h
+ create mode 100644 fs/ntfs/inode.c
+ create mode 100644 fs/ntfs/inode.h
+ create mode 100644 fs/ntfs/iomap.c
+ create mode 100644 fs/ntfs/iomap.h
+ create mode 100644 fs/ntfs/layout.h
+ create mode 100644 fs/ntfs/lcnalloc.c
+ create mode 100644 fs/ntfs/lcnalloc.h
+ create mode 100644 fs/ntfs/logfile.c
+ create mode 100644 fs/ntfs/logfile.h
+ create mode 100644 fs/ntfs/malloc.h
+ create mode 100644 fs/ntfs/mft.c
+ create mode 100644 fs/ntfs/mft.h
+ create mode 100644 fs/ntfs/mst.c
+ create mode 100644 fs/ntfs/namei.c
+ create mode 100644 fs/ntfs/ntfs.h
+ create mode 100644 fs/ntfs/object_id.c
+ create mode 100644 fs/ntfs/object_id.h
+ create mode 100644 fs/ntfs/quota.c
+ create mode 100644 fs/ntfs/quota.h
+ create mode 100644 fs/ntfs/reparse.c
+ create mode 100644 fs/ntfs/reparse.h
+ create mode 100644 fs/ntfs/runlist.c
+ create mode 100644 fs/ntfs/runlist.h
+ create mode 100644 fs/ntfs/super.c
+ create mode 100644 fs/ntfs/sysctl.c
+ create mode 100644 fs/ntfs/sysctl.h
+ create mode 100644 fs/ntfs/time.h
+ create mode 100644 fs/ntfs/unistr.c
+ create mode 100644 fs/ntfs/upcase.c
+ create mode 100644 fs/ntfs/volume.h
+Merging ntfs3/master (e98266e823a1f fs/ntfs3: prevent uninitialized lcn caused by zero len)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Auto-merging fs/ntfs3/inode.c
+Auto-merging fs/ntfs3/ntfs_fs.h
+Merge made by the 'ort' strategy.
+ fs/ntfs3/Makefile  | 20 ++++++++++++++
+ fs/ntfs3/frecord.c | 47 ++++++++++++++++++---------------
+ fs/ntfs3/index.c   | 76 +++++++++++++++++++++++++++++++++++-------------------
+ fs/ntfs3/inode.c   | 15 +++++++----
+ fs/ntfs3/ntfs_fs.h |  3 ---
+ 5 files changed, 105 insertions(+), 56 deletions(-)
+$ git am -3 ../patches/0001-ntfs3-Fix-merge-issue-with-ntfs3.patch
+Applying: ntfs3: Fix merge issue with ntfs3
+Using index info to reconstruct a base tree...
+M	fs/ntfs3/inode.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/ntfs3/inode.c
+No changes -- Patch already applied.
+Merging orangefs/for-next (9e835108a9ae1 fs/orangefs: Replace deprecated strcpy with memcpy + strscpy)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git orangefs/for-next
+Already up to date.
+Merging overlayfs/overlayfs-next (869056dbbd636 ovl: relax requirement for uuid=off,index=on)
+$ git merge -m Merge branch 'overlayfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Already up to date.
+Merging ubifs/next (11efa98bcc0d0 ubi: ubi.h: fix kernel-doc warnings)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Auto-merging drivers/mtd/ubi/ubi.h
+Merge made by the 'ort' strategy.
+ drivers/mtd/ubi/ubi.h | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+Merging v9fs/9p-next (da2519ee01067 9p: fix memory leak in v9fs_init_fs_context error path)
+$ git merge -m Merge branch '9p-next' of https://github.com/martinetd/linux v9fs/9p-next
+Auto-merging fs/9p/vfs_super.c
+Merge made by the 'ort' strategy.
+ fs/9p/vfs_super.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging v9fs-ericvh/ericvh/for-next (2014c95afecee Linux 6.14-rc1)
+$ git merge -m Merge branch 'ericvh/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (2c0ff6151c7bc Merge branch 'xfs-7.1-merge' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Merge made by the 'ort' strategy.
+ fs/xfs/libxfs/xfs_attr.h  |   3 +-
+ fs/xfs/libxfs/xfs_fs.h    |   5 +-
+ fs/xfs/scrub/quota.c      |   4 +-
+ fs/xfs/xfs_dquot_item.c   |   9 +++-
+ fs/xfs/xfs_file.c         |  95 ++++++++++++++++++++++++----------
+ fs/xfs/xfs_handle.c       |   2 +-
+ fs/xfs/xfs_inode_item.c   |   9 +++-
+ fs/xfs/xfs_ioctl.c        |  19 +++++++
+ fs/xfs/xfs_mount.c        |   7 +--
+ fs/xfs/xfs_trace.h        |  36 +++++++++++--
+ fs/xfs/xfs_trans_ail.c    | 127 +++++++++++++++++++++++++++-------------------
+ fs/xfs/xfs_verify_media.c |  18 +++----
+ fs/xfs/xfs_xattr.c        |   2 +-
+ fs/xfs/xfs_zone_alloc.c   |  44 ++++++++--------
+ fs/xfs/xfs_zone_gc.c      |  24 ++++++---
+ fs/xfs/xfs_zone_info.c    |   7 ++-
+ 16 files changed, 277 insertions(+), 134 deletions(-)
+Merging zonefs/for-next (a42938e80357a zonefs: correct some spelling mistakes)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Already up to date.
+Merging vfs-brauner/vfs.all (2a67c4361b77c Merge branch 'vfs-7.1.directory' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging fs/crypto/keysetup.c
+Auto-merging fs/ext2/inode.c
+Auto-merging fs/ext4/inode.c
+Auto-merging fs/ext4/page-io.c
+Auto-merging fs/f2fs/f2fs.h
+Auto-merging fs/f2fs/file.c
+Auto-merging fs/f2fs/gc.c
+Auto-merging fs/f2fs/inline.c
+Auto-merging fs/f2fs/namei.c
+Auto-merging fs/f2fs/node.c
+Auto-merging fs/fuse/file.c
+Auto-merging fs/gfs2/super.c
+Auto-merging fs/iomap/direct-io.c
+Auto-merging fs/jfs/jfs_imap.c
+Auto-merging fs/jfs/jfs_metapage.c
+Auto-merging fs/lockd/svclock.c
+Auto-merging fs/lockd/svcsubs.c
+Auto-merging fs/locks.c
+Auto-merging fs/nfsd/export.c
+Auto-merging fs/nfsd/nfs4state.c
+Auto-merging fs/nfsd/nfsfh.c
+Auto-merging fs/ntfs3/inode.c
+Auto-merging fs/ntfs3/super.c
+Auto-merging fs/udf/inode.c
+Auto-merging fs/udf/super.c
+Auto-merging kernel/fork.c
+Auto-merging net/rose/af_rose.c
+Auto-merging net/unix/af_unix.c
+Auto-merging security/apparmor/apparmorfs.c
+CONFLICT (content): Merge conflict in security/apparmor/apparmorfs.c
+Resolved 'security/apparmor/apparmorfs.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[fs-next 70de0a03d751a] Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/mount_api.rst            |    4 -
+ Documentation/filesystems/porting.rst              |   14 +
+ block/Makefile                                     |    2 +-
+ block/bio-integrity-auto.c                         |   80 +-
+ block/bio-integrity-fs.c                           |   81 ++
+ block/bio-integrity.c                              |   64 ++
+ block/bio.c                                        |   17 +-
+ block/blk-mq.c                                     |    6 +-
+ block/blk-settings.c                               |   13 -
+ block/blk.h                                        |    6 +-
+ block/t10-pi.c                                     |   12 +-
+ drivers/dma-buf/dma-buf.c                          |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    4 +-
+ drivers/nvdimm/btt.c                               |    6 +-
+ fs/9p/vfs_addr.c                                   |    4 +-
+ fs/9p/vfs_inode.c                                  |    6 +-
+ fs/9p/vfs_inode_dotl.c                             |    6 +-
+ fs/affs/amigaffs.c                                 |   10 +-
+ fs/affs/bitmap.c                                   |    2 +-
+ fs/affs/dir.c                                      |    2 +-
+ fs/affs/file.c                                     |   20 +-
+ fs/affs/inode.c                                    |   12 +-
+ fs/affs/namei.c                                    |   14 +-
+ fs/affs/symlink.c                                  |    2 +-
+ fs/afs/dir.c                                       |   10 +-
+ fs/afs/dir_search.c                                |    2 +-
+ fs/afs/dynroot.c                                   |    2 +-
+ fs/afs/inode.c                                     |    2 +-
+ fs/autofs/inode.c                                  |    2 +-
+ fs/befs/linuxvfs.c                                 |   28 +-
+ fs/bfs/dir.c                                       |    4 +-
+ fs/binfmt_elf_fdpic.c                              |    2 +-
+ fs/cachefiles/io.c                                 |    6 +-
+ fs/cachefiles/namei.c                              |  109 +-
+ fs/cachefiles/xattr.c                              |    2 +-
+ fs/ceph/crypto.c                                   |    4 +-
+ fs/coda/dir.c                                      |    2 +-
+ fs/coda/inode.c                                    |    2 +-
+ fs/cramfs/inode.c                                  |    2 +-
+ fs/crypto/crypto.c                                 |    2 +-
+ fs/crypto/hooks.c                                  |    2 +-
+ fs/crypto/keyring.c                                |    4 +-
+ fs/crypto/keysetup.c                               |    2 +-
+ fs/dcache.c                                        |    8 +-
+ fs/ecryptfs/crypto.c                               |    6 +-
+ fs/ecryptfs/file.c                                 |    2 +-
+ fs/efs/inode.c                                     |    6 +-
+ fs/eventpoll.c                                     |    2 +-
+ fs/exportfs/expfs.c                                |    4 +-
+ fs/ext2/dir.c                                      |   10 +-
+ fs/ext2/ialloc.c                                   |    9 +-
+ fs/ext2/inode.c                                    |    2 +-
+ fs/ext2/trace.h                                    |    8 +-
+ fs/ext2/xattr.c                                    |   16 +-
+ fs/ext4/dir.c                                      |    2 +-
+ fs/ext4/ext4.h                                     |    4 +-
+ fs/ext4/extents.c                                  |    8 +-
+ fs/ext4/extents_status.c                           |   28 +-
+ fs/ext4/fast_commit.c                              |    8 +-
+ fs/ext4/ialloc.c                                   |   10 +-
+ fs/ext4/indirect.c                                 |    2 +-
+ fs/ext4/inline.c                                   |   14 +-
+ fs/ext4/inode.c                                    |   22 +-
+ fs/ext4/ioctl.c                                    |    4 +-
+ fs/ext4/mballoc.c                                  |    6 +-
+ fs/ext4/migrate.c                                  |    2 +-
+ fs/ext4/move_extent.c                              |   20 +-
+ fs/ext4/namei.c                                    |   10 +-
+ fs/ext4/orphan.c                                   |   16 +-
+ fs/ext4/page-io.c                                  |   10 +-
+ fs/ext4/super.c                                    |   22 +-
+ fs/ext4/xattr.c                                    |   10 +-
+ fs/f2fs/compress.c                                 |    4 +-
+ fs/f2fs/dir.c                                      |    2 +-
+ fs/f2fs/extent_cache.c                             |    8 +-
+ fs/f2fs/f2fs.h                                     |    6 +-
+ fs/f2fs/file.c                                     |   12 +-
+ fs/f2fs/gc.c                                       |    2 +-
+ fs/f2fs/inline.c                                   |    4 +-
+ fs/f2fs/inode.c                                    |   48 +-
+ fs/f2fs/namei.c                                    |    8 +-
+ fs/f2fs/node.c                                     |   16 +-
+ fs/f2fs/recovery.c                                 |   10 +-
+ fs/f2fs/segment.h                                  |    2 +-
+ fs/f2fs/xattr.c                                    |   10 +-
+ fs/freevxfs/vxfs_bmap.c                            |    4 +-
+ fs/fs_context.c                                    |    1 -
+ fs/fs_parser.c                                     |   19 +-
+ fs/fs_struct.c                                     |   34 +-
+ fs/fserror.c                                       |    2 +-
+ fs/fuse/file.c                                     |    5 +-
+ fs/gfs2/super.c                                    |    2 +-
+ fs/hfs/catalog.c                                   |    2 +-
+ fs/hfs/extent.c                                    |    4 +-
+ fs/hfs/inode.c                                     |    4 +-
+ fs/hfsplus/attributes.c                            |   10 +-
+ fs/hfsplus/catalog.c                               |    2 +-
+ fs/hfsplus/dir.c                                   |    6 +-
+ fs/hfsplus/extents.c                               |    6 +-
+ fs/hfsplus/inode.c                                 |    8 +-
+ fs/hfsplus/super.c                                 |    6 +-
+ fs/hfsplus/xattr.c                                 |   10 +-
+ fs/hpfs/dir.c                                      |    4 +-
+ fs/hpfs/dnode.c                                    |    4 +-
+ fs/hpfs/ea.c                                       |    4 +-
+ fs/hpfs/inode.c                                    |    4 +-
+ fs/inode.c                                         |   49 +-
+ fs/iomap/bio.c                                     |  135 ++-
+ fs/iomap/buffered-io.c                             |    8 +-
+ fs/iomap/direct-io.c                               |   15 +-
+ fs/iomap/internal.h                                |   14 +
+ fs/iomap/ioend.c                                   |   32 +-
+ fs/iomap/trace.h                                   |    8 +-
+ fs/isofs/compress.c                                |    2 +-
+ fs/isofs/dir.c                                     |    2 +-
+ fs/isofs/inode.c                                   |    8 +-
+ fs/isofs/namei.c                                   |    2 +-
+ fs/jbd2/journal.c                                  |    4 +-
+ fs/jbd2/transaction.c                              |    2 +-
+ fs/jffs2/dir.c                                     |    4 +-
+ fs/jffs2/file.c                                    |    4 +-
+ fs/jffs2/fs.c                                      |   18 +-
+ fs/jfs/inode.c                                     |    2 +-
+ fs/jfs/jfs_imap.c                                  |    2 +-
+ fs/jfs/jfs_metapage.c                              |    2 +-
+ fs/kernfs/dir.c                                    |   15 +-
+ fs/kernfs/inode.c                                  |   99 +-
+ fs/kernfs/kernfs-internal.h                        |    5 +-
+ fs/libfs.c                                         |    3 +-
+ fs/lockd/svclock.c                                 |    8 +-
+ fs/lockd/svcsubs.c                                 |    2 +-
+ fs/locks.c                                         |    6 +-
+ fs/minix/inode.c                                   |   10 +-
+ fs/namei.c                                         |   70 +-
+ fs/namespace.c                                     |  214 ++--
+ fs/nfs/dir.c                                       |   20 +-
+ fs/nfs/file.c                                      |    8 +-
+ fs/nfs/filelayout/filelayout.c                     |    8 +-
+ fs/nfs/flexfilelayout/flexfilelayout.c             |    8 +-
+ fs/nfs/inode.c                                     |    6 +-
+ fs/nfs/internal.h                                  |    2 +-
+ fs/nfs/nfs4proc.c                                  |    4 +-
+ fs/nfs/pnfs.c                                      |   12 +-
+ fs/nfs/write.c                                     |    3 +-
+ fs/nfsd/export.c                                   |    2 +-
+ fs/nfsd/nfs4recover.c                              |    6 +-
+ fs/nfsd/nfs4state.c                                |    4 +-
+ fs/nfsd/nfsfh.c                                    |    4 +-
+ fs/nfsd/vfs.c                                      |    2 +-
+ fs/nilfs2/alloc.c                                  |   10 +-
+ fs/nilfs2/bmap.c                                   |    2 +-
+ fs/nilfs2/btnode.c                                 |    2 +-
+ fs/nilfs2/btree.c                                  |   12 +-
+ fs/nilfs2/dir.c                                    |   12 +-
+ fs/nilfs2/direct.c                                 |    4 +-
+ fs/nilfs2/gcinode.c                                |    2 +-
+ fs/nilfs2/inode.c                                  |    8 +-
+ fs/nilfs2/mdt.c                                    |    2 +-
+ fs/nilfs2/namei.c                                  |    2 +-
+ fs/nilfs2/segment.c                                |    2 +-
+ fs/notify/fdinfo.c                                 |    4 +-
+ fs/nsfs.c                                          |    4 +-
+ fs/ntfs3/inode.c                                   |   57 +-
+ fs/ntfs3/super.c                                   |    2 +-
+ fs/ocfs2/alloc.c                                   |    2 +-
+ fs/ocfs2/aops.c                                    |    4 +-
+ fs/ocfs2/dir.c                                     |    8 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |   10 +-
+ fs/ocfs2/extent_map.c                              |   12 +-
+ fs/ocfs2/inode.c                                   |    2 +-
+ fs/ocfs2/quota_local.c                             |    2 +-
+ fs/ocfs2/refcounttree.c                            |   10 +-
+ fs/ocfs2/xattr.c                                   |    4 +-
+ fs/orangefs/inode.c                                |    2 +-
+ fs/overlayfs/dir.c                                 |   51 +-
+ fs/overlayfs/export.c                              |   81 +-
+ fs/overlayfs/namei.c                               |    4 +-
+ fs/overlayfs/overlayfs.h                           |   10 +-
+ fs/overlayfs/super.c                               |   16 +-
+ fs/overlayfs/util.c                                |   27 +-
+ fs/pidfs.c                                         |  102 +-
+ fs/pipe.c                                          |    2 +-
+ fs/proc/fd.c                                       |    2 +-
+ fs/proc/self.c                                     |    3 -
+ fs/proc/task_mmu.c                                 |    4 +-
+ fs/proc/thread_self.c                              |    3 -
+ fs/qnx4/inode.c                                    |    4 +-
+ fs/qnx6/inode.c                                    |    2 +-
+ fs/ubifs/debug.c                                   |    8 +-
+ fs/ubifs/dir.c                                     |   28 +-
+ fs/ubifs/file.c                                    |   28 +-
+ fs/ubifs/journal.c                                 |    6 +-
+ fs/ubifs/super.c                                   |   16 +-
+ fs/ubifs/tnc.c                                     |    4 +-
+ fs/ubifs/xattr.c                                   |   14 +-
+ fs/udf/directory.c                                 |   18 +-
+ fs/udf/file.c                                      |    2 +-
+ fs/udf/inode.c                                     |   12 +-
+ fs/udf/namei.c                                     |    8 +-
+ fs/udf/super.c                                     |    2 +-
+ fs/ufs/balloc.c                                    |    6 +-
+ fs/ufs/dir.c                                       |   10 +-
+ fs/ufs/ialloc.c                                    |    6 +-
+ fs/ufs/inode.c                                     |   18 +-
+ fs/ufs/ufs_fs.h                                    |    6 +-
+ fs/ufs/util.c                                      |    2 +-
+ fs/verity/init.c                                   |    2 +-
+ fs/xattr.c                                         |  425 +++++---
+ fs/xfs/xfs_aops.c                                  |   47 +-
+ fs/xfs/xfs_iomap.c                                 |   11 +-
+ fs/zonefs/super.c                                  |    8 +-
+ fs/zonefs/trace.h                                  |   18 +-
+ include/linux/audit.h                              |    2 +-
+ include/linux/backing-dev.h                        |   13 +
+ include/linux/bio-integrity.h                      |   12 +-
+ include/linux/bio.h                                |    2 +-
+ include/linux/blk-integrity.h                      |   28 +-
+ include/linux/blkdev.h                             |   34 +-
+ include/linux/fs.h                                 |   88 +-
+ include/linux/fs_parser.h                          |    8 +-
+ include/linux/fs_struct.h                          |   43 +-
+ include/linux/iomap.h                              |   20 +-
+ include/linux/kernfs.h                             |    2 -
+ include/linux/namei.h                              |    6 -
+ include/linux/sched/signal.h                       |    1 +
+ include/linux/shmem_fs.h                           |    2 +-
+ include/linux/xattr.h                              |   47 +-
+ include/net/sock.h                                 |    4 +-
+ include/trace/events/cachefiles.h                  |   18 +-
+ include/trace/events/ext4.h                        |  550 +++++-----
+ include/trace/events/f2fs.h                        |  248 ++---
+ include/trace/events/filelock.h                    |   34 +-
+ include/trace/events/filemap.h                     |   20 +-
+ include/trace/events/fs_dax.h                      |   20 +-
+ include/trace/events/fsverity.h                    |   30 +-
+ include/trace/events/hugetlbfs.h                   |   42 +-
+ include/trace/events/netfs.h                       |    8 +-
+ include/trace/events/nilfs2.h                      |   12 +-
+ include/trace/events/readahead.h                   |   18 +-
+ include/trace/events/timestamp.h                   |   16 +-
+ include/trace/events/writeback.h                   |  162 +--
+ include/uapi/linux/mount.h                         |    1 +
+ include/uapi/linux/pidfd.h                         |    1 +
+ include/uapi/linux/sched.h                         |   14 +-
+ kernel/audit.h                                     |   13 +-
+ kernel/audit_fsnotify.c                            |    4 +-
+ kernel/audit_watch.c                               |   12 +-
+ kernel/auditsc.c                                   |   11 +-
+ kernel/events/uprobes.c                            |    4 +-
+ kernel/fork.c                                      |   68 +-
+ kernel/nsproxy.c                                   |   21 +-
+ kernel/ptrace.c                                    |    3 +-
+ kernel/signal.c                                    |    4 +
+ mm/shmem.c                                         |   46 +-
+ net/ax25/af_ax25.c                                 |    2 +-
+ net/bluetooth/af_bluetooth.c                       |    4 +-
+ net/can/bcm.c                                      |    2 +-
+ net/ipv4/ping.c                                    |    2 +-
+ net/ipv4/raw.c                                     |    2 +-
+ net/ipv4/tcp_ipv4.c                                |    2 +-
+ net/ipv4/udp.c                                     |    2 +-
+ net/ipv6/datagram.c                                |    2 +-
+ net/ipv6/tcp_ipv6.c                                |    2 +-
+ net/key/af_key.c                                   |    2 +-
+ net/netlink/af_netlink.c                           |    2 +-
+ net/netlink/diag.c                                 |    2 +-
+ net/netrom/af_netrom.c                             |    4 +-
+ net/packet/af_packet.c                             |    2 +-
+ net/packet/diag.c                                  |    2 +-
+ net/phonet/socket.c                                |    4 +-
+ net/rose/af_rose.c                                 |    4 +-
+ net/sctp/proc.c                                    |    4 +-
+ net/socket.c                                       |  121 ++-
+ net/unix/af_unix.c                                 |    2 +-
+ net/unix/diag.c                                    |    6 +-
+ net/x25/x25_proc.c                                 |    4 +-
+ net/xdp/xsk_diag.c                                 |    2 +-
+ security/apparmor/apparmorfs.c                     |   38 +-
+ security/integrity/integrity_audit.c               |    2 +-
+ security/ipe/audit.c                               |    2 +-
+ security/lsm_audit.c                               |   10 +-
+ security/selinux/hooks.c                           |   10 +-
+ security/selinux/selinuxfs.c                       |   17 +-
+ security/smack/smack_lsm.c                         |   12 +-
+ tools/include/uapi/linux/mount.h                   |   14 +-
+ tools/testing/selftests/Makefile                   |    1 +
+ tools/testing/selftests/filesystems/devpts_pts.c   |    2 -
+ .../selftests/filesystems/empty_mntns/.gitignore   |    4 +
+ .../selftests/filesystems/empty_mntns/Makefile     |   12 +
+ .../empty_mntns/clone3_empty_mntns_test.c          |  938 ++++++++++++++++
+ .../filesystems/empty_mntns/empty_mntns.h          |   50 +
+ .../filesystems/empty_mntns/empty_mntns_test.c     |  725 +++++++++++++
+ .../empty_mntns/overmount_chroot_test.c            |  225 ++++
+ .../selftests/filesystems/fsmount_ns/.gitignore    |    1 +
+ .../selftests/filesystems/fsmount_ns/Makefile      |   10 +
+ .../filesystems/fsmount_ns/fsmount_ns_test.c       | 1138 ++++++++++++++++++++
+ .../selftests/filesystems/move_mount/.gitignore    |    2 +
+ .../selftests/filesystems/move_mount/Makefile      |   10 +
+ .../filesystems/move_mount/move_mount_test.c       |  492 +++++++++
+ .../selftests/filesystems/open_tree_ns/Makefile    |    2 +-
+ .../filesystems/open_tree_ns/open_tree_ns_test.c   |   43 +-
+ .../selftests/filesystems/statmount/statmount.h    |   51 +
+ .../filesystems/statmount/statmount_test.c         |   45 +-
+ tools/testing/selftests/filesystems/utils.c        |    4 +-
+ tools/testing/selftests/filesystems/utils.h        |    2 +
+ .../testing/selftests/filesystems/xattr/.gitignore |    3 +
+ tools/testing/selftests/filesystems/xattr/Makefile |    6 +
+ .../filesystems/xattr/xattr_socket_test.c          |  470 ++++++++
+ .../filesystems/xattr/xattr_socket_types_test.c    |  177 +++
+ .../filesystems/xattr/xattr_sockfs_test.c          |  363 +++++++
+ tools/testing/selftests/pidfd/.gitignore           |    1 +
+ tools/testing/selftests/pidfd/Makefile             |    2 +-
+ .../testing/selftests/pidfd/pidfd_autoreap_test.c  |  900 ++++++++++++++++
+ 313 files changed, 8483 insertions(+), 2327 deletions(-)
+ create mode 100644 block/bio-integrity-fs.c
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/clone3_empty_mntns_test.c
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/empty_mntns.h
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/empty_mntns_test.c
+ create mode 100644 tools/testing/selftests/filesystems/empty_mntns/overmount_chroot_test.c
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/fsmount_ns_test.c
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/move_mount_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/xattr/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_types_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_sockfs_test.c
+ create mode 100644 tools/testing/selftests/pidfd/pidfd_autoreap_test.c
+$ git am -3 ../patches/0001-ntfs-Fixup-merge-issue-with-vfs.all-changes.patch
+Applying: ntfs: Fixup merge issue with vfs.all changes
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/ntfs/mft.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[fs-next 3ff384008ec60] Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+ Date: Fri Mar 20 12:49:16 2026 +0000
+$ git reset --hard HEAD^
+HEAD is now at 7a97780ddf23a Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git
+Merging next-20260312 version of vfs-brauner
+$ git merge -m next-20260312/vfs-brauner a4e79b1233089509af213d59f7aeef3926c15599
+Auto-merging fs/crypto/keysetup.c
+Auto-merging fs/ext2/inode.c
+Auto-merging fs/ext4/inode.c
+Auto-merging fs/ext4/page-io.c
+Auto-merging fs/f2fs/f2fs.h
+Auto-merging fs/f2fs/file.c
+Auto-merging fs/f2fs/gc.c
+Auto-merging fs/f2fs/inline.c
+Auto-merging fs/f2fs/namei.c
+Auto-merging fs/f2fs/node.c
+Auto-merging fs/fuse/file.c
+Auto-merging fs/gfs2/super.c
+Auto-merging fs/iomap/direct-io.c
+Auto-merging fs/jfs/jfs_imap.c
+Auto-merging fs/jfs/jfs_metapage.c
+Auto-merging fs/lockd/svclock.c
+Auto-merging fs/lockd/svcsubs.c
+Auto-merging fs/locks.c
+Auto-merging fs/nfsd/export.c
+Auto-merging fs/nfsd/nfs4state.c
+Auto-merging fs/nfsd/nfsfh.c
+Auto-merging fs/ntfs3/inode.c
+Auto-merging fs/ntfs3/super.c
+Auto-merging fs/udf/inode.c
+Auto-merging fs/udf/super.c
+Auto-merging kernel/fork.c
+Auto-merging net/rose/af_rose.c
+Auto-merging net/unix/af_unix.c
+Auto-merging security/apparmor/apparmorfs.c
+CONFLICT (content): Merge conflict in security/apparmor/apparmorfs.c
+Resolved 'security/apparmor/apparmorfs.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[fs-next 138338e9e3cd7] next-20260312/vfs-brauner
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/mount_api.rst            |    4 -
+ Documentation/filesystems/porting.rst              |   14 +
+ block/Makefile                                     |    2 +-
+ block/bio-integrity-auto.c                         |   80 +-
+ block/bio-integrity-fs.c                           |   81 ++
+ block/bio-integrity.c                              |   64 ++
+ block/bio.c                                        |   17 +-
+ block/blk-mq.c                                     |    6 +-
+ block/blk-settings.c                               |   13 -
+ block/blk.h                                        |    6 +-
+ block/t10-pi.c                                     |   12 +-
+ drivers/dma-buf/dma-buf.c                          |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    4 +-
+ drivers/nvdimm/btt.c                               |    6 +-
+ fs/9p/vfs_addr.c                                   |    4 +-
+ fs/9p/vfs_inode.c                                  |    6 +-
+ fs/9p/vfs_inode_dotl.c                             |    6 +-
+ fs/affs/amigaffs.c                                 |   10 +-
+ fs/affs/bitmap.c                                   |    2 +-
+ fs/affs/dir.c                                      |    2 +-
+ fs/affs/file.c                                     |   20 +-
+ fs/affs/inode.c                                    |   12 +-
+ fs/affs/namei.c                                    |   14 +-
+ fs/affs/symlink.c                                  |    2 +-
+ fs/afs/dir.c                                       |   10 +-
+ fs/afs/dir_search.c                                |    2 +-
+ fs/afs/dynroot.c                                   |    2 +-
+ fs/afs/inode.c                                     |    2 +-
+ fs/autofs/inode.c                                  |    2 +-
+ fs/befs/linuxvfs.c                                 |   28 +-
+ fs/bfs/dir.c                                       |    4 +-
+ fs/binfmt_elf_fdpic.c                              |    2 +-
+ fs/cachefiles/io.c                                 |    6 +-
+ fs/cachefiles/namei.c                              |  109 +-
+ fs/cachefiles/xattr.c                              |    2 +-
+ fs/ceph/crypto.c                                   |    4 +-
+ fs/coda/dir.c                                      |    2 +-
+ fs/coda/inode.c                                    |    2 +-
+ fs/cramfs/inode.c                                  |    2 +-
+ fs/crypto/crypto.c                                 |    2 +-
+ fs/crypto/hooks.c                                  |    2 +-
+ fs/crypto/keyring.c                                |    4 +-
+ fs/crypto/keysetup.c                               |    2 +-
+ fs/dcache.c                                        |    8 +-
+ fs/ecryptfs/crypto.c                               |    6 +-
+ fs/ecryptfs/file.c                                 |    2 +-
+ fs/efs/inode.c                                     |    6 +-
+ fs/eventpoll.c                                     |    2 +-
+ fs/exportfs/expfs.c                                |    4 +-
+ fs/ext2/dir.c                                      |   10 +-
+ fs/ext2/ialloc.c                                   |    9 +-
+ fs/ext2/inode.c                                    |    2 +-
+ fs/ext2/trace.h                                    |    8 +-
+ fs/ext2/xattr.c                                    |   16 +-
+ fs/ext4/dir.c                                      |    2 +-
+ fs/ext4/ext4.h                                     |    4 +-
+ fs/ext4/extents.c                                  |    8 +-
+ fs/ext4/extents_status.c                           |   28 +-
+ fs/ext4/fast_commit.c                              |    8 +-
+ fs/ext4/ialloc.c                                   |   10 +-
+ fs/ext4/indirect.c                                 |    2 +-
+ fs/ext4/inline.c                                   |   14 +-
+ fs/ext4/inode.c                                    |   22 +-
+ fs/ext4/ioctl.c                                    |    4 +-
+ fs/ext4/mballoc.c                                  |    6 +-
+ fs/ext4/migrate.c                                  |    2 +-
+ fs/ext4/move_extent.c                              |   20 +-
+ fs/ext4/namei.c                                    |   10 +-
+ fs/ext4/orphan.c                                   |   16 +-
+ fs/ext4/page-io.c                                  |   10 +-
+ fs/ext4/super.c                                    |   22 +-
+ fs/ext4/xattr.c                                    |   10 +-
+ fs/f2fs/compress.c                                 |    4 +-
+ fs/f2fs/dir.c                                      |    2 +-
+ fs/f2fs/extent_cache.c                             |    8 +-
+ fs/f2fs/f2fs.h                                     |    6 +-
+ fs/f2fs/file.c                                     |   12 +-
+ fs/f2fs/gc.c                                       |    2 +-
+ fs/f2fs/inline.c                                   |    4 +-
+ fs/f2fs/inode.c                                    |   48 +-
+ fs/f2fs/namei.c                                    |    8 +-
+ fs/f2fs/node.c                                     |   16 +-
+ fs/f2fs/recovery.c                                 |   10 +-
+ fs/f2fs/segment.h                                  |    2 +-
+ fs/f2fs/xattr.c                                    |   10 +-
+ fs/freevxfs/vxfs_bmap.c                            |    4 +-
+ fs/fs_context.c                                    |    1 -
+ fs/fs_parser.c                                     |   19 +-
+ fs/fs_struct.c                                     |   34 +-
+ fs/fserror.c                                       |    2 +-
+ fs/fuse/file.c                                     |    5 +-
+ fs/gfs2/super.c                                    |    2 +-
+ fs/hfs/catalog.c                                   |    2 +-
+ fs/hfs/extent.c                                    |    4 +-
+ fs/hfs/inode.c                                     |    4 +-
+ fs/hfsplus/attributes.c                            |   10 +-
+ fs/hfsplus/catalog.c                               |    2 +-
+ fs/hfsplus/dir.c                                   |    6 +-
+ fs/hfsplus/extents.c                               |    6 +-
+ fs/hfsplus/inode.c                                 |    8 +-
+ fs/hfsplus/super.c                                 |    6 +-
+ fs/hfsplus/xattr.c                                 |   10 +-
+ fs/hpfs/dir.c                                      |    4 +-
+ fs/hpfs/dnode.c                                    |    4 +-
+ fs/hpfs/ea.c                                       |    4 +-
+ fs/hpfs/inode.c                                    |    4 +-
+ fs/inode.c                                         |   49 +-
+ fs/iomap/bio.c                                     |  135 ++-
+ fs/iomap/buffered-io.c                             |    8 +-
+ fs/iomap/direct-io.c                               |   15 +-
+ fs/iomap/internal.h                                |   14 +
+ fs/iomap/ioend.c                                   |   32 +-
+ fs/iomap/trace.h                                   |    8 +-
+ fs/isofs/compress.c                                |    2 +-
+ fs/isofs/dir.c                                     |    2 +-
+ fs/isofs/inode.c                                   |    8 +-
+ fs/isofs/namei.c                                   |    2 +-
+ fs/jbd2/journal.c                                  |    4 +-
+ fs/jbd2/transaction.c                              |    2 +-
+ fs/jffs2/dir.c                                     |    4 +-
+ fs/jffs2/file.c                                    |    4 +-
+ fs/jffs2/fs.c                                      |   18 +-
+ fs/jfs/inode.c                                     |    2 +-
+ fs/jfs/jfs_imap.c                                  |    2 +-
+ fs/jfs/jfs_metapage.c                              |    2 +-
+ fs/kernfs/dir.c                                    |   15 +-
+ fs/kernfs/inode.c                                  |   99 +-
+ fs/kernfs/kernfs-internal.h                        |    5 +-
+ fs/libfs.c                                         |    3 +-
+ fs/lockd/svclock.c                                 |    8 +-
+ fs/lockd/svcsubs.c                                 |    2 +-
+ fs/locks.c                                         |    6 +-
+ fs/minix/inode.c                                   |   10 +-
+ fs/namei.c                                         |   70 +-
+ fs/namespace.c                                     |  125 ++-
+ fs/nfs/dir.c                                       |   20 +-
+ fs/nfs/file.c                                      |    8 +-
+ fs/nfs/filelayout/filelayout.c                     |    8 +-
+ fs/nfs/flexfilelayout/flexfilelayout.c             |    8 +-
+ fs/nfs/inode.c                                     |    6 +-
+ fs/nfs/internal.h                                  |    2 +-
+ fs/nfs/nfs4proc.c                                  |    4 +-
+ fs/nfs/pnfs.c                                      |   12 +-
+ fs/nfs/write.c                                     |    3 +-
+ fs/nfsd/export.c                                   |    2 +-
+ fs/nfsd/nfs4recover.c                              |    6 +-
+ fs/nfsd/nfs4state.c                                |    4 +-
+ fs/nfsd/nfsfh.c                                    |    4 +-
+ fs/nfsd/vfs.c                                      |    2 +-
+ fs/nilfs2/alloc.c                                  |   10 +-
+ fs/nilfs2/bmap.c                                   |    2 +-
+ fs/nilfs2/btnode.c                                 |    2 +-
+ fs/nilfs2/btree.c                                  |   12 +-
+ fs/nilfs2/dir.c                                    |   12 +-
+ fs/nilfs2/direct.c                                 |    4 +-
+ fs/nilfs2/gcinode.c                                |    2 +-
+ fs/nilfs2/inode.c                                  |    8 +-
+ fs/nilfs2/mdt.c                                    |    2 +-
+ fs/nilfs2/namei.c                                  |    2 +-
+ fs/nilfs2/segment.c                                |    2 +-
+ fs/notify/fdinfo.c                                 |    4 +-
+ fs/nsfs.c                                          |    4 +-
+ fs/ntfs3/inode.c                                   |   57 +-
+ fs/ntfs3/super.c                                   |    2 +-
+ fs/ocfs2/alloc.c                                   |    2 +-
+ fs/ocfs2/aops.c                                    |    4 +-
+ fs/ocfs2/dir.c                                     |    8 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |   10 +-
+ fs/ocfs2/extent_map.c                              |   12 +-
+ fs/ocfs2/inode.c                                   |    2 +-
+ fs/ocfs2/quota_local.c                             |    2 +-
+ fs/ocfs2/refcounttree.c                            |   10 +-
+ fs/ocfs2/xattr.c                                   |    4 +-
+ fs/orangefs/inode.c                                |    2 +-
+ fs/overlayfs/dir.c                                 |   51 +-
+ fs/overlayfs/export.c                              |   81 +-
+ fs/overlayfs/namei.c                               |    4 +-
+ fs/overlayfs/overlayfs.h                           |   10 +-
+ fs/overlayfs/super.c                               |   16 +-
+ fs/overlayfs/util.c                                |   27 +-
+ fs/pidfs.c                                         |  102 +-
+ fs/pipe.c                                          |    2 +-
+ fs/proc/fd.c                                       |    2 +-
+ fs/proc/self.c                                     |    3 -
+ fs/proc/task_mmu.c                                 |    4 +-
+ fs/proc/thread_self.c                              |    3 -
+ fs/qnx4/inode.c                                    |    4 +-
+ fs/qnx6/inode.c                                    |    2 +-
+ fs/ubifs/debug.c                                   |    8 +-
+ fs/ubifs/dir.c                                     |   28 +-
+ fs/ubifs/file.c                                    |   28 +-
+ fs/ubifs/journal.c                                 |    6 +-
+ fs/ubifs/super.c                                   |   16 +-
+ fs/ubifs/tnc.c                                     |    4 +-
+ fs/ubifs/xattr.c                                   |   14 +-
+ fs/udf/directory.c                                 |   18 +-
+ fs/udf/file.c                                      |    2 +-
+ fs/udf/inode.c                                     |   12 +-
+ fs/udf/namei.c                                     |    8 +-
+ fs/udf/super.c                                     |    2 +-
+ fs/ufs/balloc.c                                    |    6 +-
+ fs/ufs/dir.c                                       |   10 +-
+ fs/ufs/ialloc.c                                    |    6 +-
+ fs/ufs/inode.c                                     |   18 +-
+ fs/ufs/ufs_fs.h                                    |    6 +-
+ fs/ufs/util.c                                      |    2 +-
+ fs/verity/init.c                                   |    2 +-
+ fs/xattr.c                                         |  425 +++++---
+ fs/xfs/xfs_aops.c                                  |   47 +-
+ fs/xfs/xfs_iomap.c                                 |   11 +-
+ fs/zonefs/super.c                                  |    8 +-
+ fs/zonefs/trace.h                                  |   18 +-
+ include/linux/audit.h                              |    2 +-
+ include/linux/backing-dev.h                        |   13 +
+ include/linux/bio-integrity.h                      |   12 +-
+ include/linux/bio.h                                |    2 +-
+ include/linux/blk-integrity.h                      |   28 +-
+ include/linux/blkdev.h                             |   34 +-
+ include/linux/fs.h                                 |   88 +-
+ include/linux/fs_parser.h                          |    8 +-
+ include/linux/fs_struct.h                          |   43 +-
+ include/linux/iomap.h                              |   20 +-
+ include/linux/kernfs.h                             |    2 -
+ include/linux/namei.h                              |    6 -
+ include/linux/sched/signal.h                       |    1 +
+ include/linux/shmem_fs.h                           |    2 +-
+ include/linux/xattr.h                              |   47 +-
+ include/net/sock.h                                 |    4 +-
+ include/trace/events/cachefiles.h                  |   18 +-
+ include/trace/events/ext4.h                        |  550 +++++-----
+ include/trace/events/f2fs.h                        |  248 ++---
+ include/trace/events/filelock.h                    |   34 +-
+ include/trace/events/filemap.h                     |   20 +-
+ include/trace/events/fs_dax.h                      |   20 +-
+ include/trace/events/fsverity.h                    |   30 +-
+ include/trace/events/hugetlbfs.h                   |   42 +-
+ include/trace/events/netfs.h                       |    8 +-
+ include/trace/events/nilfs2.h                      |   12 +-
+ include/trace/events/readahead.h                   |   18 +-
+ include/trace/events/timestamp.h                   |   16 +-
+ include/trace/events/writeback.h                   |  162 +--
+ include/uapi/linux/mount.h                         |    1 +
+ include/uapi/linux/pidfd.h                         |    1 +
+ include/uapi/linux/sched.h                         |    3 +
+ kernel/audit.h                                     |   13 +-
+ kernel/audit_fsnotify.c                            |    4 +-
+ kernel/audit_watch.c                               |   12 +-
+ kernel/auditsc.c                                   |   11 +-
+ kernel/events/uprobes.c                            |    4 +-
+ kernel/fork.c                                      |   52 +-
+ kernel/ptrace.c                                    |    3 +-
+ kernel/signal.c                                    |    4 +
+ mm/shmem.c                                         |   46 +-
+ net/ax25/af_ax25.c                                 |    2 +-
+ net/bluetooth/af_bluetooth.c                       |    4 +-
+ net/can/bcm.c                                      |    2 +-
+ net/ipv4/ping.c                                    |    2 +-
+ net/ipv4/raw.c                                     |    2 +-
+ net/ipv4/tcp_ipv4.c                                |    2 +-
+ net/ipv4/udp.c                                     |    2 +-
+ net/ipv6/datagram.c                                |    2 +-
+ net/ipv6/tcp_ipv6.c                                |    2 +-
+ net/key/af_key.c                                   |    2 +-
+ net/netlink/af_netlink.c                           |    2 +-
+ net/netlink/diag.c                                 |    2 +-
+ net/netrom/af_netrom.c                             |    4 +-
+ net/packet/af_packet.c                             |    2 +-
+ net/packet/diag.c                                  |    2 +-
+ net/phonet/socket.c                                |    4 +-
+ net/rose/af_rose.c                                 |    4 +-
+ net/sctp/proc.c                                    |    4 +-
+ net/socket.c                                       |  121 ++-
+ net/unix/af_unix.c                                 |    2 +-
+ net/unix/diag.c                                    |    6 +-
+ net/x25/x25_proc.c                                 |    4 +-
+ net/xdp/xsk_diag.c                                 |    2 +-
+ security/apparmor/apparmorfs.c                     |   38 +-
+ security/integrity/integrity_audit.c               |    2 +-
+ security/ipe/audit.c                               |    2 +-
+ security/lsm_audit.c                               |   10 +-
+ security/selinux/hooks.c                           |   10 +-
+ security/selinux/selinuxfs.c                       |   17 +-
+ security/smack/smack_lsm.c                         |   12 +-
+ tools/include/uapi/linux/mount.h                   |   14 +-
+ tools/testing/selftests/Makefile                   |    1 +
+ tools/testing/selftests/filesystems/devpts_pts.c   |    2 -
+ .../selftests/filesystems/fsmount_ns/.gitignore    |    1 +
+ .../selftests/filesystems/fsmount_ns/Makefile      |   10 +
+ .../filesystems/fsmount_ns/fsmount_ns_test.c       | 1138 ++++++++++++++++++++
+ .../selftests/filesystems/move_mount/.gitignore    |    2 +
+ .../selftests/filesystems/move_mount/Makefile      |   10 +
+ .../filesystems/move_mount/move_mount_test.c       |  492 +++++++++
+ .../selftests/filesystems/open_tree_ns/Makefile    |    2 +-
+ .../filesystems/open_tree_ns/open_tree_ns_test.c   |   43 +-
+ .../selftests/filesystems/statmount/statmount.h    |   51 +
+ .../filesystems/statmount/statmount_test.c         |   45 +-
+ .../testing/selftests/filesystems/xattr/.gitignore |    3 +
+ tools/testing/selftests/filesystems/xattr/Makefile |    6 +
+ .../filesystems/xattr/xattr_socket_test.c          |  470 ++++++++
+ .../filesystems/xattr/xattr_socket_types_test.c    |  177 +++
+ .../filesystems/xattr/xattr_sockfs_test.c          |  363 +++++++
+ tools/testing/selftests/pidfd/.gitignore           |    1 +
+ tools/testing/selftests/pidfd/Makefile             |    2 +-
+ .../testing/selftests/pidfd/pidfd_autoreap_test.c  |  900 ++++++++++++++++
+ 304 files changed, 6425 insertions(+), 2288 deletions(-)
+ create mode 100644 block/bio-integrity-fs.c
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/fsmount_ns_test.c
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/move_mount_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/xattr/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_types_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_sockfs_test.c
+ create mode 100644 tools/testing/selftests/pidfd/pidfd_autoreap_test.c
+$ git am -3 ../patches/0001-ntfs-Fixup-merge-issue-with-vfs.all-changes.patch
+Applying: ntfs: Fixup merge issue with vfs.all changes
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/ntfs/mft.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[fs-next 40cdae4812a3f] next-20260312/vfs-brauner
+ Date: Fri Mar 20 12:52:23 2026 +0000
+Merging vfs/for-next (0787a93baa1aa sysfs(2): fs_index() argument is _not_ a pathname)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Already up to date.
+Merging mm-hotfixes/mm-hotfixes-unstable (d2c7d0f86d994 mm/huge_memory: fix folio isn't locked in softleaf_to_folio())
+$ git merge -m Merge branch 'mm-hotfixes-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-hotfixes/mm-hotfixes-unstable
+Merge made by the 'ort' strategy.
+ .mailmap                      |  1 +
+ arch/riscv/kernel/patch.c     | 21 +++++++++--------
+ drivers/block/zram/zram_drv.c | 39 ++++++++++++-------------------
+ include/linux/damon.h         |  6 +++++
+ include/linux/leafops.h       | 28 ++++++++++++++---------
+ mm/damon/core.c               |  8 +++++++
+ mm/damon/stat.c               | 53 ++++++++++++++++++++++++++++++++++++++++---
+ mm/filemap.c                  | 11 ++++++---
+ mm/pagewalk.c                 | 20 +++++++++++++---
+ mm/rmap.c                     |  7 ++++++
+ mm/zswap.c                    |  8 ++++++-
+ 11 files changed, 146 insertions(+), 56 deletions(-)
+Merging fs-current (8b5f70b18f13d Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git)
+$ git merge -m Merge branch 'fs-current' of linux-next fs-current
+Merge made by the 'ort' strategy.
+ fs/btrfs/backref.c         | 28 +++++++++++++
+ fs/btrfs/block-group.c     | 36 +++++++++++++++++
+ fs/btrfs/compression.c     | 11 ++++--
+ fs/btrfs/disk-io.c         | 20 ++++++++--
+ fs/btrfs/extent-tree.c     | 98 +++++++++++++++++++++++++++++++++++++++++++---
+ fs/btrfs/file-item.c       |  7 ++++
+ fs/btrfs/free-space-tree.c |  9 ++++-
+ fs/btrfs/inode.c           | 25 +++++++++---
+ fs/btrfs/ioctl.c           | 12 ++++--
+ fs/btrfs/lzo.c             |  4 +-
+ fs/btrfs/qgroup.c          |  8 ++++
+ fs/btrfs/raid56.c          | 12 +++++-
+ fs/btrfs/relocation.c      | 39 ++++++++++++++----
+ fs/btrfs/tree-checker.c    | 17 ++++++++
+ fs/btrfs/tree-log.c        | 21 ++++++++++
+ fs/btrfs/volumes.c         | 25 ++++++++----
+ fs/btrfs/zoned.c           |  7 ++++
+ fs/btrfs/zstd.c            |  2 +-
+ fs/erofs/fileio.c          |  6 +--
+ fs/erofs/xattr.c           |  8 ++++
+ fs/erofs/zdata.c           |  3 ++
+ fs/netfs/direct_write.c    | 14 +++++--
+ fs/netfs/iterator.c        | 43 ++++++++++++++++++++
+ fs/udf/super.c             |  2 +-
+ 24 files changed, 409 insertions(+), 48 deletions(-)
+Merging kbuild-current/kbuild-fixes-for-next (775af5cbb22c1 scripts: kconfig: merge_config.sh: fix indentation)
+$ git merge -m Merge branch 'kbuild-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild-current/kbuild-fixes-for-next
+Auto-merging Makefile
+Merge made by the 'ort' strategy.
+ Makefile                        |  2 +-
+ scripts/kconfig/merge_config.sh | 24 ++++++++++--------------
+ 2 files changed, 11 insertions(+), 15 deletions(-)
+Merging arc-current/for-curr (d7b8f8e20813f Linux 6.16-rc5)
+$ git merge -m Merge branch 'for-curr' of https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (23ea2a4c72323 ARM: 9468/1: fix memset64() on big-endian)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (8c6e9b60f5c79 arm64: realm: Fix PTE_NS_SHARED for 52bit PA support)
+$ git merge -m Merge branch 'for-next/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/kernel/pi/patch-scs.c    |  8 ++++++++
+ arch/arm64/kernel/rsi.c             |  3 ++-
+ drivers/resctrl/mpam_devices.c      |  2 ++
+ drivers/resctrl/test_mpam_devices.c | 22 +++++++++++++++-------
+ 4 files changed, 27 insertions(+), 8 deletions(-)
+Merging arm-soc-fixes/arm/fixes (df3ef89d7ef81 Merge tag 'scmi-fixes-7.0' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into arm/fixes)
+$ git merge -m Merge branch 'arm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Already up to date.
+Merging davinci-current/davinci/for-current (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging drivers-memory-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging sophgo-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/sophgo/linux.git sophgo-fixes/fixes
+Already up to date.
+Merging sophgo-soc-fixes/soc-fixes (0af2f6be1b428 Linux 6.15-rc1)
+$ git merge -m Merge branch 'soc-fixes' of https://github.com/sophgo/linux.git sophgo-soc-fixes/soc-fixes
+Already up to date.
+Merging m68k-current/for-linus (cb39cf99d88e8 m68k: sun3: Replace vsprintf() with bounded vsnprintf())
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Already up to date.
+Merging s390-fixes/fixes (598bbefa8032c s390/zcrypt: Enable AUTOSEL_DOM for CCA serialnr sysfs attribute)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Already up to date.
+Merging net/main (7c770dadfda5c net: openvswitch: Avoid releasing netdev before teardown completes)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Merge made by the 'ort' strategy.
+ drivers/bluetooth/hci_ll.c                      |  2 ++
+ drivers/net/ethernet/pensando/ionic/ionic_lif.c | 17 +++++++----
+ include/net/bluetooth/l2cap.h                   |  1 +
+ net/bluetooth/hci_conn.c                        |  2 +-
+ net/bluetooth/hci_core.c                        |  2 +-
+ net/bluetooth/hci_sync.c                        | 20 ++++++-------
+ net/bluetooth/l2cap_core.c                      | 40 +++++++++++++++++++++----
+ net/bluetooth/l2cap_sock.c                      |  3 ++
+ net/bluetooth/mgmt.c                            |  2 +-
+ net/bluetooth/sco.c                             | 10 +++++--
+ net/nfc/nci/core.c                              | 10 ++++---
+ net/openvswitch/vport-netdev.c                  |  9 +++++-
+ 12 files changed, 85 insertions(+), 33 deletions(-)
+Merging bpf/master (146bd2a87a65a bpf: Release module BTF IDR before module unload)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/ bpf/master
+Merge made by the 'ort' strategy.
+ kernel/bpf/btf.c                                   | 24 ++++++++++++++++++----
+ kernel/bpf/core.c                                  | 21 +++++++++++++++++++
+ kernel/bpf/verifier.c                              |  7 +++++++
+ tools/testing/selftests/bpf/Makefile               |  2 +-
+ tools/testing/selftests/bpf/progs/verifier_bswap.c | 22 ++++++++++++++++++++
+ 5 files changed, 71 insertions(+), 5 deletions(-)
+Merging ipsec/master (d849a2f7309fc xfrm: iptfs: only publish mode_data after clone setup)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Auto-merging net/key/af_key.c
+Auto-merging net/xfrm/xfrm_iptfs.c
+Auto-merging net/xfrm/xfrm_policy.c
+Merge made by the 'ort' strategy.
+ include/net/netns/xfrm.h      |   2 +-
+ net/ipv4/esp4.c               |   9 ++--
+ net/ipv6/esp6.c               |   9 ++--
+ net/key/af_key.c              |  19 ++++---
+ net/xfrm/xfrm_input.c         |   5 +-
+ net/xfrm/xfrm_iptfs.c         |  17 +++++--
+ net/xfrm/xfrm_nat_keepalive.c |   2 +-
+ net/xfrm/xfrm_policy.c        |  12 +++--
+ net/xfrm/xfrm_state.c         | 116 +++++++++++++++++++++++-------------------
+ net/xfrm/xfrm_user.c          |  32 ++++++++----
+ 10 files changed, 137 insertions(+), 86 deletions(-)
+Merging netfilter/main (e7577a06ae282 Merge tag 'nf-26-03-19' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (6cfc3bc02b977 net: bcmgenet: increase WoL poll timeout)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging wireless/for-next (a1d9d8e833781 Merge tag 'net-7.0-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Already up to date.
+Merging ath/for-current (6dccbc9f3e1d3 wifi: cfg80211: cancel pmsr_free_wk in cfg80211_pmsr_wdev_down)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath/for-current
+Already up to date.
+Merging iwlwifi/fixes (fb7f54aa2a99b wifi: iwlwifi: mvm: pause TCM on fast resume)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi/fixes
+Already up to date.
+Merging wpan/master (8ce4f287524c7 net: libwx: fix firmware mailbox abnormal return)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (0f2055db7b630 RDMA/efa: Fix possible deadlock)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Merge made by the 'ort' strategy.
+ drivers/infiniband/core/rw.c                    | 43 ++++++++----
+ drivers/infiniband/hw/bng_re/bng_dev.c          | 14 ++--
+ drivers/infiniband/hw/efa/efa_com.c             | 88 +++++++++++--------------
+ drivers/infiniband/hw/ionic/ionic_controlpath.c |  4 +-
+ drivers/infiniband/hw/irdma/verbs.c             |  1 +
+ 5 files changed, 83 insertions(+), 67 deletions(-)
+Merging sound-current/for-linus (591721223be9e ALSA: asihpi: avoid write overflow check warning)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ sound/firewire/amdtp-stream.c     |  2 +-
+ sound/hda/codecs/realtek/alc269.c | 42 +++++++++++++++++++++++++++++++++------
+ sound/pci/asihpi/hpimsgx.c        |  6 ++++--
+ sound/usb/Kconfig                 |  1 +
+ sound/usb/qcom/qc_audio_offload.c |  2 +-
+ sound/usb/quirks.c                |  1 +
+ 6 files changed, 44 insertions(+), 10 deletions(-)
+Merging sound-asoc-fixes/for-linus (a437601a0a138 ASoC: tas2781: Add null check for calibration data)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/rockchip-spdif.yaml  |  1 +
+ sound/soc/amd/acp-config.c                         | 15 +++++-
+ sound/soc/amd/acp/amd-acp70-acpi-match.c           | 61 ++++++++++++++++++++++
+ sound/soc/amd/yc/acp6x-mach.c                      | 14 +++++
+ sound/soc/codecs/sma1307.c                         |  6 ++-
+ sound/soc/codecs/tas2781-fmwlib.c                  |  3 ++
+ sound/soc/fsl/imx-card.c                           |  2 +
+ sound/soc/generic/simple-card-utils.c              |  4 +-
+ 8 files changed, 101 insertions(+), 5 deletions(-)
+Merging regmap-fixes/for-linus (09e70e4f119ff regmap: Synchronize cache for the page selector)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/regmap.c | 30 ++++++++++++++++++++++++++----
+ 1 file changed, 26 insertions(+), 4 deletions(-)
+Merging regulator-fixes/for-linus (4221f30e3e0a2 regulator: dt-bindings: fix typos in regulator-uv-* descriptions)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/regulator/regulator.yaml | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging spi-fixes/for-linus (8c89a077ca796 spi: geni-qcom: Check DMA interrupts early in ISR)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/spi/spi-amlogic-spifc-a4.c | 46 ++++++--------------------------------
+ drivers/spi/spi-amlogic-spisg.c    | 12 ++++------
+ drivers/spi/spi-axiado.c           | 31 +++++++++----------------
+ drivers/spi/spi-geni-qcom.c        | 13 ++++++-----
+ drivers/spi/spi.c                  | 25 ++++++++++-----------
+ 5 files changed, 41 insertions(+), 86 deletions(-)
+Merging pci-current/for-linus (21647677ba9af PCI: endpoint: pci-epf-test: Roll back BAR mapping when subrange setup fails)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Already up to date.
+Merging driver-core.current/driver-core-linus (2b38efc05bf7a driver core: platform: use generic driver_override infrastructure)
+$ git merge -m Merge branch 'driver-core-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core.current/driver-core-linus
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/driver-model/binding.rst | 48 ++++++++++++++++++
+ arch/sh/drivers/platform_early.c                  |  4 --
+ drivers/base/bus.c                                | 43 +++++++++++++++-
+ drivers/base/core.c                               |  2 +
+ drivers/base/dd.c                                 | 60 +++++++++++++++++++++++
+ drivers/base/platform.c                           | 37 ++------------
+ drivers/bus/simple-pm-bus.c                       |  4 +-
+ drivers/clk/imx/clk-scu.c                         |  3 +-
+ drivers/hwmon/axi-fan-control.c                   |  2 +-
+ drivers/slimbus/qcom-ngd-ctrl.c                   |  6 +--
+ include/linux/device.h                            | 54 ++++++++++++++++++++
+ include/linux/device/bus.h                        |  4 ++
+ include/linux/platform_device.h                   |  5 --
+ sound/soc/samsung/i2s.c                           |  6 +--
+ 14 files changed, 224 insertions(+), 54 deletions(-)
+Merging tty.current/tty-linus (5eb608319bb56 vt: save/restore unicode screen buffer for alternate screen)
+$ git merge -m Merge branch 'tty-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Merge made by the 'ort' strategy.
+ drivers/tty/serial/8250/8250.h      |  25 +++
+ drivers/tty/serial/8250/8250_dma.c  |  15 ++
+ drivers/tty/serial/8250/8250_dw.c   | 304 ++++++++++++++++++++++++++++--------
+ drivers/tty/serial/8250/8250_pci.c  |  17 ++
+ drivers/tty/serial/8250/8250_port.c |  75 +++++----
+ drivers/tty/serial/serial_core.c    |   5 +-
+ drivers/tty/serial/uartlite.c       |   1 +
+ drivers/tty/vt/vt.c                 |   8 +
+ include/linux/console_struct.h      |   1 +
+ include/linux/serial_8250.h         |   1 +
+ 10 files changed, 356 insertions(+), 96 deletions(-)
+Merging usb.current/usb-linus (f50200dd44125 usb: gadget: dummy_hcd: fix premature URB completion when ZLP follows partial transfer)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/nhi.c             |  2 +-
+ drivers/usb/class/cdc-acm.c           |  9 ++++++++
+ drivers/usb/class/cdc-acm.h           |  1 +
+ drivers/usb/class/usbtmc.c            |  3 +++
+ drivers/usb/core/hcd.c                |  2 +-
+ drivers/usb/dwc3/dwc3-google.c        |  5 +++--
+ drivers/usb/gadget/function/u_ether.c | 16 +++++++------
+ drivers/usb/gadget/udc/dummy_hcd.c    | 42 ++++++++++++++++++++++-------------
+ drivers/usb/typec/ucsi/ucsi.c         |  9 ++++++--
+ 9 files changed, 60 insertions(+), 29 deletions(-)
+Merging usb-serial-fixes/usb-linus (0e01c3416eb86 USB: serial: io_edgeport: add support for Blackbox IC135A)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/io_edgeport.c | 3 +++
+ drivers/usb/serial/io_usbvend.h  | 1 +
+ drivers/usb/serial/option.c      | 1 +
+ 3 files changed, 5 insertions(+)
+Merging phy/fixes (81af9e40e2e4e phy: qcom: qmp-ufs: Fix SM8650 PCS table for Gear 4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Merge made by the 'ort' strategy.
+ drivers/phy/Kconfig                      |  5 ++---
+ drivers/phy/freescale/phy-fsl-lynx-28g.c |  2 ++
+ drivers/phy/qualcomm/phy-qcom-qmp-ufs.c  |  3 +--
+ drivers/phy/spacemit/phy-k1-usb2.c       | 14 ++++++++++++++
+ drivers/phy/ti/phy-j721e-wiz.c           |  2 ++
+ 5 files changed, 21 insertions(+), 5 deletions(-)
+Merging staging.current/staging-linus (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'staging-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Already up to date.
+Merging iio-fixes/fixes-togreg (dd154646d292c iio: dac: mcp47feb02: Fix Vref validation [1-999] case)
+$ git merge -m Merge branch 'fixes-togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Merge made by the 'ort' strategy.
+ drivers/iio/accel/adxl380.c                        |  2 +-
+ drivers/iio/adc/ad4062.c                           |  8 ++--
+ drivers/iio/adc/ade9000.c                          | 12 ++---
+ drivers/iio/adc/aspeed_adc.c                       |  1 +
+ drivers/iio/adc/ti-ads1018.c                       |  2 +-
+ drivers/iio/adc/ti-ads1119.c                       | 11 ++---
+ drivers/iio/adc/ti-ads7950.c                       |  8 ++--
+ .../iio/common/hid-sensors/hid-sensor-trigger.c    | 48 ++++++++++++--------
+ drivers/iio/dac/mcp47feb02.c                       | 51 ++++++++++------------
+ drivers/iio/gyro/mpu3050-core.c                    | 32 +++++++++-----
+ drivers/iio/imu/adis16550.c                        |  8 ++--
+ drivers/iio/imu/bmi160/bmi160_core.c               | 15 +++----
+ drivers/iio/imu/bno055/bno055.c                    |  2 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c     | 15 ++++++-
+ drivers/iio/orientation/hid-sensor-rotation.c      | 24 ++++++++--
+ drivers/iio/pressure/abp2030pa.c                   |  2 +-
+ drivers/iio/proximity/rfd77402.c                   |  7 +--
+ include/linux/iio/iio.h                            | 12 +++++
+ 18 files changed, 157 insertions(+), 103 deletions(-)
+Merging counter-current/counter-current (9448598b22c50 Linux 6.19-rc2)
+$ git merge -m Merge branch 'counter-current' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'char-misc-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Already up to date.
+Merging soundwire-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (73a505dc48144 thunderbolt: Fix property read in nhi_wake_supported())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Already up to date.
+Merging input-current/for-linus (7adaaee5edd35 Input: synaptics-rmi4 - fix a locking bug in an error path)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Auto-merging drivers/input/mouse/bcm5974.c
+Merge made by the 'ort' strategy.
+ drivers/input/mouse/bcm5974.c         | 42 ++++++++++++++++++++++++++++++++++-
+ drivers/input/rmi4/rmi_f54.c          |  4 ++--
+ drivers/input/serio/i8042-acpipnpio.h |  7 ++++++
+ 3 files changed, 50 insertions(+), 3 deletions(-)
+Merging crypto-current/master (5c52607c43c39 crypto: ccp - Fix leaking the same page twice)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Already up to date.
+Merging libcrypto-fixes/libcrypto-fixes (d5b66179b0e27 lib/crypto: powerpc: Add powerpc/aesp8-ppc.S to clean-files)
+$ git merge -m Merge branch 'libcrypto-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes/libcrypto-fixes
+Already up to date.
+Merging vfio-fixes/for-linus (61ceaf236115f vfio: Prevent from pinned DMABUF importers to attach to VFIO DMABUF)
+$ git merge -m Merge branch 'for-linus' of https://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging dmaengine-fixes/fixes (a17ce4bc6f4f9 dmaengine: xilinx_dma: Fix reset related timeout with two-channel AXIDMA)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/dma/dw-edma/dw-edma-core.c    |  8 +++--
+ drivers/dma/dw-edma/dw-hdma-v0-core.c |  6 ++--
+ drivers/dma/fsl-edma-main.c           | 26 ++++++--------
+ drivers/dma/idxd/cdev.c               |  8 ++---
+ drivers/dma/idxd/device.c             | 45 +++++++++++++++--------
+ drivers/dma/idxd/dma.c                | 18 ++++++++++
+ drivers/dma/idxd/idxd.h               |  1 +
+ drivers/dma/idxd/init.c               | 14 ++++----
+ drivers/dma/idxd/irq.c                | 16 +++++++++
+ drivers/dma/idxd/submit.c             |  2 +-
+ drivers/dma/idxd/sysfs.c              |  1 +
+ drivers/dma/sh/rz-dmac.c              | 68 +++++++++++++++++++----------------
+ drivers/dma/xilinx/xdma.c             |  4 +--
+ drivers/dma/xilinx/xilinx_dma.c       | 46 +++++++++++++++---------
+ 14 files changed, 168 insertions(+), 95 deletions(-)
+Merging backlight-fixes/for-backlight-fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-backlight-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (b9465b04de4b9 mtd: rawnand: pl353: make sure optimal timings are applied)
+$ git merge -m Merge branch 'mtd/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mtd/st,spear600-smi.yaml   | 26 ++++++++++++++++------
+ MAINTAINERS                                        |  2 +-
+ drivers/mtd/nand/raw/brcmnand/brcmnand.c           |  6 ++---
+ drivers/mtd/nand/raw/cadence-nand-controller.c     |  2 +-
+ drivers/mtd/nand/raw/nand_base.c                   | 14 ++++++++++--
+ drivers/mtd/nand/raw/pl35x-nand-controller.c       |  3 +++
+ drivers/mtd/parsers/redboot.c                      |  6 ++---
+ drivers/mtd/spi-nor/core.c                         | 14 ++++++------
+ 8 files changed, 48 insertions(+), 25 deletions(-)
+Merging mfd-fixes/for-mfd-fixes (aa6e63373d91f mfd: ls2kbmc: Use PCI API instead of direct accesses)
+$ git merge -m Merge branch 'for-mfd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Merge made by the 'ort' strategy.
+Merging v4l-dvb-fixes/fixes (e8d97c270cb46 media: verisilicon: Fix kernel panic due to __initconst misuse)
+$ git merge -m Merge branch 'fixes' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/media/mc/mc-request.c                      |  5 +++
+ .../platform/rockchip/rkvdec/rkvdec-hevc-common.c  |  4 ++
+ .../platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c | 50 ++++++++++++----------
+ .../media/platform/rockchip/rkvdec/rkvdec-vp9.c    |  3 +-
+ drivers/media/platform/synopsys/Kconfig            |  1 +
+ drivers/media/platform/synopsys/dw-mipi-csi2rx.c   |  2 +-
+ drivers/media/platform/verisilicon/imx8m_vpu_hw.c  |  2 +-
+ drivers/media/v4l2-core/v4l2-ioctl.c               |  5 ++-
+ 8 files changed, 44 insertions(+), 28 deletions(-)
+Merging reset-fixes/reset/fixes (c961cc86af012 reset: rzg2l-usbphy-ctrl: Fix malformed MODULE_AUTHOR string)
+$ git merge -m Merge branch 'reset/fixes' of https://git.pengutronix.de/git/pza/linux reset-fixes/reset/fixes
+Merge made by the 'ort' strategy.
+ drivers/reset/reset-rzg2l-usbphy-ctrl.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging mips-fixes/mips-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'mips-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Already up to date.
+Merging at91-fixes/at91-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'at91-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Already up to date.
+Merging omap-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (d2ea4ff1ce507 KVM: selftests: Verify SEV+ guests can read and write EFER, CR0, CR4, and CR8)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Already up to date.
+Merging kvms390-fixes/master (ab5119735e984 KVM: s390: vsie: Avoid injecting machine check on signal)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Auto-merging arch/s390/kvm/kvm-s390.c
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Merge made by the 'ort' strategy.
+ arch/s390/include/asm/kvm_host.h               |  3 ++
+ arch/s390/include/asm/stacktrace.h             |  2 +-
+ arch/s390/kernel/asm-offsets.c                 |  2 +-
+ arch/s390/kernel/entry.S                       |  4 +-
+ arch/s390/kernel/nmi.c                         |  4 +-
+ arch/s390/kvm/gaccess.c                        |  6 ++-
+ arch/s390/kvm/interrupt.c                      | 18 +++++++
+ arch/s390/kvm/kvm-s390.c                       | 16 +++---
+ arch/s390/kvm/vsie.c                           |  8 +--
+ arch/s390/mm/fault.c                           | 11 +++-
+ tools/testing/selftests/kvm/Makefile.kvm       |  1 +
+ tools/testing/selftests/kvm/s390/irq_routing.c | 75 ++++++++++++++++++++++++++
+ 12 files changed, 129 insertions(+), 21 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/s390/irq_routing.c
+Merging kvm-arm-fixes/fixes (e3d585ed3ff89 Merge branch kvm-arm64/vgic-v5-ppi into kvmarm-master/next)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm-fixes/fixes
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Merge made by the 'ort' strategy.
+ Documentation/trace/index.rst                      |   11 +
+ Documentation/trace/remotes.rst                    |   66 +
+ Documentation/virt/kvm/api.rst                     |    6 +-
+ Documentation/virt/kvm/devices/arm-vgic-v5.rst     |   50 +
+ Documentation/virt/kvm/devices/index.rst           |    1 +
+ Documentation/virt/kvm/devices/vcpu.rst            |    5 +-
+ arch/arm64/include/asm/el2_setup.h                 |    2 +
+ arch/arm64/include/asm/kvm_asm.h                   |   10 +
+ arch/arm64/include/asm/kvm_define_hypevents.h      |   16 +
+ arch/arm64/include/asm/kvm_host.h                  |   37 +
+ arch/arm64/include/asm/kvm_hyp.h                   |   14 +-
+ arch/arm64/include/asm/kvm_hypevents.h             |   60 +
+ arch/arm64/include/asm/kvm_hyptrace.h              |   26 +
+ arch/arm64/include/asm/sysreg.h                    |    7 +
+ arch/arm64/include/asm/vncr_mapping.h              |    3 +
+ arch/arm64/include/uapi/asm/kvm.h                  |    1 +
+ arch/arm64/kernel/image-vars.h                     |    4 +
+ arch/arm64/kernel/vmlinux.lds.S                    |   18 +
+ arch/arm64/kvm/Kconfig                             |   68 +-
+ arch/arm64/kvm/Makefile                            |    2 +
+ arch/arm64/kvm/arch_timer.c                        |  108 +-
+ arch/arm64/kvm/arm.c                               |   56 +-
+ arch/arm64/kvm/config.c                            |  123 +-
+ arch/arm64/kvm/emulate-nested.c                    |   68 +
+ arch/arm64/kvm/handle_exit.c                       |    2 +-
+ arch/arm64/kvm/hyp/include/hyp/switch.h            |   27 +
+ arch/arm64/kvm/hyp/include/nvhe/arm-smccc.h        |   23 +
+ arch/arm64/kvm/hyp/include/nvhe/clock.h            |   16 +
+ arch/arm64/kvm/hyp/include/nvhe/define_events.h    |   14 +
+ arch/arm64/kvm/hyp/include/nvhe/mem_protect.h      |    2 -
+ arch/arm64/kvm/hyp/include/nvhe/trace.h            |   70 +
+ arch/arm64/kvm/hyp/nvhe/Makefile                   |    8 +-
+ arch/arm64/kvm/hyp/nvhe/clock.c                    |   65 +
+ arch/arm64/kvm/hyp/nvhe/events.c                   |   25 +
+ arch/arm64/kvm/hyp/nvhe/ffa.c                      |   28 +-
+ arch/arm64/kvm/hyp/nvhe/host.S                     |    2 +-
+ arch/arm64/kvm/hyp/nvhe/hyp-main.c                 |  103 +-
+ arch/arm64/kvm/hyp/nvhe/hyp.lds.S                  |    6 +
+ arch/arm64/kvm/hyp/nvhe/mm.c                       |    4 +-
+ arch/arm64/kvm/hyp/nvhe/psci-relay.c               |    7 +-
+ arch/arm64/kvm/hyp/nvhe/setup.c                    |    4 +-
+ arch/arm64/kvm/hyp/nvhe/stacktrace.c               |    6 +-
+ arch/arm64/kvm/hyp/nvhe/switch.c                   |   20 +-
+ arch/arm64/kvm/hyp/nvhe/sys_regs.c                 |    8 +
+ arch/arm64/kvm/hyp/nvhe/trace.c                    |  306 +++++
+ arch/arm64/kvm/hyp/vgic-v5-sr.c                    |  166 +++
+ arch/arm64/kvm/hyp/vhe/Makefile                    |    2 +-
+ arch/arm64/kvm/hyp_trace.c                         |  442 +++++++
+ arch/arm64/kvm/hyp_trace.h                         |   11 +
+ arch/arm64/kvm/nested.c                            |    5 +
+ arch/arm64/kvm/pmu-emul.c                          |   20 +-
+ arch/arm64/kvm/stacktrace.c                        |    8 +-
+ arch/arm64/kvm/sys_regs.c                          |  176 ++-
+ arch/arm64/kvm/vgic/vgic-init.c                    |  216 +++-
+ arch/arm64/kvm/vgic/vgic-kvm-device.c              |  107 +-
+ arch/arm64/kvm/vgic/vgic-mmio.c                    |   40 +-
+ arch/arm64/kvm/vgic/vgic-v3.c                      |    2 +-
+ arch/arm64/kvm/vgic/vgic-v5.c                      |  503 ++++++-
+ arch/arm64/kvm/vgic/vgic.c                         |  173 ++-
+ arch/arm64/kvm/vgic/vgic.h                         |   53 +-
+ arch/arm64/tools/sysreg                            |  480 +++++++
+ drivers/irqchip/irq-gic-v5.c                       |   18 +
+ fs/tracefs/inode.c                                 |    1 +
+ include/kvm/arm_arch_timer.h                       |   11 +-
+ include/kvm/arm_pmu.h                              |    5 +-
+ include/kvm/arm_vgic.h                             |  191 ++-
+ include/linux/irqchip/arm-gic-v5.h                 |   27 +
+ include/linux/kvm_host.h                           |    1 +
+ include/linux/ring_buffer.h                        |   58 +
+ include/linux/ring_buffer_types.h                  |   41 +
+ include/linux/simple_ring_buffer.h                 |   65 +
+ include/linux/trace_remote.h                       |   48 +
+ include/linux/trace_remote_event.h                 |   33 +
+ include/trace/define_remote_events.h               |   73 ++
+ include/uapi/linux/kvm.h                           |    2 +
+ include/uapi/linux/trace_mmap.h                    |    8 +-
+ kernel/trace/Kconfig                               |   14 +
+ kernel/trace/Makefile                              |   53 +
+ kernel/trace/remote_test.c                         |  261 ++++
+ kernel/trace/remote_test_events.h                  |   10 +
+ kernel/trace/ring_buffer.c                         |  354 ++++-
+ kernel/trace/simple_ring_buffer.c                  |  517 ++++++++
+ kernel/trace/trace.c                               |    4 +-
+ kernel/trace/trace.h                               |    7 +
+ kernel/trace/trace_remote.c                        | 1368 ++++++++++++++++++++
+ tools/arch/arm64/include/uapi/asm/kvm.h            |    1 +
+ tools/include/uapi/linux/kvm.h                     |    2 +
+ .../selftests/ftrace/test.d/remotes/buffer_size.tc |   25 +
+ .../selftests/ftrace/test.d/remotes/functions      |   88 ++
+ .../test.d/remotes/hypervisor/buffer_size.tc       |   11 +
+ .../ftrace/test.d/remotes/hypervisor/reset.tc      |   11 +
+ .../ftrace/test.d/remotes/hypervisor/trace.tc      |   11 +
+ .../ftrace/test.d/remotes/hypervisor/trace_pipe.tc |   11 +
+ .../ftrace/test.d/remotes/hypervisor/unloading.tc  |   11 +
+ .../selftests/ftrace/test.d/remotes/reset.tc       |   90 ++
+ .../selftests/ftrace/test.d/remotes/trace.tc       |  127 ++
+ .../selftests/ftrace/test.d/remotes/trace_pipe.tc  |  127 ++
+ .../selftests/ftrace/test.d/remotes/unloading.tc   |   41 +
+ tools/testing/selftests/kvm/Makefile.kvm           |    3 +-
+ tools/testing/selftests/kvm/arm64/no-vgic-v3.c     |  177 ---
+ tools/testing/selftests/kvm/arm64/no-vgic.c        |  297 +++++
+ tools/testing/selftests/kvm/arm64/vgic_v5.c        |  228 ++++
+ tools/testing/selftests/kvm/include/arm64/gic_v5.h |  150 +++
+ 103 files changed, 7988 insertions(+), 505 deletions(-)
+ create mode 100644 Documentation/trace/remotes.rst
+ create mode 100644 Documentation/virt/kvm/devices/arm-vgic-v5.rst
+ create mode 100644 arch/arm64/include/asm/kvm_define_hypevents.h
+ create mode 100644 arch/arm64/include/asm/kvm_hypevents.h
+ create mode 100644 arch/arm64/include/asm/kvm_hyptrace.h
+ create mode 100644 arch/arm64/kvm/hyp/include/nvhe/arm-smccc.h
+ create mode 100644 arch/arm64/kvm/hyp/include/nvhe/clock.h
+ create mode 100644 arch/arm64/kvm/hyp/include/nvhe/define_events.h
+ create mode 100644 arch/arm64/kvm/hyp/include/nvhe/trace.h
+ create mode 100644 arch/arm64/kvm/hyp/nvhe/clock.c
+ create mode 100644 arch/arm64/kvm/hyp/nvhe/events.c
+ create mode 100644 arch/arm64/kvm/hyp/nvhe/trace.c
+ create mode 100644 arch/arm64/kvm/hyp/vgic-v5-sr.c
+ create mode 100644 arch/arm64/kvm/hyp_trace.c
+ create mode 100644 arch/arm64/kvm/hyp_trace.h
+ create mode 100644 include/linux/ring_buffer_types.h
+ create mode 100644 include/linux/simple_ring_buffer.h
+ create mode 100644 include/linux/trace_remote.h
+ create mode 100644 include/linux/trace_remote_event.h
+ create mode 100644 include/trace/define_remote_events.h
+ create mode 100644 kernel/trace/remote_test.c
+ create mode 100644 kernel/trace/remote_test_events.h
+ create mode 100644 kernel/trace/simple_ring_buffer.c
+ create mode 100644 kernel/trace/trace_remote.c
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/buffer_size.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/functions
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/hypervisor/buffer_size.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/hypervisor/reset.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/hypervisor/trace.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/hypervisor/trace_pipe.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/hypervisor/unloading.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/reset.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/trace.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/trace_pipe.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/remotes/unloading.tc
+ delete mode 100644 tools/testing/selftests/kvm/arm64/no-vgic-v3.c
+ create mode 100644 tools/testing/selftests/kvm/arm64/no-vgic.c
+ create mode 100644 tools/testing/selftests/kvm/arm64/vgic_v5.c
+ create mode 100644 tools/testing/selftests/kvm/include/arm64/gic_v5.h
+Merging hwmon-fixes/hwmon (88838ea00402c hwmon: (max6639) Fix range check for pulses-per-revolution property)
+$ git merge -m Merge branch 'hwmon' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Merge made by the 'ort' strategy.
+ drivers/hwmon/max6639.c        |  2 +-
+ drivers/hwmon/pmbus/hac300s.c  |  2 ++
+ drivers/hwmon/pmbus/ina233.c   |  2 ++
+ drivers/hwmon/pmbus/isl68137.c |  7 +++++--
+ drivers/hwmon/pmbus/mp2869.c   | 35 +++++++++++++++++++++--------------
+ drivers/hwmon/pmbus/mp2975.c   |  2 ++
+ 6 files changed, 33 insertions(+), 17 deletions(-)
+Merging nvdimm-fixes/libnvdimm-fixes (a8aec14230322 nvdimm/bus: Fix potential use after free in asynchronous initialization)
+$ git merge -m Merge branch 'libnvdimm-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (be5c5280cf2b2 cxl: Adjust the startup priority of cxl_pmem to be higher than that of cxl_acpi)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/cxl/Kconfig       |  1 +
+ drivers/cxl/core/hdm.c    | 25 +++++++++----------------
+ drivers/cxl/core/mbox.c   |  2 +-
+ drivers/cxl/core/port.c   |  8 ++++++--
+ drivers/cxl/core/region.c |  4 +++-
+ drivers/cxl/pmem.c        |  2 +-
+ 6 files changed, 21 insertions(+), 21 deletions(-)
+Merging dma-mapping-fixes/dma-mapping-fixes (f5ebf241c407d mm/hmm: Indicate that HMM requires DMA coherency)
+$ git merge -m Merge branch 'dma-mapping-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping-fixes/dma-mapping-fixes
+Merge made by the 'ort' strategy.
+ Documentation/core-api/dma-attributes.rst | 36 +++++++++++++++++++++++++------
+ drivers/infiniband/core/umem.c            |  5 +++--
+ drivers/iommu/dma-iommu.c                 | 21 ++++++++++++++----
+ drivers/virtio/virtio_ring.c              | 10 ++++-----
+ include/linux/dma-mapping.h               | 15 +++++++++----
+ include/trace/events/dma.h                |  4 +++-
+ kernel/dma/debug.c                        |  9 ++++----
+ kernel/dma/direct.h                       |  7 +++---
+ kernel/dma/mapping.c                      |  6 ++++++
+ kernel/dma/swiotlb.c                      | 21 ++++++++++++++++--
+ mm/hmm.c                                  |  4 ++--
+ 11 files changed, 104 insertions(+), 34 deletions(-)
+Merging drivers-x86-fixes/fixes (58a1bf172388b platform/x86: intel-hid: disable wakeup_mode during hibernation)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |  2 +-
+ drivers/platform/olpc/olpc-xo175-ec.c              |  2 +-
+ drivers/platform/x86/amd/hsmp/hsmp.c               |  2 +-
+ drivers/platform/x86/asus-armoury.h                | 77 ++++++++++++++++++++++
+ drivers/platform/x86/asus-nb-wmi.c                 |  2 +-
+ drivers/platform/x86/hp/hp-wmi.c                   | 19 ++++++
+ drivers/platform/x86/intel/hid.c                   | 10 ++-
+ .../x86/intel/speed_select_if/isst_tpmi_core.c     |  3 +
+ drivers/platform/x86/lenovo/wmi-gamezone.c         |  2 -
+ 9 files changed, 112 insertions(+), 7 deletions(-)
+Merging samsung-krzk-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging devicetree-fixes/dt/linus (81122fba08fa3 of: fix reference count leak in of_alias_scan())
+$ git merge -m Merge branch 'dt/linus' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Already up to date.
+Merging dt-krzk-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (01f784fc9d0ab scsi: target: file: Use kzalloc_flex for aio_cmd)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/scsi/ibmvscsi/ibmvfc.c     |  3 ++-
+ drivers/scsi/scsi_transport_sas.c  |  2 +-
+ drivers/scsi/ses.c                 |  2 +-
+ drivers/target/loopback/tcm_loop.c | 52 +++++++++++++++++++++++++++++++++-----
+ drivers/target/target_core_file.c  |  2 +-
+ 5 files changed, 51 insertions(+), 10 deletions(-)
+Merging drm-fixes/drm-fixes (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'drm-fixes' of https://gitlab.freedesktop.org/drm/kernel.git drm-fixes/drm-fixes
+Already up to date.
+Merging drm-intel-fixes/for-linux-next-fixes (0162ab3220bac drm/i915/gt: Check set_default_submission() before deferencing)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/intel_display_power_well.c | 2 +-
+ drivers/gpu/drm/i915/display/intel_display_types.h      | 1 +
+ drivers/gpu/drm/i915/display/intel_dmc.c                | 3 +--
+ drivers/gpu/drm/i915/display/intel_psr.c                | 7 +++++--
+ drivers/gpu/drm/i915/gt/intel_engine_cs.c               | 3 ++-
+ 5 files changed, 10 insertions(+), 6 deletions(-)
+Merging mmc-fixes/fixes (5e3486e64094c mmc: sdhci: fix timing selection for 1-bit bus width)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/mmc/host/sdhci-pci-gli.c | 9 +++++++++
+ drivers/mmc/host/sdhci.c         | 9 ++++++++-
+ 2 files changed, 17 insertions(+), 1 deletion(-)
+Merging rtc-fixes/rtc-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'rtc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'gnss-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (c0e296f257671 mshv: Fix error handling in mshv_region_pin)
+$ git merge -m Merge branch 'hyperv-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                    |   4 +-
+ arch/x86/hyperv/hv_crash.c     | 118 +++++++++++++-------------
+ arch/x86/kernel/cpu/mshyperv.c |   5 +-
+ drivers/hv/mshv_regions.c      |   6 +-
+ drivers/hv/mshv_root.h         |   5 +-
+ drivers/hv/mshv_root_main.c    |  93 +++++---------------
+ drivers/hv/mshv_synic.c        | 188 +++++++++++++++++++++++++++++++++++++----
+ include/hyperv/hvgdk_mini.h    |   3 +-
+ 8 files changed, 270 insertions(+), 152 deletions(-)
+Merging risc-v-fixes/fixes (11212007bf29a riscv: kgdb: fix several debug register assignment bugs)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Auto-merging arch/riscv/kvm/vcpu_pmu.c
+Merge made by the 'ort' strategy.
+ arch/riscv/errata/thead/errata.c     | 2 +-
+ arch/riscv/include/asm/atomic.h      | 4 ++--
+ arch/riscv/include/asm/elf.h         | 4 ++--
+ arch/riscv/include/asm/page.h        | 5 ++++-
+ arch/riscv/include/asm/processor.h   | 2 +-
+ arch/riscv/include/asm/thread_info.h | 2 +-
+ arch/riscv/kernel/acpi.c             | 2 +-
+ arch/riscv/kernel/ftrace.c           | 2 +-
+ arch/riscv/kernel/head.S             | 2 +-
+ arch/riscv/kernel/kgdb.c             | 7 ++++---
+ arch/riscv/kernel/mcount-dyn.S       | 2 +-
+ arch/riscv/kernel/module-sections.c  | 2 +-
+ arch/riscv/kernel/probes/kprobes.c   | 2 +-
+ arch/riscv/kernel/probes/uprobes.c   | 2 +-
+ arch/riscv/kernel/smpboot.c          | 8 ++------
+ arch/riscv/kernel/soc.c              | 2 +-
+ arch/riscv/kernel/suspend.c          | 2 +-
+ arch/riscv/kernel/vmcore_info.c      | 7 +++++++
+ arch/riscv/kvm/tlb.c                 | 2 +-
+ arch/riscv/kvm/vcpu_pmu.c            | 2 +-
+ arch/riscv/lib/csum.c                | 2 +-
+ arch/riscv/lib/memmove.S             | 4 ++--
+ arch/riscv/mm/cacheflush.c           | 2 +-
+ arch/riscv/mm/init.c                 | 5 -----
+ arch/riscv/mm/physaddr.c             | 4 ++--
+ 25 files changed, 41 insertions(+), 39 deletions(-)
+Merging riscv-dt-fixes/riscv-dt-fixes (305f2865bd034 riscv: dts: starfive: jh7110-common: fix jh7110 SoC boot from SD-card.)
+$ git merge -m Merge branch 'riscv-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/starfive/jh7110-common.dtsi           | 1 +
+ arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts | 2 +-
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+Merging riscv-soc-fixes/riscv-soc-fixes (31c139acb6f54 firmware: microchip: auto-update: silence error print when flash not found)
+$ git merge -m Merge branch 'riscv-soc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Auto-merging drivers/firmware/microchip/mpfs-auto-update.c
+Merge made by the 'ort' strategy.
+ drivers/firmware/microchip/mpfs-auto-update.c | 10 ++++++----
+ drivers/soc/microchip/mpfs-control-scb.c      |  6 ++++--
+ drivers/soc/microchip/mpfs-mss-top-sysreg.c   |  6 ++++--
+ 3 files changed, 14 insertions(+), 8 deletions(-)
+Merging fpga-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (ebf1bafd09079 LICENSES: Explicitly allow SPDX-FileCopyrightText)
+$ git merge -m Merge branch 'spdx-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Merge made by the 'ort' strategy.
+ Documentation/process/license-rules.rst | 7 +++++--
+ scripts/checkpatch.pl                   | 8 ++++++++
+ 2 files changed, 13 insertions(+), 2 deletions(-)
+Merging gpio-brgl-fixes/gpio/for-current (09c8ef6236004 Merge branch 'gpio/dev-init-rework' into gpio/for-current)
+$ git merge -m Merge branch 'gpio/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpiolib.c            | 129 +++++++++++++++++++-------------------
+ include/linux/gpio/gpio-nomadik.h |  12 ++--
+ include/linux/timb_gpio.h         |   6 +-
+ 3 files changed, 71 insertions(+), 76 deletions(-)
+Merging gpio-intel-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Already up to date.
+Merging pinctrl-intel-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging auxdisplay-fixes/fixes (995a418a6ca33 auxdisplay: lcd2s: add error handling for i2c transfers)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay-fixes/fixes
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/auxdisplay/holtek,ht16k33.yaml    |  2 +-
+ drivers/auxdisplay/lcd2s.c                                | 15 ++++++++++++---
+ 2 files changed, 13 insertions(+), 4 deletions(-)
+Merging kunit-fixes/kunit-fixes (8b8f1d5e350ac kunit: Add documentation of --list_suites)
+$ git merge -m Merge branch 'kunit-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Already up to date.
+Merging memblock-fixes/fixes (f043a93fff9e3 mm: numa_memblks: Identify the accurate NUMA ID of CFMW)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock-fixes/fixes
+Already up to date.
+Merging renesas-fixes/fixes (418a9d4aa02d2 Merge branch 'renesas-pinctrl-fixes-for-v7.0' into renesas-fixes)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/r9a09g056-cpg.c     | 36 ++++++++++-----------
+ drivers/clk/renesas/r9a09g057-cpg.c     | 55 ++++++++++++---------------------
+ drivers/pinctrl/renesas/pinctrl-rza1.c  |  2 +-
+ drivers/pinctrl/renesas/pinctrl-rzt2h.c | 15 ++++-----
+ 4 files changed, 47 insertions(+), 61 deletions(-)
+Merging perf-current/perf-tools (563d39928db60 perf kvm stat: Fix relative paths for including headers)
+$ git merge -m Merge branch 'perf-tools' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-current/perf-tools
+Merge made by the 'ort' strategy.
+ tools/perf/check-headers.sh                  |  1 -
+ tools/perf/util/kvm-stat-arch/kvm-stat-x86.c |  6 +-
+ tools/perf/util/metricgroup.c                |  6 +-
+ tools/perf/util/parse-events.c               | 82 ++++++++++++++++++++++------
+ 4 files changed, 71 insertions(+), 24 deletions(-)
+Merging efi-fixes/urgent (a4b0bf6a40f3c x86/efi: defer freeing of boot services memory)
+$ git merge -m Merge branch 'urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Already up to date.
+Merging battery-fixes/fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Already up to date.
+Merging iommufd-fixes/for-rc (11439c4635edd Linux 7.0-rc2)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Already up to date.
+Merging rust-fixes/rust-fixes (a989fde763f4f Merge tag 'libnvdimm-fixes-7.0-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Already up to date.
+Merging w1-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (b826d2c0b0ecb pmdomain: bcm: bcm2835-power: Increase ASB control timeout)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/pmdomain/bcm/bcm2835-power.c       | 12 ++++--------
+ drivers/pmdomain/mediatek/mtk-pm-domains.c |  2 +-
+ 2 files changed, 5 insertions(+), 9 deletions(-)
+Merging i2c-host-fixes/i2c/i2c-host-fixes (78a6ee14f8b9e i2c: pxa: defer reset on Armada 3700 when recovery is used)
+$ git merge -m Merge branch 'i2c/i2c-host-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host-fixes/i2c/i2c-host-fixes
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-cp2615.c |  3 +++
+ drivers/i2c/busses/i2c-fsi.c    |  1 +
+ drivers/i2c/busses/i2c-pxa.c    | 17 ++++++++++++++++-
+ 3 files changed, 20 insertions(+), 1 deletion(-)
+Merging sparc-fixes/for-linus (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc-fixes/for-linus
+Already up to date.
+Merging clk-fixes/clk-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-fixes/clk-fixes
+Already up to date.
+Merging thead-clk-fixes/thead-clk-fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk-fixes/thead-clk-fixes
+Already up to date.
+Merging tenstorrent-clk-fixes/tenstorrent-clk-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk-fixes/tenstorrent-clk-fixes
+Already up to date.
+Merging pwrseq-fixes/pwrseq/for-current (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'pwrseq/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq-fixes/pwrseq/for-current
+Already up to date.
+Merging thead-dt-fixes/thead-dt-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'thead-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt-fixes/thead-dt-fixes
+Already up to date.
+Merging ftrace-fixes/ftrace/fixes (1650a1b6cb1ae fgraph: Check ftrace_pids_enabled on registration for early filtering)
+$ git merge -m Merge branch 'ftrace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace-fixes/ftrace/fixes
+Already up to date.
+Merging ring-buffer-fixes/ring-buffer/fixes (b042fdf18e89a tracing: Fix WARN_ON in tracing_buffers_mmap_close for split VMAs)
+$ git merge -m Merge branch 'ring-buffer/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ring-buffer-fixes/ring-buffer/fixes
+Already up to date.
+Merging trace-fixes/trace/fixes (d008ba8be8984 tracing: Fix trace_buf_size= cmdline parameter with sizes >= 2G)
+$ git merge -m Merge branch 'trace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git trace-fixes/trace/fixes
+Already up to date.
+Merging tracefs-fixes/tracefs/fixes (8b55572e51805 tracing/selftests: Add tracefs mount options test)
+$ git merge -m Merge branch 'tracefs/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git tracefs-fixes/tracefs/fixes
+Already up to date.
+Merging spacemit-fixes/fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/spacemit-com/linux spacemit-fixes/fixes
+Already up to date.
+Merging tip-fixes/tip/urgent (5f6970056f68f Merge branch into tip/master: 'x86/urgent')
+$ git merge -m Merge branch 'tip/urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip-fixes/tip/urgent
+Merge made by the 'ort' strategy.
+ arch/x86/events/core.c                  |  7 +++++--
+ arch/x86/events/intel/core.c            | 31 +++++++++++++++++++++----------
+ arch/x86/events/intel/ds.c              | 11 +++++++----
+ arch/x86/kernel/cpu/mce/amd.c           | 17 +++++++++++------
+ drivers/irqchip/irq-riscv-rpmi-sysmsi.c |  1 +
+ include/linux/local_lock_internal.h     |  2 +-
+ kernel/events/core.c                    | 19 ++++++++-----------
+ scripts/livepatch/klp-build             |  9 ++++-----
+ tools/objtool/check.c                   |  5 ++---
+ tools/objtool/elf.c                     | 23 +++--------------------
+ tools/objtool/klp-diff.c                |  3 ++-
+ 11 files changed, 65 insertions(+), 63 deletions(-)
+Merging slab-fixes/slab/for-next-fixes (464b1c115852f slab: fix memory leak when refill_sheaf() fails)
+$ git merge -m Merge branch 'slab/for-next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab-fixes/slab/for-next-fixes
+Already up to date.
+Merging drm-msm-fixes/msm-fixes (e4eb11b34d6c8 drm/msm/dsi: fix pclk rate calculation for bonded dsi)
+$ git merge -m Merge branch 'msm-fixes' of https://gitlab.freedesktop.org/drm/msm.git drm-msm-fixes/msm-fixes
+Already up to date.
+Merging uml-fixes/fixes (df447a3b4a4b9 um: Fix FD copy size in os_rcv_fd_msg())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging fwctl-fixes/for-rc (11439c4635edd Linux 7.0-rc2)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl-fixes/for-rc
+Already up to date.
+Merging devsec-tsm-fixes/fixes (c201226304768 crypto/ccp: Allow multiple streams on the same root bridge)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm-fixes/fixes
+Already up to date.
+Merging drm-rust-fixes/drm-rust-fixes (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'drm-rust-fixes' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-fixes/drm-rust-fixes
+Already up to date.
+Merging tenstorrent-dt-fixes/tenstorrent-dt-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt-fixes/tenstorrent-dt-fixes
+Already up to date.
+Merging drm-misc-fixes/for-linux-next-fixes (fc3bbf34e643f drm/shmem-helper: Fix huge page mapping in fault handler)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c |  2 +-
+ drivers/gpu/drm/drm_file.c                   |  5 +-
+ drivers/gpu/drm/drm_gem_shmem_helper.c       | 48 +++++++-------
+ drivers/gpu/drm/drm_mode_config.c            |  9 ++-
+ drivers/gpu/drm/drm_pagemap_util.c           | 14 ++---
+ drivers/gpu/drm/imagination/pvr_device.c     | 17 -----
+ drivers/gpu/drm/imagination/pvr_power.c      | 51 +++++++++++----
+ drivers/gpu/drm/vmwgfx/vmwgfx_drv.h          | 93 +++++++++++++++++-----------
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.c          |  3 +-
+ 9 files changed, 141 insertions(+), 101 deletions(-)
+Merging rust/rust-next (79e25710e7227 rust: types: remove temporary re-exports of ARef and AlwaysRefCounted)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Auto-merging rust/kernel/str.rs
+Merge made by the 'ort' strategy.
+ .clippy.toml              | 10 ++++++++++
+ rust/helpers/clk.c        | 24 +++++++++++++-----------
+ rust/helpers/jump_label.c |  2 +-
+ rust/kernel/i2c.rs        | 13 ++++++-------
+ rust/kernel/str.rs        | 22 +++++++++++++++++++---
+ rust/kernel/task.rs       |  7 +++----
+ rust/kernel/types.rs      |  2 --
+ rust/kernel/usb.rs        |  6 ++----
+ 8 files changed, 54 insertions(+), 32 deletions(-)
+Merging rust-interop/interop-next (05f7e89ab9731 Linux 6.19)
+$ git merge -m Merge branch 'interop-next' of https://github.com/Rust-for-Linux/linux.git rust-interop/interop-next
+Already up to date.
+Merging rust-alloc/alloc-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'alloc-next' of https://github.com/Rust-for-Linux/linux.git rust-alloc/alloc-next
+Already up to date.
+Merging rust-io/io-next (86731a2a651e5 Linux 6.16-rc3)
+$ git merge -m Merge branch 'io-next' of https://github.com/Rust-for-Linux/linux.git rust-io/io-next
+Already up to date.
+Merging rust-pin-init/pin-init-next (aeb5ecad5316f rust: pin-init: Implement `InPlaceWrite<T>` for `&'static mut MaybeUninit<T>`)
+$ git merge -m Merge branch 'pin-init-next' of https://github.com/Rust-for-Linux/linux.git rust-pin-init/pin-init-next
+Already up to date.
+Merging rust-timekeeping/timekeeping-next (4521438fb076f rust: time: Implement basic arithmetic operations for Delta)
+$ git merge -m Merge branch 'timekeeping-next' of https://github.com/Rust-for-Linux/linux.git rust-timekeeping/timekeeping-next
+Already up to date.
+Merging rust-xarray/xarray-next (c455f19bbe610 rust: xarray: add __rust_helper to helpers)
+$ git merge -m Merge branch 'xarray-next' of https://github.com/Rust-for-Linux/linux.git rust-xarray/xarray-next
+Already up to date.
+Merging rust-analyzer/rust-analyzer-next (5c8d16ac49405 scripts: generate_rust_analyzer.py: reduce cfg plumbing)
+$ git merge -m Merge branch 'rust-analyzer-next' of https://github.com/Rust-for-Linux/linux.git rust-analyzer/rust-analyzer-next
+Auto-merging rust/Makefile
+Merge made by the 'ort' strategy.
+ rust/Makefile                     |   2 +
+ scripts/generate_rust_analyzer.py | 286 ++++++++++++++++++++++++++++----------
+ 2 files changed, 211 insertions(+), 77 deletions(-)
+Merging mm-stable/mm-stable (f0caa1d49cc07 Merge tag 'hid-for-linus-2026031701' of git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid)
+$ git merge -m Merge branch 'mm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-stable/mm-stable
+Already up to date.
+Merging mm-nonmm-stable/mm-nonmm-stable (f0caa1d49cc07 Merge tag 'hid-for-linus-2026031701' of git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid)
+$ git merge -m Merge branch 'mm-nonmm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm-unstable/mm-unstable (a12ff2f1b5474 mm/swapfile: remove duplicate include of swap_table.h)
+$ git merge -m Merge branch 'mm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-unstable/mm-unstable
+Auto-merging MAINTAINERS
+Auto-merging arch/riscv/include/asm/page.h
+Auto-merging arch/riscv/mm/init.c
+Auto-merging fs/btrfs/compression.c
+Auto-merging kernel/events/core.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-kernel-mm-damon    |    6 +
+ Documentation/admin-guide/blockdev/zram.rst        |   47 +-
+ Documentation/admin-guide/cgroup-v2.rst            |    5 +
+ Documentation/admin-guide/kdump/vmcoreinfo.rst     |    2 +-
+ Documentation/admin-guide/kernel-parameters.txt    |    6 +
+ Documentation/admin-guide/mm/damon/lru_sort.rst    |    9 +-
+ Documentation/admin-guide/mm/damon/reclaim.rst     |    5 +
+ Documentation/admin-guide/mm/damon/stat.rst        |    5 +
+ Documentation/admin-guide/mm/damon/usage.rst       |   16 +-
+ Documentation/admin-guide/mm/kho.rst               |   53 +-
+ .../admin-guide/mm/numa_memory_policy.rst          |    2 +-
+ Documentation/core-api/kho/abi.rst                 |    6 +
+ Documentation/core-api/kho/index.rst               |   18 +-
+ Documentation/core-api/percpu-counter-tree.rst     |   75 +
+ Documentation/dev-tools/kasan.rst                  |    3 -
+ Documentation/dev-tools/kfence.rst                 |    7 +
+ Documentation/driver-api/vme.rst                   |    2 +-
+ Documentation/filesystems/index.rst                |    1 +
+ Documentation/filesystems/mmap_prepare.rst         |  168 ++
+ Documentation/mm/damon/design.rst                  |   18 +
+ Documentation/mm/damon/index.rst                   |    2 +-
+ Documentation/mm/damon/maintainer-profile.rst      |    8 +-
+ Documentation/mm/hugetlbfs_reserv.rst              |    2 +-
+ Documentation/mm/physical_memory.rst               |    4 +-
+ Documentation/mm/vmemmap_dedup.rst                 |   62 +-
+ Documentation/trace/events-kmem.rst                |    8 +-
+ MAINTAINERS                                        |    7 +
+ arch/alpha/include/asm/pgtable.h                   |    6 -
+ arch/arc/include/asm/page.h                        |    2 +-
+ arch/arc/include/asm/pgtable.h                     |    3 -
+ arch/arc/mm/init.c                                 |    2 -
+ arch/arm/include/asm/page.h                        |    2 +-
+ arch/arm/include/asm/pgtable.h                     |    9 -
+ arch/arm/mm/mmu.c                                  |    7 -
+ arch/arm/mm/nommu.c                                |    7 -
+ arch/arm64/include/asm/page.h                      |    7 +-
+ arch/arm64/include/asm/pgtable.h                   |   25 +-
+ arch/arm64/include/asm/pkeys.h                     |    3 +-
+ arch/arm64/mm/gcs.c                                |   14 +-
+ arch/arm64/mm/init.c                               |    5 +
+ arch/arm64/mm/mmu.c                                |    9 +-
+ arch/csky/include/asm/pgtable.h                    |    3 -
+ arch/csky/mm/init.c                                |    3 -
+ arch/hexagon/include/asm/page.h                    |    2 +-
+ arch/hexagon/include/asm/pgtable.h                 |    6 -
+ arch/hexagon/kernel/head.S                         |    5 -
+ arch/hexagon/kernel/hexagon_ksyms.c                |    1 -
+ arch/loongarch/include/asm/page.h                  |    2 +-
+ arch/loongarch/include/asm/pgtable.h               |   12 +-
+ arch/loongarch/mm/init.c                           |    3 -
+ arch/m68k/include/asm/pgtable_mm.h                 |    9 -
+ arch/m68k/include/asm/pgtable_no.h                 |    7 -
+ arch/m68k/mm/init.c                                |    9 -
+ arch/m68k/mm/mcfmmu.c                              |    2 -
+ arch/m68k/mm/motorola.c                            |    6 -
+ arch/m68k/mm/sun3mmu.c                             |    2 -
+ arch/microblaze/include/asm/pgtable.h              |   10 -
+ arch/microblaze/kernel/head.S                      |    4 -
+ arch/microblaze/kernel/microblaze_ksyms.c          |    2 -
+ arch/mips/include/asm/page.h                       |    2 +-
+ arch/mips/mm/init.c                                |   11 +-
+ arch/nios2/include/asm/page.h                      |    2 +-
+ arch/nios2/include/asm/pgtable.h                   |    7 -
+ arch/nios2/kernel/head.S                           |   10 -
+ arch/nios2/kernel/nios2_ksyms.c                    |    1 -
+ arch/openrisc/include/asm/pgtable.h                |    4 -
+ arch/openrisc/kernel/head.S                        |    3 -
+ arch/openrisc/kernel/or32_ksyms.c                  |    1 -
+ arch/openrisc/mm/init.c                            |    3 -
+ arch/parisc/include/asm/pgtable.h                  |   11 -
+ arch/parisc/mm/init.c                              |    6 -
+ arch/powerpc/include/asm/page.h                    |    4 +-
+ arch/powerpc/include/asm/page_32.h                 |    2 +-
+ arch/powerpc/include/asm/page_64.h                 |   12 +-
+ arch/powerpc/include/asm/pgtable.h                 |    6 -
+ arch/powerpc/include/asm/pkeys.h                   |    8 +-
+ arch/powerpc/kvm/book3s_hv.c                       |    1 -
+ arch/powerpc/mm/book3s64/pkeys.c                   |    3 +-
+ arch/powerpc/mm/mem.c                              |    3 -
+ arch/powerpc/platforms/book3s/vas-api.c            |    2 +-
+ arch/powerpc/platforms/pseries/vas.c               |    2 +-
+ arch/riscv/include/asm/page.h                      |    2 +-
+ arch/riscv/include/asm/pgtable.h                   |    7 -
+ arch/riscv/kernel/usercfi.c                        |   12 +-
+ arch/riscv/mm/init.c                               |    7 +-
+ arch/s390/include/asm/page.h                       |    2 +-
+ arch/s390/mm/gmap_helpers.c                        |    2 +-
+ arch/s390/mm/init.c                                |    4 +-
+ arch/sh/include/asm/pgtable.h                      |    8 -
+ arch/sh/include/asm/setup.h                        |    3 +-
+ arch/sh/kernel/head_32.S                           |    4 +-
+ arch/sh/kernel/sh_ksyms_32.c                       |    1 -
+ arch/sh/mm/init.c                                  |    1 -
+ arch/sparc/include/asm/pgtable_32.h                |    8 -
+ arch/sparc/include/asm/pgtable_64.h                |    3 -
+ arch/sparc/include/asm/setup.h                     |    2 -
+ arch/sparc/kernel/head_32.S                        |    7 -
+ arch/sparc/mm/init_32.c                            |    4 -
+ arch/sparc/mm/init_64.c                            |   71 +-
+ arch/um/include/asm/pgtable.h                      |    9 -
+ arch/um/include/shared/kern_util.h                 |    1 -
+ arch/um/kernel/mem.c                               |   16 -
+ arch/um/kernel/um_arch.c                           |    1 -
+ arch/x86/entry/vdso/vdso32/fake_32bit_build.h      |    1 +
+ arch/x86/include/asm/page_types.h                  |    2 +-
+ arch/x86/include/asm/pgtable.h                     |    8 -
+ arch/x86/include/asm/pkeys.h                       |    3 +-
+ arch/x86/kernel/cpu/sgx/encl.c                     |    2 +-
+ arch/x86/kernel/fpu/xstate.c                       |    3 +-
+ arch/x86/kernel/head_32.S                          |    4 -
+ arch/x86/kernel/head_64.S                          |    7 -
+ arch/x86/kernel/shstk.c                            |   12 +-
+ arch/x86/kvm/mmu/spte.h                            |    2 +-
+ arch/x86/mm/pkeys.c                                |    3 +-
+ arch/x86/um/asm/vm-flags.h                         |    4 +-
+ arch/xtensa/include/asm/pgtable.h                  |    4 -
+ arch/xtensa/kernel/head.S                          |    3 -
+ arch/xtensa/kernel/xtensa_ksyms.c                  |    2 -
+ drivers/android/binder/page_range.rs               |    4 +-
+ drivers/android/binder_alloc.c                     |    2 +-
+ drivers/base/memory.c                              |    2 +-
+ drivers/block/zram/backend_lz4.c                   |   29 +-
+ drivers/block/zram/zcomp.c                         |    9 +-
+ drivers/block/zram/zcomp.h                         |    2 +-
+ drivers/block/zram/zram_drv.c                      |  273 ++-
+ drivers/block/zram/zram_drv.h                      |    1 -
+ drivers/char/hpet.c                                |   12 +-
+ drivers/char/mem.c                                 |    2 +-
+ drivers/comedi/comedi_fops.c                       |    2 +-
+ drivers/dax/device.c                               |    2 +-
+ drivers/gpu/drm/drm_gem.c                          |    2 +-
+ drivers/gpu/drm/i915/gem/i915_gem_shmem.c          |    2 +-
+ drivers/gpu/drm/i915/gt/intel_gtt.h                |    2 +-
+ drivers/gpu/drm/i915/i915_gpu_error.c              |    2 +-
+ drivers/gpu/drm/i915/i915_mm.c                     |    4 +-
+ drivers/hv/hv_balloon.c                            |    2 +-
+ drivers/hv/hyperv_vmbus.h                          |    4 +-
+ drivers/hv/vmbus_drv.c                             |   31 +-
+ drivers/hwtracing/stm/core.c                       |   31 +-
+ drivers/infiniband/core/uverbs_main.c              |    6 +-
+ drivers/misc/open-dice.c                           |   19 +-
+ drivers/misc/sgi-gru/grumain.c                     |    2 +-
+ drivers/mtd/mtdchar.c                              |   21 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c   |    2 +-
+ drivers/staging/vme_user/vme.c                     |   20 +-
+ drivers/staging/vme_user/vme.h                     |    2 +-
+ drivers/staging/vme_user/vme_user.c                |   51 +-
+ drivers/target/target_core_user.c                  |   26 +-
+ drivers/uio/uio.c                                  |   10 +-
+ drivers/uio/uio_hv_generic.c                       |   11 +-
+ drivers/virtio/virtio_balloon.c                    |   10 +-
+ fs/afs/file.c                                      |   36 +-
+ fs/afs/internal.h                                  |    1 -
+ fs/afs/write.c                                     |    1 -
+ fs/binfmt_elf.c                                    |    3 +-
+ fs/btrfs/compression.c                             |    2 +-
+ fs/btrfs/extent_io.c                               |    6 +-
+ fs/btrfs/tests/extent-io-tests.c                   |    2 +-
+ fs/buffer.c                                        |    6 +-
+ fs/ceph/addr.c                                     |    2 +-
+ fs/dax.c                                           |    3 +-
+ fs/erofs/data.c                                    |    3 +-
+ fs/ext4/file.c                                     |    1 -
+ fs/ext4/inode.c                                    |    2 +-
+ fs/ext4/page-io.c                                  |    1 -
+ fs/ext4/readpage.c                                 |    1 -
+ fs/f2fs/checkpoint.c                               |    2 +-
+ fs/f2fs/compress.c                                 |    2 +-
+ fs/f2fs/data.c                                     |    2 +-
+ fs/f2fs/f2fs.h                                     |    2 -
+ fs/f2fs/file.c                                     |    1 -
+ fs/f2fs/node.c                                     |    2 +-
+ fs/fs-writeback.c                                  |   22 +-
+ fs/gfs2/aops.c                                     |    2 +-
+ fs/hugetlbfs/inode.c                               |   22 +-
+ fs/mpage.c                                         |    1 -
+ fs/netfs/buffered_write.c                          |    1 -
+ fs/nfs/blocklayout/blocklayout.c                   |    1 -
+ fs/nfs/dir.c                                       |    1 -
+ fs/nilfs2/btree.c                                  |    2 +-
+ fs/nilfs2/page.c                                   |    2 +-
+ fs/nilfs2/segment.c                                |    2 +-
+ fs/ntfs3/file.c                                    |    2 +-
+ fs/ocfs2/refcounttree.c                            |    1 -
+ fs/proc/meminfo.c                                  |    3 +-
+ fs/proc/vmcore.c                                   |    2 +-
+ fs/ramfs/file-nommu.c                              |    2 +-
+ fs/resctrl/pseudo_lock.c                           |    2 +-
+ fs/smb/client/connect.c                            |    1 -
+ fs/smb/client/file.c                               |    1 -
+ fs/zonefs/file.c                                   |    3 +-
+ include/asm-generic/tlb.h                          |    4 +
+ include/linux/damon.h                              |   18 +-
+ include/linux/dax.h                                |    4 +-
+ include/linux/{pagevec.h => folio_batch.h}         |   16 +-
+ include/linux/folio_queue.h                        |    8 +-
+ include/linux/fs.h                                 |   14 +-
+ include/linux/huge_mm.h                            |    5 +
+ include/linux/hugetlb.h                            |   31 +-
+ include/linux/hugetlb_inline.h                     |    4 +-
+ include/linux/hyperv.h                             |    4 +-
+ include/linux/iomap.h                              |    2 +-
+ include/linux/kasan.h                              |    8 +-
+ include/linux/kho/abi/kexec_handover.h             |  144 +-
+ include/linux/kho/abi/memfd.h                      |   18 +-
+ include/linux/kho_radix_tree.h                     |   70 +
+ include/linux/ksm.h                                |   10 +-
+ include/linux/liveupdate.h                         |   17 +-
+ include/linux/maple_tree.h                         |   42 +
+ include/linux/memcontrol.h                         |  193 +-
+ include/linux/memfd.h                              |   12 +
+ include/linux/memory.h                             |    3 +-
+ include/linux/memory_hotplug.h                     |   16 +-
+ include/linux/mm.h                                 |  708 +++++--
+ include/linux/mm_inline.h                          |    6 +
+ include/linux/mm_types.h                           |  145 +-
+ include/linux/mman.h                               |   49 -
+ include/linux/mmdebug.h                            |    8 +-
+ include/linux/mmu_notifier.h                       |   54 -
+ include/linux/mmzone.h                             |  110 +-
+ include/linux/mmzone_lock.h                        |  100 +
+ include/linux/page-flags.h                         |  177 +-
+ include/linux/page_ref.h                           |   18 +-
+ include/linux/page_reporting.h                     |    1 +
+ include/linux/pagewalk.h                           |    8 +-
+ include/linux/percpu_counter_tree.h                |  367 ++++
+ include/linux/pgtable.h                            |   90 +-
+ include/linux/sched.h                              |    2 +-
+ include/linux/shmem_fs.h                           |   14 -
+ include/linux/sunrpc/svc.h                         |    2 +-
+ include/linux/swap.h                               |   55 +-
+ include/linux/types.h                              |    2 +-
+ include/linux/uio_driver.h                         |    4 +-
+ include/linux/userfaultfd_k.h                      |   76 +-
+ include/linux/vm_event_item.h                      |   13 -
+ include/linux/vmalloc.h                            |    3 -
+ include/linux/writeback.h                          |    2 +-
+ include/net/netmem.h                               |   15 +-
+ include/trace/events/huge_memory.h                 |   26 +
+ include/trace/events/kmem.h                        |    2 +-
+ include/trace/events/vmscan.h                      |   95 +-
+ include/trace/events/writeback.h                   |    3 +
+ include/trace/events/zone_lock.h                   |   64 +
+ include/trace/stages/stage3_trace_output.h         |    8 +
+ include/trace/stages/stage7_class_define.h         |   19 +
+ init/main.c                                        |    2 +
+ kernel/bpf/arena.c                                 |    3 +-
+ kernel/cgroup/cgroup.c                             |    9 +-
+ kernel/events/core.c                               |    2 +-
+ kernel/fork.c                                      |   22 +-
+ kernel/liveupdate/kexec_handover.c                 |  711 ++++---
+ kernel/liveupdate/kexec_handover_debugfs.c         |   26 +-
+ kernel/liveupdate/kexec_handover_internal.h        |    3 -
+ kernel/liveupdate/luo_core.c                       |   12 +-
+ kernel/liveupdate/luo_file.c                       |  102 +-
+ kernel/liveupdate/luo_flb.c                        |  182 +-
+ kernel/liveupdate/luo_internal.h                   |    5 +-
+ kernel/liveupdate/luo_session.c                    |   44 -
+ kernel/power/snapshot.c                            |    5 +-
+ kernel/vmcore_info.c                               |    2 +-
+ lib/Kconfig                                        |   12 +
+ lib/Makefile                                       |    2 +
+ lib/maple_tree.c                                   | 2200 +++++++++-----------
+ lib/percpu_counter_tree.c                          |  702 +++++++
+ lib/test_maple_tree.c                              |   55 +-
+ lib/tests/Makefile                                 |    2 +
+ lib/tests/liveupdate.c                             |   18 -
+ lib/tests/percpu_counter_tree_kunit.c              |  595 ++++++
+ lib/vdso/datastore.c                               |    2 +-
+ mm/Makefile                                        |    2 +-
+ mm/compaction.c                                    |   99 +-
+ mm/damon/Kconfig                                   |   11 +
+ mm/damon/core.c                                    |  345 ++-
+ mm/damon/lru_sort.c                                |    7 +-
+ mm/damon/ops-common.c                              |    2 +-
+ mm/damon/paddr.c                                   |    3 +-
+ mm/damon/reclaim.c                                 |    7 +-
+ mm/damon/stat.c                                    |    4 +-
+ mm/damon/sysfs-schemes.c                           |   58 +
+ mm/damon/tests/.kunitconfig                        |    3 +
+ mm/damon/tests/core-kunit.h                        |  100 +
+ mm/damon/tests/vaddr-kunit.h                       |   76 -
+ mm/damon/vaddr.c                                   |   73 +-
+ mm/debug_vm_pgtable.c                              |    4 +-
+ mm/fadvise.c                                       |    2 +-
+ mm/filemap.c                                       |    3 +-
+ mm/gup.c                                           |    2 +-
+ mm/huge_memory.c                                   |  173 +-
+ mm/hugetlb.c                                       |   63 +-
+ mm/hugetlb_vmemmap.c                               |  362 ++--
+ mm/internal.h                                      |  164 +-
+ mm/interval_tree.c                                 |    5 -
+ mm/kasan/init.c                                    |    8 +-
+ mm/kasan/report.c                                  |   13 +-
+ mm/kfence/core.c                                   |   23 +-
+ mm/kfence/kfence.h                                 |   16 +-
+ mm/kfence/report.c                                 |   49 +-
+ mm/khugepaged.c                                    |  280 +--
+ mm/kmemleak.c                                      |    6 +-
+ mm/ksm.c                                           |   57 +-
+ mm/madvise.c                                       |   34 +-
+ mm/memcontrol-v1.c                                 |   36 +-
+ mm/memcontrol-v1.h                                 |   11 +-
+ mm/memcontrol.c                                    |  835 +++++---
+ mm/memfd.c                                         |    4 +-
+ mm/memfd_luo.c                                     |   35 +-
+ mm/memory.c                                        |  536 +++--
+ mm/memory_hotplug.c                                |   19 +-
+ mm/mempolicy.c                                     |    2 +-
+ mm/migrate.c                                       |   21 +-
+ mm/migrate_device.c                                |    4 +
+ mm/mlock.c                                         |   48 +-
+ mm/mm_init.c                                       |   33 +-
+ mm/mmap.c                                          |   21 +-
+ mm/mmu_gather.c                                    |   21 +-
+ mm/mmu_notifier.c                                  |   18 +-
+ mm/mprotect.c                                      |   55 +-
+ mm/mremap.c                                        |  106 +-
+ mm/mseal.c                                         |   11 +-
+ mm/nommu.c                                         |    2 +-
+ mm/oom_kill.c                                      |   24 +-
+ mm/page-writeback.c                                |    4 +-
+ mm/page_alloc.c                                    |  264 +--
+ mm/page_idle.c                                     |    2 +-
+ mm/page_io.c                                       |    8 +-
+ mm/page_isolation.c                                |   23 +-
+ mm/page_owner.c                                    |    2 +-
+ mm/page_reporting.c                                |   20 +-
+ mm/page_vma_mapped.c                               |   11 +-
+ mm/pagewalk.c                                      |   40 +-
+ mm/percpu-internal.h                               |    2 +-
+ mm/percpu.c                                        |    2 +-
+ mm/rmap.c                                          |   31 +-
+ mm/secretmem.c                                     |    2 +-
+ mm/shmem.c                                         |  203 +-
+ mm/show_mem.c                                      |    5 +-
+ mm/shrinker.c                                      |   20 +-
+ mm/shuffle.c                                       |    9 +-
+ mm/slab.h                                          |    8 +-
+ mm/sparse-vmemmap.c                                |  120 +-
+ mm/sparse.c                                        |    5 +
+ mm/swap.c                                          |   63 +-
+ mm/swap.h                                          |   22 +-
+ mm/swap_state.c                                    |   76 +-
+ mm/swap_table.h                                    |  140 +-
+ mm/swapfile.c                                      | 1156 +++++-----
+ mm/truncate.c                                      |    8 +-
+ mm/userfaultfd.c                                   |  726 ++++---
+ mm/util.c                                          |  292 ++-
+ mm/vma.c                                           |  295 ++-
+ mm/vma.h                                           |   88 +-
+ mm/vma_exec.c                                      |    5 +-
+ mm/vmalloc.c                                       |   48 +-
+ mm/vmscan.c                                        |  441 ++--
+ mm/vmstat.c                                        |   38 +-
+ mm/workingset.c                                    |   77 +-
+ mm/zone_lock.c                                     |   28 +
+ mm/zswap.c                                         |  224 +-
+ net/core/netmem_priv.h                             |   23 +-
+ net/core/page_pool.c                               |   24 +-
+ net/ipv4/tcp.c                                     |    7 +-
+ rust/kernel/mm/virt.rs                             |    6 +-
+ security/selinux/hooks.c                           |    4 +-
+ tools/include/linux/bitmap.h                       |   13 +
+ tools/lib/bitmap.c                                 |   10 +
+ tools/testing/radix-tree/maple.c                   |  308 ++-
+ .../testing/selftests/cgroup/test_hugetlb_memcg.c  |   66 +-
+ tools/testing/selftests/cgroup/test_memcontrol.c   |   11 +-
+ tools/testing/selftests/cgroup/test_zswap.c        |  136 ++
+ tools/testing/selftests/damon/_damon_sysfs.py      |   12 +-
+ tools/testing/selftests/damon/config               |    1 +
+ .../selftests/damon/drgn_dump_damon_status.py      |    1 +
+ tools/testing/selftests/damon/sysfs.py             |    7 +
+ tools/testing/selftests/kho/init.c                 |   20 -
+ tools/testing/selftests/kvm/guest_memfd_test.c     |  191 ++
+ .../selftests/mm/charge_reserved_hugetlb.sh        |   44 +-
+ tools/testing/selftests/mm/compaction_test.c       |    3 -
+ tools/testing/selftests/mm/hugepage-mremap.c       |   32 +-
+ .../selftests/mm/hugetlb_reparenting_test.sh       |   31 +-
+ tools/testing/selftests/mm/memory-failure.c        |    1 -
+ tools/testing/selftests/mm/migration.c             |    3 +-
+ tools/testing/selftests/mm/mlock2-tests.c          |   80 +-
+ tools/testing/selftests/mm/pagemap_ioctl.c         |   20 +-
+ tools/testing/selftests/mm/run_vmtests.sh          |   59 +-
+ tools/testing/selftests/mm/split_huge_page_test.c  |    5 +-
+ tools/testing/selftests/mm/uffd-stress.c           |    6 +-
+ tools/testing/selftests/mm/uffd-wp-mremap.c        |   13 +
+ tools/testing/selftests/mm/vm_util.c               |    4 +-
+ tools/testing/selftests/mm/write_to_hugetlbfs.c    |    5 +-
+ tools/testing/vma/include/custom.h                 |   26 +-
+ tools/testing/vma/include/dup.h                    |  486 +++--
+ tools/testing/vma/include/stubs.h                  |   25 +-
+ tools/testing/vma/main.c                           |    2 +
+ tools/testing/vma/shared.c                         |    8 +-
+ tools/testing/vma/shared.h                         |   22 +-
+ tools/testing/vma/tests/merge.c                    |  311 +--
+ tools/testing/vma/tests/mmap.c                     |   18 +-
+ tools/testing/vma/tests/vma.c                      |  395 +++-
+ tools/testing/vma/vma_internal.h                   |    6 +
+ virt/kvm/guest_memfd.c                             |   84 +-
+ virt/kvm/kvm_main.c                                |    1 -
+ 401 files changed, 12865 insertions(+), 7505 deletions(-)
+ create mode 100644 Documentation/core-api/percpu-counter-tree.rst
+ create mode 100644 Documentation/filesystems/mmap_prepare.rst
+ rename include/linux/{pagevec.h => folio_batch.h} (89%)
+ create mode 100644 include/linux/kho_radix_tree.h
+ create mode 100644 include/linux/mmzone_lock.h
+ create mode 100644 include/linux/percpu_counter_tree.h
+ create mode 100644 include/trace/events/zone_lock.h
+ create mode 100644 lib/percpu_counter_tree.c
+ create mode 100644 lib/tests/percpu_counter_tree_kunit.c
+ create mode 100644 mm/zone_lock.c
+Merging mm-nonmm-unstable/mm-nonmm-unstable (ed597c460186e kallsyms: add KUnit tests for lineinfo feature)
+$ git merge -m Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-unstable/mm-nonmm-unstable
+Auto-merging MAINTAINERS
+Auto-merging arch/loongarch/Kconfig
+Auto-merging fs/btrfs/raid56.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/vmcore_info.c
+Auto-merging lib/Kconfig
+Auto-merging lib/Makefile
+Auto-merging lib/tests/Makefile
+Auto-merging scripts/checkpatch.pl
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   5 +-
+ CREDITS                                            |   7 +-
+ Documentation/admin-guide/index.rst                |   1 +
+ Documentation/admin-guide/kallsyms-lineinfo.rst    |  97 +++
+ Documentation/admin-guide/lockup-watchdogs.rst     | 132 +++-
+ Documentation/admin-guide/sysctl/kernel.rst        |   3 +-
+ MAINTAINERS                                        |  10 +
+ arch/alpha/Kconfig                                 |   1 +
+ arch/arm/Kconfig                                   |   1 +
+ arch/arm/lib/Makefile                              |   5 -
+ arch/arm64/Kconfig                                 |   1 +
+ arch/arm64/include/asm/xor.h                       |  73 --
+ arch/arm64/kernel/machine_kexec_file.c             |   4 +
+ arch/arm64/lib/Makefile                            |   6 -
+ arch/loongarch/Kconfig                             |   1 +
+ arch/loongarch/include/asm/xor.h                   |  68 --
+ arch/loongarch/include/asm/xor_simd.h              |  34 -
+ arch/loongarch/lib/Makefile                        |   2 -
+ arch/loongarch/lib/xor_simd_glue.c                 |  72 --
+ arch/powerpc/Kconfig                               |   1 +
+ arch/powerpc/include/asm/xor.h                     |  47 --
+ arch/powerpc/include/asm/xor_altivec.h             |  22 -
+ arch/powerpc/kexec/elf_64.c                        |   4 +
+ arch/powerpc/lib/Makefile                          |   5 -
+ arch/powerpc/lib/xor_vmx.h                         |  22 -
+ arch/powerpc/lib/xor_vmx_glue.c                    |  63 --
+ arch/riscv/Kconfig                                 |   1 +
+ arch/riscv/include/asm/xor.h                       |  68 --
+ arch/riscv/lib/Makefile                            |   1 -
+ arch/s390/Kconfig                                  |   1 +
+ arch/s390/include/asm/xor.h                        |  21 -
+ arch/s390/lib/Makefile                             |   2 +-
+ arch/sparc/Kconfig                                 |   1 +
+ arch/sparc/include/asm/asm-prototypes.h            |   1 -
+ arch/sparc/include/asm/xor.h                       |   9 -
+ arch/sparc/lib/Makefile                            |   2 +-
+ arch/um/Kconfig                                    |   1 +
+ arch/um/include/asm/xor.h                          |  24 -
+ arch/x86/Kconfig                                   |   1 +
+ arch/x86/include/asm/xor_64.h                      |  28 -
+ arch/x86/kernel/kexec-bzimage64.c                  |   6 +-
+ crypto/Kconfig                                     |   2 -
+ crypto/Makefile                                    |   1 -
+ crypto/async_tx/async_xor.c                        |  34 +-
+ crypto/xor.c                                       | 174 -----
+ drivers/char/random.c                              |  26 +-
+ drivers/of/fdt.c                                   |  21 +
+ drivers/of/kexec.c                                 |  19 +
+ fs/btrfs/raid56.c                                  |  27 +-
+ fs/ocfs2/aops.c                                    |   3 +-
+ fs/ocfs2/dlm/dlmdomain.c                           |  10 +-
+ fs/ocfs2/dlm/dlmmaster.c                           |   1 -
+ fs/ocfs2/quota_global.c                            |  16 +-
+ fs/ocfs2/quota_local.c                             |   4 +-
+ fs/proc/array.c                                    |   2 +-
+ fs/squashfs/xz_wrapper.c                           |   6 +-
+ include/asm-generic/Kbuild                         |   1 -
+ include/asm-generic/xor.h                          | 738 ------------------
+ include/linux/crash_core.h                         |  14 +-
+ include/linux/kallsyms.h                           |  17 +-
+ include/linux/mod_lineinfo.h                       | 243 ++++++
+ include/linux/module.h                             |   5 +
+ include/linux/nmi.h                                |   1 +
+ include/linux/raid/xor.h                           |  27 +-
+ include/linux/random.h                             |   1 +
+ init/Kconfig                                       |  35 +
+ kernel/crash_core.c                                |   2 -
+ kernel/crash_dump_dm_crypt.c                       |  22 +-
+ kernel/crash_reserve.c                             |   2 -
+ kernel/exit.c                                      |   8 +-
+ kernel/fork.c                                      |  41 +-
+ kernel/hung_task.c                                 | 106 ++-
+ kernel/kallsyms.c                                  |  58 ++
+ kernel/kallsyms_internal.h                         |  11 +
+ kernel/kexec_core.c                                |   1 -
+ kernel/module/kallsyms.c                           |  76 ++
+ kernel/module/main.c                               |   3 +
+ kernel/panic.c                                     |  47 +-
+ kernel/pid.c                                       |  18 +-
+ kernel/signal.c                                    |  12 +-
+ kernel/vmcore_info.c                               |   2 -
+ kernel/watchdog.c                                  | 158 ++--
+ kernel/watchdog_buddy.c                            |   9 +-
+ lib/Kconfig                                        |   4 +-
+ lib/Kconfig.debug                                  |  42 +
+ lib/Makefile                                       |   4 +-
+ lib/bch.c                                          |   6 +-
+ lib/bug.c                                          |   6 +-
+ lib/cmdline.c                                      |  30 +-
+ lib/debugobjects.c                                 |  15 +-
+ lib/decompress_bunzip2.c                           |   4 +-
+ lib/glob.c                                         |  11 +-
+ lib/idr.c                                          |  55 +-
+ lib/inflate.c                                      |  13 +-
+ lib/kstrtox.c                                      |  37 +-
+ lib/math/Kconfig                                   |   3 +
+ lib/math/Makefile                                  |   1 +
+ lib/{ => math}/polynomial.c                        |  23 +-
+ lib/raid/Kconfig                                   |  19 +
+ lib/raid/Makefile                                  |   3 +
+ lib/raid/xor/Makefile                              |  43 ++
+ .../include/asm/xor.h => lib/raid/xor/alpha/xor.c  |  48 +-
+ lib/raid/xor/alpha/xor_arch.h                      |  22 +
+ lib/raid/xor/arm/xor-neon-glue.c                   |  19 +
+ {arch/arm/lib => lib/raid/xor/arm}/xor-neon.c      |  22 +-
+ .../include/asm/xor.h => lib/raid/xor/arm/xor.c    | 107 +--
+ lib/raid/xor/arm/xor_arch.h                        |  22 +
+ lib/raid/xor/arm64/xor-neon-glue.c                 |  26 +
+ {arch/arm64/lib => lib/raid/xor/arm64}/xor-neon.c  |  94 +--
+ lib/raid/xor/arm64/xor-neon.h                      |   6 +
+ lib/raid/xor/arm64/xor_arch.h                      |  21 +
+ lib/raid/xor/loongarch/xor_arch.h                  |  33 +
+ .../lib => lib/raid/xor/loongarch}/xor_simd.c      |   0
+ .../lib => lib/raid/xor/loongarch}/xor_simd.h      |   0
+ lib/raid/xor/loongarch/xor_simd_glue.c             |  37 +
+ .../lib => lib/raid/xor/loongarch}/xor_template.c  |   0
+ lib/raid/xor/powerpc/xor_arch.h                    |  22 +
+ .../powerpc/lib => lib/raid/xor/powerpc}/xor_vmx.c |  40 +-
+ lib/raid/xor/powerpc/xor_vmx.h                     |  10 +
+ lib/raid/xor/powerpc/xor_vmx_glue.c                |  28 +
+ lib/raid/xor/riscv/xor-glue.c                      |  25 +
+ {arch/riscv/lib => lib/raid/xor/riscv}/xor.S       |   4 -
+ lib/raid/xor/riscv/xor_arch.h                      |  17 +
+ {arch/s390/lib => lib/raid/xor/s390}/xor.c         |  15 +-
+ lib/raid/xor/s390/xor_arch.h                       |  13 +
+ .../xor_32.h => lib/raid/xor/sparc/xor-sparc32.c   |  32 +-
+ .../raid/xor/sparc/xor-sparc64-glue.c              |  48 +-
+ .../lib/xor.S => lib/raid/xor/sparc/xor-sparc64.S  |  10 -
+ lib/raid/xor/sparc/xor_arch.h                      |  35 +
+ lib/raid/xor/tests/Makefile                        |   3 +
+ lib/raid/xor/tests/xor_kunit.c                     | 180 +++++
+ lib/raid/xor/um/xor_arch.h                         |   2 +
+ .../asm/xor_avx.h => lib/raid/xor/x86/xor-avx.c    |  52 +-
+ .../asm/xor_32.h => lib/raid/xor/x86/xor-mmx.c     | 122 +--
+ .../asm/xor.h => lib/raid/xor/x86/xor-sse.c        | 105 +--
+ lib/raid/xor/x86/xor_arch.h                        |  36 +
+ lib/raid/xor/xor-32regs-prefetch.c                 | 267 +++++++
+ lib/raid/xor/xor-32regs.c                          | 217 ++++++
+ lib/raid/xor/xor-8regs-prefetch.c                  | 146 ++++
+ lib/raid/xor/xor-8regs.c                           | 103 +++
+ lib/raid/xor/xor-core.c                            | 191 +++++
+ lib/raid/xor/xor_impl.h                            |  60 ++
+ lib/test-kstrtox.c                                 |   6 +
+ lib/test_ida.c                                     |  14 +
+ lib/tests/Makefile                                 |   3 +
+ lib/tests/cmdline_kunit.c                          | 118 +++
+ lib/tests/lineinfo_kunit.c                         | 813 ++++++++++++++++++++
+ lib/ts_bm.c                                        |  18 +-
+ lib/ts_kmp.c                                       |  18 +-
+ lib/uuid.c                                         |   2 +-
+ scripts/.gitignore                                 |   1 +
+ scripts/Makefile                                   |   4 +
+ scripts/Makefile.modfinal                          |   6 +
+ scripts/bloat-o-meter                              |   6 +-
+ scripts/checkpatch.pl                              |  10 +
+ scripts/decode_stacktrace.sh                       |  26 +-
+ scripts/decodecode                                 |   3 +-
+ scripts/empty_lineinfo.S                           |  38 +
+ scripts/gdb/linux/symbols.py                       |   2 +-
+ scripts/gen-mod-lineinfo.sh                        |  48 ++
+ scripts/gen_lineinfo.c                             | 850 +++++++++++++++++++++
+ scripts/get_maintainer.pl                          |   9 +-
+ scripts/kallsyms.c                                 |  17 +
+ scripts/link-vmlinux.sh                            |  43 +-
+ scripts/spelling.txt                               | 336 ++++----
+ tools/accounting/Makefile                          |   2 +-
+ tools/include/uapi/linux/taskstats.h               | 291 +++++++
+ tools/testing/selftests/breakpoints/Makefile       |   4 +-
+ tools/testing/selftests/fchmodat2/fchmodat2_test.c | 158 ++--
+ tools/testing/selftests/ipc/Makefile               |   8 +-
+ tools/testing/selftests/ipc/msgque.c               |   3 +
+ tools/testing/selftests/prctl/Makefile             |   4 +-
+ tools/testing/selftests/sparc64/Makefile           |   4 +-
+ .../selftests/thermal/intel/power_floor/Makefile   |   4 +-
+ .../selftests/thermal/intel/workload_hint/Makefile |   4 +-
+ 175 files changed, 5652 insertions(+), 2638 deletions(-)
+ create mode 100644 Documentation/admin-guide/kallsyms-lineinfo.rst
+ delete mode 100644 arch/arm64/include/asm/xor.h
+ delete mode 100644 arch/loongarch/include/asm/xor.h
+ delete mode 100644 arch/loongarch/include/asm/xor_simd.h
+ delete mode 100644 arch/loongarch/lib/xor_simd_glue.c
+ delete mode 100644 arch/powerpc/include/asm/xor.h
+ delete mode 100644 arch/powerpc/include/asm/xor_altivec.h
+ delete mode 100644 arch/powerpc/lib/xor_vmx.h
+ delete mode 100644 arch/powerpc/lib/xor_vmx_glue.c
+ delete mode 100644 arch/riscv/include/asm/xor.h
+ delete mode 100644 arch/s390/include/asm/xor.h
+ delete mode 100644 arch/sparc/include/asm/xor.h
+ delete mode 100644 arch/um/include/asm/xor.h
+ delete mode 100644 arch/x86/include/asm/xor_64.h
+ delete mode 100644 crypto/xor.c
+ delete mode 100644 include/asm-generic/xor.h
+ create mode 100644 include/linux/mod_lineinfo.h
+ rename lib/{ => math}/polynomial.c (88%)
+ create mode 100644 lib/raid/Kconfig
+ create mode 100644 lib/raid/Makefile
+ create mode 100644 lib/raid/xor/Makefile
+ rename arch/alpha/include/asm/xor.h => lib/raid/xor/alpha/xor.c (95%)
+ create mode 100644 lib/raid/xor/alpha/xor_arch.h
+ create mode 100644 lib/raid/xor/arm/xor-neon-glue.c
+ rename {arch/arm/lib => lib/raid/xor/arm}/xor-neon.c (53%)
+ rename arch/arm/include/asm/xor.h => lib/raid/xor/arm/xor.c (59%)
+ create mode 100644 lib/raid/xor/arm/xor_arch.h
+ create mode 100644 lib/raid/xor/arm64/xor-neon-glue.c
+ rename {arch/arm64/lib => lib/raid/xor/arm64}/xor-neon.c (76%)
+ create mode 100644 lib/raid/xor/arm64/xor-neon.h
+ create mode 100644 lib/raid/xor/arm64/xor_arch.h
+ create mode 100644 lib/raid/xor/loongarch/xor_arch.h
+ rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.c (100%)
+ rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.h (100%)
+ create mode 100644 lib/raid/xor/loongarch/xor_simd_glue.c
+ rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_template.c (100%)
+ create mode 100644 lib/raid/xor/powerpc/xor_arch.h
+ rename {arch/powerpc/lib => lib/raid/xor/powerpc}/xor_vmx.c (69%)
+ create mode 100644 lib/raid/xor/powerpc/xor_vmx.h
+ create mode 100644 lib/raid/xor/powerpc/xor_vmx_glue.c
+ create mode 100644 lib/raid/xor/riscv/xor-glue.c
+ rename {arch/riscv/lib => lib/raid/xor/riscv}/xor.S (92%)
+ create mode 100644 lib/raid/xor/riscv/xor_arch.h
+ rename {arch/s390/lib => lib/raid/xor/s390}/xor.c (93%)
+ create mode 100644 lib/raid/xor/s390/xor_arch.h
+ rename arch/sparc/include/asm/xor_32.h => lib/raid/xor/sparc/xor-sparc32.c (92%)
+ rename arch/sparc/include/asm/xor_64.h => lib/raid/xor/sparc/xor-sparc64-glue.c (63%)
+ rename arch/sparc/lib/xor.S => lib/raid/xor/sparc/xor-sparc64.S (98%)
+ create mode 100644 lib/raid/xor/sparc/xor_arch.h
+ create mode 100644 lib/raid/xor/tests/Makefile
+ create mode 100644 lib/raid/xor/tests/xor_kunit.c
+ create mode 100644 lib/raid/xor/um/xor_arch.h
+ rename arch/x86/include/asm/xor_avx.h => lib/raid/xor/x86/xor-avx.c (84%)
+ rename arch/x86/include/asm/xor_32.h => lib/raid/xor/x86/xor-mmx.c (87%)
+ rename arch/x86/include/asm/xor.h => lib/raid/xor/x86/xor-sse.c (90%)
+ create mode 100644 lib/raid/xor/x86/xor_arch.h
+ create mode 100644 lib/raid/xor/xor-32regs-prefetch.c
+ create mode 100644 lib/raid/xor/xor-32regs.c
+ create mode 100644 lib/raid/xor/xor-8regs-prefetch.c
+ create mode 100644 lib/raid/xor/xor-8regs.c
+ create mode 100644 lib/raid/xor/xor-core.c
+ create mode 100644 lib/raid/xor/xor_impl.h
+ create mode 100644 lib/tests/lineinfo_kunit.c
+ create mode 100644 scripts/empty_lineinfo.S
+ create mode 100755 scripts/gen-mod-lineinfo.sh
+ create mode 100644 scripts/gen_lineinfo.c
+ create mode 100644 tools/include/uapi/linux/taskstats.h
+Merging kbuild/kbuild-for-next (c9bb03ac2c66b kbuild: reduce output spam when building out of tree)
+$ git merge -m Merge branch 'kbuild-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild/kbuild-for-next
+Auto-merging Makefile
+Auto-merging arch/x86/boot/compressed/Makefile
+Auto-merging init/Kconfig
+Merge made by the 'ort' strategy.
+ Documentation/kbuild/llvm.rst                      |  4 ---
+ Makefile                                           | 31 ++++++++++++++------
+ arch/arm64/kernel/vdso32/Makefile                  |  3 +-
+ arch/hexagon/include/uapi/asm/sigcontext.h         |  2 +-
+ arch/loongarch/vdso/Makefile                       |  2 +-
+ arch/parisc/boot/compressed/Makefile               |  2 +-
+ arch/powerpc/boot/Makefile                         |  3 +-
+ arch/s390/Makefile                                 |  3 +-
+ arch/s390/purgatory/Makefile                       |  3 +-
+ arch/x86/Makefile                                  |  6 +---
+ arch/x86/boot/compressed/Makefile                  |  6 +---
+ drivers/firmware/efi/libstub/Makefile              |  3 +-
+ init/Kconfig                                       |  5 ++++
+ scripts/Makefile.warn                              |  5 ----
+ scripts/kconfig/lexer.l                            |  4 +--
+ scripts/kconfig/lkc.h                              |  3 +-
+ scripts/kconfig/tests/err_repeated_inc/Kconfig     |  3 ++
+ .../kconfig/tests/err_repeated_inc/Kconfig.inc1    |  4 +++
+ .../kconfig/tests/err_repeated_inc/Kconfig.inc2    |  3 ++
+ .../kconfig/tests/err_repeated_inc/Kconfig.inc3    |  1 +
+ scripts/kconfig/tests/err_repeated_inc/__init__.py | 10 +++++++
+ .../kconfig/tests/err_repeated_inc/expected_stderr |  2 ++
+ scripts/kconfig/util.c                             | 33 +++++++++++++++++++---
+ tools/scripts/Makefile.include                     |  2 ++
+ usr/dummy-include/endian.h                         |  0
+ usr/dummy-include/limits.h                         |  8 ++++++
+ usr/dummy-include/netinet/if_ether.h               |  0
+ usr/dummy-include/netinet/in.h                     |  0
+ usr/dummy-include/stdbool.h                        |  7 -----
+ usr/dummy-include/stddef.h                         |  8 ++++++
+ usr/dummy-include/stdint.h                         | 17 +++++++++++
+ usr/dummy-include/stdlib.h                         |  7 -----
+ usr/dummy-include/string.h                         | 12 ++++++++
+ usr/dummy-include/sys/ioctl.h                      |  0
+ usr/dummy-include/sys/socket.h                     | 12 ++++++++
+ usr/dummy-include/sys/time.h                       |  3 ++
+ usr/dummy-include/sys/types.h                      |  0
+ usr/dummy-include/time.h                           |  0
+ usr/dummy-include/unistd.h                         |  0
+ usr/include/Makefile                               | 24 ++++------------
+ 40 files changed, 160 insertions(+), 81 deletions(-)
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/Kconfig
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/Kconfig.inc1
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/Kconfig.inc2
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/Kconfig.inc3
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/__init__.py
+ create mode 100644 scripts/kconfig/tests/err_repeated_inc/expected_stderr
+ create mode 100644 usr/dummy-include/endian.h
+ create mode 100644 usr/dummy-include/limits.h
+ create mode 100644 usr/dummy-include/netinet/if_ether.h
+ create mode 100644 usr/dummy-include/netinet/in.h
+ delete mode 100644 usr/dummy-include/stdbool.h
+ create mode 100644 usr/dummy-include/stddef.h
+ create mode 100644 usr/dummy-include/stdint.h
+ delete mode 100644 usr/dummy-include/stdlib.h
+ create mode 100644 usr/dummy-include/string.h
+ create mode 100644 usr/dummy-include/sys/ioctl.h
+ create mode 100644 usr/dummy-include/sys/socket.h
+ create mode 100644 usr/dummy-include/sys/time.h
+ create mode 100644 usr/dummy-include/sys/types.h
+ create mode 100644 usr/dummy-include/time.h
+ create mode 100644 usr/dummy-include/unistd.h
+Merging clang-format/clang-format (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (616cd6047cbf7 perf symbol: Add RISCV case in get_plt_sizes)
+$ git merge -m Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Auto-merging tools/perf/util/cs-etm-decoder/cs-etm-decoder.c
+Auto-merging tools/perf/util/disasm.c
+Merge made by the 'ort' strategy.
+ tools/perf/Documentation/perf-annotate.txt         |   7 +-
+ tools/perf/Documentation/perf-bench.txt            |   4 +
+ tools/perf/Documentation/perf-diff.txt             |   7 +-
+ tools/perf/Documentation/perf-kwork.txt            |   7 +-
+ tools/perf/Documentation/perf-probe.txt            |   6 +
+ tools/perf/Documentation/perf-report.txt           |   7 +-
+ tools/perf/Documentation/perf-sched.txt            |   7 +-
+ tools/perf/Documentation/perf-script.txt           |   7 +-
+ tools/perf/Documentation/perf-timechart.txt        |   7 +-
+ tools/perf/Documentation/tips.txt                  |   2 +-
+ tools/perf/arch/x86/util/pmu.c                     |  12 +-
+ tools/perf/bench/mem-functions.c                   | 109 +++-
+ tools/perf/builtin-annotate.c                      |   3 +-
+ tools/perf/builtin-diff.c                          |   3 +-
+ tools/perf/builtin-kwork.c                         |   4 +-
+ tools/perf/builtin-probe.c                         |   4 +-
+ tools/perf/builtin-record.c                        |   1 -
+ tools/perf/builtin-report.c                        |   3 +-
+ tools/perf/builtin-sched.c                         |   4 +-
+ tools/perf/builtin-script.c                        |  15 +-
+ tools/perf/builtin-timechart.c                     |   3 +-
+ tools/perf/builtin-trace.c                         |  13 +-
+ tools/perf/perf.c                                  |  12 +-
+ .../arch/arm64/common-and-microarch.json           |  85 +++
+ tools/perf/pmu-events/arch/arm64/mapfile.csv       |   1 +
+ .../pmu-events/arch/arm64/nvidia/t410/branch.json  |  45 ++
+ .../pmu-events/arch/arm64/nvidia/t410/brbe.json    |   6 +
+ .../pmu-events/arch/arm64/nvidia/t410/bus.json     |  48 ++
+ .../arch/arm64/nvidia/t410/exception.json          |  62 ++
+ .../arch/arm64/nvidia/t410/fp_operation.json       |  78 +++
+ .../pmu-events/arch/arm64/nvidia/t410/general.json |  15 +
+ .../arch/arm64/nvidia/t410/l1d_cache.json          | 122 ++++
+ .../arch/arm64/nvidia/t410/l1i_cache.json          | 114 ++++
+ .../arch/arm64/nvidia/t410/l2d_cache.json          | 134 ++++
+ .../arch/arm64/nvidia/t410/ll_cache.json           | 107 +++
+ .../pmu-events/arch/arm64/nvidia/t410/memory.json  |  46 ++
+ .../pmu-events/arch/arm64/nvidia/t410/metrics.json | 722 +++++++++++++++++++++
+ .../pmu-events/arch/arm64/nvidia/t410/misc.json    | 642 ++++++++++++++++++
+ .../pmu-events/arch/arm64/nvidia/t410/retired.json |  94 +++
+ .../pmu-events/arch/arm64/nvidia/t410/spe.json     |  42 ++
+ .../arch/arm64/nvidia/t410/spec_operation.json     | 230 +++++++
+ .../pmu-events/arch/arm64/nvidia/t410/stall.json   | 145 +++++
+ .../pmu-events/arch/arm64/nvidia/t410/tlb.json     | 158 +++++
+ .../perf/pmu-events/arch/x86/alderlake/cache.json  |  27 +-
+ .../pmu-events/arch/x86/alderlake/frontend.json    |  18 +
+ .../pmu-events/arch/x86/alderlake/pipeline.json    |  66 +-
+ .../perf/pmu-events/arch/x86/alderlaken/cache.json |  27 +-
+ .../pmu-events/arch/x86/alderlaken/pipeline.json   |  60 +-
+ .../perf/pmu-events/arch/x86/arrowlake/cache.json  | 103 ++-
+ .../pmu-events/arch/x86/arrowlake/frontend.json    |  18 +
+ .../pmu-events/arch/x86/arrowlake/pipeline.json    |  40 +-
+ .../pmu-events/arch/x86/emeraldrapids/cache.json   |   4 +-
+ .../arch/x86/emeraldrapids/frontend.json           |  16 +
+ .../arch/x86/emeraldrapids/uncore-cache.json       |   4 +-
+ .../arch/x86/emeraldrapids/uncore-io.json          |  17 +-
+ .../perf/pmu-events/arch/x86/grandridge/cache.json |  42 +-
+ .../pmu-events/arch/x86/grandridge/pipeline.json   |  42 +-
+ .../arch/x86/graniterapids/frontend.json           |  16 +
+ .../arch/x86/graniterapids/gnr-metrics.json        |  27 +
+ .../perf/pmu-events/arch/x86/lunarlake/cache.json  |  36 +-
+ .../pmu-events/arch/x86/lunarlake/frontend.json    |  27 +
+ .../pmu-events/arch/x86/lunarlake/pipeline.json    |  10 +-
+ tools/perf/pmu-events/arch/x86/mapfile.csv         |  22 +-
+ .../perf/pmu-events/arch/x86/meteorlake/cache.json |  67 +-
+ .../pmu-events/arch/x86/meteorlake/frontend.json   |  18 +
+ .../pmu-events/arch/x86/meteorlake/pipeline.json   |  46 +-
+ .../pmu-events/arch/x86/pantherlake/cache.json     | 159 ++++-
+ .../arch/x86/pantherlake/floating-point.json       |  28 +
+ .../pmu-events/arch/x86/pantherlake/frontend.json  |  36 +
+ .../pmu-events/arch/x86/pantherlake/memory.json    |  27 +
+ .../pmu-events/arch/x86/pantherlake/other.json     |  10 +
+ .../pmu-events/arch/x86/pantherlake/pipeline.json  | 200 +++++-
+ .../arch/x86/pantherlake/virtual-memory.json       |  30 +
+ .../pmu-events/arch/x86/sapphirerapids/cache.json  |   4 +-
+ .../arch/x86/sapphirerapids/frontend.json          |  16 +
+ .../arch/x86/sapphirerapids/uncore-cache.json      |   4 +-
+ .../arch/x86/sapphirerapids/uncore-io.json         |  17 +-
+ .../pmu-events/arch/x86/sierraforest/cache.json    |  22 +-
+ .../pmu-events/arch/x86/sierraforest/pipeline.json |  42 +-
+ tools/perf/tests/shell/data_type_profiling.sh      |   2 +-
+ tools/perf/tests/workloads/datasym.c               |   6 +-
+ tools/perf/util/Build                              |   2 -
+ tools/perf/util/bpf_map.c                          |  70 --
+ tools/perf/util/bpf_map.h                          |  23 -
+ tools/perf/util/bpf_skel/syscall_summary.bpf.c     |  18 +-
+ tools/perf/util/branch.h                           |   3 +
+ tools/perf/util/cpu-set-sched.h                    |  50 --
+ tools/perf/util/cs-etm-decoder/cs-etm-decoder.c    |  56 +-
+ tools/perf/util/disasm.c                           |   8 +-
+ tools/perf/util/drm_pmu.c                          |  36 +-
+ tools/perf/util/dump-insn.c                        |  23 -
+ tools/perf/util/intel-tpebs.c                      |   1 -
+ tools/perf/util/llvm.c                             |  10 +-
+ tools/perf/util/symbol-elf.c                       |  12 +-
+ tools/perf/util/symbol.c                           |  36 +-
+ tools/perf/util/symbol.h                           |  19 +-
+ tools/perf/util/symbol_conf.h                      |   1 +
+ 97 files changed, 4349 insertions(+), 518 deletions(-)
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/branch.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/brbe.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/bus.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/exception.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/fp_operation.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/general.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/l1d_cache.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/l1i_cache.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/l2d_cache.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/ll_cache.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/memory.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/metrics.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/misc.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/retired.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/spe.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/spec_operation.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/stall.json
+ create mode 100644 tools/perf/pmu-events/arch/arm64/nvidia/t410/tlb.json
+ delete mode 100644 tools/perf/util/bpf_map.c
+ delete mode 100644 tools/perf/util/bpf_map.h
+ delete mode 100644 tools/perf/util/cpu-set-sched.h
+ delete mode 100644 tools/perf/util/dump-insn.c
+Merging compiler-attributes/compiler-attributes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/dma-mapping-for-next (661f8a193d48d dma-mapping: Support batch mode for dma_direct_{map,unmap}_sg)
+$ git merge -m Merge branch 'dma-mapping-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping/dma-mapping-for-next
+Auto-merging arch/arm64/Kconfig
+Auto-merging drivers/iommu/dma-iommu.c
+Auto-merging kernel/dma/direct.h
+Auto-merging kernel/dma/mapping.c
+Auto-merging kernel/dma/swiotlb.c
+Merge made by the 'ort' strategy.
+ arch/arm64/Kconfig                  |   1 +
+ arch/arm64/include/asm/assembler.h  |  25 +++-
+ arch/arm64/include/asm/cache.h      |   5 +
+ arch/arm64/include/asm/cacheflush.h |   2 +
+ arch/arm64/kernel/relocate_kernel.S |   3 +-
+ arch/arm64/mm/cache.S               |  57 ++++++--
+ arch/arm64/mm/dma-mapping.c         |   4 +-
+ drivers/iommu/dma-iommu.c           |  35 +++--
+ drivers/xen/swiotlb-xen.c           |  24 ++--
+ include/linux/dma-map-ops.h         |   6 +
+ include/uapi/linux/map_benchmark.h  |  13 +-
+ kernel/dma/Kconfig                  |   3 +
+ kernel/dma/direct.c                 |  23 +++-
+ kernel/dma/direct.h                 |  21 ++-
+ kernel/dma/map_benchmark.c          | 250 ++++++++++++++++++++++++++++++++----
+ kernel/dma/mapping.c                |   6 +-
+ kernel/dma/swiotlb.c                |   7 +-
+ tools/dma/dma_map_benchmark.c       |  23 +++-
+ 18 files changed, 427 insertions(+), 81 deletions(-)
+Merging asm-generic/master (adbbd9714f805 scripts: headers_install.sh: Remove config leak ignore machinery)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic asm-generic/master
+Already up to date.
+Merging alpha/alpha-next (d58041d2c63e0 MAINTAINERS: Add Magnus Lindholm as maintainer for alpha port)
+$ git merge -m Merge branch 'alpha-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git alpha/alpha-next
+Already up to date.
+Merging arm/for-next (b9cbb9e3bed3f Merge branches 'fixes' and 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm/for-next
+Merge made by the 'ort' strategy.
+Merging arm64/for-next/core (d118f32246fda Merge branches 'for-next/misc' and 'for-next/tlbflush', remote-tracking branch 'arm64/for-next/read-once' into for-next/core)
+$ git merge -m Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm64/include/asm/pgtable.h
+Auto-merging arch/arm64/kvm/hyp/nvhe/mm.c
+Auto-merging arch/arm64/mm/mmu.c
+Merge made by the 'ort' strategy.
+ arch/arm64/Kconfig                |  26 ---
+ arch/arm64/include/asm/hugetlb.h  |  12 +-
+ arch/arm64/include/asm/pgtable.h  |  13 +-
+ arch/arm64/include/asm/rwonce.h   |  24 +-
+ arch/arm64/include/asm/scs.h      |   8 +
+ arch/arm64/include/asm/tlb.h      |   6 +-
+ arch/arm64/include/asm/tlbflush.h | 471 +++++++++++++++++++++-----------------
+ arch/arm64/kernel/entry.S         |   4 +-
+ arch/arm64/kernel/sys_compat.c    |   2 +-
+ arch/arm64/kvm/hyp/nvhe/mm.c      |   2 +-
+ arch/arm64/kvm/hyp/nvhe/tlb.c     |   2 -
+ arch/arm64/kvm/hyp/pgtable.c      |   4 +-
+ arch/arm64/kvm/hyp/vhe/tlb.c      |   2 -
+ arch/arm64/mm/contpte.c           |  12 +-
+ arch/arm64/mm/fault.c             |   2 +-
+ arch/arm64/mm/hugetlbpage.c       |   4 +-
+ arch/arm64/mm/mmu.c               |   2 +-
+ 17 files changed, 323 insertions(+), 273 deletions(-)
+Merging arm-perf/for-next/perf (ab26d9c85554c perf/cxlpmu: Replace IRQF_ONESHOT with IRQF_NO_THREAD)
+$ git merge -m Merge branch 'for-next/perf' of https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (2962cc9b98a52 soc: document merges)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/arm/marvell/armada-7k-8k.yaml         |  11 ++
+ .../bindings/soc/renesas/renesas,rzg2l-sysc.yaml   |   1 +
+ .../devicetree/bindings/soc/renesas/renesas.yaml   |  13 +++
+ arch/arm/arm-soc-for-next-contents.txt             |  40 ++++++++
+ arch/arm/boot/dts/arm/arm-realview-eb.dtsi         |   2 +-
+ arch/arm/boot/dts/arm/arm-realview-pb1176.dts      |   2 +-
+ arch/arm/boot/dts/arm/arm-realview-pb11mp.dts      |   2 +-
+ arch/arm/boot/dts/arm/arm-realview-pbx.dtsi        |   2 +-
+ .../boot/dts/renesas/r8a7740-armadillo800eva.dts   |  32 ++++++
+ .../arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts | 111 +++++++++++++++++++--
+ arch/arm/boot/dts/renesas/r9a06g032.dtsi           |  47 +++++++++
+ arch/arm/configs/shmobile_defconfig                |  11 +-
+ arch/arm64/boot/dts/renesas/ebisu.dtsi             |  43 +++++++-
+ arch/arm64/boot/dts/renesas/r8a77951.dtsi          |  20 ++++
+ arch/arm64/boot/dts/renesas/r8a77960.dtsi          |  20 ++++
+ arch/arm64/boot/dts/renesas/r8a77961.dtsi          |  20 ++++
+ arch/arm64/boot/dts/renesas/r8a77965.dtsi          |  20 ++++
+ arch/arm64/boot/dts/renesas/r8a77990.dtsi          |  27 +++++
+ .../boot/dts/renesas/r8a779g3-sparrow-hawk.dts     |   5 +
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         |  36 +++++++
+ .../boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts    |   4 +
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         |  21 ++++
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         |  16 +++
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         |  16 +++
+ arch/arm64/boot/dts/renesas/salvator-common.dtsi   |  28 +++++-
+ arch/arm64/boot/dts/renesas/ulcb-kf.dtsi           |  21 +++-
+ arch/arm64/boot/dts/renesas/ulcb.dtsi              |  13 +++
+ drivers/soc/renesas/Kconfig                        |  12 +++
+ drivers/soc/renesas/Makefile                       |   1 +
+ drivers/soc/renesas/r9a08g046-sysc.c               |  91 +++++++++++++++++
+ drivers/soc/renesas/rz-sysc.c                      |   3 +
+ drivers/soc/renesas/rz-sysc.h                      |   1 +
+ 32 files changed, 672 insertions(+), 20 deletions(-)
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+ create mode 100644 drivers/soc/renesas/r9a08g046-sysc.c
+Merging amlogic/for-next (e3194dfb77230 Merge branch 'v6.20/arm64-dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+Merging asahi-soc/asahi-soc/for-next (d31647c698f63 Merge branch 'apple-soc/drivers-6.20' into asahi-soc/for-next)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Merge made by the 'ort' strategy.
+Merging at91/at91-next (112090051aa2f Merge branch 'microchip-soc' into at91-next)
+$ git merge -m Merge branch 'at91-next' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+ .../bindings/arm/atmel,at91rm9200-sdramc.yaml      |  66 ++
+ .../bindings/arm/atmel,at91rm9200-st.yaml          |  69 ++
+ .../bindings/arm/atmel,at91sam9260-pit.yaml        |  49 ++
+ .../devicetree/bindings/arm/atmel-at91.yaml        |   6 +
+ .../devicetree/bindings/arm/atmel-sysregs.txt      |  48 --
+ .../bindings/arm/microchip,sam9x60-pit64b.yaml     |  68 ++
+ .../bindings/arm/microchip,sama7g5-chipid.yaml     |  41 ++
+ arch/arm/boot/dts/microchip/sam9x60.dtsi           |   2 -
+ arch/arm/boot/dts/microchip/sama7d65.dtsi          |  35 +
+ arch/arm/configs/sama7_defconfig                   |  10 +
+ arch/arm64/Kconfig.platforms                       |  10 +-
+ arch/arm64/boot/dts/microchip/Makefile             |   1 +
+ arch/arm64/boot/dts/microchip/clk-lan9691.h        |  24 +
+ arch/arm64/boot/dts/microchip/lan9691.dtsi         | 488 +++++++++++++
+ arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts | 756 +++++++++++++++++++++
+ arch/arm64/configs/defconfig                       |   1 +
+ drivers/clk/microchip/clk-mpfs-ccc.c               |   6 +-
+ 17 files changed, 1623 insertions(+), 57 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/arm/atmel,at91rm9200-sdramc.yaml
+ create mode 100644 Documentation/devicetree/bindings/arm/atmel,at91rm9200-st.yaml
+ create mode 100644 Documentation/devicetree/bindings/arm/atmel,at91sam9260-pit.yaml
+ delete mode 100644 Documentation/devicetree/bindings/arm/atmel-sysregs.txt
+ create mode 100644 Documentation/devicetree/bindings/arm/microchip,sam9x60-pit64b.yaml
+ create mode 100644 Documentation/devicetree/bindings/arm/microchip,sama7g5-chipid.yaml
+ create mode 100644 arch/arm64/boot/dts/microchip/clk-lan9691.h
+ create mode 100644 arch/arm64/boot/dts/microchip/lan9691.dtsi
+ create mode 100644 arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
+Merging bmc/for-next (710dbb13377c8 Merge branches 'aspeed/arm/dt', 'aspeed/fixes' and 'nuvoton/arm64/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git bmc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/aspeed/aspeed.yaml     |   2 +
+ arch/arm/boot/dts/aspeed/Makefile                  |   2 +
+ .../aspeed/aspeed-bmc-asrock-paul-ipmi-card.dts    | 131 +++++++++++++++++++++
+ .../aspeed/aspeed-bmc-asus-kommando-ipmi-card.dts  | 117 ++++++++++++++++++
+ .../dts/aspeed/aspeed-bmc-facebook-anacapa.dts     |  22 ++++
+ .../boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts |   1 +
+ .../dts/aspeed/openbmc-flash-layout-128-alt.dtsi   |  32 +++++
+ .../boot/dts/nuvoton/nuvoton-common-npcm8xx.dtsi   |   3 -
+ drivers/soc/aspeed/aspeed-socinfo.c                |   2 +-
+ 9 files changed, 308 insertions(+), 4 deletions(-)
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-paul-ipmi-card.dts
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-asus-kommando-ipmi-card.dts
+ create mode 100644 arch/arm/boot/dts/aspeed/openbmc-flash-layout-128-alt.dtsi
+Merging broadcom/next (ac6f668a23948 Merge branch 'devicetree-arm64/next' into next)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/broadcom/bcm-ns.dtsi             |  58 +++++++++++-
+ arch/arm/boot/dts/broadcom/bcm2835-common.dtsi     |   7 +-
+ arch/arm/boot/dts/broadcom/bcm2835-rpi.dtsi        |  16 ++--
+ .../boot/dts/broadcom/bcm4709-linksys-ea9200.dts   |   7 ++
+ .../boot/dts/broadcom/bcm4709-netgear-r8000.dts    |  73 ++++++--------
+ .../boot/dts/broadcom/bcm47094-asus-rt-ac3100.dtsi |   7 ++
+ .../boot/dts/broadcom/bcm47094-asus-rt-ac5300.dts  |   3 +
+ .../dts/broadcom/bcm47094-linksys-panamera.dts     |   7 ++
+ .../boot/dts/broadcom/bcm47094-luxul-xap-1610.dts  |  36 ++-----
+ .../dts/broadcom/bcm47094-luxul-xwr-3150-v1.dts    |  36 ++-----
+ arch/arm/boot/dts/broadcom/bcm63138.dtsi           |  11 ++-
+ arch/arm/boot/dts/broadcom/bcm63148.dtsi           |  11 ++-
+ arch/arm/boot/dts/broadcom/bcm6846.dtsi            |   9 ++
+ arch/arm/boot/dts/broadcom/bcm6855.dtsi            |  18 ++++
+ arch/arm/boot/dts/broadcom/bcm6878.dtsi            |  11 ++-
+ arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts |  14 +++
+ .../boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi    |  55 +++++------
+ arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts   |  24 +++++
+ arch/arm64/boot/dts/broadcom/bcm2712.dtsi          |  29 +++++-
+ arch/arm64/boot/dts/broadcom/rp1-common.dtsi       | 105 +++++++++++++++++++++
+ 20 files changed, 392 insertions(+), 145 deletions(-)
+Merging cix/for-next (b6812b03e30e6 arm64: dts: cix: add support for cix sky1 resets)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git cix/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/cix/sky1.dtsi | 12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+Merging davinci/davinci/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (82169065ffb07 memory: tegra: Add MC error logging support for Tegra264)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/cache/baikal,bt1-l2-ctl.yaml          |  63 ----
+ drivers/memory/Kconfig                             |  11 -
+ drivers/memory/Makefile                            |   1 -
+ drivers/memory/brcmstb_memc.c                      |   8 +-
+ drivers/memory/bt1-l2-ctl.c                        | 323 ----------------
+ drivers/memory/renesas-rpc-if.c                    |   8 +-
+ drivers/memory/tegra/mc.c                          | 135 ++++---
+ drivers/memory/tegra/mc.h                          | 155 +++++---
+ drivers/memory/tegra/tegra114.c                    |  18 +-
+ drivers/memory/tegra/tegra124-emc.c                |   2 +-
+ drivers/memory/tegra/tegra124.c                    |  40 +-
+ drivers/memory/tegra/tegra186-emc.c                |   8 +
+ drivers/memory/tegra/tegra186.c                    |  22 +-
+ drivers/memory/tegra/tegra194.c                    |  22 +-
+ drivers/memory/tegra/tegra20.c                     |  31 +-
+ drivers/memory/tegra/tegra210.c                    |  21 +-
+ drivers/memory/tegra/tegra234.c                    |  22 +-
+ drivers/memory/tegra/tegra264.c                    | 420 ++++++++++++++++++++-
+ drivers/memory/tegra/tegra30-emc.c                 |   6 +-
+ drivers/memory/tegra/tegra30.c                     |  18 +-
+ include/soc/tegra/mc.h                             |  40 +-
+ 21 files changed, 809 insertions(+), 565 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/cache/baikal,bt1-l2-ctl.yaml
+ delete mode 100644 drivers/memory/bt1-l2-ctl.c
+Merging fsl/soc_fsl (cf8e9203cc653 virt: fsl_hypervisor: fix header kernel-doc warnings)
+$ git merge -m Merge branch 'soc_fsl' of https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git fsl/soc_fsl
+Merge made by the 'ort' strategy.
+ drivers/base/platform-msi.c                 |   4 -
+ drivers/bus/fsl-mc/dprc-driver.c            |  14 +--
+ drivers/bus/fsl-mc/fsl-mc-bus.c             |   4 +-
+ drivers/bus/fsl-mc/fsl-mc-msi.c             | 166 +++++----------------------
+ drivers/bus/fsl-mc/fsl-mc-private.h         |   2 +-
+ drivers/irqchip/Kconfig                     |   6 -
+ drivers/irqchip/Makefile                    |   1 -
+ drivers/irqchip/irq-gic-its-msi-parent.c    |   7 +-
+ drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c | 168 ----------------------------
+ include/linux/fsl/mc.h                      |   6 +-
+ include/linux/irqdomain_defs.h              |   1 -
+ include/uapi/linux/fsl_hypervisor.h         |   4 +-
+ 12 files changed, 41 insertions(+), 342 deletions(-)
+ delete mode 100644 drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
+Merging imx-mxs/for-next (aa31e3bb9b98a Merge branches 'imx/bindings', 'imx/dt', 'imx/dt64', 'imx/fixes' and 'imx/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git imx-mxs/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/arm/freescale/fsl,imx7ulp-pm.yaml     |   5 +
+ Documentation/devicetree/bindings/arm/fsl.yaml     |  48 ++
+ .../interrupt-controller/fsl,irqsteer.yaml         |   4 +-
+ .../bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml |  78 ++
+ MAINTAINERS                                        |   3 +-
+ arch/arm/boot/dts/nxp/imx/imx1-ads.dts             | 100 ++-
+ arch/arm/boot/dts/nxp/imx/imx1-apf9328.dts         |  92 +--
+ arch/arm/boot/dts/nxp/imx/imx1.dtsi                |   2 +-
+ .../boot/dts/nxp/imx/imx25-eukrea-cpuimx25.dtsi    |  38 +-
+ .../imx25-eukrea-mbimxsd25-baseboard-cmo-qvga.dts  |   6 +-
+ .../nxp/imx/imx25-eukrea-mbimxsd25-baseboard.dts   | 134 ++-
+ arch/arm/boot/dts/nxp/imx/imx25-pdk.dts            | 190 +++--
+ arch/arm/boot/dts/nxp/imx/imx25.dtsi               |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx27-apf27.dts          |  56 +-
+ arch/arm/boot/dts/nxp/imx/imx27-apf27dev.dts       | 194 +++--
+ .../boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi    | 244 +++---
+ .../nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts   | 196 +++--
+ arch/arm/boot/dts/nxp/imx/imx27-pdk.dts            | 132 ++-
+ .../dts/nxp/imx/imx27-phytec-phycard-s-rdk.dts     |  92 +--
+ .../dts/nxp/imx/imx27-phytec-phycard-s-som.dtsi    | 162 ++--
+ .../boot/dts/nxp/imx/imx27-phytec-phycore-rdk.dts  | 206 +++--
+ .../boot/dts/nxp/imx/imx27-phytec-phycore-som.dtsi | 154 ++--
+ arch/arm/boot/dts/nxp/imx/imx27.dtsi               |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx31.dtsi               |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx35.dtsi               |  30 +-
+ arch/arm/boot/dts/nxp/imx/imx51-babbage.dts        |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx51-ts4800.dts         |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx53-qsb.dts            |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx53-smd.dts            |  18 +
+ .../arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx6-logicpd-som.dtsi    |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-alti6p.dts        |   4 +-
+ .../boot/dts/nxp/imx/imx6dl-eckelmann-ci4x10.dts   |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-lanmcu.dts        |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-plybas.dts        |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts        |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-prtmvt.dts        |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-qmx6.dtsi         |   5 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts        |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6q-b450v3.dts         |   1 +
+ arch/arm/boot/dts/nxp/imx/imx6q-b650v3.dts         |   1 +
+ arch/arm/boot/dts/nxp/imx/imx6q-b850v3.dts         |  53 +-
+ arch/arm/boot/dts/nxp/imx/imx6q-bx50v3.dtsi        |   4 +
+ arch/arm/boot/dts/nxp/imx/imx6q-prtwd2.dts         |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-icore.dtsi       |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-mba6.dtsi        |  12 +
+ .../boot/dts/nxp/imx/imx6qdl-phytec-pfla02.dtsi    |   6 +-
+ .../dts/nxp/imx/imx6qdl-phytec-phycore-som.dtsi    |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi    |  10 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-sr-som-ti.dtsi   |   8 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6.dtsi       |  11 +
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi      |   5 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi      |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tx6.dtsi         |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi             |  24 +-
+ arch/arm/boot/dts/nxp/imx/imx6ul-geam.dts          |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6ul-isiot.dtsi        |   6 +-
+ .../dts/nxp/imx/imx6ul-phytec-phycore-som.dtsi     |   6 +-
+ .../boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi    |  10 +
+ arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2.dtsi     |   1 +
+ .../boot/dts/nxp/imx/imx6ul-tqma6ul2l-mba6ulx.dts  |   4 +-
+ arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l.dtsi    |   1 +
+ .../boot/dts/nxp/imx/imx6ul-tqma6ulx-common.dtsi   |   1 +
+ .../boot/dts/nxp/imx/imx6ul-tqma6ulxl-common.dtsi  |   1 +
+ arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi        |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6ul.dtsi              |   7 +
+ arch/arm/boot/dts/nxp/imx/imx6ull-colibri.dtsi     |  12 +-
+ arch/arm/boot/dts/nxp/imx/imx6ull-dhcor-som.dtsi   |   4 +-
+ .../dts/nxp/imx/imx6ull-engicam-microgea-bmm.dts   |   1 +
+ .../boot/dts/nxp/imx/imx6ull-engicam-microgea.dtsi |  12 +-
+ .../boot/dts/nxp/imx/imx6ull-myir-mys-6ulx.dtsi    |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2.dtsi   |   1 +
+ .../dts/nxp/imx/imx6ull-tqma6ull2l-mba6ulx.dts     |   2 +-
+ arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l.dtsi  |   3 +-
+ arch/arm/boot/dts/nxp/imx/imx6ull.dtsi             |   1 +
+ arch/arm/boot/dts/nxp/imx/imx6ulz-bsh-smm-m2.dts   |   6 +-
+ arch/arm/boot/dts/nxp/imx/imx7-colibri.dtsi        |   8 +-
+ arch/arm/boot/dts/nxp/imx/imx7-mba7.dtsi           |  13 +
+ arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi          |   5 +
+ arch/arm/boot/dts/nxp/imx/imx7s-warp.dts           |   2 -
+ arch/arm/boot/dts/nxp/imx/imx7s.dtsi               |   5 +
+ arch/arm/boot/dts/nxp/imx/imx7ulp.dtsi             |  28 +
+ arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi             |   6 +
+ arch/arm/boot/dts/nxp/mxs/imx23-olinuxino.dts      |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx23.dtsi               |   6 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-apf28dev.dts       |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-apx4devkit.dts     |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-cfa10036.dts       |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-cfa10049.dts       |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-cfa10055.dts       |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-cfa10056.dts       |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-485.dts |   4 +-
+ .../boot/dts/nxp/mxs/imx28-duckbill-2-enocean.dts  |   6 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2.dts     |   4 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-duckbill.dts       |   4 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-evk.dts            |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-m28cu3.dts         |   4 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-sps1.dts           |   2 +-
+ arch/arm/boot/dts/nxp/mxs/imx28-tx28.dts           |   9 +-
+ arch/arm64/boot/dts/freescale/Makefile             |  19 +
+ .../arm64/boot/dts/freescale/imx8-apalis-eval.dtsi |   4 -
+ .../boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi |  10 +-
+ .../boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi |  10 +-
+ arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi      |  78 +-
+ arch/arm64/boot/dts/freescale/imx8mm-pinfunc.h     |  33 +
+ .../imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtso  |   6 +-
+ .../imx8mm-tx8m-1610-moduline-iv-306-d.dts         | 799 ++++++++++++++++++
+ .../imx8mm-tx8m-1610-moduline-mini-111.dts         | 687 ++++++++++++++++
+ .../arm64/boot/dts/freescale/imx8mm-tx8m-1610.dtsi | 444 ++++++++++
+ arch/arm64/boot/dts/freescale/imx8mm.dtsi          |   2 +-
+ arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi      |   4 +
+ .../imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtso  |   6 +-
+ arch/arm64/boot/dts/freescale/imx8mp-ab2.dts       | 912 +++++++++++++++++++++
+ .../arm64/boot/dts/freescale/imx8mp-beacon-kit.dts |   3 +-
+ .../boot/dts/freescale/imx8mp-beacon-som.dtsi      |   3 +-
+ arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts   |   2 +-
+ .../arm64/boot/dts/freescale/imx8mp-dhcom-pdk3.dts |   2 +-
+ arch/arm64/boot/dts/freescale/imx8mp-evk.dts       |  95 ++-
+ arch/arm64/boot/dts/freescale/imx8mp-frdm.dts      | 132 +--
+ .../imx8mp-hummingboard-pulse-common.dtsi          |   2 +-
+ .../imx8mp-hummingboard-pulse-mini-hdmi.dtsi       |  13 +-
+ .../boot/dts/freescale/imx8mp-kontron-dl.dtso      |  26 +-
+ .../imx8mp-kontron-smarc-eval-carrier.dts          |   1 -
+ arch/arm64/boot/dts/freescale/imx8mp-pinfunc.h     |   2 +-
+ .../freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts   |  71 +-
+ .../imx8mp-tqma8mpql-mba8mpxl-lvds-g133han01.dtso  |   6 +-
+ ...imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtso |   6 +-
+ .../dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts    |  67 ++
+ .../imx8mp-tx8p-ml81-moduline-display-106.dts      |  16 +-
+ .../arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi |   8 +-
+ .../arm64/boot/dts/freescale/imx8mq-librem5-r3.dts |   2 +-
+ arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi  |  24 +-
+ .../imx8mq-tqma8mq-mba8mx-lvds-tm070jvhg33.dtso    |   6 +-
+ arch/arm64/boot/dts/freescale/imx8mq.dtsi          |   2 +-
+ arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi  |   5 +-
+ arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts  |  98 +++
+ .../boot/dts/freescale/imx91-11x11-frdm-s.dts      | 769 +++++++++++++++++
+ arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi  |  20 +-
+ arch/arm64/boot/dts/freescale/imx91.dtsi           |   2 +-
+ arch/arm64/boot/dts/freescale/imx91_93_common.dtsi |  54 ++
+ .../boot/dts/freescale/imx93-11x11-evk-common.dtsi | 861 +++++++++++++++++++
+ arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts  | 855 +------------------
+ .../dts/freescale/imx93-11x11-frdm-pixpaper.dtso   |  50 ++
+ arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts  |  12 +
+ .../imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso       | 110 +++
+ arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts    |   8 +
+ .../boot/dts/freescale/imx93-phyboard-nash.dts     |  12 +-
+ .../freescale/imx93-phyboard-segin-peb-av-02.dtso  | 146 ++++
+ .../boot/dts/freescale/imx93-phyboard-segin.dts    |  12 +-
+ .../boot/dts/freescale/imx93-phycore-som.dtsi      |  12 +-
+ .../dts/freescale/imx93-tqma9352-mba91xxca.dts     |   4 +-
+ .../dts/freescale/imx93-tqma9352-mba93xxca.dts     |   4 +-
+ .../dts/freescale/imx93-tqma9352-mba93xxla.dts     |   4 +-
+ arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi  |  26 +-
+ arch/arm64/boot/dts/freescale/imx93.dtsi           |  12 +
+ arch/arm64/boot/dts/freescale/imx93w-evk.dts       |  28 +
+ arch/arm64/boot/dts/freescale/imx93w.dtsi          | 110 +++
+ arch/arm64/boot/dts/freescale/imx94.dtsi           |  61 ++
+ arch/arm64/boot/dts/freescale/imx943-evk.dts       | 182 ++++
+ arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts |  73 ++
+ .../boot/dts/freescale/imx95-19x19-evk-sof.dts     |   7 +-
+ arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts  |   2 +-
+ .../boot/dts/freescale/imx95-toradex-smarc.dtsi    |  49 +-
+ arch/arm64/boot/dts/freescale/imx95.dtsi           |   7 +
+ arch/arm64/boot/dts/freescale/imx952-evk.dts       |  41 +
+ arch/arm64/boot/dts/freescale/imx952.dtsi          |  19 +
+ arch/arm64/boot/dts/freescale/s32n79-rdb.dts       |  70 ++
+ arch/arm64/boot/dts/freescale/s32n79.dtsi          | 362 ++++++++
+ drivers/bus/imx-aipstz.c                           |  15 +
+ 169 files changed, 7953 insertions(+), 2273 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-iv-306-d.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-mini-111.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-11x11-frdm-pixpaper.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-phyboard-segin-peb-av-02.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93w-evk.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93w.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/s32n79-rdb.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/s32n79.dtsi
+Merging mediatek/for-next (405c09548a695 Merge branch 'v7.0-next/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git mediatek/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/arm/mediatek/mediatek,audsys.yaml     | 59 +++++++++---------
+ arch/arm/boot/dts/mediatek/mt7623.dtsi             |  2 +-
+ arch/arm64/boot/dts/mediatek/mt6359.dtsi           |  4 +-
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts    |  2 +-
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts    |  2 +-
+ arch/arm64/boot/dts/mediatek/mt8167.dtsi           | 70 +++++++++++-----------
+ .../boot/dts/mediatek/mt8195-cherry-dojo-r1.dts    | 38 ++++++++++++
+ arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi    | 58 +++++++++++-------
+ arch/arm64/boot/dts/mediatek/mt8365-evk.dts        |  4 +-
+ arch/arm64/boot/dts/mediatek/mt8365.dtsi           |  5 +-
+ .../boot/dts/mediatek/mt8395-genio-common.dtsi     |  4 +-
+ .../boot/dts/mediatek/mt8395-radxa-nio-12l.dts     |  4 +-
+ 12 files changed, 155 insertions(+), 97 deletions(-)
+Merging mvebu/for-next (6b14ccadcd807 Merge branch 'mvebu/dt64' into mvebu/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   1 -
+ arch/arm/boot/dts/marvell/armada-380.dtsi          | 148 ---------------------
+ .../boot/dts/marvell/armada-3720-atlas-v5.dts      |   1 -
+ .../boot/dts/marvell/armada-3720-espressobin.dtsi  |   1 -
+ arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi  |   5 +
+ arch/arm64/boot/dts/marvell/armada-37xx.dtsi       |   3 +-
+ .../boot/dts/marvell/armada-7020-comexpress.dtsi   |   2 +-
+ arch/arm64/boot/dts/marvell/armada-8020.dtsi       |  20 ---
+ .../boot/dts/marvell/cn9130-db-comexpress.dtsi     |  96 -------------
+ 9 files changed, 7 insertions(+), 270 deletions(-)
+ delete mode 100644 arch/arm/boot/dts/marvell/armada-380.dtsi
+ delete mode 100644 arch/arm64/boot/dts/marvell/armada-8020.dtsi
+ delete mode 100644 arch/arm64/boot/dts/marvell/cn9130-db-comexpress.dtsi
+Merging omap/for-next (adaad0760ba7c Merge branch 'omap-for-v7.1/soc' into tmp/omap-next-20260312.172622)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap/for-next
+Auto-merging arch/arm/configs/multi_v7_defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/ti/omap.yaml |   2 +
+ .../bindings/display/bridge/lvds-codec.yaml        |   1 +
+ .../bindings/display/panel/panel-lvds.yaml         |   4 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ arch/arm/boot/dts/ti/omap/Makefile                 |  10 +
+ arch/arm/boot/dts/ti/omap/dm816x.dtsi              |   4 +-
+ arch/arm/boot/dts/ti/omap/dra7-l4.dtsi             |   4 +-
+ arch/arm/boot/dts/ti/omap/omap4-l4.dtsi            |   4 +-
+ .../dts/ti/omap/omap4-samsung-espresso-common.dtsi | 744 +++++++++++++++++++++
+ .../boot/dts/ti/omap/omap4-samsung-espresso10.dts  | 101 +++
+ .../boot/dts/ti/omap/omap4-samsung-espresso7.dts   |  70 ++
+ arch/arm/boot/dts/ti/omap/omap5-l4.dtsi            |   2 +-
+ arch/arm/boot/dts/ti/omap/twl6032.dtsi             |  77 +++
+ arch/arm/configs/multi_v7_defconfig                |  28 +
+ arch/arm/configs/omap2plus_defconfig               |   1 +
+ arch/arm/mach-omap2/id.c                           |   3 +-
+ include/linux/platform_data/voltage-omap.h         |   4 +-
+ 17 files changed, 1051 insertions(+), 10 deletions(-)
+ create mode 100644 arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
+ create mode 100644 arch/arm/boot/dts/ti/omap/omap4-samsung-espresso10.dts
+ create mode 100644 arch/arm/boot/dts/ti/omap/omap4-samsung-espresso7.dts
+ create mode 100644 arch/arm/boot/dts/ti/omap/twl6032.dtsi
+Merging qcom/for-next (3b11871f8e473 Merge branches 'arm32-for-7.1', 'arm64-defconfig-for-7.1', 'arm64-fixes-for-7.0', 'arm64-for-7.1', 'clk-fixes-for-7.0', 'clk-for-7.1', 'drivers-fixes-for-7.0' and 'drivers-for-7.1' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/cpus.yaml    |    7 +
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   36 +
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |    2 +
+ .../bindings/clock/qcom,glymur-dispcc.yaml         |    4 +-
+ .../bindings/clock/qcom,ipq5210-gcc.yaml           |   62 +
+ .../bindings/clock/qcom,kaanapali-gxclkctl.yaml    |    1 +
+ .../devicetree/bindings/clock/qcom,milos-gcc.yaml  |    9 +-
+ .../devicetree/bindings/clock/qcom,rpmhcc.yaml     |    1 +
+ .../bindings/clock/qcom,sm8450-gpucc.yaml          |    4 +-
+ .../bindings/clock/qcom,sm8450-videocc.yaml        |    3 +
+ .../bindings/clock/qcom,sm8550-tcsr.yaml           |    2 +
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |    1 +
+ .../bindings/interconnect/qcom,eliza-rpmh.yaml     |  142 +
+ .../bindings/interconnect/qcom,osm-l3.yaml         |    1 +
+ .../bindings/soc/qcom/qcom,pmic-glink.yaml         |    2 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ arch/arm/boot/dts/qcom/qcom-ipq8062-smb208.dtsi    |   37 -
+ arch/arm/boot/dts/qcom/qcom-ipq8062.dtsi           |    8 -
+ arch/arm/boot/dts/qcom/qcom-ipq8064-smb208.dtsi    |   37 -
+ .../boot/dts/qcom/qcom-ipq8064-v2.0-smb208.dtsi    |   37 -
+ arch/arm/boot/dts/qcom/qcom-ipq8065-smb208.dtsi    |   37 -
+ arch/arm/boot/dts/qcom/qcom-ipq8065.dtsi           |    8 -
+ arch/arm64/boot/dts/qcom/Makefile                  |   36 +-
+ arch/arm64/boot/dts/qcom/apq8096-db820c.dts        | 1126 +---
+ arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi       | 1133 ++++
+ arch/arm64/boot/dts/qcom/apq8096sg-db820c.dts      |   15 +
+ arch/arm64/boot/dts/qcom/eliza-mtp.dts             |  407 ++
+ arch/arm64/boot/dts/qcom/eliza.dtsi                | 1317 +++++
+ arch/arm64/boot/dts/qcom/glymur-crd.dts            |   14 +
+ arch/arm64/boot/dts/qcom/glymur-crd.dtsi           |  591 ++
+ arch/arm64/boot/dts/qcom/glymur.dtsi               | 5913 ++++++++++++++++++++
+ arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts         |  123 +-
+ arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi        |    6 +
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |   40 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi   |   66 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp418-emmc.dts   |   20 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts        |   44 +-
+ .../arm64/boot/dts/qcom/ipq9574-rdp433-common.dtsi |  121 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp433-emmc.dts   |   20 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts        |  114 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts        |    4 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts        |    4 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts        |    4 +
+ arch/arm64/boot/dts/qcom/ipq9574.dtsi              |   13 +-
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            |    2 +-
+ arch/arm64/boot/dts/qcom/kodiak.dtsi               |   63 +-
+ arch/arm64/boot/dts/qcom/lemans-el2.dtso           |    4 +
+ arch/arm64/boot/dts/qcom/lemans.dtsi               |   14 +-
+ arch/arm64/boot/dts/qcom/mahua-crd.dts             |   21 +
+ arch/arm64/boot/dts/qcom/mahua.dtsi                |  299 +
+ arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts   |   21 +-
+ arch/arm64/boot/dts/qcom/milos.dtsi                |  241 +-
+ arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts  |  466 ++
+ arch/arm64/boot/dts/qcom/monaco-el2.dtso           |   29 +
+ .../boot/dts/qcom/monaco-evk-camera-imx577.dtso    |   66 +
+ arch/arm64/boot/dts/qcom/monaco-evk.dts            |  215 +-
+ arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi     |  323 ++
+ arch/arm64/boot/dts/qcom/monaco.dtsi               |  563 +-
+ arch/arm64/boot/dts/qcom/msm8916-wiko-chuppito.dts |  314 ++
+ arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts   |    2 +-
+ arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts     |   18 +
+ arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts  |    2 +-
+ arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts  |    2 +-
+ arch/arm64/boot/dts/qcom/msm8996-v3.0.dtsi         |   63 -
+ arch/arm64/boot/dts/qcom/pm6125.dtsi               |    1 -
+ arch/arm64/boot/dts/qcom/pmcx0102.dtsi             |  187 +
+ arch/arm64/boot/dts/qcom/pmh0101.dtsi              |   68 +
+ arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi       |  144 +
+ arch/arm64/boot/dts/qcom/pmh0110-glymur.dtsi       |  144 +
+ arch/arm64/boot/dts/qcom/pmk8550.dtsi              |   10 +
+ arch/arm64/boot/dts/qcom/pmk8850.dtsi              |   70 +
+ arch/arm64/boot/dts/qcom/purwa-iot-evk.dts         | 1549 +++++
+ arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi        |  685 +++
+ arch/arm64/boot/dts/qcom/qcm6490-idp.dts           |  171 +-
+ arch/arm64/boot/dts/qcom/qcs615-ride.dts           |   12 +-
+ .../qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso |  269 +
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |    2 +-
+ .../dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts  | 1093 ++++
+ .../boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts |    8 +-
+ arch/arm64/boot/dts/qcom/qcs8300-ride.dts          |  336 ++
+ arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts |  126 +-
+ arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts |  616 ++
+ arch/arm64/boot/dts/qcom/sc7180.dtsi               |    4 +-
+ arch/arm64/boot/dts/qcom/sc8180x.dtsi              |   12 +-
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |   32 +-
+ arch/arm64/boot/dts/qcom/sdm630.dtsi               |   34 +
+ arch/arm64/boot/dts/qcom/sdm845.dtsi               |   56 +
+ arch/arm64/boot/dts/qcom/sm6115.dtsi               |    2 +
+ .../boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi |  313 ++
+ arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts  |  285 +-
+ .../boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts  |   73 +
+ arch/arm64/boot/dts/qcom/sm6125-xiaomi-willow.dts  |   15 +
+ arch/arm64/boot/dts/qcom/sm6125.dtsi               |   28 +
+ arch/arm64/boot/dts/qcom/sm6350.dtsi               |    4 +-
+ arch/arm64/boot/dts/qcom/sm8150.dtsi               |    4 +-
+ arch/arm64/boot/dts/qcom/sm8250.dtsi               |    4 +-
+ arch/arm64/boot/dts/qcom/sm8350.dtsi               |    4 +-
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |    9 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |   77 +-
+ arch/arm64/boot/dts/qcom/sm8650.dtsi               |   15 +-
+ arch/arm64/boot/dts/qcom/sm8750-mtp.dts            |  122 +
+ arch/arm64/boot/dts/qcom/sm8750.dtsi               |  448 +-
+ arch/arm64/boot/dts/qcom/smb2370.dtsi              |   45 +
+ arch/arm64/boot/dts/qcom/talos.dtsi                |  131 +-
+ arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi | 1362 +++++
+ arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi  |   79 +
+ arch/arm64/boot/dts/qcom/x1-crd.dtsi               |   32 +
+ arch/arm64/boot/dts/qcom/x1-el2.dtso               |    8 +
+ .../boot/dts/qcom/x1e80100-asus-vivobook-s15.dts   | 1346 +----
+ .../boot/dts/qcom/x1p42100-asus-vivobook-s15.dts   |   43 +
+ arch/arm64/configs/defconfig                       |   17 +
+ drivers/clk/qcom/Kconfig                           |   59 +-
+ drivers/clk/qcom/Makefile                          |    5 +
+ drivers/clk/qcom/cambistmclkcc-kaanapali.c         |    2 -
+ drivers/clk/qcom/camcc-kaanapali.c                 |    2 -
+ drivers/clk/qcom/camcc-sc8180x.c                   |   67 +-
+ drivers/clk/qcom/clk-rpmh.c                        |   26 +
+ drivers/clk/qcom/dispcc-glymur.c                   |    8 +-
+ drivers/clk/qcom/dispcc-kaanapali.c                |    4 -
+ drivers/clk/qcom/dispcc-milos.c                    |    5 +-
+ drivers/clk/qcom/dispcc-sc8280xp.c                 |    4 -
+ drivers/clk/qcom/dispcc-sm4450.c                   |    1 -
+ drivers/clk/qcom/dispcc-sm6115.c                   |    7 +
+ drivers/clk/qcom/dispcc-sm6125.c                   |    7 +
+ drivers/clk/qcom/dispcc-sm8250.c                   |    6 +-
+ drivers/clk/qcom/dispcc-sm8450.c                   |    2 +-
+ drivers/clk/qcom/dispcc0-sa8775p.c                 |    2 -
+ drivers/clk/qcom/dispcc1-sa8775p.c                 |    2 -
+ drivers/clk/qcom/gcc-eliza.c                       | 3104 ++++++++++
+ drivers/clk/qcom/gcc-glymur.c                      |    4 +-
+ drivers/clk/qcom/gcc-ipq5210.c                     | 2661 +++++++++
+ drivers/clk/qcom/gcc-ipq6018.c                     |    2 +-
+ drivers/clk/qcom/gcc-kaanapali.c                   |    1 -
+ drivers/clk/qcom/gcc-sc8180x.c                     |  126 +-
+ drivers/clk/qcom/gpucc-glymur.c                    |  618 ++
+ drivers/clk/qcom/gpucc-kaanapali.c                 |    1 -
+ drivers/clk/qcom/gxclkctl-kaanapali.c              |    2 +-
+ drivers/clk/qcom/tcsrcc-eliza.c                    |  179 +
+ drivers/clk/qcom/tcsrcc-glymur.c                   |    3 +-
+ drivers/clk/qcom/tcsrcc-kaanapali.c                |    1 -
+ drivers/clk/qcom/tcsrcc-sm8750.c                   |    2 +-
+ drivers/clk/qcom/videocc-glymur.c                  |  532 ++
+ drivers/clk/qcom/videocc-sm8750.c                  |    1 -
+ drivers/firmware/qcom/qcom_qseecom_uefisecapp.c    |    9 +-
+ drivers/firmware/qcom/qcom_scm.c                   |   13 +-
+ drivers/soc/qcom/llcc-qcom.c                       |  186 +-
+ drivers/soc/qcom/pdr_interface.c                   |    4 +-
+ drivers/soc/qcom/pdr_internal.h                    |    5 +-
+ drivers/soc/qcom/pmic_glink.c                      |   66 +-
+ drivers/soc/qcom/pmic_glink_altmode.c              |   39 +-
+ drivers/soc/qcom/qcom_pd_mapper.c                  |   22 +-
+ drivers/soc/qcom/qcom_pdr_msg.c                    |    2 +-
+ drivers/soc/qcom/smp2p.c                           |  103 +-
+ drivers/soc/qcom/socinfo.c                         |   10 +
+ include/dt-bindings/arm/qcom,ids.h                 |    9 +
+ include/dt-bindings/clock/qcom,dispcc-sm6125.h     |    6 +-
+ include/dt-bindings/clock/qcom,eliza-gcc.h         |  210 +
+ include/dt-bindings/clock/qcom,eliza-tcsr.h        |   17 +
+ include/dt-bindings/clock/qcom,gcc-sc8180x.h       |    5 +
+ include/dt-bindings/clock/qcom,glymur-gcc.h        |    1 +
+ include/dt-bindings/clock/qcom,glymur-gpucc.h      |   51 +
+ include/dt-bindings/clock/qcom,glymur-videocc.h    |   45 +
+ include/dt-bindings/clock/qcom,ipq5210-gcc.h       |  126 +
+ include/dt-bindings/clock/qcom,sm6115-dispcc.h     |    7 +-
+ include/dt-bindings/interconnect/qcom,eliza-rpmh.h |  136 +
+ include/dt-bindings/reset/qcom,ipq5210-gcc.h       |  127 +
+ include/linux/soc/qcom/llcc-qcom.h                 |    8 +-
+ include/linux/soc/qcom/pdr.h                       |    1 +
+ include/linux/soc/qcom/qmi.h                       |   12 +
+ samples/qmi/qmi_sample_client.c                    |    2 +-
+ 170 files changed, 29443 insertions(+), 3622 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,ipq5210-gcc.yaml
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,eliza-rpmh.yaml
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8062-smb208.dtsi
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8062.dtsi
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8064-smb208.dtsi
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8064-v2.0-smb208.dtsi
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8065-smb208.dtsi
+ delete mode 100644 arch/arm/boot/dts/qcom/qcom-ipq8065.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/apq8096sg-db820c.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-mtp.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-crd.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-crd.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9574-rdp418-emmc.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9574-rdp433-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9574-rdp433-emmc.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/mahua-crd.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/mahua.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/monaco-el2.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/msm8916-wiko-chuppito.dts
+ delete mode 100644 arch/arm64/boot/dts/qcom/msm8996-v3.0.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmcx0102.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmh0101.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmh0110-glymur.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmk8850.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sm6125-xiaomi-willow.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/smb2370.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/x1p42100-asus-vivobook-s15.dts
+ create mode 100644 drivers/clk/qcom/gcc-eliza.c
+ create mode 100644 drivers/clk/qcom/gcc-ipq5210.c
+ create mode 100644 drivers/clk/qcom/gpucc-glymur.c
+ create mode 100644 drivers/clk/qcom/tcsrcc-eliza.c
+ create mode 100644 drivers/clk/qcom/videocc-glymur.c
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-gcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-tcsr.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-gpucc.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-videocc.h
+ create mode 100644 include/dt-bindings/clock/qcom,ipq5210-gcc.h
+ create mode 100644 include/dt-bindings/interconnect/qcom,eliza-rpmh.h
+ create mode 100644 include/dt-bindings/reset/qcom,ipq5210-gcc.h
+Merging renesas/next (d158b37b9b489 Merge branches 'renesas-arm-defconfig-for-v7.1', 'renesas-arm-soc-for-v7.1', 'renesas-drivers-for-v7.1', 'renesas-dt-bindings-for-v7.1' and 'renesas-dts-for-v7.1' into renesas-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Merge made by the 'ort' strategy.
+ arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c | 16 ++---
+ arch/arm64/boot/dts/renesas/Makefile               |  3 +
+ .../dts/renesas/r9a09g057h48-kakip-pixpaper.dtso   | 39 +++++++++++
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         | 18 ++++++
+ .../boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts    | 71 +++++++++++---------
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         | 18 ++++++
+ .../boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts    | 75 ++++++++++++----------
+ drivers/soc/renesas/r9a09g047-sys.c                |  2 +-
+ drivers/soc/renesas/r9a09g056-sys.c                |  2 +-
+ drivers/soc/renesas/r9a09g057-sys.c                |  2 +-
+ 10 files changed, 165 insertions(+), 81 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a09g057h48-kakip-pixpaper.dtso
+Merging reset/reset/next (223af4a569d1f reset: sti: kzalloc + kcalloc to kzalloc)
+$ git merge -m Merge branch 'reset/next' of https://git.pengutronix.de/git/pza/linux reset/reset/next
+Merge made by the 'ort' strategy.
+ .../bindings/soc/cix/cix,sky1-system-control.yaml  |  42 ++
+ Documentation/driver-api/reset.rst                 |   1 -
+ drivers/reset/Kconfig                              |   7 +
+ drivers/reset/Makefile                             |   1 +
+ drivers/reset/core.c                               | 516 ++++++++++++++-------
+ drivers/reset/reset-ath79.c                        |  12 +-
+ drivers/reset/reset-gpio.c                         |  27 +-
+ drivers/reset/reset-intel-gw.c                     |  11 +-
+ drivers/reset/reset-lpc18xx.c                      |  12 +-
+ drivers/reset/reset-ma35d1.c                       |  11 +-
+ drivers/reset/reset-npcm.c                         |  12 +-
+ drivers/reset/reset-sky1.c                         | 367 +++++++++++++++
+ drivers/reset/reset-sunplus.c                      |  12 +-
+ drivers/reset/sti/reset-syscfg.c                   |   9 +-
+ .../dt-bindings/reset/cix,sky1-s5-system-control.h | 163 +++++++
+ .../dt-bindings/reset/cix,sky1-system-control.h    |  41 ++
+ include/linux/reset-controller.h                   |  21 +-
+ include/linux/reset.h                              |  43 +-
+ 18 files changed, 1034 insertions(+), 274 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/soc/cix/cix,sky1-system-control.yaml
+ create mode 100644 drivers/reset/reset-sky1.c
+ create mode 100644 include/dt-bindings/reset/cix,sky1-s5-system-control.h
+ create mode 100644 include/dt-bindings/reset/cix,sky1-system-control.h
+Merging rockchip/for-next (74b04802e23d9 Merge branch 'v7.1-armsoc/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/rockchip.yaml          |    9 +-
+ .../bindings/clock/rockchip,rv1126b-cru.yaml       |    1 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ arch/arm/boot/dts/rockchip/rk3036-evb.dts          |    4 +-
+ arch/arm/boot/dts/rockchip/rk3036-kylin.dts        |    4 +-
+ arch/arm/boot/dts/rockchip/rk3188-bqedison2qc.dts  |    2 +-
+ .../boot/dts/rockchip/rk3288-firefly-reload.dts    |    3 +-
+ arch/arm/boot/dts/rockchip/rk3288-phycore-rdk.dts  |   16 +-
+ arch/arm/boot/dts/rockchip/rk3288-phycore-som.dtsi |    2 +-
+ arch/arm/boot/dts/rockchip/rk3288-veyron-brain.dts |    2 -
+ .../arm/boot/dts/rockchip/rk3288-veyron-fievel.dts |    5 +-
+ arch/arm/boot/dts/rockchip/rk3288-veyron-jaq.dts   |    2 +-
+ arch/arm/boot/dts/rockchip/rk3288-veyron-jerry.dts |    2 +-
+ .../arm/boot/dts/rockchip/rk3288-veyron-mickey.dts |    2 -
+ arch/arm/boot/dts/rockchip/rk3288.dtsi             |    1 -
+ arch/arm64/boot/dts/rockchip/Makefile              |   22 +
+ arch/arm64/boot/dts/rockchip/rk3328-a1.dts         |   23 -
+ .../boot/dts/rockchip/rk3328-nanopi-r2s-plus.dts   |   17 -
+ arch/arm64/boot/dts/rockchip/rk3399-base.dtsi      |    3 +-
+ .../boot/dts/rockchip/rk3399-gru-chromebook.dtsi   |    2 +
+ .../boot/dts/rockchip/rk3399-nanopc-t4-hd702e.dtso |   96 ++
+ arch/arm64/boot/dts/rockchip/rk3399-nanopi4.dtsi   |    2 +
+ .../boot/dts/rockchip/rk3399-pinebook-pro.dts      |   20 -
+ .../dts/rockchip/rk3399-sapphire-excavator.dts     |    2 +
+ arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts |    4 +-
+ .../dts/rockchip/rk3566-onething-edge-cube.dts     |  342 +++++++
+ arch/arm64/boot/dts/rockchip/rk3566-pinenote.dtsi  |   31 +
+ .../dts/rockchip/rk3566-qnap-ts133-pcb-13.dtso     |   64 ++
+ .../boot/dts/rockchip/rk3568-anbernic-rg-ds.dts    |   44 +-
+ .../dts/rockchip/rk3568-qnap-ts233-pcb-12-11.dtso  |   93 ++
+ .../dts/rockchip/rk3568-qnap-ts433-pcb-12-10.dtso  |  151 +++
+ .../rk3568-wolfvision-pf5-io-expander.dtso         |   20 +
+ arch/arm64/boot/dts/rockchip/rk356x-base.dtsi      |   27 +-
+ arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts   |    1 +
+ arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts   | 1071 ++++++++++++++++++++
+ arch/arm64/boot/dts/rockchip/rk3576.dtsi           |   28 +
+ arch/arm64/boot/dts/rockchip/rk3588-base.dtsi      |    8 +-
+ arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts   |   16 +
+ arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts     |   21 +-
+ arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi |   39 +
+ arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi     |   27 +-
+ arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts  |   16 +
+ .../boot/dts/rockchip/rk3588s-gameforce-ace.dts    |   12 +-
+ drivers/clk/rockchip/Kconfig                       |    7 +
+ drivers/clk/rockchip/Makefile                      |    1 +
+ drivers/clk/rockchip/clk-rk3568.c                  |    6 +
+ drivers/clk/rockchip/clk-rv1103b.c                 |  658 ++++++++++++
+ drivers/clk/rockchip/clk.h                         |   49 +
+ include/dt-bindings/clock/rockchip,rv1103b-cru.h   |  220 ++++
+ 49 files changed, 3064 insertions(+), 136 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3399-nanopc-t4-hd702e.dtso
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3566-onething-edge-cube.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3566-qnap-ts133-pcb-13.dtso
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3568-qnap-ts233-pcb-12-11.dtso
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3568-qnap-ts433-pcb-12-10.dtso
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
+ create mode 100644 drivers/clk/rockchip/clk-rv1103b.c
+ create mode 100644 include/dt-bindings/clock/rockchip,rv1103b-cru.h
+Merging samsung-krzk/for-next (dcba46f4b7a86 Merge branches 'next/dt' and 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/axis.yaml    |    6 +
+ .../bindings/arm/samsung/samsung-boards.yaml       |    3 +
+ .../bindings/clock/axis,artpec9-clock.yaml         |  232 ++++
+ .../clock/samsung,exynosautov920-clock.yaml        |   21 +
+ .../bindings/firmware/google,gs101-acpm-ipc.yaml   |   50 +-
+ arch/arm/boot/dts/samsung/Makefile                 |    1 +
+ arch/arm/boot/dts/samsung/exynos3250-artik5.dtsi   |    2 -
+ arch/arm/boot/dts/samsung/exynos3250-monk.dts      |    4 -
+ arch/arm/boot/dts/samsung/exynos3250-rinato.dts    |    4 -
+ arch/arm/boot/dts/samsung/exynos4210-smdkv310.dts  |    2 -
+ .../boot/dts/samsung/exynos4412-itop-scp-core.dtsi |    2 -
+ arch/arm/boot/dts/samsung/exynos4412-origen.dts    |    2 -
+ arch/arm/boot/dts/samsung/exynos5250-manta.dts     |  564 +++++++++
+ arch/arm/mach-exynos/firmware.c                    |    4 +-
+ arch/arm64/boot/dts/exynos/Makefile                |    2 +
+ arch/arm64/boot/dts/exynos/axis/Makefile           |    3 +-
+ arch/arm64/boot/dts/exynos/axis/artpec9-alfred.dts |   36 +
+ .../boot/dts/exynos/axis/artpec9-pinctrl.dtsi      |  115 ++
+ arch/arm64/boot/dts/exynos/axis/artpec9.dtsi       |  277 +++++
+ .../arm64/boot/dts/exynos/exynos7870-a2corelte.dts |    9 -
+ arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts |  523 +++++++++
+ arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts    |    6 -
+ arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts  |  494 ++++++++
+ arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts |    9 -
+ arch/arm64/boot/dts/exynos/exynos7870.dtsi         |   48 +
+ arch/arm64/boot/dts/exynos/exynos8895-dreamlte.dts |    2 -
+ arch/arm64/boot/dts/exynos/exynos8895.dtsi         |  128 ++
+ arch/arm64/boot/dts/exynos/exynosautov920.dtsi     |   13 +
+ arch/arm64/boot/dts/exynos/google/gs101-oriole.dts |   22 +
+ .../boot/dts/exynos/google/gs101-pixel-common.dtsi |  528 ++++++++-
+ arch/arm64/boot/dts/exynos/google/gs101-raven.dts  |   18 +
+ drivers/clk/samsung/Makefile                       |    1 +
+ drivers/clk/samsung/clk-acpm.c                     |    4 +-
+ drivers/clk/samsung/clk-artpec9.c                  | 1224 ++++++++++++++++++++
+ drivers/clk/samsung/clk-exynosautov920.c           |   52 +
+ drivers/clk/samsung/clk-gs101.c                    |   52 +-
+ drivers/clk/samsung/clk-pll.c                      |  185 ++-
+ drivers/clk/samsung/clk-pll.h                      |   17 +
+ drivers/clk/samsung/clk.c                          |    4 +-
+ drivers/firmware/samsung/exynos-acpm-dvfs.c        |   13 +-
+ drivers/firmware/samsung/exynos-acpm-dvfs.h        |    4 +-
+ drivers/firmware/samsung/exynos-acpm-pmic.c        |   26 +-
+ drivers/firmware/samsung/exynos-acpm-pmic.h        |   10 +-
+ drivers/firmware/samsung/exynos-acpm.c             |   30 +-
+ drivers/firmware/samsung/exynos-acpm.h             |   10 +-
+ drivers/mfd/sec-acpm.c                             |   10 +-
+ include/dt-bindings/clock/axis,artpec9-clk.h       |  195 ++++
+ include/dt-bindings/clock/samsung,exynosautov920.h |    6 +
+ .../linux/firmware/samsung/exynos-acpm-protocol.h  |   40 +-
+ 49 files changed, 4854 insertions(+), 159 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/axis,artpec9-clock.yaml
+ create mode 100644 arch/arm/boot/dts/samsung/exynos5250-manta.dts
+ create mode 100644 arch/arm64/boot/dts/exynos/axis/artpec9-alfred.dts
+ create mode 100644 arch/arm64/boot/dts/exynos/axis/artpec9-pinctrl.dtsi
+ create mode 100644 arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
+ create mode 100644 drivers/clk/samsung/clk-artpec9.c
+ create mode 100644 include/dt-bindings/clock/axis,artpec9-clk.h
+Merging scmi/for-linux-next (351139c6984d7 Merge branches 'for-next/juno/updates' and 'for-next/scmi/updates', tags 'ffa-fix-7.0' and 'scmi-fixes-7.0' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into for-linux-next)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/arm,vexpress-juno.yaml |  10 +
+ .../devicetree/bindings/firmware/arm,scmi.yaml     |  10 +
+ arch/arm64/boot/dts/arm/Makefile                   |   1 +
+ arch/arm64/boot/dts/arm/zena-css-fvp.dts           |  63 ++
+ arch/arm64/boot/dts/arm/zena-css.dtsi              | 769 +++++++++++++++++++++
+ drivers/clk/clk-scmi.c                             |  48 +-
+ drivers/firmware/arm_scmi/base.c                   |   4 +-
+ drivers/firmware/arm_scmi/clock.c                  | 316 +++++++--
+ drivers/firmware/arm_scmi/common.h                 |   4 +
+ drivers/firmware/arm_scmi/driver.c                 |  77 ++-
+ drivers/firmware/arm_scmi/protocols.h              |  13 +-
+ drivers/firmware/arm_scmi/quirks.h                 |   8 +-
+ .../firmware/arm_scmi/vendors/imx/imx-sm-misc.c    |  86 +++
+ drivers/firmware/imx/sm-misc.c                     |  73 ++
+ include/linux/scmi_imx_protocol.h                  |  14 +
+ include/linux/scmi_protocol.h                      |  29 +-
+ 16 files changed, 1369 insertions(+), 156 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/arm/zena-css-fvp.dts
+ create mode 100644 arch/arm64/boot/dts/arm/zena-css.dtsi
+Merging sophgo/for-next (9aa6068586a9b Merge branch 'dt/riscv' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/sophgo/linux.git sophgo/for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/sophgo/cv180x.dtsi | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging sophgo-soc/soc-for-next (c8754c7deab4c soc: sophgo: cv1800: rtcsys: New driver (handling RTC only))
+$ git merge -m Merge branch 'soc-for-next' of https://github.com/sophgo/linux.git sophgo-soc/soc-for-next
+Already up to date.
+Merging spacemit/for-next (28a7f755d7c9a riscv: dts: spacemit: k3: add full resource to UART)
+$ git merge -m Merge branch 'for-next' of https://github.com/spacemit-com/linux spacemit/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/serial/8250.yaml |   5 +-
+ arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts    |  24 +++-
+ arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts  |  16 ++-
+ arch/riscv/boot/dts/spacemit/k1.dtsi               |   4 +
+ arch/riscv/boot/dts/spacemit/k3-pico-itx.dts       |   3 +
+ arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi       |  24 ++++
+ arch/riscv/boot/dts/spacemit/k3.dtsi               | 152 +++++++++++++++++++--
+ 7 files changed, 206 insertions(+), 22 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
+Merging stm32/stm32-next (1dd6f0f026268 ARM: dts: stm32: fix misalignments in nodes of stm32mp131)
+$ git merge -m Merge branch 'stm32-next' of https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/arm,coresight-cti.yaml |   3 +
+ .../bindings/arm/arm,coresight-dynamic-funnel.yaml |   3 +
+ .../devicetree/bindings/arm/arm,coresight-etm.yaml |   3 +
+ .../devicetree/bindings/arm/arm,coresight-stm.yaml |   3 +
+ .../devicetree/bindings/arm/arm,coresight-tmc.yaml |   3 +
+ .../bindings/arm/arm,coresight-tpiu.yaml           |   3 +
+ .../devicetree/bindings/arm/stm32/stm32.yaml       |   8 +-
+ .../bindings/bus/st,stm32mp131-dbg-bus.yaml        |  76 +++++
+ .../devicetree/bindings/pinctrl/st,stm32-hdp.yaml  |   6 +
+ MAINTAINERS                                        |   1 +
+ arch/arm/boot/dts/st/Makefile                      | 129 +++++++-
+ arch/arm/boot/dts/st/stm32mp131.dtsi               | 156 ++++++++--
+ arch/arm/boot/dts/st/stm32mp135f-dk.dts            |  24 ++
+ ...tm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtso |  85 ++++++
+ arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi        | 164 ++++++++++
+ arch/arm/boot/dts/st/stm32mp151.dtsi               | 236 ++++++++++++--
+ arch/arm/boot/dts/st/stm32mp153.dtsi               |  68 +++++
+ arch/arm/boot/dts/st/stm32mp157c-ev1.dts           |  59 ++++
+ ...1-3.dts => stm32mp157c-phyboard-sargas-rdk.dts} |  24 +-
+ ...15xx-avenger96-overlay-644-100-x6-otm8009a.dtso |  61 ++++
+ ...15xx-avenger96-overlay-644-100-x6-rpi7inch.dtso |  31 ++
+ .../stm32mp15xx-avenger96-overlay-fdcan1-x6.dtso   |  10 +
+ .../stm32mp15xx-avenger96-overlay-fdcan2-x6.dtso   |  10 +
+ ...m32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtso |  17 ++
+ ...m32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtso |  17 ++
+ .../stm32mp15xx-avenger96-overlay-ov5640-x7.dtso   |  89 ++++++
+ ...m32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtso |  24 ++
+ .../stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtso  |  10 +
+ ...mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi |  75 +++++
+ .../st/stm32mp15xx-dhcom-overlay-panel-dpi.dtsi    |  74 +++++
+ ...stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtso |  27 ++
+ ...stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtso |  24 ++
+ ...dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtso |  26 ++
+ ...stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtso |  35 +++
+ ...stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtso |  19 ++
+ ...stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtso |  66 ++++
+ ...xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtso |  28 ++
+ ...stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtso |  13 +
+ arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2.dtsi   |  72 -----
+ ...com-picoitx-overlay-548-200-x2-mi0700s4t-6.dtso |  35 +++
+ ...m-picoitx-overlay-553-100-x2-tst043015cmhx.dtso |  35 +++
+ ...com-picoitx-overlay-626-100-x2-ch101olhlwh.dtso |   8 +
+ .../boot/dts/st/stm32mp15xx-dhcor-avenger96.dtsi   |   6 -
+ ...m32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi |  97 ++++++
+ arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi          |  40 +++
+ .../boot/dts/st/stm32mp15xx-phyboard-sargas.dtsi   | 285 +++++++++++++++++
+ ...2mp15-som.dtsi => stm32mp15xx-phycore-som.dtsi} | 339 ++++-----------------
+ arch/arm64/boot/dts/st/stm32mp231.dtsi             |  12 +-
+ arch/arm64/boot/dts/st/stm32mp235f-dk.dts          |  26 ++
+ arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi      |  17 ++
+ arch/arm64/boot/dts/st/stm32mp251.dtsi             |  24 +-
+ arch/arm64/boot/dts/st/stm32mp257f-dk.dts          |  26 ++
+ arch/arm64/boot/dts/st/stm32mp257f-ev1.dts         |   6 +
+ drivers/bus/Kconfig                                |  10 +
+ drivers/bus/Makefile                               |   1 +
+ drivers/bus/stm32_dbg_bus.c                        | 250 +++++++++++++++
+ drivers/bus/stm32_etzpc.c                          |   3 +-
+ drivers/bus/stm32_firewall.c                       |  59 +++-
+ drivers/bus/stm32_rifsc.c                          |  61 ++--
+ drivers/pinctrl/stm32/pinctrl-stm32-hdp.c          |  14 +
+ {drivers => include/linux}/bus/stm32_firewall.h    |   0
+ include/linux/bus/stm32_firewall_device.h          |  26 ++
+ 62 files changed, 2684 insertions(+), 478 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/bus/st,stm32mp131-dbg-bus.yaml
+ create mode 100644 arch/arm/boot/dts/st/stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtso
+ rename arch/arm/boot/dts/st/{stm32mp157c-phycore-stm32mp1-3.dts => stm32mp157c-phyboard-sargas-rdk.dts} (58%)
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan1-x6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan2-x6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-ov5640-x7.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi.dtsi
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtso
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi
+ create mode 100644 arch/arm/boot/dts/st/stm32mp15xx-phyboard-sargas.dtsi
+ rename arch/arm/boot/dts/st/{stm32mp157c-phycore-stm32mp15-som.dtsi => stm32mp15xx-phycore-som.dtsi} (53%)
+ create mode 100644 drivers/bus/stm32_dbg_bus.c
+ rename {drivers => include/linux}/bus/stm32_firewall.h (100%)
+Merging sunxi/sunxi/for-next (2657afa4d7d16 Merge branch 'sunxi/dt-for-7.1' into sunxi/for-next)
+$ git merge -m Merge branch 'sunxi/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/sunxi.yaml   |   5 +
+ .../bindings/leds/allwinner,sun50i-a100-ledc.yaml  |   1 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ arch/arm64/boot/dts/allwinner/Makefile             |   1 +
+ .../boot/dts/allwinner/sun50i-h6-taiqicat-a01.dts  | 361 +++++++++++++++++++++
+ arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi     |  57 ++++
+ .../boot/dts/allwinner/sun55i-t527-avaota-a1.dts   |  56 ++++
+ drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c           |  17 +-
+ 8 files changed, 499 insertions(+), 1 deletion(-)
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-h6-taiqicat-a01.dts
+Merging tee/next (68ab55bcc5f51 Merge branch 'tee_for_v7.1' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ drivers/tee/optee/device.c |   5 +--
+ drivers/tee/optee/supp.c   | 107 +++++++++++++++++++++++++++++++--------------
+ include/linux/tee_core.h   |  30 +++++++------
+ 3 files changed, 91 insertions(+), 51 deletions(-)
+Merging tegra/for-next (5d341fe395182 Merge branch for-7.1/firmware into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Merge made by the 'ort' strategy.
+ drivers/soc/tegra/Kconfig    |  9 +++++++++
+ include/soc/tegra/bpmp-abi.h | 48 ++++++++++++++++++++++----------------------
+ 2 files changed, 33 insertions(+), 24 deletions(-)
+Merging tenstorrent-dt/tenstorrent-dt-for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt/tenstorrent-dt-for-next
+Already up to date.
+Merging thead-dt/thead-dt-for-next (9c99a784d9117 riscv: dts: thead: lichee-pi-4a: enable HDMI)
+$ git merge -m Merge branch 'thead-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt/thead-dt-for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts | 25 +++++++++
+ arch/riscv/boot/dts/thead/th1520.dtsi             | 66 +++++++++++++++++++++++
+ 2 files changed, 91 insertions(+)
+Merging ti/ti-next (def2848b73894 Merge branch 'ti-k3-config-next' into ti-next)
+$ git merge -m Merge branch 'ti-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/ti/k3.yaml   |    1 -
+ .../embedded-controller/kontron,sl28cpld.yaml      |    7 +-
+ arch/arm64/boot/dts/ti/Makefile                    |   13 -
+ arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi    |    7 +
+ arch/arm64/boot/dts/ti/k3-am62a7-sk.dts            |    2 +-
+ arch/arm64/boot/dts/ti/k3-am62d2-evm.dts           |   69 ++
+ arch/arm64/boot/dts/ti/k3-am62l-main.dtsi          |    7 +
+ arch/arm64/boot/dts/ti/k3-am62l-wakeup.dtsi        |    7 +-
+ arch/arm64/boot/dts/ti/k3-am62l.dtsi               |    5 +-
+ arch/arm64/boot/dts/ti/k3-am62l3-evm.dts           |   15 +
+ arch/arm64/boot/dts/ti/k3-am62l3.dtsi              |   47 +
+ arch/arm64/boot/dts/ti/k3-am642-evm.dts            |    1 +
+ arch/arm64/boot/dts/ti/k3-am642-sk.dts             |    2 +
+ arch/arm64/boot/dts/ti/k3-am654-base-board.dts     |    1 +
+ .../boot/dts/ti/k3-am67a-kontron-sa67-ads2.dtso    |  146 ---
+ .../boot/dts/ti/k3-am67a-kontron-sa67-base.dts     | 1091 --------------------
+ .../boot/dts/ti/k3-am67a-kontron-sa67-gbe1.dtso    |   26 -
+ .../boot/dts/ti/k3-am67a-kontron-sa67-gpios.dtso   |   61 --
+ .../dts/ti/k3-am67a-kontron-sa67-rtc-rv8263.dtso   |   31 -
+ arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts   |    2 +-
+ arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts      |    2 +-
+ .../boot/dts/ti/k3-j7200-common-proc-board.dts     |    1 +
+ .../boot/dts/ti/k3-j721e-common-proc-board.dts     |    1 +
+ .../boot/dts/ti/k3-j721s2-common-proc-board.dts    |   42 +
+ arch/arm64/configs/defconfig                       |    4 +-
+ 25 files changed, 209 insertions(+), 1382 deletions(-)
+ delete mode 100644 arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-ads2.dtso
+ delete mode 100644 arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts
+ delete mode 100644 arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gbe1.dtso
+ delete mode 100644 arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gpios.dtso
+ delete mode 100644 arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-rtc-rv8263.dtso
+Merging xilinx/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Already up to date.
+Merging socfpga/for-next (9d29bcefcdbec arm64: dts: intel: agilex5: Drop CPU masks from GICv3 PPI interrupts)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git socfpga/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/altera.yaml  |  7 ++
+ arch/arm64/boot/dts/altera/Makefile                |  1 +
+ .../boot/dts/altera/socfpga_stratix10_socdk.dts    | 67 +-----------------
+ .../boot/dts/altera/socfpga_stratix10_socdk.dtsi   | 71 +++++++++++++++++++
+ .../dts/altera/socfpga_stratix10_socdk_emmc.dts    | 81 ++++++++++++++++++++++
+ arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi     |  8 +--
+ 6 files changed, 166 insertions(+), 69 deletions(-)
+ create mode 100755 arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi
+ create mode 100755 arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts
+Merging clk/clk-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'clk-next' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Already up to date.
+Merging clk-imx/for-next (d16f57caa7877 clk: imx8mq: Correct the CSI PHY sels)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/clock/imx6q-clock.yaml  |  4 ++++
+ Documentation/devicetree/bindings/clock/imx6ul-clock.yaml |  4 ++++
+ drivers/clk/imx/clk-fracn-gppll.c                         |  2 ++
+ drivers/clk/imx/clk-imx6q.c                               | 12 +++++++++++-
+ drivers/clk/imx/clk-imx8-acm.c                            |  3 ++-
+ drivers/clk/imx/clk-imx8mq.c                              |  4 ++--
+ drivers/clk/imx/clk-pll14xx.c                             |  6 +++---
+ drivers/clk/imx/clk-vf610.c                               | 12 ++++++++++++
+ include/dt-bindings/clock/vf610-clock.h                   |  6 +++++-
+ 9 files changed, 45 insertions(+), 8 deletions(-)
+Merging clk-renesas/renesas-clk (c8d5972a25408 clk: renesas: r9a09g056: Add clock and reset entries for RTC)
+$ git merge -m Merge branch 'renesas-clk' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/r9a09g056-cpg.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+Merging thead-clk/thead-clk-for-next (baf4fc7c03bd0 clk: thead: th1520-ap: Support CPU frequency scaling)
+$ git merge -m Merge branch 'thead-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk/thead-clk-for-next
+Already up to date.
+Merging tenstorrent-clk/tenstorrent-clk-for-next (23c8ebc952849 clk: tenstorrent: Add Atlantis clock controller driver)
+$ git merge -m Merge branch 'tenstorrent-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk/tenstorrent-clk-for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/reset/Kconfig
+Auto-merging drivers/reset/Makefile
+Merge made by the 'ort' strategy.
+ .../clock/tenstorrent,atlantis-prcm-rcpu.yaml      |  54 ++
+ MAINTAINERS                                        |   4 +
+ drivers/clk/Kconfig                                |   1 +
+ drivers/clk/Makefile                               |   1 +
+ drivers/clk/tenstorrent/Kconfig                    |  14 +
+ drivers/clk/tenstorrent/Makefile                   |   3 +
+ drivers/clk/tenstorrent/atlantis-prcm.c            | 870 +++++++++++++++++++++
+ drivers/reset/Kconfig                              |  11 +
+ drivers/reset/Makefile                             |   1 +
+ drivers/reset/reset-tenstorrent-atlantis.c         | 173 ++++
+ .../clock/tenstorrent,atlantis-prcm-rcpu.h         | 103 +++
+ 11 files changed, 1235 insertions(+)
+ create mode 100644 Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml
+ create mode 100644 drivers/clk/tenstorrent/Kconfig
+ create mode 100644 drivers/clk/tenstorrent/Makefile
+ create mode 100644 drivers/clk/tenstorrent/atlantis-prcm.c
+ create mode 100644 drivers/reset/reset-tenstorrent-atlantis.c
+ create mode 100644 include/dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h
+Merging csky/linux-next (817d21578d51e csky: Remove compile warning for CONFIG_SMP)
+$ git merge -m Merge branch 'linux-next' of https://github.com/c-sky/csky-linux.git csky/linux-next
+Already up to date.
+Merging loongarch/loongarch-next (c252c12d1f55b LoongArch: KVM: Fix typo issue in kvm_vm_init_features())
+$ git merge -m Merge branch 'loongarch-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Already up to date.
+Merging m68k/for-next (cb39cf99d88e8 m68k: sun3: Replace vsprintf() with bounded vsnprintf())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Already up to date.
+Merging m68knommu/for-next (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Already up to date.
+Merging microblaze/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Already up to date.
+Merging mips/mips-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'mips-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Already up to date.
+Merging openrisc/for-next (0dfffa5479d62 openrisc: define arch-specific version of nop())
+$ git merge -m Merge branch 'for-next' of https://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (2c98a8fbd6aa6 parisc: Flush correct cache in cacheflush() syscall)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Already up to date.
+Merging powerpc/next (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Already up to date.
+Merging risc-v/for-next (18be4ca5cb4e5 riscv: lib: optimize strlen loop efficiency)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Already up to date.
+Merging riscv-dt/riscv-dt-for-next (d95214e862ab5 riscv: dts: microchip: add tsu clock to macb on mpfs)
+$ git merge -m Merge branch 'riscv-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Auto-merging arch/riscv/boot/dts/microchip/mpfs.dtsi
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/riscv/extensions.yaml      |  27 +
+ .../devicetree/bindings/riscv/microchip.yaml       |   7 +-
+ .../devicetree/bindings/timer/sifive,clint.yaml    |   1 +
+ arch/riscv/boot/dts/microchip/Makefile             |  17 +-
+ .../boot/dts/microchip/mpfs-icicle-kit-common.dtsi |   1 -
+ .../boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi |  63 +++
+ arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi    | 167 ++++++
+ arch/riscv/boot/dts/microchip/mpfs.dtsi            |  24 +-
+ .../boot/dts/microchip/pic64gx-curiosity-kit.dts   | 165 ++++++
+ arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi | 177 ++++++
+ arch/riscv/boot/dts/microchip/pic64gx.dtsi         | 630 +++++++++++++++++++++
+ 11 files changed, 1264 insertions(+), 15 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi
+ create mode 100644 arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
+ create mode 100644 arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi
+ create mode 100644 arch/riscv/boot/dts/microchip/pic64gx.dtsi
+Merging riscv-soc/riscv-soc-for-next (9a9d7b6a45f5f Merge branch 'k230-basic' into riscv-soc-for-next)
+$ git merge -m Merge branch 'riscv-soc-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Auto-merging Documentation/devicetree/bindings/timer/sifive,clint.yaml
+Auto-merging drivers/soc/microchip/mpfs-sys-controller.c
+Merge made by the 'ort' strategy.
+ .../interrupt-controller/sifive,plic-1.0.0.yaml    |   1 +
+ .../devicetree/bindings/riscv/canaan.yaml          |   8 +-
+ .../microchip/microchip,mpfs-mss-top-sysreg.yaml   |  14 +-
+ .../microchip/microchip,mpfs-sys-controller.yaml   |   4 +-
+ .../devicetree/bindings/timer/sifive,clint.yaml    |   1 +
+ arch/riscv/boot/dts/canaan/Makefile                |   2 +
+ arch/riscv/boot/dts/canaan/k230-canmv.dts          | 332 +++++++++++++++++++++
+ arch/riscv/boot/dts/canaan/k230-evb.dts            |  28 ++
+ arch/riscv/boot/dts/canaan/k230-pinctrl.h          |  18 ++
+ arch/riscv/boot/dts/canaan/k230.dtsi               | 157 ++++++++++
+ drivers/soc/microchip/mpfs-sys-controller.c        |  74 +++--
+ 11 files changed, 613 insertions(+), 26 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-canmv.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-evb.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-pinctrl.h
+ create mode 100644 arch/riscv/boot/dts/canaan/k230.dtsi
+Merging s390/for-next (efa9a04bf4945 Merge branch 'features' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Auto-merging arch/arm64/include/asm/pgtable.h
+Auto-merging arch/powerpc/include/asm/pgtable.h
+Auto-merging arch/riscv/include/asm/pgtable.h
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/x86/include/asm/pgtable.h
+Merge made by the 'ort' strategy.
+ arch/arm64/include/asm/pgtable.h             |  6 +--
+ arch/powerpc/include/asm/book3s/32/pgtable.h |  2 +-
+ arch/powerpc/include/asm/book3s/64/pgtable.h | 10 ++---
+ arch/powerpc/include/asm/nohash/pgtable.h    |  2 +-
+ arch/powerpc/include/asm/pgtable.h           |  4 +-
+ arch/riscv/include/asm/pgtable.h             |  6 +--
+ arch/s390/Kconfig                            |  3 +-
+ arch/s390/boot/startup.c                     |  3 +-
+ arch/s390/configs/debug_defconfig            |  2 +
+ arch/s390/include/asm/pgtable.h              | 60 ++++++++++++++++++++++++----
+ arch/s390/include/asm/setup.h                |  1 -
+ arch/x86/include/asm/pgtable.h               |  6 +--
+ mm/page_table_check.c                        | 15 +++----
+ 13 files changed, 83 insertions(+), 37 deletions(-)
+Merging sh/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Already up to date.
+Merging sparc/for-next (d844152d85cfc sparc: remove unused variable strtab)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc/for-next
+Already up to date.
+Merging uml/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Already up to date.
+Merging xtensa/xtensa-for-next (aa5903b47d4bc xtensa: uapi: Reuse asm-generic ucontext.h)
+$ git merge -m Merge branch 'xtensa-for-next' of https://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/include/asm/ucontext.h   | 22 ----------------------
+ arch/xtensa/include/uapi/asm/Kbuild  |  1 +
+ arch/xtensa/platforms/iss/setup.c    | 11 ++++-------
+ arch/xtensa/platforms/xt2000/setup.c | 11 ++++-------
+ arch/xtensa/platforms/xtfpga/setup.c | 11 ++++-------
+ 5 files changed, 13 insertions(+), 43 deletions(-)
+ delete mode 100644 arch/xtensa/include/asm/ucontext.h
+Merging fs-next (40cdae4812a3f next-20260312/vfs-brauner)
+$ git merge -m Merge branch 'fs-next' of linux-next fs-next
+Auto-merging CREDITS
+Auto-merging Documentation/filesystems/index.rst
+Auto-merging MAINTAINERS
+Auto-merging fs/btrfs/compression.c
+Auto-merging fs/btrfs/extent_io.c
+Auto-merging fs/btrfs/raid56.c
+Auto-merging fs/buffer.c
+Auto-merging fs/ceph/addr.c
+Auto-merging fs/ext4/inode.c
+Auto-merging fs/ext4/page-io.c
+Auto-merging fs/ext4/readpage.c
+Auto-merging fs/f2fs/compress.c
+Auto-merging fs/f2fs/data.c
+Auto-merging fs/f2fs/f2fs.h
+Auto-merging fs/f2fs/file.c
+Auto-merging fs/f2fs/node.c
+Auto-merging fs/gfs2/aops.c
+Auto-merging fs/nfs/blocklayout/blocklayout.c
+Auto-merging fs/nfs/dir.c
+Auto-merging fs/nilfs2/btree.c
+Auto-merging fs/nilfs2/segment.c
+Auto-merging fs/ntfs3/file.c
+Auto-merging fs/ocfs2/aops.c
+Auto-merging fs/ocfs2/quota_local.c
+Auto-merging fs/ocfs2/refcounttree.c
+Auto-merging fs/smb/client/connect.c
+Auto-merging fs/smb/client/file.c
+Auto-merging include/linux/fs.h
+Auto-merging include/linux/iomap.h
+Auto-merging include/linux/shmem_fs.h
+Auto-merging include/linux/sunrpc/svc.h
+Auto-merging include/trace/events/writeback.h
+Auto-merging kernel/fork.c
+Auto-merging kernel/signal.c
+Auto-merging mm/shmem.c
+Auto-merging net/key/af_key.c
+Auto-merging security/selinux/hooks.c
+Merge made by the 'ort' strategy.
+ CREDITS                                            |    9 +-
+ Documentation/ABI/testing/sysfs-fs-f2fs            |    6 +
+ .../admin-guide/nfs/pnfs-block-server.rst          |   30 +
+ Documentation/admin-guide/nfs/pnfs-scsi-server.rst |   31 +
+ Documentation/filesystems/index.rst                |    1 +
+ Documentation/filesystems/locking.rst              |    2 +
+ Documentation/filesystems/mount_api.rst            |    4 -
+ Documentation/filesystems/ntfs.rst                 |  159 +
+ Documentation/filesystems/porting.rst              |   14 +
+ Documentation/sunrpc/xdr/nlm4.x                    |  211 +
+ MAINTAINERS                                        |    9 +
+ block/Makefile                                     |    2 +-
+ block/bio-integrity-auto.c                         |   80 +-
+ block/bio-integrity-fs.c                           |   81 +
+ block/bio-integrity.c                              |   64 +
+ block/bio.c                                        |   17 +-
+ block/blk-mq.c                                     |    6 +-
+ block/blk-settings.c                               |   13 -
+ block/blk.h                                        |    6 +-
+ block/t10-pi.c                                     |   12 +-
+ drivers/dma-buf/dma-buf.c                          |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    4 +-
+ drivers/mtd/ubi/ubi.h                              |    9 +-
+ drivers/nvdimm/btt.c                               |    6 +-
+ fs/9p/vfs_addr.c                                   |    4 +-
+ fs/9p/vfs_inode.c                                  |    6 +-
+ fs/9p/vfs_inode_dotl.c                             |    6 +-
+ fs/9p/vfs_super.c                                  |    6 +-
+ fs/Kconfig                                         |    1 +
+ fs/Makefile                                        |    1 +
+ fs/affs/amigaffs.c                                 |   10 +-
+ fs/affs/bitmap.c                                   |    2 +-
+ fs/affs/dir.c                                      |    2 +-
+ fs/affs/file.c                                     |   20 +-
+ fs/affs/inode.c                                    |   12 +-
+ fs/affs/namei.c                                    |   14 +-
+ fs/affs/symlink.c                                  |    2 +-
+ fs/afs/dir.c                                       |   10 +-
+ fs/afs/dir_search.c                                |    2 +-
+ fs/afs/dynroot.c                                   |    2 +-
+ fs/afs/inode.c                                     |    2 +-
+ fs/autofs/inode.c                                  |    2 +-
+ fs/befs/linuxvfs.c                                 |   28 +-
+ fs/bfs/dir.c                                       |    4 +-
+ fs/binfmt_elf_fdpic.c                              |    2 +-
+ fs/btrfs/Kconfig                                   |    1 +
+ fs/btrfs/Makefile                                  |    4 +
+ fs/btrfs/backref.c                                 |   10 -
+ fs/btrfs/bio.c                                     |    3 +-
+ fs/btrfs/block-group.c                             |  329 +-
+ fs/btrfs/block-group.h                             |    1 +
+ fs/btrfs/block-rsv.c                               |   25 +
+ fs/btrfs/compression.c                             |    5 +-
+ fs/btrfs/compression.h                             |    6 +-
+ fs/btrfs/ctree.c                                   |   36 +-
+ fs/btrfs/delayed-ref.c                             |   28 +
+ fs/btrfs/dev-replace.c                             |    4 +-
+ fs/btrfs/dir-item.c                                |    4 +-
+ fs/btrfs/direct-io.c                               |   11 +-
+ fs/btrfs/disk-io.c                                 |  228 +-
+ fs/btrfs/disk-io.h                                 |    7 +-
+ fs/btrfs/extent-tree.c                             |   10 +-
+ fs/btrfs/extent_io.c                               |  161 +-
+ fs/btrfs/extent_io.h                               |    8 +-
+ fs/btrfs/file-item.c                               |    6 +-
+ fs/btrfs/file-item.h                               |    6 +-
+ fs/btrfs/file.c                                    |    4 +-
+ fs/btrfs/fs.h                                      |    9 +-
+ fs/btrfs/inode.c                                   |  298 +-
+ fs/btrfs/ioctl.c                                   |    4 +-
+ fs/btrfs/lzo.c                                     |    7 +-
+ fs/btrfs/messages.h                                |    8 +-
+ fs/btrfs/misc.h                                    |   18 +-
+ fs/btrfs/ordered-data.c                            |   67 +-
+ fs/btrfs/ordered-data.h                            |   66 +-
+ fs/btrfs/print-tree.c                              |    4 -
+ fs/btrfs/qgroup.c                                  |   10 +-
+ fs/btrfs/raid-stripe-tree.c                        |    2 +-
+ fs/btrfs/raid56.c                                  |   17 +-
+ fs/btrfs/reflink.c                                 |    4 +-
+ fs/btrfs/relocation.c                              |   12 +-
+ fs/btrfs/scrub.c                                   |   22 +-
+ fs/btrfs/send.c                                    |    6 +-
+ fs/btrfs/space-info.c                              |   63 +-
+ fs/btrfs/space-info.h                              |   22 +-
+ fs/btrfs/super.h                                   |    2 +-
+ fs/btrfs/tests/btrfs-tests.c                       |    3 +
+ fs/btrfs/tests/btrfs-tests.h                       |   10 +
+ fs/btrfs/tests/zoned-tests.c                       |  675 +++
+ fs/btrfs/transaction.c                             |   78 +-
+ fs/btrfs/transaction.h                             |    3 +
+ fs/btrfs/tree-checker.c                            |  122 +
+ fs/btrfs/tree-log.c                                |  158 +-
+ fs/btrfs/tree-mod-log.c                            |    8 +-
+ fs/btrfs/uuid-tree.c                               |    7 +-
+ fs/btrfs/volumes.c                                 |  186 +-
+ fs/btrfs/volumes.h                                 |   34 +
+ fs/btrfs/zlib.c                                    |    2 +-
+ fs/btrfs/zoned.c                                   |   15 +-
+ fs/btrfs/zstd.c                                    |    2 +-
+ fs/buffer.c                                        |   18 +-
+ fs/cachefiles/io.c                                 |    6 +-
+ fs/cachefiles/namei.c                              |  109 +-
+ fs/cachefiles/xattr.c                              |    2 +-
+ fs/ceph/addr.c                                     |    4 +
+ fs/ceph/crypto.c                                   |    4 +-
+ fs/coda/dir.c                                      |    2 +-
+ fs/coda/inode.c                                    |    2 +-
+ fs/cramfs/inode.c                                  |    2 +-
+ fs/crypto/bio.c                                    |   38 +-
+ fs/crypto/crypto.c                                 |    2 +-
+ fs/crypto/fscrypt_private.h                        |    3 -
+ fs/crypto/hooks.c                                  |    2 +-
+ fs/crypto/inline_crypt.c                           |   86 +-
+ fs/crypto/keyring.c                                |    4 +-
+ fs/crypto/keysetup.c                               |    4 +-
+ fs/dcache.c                                        |    8 +-
+ fs/ecryptfs/crypto.c                               |    6 +-
+ fs/ecryptfs/file.c                                 |    2 +-
+ fs/efs/inode.c                                     |    6 +-
+ fs/eventpoll.c                                     |    2 +-
+ fs/exfat/balloc.c                                  |   18 +-
+ fs/exfat/dir.c                                     |   54 +-
+ fs/exfat/exfat_fs.h                                |   13 +-
+ fs/exfat/exfat_raw.h                               |    1 +
+ fs/exfat/fatent.c                                  |   95 +-
+ fs/exfat/file.c                                    |   42 +
+ fs/exfat/inode.c                                   |    8 +-
+ fs/exfat/misc.c                                    |    8 +-
+ fs/exfat/namei.c                                   |    3 +-
+ fs/exfat/super.c                                   |   11 +-
+ fs/exportfs/expfs.c                                |    4 +-
+ fs/ext2/balloc.c                                   |    4 +-
+ fs/ext2/dir.c                                      |   10 +-
+ fs/ext2/ialloc.c                                   |    9 +-
+ fs/ext2/inode.c                                    |    5 +-
+ fs/ext2/namei.c                                    |    7 +-
+ fs/ext2/trace.h                                    |    8 +-
+ fs/ext2/xattr.c                                    |   16 +-
+ fs/ext4/dir.c                                      |    2 +-
+ fs/ext4/ext4.h                                     |    4 +-
+ fs/ext4/extents-test.c                             |    4 +-
+ fs/ext4/extents.c                                  |    8 +-
+ fs/ext4/extents_status.c                           |   28 +-
+ fs/ext4/fast_commit.c                              |    8 +-
+ fs/ext4/ialloc.c                                   |   10 +-
+ fs/ext4/indirect.c                                 |    2 +-
+ fs/ext4/inline.c                                   |   14 +-
+ fs/ext4/inode.c                                    |   27 +-
+ fs/ext4/ioctl.c                                    |    4 +-
+ fs/ext4/mballoc.c                                  |    6 +-
+ fs/ext4/migrate.c                                  |    2 +-
+ fs/ext4/move_extent.c                              |   20 +-
+ fs/ext4/namei.c                                    |   10 +-
+ fs/ext4/orphan.c                                   |   16 +-
+ fs/ext4/page-io.c                                  |   38 +-
+ fs/ext4/readpage.c                                 |   10 +-
+ fs/ext4/super.c                                    |   22 +-
+ fs/ext4/xattr.c                                    |   10 +-
+ fs/f2fs/compress.c                                 |    4 +-
+ fs/f2fs/data.c                                     |   25 +-
+ fs/f2fs/debug.c                                    |    1 +
+ fs/f2fs/dir.c                                      |    2 +-
+ fs/f2fs/extent_cache.c                             |    8 +-
+ fs/f2fs/f2fs.h                                     |   13 +-
+ fs/f2fs/file.c                                     |   20 +-
+ fs/f2fs/gc.c                                       |   12 +-
+ fs/f2fs/inline.c                                   |   17 +-
+ fs/f2fs/inode.c                                    |   48 +-
+ fs/f2fs/namei.c                                    |    9 +-
+ fs/f2fs/node.c                                     |   20 +-
+ fs/f2fs/recovery.c                                 |   10 +-
+ fs/f2fs/segment.c                                  |    2 +-
+ fs/f2fs/segment.h                                  |    2 +-
+ fs/f2fs/sysfs.c                                    |   10 +
+ fs/f2fs/xattr.c                                    |   10 +-
+ fs/freevxfs/vxfs_bmap.c                            |    4 +-
+ fs/fs_context.c                                    |    1 -
+ fs/fs_parser.c                                     |   19 +-
+ fs/fs_struct.c                                     |   34 +-
+ fs/fserror.c                                       |    2 +-
+ fs/fuse/control.c                                  |    4 +-
+ fs/fuse/dev.c                                      |   61 +-
+ fs/fuse/dev_uring.c                                |   36 +-
+ fs/fuse/dir.c                                      |   14 +-
+ fs/fuse/file.c                                     |   23 +-
+ fs/fuse/fuse_i.h                                   |    1 +
+ fs/fuse/inode.c                                    |    1 +
+ fs/fuse/readdir.c                                  |    8 +-
+ fs/fuse/virtio_fs.c                                |   29 +-
+ fs/gfs2/aops.c                                     |    5 +-
+ fs/gfs2/inode.c                                    |    3 +-
+ fs/gfs2/super.c                                    |   56 +-
+ fs/hfs/catalog.c                                   |    2 +-
+ fs/hfs/extent.c                                    |    4 +-
+ fs/hfs/inode.c                                     |    4 +-
+ fs/hfsplus/attributes.c                            |   10 +-
+ fs/hfsplus/catalog.c                               |    2 +-
+ fs/hfsplus/dir.c                                   |    6 +-
+ fs/hfsplus/extents.c                               |    6 +-
+ fs/hfsplus/inode.c                                 |    8 +-
+ fs/hfsplus/super.c                                 |    6 +-
+ fs/hfsplus/xattr.c                                 |   10 +-
+ fs/hpfs/dir.c                                      |    4 +-
+ fs/hpfs/dnode.c                                    |    4 +-
+ fs/hpfs/ea.c                                       |    4 +-
+ fs/hpfs/inode.c                                    |    4 +-
+ fs/inode.c                                         |   49 +-
+ fs/iomap/bio.c                                     |  135 +-
+ fs/iomap/buffered-io.c                             |    8 +-
+ fs/iomap/direct-io.c                               |   21 +-
+ fs/iomap/internal.h                                |   14 +
+ fs/iomap/ioend.c                                   |   32 +-
+ fs/iomap/trace.h                                   |    8 +-
+ fs/isofs/compress.c                                |    2 +-
+ fs/isofs/dir.c                                     |    2 +-
+ fs/isofs/inode.c                                   |    8 +-
+ fs/isofs/namei.c                                   |    2 +-
+ fs/jbd2/journal.c                                  |    4 +-
+ fs/jbd2/transaction.c                              |    2 +-
+ fs/jffs2/dir.c                                     |    4 +-
+ fs/jffs2/file.c                                    |    4 +-
+ fs/jffs2/fs.c                                      |   18 +-
+ fs/jfs/inode.c                                     |    2 +-
+ fs/jfs/jfs_dmap.c                                  |  114 +-
+ fs/jfs/jfs_dtree.c                                 |  192 +-
+ fs/jfs/jfs_dtree.h                                 |    4 +
+ fs/jfs/jfs_imap.c                                  |    6 +-
+ fs/jfs/jfs_logmgr.c                                |   34 +-
+ fs/jfs/jfs_logmgr.h                                |    7 +
+ fs/jfs/jfs_metapage.c                              |    5 +-
+ fs/jfs/jfs_mount.c                                 |    3 +-
+ fs/jfs/jfs_txnmgr.c                                |    3 +-
+ fs/jfs/jfs_umount.c                                |   10 +
+ fs/kernfs/dir.c                                    |   15 +-
+ fs/kernfs/inode.c                                  |   99 +-
+ fs/kernfs/kernfs-internal.h                        |    5 +-
+ fs/libfs.c                                         |    3 +-
+ fs/lockd/Makefile                                  |   30 +-
+ fs/lockd/clnt4xdr.c                                |    5 +-
+ fs/lockd/clntlock.c                                |    2 +-
+ fs/lockd/clntproc.c                                |    2 +-
+ fs/lockd/clntxdr.c                                 |    3 +-
+ fs/lockd/host.c                                    |   31 +-
+ {include/linux => fs}/lockd/lockd.h                |  101 +-
+ fs/lockd/mon.c                                     |    2 +-
+ {include/linux => fs}/lockd/nlm.h                  |    8 +-
+ fs/lockd/nlm4xdr_gen.c                             |  724 +++
+ fs/lockd/nlm4xdr_gen.h                             |   32 +
+ {include/linux => fs}/lockd/share.h                |   19 +-
+ fs/lockd/svc.c                                     |   50 +-
+ fs/lockd/svc4proc.c                                | 1786 +++++--
+ fs/lockd/svclock.c                                 |   29 +-
+ fs/lockd/svcproc.c                                 |  126 +-
+ fs/lockd/svcshare.c                                |   40 +-
+ fs/lockd/svcsubs.c                                 |   34 +-
+ fs/lockd/trace.h                                   |    3 +-
+ fs/lockd/xdr.c                                     |    6 +-
+ {include/linux => fs}/lockd/xdr.h                  |   15 +-
+ fs/lockd/xdr4.c                                    |  347 --
+ fs/locks.c                                         |   32 +-
+ fs/minix/inode.c                                   |   10 +-
+ fs/namei.c                                         |   70 +-
+ fs/namespace.c                                     |  125 +-
+ fs/nfs/blocklayout/blocklayout.c                   |    4 +-
+ fs/nfs/dir.c                                       |   20 +-
+ fs/nfs/file.c                                      |    8 +-
+ fs/nfs/filelayout/filelayout.c                     |    8 +-
+ fs/nfs/flexfilelayout/flexfilelayout.c             |    8 +-
+ fs/nfs/inode.c                                     |    6 +-
+ fs/nfs/internal.h                                  |    2 +-
+ fs/nfs/nfs3proc.c                                  |    1 +
+ fs/nfs/nfs4proc.c                                  |    4 +-
+ fs/nfs/pnfs.c                                      |   12 +-
+ fs/nfs/proc.c                                      |    1 +
+ fs/nfs/sysfs.c                                     |    4 +-
+ fs/nfs/write.c                                     |    3 +-
+ fs/nfsd/Kconfig                                    |   10 -
+ fs/nfsd/blocklayout.c                              |  102 +-
+ fs/nfsd/debugfs.c                                  |    4 +
+ fs/nfsd/export.c                                   |    2 +-
+ fs/nfsd/lockd.c                                    |   50 +-
+ fs/nfsd/nfs4layouts.c                              |  152 +-
+ fs/nfsd/nfs4proc.c                                 |    3 +-
+ fs/nfsd/nfs4recover.c                              |    6 +-
+ fs/nfsd/nfs4state.c                                |   39 +-
+ fs/nfsd/nfs4xdr.c                                  |    3 +-
+ fs/nfsd/nfscache.c                                 |    3 +-
+ fs/nfsd/nfsctl.c                                   |    2 +-
+ fs/nfsd/nfsd.h                                     |    6 +
+ fs/nfsd/nfsfh.c                                    |   13 +-
+ fs/nfsd/nfssvc.c                                   |   10 +-
+ fs/nfsd/pnfs.h                                     |    5 +-
+ fs/nfsd/state.h                                    |    9 +
+ fs/nfsd/vfs.c                                      |    2 +-
+ fs/nilfs2/alloc.c                                  |   10 +-
+ fs/nilfs2/bmap.c                                   |    2 +-
+ fs/nilfs2/btnode.c                                 |    2 +-
+ fs/nilfs2/btree.c                                  |   12 +-
+ fs/nilfs2/dir.c                                    |   12 +-
+ fs/nilfs2/direct.c                                 |    4 +-
+ fs/nilfs2/gcinode.c                                |    2 +-
+ fs/nilfs2/inode.c                                  |    8 +-
+ fs/nilfs2/mdt.c                                    |    2 +-
+ fs/nilfs2/namei.c                                  |    2 +-
+ fs/nilfs2/segment.c                                |    2 +-
+ fs/notify/fanotify/fanotify_user.c                 |   50 +-
+ fs/notify/fdinfo.c                                 |    4 +-
+ fs/notify/inotify/inotify_user.c                   |    3 +-
+ fs/nsfs.c                                          |    4 +-
+ fs/ntfs/Kconfig                                    |   48 +
+ fs/ntfs/Makefile                                   |   10 +
+ fs/ntfs/aops.c                                     |  263 +
+ fs/ntfs/attrib.c                                   | 5424 ++++++++++++++++++++
+ fs/ntfs/attrib.h                                   |  164 +
+ fs/ntfs/attrlist.c                                 |  289 ++
+ fs/ntfs/attrlist.h                                 |   20 +
+ fs/ntfs/bdev-io.c                                  |  117 +
+ fs/ntfs/bitmap.c                                   |  287 ++
+ fs/ntfs/bitmap.h                                   |  100 +
+ fs/ntfs/collate.c                                  |  146 +
+ fs/ntfs/collate.h                                  |   36 +
+ fs/ntfs/compress.c                                 | 1577 ++++++
+ fs/ntfs/debug.c                                    |  171 +
+ fs/ntfs/debug.h                                    |   63 +
+ fs/ntfs/dir.c                                      | 1238 +++++
+ fs/ntfs/dir.h                                      |   32 +
+ fs/ntfs/ea.c                                       |  954 ++++
+ fs/ntfs/ea.h                                       |   30 +
+ fs/ntfs/file.c                                     | 1161 +++++
+ fs/ntfs/index.c                                    | 2117 ++++++++
+ fs/ntfs/index.h                                    |  111 +
+ fs/ntfs/inode.c                                    | 3815 ++++++++++++++
+ fs/ntfs/inode.h                                    |  358 ++
+ fs/ntfs/iomap.c                                    |  870 ++++
+ fs/ntfs/iomap.h                                    |   23 +
+ fs/ntfs/layout.h                                   | 2346 +++++++++
+ fs/ntfs/lcnalloc.c                                 | 1049 ++++
+ fs/ntfs/lcnalloc.h                                 |  134 +
+ fs/ntfs/logfile.c                                  |  778 +++
+ fs/ntfs/logfile.h                                  |  245 +
+ fs/ntfs/malloc.h                                   |   77 +
+ fs/ntfs/mft.c                                      | 2924 +++++++++++
+ fs/ntfs/mft.h                                      |   91 +
+ fs/ntfs/mst.c                                      |  194 +
+ fs/ntfs/namei.c                                    | 1692 ++++++
+ fs/ntfs/ntfs.h                                     |  294 ++
+ fs/ntfs/object_id.c                                |  158 +
+ fs/ntfs/object_id.h                                |   14 +
+ fs/ntfs/quota.c                                    |   95 +
+ fs/ntfs/quota.h                                    |   15 +
+ fs/ntfs/reparse.c                                  |  573 +++
+ fs/ntfs/reparse.h                                  |   20 +
+ fs/ntfs/runlist.c                                  | 2066 ++++++++
+ fs/ntfs/runlist.h                                  |   97 +
+ fs/ntfs/super.c                                    | 2771 ++++++++++
+ fs/ntfs/sysctl.c                                   |   54 +
+ fs/ntfs/sysctl.h                                   |   26 +
+ fs/ntfs/time.h                                     |   87 +
+ fs/ntfs/unistr.c                                   |  477 ++
+ fs/ntfs/upcase.c                                   |   70 +
+ fs/ntfs/volume.h                                   |  296 ++
+ fs/ntfs3/Kconfig                                   |   10 +-
+ fs/ntfs3/Makefile                                  |   20 +
+ fs/ntfs3/dir.c                                     |   10 -
+ fs/ntfs3/file.c                                    |   11 -
+ fs/ntfs3/frecord.c                                 |   47 +-
+ fs/ntfs3/index.c                                   |   76 +-
+ fs/ntfs3/inode.c                                   |   88 +-
+ fs/ntfs3/ntfs_fs.h                                 |   14 -
+ fs/ntfs3/super.c                                   |   61 +-
+ fs/ocfs2/alloc.c                                   |    2 +-
+ fs/ocfs2/aops.c                                    |    4 +-
+ fs/ocfs2/dir.c                                     |    8 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |   10 +-
+ fs/ocfs2/extent_map.c                              |   12 +-
+ fs/ocfs2/inode.c                                   |    2 +-
+ fs/ocfs2/quota_local.c                             |    2 +-
+ fs/ocfs2/refcounttree.c                            |   10 +-
+ fs/ocfs2/xattr.c                                   |    4 +-
+ fs/orangefs/inode.c                                |    2 +-
+ fs/overlayfs/dir.c                                 |   51 +-
+ fs/overlayfs/export.c                              |   81 +-
+ fs/overlayfs/namei.c                               |    4 +-
+ fs/overlayfs/overlayfs.h                           |   10 +-
+ fs/overlayfs/super.c                               |   16 +-
+ fs/overlayfs/util.c                                |   27 +-
+ fs/pidfs.c                                         |  102 +-
+ fs/pipe.c                                          |    2 +-
+ fs/proc/fd.c                                       |    2 +-
+ fs/proc/self.c                                     |    3 -
+ fs/proc/task_mmu.c                                 |    4 +-
+ fs/proc/thread_self.c                              |    3 -
+ fs/qnx4/inode.c                                    |    4 +-
+ fs/qnx6/inode.c                                    |    2 +-
+ fs/smb/Kconfig                                     |    1 +
+ fs/smb/client/Kconfig                              |    4 +-
+ fs/smb/client/cifs_debug.c                         |   67 +-
+ fs/smb/client/cifsglob.h                           |    6 +
+ fs/smb/client/connect.c                            |    4 +
+ fs/smb/client/file.c                               |    1 -
+ fs/smb/client/inode.c                              |   21 +-
+ fs/smb/client/smb1transport.c                      |    2 +-
+ fs/smb/client/smb2ops.c                            |   20 +-
+ fs/smb/client/smb2pdu.c                            |    9 +-
+ fs/smb/client/smbdirect.c                          | 3116 +----------
+ fs/smb/client/smbdirect.h                          |   19 +-
+ fs/smb/common/Makefile                             |    1 +
+ fs/smb/common/smbdirect/Kconfig                    |    9 +
+ fs/smb/common/smbdirect/Makefile                   |   18 +
+ fs/smb/common/smbdirect/smbdirect.h                |    9 +
+ fs/smb/common/smbdirect/smbdirect_accept.c         |  857 ++++
+ fs/smb/common/smbdirect/smbdirect_connect.c        |  925 ++++
+ fs/smb/common/smbdirect/smbdirect_connection.c     | 2175 ++++++++
+ fs/smb/common/smbdirect/smbdirect_debug.c          |   88 +
+ fs/smb/common/smbdirect/smbdirect_devices.c        |  277 +
+ fs/smb/common/smbdirect/smbdirect_internal.h       |  141 +
+ fs/smb/common/smbdirect/smbdirect_listen.c         |  308 ++
+ fs/smb/common/smbdirect/smbdirect_main.c           |  121 +
+ fs/smb/common/smbdirect/smbdirect_mr.c             |  485 ++
+ fs/smb/common/smbdirect/smbdirect_pdu.h            |    4 +
+ fs/smb/common/smbdirect/smbdirect_public.h         |  148 +
+ fs/smb/common/smbdirect/smbdirect_rw.c             |  255 +
+ fs/smb/common/smbdirect/smbdirect_socket.c         |  724 +++
+ fs/smb/common/smbdirect/smbdirect_socket.h         |  303 +-
+ fs/smb/server/Kconfig                              |    5 +-
+ fs/smb/server/connection.c                         |    5 -
+ fs/smb/server/connection.h                         |    1 -
+ fs/smb/server/mgmt/tree_connect.c                  |    9 +-
+ fs/smb/server/oplock.c                             |   72 +-
+ fs/smb/server/smb2pdu.c                            |   55 +-
+ fs/smb/server/transport_rdma.c                     | 2958 +----------
+ fs/smb/server/transport_rdma.h                     |    4 +-
+ fs/ubifs/debug.c                                   |    8 +-
+ fs/ubifs/dir.c                                     |   28 +-
+ fs/ubifs/file.c                                    |   28 +-
+ fs/ubifs/journal.c                                 |    6 +-
+ fs/ubifs/super.c                                   |   16 +-
+ fs/ubifs/tnc.c                                     |    4 +-
+ fs/ubifs/xattr.c                                   |   14 +-
+ fs/udf/directory.c                                 |   18 +-
+ fs/udf/file.c                                      |    2 +-
+ fs/udf/inode.c                                     |   23 +-
+ fs/udf/namei.c                                     |    8 +-
+ fs/udf/super.c                                     |    6 +-
+ fs/ufs/balloc.c                                    |    6 +-
+ fs/ufs/dir.c                                       |   10 +-
+ fs/ufs/ialloc.c                                    |    6 +-
+ fs/ufs/inode.c                                     |   18 +-
+ fs/ufs/ufs_fs.h                                    |    6 +-
+ fs/ufs/util.c                                      |    2 +-
+ fs/verity/init.c                                   |    2 +-
+ fs/xattr.c                                         |  425 +-
+ fs/xfs/libxfs/xfs_attr.h                           |    3 +-
+ fs/xfs/libxfs/xfs_fs.h                             |    5 +-
+ fs/xfs/scrub/quota.c                               |    4 +-
+ fs/xfs/xfs_aops.c                                  |   47 +-
+ fs/xfs/xfs_dquot_item.c                            |    9 +-
+ fs/xfs/xfs_file.c                                  |   95 +-
+ fs/xfs/xfs_handle.c                                |    2 +-
+ fs/xfs/xfs_inode_item.c                            |    9 +-
+ fs/xfs/xfs_ioctl.c                                 |   19 +
+ fs/xfs/xfs_iomap.c                                 |   11 +-
+ fs/xfs/xfs_mount.c                                 |    7 +-
+ fs/xfs/xfs_trace.h                                 |   36 +-
+ fs/xfs/xfs_trans_ail.c                             |  127 +-
+ fs/xfs/xfs_verify_media.c                          |   18 +-
+ fs/xfs/xfs_xattr.c                                 |    2 +-
+ fs/xfs/xfs_zone_alloc.c                            |   44 +-
+ fs/xfs/xfs_zone_gc.c                               |   24 +-
+ fs/xfs/xfs_zone_info.c                             |    7 +-
+ fs/zonefs/super.c                                  |    8 +-
+ fs/zonefs/trace.h                                  |   18 +-
+ include/linux/audit.h                              |    2 +-
+ include/linux/backing-dev.h                        |   13 +
+ include/linux/bio-integrity.h                      |   12 +-
+ include/linux/bio.h                                |    2 +-
+ include/linux/blk-integrity.h                      |   28 +-
+ include/linux/blkdev.h                             |   34 +-
+ include/linux/filelock.h                           |    1 +
+ include/linux/fs.h                                 |   88 +-
+ include/linux/fs_parser.h                          |    8 +-
+ include/linux/fs_struct.h                          |   43 +-
+ include/linux/fscrypt.h                            |   37 +-
+ include/linux/iomap.h                              |   20 +-
+ include/linux/kernfs.h                             |    2 -
+ include/linux/lockd/bind.h                         |   26 +-
+ include/linux/lockd/debug.h                        |   40 -
+ include/linux/lockd/xdr4.h                         |   43 -
+ include/linux/namei.h                              |    6 -
+ include/linux/sched/signal.h                       |    1 +
+ include/linux/shmem_fs.h                           |    2 +-
+ include/linux/sunrpc/debug.h                       |   10 +-
+ include/linux/sunrpc/sched.h                       |    3 -
+ include/linux/sunrpc/svc.h                         |    6 +-
+ include/linux/sunrpc/xdrgen/nlm4.h                 |  233 +
+ include/linux/xattr.h                              |   47 +-
+ include/net/sock.h                                 |    4 +-
+ include/trace/events/btrfs.h                       |   24 +
+ include/trace/events/cachefiles.h                  |   18 +-
+ include/trace/events/ext4.h                        |  550 +-
+ include/trace/events/f2fs.h                        |  248 +-
+ include/trace/events/filelock.h                    |   34 +-
+ include/trace/events/filemap.h                     |   20 +-
+ include/trace/events/fs_dax.h                      |   20 +-
+ include/trace/events/fsverity.h                    |   30 +-
+ include/trace/events/hugetlbfs.h                   |   42 +-
+ include/trace/events/netfs.h                       |    8 +-
+ include/trace/events/nilfs2.h                      |   12 +-
+ include/trace/events/readahead.h                   |   18 +-
+ include/trace/events/sunrpc.h                      |    4 +-
+ include/trace/events/timestamp.h                   |   16 +-
+ include/trace/events/writeback.h                   |  162 +-
+ include/uapi/linux/btrfs_tree.h                    |    3 +-
+ include/uapi/linux/fs.h                            |   12 +
+ include/uapi/linux/mount.h                         |    1 +
+ include/uapi/linux/pidfd.h                         |    1 +
+ include/uapi/linux/sched.h                         |    3 +
+ kernel/audit.h                                     |   13 +-
+ kernel/audit_fsnotify.c                            |    4 +-
+ kernel/audit_watch.c                               |   12 +-
+ kernel/auditsc.c                                   |   11 +-
+ kernel/events/uprobes.c                            |    4 +-
+ kernel/fork.c                                      |   52 +-
+ kernel/ptrace.c                                    |    3 +-
+ kernel/signal.c                                    |    4 +
+ mm/shmem.c                                         |   46 +-
+ net/ax25/af_ax25.c                                 |    2 +-
+ net/bluetooth/af_bluetooth.c                       |    4 +-
+ net/can/bcm.c                                      |    2 +-
+ net/ceph/auth.c                                    |    2 +-
+ net/ceph/messenger_v2.c                            |   13 +-
+ net/ipv4/ping.c                                    |    2 +-
+ net/ipv4/raw.c                                     |    2 +-
+ net/ipv4/tcp_ipv4.c                                |    2 +-
+ net/ipv4/udp.c                                     |    2 +-
+ net/ipv6/datagram.c                                |    2 +-
+ net/ipv6/tcp_ipv6.c                                |    2 +-
+ net/key/af_key.c                                   |    2 +-
+ net/netlink/af_netlink.c                           |    2 +-
+ net/netlink/diag.c                                 |    2 +-
+ net/netrom/af_netrom.c                             |    4 +-
+ net/packet/af_packet.c                             |    2 +-
+ net/packet/diag.c                                  |    2 +-
+ net/phonet/socket.c                                |    4 +-
+ net/rose/af_rose.c                                 |    4 +-
+ net/sctp/proc.c                                    |    4 +-
+ net/socket.c                                       |  121 +-
+ net/sunrpc/auth_gss/gss_krb5_test.c                |   93 +-
+ net/sunrpc/cache.c                                 |   62 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c           |   27 +-
+ net/unix/af_unix.c                                 |    2 +-
+ net/unix/diag.c                                    |    6 +-
+ net/x25/x25_proc.c                                 |    4 +-
+ net/xdp/xsk_diag.c                                 |    2 +-
+ security/apparmor/apparmorfs.c                     |   38 +-
+ security/integrity/integrity_audit.c               |    2 +-
+ security/ipe/audit.c                               |    2 +-
+ security/lsm_audit.c                               |   10 +-
+ security/selinux/hooks.c                           |   10 +-
+ security/selinux/selinuxfs.c                       |   17 +-
+ security/smack/smack_lsm.c                         |   12 +-
+ tools/include/uapi/linux/mount.h                   |   14 +-
+ tools/testing/selftests/Makefile                   |    1 +
+ tools/testing/selftests/filesystems/devpts_pts.c   |    2 -
+ .../selftests/filesystems/fsmount_ns/.gitignore    |    1 +
+ .../selftests/filesystems/fsmount_ns/Makefile      |   10 +
+ .../filesystems/fsmount_ns/fsmount_ns_test.c       | 1138 ++++
+ .../selftests/filesystems/move_mount/.gitignore    |    2 +
+ .../selftests/filesystems/move_mount/Makefile      |   10 +
+ .../filesystems/move_mount/move_mount_test.c       |  492 ++
+ .../selftests/filesystems/open_tree_ns/Makefile    |    2 +-
+ .../filesystems/open_tree_ns/open_tree_ns_test.c   |   43 +-
+ .../selftests/filesystems/statmount/statmount.h    |   51 +
+ .../filesystems/statmount/statmount_test.c         |   45 +-
+ .../testing/selftests/filesystems/xattr/.gitignore |    3 +
+ tools/testing/selftests/filesystems/xattr/Makefile |    6 +
+ .../filesystems/xattr/xattr_socket_test.c          |  470 ++
+ .../filesystems/xattr/xattr_socket_types_test.c    |  177 +
+ .../filesystems/xattr/xattr_sockfs_test.c          |  363 ++
+ tools/testing/selftests/pidfd/.gitignore           |    1 +
+ tools/testing/selftests/pidfd/Makefile             |    2 +-
+ .../testing/selftests/pidfd/pidfd_autoreap_test.c  |  900 ++++
+ 583 files changed, 56628 insertions(+), 11075 deletions(-)
+ create mode 100644 Documentation/filesystems/ntfs.rst
+ create mode 100644 Documentation/sunrpc/xdr/nlm4.x
+ create mode 100644 block/bio-integrity-fs.c
+ create mode 100644 fs/btrfs/tests/zoned-tests.c
+ rename {include/linux => fs}/lockd/lockd.h (83%)
+ rename {include/linux => fs}/lockd/nlm.h (91%)
+ create mode 100644 fs/lockd/nlm4xdr_gen.c
+ create mode 100644 fs/lockd/nlm4xdr_gen.h
+ rename {include/linux => fs}/lockd/share.h (58%)
+ rename {include/linux => fs}/lockd/xdr.h (91%)
+ delete mode 100644 fs/lockd/xdr4.c
+ create mode 100644 fs/ntfs/Kconfig
+ create mode 100644 fs/ntfs/Makefile
+ create mode 100644 fs/ntfs/aops.c
+ create mode 100644 fs/ntfs/attrib.c
+ create mode 100644 fs/ntfs/attrib.h
+ create mode 100644 fs/ntfs/attrlist.c
+ create mode 100644 fs/ntfs/attrlist.h
+ create mode 100644 fs/ntfs/bdev-io.c
+ create mode 100644 fs/ntfs/bitmap.c
+ create mode 100644 fs/ntfs/bitmap.h
+ create mode 100644 fs/ntfs/collate.c
+ create mode 100644 fs/ntfs/collate.h
+ create mode 100644 fs/ntfs/compress.c
+ create mode 100644 fs/ntfs/debug.c
+ create mode 100644 fs/ntfs/debug.h
+ create mode 100644 fs/ntfs/dir.c
+ create mode 100644 fs/ntfs/dir.h
+ create mode 100644 fs/ntfs/ea.c
+ create mode 100644 fs/ntfs/ea.h
+ create mode 100644 fs/ntfs/file.c
+ create mode 100644 fs/ntfs/index.c
+ create mode 100644 fs/ntfs/index.h
+ create mode 100644 fs/ntfs/inode.c
+ create mode 100644 fs/ntfs/inode.h
+ create mode 100644 fs/ntfs/iomap.c
+ create mode 100644 fs/ntfs/iomap.h
+ create mode 100644 fs/ntfs/layout.h
+ create mode 100644 fs/ntfs/lcnalloc.c
+ create mode 100644 fs/ntfs/lcnalloc.h
+ create mode 100644 fs/ntfs/logfile.c
+ create mode 100644 fs/ntfs/logfile.h
+ create mode 100644 fs/ntfs/malloc.h
+ create mode 100644 fs/ntfs/mft.c
+ create mode 100644 fs/ntfs/mft.h
+ create mode 100644 fs/ntfs/mst.c
+ create mode 100644 fs/ntfs/namei.c
+ create mode 100644 fs/ntfs/ntfs.h
+ create mode 100644 fs/ntfs/object_id.c
+ create mode 100644 fs/ntfs/object_id.h
+ create mode 100644 fs/ntfs/quota.c
+ create mode 100644 fs/ntfs/quota.h
+ create mode 100644 fs/ntfs/reparse.c
+ create mode 100644 fs/ntfs/reparse.h
+ create mode 100644 fs/ntfs/runlist.c
+ create mode 100644 fs/ntfs/runlist.h
+ create mode 100644 fs/ntfs/super.c
+ create mode 100644 fs/ntfs/sysctl.c
+ create mode 100644 fs/ntfs/sysctl.h
+ create mode 100644 fs/ntfs/time.h
+ create mode 100644 fs/ntfs/unistr.c
+ create mode 100644 fs/ntfs/upcase.c
+ create mode 100644 fs/ntfs/volume.h
+ create mode 100644 fs/smb/common/smbdirect/Kconfig
+ create mode 100644 fs/smb/common/smbdirect/Makefile
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_accept.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_connect.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_connection.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_debug.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_devices.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_internal.h
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_listen.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_main.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_mr.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_public.h
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_rw.c
+ create mode 100644 fs/smb/common/smbdirect/smbdirect_socket.c
+ delete mode 100644 include/linux/lockd/debug.h
+ delete mode 100644 include/linux/lockd/xdr4.h
+ create mode 100644 include/linux/sunrpc/xdrgen/nlm4.h
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/fsmount_ns/fsmount_ns_test.c
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/move_mount/move_mount_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/xattr/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_socket_types_test.c
+ create mode 100644 tools/testing/selftests/filesystems/xattr/xattr_sockfs_test.c
+ create mode 100644 tools/testing/selftests/pidfd/pidfd_autoreap_test.c
+$ git am -3 ../patches/0001-ntfs-Fix-up-mm-merge.patch
+Applying: ntfs: Fix up mm merge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/ntfs/file.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master b640d10f8d17a] Merge branch 'fs-next' of linux-next
+ Date: Fri Mar 20 13:53:11 2026 +0000
+Merging printk/for-next (9095f233c0258 printk: Fix _DESCS_COUNT type for 64-bit systems)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Auto-merging kernel/printk/printk_ringbuffer.h
+Merge made by the 'ort' strategy.
+ kernel/printk/printk_ringbuffer.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging pci/next (13292811c9d46 Merge branch 'pci/misc')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/net/ethernet/intel/ice/ice_main.c
+Auto-merging drivers/net/ethernet/intel/igc/igc_main.c
+Auto-merging drivers/pci/controller/dwc/pcie-designware-ep.c
+Auto-merging drivers/pci/endpoint/functions/pci-epf-test.c
+Merge made by the 'ort' strategy.
+ Documentation/PCI/msi-howto.rst                    |   7 +-
+ .../bindings/pci/andestech,qilai-pcie.yaml         |  89 +++
+ .../devicetree/bindings/pci/baikal,bt1-pcie.yaml   | 168 ------
+ .../devicetree/bindings/pci/eswin,pcie.yaml        | 166 ++++++
+ .../bindings/pci/renesas,r9a08g045-pcie.yaml       | 121 +++-
+ MAINTAINERS                                        |  14 +
+ drivers/misc/pci_endpoint_test.c                   |  24 +-
+ drivers/net/ethernet/intel/ice/ice_main.c          |   2 +-
+ drivers/net/ethernet/intel/idpf/idpf_main.c        |   2 +-
+ drivers/net/ethernet/intel/igc/igc_main.c          |   2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   2 +-
+ drivers/pci/controller/Kconfig                     |   1 +
+ drivers/pci/controller/dwc/Kconfig                 |  28 +-
+ drivers/pci/controller/dwc/Makefile                |   3 +-
+ drivers/pci/controller/dwc/pci-dra7xx.c            |   4 -
+ drivers/pci/controller/dwc/pci-imx6.c              |  26 +-
+ drivers/pci/controller/dwc/pci-keystone.c          |  12 +
+ drivers/pci/controller/dwc/pci-layerscape-ep.c     |   6 -
+ drivers/pci/controller/dwc/pci-layerscape.c        |  16 +-
+ drivers/pci/controller/dwc/pcie-amd-mdb.c          |   2 +-
+ drivers/pci/controller/dwc/pcie-andes-qilai.c      | 197 +++++++
+ drivers/pci/controller/dwc/pcie-artpec6.c          |   4 -
+ drivers/pci/controller/dwc/pcie-bt1.c              | 645 ---------------------
+ drivers/pci/controller/dwc/pcie-designware-ep.c    |  55 +-
+ drivers/pci/controller/dwc/pcie-designware-host.c  |  25 +-
+ drivers/pci/controller/dwc/pcie-designware-plat.c  |  10 -
+ drivers/pci/controller/dwc/pcie-dw-rockchip.c      |  24 +-
+ drivers/pci/controller/dwc/pcie-eswin.c            | 408 +++++++++++++
+ drivers/pci/controller/dwc/pcie-keembay.c          |   3 -
+ drivers/pci/controller/dwc/pcie-qcom-ep.c          |  12 -
+ drivers/pci/controller/dwc/pcie-rcar-gen4.c        |  18 +-
+ drivers/pci/controller/dwc/pcie-stm32-ep.c         |  10 -
+ drivers/pci/controller/dwc/pcie-tegra194.c         |  19 +-
+ drivers/pci/controller/dwc/pcie-uniphier-ep.c      |  19 +-
+ drivers/pci/controller/pcie-mediatek-gen3.c        | 185 +++---
+ drivers/pci/controller/pcie-rcar-ep.c              |   3 -
+ drivers/pci/controller/pcie-rzg3s-host.c           | 371 +++++++++---
+ drivers/pci/endpoint/functions/pci-epf-mhi.c       |   4 +
+ drivers/pci/endpoint/functions/pci-epf-ntb.c       |  56 +-
+ drivers/pci/endpoint/functions/pci-epf-test.c      |  36 +-
+ drivers/pci/endpoint/functions/pci-epf-vntb.c      |  60 +-
+ drivers/pci/endpoint/pci-ep-cfs.c                  |  14 +-
+ drivers/pci/endpoint/pci-ep-msi.c                  |   5 +
+ drivers/pci/endpoint/pci-epc-core.c                |   5 +-
+ drivers/pci/endpoint/pci-epf-core.c                |   2 +-
+ drivers/pci/hotplug/pciehp_core.c                  |   3 +-
+ drivers/pci/hotplug/pnv_php.c                      |  19 +-
+ drivers/pci/hotplug/rpaphp_slot.c                  |   4 +-
+ drivers/pci/msi/api.c                              |   5 +
+ drivers/pci/msi/msi.c                              |  10 +
+ drivers/pci/of.c                                   |   2 +-
+ drivers/pci/pci-sysfs.c                            |   3 +-
+ drivers/pci/pci.c                                  | 267 ++++-----
+ drivers/pci/pci.h                                  |   3 +-
+ drivers/pci/pcie/aer.c                             |  35 +-
+ drivers/pci/pcie/aspm.c                            |  17 +-
+ drivers/pci/pcie/dpc.c                             |   3 +
+ drivers/pci/pcie/ptm.c                             |  77 +--
+ drivers/pci/pwrctrl/Kconfig                        |  13 +-
+ drivers/pci/pwrctrl/Makefile                       |   4 +-
+ drivers/pci/pwrctrl/{slot.c => generic.c}          |  13 +-
+ drivers/pci/quirks.c                               |   3 +
+ drivers/pci/slot.c                                 |  31 +-
+ drivers/pci/tph.c                                  |   9 +-
+ include/linux/pci-epc.h                            |  46 +-
+ include/linux/pci.h                                |  16 +-
+ .../selftests/pci_endpoint/pci_endpoint_test.c     |   6 +
+ 67 files changed, 1983 insertions(+), 1491 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pci/andestech,qilai-pcie.yaml
+ delete mode 100644 Documentation/devicetree/bindings/pci/baikal,bt1-pcie.yaml
+ create mode 100644 Documentation/devicetree/bindings/pci/eswin,pcie.yaml
+ create mode 100644 drivers/pci/controller/dwc/pcie-andes-qilai.c
+ delete mode 100644 drivers/pci/controller/dwc/pcie-bt1.c
+ create mode 100644 drivers/pci/controller/dwc/pcie-eswin.c
+ rename drivers/pci/pwrctrl/{slot.c => generic.c} (93%)
+Merging pstore/for-next/pstore (bb5278a89dad2 pstore/ram: fix resource leak when ioremap() fails)
+$ git merge -m Merge branch 'for-next/pstore' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Auto-merging fs/pstore/inode.c
+Auto-merging fs/pstore/ram.c
+Auto-merging fs/pstore/ram_core.c
+Merge made by the 'ort' strategy.
+ fs/pstore/inode.c    | 6 +++---
+ fs/pstore/ram.c      | 2 +-
+ fs/pstore/ram_core.c | 5 ++++-
+ 3 files changed, 8 insertions(+), 5 deletions(-)
+Merging hid/for-next (2bbe306cc6f97 Merge branch 'for-7.1/core-v2' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Auto-merging MAINTAINERS
+Auto-merging include/linux/device.h
+Merge made by the 'ort' strategy.
+ .../ABI/testing/sysfs-driver-hid-lenovo-go         |  724 ++++++
+ .../ABI/testing/sysfs-driver-hid-lenovo-go-s       |  304 +++
+ MAINTAINERS                                        |   28 +-
+ drivers/hid/Kconfig                                |   24 +
+ drivers/hid/Makefile                               |    2 +
+ drivers/hid/hid-apple.c                            |   10 +-
+ drivers/hid/hid-asus.c                             |  108 +-
+ drivers/hid/hid-core.c                             |    9 +
+ drivers/hid/hid-ids.h                              |    7 +
+ drivers/hid/hid-input-test.c                       |   39 +-
+ drivers/hid/hid-input.c                            |  210 +-
+ drivers/hid/hid-lenovo-go-s.c                      | 1504 ++++++++++++
+ drivers/hid/hid-lenovo-go.c                        | 2500 ++++++++++++++++++++
+ drivers/hid/hid-magicmouse.c                       |   10 +-
+ drivers/hid/hid-mcp2221.c                          |   36 +-
+ drivers/hid/hid-pl.c                               |   12 +-
+ .../intel-thc-hid/intel-quicki2c/pci-quicki2c.c    |    7 +
+ .../intel-thc-hid/intel-quicki2c/quicki2c-dev.h    |    4 +
+ .../intel-thc-hid/intel-quickspi/pci-quickspi.c    |    6 +
+ .../intel-thc-hid/intel-quickspi/quickspi-dev.h    |    2 +
+ drivers/hid/usbhid/hid-pidff.c                     |   57 +-
+ drivers/hid/usbhid/hid-pidff.h                     |    9 +
+ include/linux/device.h                             |   46 +
+ include/linux/hid.h                                |   58 +-
+ 24 files changed, 5476 insertions(+), 240 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-driver-hid-lenovo-go
+ create mode 100644 Documentation/ABI/testing/sysfs-driver-hid-lenovo-go-s
+ create mode 100644 drivers/hid/hid-lenovo-go-s.c
+ create mode 100644 drivers/hid/hid-lenovo-go.c
+Merging i2c/i2c/for-next (e9b13580b3340 Merge branch 'i2c/for-current' into i2c/for-next)
+$ git merge -m Merge branch 'i2c/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-diolan-u2c.c     | 12 +++---------
+ drivers/i2c/busses/i2c-robotfuzz-osif.c |  4 +---
+ drivers/i2c/busses/i2c-tiny-usb.c       | 13 +++----------
+ 3 files changed, 7 insertions(+), 22 deletions(-)
+Merging i2c-host/i2c/i2c-host (08e392059b755 i2c: npcm7xx: Use NULL instead of 0 for pointer)
+$ git merge -m Merge branch 'i2c/i2c-host' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host/i2c/i2c-host
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-npcm7xx.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging i3c/i3c/next (8ecd876fdaeda i3c: master: use kzalloc_flex)
+$ git merge -m Merge branch 'i3c/next' of https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-bus-i3c            | 20 +++++++++++
+ drivers/i3c/master.c                               | 41 ++++++++++++++++------
+ drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c |  3 ++
+ drivers/i3c/master/svc-i3c-master.c                |  6 ++--
+ 4 files changed, 57 insertions(+), 13 deletions(-)
+Merging dmi/dmi-for-next (1afafbaf749d8 firmware/dmi: Include product_family info to modalias)
+$ git merge -m Merge branch 'dmi-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (be8aad7a8a141 hwmon: (pmbus/isl68137) Remove unused enum chips)
+$ git merge -m Merge branch 'hwmon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/hwmon/Kconfig
+Auto-merging drivers/hwmon/pmbus/isl68137.c
+Auto-merging drivers/hwmon/pmbus/q54sj108a2.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/hwmon/baikal,bt1-pvt.yaml  |  105 --
+ .../devicetree/bindings/hwmon/moortec,mr75203.yaml |    4 +-
+ .../devicetree/bindings/hwmon/npcm750-pwm-fan.txt  |   88 --
+ .../bindings/hwmon/nuvoton,npcm750-pwm-fan.yaml    |  139 +++
+ .../devicetree/bindings/hwmon/ti,ina2xx.yaml       |    3 +
+ .../devicetree/bindings/trivial-devices.yaml       |   11 +
+ Documentation/hwmon/asus_ec_sensors.rst            |    3 +
+ Documentation/hwmon/bt1-pvt.rst                    |  117 --
+ Documentation/hwmon/ina2xx.rst                     |   25 +-
+ Documentation/hwmon/index.rst                      |    2 +-
+ Documentation/hwmon/lattepanda-sigma-ec.rst        |   61 +
+ Documentation/hwmon/ltc4282.rst                    |    3 +-
+ Documentation/hwmon/tmp102.rst                     |   21 +
+ MAINTAINERS                                        |    7 +
+ drivers/hwmon/Kconfig                              |   50 +-
+ drivers/hwmon/Makefile                             |    2 +-
+ drivers/hwmon/acpi_power_meter.c                   |   96 +-
+ drivers/hwmon/ads7828.c                            |    4 +-
+ drivers/hwmon/ads7871.c                            |   18 +-
+ drivers/hwmon/aht10.c                              |   10 +
+ drivers/hwmon/aspeed-g6-pwm-tach.c                 |    8 -
+ drivers/hwmon/asus-ec-sensors.c                    |   29 +
+ drivers/hwmon/asus_atk0110.c                       |   92 +-
+ drivers/hwmon/bt1-pvt.c                            | 1171 --------------------
+ drivers/hwmon/bt1-pvt.h                            |  247 -----
+ drivers/hwmon/emc1403.c                            |    2 +-
+ drivers/hwmon/gpd-fan.c                            |    8 +
+ drivers/hwmon/ina2xx.c                             |   76 +-
+ drivers/hwmon/lattepanda-sigma-ec.c                |  359 ++++++
+ drivers/hwmon/ltc4282.c                            |   15 +-
+ drivers/hwmon/max31722.c                           |    3 +-
+ drivers/hwmon/max6650.c                            |    3 +-
+ drivers/hwmon/pmbus/Kconfig                        |    9 +
+ drivers/hwmon/pmbus/Makefile                       |    1 +
+ drivers/hwmon/pmbus/bel-pfe.c                      |    5 +-
+ drivers/hwmon/pmbus/fsp-3y.c                       |   13 +-
+ drivers/hwmon/pmbus/ibm-cffps.c                    |   16 +-
+ drivers/hwmon/pmbus/isl68137.c                     |   50 +-
+ drivers/hwmon/pmbus/ltc2978.c                      |    2 +-
+ drivers/hwmon/pmbus/max16601.c                     |    2 +-
+ drivers/hwmon/pmbus/max20730.c                     |    5 +-
+ drivers/hwmon/pmbus/max31785.c                     |    2 +-
+ drivers/hwmon/pmbus/max34440.c                     |    4 +-
+ drivers/hwmon/pmbus/pmbus.c                        |    4 +-
+ drivers/hwmon/pmbus/pmbus.h                        |    2 +-
+ drivers/hwmon/pmbus/pmbus_core.c                   |    4 +
+ drivers/hwmon/pmbus/q54sj108a2.c                   |   23 +-
+ drivers/hwmon/pmbus/tps53679.c                     |    5 +-
+ drivers/hwmon/pmbus/xdpe1a2g7b.c                   |  119 ++
+ drivers/hwmon/tc74.c                               |    2 +-
+ 50 files changed, 1038 insertions(+), 2012 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/hwmon/baikal,bt1-pvt.yaml
+ delete mode 100644 Documentation/devicetree/bindings/hwmon/npcm750-pwm-fan.txt
+ create mode 100644 Documentation/devicetree/bindings/hwmon/nuvoton,npcm750-pwm-fan.yaml
+ delete mode 100644 Documentation/hwmon/bt1-pvt.rst
+ create mode 100644 Documentation/hwmon/lattepanda-sigma-ec.rst
+ delete mode 100644 drivers/hwmon/bt1-pvt.c
+ delete mode 100644 drivers/hwmon/bt1-pvt.h
+ create mode 100644 drivers/hwmon/lattepanda-sigma-ec.c
+ create mode 100644 drivers/hwmon/pmbus/xdpe1a2g7b.c
+Merging jc_docs/docs-next (98e7b5752898f scripts: ver_linux: expand and fix list)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |   2 +-
+ Documentation/admin-guide/sysctl/crypto.rst        |  47 ++
+ Documentation/admin-guide/sysctl/debug.rst         |  52 ++
+ Documentation/admin-guide/sysctl/index.rst         |   6 +-
+ Documentation/admin-guide/tainted-kernels.rst      |   2 +-
+ Documentation/core-api/housekeeping.rst            |   2 +-
+ Documentation/core-api/memory-hotplug.rst          |   2 +-
+ Documentation/core-api/printk-basics.rst           |  36 ++
+ Documentation/dev-tools/coccinelle.rst             |   3 +-
+ .../early-userspace/early_userspace_support.rst    |   2 +-
+ Documentation/driver-api/interconnect.rst          |  16 +-
+ .../features/perf/perf-regs/arch-support.txt       |   2 +-
+ .../features/perf/perf-stackdump/arch-support.txt  |   2 +-
+ Documentation/filesystems/proc.rst                 |  40 +-
+ Documentation/mm/hwpoison.rst                      |   2 +-
+ Documentation/mm/numa.rst                          |   2 +-
+ Documentation/process/2.Process.rst                |   2 +-
+ Documentation/process/backporting.rst              |   2 +-
+ Documentation/process/changes.rst                  |   2 +-
+ Documentation/process/handling-regressions.rst     | 695 ++++++++++++++-------
+ Documentation/process/maintainer-handbooks.rst     |   2 -
+ Documentation/process/maintainer-tip.rst           |   2 +-
+ Documentation/process/submitting-patches.rst       |   2 +-
+ Documentation/scheduler/sched-deadline.rst         |  22 +-
+ Documentation/sphinx/translations.py               |   1 +
+ Documentation/tools/kdoc_parser.rst                |   8 +
+ Documentation/tools/rtla/rtla.rst                  |   6 +-
+ Documentation/translations/index.rst               |   4 +-
+ .../ja_JP/process/submitting-patches.rst           |  46 +-
+ .../translations/pt_BR/disclaimer-pt_BR.rst        |  11 +
+ Documentation/translations/pt_BR/index.rst         |  72 +++
+ .../translations/pt_BR/process/changes.rst         | 576 +++++++++++++++++
+ Documentation/translations/pt_BR/process/howto.rst | 637 +++++++++++++++++++
+ .../pt_BR/process/maintainer-handbooks.rst         |  17 +
+ .../pt_BR/process/maintainer-netdev.rst            | 596 ++++++++++++++++++
+ .../sp_SP/process/submitting-patches.rst           |   2 +-
+ .../translations/zh_TW/admin-guide/README.rst      |   2 +-
+ MAINTAINERS                                        |   7 +-
+ scripts/ver_linux                                  |  50 +-
+ tools/docs/checktransupdate.py                     |  14 +-
+ tools/docs/kernel-doc                              |   1 -
+ tools/docs/sphinx-build-wrapper                    |  10 +-
+ tools/lib/python/jobserver.py                      |   4 +-
+ tools/lib/python/kdoc/kdoc_files.py                |  54 +-
+ tools/lib/python/kdoc/kdoc_output.py               | 297 ++++++++-
+ tools/lib/python/kdoc/kdoc_parser.py               | 172 +----
+ tools/lib/python/kdoc/kdoc_re.py                   | 115 +++-
+ tools/lib/python/kdoc/xforms_lists.py              | 170 +++++
+ 48 files changed, 3315 insertions(+), 504 deletions(-)
+ create mode 100644 Documentation/admin-guide/sysctl/crypto.rst
+ create mode 100644 Documentation/admin-guide/sysctl/debug.rst
+ create mode 100644 Documentation/translations/pt_BR/disclaimer-pt_BR.rst
+ create mode 100644 Documentation/translations/pt_BR/index.rst
+ create mode 100644 Documentation/translations/pt_BR/process/changes.rst
+ create mode 100644 Documentation/translations/pt_BR/process/howto.rst
+ create mode 100644 Documentation/translations/pt_BR/process/maintainer-handbooks.rst
+ create mode 100644 Documentation/translations/pt_BR/process/maintainer-netdev.rst
+ create mode 100644 tools/lib/python/kdoc/xforms_lists.py
+Merging v4l-dvb/next (0e2c4117c3512 staging: media: tegra-video: add CSI support for Tegra20 and Tegra30)
+$ git merge -m Merge branch 'next' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb/next
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging drivers/media/platform/synopsys/Kconfig
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   1 +
+ Documentation/admin-guide/media/mgb4.rst           |   8 +
+ Documentation/admin-guide/media/starfive_camss.rst |  72 --
+ .../admin-guide/media/starfive_camss_graph.dot     |  12 -
+ Documentation/admin-guide/media/v4l-drivers.rst    |   1 -
+ .../media/i2c/alliedvision,alvium-csi2.yaml        |   2 +-
+ .../bindings/media/i2c/onnn,mt9m114.yaml           |   4 +-
+ .../devicetree/bindings/media/i2c/sony,imx355.yaml | 111 +++
+ .../bindings/media/i2c/ti,ds90ub960.yaml           | 221 ++---
+ .../bindings/media/qcom,sdm670-camss.yaml          |   3 -
+ .../devicetree/bindings/media/st,stm32-dcmi.yaml   |  11 +-
+ .../bindings/media/starfive,jh7110-camss.yaml      | 180 ----
+ Documentation/driver-api/media/index.rst           |   1 +
+ .../driver-api/media/maintainer-entry-profile.rst  | 463 +++++++++--
+ .../driver-api/media/media-committers.rst          | 203 +++++
+ Documentation/process/maintainer-pgp-guide.rst     |   2 +
+ .../userspace-api/media/dvb/legacy_dvb_audio.rst   |   2 +-
+ MAINTAINERS                                        |  15 +-
+ drivers/gpu/drm/bridge/sil-sii8620.c               |   1 +
+ drivers/gpu/drm/tegra/dsi.c                        |   1 +
+ drivers/gpu/host1x/Makefile                        |   1 +
+ drivers/gpu/host1x/mipi.c                          | 612 +++-----------
+ drivers/gpu/host1x/tegra114-mipi.c                 | 483 +++++++++++
+ drivers/hid/hid-picolcd_cir.c                      |   1 +
+ drivers/media/cec/core/cec-core.c                  |   2 +-
+ drivers/media/cec/usb/pulse8/pulse8-cec.c          |   7 +
+ drivers/media/common/b2c2/flexcop.c                |  14 -
+ drivers/media/common/saa7146/saa7146_fops.c        |  13 -
+ drivers/media/common/siano/smsir.c                 |   1 +
+ drivers/media/common/videobuf2/videobuf2-dma-sg.c  |   1 +
+ drivers/media/dvb-frontends/au8522_decoder.c       |   1 +
+ drivers/media/dvb-frontends/dib8000.c              |   4 +-
+ drivers/media/dvb-frontends/si2168.c               |   8 +-
+ drivers/media/i2c/Kconfig                          |   6 +-
+ drivers/media/i2c/alvium-csi2.c                    |   2 +-
+ drivers/media/i2c/ar0521.c                         |   3 +
+ drivers/media/i2c/ccs-pll.c                        |   5 +-
+ drivers/media/i2c/cx25840/cx25840-core.c           |  29 +-
+ drivers/media/i2c/ds90ub913.c                      |  59 +-
+ drivers/media/i2c/ds90ub953.c                      |  61 +-
+ drivers/media/i2c/ds90ub960.c                      | 214 +++--
+ drivers/media/i2c/imx219.c                         |   3 +
+ drivers/media/i2c/imx258.c                         |  14 +-
+ drivers/media/i2c/imx283.c                         |  15 +-
+ drivers/media/i2c/imx355.c                         | 111 ++-
+ drivers/media/i2c/imx412.c                         |   8 +-
+ drivers/media/i2c/ir-kbd-i2c.c                     |   2 +
+ drivers/media/i2c/max9286.c                        |   2 +-
+ drivers/media/i2c/mt9m114.c                        |  35 +-
+ drivers/media/i2c/mt9p031.c                        |   4 +
+ drivers/media/i2c/og01a1b.c                        | 907 ++++++++++-----------
+ drivers/media/i2c/ov5647.c                         |  12 +-
+ drivers/media/i2c/ov9282.c                         | 681 ++++++----------
+ drivers/media/i2c/vgxy61.c                         |   3 +
+ drivers/media/pci/bt8xx/bttv-input.c               |   3 +-
+ drivers/media/pci/cx23885/cx23885-input.c          |   1 +
+ drivers/media/pci/cx25821/cx25821-alsa.c           |   6 +-
+ drivers/media/pci/cx88/cx88-input.c                |   3 +-
+ drivers/media/pci/dm1105/dm1105.c                  |   1 +
+ drivers/media/pci/intel/ipu-bridge.c               |  16 +
+ drivers/media/pci/intel/ipu6/ipu6.c                |   2 +-
+ drivers/media/pci/mantis/mantis_input.c            |   1 +
+ drivers/media/pci/mgb4/mgb4_core.c                 |   3 +-
+ drivers/media/pci/mgb4/mgb4_sysfs_in.c             |  18 +-
+ drivers/media/pci/mgb4/mgb4_sysfs_out.c            |  62 ++
+ drivers/media/pci/saa7134/saa7134-i2c.c            |  26 +-
+ drivers/media/pci/saa7134/saa7134-input.c          |   1 +
+ drivers/media/pci/saa7164/saa7164-core.c           |  47 +-
+ drivers/media/pci/saa7164/saa7164-fw.c             |   4 +-
+ drivers/media/pci/smipcie/smipcie-ir.c             |   1 +
+ drivers/media/pci/ttpci/budget-ci.c                |   1 +
+ drivers/media/pci/zoran/zoran_card.c               |   2 +-
+ drivers/media/platform/broadcom/Kconfig            |   5 -
+ drivers/media/platform/broadcom/bcm2835-unicam.c   |   4 -
+ drivers/media/platform/renesas/rcar-csi2.c         | 116 +--
+ drivers/media/platform/renesas/rcar-isp/csisp.c    |  13 +-
+ .../platform/rockchip/rkcif/rkcif-capture-mipi.c   |  10 +-
+ .../platform/rockchip/rkcif/rkcif-interface.c      |   3 +-
+ .../media/platform/rockchip/rkcif/rkcif-stream.c   |  46 +-
+ .../media/platform/samsung/exynos4-is/fimc-core.c  |   6 +-
+ .../media/platform/samsung/exynos4-is/fimc-is.c    |  11 +-
+ drivers/media/platform/st/stm32/stm32-dcmi.c       | 475 +++++++----
+ drivers/media/platform/synopsys/Kconfig            |   1 +
+ .../media/platform/synopsys/hdmirx/snps_hdmirx.c   |   6 +-
+ drivers/media/platform/ti/omap3isp/ispvideo.c      |   1 +
+ drivers/media/platform/ti/vpe/vip.c                |   1 +
+ drivers/media/rc/ati_remote.c                      |   6 +-
+ drivers/media/rc/ene_ir.c                          |   2 +-
+ drivers/media/rc/fintek-cir.c                      |   3 +-
+ drivers/media/rc/gpio-ir-tx.c                      |   4 +-
+ drivers/media/rc/igorplugusb.c                     |  17 +-
+ drivers/media/rc/iguanair.c                        |   1 +
+ drivers/media/rc/img-ir/img-ir-hw.c                |   3 +-
+ drivers/media/rc/img-ir/img-ir-raw.c               |   3 +-
+ drivers/media/rc/imon.c                            |  16 +-
+ drivers/media/rc/ir-hix5hd2.c                      |   2 +-
+ drivers/media/rc/ir_toy.c                          |   1 +
+ drivers/media/rc/ite-cir.c                         |   2 +-
+ drivers/media/rc/mceusb.c                          |   5 +-
+ drivers/media/rc/rc-ir-raw.c                       |   5 -
+ drivers/media/rc/rc-loopback.c                     |   1 +
+ drivers/media/rc/rc-main.c                         |   6 +-
+ drivers/media/rc/redrat3.c                         |   4 +-
+ drivers/media/rc/st_rc.c                           |   2 +-
+ drivers/media/rc/streamzap.c                       |  19 +-
+ drivers/media/rc/sunxi-cir.c                       |   1 +
+ drivers/media/rc/ttusbir.c                         |  15 +-
+ drivers/media/rc/winbond-cir.c                     |   2 +-
+ drivers/media/rc/xbox_remote.c                     |  14 +-
+ drivers/media/test-drivers/vidtv/vidtv_bridge.c    |   4 +-
+ drivers/media/test-drivers/vidtv/vidtv_channel.c   |   4 +
+ drivers/media/test-drivers/vidtv/vidtv_mux.c       |   4 +-
+ drivers/media/test-drivers/vidtv/vidtv_s302m.c     |   6 +-
+ drivers/media/test-drivers/vidtv/vidtv_ts.c        |  48 +-
+ drivers/media/test-drivers/vidtv/vidtv_ts.h        |   4 +-
+ drivers/media/test-drivers/vimc/vimc-common.h      |  41 +
+ drivers/media/test-drivers/vimc/vimc-sensor.c      | 114 ++-
+ drivers/media/test-drivers/vimc/vimc-streamer.c    |  33 +-
+ drivers/media/tuners/mxl5005s.c                    |   6 +
+ drivers/media/tuners/si2157.c                      |  15 +-
+ drivers/media/tuners/xc5000.c                      |  12 +-
+ drivers/media/usb/as102/as102_usb_drv.c            |   2 +
+ drivers/media/usb/au0828/au0828-cards.c            |  50 ++
+ drivers/media/usb/au0828/au0828-cards.h            |   2 +
+ drivers/media/usb/au0828/au0828-dvb.c              |   1 +
+ drivers/media/usb/au0828/au0828-input.c            |   2 +
+ drivers/media/usb/cx231xx/cx231xx-avcore.c         |   7 +-
+ drivers/media/usb/cx231xx/cx231xx-cards.c          |   7 +-
+ drivers/media/usb/dvb-usb-v2/dvb_usb_core.c        |   1 +
+ drivers/media/usb/dvb-usb/dvb-usb-remote.c         |   6 +-
+ drivers/media/usb/em28xx/em28xx-cards.c            |  19 +-
+ drivers/media/usb/em28xx/em28xx-input.c            |   1 +
+ drivers/media/usb/em28xx/em28xx-video.c            |  14 +-
+ drivers/media/usb/go7007/go7007-loader.c           |   6 +-
+ drivers/media/usb/gspca/gspca.c                    |  13 -
+ drivers/media/usb/hackrf/hackrf.c                  |   7 +-
+ drivers/media/usb/usbtv/usbtv-core.c               |   4 +-
+ drivers/media/v4l2-core/v4l2-subdev.c              | 113 +++
+ drivers/staging/media/Kconfig                      |   2 -
+ drivers/staging/media/Makefile                     |   1 -
+ drivers/staging/media/atomisp/pci/atomisp_ioctl.c  |   4 +
+ drivers/staging/media/av7110/av7110.c              |   2 +-
+ drivers/staging/media/av7110/av7110_ir.c           |   1 +
+ drivers/staging/media/ipu3/ipu3.c                  |   2 +-
+ drivers/staging/media/starfive/Kconfig             |   5 -
+ drivers/staging/media/starfive/Makefile            |   2 -
+ drivers/staging/media/starfive/camss/Kconfig       |  18 -
+ drivers/staging/media/starfive/camss/Makefile      |  13 -
+ drivers/staging/media/starfive/camss/TODO.txt      |   4 -
+ drivers/staging/media/starfive/camss/stf-camss.c   | 438 ----------
+ drivers/staging/media/starfive/camss/stf-camss.h   | 134 ---
+ drivers/staging/media/starfive/camss/stf-capture.c | 605 --------------
+ drivers/staging/media/starfive/camss/stf-capture.h |  86 --
+ .../staging/media/starfive/camss/stf-isp-hw-ops.c  | 445 ----------
+ drivers/staging/media/starfive/camss/stf-isp.c     | 379 ---------
+ drivers/staging/media/starfive/camss/stf-isp.h     | 428 ----------
+ drivers/staging/media/starfive/camss/stf-video.c   | 570 -------------
+ drivers/staging/media/starfive/camss/stf-video.h   | 100 ---
+ drivers/staging/media/tegra-video/Makefile         |   1 +
+ drivers/staging/media/tegra-video/csi.c            |  68 +-
+ drivers/staging/media/tegra-video/csi.h            |  22 +
+ drivers/staging/media/tegra-video/tegra20.c        | 840 ++++++++++++++++---
+ drivers/staging/media/tegra-video/vi.c             |  58 +-
+ drivers/staging/media/tegra-video/vi.h             |   6 +-
+ drivers/staging/media/tegra-video/video.c          |   8 +-
+ drivers/staging/media/tegra-video/vip.c            |   2 +-
+ drivers/staging/media/tegra-video/vip.h            |   2 +-
+ include/linux/host1x.h                             |  10 -
+ include/linux/tegra-mipi-cal.h                     |  57 ++
+ include/media/rc-core.h                            |   4 +-
+ include/media/v4l2-subdev.h                        |  22 +
+ 171 files changed, 4524 insertions(+), 6029 deletions(-)
+ delete mode 100644 Documentation/admin-guide/media/starfive_camss.rst
+ delete mode 100644 Documentation/admin-guide/media/starfive_camss_graph.dot
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
+ delete mode 100644 Documentation/devicetree/bindings/media/starfive,jh7110-camss.yaml
+ create mode 100644 Documentation/driver-api/media/media-committers.rst
+ create mode 100644 drivers/gpu/host1x/tegra114-mipi.c
+ delete mode 100644 drivers/staging/media/starfive/Kconfig
+ delete mode 100644 drivers/staging/media/starfive/Makefile
+ delete mode 100644 drivers/staging/media/starfive/camss/Kconfig
+ delete mode 100644 drivers/staging/media/starfive/camss/Makefile
+ delete mode 100644 drivers/staging/media/starfive/camss/TODO.txt
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-camss.c
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-camss.h
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-capture.c
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-capture.h
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-isp-hw-ops.c
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-isp.c
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-isp.h
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-video.c
+ delete mode 100644 drivers/staging/media/starfive/camss/stf-video.h
+ create mode 100644 include/linux/tegra-mipi-cal.h
+Merging v4l-dvb-next/master (adc218676eef2 Linux 6.12)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (b9d1532ab4db3 Merge branch 'acpi-cppc' into linux-next)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging drivers/platform/x86/hp/hp-wmi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-devices-system-cpu |  18 +
+ Documentation/admin-guide/kernel-parameters.txt    |   8 +
+ arch/x86/kernel/rtc.c                              |  19 +-
+ arch/x86/power/hibernate_64.c                      |   2 -
+ drivers/acpi/ac.c                                  |  31 +-
+ drivers/acpi/acpi_memhotplug.c                     |   4 -
+ drivers/acpi/acpi_pad.c                            |  28 +-
+ drivers/acpi/acpi_pnp.c                            |  22 +-
+ drivers/acpi/acpi_processor.c                      |   2 -
+ drivers/acpi/acpi_tad.c                            | 279 +++++---
+ drivers/acpi/acpi_video.c                          | 100 ++-
+ drivers/acpi/battery.c                             |   9 +-
+ drivers/acpi/button.c                              |  11 +-
+ drivers/acpi/cppc_acpi.c                           | 293 ++++++--
+ drivers/acpi/ec.c                                  |   6 -
+ drivers/acpi/event.c                               |   7 +-
+ drivers/acpi/osl.c                                 |  19 +-
+ drivers/acpi/pci_link.c                            |   4 -
+ drivers/acpi/pci_root.c                            |   9 +-
+ drivers/acpi/power.c                               |   4 -
+ drivers/acpi/processor_driver.c                    |  22 +-
+ drivers/acpi/processor_idle.c                      |  21 +-
+ drivers/acpi/sbs.c                                 |   4 -
+ drivers/acpi/sbshc.c                               |   6 -
+ drivers/acpi/thermal.c                             |  13 +-
+ drivers/acpi/x86/cmos_rtc.c                        |  88 +--
+ drivers/base/auxiliary.c                           |  10 +
+ drivers/cpufreq/cppc_cpufreq.c                     | 104 ++-
+ drivers/cpufreq/cpufreq.c                          |   4 +-
+ drivers/cpufreq/cpufreq_governor.h                 |   5 +-
+ drivers/cpufreq/intel_pstate.c                     |   2 +-
+ drivers/cpuidle/governors/gov.h                    |   5 +
+ drivers/cpuidle/governors/menu.c                   |  15 +-
+ drivers/cpuidle/governors/teo.c                    |  81 +--
+ drivers/gpu/drm/amd/include/amd_acpi.h             |   2 -
+ drivers/gpu/drm/radeon/radeon_acpi.c               |   2 -
+ drivers/idle/intel_idle.c                          |  42 ++
+ drivers/platform/x86/hp/hp-wmi.c                   |   2 -
+ drivers/platform/x86/lenovo/wmi-capdata.c          |   1 -
+ drivers/powercap/intel_rapl_common.c               | 757 +++++++--------------
+ drivers/powercap/intel_rapl_msr.c                  | 251 ++++++-
+ drivers/powercap/intel_rapl_tpmi.c                 |  48 ++
+ drivers/rtc/rtc-cmos.c                             | 138 +---
+ .../intel/int340x_thermal/processor_thermal_rapl.c |   9 +
+ .../intel/int340x_thermal/processor_thermal_rfim.c |  25 +-
+ drivers/thermal/intel/intel_hfi.c                  |   2 +-
+ drivers/thermal/thermal_core.c                     |   2 +-
+ drivers/thermal/thermal_sysfs.c                    |   7 +-
+ include/acpi/acpi_bus.h                            |  14 +-
+ include/acpi/cppc_acpi.h                           |  22 +-
+ include/acpi/processor.h                           |   2 -
+ include/linux/acpi.h                               |  10 +
+ include/linux/auxiliary_bus.h                      |   2 +
+ include/linux/cpufreq.h                            |   2 +-
+ include/linux/intel_rapl.h                         |  17 +-
+ include/linux/powercap.h                           |   4 +-
+ include/linux/units.h                              |   3 +
+ 57 files changed, 1465 insertions(+), 1154 deletions(-)
+Merging cpufreq-arm/cpufreq/arm/linux-next (8655a4e35cda5 cpufreq: tegra194: remove COMPILE_TEST)
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../arm/tegra/nvidia,tegra-ccplex-cluster.yaml     |  1 +
+ .../bindings/cpufreq/cpufreq-qcom-hw.yaml          |  1 +
+ MAINTAINERS                                        | 23 ++++++++++++++++------
+ drivers/cpufreq/Kconfig.arm                        |  2 +-
+ drivers/cpufreq/cpufreq-dt-platdev.c               |  1 +
+ drivers/cpufreq/tegra194-cpufreq.c                 |  4 ++--
+ 6 files changed, 23 insertions(+), 9 deletions(-)
+Merging cpupower/cpupower (3817b1d34432d cpupower: Add intel_pstate turbo boost support for Intel platforms)
+$ git merge -m Merge branch 'cpupower' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Already up to date.
+Merging devfreq/devfreq-next (85eaa767662a6 PM / devfreq: use _visible attribute to replace create/remove_sysfs_files())
+$ git merge -m Merge branch 'devfreq-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Merge made by the 'ort' strategy.
+ drivers/devfreq/devfreq.c | 108 ++++++++++++++++++++++++++--------------------
+ 1 file changed, 62 insertions(+), 46 deletions(-)
+Merging pmdomain/next (bc330699801d3 pmdomain: qcom: cpr: simplify main allocation)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Merge made by the 'ort' strategy.
+ .../bindings/power/allwinner,sun20i-d1-ppu.yaml    |  17 +-
+ .../bindings/power/mediatek,mt8196-gpufreq.yaml    |  13 +
+ .../bindings/power/mediatek,power-controller.yaml  |   1 +
+ .../devicetree/bindings/power/qcom,rpmpd.yaml      |   1 +
+ drivers/pmdomain/arm/scmi_pm_domain.c              |   1 +
+ drivers/pmdomain/bcm/bcm2835-power.c               |  25 +-
+ drivers/pmdomain/core.c                            |  59 ++-
+ drivers/pmdomain/imx/imx93-blk-ctrl.c              |  77 ++--
+ drivers/pmdomain/imx/scu-pd.c                      |   1 +
+ .../pmdomain/marvell/pxa1908-power-controller.c    |  39 +-
+ drivers/pmdomain/mediatek/mt8189-pm-domains.h      | 485 +++++++++++++++++++++
+ drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c       |  59 +++
+ drivers/pmdomain/mediatek/mtk-pm-domains.c         |  44 +-
+ drivers/pmdomain/mediatek/mtk-pm-domains.h         |   5 +
+ drivers/pmdomain/mediatek/mtk-scpsys.c             |  10 +
+ drivers/pmdomain/qcom/cpr.c                        |  13 +-
+ drivers/pmdomain/qcom/rpmhpd.c                     |  20 +
+ drivers/pmdomain/qcom/rpmpd.c                      |   2 +-
+ drivers/pmdomain/rockchip/pm-domains.c             |   7 +-
+ drivers/pmdomain/sunxi/sun55i-pck600.c             |  35 +-
+ drivers/pmdomain/ti/omap_prm.c                     |   1 +
+ drivers/pmdomain/ti/ti_sci_pm_domains.c            |   5 +-
+ .../power/allwinner,sun60i-a733-pck-600.h          |  18 +
+ include/dt-bindings/power/marvell,pxa1908-power.h  |   1 +
+ include/dt-bindings/power/mediatek,mt8189-power.h  |  38 ++
+ include/dt-bindings/power/mt7622-power.h           |   1 +
+ include/linux/pm_domain.h                          |   5 +-
+ 27 files changed, 880 insertions(+), 103 deletions(-)
+ create mode 100644 drivers/pmdomain/mediatek/mt8189-pm-domains.h
+ create mode 100644 include/dt-bindings/power/allwinner,sun60i-a733-pck-600.h
+ create mode 100644 include/dt-bindings/power/mediatek,mt8189-power.h
+Merging opp/opp/linux-next (3d2398f44a2d4 OPP: Move break out of scoped_guard in dev_pm_opp_xlate_required_opp())
+$ git merge -m Merge branch 'opp/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Merge made by the 'ort' strategy.
+ drivers/opp/core.c    |  2 +-
+ drivers/opp/debugfs.c | 20 +++++++++++---------
+ 2 files changed, 12 insertions(+), 10 deletions(-)
+Merging thermal/thermal/linux-next (b3414148bbc1f thermal/drivers/sprd: Fix raw temperature clamping in sprd_thm_rawdata_to_temp)
+$ git merge -m Merge branch 'thermal/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Merge made by the 'ort' strategy.
+ drivers/thermal/imx91_thermal.c |  4 ++++
+ drivers/thermal/sprd_thermal.c  |  4 ++--
+ drivers/thermal/thermal_of.c    | 20 ++++++++++----------
+ 3 files changed, 16 insertions(+), 12 deletions(-)
+Merging rdma/for-next (1487bad4ea518 RDMA/rdmavt: Add driver mmap callback)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/infiniband/core/umem.c
+Auto-merging drivers/infiniband/hw/irdma/verbs.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/main.c
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/infiniband.rst            |   15 -
+ Documentation/infiniband/index.rst                 |    1 -
+ Documentation/infiniband/opa_vnic.rst              |  159 ---
+ .../translations/zh_CN/infiniband/index.rst        |    1 -
+ .../translations/zh_CN/infiniband/opa_vnic.rst     |  156 ---
+ MAINTAINERS                                        |    6 -
+ drivers/infiniband/Kconfig                         |    2 -
+ drivers/infiniband/core/Makefile                   |    6 +-
+ drivers/infiniband/core/cq.c                       |    3 +
+ drivers/infiniband/core/device.c                   |    4 +-
+ drivers/infiniband/core/frmr_pools.c               |  547 +++++++++
+ drivers/infiniband/core/frmr_pools.h               |   63 ++
+ drivers/infiniband/core/ib_core_uverbs.c           |   27 +
+ drivers/infiniband/core/iter.c                     |   43 +
+ drivers/infiniband/core/nldev.c                    |  286 +++++
+ drivers/infiniband/core/rdma_core.h                |    3 +
+ drivers/infiniband/core/umem.c                     |    2 +-
+ drivers/infiniband/core/umem_dmabuf.c              |  138 ++-
+ drivers/infiniband/core/uverbs_cmd.c               |   33 +-
+ drivers/infiniband/core/uverbs_ioctl.c             |   87 ++
+ drivers/infiniband/core/uverbs_std_types_cq.c      |   38 +-
+ drivers/infiniband/core/uverbs_std_types_device.c  |    8 +
+ drivers/infiniband/core/verbs.c                    |   49 +-
+ drivers/infiniband/hw/bnxt_re/Makefile             |    2 +-
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c           |  825 +++++++-------
+ drivers/infiniband/hw/bnxt_re/ib_verbs.h           |   13 +-
+ drivers/infiniband/hw/bnxt_re/main.c               |    2 +
+ drivers/infiniband/hw/bnxt_re/qplib_fp.c           |  305 ++----
+ drivers/infiniband/hw/bnxt_re/qplib_fp.h           |    8 +
+ drivers/infiniband/hw/bnxt_re/qplib_res.c          |   47 +-
+ drivers/infiniband/hw/bnxt_re/qplib_res.h          |   10 +
+ drivers/infiniband/hw/bnxt_re/uapi.c               |  469 ++++++++
+ drivers/infiniband/hw/cxgb4/mem.c                  |    2 +-
+ drivers/infiniband/hw/efa/efa.h                    |    6 +-
+ drivers/infiniband/hw/efa/efa_admin_cmds_defs.h    |   23 +-
+ drivers/infiniband/hw/efa/efa_com_cmd.c            |   55 +-
+ drivers/infiniband/hw/efa/efa_com_cmd.h            |    3 +-
+ drivers/infiniband/hw/efa/efa_main.c               |    3 +-
+ drivers/infiniband/hw/efa/efa_verbs.c              |   51 +-
+ drivers/infiniband/hw/erdma/erdma.h                |    1 -
+ drivers/infiniband/hw/erdma/erdma_eq.c             |    3 +-
+ drivers/infiniband/hw/erdma/erdma_main.c           |    1 -
+ drivers/infiniband/hw/erdma/erdma_verbs.c          |    2 +-
+ drivers/infiniband/hw/hfi1/Makefile                |    4 +-
+ drivers/infiniband/hw/hfi1/aspm.c                  |    2 +-
+ drivers/infiniband/hw/hfi1/chip.c                  |   54 +-
+ drivers/infiniband/hw/hfi1/chip.h                  |    2 -
+ drivers/infiniband/hw/hfi1/driver.c                |   13 +-
+ drivers/infiniband/hw/hfi1/hfi.h                   |   20 -
+ drivers/infiniband/hw/hfi1/init.c                  |    4 +-
+ drivers/infiniband/hw/hfi1/mad.c                   |    1 -
+ drivers/infiniband/hw/hfi1/msix.c                  |    4 +-
+ drivers/infiniband/hw/hfi1/netdev.h                |    8 +-
+ drivers/infiniband/hw/hfi1/netdev_rx.c             |    3 +-
+ drivers/infiniband/hw/hfi1/user_exp_rcv.c          |   14 +-
+ drivers/infiniband/hw/hfi1/user_exp_rcv.h          |    2 +-
+ drivers/infiniband/hw/hfi1/verbs.c                 |    2 -
+ drivers/infiniband/hw/hfi1/vnic.h                  |  126 ---
+ drivers/infiniband/hw/hfi1/vnic_main.c             |  615 -----------
+ drivers/infiniband/hw/hfi1/vnic_sdma.c             |  282 -----
+ drivers/infiniband/hw/hns/hns_roce_alloc.c         |    2 +-
+ drivers/infiniband/hw/hns/hns_roce_hem.c           |    7 +-
+ drivers/infiniband/hw/hns/hns_roce_qp.c            |   13 +-
+ drivers/infiniband/hw/ionic/ionic_ibdev.h          |    2 +-
+ drivers/infiniband/hw/irdma/hw.c                   |    4 +-
+ drivers/infiniband/hw/irdma/main.h                 |    2 +-
+ drivers/infiniband/hw/irdma/verbs.c                |  105 +-
+ drivers/infiniband/hw/mana/mana_ib.h               |    2 +-
+ drivers/infiniband/hw/mlx4/cq.c                    |  263 +++--
+ drivers/infiniband/hw/mlx4/main.c                  |    3 +-
+ drivers/infiniband/hw/mlx4/mlx4_ib.h               |    5 +-
+ drivers/infiniband/hw/mlx4/mr.c                    |    1 +
+ drivers/infiniband/hw/mlx5/cq.c                    |  159 ++-
+ drivers/infiniband/hw/mlx5/devx.c                  |    6 +-
+ drivers/infiniband/hw/mlx5/dm.c                    |    2 +-
+ drivers/infiniband/hw/mlx5/fs.c                    |    6 +-
+ drivers/infiniband/hw/mlx5/main.c                  |  168 ++-
+ drivers/infiniband/hw/mlx5/mem.c                   |    1 +
+ drivers/infiniband/hw/mlx5/mlx5_ib.h               |   99 +-
+ drivers/infiniband/hw/mlx5/mr.c                    | 1158 ++++----------------
+ drivers/infiniband/hw/mlx5/odp.c                   |   19 -
+ drivers/infiniband/hw/mlx5/qos.c                   |    2 +-
+ drivers/infiniband/hw/mlx5/qp.c                    |    2 +-
+ drivers/infiniband/hw/mlx5/umr.c                   |    1 +
+ drivers/infiniband/hw/mlx5/umr.h                   |    1 +
+ drivers/infiniband/hw/mthca/mthca_provider.c       |    2 +-
+ drivers/infiniband/hw/ocrdma/ocrdma.h              |    2 +-
+ drivers/infiniband/hw/ocrdma/ocrdma_verbs.c        |   30 +-
+ drivers/infiniband/hw/qedr/verbs.c                 |    2 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma.h          |    2 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c     |    3 +-
+ drivers/infiniband/sw/rdmavt/mmap.c                |   22 +-
+ drivers/infiniband/sw/rdmavt/qp.c                  |    2 +-
+ drivers/infiniband/sw/rdmavt/vt.c                  |    8 +
+ drivers/infiniband/sw/rxe/rxe_resp.c               |   56 +-
+ drivers/infiniband/sw/rxe/rxe_verbs.h              |    1 +
+ drivers/infiniband/ulp/Makefile                    |    1 -
+ drivers/infiniband/ulp/opa_vnic/Kconfig            |    9 -
+ drivers/infiniband/ulp/opa_vnic/Makefile           |    9 -
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c   |  513 ---------
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h   |  524 ---------
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c |  183 ----
+ .../infiniband/ulp/opa_vnic/opa_vnic_internal.h    |  329 ------
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c  |  400 -------
+ drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c    | 1056 ------------------
+ .../infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c  |  390 -------
+ drivers/infiniband/ulp/rtrs/rtrs-clt.c             |    2 +-
+ drivers/infiniband/ulp/rtrs/rtrs-srv.c             |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fw.c       |    6 +
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   68 +-
+ include/linux/mlx5/device.h                        |    9 +
+ include/linux/mlx5/driver.h                        |   11 -
+ include/linux/mlx5/mlx5_ifc.h                      |   23 +-
+ include/rdma/frmr_pools.h                          |   39 +
+ include/rdma/ib_cache.h                            |    4 +-
+ include/rdma/ib_umem.h                             |   65 +-
+ include/rdma/ib_verbs.h                            |  182 +--
+ include/rdma/iter.h                                |   88 ++
+ include/rdma/iw_cm.h                               |   14 +-
+ include/rdma/opa_port_info.h                       |    8 +-
+ include/rdma/opa_vnic.h                            |   96 --
+ include/rdma/rdma_vt.h                             |   10 +
+ include/rdma/restrack.h                            |    4 +-
+ include/rdma/uverbs_ioctl.h                        |  101 ++
+ include/uapi/rdma/bnxt_re-abi.h                    |   36 +-
+ include/uapi/rdma/efa-abi.h                        |    5 +-
+ include/uapi/rdma/ib_user_ioctl_verbs.h            |    1 +
+ include/uapi/rdma/mlx5_user_ioctl_cmds.h           |    1 +
+ include/uapi/rdma/mlx5_user_ioctl_verbs.h          |    4 +
+ include/uapi/rdma/rdma_netlink.h                   |   22 +
+ 130 files changed, 3687 insertions(+), 7355 deletions(-)
+ delete mode 100644 Documentation/infiniband/opa_vnic.rst
+ delete mode 100644 Documentation/translations/zh_CN/infiniband/opa_vnic.rst
+ create mode 100644 drivers/infiniband/core/frmr_pools.c
+ create mode 100644 drivers/infiniband/core/frmr_pools.h
+ create mode 100644 drivers/infiniband/core/iter.c
+ create mode 100644 drivers/infiniband/hw/bnxt_re/uapi.c
+ delete mode 100644 drivers/infiniband/hw/hfi1/vnic.h
+ delete mode 100644 drivers/infiniband/hw/hfi1/vnic_main.c
+ delete mode 100644 drivers/infiniband/hw/hfi1/vnic_sdma.c
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/Kconfig
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/Makefile
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+ delete mode 100644 drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c
+ create mode 100644 include/rdma/frmr_pools.h
+ create mode 100644 include/rdma/iter.h
+ delete mode 100644 include/rdma/opa_vnic.h
+Merging net-next/main (ca7e99335aea7 net: fjes: Drop fjes_acpi_driver and rework initialization)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/qcom/Kconfig
+Auto-merging drivers/infiniband/hw/mlx5/main.c
+Auto-merging drivers/infiniband/hw/mlx5/mlx5_ib.h
+Auto-merging drivers/infiniband/hw/mlx5/mr.c
+CONFLICT (content): Merge conflict in drivers/infiniband/hw/mlx5/mr.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/main.c
+Auto-merging include/linux/mlx5/driver.h
+Auto-merging include/net/netmem.h
+CONFLICT (content): Merge conflict in include/net/netmem.h
+Auto-merging include/net/sock.h
+Auto-merging net/ipv4/ping.c
+Auto-merging net/ipv4/raw.c
+Auto-merging net/ipv4/tcp.c
+Auto-merging net/ipv4/tcp_ipv4.c
+Auto-merging net/ipv4/udp.c
+Auto-merging net/ipv6/datagram.c
+Auto-merging net/ipv6/tcp_ipv6.c
+Auto-merging net/netlink/af_netlink.c
+Auto-merging net/socket.c
+Auto-merging net/unix/af_unix.c
+Auto-merging security/smack/smack_lsm.c
+Resolved 'drivers/infiniband/hw/mlx5/mr.c' using previous resolution.
+Resolved 'include/net/netmem.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 1c55a44cc17f2] Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/sysctl/net.rst           |   28 +
+ .../devicetree/bindings/net/cdns,macb.yaml         |   19 +
+ .../bindings/net/dsa/maxlinear,mxl862xx.yaml       |    1 -
+ Documentation/devicetree/bindings/net/micrel.yaml  |    6 +-
+ .../devicetree/bindings/net/nxp,s32-dwmac.yaml     |   47 +-
+ .../devicetree/bindings/net/qcom,ipa.yaml          |   11 +
+ .../devicetree/bindings/net/snps,dwmac.yaml        |    2 +
+ .../devicetree/bindings/net/spacemit,k3-dwmac.yaml |  102 +
+ .../devicetree/bindings/sram/qcom,imem.yaml        |   14 +
+ Documentation/netlink/genetlink.yaml               |    2 +-
+ Documentation/netlink/netlink-raw.yaml             |   11 +-
+ Documentation/netlink/specs/devlink.yaml           |   58 +
+ Documentation/netlink/specs/ethtool.yaml           |   25 +-
+ Documentation/netlink/specs/nftables.yaml          |  691 ++-
+ Documentation/netlink/specs/ovpn.yaml              |   23 +-
+ .../device_drivers/ethernet/marvell/octeontx2.rst  |    2 +-
+ .../networking/devlink/devlink-shared.rst          |   97 +
+ Documentation/networking/devlink/index.rst         |    1 +
+ Documentation/networking/ethtool-netlink.rst       |   23 +-
+ Documentation/networking/ip-sysctl.rst             |   40 +-
+ .../net_cachelines/netns_ipv4_sysctl.rst           |    2 +
+ .../networking/net_cachelines/tcp_sock.rst         |    1 +
+ Documentation/networking/scaling.rst               |   13 +-
+ Documentation/networking/smc-sysctl.rst            |   43 +-
+ Documentation/networking/tls-handshake.rst         |    2 +-
+ Documentation/process/maintainer-netdev.rst        |   10 +-
+ MAINTAINERS                                        |    8 +-
+ drivers/base/regmap/Kconfig                        |    2 +-
+ drivers/clk/qcom/Kconfig                           |    2 +-
+ drivers/dibs/dibs_main.c                           |   14 +-
+ drivers/dpll/zl3073x/Makefile                      |    4 +-
+ drivers/dpll/zl3073x/chan.c                        |  165 +
+ drivers/dpll/zl3073x/chan.h                        |  179 +
+ drivers/dpll/zl3073x/core.c                        |  152 +-
+ drivers/dpll/zl3073x/core.h                        |   71 +-
+ drivers/dpll/zl3073x/dpll.c                        |  514 +--
+ drivers/dpll/zl3073x/dpll.h                        |    6 +-
+ drivers/dpll/zl3073x/i2c.c                         |   37 +-
+ drivers/dpll/zl3073x/out.c                         |   27 +-
+ drivers/dpll/zl3073x/out.h                         |   21 +-
+ drivers/dpll/zl3073x/ref.c                         |   58 +-
+ drivers/dpll/zl3073x/ref.h                         |   33 +-
+ drivers/dpll/zl3073x/regs.h                        |   14 +
+ drivers/dpll/zl3073x/spi.c                         |   37 +-
+ drivers/dpll/zl3073x/synth.h                       |   16 +-
+ drivers/gpio/TODO                                  |   28 -
+ drivers/gpio/gpiolib-of.c                          |   31 +-
+ drivers/infiniband/hw/mlx5/ib_rep.c                |   24 +-
+ drivers/infiniband/hw/mlx5/main.c                  |   21 +-
+ drivers/infiniband/hw/mlx5/mlx5_ib.h               |    1 -
+ drivers/infiniband/hw/mlx5/mr.c                    |    1 -
+ drivers/net/Kconfig                                |    1 +
+ drivers/net/bonding/bond_main.c                    |    4 +-
+ drivers/net/bonding/bond_netlink.c                 |    9 +
+ drivers/net/dsa/mv88e6xxx/Makefile                 |    2 +
+ drivers/net/dsa/mv88e6xxx/chip.c                   |   35 +
+ drivers/net/dsa/mv88e6xxx/chip.h                   |   52 +
+ drivers/net/dsa/mv88e6xxx/port.c                   |   28 +-
+ drivers/net/dsa/mv88e6xxx/port.h                   |    7 +-
+ drivers/net/dsa/mv88e6xxx/tcam.c                   |  338 ++
+ drivers/net/dsa/mv88e6xxx/tcam.h                   |   41 +
+ drivers/net/dsa/mv88e6xxx/tcflower.c               |  167 +
+ drivers/net/dsa/mv88e6xxx/tcflower.h               |   14 +
+ drivers/net/dsa/mxl862xx/mxl862xx.c                |   32 +-
+ drivers/net/ethernet/airoha/airoha_eth.c           |   56 +-
+ drivers/net/ethernet/airoha/airoha_eth.h           |    3 +-
+ drivers/net/ethernet/airoha/airoha_ppe.c           |    2 +-
+ drivers/net/ethernet/airoha/airoha_regs.h          |    5 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-common.h        |    4 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-dev.c           |   39 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-drv.c           |   47 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-pci.c           |   91 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c        |   22 +-
+ drivers/net/ethernet/amd/xgbe/xgbe-platform.c      |   14 +-
+ drivers/net/ethernet/amd/xgbe/xgbe.h               |   34 +-
+ .../net/ethernet/aquantia/atlantic/aq_ethtool.c    |   55 +-
+ drivers/net/ethernet/aquantia/atlantic/aq_hw.h     |    3 +
+ .../aquantia/atlantic/hw_atl/hw_atl_utils.h        |    7 +
+ .../aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c   |   80 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |   18 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.h          |    8 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c  |   20 +-
+ drivers/net/ethernet/cadence/macb.h                |   20 +
+ drivers/net/ethernet/cadence/macb_main.c           |  317 +-
+ .../ethernet/chelsio/inline_crypto/chtls/chtls.h   |    2 +-
+ .../chelsio/inline_crypto/chtls/chtls_io.c         |    8 +-
+ drivers/net/ethernet/cisco/enic/enic.h             |   39 +
+ drivers/net/ethernet/cisco/enic/enic_ethtool.c     |  155 +-
+ drivers/net/ethernet/freescale/Makefile            |    3 +-
+ drivers/net/ethernet/freescale/dpaa2/Kconfig       |    4 +
+ drivers/net/ethernet/freescale/enetc/ntmp.c        |    2 +-
+ .../net/ethernet/freescale/fs_enet/fs_enet-main.c  |    4 +-
+ drivers/net/ethernet/freescale/ucc_geth.c          |    3 +-
+ drivers/net/ethernet/google/gve/gve.h              |   18 +-
+ drivers/net/ethernet/google/gve/gve_adminq.c       |   30 +-
+ .../net/ethernet/google/gve/gve_buffer_mgmt_dqo.c  |    2 +-
+ drivers/net/ethernet/google/gve/gve_main.c         |   56 +-
+ drivers/net/ethernet/google/gve/gve_rx.c           |    5 +-
+ drivers/net/ethernet/google/gve/gve_rx_dqo.c       |   43 +-
+ drivers/net/ethernet/google/gve/gve_tx.c           |    5 +-
+ drivers/net/ethernet/google/gve/gve_tx_dqo.c       |   37 +-
+ drivers/net/ethernet/huawei/hinic3/Makefile        |    1 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c   |  195 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h   |   15 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_common.c |    6 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_common.h |    1 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_csr.h    |    2 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c    |   65 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h    |    5 +
+ .../net/ethernet/huawei/hinic3/hinic3_ethtool.c    |  425 ++
+ drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c |   47 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h |    8 +
+ .../net/ethernet/huawei/hinic3/hinic3_hw_comm.c    |   69 +
+ .../net/ethernet/huawei/hinic3/hinic3_hw_comm.h    |    2 +
+ .../net/ethernet/huawei/hinic3/hinic3_hw_intf.h    |   12 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c  |    9 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h  |    1 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c   |   23 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_lld.c    |    1 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_main.c   |    3 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c   |   39 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h   |    4 +
+ .../ethernet/huawei/hinic3/hinic3_mgmt_interface.h |   16 +-
+ .../net/ethernet/huawei/hinic3/hinic3_netdev_ops.c |   16 +-
+ .../net/ethernet/huawei/hinic3/hinic3_nic_cfg.c    |   77 +
+ .../net/ethernet/huawei/hinic3/hinic3_nic_cfg.h    |  110 +
+ .../net/ethernet/huawei/hinic3/hinic3_nic_dev.h    |    3 +
+ .../net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h |    1 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_rss.c    |    2 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_rx.c     |   15 +
+ drivers/net/ethernet/huawei/hinic3/hinic3_rx.h     |    3 +
+ drivers/net/ethernet/ibm/ibmveth.c                 |    4 +-
+ drivers/net/ethernet/intel/i40e/i40e_ethtool.c     |   21 +-
+ drivers/net/ethernet/intel/i40e/i40e_hmc.h         |    2 +
+ drivers/net/ethernet/intel/ice/ice_ptp_hw.h        |    2 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe.h           |   27 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c       |    4 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c      |   56 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c       |    2 +-
+ drivers/net/ethernet/litex/litex_liteeth.c         |   21 +-
+ drivers/net/ethernet/marvell/mvneta.c              |    3 +
+ drivers/net/ethernet/marvell/octeontx2/af/Makefile |    2 +-
+ drivers/net/ethernet/marvell/octeontx2/af/cgx.c    |    2 +-
+ .../ethernet/marvell/octeontx2/af/cn20k/debugfs.c  |  273 ++
+ .../ethernet/marvell/octeontx2/af/cn20k/debugfs.h  |    3 +
+ .../marvell/octeontx2/af/cn20k/mbox_init.c         |   17 +
+ .../net/ethernet/marvell/octeontx2/af/cn20k/npc.c  | 4522 ++++++++++++++++++++
+ .../net/ethernet/marvell/octeontx2/af/cn20k/npc.h  |  340 ++
+ .../net/ethernet/marvell/octeontx2/af/cn20k/reg.h  |   65 +
+ drivers/net/ethernet/marvell/octeontx2/af/common.h |    4 -
+ drivers/net/ethernet/marvell/octeontx2/af/mbox.h   |  172 +-
+ drivers/net/ethernet/marvell/octeontx2/af/npc.h    |    2 +
+ .../ethernet/marvell/octeontx2/af/npc_profile.h    |   84 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rvu.c    |  114 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rvu.h    |   18 +-
+ .../ethernet/marvell/octeontx2/af/rvu_debugfs.c    |   76 +-
+ .../ethernet/marvell/octeontx2/af/rvu_devlink.c    |   95 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_nix.c    |    5 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc.c    |  600 ++-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc.h    |   21 +
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c |  708 ++-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h |   14 +-
+ .../ethernet/marvell/octeontx2/af/rvu_npc_hash.c   |  120 +-
+ .../ethernet/marvell/octeontx2/af/rvu_npc_hash.h   |    2 +-
+ drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c |  266 ++
+ drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h |   14 +
+ .../ethernet/marvell/octeontx2/nic/otx2_common.h   |   35 +
+ .../ethernet/marvell/octeontx2/nic/otx2_flows.c    |  267 +-
+ .../net/ethernet/marvell/octeontx2/nic/otx2_tc.c   |   79 +-
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c        |   34 +-
+ drivers/net/ethernet/mellanox/mlx5/core/Makefile   |    5 +-
+ .../mellanox/mlx5/core/diag/fs_tracepoint.c        |    3 +
+ drivers/net/ethernet/mellanox/mlx5/core/en.h       |    9 +-
+ .../net/ethernet/mellanox/mlx5/core/en/params.c    |  413 +-
+ .../net/ethernet/mellanox/mlx5/core/en/params.h    |   45 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c   |    4 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c   |    5 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h   |    3 +-
+ .../net/ethernet/mellanox/mlx5/core/en/xsk/pool.c  |   23 +-
+ .../net/ethernet/mellanox/mlx5/core/en/xsk/setup.c |   59 +-
+ .../net/ethernet/mellanox/mlx5/core/en/xsk/setup.h |    6 +-
+ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |    6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_fs.c    |    1 -
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |  194 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_rx.c    |   36 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_stats.c |   30 +
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |    9 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   14 +-
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |  103 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c   |    6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.c  |   17 +-
+ .../net/ethernet/mellanox/mlx5/core/lag/debugfs.c  |    3 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |  704 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h  |   49 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c   |   20 +-
+ .../net/ethernet/mellanox/mlx5/core/lag/mpesw.c    |   15 +-
+ .../net/ethernet/mellanox/mlx5/core/lag/port_sel.c |   28 +-
+ .../net/ethernet/mellanox/mlx5/core/lib/clock.c    |   40 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c   |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   17 +
+ .../net/ethernet/mellanox/mlx5/core/sh_devlink.c   |   61 +
+ .../net/ethernet/mellanox/mlx5/core/sh_devlink.h   |   12 +
+ drivers/net/ethernet/meta/fbnic/fbnic.h            |   35 +
+ drivers/net/ethernet/meta/fbnic/fbnic_csr.c        |  128 +
+ drivers/net/ethernet/meta/fbnic/fbnic_csr.h        |   30 +
+ drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c    |  139 +
+ drivers/net/ethernet/meta/fbnic/fbnic_fw.c         |  100 +
+ drivers/net/ethernet/meta/fbnic/fbnic_fw.h         |   27 +
+ drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h   |    1 +
+ drivers/net/ethernet/meta/fbnic/fbnic_irq.c        |  156 +
+ drivers/net/ethernet/meta/fbnic/fbnic_mac.c        |  111 +
+ drivers/net/ethernet/meta/fbnic/fbnic_mac.h        |   27 +
+ drivers/net/ethernet/meta/fbnic/fbnic_pci.c        |    5 +
+ drivers/net/ethernet/meta/fbnic/fbnic_tlv.c        |  276 ++
+ drivers/net/ethernet/meta/fbnic/fbnic_tlv.h        |   27 +
+ drivers/net/ethernet/microchip/lan743x_ethtool.c   |    8 +
+ .../ethernet/microchip/sparx5/sparx5_calendar.c    |   15 +-
+ .../net/ethernet/microchip/sparx5/sparx5_ethtool.c |    9 +-
+ .../ethernet/microchip/sparx5/sparx5_mactable.c    |   34 +-
+ .../net/ethernet/microchip/sparx5/sparx5_main.c    |  253 +-
+ .../net/ethernet/microchip/sparx5/sparx5_main.h    |   12 +-
+ drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c |   18 +
+ .../ethernet/microchip/sparx5/sparx5_vcap_impl.c   |    2 +-
+ drivers/net/ethernet/microsoft/mana/gdma_main.c    |  125 +-
+ drivers/net/ethernet/microsoft/mana/hw_channel.c   |   12 +-
+ drivers/net/ethernet/microsoft/mana/mana_en.c      |  105 +-
+ drivers/net/ethernet/microsoft/mana/mana_ethtool.c |   75 +-
+ .../net/ethernet/pensando/ionic/ionic_ethtool.c    |    9 +-
+ drivers/net/ethernet/pensando/ionic/ionic_if.h     |    6 +
+ drivers/net/ethernet/qlogic/qed/qed_l2.c           |   53 +-
+ drivers/net/ethernet/qlogic/qlcnic/qlcnic.h        |    2 +-
+ drivers/net/ethernet/realtek/r8169.h               |    1 +
+ drivers/net/ethernet/realtek/r8169_main.c          |    7 +
+ drivers/net/ethernet/realtek/r8169_phy_config.c    |   26 +
+ drivers/net/ethernet/renesas/ravb_main.c           |    1 +
+ drivers/net/ethernet/rocker/rocker_ofdpa.c         |   31 +-
+ drivers/net/ethernet/spacemit/k1_emac.c            |    3 -
+ drivers/net/ethernet/stmicro/stmmac/Kconfig        |   12 +
+ drivers/net/ethernet/stmicro/stmmac/Makefile       |    1 +
+ drivers/net/ethernet/stmicro/stmmac/chain_mode.c   |    2 +-
+ drivers/net/ethernet/stmicro/stmmac/common.h       |   17 +-
+ drivers/net/ethernet/stmicro/stmmac/descs.h        |    2 +
+ .../ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c    |    5 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c    |   13 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c  |   15 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-loongson.c   |   20 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-mediatek.c   |    8 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-motorcomm.c  |    4 -
+ .../ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c    |  160 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c    |   38 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-socfpga.c    |    5 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-spacemit.c   |  227 +
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c    |   11 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c  |   13 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c  |    2 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c  |    2 +-
+ .../net/ethernet/stmicro/stmmac/dwmac1000_core.c   |   11 +-
+ .../net/ethernet/stmicro/stmmac/dwmac100_core.c    |   11 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c  |   17 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c |   33 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c   |    2 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h   |    8 -
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c   |   54 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac5.c       |    2 -
+ .../net/ethernet/stmicro/stmmac/dwxgmac2_core.c    |   24 +-
+ .../net/ethernet/stmicro/stmmac/dwxgmac2_descs.c   |   30 +-
+ drivers/net/ethernet/stmicro/stmmac/enh_desc.c     |   38 +-
+ drivers/net/ethernet/stmicro/stmmac/hwif.c         |   13 +-
+ drivers/net/ethernet/stmicro/stmmac/hwif.h         |   25 +-
+ drivers/net/ethernet/stmicro/stmmac/norm_desc.c    |   39 +-
+ drivers/net/ethernet/stmicro/stmmac/ring_mode.c    |    2 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac.h       |   56 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c   |    5 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  | 1011 +++--
+ drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c  |  113 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c   |   11 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_platform.c  |  104 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c   |   21 +-
+ drivers/net/ethernet/ti/am65-cpsw-nuss.c           |    4 +-
+ drivers/net/ethernet/ti/am65-cpts.c                |   23 +-
+ drivers/net/ethernet/ti/am65-cpts.h                |    8 +-
+ drivers/net/ethernet/ti/davinci_emac.c             |   39 +-
+ drivers/net/ethernet/ti/icssg/icssg_classifier.c   |   69 +-
+ drivers/net/ethernet/ti/icssg/icssg_prueth.c       |  255 +-
+ drivers/net/ethernet/ti/icssg/icssg_prueth.h       |    2 +
+ drivers/net/fjes/fjes_main.c                       |  120 +-
+ drivers/net/ipa/ipa_data.h                         |    9 +-
+ drivers/net/ipa/ipa_mem.c                          |   24 +-
+ drivers/net/macvlan.c                              |   12 +-
+ drivers/net/mdio/Kconfig                           |    1 -
+ drivers/net/mdio/mdio-gpio.c                       |   12 +-
+ drivers/net/mdio/mdio-mvusb.c                      |   14 +-
+ drivers/net/mdio/mdio-xgene.c                      |    2 +-
+ drivers/net/ntb_netdev.c                           |  542 ++-
+ drivers/net/ovpn/crypto_aead.c                     |  162 +-
+ drivers/net/ovpn/io.c                              |    8 +-
+ drivers/net/ovpn/netlink-gen.c                     |   13 +-
+ drivers/net/ovpn/netlink-gen.h                     |    6 +-
+ drivers/net/ovpn/netlink.c                         |   98 +-
+ drivers/net/ovpn/netlink.h                         |    2 +
+ drivers/net/ovpn/peer.c                            |    6 +
+ drivers/net/ovpn/peer.h                            |    4 +-
+ drivers/net/ovpn/pktid.c                           |   11 +-
+ drivers/net/ovpn/pktid.h                           |    2 +-
+ drivers/net/ovpn/skb.h                             |   13 +-
+ drivers/net/ovpn/tcp.c                             |    2 +-
+ drivers/net/phy/Kconfig                            |    6 -
+ drivers/net/phy/Makefile                           |    6 +-
+ drivers/net/phy/mdio-private.h                     |   11 -
+ drivers/net/phy/mdio_bus.c                         |  412 --
+ drivers/net/phy/mdio_bus_provider.c                |  319 ++
+ drivers/net/phy/mdio_device.c                      |  171 +-
+ drivers/net/phy/micrel.c                           |  152 +
+ drivers/net/phy/mxl-gpy.c                          |   66 +-
+ drivers/net/phy/phy_device.c                       |   51 +-
+ drivers/net/phy/phylib-internal.h                  |    7 +
+ drivers/net/phy/realtek/realtek_main.c             |    3 +
+ drivers/net/phy/vitesse.c                          |   41 +
+ drivers/net/ppp/ppp_async.c                        |    2 +-
+ drivers/net/ppp/ppp_generic.c                      |  215 +-
+ drivers/net/ppp/ppp_synctty.c                      |    2 +-
+ drivers/net/ppp/pppoe.c                            |   81 +-
+ drivers/net/ppp/pppox.c                            |    3 -
+ drivers/net/usb/cdc_ether.c                        |   54 +-
+ drivers/net/usb/lan78xx.c                          |    9 +-
+ drivers/net/usb/r8152.c                            |   50 +-
+ drivers/net/virtio_net.c                           |  536 +--
+ drivers/net/vmxnet3/vmxnet3_drv.c                  |    4 +-
+ drivers/net/wireless/ath/ath11k/ahb.c              |   10 +-
+ drivers/net/wireless/ath/ath11k/mac.c              |   36 +-
+ drivers/net/wireless/ath/ath12k/Makefile           |    1 +
+ drivers/net/wireless/ath/ath12k/core.c             |   23 +-
+ drivers/net/wireless/ath/ath12k/core.h             |    3 +
+ drivers/net/wireless/ath/ath12k/dp_rx.c            |    2 -
+ drivers/net/wireless/ath/ath12k/dp_rx.h            |    2 -
+ drivers/net/wireless/ath/ath12k/mac.c              |   14 +-
+ drivers/net/wireless/ath/ath12k/qmi.c              |    2 +-
+ drivers/net/wireless/ath/ath12k/thermal.c          |  124 +
+ drivers/net/wireless/ath/ath12k/thermal.h          |   40 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c      |   25 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c         |    6 +-
+ drivers/net/wireless/ath/ath12k/wmi.c              |   57 +-
+ drivers/net/wireless/ath/ath6kl/cfg80211.c         |   25 +-
+ drivers/net/wireless/ath/ath6kl/main.c             |    4 +-
+ drivers/net/wireless/ath/ath9k/ath9k.h             |    2 +-
+ drivers/net/wireless/ath/wil6210/cfg80211.c        |   33 +-
+ drivers/net/wireless/ath/wil6210/main.c            |    3 +-
+ drivers/net/wireless/ath/wil6210/wmi.c             |    5 +-
+ drivers/net/wireless/atmel/at76c50x-usb.c          |   12 +-
+ drivers/net/wireless/broadcom/b43/phy_n.c          |   24 +-
+ .../broadcom/brcm80211/brcmfmac/cfg80211.c         |   41 +-
+ drivers/net/wireless/intel/iwlegacy/3945.h         |    4 +-
+ drivers/net/wireless/intel/iwlegacy/4965-mac.c     |    2 +-
+ drivers/net/wireless/intel/iwlegacy/commands.h     |    9 +-
+ drivers/net/wireless/intel/iwlegacy/common.h       |    4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/nan.c       |    5 +-
+ drivers/net/wireless/intel/iwlwifi/mld/time_sync.c |    6 +-
+ .../net/wireless/intel/iwlwifi/mvm/ftm-initiator.c |    7 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c |    6 +-
+ drivers/net/wireless/marvell/libertas/cfg.c        |    8 +-
+ drivers/net/wireless/marvell/libertas/if_usb.c     |   35 +-
+ drivers/net/wireless/marvell/libertas/if_usb.h     |    3 +
+ drivers/net/wireless/marvell/libertas_tf/if_usb.c  |    2 -
+ drivers/net/wireless/marvell/mwifiex/11n_aggr.c    |    1 +
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |   36 +-
+ drivers/net/wireless/marvell/mwifiex/init.c        |    2 +-
+ drivers/net/wireless/marvell/mwifiex/tdls.c        |   12 +-
+ drivers/net/wireless/marvell/mwifiex/uap_event.c   |    7 +-
+ drivers/net/wireless/marvell/mwifiex/usb.c         |    4 -
+ drivers/net/wireless/marvell/mwl8k.c               |    4 +-
+ .../net/wireless/mediatek/mt76/mt76_connac_mac.c   |    6 +-
+ drivers/net/wireless/mediatek/mt76/mt7915/mcu.c    |    4 +-
+ drivers/net/wireless/mediatek/mt76/mt7925/mac.c    |    4 +-
+ drivers/net/wireless/mediatek/mt76/mt7996/mac.c    |    4 +-
+ drivers/net/wireless/mediatek/mt76/mt7996/mcu.c    |    6 +-
+ drivers/net/wireless/microchip/wilc1000/cfg80211.c |   44 +-
+ drivers/net/wireless/quantenna/qtnfmac/cfg80211.c  |   38 +-
+ drivers/net/wireless/quantenna/qtnfmac/core.c      |    4 +-
+ drivers/net/wireless/quantenna/qtnfmac/event.c     |    6 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800lib.c     |    4 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00usb.c     |   12 +-
+ drivers/net/wireless/realtek/rtl8xxxu/core.c       |   14 +-
+ drivers/net/wireless/realtek/rtlwifi/base.c        |   28 +-
+ drivers/net/wireless/realtek/rtlwifi/pci.c         |    2 +-
+ drivers/net/wireless/rsi/rsi_91x_mac80211.c        |   17 +-
+ drivers/net/wireless/rsi/rsi_91x_usb.c             |    2 +
+ drivers/net/wireless/rsi/rsi_common.h              |    1 +
+ drivers/net/wireless/silabs/wfx/data_rx.c          |    8 +-
+ drivers/net/wireless/virtual/mac80211_hwsim.c      |  205 +-
+ drivers/net/wireless/virtual/mac80211_hwsim.h      |    2 +
+ drivers/net/wireless/virtual/virt_wifi.c           |   12 +-
+ drivers/nfc/nfcmrvl/fw_dnld.c                      |    2 +-
+ drivers/nfc/nfcmrvl/main.c                         |   47 +-
+ drivers/nfc/nfcmrvl/nfcmrvl.h                      |    4 +-
+ drivers/nfc/nfcmrvl/uart.c                         |   23 +-
+ drivers/nfc/nfcmrvl/usb.c                          |    2 +-
+ drivers/nfc/nxp-nci/firmware.c                     |    2 +-
+ drivers/nfc/pn533/usb.c                            |    4 +-
+ drivers/nfc/pn544/i2c.c                            |    2 +-
+ drivers/nfc/port100.c                              |    4 +-
+ drivers/nfc/s3fwrn5/firmware.c                     |    2 +-
+ drivers/nfc/s3fwrn5/i2c.c                          |   54 +-
+ drivers/nfc/s3fwrn5/phy_common.c                   |   11 +-
+ drivers/nfc/s3fwrn5/phy_common.h                   |    5 +-
+ drivers/nfc/s3fwrn5/uart.c                         |   43 +-
+ drivers/phy/broadcom/Kconfig                       |    4 +-
+ drivers/phy/qualcomm/phy-qcom-sgmii-eth.c          |   62 +-
+ drivers/ptp/ptp_ocp.c                              |  365 +-
+ drivers/ptp/ptp_vmw.c                              |   23 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |   33 +-
+ drivers/vhost/vsock.c                              |   13 +
+ include/linux/atmdev.h                             |    6 +-
+ include/linux/ethtool.h                            |    8 +-
+ include/linux/ieee80211-eht.h                      |    4 +-
+ include/linux/ieee80211-ht.h                       |    3 +
+ include/linux/ieee80211-uhr.h                      |  275 +-
+ include/linux/ieee80211.h                          |   87 +-
+ include/linux/if_pppox.h                           |    6 -
+ include/linux/mdio-gpio.h                          |    9 -
+ include/linux/mdio.h                               |    2 -
+ include/linux/micrel_phy.h                         |    1 +
+ include/linux/mlx5/device.h                        |    1 +
+ include/linux/mlx5/driver.h                        |    1 +
+ include/linux/mlx5/fs.h                            |   10 +-
+ include/linux/mlx5/lag.h                           |   21 +
+ include/linux/mlx5/mlx5_ifc.h                      |   26 +-
+ include/linux/mroute_base.h                        |    9 +-
+ include/linux/net.h                                |    2 +
+ include/linux/of_gpio.h                            |   38 -
+ include/linux/phy.h                                |    5 -
+ include/linux/platform_data/mdio-gpio.h            |   14 -
+ include/linux/ppp_channel.h                        |    6 +-
+ include/linux/rculist_bl.h                         |   49 +-
+ include/linux/skbuff.h                             |   13 +-
+ include/linux/socket.h                             |    2 +-
+ include/linux/stmmac.h                             |  111 +-
+ include/linux/tcp.h                                |   14 +-
+ include/linux/udp.h                                |   10 +-
+ include/net/af_vsock.h                             |    9 +
+ include/net/bonding.h                              |   10 +-
+ include/net/cfg80211.h                             |   63 +-
+ include/net/devlink.h                              |   10 +
+ include/net/dropreason-core.h                      |   51 +-
+ include/net/dropreason-qdisc.h                     |  114 +
+ include/net/dropreason.h                           |    6 +
+ include/net/hotdata.h                              |    5 +-
+ include/net/inet6_connection_sock.h                |    4 +-
+ include/net/inet6_hashtables.h                     |    2 +
+ include/net/inet_common.h                          |    3 +-
+ include/net/inet_connection_sock.h                 |    3 +-
+ include/net/inet_hashtables.h                      |    1 -
+ include/net/ip.h                                   |    2 +-
+ include/net/ip6_checksum.h                         |    2 -
+ include/net/ip6_route.h                            |   15 +-
+ include/net/ip6_tunnel.h                           |    2 +-
+ include/net/ip_vs.h                                |  404 +-
+ include/net/ipv6.h                                 |    8 +-
+ include/net/ipv6_stubs.h                           |    7 +-
+ include/net/mac80211.h                             |   19 +-
+ include/net/mana/gdma.h                            |   20 +-
+ include/net/mana/mana.h                            |   17 +-
+ include/net/netdev_rx_queue.h                      |    3 +-
+ include/net/netfilter/nf_tables.h                  |   32 -
+ include/net/netfilter/nft_fib.h                    |    2 -
+ include/net/netfilter/nft_meta.h                   |    3 -
+ include/net/netmem.h                               |   39 +-
+ include/net/netns/ipv4.h                           |    9 +-
+ include/net/netns/ipv6.h                           |    2 +-
+ include/net/netns/mib.h                            |    5 -
+ include/net/netns/vsock.h                          |    2 +
+ include/net/page_pool/types.h                      |   11 +-
+ include/net/ping.h                                 |    5 +-
+ include/net/rps-types.h                            |   24 +
+ include/net/rps.h                                  |   49 +-
+ include/net/sch_generic.h                          |   47 +-
+ include/net/sock.h                                 |   17 +-
+ include/net/switchdev.h                            |    1 +
+ include/net/tc_wrapper.h                           |   47 +-
+ include/net/tcp.h                                  |   88 +-
+ include/net/transp_v6.h                            |    3 -
+ include/net/udp.h                                  |   89 +-
+ include/net/udplite.h                              |   88 -
+ include/net/xsk_buff_pool.h                        |    7 -
+ include/trace/events/devlink.h                     |   36 +-
+ include/trace/events/qdisc.h                       |   51 +
+ include/uapi/linux/devlink.h                       |    4 +
+ include/uapi/linux/ethtool.h                       |    2 +-
+ include/uapi/linux/ethtool_netlink_generated.h     |    3 +
+ include/uapi/linux/if_link.h                       |    2 +
+ include/uapi/linux/if_pppox.h                      |   14 +-
+ include/uapi/linux/inet_diag.h                     |    9 +
+ include/uapi/linux/nfc.h                           |    6 +-
+ include/uapi/linux/nl80211.h                       |   40 +
+ include/uapi/linux/openvswitch.h                   |   76 +-
+ include/uapi/linux/ovpn.h                          |    2 +
+ include/uapi/linux/udp.h                           |    2 +
+ net/atm/pppoatm.c                                  |    2 +-
+ net/bridge/br_private.h                            |   10 +
+ net/bridge/br_switchdev.c                          |   15 +
+ net/bridge/br_vlan.c                               |    7 +-
+ net/bridge/netfilter/nft_meta_bridge.c             |   20 -
+ net/bridge/netfilter/nft_reject_bridge.c           |    1 -
+ net/core/dev.c                                     |  178 +-
+ net/core/failover.c                                |   14 +-
+ net/core/fib_notifier.c                            |    1 +
+ net/core/filter.c                                  |    5 +-
+ net/core/link_watch.c                              |    2 +-
+ net/core/net-sysfs.c                               |   85 +-
+ net/core/net-sysfs.h                               |    1 +
+ net/core/scm.c                                     |   12 +-
+ net/core/skbuff.c                                  |   16 +-
+ net/core/sock.c                                    |   45 +-
+ net/core/sock_diag.c                               |    3 +-
+ net/core/sysctl_net_core.c                         |  117 +-
+ net/devlink/Makefile                               |    2 +-
+ net/devlink/core.c                                 |   91 +-
+ net/devlink/dev.c                                  |    8 +-
+ net/devlink/devl_internal.h                        |   34 +-
+ net/devlink/netlink.c                              |   52 +-
+ net/devlink/netlink_gen.c                          |  355 +-
+ net/devlink/port.c                                 |   19 +-
+ net/devlink/sh_dev.c                               |  161 +
+ net/dns_resolver/dns_key.c                         |   16 +-
+ net/dns_resolver/dns_query.c                       |   14 +-
+ net/dns_resolver/internal.h                        |   14 +-
+ net/ethtool/coalesce.c                             |   14 +-
+ net/ethtool/pause.c                                |    4 +-
+ net/hsr/hsr_device.c                               |    2 +-
+ net/hsr/hsr_framereg.c                             |    4 +-
+ net/hsr/hsr_main.c                                 |    2 +-
+ net/ieee802154/socket.c                            |    6 +-
+ net/ipv4/Makefile                                  |    2 +-
+ net/ipv4/af_inet.c                                 |   33 +-
+ net/ipv4/devinet.c                                 |   55 +-
+ net/ipv4/fou_core.c                                |    3 +-
+ net/ipv4/icmp.c                                    |    5 +-
+ net/ipv4/inet_connection_sock.c                    |    4 +-
+ net/ipv4/inet_diag.c                               |   13 +-
+ net/ipv4/inet_hashtables.c                         |   79 +-
+ net/ipv4/inetpeer.c                                |    3 +-
+ net/ipv4/ip_input.c                                |   39 +
+ net/ipv4/ip_sockglue.c                             |    4 +-
+ net/ipv4/ip_tunnel_core.c                          |    2 +-
+ net/ipv4/ipmr.c                                    |  265 +-
+ net/ipv4/ipmr_base.c                               |    5 +-
+ net/ipv4/netfilter.c                               |    3 +-
+ net/ipv4/netfilter/nft_dup_ipv4.c                  |    1 -
+ net/ipv4/netfilter/nft_fib_ipv4.c                  |    2 -
+ net/ipv4/netfilter/nft_reject_ipv4.c               |    1 -
+ net/ipv4/ping.c                                    |    9 +-
+ net/ipv4/proc.c                                    |   16 -
+ net/ipv4/raw.c                                     |    6 +-
+ net/ipv4/sysctl_net_ipv4.c                         |    7 +
+ net/ipv4/tcp.c                                     |  122 +-
+ net/ipv4/tcp_bbr.c                                 |    1 +
+ net/ipv4/tcp_bpf.c                                 |   17 +-
+ net/ipv4/tcp_dctcp.h                               |    1 +
+ net/ipv4/tcp_diag.c                                |    4 +-
+ net/ipv4/tcp_fastopen.c                            |    1 +
+ net/ipv4/tcp_input.c                               |   67 +-
+ net/ipv4/tcp_ipv4.c                                |   59 +-
+ net/ipv4/tcp_minisocks.c                           |    1 +
+ net/ipv4/tcp_output.c                              |   77 +-
+ net/ipv4/tcp_plb.c                                 |    1 +
+ net/ipv4/udp.c                                     |  443 +-
+ net/ipv4/udp_bpf.c                                 |   18 +-
+ net/ipv4/udp_diag.c                                |  128 +-
+ net/ipv4/udp_impl.h                                |   28 -
+ net/ipv4/udp_offload.c                             |    3 +-
+ net/ipv4/udplite.c                                 |  135 -
+ net/ipv6/Makefile                                  |    2 +-
+ net/ipv6/af_inet6.c                                |   41 +-
+ net/ipv6/datagram.c                                |    9 +-
+ net/ipv6/fib6_notifier.c                           |    1 +
+ net/ipv6/fib6_rules.c                              |    3 +
+ net/ipv6/fou6.c                                    |    3 +-
+ net/ipv6/ila/ila_common.c                          |    1 +
+ net/ipv6/inet6_connection_sock.c                   |   19 +-
+ net/ipv6/inet6_hashtables.c                        |   51 +-
+ net/ipv6/ip6_checksum.c                            |   49 +-
+ net/ipv6/ip6_fib.c                                 |    3 +
+ net/ipv6/ip6_input.c                               |   40 +
+ net/ipv6/ip6mr.c                                   |    4 +-
+ net/ipv6/ipv6_sockglue.c                           |   17 +-
+ net/ipv6/netfilter.c                               |    3 +-
+ net/ipv6/netfilter/nft_dup_ipv6.c                  |    1 -
+ net/ipv6/netfilter/nft_fib_ipv6.c                  |   81 +-
+ net/ipv6/netfilter/nft_reject_ipv6.c               |    1 -
+ net/ipv6/ping.c                                    |    3 +-
+ net/ipv6/proc.c                                    |   16 -
+ net/ipv6/raw.c                                     |    8 +-
+ net/ipv6/reassembly.c                              |    6 +
+ net/ipv6/sit.c                                     |   11 +-
+ net/ipv6/tcp_ipv6.c                                |   64 +-
+ net/ipv6/udp.c                                     |  330 +-
+ net/ipv6/udp_impl.h                                |   32 -
+ net/ipv6/udp_offload.c                             |    3 +-
+ net/ipv6/udplite.c                                 |  139 -
+ net/l2tp/l2tp_eth.c                                |    5 +-
+ net/l2tp/l2tp_ip.c                                 |    4 +-
+ net/l2tp/l2tp_ip6.c                                |    6 +-
+ net/l2tp/l2tp_ppp.c                                |   15 +-
+ net/mac80211/agg-rx.c                              |   49 +-
+ net/mac80211/agg-tx.c                              |   39 +-
+ net/mac80211/cfg.c                                 |   99 +-
+ net/mac80211/chan.c                                |   90 +-
+ net/mac80211/debugfs.c                             |    1 +
+ net/mac80211/eht.c                                 |   21 +-
+ net/mac80211/ht.c                                  |   37 +-
+ net/mac80211/ibss.c                                |   18 +-
+ net/mac80211/ieee80211_i.h                         |    8 +-
+ net/mac80211/iface.c                               |   21 +-
+ net/mac80211/main.c                                |   28 +-
+ net/mac80211/mesh.c                                |   14 +-
+ net/mac80211/mesh_hwmp.c                           |   20 +-
+ net/mac80211/mesh_plink.c                          |   21 +-
+ net/mac80211/mlme.c                                |  194 +-
+ net/mac80211/rx.c                                  |  139 +-
+ net/mac80211/s1g.c                                 |   36 +-
+ net/mac80211/spectmgmt.c                           |   31 +-
+ net/mac80211/sta_info.c                            |    4 +-
+ net/mac80211/sta_info.h                            |    3 +-
+ net/mac80211/tdls.c                                |   29 +-
+ net/mac80211/trace.h                               |   94 +-
+ net/mac80211/tx.c                                  |   77 +-
+ net/mac80211/util.c                                |    5 +-
+ net/mac80211/vht.c                                 |   10 +-
+ net/mac80211/wpa.c                                 |   55 +-
+ net/mpls/af_mpls.c                                 |    3 -
+ net/mptcp/options.c                                |    6 +-
+ net/mptcp/protocol.c                               |    4 +-
+ net/mptcp/subflow.c                                |    1 -
+ net/netfilter/core.c                               |    3 +-
+ net/netfilter/ipvs/ip_vs_conn.c                    | 1055 +++--
+ net/netfilter/ipvs/ip_vs_core.c                    |  181 +-
+ net/netfilter/ipvs/ip_vs_ctl.c                     |  981 +++--
+ net/netfilter/ipvs/ip_vs_est.c                     |   18 +-
+ net/netfilter/ipvs/ip_vs_pe_sip.c                  |    4 +-
+ net/netfilter/ipvs/ip_vs_sync.c                    |   23 +
+ net/netfilter/ipvs/ip_vs_xmit.c                    |   12 +-
+ net/netfilter/nf_conntrack_netlink.c               |    4 +-
+ net/netfilter/nf_flow_table_offload.c              |    1 +
+ net/netfilter/nf_log_syslog.c                      |   16 +-
+ net/netfilter/nf_queue.c                           |    1 +
+ net/netfilter/nf_tables_api.c                      |   78 -
+ net/netfilter/nfnetlink.c                          |    4 +-
+ net/netfilter/nfnetlink_log.c                      |   19 +-
+ net/netfilter/nfnetlink_queue.c                    |   24 +-
+ net/netfilter/nft_bitwise.c                        |  104 -
+ net/netfilter/nft_byteorder.c                      |   11 -
+ net/netfilter/nft_chain_filter.c                   |    1 +
+ net/netfilter/nft_cmp.c                            |    3 -
+ net/netfilter/nft_compat.c                         |   10 -
+ net/netfilter/nft_connlimit.c                      |    1 -
+ net/netfilter/nft_counter.c                        |    1 -
+ net/netfilter/nft_ct.c                             |   46 -
+ net/netfilter/nft_dup_netdev.c                     |    1 -
+ net/netfilter/nft_dynset.c                         |    1 -
+ net/netfilter/nft_exthdr.c                         |   34 -
+ net/netfilter/nft_fib.c                            |   42 -
+ net/netfilter/nft_fib_inet.c                       |    1 -
+ net/netfilter/nft_fib_netdev.c                     |    1 -
+ net/netfilter/nft_flow_offload.c                   |    1 -
+ net/netfilter/nft_fwd_netdev.c                     |    2 -
+ net/netfilter/nft_hash.c                           |   36 -
+ net/netfilter/nft_immediate.c                      |   12 -
+ net/netfilter/nft_last.c                           |    1 -
+ net/netfilter/nft_limit.c                          |    2 -
+ net/netfilter/nft_log.c                            |    1 -
+ net/netfilter/nft_lookup.c                         |   12 -
+ net/netfilter/nft_masq.c                           |    3 -
+ net/netfilter/nft_meta.c                           |   68 +-
+ net/netfilter/nft_nat.c                            |    2 -
+ net/netfilter/nft_numgen.c                         |   22 -
+ net/netfilter/nft_objref.c                         |    2 -
+ net/netfilter/nft_osf.c                            |   25 -
+ net/netfilter/nft_payload.c                        |   47 -
+ net/netfilter/nft_queue.c                          |    2 -
+ net/netfilter/nft_quota.c                          |    1 -
+ net/netfilter/nft_range.c                          |    1 -
+ net/netfilter/nft_redir.c                          |    3 -
+ net/netfilter/nft_reject_inet.c                    |    1 -
+ net/netfilter/nft_reject_netdev.c                  |    1 -
+ net/netfilter/nft_rt.c                             |    1 -
+ net/netfilter/nft_set_rbtree.c                     |   23 +-
+ net/netfilter/nft_socket.c                         |   26 -
+ net/netfilter/nft_synproxy.c                       |    1 -
+ net/netfilter/nft_tproxy.c                         |    1 -
+ net/netfilter/nft_tunnel.c                         |   26 -
+ net/netfilter/nft_xfrm.c                           |   27 -
+ net/netfilter/xt_connbytes.c                       |    3 +-
+ net/netfilter/xt_connlimit.c                       |    3 +-
+ net/netfilter/xt_owner.c                           |   28 +-
+ net/netfilter/xt_time.c                            |    3 +-
+ net/netlink/af_netlink.c                           |    4 +-
+ net/netlink/genetlink.c                            |   20 +-
+ net/netlink/policy.c                               |    4 +-
+ net/phonet/datagram.c                              |    4 +-
+ net/phonet/pep.c                                   |    2 +-
+ net/psp/psp_main.c                                 |    2 +-
+ net/rds/send.c                                     |    2 +-
+ net/rxrpc/output.c                                 |    2 -
+ net/sched/cls_flower.c                             |   12 +-
+ net/sched/cls_u32.c                                |    5 +-
+ net/sched/sch_api.c                                |    5 +-
+ net/sched/sch_cake.c                               |   26 +-
+ net/sched/sch_codel.c                              |    5 +-
+ net/sched/sch_dualpi2.c                            |   18 +-
+ net/sched/sch_fq.c                                 |   10 +-
+ net/sched/sch_fq_codel.c                           |    4 +-
+ net/sched/sch_fq_pie.c                             |    4 +-
+ net/sched/sch_generic.c                            |   51 +-
+ net/sched/sch_gred.c                               |    4 +-
+ net/sched/sch_htb.c                                |    4 +-
+ net/sched/sch_mq.c                                 |    2 +-
+ net/sched/sch_mqprio.c                             |    2 +-
+ net/sched/sch_pie.c                                |    4 +-
+ net/sched/sch_red.c                                |    4 +-
+ net/sched/sch_sfb.c                                |    4 +-
+ net/sched/sch_sfq.c                                |    8 +-
+ net/sched/sch_taprio.c                             |    2 +-
+ net/sctp/socket.c                                  |   12 +-
+ net/socket.c                                       |   51 +-
+ net/switchdev/switchdev.c                          |    2 +-
+ net/tls/Kconfig                                    |    1 -
+ net/tls/tls.h                                      |    2 +-
+ net/tls/tls_device_fallback.c                      |    3 -
+ net/tls/tls_sw.c                                   |    3 +-
+ net/unix/af_unix.c                                 |    4 +-
+ net/unix/unix_bpf.c                                |    2 +-
+ net/vmw_vsock/af_vsock.c                           |   39 +-
+ net/vmw_vsock/virtio_transport.c                   |    7 +
+ net/vmw_vsock/vsock_bpf.c                          |    2 +-
+ net/wireless/chan.c                                |  139 +-
+ net/wireless/core.c                                |    1 -
+ net/wireless/core.h                                |    4 +
+ net/wireless/ibss.c                                |    4 +-
+ net/wireless/mlme.c                                |   51 +-
+ net/wireless/nl80211.c                             |  316 +-
+ net/wireless/nl80211.h                             |    5 +-
+ net/wireless/of.c                                  |   13 +-
+ net/wireless/pmsr.c                                |    5 +-
+ net/wireless/radiotap.c                            |   10 +-
+ net/wireless/rdev-ops.h                            |   62 +-
+ net/wireless/reg.c                                 |   13 +-
+ net/wireless/reg.h                                 |   13 +-
+ net/wireless/sme.c                                 |    4 +-
+ net/wireless/sysfs.c                               |   33 +-
+ net/wireless/trace.c                               |    1 +
+ net/wireless/trace.h                               |  185 +-
+ net/wireless/util.c                                |    4 +-
+ net/wireless/wext-compat.c                         |   12 +-
+ net/wireless/wext-core.c                           |    3 +-
+ net/wireless/wext-priv.c                           |    3 +-
+ net/wireless/wext-proc.c                           |    3 +-
+ net/xdp/xsk.h                                      |    7 -
+ net/xfrm/espintcp.c                                |    2 +-
+ security/smack/smack_lsm.c                         |    4 +-
+ tools/net/ynl/Makefile                             |    4 +-
+ tools/net/ynl/pyynl/cli.py                         |   14 +-
+ tools/net/ynl/pyynl/lib/__init__.py                |    5 +-
+ tools/net/ynl/pyynl/lib/ynl.py                     |  279 +-
+ tools/net/ynl/samples/Makefile                     |   36 -
+ tools/net/ynl/samples/devlink.c                    |   61 -
+ tools/net/ynl/samples/ethtool.c                    |   65 -
+ tools/net/ynl/samples/netdev.c                     |  128 -
+ tools/net/ynl/samples/ovs.c                        |   60 -
+ tools/net/ynl/samples/rt-addr.c                    |   80 -
+ tools/net/ynl/samples/rt-link.c                    |  184 -
+ tools/net/ynl/samples/rt-route.c                   |   80 -
+ tools/net/ynl/samples/tc-filter-add.c              |  335 --
+ tools/net/ynl/samples/tc.c                         |   80 -
+ tools/net/ynl/{samples => tests}/.gitignore        |    3 +-
+ tools/net/ynl/tests/Makefile                       |   94 +-
+ tools/net/ynl/tests/config                         |    8 +
+ tools/net/ynl/tests/devlink.c                      |  101 +
+ tools/net/ynl/tests/devlink.sh                     |    5 +
+ tools/net/ynl/tests/ethtool.c                      |   92 +
+ tools/net/ynl/tests/ethtool.sh                     |    5 +
+ tools/net/ynl/tests/netdev.c                       |  231 +
+ tools/net/ynl/tests/ovs.c                          |  108 +
+ tools/net/ynl/tests/rt-addr.c                      |  111 +
+ tools/net/ynl/tests/rt-addr.sh                     |    5 +
+ tools/net/ynl/tests/rt-link.c                      |  206 +
+ tools/net/ynl/tests/rt-route.c                     |  113 +
+ tools/net/ynl/tests/rt-route.sh                    |    5 +
+ tools/net/ynl/tests/tc.c                           |  409 ++
+ tools/net/ynl/tests/ynl_nsim_lib.sh                |   35 +
+ tools/testing/selftests/drivers/net/.gitignore     |    1 -
+ tools/testing/selftests/drivers/net/Makefile       |    1 -
+ tools/testing/selftests/drivers/net/README.rst     |  128 +-
+ tools/testing/selftests/drivers/net/gro.py         |  203 +-
+ tools/testing/selftests/drivers/net/hw/Makefile    |    2 +
+ tools/testing/selftests/drivers/net/hw/config      |    3 +
+ tools/testing/selftests/drivers/net/hw/gro_hw.py   |  294 ++
+ tools/testing/selftests/drivers/net/hw/iou-zcrx.py |   58 +-
+ .../selftests/drivers/net/hw/lib/py/__init__.py    |   12 +-
+ .../selftests/drivers/net/hw/nk_forward.bpf.c      |   49 +
+ tools/testing/selftests/drivers/net/hw/nk_netns.py |   29 +
+ tools/testing/selftests/drivers/net/hw/rss_ctx.py  |   49 +-
+ tools/testing/selftests/drivers/net/hw/tso.py      |    7 +-
+ .../selftests/drivers/net/lib/py/__init__.py       |   15 +-
+ tools/testing/selftests/drivers/net/lib/py/env.py  |  211 +-
+ .../drivers/net/netconsole/netcons_basic.sh        |    6 +-
+ tools/testing/selftests/net/Makefile               |    4 +
+ tools/testing/selftests/net/af_unix/so_peek_off.c  |   46 +
+ tools/testing/selftests/net/config                 |    2 +
+ tools/testing/selftests/net/fib_tests.sh           |   19 +-
+ tools/testing/selftests/net/forwarding/.gitignore  |    1 +
+ tools/testing/selftests/net/forwarding/Makefile    |    4 +
+ .../selftests/net/forwarding/gre_multipath.sh      |    2 +-
+ .../selftests/net/forwarding/gre_multipath_nh.sh   |    2 +-
+ .../net/forwarding/gre_multipath_nh_res.sh         |    2 +-
+ tools/testing/selftests/net/forwarding/ipip_lib.sh |    2 +-
+ tools/testing/selftests/net/forwarding/ipmr.c      |  455 ++
+ .../selftests/net/forwarding/local_termination.sh  |   25 +
+ tools/testing/selftests/net/ip6_tunnel.sh          |   44 +
+ tools/testing/selftests/net/lib/.gitignore         |    1 +
+ tools/testing/selftests/net/lib/Makefile           |    1 +
+ .../selftests/{drivers/net => net/lib}/gro.c       |  252 +-
+ tools/testing/selftests/net/lib/py/__init__.py     |   12 +-
+ tools/testing/selftests/net/lib/py/utils.py        |   75 +-
+ tools/testing/selftests/net/lib/py/ynl.py          |   18 +-
+ .../selftests/net/macvlan_mcast_shared_mac.sh      |   93 +
+ tools/testing/selftests/net/nl_netdev.py           |   32 +-
+ tools/testing/selftests/net/nl_nlctrl.py           |  131 +
+ .../testing/selftests/net/openvswitch/ovs-dpctl.py |    2 +-
+ tools/testing/selftests/net/ovpn/Makefile          |   31 +-
+ tools/testing/selftests/net/ovpn/common.sh         |  101 +-
+ tools/testing/selftests/net/ovpn/data64.key        |    6 +-
+ .../selftests/net/ovpn/json/peer0-float.json       |    9 +
+ .../selftests/net/ovpn/json/peer0-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer0-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer0.json   |    6 +
+ .../selftests/net/ovpn/json/peer1-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer1-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer1-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer1.json   |    1 +
+ .../selftests/net/ovpn/json/peer2-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer2-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer2-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer2.json   |    1 +
+ .../selftests/net/ovpn/json/peer3-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer3-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer3-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer3.json   |    1 +
+ .../selftests/net/ovpn/json/peer4-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer4-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer4-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer4.json   |    1 +
+ .../selftests/net/ovpn/json/peer5-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer5-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer5-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer5.json   |    1 +
+ .../selftests/net/ovpn/json/peer6-float.json       |    1 +
+ .../selftests/net/ovpn/json/peer6-symm-float.json  |    1 +
+ .../selftests/net/ovpn/json/peer6-symm.json        |    1 +
+ tools/testing/selftests/net/ovpn/json/peer6.json   |    1 +
+ tools/testing/selftests/net/ovpn/ovpn-cli.c        |  152 +-
+ tools/testing/selftests/net/ovpn/tcp_peers.txt     |   11 +-
+ .../selftests/net/ovpn/test-close-socket.sh        |    2 +-
+ tools/testing/selftests/net/ovpn/test-mark.sh      |   96 +
+ .../selftests/net/ovpn/test-symmetric-id-float.sh  |   11 +
+ .../selftests/net/ovpn/test-symmetric-id-tcp.sh    |   11 +
+ .../selftests/net/ovpn/test-symmetric-id.sh        |   10 +
+ tools/testing/selftests/net/ovpn/test.sh           |   76 +-
+ tools/testing/selftests/net/ovpn/udp_peers.txt     |   12 +-
+ .../packetdrill/tcp_disorder_fin_in_FIN_WAIT.pkt   |   33 +
+ .../net/packetdrill/tcp_rcv_big_endseq.pkt         |    2 +-
+ .../net/packetdrill/tcp_rcv_neg_window.pkt         |   34 +
+ .../net/packetdrill/tcp_rcv_wnd_shrink_allowed.pkt |   40 +
+ .../net/packetdrill/tcp_rcv_wnd_shrink_nomem.pkt   |  132 +
+ tools/testing/selftests/net/rds/Makefile           |    1 +
+ tools/testing/selftests/net/rds/run.sh             |    7 +-
+ tools/testing/selftests/net/rds/settings           |    1 +
+ tools/testing/selftests/net/rds/test.py            |  108 +-
+ tools/testing/selftests/net/xfrm_state.sh          |  613 +++
+ .../tc-testing/tc-tests/infra/qdiscs.json          |    2 +-
+ tools/testing/selftests/tc-testing/tdc_helper.py   |    4 +-
+ tools/testing/selftests/vsock/vmtest.sh            |   19 +
+ 880 files changed, 31992 insertions(+), 12003 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/net/spacemit,k3-dwmac.yaml
+ create mode 100644 Documentation/networking/devlink/devlink-shared.rst
+ create mode 100644 drivers/dpll/zl3073x/chan.c
+ create mode 100644 drivers/dpll/zl3073x/chan.h
+ create mode 100644 drivers/net/dsa/mv88e6xxx/tcam.c
+ create mode 100644 drivers/net/dsa/mv88e6xxx/tcam.h
+ create mode 100644 drivers/net/dsa/mv88e6xxx/tcflower.c
+ create mode 100644 drivers/net/dsa/mv88e6xxx/tcflower.h
+ create mode 100644 drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c
+ create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
+ create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
+ create mode 100644 drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
+ create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h
+ create mode 100644 drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c
+ delete mode 100644 drivers/net/phy/mdio-private.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/thermal.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/thermal.h
+ delete mode 100644 include/linux/mdio-gpio.h
+ create mode 100644 include/linux/mlx5/lag.h
+ delete mode 100644 include/linux/of_gpio.h
+ delete mode 100644 include/linux/platform_data/mdio-gpio.h
+ create mode 100644 include/net/dropreason-qdisc.h
+ create mode 100644 include/net/rps-types.h
+ delete mode 100644 include/net/udplite.h
+ create mode 100644 net/devlink/sh_dev.c
+ delete mode 100644 net/ipv4/udp_impl.h
+ delete mode 100644 net/ipv4/udplite.c
+ delete mode 100644 net/ipv6/udp_impl.h
+ delete mode 100644 net/ipv6/udplite.c
+ delete mode 100644 tools/net/ynl/samples/Makefile
+ delete mode 100644 tools/net/ynl/samples/devlink.c
+ delete mode 100644 tools/net/ynl/samples/ethtool.c
+ delete mode 100644 tools/net/ynl/samples/netdev.c
+ delete mode 100644 tools/net/ynl/samples/ovs.c
+ delete mode 100644 tools/net/ynl/samples/rt-addr.c
+ delete mode 100644 tools/net/ynl/samples/rt-link.c
+ delete mode 100644 tools/net/ynl/samples/rt-route.c
+ delete mode 100644 tools/net/ynl/samples/tc-filter-add.c
+ delete mode 100644 tools/net/ynl/samples/tc.c
+ rename tools/net/ynl/{samples => tests}/.gitignore (87%)
+ create mode 100644 tools/net/ynl/tests/devlink.c
+ create mode 100755 tools/net/ynl/tests/devlink.sh
+ create mode 100644 tools/net/ynl/tests/ethtool.c
+ create mode 100755 tools/net/ynl/tests/ethtool.sh
+ create mode 100644 tools/net/ynl/tests/netdev.c
+ create mode 100644 tools/net/ynl/tests/ovs.c
+ create mode 100644 tools/net/ynl/tests/rt-addr.c
+ create mode 100755 tools/net/ynl/tests/rt-addr.sh
+ create mode 100644 tools/net/ynl/tests/rt-link.c
+ create mode 100644 tools/net/ynl/tests/rt-route.c
+ create mode 100755 tools/net/ynl/tests/rt-route.sh
+ create mode 100644 tools/net/ynl/tests/tc.c
+ create mode 100644 tools/net/ynl/tests/ynl_nsim_lib.sh
+ create mode 100755 tools/testing/selftests/drivers/net/hw/gro_hw.py
+ create mode 100644 tools/testing/selftests/drivers/net/hw/nk_forward.bpf.c
+ create mode 100755 tools/testing/selftests/drivers/net/hw/nk_netns.py
+ create mode 100644 tools/testing/selftests/net/forwarding/ipmr.c
+ create mode 100755 tools/testing/selftests/net/ip6_tunnel.sh
+ rename tools/testing/selftests/{drivers/net => net/lib}/gro.c (86%)
+ create mode 100755 tools/testing/selftests/net/macvlan_mcast_shared_mac.sh
+ create mode 100755 tools/testing/selftests/net/nl_nlctrl.py
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer0-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer0-symm-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer0-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer0.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer1-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer1-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer1-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer1.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer2-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer2-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer2-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer2.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer3-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer3-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer3-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer3.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer4-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer4-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer4-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer4.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer5-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer5-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer5-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer5.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer6-float.json
+ create mode 120000 tools/testing/selftests/net/ovpn/json/peer6-symm-float.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer6-symm.json
+ create mode 100644 tools/testing/selftests/net/ovpn/json/peer6.json
+ create mode 100755 tools/testing/selftests/net/ovpn/test-mark.sh
+ create mode 100755 tools/testing/selftests/net/ovpn/test-symmetric-id-float.sh
+ create mode 100755 tools/testing/selftests/net/ovpn/test-symmetric-id-tcp.sh
+ create mode 100755 tools/testing/selftests/net/ovpn/test-symmetric-id.sh
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_disorder_fin_in_FIN_WAIT.pkt
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_rcv_neg_window.pkt
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_rcv_wnd_shrink_allowed.pkt
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_rcv_wnd_shrink_nomem.pkt
+ create mode 100644 tools/testing/selftests/net/rds/settings
+ create mode 100755 tools/testing/selftests/net/xfrm_state.sh
+$ git am -3 ../patches/0001-net-mlx5e-Fix-up-merge.patch
+Applying: net/mlx5e: Fix up merge
+Using index info to reconstruct a base tree...
+M	drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging bpf-next/for-next (4b21ea5024830 bpf: Add warning to detect memory leak in bpf_selem_unlink_nofail())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging kernel/bpf/btf.c
+Auto-merging kernel/bpf/core.c
+Auto-merging kernel/bpf/verifier.c
+Auto-merging tools/testing/selftests/bpf/Makefile
+Merge made by the 'ort' strategy.
+ arch/arm64/net/bpf_jit.h                           |   4 +-
+ arch/s390/kernel/Makefile                          |   2 +
+ arch/s390/kernel/bpf.c                             |  12 +
+ arch/s390/net/bpf_jit_comp.c                       | 157 +++-
+ drivers/net/ethernet/netronome/nfp/bpf/jit.c       |  18 +-
+ drivers/net/ethernet/netronome/nfp/bpf/verifier.c  |  12 +-
+ include/linux/bpf_verifier.h                       |  10 +-
+ kernel/bpf/bpf_local_storage.c                     |  11 +
+ kernel/bpf/btf.c                                   |   7 +-
+ kernel/bpf/core.c                                  |   4 +-
+ kernel/bpf/cpumap.c                                |   5 +-
+ kernel/bpf/helpers.c                               |   3 +
+ kernel/bpf/log.c                                   |  10 +-
+ kernel/bpf/verifier.c                              | 981 ++++++++++++---------
+ net/bpf/test_run.c                                 |  15 +
+ net/xdp/xsk.c                                      |  18 +-
+ net/xdp/xsk_queue.h                                |  16 +-
+ tools/bpf/bpftool/Documentation/bpftool-btf.rst    |  11 +-
+ tools/bpf/bpftool/Makefile                         |  30 +-
+ tools/bpf/bpftool/bash-completion/bpftool          |   6 +-
+ tools/bpf/bpftool/btf.c                            | 121 ++-
+ tools/bpf/bpftool/main.c                           |   7 +
+ tools/bpf/bpftool/main.h                           |  14 +
+ tools/lib/bpf/btf.c                                |  27 +-
+ tools/lib/bpf/features.c                           |  24 +
+ tools/lib/bpf/libbpf.c                             |  19 +-
+ tools/lib/bpf/libbpf.map                           |   3 +
+ tools/lib/bpf/libbpf_internal.h                    |   2 +
+ tools/lib/bpf/libbpf_version.h                     |   2 +-
+ tools/lib/bpf/usdt.c                               |  47 +-
+ tools/testing/selftests/bpf/.gitignore             |   2 +
+ tools/testing/selftests/bpf/Makefile               |  17 +-
+ tools/testing/selftests/bpf/bench.c                |   4 +
+ tools/testing/selftests/bpf/benchs/bench_trigger.c |  60 ++
+ .../selftests/bpf/benchs/run_bench_uprobes.sh      |   2 +-
+ tools/testing/selftests/bpf/bpf_experimental.h     |  18 +
+ tools/testing/selftests/bpf/bpftool_helpers.c      |  15 +-
+ tools/testing/selftests/bpf/cgroup_iter_memcg.h    |   2 -
+ tools/testing/selftests/bpf/prog_tests/align.c     | 712 ---------------
+ .../testing/selftests/bpf/prog_tests/bpf_cookie.c  |  29 +-
+ .../selftests/bpf/prog_tests/bpf_insn_array.c      |   2 +-
+ tools/testing/selftests/bpf/prog_tests/btf_write.c | 111 +++
+ .../selftests/bpf/prog_tests/cgroup_iter_memcg.c   |  28 -
+ .../testing/selftests/bpf/prog_tests/exceptions.c  |   1 +
+ .../selftests/bpf/prog_tests/fexit_bpf2bpf.c       |  26 +-
+ .../selftests/bpf/prog_tests/get_func_args_test.c  |  25 +-
+ .../selftests/bpf/prog_tests/get_func_ip_test.c    |  28 +-
+ .../testing/selftests/bpf/prog_tests/kfunc_call.c  |   2 +
+ .../selftests/bpf/prog_tests/kprobe_multi_test.c   |  33 +-
+ .../testing/selftests/bpf/prog_tests/linked_list.c |   4 +-
+ .../bpf/prog_tests/livepatch_trampoline.c          |  20 +-
+ tools/testing/selftests/bpf/prog_tests/lwt_misc.c  |   9 +
+ .../selftests/bpf/prog_tests/modify_return.c       |   8 +-
+ .../selftests/bpf/prog_tests/module_attach.c       | 191 ++--
+ tools/testing/selftests/bpf/prog_tests/perf_link.c |  20 +-
+ .../testing/selftests/bpf/prog_tests/probe_user.c  |  29 +-
+ tools/testing/selftests/bpf/prog_tests/rbtree.c    |   6 +
+ .../selftests/bpf/prog_tests/task_local_data.h     |   2 +-
+ .../selftests/bpf/prog_tests/task_local_storage.c  |  16 +-
+ .../selftests/bpf/prog_tests/test_bpf_smc.c        |   6 +-
+ .../selftests/bpf/prog_tests/test_global_funcs.c   |   2 +
+ tools/testing/selftests/bpf/prog_tests/usdt.c      |  92 ++
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |   2 +
+ tools/testing/selftests/bpf/progs/bpf_misc.h       |   4 +
+ tools/testing/selftests/bpf/progs/bpf_smc.c        |  28 +-
+ .../selftests/bpf/progs/cgroup_iter_memcg.c        |  18 +-
+ tools/testing/selftests/bpf/progs/exceptions.c     |  14 +
+ .../selftests/bpf/progs/exceptions_assert.c        |   2 +-
+ .../testing/selftests/bpf/progs/exceptions_fail.c  |  35 +-
+ .../selftests/bpf/progs/freplace_int_with_void.c   |  11 +
+ tools/testing/selftests/bpf/progs/freplace_void.c  |  10 +
+ .../bpf/progs/get_func_args_fsession_test.c        |  37 +
+ .../selftests/bpf/progs/get_func_args_test.c       |  38 -
+ .../bpf/progs/get_func_ip_fsession_test.c          |  21 +
+ .../testing/selftests/bpf/progs/get_func_ip_test.c |  23 -
+ tools/testing/selftests/bpf/progs/iters.c          |   6 +-
+ .../testing/selftests/bpf/progs/kfunc_call_test.c  |  98 ++
+ .../selftests/bpf/progs/kprobe_multi_session.c     |  10 +
+ tools/testing/selftests/bpf/progs/lwt_misc.c       |  22 +
+ .../selftests/bpf/progs/mem_rdonly_untrusted.c     |   2 +-
+ tools/testing/selftests/bpf/progs/modify_return.c  |  13 +
+ .../selftests/bpf/progs/rbtree_search_kptr.c       | 290 ++++++
+ .../testing/selftests/bpf/progs/refcounted_kptr.c  |   4 +-
+ .../selftests/bpf/progs/refcounted_kptr_fail.c     |   2 +-
+ tools/testing/selftests/bpf/progs/tailcall3.c      |   5 +-
+ .../selftests/bpf/progs/task_local_data.bpf.h      |   2 +-
+ .../selftests/bpf/progs/test_global_func3.c        |  18 +-
+ .../selftests/bpf/progs/test_global_func7.c        |   2 +-
+ .../bpf/progs/test_global_func_deep_stack.c        |  95 ++
+ .../selftests/bpf/progs/test_module_attach.c       |  63 +-
+ .../testing/selftests/bpf/progs/test_probe_user.c  |  13 +-
+ tools/testing/selftests/bpf/progs/test_usdt.c      |  12 +
+ tools/testing/selftests/bpf/progs/trigger_bench.c  |  10 +-
+ tools/testing/selftests/bpf/progs/verifier_align.c | 581 ++++++++++++
+ .../testing/selftests/bpf/progs/verifier_bounds.c  |  35 +-
+ tools/testing/selftests/bpf/progs/verifier_ctx.c   |  76 ++
+ .../bpf/progs/verifier_direct_packet_access.c      |   4 +-
+ .../selftests/bpf/progs/verifier_global_subprogs.c |  19 +
+ tools/testing/selftests/bpf/progs/verifier_gotox.c |   4 +-
+ .../bpf/progs/verifier_helper_packet_access.c      |   2 +-
+ .../bpf/progs/verifier_helper_value_access.c       |   4 +-
+ .../testing/selftests/bpf/progs/verifier_int_ptr.c |   2 +-
+ .../bpf/progs/verifier_jeq_infer_not_null.c        |  54 ++
+ .../selftests/bpf/progs/verifier_meta_access.c     |   2 +-
+ .../selftests/bpf/progs/verifier_private_stack.c   |   8 +-
+ .../selftests/bpf/progs/verifier_spill_fill.c      |   8 +-
+ .../selftests/bpf/progs/verifier_stack_ptr.c       |   4 +-
+ .../selftests/bpf/progs/verifier_value_ptr_arith.c |  10 +-
+ .../bpf/progs/verifier_xdp_direct_packet_access.c  |  64 +-
+ tools/testing/selftests/bpf/test_bpftool.py        | 174 ----
+ tools/testing/selftests/bpf/test_bpftool.sh        |  11 -
+ .../testing/selftests/bpf/test_kmods/bpf_testmod.c |  54 +-
+ .../selftests/bpf/test_kmods/bpf_testmod_kfunc.h   |   1 +
+ tools/testing/selftests/bpf/testing_helpers.c      |   9 +-
+ tools/testing/selftests/bpf/testing_helpers.h      |   1 +
+ tools/testing/selftests/bpf/uprobe_multi.c         |  19 +-
+ tools/testing/selftests/bpf/uprobe_multi.ld        |   4 +-
+ tools/testing/selftests/bpf/usdt.h                 |   2 +
+ tools/testing/selftests/bpf/usdt_1.c               |  18 +
+ tools/testing/selftests/bpf/usdt_2.c               |  16 +
+ tools/testing/selftests/bpf/verifier/calls.c       |   2 +-
+ 121 files changed, 3441 insertions(+), 1777 deletions(-)
+ create mode 100644 arch/s390/kernel/bpf.c
+ delete mode 100644 tools/testing/selftests/bpf/prog_tests/align.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/lwt_misc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/freplace_int_with_void.c
+ create mode 100644 tools/testing/selftests/bpf/progs/freplace_void.c
+ create mode 100644 tools/testing/selftests/bpf/progs/get_func_args_fsession_test.c
+ create mode 100644 tools/testing/selftests/bpf/progs/get_func_ip_fsession_test.c
+ create mode 100644 tools/testing/selftests/bpf/progs/lwt_misc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/rbtree_search_kptr.c
+ create mode 100644 tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_align.c
+ delete mode 100644 tools/testing/selftests/bpf/test_bpftool.py
+ delete mode 100755 tools/testing/selftests/bpf/test_bpftool.sh
+ create mode 100644 tools/testing/selftests/bpf/usdt_1.c
+ create mode 100644 tools/testing/selftests/bpf/usdt_2.c
+Merging ipsec-next/master (be14d13625c9b xfrm: update outdated comment)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Auto-merging net/xfrm/xfrm_policy.c
+Merge made by the 'ort' strategy.
+ net/xfrm/xfrm_policy.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging mlx5-next/mlx5-next (4dd2115f43594 net/mlx5: Expose MLX5_UMR_ALIGN definition)
+$ git merge -m Merge branch 'mlx5-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Already up to date.
+Merging netfilter-next/main (9ac76f3d0bb29 Merge tag 'wireless-next-2026-03-19' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (4686679a14d26 selftests/net: packetdrill: add tcp_disorder_fin_in_FIN_WAIT.pkt)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (eae2412ea6f80 Bluetooth: L2CAP: Fix regressions caused by reusing ident)
+  0f1a322270bb6 ("Bluetooth: hci_ll: Fix firmware leak on error path")
+  3c2dbfa16ca67 ("Bluetooth: MGMT: Fix dangling pointer on mgmt_add_adv_patterns_monitor_complete")
+  506c028241313 ("Bluetooth: SCO: Fix use-after-free in sco_recv_frame() due to missing sock_hold")
+  6ec1f2e822b22 ("Bluetooth: L2CAP: Fix stack-out-of-bounds read in l2cap_ecred_conn_req")
+  9adcc51be5497 ("Bluetooth: hci_sync: annotate data-races around hdev->req_status")
+  a480f2bbc7c2b ("Bluetooth: L2CAP: Validate PDU length before reading SDU length in l2cap_ecred_data_rcv()")
+  bc280a5ed9efb ("Bluetooth: L2CAP: Fix null-ptr-deref on l2cap_sock_ready_cb")
+  eae2412ea6f80 ("Bluetooth: L2CAP: Fix regressions caused by reusing ident")
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Auto-merging drivers/bluetooth/btqca.c
+Auto-merging net/bluetooth/hci_core.c
+Auto-merging net/bluetooth/l2cap_core.c
+CONFLICT (content): Merge conflict in net/bluetooth/l2cap_core.c
+Resolved 'net/bluetooth/l2cap_core.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master d11a8b6ca7f3d] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/bluetooth/btbcm.c        |  11 ++--
+ drivers/bluetooth/btintel.c      | 109 +++++++++++++++++++++++++++++++++------
+ drivers/bluetooth/btintel.h      |  20 +++++--
+ drivers/bluetooth/btintel_pcie.c |  24 +++++++--
+ drivers/bluetooth/btmtk.c        |  23 ++++++++-
+ drivers/bluetooth/btmtk.h        |   2 +
+ drivers/bluetooth/btmtksdio.c    |  42 ++++++++++-----
+ drivers/bluetooth/btqca.c        |  37 ++++++-------
+ drivers/bluetooth/btusb.c        |  19 ++++++-
+ drivers/bluetooth/hci_qca.c      |  77 +++++++++++++++------------
+ include/linux/mmc/sdio_ids.h     |   1 +
+ net/bluetooth/hci_core.c         |   4 +-
+ net/bluetooth/l2cap_core.c       |   9 +++-
+ 13 files changed, 276 insertions(+), 102 deletions(-)
+Merging wireless-next/for-next (9ac76f3d0bb29 Merge tag 'wireless-next-2026-03-19' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Already up to date.
+Merging ath-next/for-next (7bbb578fc43e7 wifi: ath12k: Remove unused DFS Unit Test definitions)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath-next/for-next
+Auto-merging drivers/net/wireless/ath/ath12k/mac.c
+Auto-merging drivers/net/wireless/ath/ath12k/wmi.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath10k/usb.c          |  8 +--
+ drivers/net/wireless/ath/ath12k/ahb.h          |  4 +-
+ drivers/net/wireless/ath/ath12k/core.h         |  2 +-
+ drivers/net/wireless/ath/ath12k/dp_htt.c       | 24 ++++----
+ drivers/net/wireless/ath/ath12k/hal.h          | 31 +++++++----
+ drivers/net/wireless/ath/ath12k/mac.c          | 67 +++++++++++++++--------
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c | 76 ++++++++++++++++++++------
+ drivers/net/wireless/ath/ath12k/wmi.c          | 58 +++++++++-----------
+ drivers/net/wireless/ath/ath12k/wmi.h          | 14 +++--
+ drivers/net/wireless/ath/ath6kl/usb.c          | 16 ++----
+ drivers/net/wireless/ath/ath6kl/wmi.h          | 11 ----
+ drivers/net/wireless/ath/ath9k/hif_usb.c       |  4 --
+ 12 files changed, 180 insertions(+), 135 deletions(-)
+Merging iwlwifi-next/next (cdba9ccc1fe40 wifi: iwlwifi: mld: remove SCAN_TIMEOUT_MSEC)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi-next/next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/fw/acpi.c       | 132 ++++++++--
+ drivers/net/wireless/intel/iwlwifi/fw/acpi.h       |  28 +-
+ .../net/wireless/intel/iwlwifi/fw/api/mac-cfg.h    |  94 ++++++-
+ .../net/wireless/intel/iwlwifi/fw/api/nvm-reg.h    |  14 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/power.h  |  37 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/scan.h   |  45 ++++
+ drivers/net/wireless/intel/iwlwifi/fw/dbg.c        |   4 +-
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.c | 151 ++---------
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.h |  14 +-
+ drivers/net/wireless/intel/iwlwifi/fw/runtime.h    |  10 +-
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.c       | 238 ++++++++++++++---
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.h       | 141 +++++++---
+ drivers/net/wireless/intel/iwlwifi/iwl-config.h    |   1 -
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.c     |  10 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.h     |  20 +-
+ drivers/net/wireless/intel/iwlwifi/mld/constants.h |   1 -
+ drivers/net/wireless/intel/iwlwifi/mld/fw.c        |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/iface.c     | 101 ++++---
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |  19 ++
+ drivers/net/wireless/intel/iwlwifi/mld/mlo.c       |   1 -
+ drivers/net/wireless/intel/iwlwifi/mld/power.c     |   5 +-
+ .../net/wireless/intel/iwlwifi/mld/regulatory.c    | 162 ++++++++++--
+ .../net/wireless/intel/iwlwifi/mld/regulatory.h    |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/scan.c      | 220 ++++++++++++----
+ drivers/net/wireless/intel/iwlwifi/mld/scan.h      |   2 +
+ drivers/net/wireless/intel/iwlwifi/mld/sta.c       |  42 ++-
+ drivers/net/wireless/intel/iwlwifi/mvm/fw.c        | 157 ++++++++++-
+ drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c   |  46 ----
+ .../net/wireless/intel/iwlwifi/mvm/mld-mac80211.c  | 132 ----------
+ drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c   | 291 +--------------------
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |   5 -
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.h       |   4 -
+ drivers/net/wireless/intel/iwlwifi/mvm/tdls.c      |   6 +-
+ .../net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c |   2 +-
+ 34 files changed, 1280 insertions(+), 859 deletions(-)
+Merging wpan-next/master (1dd9291eb9037 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Merge made by the 'ort' strategy.
+ net/ieee802154/core.c | 49 ++++++++++++++++++++++++++++---------------------
+ 1 file changed, 28 insertions(+), 21 deletions(-)
+Merging wpan-staging/staging (1dd9291eb9037 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'staging' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging drbd/drbd-next (945807e66a140 drbd: remove BROKEN for DRBD)
+$ git merge -m Merge branch 'drbd-next' of https://github.com/LINBIT/linux-drbd.git drbd/drbd-next
+Merge made by the 'ort' strategy.
+ Documentation/block/drbd-app-resync-io.rst   |   208 +
+ Documentation/block/index.rst                |     1 +
+ drivers/block/drbd/Kconfig                   |    59 +
+ drivers/block/drbd/Makefile                  |     9 +-
+ drivers/block/drbd/drbd_actlog.c             |  1202 +--
+ drivers/block/drbd/drbd_bitmap.c             |  1817 ++--
+ drivers/block/drbd/drbd_dax_pmem.c           |   158 +
+ drivers/block/drbd/drbd_dax_pmem.h           |    40 +
+ drivers/block/drbd/drbd_debugfs.c            |  1643 +++-
+ drivers/block/drbd/drbd_debugfs.h            |     2 +
+ drivers/block/drbd/drbd_int.h                |  3261 ++++---
+ drivers/block/drbd/drbd_interval.c           |    35 +-
+ drivers/block/drbd/drbd_interval.h           |   157 +-
+ drivers/block/drbd/drbd_legacy_84.c          |   559 ++
+ drivers/block/drbd/drbd_legacy_84.h          |    27 +
+ drivers/block/drbd/drbd_main.c               |  5922 +++++++++----
+ drivers/block/drbd/drbd_meta_data.h          |   143 +
+ drivers/block/drbd/drbd_nl.c                 |  7260 +++++++++++-----
+ drivers/block/drbd/drbd_nla.h                |     7 +-
+ drivers/block/drbd/drbd_polymorph_printk.h   |   253 +-
+ drivers/block/drbd/drbd_proc.c               |   316 +-
+ drivers/block/drbd/drbd_protocol.h           |   428 -
+ drivers/block/drbd/drbd_receiver.c           | 11480 ++++++++++++++++++-------
+ drivers/block/drbd/drbd_req.c                |  3001 +++++--
+ drivers/block/drbd/drbd_req.h                |   315 +-
+ drivers/block/drbd/drbd_sender.c             |  3826 ++++++++
+ drivers/block/drbd/drbd_state.c              |  7723 +++++++++++++----
+ drivers/block/drbd/drbd_state.h              |   284 +-
+ drivers/block/drbd/drbd_state_change.h       |    66 +-
+ drivers/block/drbd/drbd_strings.c            |   219 +-
+ drivers/block/drbd/drbd_strings.h            |    25 +-
+ drivers/block/drbd/drbd_transport.c          |   379 +
+ drivers/block/drbd/drbd_transport.h          |   443 +
+ drivers/block/drbd/drbd_transport_lb-tcp.c   |  1905 ++++
+ drivers/block/drbd/drbd_transport_rdma.c     |  3531 ++++++++
+ drivers/block/drbd/drbd_transport_tcp.c      |  1668 ++++
+ drivers/block/drbd/drbd_transport_template.c |   160 +
+ drivers/block/drbd/drbd_worker.c             |  2223 -----
+ include/linux/drbd.h                         |   188 +-
+ include/linux/drbd_config.h                  |    33 +-
+ include/linux/drbd_genl.h                    |   350 +-
+ include/linux/drbd_genl_api.h                |    19 +-
+ include/linux/drbd_limits.h                  |   105 +-
+ include/linux/drbd_protocol.h                |   721 ++
+ include/linux/genl_magic_func.h              |    50 +-
+ net/Kconfig                                  |     4 +-
+ 46 files changed, 46001 insertions(+), 16224 deletions(-)
+ create mode 100644 Documentation/block/drbd-app-resync-io.rst
+ create mode 100644 drivers/block/drbd/drbd_dax_pmem.c
+ create mode 100644 drivers/block/drbd/drbd_dax_pmem.h
+ create mode 100644 drivers/block/drbd/drbd_legacy_84.c
+ create mode 100644 drivers/block/drbd/drbd_legacy_84.h
+ create mode 100644 drivers/block/drbd/drbd_meta_data.h
+ delete mode 100644 drivers/block/drbd/drbd_protocol.h
+ create mode 100644 drivers/block/drbd/drbd_sender.c
+ create mode 100644 drivers/block/drbd/drbd_transport.c
+ create mode 100644 drivers/block/drbd/drbd_transport.h
+ create mode 100644 drivers/block/drbd/drbd_transport_lb-tcp.c
+ create mode 100644 drivers/block/drbd/drbd_transport_rdma.c
+ create mode 100644 drivers/block/drbd/drbd_transport_tcp.c
+ create mode 100644 drivers/block/drbd/drbd_transport_template.c
+ delete mode 100644 drivers/block/drbd/drbd_worker.c
+ create mode 100644 include/linux/drbd_protocol.h
+Merging mtd/mtd/next (b800359a4dfac mtd: cmdlinepart: use a flexible array member)
+$ git merge -m Merge branch 'mtd/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Merge made by the 'ort' strategy.
+ .../bindings/mtd/partitions/partition.yaml         |  20 ++
+ drivers/mtd/Kconfig                                |   9 +
+ drivers/mtd/Makefile                               |   1 +
+ drivers/mtd/devices/docg3.c                        |   3 +-
+ drivers/mtd/maps/Kconfig                           |  11 -
+ drivers/mtd/maps/Makefile                          |   1 -
+ drivers/mtd/maps/physmap-bt1-rom.c                 | 125 --------
+ drivers/mtd/maps/physmap-bt1-rom.h                 |  17 -
+ drivers/mtd/maps/physmap-core.c                    |   5 -
+ drivers/mtd/maps/physmap-gemini.c                  |   2 +-
+ drivers/mtd/mtd_virt_concat.c                      | 350 +++++++++++++++++++++
+ drivers/mtd/mtdconcat.c                            |  17 +-
+ drivers/mtd/mtdcore.c                              |  21 ++
+ drivers/mtd/mtdpart.c                              |   6 +
+ drivers/mtd/parsers/cmdlinepart.c                  |   3 +-
+ drivers/mtd/parsers/ofpart_core.c                  |   4 +-
+ include/linux/mtd/concat.h                         |  63 +++-
+ 17 files changed, 475 insertions(+), 183 deletions(-)
+ delete mode 100644 drivers/mtd/maps/physmap-bt1-rom.c
+ delete mode 100644 drivers/mtd/maps/physmap-bt1-rom.h
+ create mode 100644 drivers/mtd/mtd_virt_concat.c
+Merging nand/nand/next (ee78d466db8a0 mtd: rawnand: ifc: set chip->of_node to nand@0 child node if present)
+$ git merge -m Merge branch 'nand/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Auto-merging drivers/mtd/nand/raw/nand_base.c
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/mtd/mxc-nand.yaml |  7 ++++++-
+ drivers/mtd/nand/ecc-realtek.c                      | 18 ++++++++++--------
+ drivers/mtd/nand/raw/cafe_nand.c                    |  7 +++++--
+ drivers/mtd/nand/raw/fsl_ifc_nand.c                 | 10 +++++++++-
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c          | 11 ++++++++++-
+ drivers/mtd/nand/raw/mxc_nand.c                     | 10 +++++++++-
+ drivers/mtd/nand/raw/nand_base.c                    | 19 ++++++++++---------
+ include/linux/mtd/spinand.h                         |  5 +++--
+ 8 files changed, 62 insertions(+), 25 deletions(-)
+Merging spi-nor/spi-nor/next (94645aa41bf9e mtd: spi-nor: swp: check SR_TB flag when getting tb_mask)
+$ git merge -m Merge branch 'spi-nor/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Auto-merging drivers/mtd/spi-nor/core.c
+Merge made by the 'ort' strategy.
+ drivers/mtd/spi-nor/core.c      |  2 +-
+ drivers/mtd/spi-nor/core.h      |  2 +-
+ drivers/mtd/spi-nor/micron-st.c | 10 ++++++++++
+ drivers/mtd/spi-nor/swp.c       |  4 +++-
+ 4 files changed, 15 insertions(+), 3 deletions(-)
+Merging crypto/master (c708d3fad4217 crypto: atmel - use list_first_entry_or_null to simplify find_dev)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Auto-merging MAINTAINERS
+Auto-merging crypto/testmgr.c
+Auto-merging drivers/crypto/ccp/sev-dev.c
+Merge made by the 'ort' strategy.
+ Documentation/crypto/userspace-if.rst              |    4 +-
+ .../bindings/crypto/inside-secure,safexcel.yaml    |    5 +-
+ .../bindings/crypto/qcom,inline-crypto-engine.yaml |   27 +
+ .../devicetree/bindings/rng/atmel,at91-trng.yaml   |    1 +
+ MAINTAINERS                                        |   13 +-
+ crypto/tcrypt.c                                    |   15 +-
+ crypto/testmgr.c                                   |   39 +-
+ crypto/testmgr.h                                   | 1302 ++++++++++++++++++++
+ drivers/crypto/Makefile                            |    1 -
+ drivers/crypto/atmel-ecc.c                         |    1 +
+ drivers/crypto/atmel-i2c.c                         |    6 +-
+ drivers/crypto/atmel-sha.c                         |   17 +-
+ drivers/crypto/atmel-sha204a.c                     |   35 +-
+ drivers/crypto/atmel-tdes.c                        |    8 +-
+ drivers/crypto/caam/caamalg_qi2.c                  |   13 +-
+ drivers/crypto/caam/caamhash.c                     |   12 +-
+ drivers/crypto/ccp/sev-dev.c                       |   27 +-
+ drivers/crypto/inside-secure/eip93/eip93-main.c    |   16 +-
+ drivers/crypto/inside-secure/eip93/eip93-regs.h    |    2 +-
+ drivers/crypto/inside-secure/safexcel.c            |    8 +-
+ drivers/crypto/inside-secure/safexcel.h            |    8 +-
+ drivers/crypto/inside-secure/safexcel_cipher.c     |  149 +++
+ .../crypto/intel/qat/qat_common/adf_accel_engine.c |    7 +
+ drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c |   10 +-
+ .../qat/qat_common/icp_qat_fw_loader_handle.h      |    1 +
+ drivers/crypto/intel/qat/qat_common/qat_hal.c      |   27 +-
+ drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c  |    8 +-
+ drivers/crypto/qce/aead.c                          |   18 +-
+ drivers/crypto/qce/sha.c                           |    6 +-
+ drivers/crypto/qce/skcipher.c                      |    6 +-
+ drivers/crypto/ti/Kconfig                          |    1 +
+ drivers/crypto/ti/dthev2-aes.c                     |  173 ++-
+ drivers/crypto/ti/dthev2-common.c                  |   19 +
+ drivers/crypto/ti/dthev2-common.h                  |   17 +
+ drivers/crypto/virtio/virtio_crypto_common.h       |    3 +-
+ drivers/crypto/virtio/virtio_crypto_core.c         |   11 +-
+ include/crypto/acompress.h                         |    5 +-
+ include/crypto/des.h                               |    8 +-
+ include/crypto/internal/ecc.h                      |   22 +-
+ include/linux/rhashtable.h                         |   13 +-
+ include/uapi/linux/psp-sev.h                       |    2 +-
+ kernel/padata.c                                    |   26 +-
+ 42 files changed, 1864 insertions(+), 228 deletions(-)
+Merging libcrypto/libcrypto-next (6bc9effb4cbf9 coco/guest: Remove unneeded selection of CRYPTO)
+$ git merge -m Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto/libcrypto-next
+Auto-merging arch/arm64/crypto/aes-neonbs-glue.c
+Auto-merging crypto/Kconfig
+Auto-merging crypto/testmgr.c
+Auto-merging lib/crypto/Makefile
+Auto-merging net/mac80211/wpa.c
+Merge made by the 'ort' strategy.
+ arch/arm64/crypto/Kconfig                          |   2 +-
+ arch/arm64/crypto/Makefile                         |   4 +-
+ arch/arm64/crypto/aes-ce-ccm-glue.c                |   4 -
+ arch/arm64/crypto/aes-glue.c                       | 261 +--------------------
+ arch/arm64/crypto/aes-neonbs-glue.c                |  15 +-
+ crypto/Kconfig                                     |   4 +-
+ crypto/aes.c                                       | 183 ++++++++++++++-
+ crypto/jitterentropy-kcapi.c                       | 116 +++------
+ crypto/jitterentropy.c                             |  25 +-
+ crypto/jitterentropy.h                             |  19 +-
+ crypto/testmgr.c                                   |  10 +-
+ drivers/crypto/starfive/jh7110-aes.c               |   2 +-
+ drivers/virt/coco/guest/Kconfig                    |   1 -
+ include/crypto/aes-cbc-macs.h                      | 154 ++++++++++++
+ include/crypto/aes.h                               |  66 ++++++
+ include/crypto/internal/blockhash.h                |  52 ----
+ lib/crypto/.kunitconfig                            |  21 +-
+ lib/crypto/Kconfig                                 |  22 +-
+ lib/crypto/Makefile                                |  15 +-
+ lib/crypto/aes.c                                   | 231 +++++++++++++++++-
+ {arch/arm64/crypto => lib/crypto/arm64}/aes-ce.S   |   3 +-
+ .../arm64/crypto => lib/crypto/arm64}/aes-modes.S  |  25 +-
+ {arch/arm64/crypto => lib/crypto/arm64}/aes-neon.S |   2 +-
+ lib/crypto/arm64/aes.h                             |  84 ++++++-
+ lib/crypto/arm64/sha256.h                          |   8 +-
+ lib/crypto/arm64/sha512.h                          |   5 +-
+ lib/crypto/fips.h                                  |   5 +
+ lib/crypto/tests/Kconfig                           |  57 ++++-
+ lib/crypto/tests/Makefile                          |   1 +
+ lib/crypto/tests/aes-cmac-testvecs.h               | 181 ++++++++++++++
+ lib/crypto/tests/aes_cbc_macs_kunit.c              | 228 ++++++++++++++++++
+ lib/crypto/x86/sha256.h                            |  25 ++
+ net/mac80211/Kconfig                               |   2 +-
+ net/mac80211/aes_cmac.c                            |  65 +----
+ net/mac80211/aes_cmac.h                            |  12 +-
+ net/mac80211/fils_aead.c                           |  48 ++--
+ net/mac80211/key.c                                 |  11 +-
+ net/mac80211/key.h                                 |   3 +-
+ net/mac80211/wpa.c                                 |  13 +-
+ samples/Kconfig                                    |   2 +
+ samples/tsm-mr/tsm_mr_sample.c                     |  66 +++---
+ scripts/crypto/gen-fips-testvecs.py                |  10 +
+ scripts/crypto/gen-hash-testvecs.py                |  31 ++-
+ tools/testing/kunit/configs/all_tests.config       |   2 +
+ 44 files changed, 1417 insertions(+), 679 deletions(-)
+ create mode 100644 include/crypto/aes-cbc-macs.h
+ delete mode 100644 include/crypto/internal/blockhash.h
+ rename {arch/arm64/crypto => lib/crypto/arm64}/aes-ce.S (96%)
+ rename {arch/arm64/crypto => lib/crypto/arm64}/aes-modes.S (98%)
+ rename {arch/arm64/crypto => lib/crypto/arm64}/aes-neon.S (99%)
+ create mode 100644 lib/crypto/tests/aes-cmac-testvecs.h
+ create mode 100644 lib/crypto/tests/aes_cbc_macs_kunit.c
+Merging drm/drm-next (d93f8ea0e5ad4 Merge tag 'drm-intel-next-2026-03-16' of https://gitlab.freedesktop.org/drm/i915/kernel into drm-next)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git drm/drm-next
+Auto-merging Documentation/devicetree/bindings/arm/qcom.yaml
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging drivers/dma-buf/dma-buf.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+Auto-merging drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
+Auto-merging drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+Auto-merging drivers/gpu/drm/drm_gem.c
+Auto-merging drivers/gpu/drm/drm_gem_shmem_helper.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/drm_gem_shmem_helper.c
+Auto-merging drivers/gpu/drm/drm_mode_config.c
+Auto-merging drivers/gpu/drm/gud/gud_pipe.c
+Auto-merging drivers/gpu/drm/i915/display/intel_alpm.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_power_well.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_types.h
+Auto-merging drivers/gpu/drm/i915/display/intel_dmc.c
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Auto-merging drivers/gpu/drm/i915/display/intel_vdsc.c
+Auto-merging drivers/gpu/drm/i915/display/intel_vdsc.h
+CONFLICT (content): Merge conflict in drivers/gpu/drm/i915/display/intel_vdsc.h
+Auto-merging drivers/gpu/drm/i915/display/intel_vrr.c
+Auto-merging drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+Auto-merging drivers/gpu/drm/i915/gt/intel_engine_cs.c
+Auto-merging drivers/gpu/drm/i915/i915_gpu_error.c
+Auto-merging drivers/gpu/drm/imagination/pvr_power.c
+Auto-merging drivers/infiniband/core/ib_core_uverbs.c
+Auto-merging drivers/infiniband/core/umem_dmabuf.c
+Auto-merging drivers/infiniband/hw/mlx5/mr.c
+Auto-merging include/linux/mmu_notifier.h
+CONFLICT (content): Merge conflict in include/linux/mmu_notifier.h
+Auto-merging mm/mmu_notifier.c
+Resolved 'drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c' using previous resolution.
+Recorded preimage for 'drivers/gpu/drm/drm_gem_shmem_helper.c'
+Resolved 'drivers/gpu/drm/i915/display/intel_vdsc.h' using previous resolution.
+Resolved 'include/linux/mmu_notifier.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/gpu/drm/drm_gem_shmem_helper.c'.
+[master 70f8d66613103] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ .../ABI/testing/sysfs-driver-intel-xe-sriov        |    31 +
+ Documentation/devicetree/bindings/arm/qcom.yaml    |     6 +
+ .../bindings/display/bridge/analogix,anx7625.yaml  |    98 +-
+ .../bindings/display/bridge/fsl,ldb.yaml           |    18 +
+ .../bindings/display/bridge/lontium,lt9611.yaml    |    15 +-
+ .../display/bridge/thead,th1520-dw-hdmi.yaml       |   120 +
+ .../bindings/display/bridge/waveshare,dsi2dpi.yaml |     3 +
+ .../bindings/display/panel/abt,y030xx067a.yaml     |     5 -
+ .../display/panel/advantech,idk-1110wr.yaml        |     4 +-
+ .../display/panel/advantech,idk-2121wr.yaml        |     4 +-
+ .../display/panel/bananapi,s070wv20-ct16.yaml      |     4 +-
+ .../bindings/display/panel/dlc,dlc0700yzg-1.yaml   |     4 +-
+ .../bindings/display/panel/himax,hx83112a.yaml     |     2 -
+ .../bindings/display/panel/himax,hx8394.yaml       |     4 +-
+ .../bindings/display/panel/ilitek,ili9163.yaml     |     4 -
+ .../bindings/display/panel/ilitek,ili9322.yaml     |     3 -
+ .../bindings/display/panel/ilitek,ili9341.yaml     |     6 +-
+ .../bindings/display/panel/innolux,ej030na.yaml    |     5 -
+ .../bindings/display/panel/innolux,p097pfg.yaml    |    26 +-
+ .../bindings/display/panel/jadard,jd9365da-h3.yaml |     4 +-
+ .../display/panel/kingdisplay,kd035g6-54nt.yaml    |     5 -
+ .../bindings/display/panel/lgphilips,lb035q02.yaml |     4 -
+ .../bindings/display/panel/lxd,m9189a.yaml         |    64 +
+ .../display/panel/mitsubishi,aa104xd12.yaml        |     4 +-
+ .../display/panel/mitsubishi,aa121td01.yaml        |     4 +-
+ .../bindings/display/panel/motorola,mot-panel.yaml |    69 +
+ .../bindings/display/panel/nec,nl8048hl11.yaml     |     4 -
+ .../bindings/display/panel/novatek,nt36523.yaml    |     3 -
+ .../bindings/display/panel/novatek,nt36672a.yaml   |     3 -
+ .../bindings/display/panel/orisetech,otm8009a.yaml |     4 +-
+ .../bindings/display/panel/panel-edp-legacy.yaml   |     2 +
+ .../bindings/display/panel/panel-simple-dsi.yaml   |     2 +
+ .../bindings/display/panel/panel-simple.yaml       |     6 +-
+ .../bindings/display/panel/pda,91-00156-a0.yaml    |     4 +-
+ .../bindings/display/panel/raydium,rm68200.yaml    |     4 +-
+ .../bindings/display/panel/raydium,rm692e5.yaml    |     2 -
+ .../bindings/display/panel/samsung,atna33xc20.yaml |     4 +-
+ .../bindings/display/panel/samsung,ld9040.yaml     |     4 -
+ .../bindings/display/panel/samsung,lms380kf01.yaml |     6 -
+ .../bindings/display/panel/samsung,lms397kf04.yaml |     6 -
+ .../bindings/display/panel/samsung,s6d27a1.yaml    |     6 -
+ .../bindings/display/panel/samsung,s6d7aa0.yaml    |     8 +
+ .../bindings/display/panel/samsung,s6e3ha8.yaml    |     4 -
+ .../bindings/display/panel/samsung,s6e63m0.yaml    |     2 -
+ .../bindings/display/panel/sgd,gktw70sdae4se.yaml  |     4 +-
+ .../bindings/display/panel/sitronix,st7701.yaml    |     6 -
+ .../bindings/display/panel/sitronix,st7789v.yaml   |     6 -
+ .../bindings/display/panel/sony,acx565akm.yaml     |     4 -
+ .../display/panel/startek,kd070fhfid015.yaml       |     4 +-
+ .../devicetree/bindings/display/panel/tpo,td.yaml  |     5 -
+ .../bindings/display/panel/visionox,r66451.yaml    |     4 +-
+ .../bindings/display/panel/visionox,rm69299.yaml   |     4 +-
+ .../bindings/display/panel/visionox,vtdr6130.yaml  |     4 +-
+ .../bindings/display/rockchip/rockchip,dw-dp.yaml  |    27 +-
+ .../devicetree/bindings/display/tilcdc/panel.txt   |     1 +
+ .../bindings/display/tilcdc/ti,am33xx-tilcdc.yaml  |   100 +
+ .../devicetree/bindings/display/tilcdc/tilcdc.txt  |    82 -
+ .../bindings/display/verisilicon,dc.yaml           |   122 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |     6 +
+ Documentation/gpu/drm-mm.rst                       |    10 +-
+ Documentation/gpu/drm-ras.rst                      |   103 +
+ Documentation/gpu/index.rst                        |     1 +
+ Documentation/gpu/xe/xe_firmware.rst               |     3 +
+ Documentation/netlink/specs/drm_ras.yaml           |   115 +
+ MAINTAINERS                                        |    23 +-
+ drivers/accel/amdxdna/aie2_pci.c                   |    53 +
+ drivers/accel/amdxdna/aie2_pci.h                   |    28 +
+ drivers/accel/amdxdna/amdxdna_pci_drv.c            |     4 +-
+ drivers/accel/ivpu/ivpu_drv.c                      |    94 +-
+ drivers/accel/ivpu/ivpu_drv.h                      |    26 +-
+ drivers/accel/ivpu/ivpu_gem.c                      |    12 +-
+ drivers/accel/ivpu/ivpu_job.c                      |    36 +-
+ drivers/accel/ivpu/vpu_boot_api.h                  |   211 +-
+ drivers/dma-buf/Kconfig                            |    12 -
+ drivers/dma-buf/dma-buf.c                          |    69 +-
+ drivers/dma-buf/dma-fence-array.c                  |    19 +-
+ drivers/dma-buf/dma-fence-chain.c                  |    16 +-
+ drivers/dma-buf/dma-fence.c                        |   168 +-
+ drivers/dma-buf/dma-resv.c                         |     5 +-
+ drivers/dma-buf/heaps/cma_heap.c                   |     7 +-
+ drivers/dma-buf/st-dma-fence.c                     |    50 +-
+ drivers/dma-buf/sw_sync.c                          |    14 +-
+ drivers/dma-buf/sync_debug.h                       |     2 +-
+ drivers/firmware/google/Kconfig                    |     5 +-
+ drivers/firmware/google/cbmem.c                    |     1 +
+ drivers/firmware/google/coreboot_table.c           |    13 +-
+ drivers/firmware/google/coreboot_table.h           |    59 +-
+ drivers/firmware/google/framebuffer-coreboot.c     |   126 +-
+ drivers/firmware/google/memconsole-coreboot.c      |     1 +
+ drivers/firmware/google/vpd.c                      |     1 +
+ drivers/gpu/Kconfig                                |    13 +
+ drivers/gpu/Makefile                               |     4 +-
+ drivers/gpu/buddy.c                                |  1525 +
+ drivers/gpu/drm/Kconfig                            |    18 +-
+ drivers/gpu/drm/Kconfig.debug                      |     1 -
+ drivers/gpu/drm/Makefile                           |     5 +-
+ drivers/gpu/drm/amd/amdgpu/Makefile                |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   154 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c      |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c     |    54 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |     7 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c            |     9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h            |    13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |    61 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h        |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c   |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   814 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |   530 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h      |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_display.c        |    21 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c        |    14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |    20 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |   116 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |    49 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h            |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c        |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c             |    56 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |    19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h            |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h           |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |     7 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            |    52 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c     |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c     |   959 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h     |   163 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h     |    12 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |    50 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |    42 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |   163 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    31 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |    16 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |   135 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |   184 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c            |    18 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h           |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |   195 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |    19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c          |     4 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c       |    79 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h       |    20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c           |    15 +-
+ drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h        |    28 +-
+ drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c         |    21 +-
+ drivers/gpu/drm/amd/amdgpu/atom.c                  |    28 +-
+ drivers/gpu/drm/amd/amdgpu/cik.c                   |    52 +-
+ drivers/gpu/drm/amd/amdgpu/cik_sdma.c              |    31 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |    16 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |    16 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |    16 +-
+ drivers/gpu/drm/amd/amdgpu/df_v3_6.c               |    20 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c             |    33 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c             |    33 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |   109 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |     2 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c             |    10 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c             |     4 +-
+ drivers/gpu/drm/amd/amdgpu/ih_v7_0.c               |    36 +-
+ drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c            |    99 +
+ drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.h            |    31 +
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |     4 +-
+ drivers/gpu/drm/amd/amdgpu/nv.c                    |    30 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v13_0.c             |     2 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c             |    31 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |    31 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |    54 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c           |    39 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c             |    31 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c             |    35 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c             |    33 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c             |    33 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |    33 +-
+ drivers/gpu/drm/amd/amdgpu/si.c                    |    66 +-
+ drivers/gpu/drm/amd/amdgpu/si_dma.c                |    31 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |    68 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.h                 |     1 -
+ drivers/gpu/drm/amd/amdgpu/soc15_common.h          |    25 +-
+ drivers/gpu/drm/amd/amdgpu/soc21.c                 |    30 +-
+ drivers/gpu/drm/amd/amdgpu/soc24.c                 |    20 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |    78 +-
+ drivers/gpu/drm/amd/amdgpu/vce_v1_0.c              |    33 +-
+ drivers/gpu/drm/amd/amdgpu/vi.c                    |    76 +-
+ drivers/gpu/drm/amd/amdkfd/Kconfig                 |     2 +-
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm |     3 -
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |     7 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |     2 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |    35 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |    12 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h       |     3 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c   |     1 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c   |     1 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c   |     1 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c   |     1 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c |     1 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |     8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |     4 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |     4 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |     8 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   234 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c  |    27 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |     9 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c    |     9 +-
+ drivers/gpu/drm/amd/display/dc/bios/bios_parser.c  |    39 +-
+ .../gpu/drm/amd/display/dc/bios/command_table.c    |   124 +-
+ .../amd/display/dc/bios/command_table_helper2.c    |     1 +
+ drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile    |    23 +-
+ drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c   |    30 +-
+ .../amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c    |    64 +-
+ .../amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c   |   166 -
+ .../amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c   |     1 +
+ .../drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c   |    14 +-
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c   |  1165 +
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h   |    79 +
+ .../drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c   |   431 +
+ .../drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.h   |   190 +
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   291 +-
+ .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |    60 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |     8 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |    41 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_surface.c   |     9 +
+ drivers/gpu/drm/amd/display/dc/dc.h                |    76 +-
+ drivers/gpu/drm/amd/display/dc/dc_bios_types.h     |     7 +-
+ drivers/gpu/drm/amd/display/dc/dc_dp_types.h       |     6 +-
+ drivers/gpu/drm/amd/display/dc/dc_helper.c         |     2 +
+ drivers/gpu/drm/amd/display/dc/dc_hw_types.h       |    73 +
+ drivers/gpu/drm/amd/display/dc/dc_plane.h          |     1 +
+ drivers/gpu/drm/amd/display/dc/dc_spl_translate.c  |     1 +
+ drivers/gpu/drm/amd/display/dc/dc_trace.h          |     4 +-
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |    32 +-
+ drivers/gpu/drm/amd/display/dc/dccg/Makefile       |    11 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h |    16 +-
+ .../drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c   |    20 +
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c |   333 +
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h |   272 +
+ drivers/gpu/drm/amd/display/dc/dce/dce_abm.h       |    60 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_audio.c     |   131 -
+ drivers/gpu/drm/amd/display/dc/dce/dce_audio.h     |     9 -
+ drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c   |   966 -
+ .../gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c  |    16 +
+ .../gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.h  |    31 +
+ drivers/gpu/drm/amd/display/dc/dio/Makefile        |    11 +-
+ .../display/dc/dio/dcn10/dcn10_stream_encoder.h    |    10 +-
+ .../display/dc/dio/dcn42/dcn42_dio_link_encoder.c  |   205 +
+ .../display/dc/dio/dcn42/dcn42_dio_link_encoder.h  |   140 +
+ .../dc/dio/dcn42/dcn42_dio_stream_encoder.c        |   522 +
+ .../dc/dio/dcn42/dcn42_dio_stream_encoder.h        |   206 +
+ drivers/gpu/drm/amd/display/dc/dml2_0/Makefile     |     3 +
+ .../drm/amd/display/dc/dml2_0/display_mode_core.c  |     4 +-
+ .../dc/dml2_0/dml21/dml21_translation_helper.c     |    64 +-
+ .../dc/dml2_0/dml21/dml21_translation_helper.h     |     1 +
+ .../drm/amd/display/dc/dml2_0/dml21/dml21_utils.c  |   228 +-
+ .../drm/amd/display/dc/dml2_0/dml21/dml21_utils.h  |     1 +
+ .../amd/display/dc/dml2_0/dml21/dml21_wrapper.c    |    15 +-
+ .../dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h |   263 +
+ .../dml2_0/dml21/inc/dml_top_display_cfg_types.h   |    13 +
+ .../dml2_0/dml21/inc/dml_top_soc_parameter_types.h |     2 +
+ .../display/dc/dml2_0/dml21/inc/dml_top_types.h    |     3 +
+ .../dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c |   111 +
+ .../dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.h |     1 +
+ .../dml2_0/dml21/src/dml2_core/dml2_core_factory.c |     9 +
+ .../dml2_0/dml21/src/dml2_core/dml2_core_utils.c   |    63 +-
+ .../dml2_0/dml21/src/dml2_core/dml2_core_utils.h   |     2 +
+ .../dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c |    33 +
+ .../dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.h |     1 +
+ .../dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c |     6 +
+ .../dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.h   |     2 +-
+ .../dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c  |    76 +
+ .../dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.h  |    12 +
+ .../dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c   |     6 +
+ .../dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c       |     6 +
+ .../dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c   |     4 +-
+ .../dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.h   |     2 +-
+ .../dml21/src/dml2_top/dml2_top_interfaces.c       |     2 +
+ .../dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.c |     1 -
+ .../dml21/src/inc/dml2_internal_shared_types.h     |     3 +
+ .../amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c  |     6 +-
+ .../drm/amd/display/dc/dml2_0/dml2_mall_phantom.c  |    41 +-
+ .../gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c   |     3 +-
+ .../gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h   |     1 +
+ drivers/gpu/drm/amd/display/dc/dpp/Makefile        |    10 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c   |     8 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c   |     8 +-
+ .../drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c    |     2 +
+ .../gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h |     5 +-
+ .../amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c    |    23 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c   |   333 +
+ .../gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.h   |   469 +
+ drivers/gpu/drm/amd/display/dc/gpio/Makefile       |    13 +-
+ .../amd/display/dc/gpio/dcn42/hw_factory_dcn42.c   |   254 +
+ .../amd/display/dc/gpio/dcn42/hw_factory_dcn42.h   |    31 +
+ .../amd/display/dc/gpio/dcn42/hw_translate_dcn42.c |   205 +
+ .../amd/display/dc/gpio/dcn42/hw_translate_dcn42.h |    36 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c   |     4 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c |     4 +
+ drivers/gpu/drm/amd/display/dc/hpo/Makefile        |    11 +-
+ .../dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c     |     5 +
+ .../dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.h     |    10 +-
+ .../dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.c       |    93 +
+ .../dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.h       |    17 +
+ drivers/gpu/drm/amd/display/dc/hubbub/Makefile     |    10 +-
+ .../drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h |     6 +-
+ .../amd/display/dc/hubbub/dcn401/dcn401_hubbub.c   |     4 -
+ .../drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c |   539 +
+ .../drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.h |   268 +
+ drivers/gpu/drm/amd/display/dc/hubp/Makefile       |    10 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h |     8 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |   201 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h   |    23 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c |   708 +
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h |    73 +
+ drivers/gpu/drm/amd/display/dc/hwss/Makefile       |    12 +-
+ .../gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h    |    13 +-
+ .../drm/amd/display/dc/hwss/dce110/dce110_hwseq.c  |   128 +-
+ .../drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c    |    32 +-
+ .../drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c    |     1 -
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |    12 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c  |   492 +-
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c    |  1091 +
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h    |    47 +
+ .../gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c |   173 +
+ .../gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.h |    11 +
+ drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h |    23 +-
+ .../drm/amd/display/dc/hwss/hw_sequencer_private.h |     5 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h    |    24 +-
+ .../drm/amd/display/dc/inc/hw/clk_mgr_internal.h   |   102 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h        |    10 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |    36 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h  |    18 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h        |   100 +-
+ .../drm/amd/display/dc/inc/hw/timing_generator.h   |     4 +-
+ drivers/gpu/drm/amd/display/dc/inc/resource.h      |     1 +
+ drivers/gpu/drm/amd/display/dc/irq/Makefile        |     9 +
+ .../amd/display/dc/irq/dcn42/irq_service_dcn42.c   |   412 +
+ .../amd/display/dc/irq/dcn42/irq_service_dcn42.h   |    15 +
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |    70 +-
+ drivers/gpu/drm/amd/display/dc/link/link_dpms.c    |   634 +-
+ drivers/gpu/drm/amd/display/dc/link/link_factory.c |    29 +-
+ .../display/dc/link/protocols/link_dp_capability.c |     2 +
+ .../amd/display/dc/link/protocols/link_dp_dpia.c   |     2 +-
+ .../dc/link/protocols/link_dp_irq_handler.c        |    36 +-
+ .../dc/link/protocols/link_dp_panel_replay.c       |    25 +-
+ .../dc/link/protocols/link_edp_panel_control.c     |    11 +-
+ drivers/gpu/drm/amd/display/dc/mmhubbub/Makefile   |    10 +-
+ .../amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.c |    25 +
+ .../amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.h |    13 +
+ drivers/gpu/drm/amd/display/dc/mpc/Makefile        |    10 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c   |     4 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c |   190 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h |    35 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c   |   797 +
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h   |   962 +
+ drivers/gpu/drm/amd/display/dc/optc/Makefile       |    10 +-
+ .../gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h |    26 +-
+ .../gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c |     2 +-
+ .../gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c |   291 +
+ .../gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h |   220 +
+ drivers/gpu/drm/amd/display/dc/pg/Makefile         |    10 +-
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c    |   648 +
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h    |   175 +
+ drivers/gpu/drm/amd/display/dc/resource/Makefile   |    18 +
+ .../amd/display/dc/resource/dce60/dce60_resource.c |     2 +-
+ .../amd/display/dc/resource/dcn20/dcn20_resource.c |     2 -
+ .../amd/display/dc/resource/dcn42/dcn42_resource.c |  2337 +
+ .../amd/display/dc/resource/dcn42/dcn42_resource.h |   588 +
+ .../display/dc/resource/dcn42/dcn42_resource_fpu.c |    47 +
+ .../dcn42/dcn42_resource_fpu.h}                    |    17 +-
+ .../amd/display/dc/soc_and_ip_translator/Makefile  |     3 +
+ .../dcn401/dcn401_soc_and_ip_translator.c          |     3 +
+ .../dcn42/dcn42_soc_and_ip_translator.c            |   163 +-
+ .../dcn42/dcn42_soc_and_ip_translator.h            |     1 +
+ .../soc_and_ip_translator/soc_and_ip_translator.c  |     4 +
+ drivers/gpu/drm/amd/display/dmub/dmub_srv.h        |     5 +-
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |   125 +-
+ drivers/gpu/drm/amd/display/dmub/src/Makefile      |     3 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c  |   752 +
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h  |   171 +
+ drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c    |    64 +-
+ .../drm/amd/display/include/bios_parser_types.h    |     2 +
+ drivers/gpu/drm/amd/display/include/dal_asic_id.h  |     1 +
+ drivers/gpu/drm/amd/display/include/dal_types.h    |     1 +
+ drivers/gpu/drm/amd/display/include/dpcd_defs.h    |     3 +
+ .../amd/include/asic_reg/clk/clk_15_0_0_offset.h   |    44 +
+ .../amd/include/asic_reg/clk/clk_15_0_0_sh_mask.h  |    52 +
+ .../amd/include/asic_reg/dcn/dcn_4_2_0_offset.h    | 17874 +++++
+ .../amd/include/asic_reg/dcn/dcn_4_2_0_sh_mask.h   | 67286 +++++++++++++++++++
+ .../amd/include/asic_reg/dpcs/dpcs_4_0_0_offset.h  |   142 +
+ .../amd/include/asic_reg/dpcs/dpcs_4_0_0_sh_mask.h |   688 +
+ .../include/asic_reg/lsdma/lsdma_7_1_0_offset.h    |    44 +
+ .../include/asic_reg/lsdma/lsdma_7_1_0_sh_mask.h   |   105 +
+ drivers/gpu/drm/amd/include/atomfirmware.h         |     1 +
+ drivers/gpu/drm/amd/include/discovery.h            |    13 +
+ drivers/gpu/drm/amd/include/kgd_pp_interface.h     |     5 +
+ drivers/gpu/drm/amd/pm/legacy-dpm/si_smc.c         |    16 +-
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |    12 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |     2 +
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |     7 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c  |    42 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |    14 +
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h   |    13 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c             |     4 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |     3 +
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c |    46 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c   |     3 +-
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c  |   169 +-
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h  |    14 +-
+ drivers/gpu/drm/amd/ras/rascore/Makefile           |     3 +-
+ drivers/gpu/drm/amd/ras/rascore/ras.h              |    30 +
+ drivers/gpu/drm/amd/ras/rascore/ras_aca.c          |    29 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_cmd.c          |     9 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_cmd.h          |    29 +
+ drivers/gpu/drm/amd/ras/rascore/ras_core.c         |    41 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c       |     4 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom.h       |     7 -
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c    |   520 +
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.h    |    87 +
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.c          |    37 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.h          |     2 +
+ drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.c    |    19 +-
+ .../drm/arm/display/komeda/komeda_framebuffer.c    |     6 +-
+ .../gpu/drm/arm/display/komeda/komeda_pipeline.h   |     2 +
+ .../drm/arm/display/komeda/komeda_private_obj.c    |   208 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c       |    81 +
+ drivers/gpu/drm/bridge/Kconfig                     |    10 +
+ drivers/gpu/drm/bridge/Makefile                    |     1 +
+ drivers/gpu/drm/bridge/analogix/Kconfig            |     2 +
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |   161 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.h |     4 +-
+ drivers/gpu/drm/bridge/analogix/anx7625.c          |   216 +-
+ drivers/gpu/drm/bridge/analogix/anx7625.h          |    34 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.c    |   257 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.h    |     2 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c    |    18 +-
+ drivers/gpu/drm/bridge/fsl-ldb.c                   |     7 +
+ drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c    |    50 +-
+ drivers/gpu/drm/bridge/lontium-lt9611.c            |    46 +-
+ drivers/gpu/drm/bridge/synopsys/dw-dp.c            |    12 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c       |   224 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h       |     3 +
+ drivers/gpu/drm/bridge/tda998x_drv.c               |   263 +-
+ drivers/gpu/drm/bridge/th1520-dw-hdmi.c            |   173 +
+ drivers/gpu/drm/bridge/waveshare-dsi.c             |    14 +-
+ drivers/gpu/drm/display/drm_bridge_connector.c     |    18 +-
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      |    37 +-
+ drivers/gpu/drm/display/drm_dp_tunnel.c            |    25 +-
+ drivers/gpu/drm/drm_atomic.c                       |    32 +-
+ drivers/gpu/drm/drm_atomic_helper.c                |     6 +-
+ drivers/gpu/drm/drm_atomic_state_helper.c          |    23 +
+ drivers/gpu/drm/drm_bridge.c                       |    31 +-
+ drivers/gpu/drm/drm_buddy.c                        |  1272 +-
+ drivers/gpu/drm/drm_client.c                       |     3 +-
+ drivers/gpu/drm/drm_colorop.c                      |    49 +-
+ drivers/gpu/drm/drm_connector.c                    |    33 +
+ drivers/gpu/drm/drm_crtc.c                         |     4 +-
+ drivers/gpu/drm/drm_drv.c                          |     6 +
+ drivers/gpu/drm/drm_fb_helper.c                    |    45 +-
+ drivers/gpu/drm/drm_gem.c                          |    45 +-
+ drivers/gpu/drm/drm_gem_shmem_helper.c             |    89 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |     7 +-
+ drivers/gpu/drm/drm_mode_config.c                  |     6 +
+ drivers/gpu/drm/drm_probe_helper.c                 |     2 +-
+ drivers/gpu/drm/drm_ras.c                          |   354 +
+ drivers/gpu/drm/drm_ras_genl_family.c              |    42 +
+ drivers/gpu/drm/drm_ras_nl.c                       |    56 +
+ drivers/gpu/drm/drm_ras_nl.h                       |    24 +
+ drivers/gpu/drm/drm_self_refresh_helper.c          |     2 +-
+ drivers/gpu/drm/drm_suballoc.c                     |   106 +-
+ drivers/gpu/drm/drm_writeback.c                    |     2 +-
+ drivers/gpu/drm/gma500/fbdev.c                     |   101 +-
+ drivers/gpu/drm/gma500/framebuffer.c               |   104 +-
+ drivers/gpu/drm/gma500/psb_drv.h                   |     6 -
+ drivers/gpu/drm/gud/gud_pipe.c                     |     2 +-
+ drivers/gpu/drm/i915/Makefile                      |     7 +-
+ drivers/gpu/drm/i915/display/g4x_dp.c              |     1 -
+ drivers/gpu/drm/i915/display/g4x_hdmi.c            |     1 -
+ drivers/gpu/drm/i915/display/hsw_ips.c             |    10 +-
+ drivers/gpu/drm/i915/display/i9xx_plane.c          |     1 -
+ drivers/gpu/drm/i915/display/i9xx_wm.c             |    10 +-
+ drivers/gpu/drm/i915/display/icl_dsi.c             |     7 -
+ drivers/gpu/drm/i915/display/intel_alpm.c          |   144 +-
+ drivers/gpu/drm/i915/display/intel_alpm.h          |     9 +-
+ drivers/gpu/drm/i915/display/intel_audio.c         |    24 +-
+ drivers/gpu/drm/i915/display/intel_backlight.c     |     1 -
+ drivers/gpu/drm/i915/display/intel_bios.c          |    31 +-
+ drivers/gpu/drm/i915/display/intel_bo.c            |   100 +-
+ drivers/gpu/drm/i915/display/intel_bo.h            |    14 +-
+ drivers/gpu/drm/i915/display/intel_bw.c            |    24 +-
+ drivers/gpu/drm/i915/display/intel_casf.c          |     7 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |    63 +-
+ .../gpu/drm/i915/display/intel_color_pipeline.c    |   175 +-
+ drivers/gpu/drm/i915/display/intel_colorop.c       |     6 +
+ drivers/gpu/drm/i915/display/intel_colorop.h       |     1 +
+ drivers/gpu/drm/i915/display/intel_crt.c           |    18 +-
+ drivers/gpu/drm/i915/display/intel_crt_regs.h      |     2 -
+ drivers/gpu/drm/i915/display/intel_crtc.c          |     4 +-
+ drivers/gpu/drm/i915/display/intel_cursor.c        |     3 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       |   737 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy.h       |     5 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy_regs.h  |     4 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           |    13 +-
+ drivers/gpu/drm/i915/display/intel_de.h            |    17 +-
+ drivers/gpu/drm/i915/display/intel_display.c       |   150 +-
+ drivers/gpu/drm/i915/display/intel_display.h       |    27 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |     2 +-
+ .../gpu/drm/i915/display/intel_display_device.c    |    11 +-
+ .../gpu/drm/i915/display/intel_display_device.h    |     1 +
+ .../gpu/drm/i915/display/intel_display_driver.c    |    22 +-
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |    11 +-
+ .../gpu/drm/i915/display/intel_display_limits.h    |    26 +
+ drivers/gpu/drm/i915/display/intel_display_power.c |    29 +-
+ .../gpu/drm/i915/display/intel_display_power_map.c |    13 -
+ .../drm/i915/display/intel_display_power_well.c    |    72 +-
+ .../drm/i915/display/intel_display_power_well.h    |     2 -
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |   268 +-
+ drivers/gpu/drm/i915/display/intel_display_rps.c   |     2 +-
+ .../gpu/drm/i915/display/intel_display_snapshot.c  |     4 -
+ drivers/gpu/drm/i915/display/intel_display_types.h |    12 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |    83 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.h    |    28 +-
+ drivers/gpu/drm/i915/display/intel_dmc.c           |     1 -
+ drivers/gpu/drm/i915/display/intel_dp.c            |   864 +-
+ drivers/gpu/drm/i915/display/intel_dp.h            |    20 +-
+ .../gpu/drm/i915/display/intel_dp_aux_backlight.c  |    53 +-
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |   213 +-
+ drivers/gpu/drm/i915/display/intel_dp_tunnel.c     |    96 +-
+ drivers/gpu/drm/i915/display/intel_dpll.c          |     8 +-
+ drivers/gpu/drm/i915/display/intel_dpll.h          |     1 +
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.c      |    11 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.h      |     3 -
+ drivers/gpu/drm/i915/display/intel_dpt.c           |   328 +-
+ drivers/gpu/drm/i915/display/intel_dpt.h           |    22 +-
+ drivers/gpu/drm/i915/display/intel_dpt_common.c    |    35 -
+ drivers/gpu/drm/i915/display/intel_dpt_common.h    |    13 -
+ drivers/gpu/drm/i915/display/intel_dram.c          |     9 +-
+ drivers/gpu/drm/i915/display/intel_dsb.c           |   107 +-
+ drivers/gpu/drm/i915/display/intel_dsb_buffer.h    |    22 -
+ drivers/gpu/drm/i915/display/intel_fb.c            |    41 +-
+ drivers/gpu/drm/i915/display/intel_fb_bo.c         |   101 -
+ drivers/gpu/drm/i915/display/intel_fb_bo.h         |    25 -
+ drivers/gpu/drm/i915/display/intel_fb_pin.c        |    15 +-
+ drivers/gpu/drm/i915/display/intel_fbc.c           |    17 +-
+ drivers/gpu/drm/i915/display/intel_fbdev.c         |     5 +-
+ drivers/gpu/drm/i915/display/intel_fbdev_fb.c      |    38 +-
+ drivers/gpu/drm/i915/display/intel_fbdev_fb.h      |     1 +
+ drivers/gpu/drm/i915/display/intel_fdi.c           |     1 -
+ drivers/gpu/drm/i915/display/intel_fifo_underrun.c |     1 -
+ drivers/gpu/drm/i915/display/intel_flipq.c         |    13 +-
+ drivers/gpu/drm/i915/display/intel_frontbuffer.c   |    37 +-
+ drivers/gpu/drm/i915/display/intel_frontbuffer.h   |    21 +-
+ drivers/gpu/drm/i915/display/intel_gmbus.c         |     7 +-
+ drivers/gpu/drm/i915/display/intel_hdcp.c          |    18 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |    19 +-
+ drivers/gpu/drm/i915/display/intel_hotplug_irq.c   |     1 -
+ drivers/gpu/drm/i915/display/intel_lt_phy.c        |   245 +-
+ drivers/gpu/drm/i915/display/intel_lt_phy.h        |     5 +-
+ drivers/gpu/drm/i915/display/intel_lt_phy_regs.h   |     8 +-
+ drivers/gpu/drm/i915/display/intel_modeset_setup.c |     4 +-
+ drivers/gpu/drm/i915/display/intel_oprom_regs.h    |    15 +
+ drivers/gpu/drm/i915/display/intel_overlay.c       |   584 +-
+ drivers/gpu/drm/i915/display/intel_overlay.h       |    48 -
+ drivers/gpu/drm/i915/display/intel_parent.c        |   156 +
+ drivers/gpu/drm/i915/display/intel_parent.h        |    51 +
+ drivers/gpu/drm/i915/display/intel_pch_display.c   |     1 -
+ drivers/gpu/drm/i915/display/intel_pch_refclk.c    |     1 -
+ drivers/gpu/drm/i915/display/intel_pmdemand.c      |     8 +-
+ drivers/gpu/drm/i915/display/intel_pps.c           |     1 -
+ drivers/gpu/drm/i915/display/intel_psr.c           |    57 +-
+ drivers/gpu/drm/i915/display/intel_psr.h           |     1 +
+ drivers/gpu/drm/i915/display/intel_rom.c           |     3 +-
+ drivers/gpu/drm/i915/display/intel_snps_hdmi_pll.c |     2 -
+ drivers/gpu/drm/i915/display/intel_tc.c            |     5 +-
+ drivers/gpu/drm/i915/display/intel_vbt_defs.h      |     1 +
+ drivers/gpu/drm/i915/display/intel_vdsc.c          |    73 +-
+ drivers/gpu/drm/i915/display/intel_vdsc.h          |     9 +
+ drivers/gpu/drm/i915/display/intel_vga.c           |   333 +-
+ drivers/gpu/drm/i915/display/intel_vga.h           |     5 +-
+ drivers/gpu/drm/i915/display/intel_vrr.c           |    41 +-
+ drivers/gpu/drm/i915/display/intel_vrr.h           |     1 +
+ drivers/gpu/drm/i915/display/intel_vrr_regs.h      |     1 +
+ drivers/gpu/drm/i915/display/skl_scaler.c          |     2 +-
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |     7 +-
+ drivers/gpu/drm/i915/display/skl_watermark.c       |    26 +-
+ drivers/gpu/drm/i915/display/vlv_dsi.c             |     1 -
+ drivers/gpu/drm/i915/gem/i915_gem_clflush.c        |     2 +-
+ drivers/gpu/drm/i915/gem/i915_gem_domain.c         |     6 +-
+ drivers/gpu/drm/i915/gem/i915_gem_object.c         |    30 +-
+ .../gpu/drm/i915/gem/i915_gem_object_frontbuffer.c |    69 +
+ .../gpu/drm/i915/gem/i915_gem_object_frontbuffer.h |    24 +-
+ drivers/gpu/drm/i915/gem/i915_gem_phys.c           |     4 +-
+ drivers/gpu/drm/i915/gem/i915_gem_ttm.c            |     2 +-
+ drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c       |     2 +-
+ drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c |    27 +-
+ drivers/gpu/drm/i915/gt/gen6_ppgtt.c               |     2 +-
+ drivers/gpu/drm/i915/gt/intel_breadcrumbs.c        |     2 +-
+ drivers/gpu/drm/i915/gt/intel_engine_cs.c          |    10 +-
+ drivers/gpu/drm/i915/gt/intel_engine_pm.c          |     2 +-
+ .../gpu/drm/i915/gt/intel_execlists_submission.c   |     6 +-
+ drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c       |     7 +-
+ drivers/gpu/drm/i915/gt/intel_gt.c                 |     1 +
+ drivers/gpu/drm/i915/gt/intel_gt_irq.c             |     2 +
+ drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c      |     1 +
+ drivers/gpu/drm/i915/gt/intel_gt_regs.h            |    11 +-
+ drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c        |     2 +
+ drivers/gpu/drm/i915/gt/intel_llc.c                |     2 +
+ drivers/gpu/drm/i915/gt/intel_lrc.c                |    21 +-
+ drivers/gpu/drm/i915/gt/intel_rc6.c                |    24 +-
+ drivers/gpu/drm/i915/gt/intel_reset.c              |     4 +-
+ drivers/gpu/drm/i915/gt/intel_ring_submission.c    |    21 +-
+ drivers/gpu/drm/i915/gt/intel_rps.c                |     1 +
+ drivers/gpu/drm/i915/gt/intel_workarounds.c        |    26 +-
+ drivers/gpu/drm/i915/gt/uc/intel_guc.c             |     4 +-
+ drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c          |     4 +-
+ drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c  |    10 +-
+ drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c           |     4 +-
+ drivers/gpu/drm/i915/gt/uc/selftest_guc.c          |     2 +-
+ .../gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c    |     2 +-
+ drivers/gpu/drm/i915/gvt/cmd_parser.c              |     1 +
+ drivers/gpu/drm/i915/gvt/display.c                 |     1 -
+ drivers/gpu/drm/i915/gvt/handlers.c                |    48 +-
+ drivers/gpu/drm/i915/gvt/interrupt.c               |     1 +
+ drivers/gpu/drm/i915/gvt/mmio_context.c            |     3 +-
+ drivers/gpu/drm/i915/gvt/reg.h                     |     4 +-
+ drivers/gpu/drm/i915/i915_active.c                 |    21 +-
+ drivers/gpu/drm/i915/i915_bo.c                     |   156 +
+ drivers/gpu/drm/i915/i915_bo.h                     |     9 +
+ drivers/gpu/drm/i915/i915_debugfs.c                |     1 +
+ drivers/gpu/drm/i915/i915_dpt.c                    |   277 +
+ drivers/gpu/drm/i915/i915_dpt.h                    |    20 +
+ drivers/gpu/drm/i915/i915_driver.c                 |    32 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |     5 +-
+ .../intel_dsb_buffer.c => i915_dsb_buffer.c}       |    28 +-
+ drivers/gpu/drm/i915/i915_dsb_buffer.h             |     9 +
+ drivers/gpu/drm/i915/i915_gem.c                    |     6 +-
+ drivers/gpu/drm/i915/i915_gpu_error.c              |     6 -
+ drivers/gpu/drm/i915/i915_gpu_error.h              |     1 -
+ drivers/gpu/drm/i915/i915_hwmon.c                  |     2 +
+ drivers/gpu/drm/i915/i915_initial_plane.c          |     3 +-
+ drivers/gpu/drm/i915/i915_irq.c                    |     1 +
+ drivers/gpu/drm/i915/i915_overlay.c                |   517 +
+ drivers/gpu/drm/i915/i915_overlay.h                |    11 +
+ drivers/gpu/drm/i915/i915_perf.c                   |    34 +-
+ drivers/gpu/drm/i915/i915_reg.h                    |   465 +-
+ drivers/gpu/drm/i915/i915_reg_defs.h               |   179 +-
+ drivers/gpu/drm/i915/i915_scatterlist.c            |    10 +-
+ drivers/gpu/drm/i915/i915_sw_fence_work.c          |     2 +-
+ drivers/gpu/drm/i915/i915_ttm_buddy_manager.c      |    55 +-
+ drivers/gpu/drm/i915/i915_ttm_buddy_manager.h      |     4 +-
+ drivers/gpu/drm/i915/i915_vma.c                    |    10 +
+ drivers/gpu/drm/i915/i915_vma.h                    |     7 +-
+ drivers/gpu/drm/i915/i915_vma_resource.c           |     2 +-
+ drivers/gpu/drm/i915/intel_clock_gating.c          |    42 +-
+ drivers/gpu/drm/i915/intel_gvt_mmio_table.c        |    10 +-
+ drivers/gpu/drm/i915/intel_pcode.c                 |    17 +-
+ drivers/gpu/drm/i915/intel_pcode.h                 |     9 +-
+ drivers/gpu/drm/i915/intel_uncore.c                |     4 +-
+ drivers/gpu/drm/i915/pxp/intel_pxp.c               |     6 +-
+ drivers/gpu/drm/i915/pxp/intel_pxp_irq.c           |     2 +-
+ drivers/gpu/drm/i915/selftests/i915_active.c       |     4 +-
+ drivers/gpu/drm/i915/selftests/i915_sw_fence.c     |     2 +-
+ drivers/gpu/drm/i915/selftests/igt_mmap.c          |    43 +-
+ drivers/gpu/drm/i915/selftests/igt_mmap.h          |     8 +
+ .../gpu/drm/i915/selftests/intel_memory_region.c   |    20 +-
+ drivers/gpu/drm/i915/selftests/mock_gem_device.c   |     2 +-
+ drivers/gpu/drm/i915/vlv_suspend.c                 |     1 +
+ drivers/gpu/drm/imagination/pvr_ccb.c              |    20 +-
+ drivers/gpu/drm/imagination/pvr_device.h           |    10 +-
+ drivers/gpu/drm/imagination/pvr_gem.c              |     6 +-
+ drivers/gpu/drm/imagination/pvr_power.c            |   122 +-
+ drivers/gpu/drm/ingenic/ingenic-drm-drv.c          |    28 +-
+ drivers/gpu/drm/ingenic/ingenic-ipu.c              |    28 +-
+ drivers/gpu/drm/loongson/lsdc_drv.c                |     3 +-
+ drivers/gpu/drm/loongson/lsdc_gem.c                |    15 -
+ drivers/gpu/drm/loongson/lsdc_gem.h                |     4 -
+ drivers/gpu/drm/mcde/mcde_dsi.c                    |    17 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c            |    42 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c           |    41 +-
+ drivers/gpu/drm/mxsfb/lcdif_drv.c                  |     8 +-
+ drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h   |    19 +
+ drivers/gpu/drm/nouveau/nouveau_abi16.c            |    29 +
+ drivers/gpu/drm/nouveau/nouveau_abi16.h            |     1 +
+ drivers/gpu/drm/nouveau/nouveau_bo.c               |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |    38 +-
+ drivers/gpu/drm/nouveau/nouveau_fence.c            |     3 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |    11 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c    |     4 -
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gr.c   |     9 +-
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c  |     7 +-
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gr.c   |    34 +-
+ .../drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gr.h  |    19 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h    |     2 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c    |    18 +-
+ drivers/gpu/drm/panel/Kconfig                      |    33 +
+ drivers/gpu/drm/panel/Makefile                     |     3 +
+ .../gpu/drm/panel/panel-boe-th101mb31ig002-28a.c   |     7 +-
+ drivers/gpu/drm/panel/panel-edp.c                  |    37 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9882t.c      |    14 +-
+ drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c   |   244 +
+ drivers/gpu/drm/panel/panel-jdi-lt070me05000.c     |   105 +-
+ drivers/gpu/drm/panel/panel-lxd-m9189a.c           |   243 +
+ drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c   |    15 +-
+ drivers/gpu/drm/panel/panel-motorola-mot.c         |   244 +
+ drivers/gpu/drm/panel/panel-novatek-nt37700f.c     |   299 +
+ drivers/gpu/drm/panel/panel-renesas-r61307.c       |    40 +-
+ drivers/gpu/drm/panel/panel-renesas-r69328.c       |    38 +-
+ drivers/gpu/drm/panel/panel-simple.c               |   117 +-
+ drivers/gpu/drm/panfrost/panfrost_gem.c            |     2 +-
+ drivers/gpu/drm/panthor/Makefile                   |     2 +
+ drivers/gpu/drm/panthor/panthor_device.h           |   107 +-
+ drivers/gpu/drm/panthor/panthor_fw.c               |    16 +-
+ drivers/gpu/drm/panthor/panthor_gem.c              |     2 +-
+ drivers/gpu/drm/panthor/panthor_gpu.c              |    30 +-
+ drivers/gpu/drm/panthor/panthor_gpu.h              |     2 +
+ drivers/gpu/drm/panthor/panthor_hw.c               |    62 +
+ drivers/gpu/drm/panthor/panthor_hw.h               |     8 +
+ drivers/gpu/drm/panthor/panthor_mmu.c              |    47 +-
+ drivers/gpu/drm/panthor/panthor_pwr.c              |     2 +-
+ drivers/gpu/drm/panthor/panthor_sched.c            |     2 +-
+ drivers/gpu/drm/panthor/panthor_trace.h            |    86 +
+ drivers/gpu/drm/qxl/qxl_release.c                  |     5 +-
+ drivers/gpu/drm/radeon/radeon_device.c             |     7 +-
+ drivers/gpu/drm/radeon/radeon_fbdev.c              |    17 -
+ drivers/gpu/drm/radeon/radeon_mode.h               |     5 -
+ drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c    |    35 +-
+ drivers/gpu/drm/rockchip/dw_dp-rockchip.c          |    27 +-
+ drivers/gpu/drm/rockchip/rockchip_vop2_reg.c       |     8 +-
+ drivers/gpu/drm/scheduler/sched_fence.c            |    10 +-
+ drivers/gpu/drm/scheduler/sched_main.c             |    38 +-
+ drivers/gpu/drm/scheduler/tests/tests_basic.c      |     4 +-
+ drivers/gpu/drm/sun4i/sun4i_backend.c              |     3 +
+ drivers/gpu/drm/sun4i/sun8i_mixer.c                |     2 +-
+ drivers/gpu/drm/sun4i/sunxi_engine.h               |     7 +-
+ drivers/gpu/drm/sysfb/Kconfig                      |    16 +
+ drivers/gpu/drm/sysfb/Makefile                     |     1 +
+ drivers/gpu/drm/sysfb/corebootdrm.c                |   434 +
+ drivers/gpu/drm/sysfb/drm_sysfb.c                  |    24 +
+ drivers/gpu/drm/sysfb/drm_sysfb_helper.h           |     8 +-
+ drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c      |    30 -
+ drivers/gpu/drm/sysfb/efidrm.c                     |     8 +-
+ drivers/gpu/drm/sysfb/vesadrm.c                    |     8 +-
+ drivers/gpu/drm/tests/Makefile                     |     1 -
+ drivers/gpu/drm/tests/drm_buddy_test.c             |   928 -
+ drivers/gpu/drm/tests/drm_exec_test.c              |     2 -
+ drivers/gpu/drm/tests/drm_mm_test.c                |     2 -
+ drivers/gpu/drm/tilcdc/Kconfig                     |    18 +
+ drivers/gpu/drm/tilcdc/Makefile                    |     5 +-
+ drivers/gpu/drm/tilcdc/tilcdc_crtc.c               |   192 +-
+ drivers/gpu/drm/tilcdc/tilcdc_drv.c                |   496 +-
+ drivers/gpu/drm/tilcdc/tilcdc_drv.h                |    99 +-
+ drivers/gpu/drm/tilcdc/tilcdc_encoder.c            |    69 +
+ .../tilcdc/{tilcdc_external.h => tilcdc_encoder.h} |     5 +-
+ drivers/gpu/drm/tilcdc/tilcdc_external.c           |   179 -
+ drivers/gpu/drm/tilcdc/tilcdc_panel.h              |    15 -
+ drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c       |   185 +
+ drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.dtso    |    29 +
+ drivers/gpu/drm/tilcdc/tilcdc_plane.c              |    35 +-
+ drivers/gpu/drm/tilcdc/tilcdc_regs.h               |     8 +-
+ drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c   |     4 +-
+ drivers/gpu/drm/ttm/tests/ttm_mock_manager.c       |    18 +-
+ drivers/gpu/drm/ttm/tests/ttm_mock_manager.h       |     4 +-
+ drivers/gpu/drm/ttm/ttm_bo.c                       |     6 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |     5 +-
+ drivers/gpu/drm/v3d/v3d_perfmon.c                  |    57 +-
+ drivers/gpu/drm/vc4/vc4_bo.c                       |     2 +-
+ drivers/gpu/drm/vc4/vc4_drv.h                      |     5 +-
+ drivers/gpu/drm/vc4/vc4_gem.c                      |     2 +-
+ drivers/gpu/drm/vc4/vc4_hdmi.c                     |    18 +-
+ drivers/gpu/drm/vc4/vc4_kms.c                      |    69 +-
+ drivers/gpu/drm/vc4/vc4_perfmon.c                  |    51 +-
+ drivers/gpu/drm/verisilicon/Kconfig                |    16 +
+ drivers/gpu/drm/verisilicon/Makefile               |     5 +
+ drivers/gpu/drm/verisilicon/vs_bridge.c            |   371 +
+ drivers/gpu/drm/verisilicon/vs_bridge.h            |    39 +
+ drivers/gpu/drm/verisilicon/vs_bridge_regs.h       |    54 +
+ drivers/gpu/drm/verisilicon/vs_crtc.c              |   191 +
+ drivers/gpu/drm/verisilicon/vs_crtc.h              |    31 +
+ drivers/gpu/drm/verisilicon/vs_crtc_regs.h         |    60 +
+ drivers/gpu/drm/verisilicon/vs_dc.c                |   207 +
+ drivers/gpu/drm/verisilicon/vs_dc.h                |    38 +
+ drivers/gpu/drm/verisilicon/vs_dc_top_regs.h       |    27 +
+ drivers/gpu/drm/verisilicon/vs_drm.c               |   182 +
+ drivers/gpu/drm/verisilicon/vs_drm.h               |    28 +
+ drivers/gpu/drm/verisilicon/vs_hwdb.c              |   150 +
+ drivers/gpu/drm/verisilicon/vs_hwdb.h              |    29 +
+ drivers/gpu/drm/verisilicon/vs_plane.c             |   124 +
+ drivers/gpu/drm/verisilicon/vs_plane.h             |    72 +
+ drivers/gpu/drm/verisilicon/vs_primary_plane.c     |   173 +
+ .../gpu/drm/verisilicon/vs_primary_plane_regs.h    |    53 +
+ drivers/gpu/drm/virtio/virtgpu_prime.c             |     4 +-
+ drivers/gpu/drm/vkms/vkms_colorop.c                |    16 +-
+ drivers/gpu/drm/vkms/vkms_drv.c                    |     1 -
+ drivers/gpu/drm/vmwgfx/vmwgfx_fence.c              |     3 +-
+ drivers/gpu/drm/xe/Makefile                        |    11 +-
+ drivers/gpu/drm/xe/compat-i915-headers/i915_reg.h  |     6 -
+ drivers/gpu/drm/xe/compat-i915-headers/i915_vma.h  |    36 -
+ .../gpu/drm/xe/compat-i915-headers/intel_pcode.h   |    11 -
+ .../gpu/drm/xe/compat-i915-headers/intel_uncore.h  |     9 +
+ drivers/gpu/drm/xe/display/intel_bo.c              |   109 -
+ drivers/gpu/drm/xe/display/intel_fbdev_fb.c        |    27 +-
+ drivers/gpu/drm/xe/display/xe_display.c            |     8 +
+ .../xe/display/{intel_fb_bo.c => xe_display_bo.c}  |    63 +-
+ drivers/gpu/drm/xe/display/xe_display_bo.h         |     9 +
+ drivers/gpu/drm/xe/display/xe_display_pcode.c      |    38 +
+ drivers/gpu/drm/xe/display/xe_display_pcode.h      |     9 +
+ drivers/gpu/drm/xe/display/xe_display_vma.h        |    18 +
+ drivers/gpu/drm/xe/display/xe_dsb_buffer.c         |    28 +-
+ drivers/gpu/drm/xe/display/xe_dsb_buffer.h         |     9 +
+ drivers/gpu/drm/xe/display/xe_fb_pin.c             |    31 +-
+ drivers/gpu/drm/xe/display/xe_frontbuffer.c        |    71 +
+ drivers/gpu/drm/xe/display/xe_frontbuffer.h        |     9 +
+ drivers/gpu/drm/xe/display/xe_initial_plane.c      |    27 +-
+ .../gpu/drm/xe/instructions/xe_gfxpipe_commands.h  |    10 +
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h               |    31 +-
+ drivers/gpu/drm/xe/regs/xe_guc_regs.h              |     8 +
+ drivers/gpu/drm/xe/regs/xe_hw_error_regs.h         |    86 +-
+ drivers/gpu/drm/xe/regs/xe_reg_defs.h              |     5 +-
+ drivers/gpu/drm/xe/tests/xe_dma_buf.c              |     7 +-
+ .../gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c |    98 +-
+ drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c        |     6 +-
+ drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c         |    61 +-
+ drivers/gpu/drm/xe/tests/xe_pci_test.c             |     8 +
+ drivers/gpu/drm/xe/tests/xe_rtp_test.c             |     3 +-
+ drivers/gpu/drm/xe/xe_bb.c                         |    59 +-
+ drivers/gpu/drm/xe/xe_bb.h                         |     6 +-
+ drivers/gpu/drm/xe/xe_bo.c                         |    61 +-
+ drivers/gpu/drm/xe/xe_bo.h                         |     2 +
+ drivers/gpu/drm/xe/xe_configfs.c                   |    64 +-
+ drivers/gpu/drm/xe/xe_configfs.h                   |    16 +-
+ drivers/gpu/drm/xe/xe_defaults.h                   |    26 +
+ drivers/gpu/drm/xe/xe_devcoredump.c                |     2 +-
+ drivers/gpu/drm/xe/xe_device.c                     |    60 +-
+ drivers/gpu/drm/xe/xe_device.h                     |    20 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |   206 +-
+ drivers/gpu/drm/xe/xe_device_wa_oob.rules          |     1 +
+ drivers/gpu/drm/xe/xe_dma_buf.c                    |    14 +-
+ drivers/gpu/drm/xe/xe_drm_ras.c                    |   186 +
+ drivers/gpu/drm/xe/xe_drm_ras.h                    |    15 +
+ drivers/gpu/drm/xe/xe_drm_ras_types.h              |    48 +
+ drivers/gpu/drm/xe/xe_eu_stall.c                   |    20 +-
+ drivers/gpu/drm/xe/xe_exec_queue.c                 |   244 +-
+ drivers/gpu/drm/xe/xe_exec_queue.h                 |     1 +
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |    21 +-
+ drivers/gpu/drm/xe/xe_execlist.c                   |    15 +-
+ drivers/gpu/drm/xe/xe_force_wake.c                 |    46 +-
+ drivers/gpu/drm/xe/xe_force_wake.h                 |    11 +
+ drivers/gpu/drm/xe/xe_ggtt.c                       |   375 +-
+ drivers/gpu/drm/xe/xe_ggtt.h                       |    14 +-
+ drivers/gpu/drm/xe/xe_ggtt_types.h                 |    60 +-
+ drivers/gpu/drm/xe/xe_gt.c                         |    54 +-
+ drivers/gpu/drm/xe/xe_gt_ccs_mode.c                |    42 +-
+ drivers/gpu/drm/xe/xe_gt_debugfs.c                 |    26 +
+ drivers/gpu/drm/xe/xe_gt_idle.c                    |    25 +
+ drivers/gpu/drm/xe/xe_gt_mcr.c                     |    30 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c         |   242 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h         |     4 +
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c        |     2 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c        |     2 +
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.c                |    89 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.h                |     3 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h          |     4 +-
+ drivers/gpu/drm/xe/xe_gt_stats.c                   |    64 +-
+ drivers/gpu/drm/xe/xe_gt_stats.h                   |     6 +
+ drivers/gpu/drm/xe/xe_gt_stats_types.h             |    20 +
+ drivers/gpu/drm/xe/xe_gt_topology.c                |    39 +-
+ drivers/gpu/drm/xe/xe_gt_types.h                   |    22 +-
+ drivers/gpu/drm/xe/xe_guc.c                        |    82 +-
+ drivers/gpu/drm/xe/xe_guc_ads.c                    |    11 +
+ drivers/gpu/drm/xe/xe_guc_ct.c                     |   119 +-
+ drivers/gpu/drm/xe/xe_guc_ct_types.h               |     6 +-
+ drivers/gpu/drm/xe/xe_guc_fwif.h                   |     5 +-
+ drivers/gpu/drm/xe/xe_guc_log.h                    |     6 +-
+ drivers/gpu/drm/xe/xe_guc_pagefault.c              |    18 +-
+ drivers/gpu/drm/xe/xe_guc_pc.c                     |   126 +-
+ drivers/gpu/drm/xe/xe_guc_pc.h                     |     6 +-
+ drivers/gpu/drm/xe/xe_guc_rc.c                     |   131 +
+ drivers/gpu/drm/xe/xe_guc_rc.h                     |    16 +
+ drivers/gpu/drm/xe/xe_guc_submit.c                 |   176 +-
+ drivers/gpu/drm/xe/xe_guc_tlb_inval.c              |   242 +-
+ drivers/gpu/drm/xe/xe_hw_engine.c                  |    14 +-
+ drivers/gpu/drm/xe/xe_hw_engine_group.c            |     3 +-
+ drivers/gpu/drm/xe/xe_hw_error.c                   |   451 +-
+ drivers/gpu/drm/xe/xe_hw_fence.c                   |     3 +-
+ drivers/gpu/drm/xe/xe_i2c.c                        |     2 +-
+ drivers/gpu/drm/xe/xe_lmtt.c                       |    17 +
+ drivers/gpu/drm/xe/xe_lmtt.h                       |     1 +
+ drivers/gpu/drm/xe/xe_lrc.c                        |   310 +-
+ drivers/gpu/drm/xe/xe_lrc.h                        |     5 +
+ drivers/gpu/drm/xe/xe_lrc_types.h                  |     6 +
+ drivers/gpu/drm/xe/xe_migrate.c                    |   262 +-
+ drivers/gpu/drm/xe/xe_migrate.h                    |     4 +
+ drivers/gpu/drm/xe/xe_mmio.c                       |     9 +
+ drivers/gpu/drm/xe/xe_mmio.h                       |     8 +-
+ drivers/gpu/drm/xe/xe_mmio_types.h                 |    64 +
+ drivers/gpu/drm/xe/xe_mocs.c                       |     1 +
+ drivers/gpu/drm/xe/xe_module.c                     |    44 +-
+ drivers/gpu/drm/xe/xe_nvm.c                        |    12 +-
+ drivers/gpu/drm/xe/xe_oa.c                         |    66 +-
+ drivers/gpu/drm/xe/xe_oa_types.h                   |     3 -
+ drivers/gpu/drm/xe/xe_pagefault.c                  |    31 +-
+ drivers/gpu/drm/xe/xe_pagefault_types.h            |    20 +-
+ drivers/gpu/drm/xe/xe_pat.c                        |   162 +-
+ drivers/gpu/drm/xe/xe_pci.c                        |    71 +-
+ drivers/gpu/drm/xe/xe_pci_types.h                  |     5 +
+ drivers/gpu/drm/xe/xe_pcode.c                      |    30 -
+ drivers/gpu/drm/xe/xe_pcode.h                      |     8 -
+ drivers/gpu/drm/xe/xe_platform_types.h             |     1 +
+ drivers/gpu/drm/xe/xe_pxp.c                        |     4 +-
+ drivers/gpu/drm/xe/xe_query.c                      |     5 +-
+ drivers/gpu/drm/xe/xe_reg_sr.c                     |    72 +-
+ drivers/gpu/drm/xe/xe_reg_sr.h                     |     7 +
+ drivers/gpu/drm/xe/xe_reg_whitelist.c              |    12 +-
+ drivers/gpu/drm/xe/xe_res_cursor.h                 |    34 +-
+ drivers/gpu/drm/xe/xe_rtp.c                        |    15 +-
+ drivers/gpu/drm/xe/xe_rtp.h                        |    23 +-
+ drivers/gpu/drm/xe/xe_rtp_types.h                  |     1 +
+ drivers/gpu/drm/xe/xe_sa.c                         |    36 +
+ drivers/gpu/drm/xe/xe_sa.h                         |     2 +
+ drivers/gpu/drm/xe/xe_sched_job.c                  |     4 +-
+ drivers/gpu/drm/xe/xe_sleep.h                      |    57 +
+ drivers/gpu/drm/xe/xe_soc_remapper.c               |     1 +
+ drivers/gpu/drm/xe/xe_sriov.c                      |     2 +-
+ drivers/gpu/drm/xe/xe_sriov.h                      |     7 +-
+ drivers/gpu/drm/xe/xe_sriov_pf.c                   |    10 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_helpers.h           |     3 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_provision.c         |   119 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_provision.h         |     4 +
+ drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c             |    31 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_types.h             |     3 +
+ drivers/gpu/drm/xe/xe_step.c                       |    74 +-
+ drivers/gpu/drm/xe/xe_step.h                       |    10 +-
+ drivers/gpu/drm/xe/xe_step_types.h                 |     1 +
+ drivers/gpu/drm/xe/xe_svm.c                        |    20 +-
+ drivers/gpu/drm/xe/xe_tile.h                       |     4 +-
+ drivers/gpu/drm/xe/xe_tile_sriov_vf.c              |   198 +-
+ drivers/gpu/drm/xe/xe_tile_sriov_vf.h              |     3 -
+ drivers/gpu/drm/xe/xe_tile_sysfs.c                 |     2 +-
+ drivers/gpu/drm/xe/xe_tile_types.h                 |   141 +
+ drivers/gpu/drm/xe/xe_tlb_inval.c                  |   117 +
+ drivers/gpu/drm/xe/xe_tlb_inval.h                  |     8 +
+ drivers/gpu/drm/xe/xe_tlb_inval_types.h            |    14 +
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr.c               |    85 +-
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h         |     4 +-
+ drivers/gpu/drm/xe/xe_tuning.c                     |    36 +-
+ drivers/gpu/drm/xe/xe_uc.c                         |    10 +-
+ drivers/gpu/drm/xe/xe_uc.h                         |     1 -
+ drivers/gpu/drm/xe/xe_uc_fw.c                      |     4 +-
+ drivers/gpu/drm/xe/xe_userptr.c                    |   155 +-
+ drivers/gpu/drm/xe/xe_userptr.h                    |    31 +-
+ drivers/gpu/drm/xe/xe_vm.c                         |   224 +-
+ drivers/gpu/drm/xe/xe_vm.h                         |    15 +-
+ drivers/gpu/drm/xe/xe_vm_madvise.c                 |    10 +-
+ drivers/gpu/drm/xe/xe_vm_types.h                   |    20 +
+ drivers/gpu/drm/xe/xe_vram_freq.c                  |     2 +-
+ drivers/gpu/drm/xe/xe_wa.c                         |   377 +-
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |    39 +-
+ drivers/gpu/drm/xe/xe_wopcm.c                      |    15 +-
+ drivers/gpu/tests/Makefile                         |     4 +
+ drivers/gpu/tests/gpu_buddy_test.c                 |  1421 +
+ .../{drm/lib/drm_random.c => tests/gpu_random.c}   |    18 +-
+ .../{drm/lib/drm_random.h => tests/gpu_random.h}   |    18 +-
+ drivers/infiniband/core/ib_core_uverbs.c           |     2 +-
+ drivers/infiniband/core/umem_dmabuf.c              |    10 -
+ drivers/infiniband/core/uverbs_std_types_dmabuf.c  |     2 +-
+ drivers/infiniband/hw/mlx5/mr.c                    |     2 +-
+ drivers/iommu/iommufd/pages.c                      |    11 +-
+ drivers/iommu/iommufd/selftest.c                   |     2 +-
+ drivers/vfio/pci/vfio_pci_dmabuf.c                 |    80 +-
+ drivers/video/Kconfig                              |     1 +
+ include/drm/bridge/dw_dp.h                         |     7 +
+ include/drm/drm_atomic.h                           |    21 +-
+ include/drm/drm_atomic_state_helper.h              |     3 +
+ include/drm/drm_buddy.h                            |   163 +-
+ include/drm/drm_client.h                           |     3 +
+ include/drm/drm_colorop.h                          |    32 +-
+ include/drm/drm_connector.h                        |     1 +
+ include/drm/drm_fb_helper.h                        |   107 +-
+ include/drm/drm_mode_config.h                      |     4 +
+ include/drm/drm_ras.h                              |    75 +
+ include/drm/drm_ras_genl_family.h                  |    17 +
+ include/drm/drm_suballoc.h                         |     6 +
+ include/drm/intel/display_parent_interface.h       |   108 +
+ include/drm/intel/i915_drm.h                       |    82 +-
+ include/drm/intel/intel_gmd_interrupt_regs.h       |    92 +
+ include/drm/intel/intel_gmd_misc_regs.h            |    21 +
+ include/drm/intel/intel_pcode_regs.h               |   108 +
+ include/drm/intel/pciids.h                         |    12 +
+ include/drm/intel/pick.h                           |    51 +
+ include/drm/intel/reg_bits.h                       |   139 +
+ include/linux/coreboot.h                           |    90 +
+ include/linux/dma-buf.h                            |    17 +-
+ include/linux/dma-fence-array.h                    |     1 -
+ include/linux/dma-fence-chain.h                    |     1 -
+ include/linux/dma-fence.h                          |    97 +-
+ include/linux/gpu_buddy.h                          |   241 +
+ include/linux/iopoll.h                             |     8 +-
+ include/linux/mmu_notifier.h                       |    42 +
+ include/trace/events/dma_fence.h                   |    35 +-
+ include/uapi/drm/amdgpu_drm.h                      |     7 +-
+ include/uapi/drm/amdxdna_accel.h                   |     3 +-
+ include/uapi/drm/drm_fourcc.h                      |    16 +
+ include/uapi/drm/drm_mode.h                        |     4 +
+ include/uapi/drm/drm_ras.h                         |    49 +
+ include/uapi/drm/nouveau_drm.h                     |    66 +
+ include/uapi/drm/xe_drm.h                          |   109 +-
+ include/video/vga.h                                |     1 +
+ mm/mmu_notifier.c                                  |    65 +-
+ 1008 files changed, 132822 insertions(+), 17387 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/bridge/thead,th1520-dw-hdmi.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/lxd,m9189a.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/motorola,mot-panel.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/tilcdc/ti,am33xx-tilcdc.yaml
+ delete mode 100644 Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
+ create mode 100644 Documentation/devicetree/bindings/display/verisilicon,dc.yaml
+ create mode 100644 Documentation/gpu/drm-ras.rst
+ create mode 100644 Documentation/netlink/specs/drm_ras.yaml
+ create mode 100644 drivers/gpu/Kconfig
+ create mode 100644 drivers/gpu/buddy.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
+ rename drivers/gpu/drm/amd/display/dc/{clk_mgr/dce60/dce60_clk_mgr.h => resource/dcn42/dcn42_resource_fpu.h} (78%)
+ create mode 100644 drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
+ create mode 100644 drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
+ create mode 100644 drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.h
+ create mode 100644 drivers/gpu/drm/bridge/th1520-dw-hdmi.c
+ create mode 100644 drivers/gpu/drm/drm_ras.c
+ create mode 100644 drivers/gpu/drm/drm_ras_genl_family.c
+ create mode 100644 drivers/gpu/drm/drm_ras_nl.c
+ create mode 100644 drivers/gpu/drm/drm_ras_nl.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_dpt_common.c
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_dpt_common.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_dsb_buffer.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fb_bo.c
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fb_bo.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_oprom_regs.h
+ create mode 100644 drivers/gpu/drm/i915/i915_bo.c
+ create mode 100644 drivers/gpu/drm/i915/i915_bo.h
+ create mode 100644 drivers/gpu/drm/i915/i915_dpt.c
+ create mode 100644 drivers/gpu/drm/i915/i915_dpt.h
+ rename drivers/gpu/drm/i915/{display/intel_dsb_buffer.c => i915_dsb_buffer.c} (63%)
+ create mode 100644 drivers/gpu/drm/i915/i915_dsb_buffer.h
+ create mode 100644 drivers/gpu/drm/i915/i915_overlay.c
+ create mode 100644 drivers/gpu/drm/i915/i915_overlay.h
+ create mode 100644 drivers/gpu/drm/panel/panel-lxd-m9189a.c
+ create mode 100644 drivers/gpu/drm/panel/panel-motorola-mot.c
+ create mode 100644 drivers/gpu/drm/panel/panel-novatek-nt37700f.c
+ create mode 100644 drivers/gpu/drm/panthor/panthor_trace.h
+ create mode 100644 drivers/gpu/drm/sysfb/corebootdrm.c
+ delete mode 100644 drivers/gpu/drm/tests/drm_buddy_test.c
+ create mode 100644 drivers/gpu/drm/tilcdc/tilcdc_encoder.c
+ rename drivers/gpu/drm/tilcdc/{tilcdc_external.h => tilcdc_encoder.h} (51%)
+ delete mode 100644 drivers/gpu/drm/tilcdc/tilcdc_external.c
+ delete mode 100644 drivers/gpu/drm/tilcdc/tilcdc_panel.h
+ create mode 100644 drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c
+ create mode 100644 drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.dtso
+ create mode 100644 drivers/gpu/drm/verisilicon/Kconfig
+ create mode 100644 drivers/gpu/drm/verisilicon/Makefile
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_bridge.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_bridge.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_bridge_regs.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_crtc.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_crtc.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_crtc_regs.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_dc.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_dc.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_dc_top_regs.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_drm.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_drm.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_hwdb.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_hwdb.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_plane.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_plane.h
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_primary_plane.c
+ create mode 100644 drivers/gpu/drm/verisilicon/vs_primary_plane_regs.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_reg.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_vma.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h
+ delete mode 100644 drivers/gpu/drm/xe/display/intel_bo.c
+ rename drivers/gpu/drm/xe/display/{intel_fb_bo.c => xe_display_bo.c} (57%)
+ create mode 100644 drivers/gpu/drm/xe/display/xe_display_bo.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_display_pcode.c
+ create mode 100644 drivers/gpu/drm/xe/display/xe_display_pcode.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_display_vma.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_dsb_buffer.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_frontbuffer.c
+ create mode 100644 drivers/gpu/drm/xe/display/xe_frontbuffer.h
+ create mode 100644 drivers/gpu/drm/xe/xe_defaults.h
+ create mode 100644 drivers/gpu/drm/xe/xe_drm_ras.c
+ create mode 100644 drivers/gpu/drm/xe/xe_drm_ras.h
+ create mode 100644 drivers/gpu/drm/xe/xe_drm_ras_types.h
+ create mode 100644 drivers/gpu/drm/xe/xe_guc_rc.c
+ create mode 100644 drivers/gpu/drm/xe/xe_guc_rc.h
+ create mode 100644 drivers/gpu/drm/xe/xe_mmio_types.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sleep.h
+ create mode 100644 drivers/gpu/drm/xe/xe_tile_types.h
+ create mode 100644 drivers/gpu/tests/Makefile
+ create mode 100644 drivers/gpu/tests/gpu_buddy_test.c
+ rename drivers/gpu/{drm/lib/drm_random.c => tests/gpu_random.c} (59%)
+ rename drivers/gpu/{drm/lib/drm_random.h => tests/gpu_random.h} (53%)
+ create mode 100644 include/drm/drm_ras.h
+ create mode 100644 include/drm/drm_ras_genl_family.h
+ create mode 100644 include/drm/intel/intel_gmd_interrupt_regs.h
+ create mode 100644 include/drm/intel/intel_gmd_misc_regs.h
+ create mode 100644 include/drm/intel/intel_pcode_regs.h
+ create mode 100644 include/drm/intel/pick.h
+ create mode 100644 include/drm/intel/reg_bits.h
+ create mode 100644 include/linux/coreboot.h
+ create mode 100644 include/linux/gpu_buddy.h
+ create mode 100644 include/uapi/drm/drm_ras.h
+$ git am -3 ../patches/0001-drm-rdma-Fix-up-merge.patch
+Applying: drm/rdma: Fix up merge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/infiniband/core/umem_dmabuf.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 5d8f5166992a1] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+ Date: Fri Mar 20 14:17:40 2026 +0000
+Merging drm-exynos/for-linux-next (3a8660878839f Linux 6.18-rc1)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git drm-exynos/for-linux-next
+Already up to date.
+Merging drm-misc/for-linux-next (fbef867cf6614 drm/bridge: samsung-dsim: use drm_bridge_clear_and_put() to put the next bridge)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc/for-linux-next
+Auto-merging drivers/accel/amdxdna/aie2_ctx.c
+Auto-merging drivers/accel/amdxdna/amdxdna_ctx.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Auto-merging drivers/gpu/drm/drm_mode_config.c
+Merge made by the 'ort' strategy.
+ .../bindings/display/bridge/lontium,lt8713sx.yaml  | 113 ++++
+ drivers/accel/amdxdna/Makefile                     |   1 +
+ drivers/accel/amdxdna/aie2_ctx.c                   |  91 +++-
+ drivers/accel/amdxdna/aie2_error.c                 |   5 +-
+ drivers/accel/amdxdna/aie2_message.c               | 105 +++-
+ drivers/accel/amdxdna/aie2_msg_priv.h              |  52 ++
+ drivers/accel/amdxdna/aie2_pci.c                   |  14 +
+ drivers/accel/amdxdna/aie2_pci.h                   |  10 +-
+ drivers/accel/amdxdna/amdxdna_ctx.c                |   6 +-
+ drivers/accel/amdxdna/amdxdna_ctx.h                |  18 +-
+ drivers/accel/amdxdna/amdxdna_gem.c                |  45 +-
+ drivers/accel/amdxdna/amdxdna_gem.h                |  10 +
+ drivers/accel/amdxdna/amdxdna_iommu.c              | 184 +++++++
+ drivers/accel/amdxdna/amdxdna_pci_drv.c            |  45 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.h            |  24 +
+ drivers/accel/amdxdna/npu4_regs.c                  |   3 +-
+ drivers/accel/ivpu/ivpu_debugfs.c                  |  14 +-
+ drivers/accel/ivpu/ivpu_drv.c                      |   1 +
+ drivers/accel/ivpu/ivpu_drv.h                      |   3 +-
+ drivers/accel/ivpu/ivpu_job.c                      |  50 +-
+ drivers/accel/ivpu/ivpu_jsm_msg.c                  |  19 +-
+ drivers/accel/ivpu/ivpu_jsm_msg.h                  |   3 +-
+ drivers/accel/ivpu/ivpu_mmu.c                      |   3 +-
+ drivers/accel/ivpu/ivpu_pm.c                       |  15 +-
+ drivers/accel/ivpu/ivpu_pm.h                       |   1 +
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |  51 +-
+ .../drm/arm/display/komeda/komeda_private_obj.c    |  16 +-
+ drivers/gpu/drm/bridge/Kconfig                     |  10 +
+ drivers/gpu/drm/bridge/Makefile                    |   1 +
+ drivers/gpu/drm/bridge/lontium-lt8713sx.c          | 598 +++++++++++++++++++++
+ drivers/gpu/drm/bridge/samsung-dsim.c              |   7 +-
+ drivers/gpu/drm/bridge/waveshare-dsi.c             |   8 +-
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      |   1 -
+ drivers/gpu/drm/display/drm_dp_tunnel.c            |   2 +-
+ drivers/gpu/drm/drm_atomic.c                       |  30 +-
+ drivers/gpu/drm/drm_atomic_state_helper.c          |   1 +
+ drivers/gpu/drm/drm_atomic_uapi.c                  |   4 +
+ drivers/gpu/drm/drm_blend.c                        |  39 +-
+ drivers/gpu/drm/drm_bridge.c                       |  35 +-
+ drivers/gpu/drm/drm_colorop.c                      |  28 +-
+ drivers/gpu/drm/drm_mode_config.c                  |   6 +
+ drivers/gpu/drm/imx/ipuv3/Kconfig                  |   1 +
+ drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c            |  23 +-
+ drivers/gpu/drm/imx/ipuv3/parallel-display.c       |  15 +-
+ drivers/gpu/drm/ingenic/ingenic-drm-drv.c          |   2 +-
+ drivers/gpu/drm/ingenic/ingenic-ipu.c              |   2 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c            |   1 -
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c           |   1 -
+ drivers/gpu/drm/omapdrm/omap_drv.c                 |  22 +-
+ drivers/gpu/drm/panel/panel-edp.c                  |   3 +
+ drivers/gpu/drm/panel/panel-novatek-nt36672a.c     |  94 ++--
+ drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c     |   3 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.c       |  24 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.h       |   4 +
+ drivers/gpu/drm/sun4i/sun4i_backend.c              |   3 +-
+ drivers/gpu/drm/sun4i/sun8i_ui_layer.c             |  16 +-
+ drivers/gpu/drm/sun4i/sun8i_vi_layer.c             |  27 +-
+ drivers/gpu/drm/tegra/hub.c                        |  22 +-
+ drivers/gpu/drm/tilcdc/tilcdc_panel.c              | 408 --------------
+ drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c       |   4 +-
+ drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c   |  41 ++
+ drivers/gpu/drm/ttm/ttm_backup.c                   |   6 +-
+ drivers/gpu/drm/ttm/ttm_bo.c                       |   6 +-
+ drivers/gpu/drm/ttm/ttm_pool.c                     |   7 +-
+ drivers/gpu/drm/v3d/v3d_drv.c                      |  55 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |  46 +-
+ drivers/gpu/drm/v3d/v3d_fence.c                    |   2 +-
+ drivers/gpu/drm/v3d/v3d_gem.c                      |  43 +-
+ drivers/gpu/drm/v3d/v3d_irq.c                      |   2 +-
+ drivers/gpu/drm/v3d/v3d_sched.c                    | 133 ++---
+ drivers/gpu/drm/v3d/v3d_submit.c                   |   6 +
+ drivers/gpu/drm/v3d/v3d_sysfs.c                    |   2 +-
+ drivers/gpu/drm/vc4/vc4_kms.c                      |   4 +-
+ drivers/gpu/drm/vkms/vkms_composer.c               |   8 +-
+ drivers/gpu/drm/vkms/vkms_crtc.c                   |   3 +
+ include/drm/drm_atomic.h                           |   1 -
+ include/drm/drm_blend.h                            |   4 +-
+ include/drm/drm_bridge.h                           |   1 +
+ include/drm/drm_crtc.h                             |  12 +
+ include/drm/drm_mode_config.h                      |   5 +
+ include/drm/ttm/ttm_backup.h                       |   2 +-
+ include/drm/ttm/ttm_bo.h                           |  28 +-
+ include/linux/math.h                               |  18 +-
+ include/uapi/drm/drm_mode.h                        |  80 +++
+ include/uapi/linux/const.h                         |  18 +
+ 85 files changed, 2014 insertions(+), 866 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/bridge/lontium,lt8713sx.yaml
+ create mode 100644 drivers/accel/amdxdna/amdxdna_iommu.c
+ create mode 100644 drivers/gpu/drm/bridge/lontium-lt8713sx.c
+ delete mode 100644 drivers/gpu/drm/tilcdc/tilcdc_panel.c
+Merging amdgpu/drm-next (daf9f9dd50911 drm/amdgpu: Add client ids for gmcv9 mmhubs)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git amdgpu/drm-next
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/amd/amdgpu/Makefile                |    2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |    6 +
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c   |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c        |    4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c     |    2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |    7 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c   |   83 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h   |    7 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |   41 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |   10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c |  237 ++--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h |   55 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c     |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |   17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |    5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h          |   26 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            |    7 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c     |    1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c            |   11 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h            |    3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |   64 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          |    3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |  642 ++++++-----
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |  181 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |   17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c          |    4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c           |   12 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |   80 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h         |  223 ++--
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c          |  193 ++--
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c             |   76 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c             |   29 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |   78 +-
+ drivers/gpu/drm/amd/amdgpu/isp_v4_1_1.c            |    4 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c           |  840 ++++++++++++++
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.h           |  111 ++
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |  392 ++++++-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c            |   46 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c            |   17 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c            |   16 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c          |   17 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c          |    9 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c            |   55 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c          |   15 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c          |  194 ++--
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |   21 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |   69 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.h              |    2 +
+ drivers/gpu/drm/amd/amdgpu/umc_v12_0.c             |  117 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c            | 1219 ++++++++++++++++++++
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.h            |   29 +
+ drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c   |    6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.c            |   12 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.h            |    3 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c   |    6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c   |    6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c |   18 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c    |    6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |    2 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |    3 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_svm.c               |   30 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   64 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |   21 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c  |    6 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c  |   52 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c    |   11 +-
+ drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c   |    8 +-
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   10 +
+ drivers/gpu/drm/amd/display/dc/dc.h                |   11 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |   27 +
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h       |    9 +
+ .../amd/display/dc/dml/dcn21/display_mode_vba_21.c |    2 +-
+ .../dc/dml/dcn32/display_mode_vba_util_32.c        |    6 +-
+ .../dml2_0/dml21/inc/dml_top_soc_parameter_types.h |    1 +
+ .../dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c |    1 +
+ .../dml21/src/dml2_core/dml2_core_dcn4_calcs.c     |   22 +
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |    7 +-
+ drivers/gpu/drm/amd/display/dc/inc/core_types.h    |    1 +
+ .../amd/display/dc/resource/dcn10/dcn10_resource.c |    9 +-
+ .../amd/display/dc/resource/dcn10/dcn10_resource.h |    2 +
+ .../amd/display/dc/resource/dcn20/dcn20_resource.c |    3 +-
+ .../display/dc/resource/dcn201/dcn201_resource.c   |    3 +-
+ .../amd/display/dc/resource/dcn21/dcn21_resource.c |    3 +-
+ .../amd/display/dc/resource/dcn30/dcn30_resource.c |    3 +-
+ .../display/dc/resource/dcn301/dcn301_resource.c   |    3 +-
+ .../display/dc/resource/dcn302/dcn302_resource.c   |    3 +-
+ .../display/dc/resource/dcn303/dcn303_resource.c   |    3 +-
+ .../amd/display/dc/resource/dcn31/dcn31_resource.c |    1 +
+ .../display/dc/resource/dcn314/dcn314_resource.c   |    3 +-
+ .../display/dc/resource/dcn315/dcn315_resource.c   |    3 +-
+ .../display/dc/resource/dcn316/dcn316_resource.c   |    3 +-
+ .../amd/display/dc/resource/dcn32/dcn32_resource.c |    4 +
+ .../display/dc/resource/dcn321/dcn321_resource.c   |    1 +
+ .../amd/display/dc/resource/dcn35/dcn35_resource.c |    3 +-
+ .../display/dc/resource/dcn351/dcn351_resource.c   |    3 +-
+ .../amd/display/dc/resource/dcn36/dcn36_resource.c |    3 +-
+ .../display/dc/resource/dcn401/dcn401_resource.c   |    9 +-
+ .../display/dc/resource/dcn401/dcn401_resource.h   |    2 +
+ drivers/gpu/drm/amd/include/atomfirmware.h         |    5 +-
+ drivers/gpu/drm/amd/pm/amdgpu_pm.c                 |   36 +-
+ drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c         |    4 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |    1 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h       |    2 -
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h       |    2 -
+ drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c |    2 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |   43 -
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c   |    2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c   |    2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c   |    2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |    2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c   |    2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c   |    2 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c     |   52 -
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c   |    3 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c             |   29 +
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h             |    1 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |   31 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h   |    1 +
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c   |   10 +
+ drivers/gpu/drm/amd/ras/rascore/ras_core.c         |   11 +-
+ drivers/gpu/drm/radeon/si_dpm.c                    |    4 +-
+ 126 files changed, 4588 insertions(+), 1421 deletions(-)
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.h
+$ git am -3 ../patches/0001-drm-fbdev-emulation-restore-drm_fb_helper_gem_is_fb-.patch
+Applying: drm/fbdev-emulation: restore drm_fb_helper_gem_is_fb() dummy
+$ git reset HEAD^
+Unstaged changes after reset:
+M	include/drm/drm_fb_helper.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 4f06549e47fd1] Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git
+ Date: Fri Mar 20 14:17:49 2026 +0000
+Merging drm-intel/for-linux-next (a464bace0482a drm/i915: Order OP vs. timeout correctly in __wait_for())
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel/for-linux-next
+Auto-merging drivers/gpu/drm/i915/display/intel_display_device.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/intel_display.c       |  3 +-
+ .../gpu/drm/i915/display/intel_display_device.c    | 28 ++++++++--
+ .../gpu/drm/i915/display/intel_display_device.h    |  1 +
+ drivers/gpu/drm/i915/display/intel_dmc.c           | 19 +++----
+ .../gpu/drm/i915/display/intel_dp_aux_backlight.c  | 32 ++++++++++-
+ drivers/gpu/drm/i915/display/intel_gmbus.c         |  6 ++-
+ drivers/gpu/drm/i915/display/intel_psr.c           |  7 +++
+ drivers/gpu/drm/i915/display/intel_quirks.c        | 17 +++++-
+ drivers/gpu/drm/i915/display/intel_quirks.h        |  1 +
+ drivers/gpu/drm/i915/gvt/edid.c                    | 18 +++----
+ drivers/gpu/drm/i915/i915_initial_plane.c          |  3 +-
+ drivers/gpu/drm/i915/i915_wait_util.h              |  2 +-
+ drivers/gpu/drm/i915/intel_step.h                  | 57 +-------------------
+ .../gpu/drm/xe/compat-i915-headers/intel_step.h    |  3 +-
+ include/drm/intel/step.h                           | 62 ++++++++++++++++++++++
+ 15 files changed, 171 insertions(+), 88 deletions(-)
+ create mode 100644 include/drm/intel/step.h
+Merging drm-msm/msm-next (50c4a49f7292b drm/msm/a8xx: Add UBWC v6 support)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Already up to date.
+Merging drm-msm-lumag/msm-next-lumag (50c4a49f7292b drm/msm/a8xx: Add UBWC v6 support)
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Already up to date.
+Merging drm-xe/drm-xe-next (2c440f2fccf35 drm/xe: Extend Wa_14026781792 for xe3lpg)
+$ git merge -m Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git drm-xe/drm-xe-next
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/drm_gpusvm.c         |   7 +-
+ drivers/gpu/drm/drm_pagemap.c        | 157 ++++++++++++++++++++++++++++-------
+ drivers/gpu/drm/ttm/ttm_resource.c   |  28 +++----
+ drivers/gpu/drm/xe/xe_ggtt.c         |  15 ++--
+ drivers/gpu/drm/xe/xe_gt_ccs_mode.c  |   2 +
+ drivers/gpu/drm/xe/xe_guc.c          |  32 +++++--
+ drivers/gpu/drm/xe/xe_guc.h          |   1 +
+ drivers/gpu/drm/xe/xe_guc_ct.c       |  24 ++----
+ drivers/gpu/drm/xe/xe_guc_submit.c   |  87 +++++++++++++------
+ drivers/gpu/drm/xe/xe_i2c.c          |  15 +++-
+ drivers/gpu/drm/xe/xe_lrc.c          |  69 ++++++++-------
+ drivers/gpu/drm/xe/xe_oa.c           |   7 +-
+ drivers/gpu/drm/xe/xe_page_reclaim.c |  26 ++++++
+ drivers/gpu/drm/xe/xe_page_reclaim.h |   3 +
+ drivers/gpu/drm/xe/xe_pt.c           |  38 +++++++--
+ drivers/gpu/drm/xe/xe_uc.c           |  22 ++---
+ drivers/gpu/drm/xe/xe_wa.c           |  24 ++----
+ include/drm/drm_pagemap.h            |  21 +++++
+ include/uapi/drm/xe_drm.h            |   2 +-
+ 19 files changed, 407 insertions(+), 173 deletions(-)
+$ git am -3 ../patches/0001-drm-xe-Fix-up-merge-issue.patch
+Applying: drm: xe: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging etnaviv/etnaviv/next (6bde14ba5f7ef drm/etnaviv: add optional reset support)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Already up to date.
+Merging drm-rust/drm-rust-next (a19457958c301 gpu: nova-core: gsp: add mutex locking to Cmdq)
+$ git merge -m Merge branch 'drm-rust-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust/drm-rust-next
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/tyr/driver.rs                      |   98 +-
+ drivers/gpu/drm/tyr/file.rs                        |   34 +-
+ drivers/gpu/drm/tyr/gem.rs                         |   17 +-
+ drivers/gpu/drm/tyr/gpu.rs                         |   56 +-
+ drivers/gpu/drm/tyr/regs.rs                        |   16 +-
+ drivers/gpu/drm/tyr/tyr.rs                         |    4 +-
+ drivers/gpu/nova-core/Kconfig                      |    2 +-
+ drivers/gpu/nova-core/driver.rs                    |   17 +-
+ drivers/gpu/nova-core/falcon.rs                    |  315 ++++-
+ drivers/gpu/nova-core/falcon/hal.rs                |    6 +-
+ drivers/gpu/nova-core/fb.rs                        |   89 +-
+ drivers/gpu/nova-core/firmware.rs                  |  116 +-
+ drivers/gpu/nova-core/firmware/booter.rs           |   87 +-
+ drivers/gpu/nova-core/firmware/fwsec.rs            |  181 ++-
+ drivers/gpu/nova-core/firmware/fwsec/bootloader.rs |  347 ++++++
+ drivers/gpu/nova-core/firmware/riscv.rs            |    6 +-
+ drivers/gpu/nova-core/gpu.rs                       |   13 +-
+ drivers/gpu/nova-core/gsp.rs                       |    5 +-
+ drivers/gpu/nova-core/gsp/boot.rs                  |   64 +-
+ drivers/gpu/nova-core/gsp/cmdq.rs                  |  359 ++++--
+ drivers/gpu/nova-core/gsp/cmdq/continuation.rs     |  307 +++++
+ drivers/gpu/nova-core/gsp/commands.rs              |   23 +-
+ drivers/gpu/nova-core/gsp/fw.rs                    |  179 +--
+ drivers/gpu/nova-core/gsp/fw/commands.rs           |   17 +-
+ drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs  |    1 +
+ drivers/gpu/nova-core/gsp/sequencer.rs             |   22 +-
+ drivers/gpu/nova-core/nova_core.rs                 |    2 +-
+ drivers/gpu/nova-core/regs.rs                      |   30 +
+ rust/kernel/drm/device.rs                          |   26 +-
+ rust/kernel/drm/driver.rs                          |   10 +-
+ rust/kernel/drm/file.rs                            |    8 +-
+ rust/kernel/drm/gem/mod.rs                         |   23 +-
+ rust/kernel/error.rs                               |    1 +
+ rust/kernel/io.rs                                  |  782 +++++++-----
+ rust/kernel/io/mem.rs                              |   10 +-
+ rust/kernel/io/register.rs                         | 1260 ++++++++++++++++++++
+ rust/kernel/lib.rs                                 |    3 +
+ rust/kernel/num/bounded.rs                         |   70 +-
+ rust/kernel/pci/io.rs                              |   99 +-
+ samples/rust/rust_driver_pci.rs                    |   90 +-
+ scripts/Makefile.build                             |    3 +-
+ 41 files changed, 3745 insertions(+), 1053 deletions(-)
+ create mode 100644 drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
+ create mode 100644 drivers/gpu/nova-core/gsp/cmdq/continuation.rs
+ create mode 100644 rust/kernel/io/register.rs
+$ git am -3 ../patches/0001-drm-nova-Fix-up-merge-issue.patch
+Applying: drm: nova: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/nova-core/gsp/cmdq.rs
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/nova-core/gsp/cmdq.rs
+No changes -- Patch already applied.
+Merging drm-nova/nova-next (93296e9d9528f gpu: nova-core: vbios: store reference to Device where relevant)
+$ git merge -m Merge branch 'nova-next' of https://gitlab.freedesktop.org/drm/nova.git drm-nova/nova-next
+Already up to date.
+Merging fbdev/for-next (d3da7f00e21c2 fbdev: omapfb: Add missing error check for clk_get())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Auto-merging include/linux/console_struct.h
+Merge made by the 'ort' strategy.
+ drivers/video/console/newport_con.c |  61 +++------
+ drivers/video/fbdev/core/bitblit.c  |  11 +-
+ drivers/video/fbdev/core/fb_defio.c | 266 ++++++++++++++++++++++++++----------
+ drivers/video/fbdev/core/fbcon.c    | 194 +++++++-------------------
+ drivers/video/fbdev/core/fbcon.h    |   8 +-
+ drivers/video/fbdev/core/fbmem.c    |   6 +-
+ drivers/video/fbdev/goldfishfb.c    |  14 +-
+ drivers/video/fbdev/macfb.c         |  38 +++---
+ drivers/video/fbdev/omap/hwa742.c   |   4 +
+ drivers/video/fbdev/tdfxfb.c        | 103 +++++++-------
+ drivers/video/fbdev/via/lcd.c       |   3 +
+ drivers/video/fbdev/wmt_ge_rops.c   |  18 +--
+ include/linux/console_struct.h      |  59 +++++++-
+ include/linux/fb.h                  |  19 ++-
+ include/linux/font.h                | 115 ++++++++++------
+ lib/fonts/font.h                    |  38 ++++++
+ lib/fonts/font_10x18.c              |   2 +-
+ lib/fonts/font_6x10.c               |   3 +-
+ lib/fonts/font_6x11.c               |   2 +-
+ lib/fonts/font_6x8.c                |   3 +-
+ lib/fonts/font_7x14.c               |   2 +-
+ lib/fonts/font_8x16.c               |   3 +-
+ lib/fonts/font_8x8.c                |   2 +-
+ lib/fonts/font_acorn_8x8.c          |   4 +-
+ lib/fonts/font_mini_4x6.c           |  10 +-
+ lib/fonts/font_pearl_8x8.c          |   2 +-
+ lib/fonts/font_sun12x22.c           |   3 +-
+ lib/fonts/font_sun8x16.c            |   3 +-
+ lib/fonts/font_ter10x18.c           |   4 +-
+ lib/fonts/font_ter16x32.c           |   4 +-
+ lib/fonts/fonts.c                   | 232 ++++++++++++++++++++++++++++++-
+ 31 files changed, 794 insertions(+), 442 deletions(-)
+ create mode 100644 lib/fonts/font.h
+$ git am -3 ../patches/0001-fix-up-for-drm-hyperv-Remove-reference-to-hyperv_fb-.patch
+Applying: fix up for "drm/hyperv: Remove reference to hyperv_fb driver"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/hyperv/Kconfig
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master e19fa2d2679c8] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git
+ Date: Fri Mar 20 14:18:04 2026 +0000
+Merging regmap/for-next (10ec8c4403296 Merge remote-tracking branch 'regmap/for-7.1' into regmap-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/internal.h |   2 +-
+ drivers/base/regmap/regcache.c | 102 ++++++++++++++++++++++-------------------
+ drivers/base/regmap/regmap.c   |  23 ++++------
+ include/linux/regmap.h         |  15 +++---
+ 4 files changed, 74 insertions(+), 68 deletions(-)
+Merging sound/for-next (df3eec203b940 ALSA: usb-audio: validate full match when resolving quirk aliases)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Auto-merging sound/core/pcm_native.c
+Auto-merging sound/usb/quirks.c
+Merge made by the 'ort' strategy.
+ Documentation/sound/alsa-configuration.rst |  7 +++
+ include/sound/core.h                       |  3 ++
+ include/sound/pcm.h                        |  4 ++
+ include/sound/timer.h                      |  1 +
+ sound/aoa/aoa.h                            |  2 +-
+ sound/aoa/fabrics/layout.c                 | 28 +++++-----
+ sound/aoa/soundbus/i2sbus/core.c           |  3 ++
+ sound/aoa/soundbus/i2sbus/pcm.c            | 16 ++----
+ sound/core/Makefile                        |  1 +
+ sound/core/control.c                       | 76 ++++++++++++++++++++++++++-
+ sound/core/control_trace.h                 | 55 +++++++++++++++++++
+ sound/core/init.c                          |  8 +++
+ sound/core/oss/pcm_oss.c                   | 44 +++++++++-------
+ sound/core/pcm_native.c                    | 23 +++++++-
+ sound/core/timer.c                         | 29 +++++++----
+ sound/hda/codecs/senarytech.c              | 73 +++++++++++++++++++++++---
+ sound/usb/card.c                           | 44 ++++++++--------
+ sound/usb/format.c                         |  4 ++
+ sound/usb/midi2.c                          |  6 +--
+ sound/usb/mixer.c                          | 84 ++++++++++++++++++++++++------
+ sound/usb/mixer_quirks.c                   | 59 +++++++++++++++++++++
+ sound/usb/mixer_s1810c.c                   |  2 +
+ sound/usb/quirks.c                         | 75 ++++++++++++++++++++++++++
+ sound/usb/stream.c                         |  8 +++
+ sound/usb/usbaudio.h                       | 12 +++++
+ 25 files changed, 563 insertions(+), 104 deletions(-)
+ create mode 100644 sound/core/control_trace.h
+Merging ieee1394/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Already up to date.
+Merging sound-asoc/for-next (5113dd03be757 Merge remote-tracking branch 'asoc/for-7.1' into asoc-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/adi,ssm2305.txt      |   14 -
+ .../devicetree/bindings/sound/adi,ssm2305.yaml     |   46 +
+ .../devicetree/bindings/sound/awinic,aw88395.yaml  |    4 +
+ .../devicetree/bindings/sound/cirrus,cs42l43.yaml  |    3 +
+ .../devicetree/bindings/sound/imx-audio-card.yaml  |    9 +-
+ .../bindings/sound/nvidia,tegra-audio-cpcap.yaml   |   90 ++
+ .../sound/nvidia,tegra-audio-max9808x.yaml         |    5 +
+ .../bindings/sound/nvidia,tegra-audio-wm8962.yaml  |   88 ++
+ .../devicetree/bindings/sound/simple-card.yaml     |   14 +-
+ .../devicetree/bindings/sound/tdm-slot.txt         |   29 -
+ .../devicetree/bindings/sound/tdm-slot.yaml        |   52 +
+ .../devicetree/bindings/sound/ti,tas2770.yaml      |    4 +-
+ arch/arm/mach-s3c/mach-crag6410-module.c           |    6 +-
+ drivers/firmware/cirrus/cs_dsp.c                   |  117 +-
+ drivers/firmware/cirrus/cs_dsp.h                   |    4 +-
+ drivers/firmware/cirrus/test/cs_dsp_test_bin.c     |   25 +-
+ .../firmware/cirrus/test/cs_dsp_test_bin_error.c   |   22 +-
+ .../cirrus/test/cs_dsp_test_control_cache.c        |    5 +
+ .../cirrus/test/cs_dsp_test_control_parse.c        |    5 +
+ .../firmware/cirrus/test/cs_dsp_test_control_rw.c  |    5 +
+ drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c    |   32 +-
+ .../firmware/cirrus/test/cs_dsp_test_wmfw_error.c  |   30 +-
+ drivers/mfd/cs42l43-i2c.c                          |    7 +-
+ drivers/mfd/cs42l43-sdw.c                          |    4 +-
+ drivers/mfd/cs42l43.c                              |   93 +-
+ drivers/mfd/cs42l43.h                              |    2 +-
+ include/linux/firmware/cirrus/cs_dsp.h             |    3 +
+ include/linux/mfd/arizona/pdata.h                  |   10 -
+ include/linux/mfd/cs42l43-regs.h                   |   76 ++
+ include/linux/mfd/cs42l43.h                        |    1 +
+ include/sound/cs35l56.h                            |    6 +-
+ include/sound/simple_card_utils.h                  |    6 +
+ include/sound/soc-component.h                      |    8 +-
+ include/sound/soc-dai.h                            |   22 +
+ include/sound/soc-dapm.h                           |    1 +
+ include/sound/soc.h                                |   13 +-
+ include/sound/soc_sdw_utils.h                      |    2 +
+ sound/hda/codecs/side-codecs/Kconfig               |    1 +
+ sound/soc/amd/Kconfig                              |   10 -
+ sound/soc/amd/Makefile                             |    1 -
+ sound/soc/amd/acp-da7219-max98357a.c               |    1 -
+ sound/soc/amd/acp-es8336.c                         |    1 -
+ sound/soc/amd/acp-pcm-dma.c                        |    2 +-
+ sound/soc/amd/acp-rt5645.c                         |   22 +-
+ sound/soc/amd/acp/acp-platform.c                   |    2 +-
+ sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c      |    1 -
+ sound/soc/amd/acp3x-rt5682-max9836.c               |    1 -
+ sound/soc/amd/include/acp_2_2_d.h                  |   18 +-
+ sound/soc/amd/include/acp_2_2_enum.h               |   18 +-
+ sound/soc/amd/include/acp_2_2_sh_mask.h            |   18 +-
+ sound/soc/amd/ps/ps-pdm-dma.c                      |    2 +-
+ sound/soc/amd/ps/ps-sdw-dma.c                      |    2 +-
+ sound/soc/amd/raven/acp3x-pcm-dma.c                |    2 +-
+ sound/soc/amd/renoir/acp3x-pdm-dma.c               |    2 +-
+ sound/soc/amd/rpl/Makefile                         |    5 -
+ sound/soc/amd/rpl/rpl-pci-acp6x.c                  |  227 ----
+ sound/soc/amd/rpl/rpl_acp6x.h                      |   36 -
+ sound/soc/amd/rpl/rpl_acp6x_chip_offset_byte.h     |   30 -
+ sound/soc/amd/vangogh/acp5x-pcm-dma.c              |    2 +-
+ sound/soc/amd/yc/acp6x-mach.c                      |    7 +
+ sound/soc/amd/yc/acp6x-pdm-dma.c                   |    2 +-
+ sound/soc/amd/yc/pci-acp6x.c                       |   12 +
+ sound/soc/apple/Kconfig                            |    1 +
+ sound/soc/apple/Makefile                           |    1 +
+ sound/soc/apple/mca.c                              |    4 +-
+ sound/soc/atmel/atmel-pcm-pdc.c                    |    2 +-
+ sound/soc/au1x/dbdma2.c                            |    2 +-
+ sound/soc/au1x/dma.c                               |    2 +-
+ sound/soc/bcm/bcm63xx-pcm-whistler.c               |    2 +-
+ sound/soc/bcm/cygnus-pcm.c                         |    2 +-
+ sound/soc/codecs/Kconfig                           |   14 +-
+ sound/soc/codecs/Makefile                          |    2 +
+ sound/soc/codecs/arizona-jack.c                    |   95 +-
+ sound/soc/codecs/arizona.h                         |    1 +
+ sound/soc/codecs/aw88166.c                         |   12 +-
+ sound/soc/codecs/aw88261.c                         |   13 +-
+ sound/soc/codecs/cros_ec_codec.c                   |    2 +-
+ sound/soc/codecs/cs-amp-lib-test.c                 |    1 -
+ sound/soc/codecs/cs-amp-lib.c                      |    2 +
+ sound/soc/codecs/cs35l41.c                         |    5 +
+ sound/soc/codecs/cs35l45.c                         |    3 +
+ sound/soc/codecs/cs35l56-sdw.c                     |   34 +-
+ sound/soc/codecs/cs35l56-shared-test.c             |  240 ++++
+ sound/soc/codecs/cs35l56-test.c                    |  101 +-
+ sound/soc/codecs/cs35l56.c                         |    3 +-
+ sound/soc/codecs/cs35l56.h                         |    2 +
+ sound/soc/codecs/cs4270.c                          |    6 +-
+ sound/soc/codecs/cs42l43.c                         |  756 +++++++++---
+ sound/soc/codecs/cs42l43.h                         |    4 +-
+ sound/soc/codecs/cs42l84.c                         |   20 +-
+ sound/soc/codecs/cs42xx8-i2c.c                     |    5 +-
+ sound/soc/codecs/cs42xx8.c                         |   27 +-
+ sound/soc/codecs/cs42xx8.h                         |    5 +-
+ sound/soc/codecs/da7219-aad.c                      |    3 +-
+ sound/soc/codecs/da7219-aad.h                      |    5 +
+ sound/soc/codecs/peb2466.c                         |   19 +-
+ sound/soc/codecs/rt5514-spi.c                      |    2 +-
+ sound/soc/codecs/rt5575.c                          |    1 -
+ sound/soc/codecs/rt5677-spi.c                      |    9 +-
+ sound/soc/codecs/rt5677.c                          |   40 +-
+ sound/soc/codecs/rt5677.h                          |    2 +-
+ sound/soc/codecs/sta529.c                          |    5 +-
+ sound/soc/codecs/tas2552.c                         |   10 +-
+ sound/soc/codecs/tas2764.c                         |   95 ++
+ sound/soc/codecs/tas2764.h                         |   11 +
+ sound/soc/codecs/tas2770.c                         |   75 ++
+ sound/soc/codecs/tas2770.h                         |   12 +
+ sound/soc/codecs/wcd-clsh-v2.c                     |    3 -
+ sound/soc/codecs/wcd9335.c                         |    2 +-
+ sound/soc/codecs/wm_adsp.c                         |  242 ++--
+ sound/soc/codecs/wm_adsp.h                         |   20 +
+ sound/soc/codecs/wm_adsp_fw_find_test.c            | 1222 ++++++++++++++++++++
+ sound/soc/dwc/dwc-i2s.c                            |    5 +-
+ sound/soc/dwc/dwc-pcm.c                            |    7 +-
+ sound/soc/dwc/local.h                              |    5 +-
+ sound/soc/fsl/efika-audio-fabric.c                 |    5 +-
+ sound/soc/fsl/fsl_asrc_dma.c                       |    2 +-
+ sound/soc/fsl/fsl_dma.c                            |    2 +-
+ sound/soc/fsl/fsl_easrc.c                          |    2 +-
+ sound/soc/fsl/fsl_qmc_audio.c                      |    6 +-
+ sound/soc/fsl/fsl_sai.c                            |   66 ++
+ sound/soc/fsl/fsl_sai.h                            |    4 +
+ sound/soc/fsl/fsl_utils.c                          |  131 +++
+ sound/soc/fsl/fsl_utils.h                          |   48 +
+ sound/soc/fsl/fsl_xcvr.c                           |   64 +
+ sound/soc/fsl/fsl_xcvr.h                           |   18 +
+ sound/soc/fsl/imx-pcm-fiq.c                        |    4 +-
+ sound/soc/fsl/imx-pcm-rpmsg.c                      |    2 +-
+ sound/soc/fsl/mpc5200_dma.c                        |    2 +-
+ sound/soc/generic/audio-graph-card.c               |    2 +-
+ sound/soc/generic/simple-card-utils.c              |   41 +-
+ sound/soc/generic/test-component.c                 |   12 +-
+ sound/soc/google/chv3-i2s.c                        |    6 +-
+ sound/soc/intel/atom/sst-mfld-platform-pcm.c       |    2 +-
+ sound/soc/intel/avs/pcm.c                          |    8 +-
+ sound/soc/intel/catpt/Makefile                     |    1 +
+ sound/soc/intel/catpt/core.h                       |    2 +-
+ sound/soc/intel/catpt/device.c                     |    2 +-
+ sound/soc/intel/catpt/ipc.c                        |    3 +
+ sound/soc/intel/catpt/loader.c                     |    2 +
+ sound/soc/intel/catpt/messages.h                   |    3 +
+ sound/soc/intel/catpt/pcm.c                        |  414 +++----
+ sound/soc/intel/keembay/Makefile                   |    1 +
+ sound/soc/intel/keembay/kmb_platform.c             |    2 +-
+ sound/soc/kirkwood/kirkwood-dma.c                  |    2 +-
+ sound/soc/loongson/loongson_dma.c                  |    2 +-
+ .../soc/mediatek/common/mtk-afe-platform-driver.c  |    2 +-
+ sound/soc/mediatek/mt8186/mt8186-afe-pcm.c         |    2 +-
+ sound/soc/mediatek/mt8189/mt8189-afe-pcm.c         |    4 +-
+ sound/soc/pxa/pxa-ssp.c                            |    2 +-
+ sound/soc/pxa/pxa2xx-ac97.c                        |    2 +-
+ sound/soc/pxa/pxa2xx-i2s.c                         |    2 +-
+ sound/soc/pxa/pxa2xx-pcm.c                         |    2 +-
+ sound/soc/qcom/lpass-platform.c                    |    2 +-
+ sound/soc/qcom/qdsp6/q6apm-dai.c                   |    2 +-
+ sound/soc/qcom/qdsp6/q6asm-dai.c                   |    2 +-
+ sound/soc/renesas/dma-sh7760.c                     |    2 +-
+ sound/soc/renesas/fsi.c                            |    2 +-
+ sound/soc/renesas/rcar/msiof.c                     |    2 +-
+ sound/soc/renesas/rz-ssi.c                         |    2 +-
+ sound/soc/renesas/siu_pcm.c                        |    4 +-
+ sound/soc/samsung/idma.c                           |    4 +-
+ sound/soc/sdca/sdca_asoc.c                         |  116 +-
+ sound/soc/sdca/sdca_class.c                        |    2 +
+ sound/soc/sdca/sdca_fdl.c                          |    5 -
+ sound/soc/sdca/sdca_functions.c                    |    6 +-
+ sound/soc/sdw_utils/Makefile                       |    1 +
+ sound/soc/sdw_utils/soc_sdw_cs47l47.c              |   80 ++
+ sound/soc/sdw_utils/soc_sdw_utils.c                |  160 ++-
+ sound/soc/soc-component.c                          |    8 +-
+ sound/soc/soc-compress.c                           |    4 +-
+ sound/soc/soc-core.c                               |   81 +-
+ sound/soc/soc-dai.c                                |   40 +
+ sound/soc/soc-dapm.c                               |   11 +
+ sound/soc/soc-generic-dmaengine-pcm.c              |    4 +-
+ sound/soc/soc-ops.c                                |   64 +-
+ sound/soc/soc-pcm.c                                |    4 +-
+ sound/soc/sof/compress.c                           |    2 +-
+ sound/soc/sof/ipc4-topology.c                      |   13 +-
+ sound/soc/sof/pcm.c                                |    8 +-
+ sound/soc/sof/sof-audio.h                          |   11 +-
+ sound/soc/sof/topology.c                           |   10 +-
+ sound/soc/spear/spdif_in.c                         |    5 +-
+ sound/soc/spear/spdif_out.c                        |    5 +-
+ sound/soc/spear/spear_pcm.c                        |    5 +-
+ sound/soc/sprd/sprd-pcm-dma.c                      |    2 +-
+ sound/soc/starfive/Makefile                        |    1 +
+ sound/soc/sti/uniperif_player.c                    |    9 +-
+ sound/soc/stm/stm32_adfsdm.c                       |    2 +-
+ sound/soc/tegra/Kconfig                            |   19 +
+ sound/soc/tegra/Makefile                           |    2 +
+ sound/soc/tegra/tegra210_admaif.c                  |    6 +-
+ sound/soc/tegra/tegra_asoc_machine.c               |   52 +
+ sound/soc/tegra/tegra_pcm.c                        |    6 +-
+ sound/soc/tegra/tegra_pcm.h                        |    4 +-
+ sound/soc/tegra/tegra_wm8962.c                     |  165 +++
+ sound/soc/ti/davinci-mcasp.c                       |   47 +-
+ sound/soc/uniphier/aio-dma.c                       |    2 +-
+ sound/soc/xilinx/xlnx_formatter_pcm.c              |    2 +-
+ sound/soc/xtensa/xtfpga-i2s.c                      |    2 +-
+ sound/usb/qcom/Makefile                            |    1 +
+ sound/x86/intel_hdmi_audio.h                       |   22 +-
+ 202 files changed, 4837 insertions(+), 1503 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/sound/adi,ssm2305.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/adi,ssm2305.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/nvidia,tegra-audio-cpcap.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8962.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/tdm-slot.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/tdm-slot.yaml
+ delete mode 100644 sound/soc/amd/rpl/Makefile
+ delete mode 100644 sound/soc/amd/rpl/rpl-pci-acp6x.c
+ delete mode 100644 sound/soc/amd/rpl/rpl_acp6x.h
+ delete mode 100644 sound/soc/amd/rpl/rpl_acp6x_chip_offset_byte.h
+ create mode 100644 sound/soc/codecs/wm_adsp_fw_find_test.c
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_cs47l47.c
+ create mode 100644 sound/soc/tegra/tegra_wm8962.c
+Merging modules/modules-next (f15dbe8a94b6e docs: symbol-namespaces: mention sysfs attribute)
+$ git merge -m Merge branch 'modules-next' of https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git modules/modules-next
+Auto-merging include/linux/module.h
+Auto-merging kernel/module/main.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-module       |   9 +++
+ Documentation/core-api/symbol-namespaces.rst |   5 ++
+ include/asm-generic/codetag.lds.h            |   2 +-
+ include/linux/module.h                       |   1 +
+ include/linux/moduleparam.h                  | 100 +++++++++++++--------------
+ kernel/module/main.c                         |  73 ++++++++++++++++++-
+ kernel/params.c                              |  29 +++++---
+ scripts/module.lds.S                         |  12 ++--
+ 8 files changed, 158 insertions(+), 73 deletions(-)
+Merging input/next (b8303880b641f Input: atlas - convert ACPI driver to a platform one)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/input/parade,tc3408.yaml   |   68 ++
+ .../input/touchscreen/sitronix,st1232.yaml         |    4 +
+ drivers/hid/i2c-hid/i2c-hid-of-elan.c              |    8 +
+ drivers/input/input.c                              |   23 +-
+ drivers/input/keyboard/atkbd.c                     |  133 ++-
+ drivers/input/keyboard/cros_ec_keyb.c              |  259 +++--
+ drivers/input/keyboard/mpr121_touchkey.c           |    8 -
+ drivers/input/misc/adxl34x-i2c.c                   |    5 +-
+ drivers/input/misc/atlas_btns.c                    |   22 +-
+ drivers/input/misc/drv260x.c                       |   50 +-
+ drivers/input/mouse/Kconfig                        |   10 -
+ drivers/input/mouse/Makefile                       |    1 -
+ drivers/input/mouse/alps.c                         |   11 +-
+ drivers/input/mouse/alps.h                         |    4 +-
+ drivers/input/mouse/hgpk.c                         | 1063 --------------------
+ drivers/input/mouse/hgpk.h                         |   61 --
+ drivers/input/mouse/psmouse-base.c                 |   51 +-
+ drivers/input/mouse/psmouse.h                      |    6 +-
+ drivers/input/serio/libps2.c                       |   12 +-
+ drivers/input/touchscreen/st1232.c                 |   67 +-
+ include/linux/input.h                              |    4 +
+ 21 files changed, 508 insertions(+), 1362 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/input/parade,tc3408.yaml
+ delete mode 100644 drivers/input/mouse/hgpk.c
+ delete mode 100644 drivers/input/mouse/hgpk.h
+Merging block/for-next (c4ae84167d025 Merge branch 'io_uring-7.0' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git block/for-next
+Auto-merging drivers/target/target_core_file.c
+Auto-merging fs/btrfs/volumes.c
+Auto-merging fs/ext4/mballoc.c
+Auto-merging mm/swapfile.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/stable/sysfs-block |  15 +
+ block/bio.c                          | 224 ++++-----
+ block/blk-cgroup.c                   |  15 +
+ block/blk-crypto-sysfs.c             |  40 +-
+ block/blk-ia-ranges.c                |   6 +-
+ block/blk-mq-debugfs.c               |   1 +
+ block/blk-mq-sysfs.c                 |  10 +-
+ block/blk-mq.c                       |  19 +
+ block/blk-settings.c                 |  12 +-
+ block/blk-sysfs.c                    |  89 +++-
+ block/blk-zoned.c                    | 410 ++++++++++-------
+ block/blk.h                          |   5 -
+ block/bsg-lib.c                      |   2 +-
+ block/bsg.c                          |  33 +-
+ block/opal_proto.h                   |   4 +
+ block/sed-opal.c                     | 399 ++++++++++++++--
+ block/t10-pi.c                       | 854 +++++++++++++++++++----------------
+ drivers/block/ublk_drv.c             |   2 +-
+ drivers/md/raid1.c                   |   2 +-
+ drivers/md/raid10.c                  |   2 +-
+ drivers/md/raid5.c                   |   2 +-
+ drivers/nvme/host/core.c             |   1 +
+ drivers/nvme/host/ioctl.c            |   4 -
+ drivers/nvme/host/multipath.c        |  15 +-
+ drivers/scsi/scsi_bsg.c              | 176 +++++++-
+ drivers/target/target_core_file.c    |   2 +-
+ drivers/target/target_core_iblock.c  |   2 +-
+ fs/btrfs/volumes.c                   |   4 +-
+ fs/ext4/mballoc-test.c               |   2 +-
+ fs/ext4/mballoc.c                    |   2 +-
+ include/linux/bio.h                  |   3 +-
+ include/linux/blk-integrity.h        |   1 +
+ include/linux/blkdev.h               |  17 +-
+ include/linux/bsg.h                  |   6 +-
+ include/linux/io_uring_types.h       |  50 +-
+ include/linux/sed-opal.h             |   4 +
+ include/uapi/linux/bsg.h             |  75 +++
+ include/uapi/linux/io_uring.h        | 101 +----
+ include/uapi/linux/io_uring/zcrx.h   | 108 +++++
+ include/uapi/linux/sed-opal.h        |  29 ++
+ io_uring/Kconfig                     |   5 +
+ io_uring/Makefile                    |   3 +-
+ io_uring/bpf-ops.c                   | 270 +++++++++++
+ io_uring/bpf-ops.h                   |  28 ++
+ io_uring/cmd_net.c                   |  34 +-
+ io_uring/eventfd.c                   |   4 +-
+ io_uring/io_uring.c                  | 120 ++---
+ io_uring/io_uring.h                  |  10 +-
+ io_uring/kbuf.c                      |  18 +-
+ io_uring/loop.c                      |  91 ++++
+ io_uring/loop.h                      |  27 ++
+ io_uring/msg_ring.c                  |   2 +-
+ io_uring/net.c                       | 162 +++----
+ io_uring/net.h                       |   1 -
+ io_uring/opdef.c                     |  12 +-
+ io_uring/opdef.h                     |   2 -
+ io_uring/poll.c                      |  17 +-
+ io_uring/query.c                     |   4 +-
+ io_uring/register.c                  |   8 +-
+ io_uring/rsrc.c                      |   8 +-
+ io_uring/rw.c                        |  13 +-
+ io_uring/tctx.c                      |   2 +-
+ io_uring/timeout.c                   |  74 ++-
+ io_uring/timeout.h                   |   2 +-
+ io_uring/tw.c                        |   2 +-
+ io_uring/uring_cmd.c                 |   9 +-
+ io_uring/wait.h                      |   1 +
+ io_uring/zcrx.c                      |   4 +-
+ io_uring/zcrx.h                      |   3 +
+ mm/swapfile.c                        |   2 +-
+ 70 files changed, 2521 insertions(+), 1165 deletions(-)
+ create mode 100644 include/uapi/linux/io_uring/zcrx.h
+ create mode 100644 io_uring/bpf-ops.c
+ create mode 100644 io_uring/bpf-ops.h
+ create mode 100644 io_uring/loop.c
+ create mode 100644 io_uring/loop.h
+$ git am -3 ../patches/0001-Merge-fixup-for-Rust-error.patch
+Applying: Merge fixup for Rust error
+Using index info to reconstruct a base tree...
+M	drivers/block/rnull/configfs.rs
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/block/rnull/configfs.rs
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-Revert-block-remove-bio_last_bvec_all.patch
+Applying: Revert "block: remove bio_last_bvec_all"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	Documentation/block/biovecs.rst
+M	include/linux/bio.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 077772e240b65] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
+ Date: Fri Mar 20 14:18:19 2026 +0000
+Merging device-mapper/for-next (23e6e57a93bca dm: make "dmsetup remove_all" interruptible)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/device-mapper/verity.rst | 122 +++++--
+ drivers/md/Kconfig                                 |   1 +
+ drivers/md/dm-cache-metadata.c                     |  33 +-
+ drivers/md/dm-cache-metadata.h                     |  10 +-
+ drivers/md/dm-cache-policy-smq.c                   |   4 +
+ drivers/md/dm-cache-target.c                       |  90 +++--
+ drivers/md/dm-ima.c                                |  54 +--
+ drivers/md/dm-ima.h                                |   1 -
+ drivers/md/dm-init.c                               |   4 +-
+ drivers/md/dm-ioctl.c                              |  35 +-
+ drivers/md/dm-log.c                                |   6 +-
+ drivers/md/dm-mpath.c                              |   9 +-
+ drivers/md/dm-raid1.c                              |   6 +-
+ drivers/md/dm-vdo/action-manager.c                 |   2 +-
+ drivers/md/dm-vdo/block-map.c                      |  32 +-
+ drivers/md/dm-vdo/block-map.h                      |   2 +-
+ drivers/md/dm-vdo/data-vio.c                       |   3 +-
+ drivers/md/dm-vdo/dedupe.c                         |   8 +-
+ drivers/md/dm-vdo/dm-vdo-target.c                  |  12 +-
+ drivers/md/dm-vdo/encodings.c                      |   2 +-
+ drivers/md/dm-vdo/flush.c                          |   4 +-
+ drivers/md/dm-vdo/funnel-queue.c                   |   2 +-
+ drivers/md/dm-vdo/funnel-workqueue.c               |   8 +-
+ drivers/md/dm-vdo/indexer/chapter-index.c          |   2 +-
+ drivers/md/dm-vdo/indexer/config.c                 |   2 +-
+ drivers/md/dm-vdo/indexer/delta-index.c            |  13 +-
+ drivers/md/dm-vdo/indexer/funnel-requestqueue.c    |   2 +-
+ drivers/md/dm-vdo/indexer/geometry.c               |   2 +-
+ drivers/md/dm-vdo/indexer/index-layout.c           |  28 +-
+ drivers/md/dm-vdo/indexer/index-page-map.c         |   8 +-
+ drivers/md/dm-vdo/indexer/index-session.c          |   2 +-
+ drivers/md/dm-vdo/indexer/index.c                  |  14 +-
+ drivers/md/dm-vdo/indexer/index.h                  |   2 +-
+ drivers/md/dm-vdo/indexer/io-factory.c             |   6 +-
+ drivers/md/dm-vdo/indexer/open-chapter.c           |   4 +-
+ drivers/md/dm-vdo/indexer/open-chapter.h           |   2 +-
+ drivers/md/dm-vdo/indexer/radix-sort.c             |   3 +-
+ drivers/md/dm-vdo/indexer/sparse-cache.c           |  10 +-
+ drivers/md/dm-vdo/indexer/volume-index.c           |  10 +-
+ drivers/md/dm-vdo/indexer/volume.c                 |  22 +-
+ drivers/md/dm-vdo/int-map.c                        |   5 +-
+ drivers/md/dm-vdo/io-submitter.c                   |   3 +-
+ drivers/md/dm-vdo/logical-zone.c                   |   3 +-
+ drivers/md/dm-vdo/logical-zone.h                   |   2 +-
+ drivers/md/dm-vdo/memory-alloc.c                   |   8 +-
+ drivers/md/dm-vdo/memory-alloc.h                   |  74 +---
+ drivers/md/dm-vdo/message-stats.c                  |   2 +-
+ drivers/md/dm-vdo/packer.c                         |   9 +-
+ drivers/md/dm-vdo/physical-zone.c                  |   8 +-
+ drivers/md/dm-vdo/priority-table.c                 |   3 +-
+ drivers/md/dm-vdo/recovery-journal.c               |  23 +-
+ drivers/md/dm-vdo/repair.c                         |  17 +-
+ drivers/md/dm-vdo/slab-depot.c                     |  41 +--
+ drivers/md/dm-vdo/slab-depot.h                     |   2 +-
+ drivers/md/dm-vdo/thread-utils.c                   |   2 +-
+ drivers/md/dm-vdo/vdo.c                            |  32 +-
+ drivers/md/dm-vdo/vio.c                            |  12 +-
+ drivers/md/dm-verity-fec.c                         | 384 ++++++++++-----------
+ drivers/md/dm-verity-fec.h                         |  28 +-
+ drivers/md/dm-verity-target.c                      |   8 +-
+ drivers/md/dm-verity.h                             |   4 +-
+ 61 files changed, 605 insertions(+), 647 deletions(-)
+Merging libata/for-next (d78c5bbf098fd Merge branch 'for-7.0-fixes' into for-7.1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux libata/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/ata/baikal,bt1-ahci.yaml   | 115 -------------
+ drivers/ata/Kconfig                                |   1 -
+ drivers/ata/ahci_dwc.c                             |  55 -------
+ drivers/ata/libata-core.c                          |  12 +-
+ drivers/ata/libata-eh.c                            |  28 ++--
+ drivers/ata/libata-sata.c                          |  14 +-
+ drivers/ata/libata-scsi.c                          | 178 ++++++++++-----------
+ include/linux/libata.h                             |   4 +-
+ 8 files changed, 124 insertions(+), 283 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml
+Merging pcmcia/pcmcia-next (4a81f78caa53e pcmcia: Add error handling for add_interval() in do_validate_mem())
+$ git merge -m Merge branch 'pcmcia-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (d75c6c7d8e985 mmc: sdhci-pltfm: remove duplicate DTS property parsing)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Auto-merging MAINTAINERS
+Auto-merging include/linux/mmc/sdio_ids.h
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mmc/arasan,sdhci.yaml      |   5 +
+ .../devicetree/bindings/mmc/arm,pl18x.yaml         |   2 +-
+ .../devicetree/bindings/mmc/brcm,iproc-sdhci.yaml  |   5 +
+ .../devicetree/bindings/mmc/bst,c1200-sdhci.yaml   |  70 ++
+ .../devicetree/bindings/mmc/cdns,sdhci.yaml        |   2 -
+ .../devicetree/bindings/mmc/fsl-imx-esdhc.yaml     |   1 +
+ .../bindings/mmc/loongson,ls2k0500-mmc.yaml        |   1 +
+ Documentation/devicetree/bindings/mmc/mtk-sd.yaml  |   3 +
+ .../devicetree/bindings/mmc/renesas,sdhi.yaml      |   6 +
+ .../devicetree/bindings/mmc/rockchip-dw-mshc.yaml  |   1 +
+ .../devicetree/bindings/mmc/sdhci-msm.yaml         |   1 +
+ .../bindings/mmc/snps,dwcmshc-sdhci.yaml           |   3 +
+ .../devicetree/bindings/mmc/spacemit,sdhci.yaml    |  14 +-
+ MAINTAINERS                                        |   2 +
+ arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts  |   4 +-
+ arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts  |   4 +-
+ drivers/i2c/busses/i2c-omap.c                      |  24 +-
+ drivers/mmc/core/card.h                            |  11 +
+ drivers/mmc/core/host.c                            |  14 +-
+ drivers/mmc/core/host.h                            |   6 +-
+ drivers/mmc/core/mmc.c                             |  12 +
+ drivers/mmc/core/queue.c                           |   9 +-
+ drivers/mmc/core/quirks.h                          |  12 +
+ drivers/mmc/core/sdio_io.c                         |   6 +-
+ drivers/mmc/host/Kconfig                           |  16 +-
+ drivers/mmc/host/Makefile                          |   1 +
+ drivers/mmc/host/atmel-mci.c                       |  12 +-
+ drivers/mmc/host/cavium-octeon.c                   |   5 +-
+ drivers/mmc/host/cavium.c                          |   4 +-
+ drivers/mmc/host/dw_mmc-bluefield.c                |   2 +-
+ drivers/mmc/host/dw_mmc-exynos.c                   |   9 +-
+ drivers/mmc/host/dw_mmc-hi3798cv200.c              |   6 +-
+ drivers/mmc/host/dw_mmc-hi3798mv200.c              |  28 +-
+ drivers/mmc/host/dw_mmc-k3.c                       |  51 +-
+ drivers/mmc/host/dw_mmc-pci.c                      |  37 +-
+ drivers/mmc/host/dw_mmc-pltfm.c                    |  19 +-
+ drivers/mmc/host/dw_mmc-pltfm.h                    |   2 +-
+ drivers/mmc/host/dw_mmc-rockchip.c                 |  36 +-
+ drivers/mmc/host/dw_mmc-starfive.c                 |   5 +-
+ drivers/mmc/host/dw_mmc.c                          | 781 ++++++++-------------
+ drivers/mmc/host/dw_mmc.h                          | 148 +---
+ drivers/mmc/host/jz4740_mmc.c                      |   2 +-
+ drivers/mmc/host/loongson2-mmc.c                   |  60 +-
+ drivers/mmc/host/mtk-sd.c                          |  32 +-
+ drivers/mmc/host/renesas_sdhi_core.c               |   6 +
+ drivers/mmc/host/renesas_sdhi_sys_dmac.c           |  12 +-
+ drivers/mmc/host/rtsx_pci_sdmmc.c                  |  88 +--
+ drivers/mmc/host/sdhci-esdhc-imx.c                 |  33 +-
+ drivers/mmc/host/sdhci-msm.c                       |  77 ++
+ drivers/mmc/host/sdhci-of-arasan.c                 |  15 +
+ drivers/mmc/host/sdhci-of-aspeed.c                 |   5 +-
+ drivers/mmc/host/sdhci-of-bst.c                    | 523 ++++++++++++++
+ drivers/mmc/host/sdhci-of-k1.c                     |  39 +-
+ drivers/mmc/host/sdhci-pic32.c                     |   5 +-
+ drivers/mmc/host/sdhci-pltfm.c                     |  20 -
+ drivers/mmc/host/sdhci-uhs2.c                      |  13 +-
+ drivers/mmc/host/sdhci.c                           |  23 +-
+ drivers/mmc/host/tifm_sd.c                         |   4 +-
+ drivers/mux/Kconfig                                |  17 +-
+ drivers/mux/core.c                                 | 198 +++++-
+ drivers/phy/phy-can-transceiver.c                  |  10 -
+ drivers/phy/renesas/phy-rcar-gen3-usb2.c           |  30 +-
+ include/linux/mmc/card.h                           |   2 +
+ include/linux/mmc/sdio_ids.h                       |   3 +
+ include/linux/mux/consumer.h                       | 108 ++-
+ 65 files changed, 1693 insertions(+), 1012 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mmc/bst,c1200-sdhci.yaml
+ create mode 100644 drivers/mmc/host/sdhci-of-bst.c
+Merging mfd/for-mfd-next (405c1338219f0 dt-bindings: mfd: bd72720: Add ROHM BD73900)
+$ git merge -m Merge branch 'for-mfd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mfd/fsl,imx25-tsadc.yaml   | 97 ++++++++++++++++++++++
+ .../devicetree/bindings/mfd/fsl-imx25-tsadc.txt    | 47 -----------
+ .../devicetree/bindings/mfd/rohm,bd72720-pmic.yaml | 29 ++++---
+ .../devicetree/bindings/mfd/spacemit,p1.yaml       | 49 ++++++++++-
+ drivers/mfd/atmel-hlcdc.c                          | 13 ++-
+ drivers/mfd/dln2.c                                 | 21 ++---
+ drivers/mfd/lpc_ich.c                              |  7 ++
+ drivers/mfd/macsmc.c                               |  1 +
+ drivers/mfd/max77705.c                             |  2 +-
+ drivers/mfd/mc13xxx-core.c                         |  2 +-
+ drivers/mfd/mt6397-core.c                          |  4 +-
+ drivers/mfd/rohm-bd71828.c                         |  1 +
+ drivers/mfd/stpmic1.c                              | 20 ++++-
+ drivers/mfd/viperboard.c                           |  7 +-
+ include/linux/mfd/atmel-hlcdc.h                    |  1 +
+ include/linux/mfd/cgbc.h                           |  4 +-
+ include/linux/mfd/kempld.h                         |  6 +-
+ include/linux/mfd/lpc_ich.h                        |  2 +
+ include/linux/mfd/mt6397/core.h                    |  6 +-
+ include/linux/mfd/rsmu.h                           |  1 -
+ include/linux/mfd/si476x-core.h                    | 17 ++--
+ 21 files changed, 237 insertions(+), 100 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mfd/fsl,imx25-tsadc.yaml
+ delete mode 100644 Documentation/devicetree/bindings/mfd/fsl-imx25-tsadc.txt
+Merging backlight/for-backlight-next (797cc011ae02b backlight: sky81452-backlight: Check return value of devm_gpiod_get_optional() in sky81452_bl_parse_dt())
+$ git merge -m Merge branch 'for-backlight-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Merge made by the 'ort' strategy.
+ drivers/video/backlight/sky81452-backlight.c | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging battery/for-next (aa2132799817f power: supply: add support for S2MU005 battery fuel gauge device)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Auto-merging MAINTAINERS
+Auto-merging kernel/workqueue.c
+Merge made by the 'ort' strategy.
+ .../testing/sysfs-class-reboot-mode-reboot_modes   |  36 +
+ .../bindings/power/supply/cpcap-battery.yaml       |   1 +
+ .../bindings/power/supply/maxim,max17042.yaml      |  21 +-
+ .../power/supply/samsung,s2mu005-fuel-gauge.yaml   |  49 ++
+ Documentation/driver-api/driver-model/devres.rst   |   4 +
+ MAINTAINERS                                        |   1 +
+ drivers/power/reset/keystone-reset.c               |  11 +-
+ drivers/power/reset/reboot-mode.c                  | 151 +++-
+ drivers/power/supply/Kconfig                       |  22 +
+ drivers/power/supply/Makefile                      |   2 +
+ drivers/power/supply/axp288_charger.c              |  71 +-
+ drivers/power/supply/bq24190_charger.c             |   9 +-
+ drivers/power/supply/cpcap-battery.c               |  31 +-
+ drivers/power/supply/cw2015_battery.c              |   3 +-
+ drivers/power/supply/ipaq_micro_battery.c          |  50 +-
+ drivers/power/supply/macsmc-power.c                | 855 +++++++++++++++++++++
+ drivers/power/supply/max17042_battery.c            | 130 +++-
+ drivers/power/supply/max77705_charger.c            |  36 +-
+ drivers/power/supply/mt6370-charger.c              |  13 +-
+ drivers/power/supply/qcom_battmgr.c                |   2 +
+ drivers/power/supply/s2mu005-battery.c             | 307 ++++++++
+ drivers/power/supply/sbs-manager.c                 |   2 +-
+ drivers/power/supply/twl4030_madc_battery.c        |   2 -
+ include/linux/power/max17042_battery.h             |  25 +-
+ include/linux/workqueue.h                          |  22 +
+ kernel/workqueue.c                                 |  28 +
+ 26 files changed, 1724 insertions(+), 160 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
+ create mode 100644 Documentation/devicetree/bindings/power/supply/samsung,s2mu005-fuel-gauge.yaml
+ create mode 100644 drivers/power/supply/macsmc-power.c
+ create mode 100644 drivers/power/supply/s2mu005-battery.c
+Merging regulator/for-next (f33d6582eb207 Merge remote-tracking branch 'regulator/for-7.1' into regulator-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/regulator/cpcap-regulator.txt         |  35 -------
+ .../bindings/regulator/fitipower,fp9931.yaml       |   1 +
+ .../regulator/motorola,cpcap-regulator.yaml        |  47 +++++++++
+ drivers/regulator/cpcap-regulator.c                | 105 +++++++++++++++++++++
+ drivers/regulator/da9063-regulator.c               |   9 +-
+ drivers/regulator/fixed.c                          |   1 -
+ drivers/regulator/fp9931.c                         |   2 +-
+ drivers/regulator/max20411-regulator.c             |   2 +-
+ drivers/regulator/pf0900-regulator.c               |   2 +-
+ drivers/regulator/pf1550-regulator.c               |   1 -
+ drivers/regulator/pf9453-regulator.c               |   5 +-
+ drivers/regulator/spacemit-p1.c                    |  25 ++---
+ 12 files changed, 176 insertions(+), 59 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/regulator/cpcap-regulator.txt
+ create mode 100644 Documentation/devicetree/bindings/regulator/motorola,cpcap-regulator.yaml
+Merging security/next (306dc96b226af Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Auto-merging fs/overlayfs/dir.c
+Auto-merging security/selinux/hooks.c
+Merge made by the 'ort' strategy.
+ fs/backing-file.c             |  34 +++++++++-----
+ fs/erofs/ishare.c             |  20 ++++++--
+ fs/file_table.c               |  53 ++++++++++++++++-----
+ fs/fuse/passthrough.c         |   3 +-
+ fs/internal.h                 |   5 +-
+ fs/overlayfs/dir.c            |   3 +-
+ fs/overlayfs/file.c           |   1 +
+ include/linux/backing-file.h  |  29 ++++++++++--
+ include/linux/cred.h          |  10 ++--
+ include/linux/file_ref.h      |   4 +-
+ include/linux/lsm_audit.h     |   2 +-
+ include/linux/lsm_hook_defs.h |   2 +
+ include/linux/security.h      |  10 ++++
+ security/inode.c              |  10 ++--
+ security/security.c           |  26 +++++++++++
+ security/selinux/hooks.c      | 104 ++++++++++++++++++++++++++++++++++--------
+ 16 files changed, 254 insertions(+), 62 deletions(-)
+Merging apparmor/apparmor-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'apparmor-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Already up to date.
+Merging integrity/next-integrity (7a60fe48af206 ima: remove buggy support for asynchronous hashes)
+$ git merge -m Merge branch 'next-integrity' of https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/powerpc/Kconfig
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/s390/kernel/Makefile
+Auto-merging arch/x86/include/asm/efi.h
+Auto-merging arch/x86/platform/efi/efi.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt |  17 --
+ MAINTAINERS                                     |   1 +
+ arch/Kconfig                                    |   3 +
+ arch/powerpc/Kconfig                            |   1 +
+ arch/powerpc/kernel/ima_arch.c                  |   8 +-
+ arch/powerpc/kernel/secure_boot.c               |   6 +
+ arch/s390/Kconfig                               |   2 +-
+ arch/s390/kernel/Makefile                       |   1 -
+ arch/s390/kernel/ima_arch.c                     |  14 -
+ arch/s390/kernel/ipl.c                          |   6 +
+ arch/x86/include/asm/efi.h                      |   4 +-
+ arch/x86/platform/efi/efi.c                     |   2 +-
+ include/linux/evm.h                             |   8 +
+ include/linux/ima.h                             |   7 +-
+ include/linux/secure_boot.h                     |  23 ++
+ security/integrity/Makefile                     |   1 +
+ security/integrity/efi_secureboot.c             |  56 ++++
+ security/integrity/evm/evm_main.c               |  56 +++-
+ security/integrity/ima/ima.h                    |  28 +-
+ security/integrity/ima/ima_api.c                |  13 +-
+ security/integrity/ima/ima_appraise.c           |   7 +-
+ security/integrity/ima/ima_crypto.c             | 390 +-----------------------
+ security/integrity/ima/ima_efi.c                |  53 +---
+ security/integrity/ima/ima_fs.c                 |  34 ++-
+ security/integrity/ima/ima_main.c               |  37 ++-
+ security/integrity/integrity.h                  |   1 +
+ security/integrity/platform_certs/load_uefi.c   |   2 +-
+ 27 files changed, 267 insertions(+), 514 deletions(-)
+ delete mode 100644 arch/s390/kernel/ima_arch.c
+ create mode 100644 include/linux/secure_boot.h
+ create mode 100644 security/integrity/efi_secureboot.c
+Merging selinux/next (8dc51459ef702 selinux: annotate intentional data race in inode_doinit_with_dentry())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Auto-merging security/selinux/hooks.c
+Merge made by the 'ort' strategy.
+ security/selinux/hooks.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging smack/next (33d589ed60ae4 smack: /smack/doi: accept previously used values)
+$ git merge -m Merge branch 'next' of https://github.com/cschaufler/smack-next smack/next
+Already up to date.
+Merging tomoyo/master (824a14076c358 hfs: update sanity check of the root record)
+$ git merge -m Merge branch 'master' of git://git.code.sf.net/p/tomoyo/tomoyo.git tomoyo/master
+Auto-merging include/linux/netdevice.h
+Auto-merging include/net/sock.h
+Auto-merging kernel/workqueue.c
+Auto-merging net/core/dev.c
+Auto-merging net/socket.c
+Merge made by the 'ort' strategy.
+ fs/hfs/super.c            |   2 +-
+ include/linux/netdevice.h |  15 ++++
+ include/net/dst.h         |  22 +++++-
+ include/net/sock.h        |   8 +-
+ kernel/softirq.c          |   4 +
+ kernel/workqueue.c        |  28 +++++++
+ net/bridge/br_nf_core.c   |   1 +
+ net/core/dev.c            | 188 ++++++++++++++++++++++++++++++++++++++++++++++
+ net/core/dst.c            |  85 +++++++++++++++++++++
+ net/core/lock_debug.c     |   1 +
+ net/socket.c              |  32 ++++++--
+ 11 files changed, 374 insertions(+), 12 deletions(-)
+Merging tpmdd/next (d79526b89571a Merge tag 'spi-fix-v7.0-merge-window' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd/next
+Already up to date.
+Merging watchdog/master (450517fa5aae9 dt-bindings: watchdog: qcom-wdt: Document Glymur watchdog)
+$ git merge -m Merge branch 'master' of git://www.linux-watchdog.org/linux-watchdog-next.git watchdog/master
+Already up to date.
+Merging iommu/next (ca3bbc9287400 Merge branches 'fixes', 'arm/smmu/updates', 'arm/smmu/bindings', 'riscv', 'amd/amd-vi' and 'core' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git iommu/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/iommu/arm,smmu.yaml        |   1 +
+ drivers/iommu/amd/init.c                           |   9 +-
+ drivers/iommu/amd/iommu.c                          |  21 +-
+ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c    |  35 +-
+ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c   | 135 +++
+ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c        | 938 +++++++++++++++++----
+ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h        | 142 +++-
+ drivers/iommu/generic_pt/.kunitconfig              |   1 +
+ drivers/iommu/generic_pt/Kconfig                   |  11 +
+ drivers/iommu/generic_pt/fmt/Makefile              |   2 +
+ drivers/iommu/generic_pt/fmt/defs_riscv.h          |  29 +
+ drivers/iommu/generic_pt/fmt/iommu_riscv64.c       |  11 +
+ drivers/iommu/generic_pt/fmt/riscv.h               | 313 +++++++
+ drivers/iommu/generic_pt/iommu_pt.h                | 189 +++--
+ drivers/iommu/generic_pt/kunit_generic_pt.h        |  12 +
+ drivers/iommu/generic_pt/pt_iter.h                 |  22 +
+ drivers/iommu/intel/dmar.c                         |   3 +-
+ drivers/iommu/intel/iommu.c                        |   2 +
+ drivers/iommu/intel/svm.c                          |  12 +-
+ drivers/iommu/iommu-sva.c                          |  12 +-
+ drivers/iommu/iommu.c                              |  72 +-
+ drivers/iommu/iommufd/device.c                     |   4 +
+ drivers/iommu/iova.c                               |   3 +-
+ drivers/iommu/riscv/Kconfig                        |   8 +-
+ drivers/iommu/riscv/iommu-bits.h                   |   4 +-
+ drivers/iommu/riscv/iommu-platform.c               |  14 +-
+ drivers/iommu/riscv/iommu.c                        | 375 +++-----
+ include/linux/generic_pt/common.h                  |  16 +
+ include/linux/generic_pt/iommu.h                   |  80 +-
+ include/linux/io-pgtable.h                         |  10 +-
+ include/linux/iommu.h                              |   3 +
+ include/uapi/linux/iommufd.h                       |   4 +
+ 32 files changed, 1911 insertions(+), 582 deletions(-)
+ create mode 100644 drivers/iommu/generic_pt/fmt/defs_riscv.h
+ create mode 100644 drivers/iommu/generic_pt/fmt/iommu_riscv64.c
+ create mode 100644 drivers/iommu/generic_pt/fmt/riscv.h
+Merging audit/next (360160f75592b audit: handle unknown status requests in audit_receive_msg())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Auto-merging include/linux/audit.h
+CONFLICT (content): Merge conflict in include/linux/audit.h
+Resolved 'include/linux/audit.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 4ab637d773748] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git
+$ git diff -M --stat --summary HEAD^..
+ include/linux/audit.h      | 9 +++++++++
+ include/uapi/linux/audit.h | 8 ++++----
+ kernel/audit.c             | 8 +++++---
+ kernel/auditfilter.c       | 3 +--
+ lib/audit.c                | 2 +-
+ 5 files changed, 20 insertions(+), 10 deletions(-)
+Merging devicetree/for-next (3ee3d8a44976a soc: qcom: pd-mapper: Convert to of_machine_get_match())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Auto-merging Documentation/devicetree/bindings/arm/cpus.yaml
+CONFLICT (content): Merge conflict in Documentation/devicetree/bindings/arm/cpus.yaml
+Auto-merging drivers/bus/Kconfig
+Auto-merging drivers/bus/Makefile
+Auto-merging drivers/soc/qcom/qcom_pd_mapper.c
+CONFLICT (content): Merge conflict in drivers/soc/qcom/qcom_pd_mapper.c
+Auto-merging scripts/checkpatch.pl
+Resolved 'Documentation/devicetree/bindings/arm/cpus.yaml' using previous resolution.
+Resolved 'drivers/soc/qcom/qcom_pd_mapper.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 70c50326631e5] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/devicetree/bindings/arm/cpus.yaml    | 304 ++++++++--------
+ .../bindings/arm/freescale/fsl,imx51-m4if.yaml     |   1 +
+ .../devicetree/bindings/bus/baikal,bt1-apb.yaml    |  90 -----
+ .../devicetree/bindings/bus/baikal,bt1-axi.yaml    | 107 ------
+ .../bindings/input/touchscreen/fsl,imx25-tcq.yaml  |  69 ++++
+ .../bindings/input/touchscreen/fsl-mx25-tcq.txt    |  34 --
+ .../bindings/interrupt-controller/arm,gic-v3.yaml  |   2 +-
+ .../bindings/interrupt-controller/qcom,pdc.yaml    |   1 +
+ .../devicetree/bindings/mux/mux-controller.yaml    |   6 -
+ Documentation/devicetree/of_unittest.rst           |  20 +-
+ .../translations/zh_CN/devicetree/of_unittest.rst  |  21 +-
+ drivers/bus/Kconfig                                |  30 --
+ drivers/bus/Makefile                               |   2 -
+ drivers/bus/bt1-apb.c                              | 396 ---------------------
+ drivers/bus/bt1-axi.c                              | 292 ---------------
+ drivers/cpufreq/airoha-cpufreq.c                   |   7 +-
+ drivers/cpufreq/qcom-cpufreq-nvmem.c               |  16 +-
+ drivers/cpufreq/ti-cpufreq.c                       |  12 +-
+ drivers/of/base.c                                  |  20 +-
+ drivers/of/platform.c                              |  12 +-
+ drivers/of/property.c                              |   2 +-
+ drivers/soc/qcom/qcom_pd_mapper.c                  |   2 +-
+ include/linux/of.h                                 |  11 +-
+ scripts/checkpatch.pl                              |   2 +-
+ 24 files changed, 278 insertions(+), 1181 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml
+ delete mode 100644 Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml
+ create mode 100644 Documentation/devicetree/bindings/input/touchscreen/fsl,imx25-tcq.yaml
+ delete mode 100644 Documentation/devicetree/bindings/input/touchscreen/fsl-mx25-tcq.txt
+ delete mode 100644 drivers/bus/bt1-apb.c
+ delete mode 100644 drivers/bus/bt1-axi.c
+Merging dt-krzk/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Already up to date.
+Merging mailbox/for-next (57df858a46f0a mailbox: add API to query available TX queue slots)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Merge made by the 'ort' strategy.
+ drivers/mailbox/mailbox.c      | 23 +++++++++++++++++++++++
+ include/linux/mailbox_client.h |  1 +
+ 2 files changed, 24 insertions(+)
+Merging spi/for-next (01ad82a9260b3 Merge remote-tracking branch 'spi/for-7.1' into spi-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/spi/microchip,mpfs-spi.yaml           |   4 +-
+ .../bindings/spi/renesas,rzv2h-rspi.yaml           |  23 ++++-
+ drivers/spi/spi-amlogic-spisg.c                    |   6 +-
+ drivers/spi/spi-atcspi200.c                        |  29 ++----
+ drivers/spi/spi-axiado.c                           |   1 -
+ drivers/spi/spi-cs42l43.c                          |   4 +-
+ drivers/spi/spi-fsl-qspi.c                         |   3 +-
+ drivers/spi/spi-nxp-fspi.c                         |   3 +-
+ drivers/spi/spi-nxp-xspi.c                         |   3 +-
+ drivers/spi/spi-omap2-mcspi.c                      |  17 +++-
+ drivers/spi/spi-pxa2xx.c                           |  11 +--
+ drivers/spi/spi-stm32-ospi.c                       |   5 -
+ drivers/spi/spi-stm32.c                            |  10 +-
+ drivers/spi/spi-tegra210-quad.c                    |  16 +++-
+ drivers/spi/spi.c                                  | 106 ++++++++++++++++-----
+ include/linux/spi/spi-mem.h                        |   8 +-
+ include/linux/spi/spi.h                            |   1 +
+ 17 files changed, 164 insertions(+), 86 deletions(-)
+Merging tip/master (00296b545b4aa Merge branch into tip/master: 'x86/tdx')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/s390/Makefile
+Auto-merging arch/sparc/Kconfig
+Auto-merging arch/x86/Kconfig
+Auto-merging drivers/acpi/osl.c
+Auto-merging drivers/char/random.c
+Auto-merging drivers/irqchip/Kconfig
+Auto-merging include/linux/sched/signal.h
+Auto-merging include/trace/stages/stage3_trace_output.h
+Auto-merging include/uapi/linux/sched.h
+Auto-merging init/main.c
+CONFLICT (content): Merge conflict in init/main.c
+Auto-merging kernel/events/core.c
+Auto-merging kernel/softirq.c
+Auto-merging lib/Kconfig.debug
+Auto-merging lib/vdso/datastore.c
+Resolved 'init/main.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master a91f2cbe6b91a] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |    4 +-
+ .../bindings/interrupt-controller/apple,aic2.yaml  |   30 +-
+ MAINTAINERS                                        |   15 +
+ arch/arm/include/asm/vdso/gettimeofday.h           |    2 +
+ arch/arm64/include/asm/vdso/compat_gettimeofday.h  |    3 +
+ arch/arm64/include/asm/vdso/gettimeofday.h         |    2 +
+ arch/loongarch/include/asm/irq.h                   |    1 -
+ arch/loongarch/kernel/process.c                    |    1 +
+ arch/loongarch/kernel/vdso.c                       |    1 +
+ arch/mips/Kconfig                                  |    1 -
+ arch/mips/include/asm/vdso/vdso.h                  |    5 +
+ arch/mips/kernel/vdso.c                            |    1 +
+ arch/mips/pic32/Kconfig                            |    1 -
+ arch/parisc/kernel/time.c                          |    5 +-
+ arch/powerpc/include/asm/vdso/gettimeofday.h       |    1 +
+ arch/powerpc/include/asm/vdso/processor.h          |    3 +
+ arch/powerpc/kernel/compat_audit.c                 |    3 +-
+ arch/s390/Makefile                                 |    3 +-
+ arch/sparc/Kconfig                                 |    3 +-
+ arch/sparc/include/asm/clocksource.h               |    9 -
+ arch/sparc/include/asm/processor.h                 |    3 +
+ arch/sparc/include/asm/processor_32.h              |    2 -
+ arch/sparc/include/asm/processor_64.h              |   25 -
+ arch/sparc/include/asm/vdso.h                      |    2 -
+ arch/sparc/include/asm/vdso/clocksource.h          |   10 +
+ arch/sparc/include/asm/vdso/gettimeofday.h         |  184 ++++
+ arch/sparc/include/asm/vdso/processor.h            |   41 +
+ arch/sparc/include/asm/vdso/vsyscall.h             |   10 +
+ arch/sparc/include/asm/vvar.h                      |   75 --
+ arch/sparc/kernel/Makefile                         |    1 -
+ arch/sparc/kernel/time_64.c                        |    6 +-
+ arch/sparc/kernel/vdso.c                           |   69 --
+ arch/sparc/vdso/Makefile                           |   11 +-
+ arch/sparc/vdso/vclock_gettime.c                   |  418 +------
+ arch/sparc/vdso/vdso-layout.lds.S                  |   26 +-
+ arch/sparc/vdso/vdso.lds.S                         |    2 -
+ arch/sparc/vdso/vdso2c.c                           |   24 -
+ arch/sparc/vdso/vdso2c.h                           |   45 +-
+ arch/sparc/vdso/vdso32/vdso32.lds.S                |    4 +-
+ arch/sparc/vdso/vma.c                              |  274 +----
+ arch/x86/Kconfig                                   |    2 +
+ arch/x86/boot/compressed/sev.c                     |    6 +-
+ arch/x86/coco/tdx/debug.c                          |   26 +-
+ arch/x86/coco/tdx/tdx.c                            |    8 +-
+ arch/x86/configs/x86_64_defconfig                  |    1 +
+ arch/x86/entry/vdso/vdso32/Makefile                |    4 +
+ arch/x86/entry/vsyscall/vsyscall_64.c              |   91 +-
+ arch/x86/events/amd/ibs.c                          |  264 ++++-
+ arch/x86/events/intel/core.c                       |   22 +-
+ arch/x86/events/perf_event_flags.h                 |    1 +
+ arch/x86/include/asm/amd/ibs.h                     |    6 +-
+ arch/x86/include/asm/clock_inlined.h               |   22 +
+ arch/x86/include/asm/io.h                          |    5 +-
+ arch/x86/include/asm/local.h                       |    2 +-
+ arch/x86/include/asm/mce.h                         |   62 +-
+ arch/x86/include/asm/microcode.h                   |    2 +
+ arch/x86/include/asm/mmu_context.h                 |    3 -
+ arch/x86/include/asm/msr-index.h                   |    2 +
+ arch/x86/include/asm/orc_types.h                   |    9 +-
+ arch/x86/include/asm/perf_event.h                  |   55 +-
+ arch/x86/include/asm/processor.h                   |    5 +
+ arch/x86/include/asm/shared/tdx.h                  |   50 +-
+ arch/x86/include/asm/tdx_global_metadata.h         |    7 +
+ arch/x86/include/asm/tlbflush.h                    |   26 +
+ arch/x86/include/asm/vsyscall.h                    |   13 +-
+ arch/x86/kernel/apic/apic.c                        |   41 +-
+ arch/x86/kernel/cpu/bus_lock.c                     |   49 +-
+ arch/x86/kernel/cpu/common.c                       |   35 +-
+ arch/x86/kernel/cpu/intel.c                        |    1 +
+ arch/x86/kernel/cpu/match.c                        |    3 +
+ arch/x86/kernel/cpu/mce/amd.c                      |  138 +--
+ arch/x86/kernel/cpu/microcode/intel-ucode-defs.h   |  398 ++++---
+ arch/x86/kernel/cpu/microcode/intel.c              |   54 +-
+ arch/x86/kernel/cpu/mtrr/cleanup.c                 |    7 +-
+ arch/x86/kernel/reboot.c                           |    5 +-
+ arch/x86/kernel/traps.c                            |   12 +-
+ arch/x86/kernel/tsc.c                              |   14 +-
+ arch/x86/kernel/umip.c                             |    3 +
+ arch/x86/kernel/unwind_orc.c                       |   32 +-
+ arch/x86/kvm/vmx/tdx.c                             |    4 +-
+ arch/x86/kvm/vmx/tdx_arch.h                        |    6 -
+ arch/x86/mm/fault.c                                |    2 +-
+ arch/x86/mm/pat/set_memory.c                       |   37 +-
+ arch/x86/mm/tlb.c                                  |   21 -
+ arch/x86/platform/efi/efi_64.c                     |   35 +
+ arch/x86/virt/svm/sev.c                            |   77 +-
+ arch/x86/virt/vmx/tdx/tdx_global_metadata.c        |   22 +
+ drivers/acpi/osl.c                                 |    2 +-
+ drivers/char/random.c                              |   16 +-
+ drivers/clocksource/Kconfig                        |    1 -
+ drivers/edac/mce_amd.c                             |   48 +-
+ drivers/irqchip/Kconfig                            |   11 +-
+ drivers/irqchip/irq-apple-aic.c                    |   24 +-
+ drivers/irqchip/irq-gic-v3.c                       |    2 +-
+ drivers/irqchip/irq-imx-irqsteer.c                 |   55 +-
+ drivers/irqchip/irq-msi-lib.c                      |    3 +
+ drivers/irqchip/irq-mvebu-odmi.c                   |    2 +-
+ drivers/irqchip/irq-pic32-evic.c                   |    8 +-
+ drivers/irqchip/irq-renesas-rzv2h.c                |  482 +++++++--
+ drivers/virt/coco/tdx-guest/tdx-guest.c            |    5 +
+ include/asm-generic/bitsperlong.h                  |    9 +
+ include/asm-generic/futex.h                        |    4 +-
+ include/asm-generic/thread_info_tif.h              |    5 +-
+ include/linux/cleanup.h                            |   19 +-
+ include/linux/clockchips.h                         |    8 +-
+ include/linux/clocksource.h                        |    9 +-
+ include/linux/compiler-context-analysis.h          |   32 +
+ include/linux/energy_model.h                       |    4 +-
+ include/linux/hrtimer.h                            |   64 +-
+ include/linux/hrtimer_defs.h                       |   83 +-
+ include/linux/hrtimer_rearm.h                      |   83 ++
+ include/linux/hrtimer_types.h                      |   19 +-
+ include/linux/irq-entry-common.h                   |   25 +-
+ include/linux/irq.h                                |    4 +
+ include/linux/jiffies.h                            |    6 +-
+ include/linux/jump_label.h                         |   22 +-
+ include/linux/mod_devicetable.h                    |    2 +
+ include/linux/mutex.h                              |   13 +-
+ include/linux/mutex_types.h                        |    2 +-
+ include/linux/rbtree.h                             |   81 +-
+ include/linux/rbtree_types.h                       |   16 +
+ include/linux/rseq_entry.h                         |   16 +-
+ include/linux/rtmutex.h                            |    8 +-
+ include/linux/rwlock.h                             |   18 +-
+ include/linux/rwlock_api_smp.h                     |    6 +-
+ include/linux/rwsem.h                              |    8 +-
+ include/linux/sched/signal.h                       |    2 +-
+ include/linux/sched/topology.h                     |    2 +-
+ include/linux/semaphore.h                          |    4 +-
+ include/linux/spinlock.h                           |    3 +-
+ include/linux/spinlock_up.h                        |   20 +-
+ include/linux/timekeeper_internal.h                |    8 +
+ include/linux/timerqueue.h                         |   56 +-
+ include/linux/timerqueue_types.h                   |   15 +-
+ include/linux/trace_events.h                       |   13 +-
+ include/linux/vdso_datastore.h                     |    6 +
+ include/linux/wait_bit.h                           |    4 +-
+ include/linux/ww_mutex.h                           |    4 +-
+ include/trace/events/timer.h                       |   42 +-
+ include/trace/stages/stage3_trace_output.h         |   40 +-
+ include/uapi/linux/sched.h                         |    3 +
+ include/vdso/datapage.h                            |   27 +-
+ include/vdso/helpers.h                             |   31 +-
+ init/main.c                                        |    2 +
+ kernel/entry/common.c                              |    4 +-
+ kernel/events/core.c                               |   37 +-
+ kernel/futex/Makefile                              |    2 +
+ kernel/futex/core.c                                |    9 +-
+ kernel/futex/futex.h                               |   17 +-
+ kernel/futex/pi.c                                  |    9 +
+ kernel/futex/waitwake.c                            |    4 +
+ kernel/irq/matrix.c                                |    2 +-
+ kernel/jump_label.c                                |    9 -
+ kernel/locking/Makefile                            |    5 +
+ kernel/locking/mutex-debug.c                       |    5 +-
+ kernel/locking/mutex.c                             |   82 +-
+ kernel/locking/mutex.h                             |    1 +
+ kernel/locking/rtmutex.c                           |   18 +-
+ kernel/locking/rtmutex_api.c                       |    2 +
+ kernel/locking/rtmutex_common.h                    |   27 +-
+ kernel/locking/rwbase_rt.c                         |    1 +
+ kernel/locking/rwsem.c                             |  113 +-
+ kernel/locking/semaphore.c                         |   39 +-
+ kernel/locking/spinlock.c                          |   12 +-
+ kernel/locking/ww_mutex.h                          |   49 +-
+ kernel/locking/ww_rt_mutex.c                       |    1 +
+ kernel/sched/core.c                                |  104 +-
+ kernel/sched/deadline.c                            |   25 +-
+ kernel/sched/debug.c                               |   14 +-
+ kernel/sched/ext.c                                 |    4 +-
+ kernel/sched/fair.c                                |  454 +++++---
+ kernel/sched/features.h                            |    8 +
+ kernel/sched/rt.c                                  |    7 +-
+ kernel/sched/sched.h                               |   59 +-
+ kernel/sched/syscalls.c                            |   16 +-
+ kernel/sched/topology.c                            |  266 +++--
+ kernel/softirq.c                                   |   15 +-
+ kernel/time/.kunitconfig                           |    2 +
+ kernel/time/Kconfig                                |   20 +-
+ kernel/time/clockevents.c                          |   48 +-
+ kernel/time/clocksource.c                          |   14 +
+ kernel/time/hrtimer.c                              | 1139 +++++++++++---------
+ kernel/time/tick-broadcast-hrtimer.c               |    1 -
+ kernel/time/tick-sched.c                           |   27 +-
+ kernel/time/timekeeping.c                          |  207 +++-
+ kernel/time/timekeeping.h                          |    2 +
+ kernel/time/timer_list.c                           |   16 +-
+ kernel/trace/trace_events_synth.c                  |    4 +-
+ kernel/trace/trace_output.c                        |   20 +-
+ kernel/trace/trace_syscalls.c                      |    3 +-
+ lib/Kconfig.debug                                  |    8 +
+ lib/rbtree.c                                       |   17 +
+ lib/timerqueue.c                                   |   14 +
+ lib/vdso/datastore.c                               |   97 +-
+ lib/vdso/getrandom.c                               |    3 +
+ lib/vdso/gettimeofday.c                            |   99 +-
+ rust/helpers/atomic_ext.c                          |  152 +--
+ rust/kernel/list/arc.rs                            |    8 +-
+ rust/kernel/sync/atomic.rs                         |  310 +++++-
+ rust/kernel/sync/atomic/internal.rs                |   44 +-
+ rust/kernel/sync/atomic/predefine.rs               |  109 ++
+ scripts/context-analysis-suppression.txt           |    1 +
+ scripts/gdb/linux/timerlist.py                     |    2 +-
+ scripts/livepatch/Makefile                         |   20 +
+ scripts/livepatch/klp-build                        |  131 ++-
+ tools/arch/x86/include/asm/amd/ibs.h               |    2 +-
+ tools/arch/x86/include/asm/orc_types.h             |    9 +-
+ tools/objtool/arch/x86/decode.c                    |   18 +-
+ tools/objtool/arch/x86/orc.c                       |   31 +-
+ tools/objtool/check.c                              |    4 +-
+ tools/objtool/disas.c                              |    6 +-
+ tools/objtool/elf.c                                |  101 +-
+ tools/objtool/include/objtool/elf.h                |    3 +
+ tools/objtool/klp-diff.c                           |   99 +-
+ .../selftests/futex/functional/futex_requeue.c     |   49 +-
+ tools/testing/selftests/vDSO/Makefile              |    6 +-
+ tools/testing/selftests/vDSO/parse_vdso.c          |    3 +-
+ .../testing/selftests/vDSO/vdso_test_correctness.c |  112 +-
+ .../selftests/vDSO/vdso_test_gettimeofday.c        |    2 -
+ 219 files changed, 5563 insertions(+), 3343 deletions(-)
+ create mode 100644 arch/sparc/include/asm/vdso/clocksource.h
+ create mode 100644 arch/sparc/include/asm/vdso/gettimeofday.h
+ create mode 100644 arch/sparc/include/asm/vdso/processor.h
+ create mode 100644 arch/sparc/include/asm/vdso/vsyscall.h
+ delete mode 100644 arch/sparc/include/asm/vvar.h
+ delete mode 100644 arch/sparc/kernel/vdso.c
+ create mode 100644 arch/x86/include/asm/clock_inlined.h
+ create mode 100644 include/linux/hrtimer_rearm.h
+ create mode 100644 kernel/time/.kunitconfig
+ create mode 100644 scripts/livepatch/Makefile
+Merging clockevents/timers/drivers/next (a83f9c04be4e8 clocksource/drivers/mips-gic-timer: Move GIC timer to request_percpu_irq())
+$ git merge -m Merge branch 'timers/drivers/next' of https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Already up to date.
+Merging edac/edac-for-next (b7699a3214441 Merge ras/edac-drivers into for-next)
+$ git merge -m Merge branch 'edac-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Merge made by the 'ort' strategy.
+ drivers/edac/altera_edac.h       |   1 -
+ drivers/edac/amd64_edac.c        |   3 +
+ drivers/edac/edac_device.h       |   2 +-
+ drivers/edac/edac_device_sysfs.c |  24 +++--
+ drivers/edac/edac_pci_sysfs.c    |  10 +--
+ drivers/edac/i10nm_base.c        |  14 +++
+ drivers/edac/i7core_edac.c       |  15 +---
+ drivers/edac/mpc85xx_edac.c      |   2 +-
+ drivers/edac/sb_edac.c           |  14 +--
+ drivers/edac/versalnet_edac.c    | 186 ++++++++++++++++++++++-----------------
+ 10 files changed, 144 insertions(+), 127 deletions(-)
+Merging ftrace/for-next (60447f5637950 Merge ring-buffer/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Merge made by the 'ort' strategy.
+$ git am -3 ../patches/0001-ftrace-Fix-semantic-conflict-with-mm-tree.patch
+Applying: ftrace: Fix semantic conflict with mm tree
+Using index info to reconstruct a base tree...
+M	kernel/trace/trace_printk.c
+Falling back to patching base and 3-way merge...
+Auto-merging kernel/trace/trace_printk.c
+No changes -- Patch already applied.
+Merging rcu/next (ed062c41dfda2 Merge branch 'rcu-nocb.20260123a')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux rcu/next
+Already up to date.
+Merging paulmck/non-rcu/next (0075737499dc8 kernel/smp: Improve smp_call_function_single() CSD-lock diagnostics)
+$ git merge -m Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git paulmck/non-rcu/next
+Merge made by the 'ort' strategy.
+ include/linux/stop_machine.h |  4 ++--
+ kernel/smp.c                 | 16 +++++++++++++++-
+ 2 files changed, 17 insertions(+), 3 deletions(-)
+Merging kvm/next (d2ea4ff1ce507 KVM: selftests: Verify SEV+ guests can read and write EFER, CR0, CR4, and CR8)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Already up to date.
+Merging kvm-arm/next (e3d585ed3ff89 Merge branch kvm-arm64/vgic-v5-ppi into kvmarm-master/next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Already up to date.
+Merging kvms390/next (e3372ffb5f9e2 KVM: s390: Increase permitted SE header size to 1 MiB)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Already up to date.
+Merging kvm-ppc/topic/ppc-kvm (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'topic/ppc-kvm' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (c61ec3e8cc5d4 RISC-V: KVM: Check host Ssaia extension when creating AIA irqchip)
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Already up to date.
+Merging kvm-x86/next (3d6cdcc8883b5 Merge branch 'vmxon')
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Auto-merging arch/x86/kernel/cpu/common.c
+Auto-merging arch/x86/kernel/reboot.c
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+CONFLICT (content): Merge conflict in arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+Auto-merging include/linux/kvm_host.h
+Auto-merging include/uapi/linux/kvm.h
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Auto-merging tools/testing/selftests/kvm/guest_memfd_test.c
+Auto-merging virt/kvm/guest_memfd.c
+CONFLICT (content): Merge conflict in virt/kvm/guest_memfd.c
+Auto-merging virt/kvm/kvm_main.c
+Resolved 'arch/x86/virt/vmx/tdx/tdx_global_metadata.c' using previous resolution.
+Resolved 'virt/kvm/guest_memfd.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master a0ac0b3df821e] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/arch/x86/tdx.rst                     |  36 +-
+ arch/x86/events/intel/pt.c                         |   1 -
+ arch/x86/include/asm/cpufeatures.h                 |   1 +
+ arch/x86/include/asm/kvm-x86-ops.h                 |  10 +-
+ arch/x86/include/asm/kvm-x86-pmu-ops.h             |   8 +-
+ arch/x86/include/asm/kvm_host.h                    |  37 +-
+ arch/x86/include/asm/reboot.h                      |  11 -
+ arch/x86/include/asm/svm.h                         |  20 +-
+ arch/x86/include/asm/tdx.h                         |   4 -
+ arch/x86/include/asm/virt.h                        |  26 +
+ arch/x86/include/asm/vmx.h                         |  11 +
+ arch/x86/include/uapi/asm/kvm.h                    |  12 +-
+ arch/x86/kernel/cpu/common.c                       |   2 +
+ arch/x86/kernel/crash.c                            |   3 +-
+ arch/x86/kernel/reboot.c                           |  63 +--
+ arch/x86/kernel/smp.c                              |   5 +-
+ arch/x86/kvm/cpuid.c                               |   6 +-
+ arch/x86/kvm/emulate.c                             |  23 +-
+ arch/x86/kvm/hyperv.h                              |   8 -
+ arch/x86/kvm/lapic.c                               |  13 +-
+ arch/x86/kvm/mmu/mmu.c                             |  14 +
+ arch/x86/kvm/mmu/tdp_mmu.c                         |   2 +-
+ arch/x86/kvm/pmu.c                                 |   2 +-
+ arch/x86/kvm/svm/avic.c                            |  17 +-
+ arch/x86/kvm/svm/hyperv.h                          |   9 +-
+ arch/x86/kvm/svm/nested.c                          | 604 ++++++++++++---------
+ arch/x86/kvm/svm/sev.c                             |  36 +-
+ arch/x86/kvm/svm/svm.c                             | 449 +++++++++------
+ arch/x86/kvm/svm/svm.h                             |  78 ++-
+ arch/x86/kvm/svm/vmenter.S                         |  10 +-
+ arch/x86/kvm/vmx/capabilities.h                    |   2 +-
+ arch/x86/kvm/vmx/main.c                            |  21 +-
+ arch/x86/kvm/vmx/tdx.c                             | 224 +-------
+ arch/x86/kvm/vmx/tdx.h                             |   8 +-
+ arch/x86/kvm/vmx/vmcs.h                            |  11 -
+ arch/x86/kvm/vmx/vmcs_shadow_fields.h              |   5 +-
+ arch/x86/kvm/vmx/vmenter.S                         |   2 +-
+ arch/x86/kvm/vmx/vmx.c                             | 142 +----
+ arch/x86/kvm/vmx/vmx_ops.h                         |   5 +-
+ arch/x86/kvm/x86.c                                 | 460 +++++++++-------
+ arch/x86/kvm/x86.h                                 |  32 +-
+ arch/x86/virt/Makefile                             |   2 +
+ arch/x86/virt/hw.c                                 | 359 ++++++++++++
+ arch/x86/virt/vmx/tdx/tdx.c                        | 326 ++++++-----
+ arch/x86/virt/vmx/tdx/tdx.h                        |   8 -
+ arch/x86/virt/vmx/tdx/tdx_global_metadata.c        |  10 +-
+ include/linux/kvm_host.h                           |  19 +-
+ include/uapi/linux/kvm.h                           |  11 +-
+ tools/testing/selftests/kvm/Makefile.kvm           |   3 +
+ tools/testing/selftests/kvm/guest_memfd_test.c     |  70 ++-
+ tools/testing/selftests/kvm/include/kvm_syscalls.h |   1 +
+ .../testing/selftests/kvm/include/x86/processor.h  |  12 +
+ tools/testing/selftests/kvm/include/x86/svm.h      |  14 +-
+ tools/testing/selftests/kvm/lib/x86/processor.c    |  15 +-
+ tools/testing/selftests/kvm/lib/x86/svm.c          |   2 +-
+ .../testing/selftests/kvm/x86/fix_hypercall_test.c |   2 +-
+ tools/testing/selftests/kvm/x86/msrs_test.c        |   4 +-
+ .../selftests/kvm/x86/nested_vmsave_vmload_test.c  |  16 +-
+ .../selftests/kvm/x86/pmu_event_filter_test.c      |   3 +-
+ tools/testing/selftests/kvm/x86/state_test.c       |  35 ++
+ .../selftests/kvm/x86/svm_lbr_nested_state.c       | 145 +++++
+ .../selftests/kvm/x86/svm_nested_clear_efer_svme.c |  55 ++
+ .../kvm/x86/svm_nested_invalid_vmcb12_gpa.c        |  98 ++++
+ tools/testing/selftests/kvm/x86/xapic_state_test.c |   2 +-
+ virt/kvm/guest_memfd.c                             |   5 +-
+ virt/kvm/kvm_main.c                                |  43 +-
+ 66 files changed, 2294 insertions(+), 1399 deletions(-)
+ create mode 100644 arch/x86/include/asm/virt.h
+ create mode 100644 arch/x86/virt/hw.c
+ create mode 100644 tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
+ create mode 100644 tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
+ create mode 100644 tools/testing/selftests/kvm/x86/svm_nested_invalid_vmcb12_gpa.c
+$ git am -3 ../patches/0001-KVM-selftests-Fix-up-semantic-changes.patch
+Applying: KVM: selftests: Fix up semantic changes
+Using index info to reconstruct a base tree...
+M	tools/testing/selftests/kvm/lib/kvm_util.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+$ git am -3 ../patches/kvm-x86-init
+Applying: Fix up merge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master b741827cfcb6b] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+ Date: Fri Mar 20 14:19:20 2026 +0000
+$ git am -3 ../patches/0001-kvm-fix-up-merge-issue.patch
+Applying: kvm: fix up merge issue
+$ git reset HEAD^
+Unstaged changes after reset:
+M	virt/kvm/guest_memfd.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master df8a5fa879b6d] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+ Date: Fri Mar 20 14:19:20 2026 +0000
+Merging xen-tip/linux-next (e2dcf9065536a xen/xenbus: better handle backend crash)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Already up to date.
+Merging percpu/for-next (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Already up to date.
+Merging workqueues/for-next (5fb24c9823490 Merge branch 'for-7.1' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Auto-merging kernel/workqueue.c
+Merge made by the 'ort' strategy.
+ drivers/firmware/efi/efi.c |  2 +-
+ include/linux/workqueue.h  | 22 ++++++++++++++--------
+ kernel/workqueue.c         | 44 ++++++++++++++++++++++++++------------------
+ tools/workqueue/wq_dump.py | 17 ++++++-----------
+ 4 files changed, 47 insertions(+), 38 deletions(-)
+Merging sched-ext/for-next (f6689792ffc4b selftests/sched_ext: Show failed test names in summary)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git sched-ext/for-next
+Auto-merging init/Kconfig
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/sched/core.c
+Auto-merging kernel/sched/sched.h
+Merge made by the 'ort' strategy.
+ Documentation/scheduler/sched-ext.rst              |   78 +-
+ include/linux/cgroup-defs.h                        |    4 +
+ include/linux/cgroup.h                             |   65 +-
+ include/linux/sched/ext.h                          |   81 +-
+ init/Kconfig                                       |    4 +
+ kernel/cgroup/cgroup-internal.h                    |    6 -
+ kernel/cgroup/cgroup.c                             |   55 -
+ kernel/fork.c                                      |    6 +-
+ kernel/sched/core.c                                |    2 +-
+ kernel/sched/ext.c                                 | 3596 ++++++++++++++++----
+ kernel/sched/ext.h                                 |    4 +-
+ kernel/sched/ext_idle.c                            |  106 +-
+ kernel/sched/ext_internal.h                        |  342 +-
+ kernel/sched/sched.h                               |   11 +-
+ tools/sched_ext/include/scx/bpf_arena_common.bpf.h |    8 +-
+ tools/sched_ext/include/scx/common.bpf.h           |  278 ++
+ tools/sched_ext/include/scx/common.h               |    4 -
+ tools/sched_ext/include/scx/compat.bpf.h           |   49 +-
+ tools/sched_ext/include/scx/compat.h               |   36 +-
+ tools/sched_ext/include/scx/enum_defs.autogen.h    |   51 +-
+ tools/sched_ext/include/scx/enums.autogen.bpf.h    |    2 +
+ tools/sched_ext/include/scx/enums.autogen.h        |    1 +
+ tools/sched_ext/scx_central.bpf.c                  |    4 +-
+ tools/sched_ext/scx_central.c                      |    2 +-
+ tools/sched_ext/scx_cpu0.bpf.c                     |    2 +-
+ tools/sched_ext/scx_flatcg.bpf.c                   |   22 +-
+ tools/sched_ext/scx_pair.c                         |    2 +-
+ tools/sched_ext/scx_qmap.bpf.c                     |  174 +-
+ tools/sched_ext/scx_qmap.c                         |   29 +-
+ tools/sched_ext/scx_sdt.bpf.c                      |    2 +-
+ tools/sched_ext/scx_sdt.c                          |    2 +-
+ tools/sched_ext/scx_simple.bpf.c                   |    8 +-
+ tools/sched_ext/scx_userland.c                     |    2 +-
+ tools/testing/selftests/sched_ext/Makefile         |    1 +
+ tools/testing/selftests/sched_ext/dequeue.bpf.c    |  389 +++
+ tools/testing/selftests/sched_ext/dequeue.c        |  274 ++
+ tools/testing/selftests/sched_ext/exit.bpf.c       |    2 +-
+ tools/testing/selftests/sched_ext/exit.c           |    2 +-
+ tools/testing/selftests/sched_ext/exit_test.h      |    2 +-
+ tools/testing/selftests/sched_ext/maximal.bpf.c    |   17 +-
+ tools/testing/selftests/sched_ext/maximal.c        |    3 +
+ tools/testing/selftests/sched_ext/numa.bpf.c       |    2 +-
+ tools/testing/selftests/sched_ext/peek_dsq.bpf.c   |    8 +-
+ tools/testing/selftests/sched_ext/reload_loop.c    |    3 +
+ tools/testing/selftests/sched_ext/runner.c         |   10 +-
+ .../selftests/sched_ext/select_cpu_vtime.bpf.c     |    8 +-
+ tools/testing/selftests/sched_ext/util.h           |    2 +-
+ 47 files changed, 4740 insertions(+), 1021 deletions(-)
+ create mode 100644 tools/testing/selftests/sched_ext/dequeue.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/dequeue.c
+Merging drivers-x86/for-next (6da22b031a3cf platform/x86: fujitsu: Convert laptop driver to a platform one)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Auto-merging drivers/platform/x86/lenovo/thinkpad_acpi.c
+Merge made by the 'ort' strategy.
+ .../admin-guide/laptops/thinkpad-acpi.rst          |  21 +
+ drivers/gpu/drm/xe/xe_debugfs.c                    |   2 +-
+ drivers/gpu/drm/xe/xe_hwmon.c                      |   2 +-
+ drivers/gpu/drm/xe/xe_vsec.c                       |   7 +-
+ drivers/gpu/drm/xe/xe_vsec.h                       |   4 +-
+ drivers/input/mouse/trackpoint.c                   |  46 ++
+ drivers/input/mouse/trackpoint.h                   |   5 +
+ drivers/platform/surface/surfacepro3_button.c      |  71 +--
+ drivers/platform/x86/acer-wireless.c               |  48 +-
+ drivers/platform/x86/asus-wmi.c                    |  14 -
+ drivers/platform/x86/dell/dell-wmi-sysman/sysman.c |  44 +-
+ drivers/platform/x86/eeepc-laptop.c                |  45 +-
+ drivers/platform/x86/fujitsu-laptop.c              | 519 +++++++++++----------
+ drivers/platform/x86/fujitsu-tablet.c              |  30 +-
+ drivers/platform/x86/intel/pmc/core.c              |   4 +-
+ drivers/platform/x86/intel/pmc/ssram_telemetry.c   |   2 +-
+ drivers/platform/x86/intel/pmt/class.c             |   8 +-
+ drivers/platform/x86/intel/pmt/class.h             |   5 +-
+ drivers/platform/x86/intel/pmt/discovery.c         |   4 +-
+ drivers/platform/x86/intel/pmt/telemetry.c         |  13 +-
+ drivers/platform/x86/intel/pmt/telemetry.h         |  12 +-
+ drivers/platform/x86/intel/rst.c                   |  23 +-
+ drivers/platform/x86/intel/sdsi.c                  |   5 +-
+ drivers/platform/x86/intel/smartconnect.c          |  23 +-
+ drivers/platform/x86/intel/vsec.c                  | 121 +++--
+ drivers/platform/x86/intel/vsec_tpmi.c             |   8 +-
+ drivers/platform/x86/lenovo/thinkpad_acpi.c        | 128 +++--
+ drivers/platform/x86/lg-laptop.c                   |  51 +-
+ drivers/platform/x86/mxm-wmi.c                     |  12 -
+ drivers/platform/x86/pcengines-apuv2.c             |   3 +-
+ drivers/platform/x86/sony-laptop.c                 | 122 ++---
+ drivers/platform/x86/topstar-laptop.c              |  43 +-
+ drivers/platform/x86/wireless-hotkey.c             |  49 +-
+ include/linux/intel_vsec.h                         |  39 +-
+ 34 files changed, 837 insertions(+), 696 deletions(-)
+$ git am -3 ../patches/0001-hid-Fix-up-mismerge.patch
+Applying: hid: Fix up mismerge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/hid/hid-asus.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 661f42536352f] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git
+ Date: Fri Mar 20 14:19:32 2026 +0000
+Merging chrome-platform/for-next (168e4b208ca8c platform/chrome: cros_usbpd_logger: Simplify with devm)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Merge made by the 'ort' strategy.
+ drivers/platform/chrome/chromeos_privacy_screen.c | 41 ++++++-------
+ drivers/platform/chrome/chromeos_tbmc.c           | 55 +++++++++++------
+ drivers/platform/chrome/cros_ec_lightbar.c        |  3 +
+ drivers/platform/chrome/cros_usbpd_logger.c       | 18 +++---
+ drivers/platform/chrome/wilco_ec/event.c          | 73 +++++++++++++----------
+ 5 files changed, 107 insertions(+), 83 deletions(-)
+Merging chrome-platform-firmware/for-firmware-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-firmware-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Already up to date.
+Merging hsi/for-next (e61af3ca89336 hsi: hsi_core: use kzalloc_flex)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Merge made by the 'ort' strategy.
+ drivers/hsi/hsi_core.c  | 37 +++++++++++++++----------------------
+ include/linux/hsi/hsi.h |  2 +-
+ 2 files changed, 16 insertions(+), 23 deletions(-)
+Merging leds-lj/for-leds-next (e68f95a51d1a8 leds: Prefer IS_ERR_OR_NULL over manual NULL check)
+$ git merge -m Merge branch 'for-leds-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/leds/sprd,sc2731-bltc.yaml |   7 +-
+ drivers/leds/blink/leds-lgm-sso.c                  |   2 -
+ drivers/leds/flash/leds-ktd2692.c                  |   2 +-
+ drivers/leds/led-class-multicolor.c                |   4 +-
+ drivers/leds/leds-lp5569.c                         |   4 +-
+ drivers/leds/leds-lp8860.c                         | 119 +++++++++------------
+ drivers/leds/rgb/leds-qcom-lpg.c                   |   7 +-
+ drivers/leds/trigger/ledtrig-tty.c                 |   2 +-
+ 8 files changed, 71 insertions(+), 76 deletions(-)
+Merging ipmi/for-next (122d16da1313f ipmi: Replace use of system_wq with system_percpu_wq)
+$ git merge -m Merge branch 'for-next' of https://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Merge made by the 'ort' strategy.
+ drivers/char/ipmi/ipmi_msghandler.c | 10 +++++-----
+ 1 file changed, 5 insertions(+), 5 deletions(-)
+Merging driver-core/driver-core-next (9aa64d2503c6f rust: devres: embed struct devres_node directly)
+$ git merge -m Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core/driver-core-next
+Auto-merging drivers/base/core.c
+Auto-merging drivers/base/platform.c
+Auto-merging drivers/of/base.c
+CONFLICT (content): Merge conflict in drivers/of/base.c
+Auto-merging fs/kernfs/dir.c
+Auto-merging fs/kernfs/inode.c
+Auto-merging include/linux/device.h
+Auto-merging include/linux/of.h
+Auto-merging include/linux/platform_device.h
+Auto-merging tools/testing/selftests/cgroup/test_memcontrol.c
+Resolved 'drivers/of/base.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 4d112a4279eec] Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/base/base.h                              |  97 +++++---
+ drivers/base/core.c                              |   5 +-
+ drivers/base/devres.c                            | 281 +++++++++++++++--------
+ drivers/base/platform.c                          |  58 ++---
+ drivers/base/soc.c                               |  29 ++-
+ drivers/of/base.c                                |  28 +++
+ drivers/soc/fsl/guts.c                           |  12 +-
+ drivers/soc/imx/soc-imx8m.c                      |  11 +-
+ drivers/soc/imx/soc-imx9.c                       |   4 +-
+ drivers/soc/sunxi/sunxi_mbus.c                   |   2 +-
+ fs/kernfs/dir.c                                  |  58 ++++-
+ fs/kernfs/inode.c                                |   2 +-
+ fs/sysfs/group.c                                 |  10 +-
+ include/linux/device.h                           |   5 +-
+ include/linux/device/class.h                     |   4 +-
+ include/linux/of.h                               |  14 ++
+ include/linux/platform_device.h                  |  58 ++++-
+ include/linux/sys_soc.h                          |  10 +
+ include/linux/sysfs.h                            |  16 +-
+ rust/kernel/devres.rs                            | 185 +++++++++++----
+ rust/kernel/irq/request.rs                       |  28 +--
+ tools/testing/selftests/cgroup/test_memcontrol.c | 112 +++++++++
+ 22 files changed, 732 insertions(+), 297 deletions(-)
+Merging usb/usb-next (84db3719d2733 usb: dwc3: imx: avoid calling imx suspend/resume callbacks twice)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging drivers/usb/typec/ucsi/ucsi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-power        |   3 +-
+ .../bindings/connector/usb-connector.yaml          |  39 +-
+ .../devicetree/bindings/usb/cdns,usb3.yaml         |   1 +
+ .../devicetree/bindings/usb/corechips,sl6341.yaml  |  79 ++
+ .../devicetree/bindings/usb/fsl,imx8mp-dwc3.yaml   |   2 +
+ .../devicetree/bindings/usb/maxim,max33359.yaml    |   4 +
+ .../devicetree/bindings/usb/maxim,max3421.txt      |  23 -
+ .../devicetree/bindings/usb/maxim,max3421.yaml     |  67 ++
+ .../bindings/usb/microchip,mpfs-musb.yaml          |   3 +
+ .../devicetree/bindings/usb/nxp,imx-dwc3.yaml      | 123 +++
+ Documentation/devicetree/bindings/usb/ohci-st.txt  |  36 -
+ Documentation/devicetree/bindings/usb/omap-usb.txt |  80 --
+ .../devicetree/bindings/usb/qcom,snps-dwc3.yaml    |   2 +-
+ .../devicetree/bindings/usb/richtek,rt1711h.yaml   |   3 +-
+ .../devicetree/bindings/usb/st,st-ohci-300x.yaml   |  85 ++
+ Documentation/devicetree/bindings/usb/ti,dwc3.yaml | 100 +++
+ .../devicetree/bindings/usb/ti,omap4-musb.yaml     | 120 +++
+ .../devicetree/bindings/vendor-prefixes.yaml       |   4 +
+ drivers/power/supply/power_supply_sysfs.c          |   2 +
+ drivers/usb/core/endpoint.c                        |   8 -
+ drivers/usb/core/message.c                         |  20 +-
+ drivers/usb/core/port.c                            |  21 +-
+ drivers/usb/dwc3/Kconfig                           |  12 +
+ drivers/usb/dwc3/Makefile                          |   1 +
+ drivers/usb/dwc3/core.c                            |  29 +-
+ drivers/usb/dwc3/core.h                            |   9 +-
+ drivers/usb/dwc3/dwc3-imx.c                        | 448 +++++++++++
+ drivers/usb/dwc3/dwc3-qcom.c                       |   4 +-
+ drivers/usb/dwc3/gadget.c                          |   2 +-
+ drivers/usb/dwc3/glue.h                            |   3 +
+ drivers/usb/dwc3/ulpi.c                            |  25 +
+ drivers/usb/gadget/function/f_midi2.c              |   6 +-
+ drivers/usb/gadget/function/u_serial.c             |   2 +-
+ drivers/usb/gadget/udc/pxa27x_udc.c                |  68 +-
+ drivers/usb/gadget/udc/snps_udc_core.c             |   2 +-
+ drivers/usb/host/ehci-orion.c                      |   9 -
+ drivers/usb/host/fhci-hcd.c                        |  15 +-
+ drivers/usb/host/fhci.h                            |   3 +-
+ drivers/usb/host/xhci-tegra.c                      |   4 -
+ drivers/usb/image/microtek.c                       |  79 +-
+ drivers/usb/misc/apple-mfi-fastcharge.c            |   3 +-
+ drivers/usb/misc/cypress_cy7c63.c                  |   4 +-
+ drivers/usb/misc/cytherm.c                         |   4 +-
+ drivers/usb/misc/iowarrior.c                       |  34 +-
+ drivers/usb/misc/onboard_usb_dev.c                 |   3 +
+ drivers/usb/misc/onboard_usb_dev.h                 |  39 +-
+ drivers/usb/misc/trancevibrator.c                  |   3 +-
+ drivers/usb/misc/usb-ljca.c                        |  15 +-
+ drivers/usb/misc/usbsevseg.c                       |   3 +-
+ drivers/usb/misc/uss720.c                          |  27 +-
+ drivers/usb/musb/musb_core.c                       |   2 +-
+ drivers/usb/renesas_usbhs/mod_gadget.c             |  16 +-
+ drivers/usb/serial/ti_usb_3410_5052.c              |  16 +-
+ drivers/usb/storage/uas.c                          |   4 +-
+ drivers/usb/typec/mux.c                            |  25 +-
+ drivers/usb/typec/mux/ps883x.c                     |   1 +
+ drivers/usb/typec/tcpm/Kconfig                     |   2 +
+ drivers/usb/typec/tcpm/fusb302.c                   |  13 +
+ drivers/usb/typec/tcpm/tcpci.c                     |   2 +-
+ drivers/usb/typec/tcpm/tcpci_rt1711h.c             |  16 +-
+ drivers/usb/typec/tcpm/tcpm.c                      | 889 ++++++++++++++++++---
+ drivers/usb/typec/tipd/core.c                      |   6 +-
+ drivers/usb/typec/ucsi/ucsi.c                      |   8 +-
+ drivers/usb/typec/ucsi/ucsi.h                      |   3 +
+ drivers/usb/typec/ucsi/ucsi_glink.c                |   4 +
+ drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c        |   2 +-
+ drivers/usb/usbip/stub_dev.c                       |   5 +-
+ drivers/usb/usbip/vhci_sysfs.c                     |   2 +-
+ include/dt-bindings/usb/pd.h                       |  36 +
+ include/linux/power_supply.h                       |   3 +
+ include/linux/usb/cdc_ncm.h                        |   4 +-
+ include/linux/usb/pd.h                             | 108 ++-
+ include/linux/usb/tcpm.h                           |   2 +-
+ include/linux/usb/typec_altmode.h                  |   3 +
+ 74 files changed, 2338 insertions(+), 515 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/usb/corechips,sl6341.yaml
+ delete mode 100644 Documentation/devicetree/bindings/usb/maxim,max3421.txt
+ create mode 100644 Documentation/devicetree/bindings/usb/maxim,max3421.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/nxp,imx-dwc3.yaml
+ delete mode 100644 Documentation/devicetree/bindings/usb/ohci-st.txt
+ delete mode 100644 Documentation/devicetree/bindings/usb/omap-usb.txt
+ create mode 100644 Documentation/devicetree/bindings/usb/st,st-ohci-300x.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,dwc3.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
+ create mode 100644 drivers/usb/dwc3/dwc3-imx.c
+Merging thunderbolt/next (500e54d449f60 thunderbolt: dma_port: kmalloc_array + kzalloc to flex)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                    |  2 +-
+ drivers/thunderbolt/dma_port.c | 15 +++------------
+ drivers/thunderbolt/quirks.c   |  7 +++++++
+ drivers/thunderbolt/switch.c   | 30 ++++++++++++++++++++++++++----
+ 4 files changed, 37 insertions(+), 17 deletions(-)
+Merging usb-serial/usb-next (5a6a33b564021 USB: serial: ti_usb_3410_5052: use strscpy() instead of strcpy())
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Merge made by the 'ort' strategy.
+Merging tty/tty-next (eb3b0d92c9c39 tty: tty_port: add workqueue to flip TTY buffer)
+$ git merge -m Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Auto-merging drivers/tty/serial/8250/8250_pci.c
+Auto-merging drivers/tty/serial/8250/8250_port.c
+Auto-merging drivers/tty/vt/vt.c
+Merge made by the 'ort' strategy.
+ .../bindings/serial/amlogic,meson-uart.yaml        |   1 +
+ .../bindings/serial/atmel,at91-usart.yaml          |   1 +
+ .../bindings/serial/snps-dw-apb-uart.yaml          |   1 +
+ drivers/tty/hvc/hvc_iucv.c                         |   2 +-
+ drivers/tty/pty.c                                  |  12 +-
+ drivers/tty/serial/8250/8250_core.c                |   2 +-
+ drivers/tty/serial/8250/8250_pci.c                 |  51 ++
+ drivers/tty/serial/8250/8250_port.c                |   4 +-
+ drivers/tty/serial/Kconfig                         |  33 +-
+ drivers/tty/serial/Makefile                        |   2 -
+ drivers/tty/serial/amba-pl011.c                    | 153 +++-
+ drivers/tty/serial/apbuart.c                       |   2 +-
+ drivers/tty/serial/dz.c                            |   2 +-
+ drivers/tty/serial/esp32_acm.c                     | 459 ------------
+ drivers/tty/serial/esp32_uart.c                    | 779 ---------------------
+ drivers/tty/serial/imx.c                           |   7 +-
+ drivers/tty/serial/ip22zilog.c                     |   2 +-
+ drivers/tty/serial/mxs-auart.c                     |   3 +-
+ drivers/tty/serial/qcom_geni_serial.c              |  19 +-
+ drivers/tty/serial/samsung_tty.c                   |  10 +-
+ drivers/tty/serial/zs.c                            |   2 +-
+ drivers/tty/tty_buffer.c                           |  15 +-
+ drivers/tty/tty_io.c                               |  25 +-
+ drivers/tty/tty_ldisc.c                            |  16 +-
+ drivers/tty/tty_port.c                             |  22 +
+ drivers/tty/vt/keyboard.c                          |  80 ++-
+ drivers/tty/vt/vt.c                                |  48 +-
+ include/linux/serdev.h                             |   9 +-
+ include/linux/tty_buffer.h                         |   1 +
+ include/linux/tty_driver.h                         |   7 +
+ include/linux/tty_ldisc.h                          |   6 +-
+ include/linux/tty_port.h                           |  13 +
+ include/uapi/linux/keyboard.h                      |  29 +
+ 33 files changed, 464 insertions(+), 1354 deletions(-)
+ delete mode 100644 drivers/tty/serial/esp32_acm.c
+ delete mode 100644 drivers/tty/serial/esp32_uart.c
+Merging char-misc/char-misc-next (88cf8a9ad32f8 mux: mmio: Zero the allocated memory)
+$ git merge -m Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+Merge made by the 'ort' strategy.
+ drivers/android/binder.c                     |   2 +-
+ drivers/android/binder/context.rs            |  11 ++
+ drivers/android/binder/process.rs            |   3 +
+ drivers/android/binder/thread.rs             |   6 +
+ drivers/android/binder/transaction.rs        |   4 +-
+ drivers/gpib/agilent_82357a/agilent_82357a.c |   5 +-
+ drivers/gpib/common/gpib_os.c                |  17 ++-
+ drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c   | 206 +++++++++++++------------
+ drivers/gpib/nec7210/nec7210.c               |  12 --
+ drivers/gpib/ni_usb/ni_usb_gpib.c            |   4 -
+ drivers/gpib/tms9914/tms9914.c               |  13 --
+ drivers/mux/mmio.c                           |   8 +-
+ rust/kernel/alloc/kvec.rs                    | 216 ++++++++++++++++++++++++++-
+ 13 files changed, 355 insertions(+), 152 deletions(-)
+$ git am -3 ../patches/0001-bus-mhi-Fix-up-interaction-with-net-next-tree.patch
+Applying: bus: mhi: Fix up interaction with net-next tree
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging coresight/next (a90863095f84f MAINTAINERS: coresight: Add Leo Yan as Reviewer)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   1 +
+ MAINTAINERS                                        |   5 +-
+ drivers/hwtracing/coresight/coresight-catu.c       |   4 +-
+ drivers/hwtracing/coresight/coresight-core.c       | 216 ++++++++------
+ drivers/hwtracing/coresight/coresight-ctcu-core.c  |   5 +-
+ drivers/hwtracing/coresight/coresight-cti-core.c   | 321 ++++-----------------
+ drivers/hwtracing/coresight/coresight-cti-sysfs.c  | 171 +++++------
+ drivers/hwtracing/coresight/coresight-cti.h        |  15 +-
+ drivers/hwtracing/coresight/coresight-dummy.c      |   7 +-
+ drivers/hwtracing/coresight/coresight-etb10.c      |   4 +-
+ drivers/hwtracing/coresight/coresight-etm-perf.c   |   5 +-
+ drivers/hwtracing/coresight/coresight-funnel.c     |   4 +-
+ drivers/hwtracing/coresight/coresight-platform.c   |   2 +-
+ drivers/hwtracing/coresight/coresight-priv.h       |   3 +-
+ drivers/hwtracing/coresight/coresight-replicator.c |   4 +-
+ drivers/hwtracing/coresight/coresight-stm.c        |   4 +-
+ drivers/hwtracing/coresight/coresight-tmc-core.c   |  15 +-
+ drivers/hwtracing/coresight/coresight-tnoc.c       |   4 +-
+ drivers/hwtracing/coresight/coresight-tpda.c       |   4 +-
+ drivers/hwtracing/coresight/coresight-tpdm.c       |   4 +-
+ drivers/hwtracing/coresight/coresight-tpiu.c       |   4 +-
+ drivers/hwtracing/coresight/ultrasoc-smb.c         |   4 +-
+ include/linux/coresight.h                          |  14 +-
+ 23 files changed, 314 insertions(+), 506 deletions(-)
+Merging fastrpc/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Already up to date.
+Merging fpga/for-next (26e5e1be2e640 fpga: bridge: Use sysfs_emit() instead of sprintf())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Merge made by the 'ort' strategy.
+ drivers/fpga/fpga-bridge.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging icc/icc-next (89b6f071c4a07 Merge branch 'icc-fixes' into icc-next)
+$ git merge -m Merge branch 'icc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Merge made by the 'ort' strategy.
+ .../bindings/interconnect/qcom,glymur-rpmh.yaml    |  136 +-
+ .../bindings/interconnect/qcom,osm-l3.yaml         |    1 +
+ .../bindings/interconnect/qcom,qcs615-rpmh.yaml    |   23 +
+ .../bindings/interconnect/qcom,qcs8300-rpmh.yaml   |   63 +
+ drivers/interconnect/qcom/Kconfig                  |   13 +-
+ drivers/interconnect/qcom/Makefile                 |    2 +
+ drivers/interconnect/qcom/eliza.c                  | 1585 ++++++++++++++++++++
+ drivers/interconnect/qcom/glymur.c                 |   40 +-
+ drivers/interconnect/qcom/qcs615.c                 |  247 +++
+ drivers/interconnect/qcom/qcs8300.c                |  375 +++++
+ drivers/interconnect/qcom/sm8450.c                 |    4 +-
+ 11 files changed, 2454 insertions(+), 35 deletions(-)
+ create mode 100644 drivers/interconnect/qcom/eliza.c
+Merging iio/togreg (ff0843ceb1fb1 iio: adc: ti-ads7950: remove chip_info[])
+$ git merge -m Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Auto-merging MAINTAINERS
+Auto-merging drivers/iio/adc/ad4062.c
+Auto-merging drivers/iio/adc/ad7768-1.c
+Auto-merging drivers/iio/adc/ade9000.c
+Auto-merging drivers/iio/adc/ti-ads1119.c
+Auto-merging drivers/iio/adc/ti-ads7950.c
+Auto-merging drivers/iio/frequency/adf4377.c
+Auto-merging drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+Auto-merging drivers/iio/proximity/hx9023s.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-bus-iio            |   2 +-
+ .../bindings/iio/accel/bosch,bma255.yaml           |  38 +-
+ .../devicetree/bindings/iio/adc/adi,ad4030.yaml    |  64 ++
+ .../devicetree/bindings/iio/adc/adi,ad4080.yaml    |   3 +
+ .../devicetree/bindings/iio/adc/adi,ad7380.yaml    |  23 +
+ .../bindings/iio/adc/motorola,cpcap-adc.yaml       |   1 +
+ .../bindings/iio/adc/qcom,spmi-adc5-gen3.yaml      | 151 ++++
+ .../bindings/iio/adc/qcom,spmi-vadc-common.yaml    |  84 ++
+ .../bindings/iio/adc/qcom,spmi-vadc.yaml           |  78 +-
+ .../bindings/iio/amplifiers/adi,ad8366.yaml        |  97 +++
+ .../devicetree/bindings/iio/dac/ti,dac7612.yaml    |   2 +-
+ .../bindings/iio/gyroscope/bosch,bmg160.yaml       |  12 +-
+ .../bindings/iio/light/vishay,vcnl4000.yaml        |  17 +-
+ .../iio/magnetometer/bosch,bmc150_magn.yaml        |  14 +-
+ .../bindings/iio/proximity/tyhx,hx9023s.yaml       |   4 +
+ .../devicetree/bindings/mfd/qcom,spmi-pmic.yaml    |   1 +
+ Documentation/iio/ad4030.rst                       |  39 +
+ Documentation/iio/adxl345.rst                      |  89 ++-
+ MAINTAINERS                                        |  17 +
+ drivers/iio/accel/adxl313_core.c                   |  11 +-
+ drivers/iio/accel/adxl345_core.c                   |  32 +-
+ drivers/iio/accel/adxl372.c                        |  15 +-
+ drivers/iio/accel/sca3000.c                        | 215 +++---
+ drivers/iio/adc/Kconfig                            |  33 +
+ drivers/iio/adc/Makefile                           |   1 +
+ drivers/iio/adc/ad4030.c                           | 618 ++++++++++++++-
+ drivers/iio/adc/ad4062.c                           |  12 +-
+ drivers/iio/adc/ad4080.c                           |  46 +-
+ drivers/iio/adc/ad4170-4.c                         |   6 +-
+ drivers/iio/adc/ad7173.c                           |   3 +-
+ drivers/iio/adc/ad7192.c                           |   3 -
+ drivers/iio/adc/ad7266.c                           |   6 +-
+ drivers/iio/adc/ad7380.c                           |  53 +-
+ drivers/iio/adc/ad7768-1.c                         | 221 +++++-
+ drivers/iio/adc/ad7793.c                           |   3 +-
+ drivers/iio/adc/ad7887.c                           |   2 +-
+ drivers/iio/adc/ad7923.c                           |   4 +-
+ drivers/iio/adc/ad9467.c                           |  95 ++-
+ drivers/iio/adc/ade9000.c                          |  13 +-
+ drivers/iio/adc/adi-axi-adc.c                      |   5 +
+ drivers/iio/adc/at91-sama5d2_adc.c                 |   5 +-
+ drivers/iio/adc/at91_adc.c                         |  12 +-
+ drivers/iio/adc/cpcap-adc.c                        |  15 +
+ drivers/iio/adc/fsl-imx25-gcq.c                    |   2 +-
+ drivers/iio/adc/max1363.c                          |  87 +--
+ drivers/iio/adc/mcp3564.c                          |   2 +-
+ drivers/iio/adc/men_z188_adc.c                     |   2 +-
+ drivers/iio/adc/meson_saradc.c                     |  13 +-
+ drivers/iio/adc/nau7802.c                          |   3 +-
+ drivers/iio/adc/npcm_adc.c                         |   2 +-
+ drivers/iio/adc/nxp-sar-adc.c                      |   3 +-
+ drivers/iio/adc/pac1921.c                          |   2 +-
+ drivers/iio/adc/pac1934.c                          |   2 +-
+ drivers/iio/adc/palmas_gpadc.c                     |   8 +-
+ drivers/iio/adc/qcom-spmi-adc5-gen3.c              | 860 +++++++++++++++++++++
+ drivers/iio/adc/rohm-bd79124.c                     |   4 +-
+ drivers/iio/adc/sd_adc_modulator.c                 |   1 +
+ drivers/iio/adc/spear_adc.c                        |   2 +-
+ drivers/iio/adc/stm32-adc-core.c                   |   2 +-
+ drivers/iio/adc/stm32-adc.c                        |   2 +-
+ drivers/iio/adc/sun20i-gpadc-iio.c                 |   2 +-
+ drivers/iio/adc/ti-ads1015.c                       |   1 -
+ drivers/iio/adc/ti-ads1119.c                       |   3 +-
+ drivers/iio/adc/ti-ads7950.c                       | 172 ++---
+ drivers/iio/adc/twl4030-madc.c                     |   2 +-
+ drivers/iio/adc/twl6030-gpadc.c                    |   2 +-
+ drivers/iio/addac/ad74413r.c                       |   9 +-
+ drivers/iio/amplifiers/Kconfig                     |   6 +
+ drivers/iio/amplifiers/ad8366.c                    | 387 +++++-----
+ drivers/iio/buffer/industrialio-hw-consumer.c      |  11 +-
+ drivers/iio/buffer/kfifo_buf.c                     |  41 +-
+ drivers/iio/dac/ad5360.c                           |  29 +-
+ drivers/iio/dac/ad7293.c                           |  31 +-
+ drivers/iio/dac/adi-axi-dac.c                      |   3 +-
+ drivers/iio/dac/max5522.c                          |  22 +-
+ drivers/iio/dac/ti-dac5571.c                       |   1 -
+ drivers/iio/filter/admv8818.c                      |  60 +-
+ drivers/iio/frequency/ad9523.c                     |  70 +-
+ drivers/iio/frequency/adf4350.c                    |   2 +-
+ drivers/iio/frequency/adf4377.c                    |  56 +-
+ drivers/iio/frequency/admv1013.c                   | 104 +--
+ drivers/iio/frequency/admv1014.c                   |  89 +--
+ drivers/iio/frequency/admv4420.c                   |   2 +-
+ drivers/iio/frequency/adrf6780.c                   |  58 +-
+ drivers/iio/gyro/hid-sensor-gyro-3d.c              |   6 +-
+ drivers/iio/humidity/hdc2010.c                     |   1 -
+ drivers/iio/imu/bmi323/bmi323_core.c               |   1 -
+ drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c  |   2 +-
+ drivers/iio/industrialio-backend.c                 |  16 +
+ drivers/iio/industrialio-event.c                   |   1 +
+ drivers/iio/industrialio-trigger.c                 |  33 +-
+ drivers/iio/light/acpi-als.c                       |  42 +-
+ drivers/iio/light/apds9306.c                       |   4 +-
+ drivers/iio/light/gp2ap020a00f.c                   | 310 +++-----
+ drivers/iio/light/ltr501.c                         |   6 +-
+ drivers/iio/light/vcnl4000.c                       |  41 +
+ drivers/iio/light/zopt2201.c                       |  47 +-
+ drivers/iio/magnetometer/bmc150_magn.c             | 145 ++--
+ drivers/iio/magnetometer/si7210.c                  |  21 +-
+ drivers/iio/pressure/hsc030pa_i2c.c                |   7 +-
+ drivers/iio/proximity/hx9023s.c                    |   5 +-
+ drivers/staging/iio/adc/ad7816.c                   |  16 +-
+ drivers/staging/iio/impedance-analyzer/ad5933.c    |   4 +-
+ include/linux/iio/adc/qcom-adc5-gen3-common.h      | 211 +++++
+ include/linux/iio/backend.h                        |  24 +
+ include/linux/iio/types.h                          |   1 +
+ include/linux/platform_data/tsl2772.h              |   4 +-
+ 107 files changed, 3811 insertions(+), 1462 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/qcom,spmi-adc5-gen3.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc-common.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/amplifiers/adi,ad8366.yaml
+ create mode 100644 drivers/iio/adc/qcom-spmi-adc5-gen3.c
+ create mode 100644 include/linux/iio/adc/qcom-adc5-gen3-common.h
+Merging phy-next/next (caf08514bbee0 dt-bindings: phy: qcom,sc8280xp-qmp-ufs-phy: document the Eliza QMP UFS PHY)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Auto-merging drivers/phy/Kconfig
+Merge made by the 'ort' strategy.
+ .../bindings/phy/canaan,k230-usb-phy.yaml          |  35 +++
+ .../bindings/phy/eswin,eic7700-sata-phy.yaml       |  92 +++++++
+ .../devicetree/bindings/phy/mediatek,dsi-phy.yaml  |   1 +
+ .../bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml    |   8 +-
+ drivers/phy/Kconfig                                | 102 ++++----
+ drivers/phy/Makefile                               |  12 +-
+ drivers/phy/apple/atc.c                            |  10 +-
+ drivers/phy/canaan/Kconfig                         |  14 +
+ drivers/phy/canaan/Makefile                        |   2 +
+ drivers/phy/canaan/phy-k230-usb.c                  | 284 +++++++++++++++++++++
+ drivers/phy/eswin/Kconfig                          |  14 +
+ drivers/phy/eswin/Makefile                         |   2 +
+ drivers/phy/eswin/phy-eic7700-sata.c               | 273 ++++++++++++++++++++
+ drivers/phy/marvell/phy-mmp3-hsic.c                |  24 +-
+ drivers/phy/mediatek/phy-mtk-tphy.c                |   4 +-
+ drivers/phy/qualcomm/phy-qcom-m31-eusb2.c          |   2 +-
+ drivers/phy/qualcomm/phy-qcom-qmp-usbc.c           |   6 +-
+ drivers/phy/spacemit/Kconfig                       |  12 +
+ drivers/phy/spacemit/Makefile                      |   1 +
+ .../phy-k1-pcie.c}                                 |   0
+ 20 files changed, 817 insertions(+), 81 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/phy/canaan,k230-usb-phy.yaml
+ create mode 100644 Documentation/devicetree/bindings/phy/eswin,eic7700-sata-phy.yaml
+ create mode 100644 drivers/phy/canaan/Kconfig
+ create mode 100644 drivers/phy/canaan/Makefile
+ create mode 100644 drivers/phy/canaan/phy-k230-usb.c
+ create mode 100644 drivers/phy/eswin/Kconfig
+ create mode 100644 drivers/phy/eswin/Makefile
+ create mode 100644 drivers/phy/eswin/phy-eic7700-sata.c
+ rename drivers/phy/{phy-spacemit-k1-pcie.c => spacemit/phy-k1-pcie.c} (100%)
+Merging soundwire/next (b8f2d65fec19f soundwire: intel_auxdevice: Add cs42l49 to wake_capable_list)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Merge made by the 'ort' strategy.
+ drivers/soundwire/amd_manager.c     | 100 ++++++++++++++++++++++++++++++++----
+ drivers/soundwire/amd_manager.h     |   4 --
+ drivers/soundwire/bus.c             |   8 +--
+ drivers/soundwire/cadence_master.c  |   8 +++
+ drivers/soundwire/intel_ace2x.c     |   5 ++
+ drivers/soundwire/intel_auxdevice.c |   2 +
+ drivers/soundwire/slave.c           |   3 ++
+ drivers/soundwire/stream.c          |  22 +++++---
+ include/linux/soundwire/sdw.h       |  18 +++----
+ include/linux/soundwire/sdw_amd.h   |   4 ++
+ 10 files changed, 138 insertions(+), 36 deletions(-)
+Merging extcon/extcon-next (1bf0ba46d9d2c extcon: usbc-tusb320: Make typec-power-opmode optional)
+$ git merge -m Merge branch 'extcon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/extcon/extcon-ptn5150.yaml |  3 ++
+ drivers/extcon/Kconfig                             |  1 +
+ drivers/extcon/extcon-intel-int3496.c              |  4 +-
+ drivers/extcon/extcon-ptn5150.c                    | 56 ++++++++++++++++++++++
+ drivers/extcon/extcon-usbc-tusb320.c               | 24 +++++-----
+ drivers/extcon/extcon.c                            |  4 +-
+ 6 files changed, 75 insertions(+), 17 deletions(-)
+Merging gnss/gnss-next (f338e77383789 Linux 7.0-rc4)
+$ git merge -m Merge branch 'gnss-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (96ca4caf9066f vfio/fsl-mc: add myself as maintainer)
+$ git merge -m Merge branch 'next' of https://github.com/awilliam/linux-vfio.git vfio/next
+Already up to date.
+Merging w1/for-next (f2c1ae05fe820 w1: ds2490: drop redundant device reference)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Merge made by the 'ort' strategy.
+ drivers/w1/masters/ds2490.c | 11 +++--------
+ 1 file changed, 3 insertions(+), 8 deletions(-)
+Merging spmi/spmi-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'spmi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git spmi/spmi-next
+Already up to date.
+Merging staging/staging-next (82e1c68ac206e staging: rtl8723bs: add missing blank lines after declarations)
+$ git merge -m Merge branch 'staging-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Auto-merging drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+Merge made by the 'ort' strategy.
+ drivers/staging/axis-fifo/axis-fifo.c              |   59 +-
+ drivers/staging/fbtft/Kconfig                      |    6 +-
+ drivers/staging/fbtft/README                       |   29 +-
+ drivers/staging/fbtft/fb_agm1264k-fl.c             |    2 +-
+ drivers/staging/fbtft/fb_hx8340bn.c                |    8 -
+ drivers/staging/fbtft/fb_hx8353d.c                 |   13 -
+ drivers/staging/fbtft/fb_hx8357d.c                 |   14 -
+ drivers/staging/fbtft/fb_ili9163.c                 |    2 +-
+ drivers/staging/fbtft/fb_ili9340.c                 |   12 -
+ drivers/staging/fbtft/fb_ili9341.c                 |   12 -
+ drivers/staging/fbtft/fb_ili9481.c                 |   12 -
+ drivers/staging/fbtft/fb_ili9486.c                 |   12 -
+ drivers/staging/fbtft/fb_s6d02a1.c                 |   12 -
+ drivers/staging/fbtft/fb_st7735r.c                 |   12 -
+ drivers/staging/fbtft/fb_tinylcd.c                 |   14 +-
+ drivers/staging/fbtft/fbtft-core.c                 |   13 +-
+ .../greybus/Documentation/firmware/authenticate.c  |    2 +-
+ drivers/staging/greybus/Kconfig                    |    2 +-
+ drivers/staging/greybus/arche-apb-ctrl.c           |   10 +-
+ drivers/staging/greybus/arche-platform.c           |   10 +-
+ drivers/staging/greybus/audio_manager.c            |   12 -
+ drivers/staging/greybus/audio_manager.h            |    7 -
+ drivers/staging/greybus/audio_manager_module.c     |   13 +-
+ drivers/staging/greybus/loopback.c                 |   18 +-
+ drivers/staging/greybus/pwm.c                      |    2 +-
+ drivers/staging/greybus/sdio.c                     |    4 +-
+ .../staging/most/Documentation/driver_usage.txt    |   10 +-
+ drivers/staging/most/dim2/dim2.c                   |   76 +-
+ drivers/staging/most/dim2/hal.c                    |    7 +-
+ drivers/staging/most/net/net.c                     |    1 -
+ drivers/staging/nvec/nvec.c                        |   21 +-
+ drivers/staging/octeon/ethernet-rx.c               |    2 +-
+ drivers/staging/octeon/ethernet-tx.c               |    2 -
+ drivers/staging/octeon/octeon-stubs.h              | 1614 ++++++++++----------
+ drivers/staging/rtl8723bs/core/rtw_ap.c            |   39 +-
+ drivers/staging/rtl8723bs/core/rtw_btcoex.c        |    1 -
+ drivers/staging/rtl8723bs/core/rtw_cmd.c           |  111 +-
+ drivers/staging/rtl8723bs/core/rtw_efuse.c         |   24 +-
+ drivers/staging/rtl8723bs/core/rtw_ieee80211.c     |  131 +-
+ drivers/staging/rtl8723bs/core/rtw_io.c            |    9 +-
+ drivers/staging/rtl8723bs/core/rtw_ioctl_set.c     |   13 +-
+ drivers/staging/rtl8723bs/core/rtw_mlme.c          |   48 +-
+ drivers/staging/rtl8723bs/core/rtw_mlme_ext.c      |   63 +-
+ drivers/staging/rtl8723bs/core/rtw_pwrctrl.c       |  104 +-
+ drivers/staging/rtl8723bs/core/rtw_recv.c          |   93 +-
+ drivers/staging/rtl8723bs/core/rtw_security.c      |   16 +-
+ drivers/staging/rtl8723bs/core/rtw_sta_mgt.c       |   36 +-
+ drivers/staging/rtl8723bs/core/rtw_wlan_util.c     |   16 +-
+ drivers/staging/rtl8723bs/core/rtw_xmit.c          |   10 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c    |   60 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |   36 +-
+ drivers/staging/rtl8723bs/hal/hal_btcoex.c         |    7 +-
+ drivers/staging/rtl8723bs/hal/hal_com.c            |   14 +-
+ drivers/staging/rtl8723bs/hal/hal_com_phycfg.c     |   24 +-
+ drivers/staging/rtl8723bs/hal/odm.h                |    9 +-
+ drivers/staging/rtl8723bs/hal/odm_DIG.c            |   28 +-
+ drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c |    6 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c       |    2 +
+ drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c  |    9 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c    |    1 +
+ drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c     |    1 +
+ drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c     |    6 +-
+ drivers/staging/rtl8723bs/hal/sdio_halinit.c       |   11 +-
+ drivers/staging/rtl8723bs/hal/sdio_ops.c           |   12 +-
+ drivers/staging/rtl8723bs/include/hal_com.h        |    2 -
+ drivers/staging/rtl8723bs/include/ieee80211.h      |   15 +-
+ drivers/staging/rtl8723bs/include/ioctl_cfg80211.h |   60 -
+ drivers/staging/rtl8723bs/include/rtl8192c_recv.h  |    4 -
+ drivers/staging/rtl8723bs/include/rtw_cmd.h        |    2 +-
+ drivers/staging/rtl8723bs/include/rtw_efuse.h      |   19 -
+ drivers/staging/rtl8723bs/include/rtw_mlme.h       |   36 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme_ext.h   |    4 +-
+ drivers/staging/rtl8723bs/include/wlan_bssdef.h    |    5 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |   43 +-
+ drivers/staging/rtl8723bs/os_dep/os_intfs.c        |   10 +-
+ drivers/staging/sm750fb/ddk750_mode.c              |    2 +-
+ drivers/staging/sm750fb/ddk750_swi2c.c             |    2 +-
+ drivers/staging/sm750fb/sm750.c                    |   88 +-
+ drivers/staging/sm750fb/sm750_hw.c                 |   26 +-
+ drivers/staging/vme_user/vme_user.h                |   10 -
+ 80 files changed, 1430 insertions(+), 1883 deletions(-)
+Merging counter-next/counter-next (ad415677b7e3b MAINTAINERS: Update Intel Quadrature Encoder Peripheral maintainer)
+$ git merge -m Merge branch 'counter-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Already up to date.
+Merging mux/for-next (59b723cd2adba Linux 6.12-rc6)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Already up to date.
+Merging dmaengine/next (b7560798466a0 dmaengine: dw-edma: Add non-LL mode)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/dma/xilinx/xilinx_dma.c
+Merge made by the 'ort' strategy.
+ .../bindings/dma/loongson,ls2k0300-dma.yaml        |   81 ++
+ .../devicetree/bindings/dma/renesas,rz-dmac.yaml   |  102 +-
+ .../devicetree/bindings/dma/snps,dw-axi-dmac.yaml  |    3 +
+ .../devicetree/bindings/dma/xilinx/xilinx_dma.txt  |  111 --
+ .../bindings/dma/xilinx/xlnx,axi-dma.yaml          |  299 ++++
+ MAINTAINERS                                        |   14 +-
+ drivers/dma/Kconfig                                |   36 +-
+ drivers/dma/Makefile                               |    4 +-
+ drivers/dma/dma-axi-dmac.c                         |  170 ++-
+ drivers/dma/dmaengine.c                            |   50 +-
+ drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c     |   42 +-
+ drivers/dma/dw-axi-dmac/dw-axi-dmac.h              |    1 +
+ drivers/dma/dw-edma/dw-edma-core.c                 |  174 ++-
+ drivers/dma/dw-edma/dw-edma-core.h                 |   18 +
+ drivers/dma/dw-edma/dw-edma-pcie.c                 |  220 ++-
+ drivers/dma/dw-edma/dw-edma-v0-core.c              |   21 +
+ drivers/dma/dw-edma/dw-hdma-v0-core.c              |   71 +-
+ drivers/dma/dw-edma/dw-hdma-v0-regs.h              |    1 +
+ drivers/dma/fsl-edma-common.c                      |    3 +-
+ drivers/dma/fsl-edma-main.c                        |   55 +-
+ drivers/dma/fsl-qdma.c                             |   47 +-
+ drivers/dma/imx-sdma.c                             |   56 +-
+ drivers/dma/ioat/dma.h                             |   13 +-
+ drivers/dma/ioat/sysfs.c                           |   32 +-
+ drivers/dma/loongson/Kconfig                       |   41 +
+ drivers/dma/loongson/Makefile                      |    4 +
+ drivers/dma/{ => loongson}/loongson1-apb-dma.c     |    4 +-
+ drivers/dma/loongson/loongson2-apb-cmc-dma.c       |  730 ++++++++++
+ drivers/dma/{ => loongson}/loongson2-apb-dma.c     |   93 +-
+ drivers/dma/mxs-dma.c                              |   37 +-
+ drivers/dma/sh/rz-dmac.c                           |  304 ++++-
+ drivers/dma/switchtec_dma.c                        | 1437 ++++++++++++++++++++
+ drivers/dma/xilinx/xdma.c                          |    2 +
+ drivers/dma/xilinx/xilinx_dma.c                    |    8 +-
+ include/linux/dma/edma.h                           |    7 +
+ include/linux/dma/qcom-gpi-dma.h                   |    5 +-
+ include/linux/dma/ti-cppi5.h                       |   53 +-
+ include/linux/dmaengine.h                          |    3 +-
+ include/linux/of_dma.h                             |   29 +
+ include/linux/platform_data/dma-mcf-edma.h         |    5 +-
+ 40 files changed, 3852 insertions(+), 534 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/dma/loongson,ls2k0300-dma.yaml
+ delete mode 100644 Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt
+ create mode 100644 Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
+ create mode 100644 drivers/dma/loongson/Kconfig
+ create mode 100644 drivers/dma/loongson/Makefile
+ rename drivers/dma/{ => loongson}/loongson1-apb-dma.c (99%)
+ create mode 100644 drivers/dma/loongson/loongson2-apb-cmc-dma.c
+ rename drivers/dma/{ => loongson}/loongson2-apb-dma.c (91%)
+ create mode 100644 drivers/dma/switchtec_dma.c
+Merging cgroup/for-next (4c7c17b16c98e Merge branch 'for-7.1' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Auto-merging include/linux/cgroup-defs.h
+Auto-merging kernel/cgroup/cgroup.c
+Merge made by the 'ort' strategy.
+ include/linux/cgroup-defs.h |  1 +
+ kernel/cgroup/cgroup.c      | 55 ++++++++++++++++++++++++---------------------
+ 2 files changed, 30 insertions(+), 26 deletions(-)
+Merging scsi/for-next (8387c81a4eb9e Merge branch 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Auto-merging drivers/target/loopback/tcm_loop.c
+Auto-merging drivers/usb/gadget/function/f_tcm.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-driver-ufs         |  23 +++++
+ .../devicetree/bindings/ufs/qcom,sc7180-ufshc.yaml |  38 ++++---
+ drivers/infiniband/ulp/srpt/ib_srpt.c              |   1 +
+ drivers/scsi/BusLogic.c                            |   4 +-
+ drivers/scsi/elx/efct/efct_lio.c                   |   2 +
+ drivers/scsi/fnic/fdls_disc.c                      |   4 +-
+ drivers/scsi/fnic/fip.c                            |   2 +-
+ drivers/scsi/fnic/fnic.h                           |   7 +-
+ drivers/scsi/fnic/fnic_fcs.c                       | 112 +++++++++++++++++----
+ drivers/scsi/fnic/fnic_fdls.h                      |   2 +-
+ drivers/scsi/fnic/fnic_main.c                      |  28 +++++-
+ drivers/scsi/fnic/fnic_scsi.c                      |  70 ++-----------
+ drivers/scsi/hisi_sas/hisi_sas_main.c              |   2 +-
+ drivers/scsi/hisi_sas/hisi_sas_v3_hw.c             |  12 +--
+ drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c           |   1 +
+ drivers/scsi/lpfc/lpfc.h                           |  11 +-
+ drivers/scsi/lpfc/lpfc_crtn.h                      |   3 +-
+ drivers/scsi/lpfc/lpfc_ct.c                        |  13 +--
+ drivers/scsi/lpfc/lpfc_disc.h                      |   5 +-
+ drivers/scsi/lpfc/lpfc_els.c                       |  30 ++++--
+ drivers/scsi/lpfc/lpfc_hbadisc.c                   |  29 +++---
+ drivers/scsi/lpfc/lpfc_init.c                      |   6 +-
+ drivers/scsi/lpfc/lpfc_nvme.c                      |  50 ++++++++-
+ drivers/scsi/lpfc/lpfc_scsi.c                      |  10 +-
+ drivers/scsi/lpfc/lpfc_sli.c                       |  85 +++++++---------
+ drivers/scsi/lpfc/lpfc_sli4.h                      |   5 +-
+ drivers/scsi/lpfc/lpfc_version.h                   |   6 +-
+ drivers/scsi/megaraid/megaraid_sas_base.c          |  15 +--
+ drivers/scsi/qla2xxx/qla_attr.c                    |  62 +++++++++++-
+ drivers/scsi/qla2xxx/qla_init.c                    |   2 +-
+ drivers/scsi/qla2xxx/qla_mbx.c                     |   9 ++
+ drivers/scsi/qla2xxx/tcm_qla2xxx.c                 |   2 +
+ drivers/scsi/scsi_lib.c                            |  47 +++++++++
+ drivers/scsi/scsi_scan.c                           |  10 +-
+ drivers/scsi/scsi_sysfs.c                          |  16 +++
+ drivers/scsi/sd.c                                  |  80 ++++++++++++---
+ drivers/scsi/sg.c                                  |  88 +++++++---------
+ drivers/target/iscsi/iscsi_target_configfs.c       |   1 +
+ drivers/target/loopback/tcm_loop.c                 |   1 +
+ drivers/target/sbp/sbp_target.c                    |   1 +
+ drivers/target/target_core_configfs.c              |  22 ++++
+ drivers/target/target_core_device.c                |   1 +
+ drivers/target/target_core_fabric_configfs.c       |  24 +++++
+ drivers/target/target_core_sbc.c                   |   3 +-
+ drivers/target/target_core_transport.c             |  60 +++++++++--
+ drivers/target/tcm_fc/tfc_conf.c                   |   1 +
+ drivers/ufs/core/ufs-mcq.c                         |   9 +-
+ drivers/ufs/core/ufs-sysfs.c                       |  30 ++++++
+ drivers/ufs/core/ufshcd.c                          |  49 +++++++--
+ drivers/ufs/host/ufs-mediatek.c                    |  11 ++
+ drivers/ufs/host/ufs-mediatek.h                    |   4 +
+ drivers/ufs/host/ufshcd-pci.c                      |   1 +
+ drivers/usb/gadget/function/f_tcm.c                |   1 +
+ drivers/vhost/scsi.c                               |   2 +
+ drivers/xen/xen-scsiback.c                         |   1 +
+ include/scsi/scsi_device.h                         |   1 +
+ include/scsi/scsi_host.h                           |   7 +-
+ include/target/target_core_base.h                  |  10 ++
+ include/target/target_core_fabric.h                |  12 ++-
+ include/ufs/ufshcd.h                               |  15 +++
+ include/ufs/ufshci.h                               |   1 +
+ 61 files changed, 843 insertions(+), 317 deletions(-)
+Merging scsi-mkp/for-next (48b2de8505437 scsi: lpfc: Use the crc32c() function)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Auto-merging drivers/scsi/lpfc/lpfc_init.c
+Auto-merging drivers/ufs/core/ufshcd.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml | 14 ++++++
+ .../bindings/ufs/rockchip,rk3576-ufshc.yaml        |  7 +--
+ drivers/scsi/Kconfig                               |  1 +
+ drivers/scsi/lpfc/lpfc.h                           |  2 -
+ drivers/scsi/lpfc/lpfc_crtn.h                      |  2 +-
+ drivers/scsi/lpfc/lpfc_els.c                       |  6 +--
+ drivers/scsi/lpfc/lpfc_init.c                      | 58 +++-------------------
+ drivers/ufs/core/ufs-mcq.c                         | 13 ++++-
+ drivers/ufs/core/ufshcd-priv.h                     |  2 +
+ drivers/ufs/core/ufshcd.c                          | 16 ++++--
+ include/ufs/ufshci.h                               |  2 +
+ 11 files changed, 59 insertions(+), 64 deletions(-)
+Merging vhost/linux-next (ebcff9dacaf2c vduse: avoid adding implicit padding)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Already up to date.
+Merging rpmsg/for-next (dfc5c561d9fc7 Merge branches 'rpmsg-next', 'rproc-fixes' and 'rproc-next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/remoteproc/qcom,msm8916-mss-pil.yaml  |  13 +-
+ .../bindings/remoteproc/qcom,sm8550-pas.yaml       |  14 +-
+ drivers/remoteproc/da8xx_remoteproc.c              |  97 ++++-----
+ drivers/remoteproc/imx_rproc.c                     |   8 +-
+ drivers/remoteproc/keystone_remoteproc.c           |  43 ++--
+ drivers/remoteproc/pru_rproc.c                     |  17 +-
+ drivers/remoteproc/qcom_q6v5_mss.c                 | 237 ++++++++++++++++++++-
+ drivers/remoteproc/qcom_sysmon.c                   |   2 +-
+ drivers/remoteproc/ti_k3_common.c                  |   2 +-
+ drivers/remoteproc/xlnx_r5_remoteproc.c            |  28 ++-
+ drivers/rpmsg/rpmsg_core.c                         |   4 +-
+ 11 files changed, 340 insertions(+), 125 deletions(-)
+Merging gpio-brgl/gpio/for-next (a6e53d05ab849 gpio: cs5535: use dynamically allocated priv struct)
+$ git merge -m Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Auto-merging drivers/gpio/gpiolib.c
+Auto-merging drivers/regulator/Kconfig
+Auto-merging include/linux/mfd/kempld.h
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/gpio/gpio-delay.yaml       |   2 +-
+ .../bindings/gpio/microchip,mpfs-gpio.yaml         |   3 +
+ .../bindings/gpio/realtek,otto-gpio.yaml           |   1 +
+ Documentation/driver-api/gpio/board.rst            |  16 --
+ Documentation/driver-api/gpio/driver.rst           |  27 ++
+ arch/arc/plat-axs10x/Kconfig                       |   1 -
+ arch/arm/mach-omap1/board-ams-delta.c              |  32 ++-
+ arch/powerpc/platforms/85xx/Kconfig                |   1 -
+ drivers/gpio/Kconfig                               |   4 +-
+ drivers/gpio/gpio-bcm-kona.c                       |  37 ++-
+ drivers/gpio/gpio-bd9571mwv.c                      |   2 +-
+ drivers/gpio/gpio-brcmstb.c                        | 125 ++++++---
+ drivers/gpio/gpio-cgbc.c                           |   4 +-
+ drivers/gpio/gpio-cs5535.c                         |  48 ++--
+ drivers/gpio/gpio-da9055.c                         |   2 +-
+ drivers/gpio/gpio-htc-egpio.c                      |  13 +-
+ drivers/gpio/gpio-kempld.c                         | 289 +++++++++++++++++++-
+ drivers/gpio/gpio-lp873x.c                         |   2 +-
+ drivers/gpio/gpio-max732x.c                        |  24 +-
+ drivers/gpio/gpio-mmio.c                           |   4 +-
+ drivers/gpio/gpio-mpsse.c                          |  13 +-
+ drivers/gpio/gpio-realtek-otto.c                   |   4 +
+ drivers/gpio/gpio-sim.c                            | 200 +++++++-------
+ drivers/gpio/gpio-stp-xway.c                       |   2 +-
+ drivers/gpio/gpio-tegra186.c                       |  45 ++--
+ drivers/gpio/gpio-tps65086.c                       |   2 +-
+ drivers/gpio/gpio-ts4800.c                         |  10 -
+ drivers/gpio/gpio-viperboard.c                     |   2 +-
+ drivers/gpio/gpiolib-acpi-core.c                   |  70 -----
+ drivers/gpio/gpiolib-cdev.c                        | 159 ++++-------
+ drivers/gpio/gpiolib-cdev.h                        |   2 +-
+ drivers/gpio/gpiolib-of.c                          | 152 ++---------
+ drivers/gpio/gpiolib-of.h                          |  10 +
+ drivers/gpio/gpiolib-sysfs.c                       |  30 +--
+ drivers/gpio/gpiolib-sysfs.h                       |   8 +-
+ drivers/gpio/gpiolib.c                             | 198 ++++++++------
+ drivers/gpio/gpiolib.h                             |   6 +-
+ drivers/regulator/Kconfig                          |   2 -
+ include/linux/gpio/consumer.h                      |   2 +
+ include/linux/gpio/defs.h                          |   9 +
+ include/linux/gpio/driver.h                        |  15 +-
+ include/linux/gpio/generic.h                       |   8 +-
+ include/linux/gpio/machine.h                       |  33 ---
+ include/linux/mfd/kempld.h                         |   1 +
+ tools/testing/selftests/gpio/Makefile              |   5 +-
+ tools/testing/selftests/gpio/gpio-cdev-uaf.c       | 292 +++++++++++++++++++++
+ tools/testing/selftests/gpio/gpio-cdev-uaf.sh      |  63 +++++
+ 47 files changed, 1217 insertions(+), 763 deletions(-)
+ create mode 100644 include/linux/gpio/defs.h
+ create mode 100644 tools/testing/selftests/gpio/gpio-cdev-uaf.c
+ create mode 100755 tools/testing/selftests/gpio/gpio-cdev-uaf.sh
+Merging gpio-intel/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Already up to date.
+Merging pinctrl/for-next (7c839b398814a Merge branch 'devel' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/pinctrl/fsl,imx27-iomuxc.yaml         |  126 ++
+ .../bindings/pinctrl/fsl,imx27-pinctrl.txt         |  121 --
+ .../bindings/pinctrl/fsl,imx35-pinctrl.yaml        |    1 +
+ .../pinctrl/marvell,armada3710-xb-pinctrl.yaml     |   11 +-
+ .../devicetree/bindings/pinctrl/pincfg-node.yaml   |  101 +-
+ .../bindings/pinctrl/qcom,eliza-tlmm.yaml          |  138 ++
+ .../pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml      |  109 ++
+ .../pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml     |    8 +-
+ .../pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml     |    4 +-
+ .../bindings/pinctrl/rockchip,pinctrl.yaml         |    1 +
+ drivers/gpio/gpio-rockchip.c                       |    2 +-
+ drivers/gpio/gpio-spacemit-k1.c                    |    1 +
+ drivers/pinctrl/Kconfig                            |   15 +-
+ drivers/pinctrl/bcm/Kconfig                        |    4 +-
+ drivers/pinctrl/core.c                             |    5 +-
+ drivers/pinctrl/devicetree.c                       |    2 +-
+ drivers/pinctrl/freescale/Kconfig                  |    2 +-
+ drivers/pinctrl/meson/Kconfig                      |    1 -
+ drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c     |    2 +-
+ drivers/pinctrl/nxp/pinctrl-s32.h                  |    2 +-
+ drivers/pinctrl/pinconf-generic.c                  |   92 +-
+ drivers/pinctrl/pinctrl-cy8c95x0.c                 |   93 +-
+ drivers/pinctrl/pinctrl-eyeq5.c                    |  437 ++++--
+ drivers/pinctrl/pinctrl-pic32.c                    |   64 +-
+ drivers/pinctrl/pinctrl-rockchip.c                 |  313 +++-
+ drivers/pinctrl/pinctrl-rockchip.h                 |    1 +
+ drivers/pinctrl/pinctrl-single.c                   |    1 +
+ drivers/pinctrl/qcom/Kconfig                       |   10 +
+ drivers/pinctrl/qcom/Kconfig.msm                   |   10 +
+ drivers/pinctrl/qcom/Makefile                      |    2 +
+ drivers/pinctrl/qcom/pinctrl-eliza.c               | 1548 ++++++++++++++++++++
+ drivers/pinctrl/qcom/pinctrl-glymur.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c     |  217 +++
+ drivers/pinctrl/qcom/pinctrl-spmi-gpio.c           |   16 +
+ drivers/pinctrl/realtek/pinctrl-rtd.c              |  156 +-
+ drivers/pinctrl/realtek/pinctrl-rtd.h              |   13 +
+ drivers/pinctrl/spacemit/pinctrl-k1.c              |   21 +-
+ drivers/pinctrl/starfive/Kconfig                   |    2 -
+ drivers/pinctrl/stm32/Kconfig                      |    1 +
+ drivers/pinctrl/sunplus/Kconfig                    |    1 -
+ drivers/pinctrl/tegra/pinctrl-tegra.c              |   10 +-
+ drivers/pinctrl/tegra/pinctrl-tegra.h              |    4 +-
+ include/linux/pinctrl/pinconf-generic.h            |    2 +-
+ 43 files changed, 3213 insertions(+), 459 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/fsl,imx27-iomuxc.yaml
+ delete mode 100644 Documentation/devicetree/bindings/pinctrl/fsl,imx27-pinctrl.txt
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-eliza.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
+Merging pinctrl-intel/for-next (1ca468e78ea97 pinctrl: intel: Enable 3-bit PAD_OWN feature)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/intel/pinctrl-intel.c | 36 +++++++++++++++++++++++++----------
+ drivers/pinctrl/intel/pinctrl-intel.h |  1 +
+ 2 files changed, 27 insertions(+), 10 deletions(-)
+Merging pinctrl-renesas/renesas-pinctrl (fb22bb9701d48 pinctrl: renesas: rza1: Normalize return value of gpio_get())
+$ git merge -m Merge branch 'renesas-pinctrl' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Already up to date.
+Merging pinctrl-samsung/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Already up to date.
+Merging pwm/pwm/for-next (07bee454fbf45 MAINTAINERS: Add #linux-pwm irc channel to pwm entry)
+$ git merge -m Merge branch 'pwm/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git pwm/pwm/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml | 3 +++
+ MAINTAINERS                                            | 1 +
+ drivers/pwm/pwm-imx-tpm.c                              | 9 ++++++++-
+ 3 files changed, 12 insertions(+), 1 deletion(-)
+Merging ktest/for-next (229517449879b ktest: Add a --dry-run mode)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Merge made by the 'ort' strategy.
+ tools/testing/ktest/ktest.pl    | 158 +++++++++++++++++++++++++++++-----------
+ tools/testing/ktest/sample.conf |   6 ++
+ 2 files changed, 122 insertions(+), 42 deletions(-)
+Merging kselftest/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Already up to date.
+Merging kunit/test (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'test' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (fa2c3ed5f079d kunit: tool: Terminate kernel under test on SIGINT)
+$ git merge -m Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Auto-merging tools/testing/kunit/kunit_kernel.py
+Auto-merging tools/testing/kunit/kunit_tool_test.py
+CONFLICT (content): Merge conflict in tools/testing/kunit/kunit_tool_test.py
+Resolved 'tools/testing/kunit/kunit_tool_test.py' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 82f79163b6310] Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git
+$ git diff -M --stat --summary HEAD^..
+ tools/testing/kunit/kunit.py           | 16 ++++++++--
+ tools/testing/kunit/kunit_kernel.py    | 38 +++++++++++++++-------
+ tools/testing/kunit/kunit_parser.py    |  3 +-
+ tools/testing/kunit/kunit_tool_test.py | 58 ++++++++++++++++++++++++++++++++--
+ 4 files changed, 98 insertions(+), 17 deletions(-)
+Merging livepatching/for-next (920e5001f4beb selftests: livepatch: test-ftrace: livepatch a traced function)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git livepatching/for-next
+Merge made by the 'ort' strategy.
+ tools/testing/selftests/livepatch/test-ftrace.sh | 36 ++++++++++++++++++++++++
+ 1 file changed, 36 insertions(+)
+Merging rtc/rtc-next (5ff89ef425d17 dt-bindings: rtc: isl12026: convert to YAML schema)
+$ git merge -m Merge branch 'rtc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/rtc/isil,isl12026.txt      | 28 ----------
+ .../devicetree/bindings/rtc/isil,isl12026.yaml     | 59 ++++++++++++++++++++++
+ .../bindings/rtc/microchip,mpfs-rtc.yaml           |  3 ++
+ .../bindings/rtc/microcrystal,rv3028.yaml          |  2 +
+ drivers/rtc/rtc-abx80x.c                           |  3 +-
+ drivers/rtc/rtc-m41t80.c                           |  8 +--
+ drivers/rtc/rtc-max77686.c                         | 14 ++++-
+ drivers/rtc/rtc-pcf2127.c                          | 23 +++------
+ drivers/rtc/rtc-rs5c372.c                          |  7 +--
+ drivers/rtc/rtc-rv8803.c                           |  8 +--
+ drivers/rtc/rtc-rx8025.c                           |  4 +-
+ 11 files changed, 88 insertions(+), 71 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/rtc/isil,isl12026.txt
+ create mode 100644 Documentation/devicetree/bindings/rtc/isil,isl12026.yaml
+Merging nvdimm/libnvdimm-for-next (a9ba6733c7f10 nvdimm: virtio_pmem: serialize flush requests)
+$ git merge -m Merge branch 'libnvdimm-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Already up to date.
+Merging at24/at24/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'at24/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Already up to date.
+Merging ntb/ntb-next (7b3302c687cae ntb_hw_amd: Fix incorrect debug message in link disable path)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Merge made by the 'ort' strategy.
+ drivers/ntb/hw/amd/ntb_hw_amd.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging seccomp/for-next/seccomp (7d0a66e4bb908 Linux 6.18)
+$ git merge -m Merge branch 'for-next/seccomp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Already up to date.
+Merging slimbus/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Already up to date.
+Merging nvmem/for-next (8d48419d141cb Merge branches 'nvmem-fixes' and 'nvmem-for-7.1' into nvmem-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/nvmem/layouts/kontron,sl28-vpd.yaml   |   7 +-
+ .../devicetree/bindings/nvmem/qcom,qfprom.yaml     |   1 +
+ .../devicetree/bindings/nvmem/rockchip,otp.yaml    |  83 +++++++++--
+ drivers/nvmem/imx-ocotp-ele.c                      |   1 +
+ drivers/nvmem/imx-ocotp.c                          |   1 +
+ drivers/nvmem/qnap-mcu-eeprom.c                    |   8 +-
+ drivers/nvmem/rockchip-otp.c                       | 159 +++++++++++++++++----
+ drivers/nvmem/zynqmp_nvmem.c                       |   8 +-
+ 8 files changed, 213 insertions(+), 55 deletions(-)
+Merging hyperv/hyperv-next (e733a9e281805 mshv: Introduce tracing support)
+$ git merge -m Merge branch 'hyperv-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Auto-merging drivers/hv/hyperv_vmbus.h
+Auto-merging drivers/hv/mshv_root.h
+Auto-merging drivers/hv/mshv_root_main.c
+Auto-merging drivers/hv/vmbus_drv.c
+Merge made by the 'ort' strategy.
+ drivers/hv/Makefile            |   1 +
+ drivers/hv/channel_mgmt.c      |  16 +-
+ drivers/hv/hyperv_vmbus.h      |   3 +-
+ drivers/hv/mshv_eventfd.c      |  14 ++
+ drivers/hv/mshv_irq.c          |   4 +
+ drivers/hv/mshv_root.h         |   1 +
+ drivers/hv/mshv_root_hv_call.c |  22 +-
+ drivers/hv/mshv_root_main.c    |  78 ++++++-
+ drivers/hv/mshv_trace.c        |   9 +
+ drivers/hv/mshv_trace.h        | 515 +++++++++++++++++++++++++++++++++++++++++
+ drivers/hv/vmbus_drv.c         |   7 +-
+ 11 files changed, 644 insertions(+), 26 deletions(-)
+ create mode 100644 drivers/hv/mshv_trace.c
+ create mode 100644 drivers/hv/mshv_trace.h
+Merging auxdisplay/for-next (995a418a6ca33 auxdisplay: lcd2s: add error handling for i2c transfers)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay/for-next
+Already up to date.
+Merging kgdb/kgdb/for-next (fdbdd0ccb30af kdb: remove redundant check for scancode 0xe0)
+$ git merge -m Merge branch 'kgdb/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'hmm' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'cfi/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (f2d1643ddc0f3 bus: mhi: ep: Test for non-zero return value where applicable)
+$ git merge -m Merge branch 'mhi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Merge made by the 'ort' strategy.
+ drivers/bus/mhi/ep/main.c          | 10 +++++-----
+ drivers/bus/mhi/ep/ring.c          |  6 +++---
+ drivers/bus/mhi/host/boot.c        | 22 +++-------------------
+ drivers/bus/mhi/host/pci_generic.c | 27 ++++++++++++++++++++++++++-
+ include/linux/mhi.h                | 34 +++++++++++++++++-----------------
+ 5 files changed, 54 insertions(+), 45 deletions(-)
+$ git am -3 ../patches/0001-fix-up-for-net-qrtr-Drop-the-MHI-auto_queue-feature-.patch
+Applying: fix up for "net: qrtr: Drop the MHI auto_queue feature for IPCR DL channels"
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging memblock/for-next (f56ccc32468ee mm/memtest: add underflow detection for size calculation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock/for-next
+Already up to date.
+Merging cxl/next (d5f9bfc37906b Merge branch 'for-7.1/cxl-type2-support' into cxl-for-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Auto-merging drivers/cxl/core/mbox.c
+Auto-merging drivers/cxl/core/region.c
+Merge made by the 'ort' strategy.
+ drivers/acpi/numa/srat.c     |   7 +-
+ drivers/cxl/core/core.h      |   2 +
+ drivers/cxl/core/mbox.c      |  17 ++--
+ drivers/cxl/core/memdev.c    |  24 +++++
+ drivers/cxl/core/pci.c       |  57 +++++++++++
+ drivers/cxl/core/region.c    |  95 +++++++++++-------
+ drivers/cxl/core/regs.c      |   1 -
+ drivers/cxl/cxl.h            |  99 +------------------
+ drivers/cxl/cxlmem.h         |  88 +----------------
+ drivers/cxl/cxlpci.h         |  13 +++
+ drivers/cxl/pci.c            |  87 ++---------------
+ include/cxl/cxl.h            | 226 +++++++++++++++++++++++++++++++++++++++++++
+ tools/testing/cxl/test/mem.c |   3 +-
+ 13 files changed, 411 insertions(+), 308 deletions(-)
+ create mode 100644 include/cxl/cxl.h
+Merging zstd/zstd-next (65d1f5507ed2c zstd: Import upstream v1.5.7)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Already up to date.
+Merging efi/next (56e2ef8f5f916 efi: libstub: fix type of fdt 32 and 64bit variables)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Auto-merging drivers/firmware/efi/efi.c
+Merge made by the 'ort' strategy.
+ drivers/firmware/efi/efi-bgrt.c    |  7 +++++--
+ drivers/firmware/efi/efi.c         | 27 +++++++++------------------
+ drivers/firmware/efi/libstub/fdt.c | 12 ++++++------
+ include/linux/efi.h                | 21 ---------------------
+ 4 files changed, 20 insertions(+), 47 deletions(-)
+Merging unicode/for-next (bcfee135d5847 utf8: Remove unused utf8_normalize)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Auto-merging fs/unicode/utf8-core.c
+Merge made by the 'ort' strategy.
+ fs/unicode/utf8-core.c  | 22 ----------------------
+ include/linux/unicode.h |  3 ---
+ 2 files changed, 25 deletions(-)
+Merging slab/slab/for-next (ac9004d2224a4 Merge branch 'slab/for-7.1/misc' into slab/for-next)
+$ git merge -m Merge branch 'slab/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab/slab/for-next
+Auto-merging mm/slab.h
+Merge made by the 'ort' strategy.
+ mm/slab.h |   7 +-
+ mm/slub.c | 339 +++++++++++++++++++++++++++++++++++++++-----------------------
+ 2 files changed, 220 insertions(+), 126 deletions(-)
+Merging random/master (bb9ff576fdff4 virt: vmgenid: remap memory as decrypted)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Merge made by the 'ort' strategy.
+ drivers/virt/vmgenid.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging landlock/next (0955b24481490 selftests/landlock: Drain stale audit records on init)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Merge made by the 'ort' strategy.
+ Documentation/userspace-api/landlock.rst      |  23 ++-
+ samples/landlock/sandboxer.c                  |   5 +-
+ security/landlock/cred.h                      |   2 +-
+ security/landlock/domain.c                    |   9 +-
+ security/landlock/fs.c                        |  28 ++--
+ security/landlock/id.c                        |   2 +-
+ security/landlock/ruleset.c                   |  23 +--
+ security/landlock/ruleset.h                   |   2 +-
+ security/landlock/syscalls.c                  |  17 ++-
+ security/landlock/task.c                      |  13 +-
+ security/landlock/tsync.c                     | 204 +++++++++++++++++---------
+ tools/testing/selftests/landlock/audit.h      |  29 +++-
+ tools/testing/selftests/landlock/audit_test.c |   2 -
+ tools/testing/selftests/landlock/tsync_test.c |  93 +++++++++++-
+ 14 files changed, 317 insertions(+), 135 deletions(-)
+Merging sysctl/sysctl-next (6932256d3a376 time/jiffies: Fix sysctl file error on configurations where USER_HZ < HZ)
+$ git merge -m Merge branch 'sysctl-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git sysctl/sysctl-next
+Already up to date.
+Merging execve/for-next/execve (46a03ea50b5f3 fs/tests: exec: drop duplicate bprm_stack_limits test vectors)
+$ git merge -m Merge branch 'for-next/execve' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (32a55dedc06ae bitops: fix kernel-doc parameter name for parity8())
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Merge made by the 'ort' strategy.
+ arch/x86/kernel/cpu/topology.c |  8 ++-----
+ include/linux/bitmap.h         | 33 ++++++++++++++++++++++++++
+ include/linux/bitops.h         |  2 +-
+ include/linux/count_zeros.h    | 13 ++++-------
+ include/linux/find.h           |  2 --
+ lib/crypto/mpi/longlong.h      |  8 +++----
+ lib/find_bit.c                 |  7 ------
+ lib/find_bit_benchmark.c       | 15 +++++++-----
+ lib/test_bitmap.c              | 53 ++++++++++++++++++++++++++++++++----------
+ 9 files changed, 94 insertions(+), 47 deletions(-)
+Merging hte/for-next (92dfd92f74769 hte: tegra194: remove Kconfig dependency on Tegra194 SoC)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Merge made by the 'ort' strategy.
+ drivers/hte/Kconfig | 6 +++---
+ drivers/hte/hte.c   | 2 +-
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+Merging kspp/for-next/kspp (54031989e49fd Merge branches 'for-linus/execve' and 'for-next/execve' into for-next/kspp)
+$ git merge -m Merge branch 'for-next/kspp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Auto-merging fs/binfmt_elf.c
+Auto-merging fs/binfmt_elf_fdpic.c
+Auto-merging include/linux/mm_types.h
+Auto-merging kernel/fork.c
+Merge made by the 'ort' strategy.
+ fs/binfmt_elf.c                              |   8 +-
+ fs/binfmt_elf_fdpic.c                        |  14 +++-
+ fs/exec.c                                    |  65 +++++++++++++++++
+ fs/proc/base.c                               |  12 ++-
+ fs/tests/exec_kunit.c                        |   3 -
+ include/linux/auxvec.h                       |   2 +-
+ include/linux/binfmts.h                      |  11 +++
+ include/linux/mm_types.h                     |   3 +-
+ kernel/fork.c                                |   8 ++
+ kernel/sys.c                                 |  30 ++++----
+ tools/testing/selftests/exec/.gitignore      |   1 +
+ tools/testing/selftests/exec/Makefile        |   1 +
+ tools/testing/selftests/exec/hwcap_inherit.c | 105 +++++++++++++++++++++++++++
+ 13 files changed, 233 insertions(+), 30 deletions(-)
+ create mode 100644 tools/testing/selftests/exec/hwcap_inherit.c
+Merging nolibc/for-next (d59131de97ee6 selftests/nolibc: validate NOLIBC_IGNORE_ERRNO compilation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git nolibc/for-next
+Merge made by the 'ort' strategy.
+ tools/include/nolibc/arch-arm.h                    |  14 +-
+ tools/include/nolibc/arch-arm64.h                  |  14 +-
+ tools/include/nolibc/arch-loongarch.h              |  14 +-
+ tools/include/nolibc/arch-m68k.h                   |  14 +-
+ tools/include/nolibc/arch-mips.h                   |  33 +-
+ tools/include/nolibc/arch-powerpc.h                |  14 +-
+ tools/include/nolibc/arch-riscv.h                  |  14 +-
+ tools/include/nolibc/arch-s390.h                   |  20 +-
+ tools/include/nolibc/arch-sh.h                     |  14 +-
+ tools/include/nolibc/arch-sparc.h                  |  18 +-
+ tools/include/nolibc/arch-x86.h                    |  68 +--
+ tools/include/nolibc/compiler.h                    |   3 +
+ tools/include/nolibc/fcntl.h                       |   4 +-
+ tools/include/nolibc/nolibc.h                      |   6 +-
+ tools/include/nolibc/poll.h                        |   4 +-
+ tools/include/nolibc/sched.h                       |   4 +-
+ tools/include/nolibc/stackprotector.h              |   8 +-
+ tools/include/nolibc/stdio.h                       | 535 +++++++++++++++------
+ tools/include/nolibc/stdlib.h                      | 173 ++++---
+ tools/include/nolibc/sys.h                         | 110 ++---
+ tools/include/nolibc/sys/ioctl.h                   |   2 +-
+ tools/include/nolibc/sys/mman.h                    |   8 +-
+ tools/include/nolibc/sys/mount.h                   |   2 +-
+ tools/include/nolibc/sys/prctl.h                   |   2 +-
+ tools/include/nolibc/sys/ptrace.h                  |   2 +-
+ tools/include/nolibc/sys/random.h                  |   2 +-
+ tools/include/nolibc/sys/reboot.h                  |   2 +-
+ tools/include/nolibc/sys/resource.h                |   2 +-
+ tools/include/nolibc/sys/select.h                  |   6 +-
+ tools/include/nolibc/sys/stat.h                    |   2 +-
+ tools/include/nolibc/sys/syscall.h                 |   2 +-
+ tools/include/nolibc/sys/timerfd.h                 |  10 +-
+ tools/include/nolibc/sys/uio.h                     |   4 +-
+ tools/include/nolibc/sys/utsname.h                 |   2 +-
+ tools/include/nolibc/sys/wait.h                    |   2 +-
+ tools/include/nolibc/time.h                        |  28 +-
+ tools/include/nolibc/unistd.h                      |   2 +-
+ tools/testing/selftests/nolibc/Makefile            |   4 +-
+ tools/testing/selftests/nolibc/Makefile.include    |   2 +
+ tools/testing/selftests/nolibc/Makefile.nolibc     |   8 +-
+ .../selftests/nolibc/nolibc-test-ignore-errno.c    |   6 +
+ tools/testing/selftests/nolibc/nolibc-test.c       | 173 ++++---
+ 42 files changed, 834 insertions(+), 523 deletions(-)
+ create mode 100644 tools/testing/selftests/nolibc/nolibc-test-ignore-errno.c
+Merging iommufd/for-next (5e3387f809556 iommufd/selftest: Fix page leaks in mock_viommu_{init,destroy})
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Auto-merging drivers/iommu/iommufd/pages.c
+Auto-merging drivers/iommu/iommufd/selftest.c
+Merge made by the 'ort' strategy.
+ drivers/iommu/iommufd/pages.c       | 2 +-
+ drivers/iommu/iommufd/selftest.c    | 4 ++--
+ drivers/iommu/iommufd/vfio_compat.c | 2 +-
+ 3 files changed, 4 insertions(+), 4 deletions(-)
+Merging turbostat/next (a23c1298f1593 tools/power turbostat: Cleanup print helper functions)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat/next
+Merge made by the 'ort' strategy.
+ tools/power/x86/turbostat/turbostat.c | 91 ++++++++++++++++++++---------------
+ 1 file changed, 53 insertions(+), 38 deletions(-)
+Merging pwrseq/pwrseq/for-next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'pwrseq/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq/pwrseq/for-next
+Already up to date.
+Merging capabilities-next/caps-next (0715881360074 ipc: don't audit capability check in ipc_permissions())
+$ git merge -m Merge branch 'caps-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git capabilities-next/caps-next
+Already up to date.
+$ git am -3 ../patches/0001-sign-file-Fix-up-merge-issue.patch
+Applying: sign-file: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	scripts/sign-file.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging ipe/next (f37070985ed91 docs: security: ipe: fix typos and grammar)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git ipe/next
+Merge made by the 'ort' strategy.
+ Documentation/security/ipe.rst | 10 +++++-----
+ 1 file changed, 5 insertions(+), 5 deletions(-)
+Merging kcsan/next (dcb6fa37fd7bc Linux 6.18-rc3)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git kcsan/next
+Already up to date.
+Merging crc/crc-next (6e4d63e8993c6 lib/crc: arm64: Drop check for CONFIG_KERNEL_MODE_NEON)
+$ git merge -m Merge branch 'crc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc/crc-next
+Auto-merging tools/testing/kunit/configs/all_tests.config
+Merge made by the 'ort' strategy.
+ crypto/crc32c.c                              | 19 +------------------
+ lib/crc/.kunitconfig                         |  3 +++
+ lib/crc/Kconfig                              | 23 ++++++++++++++++-------
+ lib/crc/tests/crc_kunit.c                    | 28 ++++++++++++++++++++++------
+ tools/testing/kunit/configs/all_tests.config |  2 ++
+ 5 files changed, 44 insertions(+), 31 deletions(-)
+ create mode 100644 lib/crc/.kunitconfig
+Merging keys-next/keys-next (965e9a2cf23b0 pkcs7: Change a pr_warn() to pr_warn_once())
+$ git merge -m Merge branch 'keys-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git keys-next/keys-next
+Already up to date.
+Merging fwctl/for-next (11439c4635edd Linux 7.0-rc2)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl/for-next
+Already up to date.
+Merging devsec-tsm/next (43185067c6fd5 configfs-tsm-report: tdx_guest: Increase Quote buffer size to 128KB)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm/next
+Already up to date.
+Merging hisilicon/for-next (6c177775dcc5e Merge branch 'next/drivers' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/hisilicon/linux-hisi.git hisilicon/for-next
+Merge made by the 'ort' strategy.
+Merging kthread/for-next (fa39ec4f89f26 doc: Add housekeeping documentation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git kthread/for-next
+Already up to date.
diff --git a/arch/Kconfig b/arch/Kconfig
index 102ddbd4298ef..a6d1c8cc1d64b 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1841,4 +1841,7 @@ config ARCH_WANTS_PRE_LINK_VMLINUX
 config ARCH_HAS_CPU_ATTACK_VECTORS
 	bool
 
+config HAVE_ARCH_GET_SECUREBOOT
+	def_bool EFI
+
 endmenu
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 6c7dbf0adad62..8b9d7005bcd56 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -40,6 +40,7 @@ config ALPHA
 	select MMU_GATHER_NO_RANGE
 	select MMU_GATHER_RCU_TABLE_FREE
 	select SPARSEMEM_EXTREME if SPARSEMEM
+	select XOR_BLOCKS_ARCH
 	select ZONE_DMA
 	help
 	  The Alpha is a 64-bit general-purpose processor designed and
diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h
index e9368c54be452..268ddde336171 100644
--- a/arch/alpha/include/asm/pgtable.h
+++ b/arch/alpha/include/asm/pgtable.h
@@ -126,12 +126,6 @@ struct vm_area_struct;
  */
 #define pgprot_noncached(prot)	(prot)
 
-/*
- * ZERO_PAGE is a global shared page that is always zero:  used
- * for zero-mapped memory areas etc..
- */
-#define ZERO_PAGE(vaddr)	(virt_to_page(ZERO_PGE))
-
 /*
  * On certain platforms whose physical address space can overlap KSEG,
  * namely EV6 and above, we must re-twiddle the physaddr to restore the
diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h
deleted file mode 100644
index e0de0c233ab92..0000000000000
--- a/arch/alpha/include/asm/xor.h
+++ /dev/null
@@ -1,866 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm-alpha/xor.h
- *
- * Optimized RAID-5 checksumming functions for alpha EV5 and EV6
- */
-
-extern void
-xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2);
-extern void
-xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3);
-extern void
-xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3,
-	    const unsigned long * __restrict p4);
-extern void
-xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3,
-	    const unsigned long * __restrict p4,
-	    const unsigned long * __restrict p5);
-
-extern void
-xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2);
-extern void
-xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3);
-extern void
-xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4);
-extern void
-xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4,
-		     const unsigned long * __restrict p5);
-
-asm("								\n\
-	.text							\n\
-	.align 3						\n\
-	.ent xor_alpha_2					\n\
-xor_alpha_2:							\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-	.align 4						\n\
-2:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,8($17)						\n\
-	ldq $3,8($18)						\n\
-								\n\
-	ldq $4,16($17)						\n\
-	ldq $5,16($18)						\n\
-	ldq $6,24($17)						\n\
-	ldq $7,24($18)						\n\
-								\n\
-	ldq $19,32($17)						\n\
-	ldq $20,32($18)						\n\
-	ldq $21,40($17)						\n\
-	ldq $22,40($18)						\n\
-								\n\
-	ldq $23,48($17)						\n\
-	ldq $24,48($18)						\n\
-	ldq $25,56($17)						\n\
-	xor $0,$1,$0		# 7 cycles from $1 load		\n\
-								\n\
-	ldq $27,56($18)						\n\
-	xor $2,$3,$2						\n\
-	stq $0,0($17)						\n\
-	xor $4,$5,$4						\n\
-								\n\
-	stq $2,8($17)						\n\
-	xor $6,$7,$6						\n\
-	stq $4,16($17)						\n\
-	xor $19,$20,$19						\n\
-								\n\
-	stq $6,24($17)						\n\
-	xor $21,$22,$21						\n\
-	stq $19,32($17)						\n\
-	xor $23,$24,$23						\n\
-								\n\
-	stq $21,40($17)						\n\
-	xor $25,$27,$25						\n\
-	stq $23,48($17)						\n\
-	subq $16,1,$16						\n\
-								\n\
-	stq $25,56($17)						\n\
-	addq $17,64,$17						\n\
-	addq $18,64,$18						\n\
-	bgt $16,2b						\n\
-								\n\
-	ret							\n\
-	.end xor_alpha_2					\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_3					\n\
-xor_alpha_3:							\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-	.align 4						\n\
-3:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,8($17)						\n\
-								\n\
-	ldq $4,8($18)						\n\
-	ldq $6,16($17)						\n\
-	ldq $7,16($18)						\n\
-	ldq $21,24($17)						\n\
-								\n\
-	ldq $22,24($18)						\n\
-	ldq $24,32($17)						\n\
-	ldq $25,32($18)						\n\
-	ldq $5,8($19)						\n\
-								\n\
-	ldq $20,16($19)						\n\
-	ldq $23,24($19)						\n\
-	ldq $27,32($19)						\n\
-	nop							\n\
-								\n\
-	xor $0,$1,$1		# 8 cycles from $0 load		\n\
-	xor $3,$4,$4		# 6 cycles from $4 load		\n\
-	xor $6,$7,$7		# 6 cycles from $7 load		\n\
-	xor $21,$22,$22		# 5 cycles from $22 load	\n\
-								\n\
-	xor $1,$2,$2		# 9 cycles from $2 load		\n\
-	xor $24,$25,$25		# 5 cycles from $25 load	\n\
-	stq $2,0($17)						\n\
-	xor $4,$5,$5		# 6 cycles from $5 load		\n\
-								\n\
-	stq $5,8($17)						\n\
-	xor $7,$20,$20		# 7 cycles from $20 load	\n\
-	stq $20,16($17)						\n\
-	xor $22,$23,$23		# 7 cycles from $23 load	\n\
-								\n\
-	stq $23,24($17)						\n\
-	xor $25,$27,$27		# 7 cycles from $27 load	\n\
-	stq $27,32($17)						\n\
-	nop							\n\
-								\n\
-	ldq $0,40($17)						\n\
-	ldq $1,40($18)						\n\
-	ldq $3,48($17)						\n\
-	ldq $4,48($18)						\n\
-								\n\
-	ldq $6,56($17)						\n\
-	ldq $7,56($18)						\n\
-	ldq $2,40($19)						\n\
-	ldq $5,48($19)						\n\
-								\n\
-	ldq $20,56($19)						\n\
-	xor $0,$1,$1		# 4 cycles from $1 load		\n\
-	xor $3,$4,$4		# 5 cycles from $4 load		\n\
-	xor $6,$7,$7		# 5 cycles from $7 load		\n\
-								\n\
-	xor $1,$2,$2		# 4 cycles from $2 load		\n\
-	xor $4,$5,$5		# 5 cycles from $5 load		\n\
-	stq $2,40($17)						\n\
-	xor $7,$20,$20		# 4 cycles from $20 load	\n\
-								\n\
-	stq $5,48($17)						\n\
-	subq $16,1,$16						\n\
-	stq $20,56($17)						\n\
-	addq $19,64,$19						\n\
-								\n\
-	addq $18,64,$18						\n\
-	addq $17,64,$17						\n\
-	bgt $16,3b						\n\
-	ret							\n\
-	.end xor_alpha_3					\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_4					\n\
-xor_alpha_4:							\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-	.align 4						\n\
-4:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,0($20)						\n\
-								\n\
-	ldq $4,8($17)						\n\
-	ldq $5,8($18)						\n\
-	ldq $6,8($19)						\n\
-	ldq $7,8($20)						\n\
-								\n\
-	ldq $21,16($17)						\n\
-	ldq $22,16($18)						\n\
-	ldq $23,16($19)						\n\
-	ldq $24,16($20)						\n\
-								\n\
-	ldq $25,24($17)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-	ldq $27,24($18)						\n\
-	xor $2,$3,$3		# 6 cycles from $3 load		\n\
-								\n\
-	ldq $0,24($19)						\n\
-	xor $1,$3,$3						\n\
-	ldq $1,24($20)						\n\
-	xor $4,$5,$5		# 7 cycles from $5 load		\n\
-								\n\
-	stq $3,0($17)						\n\
-	xor $6,$7,$7						\n\
-	xor $21,$22,$22		# 7 cycles from $22 load	\n\
-	xor $5,$7,$7						\n\
-								\n\
-	stq $7,8($17)						\n\
-	xor $23,$24,$24		# 7 cycles from $24 load	\n\
-	ldq $2,32($17)						\n\
-	xor $22,$24,$24						\n\
-								\n\
-	ldq $3,32($18)						\n\
-	ldq $4,32($19)						\n\
-	ldq $5,32($20)						\n\
-	xor $25,$27,$27		# 8 cycles from $27 load	\n\
-								\n\
-	ldq $6,40($17)						\n\
-	ldq $7,40($18)						\n\
-	ldq $21,40($19)						\n\
-	ldq $22,40($20)						\n\
-								\n\
-	stq $24,16($17)						\n\
-	xor $0,$1,$1		# 9 cycles from $1 load		\n\
-	xor $2,$3,$3		# 5 cycles from $3 load		\n\
-	xor $27,$1,$1						\n\
-								\n\
-	stq $1,24($17)						\n\
-	xor $4,$5,$5		# 5 cycles from $5 load		\n\
-	ldq $23,48($17)						\n\
-	ldq $24,48($18)						\n\
-								\n\
-	ldq $25,48($19)						\n\
-	xor $3,$5,$5						\n\
-	ldq $27,48($20)						\n\
-	ldq $0,56($17)						\n\
-								\n\
-	ldq $1,56($18)						\n\
-	ldq $2,56($19)						\n\
-	xor $6,$7,$7		# 8 cycles from $6 load		\n\
-	ldq $3,56($20)						\n\
-								\n\
-	stq $5,32($17)						\n\
-	xor $21,$22,$22		# 8 cycles from $22 load	\n\
-	xor $7,$22,$22						\n\
-	xor $23,$24,$24		# 5 cycles from $24 load	\n\
-								\n\
-	stq $22,40($17)						\n\
-	xor $25,$27,$27		# 5 cycles from $27 load	\n\
-	xor $24,$27,$27						\n\
-	xor $0,$1,$1		# 5 cycles from $1 load		\n\
-								\n\
-	stq $27,48($17)						\n\
-	xor $2,$3,$3		# 4 cycles from $3 load		\n\
-	xor $1,$3,$3						\n\
-	subq $16,1,$16						\n\
-								\n\
-	stq $3,56($17)						\n\
-	addq $20,64,$20						\n\
-	addq $19,64,$19						\n\
-	addq $18,64,$18						\n\
-								\n\
-	addq $17,64,$17						\n\
-	bgt $16,4b						\n\
-	ret							\n\
-	.end xor_alpha_4					\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_5					\n\
-xor_alpha_5:							\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-	.align 4						\n\
-5:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,0($20)						\n\
-								\n\
-	ldq $4,0($21)						\n\
-	ldq $5,8($17)						\n\
-	ldq $6,8($18)						\n\
-	ldq $7,8($19)						\n\
-								\n\
-	ldq $22,8($20)						\n\
-	ldq $23,8($21)						\n\
-	ldq $24,16($17)						\n\
-	ldq $25,16($18)						\n\
-								\n\
-	ldq $27,16($19)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-	ldq $28,16($20)						\n\
-	xor $2,$3,$3		# 6 cycles from $3 load		\n\
-								\n\
-	ldq $0,16($21)						\n\
-	xor $1,$3,$3						\n\
-	ldq $1,24($17)						\n\
-	xor $3,$4,$4		# 7 cycles from $4 load		\n\
-								\n\
-	stq $4,0($17)						\n\
-	xor $5,$6,$6		# 7 cycles from $6 load		\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	xor $6,$23,$23		# 7 cycles from $23 load	\n\
-								\n\
-	ldq $2,24($18)						\n\
-	xor $22,$23,$23						\n\
-	ldq $3,24($19)						\n\
-	xor $24,$25,$25		# 8 cycles from $25 load	\n\
-								\n\
-	stq $23,8($17)						\n\
-	xor $25,$27,$27		# 8 cycles from $27 load	\n\
-	ldq $4,24($20)						\n\
-	xor $28,$0,$0		# 7 cycles from $0 load		\n\
-								\n\
-	ldq $5,24($21)						\n\
-	xor $27,$0,$0						\n\
-	ldq $6,32($17)						\n\
-	ldq $7,32($18)						\n\
-								\n\
-	stq $0,16($17)						\n\
-	xor $1,$2,$2		# 6 cycles from $2 load		\n\
-	ldq $22,32($19)						\n\
-	xor $3,$4,$4		# 4 cycles from $4 load		\n\
-								\n\
-	ldq $23,32($20)						\n\
-	xor $2,$4,$4						\n\
-	ldq $24,32($21)						\n\
-	ldq $25,40($17)						\n\
-								\n\
-	ldq $27,40($18)						\n\
-	ldq $28,40($19)						\n\
-	ldq $0,40($20)						\n\
-	xor $4,$5,$5		# 7 cycles from $5 load		\n\
-								\n\
-	stq $5,24($17)						\n\
-	xor $6,$7,$7		# 7 cycles from $7 load		\n\
-	ldq $1,40($21)						\n\
-	ldq $2,48($17)						\n\
-								\n\
-	ldq $3,48($18)						\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	ldq $4,48($19)						\n\
-	xor $23,$24,$24		# 6 cycles from $24 load	\n\
-								\n\
-	ldq $5,48($20)						\n\
-	xor $22,$24,$24						\n\
-	ldq $6,48($21)						\n\
-	xor $25,$27,$27		# 7 cycles from $27 load	\n\
-								\n\
-	stq $24,32($17)						\n\
-	xor $27,$28,$28		# 8 cycles from $28 load	\n\
-	ldq $7,56($17)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-								\n\
-	ldq $22,56($18)						\n\
-	ldq $23,56($19)						\n\
-	ldq $24,56($20)						\n\
-	ldq $25,56($21)						\n\
-								\n\
-	xor $28,$1,$1						\n\
-	xor $2,$3,$3		# 9 cycles from $3 load		\n\
-	xor $3,$4,$4		# 9 cycles from $4 load		\n\
-	xor $5,$6,$6		# 8 cycles from $6 load		\n\
-								\n\
-	stq $1,40($17)						\n\
-	xor $4,$6,$6						\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	xor $23,$24,$24		# 6 cycles from $24 load	\n\
-								\n\
-	stq $6,48($17)						\n\
-	xor $22,$24,$24						\n\
-	subq $16,1,$16						\n\
-	xor $24,$25,$25		# 8 cycles from $25 load	\n\
-								\n\
-	stq $25,56($17)						\n\
-	addq $21,64,$21						\n\
-	addq $20,64,$20						\n\
-	addq $19,64,$19						\n\
-								\n\
-	addq $18,64,$18						\n\
-	addq $17,64,$17						\n\
-	bgt $16,5b						\n\
-	ret							\n\
-	.end xor_alpha_5					\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_prefetch_2				\n\
-xor_alpha_prefetch_2:						\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-								\n\
-	ldq $31, 0($17)						\n\
-	ldq $31, 0($18)						\n\
-								\n\
-	ldq $31, 64($17)					\n\
-	ldq $31, 64($18)					\n\
-								\n\
-	ldq $31, 128($17)					\n\
-	ldq $31, 128($18)					\n\
-								\n\
-	ldq $31, 192($17)					\n\
-	ldq $31, 192($18)					\n\
-	.align 4						\n\
-2:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,8($17)						\n\
-	ldq $3,8($18)						\n\
-								\n\
-	ldq $4,16($17)						\n\
-	ldq $5,16($18)						\n\
-	ldq $6,24($17)						\n\
-	ldq $7,24($18)						\n\
-								\n\
-	ldq $19,32($17)						\n\
-	ldq $20,32($18)						\n\
-	ldq $21,40($17)						\n\
-	ldq $22,40($18)						\n\
-								\n\
-	ldq $23,48($17)						\n\
-	ldq $24,48($18)						\n\
-	ldq $25,56($17)						\n\
-	ldq $27,56($18)						\n\
-								\n\
-	ldq $31,256($17)					\n\
-	xor $0,$1,$0		# 8 cycles from $1 load		\n\
-	ldq $31,256($18)					\n\
-	xor $2,$3,$2						\n\
-								\n\
-	stq $0,0($17)						\n\
-	xor $4,$5,$4						\n\
-	stq $2,8($17)						\n\
-	xor $6,$7,$6						\n\
-								\n\
-	stq $4,16($17)						\n\
-	xor $19,$20,$19						\n\
-	stq $6,24($17)						\n\
-	xor $21,$22,$21						\n\
-								\n\
-	stq $19,32($17)						\n\
-	xor $23,$24,$23						\n\
-	stq $21,40($17)						\n\
-	xor $25,$27,$25						\n\
-								\n\
-	stq $23,48($17)						\n\
-	subq $16,1,$16						\n\
-	stq $25,56($17)						\n\
-	addq $17,64,$17						\n\
-								\n\
-	addq $18,64,$18						\n\
-	bgt $16,2b						\n\
-	ret							\n\
-	.end xor_alpha_prefetch_2				\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_prefetch_3				\n\
-xor_alpha_prefetch_3:						\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-								\n\
-	ldq $31, 0($17)						\n\
-	ldq $31, 0($18)						\n\
-	ldq $31, 0($19)						\n\
-								\n\
-	ldq $31, 64($17)					\n\
-	ldq $31, 64($18)					\n\
-	ldq $31, 64($19)					\n\
-								\n\
-	ldq $31, 128($17)					\n\
-	ldq $31, 128($18)					\n\
-	ldq $31, 128($19)					\n\
-								\n\
-	ldq $31, 192($17)					\n\
-	ldq $31, 192($18)					\n\
-	ldq $31, 192($19)					\n\
-	.align 4						\n\
-3:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,8($17)						\n\
-								\n\
-	ldq $4,8($18)						\n\
-	ldq $6,16($17)						\n\
-	ldq $7,16($18)						\n\
-	ldq $21,24($17)						\n\
-								\n\
-	ldq $22,24($18)						\n\
-	ldq $24,32($17)						\n\
-	ldq $25,32($18)						\n\
-	ldq $5,8($19)						\n\
-								\n\
-	ldq $20,16($19)						\n\
-	ldq $23,24($19)						\n\
-	ldq $27,32($19)						\n\
-	nop							\n\
-								\n\
-	xor $0,$1,$1		# 8 cycles from $0 load		\n\
-	xor $3,$4,$4		# 7 cycles from $4 load		\n\
-	xor $6,$7,$7		# 6 cycles from $7 load		\n\
-	xor $21,$22,$22		# 5 cycles from $22 load	\n\
-								\n\
-	xor $1,$2,$2		# 9 cycles from $2 load		\n\
-	xor $24,$25,$25		# 5 cycles from $25 load	\n\
-	stq $2,0($17)						\n\
-	xor $4,$5,$5		# 6 cycles from $5 load		\n\
-								\n\
-	stq $5,8($17)						\n\
-	xor $7,$20,$20		# 7 cycles from $20 load	\n\
-	stq $20,16($17)						\n\
-	xor $22,$23,$23		# 7 cycles from $23 load	\n\
-								\n\
-	stq $23,24($17)						\n\
-	xor $25,$27,$27		# 7 cycles from $27 load	\n\
-	stq $27,32($17)						\n\
-	nop							\n\
-								\n\
-	ldq $0,40($17)						\n\
-	ldq $1,40($18)						\n\
-	ldq $3,48($17)						\n\
-	ldq $4,48($18)						\n\
-								\n\
-	ldq $6,56($17)						\n\
-	ldq $7,56($18)						\n\
-	ldq $2,40($19)						\n\
-	ldq $5,48($19)						\n\
-								\n\
-	ldq $20,56($19)						\n\
-	ldq $31,256($17)					\n\
-	ldq $31,256($18)					\n\
-	ldq $31,256($19)					\n\
-								\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-	xor $3,$4,$4		# 5 cycles from $4 load		\n\
-	xor $6,$7,$7		# 5 cycles from $7 load		\n\
-	xor $1,$2,$2		# 4 cycles from $2 load		\n\
-								\n\
-	xor $4,$5,$5		# 5 cycles from $5 load		\n\
-	xor $7,$20,$20		# 4 cycles from $20 load	\n\
-	stq $2,40($17)						\n\
-	subq $16,1,$16						\n\
-								\n\
-	stq $5,48($17)						\n\
-	addq $19,64,$19						\n\
-	stq $20,56($17)						\n\
-	addq $18,64,$18						\n\
-								\n\
-	addq $17,64,$17						\n\
-	bgt $16,3b						\n\
-	ret							\n\
-	.end xor_alpha_prefetch_3				\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_prefetch_4				\n\
-xor_alpha_prefetch_4:						\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-								\n\
-	ldq $31, 0($17)						\n\
-	ldq $31, 0($18)						\n\
-	ldq $31, 0($19)						\n\
-	ldq $31, 0($20)						\n\
-								\n\
-	ldq $31, 64($17)					\n\
-	ldq $31, 64($18)					\n\
-	ldq $31, 64($19)					\n\
-	ldq $31, 64($20)					\n\
-								\n\
-	ldq $31, 128($17)					\n\
-	ldq $31, 128($18)					\n\
-	ldq $31, 128($19)					\n\
-	ldq $31, 128($20)					\n\
-								\n\
-	ldq $31, 192($17)					\n\
-	ldq $31, 192($18)					\n\
-	ldq $31, 192($19)					\n\
-	ldq $31, 192($20)					\n\
-	.align 4						\n\
-4:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,0($20)						\n\
-								\n\
-	ldq $4,8($17)						\n\
-	ldq $5,8($18)						\n\
-	ldq $6,8($19)						\n\
-	ldq $7,8($20)						\n\
-								\n\
-	ldq $21,16($17)						\n\
-	ldq $22,16($18)						\n\
-	ldq $23,16($19)						\n\
-	ldq $24,16($20)						\n\
-								\n\
-	ldq $25,24($17)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-	ldq $27,24($18)						\n\
-	xor $2,$3,$3		# 6 cycles from $3 load		\n\
-								\n\
-	ldq $0,24($19)						\n\
-	xor $1,$3,$3						\n\
-	ldq $1,24($20)						\n\
-	xor $4,$5,$5		# 7 cycles from $5 load		\n\
-								\n\
-	stq $3,0($17)						\n\
-	xor $6,$7,$7						\n\
-	xor $21,$22,$22		# 7 cycles from $22 load	\n\
-	xor $5,$7,$7						\n\
-								\n\
-	stq $7,8($17)						\n\
-	xor $23,$24,$24		# 7 cycles from $24 load	\n\
-	ldq $2,32($17)						\n\
-	xor $22,$24,$24						\n\
-								\n\
-	ldq $3,32($18)						\n\
-	ldq $4,32($19)						\n\
-	ldq $5,32($20)						\n\
-	xor $25,$27,$27		# 8 cycles from $27 load	\n\
-								\n\
-	ldq $6,40($17)						\n\
-	ldq $7,40($18)						\n\
-	ldq $21,40($19)						\n\
-	ldq $22,40($20)						\n\
-								\n\
-	stq $24,16($17)						\n\
-	xor $0,$1,$1		# 9 cycles from $1 load		\n\
-	xor $2,$3,$3		# 5 cycles from $3 load		\n\
-	xor $27,$1,$1						\n\
-								\n\
-	stq $1,24($17)						\n\
-	xor $4,$5,$5		# 5 cycles from $5 load		\n\
-	ldq $23,48($17)						\n\
-	xor $3,$5,$5						\n\
-								\n\
-	ldq $24,48($18)						\n\
-	ldq $25,48($19)						\n\
-	ldq $27,48($20)						\n\
-	ldq $0,56($17)						\n\
-								\n\
-	ldq $1,56($18)						\n\
-	ldq $2,56($19)						\n\
-	ldq $3,56($20)						\n\
-	xor $6,$7,$7		# 8 cycles from $6 load		\n\
-								\n\
-	ldq $31,256($17)					\n\
-	xor $21,$22,$22		# 8 cycles from $22 load	\n\
-	ldq $31,256($18)					\n\
-	xor $7,$22,$22						\n\
-								\n\
-	ldq $31,256($19)					\n\
-	xor $23,$24,$24		# 6 cycles from $24 load	\n\
-	ldq $31,256($20)					\n\
-	xor $25,$27,$27		# 6 cycles from $27 load	\n\
-								\n\
-	stq $5,32($17)						\n\
-	xor $24,$27,$27						\n\
-	xor $0,$1,$1		# 7 cycles from $1 load		\n\
-	xor $2,$3,$3		# 6 cycles from $3 load		\n\
-								\n\
-	stq $22,40($17)						\n\
-	xor $1,$3,$3						\n\
-	stq $27,48($17)						\n\
-	subq $16,1,$16						\n\
-								\n\
-	stq $3,56($17)						\n\
-	addq $20,64,$20						\n\
-	addq $19,64,$19						\n\
-	addq $18,64,$18						\n\
-								\n\
-	addq $17,64,$17						\n\
-	bgt $16,4b						\n\
-	ret							\n\
-	.end xor_alpha_prefetch_4				\n\
-								\n\
-	.align 3						\n\
-	.ent xor_alpha_prefetch_5				\n\
-xor_alpha_prefetch_5:						\n\
-	.prologue 0						\n\
-	srl $16, 6, $16						\n\
-								\n\
-	ldq $31, 0($17)						\n\
-	ldq $31, 0($18)						\n\
-	ldq $31, 0($19)						\n\
-	ldq $31, 0($20)						\n\
-	ldq $31, 0($21)						\n\
-								\n\
-	ldq $31, 64($17)					\n\
-	ldq $31, 64($18)					\n\
-	ldq $31, 64($19)					\n\
-	ldq $31, 64($20)					\n\
-	ldq $31, 64($21)					\n\
-								\n\
-	ldq $31, 128($17)					\n\
-	ldq $31, 128($18)					\n\
-	ldq $31, 128($19)					\n\
-	ldq $31, 128($20)					\n\
-	ldq $31, 128($21)					\n\
-								\n\
-	ldq $31, 192($17)					\n\
-	ldq $31, 192($18)					\n\
-	ldq $31, 192($19)					\n\
-	ldq $31, 192($20)					\n\
-	ldq $31, 192($21)					\n\
-	.align 4						\n\
-5:								\n\
-	ldq $0,0($17)						\n\
-	ldq $1,0($18)						\n\
-	ldq $2,0($19)						\n\
-	ldq $3,0($20)						\n\
-								\n\
-	ldq $4,0($21)						\n\
-	ldq $5,8($17)						\n\
-	ldq $6,8($18)						\n\
-	ldq $7,8($19)						\n\
-								\n\
-	ldq $22,8($20)						\n\
-	ldq $23,8($21)						\n\
-	ldq $24,16($17)						\n\
-	ldq $25,16($18)						\n\
-								\n\
-	ldq $27,16($19)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-	ldq $28,16($20)						\n\
-	xor $2,$3,$3		# 6 cycles from $3 load		\n\
-								\n\
-	ldq $0,16($21)						\n\
-	xor $1,$3,$3						\n\
-	ldq $1,24($17)						\n\
-	xor $3,$4,$4		# 7 cycles from $4 load		\n\
-								\n\
-	stq $4,0($17)						\n\
-	xor $5,$6,$6		# 7 cycles from $6 load		\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	xor $6,$23,$23		# 7 cycles from $23 load	\n\
-								\n\
-	ldq $2,24($18)						\n\
-	xor $22,$23,$23						\n\
-	ldq $3,24($19)						\n\
-	xor $24,$25,$25		# 8 cycles from $25 load	\n\
-								\n\
-	stq $23,8($17)						\n\
-	xor $25,$27,$27		# 8 cycles from $27 load	\n\
-	ldq $4,24($20)						\n\
-	xor $28,$0,$0		# 7 cycles from $0 load		\n\
-								\n\
-	ldq $5,24($21)						\n\
-	xor $27,$0,$0						\n\
-	ldq $6,32($17)						\n\
-	ldq $7,32($18)						\n\
-								\n\
-	stq $0,16($17)						\n\
-	xor $1,$2,$2		# 6 cycles from $2 load		\n\
-	ldq $22,32($19)						\n\
-	xor $3,$4,$4		# 4 cycles from $4 load		\n\
-								\n\
-	ldq $23,32($20)						\n\
-	xor $2,$4,$4						\n\
-	ldq $24,32($21)						\n\
-	ldq $25,40($17)						\n\
-								\n\
-	ldq $27,40($18)						\n\
-	ldq $28,40($19)						\n\
-	ldq $0,40($20)						\n\
-	xor $4,$5,$5		# 7 cycles from $5 load		\n\
-								\n\
-	stq $5,24($17)						\n\
-	xor $6,$7,$7		# 7 cycles from $7 load		\n\
-	ldq $1,40($21)						\n\
-	ldq $2,48($17)						\n\
-								\n\
-	ldq $3,48($18)						\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	ldq $4,48($19)						\n\
-	xor $23,$24,$24		# 6 cycles from $24 load	\n\
-								\n\
-	ldq $5,48($20)						\n\
-	xor $22,$24,$24						\n\
-	ldq $6,48($21)						\n\
-	xor $25,$27,$27		# 7 cycles from $27 load	\n\
-								\n\
-	stq $24,32($17)						\n\
-	xor $27,$28,$28		# 8 cycles from $28 load	\n\
-	ldq $7,56($17)						\n\
-	xor $0,$1,$1		# 6 cycles from $1 load		\n\
-								\n\
-	ldq $22,56($18)						\n\
-	ldq $23,56($19)						\n\
-	ldq $24,56($20)						\n\
-	ldq $25,56($21)						\n\
-								\n\
-	ldq $31,256($17)					\n\
-	xor $28,$1,$1						\n\
-	ldq $31,256($18)					\n\
-	xor $2,$3,$3		# 9 cycles from $3 load		\n\
-								\n\
-	ldq $31,256($19)					\n\
-	xor $3,$4,$4		# 9 cycles from $4 load		\n\
-	ldq $31,256($20)					\n\
-	xor $5,$6,$6		# 8 cycles from $6 load		\n\
-								\n\
-	stq $1,40($17)						\n\
-	xor $4,$6,$6						\n\
-	xor $7,$22,$22		# 7 cycles from $22 load	\n\
-	xor $23,$24,$24		# 6 cycles from $24 load	\n\
-								\n\
-	stq $6,48($17)						\n\
-	xor $22,$24,$24						\n\
-	ldq $31,256($21)					\n\
-	xor $24,$25,$25		# 8 cycles from $25 load	\n\
-								\n\
-	stq $25,56($17)						\n\
-	subq $16,1,$16						\n\
-	addq $21,64,$21						\n\
-	addq $20,64,$20						\n\
-								\n\
-	addq $19,64,$19						\n\
-	addq $18,64,$18						\n\
-	addq $17,64,$17						\n\
-	bgt $16,5b						\n\
-								\n\
-	ret							\n\
-	.end xor_alpha_prefetch_5				\n\
-");
-
-static struct xor_block_template xor_block_alpha = {
-	.name	= "alpha",
-	.do_2	= xor_alpha_2,
-	.do_3	= xor_alpha_3,
-	.do_4	= xor_alpha_4,
-	.do_5	= xor_alpha_5,
-};
-
-static struct xor_block_template xor_block_alpha_prefetch = {
-	.name	= "alpha prefetch",
-	.do_2	= xor_alpha_prefetch_2,
-	.do_3	= xor_alpha_prefetch_3,
-	.do_4	= xor_alpha_prefetch_4,
-	.do_5	= xor_alpha_prefetch_5,
-};
-
-/* For grins, also test the generic routines.  */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-	do {						\
-		xor_speed(&xor_block_8regs);		\
-		xor_speed(&xor_block_32regs);		\
-		xor_speed(&xor_block_alpha);		\
-		xor_speed(&xor_block_alpha_prefetch);	\
-	} while (0)
-
-/* Force the use of alpha_prefetch if EV6, as it is significantly
-   faster in the cold cache case.  */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
-	(implver() == IMPLVER_EV6 ? &xor_block_alpha_prefetch : FASTEST)
diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h
index 38214e126c6de..facc7a03b2501 100644
--- a/arch/arc/include/asm/page.h
+++ b/arch/arc/include/asm/page.h
@@ -131,7 +131,7 @@ static inline unsigned long virt_to_pfn(const void *kaddr)
 #define virt_addr_valid(kaddr)  pfn_valid(virt_to_pfn(kaddr))
 
 /* Default Permissions for stack/heaps pages (Non Executable) */
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_NON_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_NON_EXEC
 
 #define WANT_PAGE_VIRTUAL   1
 
diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h
index bd580e2b62d7c..0fdaea81b5fa5 100644
--- a/arch/arc/include/asm/pgtable.h
+++ b/arch/arc/include/asm/pgtable.h
@@ -21,9 +21,6 @@
 
 #ifndef __ASSEMBLER__
 
-extern char empty_zero_page[PAGE_SIZE];
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-
 extern pgd_t swapper_pg_dir[] __aligned(PAGE_SIZE);
 
 /* to cope with aliasing VIPT cache */
diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c
index a5e92f46e5d13..d6b5c27a0098f 100644
--- a/arch/arc/mm/init.c
+++ b/arch/arc/mm/init.c
@@ -19,8 +19,6 @@
 #include <asm/arcregs.h>
 
 pgd_t swapper_pg_dir[PTRS_PER_PGD] __aligned(PAGE_SIZE);
-char empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE);
-EXPORT_SYMBOL(empty_zero_page);
 
 static const unsigned long low_mem_start = CONFIG_LINUX_RAM_BASE;
 static unsigned long low_mem_sz;
diff --git a/arch/arc/plat-axs10x/Kconfig b/arch/arc/plat-axs10x/Kconfig
index b9652c69d1b90..40f2a74d404ad 100644
--- a/arch/arc/plat-axs10x/Kconfig
+++ b/arch/arc/plat-axs10x/Kconfig
@@ -7,7 +7,6 @@ menuconfig ARC_PLAT_AXS10X
 	bool "Synopsys ARC AXS10x Software Development Platforms"
 	select DW_APB_ICTL
 	select GPIO_DWAPB
-	select OF_GPIO
 	select HAVE_PCI
 	select GENERIC_IRQ_CHIP
 	select GPIOLIB
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ec33376f8e2ba..92917231789d9 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -159,6 +159,7 @@ config ARM
 	select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS
 	select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M
 	select USE_OF if !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100)
+	select XOR_BLOCKS_ARCH
 	# Above selects are sorted alphabetically; please add new ones
 	# according to that.  Thanks.
 	help
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 0000000000000..fa20f522c6eeb
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,40 @@
+soc/arm
+
+soc/dt
+	patch
+		ARM: dts: arm: Use lowercase hex
+	renesas/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v7.1-tag1
+		contains renesas/fixes
+
+soc/drivers
+	renesas/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-drivers-for-v7.1-tag1
+	renesas/drivers-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dt-bindings-for-v7.1-tag1
+
+soc/defconfig
+	renesas/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-arm-defconfig-for-v7.1-tag1
+
+soc/late
+
+arm/fixes
+	riscv/soc-fixes
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-soc-fixes-for-v7.0-rc1
+	fsl/soc-driver-fixes
+		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux tags/soc_fsl-7.0-2
+	reset/fixes
+		https://git.pengutronix.de/git/pza/linux tags/reset-fixes-for-v7.0
+	renesas/fixes
+		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-fixes-for-v7.0-tag1
+	patch
+		ARM: multi_v7_defconfig: Drop duplicate CONFIG_TI_PRUSS=m
+	rockchip/fixes
+		ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v7.0-rockchip-drvfixes1
+	<no branch> (08d9a4580f71120be3c5b221af32dca00a48ceb0)
+		git://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee tags/tee-fix-for-v7.0
+	<no branch> (a4e8473b775160f3ce978f621cf8dea2c7250433)
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/ffa-fix-7.0
+	<no branch> (4e701b47c3ba8f4eaf51d676732b11204bc75b35)
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/scmi-fixes-7.0
diff --git a/arch/arm/boot/dts/arm/arm-realview-eb.dtsi b/arch/arm/boot/dts/arm/arm-realview-eb.dtsi
index 16f784da5a556..ae370d4eb9d83 100644
--- a/arch/arm/boot/dts/arm/arm-realview-eb.dtsi
+++ b/arch/arm/boot/dts/arm/arm-realview-eb.dtsi
@@ -234,7 +234,7 @@ oscclk0: clock-controller@c {
 				reg = <0x0c 0x04>;
 				#clock-cells = <0>;
 				lock-offset = <0x20>;
-				vco-offset = <0x0C>;
+				vco-offset = <0x0c>;
 				clocks = <&xtal24mhz>;
 			};
 			oscclk1: clock-controller@10 {
diff --git a/arch/arm/boot/dts/arm/arm-realview-pb1176.dts b/arch/arm/boot/dts/arm/arm-realview-pb1176.dts
index b9b10cbd65aa8..a75d8b107fe18 100644
--- a/arch/arm/boot/dts/arm/arm-realview-pb1176.dts
+++ b/arch/arm/boot/dts/arm/arm-realview-pb1176.dts
@@ -260,7 +260,7 @@ oscclk0: clock-controller@c {
 				reg = <0x0c 0x04>;
 				#clock-cells = <0>;
 				lock-offset = <0x20>;
-				vco-offset = <0x0C>;
+				vco-offset = <0x0c>;
 				clocks = <&xtal24mhz>;
 			};
 			oscclk1: clock-controller@10 {
diff --git a/arch/arm/boot/dts/arm/arm-realview-pb11mp.dts b/arch/arm/boot/dts/arm/arm-realview-pb11mp.dts
index db1b6793cd2c6..59c78def83f2a 100644
--- a/arch/arm/boot/dts/arm/arm-realview-pb11mp.dts
+++ b/arch/arm/boot/dts/arm/arm-realview-pb11mp.dts
@@ -343,7 +343,7 @@ oscclk0: clock-controller@c {
 				reg = <0x0c 0x04>;
 				#clock-cells = <0>;
 				lock-offset = <0x20>;
-				vco-offset = <0x0C>;
+				vco-offset = <0x0c>;
 				clocks = <&xtal24mhz>;
 			};
 			oscclk1: clock-controller@10 {
diff --git a/arch/arm/boot/dts/arm/arm-realview-pbx.dtsi b/arch/arm/boot/dts/arm/arm-realview-pbx.dtsi
index e625403a9456f..0d3d96ffa6e63 100644
--- a/arch/arm/boot/dts/arm/arm-realview-pbx.dtsi
+++ b/arch/arm/boot/dts/arm/arm-realview-pbx.dtsi
@@ -256,7 +256,7 @@ oscclk0: clock-controller@c {
 				reg = <0x0c 0x04>;
 				#clock-cells = <0>;
 				lock-offset = <0x20>;
-				vco-offset = <0x0C>;
+				vco-offset = <0x0c>;
 				clocks = <&xtal24mhz>;
 			};
 			oscclk1: clock-controller@10 {
diff --git a/arch/arm/boot/dts/aspeed/Makefile b/arch/arm/boot/dts/aspeed/Makefile
index 20e5b64280a5e..767f7c7652d55 100644
--- a/arch/arm/boot/dts/aspeed/Makefile
+++ b/arch/arm/boot/dts/aspeed/Makefile
@@ -12,9 +12,11 @@ dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-bmc-asrock-altrad8.dtb \
 	aspeed-bmc-asrock-e3c246d4i.dtb \
 	aspeed-bmc-asrock-e3c256d4i.dtb \
+	aspeed-bmc-asrock-paul-ipmi-card.dtb \
 	aspeed-bmc-asrock-romed8hm3.dtb \
 	aspeed-bmc-asrock-spc621d8hm3.dtb \
 	aspeed-bmc-asrock-x570d4u.dtb \
+	aspeed-bmc-asus-kommando-ipmi-card.dtb \
 	aspeed-bmc-asus-x4tf.dtb \
 	aspeed-bmc-bytedance-g220a.dtb \
 	aspeed-bmc-delta-ahe50dc.dtb \
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-paul-ipmi-card.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-paul-ipmi-card.dts
new file mode 100644
index 0000000000000..f74f8fee9e1ed
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-paul-ipmi-card.dts
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright 2025 Anirudh Srinivasan
+
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/{
+	model = "ASRock Paul IPMI Card";
+	compatible = "asrock,ast2500-paul-ipmi-card", "aspeed,ast2500";
+
+	aliases {
+		serial4 = &uart5;
+	};
+
+	chosen {
+		stdout-path = &uart5;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-fan-1 {
+			gpios = <&gpio ASPEED_GPIO(AA, 0) GPIO_ACTIVE_LOW>;
+			label = "fan1_red";
+			default-state = "off";
+		};
+
+		led-fan-2 {
+			gpios = <&gpio ASPEED_GPIO(AA, 1) GPIO_ACTIVE_LOW>;
+			label = "fan2_red";
+			default-state = "off";
+		};
+
+		led-fault {
+			gpios = <&gpio ASPEED_GPIO(Y, 3) GPIO_ACTIVE_LOW>;
+			label = "panic_red";
+			panic-indicator;
+			default-state = "off";
+		};
+
+		led-heartbeat {
+			gpios = <&gpio ASPEED_GPIO(H, 6) GPIO_ACTIVE_LOW>;
+			label = "heartbeat_green";
+			linux,default-trigger = "timer";
+		};
+	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		video_engine_memory: video {
+			size = <0x02000000>;
+			alignment = <0x01000000>;
+			compatible = "shared-dma-pool";
+			reusable;
+		};
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		spi-max-frequency = <50000000>; /* 50 MHz */
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&gpio {
+	status = "okay";
+	gpio-line-names =
+		/*  A */ "", "", "", "", "", "", "", "",
+		/*  B */ "", "", "", "", "", "", "", "",
+		/*  C */ "", "", "", "", "", "", "", "",
+		/*  D */ "", "BMC_PWRBTN", "", "BMC_RESETCON", "", "", "", "",
+		/*  E */ "", "", "", "", "", "", "", "",
+		/*  F */ "", "", "", "", "", "", "", "",
+		/*  G */ "", "", "", "", "", "", "", "",
+		/*  H */ "", "", "", "", "", "", "BMC_LED1", "",
+		/*  I */ "", "", "", "", "", "", "", "",
+		/*  J */ "", "", "", "", "", "", "", "",
+		/*  K */ "", "", "", "", "", "", "", "",
+		/*  L */ "", "", "", "", "", "", "", "",
+		/*  M */ "", "", "", "", "", "", "", "",
+		/*  N */ "", "", "", "", "", "", "", "",
+		/*  O */ "", "", "", "", "", "", "", "",
+		/*  P */ "", "", "", "", "", "", "", "",
+		/*  Q */ "", "", "", "", "", "", "", "",
+		/*  R */ "", "", "", "", "", "", "", "",
+		/*  S */ "", "", "", "", "", "", "", "",
+		/*  T */ "", "", "", "", "", "", "", "",
+		/*  U */ "", "", "", "", "", "", "", "",
+		/*  V */ "", "", "", "", "", "", "", "",
+		/*  W */ "", "", "", "", "", "", "", "",
+		/*  X */ "", "", "", "", "", "PCIE_STNDBY", "", "",
+		/*  Y */ "", "", "", "SYSTEM_ERR_LED", "", "", "", "",
+		/*  Z */ "", "", "", "", "", "", "", "",
+		/* AA */ "FAN_1_LED", "FAN_2_LED", "", "", "", "", "", "",
+		/* AB */ "", "", "", "", "", "", "", "",
+		/* AC */ "", "", "", "", "", "", "", "";
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii1_default &pinctrl_mdio1_default>;
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&vhub {
+	status = "okay";
+};
+
+&video {
+	status = "okay";
+	memory-region = <&video_engine_memory>;
+};
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-asus-kommando-ipmi-card.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-asus-kommando-ipmi-card.dts
new file mode 100644
index 0000000000000..ab7ad320067c1
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-asus-kommando-ipmi-card.dts
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright 2025 Anirudh Srinivasan
+
+/dts-v1/;
+
+#include "aspeed-g6.dtsi"
+#include "aspeed-g6-pinctrl.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Asus Kommando IPMI Expansion Card";
+	compatible = "asus,ast2600-kommando-ipmi-card", "aspeed,ast2600";
+
+	aliases {
+		serial4 = &uart5;
+	};
+
+	chosen {
+		stdout-path = "serial4:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-fault {
+			gpios = <&gpio1 ASPEED_GPIO(C, 5) GPIO_ACTIVE_HIGH>;
+			label = "panic_amber";
+			panic-indicator;
+			default-state = "off";
+		};
+
+		led-heartbeat {
+			gpios = <&gpio0 ASPEED_GPIO(A, 7) GPIO_ACTIVE_LOW>;
+			label = "heartbeat_green";
+			linux,default-trigger = "timer";
+		};
+
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		video_engine_memory: video {
+			size = <0x04000000>;
+			alignment = <0x01000000>;
+			compatible = "shared-dma-pool";
+			reusable;
+		};
+	};
+};
+
+&fmc {
+	status = "okay";
+
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		spi-max-frequency = <50000000>;
+#include "openbmc-flash-layout-64.dtsi"
+	};
+};
+
+&gpio0 {
+	gpio-line-names =
+	/*A0 0*/	"", "", "", "", "", "", "", "BMC_HBLED",
+	/*B0 8*/	"", "", "", "", "", "", "", "",
+	/*C0 16*/	"", "", "", "", "", "", "", "",
+	/*D0 24*/	"", "", "", "", "", "", "", "",
+	/*E0 32*/	"", "", "", "", "", "", "", "",
+	/*F0 40*/	"", "", "", "", "", "", "", "",
+	/*G0 48*/	"", "", "", "", "", "", "", "",
+	/*H0 56*/	"", "", "", "", "", "", "", "",
+	/*I0 64*/	"", "", "", "BMC_RSTBTN", "", "", "", "",
+	/*J0 72*/	"", "", "", "", "", "", "", "",
+	/*K0 80*/	"", "", "", "", "", "", "", "",
+	/*L0 88*/	"", "", "", "", "", "", "", "",
+	/*M0 96*/	"", "", "", "", "", "", "", "",
+	/*N0 104*/	"", "", "", "", "", "", "", "",
+	/*O0 112*/	"", "", "", "", "", "", "", "",
+	/*P0 120*/	"", "", "", "", "", "", "", "",
+	/*Q0 128*/	"", "", "", "", "", "", "", "",
+	/*R0 136*/	"", "", "", "", "", "", "", "",
+	/*S0 144*/	"", "", "", "", "", "", "", "",
+	/*T0 152*/	"", "", "", "", "", "", "", "",
+	/*U0 160*/	"", "", "", "", "", "", "", "",
+	/*V0 168*/	"", "", "", "", "BMC_PWRBTN", "", "MB_S0_DETECT", "",
+	/*W0 176*/	"", "", "", "", "", "", "", "",
+	/*X0 184*/	"", "", "", "", "", "", "", "",
+	/*Y0 192*/	"", "", "", "", "", "", "", "",
+	/*Z0 200*/	"", "", "", "", "", "", "", "";
+};
+
+&gpio1 {
+	gpio-line-names =
+	/*18A0 0*/  "","","","","","","","",
+	/*18B0 8*/  "","","","","","","","",
+	/*18C0 16*/ "","","","","","BMC_MLED","","",
+	/*18D0 24*/ "","","","","","","","",
+	/*18E0 32*/ "","","","","","","","";
+};
+
+&vhub {
+	status = "okay";
+};
+
+&video {
+	status = "okay";
+	memory-region = <&video_engine_memory>;
+};
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
index 221af858cb6b8..2cb7bd128d24e 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
@@ -300,6 +300,12 @@ &gpio1 {
 &i2c0 {
 	status = "okay";
 
+	eeprom@50 {
+		compatible = "atmel,24c2048";
+		reg = <0x50>;
+		pagesize = <128>;
+	};
+
 	i2c-mux@70 {
 		compatible = "nxp,pca9546";
 		reg = <0x70>;
@@ -334,6 +340,12 @@ i2c0mux0ch3: i2c@3 {
 &i2c1 {
 	status = "okay";
 
+	eeprom@50 {
+		compatible = "atmel,24c2048";
+		reg = <0x50>;
+		pagesize = <128>;
+	};
+
 	i2c-mux@70 {
 		compatible = "nxp,pca9546";
 		reg = <0x70>;
@@ -802,6 +814,16 @@ i2c13mux0ch7: i2c@7 {
 			reg = <7>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+
+			nfc@28 {
+				compatible = "nxp,nxp-nci-i2c";
+				reg = <0x28>;
+
+				interrupt-parent = <&sgpiom0>;
+				interrupts = <156 IRQ_TYPE_LEVEL_HIGH>;
+
+				enable-gpios = <&sgpiom0 241 GPIO_ACTIVE_HIGH>;
+			};
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
index 44f95a3986cb9..a12d4164de4ac 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
@@ -67,6 +67,7 @@ flash@1 {
 		spi-tx-bus-width = <1>;
 		spi-rx-bus-width = <1>;
 		status = "okay";
+		#include "openbmc-flash-layout-128-alt.dtsi"
 	};
 };
 
diff --git a/arch/arm/boot/dts/aspeed/openbmc-flash-layout-128-alt.dtsi b/arch/arm/boot/dts/aspeed/openbmc-flash-layout-128-alt.dtsi
new file mode 100644
index 0000000000000..08ce2eab472c3
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/openbmc-flash-layout-128-alt.dtsi
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+partitions {
+	compatible = "fixed-partitions";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	u-boot@0 {
+		reg = <0x0 0xe0000>; // 896KB
+		label = "alt-u-boot";
+	};
+
+	u-boot-env@e0000 {
+		reg = <0xe0000 0x20000>; // 128KB
+		label = "alt-u-boot-env";
+	};
+
+	kernel@100000 {
+		reg = <0x100000 0x900000>; // 9MB
+		label = "alt-kernel";
+	};
+
+	rofs@a00000 {
+		reg = <0xa00000 0x5600000>; // 86MB
+		label = "alt-rofs";
+	};
+
+	rwfs@6000000 {
+		reg = <0x6000000 0x2000000>; // 32MB
+		label = "alt-rwfs";
+	};
+};
diff --git a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
index d0d5f7e52a917..392a257136696 100644
--- a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
@@ -95,7 +95,10 @@ L2: cache-controller@22000 {
 	axi@18000000 {
 		compatible = "brcm,bus-axi";
 		reg = <0x18000000 0x1000>;
-		ranges = <0x00000000 0x18000000 0x00100000>;
+		ranges = <0x00000000 0x18000000 0x00100000>,
+			 <0x08000000 0x08000000 0x08000000>,
+			 <0x20000000 0x20000000 0x08000000>,
+			 <0x28000000 0x28000000 0x08000000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
 
@@ -182,24 +185,75 @@ chipcommon: chipcommon@0 {
 		};
 
 		pcie0: pcie@12000 {
+			compatible = "brcm,iproc-pcie";
 			reg = <0x00012000 0x1000>;
+			ranges = <0x82000000 0 0x08000000  0x08000000  0 0x08000000>;
+			interrupt-map-mask = <0 0 0 0>;
+			interrupt-map = <0 0 0 0 &gic GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>;
+			bus-range = <0x00 0xff>;
+			device_type = "pci";
 
+			#interrupt-cells = <1>;
 			#address-cells = <3>;
 			#size-cells = <2>;
+
+			pcie_bridge0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0000 0 0 0 0>;
+				bus-range = <0x00 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pcie1: pcie@13000 {
+			compatible = "brcm,iproc-pcie";
 			reg = <0x00013000 0x1000>;
+			ranges = <0x82000000 0 0x20000000  0x20000000  0 0x08000000>;
+			interrupt-map-mask = <0 0 0 0>;
+			interrupt-map = <0 0 0 0 &gic GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>;
+			bus-range = <0x00 0xff>;
+			device_type = "pci";
 
+			#interrupt-cells = <1>;
 			#address-cells = <3>;
 			#size-cells = <2>;
+
+			pcie_bridge1: pcie@0 {
+				device_type = "pci";
+				reg = <0x0000 0 0 0 0>;
+				bus-range = <0x00 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pcie2: pcie@14000 {
+			compatible = "brcm,iproc-pcie";
 			reg = <0x00014000 0x1000>;
+			ranges = <0x82000000 0 0x28000000  0x28000000  0 0x08000000>;
+			interrupt-map-mask = <0 0 0 0>;
+			interrupt-map = <0 0 0 0 &gic GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
+			bus-range = <0x00 0xff>;
+			device_type = "pci";
 
+			#interrupt-cells = <1>;
 			#address-cells = <3>;
 			#size-cells = <2>;
+
+			pcie_bridge2: pcie@0 {
+				device_type = "pci";
+				reg = <0x0000 0 0 0 0>;
+				bus-range = <0x00 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		usb2: usb2@21000 {
@@ -479,7 +533,7 @@ thermal: thermal@2c0 {
 	};
 
 	nand_controller: nand-controller@18028000 {
-		compatible = "brcm,nand-iproc", "brcm,brcmnand-v6.1", "brcm,brcmnand";
+		compatible = "brcm,nand-iproc", "brcm,brcmnand-v6.1";
 		reg = <0x18028000 0x600>, <0x1811a408 0x600>, <0x18028f00 0x20>;
 		reg-names = "nand", "iproc-idm", "iproc-ext";
 		interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi b/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
index 9261b67dbee1d..1e76b290510d3 100644
--- a/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
@@ -141,9 +141,10 @@ v3d: v3d@7ec00000 {
 			interrupts = <1 10>;
 		};
 
-		vc4: gpu {
-			compatible = "brcm,bcm2835-vc4";
-		};
+	};
+
+	vc4: gpu {
+		compatible = "brcm,bcm2835-vc4";
 	};
 };
 
diff --git a/arch/arm/boot/dts/broadcom/bcm2835-rpi.dtsi b/arch/arm/boot/dts/broadcom/bcm2835-rpi.dtsi
index e9bf41b9f5c18..46c91468f4c54 100644
--- a/arch/arm/boot/dts/broadcom/bcm2835-rpi.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm2835-rpi.dtsi
@@ -1,8 +1,8 @@
 #include <dt-bindings/power/raspberrypi-power.h>
 
 / {
-	soc {
-		firmware: firmware {
+	firmware {
+		firmware: rpi-firmware {
 			compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
 			mboxes = <&mailbox>;
 
@@ -10,14 +10,16 @@ firmware_clocks: clocks {
 				compatible = "raspberrypi,firmware-clocks";
 				#clock-cells = <1>;
 			};
-		};
 
-		power: power {
-			compatible = "raspberrypi,bcm2835-power";
-			firmware = <&firmware>;
-			#power-domain-cells = <1>;
+			power: power {
+				compatible = "raspberrypi,bcm2835-power";
+				firmware = <&firmware>;
+				#power-domain-cells = <1>;
+			};
 		};
+	};
 
+	soc {
 		vchiq: mailbox@7e00b840 {
 			compatible = "brcm,bcm2835-vchiq";
 			reg = <0x7e00b840 0x3c>;
diff --git a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
index 2ba5adf2b7e70..7c6ad77879459 100644
--- a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
@@ -25,6 +25,10 @@ memory@0 {
 	nvram@1c080000 {
 		compatible = "brcm,nvram";
 		reg = <0x1c080000 0x180000>;
+
+		et2macaddr: et2macaddr {
+			#nvmem-cell-cells = <1>;
+		};
 	};
 
 	gpio-keys {
@@ -70,6 +74,9 @@ port@3 {
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et2macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts b/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
index 127ca87412201..d170c71cbd763 100644
--- a/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
@@ -126,66 +126,53 @@ button-brightness {
 	};
 };
 
-&pcie0 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@0,0,0 {
+&pcie_bridge0 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0000 0 0 0 0>;
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		wifi@0,1,0 {
-			compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-			reg = <0x0000 0 0 0 0>;
-			ieee80211-freq-limit = <5735000 5835000>;
-			brcm,ccode-map = "JP-JP-78", "US-Q2-86";
-		};
+		ieee80211-freq-limit = <5735000 5835000>;
+		brcm,ccode-map = "JP-JP-78", "US-Q2-86";
 	};
 };
 
-&pcie1 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@1,0,0 {
+&pcie_bridge1 {
+	pcie@0,0 {
+		device_type = "pci";
 		reg = <0x0000 0 0 0 0>;
+		bus-range = <0x01 0xff>;
 
 		#address-cells = <3>;
 		#size-cells = <2>;
+		ranges;
 
-		bridge@1,1,0 {
-			reg = <0x0000 0 0 0 0>;
+		pcie@1,0 {
+			device_type = "pci";
+			reg = <0x800 0 0 0 0>;
 
 			#address-cells = <3>;
 			#size-cells = <2>;
+			ranges;
 
-			bridge@1,0 {
-				reg = <0x800 0 0 0 0>;
-
-				#address-cells = <3>;
-				#size-cells = <2>;
-
-				wifi@0,0 {
-					compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-					reg = <0x0000 0 0 0 0>;
-					brcm,ccode-map = "JP-JP-78", "US-Q2-86";
-				};
+			wifi@0,0 {
+				compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
+				reg = <0x0000 0 0 0 0>;
+				brcm,ccode-map = "JP-JP-78", "US-Q2-86";
 			};
+		};
 
-			bridge@1,2,2 {
-				reg = <0x1000 0 0 0 0>;
+		pcie@2,0 {
+			device_type = "pci";
+			reg = <0x1000 0 0 0 0>;
 
-				#address-cells = <3>;
-				#size-cells = <2>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
 
-				wifi@1,4,0 {
-					compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-					reg = <0x0000 0 0 0 0>;
-					ieee80211-freq-limit = <5170000 5730000>;
-					brcm,ccode-map = "JP-JP-78", "US-Q2-86";
-				};
+			wifi@0,0 {
+				compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
+				reg = <0x0000 0 0 0 0>;
+				ieee80211-freq-limit = <5170000 5730000>;
+				brcm,ccode-map = "JP-JP-78", "US-Q2-86";
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac3100.dtsi b/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac3100.dtsi
index 2cfaaabc7a6a8..2d2e7e5812914 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac3100.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac3100.dtsi
@@ -18,6 +18,10 @@ memory@0 {
 	nvram@1c080000 {
 		compatible = "brcm,nvram";
 		reg = <0x1c080000 0x00180000>;
+
+		et0macaddr: et0macaddr {
+			#nvmem-cell-cells = <1>;
+		};
 	};
 
 	gpio-keys {
@@ -143,6 +147,9 @@ port@3 {
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et0macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac5300.dts b/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac5300.dts
index 01ec8c03686a6..8bf623c67de00 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac5300.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-asus-rt-ac5300.dts
@@ -127,6 +127,9 @@ &srab {
 	ports {
 		port@0 {
 			label = "wan";
+
+			nvmem-cells = <&et1macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@1 {
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts b/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
index 2b5c80d835e9a..74161b76008ad 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
@@ -25,6 +25,10 @@ memory@0 {
 	nvram@1c080000 {
 		compatible = "brcm,nvram";
 		reg = <0x1c080000 0x100000>;
+
+		et2macaddr: et2macaddr {
+			#nvmem-cell-cells = <1>;
+		};
 	};
 
 	gpio-keys {
@@ -230,6 +234,9 @@ port@3 {
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et2macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-luxul-xap-1610.dts b/arch/arm/boot/dts/broadcom/bcm47094-luxul-xap-1610.dts
index badafa024d24c..3a33705c29693 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-luxul-xap-1610.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-luxul-xap-1610.dts
@@ -65,39 +65,19 @@ &gmac0 {
 };
 
 
-&pcie0 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@0,0 {
+&pcie_bridge0 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0000 0 0 0 0>;
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		wifi@0,0 {
-			compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-			reg = <0x0000 0 0 0 0>;
-			brcm,ccode-map = "AU-AU-920", "CA-CA-892", "GB-DE-964", "NZ-AU-920", "US-US-825";
-		};
+		brcm,ccode-map = "AU-AU-920", "CA-CA-892", "GB-DE-964", "NZ-AU-920", "US-US-825";
 	};
 };
 
-&pcie1 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@0,0 {
+&pcie_bridge1 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0000 0 0 0 0>;
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		wifi@0,0 {
-			compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-			reg = <0x0000 0 0 0 0>;
-			brcm,ccode-map = "AU-AU-920", "CA-CA-892", "GB-DE-964", "NZ-AU-920", "US-US-825";
-		};
+		brcm,ccode-map = "AU-AU-920", "CA-CA-892", "GB-DE-964", "NZ-AU-920", "US-US-825";
 	};
 };
 
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-luxul-xwr-3150-v1.dts b/arch/arm/boot/dts/broadcom/bcm47094-luxul-xwr-3150-v1.dts
index 83c429afc2974..8e487f60a2cca 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-luxul-xwr-3150-v1.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-luxul-xwr-3150-v1.dts
@@ -81,39 +81,19 @@ &gmac0 {
 	nvmem-cell-names = "mac-address";
 };
 
-&pcie0 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@0,0 {
+&pcie_bridge0 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0000 0 0 0 0>;
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		wifi@0,0 {
-			compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-			reg = <0x0000 0 0 0 0>;
-			brcm,ccode-map = "AU-AU-953", "CA-CA-946", "GB-E0-846", "NZ-AU-953", "US-Q2-930";
-		};
+		brcm,ccode-map = "AU-AU-953", "CA-CA-946", "GB-E0-846", "NZ-AU-953", "US-Q2-930";
 	};
 };
 
-&pcie1 {
-	#address-cells = <3>;
-	#size-cells = <2>;
-
-	bridge@0,0 {
+&pcie_bridge1 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0000 0 0 0 0>;
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		wifi@0,0 {
-			compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
-			reg = <0x0000 0 0 0 0>;
-			brcm,ccode-map = "AU-AU-953", "CA-CA-946", "GB-E0-846", "NZ-AU-953", "US-Q2-930";
-		};
+		brcm,ccode-map = "AU-AU-953", "CA-CA-946", "GB-E0-846", "NZ-AU-953", "US-Q2-930";
 	};
 };
 
diff --git a/arch/arm/boot/dts/broadcom/bcm63138.dtsi b/arch/arm/boot/dts/broadcom/bcm63138.dtsi
index 4ec568586b14c..5b084d61edd71 100644
--- a/arch/arm/boot/dts/broadcom/bcm63138.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm63138.dtsi
@@ -312,12 +312,21 @@ bootlut: bootlut@8000 {
 			reg = <0x8000 0x50>;
 		};
 
+		i2c0: i2c@3e00 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x3e00 0x60>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		pl081_dma: dma-controller@d000 {
 			compatible = "arm,pl081", "arm,primecell";
 			// The magic B105F00D info is missing
 			arm,primecell-periphid = <0x00041081>;
 			reg = <0xd000 0x1000>;
-			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 			memcpy-burst-size = <256>;
 			memcpy-bus-width = <32>;
 			clocks = <&periph_clk>;
diff --git a/arch/arm/boot/dts/broadcom/bcm63148.dtsi b/arch/arm/boot/dts/broadcom/bcm63148.dtsi
index e071cddb28fc2..b2ddc2f583e1a 100644
--- a/arch/arm/boot/dts/broadcom/bcm63148.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm63148.dtsi
@@ -97,7 +97,7 @@ bus@ff800000 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0 0xfffe8000 0x8000>;
+		ranges = <0 0xfffe8000 0x20000>;
 
 		/* GPIOs 0 .. 31 */
 		gpio0: gpio@100 {
@@ -197,5 +197,14 @@ nandcs: nand@0 {
 				reg = <0>;
 			};
 		};
+
+		i2c0: i2c@3e00 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x3e00 0x60>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/broadcom/bcm6846.dtsi b/arch/arm/boot/dts/broadcom/bcm6846.dtsi
index f5591a45d2e4a..47f177323c188 100644
--- a/arch/arm/boot/dts/broadcom/bcm6846.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm6846.dtsi
@@ -242,6 +242,15 @@ mdio: mdio@2060 {
 			status = "disabled";
 		};
 
+		i2c0: i2c@2100 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x2100 0x60>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		pl081_dma: dma-controller@59000 {
 			compatible = "arm,pl081", "arm,primecell";
 			// The magic B105F00D info is missing
diff --git a/arch/arm/boot/dts/broadcom/bcm6855.dtsi b/arch/arm/boot/dts/broadcom/bcm6855.dtsi
index a88c3f0fbcb03..b3c4cd24ce531 100644
--- a/arch/arm/boot/dts/broadcom/bcm6855.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm6855.dtsi
@@ -240,6 +240,15 @@ nandcs: nand@0 {
 			};
 		};
 
+		i2c0: i2c@2100 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x2100 0x60>;
+			interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		leds: led-controller@3000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -278,5 +287,14 @@ uart1: serial@13000 {
 			clock-names = "uartclk", "apb_pclk";
 			status = "disabled";
 		};
+
+		i2c1: i2c@5a800 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x5a800 0x60>;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/broadcom/bcm6878.dtsi b/arch/arm/boot/dts/broadcom/bcm6878.dtsi
index dd837bf693905..e7fb45ae1d493 100644
--- a/arch/arm/boot/dts/broadcom/bcm6878.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm6878.dtsi
@@ -239,12 +239,21 @@ nandcs: nand@0 {
 			};
 		};
 
+		i2c0: i2c@2100 {
+			compatible = "brcm,brcmper-i2c";
+			reg = <0x2100 0x60>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		pl081_dma: dma-controller@11000 {
 			compatible = "arm,pl081", "arm,primecell";
 			// The magic B105F00D info is missing
 			arm,primecell-periphid = <0x00041081>;
 			reg = <0x11000 0x1000>;
-			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
 			memcpy-burst-size = <256>;
 			memcpy-bus-width = <32>;
 			clocks = <&periph_clk>;
diff --git a/arch/arm/boot/dts/marvell/armada-380.dtsi b/arch/arm/boot/dts/marvell/armada-380.dtsi
deleted file mode 100644
index e94f22b0e9b5e..0000000000000
--- a/arch/arm/boot/dts/marvell/armada-380.dtsi
+++ /dev/null
@@ -1,148 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-/*
- * Device Tree Include file for Marvell Armada 380 SoC.
- *
- * Copyright (C) 2014 Marvell
- *
- * Lior Amsalem <alior@marvell.com>
- * Gregory CLEMENT <gregory.clement@free-electrons.com>
- * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
- */
-
-#include "armada-38x.dtsi"
-
-/ {
-	model = "Marvell Armada 380 family SoC";
-	compatible = "marvell,armada380";
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		enable-method = "marvell,armada-380-smp";
-
-		cpu@0 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0>;
-		};
-	};
-
-	soc {
-		internal-regs {
-			pinctrl@18000 {
-				compatible = "marvell,mv88f6810-pinctrl";
-			};
-		};
-
-		pcie {
-			compatible = "marvell,armada-370-pcie";
-			status = "disabled";
-			device_type = "pci";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-
-			msi-parent = <&mpic>;
-			bus-range = <0x00 0xff>;
-
-			ranges =
-			       <0x82000000 0 0x80000 MBUS_ID(0xf0, 0x01) 0x80000 0 0x00002000
-				0x82000000 0 0x40000 MBUS_ID(0xf0, 0x01) 0x40000 0 0x00002000
-				0x82000000 0 0x44000 MBUS_ID(0xf0, 0x01) 0x44000 0 0x00002000
-				0x82000000 0 0x48000 MBUS_ID(0xf0, 0x01) 0x48000 0 0x00002000
-				0x82000000 0x1 0     MBUS_ID(0x08, 0xe8) 0 1 0 /* Port 0 MEM */
-				0x81000000 0x1 0     MBUS_ID(0x08, 0xe0) 0 1 0 /* Port 0 IO  */
-				0x82000000 0x2 0     MBUS_ID(0x04, 0xe8) 0 1 0 /* Port 1 MEM */
-				0x81000000 0x2 0     MBUS_ID(0x04, 0xe0) 0 1 0 /* Port 1 IO  */
-				0x82000000 0x3 0     MBUS_ID(0x04, 0xd8) 0 1 0 /* Port 2 MEM */
-				0x81000000 0x3 0     MBUS_ID(0x04, 0xd0) 0 1 0 /* Port 2 IO  */>;
-
-			/* x1 port */
-			pcie@1,0 {
-				device_type = "pci";
-				assigned-addresses = <0x82000800 0 0x80000 0 0x2000>;
-				reg = <0x0800 0 0 0 0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				interrupt-names = "intx";
-				interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
-				#interrupt-cells = <1>;
-				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
-					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
-				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 7>;
-				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
-						<0 0 0 2 &pcie1_intc 1>,
-						<0 0 0 3 &pcie1_intc 2>,
-						<0 0 0 4 &pcie1_intc 3>;
-				marvell,pcie-port = <0>;
-				marvell,pcie-lane = <0>;
-				clocks = <&gateclk 8>;
-				status = "disabled";
-
-				pcie1_intc: interrupt-controller {
-					interrupt-controller;
-					#interrupt-cells = <1>;
-				};
-			};
-
-			/* x1 port */
-			pcie@2,0 {
-				device_type = "pci";
-				assigned-addresses = <0x82001000 0 0x40000 0 0x2000>;
-				reg = <0x1000 0 0 0 0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				interrupt-names = "intx";
-				interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
-				#interrupt-cells = <1>;
-				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
-					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
-				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 7>;
-				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
-						<0 0 0 2 &pcie2_intc 1>,
-						<0 0 0 3 &pcie2_intc 2>,
-						<0 0 0 4 &pcie2_intc 3>;
-				marvell,pcie-port = <1>;
-				marvell,pcie-lane = <0>;
-				clocks = <&gateclk 5>;
-				status = "disabled";
-
-				pcie2_intc: interrupt-controller {
-					interrupt-controller;
-					#interrupt-cells = <1>;
-				};
-			};
-
-			/* x1 port */
-			pcie@3,0 {
-				device_type = "pci";
-				assigned-addresses = <0x82001800 0 0x44000 0 0x2000>;
-				reg = <0x1800 0 0 0 0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				interrupt-names = "intx";
-				interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
-				#interrupt-cells = <1>;
-				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
-					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
-				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 7>;
-				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
-						<0 0 0 2 &pcie3_intc 1>,
-						<0 0 0 3 &pcie3_intc 2>,
-						<0 0 0 4 &pcie3_intc 3>;
-				marvell,pcie-port = <2>;
-				marvell,pcie-lane = <0>;
-				clocks = <&gateclk 6>;
-				status = "disabled";
-
-				pcie3_intc: interrupt-controller {
-					interrupt-controller;
-					#interrupt-cells = <1>;
-				};
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/mediatek/mt7623.dtsi b/arch/arm/boot/dts/mediatek/mt7623.dtsi
index 4b1685b939891..71ac2b94c6ba3 100644
--- a/arch/arm/boot/dts/mediatek/mt7623.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt7623.dtsi
@@ -328,7 +328,7 @@ sysirq: interrupt-controller@10200100 {
 
 	efuse: efuse@10206000 {
 		compatible = "mediatek,mt7623-efuse",
-			     "mediatek,mt8173-efuse";
+			     "mediatek,efuse";
 		reg = <0 0x10206000 0 0x1000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/microchip/sam9x60.dtsi b/arch/arm/boot/dts/microchip/sam9x60.dtsi
index b075865e6a768..e708b3df4ccd3 100644
--- a/arch/arm/boot/dts/microchip/sam9x60.dtsi
+++ b/arch/arm/boot/dts/microchip/sam9x60.dtsi
@@ -75,8 +75,6 @@ ahb {
 		ranges;
 
 		usb0: gadget@500000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
 			compatible = "microchip,sam9x60-udc";
 			reg = <0x00500000 0x100000
 				0xf803c000 0x400>;
diff --git a/arch/arm/boot/dts/microchip/sama7d65.dtsi b/arch/arm/boot/dts/microchip/sama7d65.dtsi
index e21556f463843..80cfdec42ec42 100644
--- a/arch/arm/boot/dts/microchip/sama7d65.dtsi
+++ b/arch/arm/boot/dts/microchip/sama7d65.dtsi
@@ -278,6 +278,41 @@ sdmmc1: mmc@e1208000 {
 			status = "disabled";
 		};
 
+		xlcdc: lcd-controller@e1400000 {
+			compatible = "microchip,sama7d65-xlcdc";
+			reg = <0xe1400000 0x2000>;
+			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&pmc PMC_TYPE_PERIPHERAL 56>, <&pmc PMC_TYPE_GCK 56>, <&clk32k 1>;
+			clock-names = "periph_clk", "sys_clk", "slow_clk";
+			status = "disabled";
+
+			display-controller {
+				compatible = "atmel,hlcdc-display-controller";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+				};
+			};
+
+			pwm {
+				compatible = "atmel,hlcdc-pwm";
+				#pwm-cells = <3>;
+			};
+		};
+
+		lvdsc: lvds-controller@e1408000 {
+			compatible = "microchip,sama7d65-lvds", "microchip,sam9x75-lvds";
+			reg = <0xe1408000 0x100>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&pmc PMC_TYPE_PERIPHERAL 104>;
+			clock-names = "pclk";
+			status = "disabled";
+		};
+
 		aes: crypto@e1600000 {
 			compatible = "microchip,sama7d65-aes", "atmel,at91sam9g46-aes";
 			reg = <0xe1600000 0x100>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx1-ads.dts b/arch/arm/boot/dts/nxp/imx/imx1-ads.dts
index 2c817c4a4c68f..823e7c42910b8 100644
--- a/arch/arm/boot/dts/nxp/imx/imx1-ads.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx1-ads.dts
@@ -76,60 +76,58 @@ nor: flash@0,0 {
 };
 
 &iomuxc {
-	imx1-ads {
-		pinctrl_cspi1: cspi1grp {
-			fsl,pins = <
-				MX1_PAD_SPI1_MISO__SPI1_MISO	0x0
-				MX1_PAD_SPI1_MOSI__SPI1_MOSI	0x0
-				MX1_PAD_SPI1_RDY__SPI1_RDY	0x0
-				MX1_PAD_SPI1_SCLK__SPI1_SCLK	0x0
-				MX1_PAD_SPI1_SS__GPIO3_15	0x0
-			>;
-		};
-
-		pinctrl_i2c: i2cgrp {
-			fsl,pins = <
-				MX1_PAD_I2C_SCL__I2C_SCL	0x0
-				MX1_PAD_I2C_SDA__I2C_SDA	0x0
-			>;
-		};
-
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX1_PAD_UART1_TXD__UART1_TXD	0x0
-				MX1_PAD_UART1_RXD__UART1_RXD	0x0
-				MX1_PAD_UART1_CTS__UART1_CTS	0x0
-				MX1_PAD_UART1_RTS__UART1_RTS	0x0
-			>;
-		};
-
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX1_PAD_UART2_TXD__UART2_TXD	0x0
-				MX1_PAD_UART2_RXD__UART2_RXD	0x0
-				MX1_PAD_UART2_CTS__UART2_CTS	0x0
-				MX1_PAD_UART2_RTS__UART2_RTS	0x0
-			>;
-		};
-
-		pinctrl_weim: weimgrp {
-			fsl,pins = <
-				MX1_PAD_A0__A0			0x0
-				MX1_PAD_A16__A16		0x0
-				MX1_PAD_A17__A17		0x0
-				MX1_PAD_A18__A18		0x0
-				MX1_PAD_A19__A19		0x0
-				MX1_PAD_A20__A20		0x0
-				MX1_PAD_A21__A21		0x0
-				MX1_PAD_A22__A22		0x0
-				MX1_PAD_A23__A23		0x0
-				MX1_PAD_A24__A24		0x0
-				MX1_PAD_BCLK__BCLK		0x0
-				MX1_PAD_CS4__CS4		0x0
-				MX1_PAD_DTACK__DTACK		0x0
-				MX1_PAD_ECB__ECB		0x0
-				MX1_PAD_LBA__LBA		0x0
-			>;
-		};
+	pinctrl_cspi1: cspi1grp {
+		fsl,pins = <
+			MX1_PAD_SPI1_MISO__SPI1_MISO	0x0
+			MX1_PAD_SPI1_MOSI__SPI1_MOSI	0x0
+			MX1_PAD_SPI1_RDY__SPI1_RDY	0x0
+			MX1_PAD_SPI1_SCLK__SPI1_SCLK	0x0
+			MX1_PAD_SPI1_SS__GPIO3_15	0x0
+		>;
+	};
+
+	pinctrl_i2c: i2cgrp {
+		fsl,pins = <
+			MX1_PAD_I2C_SCL__I2C_SCL	0x0
+			MX1_PAD_I2C_SDA__I2C_SDA	0x0
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX1_PAD_UART1_TXD__UART1_TXD	0x0
+			MX1_PAD_UART1_RXD__UART1_RXD	0x0
+			MX1_PAD_UART1_CTS__UART1_CTS	0x0
+			MX1_PAD_UART1_RTS__UART1_RTS	0x0
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX1_PAD_UART2_TXD__UART2_TXD	0x0
+			MX1_PAD_UART2_RXD__UART2_RXD	0x0
+			MX1_PAD_UART2_CTS__UART2_CTS	0x0
+			MX1_PAD_UART2_RTS__UART2_RTS	0x0
+		>;
+	};
+
+	pinctrl_weim: weimgrp {
+		fsl,pins = <
+			MX1_PAD_A0__A0			0x0
+			MX1_PAD_A16__A16		0x0
+			MX1_PAD_A17__A17		0x0
+			MX1_PAD_A18__A18		0x0
+			MX1_PAD_A19__A19		0x0
+			MX1_PAD_A20__A20		0x0
+			MX1_PAD_A21__A21		0x0
+			MX1_PAD_A22__A22		0x0
+			MX1_PAD_A23__A23		0x0
+			MX1_PAD_A24__A24		0x0
+			MX1_PAD_BCLK__BCLK		0x0
+			MX1_PAD_CS4__CS4		0x0
+			MX1_PAD_DTACK__DTACK		0x0
+			MX1_PAD_ECB__ECB		0x0
+			MX1_PAD_LBA__LBA		0x0
+		>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx1-apf9328.dts b/arch/arm/boot/dts/nxp/imx/imx1-apf9328.dts
index 058e9435524fe..794e5bfee3670 100644
--- a/arch/arm/boot/dts/nxp/imx/imx1-apf9328.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx1-apf9328.dts
@@ -67,56 +67,54 @@ eth: ethernet@4,c00000 {
 };
 
 &iomuxc {
-	imx1-apf9328 {
-		pinctrl_eth: ethgrp {
-			fsl,pins = <
-				MX1_PAD_SIM_SVEN__GPIO2_14	0x0
-			>;
-		};
+	pinctrl_eth: ethgrp {
+		fsl,pins = <
+			MX1_PAD_SIM_SVEN__GPIO2_14	0x0
+		>;
+	};
 
-		pinctrl_i2c: i2cgrp {
-			fsl,pins = <
-				MX1_PAD_I2C_SCL__I2C_SCL	0x0
-				MX1_PAD_I2C_SDA__I2C_SDA	0x0
-			>;
-		};
+	pinctrl_i2c: i2cgrp {
+		fsl,pins = <
+			MX1_PAD_I2C_SCL__I2C_SCL	0x0
+			MX1_PAD_I2C_SDA__I2C_SDA	0x0
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX1_PAD_UART1_TXD__UART1_TXD	0x0
-				MX1_PAD_UART1_RXD__UART1_RXD	0x0
-				MX1_PAD_UART1_CTS__UART1_CTS	0x0
-				MX1_PAD_UART1_RTS__UART1_RTS	0x0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX1_PAD_UART1_TXD__UART1_TXD	0x0
+			MX1_PAD_UART1_RXD__UART1_RXD	0x0
+			MX1_PAD_UART1_CTS__UART1_CTS	0x0
+			MX1_PAD_UART1_RTS__UART1_RTS	0x0
+		>;
+	};
 
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX1_PAD_UART2_TXD__UART2_TXD	0x0
-				MX1_PAD_UART2_RXD__UART2_RXD	0x0
-				MX1_PAD_UART2_CTS__UART2_CTS	0x0
-				MX1_PAD_UART2_RTS__UART2_RTS	0x0
-			>;
-		};
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX1_PAD_UART2_TXD__UART2_TXD	0x0
+			MX1_PAD_UART2_RXD__UART2_RXD	0x0
+			MX1_PAD_UART2_CTS__UART2_CTS	0x0
+			MX1_PAD_UART2_RTS__UART2_RTS	0x0
+		>;
+	};
 
-		pinctrl_weim: weimgrp {
-			fsl,pins = <
-				MX1_PAD_A0__A0			0x0
-				MX1_PAD_A16__A16		0x0
-				MX1_PAD_A17__A17		0x0
-				MX1_PAD_A18__A18		0x0
-				MX1_PAD_A19__A19		0x0
-				MX1_PAD_A20__A20		0x0
-				MX1_PAD_A21__A21		0x0
-				MX1_PAD_A22__A22		0x0
-				MX1_PAD_A23__A23		0x0
-				MX1_PAD_A24__A24		0x0
-				MX1_PAD_BCLK__BCLK		0x0
-				MX1_PAD_CS4__CS4		0x0
-				MX1_PAD_DTACK__DTACK		0x0
-				MX1_PAD_ECB__ECB		0x0
-				MX1_PAD_LBA__LBA		0x0
-			>;
-		};
+	pinctrl_weim: weimgrp {
+		fsl,pins = <
+			MX1_PAD_A0__A0			0x0
+			MX1_PAD_A16__A16		0x0
+			MX1_PAD_A17__A17		0x0
+			MX1_PAD_A18__A18		0x0
+			MX1_PAD_A19__A19		0x0
+			MX1_PAD_A20__A20		0x0
+			MX1_PAD_A21__A21		0x0
+			MX1_PAD_A22__A22		0x0
+			MX1_PAD_A23__A23		0x0
+			MX1_PAD_A24__A24		0x0
+			MX1_PAD_BCLK__BCLK		0x0
+			MX1_PAD_CS4__CS4		0x0
+			MX1_PAD_DTACK__DTACK		0x0
+			MX1_PAD_ECB__ECB		0x0
+			MX1_PAD_LBA__LBA		0x0
+		>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx1.dtsi b/arch/arm/boot/dts/nxp/imx/imx1.dtsi
index a1a89ccacf055..ed04a907b3f79 100644
--- a/arch/arm/boot/dts/nxp/imx/imx1.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx1.dtsi
@@ -202,7 +202,7 @@ clks: ccm@21b000 {
 				#clock-cells = <1>;
 			};
 
-			iomuxc: iomuxc@21c000 {
+			iomuxc: pinmux@21c000 {
 				compatible = "fsl,imx1-iomuxc";
 				reg = <0x0021c000 0x1000>;
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-cpuimx25.dtsi b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-cpuimx25.dtsi
index 93a6e4e680b45..31dc2a6403628 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-cpuimx25.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-cpuimx25.dtsi
@@ -34,27 +34,25 @@ rtc@51 {
 };
 
 &iomuxc {
-	imx25-eukrea-cpuimx25 {
-		pinctrl_fec: fecgrp {
-			fsl,pins = <
-				MX25_PAD_FEC_MDC__FEC_MDC		0x80000000
-				MX25_PAD_FEC_MDIO__FEC_MDIO		0x400001e0
-				MX25_PAD_FEC_TDATA0__FEC_TDATA0		0x80000000
-				MX25_PAD_FEC_TDATA1__FEC_TDATA1		0x80000000
-				MX25_PAD_FEC_TX_EN__FEC_TX_EN		0x80000000
-				MX25_PAD_FEC_RDATA0__FEC_RDATA0		0x80000000
-				MX25_PAD_FEC_RDATA1__FEC_RDATA1		0x80000000
-				MX25_PAD_FEC_RX_DV__FEC_RX_DV		0x80000000
-				MX25_PAD_FEC_TX_CLK__FEC_TX_CLK		0x1c0
-			>;
-		};
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX25_PAD_FEC_MDC__FEC_MDC		0x80000000
+			MX25_PAD_FEC_MDIO__FEC_MDIO		0x400001e0
+			MX25_PAD_FEC_TDATA0__FEC_TDATA0		0x80000000
+			MX25_PAD_FEC_TDATA1__FEC_TDATA1		0x80000000
+			MX25_PAD_FEC_TX_EN__FEC_TX_EN		0x80000000
+			MX25_PAD_FEC_RDATA0__FEC_RDATA0		0x80000000
+			MX25_PAD_FEC_RDATA1__FEC_RDATA1		0x80000000
+			MX25_PAD_FEC_RX_DV__FEC_RX_DV		0x80000000
+			MX25_PAD_FEC_TX_CLK__FEC_TX_CLK		0x1c0
+		>;
+	};
 
-		pinctrl_i2c1: i2c1grp {
-			fsl,pins = <
-				MX25_PAD_I2C1_CLK__I2C1_CLK		0x80000000
-				MX25_PAD_I2C1_DAT__I2C1_DAT		0x80000000
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX25_PAD_I2C1_CLK__I2C1_CLK		0x80000000
+			MX25_PAD_I2C1_DAT__I2C1_DAT		0x80000000
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard-cmo-qvga.dts b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard-cmo-qvga.dts
index 6cddb2cc36fe2..e08fcbfef4d5d 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard-cmo-qvga.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard-cmo-qvga.dts
@@ -43,10 +43,8 @@ reg_lcd_3v3: regulator-0 {
 };
 
 &iomuxc {
-	imx25-eukrea-mbimxsd25-baseboard-cmo-qvga {
-		pinctrl_reg_lcd_3v3: reg_lcd_3v3 {
-			fsl,pins = <MX25_PAD_PWM__GPIO_1_26 0x80000000>;
-		};
+	pinctrl_reg_lcd_3v3: reg_lcd_3v3grp {
+		fsl,pins = <MX25_PAD_PWM__GPIO_1_26 0x80000000>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard.dts b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard.dts
index c7207ea437c40..cf127e00793ef 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx25-eukrea-mbimxsd25-baseboard.dts
@@ -68,80 +68,78 @@ tlv320aic23: codec@1a {
 };
 
 &iomuxc {
-	imx25-eukrea-mbimxsd25-baseboard {
-		pinctrl_audmux: audmuxgrp {
-			fsl,pins = <
-				MX25_PAD_KPP_COL3__AUD5_TXFS		0xe0
-				MX25_PAD_KPP_COL2__AUD5_TXC		0xe0
-				MX25_PAD_KPP_COL1__AUD5_RXD		0xe0
-				MX25_PAD_KPP_COL0__AUD5_TXD		0xe0
-			>;
-		};
+	pinctrl_audmux: audmuxgrp {
+		fsl,pins = <
+			MX25_PAD_KPP_COL3__AUD5_TXFS		0xe0
+			MX25_PAD_KPP_COL2__AUD5_TXC		0xe0
+			MX25_PAD_KPP_COL1__AUD5_RXD		0xe0
+			MX25_PAD_KPP_COL0__AUD5_TXD		0xe0
+		>;
+	};
 
-		pinctrl_esdhc1: esdhc1grp {
-			fsl,pins = <
-				MX25_PAD_SD1_CMD__ESDHC1_CMD		0x400000c0
-				MX25_PAD_SD1_CLK__ESDHC1_CLK		0x400000c0
-				MX25_PAD_SD1_DATA0__ESDHC1_DAT0		0x400000c0
-				MX25_PAD_SD1_DATA1__ESDHC1_DAT1		0x400000c0
-				MX25_PAD_SD1_DATA2__ESDHC1_DAT2		0x400000c0
-				MX25_PAD_SD1_DATA3__ESDHC1_DAT3		0x400000c0
-			>;
-		};
+	pinctrl_esdhc1: esdhc1grp {
+		fsl,pins = <
+			MX25_PAD_SD1_CMD__ESDHC1_CMD		0x400000c0
+			MX25_PAD_SD1_CLK__ESDHC1_CLK		0x400000c0
+			MX25_PAD_SD1_DATA0__ESDHC1_DAT0		0x400000c0
+			MX25_PAD_SD1_DATA1__ESDHC1_DAT1		0x400000c0
+			MX25_PAD_SD1_DATA2__ESDHC1_DAT2		0x400000c0
+			MX25_PAD_SD1_DATA3__ESDHC1_DAT3		0x400000c0
+		>;
+	};
 
-		pinctrl_gpiokeys: gpiokeysgrp {
-			fsl,pins = <MX25_PAD_VSTBY_ACK__GPIO_3_18 0x80000000>;
-		};
+	pinctrl_gpiokeys: gpiokeysgrp {
+		fsl,pins = <MX25_PAD_VSTBY_ACK__GPIO_3_18 0x80000000>;
+	};
 
-		pinctrl_gpioled: gpioledgrp {
-			fsl,pins = <MX25_PAD_POWER_FAIL__GPIO_3_19 0x80000000>;
-		};
+	pinctrl_gpioled: gpioledgrp {
+		fsl,pins = <MX25_PAD_POWER_FAIL__GPIO_3_19 0x80000000>;
+	};
 
-		pinctrl_lcdc: lcdcgrp {
-			fsl,pins = <
-				MX25_PAD_LD0__LD0			0x1
-				MX25_PAD_LD1__LD1			0x1
-				MX25_PAD_LD2__LD2			0x1
-				MX25_PAD_LD3__LD3			0x1
-				MX25_PAD_LD4__LD4			0x1
-				MX25_PAD_LD5__LD5			0x1
-				MX25_PAD_LD6__LD6			0x1
-				MX25_PAD_LD7__LD7			0x1
-				MX25_PAD_LD8__LD8			0x1
-				MX25_PAD_LD9__LD9			0x1
-				MX25_PAD_LD10__LD10			0x1
-				MX25_PAD_LD11__LD11			0x1
-				MX25_PAD_LD12__LD12			0x1
-				MX25_PAD_LD13__LD13			0x1
-				MX25_PAD_LD14__LD14			0x1
-				MX25_PAD_LD15__LD15			0x1
-				MX25_PAD_GPIO_E__LD16			0x1
-				MX25_PAD_GPIO_F__LD17			0x1
-				MX25_PAD_HSYNC__HSYNC			0x80000000
-				MX25_PAD_VSYNC__VSYNC			0x80000000
-				MX25_PAD_LSCLK__LSCLK			0x80000000
-				MX25_PAD_OE_ACD__OE_ACD			0x80000000
-				MX25_PAD_CONTRAST__CONTRAST		0x80000000
-			>;
-		};
+	pinctrl_lcdc: lcdcgrp {
+		fsl,pins = <
+			MX25_PAD_LD0__LD0			0x1
+			MX25_PAD_LD1__LD1			0x1
+			MX25_PAD_LD2__LD2			0x1
+			MX25_PAD_LD3__LD3			0x1
+			MX25_PAD_LD4__LD4			0x1
+			MX25_PAD_LD5__LD5			0x1
+			MX25_PAD_LD6__LD6			0x1
+			MX25_PAD_LD7__LD7			0x1
+			MX25_PAD_LD8__LD8			0x1
+			MX25_PAD_LD9__LD9			0x1
+			MX25_PAD_LD10__LD10			0x1
+			MX25_PAD_LD11__LD11			0x1
+			MX25_PAD_LD12__LD12			0x1
+			MX25_PAD_LD13__LD13			0x1
+			MX25_PAD_LD14__LD14			0x1
+			MX25_PAD_LD15__LD15			0x1
+			MX25_PAD_GPIO_E__LD16			0x1
+			MX25_PAD_GPIO_F__LD17			0x1
+			MX25_PAD_HSYNC__HSYNC			0x80000000
+			MX25_PAD_VSYNC__VSYNC			0x80000000
+			MX25_PAD_LSCLK__LSCLK			0x80000000
+			MX25_PAD_OE_ACD__OE_ACD			0x80000000
+			MX25_PAD_CONTRAST__CONTRAST		0x80000000
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX25_PAD_UART1_RTS__UART1_RTS		0xe0
-				MX25_PAD_UART1_CTS__UART1_CTS		0xe0
-				MX25_PAD_UART1_TXD__UART1_TXD		0x80000000
-				MX25_PAD_UART1_RXD__UART1_RXD		0xc0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX25_PAD_UART1_RTS__UART1_RTS		0xe0
+			MX25_PAD_UART1_CTS__UART1_CTS		0xe0
+			MX25_PAD_UART1_TXD__UART1_TXD		0x80000000
+			MX25_PAD_UART1_RXD__UART1_RXD		0xc0
+		>;
+	};
 
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX25_PAD_UART2_RXD__UART2_RXD		0x80000000
-				MX25_PAD_UART2_TXD__UART2_TXD		0x80000000
-				MX25_PAD_UART2_RTS__UART2_RTS		0x80000000
-				MX25_PAD_UART2_CTS__UART2_CTS		0x80000000
-			>;
-		};
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX25_PAD_UART2_RXD__UART2_RXD		0x80000000
+			MX25_PAD_UART2_TXD__UART2_TXD		0x80000000
+			MX25_PAD_UART2_RTS__UART2_RTS		0x80000000
+			MX25_PAD_UART2_CTS__UART2_CTS		0x80000000
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx25-pdk.dts b/arch/arm/boot/dts/nxp/imx/imx25-pdk.dts
index dd176fb54e585..a35778ba6ffa0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25-pdk.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx25-pdk.dts
@@ -130,109 +130,107 @@ codec: sgtl5000@a {
 };
 
 &iomuxc {
-	imx25-pdk {
-		pinctrl_audmux: audmuxgrp {
-			fsl,pins = <
-				MX25_PAD_RW__AUD4_TXFS			0xe0
-				MX25_PAD_OE__AUD4_TXC			0xe0
-				MX25_PAD_EB0__AUD4_TXD			0xe0
-				MX25_PAD_EB1__AUD4_RXD			0xe0
-			>;
-		};
+	pinctrl_audmux: audmuxgrp {
+		fsl,pins = <
+			MX25_PAD_RW__AUD4_TXFS			0xe0
+			MX25_PAD_OE__AUD4_TXC			0xe0
+			MX25_PAD_EB0__AUD4_TXD			0xe0
+			MX25_PAD_EB1__AUD4_RXD			0xe0
+		>;
+	};
 
-		pinctrl_can1: can1grp {
-			fsl,pins = <
-				MX25_PAD_GPIO_A__CAN1_TX		0x0
-				MX25_PAD_GPIO_B__CAN1_RX		0x0
-				MX25_PAD_D14__GPIO_4_6 			0x80000000
-			>;
-		};
+	pinctrl_can1: can1grp {
+		fsl,pins = <
+			MX25_PAD_GPIO_A__CAN1_TX		0x0
+			MX25_PAD_GPIO_B__CAN1_RX		0x0
+			MX25_PAD_D14__GPIO_4_6			0x80000000
+		>;
+	};
 
-		pinctrl_esdhc1: esdhc1grp {
-			fsl,pins = <
-				MX25_PAD_SD1_CMD__ESDHC1_CMD		0x80000000
-				MX25_PAD_SD1_CLK__ESDHC1_CLK		0x80000000
-				MX25_PAD_SD1_DATA0__ESDHC1_DAT0		0x80000000
-				MX25_PAD_SD1_DATA1__ESDHC1_DAT1		0x80000000
-				MX25_PAD_SD1_DATA2__ESDHC1_DAT2		0x80000000
-				MX25_PAD_SD1_DATA3__ESDHC1_DAT3		0x80000000
-				MX25_PAD_A14__GPIO_2_0			0x80000000
-				MX25_PAD_A15__GPIO_2_1			0x80000000
-			>;
-		};
+	pinctrl_esdhc1: esdhc1grp {
+		fsl,pins = <
+			MX25_PAD_SD1_CMD__ESDHC1_CMD		0x80000000
+			MX25_PAD_SD1_CLK__ESDHC1_CLK		0x80000000
+			MX25_PAD_SD1_DATA0__ESDHC1_DAT0		0x80000000
+			MX25_PAD_SD1_DATA1__ESDHC1_DAT1		0x80000000
+			MX25_PAD_SD1_DATA2__ESDHC1_DAT2		0x80000000
+			MX25_PAD_SD1_DATA3__ESDHC1_DAT3		0x80000000
+			MX25_PAD_A14__GPIO_2_0			0x80000000
+			MX25_PAD_A15__GPIO_2_1			0x80000000
+		>;
+	};
 
-		pinctrl_fec: fecgrp {
-			fsl,pins = <
-				MX25_PAD_FEC_MDC__FEC_MDC		0x80000000
-				MX25_PAD_FEC_MDIO__FEC_MDIO		0x400001e0
-				MX25_PAD_FEC_TDATA0__FEC_TDATA0		0x80000000
-				MX25_PAD_FEC_TDATA1__FEC_TDATA1		0x80000000
-				MX25_PAD_FEC_TX_EN__FEC_TX_EN		0x80000000
-				MX25_PAD_FEC_RDATA0__FEC_RDATA0		0x80000000
-				MX25_PAD_FEC_RDATA1__FEC_RDATA1		0x80000000
-				MX25_PAD_FEC_RX_DV__FEC_RX_DV		0x80000000
-				MX25_PAD_FEC_TX_CLK__FEC_TX_CLK		0x1c0
-				MX25_PAD_A17__GPIO_2_3			0x80000000
-				MX25_PAD_D12__GPIO_4_8			0x80000000
-			>;
-		};
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX25_PAD_FEC_MDC__FEC_MDC		0x80000000
+			MX25_PAD_FEC_MDIO__FEC_MDIO		0x400001e0
+			MX25_PAD_FEC_TDATA0__FEC_TDATA0		0x80000000
+			MX25_PAD_FEC_TDATA1__FEC_TDATA1		0x80000000
+			MX25_PAD_FEC_TX_EN__FEC_TX_EN		0x80000000
+			MX25_PAD_FEC_RDATA0__FEC_RDATA0		0x80000000
+			MX25_PAD_FEC_RDATA1__FEC_RDATA1		0x80000000
+			MX25_PAD_FEC_RX_DV__FEC_RX_DV		0x80000000
+			MX25_PAD_FEC_TX_CLK__FEC_TX_CLK		0x1c0
+			MX25_PAD_A17__GPIO_2_3			0x80000000
+			MX25_PAD_D12__GPIO_4_8			0x80000000
+		>;
+	};
 
-		pinctrl_i2c1: i2c1grp {
-			fsl,pins = <
-				MX25_PAD_I2C1_CLK__I2C1_CLK		0x80000000
-				MX25_PAD_I2C1_DAT__I2C1_DAT		0x80000000
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX25_PAD_I2C1_CLK__I2C1_CLK		0x80000000
+			MX25_PAD_I2C1_DAT__I2C1_DAT		0x80000000
+		>;
+	};
 
-		pinctrl_kpp: kppgrp {
-			fsl,pins = <
-				MX25_PAD_KPP_ROW0__KPP_ROW0	0x80000000
-				MX25_PAD_KPP_ROW1__KPP_ROW1	0x80000000
-				MX25_PAD_KPP_ROW2__KPP_ROW2	0x80000000
-				MX25_PAD_KPP_ROW3__KPP_ROW3	0x80000000
-				MX25_PAD_KPP_COL0__KPP_COL0	0x80000000
-				MX25_PAD_KPP_COL1__KPP_COL1	0x80000000
-				MX25_PAD_KPP_COL2__KPP_COL2	0x80000000
-				MX25_PAD_KPP_COL3__KPP_COL3	0x80000000
-			>;
-		};
+	pinctrl_kpp: kppgrp {
+		fsl,pins = <
+			MX25_PAD_KPP_ROW0__KPP_ROW0	0x80000000
+			MX25_PAD_KPP_ROW1__KPP_ROW1	0x80000000
+			MX25_PAD_KPP_ROW2__KPP_ROW2	0x80000000
+			MX25_PAD_KPP_ROW3__KPP_ROW3	0x80000000
+			MX25_PAD_KPP_COL0__KPP_COL0	0x80000000
+			MX25_PAD_KPP_COL1__KPP_COL1	0x80000000
+			MX25_PAD_KPP_COL2__KPP_COL2	0x80000000
+			MX25_PAD_KPP_COL3__KPP_COL3	0x80000000
+		>;
+	};
 
-		pinctrl_lcd: lcdgrp {
-			fsl,pins = <
-				MX25_PAD_LD0__LD0		0xe0
-				MX25_PAD_LD1__LD1		0xe0
-				MX25_PAD_LD2__LD2		0xe0
-				MX25_PAD_LD3__LD3		0xe0
-				MX25_PAD_LD4__LD4		0xe0
-				MX25_PAD_LD5__LD5		0xe0
-				MX25_PAD_LD6__LD6		0xe0
-				MX25_PAD_LD7__LD7		0xe0
-				MX25_PAD_LD8__LD8		0xe0
-				MX25_PAD_LD9__LD9		0xe0
-				MX25_PAD_LD10__LD10		0xe0
-				MX25_PAD_LD11__LD11		0xe0
-				MX25_PAD_LD12__LD12		0xe0
-				MX25_PAD_LD13__LD13		0xe0
-				MX25_PAD_LD14__LD14		0xe0
-				MX25_PAD_LD15__LD15		0xe0
-				MX25_PAD_GPIO_E__LD16		0xe0
-				MX25_PAD_GPIO_F__LD17		0xe0
-				MX25_PAD_HSYNC__HSYNC		0xe0
-				MX25_PAD_VSYNC__VSYNC		0xe0
-				MX25_PAD_LSCLK__LSCLK		0xe0
-				MX25_PAD_OE_ACD__OE_ACD		0xe0
-				MX25_PAD_CONTRAST__CONTRAST	0xe0
-			>;
-		};
+	pinctrl_lcd: lcdgrp {
+		fsl,pins = <
+			MX25_PAD_LD0__LD0		0xe0
+			MX25_PAD_LD1__LD1		0xe0
+			MX25_PAD_LD2__LD2		0xe0
+			MX25_PAD_LD3__LD3		0xe0
+			MX25_PAD_LD4__LD4		0xe0
+			MX25_PAD_LD5__LD5		0xe0
+			MX25_PAD_LD6__LD6		0xe0
+			MX25_PAD_LD7__LD7		0xe0
+			MX25_PAD_LD8__LD8		0xe0
+			MX25_PAD_LD9__LD9		0xe0
+			MX25_PAD_LD10__LD10		0xe0
+			MX25_PAD_LD11__LD11		0xe0
+			MX25_PAD_LD12__LD12		0xe0
+			MX25_PAD_LD13__LD13		0xe0
+			MX25_PAD_LD14__LD14		0xe0
+			MX25_PAD_LD15__LD15		0xe0
+			MX25_PAD_GPIO_E__LD16		0xe0
+			MX25_PAD_GPIO_F__LD17		0xe0
+			MX25_PAD_HSYNC__HSYNC		0xe0
+			MX25_PAD_VSYNC__VSYNC		0xe0
+			MX25_PAD_LSCLK__LSCLK		0xe0
+			MX25_PAD_OE_ACD__OE_ACD		0xe0
+			MX25_PAD_CONTRAST__CONTRAST	0xe0
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX25_PAD_UART1_RTS__UART1_RTS		0xe0
-				MX25_PAD_UART1_CTS__UART1_CTS		0xe0
-				MX25_PAD_UART1_TXD__UART1_TXD		0x80000000
-				MX25_PAD_UART1_RXD__UART1_RXD		0xc0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX25_PAD_UART1_RTS__UART1_RTS		0xe0
+			MX25_PAD_UART1_CTS__UART1_CTS		0xe0
+			MX25_PAD_UART1_TXD__UART1_TXD		0x80000000
+			MX25_PAD_UART1_RXD__UART1_RXD		0xc0
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx25.dtsi b/arch/arm/boot/dts/nxp/imx/imx25.dtsi
index 82601a4b7b4b5..54a116533c458 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx25.dtsi
@@ -195,7 +195,7 @@ kpp: kpp@43fa8000 {
 				status = "disabled";
 			};
 
-			iomuxc: iomuxc@43fac000 {
+			iomuxc: pinmux@43fac000 {
 				compatible = "fsl,imx25-iomuxc";
 				reg = <0x43fac000 0x4000>;
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-apf27.dts b/arch/arm/boot/dts/nxp/imx/imx27-apf27.dts
index 745d5d4099525..b67bb21af3ded 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-apf27.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-apf27.dts
@@ -24,36 +24,34 @@ &clk_osc26m {
 };
 
 &iomuxc {
-	imx27-apf27 {
-		pinctrl_fec1: fec1grp {
-			fsl,pins = <
-				MX27_PAD_SD3_CMD__FEC_TXD0 0x0
-				MX27_PAD_SD3_CLK__FEC_TXD1 0x0
-				MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
-				MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
-				MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
-				MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
-				MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
-				MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
-				MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
-				MX27_PAD_ATA_DATA7__FEC_MDC 0x0
-				MX27_PAD_ATA_DATA8__FEC_CRS 0x0
-				MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
-				MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
-				MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
-				MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
-				MX27_PAD_ATA_DATA13__FEC_COL 0x0
-				MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
-				MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
-			>;
-		};
-
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD 0x0
-				MX27_PAD_UART1_RXD__UART1_RXD 0x0
-			>;
-		};
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX27_PAD_SD3_CMD__FEC_TXD0 0x0
+			MX27_PAD_SD3_CLK__FEC_TXD1 0x0
+			MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
+			MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
+			MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
+			MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
+			MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
+			MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
+			MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
+			MX27_PAD_ATA_DATA7__FEC_MDC 0x0
+			MX27_PAD_ATA_DATA8__FEC_CRS 0x0
+			MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
+			MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
+			MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
+			MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
+			MX27_PAD_ATA_DATA13__FEC_COL 0x0
+			MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
+			MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX27_PAD_UART1_TXD__UART1_TXD 0x0
+			MX27_PAD_UART1_RXD__UART1_RXD 0x0
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-apf27dev.dts b/arch/arm/boot/dts/nxp/imx/imx27-apf27dev.dts
index 849306cb4532d..dba97912cfd75 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-apf27dev.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-apf27dev.dts
@@ -122,116 +122,114 @@ &i2c2 {
 };
 
 &iomuxc {
-	imx27-apf27dev {
-		pinctrl_cspi1: cspi1grp {
-			fsl,pins = <
-				MX27_PAD_CSPI1_MISO__CSPI1_MISO 0x0
-				MX27_PAD_CSPI1_MOSI__CSPI1_MOSI 0x0
-				MX27_PAD_CSPI1_SCLK__CSPI1_SCLK 0x0
-			>;
-		};
+	pinctrl_cspi1: cspi1grp {
+		fsl,pins = <
+			MX27_PAD_CSPI1_MISO__CSPI1_MISO 0x0
+			MX27_PAD_CSPI1_MOSI__CSPI1_MOSI 0x0
+			MX27_PAD_CSPI1_SCLK__CSPI1_SCLK 0x0
+		>;
+	};
 
-		pinctrl_cspi1_cs: cspi1csgrp {
-			fsl,pins = <MX27_PAD_CSPI1_SS0__GPIO4_28 0x0>;
-		};
+	pinctrl_cspi1_cs: cspi1csgrp {
+		fsl,pins = <MX27_PAD_CSPI1_SS0__GPIO4_28 0x0>;
+	};
 
-		pinctrl_cspi2: cspi2grp {
-			fsl,pins = <
-				MX27_PAD_CSPI2_MISO__CSPI2_MISO 0x0
-				MX27_PAD_CSPI2_MOSI__CSPI2_MOSI 0x0
-				MX27_PAD_CSPI2_SCLK__CSPI2_SCLK 0x0
-			>;
-		};
+	pinctrl_cspi2: cspi2grp {
+		fsl,pins = <
+			MX27_PAD_CSPI2_MISO__CSPI2_MISO 0x0
+			MX27_PAD_CSPI2_MOSI__CSPI2_MOSI 0x0
+			MX27_PAD_CSPI2_SCLK__CSPI2_SCLK 0x0
+		>;
+	};
 
-		pinctrl_cspi2_cs: cspi2csgrp {
-			fsl,pins = <
-				MX27_PAD_CSI_D5__GPIO2_17 0x0
-				MX27_PAD_CSPI2_SS0__GPIO4_21 0x0
-				MX27_PAD_CSPI1_SS1__GPIO4_27 0x0
-			>;
-		};
+	pinctrl_cspi2_cs: cspi2csgrp {
+		fsl,pins = <
+			MX27_PAD_CSI_D5__GPIO2_17 0x0
+			MX27_PAD_CSPI2_SS0__GPIO4_21 0x0
+			MX27_PAD_CSPI1_SS1__GPIO4_27 0x0
+		>;
+	};
 
-		pinctrl_gpio_leds: gpioledsgrp {
-			fsl,pins = <MX27_PAD_PC_VS1__GPIO6_14 0x0>;
-		};
+	pinctrl_gpio_leds: gpioledsgrp {
+		fsl,pins = <MX27_PAD_PC_VS1__GPIO6_14 0x0>;
+	};
 
-		pinctrl_gpio_keys: gpiokeysgrp {
-			fsl,pins = <MX27_PAD_PC_VS2__GPIO6_13 0x0>;
-		};
+	pinctrl_gpio_keys: gpiokeysgrp {
+		fsl,pins = <MX27_PAD_PC_VS2__GPIO6_13 0x0>;
+	};
 
-		pinctrl_imxfb1: imxfbgrp {
-			fsl,pins = <
-				MX27_PAD_CLS__CLS 0x0
-				MX27_PAD_CONTRAST__CONTRAST 0x0
-				MX27_PAD_LD0__LD0 0x0
-				MX27_PAD_LD1__LD1 0x0
-				MX27_PAD_LD2__LD2 0x0
-				MX27_PAD_LD3__LD3 0x0
-				MX27_PAD_LD4__LD4 0x0
-				MX27_PAD_LD5__LD5 0x0
-				MX27_PAD_LD6__LD6 0x0
-				MX27_PAD_LD7__LD7 0x0
-				MX27_PAD_LD8__LD8 0x0
-				MX27_PAD_LD9__LD9 0x0
-				MX27_PAD_LD10__LD10 0x0
-				MX27_PAD_LD11__LD11 0x0
-				MX27_PAD_LD12__LD12 0x0
-				MX27_PAD_LD13__LD13 0x0
-				MX27_PAD_LD14__LD14 0x0
-				MX27_PAD_LD15__LD15 0x0
-				MX27_PAD_LD16__LD16 0x0
-				MX27_PAD_LD17__LD17 0x0
-				MX27_PAD_LSCLK__LSCLK 0x0
-				MX27_PAD_OE_ACD__OE_ACD 0x0
-				MX27_PAD_PS__PS 0x0
-				MX27_PAD_REV__REV 0x0
-				MX27_PAD_SPL_SPR__SPL_SPR 0x0
-				MX27_PAD_HSYNC__HSYNC 0x0
-				MX27_PAD_VSYNC__VSYNC 0x0
-			>;
-		};
+	pinctrl_imxfb1: imxfbgrp {
+		fsl,pins = <
+			MX27_PAD_CLS__CLS 0x0
+			MX27_PAD_CONTRAST__CONTRAST 0x0
+			MX27_PAD_LD0__LD0 0x0
+			MX27_PAD_LD1__LD1 0x0
+			MX27_PAD_LD2__LD2 0x0
+			MX27_PAD_LD3__LD3 0x0
+			MX27_PAD_LD4__LD4 0x0
+			MX27_PAD_LD5__LD5 0x0
+			MX27_PAD_LD6__LD6 0x0
+			MX27_PAD_LD7__LD7 0x0
+			MX27_PAD_LD8__LD8 0x0
+			MX27_PAD_LD9__LD9 0x0
+			MX27_PAD_LD10__LD10 0x0
+			MX27_PAD_LD11__LD11 0x0
+			MX27_PAD_LD12__LD12 0x0
+			MX27_PAD_LD13__LD13 0x0
+			MX27_PAD_LD14__LD14 0x0
+			MX27_PAD_LD15__LD15 0x0
+			MX27_PAD_LD16__LD16 0x0
+			MX27_PAD_LD17__LD17 0x0
+			MX27_PAD_LSCLK__LSCLK 0x0
+			MX27_PAD_OE_ACD__OE_ACD 0x0
+			MX27_PAD_PS__PS 0x0
+			MX27_PAD_REV__REV 0x0
+			MX27_PAD_SPL_SPR__SPL_SPR 0x0
+			MX27_PAD_HSYNC__HSYNC 0x0
+			MX27_PAD_VSYNC__VSYNC 0x0
+		>;
+	};
 
-		pinctrl_i2c1: i2c1grp {
-			fsl,pins = <
-				MX27_PAD_I2C_DATA__I2C_DATA 0x0
-				MX27_PAD_I2C_CLK__I2C_CLK 0x0
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX27_PAD_I2C_DATA__I2C_DATA 0x0
+			MX27_PAD_I2C_CLK__I2C_CLK 0x0
+		>;
+	};
 
-		pinctrl_i2c2: i2c2grp {
-			fsl,pins = <
-				MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
-				MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
-			>;
-		};
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
+			MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
+		>;
+	};
 
-		pinctrl_max1027: max1027 {
-			 fsl,pins = <
-				 MX27_PAD_UART1_CTS__GPIO5_14 0x0 /* CNVST */
-				 MX27_PAD_UART1_RTS__GPIO5_15 0x0 /* EOC */
-			>;
-		};
+	pinctrl_max1027: max1027grp {
+		 fsl,pins = <
+			 MX27_PAD_UART1_CTS__GPIO5_14 0x0 /* CNVST */
+			 MX27_PAD_UART1_RTS__GPIO5_15 0x0 /* EOC */
+		>;
+	};
 
-		pinctrl_pwm: pwmgrp {
-			fsl,pins = <
-				MX27_PAD_PWMO__PWMO 0x0
-			>;
-		};
+	pinctrl_pwm: pwmgrp {
+		fsl,pins = <
+			MX27_PAD_PWMO__PWMO 0x0
+		>;
+	};
 
-		pinctrl_sdhc2: sdhc2grp {
-			fsl,pins = <
-				MX27_PAD_SD2_CLK__SD2_CLK 0x0
-				MX27_PAD_SD2_CMD__SD2_CMD 0x0
-				MX27_PAD_SD2_D0__SD2_D0 0x0
-				MX27_PAD_SD2_D1__SD2_D1 0x0
-				MX27_PAD_SD2_D2__SD2_D2 0x0
-				MX27_PAD_SD2_D3__SD2_D3 0x0
-			>;
-		};
+	pinctrl_sdhc2: sdhc2grp {
+		fsl,pins = <
+			MX27_PAD_SD2_CLK__SD2_CLK 0x0
+			MX27_PAD_SD2_CMD__SD2_CMD 0x0
+			MX27_PAD_SD2_D0__SD2_D0 0x0
+			MX27_PAD_SD2_D1__SD2_D1 0x0
+			MX27_PAD_SD2_D2__SD2_D2 0x0
+			MX27_PAD_SD2_D3__SD2_D3 0x0
+		>;
+	};
 
-		pinctrl_sdhc2_cd: sdhc2cdgrp {
-			fsl,pins = <MX27_PAD_TOUT__GPIO3_14 0x0>;
-		};
+	pinctrl_sdhc2_cd: sdhc2cdgrp {
+		fsl,pins = <MX27_PAD_TOUT__GPIO3_14 0x0>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi
index c7e9235848782..46acd0dfc5898 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi
@@ -100,52 +100,52 @@ nor: flash@0,0 {
 		fsl,weim-cs-timing = <0x00008f03 0xa0330d01 0x002208c0>;
 	};
 
-	uart8250@3,200000 {
+	serial@3,200000 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_uart8250_1>;
 		compatible = "ns8250";
 		clocks = <&clk14745600>;
 		fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>;
-		interrupts = <&gpio2 23 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&gpio2 23 IRQ_TYPE_LEVEL_LOW>;
 		reg = <3 0x200000 0x1000>;
 		reg-shift = <1>;
 		reg-io-width = <1>;
 		no-loopback-test;
 	};
 
-	uart8250@3,400000 {
+	serial@3,400000 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_uart8250_2>;
 		compatible = "ns8250";
 		clocks = <&clk14745600>;
 		fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>;
-		interrupts = <&gpio2 22 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&gpio2 22 IRQ_TYPE_LEVEL_LOW>;
 		reg = <3 0x400000 0x1000>;
 		reg-shift = <1>;
 		reg-io-width = <1>;
 		no-loopback-test;
 	};
 
-	uart8250@3,800000 {
+	serial@3,800000 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_uart8250_3>;
 		compatible = "ns8250";
 		clocks = <&clk14745600>;
 		fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>;
-		interrupts = <&gpio2 27 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&gpio2 27 IRQ_TYPE_LEVEL_LOW>;
 		reg = <3 0x800000 0x1000>;
 		reg-shift = <1>;
 		reg-io-width = <1>;
 		no-loopback-test;
 	};
 
-	uart8250@3,1000000 {
+	serial@3,1000000 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_uart8250_4>;
 		compatible = "ns8250";
 		clocks = <&clk14745600>;
 		fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>;
-		interrupts = <&gpio2 30 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&gpio2 30 IRQ_TYPE_LEVEL_LOW>;
 		reg = <3 0x1000000 0x1000>;
 		reg-shift = <1>;
 		reg-io-width = <1>;
@@ -154,131 +154,129 @@ uart8250@3,1000000 {
 };
 
 &iomuxc {
-	imx27-eukrea-cpuimx27 {
-		pinctrl_fec: fecgrp {
-			fsl,pins = <
-				MX27_PAD_SD3_CMD__FEC_TXD0		0x0
-				MX27_PAD_SD3_CLK__FEC_TXD1		0x0
-				MX27_PAD_ATA_DATA0__FEC_TXD2		0x0
-				MX27_PAD_ATA_DATA1__FEC_TXD3		0x0
-				MX27_PAD_ATA_DATA2__FEC_RX_ER		0x0
-				MX27_PAD_ATA_DATA3__FEC_RXD1		0x0
-				MX27_PAD_ATA_DATA4__FEC_RXD2		0x0
-				MX27_PAD_ATA_DATA5__FEC_RXD3		0x0
-				MX27_PAD_ATA_DATA6__FEC_MDIO		0x0
-				MX27_PAD_ATA_DATA7__FEC_MDC		0x0
-				MX27_PAD_ATA_DATA8__FEC_CRS		0x0
-				MX27_PAD_ATA_DATA9__FEC_TX_CLK		0x0
-				MX27_PAD_ATA_DATA10__FEC_RXD0		0x0
-				MX27_PAD_ATA_DATA11__FEC_RX_DV		0x0
-				MX27_PAD_ATA_DATA12__FEC_RX_CLK		0x0
-				MX27_PAD_ATA_DATA13__FEC_COL		0x0
-				MX27_PAD_ATA_DATA14__FEC_TX_ER		0x0
-				MX27_PAD_ATA_DATA15__FEC_TX_EN		0x0
-			>;
-		};
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX27_PAD_SD3_CMD__FEC_TXD0		0x0
+			MX27_PAD_SD3_CLK__FEC_TXD1		0x0
+			MX27_PAD_ATA_DATA0__FEC_TXD2		0x0
+			MX27_PAD_ATA_DATA1__FEC_TXD3		0x0
+			MX27_PAD_ATA_DATA2__FEC_RX_ER		0x0
+			MX27_PAD_ATA_DATA3__FEC_RXD1		0x0
+			MX27_PAD_ATA_DATA4__FEC_RXD2		0x0
+			MX27_PAD_ATA_DATA5__FEC_RXD3		0x0
+			MX27_PAD_ATA_DATA6__FEC_MDIO		0x0
+			MX27_PAD_ATA_DATA7__FEC_MDC		0x0
+			MX27_PAD_ATA_DATA8__FEC_CRS		0x0
+			MX27_PAD_ATA_DATA9__FEC_TX_CLK		0x0
+			MX27_PAD_ATA_DATA10__FEC_RXD0		0x0
+			MX27_PAD_ATA_DATA11__FEC_RX_DV		0x0
+			MX27_PAD_ATA_DATA12__FEC_RX_CLK		0x0
+			MX27_PAD_ATA_DATA13__FEC_COL		0x0
+			MX27_PAD_ATA_DATA14__FEC_TX_ER		0x0
+			MX27_PAD_ATA_DATA15__FEC_TX_EN		0x0
+		>;
+	};
 
-		pinctrl_i2c1: i2c1grp {
-			fsl,pins = <
-				MX27_PAD_I2C_DATA__I2C_DATA		0x0
-				MX27_PAD_I2C_CLK__I2C_CLK		0x0
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX27_PAD_I2C_DATA__I2C_DATA		0x0
+			MX27_PAD_I2C_CLK__I2C_CLK		0x0
+		>;
+	};
 
-		pinctrl_nfc: nfcgrp {
-			fsl,pins = <
-				MX27_PAD_NFRB__NFRB			0x0
-				MX27_PAD_NFCLE__NFCLE			0x0
-				MX27_PAD_NFWP_B__NFWP_B			0x0
-				MX27_PAD_NFCE_B__NFCE_B			0x0
-				MX27_PAD_NFALE__NFALE			0x0
-				MX27_PAD_NFRE_B__NFRE_B			0x0
-				MX27_PAD_NFWE_B__NFWE_B			0x0
-			>;
-		};
+	pinctrl_nfc: nfcgrp {
+		fsl,pins = <
+			MX27_PAD_NFRB__NFRB			0x0
+			MX27_PAD_NFCLE__NFCLE			0x0
+			MX27_PAD_NFWP_B__NFWP_B			0x0
+			MX27_PAD_NFCE_B__NFCE_B			0x0
+			MX27_PAD_NFALE__NFALE			0x0
+			MX27_PAD_NFRE_B__NFRE_B			0x0
+			MX27_PAD_NFWE_B__NFWE_B			0x0
+		>;
+	};
 
-		pinctrl_owire: owiregrp {
-			fsl,pins = <
-				MX27_PAD_RTCK__OWIRE			0x0
-			>;
-		};
+	pinctrl_owire: owiregrp {
+		fsl,pins = <
+			MX27_PAD_RTCK__OWIRE			0x0
+		>;
+	};
 
-		pinctrl_sdhc2: sdhc2grp {
-			fsl,pins = <
-				MX27_PAD_SD2_CLK__SD2_CLK		0x0
-				MX27_PAD_SD2_CMD__SD2_CMD		0x0
-				MX27_PAD_SD2_D0__SD2_D0			0x0
-				MX27_PAD_SD2_D1__SD2_D1			0x0
-				MX27_PAD_SD2_D2__SD2_D2			0x0
-				MX27_PAD_SD2_D3__SD2_D3			0x0
-			>;
-		};
+	pinctrl_sdhc2: sdhc2grp {
+		fsl,pins = <
+			MX27_PAD_SD2_CLK__SD2_CLK		0x0
+			MX27_PAD_SD2_CMD__SD2_CMD		0x0
+			MX27_PAD_SD2_D0__SD2_D0			0x0
+			MX27_PAD_SD2_D1__SD2_D1			0x0
+			MX27_PAD_SD2_D2__SD2_D2			0x0
+			MX27_PAD_SD2_D3__SD2_D3			0x0
+		>;
+	};
 
-		pinctrl_uart4: uart4grp {
-			fsl,pins = <
-				MX27_PAD_USBH1_TXDM__UART4_TXD		0x0
-				MX27_PAD_USBH1_RXDP__UART4_RXD		0x0
-				MX27_PAD_USBH1_TXDP__UART4_CTS		0x0
-				MX27_PAD_USBH1_FS__UART4_RTS		0x0
-			>;
-		};
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX27_PAD_USBH1_TXDM__UART4_TXD		0x0
+			MX27_PAD_USBH1_RXDP__UART4_RXD		0x0
+			MX27_PAD_USBH1_TXDP__UART4_CTS		0x0
+			MX27_PAD_USBH1_FS__UART4_RTS		0x0
+		>;
+	};
 
-		pinctrl_uart8250_1: uart82501grp {
-			fsl,pins = <
-				MX27_PAD_USB_PWR__GPIO2_23		0x0
-			>;
-		};
+	pinctrl_uart8250_1: uart82501grp {
+		fsl,pins = <
+			MX27_PAD_USB_PWR__GPIO2_23		0x0
+		>;
+	};
 
-		pinctrl_uart8250_2: uart82502grp {
-			fsl,pins = <
-				MX27_PAD_USBH1_SUSP__GPIO2_22		0x0
-			>;
-		};
+	pinctrl_uart8250_2: uart82502grp {
+		fsl,pins = <
+			MX27_PAD_USBH1_SUSP__GPIO2_22		0x0
+		>;
+	};
 
-		pinctrl_uart8250_3: uart82503grp {
-			fsl,pins = <
-				MX27_PAD_USBH1_OE_B__GPIO2_27		0x0
-			>;
-		};
+	pinctrl_uart8250_3: uart82503grp {
+		fsl,pins = <
+			MX27_PAD_USBH1_OE_B__GPIO2_27		0x0
+		>;
+	};
 
-		pinctrl_uart8250_4: uart82504grp {
-			fsl,pins = <
-				MX27_PAD_USBH1_RXDM__GPIO2_30		0x0
-			>;
-		};
+	pinctrl_uart8250_4: uart82504grp {
+		fsl,pins = <
+			MX27_PAD_USBH1_RXDM__GPIO2_30		0x0
+		>;
+	};
 
-		pinctrl_usbh2: usbh2grp {
-			fsl,pins = <
-				MX27_PAD_USBH2_CLK__USBH2_CLK		0x0
-				MX27_PAD_USBH2_DIR__USBH2_DIR		0x0
-				MX27_PAD_USBH2_NXT__USBH2_NXT		0x0
-				MX27_PAD_USBH2_STP__USBH2_STP		0x0
-				MX27_PAD_CSPI2_SCLK__USBH2_DATA0	0x0
-				MX27_PAD_CSPI2_MOSI__USBH2_DATA1	0x0
-				MX27_PAD_CSPI2_MISO__USBH2_DATA2	0x0
-				MX27_PAD_CSPI2_SS1__USBH2_DATA3		0x0
-				MX27_PAD_CSPI2_SS2__USBH2_DATA4		0x0
-				MX27_PAD_CSPI1_SS2__USBH2_DATA5		0x0
-				MX27_PAD_CSPI2_SS0__USBH2_DATA6		0x0
-				MX27_PAD_USBH2_DATA7__USBH2_DATA7	0x0
-			>;
-		};
+	pinctrl_usbh2: usbh2grp {
+		fsl,pins = <
+			MX27_PAD_USBH2_CLK__USBH2_CLK		0x0
+			MX27_PAD_USBH2_DIR__USBH2_DIR		0x0
+			MX27_PAD_USBH2_NXT__USBH2_NXT		0x0
+			MX27_PAD_USBH2_STP__USBH2_STP		0x0
+			MX27_PAD_CSPI2_SCLK__USBH2_DATA0	0x0
+			MX27_PAD_CSPI2_MOSI__USBH2_DATA1	0x0
+			MX27_PAD_CSPI2_MISO__USBH2_DATA2	0x0
+			MX27_PAD_CSPI2_SS1__USBH2_DATA3		0x0
+			MX27_PAD_CSPI2_SS2__USBH2_DATA4		0x0
+			MX27_PAD_CSPI1_SS2__USBH2_DATA5		0x0
+			MX27_PAD_CSPI2_SS0__USBH2_DATA6		0x0
+			MX27_PAD_USBH2_DATA7__USBH2_DATA7	0x0
+		>;
+	};
 
-		pinctrl_usbotg: usbotggrp {
-			fsl,pins = <
-				MX27_PAD_USBOTG_CLK__USBOTG_CLK		0x0
-				MX27_PAD_USBOTG_DIR__USBOTG_DIR		0x0
-				MX27_PAD_USBOTG_NXT__USBOTG_NXT		0x0
-				MX27_PAD_USBOTG_STP__USBOTG_STP		0x0
-				MX27_PAD_USBOTG_DATA0__USBOTG_DATA0	0x0
-				MX27_PAD_USBOTG_DATA1__USBOTG_DATA1	0x0
-				MX27_PAD_USBOTG_DATA2__USBOTG_DATA2	0x0
-				MX27_PAD_USBOTG_DATA3__USBOTG_DATA3	0x0
-				MX27_PAD_USBOTG_DATA4__USBOTG_DATA4	0x0
-				MX27_PAD_USBOTG_DATA5__USBOTG_DATA5	0x0
-				MX27_PAD_USBOTG_DATA6__USBOTG_DATA6	0x0
-				MX27_PAD_USBOTG_DATA7__USBOTG_DATA7	0x0
-			>;
-		};
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX27_PAD_USBOTG_CLK__USBOTG_CLK		0x0
+			MX27_PAD_USBOTG_DIR__USBOTG_DIR		0x0
+			MX27_PAD_USBOTG_NXT__USBOTG_NXT		0x0
+			MX27_PAD_USBOTG_STP__USBOTG_STP		0x0
+			MX27_PAD_USBOTG_DATA0__USBOTG_DATA0	0x0
+			MX27_PAD_USBOTG_DATA1__USBOTG_DATA1	0x0
+			MX27_PAD_USBOTG_DATA2__USBOTG_DATA2	0x0
+			MX27_PAD_USBOTG_DATA3__USBOTG_DATA3	0x0
+			MX27_PAD_USBOTG_DATA4__USBOTG_DATA4	0x0
+			MX27_PAD_USBOTG_DATA5__USBOTG_DATA5	0x0
+			MX27_PAD_USBOTG_DATA6__USBOTG_DATA6	0x0
+			MX27_PAD_USBOTG_DATA7__USBOTG_DATA7	0x0
+		>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts
index d78793601306c..26833ed3339e2 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts
@@ -76,7 +76,7 @@ ads7846@0 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_touch>;
 		reg = <0>;
-		interrupts = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>;
 		spi-cpol;
 		spi-max-frequency = <1500000>;
 		ti,keep-vref-on;
@@ -147,113 +147,111 @@ &uart3 {
 };
 
 &iomuxc {
-	imx27-eukrea-cpuimx27-baseboard {
-		pinctrl_cspi1: cspi1grp {
-			fsl,pins = <
-				MX27_PAD_CSPI1_MISO__CSPI1_MISO	0x0
-				MX27_PAD_CSPI1_MOSI__CSPI1_MOSI	0x0
-				MX27_PAD_CSPI1_SCLK__CSPI1_SCLK	0x0
-				MX27_PAD_CSPI1_SS0__GPIO4_28	0x0 /* CS0 */
-			>;
-		};
+	pinctrl_cspi1: cspi1grp {
+		fsl,pins = <
+			MX27_PAD_CSPI1_MISO__CSPI1_MISO	0x0
+			MX27_PAD_CSPI1_MOSI__CSPI1_MOSI	0x0
+			MX27_PAD_CSPI1_SCLK__CSPI1_SCLK	0x0
+			MX27_PAD_CSPI1_SS0__GPIO4_28	0x0 /* CS0 */
+		>;
+	};
 
-		pinctrl_backlight: backlightgrp {
-			fsl,pins = <
-				MX27_PAD_PWMO__GPIO5_5		0x0
-			>;
-		};
+	pinctrl_backlight: backlightgrp {
+		fsl,pins = <
+			MX27_PAD_PWMO__GPIO5_5		0x0
+		>;
+	};
 
-		pinctrl_gpioleds: gpioledsgrp {
-			fsl,pins = <
-				MX27_PAD_PC_PWRON__GPIO6_16	0x0
-				MX27_PAD_PC_CD2_B__GPIO6_19	0x0
-			>;
-		};
+	pinctrl_gpioleds: gpioledsgrp {
+		fsl,pins = <
+			MX27_PAD_PC_PWRON__GPIO6_16	0x0
+			MX27_PAD_PC_CD2_B__GPIO6_19	0x0
+		>;
+	};
 
-		pinctrl_imxfb: imxfbgrp {
-			fsl,pins = <
-				MX27_PAD_LD0__LD0		0x0
-				MX27_PAD_LD1__LD1		0x0
-				MX27_PAD_LD2__LD2		0x0
-				MX27_PAD_LD3__LD3		0x0
-				MX27_PAD_LD4__LD4		0x0
-				MX27_PAD_LD5__LD5		0x0
-				MX27_PAD_LD6__LD6		0x0
-				MX27_PAD_LD7__LD7		0x0
-				MX27_PAD_LD8__LD8		0x0
-				MX27_PAD_LD9__LD9		0x0
-				MX27_PAD_LD10__LD10		0x0
-				MX27_PAD_LD11__LD11		0x0
-				MX27_PAD_LD12__LD12		0x0
-				MX27_PAD_LD13__LD13		0x0
-				MX27_PAD_LD14__LD14		0x0
-				MX27_PAD_LD15__LD15		0x0
-				MX27_PAD_LD16__LD16		0x0
-				MX27_PAD_LD17__LD17		0x0
-				MX27_PAD_CONTRAST__CONTRAST	0x0
-				MX27_PAD_OE_ACD__OE_ACD		0x0
-				MX27_PAD_HSYNC__HSYNC		0x0
-				MX27_PAD_VSYNC__VSYNC		0x0
-			>;
-		};
+	pinctrl_imxfb: imxfbgrp {
+		fsl,pins = <
+			MX27_PAD_LD0__LD0		0x0
+			MX27_PAD_LD1__LD1		0x0
+			MX27_PAD_LD2__LD2		0x0
+			MX27_PAD_LD3__LD3		0x0
+			MX27_PAD_LD4__LD4		0x0
+			MX27_PAD_LD5__LD5		0x0
+			MX27_PAD_LD6__LD6		0x0
+			MX27_PAD_LD7__LD7		0x0
+			MX27_PAD_LD8__LD8		0x0
+			MX27_PAD_LD9__LD9		0x0
+			MX27_PAD_LD10__LD10		0x0
+			MX27_PAD_LD11__LD11		0x0
+			MX27_PAD_LD12__LD12		0x0
+			MX27_PAD_LD13__LD13		0x0
+			MX27_PAD_LD14__LD14		0x0
+			MX27_PAD_LD15__LD15		0x0
+			MX27_PAD_LD16__LD16		0x0
+			MX27_PAD_LD17__LD17		0x0
+			MX27_PAD_CONTRAST__CONTRAST	0x0
+			MX27_PAD_OE_ACD__OE_ACD		0x0
+			MX27_PAD_HSYNC__HSYNC		0x0
+			MX27_PAD_VSYNC__VSYNC		0x0
+		>;
+	};
 
-		pinctrl_lcdreg: lcdreggrp {
-			fsl,pins = <
-				MX27_PAD_CLS__GPIO1_25		0x0
-			>;
-		};
+	pinctrl_lcdreg: lcdreggrp {
+		fsl,pins = <
+			MX27_PAD_CLS__GPIO1_25		0x0
+		>;
+	};
 
-		pinctrl_sdhc1: sdhc1grp {
-			fsl,pins = <
-				MX27_PAD_SD1_CLK__SD1_CLK	0x0
-				MX27_PAD_SD1_CMD__SD1_CMD	0x0
-				MX27_PAD_SD1_D0__SD1_D0		0x0
-				MX27_PAD_SD1_D1__SD1_D1		0x0
-				MX27_PAD_SD1_D2__SD1_D2		0x0
-				MX27_PAD_SD1_D3__SD1_D3		0x0
-			>;
-		};
+	pinctrl_sdhc1: sdhc1grp {
+		fsl,pins = <
+			MX27_PAD_SD1_CLK__SD1_CLK	0x0
+			MX27_PAD_SD1_CMD__SD1_CMD	0x0
+			MX27_PAD_SD1_D0__SD1_D0		0x0
+			MX27_PAD_SD1_D1__SD1_D1		0x0
+			MX27_PAD_SD1_D2__SD1_D2		0x0
+			MX27_PAD_SD1_D3__SD1_D3		0x0
+		>;
+	};
 
-		pinctrl_ssi1: ssi1grp {
-			fsl,pins = <
-				MX27_PAD_SSI4_CLK__SSI4_CLK	0x0
-				MX27_PAD_SSI4_FS__SSI4_FS	0x0
-				MX27_PAD_SSI4_RXDAT__SSI4_RXDAT	0x1
-				MX27_PAD_SSI4_TXDAT__SSI4_TXDAT	0x1
-			>;
-		};
+	pinctrl_ssi1: ssi1grp {
+		fsl,pins = <
+			MX27_PAD_SSI4_CLK__SSI4_CLK	0x0
+			MX27_PAD_SSI4_FS__SSI4_FS	0x0
+			MX27_PAD_SSI4_RXDAT__SSI4_RXDAT	0x1
+			MX27_PAD_SSI4_TXDAT__SSI4_TXDAT	0x1
+		>;
+	};
 
-		pinctrl_touch: touchgrp {
-			fsl,pins = <
-				MX27_PAD_CSPI1_RDY__GPIO4_25	0x0 /* IRQ */
-			>;
-		};
+	pinctrl_touch: touchgrp {
+		fsl,pins = <
+			MX27_PAD_CSPI1_RDY__GPIO4_25	0x0 /* IRQ */
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD	0x0
-				MX27_PAD_UART1_RXD__UART1_RXD	0x0
-				MX27_PAD_UART1_CTS__UART1_CTS	0x0
-				MX27_PAD_UART1_RTS__UART1_RTS	0x0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX27_PAD_UART1_TXD__UART1_TXD	0x0
+			MX27_PAD_UART1_RXD__UART1_RXD	0x0
+			MX27_PAD_UART1_CTS__UART1_CTS	0x0
+			MX27_PAD_UART1_RTS__UART1_RTS	0x0
+		>;
+	};
 
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX27_PAD_UART2_TXD__UART2_TXD	0x0
-				MX27_PAD_UART2_RXD__UART2_RXD	0x0
-				MX27_PAD_UART2_CTS__UART2_CTS	0x0
-				MX27_PAD_UART2_RTS__UART2_RTS	0x0
-			>;
-		};
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX27_PAD_UART2_TXD__UART2_TXD	0x0
+			MX27_PAD_UART2_RXD__UART2_RXD	0x0
+			MX27_PAD_UART2_CTS__UART2_CTS	0x0
+			MX27_PAD_UART2_RTS__UART2_RTS	0x0
+		>;
+	};
 
-		pinctrl_uart3: uart3grp {
-			fsl,pins = <
-				MX27_PAD_UART3_TXD__UART3_TXD	0x0
-				MX27_PAD_UART3_RXD__UART3_RXD	0x0
-				MX27_PAD_UART3_CTS__UART3_CTS	0x0
-				MX27_PAD_UART3_RTS__UART3_RTS	0x0
-			>;
-		};
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX27_PAD_UART3_TXD__UART3_TXD	0x0
+			MX27_PAD_UART3_RXD__UART3_RXD	0x0
+			MX27_PAD_UART3_CTS__UART3_CTS	0x0
+			MX27_PAD_UART3_RTS__UART3_RTS	0x0
+		>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-pdk.dts b/arch/arm/boot/dts/nxp/imx/imx27-pdk.dts
index 21d436972aa47..2fc4ea5b9501f 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-pdk.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-pdk.dts
@@ -110,76 +110,74 @@ &usbotg {
 };
 
 &iomuxc {
-	imx27-pdk {
-		pinctrl_cspi2: cspi2grp {
-			fsl,pins = <
-				MX27_PAD_CSPI2_MISO__CSPI2_MISO 0x0
-				MX27_PAD_CSPI2_MOSI__CSPI2_MOSI 0x0
-				MX27_PAD_CSPI2_SCLK__CSPI2_SCLK 0x0
-				MX27_PAD_CSPI2_SS0__GPIO4_21	0x0 /* SPI2 CS0 */
-				MX27_PAD_TOUT__GPIO3_14		0x0 /* PMIC IRQ */
-			>;
-		};
+	pinctrl_cspi2: cspi2grp {
+		fsl,pins = <
+			MX27_PAD_CSPI2_MISO__CSPI2_MISO 0x0
+			MX27_PAD_CSPI2_MOSI__CSPI2_MOSI 0x0
+			MX27_PAD_CSPI2_SCLK__CSPI2_SCLK 0x0
+			MX27_PAD_CSPI2_SS0__GPIO4_21	0x0 /* SPI2 CS0 */
+			MX27_PAD_TOUT__GPIO3_14		0x0 /* PMIC IRQ */
+		>;
+	};
 
-		pinctrl_fec: fecgrp {
-			fsl,pins = <
-				MX27_PAD_SD3_CMD__FEC_TXD0 0x0
-				MX27_PAD_SD3_CLK__FEC_TXD1 0x0
-				MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
-				MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
-				MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
-				MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
-				MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
-				MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
-				MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
-				MX27_PAD_ATA_DATA7__FEC_MDC 0x0
-				MX27_PAD_ATA_DATA8__FEC_CRS 0x0
-				MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
-				MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
-				MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
-				MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
-				MX27_PAD_ATA_DATA13__FEC_COL 0x0
-				MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
-				MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
-			>;
-		};
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX27_PAD_SD3_CMD__FEC_TXD0 0x0
+			MX27_PAD_SD3_CLK__FEC_TXD1 0x0
+			MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
+			MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
+			MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
+			MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
+			MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
+			MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
+			MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
+			MX27_PAD_ATA_DATA7__FEC_MDC 0x0
+			MX27_PAD_ATA_DATA8__FEC_CRS 0x0
+			MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
+			MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
+			MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
+			MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
+			MX27_PAD_ATA_DATA13__FEC_COL 0x0
+			MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
+			MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
+		>;
+	};
 
-		pinctrl_nand: nandgrp {
-			fsl,pins = <
-				MX27_PAD_NFRB__NFRB	0x0
-				MX27_PAD_NFCLE__NFCLE	0x0
-				MX27_PAD_NFWP_B__NFWP_B	0x0
-				MX27_PAD_NFCE_B__NFCE_B	0x0
-				MX27_PAD_NFALE__NFALE	0x0
-				MX27_PAD_NFRE_B__NFRE_B	0x0
-				MX27_PAD_NFWE_B__NFWE_B	0x0
-			>;
-		};
+	pinctrl_nand: nandgrp {
+		fsl,pins = <
+			MX27_PAD_NFRB__NFRB	0x0
+			MX27_PAD_NFCLE__NFCLE	0x0
+			MX27_PAD_NFWP_B__NFWP_B	0x0
+			MX27_PAD_NFCE_B__NFCE_B	0x0
+			MX27_PAD_NFALE__NFALE	0x0
+			MX27_PAD_NFRE_B__NFRE_B	0x0
+			MX27_PAD_NFWE_B__NFWE_B	0x0
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD 0x0
-				MX27_PAD_UART1_RXD__UART1_RXD 0x0
-				MX27_PAD_UART1_CTS__UART1_CTS 0x0
-				MX27_PAD_UART1_RTS__UART1_RTS 0x0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX27_PAD_UART1_TXD__UART1_TXD 0x0
+			MX27_PAD_UART1_RXD__UART1_RXD 0x0
+			MX27_PAD_UART1_CTS__UART1_CTS 0x0
+			MX27_PAD_UART1_RTS__UART1_RTS 0x0
+		>;
+	};
 
-		pinctrl_usbotg: usbotggrp {
-			fsl,pins = <
-				MX27_PAD_USBOTG_NXT__USBOTG_NXT 0x0
-				MX27_PAD_USBOTG_STP__USBOTG_STP 0x0
-				MX27_PAD_USBOTG_DIR__USBOTG_DIR 0x0
-				MX27_PAD_USBOTG_CLK__USBOTG_CLK 0x0
-				MX27_PAD_USBOTG_DATA0__USBOTG_DATA0 0x0
-				MX27_PAD_USBOTG_DATA1__USBOTG_DATA1 0x0
-				MX27_PAD_USBOTG_DATA2__USBOTG_DATA2 0x0
-				MX27_PAD_USBOTG_DATA3__USBOTG_DATA3 0x0
-				MX27_PAD_USBOTG_DATA4__USBOTG_DATA4 0x0
-				MX27_PAD_USBOTG_DATA5__USBOTG_DATA5 0x0
-				MX27_PAD_USBOTG_DATA6__USBOTG_DATA6 0x0
-				MX27_PAD_USBOTG_DATA7__USBOTG_DATA7 0x0
-			>;
-		};
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX27_PAD_USBOTG_NXT__USBOTG_NXT 0x0
+			MX27_PAD_USBOTG_STP__USBOTG_STP 0x0
+			MX27_PAD_USBOTG_DIR__USBOTG_DIR 0x0
+			MX27_PAD_USBOTG_CLK__USBOTG_CLK 0x0
+			MX27_PAD_USBOTG_DATA0__USBOTG_DATA0 0x0
+			MX27_PAD_USBOTG_DATA1__USBOTG_DATA1 0x0
+			MX27_PAD_USBOTG_DATA2__USBOTG_DATA2 0x0
+			MX27_PAD_USBOTG_DATA3__USBOTG_DATA3 0x0
+			MX27_PAD_USBOTG_DATA4__USBOTG_DATA4 0x0
+			MX27_PAD_USBOTG_DATA5__USBOTG_DATA5 0x0
+			MX27_PAD_USBOTG_DATA6__USBOTG_DATA6 0x0
+			MX27_PAD_USBOTG_DATA7__USBOTG_DATA7 0x0
+		>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-rdk.dts b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-rdk.dts
index 27c93b9fe0499..2b884cb3e3818 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-rdk.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-rdk.dts
@@ -65,58 +65,56 @@ adc@64 {
 };
 
 &iomuxc {
-	imx27-phycard-s-rdk {
-		pinctrl_i2c1: i2c1grp {
-			fsl,pins = <
-				MX27_PAD_I2C_DATA__I2C_DATA 0x0
-				MX27_PAD_I2C_CLK__I2C_CLK 0x0
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX27_PAD_I2C_DATA__I2C_DATA 0x0
+			MX27_PAD_I2C_CLK__I2C_CLK 0x0
+		>;
+	};
 
-		pinctrl_owire1: owire1grp {
-			fsl,pins = <
-				MX27_PAD_RTCK__OWIRE 0x0
-			>;
-		};
+	pinctrl_owire1: owire1grp {
+		fsl,pins = <
+			MX27_PAD_RTCK__OWIRE 0x0
+		>;
+	};
 
-		pinctrl_sdhc2: sdhc2grp {
-			fsl,pins = <
-				MX27_PAD_SD2_CLK__SD2_CLK 0x0
-				MX27_PAD_SD2_CMD__SD2_CMD 0x0
-				MX27_PAD_SD2_D0__SD2_D0 0x0
-				MX27_PAD_SD2_D1__SD2_D1 0x0
-				MX27_PAD_SD2_D2__SD2_D2 0x0
-				MX27_PAD_SD2_D3__SD2_D3 0x0
-				MX27_PAD_SSI3_RXDAT__GPIO3_29 0x0 /* CD */
-			>;
-		};
+	pinctrl_sdhc2: sdhc2grp {
+		fsl,pins = <
+			MX27_PAD_SD2_CLK__SD2_CLK 0x0
+			MX27_PAD_SD2_CMD__SD2_CMD 0x0
+			MX27_PAD_SD2_D0__SD2_D0 0x0
+			MX27_PAD_SD2_D1__SD2_D1 0x0
+			MX27_PAD_SD2_D2__SD2_D2 0x0
+			MX27_PAD_SD2_D3__SD2_D3 0x0
+			MX27_PAD_SSI3_RXDAT__GPIO3_29 0x0 /* CD */
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD 0x0
-				MX27_PAD_UART1_RXD__UART1_RXD 0x0
-				MX27_PAD_UART1_CTS__UART1_CTS 0x0
-				MX27_PAD_UART1_RTS__UART1_RTS 0x0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX27_PAD_UART1_TXD__UART1_TXD 0x0
+			MX27_PAD_UART1_RXD__UART1_RXD 0x0
+			MX27_PAD_UART1_CTS__UART1_CTS 0x0
+			MX27_PAD_UART1_RTS__UART1_RTS 0x0
+		>;
+	};
 
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX27_PAD_UART2_TXD__UART2_TXD 0x0
-				MX27_PAD_UART2_RXD__UART2_RXD 0x0
-				MX27_PAD_UART2_CTS__UART2_CTS 0x0
-				MX27_PAD_UART2_RTS__UART2_RTS 0x0
-			>;
-		};
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX27_PAD_UART2_TXD__UART2_TXD 0x0
+			MX27_PAD_UART2_RXD__UART2_RXD 0x0
+			MX27_PAD_UART2_CTS__UART2_CTS 0x0
+			MX27_PAD_UART2_RTS__UART2_RTS 0x0
+		>;
+	};
 
-		pinctrl_uart3: uart3grp {
-			fsl,pins = <
-				MX27_PAD_UART3_TXD__UART3_TXD 0x0
-				MX27_PAD_UART3_RXD__UART3_RXD 0x0
-				MX27_PAD_UART3_CTS__UART3_CTS 0x0
-				MX27_PAD_UART3_RTS__UART3_RTS 0x0
-			>;
-		};
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX27_PAD_UART3_TXD__UART3_TXD 0x0
+			MX27_PAD_UART3_RXD__UART3_RXD 0x0
+			MX27_PAD_UART3_CTS__UART3_CTS 0x0
+			MX27_PAD_UART3_RTS__UART3_RTS 0x0
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-som.dtsi
index 31b3fc972abbf..2f60b3809f39b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycard-s-som.dtsi
@@ -58,94 +58,92 @@ eeprom@52 {
 };
 
 &iomuxc {
-	imx27-phycard-s-som {
-		pinctrl_fec1: fec1grp {
-			fsl,pins = <
-				MX27_PAD_SD3_CMD__FEC_TXD0 0x0
-				MX27_PAD_SD3_CLK__FEC_TXD1 0x0
-				MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
-				MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
-				MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
-				MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
-				MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
-				MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
-				MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
-				MX27_PAD_ATA_DATA7__FEC_MDC 0x0
-				MX27_PAD_ATA_DATA8__FEC_CRS 0x0
-				MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
-				MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
-				MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
-				MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
-				MX27_PAD_ATA_DATA13__FEC_COL 0x0
-				MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
-				MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
-			>;
-		};
-
-		pinctrl_i2c2: i2c2grp {
-			fsl,pins = <
-				MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
-				MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
-			>;
-		};
-
-		pinctrl_nfc: nfcgrp {
-			fsl,pins = <
-				MX27_PAD_NFRB__NFRB 0x0
-				MX27_PAD_NFCLE__NFCLE 0x0
-				MX27_PAD_NFWP_B__NFWP_B 0x0
-				MX27_PAD_NFCE_B__NFCE_B 0x0
-				MX27_PAD_NFALE__NFALE 0x0
-				MX27_PAD_NFRE_B__NFRE_B 0x0
-				MX27_PAD_NFWE_B__NFWE_B 0x0
-			>;
-		};
-
-		pinctrl_usbotgphy: usbotgphygrp {
-			fsl,pins = <
-				MX27_PAD_USBH1_RCV__GPIO2_25		0x1 /* reset gpio */
-			>;
-		};
-
-		pinctrl_usbotg: usbotggrp {
-			fsl,pins = <
-				MX27_PAD_USBOTG_CLK__USBOTG_CLK		0x0
-				MX27_PAD_USBOTG_DIR__USBOTG_DIR		0x0
-				MX27_PAD_USBOTG_NXT__USBOTG_NXT		0x0
-				MX27_PAD_USBOTG_STP__USBOTG_STP		0x0
-				MX27_PAD_USBOTG_DATA0__USBOTG_DATA0	0x0
-				MX27_PAD_USBOTG_DATA1__USBOTG_DATA1	0x0
-				MX27_PAD_USBOTG_DATA2__USBOTG_DATA2	0x0
-				MX27_PAD_USBOTG_DATA3__USBOTG_DATA3	0x0
-				MX27_PAD_USBOTG_DATA4__USBOTG_DATA4	0x0
-				MX27_PAD_USBOTG_DATA5__USBOTG_DATA5	0x0
-				MX27_PAD_USBOTG_DATA6__USBOTG_DATA6	0x0
-				MX27_PAD_USBOTG_DATA7__USBOTG_DATA7	0x0
-			>;
-		};
-
-		pinctrl_usbh2phy: usbh2phygrp {
-			fsl,pins = <
-				MX27_PAD_USBH1_SUSP__GPIO2_22		0x0 /* reset gpio */
-			>;
-		};
-
-		pinctrl_usbh2: usbh2grp {
-			fsl,pins = <
-				MX27_PAD_USBH2_CLK__USBH2_CLK		0x0
-				MX27_PAD_USBH2_DIR__USBH2_DIR		0x0
-				MX27_PAD_USBH2_NXT__USBH2_NXT		0x0
-				MX27_PAD_USBH2_STP__USBH2_STP		0x0
-				MX27_PAD_CSPI2_SCLK__USBH2_DATA0	0x0
-				MX27_PAD_CSPI2_MOSI__USBH2_DATA1	0x0
-				MX27_PAD_CSPI2_MISO__USBH2_DATA2	0x0
-				MX27_PAD_CSPI2_SS1__USBH2_DATA3		0x0
-				MX27_PAD_CSPI2_SS2__USBH2_DATA4		0x0
-				MX27_PAD_CSPI1_SS2__USBH2_DATA5		0x0
-				MX27_PAD_CSPI2_SS0__USBH2_DATA6		0x0
-				MX27_PAD_USBH2_DATA7__USBH2_DATA7	0x0
-			>;
-		};
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX27_PAD_SD3_CMD__FEC_TXD0 0x0
+			MX27_PAD_SD3_CLK__FEC_TXD1 0x0
+			MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
+			MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
+			MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
+			MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
+			MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
+			MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
+			MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
+			MX27_PAD_ATA_DATA7__FEC_MDC 0x0
+			MX27_PAD_ATA_DATA8__FEC_CRS 0x0
+			MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
+			MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
+			MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
+			MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
+			MX27_PAD_ATA_DATA13__FEC_COL 0x0
+			MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
+			MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
+			MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
+		>;
+	};
+
+	pinctrl_nfc: nfcgrp {
+		fsl,pins = <
+			MX27_PAD_NFRB__NFRB 0x0
+			MX27_PAD_NFCLE__NFCLE 0x0
+			MX27_PAD_NFWP_B__NFWP_B 0x0
+			MX27_PAD_NFCE_B__NFCE_B 0x0
+			MX27_PAD_NFALE__NFALE 0x0
+			MX27_PAD_NFRE_B__NFRE_B 0x0
+			MX27_PAD_NFWE_B__NFWE_B 0x0
+		>;
+	};
+
+	pinctrl_usbotgphy: usbotgphygrp {
+		fsl,pins = <
+			MX27_PAD_USBH1_RCV__GPIO2_25		0x1 /* reset gpio */
+		>;
+	};
+
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX27_PAD_USBOTG_CLK__USBOTG_CLK		0x0
+			MX27_PAD_USBOTG_DIR__USBOTG_DIR		0x0
+			MX27_PAD_USBOTG_NXT__USBOTG_NXT		0x0
+			MX27_PAD_USBOTG_STP__USBOTG_STP		0x0
+			MX27_PAD_USBOTG_DATA0__USBOTG_DATA0	0x0
+			MX27_PAD_USBOTG_DATA1__USBOTG_DATA1	0x0
+			MX27_PAD_USBOTG_DATA2__USBOTG_DATA2	0x0
+			MX27_PAD_USBOTG_DATA3__USBOTG_DATA3	0x0
+			MX27_PAD_USBOTG_DATA4__USBOTG_DATA4	0x0
+			MX27_PAD_USBOTG_DATA5__USBOTG_DATA5	0x0
+			MX27_PAD_USBOTG_DATA6__USBOTG_DATA6	0x0
+			MX27_PAD_USBOTG_DATA7__USBOTG_DATA7	0x0
+		>;
+	};
+
+	pinctrl_usbh2phy: usbh2phygrp {
+		fsl,pins = <
+			MX27_PAD_USBH1_SUSP__GPIO2_22		0x0 /* reset gpio */
+		>;
+	};
+
+	pinctrl_usbh2: usbh2grp {
+		fsl,pins = <
+			MX27_PAD_USBH2_CLK__USBH2_CLK		0x0
+			MX27_PAD_USBH2_DIR__USBH2_DIR		0x0
+			MX27_PAD_USBH2_NXT__USBH2_NXT		0x0
+			MX27_PAD_USBH2_STP__USBH2_STP		0x0
+			MX27_PAD_CSPI2_SCLK__USBH2_DATA0	0x0
+			MX27_PAD_CSPI2_MOSI__USBH2_DATA1	0x0
+			MX27_PAD_CSPI2_MISO__USBH2_DATA2	0x0
+			MX27_PAD_CSPI2_SS1__USBH2_DATA3		0x0
+			MX27_PAD_CSPI2_SS2__USBH2_DATA4		0x0
+			MX27_PAD_CSPI1_SS2__USBH2_DATA5		0x0
+			MX27_PAD_CSPI2_SS0__USBH2_DATA6		0x0
+			MX27_PAD_USBH2_DATA7__USBH2_DATA7	0x0
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-rdk.dts b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-rdk.dts
index 5398e9067e60f..d7136c399ae29 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-rdk.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-rdk.dts
@@ -89,119 +89,117 @@ camgpio: pca9536@41 {
 };
 
 &iomuxc {
-	imx27_phycore_rdk {
-		pinctrl_csien: csiengrp {
-			fsl,pins = <
-				MX27_PAD_USB_OC_B__GPIO2_24 0x0
-			>;
-		};
+	pinctrl_csien: csiengrp {
+		fsl,pins = <
+			MX27_PAD_USB_OC_B__GPIO2_24 0x0
+		>;
+	};
 
-		pinctrl_cspi1cs1: cspi1cs1grp {
-			fsl,pins = <
-				MX27_PAD_CSPI1_SS1__GPIO4_27 0x0
-			>;
-		};
+	pinctrl_cspi1cs1: cspi1cs1grp {
+		fsl,pins = <
+			MX27_PAD_CSPI1_SS1__GPIO4_27 0x0
+		>;
+	};
 
-		pinctrl_imxfb1: imxfbgrp {
-			fsl,pins = <
-				MX27_PAD_LD0__LD0 0x0
-				MX27_PAD_LD1__LD1 0x0
-				MX27_PAD_LD2__LD2 0x0
-				MX27_PAD_LD3__LD3 0x0
-				MX27_PAD_LD4__LD4 0x0
-				MX27_PAD_LD5__LD5 0x0
-				MX27_PAD_LD6__LD6 0x0
-				MX27_PAD_LD7__LD7 0x0
-				MX27_PAD_LD8__LD8 0x0
-				MX27_PAD_LD9__LD9 0x0
-				MX27_PAD_LD10__LD10 0x0
-				MX27_PAD_LD11__LD11 0x0
-				MX27_PAD_LD12__LD12 0x0
-				MX27_PAD_LD13__LD13 0x0
-				MX27_PAD_LD14__LD14 0x0
-				MX27_PAD_LD15__LD15 0x0
-				MX27_PAD_LD16__LD16 0x0
-				MX27_PAD_LD17__LD17 0x0
-				MX27_PAD_CLS__CLS 0x0
-				MX27_PAD_CONTRAST__CONTRAST 0x0
-				MX27_PAD_LSCLK__LSCLK 0x0
-				MX27_PAD_OE_ACD__OE_ACD 0x0
-				MX27_PAD_PS__PS 0x0
-				MX27_PAD_REV__REV 0x0
-				MX27_PAD_SPL_SPR__SPL_SPR 0x0
-				MX27_PAD_HSYNC__HSYNC 0x0
-				MX27_PAD_VSYNC__VSYNC 0x0
-			>;
-		};
+	pinctrl_imxfb1: imxfbgrp {
+		fsl,pins = <
+			MX27_PAD_LD0__LD0 0x0
+			MX27_PAD_LD1__LD1 0x0
+			MX27_PAD_LD2__LD2 0x0
+			MX27_PAD_LD3__LD3 0x0
+			MX27_PAD_LD4__LD4 0x0
+			MX27_PAD_LD5__LD5 0x0
+			MX27_PAD_LD6__LD6 0x0
+			MX27_PAD_LD7__LD7 0x0
+			MX27_PAD_LD8__LD8 0x0
+			MX27_PAD_LD9__LD9 0x0
+			MX27_PAD_LD10__LD10 0x0
+			MX27_PAD_LD11__LD11 0x0
+			MX27_PAD_LD12__LD12 0x0
+			MX27_PAD_LD13__LD13 0x0
+			MX27_PAD_LD14__LD14 0x0
+			MX27_PAD_LD15__LD15 0x0
+			MX27_PAD_LD16__LD16 0x0
+			MX27_PAD_LD17__LD17 0x0
+			MX27_PAD_CLS__CLS 0x0
+			MX27_PAD_CONTRAST__CONTRAST 0x0
+			MX27_PAD_LSCLK__LSCLK 0x0
+			MX27_PAD_OE_ACD__OE_ACD 0x0
+			MX27_PAD_PS__PS 0x0
+			MX27_PAD_REV__REV 0x0
+			MX27_PAD_SPL_SPR__SPL_SPR 0x0
+			MX27_PAD_HSYNC__HSYNC 0x0
+			MX27_PAD_VSYNC__VSYNC 0x0
+		>;
+	};
 
-		pinctrl_i2c1: i2c1grp {
-			/* Add pullup to DATA line */
-			fsl,pins = <
-				MX27_PAD_I2C_DATA__I2C_DATA	0x1
-				MX27_PAD_I2C_CLK__I2C_CLK	0x0
-			>;
-		};
+	pinctrl_i2c1: i2c1grp {
+		/* Add pullup to DATA line */
+		fsl,pins = <
+			MX27_PAD_I2C_DATA__I2C_DATA	0x1
+			MX27_PAD_I2C_CLK__I2C_CLK	0x0
+		>;
+	};
 
-		pinctrl_owire1: owire1grp {
-			fsl,pins = <
-				MX27_PAD_RTCK__OWIRE 0x0
-			>;
-		};
+	pinctrl_owire1: owire1grp {
+		fsl,pins = <
+			MX27_PAD_RTCK__OWIRE 0x0
+		>;
+	};
 
-		pinctrl_sdhc2: sdhc2grp {
-			fsl,pins = <
-				MX27_PAD_SD2_CLK__SD2_CLK 0x0
-				MX27_PAD_SD2_CMD__SD2_CMD 0x0
-				MX27_PAD_SD2_D0__SD2_D0 0x0
-				MX27_PAD_SD2_D1__SD2_D1 0x0
-				MX27_PAD_SD2_D2__SD2_D2 0x0
-				MX27_PAD_SD2_D3__SD2_D3 0x0
-				MX27_PAD_SSI3_FS__GPIO3_28	0x0 /* WP */
-				MX27_PAD_SSI3_RXDAT__GPIO3_29	0x0 /* CD */
-			>;
-		};
+	pinctrl_sdhc2: sdhc2grp {
+		fsl,pins = <
+			MX27_PAD_SD2_CLK__SD2_CLK 0x0
+			MX27_PAD_SD2_CMD__SD2_CMD 0x0
+			MX27_PAD_SD2_D0__SD2_D0 0x0
+			MX27_PAD_SD2_D1__SD2_D1 0x0
+			MX27_PAD_SD2_D2__SD2_D2 0x0
+			MX27_PAD_SD2_D3__SD2_D3 0x0
+			MX27_PAD_SSI3_FS__GPIO3_28	0x0 /* WP */
+			MX27_PAD_SSI3_RXDAT__GPIO3_29	0x0 /* CD */
+		>;
+	};
 
-		pinctrl_uart1: uart1grp {
-			fsl,pins = <
-				MX27_PAD_UART1_TXD__UART1_TXD 0x0
-				MX27_PAD_UART1_RXD__UART1_RXD 0x0
-				MX27_PAD_UART1_CTS__UART1_CTS 0x0
-				MX27_PAD_UART1_RTS__UART1_RTS 0x0
-			>;
-		};
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX27_PAD_UART1_TXD__UART1_TXD 0x0
+			MX27_PAD_UART1_RXD__UART1_RXD 0x0
+			MX27_PAD_UART1_CTS__UART1_CTS 0x0
+			MX27_PAD_UART1_RTS__UART1_RTS 0x0
+		>;
+	};
 
-		pinctrl_uart2: uart2grp {
-			fsl,pins = <
-				MX27_PAD_UART2_TXD__UART2_TXD 0x0
-				MX27_PAD_UART2_RXD__UART2_RXD 0x0
-				MX27_PAD_UART2_CTS__UART2_CTS 0x0
-				MX27_PAD_UART2_RTS__UART2_RTS 0x0
-			>;
-		};
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX27_PAD_UART2_TXD__UART2_TXD 0x0
+			MX27_PAD_UART2_RXD__UART2_RXD 0x0
+			MX27_PAD_UART2_CTS__UART2_CTS 0x0
+			MX27_PAD_UART2_RTS__UART2_RTS 0x0
+		>;
+	};
 
-		pinctrl_usbh2: usbh2grp {
-			fsl,pins = <
-				MX27_PAD_USBH2_CLK__USBH2_CLK 0x0
-				MX27_PAD_USBH2_DIR__USBH2_DIR 0x0
-				MX27_PAD_USBH2_NXT__USBH2_NXT 0x0
-				MX27_PAD_USBH2_STP__USBH2_STP 0x0
-				MX27_PAD_CSPI2_SCLK__USBH2_DATA0 0x0
-				MX27_PAD_CSPI2_MOSI__USBH2_DATA1 0x0
-				MX27_PAD_CSPI2_MISO__USBH2_DATA2 0x0
-				MX27_PAD_CSPI2_SS1__USBH2_DATA3 0x0
-				MX27_PAD_CSPI2_SS2__USBH2_DATA4 0x0
-				MX27_PAD_CSPI1_SS2__USBH2_DATA5 0x0
-				MX27_PAD_CSPI2_SS0__USBH2_DATA6 0x0
-				MX27_PAD_USBH2_DATA7__USBH2_DATA7 0x0
-			>;
-		};
+	pinctrl_usbh2: usbh2grp {
+		fsl,pins = <
+			MX27_PAD_USBH2_CLK__USBH2_CLK 0x0
+			MX27_PAD_USBH2_DIR__USBH2_DIR 0x0
+			MX27_PAD_USBH2_NXT__USBH2_NXT 0x0
+			MX27_PAD_USBH2_STP__USBH2_STP 0x0
+			MX27_PAD_CSPI2_SCLK__USBH2_DATA0 0x0
+			MX27_PAD_CSPI2_MOSI__USBH2_DATA1 0x0
+			MX27_PAD_CSPI2_MISO__USBH2_DATA2 0x0
+			MX27_PAD_CSPI2_SS1__USBH2_DATA3 0x0
+			MX27_PAD_CSPI2_SS2__USBH2_DATA4 0x0
+			MX27_PAD_CSPI1_SS2__USBH2_DATA5 0x0
+			MX27_PAD_CSPI2_SS0__USBH2_DATA6 0x0
+			MX27_PAD_USBH2_DATA7__USBH2_DATA7 0x0
+		>;
+	};
 
-		pinctrl_weim: weimgrp {
-			fsl,pins = <
-				MX27_PAD_CS4_B__CS4_B		0x0 /* CS4 */
-				MX27_PAD_SD1_D1__GPIO5_19	0x0 /* CAN IRQ */
-			>;
-		};
+	pinctrl_weim: weimgrp {
+		fsl,pins = <
+			MX27_PAD_CS4_B__CS4_B		0x0 /* CS4 */
+			MX27_PAD_SD1_D1__GPIO5_19	0x0 /* CAN IRQ */
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-som.dtsi
index e958d7286ae9d..7d5d24c781b98 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx27-phytec-phycore-som.dtsi
@@ -192,90 +192,88 @@ lm75@4a {
 };
 
 &iomuxc {
-	imx27_phycore_som {
-		pinctrl_cspi1: cspi1grp {
-			fsl,pins = <
-				MX27_PAD_CSPI1_MISO__CSPI1_MISO 0x0
-				MX27_PAD_CSPI1_MOSI__CSPI1_MOSI 0x0
-				MX27_PAD_CSPI1_SCLK__CSPI1_SCLK 0x0
-				MX27_PAD_CSPI1_SS0__GPIO4_28	0x0 /* SPI1 CS0 */
-			>;
-		};
+	pinctrl_cspi1: cspi1grp {
+		fsl,pins = <
+			MX27_PAD_CSPI1_MISO__CSPI1_MISO 0x0
+			MX27_PAD_CSPI1_MOSI__CSPI1_MOSI 0x0
+			MX27_PAD_CSPI1_SCLK__CSPI1_SCLK 0x0
+			MX27_PAD_CSPI1_SS0__GPIO4_28	0x0 /* SPI1 CS0 */
+		>;
+	};
 
-		pinctrl_fec1: fec1grp {
-			fsl,pins = <
-				MX27_PAD_SD3_CMD__FEC_TXD0 0x0
-				MX27_PAD_SD3_CLK__FEC_TXD1 0x0
-				MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
-				MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
-				MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
-				MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
-				MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
-				MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
-				MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
-				MX27_PAD_ATA_DATA7__FEC_MDC 0x0
-				MX27_PAD_ATA_DATA8__FEC_CRS 0x0
-				MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
-				MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
-				MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
-				MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
-				MX27_PAD_ATA_DATA13__FEC_COL 0x0
-				MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
-				MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
-				MX27_PAD_SSI3_TXDAT__GPIO3_30	0x0 /* FEC RST */
-			>;
-		};
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX27_PAD_SD3_CMD__FEC_TXD0 0x0
+			MX27_PAD_SD3_CLK__FEC_TXD1 0x0
+			MX27_PAD_ATA_DATA0__FEC_TXD2 0x0
+			MX27_PAD_ATA_DATA1__FEC_TXD3 0x0
+			MX27_PAD_ATA_DATA2__FEC_RX_ER 0x0
+			MX27_PAD_ATA_DATA3__FEC_RXD1 0x0
+			MX27_PAD_ATA_DATA4__FEC_RXD2 0x0
+			MX27_PAD_ATA_DATA5__FEC_RXD3 0x0
+			MX27_PAD_ATA_DATA6__FEC_MDIO 0x0
+			MX27_PAD_ATA_DATA7__FEC_MDC 0x0
+			MX27_PAD_ATA_DATA8__FEC_CRS 0x0
+			MX27_PAD_ATA_DATA9__FEC_TX_CLK 0x0
+			MX27_PAD_ATA_DATA10__FEC_RXD0 0x0
+			MX27_PAD_ATA_DATA11__FEC_RX_DV 0x0
+			MX27_PAD_ATA_DATA12__FEC_RX_CLK 0x0
+			MX27_PAD_ATA_DATA13__FEC_COL 0x0
+			MX27_PAD_ATA_DATA14__FEC_TX_ER 0x0
+			MX27_PAD_ATA_DATA15__FEC_TX_EN 0x0
+			MX27_PAD_SSI3_TXDAT__GPIO3_30	0x0 /* FEC RST */
+		>;
+	};
 
-		pinctrl_i2c2: i2c2grp {
-			fsl,pins = <
-				MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
-				MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
-			>;
-		};
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX27_PAD_I2C2_SDA__I2C2_SDA 0x0
+			MX27_PAD_I2C2_SCL__I2C2_SCL 0x0
+		>;
+	};
 
-		pinctrl_nfc: nfcgrp {
-			fsl,pins = <
-				MX27_PAD_NFRB__NFRB 0x0
-				MX27_PAD_NFCLE__NFCLE 0x0
-				MX27_PAD_NFWP_B__NFWP_B 0x0
-				MX27_PAD_NFCE_B__NFCE_B 0x0
-				MX27_PAD_NFALE__NFALE 0x0
-				MX27_PAD_NFRE_B__NFRE_B 0x0
-				MX27_PAD_NFWE_B__NFWE_B 0x0
-			>;
-		};
+	pinctrl_nfc: nfcgrp {
+		fsl,pins = <
+			MX27_PAD_NFRB__NFRB 0x0
+			MX27_PAD_NFCLE__NFCLE 0x0
+			MX27_PAD_NFWP_B__NFWP_B 0x0
+			MX27_PAD_NFCE_B__NFCE_B 0x0
+			MX27_PAD_NFALE__NFALE 0x0
+			MX27_PAD_NFRE_B__NFRE_B 0x0
+			MX27_PAD_NFWE_B__NFWE_B 0x0
+		>;
+	};
 
-		pinctrl_pmic: pmicgrp {
-			fsl,pins = <
-				MX27_PAD_USB_PWR__GPIO2_23	0x0 /* PMIC IRQ */
-			>;
-		};
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX27_PAD_USB_PWR__GPIO2_23	0x0 /* PMIC IRQ */
+		>;
+	};
 
-		pinctrl_ssi1: ssi1grp {
-			fsl,pins = <
-				MX27_PAD_SSI1_FS__SSI1_FS 0x0
-				MX27_PAD_SSI1_RXDAT__SSI1_RXDAT 0x0
-				MX27_PAD_SSI1_TXDAT__SSI1_TXDAT 0x0
-				MX27_PAD_SSI1_CLK__SSI1_CLK 0x0
-			>;
-		};
+	pinctrl_ssi1: ssi1grp {
+		fsl,pins = <
+			MX27_PAD_SSI1_FS__SSI1_FS 0x0
+			MX27_PAD_SSI1_RXDAT__SSI1_RXDAT 0x0
+			MX27_PAD_SSI1_TXDAT__SSI1_TXDAT 0x0
+			MX27_PAD_SSI1_CLK__SSI1_CLK 0x0
+		>;
+	};
 
-		pinctrl_usbotg: usbotggrp {
-			fsl,pins = <
-				MX27_PAD_USBOTG_CLK__USBOTG_CLK 0x0
-				MX27_PAD_USBOTG_DIR__USBOTG_DIR 0x0
-				MX27_PAD_USBOTG_NXT__USBOTG_NXT 0x0
-				MX27_PAD_USBOTG_STP__USBOTG_STP 0x0
-				MX27_PAD_USBOTG_DATA0__USBOTG_DATA0 0x0
-				MX27_PAD_USBOTG_DATA1__USBOTG_DATA1 0x0
-				MX27_PAD_USBOTG_DATA2__USBOTG_DATA2 0x0
-				MX27_PAD_USBOTG_DATA3__USBOTG_DATA3 0x0
-				MX27_PAD_USBOTG_DATA4__USBOTG_DATA4 0x0
-				MX27_PAD_USBOTG_DATA5__USBOTG_DATA5 0x0
-				MX27_PAD_USBOTG_DATA6__USBOTG_DATA6 0x0
-				MX27_PAD_USBOTG_DATA7__USBOTG_DATA7 0x0
-			>;
-		};
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX27_PAD_USBOTG_CLK__USBOTG_CLK 0x0
+			MX27_PAD_USBOTG_DIR__USBOTG_DIR 0x0
+			MX27_PAD_USBOTG_NXT__USBOTG_NXT 0x0
+			MX27_PAD_USBOTG_STP__USBOTG_STP 0x0
+			MX27_PAD_USBOTG_DATA0__USBOTG_DATA0 0x0
+			MX27_PAD_USBOTG_DATA1__USBOTG_DATA1 0x0
+			MX27_PAD_USBOTG_DATA2__USBOTG_DATA2 0x0
+			MX27_PAD_USBOTG_DATA3__USBOTG_DATA3 0x0
+			MX27_PAD_USBOTG_DATA4__USBOTG_DATA4 0x0
+			MX27_PAD_USBOTG_DATA5__USBOTG_DATA5 0x0
+			MX27_PAD_USBOTG_DATA6__USBOTG_DATA6 0x0
+			MX27_PAD_USBOTG_DATA7__USBOTG_DATA7 0x0
+		>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx27.dtsi b/arch/arm/boot/dts/nxp/imx/imx27.dtsi
index 989b7659b6692..28403a34638e6 100644
--- a/arch/arm/boot/dts/nxp/imx/imx27.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx27.dtsi
@@ -49,7 +49,7 @@ aitc: aitc-interrupt-controller@10040000 {
 
 	clocks {
 		clk_osc26m: osc26m {
-			compatible = "fsl,imx-osc26m", "fixed-clock";
+			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <26000000>;
 		};
@@ -289,7 +289,7 @@ sdhci2: mmc@10014000 {
 				status = "disabled";
 			};
 
-			iomuxc: iomuxc@10015000 {
+			iomuxc: pinmux@10015000 {
 				compatible = "fsl,imx27-iomuxc";
 				reg = <0x10015000 0x600>;
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx31.dtsi b/arch/arm/boot/dts/nxp/imx/imx31.dtsi
index 8541a666747a9..c58f855ea8514 100644
--- a/arch/arm/boot/dts/nxp/imx/imx31.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx31.dtsi
@@ -333,7 +333,7 @@ pwm: pwm@53fe0000 {
 			};
 		};
 
-		emi@b8000000 { /* External Memory Interface */
+		emi-bus@b8000000 { /* External Memory Interface */
 			compatible = "simple-bus";
 			reg = <0xb8000000 0x5000>;
 			ranges;
diff --git a/arch/arm/boot/dts/nxp/imx/imx35.dtsi b/arch/arm/boot/dts/nxp/imx/imx35.dtsi
index 111d7c0331f50..ab7b646399894 100644
--- a/arch/arm/boot/dts/nxp/imx/imx35.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx35.dtsi
@@ -79,7 +79,7 @@ i2c1: i2c@43f80000 {
 				compatible = "fsl,imx35-i2c", "fsl,imx1-i2c";
 				reg = <0x43f80000 0x4000>;
 				clocks = <&clks 51>;
-				clock-names = "ipg_per";
+				clock-names = "ipg";
 				interrupts = <10>;
 				status = "disabled";
 			};
@@ -90,7 +90,7 @@ i2c3: i2c@43f84000 {
 				compatible = "fsl,imx35-i2c", "fsl,imx1-i2c";
 				reg = <0x43f84000 0x4000>;
 				clocks = <&clks 53>;
-				clock-names = "ipg_per";
+				clock-names = "ipg";
 				interrupts = <3>;
 				status = "disabled";
 			};
@@ -119,7 +119,7 @@ i2c2: i2c@43f98000 {
 				compatible = "fsl,imx35-i2c", "fsl,imx1-i2c";
 				reg = <0x43f98000 0x4000>;
 				clocks = <&clks 52>;
-				clock-names = "ipg_per";
+				clock-names = "ipg";
 				interrupts = <4>;
 				status = "disabled";
 			};
@@ -356,7 +356,7 @@ usbmisc: usbmisc@53ff4600 {
 			};
 		};
 
-		emi@80000000 { /* External Memory Interface */
+		emi-bus@80000000 { /* External Memory Interface */
 			compatible = "fsl,emi", "simple-bus";
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -393,21 +393,13 @@ weim: memory-controller@b8002000 {
 		};
 	};
 
-	usbphy {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		usbphy0: usb-phy@0 {
-			reg = <0>;
-			compatible = "usb-nop-xceiv";
-			#phy-cells = <0>;
-		};
+	usbphy0: usb-phy0 {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
+	};
 
-		usbphy1: usb-phy@1 {
-			reg = <1>;
-			compatible = "usb-nop-xceiv";
-			#phy-cells = <0>;
-		};
+	usbphy1: usb-phy1 {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx51-babbage.dts b/arch/arm/boot/dts/nxp/imx/imx51-babbage.dts
index 1b6ec55f90683..b17264e06e691 100644
--- a/arch/arm/boot/dts/nxp/imx/imx51-babbage.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx51-babbage.dts
@@ -327,7 +327,7 @@ vgen3_reg: vgen3 {
 		};
 	};
 
-	flash: at45db321d@1 {
+	flash: flash@1 {
 		#address-cells = <1>;
 		#size-cells = <1>;
 		compatible = "atmel,at45db321d", "atmel,at45", "atmel,dataflash";
diff --git a/arch/arm/boot/dts/nxp/imx/imx51-ts4800.dts b/arch/arm/boot/dts/nxp/imx/imx51-ts4800.dts
index 079bd3d14999f..5118a68dbbdc3 100644
--- a/arch/arm/boot/dts/nxp/imx/imx51-ts4800.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx51-ts4800.dts
@@ -141,7 +141,7 @@ &weim {
 	pinctrl-0 = <&pinctrl_weim>;
 	status = "okay";
 
-	fpga@0 {
+	fpga-bus@0,0 {
 		compatible = "simple-bus";
 		fsl,weim-cs-timing = <0x0061008F 0x00000002 0x1c022000
 				      0x00000000 0x1c092480 0x00000000>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts b/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
index 6831836bd726c..e7e15d2b82c44 100644
--- a/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
@@ -13,7 +13,7 @@ / {
 
 &i2c1 {
 	pmic: dialog@48 {
-		compatible = "dlg,da9053-aa", "dlg,da9052";
+		compatible = "dlg,da9053-aa";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
diff --git a/arch/arm/boot/dts/nxp/imx/imx53-smd.dts b/arch/arm/boot/dts/nxp/imx/imx53-smd.dts
index 386371c816f4d..a1e19f9709b2c 100644
--- a/arch/arm/boot/dts/nxp/imx/imx53-smd.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx53-smd.dts
@@ -31,6 +31,20 @@ key-volume-down {
 			linux,code = <KEY_VOLUMEDOWN>;
 		};
 	};
+
+	reg_1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	reg_3v3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
 };
 
 &esdhc1 {
@@ -253,6 +267,10 @@ &i2c2 {
 	codec: sgtl5000@a {
 		compatible = "fsl,sgtl5000";
 		reg = <0x0a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clks IMX5_CLK_DUMMY>;
+		VDDA-supply = <&reg_1v8>;
+		VDDIO-supply = <&reg_3v3>;
 	};
 
 	magnetometer: mag3110@e {
diff --git a/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi b/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
index 6dc70a92d831b..6eed683d92a83 100644
--- a/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
@@ -128,7 +128,7 @@ &i2c1 {
 	status = "okay";
 
 	pmic: dialog@48 {
-		compatible = "dlg,da9053-aa", "dlg,da9052";
+		compatible = "dlg,da9053-aa";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
diff --git a/arch/arm/boot/dts/nxp/imx/imx6-logicpd-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6-logicpd-som.dtsi
index f452764fae00e..547fb141ec0c9 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6-logicpd-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6-logicpd-som.dtsi
@@ -36,12 +36,8 @@ &clks {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c3 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-alti6p.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-alti6p.dts
index 9bb36db131c21..aed4fb7843e2f 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-alti6p.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-alti6p.dts
@@ -193,8 +193,8 @@ &can1 {
 };
 
 &clks {
-	clocks = <&clock_ksz8081>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clock_ksz8081>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clock_ksz8081>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-eckelmann-ci4x10.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-eckelmann-ci4x10.dts
index 5ed55f74b398f..5f61eeb9fad0c 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-eckelmann-ci4x10.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-eckelmann-ci4x10.dts
@@ -66,8 +66,8 @@ &can2 {
 };
 
 &clks {
-	clocks = <&rmii_clk>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&rmii_clk>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&rmii_clk>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-lanmcu.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-lanmcu.dts
index 47a6d63c8e04c..9bde654625582 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-lanmcu.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-lanmcu.dts
@@ -119,8 +119,8 @@ &can2 {
 };
 
 &clks {
-	clocks = <&clock_ksz8081>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clock_ksz8081>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clock_ksz8081>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-plybas.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-plybas.dts
index 84f34da06267e..69e790ba56624 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-plybas.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-plybas.dts
@@ -101,8 +101,8 @@ &can2 {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clk50m_phy>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
index 0ef24a07dedf9..fbff77944ce32 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
@@ -199,8 +199,8 @@ &can1 {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clk50m_phy>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-prtmvt.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-prtmvt.dts
index 2160b71778355..dcd5a4099c602 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-prtmvt.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-prtmvt.dts
@@ -294,8 +294,8 @@ &can2 {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>, <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL5_VIDEO_DIV>, <&clk50m_phy>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-qmx6.dtsi b/arch/arm/boot/dts/nxp/imx/imx6dl-qmx6.dtsi
index d5baec5e7a782..fb674ac2c2485 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-qmx6.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-qmx6.dtsi
@@ -71,8 +71,9 @@ IMX_AUDMUX_V2_PDCR_RXDSEL(MX51_AUDMUX_PORT1_SSI0)
 };
 
 &clks {
-	clocks = <&rtc_sqw>;
-	clock-names = "ckil";
+	clocks = <&osc>, <&rtc_sqw>, <&ckih1>, <&anaclk1>, <&anaclk2>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2";
+
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
 			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL2_PFD0_352M>,
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
index 76b0007d20ad2..18019a6bb3af0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
@@ -160,8 +160,8 @@ vdiv_hitch_pos: voltage-divider-hitch-pos {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clk50m_phy>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-b450v3.dts b/arch/arm/boot/dts/nxp/imx/imx6q-b450v3.dts
index d994b32ad825c..7326b8ad08cb5 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-b450v3.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-b450v3.dts
@@ -140,6 +140,7 @@ port@3 {
 	port@4 {
 		reg = <4>;
 		label = "cpu";
+		phy-mode = "gmii";
 		ethernet = <&switch_nic>;
 		phy-handle = <&switchphy4>;
 	};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-b650v3.dts b/arch/arm/boot/dts/nxp/imx/imx6q-b650v3.dts
index b0d345f5d0712..9a43935be11ca 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-b650v3.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-b650v3.dts
@@ -139,6 +139,7 @@ port@3 {
 	port@4 {
 		reg = <4>;
 		label = "cpu";
+		phy-mode = "gmii";
 		ethernet = <&switch_nic>;
 		phy-handle = <&switchphy4>;
 	};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-b850v3.dts b/arch/arm/boot/dts/nxp/imx/imx6q-b850v3.dts
index cad112e054758..d00236a07dc4b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-b850v3.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-b850v3.dts
@@ -53,23 +53,20 @@ chosen {
 	};
 };
 
-&ldb {
-	fsl,dual-channel;
-	status = "okay";
-
-	lvds0: lvds-channel@0 {
-		fsl,data-mapping = "spwg";
-		fsl,data-width = <24>;
-		status = "okay";
-
-		port@4 {
-			reg = <4>;
+&gpio4 {
+	gpio-line-names =
+		"", "", "", "",
+		"", "", "", "",
+		"", "", "", "",
+		"", "", "PWGIN", "",
+		"", "", "", "",
+		"", "", "", "",
+		"", "", "", "",
+		"", "", "", "";
+};
 
-			lvds0_out: endpoint {
-				remote-endpoint = <&stdp4028_in>;
-			};
-		};
-	};
+&hdmi {
+	ddc-i2c-bus = <&mux2_i2c1>;
 };
 
 &i2c2 {
@@ -129,8 +126,23 @@ mux2_i2c8: i2c@7 {
 	};
 };
 
-&hdmi {
-	ddc-i2c-bus = <&mux2_i2c1>;
+&ldb {
+	fsl,dual-channel;
+	status = "okay";
+
+	lvds0: lvds-channel@0 {
+		fsl,data-mapping = "spwg";
+		fsl,data-width = <24>;
+		status = "okay";
+
+		port@4 {
+			reg = <4>;
+
+			lvds0_out: endpoint {
+				remote-endpoint = <&stdp4028_in>;
+			};
+		};
+	};
 };
 
 &mux1_i2c1 {
@@ -286,7 +298,12 @@ port@3 {
 	port@4 {
 		reg = <4>;
 		label = "cpu";
+		phy-mode = "gmii";
 		ethernet = <&switch_nic>;
 		phy-handle = <&switchphy4>;
 	};
 };
+
+&usdhc4 {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-bx50v3.dtsi b/arch/arm/boot/dts/nxp/imx/imx6q-bx50v3.dtsi
index 1e2266a2368be..2c8d2ab8cda13 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-bx50v3.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-bx50v3.dtsi
@@ -124,24 +124,28 @@ switchphy0: switchphy@0 {
 					reg = <0>;
 					interrupt-parent = <&switch>;
 					interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
+					max-speed = <100>; /* only 100Mbit/s lanes are routed */
 				};
 
 				switchphy1: switchphy@1 {
 					reg = <1>;
 					interrupt-parent = <&switch>;
 					interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+					max-speed = <100>; /* only 100Mbit/s lanes are routed */
 				};
 
 				switchphy2: switchphy@2 {
 					reg = <2>;
 					interrupt-parent = <&switch>;
 					interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
+					max-speed = <100>; /* only 100Mbit/s lanes are routed */
 				};
 
 				switchphy3: switchphy@3 {
 					reg = <3>;
 					interrupt-parent = <&switch>;
 					interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
+					max-speed = <100>; /* only 100Mbit/s lanes are routed */
 				};
 
 				switchphy4: switchphy@4 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-prtwd2.dts b/arch/arm/boot/dts/nxp/imx/imx6q-prtwd2.dts
index 0e02e448db108..5ef1ce8086991 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-prtwd2.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-prtwd2.dts
@@ -57,8 +57,8 @@ &can1 {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clk50m_phy>;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-icore.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-icore.dtsi
index 58ecdb87c6d40..9975b6ee433d1 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-icore.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-icore.dtsi
@@ -172,12 +172,8 @@ eth_phy: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-mba6.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-mba6.dtsi
index ee2c6bec92e8e..74ccfe56828fa 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-mba6.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-mba6.dtsi
@@ -206,6 +206,10 @@ ethphy: ethernet-phy@3 {
 	};
 };
 
+&gpio1 {
+	bootph-pre-ram;
+};
+
 &hdmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_hdmi>;
@@ -214,6 +218,8 @@ &hdmi {
 };
 
 &i2c1 {
+	bootph-pre-ram;
+
 	tlv320aic32x4: audio-codec@18 {
 		compatible = "ti,tlv320aic32x4";
 		reg = <0x18>;
@@ -274,6 +280,7 @@ &ssi1 {
 &uart2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart2>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -346,6 +353,7 @@ &usdhc2 {
 	no-sdio;
 	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;
+	bootph-all;
 	status = "okay";
 };
 
@@ -354,6 +362,7 @@ &wdog1 {
 	pinctrl-0 = <&pinctrl_wdog1>;
 	/* does not work on unmodified starter kit */
 	/* fsl,ext-reset-output; */
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -544,6 +553,7 @@ pinctrl_uart2: uart2grp {
 			MX6QDL_PAD_SD4_DAT4__UART2_RX_DATA 0x1b099
 			MX6QDL_PAD_SD4_DAT7__UART2_TX_DATA 0x1b099
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_uart3: uart3grp {
@@ -587,6 +597,7 @@ MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x00017059
 			MX6QDL_PAD_GPIO_4__GPIO1_IO04  0x0001b099 /* usdhc2 CD */
 			MX6QDL_PAD_GPIO_2__GPIO1_IO02  0x0001b099 /* usdhc2 WP */
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usbotg: usbotggrp {
@@ -602,5 +613,6 @@ pinctrl_wdog1: wdog1grp {
 			 /* Watchdog out */
 			MX6QDL_PAD_SD1_DAT2__WDOG1_B 0x0000b099
 		>;
+		bootph-pre-ram;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-pfla02.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-pfla02.dtsi
index 6f3becd33a5b5..aa9a442852f41 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-pfla02.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-pfla02.dtsi
@@ -102,12 +102,8 @@ ethphy: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-phycore-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-phycore-som.dtsi
index f2140dd8525f8..85e278eb20161 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-phycore-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-phytec-phycore-som.dtsi
@@ -73,12 +73,8 @@ ethphy: ethernet-phy@3 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "disabled";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c3 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
index 131a3428ddb86..e2631db798a8f 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
@@ -238,8 +238,8 @@ adc: adc@0 {
 };
 
 &clks {
-	clocks = <&clk50m_phy>;
-	clock-names = "enet_ref_pad";
+	clocks = <&osc>, <&ckil>, <&ckih1>, <&anaclk1>, <&anaclk2>, <&clk50m_phy>;
+	clock-names = "osc", "ckil", "ckih1", "anaclk1", "anaclk2", "enet_ref_pad";
 	assigned-clocks = <&clks IMX6QDL_CLK_ENET_REF_SEL>;
 	assigned-clock-parents = <&clk50m_phy>;
 };
@@ -260,14 +260,10 @@ fixed-link {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	#address-cells = <1>;
 	#size-cells = <0>;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c3 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-sr-som-ti.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-sr-som-ti.dtsi
index cd1e682f11add..8192344d52946 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-sr-som-ti.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-sr-som-ti.dtsi
@@ -41,6 +41,12 @@
 #include <dt-bindings/gpio/gpio.h>
 
 / {
+	clk32k: clock-32k {
+		compatible = "fixed-clock";
+		clock-frequency = <32768>;
+		#clock-cells = <0>;
+	};
+
 	nvcc_sd1: regulator-nvcc-sd1 {
 		compatible = "regulator-fixed";
 		regulator-always-on;
@@ -58,7 +64,7 @@ clk_ti_wifi: ti-wifi-clock {
 		 */
 		compatible = "gpio-gate-clock";
 		#clock-cells = <0>;
-		clock-frequency = <32768>;
+		clocks = <&clk32k>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_microsom_ti_clk>;
 		enable-gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6.dtsi
index 07492f63a1f80..14676d1d905a1 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6.dtsi
@@ -11,6 +11,7 @@ &ecspi1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_ecspi1>;
 	cs-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
+	bootph-pre-ram;
 	status = "okay";
 
 	m25p80: flash@0 {
@@ -19,6 +20,7 @@ m25p80: flash@0 {
 		spi-max-frequency = <50000000>;
 		vcc-supply = <&sw4_reg>;
 		m25p,fast-read;
+		bootph-pre-ram;
 
 		partitions {
 			compatible = "fixed-partitions";
@@ -28,6 +30,10 @@ partitions {
 	};
 };
 
+&gpio3 {
+	bootph-pre-ram;
+};
+
 &iomuxc {
 	pinctrl_ecspi1: ecspi1grp {
 		fsl,pins = <
@@ -38,6 +44,7 @@ MX6QDL_PAD_EIM_D16__ECSPI1_SCLK 0xb099
 			 /* eCSPI1 SS1 */
 			MX6QDL_PAD_EIM_D19__GPIO3_IO19 0xb099
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_i2c1: i2c1grp {
@@ -45,6 +52,7 @@ pinctrl_i2c1: i2c1grp {
 			MX6QDL_PAD_CSI0_DAT8__I2C1_SDA 0x4001b899
 			MX6QDL_PAD_CSI0_DAT9__I2C1_SCL 0x4001b899
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_i2c1_recovery: i2c1recoverygrp {
@@ -73,6 +81,7 @@ MX6QDL_PAD_SD3_DAT5__SD3_DATA5 0x17059
 			MX6QDL_PAD_SD3_DAT6__SD3_DATA6 0x17059
 			MX6QDL_PAD_SD3_DAT7__SD3_DATA7 0x17059
 		>;
+		bootph-all;
 	};
 };
 
@@ -117,6 +126,7 @@ sw4_reg: sw4 {
 			regulator-min-microvolt = <3300000>;
 			regulator-max-microvolt = <3300000>;
 			regulator-always-on;
+			bootph-pre-ram;
 		};
 
 		reg_5v_600mA: swbst {
@@ -186,6 +196,7 @@ &usdhc3 {
 	bus-width = <8>;
 	#address-cells = <1>;
 	#size-cells = <0>;
+	bootph-all;
 	status = "okay";
 
 	mmccard: mmccard@0 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
index e8fd37dd88355..67f8f59aff5ae 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
@@ -20,6 +20,7 @@ &i2c1 {
 	scl-gpios = <&gpio5 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	sda-gpios = <&gpio5 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	clock-frequency = <100000>;
+	bootph-pre-ram;
 	status = "okay";
 
 	pmic: pmic@8 {
@@ -27,13 +28,13 @@ pmic: pmic@8 {
 		reg = <0x08>;
 	};
 
-	temperature-sensor@48 {
+	lm75_48: temperature-sensor@48 {
 		compatible = "national,lm75a";
 		reg = <0x48>;
 		vs-supply = <&sw4_reg>;
 	};
 
-	eeprom@50 {
+	m24c64_50: eeprom@50 {
 		compatible = "st,24c64", "atmel,24c64";
 		reg = <0x50>;
 		pagesize = <32>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
index 0e404c1f62f26..db552802554d4 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
@@ -13,6 +13,7 @@ &i2c3 {
 	scl-gpios = <&gpio1 5 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	sda-gpios = <&gpio1 6 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	clock-frequency = <100000>;
+	bootph-pre-ram;
 	status = "okay";
 
 	pmic: pmic@8 {
@@ -20,13 +21,13 @@ pmic: pmic@8 {
 		reg = <0x08>;
 	};
 
-	temperature-sensor@48 {
+	lm75_48: temperature-sensor@48 {
 		compatible = "national,lm75a";
 		reg = <0x48>;
 		vs-supply = <&sw4_reg>;
 	};
 
-	eeprom@50 {
+	m24c64_50: eeprom@50 {
 		compatible = "st,24c64", "atmel,24c64";
 		reg = <0x50>;
 		pagesize = <32>;
@@ -40,6 +41,7 @@ pinctrl_i2c3: i2c3grp {
 			MX6QDL_PAD_GPIO_5__I2C3_SCL 0x4001b899
 			MX6QDL_PAD_GPIO_6__I2C3_SDA 0x4001b899
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_i2c3_recovery: i2c3recoverygrp {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tx6.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tx6.dtsi
index d29adfef5fdba..57297d6521cf0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tx6.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tx6.dtsi
@@ -252,13 +252,9 @@ etnphy: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	fsl,no-blockmark-swap;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
index 76e6043e1f918..4dc2c410cf613 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
@@ -54,23 +54,35 @@ aliases {
 	};
 
 	clocks {
-		ckil {
+		ckil: ckil {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <32768>;
 		};
 
-		ckih1 {
+		ckih1: ckih1 {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <0>;
 		};
 
-		osc {
+		osc: osc {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <24000000>;
 		};
+
+		anaclk1: anaclk1 {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <0>;
+		};
+
+		anaclk2: anaclk2 {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <0>;
+		};
 	};
 
 	ldb: ldb {
@@ -149,6 +161,7 @@ soc: soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gpc>;
 		ranges;
+		bootph-all;
 
 		dma_apbh: dma-controller@110000 {
 			compatible = "fsl,imx6q-dma-apbh", "fsl,imx28-dma-apbh";
@@ -297,6 +310,7 @@ aips1: bus@2000000 { /* AIPS1 */
 			#size-cells = <1>;
 			reg = <0x02000000 0x100000>;
 			ranges;
+			bootph-pre-ram;
 
 			spba-bus@2000000 {
 				compatible = "fsl,spba-bus", "simple-bus";
@@ -304,6 +318,7 @@ spba-bus@2000000 {
 				#size-cells = <1>;
 				reg = <0x02000000 0x40000>;
 				ranges;
+				bootph-pre-ram;
 
 				spdif: spdif@2004000 {
 					compatible = "fsl,imx35-spdif";
@@ -920,6 +935,7 @@ mux: mux-controller {
 			iomuxc: pinctrl@20e0000 {
 				compatible = "fsl,imx6dl-iomuxc", "fsl,imx6q-iomuxc";
 				reg = <0x20e0000 0x4000>;
+				bootph-pre-ram;
 			};
 
 			dcic1: dcic@20e4000 {
@@ -950,6 +966,7 @@ aips2: bus@2100000 { /* AIPS2 */
 			#size-cells = <1>;
 			reg = <0x02100000 0x100000>;
 			ranges;
+			bootph-pre-ram;
 
 			crypto: crypto@2100000 {
 				compatible = "fsl,sec-v4.0";
@@ -1320,6 +1337,7 @@ ipu1: ipu@2400000 {
 				 <&clks IMX6QDL_CLK_IPU1_DI1>;
 			clock-names = "bus", "di0", "di1";
 			resets = <&src 2>;
+			bootph-all;
 
 			ipu1_csi0: port@0 {
 				reg = <0>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-geam.dts b/arch/arm/boot/dts/nxp/imx/imx6ul-geam.dts
index 40d530c1dc29f..2a6bb5ff808ad 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-geam.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-geam.dts
@@ -133,12 +133,8 @@ ethphy1: ethernet-phy@1 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-isiot.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-isiot.dtsi
index 776f6f78ee463..e34c8cbe36aec 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-isiot.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-isiot.dtsi
@@ -101,12 +101,8 @@ ethphy0: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "disabled";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-phytec-phycore-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-phytec-phycore-som.dtsi
index 27e4d2aec137f..a3ea1b2084625 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-phytec-phycore-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-phytec-phycore-som.dtsi
@@ -63,12 +63,8 @@ ethphy1: ethernet-phy@1 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "disabled";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
index 2dd635a615cb8..4fa98e6a66d7c 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
@@ -26,6 +26,7 @@ &i2c4 {
 	pinctrl-1 = <&pinctrl_i2c4_recovery>;
 	scl-gpios = <&gpio1 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	sda-gpios = <&gpio1 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	bootph-pre-ram;
 	status = "okay";
 
 	pfuze3000: pmic@8 {
@@ -140,9 +141,14 @@ rtc0: rtc@68 {
 	};
 };
 
+&gpio1 {
+	bootph-pre-ram;
+};
+
 &gpio4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_pmic>;
+	bootph-pre-ram;
 
 	/*
 	 * PMIC & temperature sensor IRQ
@@ -159,6 +165,7 @@ pmic-int-hog {
 &qspi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_qspi>;
+	bootph-pre-ram;
 	status = "okay";
 
 	flash0: flash@0 {
@@ -168,6 +175,7 @@ flash0: flash@0 {
 		spi-rx-bus-width = <4>;
 		spi-tx-bus-width = <1>;
 		vcc-supply = <&reg_vldo4>;
+		bootph-pre-ram;
 
 		partitions {
 			compatible = "fixed-partitions";
@@ -189,6 +197,7 @@ &usdhc2 {
 	non-removable;
 	no-sdio;
 	no-sd;
+	bootph-all;
 	status = "okay";
 };
 
@@ -212,5 +221,6 @@ pinctrl_pmic: pmicgrp {
 			/* PMIC irq */
 			MX6UL_PAD_CSI_DATA03__GPIO4_IO24	0x1b099
 		>;
+		bootph-pre-ram;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2.dtsi
index e2e95dd92263e..f81cd09fe0c7f 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2.dtsi
@@ -33,6 +33,7 @@ MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x00017051
 			/* rst */
 			MX6UL_PAD_NAND_ALE__GPIO4_IO10		0x0001b051
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l-mba6ulx.dts b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l-mba6ulx.dts
index 9d9b6b744a1c0..9d637c0a12ece 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l-mba6ulx.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l-mba6ulx.dts
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright 2018-2022 TQ Systems GmbH
+ * Copyright 2018-2022 TQ-Systems GmbH
  * Author: Markus Niebel <Markus.Niebel@tq-group.com>
  */
 
@@ -10,6 +10,6 @@
 #include "mba6ulx.dtsi"
 
 / {
-	model = "TQ Systems TQMa6UL2L SoM on MBa6ULx board";
+	model = "TQ-Systems TQMa6UL2L SoM on MBa6ULx board";
 	compatible = "tq,imx6ul-tqma6ul2l-mba6ulx", "tq,imx6ul-tqma6ul2l", "fsl,imx6ul";
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l.dtsi
index 4b87e2dc70dcd..11c8f1af41732 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul2l.dtsi
@@ -33,6 +33,7 @@ MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x00017051
 			/* rst */
 			MX6UL_PAD_NAND_ALE__GPIO4_IO10		0x0001b051
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulx-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulx-common.dtsi
index 5afb9046c202a..5c90d0a3ee2e7 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulx-common.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulx-common.dtsi
@@ -39,5 +39,6 @@ MX6UL_PAD_NAND_CE1_B__QSPI_A_DATA02   0x70b9
 			MX6UL_PAD_NAND_CLE__QSPI_A_DATA03     0x70b9
 			MX6UL_PAD_NAND_DQS__QSPI_A_SS0_B      0x70a1
 		>;
+		bootph-pre-ram;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulxl-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulxl-common.dtsi
index ba84a4f70ebde..133961ee72831 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulxl-common.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ulxl-common.dtsi
@@ -44,5 +44,6 @@ MX6UL_PAD_NAND_CE1_B__QSPI_A_DATA02   0x70a9
 			MX6UL_PAD_NAND_CLE__QSPI_A_DATA03     0x70a9
 			MX6UL_PAD_NAND_DQS__QSPI_A_SS0_B      0x70a1
 		>;
+		bootph-pre-ram;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
index dc53f9286ffe2..1992dfb53b45c 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
@@ -296,13 +296,9 @@ &fec2 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	fsl,no-blockmark-swap;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &i2c2 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
index 6eb80f867f501..24541fdf49ceb 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
@@ -115,6 +115,7 @@ osc: clock-osc {
 		#clock-cells = <0>;
 		clock-frequency = <24000000>;
 		clock-output-names = "osc";
+		bootph-pre-ram;
 	};
 
 	ipp_di0: clock-di0 {
@@ -143,6 +144,7 @@ soc: soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gpc>;
 		ranges;
+		bootph-pre-ram;
 
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
@@ -202,6 +204,7 @@ aips1: bus@2000000 {
 			#size-cells = <1>;
 			reg = <0x02000000 0x100000>;
 			ranges;
+			bootph-pre-ram;
 
 			spba-bus@2000000 {
 				compatible = "fsl,spba-bus", "simple-bus";
@@ -209,6 +212,7 @@ spba-bus@2000000 {
 				#size-cells = <1>;
 				reg = <0x02000000 0x40000>;
 				ranges;
+				bootph-pre-ram;
 
 				ecspi1: spi@2008000 {
 					#address-cells = <1>;
@@ -580,6 +584,7 @@ clks: clock-controller@20c4000 {
 				#clock-cells = <1>;
 				clocks = <&ckil>, <&osc>, <&ipp_di0>, <&ipp_di1>;
 				clock-names = "ckil", "osc", "ipp_di0", "ipp_di1";
+				bootph-pre-ram;
 			};
 
 			anatop: anatop@20c8000 {
@@ -745,6 +750,7 @@ power-domain@0 {
 			iomuxc: pinctrl@20e0000 {
 				compatible = "fsl,imx6ul-iomuxc";
 				reg = <0x020e0000 0x4000>;
+				bootph-pre-ram;
 			};
 
 			gpr: iomuxc-gpr@20e4000 {
@@ -826,6 +832,7 @@ aips2: bus@2100000 {
 			#size-cells = <1>;
 			reg = <0x02100000 0x100000>;
 			ranges;
+			bootph-pre-ram;
 
 			crypto: crypto@2140000 {
 				compatible = "fsl,imx6ul-caam", "fsl,sec-v4.0";
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-colibri.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-colibri.dtsi
index eaed2cbf0c82a..ec3c1e7301f48 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-colibri.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-colibri.dtsi
@@ -160,15 +160,11 @@ &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
 	fsl,use-minimum-ecc;
+	nand-on-flash-bbt;
+	nand-ecc-mode = "hw";
+	nand-ecc-strength = <8>;
+	nand-ecc-step-size = <512>;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-		nand-ecc-mode = "hw";
-		nand-ecc-strength = <8>;
-		nand-ecc-step-size = <512>;
-	};
 };
 
 /* I2C3_SDA/SCL on SODIMM 194/196 (e.g. RTC on carrier board) */
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-dhcor-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-dhcor-som.dtsi
index 75486e1b0c15f..a0adcd3fe1229 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-dhcor-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-dhcor-som.dtsi
@@ -201,7 +201,7 @@ &uart2 {
 	 * the speed can be increased accordingly.
 	 */
 	bluetooth: bluetooth {
-		compatible = "brcm,bcm43430a1-bt"; /* muRata 1DX */
+		compatible = "brcm,bcm4329-bt"; /* muRata 1DX or 1YN */
 		max-speed = <3000000>;
 		vbat-supply = <&vcc_3v3>;
 		vddio-supply = <&vcc_3v3>;
@@ -222,7 +222,7 @@ &usdhc1 {
 	status = "okay";
 
 	brcmf: wifi@1 {
-		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac"; /* muRata 1DX */
+		compatible = "brcm,bcm4329-fmac"; /* muRata 1DX or 1YN */
 		reg = <1>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea-bmm.dts b/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea-bmm.dts
index 279d46c22cd79..f251a10283552 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea-bmm.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea-bmm.dts
@@ -154,6 +154,7 @@ &tsc {
 	pinctrl-0 = <&pinctrl_tsc>;
 	measure-delay-time = <0x9ffff>;
 	pre-charge-time = <0xfff>;
+	debounce-delay-us = <62>;
 	xnur-gpios = <&gpio1 3 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea.dtsi
index 3dfd43b320553..43518bf076024 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-engicam-microgea.dtsi
@@ -43,15 +43,11 @@ ethphy0: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-ecc-mode = "hw";
+	nand-ecc-strength = <0>;
+	nand-ecc-step-size = <0>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-ecc-mode = "hw";
-		nand-ecc-strength = <0>;
-		nand-ecc-step-size = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &iomuxc {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-myir-mys-6ulx.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-myir-mys-6ulx.dtsi
index fc298f57bfff3..83b9de17cee2d 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-myir-mys-6ulx.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-myir-mys-6ulx.dtsi
@@ -60,12 +60,8 @@ ethphy0: ethernet-phy@0 {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "disabled";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &uart1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2.dtsi
index 8541cb3f3b3e2..1224ef1324397 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2.dtsi
@@ -38,6 +38,7 @@ MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x00017039
 			/* rst */
 			MX6UL_PAD_NAND_ALE__GPIO4_IO10		0x0001b051
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l-mba6ulx.dts b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l-mba6ulx.dts
index 33437aae98224..5676904820a93 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l-mba6ulx.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l-mba6ulx.dts
@@ -10,6 +10,6 @@
 #include "mba6ulx.dtsi"
 
 / {
-	model = "TQ Systems TQMa6ULL2L SoM on MBa6ULx board";
+	model = "TQ-Systems TQMa6ULL2L SoM on MBa6ULx board";
 	compatible = "tq,imx6ull-tqma6ull2l-mba6ulx", "tq,imx6ull-tqma6ull2l", "fsl,imx6ull";
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l.dtsi
index be593d47e3b1e..ac18caf5a76c8 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-tqma6ull2l.dtsi
@@ -9,7 +9,7 @@
 #include "imx6ul-tqma6ulxl-common.dtsi"
 
 / {
-	model = "TQ Systems TQMa6ULL2L SoM";
+	model = "TQ-Systems TQMa6ULL2L SoM";
 	compatible = "tq,imx6ull-tqma6ull2l", "fsl,imx6ull";
 };
 
@@ -38,6 +38,7 @@ MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x00017039
 			/* rst */
 			MX6UL_PAD_NAND_ALE__GPIO4_IO10		0x0001b051
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull.dtsi
index db0c339022acc..ba0ea10c7b74f 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ull.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull.dtsi
@@ -57,6 +57,7 @@ aips3: bus@2200000 {
 			#size-cells = <1>;
 			reg = <0x02200000 0x100000>;
 			ranges;
+			bootph-pre-ram;
 
 			dcp: crypto@2280000 {
 				compatible = "fsl,imx6ull-dcp", "fsl,imx28-dcp";
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ulz-bsh-smm-m2.dts b/arch/arm/boot/dts/nxp/imx/imx6ulz-bsh-smm-m2.dts
index 8ec18eae98a46..2d9f495660c9a 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ulz-bsh-smm-m2.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ulz-bsh-smm-m2.dts
@@ -25,12 +25,8 @@ usdhc2_pwrseq: usdhc2-pwrseq {
 &gpmi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
 	status = "okay";
-
-	nand@0 {
-		reg = <0>;
-		nand-on-flash-bbt;
-	};
 };
 
 &snvs_poweroff {
diff --git a/arch/arm/boot/dts/nxp/imx/imx7-colibri.dtsi b/arch/arm/boot/dts/nxp/imx/imx7-colibri.dtsi
index a41dc4edfc0dd..8666dcd7fe974 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7-colibri.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7-colibri.dtsi
@@ -375,14 +375,10 @@ &gpio7 {
 /* NAND on such SKUs */
 &gpmi {
 	fsl,use-minimum-ecc;
+	nand-ecc-mode = "hw";
+	nand-on-flash-bbt;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpmi_nand>;
-
-	nand@0 {
-		reg = <0>;
-		nand-ecc-mode = "hw";
-		nand-on-flash-bbt;
-	};
 };
 
 /* On-module Power I2C */
diff --git a/arch/arm/boot/dts/nxp/imx/imx7-mba7.dtsi b/arch/arm/boot/dts/nxp/imx/imx7-mba7.dtsi
index 4d948a9757f9f..4192adb27223b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7-mba7.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7-mba7.dtsi
@@ -287,6 +287,15 @@ &flexcan2 {
 	status = "okay";
 };
 
+&gpio4 {
+	/* Deassert BOOT_EN after boot to separate BOOT_CFG circuits from LCD signals */
+	boot-en-hog {
+		gpio-hog;
+		gpios = <3 GPIO_ACTIVE_LOW>;
+		output-low;
+	};
+};
+
 &i2c1 {
 	lm75: temperature-sensor@49 {
 		compatible = "national,lm75a";
@@ -477,6 +486,7 @@ pinctrl_uart5: uart5grp {
 	};
 
 	pinctrl_uart6: uart6grp {
+		bootph-pre-ram;
 		fsl,pins =
 			<MX7D_PAD_EPDC_DATA08__UART6_DCE_RX	0x7d>,
 			<MX7D_PAD_EPDC_DATA09__UART6_DCE_TX	0x75>,
@@ -511,6 +521,7 @@ pinctrl_usdhc1: usdhc1grp {
 			<MX7D_PAD_SD1_DATA1__SD1_DATA1		0x5e>,
 			<MX7D_PAD_SD1_DATA2__SD1_DATA2		0x5e>,
 			<MX7D_PAD_SD1_DATA3__SD1_DATA3		0x5e>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc1_100mhz: usdhc1_100mhzgrp {
@@ -602,6 +613,7 @@ &uart6 {
 	pinctrl-0 = <&pinctrl_uart6>;
 	assigned-clocks = <&clks IMX7D_UART6_ROOT_SRC>;
 	assigned-clock-parents = <&clks IMX7D_OSC_24M_CLK>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -646,6 +658,7 @@ &usdhc1 {
 	no-1-8-v;
 	no-sdio;
 	no-mmc;
+	bootph-pre-ram;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi b/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
index 2966a33bc528a..f3d7a2d0cb7b8 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
@@ -193,6 +193,7 @@ pinctrl_qspi: qspigrp {
 			<MX7D_PAD_EPDC_DATA05__QSPI_A_SCLK	0x11>,
 			<MX7D_PAD_EPDC_DATA06__QSPI_A_SS0_B	0x54>,
 			<MX7D_PAD_EPDC_DATA07__QSPI_A_SS1_B	0x54>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_qspi_reset: qspi_resetgrp {
@@ -214,6 +215,7 @@ pinctrl_usdhc3: usdhc3grp {
 			<MX7D_PAD_SD3_DATA6__SD3_DATA6		0x59>,
 			<MX7D_PAD_SD3_DATA7__SD3_DATA7		0x59>,
 			<MX7D_PAD_SD3_STROBE__SD3_STROBE	0x19>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc3_100mhz: usdhc3_100mhzgrp {
@@ -257,6 +259,7 @@ pinctrl_wdog1: wdog1grp {
 &qspi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_qspi &pinctrl_qspi_reset>;
+	bootph-pre-ram;
 	status = "okay";
 
 	flash0: flash@0 {
@@ -266,6 +269,7 @@ flash0: flash@0 {
 		spi-rx-bus-width = <4>;
 		spi-tx-bus-width = <4>;
 		vcc-supply = <&vgen4_reg>;
+		bootph-pre-ram;
 
 		partitions {
 			compatible = "fixed-partitions";
@@ -288,6 +292,7 @@ &usdhc3 {
 	no-sdio;
 	vmmc-supply = <&vgen4_reg>;
 	vqmmc-supply = <&sw2_reg>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts b/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
index 92b6258059ee0..25f38acc53501 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
@@ -199,8 +199,6 @@ ov2680: camera@36 {
 		port {
 			ov2680_to_mipi: endpoint {
 				remote-endpoint = <&mipi_from_sensor>;
-				clock-lanes = <0>;
-				data-lanes = <1>;
 				link-frequencies = /bits/ 64 <330000000>;
 			};
 		};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7s.dtsi b/arch/arm/boot/dts/nxp/imx/imx7s.dtsi
index 9235dd7e93bb7..5c9be24ba7c5b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7s.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7s.dtsi
@@ -209,6 +209,7 @@ soc: soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gpc>;
 		ranges;
+		bootph-pre-ram;
 
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
@@ -371,6 +372,7 @@ aips1: bus@30000000 {
 			#size-cells = <1>;
 			reg = <0x30000000 0x400000>;
 			ranges;
+			bootph-pre-ram;
 
 			gpio1: gpio@30200000 {
 				compatible = "fsl,imx7d-gpio", "fsl,imx35-gpio";
@@ -543,6 +545,7 @@ kpp: keypad@30320000 {
 			iomuxc: pinctrl@30330000 {
 				compatible = "fsl,imx7d-iomuxc";
 				reg = <0x30330000 0x10000>;
+				bootph-pre-ram;
 			};
 
 			gpr: iomuxc-gpr@30340000 {
@@ -712,6 +715,7 @@ aips2: bus@30400000 {
 			#size-cells = <1>;
 			reg = <0x30400000 0x400000>;
 			ranges;
+			bootph-pre-ram;
 
 			adc1: adc@30610000 {
 				compatible = "fsl,imx7d-adc";
@@ -902,6 +906,7 @@ aips3: bus@30800000 {
 			#size-cells = <1>;
 			reg = <0x30800000 0x400000>;
 			ranges;
+			bootph-pre-ram;
 
 			spba-bus@30800000 {
 				compatible = "fsl,spba-bus", "simple-bus";
diff --git a/arch/arm/boot/dts/nxp/imx/imx7ulp.dtsi b/arch/arm/boot/dts/nxp/imx/imx7ulp.dtsi
index 880b9a4f32b08..1355feda1aa72 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7ulp.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7ulp.dtsi
@@ -41,6 +41,34 @@ cpu0: cpu@f00 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <0xf00>;
+			clocks = <&smc1 IMX7ULP_CLK_ARM>,
+				 <&scg1 IMX7ULP_CLK_CORE>,
+				 <&scg1 IMX7ULP_CLK_SYS_SEL>,
+				 <&scg1 IMX7ULP_CLK_HSRUN_CORE>,
+				 <&scg1 IMX7ULP_CLK_HSRUN_SYS_SEL>,
+				 <&scg1 IMX7ULP_CLK_FIRC>;
+			clock-names = "arm", "core", "scs_sel",
+				      "hsrun_core", "hsrun_scs_sel",
+				      "firc";
+			operating-points-v2 = <&cpu0_opp_table>;
+		};
+	};
+
+	cpu0_opp_table: opp-table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-500210000 {
+			opp-hz = /bits/ 64 <500210000>;
+			opp-microvolt = <1025000>;
+			clock-latency-ns = <150000>;
+			opp-suspend;
+		};
+
+		opp-720000000 {
+			opp-hz = /bits/ 64 <720000000>;
+			opp-microvolt = <1125000>;
+			clock-latency-ns = <150000>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi b/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
index 65fde4f52587c..1fda60d62ffee 100644
--- a/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
@@ -95,6 +95,7 @@ reg_mba6ul_3v3: regulator-mba6ul-3v3 {
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 		regulator-always-on;
+		bootph-pre-ram;
 	};
 
 	reg_mba6ul_5v0: regulator-mba6ul-5v0 {
@@ -336,6 +337,7 @@ &sai1 {
 &uart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart1>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -392,6 +394,7 @@ &usdhc1 {
 	no-1-8-v;
 	no-mmc;
 	no-sdio;
+	bootph-all;
 	status = "okay";
 };
 
@@ -399,6 +402,7 @@ &wdog1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_wdog1>;
 	fsl,ext-reset-output;
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -494,6 +498,7 @@ pinctrl_uart1: uart1grp {
 			MX6UL_PAD_UART1_TX_DATA__UART1_DCE_TX	0x1b0b1
 			MX6UL_PAD_UART1_RX_DATA__UART1_DCE_RX	0x1b0b1
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_uart3: uart3grp {
@@ -542,6 +547,7 @@ MX6UL_PAD_UART1_CTS_B__GPIO1_IO18	0x0001b099
 			/* CD */
 			MX6UL_PAD_UART1_RTS_B__GPIO1_IO19	0x0001b099
 		>;
+		bootph-all;
 	};
 
 	pinctrl_usdhc1_100mhz: usdhc1-100mhz-grp {
diff --git a/arch/arm/boot/dts/nxp/mxs/imx23-olinuxino.dts b/arch/arm/boot/dts/nxp/mxs/imx23-olinuxino.dts
index e372e9327a475..38be7dfabcc09 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx23-olinuxino.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx23-olinuxino.dts
@@ -116,7 +116,7 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pin_gpio2_1>;
 
-		user {
+		led-user {
 			label = "green";
 			gpios = <&gpio2 1 GPIO_ACTIVE_HIGH>;
 		};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx23.dtsi b/arch/arm/boot/dts/nxp/mxs/imx23.dtsi
index 5e21252fb7c96..58575e2c1ca5d 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx23.dtsi
+++ b/arch/arm/boot/dts/nxp/mxs/imx23.dtsi
@@ -420,9 +420,9 @@ dma_apbx: dma-controller@80024000 {
 					     <60>, <58>, <9>, <0>,
 					     <0>, <0>, <0>, <0>;
 				interrupt-names = "audio-adc", "audio-dac", "spdif-tx", "i2c",
-						  "saif0", "empty", "auart0-rx", "auart0-tx",
-						  "auart1-rx", "auart1-tx", "saif1", "empty",
-						  "empty", "empty", "empty", "empty";
+						  "saif0", "empty0", "auart0-rx", "auart0-tx",
+						  "auart1-rx", "auart1-tx", "saif1", "empty1",
+						  "empty2", "empty3", "empty4", "empty5";
 				#dma-cells = <1>;
 				dma-channels = <16>;
 				clocks = <&clks 16>;
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-apf28dev.dts b/arch/arm/boot/dts/nxp/mxs/imx28-apf28dev.dts
index 6c87266eb1353..63bbaa15c54d9 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-apf28dev.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-apf28dev.dts
@@ -29,7 +29,7 @@ reg_can0_vcc: regulator-1 {
 	leds {
 		compatible = "gpio-leds";
 
-		user {
+		led-user {
 			label = "Heartbeat";
 			gpios = <&gpio0 21 0>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-apx4devkit.dts b/arch/arm/boot/dts/nxp/mxs/imx28-apx4devkit.dts
index 0d845ca81e899..774eb49cfbc0f 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-apx4devkit.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-apx4devkit.dts
@@ -31,7 +31,7 @@ sound {
 	leds {
 		compatible = "gpio-leds";
 
-		user {
+		led-user {
 			label = "Heartbeat";
 			gpios = <&gpio3 28 0>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10036.dts b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10036.dts
index f170df37b3f88..f4485334e9f83 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10036.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10036.dts
@@ -21,7 +21,7 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pins_cfa10036>;
 
-		power {
+		led-power {
 			gpios = <&gpio3 4 1>;
 			default-state = "on";
 		};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10049.dts b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10049.dts
index f0ce897b9d5c4..dfe3f08426ac6 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10049.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10049.dts
@@ -107,7 +107,7 @@ hx8357: hx8357@0 {
 			spi-max-frequency = <100000>;
 			spi-cpol;
 			spi-cpha;
-			gpios-reset = <&gpio3 30 0>;
+			reset-gpios = <&gpio3 30 0>;
 			im-gpios = <&gpio5 4 0 &gpio5 5 0 &gpio5 6 0>;
 		};
 	};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10055.dts b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10055.dts
index cb68edd6101bc..aeac1d29058d1 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10055.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10055.dts
@@ -33,7 +33,7 @@ hx8357: hx8357@0 {
 			spi-max-frequency = <100000>;
 			spi-cpol;
 			spi-cpha;
-			gpios-reset = <&gpio3 30 0>;
+			reset-gpios = <&gpio3 30 0>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10056.dts b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10056.dts
index bc2d6fcad12f9..add6a23728e7d 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-cfa10056.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-cfa10056.dts
@@ -32,7 +32,7 @@ hx8369: hx8369@0 {
 			spi-max-frequency = <100000>;
 			spi-cpol;
 			spi-cpha;
-			gpios-reset = <&gpio3 30 0>;
+			reset-gpios = <&gpio3 30 0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-485.dts b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-485.dts
index b73020ff1053d..5e3f09109605a 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-485.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-485.dts
@@ -12,12 +12,12 @@ / {
 	compatible = "i2se,duckbill-2-485", "i2se,duckbill-2", "fsl,imx28";
 
 	leds {
-		rs485-red {
+		led-rs485-red {
 			label = "duckbill:red:rs485";
 			gpios = <&gpio3 24 GPIO_ACTIVE_LOW>;
 		};
 
-		rs485-green {
+		led-rs485-green {
 			label = "duckbill:green:rs485";
 			gpios = <&gpio3 25 GPIO_ACTIVE_LOW>;
 		};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-enocean.dts b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-enocean.dts
index 473d99b9b42f5..2168a59a33b34 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-enocean.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2-enocean.dts
@@ -13,17 +13,17 @@ / {
 	compatible = "i2se,duckbill-2-enocean", "i2se,duckbill-2", "fsl,imx28";
 
 	leds {
-		enocean-blue {
+		led-enocean-blue {
 			label = "duckbill:blue:enocean";
 			gpios = <&gpio3 24 GPIO_ACTIVE_LOW>;
 		};
 
-		enocean-red {
+		led-enocean-red {
 			label = "duckbill:red:enocean";
 			gpios = <&gpio3 25 GPIO_ACTIVE_LOW>;
 		};
 
-		enocean-green {
+		led-enocean-green {
 			label = "duckbill:green:enocean";
 			gpios = <&gpio3 2 GPIO_ACTIVE_LOW>;
 		};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2.dts b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2.dts
index 4e28212e96260..9bc703e7b2bc4 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill-2.dts
@@ -31,13 +31,13 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pins>;
 
-		status-red {
+		led-status-red {
 			label = "duckbill:red:status";
 			gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "default-on";
 		};
 
-		status-green {
+		led-status-green {
 			label = "duckbill:green:status";
 			gpios = <&gpio3 20 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill.dts b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill.dts
index 13ffd533fdea7..d5a1983f92890 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-duckbill.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-duckbill.dts
@@ -30,13 +30,13 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pins>;
 
-		status-red {
+		led-status-red {
 			label = "duckbill:red:status";
 			gpios = <&gpio3 4 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "default-on";
 		};
 
-		status-green {
+		led-status-green {
 			label = "duckbill:green:status";
 			gpios = <&gpio3 5 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-evk.dts b/arch/arm/boot/dts/nxp/mxs/imx28-evk.dts
index 330d3aff6b6c2..d1ff001b40379 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-evk.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-evk.dts
@@ -108,7 +108,7 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pin_gpio3_5>;
 
-		user {
+		led-user {
 			label = "Heartbeat";
 			gpios = <&gpio3 5 0>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-m28cu3.dts b/arch/arm/boot/dts/nxp/mxs/imx28-m28cu3.dts
index 34b4d3246db19..0f2bfb4845a9b 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-m28cu3.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-m28cu3.dts
@@ -27,13 +27,13 @@ leds {
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pins_gpio>;
 
-		user1 {
+		led-user1 {
 			label = "sd0-led";
 			gpios = <&gpio2 26 0>;
 			linux,default-trigger = "mmc0";
 		};
 
-		user2 {
+		led-user2 {
 			label = "sd1-led";
 			gpios = <&gpio2 24 0>;
 			linux,default-trigger = "mmc2";
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-sps1.dts b/arch/arm/boot/dts/nxp/mxs/imx28-sps1.dts
index ca62e7933116b..9e19c63a4d511 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-sps1.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-sps1.dts
@@ -127,7 +127,7 @@ &ssp2 {
 	flash: flash@0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
-		compatible = "everspin,mr25h256", "mr25h256";
+		compatible = "everspin,mr25h256";
 		spi-max-frequency = <40000000>;
 		reg = <0>;
 	};
diff --git a/arch/arm/boot/dts/nxp/mxs/imx28-tx28.dts b/arch/arm/boot/dts/nxp/mxs/imx28-tx28.dts
index 9290635352f12..197eba22f39c8 100644
--- a/arch/arm/boot/dts/nxp/mxs/imx28-tx28.dts
+++ b/arch/arm/boot/dts/nxp/mxs/imx28-tx28.dts
@@ -20,11 +20,6 @@ aliases {
 		ds1339 = &ds1339;
 		gpio5 = &gpio5;
 		lcdif = &lcdif;
-		lcdif_23bit_pins = &tx28_lcdif_23bit_pins;
-		lcdif_24bit_pins = &lcdif_24bit_pins_a;
-		reg_can_xcvr = &reg_can_xcvr;
-		spi_gpio = &spi_gpio;
-		spi_mxs = &ssp3;
 		stk5led = &user_led;
 		usbotg = &usb0;
 	};
@@ -122,7 +117,7 @@ sound {
 	leds {
 		compatible = "gpio-leds";
 
-		user_led: user {
+		user_led: led-user {
 			label = "Heartbeat";
 			gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
@@ -280,7 +275,7 @@ gpio5: pca953x@20 {
 	};
 
 	polytouch: edt-ft5x06@38 {
-		compatible = "edt,edt-ft5x06";
+		compatible = "edt,edt-ft5206";
 		reg = <0x38>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tx28_edt_ft5x06_pins>;
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8062-smb208.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8062-smb208.dtsi
deleted file mode 100644
index 9d06255104c71..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8062-smb208.dtsi
+++ /dev/null
@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include "qcom-ipq8062.dtsi"
-
-&rpm {
-	smb208_regulators: regulators {
-		compatible = "qcom,rpm-smb208-regulators";
-
-		smb208_s1a: s1a {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s1b: s1b {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2a: s2a {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2b: s2b {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8062.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8062.dtsi
deleted file mode 100644
index 5d3ebd3e2e514..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8062.dtsi
+++ /dev/null
@@ -1,8 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include "qcom-ipq8064-v2.0.dtsi"
-
-/ {
-	model = "Qualcomm Technologies, Inc. IPQ8062";
-	compatible = "qcom,ipq8062", "qcom,ipq8064";
-};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8064-smb208.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8064-smb208.dtsi
deleted file mode 100644
index ac9c44f0c1647..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8064-smb208.dtsi
+++ /dev/null
@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include "qcom-ipq8064.dtsi"
-
-&rpm {
-	smb208_regulators: regulators {
-		compatible = "qcom,rpm-smb208-regulators";
-
-		smb208_s1a: s1a {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s1b: s1b {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2a: s2a {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1250000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2b: s2b {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1250000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8064-v2.0-smb208.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8064-v2.0-smb208.dtsi
deleted file mode 100644
index 0442580b22ded..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8064-v2.0-smb208.dtsi
+++ /dev/null
@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include "qcom-ipq8064-v2.0.dtsi"
-
-&rpm {
-	smb208_regulators: regulators {
-		compatible = "qcom,rpm-smb208-regulators";
-
-		smb208_s1a: s1a {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s1b: s1b {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2a: s2a {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1250000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2b: s2b {
-			regulator-min-microvolt = < 800000>;
-			regulator-max-microvolt = <1250000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8065-smb208.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8065-smb208.dtsi
deleted file mode 100644
index 803e6ff99ef8d..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8065-smb208.dtsi
+++ /dev/null
@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include "qcom-ipq8065.dtsi"
-
-&rpm {
-	smb208_regulators: regulators {
-		compatible = "qcom,rpm-smb208-regulators";
-
-		smb208_s1a: s1a {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s1b: s1b {
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1150000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2a: s2a {
-			regulator-min-microvolt = <775000>;
-			regulator-max-microvolt = <1275000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-
-		smb208_s2b: s2b {
-			regulator-min-microvolt = <775000>;
-			regulator-max-microvolt = <1275000>;
-
-			qcom,switch-mode-frequency = <1200000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8065.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8065.dtsi
deleted file mode 100644
index ea49f6cc416d7..0000000000000
--- a/arch/arm/boot/dts/qcom/qcom-ipq8065.dtsi
+++ /dev/null
@@ -1,8 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include "qcom-ipq8064-v2.0.dtsi"
-
-/ {
-	model = "Qualcomm Technologies, Inc. IPQ8065";
-	compatible = "qcom,ipq8065", "qcom,ipq8064";
-};
diff --git a/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts b/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
index 04d24b6d8056f..1d56bdef54539 100644
--- a/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
+++ b/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
@@ -65,6 +65,17 @@ vccq_sdhi0: regulator-vccq-sdhi0 {
 		enable-active-high;
 	};
 
+	vcc_sdhi1: regulator-vcc-sdhi1 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "SDHI1 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pfc 16 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_5p0v: regulator-5p0v {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-5.0V";
@@ -228,6 +239,7 @@ touchscreen@55 {
 		pinctrl-0 = <&st1232_pins>;
 		pinctrl-names = "default";
 		gpios = <&pfc 166 GPIO_ACTIVE_LOW>;
+		wakeup-source;
 	};
 };
 
@@ -285,6 +297,11 @@ sdhi0_pins: sd0 {
 		function = "sdhi0";
 	};
 
+	sdhi1_pins: sd1 {
+		groups = "sdhi1_data4", "sdhi1_ctrl", "sdhi1_cd", "sdhi1_wp";
+		function = "sdhi1";
+	};
+
 	fsia_pins: sounda {
 		groups = "fsia_sclk_in", "fsia_mclk_out",
 			 "fsia_data_in_1", "fsia_data_out_0";
@@ -302,6 +319,12 @@ lcd0-mux-hog {
 		gpios = <176 0>;
 		output-high;
 	};
+
+	sdhi1-select-hog {
+		gpio-hog;
+		gpios = <6 0>;
+		input;
+	};
 };
 
 &tpu {
@@ -336,6 +359,15 @@ &sdhi0 {
 	status = "okay";
 };
 
+&sdhi1 {
+	pinctrl-0 = <&sdhi1_pins>;
+	pinctrl-names = "default";
+
+	vmmc-supply = <&vcc_sdhi1>;
+	bus-width = <4>;
+	status = "okay";
+};
+
 &sh_fsi2 {
 	pinctrl-0 = <&fsia_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
index 4a72aa7663f25..5626d7fd6c3ee 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
+++ b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
@@ -185,6 +185,18 @@ fixed-link {
 	};
 };
 
+&gpioirqmux {
+	interrupt-map = <89 &gic GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>, /* pin 147: phy@4 */
+			<91 &gic GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>; /* pin 149: phy@5 */
+	status = "okay";
+};
+
+&gpio2 {
+	pinctrl-0 = <&pins_gpio2>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &i2c2 {
 	pinctrl-0 = <&pins_i2c2>;
 	pinctrl-names = "default";
@@ -237,13 +249,13 @@ &pinctrl {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pins_cpld>;
 
-	pins_can0: pins_can0 {
+	pins_can0: pins-can0 {
 		pinmux = <RZN1_PINMUX(162, RZN1_FUNC_CAN)>,	/* CAN0_TXD */
 			 <RZN1_PINMUX(163, RZN1_FUNC_CAN)>;	/* CAN0_RXD */
 		drive-strength = <6>;
 	};
 
-	pins_can1: pins_can1 {
+	pins_can1: pins-can1 {
 		pinmux = <RZN1_PINMUX(109, RZN1_FUNC_CAN)>,	/* CAN1_TXD */
 			 <RZN1_PINMUX(110, RZN1_FUNC_CAN)>;	/* CAN1_RXD */
 		drive-strength = <6>;
@@ -256,7 +268,7 @@ pins_cpld: pins-cpld {
 			 <RZN1_PINMUX(122, RZN1_FUNC_USB)>;
 	};
 
-	pins_eth3: pins_eth3 {
+	pins_eth3: pins-eth3 {
 		pinmux = <RZN1_PINMUX(36, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(37, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(38, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
@@ -273,7 +285,7 @@ pins_eth3: pins_eth3 {
 		bias-disable;
 	};
 
-	pins_eth4: pins_eth4 {
+	pins_eth4: pins-eth4 {
 		pinmux = <RZN1_PINMUX(48, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(49, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(50, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
@@ -290,16 +302,101 @@ pins_eth4: pins_eth4 {
 		bias-disable;
 	};
 
-	pins_i2c2: pins_i2c2 {
+	pins_gpio2: pins-gpio2 {
+		pinmux = <RZN1_PINMUX(147, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(149, RZN1_FUNC_GPIO)>;
+		drive-strength = <6>;
+		bias-pull-up;
+	};
+
+	pins_i2c2: pins-i2c2 {
 		pinmux = <RZN1_PINMUX(115, RZN1_FUNC_I2C)>,
 			 <RZN1_PINMUX(116, RZN1_FUNC_I2C)>;
 		drive-strength = <12>;
 	};
 
-	pins_mdio1: pins_mdio1 {
+	pins_mdio1: pins-mdio1 {
 		pinmux = <RZN1_PINMUX(152, RZN1_FUNC_MDIO1_SWITCH)>,
 			 <RZN1_PINMUX(153, RZN1_FUNC_MDIO1_SWITCH)>;
 	};
+
+	pins_qspi0: pins-qspi0 {
+		pinmux = <RZN1_PINMUX(74, RZN1_FUNC_QSPI)>,
+			 <RZN1_PINMUX(75, RZN1_FUNC_QSPI)>,
+			 <RZN1_PINMUX(76, RZN1_FUNC_QSPI)>,
+			 <RZN1_PINMUX(77, RZN1_FUNC_QSPI)>,
+			 <RZN1_PINMUX(78, RZN1_FUNC_QSPI)>,
+			 <RZN1_PINMUX(79, RZN1_FUNC_QSPI)>;
+		bias-disable;
+	};
+};
+
+&qspi0 {
+	pinctrl-0 = <&pins_qspi0>;
+	pinctrl-names = "default";
+	status = "okay";
+	bootph-all;
+
+	flash@0 {
+		reg = <0>;
+		compatible = "jedec,spi-nor";
+		spi-max-frequency = <62500000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		cdns,read-delay = <1>;
+		cdns,tshsl-ns = <200>;
+		cdns,tsd2d-ns = <255>;
+		cdns,tchsh-ns = <20>;
+		cdns,tslch-ns = <20>;
+		bootph-all;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				/* 64KiB */
+				label = "spl";
+				reg = <0x0000000 0x00010000>;
+			};
+			partition@10000 {
+				/* 64KiB */
+				label = "pkgt";
+				reg = <0x0010000 0x00010000>;
+			};
+			partition@20000 {
+				/* 512KiB */
+				label = "u-boot";
+				reg = <0x0020000 0x00080000>;
+			};
+			partition@a0000 {
+				/* 64KiB */
+				label = "env";
+				reg = <0x00a0000 0x00010000>;
+			};
+			partition@b0000 {
+				/* 128KiB */
+				label = "dtb";
+				reg = <0x00b0000 0x00020000>;
+			};
+			partition@d0000 {
+				/* 1MiB */
+				label = "cm3";
+				reg = <0x00d0000 0x00100000>;
+			};
+			partition@1d0000 {
+				/* 6MiB */
+				label = "kernel";
+				reg = <0x01d0000 0x00600000>;
+			};
+			partition@7d0000 {
+				/* Remaining */
+				label = "data";
+				reg = <0x07d0000 0x1830000>;
+			};
+		};
+	};
 };
 
 &rtc0 {
@@ -323,11 +420,13 @@ mdio {
 		switch0phy4: ethernet-phy@4 {
 			reg = <4>;
 			micrel,led-mode = <1>;
+			interrupts-extended = <&gpio2a 25 IRQ_TYPE_LEVEL_LOW>;
 		};
 
 		switch0phy5: ethernet-phy@5 {
 			reg = <5>;
 			micrel,led-mode = <1>;
+			interrupts-extended = <&gpio2a 27 IRQ_TYPE_LEVEL_LOW>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/renesas/r9a06g032.dtsi b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
index f4f760aff28bd..daa7d5de575de 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032.dtsi
+++ b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
@@ -15,6 +15,39 @@ / {
 	#size-cells = <1>;
 	interrupt-parent = <&gic>;
 
+	/*
+	 * The CPUs clock is based on the 'ref' clock (output of OPPDIV divisor)
+	 * with x1, x2 or x4 ratio between the CPUs clock frequency and this
+	 * 'ref' clock frequency.
+	 *
+	 * The table below is built on the assumption that the 'ref' clock
+	 * frequency is set to 500MHz which is its default value.
+	 *
+	 * The table should be overridden in the board device-tree file based
+	 * on the 'ref' clock frequency if this frequency value is not the
+	 * default one.
+	 */
+	cpu_opp_table: opp-table-cpu {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-125000000 {
+			opp-hz = /bits/ 64 <125000000>;
+			/* ~35 clocks cycles at 125mhz */
+			clock-latency-ns = <300>;
+		};
+
+		opp-250000000 {
+			opp-hz = /bits/ 64 <250000000>;
+			clock-latency-ns = <300>;
+		};
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -24,6 +57,7 @@ cpu@0 {
 			compatible = "arm,cortex-a7";
 			reg = <0>;
 			clocks = <&sysctrl R9A06G032_CLK_A7MP>;
+			operating-points-v2 = <&cpu_opp_table>;
 		};
 
 		cpu@1 {
@@ -33,6 +67,7 @@ cpu@1 {
 			clocks = <&sysctrl R9A06G032_CLK_A7MP>;
 			enable-method = "renesas,r9a06g032-smp";
 			cpu-release-addr = <0 0x4000c204>;
+			operating-points-v2 = <&cpu_opp_table>;
 		};
 	};
 
@@ -66,6 +101,18 @@ soc {
 		#size-cells = <1>;
 		ranges;
 
+		qspi0: spi@40005000 {
+			compatible = "renesas,r9a06g032-qspi", "renesas,rzn1-qspi";
+			reg = <0x40005000 0x1000>, <0x10000000 0x10000000>;
+			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_QSPI0>, <&sysctrl R9A06G032_HCLK_QSPI0>,
+				 <&sysctrl R9A06G032_HCLK_QSPI0>;
+			clock-names = "ref", "ahb", "apb";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		rtc0: rtc@40006000 {
 			compatible = "renesas,r9a06g032-rtc", "renesas,rzn1-rtc";
 			reg = <0x40006000 0x1000>;
diff --git a/arch/arm/boot/dts/rockchip/rk3036-evb.dts b/arch/arm/boot/dts/rockchip/rk3036-evb.dts
index becdc0b664bfa..c8100dc4c7ce3 100644
--- a/arch/arm/boot/dts/rockchip/rk3036-evb.dts
+++ b/arch/arm/boot/dts/rockchip/rk3036-evb.dts
@@ -16,8 +16,6 @@ memory@60000000 {
 
 &emac {
 	phy = <&phy0>;
-	phy-reset-duration = <10>; /* millisecond */
-	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
 	pinctrl-names = "default";
 	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	status = "okay";
@@ -28,6 +26,8 @@ mdio {
 
 		phy0: ethernet-phy@0 {
 			reg = <0>;
+			reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rockchip/rk3036-kylin.dts b/arch/arm/boot/dts/rockchip/rk3036-kylin.dts
index ae2f84a4e922a..bc6e6468fcc4f 100644
--- a/arch/arm/boot/dts/rockchip/rk3036-kylin.dts
+++ b/arch/arm/boot/dts/rockchip/rk3036-kylin.dts
@@ -102,8 +102,6 @@ &acodec {
 
 &emac {
 	phy = <&phy0>;
-	phy-reset-duration = <10>; /* millisecond */
-	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
 	pinctrl-names = "default";
 	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	status = "okay";
@@ -114,6 +112,8 @@ mdio {
 
 		phy0: ethernet-phy@0 {
 			reg = <0>;
+			reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rockchip/rk3188-bqedison2qc.dts b/arch/arm/boot/dts/rockchip/rk3188-bqedison2qc.dts
index edc2b7f9112dd..b56095fc24414 100644
--- a/arch/arm/boot/dts/rockchip/rk3188-bqedison2qc.dts
+++ b/arch/arm/boot/dts/rockchip/rk3188-bqedison2qc.dts
@@ -262,7 +262,7 @@ lis3de: accelerometer@29 {
 		interrupts = <RK_PB7 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&gsensor_int>;
-		rotation-matrix = "1", "0", "0",
+		mount-matrix = "1", "0", "0",
 				  "0", "-1", "0",
 				  "0", "0", "1";
 		vdd-supply = <&vcc_io>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-firefly-reload.dts b/arch/arm/boot/dts/rockchip/rk3288-firefly-reload.dts
index a552706727321..8b491b002992e 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-firefly-reload.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-firefly-reload.dts
@@ -197,11 +197,10 @@ &hdmi {
 };
 
 &i2c0 {
-	hym8563: hym8563@51 {
+	hym8563: rtc@51 {
 		compatible = "haoyu,hym8563";
 		reg = <0x51>;
 		#clock-cells = <0>;
-		clock-frequency = <32768>;
 		clock-output-names = "xin32k";
 		interrupt-parent = <&gpio7>;
 		interrupts = <RK_PA4 IRQ_TYPE_EDGE_FALLING>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-phycore-rdk.dts b/arch/arm/boot/dts/rockchip/rk3288-phycore-rdk.dts
index 10ce0554d4fc9..46362e804dafc 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-phycore-rdk.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-phycore-rdk.dts
@@ -86,6 +86,10 @@ &i2c1 {
 	touchscreen@44 {
 		compatible = "st,stmpe811";
 		reg = <0x44>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <RK_PB4 IRQ_TYPE_EDGE_FALLING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts_irq_pin>;
 	};
 
 	adc@64 {
@@ -94,7 +98,7 @@ adc@64 {
 	};
 
 	i2c_rtc: rtc@68 {
-		compatible = "rv4162";
+		compatible = "microcrystal,rv4162";
 		reg = <0x68>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&i2c_rtc_int>;
@@ -121,25 +125,25 @@ leddim: leddimmer@62 {
 		compatible = "nxp,pca9533";
 		reg = <0x62>;
 
-		led1 {
+		led-1 {
 			label = "red:user1";
 			linux,default-trigger = "none";
 			type = <PCA9532_TYPE_LED>;
 		};
 
-		led2 {
+		led-2 {
 			label = "green:user2";
 			linux,default-trigger = "none";
 			type = <PCA9532_TYPE_LED>;
 		};
 
-		led3 {
+		led-3 {
 			label = "blue:user3";
 			linux,default-trigger = "none";
 			type = <PCA9532_TYPE_LED>;
 		};
 
-		led4 {
+		led-4 {
 			label = "red:user4";
 			linux,default-trigger = "none";
 			type = <PCA9532_TYPE_LED>;
@@ -199,7 +203,7 @@ sdmmc_pwr: sdmmc-pwr {
 
 	touchscreen {
 		ts_irq_pin: ts-irq-pin {
-			rockchip,pins = <5 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
+			rockchip,pins = <5 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/rockchip/rk3288-phycore-som.dtsi b/arch/arm/boot/dts/rockchip/rk3288-phycore-som.dtsi
index 12ab10c4addef..0816e388852f3 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-phycore-som.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288-phycore-som.dtsi
@@ -100,7 +100,7 @@ &gmac {
 	tx_delay = <0x0>;
 	rx_delay = <0x0>;
 
-	mdio0 {
+	mdio {
 		compatible = "snps,dwmac-mdio";
 		#address-cells = <1>;
 		#size-cells = <0>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-brain.dts b/arch/arm/boot/dts/rockchip/rk3288-veyron-brain.dts
index ade9cc291813a..d7790eebfdcd3 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-brain.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-brain.dts
@@ -91,14 +91,12 @@ vdd10_lcd: LDO_REG7 {
 			regulator-min-microvolt = <1000000>;
 			regulator-max-microvolt = <1000000>;
 			regulator-name = "vdd10_lcd";
-			regulator-suspend-mem-disabled;
 		};
 
 		vcc18_hdmi: SWITCH_REG2 {
 			regulator-always-on;
 			regulator-boot-on;
 			regulator-name = "vcc18_hdmi";
-			regulator-suspend-mem-disabled;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-fievel.dts b/arch/arm/boot/dts/rockchip/rk3288-veyron-fievel.dts
index 6a0844e162793..3da105060302e 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-fievel.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-fievel.dts
@@ -98,9 +98,8 @@ &gmac {
 	snps,reset-gpio = <&gpio4 RK_PB0 0>;
 	snps,reset-active-low;
 	snps,reset-delays-us = <0 10000 30000>;
-	wakeup-source;
 
-	mdio0 {
+	mdio {
 		compatible = "snps,dwmac-mdio";
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -177,7 +176,7 @@ &sdio0 {
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	btmrvl: btmrvl@2 {
+	btmrvl: bluetooth@2 {
 		compatible = "marvell,sd8897-bt";
 		reg = <2>;
 		interrupt-parent = <&gpio4>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-jaq.dts b/arch/arm/boot/dts/rockchip/rk3288-veyron-jaq.dts
index 0d4c50e055587..cba2898f8b7df 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-jaq.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-jaq.dts
@@ -48,7 +48,7 @@ &sdio0 {
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	btmrvl: btmrvl@2 {
+	btmrvl: bluetooth@2 {
 		compatible = "marvell,sd8897-bt";
 		reg = <2>;
 		interrupt-parent = <&gpio4>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-jerry.dts b/arch/arm/boot/dts/rockchip/rk3288-veyron-jerry.dts
index 6894763979f03..0bf03b1ff2aba 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-jerry.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-jerry.dts
@@ -488,7 +488,7 @@ trackpad@2c {
 		interrupts = <RK_PA3 IRQ_TYPE_EDGE_FALLING>;
 		reg = <0x2c>;
 		hid-descr-addr = <0x0020>;
-		vcc-supply = <&vcc33_io>;
+		vdd-supply = <&vcc33_io>;
 		wakeup-source;
 	};
 };
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-mickey.dts b/arch/arm/boot/dts/rockchip/rk3288-veyron-mickey.dts
index d665c3e8862cd..20fe846839285 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-mickey.dts
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-mickey.dts
@@ -246,7 +246,6 @@ vdd10_lcd: LDO_REG7 {
 			regulator-min-microvolt = <1000000>;
 			regulator-max-microvolt = <1000000>;
 			regulator-name = "vdd10_lcd";
-			regulator-suspend-mem-disabled;
 		};
 
 		vcc18_lcd: LDO_REG8 {
@@ -255,7 +254,6 @@ vcc18_lcd: LDO_REG8 {
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-name = "vcc18_lcd";
-			regulator-suspend-mem-disabled;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rockchip/rk3288.dtsi b/arch/arm/boot/dts/rockchip/rk3288.dtsi
index 4e5e7509de487..4f2c048aee548 100644
--- a/arch/arm/boot/dts/rockchip/rk3288.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288.dtsi
@@ -551,7 +551,6 @@ tsadc: tsadc@ff280000 {
 		pinctrl-1 = <&otp_out>;
 		pinctrl-2 = <&otp_pin>;
 		#thermal-sensor-cells = <1>;
-		rockchip,grf = <&grf>;
 		rockchip,hw-tshut-temp = <95000>;
 		status = "disabled";
 	};
diff --git a/arch/arm/boot/dts/samsung/Makefile b/arch/arm/boot/dts/samsung/Makefile
index 7becf36656b1a..e0143ee8c82de 100644
--- a/arch/arm/boot/dts/samsung/Makefile
+++ b/arch/arm/boot/dts/samsung/Makefile
@@ -26,6 +26,7 @@ dtb-$(CONFIG_ARCH_EXYNOS4) += \
 	exynos4412-trats2.dtb
 dtb-$(CONFIG_ARCH_EXYNOS5) += \
 	exynos5250-arndale.dtb \
+	exynos5250-manta.dtb \
 	exynos5250-smdk5250.dtb \
 	exynos5250-snow.dtb \
 	exynos5250-snow-rev5.dtb \
diff --git a/arch/arm/boot/dts/samsung/exynos3250-artik5.dtsi b/arch/arm/boot/dts/samsung/exynos3250-artik5.dtsi
index 3fdd922e635ce..059c0f44e164d 100644
--- a/arch/arm/boot/dts/samsung/exynos3250-artik5.dtsi
+++ b/arch/arm/boot/dts/samsung/exynos3250-artik5.dtsi
@@ -74,8 +74,6 @@ &gpu {
 };
 
 &i2c_0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-slave-addr = <0x10>;
 	samsung,i2c-max-bus-freq = <100000>;
diff --git a/arch/arm/boot/dts/samsung/exynos3250-monk.dts b/arch/arm/boot/dts/samsung/exynos3250-monk.dts
index 68236c7297d7c..d59ed46614bda 100644
--- a/arch/arm/boot/dts/samsung/exynos3250-monk.dts
+++ b/arch/arm/boot/dts/samsung/exynos3250-monk.dts
@@ -191,8 +191,6 @@ &hsotg {
 };
 
 &i2c_0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-slave-addr = <0x10>;
 	samsung,i2c-max-bus-freq = <100000>;
@@ -414,8 +412,6 @@ buck5_reg: BUCK5 {
 };
 
 &i2c_1 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-slave-addr = <0x10>;
 	samsung,i2c-max-bus-freq = <400000>;
diff --git a/arch/arm/boot/dts/samsung/exynos3250-rinato.dts b/arch/arm/boot/dts/samsung/exynos3250-rinato.dts
index 36d2171c1ce86..fa983b732898a 100644
--- a/arch/arm/boot/dts/samsung/exynos3250-rinato.dts
+++ b/arch/arm/boot/dts/samsung/exynos3250-rinato.dts
@@ -263,8 +263,6 @@ &gpu {
 };
 
 &i2c_0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-slave-addr = <0x10>;
 	samsung,i2c-max-bus-freq = <100000>;
@@ -594,8 +592,6 @@ regulator-state-mem {
 };
 
 &i2c_1 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-slave-addr = <0x10>;
 	samsung,i2c-max-bus-freq = <400000>;
diff --git a/arch/arm/boot/dts/samsung/exynos4210-smdkv310.dts b/arch/arm/boot/dts/samsung/exynos4210-smdkv310.dts
index 18f4f494093ba..a6b73a8967c6a 100644
--- a/arch/arm/boot/dts/samsung/exynos4210-smdkv310.dts
+++ b/arch/arm/boot/dts/samsung/exynos4210-smdkv310.dts
@@ -68,8 +68,6 @@ map1 {
 };
 
 &i2c_0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <100000>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/samsung/exynos4412-itop-scp-core.dtsi b/arch/arm/boot/dts/samsung/exynos4412-itop-scp-core.dtsi
index 7bc6968af9c33..223907e5919c3 100644
--- a/arch/arm/boot/dts/samsung/exynos4412-itop-scp-core.dtsi
+++ b/arch/arm/boot/dts/samsung/exynos4412-itop-scp-core.dtsi
@@ -130,8 +130,6 @@ &hsotg {
 };
 
 &i2c_1 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 	pinctrl-0 = <&i2c1_bus>;
diff --git a/arch/arm/boot/dts/samsung/exynos4412-origen.dts b/arch/arm/boot/dts/samsung/exynos4412-origen.dts
index 10ab7bc90f502..c5fb551ab3523 100644
--- a/arch/arm/boot/dts/samsung/exynos4412-origen.dts
+++ b/arch/arm/boot/dts/samsung/exynos4412-origen.dts
@@ -113,8 +113,6 @@ &fimd {
 };
 
 &i2c_0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <20000>;
 	pinctrl-0 = <&i2c0_bus>;
diff --git a/arch/arm/boot/dts/samsung/exynos5250-manta.dts b/arch/arm/boot/dts/samsung/exynos5250-manta.dts
new file mode 100644
index 0000000000000..24a27b342227b
--- /dev/null
+++ b/arch/arm/boot/dts/samsung/exynos5250-manta.dts
@@ -0,0 +1,564 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Google Manta (Nexus 10) board device tree source
+ *
+ * Copyright (c) 2023-2026 Alexandre Marquet
+ * Copyright (c) 2025-2026 Lukas Timmermann
+ */
+
+/dts-v1/;
+#include "exynos5250.dtsi"
+#include "exynos-pinctrl.h"
+#include "exynos-mfc-reserved-memory.dtsi"
+
+#include <dt-bindings/clock/maxim,max77686.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+
+/ {
+	model = "Google Nexus 10";
+	compatible = "google,manta", "samsung,exynos5250", "samsung,exynos5";
+
+	aliases {
+		mmc0 = &mmc_0; /* eMMC */
+		mmc1 = &mmc_1; /* WiFi */
+	};
+
+	/* Voltage source unknown */
+	bmp180_vdda_reg: regulator-bmp180-vdda {
+		compatible = "regulator-fixed";
+		regulator-name = "BMP180_VDDA";
+	};
+
+	/* Voltage source unknown */
+	bmp180_vddd_reg: regulator-bmp180-vddd {
+		compatible = "regulator-fixed";
+		regulator-name = "BMP180_VDDD";
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	flash-controller {
+		compatible = "gpio-leds";
+
+		led-flash {
+			function = LED_FUNCTION_FLASH;
+			color = <LED_COLOR_ID_WHITE>;
+			linux,default-trigger = "flash";
+			gpios = <&gpe0 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	fixed-rate-clocks {
+		xxti {
+			compatible = "samsung,clock-xxti";
+			clock-frequency = <24000000>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&gpio_keys>;
+		pinctrl-names = "default";
+
+		button-volume-down {
+			label = "Volume Down";
+			gpios = <&gpx2 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			debounce-interval = <5>;
+		};
+
+		button-volume-up {
+			label = "Volume Up";
+			gpios = <&gpx2 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			debounce-interval = <5>;
+		};
+
+		button-power {
+			label = "Power";
+			gpios = <&gpx2 7 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
+			debounce-interval = <5>;
+			wakeup-source;
+		};
+
+		lid-switch {
+			label = "Hall Effect Sensor";
+			gpios = <&gpx1 3 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			debounce-interval = <10>;
+			wakeup-source;
+		};
+	};
+
+	multi-led {
+		compatible = "leds-group-multicolor";
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+		leds = <&status_red>, <&status_green>, <&status_blue>, <&status_white>;
+	};
+
+	pwrseq: mmc1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+
+		reset-gpios = <&gpv1 0 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&wlan_pmena>;
+		pinctrl-names = "default";
+
+		clocks = <&max77686 MAX77686_CLK_PMIC>;
+		clock-names = "ext_clock";
+
+		post-power-on-delay-ms = <300>;
+		power-off-delay-us = <50>;
+	};
+
+	firmware@204efff {
+		compatible = "samsung,secure-firmware";
+		reg = <0x0204efff 0x1000>;
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x40000000 0x20000000>,
+		      <0x60000000 0x20000000>,
+		      <0x80000000 0x20000000>,
+		      <0xa0000000 0x1ff00000>;
+	};
+};
+
+&clock {
+	assigned-clocks = <&clock CLK_FOUT_APLL>;
+	assigned-clock-rates = <1000000000>;
+};
+
+&cpu0 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&cpu1 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&ehci {
+	status = "disabled";
+};
+
+&i2c_1 {
+	status = "okay";
+
+	pressure-sensor@77 {
+		compatible = "bosch,bmp180";
+		reg = <0x77>;
+		vddd-supply = <&bmp180_vddd_reg>;
+		vdda-supply = <&bmp180_vdda_reg>;
+	};
+
+	accelerometer@68 {
+		compatible = "invensense,mpu6050";
+		reg = <0x68>;
+
+		pinctrl-0 = <&acc_int>;
+		pinctrl-names = "default";
+
+		interrupt-parent = <&gpx1>;
+		interrupts = <4 IRQ_TYPE_EDGE_RISING>;
+
+		mount-matrix = "0", "-1",  "0",
+			      "-1",  "0",  "0",
+			       "0",  "0", "-1";
+
+		i2c-gate {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			magnetometer@c {
+				compatible = "asahi-kasei,ak8963";
+				reg = <0x0c>;
+
+				pinctrl-0 = <&msense_reset>;
+				pinctrl-names = "default";
+
+				mount-matrix = "-1", "0",  "0",
+					       "0",  "1",  "0",
+					       "0",  "0", "-1";
+			};
+		};
+	};
+
+	led-controller@42 {
+		compatible = "ams,as3668";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x42>;
+
+		status_red: led@0 {
+			reg = <0>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+		};
+
+		status_green: led@1 {
+			reg = <1>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		status_blue: led@2 {
+			reg = <2>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_BLUE>;
+		};
+
+		status_white: led@3 {
+			reg = <3>;
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_WHITE>;
+		};
+	};
+};
+
+&i2c_2 {
+	status = "okay";
+
+	light-sensor@23 {
+		compatible = "rohm,bh1721";
+		reg = <0x23>;
+
+		pinctrl-0 = <&bh1721fvc_reset>;
+		pinctrl-names = "default";
+
+		reset-gpios = <&gph1 2 GPIO_ACTIVE_LOW>;
+	};
+
+	onewire@18 {
+		compatible = "maxim,ds2484";
+		reg = <0x18>;
+
+		pinctrl-0 = <&onewire_sleep>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c_5 {
+	samsung,i2c-sda-delay = <100>;
+	status = "okay";
+
+	max77686: pmic@9 {
+		compatible = "maxim,max77686";
+		reg = <0x09>;
+
+		interrupt-parent = <&gpx0>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&max77686_irq>;
+		pinctrl-names = "default";
+
+		#clock-cells = <1>;
+		wakeup-source;
+
+		voltage-regulators {
+			buck1_reg: BUCK1 {
+				regulator-name = "vdd_mif";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2_reg: BUCK2 {
+				regulator-name = "vdd_arm";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3_reg: BUCK3 {
+				regulator-name = "vdd_int";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4_reg: BUCK4 {
+				regulator-name = "vdd_g3d";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-boot-on;
+			};
+
+			ldo3_reg: LDO3 {
+				regulator-name = "VCC_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo8_reg: LDO8 {
+				regulator-name = "VMIPI_1.0V";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo9_reg: LDO9 {
+				regulator-name = "TOUCH_VDD_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo10_reg: LDO10 {
+				regulator-name = "VMIPI_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo12_reg: LDO12 {
+				regulator-name = "VUOTG_3.0V";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo15_reg: LDO15 {
+				regulator-name = "VHSIC_1.0V";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo16_reg: LDO16 {
+				regulator-name = "VHSIC_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo17_reg: LDO17 {
+				regulator-name = "5M_CORE_1.5V";
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+			};
+
+			ldo18_reg: LDO18 {
+				regulator-name = "CAM_IO_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo19_reg: LDO19 {
+				regulator-name = "VT_CAM_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo20_reg: LDO20 {
+				regulator-name = "TA_CHECK_1.35V";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <1350000>;
+			};
+
+			ldo23_reg: LDO23 {
+				regulator-name = "TSP_AVDD_2.8V";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo24_reg: LDO24 {
+				regulator-name = "CAM_AF_2.8V";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo25_reg: LDO25 {
+				regulator-name = "VADC_3.3V";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+
+&mixer {
+	status = "okay";
+};
+
+/* eMMC */
+&mmc_0 {
+	non-removable;
+	max-frequency = <200000000>;
+	sd-uhs-ddr50;
+	mmc-ddr-1_8v;
+	cap-mmc-hw-reset;
+	mmc-hs200-1_8v;
+	bus-width = <8>;
+	card-detect-delay = <200>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <1 2>;
+	samsung,dw-mshc-ddr-timing = <2 3>;
+
+	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+/* WiFi */
+&mmc_1 {
+	non-removable;
+	max-frequency = <100000000>;
+	sd-uhs-sdr50;
+	cap-sd-highspeed;
+	keep-power-in-suspend;
+	bus-width = <4>;
+	card-detect-delay = <0>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <2 4>;
+	samsung,dw-mshc-ddr-timing = <2 3>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus4>;
+	mmc-pwrseq = <&pwrseq>;
+
+	status = "okay";
+
+	wifi@1 {
+		compatible = "brcm,bcm4330-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpx2>;
+		interrupts = <5 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-0 = <&wlan_irq>;
+		pinctrl-names = "default";
+	};
+};
+
+&ohci {
+	status = "disabled";
+};
+
+&pinctrl_0 {
+	acc_int: acc-int-pins {
+		samsung,pins = "gpx1-4";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+	};
+
+	max77686_irq: max77686-irq-pins {
+		samsung,pins = "gpx0-2";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_F>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+	};
+
+	gpio_keys: gpio-keys-pins {
+		samsung,pins = "gpx2-0", "gpx2-1", "gpx2-7", "gpx1-3";
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+	};
+
+	wlan_irq: wlan-irq-pins {
+		samsung,pins = "gpx2-5";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV3>;
+	};
+};
+
+&pinctrl_1 {
+	bh1721fvc_reset: bh1721fvc-reset-pins {
+		samsung,pins = "gph1-2";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+	};
+
+	msense_reset: msense-reset-pins {
+		samsung,pins = "gpg2-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+	};
+
+	onewire_sleep: onewire-sleep-pins {
+		samsung,pins = "gpg0-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-con-pdn = <EXYNOS_PIN_PDN_INPUT>;
+		samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+	};
+};
+
+&pinctrl_2 {
+	wlan_pmena: wlan-pmena-pins {
+		samsung,pins = "gpv1-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV3>;
+		samsung,pin-con-pdn = <EXYNOS_PIN_PDN_PREV>;
+		samsung,pin-val = <0>;
+	};
+};
+
+&pmu_system_controller {
+	assigned-clocks = <&pmu_system_controller 0>;
+	assigned-clock-parents = <&clock CLK_FIN_PLL>;
+};
+
+&rtc {
+	clocks = <&clock CLK_RTC>, <&max77686 MAX77686_CLK_AP>;
+	clock-names = "rtc", "rtc_src";
+	status = "okay";
+};
+
+&sd1_bus4 {
+	samsung,pin-con-pdn = <EXYNOS_PIN_PDN_INPUT>;
+	samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+};
+
+&sd1_cmd {
+	samsung,pin-con-pdn = <EXYNOS_PIN_PDN_INPUT>;
+	samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+};
+
+/* Bluetooth */
+&serial_0 {
+	status = "disabled";
+};
+
+/* GPS */
+&serial_1 {
+	status = "disabled";
+};
+
+&serial_2 {
+	pinctrl-0 = <&uart2_data>;
+	pinctrl-1 = <&uart2_data>, <&uart2_fctl>;
+	pinctrl-names = "default", "flow-control";
+	status = "okay";
+};
+
+&usbdrd {
+	status = "disabled";
+};
+
+&usbdrd_dwc3 {
+	status = "disabled";
+};
+
+&usbdrd_phy {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/st/Makefile b/arch/arm/boot/dts/st/Makefile
index e906bf6ba0041..10c5d75f6169c 100644
--- a/arch/arm/boot/dts/st/Makefile
+++ b/arch/arm/boot/dts/st/Makefile
@@ -16,6 +16,91 @@ dtb-$(CONFIG_ARCH_STI) += \
 	stih410-b2260.dtb \
 	stih418-b2199.dtb \
 	stih418-b2264.dtb
+
+stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2-dtbs := \
+	stm32mp135f-dhcor-dhsbc.dtb \
+	stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtbo
+
+stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtbo
+
+stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtbo
+
+stm32mp15xx-avenger96-overlay-fdcan1-x6-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-fdcan1-x6.dtbo
+
+stm32mp15xx-avenger96-overlay-fdcan2-x6-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-fdcan2-x6.dtbo
+
+stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtbo
+
+stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtbo
+
+stm32mp15xx-avenger96-overlay-ov5640-x7-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-ov5640-x7.dtbo
+
+stm32mp15xx-avenger96-overlay-spi2-eeprom-x6-dtbs := \
+	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtbo
+
+stm32mp15xx-dhcom-drc02-overlay-wifi-rsi-dtbs := \
+	stm32mp153c-dhcom-drc02.dtb \
+	stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtbo
+
+stm32mp15xx-dhcom-pdk2-overlay-460-200-x11-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-497-200-x12-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-531-100-x21-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-531-100-x22-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-560-200-x12-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtbo \
+
+stm32mp15xx-dhcom-pdk2-overlay-672-100-x18-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtbo \
+
+stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6-dtbs := \
+	stm32mp157c-dhcom-picoitx.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtbo \
+
+stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtbo \
+
+stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh-dtbs := \
+	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtbo \
+
 dtb-$(CONFIG_ARCH_STM32) += \
 	stm32f429-disco.dtb \
 	stm32f469-disco.dtb \
@@ -30,6 +115,8 @@ dtb-$(CONFIG_ARCH_STM32) += \
 	stm32h750i-art-pi.dtb \
 	stm32mp133c-prihmb.dtb \
 	stm32mp135f-dhcor-dhsbc.dtb \
+	stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtb \
+	stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtbo \
 	stm32mp135f-dk.dtb \
 	stm32mp151a-prtt1a.dtb \
 	stm32mp151a-prtt1c.dtb \
@@ -39,12 +126,30 @@ dtb-$(CONFIG_ARCH_STM32) += \
 	stm32mp151c-mect1s.dtb \
 	stm32mp151c-plyaqm.dtb \
 	stm32mp153c-dhcom-drc02.dtb \
+	stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtb \
+	stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtbo \
 	stm32mp153c-dhcor-drc-compact.dtb \
 	stm32mp153c-lxa-fairytux2-gen1.dtb \
 	stm32mp153c-lxa-fairytux2-gen2.dtb \
 	stm32mp153c-lxa-tac-gen3.dtb \
 	stm32mp153c-mecio1r1.dtb \
 	stm32mp157a-avenger96.dtb \
+	stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtb \
+	stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtbo \
+	stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtb \
+	stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtbo \
+	stm32mp15xx-avenger96-overlay-fdcan1-x6.dtb \
+	stm32mp15xx-avenger96-overlay-fdcan1-x6.dtbo \
+	stm32mp15xx-avenger96-overlay-fdcan2-x6.dtb \
+	stm32mp15xx-avenger96-overlay-fdcan2-x6.dtbo \
+	stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtb \
+	stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtbo \
+	stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtb \
+	stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtbo \
+	stm32mp15xx-avenger96-overlay-ov5640-x7.dtb \
+	stm32mp15xx-avenger96-overlay-ov5640-x7.dtbo \
+	stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtb \
+	stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtbo \
 	stm32mp157a-dhcor-avenger96.dtb \
 	stm32mp157a-dk1.dtb \
 	stm32mp157a-dk1-scmi.dtb \
@@ -56,7 +161,29 @@ dtb-$(CONFIG_ARCH_STM32) += \
 	stm32mp157a-icore-stm32mp1-edimm2.2.dtb \
 	stm32mp157a-stinger96.dtb \
 	stm32mp157c-dhcom-pdk2.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtbo \
+	stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtb \
+	stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtbo \
 	stm32mp157c-dhcom-picoitx.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtbo \
+	stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtbo \
+	stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtb \
+	stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtbo \
 	stm32mp157c-dk2.dtb \
 	stm32mp157c-dk2-scmi.dtb \
 	stm32mp157c-ed1.dtb \
@@ -69,7 +196,7 @@ dtb-$(CONFIG_ARCH_STM32) += \
 	stm32mp157c-lxa-tac-gen2.dtb \
 	stm32mp157c-odyssey.dtb \
 	stm32mp157c-osd32mp1-red.dtb \
-	stm32mp157c-phycore-stm32mp1-3.dtb \
+	stm32mp157c-phyboard-sargas-rdk.dtb \
 	stm32mp157c-ultra-fly-sbc.dtb \
 	stm32mp157f-dk2.dtb
 dtb-$(CONFIG_ARCH_U8500) += \
diff --git a/arch/arm/boot/dts/st/stm32mp131.dtsi b/arch/arm/boot/dts/st/stm32mp131.dtsi
index b9657ff91c234..83ae59b73dd09 100644
--- a/arch/arm/boot/dts/st/stm32mp131.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp131.dtsi
@@ -3,6 +3,7 @@
  * Copyright (C) STMicroelectronics 2021 - All Rights Reserved
  * Author: Alexandre Torgue <alexandre.torgue@foss.st.com> for STMicroelectronics.
  */
+#include <dt-bindings/arm/coresight-cti-dt.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/stm32mp13-clks.h>
 #include <dt-bindings/reset/stm32mp13-resets.h>
@@ -469,8 +470,8 @@ i2c1: i2c@40012000 {
 			compatible = "st,stm32mp13-i2c";
 			reg = <0x40012000 0x400>;
 			interrupt-names = "event", "error";
-			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts-extended = <&exti 21 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&rcc I2C1_K>;
 			resets = <&rcc I2C1_R>;
 			#address-cells = <1>;
@@ -478,6 +479,7 @@ i2c1: i2c@40012000 {
 			dmas = <&dmamux1 33 0x400 0x1>,
 			       <&dmamux1 34 0x400 0x1>;
 			dma-names = "rx", "tx";
+			wakeup-source;
 			st,syscfg-fmp = <&syscfg 0x4 0x1>;
 			i2c-analog-filter;
 			status = "disabled";
@@ -487,8 +489,8 @@ i2c2: i2c@40013000 {
 			compatible = "st,stm32mp13-i2c";
 			reg = <0x40013000 0x400>;
 			interrupt-names = "event", "error";
-			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts-extended = <&exti 22 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&rcc I2C2_K>;
 			resets = <&rcc I2C2_R>;
 			#address-cells = <1>;
@@ -497,6 +499,7 @@ i2c2: i2c@40013000 {
 			       <&dmamux1 36 0x400 0x1>;
 			dma-names = "rx", "tx";
 			st,syscfg-fmp = <&syscfg 0x4 0x2>;
+			wakeup-source;
 			i2c-analog-filter;
 			status = "disabled";
 		};
@@ -964,9 +967,125 @@ hdp: pinctrl@5002a000 {
 			compatible = "st,stm32mp131-hdp";
 			reg = <0x5002a000 0x400>;
 			clocks = <&rcc HDP>;
+			access-controllers = <&dbg_bus 1>;
 			status = "disabled";
 		};
 
+		dbg_bus: bus@50080000 {
+			compatible = "st,stm32mp131-dbg-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			#access-controller-cells = <1>;
+			ranges = <0x50080000 0x50080000 0x3f80000>;
+			status = "disabled";
+
+			cs_etf: etf@50092000 {
+				compatible = "arm,coresight-tmc", "arm,primecell";
+				reg = <0x50092000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				in-ports {
+					port {
+						etf_in_port: endpoint {
+							remote-endpoint = <&etm0_out_port>;
+						};
+					};
+				};
+
+				out-ports {
+					port {
+						etf_out_port: endpoint {
+							remote-endpoint = <&tpiu_in_port>;
+						};
+					};
+				};
+			};
+
+			cs_tpiu: tpiu@50093000 {
+				compatible = "arm,coresight-tpiu", "arm,primecell";
+				reg = <0x50093000 0x1000>;
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				in-ports {
+					port {
+						tpiu_in_port: endpoint {
+							remote-endpoint = <&etf_out_port>;
+						};
+					};
+				};
+			};
+
+			cs_cti_trace: cti@50094000 {
+				compatible = "arm,coresight-cti", "arm,primecell";
+				reg = <0x50094000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+			};
+
+			cs_cti_cpu0: cti@500d8000 {
+				compatible = "arm,coresight-cti", "arm,primecell";
+				reg = <0x500d8000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				trig-conns@0 {
+					reg = <0>;
+					arm,trig-in-sigs = <0 4 5>;
+					arm,trig-in-types = <PE_DBGTRIGGER
+							     GEN_IO
+							     GEN_IO>;
+					arm,trig-out-sigs = <0 7>;
+					arm,trig-out-types = <PE_EDBGREQ
+							      PE_DBGRESTART>;
+					cpu = <&cpu0>;
+				};
+
+				trig-conns@2 {
+					reg = <2>;
+					arm,trig-in-sigs = <2 3 6>;
+					arm,trig-in-types = <ETM_EXTOUT
+							     ETM_EXTOUT
+							     ETM_EXTOUT>;
+					arm,trig-out-sigs = <1 2 3 4>;
+					arm,trig-out-types = <ETM_EXTIN
+							      ETM_EXTIN
+							      ETM_EXTIN
+							      ETM_EXTIN>;
+					arm,cs-dev-assoc = <&cs_etm0>;
+				};
+			};
+
+			cs_etm0: etm@500dc000 {
+				compatible = "arm,coresight-etm3x", "arm,primecell";
+				reg = <0x500dc000 0x1000>;
+				cpu = <&cpu0>;
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				out-ports {
+					port {
+						etm0_out_port: endpoint {
+							remote-endpoint = <&etf_in_port>;
+						};
+					};
+				};
+			};
+		};
+
 		mdma: dma-controller@58000000 {
 			compatible = "st,stm32h7-mdma";
 			reg = <0x58000000 0x1000>;
@@ -1129,7 +1248,7 @@ usart1: serial@4c000000 {
 				resets = <&rcc USART1_R>;
 				wakeup-source;
 				dmas = <&dmamux1 41 0x400 0x5>,
-				<&dmamux1 42 0x400 0x1>;
+				       <&dmamux1 42 0x400 0x1>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 16>;
 				status = "disabled";
@@ -1143,7 +1262,7 @@ usart2: serial@4c001000 {
 				resets = <&rcc USART2_R>;
 				wakeup-source;
 				dmas = <&dmamux1 43 0x400 0x5>,
-				<&dmamux1 44 0x400 0x1>;
+				       <&dmamux1 44 0x400 0x1>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 17>;
 				status = "disabled";
@@ -1155,7 +1274,7 @@ i2s4: audio-controller@4c002000 {
 				#sound-dai-cells = <0>;
 				interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&dmamux1 83 0x400 0x01>,
-				<&dmamux1 84 0x400 0x01>;
+				       <&dmamux1 84 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 13>;
 				status = "disabled";
@@ -1195,8 +1314,8 @@ i2c3: i2c@4c004000 {
 				compatible = "st,stm32mp13-i2c";
 				reg = <0x4c004000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 23 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C3_K>;
 				resets = <&rcc I2C3_R>;
 				#address-cells = <1>;
@@ -1205,6 +1324,7 @@ i2c3: i2c@4c004000 {
 				       <&dmamux1 74 0x400 0x1>;
 				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x4>;
+				wakeup-source;
 				i2c-analog-filter;
 				access-controllers = <&etzpc 20>;
 				status = "disabled";
@@ -1214,8 +1334,8 @@ i2c4: i2c@4c005000 {
 				compatible = "st,stm32mp13-i2c";
 				reg = <0x4c005000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 24 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C4_K>;
 				resets = <&rcc I2C4_R>;
 				#address-cells = <1>;
@@ -1224,6 +1344,7 @@ i2c4: i2c@4c005000 {
 				       <&dmamux1 76 0x400 0x1>;
 				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x8>;
+				wakeup-source;
 				i2c-analog-filter;
 				access-controllers = <&etzpc 21>;
 				status = "disabled";
@@ -1233,8 +1354,8 @@ i2c5: i2c@4c006000 {
 				compatible = "st,stm32mp13-i2c";
 				reg = <0x4c006000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 25 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C5_K>;
 				resets = <&rcc I2C5_R>;
 				#address-cells = <1>;
@@ -1243,6 +1364,7 @@ i2c5: i2c@4c006000 {
 				       <&dmamux1 116 0x400 0x1>;
 				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x10>;
+				wakeup-source;
 				i2c-analog-filter;
 				access-controllers = <&etzpc 22>;
 				status = "disabled";
@@ -1348,9 +1470,9 @@ timers15: timer@4c00a000 {
 				clocks = <&rcc TIM15_K>;
 				clock-names = "int";
 				dmas = <&dmamux1 105 0x400 0x1>,
-				<&dmamux1 106 0x400 0x1>,
-				<&dmamux1 107 0x400 0x1>,
-				<&dmamux1 108 0x400 0x1>;
+				       <&dmamux1 106 0x400 0x1>,
+				       <&dmamux1 107 0x400 0x1>,
+				       <&dmamux1 108 0x400 0x1>;
 				dma-names = "ch1", "up", "trig", "com";
 				access-controllers = <&etzpc 26>;
 				status = "disabled";
@@ -1383,7 +1505,7 @@ timers16: timer@4c00b000 {
 				clocks = <&rcc TIM16_K>;
 				clock-names = "int";
 				dmas = <&dmamux1 109 0x400 0x1>,
-				<&dmamux1 110 0x400 0x1>;
+				       <&dmamux1 110 0x400 0x1>;
 				dma-names = "ch1", "up";
 				access-controllers = <&etzpc 27>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/st/stm32mp135f-dk.dts b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
index 8dcf68b212b45..4d4cec8b86aca 100644
--- a/arch/arm/boot/dts/st/stm32mp135f-dk.dts
+++ b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
@@ -187,6 +187,30 @@ &cryp {
 	status = "okay";
 };
 
+&cs_cti_trace {
+	status = "okay";
+};
+
+&cs_cti_cpu0 {
+	status = "okay";
+};
+
+&cs_etf {
+	status = "okay";
+};
+
+&cs_etm0 {
+	status = "okay";
+};
+
+&cs_tpiu {
+	status = "okay";
+};
+
+&dbg_bus {
+	status = "okay";
+};
+
 &dcmipp {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&dcmipp_pins_a>;
diff --git a/arch/arm/boot/dts/st/stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtso b/arch/arm/boot/dts/st/stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtso
new file mode 100644
index 0000000000000..3801dab141e8f
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp13xx-dhcor-dhsbc-overlay-rb-tft32-v2.dtso
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2024 Marek Vasut
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/input/linux-event-codes.h>
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-1 {
+			label = "KEY2";
+			linux,code = <KEY_2>;
+			gpios = <&gpiog 10 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
+	gpio-keys-polled {
+		compatible = "gpio-keys-polled";
+		poll-interval = <20>;
+
+		button-0 {
+			label = "KEY1";
+			linux,code = <KEY_1>;
+			/* IRQ bank A shared with PA1 touch controller */
+			gpios = <&gpioa 4 GPIO_ACTIVE_LOW>;
+		};
+
+		button-2 {
+			label = "KEY3";
+			linux,code = <KEY_3>;
+			/* IRQ line 0 taken by PI0 / SoM RTC IRQ */
+			gpios = <&gpiod 0 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&m_can1 {
+	/* Collides with KEY2/PG10 KEY3/PD0 */
+	status = "disabled";
+};
+
+&m_can2 {
+	/* Collides with TP_CS/PE6 */
+	status = "disabled";
+};
+
+&usart2 {
+	/* Collides with TP_IRQ/PA1 */
+	status = "disabled";
+};
+
+&spi3 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	cs-gpios = <&gpiof 3 0>, <&gpioe 0 0>;
+	status = "okay";
+
+	lcd@0 {
+		compatible = "adafruit,yx240qv29", "ilitek,ili9341";
+		reg = <0>;
+		spi-max-frequency = <10000000>;
+		dc-gpios = <&gpioe 4 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpiod 3 GPIO_ACTIVE_HIGH>;
+		rotation = <90>;
+	};
+
+	tp@1 {
+		compatible = "ti,tsc2046";
+		reg = <1>;
+		interrupt-parent = <&gpioa>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+		pendown-gpio = <&gpioa 1 GPIO_ACTIVE_LOW>;
+		spi-max-frequency = <500000>;
+		ti,pressure-max = /bits/ 16 <255>;
+		ti,x-plate-ohms = /bits/ 16 <60>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
index 8613a6a17ee98..aaa91b634c12f 100644
--- a/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
@@ -231,6 +231,45 @@ pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	dcmi_pins_d: dcmi-3 {
+		pins {
+			pinmux = <STM32_PINMUX('H', 8,  AF13)>,/* DCMI_HSYNC */
+				 <STM32_PINMUX('B', 7,  AF13)>,/* DCMI_VSYNC */
+				 <STM32_PINMUX('A', 6,  AF13)>,/* DCMI_PIXCLK */
+				 <STM32_PINMUX('H', 9,  AF13)>,/* DCMI_D0 */
+				 <STM32_PINMUX('C', 7,  AF13)>,/* DCMI_D1 */
+				 <STM32_PINMUX('E', 0,  AF13)>,/* DCMI_D2 */
+				 <STM32_PINMUX('E', 1,  AF13)>,/* DCMI_D3 */
+				 <STM32_PINMUX('H', 14, AF13)>,/* DCMI_D4 */
+				 <STM32_PINMUX('I', 4,  AF13)>,/* DCMI_D5 */
+				 <STM32_PINMUX('E', 5,  AF13)>,/* DCMI_D6 */
+				 <STM32_PINMUX('I', 7,  AF13)>,/* DCMI_D7 */
+				 <STM32_PINMUX('I', 1,  AF13)>,/* DCMI_D8 */
+				 <STM32_PINMUX('H', 7,  AF13)>;/* DCMI_D9 */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	dcmi_sleep_pins_d: dcmi-sleep-3 {
+		pins {
+			pinmux = <STM32_PINMUX('H', 8,  ANALOG)>,/* DCMI_HSYNC */
+				 <STM32_PINMUX('B', 7,  ANALOG)>,/* DCMI_VSYNC */
+				 <STM32_PINMUX('A', 6,  ANALOG)>,/* DCMI_PIXCLK */
+				 <STM32_PINMUX('H', 9,  ANALOG)>,/* DCMI_D0 */
+				 <STM32_PINMUX('C', 7,  ANALOG)>,/* DCMI_D1 */
+				 <STM32_PINMUX('E', 0,  ANALOG)>,/* DCMI_D2 */
+				 <STM32_PINMUX('E', 1,  ANALOG)>,/* DCMI_D3 */
+				 <STM32_PINMUX('H', 14, ANALOG)>,/* DCMI_D4 */
+				 <STM32_PINMUX('I', 4,  ANALOG)>,/* DCMI_D5 */
+				 <STM32_PINMUX('E', 5,  ANALOG)>,/* DCMI_D6 */
+				 <STM32_PINMUX('I', 7,  ANALOG)>,/* DCMI_D7 */
+				 <STM32_PINMUX('I', 1,  ANALOG)>,/* DCMI_D8 */
+				 <STM32_PINMUX('H', 7,  ANALOG)>;/* DCMI_D9 */
+		};
+	};
+
 	/omit-if-no-ref/
 	ethernet0_rgmii_pins_a: rgmii-0 {
 		pins1 {
@@ -394,6 +433,7 @@ pins1 {
 	ethernet0_rgmii_pins_d: rgmii-3 {
 		pins1 {
 			pinmux = <STM32_PINMUX('G', 5, AF11)>, /* ETH_RGMII_CLK125 */
+				 <STM32_PINMUX('G', 4, AF11)>, /* ETH_RGMII_GTX_CLK */
 				 <STM32_PINMUX('G', 13, AF11)>,	/* ETH_RGMII_TXD0 */
 				 <STM32_PINMUX('G', 14, AF11)>,	/* ETH_RGMII_TXD1 */
 				 <STM32_PINMUX('C', 2, AF11)>, /* ETH_RGMII_TXD2 */
@@ -1343,6 +1383,65 @@ pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	ltdc_pins_f: ltdc-5 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 7,  AF14)>, /* LCD_CLK */
+				 <STM32_PINMUX('I', 10, AF14)>, /* LCD_HSYNC */
+				 <STM32_PINMUX('I', 9,  AF14)>, /* LCD_VSYNC */
+				 <STM32_PINMUX('E', 13, AF14)>, /* LCD_DE */
+				 <STM32_PINMUX('C', 10, AF14)>, /* LCD_R2 */
+				 <STM32_PINMUX('B', 0,   AF9)>, /* LCD_R3 */
+				 <STM32_PINMUX('H', 10, AF14)>, /* LCD_R4 */
+				 <STM32_PINMUX('H', 11, AF14)>, /* LCD_R5 */
+				 <STM32_PINMUX('H', 12, AF14)>, /* LCD_R6 */
+				 <STM32_PINMUX('E', 15, AF14)>, /* LCD_R7 */
+				 <STM32_PINMUX('H', 13, AF14)>, /* LCD_G2 */
+				 <STM32_PINMUX('E', 11, AF14)>, /* LCD_G3 */
+				 <STM32_PINMUX('H', 15, AF14)>, /* LCD_G4 */
+				 <STM32_PINMUX('H', 4,   AF9)>, /* LCD_G5 */
+				 <STM32_PINMUX('I', 11,  AF9)>, /* LCD_G6 */
+				 <STM32_PINMUX('I', 2,  AF14)>, /* LCD_G7 */
+				 <STM32_PINMUX('G', 10, AF14)>, /* LCD_B2 */
+				 <STM32_PINMUX('G', 11, AF14)>, /* LCD_B3 */
+				 <STM32_PINMUX('E', 12, AF14)>, /* LCD_B4 */
+				 <STM32_PINMUX('I', 5,  AF14)>, /* LCD_B5 */
+				 <STM32_PINMUX('B', 8,  AF14)>, /* LCD_B6 */
+				 <STM32_PINMUX('D', 8,  AF14)>; /* LCD_B7 */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+	};
+
+	/omit-if-no-ref/
+	ltdc_sleep_pins_f: ltdc-sleep-5 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 7,  ANALOG)>, /* LCD_CLK */
+				 <STM32_PINMUX('I', 10, ANALOG)>, /* LCD_HSYNC */
+				 <STM32_PINMUX('I', 9,  ANALOG)>, /* LCD_VSYNC */
+				 <STM32_PINMUX('E', 13, ANALOG)>, /* LCD_DE */
+				 <STM32_PINMUX('C', 10, ANALOG)>, /* LCD_R2 */
+				 <STM32_PINMUX('B', 0,  ANALOG)>, /* LCD_R3 */
+				 <STM32_PINMUX('H', 10, ANALOG)>, /* LCD_R4 */
+				 <STM32_PINMUX('H', 11, ANALOG)>, /* LCD_R5 */
+				 <STM32_PINMUX('H', 12, ANALOG)>, /* LCD_R6 */
+				 <STM32_PINMUX('E', 15, ANALOG)>, /* LCD_R7 */
+				 <STM32_PINMUX('H', 13, ANALOG)>, /* LCD_G2 */
+				 <STM32_PINMUX('E', 11, ANALOG)>, /* LCD_G3 */
+				 <STM32_PINMUX('H', 15, ANALOG)>, /* LCD_G4 */
+				 <STM32_PINMUX('H', 4,  ANALOG)>, /* LCD_G5 */
+				 <STM32_PINMUX('I', 11, ANALOG)>, /* LCD_G6 */
+				 <STM32_PINMUX('I', 2,  ANALOG)>, /* LCD_G7 */
+				 <STM32_PINMUX('G', 10, ANALOG)>, /* LCD_B2 */
+				 <STM32_PINMUX('G', 11, ANALOG)>, /* LCD_B3 */
+				 <STM32_PINMUX('E', 12, ANALOG)>, /* LCD_B4 */
+				 <STM32_PINMUX('I', 5,  ANALOG)>, /* LCD_B5 */
+				 <STM32_PINMUX('B', 8,  ANALOG)>, /* LCD_B6 */
+				 <STM32_PINMUX('D', 8,  ANALOG)>; /* LCD_B7 */
+		};
+	};
+
 	/omit-if-no-ref/
 	mco1_pins_a: mco1-0 {
 		pins {
@@ -1683,6 +1782,23 @@ pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	pwm5_pins_c: pwm5-2 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 0, AF2)>; /* TIM5_CH4 */
+			bias-pull-down;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	pwm5_sleep_pins_c: pwm5-sleep-2 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 0, ANALOG)>; /* TIM5_CH4 */
+		};
+	};
+
 	/omit-if-no-ref/
 	pwm8_pins_a: pwm8-0 {
 		pins {
@@ -1908,6 +2024,21 @@ pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	sai2a_pins_d: sai2a-3 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 6, AF10)>; /* SAI2_SD_A */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	sai2a_sleep_pins_d: sai2a-3 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 6, ANALOG)>; /* SAI2_SD_A */
+		};
+	};
+
 	/omit-if-no-ref/
 	sai2b_pins_a: sai2b-0 {
 		pins1 {
@@ -2895,6 +3026,39 @@ pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	uart4_pins_f: uart4-5 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 9, AF8)>; /* UART4_TX */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 2, AF8)>; /* UART4_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart4_idle_pins_f: uart4-idle-5 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 9, ANALOG)>; /* UART4_TX */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 2, AF8)>; /* UART4_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart4_sleep_pins_f: uart4-sleep-5 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 9, ANALOG)>, /* UART4_TX */
+				 <STM32_PINMUX('B', 2, ANALOG)>; /* UART4_RX */
+		};
+	};
+
 	/omit-if-no-ref/
 	uart5_pins_a: uart5-0 {
 		pins1 {
diff --git a/arch/arm/boot/dts/st/stm32mp151.dtsi b/arch/arm/boot/dts/st/stm32mp151.dtsi
index b1b568dfd1261..84f68e8563d85 100644
--- a/arch/arm/boot/dts/st/stm32mp151.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp151.dtsi
@@ -3,6 +3,7 @@
  * Copyright (C) STMicroelectronics 2017 - All Rights Reserved
  * Author: Ludovic Barre <ludovic.barre@st.com> for STMicroelectronics.
  */
+#include <dt-bindings/arm/coresight-cti-dt.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/stm32mp1-clks.h>
 #include <dt-bindings/reset/stm32mp1-resets.h>
@@ -123,6 +124,14 @@ soc {
 		interrupt-parent = <&intc>;
 		ranges;
 
+		sram4: sram@10050000 {
+			compatible = "mmio-sram";
+			reg = <0x10050000 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x10050000 0x10000>;
+		};
+
 		ipcc: mailbox@4c001000 {
 			compatible = "st,stm32mp1-ipcc";
 			#mbox-cells = <1>;
@@ -274,9 +283,180 @@ hdp: pinctrl@5002a000 {
 			compatible = "st,stm32mp151-hdp";
 			reg = <0x5002a000 0x400>;
 			clocks = <&rcc HDP>;
+			access-controllers = <&dbg_bus 1>;
 			status = "disabled";
 		};
 
+		dbg_bus: bus@50080000 {
+			compatible = "st,stm32mp151-dbg-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			#access-controller-cells = <1>;
+			ranges = <0x50080000 0x50080000 0x3f80000>,
+				 <0x90000000 0x90000000 0x1000000>;
+			status = "disabled";
+
+			cs_funnel: funnel@50091000 {
+				compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+				reg = <0x50091000 0x1000>;
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				in-ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					port@0 {
+						reg = <0>;
+						funnel_in_port0: endpoint {
+							remote-endpoint = <&stm_out_port>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+						funnel_in_port1: endpoint {
+							remote-endpoint = <&etm0_out>;
+						};
+					};
+				};
+
+				out-ports {
+					port {
+						funnel_out_port: endpoint {
+							remote-endpoint = <&etf_in_port>;
+						};
+					};
+				};
+			};
+
+			cs_etf: etf@50092000 {
+				compatible = "arm,coresight-tmc", "arm,primecell";
+				reg = <0x50092000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				in-ports {
+					port {
+						etf_in_port: endpoint {
+							remote-endpoint = <&funnel_out_port>;
+						};
+					};
+				};
+
+				out-ports {
+					port {
+						etf_out_port: endpoint {
+							remote-endpoint = <&tpiu_in_port>;
+						};
+					};
+				};
+			};
+
+			cs_tpiu: tpiu@50093000 {
+				compatible = "arm,coresight-tpiu", "arm,primecell";
+				reg = <0x50093000 0x1000>;
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				in-ports {
+					port {
+						tpiu_in_port: endpoint {
+							remote-endpoint = <&etf_out_port>;
+						};
+					};
+				};
+			};
+
+			cs_cti_trace: cti@50094000 {
+				compatible = "arm,coresight-cti", "arm,primecell";
+				reg = <0x50094000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+			};
+
+			cs_stm: stm@500a0000 {
+				compatible = "arm,coresight-stm", "arm,primecell";
+				reg = <0x500a0000 0x00001000>,
+				      <0x90000000 0x01000000>;
+				reg-names = "stm-base", "stm-stimulus-base";
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				out-ports {
+					port {
+						stm_out_port: endpoint {
+							remote-endpoint = <&funnel_in_port0>;
+						};
+					};
+				};
+			};
+
+			cs_cti_cpu0: cti@500d8000 {
+				compatible = "arm,coresight-cti", "arm,primecell";
+				reg = <0x500d8000 0x1000>;
+				clocks = <&rcc CK_DBG>;
+				clock-names = "apb_pclk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				trig-conns@0 {
+					reg = <0>;
+					arm,trig-in-sigs = <0 4 5>;
+					arm,trig-in-types = <PE_DBGTRIGGER
+							     GEN_IO
+							     GEN_IO>;
+					arm,trig-out-sigs = <0 7>;
+					arm,trig-out-types = <PE_EDBGREQ
+							      PE_DBGRESTART>;
+					cpu = <&cpu0>;
+				};
+
+				trig-conns@2 {
+					reg = <2>;
+					arm,trig-in-sigs = <2 3 6>;
+					arm,trig-in-types = <ETM_EXTOUT
+							     ETM_EXTOUT
+							     ETM_EXTOUT>;
+					arm,trig-out-sigs = <1 2 3 4>;
+					arm,trig-out-types = <ETM_EXTIN
+							      ETM_EXTIN
+							      ETM_EXTIN
+							      ETM_EXTIN>;
+					arm,cs-dev-assoc = <&cs_etm0>;
+				};
+			};
+
+			cs_etm0: etm@500dc000 {
+				compatible = "arm,coresight-etm3x", "arm,primecell";
+				reg = <0x500dc000 0x1000>;
+				cpu = <&cpu0>;
+				clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+				clock-names = "apb_pclk", "atclk";
+				access-controllers = <&dbg_bus 0>;
+				status = "disabled";
+
+				out-ports {
+					port {
+						etm0_out: endpoint {
+							remote-endpoint = <&funnel_in_port1>;
+						};
+					};
+				};
+			};
+		};
+
 		mdma1: dma-controller@58000000 {
 			compatible = "st,stm32h7-mdma";
 			reg = <0x58000000 0x1000>;
@@ -867,12 +1047,15 @@ i2c1: i2c@40012000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x40012000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 21 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C1_K>;
 				resets = <&rcc I2C1_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&dmamux1 33 0x400 0x1>,
+				       <&dmamux1 34 0x400 0x1>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x1>;
 				wakeup-source;
 				i2c-analog-filter;
@@ -884,12 +1067,15 @@ i2c2: i2c@40013000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x40013000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 22 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C2_K>;
 				resets = <&rcc I2C2_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&dmamux1 35 0x400 0x1>,
+				       <&dmamux1 36 0x400 0x1>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x2>;
 				wakeup-source;
 				i2c-analog-filter;
@@ -901,12 +1087,15 @@ i2c3: i2c@40014000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x40014000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 23 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C3_K>;
 				resets = <&rcc I2C3_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&dmamux1 73 0x400 0x1>,
+				       <&dmamux1 74 0x400 0x1>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x4>;
 				wakeup-source;
 				i2c-analog-filter;
@@ -918,12 +1107,15 @@ i2c5: i2c@40015000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x40015000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 25 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C5_K>;
 				resets = <&rcc I2C5_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&dmamux1 115 0x400 0x1>,
+				       <&dmamux1 116 0x400 0x1>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x10>;
 				wakeup-source;
 				i2c-analog-filter;
@@ -1083,7 +1275,7 @@ usart6: serial@44003000 {
 				clocks = <&rcc USART6_K>;
 				wakeup-source;
 				dmas = <&dmamux1 71 0x400 0x15>,
-				<&dmamux1 72 0x400 0x11>;
+				       <&dmamux1 72 0x400 0x11>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 51>;
 				status = "disabled";
@@ -1095,7 +1287,7 @@ i2s1: audio-controller@44004000 {
 				reg = <0x44004000 0x400>;
 				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&dmamux1 37 0x400 0x01>,
-				<&dmamux1 38 0x400 0x01>;
+				       <&dmamux1 38 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 52>;
 				status = "disabled";
@@ -1110,7 +1302,7 @@ spi1: spi@44004000 {
 				clocks = <&rcc SPI1_K>;
 				resets = <&rcc SPI1_R>;
 				dmas = <&dmamux1 37 0x400 0x05>,
-				<&dmamux1 38 0x400 0x05>;
+				       <&dmamux1 38 0x400 0x05>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 52>;
 				status = "disabled";
@@ -1125,7 +1317,7 @@ spi4: spi@44005000 {
 				clocks = <&rcc SPI4_K>;
 				resets = <&rcc SPI4_R>;
 				dmas = <&dmamux1 83 0x400 0x05>,
-				<&dmamux1 84 0x400 0x05>;
+				       <&dmamux1 84 0x400 0x05>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 53>;
 				status = "disabled";
@@ -1176,7 +1368,7 @@ timers16: timer@44007000 {
 				clocks = <&rcc TIM16_K>;
 				clock-names = "int";
 				dmas = <&dmamux1 109 0x400 0x1>,
-				<&dmamux1 110 0x400 0x1>;
+				       <&dmamux1 110 0x400 0x1>;
 				dma-names = "ch1", "up";
 				access-controllers = <&etzpc 55>;
 				status = "disabled";
@@ -1209,7 +1401,7 @@ timers17: timer@44008000 {
 				clocks = <&rcc TIM17_K>;
 				clock-names = "int";
 				dmas = <&dmamux1 111 0x400 0x1>,
-				<&dmamux1 112 0x400 0x1>;
+				       <&dmamux1 112 0x400 0x1>;
 				dma-names = "ch1", "up";
 				access-controllers = <&etzpc 56>;
 				status = "disabled";
@@ -1241,7 +1433,7 @@ spi5: spi@44009000 {
 				clocks = <&rcc SPI5_K>;
 				resets = <&rcc SPI5_R>;
 				dmas = <&dmamux1 85 0x400 0x05>,
-				<&dmamux1 86 0x400 0x05>;
+				       <&dmamux1 86 0x400 0x05>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 57>;
 				status = "disabled";
@@ -1829,12 +2021,15 @@ i2c4: i2c@5c002000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x5c002000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 24 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C4_K>;
 				resets = <&rcc I2C4_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&mdma1 36 0x0 0x40008 0x0 0x0>,
+				       <&mdma1 37 0x0 0x40002 0x0 0x0>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x8>;
 				wakeup-source;
 				i2c-analog-filter;
@@ -1846,12 +2041,15 @@ i2c6: i2c@5c009000 {
 				compatible = "st,stm32mp15-i2c";
 				reg = <0x5c009000 0x400>;
 				interrupt-names = "event", "error";
-				interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>,
-					     <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti 54 IRQ_TYPE_LEVEL_HIGH>,
+						      <&intc GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc I2C6_K>;
 				resets = <&rcc I2C6_R>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				dmas = <&mdma1 38 0x0 0x40008 0x0 0x0>,
+				       <&mdma1 39 0x0 0x40002 0x0 0x0>;
+				dma-names = "rx", "tx";
 				st,syscfg-fmp = <&syscfg 0x4 0x20>;
 				wakeup-source;
 				i2c-analog-filter;
diff --git a/arch/arm/boot/dts/st/stm32mp153.dtsi b/arch/arm/boot/dts/st/stm32mp153.dtsi
index 92794b942ab22..17d52d93695e1 100644
--- a/arch/arm/boot/dts/st/stm32mp153.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp153.dtsi
@@ -30,6 +30,74 @@ timer {
 	};
 };
 
+&cs_funnel {
+	in-ports {
+		port@2 {
+			reg = <2>;
+			funnel_in_port2: endpoint {
+				remote-endpoint = <&etm1_out>;
+			};
+		};
+	};
+};
+
+&dbg_bus {
+	cs_cti_cpu1: cti@500d9000 {
+		compatible = "arm,coresight-cti", "arm,primecell";
+		reg = <0x500d9000 0x1000>;
+		clocks = <&rcc CK_DBG>;
+		clock-names = "apb_pclk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		access-controllers = <&dbg_bus 0>;
+		status = "disabled";
+
+		trig-conns@0 {
+			reg = <0>;
+			arm,trig-in-sigs = <0 4 5>;
+			arm,trig-in-types = <PE_DBGTRIGGER
+					     GEN_IO
+					     GEN_IO>;
+			arm,trig-out-sigs = <0 7>;
+			arm,trig-out-types = <PE_EDBGREQ
+					      PE_DBGRESTART>;
+			cpu = <&cpu1>;
+		};
+
+		trig-conns@2 {
+			reg = <2>;
+			arm,trig-in-sigs = <2 3 6>;
+			arm,trig-in-types = <ETM_EXTOUT
+					     ETM_EXTOUT
+					     ETM_EXTOUT>;
+			arm,trig-out-sigs = <1 2 3 4>;
+			arm,trig-out-types = <ETM_EXTIN
+					      ETM_EXTIN
+					      ETM_EXTIN
+					      ETM_EXTIN>;
+			arm,cs-dev-assoc = <&cs_etm1>;
+		};
+	};
+
+	cs_etm1: etm@500dd000 {
+		compatible = "arm,coresight-etm3x", "arm,primecell";
+		reg = <0x500dd000 0x1000>;
+		cpu = <&cpu1>;
+		clocks = <&rcc CK_DBG>, <&rcc CK_TRACE>;
+		clock-names = "apb_pclk", "atclk";
+		access-controllers = <&dbg_bus 0>;
+		status = "disabled";
+
+		out-ports {
+			port {
+				etm1_out: endpoint {
+					remote-endpoint = <&funnel_in_port2>;
+				};
+			};
+		};
+	};
+};
+
 &etzpc {
 	m_can1: can@4400e000 {
 		compatible = "bosch,m_can";
diff --git a/arch/arm/boot/dts/st/stm32mp157c-ev1.dts b/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
index 4e46d58bf61f4..0e65a1862eb53 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
@@ -81,11 +81,59 @@ &cec {
 	status = "okay";
 };
 
+&cs_cti_trace {
+	status = "okay";
+};
+
+&cs_cti_cpu0 {
+	status = "okay";
+};
+
+&cs_cti_cpu1 {
+	status = "okay";
+};
+
+&cs_etf {
+	status = "okay";
+};
+
+&cs_etm0 {
+	status = "okay";
+};
+
+&cs_etm1 {
+	status = "okay";
+};
+
+&cs_funnel {
+	status = "okay";
+};
+
+&cs_stm {
+	status = "okay";
+};
+
+&cs_tpiu {
+	status = "okay";
+};
+
+&dbg_bus {
+	status = "okay";
+};
+
 &dcmi {
 	status = "okay";
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&dcmi_pins_a>;
 	pinctrl-1 = <&dcmi_sleep_pins_a>;
+	/*
+	 * Enable DMA-MDMA chaining by adding a SRAM pool and
+	 * a MDMA channel
+	 */
+	sram = <&dcmi_pool>;
+
+	dmas = <&dmamux1 75 0x400 0x01>, <&mdma1 0 0x3 0x1200000a 0 0>;
+	dma-names = "tx", "mdma_tx";
 
 	port {
 		dcmi_0: endpoint {
@@ -171,6 +219,8 @@ &i2c2 {
 	pinctrl-1 = <&i2c2_sleep_pins_a>;
 	i2c-scl-rising-time-ns = <185>;
 	i2c-scl-falling-time-ns = <20>;
+	/delete-property/dmas;
+	/delete-property/dma-names;
 	status = "okay";
 
 	ov5640: camera@3c {
@@ -227,6 +277,8 @@ &i2c5 {
 	pinctrl-1 = <&i2c5_sleep_pins_a>;
 	i2c-scl-rising-time-ns = <185>;
 	i2c-scl-falling-time-ns = <20>;
+	/delete-property/dmas;
+	/delete-property/dma-names;
 	status = "okay";
 };
 
@@ -302,6 +354,13 @@ &spi1 {
 	status = "disabled";
 };
 
+&sram4 {
+	dcmi_pool: dcmi-sram@0 {
+		reg = <0x0 0x8000>;
+		pool;
+	};
+};
+
 &timers2 {
 	/* spare dmas for other usage (un-delete to enable pwm capture) */
 	/delete-property/dmas;
diff --git a/arch/arm/boot/dts/st/stm32mp157c-phyboard-sargas-rdk.dts b/arch/arm/boot/dts/st/stm32mp157c-phyboard-sargas-rdk.dts
new file mode 100644
index 0000000000000..c18a372660837
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp157c-phyboard-sargas-rdk.dts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) Phytec GmbH 2019-2020 - All Rights Reserved
+ * Author: Dom VOVARD <dom.vovard@linrt.com>.
+ */
+
+/dts-v1/;
+
+#include "stm32mp157.dtsi"
+#include "stm32mp15xc.dtsi"
+#include "stm32mp15xx-phycore-som.dtsi"
+#include "stm32mp15xx-phyboard-sargas.dtsi"
+
+/ {
+	model = "PHYTEC phyBOARD-Sargas STM32MP157C";
+	compatible = "phytec,phycore-stm32mp1-3",
+		     "phytec,phycore-stm32mp157c-som", "st,stm32mp157";
+};
+
+&cryp1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&i2c4_eeprom {
+	status = "okay";
+};
+
+&i2c4_rtc {
+	status = "okay";
+};
+
+&qspi {
+	status = "okay";
+};
+
+&sdmmc2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp1-3.dts b/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp1-3.dts
deleted file mode 100644
index 28d7203264cea..0000000000000
--- a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp1-3.dts
+++ /dev/null
@@ -1,60 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
-/*
- * Copyright (C) Phytec GmbH 2019-2020 - All Rights Reserved
- * Author: Dom VOVARD <dom.vovard@linrt.com>.
- */
-
-/dts-v1/;
-
-#include <dt-bindings/pinctrl/stm32-pinfunc.h>
-#include "stm32mp157.dtsi"
-#include "stm32mp15xc.dtsi"
-#include "stm32mp15xxac-pinctrl.dtsi"
-#include "stm32mp157c-phycore-stm32mp15-som.dtsi"
-
-/ {
-	model = "PHYTEC phyCORE-STM32MP1-3 Dev Board";
-	compatible = "phytec,phycore-stm32mp1-3",
-		     "phytec,phycore-stm32mp157c-som", "st,stm32mp157";
-
-	aliases {
-		mmc0 = &sdmmc1;
-		mmc1 = &sdmmc2;
-		mmc2 = &sdmmc3;
-		serial0 = &uart4;
-		serial1 = &usart3;
-		serial2 = &usart1;
-	};
-};
-
-&cryp1 {
-	status = "okay";
-};
-
-&dts {
-	status = "okay";
-};
-
-&fmc {
-	status = "disabled";
-};
-
-&gpu {
-	status = "okay";
-};
-
-&i2c4_eeprom {
-	status = "okay";
-};
-
-&i2c4_rtc {
-	status = "okay";
-};
-
-&qspi {
-	status = "okay";
-};
-
-&sdmmc2 {
-	status = "okay";
-};
diff --git a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi b/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi
deleted file mode 100644
index 370b2afbf15bf..0000000000000
--- a/arch/arm/boot/dts/st/stm32mp157c-phycore-stm32mp15-som.dtsi
+++ /dev/null
@@ -1,573 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
-/*
- * Copyright (C) 2022-2023 Steffen Trumtrar <kernel@pengutronix.de>
- * Copyright (C) Phytec GmbH 2019-2020 - All Rights Reserved
- * Author: Dom VOVARD <dom.vovard@linrt.com>.
- */
-
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/input/input.h>
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/interrupt-controller/irq.h>
-#include <dt-bindings/leds/common.h>
-#include <dt-bindings/leds/leds-pca9532.h>
-#include <dt-bindings/mfd/st,stpmic1.h>
-#include <dt-bindings/net/ti-dp83867.h>
-#include "stm32mp15-pinctrl.dtsi"
-
-/ {
-	model = "PHYTEC phyCORE-STM32MP15 SOM";
-	compatible = "phytec,phycore-stm32mp157c-som", "st,stm32mp157";
-
-	aliases {
-		ethernet0 = &ethernet0;
-		rtc0 = &i2c4_rtc;
-		rtc1 = &rtc;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-
-		key-home {
-			label = "Home";
-			gpios = <&gpioa 13 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_HOME>;
-		};
-
-		key-enter {
-			label = "Enter";
-			gpios = <&gpioa 14 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_ENTER>;
-		};
-	};
-
-	reserved-memory {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		retram: retram@38000000 {
-			compatible = "shared-dma-pool";
-			reg = <0x38000000 0x10000>;
-			no-map;
-		};
-
-		mcuram: mcuram@30000000 {
-			compatible = "shared-dma-pool";
-			reg = <0x30000000 0x40000>;
-			no-map;
-		};
-
-		mcuram2: mcuram2@10000000 {
-			compatible = "shared-dma-pool";
-			reg = <0x10000000 0x40000>;
-			no-map;
-		};
-
-		vdev0vring0: vdev0vring0@10040000 {
-			compatible = "shared-dma-pool";
-			reg = <0x10040000 0x1000>;
-			no-map;
-		};
-
-		vdev0vring1: vdev0vring1@10041000 {
-			compatible = "shared-dma-pool";
-			reg = <0x10041000 0x1000>;
-			no-map;
-		};
-
-		vdev0buffer: vdev0buffer@10042000 {
-			compatible = "shared-dma-pool";
-			reg = <0x10042000 0x4000>;
-			no-map;
-		};
-	};
-
-	sound {
-		compatible = "audio-graph-card";
-		label = "STM32MP1-PHYCORE";
-		routing =
-			"Playback", "MCLK", /* Set a route between "MCLK" and "playback" widgets */
-			"Capture", "MCLK";
-		dais = <&sai2b_port>,
-		       <&sai2a_port>;
-	};
-
-	regulator_vin: regulator {
-		compatible = "regulator-fixed";
-		regulator-name = "vin";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-always-on;
-	};
-};
-
-&ethernet0 {
-	pinctrl-0 = <&ethernet0_rgmii_pins_d>;
-	pinctrl-1 = <&ethernet0_rgmii_sleep_pins_d>;
-	pinctrl-names = "default", "sleep";
-	phy-mode = "rgmii-id";
-	max-speed = <1000>;
-	phy-handle = <&phy0>;
-	st,eth-clk-sel;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "snps,dwmac-mdio";
-
-		phy0: ethernet-phy@1 {
-			compatible = "ethernet-phy-ieee802.3-c22";
-			reg = <1>;
-			interrupt-parent = <&gpiog>;
-			interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
-			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
-			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
-			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
-			ti,min-output-impedance;
-			enet-phy-lane-no-swap;
-			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
-		};
-	};
-};
-
-&i2c1 {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&i2c1_pins_b>;
-	pinctrl-1 = <&i2c1_sleep_pins_b>;
-	i2c-scl-rising-time-ns = <100>;
-	i2c-scl-falling-time-ns = <7>;
-	status = "okay";
-
-	codec@18 {
-		compatible = "ti,tlv320aic3007";
-		reg = <0x18>;
-		#sound-dai-cells = <0>;
-
-		ai3x-micbias-vg = <2>;
-
-		AVDD-supply = <&v3v3>;
-		IOVDD-supply = <&v3v3>;
-		DRVDD-supply = <&v3v3>;
-		DVDD-supply = <&v1v8_audio>;
-
-		clocks = <&sai2b>;
-
-		port {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			tlv320_tx_endpoint: endpoint@0 {
-				reg = <0>;
-				remote-endpoint = <&sai2b_endpoint>;
-				frame-master;
-				bitclock-master;
-			};
-
-			tlv320_rx_endpoint: endpoint@1 {
-				reg = <1>;
-				remote-endpoint = <&sai2a_endpoint>;
-				frame-master;
-				bitclock-master;
-			};
-		};
-	};
-
-	touch@44 {
-		compatible = "st,stmpe811";
-		reg = <0x44>;
-		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
-		interrupt-parent = <&gpioi>;
-		vio-supply = <&v3v3>;
-		vcc-supply = <&v3v3>;
-		st,sample-time = <4>;
-		st,mod-12b = <1>;
-		st,ref-sel = <0>;
-		st,adc-freq = <1>;
-
-		touchscreen {
-			compatible = "st,stmpe-ts";
-			st,ave-ctrl = <1>;
-			st,touch-det-delay = <2>;
-			st,settling = <2>;
-			st,fraction-z = <7>;
-			st,i-drive = <1>;
-		};
-	};
-
-	leds@62 {
-		compatible = "nxp,pca9533";
-		reg = <0x62>;
-
-		led-0 {
-			color = <LED_COLOR_ID_RED>;
-			function = LED_FUNCTION_POWER;
-			type = <PCA9532_TYPE_LED>;
-		};
-
-		led-1 {
-			color = <LED_COLOR_ID_GREEN>;
-			function = LED_FUNCTION_POWER;
-			type = <PCA9532_TYPE_LED>;
-		};
-
-		led-2 {
-			color = <LED_COLOR_ID_BLUE>;
-			function = LED_FUNCTION_HEARTBEAT;
-			type = <PCA9532_TYPE_LED>;
-			linux,default-trigger = "heartbeat";
-		};
-	};
-};
-
-&i2c4 {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&i2c4_pins_a>;
-	pinctrl-1 = <&i2c4_sleep_pins_a>;
-	i2c-scl-rising-time-ns = <185>;
-	i2c-scl-falling-time-ns = <20>;
-	status = "okay";
-
-	pmic@33 {
-		compatible = "st,stpmic1";
-		reg = <0x33>;
-		interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>;
-		interrupt-controller;
-		#interrupt-cells = <2>;
-
-		regulators {
-			compatible = "st,stpmic1-regulators";
-			buck1-supply = <&regulator_vin>;
-			buck2-supply = <&regulator_vin>;
-			buck3-supply = <&regulator_vin>;
-			buck4-supply = <&regulator_vin>;
-			ldo1-supply = <&v3v3>;
-			ldo2-supply = <&v3v3>;
-			ldo3-supply = <&vdd_ddr>;
-			ldo4-supply = <&regulator_vin>;
-			ldo5-supply = <&v3v3>;
-			ldo6-supply = <&v3v3>;
-			boost-supply = <&regulator_vin>;
-			pwr_sw1-supply = <&bst_out>;
-			pwr_sw2-supply = <&bst_out>;
-
-			vddcore: buck1 {
-				regulator-name = "vddcore";
-				regulator-min-microvolt = <1200000>;
-				regulator-max-microvolt = <1350000>;
-				regulator-always-on;
-				regulator-initial-mode = <0>;
-			};
-
-			vdd_ddr: buck2 {
-				regulator-name = "vdd_ddr";
-				regulator-min-microvolt = <1350000>;
-				regulator-max-microvolt = <1350000>;
-				regulator-always-on;
-				regulator-initial-mode = <0>;
-			};
-
-			vdd: buck3 {
-				regulator-name = "vdd";
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-always-on;
-				st,mask-reset;
-				regulator-initial-mode = <0>;
-			};
-
-			v3v3: buck4 {
-				regulator-name = "v3v3";
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-always-on;
-				regulator-initial-mode = <0>;
-			};
-
-			v1v8_audio: ldo1 {
-				regulator-name = "v1v8_audio";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-always-on;
-				interrupts = <IT_CURLIM_LDO1 0>;
-
-			};
-
-			vdd_eth_2v5: ldo2 {
-				regulator-name = "dd_eth_2v5";
-				regulator-min-microvolt = <2500000>;
-				regulator-max-microvolt = <2500000>;
-				regulator-always-on;
-				interrupts = <IT_CURLIM_LDO2 0>;
-
-			};
-
-			vtt_ddr: ldo3 {
-				regulator-name = "vtt_ddr";
-				regulator-min-microvolt = <500000>;
-				regulator-max-microvolt = <750000>;
-				regulator-always-on;
-				regulator-over-current-protection;
-			};
-
-			vdd_usb: ldo4 {
-				regulator-name = "vdd_usb";
-				interrupts = <IT_CURLIM_LDO4 0>;
-			};
-
-			vdda: ldo5 {
-				regulator-name = "vdda";
-				regulator-min-microvolt = <2900000>;
-				regulator-max-microvolt = <2900000>;
-				interrupts = <IT_CURLIM_LDO5 0>;
-				regulator-boot-on;
-			};
-
-			vdd_eth_1v0: ldo6 {
-				regulator-name = "vdd_eth_1v0";
-				regulator-min-microvolt = <1000000>;
-				regulator-max-microvolt = <1000000>;
-				regulator-always-on;
-				interrupts = <IT_CURLIM_LDO6 0>;
-
-			};
-
-			vref_ddr: vref_ddr {
-				regulator-name = "vref_ddr";
-				regulator-always-on;
-			};
-
-			bst_out: boost {
-				regulator-name = "bst_out";
-				interrupts = <IT_OCP_BOOST 0>;
-			};
-
-			vbus_otg: pwr_sw1 {
-				regulator-name = "vbus_otg";
-				interrupts = <IT_OCP_OTG 0>;
-				regulator-active-discharge = <1>;
-			};
-
-			vbus_sw: pwr_sw2 {
-				regulator-name = "vbus_sw";
-				interrupts = <IT_OCP_SWOUT 0>;
-				regulator-active-discharge = <1>;
-			};
-		};
-
-		onkey {
-			compatible = "st,stpmic1-onkey";
-			interrupts = <IT_PONKEY_F 0>,
-				     <IT_PONKEY_R 0>;
-			interrupt-names = "onkey-falling",
-					  "onkey-rising";
-			power-off-time-sec = <10>;
-		};
-
-		watchdog {
-			compatible = "st,stpmic1-wdt";
-		};
-	};
-
-	i2c4_eeprom: eeprom@50 {
-		compatible = "microchip,24c32",
-			     "atmel,24c32";
-		reg = <0x50>;
-	};
-
-	i2c4_rtc: rtc@52 {
-		compatible = "microcrystal,rv3028";
-		reg = <0x52>;
-	};
-};
-
-&ipcc {
-	status = "okay";
-};
-
-&iwdg2 {
-	timeout-sec = <32>;
-	status = "okay";
-};
-
-&m_can2 {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&m_can2_pins_a>;
-	pinctrl-1 = <&m_can2_sleep_pins_a>;
-	status = "okay";
-};
-
-&m4_rproc {
-	memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>,
-			<&vdev0vring1>, <&vdev0buffer>;
-	mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>, <&ipcc 3>;
-	mbox-names = "vq0", "vq1", "shutdown", "detach";
-	interrupt-parent = <&exti>;
-	interrupts = <68 1>;
-	status = "okay";
-};
-
-&pwr_regulators {
-	vdd-supply = <&vdd>;
-	vdd_3v3_usbfs-supply = <&vdd_usb>;
-};
-
-&qspi {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a>;
-	pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a>;
-	status = "okay";
-
-	flash0: flash@0 {
-		compatible = "winbond,w25q128", "jedec,spi-nor";
-		reg = <0>;
-		spi-rx-bus-width = <4>;
-		spi-max-frequency = <50000000>;
-		m25p,fast-read;
-		#address-cells = <1>;
-		#size-cells = <1>;
-	};
-};
-
-&rng1 {
-	status = "okay";
-};
-
-&rtc {
-	status = "okay";
-};
-
-&sai2 {
-	clocks = <&rcc SAI2>, <&rcc PLL3_Q>, <&rcc PLL3_R>;
-	clock-names = "pclk", "x8k", "x11k";
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&sai2a_pins_b>, <&sai2b_pins_d>;
-	pinctrl-1 = <&sai2a_sleep_pins_b>, <&sai2b_sleep_pins_d>;
-	status = "okay";
-};
-
-&sai2a {
-	dma-names = "rx";
-	st,sync = <&sai2b 2>;
-	clocks = <&rcc SAI2_K>, <&sai2b>;
-	clock-names = "sai_ck", "MCLK";
-	#clock-cells = <0>;
-
-	sai2a_port: port {
-		sai2a_endpoint: endpoint {
-			remote-endpoint = <&tlv320_rx_endpoint>;
-			mclk-fs = <256>;
-			dai-tdm-slot-num = <2>;
-			dai-tdm-slot-width = <16>;
-		};
-	};
-};
-
-&sai2b {
-	dma-names = "tx";
-	#clock-cells = <0>;
-
-	sai2b_port: port {
-		sai2b_endpoint: endpoint {
-			remote-endpoint = <&tlv320_tx_endpoint>;
-			mclk-fs = <256>;
-			dai-tdm-slot-num = <2>;
-			dai-tdm-slot-width = <16>;
-		};
-	};
-};
-
-&sdmmc1 {
-	pinctrl-names = "default", "opendrain", "sleep";
-	pinctrl-0 = <&sdmmc1_b4_pins_b>;
-	pinctrl-1 = <&sdmmc1_b4_od_pins_b>;
-	pinctrl-2 = <&sdmmc1_b4_sleep_pins_b>;
-	cd-gpios = <&gpiof 3 GPIO_ACTIVE_LOW>;
-	disable-wp;
-	st,neg-edge;
-	bus-width = <4>;
-	vmmc-supply = <&v3v3>;
-	status = "okay";
-};
-
-&sdmmc2 {
-	pinctrl-names = "default", "opendrain", "sleep";
-	pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_e>;
-	pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_e>;
-	pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_e>;
-	non-removable;
-	no-sd;
-	no-sdio;
-	st,neg-edge;
-	bus-width = <8>;
-	vmmc-supply = <&v3v3>;
-	vqmmc-supply = <&v3v3>;
-	mmc-ddr-3_3v;
-};
-
-&spi1 {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&spi1_pins_a>;
-	pinctrl-1 = <&spi1_sleep_pins_a>;
-	cs-gpios = <&gpioz 3 0>;
-	status = "okay";
-};
-
-&uart4 {
-	pinctrl-names = "default", "sleep", "idle";
-	pinctrl-0 = <&uart4_pins_a>;
-	pinctrl-1 = <&uart4_sleep_pins_a>;
-	pinctrl-2 = <&uart4_idle_pins_a>;
-	pinctrl-3 = <&uart4_pins_a>;
-	/delete-property/dmas;
-	/delete-property/dma-names;
-	status = "okay";
-};
-
-&usart1 {
-	pinctrl-names = "default", "sleep", "idle";
-	pinctrl-0 = <&usart1_pins_b &usart1_pins_a>;
-	pinctrl-1 = <&usart1_sleep_pins_b &usart1_sleep_pins_a>;
-	pinctrl-2 = <&usart1_idle_pins_b &usart1_idle_pins_a>;
-	uart-has-rtscts;
-	status = "okay";
-};
-
-&usart3 {
-	pinctrl-names = "default", "sleep", "idle";
-	pinctrl-0 = <&usart3_pins_a>;
-	pinctrl-1 = <&usart3_sleep_pins_a>;
-	pinctrl-2 = <&usart3_idle_pins_a>;
-	status = "okay";
-};
-
-&usbh_ehci {
-	status = "okay";
-};
-
-&usbh_ohci {
-	status = "okay";
-};
-
-&usbotg_hs {
-	phys = <&usbphyc_port1 0>;
-	phy-names = "usb2-phy";
-	status = "okay";
-};
-
-&usbphyc {
-	status = "okay";
-};
-
-&usbphyc_port0 {
-	phy-supply = <&vdd_usb>;
-};
-
-&usbphyc_port1 {
-	phy-supply = <&vdd_usb>;
-};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtso
new file mode 100644
index 0000000000000..103a2f0cf57b0
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-otm8009a.dtso
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+#include <dt-bindings/gpio/gpio.h>
+
+/dts-v1/;
+/plugin/;
+
+&dsi {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	phy-dsi-supply = <&reg18>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			dsi_in: endpoint {
+				remote-endpoint = <&ltdc_ep1_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			dsi_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel@0 {
+		compatible = "orisetech,otm8009a";
+		reg = <0>;
+		reset-gpios = <&gpiod 8 GPIO_ACTIVE_LOW>;
+		power-supply = <&v3v3>;
+		status = "okay";
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi_out>;
+			};
+		};
+	};
+};
+
+&ltdc {
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ltdc_ep1_out: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&dsi_in>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtso
new file mode 100644
index 0000000000000..cde2f8f68f86b
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-644-100-x6-rpi7inch.dtso
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen: touchscreen@38 {
+	};
+
+	attiny: regulator@45 {
+	};
+};
+
+#include "stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi"
+
+&ltdc {
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ltdc_ep_out: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&dsi_in>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan1-x6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan1-x6.dtso
new file mode 100644
index 0000000000000..a9916aa8df752
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan1-x6.dtso
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&m_can1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan2-x6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan2-x6.dtso
new file mode 100644
index 0000000000000..c994ff0552272
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-fdcan2-x6.dtso
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&m_can2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtso
new file mode 100644
index 0000000000000..1edfff973a815
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c1-eeprom-x6.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@56 {
+		compatible = "atmel,24c04";
+		reg = <0x56>;
+		pagesize = <16>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtso
new file mode 100644
index 0000000000000..bb3db38e3e63c
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-i2c2-eeprom-x6.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@56 {
+		compatible = "atmel,24c04";
+		reg = <0x56>;
+		pagesize = <16>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-ov5640-x7.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-ov5640-x7.dtso
new file mode 100644
index 0000000000000..3056be6cd1962
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-ov5640-x7.dtso
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+#include <dt-bindings/clock/stm32mp1-clks.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	camera0_1v5_pwr: regulator-camera0-1v5 {
+		compatible = "regulator-fixed";
+		regulator-name = "camera0-1v5-reg";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		regulator-always-on;
+	};
+
+	camera0_1v8_pwr: regulator-camera0-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "camera0-1v8-reg";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	camera0_2v8_pwr: regulator-camera0-2v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "camera0-2v8-reg";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		regulator-always-on;
+	};
+};
+
+&dcmi {
+	status = "okay";
+};
+
+&dcmi_0 {
+	hsync-active = <0>;
+	vsync-active = <0>;
+	pclk-sample = <0>;
+};
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@3c {
+		compatible = "ovti,ov5640";
+		reg = <0x3c>;
+		clocks = <&rcc CK_MCO1>;
+		clock-names = "xclk";
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&mco1_pins_a>;
+		pinctrl-1 = <&mco1_sleep_pins_a>;
+		assigned-clocks = <&rcc CK_MCO1>;
+		assigned-clock-parents = <&rcc CK_HSE>;
+		assigned-clock-rates = <24000000>;
+		AVDD-supply = <&camera0_2v8_pwr>;
+		DOVDD-supply = <&camera0_1v8_pwr>;
+		DVDD-supply = <&camera0_1v5_pwr>;
+		/* GPIO-J on the Dragonboard Dual-Leopard OV5640 board */
+		powerdown-gpios = <&gpiob 5 GPIO_ACTIVE_HIGH>;
+		/* GPIO-I on the Dragonboard Dual-Leopard OV5640 board */
+		reset-gpios = <&gpioa 12 GPIO_ACTIVE_LOW>;
+		rotation = <180>;
+		status = "okay";
+
+		port {
+			ov5640_0: endpoint {
+				remote-endpoint = <&stmipi_0>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+			};
+		};
+	};
+};
+
+&stmipi {
+	status = "okay";
+};
+
+&stmipi_0 {
+	data-lanes = <1 2>;
+	remote-endpoint = <&ov5640_0>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtso
new file mode 100644
index 0000000000000..acfd25c5bbcd2
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-avenger96-overlay-spi2-eeprom-x6.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&spi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi2_pins_a>;
+	status = "okay";
+	cs-gpios = <&gpioi 0 0>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@0 {
+		compatible = "microchip,25aa010a", "atmel,at25";
+		reg = <0>;
+		address-width = <8>;
+		pagesize = <16>;
+		size = <128>;
+		spi-max-frequency = <5000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtso
new file mode 100644
index 0000000000000..aa79f95906f8c
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-drc02-overlay-wifi-rsi.dtso
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2021 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&sdmmc3 {
+	broken-cd;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi
new file mode 100644
index 0000000000000..be9eb1e11ecd2
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2021 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi.dtsi"
+
+&{/} {
+	lvds-encoder {
+		compatible = "onnn,fin3385", "lvds-encoder";
+		pclk-sample = <1>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lvds_bridge_in: endpoint {
+					remote-endpoint = <&ltdc_dpi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				lvds_bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+};
+
+&display_bl {
+	pwms = <&pwm2 3 5000000 0>;
+};
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	ili251x@41 {
+		compatible = "ilitek,ili251x";
+		reg = <0x41>;
+		interrupt-parent = <&gpioi>;
+		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpiod 6 GPIO_ACTIVE_LOW>;
+		touchscreen-size-x = <16384>;
+		touchscreen-size-y = <9600>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c04";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&ltdc_dpi_out {
+	remote-endpoint = <&lvds_bridge_in>;
+};
+
+&panel {
+	compatible = "chefree,ch101olhlwh-002";
+};
+
+&panel_in {
+	remote-endpoint = <&lvds_bridge_out>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi.dtsi
new file mode 100644
index 0000000000000..41229ec680cc4
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-overlay-panel-dpi.dtsi
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2021 Marek Vasut
+ */
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/pwm/pwm.h>
+
+&{/} {
+	display_bl: display-bl {
+		compatible = "pwm-backlight";
+		brightness-levels = <0 16 22 30 40 55 75 102 138 188 255>;
+		default-brightness-level = <8>;
+		enable-gpios = <&gpioi 0 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_panel_bl>;
+		status = "okay";
+	};
+
+	panel: panel {
+		backlight = <&display_bl>;
+		power-supply = <&reg_panel_bl>;
+
+		port {
+			panel_in: endpoint {
+			};
+		};
+	};
+
+	reg_panel_bl: regulator-panel-bl {
+		compatible = "regulator-fixed";
+		regulator-name = "panel_backlight";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_panel_supply>;
+	};
+
+	reg_panel_supply: regulator-panel-supply {
+		compatible = "regulator-fixed";
+		regulator-name = "panel_supply";
+		regulator-min-microvolt = <24000000>;
+		regulator-max-microvolt = <24000000>;
+	};
+};
+
+&timers2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	pwm2: pwm {
+		#pwm-cells = <3>;
+		pinctrl-0 = <&pwm2_pins_a>;
+		pinctrl-names = "default";
+		status = "okay";
+	};
+
+	timer@1 {
+		reg = <1>;
+		status = "okay";
+	};
+};
+
+&ltdc {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&ltdc_pins_b>;
+	pinctrl-1 = <&ltdc_sleep_pins_b>;
+	status = "okay";
+
+	port {
+		ltdc_dpi_out: endpoint {
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtso
new file mode 100644
index 0000000000000..161e401f5e21b
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-460-200-x11.dtso
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&fmc {
+	#address-cells = <2>;
+	#size-cells = <1>;
+
+	sram@3,0 {
+		compatible = "mtd-ram";
+		reg = <3 0x0 0x80000>;
+		bank-width = <2>;
+
+		/* Timing values are in nS */
+		st,fmc2-ebi-cs-mux-enable;
+		st,fmc2-ebi-cs-transaction-type = <4>;
+		st,fmc2-ebi-cs-buswidth = <16>;
+		st,fmc2-ebi-cs-address-setup-ns = <6>;
+		st,fmc2-ebi-cs-address-hold-ns = <6>;
+		st,fmc2-ebi-cs-data-setup-ns = <127>;
+		st,fmc2-ebi-cs-bus-turnaround-ns = <9>;
+		st,fmc2-ebi-cs-data-hold-ns = <9>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtso
new file mode 100644
index 0000000000000..1de2445450540
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-497-200-x12.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi.dtsi"
+
+&display_bl {
+	pwms = <&pwm2 3 500000 PWM_POLARITY_INVERTED>;
+};
+
+&ltdc_dpi_out {
+	remote-endpoint = <&panel_in>;
+};
+
+&panel {
+	compatible = "dataimage,scf0700c48ggu18";
+};
+
+&panel_in {
+	remote-endpoint = <&ltdc_dpi_out>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtso
new file mode 100644
index 0000000000000..6ef9bcf527ad0
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-505-200-x12-ch101olhlwh.dtso
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi"
+
+&{/} {
+	gpio-keys-polled {
+		/* BUTTON1 GPIO-B conflicts with touchscreen reset */
+		button-1 {
+			/* Use status as /delete-node/ does not work in DTOs */
+			status = "disabled";
+		};
+	};
+
+	led {
+		/* LED7 GPIO-H conflicts with touchscreen IRQ */
+		led-2 {
+			/* Use status as /delete-node/ does not work in DTOs */
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtso
new file mode 100644
index 0000000000000..ce291736abbb2
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x21.dtso
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@56 {
+		compatible = "atmel,24c04";
+		reg = <0x56>;
+		pagesize = <16>;
+	};
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_pins_a>;
+	status = "okay";
+	cs-gpios = <&gpioz 3 0>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@0 {
+		compatible = "microchip,25aa010a", "atmel,at25";
+		reg = <0>;
+		address-width = <8>;
+		pagesize = <16>;
+		size = <128>;
+		spi-max-frequency = <5000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtso
new file mode 100644
index 0000000000000..7e040b2d8f248
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-531-100-x22.dtso
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@56 {
+		compatible = "atmel,24c04";
+		reg = <0x56>;
+		pagesize = <16>;
+	};
+};
+
+/* SPI2 is not connected on STM32MP1 DHCOM SoM */
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtso
new file mode 100644
index 0000000000000..a5cef9ba7dd2a
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-560-200-x12.dtso
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi.dtsi"
+
+&{/} {
+	gpio-keys {
+		/*
+		 * The EXTi IRQ line 6 is shared with touchscreen IRQ,
+		 * so operate button-1 as polled GPIO key.
+		 */
+		button-1 {
+			/* Use status as /delete-node/ does not work in DTOs */
+			status = "disabled";
+		};
+	};
+
+	gpio-keys-polled {
+		button-1 {
+			label = "TA2-GPIO-B";
+			linux,code = <KEY_B>;
+			gpios = <&gpiod 6 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	led {
+		/* LED5 GPIO-E conflicts with touchscreen IRQ */
+		led-0 {
+			/* Use status as /delete-node/ does not work in DTOs */
+			status = "disabled";
+		};
+	};
+};
+
+&display_bl {
+	pwms = <&pwm2 3 500000 PWM_POLARITY_INVERTED>;
+};
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen@38 {
+		compatible = "edt,edt-ft5406";
+		reg = <0x38>;
+		/* Touchscreen IRQ GPIO-E conflicts with LED5 GPIO */
+		interrupt-parent = <&gpioc>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>; /* GPIO E */
+	};
+};
+
+&ltdc_dpi_out {
+	remote-endpoint = <&panel_in>;
+};
+
+&panel {
+	compatible = "edt,etm0700g0edh6";
+};
+
+&panel_in {
+	remote-endpoint = <&ltdc_dpi_out>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtso
new file mode 100644
index 0000000000000..ee8a2d1a7b87a
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-638-100-x12-rpi7inch.dtso
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen: touchscreen@38 {
+	};
+
+	attiny: regulator@45 {
+	};
+};
+
+#include "stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi"
+
+&ltdc {
+	status = "okay";
+	port {
+		ltdc_ep_out: endpoint {
+			remote-endpoint = <&dsi_in>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtso
new file mode 100644
index 0000000000000..41e473986189b
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2-overlay-672-100-x18.dtso
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&m_can2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&m_can2_pins_a>;
+	pinctrl-1 = <&m_can2_sleep_pins_a>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2.dtsi
index 5c77202ee1966..0075d93911812 100644
--- a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-pdk2.dtsi
@@ -13,15 +13,6 @@ clk_ext_audio_codec: clock-codec {
 		clock-frequency = <24000000>;
 	};
 
-	display_bl: display-bl {
-		compatible = "pwm-backlight";
-		pwms = <&pwm2 3 500000 PWM_POLARITY_INVERTED>;
-		brightness-levels = <0 16 22 30 40 55 75 102 138 188 255>;
-		default-brightness-level = <8>;
-		enable-gpios = <&gpioi 0 GPIO_ACTIVE_HIGH>;
-		power-supply = <&reg_panel_bl>;
-	};
-
 	gpio-keys-polled {
 		compatible = "gpio-keys-polled";
 		poll-interval = <20>;
@@ -75,7 +66,6 @@ led-0 {
 			label = "green:led5";
 			gpios = <&gpioc 6 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
-			status = "disabled";
 		};
 
 		led-1 {
@@ -97,33 +87,6 @@ led-3 {
 		};
 	};
 
-	panel {
-		compatible = "edt,etm0700g0edh6";
-		backlight = <&display_bl>;
-		power-supply = <&reg_panel_bl>;
-
-		port {
-			lcd_panel_in: endpoint {
-				remote-endpoint = <&lcd_display_out>;
-			};
-		};
-	};
-
-	reg_panel_bl: regulator-panel-bl {
-		compatible = "regulator-fixed";
-		regulator-name = "panel_backlight";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&reg_panel_supply>;
-	};
-
-	reg_panel_supply: regulator-panel-supply {
-		compatible = "regulator-fixed";
-		regulator-name = "panel_supply";
-		regulator-min-microvolt = <24000000>;
-		regulator-max-microvolt = <24000000>;
-	};
-
 	sound {
 		compatible = "audio-graph-card";
 		widgets = "Headphone", "Headphone Jack",
@@ -188,26 +151,6 @@ sgtl5000_rx_endpoint: endpoint@1 {
 		};
 
 	};
-
-	touchscreen@38 {
-		compatible = "edt,edt-ft5406";
-		reg = <0x38>;
-		interrupt-parent = <&gpioc>;
-		interrupts = <6 IRQ_TYPE_EDGE_FALLING>; /* GPIO E */
-	};
-};
-
-&ltdc {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&ltdc_pins_b>;
-	pinctrl-1 = <&ltdc_sleep_pins_b>;
-	status = "okay";
-
-	port {
-		lcd_display_out: endpoint {
-			remote-endpoint = <&lcd_panel_in>;
-		};
-	};
 };
 
 &sai2 {
@@ -259,21 +202,6 @@ sai2b_endpoint: endpoint {
 	};
 };
 
-&timers2 {
-	/* spare dmas for other usage (un-delete to enable pwm capture) */
-	/delete-property/dmas;
-	/delete-property/dma-names;
-	status = "okay";
-	pwm2: pwm {
-		pinctrl-0 = <&pwm2_pins_a>;
-		pinctrl-names = "default";
-		status = "okay";
-	};
-	timer@1 {
-		status = "okay";
-	};
-};
-
 &usart3 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&usart3_pins_a>;
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtso
new file mode 100644
index 0000000000000..c462c6a08833d
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-548-200-x2-mi0700s4t-6.dtso
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2021 Andreas Geisreiter <ageisreiter@dh-electronics.com>
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi.dtsi"
+
+&display_bl {
+	pwms = <&pwm2 3 10000000 0>;
+};
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@50 {
+		compatible = "atmel,24c04";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&ltdc_dpi_out {
+	remote-endpoint = <&panel_in>;
+};
+
+&panel {
+	compatible = "multi-inno,mi0700s4t-6";
+};
+
+&panel_in {
+	remote-endpoint = <&ltdc_dpi_out>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtso
new file mode 100644
index 0000000000000..06338b7f7b679
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-553-100-x2-tst043015cmhx.dtso
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2023 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi.dtsi"
+
+&display_bl {
+	pwms = <&pwm2 3 10000000 0>;
+};
+
+&i2c5 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	eeprom@50 {
+		compatible = "atmel,24c04";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&ltdc_dpi_out {
+	remote-endpoint = <&panel_in>;
+};
+
+&panel {
+	compatible = "team-source-display,tst043015cmhx";
+};
+
+&panel_in {
+	remote-endpoint = <&ltdc_dpi_out>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtso b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtso
new file mode 100644
index 0000000000000..bf5c1f6eece0d
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-picoitx-overlay-626-100-x2-ch101olhlwh.dtso
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2020 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "stm32mp15xx-dhcom-overlay-panel-dpi-ch101olhlwh.dtsi"
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcor-avenger96.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhcor-avenger96.dtsi
index aceeff6c38ba1..85d93ddfa12a1 100644
--- a/arch/arm/boot/dts/st/stm32mp15xx-dhcor-avenger96.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcor-avenger96.dtsi
@@ -258,15 +258,9 @@ &i2c2 {	/* X6 I2C2 */
 &i2c4 {
 	stmipi: stmipi@14 {
 		compatible = "st,st-mipid02";
-		pinctrl-names = "default", "sleep";
-		pinctrl-0 = <&mco1_pins_a>;
-		pinctrl-1 = <&mco1_sleep_pins_a>;
 		reg = <0x14>;
 		clocks = <&rcc CK_MCO1>;
 		clock-names = "xclk";
-		assigned-clocks = <&rcc CK_MCO1>;
-		assigned-clock-parents = <&rcc CK_HSE>;
-		assigned-clock-rates = <24000000>;
 		VDDE-supply = <&v1v8>;
 		VDDIN-supply = <&v1v8>;
 		reset-gpios = <&gpioz 0 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi
new file mode 100644
index 0000000000000..518c269a1dba1
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhsom-overlay-panel-dsi-rpi7inch.dtsi
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2021 Marek Vasut
+ */
+#include <dt-bindings/gpio/gpio.h>
+
+&{/} {
+	panel {
+		compatible = "powertip,ph800480t013-idf02";
+		backlight = <&attiny>;
+		power-supply = <&attiny>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&bridge_out>;
+			};
+		};
+	};
+};
+
+&attiny {
+	compatible = "raspberrypi,7inch-touchscreen-panel-regulator";
+	gpio-controller;
+	#gpio-cells = <2>;
+	reg = <0x45>;
+};
+
+&dsi {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	phy-dsi-supply = <&reg18>;
+	status = "okay";
+
+	bridge@0 {
+		compatible = "toshiba,tc358762";
+		reg = <0>;
+		reset-gpios = <&attiny 0 GPIO_ACTIVE_HIGH>;
+		vddc-supply = <&attiny>;
+		status = "okay";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				bridge_in: endpoint {
+					remote-endpoint = <&dsi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			dsi_in: endpoint {
+				remote-endpoint = <&ltdc_ep_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			dsi_out: endpoint {
+				remote-endpoint = <&bridge_in>;
+			};
+		};
+	};
+};
+
+&touchscreen {
+	compatible = "edt,edt-ft5406";
+	reg = <0x38>;
+	reset-gpios = <&attiny 1 GPIO_ACTIVE_LOW>;
+	/*
+	 * Disabled, since the IRQ line is not on
+	 * the FPC cable, so we cannot get touch
+	 * IRQs unless its connected otherwise. In
+	 * that case, add entry like this one and
+	 * enable below.
+	 *
+	 * interrupt-parent = <&gpiog>;
+	 * interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+	 */
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
index 7ed2b01958fef..599ea07bdb19c 100644
--- a/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
@@ -155,6 +155,46 @@ &crc1 {
 	status = "okay";
 };
 
+&cs_cti_trace {
+	status = "okay";
+};
+
+&cs_cti_cpu0 {
+	status = "okay";
+};
+
+&cs_cti_cpu1 {
+	status = "okay";
+};
+
+&cs_etf {
+	status = "okay";
+};
+
+&cs_etm0 {
+	status = "okay";
+};
+
+&cs_etm1 {
+	status = "okay";
+};
+
+&cs_funnel {
+	status = "okay";
+};
+
+&cs_stm {
+	status = "okay";
+};
+
+&cs_tpiu {
+	status = "okay";
+};
+
+&dbg_bus {
+	status = "okay";
+};
+
 &dts {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-phyboard-sargas.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-phyboard-sargas.dtsi
new file mode 100644
index 0000000000000..f7e7aa9b916a1
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-phyboard-sargas.dtsi
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) 2022-2023 Steffen Trumtrar <kernel@pengutronix.de>
+ * Copyright (C) Phytec GmbH 2019-2020 - All Rights Reserved
+ * Author: Dom VOVARD <dom.vovard@linrt.com>.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/leds/leds-pca9532.h>
+
+/ {
+	aliases {
+		mmc0 = &sdmmc1;
+		mmc1 = &sdmmc2;
+		mmc2 = &sdmmc3;
+		serial0 = &uart4;
+		serial1 = &usart3;
+		serial2 = &usart1;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		key-home {
+			label = "Home";
+			gpios = <&gpioa 13 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOME>;
+		};
+
+		key-enter {
+			label = "Enter";
+			gpios = <&gpioa 14 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_ENTER>;
+		};
+	};
+
+	sound {
+		compatible = "audio-graph-card";
+		label = "STM32MP1-PHYCORE";
+		routing =
+			"Playback", "MCLK", /* Set a route between "MCLK" and "playback" widgets */
+			"Capture", "MCLK";
+		dais = <&sai2b_port>,
+		       <&sai2a_port>;
+	};
+};
+
+&dcmi {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&dcmi_pins_d>;
+	pinctrl-1 = <&dcmi_sleep_pins_d>;
+};
+
+&i2c1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c1_pins_b>;
+	pinctrl-1 = <&i2c1_sleep_pins_b>;
+	i2c-scl-rising-time-ns = <100>;
+	i2c-scl-falling-time-ns = <7>;
+	status = "okay";
+
+	codec@18 {
+		compatible = "ti,tlv320aic3007";
+		reg = <0x18>;
+		#sound-dai-cells = <0>;
+
+		ai3x-micbias-vg = <2>;
+
+		AVDD-supply = <&v3v3>;
+		IOVDD-supply = <&v3v3>;
+		DRVDD-supply = <&v3v3>;
+		DVDD-supply = <&v1v8_audio>;
+
+		clocks = <&sai2b>;
+
+		port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			tlv320_tx_endpoint: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&sai2b_endpoint>;
+				frame-master;
+				bitclock-master;
+			};
+
+			tlv320_rx_endpoint: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&sai2a_endpoint>;
+				frame-master;
+				bitclock-master;
+			};
+		};
+	};
+
+	touch@44 {
+		compatible = "st,stmpe811";
+		reg = <0x44>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-parent = <&gpioi>;
+		vio-supply = <&v3v3>;
+		vcc-supply = <&v3v3>;
+		st,sample-time = <4>;
+		st,mod-12b = <1>;
+		st,ref-sel = <0>;
+		st,adc-freq = <1>;
+
+		touchscreen {
+			compatible = "st,stmpe-ts";
+			st,ave-ctrl = <1>;
+			st,touch-det-delay = <2>;
+			st,settling = <2>;
+			st,fraction-z = <7>;
+			st,i-drive = <1>;
+		};
+	};
+
+	leds@62 {
+		compatible = "nxp,pca9533";
+		reg = <0x62>;
+
+		led-0 {
+			color = <LED_COLOR_ID_RED>;
+			function = LED_FUNCTION_POWER;
+			type = <PCA9532_TYPE_LED>;
+		};
+
+		led-1 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_POWER;
+			type = <PCA9532_TYPE_LED>;
+		};
+
+		led-2 {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_HEARTBEAT;
+			type = <PCA9532_TYPE_LED>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+};
+
+&ltdc {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&ltdc_pins_f>;
+	pinctrl-1 = <&ltdc_sleep_pins_f>;
+};
+
+&m_can2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&m_can2_pins_a>;
+	pinctrl-1 = <&m_can2_sleep_pins_a>;
+	status = "okay";
+};
+
+&sai2 {
+	clocks = <&rcc SAI2>, <&rcc PLL3_Q>, <&rcc PLL3_R>;
+	clock-names = "pclk", "x8k", "x11k";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&sai2a_pins_d>, <&sai2b_pins_d>;
+	pinctrl-1 = <&sai2a_sleep_pins_d>, <&sai2b_sleep_pins_d>;
+	status = "okay";
+};
+
+&sai2a {
+	dma-names = "rx";
+	clocks = <&rcc SAI2_K>, <&sai2b>;
+	clock-names = "sai_ck", "MCLK";
+	#clock-cells = <0>;
+	st,sync = <&sai2b 2>;
+
+	sai2a_port: port {
+		sai2a_endpoint: endpoint {
+			remote-endpoint = <&tlv320_rx_endpoint>;
+			mclk-fs = <256>;
+			dai-tdm-slot-num = <2>;
+			dai-tdm-slot-width = <16>;
+		};
+	};
+};
+
+&sai2b {
+	dma-names = "tx";
+	#clock-cells = <0>;
+
+	sai2b_port: port {
+		sai2b_endpoint: endpoint {
+			remote-endpoint = <&tlv320_tx_endpoint>;
+			mclk-fs = <256>;
+			dai-tdm-slot-num = <2>;
+			dai-tdm-slot-width = <16>;
+		};
+	};
+};
+
+&sdmmc1 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc1_b4_pins_b>;
+	pinctrl-1 = <&sdmmc1_b4_od_pins_b>;
+	pinctrl-2 = <&sdmmc1_b4_sleep_pins_b>;
+	cd-gpios = <&gpiof 3 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	bus-width = <4>;
+	vmmc-supply = <&v3v3>;
+	st,neg-edge;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi1_pins_a>;
+	pinctrl-1 = <&spi1_sleep_pins_a>;
+	cs-gpios = <&gpioz 3 0>;
+	status = "okay";
+};
+
+&timers5 {
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+	pwm5: pwm {
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&pwm5_pins_c>;
+		pinctrl-1 = <&pwm5_sleep_pins_c>;
+	};
+};
+
+&uart4 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&uart4_pins_f>;
+	pinctrl-1 = <&uart4_sleep_pins_f>;
+	pinctrl-2 = <&uart4_idle_pins_f>;
+	/delete-property/dmas;
+	/delete-property/dma-names;
+	status = "okay";
+};
+
+&usart1 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&usart1_pins_b &usart1_pins_a>;
+	pinctrl-1 = <&usart1_sleep_pins_b &usart1_sleep_pins_a>;
+	pinctrl-2 = <&usart1_idle_pins_b &usart1_idle_pins_a>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usart3 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&usart3_pins_a>;
+	pinctrl-1 = <&usart3_sleep_pins_a>;
+	pinctrl-2 = <&usart3_idle_pins_a>;
+	status = "okay";
+};
+
+&usbh_ehci {
+	status = "okay";
+};
+
+&usbh_ohci {
+	status = "okay";
+};
+
+&usbotg_hs {
+	phys = <&usbphyc_port1 0>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usbphyc {
+	status = "okay";
+};
+
+&usbphyc_port0 {
+	phy-supply = <&vdd_usb>;
+};
+
+&usbphyc_port1 {
+	phy-supply = <&vdd_usb>;
+};
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-phycore-som.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-phycore-som.dtsi
new file mode 100644
index 0000000000000..126446c4198f2
--- /dev/null
+++ b/arch/arm/boot/dts/st/stm32mp15xx-phycore-som.dtsi
@@ -0,0 +1,364 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (C) Phytec GmbH 2019-2020 - All Rights Reserved
+ * Author: Dom VOVARD <dom.vovard@linrt.com>
+ * Copyright (C) 2022-2023 Steffen Trumtrar <kernel@pengutronix.de>
+ * Copyright (C) 2024 PHYTEC Messtechnik GmbH
+ * Author: Christophe Parant <c.parant@phytec.fr>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/mfd/st,stpmic1.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include "stm32mp15-pinctrl.dtsi"
+#include "stm32mp15xxac-pinctrl.dtsi"
+
+/ {
+
+	aliases {
+		ethernet0 = &ethernet0;
+		rtc0 = &i2c4_rtc;
+		rtc1 = &rtc;
+	};
+
+	/*
+	 * Set the minimum memory size here and
+	 * let the bootloader set the real size.
+	 */
+	memory@c0000000 {
+		device_type = "memory";
+		reg = <0xc0000000 0x20000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		mcuram2: mcuram2@10000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x10000000 0x40000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@10040000 {
+			compatible = "shared-dma-pool";
+			reg = <0x10040000 0x1000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@10041000 {
+			compatible = "shared-dma-pool";
+			reg = <0x10041000 0x1000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@10042000 {
+			compatible = "shared-dma-pool";
+			reg = <0x10042000 0x4000>;
+			no-map;
+		};
+
+		mcuram: mcuram@30000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x30000000 0x40000>;
+			no-map;
+		};
+
+		retram: retram@38000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x38000000 0x10000>;
+			no-map;
+		};
+	};
+
+	regulator_vin: regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VIN";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+};
+
+&ethernet0 {
+	pinctrl-0 = <&ethernet0_rgmii_pins_d>;
+	pinctrl-1 = <&ethernet0_rgmii_sleep_pins_d>;
+	pinctrl-names = "default", "sleep";
+	phy-mode = "rgmii-id";
+	max-speed = <1000>;
+	phy-handle = <&phy0>;
+	st,eth-clk-sel;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		phy0: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			interrupt-parent = <&gpiog>;
+			interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+			enet-phy-lane-no-swap;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,min-output-impedance;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+		};
+	};
+};
+
+&fmc {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&fmc_pins_a>;
+	pinctrl-1 = <&fmc_sleep_pins_a>;
+	status = "disabled";
+
+	nand-controller@4,0 {
+		nand0: nand@0 {
+			reg = <0>;
+			nand-on-flash-bbt;
+			nand-ecc-strength = <4>;
+			nand-ecc-step-size = <512>;
+		};
+	};
+};
+
+&i2c4 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c4_pins_a>;
+	pinctrl-1 = <&i2c4_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+
+	pmic@33 {
+		compatible = "st,stpmic1";
+		reg = <0x33>;
+		interrupts-extended = <&gpioa 0 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		regulators {
+			compatible = "st,stpmic1-regulators";
+			buck1-supply = <&regulator_vin>;
+			buck2-supply = <&regulator_vin>;
+			buck3-supply = <&regulator_vin>;
+			buck4-supply = <&regulator_vin>;
+			ldo1-supply = <&v3v3>;
+			ldo2-supply = <&v3v3>;
+			ldo3-supply = <&vdd_ddr>;
+			ldo4-supply = <&regulator_vin>;
+			ldo5-supply = <&v3v3>;
+			ldo6-supply = <&v3v3>;
+			boost-supply = <&regulator_vin>;
+			pwr_sw1-supply = <&bst_out>;
+			pwr_sw2-supply = <&bst_out>;
+
+			vddcore: buck1 {
+				regulator-name = "VDD_CORE";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-initial-mode = <0>;
+			};
+
+			vdd_ddr: buck2 {
+				regulator-name = "VDD_DDR";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-initial-mode = <0>;
+			};
+
+			vdd: buck3 {
+				regulator-name = "VDD";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				st,mask-reset;
+				regulator-initial-mode = <0>;
+			};
+
+			v3v3: buck4 {
+				regulator-name = "VDD_BUCK4";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-initial-mode = <0>;
+			};
+
+			v1v8_audio: ldo1 {
+				regulator-name = "VDD_LDO1";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				interrupts = <IT_CURLIM_LDO1 0>;
+
+			};
+
+			vdd_eth_2v5: ldo2 {
+				regulator-name = "VDD_ETH_2V5";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <2500000>;
+				regulator-always-on;
+				interrupts = <IT_CURLIM_LDO2 0>;
+
+			};
+
+			vtt_ddr: ldo3 {
+				regulator-name = "VTT_DDR";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <750000>;
+				regulator-always-on;
+				regulator-over-current-protection;
+			};
+
+			vdd_usb: ldo4 {
+				regulator-name = "VDD_USB";
+				interrupts = <IT_CURLIM_LDO4 0>;
+			};
+
+			vdda: ldo5 {
+				regulator-name = "VDDA";
+				regulator-min-microvolt = <2900000>;
+				regulator-max-microvolt = <2900000>;
+				interrupts = <IT_CURLIM_LDO5 0>;
+				regulator-boot-on;
+			};
+
+			vdd_eth_1v0: ldo6 {
+				regulator-name = "VDD_ETH_1V0";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				interrupts = <IT_CURLIM_LDO6 0>;
+
+			};
+
+			vref_ddr: vref_ddr {
+				regulator-name = "VDD_REFDDR";
+				regulator-always-on;
+			};
+
+			bst_out: boost {
+				regulator-name = "BST_OUT";
+				interrupts = <IT_OCP_BOOST 0>;
+			};
+
+			vbus_otg: pwr_sw1 {
+				regulator-name = "VBUS_OTG";
+				interrupts = <IT_OCP_OTG 0>;
+				regulator-active-discharge = <1>;
+			};
+
+			vbus_sw: pwr_sw2 {
+				regulator-name = "VBUS_SW";
+				interrupts = <IT_OCP_SWOUT 0>;
+				regulator-active-discharge = <1>;
+			};
+		};
+
+		onkey {
+			compatible = "st,stpmic1-onkey";
+			interrupts = <IT_PONKEY_F 0>,
+				     <IT_PONKEY_R 0>;
+			interrupt-names = "onkey-falling",
+					  "onkey-rising";
+			power-off-time-sec = <10>;
+		};
+
+		watchdog {
+			compatible = "st,stpmic1-wdt";
+		};
+	};
+
+	i2c4_eeprom: eeprom@50 {
+		compatible = "atmel,24c32";
+		reg = <0x50>;
+		status = "disabled";
+	};
+
+	i2c4_rtc: rtc@52 {
+		compatible = "microcrystal,rv3028";
+		reg = <0x52>;
+		status = "disabled";
+	};
+};
+
+&ipcc {
+	status = "okay";
+};
+
+&iwdg2 {
+	timeout-sec = <32>;
+	status = "okay";
+};
+
+&m4_rproc {
+	memory-region = <&retram>, <&mcuram>, <&mcuram2>, <&vdev0vring0>,
+			<&vdev0vring1>, <&vdev0buffer>;
+	mboxes = <&ipcc 0>, <&ipcc 1>, <&ipcc 2>, <&ipcc 3>;
+	mbox-names = "vq0", "vq1", "shutdown", "detach";
+	interrupt-parent = <&exti>;
+	interrupts = <68 1>;
+	status = "okay";
+};
+
+&pwr_regulators {
+	vdd-supply = <&vdd>;
+	vdd_3v3_usbfs-supply = <&vdd_usb>;
+};
+
+&qspi {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&qspi_clk_pins_a
+		     &qspi_bk1_pins_a
+		     &qspi_cs1_pins_a>;
+	pinctrl-1 = <&qspi_clk_sleep_pins_a
+		     &qspi_bk1_sleep_pins_a
+		     &qspi_cs1_sleep_pins_a>;
+	reg = <0x58003000 0x1000>,
+	      <0x70000000 0x1000000>;
+	status = "disabled";
+
+	flash0: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+	};
+};
+
+&rng1 {
+	status = "okay";
+};
+
+&rtc {
+	status = "okay";
+};
+
+&dts {
+	status = "okay";
+};
+
+&sdmmc2 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_e>;
+	pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_e>;
+	pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_e>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	bus-width = <8>;
+	vmmc-supply = <&v3v3>;
+	vqmmc-supply = <&v3v3>;
+	mmc-ddr-3_3v;
+	st,neg-edge;
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/ti/omap/Makefile b/arch/arm/boot/dts/ti/omap/Makefile
index 3f54b515c471c..4c577214f5122 100644
--- a/arch/arm/boot/dts/ti/omap/Makefile
+++ b/arch/arm/boot/dts/ti/omap/Makefile
@@ -80,6 +80,8 @@ dtb-$(CONFIG_ARCH_OMAP4) += \
 	omap4-sdp-es23plus.dtb \
 	omap4-var-dvk-om44.dtb \
 	omap4-var-stk-om44.dtb \
+	omap4-samsung-espresso7.dtb \
+	omap4-samsung-espresso10.dtb \
 	omap4-xyboard-mz609.dtb \
 	omap4-xyboard-mz617.dtb
 dtb-$(CONFIG_SOC_AM33XX) += \
@@ -174,3 +176,11 @@ dtb-$(CONFIG_SOC_TI81XX) += \
 	dm8148-t410.dtb \
 	dm8168-evm.dtb \
 	dra62x-j5eco-evm.dtb
+
+# Enable support for device-tree overlays
+DTC_FLAGS_am335x-bone += -@
+DTC_FLAGS_am335x-boneblack += -@
+DTC_FLAGS_am335x-boneblack-wireless += -@
+DTC_FLAGS_am335x-bonegreen += -@
+DTC_FLAGS_am335x-bonegreen-wireless += -@
+DTC_FLAGS_am335x-bonegreen-eco += -@
diff --git a/arch/arm/boot/dts/ti/omap/dm816x.dtsi b/arch/arm/boot/dts/ti/omap/dm816x.dtsi
index 407d7bc5b13a3..a1e0e904e0f05 100644
--- a/arch/arm/boot/dts/ti/omap/dm816x.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dm816x.dtsi
@@ -94,8 +94,8 @@ dm816x_pinmux: pinmux@800 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				#pinctrl-cells = <1>;
-				pinctrl-single,register-width = <16>;
-				pinctrl-single,function-mask = <0xf>;
+				pinctrl-single,register-width = <32>;
+				pinctrl-single,function-mask = <0x1f>;
 			};
 
 			/* Device Configuration Registers */
diff --git a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
index db6c53bbaf51c..c8d325b0f57b2 100644
--- a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
@@ -327,7 +327,7 @@ ocp2scp@0 {
 				ranges = <0 0 0x8000>;
 				reg = <0x0 0x20>;
 
-				pcie1_phy: pciephy@4000 {
+				pcie1_phy: pcie-phy@4000 {
 					compatible = "ti,phy-pipe3-pcie";
 					reg = <0x4000 0x80>, /* phy_rx */
 					      <0x4400 0x64>; /* phy_tx */
@@ -347,7 +347,7 @@ pcie1_phy: pciephy@4000 {
 					#phy-cells = <0>;
 				};
 
-				pcie2_phy: pciephy@5000 {
+				pcie2_phy: pcie-phy@5000 {
 					compatible = "ti,phy-pipe3-pcie";
 					reg = <0x5000 0x80>, /* phy_rx */
 					      <0x5400 0x64>; /* phy_tx */
diff --git a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
index 4881dd6743930..4c78a0b28fab3 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
@@ -72,13 +72,13 @@ scm_conf: scm_conf@0 {
 					#size-cells = <1>;
 				};
 
-				omap_control_usb2phy: control-phy@300 {
+				omap_control_usb2phy: phy@300 {
 					compatible = "ti,control-phy-usb2";
 					reg = <0x300 0x4>;
 					reg-names = "power";
 				};
 
-				omap_control_usbotg: control-phy@33c {
+				omap_control_usbotg: phy@33c {
 					compatible = "ti,control-phy-otghs";
 					reg = <0x33c 0x4>;
 					reg-names = "otghs_control";
diff --git a/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
new file mode 100644
index 0000000000000..06651072f0007
--- /dev/null
+++ b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
@@ -0,0 +1,744 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/dts-v1/;
+#include "dt-bindings/gpio/gpio.h"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/input/input.h>
+#include "omap443x.dtsi"
+
+/ {
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x40000000>; /* 1 GB */
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		continuous_splash: framebuffer@bef00000{
+			reg = <0xbef00000 (1024 * 600 * 4)>;
+			no-map;
+		};
+	};
+
+	chosen {
+		stdout-path = &uart3;
+		#address-cells = <1>;
+	};
+
+	i2c-gpio5 {
+		compatible = "i2c-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2c5_pins>;
+		sda-gpios = <&gpio4 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio4 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		i2c-gpio,delay-us = <10>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* TODO: SMB136 Charger for 7" variant at 0x4d */
+	};
+
+	i2c-gpio6 {
+		compatible = "i2c-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2c6_pins>;
+		sda-gpios = <&gpio3 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio3 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		i2c-gpio,delay-us = <10>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* TODO: STMPE811 ADC at 0x41 */
+	};
+
+	i2c-gpio7 {
+		compatible = "i2c-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2c7_pins>;
+		sda-gpios = <&gpio2 30 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		i2c-gpio,delay-us = <3>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		fuel-gauge@36 {
+			compatible = "maxim,max17042";
+			reg = <0x36>;
+			pinctrl-0 = <&fuel_alert_irq>;
+			pinctrl-names = "default";
+			interrupt-parent = <&gpio2>;
+			interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
+			maxim,rsns-microohm = <10000>;
+			maxim,over-heat-temp = <500>;
+			maxim,dead-volt = <2500>;
+			maxim,over-volt = <4300>;
+		};
+	};
+
+	reg_espresso_wlan: regulator-espresso-wlan {
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlanen_gpio>;
+		compatible = "regulator-fixed";
+		regulator-name = "espresso_wlan";
+		regulator-max-microvolt = <2000000>;
+		regulator-min-microvolt = <2000000>;
+		gpio = <&gpio4 8 GPIO_ACTIVE_HIGH>; /* GPIO_104 */
+		startup-delay-us = <70000>;
+		regulator-always-on;
+		enable-active-high;
+	};
+
+	wlan_pwrseq: wlan-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&twl 0>;
+		clock-names = "ext_clock";
+	};
+
+	reg_espresso_internal: regulator-espresso-internal {
+		compatible = "regulator-fixed";
+		regulator-name = "eMMC_LDO";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; /* GPIO_63 */
+		startup-delay-us = <100000>;
+		regulator-boot-on;
+		regulator-always-on;
+		enable-active-high;
+	};
+
+	reg_espresso_external: regulator-espresso-external {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc1";
+		regulator-max-microvolt = <2800000>;
+		regulator-min-microvolt = <2800000>;
+		gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>; /* GPIO_34 */
+		enable-active-high;
+	};
+
+	reg_touch_ldo_en: regulator-touch-ldo-en  {
+		compatible = "regulator-fixed";
+		regulator-name = "touch_ldo_en";
+		regulator-max-microvolt = <2800000>;
+		regulator-min-microvolt = <2800000>;
+		gpios = <&gpio2 22 GPIO_ACTIVE_HIGH>; /* GPIO_54 */
+		regulator-always-on;
+		enable-active-high;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_keys>;
+
+		key-power {
+			label = "power";
+
+			gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; /* GPIO_wk3 */
+			linux,code = <KEY_POWER>;
+			wakeup-source;
+		};
+
+		button-volup {
+			linux,code = <KEY_VOLUMEUP>;
+			label = "volume_up"; /* GPIO_wk30 */
+			gpios = <&gpio1 30 GPIO_ACTIVE_LOW>;
+		};
+
+		button-voldown {
+			linux,code = <KEY_VOLUMEDOWN>;
+			label = "volume_down"; /* GPIO_wk8 */
+			gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	reg_lcd: regulator-lcd	{
+		compatible = "regulator-fixed";
+		regulator-name = "lcd_en";
+		gpios = <&gpio5 7 GPIO_ACTIVE_HIGH>; /* GPIO_135 */
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+	};
+
+	pwm10: pwm-10 {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwm10_default>;
+		compatible = "ti,omap-dmtimer-pwm";
+		#pwm-cells = <3>;
+		ti,timers = <&timer10>;
+		ti,clock-source = <0x00>;
+	};
+
+	lvds-encoder {
+		compatible = "doestek,dtc34lm85am", "lvds-encoder";
+		powerdown-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>; /* GPIO_136 */
+		power-supply = <&reg_lcd>;
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				bridge_in: endpoint {
+					remote-endpoint = <&dpi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				bridge_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
+	};
+
+	vibrator {
+		compatible = "gpio-vibrator";
+		enable-gpios = <&gpio2 6 GPIO_ACTIVE_HIGH>; /* GPIO_38 */
+		pinctrl-names = "default";
+		pinctrl-0 = <&vibrator_default>;
+	};
+
+	gp2a_shunt: current-sense-shunt {
+		compatible = "current-sense-shunt";
+		io-channels = <&gpadc 4>;
+		shunt-resistor-micro-ohms = <24000000>; /* 24 ohms */
+		#io-channel-cells = <0>;
+	};
+
+	led-ir {
+		compatible = "gpio-ir-tx";
+		gpios = <&gpio2 27 GPIO_ACTIVE_HIGH>; /* GPIO_59 */
+		pinctrl-names = "default";
+		pinctrl-0 = <&ledir_pins>;
+	};
+};
+
+&omap4_pmx_wkup {
+	gpio_keys: gpio-keys-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x046, PIN_INPUT | MUX_MODE3)
+			/* sim_cd.gpio_wk3 - EXT_WAKEUP */
+			OMAP4_IOPAD(0x056, PIN_INPUT | MUX_MODE3)
+			/* fref_clk3_req.gpio_wk30 - VOL_UP */
+			OMAP4_IOPAD(0x05C, PIN_INPUT | MUX_MODE3)
+			/* fref_clk4_out.gpio_wk8 - VOL_DN */
+		>;
+	};
+};
+
+&omap4_pmx_core {
+	backlight_pins: pinmux-backlight-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0X0D8, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE3)
+			/* usbb1_ulpitll_dat7.gpio_95 - LED_BACKLIGHT_RESET */
+		>;
+	};
+
+	bluetooth_pins: pinmux-bluetooth-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x094, PIN_OUTPUT | MUX_MODE3)
+			/* gpmc_ncs6.gpio_103 - BT_EN */
+			OMAP4_IOPAD(0x0be, PIN_OUTPUT | MUX_MODE3)
+			/* cam_strobe.gpio_82 - BT_nRST */
+			OMAP4_IOPAD(0x0c0, PIN_INPUT | MUX_MODE3)
+			/* cam_globalreset.gpio_83 - BT_HOST_WAKE */
+			OMAP4_IOPAD(0x0d4, PIN_OUTPUT | MUX_MODE3)
+			/* usbb1_ulpitll_dat5.gpio_93 - BT_WAKE */
+		>;
+	};
+
+	dss_dpi_pins: pinmux-dss-dpi-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x162, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data23 */
+			OMAP4_IOPAD(0x164, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data22 */
+			OMAP4_IOPAD(0x166, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data21 */
+			OMAP4_IOPAD(0x168, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data20 */
+			OMAP4_IOPAD(0x16a, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data19 */
+			OMAP4_IOPAD(0x16c, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data18 */
+			OMAP4_IOPAD(0x16e, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data15 */
+			OMAP4_IOPAD(0x170, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data14 */
+			OMAP4_IOPAD(0x172, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data13 */
+			OMAP4_IOPAD(0x174, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data12 */
+			OMAP4_IOPAD(0x176, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data11 */
+			OMAP4_IOPAD(0x1b4, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data10 */
+			OMAP4_IOPAD(0x1b6, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data9 */
+			OMAP4_IOPAD(0x1b8, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data16 */
+			OMAP4_IOPAD(0x1ba, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data17 */
+			OMAP4_IOPAD(0x1bc, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_hsync */
+			OMAP4_IOPAD(0x1be, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_pclk */
+			OMAP4_IOPAD(0x1c0, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_vsync */
+			OMAP4_IOPAD(0x1c2, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_de */
+			OMAP4_IOPAD(0x1c4, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data8 */
+			OMAP4_IOPAD(0x1c6, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data7 */
+			OMAP4_IOPAD(0x1c8, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data6 */
+			OMAP4_IOPAD(0x1ca, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data5 */
+			OMAP4_IOPAD(0x1cc, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data4 */
+			OMAP4_IOPAD(0x1ce, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data3 */
+
+			OMAP4_IOPAD(0x1d0, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data2 */
+			OMAP4_IOPAD(0x1d2, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data1 */
+			OMAP4_IOPAD(0x1d4, PIN_OFF_OUTPUT_LOW | MUX_MODE5)
+			/* dispc2_data0 */
+		>;
+	};
+
+	fuel_alert_irq: pinmux-fuel-alert-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x068, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* gpmc_a20.gpio_44 */
+		>;
+	};
+
+	i2c1_pins: pinmux-i2c1-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x122, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c1_scl */
+			OMAP4_IOPAD(0x124, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c1_sda */
+		>;
+	};
+
+	i2c2_pins: pinmux-i2c2-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x126, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c2_scl */
+			OMAP4_IOPAD(0x128, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c2_sda */
+		>;
+	};
+
+	i2c3_pins: pinmux-i2c3-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x12a, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c3_scl */
+			OMAP4_IOPAD(0x12c, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c3_sda */
+		>;
+	};
+
+	i2c4_pins: pinmux-i2c4-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x12e, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c4_scl */
+			OMAP4_IOPAD(0x130, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* i2c4_sda */
+		>;
+	};
+
+	i2c5_pins: pinmux-i2c5-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x0de, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* usbc1_icusb_dp.gpio_98 */
+			OMAP4_IOPAD(0x0e0, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* usbc1_icusb_dm.gpio_99 */
+		>;
+	};
+
+	i2c6_pins: pinmux-i2c6-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x09c, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* hdmi_ddc_scl.gpio_65 */
+			OMAP4_IOPAD(0x09e, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* hdmi_ddc_sda.gpio_66 */
+		>;
+	};
+
+	i2c7_pins: pinmux-i2c7-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x08a, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* gpmc_wait0.gpio_61 */
+			OMAP4_IOPAD(0x08c, PIN_INPUT_PULLUP | MUX_MODE3)
+			/* gpmc_wait1.gpio_62 */
+		>;
+	};
+
+	ledir_pins: pimux-ledir-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x086, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			/* gpmc_nbe0_cle.gpio_59 */
+			OMAP4_IOPAD(0x156, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			/* mcspi4_simo.gpio_152 */
+		>;
+	};
+
+	lvds_pins: pinmux-lvds-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0X136, PIN_OUTPUT | MUX_MODE3)
+			/* mcspi1_simo.gpio_136 - LVDS_nSHDN */
+		>;
+	};
+
+	mmc1_pins: pinmux-mmc1-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x0e2, PIN_INPUT_PULLDOWN | MUX_MODE0)
+			/* sdmmc1_clk */
+			OMAP4_IOPAD(0x0e4, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmcc1_cmd */
+			OMAP4_IOPAD(0x0e6, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmcc1_dat0 */
+			OMAP4_IOPAD(0x0e8, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc1_dat1 */
+			OMAP4_IOPAD(0x0ea, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc1_dat2 */
+			OMAP4_IOPAD(0x0ec, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc1_dat3 */
+		>;
+	};
+
+	mmc2_pins: pinmux-mmc2-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x040, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat0 */
+			OMAP4_IOPAD(0x042, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat1 */
+			OMAP4_IOPAD(0x044, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat2 */
+			OMAP4_IOPAD(0x046, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat3 */
+			OMAP4_IOPAD(0x048, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat4 */
+			OMAP4_IOPAD(0x04a, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat5 */
+			OMAP4_IOPAD(0x04c, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat6 */
+			OMAP4_IOPAD(0x04e, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_dat7 */
+			OMAP4_IOPAD(0x082, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_clk */
+			OMAP4_IOPAD(0x084, PIN_INPUT_PULLUP | MUX_MODE1)
+			/* sdmmc2_cmd */
+		>;
+	};
+
+	mmc5_pins: pinmux-mmc5-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x148, PIN_INPUT_PULLDOWN | MUX_MODE0)
+			/* sdmmc5_clk.sdmmc5_clk */
+			OMAP4_IOPAD(0x14a, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc5_cmd.sdmmc5_cmd */
+			OMAP4_IOPAD(0x14c, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc5_dat0.sdmmc5_dat0 */
+			OMAP4_IOPAD(0x14e, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc5_dat1.sdmmc5_dat1 */
+			OMAP4_IOPAD(0x150, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc5_dat2.sdmmc5_dat2 */
+			OMAP4_IOPAD(0x152, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* sdmmc5_dat3.sdmmc5_dat3 */
+		>;
+	};
+
+	pwm10_default: pinmux-pwm10-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0X0D6, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE1)
+			/* usbb1_ulpitll_dat6.dmtimer10_pwm_evt - LED_BACKLIGHT_PWM */
+		>;
+	};
+
+	touch_pins: pinmux-touch-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x06c, PIN_INPUT | MUX_MODE3)
+			/* gpmc_a22.gpio_46 - TSP_INT */
+		>;
+	};
+
+	uart2_pins: pinmux-uart2-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x118, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* uart2_cts.uart2_cts */
+			OMAP4_IOPAD(0x11a, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* uart2_rts.uart2_rts */
+			OMAP4_IOPAD(0x11c, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* uart2_rx.uart2_rx */
+			OMAP4_IOPAD(0x11e, PIN_INPUT_PULLUP | MUX_MODE0)
+			/* uart2_tx.uart2_tx */
+		>;
+	};
+
+	uart3_pins: pinmux-uart3-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x144, PIN_INPUT | MUX_MODE0)
+			/* uart3_rx_irrx */
+			OMAP4_IOPAD(0x146, PIN_OUTPUT | MUX_MODE0)
+			/* uart3_tx_irtx */
+		>;
+	};
+
+	vibrator_default: pinmux-vibrator-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x05c, PIN_INPUT_PULLDOWN | MUX_MODE3)
+			/* gpmc_ad14.gpio_38 - MOTOR_EN */
+		>;
+	};
+
+	wlanen_gpio: pinmux-wlanen-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x096, PIN_OUTPUT | MUX_MODE3)
+			/* gpmc_ncs7.gpio_104 */
+		>;
+	};
+
+	wlan_host_wake: pinmux-wlan-host-wake-pins {
+		pinctrl-single,pins = <
+			OMAP4_IOPAD(0x0bc, PIN_INPUT | MUX_MODE3)
+			/* cam_shutter.gpio_81 - WLAN_HOST_WAKE */
+		>;
+	};
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_pins>;
+
+	interrupts-extended = <&wakeupgen GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH
+				   &omap4_pmx_core OMAP4_UART3_RX>;
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+
+	clock-frequency = <400000>;
+
+	twl: pmic@48 {
+		reg = <0x48>;
+		#clock-cells = <1>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <
+			&twl6030_pins
+			&twl6030_wkup_pins
+		>;
+
+		/* SPI = 0, IRQ# = 7, 4 = active high level-sensitive */
+		interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; /* IRQ_SYS_1N cascaded to gic */
+		interrupt-parent = <&gic>;
+		system-power-controller;
+	};
+};
+
+#include "twl6032.dtsi"
+#include "twl6030_omap4.dtsi"
+
+&ldo1 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-always-on;
+	regulator-state-mem {
+		regulator-off-in-suspend;
+	};
+};
+
+&ldo3 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-always-on;
+	regulator-state-mem {
+		regulator-off-in-suspend;
+	};
+};
+
+&ldo4 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-always-on;
+};
+
+&ldo5 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&ldo6 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+	regulator-state-mem {
+		regulator-off-in-suspend;
+	};
+};
+
+&smps4 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-always-on;
+};
+
+&ldousb {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-always-on;
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins>;
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_pins>;
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4_pins>;
+
+	accelerometer@18 {
+		compatible = "bosch,bma254";
+		reg = <0x18>;
+		vdd-supply = <&ldo4>;
+		vddio-supply = <&ldo5>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <25 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>,
+			<26 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>;
+	};
+
+	magnetometer@2e {
+		compatible = "yamaha,yas530";
+		reg = <0x2e>;
+		vdd-supply = <&ldo4>;
+		iovdd-supply = <&ldo5>;
+		reset-gpios = <&gpio5 29 GPIO_ACTIVE_LOW>;
+		interrupts = <&gpio6 10 IRQ_TYPE_EDGE_RISING>;
+	};
+
+	light-sensor@44 {
+		compatible = "sharp,gp2ap002a00f";
+		reg = <0x44>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&ldo4>;
+		vio-supply = <&ldo4>;
+		io-channels = <&gp2a_shunt>;
+		io-channel-names = "alsout";
+		sharp,proximity-far-hysteresis = /bits/ 8 <0x40>;
+		sharp,proximity-close-hysteresis = /bits/ 8 <0x20>;
+	};
+};
+
+&dss {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_dpi_pins>;
+
+	port {
+		dpi_out: endpoint {
+			remote-endpoint = <&bridge_in>;
+			data-lines = <24>;
+		};
+	};
+};
+
+&twl_usb_comparator {
+	usb-supply = <&ldousb>;
+};
+
+&usb_otg_hs {
+	interface-type = <1>;
+	mode = <3>;
+	power = <50>;
+};
+
+&mmc1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
+
+	vmmc-supply = <&reg_espresso_external>;
+	bus-width = <4>;
+};
+
+&mmc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_pins>;
+
+	vmmc-supply = <&reg_espresso_internal>;
+	ti,non-removable;
+	bus-width = <8>;
+};
+
+&mmc3 {
+	status = "disabled";
+};
+
+&mmc4 {
+	status = "disabled";
+};
+
+&mmc5 {
+	status = "okay";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	non-removable;
+	bus-width = <4>;
+	vmmc-supply = <&reg_espresso_wlan>;
+	mmc-pwrseq = <&wlan_pwrseq>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc5_pins>;
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4330-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+
+		interrupt-parent = <&gpio3>;
+		interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlan_host_wake>;
+	};
+};
+
+&uart2 {
+	interrupts-extended = <&wakeupgen GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH
+				&omap4_pmx_core OMAP4_UART2_RX>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins>;
+	bluetooth {
+		compatible = "brcm,bcm4330-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&bluetooth_pins>;
+		shutdown-gpios = <&gpio4 7 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio3 18 GPIO_ACTIVE_LOW>;
+		device-wakeup-gpios = <&gpio3 29 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <19 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-names = "host-wakeup";
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso10.dts b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso10.dts
new file mode 100644
index 0000000000000..558dc3c029e9f
--- /dev/null
+++ b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso10.dts
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/dts-v1/;
+
+#include "omap4-samsung-espresso-common.dtsi"
+#include <dt-bindings/power/summit,smb347-charger.h>
+/ {
+	model = "Samsung Galaxy Tab 2 (10 inch)";
+	compatible = "samsung,espresso10", "ti,omap4430", "ti,omap4";
+
+	i2c-gpio5 {
+		smb347: charger@6 {
+			compatible = "summit,smb347";
+			reg = <0x6>; // 0x0C >> 1
+			interrupt-parent = <&gpio2>;
+			interrupts = <0 IRQ_TYPE_EDGE_BOTH>;
+
+			summit,enable-usb-charging;
+			summit,enable-charge-control = <SMB3XX_CHG_ENABLE_SW>;
+			summit,chip-temperature-threshold-celsius = <120>;
+			summit,usb-current-limit-microamp = <1800000>;
+		};
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		pwms = <&pwm10 0 1600 0>;
+		power-supply = <&reg_lcd>;
+		enable-gpios = <&gpio3 31 GPIO_ACTIVE_HIGH>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+	};
+
+	panel {
+		compatible = "samsung,ltn101al03", "panel-lvds";
+		power-supply = <&reg_lcd>;
+		width-mm = <223>;
+		height-mm = <125>;
+		data-mapping = "vesa-24";
+		backlight = <&backlight>;
+
+		panel-timing {
+			clock-frequency = <69818000>;
+
+			hback-porch = <64>;
+			hactive = <1280>;
+			hfront-porch = <16>;
+			hsync-len = <48>;
+
+			vback-porch = <11>;
+			vactive = <800>;
+			vfront-porch = <16>;
+			vsync-len = <3>;
+
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <1>;
+		};
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&bridge_out>;
+			};
+		};
+	};
+};
+
+&i2c3 {
+	touchscreen@20 {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x20>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		interrupt-parent = <&gpio2>;
+		interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&touch_pins>;
+
+		//avdd-supply = <&reg_touch_ldo_en>;
+		vdd-supply = <&ldo6>;
+
+		syna,reset-delay-ms = <200>;
+		syna,startup-delay-ms = <200>;
+
+		rmi4-f01@1 {
+			reg = <0x01>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f11@11 {
+			reg = <0x11>;
+			touchscreen-size-x = <1280>;
+			touchscreen-size-y = <800>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso7.dts b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso7.dts
new file mode 100644
index 0000000000000..cae37ff066c30
--- /dev/null
+++ b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso7.dts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/dts-v1/;
+
+#include "omap4-samsung-espresso-common.dtsi"
+
+/ {
+	model = "Samsung Galaxy Tab 2 (7 inch)";
+	compatible = "samsung,espresso7", "ti,omap4430", "ti,omap4";
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		pwms = <&pwm10 0 1200 0>;
+		power-supply = <&reg_lcd>;
+		enable-gpios = <&gpio3 31 GPIO_ACTIVE_HIGH>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <2>;
+	};
+
+	panel {
+		compatible = "samsung,ltn070nl01", "panel-lvds";
+		power-supply = <&reg_lcd>;
+		width-mm = <154>;
+		height-mm = <90>;
+		data-mapping = "vesa-24";
+		backlight = <&backlight>;
+
+		panel-timing {
+			clock-frequency = <47255554>;
+
+			hback-porch = <210>;
+			hactive = <1024>;
+			hfront-porch = <186>;
+			hsync-len = <50>;
+
+			vback-porch = <11>;
+			vactive = <600>;
+			vfront-porch = <24>;
+			vsync-len = <10>;
+
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <1>;
+			syncclk-active = <0>;
+		};
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&bridge_out>;
+			};
+		};
+	};
+};
+
+&i2c3 {
+	touchscreen@48 {
+		compatible = "melfas,mms136";
+		reg = <0x48>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <1024>;
+		touchscreen-size-y = <600>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&touch_pins>;
+		avdd-supply = <&reg_touch_ldo_en>;
+		vdd-supply = <&ldo6>;
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
index 487259132ebfd..915870eb5c990 100644
--- a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
@@ -472,7 +472,7 @@ usb2_phy: usb2phy@4000 {
 				#phy-cells = <0>;
 			};
 
-			usb3_phy: usb3phy@4400 {
+			usb3_phy: usb3-phy@4400 {
 				compatible = "ti,omap-usb3";
 				reg = <0x4400 0x80>,
 				<0x4800 0x64>,
diff --git a/arch/arm/boot/dts/ti/omap/twl6032.dtsi b/arch/arm/boot/dts/ti/omap/twl6032.dtsi
new file mode 100644
index 0000000000000..d599a2ca62a4f
--- /dev/null
+++ b/arch/arm/boot/dts/ti/omap/twl6032.dtsi
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Integrated Power Management Chip
+ * https://www.ti.com/lit/ds/symlink/twl6032.pdf
+ */
+
+&twl {
+	compatible = "ti,twl6032";
+	interrupt-controller;
+	#interrupt-cells = <1>;
+
+	rtc {
+		compatible = "ti,twl4030-rtc";
+		interrupts = <11>;
+	};
+
+	vio: regulator-vio {
+		compatible = "ti,twl6032-vio";
+	};
+
+	ldo1: regulator-ldo1 {
+		compatible = "ti,twl6032-ldo1";
+	};
+
+	ldo2: regulator-ldo2 {
+		compatible = "ti,twl6032-ldo2";
+	};
+
+	ldo3: regulator-ldo3 {
+		compatible = "ti,twl6032-ldo3";
+	};
+
+	ldo4: regulator-ldo4 {
+		compatible = "ti,twl6032-ldo4";
+	};
+
+	ldo5: regulator-ldo5 {
+		compatible = "ti,twl6032-ldo5";
+	};
+
+	ldo6: regulator-ldo6 {
+		compatible = "ti,twl6032-ldo6";
+	};
+
+	ldoln: regulator-ldoln {
+		compatible = "ti,twl6032-ldoln";
+	};
+
+	ldousb: regulator-ldousb {
+		compatible = "ti,twl6032-ldousb";
+	};
+
+	smps4: regulator-smps4 {
+		compatible = "ti,twl6032-smps4";
+	};
+
+	gpadc: gpadc {
+		compatible = "ti,twl6032-gpadc";
+		interrupts = <3>;
+		#io-channel-cells = <1>;
+	};
+
+	twl_usb_comparator: usb-comparator {
+		compatible = "ti,twl6030-usb";
+		interrupts = <4>, <10>;
+	};
+
+	twl_pwm: pwm {
+		compatible = "ti,twl6030-pwm";
+		#pwm-cells = <2>;
+	};
+
+	twl_pwmled: pwmled {
+		compatible = "ti,twl6030-pwmled";
+		#pwm-cells = <2>;
+	};
+};
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index f75d75cf91c88..5eb211d1f5639 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -155,6 +155,7 @@ CONFIG_QRTR_SMD=m
 CONFIG_CAN=y
 CONFIG_BT=m
 CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_LL=y
 CONFIG_BT_HCIUART_BCM=y
 CONFIG_BT_MRVL=m
 CONFIG_BT_MRVL_SDIO=m
@@ -279,7 +280,6 @@ CONFIG_TI_CPSW_SWITCHDEV=y
 CONFIG_TI_CPTS=y
 CONFIG_TI_KEYSTONE_NETCP=y
 CONFIG_TI_KEYSTONE_NETCP_ETHSS=y
-CONFIG_TI_PRUSS=m
 CONFIG_TI_PRUETH=m
 CONFIG_XILINX_EMACLITE=y
 CONFIG_SFP=m
@@ -310,6 +310,11 @@ CONFIG_MWIFIEX=m
 CONFIG_MWIFIEX_SDIO=m
 CONFIG_RT2X00=m
 CONFIG_RT2800USB=m
+CONFIG_WL1251=m
+CONFIG_WL1251_SDIO=m
+CONFIG_WL12XX=m
+CONFIG_WL18XX=m
+CONFIG_WLCORE_SDIO=m
 CONFIG_INPUT_JOYDEV=y
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_QT1070=m
@@ -318,8 +323,10 @@ CONFIG_KEYBOARD_TEGRA=y
 CONFIG_KEYBOARD_PXA27x=m
 CONFIG_KEYBOARD_SAMSUNG=m
 CONFIG_KEYBOARD_ST_KEYSCAN=y
+CONFIG_KEYBOARD_OMAP4=m
 CONFIG_KEYBOARD_SPEAR=y
 CONFIG_KEYBOARD_TM2_TOUCHKEY=m
+CONFIG_KEYBOARD_TWL4030=m
 CONFIG_KEYBOARD_CROS_EC=m
 CONFIG_MOUSE_PS2_ELANTECH=y
 CONFIG_MOUSE_CYAPA=m
@@ -342,6 +349,8 @@ CONFIG_INPUT_GPIO_DECODER=m
 CONFIG_INPUT_CPCAP_PWRBUTTON=m
 CONFIG_INPUT_TPS65219_PWRBUTTON=m
 CONFIG_INPUT_AXP20X_PEK=m
+CONFIG_INPUT_TWL4030_PWRBUTTON=m
+CONFIG_INPUT_TWL4030_VIBRA=m
 CONFIG_INPUT_DA9063_ONKEY=m
 CONFIG_INPUT_ADXL34X=m
 CONFIG_INPUT_STPMIC1_ONKEY=y
@@ -521,6 +530,8 @@ CONFIG_AXP20X_POWER=m
 CONFIG_BATTERY_MAX17040=m
 CONFIG_BATTERY_MAX17042=m
 CONFIG_CHARGER_CPCAP=m
+CONFIG_CHARGER_TWL4030=m
+CONFIG_CHARGER_TWL6030=m
 CONFIG_CHARGER_GPIO=m
 CONFIG_CHARGER_MAX14577=m
 CONFIG_CHARGER_MAX77693=m
@@ -549,6 +560,8 @@ CONFIG_ARMADA_THERMAL=y
 CONFIG_BCM2711_THERMAL=m
 CONFIG_BCM2835_THERMAL=m
 CONFIG_BRCMSTB_THERMAL=m
+CONFIG_OMAP4_THERMAL=y
+CONFIG_OMAP5_THERMAL=y
 CONFIG_ST_THERMAL_MEMMAP=y
 CONFIG_RCAR_THERMAL=y
 CONFIG_TEGRA_SOCTHERM=m
@@ -563,10 +576,12 @@ CONFIG_AT91SAM9X_WATCHDOG=y
 CONFIG_SAMA5D4_WATCHDOG=y
 CONFIG_S3C2410_WATCHDOG=m
 CONFIG_DW_WATCHDOG=y
+CONFIG_OMAP_WATCHDOG=y
 CONFIG_DAVINCI_WATCHDOG=m
 CONFIG_ORION_WATCHDOG=y
 CONFIG_RN5T618_WATCHDOG=y
 CONFIG_SUNXI_WATCHDOG=y
+CONFIG_TWL4030_WATCHDOG=m
 CONFIG_IMX2_WDT=y
 CONFIG_ST_LPC_WATCHDOG=y
 CONFIG_TEGRA_WATCHDOG=m
@@ -735,6 +750,7 @@ CONFIG_DRM_RCAR_DU=m
 CONFIG_DRM_SUN4I=m
 CONFIG_DRM_OMAP=m
 CONFIG_OMAP5_DSS_HDMI=y
+CONFIG_DRM_TILCDC=m
 CONFIG_DRM_MSM=m
 CONFIG_DRM_FSL_DCU=m
 CONFIG_DRM_TEGRA=y
@@ -751,6 +767,7 @@ CONFIG_DRM_PANEL_EDP=y
 CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_DISPLAY_CONNECTOR=m
 CONFIG_DRM_I2C_NXP_TDA998X=m
+CONFIG_DRM_ITE_IT66121=m
 CONFIG_DRM_LVDS_CODEC=m
 CONFIG_DRM_NXP_PTN3460=m
 CONFIG_DRM_PARADE_PS8622=m
@@ -845,6 +862,9 @@ CONFIG_SND_SOC_TEGRA_TRIMSLICE=m
 CONFIG_SND_SOC_TEGRA_ALC5632=m
 CONFIG_SND_SOC_TEGRA_MAX98090=m
 CONFIG_SND_SOC_DAVINCI_MCASP=m
+CONFIG_SND_SOC_OMAP_DMIC=m
+CONFIG_SND_SOC_OMAP_MCPDM=m
+CONFIG_SND_SOC_OMAP3_TWL4030=m
 CONFIG_SND_SOC_AC97_CODEC=m
 CONFIG_SND_SOC_AK4642=m
 CONFIG_SND_SOC_CPCAP=m
@@ -987,6 +1007,7 @@ CONFIG_LEDS_PCA9532=m
 CONFIG_LEDS_PCA9532_GPIO=y
 CONFIG_LEDS_GPIO=y
 CONFIG_LEDS_PWM=y
+CONFIG_LEDS_TCA6507=m
 CONFIG_LEDS_MAX8997=m
 CONFIG_LEDS_ACER_A500=m
 CONFIG_LEDS_TRIGGERS=y
@@ -1088,6 +1109,7 @@ CONFIG_COMMON_CLK_RK808=m
 CONFIG_COMMON_CLK_SCMI=y
 CONFIG_COMMON_CLK_LAN966X=y
 CONFIG_COMMON_CLK_S2MPS11=m
+CONFIG_CLK_TWL=m
 CONFIG_CLK_RASPBERRYPI=y
 CONFIG_COMMON_CLK_QCOM=y
 CONFIG_QCOM_A53PLL=y
@@ -1183,6 +1205,8 @@ CONFIG_STM32_ADC_CORE=m
 CONFIG_STM32_ADC=m
 CONFIG_STM32_DFSDM_ADC=m
 CONFIG_STMPE_ADC=m
+CONFIG_TWL4030_MADC=m
+CONFIG_TWL6030_GPADC=m
 CONFIG_VF610_ADC=m
 CONFIG_XILINX_XADC=y
 CONFIG_IIO_CROS_EC_SENSORS_CORE=m
@@ -1205,6 +1229,7 @@ CONFIG_PWM_BCM2835=y
 CONFIG_PWM_BRCMSTB=m
 CONFIG_PWM_FSL_FTM=m
 CONFIG_PWM_MESON=m
+CONFIG_PWM_OMAP_DMTIMER=m
 CONFIG_PWM_RENESAS_RCAR=m
 CONFIG_PWM_RENESAS_TPU=y
 CONFIG_PWM_ROCKCHIP=m
@@ -1215,6 +1240,8 @@ CONFIG_PWM_STM32_LP=m
 CONFIG_PWM_SUN4I=y
 CONFIG_PWM_TEGRA=y
 CONFIG_PWM_TIECAP=m
+CONFIG_PWM_TWL=m
+CONFIG_PWM_TWL_LED=m
 CONFIG_PWM_VT8500=y
 CONFIG_KEYSTONE_IRQ=y
 CONFIG_RESET_MCHP_SPARX5=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index 0464f6552169b..51a35bac7d88a 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -486,6 +486,7 @@ CONFIG_VIDEO_OMAP3=m
 CONFIG_VIDEO_MT9P031=m
 CONFIG_VIDEO_TVP5150=m
 CONFIG_DRM=m
+CONFIG_DRM_ITE_IT66121=m
 CONFIG_DRM_OMAP=m
 CONFIG_OMAP5_DSS_HDMI=y
 CONFIG_OMAP2_DSS_SDI=y
diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig
index e2ad9a05566f3..e52f671ccec4e 100644
--- a/arch/arm/configs/sama7_defconfig
+++ b/arch/arm/configs/sama7_defconfig
@@ -108,6 +108,8 @@ CONFIG_CAN_M_CAN_PLATFORM=y
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_GPIO=y
 # CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_ATMEL_MXT=y
 CONFIG_LEGACY_PTY_COUNT=4
 CONFIG_SERIAL_ATMEL=y
 CONFIG_SERIAL_ATMEL_CONSOLE=y
@@ -133,6 +135,7 @@ CONFIG_GENERIC_ADC_THERMAL=y
 CONFIG_WATCHDOG=y
 CONFIG_SAMA5D4_WATCHDOG=y
 CONFIG_MFD_ATMEL_FLEXCOM=y
+CONFIG_MFD_ATMEL_HLCDC=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
 CONFIG_REGULATOR_MCP16502=y
@@ -147,6 +150,12 @@ CONFIG_VIDEO_MICROCHIP_CSI2DC=y
 CONFIG_VIDEO_IMX219=m
 CONFIG_VIDEO_IMX274=m
 CONFIG_VIDEO_OV5647=m
+CONFIG_DRM=y
+CONFIG_DRM_ATMEL_HLCDC=y
+CONFIG_DRM_MICROCHIP_LVDS_SERIALIZER=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_BACKLIGHT_CLASS_DEVICE=y
+CONFIG_BACKLIGHT_PWM=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
@@ -197,6 +206,7 @@ CONFIG_AT91_SAMA5D2_ADC=y
 CONFIG_PAC1934=m
 CONFIG_PWM=y
 CONFIG_PWM_ATMEL=y
+CONFIG_PWM_ATMEL_HLCDC_PWM=y
 CONFIG_MCHP_EIC=y
 CONFIG_RESET_CONTROLLER=y
 CONFIG_NVMEM_MICROCHIP_OTPC=y
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index ef487eab17cc7..6f9696e9fe17d 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -140,17 +140,17 @@ CONFIG_VIDEO_ADV7604_CEC=y
 CONFIG_VIDEO_ML86V7667=y
 CONFIG_DRM=y
 CONFIG_DRM_FBDEV_EMULATION=y
-CONFIG_DRM_RCAR_DU=y
-# CONFIG_DRM_RCAR_USE_MIPI_DSI is not set
-CONFIG_DRM_SHMOBILE=y
-CONFIG_DRM_PANEL_EDP=y
-CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_DISPLAY_CONNECTOR=y
 CONFIG_DRM_LVDS_CODEC=y
 CONFIG_DRM_SII902X=y
 CONFIG_DRM_SIMPLE_BRIDGE=y
 CONFIG_DRM_I2C_ADV7511=y
 CONFIG_DRM_I2C_ADV7511_AUDIO=y
+CONFIG_DRM_PANEL_EDP=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_RCAR_DU=y
+# CONFIG_DRM_RCAR_USE_MIPI_DSI is not set
+CONFIG_DRM_SHMOBILE=y
 CONFIG_FB_DEVICE=y
 CONFIG_BACKLIGHT_PWM=y
 CONFIG_BACKLIGHT_AS3711=y
@@ -217,7 +217,6 @@ CONFIG_TMPFS=y
 CONFIG_NFS_FS=y
 CONFIG_NFS_V3_ACL=y
 CONFIG_NFS_V4=y
-CONFIG_NFS_V4_1=y
 CONFIG_ROOT_NFS=y
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index ef11b721230e2..fa4c1225dde57 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -184,7 +184,7 @@ extern int pfn_valid(unsigned long);
 
 #include <asm/memory.h>
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
 
 #include <asm-generic/getorder.h>
 #include <asm-generic/memory_model.h>
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 6fa9acd6a7f5d..982795cf45637 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -10,15 +10,6 @@
 #include <linux/const.h>
 #include <asm/proc-fns.h>
 
-#ifndef __ASSEMBLY__
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-#endif
-
 #include <asm-generic/pgtable-nopud.h>
 
 #ifndef CONFIG_MMU
diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h
index 1e9f81639c88c..26da5d8621cc9 100644
--- a/arch/arm/include/asm/vdso/gettimeofday.h
+++ b/arch/arm/include/asm/vdso/gettimeofday.h
@@ -11,6 +11,8 @@
 #include <asm/errno.h>
 #include <asm/unistd.h>
 #include <asm/vdso/cp15.h>
+#include <vdso/clocksource.h>
+#include <vdso/time32.h>
 #include <uapi/linux/time.h>
 
 #define VDSO_HAS_CLOCK_GETRES		1
diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
deleted file mode 100644
index 934b549905f5c..0000000000000
--- a/arch/arm/include/asm/xor.h
+++ /dev/null
@@ -1,225 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- *  arch/arm/include/asm/xor.h
- *
- *  Copyright (C) 2001 Russell King
- */
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-
-#define __XOR(a1, a2) a1 ^= a2
-
-#define GET_BLOCK_2(dst) \
-	__asm__("ldmia	%0, {%1, %2}" \
-		: "=r" (dst), "=r" (a1), "=r" (a2) \
-		: "0" (dst))
-
-#define GET_BLOCK_4(dst) \
-	__asm__("ldmia	%0, {%1, %2, %3, %4}" \
-		: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
-		: "0" (dst))
-
-#define XOR_BLOCK_2(src) \
-	__asm__("ldmia	%0!, {%1, %2}" \
-		: "=r" (src), "=r" (b1), "=r" (b2) \
-		: "0" (src)); \
-	__XOR(a1, b1); __XOR(a2, b2);
-
-#define XOR_BLOCK_4(src) \
-	__asm__("ldmia	%0!, {%1, %2, %3, %4}" \
-		: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
-		: "0" (src)); \
-	__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
-
-#define PUT_BLOCK_2(dst) \
-	__asm__ __volatile__("stmia	%0!, {%2, %3}" \
-		: "=r" (dst) \
-		: "0" (dst), "r" (a1), "r" (a2))
-
-#define PUT_BLOCK_4(dst) \
-	__asm__ __volatile__("stmia	%0!, {%2, %3, %4, %5}" \
-		: "=r" (dst) \
-		: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
-
-static void
-xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2)
-{
-	unsigned int lines = bytes / sizeof(unsigned long) / 4;
-	register unsigned int a1 __asm__("r4");
-	register unsigned int a2 __asm__("r5");
-	register unsigned int a3 __asm__("r6");
-	register unsigned int a4 __asm__("r10");
-	register unsigned int b1 __asm__("r8");
-	register unsigned int b2 __asm__("r9");
-	register unsigned int b3 __asm__("ip");
-	register unsigned int b4 __asm__("lr");
-
-	do {
-		GET_BLOCK_4(p1);
-		XOR_BLOCK_4(p2);
-		PUT_BLOCK_4(p1);
-	} while (--lines);
-}
-
-static void
-xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3)
-{
-	unsigned int lines = bytes / sizeof(unsigned long) / 4;
-	register unsigned int a1 __asm__("r4");
-	register unsigned int a2 __asm__("r5");
-	register unsigned int a3 __asm__("r6");
-	register unsigned int a4 __asm__("r10");
-	register unsigned int b1 __asm__("r8");
-	register unsigned int b2 __asm__("r9");
-	register unsigned int b3 __asm__("ip");
-	register unsigned int b4 __asm__("lr");
-
-	do {
-		GET_BLOCK_4(p1);
-		XOR_BLOCK_4(p2);
-		XOR_BLOCK_4(p3);
-		PUT_BLOCK_4(p1);
-	} while (--lines);
-}
-
-static void
-xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4)
-{
-	unsigned int lines = bytes / sizeof(unsigned long) / 2;
-	register unsigned int a1 __asm__("r8");
-	register unsigned int a2 __asm__("r9");
-	register unsigned int b1 __asm__("ip");
-	register unsigned int b2 __asm__("lr");
-
-	do {
-		GET_BLOCK_2(p1);
-		XOR_BLOCK_2(p2);
-		XOR_BLOCK_2(p3);
-		XOR_BLOCK_2(p4);
-		PUT_BLOCK_2(p1);
-	} while (--lines);
-}
-
-static void
-xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4,
-	       const unsigned long * __restrict p5)
-{
-	unsigned int lines = bytes / sizeof(unsigned long) / 2;
-	register unsigned int a1 __asm__("r8");
-	register unsigned int a2 __asm__("r9");
-	register unsigned int b1 __asm__("ip");
-	register unsigned int b2 __asm__("lr");
-
-	do {
-		GET_BLOCK_2(p1);
-		XOR_BLOCK_2(p2);
-		XOR_BLOCK_2(p3);
-		XOR_BLOCK_2(p4);
-		XOR_BLOCK_2(p5);
-		PUT_BLOCK_2(p1);
-	} while (--lines);
-}
-
-static struct xor_block_template xor_block_arm4regs = {
-	.name	= "arm4regs",
-	.do_2	= xor_arm4regs_2,
-	.do_3	= xor_arm4regs_3,
-	.do_4	= xor_arm4regs_4,
-	.do_5	= xor_arm4regs_5,
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES			\
-	do {					\
-		xor_speed(&xor_block_arm4regs);	\
-		xor_speed(&xor_block_8regs);	\
-		xor_speed(&xor_block_32regs);	\
-		NEON_TEMPLATES;			\
-	} while (0)
-
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_neon_inner;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2)
-{
-	if (in_interrupt()) {
-		xor_arm4regs_2(bytes, p1, p2);
-	} else {
-		kernel_neon_begin();
-		xor_block_neon_inner.do_2(bytes, p1, p2);
-		kernel_neon_end();
-	}
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3)
-{
-	if (in_interrupt()) {
-		xor_arm4regs_3(bytes, p1, p2, p3);
-	} else {
-		kernel_neon_begin();
-		xor_block_neon_inner.do_3(bytes, p1, p2, p3);
-		kernel_neon_end();
-	}
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3,
-	   const unsigned long * __restrict p4)
-{
-	if (in_interrupt()) {
-		xor_arm4regs_4(bytes, p1, p2, p3, p4);
-	} else {
-		kernel_neon_begin();
-		xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
-		kernel_neon_end();
-	}
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3,
-	   const unsigned long * __restrict p4,
-	   const unsigned long * __restrict p5)
-{
-	if (in_interrupt()) {
-		xor_arm4regs_5(bytes, p1, p2, p3, p4, p5);
-	} else {
-		kernel_neon_begin();
-		xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
-		kernel_neon_end();
-	}
-}
-
-static struct xor_block_template xor_block_neon = {
-	.name	= "neon",
-	.do_2	= xor_neon_2,
-	.do_3	= xor_neon_3,
-	.do_4	= xor_neon_4,
-	.do_5	= xor_neon_5
-};
-
-#define NEON_TEMPLATES	\
-	do { if (cpu_has_neon()) xor_speed(&xor_block_neon); } while (0)
-#else
-#define NEON_TEMPLATES
-#endif
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 0ca5aae1bcc3e..9295055cdfc92 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -39,9 +39,4 @@ endif
 $(obj)/csumpartialcopy.o:	$(obj)/csumpartialcopygeneric.S
 $(obj)/csumpartialcopyuser.o:	$(obj)/csumpartialcopygeneric.S
 
-ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
-  CFLAGS_xor-neon.o		+= $(CC_FLAGS_FPU)
-  obj-$(CONFIG_XOR_BLOCKS)	+= xor-neon.o
-endif
-
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c
deleted file mode 100644
index cf57fca979089..0000000000000
--- a/arch/arm/lib/xor-neon.c
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * linux/arch/arm/lib/xor-neon.c
- *
- * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/raid/xor.h>
-#include <linux/module.h>
-
-MODULE_DESCRIPTION("NEON accelerated XOR implementation");
-MODULE_LICENSE("GPL");
-
-#ifndef __ARM_NEON__
-#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
-#endif
-
-/*
- * Pull in the reference implementations while instructing GCC (through
- * -ftree-vectorize) to attempt to exploit implicit parallelism and emit
- * NEON instructions. Clang does this by default at O2 so no pragma is
- * needed.
- */
-#ifdef CONFIG_CC_IS_GCC
-#pragma GCC optimize "tree-vectorize"
-#endif
-
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#include <asm-generic/xor.h>
-
-struct xor_block_template const xor_block_neon_inner = {
-	.name	= "__inner_neon__",
-	.do_2	= xor_8regs_2,
-	.do_3	= xor_8regs_3,
-	.do_4	= xor_8regs_4,
-	.do_5	= xor_8regs_5,
-};
-EXPORT_SYMBOL(xor_block_neon_inner);
diff --git a/arch/arm/mach-exynos/firmware.c b/arch/arm/mach-exynos/firmware.c
index a5e22678e27be..e9b0ed07bb90f 100644
--- a/arch/arm/mach-exynos/firmware.c
+++ b/arch/arm/mach-exynos/firmware.c
@@ -61,10 +61,10 @@ static int exynos_cpu_boot(int cpu)
 	 * Exynos3250 doesn't need to send smc command for secondary CPU boot
 	 * because Exynos3250 removes WFE in secure mode.
 	 *
-	 * On Exynos5 devices the call is ignored by trustzone firmware.
+	 * On most Exynos5 devices the call is ignored by trustzone firmware.
 	 */
 	if (!soc_is_exynos4210() && !soc_is_exynos4212() &&
-	    !soc_is_exynos4412())
+	    !soc_is_exynos4412() && !of_machine_is_compatible("google,manta"))
 		return 0;
 
 	/*
diff --git a/arch/arm/mach-omap1/board-ams-delta.c b/arch/arm/mach-omap1/board-ams-delta.c
index 16392720296cd..1bec4fa0bd5e8 100644
--- a/arch/arm/mach-omap1/board-ams-delta.c
+++ b/arch/arm/mach-omap1/board-ams-delta.c
@@ -556,10 +556,30 @@ static struct gpiod_lookup_table *ams_delta_gpio_tables[] __initdata = {
 	&ams_delta_nand_gpio_table,
 };
 
-static struct gpiod_hog ams_delta_gpio_hogs[] = {
-	GPIO_HOG(LATCH2_LABEL, LATCH2_PIN_KEYBRD_DATAOUT, "keybrd_dataout",
-		 GPIO_ACTIVE_HIGH, GPIOD_OUT_LOW),
-	{},
+static const struct software_node latch2_gpio_swnode = {
+	.name = LATCH2_LABEL,
+};
+
+static const u32 latch2_hog_gpios[] = { LATCH2_PIN_KEYBRD_DATAOUT, 0 };
+
+static const struct property_entry latch2_gpio_hog_props[] = {
+	PROPERTY_ENTRY_BOOL("gpio-hog"),
+	PROPERTY_ENTRY_U32_ARRAY("gpios", latch2_hog_gpios),
+	PROPERTY_ENTRY_STRING("line-name", "keybrd_dataout"),
+	PROPERTY_ENTRY_BOOL("output-low"),
+	{ }
+};
+
+static const struct software_node latch2_gpio_hog_swnode = {
+	.parent = &latch2_gpio_swnode,
+	.name = "latch2-hog",
+	.properties = latch2_gpio_hog_props,
+};
+
+static const struct software_node *const latch2_gpio_swnodes[] = {
+	&latch2_gpio_swnode,
+	&latch2_gpio_hog_swnode,
+	NULL
 };
 
 static struct plat_serial8250_port ams_delta_modem_ports[];
@@ -684,7 +704,6 @@ static void __init ams_delta_init(void)
 
 	omap_gpio_deps_init();
 	ams_delta_latch2_init();
-	gpiod_add_hogs(ams_delta_gpio_hogs);
 
 	omap_serial_init();
 	omap_register_i2c_bus(1, 100, NULL, 0);
@@ -693,6 +712,9 @@ static void __init ams_delta_init(void)
 	platform_add_devices(ams_delta_devices, ARRAY_SIZE(ams_delta_devices));
 
 	platform_device_register_full(&latch1_gpio_devinfo);
+
+	software_node_register_node_group(latch2_gpio_swnodes);
+	latch2_gpio_devinfo.fwnode = software_node_fwnode(&latch2_gpio_swnode);
 	platform_device_register_full(&latch2_gpio_devinfo);
 
 	/*
diff --git a/arch/arm/mach-omap2/id.c b/arch/arm/mach-omap2/id.c
index 43f5944850e61..cf2bfb447ee28 100644
--- a/arch/arm/mach-omap2/id.c
+++ b/arch/arm/mach-omap2/id.c
@@ -17,6 +17,7 @@
 #include <linux/io.h>
 #include <linux/random.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 
 #ifdef CONFIG_SOC_BUS
 #include <linux/sys_soc.h>
@@ -250,7 +251,7 @@ static void __init omap3_cpuinfo(void)
 		cpu_name = "OMAP3503";
 	}
 
-	scnprintf(soc_name, sizeof(soc_name), "%s", cpu_name);
+	strscpy(soc_name, cpu_name);
 
 	/* Print verbose information */
 	n += scnprintf(buf, sizeof(buf) - n, "%s %s (", soc_name, soc_rev);
diff --git a/arch/arm/mach-s3c/mach-crag6410-module.c b/arch/arm/mach-s3c/mach-crag6410-module.c
index 4ffcf024b09df..14b0f9cc103e0 100644
--- a/arch/arm/mach-s3c/mach-crag6410-module.c
+++ b/arch/arm/mach-s3c/mach-crag6410-module.c
@@ -239,7 +239,6 @@ static struct gpiod_lookup_table wm8994_gpiod_table = {
 static struct arizona_pdata wm5102_reva_pdata = {
 	.gpio_base = CODEC_GPIO_BASE,
 	.irq_flags = IRQF_TRIGGER_HIGH,
-	.micd_pol_gpio = CODEC_GPIO_BASE + 4,
 	.micd_rate = 6,
 	.gpio_defaults = {
 		[2] = 0x10000, /* AIF3TXLRCLK */
@@ -265,6 +264,8 @@ static struct gpiod_lookup_table wm5102_reva_gpiod_table = {
 	.table = {
 		GPIO_LOOKUP("GPION", 7,
 			    "wlf,ldoena", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("arizona", 4,
+			    "wlf,micd-pol", GPIO_ACTIVE_HIGH),
 		{ },
 	},
 };
@@ -272,7 +273,6 @@ static struct gpiod_lookup_table wm5102_reva_gpiod_table = {
 static struct arizona_pdata wm5102_pdata = {
 	.gpio_base = CODEC_GPIO_BASE,
 	.irq_flags = IRQF_TRIGGER_HIGH,
-	.micd_pol_gpio = CODEC_GPIO_BASE + 2,
 	.gpio_defaults = {
 		[2] = 0x10000, /* AIF3TXLRCLK */
 		[3] = 0x4,     /* OPCLK */
@@ -297,6 +297,8 @@ static struct gpiod_lookup_table wm5102_gpiod_table = {
 	.table = {
 		GPIO_LOOKUP("GPION", 7,
 			    "wlf,ldo1ena", GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP("arizona", 2,
+			    "wlf,micd-pol", GPIO_ACTIVE_HIGH),
 		{ },
 	},
 };
diff --git a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
index 4277ba5b3ae05..747a96c58df15 100644
--- a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
+++ b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
@@ -141,7 +141,7 @@ static struct notifier_block regulator_quirk_nb = {
 static int __init rcar_gen2_regulator_quirk(void)
 {
 	struct regulator_quirk *quirk, *pos, *tmp;
-	struct of_phandle_args *argsa, *argsb;
+	struct of_phandle_args *args;
 	const struct of_device_id *id;
 	struct device_node *np;
 	u32 mon, addr;
@@ -171,14 +171,14 @@ static int __init rcar_gen2_regulator_quirk(void)
 			goto err_mem;
 		}
 
-		argsa = &quirk->irq_args;
+		args = &quirk->irq_args;
 		memcpy(&quirk->i2c_msg, id->data, sizeof(quirk->i2c_msg));
 
 		quirk->id = id;
 		quirk->np = of_node_get(np);
 		quirk->i2c_msg.addr = addr;
 
-		ret = of_irq_parse_one(np, 0, argsa);
+		ret = of_irq_parse_one(np, 0, args);
 		if (ret) {	/* Skip invalid entry and continue */
 			of_node_put(np);
 			kfree(quirk);
@@ -186,15 +186,7 @@ static int __init rcar_gen2_regulator_quirk(void)
 		}
 
 		list_for_each_entry(pos, &quirk_list, list) {
-			argsb = &pos->irq_args;
-
-			if (argsa->args_count != argsb->args_count)
-				continue;
-
-			ret = memcmp(argsa->args, argsb->args,
-				     argsa->args_count *
-				     sizeof(argsa->args[0]));
-			if (!ret) {
+			if (of_phandle_args_equal(args, &pos->irq_args)) {
 				pos->shared = true;
 				quirk->shared = true;
 			}
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 518def8314e77..23b87b5ef7f1b 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -41,13 +41,6 @@
 
 extern unsigned long __atags_pointer;
 
-/*
- * empty_zero_page is a special page that is used for
- * zero-initialized data and COW.
- */
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 /*
  * The pmd table for the upper-most set of pages.
  */
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 7e42d8accec6b..040ea43cce32e 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -27,13 +27,6 @@
 
 unsigned long vectors_base;
 
-/*
- * empty_zero_page is a special page that is used for
- * zero-initialized data and COW.
- */
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 #ifdef CONFIG_ARM_MPU
 struct mpu_rgn_info mpu_rgn_info;
 #endif
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 38dba5f7e4d2d..b11955714af54 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -55,38 +55,13 @@ config ARM64
 	select ARCH_HAS_STRICT_MODULE_RWX
 	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_BATCHED_DMA_SYNC
 	select ARCH_HAS_SYSCALL_WRAPPER
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_ZONE_DMA_SET if EXPERT
 	select ARCH_HAVE_ELF_PROT
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_HAVE_TRACE_MMIO_ACCESS
-	select ARCH_INLINE_READ_LOCK if !PREEMPTION
-	select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION
-	select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_READ_LOCK_IRQSAVE if !PREEMPTION
-	select ARCH_INLINE_READ_UNLOCK if !PREEMPTION
-	select ARCH_INLINE_READ_UNLOCK_BH if !PREEMPTION
-	select ARCH_INLINE_READ_UNLOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_READ_UNLOCK_IRQRESTORE if !PREEMPTION
-	select ARCH_INLINE_WRITE_LOCK if !PREEMPTION
-	select ARCH_INLINE_WRITE_LOCK_BH if !PREEMPTION
-	select ARCH_INLINE_WRITE_LOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_WRITE_LOCK_IRQSAVE if !PREEMPTION
-	select ARCH_INLINE_WRITE_UNLOCK if !PREEMPTION
-	select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPTION
-	select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPTION
-	select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPTION
-	select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPTION
-	select ARCH_INLINE_SPIN_LOCK if !PREEMPTION
-	select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPTION
-	select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPTION
-	select ARCH_INLINE_SPIN_UNLOCK if !PREEMPTION
-	select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION
-	select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION
-	select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION
 	select ARCH_KEEP_MEMBLOCK
 	select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
 	select ARCH_USE_CMPXCHG_LOCKREF
@@ -285,6 +260,7 @@ config ARM64
 	select USER_STACKTRACE_SUPPORT
 	select VDSO_GETRANDOM
 	select VMAP_STACK
+	select XOR_BLOCKS_ARCH
 	help
 	  ARM 64-bit (AArch64) Linux support.
 
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 54eb1d7fd419b..72c812e76b0b1 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -188,16 +188,15 @@ config ARCH_MESON
 	  This enables support for the arm64 based Amlogic SoCs
 	  such as the s905, S905X/D, S912, A113X/D or S905X/D2
 
-menu "Microchip SoC support"
+menuconfig ARCH_MICROCHIP
+	bool "Microchip SoC support"
 
-config ARCH_MICROCHIP
-	bool
+if ARCH_MICROCHIP
 
 config ARCH_LAN969X
 	bool "Microchip LAN969X SoC family"
 	select PINCTRL
 	select DW_APB_TIMER_OF
-	select ARCH_MICROCHIP
 	help
 	  This enables support for the Microchip LAN969X ARMv8-based
 	  SoC family of TSN-capable gigabit switches.
@@ -212,7 +211,6 @@ config ARCH_SPARX5
 	bool "Microchip Sparx5 SoC family"
 	select PINCTRL
 	select DW_APB_TIMER_OF
-	select ARCH_MICROCHIP
 	help
 	  This enables support for the Microchip Sparx5 ARMv8-based
 	  SoC family of TSN-capable gigabit switches.
@@ -223,7 +221,7 @@ config ARCH_SPARX5
 	  security through TCAM-based frame processing using versatile
 	  content aware processor (VCAP).
 
-endmenu
+endif
 
 config ARCH_MMP
 	bool "Marvell MMP SoC Family"
diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile
index 2edfa7bf4ab31..d116864b6c2b3 100644
--- a/arch/arm64/boot/dts/allwinner/Makefile
+++ b/arch/arm64/boot/dts/allwinner/Makefile
@@ -38,6 +38,7 @@ dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-orangepi-lite2.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-orangepi-one-plus.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-pine-h64.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-pine-h64-model-b.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-taiqicat-a01.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-tanix-tx6.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-tanix-tx6-mini.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h313-tanix-tx1.dtb
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-taiqicat-a01.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-taiqicat-a01.dts
new file mode 100644
index 0000000000000..225f42bd3b9e5
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-taiqicat-a01.dts
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+// Copyright (C) 2026 Jun Yan <jerrysteve1101@gmail.com>
+
+/dts-v1/;
+
+#include "sun50i-h6.dtsi"
+#include "sun50i-h6-cpu-opp.dtsi"
+#include "sun50i-h6-gpu-opp.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "TaiqiCat (TQC) A01";
+	compatible = "ultrapower,taiqicat-a01", "allwinner,sun50i-h6";
+
+	aliases {
+		ethernet1 = &sdio_wifi;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	connector {
+		compatible = "hdmi-connector";
+		ddc-en-gpios = <&pio 7 2 GPIO_ACTIVE_HIGH>; /* PH2 */
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	ext_osc32k: ext-osc32k-clk {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <32768>;
+		clock-output-names = "ext_osc32k";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			label = "taiqicat:blue:power";
+			gpios = <&r_pio 0 4 GPIO_ACTIVE_HIGH>; /* PL4 */
+			default-state = "on";
+		};
+	};
+
+	reg_vcc5v: vcc5v {
+		/* board wide 5V supply directly from the DC jack */
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-5v";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+
+	sound-spdif {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "sun50i-h6-spdif";
+
+		simple-audio-card,cpu {
+			sound-dai = <&spdif>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&spdif_out>;
+		};
+	};
+
+	spdif_out: spdif-out {
+		#sound-dai-cells = <0>;
+		compatible = "linux,spdif-dit";
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rtc CLK_OSC32K_FANOUT>;
+		clock-names = "ext_clock";
+		reset-gpios = <&r_pio 0 8 GPIO_ACTIVE_LOW>; /* PL8 */
+		post-power-on-delay-ms = <200>;
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&reg_dcdca>;
+};
+
+&de {
+	status = "okay";
+};
+
+&dwc3 {
+	status = "okay";
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci3 {
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&reg_dcdcc>;
+	status = "okay";
+};
+
+&hdmi {
+	hvcc-supply = <&reg_bldo2>;
+	status = "okay";
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&mmc0 {
+	vmmc-supply = <&reg_cldo1>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
+	bus-width = <4>;
+	disable-wp;
+	status = "okay";
+};
+
+&mmc1 {
+	vmmc-supply = <&reg_cldo3>;
+	vqmmc-supply = <&reg_bldo3>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	keep-power-in-suspend;
+	status = "okay";
+
+	sdio_wifi: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
+		interrupt-parent = <&r_pio>;
+		interrupts = <1 0 IRQ_TYPE_LEVEL_LOW>; /* PM0 */
+		interrupt-names = "host-wake";
+	};
+};
+
+&mmc2 {
+	vmmc-supply = <&reg_cldo1>;
+	vqmmc-supply = <&reg_bldo2>;
+	cap-mmc-hw-reset;
+	non-removable;
+	mmc-hs200-1_8v;
+	bus-width = <8>;
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci3 {
+	status = "okay";
+};
+
+&pio {
+	vcc-pc-supply = <&reg_bldo2>;
+	vcc-pd-supply = <&reg_cldo1>;
+	vcc-pg-supply = <&reg_bldo3>;
+};
+
+&r_i2c {
+	status = "okay";
+
+	axp805: pmic@36 {
+		compatible = "x-powers,axp805", "x-powers,axp806";
+		reg = <0x36>;
+		interrupt-parent = <&r_intc>;
+		interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-controller;
+		#interrupt-cells = <1>;
+		x-powers,self-working-mode;
+		vina-supply = <&reg_vcc5v>;
+		vinb-supply = <&reg_vcc5v>;
+		vinc-supply = <&reg_vcc5v>;
+		vind-supply = <&reg_vcc5v>;
+		vine-supply = <&reg_vcc5v>;
+		aldoin-supply = <&reg_vcc5v>;
+		bldoin-supply = <&reg_vcc5v>;
+		cldoin-supply = <&reg_vcc5v>;
+
+		regulators {
+			reg_aldo1: aldo1 {
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc-pl-led-ir-pg-pm-ts";
+			};
+
+			reg_aldo2: aldo2 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc-ac200";
+				regulator-enable-ramp-delay = <100000>;
+			};
+
+			aldo3 {
+				/* unused */
+			};
+
+			reg_bldo1: bldo1 {
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc18-dram-bias-pll";
+			};
+
+			reg_bldo2: bldo2 {
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc-pc-emmc-efuse-hdmi";
+			};
+
+			reg_bldo3: bldo3 {
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc-pg-wifiio";
+			};
+
+			bldo4 {
+				/* unused */
+			};
+
+			reg_cldo1: cldo1 {
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc33-io-pd-emmc-sd-usb-uart";
+			};
+
+			/* This regulator is connected with CLDO3 */
+			reg_cldo2: cldo2 {
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc-wifi-1";
+			};
+
+			reg_cldo3: cldo3 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc-wifi-2";
+			};
+
+			reg_dcdca: dcdca {
+				regulator-always-on;
+				regulator-min-microvolt = <810000>;
+				regulator-max-microvolt = <1160000>;
+				regulator-ramp-delay = <2500>;
+				regulator-name = "vdd-cpu";
+			};
+
+			reg_dcdcc: dcdcc {
+				regulator-enable-ramp-delay = <32000>;
+				regulator-min-microvolt = <810000>;
+				regulator-max-microvolt = <1080000>;
+				regulator-ramp-delay = <2500>;
+				regulator-name = "vdd-gpu";
+			};
+
+			reg_dcdcd: dcdcd {
+				regulator-always-on;
+				regulator-min-microvolt = <960000>;
+				regulator-max-microvolt = <960000>;
+				regulator-name = "vdd-sys-hdmi-usb";
+			};
+
+			reg_dcdce: dcdce {
+				regulator-always-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vcc-dram";
+			};
+
+			sw {
+				/* unused */
+			};
+		};
+	};
+};
+
+&r_ir {
+	status = "okay";
+};
+
+&r_pio {
+	/*
+	 * PL0 and PL1 are used for PMIC I2C
+	 * don't enable the pl-supply else
+	 * it will fail at boot
+	 *
+	 * vcc-pl-supply = <&reg_aldo1>;
+	 */
+	vcc-pm-supply = <&reg_aldo1>;
+};
+
+&rtc {
+	clocks = <&ext_osc32k>;
+};
+
+&spdif {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spdif_tx_pin>;
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_ph_pins>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins>, <&uart1_rts_cts_pins>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm43430a1-bt";
+		clocks = <&rtc CLK_OSC32K_FANOUT>;
+		clock-names = "lpo";
+		vbat-supply = <&reg_cldo3>;
+		vddio-supply = <&reg_bldo3>;
+		device-wakeup-gpios = <&r_pio 1 2 GPIO_ACTIVE_HIGH>; /* PM2 */
+		host-wakeup-gpios = <&r_pio 1 1 GPIO_ACTIVE_HIGH>; /* PM1 */
+		shutdown-gpios = <&r_pio 1 4 GPIO_ACTIVE_HIGH>; /* PM4 */
+		max-speed = <1500000>;
+	};
+};
+
+&usb2otg {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usb2phy {
+	usb0_vbus-supply = <&reg_vcc5v>;
+	usb3_vbus-supply = <&reg_vcc5v>;
+	status = "okay";
+};
+
+&usb3phy {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi b/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi
index 9335977751e28..da85cecb66c05 100644
--- a/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun55i-a523.dtsi
@@ -153,6 +153,13 @@ i2s2_pi_pins: i2s2-pi-pins {
 				bias-disable;
 			};
 
+			/omit-if-no-ref/
+			ledc_ph_pin: ledc-ph-pin {
+				pins = "PH19";
+				function = "ledc";
+				allwinner,pinmux = <5>;
+			};
+
 			mmc0_pins: mmc0-pins {
 				pins = "PF0" ,"PF1", "PF2", "PF3", "PF4", "PF5";
 				allwinner,pinmux = <2>;
@@ -221,6 +228,13 @@ spi0_pc_pins: spi0-pc-pins {
 				allwinner,pinmux = <4>;
 			};
 
+			/omit-if-no-ref/
+			spi0_pj_pins: spi0-pj-pins {
+				pins = "PJ21", "PJ22", "PJ23";
+				function = "spi0";
+				allwinner,pinmux = <5>;
+			};
+
 			/omit-if-no-ref/
 			spi0_cs0_pc_pin: spi0-cs0-pc-pin {
 				pins = "PC3";
@@ -228,6 +242,13 @@ spi0_cs0_pc_pin: spi0-cs0-pc-pin {
 				allwinner,pinmux = <4>;
 			};
 
+			/omit-if-no-ref/
+			spi0_cs0_pj_pin: spi0-cs0-pj-pin {
+				pins = "PJ20";
+				function = "spi0";
+				allwinner,pinmux = <5>;
+			};
+
 			/omit-if-no-ref/
 			spi0_cs1_pc_pin: spi0-cs1-pc-pin {
 				pins = "PC7";
@@ -235,6 +256,13 @@ spi0_cs1_pc_pin: spi0-cs1-pc-pin {
 				allwinner,pinmux = <4>;
 			};
 
+			/omit-if-no-ref/
+			spi0_cs1_pj_pin: spi0-cs1-pj-pin {
+				pins = "PJ24";
+				function = "spi0";
+				allwinner,pinmux = <5>;
+			};
+
 			/omit-if-no-ref/
 			spi0_hold_pc_pin: spi0-hold-pc-pin {
 				/* conflicts with eMMC D7 */
@@ -243,6 +271,13 @@ spi0_hold_pc_pin: spi0-hold-pc-pin {
 				allwinner,pinmux = <4>;
 			};
 
+			/omit-if-no-ref/
+			spi0_hold_pj_pin: spi0-hold-pj-pin {
+				pins = "PJ26";
+				function = "spi0";
+				allwinner,pinmux = <5>;
+			};
+
 			/omit-if-no-ref/
 			spi0_wp_pc_pin: spi0-wp-pc-pin {
 				/* conflicts with eMMC D2 */
@@ -251,6 +286,13 @@ spi0_wp_pc_pin: spi0-wp-pc-pin {
 				allwinner,pinmux = <4>;
 			};
 
+			/omit-if-no-ref/
+			spi0_wp_pj_pin: spi0-wp-pj-pin {
+				pins = "PJ25";
+				function = "spi0";
+				allwinner,pinmux = <5>;
+			};
+
 			uart0_pb_pins: uart0-pb-pins {
 				pins = "PB9", "PB10";
 				allwinner,pinmux = <2>;
@@ -283,6 +325,21 @@ ccu: clock-controller@2001000 {
 			#reset-cells = <1>;
 		};
 
+		ledc: led-controller@2008000 {
+			compatible = "allwinner,sun55i-a523-ledc",
+				     "allwinner,sun50i-a100-ledc";
+			reg = <0x02008000 0x400>;
+			interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_LEDC>, <&ccu CLK_LEDC>;
+			clock-names = "bus", "mod";
+			resets = <&ccu RST_BUS_LEDC>;
+			dmas = <&dma 42>;
+			dma-names = "tx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		wdt: watchdog@2050000 {
 			compatible = "allwinner,sun55i-a523-wdt";
 			reg = <0x2050000 0x20>;
diff --git a/arch/arm64/boot/dts/allwinner/sun55i-t527-avaota-a1.dts b/arch/arm64/boot/dts/allwinner/sun55i-t527-avaota-a1.dts
index 054d0357c139f..474354fbfcec1 100644
--- a/arch/arm64/boot/dts/allwinner/sun55i-t527-avaota-a1.dts
+++ b/arch/arm64/boot/dts/allwinner/sun55i-t527-avaota-a1.dts
@@ -6,6 +6,7 @@
 #include "sun55i-a523.dtsi"
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
 
 / {
 	model = "Avaota A1";
@@ -99,6 +100,46 @@ &gpu {
 	status = "okay";
 };
 
+&ledc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ledc_ph_pin>;
+	allwinner,pixel-format = "grb";
+	allwinner,t0l-ns = <800>;
+	allwinner,t0h-ns = <300>;
+	allwinner,t1l-ns = <320>;
+	allwinner,t1h-ns = <800>;
+	allwinner,treset-ns = <84>;
+	status = "okay";
+
+	multi-led@0 {
+		reg = <0x0>;
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_INDICATOR;
+		function-enumerator = <0>;
+	};
+
+	multi-led@1 {
+		reg = <0x1>;
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_INDICATOR;
+		function-enumerator = <1>;
+	};
+
+	multi-led@2 {
+		reg = <0x2>;
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_INDICATOR;
+		function-enumerator = <2>;
+	};
+
+	multi-led@3 {
+		reg = <0x3>;
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_INDICATOR;
+		function-enumerator = <3>;
+	};
+};
+
 &mdio0 {
 	ext_rgmii0_phy: ethernet-phy@1 {
 		compatible = "ethernet-phy-ieee802.3-c22";
@@ -362,6 +403,21 @@ &rtc {
 	assigned-clock-rates = <32768>;
 };
 
+&spi0  {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pj_pins>, <&spi0_cs0_pj_pin>,
+		    <&spi0_hold_pj_pin>, <&spi0_wp_pj_pin>;
+	status = "okay";
+
+	nand@0 {
+		compatible = "spi-nand";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pb_pins>;
diff --git a/arch/arm64/boot/dts/altera/Makefile b/arch/arm64/boot/dts/altera/Makefile
index 1bf0c472f6b4a..540bb5ae746b3 100644
--- a/arch/arm64/boot/dts/altera/Makefile
+++ b/arch/arm64/boot/dts/altera/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
 dtb-$(CONFIG_ARCH_INTEL_SOCFPGA) += socfpga_stratix10_socdk.dtb \
+				socfpga_stratix10_socdk_emmc.dtb \
 				socfpga_stratix10_socdk_nand.dtb \
 				socfpga_stratix10_swvp.dtb
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
index 4ae18a013bbed..e2a1cea7f3da9 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
@@ -3,53 +3,11 @@
  * Copyright Altera Corporation (C) 2015. All rights reserved.
  */
 
-#include "socfpga_stratix10.dtsi"
+#include "socfpga_stratix10_socdk.dtsi"
 
 / {
 	model = "SoCFPGA Stratix 10 SoCDK";
 	compatible = "altr,socfpga-stratix10-socdk", "altr,socfpga-stratix10";
-
-	aliases {
-		serial0 = &uart0;
-		ethernet0 = &gmac0;
-		ethernet1 = &gmac1;
-		ethernet2 = &gmac2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		led-hps0 {
-			label = "hps_led0";
-			gpios = <&portb 20 GPIO_ACTIVE_HIGH>;
-		};
-
-		led-hps1 {
-			label = "hps_led1";
-			gpios = <&portb 19 GPIO_ACTIVE_HIGH>;
-		};
-
-		led-hps2 {
-			label = "hps_led2";
-			gpios = <&portb 21 GPIO_ACTIVE_HIGH>;
-		};
-	};
-
-	memory@80000000 {
-		device_type = "memory";
-		/* We expect the bootloader to fill in the reg */
-		reg = <0 0x80000000 0 0>;
-	};
-
-	ref_033v: regulator-v-ref {
-		compatible = "regulator-fixed";
-		regulator-name = "0.33V";
-		regulator-min-microvolt = <330000>;
-		regulator-max-microvolt = <330000>;
-	};
 };
 
 &pinctrl0 {
@@ -68,10 +26,6 @@ i2c1_pmx_func_gpio: i2c1-pmx-func-gpio-pins {
 	};
 };
 
-&gpio1 {
-	status = "okay";
-};
-
 &gmac0 {
 	status = "okay";
 	phy-mode = "rgmii";
@@ -83,7 +37,7 @@ mdio0 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "snps,dwmac-mdio";
-		phy0: ethernet-phy@0 {
+		phy0: ethernet-phy@4 {
 			reg = <4>;
 
 			txd0-skew-ps = <0>; /* -420ps */
@@ -111,23 +65,6 @@ &mmc {
 	clk-phase-sd-hs = <0>, <135>;
 };
 
-&osc1 {
-	clock-frequency = <25000000>;
-};
-
-&uart0 {
-	status = "okay";
-};
-
-&usb0 {
-	status = "okay";
-	disable-over-current;
-};
-
-&watchdog0 {
-	status = "okay";
-};
-
 &i2c1 {
 	status = "okay";
 	clock-frequency = <100000>;
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi
new file mode 100755
index 0000000000000..1d50f7b21160c
--- /dev/null
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dtsi
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright Altera Corporation (C) 2026. All rights reserved.
+ */
+
+#include "socfpga_stratix10.dtsi"
+
+/ {
+	aliases {
+		serial0 = &uart0;
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
+		ethernet2 = &gmac2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		led-hps0 {
+			label = "hps_led0";
+			gpios = <&portb 20 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-hps1 {
+			label = "hps_led1";
+			gpios = <&portb 19 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-hps2 {
+			label = "hps_led2";
+			gpios = <&portb 21 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the reg */
+		reg = <0 0x80000000 0 0>;
+	};
+
+	ref_033v: regulator-0v33-ref {
+		compatible = "regulator-fixed";
+		regulator-name = "0.33V";
+		regulator-min-microvolt = <330000>;
+		regulator-max-microvolt = <330000>;
+	};
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&osc1 {
+	clock-frequency = <25000000>;
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+	disable-over-current;
+};
+
+&watchdog0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts
new file mode 100755
index 0000000000000..b2a3449638dd3
--- /dev/null
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_emmc.dts
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright Altera Corporation (C) 2026. All rights reserved.
+ */
+
+#include "socfpga_stratix10_socdk.dtsi"
+
+/ {
+	model = "SoCFPGA Stratix 10 SoCDK eMMC daughter board";
+	compatible = "altr,socfpga-stratix10-socdk-emmc",
+			"altr,socfpga-stratix10-socdk",
+			"altr,socfpga-stratix10";
+};
+
+&gmac2 {
+	status = "okay";
+	/* PHY delays is configured via skew properties */
+	phy-mode = "rgmii";
+	phy-handle = <&phy0>;
+
+	max-frame-size = <9000>;
+
+	mdio0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+		phy0: ethernet-phy@4 {
+			reg = <4>;
+
+			txd0-skew-ps = <0>; /* -420ps */
+			txd1-skew-ps = <0>; /* -420ps */
+			txd2-skew-ps = <0>; /* -420ps */
+			txd3-skew-ps = <0>; /* -420ps */
+			rxd0-skew-ps = <420>; /* 0ps */
+			rxd1-skew-ps = <420>; /* 0ps */
+			rxd2-skew-ps = <420>; /* 0ps */
+			rxd3-skew-ps = <420>; /* 0ps */
+			txen-skew-ps = <0>; /* -420ps */
+			txc-skew-ps = <900>; /* 0ps */
+			rxdv-skew-ps = <420>; /* 0ps */
+			rxc-skew-ps = <1680>; /* 780ps */
+		};
+	};
+};
+
+&mmc {
+	status = "okay";
+	cap-mmc-highspeed;
+	broken-cd;
+	bus-width = <4>;
+	clk-phase-sd-hs = <0>, <135>;
+};
+
+&i2c2 {
+	status = "okay";
+	clock-frequency = <100000>;
+	i2c-sda-falling-time-ns = <890>;  /* hcnt */
+	i2c-scl-falling-time-ns = <890>;  /* lcnt */
+
+	adc@14 {
+		compatible = "lltc,ltc2497";
+		reg = <0x14>;
+		vref-supply = <&ref_033v>;
+	};
+
+	temp@4c {
+		compatible = "maxim,max1619";
+		reg = <0x4c>;
+	};
+
+	eeprom@51 {
+		compatible = "atmel,24c32";
+		reg = <0x51>;
+		pagesize = <32>;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1339";
+		reg = <0x68>;
+	};
+};
diff --git a/arch/arm64/boot/dts/arm/Makefile b/arch/arm64/boot/dts/arm/Makefile
index f30ee045dc959..770fb145b4a92 100644
--- a/arch/arm64/boot/dts/arm/Makefile
+++ b/arch/arm64/boot/dts/arm/Makefile
@@ -8,3 +8,4 @@ dtb-$(CONFIG_ARCH_VEXPRESS) += vexpress-v2f-1xv7-ca53x2.dtb
 dtb-$(CONFIG_ARCH_VEXPRESS) += fvp-base-revc.dtb
 dtb-$(CONFIG_ARCH_VEXPRESS) += corstone1000-fvp.dtb corstone1000-mps3.dtb
 dtb-$(CONFIG_ARCH_VEXPRESS) += morello-sdp.dtb morello-fvp.dtb
+dtb-$(CONFIG_ARCH_VEXPRESS) += zena-css-fvp.dtb
diff --git a/arch/arm64/boot/dts/arm/zena-css-fvp.dts b/arch/arm64/boot/dts/arm/zena-css-fvp.dts
new file mode 100644
index 0000000000000..53c5412d92b21
--- /dev/null
+++ b/arch/arm64/boot/dts/arm/zena-css-fvp.dts
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright (c) 2025, Arm Limited. All rights reserved.
+ */
+
+/dts-v1/;
+
+#include "zena-css.dtsi"
+
+/ {
+	model = "Zena CSS Fixed Virtual Platform";
+	compatible = "arm,zena-css-fvp", "arm,zena-css", "arm,vexpress";
+
+	chosen {
+		stdout-path = &soc_serial0;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+
+		/* ~2GB mapped at 2GB, another 2GB at 2TB */
+		reg = <0x00000000 0x80000000 0x00000000 0x7f000000>,
+		      <0x00000200 0x00000000 0x00000000 0x80000000>;
+	};
+};
+
+&soc {
+	virtio@30020000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30020000 0x0 0x10000>;
+		interrupts = <GIC_SPI 257 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	virtio@30030000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30030000 0x0 0x10000>;
+		interrupts = <GIC_SPI 258 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	virtio@30040000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30040000 0x0 0x10000>;
+		interrupts = <GIC_SPI 259 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	virtio@30050000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30050000 0x0 0x10000>;
+		interrupts = <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	virtio@30060000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30060000 0x0 0x10000>;
+		interrupts = <GIC_SPI 261 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	virtio@30080000 {
+		compatible = "virtio,mmio";
+		reg = <0x0 0x30080000 0x0 0x10000>;
+		interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
diff --git a/arch/arm64/boot/dts/arm/zena-css.dtsi b/arch/arm64/boot/dts/arm/zena-css.dtsi
new file mode 100644
index 0000000000000..0b41ee4bf4c6e
--- /dev/null
+++ b/arch/arm64/boot/dts/arm/zena-css.dtsi
@@ -0,0 +1,769 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright (c) 2025, Arm Limited. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	interrupt-parent = <&gic>;
+
+	soc_clk24mhz: clock-24000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "refclk24mhz";
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		/*
+		 * The latency and residency numbers below are for illustrative
+		 * purposes only and may vary on actual silicon. These values are
+		 * considered just to demonstrate that the cpuidle governor logic
+		 * works.
+		 */
+		idle-states {
+			entry-method = "psci";
+
+			cpu_sleep: cpu-sleep {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x10000>;
+				entry-latency-us = <800>;
+				exit-latency-us = <3200>;
+				local-timer-stop;
+				min-residency-us = <4200>;
+			};
+
+			cluster_sleep: cluster-sleep {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x1010000>;
+				entry-latency-us = <1000>;
+				exit-latency-us = <3200>;
+				local-timer-stop;
+				min-residency-us = <4500>;
+			};
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 { cpu = <&cpu0>; };
+				core1 { cpu = <&cpu1>; };
+				core2 { cpu = <&cpu2>; };
+				core3 { cpu = <&cpu3>; };
+			};
+
+			cluster1 {
+				core0 { cpu = <&cpu4>; };
+				core1 { cpu = <&cpu5>; };
+				core2 { cpu = <&cpu6>; };
+				core3 { cpu = <&cpu7>; };
+			};
+
+			cluster2 {
+				core0 { cpu = <&cpu8>; };
+				core1 { cpu = <&cpu9>; };
+				core2 { cpu = <&cpu10>; };
+				core3 { cpu = <&cpu11>; };
+			};
+
+			cluster3 {
+				core0 { cpu = <&cpu12>; };
+				core1 { cpu = <&cpu13>; };
+				core2 { cpu = <&cpu14>; };
+				core3 { cpu = <&cpu15>; };
+			};
+		};
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x0000>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl0_l2_0>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl0_l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl0_l3>;
+			};
+		};
+
+		cpu1: cpu@100 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x0100>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl0_l2_1>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl0_l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl0_l3>;
+			};
+		};
+
+		cpu2: cpu@200 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x0200>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl0_l2_2>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl0_l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl0_l3>;
+			};
+		};
+
+		cpu3: cpu@300 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x0300>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl0_l2_3>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl0_l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl0_l3>;
+			};
+		};
+
+		cpu4: cpu@10000 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x10000>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl1_l2_0>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl1_l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl1_l3>;
+			};
+		};
+
+		cpu5: cpu@10100 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x10100>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl1_l2_1>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl1_l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl1_l3>;
+			};
+		};
+
+		cpu6: cpu@10200 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x10200>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl1_l2_2>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl1_l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl1_l3>;
+			};
+		};
+
+		cpu7: cpu@10300 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x10300>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl1_l2_3>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl1_l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl1_l3>;
+			};
+		};
+
+		cpu8: cpu@20000 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x20000>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl2_l2_0>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl2_l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl2_l3>;
+			};
+		};
+
+		cpu9: cpu@20100 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x20100>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl2_l2_1>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl2_l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl2_l3>;
+			};
+		};
+
+		cpu10: cpu@20200 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x20200>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl2_l2_2>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl2_l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl2_l3>;
+			};
+		};
+
+		cpu11: cpu@20300 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x20300>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl2_l2_3>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl2_l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl2_l3>;
+			};
+		};
+
+		cpu12: cpu@30000 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x30000>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl3_l2_0>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl3_l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl3_l3>;
+			};
+		};
+
+		cpu13: cpu@30100 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x30100>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl3_l2_1>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl3_l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl3_l3>;
+			};
+		};
+
+		cpu14: cpu@30200 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x30200>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl3_l2_2>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl3_l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl3_l3>;
+			};
+		};
+
+		cpu15: cpu@30300 {
+			compatible = "arm,cortex-a720ae";
+			device_type = "cpu";
+			reg = <0x00 0x30300>;
+			enable-method = "psci";
+
+			clocks = <&scmi_dvfs 0>;
+			cpu-idle-states = <&cpu_sleep &cluster_sleep>;
+			next-level-cache = <&cl3_l2_3>;
+
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			i-cache-size = <0x10000>;
+
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x10000>;
+
+			cl3_l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-line-size = <64>;
+				cache-sets = <0x400>;	/* 8-way set */
+				cache-size = <0x80000>;	/* 512KB */
+				cache-unified;
+				next-level-cache = <&cl3_l3>;
+			};
+		};
+
+		cl0_l3: l3-cache0 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <0x1000>;		/* 16-way set */
+			cache-size = <0x400000>;	/* 4MB */
+			cache-unified;
+		};
+
+		cl1_l3: l3-cache1 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <0x1000>;		/* 16-way set */
+			cache-size = <0x400000>;	/* 4MB */
+			cache-unified;
+		};
+
+		cl2_l3: l3-cache2 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <0x1000>;		/* 16-way set */
+			cache-size = <0x400000>;	/* 4MB */
+			cache-unified;
+		};
+
+		cl3_l3: l3-cache3 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <0x1000>;		/* 16-way set */
+			cache-size = <0x400000>;	/* 4MB */
+			cache-unified;
+		};
+	};
+
+	firmware {
+		scmi {
+			compatible = "arm,scmi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mbox-names = "tx", "tx_reply", "rx";
+			mboxes = <&mbox_db_tx 0 0 0>,
+				 <&mbox_db_rx 0 0 0>,
+				 <&mbox_db_rx 0 0 2>;
+			shmem = <&scmi_shmem_tx &scmi_shmem_rx>;
+
+			scmi_dvfs: protocol@13 {
+				reg = <0x13>;
+				#clock-cells = <1>;
+			};
+		};
+	};
+
+	dsu-pmu-0 {
+		compatible = "arm,dsu-pmu";
+		cpus = <&cpu0 &cpu1 &cpu2 &cpu3>;
+		interrupts = <GIC_SPI 184 IRQ_TYPE_EDGE_RISING>;
+	};
+
+	dsu-pmu-1 {
+		compatible = "arm,dsu-pmu";
+		cpus = <&cpu4 &cpu5 &cpu6 &cpu7>;
+		interrupts = <GIC_SPI 185 IRQ_TYPE_EDGE_RISING>;
+	};
+
+	dsu-pmu-2 {
+		compatible = "arm,dsu-pmu";
+		cpus = <&cpu8 &cpu9 &cpu10 &cpu11>;
+		interrupts = <GIC_SPI 186 IRQ_TYPE_EDGE_RISING>;
+	};
+
+	dsu-pmu-3 {
+		compatible = "arm,dsu-pmu";
+		cpus = <&cpu12 &cpu13 &cpu14 &cpu15>;
+		interrupts = <GIC_SPI 187 IRQ_TYPE_EDGE_RISING>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		sram: sram@104000 {
+			compatible = "mmio-sram";
+			reg = <0x0 0x00104000 0x0 0x00001000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x0 0x00104000 0x00001000>;
+
+			scmi_shmem_tx: scpshmem-sram-section@0 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x0 0x100>;
+			};
+
+			scmi_shmem_rx: scpshmem-sram-section@100 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x100 0x100>;
+			};
+		};
+
+		timer@1a810000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x1a810000 0x0 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			/*
+			 * Map child space [0x0..0x30000) to parent @ 0x1a810000
+			 */
+			ranges = <0x0 0x0 0x1a810000 0x00030000>;
+
+			frame@20000 {
+				reg = <0x20000 0x10000>;
+				frame-number = <0>;
+				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
+		gic: interrupt-controller@20800000 {
+			compatible = "arm,gic-v3";
+			#interrupt-cells = <3>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			#redistributor-regions = <16>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			ranges;
+
+			/*
+			 * With GIC-A720AE multiview enabled, GICR_TYPER.Last is
+			 * always reported as 1 on redistributor views other than
+			 * view 0. This breaks discovery of a single contiguous
+			 * GICR frame region, so each core is described with its own
+			 * redistributor region.
+			 */
+			reg = <0x0 0x20800000 0x0 0x10000>,	/* GICD */
+			      <0x0 0x20880000 0x0 0x40000>,	/* 16 * GICR */
+			      <0x0 0x208c0000 0x0 0x40000>,
+			      <0x0 0x20900000 0x0 0x40000>,
+			      <0x0 0x20940000 0x0 0x40000>,
+			      <0x0 0x20980000 0x0 0x40000>,
+			      <0x0 0x209c0000 0x0 0x40000>,
+			      <0x0 0x20a00000 0x0 0x40000>,
+			      <0x0 0x20a40000 0x0 0x40000>,
+			      <0x0 0x20a80000 0x0 0x40000>,
+			      <0x0 0x20ac0000 0x0 0x40000>,
+			      <0x0 0x20b00000 0x0 0x40000>,
+			      <0x0 0x20b40000 0x0 0x40000>,
+			      <0x0 0x20b80000 0x0 0x40000>,
+			      <0x0 0x20bc0000 0x0 0x40000>,
+			      <0x0 0x20c00000 0x0 0x40000>,
+			      <0x0 0x20c40000 0x0 0x40000>;
+
+			its: msi-controller@20840000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x20840000 0x0 0x40000>;
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		/*
+		 * UART is fixed at 24MHz, both UARTCLK and PCLK.
+		 */
+		soc_serial0: serial@1a400000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x0 0x1a400000 0x0 0x10000>;
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&soc_clk24mhz>, <&soc_clk24mhz>;
+			clock-names = "uartclk", "apb_pclk";
+		};
+
+		watchdog@1a420000 {
+			compatible = "arm,sbsa-gwdt";
+			reg = <0x0 0x1a420000 0x0 0x10000>,
+			      <0x0 0x1a430000 0x0 0x10000>;
+			interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		rtc@300d0000 {
+			compatible = "arm,pl031", "arm,primecell";
+			reg = <0x0 0x300d0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&soc_clk24mhz>;
+			clock-names = "apb_pclk";
+		};
+
+		mbox_db_tx: mailbox@40020000 {
+			compatible = "arm,mhuv3";
+			reg = <0x0 0x40020000 0x0 0x30000>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "combined";
+			clocks = <&soc_clk24mhz>;
+			#mbox-cells = <3>;
+		};
+
+		mbox_db_rx: mailbox@40060000 {
+			compatible = "arm,mhuv3";
+			reg = <0x0 0x40060000 0x0 0x30000>;
+			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "combined";
+			clocks = <&soc_clk24mhz>;
+			#mbox-cells = <3>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 12 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
index 7de24d60bcd1a..127be0fc27c22 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
+++ b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
@@ -35,3 +35,17 @@ &gio_aon {
 		"PMIC_SCL", // AON_SGPIO_04
 		"PMIC_SDA"; // AON_SGPIO_05
 };
+
+&pinctrl {
+	compatible = "brcm,bcm2712d0-pinctrl";
+	reg = <0x7d504100 0x20>;
+};
+
+&pinctrl_aon {
+	compatible = "brcm,bcm2712d0-aon-pinctrl";
+	reg = <0x7d510700 0x1c>;
+};
+
+&uart10 {
+	interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
index 04738bf281ebd..b7a6bc34ae1ab 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
@@ -44,6 +44,30 @@ power_button: power-button {
 		};
 	};
 
+	firmware {
+		firmware: rpi-firmware {
+			compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
+
+			mboxes = <&mailbox>;
+
+			firmware_clocks: clocks {
+				compatible = "raspberrypi,firmware-clocks";
+				#clock-cells = <1>;
+			};
+
+			reset: reset {
+				compatible = "raspberrypi,firmware-reset";
+				#reset-cells = <1>;
+			};
+
+			power: power {
+				compatible = "raspberrypi,bcm2835-power";
+				firmware = <&firmware>;
+				#power-domain-cells = <1>;
+			};
+		};
+	};
+
 	sd_io_1v8_reg: sd-io-1v8-reg {
 		compatible = "regulator-gpio";
 		regulator-name = "vdd-sd-io";
@@ -189,33 +213,6 @@ wifi: wifi@1 {
 	};
 };
 
-&soc {
-	firmware: firmware {
-		compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
-		#address-cells = <1>;
-		#size-cells = <1>;
-
-		mboxes = <&mailbox>;
-		dma-ranges;
-
-		firmware_clocks: clocks {
-			compatible = "raspberrypi,firmware-clocks";
-			#clock-cells = <1>;
-		};
-
-		reset: reset {
-			compatible = "raspberrypi,firmware-reset";
-			#reset-cells = <1>;
-		};
-	};
-
-	power: power {
-		compatible = "raspberrypi,bcm2835-power";
-		firmware = <&firmware>;
-		#power-domain-cells = <1>;
-	};
-};
-
 /* uarta communicates with the BT module */
 &uarta {
 	uart-has-rtscts;
@@ -252,3 +249,7 @@ &pcie1 {
 &pcie2 {
 	status = "okay";
 };
+
+&v3d {
+	clocks = <&firmware_clocks 5>;
+};
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts
index 2856082814462..0fc57e72632ed 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts
+++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts
@@ -68,6 +68,30 @@ usb_vbus_default_state: usb-vbus-default-state {
 		function = "vbus1";
 		groups = "vbus1";
 	};
+
+	rp1_i2c4_default_state: rp1-i2c4-default-state {
+		function = "i2c4";
+		groups = "i2c4_2";
+		drive-strength = <12>;
+		bias-pull-up;
+	};
+
+	rp1_i2c6_default_state: rp1-i2c6-default-state {
+		function = "i2c6";
+		groups = "i2c6_0";
+		drive-strength = <12>;
+		bias-pull-up;
+	};
+};
+
+&rp1_i2c4 {
+	pinctrl-0 = <&rp1_i2c4_default_state>;
+	pinctrl-names = "default";
+};
+
+&rp1_i2c6 {
+	pinctrl-0 = <&rp1_i2c6_default_state>;
+	pinctrl-names = "default";
 };
 
 &rp1_usb0 {
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
index d57a9b1bff70c..761c59d90ffc9 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/soc/bcm2835-pm.h>
 
 / {
 	compatible = "brcm,bcm2712";
@@ -508,10 +509,6 @@ axi: axi {
 			     <0x18 0x00000000  0x18 0x00000000  0x04 0x00000000>,
 			     <0x1c 0x00000000  0x1c 0x00000000  0x04 0x00000000>;
 
-		vc4: gpu {
-			compatible = "brcm,bcm2712-vc6";
-		};
-
 		pcie0: pcie@1000100000 {
 			compatible = "brcm,bcm2712-pcie";
 			reg = <0x10 0x00100000 0x00 0x9310>;
@@ -646,6 +643,30 @@ mip1: msi-controller@1000131000 {
 			msi-ranges = <&gicv2 GIC_SPI 247 IRQ_TYPE_EDGE_RISING 8>;
 			brcm,msi-offset = <8>;
 		};
+
+		isp: isp@1000880000  {
+		     compatible = "brcm,bcm2712-pispbe", "raspberrypi,pispbe";
+		     reg = <0x10 0x00880000 0x0 0x4000>;
+		     interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+		     clocks = <&firmware_clocks 7>;
+		};
+
+		v3d: gpu@1002000000 {
+			compatible = "brcm,2712-v3d";
+			reg = <0x10 0x02000000 0x00 0x4000>,
+			      <0x10 0x02008000 0x00 0x6000>,
+			      <0x10 0x02030800 0x00 0x0700>;
+			reg-names = "hub", "core0", "sms";
+
+			power-domains = <&pm BCM2835_POWER_DOMAIN_GRAFX_V3D>;
+			resets = <&pm BCM2835_RESET_V3D>;
+			interrupts = <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+
+	vc4: gpu {
+		compatible = "brcm,bcm2712-vc6";
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/broadcom/rp1-common.dtsi b/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
index 5a815c3797945..16f5359395835 100644
--- a/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
+++ b/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
@@ -26,6 +26,83 @@ rp1_clocks: clocks@40018000 {
 				       <200000000>;  // RP1_CLK_SYS
 	};
 
+	rp1_i2c0: i2c@40070000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40070000  0x0 0x1000>;
+		interrupts = <7 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c1: i2c@40074000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40074000  0x0 0x1000>;
+		interrupts = <8 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c2: i2c@40078000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40078000  0x0 0x1000>;
+		interrupts = <9 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c3: i2c@4007c000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x4007c000  0x0 0x1000>;
+		interrupts = <10 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c4: i2c@40080000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40080000  0x0 0x1000>;
+		interrupts = <11 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c5: i2c@40084000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40084000  0x0 0x1000>;
+		interrupts = <12 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
+	rp1_i2c6: i2c@40088000 {
+		compatible = "snps,designware-i2c";
+		reg = <0x0 0x40088000  0x0 0x1000>;
+		interrupts = <13 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		i2c-scl-rising-time-ns = <65>;
+		i2c-scl-falling-time-ns = <100>;
+
+		status = "disabled";
+	};
+
 	rp1_gpio: pinctrl@400d0000 {
 		compatible = "raspberrypi,rp1-gpio";
 		reg = <0x00 0x400d0000  0x0 0xc000>,
@@ -56,6 +133,34 @@ rp1_eth: ethernet@40100000 {
 		#size-cells = <0>;
 	};
 
+	rp1_csi0: csi@40110000 {
+		compatible = "raspberrypi,rp1-cfe";
+		reg = <0x0 0x40110000  0x0 0x100>, // CSI2 DMA address
+		      <0x0 0x40114000  0x0 0x100>, // PHY/CSI Host address
+		      <0x0 0x40120000  0x0 0x100>, // MIPI CFG address
+		      <0x0 0x40124000  0x0 0x1000>; // PiSP FE address
+		interrupts = <47 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_MIPI0_CFG>;
+		assigned-clocks = <&rp1_clocks RP1_CLK_MIPI0_CFG>;
+		assigned-clock-rates = <25000000>;
+
+		status = "disabled";
+	};
+
+	rp1_csi1: csi@40128000 {
+		compatible = "raspberrypi,rp1-cfe";
+		reg = <0x0 0x40128000  0x0 0x100>, // CSI2 DMA address
+		      <0x0 0x4012c000  0x0 0x100>, // PHY/CSI Host address
+		      <0x0 0x40138000  0x0 0x100>, // MIPI CFG address
+		      <0x0 0x4013c000  0x0 0x1000>; // PiSP FE address
+		interrupts = <48 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_MIPI1_CFG>;
+		assigned-clocks = <&rp1_clocks RP1_CLK_MIPI1_CFG>;
+		assigned-clock-rates = <25000000>;
+
+		status = "disabled";
+	};
+
 	rp1_usb0: usb@40200000 {
 		compatible = "snps,dwc3";
 		reg = <0x00 0x40200000  0x0 0x100000>;
diff --git a/arch/arm64/boot/dts/cix/sky1.dtsi b/arch/arm64/boot/dts/cix/sky1.dtsi
index fb8c826bbc97e..310dd1aadf6de 100644
--- a/arch/arm64/boot/dts/cix/sky1.dtsi
+++ b/arch/arm64/boot/dts/cix/sky1.dtsi
@@ -348,6 +348,12 @@ i3c1: i3c@4100000 {
 			status = "disabled";
 		};
 
+		syscon: syscon@4160000 {
+			compatible = "cix,sky1-system-control", "syscon";
+			reg = <0x0 0x4160000 0x0 0x100>;
+			#reset-cells = <1>;
+		};
+
 		iomuxc: pinctrl@4170000 {
 			compatible = "cix,sky1-pinctrl";
 			reg = <0x0 0x04170000 0x0 0x1000>;
@@ -568,6 +574,12 @@ ppi_partition1: interrupt-partition-1 {
 			};
 		};
 
+		s5_syscon: syscon@16000000 {
+			compatible = "cix,sky1-s5-system-control", "syscon";
+			reg = <0x0 0x16000000 0x0 0x1000>;
+			#reset-cells = <1>;
+		};
+
 		iomuxc_s5: pinctrl@16007000 {
 			compatible = "cix,sky1-pinctrl-s5";
 			reg = <0x0 0x16007000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile
index bcca631365578..76cc23acb9b29 100644
--- a/arch/arm64/boot/dts/exynos/Makefile
+++ b/arch/arm64/boot/dts/exynos/Makefile
@@ -8,7 +8,9 @@ dtb-$(CONFIG_ARCH_EXYNOS) += \
 	exynos5433-tm2e.dtb		\
 	exynos7-espresso.dtb		\
 	exynos7870-a2corelte.dtb	\
+	exynos7870-j5y17lte.dtb		\
 	exynos7870-j6lte.dtb		\
+	exynos7870-j7xelte.dtb		\
 	exynos7870-on7xelte.dtb		\
 	exynos7885-jackpotlte.dtb	\
 	exynos850-e850-96.dtb		\
diff --git a/arch/arm64/boot/dts/exynos/axis/Makefile b/arch/arm64/boot/dts/exynos/axis/Makefile
index ccf00de640166..da6a426516fca 100644
--- a/arch/arm64/boot/dts/exynos/axis/Makefile
+++ b/arch/arm64/boot/dts/exynos/axis/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
 dtb-$(CONFIG_ARCH_ARTPEC) += \
-	artpec8-grizzly.dtb
+	artpec8-grizzly.dtb \
+	artpec9-alfred.dtb
diff --git a/arch/arm64/boot/dts/exynos/axis/artpec9-alfred.dts b/arch/arm64/boot/dts/exynos/axis/artpec9-alfred.dts
new file mode 100644
index 0000000000000..5a779f1acf3ba
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/axis/artpec9-alfred.dts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Axis ARTPEC-9 Alfred board device tree source
+ *
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd.
+ *             https://www.samsung.com
+ * Copyright (c) 2025  Axis Communications AB.
+ *             https://www.axis.com
+ */
+
+/dts-v1/;
+#include "artpec9.dtsi"
+#include "artpec9-pinctrl.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "ARTPEC-9 alfred board";
+	compatible = "axis,artpec9-alfred", "axis,artpec9";
+
+	aliases {
+		serial0 = &serial_0;
+	};
+
+	chosen {
+		stdout-path = &serial_0;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x80000000>;
+	};
+};
+
+&osc_clk {
+	clock-frequency = <50000000>;
+};
diff --git a/arch/arm64/boot/dts/exynos/axis/artpec9-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/axis/artpec9-pinctrl.dtsi
new file mode 100644
index 0000000000000..a9fbdf7734d42
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/axis/artpec9-pinctrl.dtsi
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Axis ARTPEC-9 SoC pin-mux and pin-config device tree source
+ *
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd.
+ *             https://www.samsung.com
+ * Copyright (c) 2025  Axis Communications AB.
+ *             https://www.axis.com
+ */
+
+#include "artpec-pinctrl.h"
+
+&pinctrl_fsys0 {
+	gpe0: gpe0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpe1: gpe1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpe2: gpe2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpe3: gpe3-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpe4: gpe4-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf0: gpf0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf1: gpf1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpi0: gpi0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gps0: gps0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gps1: gps1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_fsys1 {
+	gpu0: gpu0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	serial0_bus: serial0-bus-pins {
+		samsung,pins = "gpu0-0", "gpu0-1";
+		samsung,pin-function = <ARTPEC_PIN_FUNC_2>;
+		samsung,pin-pud = <ARTPEC_PIN_PULL_UP>;
+		samsung,pin-drv = <ARTPEC_PIN_DRV_SR3>;
+	};
+};
+
+&pinctrl_peric {
+	gpa0: gpa0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpa1: gpa1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
diff --git a/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi b/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
new file mode 100644
index 0000000000000..f8ed43c6e8258
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Axis ARTPEC-9 SoC device tree source
+ *
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd.
+ *             https://www.samsung.com
+ * Copyright (c) 2025  Axis Communications AB.
+ *             https://www.axis.com
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/axis,artpec9-clk.h>
+
+/ {
+	compatible = "axis,artpec9";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		pinctrl0 = &pinctrl_fsys0;
+		pinctrl1 = &pinctrl_fsys1;
+		pinctrl2 = &pinctrl_peric;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x0>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+			clocks = <&cmu_cpucl CLK_GOUT_CPUCL_CLUSTER_CPU>;
+			clock-names = "cpu";
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x100>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x200>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x300>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x400>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x500>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cpu_sleep: cpu-sleep {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x0010000>;
+				local-timer-stop;
+				entry-latency-us = <300>;
+				exit-latency-us = <1200>;
+				min-residency-us = <2000>;
+			};
+		};
+	};
+
+	fin_pll: clock-finpll {
+		compatible = "fixed-factor-clock";
+		clocks = <&osc_clk>;
+		#clock-cells = <0>;
+		clock-div = <2>;
+		clock-mult = <1>;
+		clock-output-names = "fin_pll";
+	};
+
+	osc_clk: clock-osc {
+		/* XXTI */
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-output-names = "osc_clk";
+	};
+
+	pmu {
+		compatible = "arm,cortex-a55-pmu";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>, <&cpu4>, <&cpu5>;
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		ranges = <0x0 0x0 0x0 0x0 0x0 0x17000000>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		cmu_imem: clock-controller@10010000 {
+			compatible = "axis,artpec9-cmu-imem";
+			reg = <0x0 0x10010000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_IMEM_ACLK>,
+				 <&cmu_cmu CLK_DOUT_CMU_IMEM_CA5>,
+				 <&cmu_cmu CLK_DOUT_CMU_IMEM_JPEG>,
+				 <&cmu_cmu CLK_DOUT_CMU_IMEM_SSS>;
+			clock-names = "fin_pll", "aclk", "ca5", "jpeg", "sss";
+		};
+
+		timer@10040000 {
+			compatible = "axis,artpec9-mct", "samsung,exynos4210-mct";
+			reg = <0x0 0x10040000 0x0 0x1000>;
+			clocks = <&fin_pll>, <&cmu_imem CLK_GOUT_IMEM_MCT0_PCLK>;
+			clock-names = "fin_pll", "mct";
+			interrupts = <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		gic: interrupt-controller@10400000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x10400000 0x0 0x00040000>,
+			      <0x0 0x10440000 0x0 0x000c0000>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			redistributor-stride = <0x0 0x20000>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		cmu_cpucl: clock-controller@12810000 {
+			compatible = "axis,artpec9-cmu-cpucl";
+			reg = <0x0 0x12810000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_CPUCL_SWITCH>;
+			clock-names = "fin_pll", "switch";
+		};
+
+		cmu_cmu: clock-controller@12c00000 {
+			compatible = "axis,artpec9-cmu-cmu";
+			reg = <0x0 0x12c00000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>;
+			clock-names = "fin_pll";
+		};
+
+		cmu_core: clock-controller@12c10000 {
+			compatible = "axis,artpec9-cmu-core";
+			reg = <0x0 0x12c10000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_CORE_MAIN>;
+			clock-names = "fin_pll", "main";
+		};
+
+		cmu_bus: clock-controller@13410000 {
+			compatible = "axis,artpec9-cmu-bus";
+			reg = <0x0 0x13410000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_BUS>;
+			clock-names = "fin_pll", "bus";
+		};
+
+		cmu_peri: clock-controller@14010000 {
+			compatible = "axis,artpec9-cmu-peri";
+			reg = <0x0 0x14010000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_PERI_IP>,
+				 <&cmu_cmu CLK_DOUT_CMU_PERI_DISP>;
+			clock-names = "fin_pll", "ip", "disp";
+		};
+
+		pinctrl_peric: pinctrl@141f0000 {
+			compatible = "axis,artpec9-pinctrl";
+			reg = <0x0 0x141f0000 0x0 0x1000>;
+			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		cmu_fsys0: clock-controller@14410000 {
+			compatible = "axis,artpec9-cmu-fsys0";
+			reg = <0x0 0x14410000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_FSYS0_BUS>,
+				 <&cmu_cmu CLK_DOUT_CMU_FSYS0_IP>;
+			clock-names = "fin_pll", "bus", "ip";
+		};
+
+		pinctrl_fsys0: pinctrl@14430000 {
+			compatible = "axis,artpec9-pinctrl";
+			reg = <0x0 0x14430000 0x0 0x1000>;
+			interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		cmu_fsys1: clock-controller@14c10000 {
+			compatible = "axis,artpec9-cmu-fsys1";
+			reg = <0x0 0x14c10000 0x0 0x4000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&cmu_cmu CLK_DOUT_CMU_FSYS1_SCAN0>,
+				 <&cmu_cmu CLK_DOUT_CMU_FSYS1_SCAN1>,
+				 <&cmu_cmu CLK_DOUT_CMU_FSYS1_BUS>;
+			clock-names = "fin_pll", "scan0", "scan1", "bus";
+		};
+
+		pinctrl_fsys1: pinctrl@14c30000 {
+			compatible = "axis,artpec9-pinctrl";
+			reg = <0x0 0x14c30000 0x0 0x1000>;
+			interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pmu_system_controller: system-controller@14c40000 {
+			compatible = "axis,artpec9-pmu", "samsung,exynos7-pmu", "syscon";
+			reg = <0x0 0x14c40000 0x0 0x10000>;
+		};
+
+		serial_0: serial@14c70000 {
+			compatible = "axis,artpec9-uart", "samsung,exynos8895-uart";
+			reg = <0x0 0x14c70000 0x0 0x100>;
+			clocks = <&cmu_fsys1 CLK_GOUT_FSYS1_UART0_PCLK>,
+				 <&cmu_fsys1 CLK_GOUT_FSYS1_UART0_SCLK_UART>;
+			clock-names = "uart", "clk_uart_baud0";
+			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&serial0_bus>;
+			samsung,uart-fifosize = <64>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
index 6f40ca4350ed7..0888cd2faca4a 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
@@ -158,9 +158,6 @@ &gpu {
 };
 
 &hsi2c0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	status = "okay";
 
 	pmic@66 {
@@ -394,9 +391,6 @@ vdd_ldo35: ldo35 {
 };
 
 &i2c5 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
@@ -428,9 +422,6 @@ proximity@48 {
 };
 
 &i2c6 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
new file mode 100644
index 0000000000000..36c327668db51
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
@@ -0,0 +1,523 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung Galaxy J5 (2017) (j5y17lte) device tree source
+ *
+ * Copyright (c) 2024 Andras Sebok <sebokandris2009@gmail.com>
+ */
+
+/dts-v1/;
+#include "exynos7870.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	model = "Samsung Galaxy J5 (2017)";
+	compatible = "samsung,j5y17lte", "samsung,exynos7870";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &mmc0;
+		mmc1 = &mmc1;
+		mmc2 = &mmc2;
+		serial0 = &serial0;
+		serial1 = &serial1;
+		serial2 = &serial2;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges;
+
+		stdout-path = &serial2;
+
+		framebuffer@67000000 {
+			compatible = "simple-framebuffer";
+			reg = <0x0 0x67000000 (720 * 1280 * 4)>;
+			width = <720>;
+			height = <1280>;
+			stride = <(720 * 4)>;
+			format = "a8r8g8b8";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_power &key_voldown &key_volup>;
+
+		key-home {
+			interrupt-parent = <&gpa1>;
+			linux,code = <KEY_HOMEPAGE>;
+			label = "gpio-keys: KEY_HOMEPAGE";
+			gpios = <&gpa1 7 GPIO_ACTIVE_LOW>;
+		};
+
+		key-power {
+			interrupt-parent = <&gpa0>;
+			linux,code = <KEY_POWER>;
+			label = "gpio-keys: KEY_POWER";
+			gpios = <&gpa0 0 GPIO_ACTIVE_LOW>;
+		};
+
+		key-voldown {
+			interrupt-parent = <&gpa2>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			label = "gpio-keys: KEY_VOLUMEDOWN";
+			gpios = <&gpa2 1 GPIO_ACTIVE_LOW>;
+		};
+
+		key-volup {
+			interrupt-parent = <&gpa2>;
+			linux,code = <KEY_VOLUMEUP>;
+			label = "gpio-keys: KEY_VOLUMEUP";
+			gpios = <&gpa2 0 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0x3e400000>,
+		      <0x0 0x80000000 0x40000000>;
+	};
+
+	pwrseq_mmc1: pwrseq-mmc1 {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
+	};
+
+	vdd_fixed_mmc2: regulator-fixed-mmc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_fixed_mmc2";
+		regulator-max-microvolt = <2800000>;
+		regulator-min-microvolt = <2800000>;
+
+		gpio = <&gpc0 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges;
+
+		ramoops@46e00000 {
+			compatible = "ramoops";
+			reg = <0x0 0x46e00000 0x8000>;
+			console-size = <0x4000>;
+			pmsg-size = <0x4000>;
+		};
+
+		framebuffer@67000000 {
+			reg = <0x0 0x67000000 (720 * 1280 * 4)>;
+			no-map;
+		};
+	};
+
+	vibrator {
+		compatible = "regulator-haptic";
+		haptic-supply = <&vdd_ldo32>;
+		min-microvolt = <3300000>;
+		max-microvolt = <3300000>;
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&hsi2c0 {
+	status = "okay";
+
+	pmic@66 {
+		compatible = "samsung,s2mpu05-pmic";
+		reg = <0x66>;
+
+		interrupt-parent = <&gpa0>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_irq>;
+
+		regulators {
+			vdd_buck1: buck1 {
+				regulator-name = "vdd_buck1";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_buck2: buck2 {
+				regulator-name = "vdd_buck2";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_buck3: buck3 {
+				regulator-name = "vdd_buck3";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_buck4: buck4 {
+				regulator-name = "vdd_buck4";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_buck5: buck5 {
+				regulator-name = "vdd_buck5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo1: ldo1 {
+				regulator-name = "vdd_ldo1";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo2: ldo2 {
+				regulator-name = "vdd_ldo2";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-ramp-delay = <12000>;
+			};
+
+			vdd_ldo3: ldo3 {
+				regulator-name = "vdd_ldo3";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo4: ldo4 {
+				regulator-name = "vdd_ldo4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo5: ldo5 {
+				regulator-name = "vdd_ldo5";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo6: ldo6 {
+				regulator-name = "vdd_ldo6";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo7: ldo7 {
+				regulator-name = "vdd_ldo7";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo8: ldo8 {
+				regulator-name = "vdd_ldo8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3375000>;
+				regulator-ramp-delay = <12000>;
+			};
+
+			vdd_ldo9: ldo9 {
+				regulator-name = "vdd_ldo9";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo10: ldo10 {
+				regulator-name = "vdd_ldo10";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo25: ldo25 {
+				regulator-name = "vdd_ldo25";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo26: ldo26 {
+				regulator-name = "vdd_ldo26";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3375000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo27: ldo27 {
+				regulator-name = "vdd_ldo27";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo29: ldo29 {
+				regulator-name = "vdd_ldo29";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo30: ldo30 {
+				regulator-name = "vdd_ldo30";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo31: ldo31 {
+				regulator-name = "vdd_ldo31";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-ramp-delay = <12000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_ldo32: ldo32 {
+				regulator-name = "vdd_ldo32";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <12000>;
+			};
+
+			vdd_ldo33: ldo33 {
+				regulator-name = "vdd_ldo33";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <12000>;
+			};
+
+			vdd_ldo34: ldo34 {
+				regulator-name = "vdd_ldo34";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			vdd_ldo35: ldo35 {
+				regulator-name = "vdd_ldo35";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&i2c2 {
+	samsung,i2c-sda-delay = <100>;
+	samsung,i2c-max-bus-freq = <400000>;
+
+	status = "okay";
+
+	touchscreen@50 {
+		compatible = "imagis,ist3038h";
+		reg = <0x50>;
+
+		interrupt-parent = <&gpa0>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&touch_irq>;
+
+		touchscreen-size-x = <720>;
+		touchscreen-size-y = <1280>;
+
+		vdd-supply = <&vdd_ldo34>;
+	};
+};
+
+&mmc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+
+	vmmc-supply = <&vdd_ldo26>;
+	vqmmc-supply = <&vdd_ldo27>;
+
+	fifo-depth = <64>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <2 4>;
+	non-removable;
+
+	status = "okay";
+};
+
+&mmc1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+
+	mmc-pwrseq = <&pwrseq_mmc1>;
+
+	bus-width = <4>;
+	fifo-depth = <64>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+	non-removable;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+
+	status = "okay";
+
+	wifi@0 {
+		compatible = "brcm,bcm43455-fmac", "brcm,bcm4329-fmac";
+		reg = <0x0>;
+
+		interrupt-names = "host-wake";
+		interrupt-parent = <&gpa2>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&oscclk {
+	clock-frequency = <26000000>;
+};
+
+&pinctrl_alive {
+	accel_irq: accel-irq-pins {
+		samsung,pins = "gpa2-3";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	dwmmc2_irq: dwmmc2-irq-pins {
+		samsung,pins = "gpa0-1";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	fuel_irq: fuel-irq-pins {
+		samsung,pins = "gpa0-3";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	hall_irq: hall-irq-pins {
+		samsung,pins = "gpa1-3";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_power: key-power-pins {
+		samsung,pins = "gpa0-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_voldown: key-voldown-pins {
+		samsung,pins = "gpa2-1";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_volup: key-volup-pins {
+		samsung,pins = "gpa2-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	pmic_irq: pmic-irq-pins {
+		samsung,pins = "gpa0-2";
+		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR4>;
+	};
+
+	touch_irq: touch-irq-pins {
+		samsung,pins = "gpa0-6";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	wlan_hostwake: wlan-hostwake-pins {
+		samsung,pins = "gpa2-2";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+	};
+};
+
+&pinctrl_top {
+	wlan_enable: wlan-enable-pins {
+		samsung,pins = "gpd3-6";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-con-pdn = <EXYNOS_PIN_PDN_PREV>;
+		samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR4>;
+		samsung,pin-val = <0>;
+	};
+};
+
+&serial2 {
+	status = "okay";
+};
+
+&usbdrd {
+	vdd33-supply = <&vdd_ldo8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
index 09f2367cfec93..de30d0970336f 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
@@ -383,9 +383,6 @@ vdd_ldo35: ldo35 {
 };
 
 &i2c5 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
@@ -409,9 +406,6 @@ accelerometer@1d {
 };
 
 &i2c6 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
new file mode 100644
index 0000000000000..079b2b581307c
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
@@ -0,0 +1,494 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung Galaxy J7 (2016) (j7xelte) device tree source
+ *
+ * Copyright (c) 2025 Rayan Marzouk <rayanmarzouk743@gmail.com>
+ */
+
+/dts-v1/;
+#include "exynos7870.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	model = "Samsung Galaxy J7 (2016)";
+	compatible = "samsung,j7xelte", "samsung,exynos7870";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &mmc0;
+		mmc1 = &mmc1;
+		mmc2 = &mmc2;
+		serial0 = &serial0;
+		serial1 = &serial1;
+		serial2 = &serial2;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges;
+
+		stdout-path = &serial2;
+
+		framebuffer@67000000 {
+			compatible = "simple-framebuffer";
+			reg = <0x0 0x67000000 (720 * 1280 * 4)>;
+			width = <720>;
+			height = <1280>;
+			stride = <(720 * 4)>;
+			format = "a8r8g8b8";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		label = "GPIO Keys";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_home &key_power &key_voldown &key_volup>;
+
+		key-home {
+			label = "Home Key";
+			gpios = <&gpa1 7 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOMEPAGE>;
+		};
+
+		key-power {
+			label = "Power Key";
+			gpios = <&gpa0 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
+		};
+
+		key-voldown {
+			label = "Volume Down Key";
+			gpios = <&gpa2 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+
+		key-volup {
+			label = "Volume Up Key";
+			gpios = <&gpa2 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0x3e400000>,
+		      <0x0 0x80000000 0x40000000>;
+	};
+
+	pwrseq_mmc1: pwrseq-mmc1 {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges;
+
+		ramoops@46e00000 {
+			compatible = "ramoops";
+			reg = <0x0 0x46e00000 0x8000>;
+			console-size = <0x4000>;
+			pmsg-size = <0x4000>;
+		};
+
+		cont_splash_mem: framebuffer@67000000 {
+			reg = <0x0 0x67000000 0x00384000>;
+			no-map;
+		};
+	};
+
+	vdd_fixed_mmc2: regulator-fixed-mmc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_fixed_mmc2";
+		regulator-max-microvolt = <2800000>;
+		regulator-min-microvolt = <2800000>;
+		gpio = <&gpc0 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vibrator {
+		compatible = "regulator-haptic";
+		haptic-supply = <&vdd_ldo32>;
+		min-microvolt = <3300000>;
+		max-microvolt = <3300000>;
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&hsi2c0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+
+	pmic@66 {
+		compatible = "samsung,s2mpu05-pmic";
+		reg = <0x66>;
+		interrupt-parent = <&gpa0>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_irq>;
+
+		regulators {
+			vdd_buck1: buck1 {
+				regulator-name = "vdd_buck1";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_buck2: buck2 {
+				regulator-name = "vdd_buck2";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_buck3: buck3 {
+				regulator-name = "vdd_buck3";
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_buck4: buck4 {
+				regulator-name = "vdd_buck4";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_buck5: buck5 {
+				regulator-name = "vdd_buck5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo1: ldo1 {
+				regulator-name = "vdd_ldo1";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo2: ldo2 {
+				regulator-name = "vdd_ldo2";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			vdd_ldo3: ldo3 {
+				regulator-name = "vdd_ldo3";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo4: ldo4 {
+				regulator-name = "vdd_ldo4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo5: ldo5 {
+				regulator-name = "vdd_ldo5";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo6: ldo6 {
+				regulator-name = "vdd_ldo6";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo7: ldo7 {
+				regulator-name = "vdd_ldo7";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo8: ldo8 {
+				regulator-name = "vdd_ldo8";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			vdd_ldo9: ldo9 {
+				regulator-name = "vdd_ldo9";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo10: ldo10 {
+				regulator-name = "vdd_ldo10";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo25: ldo25 {
+				regulator-name = "vdd_ldo25";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo26: ldo26 {
+				regulator-name = "vdd_ldo26";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3375000>;
+			};
+
+			vdd_ldo27: ldo27 {
+				regulator-name = "vdd_ldo27";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <2375000>;
+			};
+
+			vdd_ldo29: ldo29 {
+				regulator-name = "vdd_ldo29";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo30: ldo30 {
+				regulator-name = "vdd_ldo30";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo31: ldo31 {
+				regulator-name = "vdd_ldo31";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo32: ldo32 {
+				regulator-name = "vdd_ldo32";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			vdd_ldo33: ldo33 {
+				regulator-name = "vdd_ldo33";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo34: ldo34 {
+				regulator-name = "vdd_ldo34";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			vdd_ldo35: ldo35 {
+				regulator-name = "vdd_ldo35";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+
+&i2c1 {
+	samsung,i2c-sda-delay = <100>;
+	samsung,i2c-max-bus-freq = <400000>;
+
+	status = "okay";
+
+	touchscreen@48 {
+		compatible = "melfas,mip4_ts";
+		reg = <0x48>;
+
+		interrupt-parent = <&gpc3>;
+		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&touch_irq>;
+	};
+};
+
+&mmc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+
+	vmmc-supply = <&vdd_ldo26>;
+	vqmmc-supply = <&vdd_ldo27>;
+
+	fifo-depth = <64>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <2 4>;
+	non-removable;
+
+	status = "okay";
+};
+
+&mmc1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+
+	mmc-pwrseq = <&pwrseq_mmc1>;
+
+	bus-width = <4>;
+	fifo-depth = <64>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+	non-removable;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+
+	status = "okay";
+
+	wifi@1 {
+		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
+		reg = <0x1>;
+
+		interrupt-names = "host-wake";
+		interrupt-parent = <&gpa2>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&mmc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4 &dwmmc2_irq>;
+
+	vmmc-supply = <&vdd_fixed_mmc2>;
+	vqmmc-supply = <&vdd_ldo2>;
+
+	bus-width = <4>;
+	card-detect-delay = <200>;
+	fifo-depth = <64>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	broken-cd;
+	disable-wp;
+
+	status = "okay";
+};
+
+&oscclk {
+	clock-frequency = <26000000>;
+};
+
+&pinctrl_alive {
+	dwmmc2_irq: dwmmc2-irq-pins {
+		samsung,pins = "gpa0-1";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	fuel_irq: fuel-irq-pins {
+		samsung,pins = "gpa0-3";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_home: key-home-pins {
+		samsung,pins = "gpa1-7";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_power: key-power-pins {
+		samsung,pins = "gpa0-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_voldown: key-voldown-pins {
+		samsung,pins = "gpa2-1";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	key_volup: key-volup-pins {
+		samsung,pins = "gpa2-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+
+	pmic_irq: pmic-irq-pins {
+		samsung,pins = "gpa0-2";
+		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR3>;
+	};
+};
+
+&pinctrl_touch {
+	touch_irq: touch-irq-pins {
+		samsung,pins = "gpc3-2";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+	};
+};
+
+&serial2 {
+	status = "okay";
+};
+
+&usbdrd {
+	vdd33-supply = <&vdd_ldo8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
index 29e124c72e9d2..29be4764f84d3 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
@@ -398,9 +398,6 @@ vdd_ldo35: ldo35 {
 };
 
 &i2c1 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
@@ -438,9 +435,6 @@ rmi4-f12@12 {
 };
 
 &i2c7 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
@@ -464,9 +458,6 @@ accelerometer@1d {
 };
 
 &i2c8 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
 	samsung,i2c-sda-delay = <100>;
 	samsung,i2c-max-bus-freq = <400000>;
 
diff --git a/arch/arm64/boot/dts/exynos/exynos7870.dtsi b/arch/arm64/boot/dts/exynos/exynos7870.dtsi
index 2827e10d69625..19f9ae7835873 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7870.dtsi
@@ -220,6 +220,9 @@ hsi2c0: i2c@10510000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_mif CLK_GOUT_MIF_HSI2C_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -422,6 +425,9 @@ i2c0: i2c@13830000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C0_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -437,6 +443,9 @@ i2c1: i2c@13840000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C1_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -452,6 +461,9 @@ i2c2: i2c@13850000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C2_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -467,6 +479,9 @@ i2c3: i2c@13860000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C3_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -482,6 +497,9 @@ i2c4: i2c@13870000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C4_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -497,6 +515,9 @@ i2c5: i2c@13880000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C5_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -512,6 +533,9 @@ i2c6: i2c@13890000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C6_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -527,6 +551,9 @@ hsi2c1: i2c@138a0000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C1_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -542,6 +569,9 @@ hsi2c2: i2c@138b0000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C2_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -557,6 +587,9 @@ hsi2c3: i2c@138c0000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C3_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -572,6 +605,9 @@ i2c7: i2c@138d0000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C7_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -587,6 +623,9 @@ i2c8: i2c@138e0000 {
 			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C8_PCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -602,6 +641,9 @@ hsi2c4: i2c@138f0000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C4_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -617,6 +659,9 @@ hsi2c5: i2c@13950000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C5_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -632,6 +677,9 @@ hsi2c6: i2c@13960000 {
 			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C6_IPCLK>;
 
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/exynos/exynos8895-dreamlte.dts b/arch/arm64/boot/dts/exynos/exynos8895-dreamlte.dts
index 61e064af3337d..305dc72f93d84 100644
--- a/arch/arm64/boot/dts/exynos/exynos8895-dreamlte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos8895-dreamlte.dts
@@ -103,8 +103,6 @@ reg_placeholder: regulator-0 {
 };
 
 &hsi2c_23 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	status = "okay";
 
 	touchscreen@48 {
diff --git a/arch/arm64/boot/dts/exynos/exynos8895.dtsi b/arch/arm64/boot/dts/exynos/exynos8895.dtsi
index f92d2a8a20a2d..ff114cd6c9eaa 100644
--- a/arch/arm64/boot/dts/exynos/exynos8895.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos8895.dtsi
@@ -266,6 +266,10 @@ hsi2c_5: i2c@0 {
 				interrupts = <GIC_SPI 364 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c5_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -305,6 +309,10 @@ hsi2c_6: i2c@10000 {
 				interrupts = <GIC_SPI 365 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c6_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -328,6 +336,10 @@ hsi2c_7: i2c@0 {
 				interrupts = <GIC_SPI 368 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c5_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -367,6 +379,10 @@ hsi2c_8: i2c@10000 {
 				interrupts = <GIC_SPI 369 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c8_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -390,6 +406,10 @@ hsi2c_9: i2c@0 {
 				interrupts = <GIC_SPI 372 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c9_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -429,6 +449,10 @@ hsi2c_10: i2c@10000 {
 				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c10_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -452,6 +476,10 @@ hsi2c_11: i2c@0 {
 				interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c11_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -491,6 +519,10 @@ hsi2c_12: i2c@10000 {
 				interrupts = <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c12_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -565,6 +597,10 @@ hsi2c_13: i2c@0 {
 				interrupts = <GIC_SPI 390 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c13_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -604,6 +640,10 @@ hsi2c_14: i2c@10000 {
 				interrupts = <GIC_SPI 391 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c14_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -627,6 +667,10 @@ hsi2c_15: i2c@0 {
 				interrupts = <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c15_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -666,6 +710,10 @@ hsi2c_16: i2c@10000 {
 				interrupts = <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c16_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -689,6 +737,10 @@ hsi2c_17: i2c@0 {
 				interrupts = <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c17_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -728,6 +780,10 @@ hsi2c_18: i2c@10000 {
 				interrupts = <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c18_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -751,6 +807,10 @@ hsi2c_19: i2c@0 {
 				interrupts = <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c19_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -790,6 +850,10 @@ hsi2c_20: i2c@10000 {
 				interrupts = <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c20_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -813,6 +877,10 @@ hsi2c_21: i2c@0 {
 				interrupts = <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c21_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -852,6 +920,10 @@ hsi2c_22: i2c@10000 {
 				interrupts = <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c22_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -875,6 +947,10 @@ hsi2c_23: i2c@0 {
 				interrupts = <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c23_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -914,6 +990,10 @@ hsi2c_24: i2c@10000 {
 				interrupts = <GIC_SPI 411 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c24_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -937,6 +1017,10 @@ hsi2c_25: i2c@0 {
 				interrupts = <GIC_SPI 414 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c25_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -976,6 +1060,10 @@ hsi2c_26: i2c@10000 {
 				interrupts = <GIC_SPI 415 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c26_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -999,6 +1087,10 @@ hsi2c_27: i2c@0 {
 				interrupts = <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c27_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -1038,6 +1130,10 @@ hsi2c_28: i2c@10000 {
 				interrupts = <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c28_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -1061,6 +1157,10 @@ hsi2c_29: i2c@0 {
 				interrupts = <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c29_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -1100,6 +1200,10 @@ hsi2c_30: i2c@10000 {
 				interrupts = <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c30_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -1123,6 +1227,10 @@ hsi2c_31: i2c@0 {
 				interrupts = <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c31_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 
@@ -1162,6 +1270,10 @@ hsi2c_32: i2c@10000 {
 				interrupts = <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>;
 				pinctrl-0 = <&hsi2c32_bus>;
 				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
 				status = "disabled";
 			};
 		};
@@ -1180,6 +1292,10 @@ hsi2c_1: i2c@10990000 {
 			interrupts = <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&hsi2c1_bus>;
 			pinctrl-names = "default";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -1191,6 +1307,10 @@ hsi2c_2: i2c@109a0000 {
 			interrupts = <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&hsi2c2_bus>;
 			pinctrl-names = "default";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -1202,6 +1322,10 @@ hsi2c_3: i2c@109b0000 {
 			interrupts = <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&hsi2c3_bus>;
 			pinctrl-names = "default";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
@@ -1213,6 +1337,10 @@ hsi2c_4: i2c@109c0000 {
 			interrupts = <GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&hsi2c4_bus>;
 			pinctrl-names = "default";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
index 02bf2ca52fdce..0bf7c4cb98461 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
@@ -1473,6 +1473,19 @@ cmu_mfd: clock-controller@19e00000 {
 				      "noc";
 		};
 
+		cmu_g3d: clock-controller@1a000000 {
+			compatible = "samsung,exynosautov920-cmu-g3d";
+			reg = <0x1a000000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&xtcxo>,
+				 <&cmu_top DOUT_CLKCMU_G3D_SWITCH>,
+				 <&cmu_top DOUT_CLKCMU_G3D_NOCP>;
+			clock-names = "oscclk",
+				      "switch",
+				      "nocp";
+		};
+
 		pinctrl_aud: pinctrl@1a460000 {
 			compatible = "samsung,exynosautov920-pinctrl";
 			reg = <0x1a460000 0x10000>;
diff --git a/arch/arm64/boot/dts/exynos/google/gs101-oriole.dts b/arch/arm64/boot/dts/exynos/google/gs101-oriole.dts
index 8df42bedbc036..36721adcaa239 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101-oriole.dts
+++ b/arch/arm64/boot/dts/exynos/google/gs101-oriole.dts
@@ -8,6 +8,7 @@
 
 /dts-v1/;
 
+#include <dt-bindings/regulator/samsung,s2mpg10-regulator.h>
 #include "gs101-pixel-common.dtsi"
 
 / {
@@ -15,6 +16,27 @@ / {
 	compatible = "google,gs101-oriole", "google,gs101";
 };
 
+&acpm_ipc {
+	pmic-1 {
+		regulators {
+			ldo14m {
+				/* PLL */
+				regulator-name = "avdd18_tcxo";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_TCXO_ON>;
+			};
+
+			ldo31m {
+				regulator-name = "nfc";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+		};
+	};
+};
+
 &cont_splash_mem {
 	reg = <0x0 0xfac00000 (1080 * 2400 * 4)>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi b/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
index 93892adaa6794..5227cd4e314b8 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
+++ b/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
@@ -10,6 +10,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/regulator/samsung,s2mpg10-regulator.h>
 #include <dt-bindings/usb/pd.h>
 #include "gs101-pinctrl.h"
 #include "gs101.dtsi"
@@ -28,6 +29,8 @@ chosen {
 		framebuffer0: framebuffer-0 {
 			compatible = "simple-framebuffer";
 			memory-region = <&cont_splash_mem>;
+			vci-supply = <&s2mpg10_ldo22m>;
+			vddi-supply = <&s2mpg11_bucka>;
 			/* format properties to be added by actual board */
 			status = "disabled";
 		};
@@ -101,7 +104,7 @@ cont_splash_mem: splash@fac00000 {
 };
 
 &acpm_ipc {
-	pmic {
+	pmic-1 {
 		compatible = "samsung,s2mpg10-pmic";
 		interrupts-extended = <&gpa0 6 IRQ_TYPE_LEVEL_LOW>;
 		pinctrl-names = "default";
@@ -109,6 +112,22 @@ pmic {
 		system-power-controller;
 		wakeup-source;
 
+		vinl1m-supply = <&s2mpg11_buck3s>;
+		vinl2m-supply = <&s2mpg11_buck3s>;
+		vinl3m-supply = <&s2mpg10_buck8m>;
+		vinl4m-supply = <&s2mpg10_buck9m>;
+		vinl5m-supply = <&s2mpg10_buck9m>;
+		vinl6m-supply = <&s2mpg10_buck9m>;
+		vinl7m-supply = <&s2mpg11_buck6s>;
+		vinl8m-supply = <&s2mpg11_buck6s>;
+		vinl9m-supply = <&s2mpg11_buck7s>;
+		vinl10m-supply = <&s2mpg11_buck7s>;
+		vinl11m-supply = <&s2mpg11_buck7s>;
+		vinl12m-supply = <&s2mpg11_bucka>;
+		vinl13m-supply = <&s2mpg11_bucka>;
+		vinl14m-supply = <&s2mpg11_buckboost>;
+		vinl15m-supply = <&s2mpg11_buckboost>;
+
 		clocks {
 			compatible = "samsung,s2mpg10-clk";
 			#clock-cells = <1>;
@@ -117,6 +136,489 @@ clocks {
 		};
 
 		regulators {
+			buck1m {
+				regulator-name = "vdd_mif";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			buck2m {
+				regulator-name = "vdd_cpucl2";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_CPUCL2_EN2>;
+			};
+
+			buck3m {
+				regulator-name = "vdd_cpucl1";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_CPUCL1_EN2>;
+			};
+
+			buck4m {
+				regulator-name = "vdd_cpucl0";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN>;
+			};
+
+			buck5m {
+				regulator-name = "vdd_int";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			buck6m {
+				regulator-name = "vdd_cpucl2_m";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_CPUCL2_EN2>;
+			};
+
+			buck7m {
+				/* GPU */
+				regulator-name = "vdd_int_m";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			s2mpg10_buck8m: buck8m {
+				regulator-name = "lldo2";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+			};
+
+			s2mpg10_buck9m: buck9m {
+				regulator-name = "lldo3";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+			};
+
+			buck10m {
+				regulator-name = "vdd_tpu";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_TPU_EN>;
+			};
+
+			ldo1m {
+				/* ALIVE, AOC PLL */
+				regulator-name = "vdd_l1m_alive";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+			};
+
+			ldo2m {
+				/* lots, DDR */
+				regulator-name = "vdd_l2m_alive";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+				regulator-always-on;
+			};
+
+			ldo3m {
+				/* AVDD: MIPI CSI & DSI, PLL: CPUCL SHARED TPU UFS */
+				regulator-name = "ldo3m";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			ldo4m {
+				/* AVDD: MIPI CSI & DSI, UFS, OTP, TS_SUB, TS_TOP, XOTP */
+				regulator-name = "ldo4m";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			ldo5m {
+				/* VDD: ADD, AVDD: TCXO & TCXO_FAR */
+				regulator-name = "avdd075_tcxo";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_TCXO_ON>;
+			};
+
+			ldo6m {
+				/* PLL CPUCL & MIFx, UFS clk, MIPI DSI */
+				regulator-name = "vdd_pll";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN_MIF>;
+			};
+
+			ldo7m {
+				/* IO (HSI (USB)) */
+				regulator-name = "vdd_hsi";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <6250>;
+				/*
+				 * TODO: link to HSI power domain, without this,
+				 * Linux hangs during USB access.
+				 */
+				regulator-always-on;
+			};
+
+			ldo8m {
+				regulator-name = "vdd085_usb";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+			};
+
+			ldo9m {
+				regulator-name = "vdd18_usb";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+				regulator-always-on;
+			};
+
+			ldo10m {
+				regulator-name = "vdd33_usb";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3350000>;
+				regulator-always-on;
+			};
+
+			ldo11m {
+				regulator-name = "vdd_cpucl1_m";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_CPUCL1_EN2>;
+			};
+
+			ldo12m {
+				regulator-name = "vdd_cpucl0_m";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_PWREN>;
+			};
+
+			ldo13m {
+				regulator-name = "vdd_tpu_m";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_TPU_EN>;
+			};
+
+			/* ldo14m is board specific */
+
+			ldo15m {
+				regulator-name = "vdd_slc_m";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+			};
+
+			ldo16m {
+				regulator-name = "vdd085_pcie0";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+			};
+
+			ldo17m {
+				regulator-name = "vdd085_pcie1";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+			};
+
+			ldo18m {
+				regulator-name = "vdd18_pcie0";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			ldo19m {
+				regulator-name = "vdd18_pcie1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			ldo20m {
+				/* DMIC, memory power */
+				regulator-name = "vddq_aoc_pdm";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG10_EXTCTRL_LDO20M_EN2>;
+			};
+
+			ldo21m {
+				/* Dauntless  */
+				regulator-name = "vdd_dtls";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			s2mpg10_ldo22m: ldo22m {
+				/* display */
+				regulator-name = "vci_disp";
+				regulator-min-microvolt = <3025000>;
+				regulator-max-microvolt = <3025000>;
+			};
+
+			/* ldo23m & ldo24m are unused */
+
+			ldo25m {
+				/* touch */
+				regulator-name = "dvdd_tsp";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			ldo26m {
+				/* touch */
+				regulator-name = "avdd_ts";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			ldo27m {
+				/* under-display fingerprint scanner */
+				regulator-name = "avdd_udfps";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			/*
+			 *ldo28m .. ldo30m are unused, ldo31m is board specific
+			 */
+		};
+	};
+
+	pmic-2 {
+		compatible = "samsung,s2mpg11-pmic";
+		interrupts-extended = <&gpa0 7 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sub_pmic_int>;
+		wakeup-source;
+
+		vinl1s-supply = <&s2mpg10_buck8m>;
+		vinl2s-supply = <&s2mpg11_buck6s>;
+		vinl3s-supply = <&s2mpg11_buck7s>;
+		vinl4s-supply = <&s2mpg11_buck7s>;
+		vinl5s-supply = <&s2mpg11_buckboost>;
+		vinl6s-supply = <&s2mpg11_buckboost>;
+		vinbd-supply = <&s2mpg11_buckboost>;
+
+		regulators {
+			buck1s {
+				/* multimedia */
+				regulator-name = "vdd_cam";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_PWREN_MIF>;
+			};
+
+			buck2s {
+				regulator-name = "vdd_g3d";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_G3D_EN>;
+			};
+
+			s2mpg11_buck3s: buck3s {
+				regulator-name = "lldo1";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+			};
+
+			buck4s {
+				/* DDR */
+				regulator-name = "vdd2h_mem";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+			};
+
+			buck5s {
+				/* DDR */
+				regulator-name = "vddq_mem";
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_PWREN_MIF>;
+			};
+
+			s2mpg11_buck6s: buck6s {
+				regulator-name = "lldo4";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-ramp-delay = <6250>;
+			};
+
+			s2mpg11_buck7s: buck7s {
+				regulator-name = "mldo";
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <2200000>;
+				regulator-ramp-delay = <6250>;
+			};
+
+			buck8s {
+				regulator-name = "vdd_g3d_l2";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_G3D_EN>;
+			};
+
+			buck9s {
+				regulator-name = "vdd_aoc";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+			};
+
+			buck10s {
+				/* DDR */
+				regulator-name = "vdd2l_mem";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+			};
+
+			buckd {
+				regulator-name = "vcc_ufs";
+				regulator-min-microvolt = <2400000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_UFS_EN>;
+			};
+
+			s2mpg11_bucka: bucka {
+				/* lots, IO */
+				regulator-name = "bucka";
+				regulator-min-microvolt = <1700000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+			};
+
+			s2mpg11_buckboost: buckboost {
+				regulator-name = "buckboost";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3600000>;
+			};
+
+			ldo1s {
+				regulator-name = "vdd_g3d_m";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_G3D_EN>;
+			};
+
+			ldo2s {
+				regulator-name = "vdd_aoc_ret";
+				regulator-min-microvolt = <450000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-always-on;
+			};
+
+			/* ldo3s & ldo5s are unused, ldo4s is board specific */
+
+			ldo6s {
+				/* sensors */
+				regulator-name = "vdd_prox";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			ldo7s {
+				/* sensors */
+				regulator-name = "vdd_sensors";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			ldo8s {
+				regulator-name = "vccq_ufs";
+				regulator-min-microvolt = <1130400>;
+				regulator-max-microvolt = <1281200>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_UFS_EN>;
+			};
+
+			ldo9s {
+				regulator-name = "vdd_gnss";
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1300000>;
+			};
+
+			ldo10s {
+				regulator-name = "vdd_gnss_rf";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			ldo11s {
+				regulator-name = "vdd_gnss_aux";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+
+			/* ldo12s is unused */
+
+			ldo13s {
+				regulator-name = "vddq_mmc";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3350000>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_LDO13S_EN>;
+			};
+
+			/* ldo14s is board specific, ldo15s is unused */
 		};
 	};
 };
@@ -225,6 +727,17 @@ usbc0_role_sw: endpoint {
 		};
 	};
 
+	fuel-gauge@36 {
+		compatible = "maxim,max77759-fg";
+		reg = <0x36>;
+
+		pinctrl-0 = <&if_pmic_fg_int>;
+		pinctrl-names = "default";
+		interrupts-extended = <&gpa9 3 IRQ_TYPE_LEVEL_LOW>;
+
+		shunt-resistor-micro-ohms = <5000>;
+	};
+
 	pmic@66 {
 		compatible = "maxim,max77759";
 		reg = <0x66>;
@@ -317,6 +830,19 @@ pmic_int: pmic-int-pins {
 		samsung,pin-pud = <GS101_PIN_PULL_NONE>;
 	};
 
+	sub_pmic_int: sub-pmic-int-pins {
+		samsung,pins = "gpa0-7";
+		samsung,pin-function = <GS101_PIN_FUNC_EINT>;
+		samsung,pin-pud = <GS101_PIN_PULL_NONE>;
+	};
+
+	if_pmic_fg_int: if-pmic-fg-int-pins {
+		samsung,pins = "gpa9-3";
+		samsung,pin-function = <GS101_PIN_FUNC_EINT>;
+		samsung,pin-pud = <GS101_PIN_PULL_UP>;
+		samsung,pin-drv = <GS101_PIN_DRV_2_5_MA>;
+	};
+
 	key_power: key-power-pins {
 		samsung,pins = "gpa10-1";
 		samsung,pin-function = <GS101_PIN_FUNC_EINT>;
diff --git a/arch/arm64/boot/dts/exynos/google/gs101-raven.dts b/arch/arm64/boot/dts/exynos/google/gs101-raven.dts
index 1e7e6b34b8649..a422542715f27 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101-raven.dts
+++ b/arch/arm64/boot/dts/exynos/google/gs101-raven.dts
@@ -15,6 +15,24 @@ / {
 	compatible = "google,gs101-raven", "google,gs101";
 };
 
+&acpm_ipc {
+	pmic-2 {
+		regulators {
+			ldo4s {
+				regulator-name = "vdd2_uwb";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			ldo14s {
+				regulator-name = "vdd3_uwb";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1950000>;
+			};
+		};
+	};
+};
+
 &cont_splash_mem {
 	reg = <0x0 0xfac00000 (1440 * 3120 * 4)>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 700bab4d3e600..bae24b53bce6d 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -158,6 +158,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-phycore-rpmsg.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-phygate-tauri-l.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-prt8mm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-tqma8mqml-mba8mx.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-tx8m-1610-moduline-iv-306-d.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-tx8m-1610-moduline-mini-111.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw71xx-0x.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw72xx-0x.dtb
@@ -217,12 +219,14 @@ imx8mn-tqma8mqnl-mba8mx-usbotg-dtbs += imx8mn-tqma8mqnl-mba8mx.dtb imx8mn-tqma8m
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-tqma8mqnl-mba8mx-usbotg.dtb
 
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-ab2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-aristainetos3-adpismarc.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-aristainetos3-helios.dtb
 imx8mp-aristainetos3-helios-lvds-dtbs += imx8mp-aristainetos3-helios.dtb imx8mp-aristainetos3-helios-lvds.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-aristainetos3-helios-lvds.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-aristainetos3-proton2s.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-beacon-kit.dtb
+DTC_FLAGS_imx8mp-cubox-m := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-cubox-m.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-data-modul-edm-sbc.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-debix-model-a.dtb
@@ -234,9 +238,13 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-dhcom-picoitx.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-edm-g-wb.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-frdm.dtb
+DTC_FLAGS_imx8mp-hummingboard-mate := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-hummingboard-mate.dtb
+DTC_FLAGS_imx8mp-hummingboard-pro := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-hummingboard-pro.dtb
+DTC_FLAGS_imx8mp-hummingboard-pulse := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-hummingboard-pulse.dtb
+DTC_FLAGS_imx8mp-hummingboard-ripple := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-hummingboard-ripple.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-icore-mx8mp-edimm2.2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-iota2-lumpy.dtb
@@ -406,17 +414,24 @@ dtb-$(CONFIG_ARCH_MXC) += imx8ulp-9x9-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm-s.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-phyboard-segin.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-tqma9131-mba91xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb.dtb
 
 imx93-9x9-qsb-can1-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-can1.dtbo
 imx93-9x9-qsb-i3c-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-i3c.dtbo
+imx93-9x9-qsb-ontat-kd50g21-40nt-a1-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-can1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-i3c.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm.dtb
+
+imx93-11x11-frdm-pixpaper-dtbs += imx93-11x11-frdm.dtb imx93-11x11-frdm-pixpaper.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm-pixpaper.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx93-14x14-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-kontron-bl-osm-s.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-nash.dtb
@@ -425,12 +440,14 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin.dtb
 imx93-phyboard-nash-jtag-dtbs += imx93-phyboard-nash.dtb imx93-phyboard-nash-jtag.dtbo
 imx93-phyboard-nash-peb-wlbt-07-dtbs += imx93-phyboard-nash.dtb imx93-phyboard-nash-peb-wlbt-07.dtbo
 imx93-phyboard-nash-pwm-fan-dtbs += imx93-phyboard-nash.dtb imx93-phyboard-nash-pwm-fan.dtbo
+imx93-phyboard-segin-peb-av-02-dtbs += imx93-phyboard-segin.dtb imx93-phyboard-segin-peb-av-02.dtbo
 imx93-phyboard-segin-peb-eval-01-dtbs += imx93-phyboard-segin.dtb imx93-phyboard-segin-peb-eval-01.dtbo
 imx93-phyboard-segin-peb-wlbt-05-dtbs += imx93-phyboard-segin.dtb imx93-phyboard-segin-peb-wlbt-05.dtbo
 imx93-phycore-rpmsg-dtbs += imx93-phyboard-nash.dtb imx93-phyboard-segin.dtb imx93-phycore-rpmsg.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-nash-jtag.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-nash-peb-wlbt-07.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-nash-pwm-fan.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin-peb-av-02.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin-peb-eval-01.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin-peb-wlbt-05.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phycore-rpmsg.dtb
@@ -439,6 +456,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba91xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-som-symphony.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93w-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx943-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-frdm.dtb
@@ -501,4 +519,5 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-phygate-tauri-l-rs232-rs485.dtb
 dtb-$(CONFIG_ARCH_S32) += s32g274a-evb.dtb
 dtb-$(CONFIG_ARCH_S32) += s32g274a-rdb2.dtb
 dtb-$(CONFIG_ARCH_S32) += s32g399a-rdb3.dtb
+dtb-$(CONFIG_ARCH_S32) += s32n79-rdb.dtb
 dtb-$(CONFIG_ARCH_S32) += s32v234-evb.dtb
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
index 06790255a764a..6f5af37ba9af9 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
@@ -22,10 +22,6 @@ &adc1 {
 	status = "okay";
 };
 
-&amix {
-	status = "okay";
-};
-
 &asrc0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
index 7022de46b8bff..93f485140b20c 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
@@ -21,6 +21,7 @@ led-1 {
 			color = <LED_COLOR_ID_GREEN>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
 			gpios = <&lsio_gpio5 27 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -29,6 +30,7 @@ led-2 {
 			color = <LED_COLOR_ID_RED>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
 			gpios = <&lsio_gpio5 29 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -37,6 +39,7 @@ led-3 {
 			color = <LED_COLOR_ID_GREEN>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <2>;
 			gpios = <&lsio_gpio5 20 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -45,6 +48,7 @@ led-4 {
 			color = <LED_COLOR_ID_RED>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <2>;
 			gpios = <&lsio_gpio5 21 GPIO_ACTIVE_HIGH>;
 		};
 	};
@@ -62,10 +66,6 @@ &adc1 {
 	status = "okay";
 };
 
-&amix {
-	status = "okay";
-};
-
 &asrc0 {
 	status = "okay";
 };
@@ -89,8 +89,6 @@ &flexcan2 {
 	status = "okay";
 };
 
-/* TODO: GPU */
-
 /* Apalis I2C1 */
 &i2c2 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
index 12732ed7f8119..5c86bcee55fb7 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
@@ -21,6 +21,7 @@ led-1 {
 			color = <LED_COLOR_ID_GREEN>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
 			gpios = <&lsio_gpio5 27 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -29,6 +30,7 @@ led-2 {
 			color = <LED_COLOR_ID_RED>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
 			gpios = <&lsio_gpio5 29 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -37,6 +39,7 @@ led-3 {
 			color = <LED_COLOR_ID_GREEN>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <2>;
 			gpios = <&lsio_gpio5 20 GPIO_ACTIVE_HIGH>;
 		};
 
@@ -45,6 +48,7 @@ led-4 {
 			color = <LED_COLOR_ID_RED>;
 			default-state = "off";
 			function = LED_FUNCTION_STATUS;
+			function-enumerator = <2>;
 			gpios = <&lsio_gpio5 21 GPIO_ACTIVE_HIGH>;
 		};
 	};
@@ -94,10 +98,6 @@ &adc1 {
 	status = "okay";
 };
 
-&amix {
-	status = "okay";
-};
-
 &asrc0 {
 	status = "okay";
 };
@@ -123,8 +123,6 @@ &flexcan2 {
 	status = "okay";
 };
 
-/* TODO: GPU */
-
 /* Apalis I2C1 */
 &i2c2 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
index 6eab8a6001dbf..8be44eaf4e1e0 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
@@ -608,12 +608,34 @@ &spdif1 {
 	status = "okay";
 };
 
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	assigned-clocks = <&clk IMX8MM_CLK_UART1>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_80M>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
 &uart2 { /* console */
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart2>;
 	status = "okay";
 };
 
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	assigned-clocks = <&clk IMX8MM_CLK_UART3>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_80M>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
 &usbphynop1 {
 	wakeup-source;
 };
@@ -691,7 +713,7 @@ MX8MM_IOMUXC_NAND_READY_B_GPIO3_IO16	0x19
 
 	pinctrl_ir: irgrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_GPIO1_IO13_GPIO1_IO13              0x4f
+			MX8MM_IOMUXC_GPIO1_IO13_GPIO1_IO13	0x4f
 		>;
 	};
 
@@ -724,26 +746,26 @@ MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA			0x400001c3
 
 	pinctrl_pcie0: pcie0grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_I2C4_SCL_PCIE1_CLKREQ_B    0x61
-			MX8MM_IOMUXC_SAI2_RXFS_GPIO4_IO21       0x41
+			MX8MM_IOMUXC_I2C4_SCL_PCIE1_CLKREQ_B	0x61
+			MX8MM_IOMUXC_SAI2_RXFS_GPIO4_IO21	0x41
 		>;
 	};
 
 	pinctrl_pcie0_reg: pcie0reggrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_GPIO1_IO05_GPIO1_IO5       0x41
+			MX8MM_IOMUXC_GPIO1_IO05_GPIO1_IO5	0x41
 		>;
 	};
 
 	pinctrl_pdm: pdmgrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI5_MCLK_SAI5_MCLK        0xd6
-			MX8MM_IOMUXC_SAI5_RXC_PDM_CLK           0xd6
-			MX8MM_IOMUXC_SAI5_RXFS_SAI5_RX_SYNC     0xd6
-			MX8MM_IOMUXC_SAI5_RXD0_PDM_DATA0        0xd6
-			MX8MM_IOMUXC_SAI5_RXD1_PDM_DATA1        0xd6
-			MX8MM_IOMUXC_SAI5_RXD2_PDM_DATA2        0xd6
-			MX8MM_IOMUXC_SAI5_RXD3_PDM_DATA3        0xd6
+			MX8MM_IOMUXC_SAI5_MCLK_SAI5_MCLK	0xd6
+			MX8MM_IOMUXC_SAI5_RXC_PDM_CLK		0xd6
+			MX8MM_IOMUXC_SAI5_RXFS_SAI5_RX_SYNC	0xd6
+			MX8MM_IOMUXC_SAI5_RXD0_PDM_DATA0	0xd6
+			MX8MM_IOMUXC_SAI5_RXD1_PDM_DATA1	0xd6
+			MX8MM_IOMUXC_SAI5_RXD2_PDM_DATA2	0xd6
+			MX8MM_IOMUXC_SAI5_RXD3_PDM_DATA3	0xd6
 		>;
 	};
 
@@ -761,19 +783,19 @@ MX8MM_IOMUXC_SD2_RESET_B_GPIO2_IO19	0x41
 
 	pinctrl_sai2: sai2grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI2_TXC_SAI2_TX_BCLK      0xd6
-			MX8MM_IOMUXC_SAI2_TXFS_SAI2_TX_SYNC     0xd6
-			MX8MM_IOMUXC_SAI2_TXD0_SAI2_TX_DATA0    0xd6
-			MX8MM_IOMUXC_SAI2_RXD0_SAI2_RX_DATA0    0xd6
+			MX8MM_IOMUXC_SAI2_TXC_SAI2_TX_BCLK	0xd6
+			MX8MM_IOMUXC_SAI2_TXFS_SAI2_TX_SYNC	0xd6
+			MX8MM_IOMUXC_SAI2_TXD0_SAI2_TX_DATA0	0xd6
+			MX8MM_IOMUXC_SAI2_RXD0_SAI2_RX_DATA0	0xd6
 		>;
 	};
 
 	pinctrl_sai3: sai3grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC     0xd6
-			MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK      0xd6
-			MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK        0xd6
-			MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0     0xd6
+			MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC	0xd6
+			MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK	0xd6
+			MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK	0xd6
+			MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0	0xd6
 		>;
 	};
 
@@ -790,6 +812,15 @@ MX8MM_IOMUXC_SD1_STROBE_GPIO2_IO11	0x159
 		>;
 	};
 
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART1_RXD_UART1_DCE_RX	0x140
+			MX8MM_IOMUXC_UART1_TXD_UART1_DCE_TX	0x140
+			MX8MM_IOMUXC_UART3_RXD_UART1_DCE_CTS_B	0x140
+			MX8MM_IOMUXC_UART3_TXD_UART1_DCE_RTS_B	0x140
+		>;
+	};
+
 	pinctrl_uart2: uart2grp {
 		fsl,pins = <
 			MX8MM_IOMUXC_UART2_RXD_UART2_DCE_RX	0x140
@@ -797,6 +828,15 @@ MX8MM_IOMUXC_UART2_TXD_UART2_DCE_TX	0x140
 		>;
 	};
 
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI1_SCLK_UART3_DCE_RX		0x140
+			MX8MM_IOMUXC_ECSPI1_MOSI_UART3_DCE_TX		0x140
+			MX8MM_IOMUXC_ECSPI1_SS0_UART3_DCE_RTS_B		0x140
+			MX8MM_IOMUXC_ECSPI1_MISO_UART3_DCE_CTS_B	0x140
+		>;
+	};
+
 	pinctrl_usdhc2_gpio: usdhc2grpgpiogrp {
 		fsl,pins = <
 			MX8MM_IOMUXC_GPIO1_IO15_GPIO1_IO15	0x1c4
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-pinfunc.h b/arch/arm64/boot/dts/freescale/imx8mm-pinfunc.h
index b1f11098d248e..31557b7b9ccc1 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-pinfunc.h
+++ b/arch/arm64/boot/dts/freescale/imx8mm-pinfunc.h
@@ -6,6 +6,39 @@
 #ifndef __DTS_IMX8MM_PINFUNC_H
 #define __DTS_IMX8MM_PINFUNC_H
 
+/* Drive Strength */
+#define MX8MM_DSE_X1		0x0
+#define MX8MM_DSE_X2		0x4
+#define MX8MM_DSE_X4		0x2
+#define MX8MM_DSE_X6		0x6
+
+/* Slew Rate */
+#define MX8MM_FSEL_FAST		0x10
+#define MX8MM_FSEL_SLOW		0x0
+
+/* Open Drain */
+#define MX8MM_ODE_ENABLE	0x20
+#define MX8MM_ODE_DISABLE	0x0
+
+#define MX8MM_PULL_DOWN		0x0
+#define MX8MM_PULL_UP		0x40
+
+/* Hysteresis */
+#define MX8MM_HYS_CMOS		0x0
+#define MX8MM_HYS_SCHMITT	0x80
+
+#define MX8MM_PULL_ENABLE	0x100
+#define MX8MM_PULL_DISABLE	0x0
+
+/* SION force input mode */
+#define MX8MM_SION		0x40000000
+
+/* long defaults */
+#define MX8MM_USDHC_DATA_DEFAULT (MX8MM_FSEL_FAST | MX8MM_PULL_UP | \
+				  MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+#define MX8MM_I2C_DEFAULT (MX8MM_DSE_X6 | MX8MM_PULL_UP | MX8MM_HYS_SCHMITT | \
+			   MX8MM_PULL_ENABLE | MX8MM_SION)
+
 /*
  * The pin function ID is a tuple of
  * <mux_reg conf_reg input_reg mux_mode input_val>
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtso
index e44249c6d8a09..e5425653ea7ed 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtso
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright (c) 2022-2023 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * Copyright (c) 2022-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
  * D-82229 Seefeld, Germany.
  * Author: Alexander Stein
  */
@@ -10,10 +10,6 @@
 
 #include <dt-bindings/gpio/gpio.h>
 
-&{/} {
-	compatible = "tq,imx8mm-tqma8mqml-mba8mx", "tq,imx8mm-tqma8mqml", "fsl,imx8mm";
-};
-
 &backlight_lvds {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-iv-306-d.dts b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-iv-306-d.dts
new file mode 100644
index 0000000000000..6cc04aa90f213
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-iv-306-d.dts
@@ -0,0 +1,799 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2025 Maud Spierings <maudspierings@gocontroll.com>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+
+#include "imx8mm-tx8m-1610.dtsi"
+
+/ {
+	chassis-type = "embedded";
+	compatible = "gocontroll,moduline-iv-306-d", "karo,tx8m-1610", "fsl,imx8mm";
+	hardware = "Moduline IV V3.06-D";
+	model = "GOcontroll Moduline IV";
+
+	aliases {
+		spi0 = &ecspi2; /* spidev number compatibility */
+		spi1 = &ecspi3; /* spidev number compatibility */
+		spi2 = &ecspi1; /* spidev number compatibility */
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	mcp_clock: mcp-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <20000000>;
+	};
+
+	reg_3v3_m2: regulator-3v3-m2 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 8 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&pinctrl_reg_m2>;
+		pinctrl-names = "default";
+		power-supply = <&reg_6v4>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "3v3-m.2";
+	};
+
+	reg_5v0: regulator-5v0 {
+		compatible = "regulator-fixed";
+		power-supply = <&reg_6v4>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "5v0";
+	};
+
+	reg_6v4: regulator-6v4 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <6400000>;
+		regulator-min-microvolt = <6400000>;
+		regulator-name = "6v4";
+	};
+
+	reg_can1_stby: regulator-can1-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 16 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can1_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can1-stby";
+	};
+
+	reg_can2_stby: regulator-can2-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 17 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can2_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can2-stby";
+	};
+
+	reg_can3_stby: regulator-can3-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio1 11 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can3_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can3-stby";
+	};
+
+	reg_can4_stby: regulator-can4-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 8 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can4_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can4-stby";
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-0 = <&pinctrl_wl_reg>;
+		pinctrl-names = "default";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <500000>;
+		reset-gpios = <&gpio3 3 GPIO_ACTIVE_LOW>;
+	};
+};
+
+/* SPI 2 */
+&ecspi1 {
+	cs-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>,
+		   <&gpio1 0 GPIO_ACTIVE_LOW>,
+		   <&gpio5 2 GPIO_ACTIVE_LOW>,
+		   <&gpio4 27 GPIO_ACTIVE_LOW>,
+		   <&gpio3 1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	connector@0 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <0>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio1 5 GPIO_ACTIVE_LOW>;
+		slot-number = <3>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@1 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <1>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <21 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio3 15 GPIO_ACTIVE_LOW>;
+		slot-number = <4>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@2 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <2>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio4 25 GPIO_ACTIVE_LOW>;
+		slot-number = <5>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@3 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <3>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio4 28 GPIO_ACTIVE_LOW>;
+		slot-number = <6>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	adc@4 {
+		compatible = "microchip,mcp3004";
+		reg = <4>;
+		spi-max-frequency = <2300000>;
+		vref-supply = <&reg_vdd_3v3>;
+	};
+};
+
+&ecspi2 {
+	cs-gpios = <&gpio3 23 GPIO_ACTIVE_LOW>,
+		   <&gpio5 9 GPIO_ACTIVE_LOW>,
+		   <&gpio3 2 GPIO_ACTIVE_LOW>,
+		   <&gpio5 25 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	connector@0 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <0>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <19 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio3 24 GPIO_ACTIVE_LOW>;
+		slot-number = <7>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@1 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <1>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <22 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio3 21 GPIO_ACTIVE_LOW>;
+		slot-number = <8>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	can@2 {
+		compatible = "microchip,mcp25625";
+		reg = <2>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can1>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can1_stby>;
+	};
+
+	can@3 {
+		compatible = "microchip,mcp25625";
+		reg = <3>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <13 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can2>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can2_stby>;
+	};
+};
+
+&ecspi3 {
+	cs-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>,
+		   <&gpio1 10 GPIO_ACTIVE_LOW>,
+		   <&gpio5 5 GPIO_ACTIVE_LOW>,
+		   <&gpio5 4 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi3>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	connector@0 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <0>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+		slot-number = <1>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@1 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <1>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <20 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio3 11 GPIO_ACTIVE_LOW>;
+		slot-number = <2>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	can@2 {
+		compatible = "microchip,mcp25625";
+		reg = <2>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can3>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can3_stby>;
+	};
+
+	can@3 {
+		compatible = "microchip,mcp25625";
+		reg = <3>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can4>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can4_stby>;
+	};
+};
+
+&gpu_2d {
+	status = "disabled";
+};
+
+&gpu_3d {
+	status = "disabled";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio5 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	pinctrl-names = "default", "gpio";
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	lp5012@14 {
+		compatible = "ti,lp5012";
+		reg = <0x14>;
+		vled-supply = <&reg_6v4>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		multi-led@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led1";
+
+			led@0 {
+				color = <LED_COLOR_ID_RED>;
+				reg = <0>;
+			};
+
+			led@1 {
+				color = <LED_COLOR_ID_GREEN>;
+				reg = <1>;
+			};
+
+			led@2 {
+				color = <LED_COLOR_ID_BLUE>;
+				reg = <2>;
+			};
+		};
+
+		multi-led@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led2";
+
+			led@0 {
+				color = <LED_COLOR_ID_RED>;
+				reg = <0>;
+			};
+
+			led@1 {
+				color = <LED_COLOR_ID_GREEN>;
+				reg = <1>;
+			};
+
+			led@2 {
+				color = <LED_COLOR_ID_BLUE>;
+				reg = <2>;
+			};
+		};
+
+		multi-led@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led3";
+
+			led@0 {
+				color = <LED_COLOR_ID_RED>;
+				reg = <0>;
+			};
+
+			led@1 {
+				color = <LED_COLOR_ID_GREEN>;
+				reg = <1>;
+			};
+
+			led@2 {
+				color = <LED_COLOR_ID_BLUE>;
+				reg = <2>;
+			};
+		};
+
+		multi-led@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led4";
+
+			led@0 {
+				color = <LED_COLOR_ID_RED>;
+				reg = <0>;
+			};
+
+			led@1 {
+				color = <LED_COLOR_ID_GREEN>;
+				reg = <1>;
+			};
+
+			led@2 {
+				color = <LED_COLOR_ID_BLUE>;
+				reg = <2>;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO01_GPIO1_IO1
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI5_MCLK_GPIO3_IO25
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_can1: can1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_ALE_GPIO3_IO0
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_NAND_CE3_B_GPIO3_IO4
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can1_reg: can1reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_READY_B_GPIO3_IO16
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_can2: can2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_NAND_DATA07_GPIO3_IO13
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can2_reg: can2reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_GPIO3_IO17
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_can3: can3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SPDIF_TX_GPIO5_IO3
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_NAND_DATA06_GPIO3_IO12
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can3_reg: can3reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_can4: can4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_DQS_GPIO3_IO14
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_NAND_WP_B_GPIO3_IO18
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can4_reg: can4reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_DATA02_GPIO3_IO8
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI1_MOSI_ECSPI1_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_ECSPI1_MISO_ECSPI1_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI1_SCLK_ECSPI1_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI3_MCLK_GPIO5_IO2
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_NAND_CE0_B_GPIO3_IO1
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_SAI5_RXD2_GPIO3_IO23
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_ECSPI1_SS0_GPIO5_IO9
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_NAND_CE1_B_GPIO3_IO2
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_UART2_TXD_GPIO5_IO25
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_ecspi3: ecspi3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART1_TXD_ECSPI3_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_UART2_RXD_ECSPI3_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_UART1_RXD_ECSPI3_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_GPIO1_IO04_GPIO1_IO4
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SPDIF_EXT_CLK_GPIO5_IO5
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SPDIF_RX_GPIO5_IO4
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_I2C2_SCL
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C2_SDA_I2C2_SDA
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_GPIO5_IO16
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C2_SDA_GPIO5_IO17
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c3_gpio: i2c3-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_GPIO5_IO18
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C3_SDA_GPIO5_IO19
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_reg_m2: reg-m2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI1_RXD6_GPIO4_IO8
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_RXC_UART1_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_RXFS_UART1_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_RXD0_UART1_DCE_RTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_TXFS_UART1_DCE_CTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_TXC_UART2_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_RXD_UART2_DCE_RTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_RXC_UART2_DCE_CTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART4_RXD_UART4_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_UART4_TXD_UART4_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_usdhc2: pinctrlusdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+		>;
+	};
+
+	pinctrl_wl_int: wlintgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI5_RXC_GPIO3_IO20
+				(MX8MM_PULL_UP | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_wl_reg: wlreggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_CE2_B_GPIO3_IO3
+				MX8MM_DSE_X1
+		>;
+	};
+};
+
+&uart1 {
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "infineon,cyw43439-bt", "brcm,bcm4329-bt";
+		device-wakeup-gpios = <&gpio5 13 GPIO_ACTIVE_HIGH>;
+		interrupt-names = "host-wakeup";
+		interrupt-parent = <&gpio3>;
+		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
+		max-speed = <921600>;
+		pinctrl-0 = <&pinctrl_bt>;
+		pinctrl-names = "default";
+		shutdown-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_3v3_m2>;
+		vddio-supply = <&reg_3v3_m2>;
+	};
+};
+
+&uart2 {
+	pinctrl-0 = <&pinctrl_uart2>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-0 = <&pinctrl_uart3>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-0 = <&pinctrl_uart4>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usbotg1 {
+	disable-over-current;
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_5v0>;
+	status = "okay";
+};
+
+&usdhc2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	max-frequency = <50000000>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	pinctrl-names = "default";
+	sd-uhs-sdr25;
+	vmmc-supply = <&reg_3v3_m2>;
+	status = "okay";
+
+	wifi@1 {
+		compatible = "infineon,cyw43439-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		pinctrl-0 = <&pinctrl_wl_int>;
+		pinctrl-names = "default";
+		interrupt-names = "host-wake";
+		interrupt-parent = <&gpio3>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		brcm,board-type = "GOcontroll,moduline";
+	};
+};
+
+&vpu_blk_ctrl {
+	status = "disabled";
+};
+
+&vpu_g1 {
+	status = "disabled";
+};
+
+&vpu_g2 {
+	status = "disabled";
+};
+
+&wdog1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-mini-111.dts b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-mini-111.dts
new file mode 100644
index 0000000000000..39b7d9077e8e2
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610-moduline-mini-111.dts
@@ -0,0 +1,687 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2025 Maud Spierings <maudspierings@gocontroll.com>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+
+#include "imx8mm-tx8m-1610.dtsi"
+
+/ {
+	chassis-type = "embedded";
+	compatible = "gocontroll,moduline-mini-111", "karo,tx8m-1610", "fsl,imx8mm";
+	hardware = "Moduline Mini V1.11";
+	model = "GOcontroll Moduline Mini";
+
+	aliases {
+		spi0 = &ecspi2; /* spidev number compatibility */
+		spi1 = &ecspi3; /* spidev number compatibility */
+		spi2 = &ecspi1; /* spidev number compatibility */
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	mcp_clock: mcp-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <20000000>;
+	};
+
+	reg_3v3_comm: regulator-3v3-communication {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 11 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&pinctrl_reg_comm>;
+		pinctrl-names = "default";
+		power-supply = <&reg_6v4>;
+		/* also powers the cellular modem which can't vote on the regulator */
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "3v3_comm";
+	};
+
+	reg_5v0: regulator-5v0 {
+		compatible = "regulator-fixed";
+		power-supply = <&reg_6v4>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "5v0";
+	};
+
+	reg_6v4: regulator-6v4 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <6400000>;
+		regulator-min-microvolt = <6400000>;
+		regulator-name = "6v4";
+	};
+
+	reg_can1_stby: regulator-can1-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio2 12 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can1_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can1-stby";
+	};
+
+	reg_can2_stby: regulator-can2-stby {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 15 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_can2_reg>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "can2-stby";
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-0 = <&pinctrl_wl_reg>;
+		pinctrl-names = "default";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <500000>;
+		reset-gpios = <&gpio5 28 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&ecspi1 {
+	cs-gpios = <&gpio4 27 GPIO_ACTIVE_LOW>,
+		   <&gpio3 23 GPIO_ACTIVE_LOW>,
+		   <&gpio3 1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	connector@0 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <0>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio4 28 GPIO_ACTIVE_LOW>;
+		slot-number = <3>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@1 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <1>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <19 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio3 21 GPIO_ACTIVE_LOW>;
+		slot-number = <4>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	adc@2 {
+		compatible = "microchip,mcp3004";
+		reg = <2>;
+		spi-max-frequency = <2300000>;
+		vref-supply = <&reg_vdd_3v3>;
+	};
+};
+
+&ecspi2 {
+	cs-gpios = <&gpio3 24 GPIO_ACTIVE_LOW>,
+		   <&gpio3 9 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	can@0 {
+		compatible = "microchip,mcp25625";
+		reg = <0>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <22 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can1>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can1_stby>;
+	};
+
+	can@1 {
+		compatible = "microchip,mcp25625";
+		reg = <1>;
+		clocks = <&mcp_clock>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_can2>;
+		pinctrl-names = "default";
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_can2_stby>;
+	};
+};
+
+&ecspi3 {
+	cs-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>,
+		   <&gpio1 2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_ecspi3>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	connector@0 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <0>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;
+		slot-number = <1>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	connector@1 {
+		compatible = "gocontroll,moduline-module-slot";
+		reg = <1>;
+		i2c-bus = <&i2c2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio5 21 GPIO_ACTIVE_LOW>;
+		slot-number = <2>;
+		spi-max-frequency = <54000000>;
+		sync-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		vddhpp-supply = <&reg_6v4>;
+		vddp-supply = <&reg_5v0>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+};
+
+&gpu_2d {
+	status = "disabled";
+};
+
+&gpu_3d {
+	status = "disabled";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	pinctrl-names = "default", "gpio";
+	scl-gpios = <&gpio5 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	pinctrl-names = "default", "gpio";
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	lp5012@14 {
+		compatible = "ti,lp5012";
+		reg = <0x14>;
+		vled-supply = <&reg_6v4>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		multi-led@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led1";
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led2";
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led3";
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "case-led4";
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+	};
+
+	accelerometer@18 {
+		compatible = "st,lis2dw12";
+		reg = <0x18>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <3 IRQ_TYPE_EDGE_RISING>, <5 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-0 = <&pinctrl_lis_int>;
+		pinctrl-names = "default";
+		vddio-supply = <&reg_vdd_3v3>;
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+
+	humidity-sensor@5f {
+		compatible = "st,hts221";
+		reg = <0x5f>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <10 IRQ_TYPE_EDGE_RISING>;
+		pinctrl-0 = <&pinctrl_hts_int>;
+		pinctrl-names = "default";
+		vdd-supply = <&reg_vdd_3v3>;
+	};
+};
+
+&iomuxc {
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO01_GPIO1_IO1
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI5_MCLK_GPIO3_IO25
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_can1: can1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI5_RXD1_GPIO3_IO22
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can1_reg: can1reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_can2: can2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_CLE_GPIO3_IO5
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_NAND_DATA00_GPIO3_IO6
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_can2_reg: can2reggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_RE_B_GPIO3_IO15
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI1_MOSI_ECSPI1_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_ECSPI1_MISO_ECSPI1_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI1_SCLK_ECSPI1_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI5_RXD2_GPIO3_IO23
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_NAND_CE0_B_GPIO3_IO1
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_SAI5_RXD3_GPIO3_IO24
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_NAND_DATA03_GPIO3_IO9
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_ecspi3: ecspi3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART1_TXD_ECSPI3_MOSI
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_UART2_RXD_ECSPI3_MISO
+				(MX8MM_DSE_X4 | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_UART1_RXD_ECSPI3_SCLK
+				MX8MM_DSE_X4
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO02_GPIO1_IO2
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_hts_int: htsintgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_DATA04_GPIO3_IO10
+				(MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_I2C2_SCL
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C2_SDA_I2C2_SDA
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_GPIO5_IO16
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C2_SDA_GPIO5_IO17
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c3_gpio: i2c3-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_GPIO5_IO18
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C3_SDA_GPIO5_IO19
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_lis_int: lisintgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SPDIF_TX_GPIO5_IO3
+				(MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_SPDIF_EXT_CLK_GPIO5_IO5
+				(MX8MM_PULL_ENABLE | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_reg_comm: reg_commgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_sysfs_gpios: sysfsgpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_I2C4_SDA_GPIO5_IO21
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SAI5_RXD0_GPIO3_IO21
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_SD2_WP_GPIO2_IO20
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO04_GPIO1_IO4
+				MX8MM_DSE_X1
+			MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6
+				MX8MM_DSE_X1
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_RXC_UART1_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_RXFS_UART1_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_RXD0_UART1_DCE_RTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI2_TXFS_UART1_DCE_CTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_TXC_UART2_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_RXD_UART2_DCE_RTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SAI3_RXC_UART2_DCE_CTS_B
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_usdhc2: pinctrlusdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+		>;
+	};
+
+	pinctrl_wl_int: wlintgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI5_RXC_GPIO3_IO20
+				(MX8MM_PULL_UP | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_wl_reg: wlreggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART4_RXD_GPIO5_IO28
+				MX8MM_DSE_X1
+		>;
+	};
+};
+
+&uart1 {
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "infineon,cyw43439-bt", "brcm,bcm4329-bt";
+		device-wakeup-gpios = <&gpio5 13 GPIO_ACTIVE_HIGH>;
+		interrupt-names = "host-wakeup";
+		interrupt-parent = <&gpio3>;
+		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
+		max-speed = <921600>;
+		pinctrl-0 = <&pinctrl_bt>;
+		pinctrl-names = "default";
+		shutdown-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_3v3_comm>;
+		vddio-supply = <&reg_3v3_comm>;
+	};
+};
+
+&uart2 {
+	pinctrl-0 = <&pinctrl_uart2>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-0 = <&pinctrl_uart3>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usbotg1 {
+	disable-over-current;
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_5v0>;
+	status = "okay";
+};
+
+&usdhc2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	max-frequency = <50000000>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	pinctrl-names = "default";
+	sd-uhs-sdr25;
+	vmmc-supply = <&reg_3v3_comm>;
+	status = "okay";
+
+	wifi@1 {
+		compatible = "infineon,cyw43439-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		pinctrl-0 = <&pinctrl_wl_int>;
+		pinctrl-names = "default";
+		interrupt-names = "host-wake";
+		interrupt-parent = <&gpio3>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		brcm,board-type = "GOcontroll,moduline";
+	};
+};
+
+&vpu_blk_ctrl {
+	status = "disabled";
+};
+
+&vpu_g1 {
+	status = "disabled";
+};
+
+&vpu_g2 {
+	status = "disabled";
+};
+
+&wdog1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610.dtsi
new file mode 100644
index 0000000000000..ba00f7063476b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-tx8m-1610.dtsi
@@ -0,0 +1,444 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2021 Lothar Waßmann <LW@KARO-electronics.de>
+ * 2025 Maud Spierings <maudspierings@gocontroll.com>
+ */
+
+#include "imx8mm.dtsi"
+
+/ {
+	model = "Ka-Ro Electronics TX8M-1610";
+	compatible = "karo,tx8m-1610", "fsl,imx8mm";
+
+	reg_3v3_etn: regulator-3v3-etn {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 23 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&pinctrl_reg_3v3_etn>;
+		pinctrl-names = "default";
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "3v3-etn";
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_1 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_2 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_3 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&ddrc {
+	operating-points-v2 = <&ddrc_opp_table>;
+
+	ddrc_opp_table: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+		};
+	};
+};
+
+&fec1 {
+	assigned-clocks = <&clk IMX8MM_CLK_ENET_AXI>,
+			  <&clk IMX8MM_CLK_ENET_TIMER>,
+			  <&clk IMX8MM_CLK_ENET_REF>,
+			  <&clk IMX8MM_CLK_ENET_REF>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_266M>,
+				 <&clk IMX8MM_SYS_PLL2_100M>,
+				 <&clk IMX8MM_SYS_PLL2_50M>,
+				 <&clk IMX8MM_SYS_PLL2_50M>;
+	assigned-clock-rates = <0>, <100000000>, <50000000>, <50000000>;
+	clocks = <&clk IMX8MM_CLK_ENET1_ROOT>,
+		 <&clk IMX8MM_CLK_ENET1_ROOT>,
+		 <&clk IMX8MM_CLK_ENET_TIMER>,
+		 <&clk IMX8MM_CLK_ENET_REF>;
+	phy-handle = <&ethphy0>;
+	phy-mode = "rmii";
+	phy-reset-duration = <25>;
+	phy-reset-gpios = <&gpio1 29 GPIO_ACTIVE_LOW>;
+	phy-reset-post-delay = <1>;
+	phy-supply = <&reg_3v3_etn>;
+	pinctrl-0 = <&pinctrl_fec1>, <&pinctrl_ethphy_rst>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			reg = <0>;
+			clocks = <&clk IMX8MM_CLK_ENET_REF>;
+			interrupt-parent = <&gpio1>;
+			interrupts = <28 IRQ_TYPE_EDGE_FALLING>;
+			pinctrl-0 = <&pinctrl_ethphy_int>;
+			pinctrl-names = "default";
+			smsc,disable-energy-detect;
+		};
+	};
+};
+
+&gpio1 {
+	gpio-line-names = "SODIMM_152", "SODIMM_42", "SODIMM_153", "PMIC_IRQ_B",
+			  "SODIMM_154", "SODIMM_155", "SODIMM_156", "SODIMM_157",
+			  "SODIMM_158", "SODIMM_159", "SODIMM_161", "SODIMM_162",
+			  "SODIMM_34", "SODIMM_36", "SODIMM_27", "SODIMM_28",
+			  "", "", "", "",
+			  "", "", "", "ENET_POWER",
+			  "", "", "", "",
+			  "ENET_nINT", "ENET_nRST", "", "";
+};
+
+&gpio2 {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "SODIMM_51", "SODIMM_57", "SODIMM_56", "SODIMM_52",
+			  "SODIMM_53", "SODIMM_54", "SODIMM_55", "SODIMM_15",
+			  "SODIMM_45", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&gpio3 {
+	gpio-line-names = "SODIMM_103", "SODIMM_104", "SODIMM_105", "SODIMM_106",
+			  "SODIMM_107", "SODIMM_112", "SODIMM_108", "SODIMM_109",
+			  "SODIMM_95", "SODIMM_110", "SODIMM_96", "SODIMM_97",
+			  "SODIMM_98", "SODIMM_99", "SODIMM_113", "SODIMM_114",
+			  "SODIMM_115", "SODIMM_101", "SODIMM_100", "SODIMM_77",
+			  "SODIMM_72", "SODIMM_73", "SODIMM_74", "SODIMM_75",
+			  "SODIMM_76", "SODIMM_43", "", "",
+			  "", "", "", "";
+};
+
+&gpio4 {
+	gpio-line-names = "SODIMM_178", "SODIMM_180", "SODIMM_184", "SODIMM_185",
+			  "SODIMM_186", "SODIMM_187", "SODIMM_188", "SODIMM_189",
+			  "SODIMM_190", "SODIMM_191", "SODIMM_179", "SODIMM_181",
+			  "SODIMM_192", "SODIMM_193", "SODIMM_194", "SODIMM_195",
+			  "SODIMM_196", "SODIMM_197", "SODIMM_198", "SODIMM_199",
+			  "SODIMM_182", "SODIMM_79", "SODIMM_78", "SODIMM_84",
+			  "SODIMM_87", "SODIMM_86", "SODIMM_85", "SODIMM_83",
+			  "SODIMM_81", "SODIMM_80", "SODIMM_90", "SODIMM_93";
+};
+
+&gpio5 {
+	gpio-line-names = "SODIMM_92", "SODIMM_91", "SODIMM_89", "SODIMM_144",
+			  "SODIMM_143", "SODIMM_146", "SODIMM_68", "SODIMM_67",
+			  "SODIMM_70", "SODIMM_69", "SODIMM_48", "SODIMM_46",
+			  "SODIMM_47", "SODIMM_44", "PMIC_SCL", "PMIC_SDA",
+			  "SODIMM_41", "SODIMM_40", "SODIMM_148", "SODIMM_149",
+			  "SODIMM_150", "SODIMM_151", "SODIMM_60", "SODIMM_59",
+			  "SODIMM_64", "SODIMM_63", "SODIMM_62", "SODIMM_61",
+			  "SODIMM_66", "SODIMM_65", "", "";
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	pinctrl-names = "default", "gpio";
+	scl-gpios = <&gpio5 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pmic: pmic@4b {
+		compatible = "rohm,bd71847";
+		reg = <0x4b>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_pmic>;
+		pinctrl-names = "default";
+		rohm,reset-snvs-powered;
+
+		regulators {
+			BUCK1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <900000>;
+				regulator-min-microvolt = <780000>;
+				regulator-name = "buck1";
+				regulator-ramp-delay = <1250>;
+			};
+
+			reg_vdd_arm: BUCK2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <805000>;
+				regulator-name = "buck2";
+				regulator-ramp-delay = <1250>;
+				rohm,dvs-run-voltage = <950000>;
+				rohm,dvs-idle-voltage = <810000>;
+			};
+
+			BUCK3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <900000>;
+				regulator-min-microvolt = <805000>;
+				regulator-name = "buck3";
+			};
+
+			reg_vdd_3v3: BUCK4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-name = "buck4";
+			};
+
+			reg_vdd_1v8: BUCK5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1950000>;
+				regulator-min-microvolt = <1700000>;
+				regulator-name = "buck5";
+			};
+
+			BUCK6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <(1350000 + 100000)>;
+				regulator-min-microvolt = <(1350000 - 67000)>;
+				regulator-name = "buck6";
+				rohm,fb-pull-up-microvolt = <0>;
+				rohm,feedback-pull-up-r1-ohms = <2200>;
+				rohm,feedback-pull-up-r2-ohms = <499>;
+			};
+
+			LDO1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1980000>;
+				regulator-min-microvolt = <1620000>;
+				regulator-name = "ldo1";
+			};
+
+			LDO2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <900000>;
+				regulator-min-microvolt = <760000>;
+				regulator-name = "ldo2";
+			};
+
+			LDO3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1890000>;
+				regulator-min-microvolt = <1710000>;
+				regulator-name = "ldo3";
+			};
+
+			LDO4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1000000>;
+				regulator-min-microvolt = <855000>;
+				regulator-name = "ldo4";
+			};
+
+			LDO5 {
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "ldo5";
+			};
+
+			LDO6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1260000>;
+				regulator-min-microvolt = <1140000>;
+				regulator-name = "ldo6";
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_ethphy_int: etnphy-intgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_RD2_GPIO1_IO28
+				(MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT)
+		>;
+	};
+
+	pinctrl_ethphy_rst: etnphy-rstgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_RD3_GPIO1_IO29
+				(MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC
+				(MX8MM_DSE_X4 | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO
+				(MX8MM_DSE_X4 | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_ENET_TD2_ENET1_TX_CLK
+				(MX8MM_FSEL_FAST | MX8MM_SION)
+			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST)
+			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST)
+			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0
+				(MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1
+				(MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT)
+			MX8MM_IOMUXC_ENET_RXC_ENET1_RX_ER
+				MX8MM_FSEL_FAST
+			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL
+				MX8MM_FSEL_FAST
+			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST)
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_GPIO5_IO14
+				MX8MM_I2C_DEFAULT
+			MX8MM_IOMUXC_I2C1_SDA_GPIO5_IO15
+				MX8MM_I2C_DEFAULT
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3
+				(MX8MM_PULL_UP | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_reg_3v3_etn: reg-3v3-etngrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_TXC_GPIO1_IO23
+				(MX8MM_DSE_X4 | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK
+				(MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7
+				MX8MM_USDHC_DATA_DEFAULT
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE
+				(MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7
+				(MX8MM_DSE_X2 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE
+				(MX8MM_DSE_X2 | MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7
+				(MX8MM_DSE_X6 | MX8MM_USDHC_DATA_DEFAULT)
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST | MX8MM_HYS_SCHMITT | MX8MM_PULL_ENABLE)
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B
+				(MX8MM_DSE_X6 | MX8MM_FSEL_FAST | MX8MM_PULL_UP | MX8MM_PULL_ENABLE)
+		>;
+	};
+};
+
+&usdhc1 {
+	assigned-clocks = <&clk IMX8MM_CLK_USDHC1>;
+	assigned-clock-rates = <400000000>;
+	bus-width = <8>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	vmmc-supply = <&reg_vdd_3v3>;
+	vqmmc-supply = <&reg_vdd_1v8>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
index 9f49c0b386d31..f2e1854f38a0a 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
@@ -400,7 +400,7 @@ micfil: audio-controller@30080000 {
 				};
 
 				spdif1: spdif@30090000 {
-					compatible = "fsl,imx35-spdif";
+					compatible = "fsl,imx8mm-spdif";
 					reg = <0x30090000 0x10000>;
 					interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clk IMX8MM_CLK_AUDIO_AHB>, /* core */
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi
index 145355ff91b45..3e590afa4fabb 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi
@@ -415,6 +415,10 @@ &uart1 { /* BT */
 	assigned-clock-parents = <&clk IMX8MN_SYS_PLL1_80M>;
 	uart-has-rtscts;
 	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
 };
 
 &uart2 { /* console */
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtso
index 29235e390a5d7..0f4cf1855bf3a 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtso
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright (c) 2022-2023 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * Copyright (c) 2022-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
  * D-82229 Seefeld, Germany.
  * Author: Alexander Stein
  */
@@ -10,10 +10,6 @@
 
 #include <dt-bindings/gpio/gpio.h>
 
-&{/} {
-	compatible = "tq,imx8mn-tqma8mqnl-mba8mx", "tq,imx8mn-tqma8mqnl", "fsl,imx8mn";
-};
-
 &backlight_lvds {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
new file mode 100644
index 0000000000000..7031a33d85c8f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
@@ -0,0 +1,912 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2020-2026 NXP
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/usb/pd.h>
+#include "imx8mp.dtsi"
+
+/ {
+	compatible = "fsl,imx8mp-ab2", "fsl,imx8mp";
+	model = "NXP i.MX8MP SOM on AB2";
+
+	chosen {
+		stdout-path = &uart2;
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-0 = <&pinctrl_gpio_led>;
+		pinctrl-names = "default";
+
+		status {
+			default-state = "on";
+			gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>;
+			label = "yellow:status";
+		};
+	};
+
+	native-hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "HDMI OUT";
+		type = "a";
+
+		port {
+			hdmi_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+
+	reg_ab2_ana_pwr: regulator-ab2-ana-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "ab2_ana_pwr";
+		pinctrl-0 = <&pinctrl_ab2_ana_pwr>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_ab2_vdd_pwr_5v0: regulator-ab2-vdd-pwr-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "ab2_vdd_pwr_5v0";
+		pinctrl-0 = <&pinctrl_ab2_vdd_pwr_5v0>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio1 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VSD_3V3";
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		dsp_vdev0vring0: vdev0vring0@942f0000 {
+			reg = <0 0x942f0000 0 0x8000>;
+			no-map;
+		};
+
+		dsp_vdev0vring1: vdev0vring1@942f8000 {
+			reg = <0 0x942f8000 0 0x8000>;
+			no-map;
+		};
+
+		dsp_vdev0buffer: vdev0buffer@94300000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x94300000 0 0x100000>;
+			no-map;
+		};
+	};
+
+	sound-ak4458 {
+		compatible = "fsl,imx-audio-card";
+		model = "ak4458-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "akcodec";
+			fsl,mclk-equal-bclk;
+
+			codec {
+				sound-dai = <&ak4458_1>, <&ak4458_2>;
+			};
+
+			cpu {
+				sound-dai = <&sai1>;
+			};
+		};
+	};
+
+	sound-ak5552 {
+		compatible = "fsl,imx-audio-card";
+		model = "ak5552-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "akcodec";
+			fsl,mclk-equal-bclk;
+
+			codec {
+				sound-dai = <&ak5552>;
+			};
+
+			cpu {
+				sound-dai = <&sai3>;
+			};
+		};
+	};
+
+	sound-hdmi {
+		compatible = "fsl,imx-audio-hdmi";
+		audio-cpu = <&aud2htx>;
+		hdmi-out;
+		model = "audio-hdmi";
+	};
+
+	sound-micfil {
+		compatible = "fsl,imx-audio-card";
+		model = "micfil-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "micfil hifi";
+
+			cpu {
+				sound-dai = <&micfil>;
+			};
+		};
+	};
+
+	sound-xcvr {
+		compatible = "fsl,imx-audio-card";
+		model = "imx-audio-xcvr";
+
+		pri-dai-link {
+			link-name = "XCVR PCM";
+
+			cpu {
+				sound-dai = <&xcvr>;
+			};
+		};
+	};
+
+	memory@40000000 {
+		reg = <0x0 0x40000000 0 0xc0000000>,
+		      <0x1 0x00000000 0 0xc0000000>;
+		device_type = "memory";
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_1 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_2 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_3 {
+	cpu-supply = <&buck2>;
+};
+
+&aud2htx {
+	status = "okay";
+};
+
+&dsp {
+	memory-region = <&dsp_vdev0buffer>, <&dsp_vdev0vring0>,
+			<&dsp_vdev0vring1>, <&dsp_reserved>;
+	status = "okay";
+};
+
+&dsp_reserved {
+	status = "okay";
+};
+
+&easrc {
+	#sound-dai-cells = <0>;
+	fsl,asrc-rate = <48000>;
+	status = "okay";
+};
+
+&eqos {
+	phy-handle = <&ethphy0>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+		};
+	};
+};
+
+&flexspi {
+	pinctrl-0 = <&pinctrl_flexspi0>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	mt25qu256aba: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <80000000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <1>;
+	};
+};
+
+&hdmi_pai {
+	status = "okay";
+};
+
+&hdmi_pvi {
+	status = "okay";
+};
+
+&hdmi_tx {
+	pinctrl-0 = <&pinctrl_hdmi>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ports {
+		port@1 {
+			hdmi_tx_out: endpoint {
+				remote-endpoint = <&hdmi_in>;
+			};
+		};
+	};
+};
+
+&hdmi_tx_phy {
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pca9450: pmic@25 {
+		compatible = "nxp,pca9450c";
+		reg = <0x25>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <3 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&pinctrl_pmic>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <2187500>;
+				regulator-min-microvolt = <600000>;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <2187500>;
+				regulator-min-microvolt = <600000>;
+				regulator-ramp-delay = <3125>;
+				nxp,dvs-run-voltage = <950000>;
+				nxp,dvs-standby-voltage = <850000>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3400000>;
+				regulator-min-microvolt = <600000>;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3400000>;
+				regulator-min-microvolt = <600000>;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3400000>;
+				regulator-min-microvolt = <600000>;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1600000>;
+			};
+
+			ldo2: LDO2 {
+				regulator-name = "LDO2";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1150000>;
+				regulator-min-microvolt = <800000>;
+			};
+
+			ldo3: LDO3 {
+				regulator-name = "LDO3";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <800000>;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <800000>;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
+			};
+		};
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pca6408: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+
+	pca6416_2: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ak4458_1: audio-codec@10 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "0";
+	};
+
+	ak4458_2: audio-codec@11 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "1";
+	};
+
+	ak4458_3: audio-codec@12 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x12>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+	};
+
+	ak5552: audio-codec@13 {
+		compatible = "asahi-kasei,ak5552";
+		reg = <0x13>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 2 GPIO_ACTIVE_LOW>;
+	};
+
+	pca6416: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+};
+
+&iomuxc {
+	pinctrl-0 = <&pinctrl_hog>;
+	pinctrl-names = "default";
+
+	pinctrl_ab2_ana_pwr: ab2anapwrgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10	0xd6
+		>;
+	};
+
+	pinctrl_ab2_vdd_pwr_5v0: ab2vddpwr5v0grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07	0xd6
+		>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ENET_MDC__ENET_QOS_MDC		0x2
+			MX8MP_IOMUXC_ENET_MDIO__ENET_QOS_MDIO		0x2
+			MX8MP_IOMUXC_ENET_RD0__ENET_QOS_RGMII_RD0	0x90
+			MX8MP_IOMUXC_ENET_RD1__ENET_QOS_RGMII_RD1	0x90
+			MX8MP_IOMUXC_ENET_RD2__ENET_QOS_RGMII_RD2	0x90
+			MX8MP_IOMUXC_ENET_RD3__ENET_QOS_RGMII_RD3	0x90
+			MX8MP_IOMUXC_ENET_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x90
+			MX8MP_IOMUXC_ENET_RX_CTL__ENET_QOS_RGMII_RX_CTL	0x90
+			MX8MP_IOMUXC_ENET_TD0__ENET_QOS_RGMII_TD0	0x16
+			MX8MP_IOMUXC_ENET_TD1__ENET_QOS_RGMII_TD1	0x16
+			MX8MP_IOMUXC_ENET_TD2__ENET_QOS_RGMII_TD2	0x16
+			MX8MP_IOMUXC_ENET_TD3__ENET_QOS_RGMII_TD3	0x16
+			MX8MP_IOMUXC_ENET_TX_CTL__ENET_QOS_RGMII_TX_CTL	0x16
+			MX8MP_IOMUXC_ENET_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x16
+			MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x10
+		>;
+	};
+
+	pinctrl_flexspi0: flexspi0grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_ALE__FLEXSPI_A_SCLK           0x1c2
+			MX8MP_IOMUXC_NAND_CE0_B__FLEXSPI_A_SS0_B        0x82
+			MX8MP_IOMUXC_NAND_DATA00__FLEXSPI_A_DATA00      0x82
+			MX8MP_IOMUXC_NAND_DATA01__FLEXSPI_A_DATA01      0x82
+			MX8MP_IOMUXC_NAND_DATA02__FLEXSPI_A_DATA02      0x82
+			MX8MP_IOMUXC_NAND_DATA03__FLEXSPI_A_DATA03      0x82
+		>;
+	};
+
+	pinctrl_gpio_led: gpioledgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_READY_B__GPIO3_IO16	0x10
+		>;
+	};
+
+	pinctrl_hdmi: hdmigrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL	0x1c2
+			MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA	0x1c2
+			MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC		0x10
+		>;
+	};
+
+	pinctrl_hog: hoggrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_HPD__HDMIMIX_HDMI_HPD		0x40000010
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL		0x400001c2
+			MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA		0x400001c2
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL		0x400001c2
+			MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA		0x400001c2
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL		0x400001c2
+			MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA		0x400001c2
+		>;
+	};
+
+	pinctrl_pdm: pdmgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI5_RXC__AUDIOMIX_PDM_CLK		0xd6
+			MX8MP_IOMUXC_SAI5_RXD0__AUDIOMIX_PDM_BIT_STREAM00	0xd6
+			MX8MP_IOMUXC_SAI5_RXD1__AUDIOMIX_PDM_BIT_STREAM01	0xd6
+			MX8MP_IOMUXC_SAI5_RXD2__AUDIOMIX_PDM_BIT_STREAM02	0xd6
+			MX8MP_IOMUXC_SAI5_RXD3__AUDIOMIX_PDM_BIT_STREAM03	0xd6
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03	0x41
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO01__PWM1_OUT	0x116
+		>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO11__PWM2_OUT	0x116
+		>;
+	};
+
+	pinctrl_pwm4: pwm4grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI5_RXFS__PWM4_OUT	0x116
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_RESET_B__GPIO2_IO19	0x41
+		>;
+	};
+
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI1_MCLK__AUDIOMIX_SAI1_MCLK	0xd6
+			MX8MP_IOMUXC_SAI1_TXFS__AUDIOMIX_SAI1_TX_SYNC	0xd6
+			MX8MP_IOMUXC_SAI1_TXC__AUDIOMIX_SAI1_TX_BCLK	0xd6
+			MX8MP_IOMUXC_SAI1_TXD0__AUDIOMIX_SAI1_TX_DATA00	0xd6
+			MX8MP_IOMUXC_SAI1_TXD1__AUDIOMIX_SAI1_TX_DATA01	0xd6
+			MX8MP_IOMUXC_SAI1_TXD2__AUDIOMIX_SAI1_TX_DATA02	0xd6
+			MX8MP_IOMUXC_SAI1_TXD3__AUDIOMIX_SAI1_TX_DATA03	0xd6
+			MX8MP_IOMUXC_SAI1_TXD4__AUDIOMIX_SAI1_TX_DATA04	0xd6
+			MX8MP_IOMUXC_SAI1_TXD5__AUDIOMIX_SAI1_TX_DATA05	0xd6
+			MX8MP_IOMUXC_SAI1_TXD6__AUDIOMIX_SAI1_TX_DATA06	0xd6
+			MX8MP_IOMUXC_SAI1_TXD7__AUDIOMIX_SAI1_TX_DATA07	0xd6
+		>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI3_MCLK__AUDIOMIX_SAI3_MCLK	0xd6
+			MX8MP_IOMUXC_SAI3_RXC__AUDIOMIX_SAI3_RX_BCLK	0xd6
+			MX8MP_IOMUXC_SAI3_RXFS__AUDIOMIX_SAI3_RX_SYNC	0xd6
+			MX8MP_IOMUXC_SAI3_RXD__AUDIOMIX_SAI3_RX_DATA00	0xd6
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX	0x140
+			MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX	0x140
+			MX8MP_IOMUXC_UART3_RXD__UART1_DCE_CTS	0x140
+			MX8MP_IOMUXC_UART3_TXD__UART1_DCE_RTS	0x140
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX	0x140
+			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX	0x140
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ECSPI1_SCLK__UART3_DCE_RX		0x140
+			MX8MP_IOMUXC_ECSPI1_MOSI__UART3_DCE_TX		0x140
+			MX8MP_IOMUXC_ECSPI1_SS0__UART3_DCE_RTS		0x140
+			MX8MP_IOMUXC_ECSPI1_MISO__UART3_DCE_CTS		0x140
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x190
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d0
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d0
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d0
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d0
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d0
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x194
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d4
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d4
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d4
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d4
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d4
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x196
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d6
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d6
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d6
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d6
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d6
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CD_B__GPIO2_IO12	0x1c4
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK	0x190
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD	0x1d0
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d0
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d0
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d0
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d0
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT	0xc0
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK	0x194
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD	0x1d4
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d4
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d4
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d4
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d4
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0xc0
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK	0x196
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD	0x1d6
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d6
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d6
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d6
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d6
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT 0xc0
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x190
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d0
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d0
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d0
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d0
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d0
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d0
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d0
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d0
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d0
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x190
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x194
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d4
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d4
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d4
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d4
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d4
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d4
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d4
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d4
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d4
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x194
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x196
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d6
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d6
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d6
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d6
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d6
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d6
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d6
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d6
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d6
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x196
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO02__WDOG1_WDOG_B	0xc6
+		>;
+	};
+
+	pinctrl_xcvr: xcvrgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SPDIF_EXT_CLK__AUDIOMIX_SPDIF1_EXT_CLK	0xd6
+			MX8MP_IOMUXC_SPDIF_RX__AUDIOMIX_SPDIF1_IN	0xd6
+			MX8MP_IOMUXC_SPDIF_TX__AUDIOMIX_SPDIF1_OUT	0xd6
+		>;
+	};
+};
+
+&lcdif3 {
+	status = "okay";
+};
+
+&micfil {
+	assigned-clocks = <&clk IMX8MP_CLK_PDM>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <196608000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-0 = <&pinctrl_pwm1>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-0 = <&pinctrl_pwm2>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pwm4 {
+	pinctrl-0 = <&pinctrl_pwm4>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&sai1 {
+	clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI1_IPG>, <&clk IMX8MP_CLK_DUMMY>,
+		<&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI1_MCLK1>, <&clk IMX8MP_CLK_DUMMY>,
+		<&clk IMX8MP_CLK_DUMMY>, <&clk IMX8MP_AUDIO_PLL1_OUT>,
+		<&clk IMX8MP_AUDIO_PLL2_OUT>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
+	assigned-clocks = <&clk IMX8MP_CLK_SAI1>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <49152000>;
+	dmas = <&sdma2 0 25 0>, <&sdma2 1 25 0>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai1>;
+	pinctrl-names = "default";
+	fsl,dataline = <2 0xff 0xff>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&sai3 {
+	clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI3_IPG>, <&clk IMX8MP_CLK_DUMMY>,
+		<&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI3_MCLK1>, <&clk IMX8MP_CLK_DUMMY>,
+		<&clk IMX8MP_CLK_DUMMY>, <&clk IMX8MP_AUDIO_PLL1_OUT>,
+		<&clk IMX8MP_AUDIO_PLL2_OUT>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
+	assigned-clocks = <&clk IMX8MP_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <49152000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai3>;
+	pinctrl-names = "default";
+	fsl,sai-asynchronous;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&sdma2 {
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&uart1 {
+	assigned-clocks = <&clk IMX8MP_CLK_UART1>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-0 = <&pinctrl_uart2>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&uart3 {
+	assigned-clocks = <&clk IMX8MP_CLK_UART3>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	pinctrl-0 = <&pinctrl_uart3>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usdhc1 {
+	assigned-clocks = <&clk IMX8MP_CLK_USDHC1>;
+	assigned-clock-rates = <400000000>;
+	bus-width = <4>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	status = "okay";
+};
+
+&usdhc2 {
+	assigned-clocks = <&clk IMX8MP_CLK_USDHC2>;
+	assigned-clock-rates = <400000000>;
+	bus-width = <4>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&usdhc3 {
+	assigned-clocks = <&clk IMX8MP_CLK_USDHC3>;
+	assigned-clock-rates = <400000000>;
+	bus-width = <8>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-0 = <&pinctrl_wdog>;
+	pinctrl-names = "default";
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&xcvr {
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_xcvr>;
+	pinctrl-names = "default";
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-beacon-kit.dts b/arch/arm64/boot/dts/freescale/imx8mp-beacon-kit.dts
index 31c33acb560c6..385aa6bae5202 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-beacon-kit.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-beacon-kit.dts
@@ -266,8 +266,7 @@ mdio {
 		#size-cells = <0>;
 
 		ethphy1: ethernet-phy@3 {
-			compatible = "ethernet-phy-id0022.1640",
-				     "ethernet-phy-ieee802.3-c22";
+			compatible = "ethernet-phy-id0022.1640";
 			reg = <3>;
 			reset-gpios = <&gpio4 18 GPIO_ACTIVE_LOW>;
 			reset-assert-us = <10000>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi
index 6a62cb32e22ec..1007f7db85e93 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-beacon-som.dtsi
@@ -60,8 +60,7 @@ mdio {
 		#size-cells = <0>;
 
 		ethphy0: ethernet-phy@3 {
-			compatible = "ethernet-phy-id0022.1640",
-				     "ethernet-phy-ieee802.3-c22";
+			compatible = "ethernet-phy-id0022.1640";
 			reg = <3>;
 			reset-gpios = <&gpio4 10 GPIO_ACTIVE_LOW>;
 			interrupt-parent = <&gpio1>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts b/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts
index 8290f187b79fd..7bc213499f094 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts
@@ -68,7 +68,7 @@ vmmc: regulator-mmc {
 		regulator-name = "vmmc";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
-		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_LOW>;
 		startup-delay-us = <250>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk3.dts b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk3.dts
index ef012e8365b1f..6ad824a7e07e0 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk3.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk3.dts
@@ -345,7 +345,7 @@ &pinctrl_dhcom_l
 
 	pinctrl_ptn5150: ptn5150grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SAI2_TXC__GPIO4_IO25		0x40000000
+			MX8MP_IOMUXC_SAI2_TXC__GPIO4_IO25		MX8MP_SION
 		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
index b256be710ea12..aedc099377161 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
@@ -134,7 +134,7 @@ reg_audio_pwr: regulator-audio-pwr {
 		enable-active-high;
 	};
 
-	reg_pcie0: regulator-pcie {
+	reg_m2_wlan: reg_pcie0: regulator-pcie {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_pcie0_reg>;
@@ -250,6 +250,13 @@ cpu {
 		};
 	};
 
+	usdhc1_pwrseq: usdhc1_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usdhc1_pwrseq>;
+		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
+	};
+
 	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
@@ -825,6 +832,10 @@ &uart1 { /* BT */
 	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
 	uart-has-rtscts;
 	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
 };
 
 &uart2 {
@@ -858,6 +869,19 @@ &uart3 {
 	status = "okay";
 };
 
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	keep-power-in-suspend;
+	non-removable;
+	wakeup-source;
+	mmc-pwrseq = <&usdhc1_pwrseq>;
+	vmmc-supply = <&reg_m2_wlan>;
+	status = "okay";
+};
+
 &usdhc2 {
 	assigned-clocks = <&clk IMX8MP_CLK_USDHC2>;
 	assigned-clock-rates = <400000000>;
@@ -965,33 +989,33 @@ MX8MP_IOMUXC_SAI1_RXD0__GPIO4_IO02		0x10
 
 	pinctrl_flexcan1: flexcan1grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SPDIF_RX__CAN1_RX          0x154
-			MX8MP_IOMUXC_SPDIF_TX__CAN1_TX          0x154
+			MX8MP_IOMUXC_SPDIF_RX__CAN1_RX		0x154
+			MX8MP_IOMUXC_SPDIF_TX__CAN1_TX		0x154
 		>;
 	};
 
 	pinctrl_flexcan2: flexcan2grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SAI5_MCLK__CAN2_RX         0x154
-			MX8MP_IOMUXC_SAI5_RXD3__CAN2_TX         0x154
+			MX8MP_IOMUXC_SAI5_MCLK__CAN2_RX		0x154
+			MX8MP_IOMUXC_SAI5_RXD3__CAN2_TX		0x154
 		>;
 	};
 
 	pinctrl_flexcan_phy: flexcanphygrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SPDIF_EXT_CLK__GPIO5_IO05  0x154   /* CAN1_STBY */
-			MX8MP_IOMUXC_SAI2_MCLK__GPIO4_IO27      0x154   /* CAN2_STBY */
+			MX8MP_IOMUXC_SPDIF_EXT_CLK__GPIO5_IO05	0x154   /* CAN1_STBY */
+			MX8MP_IOMUXC_SAI2_MCLK__GPIO4_IO27	0x154   /* CAN2_STBY */
 		>;
 	};
 
 	pinctrl_flexspi0: flexspi0grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_ALE__FLEXSPI_A_SCLK           0x1c2
-			MX8MP_IOMUXC_NAND_CE0_B__FLEXSPI_A_SS0_B        0x82
-			MX8MP_IOMUXC_NAND_DATA00__FLEXSPI_A_DATA00      0x82
-			MX8MP_IOMUXC_NAND_DATA01__FLEXSPI_A_DATA01      0x82
-			MX8MP_IOMUXC_NAND_DATA02__FLEXSPI_A_DATA02      0x82
-			MX8MP_IOMUXC_NAND_DATA03__FLEXSPI_A_DATA03      0x82
+			MX8MP_IOMUXC_NAND_ALE__FLEXSPI_A_SCLK		0x1c2
+			MX8MP_IOMUXC_NAND_CE0_B__FLEXSPI_A_SS0_B	0x82
+			MX8MP_IOMUXC_NAND_DATA00__FLEXSPI_A_DATA00	0x82
+			MX8MP_IOMUXC_NAND_DATA01__FLEXSPI_A_DATA01	0x82
+			MX8MP_IOMUXC_NAND_DATA02__FLEXSPI_A_DATA02	0x82
+			MX8MP_IOMUXC_NAND_DATA03__FLEXSPI_A_DATA03	0x82
 		>;
 	};
 
@@ -1044,8 +1068,8 @@ MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA		0x400001c2
 
 	pinctrl_i2c5: i2c5grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SPDIF_RX__I2C5_SDA         0x400001c2
-			MX8MP_IOMUXC_SPDIF_TX__I2C5_SCL         0x400001c2
+			MX8MP_IOMUXC_SPDIF_RX__I2C5_SDA		0x400001c2
+			MX8MP_IOMUXC_SPDIF_TX__I2C5_SCL		0x400001c2
 		>;
 	};
 
@@ -1064,7 +1088,7 @@ MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07	0x40
 
 	pinctrl_pcie0_reg: pcie0reggrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06	0x40
+			MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06	0x140
 		>;
 	};
 
@@ -1164,6 +1188,45 @@ MX8MP_IOMUXC_ECSPI1_MISO__UART3_DCE_CTS		0x140
 		>;
 	};
 
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x190
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d0
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d0
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d0
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d0
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d0
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x194
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d4
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d4
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d4
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d4
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d4
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x196
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d6
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d6
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d6
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d6
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d6
+		>;
+	};
+
+	pinctrl_usdhc1_pwrseq: usdhc1pwrseqgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_RESET_B__GPIO2_IO10	0x140
+		>;
+	};
+
 	pinctrl_usdhc2: usdhc2grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK	0x190
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
index 55690f5e53d7e..d69d78fed4e1b 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
@@ -252,104 +252,140 @@ &usdhc3 {
 &iomuxc {
 	pinctrl_i2c1: i2c1grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL	0x400001c2
-			MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA	0x400001c2
+			MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
+			MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c2: i2c2grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL	0x400001c2
-			MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA	0x400001c2
+			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
+			MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c3: i2c3grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL	0x400001c2
-			MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA	0x400001c2
+			MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
+			MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA	(MX8MP_DSE_X4 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03	0x000001c0
+			MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03
+				(MX8MP_PULL_UP | MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
 		>;
 	};
 
 	pinctrl_pcal6416_0_int: pcal6416-0-int-grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_READY_B__GPIO3_IO16	0x146
+			MX8MP_IOMUXC_NAND_READY_B__GPIO3_IO16
+				(MX8MP_DSE_X6 | MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
 		>;
 	};
 
 	pinctrl_pcal6416_1_int: pcal6416-1-int-grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD1_STROBE__GPIO2_IO11	0x146
+			MX8MP_IOMUXC_SD1_STROBE__GPIO2_IO11
+				(MX8MP_DSE_X6 | MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+
 		>;
 	};
 
 	pinctrl_uart2: uart2grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX	0x140
-			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX	0x140
+			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+
 		>;
 	};
 
 	pinctrl_uart3: uart3grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_ECSPI1_SCLK__UART3_DCE_RX	0x140
-			MX8MP_IOMUXC_ECSPI1_MOSI__UART3_DCE_TX	0x140
-			MX8MP_IOMUXC_ECSPI1_SS0__UART3_DCE_RTS	0x140
-			MX8MP_IOMUXC_ECSPI1_MISO__UART3_DCE_CTS	0x140
+			MX8MP_IOMUXC_ECSPI1_SCLK__UART3_DCE_RX	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_ECSPI1_MOSI__UART3_DCE_TX	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_ECSPI1_SS0__UART3_DCE_RTS	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_ECSPI1_MISO__UART3_DCE_CTS	(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
 		>;
 	};
 
 	pinctrl_usdhc3: usdhc3grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x190
-			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d0
-			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d0
-			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d0
-			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d0
-			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d0
-			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d0
-			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d0
-			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d0
-			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d0
-			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x190
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK
+				(MX8MP_FSEL_FAST | MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD
+				(MX8MP_FSEL_FAST | MX8MP_PULL_UP |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	MX8MP_USDHC_DATA_DEFAULT
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE
+				(MX8MP_FSEL_FAST | MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
 		>;
 	};
 
 	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x194
-			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d4
-			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d4
-			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d4
-			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d4
-			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d4
-			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d4
-			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d4
-			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d4
-			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d4
-			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x194
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK
+				(MX8MP_DSE_X2 | MX8MP_FSEL_FAST |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7
+				(MX8MP_DSE_X2 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE
+				(MX8MP_DSE_X2 | MX8MP_FSEL_FAST |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
 		>;
 	};
 
 	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x196
-			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD	0x1d6
-			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d6
-			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d6
-			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d6
-			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d6
-			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d6
-			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d6
-			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d6
-			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7	0x1d6
-			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x196
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK
+				(MX8MP_DSE_X6 | MX8MP_FSEL_FAST |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD
+				(MX8MP_DSE_X6 | MX8MP_FSEL_FAST | MX8MP_PULL_UP |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7
+				(MX8MP_DSE_X6 | MX8MP_USDHC_DATA_DEFAULT)
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE
+				(MX8MP_DSE_X6 | MX8MP_FSEL_FAST |
+				MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
 		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi
index fa7cb9759d01c..0b4e5f300eb16 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi
@@ -73,7 +73,7 @@ vmmc: regulator-mmc {
 		regulator-name = "vmmc";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
-		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_LOW>;
 		startup-delay-us = <250>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi
index 46916ddc05335..0e5f4607c7c1b 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi
@@ -41,7 +41,7 @@ port@0 {
 				reg = <0>;
 
 				adv7535_from_dsim: endpoint {
-					remote-endpoint = <&dsim_to_adv7535>;
+					remote-endpoint = <&mipi_dsi_out>;
 				};
 			};
 
@@ -71,11 +71,8 @@ &lcdif1 {
 &mipi_dsi {
 	samsung,esc-clock-frequency = <10000000>;
 	status = "okay";
+};
 
-	port@1 {
-		dsim_to_adv7535: endpoint {
-			remote-endpoint = <&adv7535_from_dsim>;
-			attach-bridge;
-		};
-	};
+&mipi_dsi_out {
+	remote-endpoint = <&adv7535_from_dsim>;
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
index a3cba41d2b531..41a2bb74f1565 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
@@ -7,6 +7,7 @@
 /plugin/;
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 #include "imx8mp-pinfunc.h"
 
 &{/} {
@@ -77,12 +78,14 @@ &i2c1 {
 	touchscreen@5d {
 		compatible = "goodix,gt928";
 		reg = <0x5d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_touch>;
 		interrupt-parent = <&gpio1>;
-		interrupts = <6 8>;
-		irq-gpios = <&gpio1 6 0>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		irq-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>;
 		AVDD28-supply = <&reg_vcc_panel>;
 		VDDIO-supply = <&reg_vcc_panel>;
-		reset-gpios = <&gpio1 7 0>;
+		reset-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
 	};
 };
 
@@ -98,6 +101,16 @@ &lvds_bridge {
 	status = "okay";
 };
 
+/* redefine to remove touch controller GPIOs */
+&pinctrl_gpio1 {
+	fsl,pins = <
+		MX8MP_IOMUXC_GPIO1_IO00__GPIO1_IO00		0x19 /* GPIO_A_0 */
+		MX8MP_IOMUXC_GPIO1_IO01__GPIO1_IO01		0x19 /* GPIO_A_1 */
+		MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x19 /* GPIO_A_2 */
+		MX8MP_IOMUXC_GPIO1_IO08__GPIO1_IO08		0x19 /* GPIO_A_5 */
+	>;
+};
+
 &pwm1 {
 	status = "okay";
 };
@@ -108,4 +121,11 @@ pinctrl_panel_stby: panelstbygrp {
 			MX8MP_IOMUXC_SAI3_RXFS__GPIO4_IO28		0x19
 		>;
 	};
+
+	pinctrl_touch: touchgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06		0x19
+			MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07		0x150
+		>;
+	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts b/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts
index 2173a36ff6917..74d620dd06b7b 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts
@@ -249,6 +249,5 @@ &usb3_phy1 {
 };
 
 &usdhc2 {
-	vmmc-supply = <&reg_vdd_3v3>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-pinfunc.h b/arch/arm64/boot/dts/freescale/imx8mp-pinfunc.h
index 16f5899de4152..26e7a9428c4c7 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-pinfunc.h
+++ b/arch/arm64/boot/dts/freescale/imx8mp-pinfunc.h
@@ -36,7 +36,7 @@
 /* long defaults */
 #define MX8MP_USDHC_DATA_DEFAULT (MX8MP_FSEL_FAST | MX8MP_PULL_UP | \
 				  MX8MP_HYS_SCHMITT | MX8MP_PULL_ENABLE)
-#define MX8MP_I2C_DEFAULT (MX8MP_DSE_X6 | MX8MP_PULL_UP | MX8MP_HYS_SCHMITT | \
+#define MX8MP_I2C_DEFAULT (MX8MP_PULL_UP | MX8MP_HYS_SCHMITT | \
 			   MX8MP_PULL_ENABLE | MX8MP_SION)
 
 /*
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts
index b7f69c92b7748..94e2a7df35560 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts
@@ -184,6 +184,9 @@ &eqos {
 	pinctrl-0 = <&pinctrl_eqos>;
 	phy-mode = "rgmii-id";
 	phy-handle = <&ethphy3>;
+	snps,force_thresh_dma_mode;
+	snps,mtl-tx-config = <&mtl_tx_setup>;
+	snps,mtl-rx-config = <&mtl_rx_setup>;
 	status = "okay";
 
 	mdio {
@@ -209,6 +212,70 @@ ethphy3: ethernet-phy@3 {
 			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
 		};
 	};
+
+	mtl_tx_setup: tx-queues-config {
+		snps,tx-queues-to-use = <5>;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+		};
+	};
+
+	mtl_rx_setup: rx-queues-config {
+		snps,rx-queues-to-use = <5>;
+		snps,rx-sched-sp;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+			snps,map-to-dma-channel = <0>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+			snps,map-to-dma-channel = <1>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+			snps,map-to-dma-channel = <2>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+			snps,map-to-dma-channel = <3>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+			snps,map-to-dma-channel = <4>;
+		};
+	};
 };
 
 &fec {
@@ -848,8 +915,8 @@ pinctrl_tlv320aic3x04: tlv320aic3x04grp {
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <MX8MP_IOMUXC_SAI2_RXFS__UART1_DCE_TX	0x14>,
 			   <MX8MP_IOMUXC_SAI2_RXC__UART1_DCE_RX		0x14>,
-			   <MX8MP_IOMUXC_SAI2_RXD0__UART1_DTE_CTS	0x14>,
-			   <MX8MP_IOMUXC_SAI2_TXFS__UART1_DTE_RTS	0x14>;
+			   <MX8MP_IOMUXC_SAI2_RXD0__UART1_DCE_RTS	0x14>,
+			   <MX8MP_IOMUXC_SAI2_TXFS__UART1_DCE_CTS	0x14>;
 	};
 
 	pinctrl_uart1_gpio: uart1gpiogrp {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-g133han01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-g133han01.dtso
index 5058cd9409c75..129b02a69ccf0 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-g133han01.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-g133han01.dtso
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright (c) 2023 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * Copyright (c) 2023-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
  * D-82229 Seefeld, Germany.
  * Author: Alexander Stein
  */
@@ -10,10 +10,6 @@
 
 #include <dt-bindings/clock/imx8mp-clock.h>
 
-&{/} {
-	compatible = "tq,imx8mp-tqma8mpql-mba8mpxl", "tq,imx8mp-tqma8mpql", "fsl,imx8mp";
-};
-
 &backlight_lvds {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtso
index ea44d605342ba..f6aaad91d7f8e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtso
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright (c) 2022 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * Copyright (c) 2022-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
  * D-82229 Seefeld, Germany.
  * Author: Alexander Stein
  */
@@ -8,10 +8,6 @@
 /dts-v1/;
 /plugin/;
 
-&{/} {
-	compatible = "tq,imx8mp-tqma8mpql-mba8mpxl", "tq,imx8mp-tqma8mpql", "fsl,imx8mp";
-};
-
 &backlight_lvds {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
index ad49bf85a04d3..890d1e525a489 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
@@ -336,6 +336,9 @@ &eqos {
 	pinctrl-0 = <&pinctrl_eqos>, <&pinctrl_eqos_phy>;
 	phy-mode = "rgmii-id";
 	phy-handle = <&ethphy3>;
+	snps,force_thresh_dma_mode;
+	snps,mtl-tx-config = <&mtl_tx_setup>;
+	snps,mtl-rx-config = <&mtl_rx_setup>;
 	status = "okay";
 
 	mdio {
@@ -359,6 +362,70 @@ ethphy3: ethernet-phy@3 {
 			interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
 		};
 	};
+
+	mtl_tx_setup: tx-queues-config {
+		snps,tx-queues-to-use = <5>;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+		};
+	};
+
+	mtl_rx_setup: rx-queues-config {
+		snps,rx-queues-to-use = <5>;
+		snps,rx-sched-sp;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+			snps,map-to-dma-channel = <0>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+			snps,map-to-dma-channel = <1>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+			snps,map-to-dma-channel = <2>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+			snps,map-to-dma-channel = <3>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+			snps,map-to-dma-channel = <4>;
+		};
+	};
 };
 
 &fec {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts
index 399230144ce39..87b20b8564583 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106.dts
@@ -302,36 +302,36 @@ MX8MP_IOMUXC_ECSPI1_SS0__GPIO5_IO09
 	pinctrl_i2c2: i2c2grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c2_gpio: i2c2-gpiogrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C2_SCL__GPIO5_IO16
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_I2C2_SDA__GPIO5_IO17
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c4: i2c4grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_ECSPI2_MISO__I2C4_SCL
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_ECSPI2_SS0__I2C4_SDA
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c4_gpio: i2c4-gpiogrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_ECSPI2_MISO__GPIO5_IO12
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_ECSPI2_SS0__GPIO5_IO13
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi
index 761ee046eb72e..bf49ae942d411 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81.dtsi
@@ -425,18 +425,18 @@ MX8MP_IOMUXC_SAI2_RXC__GPIO4_IO22
 	pinctrl_i2c1: i2c1grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
 	pinctrl_i2c1_gpio: i2c1-gpiogrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C1_SCL__GPIO5_IO14
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 			MX8MP_IOMUXC_I2C1_SDA__GPIO5_IO15
-				MX8MP_I2C_DEFAULT
+				(MX8MP_DSE_X6 | MX8MP_I2C_DEFAULT)
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts
index 077c5cd2586f7..4533a84fb0b95 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts
@@ -7,7 +7,7 @@
 
 &a53_opp_table {
 	opp-1000000000 {
-		opp-microvolt = <950000>;
+		opp-microvolt = <1000000>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
index eee390c272105..f5d529c5baf3e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
@@ -880,9 +880,9 @@ buck1_reg: BUCK1 {
 				regulator-max-microvolt = <1300000>;
 				regulator-boot-on;
 				regulator-ramp-delay = <1250>;
-				rohm,dvs-run-voltage = <880000>;
-				rohm,dvs-idle-voltage = <820000>;
-				rohm,dvs-suspend-voltage = <810000>;
+				rohm,dvs-run-voltage = <900000>;
+				rohm,dvs-idle-voltage = <850000>;
+				rohm,dvs-suspend-voltage = <850000>;
 				regulator-always-on;
 			};
 
@@ -892,8 +892,8 @@ buck2_reg: BUCK2 {
 				regulator-max-microvolt = <1300000>;
 				regulator-boot-on;
 				regulator-ramp-delay = <1250>;
-				rohm,dvs-run-voltage = <950000>;
-				rohm,dvs-idle-voltage = <850000>;
+				rohm,dvs-run-voltage = <1000000>;
+				rohm,dvs-idle-voltage = <900000>;
 				regulator-always-on;
 			};
 
@@ -902,14 +902,14 @@ buck3_reg: BUCK3 {
 				regulator-min-microvolt = <700000>;
 				regulator-max-microvolt = <1300000>;
 				regulator-boot-on;
-				rohm,dvs-run-voltage = <850000>;
+				rohm,dvs-run-voltage = <900000>;
 			};
 
 			buck4_reg: BUCK4 {
 				regulator-name = "buck4";
 				regulator-min-microvolt = <700000>;
 				regulator-max-microvolt = <1300000>;
-				rohm,dvs-run-voltage = <930000>;
+				rohm,dvs-run-voltage = <1000000>;
 			};
 
 			buck5_reg: BUCK5 {
@@ -1448,13 +1448,3 @@ &wdog1 {
 	fsl,ext-reset-output;
 	status = "okay";
 };
-
-&a53_opp_table {
-	opp-1000000000 {
-		opp-microvolt = <850000>;
-	};
-
-	opp-1500000000 {
-		opp-microvolt = <950000>;
-	};
-};
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq-mba8mx-lvds-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq-mba8mx-lvds-tm070jvhg33.dtso
index 306977d6ba0cb..78f24f72cc690 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq-mba8mx-lvds-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq-mba8mx-lvds-tm070jvhg33.dtso
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
 /*
- * Copyright (c) 2019-2023 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * Copyright (c) 2019-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
  * D-82229 Seefeld, Germany.
  * Author: Alexander Stein
  */
@@ -10,10 +10,6 @@
 
 #include <dt-bindings/gpio/gpio.h>
 
-&{/} {
-	compatible = "tq,imx8mq-tqma8mq-mba8mx", "tq,imx8mq-tqma8mq", "fsl,imx8mq";
-};
-
 &backlight_lvds {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
index 607962f807beb..6a25e219832ce 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
@@ -1632,7 +1632,7 @@ gpu: gpu@38000000 {
 			                         <&clk IMX8MQ_GPU_PLL_OUT>,
 			                         <&clk IMX8MQ_GPU_PLL>;
 			assigned-clock-rates = <800000000>, <800000000>,
-			                       <800000000>, <800000000>, <0>;
+			                       <800000000>, <400000000>, <0>;
 			power-domains = <&pgc_gpu>;
 		};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
index bd6e0aa27efe9..f2c94cdb682b9 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8qm-ss-hsio.dtsi
@@ -20,8 +20,9 @@ pcie0: pciea: pcie@5f000000 {
 		ranges = <0x81000000 0 0x00000000 0x4ff80000 0 0x00010000>,
 			 <0x82000000 0 0x40000000 0x40000000 0 0x0ff00000>;
 		#interrupt-cells = <1>;
-		interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "msi";
+		interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "msi", "dma";
 		#address-cells = <3>;
 		#size-cells = <2>;
 		clocks = <&pciea_lpcg IMX_LPCG_CLK_6>,
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
index 03f460d62f7a5..c083b97476a51 100644
--- a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
@@ -23,6 +23,7 @@ aliases {
 		i2c2 = &lpi2c3;
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
+		mmc2 = &usdhc3;
 		rtc0 = &bbnsm_rtc;
 		serial0 = &lpuart1;
 		serial1 = &lpuart2;
@@ -57,6 +58,15 @@ reg_audio_pwr: regulator-audio-pwr {
 		enable-active-high;
 	};
 
+	reg_m2_pwr: regulator-m2-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "M.2-power";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pcal6524 13 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_usdhc2_vmmc: regulator-usdhc2 {
 		compatible = "regulator-fixed";
 		off-on-delay-us = <12000>;
@@ -69,6 +79,23 @@ reg_usdhc2_vmmc: regulator-usdhc2 {
 		enable-active-high;
 	};
 
+	reg_usdhc3_vmmc: regulator-usdhc3 {
+		compatible = "regulator-fixed";
+		regulator-name = "WLAN_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_m2_pwr>;
+		gpio = <&pcal6524 20 GPIO_ACTIVE_HIGH>;
+		/*
+		 * IW612 wifi chip needs more delay than other wifi chips to complete
+		 * the host interface initialization after power up, otherwise the
+		 * internal state of IW612 may be unstable, resulting in the failure of
+		 * the SDIO3.0 switch voltage.
+		 */
+		startup-delay-us = <20000>;
+		enable-active-high;
+	};
+
 	reserved-memory {
 		ranges;
 		#address-cells = <2>;
@@ -144,6 +171,11 @@ cpu {
 			};
 		};
 	};
+
+	usdhc3_pwrseq: usdhc3_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pcal6524 12 GPIO_ACTIVE_LOW>;
+	};
 };
 
 &adc1 {
@@ -246,6 +278,12 @@ pcal6524: gpio@22 {
 		interrupt-parent = <&gpio3>;
 		pinctrl-0 = <&pinctrl_pcal6524>;
 		pinctrl-names = "default";
+
+		m2-pcm-level-shifter-hog {
+			gpio-hog;
+			gpios = <19 GPIO_ACTIVE_HIGH>;
+			output-high;
+		};
 	};
 
 	pmic@25 {
@@ -514,6 +552,7 @@ &usdhc1 {
 	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
 	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	fsl,tuning-step = <1>;
 	status = "okay";
 };
 
@@ -528,6 +567,22 @@ &usdhc2 {
 	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
 	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
 	vmmc-supply = <&reg_usdhc2_vmmc>;
+	fsl,tuning-step = <1>;
+	status = "okay";
+};
+
+&usdhc3 {
+	bus-width = <4>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&usdhc3_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	pinctrl-3 = <&pinctrl_usdhc3_sleep>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc3_vmmc>;
+	wakeup-source;
 	status = "okay";
 };
 
@@ -850,4 +905,47 @@ MX91_PAD_SD2_VSELECT__GPIO3_IO19                        0x51e
 		>;
 	};
 
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK                            0x1582
+			MX91_PAD_SD3_CMD__USDHC3_CMD                            0x1382
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0                        0x1382
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1                        0x1382
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2                        0x1382
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3                        0x1382
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK                            0x158e
+			MX91_PAD_SD3_CMD__USDHC3_CMD                            0x138e
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0                        0x138e
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1                        0x138e
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2                        0x138e
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3                        0x138e
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK                            0x15fe
+			MX91_PAD_SD3_CMD__USDHC3_CMD                            0x13fe
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0                        0x13fe
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1                        0x13fe
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2                        0x13fe
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3                        0x13fe
+		>;
+	};
+
+	pinctrl_usdhc3_sleep: usdhc3-sleepgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__GPIO3_IO20                             0x31e
+			MX91_PAD_SD3_CMD__GPIO3_IO21                             0x31e
+			MX91_PAD_SD3_DATA0__GPIO3_IO22                           0x31e
+			MX91_PAD_SD3_DATA1__GPIO3_IO23                           0x31e
+			MX91_PAD_SD3_DATA2__GPIO3_IO24                           0x31e
+			MX91_PAD_SD3_DATA3__GPIO3_IO25                           0x31e
+		>;
+	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
new file mode 100644
index 0000000000000..62dc1dedfb0ee
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
@@ -0,0 +1,769 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2025 NXP
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/usb/pd.h>
+#include "imx91.dtsi"
+
+/ {
+	compatible = "fsl,imx91-11x11-frdm-s", "fsl,imx91";
+	model = "NXP FRDM-IMX91S board";
+
+	aliases {
+		ethernet0 = &eqos;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		gpio3 = &gpio4;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		rtc0 = &pcf2131;
+		rtc1 = &bbnsm_rtc;
+		serial0 = &lpuart1;
+		serial4 = &lpuart5;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	flexcan1_phy: can-phy {
+		compatible = "nxp,tja1051";
+		#phy-cells = <0>;
+		max-bitrate = <5000000>;
+		silent-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&pinctrl_gpio_key>;
+		pinctrl-names = "default";
+
+		button {
+			interrupt-parent = <&gpio3>;
+			interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+			gpios = <&gpio3 26 GPIO_PULL_UP>;
+			label = "User Button";
+			linux,code = <BTN_1>;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			default-state = "on";
+			gpios = <&pcal6524 7 GPIO_ACTIVE_LOW>;
+			label = "green:status";
+		};
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "vref_1v8";
+	};
+
+	reg_usdhc1_vmmc: regulator-usdhc1 {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "WLAN_EN";
+		startup-delay-us = <20000>;
+		gpio = <&pcal6524 3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		off-on-delay-us = <12000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "VSD_3V3";
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	reg_usb_vbus: regulator-vbus {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "USB_VBUS";
+		gpio = <&pcal6524 15 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x20000000>;
+			reusable;
+			size = <0 0x2000000>;
+			linux,cma-default;
+		};
+	};
+
+	soc@0 {
+		bootph-all;
+		bootph-pre-ram;
+	};
+
+	sound-mqs {
+		compatible = "fsl,imx-audio-mqs";
+		model = "mqs-audio";
+		audio-codec = <&mqs1>;
+		audio-cpu = <&sai1>;
+	};
+
+	usdhc1_pwrseq: usdhc1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pcal6524 18 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&aips1 {
+	bootph-pre-ram;
+	bootph-all;
+};
+
+&aips2 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&aips3 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&clk {
+	bootph-all;
+	bootph-pre-ram;
+};
+
+&clk_ext1 {
+	bootph-all;
+	bootph-pre-ram;
+};
+
+&eqos {
+	phy-handle = <&ethphy1>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-1 = <&pinctrl_eqos_sleep>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy1: ethernet-phy@4 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <4>;
+		};
+	};
+};
+
+&flexcan1 {
+	phys = <&flexcan1_phy>;
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	pinctrl-1 = <&pinctrl_flexcan1_sleep>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+};
+
+&flexspi1 {
+	pinctrl-0 = <&pinctrl_flexspi1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	flash@0 {
+		compatible = "spi-nand";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <104000000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				reg = <0x000000 0x800000>;
+				label = "bootloader";
+			};
+
+			partition@1 {
+				reg = <0x800000 0x800000>;
+				label = "env";
+			};
+
+			partition@2 {
+				reg = <0x1000000 0x2800000>;
+				label = "kernel";
+			};
+
+			partition@3 {
+				reg = <0x3800000 0x20000>;
+				label = "dtb";
+			};
+
+			partition@4 {
+				reg = <0x3820000 0xc7e0000>;
+				label = "rootfs";
+				linux,rootfs;
+			};
+		};
+	};
+};
+
+&gpio1 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&gpio2 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&gpio3 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&gpio4 {
+	bootph-pre-ram;
+	bootph-some-ram;
+};
+
+&iomuxc {
+	bootph-pre-ram;
+	bootph-some-ram;
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX91_PAD_ENET1_MDC__ENET1_MDC				0x57e
+			MX91_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
+			MX91_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
+			MX91_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
+			MX91_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
+			MX91_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
+			MX91_PAD_ENET1_RXC__ENET_QOS_RGMII_RXC			0x5fe
+			MX91_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
+			MX91_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
+			MX91_PAD_ENET1_TD1__ENET1_RGMII_TD1			0x57e
+			MX91_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
+			MX91_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
+			MX91_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x5fe
+			MX91_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
+		>;
+	};
+
+	pinctrl_eqos_sleep: eqossleepgrp {
+		fsl,pins = <
+			MX91_PAD_ENET1_MDC__GPIO4_IO0				0x31e
+			MX91_PAD_ENET1_MDIO__GPIO4_IO1				0x31e
+			MX91_PAD_ENET1_RD0__GPIO4_IO10				0x31e
+			MX91_PAD_ENET1_RD1__GPIO4_IO11				0x31e
+			MX91_PAD_ENET1_RD2__GPIO4_IO12				0x31e
+			MX91_PAD_ENET1_RD3__GPIO4_IO13				0x31e
+			MX91_PAD_ENET1_RXC__GPIO4_IO9				0x31e
+			MX91_PAD_ENET1_RX_CTL__GPIO4_IO8			0x31e
+			MX91_PAD_ENET1_TD0__GPIO4_IO5				0x31e
+			MX91_PAD_ENET1_TD1__GPIO4_IO4				0x31e
+			MX91_PAD_ENET1_TD2__GPIO4_IO3				0x31e
+			MX91_PAD_ENET1_TD3__GPIO4_IO2				0x31e
+			MX91_PAD_ENET1_TXC__GPIO4_IO7				0x31e
+			MX91_PAD_ENET1_TX_CTL__GPIO4_IO6			0x31e
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO28__CAN1_TX				0x139e
+			MX91_PAD_GPIO_IO29__CAN1_RX				0x139e
+			MX91_PAD_PDM_BIT_STREAM1__GPIO1_IO10			0x31e
+		>;
+	};
+
+	pinctrl_flexcan1_sleep: flexcan1sleepgrp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO28__GPIO2_IO28				0x31e
+			MX91_PAD_GPIO_IO29__GPIO2_IO29				0x31e
+			MX91_PAD_PDM_BIT_STREAM1__GPIO1_IO10			0x31e
+		>;
+	};
+
+	pinctrl_flexspi1: flexspi1grp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__FLEXSPI1_A_SCLK			0x3fe
+			MX91_PAD_SD3_CMD__FLEXSPI1_A_SS0_B			0x3fe
+			MX91_PAD_SD3_DATA0__FLEXSPI1_A_DATA0			0x3fe
+			MX91_PAD_SD3_DATA1__FLEXSPI1_A_DATA1			0x3fe
+			MX91_PAD_SD3_DATA2__FLEXSPI1_A_DATA2			0x3fe
+			MX91_PAD_SD3_DATA3__FLEXSPI1_A_DATA3			0x3fe
+		>;
+	};
+
+	pinctrl_gpio_key: gpiokeysgrp {
+		fsl,pins = <
+			MX91_PAD_CCM_CLKO1__GPIO3_IO26				0x31e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX91_PAD_I2C1_SCL__LPI2C1_SCL				0x40000b9e
+			MX91_PAD_I2C1_SDA__LPI2C1_SDA				0x40000b9e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			MX91_PAD_I2C2_SCL__LPI2C2_SCL				0x40000b9e
+			MX91_PAD_I2C2_SDA__LPI2C2_SDA				0x40000b9e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			MX91_PAD_PDM_CLK__MQS1_LEFT				0x31e
+			MX91_PAD_PDM_BIT_STREAM0__MQS1_RIGHT			0x31e
+		>;
+	};
+
+	pinctrl_pcal6524: pcal6524grp {
+		fsl,pins = <
+			MX91_PAD_CCM_CLKO2__GPIO3_IO27				0x31e
+			MX91_PAD_SD1_DATA5__GPIO3_IO15				0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_RESET_B__GPIO3_IO7				0x31e
+		>;
+		bootph-pre-ram;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX91_PAD_UART1_RXD__LPUART1_RX				0x31e
+			MX91_PAD_UART1_TXD__LPUART1_TX				0x31e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX91_PAD_DAP_TDO_TRACESWO__LPUART5_TX			0x31e
+			MX91_PAD_DAP_TDI__LPUART5_RX				0x31e
+			MX91_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B			0x31e
+			MX91_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B			0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x1582
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x1382
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x1382
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x1382
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x1382
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x1382
+			MX91_PAD_SD1_DATA4__GPIO3_IO14				0x31e
+			MX91_PAD_SD1_STROBE__GPIO3_IO18				0x31e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x158e
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x138e
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x138e
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x138e
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x138e
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x138e
+			MX91_PAD_SD1_DATA4__GPIO3_IO14				0x31e
+			MX91_PAD_SD1_STROBE__GPIO3_IO18				0x31e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x15fe
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x13fe
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x13fe
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x13fe
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x13fe
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x13fe
+			MX91_PAD_SD1_DATA4__GPIO3_IO14				0x31e
+			MX91_PAD_SD1_STROBE__GPIO3_IO18				0x31e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CD_B__GPIO3_IO0				0x31e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleepgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CD_B__GPIO3_IO0				0x51e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK				0x1582
+			MX91_PAD_SD2_CMD__USDHC2_CMD				0x1382
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0			0x1382
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1			0x1382
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2			0x1382
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3			0x1382
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+		bootph-pre-ram;
+		bootph-some-ram;
+	};
+
+	pinctrl_usdhc2_sleep: usdhc2sleepgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__GPIO3_IO1				0x51e
+			MX91_PAD_SD2_CMD__GPIO3_IO2				0x51e
+			MX91_PAD_SD2_DATA0__GPIO3_IO3				0x51e
+			MX91_PAD_SD2_DATA1__GPIO3_IO4				0x51e
+			MX91_PAD_SD2_DATA2__GPIO3_IO5				0x51e
+			MX91_PAD_SD2_DATA3__GPIO3_IO6				0x51e
+			MX91_PAD_SD2_VSELECT__GPIO3_IO19			0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK				0x158e
+			MX91_PAD_SD2_CMD__USDHC2_CMD				0x138e
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0			0x138e
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1			0x138e
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2			0x138e
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK				0x15fe
+			MX91_PAD_SD2_CMD__USDHC2_CMD				0x13fe
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0			0x13fe
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1			0x13fe
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2			0x13fe
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3			0x13fe
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX91_PAD_WDOG_ANY__WDOG1_WDOG_ANY			0x31e
+		>;
+	};
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-names = "default";
+	bootph-pre-ram;
+	bootph-some-ram;
+	status = "okay";
+
+	pcf2131: rtc@53 {
+		compatible = "nxp,pcf2131";
+		reg = <0x53>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-names = "default";
+	bootph-pre-ram;
+	bootph-some-ram;
+	status = "okay";
+
+	pcal6524: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		pinctrl-0 = <&pinctrl_pcal6524>;
+		pinctrl-names = "default";
+		reset-gpios = <&gpio3 15 GPIO_ACTIVE_LOW>;
+	};
+
+	pmic@32 {
+		compatible = "nxp,pf9453";
+		reg = <0x32>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
+		bootph-pre-ram;
+		bootph-some-ram;
+
+		regulators {
+			bootph-pre-ram;
+			bootph-some-ram;
+
+			buck1: BUCK1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3775000>;
+				regulator-min-microvolt = <600000>;
+				regulator-name = "BUCK1";
+			};
+
+			buck2: BUCK2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <2187500>;
+				regulator-min-microvolt = <600000>;
+				regulator-name = "BUCK2";
+				regulator-ramp-delay = <12500>;
+			};
+
+			buck3: BUCK3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3775000>;
+				regulator-min-microvolt = <600000>;
+				regulator-name = "BUCK3";
+			};
+
+			buck4: BUCK4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3775000>;
+				regulator-min-microvolt = <600000>;
+				regulator-name = "BUCK4";
+			};
+
+			ldo1: LDO1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <800000>;
+				regulator-name = "LDO1";
+			};
+
+			ldo2: LDO2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1950000>;
+				regulator-min-microvolt = <500000>;
+				regulator-name = "LDO2";
+			};
+
+			ldo_snvs: LDO-SNVS {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3400000>;
+				regulator-min-microvolt = <1200000>;
+				regulator-name = "LDO-SNVS";
+			};
+		};
+	};
+
+	ptn5110: tcpc@52 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x52>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+
+		typec1_con: connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			op-sink-microwatt = <15000000>;
+			power-role = "dual";
+			self-powered;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "sink";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec1_dr_sw: endpoint {
+						remote-endpoint = <&usb1_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&lpuart1 {
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	bootph-pre-ram;
+	bootph-some-ram;
+	status = "okay";
+};
+
+&lpuart5 {
+	pinctrl-0 = <&pinctrl_uart5>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&mqs1 {
+	clocks = <&clk IMX93_CLK_MQS1_GATE>;
+	clock-names = "mclk";
+	pinctrl-0 = <&pinctrl_mqs1>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&osc_32k {
+	bootph-all;
+	bootph-pre-ram;
+};
+
+&osc_24m {
+	bootph-all;
+	bootph-pre-ram;
+};
+
+&sai1 {
+	clocks = <&clk IMX93_CLK_SAI1_IPG>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_SAI1_GATE>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_DUMMY>, <&clk IMX93_CLK_AUDIO_PLL>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k";
+	assigned-clocks = <&clk IMX93_CLK_SAI1>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <24576000>;
+	#sound-dai-cells = <0>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&usbotg1 {
+	adp-disable;
+	disable-over-current;
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_usb_vbus>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+};
+
+&usdhc1 {
+	bus-width = <8>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&usdhc1_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	vmmc-supply = <&reg_usdhc1_vmmc>;
+	wakeup-source;
+	bootph-pre-ram;
+	bootph-some-ram;
+	status = "okay";
+};
+
+&usdhc2 {
+	bus-width = <4>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	no-mmc;
+	no-sdio;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	bootph-pre-ram;
+	bootph-some-ram;
+	status = "okay";
+};
+
+&wdog3 {
+	pinctrl-0 = <&pinctrl_wdog>;
+	pinctrl-names = "default";
+	fsl,ext-reset-output;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
index 5792952b7a8e1..c99d7bc168483 100644
--- a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
@@ -272,20 +272,20 @@ pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
 	/* enable SION for data and cmd pad due to ERR052021 */
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = /* PD | FSEL 3 | DSE X5 */
-			   <MX91_PAD_SD1_CLK__USDHC1_CLK		0x5be>,
+			   <MX91_PAD_SD1_CLK__USDHC1_CLK		0x59e>,
 			   /* HYS | FSEL 0 | no drive */
 			   <MX91_PAD_SD1_STROBE__USDHC1_STROBE		0x1000>,
 			   /* HYS | FSEL 3 | X5 */
-			   <MX91_PAD_SD1_CMD__USDHC1_CMD		0x400011be>,
+			   <MX91_PAD_SD1_CMD__USDHC1_CMD		0x4000139e>,
 			   /* HYS | FSEL 3 | X4 */
-			   <MX91_PAD_SD1_DATA0__USDHC1_DATA0		0x4000119e>,
-			   <MX91_PAD_SD1_DATA1__USDHC1_DATA1		0x4000119e>,
-			   <MX91_PAD_SD1_DATA2__USDHC1_DATA2		0x4000119e>,
-			   <MX91_PAD_SD1_DATA3__USDHC1_DATA3		0x4000119e>,
-			   <MX91_PAD_SD1_DATA4__USDHC1_DATA4		0x4000119e>,
-			   <MX91_PAD_SD1_DATA5__USDHC1_DATA5		0x4000119e>,
-			   <MX91_PAD_SD1_DATA6__USDHC1_DATA6		0x4000119e>,
-			   <MX91_PAD_SD1_DATA7__USDHC1_DATA7		0x4000119e>;
+			   <MX91_PAD_SD1_DATA0__USDHC1_DATA0		0x4000139e>,
+			   <MX91_PAD_SD1_DATA1__USDHC1_DATA1		0x4000139e>,
+			   <MX91_PAD_SD1_DATA2__USDHC1_DATA2		0x4000139e>,
+			   <MX91_PAD_SD1_DATA3__USDHC1_DATA3		0x4000139e>,
+			   <MX91_PAD_SD1_DATA4__USDHC1_DATA4		0x4000139e>,
+			   <MX91_PAD_SD1_DATA5__USDHC1_DATA5		0x4000139e>,
+			   <MX91_PAD_SD1_DATA6__USDHC1_DATA6		0x4000139e>,
+			   <MX91_PAD_SD1_DATA7__USDHC1_DATA7		0x4000139e>;
 	};
 
 	pinctrl_wdog: wdoggrp {
diff --git a/arch/arm64/boot/dts/freescale/imx91.dtsi b/arch/arm64/boot/dts/freescale/imx91.dtsi
index f075592bfc01f..d63569b39bbc5 100644
--- a/arch/arm64/boot/dts/freescale/imx91.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91.dtsi
@@ -11,7 +11,7 @@ thermal-zones {
 		cpu-thermal {
 			polling-delay-passive = <250>;
 			polling-delay = <2000>;
-			thermal-sensors = <&tmu 0>;
+			thermal-sensors = <&tmu>;
 
 			trips {
 				cpu_alert: cpu-alert {
diff --git a/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi b/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
index 7958cef353766..5a8813df6bc99 100644
--- a/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
@@ -1122,8 +1122,62 @@ media_blk_ctrl: system-controller@4ac10000 {
 				 <&clk IMX93_CLK_MIPI_DSI_GATE>;
 			clock-names = "apb", "axi", "nic", "disp", "cam",
 				      "pxp", "lcdif", "isi", "csi", "dsi";
+			assigned-clocks = <&clk IMX93_CLK_MEDIA_AXI>,
+					  <&clk IMX93_CLK_MEDIA_APB>,
+					  <&clk IMX93_CLK_MEDIA_DISP_PIX>;
+			assigned-clock-parents = <&clk IMX93_CLK_SYS_PLL_PFD1>,
+						 <&clk IMX93_CLK_SYS_PLL_PFD1_DIV2>,
+						 <&clk IMX93_CLK_VIDEO_PLL>;
+			assigned-clock-rates = <400000000>, <133333333>;
 			#power-domain-cells = <1>;
 			status = "disabled";
+
+			dpi_bridge: dpi-bridge {
+				compatible = "nxp,imx93-pdfc";
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						dpi_from_lcdif: endpoint {
+							remote-endpoint = <&lcdif_to_dpi>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						dpi_to_panel: endpoint {
+						};
+					};
+				};
+			};
+		};
+
+		lcdif: display-controller@4ae30000 {
+			compatible = "fsl,imx93-lcdif";
+			reg = <0x4ae30000 0x23c>;
+			interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clk IMX93_CLK_MEDIA_DISP_PIX>,
+				 <&clk IMX93_CLK_LCDIF_GATE>,
+				 <&clk IMX93_CLK_MEDIA_AXI>;
+			clock-names = "pix", "axi", "disp_axi";
+			power-domains = <&media_blk_ctrl IMX93_MEDIABLK_PD_LCDIF>;
+			status = "disabled";
+
+			port {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				lcdif_to_dpi: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&dpi_from_lcdif>;
+				};
+			};
 		};
 
 		usbotg1: usb@4c100000 {
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
new file mode 100644
index 0000000000000..7d3fc4ad7b8b7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
@@ -0,0 +1,861 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022,2026 NXP
+ */
+
+#include <dt-bindings/usb/pd.h>
+
+/ {
+	aliases {
+		ethernet0 = &fec;
+		ethernet1 = &eqos;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		rtc0 = &bbnsm_rtc;
+		serial0 = &lpuart1;
+		serial1 = &lpuart2;
+		serial2 = &lpuart3;
+		serial3 = &lpuart4;
+		serial4 = &lpuart5;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			alloc-ranges = <0 0x80000000 0 0x40000000>;
+			size = <0 0x10000000>;
+			linux,cma-default;
+		};
+
+		vdev0vring0: vdev0vring0@a4000000 {
+			reg = <0 0xa4000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@a4008000 {
+			reg = <0 0xa4008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@a4010000 {
+			reg = <0 0xa4010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@a4018000 {
+			reg = <0 0xa4018000 0 0x8000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@2021e000 {
+			reg = <0 0x2021e000 0 0x1000>;
+			no-map;
+		};
+
+		vdevbuffer: vdevbuffer@a4020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa4020000 0 0x100000>;
+			no-map;
+		};
+
+	};
+
+	flexcan_phy: can-phy {
+		compatible = "nxp,tja1057";
+		#phy-cells = <0>;
+		max-bitrate = <5000000>;
+		silent-gpios = <&adp5585 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_vdd_12v: regulator-vdd-12v {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_12V";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		gpio = <&pcal6524 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref_1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	reg_audio_pwr: regulator-audio-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "audio-pwr";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&adp5585 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		off-on-delay-us = <12000>;
+		enable-active-high;
+	};
+
+	backlight_lvds: backlight-lvds {
+		compatible = "pwm-backlight";
+		pwms = <&adp5585 0 100000 0>;
+		brightness-levels = <0 100>;
+		num-interpolated-steps = <100>;
+		default-brightness-level = <100>;
+		power-supply = <&reg_vdd_12v>;
+		enable-gpios = <&adp5585 9 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+
+	sound-wm8962 {
+		compatible = "fsl,imx-audio-wm8962";
+		model = "wm8962-audio";
+		audio-cpu = <&sai3>;
+		audio-codec = <&wm8962>;
+		hp-det-gpio = <&pcal6524 4 GPIO_ACTIVE_HIGH>;
+		audio-routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"Ext Spk", "SPKOUTL",
+			"Ext Spk", "SPKOUTR",
+			"AMIC", "MICBIAS",
+			"IN3R", "AMIC",
+			"IN1R", "AMIC";
+	};
+
+	sound-xcvr {
+		compatible = "fsl,imx-audio-card";
+		model = "imx-audio-xcvr";
+
+		pri-dai-link {
+			link-name = "XCVR PCM";
+
+			cpu {
+				sound-dai = <&xcvr>;
+			};
+		};
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&cm33 {
+	mbox-names = "tx", "rx", "rxdb";
+	mboxes = <&mu1 0 1>,
+		 <&mu1 1 1>,
+		 <&mu1 3 1>;
+	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
+	status = "okay";
+};
+
+&eqos {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-1 = <&pinctrl_eqos_sleep>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy1>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy1: ethernet-phy@1 {
+			reg = <1>;
+			reset-gpios = <&pcal6524 15 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <80000>;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_fec>;
+	pinctrl-1 = <&pinctrl_fec_sleep>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy2>;
+	fsl,magic-packet;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy2: ethernet-phy@2 {
+			reg = <2>;
+			reset-gpios = <&pcal6524 16 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <80000>;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	phys = <&flexcan_phy>;
+	status = "okay";
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	status = "okay";
+
+	wm8962: codec@1a {
+		compatible = "wlf,wm8962";
+		reg = <0x1a>;
+		clocks = <&clk IMX93_CLK_SAI3_GATE>;
+		DCVDD-supply = <&reg_audio_pwr>;
+		DBVDD-supply = <&reg_audio_pwr>;
+		AVDD-supply = <&reg_audio_pwr>;
+		CPVDD-supply = <&reg_audio_pwr>;
+		MICVDD-supply = <&reg_audio_pwr>;
+		PLLVDD-supply = <&reg_audio_pwr>;
+		SPKVDD1-supply = <&reg_audio_pwr>;
+		SPKVDD2-supply = <&reg_audio_pwr>;
+		gpio-cfg = <
+			0x0000 /* 0:Default */
+			0x0000 /* 1:Default */
+			0x0000 /* 2:FN_DMICCLK */
+			0x0000 /* 3:Default */
+			0x0000 /* 4:FN_DMICCDAT */
+			0x0000 /* 5:Default */
+		>;
+	};
+
+	inertial-meter@6a {
+		compatible = "st,lsm6dso";
+		reg = <0x6a>;
+	};
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	status = "okay";
+
+	pcal6524: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pcal6524>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pmic@25 {
+		compatible = "nxp,pca9451a";
+		reg = <0x25>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-min-microvolt = <610000>;
+				regulator-max-microvolt = <950000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <670000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-min-microvolt = <1620000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-min-microvolt = <1620000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-min-microvolt = <1060000>;
+				regulator-max-microvolt = <1140000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-min-microvolt = <1620000>;
+				regulator-max-microvolt = <1980000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <840000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+
+	adp5585: io-expander@34 {
+		compatible = "adi,adp5585-00", "adi,adp5585";
+		reg = <0x34>;
+		vdd-supply = <&buck4>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-reserved-ranges = <5 1>;
+		#pwm-cells = <3>;
+	};
+};
+
+&lpi2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+	status = "okay";
+
+	adp5585_isp: io-expander@34 {
+		compatible = "adi,adp5585-01", "adi,adp5585";
+		reg = <0x34>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		#pwm-cells = <3>;
+	};
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+
+		typec1_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			power-role = "dual";
+			data-role = "dual";
+			try-power-role = "sink";
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			op-sink-microwatt = <15000000>;
+			self-powered;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec1_dr_sw: endpoint {
+						remote-endpoint = <&usb1_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+
+	ptn5110_2: tcpc@51 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x51>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+
+		typec2_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			power-role = "dual";
+			data-role = "dual";
+			try-power-role = "sink";
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			op-sink-microwatt = <15000000>;
+			self-powered;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec2_dr_sw: endpoint {
+						remote-endpoint = <&usb2_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+
+	pcf2131: rtc@53 {
+		compatible = "nxp,pcf2131";
+		reg = <0x53>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&lpuart1 { /* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&lpuart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&mu1 {
+	status = "okay";
+};
+
+&mu2 {
+	status = "okay";
+};
+
+&sai3 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_sai3>;
+	pinctrl-1 = <&pinctrl_sai3_sleep>;
+	assigned-clocks = <&clk IMX93_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <12288000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	status = "okay";
+
+	port {
+		usb2_drd_sw: endpoint {
+			remote-endpoint = <&typec2_dr_sw>;
+		};
+	};
+};
+
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	fsl,tuning-step = <1>;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
+	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	bus-width = <4>;
+	fsl,tuning-step = <1>;
+	status = "okay";
+	no-mmc;
+};
+
+&wdog3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&xcvr {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_spdif>;
+	pinctrl-1 = <&pinctrl_spdif_sleep>;
+	assigned-clocks = <&clk IMX93_CLK_SPDIF>,
+			 <&clk IMX93_CLK_AUDIO_XCVR>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>,
+			 <&clk IMX93_CLK_SYS_PLL_PFD1_DIV2>;
+	assigned-clock-rates = <12288000>, <200000000>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__ENET_QOS_MDC			0x57e
+			MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
+			MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
+			MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
+			MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
+			MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
+			MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x58e
+			MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
+			MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
+			MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x57e
+			MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
+			MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
+			MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x58e
+			MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
+		>;
+	};
+
+	pinctrl_eqos_sleep: eqossleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__GPIO4_IO00				0x31e
+			MX93_PAD_ENET1_MDIO__GPIO4_IO01				0x31e
+			MX93_PAD_ENET1_RD0__GPIO4_IO10                          0x31e
+			MX93_PAD_ENET1_RD1__GPIO4_IO11				0x31e
+			MX93_PAD_ENET1_RD2__GPIO4_IO12				0x31e
+			MX93_PAD_ENET1_RD3__GPIO4_IO13				0x31e
+			MX93_PAD_ENET1_RXC__GPIO4_IO09                          0x31e
+			MX93_PAD_ENET1_RX_CTL__GPIO4_IO08			0x31e
+			MX93_PAD_ENET1_TD0__GPIO4_IO05                          0x31e
+			MX93_PAD_ENET1_TD1__GPIO4_IO04                          0x31e
+			MX93_PAD_ENET1_TD2__GPIO4_IO03				0x31e
+			MX93_PAD_ENET1_TD3__GPIO4_IO02				0x31e
+			MX93_PAD_ENET1_TXC__GPIO4_IO07                          0x31e
+			MX93_PAD_ENET1_TX_CTL__GPIO4_IO06                       0x31e
+		>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDC__ENET1_MDC			0x57e
+			MX93_PAD_ENET2_MDIO__ENET1_MDIO			0x57e
+			MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x57e
+			MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x57e
+			MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x57e
+			MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x57e
+			MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x58e
+			MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x57e
+			MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x57e
+			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x57e
+			MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x57e
+			MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x57e
+			MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x58e
+			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x57e
+		>;
+	};
+
+	pinctrl_fec_sleep: fecsleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDC__GPIO4_IO14			0x51e
+			MX93_PAD_ENET2_MDIO__GPIO4_IO15			0x51e
+			MX93_PAD_ENET2_RD0__GPIO4_IO24			0x51e
+			MX93_PAD_ENET2_RD1__GPIO4_IO25			0x51e
+			MX93_PAD_ENET2_RD2__GPIO4_IO26			0x51e
+			MX93_PAD_ENET2_RD3__GPIO4_IO27			0x51e
+			MX93_PAD_ENET2_RXC__GPIO4_IO23                  0x51e
+			MX93_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
+			MX93_PAD_ENET2_TD0__GPIO4_IO19			0x51e
+			MX93_PAD_ENET2_TD1__GPIO4_IO18			0x51e
+			MX93_PAD_ENET2_TD2__GPIO4_IO17			0x51e
+			MX93_PAD_ENET2_TD3__GPIO4_IO16			0x51e
+			MX93_PAD_ENET2_TXC__GPIO4_IO21                  0x51e
+			MX93_PAD_ENET2_TX_CTL__GPIO4_IO20               0x51e
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO25__CAN2_TX	0x139e
+			MX93_PAD_GPIO_IO27__CAN2_RX	0x139e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX93_PAD_UART1_RXD__LPUART1_RX			0x31e
+			MX93_PAD_UART1_TXD__LPUART1_TX			0x31e
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX93_PAD_DAP_TDO_TRACESWO__LPUART5_TX		0x31e
+			MX93_PAD_DAP_TDI__LPUART5_RX			0x31e
+			MX93_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B		0x31e
+			MX93_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B		0x31e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX93_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
+			MX93_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			MX93_PAD_I2C2_SCL__LPI2C2_SCL			0x40000b9e
+			MX93_PAD_I2C2_SDA__LPI2C2_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO28__LPI2C3_SDA			0x40000b9e
+			MX93_PAD_GPIO_IO29__LPI2C3_SCL			0x40000b9e
+		>;
+	};
+
+	pinctrl_pcal6524: pcal6524grp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x1582
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x40001382
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x40001382
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x40001382
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x40001382
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x40001382
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x40001382
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x40001382
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x40001382
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x40001382
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1582
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x158e
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000138e
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000138e
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000138e
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000138e
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000138e
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000138e
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000138e
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000138e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x158e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x15fe
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x400013fe
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x15fe
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x31e
+		>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO26__SAI3_TX_SYNC                0x31e
+			MX93_PAD_GPIO_IO16__SAI3_TX_BCLK                0x31e
+			MX93_PAD_GPIO_IO17__SAI3_MCLK           0x31e
+			MX93_PAD_GPIO_IO19__SAI3_TX_DATA00              0x31e
+			MX93_PAD_GPIO_IO20__SAI3_RX_DATA00              0x31e
+		>;
+	};
+
+	pinctrl_sai3_sleep: sai3sleepgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO26__GPIO2_IO26			0x51e
+			MX93_PAD_GPIO_IO16__GPIO2_IO16			0x51e
+			MX93_PAD_GPIO_IO17__GPIO2_IO17			0x51e
+			MX93_PAD_GPIO_IO19__GPIO2_IO19			0x51e
+			MX93_PAD_GPIO_IO20__GPIO2_IO20			0x51e
+		>;
+	};
+
+	pinctrl_spdif: spdifgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO22__SPDIF_IN		0x31e
+			MX93_PAD_GPIO_IO23__SPDIF_OUT		0x31e
+		>;
+	};
+
+	pinctrl_spdif_sleep: spdifsleepgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO22__GPIO2_IO22		0x31e
+			MX93_PAD_GPIO_IO23__GPIO2_IO23		0x31e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x1582
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x40001382
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x40001382
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x40001382
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x40001382
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x40001382
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x158e
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000138e
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x4000138e
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x4000138e
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x15fe
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x400013fe
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_sleep: usdhc2sleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__GPIO3_IO01            0x51e
+			MX93_PAD_SD2_CMD__GPIO3_IO02		0x51e
+			MX93_PAD_SD2_DATA0__GPIO3_IO03		0x51e
+			MX93_PAD_SD2_DATA1__GPIO3_IO04		0x51e
+			MX93_PAD_SD2_DATA2__GPIO3_IO05		0x51e
+			MX93_PAD_SD2_DATA3__GPIO3_IO06		0x51e
+			MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY	0x31e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
index 8dd5340e8141f..c6db9c85f2ac5 100644
--- a/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
@@ -1,116 +1,19 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 /*
- * Copyright 2022 NXP
+ * Copyright 2022,2026 NXP
  */
 
 /dts-v1/;
 
-#include <dt-bindings/usb/pd.h>
 #include "imx93.dtsi"
+#include "imx93-11x11-evk-common.dtsi"
 
 / {
 	model = "NXP i.MX93 11X11 EVK board";
 	compatible = "fsl,imx93-11x11-evk", "fsl,imx93";
 
 	aliases {
-		ethernet0 = &fec;
-		ethernet1 = &eqos;
-		gpio0 = &gpio1;
-		gpio1 = &gpio2;
-		gpio2 = &gpio3;
-		i2c0 = &lpi2c1;
-		i2c1 = &lpi2c2;
-		i2c2 = &lpi2c3;
-		mmc0 = &usdhc1;
-		mmc1 = &usdhc2;
-		rtc0 = &bbnsm_rtc;
-		serial0 = &lpuart1;
-		serial1 = &lpuart2;
-		serial2 = &lpuart3;
-		serial3 = &lpuart4;
-		serial4 = &lpuart5;
-	};
-
-	chosen {
-		stdout-path = &lpuart1;
-	};
-
-	reserved-memory {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
-		linux,cma {
-			compatible = "shared-dma-pool";
-			reusable;
-			alloc-ranges = <0 0x80000000 0 0x40000000>;
-			size = <0 0x10000000>;
-			linux,cma-default;
-		};
-
-		vdev0vring0: vdev0vring0@a4000000 {
-			reg = <0 0xa4000000 0 0x8000>;
-			no-map;
-		};
-
-		vdev0vring1: vdev0vring1@a4008000 {
-			reg = <0 0xa4008000 0 0x8000>;
-			no-map;
-		};
-
-		vdev1vring0: vdev1vring0@a4010000 {
-			reg = <0 0xa4010000 0 0x8000>;
-			no-map;
-		};
-
-		vdev1vring1: vdev1vring1@a4018000 {
-			reg = <0 0xa4018000 0 0x8000>;
-			no-map;
-		};
-
-		rsc_table: rsc-table@2021e000 {
-			reg = <0 0x2021e000 0 0x1000>;
-			no-map;
-		};
-
-		vdevbuffer: vdevbuffer@a4020000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0xa4020000 0 0x100000>;
-			no-map;
-		};
-
-	};
-
-	flexcan_phy: can-phy {
-		compatible = "nxp,tja1057";
-		#phy-cells = <0>;
-		max-bitrate = <5000000>;
-		silent-gpios = <&adp5585 6 GPIO_ACTIVE_HIGH>;
-	};
-
-	reg_vdd_12v: regulator-vdd-12v {
-		compatible = "regulator-fixed";
-		regulator-name = "VDD_12V";
-		regulator-min-microvolt = <12000000>;
-		regulator-max-microvolt = <12000000>;
-		gpio = <&pcal6524 14 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-	};
-
-	reg_vref_1v8: regulator-adc-vref {
-		compatible = "regulator-fixed";
-		regulator-name = "vref_1v8";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-	};
-
-	reg_audio_pwr: regulator-audio-pwr {
-		compatible = "regulator-fixed";
-		regulator-name = "audio-pwr";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		gpio = <&adp5585 1 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
+		mmc2 = &usdhc3;
 	};
 
 	reg_m2_pwr: regulator-m2-pwr {
@@ -122,18 +25,6 @@ reg_m2_pwr: regulator-m2-pwr {
 		enable-active-high;
 	};
 
-	reg_usdhc2_vmmc: regulator-usdhc2 {
-		compatible = "regulator-fixed";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
-		regulator-name = "VSD_3V3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
-		off-on-delay-us = <12000>;
-		enable-active-high;
-	};
-
 	reg_usdhc3_vmmc: regulator-usdhc3 {
 		compatible = "regulator-fixed";
 		regulator-name = "WLAN_EN";
@@ -156,17 +47,6 @@ usdhc3_pwrseq: usdhc3_pwrseq {
 		reset-gpios = <&pcal6524 12 GPIO_ACTIVE_LOW>;
 	};
 
-	backlight_lvds: backlight-lvds {
-		compatible = "pwm-backlight";
-		pwms = <&adp5585 0 100000 0>;
-		brightness-levels = <0 100>;
-		num-interpolated-steps = <100>;
-		default-brightness-level = <100>;
-		power-supply = <&reg_vdd_12v>;
-		enable-gpios = <&adp5585 9 GPIO_ACTIVE_HIGH>;
-		status = "disabled";
-	};
-
 	bt_sco_codec: bt-sco-codec {
 		compatible = "linux,bt-sco";
 		#sound-dai-cells = <1>;
@@ -204,348 +84,6 @@ cpu {
 			};
 		};
 	};
-
-	sound-wm8962 {
-		compatible = "fsl,imx-audio-wm8962";
-		model = "wm8962-audio";
-		audio-cpu = <&sai3>;
-		audio-codec = <&wm8962>;
-		hp-det-gpio = <&pcal6524 4 GPIO_ACTIVE_HIGH>;
-		audio-routing =
-			"Headphone Jack", "HPOUTL",
-			"Headphone Jack", "HPOUTR",
-			"Ext Spk", "SPKOUTL",
-			"Ext Spk", "SPKOUTR",
-			"AMIC", "MICBIAS",
-			"IN3R", "AMIC",
-			"IN1R", "AMIC";
-	};
-
-	sound-xcvr {
-		compatible = "fsl,imx-audio-card";
-		model = "imx-audio-xcvr";
-
-		pri-dai-link {
-			link-name = "XCVR PCM";
-
-			cpu {
-				sound-dai = <&xcvr>;
-			};
-		};
-	};
-};
-
-&adc1 {
-	vref-supply = <&reg_vref_1v8>;
-	status = "okay";
-};
-
-&cm33 {
-	mbox-names = "tx", "rx", "rxdb";
-	mboxes = <&mu1 0 1>,
-		 <&mu1 1 1>,
-		 <&mu1 3 1>;
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
-			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
-	status = "okay";
-};
-
-&eqos {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_eqos>;
-	pinctrl-1 = <&pinctrl_eqos_sleep>;
-	phy-mode = "rgmii-id";
-	phy-handle = <&ethphy1>;
-	status = "okay";
-
-	mdio {
-		compatible = "snps,dwmac-mdio";
-		#address-cells = <1>;
-		#size-cells = <0>;
-		clock-frequency = <5000000>;
-
-		ethphy1: ethernet-phy@1 {
-			reg = <1>;
-			reset-gpios = <&pcal6524 15 GPIO_ACTIVE_LOW>;
-			reset-assert-us = <10000>;
-			reset-deassert-us = <80000>;
-			realtek,clkout-disable;
-		};
-	};
-};
-
-&fec {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_fec>;
-	pinctrl-1 = <&pinctrl_fec_sleep>;
-	phy-mode = "rgmii-id";
-	phy-handle = <&ethphy2>;
-	fsl,magic-packet;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		clock-frequency = <5000000>;
-
-		ethphy2: ethernet-phy@2 {
-			reg = <2>;
-			reset-gpios = <&pcal6524 16 GPIO_ACTIVE_LOW>;
-			reset-assert-us = <10000>;
-			reset-deassert-us = <80000>;
-			realtek,clkout-disable;
-		};
-	};
-};
-
-&flexcan2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_flexcan2>;
-	phys = <&flexcan_phy>;
-	status = "okay";
-};
-
-&lpi2c1 {
-	clock-frequency = <400000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_lpi2c1>;
-	status = "okay";
-
-	wm8962: codec@1a {
-		compatible = "wlf,wm8962";
-		reg = <0x1a>;
-		clocks = <&clk IMX93_CLK_SAI3_GATE>;
-		DCVDD-supply = <&reg_audio_pwr>;
-		DBVDD-supply = <&reg_audio_pwr>;
-		AVDD-supply = <&reg_audio_pwr>;
-		CPVDD-supply = <&reg_audio_pwr>;
-		MICVDD-supply = <&reg_audio_pwr>;
-		PLLVDD-supply = <&reg_audio_pwr>;
-		SPKVDD1-supply = <&reg_audio_pwr>;
-		SPKVDD2-supply = <&reg_audio_pwr>;
-		gpio-cfg = <
-			0x0000 /* 0:Default */
-			0x0000 /* 1:Default */
-			0x0000 /* 2:FN_DMICCLK */
-			0x0000 /* 3:Default */
-			0x0000 /* 4:FN_DMICCDAT */
-			0x0000 /* 5:Default */
-		>;
-	};
-
-	inertial-meter@6a {
-		compatible = "st,lsm6dso";
-		reg = <0x6a>;
-	};
-};
-
-&lpi2c2 {
-	clock-frequency = <400000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_lpi2c2>;
-	status = "okay";
-
-	pcal6524: gpio@22 {
-		compatible = "nxp,pcal6524";
-		reg = <0x22>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_pcal6524>;
-		gpio-controller;
-		#gpio-cells = <2>;
-		interrupt-controller;
-		#interrupt-cells = <2>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
-	};
-
-	pmic@25 {
-		compatible = "nxp,pca9451a";
-		reg = <0x25>;
-		interrupt-parent = <&pcal6524>;
-		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
-
-		regulators {
-			buck1: BUCK1 {
-				regulator-name = "BUCK1";
-				regulator-min-microvolt = <610000>;
-				regulator-max-microvolt = <950000>;
-				regulator-boot-on;
-				regulator-always-on;
-				regulator-ramp-delay = <3125>;
-			};
-
-			buck2: BUCK2 {
-				regulator-name = "BUCK2";
-				regulator-min-microvolt = <600000>;
-				regulator-max-microvolt = <670000>;
-				regulator-boot-on;
-				regulator-always-on;
-				regulator-ramp-delay = <3125>;
-			};
-
-			buck4: BUCK4 {
-				regulator-name = "BUCK4";
-				regulator-min-microvolt = <1620000>;
-				regulator-max-microvolt = <3400000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			buck5: BUCK5 {
-				regulator-name = "BUCK5";
-				regulator-min-microvolt = <1620000>;
-				regulator-max-microvolt = <3400000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			buck6: BUCK6 {
-				regulator-name = "BUCK6";
-				regulator-min-microvolt = <1060000>;
-				regulator-max-microvolt = <1140000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			ldo1: LDO1 {
-				regulator-name = "LDO1";
-				regulator-min-microvolt = <1620000>;
-				regulator-max-microvolt = <1980000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			ldo4: LDO4 {
-				regulator-name = "LDO4";
-				regulator-min-microvolt = <800000>;
-				regulator-max-microvolt = <840000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			ldo5: LDO5 {
-				regulator-name = "LDO5";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-		};
-	};
-
-	adp5585: io-expander@34 {
-		compatible = "adi,adp5585-00", "adi,adp5585";
-		reg = <0x34>;
-		vdd-supply = <&buck4>;
-		gpio-controller;
-		#gpio-cells = <2>;
-		gpio-reserved-ranges = <5 1>;
-		#pwm-cells = <3>;
-	};
-};
-
-&lpi2c3 {
-	clock-frequency = <400000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_lpi2c3>;
-	status = "okay";
-
-	adp5585_isp: io-expander@34 {
-		compatible = "adi,adp5585-01", "adi,adp5585";
-		reg = <0x34>;
-		gpio-controller;
-		#gpio-cells = <2>;
-		#pwm-cells = <3>;
-	};
-
-	ptn5110: tcpc@50 {
-		compatible = "nxp,ptn5110", "tcpci";
-		reg = <0x50>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
-
-		typec1_con: connector {
-			compatible = "usb-c-connector";
-			label = "USB-C";
-			power-role = "dual";
-			data-role = "dual";
-			try-power-role = "sink";
-			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
-				     PDO_VAR(5000, 20000, 3000)>;
-			op-sink-microwatt = <15000000>;
-			self-powered;
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					typec1_dr_sw: endpoint {
-						remote-endpoint = <&usb1_drd_sw>;
-					};
-				};
-			};
-		};
-	};
-
-	ptn5110_2: tcpc@51 {
-		compatible = "nxp,ptn5110", "tcpci";
-		reg = <0x51>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
-
-		typec2_con: connector {
-			compatible = "usb-c-connector";
-			label = "USB-C";
-			power-role = "dual";
-			data-role = "dual";
-			try-power-role = "sink";
-			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
-				     PDO_VAR(5000, 20000, 3000)>;
-			op-sink-microwatt = <15000000>;
-			self-powered;
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					typec2_dr_sw: endpoint {
-						remote-endpoint = <&usb2_drd_sw>;
-					};
-				};
-			};
-		};
-	};
-
-	pcf2131: rtc@53 {
-		compatible = "nxp,pcf2131";
-		reg = <0x53>;
-		interrupt-parent = <&pcal6524>;
-		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
-	};
-};
-
-&lpuart1 { /* console */
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart1>;
-	status = "okay";
-};
-
-&lpuart5 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart5>;
-	status = "okay";
-
-	bluetooth {
-		compatible = "nxp,88w8987-bt";
-	};
 };
 
 &micfil {
@@ -558,12 +96,12 @@ &micfil {
 	status = "okay";
 };
 
-&mu1 {
-	status = "okay";
-};
-
-&mu2 {
-	status = "okay";
+&pcal6524 {
+	m2-pcm-level-shifter-hog {
+		gpio-hog;
+		gpios = <19 GPIO_ACTIVE_HIGH>;
+		output-high;
+	};
 };
 
 &sai1 {
@@ -577,76 +115,6 @@ &sai1 {
 	status = "okay";
 };
 
-&sai3 {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_sai3>;
-	pinctrl-1 = <&pinctrl_sai3_sleep>;
-	assigned-clocks = <&clk IMX93_CLK_SAI3>;
-	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
-	assigned-clock-rates = <12288000>;
-	fsl,sai-mclk-direction-output;
-	status = "okay";
-};
-
-&usbotg1 {
-	dr_mode = "otg";
-	hnp-disable;
-	srp-disable;
-	adp-disable;
-	usb-role-switch;
-	disable-over-current;
-	samsung,picophy-pre-emp-curr-control = <3>;
-	samsung,picophy-dc-vol-level-adjust = <7>;
-	status = "okay";
-
-	port {
-		usb1_drd_sw: endpoint {
-			remote-endpoint = <&typec1_dr_sw>;
-		};
-	};
-};
-
-&usbotg2 {
-	dr_mode = "otg";
-	hnp-disable;
-	srp-disable;
-	adp-disable;
-	usb-role-switch;
-	disable-over-current;
-	samsung,picophy-pre-emp-curr-control = <3>;
-	samsung,picophy-dc-vol-level-adjust = <7>;
-	status = "okay";
-
-	port {
-		usb2_drd_sw: endpoint {
-			remote-endpoint = <&typec2_dr_sw>;
-		};
-	};
-};
-
-&usdhc1 {
-	pinctrl-names = "default", "state_100mhz", "state_200mhz";
-	pinctrl-0 = <&pinctrl_usdhc1>;
-	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
-	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
-	bus-width = <8>;
-	non-removable;
-	status = "okay";
-};
-
-&usdhc2 {
-	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
-	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
-	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
-	vmmc-supply = <&reg_usdhc2_vmmc>;
-	bus-width = <4>;
-	status = "okay";
-	no-mmc;
-};
-
 &usdhc3 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
 	pinctrl-0 = <&pinctrl_usdhc3>, <&pinctrl_usdhc3_wlan>;
@@ -662,152 +130,7 @@ &usdhc3 {
 	status = "okay";
 };
 
-&wdog3 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_wdog>;
-	fsl,ext-reset-output;
-	status = "okay";
-};
-
-&xcvr {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_spdif>;
-	pinctrl-1 = <&pinctrl_spdif_sleep>;
-	assigned-clocks = <&clk IMX93_CLK_SPDIF>,
-			 <&clk IMX93_CLK_AUDIO_XCVR>;
-	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>,
-			 <&clk IMX93_CLK_SYS_PLL_PFD1_DIV2>;
-	assigned-clock-rates = <12288000>, <200000000>;
-	status = "okay";
-};
-
 &iomuxc {
-	pinctrl_eqos: eqosgrp {
-		fsl,pins = <
-			MX93_PAD_ENET1_MDC__ENET_QOS_MDC			0x57e
-			MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
-			MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
-			MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
-			MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
-			MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
-			MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x58e
-			MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
-			MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
-			MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x57e
-			MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
-			MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
-			MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x58e
-			MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
-		>;
-	};
-
-	pinctrl_eqos_sleep: eqossleepgrp {
-		fsl,pins = <
-			MX93_PAD_ENET1_MDC__GPIO4_IO00				0x31e
-			MX93_PAD_ENET1_MDIO__GPIO4_IO01				0x31e
-			MX93_PAD_ENET1_RD0__GPIO4_IO10                          0x31e
-			MX93_PAD_ENET1_RD1__GPIO4_IO11				0x31e
-			MX93_PAD_ENET1_RD2__GPIO4_IO12				0x31e
-			MX93_PAD_ENET1_RD3__GPIO4_IO13				0x31e
-			MX93_PAD_ENET1_RXC__GPIO4_IO09                          0x31e
-			MX93_PAD_ENET1_RX_CTL__GPIO4_IO08			0x31e
-			MX93_PAD_ENET1_TD0__GPIO4_IO05                          0x31e
-			MX93_PAD_ENET1_TD1__GPIO4_IO04                          0x31e
-			MX93_PAD_ENET1_TD2__GPIO4_IO03				0x31e
-			MX93_PAD_ENET1_TD3__GPIO4_IO02				0x31e
-			MX93_PAD_ENET1_TXC__GPIO4_IO07                          0x31e
-			MX93_PAD_ENET1_TX_CTL__GPIO4_IO06                       0x31e
-		>;
-	};
-
-	pinctrl_fec: fecgrp {
-		fsl,pins = <
-			MX93_PAD_ENET2_MDC__ENET1_MDC			0x57e
-			MX93_PAD_ENET2_MDIO__ENET1_MDIO			0x57e
-			MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x57e
-			MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x57e
-			MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x57e
-			MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x57e
-			MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x58e
-			MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x57e
-			MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x57e
-			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x57e
-			MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x57e
-			MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x57e
-			MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x58e
-			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x57e
-		>;
-	};
-
-	pinctrl_fec_sleep: fecsleepgrp {
-		fsl,pins = <
-			MX93_PAD_ENET2_MDC__GPIO4_IO14			0x51e
-			MX93_PAD_ENET2_MDIO__GPIO4_IO15			0x51e
-			MX93_PAD_ENET2_RD0__GPIO4_IO24			0x51e
-			MX93_PAD_ENET2_RD1__GPIO4_IO25			0x51e
-			MX93_PAD_ENET2_RD2__GPIO4_IO26			0x51e
-			MX93_PAD_ENET2_RD3__GPIO4_IO27			0x51e
-			MX93_PAD_ENET2_RXC__GPIO4_IO23                  0x51e
-			MX93_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
-			MX93_PAD_ENET2_TD0__GPIO4_IO19			0x51e
-			MX93_PAD_ENET2_TD1__GPIO4_IO18			0x51e
-			MX93_PAD_ENET2_TD2__GPIO4_IO17			0x51e
-			MX93_PAD_ENET2_TD3__GPIO4_IO16			0x51e
-			MX93_PAD_ENET2_TXC__GPIO4_IO21                  0x51e
-			MX93_PAD_ENET2_TX_CTL__GPIO4_IO20               0x51e
-		>;
-	};
-
-	pinctrl_flexcan2: flexcan2grp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO25__CAN2_TX	0x139e
-			MX93_PAD_GPIO_IO27__CAN2_RX	0x139e
-		>;
-	};
-
-	pinctrl_uart1: uart1grp {
-		fsl,pins = <
-			MX93_PAD_UART1_RXD__LPUART1_RX			0x31e
-			MX93_PAD_UART1_TXD__LPUART1_TX			0x31e
-		>;
-	};
-
-	pinctrl_uart5: uart5grp {
-		fsl,pins = <
-			MX93_PAD_DAP_TDO_TRACESWO__LPUART5_TX		0x31e
-			MX93_PAD_DAP_TDI__LPUART5_RX			0x31e
-			MX93_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B		0x31e
-			MX93_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B		0x31e
-		>;
-	};
-
-	pinctrl_lpi2c1: lpi2c1grp {
-		fsl,pins = <
-			MX93_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
-			MX93_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
-		>;
-	};
-
-	pinctrl_lpi2c2: lpi2c2grp {
-		fsl,pins = <
-			MX93_PAD_I2C2_SCL__LPI2C2_SCL			0x40000b9e
-			MX93_PAD_I2C2_SDA__LPI2C2_SDA			0x40000b9e
-		>;
-	};
-
-	pinctrl_lpi2c3: lpi2c3grp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO28__LPI2C3_SDA			0x40000b9e
-			MX93_PAD_GPIO_IO29__LPI2C3_SCL			0x40000b9e
-		>;
-	};
-
-	pinctrl_pcal6524: pcal6524grp {
-		fsl,pins = <
-			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
-		>;
-	};
-
 	pinctrl_pdm: pdmgrp {
 		fsl,pins = <
 			MX93_PAD_PDM_CLK__PDM_CLK			0x31e
@@ -842,160 +165,6 @@ MX93_PAD_SAI1_RXD0__GPIO1_IO14			0x51e
 		>;
 	};
 
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1: usdhc1grp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x1582
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x40001382
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x40001382
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x40001382
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x40001382
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x40001382
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x40001382
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x40001382
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x40001382
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x40001382
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1582
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x158e
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000138e
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000138e
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000138e
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000138e
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000138e
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000138e
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000138e
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000138e
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x158e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x15fe
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x400013fe
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x15fe
-		>;
-	};
-
-	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x31e
-		>;
-	};
-
-	pinctrl_sai3: sai3grp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO26__SAI3_TX_SYNC                0x31e
-			MX93_PAD_GPIO_IO16__SAI3_TX_BCLK                0x31e
-			MX93_PAD_GPIO_IO17__SAI3_MCLK           0x31e
-			MX93_PAD_GPIO_IO19__SAI3_TX_DATA00              0x31e
-			MX93_PAD_GPIO_IO20__SAI3_RX_DATA00              0x31e
-		>;
-	};
-
-	pinctrl_sai3_sleep: sai3sleepgrp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO26__GPIO2_IO26			0x51e
-			MX93_PAD_GPIO_IO16__GPIO2_IO16			0x51e
-			MX93_PAD_GPIO_IO17__GPIO2_IO17			0x51e
-			MX93_PAD_GPIO_IO19__GPIO2_IO19			0x51e
-			MX93_PAD_GPIO_IO20__GPIO2_IO20			0x51e
-		>;
-	};
-
-	pinctrl_spdif: spdifgrp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO22__SPDIF_IN		0x31e
-			MX93_PAD_GPIO_IO23__SPDIF_OUT		0x31e
-		>;
-	};
-
-	pinctrl_spdif_sleep: spdifsleepgrp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO22__GPIO2_IO22		0x31e
-			MX93_PAD_GPIO_IO23__GPIO2_IO23		0x31e
-		>;
-	};
-
-	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
-		>;
-	};
-
-	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleepgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2: usdhc2grp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x1582
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x40001382
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x40001382
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x40001382
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x40001382
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x40001382
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x158e
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000138e
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x4000138e
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x4000138e
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x15fe
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x400013fe
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	pinctrl_usdhc2_sleep: usdhc2sleepgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__GPIO3_IO01            0x51e
-			MX93_PAD_SD2_CMD__GPIO3_IO02		0x51e
-			MX93_PAD_SD2_DATA0__GPIO3_IO03		0x51e
-			MX93_PAD_SD2_DATA1__GPIO3_IO04		0x51e
-			MX93_PAD_SD2_DATA2__GPIO3_IO05		0x51e
-			MX93_PAD_SD2_DATA3__GPIO3_IO06		0x51e
-			MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e
-		>;
-	};
-
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc3: usdhc3grp {
 		fsl,pins = <
@@ -1048,10 +217,4 @@ pinctrl_usdhc3_wlan: usdhc3wlangrp {
 			MX93_PAD_CCM_CLKO1__GPIO3_IO26		0x31e
 		>;
 	};
-
-	pinctrl_wdog: wdoggrp {
-		fsl,pins = <
-			MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY	0x31e
-		>;
-	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-pixpaper.dtso b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-pixpaper.dtso
new file mode 100644
index 0000000000000..d220d7e07df22
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-pixpaper.dtso
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Overlay for Mayqueen (Open-EP Community) pixpaper display
+ * support on NXP FRDM i.MX 93 Development Board
+ *
+ * Copyright (C) 2026 Wig Cheng <onlywig@gmail.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include "imx93-pinfunc.h"
+
+/dts-v1/;
+/plugin/;
+
+&iomuxc {
+	pinctrl_epd_ctrl: epdctrlgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO05__GPIO2_IO05		0x31e /* DC pin */
+			MX93_PAD_GPIO_IO06__GPIO2_IO06		0x31e /* RESET pin */
+			MX93_PAD_GPIO_IO26__GPIO2_IO26		0x31e /* BUSY pin */
+		>;
+	};
+
+	pinctrl_lpspi3: lpspi3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO08__GPIO2_IO08		0x3fe /* SPI3 CE0 */
+			MX93_PAD_GPIO_IO09__LPSPI3_SIN		0x3fe /* SPI3 MISO */
+			MX93_PAD_GPIO_IO10__LPSPI3_SOUT		0x3fe /* SPI3 MOSI */
+			MX93_PAD_GPIO_IO11__LPSPI3_SCK		0x3fe /* SPI3 CLK */
+		>;
+	};
+};
+
+&lpspi3 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	cs-gpios = <&gpio2 8 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_lpspi3>, <&pinctrl_epd_ctrl>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	display@0 {
+		compatible = "mayqueen,pixpaper";
+		reg = <0>;
+		busy-gpios = <&gpio2 26 GPIO_ACTIVE_HIGH>;
+		dc-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio2 6 GPIO_ACTIVE_HIGH>;
+		spi-max-frequency = <1000000>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts b/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
index 61843b2c1b1b8..ec78c03f4788b 100644
--- a/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
@@ -25,6 +25,7 @@ aliases {
 		mmc1 = &usdhc2;
 		rtc0 = &bbnsm_rtc;
 		serial0 = &lpuart1;
+		serial4 = &lpuart5;
 	};
 
 	bt_sco_codec: bt-sco-codec {
@@ -400,6 +401,17 @@ &lpuart1 { /* console */
 	status = "okay";
 };
 
+&lpuart5 {
+	/* BT */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
 &mu1 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
new file mode 100644
index 0000000000000..d167c9fc3b8f8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include "imx93-pinfunc.h"
+
+&{/} {
+	backlight: backlight {
+		compatible = "gpio-backlight";
+		gpios = <&pcal6524 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	panel {
+		compatible = "ontat,kd50g21-40nt-a1";
+		backlight = <&backlight>;
+		power-supply = <&reg_rpi_3v3>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dpi_to_panel>;
+			};
+		};
+	};
+};
+
+&dpi_bridge {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lcdif>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			dpi_to_panel: endpoint {
+				remote-endpoint = <&panel_in>;
+				bus-width = <18>;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_lcdif: lcdifgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__MEDIAMIX_DISP_CLK		0x31e
+			MX93_PAD_GPIO_IO01__MEDIAMIX_DISP_DE		0x31e
+			MX93_PAD_GPIO_IO02__MEDIAMIX_DISP_VSYNC		0x31e
+			MX93_PAD_GPIO_IO03__MEDIAMIX_DISP_HSYNC		0x31e
+			MX93_PAD_GPIO_IO04__MEDIAMIX_DISP_DATA00	0x31e
+			MX93_PAD_GPIO_IO05__MEDIAMIX_DISP_DATA01	0x31e
+			MX93_PAD_GPIO_IO06__MEDIAMIX_DISP_DATA02	0x31e
+			MX93_PAD_GPIO_IO07__MEDIAMIX_DISP_DATA03	0x31e
+			MX93_PAD_GPIO_IO08__MEDIAMIX_DISP_DATA04	0x31e
+			MX93_PAD_GPIO_IO09__MEDIAMIX_DISP_DATA05	0x31e
+			MX93_PAD_GPIO_IO10__MEDIAMIX_DISP_DATA06	0x31e
+			MX93_PAD_GPIO_IO11__MEDIAMIX_DISP_DATA07	0x31e
+			MX93_PAD_GPIO_IO12__MEDIAMIX_DISP_DATA08	0x31e
+			MX93_PAD_GPIO_IO13__MEDIAMIX_DISP_DATA09	0x31e
+			MX93_PAD_GPIO_IO14__MEDIAMIX_DISP_DATA10	0x31e
+			MX93_PAD_GPIO_IO15__MEDIAMIX_DISP_DATA11	0x31e
+			MX93_PAD_GPIO_IO16__MEDIAMIX_DISP_DATA12	0x31e
+			MX93_PAD_GPIO_IO17__MEDIAMIX_DISP_DATA13	0x31e
+			MX93_PAD_GPIO_IO18__MEDIAMIX_DISP_DATA14	0x31e
+			MX93_PAD_GPIO_IO19__MEDIAMIX_DISP_DATA15	0x31e
+			MX93_PAD_GPIO_IO20__MEDIAMIX_DISP_DATA16	0x31e
+			MX93_PAD_GPIO_IO21__MEDIAMIX_DISP_DATA17	0x31e
+		>;
+	};
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&pcal6524 {
+	/*
+	 * exp-sel-hog has property 'output-low' while DT overlay doesn't
+	 * support /delete-property/. Both 'output-low' and 'output-high'
+	 * will exist under hog nodes if DT overlay file sets 'output-high'.
+	 * Workaround is to disable this hog and create new hog with
+	 * 'output-high'.
+	 */
+	exp-sel-hog {
+		status = "disabled";
+	};
+
+	exp-high-sel-hog {
+		gpio-hog;
+		gpios = <22 GPIO_ACTIVE_HIGH>;
+		output-high;
+	};
+};
+
+&sai3 {
+	/* disable due to GPIO12 and GPIO17~20 pin conflicts with LCDIF */
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
index 0852067eab2cb..7bcebd7021063 100644
--- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
@@ -349,6 +349,12 @@ mic-can-sel-hog {
 			gpios = <17 GPIO_ACTIVE_HIGH>;
 			output-low;
 		};
+
+		m2-pcm-level-shifter-hog {
+			gpio-hog;
+			gpios = <19 GPIO_ACTIVE_HIGH>;
+			output-high;
+		};
 	};
 
 	pmic@25 {
@@ -507,6 +513,7 @@ &usdhc1 {
 	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
 	bus-width = <8>;
 	non-removable;
+	fsl,tuning-step = <1>;
 	status = "okay";
 };
 
@@ -519,6 +526,7 @@ &usdhc2 {
 	vmmc-supply = <&reg_usdhc2_vmmc>;
 	bus-width = <4>;
 	no-mmc;
+	fsl,tuning-step = <1>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts b/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
index 9e875e082ee87..eac389ed30f36 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
@@ -144,8 +144,11 @@ &flexcan1 {
 /* I2C2 */
 &lpi2c2 {
 	clock-frequency = <400000>;
-	pinctrl-names = "default";
+	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-1 = <&pinctrl_lpi2c2_gpio>;
+	scl-gpios = <&gpio1 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
 	/* RTC */
@@ -277,6 +280,13 @@ MX93_PAD_I2C2_SDA__LPI2C2_SDA		0x40000b9e
 		>;
 	};
 
+	pinctrl_lpi2c2_gpio: lpi2c2gpiogrp {
+		fsl,pins = <
+			MX93_PAD_I2C2_SCL__GPIO1_IO02		0x31e
+			MX93_PAD_I2C2_SDA__GPIO1_IO03		0x31e
+		>;
+	};
+
 	pinctrl_lpspi6: lpspi6grp {
 		fsl,pins = <
 			MX93_PAD_GPIO_IO00__GPIO2_IO00		0x386
diff --git a/arch/arm64/boot/dts/freescale/imx93-phyboard-segin-peb-av-02.dtso b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin-peb-av-02.dtso
new file mode 100644
index 0000000000000..af330756abfd1
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin-peb-av-02.dtso
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2025 PHYTEC Messtechnik GmbH
+ * Copyright (C) 2025 Pengutronix
+ *
+ * Author: Andrej Picej <andrej.picej@norik.com>
+ * Author: Marco Felsch <kernel@pengutronix.de>
+ */
+
+#include <dt-bindings/clock/imx93-clock.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "imx93-pinfunc.h"
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <5>;
+		power-supply = <&reg_vcc_3v3_con>;
+		pwms = <&pwm7 0 5000000 0>;
+	};
+
+	panel {
+		compatible = "edt,etm0700g0edh6";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_panel>;
+
+		backlight = <&backlight>;
+		enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_vcc_3v3_con>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dpi_to_panel>;
+			};
+		};
+	};
+
+	/* TODO: Convert to FlexIO PWM once supported */
+	pwm7: pwm-7 {
+		compatible = "pwm-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pwm7>;
+		gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>;
+		#pwm-cells = <3>;
+	};
+
+	reg_vcc_3v3_con: regulator-vcc-3v3-con {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC3V3_CON";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+};
+
+&dpi_bridge {
+	status = "okay";
+};
+
+&dpi_to_panel {
+	remote-endpoint = <&panel_in>;
+};
+
+&lcdif {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lcdif>;
+	assigned-clocks = <&clk IMX93_CLK_VIDEO_PLL>;
+	assigned-clock-rates = <332600000>;
+	status = "okay";
+};
+
+&lpi2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen@38 {
+		compatible = "edt,edt-ft5406";
+		reg = <0x38>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_touchscreen>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio4 1 GPIO_ACTIVE_LOW>;
+		vcc-supply = <&reg_vcc_3v3_con>;
+		iovcc-supply = <&reg_vcc_3v3_con>;
+		touchscreen-size-x = <1792>;
+		touchscreen-size-y = <1024>;
+		wakeup-source;
+	};
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_lcdif: lcdifgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__MEDIAMIX_DISP_CLK		0x50e
+			MX93_PAD_GPIO_IO01__MEDIAMIX_DISP_DE		0x50e
+			MX93_PAD_GPIO_IO02__MEDIAMIX_DISP_VSYNC		0x50e
+			MX93_PAD_GPIO_IO03__MEDIAMIX_DISP_HSYNC		0x50e
+			MX93_PAD_GPIO_IO04__MEDIAMIX_DISP_DATA00	0x50e
+			MX93_PAD_GPIO_IO05__MEDIAMIX_DISP_DATA01	0x50e
+			MX93_PAD_GPIO_IO06__MEDIAMIX_DISP_DATA02	0x50e
+			MX93_PAD_GPIO_IO07__MEDIAMIX_DISP_DATA03	0x50e
+			MX93_PAD_GPIO_IO08__MEDIAMIX_DISP_DATA04	0x50e
+			MX93_PAD_GPIO_IO09__MEDIAMIX_DISP_DATA05	0x51e
+			MX93_PAD_GPIO_IO10__MEDIAMIX_DISP_DATA06	0x50e
+			MX93_PAD_GPIO_IO11__MEDIAMIX_DISP_DATA07	0x50e
+			MX93_PAD_GPIO_IO12__MEDIAMIX_DISP_DATA08	0x50e
+			MX93_PAD_GPIO_IO13__MEDIAMIX_DISP_DATA09	0x50e
+			MX93_PAD_GPIO_IO14__MEDIAMIX_DISP_DATA10	0x50e
+			MX93_PAD_GPIO_IO15__MEDIAMIX_DISP_DATA11	0x50e
+			MX93_PAD_GPIO_IO16__MEDIAMIX_DISP_DATA12	0x506
+			MX93_PAD_GPIO_IO17__MEDIAMIX_DISP_DATA13	0x506
+			MX93_PAD_GPIO_IO18__MEDIAMIX_DISP_DATA14	0x506
+			MX93_PAD_GPIO_IO19__MEDIAMIX_DISP_DATA15	0x506
+			MX93_PAD_GPIO_IO20__MEDIAMIX_DISP_DATA16	0x506
+			MX93_PAD_GPIO_IO21__MEDIAMIX_DISP_DATA17	0x506
+		>;
+	};
+
+	pinctrl_panel: panelgrp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO4__GPIO4_IO29			0x1133e
+		>;
+	};
+
+	pinctrl_pwm7: pwm7grp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO3__GPIO4_IO28			0x1133e
+		>;
+	};
+
+	pinctrl_touchscreen: touchscreengrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDIO__GPIO4_IO01			0x11e
+			MX93_PAD_ENET1_RD2__GPIO4_IO12			0x1133e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
index ac64abacc4a27..a982606de1ee4 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
@@ -148,8 +148,11 @@ &flexcan1 {
 /* I2C2 */
 &lpi2c2 {
 	clock-frequency = <400000>;
-	pinctrl-names = "default";
+	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-1 = <&pinctrl_lpi2c2_gpio>;
+	scl-gpios = <&gpio1 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
 	/* Codec */
@@ -262,6 +265,13 @@ MX93_PAD_I2C2_SDA__LPI2C2_SDA		0x40000b9e
 		>;
 	};
 
+	pinctrl_lpi2c2_gpio: lpi2c2gpiogrp {
+		fsl,pins = <
+			MX93_PAD_I2C2_SCL__GPIO1_IO02		0x31e
+			MX93_PAD_I2C2_SDA__GPIO1_IO03		0x31e
+		>;
+	};
+
 	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
 		fsl,pins = <
 			MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi b/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
index 3f069905cf0b5..ebc57841f27f5 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
@@ -93,8 +93,11 @@ ethphy1: ethernet-phy@1 {
 /* I2C3 */
 &lpi2c3 {
 	clock-frequency = <400000>;
-	pinctrl-names = "default";
+	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_lpi2c3>;
+	pinctrl-1 = <&pinctrl_lpi2c3_gpio>;
+	scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
 	pmic@25 {
@@ -234,6 +237,13 @@ MX93_PAD_GPIO_IO29__LPI2C3_SCL		0x40000b9e
 		>;
 	};
 
+	pinctrl_lpi2c3_gpio: lpi2c3gpiogrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO28__GPIO2_IO28		0x31e
+			MX93_PAD_GPIO_IO29__GPIO2_IO29		0x31e
+		>;
+	};
+
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
 			MX93_PAD_ENET2_RD3__GPIO4_IO27		0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba91xxca.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba91xxca.dts
index 2673d9dccbf4b..737326ba1b2a7 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba91xxca.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba91xxca.dts
@@ -232,7 +232,7 @@ ethphy_eqos: ethernet-phy@0 {
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
 			enet-phy-lane-no-swap;
 			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
 			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
@@ -265,7 +265,7 @@ ethphy_fec: ethernet-phy@0 {
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <27 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
 			enet-phy-lane-no-swap;
 			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
 			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
index 4760d07ea24b5..9108181e6592b 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
@@ -242,7 +242,7 @@ ethphy_eqos: ethernet-phy@0 {
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
 			enet-phy-lane-no-swap;
 			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
 			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
@@ -275,7 +275,7 @@ ethphy_fec: ethernet-phy@0 {
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <27 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
 			enet-phy-lane-no-swap;
 			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
 			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
index 8a88c98ac05a7..a78bbc46c59b2 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
@@ -172,7 +172,7 @@ ethphy_eqos: ethernet-phy@0 {
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_eqos_phy>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
 			reset-gpios = <&expander1 0 GPIO_ACTIVE_LOW>;
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
@@ -205,7 +205,7 @@ ethphy_fec: ethernet-phy@0 {
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_fec_phy>;
 			interrupt-parent = <&gpio3>;
-			interrupts = <27 IRQ_TYPE_EDGE_FALLING>;
+			interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
 			reset-gpios = <&expander1 1 GPIO_ACTIVE_LOW>;
 			reset-assert-us = <500000>;
 			reset-deassert-us = <50000>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi
index 3a23e2eb9febe..ce34a296495c4 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352.dtsi
@@ -271,21 +271,21 @@ MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x106
 	/* enable SION for data and cmd pad due to ERR052021 */
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
-			/* PD | FSEL 3 | DSE X5 */
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x5be
+			/* PD | FSEL 3 | DSE X4 */
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x59e
 			/* HYS | FSEL 0 | no drive */
 			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1000
-			/* HYS | FSEL 3 | X5 */
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x400011be
-			/* HYS | FSEL 3 | X4 */
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000119e
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000119e
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000119e
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000119e
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000119e
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000119e
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000119e
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000119e
+			/* HYS | PU | FSEL 3 | DSE X4 */
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000139e
+			/* HYS | PU | FSEL 3 | DSE X4 */
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000139e
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000139e
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000139e
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000139e
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000139e
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000139e
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000139e
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000139e
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi
index 7b27012dfcb56..5436b48b30e89 100644
--- a/arch/arm64/boot/dts/freescale/imx93.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93.dtsi
@@ -150,6 +150,18 @@ l3_cache: l3-cache {
 	};
 };
 
+&lcdif {
+	port {
+		lcdif_to_ldb: endpoint@1 {
+			reg = <1>;
+		};
+
+		lcdif_to_dsi: endpoint@2 {
+			reg = <2>;
+		};
+	};
+};
+
 &src {
 	mlmix: power-domain@44461800 {
 		compatible = "fsl,imx93-src-slice";
diff --git a/arch/arm64/boot/dts/freescale/imx93w-evk.dts b/arch/arm64/boot/dts/freescale/imx93w-evk.dts
new file mode 100644
index 0000000000000..8e53e73840137
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93w-evk.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+
+#include "imx93w.dtsi"
+#include "imx93-11x11-evk-common.dtsi"
+
+/ {
+	model = "NXP i.MX93W EVK board";
+	compatible = "fsl,imx93-wireless-evk", "fsl,imx93";
+};
+
+&lpi2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+};
+
+&iomuxc {
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__LPI2C3_SDA			0x40000b9e
+			MX93_PAD_GPIO_IO01__LPI2C3_SCL			0x40000b9e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93w.dtsi b/arch/arm64/boot/dts/freescale/imx93w.dtsi
new file mode 100644
index 0000000000000..95fb025c39493
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93w.dtsi
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+#include "imx93.dtsi"
+
+/ {
+	aliases {
+		mmc2 = &usdhc3;
+	};
+
+	reg_usdhc3_vmmc: regulator-usdhc3 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_usdhc3_vmmc>;
+		regulator-name = "WLAN_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 29 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	usdhc3_pwrseq: usdhc3_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usdhc3_pwrseq>;
+		reset-gpios = <&gpio1 10 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	pinctrl-3 = <&pinctrl_usdhc3_sleep>;
+	mmc-pwrseq = <&usdhc3_pwrseq>;
+	vmmc-supply = <&reg_usdhc3_vmmc>;
+	bus-width = <4>;
+	keep-power-in-suspend;
+	non-removable;
+	wakeup-source;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_reg_usdhc3_vmmc: regusdhc3vmmcgrp {
+		fsl,pins = <
+			/*
+			 * Enable open drain and internal pull-up to allow the IW610 JTAG
+			 * connector to control the PDn status.
+			 */
+			MX93_PAD_GPIO_IO29__GPIO2_IO29			0xb9e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK		0x1582
+			MX93_PAD_SD3_CMD__USDHC3_CMD		0x40001382
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0	0x40001382
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1	0x40001382
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2	0x40001382
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3	0x40001382
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK		0x158e
+			MX93_PAD_SD3_CMD__USDHC3_CMD		0x4000138e
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0	0x4000138e
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1	0x4000138e
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2	0x4000138e
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3	0x4000138e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK		0x15fe
+			MX93_PAD_SD3_CMD__USDHC3_CMD		0x400013fe
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0	0x400013fe
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1	0x400013fe
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2	0x400013fe
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3	0x400013fe
+		>;
+	};
+
+	pinctrl_usdhc3_sleep: usdhc3grpsleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__GPIO3_IO20		0x31e
+			MX93_PAD_SD3_CMD__GPIO3_IO21		0x31e
+			MX93_PAD_SD3_DATA0__GPIO3_IO22		0x31e
+			MX93_PAD_SD3_DATA1__GPIO3_IO23		0x31e
+			MX93_PAD_SD3_DATA2__GPIO3_IO24		0x31e
+			MX93_PAD_SD3_DATA3__GPIO3_IO25		0x31e
+		>;
+	};
+
+	pinctrl_usdhc3_pwrseq: usdhc3pwrseqgrp {
+		fsl,pins = <
+			MX93_PAD_PDM_BIT_STREAM1__GPIO1_IO10		0x39e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi
index d2f31c8caf6eb..38488147a3ef2 100644
--- a/arch/arm64/boot/dts/freescale/imx94.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx94.dtsi
@@ -139,6 +139,13 @@ timer {
 		arm,no-tick-in-suspend;
 	};
 
+	usbphynop: usbphynop {
+		compatible = "usb-nop-xceiv";
+		clocks = <&scmi_clk IMX94_CLK_HSIO>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+	};
+
 	gic: interrupt-controller@48000000 {
 		compatible = "arm,gic-v3";
 		reg = <0 0x48000000 0 0x10000>,
@@ -1223,6 +1230,60 @@ wdog3: watchdog@49220000 {
 			};
 		};
 
+		usb3: usb@4c100000 {
+			compatible = "nxp,imx94-dwc3", "nxp,imx8mp-dwc3";
+			reg = <0x0 0x4c100000 0x0 0x10000>,
+			      <0x0 0x4c010010 0x0 0x04>,
+			      <0x0 0x4c1f0000 0x0 0x20>;
+			reg-names = "core", "blkctl", "glue";
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_24M>,
+				 <&scmi_clk IMX94_CLK_32K>;
+			clock-names = "hsio", "bus_early", "ref", "suspend";
+			interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 386 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc_usb3", "wakeup";
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			phys = <&usb3_phy>, <&usb3_phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			snps,gfladj-refclk-lpm-sel-quirk;
+			snps,parkmode-disable-ss-quirk;
+			status = "disabled";
+		};
+
+		usb3_phy: phy@4c1f0040 {
+			compatible = "fsl,imx95-usb-phy", "fsl,imx8mp-usb-phy";
+			reg = <0x0 0x4c1f0040 0x0 0x40>,
+			      <0x0 0x4c1fc000 0x0 0x100>;
+			clocks = <&scmi_clk IMX94_CLK_HSIO>;
+			clock-names = "phy";
+			#phy-cells = <0>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			status = "disabled";
+		};
+
+		usb2: usb@4c200000 {
+			compatible = "fsl,imx95-usb", "fsl,imx7d-usb", "fsl,imx27-usb";
+			reg = <0x0 0x4c200000 0x0 0x200>;
+			interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 387 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_32K>;
+			clock-names = "usb_ctrl_root", "usb_wakeup";
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			phys = <&usbphynop>;
+			fsl,usbmisc = <&usbmisc 0>;
+			status = "disabled";
+		};
+
+		usbmisc: usbmisc@4c200200 {
+			compatible = "fsl,imx95-usbmisc", "fsl,imx7d-usbmisc", "fsl,imx6q-usbmisc";
+			reg = <0x0 0x4c200200 0x0 0x200>,
+			      <0x0 0x4c010014 0x0 0x04>;
+			#index-cells = <1>;
+		};
+
 		netc_blk_ctrl: system-controller@4ceb0000 {
 			compatible = "nxp,imx94-netc-blk-ctrl";
 			reg = <0x0 0x4ceb0000 0x0 0x10000>,
diff --git a/arch/arm64/boot/dts/freescale/imx943-evk.dts b/arch/arm64/boot/dts/freescale/imx943-evk.dts
index 31fa9675cee13..1d834379f602d 100644
--- a/arch/arm64/boot/dts/freescale/imx943-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx943-evk.dts
@@ -6,6 +6,7 @@
 /dts-v1/;
 
 #include "imx943.dtsi"
+#include <dt-bindings/usb/pd.h>
 
 / {
 	compatible = "fsl,imx943-evk", "fsl,imx94";
@@ -20,7 +21,9 @@ aliases {
 		i2c5 = &lpi2c6;
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
+		mmc2 = &usdhc3;
 		serial0 = &lpuart1;
+		serial5 = &lpuart6;
 	};
 
 	bt_sco_codec: bt-sco-codec {
@@ -53,6 +56,32 @@ dmic: dmic {
 		#sound-dai-cells = <0>;
 	};
 
+	reg_m2_pwr: regulator-m2-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "M.2-power";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pcal6416_i2c3_u46 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		/*
+		 * M.2 device only can be enabled(W_DISABLE1#) after all Power
+		 * Rails reach their minimum operating voltage (PCI Express M.2
+		 * Specification r5.1 3.1.4 Power-up Timing).
+		 * Set a delay equal to the max value of Tsettle here.
+		 */
+		startup-delay-us = <5000>;
+	};
+
+	reg_m2_wlan: regulator-wlan {
+		compatible = "regulator-fixed";
+		regulator-name = "WLAN_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_m2_pwr>;
+		gpio = <&pcal6416_i2c3_u46 5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_usdhc2_vmmc: regulator-usdhc2 {
 		compatible = "regulator-fixed";
 		off-on-delay-us = <12000>;
@@ -140,6 +169,11 @@ sound-wm8962 {
 		model = "wm8962-audio";
 	};
 
+	usdhc3_pwrseq: usdhc3_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pcal6416_i2c3_u46 4 GPIO_ACTIVE_LOW>;
+	};
+
 	memory@80000000 {
 		reg = <0x0 0x80000000 0x0 0x80000000>;
 		device_type = "memory";
@@ -197,6 +231,48 @@ pca9670_i2c3: gpio@23 {
 		gpio-controller;
 	};
 
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_typec>;
+
+		typec_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			power-role = "dual";
+			data-role = "dual";
+			try-power-role = "sink";
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			op-sink-microwatt = <0>;
+			self-powered;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec_con_hs: endpoint {
+						remote-endpoint = <&usb3_data_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					typec_con_ss: endpoint {
+						remote-endpoint = <&usb3_data_ss>;
+					};
+				};
+			};
+		};
+	};
+
 	pca9548_i2c3: i2c-mux@77 {
 		compatible = "nxp,pca9548";
 		reg = <0x77>;
@@ -253,6 +329,14 @@ wm8962: codec@1a {
 				SPKVDD1-supply = <&reg_audio_pwr>;
 				SPKVDD2-supply = <&reg_audio_pwr>;
 			};
+
+			ptn5150: tcpc@3d {
+				compatible = "nxp,ptn5150";
+				reg = <0x3d>;
+				interrupt-parent = <&pcal6408_i2c3_u172>;
+				interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+				status = "disabled";
+			};
 		};
 
 		i2c@5 {
@@ -437,6 +521,17 @@ &lpuart1 {
 	status = "okay";
 };
 
+&lpuart6 {
+	/* BT */
+	pinctrl-0 = <&pinctrl_uart6>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
 &micfil {
 	assigned-clocks = <&scmi_clk IMX94_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX94_CLK_AUDIOPLL2_VCO>,
@@ -636,6 +731,12 @@ IMX94_PAD_GPIO_IO47__SAI3_TX_DATA0	0x31e
 		>;
 	};
 
+	pinctrl_typec: typecgrp {
+		fsl,pins = <
+			IMX94_PAD_GPIO_IO44__GPIO3_IO12		0x30e
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			IMX94_PAD_UART1_TXD__LPUART1_TX		0x31e
@@ -643,6 +744,15 @@ IMX94_PAD_UART1_RXD__LPUART1_RX		0x31e
 		>;
 	};
 
+	pinctrl_uart6: uart6grp {
+		fsl,pins = <
+			IMX94_PAD_GPIO_IO04__LPUART6_TX		0x31e
+			IMX94_PAD_GPIO_IO05__LPUART6_RX		0x31e
+			IMX94_PAD_GPIO_IO06__LPUART6_CTS_B	0x31e
+			IMX94_PAD_GPIO_IO07__LPUART6_RTS_B	0x31e
+		>;
+	};
+
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
 		fsl,pins = <
 			IMX94_PAD_SD1_CLK__USDHC1_CLK		0x158e
@@ -739,6 +849,18 @@ IMX94_PAD_SD2_RESET_B__GPIO4_IO27	0x31e
 		>;
 	};
 
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			IMX94_PAD_GPIO_IO48__USDHC3_CLK		0x158e
+			/* Need to config the SION for CMD pad, refer to ERR053138 */
+			IMX94_PAD_GPIO_IO49__USDHC3_CMD		0x4000138e
+			IMX94_PAD_GPIO_IO50__USDHC3_DATA0	0x138e
+			IMX94_PAD_GPIO_IO51__USDHC3_DATA1	0x138e
+			IMX94_PAD_GPIO_IO52__USDHC3_DATA2	0x138e
+			IMX94_PAD_GPIO_IO53__USDHC3_DATA3	0x138e
+		>;
+	};
+
 	pinctrl_xspi1: xspi1grp {
 		fsl,pins = <
 			IMX94_PAD_XSPI1_SCLK__XSPI1_A_SCLK	0x3fe
@@ -756,6 +878,49 @@ IMX94_PAD_XSPI1_DQS__XSPI1_A_DQS	0x3fe
 	};
 };
 
+&usb2 {
+	dr_mode = "otg";
+	disable-over-current;
+	adp-disable;
+	hnp-disable;
+	srp-disable;
+	samsung,picophy-dc-vol-level-adjust = <10>;
+	status = "okay";
+};
+
+&usb3 {
+	dr_mode = "otg";
+	adp-disable;
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	status = "okay";
+
+	port {
+		usb3_data_hs: endpoint {
+			remote-endpoint = <&typec_con_hs>;
+		};
+	};
+};
+
+&usb3_phy {
+	orientation-switch;
+	fsl,phy-pcs-tx-deemph-3p5db-attenuation-db = <17>;
+	fsl,phy-pcs-tx-swing-full-percent = <100>;
+	fsl,phy-tx-preemp-amp-tune-microamp = <600>;
+	fsl,phy-tx-vboost-level-microvolt = <1156>;
+	fsl,phy-tx-vref-tune-percent = <100>;
+	status = "okay";
+
+	port {
+		usb3_data_ss: endpoint {
+			remote-endpoint = <&typec_con_ss>;
+		};
+	};
+};
+
 &usdhc1 {
 	pinctrl-0 = <&pinctrl_usdhc1>;
 	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
@@ -781,6 +946,23 @@ &usdhc2 {
 	status = "okay";
 };
 
+&usdhc3 {
+	/*
+	 * Only enable SDIO2.0 mode as the corresponding GPIO pads are 3.3V, the
+	 * max frequency is 50MHz.
+	 */
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3>;
+	pinctrl-names = "default", "sleep";
+	bus-width = <4>;
+	vmmc-supply = <&reg_m2_wlan>;
+	mmc-pwrseq = <&usdhc3_pwrseq>;
+	keep-power-in-suspend;
+	non-removable;
+	wakeup-source;
+	status = "okay";
+};
+
 &wdog3 {
 	fsl,ext-reset-output;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
index ca1c4966c8670..0f43e3be70589 100644
--- a/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
@@ -243,6 +243,12 @@ codec {
 		};
 	};
 
+	sound-mqs {
+		compatible = "audio-graph-card2";
+		links = <&sai1_port1>;
+		label = "mqs-audio";
+	};
+
 	usdhc3_pwrseq: usdhc3-pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&pcal6524 8 GPIO_ACTIVE_LOW>;
@@ -473,6 +479,21 @@ &mu7 {
 	status = "okay";
 };
 
+&mqs1 {
+	clocks = <&scmi_clk IMX95_CLK_SAI1>;
+	clock-names = "mclk";
+	pinctrl-0 = <&pinctrl_mqs1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	mqs1_port: port {
+		mqs1_ep: endpoint {
+			dai-format = "left_j";
+			remote-endpoint = <&sai1_port1_ep>;
+		};
+	};
+};
+
 &netc_blk_ctrl {
 	status = "okay";
 };
@@ -534,6 +555,51 @@ &pcie0 {
 	status = "okay";
 };
 
+&sai1 {
+	clocks = <&scmi_clk IMX95_CLK_BUSAON>, <&dummy>,
+		 <&scmi_clk IMX95_CLK_SAI1>, <&dummy>,
+		 <&dummy>, <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI1>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>, <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>, <3612672000>,
+			       <393216000>, <361267200>,
+			       <24576000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* leave unconnected - no RX in the context of MQS */
+		port@0 {
+			reg = <0>;
+
+			endpoint {
+			};
+		};
+
+		sai1_port1: port@1 {
+			reg = <1>;
+			mclk-fs = <512>;
+
+			sai1_port1_ep: endpoint {
+				dai-format = "left_j";
+				system-clock-direction-out;
+				bitclock-master;
+				frame-master;
+				remote-endpoint = <&mqs1_ep>;
+			};
+		};
+	};
+};
+
 &scmi_iomuxc {
 	pinctrl_emdio: emdiogrp {
 		fsl,pins = <
@@ -618,6 +684,13 @@ IMX95_PAD_GPIO_IO31__LPI2C4_SCL				0x40000b9e
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			IMX95_PAD_SAI1_TXFS__AONMIX_TOP_MQS1_LEFT		0x31e
+			IMX95_PAD_SAI1_RXD0__AONMIX_TOP_MQS1_RIGHT		0x31e
+		>;
+	};
+
 	pinctrl_pcal6524: pcal6524grp {
 		fsl,pins = <
 			IMX95_PAD_GPIO_IO34__GPIO5_IO_BIT14			0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk-sof.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk-sof.dts
index 808a9fe3ebb2b..264703f6eef64 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk-sof.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk-sof.dts
@@ -55,8 +55,11 @@ sound-wm8962 {
 };
 
 &edma2 {
-	/* channels 30 and 31 reserved for FW usage */
-	dma-channel-mask = <0xc0000000>, <0x0>;
+	/*
+	 * channels 0 and 1 reserved for V2X fast hash,
+	 * channels 30 and 31 reserved for FW usage
+	 */
+	dma-channel-mask = <0xc0000003>, <0x0>;
 };
 
 &sai3 {
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
index aaa0da55a22bc..041fd838fabba 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
@@ -475,7 +475,7 @@ &lpuart1 {
 &lpuart5 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart5>;
-	status = "disabled";
+	status = "okay";
 
 	bluetooth {
 		compatible = "nxp,88w8987-bt";
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
index 5932ba238a8ac..7a73958f6eecc 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
@@ -144,6 +144,14 @@ reg_wifi_en: regulator-wifi-en {
 		startup-delay-us = <2000>;
 	};
 
+	remoteproc-cm7 {
+		compatible = "fsl,imx95-cm7";
+		mboxes = <&mu7 0 1 &mu7 1 1 &mu7 3 1>;
+		mbox-names = "tx", "rx", "rxdb";
+		memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>, <&m7_reserved>;
+	};
+
 	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
@@ -156,6 +164,42 @@ linux_cma: linux,cma {
 			alloc-ranges = <0 0x80000000 0 0x7f000000>;
 			linux,cma-default;
 		};
+
+		m7_reserved: memory@80000000 {
+			reg = <0 0x80000000 0 0x1000000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@88220000 {
+			reg = <0 0x88220000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@88000000 {
+			reg = <0 0x88000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@88008000 {
+			reg = <0 0x88008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@88010000 {
+			reg = <0 0x88010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@88018000 {
+			reg = <0 0x88018000 0 0x8000>;
+			no-map;
+		};
+
+		vdevbuffer: vdevbuffer@88020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x88020000 0 0x100000>;
+			no-map;
+		};
 	};
 };
 
@@ -262,7 +306,6 @@ &gpio3 {
 			  "",
 			  "",
 			  "",
-			  "",
 			  "PMIC_SD2_VSEL";
 	status = "okay";
 };
@@ -572,6 +615,10 @@ &lpuart3 {
 	pinctrl-0 = <&pinctrl_uart3>;
 };
 
+&mu7 {
+	status = "okay";
+};
+
 /* SMARC MDIO, shared between all ethernet ports */
 &netc_emdio {
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 55e2da094c889..5f61866e9df9e 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -391,6 +391,11 @@ scmi_misc: protocol@84 {
 		};
 	};
 
+	mqs1: mqs-1 {
+		compatible = "fsl,imx95-aonmix-mqs";
+		status = "disabled";
+	};
+
 	pmu {
 		compatible = "arm,cortex-a55-pmu";
 		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>;
@@ -631,6 +636,8 @@ edma2: dma-controller@42000000 {
 				reg = <0x42000000 0x210000>;
 				#dma-cells = <3>;
 				dma-channels = <64>;
+				/* channels 0 and 1 reserved for V2X fast hash */
+				dma-channel-mask = <0x3>;
 				interrupts = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/freescale/imx952-evk.dts b/arch/arm64/boot/dts/freescale/imx952-evk.dts
index b838323468d41..509704c23e89d 100644
--- a/arch/arm64/boot/dts/freescale/imx952-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx952-evk.dts
@@ -593,4 +593,45 @@ pinctrl_usdhc2_gpio: usdhc2gpiogrp {
 			IMX952_PAD_SD2_CD_B__WAKEUPMIX_TOP_GPIO3_IO_0		0x31e
 		>;
 	};
+
+	pinctrl_xspi1: xspi1grp {
+		fsl,pins = <
+			IMX952_PAD_XSPI1_DATA0__WAKEUPMIX_TOP_XSPI1_A_DATA_0		0x39e
+			IMX952_PAD_XSPI1_DATA1__WAKEUPMIX_TOP_XSPI1_A_DATA_1		0x39e
+			IMX952_PAD_XSPI1_DATA2__WAKEUPMIX_TOP_XSPI1_A_DATA_2		0x39e
+			IMX952_PAD_XSPI1_DATA3__WAKEUPMIX_TOP_XSPI1_A_DATA_3		0x39e
+			IMX952_PAD_XSPI1_DATA4__WAKEUPMIX_TOP_XSPI1_A_DATA_4		0x39e
+			IMX952_PAD_XSPI1_DATA5__WAKEUPMIX_TOP_XSPI1_A_DATA_5		0x39e
+			IMX952_PAD_XSPI1_DATA6__WAKEUPMIX_TOP_XSPI1_A_DATA_6		0x39e
+			IMX952_PAD_XSPI1_DATA7__WAKEUPMIX_TOP_XSPI1_A_DATA_7		0x39e
+			IMX952_PAD_XSPI1_DQS__WAKEUPMIX_TOP_XSPI1_A_DQS			0x39e
+			IMX952_PAD_XSPI1_SCLK__WAKEUPMIX_TOP_XSPI1_A_SCLK		0x39e
+			IMX952_PAD_XSPI1_SS0_B__WAKEUPMIX_TOP_XSPI1_A_SS0_B		0x39e
+		>;
+	};
+
+	pinctrl_xspi1_reset: xspi1-reset-grp {
+		fsl,pins = <
+			IMX952_PAD_XSPI1_SS1_B__WAKEUPMIX_TOP_GPIO5_IO_11	0x39e
+		>;
+	};
+};
+
+&xspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_xspi1>;
+	status = "okay";
+
+	mt35xu01gbba: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_xspi1_reset>;
+		reset-gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <200000000>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx952.dtsi b/arch/arm64/boot/dts/freescale/imx952.dtsi
index 91fe4916ac04d..0f86b5626cdd8 100644
--- a/arch/arm64/boot/dts/freescale/imx952.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx952.dtsi
@@ -672,6 +672,25 @@ mu8: mailbox@42350000 {
 				#mbox-cells = <2>;
 				status = "disabled";
 			};
+
+			xspi1: spi@42400000 {
+				compatible = "nxp,imx952-xspi", "nxp,imx94-xspi";
+				reg = <0x42400000 0x50000>, <0x28000000 0x8000000>;
+				reg-names = "base", "mmap";
+				interrupts = <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				clocks = <&scmi_clk IMX952_CLK_XSPI1>;
+				clock-names = "per";
+				assigned-clocks = <&scmi_clk IMX952_CLK_XSPI1>;
+				assigned-clock-parents = <&scmi_clk IMX952_CLK_SYSPLL1_PFD1>;
+				assigned-clock-rates = <200000000>;
+				status = "disabled";
+			};
 		};
 
 		aips3: bus@42800000 {
diff --git a/arch/arm64/boot/dts/freescale/s32n79-rdb.dts b/arch/arm64/boot/dts/freescale/s32n79-rdb.dts
new file mode 100644
index 0000000000000..1feccd61258ef
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/s32n79-rdb.dts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright 2026 NXP
+ *
+ * NXP S32N79 Reference Design Board (S32N79-RDB)
+ */
+
+/dts-v1/;
+#include "s32n79.dtsi"
+
+/ {
+	compatible = "nxp,s32n79-rdb", "nxp,s32n79";
+	model = "NXP S32N79-RDB";
+
+	aliases {
+		serial0 = &uart0;
+		serial1 = &uart5;
+		serial2 = &uart6;
+		serial3 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		scmi_shbuf: memory@93000000 {
+			compatible = "arm,scmi-shmem";
+			reg = <0x0 0x93000000 0x0 0x80>;
+			no-map;
+		};
+	};
+
+	memory@80000000 {
+		reg = <0x00 0x80000000 0x00 0x80000000>,
+			<0x88 0x00000000 0x03 0x40000000>,
+			<0xc0 0x00000000 0x03 0x40000000>;
+		device_type = "memory";
+	};
+};
+
+&irqsteer_coss {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart6 {
+	status = "okay";
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&usdhc0 {
+	disable-wp;
+	no-sdio;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/s32n79.dtsi b/arch/arm64/boot/dts/freescale/s32n79.dtsi
new file mode 100644
index 0000000000000..94ab58783fdc8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/s32n79.dtsi
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * NXP S32N79 SoC
+ *
+ * Copyright 2026 NXP
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cis-bus {
+		compatible = "simple-bus";
+		ranges = <0x4f200000 0x0 0x4f200000 0xc00000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		gic: interrupt-controller@4f200000 {
+			compatible = "arm,gic-v3";
+			reg = <0x4f200000 0x10000>, /* GIC Dist */
+			      <0x4f260000 0x100000>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			/* GICR (RD_base + SGI_base) */
+			ranges;
+
+			its: msi-controller@4f240000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x4f240000 0x20000>;
+				#msi-cells = <1>;
+				msi-controller;
+			};
+		};
+
+		smmu: iommu@4fc00000 {
+			compatible = "arm,smmu-v3";
+			reg = <0x4fc00000 0x200000>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 1 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 2 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "eventq", "gerror", "priq", "cmdq-sync";
+			#iommu-cells = <1>;
+			dma-coherent;
+			status = "disabled";
+		};
+	};
+
+	coss-bus {
+		compatible = "simple-bus";
+		ranges = <0x4a000000 0x0 0x4a000000 0xff0000>,
+			 <0x4e000000 0x0 0x4e000000 0x1000000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		uart0: serial@4a030000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x4a030000 0x1000>;
+			interrupt-parent = <&irqsteer_coss>;
+			interrupts = <264>;
+			clocks = <&clks 0x9a>, <&clks 0x9a>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		uart5: serial@4a060000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x4a060000 0x1000>;
+			interrupt-parent = <&irqsteer_coss>;
+			interrupts = <269>;
+			clocks = <&clks 0x9a>, <&clks 0x9a>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		uart6: serial@4aa30000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x4aa30000 0x1000>;
+			interrupt-parent = <&irqsteer_coss>;
+			interrupts = <270>;
+			clocks = <&clks 0x9a>, <&clks 0x9a>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		uart7: serial@4aa40000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x4aa40000 0x1000>;
+			interrupt-parent = <&irqsteer_coss>;
+			interrupts = <271>;
+			clocks = <&clks 0x9a>, <&clks 0x9a>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		irqsteer_coss: interrupt-controller@4ed00000 {
+			compatible = "nxp,s32n79-irqsteer";
+			reg = <0x4ed00000 0x10000>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 527 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 528 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 529 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 530 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 531 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 532 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 533 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 534 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clks 0x9a>;
+			clock-names = "ipg";
+			fsl,channel = <0>;
+			fsl,num-irqs = <512>;
+			status = "disabled";
+		};
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu2>;
+				};
+
+				core1 {
+					cpu = <&cpu3>;
+				};
+			};
+
+			cluster2 {
+				core0 {
+					cpu = <&cpu4>;
+				};
+
+				core1 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster3 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		l2_0: l2-cache0 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-line-size = <64>;
+			cache-sets = <512>;
+			cache-size = <524288>;
+			cache-unified;
+			next-level-cache = <&l3_0>;
+		};
+
+		l2_1: l2-cache1 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-line-size = <64>;
+			cache-sets = <512>;
+			cache-size = <524288>;
+			cache-unified;
+			next-level-cache = <&l3_1>;
+		};
+
+		l2_2: l2-cache2 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-line-size = <64>;
+			cache-sets = <512>;
+			cache-size = <524288>;
+			cache-unified;
+			next-level-cache = <&l3_2>;
+		};
+
+		l2_3: l2-cache3 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-line-size = <64>;
+			cache-sets = <512>;
+			cache-size = <524288>;
+			cache-unified;
+			next-level-cache = <&l3_3>;
+		};
+
+		l3_0: l3-cache0 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <1024>;
+			cache-size = <1048576>;
+			cache-unified;
+		};
+
+		l3_1: l3-cache1 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <1024>;
+			cache-size = <1048576>;
+			cache-unified;
+		};
+
+		l3_2: l3-cache2 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <1024>;
+			cache-size = <1048576>;
+			cache-unified;
+		};
+
+		l3_3: l3-cache3 {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-line-size = <64>;
+			cache-sets = <1024>;
+			cache-size = <1048576>;
+			cache-unified;
+		};
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x0>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu1: cpu@100 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x100>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu2: cpu@10000 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x10000>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu3: cpu@10100 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x10100>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu4: cpu@20000 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x20000>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu5: cpu@20100 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x20100>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu6: cpu@30000 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x30000>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_3>;
+		};
+
+		cpu7: cpu@30100 {
+			compatible = "arm,cortex-a78ae";
+			reg = <0x30100>;
+			device_type = "cpu";
+			enable-method = "psci";
+			next-level-cache = <&l2_3>;
+		};
+	};
+
+	firmware {
+		psci {
+			compatible = "arm,psci-1.0";
+			method = "smc";
+		};
+
+		scmi: scmi {
+			compatible = "arm,scmi-smc";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			shmem = <&scmi_shbuf>;
+			arm,smc-id = <0xc20000fe>;
+			status = "okay";
+
+			clks: protocol@14 {
+				reg = <0x14>;
+				#clock-cells = <1>;
+			};
+		};
+	};
+
+	fss-bus {
+		compatible = "simple-bus";
+		ranges = <0x5b490000 0x0 0x5b490000 0x1000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		usdhc0: mmc@5b490000 {
+			compatible = "nxp,s32n79-usdhc";
+			reg = <0x5b490000 0x1000>;
+			interrupts = <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clks 0x58>, <&clks 0x50>, <&clks 0x5f>;
+			clock-names = "ipg", "ahb", "per";
+			bus-width = <8>;
+			status = "disabled";
+		};
+	};
+
+	pmu: pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	timer: timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts b/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
index ed84ab92fb19c..c6056a85ce802 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
@@ -20,8 +20,8 @@ / {
 	compatible = "hisilicon,hi3660-hikey960", "hisilicon,hi3660";
 
 	aliases {
-		mshc1 = &dwmmc1;
-		mshc2 = &dwmmc2;
+		mmc1 = &dwmmc1;
+		mmc2 = &dwmmc2;
 		serial0 = &uart0;
 		serial1 = &uart1;
 		serial2 = &uart2;
diff --git a/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts b/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts
index 7c32f5fd5cc5b..65764b38d2e60 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts
@@ -19,8 +19,8 @@ / {
 	compatible = "hisilicon,hi3670-hikey970", "hisilicon,hi3670";
 
 	aliases {
-		mshc1 = &dwmmc1;
-		mshc2 = &dwmmc2;
+		mmc1 = &dwmmc1;
+		mmc2 = &dwmmc2;
 		serial0 = &uart0;
 		serial1 = &uart1;
 		serial2 = &uart2;
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
index 352c96d144a84..02e62d954e949 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
@@ -152,10 +152,10 @@ qspi_clk: qspi-clk {
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupt-parent = <&intc>;
-		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
 	};
 
 	usbphy0: usbphy {
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-atlas-v5.dts b/arch/arm64/boot/dts/marvell/armada-3720-atlas-v5.dts
index 070d10a705bbd..a313d5687789e 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-atlas-v5.dts
+++ b/arch/arm64/boot/dts/marvell/armada-3720-atlas-v5.dts
@@ -82,7 +82,6 @@ &sdhci0 {
 	mmc-ddr-1_8v;
 	mmc-hs400-1_8v;
 	sd-uhs-sdr104;
-	marvell,xenon-emmc;
 	marvell,xenon-tun-count = <9>;
 	marvell,pad-type = "fixed-1-8v";
 	vqmmc-supply = <&vsdc_reg>;
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dtsi
index fed2dcecb323f..37e16fb3a3832 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dtsi
@@ -78,7 +78,6 @@ &sdhci0 {
 	bus-width = <8>;
 	mmc-ddr-1_8v;
 	mmc-hs400-1_8v;
-	marvell,xenon-emmc;
 	marvell,xenon-tun-count = <9>;
 	marvell,pad-type = "fixed-1-8v";
 
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi
index 2428208457078..cd856c0aba71e 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi
@@ -15,6 +15,11 @@
 #include "armada-372x.dtsi"
 
 / {
+	aliases {
+		ethernet0 = &eth0;
+		ethernet1 = &eth1;
+	};
+
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
index 87f9367aec122..ea1824f5321fb 100644
--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
@@ -396,7 +396,7 @@ usb2: usb@5e000 {
 				marvell,usb-misc-reg = <&usb2_syscon>;
 				interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&usb2_utmi_host_phy>;
-				phy-names = "usb2-utmi-host-phy";
+				phy-names = "usb";
 				status = "disabled";
 			};
 
@@ -534,7 +534,6 @@ firmware {
 		armada-3700-rwtm {
 			compatible = "marvell,armada-3700-rwtm-firmware";
 			mboxes = <&rwtm 0>;
-			status = "okay";
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/marvell/armada-7020-comexpress.dtsi b/arch/arm64/boot/dts/marvell/armada-7020-comexpress.dtsi
index 2b5ec4a451e38..0cfcf5f6bde13 100644
--- a/arch/arm64/boot/dts/marvell/armada-7020-comexpress.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-7020-comexpress.dtsi
@@ -70,7 +70,7 @@ &cp0_eth0 {
 
 &cp0_eth1 {
 	status = "okay";
-	phy = <&phy0>;
+	phy-handle = <&phy0>;
 	phy-mode = "rgmii-id";
 };
 
diff --git a/arch/arm64/boot/dts/marvell/armada-8020.dtsi b/arch/arm64/boot/dts/marvell/armada-8020.dtsi
deleted file mode 100644
index b6fc188760935..0000000000000
--- a/arch/arm64/boot/dts/marvell/armada-8020.dtsi
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-/*
- * Copyright (C) 2016 Marvell Technology Group Ltd.
- *
- * Device Tree file for the Armada 8020 SoC, made of an AP806 Dual and
- * two CP110.
- */
-
-#include "armada-ap806-dual.dtsi"
-#include "armada-80x0.dtsi"
-
-/* The RTC requires external oscillator. But on Aramda 80x0, the RTC clock
- * in CP master is not connected (by package) to the oscillator. So
- * disable it. However, the RTC clock in CP slave is connected to the
- * oscillator so this one is let enabled.
- */
-
-&cp0_rtc {
-	status = "disabled";
-};
diff --git a/arch/arm64/boot/dts/marvell/cn9130-db-comexpress.dtsi b/arch/arm64/boot/dts/marvell/cn9130-db-comexpress.dtsi
deleted file mode 100644
index 028496ebc4736..0000000000000
--- a/arch/arm64/boot/dts/marvell/cn9130-db-comexpress.dtsi
+++ /dev/null
@@ -1,96 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-/*
- * Copyright (C) 2023 Marvell International Ltd.
- *
- * Device tree for the CN9130-DB Com Express CPU module board.
- */
-
-#include "cn9130-db.dtsi"
-
-/ {
-	model = "Marvell Armada CN9130-DB COM EXPRESS type 7 CPU module board";
-	compatible = "marvell,cn9130-cpu-module", "marvell,cn9130",
-		     "marvell,armada-ap807-quad", "marvell,armada-ap807";
-
-};
-
-&ap0_reg_sd_vccq {
-	regulator-max-microvolt = <1800000>;
-	states = <1800000 0x1 1800000 0x0>;
-	/delete-property/ gpios;
-};
-
-&cp0_reg_usb3_vbus0 {
-	/delete-property/ gpio;
-};
-
-&cp0_reg_usb3_vbus1 {
-	/delete-property/ gpio;
-};
-
-&cp0_reg_sd_vcc {
-	status = "disabled";
-};
-
-&cp0_reg_sd_vccq {
-	status = "disabled";
-};
-
-&cp0_sdhci0 {
-	status = "disabled";
-};
-
-&cp0_eth0 {
-	status = "disabled";
-};
-
-&cp0_eth1 {
-	status = "okay";
-	phy = <&phy0>;
-	phy-mode = "rgmii-id";
-};
-
-&cp0_eth2 {
-	status = "disabled";
-};
-
-&cp0_mdio {
-	status = "okay";
-	pinctrl-0 = <&cp0_ge_mdio_pins>;
-	phy0: ethernet-phy@0 {
-		status = "okay";
-	};
-};
-
-&cp0_syscon0 {
-	cp0_pinctrl: pinctrl {
-		compatible = "marvell,cp115-standalone-pinctrl";
-
-		cp0_ge_mdio_pins: ge-mdio-pins {
-			marvell,pins = "mpp40", "mpp41";
-			marvell,function = "ge";
-		};
-	};
-};
-
-&cp0_sdhci0 {
-	status = "disabled";
-};
-
-&cp0_spi1 {
-	status = "okay";
-};
-
-&cp0_usb3_0 {
-	status = "okay";
-	usb-phy = <&cp0_usb3_0_phy0>;
-	phy-names = "usb";
-	/delete-property/ phys;
-};
-
-&cp0_usb3_1 {
-	status = "okay";
-	usb-phy = <&cp0_usb3_0_phy1>;
-	phy-names = "usb";
-	/delete-property/ phys;
-};
diff --git a/arch/arm64/boot/dts/mediatek/mt6359.dtsi b/arch/arm64/boot/dts/mediatek/mt6359.dtsi
index 467d8a4c2aa7f..45ad69ee49edd 100644
--- a/arch/arm64/boot/dts/mediatek/mt6359.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt6359.dtsi
@@ -205,7 +205,7 @@ mt6359_vrfck_ldo_reg: ldo_vrfck {
 				regulator-max-microvolt = <1700000>;
 			};
 			mt6359_vrfck_1_ldo_reg: ldo_vrfck_1 {
-				regulator-name = "vrfck";
+				regulator-name = "vrfck_1";
 				regulator-min-microvolt = <1240000>;
 				regulator-max-microvolt = <1600000>;
 			};
@@ -227,7 +227,7 @@ mt6359_vemc_ldo_reg: ldo_vemc {
 				regulator-max-microvolt = <3300000>;
 			};
 			mt6359_vemc_1_ldo_reg: ldo_vemc_1 {
-				regulator-name = "vemc";
+				regulator-name = "vemc_1";
 				regulator-min-microvolt = <2500000>;
 				regulator-max-microvolt = <3300000>;
 			};
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts
index c7ea6e88c4f48..621d01e3cd896 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts
@@ -9,7 +9,7 @@
 #include "mt7988a-bananapi-bpi-r4-pro.dtsi"
 
 / {
-	model = "Bananapi BPI-R4";
+	model = "Bananapi BPI-R4 Pro 4E";
 	compatible = "bananapi,bpi-r4-pro-4e",
 		     "bananapi,bpi-r4-pro",
 		     "mediatek,mt7988a";
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts
index c9a0e69e9dd51..bb15bfa5e6ae5 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts
@@ -9,7 +9,7 @@
 #include "mt7988a-bananapi-bpi-r4-pro.dtsi"
 
 / {
-	model = "Bananapi BPI-R4";
+	model = "Bananapi BPI-R4 Pro 8X";
 	compatible = "bananapi,bpi-r4-pro-8x",
 		     "bananapi,bpi-r4-pro",
 		     "mediatek,mt7988a";
diff --git a/arch/arm64/boot/dts/mediatek/mt8167.dtsi b/arch/arm64/boot/dts/mediatek/mt8167.dtsi
index 2374c09530575..27cf32d7ae35f 100644
--- a/arch/arm64/boot/dts/mediatek/mt8167.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8167.dtsi
@@ -29,12 +29,6 @@ infracfg: infracfg@10001000 {
 			#clock-cells = <1>;
 		};
 
-		apmixedsys: apmixedsys@10018000 {
-			compatible = "mediatek,mt8167-apmixedsys", "syscon";
-			reg = <0 0x10018000 0 0x710>;
-			#clock-cells = <1>;
-		};
-
 		scpsys: syscon@10006000 {
 			compatible = "mediatek,mt8167-scpsys", "syscon", "simple-mfd";
 			reg = <0 0x10006000 0 0x1000>;
@@ -101,18 +95,6 @@ power-domain@MT8167_POWER_DOMAIN_CONN {
 			};
 		};
 
-		imgsys: syscon@15000000 {
-			compatible = "mediatek,mt8167-imgsys", "syscon";
-			reg = <0 0x15000000 0 0x1000>;
-			#clock-cells = <1>;
-		};
-
-		vdecsys: syscon@16000000 {
-			compatible = "mediatek,mt8167-vdecsys", "syscon";
-			reg = <0 0x16000000 0 0x1000>;
-			#clock-cells = <1>;
-		};
-
 		pio: pinctrl@1000b000 {
 			compatible = "mediatek,mt8167-pinctrl";
 			reg = <0 0x1000b000 0 0x1000>;
@@ -124,12 +106,36 @@ pio: pinctrl@1000b000 {
 			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		apmixedsys: apmixedsys@10018000 {
+			compatible = "mediatek,mt8167-apmixedsys", "syscon";
+			reg = <0 0x10018000 0 0x710>;
+			#clock-cells = <1>;
+		};
+
+		iommu: m4u@10203000 {
+			compatible = "mediatek,mt8167-m4u";
+			reg = <0 0x10203000 0 0x1000>;
+			mediatek,larbs = <&larb0>, <&larb1>, <&larb2>;
+			interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_LOW>;
+			#iommu-cells = <1>;
+		};
+
 		mmsys: syscon@14000000 {
 			compatible = "mediatek,mt8167-mmsys", "syscon";
 			reg = <0 0x14000000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb0: larb@14016000 {
+			compatible = "mediatek,mt8167-smi-larb";
+			reg = <0 0x14016000 0 0x1000>;
+			mediatek,smi = <&smi_common>;
+			clocks = <&mmsys CLK_MM_SMI_LARB0>,
+				 <&mmsys CLK_MM_SMI_LARB0>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8167_POWER_DOMAIN_MM>;
+		};
+
 		smi_common: smi@14017000 {
 			compatible = "mediatek,mt8167-smi-common";
 			reg = <0 0x14017000 0 0x1000>;
@@ -139,14 +145,10 @@ smi_common: smi@14017000 {
 			power-domains = <&spm MT8167_POWER_DOMAIN_MM>;
 		};
 
-		larb0: larb@14016000 {
-			compatible = "mediatek,mt8167-smi-larb";
-			reg = <0 0x14016000 0 0x1000>;
-			mediatek,smi = <&smi_common>;
-			clocks = <&mmsys CLK_MM_SMI_LARB0>,
-				 <&mmsys CLK_MM_SMI_LARB0>;
-			clock-names = "apb", "smi";
-			power-domains = <&spm MT8167_POWER_DOMAIN_MM>;
+		imgsys: syscon@15000000 {
+			compatible = "mediatek,mt8167-imgsys", "syscon";
+			reg = <0 0x15000000 0 0x1000>;
+			#clock-cells = <1>;
 		};
 
 		larb1: larb@15001000 {
@@ -159,6 +161,12 @@ larb1: larb@15001000 {
 			power-domains = <&spm MT8167_POWER_DOMAIN_ISP>;
 		};
 
+		vdecsys: syscon@16000000 {
+			compatible = "mediatek,mt8167-vdecsys", "syscon";
+			reg = <0 0x16000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
 		larb2: larb@16010000 {
 			compatible = "mediatek,mt8167-smi-larb";
 			reg = <0 0x16010000 0 0x1000>;
@@ -168,13 +176,5 @@ larb2: larb@16010000 {
 			clock-names = "apb", "smi";
 			power-domains = <&spm MT8167_POWER_DOMAIN_VDEC>;
 		};
-
-		iommu: m4u@10203000 {
-			compatible = "mediatek,mt8167-m4u";
-			reg = <0 0x10203000 0 0x1000>;
-			mediatek,larbs = <&larb0>, <&larb1>, <&larb2>;
-			interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_LOW>;
-			#iommu-cells = <1>;
-		};
 	};
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry-dojo-r1.dts b/arch/arm64/boot/dts/mediatek/mt8195-cherry-dojo-r1.dts
index 49664de99b882..57cc329f49c45 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195-cherry-dojo-r1.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry-dojo-r1.dts
@@ -11,6 +11,28 @@ / {
 	compatible = "google,dojo-sku7", "google,dojo-sku5",
 		     "google,dojo-sku3", "google,dojo-sku1",
 		     "google,dojo", "mediatek,mt8195";
+
+	nvme-connector {
+		compatible = "pcie-m2-m-connector";
+		/* power is controlled by EC */
+		vpcie3v3-supply = <&pp3300_z2>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				nvme_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie0_ep>;
+				};
+			};
+		};
+	};
 };
 
 &audio_codec {
@@ -72,6 +94,22 @@ &pcie0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie0_pins_default>;
 	status = "okay";
+
+	pcie@0 {
+		compatible = "pciclass,0604";
+		reg = <0 0 0 0 0>;
+		device_type = "pci";
+		num-lanes = <2>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges;
+
+		port {
+			pcie0_ep: endpoint {
+				remote-endpoint = <&nvme_ep>;
+			};
+		};
+	};
 };
 
 &pciephy {
diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
index b3761b80cac7e..f1ff64a84267c 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
@@ -83,6 +83,17 @@ pp3300_s3: regulator-pp3300-s3 {
 		vin-supply = <&pp3300_z2>;
 	};
 
+	pp3300_wlan: regulator-pp3300-wlan {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pp3300_wlan_en_pin>;
+		regulator-name = "pp3300_wlan";
+		/* load switch */
+		enable-active-high;
+		gpio = <&pio 58 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&pp3300_z2>;
+	};
+
 	/* system wide 3.3V power rail */
 	pp3300_z2: regulator-pp3300-z2 {
 		compatible = "regulator-fixed";
@@ -760,10 +771,25 @@ &ovl0_in {
 };
 
 &pcie1 {
-	status = "okay";
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_pins_default>;
+	status = "okay";
+
+	pcie@0 {
+		compatible = "pciclass,0604";
+		reg = <0 0 0 0 0>;
+		device_type = "pci";
+		num-lanes = <1>;
+		vpcie3v3-supply = <&pp3300_wlan>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges;
+
+		wifi@0 {
+			reg = <0 0 0 0 0>;
+			wakeup-source;
+		};
+	};
 };
 
 &pio {
@@ -1179,12 +1205,6 @@ pins-vreg-en {
 	};
 
 	pio_default: pio-default-pins {
-		pins-wifi-enable {
-			pinmux = <PINMUX_GPIO58__FUNC_GPIO58>;
-			output-high;
-			drive-strength = <14>;
-		};
-
 		pins-low-power-pd {
 			pinmux = <PINMUX_GPIO25__FUNC_GPIO25>,
 				 <PINMUX_GPIO26__FUNC_GPIO26>,
@@ -1222,6 +1242,12 @@ pins-low-power-pupd {
 		};
 	};
 
+	pp3300_wlan_en_pin: pp3300-wlan-en-pins {
+		pins-en {
+			pinmux = <PINMUX_GPIO58__FUNC_GPIO58>;
+		};
+	};
+
 	rt1019p_pins_default: rt1019p-default-pins {
 		pins-amp-sdb {
 			pinmux = <PINMUX_GPIO100__FUNC_GPIO100>;
@@ -1495,6 +1521,7 @@ &u3phy0 {
 };
 
 &u3phy1 {
+	/* shared between xhci1 and pcie1. */
 	status = "okay";
 };
 
@@ -1563,27 +1590,16 @@ &xhci0 {
 	vbus-supply = <&usb_vbus>;
 };
 
-&xhci1 {
-	status = "okay";
-
-	phys = <&u2port1 PHY_TYPE_USB2>;
-	rx-fifo-depth = <3072>;
-	vusb33-supply = <&mt6359_vusb_ldo_reg>;
-	vbus-supply = <&usb_vbus>;
-	mediatek,u3p-dis-msk = <1>;
-};
-
 &xhci2 {
 	status = "okay";
 	vbus-supply = <&usb_vbus>;
 };
 
 &xhci3 {
-	status = "okay";
-
 	/* MT7921's USB Bluetooth has issues with USB2 LPM */
 	usb2-lpm-disable;
-	vbus-supply = <&usb_vbus>;
+	vbus-supply = <&pp3300_wlan>;
+	status = "okay";
 };
 
 #include <arm/cros-ec-keyboard.dtsi>
diff --git a/arch/arm64/boot/dts/mediatek/mt8365-evk.dts b/arch/arm64/boot/dts/mediatek/mt8365-evk.dts
index b5dd5ef9fa11a..a30ee523b0b58 100644
--- a/arch/arm64/boot/dts/mediatek/mt8365-evk.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8365-evk.dts
@@ -20,8 +20,10 @@ / {
 	compatible = "mediatek,mt8365-evk", "mediatek,mt8365";
 
 	aliases {
-		serial0 = &uart0;
 		ethernet = &ethernet;
+		mmc0 = &mmc0;
+		mmc1 = &mmc1;
+		serial0 = &uart0;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/mediatek/mt8365.dtsi b/arch/arm64/boot/dts/mediatek/mt8365.dtsi
index a5ca3cda6ef30..2e782558fb776 100644
--- a/arch/arm64/boot/dts/mediatek/mt8365.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8365.dtsi
@@ -536,10 +536,9 @@ iommu: iommu@10205000 {
 			#iommu-cells = <1>;
 		};
 
-		infracfg_nao: infracfg@1020e000 {
-			compatible = "mediatek,mt8365-infracfg", "syscon";
+		infracfg_nao: syscon@1020e000 {
+			compatible = "mediatek,mt8365-infracfg-nao", "syscon";
 			reg = <0 0x1020e000 0 0x1000>;
-			#clock-cells = <1>;
 		};
 
 		rng: rng@1020f000 {
diff --git a/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi b/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
index d849af4d36134..62c336e215009 100644
--- a/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
@@ -18,8 +18,10 @@
 
 / {
 	aliases {
-		serial0 = &uart0;
 		ethernet0 = &eth;
+		mmc0 = &mmc0;
+		mmc1 = &mmc1;
+		serial0 = &uart0;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
index 1cd4b84e98615..bf91305e8e4a5 100644
--- a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
@@ -21,12 +21,14 @@ / {
 	compatible = "radxa,nio-12l", "mediatek,mt8395", "mediatek,mt8195";
 
 	aliases {
+		ethernet0 = &eth;
 		i2c0 = &i2c2;
 		i2c1 = &i2c3;
 		i2c2 = &i2c4;
 		i2c3 = &i2c0;
 		i2c4 = &i2c1;
-		ethernet0 = &eth;
+		mmc0 = &mmc0;
+		mmc1 = &mmc1;
 		serial0 = &uart0;
 		serial1 = &uart1;
 		spi0 = &spi1;
diff --git a/arch/arm64/boot/dts/microchip/Makefile b/arch/arm64/boot/dts/microchip/Makefile
index c6e0313eea0f9..09d16fc1ce9ac 100644
--- a/arch/arm64/boot/dts/microchip/Makefile
+++ b/arch/arm64/boot/dts/microchip/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
+dtb-$(CONFIG_ARCH_LAN969X) += lan9696-ev23x71a.dtb
 dtb-$(CONFIG_ARCH_SPARX5) += sparx5_pcb125.dtb
 dtb-$(CONFIG_ARCH_SPARX5) += sparx5_pcb134.dtb sparx5_pcb134_emmc.dtb
 dtb-$(CONFIG_ARCH_SPARX5) += sparx5_pcb135.dtb sparx5_pcb135_emmc.dtb
diff --git a/arch/arm64/boot/dts/microchip/clk-lan9691.h b/arch/arm64/boot/dts/microchip/clk-lan9691.h
new file mode 100644
index 0000000000000..0f2d7a0f881ef
--- /dev/null
+++ b/arch/arm64/boot/dts/microchip/clk-lan9691.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) */
+
+#ifndef _DTS_CLK_LAN9691_H
+#define _DTS_CLK_LAN9691_H
+
+#define GCK_ID_QSPI0		0
+#define GCK_ID_QSPI2		1
+#define GCK_ID_SDMMC0		2
+#define GCK_ID_SDMMC1		3
+#define GCK_ID_MCAN0		4
+#define GCK_ID_MCAN1		5
+#define GCK_ID_FLEXCOM0		6
+#define GCK_ID_FLEXCOM1		7
+#define GCK_ID_FLEXCOM2		8
+#define GCK_ID_FLEXCOM3		9
+#define GCK_ID_TIMER		10
+#define GCK_ID_USB_REFCLK	11
+
+/* Gate clocks */
+#define GCK_GATE_USB_DRD	12
+#define GCK_GATE_MCRAMC		13
+#define GCK_GATE_HMATRIX	14
+
+#endif
diff --git a/arch/arm64/boot/dts/microchip/lan9691.dtsi b/arch/arm64/boot/dts/microchip/lan9691.dtsi
new file mode 100644
index 0000000000000..235e56bebbdbb
--- /dev/null
+++ b/arch/arm64/boot/dts/microchip/lan9691.dtsi
@@ -0,0 +1,488 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2025 Microchip Technology Inc. and its subsidiaries.
+ */
+
+#include <dt-bindings/dma/at91.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/mfd/at91-usart.h>
+#include <dt-bindings/mfd/atmel-flexcom.h>
+
+#include "clk-lan9691.h"
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	model = "Microchip LAN969x";
+	compatible = "microchip,lan9691";
+	interrupt-parent = <&gic>;
+
+	clocks {
+		fx100_clk: fx100-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <320000000>;
+		};
+
+		cpu_clk: cpu-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <1000000000>;
+		};
+
+		ddr_clk: ddr-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <600000000>;
+		};
+
+		fabric_clk: fabric-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <250000000>;
+		};
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a53";
+			device_type = "cpu";
+			reg = <0x0 0x0>;
+			next-level-cache = <&l2_0>;
+		};
+
+		l2_0: l2-cache {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-unified;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	pmu {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>, /* Secure Phys IRQ */
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* Non-secure Phys IRQ */
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* Virt IRQ */
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* Hyp IRQ */
+	};
+
+	axi: axi {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		usb: usb@300000 {
+			compatible = "microchip,lan9691-dwc3", "snps,dwc3";
+			reg = <0x300000 0x80000>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clks GCK_GATE_USB_DRD>,
+				 <&clks GCK_ID_USB_REFCLK>;
+			clock-names = "bus_early", "ref";
+			assigned-clocks = <&clks GCK_ID_USB_REFCLK>;
+			assigned-clock-rates = <60000000>;
+			maximum-speed = "high-speed";
+			dr_mode = "host";
+			status = "disabled";
+		};
+
+		flx0: flexcom@e0040000 {
+			compatible = "microchip,lan9691-flexcom", "atmel,sama5d2-flexcom";
+			reg = <0xe0040000 0x100>;
+			ranges = <0x0 0xe0040000 0x800>;
+			clocks = <&clks GCK_ID_FLEXCOM0>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			status = "disabled";
+
+			usart0: serial@200 {
+				compatible = "microchip,lan9691-usart", "atmel,at91sam9260-usart";
+				reg = <0x200 0x200>;
+				interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "usart";
+				atmel,fifo-size = <32>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
+				status = "disabled";
+			};
+
+			spi0: spi@400 {
+				compatible = "microchip,lan9691-spi", "atmel,at91rm9200-spi";
+				reg = <0x400 0x200>;
+				interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "spi_clk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				atmel,fifo-size = <32>;
+				status = "disabled";
+			};
+
+			i2c0: i2c@600 {
+				compatible = "microchip,lan9691-i2c", "microchip,sam9x60-i2c";
+				reg = <0x600 0x200>;
+				interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		flx1: flexcom@e0044000 {
+			compatible = "microchip,lan9691-flexcom", "atmel,sama5d2-flexcom";
+			reg = <0xe0044000 0x100>;
+			ranges = <0x0 0xe0044000 0x800>;
+			clocks = <&clks GCK_ID_FLEXCOM1>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			status = "disabled";
+
+			usart1: serial@200 {
+				compatible = "microchip,lan9691-usart", "atmel,at91sam9260-usart";
+				reg = <0x200 0x200>;
+				interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "usart";
+				atmel,fifo-size = <32>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
+				status = "disabled";
+			};
+
+			spi1: spi@400 {
+				compatible = "microchip,lan9691-spi", "atmel,at91rm9200-spi";
+				reg = <0x400 0x200>;
+				interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "spi_clk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				atmel,fifo-size = <32>;
+				status = "disabled";
+			};
+
+			i2c1: i2c@600 {
+				compatible = "microchip,lan9691-i2c", "microchip,sam9x60-i2c";
+				reg = <0x600 0x200>;
+				interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(3)>,
+				       <&dma AT91_XDMAC_DT_PERID(2)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		trng: rng@e0048000 {
+			compatible = "microchip,lan9691-trng", "atmel,at91sam9g45-trng";
+			reg = <0xe0048000 0x100>;
+			clocks = <&fabric_clk>;
+			status = "disabled";
+		};
+
+		aes: crypto@e004c000 {
+			compatible = "microchip,lan9691-aes", "atmel,at91sam9g46-aes";
+			reg = <0xe004c000 0x100>;
+			interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&dma AT91_XDMAC_DT_PERID(12)>,
+			       <&dma AT91_XDMAC_DT_PERID(13)>;
+			dma-names = "tx", "rx";
+			clocks = <&fabric_clk>;
+			clock-names = "aes_clk";
+			status = "disabled";
+		};
+
+		flx2: flexcom@e0060000 {
+			compatible = "microchip,lan9691-flexcom", "atmel,sama5d2-flexcom";
+			reg = <0xe0060000 0x100>;
+			ranges = <0x0 0xe0060000 0x800>;
+			clocks = <&clks GCK_ID_FLEXCOM2>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			status = "disabled";
+
+			usart2: serial@200 {
+				compatible = "microchip,lan9691-usart", "atmel,at91sam9260-usart";
+				reg = <0x200 0x200>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(7)>,
+				       <&dma AT91_XDMAC_DT_PERID(6)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "usart";
+				atmel,fifo-size = <32>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
+				status = "disabled";
+			};
+
+			spi2: spi@400 {
+				compatible = "microchip,lan9691-spi", "atmel,at91rm9200-spi";
+				reg = <0x400 0x200>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(7)>,
+				       <&dma AT91_XDMAC_DT_PERID(6)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "spi_clk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				atmel,fifo-size = <32>;
+				status = "disabled";
+			};
+
+			i2c2: i2c@600 {
+				compatible = "microchip,lan9691-i2c", "microchip,sam9x60-i2c";
+				reg = <0x600 0x200>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(7)>,
+				       <&dma AT91_XDMAC_DT_PERID(6)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		flx3: flexcom@e0064000 {
+			compatible = "microchip,lan9691-flexcom", "atmel,sama5d2-flexcom";
+			reg = <0xe0064000 0x100>;
+			ranges = <0x0 0xe0064000 0x800>;
+			clocks = <&clks GCK_ID_FLEXCOM3>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			status = "disabled";
+
+			usart3: serial@200 {
+				compatible = "microchip,lan9691-usart", "atmel,at91sam9260-usart";
+				reg = <0x200 0x200>;
+				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(9)>,
+				       <&dma AT91_XDMAC_DT_PERID(8)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "usart";
+				atmel,fifo-size = <32>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
+				status = "disabled";
+			};
+
+			spi3: spi@400 {
+				compatible = "microchip,lan9691-spi", "atmel,at91rm9200-spi";
+				reg = <0x400 0x200>;
+				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(9)>,
+				       <&dma AT91_XDMAC_DT_PERID(8)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				clock-names = "spi_clk";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				atmel,fifo-size = <32>;
+				status = "disabled";
+			};
+
+			i2c3: i2c@600 {
+				compatible = "microchip,lan9691-i2c", "microchip,sam9x60-i2c";
+				reg = <0x600 0x200>;
+				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&dma AT91_XDMAC_DT_PERID(9)>,
+				       <&dma AT91_XDMAC_DT_PERID(8)>;
+				dma-names = "tx", "rx";
+				clocks = <&fabric_clk>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		dma: dma-controller@e0068000 {
+			compatible = "microchip,lan9691-dma", "microchip,sama7g5-dma";
+			reg = <0xe0068000 0x1000>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <16>;
+			#dma-cells = <1>;
+			clocks = <&fabric_clk>;
+			clock-names = "dma_clk";
+		};
+
+		sha: crypto@e006c000 {
+			compatible = "microchip,lan9691-sha", "atmel,at91sam9g46-sha";
+			reg = <0xe006c000 0xec>;
+			interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&dma AT91_XDMAC_DT_PERID(14)>;
+			dma-names = "tx";
+			clocks = <&fabric_clk>;
+			clock-names = "sha_clk";
+			status = "disabled";
+		};
+
+		timer: timer@e008c000 {
+			compatible = "snps,dw-apb-timer";
+			reg = <0xe008c000 0x400>;
+			clocks = <&fabric_clk>;
+			clock-names = "timer";
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		watchdog: watchdog@e0090000 {
+			compatible = "snps,dw-wdt";
+			reg = <0xe0090000 0x1000>;
+			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&fabric_clk>;
+		};
+
+		cpu_ctrl: syscon@e00c0000 {
+			compatible = "microchip,lan966x-cpu-syscon", "syscon";
+			reg = <0xe00c0000 0x350>;
+		};
+
+		switch: switch@e00c0000 {
+			compatible = "microchip,lan9691-switch";
+			reg = <0xe00c0000 0x0010000>,
+			      <0xe2010000 0x1410000>;
+			reg-names = "cpu", "devices";
+			interrupt-names = "xtr", "fdma", "ptp";
+			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI  9 IRQ_TYPE_LEVEL_HIGH>;
+			resets = <&reset 0>;
+			reset-names = "switch";
+			status = "disabled";
+		};
+
+		clks: clock-controller@e00c00b4 {
+			compatible = "microchip,lan9691-gck";
+			reg = <0xe00c00b4 0x30>, <0xe00c0308 0x4>;
+			#clock-cells = <1>;
+			clocks = <&cpu_clk>, <&ddr_clk>, <&fx100_clk>;
+			clock-names = "cpu", "ddr", "sys";
+		};
+
+		reset: reset-controller@e201000c {
+			compatible = "microchip,lan9691-switch-reset",
+				     "microchip,lan966x-switch-reset";
+			reg = <0xe201000c 0x4>;
+			reg-names = "gcb";
+			#reset-cells = <1>;
+			cpu-syscon = <&cpu_ctrl>;
+		};
+
+		gpio: pinctrl@e20100d4 {
+			compatible = "microchip,lan9691-pinctrl";
+			reg = <0xe20100d4 0xd4>,
+			      <0xe2010370 0xa8>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&gpio 0 0 66>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+			#interrupt-cells = <2>;
+		};
+
+		mdio0: mdio@e20101a8 {
+			compatible = "microchip,lan9691-miim", "mscc,ocelot-miim";
+			reg = <0xe20101a8 0x24>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&fx100_clk>;
+			status = "disabled";
+		};
+
+		mdio1: mdio@e20101cc {
+			compatible = "microchip,lan9691-miim", "mscc,ocelot-miim";
+			reg = <0xe20101cc 0x24>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&fx100_clk>;
+			status = "disabled";
+		};
+
+		sgpio: gpio@e2010230 {
+			compatible = "microchip,lan9691-sgpio", "microchip,sparx5-sgpio";
+			reg = <0xe2010230 0x118>;
+			clocks = <&fx100_clk>;
+			resets = <&reset 0>;
+			reset-names = "switch";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			sgpio_in: gpio@0 {
+				compatible = "microchip,lan9691-sgpio-bank",
+					     "microchip,sparx5-sgpio-bank";
+				reg = <0>;
+				gpio-controller;
+				#gpio-cells = <3>;
+				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-controller;
+				#interrupt-cells = <3>;
+			};
+
+			sgpio_out: gpio@1 {
+				compatible = "microchip,lan9691-sgpio-bank",
+					     "microchip,sparx5-sgpio-bank";
+				reg = <1>;
+				gpio-controller;
+				#gpio-cells = <3>;
+			};
+		};
+
+		tmon: hwmon@e2020100 {
+			compatible = "microchip,lan9691-temp", "microchip,sparx5-temp";
+			reg = <0xe2020100 0xc>;
+			clocks = <&fx100_clk>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		serdes: serdes@e3410000 {
+			compatible = "microchip,lan9691-serdes";
+			reg = <0xe3410000 0x150000>;
+			#phy-cells = <1>;
+			clocks = <&fabric_clk>;
+		};
+
+		gic: interrupt-controller@e8c11000 {
+			compatible = "arm,gic-400";
+			reg = <0xe8c11000 0x1000>, /* Distributor GICD_ */
+			      <0xe8c12000 0x2000>, /* CPU interface GICC_ */
+			      <0xe8c14000 0x2000>, /* Virt interface control */
+			      <0xe8c16000 0x2000>; /* Virt CPU interface */
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts b/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
new file mode 100644
index 0000000000000..4012ea7d07bbd
--- /dev/null
+++ b/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
@@ -0,0 +1,756 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2025 Microchip Technology Inc. and its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+#include "lan9691.dtsi"
+
+/ {
+	model = "Microchip EV23X71A";
+	compatible = "microchip,ev23x71a", "microchip,lan9696", "microchip,lan9691";
+
+	aliases {
+		serial0 = &usart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-restart {
+		compatible = "gpio-restart";
+		gpios = <&gpio 60 GPIO_ACTIVE_LOW>;
+		open-source;
+		priority = <200>;
+	};
+
+	i2c-mux {
+		compatible = "i2c-mux-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		i2c-parent = <&i2c3>;
+		idle-state = <0x8>;
+		mux-gpios = <&sgpio_out 0 1 GPIO_ACTIVE_HIGH>,
+			    <&sgpio_out 0 2 GPIO_ACTIVE_HIGH>,
+			    <&sgpio_out 0 3 GPIO_ACTIVE_HIGH>;
+		settle-time-us = <100>;
+
+		i2c_sfp0: i2c@0 {
+			reg = <0x0>;
+		};
+
+		i2c_sfp1: i2c@1 {
+			reg = <0x1>;
+		};
+
+		i2c_sfp2: i2c@2 {
+			reg = <0x2>;
+		};
+
+		i2c_sfp3: i2c@3 {
+			reg = <0x3>;
+		};
+
+		i2c_poe: i2c@7 {
+			reg = <0x7>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-status {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio 61 GPIO_ACTIVE_LOW>;
+		};
+
+		led-sfp1-green {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			gpios = <&sgpio_out 6 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp1-yellow {
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			gpios = <&sgpio_out 6 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp2-green {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <1>;
+			gpios = <&sgpio_out 7 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp2-yellow {
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <1>;
+			gpios = <&sgpio_out 7 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp3-green {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <2>;
+			gpios = <&sgpio_out 8 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp3-yellow {
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <2>;
+			gpios = <&sgpio_out 8 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp4-green {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <3>;
+			gpios = <&sgpio_out 9 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-sfp4-yellow {
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <3>;
+			gpios = <&sgpio_out 9 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+	};
+
+	mux-controller {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		mux-gpios = <&sgpio_out 1 2 GPIO_ACTIVE_LOW>,
+			    <&sgpio_out 1 3 GPIO_ACTIVE_LOW>;
+	};
+
+	sfp0: sfp0 {
+		compatible = "sff,sfp";
+		i2c-bus = <&i2c_sfp0>;
+		tx-disable-gpios = <&sgpio_out 6 2 GPIO_ACTIVE_HIGH>;
+		los-gpios = <&sgpio_in 6 0 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpios = <&sgpio_in 6 1 GPIO_ACTIVE_LOW>;
+		tx-fault-gpios = <&sgpio_in 6 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	sfp1: sfp1 {
+		compatible = "sff,sfp";
+		i2c-bus = <&i2c_sfp1>;
+		tx-disable-gpios = <&sgpio_out 7 2 GPIO_ACTIVE_HIGH>;
+		los-gpios = <&sgpio_in 7 0 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpios = <&sgpio_in 7 1 GPIO_ACTIVE_LOW>;
+		tx-fault-gpios = <&sgpio_in 7 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	sfp2: sfp2 {
+		compatible = "sff,sfp";
+		i2c-bus = <&i2c_sfp2>;
+		tx-disable-gpios = <&sgpio_out 8 2 GPIO_ACTIVE_HIGH>;
+		los-gpios = <&sgpio_in 8 0 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpios = <&sgpio_in 8 1 GPIO_ACTIVE_LOW>;
+		tx-fault-gpios = <&sgpio_in 8 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	sfp3: sfp3 {
+		compatible = "sff,sfp";
+		i2c-bus = <&i2c_sfp3>;
+		tx-disable-gpios = <&sgpio_out 9 2 GPIO_ACTIVE_HIGH>;
+		los-gpios = <&sgpio_in 9 0 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpios = <&sgpio_in 9 1 GPIO_ACTIVE_LOW>;
+		tx-fault-gpios = <&sgpio_in 9 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&gpio {
+	emmc_sd_pins: emmc-sd-pins {
+		/* eMMC_SD - CMD, CLK, D0, D1, D2, D3, D4, D5, D6, D7, RSTN */
+		pins = "GPIO_14", "GPIO_15", "GPIO_16", "GPIO_17",
+		       "GPIO_18", "GPIO_19", "GPIO_20", "GPIO_21",
+		       "GPIO_22", "GPIO_23", "GPIO_24";
+		function = "emmc_sd";
+	};
+
+	fan_pins: fan-pins {
+		pins = "GPIO_25", "GPIO_26";
+		function = "fan";
+	};
+
+	fc0_pins: fc0-pins {
+		pins = "GPIO_3", "GPIO_4";
+		function = "fc";
+	};
+
+	fc2_pins: fc2-pins {
+		pins = "GPIO_64", "GPIO_65", "GPIO_66";
+		function = "fc";
+	};
+
+	fc3_pins: fc3-pins {
+		pins = "GPIO_55", "GPIO_56";
+		function = "fc";
+	};
+
+	mdio_irq_pins: mdio-irq-pins {
+		pins = "GPIO_11";
+		function = "miim_irq";
+	};
+
+	mdio_pins: mdio-pins {
+		pins = "GPIO_9", "GPIO_10";
+		function = "miim";
+	};
+
+	ptp_ext_pins: ptp-ext-pins {
+		pins = "GPIO_59";
+		function = "ptpsync_5";
+	};
+
+	ptp_out_pins: ptp-out-pins {
+		pins = "GPIO_58";
+		function = "ptpsync_4";
+	};
+
+	sgpio_pins: sgpio-pins {
+		/* SCK, D0, D1, LD */
+		pins = "GPIO_5", "GPIO_6", "GPIO_7", "GPIO_8";
+		function = "sgpio_a";
+	};
+
+	usb_over_pins: usb-over-pins {
+		pins = "GPIO_13";
+		function = "usb_over_detect";
+	};
+
+	usb_power_pins: usb-power-pins {
+		pins = "GPIO_1";
+		function = "usb_power";
+	};
+
+	usb_rst_pins: usb-rst-pins {
+		pins = "GPIO_12";
+		function = "usb2phy_rst";
+	};
+
+	usb_ulpi_pins: usb-ulpi-pins {
+		pins = "GPIO_30", "GPIO_31", "GPIO_32", "GPIO_33",
+		       "GPIO_34", "GPIO_35", "GPIO_36", "GPIO_37",
+		       "GPIO_38", "GPIO_39", "GPIO_40", "GPIO_41";
+		function = "usb_ulpi";
+	};
+};
+
+&flx0 {
+	atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_USART>;
+	status = "okay";
+};
+
+&flx2 {
+	atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_SPI>;
+	status = "okay";
+};
+
+&flx3 {
+	atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_TWI>;
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-0 = <&fc3_pins>;
+	pinctrl-names = "default";
+	i2c-analog-filter;
+	i2c-digital-filter;
+	i2c-digital-filter-width-ns = <35>;
+	i2c-sda-hold-time-ns = <1500>;
+	status = "okay";
+};
+
+&mdio0 {
+	pinctrl-0 = <&mdio_pins>, <&mdio_irq_pins>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio 62 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	phy3: phy@3 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <3>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy4: phy@4 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <4>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy5: phy@5 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <5>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy6: phy@6 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <6>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy7: phy@7 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <7>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy8: phy@8 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <8>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy9: phy@9 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <9>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy10: phy@10 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <10>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy11: phy@11 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <11>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy12: phy@12 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <12>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy13: phy@13 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <13>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy14: phy@14 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <14>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy15: phy@15 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <15>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy16: phy@16 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <16>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy17: phy@17 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <17>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy18: phy@18 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <18>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy19: phy@19 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <19>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy20: phy@20 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <20>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy21: phy@21 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <21>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy22: phy@22 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <22>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy23: phy@23 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <23>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy24: phy@24 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <24>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy25: phy@25 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <25>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy26: phy@26 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <26>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+
+	phy27: phy@27 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <27>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio>;
+	};
+};
+
+&serdes {
+	status = "okay";
+};
+
+&sgpio {
+	pinctrl-0 = <&sgpio_pins>;
+	pinctrl-names = "default";
+	microchip,sgpio-port-ranges = <0 1>, <6 9>;
+	status = "okay";
+
+	gpio@0 {
+		ngpios = <128>;
+	};
+	gpio@1 {
+		ngpios = <128>;
+	};
+};
+
+&spi2 {
+	pinctrl-0 = <&fc2_pins>;
+	pinctrl-names = "default";
+	cs-gpios = <&gpio 63 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&switch {
+	pinctrl-0 = <&ptp_out_pins>, <&ptp_ext_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ethernet-ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port0: port@0 {
+			reg = <0>;
+			phy-handle = <&phy4>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 0>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port1: port@1 {
+			reg = <1>;
+			phy-handle = <&phy5>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 0>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port2: port@2 {
+			reg = <2>;
+			phy-handle = <&phy6>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 0>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port3: port@3 {
+			reg = <3>;
+			phy-handle = <&phy7>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 0>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port4: port@4 {
+			reg = <4>;
+			phy-handle = <&phy8>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 1>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port5: port@5 {
+			reg = <5>;
+			phy-handle = <&phy9>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 1>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port6: port@6 {
+			reg = <6>;
+			phy-handle = <&phy10>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 1>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port7: port@7 {
+			reg = <7>;
+			phy-handle = <&phy11>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 1>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port8: port@8 {
+			reg = <8>;
+			phy-handle = <&phy12>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 2>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port9: port@9 {
+			reg = <9>;
+			phy-handle = <&phy13>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 2>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port10: port@10 {
+			reg = <10>;
+			phy-handle = <&phy14>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 2>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port11: port@11 {
+			reg = <11>;
+			phy-handle = <&phy15>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 2>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port12: port@12 {
+			reg = <12>;
+			phy-handle = <&phy16>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 3>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port13: port@13 {
+			reg = <13>;
+			phy-handle = <&phy17>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 3>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port14: port@14 {
+			reg = <14>;
+			phy-handle = <&phy18>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 3>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port15: port@15 {
+			reg = <15>;
+			phy-handle = <&phy19>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 3>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port16: port@16 {
+			reg = <16>;
+			phy-handle = <&phy20>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 4>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port17: port@17 {
+			reg = <17>;
+			phy-handle = <&phy21>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 4>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port18: port@18 {
+			reg = <18>;
+			phy-handle = <&phy22>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 4>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port19: port@19 {
+			reg = <19>;
+			phy-handle = <&phy23>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 4>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port20: port@20 {
+			reg = <20>;
+			phy-handle = <&phy24>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 5>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port21: port@21 {
+			reg = <21>;
+			phy-handle = <&phy25>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 5>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port22: port@22 {
+			reg = <22>;
+			phy-handle = <&phy26>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 5>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port23: port@23 {
+			reg = <23>;
+			phy-handle = <&phy27>;
+			phy-mode = "qsgmii";
+			phys = <&serdes 5>;
+			microchip,bandwidth = <1000>;
+		};
+
+		port24: port@24 {
+			reg = <24>;
+			phys = <&serdes 6>;
+			phy-mode = "10gbase-r";
+			sfp = <&sfp0>;
+			managed = "in-band-status";
+			microchip,bandwidth = <10000>;
+			microchip,sd-sgpio = <24>;
+		};
+
+		port25: port@25 {
+			reg = <25>;
+			phys = <&serdes 7>;
+			phy-mode = "10gbase-r";
+			sfp = <&sfp1>;
+			managed = "in-band-status";
+			microchip,bandwidth = <10000>;
+			microchip,sd-sgpio = <28>;
+		};
+
+		port26: port@26 {
+			reg = <26>;
+			phys = <&serdes 8>;
+			phy-mode = "10gbase-r";
+			sfp = <&sfp2>;
+			managed = "in-band-status";
+			microchip,bandwidth = <10000>;
+			microchip,sd-sgpio = <32>;
+		};
+
+		port27: port@27 {
+			reg = <27>;
+			phys = <&serdes 9>;
+			phy-mode = "10gbase-r";
+			sfp = <&sfp3>;
+			managed = "in-band-status";
+			microchip,bandwidth = <10000>;
+			microchip,sd-sgpio = <36>;
+		};
+
+		port29: port@29 {
+			reg = <29>;
+			phy-handle = <&phy3>;
+			phy-mode = "rgmii-id";
+			microchip,bandwidth = <1000>;
+		};
+	};
+};
+
+&tmon {
+	pinctrl-0 = <&fan_pins>;
+	pinctrl-names = "default";
+};
+
+&usart0 {
+	pinctrl-0 = <&fc0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usb {
+	pinctrl-0 = <&usb_ulpi_pins>, <&usb_rst_pins>, <&usb_over_pins>, <&usb_power_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/nuvoton/nuvoton-common-npcm8xx.dtsi b/arch/arm64/boot/dts/nuvoton/nuvoton-common-npcm8xx.dtsi
index 24133528b8e93..c781190b42c51 100644
--- a/arch/arm64/boot/dts/nuvoton/nuvoton-common-npcm8xx.dtsi
+++ b/arch/arm64/boot/dts/nuvoton/nuvoton-common-npcm8xx.dtsi
@@ -145,7 +145,6 @@ watchdog0: watchdog@801c {
 				reg = <0x801c 0x4>;
 				status = "disabled";
 				clocks = <&refclk>;
-				syscon = <&gcr>;
 			};
 
 			watchdog1: watchdog@901c {
@@ -154,7 +153,6 @@ watchdog1: watchdog@901c {
 				reg = <0x901c 0x4>;
 				status = "disabled";
 				clocks = <&refclk>;
-				syscon = <&gcr>;
 			};
 
 			watchdog2: watchdog@a01c {
@@ -163,7 +161,6 @@ watchdog2: watchdog@a01c {
 				reg = <0xa01c 0x4>;
 				status = "disabled";
 				clocks = <&refclk>;
-				syscon = <&gcr>;
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index f80b5d9cf1e80..2349cd200a61c 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -12,8 +12,15 @@ dtb-$(CONFIG_ARCH_QCOM)	+= apq8016-schneider-hmibsc.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8039-t2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8094-sony-xperia-kitakami-karin_windy.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-db820c.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= apq8096sg-db820c.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-ifc6640.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= eliza-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= glymur-crd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-iot-evk.dtb
+
+hamoa-iot-evk-el2-dtbs	:= hamoa-iot-evk.dtb x1-el2.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-iot-evk-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5018-rdp432-c2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5018-tplink-archer-ax55-v1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5332-rdp441.dtb
@@ -25,8 +32,8 @@ dtb-$(CONFIG_ARCH_QCOM)	+= ipq6018-cp01-c1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq8074-hk01.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq8074-hk10-c1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq8074-hk10-c2.dtb
-dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp418.dtb
-dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp433.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp418.dtb ipq9574-rdp418-emmc.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp433.dtb ipq9574-rdp433-emmc.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp449.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp453.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp454.dtb
@@ -43,8 +50,17 @@ dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk-camera.dtb
 lemans-evk-el2-dtbs := lemans-evk.dtb lemans-el2.dtbo
 
 dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk-el2.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= mahua-crd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= milos-fairphone-fp6.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= monaco-arduino-monza.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk.dtb
+
+monaco-evk-camera-imx577-dtbs	:= monaco-evk.dtb monaco-evk-camera-imx577.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk-camera-imx577.dtb
+
+monaco-evk-el2-dtbs := monaco-evk.dtb monaco-el2.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8216-samsung-fortuna3g.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-acer-a1-724.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-alcatel-idol347.dtb
@@ -75,6 +91,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-rossa.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-serranove.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-thwc-uf896.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-thwc-ufi001c.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-wiko-chuppito.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-wingtech-wt86518.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-wingtech-wt86528.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-wingtech-wt88047.dtb
@@ -130,6 +147,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-lilac.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-maple.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-poplar.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-xiaomi-sagit.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= purwa-iot-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-fairphone-fp5.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-particle-tachyon.dtb
@@ -145,8 +163,13 @@ qcs6490-rb3gen2-industrial-mezzanine-dtbs := qcs6490-rb3gen2.dtb qcs6490-rb3gen2
 
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-industrial-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-vision-mezzanine.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-minipc-g1iot.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride.dtb
+
+qcs8300-ride-el2-dtbs := qcs8300-ride.dtb monaco-el2.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-aim300-aiot.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride-r3.dtb
@@ -178,6 +201,8 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sa8775p-ride.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sa8775p-ride-r3.dtb
 sc7180-acer-aspire1-el2-dtbs	:= sc7180-acer-aspire1.dtb sc7180-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-acer-aspire1.dtb sc7180-acer-aspire1-el2.dtb
+sc7180-ecs-liva-qc710-el2-dtbs	:= sc7180-ecs-liva-qc710.dtb sc7180-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-ecs-liva-qc710.dtb sc7180-ecs-liva-qc710-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-trogdor-coachz-r1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-trogdor-coachz-r1-lte.dtb
@@ -294,8 +319,9 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm4450-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6115-fxtec-pro1x.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6115p-lenovo-j606f.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6125-sony-xperia-seine-pdx201.dtb
-dtb-$(CONFIG_ARCH_QCOM) += sm6125-xiaomi-ginkgo.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm6125-xiaomi-ginkgo.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6125-xiaomi-laurel-sprout.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm6125-xiaomi-willow.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6350-sony-xperia-lena-pdx213.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6375-sony-xperia-murray-pdx225.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7125-xiaomi-curtana.dtb
@@ -374,12 +400,16 @@ x1e80100-lenovo-yoga-slim7x-el2-dtbs	:= x1e80100-lenovo-yoga-slim7x.dtb x1-el2.d
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-lenovo-yoga-slim7x.dtb x1e80100-lenovo-yoga-slim7x-el2.dtb
 x1e80100-medion-sprchrgd-14-s1-el2-dtbs	:= x1e80100-medion-sprchrgd-14-s1.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-medion-sprchrgd-14-s1.dtb x1e80100-medion-sprchrgd-14-s1-el2.dtb
+x1e80100-microsoft-denali-oled-el2-dtbs	:= x1e80100-microsoft-denali-oled.dtb x1-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-microsoft-denali-oled.dtb x1e80100-microsoft-denali-oled-el2.dtb
 x1e80100-microsoft-romulus13-el2-dtbs	:= x1e80100-microsoft-romulus13.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-microsoft-romulus13.dtb x1e80100-microsoft-romulus13-el2.dtb
 x1e80100-microsoft-romulus15-el2-dtbs	:= x1e80100-microsoft-romulus15.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-microsoft-romulus15.dtb x1e80100-microsoft-romulus15-el2.dtb
 x1e80100-qcp-el2-dtbs	:= x1e80100-qcp.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-qcp.dtb x1e80100-qcp-el2.dtb
+x1p42100-asus-vivobook-s15-el2-dtbs	:= x1p42100-asus-vivobook-s15.dtb x1-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= x1p42100-asus-vivobook-s15.dtb x1p42100-asus-vivobook-s15-el2.dtb
 x1p42100-asus-zenbook-a14-el2-dtbs	:= x1p42100-asus-zenbook-a14.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1p42100-asus-zenbook-a14.dtb x1p42100-asus-zenbook-a14-el2.dtb
 x1p42100-asus-zenbook-a14-lcd-el2-dtbs	:= x1p42100-asus-zenbook-a14-lcd.dtb x1-el2.dtbo
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
index 9fa70ff6887b7..47b4568e40391 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
@@ -6,1133 +6,9 @@
 /dts-v1/;
 
 #include "msm8996.dtsi"
-#include "pm8994.dtsi"
-#include "pmi8994.dtsi"
-#include <dt-bindings/input/input.h>
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/leds/common.h>
-#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
-#include <dt-bindings/sound/qcom,q6afe.h>
-#include <dt-bindings/sound/qcom,q6asm.h>
-#include <dt-bindings/sound/qcom,wcd9335.h>
-
-/*
- * GPIO name legend: proper name = the GPIO line is used as GPIO
- *         NC      = not connected (pin out but not routed from the chip to
- *                   anything the board)
- *         "[PER]" = pin is muxed for [peripheral] (not GPIO)
- *         LSEC    = Low Speed External Connector
- *         P HSEC  = Primary High Speed External Connector
- *         S HSEC  = Secondary High Speed External Connector
- *         J14     = Camera Connector
- *         TP      = Test Points
- *
- * Line names are taken from the schematic "DragonBoard 820c",
- * drawing no: LM25-P2751-1
- *
- * For the lines routed to the external connectors the
- * lines are named after the 96Boards CE Specification 1.0,
- * Appendix "Expansion Connector Signal Description".
- *
- * When the 96Board naming of a line and the schematic name of
- * the same line are in conflict, the 96Board specification
- * takes precedence, which means that the external UART on the
- * LSEC is named UART0 while the schematic and SoC names this
- * UART3. This is only for the informational lines i.e. "[FOO]",
- * the GPIO named lines "GPIO-A" thru "GPIO-L" are the only
- * ones actually used for GPIO.
- */
+#include "apq8096-db820c.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. DB820c";
 	compatible = "arrow,apq8096-db820c", "qcom,apq8096-sbc", "qcom,apq8096";
-
-	aliases {
-		serial0 = &blsp2_uart2;
-		serial1 = &blsp2_uart3;
-		serial2 = &blsp1_uart2;
-		i2c0 = &blsp1_i2c3;
-		i2c1 = &blsp2_i2c1;
-		i2c2 = &blsp2_i2c1;
-		spi0 = &blsp1_spi1;
-		spi1 = &blsp2_spi6;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	div1_mclk: divclk1 {
-		compatible = "gpio-gate-clock";
-		pinctrl-0 = <&audio_mclk>;
-		pinctrl-names = "default";
-		clocks = <&rpmcc RPM_SMD_DIV_CLK1>;
-		#clock-cells = <0>;
-		enable-gpios = <&pm8994_gpios 15 0>;
-	};
-
-	divclk4: divclk4 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <32768>;
-		clock-output-names = "divclk4";
-
-		pinctrl-names = "default";
-		pinctrl-0 = <&divclk4_pin_a>;
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-		autorepeat;
-
-		pinctrl-names = "default";
-		pinctrl-0 = <&volume_up_gpio>;
-
-		button {
-			label = "Volume Up";
-			linux,code = <KEY_VOLUMEUP>;
-			gpios = <&pm8994_gpios 2 GPIO_ACTIVE_LOW>;
-		};
-	};
-
-	usb2_id: usb2-id {
-		compatible = "linux,extcon-usb-gpio";
-		id-gpios = <&pmi8994_gpios 6 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&usb2_vbus_det_gpio>;
-	};
-
-	usb3_id: usb3-id {
-		compatible = "linux,extcon-usb-gpio";
-		id-gpios = <&pm8994_gpios 22 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&usb3_vbus_det_gpio>;
-	};
-
-	vph_pwr: vph-pwr-regulator {
-		compatible = "regulator-fixed";
-		regulator-name = "vph_pwr";
-		regulator-always-on;
-		regulator-boot-on;
-
-		regulator-min-microvolt = <3700000>;
-		regulator-max-microvolt = <3700000>;
-	};
-
-	wlan_en: wlan-en-1-8v {
-		pinctrl-names = "default";
-		pinctrl-0 = <&wlan_en_gpios>;
-		compatible = "regulator-fixed";
-		regulator-name = "wlan-en-regulator";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-
-		gpio = <&pm8994_gpios 8 0>;
-
-		/* WLAN card specific delay */
-		startup-delay-us = <70000>;
-		enable-active-high;
-	};
-};
-
-&blsp1_i2c3 {
-	/* On Low speed expansion: LS-I2C0 */
-	status = "okay";
-};
-
-&blsp1_spi1 {
-	/* On Low speed expansion */
-	status = "okay";
-};
-
-&blsp1_uart2 {
-	label = "BT-UART";
-	status = "okay";
-
-	bluetooth {
-		compatible = "qcom,qca6174-bt";
-
-		/* bt_disable_n gpio */
-		enable-gpios = <&pm8994_gpios 19 GPIO_ACTIVE_HIGH>;
-
-		clocks = <&divclk4>;
-	};
-};
-
-&adsp_pil {
-	status = "okay";
-	firmware-name = "qcom/apq8096/adsp.mbn";
-};
-
-&blsp2_i2c1 {
-	/* On High speed expansion: HS-I2C2 */
-	status = "okay";
-};
-
-&blsp2_i2c1 {
-	/* On Low speed expansion: LS-I2C1 */
-	status = "okay";
-};
-
-&blsp2_spi6 {
-	/* On High speed expansion */
-	status = "okay";
-};
-
-&blsp2_uart2 {
-	label = "LS-UART1";
-	status = "okay";
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&blsp2_uart2_2pins_default>;
-	pinctrl-1 = <&blsp2_uart2_2pins_sleep>;
-};
-
-&blsp2_uart3 {
-	label = "LS-UART0";
-	status = "disabled";
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&blsp2_uart3_4pins_default>;
-	pinctrl-1 = <&blsp2_uart3_4pins_sleep>;
-};
-
-&camss {
-	vdda-supply = <&vreg_l2a_1p25>;
-};
-
-&gpu {
-	status = "okay";
-};
-
-&gpu_zap_shader {
-	firmware-name = "qcom/apq8096/a530_zap.mbn";
-};
-
-&hsusb_phy1 {
-	status = "okay";
-
-	vdd-supply = <&vreg_l28a_0p925>;
-	vdda-pll-supply = <&vreg_l12a_1p8>;
-	vdda-phy-dpdm-supply = <&vreg_l24a_3p075>;
-};
-
-&hsusb_phy2 {
-	status = "okay";
-
-	vdd-supply = <&vreg_l28a_0p925>;
-	vdda-pll-supply = <&vreg_l12a_1p8>;
-	vdda-phy-dpdm-supply = <&vreg_l24a_3p075>;
-};
-
-&mdp {
-	status = "okay";
-};
-
-&mdss {
-	status = "okay";
-};
-
-&mdss_hdmi {
-	status = "okay";
-
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&hdmi_hpd_active &hdmi_ddc_active>;
-	pinctrl-1 = <&hdmi_hpd_suspend &hdmi_ddc_suspend>;
-
-	core-vdda-supply = <&vreg_l12a_1p8>;
-	core-vcc-supply = <&vreg_s4a_1p8>;
-};
-
-&mdss_hdmi_phy {
-	status = "okay";
-
-	vddio-supply = <&vreg_l12a_1p8>;
-	vcca-supply = <&vreg_l28a_0p925>;
-	#phy-cells = <0>;
-};
-
-&mmcc {
-	vdd-gfx-supply = <&vdd_gfx>;
-};
-
-&mss_pil {
-	status = "okay";
-	pll-supply = <&vreg_l12a_1p8>;
-	firmware-name = "qcom/apq8096/mba.mbn", "qcom/apq8096/modem.mbn";
-};
-
-&pm8994_resin {
-	status = "okay";
-	linux,code = <KEY_VOLUMEDOWN>;
-};
-
-&tlmm {
-	gpio-line-names =
-		"[SPI0_DOUT]", /* GPIO_0, BLSP1_SPI_MOSI, LSEC pin 14 */
-		"[SPI0_DIN]", /* GPIO_1, BLSP1_SPI_MISO, LSEC pin 10 */
-		"[SPI0_CS]", /* GPIO_2, BLSP1_SPI_CS_N, LSEC pin 12 */
-		"[SPI0_SCLK]", /* GPIO_3, BLSP1_SPI_CLK, LSEC pin 8 */
-		"[UART1_TxD]", /* GPIO_4, BLSP8_UART_TX, LSEC pin 11 */
-		"[UART1_RxD]", /* GPIO_5, BLSP8_UART_RX, LSEC pin 13 */
-		"[I2C1_SDA]", /* GPIO_6, BLSP8_I2C_SDA, LSEC pin 21 */
-		"[I2C1_SCL]", /* GPIO_7, BLSP8_I2C_SCL, LSEC pin 19 */
-		"GPIO-H", /* GPIO_8, LCD0_RESET_N, LSEC pin 30 */
-		"TP93", /* GPIO_9 */
-		"GPIO-G", /* GPIO_10, MDP_VSYNC_P, LSEC pin 29 */
-		"[MDP_VSYNC_S]", /* GPIO_11, S HSEC pin 55 */
-		"NC", /* GPIO_12 */
-		"[CSI0_MCLK]", /* GPIO_13, CAM_MCLK0, P HSEC pin 15 */
-		"[CAM_MCLK1]", /* GPIO_14, J14 pin 11 */
-		"[CSI1_MCLK]", /* GPIO_15, CAM_MCLK2, P HSEC pin 17 */
-		"TP99", /* GPIO_16 */
-		"[I2C2_SDA]", /* GPIO_17, CCI_I2C_SDA0, P HSEC pin 34 */
-		"[I2C2_SCL]", /* GPIO_18, CCI_I2C_SCL0, P HSEC pin 32 */
-		"[CCI_I2C_SDA1]", /* GPIO_19, S HSEC pin 38 */
-		"[CCI_I2C_SCL1]", /* GPIO_20, S HSEC pin 36 */
-		"FLASH_STROBE_EN", /* GPIO_21, S HSEC pin 5 */
-		"FLASH_STROBE_TRIG", /* GPIO_22, S HSEC pin 1 */
-		"GPIO-K", /* GPIO_23, CAM2_RST_N, LSEC pin 33 */
-		"GPIO-D", /* GPIO_24, LSEC pin 26 */
-		"GPIO-I", /* GPIO_25, CAM0_RST_N, LSEC pin 31 */
-		"GPIO-J", /* GPIO_26, CAM0_STANDBY_N, LSEC pin 32 */
-		"BLSP6_I2C_SDA", /* GPIO_27 */
-		"BLSP6_I2C_SCL", /* GPIO_28 */
-		"GPIO-B", /* GPIO_29, TS0_RESET_N, LSEC pin 24 */
-		"GPIO30", /* GPIO_30, S HSEC pin 4 */
-		"HDMI_CEC", /* GPIO_31 */
-		"HDMI_DDC_CLOCK", /* GPIO_32 */
-		"HDMI_DDC_DATA", /* GPIO_33 */
-		"HDMI_HOT_PLUG_DETECT", /* GPIO_34 */
-		"PCIE0_RST_N", /* GPIO_35 */
-		"PCIE0_CLKREQ_N", /* GPIO_36 */
-		"PCIE0_WAKE", /* GPIO_37 */
-		"SD_CARD_DET_N", /* GPIO_38 */
-		"TSIF1_SYNC", /* GPIO_39, S HSEC pin 48 */
-		"W_DISABLE_N", /* GPIO_40 */
-		"[BLSP9_UART_TX]", /* GPIO_41 */
-		"[BLSP9_UART_RX]", /* GPIO_42 */
-		"[BLSP2_UART_CTS_N]", /* GPIO_43 */
-		"[BLSP2_UART_RFR_N]", /* GPIO_44 */
-		"[BLSP3_UART_TX]", /* GPIO_45 */
-		"[BLSP3_UART_RX]", /* GPIO_46 */
-		"[I2C0_SDA]", /* GPIO_47, LS_I2C0_SDA, LSEC pin 17 */
-		"[I2C0_SCL]", /* GPIO_48, LS_I2C0_SCL, LSEC pin 15 */
-		"[UART0_TxD]", /* GPIO_49, BLSP9_UART_TX, LSEC pin 5 */
-		"[UART0_RxD]", /* GPIO_50, BLSP9_UART_RX, LSEC pin 7 */
-		"[UART0_CTS]", /* GPIO_51, BLSP9_UART_CTS_N, LSEC pin 3 */
-		"[UART0_RTS]", /* GPIO_52, BLSP9_UART_RFR_N, LSEC pin 9 */
-		"[CODEC_INT1_N]", /* GPIO_53 */
-		"[CODEC_INT2_N]", /* GPIO_54 */
-		"[BLSP7_I2C_SDA]", /* GPIO_55 */
-		"[BLSP7_I2C_SCL]", /* GPIO_56 */
-		"MI2S_MCLK", /* GPIO_57, S HSEC pin 3 */
-		"[PCM_CLK]", /* GPIO_58, QUA_MI2S_SCK, LSEC pin 18 */
-		"[PCM_FS]", /* GPIO_59, QUA_MI2S_WS, LSEC pin 16 */
-		"[PCM_DO]", /* GPIO_60, QUA_MI2S_DATA0, LSEC pin 20 */
-		"[PCM_DI]", /* GPIO_61, QUA_MI2S_DATA1, LSEC pin 22 */
-		"GPIO-E", /* GPIO_62, LSEC pin 27 */
-		"TP87", /* GPIO_63 */
-		"[CODEC_RST_N]", /* GPIO_64 */
-		"[PCM1_CLK]", /* GPIO_65 */
-		"[PCM1_SYNC]", /* GPIO_66 */
-		"[PCM1_DIN]", /* GPIO_67 */
-		"[PCM1_DOUT]", /* GPIO_68 */
-		"AUDIO_REF_CLK", /* GPIO_69 */
-		"SLIMBUS_CLK", /* GPIO_70 */
-		"SLIMBUS_DATA0", /* GPIO_71 */
-		"SLIMBUS_DATA1", /* GPIO_72 */
-		"NC", /* GPIO_73 */
-		"NC", /* GPIO_74 */
-		"NC", /* GPIO_75 */
-		"NC", /* GPIO_76 */
-		"TP94", /* GPIO_77 */
-		"NC", /* GPIO_78 */
-		"TP95", /* GPIO_79 */
-		"GPIO-A", /* GPIO_80, MEMS_RESET_N, LSEC pin 23 */
-		"TP88", /* GPIO_81 */
-		"TP89", /* GPIO_82 */
-		"TP90", /* GPIO_83 */
-		"TP91", /* GPIO_84 */
-		"[SD_DAT0]", /* GPIO_85, BLSP12_SPI_MOSI, P HSEC pin 1 */
-		"[SD_CMD]", /* GPIO_86, BLSP12_SPI_MISO, P HSEC pin 11 */
-		"[SD_DAT3]", /* GPIO_87, BLSP12_SPI_CS_N, P HSEC pin 7 */
-		"[SD_SCLK]", /* GPIO_88, BLSP12_SPI_CLK, P HSEC pin 9 */
-		"TSIF1_CLK", /* GPIO_89, S HSEC pin 42 */
-		"TSIF1_EN", /* GPIO_90, S HSEC pin 46 */
-		"TSIF1_DATA", /* GPIO_91, S HSEC pin 44 */
-		"NC", /* GPIO_92 */
-		"TSIF2_CLK", /* GPIO_93, S HSEC pin 52 */
-		"TSIF2_EN", /* GPIO_94, S HSEC pin 56 */
-		"TSIF2_DATA", /* GPIO_95, S HSEC pin 54 */
-		"TSIF2_SYNC", /* GPIO_96, S HSEC pin 58 */
-		"NC", /* GPIO_97 */
-		"CAM1_STANDBY_N", /* GPIO_98 */
-		"NC", /* GPIO_99 */
-		"NC", /* GPIO_100 */
-		"[LCD1_RESET_N]", /* GPIO_101, S HSEC pin 51 */
-		"BOOT_CONFIG1", /* GPIO_102 */
-		"USB_HUB_RESET", /* GPIO_103 */
-		"CAM1_RST_N", /* GPIO_104 */
-		"NC", /* GPIO_105 */
-		"NC", /* GPIO_106 */
-		"NC", /* GPIO_107 */
-		"NC", /* GPIO_108 */
-		"NC", /* GPIO_109 */
-		"NC", /* GPIO_110 */
-		"NC", /* GPIO_111 */
-		"NC", /* GPIO_112 */
-		"PMI8994_BUA", /* GPIO_113 */
-		"PCIE2_RST_N", /* GPIO_114 */
-		"PCIE2_CLKREQ_N", /* GPIO_115 */
-		"PCIE2_WAKE", /* GPIO_116 */
-		"SSC_IRQ_0", /* GPIO_117 */
-		"SSC_IRQ_1", /* GPIO_118 */
-		"SSC_IRQ_2", /* GPIO_119 */
-		"NC", /* GPIO_120 */
-		"GPIO121", /* GPIO_121, S HSEC pin 2 */
-		"NC", /* GPIO_122 */
-		"SSC_IRQ_6", /* GPIO_123 */
-		"SSC_IRQ_7", /* GPIO_124 */
-		"GPIO-C", /* GPIO_125, TS_INT0, LSEC pin 25 */
-		"BOOT_CONFIG5", /* GPIO_126 */
-		"NC", /* GPIO_127 */
-		"NC", /* GPIO_128 */
-		"BOOT_CONFIG7", /* GPIO_129 */
-		"PCIE1_RST_N", /* GPIO_130 */
-		"PCIE1_CLKREQ_N", /* GPIO_131 */
-		"PCIE1_WAKE", /* GPIO_132 */
-		"GPIO-L", /* GPIO_133, CAM2_STANDBY_N, LSEC pin 34 */
-		"NC", /* GPIO_134 */
-		"NC", /* GPIO_135 */
-		"BOOT_CONFIG8", /* GPIO_136 */
-		"NC", /* GPIO_137 */
-		"NC", /* GPIO_138 */
-		"GPS_SSBI2", /* GPIO_139 */
-		"GPS_SSBI1", /* GPIO_140 */
-		"NC", /* GPIO_141 */
-		"NC", /* GPIO_142 */
-		"NC", /* GPIO_143 */
-		"BOOT_CONFIG6", /* GPIO_144 */
-		"NC", /* GPIO_145 */
-		"NC", /* GPIO_146 */
-		"NC", /* GPIO_147 */
-		"NC", /* GPIO_148 */
-		"NC"; /* GPIO_149 */
-
-	sdc2_cd_on: sdc2-cd-on-state {
-		pins = "gpio38";
-		function = "gpio";
-		bias-pull-up;
-		drive-strength = <16>;
-	};
-
-	sdc2_cd_off: sdc2-cd-off-state {
-		pins = "gpio38";
-		function = "gpio";
-		bias-pull-up;
-		drive-strength = <2>;
-	};
-
-	hdmi_hpd_active: hdmi-hpd-active-state {
-		pins = "gpio34";
-		function = "hdmi_hot";
-		bias-pull-down;
-		drive-strength = <16>;
-	};
-
-	hdmi_hpd_suspend: hdmi-hpd-suspend-state {
-		pins = "gpio34";
-		function = "hdmi_hot";
-		bias-pull-down;
-		drive-strength = <2>;
-	};
-
-	hdmi_ddc_active: hdmi-ddc-active-state {
-		pins = "gpio32", "gpio33";
-		function = "hdmi_ddc";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-
-	hdmi_ddc_suspend: hdmi-ddc-suspend-state {
-		pins = "gpio32", "gpio33";
-		function = "hdmi_ddc";
-		drive-strength = <2>;
-		bias-pull-down;
-	};
-};
-
-&pcie0 {
-	status = "okay";
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
-	vddpe-3v3-supply = <&wlan_en>;
-	vdda-supply = <&vreg_l28a_0p925>;
-};
-
-&pcie1 {
-	status = "okay";
-	perst-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
-	vdda-supply = <&vreg_l28a_0p925>;
-};
-
-&pcie2 {
-	status = "okay";
-	perst-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
-	vdda-supply = <&vreg_l28a_0p925>;
-};
-
-&pcie_phy {
-	status = "okay";
-
-	vdda-phy-supply = <&vreg_l28a_0p925>;
-	vdda-pll-supply = <&vreg_l12a_1p8>;
-};
-
-&pm8994_gpios {
-	gpio-line-names =
-		"NC",
-		"KEY_VOLP_N",
-		"NC",
-		"BL1_PWM",
-		"GPIO-F", /* BL0_PWM, LSEC pin 28 */
-		"BL1_EN",
-		"NC",
-		"WLAN_EN",
-		"NC",
-		"NC",
-		"NC",
-		"NC",
-		"NC",
-		"NC",
-		"DIVCLK1",
-		"DIVCLK2",
-		"DIVCLK3",
-		"DIVCLK4",
-		"BT_EN",
-		"PMIC_SLB",
-		"PMIC_BUA",
-		"USB_VBUS_DET";
-
-	pinctrl-names = "default";
-	pinctrl-0 = <&ls_exp_gpio_f &bt_en_gpios>;
-
-	ls_exp_gpio_f: pm8994-gpio5-state {
-		pinconf {
-			pins = "gpio5";
-			function = PMIC_GPIO_FUNC_NORMAL;
-			output-low;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-		};
-	};
-
-	bt_en_gpios: bt-en-pios-state {
-		pinconf {
-			pins = "gpio19";
-			function = PMIC_GPIO_FUNC_NORMAL;
-			output-low;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
-			bias-pull-down;
-		};
-	};
-
-	wlan_en_gpios: wlan-en-gpios-state {
-		pinconf {
-			pins = "gpio8";
-			function = PMIC_GPIO_FUNC_NORMAL;
-			output-low;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
-			bias-pull-down;
-		};
-	};
-
-	audio_mclk: clk-div1-state {
-		pinconf {
-			pins = "gpio15";
-			function = "func1";
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-		};
-	};
-
-	volume_up_gpio: pm8996-gpio2-state {
-		pinconf {
-			pins = "gpio2";
-			function = "normal";
-			input-enable;
-			drive-push-pull;
-			bias-pull-up;
-			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-		};
-	};
-
-	divclk4_pin_a: divclk4-state {
-		pinconf {
-			pins = "gpio18";
-			function = PMIC_GPIO_FUNC_FUNC2;
-
-			bias-disable;
-			power-source = <PM8994_GPIO_S4>;
-		};
-	};
-
-	usb3_vbus_det_gpio: pm8996-gpio22-state {
-		pinconf {
-			pins = "gpio22";
-			function = PMIC_GPIO_FUNC_NORMAL;
-			input-enable;
-			bias-pull-down;
-			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-		};
-	};
-};
-
-&pm8994_mpps {
-	gpio-line-names =
-		"VDDPX_BIAS",
-		"WIFI_LED",
-		"NC",
-		"BT_LED",
-		"PM_MPP05",
-		"PM_MPP06",
-		"PM_MPP07",
-		"NC";
-};
-
-&pm8994_spmi_regulators {
-	qcom,saw-reg = <&saw3>;
-	vdd_s11-supply = <&vph_pwr>;
-
-	s9 {
-		qcom,saw-slave;
-	};
-	s10 {
-		qcom,saw-slave;
-	};
-	s11 {
-		qcom,saw-leader;
-		regulator-name = "VDD_APCC";
-		regulator-always-on;
-		regulator-min-microvolt = <980000>;
-		regulator-max-microvolt = <980000>;
-	};
-};
-
-&pmi8994_gpios {
-	gpio-line-names =
-		"NC",
-		"SPKR_AMP_EN1",
-		"SPKR_AMP_EN2",
-		"TP61",
-		"NC",
-		"USB2_VBUS_DET",
-		"NC",
-		"NC",
-		"NC",
-		"NC";
-
-	usb2_vbus_det_gpio: pmi8996-gpio6-state {
-		pinconf {
-			pins = "gpio6";
-			function = PMIC_GPIO_FUNC_NORMAL;
-			input-enable;
-			bias-pull-down;
-			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
-			power-source = <PM8994_GPIO_S4>; /* 1.8V */
-		};
-	};
-};
-
-&pmi8994_lpg {
-	qcom,power-source = <1>;
-
-	pinctrl-names = "default";
-	pinctrl-0 = <&pmi8994_mpp2_userled4>;
-
-	qcom,dtest = <0 0>,
-		     <0 0>,
-		     <0 0>,
-		     <4 1>;
-
-	status = "okay";
-
-	led@1 {
-		reg = <1>;
-		color = <LED_COLOR_ID_GREEN>;
-		function = LED_FUNCTION_HEARTBEAT;
-		function-enumerator = <1>;
-
-		linux,default-trigger = "heartbeat";
-		default-state = "on";
-	};
-
-	led@2 {
-		reg = <2>;
-		color = <LED_COLOR_ID_GREEN>;
-		function = LED_FUNCTION_HEARTBEAT;
-		function-enumerator = <0>;
-	};
-
-	led@3 {
-		reg = <3>;
-		color = <LED_COLOR_ID_GREEN>;
-		function = LED_FUNCTION_HEARTBEAT;
-		function-enumerator = <2>;
-	};
-
-	led@4 {
-		reg = <4>;
-		color = <LED_COLOR_ID_GREEN>;
-		function = LED_FUNCTION_HEARTBEAT;
-		function-enumerator = <3>;
-	};
-};
-
-&pmi8994_mpps {
-	pmi8994_mpp2_userled4: mpp2-userled4-state {
-		pins = "mpp2";
-		function = "sink";
-
-		output-low;
-		qcom,dtest = <4>;
-	};
-};
-
-&pmi8994_spmi_regulators {
-	vdd_s2-supply = <&vph_pwr>;
-
-	vdd_gfx: s2 {
-		regulator-name = "VDD_GFX";
-		regulator-min-microvolt = <980000>;
-		regulator-max-microvolt = <980000>;
-	};
-};
-
-&rpm_requests {
-	regulators-0 {
-		compatible = "qcom,rpm-pm8994-regulators";
-
-		vdd_s1-supply = <&vph_pwr>;
-		vdd_s2-supply = <&vph_pwr>;
-		vdd_s3-supply = <&vph_pwr>;
-		vdd_s4-supply = <&vph_pwr>;
-		vdd_s5-supply = <&vph_pwr>;
-		vdd_s6-supply = <&vph_pwr>;
-		vdd_s7-supply = <&vph_pwr>;
-		vdd_s8-supply = <&vph_pwr>;
-		vdd_s9-supply = <&vph_pwr>;
-		vdd_s10-supply = <&vph_pwr>;
-		vdd_s11-supply = <&vph_pwr>;
-		vdd_s12-supply = <&vph_pwr>;
-		vdd_l1-supply = <&vreg_s1b_1p025>;
-		vdd_l2_l26_l28-supply = <&vreg_s3a_1p3>;
-		vdd_l3_l11-supply = <&vreg_s3a_1p3>;
-		vdd_l4_l27_l31-supply = <&vreg_s3a_1p3>;
-		vdd_l5_l7-supply = <&vreg_s5a_2p15>;
-		vdd_l6_l12_l32-supply = <&vreg_s5a_2p15>;
-		vdd_l8_l16_l30-supply = <&vph_pwr>;
-		vdd_l9_l10_l18_l22-supply = <&vph_pwr_bbyp>;
-		vdd_l13_l19_l23_l24-supply = <&vph_pwr_bbyp>;
-		vdd_l14_l15-supply = <&vreg_s5a_2p15>;
-		vdd_l17_l29-supply = <&vph_pwr_bbyp>;
-		vdd_l20_l21-supply = <&vph_pwr_bbyp>;
-		vdd_l25-supply = <&vreg_s3a_1p3>;
-		vdd_lvs1_2-supply = <&vreg_s4a_1p8>;
-
-		vreg_s3a_1p3: s3 {
-			regulator-name = "vreg_s3a_1p3";
-			regulator-min-microvolt = <1300000>;
-			regulator-max-microvolt = <1300000>;
-		};
-
-		/**
-		 * 1.8v required on LS expansion
-		 * for mezzanine boards
-		 */
-		vreg_s4a_1p8: s4 {
-			regulator-name = "vreg_s4a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-always-on;
-		};
-		vreg_s5a_2p15: s5 {
-			regulator-name = "vreg_s5a_2p15";
-			regulator-min-microvolt = <2150000>;
-			regulator-max-microvolt = <2150000>;
-		};
-		vreg_s7a_1p0: s7 {
-			regulator-name = "vreg_s7a_1p0";
-			regulator-min-microvolt = <800000>;
-			regulator-max-microvolt = <800000>;
-		};
-
-		vreg_l1a_1p0: l1 {
-			regulator-name = "vreg_l1a_1p0";
-			regulator-min-microvolt = <1000000>;
-			regulator-max-microvolt = <1000000>;
-		};
-		vreg_l2a_1p25: l2 {
-			regulator-name = "vreg_l2a_1p25";
-			regulator-min-microvolt = <1250000>;
-			regulator-max-microvolt = <1250000>;
-		};
-		vreg_l3a_0p875: l3 {
-			regulator-name = "vreg_l3a_0p875";
-			regulator-min-microvolt = <850000>;
-			regulator-max-microvolt = <850000>;
-		};
-		vreg_l4a_1p225: l4 {
-			regulator-name = "vreg_l4a_1p225";
-			regulator-min-microvolt = <1225000>;
-			regulator-max-microvolt = <1225000>;
-		};
-		vreg_l6a_1p2: l6 {
-			regulator-name = "vreg_l6a_1p2";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1200000>;
-		};
-		vreg_l8a_1p8: l8 {
-			regulator-name = "vreg_l8a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l9a_1p8: l9 {
-			regulator-name = "vreg_l9a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l10a_1p8: l10 {
-			regulator-name = "vreg_l10a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l11a_1p15: l11 {
-			regulator-name = "vreg_l11a_1p15";
-			regulator-min-microvolt = <1150000>;
-			regulator-max-microvolt = <1150000>;
-		};
-		vreg_l12a_1p8: l12 {
-			regulator-name = "vreg_l12a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l13a_2p95: l13 {
-			regulator-name = "vreg_l13a_2p95";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <2950000>;
-		};
-		vreg_l14a_1p8: l14 {
-			regulator-name = "vreg_l14a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l15a_1p8: l15 {
-			regulator-name = "vreg_l15a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l16a_2p7: l16 {
-			regulator-name = "vreg_l16a_2p7";
-			regulator-min-microvolt = <2700000>;
-			regulator-max-microvolt = <2700000>;
-		};
-		vreg_l17a_2p8: l17 {
-			regulator-name = "vreg_l17a_2p8";
-			regulator-min-microvolt = <2500000>;
-			regulator-max-microvolt = <2500000>;
-		};
-		vreg_l18a_2p85: l18 {
-			regulator-name = "vreg_l18a_2p85";
-			regulator-min-microvolt = <2700000>;
-			regulator-max-microvolt = <2900000>;
-		};
-		vreg_l19a_2p8: l19 {
-			regulator-name = "vreg_l19a_2p8";
-			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3000000>;
-		};
-		vreg_l20a_2p95: l20 {
-			regulator-name = "vreg_l20a_2p95";
-			regulator-min-microvolt = <2950000>;
-			regulator-max-microvolt = <2950000>;
-			regulator-allow-set-load;
-		};
-		vreg_l21a_2p95: l21 {
-			regulator-name = "vreg_l21a_2p95";
-			regulator-min-microvolt = <2950000>;
-			regulator-max-microvolt = <2950000>;
-			regulator-allow-set-load;
-			regulator-system-load = <200000>;
-		};
-		vreg_l22a_3p0: l22 {
-			regulator-name = "vreg_l22a_3p0";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-		};
-		vreg_l23a_2p8: l23 {
-			regulator-name = "vreg_l23a_2p8";
-			regulator-min-microvolt = <2800000>;
-			regulator-max-microvolt = <2800000>;
-		};
-		vreg_l24a_3p075: l24 {
-			regulator-name = "vreg_l24a_3p075";
-			regulator-min-microvolt = <3075000>;
-			regulator-max-microvolt = <3075000>;
-		};
-		vreg_l25a_1p2: l25 {
-			regulator-name = "vreg_l25a_1p2";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1200000>;
-			regulator-allow-set-load;
-		};
-		vreg_l26a_0p8: l27 {
-			regulator-name = "vreg_l26a_0p8";
-			regulator-min-microvolt = <1000000>;
-			regulator-max-microvolt = <1000000>;
-		};
-		vreg_l28a_0p925: l28 {
-			regulator-name = "vreg_l28a_0p925";
-			regulator-min-microvolt = <925000>;
-			regulator-max-microvolt = <925000>;
-			regulator-allow-set-load;
-		};
-		vreg_l29a_2p8: l29 {
-			regulator-name = "vreg_l29a_2p8";
-			regulator-min-microvolt = <2800000>;
-			regulator-max-microvolt = <2800000>;
-		};
-		vreg_l30a_1p8: l30 {
-			regulator-name = "vreg_l30a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-		vreg_l32a_1p8: l32 {
-			regulator-name = "vreg_l32a_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-		};
-
-		vreg_lvs1a_1p8: lvs1 {
-			regulator-name = "vreg_lvs1a_1p8";
-		};
-
-		vreg_lvs2a_1p8: lvs2 {
-			regulator-name = "vreg_lvs2a_1p8";
-		};
-	};
-
-	regulators-1 {
-		compatible = "qcom,rpm-pmi8994-regulators";
-
-		vdd_s1-supply = <&vph_pwr>;
-		vdd_s2-supply = <&vph_pwr>;
-		vdd_s3-supply = <&vph_pwr>;
-		vdd_bst_byp-supply = <&vph_pwr>;
-
-		vph_pwr_bbyp: boost-bypass {
-			regulator-name = "vph_pwr_bbyp";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-		};
-
-		vreg_s1b_1p025: s1 {
-			regulator-name = "vreg_s1b_1p025";
-			regulator-min-microvolt = <1025000>;
-			regulator-max-microvolt = <1025000>;
-		};
-	};
-};
-
-&sdhc2 {
-	/* External SD card */
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&sdc2_state_on &sdc2_cd_on>;
-	pinctrl-1 = <&sdc2_state_off &sdc2_cd_off>;
-	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
-	vmmc-supply = <&vreg_l21a_2p95>;
-	vqmmc-supply = <&vreg_l13a_2p95>;
-	status = "okay";
-};
-
-&q6asmdai {
-	dai@0 {
-		reg = <MSM_FRONTEND_DAI_MULTIMEDIA1>;
-	};
-
-	dai@1 {
-		reg = <MSM_FRONTEND_DAI_MULTIMEDIA2>;
-	};
-
-	dai@2 {
-		reg = <MSM_FRONTEND_DAI_MULTIMEDIA3>;
-	};
-};
-
-&slim_msm {
-	status = "okay";
-
-	slim@1 {
-		reg = <1>;
-		#address-cells = <2>;
-		#size-cells = <0>;
-
-		tasha_ifd: tas-ifd@0,0 {
-			compatible = "slim217,1a0";
-			reg = <0 0>;
-		};
-
-		wcd9335: codec@1,0 {
-			compatible = "slim217,1a0";
-			reg = <1 0>;
-
-			clock-names = "mclk", "slimbus";
-			clocks = <&div1_mclk>,
-				 <&rpmcc RPM_SMD_BB_CLK1>;
-			interrupt-parent = <&tlmm>;
-			interrupts = <54 IRQ_TYPE_LEVEL_HIGH>,
-				     <53 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "intr1", "intr2";
-			interrupt-controller;
-			#interrupt-cells = <1>;
-
-			pinctrl-0 = <&cdc_reset_active &wcd_intr_default>;
-			pinctrl-names = "default";
-
-			reset-gpios = <&tlmm 64 GPIO_ACTIVE_LOW>;
-			slim-ifc-dev = <&tasha_ifd>;
-
-			#sound-dai-cells = <1>;
-
-			vdd-buck-supply = <&vreg_s4a_1p8>;
-			vdd-buck-sido-supply = <&vreg_s4a_1p8>;
-			vdd-tx-supply = <&vreg_s4a_1p8>;
-			vdd-rx-supply = <&vreg_s4a_1p8>;
-			vdd-io-supply = <&vreg_s4a_1p8>;
-		};
-	};
-};
-
-&sound {
-	compatible = "qcom,apq8096-sndcard";
-	model = "DB820c";
-	audio-routing = "RX_BIAS", "MCLK";
-
-	mm1-dai-link {
-		link-name = "MultiMedia1";
-		cpu {
-			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA1>;
-		};
-	};
-
-	mm2-dai-link {
-		link-name = "MultiMedia2";
-		cpu {
-			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA2>;
-		};
-	};
-
-	mm3-dai-link {
-		link-name = "MultiMedia3";
-		cpu {
-			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA3>;
-		};
-	};
-
-	hdmi-dai-link {
-		link-name = "HDMI";
-		cpu {
-			sound-dai = <&q6afedai HDMI_RX>;
-		};
-
-		platform {
-			sound-dai = <&q6routing>;
-		};
-
-		codec {
-			sound-dai = <&mdss_hdmi 0>;
-		};
-	};
-
-	slim-dai-link {
-		link-name = "SLIM Playback";
-		cpu {
-			sound-dai = <&q6afedai SLIMBUS_6_RX>;
-		};
-
-		platform {
-			sound-dai = <&q6routing>;
-		};
-
-		codec {
-			sound-dai = <&wcd9335 AIF4_PB>;
-		};
-	};
-
-	slimcap-dai-link {
-		link-name = "SLIM Capture";
-		cpu {
-			sound-dai = <&q6afedai SLIMBUS_0_TX>;
-		};
-
-		platform {
-			sound-dai = <&q6routing>;
-		};
-
-		codec {
-			sound-dai = <&wcd9335 AIF1_CAP>;
-		};
-	};
-};
-
-&ufsphy {
-	status = "okay";
-
-	vdda-phy-supply = <&vreg_l28a_0p925>;
-	vdda-pll-supply = <&vreg_l12a_1p8>;
-};
-
-&ufshc {
-	status = "okay";
-
-	vcc-supply = <&vreg_l20a_2p95>;
-	vccq-supply = <&vreg_l25a_1p2>;
-	vccq2-supply = <&vreg_s4a_1p8>;
-	vdd-hba-supply = <&vreg_l25a_1p2>;
-
-	vcc-max-microamp = <600000>;
-	vccq-max-microamp = <450000>;
-	vccq2-max-microamp = <450000>;
-};
-
-&usb2 {
-	status = "okay";
-	extcon = <&usb2_id>;
-};
-
-&usb2_dwc3 {
-	extcon = <&usb2_id>;
-	dr_mode = "otg";
-	maximum-speed = "high-speed";
-};
-
-&usb3 {
-	status = "okay";
-	extcon = <&usb3_id>;
-};
-
-&usb3_dwc3 {
-	extcon = <&usb3_id>;
-	dr_mode = "otg";
-};
-
-&usb3phy {
-	status = "okay";
-
-	vdda-phy-supply = <&vreg_l28a_0p925>;
-	vdda-pll-supply = <&vreg_l12a_1p8>;
-};
-
-&venus {
-	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
new file mode 100644
index 0000000000000..0c076852b4946
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
@@ -0,0 +1,1133 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2014-2016, The Linux Foundation. All rights reserved.
+ */
+
+
+#include "pm8994.dtsi"
+#include "pmi8994.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/sound/qcom,q6afe.h>
+#include <dt-bindings/sound/qcom,q6asm.h>
+#include <dt-bindings/sound/qcom,wcd9335.h>
+
+/*
+ * GPIO name legend: proper name = the GPIO line is used as GPIO
+ *         NC      = not connected (pin out but not routed from the chip to
+ *                   anything the board)
+ *         "[PER]" = pin is muxed for [peripheral] (not GPIO)
+ *         LSEC    = Low Speed External Connector
+ *         P HSEC  = Primary High Speed External Connector
+ *         S HSEC  = Secondary High Speed External Connector
+ *         J14     = Camera Connector
+ *         TP      = Test Points
+ *
+ * Line names are taken from the schematic "DragonBoard 820c",
+ * drawing no: LM25-P2751-1
+ *
+ * For the lines routed to the external connectors the
+ * lines are named after the 96Boards CE Specification 1.0,
+ * Appendix "Expansion Connector Signal Description".
+ *
+ * When the 96Board naming of a line and the schematic name of
+ * the same line are in conflict, the 96Board specification
+ * takes precedence, which means that the external UART on the
+ * LSEC is named UART0 while the schematic and SoC names this
+ * UART3. This is only for the informational lines i.e. "[FOO]",
+ * the GPIO named lines "GPIO-A" thru "GPIO-L" are the only
+ * ones actually used for GPIO.
+ */
+
+/ {
+	aliases {
+		serial0 = &blsp2_uart2;
+		serial1 = &blsp2_uart3;
+		serial2 = &blsp1_uart2;
+		i2c0 = &blsp1_i2c3;
+		i2c1 = &blsp2_i2c1;
+		i2c2 = &blsp2_i2c1;
+		spi0 = &blsp1_spi1;
+		spi1 = &blsp2_spi6;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	div1_mclk: divclk1 {
+		compatible = "gpio-gate-clock";
+		pinctrl-0 = <&audio_mclk>;
+		pinctrl-names = "default";
+		clocks = <&rpmcc RPM_SMD_DIV_CLK1>;
+		#clock-cells = <0>;
+		enable-gpios = <&pm8994_gpios 15 0>;
+	};
+
+	divclk4: divclk4 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "divclk4";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&divclk4_pin_a>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&volume_up_gpio>;
+
+		button {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&pm8994_gpios 2 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	usb2_id: usb2-id {
+		compatible = "linux,extcon-usb-gpio";
+		id-gpios = <&pmi8994_gpios 6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb2_vbus_det_gpio>;
+	};
+
+	usb3_id: usb3-id {
+		compatible = "linux,extcon-usb-gpio";
+		id-gpios = <&pm8994_gpios 22 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb3_vbus_det_gpio>;
+	};
+
+	vph_pwr: vph-pwr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-always-on;
+		regulator-boot-on;
+
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	wlan_en: wlan-en-1-8v {
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlan_en_gpios>;
+		compatible = "regulator-fixed";
+		regulator-name = "wlan-en-regulator";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&pm8994_gpios 8 0>;
+
+		/* WLAN card specific delay */
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+};
+
+&blsp1_i2c3 {
+	/* On Low speed expansion: LS-I2C0 */
+	status = "okay";
+};
+
+&blsp1_spi1 {
+	/* On Low speed expansion */
+	status = "okay";
+};
+
+&blsp1_uart2 {
+	label = "BT-UART";
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,qca6174-bt";
+
+		/* bt_disable_n gpio */
+		enable-gpios = <&pm8994_gpios 19 GPIO_ACTIVE_HIGH>;
+
+		clocks = <&divclk4>;
+	};
+};
+
+&adsp_pil {
+	status = "okay";
+	firmware-name = "qcom/apq8096/adsp.mbn";
+};
+
+&blsp2_i2c1 {
+	/* On High speed expansion: HS-I2C2 */
+	status = "okay";
+};
+
+&blsp2_i2c1 {
+	/* On Low speed expansion: LS-I2C1 */
+	status = "okay";
+};
+
+&blsp2_spi6 {
+	/* On High speed expansion */
+	status = "okay";
+};
+
+&blsp2_uart2 {
+	label = "LS-UART1";
+	status = "okay";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&blsp2_uart2_2pins_default>;
+	pinctrl-1 = <&blsp2_uart2_2pins_sleep>;
+};
+
+&blsp2_uart3 {
+	label = "LS-UART0";
+	status = "disabled";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&blsp2_uart3_4pins_default>;
+	pinctrl-1 = <&blsp2_uart3_4pins_sleep>;
+};
+
+&camss {
+	vdda-supply = <&vreg_l2a_1p25>;
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/apq8096/a530_zap.mbn";
+};
+
+&hsusb_phy1 {
+	status = "okay";
+
+	vdd-supply = <&vreg_l28a_0p925>;
+	vdda-pll-supply = <&vreg_l12a_1p8>;
+	vdda-phy-dpdm-supply = <&vreg_l24a_3p075>;
+};
+
+&hsusb_phy2 {
+	status = "okay";
+
+	vdd-supply = <&vreg_l28a_0p925>;
+	vdda-pll-supply = <&vreg_l12a_1p8>;
+	vdda-phy-dpdm-supply = <&vreg_l24a_3p075>;
+};
+
+&mdp {
+	status = "okay";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_hdmi {
+	status = "okay";
+
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&hdmi_hpd_active &hdmi_ddc_active>;
+	pinctrl-1 = <&hdmi_hpd_suspend &hdmi_ddc_suspend>;
+
+	core-vdda-supply = <&vreg_l12a_1p8>;
+	core-vcc-supply = <&vreg_s4a_1p8>;
+};
+
+&mdss_hdmi_phy {
+	status = "okay";
+
+	vddio-supply = <&vreg_l12a_1p8>;
+	vcca-supply = <&vreg_l28a_0p925>;
+	#phy-cells = <0>;
+};
+
+&mmcc {
+	vdd-gfx-supply = <&vdd_gfx>;
+};
+
+&mss_pil {
+	status = "okay";
+	pll-supply = <&vreg_l12a_1p8>;
+	firmware-name = "qcom/apq8096/mba.mbn", "qcom/apq8096/modem.mbn";
+};
+
+&pm8994_resin {
+	status = "okay";
+	linux,code = <KEY_VOLUMEDOWN>;
+};
+
+&tlmm {
+	gpio-line-names =
+		"[SPI0_DOUT]", /* GPIO_0, BLSP1_SPI_MOSI, LSEC pin 14 */
+		"[SPI0_DIN]", /* GPIO_1, BLSP1_SPI_MISO, LSEC pin 10 */
+		"[SPI0_CS]", /* GPIO_2, BLSP1_SPI_CS_N, LSEC pin 12 */
+		"[SPI0_SCLK]", /* GPIO_3, BLSP1_SPI_CLK, LSEC pin 8 */
+		"[UART1_TxD]", /* GPIO_4, BLSP8_UART_TX, LSEC pin 11 */
+		"[UART1_RxD]", /* GPIO_5, BLSP8_UART_RX, LSEC pin 13 */
+		"[I2C1_SDA]", /* GPIO_6, BLSP8_I2C_SDA, LSEC pin 21 */
+		"[I2C1_SCL]", /* GPIO_7, BLSP8_I2C_SCL, LSEC pin 19 */
+		"GPIO-H", /* GPIO_8, LCD0_RESET_N, LSEC pin 30 */
+		"TP93", /* GPIO_9 */
+		"GPIO-G", /* GPIO_10, MDP_VSYNC_P, LSEC pin 29 */
+		"[MDP_VSYNC_S]", /* GPIO_11, S HSEC pin 55 */
+		"NC", /* GPIO_12 */
+		"[CSI0_MCLK]", /* GPIO_13, CAM_MCLK0, P HSEC pin 15 */
+		"[CAM_MCLK1]", /* GPIO_14, J14 pin 11 */
+		"[CSI1_MCLK]", /* GPIO_15, CAM_MCLK2, P HSEC pin 17 */
+		"TP99", /* GPIO_16 */
+		"[I2C2_SDA]", /* GPIO_17, CCI_I2C_SDA0, P HSEC pin 34 */
+		"[I2C2_SCL]", /* GPIO_18, CCI_I2C_SCL0, P HSEC pin 32 */
+		"[CCI_I2C_SDA1]", /* GPIO_19, S HSEC pin 38 */
+		"[CCI_I2C_SCL1]", /* GPIO_20, S HSEC pin 36 */
+		"FLASH_STROBE_EN", /* GPIO_21, S HSEC pin 5 */
+		"FLASH_STROBE_TRIG", /* GPIO_22, S HSEC pin 1 */
+		"GPIO-K", /* GPIO_23, CAM2_RST_N, LSEC pin 33 */
+		"GPIO-D", /* GPIO_24, LSEC pin 26 */
+		"GPIO-I", /* GPIO_25, CAM0_RST_N, LSEC pin 31 */
+		"GPIO-J", /* GPIO_26, CAM0_STANDBY_N, LSEC pin 32 */
+		"BLSP6_I2C_SDA", /* GPIO_27 */
+		"BLSP6_I2C_SCL", /* GPIO_28 */
+		"GPIO-B", /* GPIO_29, TS0_RESET_N, LSEC pin 24 */
+		"GPIO30", /* GPIO_30, S HSEC pin 4 */
+		"HDMI_CEC", /* GPIO_31 */
+		"HDMI_DDC_CLOCK", /* GPIO_32 */
+		"HDMI_DDC_DATA", /* GPIO_33 */
+		"HDMI_HOT_PLUG_DETECT", /* GPIO_34 */
+		"PCIE0_RST_N", /* GPIO_35 */
+		"PCIE0_CLKREQ_N", /* GPIO_36 */
+		"PCIE0_WAKE", /* GPIO_37 */
+		"SD_CARD_DET_N", /* GPIO_38 */
+		"TSIF1_SYNC", /* GPIO_39, S HSEC pin 48 */
+		"W_DISABLE_N", /* GPIO_40 */
+		"[BLSP9_UART_TX]", /* GPIO_41 */
+		"[BLSP9_UART_RX]", /* GPIO_42 */
+		"[BLSP2_UART_CTS_N]", /* GPIO_43 */
+		"[BLSP2_UART_RFR_N]", /* GPIO_44 */
+		"[BLSP3_UART_TX]", /* GPIO_45 */
+		"[BLSP3_UART_RX]", /* GPIO_46 */
+		"[I2C0_SDA]", /* GPIO_47, LS_I2C0_SDA, LSEC pin 17 */
+		"[I2C0_SCL]", /* GPIO_48, LS_I2C0_SCL, LSEC pin 15 */
+		"[UART0_TxD]", /* GPIO_49, BLSP9_UART_TX, LSEC pin 5 */
+		"[UART0_RxD]", /* GPIO_50, BLSP9_UART_RX, LSEC pin 7 */
+		"[UART0_CTS]", /* GPIO_51, BLSP9_UART_CTS_N, LSEC pin 3 */
+		"[UART0_RTS]", /* GPIO_52, BLSP9_UART_RFR_N, LSEC pin 9 */
+		"[CODEC_INT1_N]", /* GPIO_53 */
+		"[CODEC_INT2_N]", /* GPIO_54 */
+		"[BLSP7_I2C_SDA]", /* GPIO_55 */
+		"[BLSP7_I2C_SCL]", /* GPIO_56 */
+		"MI2S_MCLK", /* GPIO_57, S HSEC pin 3 */
+		"[PCM_CLK]", /* GPIO_58, QUA_MI2S_SCK, LSEC pin 18 */
+		"[PCM_FS]", /* GPIO_59, QUA_MI2S_WS, LSEC pin 16 */
+		"[PCM_DO]", /* GPIO_60, QUA_MI2S_DATA0, LSEC pin 20 */
+		"[PCM_DI]", /* GPIO_61, QUA_MI2S_DATA1, LSEC pin 22 */
+		"GPIO-E", /* GPIO_62, LSEC pin 27 */
+		"TP87", /* GPIO_63 */
+		"[CODEC_RST_N]", /* GPIO_64 */
+		"[PCM1_CLK]", /* GPIO_65 */
+		"[PCM1_SYNC]", /* GPIO_66 */
+		"[PCM1_DIN]", /* GPIO_67 */
+		"[PCM1_DOUT]", /* GPIO_68 */
+		"AUDIO_REF_CLK", /* GPIO_69 */
+		"SLIMBUS_CLK", /* GPIO_70 */
+		"SLIMBUS_DATA0", /* GPIO_71 */
+		"SLIMBUS_DATA1", /* GPIO_72 */
+		"NC", /* GPIO_73 */
+		"NC", /* GPIO_74 */
+		"NC", /* GPIO_75 */
+		"NC", /* GPIO_76 */
+		"TP94", /* GPIO_77 */
+		"NC", /* GPIO_78 */
+		"TP95", /* GPIO_79 */
+		"GPIO-A", /* GPIO_80, MEMS_RESET_N, LSEC pin 23 */
+		"TP88", /* GPIO_81 */
+		"TP89", /* GPIO_82 */
+		"TP90", /* GPIO_83 */
+		"TP91", /* GPIO_84 */
+		"[SD_DAT0]", /* GPIO_85, BLSP12_SPI_MOSI, P HSEC pin 1 */
+		"[SD_CMD]", /* GPIO_86, BLSP12_SPI_MISO, P HSEC pin 11 */
+		"[SD_DAT3]", /* GPIO_87, BLSP12_SPI_CS_N, P HSEC pin 7 */
+		"[SD_SCLK]", /* GPIO_88, BLSP12_SPI_CLK, P HSEC pin 9 */
+		"TSIF1_CLK", /* GPIO_89, S HSEC pin 42 */
+		"TSIF1_EN", /* GPIO_90, S HSEC pin 46 */
+		"TSIF1_DATA", /* GPIO_91, S HSEC pin 44 */
+		"NC", /* GPIO_92 */
+		"TSIF2_CLK", /* GPIO_93, S HSEC pin 52 */
+		"TSIF2_EN", /* GPIO_94, S HSEC pin 56 */
+		"TSIF2_DATA", /* GPIO_95, S HSEC pin 54 */
+		"TSIF2_SYNC", /* GPIO_96, S HSEC pin 58 */
+		"NC", /* GPIO_97 */
+		"CAM1_STANDBY_N", /* GPIO_98 */
+		"NC", /* GPIO_99 */
+		"NC", /* GPIO_100 */
+		"[LCD1_RESET_N]", /* GPIO_101, S HSEC pin 51 */
+		"BOOT_CONFIG1", /* GPIO_102 */
+		"USB_HUB_RESET", /* GPIO_103 */
+		"CAM1_RST_N", /* GPIO_104 */
+		"NC", /* GPIO_105 */
+		"NC", /* GPIO_106 */
+		"NC", /* GPIO_107 */
+		"NC", /* GPIO_108 */
+		"NC", /* GPIO_109 */
+		"NC", /* GPIO_110 */
+		"NC", /* GPIO_111 */
+		"NC", /* GPIO_112 */
+		"PMI8994_BUA", /* GPIO_113 */
+		"PCIE2_RST_N", /* GPIO_114 */
+		"PCIE2_CLKREQ_N", /* GPIO_115 */
+		"PCIE2_WAKE", /* GPIO_116 */
+		"SSC_IRQ_0", /* GPIO_117 */
+		"SSC_IRQ_1", /* GPIO_118 */
+		"SSC_IRQ_2", /* GPIO_119 */
+		"NC", /* GPIO_120 */
+		"GPIO121", /* GPIO_121, S HSEC pin 2 */
+		"NC", /* GPIO_122 */
+		"SSC_IRQ_6", /* GPIO_123 */
+		"SSC_IRQ_7", /* GPIO_124 */
+		"GPIO-C", /* GPIO_125, TS_INT0, LSEC pin 25 */
+		"BOOT_CONFIG5", /* GPIO_126 */
+		"NC", /* GPIO_127 */
+		"NC", /* GPIO_128 */
+		"BOOT_CONFIG7", /* GPIO_129 */
+		"PCIE1_RST_N", /* GPIO_130 */
+		"PCIE1_CLKREQ_N", /* GPIO_131 */
+		"PCIE1_WAKE", /* GPIO_132 */
+		"GPIO-L", /* GPIO_133, CAM2_STANDBY_N, LSEC pin 34 */
+		"NC", /* GPIO_134 */
+		"NC", /* GPIO_135 */
+		"BOOT_CONFIG8", /* GPIO_136 */
+		"NC", /* GPIO_137 */
+		"NC", /* GPIO_138 */
+		"GPS_SSBI2", /* GPIO_139 */
+		"GPS_SSBI1", /* GPIO_140 */
+		"NC", /* GPIO_141 */
+		"NC", /* GPIO_142 */
+		"NC", /* GPIO_143 */
+		"BOOT_CONFIG6", /* GPIO_144 */
+		"NC", /* GPIO_145 */
+		"NC", /* GPIO_146 */
+		"NC", /* GPIO_147 */
+		"NC", /* GPIO_148 */
+		"NC"; /* GPIO_149 */
+
+	sdc2_cd_on: sdc2-cd-on-state {
+		pins = "gpio38";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <16>;
+	};
+
+	sdc2_cd_off: sdc2-cd-off-state {
+		pins = "gpio38";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <2>;
+	};
+
+	hdmi_hpd_active: hdmi-hpd-active-state {
+		pins = "gpio34";
+		function = "hdmi_hot";
+		bias-pull-down;
+		drive-strength = <16>;
+	};
+
+	hdmi_hpd_suspend: hdmi-hpd-suspend-state {
+		pins = "gpio34";
+		function = "hdmi_hot";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	hdmi_ddc_active: hdmi-ddc-active-state {
+		pins = "gpio32", "gpio33";
+		function = "hdmi_ddc";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	hdmi_ddc_suspend: hdmi-ddc-suspend-state {
+		pins = "gpio32", "gpio33";
+		function = "hdmi_ddc";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+};
+
+&pcie0 {
+	status = "okay";
+	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	vddpe-3v3-supply = <&wlan_en>;
+	vdda-supply = <&vreg_l28a_0p925>;
+};
+
+&pcie1 {
+	status = "okay";
+	perst-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
+	vdda-supply = <&vreg_l28a_0p925>;
+};
+
+&pcie2 {
+	status = "okay";
+	perst-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
+	vdda-supply = <&vreg_l28a_0p925>;
+};
+
+&pcie_phy {
+	status = "okay";
+
+	vdda-phy-supply = <&vreg_l28a_0p925>;
+	vdda-pll-supply = <&vreg_l12a_1p8>;
+};
+
+&pm8994_gpios {
+	gpio-line-names =
+		"NC",
+		"KEY_VOLP_N",
+		"NC",
+		"BL1_PWM",
+		"GPIO-F", /* BL0_PWM, LSEC pin 28 */
+		"BL1_EN",
+		"NC",
+		"WLAN_EN",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"DIVCLK1",
+		"DIVCLK2",
+		"DIVCLK3",
+		"DIVCLK4",
+		"BT_EN",
+		"PMIC_SLB",
+		"PMIC_BUA",
+		"USB_VBUS_DET";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&ls_exp_gpio_f &bt_en_gpios>;
+
+	ls_exp_gpio_f: pm8994-gpio5-state {
+		pinconf {
+			pins = "gpio5";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			output-low;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+		};
+	};
+
+	bt_en_gpios: bt-en-pios-state {
+		pinconf {
+			pins = "gpio19";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			output-low;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+			bias-pull-down;
+		};
+	};
+
+	wlan_en_gpios: wlan-en-gpios-state {
+		pinconf {
+			pins = "gpio8";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			output-low;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+			bias-pull-down;
+		};
+	};
+
+	audio_mclk: clk-div1-state {
+		pinconf {
+			pins = "gpio15";
+			function = "func1";
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+		};
+	};
+
+	volume_up_gpio: pm8996-gpio2-state {
+		pinconf {
+			pins = "gpio2";
+			function = "normal";
+			input-enable;
+			drive-push-pull;
+			bias-pull-up;
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+		};
+	};
+
+	divclk4_pin_a: divclk4-state {
+		pinconf {
+			pins = "gpio18";
+			function = PMIC_GPIO_FUNC_FUNC2;
+
+			bias-disable;
+			power-source = <PM8994_GPIO_S4>;
+		};
+	};
+
+	usb3_vbus_det_gpio: pm8996-gpio22-state {
+		pinconf {
+			pins = "gpio22";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			input-enable;
+			bias-pull-down;
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+		};
+	};
+};
+
+&pm8994_mpps {
+	gpio-line-names =
+		"VDDPX_BIAS",
+		"WIFI_LED",
+		"NC",
+		"BT_LED",
+		"PM_MPP05",
+		"PM_MPP06",
+		"PM_MPP07",
+		"NC";
+};
+
+&pm8994_spmi_regulators {
+	qcom,saw-reg = <&saw3>;
+	vdd_s11-supply = <&vph_pwr>;
+
+	s9 {
+		qcom,saw-slave;
+	};
+	s10 {
+		qcom,saw-slave;
+	};
+	s11 {
+		qcom,saw-leader;
+		regulator-name = "VDD_APCC";
+		regulator-always-on;
+		regulator-min-microvolt = <980000>;
+		regulator-max-microvolt = <980000>;
+	};
+};
+
+&pmi8994_gpios {
+	gpio-line-names =
+		"NC",
+		"SPKR_AMP_EN1",
+		"SPKR_AMP_EN2",
+		"TP61",
+		"NC",
+		"USB2_VBUS_DET",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+
+	usb2_vbus_det_gpio: pmi8996-gpio6-state {
+		pinconf {
+			pins = "gpio6";
+			function = PMIC_GPIO_FUNC_NORMAL;
+			input-enable;
+			bias-pull-down;
+			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
+			power-source = <PM8994_GPIO_S4>; /* 1.8V */
+		};
+	};
+};
+
+&pmi8994_lpg {
+	qcom,power-source = <1>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pmi8994_mpp2_userled4>;
+
+	qcom,dtest = <0 0>,
+		     <0 0>,
+		     <0 0>,
+		     <4 1>;
+
+	status = "okay";
+
+	led@1 {
+		reg = <1>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_HEARTBEAT;
+		function-enumerator = <1>;
+
+		linux,default-trigger = "heartbeat";
+		default-state = "on";
+	};
+
+	led@2 {
+		reg = <2>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_HEARTBEAT;
+		function-enumerator = <0>;
+	};
+
+	led@3 {
+		reg = <3>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_HEARTBEAT;
+		function-enumerator = <2>;
+	};
+
+	led@4 {
+		reg = <4>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_HEARTBEAT;
+		function-enumerator = <3>;
+	};
+};
+
+&pmi8994_mpps {
+	pmi8994_mpp2_userled4: mpp2-userled4-state {
+		pins = "mpp2";
+		function = "sink";
+
+		output-low;
+		qcom,dtest = <4>;
+	};
+};
+
+&pmi8994_spmi_regulators {
+	vdd_s2-supply = <&vph_pwr>;
+
+	vdd_gfx: s2 {
+		regulator-name = "VDD_GFX";
+		regulator-min-microvolt = <980000>;
+		regulator-max-microvolt = <980000>;
+	};
+};
+
+&rpm_requests {
+	regulators-0 {
+		compatible = "qcom,rpm-pm8994-regulators";
+
+		vdd_s1-supply = <&vph_pwr>;
+		vdd_s2-supply = <&vph_pwr>;
+		vdd_s3-supply = <&vph_pwr>;
+		vdd_s4-supply = <&vph_pwr>;
+		vdd_s5-supply = <&vph_pwr>;
+		vdd_s6-supply = <&vph_pwr>;
+		vdd_s7-supply = <&vph_pwr>;
+		vdd_s8-supply = <&vph_pwr>;
+		vdd_s9-supply = <&vph_pwr>;
+		vdd_s10-supply = <&vph_pwr>;
+		vdd_s11-supply = <&vph_pwr>;
+		vdd_s12-supply = <&vph_pwr>;
+		vdd_l1-supply = <&vreg_s1b_1p025>;
+		vdd_l2_l26_l28-supply = <&vreg_s3a_1p3>;
+		vdd_l3_l11-supply = <&vreg_s3a_1p3>;
+		vdd_l4_l27_l31-supply = <&vreg_s3a_1p3>;
+		vdd_l5_l7-supply = <&vreg_s5a_2p15>;
+		vdd_l6_l12_l32-supply = <&vreg_s5a_2p15>;
+		vdd_l8_l16_l30-supply = <&vph_pwr>;
+		vdd_l9_l10_l18_l22-supply = <&vph_pwr_bbyp>;
+		vdd_l13_l19_l23_l24-supply = <&vph_pwr_bbyp>;
+		vdd_l14_l15-supply = <&vreg_s5a_2p15>;
+		vdd_l17_l29-supply = <&vph_pwr_bbyp>;
+		vdd_l20_l21-supply = <&vph_pwr_bbyp>;
+		vdd_l25-supply = <&vreg_s3a_1p3>;
+		vdd_lvs1_2-supply = <&vreg_s4a_1p8>;
+
+		vreg_s3a_1p3: s3 {
+			regulator-name = "vreg_s3a_1p3";
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1300000>;
+		};
+
+		/**
+		 * 1.8v required on LS expansion
+		 * for mezzanine boards
+		 */
+		vreg_s4a_1p8: s4 {
+			regulator-name = "vreg_s4a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+		};
+		vreg_s5a_2p15: s5 {
+			regulator-name = "vreg_s5a_2p15";
+			regulator-min-microvolt = <2150000>;
+			regulator-max-microvolt = <2150000>;
+		};
+		vreg_s7a_1p0: s7 {
+			regulator-name = "vreg_s7a_1p0";
+			regulator-min-microvolt = <800000>;
+			regulator-max-microvolt = <800000>;
+		};
+
+		vreg_l1a_1p0: l1 {
+			regulator-name = "vreg_l1a_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1000000>;
+		};
+		vreg_l2a_1p25: l2 {
+			regulator-name = "vreg_l2a_1p25";
+			regulator-min-microvolt = <1250000>;
+			regulator-max-microvolt = <1250000>;
+		};
+		vreg_l3a_0p875: l3 {
+			regulator-name = "vreg_l3a_0p875";
+			regulator-min-microvolt = <850000>;
+			regulator-max-microvolt = <850000>;
+		};
+		vreg_l4a_1p225: l4 {
+			regulator-name = "vreg_l4a_1p225";
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+		};
+		vreg_l6a_1p2: l6 {
+			regulator-name = "vreg_l6a_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+		vreg_l8a_1p8: l8 {
+			regulator-name = "vreg_l8a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l9a_1p8: l9 {
+			regulator-name = "vreg_l9a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l10a_1p8: l10 {
+			regulator-name = "vreg_l10a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l11a_1p15: l11 {
+			regulator-name = "vreg_l11a_1p15";
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1150000>;
+		};
+		vreg_l12a_1p8: l12 {
+			regulator-name = "vreg_l12a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l13a_2p95: l13 {
+			regulator-name = "vreg_l13a_2p95";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+		vreg_l14a_1p8: l14 {
+			regulator-name = "vreg_l14a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l15a_1p8: l15 {
+			regulator-name = "vreg_l15a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l16a_2p7: l16 {
+			regulator-name = "vreg_l16a_2p7";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <2700000>;
+		};
+		vreg_l17a_2p8: l17 {
+			regulator-name = "vreg_l17a_2p8";
+			regulator-min-microvolt = <2500000>;
+			regulator-max-microvolt = <2500000>;
+		};
+		vreg_l18a_2p85: l18 {
+			regulator-name = "vreg_l18a_2p85";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <2900000>;
+		};
+		vreg_l19a_2p8: l19 {
+			regulator-name = "vreg_l19a_2p8";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+		};
+		vreg_l20a_2p95: l20 {
+			regulator-name = "vreg_l20a_2p95";
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+		vreg_l21a_2p95: l21 {
+			regulator-name = "vreg_l21a_2p95";
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+			regulator-system-load = <200000>;
+		};
+		vreg_l22a_3p0: l22 {
+			regulator-name = "vreg_l22a_3p0";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+		};
+		vreg_l23a_2p8: l23 {
+			regulator-name = "vreg_l23a_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+		};
+		vreg_l24a_3p075: l24 {
+			regulator-name = "vreg_l24a_3p075";
+			regulator-min-microvolt = <3075000>;
+			regulator-max-microvolt = <3075000>;
+		};
+		vreg_l25a_1p2: l25 {
+			regulator-name = "vreg_l25a_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-allow-set-load;
+		};
+		vreg_l26a_0p8: l27 {
+			regulator-name = "vreg_l26a_0p8";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1000000>;
+		};
+		vreg_l28a_0p925: l28 {
+			regulator-name = "vreg_l28a_0p925";
+			regulator-min-microvolt = <925000>;
+			regulator-max-microvolt = <925000>;
+			regulator-allow-set-load;
+		};
+		vreg_l29a_2p8: l29 {
+			regulator-name = "vreg_l29a_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+		};
+		vreg_l30a_1p8: l30 {
+			regulator-name = "vreg_l30a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+		vreg_l32a_1p8: l32 {
+			regulator-name = "vreg_l32a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		vreg_lvs1a_1p8: lvs1 {
+			regulator-name = "vreg_lvs1a_1p8";
+		};
+
+		vreg_lvs2a_1p8: lvs2 {
+			regulator-name = "vreg_lvs2a_1p8";
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,rpm-pmi8994-regulators";
+
+		vdd_s1-supply = <&vph_pwr>;
+		vdd_s2-supply = <&vph_pwr>;
+		vdd_s3-supply = <&vph_pwr>;
+		vdd_bst_byp-supply = <&vph_pwr>;
+
+		vph_pwr_bbyp: boost-bypass {
+			regulator-name = "vph_pwr_bbyp";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+		};
+
+		vreg_s1b_1p025: s1 {
+			regulator-name = "vreg_s1b_1p025";
+			regulator-min-microvolt = <1025000>;
+			regulator-max-microvolt = <1025000>;
+		};
+	};
+};
+
+&sdhc2 {
+	/* External SD card */
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&sdc2_state_on &sdc2_cd_on>;
+	pinctrl-1 = <&sdc2_state_off &sdc2_cd_off>;
+	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&vreg_l21a_2p95>;
+	vqmmc-supply = <&vreg_l13a_2p95>;
+	status = "okay";
+};
+
+&q6asmdai {
+	dai@0 {
+		reg = <MSM_FRONTEND_DAI_MULTIMEDIA1>;
+	};
+
+	dai@1 {
+		reg = <MSM_FRONTEND_DAI_MULTIMEDIA2>;
+	};
+
+	dai@2 {
+		reg = <MSM_FRONTEND_DAI_MULTIMEDIA3>;
+	};
+};
+
+&slim_msm {
+	status = "okay";
+
+	slim@1 {
+		reg = <1>;
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		tasha_ifd: tas-ifd@0,0 {
+			compatible = "slim217,1a0";
+			reg = <0 0>;
+		};
+
+		wcd9335: codec@1,0 {
+			compatible = "slim217,1a0";
+			reg = <1 0>;
+
+			clock-names = "mclk", "slimbus";
+			clocks = <&div1_mclk>,
+				 <&rpmcc RPM_SMD_BB_CLK1>;
+			interrupt-parent = <&tlmm>;
+			interrupts = <54 IRQ_TYPE_LEVEL_HIGH>,
+				     <53 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "intr1", "intr2";
+			interrupt-controller;
+			#interrupt-cells = <1>;
+
+			pinctrl-0 = <&cdc_reset_active &wcd_intr_default>;
+			pinctrl-names = "default";
+
+			reset-gpios = <&tlmm 64 GPIO_ACTIVE_LOW>;
+			slim-ifc-dev = <&tasha_ifd>;
+
+			#sound-dai-cells = <1>;
+
+			vdd-buck-supply = <&vreg_s4a_1p8>;
+			vdd-buck-sido-supply = <&vreg_s4a_1p8>;
+			vdd-tx-supply = <&vreg_s4a_1p8>;
+			vdd-rx-supply = <&vreg_s4a_1p8>;
+			vdd-io-supply = <&vreg_s4a_1p8>;
+		};
+	};
+};
+
+&sound {
+	compatible = "qcom,apq8096-sndcard";
+	model = "DB820c";
+	audio-routing = "RX_BIAS", "MCLK";
+
+	mm1-dai-link {
+		link-name = "MultiMedia1";
+		cpu {
+			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA1>;
+		};
+	};
+
+	mm2-dai-link {
+		link-name = "MultiMedia2";
+		cpu {
+			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA2>;
+		};
+	};
+
+	mm3-dai-link {
+		link-name = "MultiMedia3";
+		cpu {
+			sound-dai = <&q6asmdai  MSM_FRONTEND_DAI_MULTIMEDIA3>;
+		};
+	};
+
+	hdmi-dai-link {
+		link-name = "HDMI";
+		cpu {
+			sound-dai = <&q6afedai HDMI_RX>;
+		};
+
+		platform {
+			sound-dai = <&q6routing>;
+		};
+
+		codec {
+			sound-dai = <&mdss_hdmi 0>;
+		};
+	};
+
+	slim-dai-link {
+		link-name = "SLIM Playback";
+		cpu {
+			sound-dai = <&q6afedai SLIMBUS_6_RX>;
+		};
+
+		platform {
+			sound-dai = <&q6routing>;
+		};
+
+		codec {
+			sound-dai = <&wcd9335 AIF4_PB>;
+		};
+	};
+
+	slimcap-dai-link {
+		link-name = "SLIM Capture";
+		cpu {
+			sound-dai = <&q6afedai SLIMBUS_0_TX>;
+		};
+
+		platform {
+			sound-dai = <&q6routing>;
+		};
+
+		codec {
+			sound-dai = <&wcd9335 AIF1_CAP>;
+		};
+	};
+};
+
+&ufsphy {
+	status = "okay";
+
+	vdda-phy-supply = <&vreg_l28a_0p925>;
+	vdda-pll-supply = <&vreg_l12a_1p8>;
+};
+
+&ufshc {
+	status = "okay";
+
+	vcc-supply = <&vreg_l20a_2p95>;
+	vccq-supply = <&vreg_l25a_1p2>;
+	vccq2-supply = <&vreg_s4a_1p8>;
+	vdd-hba-supply = <&vreg_l25a_1p2>;
+
+	vcc-max-microamp = <600000>;
+	vccq-max-microamp = <450000>;
+	vccq2-max-microamp = <450000>;
+};
+
+&usb2 {
+	status = "okay";
+	extcon = <&usb2_id>;
+};
+
+&usb2_dwc3 {
+	extcon = <&usb2_id>;
+	dr_mode = "otg";
+	maximum-speed = "high-speed";
+};
+
+&usb3 {
+	status = "okay";
+	extcon = <&usb3_id>;
+};
+
+&usb3_dwc3 {
+	extcon = <&usb3_id>;
+	dr_mode = "otg";
+};
+
+&usb3phy {
+	status = "okay";
+
+	vdda-phy-supply = <&vreg_l28a_0p925>;
+	vdda-pll-supply = <&vreg_l12a_1p8>;
+};
+
+&venus {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/apq8096sg-db820c.dts b/arch/arm64/boot/dts/qcom/apq8096sg-db820c.dts
new file mode 100644
index 0000000000000..f3ab5a7c6e53a
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/apq8096sg-db820c.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "msm8996pro.dtsi"
+#include "apq8096-db820c.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. DB820c";
+	compatible = "arrow,apq8096sg-db820c", "arrow,apq8096-db820c",
+		"qcom,apq8096-sbc", "qcom,apq8096sg", "qcom,apq8096";
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
new file mode 100644
index 0000000000000..90f629800cb02
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "eliza.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Eliza MTP";
+	compatible = "qcom,eliza-mtp", "qcom,eliza";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart14;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <76800000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7550-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3b>;
+		vdd-l2-l3-supply = <&vreg_s3b>;
+		vdd-l4-l5-supply = <&vreg_s2b>;
+		vdd-l6-supply = <&vreg_s2b>;
+		vdd-l7-supply = <&vreg_s1b>;
+		vdd-l8-supply = <&vreg_s1b>;
+		vdd-l9-l10-supply = <&vreg_s1b>;
+		vdd-l11-supply = <&vreg_s1b>;
+		vdd-l12-l14-supply = <&vreg_bob>;
+		vdd-l13-l16-supply = <&vreg_bob>;
+		vdd-l15-l17-l18-l19-l20-l21-l22-l23-supply = <&vreg_bob>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "b";
+
+		vreg_s1b: smps1 {
+			regulator-name = "vreg_s1b";
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <2040000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2b: smps2 {
+			regulator-name = "vreg_s2b";
+			regulator-min-microvolt = <375000>;
+			regulator-max-microvolt = <2744000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3b: smps3 {
+			regulator-name = "vreg_s3b";
+			regulator-min-microvolt = <375000>;
+			regulator-max-microvolt = <2744000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4b: smps4 {
+			regulator-name = "vreg_s4b";
+			regulator-min-microvolt = <2156000>;
+			regulator-max-microvolt = <2400000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b: ldo2 {
+			regulator-name = "vreg_l2b";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b: ldo3 {
+			regulator-name = "vreg_l3b";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b: ldo4 {
+			regulator-name = "vreg_l4b";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b: ldo6 {
+			regulator-name = "vreg_l6b";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b: ldo7 {
+			regulator-name = "vreg_l7b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b: ldo8 {
+			regulator-name = "vreg_l8b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b: ldo9 {
+			regulator-name = "vreg_l9b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b: ldo10 {
+			regulator-name = "vreg_l10b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b: ldo11 {
+			regulator-name = "vreg_l11b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b: ldo12 {
+			regulator-name = "vreg_l12b";
+			/* Voltage range for UFS 3.x and above */
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b: ldo13 {
+			regulator-name = "vreg_l13b";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b: ldo14 {
+			regulator-name = "vreg_l14b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b: ldo15 {
+			regulator-name = "vreg_l15b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b: ldo16 {
+			regulator-name = "vreg_l16b";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b: ldo17 {
+			regulator-name = "vreg_l17b";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b: ldo18 {
+			regulator-name = "vreg_l18b";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b: ldo19 {
+			regulator-name = "vreg_l19b";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l20b: ldo20 {
+			regulator-name = "vreg_l20b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l21b: ldo21 {
+			regulator-name = "vreg_l21b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l22b: ldo22 {
+			regulator-name = "vreg_l22b";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l23b: ldo23 {
+			regulator-name = "vreg_l23b";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s2b>;
+
+		qcom,pmic-id = "d";
+
+		vreg_l1d: ldo1 {
+			regulator-name = "vreg_l1d";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s2b>;
+		vdd-l3-supply = <&vreg_s2b>;
+
+		qcom,pmic-id = "g";
+
+		vreg_l1g: ldo1 {
+			regulator-name = "vreg_l1g";
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g: ldo3 {
+			regulator-name = "vreg_l3g";
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmr735d-rpmh-regulators";
+
+		vdd-l1-l2-l5-supply = <&vreg_s3b>;
+		vdd-l3-l4-supply = <&vreg_s2b>;
+		vdd-l6-supply = <&vreg_s1b>;
+		vdd-l7-supply = <&vreg_s3b>;
+
+		qcom,pmic-id = "k";
+
+		vreg_l1k: ldo1 {
+			regulator-name = "vreg_l1k";
+			regulator-min-microvolt = <488000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k: ldo2 {
+			regulator-name = "vreg_l2k";
+			regulator-min-microvolt = <920000>;
+			regulator-max-microvolt = <969000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k: ldo3 {
+			regulator-name = "vreg_l3k";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1350000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4k: ldo4 {
+			regulator-name = "vreg_l4k";
+			regulator-min-microvolt = <960000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5k: ldo5 {
+			regulator-name = "vreg_l5k";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <931000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6k: ldo6 {
+			regulator-name = "vreg_l6k";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7k: ldo7 {
+			regulator-name = "vreg_l7k";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <20 4>,   /* NFC SPI */
+			       <111 2>,  /* WCN UART1 */
+			       <118 1>;  /* NFC Secure I/O */
+};
+
+&uart14 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 185 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l12b>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l1d>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l6b>;
+	vdda-pll-supply = <&vreg_l4b>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi
new file mode 100644
index 0000000000000..190f10a77d748
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/eliza.dtsi
@@ -0,0 +1,1317 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,eliza-gcc.h>
+#include <dt-bindings/clock/qcom,eliza-tcsr.h>
+#include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/interconnect/qcom,eliza-rpmh.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interconnect/qcom,osm-l3.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/soc/qcom,rpmh-rsc.h>
+
+/ {
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x0 0x0>;
+
+			clocks = <&cpufreq_hw 0>;
+
+			power-domains = <&cpu_pd0>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+
+			qcom,freq-domain = <&cpufreq_hw 0>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+
+				l3: l3-cache {
+					compatible = "cache";
+					cache-level = <3>;
+					cache-unified;
+				};
+			};
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x0 0x100>;
+
+			clocks = <&cpufreq_hw 0>;
+
+			power-domains = <&cpu_pd1>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+
+			qcom,freq-domain = <&cpufreq_hw 0>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x0 0x200>;
+
+			clocks = <&cpufreq_hw 0>;
+
+			power-domains = <&cpu_pd2>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_2>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+
+			qcom,freq-domain = <&cpufreq_hw 0>;
+
+			l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x0 0x300>;
+
+			clocks = <&cpufreq_hw 1>;
+
+			power-domains = <&cpu_pd3>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_3>;
+			capacity-dmips-mhz = <1792>;
+			dynamic-power-coefficient = <238>;
+
+			qcom,freq-domain = <&cpufreq_hw 1>;
+
+			l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x0 0x400>;
+
+			clocks = <&cpufreq_hw 1>;
+
+			power-domains = <&cpu_pd4>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_4>;
+			capacity-dmips-mhz = <1792>;
+			dynamic-power-coefficient = <238>;
+
+			qcom,freq-domain = <&cpufreq_hw 1>;
+
+			l2_4: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x0 0x500>;
+
+			clocks = <&cpufreq_hw 1>;
+
+			power-domains = <&cpu_pd5>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_5>;
+			capacity-dmips-mhz = <1792>;
+			dynamic-power-coefficient = <238>;
+
+			qcom,freq-domain = <&cpufreq_hw 1>;
+
+			l2_5: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu6: cpu@600 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x0 0x600>;
+
+			clocks = <&cpufreq_hw 1>;
+
+			power-domains = <&cpu_pd6>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_6>;
+			capacity-dmips-mhz = <1792>;
+			dynamic-power-coefficient = <238>;
+
+			qcom,freq-domain = <&cpufreq_hw 1>;
+
+			l2_6: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu7: cpu@700 {
+			device_type = "cpu";
+			compatible = "arm,cortex-x3";
+			reg = <0x0 0x700>;
+
+			clocks = <&cpufreq_hw 2>;
+
+			power-domains = <&cpu_pd7>;
+			power-domain-names = "psci";
+
+			enable-method = "psci";
+			next-level-cache = <&l2_7>;
+			capacity-dmips-mhz = <1894>;
+			dynamic-power-coefficient = <588>;
+
+			qcom,freq-domain = <&cpufreq_hw 2>;
+
+			l2_7: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+			};
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+
+				core5 {
+					cpu = <&cpu5>;
+				};
+
+				core6 {
+					cpu = <&cpu6>;
+				};
+
+				core7 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cluster0_c4: cpu-sleep-0 {
+				compatible = "arm,idle-state";
+				idle-state-name = "silver-rail-power-collapse";
+				arm,psci-suspend-param = <0x40000004>;
+				entry-latency-us = <550>;
+				exit-latency-us = <750>;
+				min-residency-us = <6700>;
+			};
+
+			cluster1_c4: cpu-sleep-1 {
+				compatible = "arm,idle-state";
+				idle-state-name = "gold-rail-power-collapse";
+				arm,psci-suspend-param = <0x40000004>;
+				entry-latency-us = <550>;
+				exit-latency-us = <1050>;
+				min-residency-us = <7951>;
+			};
+
+			cluster2_c4: cpu-sleep-2 {
+				compatible = "arm,idle-state";
+				idle-state-name = "gold-plus-rail-power-collapse";
+				arm,psci-suspend-param = <0x40000004>;
+				entry-latency-us = <500>;
+				exit-latency-us = <1350>;
+				min-residency-us = <7480>;
+			};
+		};
+
+		domain-idle-states {
+			cluster_sleep_0: cluster-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x41000044>;
+				entry-latency-us = <750>;
+				exit-latency-us = <2350>;
+				min-residency-us = <9144>;
+			};
+
+			cluster_sleep_1: cluster-sleep-1 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x4100b344>;
+				entry-latency-us = <2800>;
+				exit-latency-us = <4400>;
+				min-residency-us = <10150>;
+			};
+		};
+	};
+
+	firmware {
+		scm: scm {
+			compatible = "qcom,scm-eliza", "qcom,scm";
+			interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			qcom,dload-mode = <&tcsr 0x1a000>;
+		};
+	};
+
+	clk_virt: interconnect-0 {
+		compatible = "qcom,eliza-clk-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	mc_virt: interconnect-1 {
+		compatible = "qcom,eliza-mc-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	memory@a0000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpu_pd0: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd1: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd2: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd3: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cpu_pd4: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cpu_pd5: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cpu_pd6: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cpu_pd7: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster2_c4>;
+		};
+
+		cluster_pd: power-domain-cluster {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_sleep_0>,
+					     <&cluster_sleep_1>;
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gunyah_hyp_mem: gunyah-hyp@80000000 {
+			reg = <0x0 0x80000000 0x0 0xe00000>;
+			no-map;
+		};
+
+		cpusys_vm_mem: cpusys-vm-mem@80e00000 {
+			reg = <0x0 0x80e00000 0x0 0x40000>;
+			no-map;
+		};
+
+		cpucp_mem: cpucp@81200000 {
+			reg = <0x0 0x81200000 0x0 0x200000>;
+			no-map;
+		};
+
+		xbl_dtlog_mem: xbl-dtlog@81a00000 {
+			reg = <0x0 0x81a00000 0x0 0x40000>;
+			no-map;
+		};
+
+		aop_image_mem: aop-image@81c00000 {
+			reg = <0x0 0x81c00000 0x0 0x60000>;
+			no-map;
+		};
+
+		aop_cmd_db_mem: aop-cmd-db@81c60000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x81c60000 0x0 0x20000>;
+			no-map;
+		};
+
+		/* Merged aop_config, tme_crash_dump, tme_log and uefi_log regions */
+		aop_tme_uefi_merged_mem: aop-tme-uefi-merged@81c80000 {
+			reg = <0x0 0x81c80000 0x0 0x74000>;
+			no-map;
+		};
+
+		/* Secdata region can be reused by apps */
+		smem_mem: smem@81d00000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0x81d00000 0x0 0x200000>;
+			hwlocks = <&tcsr_mutex 3>;
+			no-map;
+		};
+
+		cpucp_scandump_mem: cpucp-scandump@82000000 {
+			reg = <0x0 0x82200000 0x0 0x180000>;
+			no-map;
+		};
+
+		adsp_mhi_mem: adsp-mhi@82380000 {
+			reg = <0x0 0x82380000 0x0 0x20000>;
+			no-map;
+		};
+
+		soccp_sdi_mem: soccp-sdi@823a0000 {
+			reg = <0x0 0x823a0000 0x0 0x40000>;
+			no-map;
+		};
+
+		pmic_minii_dump_mem: pmic-minii-dump@823e0000 {
+			reg = <0x0 0x823e0000 0x0 0x80000>;
+			no-map;
+		};
+
+		pvmfw_mem: pvmfw@824a0000 {
+			reg = <0x0 0x824a0000 0x0 0x100000>;
+			no-map;
+		};
+
+		hyp_db_mem: hyp-db@825a0000 {
+			reg = <0x0 0x825a0000 0x0 0x60000>;
+			no-map;
+		};
+
+		global_sync_mem: global-sync@82600000 {
+			reg = <0x0 0x82600000 0x0 0x100000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat@82700000 {
+			reg = <0x0 0x82700000 0x0 0x100000>;
+			no-map;
+		};
+
+		qdss_mem: qdss@82800000 {
+			reg = <0x0 0x82800000 0x0 0x2000000>;
+			no-map;
+		};
+
+		dsm_partition_1_mem: dsm-partition-1@84a00000 {
+			reg = <0x0 0x84a00000 0x0 0x3700000>;
+			no-map;
+		};
+
+		mpss_mem: mpss@88100000 {
+			reg = <0x0 0x88100000 0x0 0xcd00000>;
+			no-map;
+		};
+
+		q6_mpss_dtb_mem: q6-mpss-dtb@94e00000 {
+			reg = <0x0 0x94e00000 0x0 0x80000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa-fw@94e80000 {
+			reg = <0x0 0x94e80000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@94e90000 {
+			reg = <0x0 0x94e90000 0x0 0xa000>;
+			no-map;
+		};
+
+		gpu_micro_code_mem: gpu-micro-code@94e9a000 {
+			reg = <0x0 0x94e9a000 0x0 0x2000>;
+			no-map;
+		};
+
+		camera_mem: camera@94f00000 {
+			reg = <0x0 0x94f00000 0x0 0x800000>;
+			no-map;
+		};
+
+		camera_2_mem: camera-2@95700000 {
+			reg = <0x0 0x95700000 0x0 0x800000>;
+			no-map;
+		};
+
+		video_mem: video@95f00000 {
+			reg = <0x0 0x95f00000 0x0 0x800000>;
+			no-map;
+		};
+
+		soccp_mem: soccp@96700000 {
+			reg = <0x0 0x96700000 0x0 0x180000>;
+			no-map;
+		};
+
+		wpss_mem: wpss@97000000 {
+			reg = <0x0 0x97000000 0x0 0x1900000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@98900000 {
+			reg = <0x0 0x98900000 0x0 0x1400000>;
+			no-map;
+		};
+
+		q6_cdsp_dtb_mem: q6-cdsp-dtb@99d00000 {
+			reg = <0x0 0x99d00000 0x0 0x80000>;
+			no-map;
+		};
+
+		q6_adsp_dtb_mem: q6-adsp-dtb@99d80000 {
+			reg = <0x0 0x99d80000 0x0 0x80000>;
+			no-map;
+		};
+
+		adspslpi_mem: adspslpi@99e00000 {
+			reg = <0x0 0x99e00000 0x0 0x2a00000>;
+			no-map;
+		};
+
+		wlan_msa_mem: wlan-msa@a6400000 {
+			reg = <0x0 0xa6400000 0x0 0xc00000>;
+			no-map;
+		};
+
+		xbl_ramdump_mem: xbl-ramdump@b8000000 {
+			reg = <0x0 0xb8000000 0x0 0x1c0000>;
+			no-map;
+		};
+
+		/* Merged tz_reserved, xbl_sc, and qtee regions */
+		tz_merged_mem: tz-merged@d8000000 {
+			reg = <0x0 0xd8000000 0x0 0x600000>;
+			no-map;
+		};
+
+		trust_ui_vm_mem: trust-ui-vm@f3800000 {
+			reg = <0x0 0xf3800000 0x0 0x4400000>;
+			no-map;
+		};
+
+		oem_vm_mem: oem-vm@f7c00000 {
+			reg = <0x0 0xf7c00000 0x0 0x4c00000>;
+			no-map;
+		};
+
+		llcc_lpi_mem: llcc-lpi@ff800000 {
+			reg = <0x0 0xff800000 0x0 0x180000>;
+			no-map;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+		ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		gcc: clock-controller@100000 {
+			compatible = "qcom,eliza-gcc";
+			reg = <0x0 0x00100000 0x0 0x1f4200>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&sleep_clk>,
+				 <0>,
+				 <0>,
+				 <&ufs_mem_phy 0>,
+				 <&ufs_mem_phy 1>,
+				 <&ufs_mem_phy 2>,
+				 <0>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		qupv3_2: geniqup@8c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x008c0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_2_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_2_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0x423 0x0>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			uart14: serial@894000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart14_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		config_noc: interconnect@1600000 {
+			compatible = "qcom,eliza-cnoc-cfg";
+			reg = <0x0 0x01600000 0x0 0x5200>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		cnoc_main: interconnect@1500000 {
+			compatible = "qcom,eliza-cnoc-main";
+			reg = <0x0 0x01500000 0x0 0x16080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		system_noc: interconnect@1680000 {
+			compatible = "qcom,eliza-system-noc";
+			reg = <0x0 0x01680000 0x0 0x40000>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie_noc: interconnect@16c0000 {
+			compatible = "qcom,eliza-pcie-anoc";
+			reg = <0x0 0x016c0000 0x0 0x11400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				<&gcc GCC_CFG_NOC_PCIE_ANOC_AHB_CLK>;
+			#interconnect-cells = <2>;
+		};
+
+		aggre1_noc: interconnect@16e0000 {
+			compatible = "qcom,eliza-aggre1-noc";
+			reg = <0x0 0x016e0000 0x0 0x16400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>;
+			#interconnect-cells = <2>;
+		};
+
+		aggre2_noc: interconnect@1700000 {
+			compatible = "qcom,eliza-aggre2-noc";
+			reg = <0x0 0x01700000 0x0 0x1f400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&rpmhcc RPMH_IPA_CLK>;
+			#interconnect-cells = <2>;
+		};
+
+		mmss_noc: interconnect@1780000 {
+			compatible = "qcom,eliza-mmss-noc";
+			reg = <0x0 0x01780000 0x0 0x7d800>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		ufs_mem_phy: phy@1d80000 {
+			compatible = "qcom,eliza-qmp-ufs-phy",
+				     "qcom,sm8650-qmp-ufs-phy";
+			reg = <0x0 0x01d80000 0x0 0x2000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&gcc GCC_UFS_PHY_PHY_AUX_CLK>,
+				 <&tcsr TCSR_UFS_CLKREF_EN>;
+
+			clock-names = "ref",
+				      "ref_aux",
+				      "qref";
+
+			resets = <&ufs_mem_hc 0>;
+			reset-names = "ufsphy";
+
+			power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>;
+
+			#clock-cells = <1>;
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		ufs_mem_hc: ufshc@1d84000 {
+			compatible = "qcom,eliza-ufshc",
+				     "qcom,ufshc",
+				     "jedec,ufs-2.0";
+			reg = <0x0 0x01d84000 0x0 0x3000>,
+			      <0x0 0x01da0000 0x0 0x15000>;
+			reg-names = "std",
+				    "mcq";
+
+			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_UFS_PHY_AHB_CLK>,
+				 <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>,
+				 <&rpmhcc RPMH_LN_BB_CLK3>,
+				 <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>;
+			clock-names = "core_clk",
+				      "bus_aggr_clk",
+				      "iface_clk",
+				      "core_clk_unipro",
+				      "ref_clk",
+				      "tx_lane0_sync_clk",
+				      "rx_lane0_sync_clk",
+				      "rx_lane1_sync_clk";
+
+			operating-points-v2 = <&ufs_opp_table>;
+
+			resets = <&gcc GCC_UFS_PHY_BCR>;
+			reset-names = "rst";
+
+			interconnects = <&aggre1_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "ufs-ddr",
+					     "cpu-ufs";
+
+			power-domains = <&gcc GCC_UFS_PHY_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
+
+			iommus = <&apps_smmu 0x60 0x0>;
+			dma-coherent;
+
+			lanes-per-direction = <2>;
+			qcom,ice = <&ice>;
+
+			phys = <&ufs_mem_phy>;
+			phy-names = "ufsphy";
+
+			#reset-cells = <1>;
+
+			status = "disabled";
+
+			ufs_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-201500000 {
+					opp-hz = /bits/ 64 <201500000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <201500000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_svs_l1>;
+				};
+
+				opp-403000000 {
+					opp-hz = /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		ice: crypto@1d88000 {
+			compatible = "qcom,eliza-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x01d88000 0x0 0x18000>;
+
+			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+		};
+
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01f40000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr: clock-controller@1fbf000 {
+			compatible = "qcom,eliza-tcsr", "syscon";
+			reg = <0x0 0x01fbf000 0x0 0x21000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		lpass_ag_noc: interconnect@7e40000 {
+			compatible = "qcom,eliza-lpass-ag-noc";
+			reg = <0x0 0x07e40000 0x0 0xe080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_lpiaon_noc: interconnect@7400000 {
+			compatible = "qcom,eliza-lpass-lpiaon-noc";
+			reg = <0x0 0x07400000 0x0 0x19080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_lpicx_noc: interconnect@7420000 {
+			compatible = "qcom,eliza-lpass-lpicx-noc";
+			reg = <0x0 0x07420000 0x0 0x44080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pdc: interrupt-controller@b220000 {
+			compatible = "qcom,eliza-pdc", "qcom,pdc";
+			reg = <0x0 0x0b220000 0x0 0x40000>,
+			      <0x0 0x174000f0 0x0 0x64>;
+
+			qcom,pdc-ranges = <0 480 8>, <8 719 1>, <9 718 1>,
+					  <10 230 1>, <11 724 1>, <12 716 1>,
+					  <13 727 1>, <14 720 1>, <15 726 1>,
+					  <16 721 1>, <17 262 1>, <18 70 1>,
+					  <19 723 1>, <20 234 1>, <22 725 1>,
+					  <23 231 1>, <24 504 5>, <30 510 8>,
+					  <40 520 6>, <51 531 4>, <58 538 2>,
+					  <61 541 5>, <66 92 1>, <67 547 13>,
+					  <80 240 1>, <81 235 1>, <82 310 2>,
+					  <84 248 1>, <85 241 1>, <86 238 2>,
+					  <88 254 1>, <89 509 1>, <90 563 1>,
+					  <91 259 2>, <93 201 1>, <94 246 1>,
+					  <95 93 1>, <96 611 29>, <125 63 1>,
+					  <126 366 2>, <128 374 1>, <129 377 1>,
+					  <130 428 1>, <131 434 2>, <133 437 1>,
+					  <134 452 2>, <136 458 2>, <138 464 11>,
+					  <149 671 1>, <150 688 1>, <151 714 2>,
+					  <153 722 1>, <154 255 1>, <155 269 2>,
+					  <157 276 1>, <158 287 1>, <159 306 4>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&intc>;
+			interrupt-controller;
+		};
+
+		spmi: arbiter@c400000 {
+			compatible = "qcom,eliza-spmi-pmic-arb",
+				     "qcom,x1e80100-spmi-pmic-arb";
+			reg = <0 0x0c400000 0 0x3000>,
+			      <0 0x0c500000 0 0x400000>,
+			      <0 0x0c440000 0 0x80000>;
+			reg-names = "core", "chnls", "obsrvr";
+
+			qcom,ee = <0>;
+			qcom,channel = <0>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			spmi_bus0: spmi@c42d000 {
+				reg = <0 0x0c42d000 0 0x4000>,
+				      <0 0x0c4c0000 0 0x10000>;
+				reg-names = "cnfg", "intr";
+
+				interrupt-names = "periph_irq";
+				interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-controller;
+				#interrupt-cells = <4>;
+
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+
+			spmi_bus1: spmi@c432000 {
+				reg = <0 0x0c432000 0 0x4000>,
+				      <0 0x0c4d0000 0 0x10000>;
+				reg-names = "cnfg", "intr";
+
+				interrupt-names = "periph_irq";
+				interrupts-extended = <&pdc 3 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-controller;
+				#interrupt-cells = <4>;
+
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+		};
+
+		apps_smmu: iommu@15000000 {
+			compatible = "qcom,eliza-smmu-500", "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x15000000 0x0 0x100000>;
+
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 690 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 691 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 692 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 693 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 694 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 695 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 696 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 697 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 488 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 490 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 491 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 492 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 493 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 494 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 497 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 498 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 499 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 500 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 501 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 502 IRQ_TYPE_LEVEL_HIGH>;
+
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			dma-coherent;
+		};
+
+		intc: interrupt-controller@17100000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x17100000 0x0 0x10000>,
+			      <0x0 0x17180000 0x0 0x200000>;
+
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+			#interrupt-cells = <3>;
+			interrupt-controller;
+
+			#redistributor-regions = <1>;
+			redistributor-stride = <0x0 0x40000>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			gic_its: msi-controller@17140000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x17140000 0x0 0x40000>;
+
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		apps_rsc: rsc@17a00000 {
+			compatible = "qcom,rpmh-rsc";
+			reg = <0x0 0x17a00000 0x0 0x10000>,
+			      <0x0 0x17a10000 0x0 0x10000>,
+			      <0x0 0x17a20000 0x0 0x10000>;
+			reg-names = "drv-0",
+				    "drv-1",
+				    "drv-2";
+
+			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+
+			power-domains = <&cluster_pd>;
+			label = "apps_rsc";
+
+			qcom,tcs-offset = <0xd00>;
+			qcom,drv-id = <2>;
+			qcom,tcs-config = <ACTIVE_TCS 3>,
+					  <SLEEP_TCS 2>,
+					  <WAKE_TCS 2>,
+					  <CONTROL_TCS 0>;
+
+			apps_bcm_voter: bcm-voter {
+				compatible = "qcom,bcm-voter";
+			};
+
+			rpmhcc: clock-controller {
+				compatible = "qcom,eliza-rpmh-clk";
+				#clock-cells = <1>;
+				clocks = <&xo_board>;
+				clock-names = "xo";
+			};
+
+			rpmhpd: power-controller {
+				compatible = "qcom,eliza-rpmhpd";
+
+				operating-points-v2 = <&rpmhpd_opp_table>;
+
+				#power-domain-cells = <1>;
+
+				rpmhpd_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					rpmhpd_opp_ret: opp-16 {
+						opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
+					};
+
+					rpmhpd_opp_min_svs: opp-48 {
+						opp-level = <RPMH_REGULATOR_LEVEL_MIN_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_d3: opp-50 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3>;
+					};
+
+					rpmhpd_opp_low_svs_d2: opp-52 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					};
+
+					rpmhpd_opp_low_svs_d1: opp-56 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					};
+
+					rpmhpd_opp_low_svs_d0: opp-60 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					};
+
+					rpmhpd_opp_low_svs: opp-64 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_l1: opp-80 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					};
+
+					rpmhpd_opp_svs: opp-128 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					};
+
+					rpmhpd_opp_svs_l0: opp-144 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					};
+
+					rpmhpd_opp_svs_l1: opp-192 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					};
+
+					rpmhpd_opp_svs_l2: opp-224 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					};
+
+					rpmhpd_opp_nom: opp-256 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					};
+
+					rpmhpd_opp_nom_l1: opp-320 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					};
+
+					rpmhpd_opp_nom_l2: opp-336 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L2>;
+					};
+
+					rpmhpd_opp_turbo: opp-384 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					};
+
+					rpmhpd_opp_turbo_l1: opp-416 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					};
+
+					rpmhpd_opp_turbo_l2: opp-432 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L2>;
+					};
+
+					rpmhpd_opp_turbo_l3: opp-448 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					};
+
+					rpmhpd_opp_turbo_l4: opp-452 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					};
+
+					rpmhpd_opp_super_turbo_no_cpr: opp-480 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SUPER_TURBO_NO_CPR>;
+					};
+				};
+			};
+		};
+
+		epss_l3: interconnect@17d90000 {
+			compatible = "qcom,eliza-epss-l3", "qcom,epss-l3";
+			reg = <0x0 0x17d90000 0x0 0x1000>;
+
+			clocks = <&bi_tcxo_div2>, <&gcc GCC_GPLL0>;
+			clock-names = "xo", "alternate";
+
+			#interconnect-cells = <1>;
+		};
+
+		cpufreq_hw: cpufreq@17d91000 {
+			compatible = "qcom,eliza-cpufreq-epss", "qcom,cpufreq-epss";
+			reg = <0x0 0x17d91000 0x0 0x1000>,
+			      <0x0 0x17d92000 0x0 0x1000>,
+			      <0x0 0x17d93000 0x0 0x1000>;
+			reg-names = "freq-domain0",
+				    "freq-domain1",
+				    "freq-domain2";
+
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dcvsh-irq-0",
+					  "dcvsh-irq-1",
+					  "dcvsh-irq-2";
+
+			clocks = <&bi_tcxo_div2>, <&gcc GCC_GPLL0>;
+			clock-names = "xo", "alternate";
+
+			#freq-domain-cells = <1>;
+			#clock-cells = <1>;
+		};
+
+		tlmm: pinctrl@f100000 {
+			compatible = "qcom,eliza-tlmm";
+			reg = <0x0 0x0f100000 0x0 0xf00000>;
+
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			gpio-ranges = <&tlmm 0 0 184>;
+			wakeup-parent = <&pdc>;
+
+			qup_uart14_default: qup-uart14-default-state {
+				/* TX, RX */
+				pins = "gpio18", "gpio19";
+				function = "qup2_se5";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+		};
+
+		gem_noc: interconnect@24100000 {
+			compatible = "qcom,eliza-gem-noc";
+			reg = <0x0 0x24100000 0x0 0x163080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		nsp_noc: interconnect@320c0000 {
+			compatible = "qcom,eliza-nsp-noc";
+			reg = <0x0 0x320c0000 0x0 0xe080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts
new file mode 100644
index 0000000000000..0efd9e27c82f6
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "glymur.dtsi"
+#include "glymur-crd.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Glymur CRD";
+	compatible = "qcom,glymur-crd", "qcom,glymur";
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
new file mode 100644
index 0000000000000..abc6cc8bb0a84
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
@@ -0,0 +1,591 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "pmcx0102.dtsi"        /* SPMI0: SID-2/3 SPMI1: SID-2/3 */
+#include "pmh0101.dtsi"         /* SPMI0: SID-1                  */
+#include "pmh0110-glymur.dtsi"  /* SPMI0: SID-5/7 SPMI1: SID-5   */
+#include "pmh0104-glymur.dtsi"  /* SPMI0: SID-8/9 SPMI1: SID-11  */
+#include "pmk8850.dtsi"         /* SPMI0: SID-0                  */
+#include "smb2370.dtsi"         /* SPMI2: SID-9/10/11            */
+
+/ {
+	model = "Qualcomm Technologies, Inc. Glymur CRD";
+	compatible = "qcom,glymur-crd", "qcom,glymur";
+
+	aliases {
+		serial0 = &uart21;
+		serial1 = &uart14;
+		i2c0 = &i2c0;
+		i2c1 = &i2c4;
+		i2c2 = &i2c5;
+		spi0 = &spi18;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&key_vol_up_default>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&pmh0101_gpios 6 GPIO_ACTIVE_LOW>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvmesec: regulator-nvmesec {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_SEC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0110_f_e1_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_sec_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wlan: regulator-wlan {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WLAN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 94 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wlan_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wwan: regulator-wwan {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WWAN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 246 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wwan_reg_en>;
+		pinctrl-names = "default";
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_bob1_e0: bob1 {
+			regulator-name = "vreg_bob1_e0";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <4224000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_bob2_e0: bob2 {
+			regulator-name = "vreg_bob2_e0";
+			regulator-min-microvolt = <2540000>;
+			regulator-max-microvolt = <3600000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l1b_e0_1p8: ldo1 {
+			regulator-name = "vreg_l1b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_e0_2p9: ldo2 {
+			regulator-name = "vreg_l2b_e0_2p9";
+			regulator-min-microvolt = <2904000>;
+			regulator-max-microvolt = <2904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_e0_2p79: ldo7 {
+			regulator-name = "vreg_l7b_e0_2p79";
+			regulator-min-microvolt = <2790000>;
+			regulator-max-microvolt = <2792000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_e0_1p50: ldo8 {
+			regulator-name = "vreg_l8b_e0_1p50";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <1504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_e0_2p7: ldo9 {
+			regulator-name = "vreg_l9b_e0_2p7";
+			regulator-min-microvolt = <2704000>;
+			regulator-max-microvolt = <2704000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_e0_1p8: ldo10 {
+			regulator-name = "vreg_l10b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_e0_1p2: ldo11 {
+			regulator-name = "vreg_l11b_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_e0_1p14: ldo12 {
+			regulator-name = "vreg_l12b_e0_1p14";
+			regulator-min-microvolt = <1144000>;
+			regulator-max-microvolt = <1144000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_e0_1p8: ldo15 {
+			regulator-name = "vreg_l15b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_e0_2p4: ldo17 {
+			regulator-name = "vreg_l17b_e0_2p4";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <2700000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_e0_1p2: ldo18 {
+			regulator-name = "vreg_l18b_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmcx0102-rpmh-regulators";
+		qcom,pmic-id = "C_E1";
+
+		vreg_l1c_e1_0p82: ldo1 {
+			regulator-name = "vreg_l1c_e1_0p82";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_e1_1p14: ldo2 {
+			regulator-name = "vreg_l2c_e1_1p14";
+			regulator-min-microvolt = <1144000>;
+			regulator-max-microvolt = <1144000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_e1_0p89: ldo3 {
+			regulator-name = "vreg_l3c_e1_0p89";
+			regulator-min-microvolt = <890000>;
+			regulator-max-microvolt = <980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_e1_0p72: ldo4 {
+			regulator-name = "vreg_l4c_e1_0p72";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <720000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s7f_e0_1p32: smps7 {
+			regulator-name = "vreg_s7f_e0_1p32";
+			regulator-min-microvolt = <1320000>;
+			regulator-max-microvolt = <1352000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_e0_0p95: smps8 {
+			regulator-name = "vreg_s8f_e0_0p95";
+			regulator-min-microvolt = <952000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s9f_e0_1p9: smps9 {
+			regulator-name = "vreg_s9f_e0_1p9";
+			regulator-min-microvolt = <1900000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e0_0p82: ldo2 {
+			regulator-name = "vreg_l2f_e0_0p82";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_e0_0p72: ldo3 {
+			regulator-name = "vreg_l3f_e0_0p72";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <720000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e0_0p3: ldo4 {
+			regulator-name = "vreg_l4f_e0_0p3";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E1";
+
+		vreg_s7f_e1_0p3: smps7 {
+			regulator-name = "vreg_s7f_e1_0p3";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_e1_0p82: ldo1 {
+			regulator-name = "vreg_l1f_e1_0p82";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e1_0p83: ldo2 {
+			regulator-name = "vreg_l2f_e1_0p83";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e1_1p08: ldo4 {
+			regulator-name = "vreg_l4f_e1_1p08";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1320000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h_e0_0p89: ldo1 {
+			regulator-name = "vreg_l1h_e0_0p89";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h_e0_0p72: ldo2 {
+			regulator-name = "vreg_l2h_e0_0p72";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3h_e0_0p32: ldo3 {
+			regulator-name = "vreg_l3h_e0_0p32";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4h_e0_1p2: ldo4 {
+			regulator-name = "vreg_l4h_e0_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1320000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&pcie3b {
+	vddpe-3v3-supply = <&vreg_nvmesec>;
+
+	pinctrl-0 = <&pcie3b_default>;
+	pinctrl-names = "default";
+};
+
+&pcie3b_phy {
+	vdda-phy-supply = <&vreg_l3c_e1_0p89>;
+	vdda-pll-supply = <&vreg_l2c_e1_1p14>;
+};
+
+&pcie3b_port0 {
+	reset-gpios = <&tlmm 155 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 157 GPIO_ACTIVE_LOW>;
+};
+
+&pcie4 {
+	vddpe-3v3-supply = <&vreg_wlan>;
+
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l1c_e1_0p82>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p08>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
+&pcie5 {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie5_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie5_phy {
+	vdda-phy-supply = <&vreg_l2f_e0_0p82>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+&pcie6 {
+	vddpe-3v3-supply = <&vreg_wwan>;
+
+	pinctrl-0 = <&pcie6_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6_phy {
+	vdda-phy-supply = <&vreg_l1c_e1_0p82>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p08>;
+
+	status = "okay";
+};
+
+&pcie6_port0 {
+	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+};
+
+&pmh0101_gpios {
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0110_f_e1_gpios {
+	nvme_sec_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0101_gpios {
+	key_vol_up_default: key-vol-up-default-state {
+		pins = "gpio6";
+		function = "normal";
+		output-disable;
+		bias-pull-up;
+	};
+};
+
+&pmk8850_rtc {
+	qcom,no-alarm;
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
+			       <10 2>, /* OOB UART */
+			       <44 4>; /* Security SPI (TPM) */
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie5_default: pcie5-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie5_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6_default: pcie6-default-state {
+		clkreq-n-pins {
+			pins = "gpio150";
+			function = "pcie6_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio149";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio151";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie3b_default: pcie3b-default-state {
+		clkreq-n-pins {
+			pins = "gpio156";
+			function = "pcie3b_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio155";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio157";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	wlan_reg_en: wlan-reg-en-state {
+		pins = "gpio94";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wwan_reg_en: wwan-reg-en-state {
+		pins = "gpio246";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi
new file mode 100644
index 0000000000000..bde287f645ee9
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur.dtsi
@@ -0,0 +1,5913 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,glymur-dispcc.h>
+#include <dt-bindings/clock/qcom,glymur-gcc.h>
+#include <dt-bindings/clock/qcom,glymur-tcsr.h>
+#include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interconnect/qcom,glymur-rpmh.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/mailbox/qcom-ipcc.h>
+#include <dt-bindings/phy/phy-qcom-qmp.h>
+#include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/soc/qcom,rpmh-rsc.h>
+#include <dt-bindings/spmi/spmi.h>
+
+#include "glymur-ipcc.h"
+
+/ {
+	interrupt-parent = <&intc>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd0>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd1>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd2>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd3>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x400>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd4>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-2";
+			reg = <0x0 0x500>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd5>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+		};
+
+		cpu6: cpu@10000 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10000>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd6>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+
+			l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu7: cpu@10100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10100>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd7>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu8: cpu@10200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10200>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd8>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu9: cpu@10300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10300>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd9>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu10: cpu@10400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10400>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd10>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu11: cpu@10500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x10500>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd11>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_1>;
+		};
+
+		cpu12: cpu@20000 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20000>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd12>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+
+			l2_2: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu13: cpu@20100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20100>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd13>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu14: cpu@20200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20200>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd14>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu15: cpu@20300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20300>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd15>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu16: cpu@20400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20400>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd16>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu17: cpu@20500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-2-1";
+			reg = <0x0 0x20500>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd17>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_2>;
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+
+				core5 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+
+				core2 {
+					cpu = <&cpu8>;
+				};
+
+				core3 {
+					cpu = <&cpu9>;
+				};
+
+				core4 {
+					cpu = <&cpu10>;
+				};
+
+				core5 {
+					cpu = <&cpu11>;
+				};
+			};
+
+			cpu_map_cluster2: cluster2 {
+				core0 {
+					cpu = <&cpu12>;
+				};
+
+				core1 {
+					cpu = <&cpu13>;
+				};
+
+				core2 {
+					cpu = <&cpu14>;
+				};
+
+				core3 {
+					cpu = <&cpu15>;
+				};
+
+				core4 {
+					cpu = <&cpu16>;
+				};
+
+				core5 {
+					cpu = <&cpu17>;
+				};
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cpu_c4: cpu-sleep-0 {
+				compatible = "arm,idle-state";
+				idle-state-name = "ret";
+				arm,psci-suspend-param = <0x00000004>;
+				entry-latency-us = <180>;
+				exit-latency-us = <320>;
+				min-residency-us = <1000>;
+			};
+		};
+
+		domain-idle-states {
+			cluster_cl5: cluster-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x01000054>;
+				entry-latency-us = <2000>;
+				exit-latency-us = <2000>;
+				min-residency-us = <9000>;
+			};
+
+			domain_ss3: domain-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x0200c354>;
+				entry-latency-us = <2800>;
+				exit-latency-us = <4400>;
+				min-residency-us = <10150>;
+			};
+		};
+	};
+
+	firmware {
+		scm: scm {
+			compatible = "qcom,scm-glymur", "qcom,scm";
+			qcom,dload-mode = <&tcsr 0x4000>;
+			interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+		};
+
+		scmi {
+			compatible = "arm,scmi";
+			mboxes = <&pdp0_mbox 0>, <&pdp0_mbox 1>;
+			mbox-names = "tx", "rx";
+			shmem = <&cpu_scp_lpri1>, <&cpu_scp_lpri0>;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			scmi_perf: protocol@13 {
+				reg = <0x13>;
+				#power-domain-cells = <1>;
+			};
+		};
+	};
+
+	clk_virt: interconnect-0 {
+		compatible = "qcom,glymur-clk-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	mc_virt: interconnect-1 {
+		compatible = "qcom,glymur-mc-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpu_pd0: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd1: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd2: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd3: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd4: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd5: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd6: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd7: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd8: power-domain-cpu8 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd9: power-domain-cpu9 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd10: power-domain-cpu10 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd11: power-domain-cpu11 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd12: power-domain-cpu12 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd13: power-domain-cpu13 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd14: power-domain-cpu14 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd15: power-domain-cpu15 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd16: power-domain-cpu16 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cpu_pd17: power-domain-cpu17 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&cpu_c4>;
+		};
+
+		cluster0_pd: power-domain-cpu-cluster0 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_cl5>;
+		};
+
+		cluster1_pd: power-domain-cpu-cluster1 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_cl5>;
+		};
+
+		cluster2_pd: power-domain-cpu-cluster2 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_cl5>;
+		};
+
+		system_pd: power-domain-system {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&domain_ss3>;
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		pdp_mem: pdp@81400000 {
+			reg = <0x0 0x81400000 0x0 0x100000>;
+			no-map;
+		};
+
+		aop_cmd_db_mem: aop-cmd-db@81c60000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x81c60000 0x0 0x20000>;
+			no-map;
+		};
+
+		pdp_ns_shared_mem: pdp-ns-shared@81e00000 {
+			reg = <0x0 0x81e00000 0x0 0x200000>;
+			no-map;
+		};
+
+		oobdaretag_mem: oobdaretag@86e10000 {
+			reg = <0x0 0x86e10000 0x0 0x360000>;
+			no-map;
+		};
+
+		oob_secure_mem: oob-secure@87170000 {
+			reg = <0x0 0x87170000 0x0 0xbc0000>;
+			no-map;
+		};
+
+		oobdtbqc_mem: oobdtbqc@87d30000 {
+			reg = <0x0 0x87d30000 0x0 0x20000>;
+			no-map;
+		};
+
+		oobdtboem_mem: oobdtboem@87d50000 {
+			reg = <0x0 0x87d50000 0x0 0x20000>;
+			no-map;
+		};
+
+		oob_nonsecure_mem: oob-nonsecure@87e00000 {
+			reg = <0x0 0x87e00000 0x0 0xc00000>;
+			no-map;
+		};
+
+		spss_region_mem: spss@88a00000 {
+			reg = <0x0 0x88a00000 0x0 0x400000>;
+			no-map;
+		};
+
+		soccpdtb_mem: soccpdtb@892e0000 {
+			reg = <0x0 0x892e0000 0x0 0x20000>;
+			no-map;
+		};
+
+		soccp_mem: soccp@89300000 {
+			reg = <0x0 0x89300000 0x0 0x400000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@89700000 {
+			reg = <0x0 0x89700000 0x0 0x700000>;
+			no-map;
+		};
+
+		adspslpi_mem: adspslpi@89e00000 {
+			reg = <0x0 0x89e00000 0x0 0x3a00000>;
+			no-map;
+		};
+
+		q6_adsp_dtb_mem: q6-adsp-dtb@8d800000 {
+			reg = <0x0 0x8d800000 0x0 0x80000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@8d900000 {
+			reg = <0x0 0x8d900000 0x0 0x4000000>;
+			no-map;
+		};
+
+		q6_cdsp_dtb_mem: q6-cdsp-dtb@91900000 {
+			reg = <0x0 0x91900000 0x0 0x80000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode@919fe000 {
+			reg = <0x0 0x919fe000 0x0 0x2000>;
+			no-map;
+		};
+
+		camera_mem: camera@91a00000 {
+			reg = <0x0 0x91a00000 0x0 0x800000>;
+			no-map;
+		};
+
+		av1_encoder_mem: av1-encoder@92200000 {
+			reg = <0x0 0x92200000 0x0 0x700000>;
+			no-map;
+		};
+
+		video_mem: video@92900000 {
+			reg = <0x0 0x92900000 0x0 0xc00000>;
+			no-map;
+		};
+
+		smem_mem: smem@ffe00000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0xffe00000 0x0 0x200000>;
+			hwlocks = <&tcsr_mutex 3>;
+			no-map;
+		};
+	};
+
+	smp2p-adsp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_LPASS
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_LPASS IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <443>, <429>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <2>;
+
+		smp2p_adsp_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_adsp_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	smp2p-cdsp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_CDSP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_CDSP IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <94>, <432>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <5>;
+
+		smp2p_cdsp_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_cdsp_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	smp2p-soccp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_SOCCP
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <617>, <616>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <19>;
+
+		soccp_smp2p_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		soccp_smp2p_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x0 0x0 0x0 0x0 0x100 0x0>;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x100 0x0>;
+
+		gcc: clock-controller@100000 {
+			compatible = "qcom,glymur-gcc";
+			reg = <0x0 0x00100000 0x0 0x1f9000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,	/* Board XO source */
+				 <&rpmhcc RPMH_CXO_CLK_A>,	/* Board XO_A source */
+				 <&sleep_clk>,			/* Sleep */
+				 <0>,				/* USB 0 Phy DP0 GMUX */
+				 <0>,				/* USB 0 Phy DP1 GMUX */
+				 <0>,				/* USB 0 Phy PCIE PIPEGMUX */
+				 <0>,				/* USB 0 Phy PIPEGMUX */
+				 <0>,				/* USB 0 Phy SYS PCIE PIPEGMUX */
+				 <0>,				/* USB 1 Phy DP0 GMUX 2 */
+				 <0>,				/* USB 1 Phy DP1 GMUX 2 */
+				 <0>,				/* USB 1 Phy PCIE PIPEGMUX */
+				 <0>,				/* USB 1 Phy PIPEGMUX */
+				 <0>,				/* USB 1 Phy SYS PCIE PIPEGMUX */
+				 <0>,				/* USB 2 Phy DP0 GMUX 2 */
+				 <0>,				/* USB 2 Phy DP1 GMUX 2 */
+				 <0>,				/* USB 2 Phy PCIE PIPEGMUX */
+				 <0>,				/* USB 2 Phy PIPEGMUX */
+				 <0>,				/* USB 2 Phy SYS PCIE PIPEGMUX */
+				 <0>,				/* PCIe 3a */
+				 <&pcie3b_phy>,			/* PCIe 3b */
+				 <&pcie4_phy>,			/* PCIe 4 */
+				 <&pcie5_phy>,			/* PCIe 5 */
+				 <&pcie6_phy>,			/* PCIe 6 */
+				 <0>,				/* QUSB4 0 PHY RX 0 */
+				 <0>,				/* QUSB4 0 PHY RX 1 */
+				 <0>,				/* QUSB4 1 PHY RX 0 */
+				 <0>,				/* QUSB4 1 PHY RX 1 */
+				 <0>,				/* QUSB4 2 PHY RX 0 */
+				 <0>,				/* QUSB4 2 PHY RX 1 */
+				 <0>,				/* UFS PHY RX Symbol 0 */
+				 <0>,				/* UFS PHY RX Symbol 1 */
+				 <0>,				/* UFS PHY TX Symbol 0 */
+				 <0>,				/* USB3 PHY 0 */
+				 <0>,				/* USB3 PHY 1 */
+				 <0>,				/* USB3 PHY 2 */
+				 <0>,				/* USB3 UNI PHY pipe 0 */
+				 <0>,				/* USB3 UNI PHY pipe 1 */
+				 <0>,				/* USB4 PHY 0 pcie pipe */
+				 <0>,				/* USB4 PHY 0 Max pipe */
+				 <0>,				/* USB4 PHY 1 pcie pipe */
+				 <0>,				/* USB4 PHY 1 Max pipe */
+				 <0>,				/* USB4 PHY 2 pcie */
+				 <0>;				/* USB4 PHY 2 Max */
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		gpi_dma2: dma-controller@800000 {
+			compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x00800000 0x0 0x60000>;
+			interrupts = <GIC_SPI 588 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 589 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 590 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 591 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 592 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 593 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 594 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 595 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 596 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 597 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 598 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 599 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 129 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 130 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 131 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 132 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <16>;
+			dma-channel-mask = <0x3f>;
+			#dma-cells = <3>;
+			iommus = <&apps_smmu 0xd76 0x0>;
+		};
+
+		qupv3_2: geniqup@8c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x008c0000 0x0 0x3000>;
+			clocks = <&gcc GCC_QUPV3_WRAP_2_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_2_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+			iommus = <&apps_smmu 0xd63 0x0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c16: i2c@880000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c16_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi16: spi@880000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi16_data_clk>, <&qup_spi16_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c17: i2c@884000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c17_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi17: spi@884000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi17_data_clk>, <&qup_spi17_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c18: i2c@888000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c18_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi18: spi@888000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi18_data_clk>, <&qup_spi18_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c19: i2c@88c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c19_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi19: spi@88c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi19_data_clk>, <&qup_spi19_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart19: serial@88c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+				pinctrl-0 = <&qup_uart19_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c20: i2c@890000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c20_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi20: spi@890000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 4 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 4 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi20_data_clk>, <&qup_spi20_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c21: i2c@894000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 5 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 5 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c21_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi21: spi@894000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 5 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 5 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi21_data_clk>, <&qup_spi21_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart21: serial@894000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+				pinctrl-0 = <&qup_uart21_default>;
+				pinctrl-names = "default";
+			};
+
+			i2c22: i2c@898000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 6 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 6 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c22_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi22: spi@898000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 6 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 6 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi22_data_clk>, <&qup_spi22_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart22: serial@898000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+				pinctrl-0 = <&qup_uart22_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c23: i2c@89c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0089c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 7 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 7 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c23_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi23: spi@89c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0089c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma2 0 7 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 7 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi23_data_clk>, <&qup_spi23_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+		};
+
+		gpi_dma1: dma-controller@a00000 {
+			compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x00a00000 0x0 0x60000>;
+			interrupts = <GIC_SPI 279 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 280 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 281 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 282 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 283 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 284 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 293 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 294 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 295 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 296 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 297 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 298 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 124 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <16>;
+			dma-channel-mask = <0x3f>;
+			#dma-cells = <3>;
+			iommus = <&apps_smmu 0xcb6 0x0>;
+		};
+
+		qupv3_1: geniqup@ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x00ac0000 0x0 0x3000>;
+			clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>,
+				<&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+			iommus = <&apps_smmu 0xca3 0x0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c8: i2c@a80000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c8_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi8: spi@a80000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi8_data_clk>, <&qup_spi8_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c9: i2c@a84000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c9_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi9: spi@a84000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi9_data_clk>, <&qup_spi9_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c10: i2c@a88000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c10_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi10: spi@a88000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi10_data_clk>, <&qup_spi10_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c11: i2c@a8c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c11_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi11: spi@a8c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi11_data_clk>, <&qup_spi11_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c12: i2c@a90000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c12_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi12: spi@a90000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 4 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi12_data_clk>, <&qup_spi12_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c13: i2c@a94000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 5 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c13_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi13: spi@a94000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 5 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi13_data_clk>, <&qup_spi13_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c14: i2c@a98000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 6 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c14_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi14: spi@a98000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 6 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi14_data_clk>, <&qup_spi14_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart14: serial@a98000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+				pinctrl-0 = <&qup_uart14_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c15: i2c@a9c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a9c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 579 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 7 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 7 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c15_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi15: spi@a9c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a9c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 579 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma1 0 7 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 7 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi15_data_clk>, <&qup_spi15_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+		};
+
+		gpi_dma0: dma-controller@b00000 {
+			compatible = "qcom,glymur-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x00b00000 0x0 0x60000>;
+			interrupts = <GIC_ESPI 76 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 77 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 78 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 79 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 80 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 81 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 83 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 86 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 87 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 88 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 89 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 90 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 91 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <16>;
+			dma-channel-mask = <0x3f>;
+			#dma-cells = <3>;
+			iommus = <&apps_smmu 0xd36 0x0>;
+		};
+
+		qupv3_0: geniqup@bc0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x00bc0000 0x0 0x3000>;
+			clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+			iommus = <&apps_smmu 0xd23 0x0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c0: i2c@b80000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b80000 0x0 0x4000>;
+				interrupts = <GIC_ESPI 92 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c0_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi0: spi@b80000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1052 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi0_data_clk>, <&qup_spi0_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c1: i2c@b84000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c1_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi1: spi@b84000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi1_data_clk>, <&qup_spi1_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c2: i2c@b88000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c2_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi2: spi@b88000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi2_data_clk>, <&qup_spi2_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart2: serial@b88000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00b88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+				pinctrl-0 = <&qup_uart2_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c3: i2c@b8c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b8c000 0x0 0x4000>;
+				interrupts = <GIC_ESPI 95 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c3_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi3: spi@b8c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b8c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1055 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi3_data_clk>, <&qup_spi3_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c4: i2c@b90000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b90000 0x0 0x4000>;
+				interrupts = <GIC_ESPI 96 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c4_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi4: spi@b90000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1056 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 4 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 4 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi4_data_clk>, <&qup_spi4_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c5: i2c@b94000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b94000 0x0 0x4000>;
+				interrupts = <GIC_ESPI 97 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 5 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 5 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c5_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi5: spi@b94000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1057 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 5 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 5 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi5_data_clk>, <&qup_spi5_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c6: i2c@b98000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 6 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 6 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c6_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi6: spi@b98000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 6 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 6 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi6_data_clk>, <&qup_spi6_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c7: i2c@b9c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00b9c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 7 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 7 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_i2c7_data_clk>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi7: spi@b9c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00b9c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>;
+				clock-names = "se";
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+						<&aggre3_noc MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+				dmas = <&gpi_dma0 0 7 QCOM_GPI_SPI>,
+				       <&gpi_dma0 1 7 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+				pinctrl-0 = <&qup_spi7_data_clk>, <&qup_spi7_cs>;
+				pinctrl-names = "default";
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+		};
+
+		cnoc_main: interconnect@1500000 {
+			compatible = "qcom,glymur-cnoc-main";
+			reg = <0x0 0x01500000 0x0 0x17080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		config_noc: interconnect@1600000 {
+			compatible = "qcom,glymur-cnoc-cfg";
+			reg = <0x0 0x01600000 0x0 0x6600>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		system_noc: interconnect@1680000 {
+			compatible = "qcom,glymur-system-noc";
+			reg = <0x0 0x01680000 0x0 0x1c080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie_west_anoc: interconnect@16c0000 {
+			compatible = "qcom,glymur-pcie-west-anoc";
+			reg = <0x0 0x016c0000 0x0 0xf580>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE_3A_WEST_SF_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_3B_WEST_SF_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_4_WEST_SF_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_6_WEST_SF_AXI_CLK>;
+		};
+
+		pcie_east_anoc: interconnect@16d0000 {
+			compatible = "qcom,glymur-pcie-east-anoc";
+			reg = <0x0 0x016d0000 0x0 0xf300>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE_5_EAST_SF_AXI_CLK>;
+		};
+
+		aggre1_noc: interconnect@16e0000 {
+			compatible = "qcom,glymur-aggre1-noc";
+			reg = <0x0 0x016e0000 0x0 0x14400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		aggre2_noc: interconnect@1720000 {
+			compatible = "qcom,glymur-aggre2-noc";
+			reg = <0x0 0x01720000 0x0 0x14400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_USB3_TERT_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_2_AXI_CLK>,
+				 <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>;
+		};
+
+		aggre3_noc: interconnect@1700000 {
+			compatible = "qcom,glymur-aggre3-noc";
+			reg = <0x0 0x01700000 0x0 0x1d400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		aggre4_noc: interconnect@1740000 {
+			compatible = "qcom,glymur-aggre4-noc";
+			reg = <0x0 0x01740000 0x0 0x14400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_0_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_1_AXI_CLK>;
+		};
+
+		mmss_noc: interconnect@1780000 {
+			compatible = "qcom,glymur-mmss-noc";
+			reg = <0x0 0x01780000 0x0 0x5b800>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie_east_slv_noc: interconnect@1900000 {
+			compatible = "qcom,glymur-pcie-east-slv-noc";
+			reg = <0x0 0x01900000 0x0 0xe080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie_west_slv_noc: interconnect@1920000 {
+			compatible = "qcom,glymur-pcie-west-slv-noc";
+			reg = <0x0 0x01920000 0x0 0xf180>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie4: pci@1bf0000 {
+			device_type = "pci";
+			compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100";
+			reg = <0x0 0x01bf0000 0x0 0x3000>,
+			      <0x0 0x78000000 0x0 0xf20>,
+			      <0x0 0x78000f40 0x0 0xa8>,
+			      <0x0 0x78001000 0x0 0x4000>,
+			      <0x0 0x78005000 0x0 0x100000>,
+			      <0x0 0x01bf3000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x78105000 0x0 0x100000>,
+				 <0x02000000 0x0 0x78205000 0x0 0x78205000 0x0 0x1dfb000>,
+				 <0x03000000 0x7 0x80000000 0x7 0x80000000 0x0 0x20000000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <4>;
+			num-lanes = <2>;
+
+			operating-points-v2 = <&pcie4_opp_table>;
+
+			msi-map = <0x0 &gic_its 0xc0000 0x10000>;
+			iommu-map = <0x0 &pcie_smmu 0x40000 0x10000>;
+
+			interrupts = <GIC_SPI 505 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 510 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 511 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 512 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 944 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 0 0 513 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 514 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 515 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 516 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_4_AUX_CLK>,
+				 <&gcc GCC_PCIE_4_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_4_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_4_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_4_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_4_WEST_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr";
+
+			assigned-clocks = <&gcc GCC_PCIE_4_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_west_anoc MASTER_PCIE_4 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+					 &pcie_west_slv_noc SLAVE_PCIE_4 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			resets = <&gcc GCC_PCIE_4_BCR>,
+				 <&gcc GCC_PCIE_4_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc GCC_PCIE_4_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55>;
+
+			status = "disabled";
+
+			pcie4_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 2 x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 4 x1 */
+				opp-16000000-4 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x2 */
+				opp-32000000-4 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <4>;
+				};
+
+			};
+
+			pcie4_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie4_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie4_phy: phy@1bf6000 {
+			compatible = "qcom,glymur-qmp-gen4x2-pcie-phy";
+			reg = <0x0 0x01bf6000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_PHY_4_AUX_CLK>,
+				 <&gcc GCC_PCIE_4_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_2_CLKREF_EN>,
+				 <&gcc GCC_PCIE_4_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_4_PIPE_CLK>,
+				 <&gcc GCC_PCIE_4_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			resets = <&gcc GCC_PCIE_4_PHY_BCR>,
+				 <&gcc GCC_PCIE_4_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			assigned-clocks = <&gcc GCC_PCIE_4_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			power-domains = <&gcc GCC_PCIE_4_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie4_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie5: pci@1b40000 {
+			device_type = "pci";
+			compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100";
+			reg = <0x0 0x01b40000 0x0 0x3000>,
+			      <0x0 0x7a000000 0x0 0xf20>,
+			      <0x0 0x7a000f40 0x0 0xa8>,
+			      <0x0 0x7a001000 0x0 0x4000>,
+			      <0x0 0x7a100000 0x0 0x100000>,
+			      <0x0 0x01b43000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x7a200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x7a300000 0x0 0x7a300000 0x0 0x3d00000>,
+				 <0x03000000 0x7 0xa0000000 0x7 0xa0000000 0x0 0x40000000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <5>;
+			num-lanes = <4>;
+
+			operating-points-v2 = <&pcie5_opp_table>;
+
+			msi-map = <0x0 &gic_its 0xd0000 0x10000>;
+			iommu-map = <0x0 &pcie_smmu 0x50000 0x10000>;
+
+			interrupts = <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 519 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 520 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 521 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 522 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 523 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 524 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 525 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 945 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 0 0 526 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 428 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 429 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 435 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_5_AUX_CLK>,
+				 <&gcc GCC_PCIE_5_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_5_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_5_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_5_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_5_EAST_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr";
+
+			assigned-clocks = <&gcc GCC_PCIE_5_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_east_anoc MASTER_PCIE_5 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+					 &pcie_east_slv_noc SLAVE_PCIE_5 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			resets = <&gcc GCC_PCIE_5_BCR>,
+				 <&gcc GCC_PCIE_5_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc GCC_PCIE_5_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
+			eq-presets-32gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
+
+			status = "disabled";
+
+			pcie5_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x4 */
+				opp-10000000-1 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 2 x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x4 */
+				opp-20000000-2 {
+					opp-hz = /bits/ 64 <20000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <2000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x4 */
+				opp-32000000-3 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 4 x1 */
+				opp-16000000-4 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x2 */
+				opp-32000000-4 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x4 */
+				opp-64000000-4 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 5 x1 */
+				opp-32000000-5 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x2 */
+				opp-64000000-5 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x4 */
+				opp-128000000-5 {
+					opp-hz = /bits/ 64 <128000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <15753000 1>;
+					opp-level = <5>;
+				};
+			};
+
+			pcie5_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie5_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie5_phy: phy@1b50000 {
+			compatible = "qcom,glymur-qmp-gen5x4-pcie-phy";
+			reg = <0x0 0x01b50000 0x0 0x10000>;
+
+			clocks = <&gcc GCC_PCIE_PHY_5_AUX_CLK>,
+				 <&gcc GCC_PCIE_5_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_1_CLKREF_EN>,
+				 <&gcc GCC_PCIE_5_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_5_PIPE_CLK>,
+				 <&gcc GCC_PCIE_5_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			resets = <&gcc GCC_PCIE_5_PHY_BCR>,
+				 <&gcc GCC_PCIE_5_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			assigned-clocks = <&gcc GCC_PCIE_5_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			power-domains = <&gcc GCC_PCIE_5_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie5_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie6: pci@1c00000 {
+			device_type = "pci";
+			compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x7e000000 0x0 0xf20>,
+			      <0x0 0x7e000f40 0x0 0xa8>,
+			      <0x0 0x7e001000 0x0 0x4000>,
+			      <0x0 0x7e100000 0x0 0x100000>,
+			      <0x0 0x01c03000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x7e200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x7e300000 0x0 0x7e300000 0x0 0x1d00000>,
+				 <0x03000000 0x7 0xe0000000 0x7 0xe0000000 0x0 0x20000000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <6>;
+			num-lanes = <2>;
+
+			operating-points-v2 = <&pcie6_opp_table>;
+
+			msi-map = <0x0 &gic_its 0xe0000 0x10000>;
+			iommu-map = <0x0 &pcie_smmu 0x60000 0x10000>;
+
+			interrupts = <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 946 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 0 0 472 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 473 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 474 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 475 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_6_AUX_CLK>,
+				 <&gcc GCC_PCIE_6_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_6_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_6_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_6_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_6_WEST_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr";
+
+			assigned-clocks = <&gcc GCC_PCIE_6_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_west_anoc MASTER_PCIE_6 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+					 &pcie_west_slv_noc SLAVE_PCIE_6 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			resets = <&gcc GCC_PCIE_6_BCR>,
+				 <&gcc GCC_PCIE_6_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc GCC_PCIE_6_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55>;
+
+			status = "disabled";
+
+			pcie6_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 2 x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 4 x1 */
+				opp-16000000-4 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x2 */
+				opp-32000000-4 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <4>;
+				};
+
+			};
+
+			pcie6_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie6_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie6_phy: phy@1c06000 {
+			compatible = "qcom,glymur-qmp-gen4x2-pcie-phy";
+			reg = <0x0 0x01c06000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_PHY_6_AUX_CLK>,
+				 <&gcc GCC_PCIE_6_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_4_CLKREF_EN>,
+				 <&gcc GCC_PCIE_6_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_6_PIPE_CLK>,
+				 <&gcc GCC_PCIE_6_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			resets = <&gcc GCC_PCIE_6_PHY_BCR>,
+				 <&gcc GCC_PCIE_6_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			assigned-clocks = <&gcc GCC_PCIE_6_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			power-domains = <&gcc GCC_PCIE_6_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie6_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie3b: pci@1b80000 {
+			device_type = "pci";
+			compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100";
+			reg = <0x0 0x01b80000 0x0 0x3000>,
+			      <0x0 0x74000000 0x0 0xf20>,
+			      <0x0 0x74000f40 0x0 0xa8>,
+			      <0x0 0x74001000 0x0 0x4000>,
+			      <0x0 0x74100000 0x0 0x100000>,
+			      <0x0 0x01b83000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x74200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x74300000 0x0 0x74300000 0x0 0x3d00000>,
+				 <0x03000000 0x7 0x40000000 0x7 0x40000000 0x0 0x40000000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <7>;
+			num-lanes = <4>;
+
+			operating-points-v2 = <&pcie3b_opp_table>;
+
+			msi-map = <0x0 &gic_its 0xf0000 0x10000>;
+			iommu-map = <0x0 &pcie_smmu 0x70000 0x10000>;
+
+			interrupts = <GIC_SPI 950 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 951 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 943 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 0 0 831 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 832 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 833 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 834 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_3B_AUX_CLK>,
+				 <&gcc GCC_PCIE_3B_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_3B_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_3B_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_3B_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_3B_WEST_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr";
+
+			assigned-clocks = <&gcc GCC_PCIE_3B_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_west_anoc MASTER_PCIE_3B QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+					 &pcie_west_slv_noc SLAVE_PCIE_3B QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			resets = <&gcc GCC_PCIE_3B_BCR>,
+				 <&gcc GCC_PCIE_3B_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc GCC_PCIE_3B_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
+			eq-presets-32gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
+
+			status = "disabled";
+
+			pcie3b_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x4 */
+				opp-10000000-1 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 2 x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x4 */
+				opp-20000000-2 {
+					opp-hz = /bits/ 64 <20000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <2000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x4 */
+				opp-32000000-3 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 4 x1 */
+				opp-16000000-4 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x2 */
+				opp-32000000-4 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x4 */
+				opp-64000000-4 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 5 x1 */
+				opp-32000000-5 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x2 */
+				opp-64000000-5 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x4 */
+				opp-128000000-5 {
+					opp-hz = /bits/ 64 <128000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <15753000 1>;
+					opp-level = <5>;
+				};
+			};
+
+			pcie3b_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie3b_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie3b_phy: phy@f10000 {
+			compatible = "qcom,glymur-qmp-gen5x4-pcie-phy";
+			reg = <0x0 0x00f10000 0x0 0x10000>;
+
+			clocks = <&gcc GCC_PCIE_PHY_3B_AUX_CLK>,
+				 <&gcc GCC_PCIE_3B_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_3_CLKREF_EN>,
+				 <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_3B_PIPE_CLK>,
+				 <&gcc GCC_PCIE_3B_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			resets = <&gcc GCC_PCIE_3B_PHY_BCR>,
+				 <&gcc GCC_PCIE_3B_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			assigned-clocks = <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			power-domains = <&gcc GCC_PCIE_3B_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie3b_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01f40000 0x0 0x20000>;
+
+			#hwlock-cells = <1>;
+		};
+
+		tcsr: clock-controller@1fd5000 {
+			compatible = "qcom,glymur-tcsr",
+				     "syscon";
+			reg = <0x0 0x1fd5000 0x0 0x21000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		hsc_noc: interconnect@2000000 {
+			compatible = "qcom,glymur-hscnoc";
+			reg = <0x0 0x02000000 0x0 0x93a080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		ipcc: mailbox@3e04000 {
+			compatible = "qcom,glymur-ipcc", "qcom,ipcc";
+			reg = <0x0 0x03e04000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+
+			#mbox-cells = <2>;
+		};
+
+		lpass_lpiaon_noc: interconnect@7400000 {
+			compatible = "qcom,glymur-lpass-lpiaon-noc";
+			reg = <0x0 0x07400000 0x0 0x19080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_lpicx_noc: interconnect@7420000 {
+			compatible = "qcom,glymur-lpass-lpicx-noc";
+			reg = <0x0 0x07420000 0x0 0x44080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_ag_noc: interconnect@7e40000 {
+			compatible = "qcom,glymur-lpass-ag-noc";
+			reg = <0x0 0x07e40000 0x0 0xe080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		dispcc: clock-controller@af00000 {
+			compatible = "qcom,glymur-dispcc";
+			reg = <0x0 0x0af00000 0x0 0x20000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&sleep_clk>,
+				 <0>, /* dp0 */
+				 <0>,
+				 <0>, /* dp1 */
+				 <0>,
+				 <0>, /* dp2 */
+				 <0>,
+				 <0>, /* dp3 */
+				 <0>,
+				 <0>, /* dsi0 */
+				 <0>,
+				 <0>, /* dsi1 */
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			power-domains = <&rpmhpd RPMHPD_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		pdc: interrupt-controller@b220000 {
+			compatible = "qcom,glymur-pdc", "qcom,pdc";
+			reg = <0x0 0x0b220000 0x0 0x10000>;
+			qcom,pdc-ranges = <0 745 51>,
+					  <51 527 47>,
+					  <98 609 32>,
+					  <130 717 12>,
+					  <142 251 5>,
+					  <147 796 16>,
+					  <171 4104 36>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&intc>;
+			interrupt-controller;
+		};
+
+		tsens0: thermal-sensor@c22c000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22c000 0x0 0x1000>,
+			      <0x0 0x0c222000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 771 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 861 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <13>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens1: thermal-sensor@c22d000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22d000 0x0 0x1000>,
+			      <0x0 0x0c223000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 772 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 862 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <9>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens2: thermal-sensor@c22e000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22e000 0x0 0x1000>,
+			      <0x0 0x0c224000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 773 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 863 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <13>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens3: thermal-sensor@c22f000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22f000 0x0 0x1000>,
+			      <0x0 0x0c225000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 774 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 864 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <8>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens4: thermal-sensor@c230000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c230000 0x0 0x1000>,
+			      <0x0 0x0c226000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 791 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 865 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <13>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens5: thermal-sensor@c231000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c231000 0x0 0x1000>,
+			      <0x0 0x0c227000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 619 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 814 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <8>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens6: thermal-sensor@c232000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c232000 0x0 0x1000>,
+			      <0x0 0x0c228000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 620 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 815 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <13>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens7: thermal-sensor@c233000 {
+			compatible = "qcom,glymur-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c233000 0x0 0x1000>,
+			      <0x0 0x0c229000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 621 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 816 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+
+			#qcom,sensors = <15>;
+
+			#thermal-sensor-cells = <1>;
+		};
+
+		aoss_qmp: power-management@c300000 {
+			compatible = "qcom,glymur-aoss-qmp", "qcom,aoss-qmp";
+			reg = <0x0 0x0c300000 0x0 0x400>;
+			interrupt-parent = <&ipcc>;
+			interrupts-extended = <&ipcc IPCC_MPROC_AOP IPCC_MPROC_SIGNAL_GLINK_QMP
+						     IRQ_TYPE_EDGE_RISING>;
+			mboxes = <&ipcc IPCC_MPROC_AOP IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+			#clock-cells = <0>;
+		};
+
+		sram@c30f000 {
+			compatible = "qcom,rpmh-stats";
+			reg = <0x0 0x0c30f000 0x0 0x400>;
+		};
+
+		arbiter@c400000 {
+			compatible = "qcom,glymur-spmi-pmic-arb";
+			reg = <0x0 0x0c400000 0x0 0x3000>,
+			      <0x0 0x0c900000 0x0 0x400000>,
+			      <0x0 0x0c4c0000 0x0 0x400000>,
+			      <0x0 0x0c403000 0x0 0x8000>;
+			reg-names = "core",
+				    "chnls",
+				    "obsrvr",
+				    "chnl_map";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			qcom,channel = <0>;
+			qcom,ee = <0>;
+
+			spmi_bus0: spmi@c426000 {
+				reg = <0x0 0x0c426000 0x0 0x4000>,
+				      <0x0 0x0c8c0000 0x0 0x10000>,
+				      <0x0 0x0c42a000 0x0 0x8000>;
+				reg-names = "cnfg",
+					    "intr",
+					    "chnl_owner";
+				interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "periph_irq";
+				interrupt-controller;
+				#interrupt-cells = <4>;
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+
+			spmi_bus1: spmi@c437000 {
+				reg = <0x0 0x0c437000 0x0 0x4000>,
+				      <0x0 0x0c8d0000 0x0 0x10000>,
+				      <0x0 0x0c43b000 0x0 0x8000>;
+				reg-names = "cnfg",
+					    "intr",
+					    "chnl_owner";
+				interrupts-extended = <&pdc 3 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "periph_irq";
+				interrupt-controller;
+				#interrupt-cells = <4>;
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+
+			spmi_bus2: spmi@c48000 {
+				reg = <0x0 0x0c448000 0x0 0x4000>,
+				      <0x0 0x0c8e0000 0x0 0x10000>,
+				      <0x0 0x0c44c000 0x0 0x8000>;
+				reg-names = "cnfg",
+					    "intr",
+					    "chnl_owner";
+				interrupts-extended = <&pdc 72 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "periph_irq";
+				interrupt-controller;
+				#interrupt-cells = <4>;
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+		};
+
+		tlmm: pinctrl@f100000 {
+			compatible = "qcom,glymur-tlmm";
+			reg = <0x0 0x0f100000 0x0 0xf00000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			gpio-ranges = <&tlmm 0 0 249>;
+			wakeup-parent = <&pdc>;
+
+			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio0", "gpio1";
+				function = "qup0_se0";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio4", "gpio5";
+				function = "qup0_se1";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c2_data_clk: qup-i2c2-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio8", "gpio9";
+				function = "qup0_se2";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c3_data_clk: qup-i2c3-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio12", "gpio13";
+				function = "qup0_se3";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c4_data_clk: qup-i2c4-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio16", "gpio17";
+				function = "qup0_se4";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c5_data_clk: qup-i2c5-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio20", "gpio21";
+				function = "qup0_se5";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c6_data_clk: qup-i2c6-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio6", "gpio7";
+				function = "qup0_se6";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c7_data_clk: qup-i2c7-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio14", "gpio15";
+				function = "qup0_se7";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c8_data_clk: qup-i2c8-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio32", "gpio33";
+				function = "qup1_se0";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c9_data_clk: qup-i2c9-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio36", "gpio37";
+				function = "qup1_se1";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c10_data_clk: qup-i2c10-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio40", "gpio41";
+				function = "qup1_se2";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c11_data_clk: qup-i2c11-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio44", "gpio45";
+				function = "qup1_se3";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c12_data_clk: qup-i2c12-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio48", "gpio49";
+				function = "qup1_se4";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c13_data_clk: qup-i2c13-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio52", "gpio53";
+				function = "qup1_se5";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c14_data_clk: qup-i2c14-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio56", "gpio57";
+				function = "qup1_se6";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c15_data_clk: qup-i2c15-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio54", "gpio55";
+				function = "qup1_se7";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c16_data_clk: qup-i2c16-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio64", "gpio65";
+				function = "qup2_se0";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c17_data_clk: qup-i2c17-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio68", "gpio69";
+				function = "qup2_se1";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c18_data_clk: qup-i2c18-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio72", "gpio73";
+				function = "qup2_se2";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c19_data_clk: qup-i2c19-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio76", "gpio77";
+				function = "qup2_se3";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c20_data_clk: qup-i2c20-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio80", "gpio81";
+				function = "qup2_se4";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c21_data_clk: qup-i2c21-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio84", "gpio85";
+				function = "qup2_se5";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c22_data_clk: qup-i2c22-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio88", "gpio89";
+				function = "qup2_se6";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_i2c23_data_clk: qup-i2c23-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio80", "gpio81";
+				function = "qup2_se7";
+				drive-strength = <2>;
+				bias-pull-up = <2200>;
+			};
+
+			qup_spi0_cs: qup-spi0-cs-state {
+				pins = "gpio3";
+				function = "qup0_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi0_data_clk: qup-spi0-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio0", "gpio1", "gpio2";
+				function = "qup0_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi1_cs: qup-spi1-cs-state {
+				pins = "gpio7";
+				function = "qup0_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi1_data_clk: qup-spi1-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio4", "gpio5", "gpio6";
+				function = "qup0_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi2_cs: qup-spi2-cs-state {
+				pins = "gpio11";
+				function = "qup0_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi2_data_clk: qup-spi2-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio8", "gpio9", "gpio10";
+				function = "qup0_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi3_cs: qup-spi3-cs-state {
+				pins = "gpio15";
+				function = "qup0_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi3_data_clk: qup-spi3-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio12", "gpio13", "gpio14";
+				function = "qup0_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi4_cs: qup-spi4-cs-state {
+				pins = "gpio19";
+				function = "qup0_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi4_data_clk: qup-spi4-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio16", "gpio17", "gpio18";
+				function = "qup0_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi5_cs: qup-spi5-cs-state {
+				pins = "gpio23";
+				function = "qup0_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi5_data_clk: qup-spi5-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio20", "gpio21", "gpio22";
+				function = "qup0_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi6_cs: qup-spi6-cs-state {
+				pins = "gpio5";
+				function = "qup0_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi6_data_clk: qup-spi6-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio6", "gpio7", "gpio4";
+				function = "qup0_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi7_cs: qup-spi7-cs-state {
+				pins = "gpio13";
+				function = "qup0_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi7_data_clk: qup-spi7-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio14", "gpio15", "gpio12";
+				function = "qup0_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi8_cs: qup-spi8-cs-state {
+				pins = "gpio35";
+				function = "qup1_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi8_data_clk: qup-spi8-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio32", "gpio33", "gpio34";
+				function = "qup1_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi9_cs: qup-spi9-cs-state {
+				pins = "gpio39";
+				function = "qup1_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi9_data_clk: qup-spi9-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio36", "gpio37", "gpio38";
+				function = "qup1_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi10_cs: qup-spi10-cs-state {
+				pins = "gpio43";
+				function = "qup1_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi10_data_clk: qup-spi10-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio40", "gpio41", "gpio42";
+				function = "qup1_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi11_cs: qup-spi11-cs-state {
+				pins = "gpio47";
+				function = "qup1_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi11_data_clk: qup-spi11-data-clk-state {
+				pins = "gpio44", "gpio45", "gpio46";
+				function = "qup1_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi12_cs: qup-spi12-cs-state {
+				pins = "gpio51";
+				function = "qup1_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi12_data_clk: qup-spi12-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio48", "gpio49", "gpio50";
+				function = "qup1_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi13_cs: qup-spi13-cs-state {
+				pins = "gpio55";
+				function = "qup1_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi13_data_clk: qup-spi13-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio52", "gpio53", "gpio54";
+				function = "qup1_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi14_cs: qup-spi14-cs-state {
+				pins = "gpio59";
+				function = "qup1_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi14_data_clk: qup-spi14-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio56", "gpio57", "gpio58";
+				function = "qup1_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi15_cs: qup-spi15-cs-state {
+				pins = "gpio53";
+				function = "qup1_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi15_data_clk: qup-spi15-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio54", "gpio55", "gpio52";
+				function = "qup1_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi16_cs: qup-spi16-cs-state {
+				pins = "gpio67";
+				function = "qup2_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi16_data_clk: qup-spi16-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio64", "gpio65", "gpio66";
+				function = "qup2_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi17_cs: qup-spi17-cs-state {
+				pins = "gpio71";
+				function = "qup2_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi17_data_clk: qup-spi17-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio68", "gpio69", "gpio70";
+				function = "qup2_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi18_cs: qup-spi18-cs-state {
+				pins = "gpio75";
+				function = "qup2_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi18_data_clk: qup-spi18-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio72", "gpio73", "gpio74";
+				function = "qup2_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi19_cs: qup-spi19-cs-state {
+				pins = "gpio79";
+				function = "qup2_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi19_data_clk: qup-spi19-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio76", "gpio77", "gpio78";
+				function = "qup2_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi20_cs: qup-spi20-cs-state {
+				pins = "gpio83";
+				function = "qup2_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi20_data_clk: qup-spi20-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio80", "gpio81", "gpio82";
+				function = "qup2_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi21_cs: qup-spi21-cs-state {
+				pins = "gpio87";
+				function = "qup2_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi21_data_clk: qup-spi21-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio84", "gpio85", "gpio86";
+				function = "qup2_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi22_cs: qup-spi22-cs-state {
+				pins = "gpio91";
+				function = "qup2_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi22_data_clk: qup-spi22-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio88", "gpio89", "gpio90";
+				function = "qup2_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi23_cs: qup-spi23-cs-state {
+				pins = "gpio83";
+				function = "qup2_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi23_data_clk: qup-spi23-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio80", "gpio81", "gpio82";
+				function = "qup2_se7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart2_default: qup-uart2-default-state {
+				tx-pins {
+					pins = "gpio10";
+					function = "qup0_se2";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio11";
+					function = "qup0_se2";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart14_default: qup-uart14-default-state {
+				cts-pins {
+					pins = "gpio56";
+					function = "qup1_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rts-pins {
+					pins = "gpio57";
+					function = "qup1_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				tx-pins {
+					pins = "gpio58";
+					function = "qup1_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio59";
+					function = "qup1_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart19_default: qup-uart19-default-state {
+				cts-pins {
+					pins = "gpio76";
+					function = "qup2_se3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rts-pins {
+					pins = "gpio77";
+					function = "qup2_se3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				tx-pins {
+					pins = "gpio78";
+					function = "qup2_se3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio79";
+					function = "qup2_se3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart21_default: qup-uart21-default-state {
+				tx-pins {
+					pins = "gpio86";
+					function = "qup2_se5";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio87";
+					function = "qup2_se5";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart22_default: qup-uart22-default-state {
+				tx-pins {
+					pins = "gpio90";
+					function = "qup2_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio91";
+					function = "qup2_se6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+		};
+
+		apps_smmu: iommu@15000000 {
+			compatible = "qcom,glymur-smmu-500",
+				     "qcom,smmu-500",
+				     "arm,mmu-500";
+			reg = <0x0 0x15000000 0x0 0x100000>;
+
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 690 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 691 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 692 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 693 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 694 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 695 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 696 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 697 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 488 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 490 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 491 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 492 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 493 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 494 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 497 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 498 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 499 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 500 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 501 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 502 IRQ_TYPE_LEVEL_HIGH>;
+
+			dma-coherent;
+		};
+
+		pcie_smmu: iommu@15480000 {
+			compatible = "arm,smmu-v3";
+			reg = <0x0 0x15480000 0x0 0x20000>;
+			interrupts = <GIC_SPI 964 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 962 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 960 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "eventq", "cmdq-sync", "gerror";
+			dma-coherent;
+			#iommu-cells = <1>;
+		};
+
+		intc: interrupt-controller@17000000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x17000000 0x0 0x10000>,
+			      <0x0 0x17080000 0x0 0x480000>;
+
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+			#interrupt-cells = <3>;
+			interrupt-controller;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			gic_its: msi-controller@17040000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x17040000 0x0 0x40000>;
+
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		watchdog@17600000 {
+			compatible = "qcom,apss-wdt-glymur", "qcom,kpss-wdt";
+			reg = <0x0 0x17600000 0x0 0x1000>;
+			clocks = <&sleep_clk>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		pdp0_mbox: mailbox@17610000 {
+			compatible = "qcom,glymur-cpucp-mbox", "qcom,x1e80100-cpucp-mbox";
+			reg = <0x0 0x17610000 0 0x8000>, <0 0x19980000 0 0x8000>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+		};
+
+		timer@17810000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x17810000 0x0 0x1000>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x0 0x0 0x20000000>;
+
+			frame@17811000 {
+				reg = <0x0 0x17811000 0x1000>,
+				      <0x0 0x17812000 0x1000>;
+
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <0>;
+			};
+
+			frame@17813000 {
+				reg = <0x0 0x17813000 0x1000>;
+
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <1>;
+
+				status = "disabled";
+			};
+
+			frame@17815000 {
+				reg = <0x0 0x17815000 0x1000>;
+
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <2>;
+
+				status = "disabled";
+			};
+
+			frame@17817000 {
+				reg = <0x0 0x17817000 0x1000>;
+
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <3>;
+
+				status = "disabled";
+			};
+
+			frame@17819000 {
+				reg = <0x0 0x17819000 0x1000>;
+
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <4>;
+
+				status = "disabled";
+			};
+
+			frame@1781b000 {
+				reg = <0x0 0x1781b000 0x1000>;
+
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <5>;
+
+				status = "disabled";
+			};
+
+			frame@1781d000 {
+				reg = <0x0 0x1781d000 0x1000>;
+
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+
+				frame-number = <6>;
+
+				status = "disabled";
+			};
+		};
+
+		apps_rsc: rsc@18900000 {
+			compatible = "qcom,rpmh-rsc";
+			label = "apps_rsc";
+			reg = <0x0 0x18900000 0x0 0x10000>,
+			      <0x0 0x18910000 0x0 0x10000>,
+			      <0x0 0x18920000 0x0 0x10000>;
+			reg-names = "drv-0",
+				    "drv-1",
+				    "drv-2";
+			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+			qcom,tcs-offset = <0xd00>;
+			qcom,drv-id = <2>;
+			qcom,tcs-config = <ACTIVE_TCS 2>,
+					  <SLEEP_TCS 3>,
+					  <WAKE_TCS 3>,
+					  <CONTROL_TCS 0>;
+			power-domains = <&system_pd>;
+
+			apps_bcm_voter: bcm-voter {
+				compatible = "qcom,bcm-voter";
+			};
+
+			rpmhcc: clock-controller {
+				compatible = "qcom,glymur-rpmh-clk";
+
+				clocks = <&xo_board>;
+				clock-names = "xo";
+
+				#clock-cells = <1>;
+			};
+
+			rpmhpd: power-controller {
+				compatible = "qcom,glymur-rpmhpd";
+
+				operating-points-v2 = <&rpmhpd_opp_table>;
+
+				#power-domain-cells = <1>;
+
+				rpmhpd_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					rpmhpd_opp_ret: opp-16 {
+						opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
+					};
+
+					rpmhpd_opp_min_svs: opp-48 {
+						opp-level = <RPMH_REGULATOR_LEVEL_MIN_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_d2: opp-52 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					};
+
+					rpmhpd_opp_low_svs_d1: opp-56 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					};
+
+					rpmhpd_opp_low_svs_d0: opp-60 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					};
+
+					rpmhpd_opp_low_svs: opp-64 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_l1: opp-80 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					};
+
+					rpmhpd_opp_svs: opp-128 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					};
+
+					rpmhpd_opp_svs_l0: opp-144 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					};
+
+					rpmhpd_opp_svs_l1: opp-192 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					};
+
+					rpmhpd_opp_nom: opp-256 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					};
+
+					rpmhpd_opp_nom_l1: opp-320 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					};
+
+					rpmhpd_opp_nom_l2: opp-336 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L2>;
+					};
+
+					rpmhpd_opp_turbo: opp-384 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					};
+
+					rpmhpd_opp_turbo_l1: opp-416 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					};
+				};
+			};
+		};
+
+		nsi_noc: interconnect@1d600000 {
+			compatible = "qcom,glymur-nsinoc";
+			reg = <0x0 0x1d600000 0x0 0x14080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		oobm_ss_noc: interconnect@1f300000 {
+			compatible = "qcom,glymur-oobm-ss-noc";
+			reg = <0x0 0x1f300000 0x0 0x49a00>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		system-cache-controller@20400000 {
+			compatible = "qcom,glymur-llcc";
+			reg = <0x0 0x21800000 0x0 0x100000>,
+			      <0x0 0x21a00000 0x0 0x100000>,
+			      <0x0 0x21c00000 0x0 0x100000>,
+			      <0x0 0x21e00000 0x0 0x100000>,
+			      <0x0 0x22800000 0x0 0x100000>,
+			      <0x0 0x22a00000 0x0 0x100000>,
+			      <0x0 0x22c00000 0x0 0x100000>,
+			      <0x0 0x22e00000 0x0 0x100000>,
+			      <0x0 0x23800000 0x0 0x100000>,
+			      <0x0 0x23a00000 0x0 0x100000>,
+			      <0x0 0x23c00000 0x0 0x100000>,
+			      <0x0 0x23e00000 0x0 0x100000>,
+			      <0x0 0x20400000 0x0 0x100000>,
+			      <0x0 0x20600000 0x0 0x100000>;
+			reg-names = "llcc0_base",
+				    "llcc1_base",
+				    "llcc2_base",
+				    "llcc3_base",
+				    "llcc4_base",
+				    "llcc5_base",
+				    "llcc6_base",
+				    "llcc7_base",
+				    "llcc8_base",
+				    "llcc9_base",
+				    "llcc10_base",
+				    "llcc11_base",
+				    "llcc_broadcast_base",
+				    "llcc_broadcast_and_base";
+
+			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		nsp_noc: interconnect@320c0000 {
+			compatible = "qcom,glymur-nsp-noc";
+			reg = <0x0 0x320c0000 0x0 0x21280>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		imem: sram@81e08000 {
+			compatible = "mmio-sram";
+			reg = <0x0 0x81e08600 0x0 0x300>;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x81e08600 0x300>;
+
+			cpu_scp_lpri0: scp-sram-section@0 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x0 0x180>;
+			};
+
+			cpu_scp_lpri1: scp-sram-section@180 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x180 0x180>;
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 12 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	thermal_zones: thermal-zones {
+		aoss-0-thermal {
+			thermal-sensors = <&tsens0 0>;
+
+			trips {
+				aoss-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-0-0-thermal {
+			thermal-sensors = <&tsens0 1>;
+
+			trips {
+				cpu-0-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-0-1-thermal {
+			thermal-sensors = <&tsens0 2>;
+
+			trips {
+				cpu-0-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-1-0-thermal {
+			thermal-sensors = <&tsens0 3>;
+
+			trips {
+				cpu-0-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-1-1-thermal {
+			thermal-sensors = <&tsens0 4>;
+
+			trips {
+				cpu-0-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-2-0-thermal {
+			thermal-sensors = <&tsens0 5>;
+
+			trips {
+				cpu-0-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-2-1-thermal {
+			thermal-sensors = <&tsens0 6>;
+
+			trips {
+				cpu-0-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-3-0-thermal {
+			thermal-sensors = <&tsens0 7>;
+
+			trips {
+				cpu-0-3-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-3-1-thermal {
+			thermal-sensors = <&tsens0 8>;
+
+			trips {
+				cpu-0-3-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-4-0-thermal {
+			thermal-sensors = <&tsens0 9>;
+
+			trips {
+				cpu-0-4-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-4-1-thermal {
+			thermal-sensors = <&tsens0 10>;
+
+			trips {
+				cpu-0-4-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-5-0-thermal {
+			thermal-sensors = <&tsens0 11>;
+
+			trips {
+				cpu-0-5-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-5-1-thermal {
+			thermal-sensors = <&tsens0 12>;
+
+			trips {
+				cpu-0-5-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-1-thermal {
+			thermal-sensors = <&tsens1 0>;
+
+			trips {
+				aoss-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-0-0-thermal {
+			thermal-sensors = <&tsens1 1>;
+
+			trips {
+				cpullc-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-0-1-thermal {
+			thermal-sensors = <&tsens1 2>;
+
+			trips {
+				cpullc-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-0-0-thermal {
+			thermal-sensors = <&tsens1 3>;
+
+			trips {
+				qmx-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-0-1-thermal {
+			thermal-sensors = <&tsens1 4>;
+
+			trips {
+				qmx-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-0-2-thermal {
+			thermal-sensors = <&tsens1 5>;
+
+			trips {
+				qmx-0-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddr-0-thermal {
+			thermal-sensors = <&tsens1 6>;
+
+			trips {
+				ddr-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		video-0-thermal {
+			thermal-sensors = <&tsens1 7>;
+
+			trips {
+				video-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		video-1-thermal {
+			thermal-sensors = <&tsens1 8>;
+
+			trips {
+				video-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-2-thermal {
+			thermal-sensors = <&tsens2 0>;
+
+			trips {
+				aoss-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-0-0-thermal {
+			thermal-sensors = <&tsens2 1>;
+
+			trips {
+				cpu-1-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-0-1-thermal {
+			thermal-sensors = <&tsens2 2>;
+
+			trips {
+				cpu-1-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-1-0-thermal {
+			thermal-sensors = <&tsens2 3>;
+
+			trips {
+				cpu-1-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-1-1-thermal {
+			thermal-sensors = <&tsens2 4>;
+
+			trips {
+				cpu-1-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-2-0-thermal {
+			thermal-sensors = <&tsens2 5>;
+
+			trips {
+				cpu-1-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-2-1-thermal {
+			thermal-sensors = <&tsens2 6>;
+
+			trips {
+				cpu-1-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-3-0-thermal {
+			thermal-sensors = <&tsens2 7>;
+
+			trips {
+				cpu-1-3-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-3-1-thermal {
+			thermal-sensors = <&tsens2 8>;
+
+			trips {
+				cpu-1-3-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-4-0-thermal {
+			thermal-sensors = <&tsens2 9>;
+
+			trips {
+				cpu-1-4-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-4-1-thermal {
+			thermal-sensors = <&tsens2 10>;
+
+			trips {
+				cpu-1-4-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-5-0-thermal {
+			thermal-sensors = <&tsens2 11>;
+
+			trips {
+				cpu-1-5-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-5-1-thermal {
+			thermal-sensors = <&tsens2 12>;
+
+			trips {
+				cpu-1-5-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-3-thermal {
+			thermal-sensors = <&tsens3 0>;
+
+			trips {
+				aoss-3-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-1-0-thermal {
+			thermal-sensors = <&tsens3 1>;
+
+			trips {
+				cpullc-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-1-1-thermal {
+			thermal-sensors = <&tsens3 2>;
+
+			trips {
+				cpullc-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-0-thermal {
+			thermal-sensors = <&tsens3 3>;
+
+			trips {
+				qmx-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-1-thermal {
+			thermal-sensors = <&tsens3 4>;
+
+			trips {
+				qmx-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-2-thermal {
+			thermal-sensors = <&tsens3 5>;
+
+			trips {
+				qmx-1-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-3-thermal {
+			thermal-sensors = <&tsens3 6>;
+
+			trips {
+				qmx-1-3-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-4-thermal {
+			thermal-sensors = <&tsens3 7>;
+
+			trips {
+				qmx-1-4-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-4-thermal {
+			thermal-sensors = <&tsens4 0>;
+
+			trips {
+				aoss-4-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-0-0-thermal {
+			thermal-sensors = <&tsens4 1>;
+
+			trips {
+				cpu-2-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-0-1-thermal {
+			thermal-sensors = <&tsens4 2>;
+
+			trips {
+				cpu-2-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-1-0-thermal {
+			thermal-sensors = <&tsens4 3>;
+
+			trips {
+				cpu-2-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-1-1-thermal {
+			thermal-sensors = <&tsens4 4>;
+
+			trips {
+				cpu-2-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-2-0-thermal {
+			thermal-sensors = <&tsens4 5>;
+
+			trips {
+				cpu-2-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-2-1-thermal {
+			thermal-sensors = <&tsens4 6>;
+
+			trips {
+				cpu-2-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-3-0-thermal {
+			thermal-sensors = <&tsens4 7>;
+
+			trips {
+				cpu-2-3-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-3-1-thermal {
+			thermal-sensors = <&tsens4 8>;
+
+			trips {
+				cpu-2-3-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-4-0-thermal {
+			thermal-sensors = <&tsens4 9>;
+
+			trips {
+				cpu-2-4-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-4-1-thermal {
+			thermal-sensors = <&tsens4 10>;
+
+			trips {
+				cpu-2-4-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-5-0-thermal {
+			thermal-sensors = <&tsens4 11>;
+
+			trips {
+				cpu-2-5-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-5-1-thermal {
+			thermal-sensors = <&tsens4 12>;
+
+			trips {
+				cpu-2-5-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-5-thermal {
+			thermal-sensors = <&tsens5 0>;
+
+			trips {
+				aoss-5-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-2-0-thermal {
+			thermal-sensors = <&tsens5 1>;
+
+			trips {
+				cpullc-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpuillc-2-1-thermal {
+			thermal-sensors = <&tsens5 2>;
+
+			trips {
+				cpullc-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-0-thermal {
+			thermal-sensors = <&tsens5 3>;
+
+			trips {
+				qmx-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-1-thermal {
+			thermal-sensors = <&tsens5 4>;
+
+			trips {
+				qmx-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-2-thermal {
+			thermal-sensors = <&tsens5 5>;
+
+			trips {
+				qmx-2-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-3-thermal {
+			thermal-sensors = <&tsens5 6>;
+
+			trips {
+				qmx-2-3-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-4-thermal {
+			thermal-sensors = <&tsens5 7>;
+
+			trips {
+				qmx-2-4-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-6-thermal {
+			thermal-sensors = <&tsens6 0>;
+
+			trips {
+				aoss-6-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphvx-0-thermal {
+			thermal-sensors = <&tsens6 1>;
+
+			trips {
+				nsphvx-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphvx-1-thermal {
+			thermal-sensors = <&tsens6 2>;
+
+			trips {
+				nsphvx-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphvx-2-thermal {
+			thermal-sensors = <&tsens6 3>;
+
+			trips {
+				nsphvx-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphvx-3-thermal {
+			thermal-sensors = <&tsens6 4>;
+
+			trips {
+				nsphvx-3-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphmx-0-thermal {
+			thermal-sensors = <&tsens6 5>;
+
+			trips {
+				nsphmx-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphmx-1-thermal {
+			thermal-sensors = <&tsens6 6>;
+
+			trips {
+				nsphmx-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphmx-2-thermal {
+			thermal-sensors = <&tsens6 7>;
+
+			trips {
+				nsphmx-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsphmx-3-thermal {
+			thermal-sensors = <&tsens6 8>;
+
+			trips {
+				nsphmx-3-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		camera-0-thermal {
+			thermal-sensors = <&tsens6 9>;
+
+			trips {
+				camera-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		camera-1-thermal {
+			thermal-sensors = <&tsens6 10>;
+
+			trips {
+				camera-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddr-1-thermal {
+			thermal-sensors = <&tsens6 11>;
+
+			trips {
+				ddr-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddr-2-thermal {
+			thermal-sensors = <&tsens6 12>;
+
+			trips {
+				ddr-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-7-thermal {
+			thermal-sensors = <&tsens7 0>;
+
+			trips {
+				aoss-7-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-0-thermal {
+			thermal-sensors = <&tsens7 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-1-thermal {
+			thermal-sensors = <&tsens7 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-2-thermal {
+			thermal-sensors = <&tsens7 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-0-thermal {
+			thermal-sensors = <&tsens7 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-1-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-1-thermal {
+			thermal-sensors = <&tsens7 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-1-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-2-thermal {
+			thermal-sensors = <&tsens7 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-1-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-0-thermal {
+			thermal-sensors = <&tsens7 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-2-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-1-thermal {
+			thermal-sensors = <&tsens7 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-2-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-2-thermal {
+			thermal-sensors = <&tsens7 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-2-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-3-0-thermal {
+			thermal-sensors = <&tsens7 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-3-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-3-1-thermal {
+			thermal-sensors = <&tsens7 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-3-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-3-2-thermal {
+			thermal-sensors = <&tsens7 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-3-2-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-0-thermal {
+			thermal-sensors = <&tsens7 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpuss-0-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-1-thermal {
+			thermal-sensors = <&tsens7 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpuss-1-critical {
+					temperature = <115000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
index 2390648a248f7..b20eb7c924265 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
+++ b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
@@ -616,6 +616,38 @@ platform {
 				sound-dai = <&q6apm>;
 			};
 		};
+
+		dp0-dai-link {
+			link-name = "DP0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		dp1-dai-link {
+			link-name = "DP1 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp1>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
 	};
 
 	usb-1-ss0-sbu-mux {
@@ -1102,9 +1134,7 @@ edp_bl_reg_en: edp-bl-reg-en-state {
 		pins = "gpio10";
 		function = "normal";
 	};
-};
 
-&pmc8380_3_gpios {
 	pm_sde7_aux_3p3_en: pcie-aux-3p3-default-state {
 		pins = "gpio8";
 		function = "normal";
@@ -1144,6 +1174,22 @@ &pmk8550_pwm {
 	status = "okay";
 };
 
+&sdhc_2 {
+	cd-gpios = <&tlmm 71 GPIO_ACTIVE_LOW>;
+
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l6b_1p8>;
+
+	no-sdio;
+	no-mmc;
+
+	pinctrl-0 = <&sdc2_default &sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep &sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
 &smb2360_0 {
 	status = "okay";
 };
@@ -1326,6 +1372,13 @@ rtmr2_default: rtmr2-reset-n-active-state {
 		bias-disable;
 	};
 
+	sdc2_card_det_n: sd-card-det-n-state {
+		pins = "gpio71";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
 	usb1_pwr_1p15_reg_en: usb1-pwr-1p15-reg-en-state {
 		pins = "gpio188";
 		function = "gpio";
@@ -1511,3 +1564,69 @@ &usb_mp_hsphy0 {
 &usb_mp_hsphy1 {
 	phys = <&eusb6_repeater>;
 };
+
+&thermal_zones {
+	gpuss-0-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-1-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-2-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-3-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-4-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-5-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-6-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-7-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
index b8e3e04a6fbd4..7292d823e13fd 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
@@ -447,14 +447,20 @@ &pcie6a_phy {
 };
 
 &qupv3_0 {
+	firmware-name = "qcom/x1e80100/qupv3fw.elf";
+
 	status = "okay";
 };
 
 &qupv3_1 {
+	firmware-name = "qcom/x1e80100/qupv3fw.elf";
+
 	status = "okay";
 };
 
 &qupv3_2 {
+	firmware-name = "qcom/x1e80100/qupv3fw.elf";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi
index db65c392e6189..0efeb7b7ff034 100644
--- a/arch/arm64/boot/dts/qcom/hamoa.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi
@@ -75,6 +75,7 @@ cpu0: cpu@0 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd0>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -91,6 +92,7 @@ cpu1: cpu@100 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd1>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu2: cpu@200 {
@@ -101,6 +103,7 @@ cpu2: cpu@200 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd2>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu3: cpu@300 {
@@ -111,6 +114,7 @@ cpu3: cpu@300 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd3>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu4: cpu@10000 {
@@ -121,6 +125,7 @@ cpu4: cpu@10000 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd4>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_1: l2-cache {
 				compatible = "cache";
@@ -137,6 +142,7 @@ cpu5: cpu@10100 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd5>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu6: cpu@10200 {
@@ -147,6 +153,7 @@ cpu6: cpu@10200 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd6>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu7: cpu@10300 {
@@ -157,6 +164,7 @@ cpu7: cpu@10300 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd7>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu8: cpu@20000 {
@@ -167,6 +175,7 @@ cpu8: cpu@20000 {
 			next-level-cache = <&l2_2>;
 			power-domains = <&cpu_pd8>, <&scmi_dvfs 2>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_2: l2-cache {
 				compatible = "cache";
@@ -183,6 +192,7 @@ cpu9: cpu@20100 {
 			next-level-cache = <&l2_2>;
 			power-domains = <&cpu_pd9>, <&scmi_dvfs 2>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu10: cpu@20200 {
@@ -193,6 +203,7 @@ cpu10: cpu@20200 {
 			next-level-cache = <&l2_2>;
 			power-domains = <&cpu_pd10>, <&scmi_dvfs 2>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu11: cpu@20300 {
@@ -203,6 +214,7 @@ cpu11: cpu@20300 {
 			next-level-cache = <&l2_2>;
 			power-domains = <&cpu_pd11>, <&scmi_dvfs 2>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu-map {
@@ -269,7 +281,7 @@ cluster_c4: cpu-sleep-0 {
 				idle-state-name = "ret";
 				arm,psci-suspend-param = <0x00000004>;
 				entry-latency-us = <180>;
-				exit-latency-us = <500>;
+				exit-latency-us = <320>;
 				min-residency-us = <600>;
 			};
 		};
@@ -4714,7 +4726,7 @@ sdhc_2: mmc@8804000 {
 
 			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
 				 <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>;
+				 <&bi_tcxo_div2>;
 			clock-names = "iface", "core", "xo";
 			iommus = <&apps_smmu 0x520 0>;
 			qcom,dll-config = <0x0007642c>;
@@ -4767,7 +4779,7 @@ sdhc_4: mmc@8844000 {
 
 			clocks = <&gcc GCC_SDCC4_AHB_CLK>,
 				 <&gcc GCC_SDCC4_APPS_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>;
+				 <&bi_tcxo_div2>;
 			clock-names = "iface", "core", "xo";
 			iommus = <&apps_smmu 0x160 0>;
 			qcom,dll-config = <0x0007642c>;
@@ -5432,19 +5444,19 @@ opp-338000000 {
 
 				opp-366000000 {
 					opp-hz = /bits/ 64 <366000000>;
-					required-opps = <&rpmhpd_opp_svs_l1>,
+					required-opps = <&rpmhpd_opp_svs>,
 							<&rpmhpd_opp_svs_l1>;
 				};
 
 				opp-444000000 {
 					opp-hz = /bits/ 64 <444000000>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-481000000 {
 					opp-hz = /bits/ 64 <481000000>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_turbo>;
 				};
 			};
@@ -5658,8 +5670,8 @@ mdss_dp0_out: endpoint {
 				mdss_dp0_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5747,8 +5759,8 @@ mdss_dp1_out: endpoint {
 				mdss_dp1_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5835,8 +5847,8 @@ mdss_dp2_out: endpoint {
 				mdss_dp2_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5918,8 +5930,8 @@ mdss_dp3_out: endpoint {
 				mdss_dp3_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
index bdb396afb9922..62877b46f9b3f 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
@@ -22,6 +22,15 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	regulator_fixed_1p8: s1800 {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-name = "fixed_1p8";
+	};
+
 	regulator_fixed_3p3: s3300 {
 		compatible = "regulator-fixed";
 		regulator-min-microvolt = <3300000>;
@@ -88,11 +97,27 @@ &blsp1_uart2 {
 	status = "okay";
 };
 
+&cpu0 {
+	cpu-supply = <&mp5496_s1>;
+};
+
+&cpu1 {
+	cpu-supply = <&mp5496_s1>;
+};
+
+&cpu2 {
+	cpu-supply = <&mp5496_s1>;
+};
+
+&cpu3 {
+	cpu-supply = <&mp5496_s1>;
+};
+
 &rpm_requests {
 	regulators {
 		compatible = "qcom,rpm-mp5496-regulators";
 
-		ipq9574_s1: s1 {
+		mp5496_s1: s1 {
 		/*
 		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
 		 * During regulator registration, kernel not knowing the initial voltage,
@@ -121,6 +146,11 @@ mp5496_l5: l5 {
 	};
 };
 
+&sdhc_1 {
+	vmmc-supply = <&regulator_fixed_3p3>;
+	vqmmc-supply = <&regulator_fixed_1p8>;
+};
+
 &sleep_clk {
 	clock-frequency = <32000>;
 };
@@ -169,6 +199,38 @@ data-pins {
 			bias-disable;
 		};
 	};
+
+	sdc_default_state: sdc-default-state {
+		clk-pins {
+			pins = "gpio5";
+			function = "sdc_clk";
+			drive-strength = <8>;
+			bias-disable;
+		};
+
+		cmd-pins {
+			pins = "gpio4";
+			function = "sdc_cmd";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		data-pins {
+			pins = "gpio0", "gpio1", "gpio2",
+			       "gpio3", "gpio6", "gpio7",
+			       "gpio8", "gpio9";
+			function = "sdc_data";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		rclk-pins {
+			pins = "gpio10";
+			function = "sdc_rclk";
+			drive-strength = <8>;
+			bias-pull-down;
+		};
+	};
 };
 
 &qpic_bam {
@@ -179,8 +241,6 @@ &qpic_nand {
 	pinctrl-0 = <&qpic_snand_default_state>;
 	pinctrl-names = "default";
 
-	status = "okay";
-
 	flash@0 {
 		compatible = "spi-nand";
 		reg = <0>;
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp418-emmc.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp418-emmc.dts
new file mode 100644
index 0000000000000..2cf35c7174116
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp418-emmc.dts
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * IPQ9574 RDP418 (eMMC variant) board device tree source
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "ipq9574-rdp-common.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C2 (eMMC)";
+	compatible = "qcom,ipq9574-ap-al02-c2-emmc", "qcom,ipq9574";
+
+};
+
+&sdhc_1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
index f4f9199d4ab1e..23d4cba7c6b65 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
@@ -16,48 +16,6 @@ / {
 
 };
 
-&sdhc_1 {
-	pinctrl-0 = <&sdc_default_state>;
-	pinctrl-names = "default";
-	mmc-ddr-1_8v;
-	mmc-hs200-1_8v;
-	mmc-hs400-1_8v;
-	mmc-hs400-enhanced-strobe;
-	max-frequency = <384000000>;
-	bus-width = <8>;
+&qpic_nand {
 	status = "okay";
 };
-
-&tlmm {
-	sdc_default_state: sdc-default-state {
-		clk-pins {
-			pins = "gpio5";
-			function = "sdc_clk";
-			drive-strength = <8>;
-			bias-disable;
-		};
-
-		cmd-pins {
-			pins = "gpio4";
-			function = "sdc_cmd";
-			drive-strength = <8>;
-			bias-pull-up;
-		};
-
-		data-pins {
-			pins = "gpio0", "gpio1", "gpio2",
-			       "gpio3", "gpio6", "gpio7",
-			       "gpio8", "gpio9";
-			function = "sdc_data";
-			drive-strength = <8>;
-			bias-pull-up;
-		};
-
-		rclk-pins {
-			pins = "gpio10";
-			function = "sdc_rclk";
-			drive-strength = <8>;
-			bias-pull-down;
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp433-common.dtsi b/arch/arm64/boot/dts/qcom/ipq9574-rdp433-common.dtsi
new file mode 100644
index 0000000000000..3422058ac4801
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp433-common.dtsi
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * IPQ9574 RDP433 board device tree source
+ *
+ * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pcie1 {
+	pinctrl-0 = <&pcie1_default>;
+	pinctrl-names = "default";
+
+	perst-gpios = <&tlmm 26 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 27 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pcie2_phy {
+	status = "okay";
+};
+
+&pcie2 {
+	pinctrl-0 = <&pcie2_default>;
+	pinctrl-names = "default";
+
+	perst-gpios = <&tlmm 29 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 30 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pcie3_phy {
+	status = "okay";
+};
+
+&pcie3 {
+	pinctrl-0 = <&pcie3_default>;
+	pinctrl-names = "default";
+
+	perst-gpios = <&tlmm 32 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 33 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&tlmm {
+
+	pcie1_default: pcie1-default-state {
+		clkreq-n-pins {
+			pins = "gpio25";
+			function = "pcie1_clk";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio26";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-down;
+			output-low;
+		};
+
+		wake-n-pins {
+			pins = "gpio27";
+			function = "pcie1_wake";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+	};
+
+	pcie2_default: pcie2-default-state {
+		clkreq-n-pins {
+			pins = "gpio28";
+			function = "pcie2_clk";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio29";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-down;
+			output-low;
+		};
+
+		wake-n-pins {
+			pins = "gpio30";
+			function = "pcie2_wake";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+	};
+
+	pcie3_default: pcie3-default-state {
+		clkreq-n-pins {
+			pins = "gpio31";
+			function = "pcie3_clk";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio32";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-up;
+			output-low;
+		};
+
+		wake-n-pins {
+			pins = "gpio33";
+			function = "pcie3_wake";
+			drive-strength = <6>;
+			bias-pull-up;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp433-emmc.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp433-emmc.dts
new file mode 100644
index 0000000000000..44884231499fa
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp433-emmc.dts
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * IPQ9574 RDP433 (eMMC variant) board device tree source
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "ipq9574-rdp-common.dtsi"
+#include "ipq9574-rdp433-common.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C7 (eMMC)";
+	compatible = "qcom,ipq9574-ap-al02-c7-emmc", "qcom,ipq9574";
+};
+
+&sdhc_1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
index 5a546a14998b0..88439697c0747 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
@@ -8,124 +8,14 @@
 
 /dts-v1/;
 
-#include <dt-bindings/gpio/gpio.h>
 #include "ipq9574-rdp-common.dtsi"
+#include "ipq9574-rdp433-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C7";
 	compatible = "qcom,ipq9574-ap-al02-c7", "qcom,ipq9574";
 };
 
-&pcie1_phy {
+&qpic_nand {
 	status = "okay";
 };
-
-&pcie1 {
-	pinctrl-0 = <&pcie1_default>;
-	pinctrl-names = "default";
-
-	perst-gpios = <&tlmm 26 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 27 GPIO_ACTIVE_LOW>;
-	status = "okay";
-};
-
-&pcie2_phy {
-	status = "okay";
-};
-
-&pcie2 {
-	pinctrl-0 = <&pcie2_default>;
-	pinctrl-names = "default";
-
-	perst-gpios = <&tlmm 29 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 30 GPIO_ACTIVE_LOW>;
-	status = "okay";
-};
-
-&pcie3_phy {
-	status = "okay";
-};
-
-&pcie3 {
-	pinctrl-0 = <&pcie3_default>;
-	pinctrl-names = "default";
-
-	perst-gpios = <&tlmm 32 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 33 GPIO_ACTIVE_LOW>;
-	status = "okay";
-};
-
-&tlmm {
-
-	pcie1_default: pcie1-default-state {
-		clkreq-n-pins {
-			pins = "gpio25";
-			function = "pcie1_clk";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-
-		perst-n-pins {
-			pins = "gpio26";
-			function = "gpio";
-			drive-strength = <8>;
-			bias-pull-down;
-			output-low;
-		};
-
-		wake-n-pins {
-			pins = "gpio27";
-			function = "pcie1_wake";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-	};
-
-	pcie2_default: pcie2-default-state {
-		clkreq-n-pins {
-			pins = "gpio28";
-			function = "pcie2_clk";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-
-		perst-n-pins {
-			pins = "gpio29";
-			function = "gpio";
-			drive-strength = <8>;
-			bias-pull-down;
-			output-low;
-		};
-
-		wake-n-pins {
-			pins = "gpio30";
-			function = "pcie2_wake";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-	};
-
-	pcie3_default: pcie3-default-state {
-		clkreq-n-pins {
-			pins = "gpio31";
-			function = "pcie3_clk";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-
-		perst-n-pins {
-			pins = "gpio32";
-			function = "gpio";
-			drive-strength = <8>;
-			bias-pull-up;
-			output-low;
-		};
-
-		wake-n-pins {
-			pins = "gpio33";
-			function = "pcie3_wake";
-			drive-strength = <6>;
-			bias-pull-up;
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
index d36d1078763ec..cbc9047cfe929 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
@@ -15,3 +15,7 @@ / {
 	compatible = "qcom,ipq9574-ap-al02-c6", "qcom,ipq9574";
 
 };
+
+&qpic_nand {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
index c30c9fbedf26b..d233ec530cc3d 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
@@ -15,3 +15,7 @@ / {
 	compatible = "qcom,ipq9574-ap-al02-c8", "qcom,ipq9574";
 
 };
+
+&qpic_nand {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
index 0dc382f5d5ecd..f2334b9e0ed49 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
@@ -14,3 +14,7 @@ / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C9";
 	compatible = "qcom,ipq9574-ap-al02-c9", "qcom,ipq9574";
 };
+
+&qpic_nand {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574.dtsi b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
index d7278f2137ac5..622cfa96ed2b3 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
@@ -56,7 +56,6 @@ cpu0: cpu@0 {
 			clocks = <&apcs_glb APCS_ALIAS0_CORE_CLK>;
 			clock-names = "cpu";
 			operating-points-v2 = <&cpu_opp_table>;
-			cpu-supply = <&ipq9574_s1>;
 			#cooling-cells = <2>;
 		};
 
@@ -69,7 +68,6 @@ cpu1: cpu@1 {
 			clocks = <&apcs_glb APCS_ALIAS0_CORE_CLK>;
 			clock-names = "cpu";
 			operating-points-v2 = <&cpu_opp_table>;
-			cpu-supply = <&ipq9574_s1>;
 			#cooling-cells = <2>;
 		};
 
@@ -82,7 +80,6 @@ cpu2: cpu@2 {
 			clocks = <&apcs_glb APCS_ALIAS0_CORE_CLK>;
 			clock-names = "cpu";
 			operating-points-v2 = <&cpu_opp_table>;
-			cpu-supply = <&ipq9574_s1>;
 			#cooling-cells = <2>;
 		};
 
@@ -95,7 +92,6 @@ cpu3: cpu@3 {
 			clocks = <&apcs_glb APCS_ALIAS0_CORE_CLK>;
 			clock-names = "cpu";
 			operating-points-v2 = <&cpu_opp_table>;
-			cpu-supply = <&ipq9574_s1>;
 			#cooling-cells = <2>;
 		};
 
@@ -467,6 +463,15 @@ sdhc_1: mmc@7804000 {
 			clock-names = "iface", "core", "xo", "ice";
 			non-removable;
 			supports-cqe;
+			pinctrl-0 = <&sdc_default_state>;
+			pinctrl-names = "default";
+			mmc-ddr-1_8v;
+			mmc-hs200-1_8v;
+			mmc-hs400-1_8v;
+			mmc-hs400-enhanced-strobe;
+			max-frequency = <384000000>;
+			bus-width = <8>;
+
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
index 9ef57ad0ca71d..9be86479ceef1 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi
+++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
@@ -1239,7 +1239,7 @@ intc: interrupt-controller@17000000 {
 
 			gic_its: msi-controller@17040000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0x0 0x17040000 0x0 0x20000>;
+				reg = <0x0 0x17040000 0x0 0x40000>;
 
 				msi-controller;
 				#msi-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi
index 6079e67ea829b..343da1b180953 100644
--- a/arch/arm64/boot/dts/qcom/kodiak.dtsi
+++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi
@@ -2445,7 +2445,7 @@ pcie1_phy: phy@1c0e000 {
 			reg = <0 0x01c0e000 0 0x1000>;
 			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
 				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
-				 <&gcc GCC_PCIE_CLKREF_EN>,
+				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&gcc GCC_PCIE1_PHY_RCHNG_CLK>,
 				 <&gcc GCC_PCIE_1_PIPE_CLK>;
 			clock-names = "aux",
@@ -3036,6 +3036,58 @@ lpass_dmic23_data: dmic23-data-state {
 				bias-pull-down;
 			};
 
+			lpass_i2s1_active: i2s1-active-state {
+				clk-pins {
+					pins = "gpio6";
+					function = "i2s1_clk";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+
+				ws-pins {
+					pins = "gpio7";
+					function = "i2s1_ws";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+
+				data-pins {
+					pins = "gpio8", "gpio9";
+					function = "i2s1_data";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+			};
+
+			lpass_i2s1_sleep: i2s1-sleep-state {
+				clk-pins {
+					pins = "gpio6";
+					function = "i2s1_clk";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+
+				ws-pins {
+					pins = "gpio7";
+					function = "i2s1_ws";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+
+				data-pins {
+					pins = "gpio8", "gpio9";
+					function = "i2s1_data";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+			};
+
 			lpass_rx_swr_clk: rx-swr-clk-state {
 				pins = "gpio3";
 				function = "swr_rx_clk";
@@ -4849,6 +4901,7 @@ usb_1: usb@a600000 {
 			phys = <&usb_1_hsphy>, <&usb_1_qmpphy QMP_USB43DP_USB3_PHY>;
 			phy-names = "usb2-phy", "usb3-phy";
 			maximum-speed = "super-speed";
+			usb-role-switch;
 
 			ports {
 				#address-cells = <1>;
@@ -5505,8 +5558,8 @@ port@1 {
 				edp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5604,8 +5657,8 @@ mdss_dp_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/lemans-el2.dtso b/arch/arm64/boot/dts/qcom/lemans-el2.dtso
index ed615dce6c789..621ad930cf547 100644
--- a/arch/arm64/boot/dts/qcom/lemans-el2.dtso
+++ b/arch/arm64/boot/dts/qcom/lemans-el2.dtso
@@ -10,6 +10,10 @@
 /dts-v1/;
 /plugin/;
 
+&gpu_zap_shader {
+	status = "disabled";
+};
+
 &iris {
 	status = "disabled";
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi
index 808827b83553d..147ebf9b1ac61 100644
--- a/arch/arm64/boot/dts/qcom/lemans.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans.dtsi
@@ -4625,19 +4625,19 @@ opp-366000000 {
 
 				opp-444000000 {
 					opp-hz = /bits/ 64 <444000000>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-533000000 {
 					opp-hz = /bits/ 64 <533000000>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo>;
 				};
 
 				opp-560000000 {
 					opp-hz = /bits/ 64 <560000000>;
-					required-opps = <&rpmhpd_opp_turbo_l1>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo_l1>;
 				};
 			};
@@ -5404,8 +5404,8 @@ port@1 {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5492,8 +5492,8 @@ port@1 {
 				dp1_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/mahua-crd.dts b/arch/arm64/boot/dts/qcom/mahua-crd.dts
new file mode 100644
index 0000000000000..9c8244e892ddd
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/mahua-crd.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "mahua.dtsi"
+#include "glymur-crd.dtsi"
+
+/delete-node/ &pmcx0102_d_e0;
+/delete-node/ &pmcx0102_d0_thermal;
+/delete-node/ &pmh0104_i_e0;
+/delete-node/ &pmh0104_i0_thermal;
+/delete-node/ &pmh0104_j_e0;
+/delete-node/ &pmh0104_j0_thermal;
+
+/ {
+	model = "Qualcomm Technologies, Inc. Mahua CRD";
+	compatible = "qcom,mahua-crd", "qcom,mahua";
+};
diff --git a/arch/arm64/boot/dts/qcom/mahua.dtsi b/arch/arm64/boot/dts/qcom/mahua.dtsi
new file mode 100644
index 0000000000000..7aa8d26b2b3a3
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/mahua.dtsi
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/* Mahua is heavily based on Glymur, with some meaningful differences */
+#include "glymur.dtsi"
+
+/delete-node/ &cluster2_pd;
+/delete-node/ &cpu_map_cluster2;
+/delete-node/ &cpu12;
+/delete-node/ &cpu13;
+/delete-node/ &cpu14;
+/delete-node/ &cpu15;
+/delete-node/ &cpu16;
+/delete-node/ &cpu17;
+/delete-node/ &cpu_pd12;
+/delete-node/ &cpu_pd13;
+/delete-node/ &cpu_pd14;
+/delete-node/ &cpu_pd15;
+/delete-node/ &cpu_pd16;
+/delete-node/ &cpu_pd17;
+/delete-node/ &tsens6;
+/delete-node/ &tsens7;
+
+&aggre1_noc {
+	compatible = "qcom,mahua-aggre1-noc", "qcom,glymur-aggre1-noc";
+};
+
+&aggre2_noc {
+	compatible = "qcom,mahua-aggre2-noc", "qcom,glymur-aggre2-noc";
+};
+
+&aggre3_noc {
+	compatible = "qcom,mahua-aggre3-noc", "qcom,glymur-aggre3-noc";
+};
+
+&aggre4_noc {
+	compatible = "qcom,mahua-aggre4-noc", "qcom,glymur-aggre4-noc";
+};
+
+&clk_virt {
+	compatible = "qcom,mahua-clk-virt", "qcom,glymur-clk-virt";
+};
+
+&cnoc_main {
+	compatible = "qcom,mahua-cnoc-main", "qcom,glymur-cnoc-main";
+};
+
+&config_noc {
+	compatible = "qcom,mahua-cnoc-cfg";
+};
+
+&hsc_noc {
+	compatible = "qcom,mahua-hscnoc";
+};
+
+&lpass_ag_noc {
+	compatible = "qcom,mahua-lpass-ag-noc", "qcom,glymur-lpass-ag-noc";
+};
+
+&lpass_lpiaon_noc {
+	compatible = "qcom,mahua-lpass-lpiaon-noc", "qcom,glymur-lpass-lpiaon-noc";
+};
+
+&lpass_lpicx_noc {
+	compatible = "qcom,mahua-lpass-lpicx-noc", "qcom,glymur-lpass-lpicx-noc";
+};
+
+&mc_virt {
+	compatible = "qcom,mahua-mc-virt";
+};
+
+&mmss_noc {
+	compatible = "qcom,mahua-mmss-noc", "qcom,glymur-mmss-noc";
+};
+
+&nsi_noc {
+	compatible = "qcom,mahua-nsinoc", "qcom,glymur-nsinoc";
+};
+
+&nsp_noc {
+	compatible = "qcom,mahua-nsp-noc", "qcom,glymur-nsp-noc";
+};
+
+&oobm_ss_noc {
+	compatible = "qcom,mahua-oobm-ss-noc", "qcom,glymur-oobm-ss-noc";
+};
+
+&pcie_east_anoc {
+	compatible = "qcom,mahua-pcie-east-anoc", "qcom,glymur-pcie-east-anoc";
+};
+
+&pcie_east_slv_noc {
+	compatible = "qcom,mahua-pcie-east-slv-noc", "qcom,glymur-pcie-east-slv-noc";
+};
+
+&pcie_west_anoc {
+	compatible = "qcom,mahua-pcie-west-anoc";
+	clocks = <&gcc GCC_AGGRE_NOC_PCIE_3B_WEST_SF_AXI_CLK>,
+		 <&gcc GCC_AGGRE_NOC_PCIE_4_WEST_SF_AXI_CLK>,
+		 <&gcc GCC_AGGRE_NOC_PCIE_6_WEST_SF_AXI_CLK>;
+};
+
+&pcie_west_slv_noc {
+	compatible = "qcom,mahua-pcie-west-slv-noc";
+};
+
+&system_noc {
+	compatible = "qcom,mahua-system-noc", "qcom,glymur-system-noc";
+};
+
+&tlmm {
+	compatible = "qcom,mahua-tlmm";
+};
+
+&thermal_zones {
+	/delete-node/ aoss-6-thermal;
+	/delete-node/ aoss-7-thermal;
+	/delete-node/ cpu-2-0-0-thermal;
+	/delete-node/ cpu-2-0-1-thermal;
+	/delete-node/ cpu-2-1-0-thermal;
+	/delete-node/ cpu-2-1-1-thermal;
+	/delete-node/ cpu-2-2-0-thermal;
+	/delete-node/ cpu-2-2-1-thermal;
+	/delete-node/ cpu-2-3-0-thermal;
+	/delete-node/ cpu-2-3-1-thermal;
+	/delete-node/ cpu-2-4-0-thermal;
+	/delete-node/ cpu-2-4-1-thermal;
+	/delete-node/ cpu-2-5-0-thermal;
+	/delete-node/ cpu-2-5-1-thermal;
+	/delete-node/ cpullc-2-0-thermal;
+	/delete-node/ cpuillc-2-1-thermal;
+	/delete-node/ ddr-2-thermal;
+	/delete-node/ gpu-3-0-thermal;
+	/delete-node/ gpu-3-1-thermal;
+	/delete-node/ gpu-3-2-thermal;
+	/delete-node/ qmx-2-0-thermal;
+	/delete-node/ qmx-2-1-thermal;
+	/delete-node/ qmx-2-2-thermal;
+	/delete-node/ qmx-2-3-thermal;
+	/delete-node/ qmx-2-4-thermal;
+	/delete-node/ video-1-thermal;
+
+	ddr-1-thermal {
+		thermal-sensors = <&tsens1 7>;
+	};
+
+	video-0-thermal {
+		thermal-sensors = <&tsens1 8>;
+	};
+
+	nsphvx-0-thermal {
+		thermal-sensors = <&tsens4 1>;
+	};
+
+	nsphvx-1-thermal {
+		thermal-sensors = <&tsens4 2>;
+	};
+
+	nsphvx-2-thermal {
+		thermal-sensors = <&tsens4 3>;
+	};
+
+	nsphvx-3-thermal {
+		thermal-sensors = <&tsens4 4>;
+	};
+
+	nsphmx-0-thermal {
+		thermal-sensors = <&tsens4 5>;
+	};
+
+	nsphmx-1-thermal {
+		thermal-sensors = <&tsens4 6>;
+	};
+
+	nsphmx-2-thermal {
+		thermal-sensors = <&tsens4 7>;
+	};
+
+	nsphmx-3-thermal {
+		thermal-sensors = <&tsens4 8>;
+	};
+
+	camera-0-thermal {
+		thermal-sensors = <&tsens4 9>;
+	};
+
+	camera-1-thermal {
+		thermal-sensors = <&tsens4 10>;
+	};
+
+	gpu-0-0-thermal {
+		thermal-sensors = <&tsens5 1>;
+	};
+
+	gpu-0-1-thermal {
+		thermal-sensors = <&tsens5 2>;
+	};
+
+	gpu-0-2-thermal {
+		thermal-sensors = <&tsens5 3>;
+	};
+
+	gpu-1-0-thermal {
+		thermal-sensors = <&tsens5 4>;
+	};
+
+	gpu-1-1-thermal {
+		thermal-sensors = <&tsens5 5>;
+	};
+
+	gpu-1-2-thermal {
+		thermal-sensors = <&tsens5 6>;
+	};
+
+	gpu-2-0-thermal {
+		thermal-sensors = <&tsens5 7>;
+	};
+
+	gpu-2-1-thermal {
+		thermal-sensors = <&tsens5 8>;
+	};
+
+	gpu-2-2-thermal {
+		thermal-sensors = <&tsens5 9>;
+	};
+
+	gpuss-0-thermal {
+		thermal-sensors = <&tsens5 10>;
+	};
+
+	gpuss-1-thermal {
+		thermal-sensors = <&tsens5 11>;
+	};
+
+	gpuss-2-thermal {
+		thermal-sensors = <&tsens5 12>;
+
+		trips {
+			trip-point0 {
+				temperature = <90000>;
+				hysteresis = <5000>;
+				type = "hot";
+			};
+
+			gpuss-2-critical {
+				temperature = <115000>;
+				hysteresis = <1000>;
+				type = "critical";
+			};
+		};
+	};
+
+	gpuss-3-thermal {
+		thermal-sensors = <&tsens5 13>;
+
+		trips {
+			trip-point0 {
+				temperature = <90000>;
+				hysteresis = <5000>;
+				type = "hot";
+			};
+
+			gpuss-3-critical {
+				temperature = <115000>;
+				hysteresis = <1000>;
+				type = "critical";
+			};
+		};
+	};
+
+	gpuss-4-thermal {
+		thermal-sensors = <&tsens5 14>;
+
+		trips {
+			trip-point0 {
+				temperature = <90000>;
+				hysteresis = <5000>;
+				type = "hot";
+			};
+
+			gpuss-4-critical {
+				temperature = <115000>;
+				hysteresis = <1000>;
+				type = "critical";
+			};
+		};
+	};
+};
+
+&tsens4 {
+	#qcom,sensors = <11>;
+};
+
+&tsens5 {
+	#qcom,sensors = <15>;
+};
+
diff --git a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
index 52895dd9e4fa1..0199253859f10 100644
--- a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
+++ b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
@@ -29,9 +29,19 @@ aliases {
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		pinctrl-0 = <&volume_up_default>;
+		pinctrl-0 = <&volume_up_default>, <&hall_sensor_default>;
 		pinctrl-names = "default";
 
+		/* Powered by the always-on vreg_l10b */
+		event-hall-sensor {
+			label = "Hall Effect Sensor";
+			gpios = <&tlmm 70 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			linux,can-disable;
+			wakeup-source;
+		};
+
 		key-volume-up {
 			label = "Volume Up";
 			gpios = <&pm7550_gpios 6 GPIO_ACTIVE_LOW>;
@@ -316,6 +326,8 @@ vreg_l10b: ldo10 {
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			/* Hall sensor VDD */
+			regulator-always-on;
 		};
 
 		vreg_l11b: ldo11 {
@@ -755,6 +767,13 @@ sdc2_card_det_n: sdc2-card-det-state {
 		bias-pull-up;
 	};
 
+	hall_sensor_default: hall-sensor-default-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	pm8008_int_default: pm8008-int-default-state {
 		pins = "gpio125";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi
index e1a51d43943ff..ed814c45ca052 100644
--- a/arch/arm64/boot/dts/qcom/milos.dtsi
+++ b/arch/arm64/boot/dts/qcom/milos.dtsi
@@ -18,6 +18,7 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 
 / {
@@ -1214,6 +1215,95 @@ IPCC_MPROC_SIGNAL_GLINK_QMP
 
 				label = "lpass";
 				qcom,remote-pid = <2>;
+
+				fastrpc {
+					compatible = "qcom,fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "adsp";
+					qcom,non-secure-domain;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+						iommus = <&apps_smmu 0x1003 0x0>,
+							 <&apps_smmu 0x1063 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+						iommus = <&apps_smmu 0x1004 0x0>,
+							 <&apps_smmu 0x1064 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+						iommus = <&apps_smmu 0x1005 0x0>,
+							 <&apps_smmu 0x1065 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+						iommus = <&apps_smmu 0x1006 0x0>,
+							 <&apps_smmu 0x1066 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+						iommus = <&apps_smmu 0x1007 0x0>,
+							 <&apps_smmu 0x1067 0x0>;
+						dma-coherent;
+					};
+				};
+
+				gpr {
+					compatible = "qcom,gpr";
+					qcom,glink-channels = "adsp_apps";
+					qcom,domain = <GPR_DOMAIN_ID_ADSP>;
+					qcom,intents = <512 20>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					q6apm: service@1 {
+						compatible = "qcom,q6apm";
+						reg = <GPR_APM_MODULE_IID>;
+						#sound-dai-cells = <0>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6apmbedai: bedais {
+							compatible = "qcom,q6apm-lpass-dais";
+							#sound-dai-cells = <1>;
+						};
+
+						q6apmdai: dais {
+							compatible = "qcom,q6apm-dais";
+							iommus = <&apps_smmu 0x1001 0x0>,
+								 <&apps_smmu 0x1061 0x0>;
+						};
+					};
+
+					q6prm: service@2 {
+						compatible = "qcom,q6prm";
+						reg = <GPR_PRM_MODULE_IID>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6prmcc: clock-controller {
+							compatible = "qcom,q6prm-lpass-clocks";
+							#clock-cells = <2>;
+						};
+					};
+				};
 			};
 		};
 
@@ -1667,6 +1757,21 @@ tlmm: pinctrl@f100000 {
 
 			wakeup-parent = <&pdc>;
 
+			qup_spi0_data_clk: qup-spi0-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio0", "gpio1", "gpio2";
+				function = "qup0_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_spi0_cs: qup-spi0-cs-state {
+				pins = "gpio3";
+				function = "qup0_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
 			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
 				/* SDA, SCL */
 				pins = "gpio4", "gpio5";
@@ -1683,29 +1788,6 @@ qup_i2c3_data_clk: qup-i2c3-data-clk-state {
 				bias-pull-up = <2200>;
 			};
 
-			qup_i2c7_data_clk: qup-i2c7-data-clk-state {
-				/* SDA, SCL */
-				pins = "gpio32", "gpio33";
-				function = "qup1_se0";
-				drive-strength = <2>;
-				bias-pull-up;
-			};
-
-			qup_spi0_cs: qup-spi0-cs-state {
-				pins = "gpio3";
-				function = "qup0_se0";
-				drive-strength = <6>;
-				bias-disable;
-			};
-
-			qup_spi0_data_clk: qup-spi0-data-clk-state {
-				/* MISO, MOSI, CLK */
-				pins = "gpio0", "gpio1", "gpio2";
-				function = "qup0_se0";
-				drive-strength = <6>;
-				bias-disable;
-			};
-
 			qup_uart5_default: qup-uart5-default-state {
 				/* TX, RX */
 				pins = "gpio25", "gpio26";
@@ -1714,10 +1796,10 @@ qup_uart5_default: qup-uart5-default-state {
 				bias-disable;
 			};
 
-			qup_uart11_default: qup-uart11-default-state {
-				/* TX, RX */
-				pins = "gpio50", "gpio51";
-				function = "qup1_se4";
+			qup_i2c7_data_clk: qup-i2c7-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio32", "gpio33";
+				function = "qup1_se0";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
@@ -1730,6 +1812,14 @@ qup_uart11_cts_rts: qup-uart11-cts-rts-state {
 				bias-pull-down;
 			};
 
+			qup_uart11_default: qup-uart11-default-state {
+				/* TX, RX */
+				pins = "gpio50", "gpio51";
+				function = "qup1_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
 			sdc2_default: sdc2-default-state {
 				clk-pins {
 					pins = "gpio62";
@@ -1911,7 +2001,7 @@ ppi_cluster1: interrupt-partition-1 {
 
 			gic_its: msi-controller@17140000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0x0 0x17140000 0x0 0x20000>;
+				reg = <0x0 0x17140000 0x0 0x40000>;
 
 				msi-controller;
 				#msi-cells = <1>;
@@ -2164,6 +2254,101 @@ IPCC_MPROC_SIGNAL_GLINK_QMP
 
 				label = "cdsp";
 				qcom,remote-pid = <5>;
+
+				fastrpc {
+					compatible = "qcom,fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "cdsp";
+					qcom,non-secure-domain;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@1 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <1>;
+						iommus = <&apps_smmu 0x0c01 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@2 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <2>;
+						iommus = <&apps_smmu 0x0c02 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+						iommus = <&apps_smmu 0x0c03 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+						iommus = <&apps_smmu 0x0c04 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+						iommus = <&apps_smmu 0x0c05 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+						iommus = <&apps_smmu 0x0c06 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+						iommus = <&apps_smmu 0x0c07 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@8 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <8>;
+						iommus = <&apps_smmu 0x0c08 0x0>;
+						dma-coherent;
+					};
+
+					/* note: secure cb9 in downstream */
+
+					compute-cb@12 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <12>;
+						iommus = <&apps_smmu 0x0c0c 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@13 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <13>;
+						iommus = <&apps_smmu 0x0c0d 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@14 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <14>;
+						iommus = <&apps_smmu 0x0c0e 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@15 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <15>;
+						iommus = <&apps_smmu 0x0c0f 0x0>;
+						dma-coherent;
+					};
+				};
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
new file mode 100644
index 0000000000000..ca14f0ea4dae6
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
@@ -0,0 +1,466 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+#include "monaco.dtsi"
+#include "monaco-pmics.dtsi"
+#include "monaco-monza-som.dtsi"
+
+/ {
+	model = "Arduino VENTUNO Q";
+	compatible = "arduino,monza", "qcom,qcs8300";
+
+	aliases {
+		ethernet0 = &ethernet0;
+		i2c1 = &i2c1;
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&gpio_keys_default>;
+		pinctrl-names = "default";
+
+		button-home {
+			label = "Home Key";
+			linux,code = <KEY_HOMEPAGE>;
+			gpios = <&tlmm 79 GPIO_ACTIVE_LOW>;
+			debounce-interval = <60>;
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&adv7535_out>;
+			};
+		};
+	};
+
+	sound {
+		compatible = "qcom,qcs8275-sndcard";
+		model = "arduino-monza";
+		audio-routing = "IN12",  "Headset Mic12",
+				"Headset Mic12",  "MICBIAS",
+				"IN56",  "Headset Mic56",
+				"Headset Mic56",  "MICBIAS",
+				"MIC1",  "MICBIAS",
+				"Headphone",  "HPL",
+				"Headphone",  "HPR",
+				"Receiver",  "RCVL",
+				"Receiver",  "RCVR",
+				"Speaker",  "SPKL",
+				"Speaker",  "SPKR";
+
+		pinctrl-0 = <&quad_mi2s_active>, <&quad_mclk_active>,  <&lpi_i2s4_active>;
+		pinctrl-names = "default";
+
+		pri-i2s-playback-dai-link {
+			link-name = "Analog Playback";
+
+			codec {
+				sound-dai = <&max98091>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai 137>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		pri-i2s-capture-dai-link {
+			link-name = "Analog Capture";
+
+			codec {
+				sound-dai = <&max98091>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai 138>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		hdmi-mi2s-playback-dai-link {
+			link-name = "HDMI Playback";
+
+			codec {
+				sound-dai = <&adv7535>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai 145>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	vdc_3v3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdc_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vdc_1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdc_1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vdc_5v: regulator-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdc_5v";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&tlmm 49 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+		startup-delay-us = <20000>;
+	};
+
+	vreg_nvme: regulator-3p3-m2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_m2_3p3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&tlmm 51 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <20000>;
+	};
+};
+
+&ethernet0 {
+	phy-mode = "2500base-x";
+	phy-handle = <&hsgmii_phy0>;
+
+	pinctrl-0 = <&ethernet0_default>;
+	pinctrl-names = "default";
+
+	snps,mtl-rx-config = <&mtl_rx_setup>;
+	snps,mtl-tx-config = <&mtl_tx_setup>;
+
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		hsgmii_phy0: ethernet-phy@1c {
+			compatible = "ethernet-phy-id004d.d101";
+			reg = <0x1c>;
+			reset-gpios = <&tlmm 50 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <11000>;
+			reset-deassert-us = <70000>;
+		};
+	};
+
+	mtl_rx_setup: rx-queues-config {
+		snps,rx-queues-to-use = <4>;
+		snps,rx-sched-sp;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,map-to-dma-channel = <0x0>;
+			snps,route-up;
+			snps,priority = <0x1>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,map-to-dma-channel = <0x1>;
+			snps,route-ptp;
+		};
+
+		queue2 {
+			snps,avb-algorithm;
+			snps,map-to-dma-channel = <0x2>;
+			snps,route-avcp;
+		};
+
+		queue3 {
+			snps,avb-algorithm;
+			snps,map-to-dma-channel = <0x3>;
+			snps,priority = <0xc>;
+		};
+	};
+
+	mtl_tx_setup: tx-queues-config {
+		snps,tx-queues-to-use = <4>;
+
+		queue0 {
+			snps,dcb-algorithm;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+		};
+
+		queue2 {
+			snps,avb-algorithm;
+			snps,send_slope = <0x1000>;
+			snps,idle_slope = <0x1000>;
+			snps,high_credit = <0x3e800>;
+			snps,low_credit = <0xffc18000>;
+		};
+
+		queue3 {
+			snps,avb-algorithm;
+			snps,send_slope = <0x1000>;
+			snps,idle_slope = <0x1000>;
+			snps,high_credit = <0x3e800>;
+			snps,low_credit = <0xffc18000>;
+		};
+	};
+};
+
+&i2c12 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	max98091: audio-codec@10 {
+		compatible = "maxim,max98091";
+		reg = <0x10>;
+		pinctrl-0 = <&max98091_default>;
+		pinctrl-names = "default";
+		interrupts-extended = <&tlmm 16 IRQ_TYPE_LEVEL_HIGH>; /* GPIO_16 */
+		clocks = <&q6prmcc LPASS_CLK_ID_MCLK_3 LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
+	};
+
+	adv7535: bridge@3d {
+		compatible = "adi,adv7535";
+		reg = <0x3d>;
+		pinctrl-0 = <&adv7535_default>;
+		pinctrl-names = "default";
+		interrupts-extended = <&tlmm 93 IRQ_TYPE_EDGE_FALLING>;
+		avdd-supply = <&vdc_1v8>;
+		dvdd-supply = <&vdc_1v8>;
+		pvdd-supply = <&vdc_1v8>;
+		a2vdd-supply = <&vdc_1v8>;
+		v3p3-supply = <&vdc_3v3>;
+		v1p2-supply = <&vdc_1v8>;
+		adi,dsi-lanes = <4>;
+		#sound-dai-cells = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7535_in: endpoint {
+					remote-endpoint = <&mdss_dsi0_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7535_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp0_phy {
+	status = "okay";
+};
+
+&mdss_dsi0 {
+	status = "okay";
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&adv7535_in>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	status = "okay";
+};
+
+&pcie0 {
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	vddpe-3v3-supply = <&vdc_3v3>;
+};
+
+&pcie1 {
+	pinctrl-0 = <&pcie1_default_state>;
+	pinctrl-names = "default";
+
+	vddpe-3v3-supply = <&vreg_nvme>;
+};
+
+&pcieport0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+
+	pci@0,0 {
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		bus-range = <0x01 0xff>;
+		ranges;
+		reg = <0x010000 0x00 0x00 0x00 0x00>;
+
+		pci@2,0 {
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			bus-range = <0x00 0xff>;
+			ranges;
+			reg = <0x021000 0x00 0x00 0x00 0x00>;
+
+			usb@0 {
+				compatible = "pci104c,8241";
+				reg = <0 0 0 0 0>;
+				ti,pwron-active-high;
+			};
+		};
+	};
+};
+
+&pcieport1 {
+	reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
+};
+
+&tlmm {
+	pcie0_default_state: pcie0-default-state {
+		wake-pins {
+			pins = "gpio0";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		clkreq-pins {
+			pins = "gpio1";
+			function = "pcie0_clkreq";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-pins {
+			pins = "gpio2";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	ethernet0_default: ethernet0-default-state {
+		ethernet0_mdc: ethernet0-mdc-pins {
+			pins = "gpio5";
+			function = "emac0_mdc";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+
+		ethernet0_mdio: ethernet0-mdio-pins {
+			pins = "gpio6";
+			function = "emac0_mdio";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+	};
+
+	max98091_default: max98091-default-state {
+		pins = "gpio16";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	pcie1_default_state: pcie1-default-state {
+		wake-pins {
+			pins = "gpio21";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		clkreq-pins {
+			pins = "gpio22";
+			function = "pcie1_clkreq";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-pins {
+			pins = "gpio23";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	gpio_keys_default: gpio-keys-default-state {
+		pins = "gpio79";
+		function = "gpio";
+		bias-disable;
+	};
+
+	adv7535_default: adv7535-default-state {
+		pins = "gpio93";
+		function = "gpio";
+		bias-pull-up;
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+/* Internally connected to the MCU (e.g. for DFU). */
+&usb_2 {
+	dr_mode = "host";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco-el2.dtso b/arch/arm64/boot/dts/qcom/monaco-el2.dtso
new file mode 100644
index 0000000000000..a7e3270f86090
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/monaco-el2.dtso
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * Monaco specific modifications required to boot in EL2.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&gpu_zap_shader {
+	status = "disabled";
+};
+
+&iris {
+	status = "disabled";
+};
+
+&remoteproc_adsp {
+	iommus = <&apps_smmu 0x2000 0x0>;
+};
+
+&remoteproc_cdsp {
+	iommus = <&apps_smmu 0x19c0 0x0400>;
+};
+
+&remoteproc_gpdsp {
+	iommus = <&apps_smmu 0x28a0 0x0>;
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso
new file mode 100644
index 0000000000000..0d5ccd020e6ee
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/monaco-evk-camera-imx577.dtso
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/qcom,sa8775p-camcc.h>
+#include <dt-bindings/gpio/gpio.h>
+
+&camss {
+	vdda-phy-supply = <&vreg_l4a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			csiphy1_ep: endpoint {
+				data-lanes = <0 1 2 3>;
+				remote-endpoint = <&imx577_ep1>;
+			};
+		};
+	};
+};
+
+&cci1 {
+	pinctrl-0 = <&cci1_0_default>;
+	pinctrl-1 = <&cci1_0_sleep>;
+
+	status = "okay";
+};
+
+&cci1_i2c0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@1a {
+		compatible = "sony,imx577";
+		reg = <0x1a>;
+
+		reset-gpios = <&expander2 1 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&cam1_default>;
+		pinctrl-names = "default";
+
+		clocks = <&camcc CAM_CC_MCLK1_CLK>;
+		assigned-clocks = <&camcc CAM_CC_MCLK1_CLK>;
+		assigned-clock-rates = <24000000>;
+
+		avdd-supply = <&vreg_cam1_2p8>;
+
+		port {
+			imx577_ep1: endpoint {
+				link-frequencies = /bits/ 64 <600000000>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&csiphy1_ep>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco-evk.dts b/arch/arm64/boot/dts/qcom/monaco-evk.dts
index 565418b86b2ad..fb687309decd0 100644
--- a/arch/arm64/boot/dts/qcom/monaco-evk.dts
+++ b/arch/arm64/boot/dts/qcom/monaco-evk.dts
@@ -27,6 +27,25 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	connector-2 {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		label = "micro-USB";
+		type = "micro";
+
+		id-gpios = <&pmm8620au_0_gpios 9 GPIO_ACTIVE_HIGH>;
+		vbus-gpios = <&expander6 7 GPIO_ACTIVE_HIGH>;
+		vbus-supply = <&usb2_vbus>;
+
+		pinctrl-0 = <&usb2_id>;
+		pinctrl-names = "default";
+
+		port {
+			usb2_con_hs_ep: endpoint {
+				remote-endpoint = <&usb_2_dwc3_hs>;
+			};
+		};
+	};
+
 	dmic: audio-codec-0 {
 		compatible = "dmic-codec";
 		#sound-dai-cells = <0>;
@@ -38,6 +57,15 @@ max98357a: audio-codec-1 {
 		#sound-dai-cells = <0>;
 	};
 
+	usb2_vbus: regulator-usb2-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb2_vbus";
+		gpio = <&pmm8650au_1_gpios 7 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+	};
+
 	sound {
 		compatible = "qcom,qcs8275-sndcard";
 		model = "MONACO-EVK";
@@ -77,6 +105,57 @@ platform {
 			};
 		};
 	};
+
+	vreg_cam0_2p8: vreg-cam0-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam0_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 73 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam0_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	vreg_cam1_2p8: vreg-cam1-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam1_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 74 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam1_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	vreg_cam2_2p8: vreg-cam2-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam2_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 75 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam2_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	/* This comes from a PMIC handled within the SAIL domain */
+	vreg_s2s: vreg-s2s {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2s";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
 };
 
 &apps_rsc {
@@ -362,6 +441,11 @@ expander0: gpio@38 {
 		reg = <0x38>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 56 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander0_int>;
+		pinctrl-names = "default";
 	};
 
 	expander1: gpio@39 {
@@ -369,6 +453,11 @@ expander1: gpio@39 {
 		reg = <0x39>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 16 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander1_int>;
+		pinctrl-names = "default";
 	};
 
 	expander2: gpio@3a {
@@ -376,6 +465,11 @@ expander2: gpio@3a {
 		reg = <0x3a>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 95 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander2_int>;
+		pinctrl-names = "default";
 	};
 
 	expander3: gpio@3b {
@@ -383,6 +477,11 @@ expander3: gpio@3b {
 		reg = <0x3b>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 24 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander3_int>;
+		pinctrl-names = "default";
 	};
 
 	expander4: gpio@3c {
@@ -390,6 +489,11 @@ expander4: gpio@3c {
 		reg = <0x3c>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 96 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander4_int>;
+		pinctrl-names = "default";
 	};
 
 	expander5: gpio@3d {
@@ -397,6 +501,11 @@ expander5: gpio@3d {
 		reg = <0x3d>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander5_int>;
+		pinctrl-names = "default";
 	};
 
 	expander6: gpio@3e {
@@ -404,6 +513,11 @@ expander6: gpio@3e {
 		reg = <0x3e>;
 		#gpio-cells = <2>;
 		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 52 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander6_int>;
+		pinctrl-names = "default";
 	};
 };
 
@@ -449,6 +563,16 @@ &pcieport1 {
 	wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
 };
 
+&pmm8620au_0_gpios {
+	usb2_id: usb2-id-state {
+		pins = "gpio9";
+		function = "normal";
+		input-enable;
+		bias-pull-up;
+		power-source = <0>;
+	};
+};
+
 &qupv3_id_0 {
 	firmware-name = "qcom/qcs8300/qupv3fw.elf";
 	status = "okay";
@@ -477,6 +601,17 @@ &remoteproc_gpdsp {
 	status = "okay";
 };
 
+&sdhc_1 {
+	vmmc-supply = <&vreg_l8a>;
+	vqmmc-supply = <&vreg_s2s>;
+
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
 &serdes0 {
 	phy-supply = <&vreg_l4a>;
 
@@ -494,7 +629,6 @@ tpm@0 {
 };
 
 &tlmm {
-
 	pcie0_default_state: pcie0-default-state {
 		wake-pins {
 			pins = "gpio0";
@@ -534,6 +668,18 @@ ethernet0_mdio: ethernet0-mdio-pins {
 		};
 	};
 
+	expander5_int: expander5-int-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander1_int: expander1-int-state {
+		pins = "gpio16";
+		function = "gpio";
+		bias-pull-up;
+	};
+
 	qup_i2c1_default: qup-i2c1-state {
 		pins = "gpio19", "gpio20";
 		function = "qup0_se1";
@@ -564,12 +710,63 @@ perst-pins {
 		};
 	};
 
+	expander3_int: expander3-int-state {
+		pins = "gpio24";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander6_int:  expander6-int-state {
+		pins = "gpio52";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander0_int: expander0-int-state {
+		pins = "gpio56";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	cam0_avdd_2v8_en_default: cam0-avdd-2v8-en-state {
+		pins = "gpio73";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam1_avdd_2v8_en_default: cam1-avdd-2v8-en-state {
+		pins = "gpio74";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam2_avdd_2v8_en_default: cam2-avdd-2v8-en-state {
+		pins = "gpio75";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	qup_i2c15_default: qup-i2c15-state {
 		pins = "gpio91", "gpio92";
 		function = "qup1_se7";
 		drive-strength = <2>;
 		bias-pull-up;
 	};
+
+	expander2_int: expander2-int-state {
+		pins = "gpio95";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander4_int: expander4-int-state {
+		pins = "gpio96";
+		function = "gpio";
+		bias-pull-up;
+	};
 };
 
 &uart7 {
@@ -613,3 +810,19 @@ &usb_qmpphy {
 
 	status = "okay";
 };
+
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3_hs {
+	remote-endpoint = <&usb2_con_hs_ep>;
+};
+
+&usb_2_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l7c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi
new file mode 100644
index 0000000000000..9b5ed55939b86
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/monaco-monza-som.dtsi
@@ -0,0 +1,323 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "monaco.dtsi"
+#include "monaco-pmics.dtsi"
+
+/ {
+	/* This comes from a PMIC handled within the SAIL domain */
+	vreg_s2s: vreg-s2s {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2s";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmm8654au-rpmh-regulators";
+		qcom,pmic-id = "a";
+
+		vreg_l3a: ldo3 {
+			regulator-name = "vreg_l3a";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l4a: ldo4 {
+			regulator-name = "vreg_l4a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5a: ldo5 {
+			regulator-name = "vreg_l5a";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6a: ldo6 {
+			regulator-name = "vreg_l6a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7a: ldo7 {
+			regulator-name = "vreg_l7a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8a: ldo8 {
+			regulator-name = "vreg_l8a";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9a: ldo9 {
+			regulator-name = "vreg_l9a";
+			regulator-min-microvolt = <2970000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmm8654au-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vreg_s5c: smps5 { /* LPDDR VDD2H */
+			regulator-name = "vreg_s5c";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c: ldo1 { /* LPDDR VDDQ */
+			regulator-name = "vreg_l1c";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <512000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c: ldo2 { /* LPDDR VDD2L */
+			regulator-name = "vreg_l2c";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c: ldo4 {
+			regulator-name = "vreg_l4c";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c: ldo7 {
+			regulator-name = "vreg_l7c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c: ldo8 { /* LPDDR VDD1 */
+			regulator-name = "vreg_l8c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c: ldo9 { /* QFPROM */
+			regulator-name = "vreg_l9c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&mdss_dp0 {
+	pinctrl-0 = <&dp_hpd>;
+	pinctrl-names = "default";
+};
+
+&mdss_dp0_phy {
+	vdda-phy-supply = <&vreg_l5a>;
+	vdda-pll-supply = <&vreg_l4a>;
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l5a>;
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l4a>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/qcs8300/a623_zap.mbn";
+};
+
+&iris {
+	status = "okay";
+};
+
+/* PCIe0 Gen4 x2 */
+&pcie0 {
+	iommu-map = <0x0 &pcie_smmu 0x0000 0x1>,
+		    <0x100 &pcie_smmu 0x0001 0x1>,
+		    <0x200 &pcie_smmu 0x0007 0x1>,
+		    <0x208 &pcie_smmu 0x0002 0x1>,
+		    <0x210 &pcie_smmu 0x0003 0x1>,
+		    <0x218 &pcie_smmu 0x0004 0x1>,
+		    <0x300 &pcie_smmu 0x0005 0x1>,
+		    <0x400 &pcie_smmu 0x0006 0x1>;
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l6a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+/* PCIe1 Gen4 x4 */
+&pcie1 {
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l6a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	firmware-name = "qcom/qcs8300/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	firmware-name = "qcom/qcs8300/qupv3fw.elf";
+	status = "okay";
+};
+
+/* There is a HW/FW issue preventing proper REFGEN hardware voting
+ * for the USB2 HS PHY. As a workaround, we force REFGEN to stay
+ * always‑on in software, matching initial bootloader config.
+ */
+&refgen {
+	regulator-always-on;
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/qcs8300/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/qcs8300/cdsp0.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_gpdsp {
+	firmware-name = "qcom/qcs8300/gpdsp0.mbn";
+
+	status = "okay";
+};
+
+/* OnSom eMMC */
+&sdhc_1 {
+	vmmc-supply = <&vreg_l8a>;
+	vqmmc-supply = <&vreg_s2s>;
+
+	bus-width = <8>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
+/* Ethernet/SGMII */
+&serdes0 {
+	phy-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&tlmm {
+	dp_hpd: dp-hpd-state {
+		pins = "gpio94";
+		function = "edp0_hot";
+		bias-disable;
+	};
+};
+
+/* USB0 HS + SS */
+&usb_1_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l7c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
+
+&usb_qmpphy {
+	vdda-phy-supply = <&vreg_l7a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+/* USB1 HS */
+&usb_2_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l7c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi
index 5d2df4305d1c1..10e799dd4a78d 100644
--- a/arch/arm64/boot/dts/qcom/monaco.dtsi
+++ b/arch/arm64/boot/dts/qcom/monaco.dtsi
@@ -3,6 +3,7 @@
  * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
+#include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,qcs8300-gcc.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/clock/qcom,sa8775p-camcc.h>
@@ -20,6 +21,7 @@
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
+#include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -765,6 +767,11 @@ smem_mem: smem@90900000 {
 			hwlocks = <&tcsr_mutex 3>;
 		};
 
+		gunyah_md_mem: gunyah-md-region@91a80000 {
+			reg = <0x0 0x91a80000 0x0 0x80000>;
+			no-map;
+		};
+
 		lpass_machine_learning_mem: lpass-machine-learning-region@93b00000 {
 			reg = <0x0 0x93b00000 0x0 0xf00000>;
 			no-map;
@@ -2234,6 +2241,10 @@ aggre1_noc: interconnect@16c0000 {
 			reg = <0x0 0x016c0000 0x0 0x17080>;
 			#interconnect-cells = <2>;
 			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_QUPV3_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB2_PRIM_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>;
 		};
 
 		aggre2_noc: interconnect@1700000 {
@@ -2241,6 +2252,7 @@ aggre2_noc: interconnect@1700000 {
 			reg = <0x0 0x01700000 0x0 0x1a080>;
 			#interconnect-cells = <2>;
 			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&rpmhcc RPMH_IPA_CLK>;
 		};
 
 		pcie_anoc: interconnect@1760000 {
@@ -2866,6 +2878,75 @@ q6prmcc: clock-controller {
 			};
 		};
 
+		lpass_tlmm: pinctrl@3440000 {
+			compatible = "qcom,qcs8300-lpass-lpi-pinctrl", "qcom,sm8450-lpass-lpi-pinctrl";
+			reg = <0x0 0x03440000 0x0 0x20000>,
+			      <0x0 0x034d0000 0x0 0x10000>;
+
+			clocks = <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+			clock-names = "core", "audio";
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&lpass_tlmm 0 0 23>;
+
+			quad_mclk_active: quad-mclk-state {
+				clk-pins {
+					pins = "gpio5";
+					function = "ext_mclk1_c";
+					drive-strength = <8>;
+					bias-disable;
+				};
+			};
+
+			quad_mi2s_active: quad-active-state {
+				data-pins {
+					pins = "gpio2", "gpio3";
+					function = "qua_mi2s_data";
+					drive-strength = <8>;
+					bias-disable;
+				};
+
+				sclk-pins {
+					pins = "gpio0";
+					function = "qua_mi2s_sclk";
+					drive-strength = <8>;
+					bias-disable;
+				};
+
+				ws-pins {
+					pins = "gpio1";
+					function = "qua_mi2s_ws";
+					drive-strength = <8>;
+					bias-disable;
+				};
+			};
+
+			lpi_i2s4_active: lpi_i2s4-active-state {
+				data0-pins {
+					pins = "gpio17";
+					function = "i2s4_data";
+					drive-strength = <8>;
+					bias-disable;
+				};
+
+				clk-pins {
+					pins = "gpio12";
+					function = "i2s4_clk";
+					drive-strength = <8>;
+					bias-disable;
+				};
+
+				ws-pins {
+					pins = "gpio13";
+					function = "i2s4_ws";
+					drive-strength = <8>;
+					bias-disable;
+				};
+			};
+		};
+
 		lpass_ag_noc: interconnect@3c40000 {
 			compatible = "qcom,qcs8300-lpass-ag-noc";
 			reg = <0x0 0x03c40000 0x0 0x17200>;
@@ -4740,11 +4821,21 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			interconnect-names = "sdhc-ddr",
 					     "cpu-sdhc";
 
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&sdc1_state_on>;
+			pinctrl-1 = <&sdc1_state_off>;
+
 			qcom,dll-config = <0x000f64ee>;
 			qcom,ddr-config = <0x80040868>;
+			bus-width = <8>;
 			supports-cqe;
 			dma-coherent;
 
+			mmc-ddr-1_8v;
+			mmc-hs200-1_8v;
+			mmc-hs400-1_8v;
+			mmc-hs400-enhanced-strobe;
+
 			status = "disabled";
 
 			sdhc1_opp_table: opp-table {
@@ -5103,6 +5194,7 @@ gem_noc: interconnect@9100000 {
 			reg = <0x0 0x9100000 0x0 0xf7080>;
 			#interconnect-cells = <2>;
 			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&gcc GCC_DDRSS_GPU_AXI_CLK>;
 		};
 
 		llcc: system-cache-controller@9200000 {
@@ -5171,9 +5263,29 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			snps,dis_u2_susphy_quirk;
 			snps,dis_u3_susphy_quirk;
 
+			usb-role-switch;
 			wakeup-source;
 
 			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_1_dwc3_hs: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_1_dwc3_ss: endpoint {
+					};
+				};
+			};
 		};
 
 		usb_2: usb@a400000 {
@@ -5232,7 +5344,14 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			qcom,select-utmi-as-pipe-clk;
 			wakeup-source;
 
+			usb-role-switch;
+
 			status = "disabled";
+
+			port {
+				usb_2_dwc3_hs: endpoint {
+				};
+			};
 		};
 
 		iris: video-codec@aa00000 {
@@ -5288,19 +5407,19 @@ opp-366000000 {
 
 				opp-444000000 {
 					opp-hz = /bits/ 64 <444000000>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-533000000 {
 					opp-hz = /bits/ 64 <533000000>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo>;
 				};
 
 				opp-560000000 {
 					opp-hz = /bits/ 64 <560000000>;
-					required-opps = <&rpmhpd_opp_turbo_l1>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo_l1>;
 				};
 			};
@@ -5319,6 +5438,117 @@ videocc: clock-controller@abf0000 {
 			#power-domain-cells = <1>;
 		};
 
+		cci0: cci@ac13000 {
+			compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci";
+			reg = <0x0 0x0ac13000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 460 IRQ_TYPE_EDGE_RISING>;
+
+			clocks = <&camcc CAM_CC_CPAS_AHB_CLK>,
+				 <&camcc CAM_CC_CCI_0_CLK>;
+			clock-names = "ahb",
+				      "cci";
+
+			power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>;
+
+			pinctrl-0 = <&cci0_0_default &cci0_1_default>;
+			pinctrl-1 = <&cci0_0_sleep &cci0_1_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci0_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci0_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		cci1: cci@ac14000 {
+			compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci";
+			reg = <0x0 0x0ac14000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 271 IRQ_TYPE_EDGE_RISING>;
+
+			clocks = <&camcc CAM_CC_CPAS_AHB_CLK>,
+				 <&camcc CAM_CC_CCI_1_CLK>;
+			clock-names = "ahb",
+				      "cci";
+
+			power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>;
+
+			pinctrl-0 = <&cci1_0_default &cci1_1_default>;
+			pinctrl-1 = <&cci1_0_sleep &cci1_1_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci1_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci1_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		cci2: cci@ac15000 {
+			compatible = "qcom,qcs8300-cci", "qcom,msm8996-cci";
+			reg = <0x0 0x0ac15000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 651 IRQ_TYPE_EDGE_RISING>;
+
+			clocks = <&camcc CAM_CC_CPAS_AHB_CLK>,
+				 <&camcc CAM_CC_CCI_2_CLK>;
+			clock-names = "ahb",
+				      "cci";
+
+			power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>;
+
+			pinctrl-0 = <&cci2_0_default &cci2_1_default>;
+			pinctrl-1 = <&cci2_0_sleep &cci2_1_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci2_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci2_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
 		camss: isp@ac78000 {
 			compatible = "qcom,qcs8300-camss";
 
@@ -5573,9 +5803,19 @@ port@0 {
 						reg = <0>;
 
 						dpu_intf0_out: endpoint {
+
 							remote-endpoint = <&mdss_dp0_in>;
 						};
 					};
+
+					port@1 {
+						reg = <1>;
+
+						dpu_intf1_out: endpoint {
+
+							remote-endpoint = <&mdss_dsi0_in>;
+						};
+					};
 				};
 
 				mdp_opp_table: opp-table {
@@ -5603,6 +5843,98 @@ opp-650000000 {
 				};
 			};
 
+			mdss_dsi0: dsi@ae94000 {
+				compatible = "qcom,qcs8300-dsi-ctrl",
+					     "qcom,sa8775p-dsi-ctrl",
+					     "qcom,mdss-dsi-ctrl";
+				reg = <0x0 0x0ae94000 0x0 0x400>;
+				reg-names = "dsi_ctrl";
+
+				interrupt-parent = <&mdss>;
+				interrupts = <4>;
+
+				clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK>,
+					 <&dispcc MDSS_DISP_CC_MDSS_BYTE0_INTF_CLK>,
+					 <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK>,
+					 <&dispcc MDSS_DISP_CC_MDSS_ESC0_CLK>,
+					 <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_DISP_HF_AXI_CLK>;
+				clock-names = "byte",
+					      "byte_intf",
+					      "pixel",
+					      "core",
+					      "iface",
+					      "bus";
+
+				assigned-clocks = <&dispcc MDSS_DISP_CC_MDSS_BYTE0_CLK_SRC>,
+						  <&dispcc MDSS_DISP_CC_MDSS_PCLK0_CLK_SRC>;
+				assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+							 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>;
+
+				phys = <&mdss_dsi0_phy>;
+
+				operating-points-v2 = <&mdss_dsi_opp_table>;
+				power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+				refgen-supply = <&refgen>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss_dsi0_in: endpoint {
+
+							remote-endpoint = <&dpu_intf1_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss_dsi0_out: endpoint {
+						};
+					};
+				};
+
+				mdss_dsi_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-358000000 {
+						opp-hz = /bits/ 64 <358000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+				};
+			};
+
+			mdss_dsi0_phy: phy@ae94400 {
+				compatible = "qcom,qcs8300-dsi-phy-5nm",
+					     "qcom,sa8775p-dsi-phy-5nm";
+				reg = <0x0 0x0ae94400 0x0 0x200>,
+				      <0x0 0x0ae94600 0x0 0x280>,
+				      <0x0 0x0ae94900 0x0 0x280>;
+				reg-names = "dsi_phy",
+					    "dsi_phy_lane",
+					    "dsi_pll";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				clocks = <&dispcc MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&rpmhcc RPMH_CXO_CLK>;
+				clock-names = "iface",
+					      "ref";
+
+				status = "disabled";
+			};
+
 			mdss_dp0_phy: phy@aec2a00 {
 				compatible = "qcom,qcs8300-edp-phy", "qcom,sa8775p-edp-phy";
 
@@ -5697,8 +6029,8 @@ port@1 {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5730,7 +6062,9 @@ dispcc: clock-controller@af00000 {
 				 <&mdss_dp0_phy 0>,
 				 <&mdss_dp0_phy 1>,
 				 <0>, <0>,
-				 <0>, <0>, <0>, <0>;
+				 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+				 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>,
+				 <0>, <0>;
 			power-domains = <&rpmhpd RPMHPD_MMCX>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
@@ -5876,6 +6210,219 @@ tlmm: pinctrl@f100000 {
 			#interrupt-cells = <2>;
 			wakeup-parent = <&pdc>;
 
+			cam0_default: cam0-default-state {
+				pins = "gpio67";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam1_default: cam1-default-state {
+				pins = "gpio68";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam2_default: cam2-default-state {
+				pins = "gpio69";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cci0_0_default: cci0-0-default-state {
+				sda-pins {
+					pins = "gpio57";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio58";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci0_0_sleep: cci0-0-sleep-state {
+				sda-pins {
+					pins = "gpio57";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio58";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
+			cci0_1_default: cci0-1-default-state {
+				sda-pins {
+					pins = "gpio29";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio30";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci0_1_sleep: cci0-1-sleep-state {
+				sda-pins {
+					pins = "gpio29";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio30";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
+			cci1_0_default: cci1-0-default-state {
+				sda-pins {
+					pins = "gpio59";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio60";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci1_0_sleep: cci1-0-sleep-state {
+				sda-pins {
+					pins = "gpio59";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio60";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
+			cci1_1_default: cci1-1-default-state {
+				sda-pins {
+					pins = "gpio31";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio32";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci1_1_sleep: cci1-1-sleep-state {
+				sda-pins {
+					pins = "gpio31";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio32";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
+			cci2_0_default: cci2-0-default-state {
+				sda-pins {
+					pins = "gpio61";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio62";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci2_0_sleep: cci2-0-sleep-state {
+				sda-pins {
+					pins = "gpio61";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio62";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
+			cci2_1_default: cci2-1-default-state {
+				sda-pins {
+					pins = "gpio54";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+
+				scl-pins {
+					pins = "gpio55";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-up = <2200>;
+				};
+			};
+
+			cci2_1_sleep: cci2-1-sleep-state {
+				sda-pins {
+					pins = "gpio54";
+					function = "cci_i2c_sda";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				scl-pins {
+					pins = "gpio55";
+					function = "cci_i2c_scl";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+			};
+
 			hs0_mi2s_active: hs0-mi2s-active-state {
 				pins = "gpio106", "gpio107", "gpio108", "gpio109";
 				function = "hs0_mi2s";
@@ -6414,12 +6961,12 @@ qup_uart10_cts: qup-uart10-cts-state {
 			};
 
 			qup_uart10_rts: qup-uart10-rts-state {
-				pins = "gpio84";
+				pins = "gpio85";
 				function = "qup1_se2";
 			};
 
 			qup_uart10_tx: qup-uart10-tx-state {
-				pins = "gpio85";
+				pins = "gpio86";
 				function = "qup1_se2";
 			};
 
diff --git a/arch/arm64/boot/dts/qcom/msm8916-wiko-chuppito.dts b/arch/arm64/boot/dts/qcom/msm8916-wiko-chuppito.dts
new file mode 100644
index 0000000000000..262d9a959e6a8
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-wiko-chuppito.dts
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/dts-v1/;
+
+#include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/qcom,pmic-mpp.h>
+
+/ {
+	model = "Wiko Pulp 4G";
+	compatible = "wiko,chuppito", "qcom,msm8916";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &sdhc_1; /* eMMC */
+		mmc1 = &sdhc_2; /* SD card */
+		serial0 = &blsp_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0";
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pm8916_pwm 0 100000>;
+		brightness-levels = <0 255>;
+		num-interpolated-steps = <255>;
+		default-brightness-level = <255>;
+		enable-gpios = <&tlmm 119 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&button_backlight_default>;
+		pinctrl-names = "default";
+	};
+
+	gpio-hall-sensor {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&gpio_hall_sensor_default>;
+		pinctrl-names = "default";
+		label = "Hall Effect Sensor";
+
+		event-hall-sensor {
+			label = "Hall Effect Sensor";
+			gpios = <&tlmm 117 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			linux,can-disable;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&gpio_keys_default>;
+		pinctrl-names = "default";
+		label = "Buttons";
+
+		button-volume-up {
+			label = "Volume up";
+			gpios = <&tlmm 107 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	usb_id: usb-id {
+		compatible = "linux,extcon-usb-gpio";
+		id-gpios = <&tlmm 110 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&usb_id_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&blsp_i2c2 {
+	status = "okay";
+
+	magnetometer@c {
+		compatible = "asahi-kasei,ak09911";
+		reg = <0x0c>;
+		vdd-supply = <&pm8916_l17>;
+		vid-supply = <&pm8916_l6>;
+		reset-gpios = <&tlmm 120 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&mag_reset_default>;
+		pinctrl-names = "default";
+		mount-matrix = "1",  "0", "0",
+			       "0", "1", "0",
+			       "0",  "0", "1";
+	};
+
+	proximity@48 {
+		compatible = "sensortek,stk3310";
+		reg = <0x48>;
+		interrupts-extended = <&tlmm 113 IRQ_TYPE_EDGE_FALLING>;
+		pinctrl-0 = <&proximity_int_default>;
+		pinctrl-names = "default";
+	};
+
+	imu@68 {
+		compatible = "invensense,mpu6880";
+		reg = <0x68>;
+		interrupts-extended = <&tlmm 115 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&pm8916_l17>;
+		vddio-supply = <&pm8916_l6>;
+		pinctrl-0 = <&imu_int_default>;
+		pinctrl-names = "default";
+		mount-matrix = "0",  "-1", "0",
+			       "-1", "0", "0",
+			       "0",  "0", "-1";
+	};
+};
+
+&blsp_i2c5 {
+	status = "okay";
+
+	touchscreen@39 {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x39>;
+		interrupts-extended = <&tlmm 13 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&pm8916_l17>;
+		vio-supply = <&pm8916_l6>;
+		pinctrl-0 = <&touchscreen_default>;
+		pinctrl-names = "default";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		syna,startup-delay-ms = <100>;
+		syna,reset-delay-ms = <160>;
+
+		rmi4-f01@1 {
+			reg = <0x1>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f11@11 {
+			reg = <0x11>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
+
+&blsp_uart2 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5600000>;
+};
+
+&pm8916_codec {
+	qcom,hphl-jack-type-normally-open;
+};
+
+&pm8916_mpps {
+	pwm_out: mpp4-state {
+		pins = "mpp4";
+		function = "digital";
+		power-source = <PM8916_MPP_VPH>;
+		output-low;
+		qcom,dtest = <1>;
+	};
+};
+
+&pm8916_pwm {
+	pinctrl-0 = <&pwm_out>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pm8916_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&pm8916_rpm_regulators {
+	pm8916_l17: l17 {
+		regulator-min-microvolt = <2850000>;
+		regulator-max-microvolt = <2850000>;
+	};
+};
+
+&pm8916_vib {
+	status = "okay";
+};
+
+&sdhc_1 {
+	status = "okay";
+};
+
+&sdhc_2 {
+	pinctrl-0 = <&sdc2_default>, <&sdc2_cd_default>;
+	pinctrl-1 = <&sdc2_sleep>, <&sdc2_cd_default>;
+	pinctrl-names = "default", "sleep";
+	cd-gpios = <&tlmm 38 GPIO_ACTIVE_HIGH>;
+
+	status = "okay";
+};
+
+&sound {
+	audio-routing = "AMIC1", "MIC BIAS Internal1",
+		"AMIC2", "MIC BIAS Internal2";
+};
+
+&tlmm {
+	button_backlight_default: button-backlight-default-state {
+		pins = "gpio119";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	gpio_hall_sensor_default: gpio-hall-sensor-default-state {
+		pins = "gpio117";
+		function = "gpio";
+		drive-strength = <6>;
+		bias-pull-up;
+	};
+
+	gpio_keys_default: gpio-keys-default-state {
+		pins = "gpio107";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	imu_int_default: imu-int-default-state {
+		pins = "gpio115";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	mag_reset_default: mag-reset-default-state {
+		pins = "gpio120";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	proximity_int_default: proximity-int-default-state {
+		pins = "gpio113";
+		function = "gpio";
+		drive-strength = <6>;
+		bias-pull-up;
+	};
+
+	sdc2_cd_default: sdc2-cd-default-state {
+		pins = "gpio38";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	touchscreen_default: touchscreen-default-state {
+		touchscreen-pins {
+			pins = "gpio13";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		reset-pins {
+			pins = "gpio12";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	usb_id_default: usb-id-default-state {
+		pins = "gpio110";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+};
+
+&usb {
+	extcon = <&usb_id>, <&usb_id>;
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	extcon = <&usb_id>;
+};
+
+&venus {
+	status = "okay";
+};
+
+&venus_mem {
+	status = "okay";
+};
+
+&wcnss {
+	status = "okay";
+};
+
+&wcnss_iris {
+	compatible = "qcom,wcn3620";
+};
+
+&wcnss_mem {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts b/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts
index 91837ff940f1b..4f301e7c65171 100644
--- a/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts
+++ b/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts
@@ -178,7 +178,7 @@ &pmi8950_wled {
 	qcom,num-strings = <2>;
 	qcom,external-pfet;
 	qcom,current-limit-microamp = <20000>;
-	qcom,ovp-millivolt = <29600>;
+	qcom,ovp-millivolt = <29500>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts b/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
index ea90b00a2c8a4..1a340fa92755f 100644
--- a/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
+++ b/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
@@ -147,6 +147,17 @@ magnetometer@c {
 		pinctrl-names = "default";
 	};
 
+	light-sensor@60 {
+		compatible = "capella,cm36686", "vishay,vcnl4040";
+		reg = <0x60>;
+
+		interrupts-extended = <&tlmm 113 IRQ_TYPE_EDGE_FALLING>;
+		proximity-near-level = <30>;
+
+		pinctrl-0 = <&light_int_default>;
+		pinctrl-names = "default";
+	};
+
 	imu@68 {
 		compatible = "invensense,mpu6515";
 		reg = <0x68>;
@@ -330,4 +341,11 @@ mag_reset_default: mag-reset-default-state {
 		drive-strength = <2>;
 		bias-disable;
 	};
+
+	light_int_default: light-int-default-state {
+		pins = "gpio113";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-pull-up;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts
index ddd7af6167942..59f873a06e4dd 100644
--- a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts
+++ b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts
@@ -157,7 +157,7 @@ &pm8953_resin {
 
 &pmi8950_wled {
 	qcom,current-limit-microamp = <20000>;
-	qcom,num-strings = <2>;
+	qcom,num-strings = <3>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts
index d46325e799176..c2a290bf493c1 100644
--- a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts
+++ b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts
@@ -169,7 +169,7 @@ &pm8953_resin {
 
 &pmi8950_wled {
 	qcom,current-limit-microamp = <20000>;
-	qcom,ovp-millivolt = <29600>;
+	qcom,ovp-millivolt = <29500>;
 	qcom,num-strings = <2>;
 	qcom,external-pfet;
 	qcom,cabc;
diff --git a/arch/arm64/boot/dts/qcom/msm8996-v3.0.dtsi b/arch/arm64/boot/dts/qcom/msm8996-v3.0.dtsi
deleted file mode 100644
index 929bdcd45d02b..0000000000000
--- a/arch/arm64/boot/dts/qcom/msm8996-v3.0.dtsi
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause
-/*
- * Copyright (c) 2021, Konrad Dybcio <konrad.dybcio@somainline.org>
- */
-
-#include "msm8996.dtsi"
-
-/ {
-	qcom,msm-id = <246 0x30000>;
-};
-
- /*
-  * This revision seems to have differ GPU CPR
-  * parameters, GPU frequencies and some differences
-  * when it comes to voltage delivery to.. once again
-  * the GPU. Funnily enough, it's simpler to make it an
-  * overlay on top of 3.1 (the final one) than vice versa.
-  * The differences will show here as more and more
-  * features get enabled upstream.
-  */
-
-gpu_opp_table_3_0: opp-table-gpu30 {
-	compatible = "operating-points-v2";
-
-	opp-624000000 {
-		opp-hz = /bits/ 64 <624000000>;
-		opp-level = <7>;
-	};
-
-	opp-560000000 {
-		opp-hz = /bits/ 64 <560000000>;
-		opp-level = <6>;
-	};
-
-	opp-510000000 {
-		opp-hz = /bits/ 64 <510000000>;
-		opp-level = <5>;
-	};
-
-	opp-401800000 {
-		opp-hz = /bits/ 64 <401800000>;
-		opp-level = <4>;
-	};
-
-	opp-315000000 {
-		opp-hz = /bits/ 64 <315000000>;
-		opp-level = <3>;
-	};
-
-	opp-214000000 {
-		opp-hz = /bits/ 64 <214000000>;
-		opp-level = <3>;
-	};
-
-	opp-133000000 {
-		opp-hz = /bits/ 64 <133000000>;
-		opp-level = <3>;
-	};
-};
-
-&gpu {
-	operating-points-v2 = <&gpu_opp_table_3_0>;
-};
diff --git a/arch/arm64/boot/dts/qcom/pm6125.dtsi b/arch/arm64/boot/dts/qcom/pm6125.dtsi
index d0db28336fa9b..cb067adb7d174 100644
--- a/arch/arm64/boot/dts/qcom/pm6125.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm6125.dtsi
@@ -138,7 +138,6 @@ pm6125_rtc: rtc@6000 {
 			reg = <0x6000>, <0x6100>;
 			reg-names = "rtc", "alarm";
 			interrupts = <0x0 0x61 0x1 IRQ_TYPE_EDGE_RISING>;
-			status = "disabled";
 		};
 
 		pm6125_gpios: gpio@c000 {
diff --git a/arch/arm64/boot/dts/qcom/pmcx0102.dtsi b/arch/arm64/boot/dts/qcom/pmcx0102.dtsi
new file mode 100644
index 0000000000000..db2da9ef4f019
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmcx0102.dtsi
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pmcx0102-c0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmcx0102_c_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmcx0102-c1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmcx0102_c_e1_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmcx0102_d0_thermal: pmcx0102-d0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmcx0102_d_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmcx0102-d1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmcx0102_d_e1_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmcx0102_c_e0: pmic@2 {
+		compatible = "qcom,pmcx0102", "qcom,spmi-pmic";
+		reg = <0x2 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmcx0102_c_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x2 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmcx0102_c_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmcx0102_c_e0_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	pmcx0102_d_e0: pmic@3 {
+		compatible = "qcom,pmcx0102", "qcom,spmi-pmic";
+		reg = <0x3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmcx0102_d_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x3 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmcx0102_d_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmcx0102_d_e0_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
+
+&spmi_bus1 {
+	pmcx0102_c_e1: pmic@2 {
+		compatible = "qcom,pmcx0102", "qcom,spmi-pmic";
+		reg = <0x2 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmcx0102_c_e1_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x2 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmcx0102_c_e1_gpios: gpio@8800 {
+			compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmcx0102_c_e1_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	pmcx0102_d_e1: pmic@3 {
+		compatible = "qcom,pmcx0102", "qcom,spmi-pmic";
+		reg = <0x3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmcx0102_d_e1_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x3 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmcx0102_d_e1_gpios: gpio@8800 {
+			compatible = "qcom,pmcx0102-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmcx0102_d_e1_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmh0101.dtsi b/arch/arm64/boot/dts/qcom/pmh0101.dtsi
new file mode 100644
index 0000000000000..b1ec413259583
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmh0101.dtsi
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pmh0101-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0101_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmic@1 {
+		compatible = "qcom,pmh0101", "qcom,spmi-pmic";
+		reg = <0x1 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0101_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x1 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0101_gpios: gpio@8800 {
+			compatible = "qcom,pmh0101-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0101_gpios 0 0 18>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		pmh0101_flash: led-controller@ee00 {
+			compatible = "qcom,pmh0101-flash-led", "qcom,spmi-flash-led";
+			reg = <0xee00>;
+			status = "disabled";
+		};
+
+		pmh0101_pwm: pwm {
+			compatible = "qcom,pmh0101-pwm", "qcom,pm8350c-pwm";
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
new file mode 100644
index 0000000000000..7a1e5f355c175
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/{
+	thermal_zones {
+		pmh0104_i0_thermal: pmh0104-i0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0104_i_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmh0104_j0_thermal: pmh0104-j0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0104_j_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmh0104-l1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0104_l_e1_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmh0104_i_e0: pmic@8 {
+		compatible = "qcom,pmh0104", "qcom,spmi-pmic";
+		reg = <0x8 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0104_i_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x8 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0104_i_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0104_i_e0_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	pmh0104_j_e0: pmic@9 {
+		compatible = "qcom,pmh0104", "qcom,spmi-pmic";
+		reg = <0x9 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0104_j_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x9 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0104_j_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0104_j_e0_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
+
+&spmi_bus1 {
+	pmh0104_l_e1: pmic@b {
+		compatible = "qcom,pmh0104", "qcom,spmi-pmic";
+		reg = <0xb SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0104_l_e1_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0xb 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0104_l_e1_gpios: gpio@8800 {
+			compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0104_l_e1_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmh0110-glymur.dtsi b/arch/arm64/boot/dts/qcom/pmh0110-glymur.dtsi
new file mode 100644
index 0000000000000..7655bc0303482
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmh0110-glymur.dtsi
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pmh0110-f0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0110_f_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmh0110-f1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0110_f_e1_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		pmh0110-h0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmh0110_h_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmh0110_f_e0: pmic@5 {
+		compatible = "qcom,pmh0110", "qcom,spmi-pmic";
+		reg = <0x5 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0110_f_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x5 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0110_f_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0110_f_e0_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	pmh0110_h_e0: pmic@7 {
+		compatible = "qcom,pmh0110", "qcom,spmi-pmic";
+		reg = <0x7 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0110_h_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x7 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0110_h_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0110_h_e0_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
+
+&spmi_bus1 {
+	pmh0110_f_e1: pmic@5 {
+		compatible = "qcom,pmh0110", "qcom,spmi-pmic";
+		reg = <0x5 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0110_f_e1_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x5 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0110_f_e1_gpios: gpio@8800 {
+			compatible = "qcom,pmh0110-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0110_f_e1_gpios 0 0 14>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmk8550.dtsi b/arch/arm64/boot/dts/qcom/pmk8550.dtsi
index 583f61fc16ad5..3049eb6b46d7a 100644
--- a/arch/arm64/boot/dts/qcom/pmk8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmk8550.dtsi
@@ -73,5 +73,15 @@ pmk8550_gpios: gpio@b800 {
 			interrupt-controller;
 			#interrupt-cells = <2>;
 		};
+
+		pmk8550_pwm: pwm {
+			compatible = "qcom,pmk8550-pwm";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#pwm-cells = <2>;
+
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/pmk8850.dtsi b/arch/arm64/boot/dts/qcom/pmk8850.dtsi
new file mode 100644
index 0000000000000..c7ba72fd48bc8
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmk8850.dtsi
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+&spmi_bus0 {
+	pmic@0 {
+		compatible = "qcom,pmk8850", "qcom,spmi-pmic";
+		reg = <0x0 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmk8850_pon: pon@1300 {
+			compatible = "qcom,pmk8350-pon";
+			reg = <0x1300>,
+			      <0x800>;
+			reg-names = "hlos",
+				    "pbs";
+
+			pon_pwrkey: pwrkey {
+				compatible = "qcom,pmk8350-pwrkey";
+				interrupts = <0x0 0x13 0x7 IRQ_TYPE_EDGE_BOTH>;
+				linux,code = <KEY_POWER>;
+			};
+
+			pon_resin: resin {
+				compatible = "qcom,pmk8350-resin";
+				interrupts = <0x0 0x13 0x6 IRQ_TYPE_EDGE_BOTH>;
+				status = "disabled";
+			};
+		};
+
+		pmk8850_gpios: gpio@b800 {
+			compatible = "qcom,pmk8850-gpio", "qcom,spmi-gpio";
+			reg = <0xb800>;
+			gpio-controller;
+			gpio-ranges = <&pmk8850_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		pmk8850_rtc: rtc@6100 {
+			compatible = "qcom,pmk8350-rtc";
+			reg = <0x6100>,
+			      <0x6200>;
+			reg-names = "rtc",
+				    "alarm";
+			interrupts = <0x0 0x62 0x1 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		pmk8850_sdam_2: nvram@7100 {
+			compatible = "qcom,spmi-sdam";
+			reg = <0x7100>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x7100 0x100>;
+
+			reboot_reason: reboot-reason@48 {
+				reg = <0x48 0x1>;
+				bits = <1 7>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
new file mode 100644
index 0000000000000..fe539b1f45677
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
@@ -0,0 +1,1549 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include "purwa-iot-som.dtsi"
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+/ {
+	model = "Qualcomm Technologies, Inc. Purwa IoT EVK";
+	compatible = "qcom,purwa-iot-evk", "qcom,purwa-iot-som", "qcom,x1p42100";
+	chassis-type = "embedded";
+
+	aliases {
+		serial0 = &uart21;
+		serial1 = &uart14;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pmk8550_pwm 0 5000000>;
+		enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>;
+		power-supply = <&vreg_edp_bl>;
+
+		pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>;
+		pinctrl-names = "default";
+	};
+
+	wcd938x: audio-codec {
+		compatible = "qcom,wcd9385-codec";
+
+		pinctrl-0 = <&wcd_default>;
+		pinctrl-names = "default";
+
+		reset-gpios = <&tlmm 191 GPIO_ACTIVE_LOW>;
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000
+							 500000 500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		vdd-buck-supply = <&vreg_l15b_1p8>;
+		vdd-rxtx-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob1>;
+
+		#sound-dai-cells = <1>;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	connector3 {
+		compatible = "usb-a-connector";
+		label = "USB-3-Type-A";
+		power-role = "source";
+
+		vbus-supply = <&regulator_usb3_vbus>;
+
+		port {
+			connector_3_in: endpoint {
+			};
+		};
+	};
+
+	connector6 {
+		compatible = "usb-a-connector";
+		label = "USB-6-Type-A";
+		power-role = "source";
+
+		vbus-supply = <&regulator_usb6_vbus>;
+
+		port {
+			connector_4_in: endpoint {
+			};
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,x1e80100-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 123 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 125 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss0_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss0_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss0_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss0_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss0_con_sbu_out>;
+					};
+				};
+			};
+		};
+
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss1_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss1_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss1_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss1_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss1_con_sbu_out>;
+					};
+				};
+			};
+		};
+
+		connector@2 {
+			compatible = "usb-c-connector";
+			reg = <2>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss2_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss2_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss2_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss2_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss2_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss2_con_sbu_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_edp_bl: regulator-edp-bl {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VBL9";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+
+		gpio = <&pmc8380_3_gpios 10 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_bl_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_pcie_12v: regulator-pcie-12v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_PCIE_12V";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+
+		gpio = <&pm8550ve_8_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&pcie_x8_12v>;
+		pinctrl-names = "default";
+	};
+
+	vreg_pcie_3v3: regulator-pcie-3v3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_PCIE_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmc8380_3_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&pm_sde7_main_3p3_en>;
+		pinctrl-names = "default";
+	};
+
+	vreg_pcie_3v3_aux: regulator-pcie-3v3-aux {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_PCIE_3P3_AUX";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmc8380_3_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&pm_sde7_aux_3p3_en>;
+		pinctrl-names = "default";
+	};
+
+	/* Left unused as the retimer is not used on this board. */
+	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&pmc8380_5_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_1p8: regulator-rtmr0-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&pm8550ve_9_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_3p3: regulator-rtmr0-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pm8550_gpios 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_1p15: regulator-rtmr1-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&tlmm 188 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_1p8: regulator-rtmr1-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&tlmm 175 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_3p3: regulator-rtmr1-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 186 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr2_1p15: regulator-rtmr2-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR2_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&tlmm 189 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb2_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr2_1p8: regulator-rtmr2-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR2_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&tlmm 126 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb2_pwr_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr2_3p3: regulator-rtmr2-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR2_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 187 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb2_pwr_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	regulator_usb3_vbus: regulator-usb3-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "USB3_VBUS";
+		gpio = <&pm8550ve_9_gpios 4 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&usb3_en>;
+		pinctrl-names = "default";
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	regulator_usb6_vbus: regulator-usb6-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "USB6_VBUS";
+		gpio = <&pm8550ve_9_gpios 5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&usb6_en>;
+		pinctrl-names = "default";
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	/*
+	 * TODO: These two regulators are actually part of the removable M.2
+	 * card and not the EVK mainboard. Need to describe this differently.
+	 * Functionally it works correctly, because all we need to do is to
+	 * turn on the actual 3.3V supply above.
+	 */
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_1p9: regulator-wcn-1p9 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_1P9";
+		regulator-min-microvolt = <1900000>;
+		regulator-max-microvolt = <1900000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_wwan: regulator-wwan {
+		compatible = "regulator-fixed";
+
+		regulator-name = "SDX_VPH_PWR";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 221 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wwan_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	sound {
+		compatible = "qcom,x1e80100-sndcard";
+		model = "X1E80100-EVK";
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK2 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"IN1_HPHL", "HPHL_OUT",
+				"IN2_HPHR", "HPHR_OUT",
+				"AMIC2", "MIC BIAS2",
+				"VA DMIC0", "MIC BIAS3",
+				"VA DMIC1", "MIC BIAS3",
+				"VA DMIC2", "MIC BIAS1",
+				"VA DMIC3", "MIC BIAS1",
+				"TX SWR_INPUT1", "ADC2_OUTPUT";
+
+		wcd-playback-dai-link {
+			link-name = "WCD Playback";
+
+			codec {
+				sound-dai = <&wcd938x 0>, <&swr1 0>, <&lpass_rxmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-capture-dai-link {
+			link-name = "WCD Capture";
+
+			codec {
+				sound-dai = <&wcd938x 1>, <&swr2 1>, <&lpass_txmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_woofer>,
+					    <&left_tweeter>,
+					    <&swr0 0>,
+					    <&lpass_wsamacro 0>,
+					    <&right_woofer>,
+					    <&right_tweeter>,
+					    <&swr3 0>,
+					    <&lpass_wsa2macro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_wcn_0p95>;
+		vdddig-supply = <&vreg_wcn_0p95>;
+		vddrfa1p2-supply = <&vreg_wcn_1p9>;
+		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x08>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK5>;
+
+		vdd-supply = <&vreg_rtmr2_1p15>;
+		vdd33-supply = <&vreg_rtmr2_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr2_3p3>;
+		vddar-supply = <&vreg_rtmr2_1p15>;
+		vddat-supply = <&vreg_rtmr2_1p15>;
+		vddio-supply = <&vreg_rtmr2_1p8>;
+
+		reset-gpios = <&tlmm 185 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr2_default>;
+		pinctrl-names = "default";
+
+		orientation-switch;
+		retimer-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss2_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss2_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss2_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss2_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss2_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss2_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x8>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK4>;
+
+		vdd-supply = <&vreg_rtmr0_1p15>;
+		vdd33-supply = <&vreg_rtmr0_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr0_3p3>;
+		vddar-supply = <&vreg_rtmr0_1p15>;
+		vddat-supply = <&vreg_rtmr0_1p15>;
+		vddio-supply = <&vreg_rtmr0_1p8>;
+
+		reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr0_default>;
+		pinctrl-names = "default";
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss0_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss0_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss0_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	eusb3_repeater: redriver@47 {
+		compatible = "nxp,ptn3222";
+		reg = <0x47>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb3_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb5_repeater: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 7 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb5_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb6_repeater: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb6_reset_n>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c7 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x8>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK4>;
+
+		vdd-supply = <&vreg_rtmr1_1p15>;
+		vdd33-supply = <&vreg_rtmr1_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr1_3p3>;
+		vddar-supply = <&vreg_rtmr1_1p15>;
+		vddat-supply = <&vreg_rtmr1_1p15>;
+		vddio-supply = <&vreg_rtmr1_1p8>;
+
+		reset-gpios = <&tlmm 176 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr1_default>;
+		pinctrl-names = "default";
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss1_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss1_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss1_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss1_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss1_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss1_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&lpass_tlmm {
+	spkr_0_sd_n_active: spkr-0-sd-n-active-state {
+		pins = "gpio12";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	spkr_1_sd_n_active: spkr-1-sd-n-active-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	spkr_2_sd_n_active: spkr-2-sd-n-active-state {
+		pins = "gpio17";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	spkr_3_sd_n_active: spkr-3-sd-n-active-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
+	pinctrl-names = "default";
+
+	vdd-micb-supply = <&vreg_l1b_1p8>;
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "edp-panel";
+
+			backlight = <&backlight>;
+			power-supply = <&vreg_edp_3p3>;
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l3j_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pcie3_port0 {
+	vpcie12v-supply = <&vreg_pcie_12v>;
+	vpcie3v3-supply = <&vreg_pcie_3v3>;
+	vpcie3v3aux-supply = <&vreg_pcie_3v3_aux>;
+
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie5 {
+	vddpe-3v3-supply = <&vreg_wwan>;
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+};
+
+&pcie6a {
+	vddpe-3v3-supply = <&vreg_nvme>;
+};
+
+&pcie6a_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+&pm8550_gpios {
+	rtmr0_default: rtmr0-reset-n-active-state {
+		pins = "gpio10";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+	usb0_3p3_reg_en: usb0-3p3-reg-en-state {
+		pins = "gpio11";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pm8550ve_8_gpios {
+	pcie_x8_12v: pcie-12v-default-state {
+		pins = "gpio8";
+		function = "normal";
+		output-enable;
+		output-high;
+		bias-pull-down;
+		power-source = <0>;
+	};
+};
+
+&pm8550ve_9_gpios {
+	usb0_1p8_reg_en: usb0-1p8-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+	usb3_en: usb3-en-state {
+		pins = "gpio4";
+		function = "normal";
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_HIGH>;
+		output-enable;
+		power-source = <0>;
+	};
+
+	usb6_en: usb6-en-state {
+		pins = "gpio5";
+		function = "normal";
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_HIGH>;
+		output-enable;
+		power-source = <0>;
+	};
+};
+
+&pm8550_pwm {
+	status = "okay";
+
+	multi-led {
+		color = <LED_COLOR_ID_MULTI>;
+		function = LED_FUNCTION_STATUS;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_RED>;
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+	};
+};
+
+&pmc8380_3_gpios {
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio4";
+		function = "normal";
+		power-source = <1>;
+		input-disable;
+		output-enable;
+	};
+
+	edp_bl_reg_en: edp-bl-reg-en-state {
+		pins = "gpio10";
+		function = "normal";
+	};
+
+	pm_sde7_aux_3p3_en: pcie-aux-3p3-default-state {
+		pins = "gpio8";
+		function = "normal";
+		output-enable;
+		bias-pull-down;
+		power-source = <0>;
+	};
+
+	pm_sde7_main_3p3_en: pcie-main-3p3-default-state {
+		pins = "gpio6";
+		function = "normal";
+		output-enable;
+		bias-pull-down;
+		power-source = <0>;
+	};
+};
+
+&pmc8380_5_gpios {
+	usb0_pwr_1p15_reg_en: usb0-pwr-1p15-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pmk8550_gpios {
+	edp_bl_pwm: edp-bl-pwm-state {
+		pins = "gpio5";
+		function = "func3";
+	};
+};
+
+&pmk8550_pwm {
+	status = "okay";
+};
+
+&smb2360_0 {
+	status = "okay";
+};
+
+&smb2360_0_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l2b_3p0>;
+};
+
+&smb2360_1 {
+	status = "okay";
+};
+
+&smb2360_1_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l14b_3p0>;
+};
+
+&smb2360_2 {
+	status = "okay";
+};
+
+&smb2360_2_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l8b_3p0>;
+};
+
+&spi11 {
+	status = "okay";
+
+	tpm@0 {
+		compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&swr0 {
+	status = "okay";
+
+	pinctrl-0 = <&wsa_swr_active>;
+	pinctrl-names = "default";
+
+	/* WSA8845, Left Woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		pinctrl-0 = <&spkr_0_sd_n_active>;
+		pinctrl-names = "default";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Left Tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		pinctrl-0 = <&spkr_1_sd_n_active>;
+		pinctrl-names = "default";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&swr1 {
+	status = "okay";
+
+	/* WCD9385 RX */
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr2 {
+	status = "okay";
+
+	/* WCD9385 TX */
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		qcom,tx-port-mapping = <2 2 3 4>;
+	};
+};
+
+&swr3 {
+	status = "okay";
+
+	pinctrl-0 = <&wsa2_swr_active>;
+	pinctrl-names = "default";
+
+	/* WSA8845, Right Woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		pinctrl-0 = <&spkr_2_sd_n_active>;
+		pinctrl-names = "default";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 17 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Right Tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		pinctrl-0 = <&spkr_3_sd_n_active>;
+		pinctrl-names = "default";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 18 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&tlmm {
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	eusb3_reset_n: eusb3-reset-n-state {
+		pins = "gpio6";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+		output-low;
+	};
+
+	eusb5_reset_n: eusb5-reset-n-state {
+		pins = "gpio7";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+		output-low;
+	};
+
+	eusb6_reset_n: eusb6-reset-n-state {
+		pins = "gpio184";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+		output-low;
+	};
+
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	rtmr1_default: rtmr1-reset-n-active-state {
+		pins = "gpio176";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	rtmr2_default: rtmr2-reset-n-active-state {
+		pins = "gpio185";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb1_pwr_1p15_reg_en: usb1-pwr-1p15-reg-en-state {
+		pins = "gpio188";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb1_pwr_1p8_reg_en: usb1-pwr-1p8-reg-en-state {
+		pins = "gpio175";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb1_pwr_3p3_reg_en: usb1-pwr-3p3-reg-en-state {
+		pins = "gpio186";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb2_pwr_1p15_reg_en: usb2-pwr-1p15-reg-en-state {
+		pins = "gpio189";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb2_pwr_1p8_reg_en: usb2-pwr-1p8-reg-en-state {
+		pins = "gpio126";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb2_pwr_3p3_reg_en: usb2-pwr-3p3-reg-en-state {
+		pins = "gpio187";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb_1_ss0_sbu_default: usb-1-ss0-sbu-state {
+		mode-pins {
+			pins = "gpio166";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+			output-high;
+		};
+
+		oe-n-pins {
+			pins = "gpio168";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+
+		sel-pins {
+			pins = "gpio167";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+	};
+
+	wcd_default: wcd-reset-n-active-state {
+		pins = "gpio191";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+		output-low;
+	};
+
+	wcn_bt_en: wcn-bt-en-state {
+		pins = "gpio116";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_en: wcn-wlan-en-state {
+		pins = "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wwan_sw_en: wwan-sw-en-state {
+		pins = "gpio221";
+		function = "gpio";
+		drive-strength = <4>;
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio214";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	/* Switches USB signal routing between the USB connector and the Wi-Fi card. */
+	wcn_usb_sw_n: wcn-usb-sw-n-state {
+		pins = "gpio225";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+		output-high;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&uart21 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss0_hs_in>;
+};
+
+&usb_1_ss0_hsphy {
+	phys = <&smb2360_0_eusb2_repeater>;
+};
+
+&usb_1_ss0_qmpphy_out {
+	remote-endpoint = <&retimer_ss0_ss_in>;
+};
+
+&usb_1_ss1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss1_hs_in>;
+};
+
+&usb_1_ss1_hsphy {
+	phys = <&smb2360_1_eusb2_repeater>;
+};
+
+&usb_1_ss1_qmpphy_out {
+	remote-endpoint = <&retimer_ss1_ss_in>;
+};
+
+&usb_1_ss2_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss2_hs_in>;
+};
+
+&usb_1_ss2_hsphy {
+	phys = <&smb2360_2_eusb2_repeater>;
+};
+
+&usb_1_ss2_qmpphy_out {
+	remote-endpoint = <&retimer_ss2_ss_in>;
+};
+
+&usb_2_hsphy {
+	phys = <&eusb5_repeater>;
+
+	pinctrl-0 = <&wcn_usb_sw_n>;
+	pinctrl-names = "default";
+};
+
+&usb_mp_hsphy0 {
+	phys = <&eusb3_repeater>;
+};
+
+&usb_mp_hsphy1 {
+	phys = <&eusb6_repeater>;
+};
diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
new file mode 100644
index 0000000000000..fb90beb1096f6
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
@@ -0,0 +1,685 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "purwa.dtsi"
+#include "hamoa-pmics.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+/delete-node/ &pmc8380_6;
+/delete-node/ &pmc8380_6_thermal;
+
+/ {
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+};
+
+&apps_rsc {
+	/* PMC8380C_B */
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob2>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s5j_1p2>;
+		vdd-l15-supply = <&vreg_s4c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p0: ldo5 {
+			regulator-name = "vreg_l5b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p8: ldo7 {
+			regulator-name = "vreg_l7b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_3p0: ldo8 {
+			regulator-name = "vreg_l8b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p2: ldo12 {
+			regulator-name = "vreg_l12b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p0: ldo14 {
+			regulator-name = "vreg_l14b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l16b_2p9: ldo16 {
+			regulator-name = "vreg_l16b_2p9";
+			regulator-min-microvolt = <2912000>;
+			regulator-max-microvolt = <2912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380VE_C */
+	regulators-1 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c_1p2: ldo1 {
+			regulator-name = "vreg_l1c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_0p8: ldo2 {
+			regulator-name = "vreg_l2c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_0p8: ldo3 {
+			regulator-name = "vreg_l3c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380_D */
+	regulators-2 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s4c_1p8>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_l1d_0p8: ldo1 {
+			regulator-name = "vreg_l1d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3d_1p8: ldo3 {
+			regulator-name = "vreg_l3d_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380_E */
+	regulators-3 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+
+		vreg_l2e_0p8: ldo2 {
+			regulator-name = "vreg_l2e_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380_F */
+	regulators-4 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_s1f_0p7: smps1 {
+			regulator-name = "vreg_s1f_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_1p0: ldo1 {
+			regulator-name = "vreg_l1f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_1p0: ldo2 {
+			regulator-name = "vreg_l2f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_1p0: ldo3 {
+			regulator-name = "vreg_l3f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380VE_I */
+	regulators-6 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "i";
+
+		vdd-l1-supply = <&vreg_s4c_1p8>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+
+		vreg_s1i_0p9: smps1 {
+			regulator-name = "vreg_s1i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2i_1p0: smps2 {
+			regulator-name = "vreg_s2i_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1i_1p8: ldo1 {
+			regulator-name = "vreg_l1i_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMC8380VE_J */
+	regulators-7 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "j";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_s5j_1p2: smps5 {
+			regulator-name = "vreg_s5j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1j_0p8: ldo1 {
+			regulator-name = "vreg_l1j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2j_1p2: ldo2 {
+			regulator-name = "vreg_l2j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3j_0p8: ldo3 {
+			regulator-name = "vreg_l3j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/x1p42100/gen71500_zap.mbn";
+};
+
+&pcie3 {
+	pinctrl-0 = <&pcie3_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie3_phy {
+	vdda-phy-supply = <&vreg_l3c_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie5 {
+	pinctrl-0 = <&pcie5_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie5_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie6a {
+	pinctrl-0 = <&pcie6a_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6a_phy {
+	vdda-phy-supply = <&vreg_l1d_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&qupv3_0 {
+	status = "okay";
+};
+
+&qupv3_1 {
+	status = "okay";
+};
+
+&qupv3_2 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/x1e80100/adsp.mbn",
+			"qcom/x1e80100/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/x1e80100/cdsp.mbn",
+			"qcom/x1e80100/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <34 2>; /* TPM LP & INT */
+
+	pcie3_default: pcie3-default-state {
+		clkreq-n-pins {
+			pins = "gpio144";
+			function = "pcie3_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio143";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio145";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie5_default: pcie5-default-state {
+		clkreq-n-pins {
+			pins = "gpio150";
+			function = "pcie5_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio149";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio151";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6a_default: pcie6a-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie6a_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+
+		};
+	};
+};
+
+&usb_1_ss0 {
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3 {
+	dr_mode = "otg";
+	usb-role-switch;
+};
+
+&usb_1_ss0_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l1j_0p8>;
+
+	status = "okay";
+};
+
+&usb_1_ss1 {
+	status = "okay";
+};
+
+&usb_1_ss1_dwc3 {
+	dr_mode = "otg";
+	usb-role-switch;
+};
+
+&usb_1_ss1_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss1_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	status = "okay";
+};
+
+&usb_1_ss2 {
+	status = "okay";
+};
+
+&usb_1_ss2_dwc3 {
+	dr_mode = "otg";
+	usb-role-switch;
+};
+
+&usb_1_ss2_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss2_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	status = "okay";
+};
+
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
index 089a027c57d5c..bdc02260f902b 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
@@ -36,6 +36,7 @@ / {
 
 	aliases {
 		serial0 = &uart5;
+		serial1 = &uart7;
 	};
 
 	pm8350c_pwm_backlight: backlight {
@@ -177,7 +178,7 @@ wcd9370: audio-codec-0 {
 		pinctrl-0 = <&wcd_default>;
 		pinctrl-names = "default";
 
-		reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&tlmm 83 GPIO_ACTIVE_LOW>;
 
 		vdd-buck-supply = <&vreg_l17b_1p7>;
 		vdd-rxtx-supply = <&vreg_l18b_1p8>;
@@ -194,6 +195,63 @@ wcd9370: audio-codec-0 {
 
 		#sound-dai-cells = <1>;
 	};
+
+	wcn6750-pmu {
+		compatible = "qcom,wcn6750-pmu";
+		pinctrl-0 = <&bt_en>;
+		pinctrl-names = "default";
+		vddaon-supply = <&vreg_s7b_0p972>;
+		vddasd-supply = <&vreg_l11c_2p8>;
+		vddpmu-supply = <&vreg_s7b_0p972>;
+		vddrfa0p8-supply = <&vreg_s7b_0p972>;
+		vddrfa1p2-supply = <&vreg_s8b_1p272>;
+		vddrfa1p7-supply = <&vreg_s1b_1p872>;
+		vddrfa2p2-supply = <&vreg_s1c_2p19>;
+
+		bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -695,6 +753,39 @@ &pon_resin {
 	status = "okay";
 };
 
+&qup_uart7_cts {
+	/*
+	 * Configure a bias-bus-hold on CTS to lower power
+	 * usage when Bluetooth is turned off. Bus hold will
+	 * maintain a low power state regardless of whether
+	 * the Bluetooth module drives the pin in either
+	 * direction or leaves the pin fully unpowered.
+	 */
+	bias-bus-hold;
+};
+
+&qup_uart7_rts {
+	/* We'll drive RTS, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart7_rx {
+	/*
+	 * Configure a pull-up on RX. This is needed to avoid
+	 * garbage data when the TX pin of the Bluetooth module is
+	 * in tri-state (module powered off or not driving the
+	 * signal yet).
+	 */
+	bias-pull-up;
+};
+
+&qup_uart7_tx {
+	/* We'll drive TX, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
 &qupv3_id_0 {
 	status = "okay";
 };
@@ -920,6 +1011,59 @@ &tlmm {
 	gpio-reserved-ranges = <32 2>, /* ADSP */
 			       <48 4>; /* NFC */
 
+	bt_en: bt-en-state {
+		pins = "gpio85";
+		function = "gpio";
+		output-low;
+		bias-disable;
+	};
+
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-state {
+		pins = "gpio28";
+		function = "gpio";
+		/*
+		 * Configure a bias-bus-hold on CTS to lower power
+		 * usage when Bluetooth is turned off. Bus hold will
+		 * maintain a low power state regardless of whether
+		 * the Bluetooth module drives the pin in either
+		 * direction or leaves the pin fully unpowered.
+		 */
+		bias-bus-hold;
+	};
+
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-state {
+		pins = "gpio29";
+		function = "gpio";
+		/*
+		 * Configure pull-down on RTS. As RTS is active low
+		 * signal, pull it low to indicate the BT SoC that it
+		 * can wakeup the system anytime from suspend state by
+		 * pulling RX low (by sending wakeup bytes).
+		 */
+		bias-pull-down;
+	};
+
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-state {
+		pins = "gpio31";
+		function = "gpio";
+		/*
+		 * Configure a pull-up on RX. This is needed to avoid
+		 * garbage data when the TX pin of the Bluetooth module
+		 * is floating which may cause spurious wakeups.
+		 */
+		bias-pull-up;
+	};
+
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-state {
+		pins = "gpio30";
+		function = "gpio";
+		/*
+		 * Configure pull-up on TX when it isn't actively driven
+		 * to prevent BT SoC from receiving garbage during sleep.
+		 */
+		bias-pull-up;
+	};
+
 	sd_cd: sd-cd-state {
 		pins = "gpio91";
 		function = "gpio";
@@ -938,6 +1082,31 @@ &uart5 {
 	status = "okay";
 };
 
+&uart7 {
+	/delete-property/ interrupts;
+	interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 31 IRQ_TYPE_EDGE_FALLING>;
+	pinctrl-1 = <&qup_uart7_sleep_cts>,
+		    <&qup_uart7_sleep_rts>,
+		    <&qup_uart7_sleep_tx>,
+		    <&qup_uart7_sleep_rx>;
+	pinctrl-names = "default",
+			"sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6750-bt";
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		max-speed = <3200000>;
+	};
+};
+
 &ufs_mem_hc {
 	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
 	vcc-supply = <&vreg_l7b_2p952>;
diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride.dts b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
index 5a24c19c415e3..4f5dca49e15a3 100644
--- a/arch/arm64/boot/dts/qcom/qcs615-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
@@ -655,11 +655,9 @@ &usb_qmpphy_2 {
 };
 
 &usb_1 {
-	status = "okay";
-};
-
-&usb_1_dwc3 {
 	dr_mode = "peripheral";
+
+	status = "okay";
 };
 
 &usb_hsphy_2 {
@@ -671,11 +669,9 @@ &usb_hsphy_2 {
 };
 
 &usb_2 {
-	status = "okay";
-};
-
-&usb_2_dwc3 {
 	dr_mode = "host";
+
+	status = "okay";
 };
 
 &ufs_mem_hc {
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
index 619a42b5ef48d..83908db335afa 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
@@ -5,9 +5,37 @@
 
 /dts-v1/;
 /plugin/;
+#include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/qcom,gcc-sc7280.h>
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 
+&{/} {
+
+	vreg_0p9: regulator-0v9 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_0P9";
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_1p8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&remoteproc_wpss {
+       status = "disabled";
+};
+
 &spi11 {
 	#address-cells = <1>;
 	#size-cells = <0>;
@@ -19,3 +47,244 @@ st33htpm0: tpm@0 {
 		spi-max-frequency = <20000000>;
 	};
 };
+
+&pcie0 {
+	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie0_reset_n>, <&pcie0_wake_n>, <&pcie0_clkreq_n>;
+	pinctrl-names = "default";
+
+	iommu-map = <0x0 &apps_smmu 0x1c00 0x1>,
+		    <0x100 &apps_smmu 0x1c01 0x1>,
+		    <0x208 &apps_smmu 0x1c04 0x1>,
+		    <0x210 &apps_smmu 0x1c05 0x1>,
+		    <0x218 &apps_smmu 0x1c06 0x1>,
+		    <0x300 &apps_smmu 0x1c07 0x1>,
+		    <0x400 &apps_smmu 0x1c08 0x1>,
+		    <0x500 &apps_smmu 0x1c09 0x1>,
+		    <0x501 &apps_smmu 0x1c10 0x1>;
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&pcie0_port {
+	#address-cells = <3>;
+	#size-cells = <2>;
+
+	pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vreg_0p9>;
+		vdd18-supply = <&vreg_1p8>;
+		vdd09-supply = <&vreg_0p9>;
+		vddio1-supply = <&vreg_1p8>;
+		vddio2-supply = <&vreg_1p8>;
+		vddio18-supply = <&vreg_1p8>;
+
+		i2c-parent = <&i2c1 0x33>;
+
+		resx-gpios = <&tlmm 78 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&pcie0_tc9563_resx_n>;
+		pinctrl-names = "default";
+
+		pcie@1,0 {
+			reg = <0x20800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x21000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+		};
+
+		pcie@3,0 {
+			reg = <0x21800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+
+			pci@0,0 {
+				reg = <0x50000 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+
+			pci@0,1 {
+				reg = <0x50100 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+		};
+
+	};
+};
+
+&pcie1 {
+	iommu-map = <0x0 &apps_smmu 0x1c80 0x1>,
+		    <0x100 &apps_smmu 0x1c81 0x1>,
+		    <0x208 &apps_smmu 0x1c84 0x1>,
+		    <0x210 &apps_smmu 0x1c85 0x1>,
+		    <0x218 &apps_smmu 0x1c86 0x1>,
+		    <0x300 &apps_smmu 0x1c87 0x1>,
+		    <0x408 &apps_smmu 0x1c90 0x1>,
+		    <0x410 &apps_smmu 0x1c91 0x1>,
+		    <0x418 &apps_smmu 0x1c92 0x1>,
+		    <0x500 &apps_smmu 0x1c93 0x1>,
+		    <0x600 &apps_smmu 0x1c94 0x1>,
+		    <0x700 &apps_smmu 0x1c95 0x1>,
+		    <0x701 &apps_smmu 0x1c96 0x1>,
+		    <0x800 &apps_smmu 0x1c97 0x1>,
+		    <0x900 &apps_smmu 0x1c98 0x1>,
+		    <0x901 &apps_smmu 0x1c99 0x1>;
+};
+
+&pcie1_switch0_dsp1 {
+	#address-cells = <3>;
+	#size-cells = <2>;
+
+	pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x30000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vdd_ntn_0p9>;
+		vdd18-supply = <&vdd_ntn_1p8>;
+		vdd09-supply = <&vdd_ntn_0p9>;
+		vddio1-supply = <&vdd_ntn_1p8>;
+		vddio2-supply = <&vdd_ntn_1p8>;
+		vddio18-supply = <&vdd_ntn_1p8>;
+
+		i2c-parent = <&i2c1 0x77>;
+
+		resx-gpios = <&tlmm 124 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&pcie1_tc9563_resx_n>;
+		pinctrl-names = "default";
+
+		pcie@1,0 {
+			reg = <0x40800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x41000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+		};
+
+		pcie@3,0 {
+			reg = <0x41800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+
+			pci@0,0 {
+				reg = <0x50000 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+
+			pci@0,1 {
+				reg = <0x50100 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+		};
+	};
+};
+
+&tlmm {
+	pcie0_tc9563_resx_n: pcie0-tc9563-resx-state {
+		pins = "gpio78";
+		function = "gpio";
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+	pcie0_reset_n: pcie0-reset-n-state {
+		pins = "gpio87";
+		function = "gpio";
+		drive-strength = <16>;
+		output-low;
+		bias-disable;
+	};
+
+	pcie0_clkreq_n: pcie0-clkreq-n-state {
+		pins = "gpio88";
+		function = "pcie0_clkreqn";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	pcie0_wake_n: pcie0-wake-n-state {
+		pins = "gpio89";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	pcie1_tc9563_resx_n: pcie1-tc9563-resx-state {
+		pins = "gpio124";
+		function = "gpio";
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+};
+
+&wifi {
+       status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
index e3d2f01881ae0..cd54525e45e0f 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
@@ -852,7 +852,7 @@ pcie@0,0 {
 		pinctrl-0 = <&tc9563_resx_n>;
 		pinctrl-names = "default";
 
-		pcie@1,0 {
+		pcie1_switch0_dsp1: pcie@1,0 {
 			reg = <0x20800 0x0 0x0 0x0 0x0>;
 			#address-cells = <3>;
 			#size-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
new file mode 100644
index 0000000000000..a5ad796cb65d0
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
@@ -0,0 +1,1093 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, Roger Shimizu <rosh@debian.org>
+ */
+
+/dts-v1/;
+
+/* PM7250B is configured to use SID8/9 */
+#define PM7250B_SID 8
+#define PM7250B_SID1 9
+
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "kodiak.dtsi"
+#include "pm7250b.dtsi"
+#include "pm7325.dtsi"
+#include "pm8350c.dtsi" /* PM7350C */
+#include "pmk8350.dtsi" /* PMK7325 */
+
+/delete-node/ &adsp_mem;
+/delete-node/ &cdsp_mem;
+/delete-node/ &ipa_fw_mem;
+/delete-node/ &mpss_mem;
+/delete-node/ &remoteproc_mpss;
+/delete-node/ &remoteproc_wpss;
+/delete-node/ &rmtfs_mem;
+/delete-node/ &video_mem;
+/delete-node/ &wifi;
+/delete-node/ &wlan_ce_mem;
+/delete-node/ &wlan_fw_mem;
+/delete-node/ &wpss_mem;
+/delete-node/ &xbl_mem;
+
+/ {
+	model = "Thundercomm AI Mini PC G1 IoT";
+	compatible = "thundercomm,minipc-g1iot", "qcom,qcm6490";
+	chassis-type = "desktop";
+
+	aliases {
+		serial0 = &uart5;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&lt9611_out>;
+			};
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,qcm6490-pmic-glink", "qcom,pmic-glink";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+		orientation-gpios = <&tlmm 140 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&redriver_usb_con_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu_in: endpoint {
+						remote-endpoint = <&redriver_usb_con_sbu>;
+					};
+				};
+			};
+		};
+	};
+
+	lt9611_1v2: regulator-lt9611-vdd12 {
+		compatible = "regulator-fixed";
+		regulator-name = "LT9611_1V2";
+
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+	};
+
+	reserved-memory {
+		xbl_mem: xbl@80700000 {
+			reg = <0x0 0x80700000 0x0 0x100000>;
+			no-map;
+		};
+
+		cdsp_secure_heap_mem: cdsp-secure-heap@81800000 {
+			reg = <0x0 0x81800000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		camera_mem: camera@84300000 {
+			reg = <0x0 0x84300000 0x0 0x500000>;
+			no-map;
+		};
+
+		adsp_mem: adsp@86100000 {
+			reg = <0x0 0x86100000 0x0 0x2800000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@88900000 {
+			reg = <0x0 0x88900000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		video_mem: video@8a700000 {
+			reg = <0x0 0x8a700000 0x0 0x700000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@8ae00000 {
+			reg = <0x0 0x8ae00000 0x0 0x500000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode@8b31a000 {
+			reg = <0x0 0x8b31a000 0x0 0x2000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x100000>;
+			no-map;
+		};
+
+		tags_mem: tags@c0100000 {
+			reg = <0x0 0xc0100000 0x0 0x1200000>;
+			no-map;
+		};
+
+		qtee_mem: qtee@c1300000 {
+			reg = <0x0 0xc1300000 0x0 0x500000>;
+			no-map;
+		};
+
+		trusted_apps_mem: trusted-apps@c1800000 {
+			reg = <0x0 0xc1800000 0x0 0x1c00000>;
+			no-map;
+		};
+
+		debug_vm_mem: debug-vm@d0600000 {
+			reg = <0x0 0xd0600000 0x0 0x100000>;
+			no-map;
+		};
+	};
+
+	vdd_ntn_0p9: regulator-vdd-ntn-0p9 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN_0P9";
+		gpio = <&pm8350c_gpios 2 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <899400>;
+		regulator-max-microvolt = <899400>;
+		enable-active-high;
+		pinctrl-0 = <&ntn_0p9_en>;
+		pinctrl-names = "default";
+		regulator-enable-ramp-delay = <4300>;
+	};
+
+	vdd_ntn_1p8: regulator-vdd-ntn-1p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN_1P8";
+		gpio = <&pm8350c_gpios 3 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		enable-active-high;
+		pinctrl-0 = <&ntn_1p8_en>;
+		pinctrl-names = "default";
+		regulator-enable-ramp-delay = <10000>;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	thermal-zones {
+		sdm-skin-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 3>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		quiet-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		xo-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p972>;
+		vdd-l2-l7-supply = <&vreg_bob_3p296>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p272>;
+		vdd-l8-supply = <&vreg_s7b_0p972>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p872>;
+		vdd-l13-supply = <&vreg_s7b_0p972>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p272>;
+
+		vreg_s1b_1p872: smps1 {
+			regulator-name = "vreg_s1b_1p872";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		vreg_s7b_0p972: smps7 {
+			regulator-name = "vreg_s7b_0p972";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vreg_s8b_1p272: smps8 {
+			regulator-name = "vreg_s8b_1p272";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p504: ldo3 {
+			regulator-name = "vreg_l3b_0p504";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <650000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p952: ldo7 {
+			regulator-name = "vreg_l7b_2p952";
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p504: ldo11 {
+			regulator-name = "vreg_l11b_1p504";
+			regulator-min-microvolt = <1776000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p751: ldo12 {
+			regulator-name = "vreg_l12b_0p751";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p53: ldo13 {
+			regulator-name = "vreg_l13b_0p53";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p08: ldo14 {
+			regulator-name = "vreg_l14b_1p08";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p765: ldo15 {
+			regulator-name = "vreg_l15b_0p765";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p1: ldo16 {
+			regulator-name = "vreg_l16b_1p1";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p7: ldo17 {
+			regulator-name = "vreg_l17b_1p7";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+		vdd-l1-l12-supply = <&vreg_s1b_1p872>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p872>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob_3p296>;
+		vdd-l6-l9-l11-supply = <&vreg_bob_3p296>;
+		vdd-l10-supply = <&vreg_s7b_0p972>;
+		vdd-bob-supply = <&vph_pwr>;
+
+		vreg_s1c_2p19: smps1 {
+			regulator-name = "vreg_s1c_2p19";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <2208000>;
+		};
+
+		vreg_s9c_1p084: smps9 {
+			regulator-name = "vreg_s9c_1p084";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_1p62: ldo2 {
+			regulator-name = "vreg_l2c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_2p8: ldo3 {
+			regulator-name = "vreg_l3c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3540000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p62: ldo4 {
+			regulator-name = "vreg_l4c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p62: ldo5 {
+			regulator-name = "vreg_l5c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p62: ldo8 {
+			regulator-name = "vreg_l8c_1p62";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_2p96: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12c_1p65: ldo12 {
+			regulator-name = "vreg_l12c_1p65";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13c_2p7: ldo13 {
+			regulator-name = "vreg_l13c_2p7";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob_3p296: bob {
+			regulator-name = "vreg_bob_3p296";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+		};
+	};
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_GPLL0_MAIN_DIV_CLK_SRC>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/qcs6490/a660_zap.mbn";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	lt9611_codec: hdmi-bridge@2b {
+		compatible = "lontium,lt9611uxc";
+		reg = <0x2b>;
+
+		interrupts-extended = <&tlmm 24 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&pm7250b_gpios 2 GPIO_ACTIVE_HIGH>;
+
+		vdd-supply = <&lt9611_1v2>;
+		vcc-supply = <&vreg_l11c_2p8>;
+
+		pinctrl-0 = <&lt9611_irq_pin &lt9611_rst_pin>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lt9611_a: endpoint {
+					remote-endpoint = <&mdss_dsi0_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				lt9611_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	typec-mux@1c {
+		compatible = "onnn,nb7vpq904m";
+		reg = <0x1c>;
+
+		vcc-supply = <&vreg_l18b_1p8>;
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				redriver_usb_con_ss: endpoint {
+					remote-endpoint = <&pmic_glink_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				redriver_phy_con_ss: endpoint {
+					remote-endpoint = <&usb_dp_qmpphy_out>;
+					data-lanes = <0 1 2 3>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				redriver_usb_con_sbu: endpoint {
+					remote-endpoint = <&pmic_glink_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp {
+	status = "okay";
+};
+
+&mdss_dp_out {
+	data-lanes = <0 1>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dsi {
+	vdda-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&lt9611_a>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi_phy {
+	vdds-supply = <&vreg_l10c_0p88>;
+
+	status = "okay";
+};
+
+&pcie0 {
+	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+
+	pinctrl-0 = <&pcie0_clkreq_n>,
+		    <&pcie0_reset_n>,
+		    <&pcie0_wake_n>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&pcie1 {
+	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+
+	pinctrl-0 = <&pcie1_clkreq_n>,
+		    <&pcie1_reset_n>,
+		    <&pcie1_wake_n>;
+	pinctrl-names = "default";
+
+	iommu-map = <0x0 &apps_smmu 0x1c80 0x1>,
+		    <0x100 &apps_smmu 0x1c81 0x1>,
+		    <0x208 &apps_smmu 0x1c84 0x1>,
+		    <0x210 &apps_smmu 0x1c85 0x1>,
+		    <0x218 &apps_smmu 0x1c86 0x1>,
+		    <0x300 &apps_smmu 0x1c87 0x1>,
+		    <0x400 &apps_smmu 0x1c88 0x1>,
+		    <0x500 &apps_smmu 0x1c89 0x1>,
+		    <0x501 &apps_smmu 0x1c90 0x1>;
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&pcie1_port0 {
+	pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vdd_ntn_0p9>;
+		vdd18-supply = <&vdd_ntn_1p8>;
+		vdd09-supply = <&vdd_ntn_0p9>;
+		vddio1-supply = <&vdd_ntn_1p8>;
+		vddio2-supply = <&vdd_ntn_1p8>;
+		vddio18-supply = <&vdd_ntn_1p8>;
+
+		i2c-parent = <&i2c0 0x77>;
+
+		resx-gpios = <&pm8350c_gpios 1 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&tc9563_resx_n>;
+		pinctrl-names = "default";
+
+		pcie@1,0 {
+			reg = <0x20800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x21000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+		};
+
+		pcie@3,0 {
+			reg = <0x21800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+
+			pci@0,0 {
+				reg = <0x50000 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+
+			pci@0,1 {
+				reg = <0x50100 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+		};
+	};
+};
+
+&pm7250b_gpios {
+	lt9611_rst_pin: lt9611-rst-state {
+		pins = "gpio2";
+		function = "normal";
+
+		output-high;
+		input-disable;
+		power-source = <0>;
+	};
+};
+
+&pm7325_temp_alarm {
+	io-channels = <&pmk8350_vadc PM7325_ADC7_DIE_TEMP>;
+	io-channel-names = "thermal";
+};
+
+&pmk8350_adc_tm {
+	status = "okay";
+
+	xo-therm@0 {
+		reg = <0>;
+		io-channels = <&pmk8350_vadc PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	quiet-therm@1 {
+		reg = <1>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	sdm-skin-therm@3 {
+		reg = <3>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pm8350c_gpios {
+	ntn_0p9_en: ntn-0p9-en-state {
+		pins = "gpio2";
+		function = "normal";
+
+		bias-disable;
+		input-disable;
+		output-enable;
+		power-source = <0>;
+	};
+
+	ntn_1p8_en: ntn-1p8-en-state {
+		pins = "gpio3";
+		function = "normal";
+
+		bias-disable;
+		input-disable;
+		output-enable;
+		power-source = <0>;
+	};
+
+	tc9563_resx_n: tc9563-resx-state {
+		pins = "gpio1";
+		function = "normal";
+
+		bias-disable;
+		input-disable;
+		output-enable;
+		power-source = <0>;
+	};
+};
+
+&pm8350c_pwm {
+	nvmem = <&pmk8350_sdam_21>,
+		<&pmk8350_sdam_22>;
+	nvmem-names = "lpg_chan_sdam",
+		      "lut_sdam";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pmk8350_vadc {
+	channel@3 {
+		reg = <PMK8350_ADC7_DIE_TEMP>;
+		label = "pmk7325_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@44 {
+		reg = <PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		label = "xo_therm";
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		qcom,ratiometric;
+	};
+
+	channel@103 {
+		reg = <PM7325_ADC7_DIE_TEMP>;
+		label = "pm7325_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@144 {
+		reg = <PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_quiet_therm";
+	};
+
+	channel@146 {
+		reg = <PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_sdm_skin_therm";
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	firmware-name = "qcom/qcs6490/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	firmware-name = "qcom/qcs6490/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/qcs6490/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/qcs6490/cdsp.mbn";
+
+	status = "okay";
+};
+
+&sdc2_clk {
+	bias-disable;
+	drive-strength = <16>;
+};
+
+&sdc2_cmd {
+	bias-pull-up;
+	drive-strength = <10>;
+};
+
+&sdc2_data {
+	bias-pull-up;
+	drive-strength = <10>;
+};
+
+&sdhc_2 {
+	pinctrl-0 = <&sdc2_clk>, <&sdc2_cmd>, <&sdc2_data>, <&sd_cd>;
+	pinctrl-1 = <&sdc2_clk_sleep>, <&sdc2_cmd_sleep>, <&sdc2_data_sleep>, <&sd_cd>;
+
+	vmmc-supply = <&vreg_l9c_2p96>;
+	vqmmc-supply = <&vreg_l6c_2p96>;
+
+	cd-gpios = <&tlmm 91 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <32 2>, /* ADSP */
+			       <48 4>; /* NFC */
+
+	lt9611_irq_pin: lt9611-irq-state {
+		pins = "gpio24";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie0_reset_n: pcie0-reset-n-state {
+		pins = "gpio87";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie0_wake_n: pcie0-wake-n-state {
+		pins = "gpio89";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	pcie1_reset_n: pcie1-reset-n-state {
+		pins = "gpio2";
+		function = "gpio";
+		drive-strength = <16>;
+		output-low;
+		bias-disable;
+	};
+
+	pcie1_wake_n: pcie1-wake-n-state {
+		pins = "gpio3";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	sd_cd: sd-cd-state {
+		pins = "gpio91";
+		function = "gpio";
+		bias-pull-up;
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vreg_l10c_0p88>;
+	vdda33-supply = <&vreg_l2b_3p072>;
+	vdda18-supply = <&vreg_l1c_1p8>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l6b_1p2>;
+	vdda-pll-supply = <&vreg_l1b_0p912>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&redriver_phy_con_ss>;
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&vreg_l7b_2p952>;
+	vcc-max-microamp = <800000>;
+	vccq-supply = <&vreg_l9b_1p2>;
+	vccq-max-microamp = <900000>;
+	vccq2-supply = <&vreg_l9b_1p2>;
+	vccq2-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&venus {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
index 0b64a0b912021..f47efca42d48d 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
@@ -755,10 +755,10 @@ ports {
 			#address-cells = <1>;
 			#size-cells = <0>;
 
-			port@0 {
-				reg = <0>;
+			port@1 {
+				reg = <1>;
 
-				lt9611_a: endpoint {
+				lt9611_b: endpoint {
 					remote-endpoint = <&mdss_dsi0_out>;
 				};
 			};
@@ -801,7 +801,7 @@ &mdss_dsi {
 };
 
 &mdss_dsi0_out {
-	remote-endpoint = <&lt9611_a>;
+	remote-endpoint = <&lt9611_b>;
 	data-lanes = <0 1 2 3>;
 };
 
diff --git a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
index c04e0ad53eecf..e9a8553a8d821 100644
--- a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
@@ -18,12 +18,76 @@ / {
 	aliases {
 		serial0 = &uart7;
 		mmc0 = &sdhc_1;
+		serial1 = &uart2;
 	};
 
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	vreg_1p0: regulator-vreg-1p0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_1P0";
+
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1000000>;
+
+		vin-supply = <&vreg_1p8>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_1p8: regulator-vreg-1p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_1P8";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		vin-supply = <&vreg_5p0>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_3p0: regulator-vreg-3p0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_3P0";
+
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+
+		vin-supply = <&vreg_12p0>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_5p0: regulator-vreg-5p0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_5P0";
+
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		vin-supply = <&vreg_12p0>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_12p0: regulator-vreg-12p0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_12P0";
+
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	dp0-connector {
 		compatible = "dp-connector";
 		label = "DP0";
@@ -36,6 +100,62 @@ dp0_connector_in: endpoint {
 		};
 	};
 
+	dp-dsi0-connector {
+		compatible = "dp-connector";
+		label = "DSI0";
+		type = "full-size";
+
+		port {
+			dp_dsi0_connector_in: endpoint {
+				remote-endpoint = <&dsi2dp_bridge_out>;
+			};
+		};
+	};
+
+	vreg_conn_1p05: regulator-conn-1p05 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_conn_1p05";
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		vin-supply = <&vreg_conn_1p8>;
+	};
+
+	vreg_conn_1p35: regulator-conn-1p35 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_conn_1p35";
+		regulator-min-microvolt = <1350000>;
+		regulator-max-microvolt = <1350000>;
+		vin-supply = <&vreg_conn_1p8>;
+	};
+
+	vreg_conn_1p8: regulator-conn-1p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_conn_1p8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		startup-delay-us = <4000>;
+		enable-active-high;
+		gpio = <&pmm8650au_1_gpios 4 GPIO_ACTIVE_HIGH>;
+	};
+
+	vreg_conn_1p95: regulator-conn-1p95 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_conn_1p95";
+		regulator-min-microvolt = <1950000>;
+		regulator-max-microvolt = <1950000>;
+		vin-supply = <&vreg_conn_1p8>;
+	};
+
+	vreg_conn_pa: regulator-conn-pa {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_conn_pa";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		startup-delay-us = <4000>;
+		enable-active-high;
+		gpio = <&pmm8650au_1_gpios 6 GPIO_ACTIVE_HIGH>;
+	};
+
 	regulator-usb2-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "USB2_VBUS";
@@ -45,6 +165,70 @@ regulator-usb2-vbus {
 		enable-active-high;
 		regulator-always-on;
 	};
+
+	wcn6855-pmu {
+		compatible = "qcom,wcn6855-pmu";
+
+		pinctrl-0 = <&wlan_en_state>;
+		pinctrl-names = "default";
+
+		vddio-supply = <&vreg_conn_pa>;
+		vddaon-supply = <&vreg_conn_1p8>;
+		vddpmu-supply = <&vreg_conn_pa>;
+		vddpmumx-supply = <&vreg_conn_1p8>;
+		vddpmucx-supply = <&vreg_conn_pa>;
+		/* WLAN rails: 1.05/1.35/1.95V (nominal 0.95/1.30/1.90V) */
+		vddrfa0p95-supply = <&vreg_conn_1p05>;
+		vddrfa1p3-supply = <&vreg_conn_1p35>;
+		vddrfa1p9-supply = <&vreg_conn_1p95>;
+		vddpcie1p3-supply = <&vreg_conn_1p35>;
+		vddpcie1p9-supply = <&vreg_conn_1p95>;
+
+		bt-enable-gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>;
+		wlan-enable-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -316,6 +500,75 @@ &gpu_zap_shader {
 	firmware-name = "qcom/qcs8300/a623_zap.mbn";
 };
 
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	io_expander: gpio@74 {
+		compatible = "ti,tca9539";
+		reg = <0x74>;
+		interrupts-extended = <&tlmm 93 IRQ_TYPE_EDGE_BOTH>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		reset-gpios = <&tlmm 66 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&io_expander_intr_active>,
+			    <&io_expander_reset_active>;
+		pinctrl-names = "default";
+	};
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9543";
+		reg = <0x70>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			bridge@58 {
+				compatible = "analogix,anx7625";
+				reg = <0x58>;
+				interrupts-extended = <&io_expander 2 IRQ_TYPE_EDGE_FALLING>;
+				enable-gpios = <&io_expander 1 GPIO_ACTIVE_HIGH>;
+				reset-gpios = <&io_expander 0 GPIO_ACTIVE_HIGH>;
+				vdd10-supply = <&vreg_1p0>;
+				vdd18-supply = <&vreg_1p8>;
+				vdd33-supply = <&vreg_3p0>;
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						dsi2dp_bridge_in: endpoint {
+
+							remote-endpoint = <&mdss_dsi0_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						dsi2dp_bridge_out: endpoint {
+
+							remote-endpoint = <&dp_dsi0_connector_in>;
+						};
+					};
+				};
+			};
+		};
+	};
+};
+
 &pmm8650au_1_gpios {
 	usb2_en: usb2-en-state {
 		pins = "gpio7";
@@ -363,6 +616,23 @@ &pcie0 {
 &pcieport0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
+
+	wifi@0 {
+		compatible = "pci17cb,1103";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+
+		qcom,calibration-variant = "QC_QCS8300_Ride";
+	};
 };
 
 &pcie0_phy {
@@ -391,10 +661,31 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l4a>;
+
+	status = "okay";
+};
+
+&mdss_dsi0_out {
+	data-lanes = <0 1 2 3>;
+	remote-endpoint = <&dsi2dp_bridge_in>;
+};
+
 &qupv3_id_0 {
 	status = "okay";
 };
 
+&qupv3_id_1 {
+	status = "okay";
+};
+
 &remoteproc_adsp {
 	firmware-name = "qcom/qcs8300/adsp.mbn";
 	status = "okay";
@@ -436,6 +727,12 @@ &sdhc_1 {
 };
 
 &tlmm {
+	bt_en_state: bt-en-state {
+		pins = "gpio55";
+		function = "gpio";
+		bias-pull-down;
+	};
+
 	pcie0_default_state: pcie0-default-state {
 		wake-pins {
 			pins = "gpio0";
@@ -498,11 +795,50 @@ perst-pins {
 		};
 	};
 
+	io_expander_reset_active: io-expander-reset-active-state {
+		pins = "gpio66";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	io_expander_intr_active: io-expander-intr-active-state {
+		pins = "gpio93";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	dp_hot_plug_det: dp-hot-plug-det-state {
 		pins = "gpio94";
 		function = "edp0_hot";
 		bias-disable;
 	};
+
+	wlan_en_state: wlan-en-state {
+		pins = "gpio54";
+		function = "gpio";
+		bias-pull-down;
+	};
+};
+
+&uart2 {
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6855-bt";
+		firmware-name = "QCA6698/hpnv21", "QCA6698/hpbtfw21.tlv";
+		max-speed = <3200000>;
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+	};
 };
 
 &uart7 {
diff --git a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
index 197ab6eb1666f..bf088fa9807f0 100644
--- a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
+++ b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
@@ -6,6 +6,7 @@
 /dts-v1/;
 
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/usb/pd.h>
 #include "agatti.dtsi"
 #include "pm4125.dtsi"
 
@@ -109,6 +110,15 @@ multi-led {
 		leds = <&ledr>, <&ledg>, <&ledb>;
 	};
 
+	vreg_anx_30: regulator-anx-30 {
+		compatible = "regulator-fixed";
+		regulator-name = "anx30";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	/* PM4125 charger out, supplied by VBAT */
 	vph_pwr: regulator-vph-pwr {
 		compatible = "regulator-fixed";
@@ -142,6 +152,86 @@ &i2c1 {
 	clock-frequency = <100000>;
 
 	status = "okay";
+
+	anx7625: encoder@58 {
+		compatible = "analogix,anx7625";
+		reg = <0x58>;
+		interrupts-extended = <&tlmm 81 IRQ_TYPE_EDGE_FALLING>;
+		vdd10-supply = <&pm4125_l11>;
+		vdd18-supply = <&pm4125_l15>;
+		vdd33-supply = <&vreg_anx_30>;
+		analogix,audio-enable;
+		analogix,lane0-swing = /bits/ 8 <0x14 0x54 0x64 0x74>;
+		analogix,lane1-swing = /bits/ 8 <0x14 0x54 0x64 0x74>;
+
+		pinctrl-0 = <&anx7625_int_pin>, <&anx7625_cable_det_pin>;
+
+		connector {
+			compatible = "usb-c-connector";
+			power-role = "sink";
+			data-role = "dual";
+			try-power-role = "sink";
+
+			pd-revision = /bits/ 8 <0x03 0x00 0x00 0x00>;
+			op-sink-microwatt = <15000000>;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					anx_hs_in: endpoint {
+						remote-endpoint = <&usb_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					anx_ss_in: endpoint {
+						remote-endpoint = <&usb_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				anx_dsi0_in: endpoint {
+					remote-endpoint = <&mdss_dsi0_out>;
+					data-lanes = <0 1 2 3>;
+				};
+			};
+		};
+	};
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&pm4125_l5>;
+
+	status = "okay";
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&anx_dsi0_in>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	status = "okay";
 };
 
 &pm4125_vbus {
@@ -325,21 +415,13 @@ &sdhc_1 {
 &spi5 {
 	status = "okay";
 
-	spidev@0 {
-		reg = <0>;
+	mcu@0 {
 		compatible = "arduino,unoq-mcu";
-		pinctrl-0 = <&spidev_cs>;
-		pinctrl-names = "default";
+		reg = <0>;
 	};
 };
 
 &tlmm {
-	spidev_cs: spidev-cs-state {
-		pins = "gpio17";
-		function = "gpio";
-		drive-strength = <16>;
-	};
-
 	jmisc_gpio18: jmisc-gpio18-state {
 		pins = "gpio18";
 		function = "gpio";
@@ -361,6 +443,22 @@ key_vold_n: key-vold-n-state {
 		output-disable;
 	};
 
+	anx7625_cable_det_pin: anx7625-cable-det-pins-state {
+		pins = "gpio46";
+		function = "gpio";
+		drive-strength = <16>;
+		output-disable;
+		bias-pull-up;
+	};
+
+	anx7625_int_pin: anx7625-int-pins-state {
+		pins = "gpio81";
+		function = "gpio";
+		drive-strength = <16>;
+		output-disable;
+		bias-pull-up;
+	};
+
 	key_volp_n: key-volp-n-state {
 		pins = "gpio96";
 		function = "gpio";
@@ -428,6 +526,10 @@ &usb {
 	status = "okay";
 };
 
+&usb_dwc3_hs {
+	remote-endpoint = <&anx_hs_in>;
+};
+
 &usb_hsphy {
 	vdd-supply = <&pm4125_l12>;
 	vdda-pll-supply = <&pm4125_l13>;
@@ -443,6 +545,10 @@ &usb_qmpphy {
 	status = "okay";
 };
 
+&usb_qmpphy_out {
+	remote-endpoint = <&anx_ss_in>;
+};
+
 &wifi {
 	vdd-0.8-cx-mx-supply = <&pm4125_l7>;
 	vdd-1.8-xo-supply = <&pm4125_l13>;
diff --git a/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts
new file mode 100644
index 0000000000000..b84da371581b9
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7180-ecs-liva-qc710.dts
@@ -0,0 +1,616 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/sound/qcom,q6asm.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "sc7180.dtsi"
+
+#include "pm6150.dtsi"
+#include "pm6150l.dtsi"
+
+/delete-node/ &tz_mem;
+/delete-node/ &ipa_fw_mem;
+
+/ {
+	model = "ECS LIVA QC710";
+	compatible = "ecs,liva-qc710", "qcom,sc7180";
+	chassis-type = "desktop";
+
+	aliases {
+		bluetooth0 = &bluetooth;
+		hsuart0 = &uart3;
+		wifi0 = &wifi;
+	};
+
+	hdmi-bridge {
+		compatible = "algoltek,ag6311";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_dp_in: endpoint {
+					remote-endpoint = <&usb_1_qmpphy_dp_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_tmds_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_tmds_out>;
+			};
+		};
+	};
+
+	reserved-memory {
+		gpu_mem: zap-shader@80840000 {
+			reg = <0x0 0x80840000 0 0x2000>;
+			no-map;
+		};
+
+		venus_mem: venus@85b00000 {
+			reg = <0x0 0x85b00000 0 0x500000>;
+			no-map;
+		};
+
+		mpss_mem: mpss@86000000 {
+			reg = <0x0 0x86000000 0x0 0x2000000>;
+			no-map;
+		};
+
+		adsp_mem: adsp@8e400000 {
+			reg = <0x0 0x8e400000 0x0 0x2800000>;
+			no-map;
+		};
+
+		wlan_mem: wlan@93900000 {
+			reg = <0x0 0x93900000 0x0 0x200000>;
+			no-map;
+		};
+	};
+
+	usb_a_connector: usb-a-connector {
+		compatible = "usb-a-connector";
+
+		port {
+			usb_a_connector_ss_in: endpoint {
+				remote-endpoint = <&usb_1_qmpphy_usb_ss_out>;
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm6150-rpmh-regulators";
+		qcom,pmic-id = "a";
+
+		vreg_s1a_1p1: smps1 {
+			regulator-min-microvolt = <1128000>;
+			regulator-max-microvolt = <1128000>;
+		};
+
+		vreg_l4a_0p8: ldo4 {
+			regulator-min-microvolt = <824000>;
+			regulator-max-microvolt = <928000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9a_0p6: ldo9 {
+			regulator-min-microvolt = <488000>;
+			regulator-max-microvolt = <800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10a_1p8: ldo10 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_l11a_1p8: ldo11 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12a_1p8: ldo12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13a_1p8: ldo13 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14a_1p8: ldo14 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15a_1p8: ldo15 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16a_2p7: ldo16 {
+			regulator-min-microvolt = <2496000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17a_3p0: ldo17 {
+			regulator-min-microvolt = <2920000>;
+			regulator-max-microvolt = <3232000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18a_2p8: ldo18 {
+			regulator-min-microvolt = <2496000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19a_2p9: ldo19 {
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm6150l-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vreg_s8c_1p3: smps8 {
+			regulator-min-microvolt = <1120000>;
+			regulator-max-microvolt = <1408000>;
+		};
+
+		vreg_l1c_1p8: ldo1 {
+			regulator-min-microvolt = <1616000>;
+			regulator-max-microvolt = <1984000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_1p3: ldo2 {
+			regulator-min-microvolt = <1168000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_1p2: ldo3 {
+			regulator-min-microvolt = <1144000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p8: ldo4 {
+			regulator-min-microvolt = <1648000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_LPM>;
+		};
+
+		vreg_l5c_1p8: ldo5 {
+			regulator-min-microvolt = <1648000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_LPM>;
+		};
+
+		vreg_l6c_2p9: ldo6 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3312000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_LPM>;
+		};
+
+		vreg_l8c_1p8: ldo8 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_2p9: ldo9 {
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10c_3p3: ldo10 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3400000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_3p3: ldo11 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3400000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob: bob {
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sc7180/ecs/liva-qc710/qcdxkmsuc7180.mbn";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp {
+	pinctrl-0 = <&dp_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss_dp_out {
+	data-lanes = <0 1>;
+	remote-endpoint = <&usb_1_qmpphy_dp_in>;
+};
+
+&pm6150_rtc {
+	qcom,uefi-rtc-info;
+
+	status = "okay";
+};
+
+&qfprom {
+	vcc-supply = <&vreg_l11a_1p8>;
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	memory-region = <&adsp_mem>;
+	firmware-name = "qcom/sc7180/ecs/liva-qc710/qcadsp7180.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sc7180/ecs/liva-qc710/qcmpss7180_nm.mbn";
+
+	status = "okay";
+};
+
+&sdhc_1 {
+	pinctrl-0 = <&sdc1_default>;
+	pinctrl-1 = <&sdc1_sleep>;
+	pinctrl-names = "default", "sleep";
+	vmmc-supply = <&vreg_l19a_2p9>;
+	vqmmc-supply = <&vreg_l12a_1p8>;
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	pinctrl-0 = <&sdc2_default>;
+	pinctrl-1 = <&sdc2_sleep>;
+	pinctrl-names = "default", "sleep";
+	vmmc-supply = <&vreg_l9c_2p9>;
+	vqmmc-supply = <&vreg_l6c_2p9>;
+
+	cd-gpios = <&tlmm 69 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&tlmm {
+	/*
+	 * The TZ seem to protect those because some boards can have
+	 * fingerprint sensor connected to this range. Not connected
+	 * on this board
+	 */
+	gpio-reserved-ranges = <58 5>;
+
+	qup_uart3_sleep: qup-uart3-sleep-state {
+		cts-pins {
+			/*
+			 * Configure a pull-down on CTS to match the pull of
+			 * the Bluetooth module.
+			 */
+			pins = "gpio38";
+			function = "gpio";
+			bias-pull-down;
+		};
+
+		rts-pins {
+			/*
+			 * Configure pull-down on RTS. As RTS is active low
+			 * signal, pull it low to indicate the BT SoC that it
+			 * can wakeup the system anytime from suspend state by
+			 * pulling RX low (by sending wakeup bytes).
+			 */
+			pins = "gpio39";
+			function = "gpio";
+			bias-pull-down;
+		};
+
+		tx-pins {
+			/*
+			 * Configure pull-up on TX when it isn't actively driven
+			 * to prevent BT SoC from receiving garbage during sleep.
+			 */
+			pins = "gpio40";
+			function = "gpio";
+			bias-pull-up;
+		};
+
+		rx-pins {
+			/*
+			 * Configure a pull-up on RX. This is needed to avoid
+			 * garbage data when the TX pin of the Bluetooth module
+			 * is floating which may cause spurious wakeups.
+			 */
+			pins = "gpio41";
+			function = "gpio";
+			bias-pull-up;
+		};
+	};
+
+	sdc1_default: sdc1-default-state {
+		clk-pins {
+			pins = "sdc1_clk";
+			drive-strength = <16>;
+			bias-disable;
+		};
+
+		cmd-pins {
+			pins = "sdc1_cmd";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+
+		data-pins {
+			pins = "sdc1_data";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+
+		rclk-pins {
+			pins = "sdc1_rclk";
+			bias-pull-down;
+		};
+	};
+
+	sdc1_sleep: sdc1-sleep-state {
+		clk-pins {
+			pins = "sdc1_clk";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		cmd-pins {
+			pins = "sdc1_cmd";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		data-pins {
+			pins = "sdc1_data";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		rclk-pins {
+			pins = "sdc1_rclk";
+			bias-pull-down;
+		};
+	};
+
+	sdc2_default: sdc2-default-state {
+		clk-pins {
+			pins = "sdc2_clk";
+			bias-disable;
+			drive-strength = <16>;
+		};
+
+		cmd-pins {
+			pins = "sdc2_cmd";
+			bias-pull-up;
+			drive-strength = <10>;
+		};
+
+		data-pins {
+			pins = "sdc2_data";
+			bias-pull-up;
+			drive-strength = <10>;
+		};
+
+		sd-cd-pins {
+			pins = "gpio69";
+			function = "gpio";
+			bias-pull-up;
+			drive-strength = <2>;
+		};
+	};
+
+	sdc2_sleep: sdc2-sleep-state {
+		clk-pins {
+			pins = "sdc2_clk";
+			bias-disable;
+			drive-strength = <2>;
+		};
+
+		cmd-pins {
+			pins = "sdc2_cmd";
+			bias-pull-up;
+			drive-strength = <2>;
+		};
+
+		data-pins {
+			pins = "sdc2_data";
+			bias-pull-up;
+			drive-strength = <2>;
+		};
+
+		sd-cd-pins {
+			pins = "gpio69";
+			function = "gpio";
+			bias-pull-up;
+			drive-strength = <2>;
+		};
+	};
+};
+&uart3 {
+	/delete-property/ interrupts;
+	interrupts-extended = <&intc GIC_SPI 604 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 41 IRQ_TYPE_EDGE_FALLING>;
+
+	pinctrl-1 = <&qup_uart3_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn3991-bt";
+		vddio-supply = <&vreg_l10a_1p8>;
+		vddxo-supply = <&vreg_l1c_1p8>;
+		vddrf-supply = <&vreg_l2c_1p3>;
+		vddch0-supply = <&vreg_l10c_3p3>;
+		max-speed = <3200000>;
+	};
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3 {
+	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	hub@1 {
+		compatible = "usb5e3,608";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* @1: 3.0 Type-A port on the back
+		 * @2: 2.0 Type-A port the side
+		 * @3: 2.0 Type-C port on the back
+		 */
+
+		ethernet@4 {
+			compatible = "usbbda,8152";
+			reg = <4>;
+		};
+	};
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l4a_0p8>;
+	vdda-pll-supply = <&vreg_l11a_1p8>;
+	vdda-phy-dpdm-supply = <&vreg_l17a_3p0>;
+	qcom,imp-res-offset-value = <8>;
+	qcom,preemphasis-level = <QUSB2_V2_PREEMPHASIS_15_PERCENT>;
+	qcom,preemphasis-width = <QUSB2_V2_PREEMPHASIS_WIDTH_HALF_BIT>;
+	qcom,bias-ctrl-value = <0x22>;
+	qcom,charge-ctrl-value = <3>;
+	qcom,hsdisc-trim-value = <0>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l3c_1p2>;
+	vdda-pll-supply = <&vreg_l4a_0p8>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_1_qmpphy_dp_out: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2>;
+				remote-endpoint = <&hdmi_bridge_dp_in>;
+			};
+
+			usb_1_qmpphy_usb_ss_out: endpoint@1 {
+				reg = <1>;
+
+				data-lanes = <1 0>;
+				remote-endpoint = <&usb_a_connector_ss_in>;
+			};
+		};
+	};
+};
+
+&usb_1_qmpphy_dp_in {
+	remote-endpoint = <&mdss_dp_out>;
+};
+
+&venus {
+	firmware-name = "qcom/sc7180/ecs/liva-qc710/qcvss7180.mbn";
+};
+
+&wifi {
+	vdd-0.8-cx-mx-supply = <&vreg_l9a_0p6>;
+	vdd-1.8-xo-supply = <&vreg_l1c_1p8>;
+	vdd-1.3-rfa-supply = <&vreg_l2c_1p3>;
+	vdd-3.3-ch0-supply = <&vreg_l10c_3p3>;
+	vdd-3.3-ch1-supply = <&vreg_l11c_3p3>;
+
+	qcom,calibration-variant = "ECS_QC710";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
index 45b9864e3304b..8d69225a4271e 100644
--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
@@ -3460,8 +3460,8 @@ port@1 {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
index 8319d892c6e49..f45deb188c6c0 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
@@ -3322,8 +3322,8 @@ mdss_dp0_out: endpoint {
 				dp0_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -3404,8 +3404,8 @@ mdss_dp1_out: endpoint {
 				dp1_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -3480,8 +3480,8 @@ mdss_edp_out: endpoint {
 				edp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index 706eb1309d3f0..58876b25dd23c 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -4769,8 +4769,8 @@ mdss0_dp0_out: endpoint {
 				mdss0_dp0_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -4851,8 +4851,8 @@ mdss0_dp1_out: endpoint {
 				mdss0_dp1_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -4931,8 +4931,8 @@ mdss0_dp2_out: endpoint {
 				mdss0_dp2_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5006,8 +5006,8 @@ mdss0_dp3_out: endpoint {
 				mdss0_dp3_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -6125,8 +6125,8 @@ mdss1_dp0_out: endpoint {
 				mdss1_dp0_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -6205,8 +6205,8 @@ mdss1_dp1_out: endpoint {
 				mdss1_dp1_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -6285,8 +6285,8 @@ mdss1_dp2_out: endpoint {
 				mdss1_dp2_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -6360,8 +6360,8 @@ mdss1_dp3_out: endpoint {
 				mdss1_dp3_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi
index f4b8e8f468f24..1e424a9503c7d 100644
--- a/arch/arm64/boot/dts/qcom/sdm630.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi
@@ -1013,6 +1013,20 @@ data-pins {
 					drive-strength = <2>;
 				};
 			};
+
+			spi7_default: spi7-default-state {
+				pins = "gpio24", "gpio25", "gpio26", "gpio27";
+				function = "blsp_spi7";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			spi7_sleep: spi7-sleep-state {
+				pins = "gpio24", "gpio25", "gpio26", "gpio27";
+				function = "blsp_spi7";
+				drive-strength = <6>;
+				bias-disable;
+			};
 		};
 
 		remoteproc_mss: remoteproc@4080000 {
@@ -1950,6 +1964,26 @@ blsp_i2c7: i2c@c1b7000 {
 			status = "disabled";
 		};
 
+		blsp_spi7: spi@c1b7000 {
+			compatible = "qcom,spi-qup-v2.2.1";
+			reg = <0x0c1b7000 0x600>;
+			interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_BLSP2_QUP3_SPI_APPS_CLK>,
+				 <&gcc GCC_BLSP2_AHB_CLK>;
+			clock-names = "core", "iface";
+
+			dmas = <&blsp2_dma 8>, <&blsp2_dma 9>;
+			dma-names = "tx", "rx";
+
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&spi7_default>;
+			pinctrl-1 = <&spi7_sleep>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		blsp_i2c8: i2c@c1b8000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x0c1b8000 0x600>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
index bf2f9c04adba7..c0f21a745fb0c 100644
--- a/arch/arm64/boot/dts/qcom/sdm845.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
@@ -2805,6 +2805,62 @@ tlmm: pinctrl@3400000 {
 			gpio-ranges = <&tlmm 0 0 151>;
 			wakeup-parent = <&pdc_intc>;
 
+			cam_mclk0_default: cam-mclk0-default-state {
+				pins = "gpio13";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk0_sleep: cam-mclk0-sleep-state {
+				pins = "gpio13";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cam_mclk1_default: cam-mclk1-default-state {
+				pins = "gpio14";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk1_sleep: cam-mclk1-sleep-state {
+				pins = "gpio14";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cam_mclk2_default: cam-mclk2-default-state {
+				pins = "gpio15";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk2_sleep: cam-mclk2-sleep-state {
+				pins = "gpio15";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio16";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk3_sleep: cam-mclk3-sleep-state {
+				pins = "gpio16";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
 			cci0_default: cci0-default-state {
 				/* SDA, SCL */
 				pins = "gpio17", "gpio18";
diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi
index e9336adbc3918..3a9a1ad8d5816 100644
--- a/arch/arm64/boot/dts/qcom/sm6115.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi
@@ -1864,6 +1864,8 @@ mdss: display-subsystem@5e00000 {
 				 <&gcc GCC_DISP_HF_AXI_CLK>,
 				 <&dispcc DISP_CC_MDSS_MDP_CLK>;
 
+			resets = <&dispcc DISP_CC_MDSS_CORE_BCR>;
+
 			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
new file mode 100644
index 0000000000000..7eecd9dc30280
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
@@ -0,0 +1,313 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2025, Gabriel Gonzales <semfault@disroot.org>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/arm/qcom,ids.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include "sm6125.dtsi"
+#include "pm6125.dtsi"
+
+/delete-node/ &adsp_pil_mem;
+/delete-node/ &cont_splash_mem;
+/delete-node/ &gpu_mem;
+/delete-node/ &ipa_fw_mem;
+/delete-node/ &ipa_gsi_mem;
+
+/ {
+	model = "Xiaomi Redmi Note 8";
+	compatible = "xiaomi,ginkgo", "qcom,sm6125";
+	chassis-type = "handset";
+
+	qcom,msm-id = <QCOM_ID_SM6125 0x10000>;
+
+	aliases {
+		serial0 = &uart4;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_mem>;
+			width = <1080>;
+			height = <2340>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+		};
+	};
+
+	reserved-memory {
+		adsp_pil_mem: adsp_pil_mem@55300000 {
+			reg = <0x0 0x55300000 0x0 0x2200000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa_fw_mem@57500000 {
+			reg = <0x0 0x57500000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa_gsi_mem@57510000 {
+			reg = <0x0 0x57510000 0x0 0x5000>;
+			no-map;
+		};
+
+		gpu_mem: gpu_mem@57515000 {
+			reg = <0x0 0x57515000 0x0 0x2000>;
+			no-map;
+		};
+
+		framebuffer_mem: framebuffer@5c000000 {
+			reg = <0x0 0x5c000000 0x0 (2340 * 1080 * 4)>;
+			no-map;
+		};
+
+		/* Matching with recovery values to be able to get the results. */
+		ramoops@61600000 {
+			compatible = "ramoops";
+			reg = <0x0 0x61600000 0x0 0x400000>;
+			record-size = <0x80000>;
+			pmsg-size = <0x200000>;
+			console-size = <0x100000>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&vol_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm6125_gpios 5 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+};
+
+&pm6125_gpios {
+	vol_up_n: vol-up-n-state {
+		pins = "gpio5";
+		function = "normal";
+		power-source = <0>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&hsusb_phy1 {
+	vdd-supply = <&vreg_l7a>;
+	vdda-pll-supply = <&vreg_l10a>;
+	vdda-phy-dpdm-supply = <&vreg_l15a>;
+	status = "okay";
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators-0 {
+		compatible = "qcom,rpm-pm6125-regulators";
+
+		vreg_s6a: s6 {
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <1422000>;
+		};
+
+		vreg_l1a: l1 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+		};
+
+		vreg_l2a: l2 {
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1056000>;
+		};
+
+		vreg_l3a: l3 {
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1064000>;
+		};
+
+		vreg_l4a: l4 {
+			regulator-min-microvolt = <872000>;
+			regulator-max-microvolt = <976000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l5a: l5 {
+			regulator-min-microvolt = <1648000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l6a: l6 {
+			regulator-min-microvolt = <576000>;
+			regulator-max-microvolt = <656000>;
+		};
+
+		vreg_l7a: l7 {
+			regulator-min-microvolt = <872000>;
+			regulator-max-microvolt = <976000>;
+		};
+
+		vreg_l8a: l8 {
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <728000>;
+		};
+
+		vreg_l9a: l9 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1896000>;
+		};
+
+		vreg_l10a: l10 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1896000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l11a: l11 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1952000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l12a: l12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1996000>;
+		};
+
+		vreg_l13a: l13 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1832000>;
+		};
+
+		vreg_l14a: l14 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1904000>;
+		};
+
+		vreg_l15a: l15 {
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3232000>;
+		};
+
+		vreg_l16a: l16 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1904000>;
+		};
+
+		vreg_l17a: l17 {
+			regulator-min-microvolt = <1248000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		vreg_l18a: l18 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1264000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l19a: l19 {
+			regulator-min-microvolt = <1648000>;
+			regulator-max-microvolt = <2952000>;
+		};
+
+		vreg_l20a: l20 {
+			regulator-min-microvolt = <1648000>;
+			regulator-max-microvolt = <2952000>;
+		};
+
+		vreg_l21a: l21 {
+			regulator-min-microvolt = <2600000>;
+			regulator-max-microvolt = <2856000>;
+		};
+
+		vreg_l22a: l22 {
+			regulator-min-microvolt = <2944000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+
+		vreg_l23a: l23 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3400000>;
+		};
+
+		vreg_l24a: l24 {
+			regulator-min-microvolt = <2944000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+
+	};
+};
+
+&sdc2_off_state {
+	sd-cd-pins {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&sdc2_on_state {
+	sd-cd-pins {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&vreg_l24a>;
+	vqmmc-supply = <&vreg_l11a>;
+	status = "okay";
+};
+
+&sdhc_2 {
+	cd-gpios = <&tlmm 98 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&vreg_l22a>;
+	vqmmc-supply = <&vreg_l5a>;
+	no-sdio;
+	no-mmc;
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <0 4>, <30 4>;
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&usb3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts
index 6b68e391cf3ea..496f33e9d73cc 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts
@@ -2,294 +2,11 @@
 /*
  * Copyright (c) 2025, Gabriel Gonzales <semfault@disroot.org>
  */
-
 /dts-v1/;
 
-#include <dt-bindings/arm/qcom,ids.h>
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/input/input.h>
-#include <dt-bindings/input/gpio-keys.h>
-#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
-#include "sm6125.dtsi"
-#include "pm6125.dtsi"
+#include "sm6125-xiaomi-ginkgo-common.dtsi"
 
 / {
 	model = "Xiaomi Redmi Note 8";
 	compatible = "xiaomi,ginkgo", "qcom,sm6125";
-	chassis-type = "handset";
-
-	/* required for bootloader to select correct board */
-	qcom,msm-id = <QCOM_ID_SM6125 0x10000>;
-	qcom,board-id = <22 0>;
-
-	chosen {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
-		framebuffer0: framebuffer@5c000000 {
-			compatible = "simple-framebuffer";
-			reg = <0 0x5c000000 0 (2340 * 1080 * 4)>;
-			width = <1080>;
-			height = <2340>;
-			stride = <(1080 * 4)>;
-			format = "a8r8g8b8";
-		};
-	};
-
-	reserved-memory {
-		debug_mem: debug@ffb00000 {
-			reg = <0x0 0xffb00000 0x0 0xc0000>;
-			no-map;
-		};
-
-		last_log_mem: lastlog@ffbc0000 {
-			reg = <0x0 0xffbc0000 0x0 0x80000>;
-			no-map;
-		};
-
-		pstore_mem: ramoops@ffc00000 {
-			compatible = "ramoops";
-			reg = <0x0 0xffc40000 0x0 0xc0000>;
-			record-size = <0x1000>;
-			console-size = <0x40000>;
-			pmsg-size = <0x20000>;
-		};
-
-		cmdline_mem: memory@ffd00000 {
-			reg = <0x0 0xffd40000 0x0 0x1000>;
-			no-map;
-		};
-	};
-
-	extcon_usb: extcon-usb {
-		compatible = "linux,extcon-usb-gpio";
-		id-gpios = <&tlmm 102 GPIO_ACTIVE_HIGH>;
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-
-		pinctrl-0 = <&vol_up_n>;
-		pinctrl-names = "default";
-
-		key-volume-up {
-			label = "Volume Up";
-			gpios = <&pm6125_gpios 6 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_VOLUMEUP>;
-			debounce-interval = <15>;
-			linux,can-disable;
-			wakeup-source;
-		};
-	};
-};
-
-&pm6125_gpios {
-	vol_up_n: vol-up-n-state {
-		pins = "gpio6";
-		function = "normal";
-		power-source = <1>;
-		bias-pull-up;
-		input-enable;
-	};
-};
-
-&hsusb_phy1 {
-	vdd-supply = <&vreg_l7a>;
-	vdda-pll-supply = <&vreg_l10a>;
-	vdda-phy-dpdm-supply = <&vreg_l15a>;
-	status = "okay";
-};
-
-&pon_pwrkey {
-	status = "okay";
-};
-
-&pon_resin {
-	linux,code = <KEY_VOLUMEDOWN>;
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators-0 {
-		compatible = "qcom,rpm-pm6125-regulators";
-
-		vreg_s6a: s6 {
-			regulator-min-microvolt = <936000>;
-			regulator-max-microvolt = <1422000>;
-		};
-
-		vreg_l1a: l1 {
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1256000>;
-		};
-
-		vreg_l2a: l2 {
-			regulator-min-microvolt = <1000000>;
-			regulator-max-microvolt = <1056000>;
-		};
-
-		vreg_l3a: l3 {
-			regulator-min-microvolt = <1000000>;
-			regulator-max-microvolt = <1064000>;
-		};
-
-		vreg_l4a: l4 {
-			regulator-min-microvolt = <872000>;
-			regulator-max-microvolt = <976000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l5a: l5 {
-			regulator-min-microvolt = <1648000>;
-			regulator-max-microvolt = <2950000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l6a: l6 {
-			regulator-min-microvolt = <576000>;
-			regulator-max-microvolt = <656000>;
-		};
-
-		vreg_l7a: l7 {
-			regulator-min-microvolt = <872000>;
-			regulator-max-microvolt = <976000>;
-		};
-
-		vreg_l8a: l8 {
-			regulator-min-microvolt = <400000>;
-			regulator-max-microvolt = <728000>;
-		};
-
-		vreg_l9a: l9 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1896000>;
-		};
-
-		vreg_l10a: l10 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1896000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l11a: l11 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1952000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l12a: l12 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1996000>;
-		};
-
-		vreg_l13a: l13 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1832000>;
-		};
-
-		vreg_l14a: l14 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1904000>;
-		};
-
-		vreg_l15a: l15 {
-			regulator-min-microvolt = <3104000>;
-			regulator-max-microvolt = <3232000>;
-		};
-
-		vreg_l16a: l16 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1904000>;
-		};
-
-		vreg_l17a: l17 {
-			regulator-min-microvolt = <1248000>;
-			regulator-max-microvolt = <1304000>;
-		};
-
-		vreg_l18a: l18 {
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1264000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l19a: l19 {
-			regulator-min-microvolt = <1648000>;
-			regulator-max-microvolt = <2952000>;
-		};
-
-		vreg_l20a: l20 {
-			regulator-min-microvolt = <1648000>;
-			regulator-max-microvolt = <2952000>;
-		};
-
-		vreg_l21a: l21 {
-			regulator-min-microvolt = <2600000>;
-			regulator-max-microvolt = <2856000>;
-		};
-
-		vreg_l22a: l22 {
-			regulator-min-microvolt = <2944000>;
-			regulator-max-microvolt = <2950000>;
-			regulator-allow-set-load;
-		};
-
-		vreg_l23a: l23 {
-			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3400000>;
-		};
-
-		vreg_l24a: l24 {
-			regulator-min-microvolt = <2944000>;
-			regulator-max-microvolt = <2950000>;
-			regulator-allow-set-load;
-		};
-
-	};
-};
-
-&sdc2_off_state {
-	sd-cd-pins {
-		pins = "gpio98";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-disable;
-	};
-};
-
-&sdc2_on_state {
-	sd-cd-pins {
-		pins = "gpio98";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-};
-
-&sdhc_1 {
-	vmmc-supply = <&vreg_l24a>;
-	vqmmc-supply = <&vreg_l11a>;
-	status = "okay";
-};
-
-&sdhc_2 {
-	cd-gpios = <&tlmm 98 GPIO_ACTIVE_HIGH>;
-	vmmc-supply = <&vreg_l22a>;
-	vqmmc-supply = <&vreg_l5a>;
-	no-sdio;
-	no-mmc;
-	status = "okay";
-};
-
-&tlmm {
-	gpio-reserved-ranges = <22 2>, <28 6>;
-};
-
-&usb3 {
-	status = "okay";
-};
-
-&usb3_dwc3 {
-	extcon = <&extcon_usb>;
 };
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
index 994fb0412fcbd..97f64cb5d570e 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
@@ -82,6 +82,19 @@ key-volume-up {
 		};
 	};
 
+	ts_vdd_supply: regulator-ts-vdd {
+		compatible = "regulator-fixed";
+		regulator-name = "ts_vdd_supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <70000>;
+
+		enable-active-high;
+		gpio = <&tlmm 83 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&ts_vdd_en>;
+		pinctrl-names = "default";
+	};
+
 	thermal-zones {
 		rf-pa0-thermal {
 			thermal-sensors = <&pm6125_adc_tm 0>;
@@ -128,6 +141,27 @@ &hsusb_phy1 {
 	status = "okay";
 };
 
+&i2c2 {
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "focaltech,ft3518";
+		reg = <0x38>;
+		interrupts-extended = <&tlmm 88 IRQ_TYPE_EDGE_FALLING>;
+
+		vcc-supply = <&ts_vdd_supply>;
+
+		pinctrl-0 = <&ts_int_active &ts_reset_active>;
+		pinctrl-1 = <&ts_int_suspend &ts_reset_suspend>;
+		pinctrl-names = "default","sleep";
+
+		reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+
+		touchscreen-size-x = <720>;
+		touchscreen-size-y = <1560>;
+	};
+};
+
 &pm6125_adc {
 	pinctrl-names = "default";
 	pinctrl-0 = <&camera_flash_therm &emmc_ufs_therm>;
@@ -220,6 +254,10 @@ &pon_resin {
 	status = "okay";
 };
 
+&qupv3_id_0 {
+	status = "okay";
+};
+
 &rpm_requests {
 	regulators-0 {
 		compatible = "qcom,rpm-pm6125-regulators";
@@ -387,6 +425,41 @@ &sdhc_2 {
 
 &tlmm {
 	gpio-reserved-ranges = <22 2>, <28 6>;
+
+	ts_vdd_en: ts-vdd-default-state {
+		pins = "gpio83";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	ts_reset_active: pmx-ts-reset-active-state {
+		pins = "gpio87";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	ts_reset_suspend: pmx-ts-reset-suspend-state {
+		pins = "gpio87";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	ts_int_active: pmx-ts-int-active-state {
+		pins = "gpio88";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	ts_int_suspend: pmx-ts-int-suspend-state {
+		pins = "gpio88";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
 };
 
 &ufs_mem_hc {
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-willow.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-willow.dts
new file mode 100644
index 0000000000000..1231e440ba2cd
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-willow.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, Barnabas Czeman
+ */
+/dts-v1/;
+
+#include "sm6125-xiaomi-ginkgo-common.dtsi"
+
+/ {
+	model = "Xiaomi Redmi Note 8T";
+	compatible = "xiaomi,willow", "qcom,sm6125";
+
+};
+
+/* Difference from Redmi Note 8 it have NFC */
diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi
index 80c42dff5399b..c84911a98fce9 100644
--- a/arch/arm64/boot/dts/qcom/sm6125.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi
@@ -661,6 +661,13 @@ qup_spi9_sleep: qup-spi9-sleep-state {
 				drive-strength = <6>;
 				bias-disable;
 			};
+
+			qup_uart4_default: qup-uart4-default-state {
+				pins = "gpio16", "gpio17";
+				function = "qup04";
+				drive-strength = <2>;
+				bias-disable;
+			};
 		};
 
 		gcc: clock-controller@1400000 {
@@ -686,6 +693,13 @@ hsusb_phy1: phy@1613000 {
 			status = "disabled";
 		};
 
+		rng: rng@1b53000 {
+			compatible = "qcom,prng-ee";
+			reg = <0x01b53000 0x1000>;
+			clocks = <&gcc GCC_PRNG_AHB_CLK>;
+			clock-names = "core";
+		};
+
 		spmi_bus: spmi@1c40000 {
 			compatible = "qcom,spmi-pmic-arb";
 			reg = <0x01c40000 0x1100>,
@@ -978,6 +992,17 @@ i2c4: i2c@4a90000 {
 				#size-cells = <0>;
 				status = "disabled";
 			};
+
+			uart4: serial@4a90000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x04a90000 0x4000>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
+				clock-names = "se";
+				interrupts = <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>;
+				pinctrl-0 = <&qup_uart4_default>;
+				pinctrl-names = "default";
+				status = "disabled";
+			};
 		};
 
 		gpi_dma1: dma-controller@4c00000 {
@@ -1238,6 +1263,8 @@ mdss: display-subsystem@5e00000 {
 				      "ahb",
 				      "core";
 
+			resets = <&dispcc DISP_CC_MDSS_CORE_BCR>;
+
 			power-domains = <&dispcc MDSS_GDSC>;
 
 			iommus = <&apps_smmu 0x400 0x0>;
@@ -1437,6 +1464,7 @@ dispcc: clock-controller@5f00000 {
 			power-domains = <&rpmpd RPMPD_VDDCX>;
 
 			#clock-cells = <1>;
+			#reset-cells = <1>;
 			#power-domain-cells = <1>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
index 9f9b9f9af0da9..fec01750e2c75 100644
--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
@@ -2346,8 +2346,8 @@ mdss_dp_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
index 97ca5275d740a..0e101096209ab 100644
--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
@@ -3954,8 +3954,8 @@ mdss_dp_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
index c7dffa4400740..01453aaa1752f 100644
--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
@@ -4826,8 +4826,8 @@ mdss_dp_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
index 5c8fe213f5e4e..28b4b79d25872 100644
--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
@@ -2925,8 +2925,8 @@ mdss_dp_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index 920a2d1c04d0c..674e888595bbb 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -3508,8 +3508,8 @@ mdss_dp0_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -5104,7 +5104,7 @@ intc: interrupt-controller@17100000 {
 
 			gic_its: msi-controller@17140000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0x0 0x17140000 0x0 0x20000>;
+				reg = <0x0 0x17140000 0x0 0x40000>;
 				msi-controller;
 				#msi-cells = <1>;
 			};
@@ -5429,9 +5429,6 @@ sdhc_2: mmc@8804000 {
 			bus-width = <4>;
 			dma-coherent;
 
-			/* Forbid SDR104/SDR50 - broken hw! */
-			sdhci-caps-mask = <0x3 0x0>;
-
 			status = "disabled";
 
 			sdhc2_opp_table: opp-table {
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index e3f93f4f412de..656d69774ead8 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -76,8 +76,8 @@ cpu0: cpu@0 {
 			power-domains = <&cpu_pd0>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 0>;
-			capacity-dmips-mhz = <1024>;
-			dynamic-power-coefficient = <100>;
+			capacity-dmips-mhz = <326>;
+			dynamic-power-coefficient = <251>;
 			#cooling-cells = <2>;
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -102,8 +102,8 @@ cpu1: cpu@100 {
 			power-domains = <&cpu_pd1>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 0>;
-			capacity-dmips-mhz = <1024>;
-			dynamic-power-coefficient = <100>;
+			capacity-dmips-mhz = <326>;
+			dynamic-power-coefficient = <251>;
 			#cooling-cells = <2>;
 			l2_100: l2-cache {
 				compatible = "cache";
@@ -123,8 +123,8 @@ cpu2: cpu@200 {
 			power-domains = <&cpu_pd2>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 0>;
-			capacity-dmips-mhz = <1024>;
-			dynamic-power-coefficient = <100>;
+			capacity-dmips-mhz = <326>;
+			dynamic-power-coefficient = <251>;
 			#cooling-cells = <2>;
 			l2_200: l2-cache {
 				compatible = "cache";
@@ -144,8 +144,8 @@ cpu3: cpu@300 {
 			power-domains = <&cpu_pd3>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 1>;
-			capacity-dmips-mhz = <1792>;
-			dynamic-power-coefficient = <270>;
+			capacity-dmips-mhz = <693>;
+			dynamic-power-coefficient = <447>;
 			#cooling-cells = <2>;
 			l2_300: l2-cache {
 				compatible = "cache";
@@ -165,8 +165,8 @@ cpu4: cpu@400 {
 			power-domains = <&cpu_pd4>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 1>;
-			capacity-dmips-mhz = <1792>;
-			dynamic-power-coefficient = <270>;
+			capacity-dmips-mhz = <693>;
+			dynamic-power-coefficient = <447>;
 			#cooling-cells = <2>;
 			l2_400: l2-cache {
 				compatible = "cache";
@@ -186,8 +186,8 @@ cpu5: cpu@500 {
 			power-domains = <&cpu_pd5>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 1>;
-			capacity-dmips-mhz = <1792>;
-			dynamic-power-coefficient = <270>;
+			capacity-dmips-mhz = <693>;
+			dynamic-power-coefficient = <447>;
 			#cooling-cells = <2>;
 			l2_500: l2-cache {
 				compatible = "cache";
@@ -207,8 +207,8 @@ cpu6: cpu@600 {
 			power-domains = <&cpu_pd6>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 1>;
-			capacity-dmips-mhz = <1792>;
-			dynamic-power-coefficient = <270>;
+			capacity-dmips-mhz = <693>;
+			dynamic-power-coefficient = <447>;
 			#cooling-cells = <2>;
 			l2_600: l2-cache {
 				compatible = "cache";
@@ -228,8 +228,8 @@ cpu7: cpu@700 {
 			power-domains = <&cpu_pd7>;
 			power-domain-names = "psci";
 			qcom,freq-domain = <&cpufreq_hw 2>;
-			capacity-dmips-mhz = <1894>;
-			dynamic-power-coefficient = <588>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <1057>;
 			#cooling-cells = <2>;
 			l2_700: l2-cache {
 				compatible = "cache";
@@ -1251,6 +1251,22 @@ &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
 				#size-cells = <0>;
 				status = "disabled";
 			};
+
+			uart15: serial@89c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0 0x0089c000 0 0x4000>;
+				clock-names = "se";
+				clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&qup_uart15_default>;
+				interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH 0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core", "qup-config";
+				status = "disabled";
+			};
 		};
 
 		i2c_master_hub_0: geniqup@9c0000 {
@@ -2503,48 +2519,56 @@ opp-680000000 {
 					opp-hz = /bits/ 64 <680000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
 					opp-peak-kBps = <16500000>;
+					qcom,opp-acd-level = <0x882e5ffd>;
 				};
 
 				opp-615000000 {
 					opp-hz = /bits/ 64 <615000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
 					opp-peak-kBps = <12449218>;
+					qcom,opp-acd-level = <0xa82f5ffd>;
 				};
 
 				opp-550000000 {
 					opp-hz = /bits/ 64 <550000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
 					opp-peak-kBps = <10687500>;
+					qcom,opp-acd-level = <0xe0285ffd>;
 				};
 
 				opp-475000000 {
 					opp-hz = /bits/ 64 <475000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
 					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xe0285ffd>;
 				};
 
 				opp-401000000 {
 					opp-hz = /bits/ 64 <401000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
 					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xc02a5ffd>;
 				};
 
 				opp-348000000 {
 					opp-hz = /bits/ 64 <348000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
 					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xe02b5ffd>;
 				};
 
 				opp-295000000 {
 					opp-hz = /bits/ 64 <295000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
 					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xe02d5ffd>;
 				};
 
 				opp-220000000 {
 					opp-hz = /bits/ 64 <220000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
 					opp-peak-kBps = <2136718>;
+					qcom,opp-acd-level = <0xc02f5ffd>;
 				};
 			};
 		};
@@ -3210,7 +3234,7 @@ sdhc_2: mmc@8804000 {
 
 			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
 				 <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>;
+				 <&bi_tcxo_div2>;
 			clock-names = "iface", "core", "xo";
 			iommus = <&apps_smmu 0x540 0>;
 			qcom,dll-config = <0x0007642c>;
@@ -3227,9 +3251,6 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			max-sd-hs-hz = <37500000>;
 			dma-coherent;
 
-			/* Forbid SDR104/SDR50 - broken hw! */
-			sdhci-caps-mask = <0x3 0>;
-
 			status = "disabled";
 
 			sdhc2_opp_table: opp-table {
@@ -3320,19 +3341,19 @@ opp-338000000 {
 
 				opp-366000000 {
 					opp-hz = /bits/ 64 <366000000>;
-					required-opps = <&rpmhpd_opp_svs_l1>,
+					required-opps = <&rpmhpd_opp_svs>,
 							<&rpmhpd_opp_svs_l1>;
 				};
 
 				opp-444000000 {
 					opp-hz = /bits/ 64 <444000000>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-533333334 {
 					opp-hz = /bits/ 64 <533333334>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo>;
 				};
 			};
@@ -5095,6 +5116,14 @@ qup_uart14_cts_rts: qup-uart14-cts-rts-state {
 				bias-pull-down;
 			};
 
+			qup_uart15_default: qup-uart15-default-state {
+				/* TX, RX */
+				pins = "gpio74", "gpio75";
+				function = "qup2_se7";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
 			sdc2_sleep: sdc2-sleep-state {
 				clk-pins {
 					pins = "sdc2_clk";
@@ -5274,7 +5303,7 @@ ppi_cluster3: interrupt-partition-3 {
 
 			gic_its: msi-controller@17140000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0 0x17140000 0 0x20000>;
+				reg = <0 0x17140000 0 0x40000>;
 				msi-controller;
 				#msi-cells = <1>;
 			};
diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi
index 357e43b907405..64a7480291d84 100644
--- a/arch/arm64/boot/dts/qcom/sm8650.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi
@@ -4957,7 +4957,7 @@ sdhc_2: mmc@8804000 {
 
 			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
 				 <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>;
+				 <&bi_tcxo_div2>;
 			clock-names = "iface",
 				      "core",
 				      "xo";
@@ -4976,9 +4976,6 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 
 			bus-width = <4>;
 
-			/* Forbid SDR104/SDR50 - broken hw! */
-			sdhci-caps-mask = <0x3 0>;
-
 			qcom,dll-config = <0x0007642c>;
 			qcom,ddr-config = <0x80040868>;
 
@@ -5236,13 +5233,13 @@ opp-196000000 {
 
 				opp-300000000 {
 					opp-hz = /bits/ 64 <300000000>;
-					required-opps = <&rpmhpd_opp_low_svs>,
+					required-opps = <&rpmhpd_opp_svs>,
 							<&rpmhpd_opp_low_svs>;
 				};
 
 				opp-380000000 {
 					opp-hz = /bits/ 64 <380000000>;
-					required-opps = <&rpmhpd_opp_svs>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_svs>;
 				};
 
@@ -5254,13 +5251,13 @@ opp-435000000 {
 
 				opp-480000000 {
 					opp-hz = /bits/ 64 <480000000>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-533333334 {
 					opp-hz = /bits/ 64 <533333334>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_turbo>;
 				};
 			};
@@ -7219,7 +7216,7 @@ ppi_cluster2: interrupt-partition-2 {
 
 			gic_its: msi-controller@17140000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0 0x17140000 0 0x20000>;
+				reg = <0 0x17140000 0 0x40000>;
 
 				msi-controller;
 				#msi-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
index cb718331496ee..3837f6785320d 100644
--- a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
@@ -54,6 +54,15 @@ wcd939x: audio-codec {
 		vdd-px-supply = <&vreg_l2i_1p2>;
 
 		#sound-dai-cells = <1>;
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			wcd_codec_headset_in: endpoint {
+				remote-endpoint = <&wcd_usbss_headset_out>;
+			};
+		};
 	};
 
 	chosen {
@@ -230,6 +239,7 @@ port@2 {
 					reg = <2>;
 
 					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&wcd_usbss_sbu_mux>;
 					};
 				};
 			};
@@ -925,6 +935,42 @@ vreg_l7n_3p3: ldo7 {
 	};
 };
 
+&i2c3 {
+	status = "okay";
+
+	wcd_usbss: typec-mux@e {
+		compatible = "qcom,wcd9395-usbss", "qcom,wcd9390-usbss";
+		reg = <0xe>;
+
+		vdd-supply = <&vreg_l15b_1p8>;
+		reset-gpios = <&tlmm 152 GPIO_ACTIVE_HIGH>;
+
+		mode-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				wcd_usbss_sbu_mux: endpoint {
+					remote-endpoint = <&pmic_glink_sbu>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				wcd_usbss_headset_out: endpoint {
+					remote-endpoint = <&wcd_codec_headset_in>;
+				};
+			};
+		};
+	};
+};
+
 &iris {
 	status = "okay";
 };
@@ -937,6 +983,56 @@ &lpass_vamacro {
 	qcom,dmic-sample-rate = <4800000>;
 };
 
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l3g_1p2>;
+
+	status = "okay";
+
+	panel@0 {
+		compatible = "novatek,nt37801";
+		reg = <0>;
+
+		reset-gpios = <&tlmm 98 GPIO_ACTIVE_LOW>;
+
+		vddio-supply = <&vreg_l12b_1p8>;
+		vci-supply = <&vreg_l13b_3p0>;
+		vdd-supply = <&vreg_l11b_1p0>;
+
+		pinctrl-0 = <&disp0_reset_n_active>, <&mdp_vsync>;
+		pinctrl-1 = <&disp0_reset_n_suspend>, <&mdp_vsync>;
+		pinctrl-names = "default", "sleep";
+
+		port {
+			panel0_in: endpoint {
+				remote-endpoint = <&mdss_dsi0_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&panel0_in>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l3i_0p88>;
+
+	status = "okay";
+};
+
 &pm8550_flash {
 	status = "okay";
 
@@ -1053,6 +1149,11 @@ &pmih0108_eusb2_repeater {
 	status = "okay";
 };
 
+&qup_i2c3_data_clk {
+	/* Use internal I2C pull-up */
+	bias-pull-up = <2200>;
+};
+
 &qupv3_1 {
 	status = "okay";
 };
@@ -1225,6 +1326,27 @@ sdc2_card_det_n: sd-card-det-n-state {
 		bias-pull-up;
 	};
 
+	mdp_vsync: mdp-vsync-state {
+		pins = "gpio86";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	disp0_reset_n_active: disp0-reset-n-active-state {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	disp0_reset_n_suspend: disp0-reset-n-suspend-state {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
 	wcd_default: wcd-reset-n-active-state {
 		pins = "gpio101";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi
index f56b1f889b857..8b1b51558dcb5 100644
--- a/arch/arm64/boot/dts/qcom/sm8750.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi
@@ -3,7 +3,9 @@
  * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
+#include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/clock/qcom,sm8750-dispcc.h>
 #include <dt-bindings/clock/qcom,sm8750-gcc.h>
 #include <dt-bindings/clock/qcom,sm8750-tcsr.h>
 #include <dt-bindings/clock/qcom,sm8750-videocc.h>
@@ -2787,6 +2789,7 @@ port@2 {
 					reg = <2>;
 
 					usb_dp_qmpphy_dp_in: endpoint {
+						remote-endpoint = <&mdss_dp0_out>;
 					};
 				};
 			};
@@ -2945,19 +2948,19 @@ iris_opp_table: opp-table {
 
 				opp-240000000 {
 					opp-hz = /bits/ 64 <240000000>;
-					required-opps = <&rpmhpd_opp_low_svs_d1>,
+					required-opps = <&rpmhpd_opp_svs>,
 							<&rpmhpd_opp_low_svs_d1>;
 				};
 
 				opp-338000000 {
 					opp-hz = /bits/ 64 <338000000>;
-					required-opps = <&rpmhpd_opp_low_svs>,
+					required-opps = <&rpmhpd_opp_svs>,
 							<&rpmhpd_opp_low_svs>;
 				};
 
 				opp-420000000 {
 					opp-hz = /bits/ 64 <420000000>;
-					required-opps = <&rpmhpd_opp_svs>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_svs>;
 				};
 
@@ -2969,19 +2972,19 @@ opp-444000000 {
 
 				opp-533333334 {
 					opp-hz = /bits/ 64 <533333334>;
-					required-opps = <&rpmhpd_opp_nom>,
+					required-opps = <&rpmhpd_opp_svs_l1>,
 							<&rpmhpd_opp_nom>;
 				};
 
 				opp-570000000 {
 					opp-hz = /bits/ 64 <570000000>;
-					required-opps = <&rpmhpd_opp_nom_l1>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_nom_l1>;
 				};
 
 				opp-630000000 {
 					opp-hz = /bits/ 64 <630000000>;
-					required-opps = <&rpmhpd_opp_turbo>,
+					required-opps = <&rpmhpd_opp_nom>,
 							<&rpmhpd_opp_turbo>;
 				};
 			};
@@ -3001,6 +3004,437 @@ videocc: clock-controller@aaf0000 {
 			#power-domain-cells = <1>;
 		};
 
+		mdss: display-subsystem@ae00000 {
+			compatible = "qcom,sm8750-mdss";
+			reg = <0x0 0x0ae00000 0x0 0x1000>;
+			reg-names = "mdss";
+
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+				 <&gcc GCC_DISP_HF_AXI_CLK>,
+				 <&dispcc DISP_CC_MDSS_MDP_CLK>;
+
+			resets = <&dispcc DISP_CC_MDSS_CORE_BCR>;
+
+			interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "mdp0-mem",
+					     "cpu-cfg";
+
+			power-domains = <&dispcc MDSS_GDSC>;
+
+			iommus = <&apps_smmu 0x800 0x2>;
+
+			interrupt-controller;
+			#interrupt-cells = <1>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			status = "disabled";
+
+			mdss_mdp: display-controller@ae01000 {
+				compatible = "qcom,sm8750-dpu";
+				reg = <0x0 0x0ae01000 0x0 0x93000>,
+				      <0x0 0x0aeb0000 0x0 0x2008>;
+				reg-names = "mdp",
+					    "vbif";
+
+				interrupts-extended = <&mdss 0>;
+
+				clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+					 <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc DISP_CC_MDSS_MDP_LUT_CLK>,
+					 <&dispcc DISP_CC_MDSS_MDP_CLK>,
+					 <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+				clock-names = "nrt_bus",
+					      "iface",
+					      "lut",
+					      "core",
+					      "vsync";
+
+				assigned-clocks = <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+				assigned-clock-rates = <19200000>;
+
+				operating-points-v2 = <&mdp_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						dpu_intf1_out: endpoint {
+							remote-endpoint = <&mdss_dsi0_in>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						dpu_intf2_out: endpoint {
+							remote-endpoint = <&mdss_dsi1_in>;
+						};
+					};
+
+					port@2 {
+						reg = <2>;
+
+						dpu_intf0_out: endpoint {
+							remote-endpoint = <&mdss_dp0_in>;
+						};
+					};
+				};
+
+				mdp_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-156000000 {
+						opp-hz = /bits/ 64 <156000000>;
+						required-opps = <&rpmhpd_opp_low_svs_d1>;
+					};
+
+					opp-207000000 {
+						opp-hz = /bits/ 64 <207000000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-337000000 {
+						opp-hz = /bits/ 64 <337000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-417000000 {
+						opp-hz = /bits/ 64 <417000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+
+					opp-532000000 {
+						opp-hz = /bits/ 64 <532000000>;
+						required-opps = <&rpmhpd_opp_nom>;
+					};
+
+					opp-575000000 {
+						opp-hz = /bits/ 64 <575000000>;
+						required-opps = <&rpmhpd_opp_nom_l1>;
+					};
+				};
+			};
+
+			mdss_dsi0: dsi@ae94000 {
+				compatible = "qcom,sm8750-dsi-ctrl", "qcom,mdss-dsi-ctrl";
+				reg = <0x0 0x0ae94000 0x0 0x400>;
+				reg-names = "dsi_ctrl";
+
+				interrupts-extended = <&mdss 4>;
+
+				clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+					 <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+					 <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+					 <&dispcc DISP_CC_MDSS_ESC0_CLK>,
+					 <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_DISP_HF_AXI_CLK>,
+					 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>,
+					 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+					 <&dispcc DISP_CC_ESYNC0_CLK>,
+					 <&dispcc DISP_CC_OSC_CLK>,
+					 <&dispcc DISP_CC_MDSS_BYTE0_CLK_SRC>,
+					 <&dispcc DISP_CC_MDSS_PCLK0_CLK_SRC>;
+				clock-names = "byte",
+					      "byte_intf",
+					      "pixel",
+					      "core",
+					      "iface",
+					      "bus",
+					      "dsi_pll_pixel",
+					      "dsi_pll_byte",
+					      "esync",
+					      "osc",
+					      "byte_src",
+					      "pixel_src";
+
+				operating-points-v2 = <&mdss_dsi_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+				phys = <&mdss_dsi0_phy>;
+				phy-names = "dsi";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss_dsi0_in: endpoint {
+							remote-endpoint = <&dpu_intf1_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss_dsi0_out: endpoint {
+						};
+					};
+				};
+
+				mdss_dsi_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-140630000 {
+						opp-hz = /bits/ 64 <140630000>;
+						required-opps = <&rpmhpd_opp_low_svs_d1>;
+					};
+
+					opp-187500000 {
+						opp-hz = /bits/ 64 <187500000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-300000000 {
+						opp-hz = /bits/ 64 <300000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-358000000 {
+						opp-hz = /bits/ 64 <358000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+				};
+			};
+
+			mdss_dsi0_phy: phy@ae95000 {
+				compatible = "qcom,sm8750-dsi-phy-3nm";
+				reg = <0x0 0x0ae95000 0x0 0x200>,
+				      <0x0 0x0ae95200 0x0 0x280>,
+				      <0x0 0x0ae95500 0x0 0x400>;
+				reg-names = "dsi_phy",
+					    "dsi_phy_lane",
+					    "dsi_pll";
+
+				clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&bi_tcxo_div2>;
+				clock-names = "iface",
+					      "ref";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				status = "disabled";
+			};
+
+			mdss_dsi1: dsi@ae96000 {
+				compatible = "qcom,sm8750-dsi-ctrl", "qcom,mdss-dsi-ctrl";
+				reg = <0x0 0x0ae96000 0x0 0x400>;
+				reg-names = "dsi_ctrl";
+
+				interrupts-extended = <&mdss 5>;
+
+				clocks = <&dispcc DISP_CC_MDSS_BYTE1_CLK>,
+					 <&dispcc DISP_CC_MDSS_BYTE1_INTF_CLK>,
+					 <&dispcc DISP_CC_MDSS_PCLK1_CLK>,
+					 <&dispcc DISP_CC_MDSS_ESC1_CLK>,
+					 <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_DISP_HF_AXI_CLK>,
+					 <&mdss_dsi1_phy DSI_PIXEL_PLL_CLK>,
+					 <&mdss_dsi1_phy DSI_BYTE_PLL_CLK>,
+					 <&dispcc DISP_CC_ESYNC1_CLK>,
+					 <&dispcc DISP_CC_OSC_CLK>,
+					 <&dispcc DISP_CC_MDSS_BYTE1_CLK_SRC>,
+					 <&dispcc DISP_CC_MDSS_PCLK1_CLK_SRC>;
+				clock-names = "byte",
+					      "byte_intf",
+					      "pixel",
+					      "core",
+					      "iface",
+					      "bus",
+					      "dsi_pll_pixel",
+					      "dsi_pll_byte",
+					      "esync",
+					      "osc",
+					      "byte_src",
+					      "pixel_src";
+
+				operating-points-v2 = <&mdss_dsi_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+				phys = <&mdss_dsi1_phy>;
+				phy-names = "dsi";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss_dsi1_in: endpoint {
+							remote-endpoint = <&dpu_intf2_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss_dsi1_out: endpoint {
+						};
+					};
+				};
+			};
+
+			mdss_dsi1_phy: phy@ae97000 {
+				compatible = "qcom,sm8750-dsi-phy-3nm";
+				reg = <0x0 0x0ae97000 0x0 0x200>,
+				      <0x0 0x0ae97200 0x0 0x280>,
+				      <0x0 0x0ae97500 0x0 0x400>;
+				reg-names = "dsi_phy",
+					    "dsi_phy_lane",
+					    "dsi_pll";
+
+				clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&rpmhcc RPMH_CXO_CLK>;
+				clock-names = "iface",
+					      "ref";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				status = "disabled";
+			};
+
+			mdss_dp0: displayport-controller@af54000 {
+				compatible = "qcom,sm8750-dp", "qcom,sm8650-dp";
+				reg = <0x0 0xaf54000 0x0 0x104>,
+				      <0x0 0xaf54200 0x0 0xc0>,
+				      <0x0 0xaf55000 0x0 0x770>,
+				      <0x0 0xaf56000 0x0 0x9c>,
+				      <0x0 0xaf57000 0x0 0x9c>;
+
+				interrupts-extended = <&mdss 12>;
+
+				clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc DISP_CC_MDSS_DPTX0_AUX_CLK>,
+					 <&dispcc DISP_CC_MDSS_DPTX0_LINK_CLK>,
+					 <&dispcc DISP_CC_MDSS_DPTX0_LINK_INTF_CLK>,
+					 <&dispcc DISP_CC_MDSS_DPTX0_PIXEL0_CLK>,
+					 <&dispcc DISP_CC_MDSS_DPTX0_PIXEL1_CLK>;
+				clock-names = "core_iface",
+					      "core_aux",
+					      "ctrl_link",
+					      "ctrl_link_iface",
+					      "stream_pixel",
+					      "stream_1_pixel";
+
+				assigned-clocks = <&dispcc DISP_CC_MDSS_DPTX0_LINK_CLK_SRC>,
+						  <&dispcc DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC>,
+						  <&dispcc DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC>;
+				assigned-clock-parents = <&usb_dp_qmpphy QMP_USB43DP_DP_LINK_CLK>,
+							 <&usb_dp_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>,
+							 <&usb_dp_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>;
+
+				operating-points-v2 = <&dp_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_MMCX>;
+
+				phys = <&usb_dp_qmpphy QMP_USB43DP_DP_PHY>;
+				phy-names = "dp";
+
+				#sound-dai-cells = <0>;
+
+				status = "disabled";
+
+				dp_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-270000000 {
+						opp-hz = /bits/ 64 <270000000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-540000000 {
+						opp-hz = /bits/ 64 <540000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+
+					opp-810000000 {
+						opp-hz = /bits/ 64 <810000000>;
+						required-opps = <&rpmhpd_opp_nom>;
+					};
+				};
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss_dp0_in: endpoint {
+							remote-endpoint = <&dpu_intf0_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss_dp0_out: endpoint {
+							data-lanes = <0 1 2 3>;
+							remote-endpoint = <&usb_dp_qmpphy_dp_in>;
+						};
+					};
+				};
+			};
+		};
+
+		dispcc: clock-controller@af00000 {
+			compatible = "qcom,sm8750-dispcc";
+			reg = <0x0 0x0af00000 0x0 0x20000>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&gcc GCC_DISP_AHB_CLK>,
+				 <&sleep_clk>,
+				 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+				 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>,
+				 <&mdss_dsi1_phy DSI_BYTE_PLL_CLK>,
+				 <&mdss_dsi1_phy DSI_PIXEL_PLL_CLK>,
+				 <&usb_dp_qmpphy QMP_USB43DP_DP_LINK_CLK>,
+				 <&usb_dp_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>,
+				 <0>, /* dp1 */
+				 <0>,
+				 <0>, /* dp2 */
+				 <0>,
+				 <0>, /* dp3 */
+				 <0>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8750-pdc", "qcom,pdc";
 			reg = <0x0 0x0b220000 0x0 0x10000>, <0x0 0x164400f0 0x0 0x64>;
@@ -4658,7 +5092,7 @@ intc: interrupt-controller@16000000 {
 
 			gic_its: msi-controller@16040000 {
 				compatible = "arm,gic-v3-its";
-				reg = <0x0 0x16040000 0x0 0x20000>;
+				reg = <0x0 0x16040000 0x0 0x40000>;
 
 				msi-controller;
 				#msi-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/smb2370.dtsi b/arch/arm64/boot/dts/qcom/smb2370.dtsi
new file mode 100644
index 0000000000000..80f3fdae57050
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/smb2370.dtsi
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+&spmi_bus2 {
+	smb2370_j_e2: pmic@9 {
+		compatible = "qcom,smb2370", "qcom,spmi-pmic";
+		reg = <0x9 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		smb2370_j_e2_eusb2_repeater: phy@fd00 {
+			compatible = "qcom,smb2370-eusb2-repeater";
+			reg = <0xfd00>;
+			#phy-cells = <0>;
+		};
+	};
+
+	smb2370_k_e2: pmic@a {
+		compatible = "qcom,smb2370", "qcom,spmi-pmic";
+		reg = <0xa SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		smb2370_k_e2_eusb2_repeater: phy@fd00 {
+			compatible = "qcom,smb2370-eusb2-repeater";
+			reg = <0xfd00>;
+			#phy-cells = <0>;
+		};
+	};
+
+	smb2370_l_e2: pmic@b {
+		compatible = "qcom,smb2370", "qcom,spmi-pmic";
+		reg = <0xb SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		smb2370_l_e2_eusb2_repeater: phy@fd00 {
+			compatible = "qcom,smb2370-eusb2-repeater";
+			reg = <0xfd00>;
+			#phy-cells = <0>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi
index 75716b4a58d6d..e0282a5f9a6ab 100644
--- a/arch/arm64/boot/dts/qcom/talos.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos.dtsi
@@ -666,6 +666,9 @@ gcc: clock-controller@100000 {
 			clocks = <&rpmhcc RPMH_CXO_CLK>,
 				 <&rpmhcc RPMH_CXO_CLK_A>,
 				 <&sleep_clk>;
+			clock-names = "bi_tcxo",
+				      "bi_tcxo_ao",
+				      "sleep_clk";
 
 			#clock-cells = <1>;
 			#reset-cells = <1>;
@@ -2253,6 +2256,14 @@ out-ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
+				port@0 {
+					reg = <0>;
+
+					replicator0_out0: endpoint {
+						remote-endpoint = <&tmc_etr_in>;
+					};
+				};
+
 				port@1 {
 					reg = <1>;
 
@@ -2287,6 +2298,25 @@ tmc_etf_out: endpoint {
 			};
 		};
 
+		tmc@6048000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x06048000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			iommus = <&apps_smmu 0x01e0 0x0>;
+			arm,scatter-gather;
+
+			in-ports {
+				port {
+					tmc_etr_in: endpoint {
+						remote-endpoint = <&replicator0_out0>;
+					};
+				};
+			};
+		};
+
 		replicator@604a000 {
 			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
 			reg = <0x0 0x0604a000 0x0 0x1000>;
@@ -3958,8 +3988,8 @@ mdss_dp0_out: endpoint {
 				dp_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-160000000 {
-						opp-hz = /bits/ 64 <160000000>;
+					opp-162000000 {
+						opp-hz = /bits/ 64 <162000000>;
 						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
@@ -4486,9 +4516,9 @@ usb_qmpphy_2: phy@88e8000 {
 			status = "disabled";
 		};
 
-		usb_1: usb@a6f8800 {
-			compatible = "qcom,qcs615-dwc3", "qcom,dwc3";
-			reg = <0x0 0x0a6f8800 0x0 0x400>;
+		usb_1: usb@a600000 {
+			compatible = "qcom,qcs615-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x0a600000 0x0 0xfc100>;
 
 			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
 				 <&gcc GCC_USB30_PRIM_MASTER_CLK>,
@@ -4507,52 +4537,46 @@ usb_1: usb@a6f8800 {
 					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
 			assigned-clock-rates = <19200000>, <200000000>;
 
-			interrupts-extended = <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH 0>,
+			interrupts-extended = <&intc GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH 0>,
 					      <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH 0>,
 					      <&pdc 9 IRQ_TYPE_EDGE_BOTH>,
 					      <&pdc 8 IRQ_TYPE_EDGE_BOTH>,
 					      <&pdc 6 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "pwr_event",
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
 					  "hs_phy_irq",
 					  "dp_hs_phy_irq",
 					  "dm_hs_phy_irq",
 					  "ss_phy_irq";
 
+			iommus = <&apps_smmu 0x140 0x0>;
+
+			phys = <&usb_1_hsphy>, <&usb_qmpphy>;
+			phy-names = "usb2-phy", "usb3-phy";
+
 			power-domains = <&gcc USB30_PRIM_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges;
-
-			status = "disabled";
-
-			usb_1_dwc3: usb@a600000 {
-				compatible = "snps,dwc3";
-				reg = <0x0 0x0a600000 0x0 0xcd00>;
+			snps,dis-u1-entry-quirk;
+			snps,dis-u2-entry-quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,has-lpm-erratum;
+			snps,hird-threshold = /bits/ 8 <0x10>;
+			snps,usb3_lpm_capable;
 
-				iommus = <&apps_smmu 0x140 0x0>;
-				interrupts = <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH 0>;
+			wakeup-source;
 
-				phys = <&usb_1_hsphy>, <&usb_qmpphy>;
-				phy-names = "usb2-phy", "usb3-phy";
-
-				snps,dis-u1-entry-quirk;
-				snps,dis-u2-entry-quirk;
-				snps,dis_u2_susphy_quirk;
-				snps,dis_u3_susphy_quirk;
-				snps,dis_enblslpm_quirk;
-				snps,has-lpm-erratum;
-				snps,hird-threshold = /bits/ 8 <0x10>;
-				snps,usb3_lpm_capable;
-			};
+			status = "disabled";
 		};
 
-		usb_2: usb@a8f8800 {
-			compatible = "qcom,qcs615-dwc3", "qcom,dwc3";
-			reg = <0x0 0x0a8f8800 0x0 0x400>;
+		usb_2: usb@a800000 {
+			compatible = "qcom,qcs615-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x0a800000 0x0 0xfc100>;
 
 			clocks = <&gcc GCC_CFG_NOC_USB2_SEC_AXI_CLK>,
 				 <&gcc GCC_USB20_SEC_MASTER_CLK>,
@@ -4571,15 +4595,22 @@ usb_2: usb@a8f8800 {
 					  <&gcc GCC_USB20_SEC_MASTER_CLK>;
 			assigned-clock-rates = <19200000>, <200000000>;
 
-			interrupts-extended = <&intc GIC_SPI 663 IRQ_TYPE_LEVEL_HIGH 0>,
+			interrupts-extended = <&intc GIC_SPI 664 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 663 IRQ_TYPE_LEVEL_HIGH 0>,
 					      <&intc GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH 0>,
 					      <&pdc 11 IRQ_TYPE_EDGE_BOTH>,
 					      <&pdc 10 IRQ_TYPE_EDGE_BOTH>;
-			interrupt-names = "pwr_event",
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
 					  "hs_phy_irq",
 					  "dp_hs_phy_irq",
 					  "dm_hs_phy_irq";
 
+			iommus = <&apps_smmu 0xe0 0x0>;
+
+			phys = <&usb_hsphy_2>;
+			phy-names = "usb2-phy";
+
 			power-domains = <&gcc USB20_SEC_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
@@ -4587,30 +4618,16 @@ usb_2: usb@a8f8800 {
 
 			qcom,select-utmi-as-pipe-clk;
 
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges;
-
-			status = "disabled";
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,has-lpm-erratum;
+			snps,hird-threshold = /bits/ 8 <0x10>;
 
-			usb_2_dwc3: usb@a800000 {
-				compatible = "snps,dwc3";
-				reg = <0x0 0x0a800000 0x0 0xcd00>;
+			maximum-speed = "high-speed";
+			wakeup-source;
 
-				iommus = <&apps_smmu 0xe0 0x0>;
-				interrupts = <GIC_SPI 664 IRQ_TYPE_LEVEL_HIGH 0>;
-
-				phys = <&usb_hsphy_2>;
-				phy-names = "usb2-phy";
-
-				snps,dis_u2_susphy_quirk;
-				snps,dis_u3_susphy_quirk;
-				snps,dis_enblslpm_quirk;
-				snps,has-lpm-erratum;
-				snps,hird-threshold = /bits/ 8 <0x10>;
-
-				maximum-speed = "high-speed";
-			};
+			status = "disabled";
 		};
 
 		tsens0: thermal-sensor@c263000 {
diff --git a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
new file mode 100644
index 0000000000000..bae6bb4f03d82
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
@@ -0,0 +1,1362 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2024, Xilin Wu <wuxilin123@gmail.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+#include "hamoa-pmics.dtsi"
+
+/ {
+	chassis-type = "laptop";
+
+	aliases {
+		serial1 = &uart14;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
+	hdmi-bridge {
+		compatible = "parade,ps185hdm";
+
+		pinctrl-0 = <&hdmi_hpd_default>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_dp_in: endpoint {
+					remote-endpoint = <&usb_1_ss2_qmpphy_out_dp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_tmds_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_tmds_out>;
+			};
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,x1e80100-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 123 GPIO_ACTIVE_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Left-side port, closer to the screen */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss0_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss0_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss0_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss0_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss0_con_sbu_out>;
+					};
+				};
+			};
+		};
+
+		/* Left-side port, farther from the screen */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss1_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss1_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss1_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss1_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss1_con_sbu_out>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&pmc8380_5_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_1p8: regulator-rtmr0-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&pm8550ve_9_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_3p3: regulator-rtmr0-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pm8550_gpios 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_1p15: regulator-rtmr1-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&tlmm 188 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_1p8: regulator-rtmr1-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&tlmm 175 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr1_3p3: regulator-rtmr1-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR1_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 186 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb1_pwr_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	/*
+	 * TODO: These two regulators are actually part of the removable M.2
+	 * card and not the CRD mainboard. Need to describe this differently.
+	 * Functionally it works correctly, because all we need to do is to
+	 * turn on the actual 3.3V supply above.
+	 */
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_1p9: regulator-wcn-1p9 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_1P9";
+		regulator-min-microvolt = <1900000>;
+		regulator-max-microvolt = <1900000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_wcn_0p95>;
+		vdddig-supply = <&vreg_wcn_0p95>;
+		vddrfa1p2-supply = <&vreg_wcn_1p9>;
+		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_en>, <&wcn_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob2>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s5j_1p2>;
+		vdd-l15-supply = <&vreg_s4c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p0: ldo14 {
+			regulator-name = "vreg_l14b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_l3c_0p8: ldo3 {
+			regulator-name = "vreg_l3c_0p8";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s4c_1p8>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_l1d_0p8: ldo1 {
+			regulator-name = "vreg_l1d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3d_1p8: ldo3 {
+			regulator-name = "vreg_l3d_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+
+		vreg_l2e_0p8: ldo2 {
+			regulator-name = "vreg_l2e_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_s1f_0p7: smps1 {
+			regulator-name = "vreg_s1f_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "i";
+
+		vdd-l1-supply = <&vreg_s4c_1p8>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "j";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_s5j_1p2: smps5 {
+			regulator-name = "vreg_s5j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1j_0p8: ldo1 {
+			regulator-name = "vreg_l1j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2j_1p2: ldo2 {
+			regulator-name = "vreg_l2j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3j_0p8: ldo3 {
+			regulator-name = "vreg_l3j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@15 {
+		compatible = "hid-over-i2c";
+		reg = <0x15>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x08>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK3>;
+
+		vdd-supply = <&vreg_rtmr0_1p15>;
+		vdd33-supply = <&vreg_rtmr0_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr0_3p3>;
+		vddar-supply = <&vreg_rtmr0_1p15>;
+		vddat-supply = <&vreg_rtmr0_1p15>;
+		vddio-supply = <&vreg_rtmr0_1p8>;
+
+		reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr0_default>;
+		pinctrl-names = "default";
+
+		orientation-switch;
+		retimer-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss0_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss0_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss0_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	eusb5_repeater: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 7 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb5_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb3_repeater: redriver@47 {
+		compatible = "nxp,ptn3222";
+		reg = <0x47>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb3_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb6_repeater: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb6_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	/* EC @ 0x76 */
+};
+
+&i2c7 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x8>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK4>;
+
+		vdd-supply = <&vreg_rtmr1_1p15>;
+		vdd33-supply = <&vreg_rtmr1_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr1_3p3>;
+		vddar-supply = <&vreg_rtmr1_1p15>;
+		vddat-supply = <&vreg_rtmr1_1p15>;
+		vddio-supply = <&vreg_rtmr1_1p8>;
+
+		reset-gpios = <&tlmm 176 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr1_default>;
+		pinctrl-names = "default";
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss1_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss1_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss1_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss1_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss1_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss1_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "samsung,atna56ac03", "samsung,atna33xc20";
+			enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp_3p3>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l3j_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie6a {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie6a_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6a_phy {
+	vdda-phy-supply = <&vreg_l1d_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pcie6a_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+&pm8550_gpios {
+	rtmr0_default: rtmr0-reset-n-active-state {
+		pins = "gpio10";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+	usb0_3p3_reg_en: usb0-3p3-reg-en-state {
+		pins = "gpio11";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pm8550ve_9_gpios {
+	usb0_1p8_reg_en: usb0-1p8-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pmc8380_3_gpios {
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio4";
+		function = "normal";
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_MED>;
+		bias-pull-down;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pmc8380_5_gpios {
+	usb0_pwr_1p15_reg_en: usb0-pwr-1p15-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&qupv3_0 {
+	status = "okay";
+};
+
+&qupv3_1 {
+	status = "okay";
+};
+
+&qupv3_2 {
+	status = "okay";
+};
+
+&smb2360_0 {
+	status = "okay";
+};
+
+&smb2360_0_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l2b_3p0>;
+};
+
+&smb2360_1 {
+	status = "okay";
+};
+
+&smb2360_1_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l14b_3p0>;
+};
+
+&tlmm {
+	gpio-reserved-ranges = <34 2>, /* Unused */
+			       <44 4>, /* SPI (TPM) */
+			       <238 1>; /* UFS Reset */
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	eusb3_reset_n: eusb3-reset-n-state {
+		pins = "gpio6";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+		output-low;
+	};
+
+	eusb5_reset_n: eusb5-reset-n-state {
+		pins = "gpio7";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+		output-low;
+	};
+
+	eusb6_reset_n: eusb6-reset-n-state {
+		pins = "gpio184";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+		output-low;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	hdmi_hpd_default: hdmi-hpd-default-state {
+		pins = "gpio126";
+		function = "usb2_dp";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6a_default: pcie6a-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie6a_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	rtmr1_default: rtmr1-reset-n-active-state {
+		pins = "gpio176";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	usb1_pwr_1p15_reg_en: usb1-pwr-1p15-reg-en-state {
+		pins = "gpio188";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	usb1_pwr_1p8_reg_en: usb1-pwr-1p8-reg-en-state {
+		pins = "gpio175";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	usb1_pwr_3p3_reg_en: usb1-pwr-3p3-reg-en-state {
+		pins = "gpio186";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	wcn_bt_en: wcn-bt-en-state {
+		pins = "gpio116";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-pull-down;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio214";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	wcn_wlan_en: wcn-wlan-en-state {
+		pins = "gpio117";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&usb_1_ss0_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	phys = <&smb2360_0_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l1j_0p8>;
+
+	status = "okay";
+};
+
+&usb_1_ss0 {
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_1_ss0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss0_hs_in>;
+};
+
+&usb_1_ss0_qmpphy_out {
+	remote-endpoint = <&retimer_ss0_ss_in>;
+};
+
+&usb_1_ss1_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	phys = <&smb2360_1_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_ss1_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	status = "okay";
+};
+
+&usb_1_ss1 {
+	status = "okay";
+};
+
+&usb_1_ss1_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_1_ss1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss1_hs_in>;
+};
+
+&usb_1_ss1_qmpphy_out {
+	remote-endpoint = <&retimer_ss1_ss_in>;
+};
+
+&usb_1_ss2_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_1_ss2_qmpphy_out_dp: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2 1 0>;
+				remote-endpoint = <&hdmi_bridge_dp_in>;
+			};
+
+			/* No USB3 lanes connected */
+		};
+	};
+};
+
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb5_repeater>;
+
+	status = "okay";
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb3_repeater>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb6_repeater>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
index 8e5c5575a5320..d441f860fbed2 100644
--- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
@@ -63,6 +63,45 @@ switch-lid {
 		};
 	};
 
+	hdmi-bridge {
+		compatible = "parade,ps185hdm";
+
+		pinctrl-0 = <&hdmi_hpd_default>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_dp_in: endpoint {
+					remote-endpoint = <&usb_1_ss2_qmpphy_out_dp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_tmds_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_tmds_out>;
+			};
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -995,6 +1034,14 @@ &mdss_dp1_out {
 	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
 };
 
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
 &mdss_dp3 {
 	/delete-property/ #sound-dai-cells;
 
@@ -1253,6 +1300,12 @@ hall_int_n_default: hall-int-n-state {
 		bias-disable;
 	};
 
+	hdmi_hpd_default: hdmi-hpd-default-state {
+		pins = "gpio126";
+		function = "usb2_dp";
+		bias-disable;
+	};
+
 	hdtl_default: hdtl-default-state {
 		pins = "gpio95";
 		function = "gpio";
@@ -1451,6 +1504,32 @@ &usb_1_ss1_qmpphy_out {
 	remote-endpoint = <&retimer_ss1_ss_in>;
 };
 
+&usb_1_ss2_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_1_ss2_qmpphy_out_dp: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2 1 0>;
+				remote-endpoint = <&hdmi_bridge_dp_in>;
+			};
+		};
+	};
+};
+
 &usb_mp {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
index ded96fb43489b..c0c7ddabd8fcf 100644
--- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
@@ -226,6 +226,38 @@ sound {
 				"VA DMIC3", "MIC BIAS1",
 				"TX SWR_INPUT1", "ADC2_OUTPUT";
 
+		displayport-0-dai-link {
+			link-name = "DisplayPort0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		displayport-1-dai-link {
+			link-name = "DisplayPort1 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp1>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
 		wcd-playback-dai-link {
 			link-name = "WCD Playback";
 
diff --git a/arch/arm64/boot/dts/qcom/x1-el2.dtso b/arch/arm64/boot/dts/qcom/x1-el2.dtso
index 175679be01eba..ee006742d6f3b 100644
--- a/arch/arm64/boot/dts/qcom/x1-el2.dtso
+++ b/arch/arm64/boot/dts/qcom/x1-el2.dtso
@@ -52,6 +52,14 @@ &pcie_smmu {
 	status = "okay";
 };
 
+&remoteproc_adsp {
+	iommus = <&apps_smmu 0x1000 0x80>;
+};
+
+&remoteproc_cdsp {
+	iommus = <&apps_smmu 0x0c00 0x0>;
+};
+
 /*
  * The "SBSA watchdog" is implemented in software in Gunyah
  * and can't be used when running in EL2.
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts
index 17269eb0638ac..519bcbc98985a 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts
@@ -6,1024 +6,22 @@
 
 /dts-v1/;
 
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/input/gpio-keys.h>
-#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
-#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
-
 #include "hamoa.dtsi"
-#include "hamoa-pmics.dtsi"
+#include "x1-asus-vivobook-s15.dtsi"
 
 / {
 	model = "ASUS Vivobook S 15";
 	compatible = "asus,vivobook-s15", "qcom,x1e80100";
 	chassis-type = "laptop";
-
-	aliases {
-		serial1 = &uart14;
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-		pinctrl-0 = <&hall_int_n_default>;
-		pinctrl-names = "default";
-
-		switch-lid {
-			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
-			linux,input-type = <EV_SW>;
-			linux,code = <SW_LID>;
-			wakeup-source;
-			wakeup-event-action = <EV_ACT_DEASSERTED>;
-		};
-	};
-
-	hdmi-bridge {
-		compatible = "parade,ps185hdm";
-
-		pinctrl-0 = <&hdmi_hpd_default>;
-		pinctrl-names = "default";
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-
-				hdmi_bridge_dp_in: endpoint {
-					remote-endpoint = <&usb_1_ss2_qmpphy_out_dp>;
-				};
-			};
-
-			port@1 {
-				reg = <1>;
-
-				hdmi_bridge_tmds_out: endpoint {
-					remote-endpoint = <&hdmi_con>;
-				};
-			};
-		};
-	};
-
-	hdmi-connector {
-		compatible = "hdmi-connector";
-		type = "a";
-
-		port {
-			hdmi_con: endpoint {
-				remote-endpoint = <&hdmi_bridge_tmds_out>;
-			};
-		};
-	};
-
-	pmic-glink {
-		compatible = "qcom,x1e80100-pmic-glink",
-			     "qcom,sm8550-pmic-glink",
-			     "qcom,pmic-glink";
-		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>,
-				    <&tlmm 123 GPIO_ACTIVE_HIGH>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		/* Left-side port, closer to the screen */
-		connector@0 {
-			compatible = "usb-c-connector";
-			reg = <0>;
-			power-role = "dual";
-			data-role = "dual";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					pmic_glink_ss0_hs_in: endpoint {
-						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-
-					pmic_glink_ss0_ss_in: endpoint {
-						remote-endpoint = <&retimer_ss0_ss_out>;
-					};
-				};
-
-				port@2 {
-					reg = <2>;
-
-					pmic_glink_ss0_con_sbu_in: endpoint {
-						remote-endpoint = <&retimer_ss0_con_sbu_out>;
-					};
-				};
-			};
-		};
-
-		/* Left-side port, farther from the screen */
-		connector@1 {
-			compatible = "usb-c-connector";
-			reg = <1>;
-			power-role = "dual";
-			data-role = "dual";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					pmic_glink_ss1_hs_in: endpoint {
-						remote-endpoint = <&usb_1_ss1_dwc3_hs>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-
-					pmic_glink_ss1_ss_in: endpoint {
-						remote-endpoint = <&retimer_ss1_ss_out>;
-					};
-				};
-
-				port@2 {
-					reg = <2>;
-
-					pmic_glink_ss1_con_sbu_in: endpoint {
-						remote-endpoint = <&retimer_ss1_con_sbu_out>;
-					};
-				};
-			};
-		};
-	};
-
-	reserved-memory {
-		linux,cma {
-			compatible = "shared-dma-pool";
-			size = <0x0 0x8000000>;
-			reusable;
-			linux,cma-default;
-		};
-	};
-
-	vreg_edp_3p3: regulator-edp-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_EDP_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&edp_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-always-on;
-		regulator-boot-on;
-	};
-
-	vreg_nvme: regulator-nvme {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_NVME_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&tlmm 18 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&nvme_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR0_1P15";
-		regulator-min-microvolt = <1150000>;
-		regulator-max-microvolt = <1150000>;
-
-		gpio = <&pmc8380_5_gpios 8 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb0_pwr_1p15_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr0_1p8: regulator-rtmr0-1p8 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR0_1P8";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-
-		gpio = <&pm8550ve_9_gpios 8 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb0_1p8_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr0_3p3: regulator-rtmr0-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR0_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&pm8550_gpios 11 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb0_3p3_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr1_1p15: regulator-rtmr1-1p15 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR1_1P15";
-		regulator-min-microvolt = <1150000>;
-		regulator-max-microvolt = <1150000>;
-
-		gpio = <&tlmm 188 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb1_pwr_1p15_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr1_1p8: regulator-rtmr1-1p8 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR1_1P8";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-
-		gpio = <&tlmm 175 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb1_pwr_1p8_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_rtmr1_3p3: regulator-rtmr1-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_RTMR1_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&tlmm 186 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&usb1_pwr_3p3_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vph_pwr: regulator-vph-pwr {
-		compatible = "regulator-fixed";
-
-		regulator-name = "vph_pwr";
-		regulator-min-microvolt = <3700000>;
-		regulator-max-microvolt = <3700000>;
-
-		regulator-always-on;
-		regulator-boot-on;
-	};
-
-	/*
-	 * TODO: These two regulators are actually part of the removable M.2
-	 * card and not the CRD mainboard. Need to describe this differently.
-	 * Functionally it works correctly, because all we need to do is to
-	 * turn on the actual 3.3V supply above.
-	 */
-	vreg_wcn_0p95: regulator-wcn-0p95 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_0P95";
-		regulator-min-microvolt = <950000>;
-		regulator-max-microvolt = <950000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
-	vreg_wcn_1p9: regulator-wcn-1p9 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_1P9";
-		regulator-min-microvolt = <1900000>;
-		regulator-max-microvolt = <1900000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
-	vreg_wcn_3p3: regulator-wcn-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&wcn_sw_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	wcn7850-pmu {
-		compatible = "qcom,wcn7850-pmu";
-
-		vdd-supply = <&vreg_wcn_0p95>;
-		vddio-supply = <&vreg_l15b_1p8>;
-		vddaon-supply = <&vreg_wcn_0p95>;
-		vdddig-supply = <&vreg_wcn_0p95>;
-		vddrfa1p2-supply = <&vreg_wcn_1p9>;
-		vddrfa1p8-supply = <&vreg_wcn_1p9>;
-
-		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
-		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
-
-		pinctrl-0 = <&wcn_wlan_en>, <&wcn_bt_en>;
-		pinctrl-names = "default";
-
-		regulators {
-			vreg_pmu_rfa_cmn: ldo0 {
-				regulator-name = "vreg_pmu_rfa_cmn";
-			};
-
-			vreg_pmu_aon_0p59: ldo1 {
-				regulator-name = "vreg_pmu_aon_0p59";
-			};
-
-			vreg_pmu_wlcx_0p8: ldo2 {
-				regulator-name = "vreg_pmu_wlcx_0p8";
-			};
-
-			vreg_pmu_wlmx_0p85: ldo3 {
-				regulator-name = "vreg_pmu_wlmx_0p85";
-			};
-
-			vreg_pmu_btcmx_0p85: ldo4 {
-				regulator-name = "vreg_pmu_btcmx_0p85";
-			};
-
-			vreg_pmu_rfa_0p8: ldo5 {
-				regulator-name = "vreg_pmu_rfa_0p8";
-			};
-
-			vreg_pmu_rfa_1p2: ldo6 {
-				regulator-name = "vreg_pmu_rfa_1p2";
-			};
-
-			vreg_pmu_rfa_1p8: ldo7 {
-				regulator-name = "vreg_pmu_rfa_1p8";
-			};
-
-			vreg_pmu_pcie_0p9: ldo8 {
-				regulator-name = "vreg_pmu_pcie_0p9";
-			};
-
-			vreg_pmu_pcie_1p8: ldo9 {
-				regulator-name = "vreg_pmu_pcie_1p8";
-			};
-		};
-	};
-};
-
-&apps_rsc {
-	regulators-0 {
-		compatible = "qcom,pm8550-rpmh-regulators";
-		qcom,pmic-id = "b";
-
-		vdd-bob1-supply = <&vph_pwr>;
-		vdd-bob2-supply = <&vph_pwr>;
-		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
-		vdd-l2-l13-l14-supply = <&vreg_bob1>;
-		vdd-l5-l16-supply = <&vreg_bob1>;
-		vdd-l6-l7-supply = <&vreg_bob2>;
-		vdd-l8-l9-supply = <&vreg_bob1>;
-		vdd-l12-supply = <&vreg_s5j_1p2>;
-		vdd-l15-supply = <&vreg_s4c_1p8>;
-		vdd-l17-supply = <&vreg_bob2>;
-
-		vreg_bob1: bob1 {
-			regulator-name = "vreg_bob1";
-			regulator-min-microvolt = <3008000>;
-			regulator-max-microvolt = <3960000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_bob2: bob2 {
-			regulator-name = "vreg_bob2";
-			regulator-min-microvolt = <2504000>;
-			regulator-max-microvolt = <3008000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l2b_3p0: ldo2 {
-			regulator-name = "vreg_l2b_3p0";
-			regulator-min-microvolt = <3072000>;
-			regulator-max-microvolt = <3100000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l4b_1p8: ldo4 {
-			regulator-name = "vreg_l4b_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l13b_3p0: ldo13 {
-			regulator-name = "vreg_l13b_3p0";
-			regulator-min-microvolt = <3072000>;
-			regulator-max-microvolt = <3072000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l14b_3p0: ldo14 {
-			regulator-name = "vreg_l14b_3p0";
-			regulator-min-microvolt = <3072000>;
-			regulator-max-microvolt = <3072000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l15b_1p8: ldo15 {
-			regulator-name = "vreg_l15b_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-1 {
-		compatible = "qcom,pm8550ve-rpmh-regulators";
-		qcom,pmic-id = "c";
-
-		vdd-l1-supply = <&vreg_s5j_1p2>;
-		vdd-l2-supply = <&vreg_s1f_0p7>;
-		vdd-l3-supply = <&vreg_s1f_0p7>;
-		vdd-s4-supply = <&vph_pwr>;
-
-		vreg_l3c_0p8: ldo3 {
-			regulator-name = "vreg_l3c_0p8";
-			regulator-min-microvolt = <912000>;
-			regulator-max-microvolt = <912000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_s4c_1p8: smps4 {
-			regulator-name = "vreg_s4c_1p8";
-			regulator-min-microvolt = <1856000>;
-			regulator-max-microvolt = <2000000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-2 {
-		compatible = "qcom,pmc8380-rpmh-regulators";
-		qcom,pmic-id = "d";
-
-		vdd-l1-supply = <&vreg_s1f_0p7>;
-		vdd-l2-supply = <&vreg_s1f_0p7>;
-		vdd-l3-supply = <&vreg_s4c_1p8>;
-		vdd-s1-supply = <&vph_pwr>;
-
-		vreg_l1d_0p8: ldo1 {
-			regulator-name = "vreg_l1d_0p8";
-			regulator-min-microvolt = <880000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l2d_0p9: ldo2 {
-			regulator-name = "vreg_l2d_0p9";
-			regulator-min-microvolt = <912000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l3d_1p8: ldo3 {
-			regulator-name = "vreg_l3d_1p8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-3 {
-		compatible = "qcom,pmc8380-rpmh-regulators";
-		qcom,pmic-id = "e";
-
-		vdd-l2-supply = <&vreg_s1f_0p7>;
-		vdd-l3-supply = <&vreg_s5j_1p2>;
-
-		vreg_l2e_0p8: ldo2 {
-			regulator-name = "vreg_l2e_0p8";
-			regulator-min-microvolt = <880000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l3e_1p2: ldo3 {
-			regulator-name = "vreg_l3e_1p2";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1200000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-4 {
-		compatible = "qcom,pmc8380-rpmh-regulators";
-		qcom,pmic-id = "f";
-
-		vdd-l1-supply = <&vreg_s5j_1p2>;
-		vdd-l2-supply = <&vreg_s5j_1p2>;
-		vdd-l3-supply = <&vreg_s5j_1p2>;
-		vdd-s1-supply = <&vph_pwr>;
-
-		vreg_s1f_0p7: smps1 {
-			regulator-name = "vreg_s1f_0p7";
-			regulator-min-microvolt = <700000>;
-			regulator-max-microvolt = <1100000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-6 {
-		compatible = "qcom,pm8550ve-rpmh-regulators";
-		qcom,pmic-id = "i";
-
-		vdd-l1-supply = <&vreg_s4c_1p8>;
-		vdd-l2-supply = <&vreg_s5j_1p2>;
-		vdd-l3-supply = <&vreg_s1f_0p7>;
-		vdd-s1-supply = <&vph_pwr>;
-		vdd-s2-supply = <&vph_pwr>;
-
-		vreg_l3i_0p8: ldo3 {
-			regulator-name = "vreg_l3i_0p8";
-			regulator-min-microvolt = <880000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-
-	regulators-7 {
-		compatible = "qcom,pm8550ve-rpmh-regulators";
-		qcom,pmic-id = "j";
-
-		vdd-l1-supply = <&vreg_s1f_0p7>;
-		vdd-l2-supply = <&vreg_s5j_1p2>;
-		vdd-l3-supply = <&vreg_s1f_0p7>;
-		vdd-s5-supply = <&vph_pwr>;
-
-		vreg_s5j_1p2: smps5 {
-			regulator-name = "vreg_s5j_1p2";
-			regulator-min-microvolt = <1256000>;
-			regulator-max-microvolt = <1304000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l1j_0p8: ldo1 {
-			regulator-name = "vreg_l1j_0p8";
-			regulator-min-microvolt = <880000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l2j_1p2: ldo2 {
-			regulator-name = "vreg_l2j_1p2";
-			regulator-min-microvolt = <1256000>;
-			regulator-max-microvolt = <1256000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-
-		vreg_l3j_0p8: ldo3 {
-			regulator-name = "vreg_l3j_0p8";
-			regulator-min-microvolt = <880000>;
-			regulator-max-microvolt = <920000>;
-			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-		};
-	};
-};
-
-&gpu {
-	status = "okay";
 };
 
 &gpu_zap_shader {
 	firmware-name = "qcom/x1e80100/ASUSTeK/vivobook-s15/qcdxkmsuc8380.mbn";
 };
 
-&i2c0 {
-	clock-frequency = <400000>;
-	status = "okay";
-
-	touchpad@15 {
-		compatible = "hid-over-i2c";
-		reg = <0x15>;
-
-		hid-descr-addr = <0x1>;
-		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
-
-		pinctrl-0 = <&tpad_default>;
-		pinctrl-names = "default";
-
-		wakeup-source;
-	};
-};
-
-&i2c1 {
-	clock-frequency = <400000>;
-	status = "okay";
-};
-
-&i2c3 {
-	clock-frequency = <400000>;
-	status = "okay";
-
-	typec-mux@8 {
-		compatible = "parade,ps8830";
-		reg = <0x08>;
-
-		clocks = <&rpmhcc RPMH_RF_CLK3>;
-
-		vdd-supply = <&vreg_rtmr0_1p15>;
-		vdd33-supply = <&vreg_rtmr0_3p3>;
-		vdd33-cap-supply = <&vreg_rtmr0_3p3>;
-		vddar-supply = <&vreg_rtmr0_1p15>;
-		vddat-supply = <&vreg_rtmr0_1p15>;
-		vddio-supply = <&vreg_rtmr0_1p8>;
-
-		reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>;
-
-		pinctrl-0 = <&rtmr0_default>;
-		pinctrl-names = "default";
-
-		orientation-switch;
-		retimer-switch;
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-
-				retimer_ss0_ss_out: endpoint {
-					remote-endpoint = <&pmic_glink_ss0_ss_in>;
-				};
-			};
-
-			port@1 {
-				reg = <1>;
-
-				retimer_ss0_ss_in: endpoint {
-					remote-endpoint = <&usb_1_ss0_qmpphy_out>;
-				};
-			};
-
-			port@2 {
-				reg = <2>;
-
-				retimer_ss0_con_sbu_out: endpoint {
-					remote-endpoint = <&pmic_glink_ss0_con_sbu_in>;
-				};
-			};
-		};
-	};
-};
-
-&i2c5 {
-	clock-frequency = <400000>;
-	status = "okay";
-
-	keyboard@3a {
-		compatible = "hid-over-i2c";
-		reg = <0x3a>;
-
-		hid-descr-addr = <0x1>;
-		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
-
-		pinctrl-0 = <&kybd_default>;
-		pinctrl-names = "default";
-
-		wakeup-source;
-	};
-
-	eusb5_repeater: redriver@43 {
-		compatible = "nxp,ptn3222";
-		reg = <0x43>;
-		#phy-cells = <0>;
-
-		vdd3v3-supply = <&vreg_l13b_3p0>;
-		vdd1v8-supply = <&vreg_l4b_1p8>;
-
-		reset-gpios = <&tlmm 7 GPIO_ACTIVE_LOW>;
-
-		pinctrl-0 = <&eusb5_reset_n>;
-		pinctrl-names = "default";
-	};
-
-	eusb3_repeater: redriver@47 {
-		compatible = "nxp,ptn3222";
-		reg = <0x47>;
-		#phy-cells = <0>;
-
-		vdd3v3-supply = <&vreg_l13b_3p0>;
-		vdd1v8-supply = <&vreg_l4b_1p8>;
-
-		reset-gpios = <&tlmm 6 GPIO_ACTIVE_LOW>;
-
-		pinctrl-0 = <&eusb3_reset_n>;
-		pinctrl-names = "default";
-	};
-
-	eusb6_repeater: redriver@4f {
-		compatible = "nxp,ptn3222";
-		reg = <0x4f>;
-		#phy-cells = <0>;
-
-		vdd3v3-supply = <&vreg_l13b_3p0>;
-		vdd1v8-supply = <&vreg_l4b_1p8>;
-
-		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
-
-		pinctrl-0 = <&eusb6_reset_n>;
-		pinctrl-names = "default";
-	};
-
-	/* EC @ 0x76 */
-};
-
-&i2c7 {
-	clock-frequency = <400000>;
-	status = "okay";
-
-	typec-mux@8 {
-		compatible = "parade,ps8830";
-		reg = <0x8>;
-
-		clocks = <&rpmhcc RPMH_RF_CLK4>;
-
-		vdd-supply = <&vreg_rtmr1_1p15>;
-		vdd33-supply = <&vreg_rtmr1_3p3>;
-		vdd33-cap-supply = <&vreg_rtmr1_3p3>;
-		vddar-supply = <&vreg_rtmr1_1p15>;
-		vddat-supply = <&vreg_rtmr1_1p15>;
-		vddio-supply = <&vreg_rtmr1_1p8>;
-
-		reset-gpios = <&tlmm 176 GPIO_ACTIVE_LOW>;
-
-		pinctrl-0 = <&rtmr1_default>;
-		pinctrl-names = "default";
-
-		retimer-switch;
-		orientation-switch;
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-
-				retimer_ss1_ss_out: endpoint {
-					remote-endpoint = <&pmic_glink_ss1_ss_in>;
-				};
-			};
-
-			port@1 {
-				reg = <1>;
-
-				retimer_ss1_ss_in: endpoint {
-					remote-endpoint = <&usb_1_ss1_qmpphy_out>;
-				};
-			};
-
-			port@2 {
-				reg = <2>;
-
-				retimer_ss1_con_sbu_out: endpoint {
-					remote-endpoint = <&pmic_glink_ss1_con_sbu_in>;
-				};
-			};
-		};
-	};
-};
-
 &iris {
 	firmware-name = "qcom/x1e80100/ASUSTeK/vivobook-s15/qcvss8380.mbn";
-	status = "okay";
-};
 
-&mdss {
-	status = "okay";
-};
-
-&mdss_dp0 {
-	status = "okay";
-};
-
-&mdss_dp0_out {
-	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
-};
-
-&mdss_dp1 {
-	status = "okay";
-};
-
-&mdss_dp1_out {
-	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
-};
-
-&mdss_dp2 {
-	status = "okay";
-};
-
-&mdss_dp2_out {
-	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
-};
-
-&mdss_dp3 {
-	/delete-property/ #sound-dai-cells;
-
-	pinctrl-0 = <&edp0_hpd_default>;
-	pinctrl-names = "default";
-
-	status = "okay";
-
-	aux-bus {
-		panel {
-			compatible = "samsung,atna56ac03", "samsung,atna33xc20";
-			enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>;
-			power-supply = <&vreg_edp_3p3>;
-
-			pinctrl-0 = <&edp_bl_en>;
-			pinctrl-names = "default";
-
-			port {
-				edp_panel_in: endpoint {
-					remote-endpoint = <&mdss_dp3_out>;
-				};
-			};
-		};
-	};
-};
-
-&mdss_dp3_out {
-	data-lanes = <0 1 2 3>;
-	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
-
-	remote-endpoint = <&edp_panel_in>;
-};
-
-&mdss_dp3_phy {
-	vdda-phy-supply = <&vreg_l3j_0p8>;
-	vdda-pll-supply = <&vreg_l2j_1p2>;
-
-	status = "okay";
-};
-
-&pcie4 {
-	pinctrl-0 = <&pcie4_default>;
-	pinctrl-names = "default";
-
-	status = "okay";
-};
-
-&pcie4_phy {
-	vdda-phy-supply = <&vreg_l3i_0p8>;
-	vdda-pll-supply = <&vreg_l3e_1p2>;
-
-	status = "okay";
-};
-
-&pcie4_port0 {
-	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
-
-	wifi@0 {
-		compatible = "pci17cb,1107";
-		reg = <0x10000 0x0 0x0 0x0 0x0>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
-		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
-	};
-};
-
-&pcie6a {
-	vddpe-3v3-supply = <&vreg_nvme>;
-
-	pinctrl-0 = <&pcie6a_default>;
-	pinctrl-names = "default";
-
-	status = "okay";
-};
-
-&pcie6a_phy {
-	vdda-phy-supply = <&vreg_l1d_0p8>;
-	vdda-pll-supply = <&vreg_l2j_1p2>;
-
-	status = "okay";
-};
-
-&pcie6a_port0 {
-	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
-};
-
-&pm8550_gpios {
-	rtmr0_default: rtmr0-reset-n-active-state {
-		pins = "gpio10";
-		function = "normal";
-		power-source = <1>; /* 1.8V */
-		bias-disable;
-		input-disable;
-		output-enable;
-	};
-
-	usb0_3p3_reg_en: usb0-3p3-reg-en-state {
-		pins = "gpio11";
-		function = "normal";
-		power-source = <1>; /* 1.8V */
-		bias-disable;
-		input-disable;
-		output-enable;
-	};
-};
-
-&pm8550ve_9_gpios {
-	usb0_1p8_reg_en: usb0-1p8-reg-en-state {
-		pins = "gpio8";
-		function = "normal";
-		power-source = <1>; /* 1.8V */
-		bias-disable;
-		input-disable;
-		output-enable;
-	};
-};
-
-&pmc8380_3_gpios {
-	edp_bl_en: edp-bl-en-state {
-		pins = "gpio4";
-		function = "normal";
-		power-source = <1>; /* 1.8 V */
-		qcom,drive-strength = <PMIC_GPIO_STRENGTH_MED>;
-		bias-pull-down;
-		input-disable;
-		output-enable;
-	};
-};
-
-&pmc8380_5_gpios {
-	usb0_pwr_1p15_reg_en: usb0-pwr-1p15-reg-en-state {
-		pins = "gpio8";
-		function = "normal";
-		power-source = <1>; /* 1.8V */
-		bias-disable;
-		input-disable;
-		output-enable;
-	};
-};
-
-&qupv3_0 {
-	status = "okay";
-};
-
-&qupv3_1 {
-	status = "okay";
-};
-
-&qupv3_2 {
 	status = "okay";
 };
 
@@ -1041,345 +39,3 @@ &remoteproc_cdsp {
 	status = "okay";
 };
 
-&smb2360_0 {
-	status = "okay";
-};
-
-&smb2360_0_eusb2_repeater {
-	vdd18-supply = <&vreg_l3d_1p8>;
-	vdd3-supply = <&vreg_l2b_3p0>;
-};
-
-&smb2360_1 {
-	status = "okay";
-};
-
-&smb2360_1_eusb2_repeater {
-	vdd18-supply = <&vreg_l3d_1p8>;
-	vdd3-supply = <&vreg_l14b_3p0>;
-};
-
-&tlmm {
-	gpio-reserved-ranges = <34 2>, /* Unused */
-			       <44 4>, /* SPI (TPM) */
-			       <238 1>; /* UFS Reset */
-
-	edp_reg_en: edp-reg-en-state {
-		pins = "gpio70";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-disable;
-	};
-
-	eusb3_reset_n: eusb3-reset-n-state {
-		pins = "gpio6";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-		output-low;
-	};
-
-	eusb5_reset_n: eusb5-reset-n-state {
-		pins = "gpio7";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-		output-low;
-	};
-
-	eusb6_reset_n: eusb6-reset-n-state {
-		pins = "gpio184";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-		output-low;
-	};
-
-	hall_int_n_default: hall-int-n-state {
-		pins = "gpio92";
-		function = "gpio";
-		bias-disable;
-	};
-
-	hdmi_hpd_default: hdmi-hpd-default-state {
-		pins = "gpio126";
-		function = "usb2_dp";
-		bias-disable;
-	};
-
-	kybd_default: kybd-default-state {
-		pins = "gpio67";
-		function = "gpio";
-		bias-disable;
-	};
-
-	nvme_reg_en: nvme-reg-en-state {
-		pins = "gpio18";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-disable;
-	};
-
-	pcie4_default: pcie4-default-state {
-		clkreq-n-pins {
-			pins = "gpio147";
-			function = "pcie4_clk";
-			drive-strength = <2>;
-			bias-pull-up;
-		};
-
-		perst-n-pins {
-			pins = "gpio146";
-			function = "gpio";
-			drive-strength = <2>;
-			bias-disable;
-		};
-
-		wake-n-pins {
-			pins = "gpio148";
-			function = "gpio";
-			drive-strength = <2>;
-			bias-pull-up;
-		};
-	};
-
-	pcie6a_default: pcie6a-default-state {
-		clkreq-n-pins {
-			pins = "gpio153";
-			function = "pcie6a_clk";
-			drive-strength = <2>;
-			bias-pull-up;
-		};
-
-		perst-n-pins {
-			pins = "gpio152";
-			function = "gpio";
-			drive-strength = <2>;
-			bias-disable;
-		};
-
-		wake-n-pins {
-			pins = "gpio154";
-			function = "gpio";
-			drive-strength = <2>;
-			bias-pull-up;
-		};
-	};
-
-	rtmr1_default: rtmr1-reset-n-active-state {
-		pins = "gpio176";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-
-	tpad_default: tpad-default-state {
-		pins = "gpio3";
-		function = "gpio";
-		bias-disable;
-	};
-
-	usb1_pwr_1p15_reg_en: usb1-pwr-1p15-reg-en-state {
-		pins = "gpio188";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-
-	usb1_pwr_1p8_reg_en: usb1-pwr-1p8-reg-en-state {
-		pins = "gpio175";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-
-	usb1_pwr_3p3_reg_en: usb1-pwr-3p3-reg-en-state {
-		pins = "gpio186";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-up;
-	};
-
-	wcn_bt_en: wcn-bt-en-state {
-		pins = "gpio116";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-pull-down;
-	};
-
-	wcn_sw_en: wcn-sw-en-state {
-		pins = "gpio214";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-disable;
-	};
-
-	wcn_wlan_en: wcn-wlan-en-state {
-		pins = "gpio117";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-disable;
-	};
-};
-
-&uart14 {
-	status = "okay";
-
-	bluetooth {
-		compatible = "qcom,wcn7850-bt";
-		max-speed = <3200000>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-	};
-};
-
-&usb_1_ss0_hsphy {
-	vdd-supply = <&vreg_l3j_0p8>;
-	vdda12-supply = <&vreg_l2j_1p2>;
-
-	phys = <&smb2360_0_eusb2_repeater>;
-
-	status = "okay";
-};
-
-&usb_1_ss0_qmpphy {
-	vdda-phy-supply = <&vreg_l2j_1p2>;
-	vdda-pll-supply = <&vreg_l1j_0p8>;
-
-	status = "okay";
-};
-
-&usb_1_ss0 {
-	status = "okay";
-};
-
-&usb_1_ss0_dwc3 {
-	dr_mode = "host";
-};
-
-&usb_1_ss0_dwc3_hs {
-	remote-endpoint = <&pmic_glink_ss0_hs_in>;
-};
-
-&usb_1_ss0_qmpphy_out {
-	remote-endpoint = <&retimer_ss0_ss_in>;
-};
-
-&usb_1_ss1_hsphy {
-	vdd-supply = <&vreg_l3j_0p8>;
-	vdda12-supply = <&vreg_l2j_1p2>;
-
-	phys = <&smb2360_1_eusb2_repeater>;
-
-	status = "okay";
-};
-
-&usb_1_ss1_qmpphy {
-	vdda-phy-supply = <&vreg_l2j_1p2>;
-	vdda-pll-supply = <&vreg_l2d_0p9>;
-
-	status = "okay";
-};
-
-&usb_1_ss1 {
-	status = "okay";
-};
-
-&usb_1_ss1_dwc3 {
-	dr_mode = "host";
-};
-
-&usb_1_ss1_dwc3_hs {
-	remote-endpoint = <&pmic_glink_ss1_hs_in>;
-};
-
-&usb_1_ss1_qmpphy_out {
-	remote-endpoint = <&retimer_ss1_ss_in>;
-};
-
-&usb_1_ss2_qmpphy {
-	vdda-phy-supply = <&vreg_l2j_1p2>;
-	vdda-pll-supply = <&vreg_l2d_0p9>;
-
-	/delete-property/ mode-switch;
-	/delete-property/ orientation-switch;
-
-	status = "okay";
-
-	ports {
-		port@0 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			/delete-node/ endpoint;
-
-			usb_1_ss2_qmpphy_out_dp: endpoint@0 {
-				reg = <0>;
-
-				data-lanes = <3 2 1 0>;
-				remote-endpoint = <&hdmi_bridge_dp_in>;
-			};
-
-			/* No USB3 lanes connected */
-		};
-	};
-};
-
-&usb_2 {
-	status = "okay";
-};
-
-&usb_2_dwc3 {
-	dr_mode = "host";
-};
-
-&usb_2_hsphy {
-	vdd-supply = <&vreg_l2e_0p8>;
-	vdda12-supply = <&vreg_l3e_1p2>;
-
-	phys = <&eusb5_repeater>;
-
-	status = "okay";
-};
-
-&usb_mp {
-	status = "okay";
-};
-
-&usb_mp_hsphy0 {
-	vdd-supply = <&vreg_l2e_0p8>;
-	vdda12-supply = <&vreg_l3e_1p2>;
-
-	phys = <&eusb3_repeater>;
-
-	status = "okay";
-};
-
-&usb_mp_hsphy1 {
-	vdd-supply = <&vreg_l2e_0p8>;
-	vdda12-supply = <&vreg_l3e_1p2>;
-
-	phys = <&eusb6_repeater>;
-
-	status = "okay";
-};
-
-&usb_mp_qmpphy0 {
-	vdda-phy-supply = <&vreg_l3e_1p2>;
-	vdda-pll-supply = <&vreg_l3c_0p8>;
-
-	status = "okay";
-};
-
-&usb_mp_qmpphy1 {
-	vdda-phy-supply = <&vreg_l3e_1p2>;
-	vdda-pll-supply = <&vreg_l3c_0p8>;
-
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-vivobook-s15.dts b/arch/arm64/boot/dts/qcom/x1p42100-asus-vivobook-s15.dts
new file mode 100644
index 0000000000000..63e29d2cc4abc
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-vivobook-s15.dts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2024, Xilin Wu <wuxilin123@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "purwa.dtsi"
+#include "x1-asus-vivobook-s15.dtsi"
+
+/delete-node/ &pmc8380_6;
+/delete-node/ &pmc8380_6_thermal;
+
+/ {
+	model = "ASUS Vivobook S 15 X1P-42-100";
+	compatible = "asus,vivobook-s15-x1p4", "qcom,x1p42100";
+	chassis-type = "laptop";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/x1p42100/ASUSTeK/vivobook-s15/qcdxkmsucpurwa.mbn";
+};
+
+&iris {
+	firmware-name = "qcom/x1p42100/ASUSTeK/vivobook-s15/qcvss8380.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/x1p42100/ASUSTeK/vivobook-s15/qcadsp8380.mbn",
+			"qcom/x1p42100/ASUSTeK/vivobook-s15/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/x1p42100/ASUSTeK/vivobook-s15/qccdsp8380.mbn",
+			"qcom/x1p42100/ASUSTeK/vivobook-s15/cdsp_dtbs.elf";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index 1fab1b50f20e6..d4dfb7fd973bf 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -202,6 +202,9 @@ dtb-$(CONFIG_ARCH_R9A09G057) += rzv2-evk-cn15-sd.dtbo
 r9a09g057h44-rzv2h-evk-cn15-sd-dtbs := r9a09g057h44-rzv2h-evk.dtb rzv2-evk-cn15-sd.dtbo
 dtb-$(CONFIG_ARCH_R9A09G057) += r9a09g057h44-rzv2h-evk-cn15-sd.dtb
 dtb-$(CONFIG_ARCH_R9A09G057) += r9a09g057h48-kakip.dtb
+dtb-$(CONFIG_ARCH_R9A09G057) += r9a09g057h48-kakip-pixpaper.dtbo
+r9a09g057h48-kakip-pixpaper-dtbs := r9a09g057h48-kakip.dtb r9a09g057h48-kakip-pixpaper.dtbo
+dtb-$(CONFIG_ARCH_R9A09G057) += r9a09g057h48-kakip-pixpaper.dtb
 
 dtb-$(CONFIG_ARCH_R9A09G077) += r9a09g077m44-rzt2h-evk.dtb
 
diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi
index 692a2b12aa035..aaedb1fb51aed 100644
--- a/arch/arm64/boot/dts/renesas/ebisu.dtsi
+++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi
@@ -53,6 +53,12 @@ backlight: backlight {
 		power-supply = <&reg_12p0v>;
 	};
 
+	pcie_usb_refclk: clk-x7 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
 	cvbs-in {
 		compatible = "composite-video-connector";
 		label = "CVBS IN";
@@ -439,6 +445,13 @@ adv7511_out: endpoint {
 		};
 	};
 
+	pcie_usb_clk: clk@68 {
+		compatible = "renesas,9fgv0841";
+		reg = <0x68>;
+		clocks = <&pcie_usb_refclk>;
+		#clock-cells = <1>;
+	};
+
 	video-receiver@70 {
 		compatible = "adi,adv7482";
 		reg = <0x70>;
@@ -577,13 +590,30 @@ &ohci0 {
 };
 
 &pcie_bus_clk {
-	clock-frequency = <100000000>;
+	status = "disabled";
 };
 
 &pciec0 {
+	clocks = <&cpg CPG_MOD 319>, <&pcie_usb_clk 1>;
 	status = "okay";
 };
 
+&pciec0_rp {
+	/*
+	 * This configuration is valid for SW49 in OFF position,
+	 * which means the PCIe signals are routed to the PCIe slot
+	 * and U11 9FGV0841 PCIe clock generator output 3 supplies
+	 * clock to the PCIe slot.
+	 *
+	 * In case the SW49 is set to ON position, which means the
+	 * PCIe signals are routed to the EX BT/WLAN expansion port,
+	 * and U11 9FGV0841 PCIe clock generator output 4 supplies
+	 * clock to the port, change clocks below to:
+	 * clocks = <&pcie_usb_clk 4>;
+	 */
+	clocks = <&pcie_usb_clk 3>;
+};
+
 &pfc {
 	avb_pins: avb {
 		groups = "avb_link", "avb_mii";
@@ -871,7 +901,18 @@ &usb2_phy0 {
 	status = "okay";
 };
 
+&usb3_phy0 {
+	clocks = <&pcie_usb_clk 6>;
+	status = "okay";
+};
+
+&usb3s0_clk {
+	status = "disabled";
+};
+
 &usb3_peri0 {
+	phys = <&usb3_phy0>;
+	phy-names = "usb";
 	companion = <&xhci0>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
index 607f62a448d89..59a0f2e1479d0 100644
--- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
@@ -2814,6 +2814,16 @@ pciec0: pcie@fe000000 {
 			iommu-map = <0 &ipmmu_hc 0 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec0_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		pciec1: pcie@ee800000 {
@@ -2843,6 +2853,16 @@ pciec1: pcie@ee800000 {
 			iommu-map = <0 &ipmmu_hc 1 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec1_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		pciec0_ep: pcie-ep@fe000000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
index e64c7b1aebc47..ad36aa8e75435 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
@@ -2619,6 +2619,16 @@ pciec0: pcie@fe000000 {
 			iommu-map = <0 &ipmmu_hc 0 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec0_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		pciec1: pcie@ee800000 {
@@ -2648,6 +2658,16 @@ pciec1: pcie@ee800000 {
 			iommu-map = <0 &ipmmu_hc 1 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec1_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		imr-lx4@fe860000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a77961.dtsi b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
index 89f6c052c5e06..9d76e39eab72e 100644
--- a/arch/arm64/boot/dts/renesas/r8a77961.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
@@ -2499,6 +2499,16 @@ pciec0: pcie@fe000000 {
 			iommu-map = <0 &ipmmu_hc 0 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec0_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		pciec1: pcie@ee800000 {
@@ -2528,6 +2538,16 @@ pciec1: pcie@ee800000 {
 			iommu-map = <0 &ipmmu_hc 1 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec1_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		fcpf0: fcp@fe950000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
index 425561e658caf..611a9335c63ad 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
@@ -2494,6 +2494,16 @@ pciec0: pcie@fe000000 {
 			iommu-map = <0 &ipmmu_hc 0 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec0_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		pciec1: pcie@ee800000 {
@@ -2523,6 +2533,16 @@ pciec1: pcie@ee800000 {
 			iommu-map = <0 &ipmmu_hc 1 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec1_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		fdp1@fe940000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
index d3698f7e494df..fadb5f4effcf0 100644
--- a/arch/arm64/boot/dts/renesas/r8a77990.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
@@ -1912,6 +1912,16 @@ pciec0: pcie@fe000000 {
 			iommu-map = <0 &ipmmu_hc 0 1>;
 			iommu-map-mask = <0>;
 			status = "disabled";
+
+			/* PCIe bridge, Root Port */
+			pciec0_rp: pci@0,0 {
+				#address-cells = <3>;
+				#size-cells = <2>;
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				ranges;
+			};
 		};
 
 		vspb0: vsp@fe960000 {
@@ -2180,4 +2190,21 @@ timer {
 			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
 		interrupt-names = "sec-phys", "phys", "virt", "hyp-phys";
 	};
+
+	/* External USB clock - to be overridden by boards that provide it */
+	usb3s0_clk: usb3s0-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	usb3_phy0: usb-phy {
+		compatible = "usb-nop-xceiv";
+		clocks = <&usb3s0_clk>;
+		clock-names = "main_clk";
+		clock-frequency = <100000000>;
+		power-domains = <&sysc R8A77990_PD_ALWAYS_ON>;
+		#phy-cells = <0>;
+		status = "disabled";
+	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
index ff07d984cbf29..3b47e3ce95d47 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
@@ -536,6 +536,10 @@ msiof1_snd_endpoint: endpoint {
 	};
 };
 
+&otp {
+	bootph-all;
+};
+
 /* Page 26 / 2230 Key M M.2 */
 &pcie0_clkref {
 	status = "disabled";
@@ -620,6 +624,7 @@ canfd4_pins: canfd4 {
 	hscif0_pins: hscif0 {
 		groups = "hscif0_data", "hscif0_ctrl";
 		function = "hscif0";
+		bootph-all;
 	};
 
 	/* Page 23 / DEBUG */
diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
index 4c97298fa7634..3e1c98903cea0 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
@@ -698,7 +698,7 @@ scif0: serial@c0700000 {
 			compatible = "renesas,scif-r8a78000",
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0700000 0 0x40>;
-			interrupts = <GIC_SPI 4074 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 10 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -708,7 +708,7 @@ scif1: serial@c0704000 {
 			compatible = "renesas,scif-r8a78000",
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0704000 0 0x40>;
-			interrupts = <GIC_SPI 4075 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 11 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -718,7 +718,7 @@ scif3: serial@c0708000 {
 			compatible = "renesas,scif-r8a78000",
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0708000 0 0x40>;
-			interrupts = <GIC_SPI 4076 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 12 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -728,7 +728,7 @@ scif4: serial@c070c000 {
 			compatible = "renesas,scif-r8a78000",
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc070c000 0 0x40>;
-			interrupts = <GIC_SPI 4077 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 13 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -738,7 +738,7 @@ hscif0: serial@c0710000 {
 			compatible = "renesas,hscif-r8a78000",
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0710000 0 0x60>;
-			interrupts = <GIC_SPI 4078 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 14 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -748,7 +748,7 @@ hscif1: serial@c0714000 {
 			compatible = "renesas,hscif-r8a78000",
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0714000 0 0x60>;
-			interrupts = <GIC_SPI 4079 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 15 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -758,7 +758,7 @@ hscif2: serial@c0718000 {
 			compatible = "renesas,hscif-r8a78000",
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0718000 0 0x60>;
-			interrupts = <GIC_SPI 4080 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 16 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
@@ -768,7 +768,7 @@ hscif3: serial@c071c000 {
 			compatible = "renesas,hscif-r8a78000",
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc071c000 0 0x60>;
-			interrupts = <GIC_SPI 4081 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 17 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index 9fb15ca24984f..9192c5bf7e592 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -639,6 +639,21 @@ wdt3: watchdog@13000400 {
 			status = "disabled";
 		};
 
+		rtc: rtc@11c00800 {
+			compatible = "renesas,r9a09g056-rtca3", "renesas,rz-rtca3";
+			reg = <0 0x11c00800 0 0x400>;
+			interrupts = <GIC_SPI 524 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 525 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 526 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "alarm", "period", "carry";
+			clocks = <&cpg CPG_MOD 0x53>, <&rtxin_clk>;
+			clock-names = "bus", "counter";
+			power-domains = <&cpg>;
+			resets = <&cpg 0x79>, <&cpg 0x7a>;
+			reset-names = "rtc", "rtest";
+			status = "disabled";
+		};
+
 		scif: serial@11c01400 {
 			compatible = "renesas,scif-r9a09g056",
 				     "renesas,scif-r9a09g057";
@@ -769,6 +784,13 @@ rspi0: spi@12800000 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7b>, <&cpg 0x7c>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x448c>, <&dmac0 0x448d>,
+			       <&dmac1 0x448c>, <&dmac1 0x448d>,
+			       <&dmac2 0x448c>, <&dmac2 0x448d>,
+			       <&dmac3 0x448c>, <&dmac3 0x448d>,
+			       <&dmac4 0x448c>, <&dmac4 0x448d>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -790,6 +812,13 @@ rspi1: spi@12800400 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7d>, <&cpg 0x7e>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x448e>, <&dmac0 0x448f>,
+			       <&dmac1 0x448e>, <&dmac1 0x448f>,
+			       <&dmac2 0x448e>, <&dmac2 0x448f>,
+			       <&dmac3 0x448e>, <&dmac3 0x448f>,
+			       <&dmac4 0x448e>, <&dmac4 0x448f>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -811,6 +840,13 @@ rspi2: spi@12800800 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7f>, <&cpg 0x80>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x4490>, <&dmac0 0x4491>,
+			       <&dmac1 0x4490>, <&dmac1 0x4491>,
+			       <&dmac2 0x4490>, <&dmac2 0x4491>,
+			       <&dmac3 0x4490>, <&dmac3 0x4491>,
+			       <&dmac4 0x4490>, <&dmac4 0x4491>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
index 9af50198d2f11..c191ecb397135 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
@@ -446,6 +446,10 @@ &qextal_clk {
 	clock-frequency = <24000000>;
 };
 
+&rtc {
+	status = "okay";
+};
+
 &rtxin_clk {
 	clock-frequency = <32768>;
 };
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 80cba9fcfe7bf..9581af58024ed 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -581,16 +581,6 @@ ostm7: timer@12c03000 {
 			status = "disabled";
 		};
 
-		wdt0: watchdog@11c00400 {
-			compatible = "renesas,r9a09g057-wdt";
-			reg = <0 0x11c00400 0 0x400>;
-			clocks = <&cpg CPG_MOD 0x4b>, <&cpg CPG_MOD 0x4c>;
-			clock-names = "pclk", "oscclk";
-			resets = <&cpg 0x75>;
-			power-domains = <&cpg>;
-			status = "disabled";
-		};
-
 		wdt1: watchdog@14400000 {
 			compatible = "renesas,r9a09g057-wdt";
 			reg = <0 0x14400000 0 0x400>;
@@ -601,26 +591,6 @@ wdt1: watchdog@14400000 {
 			status = "disabled";
 		};
 
-		wdt2: watchdog@13000000 {
-			compatible = "renesas,r9a09g057-wdt";
-			reg = <0 0x13000000 0 0x400>;
-			clocks = <&cpg CPG_MOD 0x4f>, <&cpg CPG_MOD 0x50>;
-			clock-names = "pclk", "oscclk";
-			resets = <&cpg 0x77>;
-			power-domains = <&cpg>;
-			status = "disabled";
-		};
-
-		wdt3: watchdog@13000400 {
-			compatible = "renesas,r9a09g057-wdt";
-			reg = <0 0x13000400 0 0x400>;
-			clocks = <&cpg CPG_MOD 0x51>, <&cpg CPG_MOD 0x52>;
-			clock-names = "pclk", "oscclk";
-			resets = <&cpg 0x78>;
-			power-domains = <&cpg>;
-			status = "disabled";
-		};
-
 		rtc: rtc@11c00800 {
 			compatible = "renesas,r9a09g057-rtca3", "renesas,rz-rtca3";
 			reg = <0 0x11c00800 0 0x400>;
@@ -765,6 +735,13 @@ rspi0: spi@12800000 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7b>, <&cpg 0x7c>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x448c>, <&dmac0 0x448d>,
+			       <&dmac1 0x448c>, <&dmac1 0x448d>,
+			       <&dmac2 0x448c>, <&dmac2 0x448d>,
+			       <&dmac3 0x448c>, <&dmac3 0x448d>,
+			       <&dmac4 0x448c>, <&dmac4 0x448d>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -786,6 +763,13 @@ rspi1: spi@12800400 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7d>, <&cpg 0x7e>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x448e>, <&dmac0 0x448f>,
+			       <&dmac1 0x448e>, <&dmac1 0x448f>,
+			       <&dmac2 0x448e>, <&dmac2 0x448f>,
+			       <&dmac3 0x448e>, <&dmac3 0x448f>,
+			       <&dmac4 0x448e>, <&dmac4 0x448f>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -807,6 +791,13 @@ rspi2: spi@12800800 {
 			clock-names = "pclk", "pclk_sfr", "tclk";
 			resets = <&cpg 0x7f>, <&cpg 0x80>;
 			reset-names = "presetn", "tresetn";
+			dmas = <&dmac0 0x4490>, <&dmac0 0x4491>,
+			       <&dmac1 0x4490>, <&dmac1 0x4491>,
+			       <&dmac2 0x4490>, <&dmac2 0x4491>,
+			       <&dmac3 0x4490>, <&dmac3 0x4491>,
+			       <&dmac4 0x4490>, <&dmac4 0x4491>;
+			dma-names = "rx", "tx", "rx", "tx", "rx",
+				    "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057h48-kakip-pixpaper.dtso b/arch/arm64/boot/dts/renesas/r9a09g057h48-kakip-pixpaper.dtso
new file mode 100644
index 0000000000000..7b8209494b738
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a09g057h48-kakip-pixpaper.dtso
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Overlay for Mayqueen (Open-EP Community) pixpaper display
+ * support on Renesas RZ/V2H platform (KAKIP board).
+ *
+ * Copyright (C) 2026 Wig Cheng <onlywig@gmail.com>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/pinctrl/renesas,r9a09g057-pinctrl.h>
+#include <dt-bindings/gpio/gpio.h>
+
+&pinctrl {
+	rspi0_pins: rspi0 {
+		pinmux = <RZV2H_PORT_PINMUX(9, 0, 1)>, /* SPI0 MOSI */
+			 <RZV2H_PORT_PINMUX(9, 1, 1)>, /* SPI0 MISO */
+			 <RZV2H_PORT_PINMUX(9, 2, 1)>, /* SPI0 CLK */
+			 <RZV2H_PORT_PINMUX(9, 3, 1)>; /* SPI0 CE0 */
+	};
+};
+
+&rspi0 {
+	pinctrl-0 = <&rspi0_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	display@0 {
+		compatible = "mayqueen,pixpaper";
+		reg = <0>;
+		spi-max-frequency = <1000000>;
+		reset-gpios = <&pinctrl RZV2H_GPIO(A, 7) GPIO_ACTIVE_HIGH>;
+		busy-gpios = <&pinctrl RZV2H_GPIO(B, 3) GPIO_ACTIVE_HIGH>;
+		dc-gpios = <&pinctrl RZV2H_GPIO(7, 4) GPIO_ACTIVE_HIGH>;
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
index 14d7fb6f8952e..3761551c96472 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
@@ -8,6 +8,24 @@
 #include <dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
+/* The IRQ_NS lines start at offset 16 in the ICU interrupt space */
+#define RZT2H_IRQ0	16
+#define RZT2H_IRQ1	17
+#define RZT2H_IRQ2	18
+#define RZT2H_IRQ3	19
+#define RZT2H_IRQ4	20
+#define RZT2H_IRQ5	21
+#define RZT2H_IRQ6	22
+#define RZT2H_IRQ7	23
+#define RZT2H_IRQ8	24
+#define RZT2H_IRQ9	25
+#define RZT2H_IRQ10	26
+#define RZT2H_IRQ11	27
+#define RZT2H_IRQ12	28
+#define RZT2H_IRQ13	29
+#define RZT2H_IRQ14	30
+#define RZT2H_IRQ15	31
+
 / {
 	compatible = "renesas,r9a09g077";
 	#address-cells = <2>;
@@ -200,6 +218,10 @@ rspi0: spi@80007000 {
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
 				 <&cpg CPG_MOD 104>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x267a>, <&dmac0 0x267b>,
+			       <&dmac1 0x267a>, <&dmac1 0x267b>,
+			       <&dmac2 0x267a>, <&dmac2 0x267b>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -218,6 +240,10 @@ rspi1: spi@80007400 {
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
 				 <&cpg CPG_MOD 105>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x267f>, <&dmac0 0x2680>,
+			       <&dmac1 0x267f>, <&dmac1 0x2680>,
+			       <&dmac2 0x267f>, <&dmac2 0x2680>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -236,6 +262,10 @@ rspi2: spi@80007800 {
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
 				 <&cpg CPG_MOD 106>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x2684>, <&dmac0 0x2685>,
+			       <&dmac1 0x2684>, <&dmac1 0x2685>,
+			       <&dmac2 0x2684>, <&dmac2 0x2685>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -254,6 +284,10 @@ rspi3: spi@81007000 {
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
 				 <&cpg CPG_MOD 602>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x2689>, <&dmac0 0x268a>,
+			       <&dmac1 0x2689>, <&dmac1 0x268a>,
+			       <&dmac2 0x2689>, <&dmac2 0x268a>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -974,8 +1008,8 @@ mii_conv3: mii-conv@3 {
 
 		cpg: clock-controller@80280000 {
 			compatible = "renesas,r9a09g077-cpg-mssr";
-			reg = <0 0x80280000 0 0x1000>,
-			      <0 0x81280000 0 0x9000>;
+			reg = <0 0x80280000 0 0x10000>,
+			      <0 0x81280000 0 0x10000>;
 			clocks = <&extal_clk>;
 			clock-names = "extal";
 			#clock-cells = <2>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
index e9639bbb2d703..4c0e52850ca97 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
@@ -14,12 +14,15 @@
 /*
  * SD0 can be connected to either eMMC (IC49) or SD card slot CN31
  * Lets by default enable the eMMC, note we need the below SW settings
- * for eMMC.
+ * for eMMC. Also ensure that CN78 pins 5 and 6 are connected with a jumper
+ * to provide the SD0 power supply when using eMMC.
  * SW2[1] = ON; SW2[2] = ON
  *
  * To enable SD card and disable eMMC on SDHI0 disable the below macro
- * and set the below switch setting:
- * SW2[1] = OFF; SW2[2] = ON
+ * and set the switch as follows. Also ensure that CN78 pins 3 and 4 are connected
+ * with a jumper to provide the SD0 power supply when using an SD card.
+ *
+ * SW2[1] = OFF; SW2[2] = ON.
  */
 #define SD0_EMMC	1
 #define SD0_SD		(!SD0_EMMC)
@@ -224,10 +227,12 @@ &i2c1 {
 };
 
 &mdio1_phy {
+	interrupts-extended = <&icu RZT2H_IRQ3 IRQ_TYPE_EDGE_FALLING>;
 	reset-gpios = <&pinctrl RZT2H_GPIO(32, 3) GPIO_ACTIVE_LOW>;
 };
 
 &mdio2_phy {
+	interrupts-extended = <&icu RZT2H_IRQ13 IRQ_TYPE_EDGE_FALLING>;
 	/*
 	 * PHY2 Reset Configuration:
 	 *
@@ -248,6 +253,35 @@ can0_pins: can0-pins {
 			 <RZT2H_PORT_PINMUX(24, 4, 0x19)>; /* CANTX0 */
 	};
 
+	/*
+	 * GMAC1 Pin Configuration:
+	 *
+	 * SW2[8] ON - use pins P33_2-P33_7, P34_0-P34_5, P34_7 and
+	 * P35_0-P35_2 for Ethernet port 3
+	 */
+	gmac1_pins: gmac1-pins {
+		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
+			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
+			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD1 */
+			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
+			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
+			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
+			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
+			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
+			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
+			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
+			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
+			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
+			 <RZT2H_PORT_PINMUX(34, 7, 0xf)>, /* ETH3_TXER */
+			 <RZT2H_PORT_PINMUX(35, 0, 0xf)>, /* ETH3_RXER */
+			 <RZT2H_PORT_PINMUX(35, 1, 0xf)>, /* ETH3_CRS */
+			 <RZT2H_PORT_PINMUX(35, 2, 0xf)>, /* ETH3_COL */
+			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
+			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
+			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>, /* ETH3_REFCLK */
+			 <RZT2H_PORT_PINMUX(27, 2, 0x0)>; /* IRQ3 */
+	};
+
 	/*
 	 * GMAC2 Pin Configuration:
 	 *
@@ -274,35 +308,8 @@ gmac2_pins: gmac2-pins {
 			 <RZT2H_PORT_PINMUX(31, 5, 0xf)>, /* ETH2_COL */
 			 <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
 			 <RZT2H_PORT_PINMUX(30, 6, 0x10)>, /* GMAC2_MDIO */
-			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>; /* ETH2_REFCLK */
-	};
-
-	/*
-	 * GMAC1 Pin Configuration:
-	 *
-	 * SW2[8] ON - use pins P33_2-P33_7, P34_0-P34_5, P34_7 and
-	 * P35_0-P35_2 for Ethernet port 3
-	 */
-	gmac1_pins: gmac1-pins {
-		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
-			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD1 */
-			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
-			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
-			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
-			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
-			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
-			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
-			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
-			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
-			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
-			 <RZT2H_PORT_PINMUX(34, 7, 0xf)>, /* ETH3_TXER */
-			 <RZT2H_PORT_PINMUX(35, 0, 0xf)>, /* ETH3_RXER */
-			 <RZT2H_PORT_PINMUX(35, 1, 0xf)>, /* ETH3_CRS */
-			 <RZT2H_PORT_PINMUX(35, 2, 0xf)>, /* ETH3_COL */
-			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
-			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
-			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>; /* ETH3_REFCLK */
+			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>, /* ETH2_REFCLK */
+			 <RZT2H_PORT_PINMUX(31, 1, 0x0)>; /* IRQ13 */
 	};
 
 	/*
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
index 4a13395613321..f697e9698ed39 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
@@ -8,6 +8,24 @@
 #include <dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
+/* The IRQ_NS lines start at offset 16 in the ICU interrupt space */
+#define RZN2H_IRQ0	16
+#define RZN2H_IRQ1	17
+#define RZN2H_IRQ2	18
+#define RZN2H_IRQ3	19
+#define RZN2H_IRQ4	20
+#define RZN2H_IRQ5	21
+#define RZN2H_IRQ6	22
+#define RZN2H_IRQ7	23
+#define RZN2H_IRQ8	24
+#define RZN2H_IRQ9	25
+#define RZN2H_IRQ10	26
+#define RZN2H_IRQ11	27
+#define RZN2H_IRQ12	28
+#define RZN2H_IRQ13	29
+#define RZN2H_IRQ14	30
+#define RZN2H_IRQ15	31
+
 / {
 	compatible = "renesas,r9a09g087";
 	#address-cells = <2>;
@@ -200,6 +218,10 @@ rspi0: spi@80007000 {
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
 				 <&cpg CPG_MOD 104>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x267a>, <&dmac0 0x267b>,
+			       <&dmac1 0x267a>, <&dmac1 0x267b>,
+			       <&dmac2 0x267a>, <&dmac2 0x267b>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -218,6 +240,10 @@ rspi1: spi@80007400 {
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
 				 <&cpg CPG_MOD 105>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x267f>, <&dmac0 0x2680>,
+			       <&dmac1 0x267f>, <&dmac1 0x2680>,
+			       <&dmac2 0x267f>, <&dmac2 0x2680>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -236,6 +262,10 @@ rspi2: spi@80007800 {
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
 				 <&cpg CPG_MOD 106>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x2684>, <&dmac0 0x2685>,
+			       <&dmac1 0x2684>, <&dmac1 0x2685>,
+			       <&dmac2 0x2684>, <&dmac2 0x2685>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -254,6 +284,10 @@ rspi3: spi@81007000 {
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
 				 <&cpg CPG_MOD 602>;
 			clock-names = "pclk", "pclkspi";
+			dmas = <&dmac0 0x2689>, <&dmac0 0x268a>,
+			       <&dmac1 0x2689>, <&dmac1 0x268a>,
+			       <&dmac2 0x2689>, <&dmac2 0x268a>;
+			dma-names = "rx", "tx", "rx", "tx", "rx", "tx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -977,8 +1011,8 @@ mii_conv3: mii-conv@3 {
 
 		cpg: clock-controller@80280000 {
 			compatible = "renesas,r9a09g087-cpg-mssr";
-			reg = <0 0x80280000 0 0x1000>,
-			      <0 0x81280000 0 0x9000>;
+			reg = <0 0x80280000 0 0x10000>,
+			      <0 0x81280000 0 0x10000>;
 			clocks = <&extal_clk>;
 			clock-names = "extal";
 			#clock-cells = <2>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
index 19f0a2c067530..ef6cc7497c2c4 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
@@ -14,12 +14,14 @@
 /*
  * SD0 can be connected to either eMMC (U33) or SD card slot CN21
  * Lets by default enable the eMMC, note we need the below SW settings
- * for eMMC.
+ * for eMMC. Also ensure that JP23 pins 5 and 6 are connected with a jumper
+ * to provide the SD0 power supply when using eMMC.
  * DSW5[1] = ON; DSW5[2] = ON
  * DSW17[5] = OFF; DSW17[6] = ON
  *
  * To enable SD card and disable eMMC on SDHI0 disable the below macro
- * and set the below switch setting:
+ * and set the below switch settings. Also ensure that JP23 pins 3 and 4 are
+ * connected with a jumper to provide the SD0 power supply when using an SD card.
  * DSW5[1] = OFF; DSW5[2] = ON
  * P22_6 = SD0_WP; DSW15[1] = OFF; DSW15[2] = ON
  * P22_5 = SD0_CD; DSW15[3] = OFF; DSW15[4] = ON
@@ -303,6 +305,7 @@ &i2c1 {
 };
 
 &mdio1_phy {
+	interrupts-extended = <&icu RZN2H_IRQ15 IRQ_TYPE_EDGE_FALLING>;
 	/*
 	 * PHY3 Reset Configuration:
 	 *
@@ -312,6 +315,7 @@ &mdio1_phy {
 };
 
 &mdio2_phy {
+	interrupts-extended = <&icu RZN2H_IRQ14 IRQ_TYPE_EDGE_FALLING>;
 	/*
 	 * PHY2 Reset Configuration:
 	 *
@@ -332,12 +336,43 @@ can1_pins: can1-pins {
 			 <RZT2H_PORT_PINMUX(12, 1, 0x19)>; /* CANTX1 */
 	};
 
+	/*
+	 * GMAC1 Pin Configuration:
+	 *
+	 * DSW5[8] ON - use pins P00_0-P00_2, P33_2-P33_7, P34_0-P34_6
+	 * for Ethernet port 3
+	 * DSW12[1] OFF; DSW12[2] ON - use pin P00_3 for Ethernet port 3
+	 */
+	gmac1_pins: gmac1-pins {
+		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
+			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
+			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD0 */
+			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
+			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
+			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
+			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
+			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
+			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
+			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
+			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
+			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
+			 <RZT2H_PORT_PINMUX(0, 0, 0xf)>, /* ETH3_TXER */
+			 <RZT2H_PORT_PINMUX(0, 1, 0xf)>, /* ETH3_RXER */
+			 <RZT2H_PORT_PINMUX(0, 2, 0xf)>, /* ETH3_CRS */
+			 <RZT2H_PORT_PINMUX(0, 3, 0xf)>, /* ETH3_COL */
+			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
+			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
+			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>, /* ETH3_REFCLK */
+			 <RZT2H_PORT_PINMUX(17, 3, 0x0)>; /* IRQ15 */
+	};
+
 	/*
 	 * GMAC2 Pin Configuration:
 	 *
 	 * DSW5[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2
 	 * DSW5[7] ON - use pins P29_1-P29_7, P30_0-P30_4, P30_7,
 	 * P31_2, P31_4 and P31_5 are used for Ethernet port 2
+	 * DSW13[7] OFF; DSW13[8] ON - use pin P13_7 for IRQ14
 	 */
 	gmac2_pins: gmac2-pins {
 		pinmux = <RZT2H_PORT_PINMUX(29, 1, 0xf)>, /* ETH2_TXCLK */
@@ -358,39 +393,11 @@ gmac2_pins: gmac2-pins {
 			 <RZT2H_PORT_PINMUX(31, 5, 0xf)>, /* ETH2_COL */
 			 <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
 			 <RZT2H_PORT_PINMUX(30, 6, 0x10)>, /* GMAC2_MDIO */
-			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>; /* ETH2_REFCLK */
+			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>, /* ETH2_REFCLK */
+			 <RZT2H_PORT_PINMUX(13, 7, 0x0)>; /* IRQ14 */
 
 	};
 
-	/*
-	 * GMAC2 Pin Configuration:
-	 *
-	 * DSW5[8] ON - use pins P00_0-P00_2, P33_2-P33_7, P34_0-P34_6
-	 * for Ethernet port 3
-	 * DSW12[1] OFF; DSW12[2] ON - use pin P00_3 for Ethernet port 3
-	 */
-	gmac1_pins: gmac1-pins {
-		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
-			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
-			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
-			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
-			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
-			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
-			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
-			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
-			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
-			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
-			 <RZT2H_PORT_PINMUX(0, 0, 0xf)>, /* ETH3_TXER */
-			 <RZT2H_PORT_PINMUX(0, 1, 0xf)>, /* ETH3_RXER */
-			 <RZT2H_PORT_PINMUX(0, 2, 0xf)>, /* ETH3_CRS */
-			 <RZT2H_PORT_PINMUX(0, 3, 0xf)>, /* ETH3_COL */
-			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
-			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
-			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>; /* ETH3_REFCLK */
-	};
-
 	/*
 	 * I2C0 Pin Configuration:
 	 * ------------------------
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
index 982f17aafbc50..b45acfe6288a7 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
@@ -162,7 +162,7 @@ versa3: clock-generator@68 {
 				       <100000000>;
 		renesas,settings = [
 		  80 00 11 19 4c 42 dc 2f 06 7d 20 1a 5f 1e f2 27
-		  00 40 00 00 00 00 00 00 06 0c 19 02 3f f0 90 86
+		  00 40 00 00 00 00 00 00 06 0c 19 02 3b f0 90 86
 		  a0 80 30 30 9c
 		];
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
index 510399febf295..f87c2492f414a 100644
--- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
@@ -53,6 +53,7 @@ vqmmc_sdhi0: regulator-vqmmc-sdhi0 {
 		regulator-max-microvolt = <3300000>;
 		gpios-states = <0>;
 		states = <3300000 0>, <1800000 1>;
+		regulator-ramp-delay = <60>;
 	};
 #endif
 
diff --git a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso
index 0af1e0a6c7f48..fc53c1aae3b52 100644
--- a/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso
+++ b/arch/arm64/boot/dts/renesas/rzv2-evk-cn15-sd.dtso
@@ -25,6 +25,7 @@ vqmmc_sdhi0: regulator-vqmmc-sdhi0 {
 		regulator-max-microvolt = <3300000>;
 		gpios-states = <0>;
 		states = <3300000 0>, <1800000 1>;
+		regulator-ramp-delay = <60>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index d4a921bed4c39..e505161caa675 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -75,6 +75,12 @@ backlight: backlight {
 		enable-gpios = <&gpio6 7 GPIO_ACTIVE_HIGH>;
 	};
 
+	pcie_usb_refclk: clk-x7 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
 	cvbs-in {
 		compatible = "composite-video-connector";
 		label = "CVBS IN";
@@ -523,6 +529,13 @@ pca9654: gpio@20 {
 		#gpio-cells = <2>;
 	};
 
+	pcie_usb_clk: clk@68 {
+		compatible = "renesas,9fgv0841";
+		reg = <0x68>;
+		clocks = <&pcie_usb_refclk>;
+		#clock-cells = <1>;
+	};
+
 	video-receiver@70 {
 		compatible = "adi,adv7482";
 		reg = <0x70 0x71 0x72 0x73 0x74 0x75
@@ -640,17 +653,27 @@ &ohci1 {
 };
 
 &pcie_bus_clk {
-	clock-frequency = <100000000>;
+	status = "disabled";
 };
 
 &pciec0 {
+	clocks = <&cpg CPG_MOD 319>, <&pcie_usb_clk 1>;
 	status = "okay";
 };
 
+&pciec0_rp {
+	clocks = <&pcie_usb_clk 3>;
+};
+
 &pciec1 {
+	clocks = <&cpg CPG_MOD 318>, <&pcie_usb_clk 2>;
 	status = "okay";
 };
 
+&pciec1_rp {
+	clocks = <&pcie_usb_clk 4>;
+};
+
 &pfc {
 	pinctrl-0 = <&scif_clk_pins>;
 	pinctrl-names = "default";
@@ -1038,11 +1061,12 @@ &usb3_peri0 {
 };
 
 &usb3_phy0 {
+	clocks = <&cpg CPG_MOD 328>, <&pcie_usb_clk 6>, <&usb_extal_clk>;
 	status = "okay";
 };
 
 &usb3s0_clk {
-	clock-frequency = <100000000>;
+	status = "disabled";
 };
 
 &vin0 {
diff --git a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
index 2a157d1efb3d3..97014bcfbb1d2 100644
--- a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
@@ -351,20 +351,30 @@ &ohci0 {
 };
 
 &pcie_bus_clk {
-	clock-frequency = <100000000>;
+	status = "disabled";
 };
 
 &pciec0 {
+	clocks = <&cpg CPG_MOD 319>, <&pcie_usb_clk 1>;
 	status = "okay";
 };
 
+&pciec0_rp {
+	clocks = <&pcie_usb_clk 3>;
+};
+
 &pciec1 {
+	clocks = <&cpg CPG_MOD 318>, <&pcie_usb_clk 2>;
 	status = "okay";
 
 	vpcie1v5-supply = <&pcie_1v5>;
 	vpcie3v3-supply = <&pcie_3v3>;
 };
 
+&pciec1_rp {
+	clocks = <&pcie_usb_clk 4>;
+};
+
 &pfc {
 	can0_pins: can0 {
 		groups = "can0_data_a";
@@ -475,6 +485,15 @@ &usb2_phy0 {
 	status = "okay";
 };
 
+&usb3_phy0 {
+	clocks = <&cpg CPG_MOD 328>, <&pcie_usb_clk 6>, <&usb_extal_clk>;
+	status = "okay";
+};
+
+&usb3s0_clk {
+	status = "disabled";
+};
+
 &xhci0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi
index 241caf737abbd..67fd6a65db897 100644
--- a/arch/arm64/boot/dts/renesas/ulcb.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi
@@ -47,6 +47,12 @@ audio_clkout: audio-clkout {
 		clock-frequency = <12288000>;
 	};
 
+	pcie_usb_refclk: clk-x24 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
 	hdmi0-out {
 		compatible = "hdmi-connector";
 		type = "a";
@@ -232,6 +238,13 @@ &i2c4 {
 
 	clock-frequency = <400000>;
 
+	pcie_usb_clk: clk@68 {
+		compatible = "renesas,9fgv0841";
+		reg = <0x68>;
+		clocks = <&pcie_usb_refclk>;
+		#clock-cells = <1>;
+	};
+
 	versaclock5: clock-generator@6a {
 		compatible = "idt,5p49v5925";
 		reg = <0x6a>;
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index 4d384f153c134..11a1bad352094 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -63,6 +63,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-khadas-edge-v.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-kobol-helios64.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-leez-p710.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopc-t4.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopc-t4-hd702e.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4b.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-neo4.dtb
@@ -104,6 +105,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg353v.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg353vs.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg503.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-odroid-m1s.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-onething-edge-cube.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-orangepi-3b-v1.1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-orangepi-3b-v2.1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.1.dtb
@@ -116,6 +118,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-powkiddy-rgb30.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-powkiddy-rk2023.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-powkiddy-x55.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-qnap-ts133.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-qnap-ts133-pcb-13.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-a.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-b.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-radxa-cm3-io.dtb
@@ -150,7 +153,9 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nanopi-r5s.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-odroid-m1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-photonicat.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts233.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts233-pcb-12-11.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts433.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts433-pcb-12-10.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-radxa-cm3j-rpi-cm4.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-radxa-e25.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-roc-pc.dtb
@@ -164,6 +169,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-armsom-sige5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-armsom-sige5-v1.2-wifibt.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10-pcie1.dtbo
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb2-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-luckfox-omni3576.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-m5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-r76s.dtb
@@ -245,6 +251,10 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += px30-ringneck-haikou-haikou-video-demo.dtb
 px30-ringneck-haikou-haikou-video-demo-dtbs := px30-ringneck-haikou.dtb \
 	px30-ringneck-haikou-video-demo.dtbo
 
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopc-t4-hd702e.dtb
+rk3399-nanopc-t4-hd702e-dtbs := rk3399-nanopc-t4.dtb \
+	rk3399-nanopc-t4-hd702e.dtbo
+
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-lion-haikou-haikou-video-demo.dtb
 rk3368-lion-haikou-haikou-video-demo-dtbs := rk3368-lion-haikou.dtb \
 	rk3368-lion-haikou-video-demo.dtbo
@@ -261,6 +271,18 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64-v2-screen.dtb
 rk3399-rockpro64-v2-screen-dtbs := rk3399-rockpro64-v2.dtb \
 	rk3399-rockpro64-screen.dtbo
 
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-qnap-ts133-pcb-13.dtb
+rk3566-qnap-ts133-pcb-13-dtbs := rk3566-qnap-ts133.dtb \
+	rk3566-qnap-ts133-pcb-13.dtbo
+
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts233-pcb-12-11.dtb
+rk3568-qnap-ts233-pcb-12-11-dtbs := rk3568-qnap-ts233.dtb \
+	rk3568-qnap-ts233-pcb-12-11.dtbo
+
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-qnap-ts433-pcb-12-10.dtb
+rk3568-qnap-ts433-pcb-12-10-dtbs := rk3568-qnap-ts433.dtb \
+	rk3568-qnap-ts433-pcb-12-10.dtbo
+
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-wolfvision-pf5-vz-2-uhd.dtb
 rk3568-wolfvision-pf5-vz-2-uhd-dtbs := rk3568-wolfvision-pf5.dtb \
 	rk3568-wolfvision-pf5-display-vz.dtbo \
diff --git a/arch/arm64/boot/dts/rockchip/rk3328-a1.dts b/arch/arm64/boot/dts/rockchip/rk3328-a1.dts
index 30bdb38f0727a..e810ed146451c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328-a1.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3328-a1.dts
@@ -58,24 +58,6 @@ ir-receiver {
 		gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_LOW>;
 		linux,rc-map-name = "rc-beelink-gs1";
 	};
-
-	spdif_dit: spdif-dit {
-		compatible = "linux,spdif-dit";
-		#sound-dai-cells = <0>;
-	};
-
-	spdif_sound: spdif-sound {
-		compatible = "simple-audio-card";
-		simple-audio-card,name = "SPDIF";
-
-		simple-audio-card,cpu {
-			sound-dai = <&spdif>;
-		};
-
-		simple-audio-card,codec {
-			sound-dai = <&spdif_dit>;
-		};
-	};
 };
 
 &analog_sound {
@@ -343,11 +325,6 @@ &sdmmc {
 	status = "okay";
 };
 
-&spdif {
-	pinctrl-0 = <&spdifm0_tx>;
-	status = "okay";
-};
-
 &tsadc {
 	rockchip,hw-tshut-mode = <0>;
 	rockchip,hw-tshut-polarity = <0>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s-plus.dts b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s-plus.dts
index f72b1518c14f8..4eff0503928a8 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s-plus.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s-plus.dts
@@ -29,20 +29,3 @@ &emmc {
 	pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>;
 	status = "okay";
 };
-
-&gmac2io {
-	phy-handle = <&rtl8211e>;
-	tx_delay = <0x24>;
-	rx_delay = <0x18>;
-
-	mdio {
-		rtl8211e: ethernet-phy@1 {
-			reg = <1>;
-			pinctrl-0 = <&eth_phy_reset_pin>;
-			pinctrl-names = "default";
-			reset-assert-us = <10000>;
-			reset-deassert-us = <50000>;
-			reset-gpios = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>;
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-base.dtsi
index 4dcceb9136b7f..49c68fe65de66 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-base.dtsi
@@ -2145,8 +2145,6 @@ edp: dp@ff970000 {
 		interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH 0>;
 		clocks = <&cru PCLK_EDP>, <&cru PCLK_EDP_CTRL>, <&cru PCLK_VIO_GRF>;
 		clock-names = "dp", "pclk", "grf";
-		pinctrl-names = "default";
-		pinctrl-0 = <&edp_hpd>;
 		power-domains = <&power RK3399_PD_EDP>;
 		resets = <&cru SRST_P_EDP_CTRL>;
 		reset-names = "dp";
@@ -2384,6 +2382,7 @@ cif_clkouta: cif-clkouta {
 		};
 
 		edp {
+			/omit-if-no-ref/
 			edp_hpd: edp-hpd {
 				rockchip,pins =
 					<4 RK_PC7 2 &pcfg_pull_none>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi
index 9d07353df52c8..3f3cb0eb58096 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi
@@ -241,6 +241,8 @@ &dmc {
 };
 
 &edp {
+	pinctrl-names = "default";
+	pinctrl-0 = <&edp_hpd>;
 	status = "okay";
 
 	/*
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-nanopc-t4-hd702e.dtso b/arch/arm64/boot/dts/rockchip/rk3399-nanopc-t4-hd702e.dtso
new file mode 100644
index 0000000000000..766925b635717
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-nanopc-t4-hd702e.dtso
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * FriendlyElec HD702E LCD on NanoPC-T4 board
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+
+&{/} {
+	vdd_3_3v: regulator-vdd_3_3v {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vdd_3.3v";
+		vin-supply = <&vcc12v0_sys>;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm0 0 25000 0>;
+		enable-gpios = <&gpio4 RK_PD5 GPIO_ACTIVE_HIGH>;
+		brightness-levels = <0 255>;
+		default-brightness-level = <200>;
+		num-interpolated-steps = <255>;
+		pinctrl-0 = <&bl_en>;
+		pinctrl-names = "default";
+	};
+};
+
+&edp {
+	force-hpd;
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "friendlyarm,hd702e";
+			backlight = <&backlight>;
+			no-hpd;
+			power-supply = <&vdd_3_3v>;
+
+			port {
+				panel_in_edp: endpoint {
+					remote-endpoint = <&edp_out_panel>;
+				};
+			};
+		};
+	};
+};
+
+&edp_out {
+	edp_out_panel: endpoint {
+		remote-endpoint = <&panel_in_edp>;
+	};
+};
+
+&i2c4 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen@5d {
+		compatible = "goodix,gt9271";
+		reg = <0x5d>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <RK_PC4 IRQ_TYPE_EDGE_FALLING>;
+		irq-gpios = <&gpio1 RK_PC4 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&touch_int &touch_rst>;
+		pinctrl-names = "default";
+		AVDD28-supply = <&vdd_3_3v>;
+		VDDIO-supply = <&vdd_3_3v>;
+	};
+};
+
+&pinctrl {
+	backlight {
+		bl_en: bl-en {
+			rockchip,pins = <4 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	touchscreen {
+		touch_int: touch-int {
+			rockchip,pins = <1 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		touch_rst: touch-rst {
+			rockchip,pins = <1 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-nanopi4.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-nanopi4.dtsi
index c8eb5481f43d0..9e4fea84c4fb1 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-nanopi4.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-nanopi4.dtsi
@@ -206,6 +206,8 @@ &gpu {
 };
 
 &hdmi {
+	avdd-0v9-supply = <&vcca0v9_s3>;
+	avdd-1v8-supply = <&vcca1v8_s3>;
 	ddc-i2c-bus = <&i2c7>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&hdmi_cec>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
index 753d513449540..b5c05928142c0 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts
@@ -401,8 +401,6 @@ &cpu_l3 {
 
 &edp {
 	force-hpd;
-	pinctrl-names = "default";
-	pinctrl-0 = <&edp_hpd>;
 	status = "okay";
 };
 
@@ -879,12 +877,6 @@ vcc5v0_host_en_pin: vcc5v0-host-en-pin {
 		};
 	};
 
-	wifi {
-		wifi_host_wake_l: wifi-host-wake-l {
-			rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
-		};
-	};
-
 	wireless-bluetooth {
 		bt_wake_pin: bt-wake-pin {
 			rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -942,19 +934,7 @@ &sdio0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>;
 	sd-uhs-sdr104;
-	#address-cells = <1>;
-	#size-cells = <0>;
 	status = "okay";
-
-	brcmf: wifi@1 {
-		compatible = "brcm,bcm4329-fmac";
-		reg = <1>;
-		interrupt-parent = <&gpio0>;
-		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "host-wake";
-		pinctrl-names = "default";
-		pinctrl-0 = <&wifi_host_wake_l>;
-	};
 };
 
 &sdhci {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
index a4ceafe6dd7a8..80d6ea0eda845 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator.dts
@@ -141,6 +141,8 @@ sdio_pwrseq: sdio-pwrseq {
 };
 
 &edp {
+	pinctrl-names = "default";
+	pinctrl-0 = <&edp_hpd>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts b/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts
index ed65d31204446..18a560a6e2a4a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts
@@ -635,10 +635,10 @@ &uart1 {
 	status = "okay";
 
 	bluetooth: bluetooth {
-		compatible = "brcm,bcm43438-bt";
+		compatible = "brcm,bcm43430a1-bt";
 		clocks = <&rk809 1>;
 		clock-names = "lpo";
-		max-speed = <3000000>;
+		max-speed = <1500000>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>;
 		shutdown-gpios = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-onething-edge-cube.dts b/arch/arm64/boot/dts/rockchip/rk3566-onething-edge-cube.dts
new file mode 100644
index 0000000000000..b57bf7bf10f50
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-onething-edge-cube.dts
@@ -0,0 +1,342 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/pwm/pwm.h>
+#include "rk3566.dtsi"
+
+/ {
+	model = "OneThing Edge Cube (OEC)/OEC Turbo";
+	compatible = "onething,edge-cube", "rockchip,rk3566";
+
+	aliases {
+		ethernet0 = &gmac1;
+		mmc0 = &sdhci;
+	};
+
+	chosen: chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	gmac1_clkin: external-gmac1-clock {
+		compatible = "fixed-clock";
+		clock-frequency = <125000000>;
+		clock-output-names = "gmac1_clkin";
+		#clock-cells = <0>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		rgb_led_b: led-0 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&rgb_led_b_enable_l>;
+		};
+
+		rgb_led_g: led-1 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&rgb_led_g_enable_l>;
+		};
+
+		rgb_led_r: led-2 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio0 RK_PC3 GPIO_ACTIVE_LOW>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&rgb_led_r_enable_l>;
+		};
+	};
+
+	vcc_1v8: regulator-vcc-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc_3v3: regulator-vcc-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc12v0_dcin: regulator-vcc12v0-dcin {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc12v0_dcin";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	vcc3v3_sys: regulator-vcc3v3-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc12v0_dcin>;
+	};
+
+	vcc5v0_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc12v0_dcin>;
+	};
+
+	vcc5v0_usb_host: regulator-vcc5v0-usb-host {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_usb_host_en>;
+		regulator-name = "vcc5v0_usb_host";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vdd_cpu: regulator-vdd-cpu {
+		compatible = "pwm-regulator";
+		pwms = <&pwm0 0 5000 PWM_POLARITY_INVERTED>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "vdd_cpu";
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-settling-time-up-us = <250>;
+		pwm-supply = <&vcc5v0_sys>;
+	};
+
+	vdd_fixed: regulator-vdd-fixed {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "vdd_fixed";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vdd_logic: regulator-vdd-logic {
+		compatible = "pwm-regulator";
+		pwms = <&pwm1 0 5000 PWM_POLARITY_INVERTED>;
+		regulator-name = "vdd_logic";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <1100000>;
+		regulator-settling-time-up-us = <250>;
+		pwm-supply = <&vcc5v0_sys>;
+	};
+};
+
+
+/* No hardware video output port */
+&display_subsystem {
+	status = "disabled";
+};
+
+&combphy1 {
+	status = "okay";
+};
+
+&combphy2 {
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&gmac1 {
+	assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>;
+	assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&gmac1_clkin>;
+	clock_in_out = "input";
+	phy-handle = <&rgmii_phy1>;
+	phy-mode = "rgmii-id";
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1m1_miim
+		    &gmac1m1_tx_bus2
+		    &gmac1m1_rx_bus2
+		    &gmac1m1_rgmii_clk
+		    &gmac1m1_rgmii_bus
+		    &gmac1m1_clkinout>;
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&mdio1 {
+	rgmii_phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pinctrl {
+	gmac {
+		eth_phy_reset_pin: eth-phy-reset-pin {
+			rockchip,pins = <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	leds {
+		rgb_led_b_enable_l: rgb-led-b-enable-l {
+			rockchip,pins = <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		rgb_led_g_enable_l: rgb-led-g-enable-l {
+			rockchip,pins = <3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		rgb_led_r_enable_l: rgb-led-r-enable-l {
+			rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb {
+		vcc5v0_usb_host_en: vcc5v0-usb-host-en {
+			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	pmuio1-supply = <&vcc_3v3>;
+	pmuio2-supply = <&vcc_3v3>;
+	vccio1-supply = <&vcc_1v8>;
+	vccio3-supply = <&vcc_3v3>;
+	vccio4-supply = <&vcc_1v8>;
+	vccio5-supply = <&vcc_3v3>;
+	vccio6-supply = <&vcc_1v8>;
+	vccio7-supply = <&vcc_3v3>;
+	status = "okay";
+};
+
+&pwm0 {
+	status = "okay";
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sata2 {
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	mmc-hs200-1_8v;
+	no-sd;
+	no-sdio;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_bus8 &emmc_clk &emmc_cmd &emmc_datastrobe>;
+	status = "okay";
+};
+
+&tsadc {
+	rockchip,hw-tshut-mode = <1>;
+	rockchip,hw-tshut-polarity = <0>;
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci  {
+	status = "okay";
+};
+
+&usb2phy0 {
+	status = "okay";
+};
+
+&usb2phy0_host {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
+
+&usb2phy0_otg {
+	status = "okay";
+};
+
+&usb2phy1 {
+	status = "okay";
+};
+
+&usb2phy1_host {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
+
+&usb2phy1_otg {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-pinenote.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-pinenote.dtsi
index 791719acb9dd0..071cbdc3f6d57 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-pinenote.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3566-pinenote.dtsi
@@ -40,6 +40,25 @@ spk_amp: audio-amplifier {
 		sound-name-prefix = "Speaker Amp";
 	};
 
+	battery: battery {
+		compatible = "simple-battery";
+		charge-full-design-microamp-hours = <4000000>;
+		charge-term-current-microamp = <300000>;
+		constant-charge-current-max-microamp = <2000000>;
+		constant-charge-voltage-max-microvolt = <4200000>;
+		factory-internal-resistance-micro-ohms = <96000>;
+		voltage-max-design-microvolt = <4200000>;
+		voltage-min-design-microvolt = <3500000>;
+
+		ocv-capacity-celsius = <20>;
+		ocv-capacity-table-0 = <4168000 100>, <4109000 95>, <4066000 90>, <4023000 85>,
+					<3985000 80>, <3954000 75>, <3924000 70>, <3897000 65>,
+					<3866000 60>, <3826000 55>, <3804000 50>, <3789000 45>,
+					<3777000 40>, <3770000 35>, <3763000 30>, <3750000 25>,
+					<3732000 20>, <3710000 15>, <3680000 10>, <3670000  5>,
+					<3500000  0>;
+	};
+
 	dmic_codec: dmic-codec {
 		compatible = "dmic-codec";
 		num-channels = <6>;
@@ -215,6 +234,11 @@ &cpu3 {
 	cpu-supply = <&vdd_cpu>;
 };
 
+&gpu {
+	mali-supply = <&vdd_gpu_npu>;
+	status = "okay";
+};
+
 &i2c0 {
 	status = "okay";
 
@@ -260,6 +284,13 @@ rk817: pmic@20 {
 		vcc8-supply = <&vcc_sys>;
 		vcc9-supply = <&dcdc_boost>;
 
+		charger {
+			monitored-battery = <&battery>;
+			rockchip,resistor-sense-micro-ohms = <10000>;
+			rockchip,sleep-enter-current-microamp = <150000>;
+			rockchip,sleep-filter-current-microamp = <100000>;
+		};
+
 		regulators {
 			vdd_logic: DCDC_REG1 {
 				regulator-name = "vdd_logic";
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-qnap-ts133-pcb-13.dtso b/arch/arm64/boot/dts/rockchip/rk3566-qnap-ts133-pcb-13.dtso
new file mode 100644
index 0000000000000..f9a8194f5753f
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-qnap-ts133-pcb-13.dtso
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Device tree overlay for TS133 board PCB-13 revision.
+ *
+ * Copyright (C) 2025 Heiko Stuebner <heiko@sntech.de>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+
+&{/} {
+	/*
+	 * The default hardware-state of this gpio causes the drive
+	 * to be already running when entering the kernel.
+	 * regulator-boot-on is needed to prevent one additional
+	 * power-cycle on the drive.
+	 *
+	 * With regulator-boot-on we get the expected 1 cycle
+	 * per boot, without it we end up with 2 cycles as seen
+	 * via smartctl.
+	 */
+	hdd1_pwr: regulator-hdd1-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd1_power_pin>;
+		regulator-name = "hdd1-power";
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+};
+
+&gpio2 {
+	hdd1-present-hog {
+		gpios = <RK_PA2 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd1-present";
+	};
+};
+
+&pinctrl {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdd1_present_pin>;
+
+	hdd-power {
+		hdd1_power_pin: hdd1-power-pin {
+			rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	hdd-present {
+		hdd1_present_pin: hdd1-present-pin {
+			rockchip,pins = <2 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&sata2_port0 {
+	target-supply = <&hdd1_pwr>;
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-anbernic-rg-ds.dts b/arch/arm64/boot/dts/rockchip/rk3568-anbernic-rg-ds.dts
index 6ac1fe0d3c986..8d906ab02c5fb 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-anbernic-rg-ds.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-anbernic-rg-ds.dts
@@ -354,6 +354,7 @@ sound {
 		compatible = "simple-audio-card";
 		pinctrl-0 = <&hp_det>;
 		pinctrl-names = "default";
+		simple-audio-card,aux-devs = <&aw87391_pa_l>, <&aw87391_pa_r>;
 		simple-audio-card,format = "i2s";
 		simple-audio-card,hp-det-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>;
 		simple-audio-card,mclk-fs = <256>;
@@ -363,8 +364,10 @@ sound {
 			"MICL", "Mic Jack",
 			"Headphones", "HPOL",
 			"Headphones", "HPOR",
-			"Internal Speakers", "HPOL",
-			"Internal Speakers", "HPOR";
+			"Internal Speakers", "Left Amp OUT",
+			"Internal Speakers", "Right Amp OUT",
+			"Left Amp IN", "HPOL",
+			"Right Amp IN", "HPOR";
 		simple-audio-card,widgets =
 			"Microphone", "Mic Jack",
 			"Headphone", "Headphones",
@@ -468,6 +471,18 @@ vcc_wifi: regulator-vcc-wifi {
 		regulator-max-microvolt = <3300000>;
 		regulator-name = "vcc_wifi";
 	};
+
+	vdd_amp: regulator-vcc-amp {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vdd_amp_h>;
+		pinctrl-names = "default";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vccio_acodec>;
+		regulator-name = "vdd_amp";
+	};
 };
 
 &cpu0 {
@@ -840,8 +855,22 @@ &i2c2 {
 	pinctrl-names = "default";
 	status = "okay";
 
-	/* awinic,aw87391 at 0x58 */
-	/* awinic,aw87391 at 0x5b */
+	aw87391_pa_l: audio-codec@58 {
+		compatible = "anbernic,rgds-amp", "awinic,aw87391";
+		reg = <0x58>;
+		vdd-supply = <&vdd_amp>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "Left Amp";
+	};
+
+	aw87391_pa_r: audio-codec@5b {
+		compatible = "anbernic,rgds-amp", "awinic,aw87391";
+		reg = <0x5b>;
+		vdd-supply = <&vdd_amp>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "Right Amp";
+	};
+
 	/* invensense,icm42607p at 0x68 */
 };
 
@@ -1014,6 +1043,13 @@ touch1_irq: touch1-irq {
 		};
 	};
 
+	vdd-amp {
+		vdd_amp_h: vdd-amp-h {
+			rockchip,pins =
+				<4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	vcc-lcd {
 		vdd_lcd0_h: vdd-lcd0-h {
 			rockchip,pins =
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts233-pcb-12-11.dtso b/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts233-pcb-12-11.dtso
new file mode 100644
index 0000000000000..c7987171644c1
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts233-pcb-12-11.dtso
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Device tree overlay for TS233 board PCBs-12-11 revision.
+ *
+ * Copyright (C) 2025 Heiko Stuebner <heiko@sntech.de>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+
+&{/} {
+	/*
+	 * The default hardware-state of this gpio causes the drive
+	 * to be already running when entering the kernel.
+	 * regulator-boot-on is needed to prevent one additional
+	 * power-cycle on the drive.
+	 *
+	 * With regulator-boot-on we get the expected 1 cycle
+	 * per boot, without it we end up with 2 cycles as seen
+	 * via smartctl.
+	 */
+	hdd1_pwr: regulator-hdd1-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd1_power_pin>;
+		regulator-name = "hdd1-power";
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+
+	hdd2_pwr: regulator-hdd2-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC3 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd2_power_pin>;
+		regulator-name = "hdd2-power";
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+};
+
+&gpio2 {
+	hdd1-present-hog {
+		gpios = <RK_PA2 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd1-present";
+	};
+
+	hdd2-present-hog {
+		gpios = <RK_PA1 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd2-present";
+	};
+};
+
+&pinctrl {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdd1_present_pin &hdd2_present_pin>;
+
+	hdd-power {
+		hdd1_power_pin: hdd1-power-pin {
+			rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdd2_power_pin: hdd2-power-pin {
+			rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	hdd-present {
+		hdd1_present_pin: hdd1-present-pin {
+			rockchip,pins = <2 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		hdd2_present_pin: hdd2-present-pin {
+			rockchip,pins = <2 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&sata1_port0 {
+	target-supply = <&hdd2_pwr>;
+};
+
+&sata2_port0 {
+	target-supply = <&hdd1_pwr>;
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts433-pcb-12-10.dtso b/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts433-pcb-12-10.dtso
new file mode 100644
index 0000000000000..ce0fdc9f29898
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3568-qnap-ts433-pcb-12-10.dtso
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Device tree overlay for TS433 board PCBs-12-10 revision.
+ *
+ * Copyright (C) 2025 Heiko Stuebner <heiko@sntech.de>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+
+&{/} {
+	/*
+	 * The default hardware-state of this gpio causes the drive
+	 * to be already running when entering the kernel.
+	 * regulator-boot-on is needed to prevent one additional
+	 * power-cycle on the drive.
+	 *
+	 * With regulator-boot-on we get the expected 1 cycle
+	 * per boot, without it we end up with 2 cycles as seen
+	 * via smartctl.
+	 */
+	hdd1_pwr: regulator-hdd1-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd1_power_pin>;
+		regulator-name = "hdd1-power";
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+
+	hdd2_pwr: regulator-hdd2-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC3 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd2_power_pin>;
+		regulator-name = "hdd2-power";
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+
+	/*
+	 * HDD3+4 are connected to ports of the PCIe SATA controller.
+	 * Currently there is no way to attach those, so keep them
+	 * always on.
+	 */
+	hdd3_pwr: regulator-hdd3-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd3_power_pin>;
+		regulator-name = "hdd3-power";
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+
+	hdd4_pwr: regulator-hdd4-power {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdd4_power_pin>;
+		regulator-name = "hdd4-power";
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&dc_12v>;
+	};
+};
+
+&gpio2 {
+	hdd1-present-hog {
+		gpios = <RK_PA2 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd1-present";
+	};
+
+	hdd2-present-hog {
+		gpios = <RK_PA1 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd2-present";
+	};
+
+	hdd3-present-hog {
+		gpios = <RK_PD0 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd3-present";
+	};
+
+	hdd4-present-hog {
+		gpios = <RK_PD1 GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		input;
+		line-name = "hdd4-present";
+	};
+};
+
+&pinctrl {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdd1_present_pin &hdd2_present_pin &hdd3_present_pin
+		     &hdd4_present_pin>;
+
+	hdd-power {
+		hdd1_power_pin: hdd1-power-pin {
+			rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdd2_power_pin: hdd2-power-pin {
+			rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdd3_power_pin: hdd3-power-pin {
+			rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdd4_power_pin: hdd4-power-pin {
+			rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	hdd-present {
+		hdd1_present_pin: hdd1-present-pin {
+			rockchip,pins = <2 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		hdd2_present_pin: hdd2-present-pin {
+			rockchip,pins = <2 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		hdd3_present_pin: hdd3-present-pin {
+			rockchip,pins = <2 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		hdd4_present_pin: hdd4-present-pin {
+			rockchip,pins = <2 RK_PD1 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&sata1_port0 {
+	target-supply = <&hdd2_pwr>;
+};
+
+&sata2_port0 {
+	target-supply = <&hdd1_pwr>;
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-wolfvision-pf5-io-expander.dtso b/arch/arm64/boot/dts/rockchip/rk3568-wolfvision-pf5-io-expander.dtso
index 048933de2943e..8cfce71dd3181 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-wolfvision-pf5-io-expander.dtso
+++ b/arch/arm64/boot/dts/rockchip/rk3568-wolfvision-pf5-io-expander.dtso
@@ -11,6 +11,7 @@
 #include <dt-bindings/clock/rk3568-cru.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/media/video-interfaces.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 
 &{/} {
@@ -134,3 +135,22 @@ &usb2phy0_host {
 	phy-supply = <&usb_host_vbus>;
 	status = "okay";
 };
+
+&vicap {
+	pinctrl-names = "default";
+	pinctrl-0 = <&cif_clk &cif_dvp_clk &cif_dvp_bus16>;
+	status = "okay";
+};
+
+&vicap_dvp {
+	vicap_dvp_input: endpoint {
+		bus-type = <MEDIA_BUS_TYPE_BT656>;
+		bus-width = <16>;
+		pclk-sample = <MEDIA_PCLK_SAMPLE_DUAL_EDGE>;
+		rockchip,dvp-clk-delay = <10>;
+	};
+};
+
+&vicap_mmu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi
index a2c4957a58992..68b48606f6010 100644
--- a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi
@@ -3,6 +3,7 @@
  * Copyright (c) 2021 Rockchip Electronics Co., Ltd.
  */
 
+#include <dt-bindings/ata/ahci.h>
 #include <dt-bindings/clock/rk3568-cru.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -221,11 +222,20 @@ sata1: sata@fc400000 {
 			 <&cru CLK_SATA1_RXOOB>;
 		clock-names = "sata", "pmalive", "rxoob";
 		interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
-		phys = <&combphy1 PHY_TYPE_SATA>;
-		phy-names = "sata-phy";
 		ports-implemented = <0x1>;
 		power-domains = <&power RK3568_PD_PIPE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
 		status = "disabled";
+
+		sata1_port0: sata-port@0 {
+			reg = <0>;
+			hba-port-cap = <HBA_PORT_FBSCP>;
+			phys = <&combphy1 PHY_TYPE_SATA>;
+			phy-names = "sata-phy";
+			snps,rx-ts-max = <32>;
+			snps,tx-ts-max = <32>;
+		};
 	};
 
 	sata2: sata@fc800000 {
@@ -235,11 +245,20 @@ sata2: sata@fc800000 {
 			 <&cru CLK_SATA2_RXOOB>;
 		clock-names = "sata", "pmalive", "rxoob";
 		interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
-		phys = <&combphy2 PHY_TYPE_SATA>;
-		phy-names = "sata-phy";
 		ports-implemented = <0x1>;
 		power-domains = <&power RK3568_PD_PIPE>;
+		#address-cells = <1>;
+		#size-cells = <0>;
 		status = "disabled";
+
+		sata2_port0: sata-port@0 {
+			reg = <0>;
+			hba-port-cap = <HBA_PORT_FBSCP>;
+			phys = <&combphy2 PHY_TYPE_SATA>;
+			phy-names = "sata-phy";
+			snps,rx-ts-max = <32>;
+			snps,tx-ts-max = <32>;
+		};
 	};
 
 	usb_host0_xhci: usb@fcc00000 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
index f5746bc2970b3..fb0dd1bc51482 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
@@ -270,6 +270,7 @@ sound {
 		simple-audio-card,frame-master = <&masterdai>;
 		simple-audio-card,hp-det-gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_LOW>;
 		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,pin-switches = "Headphones", "Speaker", "Main Mic", "Headset Mic";
 		simple-audio-card,routing =
 			"Headphone Power INL", "LOUT1",
 			"Headphone Power INR", "ROUT1",
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
new file mode 100644
index 0000000000000..98d5d00d63b57
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
@@ -0,0 +1,1071 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 Rockchip Electronics Co., Ltd.
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include "rk3576.dtsi"
+
+/ {
+	model = "Rockchip RK3576 EVB2 V10 Board";
+	compatible = "rockchip,rk3576-evb2-v10", "rockchip,rk3576";
+
+	aliases {
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc;
+	};
+
+	chosen {
+		stdout-path = "serial0:1500000n8";
+	};
+
+	dp-con {
+		compatible = "dp-connector";
+		dp-pwr-supply = <&vcc3v3_dp_port>;
+		label = "DP OUT";
+		type = "full-size";
+
+		port {
+			dp0_con_in: endpoint {
+				remote-endpoint = <&dp0_out_con>;
+			};
+		};
+	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 1>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-back {
+			label = "back";
+			linux,code = <KEY_BACK>;
+			press-threshold-microvolt = <1235000>;
+		};
+
+		button-menu {
+			label = "menu";
+			linux,code = <KEY_MENU>;
+			press-threshold-microvolt = <890000>;
+		};
+
+		button-vol-down {
+			label = "volume down";
+			linux,code = <KEY_VOLUMEDOWN>;
+			press-threshold-microvolt = <417000>;
+		};
+
+		button-vol-up {
+			label = "volume up";
+			linux,code = <KEY_VOLUMEUP>;
+			press-threshold-microvolt = <17000>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		work_led: led-0 {
+			gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_pwren>;
+
+		/*
+		 * On the module itself this is one of these (depending
+		 * on the actual card populated):
+		 * - SDIO_RESET_L_WL_REG_ON
+		 * - PDN (power down when low)
+		 */
+		post-power-on-delay-ms = <200>;
+		reset-gpios = <&gpio1 RK_PC6 GPIO_ACTIVE_LOW>;
+	};
+
+	vbus5v0_usb0: regulator-vbus5v0-usb0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_usb3_host0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+		gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&vcc5v0_device>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb_otg0_pwren>;
+	};
+
+	vbus5v0_usb1: regulator-vcc5v0-usb1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_usb3_host1";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+		gpio = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&vcc5v0_device>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb_host_pwren>;
+	};
+
+	vcc12v_dcin: regulator-vcc12v-dcin {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc12v_dcin";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	vcc1v2_ufs_vccq_s0: regulator-vcc1v2-ufs-vccq-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v2_ufs_vccq_s0";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc1v8_ufs_vccq2_s0: regulator-vcc1v8-ufs-vccq2-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8_ufs_vccq2_s0";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_1v8_s3>;
+	};
+
+	vcc3v3_dp_port: regulator-vcc3v3-dp-port {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_dp_port";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&vcc_3v3_s3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&image_pwren>;
+	};
+
+	vcc3v3_pcie1: regulator-vcc3v3-pcie1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_pcie1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <5000>;
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc3v3_rtc_s5: regulator-vcc3v3-rtc-s5 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_rtc_s5";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc3v3_sata_pwren: vcc3v3-sata-pwren {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_satapm";
+		enable-active-high;
+		gpio = <&gpio4 RK_PC7 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&satapm_pwren>;
+	};
+
+	vcc3v3_sd: regulator-vcc-3v3-sd {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdmmc_pwren>;
+		regulator-name = "vcc3v3_sd";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3_s0>;
+	};
+
+	vcc5v0_device: regulator-vcc5v0-device {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_device";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v1_nldo_s3";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc_1v8_s0: regulator-vcc-1v8-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v8_s0";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_1v8_s3>;
+	};
+
+	vcc_2v0_pldo_s3: regulator-vcc-2v0-pldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_2v0_pldo_s3";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <2000000>;
+		regulator-max-microvolt = <2000000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc_3v3_s0: regulator-vcc-3v3-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3_s0";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3_s3>;
+	};
+
+	vcc_ufs_s0: regulator-vcc-ufs-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_ufs_s0";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_sys>;
+	};
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b2 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b3 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&combphy0_ps {
+	status = "okay";
+};
+
+&combphy1_psu {
+	status = "okay";
+};
+
+&dp {
+	pinctrl-0 = <&dpm0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&dp0_in {
+	dp0_in_vp1: endpoint {
+		remote-endpoint = <&vp1_out_dp0>;
+	};
+};
+
+&dp0_out {
+	dp0_out_con: endpoint {
+		remote-endpoint = <&dp0_con_in>;
+	};
+};
+
+&gmac0 {
+	clock_in_out = "output";
+	phy-mode = "rgmii-id";
+	phy-handle = <&rgmii_phy0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&eth0m1_miim
+		     &eth0m1_tx_bus2
+		     &eth0m1_rx_bus2
+		     &eth0m1_rgmii_clk
+		     &eth0m1_rgmii_bus>;
+	status = "okay";
+};
+
+&gmac1 {
+	clock_in_out = "output";
+	phy-handle = <&rgmii_phy1>;
+	phy-mode = "rgmii-id";
+	pinctrl-names = "default";
+	pinctrl-0 = <&eth1m0_miim
+		     &eth1m0_tx_bus2
+		     &eth1m0_rx_bus2
+		     &eth1m0_rgmii_clk
+		     &eth1m0_rgmii_bus>;
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu_s0>;
+	status = "okay";
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&hdptxphy {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+
+	rk806: pmic@23 {
+		compatible = "rockchip,rk806";
+		reg = <0x23>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>,
+			    <&rk806_dvs2_null>, <&rk806_dvs3_null>;
+		system-power-controller;
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc5-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_sys>;
+		vcc9-supply = <&vcc_sys>;
+		vcc10-supply = <&vcc_sys>;
+		vcc11-supply = <&vcc_2v0_pldo_s3>;
+		vcc12-supply = <&vcc_sys>;
+		vcc13-supply = <&vcc_1v1_nldo_s3>;
+		vcc14-supply = <&vcc_1v1_nldo_s3>;
+		vcca-supply = <&vcc_sys>;
+
+		rk806_dvs1_null: dvs1-null-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs2_null: dvs2-null-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs3_null: dvs3-null-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs1_slp: dvs1-slp-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs1_pwrdn: dvs1-pwrdn-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs1_rst: dvs1-rst-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs2_slp: dvs2-slp-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs2_pwrdn: dvs2-pwrdn-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs2_rst: dvs2-rst-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs2_dvs: dvs2-dvs-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun4";
+		};
+
+		rk806_dvs2_gpio: dvs2-gpio-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun5";
+		};
+
+		rk806_dvs3_slp: dvs3-slp-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs3_pwrdn: dvs3-pwrdn-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs3_rst: dvs3-rst-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs3_dvs: dvs3-dvs-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun4";
+		};
+
+		rk806_dvs3_gpio: dvs3-gpio-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun5";
+		};
+
+		regulators {
+			vdd_cpu_big_s0: dcdc-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_cpu_big_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_npu_s0: dcdc-reg2 {
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_npu_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_lit_s0: dcdc-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_cpu_lit_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vcc_3v3_s3: dcdc-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_3v3_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vdd_gpu_s0: dcdc-reg5 {
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <900000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_gpu_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			vddq_ddr_s0: dcdc-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vddq_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_logic_s0: dcdc-reg7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <800000>;
+				regulator-name = "vdd_logic_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: dcdc-reg8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd2_ddr_s3: dcdc-reg9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vdd2_ddr_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vdd_ddr_s0: dcdc-reg10 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdd_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_1v8_s0: pldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo2_s0: pldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_1v2_s0: pldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdda_1v2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_3v3_s0: pldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcca_3v3_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd_s0: pldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo6_s3: pldo-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo6_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd_0v75_s3: nldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdda_ddr_pll_s0: nldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_ddr_pll_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v75_hdmi_s0: nldo-reg3 {
+				regulator-boot-on;
+				regulator-min-microvolt = <837500>;
+				regulator-max-microvolt = <837500>;
+				regulator-name = "vdda0v75_hdmi_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v85_s0: nldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_0v85_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v75_s0: nldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdda_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	status = "okay";
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		clock-output-names = "hym8563";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA5 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rtc_int>;
+		wakeup-source;
+		#clock-cells = <0>;
+	};
+};
+
+&mdio0 {
+	rgmii_phy0: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rgmii_phy0_rst>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PD3 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&mdio1 {
+	rgmii_phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rgmii_phy1_rst>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PD6 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pinctrl {
+	bluetooth {
+		bt_reg_on: bt-reg-on {
+			rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		bt_wake_host: bt-wake-host {
+			rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		host_wake_bt: host-wake-bt {
+			rockchip,pins = <1 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	dp {
+		image_pwren: image-pwren {
+			rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	hym8563 {
+		rtc_int: rtc-int {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	network {
+		rgmii_phy0_rst: rgmii-phy0-rst {
+			rockchip,pins = <3 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		rgmii_phy1_rst: rgmii-phy1-rst {
+			rockchip,pins = <3 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pcie1 {
+		pcie1_rst: pcie1-rst {
+			rockchip,pins = <4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sata {
+		satapm_pwren: satapm-pwren {
+			rockchip,pins = <4 RK_PC7 RK_FUNC_GPIO &pcfg_output_high>;
+		};
+	};
+
+	sdmmc {
+		sdmmc_pwren: sdmmc-pwren {
+			rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb {
+		usb_host_pwren: usb-host-pwren {
+			rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		usb_hub_reset: usb-hub-reset {
+			rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		usb_otg0_pwren: usb-otg0-pwren {
+			rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	wifi {
+		wifi_host_wake: wifi-host-wake {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		wifi_pwren: wifi-pwren {
+			rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&sai6 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcca_1v8_s0>;
+	status = "okay";
+};
+
+&sata0 {
+	target-supply = <&vcc3v3_sata_pwren>;
+	status = "okay";
+};
+
+&sdio {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	max-frequency = <150000000>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-sd;
+	no-mmc;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc1m0_bus4 &sdmmc1m0_clk &sdmmc1m0_cmd>;
+	sd-uhs-sdr104;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB0 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake>;
+	};
+};
+
+&sdhci {
+	bus-width = <8>;
+	full-pwr-cycle-in-suspend;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sdio;
+	no-sd;
+	non-removable;
+	status = "okay";
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	no-sdio;
+	no-mmc;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det &sdmmc0_bus4>;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sd>;
+	vqmmc-supply = <&vccio_sd_s0>;
+	status = "okay";
+};
+
+&u2phy0 {
+	status = "okay";
+};
+
+&u2phy0_otg {
+	phy-supply = <&vbus5v0_usb0>;
+	status = "okay";
+};
+
+&u2phy1 {
+	status = "okay";
+};
+
+&u2phy1_otg {
+	phy-supply = <&vbus5v0_usb1>;
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-0 = <&uart4m1_xfer &uart4m1_ctsn &uart4m1_rtsn>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		clocks = <&hym8563>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB1 IRQ_TYPE_LEVEL_HIGH>;
+		pinctrl-0 = <&bt_reg_on &bt_wake_host &host_wake_bt>;
+		pinctrl-names = "default";
+		shutdown-gpios = <&gpio1 RK_PC7 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&vcc_3v3_s3>;
+		vddio-supply = <&vcc_1v8_s3>;
+	};
+};
+
+&ufshc {
+	vcc-supply = <&vcc_ufs_s0>;
+	vccq-supply = <&vcc1v2_ufs_vccq_s0>;
+	vccq2-supply = <&vcc1v8_ufs_vccq2_s0>;
+	status = "okay";
+};
+
+&usbdp_phy {
+	rockchip,dp-lane-mux = <2 3>;
+	status = "okay";
+};
+
+&usb_drd0_dwc3 {
+	dr_mode = "otg";
+	extcon = <&u2phy0>;
+	status = "okay";
+};
+
+&usb_drd1_dwc3 {
+	dr_mode = "host";
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb_hub_reset>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb1a86,8091";
+		reg = <1>;
+		reset-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_LOW>;
+		v5-supply = <&vcc_3v3_s0>;
+		vdd33-supply = <&vcc_3v3_s0>;
+	};
+};
+
+&vop {
+	/*
+	 * If no dedicated PLL was specified, the GPLL would be automatically
+	 * assigned as the PLL source for dclk_vp1_src. As the frequency of GPLL
+	 * is 1188 MHz, we can only get typical clock frequencies such as
+	 * 74.25MHz, 148.5MHz, 297MHz, 594MHz.
+	 *
+	 * So here we set the parent clock of VP1 to VPLL so that we can get
+	 * any frequency.
+	 */
+	assigned-clocks = <&cru DCLK_VP1_SRC>;
+	assigned-clock-parents = <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
+
+&vp1 {
+	vp1_out_dp0: endpoint@a {
+		reg = <ROCKCHIP_VOP2_EP_DP0>;
+		remote-endpoint = <&dp0_in_vp1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
index 49ccdf12ef7eb..53ff6bd027af8 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
@@ -1483,6 +1483,34 @@ hdmi_out: port@1 {
 			};
 		};
 
+		dp: dp@27e40000 {
+			compatible = "rockchip,rk3576-dp";
+			reg = <0x0 0x27e40000 0x0 0x30000>;
+			interrupts = <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>;
+			assigned-clocks = <&cru CLK_AUX16MHZ_0>;
+			assigned-clock-rates = <16000000>;
+			clocks = <&cru PCLK_DP0>, <&cru CLK_AUX16MHZ_0>,
+				 <&cru ACLK_DP0>;
+			clock-names = "apb", "aux", "hdcp";
+			resets = <&cru SRST_DP0>;
+			phys = <&usbdp_phy PHY_TYPE_DP>;
+			power-domains = <&power RK3576_PD_VO1>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				dp0_in: port@0 {
+					reg = <0>;
+				};
+
+				dp0_out: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
 		sai7: sai@27ed0000 {
 			compatible = "rockchip,rk3576-sai";
 			reg = <0x0 0x27ed0000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
index 7fe9593d8c198..4fb8888c281c8 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
@@ -907,7 +907,7 @@ power-domain@RK3588_PD_VCODEC {
 				#size-cells = <0>;
 				#power-domain-cells = <0>;
 
-				power-domain@RK3588_PD_RKVDEC0 {
+				pd_rkvdec0: power-domain@RK3588_PD_RKVDEC0 {
 					reg = <RK3588_PD_RKVDEC0>;
 					clocks = <&cru HCLK_RKVDEC0>,
 						 <&cru HCLK_VDPU_ROOT>,
@@ -917,7 +917,7 @@ power-domain@RK3588_PD_RKVDEC0 {
 					pm_qos = <&qos_rkvdec0>;
 					#power-domain-cells = <0>;
 				};
-				power-domain@RK3588_PD_RKVDEC1 {
+				pd_rkvdec1: power-domain@RK3588_PD_RKVDEC1 {
 					reg = <RK3588_PD_RKVDEC1>;
 					clocks = <&cru HCLK_RKVDEC1>,
 						 <&cru HCLK_VDPU_ROOT>,
@@ -926,7 +926,7 @@ power-domain@RK3588_PD_RKVDEC1 {
 					pm_qos = <&qos_rkvdec1>;
 					#power-domain-cells = <0>;
 				};
-				power-domain@RK3588_PD_VENC0 {
+				pd_venc0: power-domain@RK3588_PD_VENC0 {
 					reg = <RK3588_PD_VENC0>;
 					clocks = <&cru HCLK_RKVENC0>,
 						 <&cru ACLK_RKVENC0>;
@@ -937,7 +937,7 @@ power-domain@RK3588_PD_VENC0 {
 					#size-cells = <0>;
 					#power-domain-cells = <0>;
 
-					power-domain@RK3588_PD_VENC1 {
+				pd_venc1: power-domain@RK3588_PD_VENC1 {
 						reg = <RK3588_PD_VENC1>;
 						clocks = <&cru HCLK_RKVENC1>,
 							 <&cru HCLK_RKVENC0>,
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
index c9d284cb738b7..09bc7b68dcc05 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
@@ -568,6 +568,22 @@ &pd_gpu {
 	domain-supply = <&vdd_gpu_s0>;
 };
 
+&pd_rkvdec0 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_rkvdec1 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_venc0 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_venc1 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
 &pinctrl {
 	audio {
 		hp_detect: headphone-detect {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
index 952affaf455cf..5f5d89a33a4ae 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
@@ -86,25 +86,16 @@ led-1 {
 		};
 	};
 
-	/*
-	 * 100MHz reference clock for PCIe peripherals from PI6C557-05BLE
-	 * clock generator.
-	 * The clock output is gated via the OE pin on the clock generator.
-	 * This is modeled as a fixed-clock plus a gpio-gate-clock.
-	 */
-	pcie_refclk_gen: pcie-refclk-gen-clock {
-		compatible = "fixed-clock";
+	/* 100MHz PCIe reference clock from PI6C557-05BLE */
+	pcie_refclk: pcie-clock-generator {
+		compatible = "gated-fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <100000000>;
-	};
-
-	pcie_refclk: pcie-refclk-clock {
-		compatible = "gpio-gate-clock";
-		clocks = <&pcie_refclk_gen>;
-		#clock-cells = <0>;
+		clock-output-names = "pcie-refclk-clock";
 		enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_LOW>; /* PCIE30X4_CLKREQN_M0 */
 		pinctrl-names = "default";
 		pinctrl-0 = <&pcie30x4_clkreqn_m0>;
+		vdd-supply = <&vcca_3v3_s0>;
 	};
 
 	pps {
@@ -588,7 +579,7 @@ led1_pin: led1-pin {
 
 	pcie30x4 {
 		pcie30x4_clkreqn_m0: pcie30x4-clkreqn-m0 {
-			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 
 		pcie30x4_perstn_m0: pcie30x4-perstn-m0 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
index 90e7fe254491b..84b6b53f016ab 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 #include <dt-bindings/soc/rockchip,vop2.h>
+#include <dt-bindings/thermal/thermal.h>
 #include <dt-bindings/usb/pd.h>
 #include "rk3588.dtsi"
 
@@ -89,6 +90,14 @@ usr_led: led-1 {
 		};
 	};
 
+	fan: pwm-fan {
+		compatible = "pwm-fan";
+		cooling-levels = <0 35 64 100 150 255>;
+		fan-supply = <&vcc5v0_sys>;
+		pwms = <&pwm1 0 50000 0>;
+		#cooling-cells = <2>;
+	};
+
 	sound {
 		compatible = "simple-audio-card";
 		pinctrl-names = "default";
@@ -590,6 +599,36 @@ &i2s6_8ch {
 	status = "okay";
 };
 
+&package_thermal {
+	polling-delay = <1000>;
+
+	trips {
+		package_warm: package-warm {
+			temperature = <55000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+
+		package_hot: package-hot {
+			temperature = <65000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+	};
+
+	cooling-maps {
+		map0 {
+			trip = <&package_warm>;
+			cooling-device = <&fan THERMAL_NO_LIMIT 1>;
+		};
+
+		map1 {
+			trip = <&package_hot>;
+			cooling-device = <&fan 2 THERMAL_NO_LIMIT>;
+		};
+	};
+};
+
 &pcie2x1l0 {
 	reset-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>;
 	vpcie3v3-supply = <&vcc_3v3_pcie20>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
index 27269b7b08aa7..a0e97481afb7d 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
@@ -47,23 +47,16 @@ led-1 {
 		};
 	};
 
-	/*
-	 * 100MHz reference clock for PCIe peripherals from PI6C557-05BLE
-	 * clock generator.
-	 * The clock output is gated via the OE pin on the clock generator.
-	 * This is modeled as a fixed-clock plus a gpio-gate-clock.
-	 */
-	pcie_refclk_gen: pcie-refclk-gen-clock {
-		compatible = "fixed-clock";
+	/* 100MHz PCIe reference clock from PI6C557-05BLE */
+	pcie_refclk: pcie-clock-generator {
+		compatible = "gated-fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <100000000>;
-	};
-
-	pcie_refclk: pcie-refclk-clock {
-		compatible = "gpio-gate-clock";
-		clocks = <&pcie_refclk_gen>;
-		#clock-cells = <0>;
+		clock-output-names = "pcie-refclk-clock";
 		enable-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; /* PCIE30X4_CLKREQN_M1_L */
+		pinctrl-names = "default";
+		pinctrl-0 = <&pcie30x4_clkreqn_m1_l>;
+		vdd-supply = <&vcca_3v3_s0>;
 	};
 
 	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
@@ -362,6 +355,12 @@ module_led_pin: module-led-pin {
 		};
 	};
 
+	pcie30x4 {
+		pcie30x4_clkreqn_m1_l: pcie30x4-clkreqn-m1-l {
+			rockchip,pins = <4 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	usb3 {
 		usb3_id: usb3-id {
 			rockchip,pins =
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
index f82050597ab31..d6b62cd1b90b5 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
@@ -381,6 +381,22 @@ &pd_gpu {
 	domain-supply = <&vdd_gpu_s0>;
 };
 
+&pd_rkvdec0 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_rkvdec1 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_venc0 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
+&pd_venc1 {
+	domain-supply = <&vdd_vdenc_s0>;
+};
+
 &pinctrl {
 	audio {
 		hp_detect: headphone-detect {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
index e8ad525ba3f9b..89618394c0bfb 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
@@ -60,8 +60,8 @@ axis@0 {
 			reg = <0>;
 			abs-flat = <40>;
 			abs-fuzz = <30>;
-			abs-range = <0 4095>;
-			linux,code = <ABS_RX>;
+			abs-range = <4095 0>;
+			linux,code = <ABS_RY>;
 		};
 
 		axis@1 {
@@ -69,7 +69,7 @@ axis@1 {
 			abs-flat = <40>;
 			abs-fuzz = <30>;
 			abs-range = <0 4095>;
-			linux,code = <ABS_RY>;
+			linux,code = <ABS_RX>;
 		};
 
 		axis@2 {
@@ -77,7 +77,7 @@ axis@2 {
 			abs-flat = <40>;
 			abs-fuzz = <30>;
 			abs-range = <0 4095>;
-			linux,code = <ABS_Y>;
+			linux,code = <ABS_X>;
 		};
 
 		axis@3 {
@@ -85,7 +85,7 @@ axis@3 {
 			abs-flat = <40>;
 			abs-fuzz = <30>;
 			abs-range = <0 4095>;
-			linux,code = <ABS_X>;
+			linux,code = <ABS_Y>;
 		};
 	};
 
@@ -318,7 +318,7 @@ pwm_fan: pwm-fan {
 		compatible = "pwm-fan";
 		#cooling-cells = <2>;
 		cooling-levels = <0 120 150 180 210 240 255>;
-		fan-supply = <&vcc5v0_sys>;
+		fan-supply = <&vcc5v0_spk>;
 		interrupt-parent = <&gpio4>;
 		interrupts = <RK_PB2 IRQ_TYPE_EDGE_RISING>;
 		pulses-per-revolution = <4>;
diff --git a/arch/arm64/boot/dts/st/stm32mp231.dtsi b/arch/arm64/boot/dts/st/stm32mp231.dtsi
index b5d81d1ee153f..1121dca7880fd 100644
--- a/arch/arm64/boot/dts/st/stm32mp231.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp231.dtsi
@@ -354,7 +354,7 @@ i2c1: i2c@40120000 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 21 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C1>;
 				resets = <&rcc I2C1_R>;
 				dmas = <&hpdma 27 0x20 0x3012>,
@@ -363,6 +363,7 @@ i2c1: i2c@40120000 {
 				access-controllers = <&rifsc 41>;
 				power-domains = <&cluster_pd>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -372,7 +373,7 @@ i2c2: i2c@40130000 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 22 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C2>;
 				resets = <&rcc I2C2_R>;
 				dmas = <&hpdma 30 0x20 0x3012>,
@@ -381,6 +382,7 @@ i2c2: i2c@40130000 {
 				access-controllers = <&rifsc 42>;
 				power-domains = <&cluster_pd>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -390,7 +392,7 @@ i2c7: i2c@40180000 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 50 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C7>;
 				resets = <&rcc I2C7_R>;
 				dmas = <&hpdma 45 0x20 0x3012>,
@@ -399,6 +401,7 @@ i2c7: i2c@40180000 {
 				access-controllers = <&rifsc 47>;
 				power-domains = <&cluster_pd>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -668,7 +671,7 @@ i2c8: i2c@46040000 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti2 25 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C8>;
 				resets = <&rcc I2C8_R>;
 				dmas = <&hpdma 168 0x20 0x3012>,
@@ -677,6 +680,7 @@ i2c8: i2c@46040000 {
 				access-controllers = <&rifsc 48>;
 				power-domains = <&cluster_pd>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
diff --git a/arch/arm64/boot/dts/st/stm32mp235f-dk.dts b/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
index 5ecc5ef615907..ff4b32a3b12c8 100644
--- a/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
+++ b/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
@@ -117,6 +117,32 @@ phy1_eth1: ethernet-phy@1 {
 	};
 };
 
+&i2c2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c2_pins_b>;
+	pinctrl-1 = <&i2c2_sleep_pins_b>;
+	i2c-scl-rising-time-ns = <108>;
+	i2c-scl-falling-time-ns = <12>;
+	clock-frequency = <400000>;
+	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
+&i2c8 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c8_pins_a>;
+	pinctrl-1 = <&i2c8_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	clock-frequency = <100000>;
+	status = "disabled";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
 &scmi_regu {
 	scmi_vddio1: regulator@0 {
 		regulator-min-microvolt = <1800000>;
diff --git a/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi b/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
index c34cd33cd855f..2f853f3fae6d8 100644
--- a/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
@@ -212,6 +212,23 @@ pins {
 		};
 	};
 
+	i2c2_pins_b: i2c2-1 {
+		pins {
+			pinmux = <STM32_PINMUX('F', 2, AF9)>, /* I2C2_SCL */
+				 <STM32_PINMUX('F', 0, AF9)>; /* I2C2_SDA */
+			bias-disable;
+			drive-open-drain;
+			slew-rate = <0>;
+		};
+	};
+
+	i2c2_sleep_pins_b: i2c2-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('F', 2, ANALOG)>, /* I2C2_SCL */
+				 <STM32_PINMUX('F', 0, ANALOG)>; /* I2C2_SDA */
+		};
+	};
+
 	ospi_port1_clk_pins_a: ospi-port1-clk-0 {
 		pins {
 			pinmux = <STM32_PINMUX('D', 0, AF10)>; /* OSPI1_CLK */
diff --git a/arch/arm64/boot/dts/st/stm32mp251.dtsi b/arch/arm64/boot/dts/st/stm32mp251.dtsi
index 8b925ed0d8818..673fbc5632e69 100644
--- a/arch/arm64/boot/dts/st/stm32mp251.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp251.dtsi
@@ -773,7 +773,7 @@ i2c1: i2c@40120000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40120000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 21 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C1>;
 				resets = <&rcc I2C1_R>;
 				#address-cells = <1>;
@@ -784,6 +784,7 @@ i2c1: i2c@40120000 {
 				access-controllers = <&rifsc 41>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -791,7 +792,7 @@ i2c2: i2c@40130000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40130000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 22 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C2>;
 				resets = <&rcc I2C2_R>;
 				#address-cells = <1>;
@@ -802,6 +803,7 @@ i2c2: i2c@40130000 {
 				access-controllers = <&rifsc 42>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -809,7 +811,7 @@ i2c3: i2c@40140000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40140000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 23 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C3>;
 				resets = <&rcc I2C3_R>;
 				#address-cells = <1>;
@@ -820,6 +822,7 @@ i2c3: i2c@40140000 {
 				access-controllers = <&rifsc 43>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -827,7 +830,7 @@ i2c4: i2c@40150000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40150000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 24 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C4>;
 				resets = <&rcc I2C4_R>;
 				#address-cells = <1>;
@@ -838,6 +841,7 @@ i2c4: i2c@40150000 {
 				access-controllers = <&rifsc 44>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -845,7 +849,7 @@ i2c5: i2c@40160000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40160000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 25 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C5>;
 				resets = <&rcc I2C5_R>;
 				#address-cells = <1>;
@@ -856,6 +860,7 @@ i2c5: i2c@40160000 {
 				access-controllers = <&rifsc 45>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -863,7 +868,7 @@ i2c6: i2c@40170000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40170000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 49 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C6>;
 				resets = <&rcc I2C6_R>;
 				#address-cells = <1>;
@@ -874,6 +879,7 @@ i2c6: i2c@40170000 {
 				access-controllers = <&rifsc 46>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -881,7 +887,7 @@ i2c7: i2c@40180000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x40180000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti1 50 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C7>;
 				resets = <&rcc I2C7_R>;
 				#address-cells = <1>;
@@ -892,6 +898,7 @@ i2c7: i2c@40180000 {
 				access-controllers = <&rifsc 47>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
@@ -1473,7 +1480,7 @@ i2c8: i2c@46040000 {
 				compatible = "st,stm32mp25-i2c";
 				reg = <0x46040000 0x400>;
 				interrupt-names = "event";
-				interrupts = <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts-extended = <&exti2 25 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc CK_KER_I2C8>;
 				resets = <&rcc I2C8_R>;
 				#address-cells = <1>;
@@ -1484,6 +1491,7 @@ i2c8: i2c@46040000 {
 				access-controllers = <&rifsc 48>;
 				power-domains = <&CLUSTER_PD>;
 				i2c-analog-filter;
+				wakeup-source;
 				status = "disabled";
 			};
 
diff --git a/arch/arm64/boot/dts/st/stm32mp257f-dk.dts b/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
index 4135e7c0d9a35..220d977e9aa9f 100644
--- a/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
+++ b/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
@@ -124,6 +124,32 @@ phy1_eth1: ethernet-phy@1 {
 	};
 };
 
+&i2c2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c2_pins_b>;
+	pinctrl-1 = <&i2c2_sleep_pins_b>;
+	i2c-scl-rising-time-ns = <108>;
+	i2c-scl-falling-time-ns = <12>;
+	clock-frequency = <400000>;
+	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
+&i2c8 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c8_pins_a>;
+	pinctrl-1 = <&i2c8_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	clock-frequency = <100000>;
+	status = "disabled";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
 &scmi_regu {
 	scmi_vddio1: regulator@0 {
 		regulator-min-microvolt = <1800000>;
diff --git a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
index 852a73b0c516e..14e033f365e39 100644
--- a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
+++ b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
@@ -261,6 +261,9 @@ &i2c2 {
 	i2c-scl-falling-time-ns = <13>;
 	clock-frequency = <400000>;
 	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
 
 	imx335: camera@1a {
 		compatible = "sony,imx335";
@@ -299,6 +302,9 @@ &i2c8 {
 	i2c-scl-falling-time-ns = <7>;
 	clock-frequency = <400000>;
 	status = "disabled";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
 };
 
 &ommanager {
diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile
index ba01a929e06f2..260e211ca277d 100644
--- a/arch/arm64/boot/dts/ti/Makefile
+++ b/arch/arm64/boot/dts/ti/Makefile
@@ -142,17 +142,7 @@ dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-pcie1-ep.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-usb0-type-a.dtbo
 
 # Boards with J722s SoC
-k3-am67a-kontron-sa67-dtbs := k3-am67a-kontron-sa67-base.dtb \
-	k3-am67a-kontron-sa67-rtc-rv8263.dtbo k3-am67a-kontron-sa67-gbe1.dtbo
-k3-am67a-kontron-sa67-ads2-dtbs := k3-am67a-kontron-sa67.dtb k3-am67a-kontron-sa67-ads2.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am67a-beagley-ai.dtb
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67.dtb
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-base.dtb
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-gbe1.dtbo
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-gpios.dtbo
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-rtc-rv8263.dtbo
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-ads2.dtb
-dtb-$(CONFIG_ARCH_K3) += k3-am67a-kontron-sa67-ads2.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j722s-evm.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-j722s-evm-csi2-quad-rpi-cam-imx219.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j722s-evm-csi2-quad-tevi-ov5640.dtbo
@@ -244,8 +234,6 @@ k3-am642-tqma64xxl-mbax4xxl-sdcard-dtbs := \
 	k3-am642-tqma64xxl-mbax4xxl.dtb k3-am64-tqma64xxl-mbax4xxl-sdcard.dtbo
 k3-am642-tqma64xxl-mbax4xxl-wlan-dtbs := \
 	k3-am642-tqma64xxl-mbax4xxl.dtb k3-am64-tqma64xxl-mbax4xxl-wlan.dtbo
-k3-am67a-kontron-sa67-base-gpios-dtbs := \
-	k3-am67a-kontron-sa67-base.dtb k3-am67a-kontron-sa67-gpios.dtbo
 k3-am68-sk-base-board-csi2-dual-imx219-dtbs := k3-am68-sk-base-board.dtb \
 	k3-j721e-sk-csi2-dual-imx219.dtbo
 k3-am68-sk-base-board-pcie1-ep-dtbs := k3-am68-sk-base-board.dtb \
@@ -318,7 +306,6 @@ dtb- += k3-am625-beagleplay-csi2-ov5640.dtb \
 	k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtb \
 	k3-am642-tqma64xxl-mbax4xxl-sdcard.dtb \
 	k3-am642-tqma64xxl-mbax4xxl-wlan.dtb \
-	k3-am67a-kontron-sa67-base-gpios.dtb \
 	k3-am68-sk-base-board-csi2-dual-imx219.dtb \
 	k3-am68-sk-base-board-pcie1-ep.dtb \
 	k3-am69-sk-csi2-dual-imx219.dtb \
diff --git a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
index e15da771bc07d..3baa653257bb6 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
@@ -37,6 +37,13 @@ reserved_memory: reserved-memory {
 		#size-cells = <2>;
 		ranges;
 
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x00 0x8000000>;
+			linux,cma-default;
+		};
+
 		ramoops@9c700000 {
 			compatible = "ramoops";
 			reg = <0x00 0x9c700000 0x00 0x00100000>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
index e99bdbc2e0cbd..b1a6f10adf26b 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
@@ -398,7 +398,7 @@ AM62AX_IOPAD(0x01d4, PIN_INPUT, 7) /* (C15) UART0_RTSn.GPIO1_23 */
 
 	vddshv_sdio_pins_default: vddshv-sdio-default-pins {
 		pinctrl-single,pins = <
-			AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (M19) GPMC0_CLK.GPIO0_31 */
+			AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (N22) GPMC0_CLK.GPIO0_31 */
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts b/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts
index a5d5dc0a7bec8..1ee851a646c05 100644
--- a/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62d2-evm.dts
@@ -199,6 +199,42 @@ led-0 {
 &mcu_pmx0 {
 	status = "okay";
 
+	mcu_mcan0_tx_pins_default: mcu-mcan0-tx-default-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x0034, PIN_OUTPUT, 0) /* (C7) MCU_MCAN0_TX */
+		>;
+	};
+
+	mcu_mcan0_rx_pins_default: mcu-mcan0-rx-default-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x0038, PIN_INPUT, 0) /* (E8) MCU_MCAN0_RX */
+		>;
+	};
+
+	mcu_mcan0_rx_pins_wakeup: mcu-mcan0-rx-wakeup-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x0038, PIN_INPUT | PIN_WKUP_EN, 0) /* (E8) MCU_MCAN0_RX */
+		>;
+	};
+
+	mcu_mcan1_tx_pins_default: mcu-mcan1-tx-default-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x003c, PIN_OUTPUT, 0) /* (D7) MCU_MCAN1_TX */
+		>;
+	};
+
+	mcu_mcan1_rx_pins_default: mcu-mcan1-rx-default-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x0040, PIN_INPUT, 0) /* (B9) MCU_MCAN1_RX */
+		>;
+	};
+
+	mcu_mcan1_rx_pins_wakeup: mcu-mcan1-rx-wakeup-pins {
+		pinctrl-single,pins = <
+			AM62DX_MCU_IOPAD(0x040, PIN_INPUT | PIN_WKUP_EN, 0) /* (B9) MCU_MCAN1_RX */
+		>;
+	};
+
 	pmic_irq_pins_default: pmic-irq-default-pins {
 		pinctrl-single,pins = <
 			AM62DX_IOPAD(0x01f4, PIN_INPUT, 7) /* (F17) EXTINTn.GPIO1_31 */
@@ -228,6 +264,10 @@ AM62DX_MCU_IOPAD(0x0050, PIN_INPUT, 0) /* (E13) WKUP_I2C0_SDA */
 &wkup_uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&wkup_uart0_pins_default>;
+	wakeup-source = <&system_io_ddr>,
+			<&system_deep_sleep>,
+			<&system_mcu_only>,
+			<&system_standby>;
 	bootph-all;
 	status = "reserved";
 };
@@ -737,4 +777,33 @@ &c7x_0 {
 	firmware-name = "am62d-c71_0-fw";
 };
 
+&mcu_mcan0 {
+	pinctrl-names = "default", "wakeup";
+	pinctrl-0 = <&mcu_mcan0_tx_pins_default>, <&mcu_mcan0_rx_pins_default>;
+	pinctrl-1 = <&mcu_mcan0_tx_pins_default>, <&mcu_mcan0_rx_pins_wakeup>;
+	wakeup-source = <&system_partial_io>,
+			<&system_io_ddr>,
+			<&system_deep_sleep>,
+			<&system_mcu_only>,
+			<&system_standby>;
+};
+
+&mcu_mcan1 {
+	pinctrl-names = "default", "wakeup";
+	pinctrl-0 = <&mcu_mcan1_tx_pins_default>, <&mcu_mcan1_rx_pins_default>;
+	pinctrl-1 = <&mcu_mcan1_tx_pins_default>, <&mcu_mcan1_rx_pins_wakeup>;
+	wakeup-source = <&system_partial_io>,
+			<&system_io_ddr>,
+			<&system_deep_sleep>,
+			<&system_mcu_only>,
+			<&system_standby>;
+};
+
+&mcu_uart0 {
+	wakeup-source = <&system_io_ddr>,
+			<&system_deep_sleep>,
+			<&system_mcu_only>,
+			<&system_standby>;
+};
+
 #include "k3-am62a-ti-ipc-firmware.dtsi"
diff --git a/arch/arm64/boot/dts/ti/k3-am62l-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62l-main.dtsi
index 883beb76ba9c4..80615ca1e01a2 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62l-main.dtsi
@@ -564,6 +564,13 @@ gpmc0: memory-controller@3b000000 {
 		status = "disabled";
 	};
 
+	rng: rng@3b100000 {
+		compatible = "inside-secure,safexcel-eip76";
+		reg = <0x00 0x3b100000 0x00 0x7d>;
+		interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+		status = "reserved"; /* Reserved for OP-TEE */
+	};
+
 	oc_sram: sram@70800000 {
 		compatible = "mmio-sram";
 		reg = <0x00 0x70800000 0x00 0x10000>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62l-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-am62l-wakeup.dtsi
index 61bfcdcfc66ea..b5b70e71a0b79 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l-wakeup.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62l-wakeup.dtsi
@@ -81,7 +81,7 @@ wkup_i2c0: i2c@2b200000 {
 		status = "disabled";
 	};
 
-	target-module@2b300050 {
+	wkup_uart0_target: target-module@2b300050 {
 		compatible = "ti,sysc-omap2", "ti,sysc";
 		reg = <0x00 0x2b300050 0x00 0x4>,
 		      <0x00 0x2b300054 0x00 0x4>,
@@ -127,6 +127,11 @@ chipid: chipid@14 {
 			bootph-all;
 		};
 
+		opp_efuse_table: syscon@18 {
+			compatible = "ti,am62-opp-efuse-table", "syscon";
+			reg = <0x18 0x4>;
+		};
+
 		cpsw_mac_syscon: ethernet-mac-syscon@2000 {
 			compatible = "ti,am62p-cpsw-mac-efuse", "syscon";
 			reg = <0x2000 0x8>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62l.dtsi b/arch/arm64/boot/dts/ti/k3-am62l.dtsi
index 23acdbb301fe3..28bb6ef2194f1 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62l.dtsi
@@ -79,6 +79,7 @@ cbass_main: bus@f0000 {
 			 <0x00 0x31000000 0x00 0x31000000 0x00 0x00050000>, /* USB0 DWC3 Core Window */
 			 <0x00 0x31100000 0x00 0x31100000 0x00 0x00050000>, /* USB1 DWC3 Core Window */
 			 <0x00 0x3b000000 0x00 0x3b000000 0x00 0x00000400>, /* GPMC0 */
+			 <0x00 0x3b100000 0x00 0x3b100000 0x00 0x0000007d>, /* RNG */
 			 <0x00 0x45810000 0x00 0x45810000 0x00 0x03170000>, /* DMSS */
 			 <0x00 0x50000000 0x00 0x50000000 0x00 0x08000000>, /* GPMC DATA */
 			 <0x00 0x60000000 0x00 0x60000000 0x00 0x08000000>, /* FSS DAT1 */
@@ -92,7 +93,7 @@ cbass_main: bus@f0000 {
 			 <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00001400>, /* VTM */
 			 <0x00 0x04080000 0x00 0x04080000 0x00 0x00008000>, /* PDCFG */
 			 <0x00 0x04201000 0x00 0x04201000 0x00 0x00000100>, /* GPIO */
-			 <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00100100>, /* Wakeup Peripheral Window */
+			 <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00200200>, /* Wakeup Peripheral Window */
 			 <0x00 0x40800000 0x00 0x40800000 0x00 0x00014000>, /* DMA */
 			 <0x00 0x43000000 0x00 0x43000000 0x00 0x00080000>; /* CTRL MMRs */
 		#address-cells = <2>;
@@ -104,7 +105,7 @@ cbass_wakeup: bus@a80000 {
 				 <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00001400>, /* VTM */
 				 <0x00 0x04080000 0x00 0x04080000 0x00 0x00008000>, /* PDCFG */
 				 <0x00 0x04201000 0x00 0x04201000 0x00 0x00000100>, /* GPIO */
-				 <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00100100>, /* Wakeup Peripheral Window */
+				 <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00200200>, /* Wakeup Peripheral Window */
 				 <0x00 0x40800000 0x00 0x40800000 0x00 0x00014000>, /* DMA */
 				 <0x00 0x43000000 0x00 0x43000000 0x00 0x00080000>; /* CTRL MMRs */
 			#address-cells = <2>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
index cae04cce33736..8ac7ad2cd1b96 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
@@ -306,6 +306,12 @@ AM62LX_IOPAD(0x0238, PIN_OUTPUT, 7) /* (D24) MMC1_SDWP.GPIO0_123 */
 		>;
 	};
 
+	wkup_uart0_pins_default: wkup-uart0-default-pins {
+		pinctrl-single,pins = <
+			AM62LX_IOPAD(0x0000, PIN_INPUT, 0) /* (Y22) WKUP_UART0_RXD */
+			AM62LX_IOPAD(0x0004, PIN_OUTPUT, 0) /* (AA23) WKUP_UART0_TXD */
+		>;
+	};
 };
 
 &sdhci0 {
@@ -359,3 +365,12 @@ &usb1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&usb1_default_pins>;
 };
+
+&wkup_uart0 {
+	pinctrl-0 = <&wkup_uart0_pins_default>;
+	pinctrl-names = "default";
+};
+
+&wkup_uart0_target {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62l3.dtsi b/arch/arm64/boot/dts/ti/k3-am62l3.dtsi
index da220b8515122..fc514e19f57d5 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l3.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62l3.dtsi
@@ -39,6 +39,8 @@ cpu0: cpu@0 {
 			d-cache-line-size = <64>;
 			d-cache-sets = <128>;
 			next-level-cache = <&l2_0>;
+			operating-points-v2 = <&a53_opp_table>;
+			clocks = <&scmi_clk 356>;
 		};
 
 		cpu1: cpu@1 {
@@ -53,6 +55,8 @@ cpu1: cpu@1 {
 			d-cache-line-size = <64>;
 			d-cache-sets = <128>;
 			next-level-cache = <&l2_0>;
+			operating-points-v2 = <&a53_opp_table>;
+			clocks = <&scmi_clk 357>;
 		};
 	};
 
@@ -64,4 +68,47 @@ l2_0: l2-cache0 {
 		cache-line-size = <64>;
 		cache-sets = <256>;
 	};
+
+	a53_opp_table: opp-table {
+		compatible = "operating-points-v2-ti-cpu";
+		opp-shared;
+		syscon = <&opp_efuse_table>;
+
+		opp-200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-supported-hw = <0x01 0x0003>;
+			clock-latency-ns = <6000000>;
+		};
+
+		opp-400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-supported-hw = <0x01 0x0003>;
+			clock-latency-ns = <6000000>;
+		};
+
+		opp-600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-supported-hw = <0x01 0x0003>;
+			clock-latency-ns = <6000000>;
+		};
+
+		opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-supported-hw = <0x01 0x0003>;
+			clock-latency-ns = <6000000>;
+		};
+
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-supported-hw = <0x01 0x0002>;
+			clock-latency-ns = <6000000>;
+		};
+
+		opp-1250000000 {
+			opp-hz = /bits/ 64 <1250000000>;
+			opp-supported-hw = <0x01 0x0002>;
+			clock-latency-ns = <6000000>;
+			opp-suspend;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/ti/k3-am642-evm.dts b/arch/arm64/boot/dts/ti/k3-am642-evm.dts
index 88093ab74502c..bc4347cf71140 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-am642-evm.dts
@@ -601,6 +601,7 @@ cpsw3g_phy0: ethernet-phy@0 {
 		reg = <0>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am642-sk.dts b/arch/arm64/boot/dts/ti/k3-am642-sk.dts
index 34bfa99bd4b88..d28a38c87f32a 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am642-sk.dts
@@ -519,12 +519,14 @@ cpsw3g_phy0: ethernet-phy@0 {
 		reg = <0>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 
 	cpsw3g_phy1: ethernet-phy@1 {
 		reg = <1>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
index e0262c2743eb2..ca1300e130936 100644
--- a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
@@ -583,6 +583,7 @@ phy0: ethernet-phy@0 {
 		reg = <0>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-ads2.dtso b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-ads2.dtso
deleted file mode 100644
index ae5e2b52594bb..0000000000000
--- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-ads2.dtso
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only OR MIT
-/*
- * Kontron SMARC-sa67 board on the Kontron Eval Carrier 2.2.
- *
- * Copyright (c) 2025 Kontron Europe GmbH
- */
-
-/dts-v1/;
-/plugin/;
-
-#include <dt-bindings/interrupt-controller/irq.h>
-#include "k3-pinctrl.h"
-
-&{/} {
-	pwm-fan {
-		compatible = "pwm-fan";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pwm_fan_pins_default>;
-		interrupts-extended = <&main_gpio1 7 IRQ_TYPE_EDGE_FALLING>;
-		#cooling-cells = <2>;
-		pwms = <&epwm2 1 4000000 0>;
-		cooling-levels = <1 128 192 255>;
-	};
-
-	sound {
-		compatible = "simple-audio-card";
-		simple-audio-card,widgets =
-			"Headphone", "Headphone Jack",
-			"Line", "Line Out Jack",
-			"Microphone", "Microphone Jack",
-			"Line", "Line In Jack";
-		simple-audio-card,routing =
-			"Line Out Jack", "LINEOUTR",
-			"Line Out Jack", "LINEOUTL",
-			"Headphone Jack", "HPOUTR",
-			"Headphone Jack", "HPOUTL",
-			"IN1L", "Line In Jack",
-			"IN1R", "Line In Jack",
-			"Microphone Jack", "MICBIAS",
-			"IN2L", "Microphone Jack",
-			"IN2R", "Microphone Jack";
-		simple-audio-card,mclk-fs = <256>;
-		simple-audio-card,format = "i2s";
-		simple-audio-card,bitclock-master = <&dailink0_master>;
-		simple-audio-card,frame-master = <&dailink0_master>;
-
-		simple-audio-card,cpu {
-			sound-dai = <&mcasp0>;
-		};
-
-		dailink0_master: simple-audio-card,codec {
-			sound-dai = <&wm8904>;
-			clocks = <&audio_refclk0>;
-		};
-	};
-
-	cvcc_1p8v_i2s: regulator-carrier-0 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_1V8_S0_I2S";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-	};
-
-	cvcc_1p8v_s0: regulator-carrier-1 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_1V8_S0";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-	};
-
-	cvcc_3p3v_s0: regulator-carrier-2 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_3V3_S0";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-	};
-};
-
-&audio_refclk0 {
-	status = "okay";
-};
-
-&epwm2 {
-	status = "okay";
-};
-
-&main_pmx0 {
-	pwm_fan_pins_default: pwm-fan-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1ec, PIN_OUTPUT, 8)	/* (A22) I2C1_SDA.EHRPWM2_B */
-			J722S_IOPAD(0x194, PIN_INPUT, 0)	/* (A25) MCASP0_AXR3.GPIO1_7 */
-		>;
-	};
-};
-
-&mcasp0 {
-	#sound-dai-cells = <0>;
-	status = "okay";
-};
-
-&mcu_i2c0 {
-	status = "okay";
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	wm8904: audio-codec@1a {
-		#sound-dai-cells = <0>;
-		compatible = "wlf,wm8904";
-		reg = <0x1a>;
-		clocks = <&audio_refclk0>;
-		clock-names = "mclk";
-		AVDD-supply = <&cvcc_1p8v_i2s>;
-		CPVDD-supply = <&cvcc_1p8v_i2s>;
-		DBVDD-supply = <&cvcc_1p8v_i2s>;
-		DCVDD-supply = <&cvcc_1p8v_i2s>;
-		MICVDD-supply = <&cvcc_1p8v_i2s>;
-	};
-};
-
-&mcu_spi0 {
-	status = "okay";
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	flash@0 {
-		compatible = "jedec,spi-nor";
-		reg = <0>;
-		spi-max-frequency = <104000000>;
-		m25p,fast-read;
-		vcc-supply = <&cvcc_1p8v_s0>;
-	};
-};
-
-&wkup_i2c0 {
-	status = "okay";
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	/* SMARC Carrier EEPROM */
-	eeprom@57 {
-		compatible = "atmel,24c32";
-		reg = <0x57>;
-		pagesize = <32>;
-		vcc-supply = <&cvcc_3p3v_s0>;
-	};
-};
diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts
deleted file mode 100644
index 95234c8460ed0..0000000000000
--- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts
+++ /dev/null
@@ -1,1091 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only OR MIT
-/*
- * Kontron SMARC-sAM67 module
- *
- * Copyright (c) 2025 Kontron Europe GmbH
- */
-
-/dts-v1/;
-
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/interrupt-controller/irq.h>
-#include <dt-bindings/phy/phy.h>
-#include "k3-j722s.dtsi"
-#include "k3-serdes.h"
-
-/ {
-	compatible = "kontron,sa67", "ti,j722s";
-	model = "Kontron SMARC-sAM67";
-
-	aliases {
-		serial0 = &mcu_uart0;
-		serial1 = &main_uart0;
-		serial2 = &main_uart5;
-		serial3 = &wkup_uart0;
-		mmc0 = &sdhci0;
-		mmc1 = &sdhci1;
-		rtc0 = &wkup_rtc0;
-	};
-
-	lcd0_backlight: backlight-1 {
-		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&lcd0_backlight_pins_default>;
-		pwms = <&epwm1 0 50000 0>;
-		brightness-levels = <0 32 64 96 128 160 192 224 255>;
-		default-brightness-level = <8>;
-		enable-gpios = <&main_gpio0 29 GPIO_ACTIVE_HIGH>;
-		status = "disabled";
-	};
-
-	lcd1_backlight: backlight-2 {
-		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&lcd1_backlight_pins_default>;
-		pwms = <&epwm1 1 50000 0>;
-		brightness-levels = <0 32 64 96 128 160 192 224 255>;
-		default-brightness-level = <8>;
-		enable-gpios = <&main_gpio1 18 GPIO_ACTIVE_HIGH>;
-		status = "disabled";
-	};
-
-	chosen {
-		stdout-path = "serial1:115200n8";
-	};
-
-	connector-1 {
-		compatible = "gpio-usb-b-connector", "usb-b-connector";
-		pinctrl-names = "default";
-		pinctrl-0 = <&usb0_connector_pins_default>;
-		type = "micro";
-		id-gpios = <&main_gpio0 34 GPIO_ACTIVE_HIGH>;
-		vbus-supply = <&vcc_usb0_vbus>;
-
-		port  {
-			usb0_connector: endpoint {
-				remote-endpoint = <&usb0_hc>;
-			};
-		};
-
-	};
-
-	memory@80000000 {
-		/* Filled in by bootloader */
-		reg = <0x00000000 0x00000000 0x00000000 0x00000000>,
-		      <0x00000000 0x00000000 0x00000000 0x00000000>;
-		device_type = "memory";
-		bootph-pre-ram;
-	};
-
-	reserved_memory: reserved-memory {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
-		linux,cma {
-			compatible = "shared-dma-pool";
-			reusable;
-			size = <0x00 0x10000000>;
-			linux,cma-default;
-		};
-
-		secure_tfa_ddr: tfa@9e780000 {
-			reg = <0x00 0x9e780000 0x00 0x80000>;
-			no-map;
-		};
-
-		secure_ddr: optee@9e800000 {
-			reg = <0x00 0x9e800000 0x00 0x01800000>;
-			no-map;
-		};
-
-		wkup_r5fss0_core0_memory_region: r5f-memory@a0100000 {
-			compatible = "shared-dma-pool";
-			reg = <0x00 0xa0100000 0x00 0xf00000>;
-			no-map;
-		};
-	};
-
-	vin_5p0: regulator-1 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_3V0_5V25_IN";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-always-on;
-		regulator-boot-on;
-		bootph-all;
-	};
-
-	vcc_3p3_s5: regulator-2 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_3V3_S5";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vin_5p0>;
-		regulator-always-on;
-		regulator-boot-on;
-		bootph-all;
-	};
-
-	vcc_1p8_s5: regulator-3 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_1V8_S5";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		vin-supply = <&vin_5p0>;
-		regulator-always-on;
-		regulator-boot-on;
-		bootph-all;
-	};
-
-	vcc_3p3_s0: regulator-4 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_3V3_S0";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_3p3_s5>;
-		regulator-always-on;
-		regulator-boot-on;
-		enable-active-high;
-		gpios = <&tps652g1 1 GPIO_ACTIVE_HIGH>;
-		bootph-all;
-	};
-
-	vcc_3p3_sd_s0: regulator-5 {
-		compatible = "regulator-fixed";
-		regulator-name = "SDIO_PWR_EN";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_3p3_sd_s0_pins_default>;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-boot-on;
-		enable-active-high;
-		gpios = <&main_gpio0 7 GPIO_ACTIVE_HIGH>;
-		bootph-all;
-	};
-
-	vcc_3p3_sd_vio_s0: regulator-6 {
-		compatible = "regulator-gpio";
-		regulator-name = "V_3V3_1V8_SD_S0";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_3p3_sd_vio_s0_pins_default>;
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_3p3_s0>;
-		regulator-boot-on;
-		enable-active-high;
-		enable-gpios = <&main_gpio0 7 GPIO_ACTIVE_HIGH>;
-		gpios = <&main_gpio0 8 GPIO_ACTIVE_HIGH>;
-		states = <3300000 0x0>,
-			 <1800000 0x1>;
-		bootph-all;
-	};
-
-	vcc_3p3_cam_s0: regulator-7 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_3V3_CAM_S0";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_3p3_cam_s0_pins_default>;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_3p3_s5>;
-		enable-active-high;
-		interrupts-extended = <&main_gpio1 30 IRQ_TYPE_EDGE_FALLING>;
-		bootph-all;
-	};
-
-	vcc_1p1_s0: regulator-8 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_1V1_S0";
-		regulator-min-microvolt = <1100000>;
-		regulator-max-microvolt = <1100000>;
-		vin-supply = <&vcc_1p1_s3>;
-		regulator-always-on;
-		regulator-boot-on;
-		enable-active-high;
-		/* shared with V_0V75_0V85_CORE_S0 */
-		gpios = <&tps652g1 4 GPIO_ACTIVE_HIGH>;
-		bootph-all;
-	};
-
-	vcc_0p85_vcore_s0: regulator-9 {
-		compatible = "regulator-fixed";
-		regulator-name = "V_0V75_0V85_CORE_S0";
-		regulator-min-microvolt = <850000>;
-		regulator-max-microvolt = <850000>;
-		vin-supply = <&vin_5p0>;
-		regulator-always-on;
-		regulator-boot-on;
-		enable-active-high;
-		gpios = <&tps652g1 4 GPIO_ACTIVE_HIGH>;
-		bootph-all;
-	};
-
-	vcc_lcd0_panel: regulator-10 {
-		compatible = "regulator-fixed";
-		regulator-name = "LCD0_VDD_EN";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_lcd0_panel_pins_default>;
-		enable-active-high;
-		gpios = <&main_gpio0 30 GPIO_ACTIVE_HIGH>;
-	};
-
-	vcc_lcd1_panel: regulator-11 {
-		compatible = "regulator-fixed";
-		regulator-name = "LCD1_VDD_EN";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_lcd1_panel_pins_default>;
-		enable-active-high;
-		gpios = <&main_gpio1 19 GPIO_ACTIVE_HIGH>;
-	};
-
-	vcc_usb0_vbus: regulator-12 {
-		compatible = "regulator-fixed";
-		regulator-name = "USB0_EN_OC#";
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc_usb0_vbus_pins_default>;
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		enable-active-high;
-		gpios = <&main_gpio1 50 GPIO_ACTIVE_HIGH>;
-	};
-};
-
-&audio_refclk0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&audio_refclk0_pins_default>;
-	status = "disabled";
-};
-
-&audio_refclk1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&audio_refclk1_pins_default>;
-	status = "disabled";
-};
-
-&cpsw3g {
-	pinctrl-names = "default";
-	pinctrl-0 = <&cpsw3g_pins_default>, <&rgmii1_pins_default>,
-		    <&rgmii2_pins_default>;
-	status = "okay";
-};
-
-&cpsw3g_mdio {
-	pinctrl-names = "default";
-	pinctrl-0 = <&cpsw3g_mdio_pins_default>;
-	status = "okay";
-
-	phy0: ethernet-phy@0 {
-		reg = <0>;
-	};
-
-	phy1: ethernet-phy@1 {
-		reg = <1>;
-	};
-};
-
-&cpsw_port1 {
-	phy-connection-type = "rgmii-id";
-	phy-handle = <&phy0>;
-	nvmem-cells = <&base_mac_address 0>;
-	nvmem-cell-names = "mac-address";
-	status = "okay";
-};
-
-&main_gpio0 {
-	gpio-line-names =
-		"", "", "", "", "", "", "", "SOC_SDIO_PWR_EN", "VSD_SEL",
-		"RESET_OUT#", "I2C_MUX_RST#", "SPI_FLASH_CS#", "QPSI_CS0#",
-		"QSPI_CS1#", "BOOT_SEL1", "BRDCFG0", "BRDCFG1", "BRDCFG2",
-		"BRDCFG3", "BRDCFG4", "", "BRDREV0", "BRDREV1", "", "", "", "",
-		"", "", "LCD0_BKLT_EN", "LCD0_VDD_EN", "GBE_INT#", "DSI0_TE",
-		"CHARGING#", "USB0_OTG_ID", "PMIC_INT#", "RTC_INT#",
-		"EDP_BRIDGE_EN", "EDP_BRIDGE_IRQ#", "", "CHARGER_PRSNT#", "",
-		"", "", "", "BOOT_SEL2#", "CAM2_RST#", "CAM2_PWR#", "",
-		"CAM3_RST#", "CAM3_PWR#", "GPIO0", "GPIO1", "", "", "", "", "",
-		"", "", "", "", "", "", "", "", "", "", "", "GPIO10", "GPIO11",
-		"SLEEP#", "LID#";
-
-	bootph-all;
-	status = "okay";
-};
-
-&main_gpio1 {
-	gpio-line-names =
-		"", "", "", "", "", "", "", "GPIO6", "GPIO7", "", "", "", "",
-		"GPIO8", "GPIO9", "PCIE_A_RST#", "", "BATLOW#", "LCD1_BKLT_EN",
-		"LCD1_VDD_EN", "", "", "", "", "GPIO2", "GPIO3", "", "",
-		"GPIO4", "GPIO5", "CAM_S0_FAULT#", "BOOT_SEL0#", "", "", "", "",
-		"", "", "", "", "", "", "", "", "", "", "", "", "SDIO_CD#", "",
-		"USB0_DRVVBUS", "USB1_DRVVBUS";
-
-	bootph-all;
-	status = "okay";
-};
-
-/* I2C_LOCAL */
-&main_i2c0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_i2c0_pins_default>;
-	clock-frequency = <100000>;
-	bootph-all;
-	status = "okay";
-
-	tps652g1: pmic@44 {
-		compatible = "ti,tps652g1";
-		reg = <0x44>;
-		ti,primary-pmic;
-		system-power-controller;
-
-		gpio-controller;
-		#gpio-cells = <2>;
-		gpio-line-names =
-			"LPM_EN#", "EN_3V3_S0", "POWER_BTN#", "CARRIER_STBY#",
-			"EN_0V75_0V85_VCORE_S0", "PMIC_WAKEUP";
-
-		pinctrl-names = "default";
-		pinctrl-0 = <&pmic_irq_pins_default>;
-		interrupts-extended = <&main_gpio0 35 IRQ_TYPE_EDGE_FALLING>;
-
-		buck1-supply = <&vin_5p0>;
-		buck2-supply = <&vin_5p0>;
-		buck3-supply = <&vin_5p0>;
-		buck4-supply = <&vin_5p0>;
-		ldo1-supply = <&vin_5p0>;
-		ldo2-supply = <&vin_5p0>;
-		ldo3-supply = <&vin_5p0>;
-
-		bootph-all;
-
-		regulators {
-			vcc_0p85_s0: buck1 {
-				regulator-name = "V_0V85_S0";
-				regulator-min-microvolt = <850000>;
-				regulator-max-microvolt = <850000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p1_s3: buck2 {
-				regulator-name = "V_1V1_S3";
-				regulator-min-microvolt = <1100000>;
-				regulator-max-microvolt = <1100000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p8_s0: buck3 {
-				regulator-name = "V_1V8_S0";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p2_s0: buck4 {
-				regulator-name = "V_1V2_S0";
-				regulator-min-microvolt = <1200000>;
-				regulator-max-microvolt = <1200000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p8_vda_pll_s0: ldo1 {
-				regulator-name = "V_1V8_VDA_PLL_S0";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p8_s3: ldo2 {
-				regulator-name = "V_1V8_S3";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			vcc_1p8_ret_s5: ldo3 {
-				regulator-name = "V_1V8_RET_S5";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-		};
-	};
-
-	system-controller@4a {
-		compatible = "kontron,sa67mcu", "kontron,sl28cpld";
-		reg = <0x4a>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		watchdog@4 {
-			compatible = "kontron,sa67mcu-wdt", "kontron,sl28cpld-wdt";
-			reg = <0x4>;
-			kontron,assert-wdt-timeout-pin;
-		};
-
-		hwmon@8 {
-			compatible = "kontron,sa67mcu-hwmon";
-			reg = <0x8>;
-		};
-	};
-};
-
-/* I2C_CAM */
-&main_i2c2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_i2c2_pins_default>;
-	clock-frequency = <100000>;
-	status = "okay";
-
-	i2c-mux@70 {
-		compatible = "nxp,pca9546";
-		reg = <0x70>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		pinctrl-names = "default";
-		pinctrl-0 = <&i2c_mux_pins_default>;
-
-		vdd-supply = <&vcc_1p8_s0>;
-		reset-gpios = <&main_gpio0 10 GPIO_ACTIVE_LOW>;
-
-		i2c_cam0: i2c@0 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			reg = <0>;
-		};
-
-		i2c_cam1: i2c@1 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			reg = <1>;
-		};
-
-		i2c_cam2: i2c@2 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			reg = <2>;
-		};
-
-		i2c_cam3: i2c@3 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			reg = <3>;
-		};
-	};
-};
-
-/* I2C_LCD */
-&main_i2c3 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_i2c3_pins_default>;
-	clock-frequency = <100000>;
-	status = "okay";
-};
-
-&main_pmx0 {
-	audio_refclk0_pins_default: audio-refclk0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0c4, PIN_OUTPUT, 5)	/* (W23) VOUT0_DATA3.AUDIO_EXT_REFCLK0 */
-		>;
-	};
-
-	audio_refclk1_pins_default: audio-refclk1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0a0, PIN_OUTPUT, 1)	/* (N24) GPMC0_WPn.AUDIO_EXT_REFCLK1 */
-		>;
-	};
-
-	cpsw3g_mdio_pins_default: cpsw3g-mdio-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x160, PIN_OUTPUT, 0)	/* (AC24) MDIO0_MDC */
-			J722S_IOPAD(0x15c, PIN_INPUT, 0)	/* (AD25) MDIO0_MDIO */
-		>;
-	};
-
-	cpsw3g_pins_default: cpsw3g-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1b8, PIN_OUTPUT, 1)	/* (C20) SPI0_CS1.CP_GEMAC_CPTS0_TS_COMP */
-		>;
-	};
-
-	edp_bridge_pins_default: edp-bridge-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x098, PIN_OUTPUT, 7)	/* (V21) GPMC0_WAIT0.GPIO0_37 */
-			J722S_IOPAD(0x09c, PIN_INPUT, 7)	/* (W26) GPMC0_WAIT1.GPIO0_38 */
-		>;
-	};
-
-	i2c_mux_pins_default: i2c-mux-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x028, PIN_OUTPUT, 7)	/* (M27) OSPI0_D7.GPIO0_10 */
-		>;
-	};
-
-	lcd0_backlight_pins_default: lcd0-backlight-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x074, PIN_OUTPUT, 7)	/* (V22) GPMC0_AD14.GPIO0_29 */
-			J722S_IOPAD(0x110, PIN_OUTPUT, 4)	/* (G27) MMC2_DAT1.EHRPWM1_A */
-		>;
-	};
-
-	lcd1_backlight_pins_default: lcd1-backlight-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1c0, PIN_OUTPUT, 7)	/* (E19) SPI0_D0.GPIO1_18 */
-			J722S_IOPAD(0x114, PIN_OUTPUT, 4)	/* (G26) MMC2_DAT0.EHRPWM1_B */
-		>;
-	};
-
-	main_i2c0_pins_default: main-i2c0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1e0, PIN_INPUT, 0)	/* (D23) I2C0_SCL */
-			J722S_IOPAD(0x1e4, PIN_INPUT, 0)	/* (B22) I2C0_SDA */
-		>;
-		bootph-all;
-	};
-
-	main_i2c2_pins_default: main-i2c2-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0b0, PIN_INPUT, 1)	/* (P22) GPMC0_CSn2.I2C2_SCL */
-			J722S_IOPAD(0x0b4, PIN_INPUT, 1)	/* (P23) GPMC0_CSn3.I2C2_SDA */
-		>;
-	};
-
-	main_i2c3_pins_default: main-i2c3-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1d0, PIN_INPUT, 2)	/* (E22) UART0_CTSn.I2C3_SCL */
-			J722S_IOPAD(0x1d4, PIN_INPUT, 2)	/* (B21) UART0_RTSn.I2C3_SDA */
-		>;
-	};
-
-	main_i2c4_pins_default: main-i2c4-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0a8, PIN_INPUT, 1)	/* (R27) GPMC0_CSn0.I2C4_SCL */
-			J722S_IOPAD(0x0ac, PIN_INPUT, 1)	/* (P21) GPMC0_CSn1.I2C4_SDA */
-		>;
-	};
-
-	main_uart0_pins_default: main-uart0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1c8, PIN_INPUT, 0)	/* (F19) UART0_RXD */
-			J722S_IOPAD(0x1cc, PIN_OUTPUT, 0)	/* (F20) UART0_TXD */
-		>;
-		bootph-all;
-	};
-
-	main_uart5_pins_default: main-uart5-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x108, PIN_INPUT, 3)	/* (J27) MMC2_DAT3.UART5_RXD */
-			J722S_IOPAD(0x10c, PIN_OUTPUT, 3)	/* (H27) MMC2_DAT2.UART5_TXD */
-			J722S_IOPAD(0x008, PIN_INPUT, 5)	/* (L22) OSPI0_DQS.UART5_CTSn */
-			J722S_IOPAD(0x004, PIN_OUTPUT, 5)	/* (L23) OSPI0_LBCLKO.UART5_RTSn */
-		>;
-	};
-
-	mcasp0_pins_default: mcasp0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1a4, PIN_INPUT, 0)	/* (D25) MCASP0_ACLKX */
-			J722S_IOPAD(0x1a8, PIN_INPUT, 0)	/* (C26) MCASP0_AFSX */
-			J722S_IOPAD(0x1a0, PIN_INPUT, 0)	/* (F23) MCASP0_AXR0 */
-			J722S_IOPAD(0x19c, PIN_OUTPUT, 0)	/* (B25) MCASP0_AXR1 */
-		>;
-	};
-
-	mcasp2_pins_default: mcasp2-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x070, PIN_INPUT, 3)	/* (V24) GPMC0_AD13.MCASP2_ACLKX */
-			J722S_IOPAD(0x06c, PIN_INPUT, 3)	/* (V26) GPMC0_AD12.MCASP2_AFSX */
-			J722S_IOPAD(0x05c, PIN_INPUT, 3)	/* (U27) GPMC0_AD8.MCASP2_AXR0 */
-			J722S_IOPAD(0x060, PIN_OUTPUT, 3)	/* (U26) GPMC0_AD9.MCASP2_AXR1 */
-		>;
-	};
-
-	oldi0_pins_default: oldi0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x260, PIN_OUTPUT, 0)	/* (AF23) OLDI0_A0N */
-			J722S_IOPAD(0x25c, PIN_OUTPUT, 0)	/* (AG24) OLDI0_A0P */
-			J722S_IOPAD(0x268, PIN_OUTPUT, 0)	/* (AG22) OLDI0_A1N */
-			J722S_IOPAD(0x264, PIN_OUTPUT, 0)	/* (AG23) OLDI0_A1P */
-			J722S_IOPAD(0x270, PIN_OUTPUT, 0)	/* (AB20) OLDI0_A2N */
-			J722S_IOPAD(0x26c, PIN_OUTPUT, 0)	/* (AB21) OLDI0_A2P */
-			J722S_IOPAD(0x278, PIN_OUTPUT, 0)	/* (AG20) OLDI0_A3N */
-			J722S_IOPAD(0x274, PIN_OUTPUT, 0)	/* (AG21) OLDI0_A3P */
-			J722S_IOPAD(0x2a0, PIN_OUTPUT, 0)	/* (AF21) OLDI0_CLK0N */
-			J722S_IOPAD(0x29c, PIN_OUTPUT, 0)	/* (AE20) OLDI0_CLK0P */
-		>;
-	};
-
-	oldi1_pins_default: oldi1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x280, PIN_OUTPUT, 0)	/* (AD21) OLDI0_A4N */
-			J722S_IOPAD(0x27c, PIN_OUTPUT, 0)	/* (AC21) OLDI0_A4P */
-			J722S_IOPAD(0x288, PIN_OUTPUT, 0)	/* (AF19) OLDI0_A5N */
-			J722S_IOPAD(0x284, PIN_OUTPUT, 0)	/* (AF18) OLDI0_A5P */
-			J722S_IOPAD(0x290, PIN_OUTPUT, 0)	/* (AG17) OLDI0_A6N */
-			J722S_IOPAD(0x28c, PIN_OUTPUT, 0)	/* (AG18) OLDI0_A6P */
-			J722S_IOPAD(0x298, PIN_OUTPUT, 0)	/* (AB19) OLDI0_A7N */
-			J722S_IOPAD(0x294, PIN_OUTPUT, 0)	/* (AA20) OLDI0_A7P */
-			J722S_IOPAD(0x2a8, PIN_OUTPUT, 0)	/* (AD20) OLDI0_CLK1N */
-			J722S_IOPAD(0x2a4, PIN_OUTPUT, 0)	/* (AE19) OLDI0_CLK1P */
-		>;
-	};
-
-	ospi0_pins_default: ospi0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x000, PIN_OUTPUT, 0)	/* (L24) OSPI0_CLK */
-			J722S_IOPAD(0x02c, PIN_OUTPUT, 0)	/* (K26) OSPI0_CSn0 */
-			J722S_IOPAD(0x030, PIN_OUTPUT, 0)	/* (K23) OSPI0_CSn1 */
-			J722S_IOPAD(0x034, PIN_OUTPUT, 0)	/* (K22) OSPI0_CSn2 */
-			J722S_IOPAD(0x00c, PIN_INPUT, 0)	/* (K27) OSPI0_D0 */
-			J722S_IOPAD(0x010, PIN_INPUT, 0)	/* (L27) OSPI0_D1 */
-			J722S_IOPAD(0x014, PIN_INPUT, 0)	/* (L26) OSPI0_D2 */
-			J722S_IOPAD(0x018, PIN_INPUT, 0)	/* (L25) OSPI0_D3 */
-		>;
-		bootph-all;
-	};
-
-	pcie0_rc_pins_default: pcie0-rc-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x2ac, PIN_OUTPUT, 0)	/* (F25) PCIE0_CLKREQn */
-			J722S_IOPAD(0x1b4, PIN_OUTPUT, 7)	/* (B20) SPI0_CS0.GPIO1_15 */
-		>;
-	};
-
-	pmic_irq_pins_default: pmic-irq-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x090, PIN_INPUT, 7)	/* (P27) GPMC0_BE0n_CLE.GPIO0_35 */
-		>;
-	};
-
-	rgmii1_pins_default: rgmii1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x14c, PIN_INPUT, 0)	/* (AC25) RGMII1_RD0 */
-			J722S_IOPAD(0x150, PIN_INPUT, 0)	/* (AD27) RGMII1_RD1 */
-			J722S_IOPAD(0x154, PIN_INPUT, 0)	/* (AE24) RGMII1_RD2 */
-			J722S_IOPAD(0x158, PIN_INPUT, 0)	/* (AE26) RGMII1_RD3 */
-			J722S_IOPAD(0x148, PIN_INPUT, 0)	/* (AE27) RGMII1_RXC */
-			J722S_IOPAD(0x144, PIN_INPUT, 0)	/* (AD23) RGMII1_RX_CTL */
-			J722S_IOPAD(0x134, PIN_OUTPUT, 0)	/* (AF27) RGMII1_TD0 */
-			J722S_IOPAD(0x138, PIN_OUTPUT, 0)	/* (AE23) RGMII1_TD1 */
-			J722S_IOPAD(0x13c, PIN_OUTPUT, 0)	/* (AG25) RGMII1_TD2 */
-			J722S_IOPAD(0x140, PIN_OUTPUT, 0)	/* (AF24) RGMII1_TD3 */
-			J722S_IOPAD(0x130, PIN_OUTPUT, 0)	/* (AG26) RGMII1_TXC */
-			J722S_IOPAD(0x12c, PIN_OUTPUT, 0)	/* (AF25) RGMII1_TX_CTL */
-		>;
-	};
-
-	rgmii2_pins_default: rgmii2-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0f8, PIN_INPUT, 2)	/* (AB24) VOUT0_HSYNC.RGMII2_RD0 */
-			J722S_IOPAD(0x0fc, PIN_INPUT, 2)	/* (AC27) VOUT0_DE.RGMII2_RD1 */
-			J722S_IOPAD(0x100, PIN_INPUT, 2)	/* (AB23) VOUT0_VSYNC.RGMII2_RD2 */
-			J722S_IOPAD(0x104, PIN_INPUT, 2)	/* (AC26) VOUT0_PCLK.RGMII2_RD3 */
-			J722S_IOPAD(0x0f4, PIN_INPUT, 2)	/* (AB27) VOUT0_DATA15.RGMII2_RXC */
-			J722S_IOPAD(0x0f0, PIN_INPUT, 2)	/* (AB26) VOUT0_DATA14.RGMII2_RX_CTL */
-			J722S_IOPAD(0x0e0, PIN_OUTPUT, 2)	/* (AA25) VOUT0_DATA10.RGMII2_TD0 */
-			J722S_IOPAD(0x0e4, PIN_OUTPUT, 2)	/* (AB25) VOUT0_DATA11.RGMII2_TD1 */
-			J722S_IOPAD(0x0e8, PIN_OUTPUT, 2)	/* (AA23) VOUT0_DATA12.RGMII2_TD2 */
-			J722S_IOPAD(0x0ec, PIN_OUTPUT, 2)	/* (AA22) VOUT0_DATA13.RGMII2_TD3 */
-			J722S_IOPAD(0x0dc, PIN_OUTPUT, 2)	/* (AA27) VOUT0_DATA9.RGMII2_TXC */
-			J722S_IOPAD(0x0d8, PIN_OUTPUT, 2)	/* (AA24) VOUT0_DATA8.RGMII2_TX_CTL */
-		>;
-	};
-
-	rtc_pins_default: rtc-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x094, PIN_INPUT, 7)	/* (P26) GPMC0_BE1n.GPIO0_36 */
-		>;
-	};
-
-	sdhci1_pins_default: sdhci1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x23c, PIN_INPUT, 0)	/* (H22) MMC1_CMD */
-			J722S_IOPAD(0x234, PIN_OUTPUT, 0)	/* (H24) MMC1_CLK */
-			J722S_IOPAD(0x230, PIN_INPUT, 0)	/* (H23) MMC1_DAT0 */
-			J722S_IOPAD(0x22c, PIN_INPUT, 0)	/* (H20) MMC1_DAT1 */
-			J722S_IOPAD(0x228, PIN_INPUT, 0)	/* (J23) MMC1_DAT2 */
-			J722S_IOPAD(0x224, PIN_INPUT, 0)	/* (H25) MMC1_DAT3 */
-			J722S_IOPAD(0x240, PIN_INPUT, 0)	/* (B24) MMC1_SDCD */
-			J722S_IOPAD(0x244, PIN_INPUT, 0)	/* (A24) MMC1_SDWP */
-		>;
-		bootph-all;
-	};
-
-	usb0_connector_pins_default: usb0-connector-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x08c, PIN_INPUT_PULLUP, 7)	/* (N23) GPMC0_WEn.GPIO0_34 */
-		>;
-	};
-
-	usb1_pins_default: usb1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x258, PIN_OUTPUT, 0)	/* (B27) USB1_DRVVBUS */
-		>;
-	};
-
-	vcc_3p3_sd_s0_pins_default: vcc-3p3-sd-s0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x01c, PIN_OUTPUT, 7)	/* (L21) OSPI0_D4.GPIO0_7 */
-		>;
-		bootph-all;
-	};
-
-	vcc_3p3_sd_vio_s0_pins_default: vcc-3p3-sd-vio-s0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x020, PIN_OUTPUT, 7)	/* (M26) OSPI0_D5.GPIO0_8 */
-		>;
-		bootph-all;
-	};
-
-	vcc_3p3_cam_s0_pins_default: vcc-3p3-cam-s0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1f0, PIN_OUTPUT, 7)	/* (A23) EXT_REFCLK1.GPIO1_30 */
-		>;
-	};
-
-	vcc_lcd0_panel_pins_default: vcc-lcd0-panel-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x078, PIN_OUTPUT, 7)	/* (V23) GPMC0_AD15.GPIO0_30 */
-		>;
-	};
-
-	vcc_lcd1_panel_pins_default: vcc-lcd1-panel-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x1c4, PIN_OUTPUT, 7)	/* (E20) SPI0_D1.GPIO1_19 */
-		>;
-	};
-
-	vcc_usb0_vbus_pins_default: vcc-usb0-vbus-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x254, PIN_OUTPUT, 7)	/* (E25) USB0_DRVVBUS.GPIO1_50 */
-		>;
-	};
-};
-
-/* SER1 */
-&main_uart0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_uart0_pins_default>;
-	bootph-all;
-	status = "okay";
-};
-
-/* SER2 */
-&main_uart5 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_uart5_pins_default>;
-	bootph-all;
-	status = "okay";
-};
-
-/* I2S0 */
-&mcasp0 {
-	#sound-dai-cells = <0>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcasp0_pins_default>;
-	op-mode = <0>; /* I2S */
-	tdm-slots = <2>;
-	serial-dir = <2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0>;
-};
-
-/* I2S2 */
-&mcasp2 {
-	#sound-dai-cells = <0>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcasp2_pins_default>;
-	op-mode = <0>; /* I2S */
-	tdm-slots = <2>;
-	serial-dir = <2 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0>;
-};
-
-/* CAN0 */
-&mcu_mcan0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_mcan0_pins_default>;
-	status = "okay";
-};
-
-/* CAN1 */
-&mcu_mcan1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_mcan1_pins_default>;
-	status = "okay";
-};
-
-&mcu_gpio0 {
-	gpio-line-names =
-		"", "", "", "", "", "", "", "", "", "", "", /* 10 */ "GPIO12",
-		"MCU_INT#", "", "", "", "", "", "", "", "", "", "", "GPIO13";
-};
-
-/* I2C_GP */
-&mcu_i2c0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_i2c0_pins_default>;
-	clock-frequency = <100000>;
-	status = "okay";
-
-	/* SMARC Module EEPROM */
-	eeprom@50 {
-		compatible = "atmel,24c32";
-		reg = <0x50>;
-		pagesize = <32>;
-		vcc-supply = <&vcc_1p8_s0>;
-	};
-};
-
-&mcu_pmx0 {
-	mcu_i2c0_pins_default: mcu-i2c0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x044, PIN_INPUT, 0)	/* (B13) MCU_I2C0_SCL */
-			J722S_MCU_IOPAD(0x048, PIN_INPUT, 0)	/* (E11) MCU_I2C0_SDA */
-		>;
-	};
-	mcu_mcan0_pins_default: mcu-mcan0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x038, PIN_INPUT, 0)	/* (D8) MCU_MCAN0_RX */
-			J722S_MCU_IOPAD(0x034, PIN_OUTPUT, 0)	/* (B2) MCU_MCAN0_TX */
-		>;
-	};
-
-	mcu_mcan1_pins_default: mcu-mcan1-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x040, PIN_INPUT, 0)	/* (B1) MCU_MCAN1_RX */
-			J722S_MCU_IOPAD(0x03c, PIN_OUTPUT, 0)	/* (C1) MCU_MCAN1_TX */
-		>;
-	};
-
-	mcu_uart0_pins_default: mcu-uart0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x014, PIN_INPUT, 0)	/* (B8) MCU_UART0_RXD */
-			J722S_MCU_IOPAD(0x018, PIN_OUTPUT, 0)	/* (B4) MCU_UART0_TXD */
-			J722S_MCU_IOPAD(0x01c, PIN_INPUT, 0)	/* (B5) MCU_UART0_CTSn */
-			J722S_MCU_IOPAD(0x020, PIN_OUTPUT, 0)	/* (C5) MCU_UART0_RTSn */
-		>;
-		bootph-all;
-	};
-
-	mcu_spi0_pins_default: mcu-spi0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x008, PIN_OUTPUT, 0)	/* (A9) MCU_SPI0_CLK */
-			J722S_MCU_IOPAD(0x000, PIN_OUTPUT, 0)	/* (C12) MCU_SPI0_CS0 */
-			J722S_MCU_IOPAD(0x004, PIN_OUTPUT, 0)	/* (A10) MCU_SPI0_CS1 */
-			J722S_MCU_IOPAD(0x00c, PIN_INPUT, 0)	/* (B12) MCU_SPI0_D0 */
-			J722S_MCU_IOPAD(0x010, PIN_OUTPUT, 0)	/* (C11) MCU_SPI0_D1 */
-		>;
-	};
-
-	wkup_uart0_pins_default: wkup-uart0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x024, PIN_INPUT, 0)	/* (B3) WKUP_UART0_RXD */
-			J722S_MCU_IOPAD(0x028, PIN_OUTPUT, 0)	/* (C8) WKUP_UART0_TXD */
-		>;
-		bootph-all;
-	};
-
-	wkup_i2c0_pins_default: wkup-i2c0-default-pins {
-		pinctrl-single,pins = <
-			J722S_MCU_IOPAD(0x04c, PIN_INPUT, 0)	/* (B9) WKUP_I2C0_SCL */
-			J722S_MCU_IOPAD(0x050, PIN_INPUT, 0)	/* (D11) WKUP_I2C0_SDA */
-		>;
-	};
-};
-
-/* SPI0 */
-&mcu_spi0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_spi0_pins_default>;
-};
-
-/* SER0 */
-&mcu_uart0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_uart0_pins_default>;
-	bootph-all;
-	status = "okay";
-};
-
-/* QSPI0 */
-&ospi0 {
-	pinctrl-0 = <&ospi0_pins_default>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	flash@0 {
-		compatible = "jedec,spi-nor";
-		reg = <0>;
-		spi-max-frequency = <104000000>;
-		spi-rx-bus-width = <2>;
-		spi-tx-bus-width = <2>;
-		m25p,fast-read;
-		cdns,tshsl-ns = <60>;
-		cdns,tsd2d-ns = <60>;
-		cdns,tchsh-ns = <60>;
-		cdns,tslch-ns = <60>;
-		cdns,read-delay = <3>;
-		vcc-supply = <&vcc_1p8_s0>;
-		bootph-all;
-
-		partitions {
-			compatible = "fixed-partitions";
-			#address-cells = <1>;
-			#size-cells = <1>;
-
-			partition@0 {
-				reg = <0x000000 0x400000>;
-				label = "failsafe bootloader";
-				read-only;
-			};
-		};
-
-		otp-1 {
-			compatible = "user-otp";
-
-			nvmem-layout {
-				compatible = "kontron,sa67-vpd", "kontron,sl28-vpd";
-
-				serial_number: serial-number {
-				};
-
-				base_mac_address: base-mac-address {
-					#nvmem-cell-cells = <1>;
-				};
-			};
-		};
-	};
-};
-
-&pcie0_rc {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pcie0_rc_pins_default>;
-
-	/*
-	 * This is low active, but the driver itself is broken and already
-	 * inverts the logic.
-	 */
-	reset-gpios = <&main_gpio1 15 GPIO_ACTIVE_HIGH>;
-	phys = <&serdes1_pcie>;
-	phy-names = "pcie-phy";
-	status = "okay";
-};
-
-&sdhci0 {
-	disable-wp;
-	bootph-all;
-	ti,driver-strength-ohm = <50>;
-	status = "okay";
-};
-
-/* SDIO */
-&sdhci1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&sdhci1_pins_default>;
-	vmmc-supply = <&vcc_3p3_sd_s0>;
-	vqmmc-supply = <&vcc_3p3_sd_vio_s0>;
-	bootph-all;
-	cd-gpios = <&main_gpio1 48 GPIO_ACTIVE_LOW>;
-	cd-debounce-delay-ms = <100>;
-	ti,fails-without-test-cd;
-	ti,driver-strength-ohm = <50>;
-	status = "okay";
-};
-
-&serdes_ln_ctrl {
-	idle-states = <J722S_SERDES0_LANE0_USB>,
-		      <J722S_SERDES1_LANE0_PCIE0_LANE0>;
-};
-
-&serdes_wiz0 {
-	status = "okay";
-};
-
-&serdes_wiz1 {
-	status = "okay";
-};
-
-&serdes0 {
-	serdes0_usb3: phy@0 {
-		reg = <0>;
-		#phy-cells = <0>;
-		resets = <&serdes_wiz0 1>;
-		cdns,num-lanes = <1>;
-		cdns,phy-type = <PHY_TYPE_USB3>;
-	};
-};
-
-&serdes1 {
-	serdes1_pcie: phy@0 {
-		reg = <0>;
-		#phy-cells = <0>;
-		resets = <&serdes_wiz1 1>;
-		cdns,num-lanes = <1>;
-		cdns,phy-type = <PHY_TYPE_PCIE>;
-	};
-};
-
-&usb0 {
-	/* dual role is implemented but not a full featured OTG */
-	adp-disable;
-	hnp-disable;
-	srp-disable;
-	dr_mode = "otg";
-	usb-role-switch;
-	role-switch-default-mode = "peripheral";
-	status = "okay";
-
-	port {
-		usb0_hc: endpoint {
-			remote-endpoint = <&usb0_connector>;
-		};
-	};
-};
-
-&usb0_phy_ctrl {
-	/*
-	 * Keep this node in the SPL to be able to use the USB controller to
-	 * boot via DFU.
-	 */
-	bootph-all;
-};
-
-&usb1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&usb1_pins_default>;
-
-	dr_mode = "host";
-	maximum-speed = "super-speed";
-	phys = <&serdes0_usb3>;
-	phy-names = "cdns3,usb3-phy";
-};
-
-&usbss0 {
-	ti,vbus-divider;
-	status = "okay";
-};
-
-&usbss1 {
-	ti,vbus-divider;
-	status = "okay";
-};
-
-/* I2C_PM */
-&wkup_i2c0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&wkup_i2c0_pins_default>;
-	clock-frequency = <100000>;
-	status = "okay";
-};
-
-/* SER3 */
-&wkup_uart0 {
-	/* WKUP UART0 is used by Device Manager firmware */
-	pinctrl-names = "default";
-	pinctrl-0 = <&wkup_uart0_pins_default>;
-	bootph-all;
-	status = "reserved";
-};
diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gbe1.dtso b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gbe1.dtso
deleted file mode 100644
index 5dfb0b8f10d2c..0000000000000
--- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gbe1.dtso
+++ /dev/null
@@ -1,26 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only OR MIT
-/*
- * Second ethernet port GBE1.
- *
- * Copyright (c) 2025 Kontron Europe GmbH
- */
-
-/dts-v1/;
-/plugin/;
-
-&cpsw3g_mdio {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	phy1: ethernet-phy@1 {
-		reg = <1>;
-	};
-};
-
-&cpsw_port2 {
-	phy-connection-type = "rgmii-id";
-	phy-handle = <&phy1>;
-	nvmem-cells = <&base_mac_address 1>;
-	nvmem-cell-names = "mac-address";
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gpios.dtso b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gpios.dtso
deleted file mode 100644
index a6ae758e0b3a9..0000000000000
--- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-gpios.dtso
+++ /dev/null
@@ -1,61 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only OR MIT
-/*
- * SMARC GPIOs.
- *
- * Copyright (c) 2025 Kontron Europe GmbH
- */
-
-/dts-v1/;
-/plugin/;
-
-#include <dt-bindings/gpio/gpio.h>
-#include "k3-pinctrl.h"
-
-&main_gpio0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_gpio0_pins_default>;
-};
-
-&main_gpio1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&main_gpio1_pins_default>;
-};
-
-&main_pmx0 {
-	main_gpio0_pins_default: main-gpio0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x0d0, PIN_INPUT, 7)	/* (Y26) VOUT0_DATA6.GPIO0_51 */
-			J722S_IOPAD(0x0d4, PIN_INPUT, 7)	/* (Y27) VOUT0_DATA7.GPIO0_52 */
-			J722S_IOPAD(0x118, PIN_INPUT, 7)	/* (H26) MMC2_CLK.GPIO0_69 */
-			J722S_IOPAD(0x120, PIN_INPUT, 7)	/* (F27) MMC2_CMD.GPIO0_70 */
-		>;
-	};
-
-	main_gpio1_pins_default: main-gpio1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x194, PIN_INPUT, 7)	/* (A25) MCASP0_AXR3.GPIO1_7 */
-			J722S_IOPAD(0x198, PIN_INPUT, 7)	/* (A26) MCASP0_AXR2.GPIO1_8 */
-			J722S_IOPAD(0x1ac, PIN_INPUT, 7)	/* (C27) MCASP0_AFSR.GPIO1_13 */
-			J722S_IOPAD(0x1b0, PIN_INPUT, 7)	/* (F24) MCASP0_ACLKR.GPIO1_14 */
-			J722S_IOPAD(0x1d8, PIN_INPUT, 7)	/* (D22) MCAN0_TX.GPIO1_24 */
-			J722S_IOPAD(0x1dc, PIN_INPUT, 7)	/* (C22) MCAN0_RX.GPIO1_25 */
-			J722S_IOPAD(0x1e8, PIN_INPUT, 7)	/* (C24) I2C1_SCL.GPIO1_28 */
-			J722S_IOPAD(0x1ec, PIN_INPUT, 7)	/* (A22) I2C1_SDA.GPIO1_29 */
-		>;
-	};
-};
-
-&mcu_gpio0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mcu_gpio0_pins_default>;
-};
-
-&mcu_pmx0 {
-	mcu_gpio0_pins_default: mcu-gpio0-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x02c, PIN_INPUT, 7)	/* (C4) WKUP_UART0_CTSn.MCU_GPIO0_11 */
-			J722S_IOPAD(0x084, PIN_INPUT, 7)	/* (F12) WKUP_CLKOUT0.MCU_GPIO0_23 */
-		>;
-	};
-
-};
diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-rtc-rv8263.dtso b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-rtc-rv8263.dtso
deleted file mode 100644
index 0a3e9f614c4c9..0000000000000
--- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-rtc-rv8263.dtso
+++ /dev/null
@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only OR MIT
-/*
- * Microcrystal RV8263 RTC variant.
- *
- * Copyright (c) 2025 Kontron Europe GmbH
- */
-
-/dts-v1/;
-/plugin/;
-
-#include <dt-bindings/interrupt-controller/irq.h>
-
-&{/} {
-	aliases {
-		rtc0 = "/bus@f0000/i2c@20000000/rtc@51"; /* &rtc */
-		rtc1 = "/bus@f0000/bus@b00000/rtc@2b1f0000"; /* &wkup_rtc0 */
-	};
-};
-
-&main_i2c0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	rtc: rtc@51 {
-		compatible = "microcrystal,rv8263";
-		reg = <0x51>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&rtc_pins_default>;
-		interrupts-extended = <&main_gpio0 36 IRQ_TYPE_EDGE_FALLING>;
-	};
-};
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
index ec8ff45877157..dc0d3cf2f985d 100644
--- a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
@@ -26,7 +26,7 @@ reg_3v3_dp: regulator-3v3-dp {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_21_dp>;
 		/* Aquila GPIO_21_DP (AQUILA B57) */
-		gpio = <&main_gpio0 37 GPIO_ACTIVE_HIGH>;
+		gpio = <&main_gpio0 21 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
 		regulator-max-microvolt = <3300000>;
 		regulator-min-microvolt = <3300000>;
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
index f48601ae38b7c..d3677c2c25473 100644
--- a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
@@ -33,7 +33,7 @@ reg_3v3_dp: regulator-3v3-dp {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_21_dp>;
 		/* Aquila GPIO_21_DP (AQUILA B57) */
-		gpio = <&main_gpio0 37 GPIO_ACTIVE_HIGH>;
+		gpio = <&main_gpio0 21 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
 		regulator-max-microvolt = <3300000>;
 		regulator-min-microvolt = <3300000>;
diff --git a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
index 3e5efdfe87f15..54dea36b153b1 100644
--- a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
@@ -331,6 +331,7 @@ phy0: ethernet-phy@0 {
 		reg = <0>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts
index 47702fb279a4a..53e7fbcef52b8 100644
--- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts
@@ -777,6 +777,7 @@ phy0: ethernet-phy@0 {
 		reg = <0>;
 		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
 		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
index 4fea995191136..19c5157510b26 100644
--- a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
@@ -530,6 +530,48 @@ flash@0 {
 		cdns,tchsh-ns = <60>;
 		cdns,tslch-ns = <60>;
 		cdns,read-delay = <2>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "qspi.tiboot3";
+				reg = <0x0 0x80000>;
+			};
+
+			partition@80000 {
+				label = "qspi.tispl";
+				reg = <0x80000 0x200000>;
+			};
+
+			partition@280000 {
+				label = "qspi.u-boot";
+				reg = <0x280000 0x400000>;
+			};
+
+			partition@680000 {
+				label = "qspi.env";
+				reg = <0x680000 0x40000>;
+			};
+
+			partition@6c0000 {
+				label = "qspi.env.backup";
+				reg = <0x6c0000 0x40000>;
+			};
+
+			partition@800000 {
+				label = "qspi.rootfs";
+				reg = <0x800000 0x37c0000>;
+			};
+
+			partition@3fc0000 {
+				label = "qspi.phypattern";
+				reg = <0x3fc0000 0x40000>;
+				bootph-all;
+			};
+		};
 	};
 };
 
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index b67d5b1fc45b0..8a5309bb20294 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -57,6 +57,7 @@ CONFIG_ARCH_HISI=y
 CONFIG_ARCH_KEEMBAY=y
 CONFIG_ARCH_MEDIATEK=y
 CONFIG_ARCH_MESON=y
+CONFIG_ARCH_MICROCHIP=y
 CONFIG_ARCH_MVEBU=y
 CONFIG_ARCH_NXP=y
 CONFIG_ARCH_LAYERSCAPE=y
@@ -416,6 +417,7 @@ CONFIG_MARVELL_88Q2XXX_PHY=y
 CONFIG_MICREL_PHY=y
 CONFIG_MICROSEMI_PHY=y
 CONFIG_AT803X_PHY=y
+CONFIG_QCA808X_PHY=m
 CONFIG_REALTEK_PHY=y
 CONFIG_ROCKCHIP_PHY=y
 CONFIG_DP83867_PHY=y
@@ -647,6 +649,8 @@ CONFIG_PINCTRL_IMX91=y
 CONFIG_PINCTRL_IMX93=y
 CONFIG_PINCTRL_IMX_SCMI=y
 CONFIG_PINCTRL_MSM=y
+CONFIG_PINCTRL_ELIZA=y
+CONFIG_PINCTRL_GLYMUR=y
 CONFIG_PINCTRL_IPQ5018=y
 CONFIG_PINCTRL_IPQ5332=y
 CONFIG_PINCTRL_IPQ5424=y
@@ -728,6 +732,8 @@ CONFIG_GPIO_MACSMC=m
 CONFIG_GPIO_MAX77620=y
 CONFIG_GPIO_SL28CPLD=m
 CONFIG_GPIO_AGGREGATOR=m
+CONFIG_POWER_RESET_GPIO=y
+CONFIG_POWER_RESET_GPIO_RESTART=y
 CONFIG_POWER_RESET_MACSMC=m
 CONFIG_POWER_RESET_MSM=y
 CONFIG_POWER_RESET_QCOM_PON=m
@@ -754,8 +760,8 @@ CONFIG_SENSORS_LM75=m
 CONFIG_SENSORS_LM90=m
 CONFIG_SENSORS_PWM_FAN=m
 CONFIG_SENSORS_RASPBERRYPI_HWMON=m
-CONFIG_SENSORS_SA67MCU=m
 CONFIG_SENSORS_SL28CPLD=m
+CONFIG_SENSORS_AMC6821=m
 CONFIG_SENSORS_INA2XX=m
 CONFIG_SENSORS_INA3221=m
 CONFIG_SENSORS_TMP102=m
@@ -943,6 +949,7 @@ CONFIG_VIDEO_IMX219=m
 CONFIG_VIDEO_IMX412=m
 CONFIG_VIDEO_OV5640=m
 CONFIG_VIDEO_OV5645=m
+CONFIG_VIDEO_S5KJN1=m
 CONFIG_DRM=m
 CONFIG_DRM_I2C_NXP_TDA998X=m
 CONFIG_DRM_HDLCD=m
@@ -1144,6 +1151,7 @@ CONFIG_SND_SOC_ES7241=m
 CONFIG_SND_SOC_ES8316=m
 CONFIG_SND_SOC_ES8328_I2C=m
 CONFIG_SND_SOC_GTM601=m
+CONFIG_SND_SOC_MAX98090=m
 CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m
 CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m
 CONFIG_SND_SOC_PCM3168A_I2C=m
@@ -1159,6 +1167,7 @@ CONFIG_SND_SOC_TLV320AIC32X4_I2C=m
 CONFIG_SND_SOC_TLV320AIC3X_I2C=m
 CONFIG_SND_SOC_WCD9335=m
 CONFIG_SND_SOC_WCD934X=m
+CONFIG_SND_SOC_WCD937X_SDW=m
 CONFIG_SND_SOC_WCD939X_SDW=m
 CONFIG_SND_SOC_WM8524=m
 CONFIG_SND_SOC_WM8904=m
@@ -1454,6 +1463,11 @@ CONFIG_COMMON_CLK_MT8192_SCP_ADSP=y
 CONFIG_COMMON_CLK_MT8192_VDECSYS=y
 CONFIG_COMMON_CLK_MT8192_VENCSYS=y
 CONFIG_COMMON_CLK_QCOM=y
+CONFIG_CLK_ELIZA_GCC=y
+CONFIG_CLK_ELIZA_TCSRCC=m
+CONFIG_CLK_GLYMUR_DISPCC=m
+CONFIG_CLK_GLYMUR_GCC=y
+CONFIG_CLK_GLYMUR_TCSRCC=m
 CONFIG_CLK_KAANAPALI_GCC=y
 CONFIG_CLK_KAANAPALI_TCSRCC=m
 CONFIG_CLK_X1E80100_CAMCC=m
@@ -1525,6 +1539,7 @@ CONFIG_SM_CAMCC_MILOS=m
 CONFIG_SM_CAMCC_8250=m
 CONFIG_SM_CAMCC_8550=m
 CONFIG_SM_CAMCC_8650=m
+CONFIG_SM_CAMCC_8750=m
 CONFIG_SM_DISPCC_6115=m
 CONFIG_SM_DISPCC_8250=y
 CONFIG_SM_DISPCC_6350=m
@@ -1550,6 +1565,7 @@ CONFIG_SM_GPUCC_8350=m
 CONFIG_SM_GPUCC_8450=m
 CONFIG_SM_GPUCC_8550=m
 CONFIG_SM_GPUCC_8650=m
+CONFIG_SM_GPUCC_8750=m
 CONFIG_SM_TCSRCC_8550=y
 CONFIG_SM_TCSRCC_8650=y
 CONFIG_SM_TCSRCC_8750=m
@@ -1558,6 +1574,7 @@ CONFIG_SM_VIDEOCC_6350=m
 CONFIG_SM_VIDEOCC_MILOS=m
 CONFIG_SM_VIDEOCC_8250=y
 CONFIG_SM_VIDEOCC_8550=m
+CONFIG_SM_VIDEOCC_8750=m
 CONFIG_QCOM_HFPLL=y
 CONFIG_CLK_GFM_LPASS_SM8250=m
 CONFIG_SM_VIDEOCC_8450=m
@@ -1834,6 +1851,7 @@ CONFIG_FPGA_MGR_ZYNQMP_FPGA=m
 CONFIG_FPGA_MGR_VERSAL_FPGA=m
 CONFIG_TEE=y
 CONFIG_OPTEE=y
+CONFIG_QCOMTEE=m
 CONFIG_MUX_GPIO=m
 CONFIG_MUX_MMIO=y
 CONFIG_SLIMBUS=m
@@ -1846,6 +1864,8 @@ CONFIG_INTERCONNECT_IMX8MN=m
 CONFIG_INTERCONNECT_IMX8MQ=m
 CONFIG_INTERCONNECT_IMX8MP=y
 CONFIG_INTERCONNECT_QCOM=y
+CONFIG_INTERCONNECT_QCOM_ELIZA=y
+CONFIG_INTERCONNECT_QCOM_GLYMUR=y
 CONFIG_INTERCONNECT_QCOM_KAANAPALI=y
 CONFIG_INTERCONNECT_QCOM_MSM8916=m
 CONFIG_INTERCONNECT_QCOM_MSM8953=y
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 81ed892b3b721..82794afaffc9d 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -144,7 +144,7 @@ config CRYPTO_AES_ARM64_CE_CCM
 	select CRYPTO_ALGAPI
 	select CRYPTO_AES_ARM64_CE_BLK
 	select CRYPTO_AEAD
-	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_AES_CBC_MACS
 	help
 	  AEAD cipher: AES cipher algorithms (FIPS-197) with
 	  CCM (Counter with Cipher Block Chaining-Message Authentication Code)
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 3574e917bc37d..8a8e3e551ed33 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -33,10 +33,10 @@ obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
 aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o
-aes-ce-blk-y := aes-glue-ce.o aes-ce.o
+aes-ce-blk-y := aes-glue-ce.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
-aes-neon-blk-y := aes-glue-neon.o aes-neon.o
+aes-neon-blk-y := aes-glue-neon.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
 aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index db371ac051fcf..45aed0073283e 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -31,10 +31,6 @@ static int num_rounds(struct crypto_aes_ctx *ctx)
 	return 6 + ctx->key_length / 4;
 }
 
-asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
-				 int blocks, u8 dg[], int enc_before,
-				 int enc_after);
-
 asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
 				   u32 const rk[], u32 rounds, u8 mac[],
 				   u8 ctr[], u8 const final_iv[]);
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
deleted file mode 100644
index b262eaa9170c3..0000000000000
--- a/arch/arm64/crypto/aes-ce.S
+++ /dev/null
@@ -1,146 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
- *                                    Crypto Extensions
- *
- * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
-#define AES_FUNC_START(func)		SYM_FUNC_START(ce_ ## func)
-#define AES_FUNC_END(func)		SYM_FUNC_END(ce_ ## func)
-
-	.arch		armv8-a+crypto
-
-	xtsmask		.req	v16
-	cbciv		.req	v16
-	vctr		.req	v16
-
-	.macro		xts_reload_mask, tmp
-	.endm
-
-	.macro		xts_cts_skip_tw, reg, lbl
-	.endm
-
-	/* preload all round keys */
-	.macro		load_round_keys, rk, nr, tmp
-	add		\tmp, \rk, \nr, sxtw #4
-	sub		\tmp, \tmp, #160
-	ld1		{v17.4s-v20.4s}, [\rk]
-	ld1		{v21.4s-v24.4s}, [\tmp], #64
-	ld1		{v25.4s-v28.4s}, [\tmp], #64
-	ld1		{v29.4s-v31.4s}, [\tmp]
-	.endm
-
-	/* prepare for encryption with key in rk[] */
-	.macro		enc_prepare, rounds, rk, temp
-	load_round_keys	\rk, \rounds, \temp
-	.endm
-
-	/* prepare for encryption (again) but with new key in rk[] */
-	.macro		enc_switch_key, rounds, rk, temp
-	load_round_keys	\rk, \rounds, \temp
-	.endm
-
-	/* prepare for decryption with key in rk[] */
-	.macro		dec_prepare, rounds, rk, temp
-	load_round_keys	\rk, \rounds, \temp
-	.endm
-
-	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
-	aes\de		\i0\().16b, \k\().16b
-	aes\mc		\i0\().16b, \i0\().16b
-	.ifnb		\i1
-	aes\de		\i1\().16b, \k\().16b
-	aes\mc		\i1\().16b, \i1\().16b
-	.ifnb		\i3
-	aes\de		\i2\().16b, \k\().16b
-	aes\mc		\i2\().16b, \i2\().16b
-	aes\de		\i3\().16b, \k\().16b
-	aes\mc		\i3\().16b, \i3\().16b
-	.ifnb		\i4
-	aes\de		\i4\().16b, \k\().16b
-	aes\mc		\i4\().16b, \i4\().16b
-	.endif
-	.endif
-	.endif
-	.endm
-
-	/* up to 5 interleaved encryption rounds with the same round key */
-	.macro		round_Nx, enc, k, i0, i1, i2, i3, i4
-	.ifc		\enc, e
-	do_enc_Nx	e, mc, \k, \i0, \i1, \i2, \i3, \i4
-	.else
-	do_enc_Nx	d, imc, \k, \i0, \i1, \i2, \i3, \i4
-	.endif
-	.endm
-
-	/* up to 5 interleaved final rounds */
-	.macro		fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
-	aes\de		\i0\().16b, \k\().16b
-	.ifnb		\i1
-	aes\de		\i1\().16b, \k\().16b
-	.ifnb		\i3
-	aes\de		\i2\().16b, \k\().16b
-	aes\de		\i3\().16b, \k\().16b
-	.ifnb		\i4
-	aes\de		\i4\().16b, \k\().16b
-	.endif
-	.endif
-	.endif
-	eor		\i0\().16b, \i0\().16b, \k2\().16b
-	.ifnb		\i1
-	eor		\i1\().16b, \i1\().16b, \k2\().16b
-	.ifnb		\i3
-	eor		\i2\().16b, \i2\().16b, \k2\().16b
-	eor		\i3\().16b, \i3\().16b, \k2\().16b
-	.ifnb		\i4
-	eor		\i4\().16b, \i4\().16b, \k2\().16b
-	.endif
-	.endif
-	.endif
-	.endm
-
-	/* up to 5 interleaved blocks */
-	.macro		do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
-	tbz		\rounds, #2, .L\@	/* 128 bits */
-	round_Nx	\enc, v17, \i0, \i1, \i2, \i3, \i4
-	round_Nx	\enc, v18, \i0, \i1, \i2, \i3, \i4
-	tbz		\rounds, #1, .L\@	/* 192 bits */
-	round_Nx	\enc, v19, \i0, \i1, \i2, \i3, \i4
-	round_Nx	\enc, v20, \i0, \i1, \i2, \i3, \i4
-.L\@:	.irp		key, v21, v22, v23, v24, v25, v26, v27, v28, v29
-	round_Nx	\enc, \key, \i0, \i1, \i2, \i3, \i4
-	.endr
-	fin_round_Nx	\enc, v30, v31, \i0, \i1, \i2, \i3, \i4
-	.endm
-
-	.macro		encrypt_block, in, rounds, t0, t1, t2
-	do_block_Nx	e, \rounds, \in
-	.endm
-
-	.macro		encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
-	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3
-	.endm
-
-	.macro		encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
-	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3, \i4
-	.endm
-
-	.macro		decrypt_block, in, rounds, t0, t1, t2
-	do_block_Nx	d, \rounds, \in
-	.endm
-
-	.macro		decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
-	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3
-	.endm
-
-	.macro		decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
-	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3, \i4
-	.endm
-
-#define MAX_STRIDE	5
-
-#include "aes-modes.S"
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 92f43e1cd0977..85497cfe76d8f 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -7,7 +7,6 @@
 
 #include <crypto/aes.h>
 #include <crypto/ctr.h>
-#include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/sha2.h>
@@ -37,7 +36,6 @@
 #define aes_xctr_encrypt	ce_aes_xctr_encrypt
 #define aes_xts_encrypt		ce_aes_xts_encrypt
 #define aes_xts_decrypt		ce_aes_xts_decrypt
-#define aes_mac_update		ce_aes_mac_update
 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions");
 #else
 #define MODE			"neon"
@@ -54,7 +52,6 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions");
 #define aes_xctr_encrypt	neon_aes_xctr_encrypt
 #define aes_xts_encrypt		neon_aes_xts_encrypt
 #define aes_xts_decrypt		neon_aes_xts_decrypt
-#define aes_mac_update		neon_aes_mac_update
 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON");
 #endif
 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
@@ -66,53 +63,11 @@ MODULE_ALIAS_CRYPTO("xctr(aes)");
 #endif
 MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
-MODULE_ALIAS_CRYPTO("cmac(aes)");
-MODULE_ALIAS_CRYPTO("xcbc(aes)");
-MODULE_ALIAS_CRYPTO("cbcmac(aes)");
 
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_IMPORT_NS("CRYPTO_INTERNAL");
 MODULE_LICENSE("GPL v2");
 
-/* defined in aes-modes.S */
-asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int blocks);
-asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int blocks);
-
-asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int blocks, u8 iv[]);
-asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int blocks, u8 iv[]);
-
-asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int bytes, u8 const iv[]);
-asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int bytes, u8 const iv[]);
-
-asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				int rounds, int bytes, u8 ctr[]);
-
-asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				 int rounds, int bytes, u8 ctr[], int byte_ctr);
-
-asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
-				int rounds, int bytes, u32 const rk2[], u8 iv[],
-				int first);
-asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
-				int rounds, int bytes, u32 const rk2[], u8 iv[],
-				int first);
-
-asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
-				      int rounds, int blocks, u8 iv[],
-				      u32 const rk2[]);
-asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
-				      int rounds, int blocks, u8 iv[],
-				      u32 const rk2[]);
-
-asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds,
-			      int blocks, u8 dg[], int enc_before,
-			      int enc_after);
-
 struct crypto_aes_xts_ctx {
 	struct crypto_aes_ctx key1;
 	struct crypto_aes_ctx __aligned(8) key2;
@@ -123,15 +78,6 @@ struct crypto_aes_essiv_cbc_ctx {
 	struct crypto_aes_ctx __aligned(8) key2;
 };
 
-struct mac_tfm_ctx {
-	struct crypto_aes_ctx key;
-	u8 __aligned(8) consts[];
-};
-
-struct mac_desc_ctx {
-	u8 dg[AES_BLOCK_SIZE];
-};
-
 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			       unsigned int key_len)
 {
@@ -762,222 +708,19 @@ static struct skcipher_alg aes_algs[] = { {
 	.decrypt	= essiv_cbc_decrypt,
 } };
 
-static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
-			 unsigned int key_len)
-{
-	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
-
-	return aes_expandkey(&ctx->key, in_key, key_len);
-}
-
-static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
-{
-	u64 a = be64_to_cpu(x->a);
-	u64 b = be64_to_cpu(x->b);
-
-	y->a = cpu_to_be64((a << 1) | (b >> 63));
-	y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
-}
-
-static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
-		       unsigned int key_len)
-{
-	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
-	be128 *consts = (be128 *)ctx->consts;
-	int rounds = 6 + key_len / 4;
-	int err;
-
-	err = cbcmac_setkey(tfm, in_key, key_len);
-	if (err)
-		return err;
-
-	/* encrypt the zero vector */
-	scoped_ksimd()
-		aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){},
-				ctx->key.key_enc, rounds, 1);
-
-	cmac_gf128_mul_by_x(consts, consts);
-	cmac_gf128_mul_by_x(consts + 1, consts);
-
-	return 0;
-}
-
-static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
-		       unsigned int key_len)
-{
-	static u8 const ks[3][AES_BLOCK_SIZE] = {
-		{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
-		{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
-		{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
-	};
-
-	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
-	int rounds = 6 + key_len / 4;
-	u8 key[AES_BLOCK_SIZE];
-	int err;
-
-	err = cbcmac_setkey(tfm, in_key, key_len);
-	if (err)
-		return err;
-
-	scoped_ksimd() {
-		aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
-		aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
-	}
-
-	return cbcmac_setkey(tfm, key, sizeof(key));
-}
-
-static int mac_init(struct shash_desc *desc)
-{
-	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	memset(ctx->dg, 0, AES_BLOCK_SIZE);
-	return 0;
-}
-
-static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
-			  u8 dg[], int enc_before)
-{
-	int rounds = 6 + ctx->key_length / 4;
-	int rem;
-
-	do {
-		scoped_ksimd()
-			rem = aes_mac_update(in, ctx->key_enc, rounds, blocks,
-					     dg, enc_before, !enc_before);
-		in += (blocks - rem) * AES_BLOCK_SIZE;
-		blocks = rem;
-	} while (blocks);
-}
-
-static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
-{
-	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-	int blocks = len / AES_BLOCK_SIZE;
-
-	len %= AES_BLOCK_SIZE;
-	mac_do_update(&tctx->key, p, blocks, ctx->dg, 0);
-	return len;
-}
-
-static int cbcmac_finup(struct shash_desc *desc, const u8 *src,
-			unsigned int len, u8 *out)
-{
-	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-
-	if (len) {
-		crypto_xor(ctx->dg, src, len);
-		mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1);
-	}
-	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
-	return 0;
-}
-
-static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len,
-		      u8 *out)
-{
-	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-	u8 *consts = tctx->consts;
-
-	crypto_xor(ctx->dg, src, len);
-	if (len != AES_BLOCK_SIZE) {
-		ctx->dg[len] ^= 0x80;
-		consts += AES_BLOCK_SIZE;
-	}
-	mac_do_update(&tctx->key, consts, 1, ctx->dg, 0);
-	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
-	return 0;
-}
-
-static struct shash_alg mac_algs[] = { {
-	.base.cra_name		= "cmac(aes)",
-	.base.cra_driver_name	= "cmac-aes-" MODE,
-	.base.cra_priority	= PRIO,
-	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-				  CRYPTO_AHASH_ALG_FINAL_NONZERO,
-	.base.cra_blocksize	= AES_BLOCK_SIZE,
-	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
-				  2 * AES_BLOCK_SIZE,
-	.base.cra_module	= THIS_MODULE,
-
-	.digestsize		= AES_BLOCK_SIZE,
-	.init			= mac_init,
-	.update			= mac_update,
-	.finup			= cmac_finup,
-	.setkey			= cmac_setkey,
-	.descsize		= sizeof(struct mac_desc_ctx),
-}, {
-	.base.cra_name		= "xcbc(aes)",
-	.base.cra_driver_name	= "xcbc-aes-" MODE,
-	.base.cra_priority	= PRIO,
-	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-				  CRYPTO_AHASH_ALG_FINAL_NONZERO,
-	.base.cra_blocksize	= AES_BLOCK_SIZE,
-	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
-				  2 * AES_BLOCK_SIZE,
-	.base.cra_module	= THIS_MODULE,
-
-	.digestsize		= AES_BLOCK_SIZE,
-	.init			= mac_init,
-	.update			= mac_update,
-	.finup			= cmac_finup,
-	.setkey			= xcbc_setkey,
-	.descsize		= sizeof(struct mac_desc_ctx),
-}, {
-	.base.cra_name		= "cbcmac(aes)",
-	.base.cra_driver_name	= "cbcmac-aes-" MODE,
-	.base.cra_priority	= PRIO,
-	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
-	.base.cra_blocksize	= AES_BLOCK_SIZE,
-	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx),
-	.base.cra_module	= THIS_MODULE,
-
-	.digestsize		= AES_BLOCK_SIZE,
-	.init			= mac_init,
-	.update			= mac_update,
-	.finup			= cbcmac_finup,
-	.setkey			= cbcmac_setkey,
-	.descsize		= sizeof(struct mac_desc_ctx),
-} };
-
 static void aes_exit(void)
 {
-	crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 }
 
 static int __init aes_init(void)
 {
-	int err;
-
-	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
-	if (err)
-		return err;
-
-	err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
-	if (err)
-		goto unregister_ciphers;
-
-	return 0;
-
-unregister_ciphers:
-	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
-	return err;
+	return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 }
 
 #ifdef USE_V8_CRYPTO_EXTENSIONS
 module_cpu_feature_match(AES, aes_init);
-EXPORT_SYMBOL_NS(ce_aes_mac_update, "CRYPTO_INTERNAL");
 #else
 module_init(aes_init);
-EXPORT_SYMBOL(neon_aes_ecb_encrypt);
-EXPORT_SYMBOL(neon_aes_cbc_encrypt);
-EXPORT_SYMBOL(neon_aes_ctr_encrypt);
-EXPORT_SYMBOL(neon_aes_xts_encrypt);
-EXPORT_SYMBOL(neon_aes_xts_decrypt);
 #endif
 module_exit(aes_exit);
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
deleted file mode 100644
index 0e834a2c062cf..0000000000000
--- a/arch/arm64/crypto/aes-modes.S
+++ /dev/null
@@ -1,866 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
- *
- * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-/* included by aes-ce.S and aes-neon.S */
-
-	.text
-	.align		4
-
-#ifndef MAX_STRIDE
-#define MAX_STRIDE	4
-#endif
-
-#if MAX_STRIDE == 4
-#define ST4(x...) x
-#define ST5(x...)
-#else
-#define ST4(x...)
-#define ST5(x...) x
-#endif
-
-SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
-	encrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
-	ret
-SYM_FUNC_END(aes_encrypt_block4x)
-
-SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
-	decrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
-	ret
-SYM_FUNC_END(aes_decrypt_block4x)
-
-#if MAX_STRIDE == 5
-SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
-	encrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
-	ret
-SYM_FUNC_END(aes_encrypt_block5x)
-
-SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
-	decrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
-	ret
-SYM_FUNC_END(aes_decrypt_block5x)
-#endif
-
-	/*
-	 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int blocks)
-	 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int blocks)
-	 */
-
-AES_FUNC_START(aes_ecb_encrypt)
-	frame_push	0
-
-	enc_prepare	w3, x2, x5
-
-.LecbencloopNx:
-	subs		w4, w4, #MAX_STRIDE
-	bmi		.Lecbenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
-ST4(	bl		aes_encrypt_block4x		)
-ST5(	ld1		{v4.16b}, [x1], #16		)
-ST5(	bl		aes_encrypt_block5x		)
-	st1		{v0.16b-v3.16b}, [x0], #64
-ST5(	st1		{v4.16b}, [x0], #16		)
-	b		.LecbencloopNx
-.Lecbenc1x:
-	adds		w4, w4, #MAX_STRIDE
-	beq		.Lecbencout
-.Lecbencloop:
-	ld1		{v0.16b}, [x1], #16		/* get next pt block */
-	encrypt_block	v0, w3, x2, x5, w6
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
-	bne		.Lecbencloop
-.Lecbencout:
-	frame_pop
-	ret
-AES_FUNC_END(aes_ecb_encrypt)
-
-
-AES_FUNC_START(aes_ecb_decrypt)
-	frame_push	0
-
-	dec_prepare	w3, x2, x5
-
-.LecbdecloopNx:
-	subs		w4, w4, #MAX_STRIDE
-	bmi		.Lecbdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
-ST4(	bl		aes_decrypt_block4x		)
-ST5(	ld1		{v4.16b}, [x1], #16		)
-ST5(	bl		aes_decrypt_block5x		)
-	st1		{v0.16b-v3.16b}, [x0], #64
-ST5(	st1		{v4.16b}, [x0], #16		)
-	b		.LecbdecloopNx
-.Lecbdec1x:
-	adds		w4, w4, #MAX_STRIDE
-	beq		.Lecbdecout
-.Lecbdecloop:
-	ld1		{v0.16b}, [x1], #16		/* get next ct block */
-	decrypt_block	v0, w3, x2, x5, w6
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
-	bne		.Lecbdecloop
-.Lecbdecout:
-	frame_pop
-	ret
-AES_FUNC_END(aes_ecb_decrypt)
-
-
-	/*
-	 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int blocks, u8 iv[])
-	 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int blocks, u8 iv[])
-	 * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
-	 *			 int rounds, int blocks, u8 iv[],
-	 *			 u32 const rk2[]);
-	 * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
-	 *			 int rounds, int blocks, u8 iv[],
-	 *			 u32 const rk2[]);
-	 */
-
-AES_FUNC_START(aes_essiv_cbc_encrypt)
-	ld1		{v4.16b}, [x5]			/* get iv */
-
-	mov		w8, #14				/* AES-256: 14 rounds */
-	enc_prepare	w8, x6, x7
-	encrypt_block	v4, w8, x6, x7, w9
-	enc_switch_key	w3, x2, x6
-	b		.Lcbcencloop4x
-
-AES_FUNC_START(aes_cbc_encrypt)
-	ld1		{v4.16b}, [x5]			/* get iv */
-	enc_prepare	w3, x2, x6
-
-.Lcbcencloop4x:
-	subs		w4, w4, #4
-	bmi		.Lcbcenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
-	eor		v0.16b, v0.16b, v4.16b		/* ..and xor with iv */
-	encrypt_block	v0, w3, x2, x6, w7
-	eor		v1.16b, v1.16b, v0.16b
-	encrypt_block	v1, w3, x2, x6, w7
-	eor		v2.16b, v2.16b, v1.16b
-	encrypt_block	v2, w3, x2, x6, w7
-	eor		v3.16b, v3.16b, v2.16b
-	encrypt_block	v3, w3, x2, x6, w7
-	st1		{v0.16b-v3.16b}, [x0], #64
-	mov		v4.16b, v3.16b
-	b		.Lcbcencloop4x
-.Lcbcenc1x:
-	adds		w4, w4, #4
-	beq		.Lcbcencout
-.Lcbcencloop:
-	ld1		{v0.16b}, [x1], #16		/* get next pt block */
-	eor		v4.16b, v4.16b, v0.16b		/* ..and xor with iv */
-	encrypt_block	v4, w3, x2, x6, w7
-	st1		{v4.16b}, [x0], #16
-	subs		w4, w4, #1
-	bne		.Lcbcencloop
-.Lcbcencout:
-	st1		{v4.16b}, [x5]			/* return iv */
-	ret
-AES_FUNC_END(aes_cbc_encrypt)
-AES_FUNC_END(aes_essiv_cbc_encrypt)
-
-AES_FUNC_START(aes_essiv_cbc_decrypt)
-	ld1		{cbciv.16b}, [x5]		/* get iv */
-
-	mov		w8, #14				/* AES-256: 14 rounds */
-	enc_prepare	w8, x6, x7
-	encrypt_block	cbciv, w8, x6, x7, w9
-	b		.Lessivcbcdecstart
-
-AES_FUNC_START(aes_cbc_decrypt)
-	ld1		{cbciv.16b}, [x5]		/* get iv */
-.Lessivcbcdecstart:
-	frame_push	0
-	dec_prepare	w3, x2, x6
-
-.LcbcdecloopNx:
-	subs		w4, w4, #MAX_STRIDE
-	bmi		.Lcbcdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
-#if MAX_STRIDE == 5
-	ld1		{v4.16b}, [x1], #16		/* get 1 ct block */
-	mov		v5.16b, v0.16b
-	mov		v6.16b, v1.16b
-	mov		v7.16b, v2.16b
-	bl		aes_decrypt_block5x
-	sub		x1, x1, #32
-	eor		v0.16b, v0.16b, cbciv.16b
-	eor		v1.16b, v1.16b, v5.16b
-	ld1		{v5.16b}, [x1], #16		/* reload 1 ct block */
-	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
-	eor		v2.16b, v2.16b, v6.16b
-	eor		v3.16b, v3.16b, v7.16b
-	eor		v4.16b, v4.16b, v5.16b
-#else
-	mov		v4.16b, v0.16b
-	mov		v5.16b, v1.16b
-	mov		v6.16b, v2.16b
-	bl		aes_decrypt_block4x
-	sub		x1, x1, #16
-	eor		v0.16b, v0.16b, cbciv.16b
-	eor		v1.16b, v1.16b, v4.16b
-	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
-	eor		v2.16b, v2.16b, v5.16b
-	eor		v3.16b, v3.16b, v6.16b
-#endif
-	st1		{v0.16b-v3.16b}, [x0], #64
-ST5(	st1		{v4.16b}, [x0], #16		)
-	b		.LcbcdecloopNx
-.Lcbcdec1x:
-	adds		w4, w4, #MAX_STRIDE
-	beq		.Lcbcdecout
-.Lcbcdecloop:
-	ld1		{v1.16b}, [x1], #16		/* get next ct block */
-	mov		v0.16b, v1.16b			/* ...and copy to v0 */
-	decrypt_block	v0, w3, x2, x6, w7
-	eor		v0.16b, v0.16b, cbciv.16b	/* xor with iv => pt */
-	mov		cbciv.16b, v1.16b		/* ct is next iv */
-	st1		{v0.16b}, [x0], #16
-	subs		w4, w4, #1
-	bne		.Lcbcdecloop
-.Lcbcdecout:
-	st1		{cbciv.16b}, [x5]		/* return iv */
-	frame_pop
-	ret
-AES_FUNC_END(aes_cbc_decrypt)
-AES_FUNC_END(aes_essiv_cbc_decrypt)
-
-
-	/*
-	 * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
-	 *		       int rounds, int bytes, u8 const iv[])
-	 * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
-	 *		       int rounds, int bytes, u8 const iv[])
-	 */
-
-AES_FUNC_START(aes_cbc_cts_encrypt)
-	adr_l		x8, .Lcts_permute_table
-	sub		x4, x4, #16
-	add		x9, x8, #32
-	add		x8, x8, x4
-	sub		x9, x9, x4
-	ld1		{v3.16b}, [x8]
-	ld1		{v4.16b}, [x9]
-
-	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
-	ld1		{v1.16b}, [x1]
-
-	ld1		{v5.16b}, [x5]			/* get iv */
-	enc_prepare	w3, x2, x6
-
-	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
-	tbl		v1.16b, {v1.16b}, v4.16b
-	encrypt_block	v0, w3, x2, x6, w7
-
-	eor		v1.16b, v1.16b, v0.16b
-	tbl		v0.16b, {v0.16b}, v3.16b
-	encrypt_block	v1, w3, x2, x6, w7
-
-	add		x4, x0, x4
-	st1		{v0.16b}, [x4]			/* overlapping stores */
-	st1		{v1.16b}, [x0]
-	ret
-AES_FUNC_END(aes_cbc_cts_encrypt)
-
-AES_FUNC_START(aes_cbc_cts_decrypt)
-	adr_l		x8, .Lcts_permute_table
-	sub		x4, x4, #16
-	add		x9, x8, #32
-	add		x8, x8, x4
-	sub		x9, x9, x4
-	ld1		{v3.16b}, [x8]
-	ld1		{v4.16b}, [x9]
-
-	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
-	ld1		{v1.16b}, [x1]
-
-	ld1		{v5.16b}, [x5]			/* get iv */
-	dec_prepare	w3, x2, x6
-
-	decrypt_block	v0, w3, x2, x6, w7
-	tbl		v2.16b, {v0.16b}, v3.16b
-	eor		v2.16b, v2.16b, v1.16b
-
-	tbx		v0.16b, {v1.16b}, v4.16b
-	decrypt_block	v0, w3, x2, x6, w7
-	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
-
-	add		x4, x0, x4
-	st1		{v2.16b}, [x4]			/* overlapping stores */
-	st1		{v0.16b}, [x0]
-	ret
-AES_FUNC_END(aes_cbc_cts_decrypt)
-
-	.section	".rodata", "a"
-	.align		6
-.Lcts_permute_table:
-	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
-	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
-	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-	.previous
-
-	/*
-	 * This macro generates the code for CTR and XCTR mode.
-	 */
-.macro ctr_encrypt xctr
-	// Arguments
-	OUT		.req x0
-	IN		.req x1
-	KEY		.req x2
-	ROUNDS_W	.req w3
-	BYTES_W		.req w4
-	IV		.req x5
-	BYTE_CTR_W 	.req w6		// XCTR only
-	// Intermediate values
-	CTR_W		.req w11	// XCTR only
-	CTR		.req x11	// XCTR only
-	IV_PART		.req x12
-	BLOCKS		.req x13
-	BLOCKS_W	.req w13
-
-	frame_push	0
-
-	enc_prepare	ROUNDS_W, KEY, IV_PART
-	ld1		{vctr.16b}, [IV]
-
-	/*
-	 * Keep 64 bits of the IV in a register.  For CTR mode this lets us
-	 * easily increment the IV.  For XCTR mode this lets us efficiently XOR
-	 * the 64-bit counter with the IV.
-	 */
-	.if \xctr
-		umov		IV_PART, vctr.d[0]
-		lsr		CTR_W, BYTE_CTR_W, #4
-	.else
-		umov		IV_PART, vctr.d[1]
-		rev		IV_PART, IV_PART
-	.endif
-
-.LctrloopNx\xctr:
-	add		BLOCKS_W, BYTES_W, #15
-	sub		BYTES_W, BYTES_W, #MAX_STRIDE << 4
-	lsr		BLOCKS_W, BLOCKS_W, #4
-	mov		w8, #MAX_STRIDE
-	cmp		BLOCKS_W, w8
-	csel		BLOCKS_W, BLOCKS_W, w8, lt
-
-	/*
-	 * Set up the counter values in v0-v{MAX_STRIDE-1}.
-	 *
-	 * If we are encrypting less than MAX_STRIDE blocks, the tail block
-	 * handling code expects the last keystream block to be in
-	 * v{MAX_STRIDE-1}.  For example: if encrypting two blocks with
-	 * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
-	 */
-	.if \xctr
-		add		CTR, CTR, BLOCKS
-	.else
-		adds		IV_PART, IV_PART, BLOCKS
-	.endif
-	mov		v0.16b, vctr.16b
-	mov		v1.16b, vctr.16b
-	mov		v2.16b, vctr.16b
-	mov		v3.16b, vctr.16b
-ST5(	mov		v4.16b, vctr.16b		)
-	.if \xctr
-		sub		x6, CTR, #MAX_STRIDE - 1
-		sub		x7, CTR, #MAX_STRIDE - 2
-		sub		x8, CTR, #MAX_STRIDE - 3
-		sub		x9, CTR, #MAX_STRIDE - 4
-ST5(		sub		x10, CTR, #MAX_STRIDE - 5	)
-		eor		x6, x6, IV_PART
-		eor		x7, x7, IV_PART
-		eor		x8, x8, IV_PART
-		eor		x9, x9, IV_PART
-ST5(		eor		x10, x10, IV_PART		)
-		mov		v0.d[0], x6
-		mov		v1.d[0], x7
-		mov		v2.d[0], x8
-		mov		v3.d[0], x9
-ST5(		mov		v4.d[0], x10			)
-	.else
-		bcs		0f
-		.subsection	1
-		/*
-		 * This subsection handles carries.
-		 *
-		 * Conditional branching here is allowed with respect to time
-		 * invariance since the branches are dependent on the IV instead
-		 * of the plaintext or key.  This code is rarely executed in
-		 * practice anyway.
-		 */
-
-		/* Apply carry to outgoing counter. */
-0:		umov		x8, vctr.d[0]
-		rev		x8, x8
-		add		x8, x8, #1
-		rev		x8, x8
-		ins		vctr.d[0], x8
-
-		/*
-		 * Apply carry to counter blocks if needed.
-		 *
-		 * Since the carry flag was set, we know 0 <= IV_PART <
-		 * MAX_STRIDE.  Using the value of IV_PART we can determine how
-		 * many counter blocks need to be updated.
-		 */
-		cbz		IV_PART, 2f
-		adr		x16, 1f
-		sub		x16, x16, IV_PART, lsl #3
-		br		x16
-		bti		c
-		mov		v0.d[0], vctr.d[0]
-		bti		c
-		mov		v1.d[0], vctr.d[0]
-		bti		c
-		mov		v2.d[0], vctr.d[0]
-		bti		c
-		mov		v3.d[0], vctr.d[0]
-ST5(		bti		c				)
-ST5(		mov		v4.d[0], vctr.d[0]		)
-1:		b		2f
-		.previous
-
-2:		rev		x7, IV_PART
-		ins		vctr.d[1], x7
-		sub		x7, IV_PART, #MAX_STRIDE - 1
-		sub		x8, IV_PART, #MAX_STRIDE - 2
-		sub		x9, IV_PART, #MAX_STRIDE - 3
-		rev		x7, x7
-		rev		x8, x8
-		mov		v1.d[1], x7
-		rev		x9, x9
-ST5(		sub		x10, IV_PART, #MAX_STRIDE - 4	)
-		mov		v2.d[1], x8
-ST5(		rev		x10, x10			)
-		mov		v3.d[1], x9
-ST5(		mov		v4.d[1], x10			)
-	.endif
-
-	/*
-	 * If there are at least MAX_STRIDE blocks left, XOR the data with
-	 * keystream and store.  Otherwise jump to tail handling.
-	 */
-	tbnz		BYTES_W, #31, .Lctrtail\xctr
-	ld1		{v5.16b-v7.16b}, [IN], #48
-ST4(	bl		aes_encrypt_block4x		)
-ST5(	bl		aes_encrypt_block5x		)
-	eor		v0.16b, v5.16b, v0.16b
-ST4(	ld1		{v5.16b}, [IN], #16		)
-	eor		v1.16b, v6.16b, v1.16b
-ST5(	ld1		{v5.16b-v6.16b}, [IN], #32	)
-	eor		v2.16b, v7.16b, v2.16b
-	eor		v3.16b, v5.16b, v3.16b
-ST5(	eor		v4.16b, v6.16b, v4.16b		)
-	st1		{v0.16b-v3.16b}, [OUT], #64
-ST5(	st1		{v4.16b}, [OUT], #16		)
-	cbz		BYTES_W, .Lctrout\xctr
-	b		.LctrloopNx\xctr
-
-.Lctrout\xctr:
-	.if !\xctr
-		st1		{vctr.16b}, [IV] /* return next CTR value */
-	.endif
-	frame_pop
-	ret
-
-.Lctrtail\xctr:
-	/*
-	 * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
-	 *
-	 * This code expects the last keystream block to be in v{MAX_STRIDE-1}.
-	 * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
-	 * v4 should have the next two counter blocks.
-	 *
-	 * This allows us to store the ciphertext by writing to overlapping
-	 * regions of memory.  Any invalid ciphertext blocks get overwritten by
-	 * correctly computed blocks.  This approach greatly simplifies the
-	 * logic for storing the ciphertext.
-	 */
-	mov		x16, #16
-	ands		w7, BYTES_W, #0xf
-	csel		x13, x7, x16, ne
-
-ST5(	cmp		BYTES_W, #64 - (MAX_STRIDE << 4))
-ST5(	csel		x14, x16, xzr, gt		)
-	cmp		BYTES_W, #48 - (MAX_STRIDE << 4)
-	csel		x15, x16, xzr, gt
-	cmp		BYTES_W, #32 - (MAX_STRIDE << 4)
-	csel		x16, x16, xzr, gt
-	cmp		BYTES_W, #16 - (MAX_STRIDE << 4)
-
-	adr_l		x9, .Lcts_permute_table
-	add		x9, x9, x13
-	ble		.Lctrtail1x\xctr
-
-ST5(	ld1		{v5.16b}, [IN], x14		)
-	ld1		{v6.16b}, [IN], x15
-	ld1		{v7.16b}, [IN], x16
-
-ST4(	bl		aes_encrypt_block4x		)
-ST5(	bl		aes_encrypt_block5x		)
-
-	ld1		{v8.16b}, [IN], x13
-	ld1		{v9.16b}, [IN]
-	ld1		{v10.16b}, [x9]
-
-ST4(	eor		v6.16b, v6.16b, v0.16b		)
-ST4(	eor		v7.16b, v7.16b, v1.16b		)
-ST4(	tbl		v3.16b, {v3.16b}, v10.16b	)
-ST4(	eor		v8.16b, v8.16b, v2.16b		)
-ST4(	eor		v9.16b, v9.16b, v3.16b		)
-
-ST5(	eor		v5.16b, v5.16b, v0.16b		)
-ST5(	eor		v6.16b, v6.16b, v1.16b		)
-ST5(	tbl		v4.16b, {v4.16b}, v10.16b	)
-ST5(	eor		v7.16b, v7.16b, v2.16b		)
-ST5(	eor		v8.16b, v8.16b, v3.16b		)
-ST5(	eor		v9.16b, v9.16b, v4.16b		)
-
-ST5(	st1		{v5.16b}, [OUT], x14		)
-	st1		{v6.16b}, [OUT], x15
-	st1		{v7.16b}, [OUT], x16
-	add		x13, x13, OUT
-	st1		{v9.16b}, [x13]		// overlapping stores
-	st1		{v8.16b}, [OUT]
-	b		.Lctrout\xctr
-
-.Lctrtail1x\xctr:
-	/*
-	 * Handle <= 16 bytes of plaintext
-	 *
-	 * This code always reads and writes 16 bytes.  To avoid out of bounds
-	 * accesses, XCTR and CTR modes must use a temporary buffer when
-	 * encrypting/decrypting less than 16 bytes.
-	 *
-	 * This code is unusual in that it loads the input and stores the output
-	 * relative to the end of the buffers rather than relative to the start.
-	 * This causes unusual behaviour when encrypting/decrypting less than 16
-	 * bytes; the end of the data is expected to be at the end of the
-	 * temporary buffer rather than the start of the data being at the start
-	 * of the temporary buffer.
-	 */
-	sub		x8, x7, #16
-	csel		x7, x7, x8, eq
-	add		IN, IN, x7
-	add		OUT, OUT, x7
-	ld1		{v5.16b}, [IN]
-	ld1		{v6.16b}, [OUT]
-ST5(	mov		v3.16b, v4.16b			)
-	encrypt_block	v3, ROUNDS_W, KEY, x8, w7
-	ld1		{v10.16b-v11.16b}, [x9]
-	tbl		v3.16b, {v3.16b}, v10.16b
-	sshr		v11.16b, v11.16b, #7
-	eor		v5.16b, v5.16b, v3.16b
-	bif		v5.16b, v6.16b, v11.16b
-	st1		{v5.16b}, [OUT]
-	b		.Lctrout\xctr
-
-	// Arguments
-	.unreq OUT
-	.unreq IN
-	.unreq KEY
-	.unreq ROUNDS_W
-	.unreq BYTES_W
-	.unreq IV
-	.unreq BYTE_CTR_W	// XCTR only
-	// Intermediate values
-	.unreq CTR_W		// XCTR only
-	.unreq CTR		// XCTR only
-	.unreq IV_PART
-	.unreq BLOCKS
-	.unreq BLOCKS_W
-.endm
-
-	/*
-	 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int bytes, u8 ctr[])
-	 *
-	 * The input and output buffers must always be at least 16 bytes even if
-	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
-	 * accesses will occur.  The data to be encrypted/decrypted is expected
-	 * to be at the end of this 16-byte temporary buffer rather than the
-	 * start.
-	 */
-
-AES_FUNC_START(aes_ctr_encrypt)
-	ctr_encrypt 0
-AES_FUNC_END(aes_ctr_encrypt)
-
-	/*
-	 * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-	 *		   int bytes, u8 const iv[], int byte_ctr)
-	 *
-	 * The input and output buffers must always be at least 16 bytes even if
-	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
-	 * accesses will occur.  The data to be encrypted/decrypted is expected
-	 * to be at the end of this 16-byte temporary buffer rather than the
-	 * start.
-	 */
-
-AES_FUNC_START(aes_xctr_encrypt)
-	ctr_encrypt 1
-AES_FUNC_END(aes_xctr_encrypt)
-
-
-	/*
-	 * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
-	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
-	 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
-	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
-	 */
-
-	.macro		next_tweak, out, in, tmp
-	sshr		\tmp\().2d,  \in\().2d,   #63
-	and		\tmp\().16b, \tmp\().16b, xtsmask.16b
-	add		\out\().2d,  \in\().2d,   \in\().2d
-	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
-	eor		\out\().16b, \out\().16b, \tmp\().16b
-	.endm
-
-	.macro		xts_load_mask, tmp
-	movi		xtsmask.2s, #0x1
-	movi		\tmp\().2s, #0x87
-	uzp1		xtsmask.4s, xtsmask.4s, \tmp\().4s
-	.endm
-
-AES_FUNC_START(aes_xts_encrypt)
-	frame_push	0
-
-	ld1		{v4.16b}, [x6]
-	xts_load_mask	v8
-	cbz		w7, .Lxtsencnotfirst
-
-	enc_prepare	w3, x5, x8
-	xts_cts_skip_tw	w7, .LxtsencNx
-	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
-	enc_switch_key	w3, x2, x8
-	b		.LxtsencNx
-
-.Lxtsencnotfirst:
-	enc_prepare	w3, x2, x8
-.LxtsencloopNx:
-	next_tweak	v4, v4, v8
-.LxtsencNx:
-	subs		w4, w4, #64
-	bmi		.Lxtsenc1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
-	next_tweak	v5, v4, v8
-	eor		v0.16b, v0.16b, v4.16b
-	next_tweak	v6, v5, v8
-	eor		v1.16b, v1.16b, v5.16b
-	eor		v2.16b, v2.16b, v6.16b
-	next_tweak	v7, v6, v8
-	eor		v3.16b, v3.16b, v7.16b
-	bl		aes_encrypt_block4x
-	eor		v3.16b, v3.16b, v7.16b
-	eor		v0.16b, v0.16b, v4.16b
-	eor		v1.16b, v1.16b, v5.16b
-	eor		v2.16b, v2.16b, v6.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
-	mov		v4.16b, v7.16b
-	cbz		w4, .Lxtsencret
-	xts_reload_mask	v8
-	b		.LxtsencloopNx
-.Lxtsenc1x:
-	adds		w4, w4, #64
-	beq		.Lxtsencout
-	subs		w4, w4, #16
-	bmi		.LxtsencctsNx
-.Lxtsencloop:
-	ld1		{v0.16b}, [x1], #16
-.Lxtsencctsout:
-	eor		v0.16b, v0.16b, v4.16b
-	encrypt_block	v0, w3, x2, x8, w7
-	eor		v0.16b, v0.16b, v4.16b
-	cbz		w4, .Lxtsencout
-	subs		w4, w4, #16
-	next_tweak	v4, v4, v8
-	bmi		.Lxtsenccts
-	st1		{v0.16b}, [x0], #16
-	b		.Lxtsencloop
-.Lxtsencout:
-	st1		{v0.16b}, [x0]
-.Lxtsencret:
-	st1		{v4.16b}, [x6]
-	frame_pop
-	ret
-
-.LxtsencctsNx:
-	mov		v0.16b, v3.16b
-	sub		x0, x0, #16
-.Lxtsenccts:
-	adr_l		x8, .Lcts_permute_table
-
-	add		x1, x1, w4, sxtw	/* rewind input pointer */
-	add		w4, w4, #16		/* # bytes in final block */
-	add		x9, x8, #32
-	add		x8, x8, x4
-	sub		x9, x9, x4
-	add		x4, x0, x4		/* output address of final block */
-
-	ld1		{v1.16b}, [x1]		/* load final block */
-	ld1		{v2.16b}, [x8]
-	ld1		{v3.16b}, [x9]
-
-	tbl		v2.16b, {v0.16b}, v2.16b
-	tbx		v0.16b, {v1.16b}, v3.16b
-	st1		{v2.16b}, [x4]			/* overlapping stores */
-	mov		w4, wzr
-	b		.Lxtsencctsout
-AES_FUNC_END(aes_xts_encrypt)
-
-AES_FUNC_START(aes_xts_decrypt)
-	frame_push	0
-
-	/* subtract 16 bytes if we are doing CTS */
-	sub		w8, w4, #0x10
-	tst		w4, #0xf
-	csel		w4, w4, w8, eq
-
-	ld1		{v4.16b}, [x6]
-	xts_load_mask	v8
-	xts_cts_skip_tw	w7, .Lxtsdecskiptw
-	cbz		w7, .Lxtsdecnotfirst
-
-	enc_prepare	w3, x5, x8
-	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
-.Lxtsdecskiptw:
-	dec_prepare	w3, x2, x8
-	b		.LxtsdecNx
-
-.Lxtsdecnotfirst:
-	dec_prepare	w3, x2, x8
-.LxtsdecloopNx:
-	next_tweak	v4, v4, v8
-.LxtsdecNx:
-	subs		w4, w4, #64
-	bmi		.Lxtsdec1x
-	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
-	next_tweak	v5, v4, v8
-	eor		v0.16b, v0.16b, v4.16b
-	next_tweak	v6, v5, v8
-	eor		v1.16b, v1.16b, v5.16b
-	eor		v2.16b, v2.16b, v6.16b
-	next_tweak	v7, v6, v8
-	eor		v3.16b, v3.16b, v7.16b
-	bl		aes_decrypt_block4x
-	eor		v3.16b, v3.16b, v7.16b
-	eor		v0.16b, v0.16b, v4.16b
-	eor		v1.16b, v1.16b, v5.16b
-	eor		v2.16b, v2.16b, v6.16b
-	st1		{v0.16b-v3.16b}, [x0], #64
-	mov		v4.16b, v7.16b
-	cbz		w4, .Lxtsdecout
-	xts_reload_mask	v8
-	b		.LxtsdecloopNx
-.Lxtsdec1x:
-	adds		w4, w4, #64
-	beq		.Lxtsdecout
-	subs		w4, w4, #16
-.Lxtsdecloop:
-	ld1		{v0.16b}, [x1], #16
-	bmi		.Lxtsdeccts
-.Lxtsdecctsout:
-	eor		v0.16b, v0.16b, v4.16b
-	decrypt_block	v0, w3, x2, x8, w7
-	eor		v0.16b, v0.16b, v4.16b
-	st1		{v0.16b}, [x0], #16
-	cbz		w4, .Lxtsdecout
-	subs		w4, w4, #16
-	next_tweak	v4, v4, v8
-	b		.Lxtsdecloop
-.Lxtsdecout:
-	st1		{v4.16b}, [x6]
-	frame_pop
-	ret
-
-.Lxtsdeccts:
-	adr_l		x8, .Lcts_permute_table
-
-	add		x1, x1, w4, sxtw	/* rewind input pointer */
-	add		w4, w4, #16		/* # bytes in final block */
-	add		x9, x8, #32
-	add		x8, x8, x4
-	sub		x9, x9, x4
-	add		x4, x0, x4		/* output address of final block */
-
-	next_tweak	v5, v4, v8
-
-	ld1		{v1.16b}, [x1]		/* load final block */
-	ld1		{v2.16b}, [x8]
-	ld1		{v3.16b}, [x9]
-
-	eor		v0.16b, v0.16b, v5.16b
-	decrypt_block	v0, w3, x2, x8, w7
-	eor		v0.16b, v0.16b, v5.16b
-
-	tbl		v2.16b, {v0.16b}, v2.16b
-	tbx		v0.16b, {v1.16b}, v3.16b
-
-	st1		{v2.16b}, [x4]			/* overlapping stores */
-	mov		w4, wzr
-	b		.Lxtsdecctsout
-AES_FUNC_END(aes_xts_decrypt)
-
-	/*
-	 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
-	 *		  int blocks, u8 dg[], int enc_before, int enc_after)
-	 */
-AES_FUNC_START(aes_mac_update)
-	ld1		{v0.16b}, [x4]			/* get dg */
-	enc_prepare	w2, x1, x7
-	cbz		w5, .Lmacloop4x
-
-	encrypt_block	v0, w2, x1, x7, w8
-
-.Lmacloop4x:
-	subs		w3, w3, #4
-	bmi		.Lmac1x
-	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */
-	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
-	encrypt_block	v0, w2, x1, x7, w8
-	eor		v0.16b, v0.16b, v2.16b
-	encrypt_block	v0, w2, x1, x7, w8
-	eor		v0.16b, v0.16b, v3.16b
-	encrypt_block	v0, w2, x1, x7, w8
-	eor		v0.16b, v0.16b, v4.16b
-	cmp		w3, wzr
-	csinv		x5, x6, xzr, eq
-	cbz		w5, .Lmacout
-	encrypt_block	v0, w2, x1, x7, w8
-	st1		{v0.16b}, [x4]			/* return dg */
-	cond_yield	.Lmacout, x7, x8
-	b		.Lmacloop4x
-.Lmac1x:
-	add		w3, w3, #4
-.Lmacloop:
-	cbz		w3, .Lmacout
-	ld1		{v1.16b}, [x0], #16		/* get next pt block */
-	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
-
-	subs		w3, w3, #1
-	csinv		x5, x6, xzr, eq
-	cbz		w5, .Lmacout
-
-.Lmacenc:
-	encrypt_block	v0, w2, x1, x7, w8
-	b		.Lmacloop
-
-.Lmacout:
-	st1		{v0.16b}, [x4]			/* return dg */
-	mov		w0, w3
-	ret
-AES_FUNC_END(aes_mac_update)
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
deleted file mode 100644
index 3a8961b6ea517..0000000000000
--- a/arch/arm64/crypto/aes-neon.S
+++ /dev/null
@@ -1,250 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
- *
- * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
-#define AES_FUNC_START(func)		SYM_FUNC_START(neon_ ## func)
-#define AES_FUNC_END(func)		SYM_FUNC_END(neon_ ## func)
-
-	xtsmask		.req	v7
-	cbciv		.req	v7
-	vctr		.req	v4
-
-	.macro		xts_reload_mask, tmp
-	xts_load_mask	\tmp
-	.endm
-
-	/* special case for the neon-bs driver calling into this one for CTS */
-	.macro		xts_cts_skip_tw, reg, lbl
-	tbnz		\reg, #1, \lbl
-	.endm
-
-	/* multiply by polynomial 'x' in GF(2^8) */
-	.macro		mul_by_x, out, in, temp, const
-	sshr		\temp, \in, #7
-	shl		\out, \in, #1
-	and		\temp, \temp, \const
-	eor		\out, \out, \temp
-	.endm
-
-	/* multiply by polynomial 'x^2' in GF(2^8) */
-	.macro		mul_by_x2, out, in, temp, const
-	ushr		\temp, \in, #6
-	shl		\out, \in, #2
-	pmul		\temp, \temp, \const
-	eor		\out, \out, \temp
-	.endm
-
-	/* preload the entire Sbox */
-	.macro		prepare, sbox, shiftrows, temp
-	movi		v12.16b, #0x1b
-	ldr_l		q13, \shiftrows, \temp
-	ldr_l		q14, .Lror32by8, \temp
-	adr_l		\temp, \sbox
-	ld1		{v16.16b-v19.16b}, [\temp], #64
-	ld1		{v20.16b-v23.16b}, [\temp], #64
-	ld1		{v24.16b-v27.16b}, [\temp], #64
-	ld1		{v28.16b-v31.16b}, [\temp]
-	.endm
-
-	/* do preload for encryption */
-	.macro		enc_prepare, ignore0, ignore1, temp
-	prepare		crypto_aes_sbox, .LForward_ShiftRows, \temp
-	.endm
-
-	.macro		enc_switch_key, ignore0, ignore1, temp
-	/* do nothing */
-	.endm
-
-	/* do preload for decryption */
-	.macro		dec_prepare, ignore0, ignore1, temp
-	prepare		crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
-	.endm
-
-	/* apply SubBytes transformation using the preloaded Sbox */
-	.macro		sub_bytes, in
-	sub		v9.16b, \in\().16b, v15.16b
-	tbl		\in\().16b, {v16.16b-v19.16b}, \in\().16b
-	sub		v10.16b, v9.16b, v15.16b
-	tbx		\in\().16b, {v20.16b-v23.16b}, v9.16b
-	sub		v11.16b, v10.16b, v15.16b
-	tbx		\in\().16b, {v24.16b-v27.16b}, v10.16b
-	tbx		\in\().16b, {v28.16b-v31.16b}, v11.16b
-	.endm
-
-	/* apply MixColumns transformation */
-	.macro		mix_columns, in, enc
-	.if		\enc == 0
-	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
-	mul_by_x2	v8.16b, \in\().16b, v9.16b, v12.16b
-	eor		\in\().16b, \in\().16b, v8.16b
-	rev32		v8.8h, v8.8h
-	eor		\in\().16b, \in\().16b, v8.16b
-	.endif
-
-	mul_by_x	v9.16b, \in\().16b, v8.16b, v12.16b
-	rev32		v8.8h, \in\().8h
-	eor		v8.16b, v8.16b, v9.16b
-	eor		\in\().16b, \in\().16b, v8.16b
-	tbl		\in\().16b, {\in\().16b}, v14.16b
-	eor		\in\().16b, \in\().16b, v8.16b
-	.endm
-
-	.macro		do_block, enc, in, rounds, rk, rkp, i
-	ld1		{v15.4s}, [\rk]
-	add		\rkp, \rk, #16
-	mov		\i, \rounds
-.La\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
-	movi		v15.16b, #0x40
-	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
-	sub_bytes	\in
-	sub		\i, \i, #1
-	ld1		{v15.4s}, [\rkp], #16
-	cbz		\i, .Lb\@
-	mix_columns	\in, \enc
-	b		.La\@
-.Lb\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
-	.endm
-
-	.macro		encrypt_block, in, rounds, rk, rkp, i
-	do_block	1, \in, \rounds, \rk, \rkp, \i
-	.endm
-
-	.macro		decrypt_block, in, rounds, rk, rkp, i
-	do_block	0, \in, \rounds, \rk, \rkp, \i
-	.endm
-
-	/*
-	 * Interleaved versions: functionally equivalent to the
-	 * ones above, but applied to AES states in parallel.
-	 */
-
-	.macro		sub_bytes_4x, in0, in1, in2, in3
-	sub		v8.16b, \in0\().16b, v15.16b
-	tbl		\in0\().16b, {v16.16b-v19.16b}, \in0\().16b
-	sub		v9.16b, \in1\().16b, v15.16b
-	tbl		\in1\().16b, {v16.16b-v19.16b}, \in1\().16b
-	sub		v10.16b, \in2\().16b, v15.16b
-	tbl		\in2\().16b, {v16.16b-v19.16b}, \in2\().16b
-	sub		v11.16b, \in3\().16b, v15.16b
-	tbl		\in3\().16b, {v16.16b-v19.16b}, \in3\().16b
-	tbx		\in0\().16b, {v20.16b-v23.16b}, v8.16b
-	tbx		\in1\().16b, {v20.16b-v23.16b}, v9.16b
-	sub		v8.16b, v8.16b, v15.16b
-	tbx		\in2\().16b, {v20.16b-v23.16b}, v10.16b
-	sub		v9.16b, v9.16b, v15.16b
-	tbx		\in3\().16b, {v20.16b-v23.16b}, v11.16b
-	sub		v10.16b, v10.16b, v15.16b
-	tbx		\in0\().16b, {v24.16b-v27.16b}, v8.16b
-	sub		v11.16b, v11.16b, v15.16b
-	tbx		\in1\().16b, {v24.16b-v27.16b}, v9.16b
-	sub		v8.16b, v8.16b, v15.16b
-	tbx		\in2\().16b, {v24.16b-v27.16b}, v10.16b
-	sub		v9.16b, v9.16b, v15.16b
-	tbx		\in3\().16b, {v24.16b-v27.16b}, v11.16b
-	sub		v10.16b, v10.16b, v15.16b
-	tbx		\in0\().16b, {v28.16b-v31.16b}, v8.16b
-	sub		v11.16b, v11.16b, v15.16b
-	tbx		\in1\().16b, {v28.16b-v31.16b}, v9.16b
-	tbx		\in2\().16b, {v28.16b-v31.16b}, v10.16b
-	tbx		\in3\().16b, {v28.16b-v31.16b}, v11.16b
-	.endm
-
-	.macro		mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
-	sshr		\tmp0\().16b, \in0\().16b, #7
-	shl		\out0\().16b, \in0\().16b, #1
-	sshr		\tmp1\().16b, \in1\().16b, #7
-	and		\tmp0\().16b, \tmp0\().16b, \const\().16b
-	shl		\out1\().16b, \in1\().16b, #1
-	and		\tmp1\().16b, \tmp1\().16b, \const\().16b
-	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
-	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
-	.endm
-
-	.macro		mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
-	ushr		\tmp0\().16b, \in0\().16b, #6
-	shl		\out0\().16b, \in0\().16b, #2
-	ushr		\tmp1\().16b, \in1\().16b, #6
-	pmul		\tmp0\().16b, \tmp0\().16b, \const\().16b
-	shl		\out1\().16b, \in1\().16b, #2
-	pmul		\tmp1\().16b, \tmp1\().16b, \const\().16b
-	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
-	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
-	.endm
-
-	.macro		mix_columns_2x, in0, in1, enc
-	.if		\enc == 0
-	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
-	mul_by_x2_2x	v8, v9, \in0, \in1, v10, v11, v12
-	eor		\in0\().16b, \in0\().16b, v8.16b
-	rev32		v8.8h, v8.8h
-	eor		\in1\().16b, \in1\().16b, v9.16b
-	rev32		v9.8h, v9.8h
-	eor		\in0\().16b, \in0\().16b, v8.16b
-	eor		\in1\().16b, \in1\().16b, v9.16b
-	.endif
-
-	mul_by_x_2x	v8, v9, \in0, \in1, v10, v11, v12
-	rev32		v10.8h, \in0\().8h
-	rev32		v11.8h, \in1\().8h
-	eor		v10.16b, v10.16b, v8.16b
-	eor		v11.16b, v11.16b, v9.16b
-	eor		\in0\().16b, \in0\().16b, v10.16b
-	eor		\in1\().16b, \in1\().16b, v11.16b
-	tbl		\in0\().16b, {\in0\().16b}, v14.16b
-	tbl		\in1\().16b, {\in1\().16b}, v14.16b
-	eor		\in0\().16b, \in0\().16b, v10.16b
-	eor		\in1\().16b, \in1\().16b, v11.16b
-	.endm
-
-	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
-	ld1		{v15.4s}, [\rk]
-	add		\rkp, \rk, #16
-	mov		\i, \rounds
-.La\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
-	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
-	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
-	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
-	movi		v15.16b, #0x40
-	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
-	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
-	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
-	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
-	sub_bytes_4x	\in0, \in1, \in2, \in3
-	sub		\i, \i, #1
-	ld1		{v15.4s}, [\rkp], #16
-	cbz		\i, .Lb\@
-	mix_columns_2x	\in0, \in1, \enc
-	mix_columns_2x	\in2, \in3, \enc
-	b		.La\@
-.Lb\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
-	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
-	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
-	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
-	.endm
-
-	.macro		encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
-	do_block_4x	1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
-	.endm
-
-	.macro		decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
-	do_block_4x	0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
-	.endm
-
-#include "aes-modes.S"
-
-	.section	".rodata", "a"
-	.align		4
-.LForward_ShiftRows:
-	.octa		0x0b06010c07020d08030e09040f0a0500
-
-.LReverse_ShiftRows:
-	.octa		0x0306090c0f0205080b0e0104070a0d00
-
-.Lror32by8:
-	.octa		0x0c0f0e0d080b0a090407060500030201
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
index cb87c8fc66b3b..5bcbac9798931 100644
--- a/arch/arm64/crypto/aes-neonbs-glue.c
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -17,6 +17,7 @@
 
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_DESCRIPTION("Bit sliced AES using NEON instructions");
+MODULE_IMPORT_NS("CRYPTO_INTERNAL");
 MODULE_LICENSE("GPL v2");
 
 MODULE_ALIAS_CRYPTO("ecb(aes)");
@@ -42,20 +43,6 @@ asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
 				  int rounds, int blocks, u8 iv[]);
 
-/* borrowed from aes-neon-blk.ko */
-asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				     int rounds, int blocks);
-asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				     int rounds, int blocks, u8 iv[]);
-asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
-				     int rounds, int bytes, u8 ctr[]);
-asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[],
-				     u32 const rk1[], int rounds, int bytes,
-				     u32 const rk2[], u8 iv[], int first);
-asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[],
-				     u32 const rk1[], int rounds, int bytes,
-				     u32 const rk2[], u8 iv[], int first);
-
 struct aesbs_ctx {
 	u8	rk[13 * (8 * AES_BLOCK_SIZE) + 32];
 	int	rounds;
@@ -76,19 +63,24 @@ static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			unsigned int key_len)
 {
 	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct crypto_aes_ctx rk;
+	struct crypto_aes_ctx *rk;
 	int err;
 
-	err = aes_expandkey(&rk, in_key, key_len);
+	rk = kmalloc(sizeof(*rk), GFP_KERNEL);
+	if (!rk)
+		return -ENOMEM;
+
+	err = aes_expandkey(rk, in_key, key_len);
 	if (err)
-		return err;
+		goto out;
 
 	ctx->rounds = 6 + key_len / 4;
 
 	scoped_ksimd()
-		aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
-
-	return 0;
+		aesbs_convert_key(ctx->rk, rk->key_enc, ctx->rounds);
+out:
+	kfree_sensitive(rk);
+	return err;
 }
 
 static int __ecb_crypt(struct skcipher_request *req,
@@ -133,22 +125,26 @@ static int aesbs_cbc_ctr_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			    unsigned int key_len)
 {
 	struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct crypto_aes_ctx rk;
+	struct crypto_aes_ctx *rk;
 	int err;
 
-	err = aes_expandkey(&rk, in_key, key_len);
+	rk = kmalloc(sizeof(*rk), GFP_KERNEL);
+	if (!rk)
+		return -ENOMEM;
+
+	err = aes_expandkey(rk, in_key, key_len);
 	if (err)
-		return err;
+		goto out;
 
 	ctx->key.rounds = 6 + key_len / 4;
 
-	memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
+	memcpy(ctx->enc, rk->key_enc, sizeof(ctx->enc));
 
 	scoped_ksimd()
-		aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
-	memzero_explicit(&rk, sizeof(rk));
-
-	return 0;
+		aesbs_convert_key(ctx->key.rk, rk->key_enc, ctx->key.rounds);
+out:
+	kfree_sensitive(rk);
+	return err;
 }
 
 static int cbc_encrypt(struct skcipher_request *req)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index d3d46e5f71884..cdbaad41bddbf 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -371,14 +371,13 @@ alternative_endif
  * [start, end) with dcache line size explicitly provided.
  *
  * 	op:		operation passed to dc instruction
- * 	domain:		domain used in dsb instruction
  * 	start:          starting virtual address of the region
  * 	end:            end virtual address of the region
  *	linesz:		dcache line size
  * 	fixup:		optional label to branch to on user fault
  * 	Corrupts:       start, end, tmp
  */
-	.macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup
+	.macro dcache_by_myline_op_nosync op, start, end, linesz, tmp, fixup
 	sub	\tmp, \linesz, #1
 	bic	\start, \start, \tmp
 alternative_if ARM64_WORKAROUND_4311569
@@ -412,14 +411,28 @@ alternative_if ARM64_WORKAROUND_4311569
 	cbnz	\start, .Ldcache_op\@
 	.endif
 alternative_else_nop_endif
-	dsb	\domain
 
 	_cond_uaccess_extable .Ldcache_op\@, \fixup
 	.endm
 
 /*
  * Macro to perform a data cache maintenance for the interval
- * [start, end)
+ * [start, end) without waiting for completion
+ *
+ * 	op:		operation passed to dc instruction
+ * 	start:          starting virtual address of the region
+ * 	end:            end virtual address of the region
+ * 	fixup:		optional label to branch to on user fault
+ * 	Corrupts:       start, end, tmp1, tmp2
+ */
+	.macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup
+	dcache_line_size \tmp1, \tmp2
+	dcache_by_myline_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup
+	.endm
+
+/*
+ * Macro to perform a data cache maintenance for the interval
+ * [start, end) and wait for completion
  *
  * 	op:		operation passed to dc instruction
  * 	domain:		domain used in dsb instruction
@@ -429,8 +442,8 @@ alternative_else_nop_endif
  * 	Corrupts:       start, end, tmp1, tmp2
  */
 	.macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
-	dcache_line_size \tmp1, \tmp2
-	dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup
+	dcache_by_line_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup
+	dsb \domain
 	.endm
 
 /*
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index dd2c8586a725e..10a7ffadee3da 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -87,6 +87,11 @@ int cache_line_size(void);
 
 #define dma_get_cache_alignment	cache_line_size
 
+static inline void arch_sync_dma_flush(void)
+{
+	dsb(sy);
+}
+
 /* Compress a u64 MPIDR value into 32 bits. */
 static inline u64 arch_compact_of_hwid(u64 id)
 {
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 28ab96e808eff..382b4ac3734dc 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -74,6 +74,8 @@ extern void icache_inval_pou(unsigned long start, unsigned long end);
 extern void dcache_clean_inval_poc(unsigned long start, unsigned long end);
 extern void dcache_inval_poc(unsigned long start, unsigned long end);
 extern void dcache_clean_poc(unsigned long start, unsigned long end);
+extern void dcache_inval_poc_nosync(unsigned long start, unsigned long end);
+extern void dcache_clean_poc_nosync(unsigned long start, unsigned long end);
 extern void dcache_clean_pop(unsigned long start, unsigned long end);
 extern void dcache_clean_pou(unsigned long start, unsigned long end);
 extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end);
diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index 85f4c1615472d..998b2a3f615a7 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -248,6 +248,8 @@
 		     ICH_HFGWTR_EL2_ICC_CR0_EL1			| \
 		     ICH_HFGWTR_EL2_ICC_APR_EL1)
 	msr_s	SYS_ICH_HFGWTR_EL2, x0		// Disable reg write traps
+	mov	x0, #(ICH_VCTLR_EL2_En)
+	msr_s	SYS_ICH_VCTLR_EL2, x0		// Enable vHPPI selection
 .Lskip_gicv5_\@:
 .endm
 
diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
index e6f8ff3cc6306..d038ff14d16ca 100644
--- a/arch/arm64/include/asm/hugetlb.h
+++ b/arch/arm64/include/asm/hugetlb.h
@@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma,
 					     unsigned long start,
 					     unsigned long end,
 					     unsigned long stride,
-					     bool last_level)
+					     tlbf_t flags)
 {
 	switch (stride) {
 #ifndef __PAGETABLE_PMD_FOLDED
 	case PUD_SIZE:
-		__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);
+		__flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);
 		break;
 #endif
 	case CONT_PMD_SIZE:
 	case PMD_SIZE:
-		__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);
+		__flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);
 		break;
 	case CONT_PTE_SIZE:
-		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);
+		__flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);
 		break;
 	default:
-		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);
+		__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);
 	}
 }
 
@@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
 {
 	unsigned long stride = huge_page_size(hstate_vma(vma));
 
-	__flush_hugetlb_tlb_range(vma, start, end, stride, false);
+	__flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);
 }
 
 #endif /* __ASM_HUGETLB_H */
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index a1ad12c72ebf1..724319298e71f 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -81,6 +81,8 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
 	__KVM_HOST_SMCCC_FUNC___pkvm_reserve_vm,
 	__KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm,
 	__KVM_HOST_SMCCC_FUNC___pkvm_init_vm,
@@ -89,6 +91,14 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load,
 	__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put,
 	__KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid,
+	__KVM_HOST_SMCCC_FUNC___tracing_load,
+	__KVM_HOST_SMCCC_FUNC___tracing_unload,
+	__KVM_HOST_SMCCC_FUNC___tracing_enable,
+	__KVM_HOST_SMCCC_FUNC___tracing_swap_reader,
+	__KVM_HOST_SMCCC_FUNC___tracing_update_clock,
+	__KVM_HOST_SMCCC_FUNC___tracing_reset,
+	__KVM_HOST_SMCCC_FUNC___tracing_enable_event,
+	__KVM_HOST_SMCCC_FUNC___tracing_write_event,
 };
 
 #define DECLARE_KVM_VHE_SYM(sym)	extern char sym[]
diff --git a/arch/arm64/include/asm/kvm_define_hypevents.h b/arch/arm64/include/asm/kvm_define_hypevents.h
new file mode 100644
index 0000000000000..77d6790252a67
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_define_hypevents.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#define REMOTE_EVENT_INCLUDE_FILE arch/arm64/include/asm/kvm_hypevents.h
+
+#define REMOTE_EVENT_SECTION "_hyp_events"
+
+#define HE_STRUCT(__args)		__args
+#define HE_PRINTK(__args...)		__args
+#define he_field			re_field
+
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
+	REMOTE_EVENT(__name, 0, RE_STRUCT(__struct), RE_PRINTK(__printk))
+
+#define HYP_EVENT_MULTI_READ
+#include <trace/define_remote_events.h>
+#undef HYP_EVENT_MULTI_READ
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 70cb9cfd760a3..7298a68eaef9a 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -287,6 +287,9 @@ enum fgt_group_id {
 	HDFGRTR2_GROUP,
 	HDFGWTR2_GROUP = HDFGRTR2_GROUP,
 	HFGITR2_GROUP,
+	ICH_HFGRTR_GROUP,
+	ICH_HFGWTR_GROUP = ICH_HFGRTR_GROUP,
+	ICH_HFGITR_GROUP,
 
 	/* Must be last */
 	__NR_FGT_GROUP_IDS__
@@ -620,6 +623,10 @@ enum vcpu_sysreg {
 	VNCR(ICH_HCR_EL2),
 	VNCR(ICH_VMCR_EL2),
 
+	VNCR(ICH_HFGRTR_EL2),
+	VNCR(ICH_HFGWTR_EL2),
+	VNCR(ICH_HFGITR_EL2),
+
 	NR_SYS_REGS	/* Nothing after this line! */
 };
 
@@ -675,6 +682,9 @@ extern struct fgt_masks hfgwtr2_masks;
 extern struct fgt_masks hfgitr2_masks;
 extern struct fgt_masks hdfgrtr2_masks;
 extern struct fgt_masks hdfgwtr2_masks;
+extern struct fgt_masks ich_hfgrtr_masks;
+extern struct fgt_masks ich_hfgwtr_masks;
+extern struct fgt_masks ich_hfgitr_masks;
 
 extern struct fgt_masks kvm_nvhe_sym(hfgrtr_masks);
 extern struct fgt_masks kvm_nvhe_sym(hfgwtr_masks);
@@ -687,6 +697,9 @@ extern struct fgt_masks kvm_nvhe_sym(hfgwtr2_masks);
 extern struct fgt_masks kvm_nvhe_sym(hfgitr2_masks);
 extern struct fgt_masks kvm_nvhe_sym(hdfgrtr2_masks);
 extern struct fgt_masks kvm_nvhe_sym(hdfgwtr2_masks);
+extern struct fgt_masks kvm_nvhe_sym(ich_hfgrtr_masks);
+extern struct fgt_masks kvm_nvhe_sym(ich_hfgwtr_masks);
+extern struct fgt_masks kvm_nvhe_sym(ich_hfgitr_masks);
 
 struct kvm_cpu_context {
 	struct user_pt_regs regs;	/* sp = sp_el0 */
@@ -787,6 +800,21 @@ struct kvm_host_data {
 
 	/* Last vgic_irq part of the AP list recorded in an LR */
 	struct vgic_irq *last_lr_irq;
+
+	/* PPI state tracking for GICv5-based guests */
+	struct {
+		/*
+		 * For tracking the PPI pending state, we need both the entry
+		 * state and exit state to correctly detect edges as it is
+		 * possible that an interrupt has been injected in software in
+		 * the interim.
+		 */
+		DECLARE_BITMAP(pendr_entry, VGIC_V5_NR_PRIVATE_IRQS);
+		DECLARE_BITMAP(pendr_exit, VGIC_V5_NR_PRIVATE_IRQS);
+
+		/* The saved state of the regs when leaving the guest */
+		DECLARE_BITMAP(activer_exit, VGIC_V5_NR_PRIVATE_IRQS);
+	} vgic_v5_ppi_state;
 };
 
 struct kvm_host_psci_config {
@@ -923,6 +951,9 @@ struct kvm_vcpu_arch {
 
 	/* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */
 	struct vncr_tlb	*vncr_tlb;
+
+	/* Hyp-readable copy of kvm_vcpu::pid */
+	pid_t pid;
 };
 
 /*
@@ -1659,6 +1690,11 @@ static __always_inline enum fgt_group_id __fgt_reg_to_group_id(enum vcpu_sysreg
 	case HDFGRTR2_EL2:
 	case HDFGWTR2_EL2:
 		return HDFGRTR2_GROUP;
+	case ICH_HFGRTR_EL2:
+	case ICH_HFGWTR_EL2:
+		return ICH_HFGRTR_GROUP;
+	case ICH_HFGITR_EL2:
+		return ICH_HFGITR_GROUP;
 	default:
 		BUILD_BUG_ON(1);
 	}
@@ -1673,6 +1709,7 @@ static __always_inline enum fgt_group_id __fgt_reg_to_group_id(enum vcpu_sysreg
 		case HDFGWTR_EL2:					\
 		case HFGWTR2_EL2:					\
 		case HDFGWTR2_EL2:					\
+		case ICH_HFGWTR_EL2:					\
 			p = &(vcpu)->arch.fgt[id].w;			\
 			break;						\
 		default:						\
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 76ce2b94bd97e..8d06b62e7188c 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -87,6 +87,15 @@ void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if);
 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if);
 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
 
+/* GICv5 */
+void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if);
+/* No hypercalls for the following */
+void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if);
+
 #ifdef __KVM_NVHE_HYPERVISOR__
 void __timer_enable_traps(struct kvm_vcpu *vcpu);
 void __timer_disable_traps(struct kvm_vcpu *vcpu);
@@ -129,13 +138,13 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
 #ifdef __KVM_NVHE_HYPERVISOR__
 void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
 		void (*fn)(void));
-int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
-		unsigned long *per_cpu_base, u32 hyp_va_bits);
+int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_base, u32 hyp_va_bits);
 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
 #endif
 
 extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val);
 extern u64 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val);
+extern u64 kvm_nvhe_sym(id_aa64pfr2_el1_sys_val);
 extern u64 kvm_nvhe_sym(id_aa64isar0_el1_sys_val);
 extern u64 kvm_nvhe_sym(id_aa64isar1_el1_sys_val);
 extern u64 kvm_nvhe_sym(id_aa64isar2_el1_sys_val);
@@ -147,5 +156,6 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val);
 extern unsigned long kvm_nvhe_sym(__icache_flags);
 extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
 extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
+extern unsigned long kvm_nvhe_sym(hyp_nr_cpus);
 
 #endif /* __ARM64_KVM_HYP_H__ */
diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h
new file mode 100644
index 0000000000000..743c49bd878f7
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_hypevents.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#if !defined(__ARM64_KVM_HYPEVENTS_H_) || defined(HYP_EVENT_MULTI_READ)
+#define __ARM64_KVM_HYPEVENTS_H_
+
+#ifdef __KVM_NVHE_HYPERVISOR__
+#include <nvhe/trace.h>
+#endif
+
+#ifndef __HYP_ENTER_EXIT_REASON
+#define __HYP_ENTER_EXIT_REASON
+enum hyp_enter_exit_reason {
+	HYP_REASON_SMC,
+	HYP_REASON_HVC,
+	HYP_REASON_PSCI,
+	HYP_REASON_HOST_ABORT,
+	HYP_REASON_GUEST_EXIT,
+	HYP_REASON_ERET_HOST,
+	HYP_REASON_ERET_GUEST,
+	HYP_REASON_UNKNOWN	/* Must be last */
+};
+#endif
+
+HYP_EVENT(hyp_enter,
+	HE_PROTO(struct kvm_cpu_context *host_ctxt, u8 reason),
+	HE_STRUCT(
+		he_field(u8, reason)
+		he_field(pid_t, vcpu)
+	),
+	HE_ASSIGN(
+		__entry->reason = reason;
+		__entry->vcpu = __tracing_get_vcpu_pid(host_ctxt);
+	),
+	HE_PRINTK("reason=%s vcpu=%d", __hyp_enter_exit_reason_str(__entry->reason), __entry->vcpu)
+);
+
+HYP_EVENT(hyp_exit,
+	HE_PROTO(struct kvm_cpu_context *host_ctxt, u8 reason),
+	HE_STRUCT(
+		he_field(u8, reason)
+		he_field(pid_t, vcpu)
+	),
+	HE_ASSIGN(
+		__entry->reason = reason;
+		__entry->vcpu = __tracing_get_vcpu_pid(host_ctxt);
+	),
+	HE_PRINTK("reason=%s vcpu=%d", __hyp_enter_exit_reason_str(__entry->reason), __entry->vcpu)
+);
+
+HYP_EVENT(selftest,
+	HE_PROTO(u64 id),
+	HE_STRUCT(
+		he_field(u64, id)
+	),
+	HE_ASSIGN(
+		__entry->id = id;
+	),
+	RE_PRINTK("id=%llu", __entry->id)
+);
+#endif
diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h
new file mode 100644
index 0000000000000..de133b735f72b
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_hyptrace.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ARM64_KVM_HYPTRACE_H_
+#define __ARM64_KVM_HYPTRACE_H_
+
+#include <linux/ring_buffer.h>
+
+struct hyp_trace_desc {
+	unsigned long			bpages_backing_start;
+	size_t				bpages_backing_size;
+	struct trace_buffer_desc	trace_buffer_desc;
+
+};
+
+struct hyp_event_id {
+	unsigned short	id;
+	atomic_t	enabled;
+};
+
+extern struct remote_event __hyp_events_start[];
+extern struct remote_event __hyp_events_end[];
+
+/* hyp_event section used by the hypervisor */
+extern struct hyp_event_id __hyp_event_ids_start[];
+extern struct hyp_event_id __hyp_event_ids_end[];
+
+#endif
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index b39cc1127e1f1..e25d0d18f6d77 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -46,7 +46,12 @@ int pfn_is_map_memory(unsigned long pfn);
 
 #endif /* !__ASSEMBLER__ */
 
-#define VM_DATA_DEFAULT_FLAGS	(VM_DATA_FLAGS_TSK_EXEC | VM_MTE_ALLOWED)
+#ifdef CONFIG_ARM64_MTE
+#define VMA_DATA_DEFAULT_FLAGS	append_vma_flags(VMA_DATA_FLAGS_TSK_EXEC, \
+						 VMA_MTE_ALLOWED_BIT)
+#else
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
+#endif
 
 #include <asm-generic/getorder.h>
 
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index b3e58735c49bd..18adbb391335d 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -89,9 +89,9 @@ static inline void arch_leave_lazy_mmu_mode(void)
 
 /* Set stride and tlb_level in flush_*_tlb_range */
 #define flush_pmd_tlb_range(vma, addr, end)	\
-	__flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
+	__flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)
 #define flush_pud_tlb_range(vma, addr, end)	\
-	__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
+	__flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 /*
@@ -101,17 +101,11 @@ static inline void arch_leave_lazy_mmu_mode(void)
  * entries exist.
  */
 #define flush_tlb_fix_spurious_fault(vma, address, ptep)	\
-	local_flush_tlb_page_nonotify(vma, address)
+	__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
 
-#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)	\
-	local_flush_tlb_page_nonotify(vma, address)
-
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr)	phys_to_page(__pa_symbol(empty_zero_page))
+#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)			\
+	__flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2,	\
+			  TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE)
 
 #define pte_ERROR(e)	\
 	pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
@@ -1263,17 +1257,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
 #endif
 
 #ifdef CONFIG_PAGE_TABLE_CHECK
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
 }
 
-static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
+static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
 {
 	return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
 }
 
-static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
+static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
 {
 	return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
 }
@@ -1320,7 +1314,7 @@ static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
 		 * context-switch, which provides a DSB to complete the TLB
 		 * invalidation.
 		 */
-		flush_tlb_page_nosync(vma, address);
+		__flush_tlb_page(vma, address, TLBF_NOSYNC);
 	}
 
 	return young;
@@ -1819,16 +1813,22 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 	return __ptep_get_and_clear(mm, addr, ptep);
 }
 
+#define test_and_clear_young_ptes test_and_clear_young_ptes
+static inline int test_and_clear_young_ptes(struct vm_area_struct *vma,
+					    unsigned long addr, pte_t *ptep,
+					    unsigned int nr)
+{
+	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+		return __ptep_test_and_clear_young(vma, addr, ptep);
+
+	return contpte_test_and_clear_young_ptes(vma, addr, ptep, nr);
+}
+
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
 				unsigned long addr, pte_t *ptep)
 {
-	pte_t orig_pte = __ptep_get(ptep);
-
-	if (likely(!pte_valid_cont(orig_pte)))
-		return __ptep_test_and_clear_young(vma, addr, ptep);
-
-	return contpte_test_and_clear_young_ptes(vma, addr, ptep, 1);
+	return test_and_clear_young_ptes(vma, addr, ptep, 1);
 }
 
 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
diff --git a/arch/arm64/include/asm/pkeys.h b/arch/arm64/include/asm/pkeys.h
index 0ca5f83ce148f..cd286d3a9c94f 100644
--- a/arch/arm64/include/asm/pkeys.h
+++ b/arch/arm64/include/asm/pkeys.h
@@ -12,8 +12,7 @@
 
 #define arch_max_pkey() 8
 
-int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-		unsigned long init_val);
+int arch_set_user_pkey_access(int pkey, unsigned long init_val);
 
 static inline bool arch_pkeys_enabled(void)
 {
diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h
index fc0fb42b0b641..0f3a01d30f661 100644
--- a/arch/arm64/include/asm/rwonce.h
+++ b/arch/arm64/include/asm/rwonce.h
@@ -19,6 +19,17 @@
 		"ldapr"	#sfx "\t" #regs,				\
 	ARM64_HAS_LDAPR)
 
+/*
+ * Replace this with typeof_unqual() when minimum compiler versions are
+ * increased to GCC 14 and Clang 19. For the time being, we need this
+ * workaround, which relies on function return values dropping qualifiers.
+ */
+#define __rwonce_typeof_unqual(x) typeof(({				\
+	__diag_push()							\
+	__diag_ignore_all("-Wignored-qualifiers", "")			\
+	((typeof(x)(*)(void))0)();					\
+	__diag_pop() }))
+
 /*
  * When building with LTO, there is an increased risk of the compiler
  * converting an address dependency headed by a READ_ONCE() invocation
@@ -31,9 +42,12 @@
  */
 #define __READ_ONCE(x)							\
 ({									\
-	typeof(&(x)) __x = &(x);					\
-	int atomic = 1;							\
-	union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u;	\
+	auto __x = &(x);						\
+	auto __ret = (__rwonce_typeof_unqual(*__x) *)__x;		\
+	/* Hides alias reassignment from Clang's -Wthread-safety. */	\
+	auto __retp = &__ret;						\
+	union { typeof(*__ret) __val; char __c[1]; } __u;		\
+	*__retp = &__u.__val;						\
 	switch (sizeof(x)) {						\
 	case 1:								\
 		asm volatile(__LOAD_RCPC(b, %w0, %1)			\
@@ -56,9 +70,9 @@
 			: "Q" (*__x) : "memory");			\
 		break;							\
 	default:							\
-		atomic = 0;						\
+		__u.__val = *(volatile typeof(*__x) *)__x;		\
 	}								\
-	atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\
+	*__ret;								\
 })
 
 #endif	/* !BUILD_VDSO */
diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 0fbc2e7867d3e..a15a2968e7b66 100644
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -10,6 +10,11 @@
 #ifdef CONFIG_SHADOW_CALL_STACK
 	scs_sp	.req	x18
 
+	.macro scs_load_current_base
+	get_current_task scs_sp
+	ldr	scs_sp, [scs_sp, #TSK_TI_SCS_BASE]
+	.endm
+
 	.macro scs_load_current
 	get_current_task scs_sp
 	ldr	scs_sp, [scs_sp, #TSK_TI_SCS_SP]
@@ -19,6 +24,9 @@
 	str	scs_sp, [\tsk, #TSK_TI_SCS_SP]
 	.endm
 #else
+	.macro scs_load_current_base
+	.endm
+
 	.macro scs_load_current
 	.endm
 
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f4436ecc630cd..938cdb248f83c 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -1052,6 +1052,7 @@
 #define GICV5_OP_GIC_CDPRI		sys_insn(1, 0, 12, 1, 2)
 #define GICV5_OP_GIC_CDRCFG		sys_insn(1, 0, 12, 1, 5)
 #define GICV5_OP_GICR_CDIA		sys_insn(1, 0, 12, 3, 0)
+#define GICV5_OP_GICR_CDNMIA		sys_insn(1, 0, 12, 3, 1)
 
 /* Definitions for GIC CDAFF */
 #define GICV5_GIC_CDAFF_IAFFID_MASK	GENMASK_ULL(47, 32)
@@ -1098,6 +1099,12 @@
 #define GICV5_GIC_CDIA_TYPE_MASK	GENMASK_ULL(31, 29)
 #define GICV5_GIC_CDIA_ID_MASK		GENMASK_ULL(23, 0)
 
+/* Definitions for GICR CDNMIA */
+#define GICV5_GICR_CDNMIA_VALID_MASK	BIT_ULL(32)
+#define GICV5_GICR_CDNMIA_VALID(r)	FIELD_GET(GICV5_GICR_CDNMIA_VALID_MASK, r)
+#define GICV5_GICR_CDNMIA_TYPE_MASK	GENMASK_ULL(31, 29)
+#define GICV5_GICR_CDNMIA_ID_MASK	GENMASK_ULL(23, 0)
+
 #define gicr_insn(insn)			read_sysreg_s(GICV5_OP_GICR_##insn)
 #define gic_insn(v, insn)		write_sysreg_s(v, GICV5_OP_GIC_##insn)
 
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 8d762607285cc..10869d7731b83 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -53,7 +53,7 @@ static inline int tlb_get_level(struct mmu_gather *tlb)
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
 	struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
-	bool last_level = !tlb->freed_tables;
+	tlbf_t flags = tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE;
 	unsigned long stride = tlb_get_unmap_size(tlb);
 	int tlb_level = tlb_get_level(tlb);
 
@@ -63,13 +63,13 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 	 * reallocate our ASID without invalidating the entire TLB.
 	 */
 	if (tlb->fullmm) {
-		if (!last_level)
+		if (tlb->freed_tables)
 			flush_tlb_mm(tlb->mm);
 		return;
 	}
 
 	__flush_tlb_range(&vma, tlb->start, tlb->end, stride,
-			  last_level, tlb_level);
+			  tlb_level, flags);
 }
 
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 1416e652612b7..47fa4d39a4619 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -97,24 +97,69 @@ static inline unsigned long get_trans_granule(void)
 
 #define TLBI_TTL_UNKNOWN	INT_MAX
 
-#define __tlbi_level(op, addr, level) do {				\
-	u64 arg = addr;							\
-									\
-	if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) &&	\
-	    level >= 0 && level <= 3) {					\
-		u64 ttl = level & 3;					\
-		ttl |= get_trans_granule() << 2;			\
-		arg &= ~TLBI_TTL_MASK;					\
-		arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);			\
-	}								\
-									\
-	__tlbi(op, arg);						\
-} while(0)
-
-#define __tlbi_user_level(op, arg, level) do {				\
-	if (arm64_kernel_unmapped_at_el0())				\
-		__tlbi_level(op, (arg | USER_ASID_FLAG), level);	\
-} while (0)
+typedef void (*tlbi_op)(u64 arg);
+
+static __always_inline void vae1is(u64 arg)
+{
+	__tlbi(vae1is, arg);
+	__tlbi_user(vae1is, arg);
+}
+
+static __always_inline void vae2is(u64 arg)
+{
+	__tlbi(vae2is, arg);
+}
+
+static __always_inline void vale1(u64 arg)
+{
+	__tlbi(vale1, arg);
+	__tlbi_user(vale1, arg);
+}
+
+static __always_inline void vale1is(u64 arg)
+{
+	__tlbi(vale1is, arg);
+	__tlbi_user(vale1is, arg);
+}
+
+static __always_inline void vale2is(u64 arg)
+{
+	__tlbi(vale2is, arg);
+}
+
+static __always_inline void vaale1is(u64 arg)
+{
+	__tlbi(vaale1is, arg);
+}
+
+static __always_inline void ipas2e1(u64 arg)
+{
+	__tlbi(ipas2e1, arg);
+}
+
+static __always_inline void ipas2e1is(u64 arg)
+{
+	__tlbi(ipas2e1is, arg);
+}
+
+static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level,
+					      u16 asid)
+{
+	u64 arg = __TLBI_VADDR(addr, asid);
+
+	if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) {
+		u64 ttl = level | (get_trans_granule() << 2);
+
+		FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl);
+	}
+
+	op(arg);
+}
+
+static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level)
+{
+	__tlbi_level_asid(op, addr, level, 0);
+}
 
 /*
  * This macro creates a properly formatted VA operand for the TLB RANGE. The
@@ -141,19 +186,6 @@ static inline unsigned long get_trans_granule(void)
 #define TLBIR_TTL_MASK		GENMASK_ULL(38, 37)
 #define TLBIR_BADDR_MASK	GENMASK_ULL(36,  0)
 
-#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl)		\
-	({								\
-		unsigned long __ta = 0;					\
-		unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0;	\
-		__ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr);		\
-		__ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl);		\
-		__ta |= FIELD_PREP(TLBIR_NUM_MASK, num);		\
-		__ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale);		\
-		__ta |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule());	\
-		__ta |= FIELD_PREP(TLBIR_ASID_MASK, asid);		\
-		__ta;							\
-	})
-
 /* These macros are used by the TLBI RANGE feature. */
 #define __TLBI_RANGE_PAGES(num, scale)	\
 	((unsigned long)((num) + 1) << (5 * (scale) + 1))
@@ -167,11 +199,7 @@ static inline unsigned long get_trans_granule(void)
  * range.
  */
 #define __TLBI_RANGE_NUM(pages, scale)					\
-	({								\
-		int __pages = min((pages),				\
-				  __TLBI_RANGE_PAGES(31, (scale)));	\
-		(__pages >> (5 * (scale) + 1)) - 1;			\
-	})
+	(((pages) >> (5 * (scale) + 1)) - 1)
 
 #define __repeat_tlbi_sync(op, arg...)						\
 do {										\
@@ -241,10 +269,7 @@ static inline void __tlbi_sync_s1ish_hyp(void)
  *		unmapping pages from vmalloc/io space.
  *
  *	flush_tlb_page(vma, addr)
- *		Invalidate a single user mapping for address 'addr' in the
- *		address space corresponding to 'vma->mm'.  Note that this
- *		operation only invalidates a single, last-level page-table
- *		entry and therefore does not affect any walk-caches.
+ *		Equivalent to __flush_tlb_page(..., flags=TLBF_NONE)
  *
  *
  *	Next, we have some undocumented invalidation routines that you probably
@@ -258,30 +283,28 @@ static inline void __tlbi_sync_s1ish_hyp(void)
  *		CPUs, ensuring that any walk-cache entries associated with the
  *		translation are also invalidated.
  *
- *	__flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
+ *	__flush_tlb_range(vma, start, end, stride, tlb_level, flags)
  *		Invalidate the virtual-address range '[start, end)' on all
  *		CPUs for the user address space corresponding to 'vma->mm'.
  *		The invalidation operations are issued at a granularity
- *		determined by 'stride' and only affect any walk-cache entries
- *		if 'last_level' is equal to false. tlb_level is the level at
+ *		determined by 'stride'. tlb_level is the level at
  *		which the invalidation must take place. If the level is wrong,
  *		no invalidation may take place. In the case where the level
  *		cannot be easily determined, the value TLBI_TTL_UNKNOWN will
- *		perform a non-hinted invalidation.
+ *		perform a non-hinted invalidation. flags may be TLBF_NONE (0) or
+ *		any combination of TLBF_NOWALKCACHE (elide eviction of walk
+ *		cache entries), TLBF_NONOTIFY (don't call mmu notifiers),
+ *		TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
+ *		(only perform the invalidation for the local cpu).
  *
- *	local_flush_tlb_page(vma, addr)
- *		Local variant of flush_tlb_page().  Stale TLB entries may
- *		remain in remote CPUs.
- *
- *	local_flush_tlb_page_nonotify(vma, addr)
- *		Same as local_flush_tlb_page() except MMU notifier will not be
- *		called.
- *
- *	local_flush_tlb_contpte(vma, addr)
- *		Invalidate the virtual-address range
- *		'[addr, addr+CONT_PTE_SIZE)' mapped with contpte on local CPU
- *		for the user address space corresponding to 'vma->mm'.  Stale
- *		TLB entries may remain in remote CPUs.
+ *	__flush_tlb_page(vma, addr, flags)
+ *		Invalidate a single user mapping for address 'addr' in the
+ *		address space corresponding to 'vma->mm'.  Note that this
+ *		operation only invalidates a single level 3 page-table entry
+ *		and therefore does not affect any walk-caches. flags may contain
+ *		any combination of TLBF_NONOTIFY (don't call mmu notifiers),
+ *		TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
+ *		(only perform the invalidation for the local cpu).
  *
  *	Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
  *	on top of these routines, since that is our interface to the mmu_gather
@@ -315,59 +338,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 	mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
 }
 
-static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,
-							  unsigned long uaddr)
-{
-	unsigned long addr;
-
-	dsb(nshst);
-	addr = __TLBI_VADDR(uaddr, ASID(mm));
-	__tlbi(vale1, addr);
-	__tlbi_user(vale1, addr);
-}
-
-static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,
-						 unsigned long uaddr)
-{
-	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
-	dsb(nsh);
-}
-
-static inline void local_flush_tlb_page(struct vm_area_struct *vma,
-					unsigned long uaddr)
-{
-	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
-	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,
-						(uaddr & PAGE_MASK) + PAGE_SIZE);
-	dsb(nsh);
-}
-
-static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
-					   unsigned long uaddr)
-{
-	unsigned long addr;
-
-	dsb(ishst);
-	addr = __TLBI_VADDR(uaddr, ASID(mm));
-	__tlbi(vale1is, addr);
-	__tlbi_user(vale1is, addr);
-	mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
-						(uaddr & PAGE_MASK) + PAGE_SIZE);
-}
-
-static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
-					 unsigned long uaddr)
-{
-	return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
-}
-
-static inline void flush_tlb_page(struct vm_area_struct *vma,
-				  unsigned long uaddr)
-{
-	flush_tlb_page_nosync(vma, uaddr);
-	__tlbi_sync_s1ish();
-}
-
 static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
 {
 	return true;
@@ -397,14 +367,13 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
 /*
  * __flush_tlb_range_op - Perform TLBI operation upon a range
  *
- * @op:	TLBI instruction that operates on a range (has 'r' prefix)
+ * @lop:	TLBI level operation to perform
+ * @rop:	TLBI range operation to perform
  * @start:	The start address of the range
  * @pages:	Range as the number of pages from 'start'
  * @stride:	Flush granularity
  * @asid:	The ASID of the task (0 for IPA instructions)
- * @tlb_level:	Translation Table level hint, if known
- * @tlbi_user:	If 'true', call an additional __tlbi_user()
- *              (typically for user ASIDs). 'flase' for IPA instructions
+ * @level:	Translation Table level hint, if known
  * @lpa2:	If 'true', the lpa2 scheme is used as set out below
  *
  * When the CPU does not support TLB range operations, flush the TLB
@@ -427,116 +396,181 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
  *    operations can only span an even number of pages. We save this for last to
  *    ensure 64KB start alignment is maintained for the LPA2 case.
  */
-#define __flush_tlb_range_op(op, start, pages, stride,			\
-				asid, tlb_level, tlbi_user, lpa2)	\
-do {									\
-	typeof(start) __flush_start = start;				\
-	typeof(pages) __flush_pages = pages;				\
-	int num = 0;							\
-	int scale = 3;							\
-	int shift = lpa2 ? 16 : PAGE_SHIFT;				\
-	unsigned long addr;						\
-									\
-	while (__flush_pages > 0) {					\
-		if (!system_supports_tlb_range() ||			\
-		    __flush_pages == 1 ||				\
-		    (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) {	\
-			addr = __TLBI_VADDR(__flush_start, asid);	\
-			__tlbi_level(op, addr, tlb_level);		\
-			if (tlbi_user)					\
-				__tlbi_user_level(op, addr, tlb_level);	\
-			__flush_start += stride;			\
-			__flush_pages -= stride >> PAGE_SHIFT;		\
-			continue;					\
-		}							\
-									\
-		num = __TLBI_RANGE_NUM(__flush_pages, scale);		\
-		if (num >= 0) {						\
-			addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \
-						scale, num, tlb_level);	\
-			__tlbi(r##op, addr);				\
-			if (tlbi_user)					\
-				__tlbi_user(r##op, addr);		\
-			__flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
-			__flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
-		}							\
-		scale--;						\
-	}								\
-} while (0)
+static __always_inline void rvae1is(u64 arg)
+{
+	__tlbi(rvae1is, arg);
+	__tlbi_user(rvae1is, arg);
+}
+
+static __always_inline void rvale1(u64 arg)
+{
+	__tlbi(rvale1, arg);
+	__tlbi_user(rvale1, arg);
+}
+
+static __always_inline void rvale1is(u64 arg)
+{
+	__tlbi(rvale1is, arg);
+	__tlbi_user(rvale1is, arg);
+}
+
+static __always_inline void rvaale1is(u64 arg)
+{
+	__tlbi(rvaale1is, arg);
+}
+
+static __always_inline void ripas2e1is(u64 arg)
+{
+	__tlbi(ripas2e1is, arg);
+}
+
+static __always_inline void __tlbi_range(tlbi_op op, u64 addr,
+					 u16 asid, int scale, int num,
+					 u32 level, bool lpa2)
+{
+	u64 arg = 0;
+
+	arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT));
+	arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level);
+	arg |= FIELD_PREP(TLBIR_NUM_MASK, num);
+	arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale);
+	arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule());
+	arg |= FIELD_PREP(TLBIR_ASID_MASK, asid);
+
+	op(arg);
+}
+
+static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop,
+						 u64 start, size_t pages,
+						 u64 stride, u16 asid,
+						 u32 level, bool lpa2)
+{
+	u64 addr = start, end = start + pages * PAGE_SIZE;
+	int scale = 3;
+
+	while (addr != end) {
+		int num;
+
+		pages = (end - addr) >> PAGE_SHIFT;
+
+		if (!system_supports_tlb_range() || pages == 1)
+			goto invalidate_one;
+
+		if (lpa2 && !IS_ALIGNED(addr, SZ_64K))
+			goto invalidate_one;
+
+		num = __TLBI_RANGE_NUM(pages, scale);
+		if (num >= 0) {
+			__tlbi_range(rop, addr, asid, scale, num, level, lpa2);
+			addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;
+		}
+
+		scale--;
+		continue;
+invalidate_one:
+		__tlbi_level_asid(lop, addr, level, asid);
+		addr += stride;
+	}
+}
+
+#define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \
+	__flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled())
 
 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
-	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled());
+	__flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled())
 
-static inline bool __flush_tlb_range_limit_excess(unsigned long start,
-		unsigned long end, unsigned long pages, unsigned long stride)
+static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
+						  unsigned long stride)
 {
 	/*
-	 * When the system does not support TLB range based flush
-	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
-	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
-	 * pages can be handled.
+	 * Assume that the worst case number of DVM ops required to flush a
+	 * given range on a system that supports tlb-range is 20 (4 scales, 1
+	 * final page, 15 for alignment on LPA2 systems), which is much smaller
+	 * than MAX_DVM_OPS.
 	 */
-	if ((!system_supports_tlb_range() &&
-	     (end - start) >= (MAX_DVM_OPS * stride)) ||
-	    pages > MAX_TLBI_RANGE_PAGES)
-		return true;
+	if (system_supports_tlb_range())
+		return pages > MAX_TLBI_RANGE_PAGES;
 
-	return false;
+	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
 }
 
-static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
-				     unsigned long start, unsigned long end,
-				     unsigned long stride, bool last_level,
-				     int tlb_level)
+typedef unsigned __bitwise tlbf_t;
+
+/* No special behaviour. */
+#define TLBF_NONE		((__force tlbf_t)0)
+
+/* Invalidate tlb entries only, leaving the page table walk cache intact. */
+#define TLBF_NOWALKCACHE	((__force tlbf_t)BIT(0))
+
+/* Skip the trailing dsb after issuing tlbi. */
+#define TLBF_NOSYNC		((__force tlbf_t)BIT(1))
+
+/* Suppress tlb notifier callbacks for this flush operation. */
+#define TLBF_NONOTIFY		((__force tlbf_t)BIT(2))
+
+/* Perform the tlbi locally without broadcasting to other CPUs. */
+#define TLBF_NOBROADCAST	((__force tlbf_t)BIT(3))
+
+static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma,
+					unsigned long start, unsigned long end,
+					unsigned long stride, int tlb_level,
+					tlbf_t flags)
 {
+	struct mm_struct *mm = vma->vm_mm;
 	unsigned long asid, pages;
 
-	start = round_down(start, stride);
-	end = round_up(end, stride);
 	pages = (end - start) >> PAGE_SHIFT;
 
-	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
+	if (__flush_tlb_range_limit_excess(pages, stride)) {
 		flush_tlb_mm(mm);
 		return;
 	}
 
-	dsb(ishst);
+	if (!(flags & TLBF_NOBROADCAST))
+		dsb(ishst);
+	else
+		dsb(nshst);
+
 	asid = ASID(mm);
 
-	if (last_level)
-		__flush_tlb_range_op(vale1is, start, pages, stride, asid,
-				     tlb_level, true, lpa2_is_enabled());
-	else
-		__flush_tlb_range_op(vae1is, start, pages, stride, asid,
-				     tlb_level, true, lpa2_is_enabled());
+	switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) {
+	case TLBF_NONE:
+		__flush_s1_tlb_range_op(vae1is, start, pages, stride,
+					asid, tlb_level);
+		break;
+	case TLBF_NOWALKCACHE:
+		__flush_s1_tlb_range_op(vale1is, start, pages, stride,
+					asid, tlb_level);
+		break;
+	case TLBF_NOBROADCAST:
+		/* Combination unused */
+		BUG();
+		break;
+	case TLBF_NOWALKCACHE | TLBF_NOBROADCAST:
+		__flush_s1_tlb_range_op(vale1, start, pages, stride,
+					asid, tlb_level);
+		break;
+	}
 
-	mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
+	if (!(flags & TLBF_NONOTIFY))
+		mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
+
+	if (!(flags & TLBF_NOSYNC)) {
+		if (!(flags & TLBF_NOBROADCAST))
+			__tlbi_sync_s1ish();
+		else
+			dsb(nsh);
+	}
 }
 
 static inline void __flush_tlb_range(struct vm_area_struct *vma,
 				     unsigned long start, unsigned long end,
-				     unsigned long stride, bool last_level,
-				     int tlb_level)
-{
-	__flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
-				 last_level, tlb_level);
-	__tlbi_sync_s1ish();
-}
-
-static inline void local_flush_tlb_contpte(struct vm_area_struct *vma,
-					   unsigned long addr)
+				     unsigned long stride, int tlb_level,
+				     tlbf_t flags)
 {
-	unsigned long asid;
-
-	addr = round_down(addr, CONT_PTE_SIZE);
-
-	dsb(nshst);
-	asid = ASID(vma->vm_mm);
-	__flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid,
-			     3, true, lpa2_is_enabled());
-	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr,
-						    addr + CONT_PTE_SIZE);
-	dsb(nsh);
+	start = round_down(start, stride);
+	end = round_up(end, stride);
+	__do_flush_tlb_range(vma, start, end, stride, tlb_level, flags);
 }
 
 static inline void flush_tlb_range(struct vm_area_struct *vma,
@@ -548,7 +582,23 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
 	 * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
 	 * information here.
 	 */
-	__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
+	__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);
+}
+
+static inline void __flush_tlb_page(struct vm_area_struct *vma,
+				    unsigned long uaddr, tlbf_t flags)
+{
+	unsigned long start = round_down(uaddr, PAGE_SIZE);
+	unsigned long end = start + PAGE_SIZE;
+
+	__do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3,
+			     TLBF_NOWALKCACHE | flags);
+}
+
+static inline void flush_tlb_page(struct vm_area_struct *vma,
+				  unsigned long uaddr)
+{
+	__flush_tlb_page(vma, uaddr, TLBF_NONE);
 }
 
 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -560,14 +610,14 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
 	end = round_up(end, stride);
 	pages = (end - start) >> PAGE_SHIFT;
 
-	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
+	if (__flush_tlb_range_limit_excess(pages, stride)) {
 		flush_tlb_all();
 		return;
 	}
 
 	dsb(ishst);
-	__flush_tlb_range_op(vaale1is, start, pages, stride, 0,
-			     TLBI_TTL_UNKNOWN, false, lpa2_is_enabled());
+	__flush_s1_tlb_range_op(vaale1is, start, pages, stride, 0,
+				TLBI_TTL_UNKNOWN);
 	__tlbi_sync_s1ish();
 	isb();
 }
@@ -589,7 +639,10 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
 		struct mm_struct *mm, unsigned long start, unsigned long end)
 {
-	__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
+	struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 };
+
+	__flush_tlb_range(&vma, start, end, PAGE_SIZE, 3,
+			  TLBF_NOWALKCACHE | TLBF_NOSYNC);
 }
 
 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
@@ -618,6 +671,8 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
 }
 #define huge_pmd_needs_flush huge_pmd_needs_flush
 
+#undef __tlbi_user
+#undef __TLBI_VADDR
 #endif
 
 #endif
diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
index 0d513f924321f..a03e34b572f19 100644
--- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
@@ -7,6 +7,9 @@
 
 #ifndef __ASSEMBLER__
 
+#include <vdso/clocksource.h>
+#include <vdso/time32.h>
+
 #include <asm/barrier.h>
 #include <asm/unistd_compat_32.h>
 #include <asm/errno.h>
diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h
index 3658a757e255d..96d2eccd4995b 100644
--- a/arch/arm64/include/asm/vdso/gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/gettimeofday.h
@@ -9,6 +9,8 @@
 
 #ifndef __ASSEMBLER__
 
+#include <vdso/clocksource.h>
+
 #include <asm/alternative.h>
 #include <asm/arch_timer.h>
 #include <asm/barrier.h>
diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h
index c2485a862e690..14366d35ce82f 100644
--- a/arch/arm64/include/asm/vncr_mapping.h
+++ b/arch/arm64/include/asm/vncr_mapping.h
@@ -108,5 +108,8 @@
 #define VNCR_MPAMVPM5_EL2       0x968
 #define VNCR_MPAMVPM6_EL2       0x970
 #define VNCR_MPAMVPM7_EL2       0x978
+#define VNCR_ICH_HFGITR_EL2	0xB10
+#define VNCR_ICH_HFGRTR_EL2	0xB18
+#define VNCR_ICH_HFGWTR_EL2	0xB20
 
 #endif /* __ARM64_VNCR_MAPPING_H__ */
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
deleted file mode 100644
index c38e3d017a79e..0000000000000
--- a/arch/arm64/include/asm/xor.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * arch/arm64/include/asm/xor.h
- *
- * Authors: Jackie Liu <liuyun01@kylinos.cn>
- * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
- */
-
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
-#include <asm/simd.h>
-
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_inner_neon;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2)
-{
-	scoped_ksimd()
-		xor_block_inner_neon.do_2(bytes, p1, p2);
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3)
-{
-	scoped_ksimd()
-		xor_block_inner_neon.do_3(bytes, p1, p2, p3);
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3,
-	   const unsigned long * __restrict p4)
-{
-	scoped_ksimd()
-		xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
-	   const unsigned long * __restrict p2,
-	   const unsigned long * __restrict p3,
-	   const unsigned long * __restrict p4,
-	   const unsigned long * __restrict p5)
-{
-	scoped_ksimd()
-		xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
-}
-
-static struct xor_block_template xor_block_arm64 = {
-	.name   = "arm64_neon",
-	.do_2   = xor_neon_2,
-	.do_3   = xor_neon_3,
-	.do_4   = xor_neon_4,
-	.do_5	= xor_neon_5
-};
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES           \
-	do {        \
-		xor_speed(&xor_block_8regs);    \
-		xor_speed(&xor_block_32regs);    \
-		if (cpu_has_neon()) { \
-			xor_speed(&xor_block_arm64);\
-		} \
-	} while (0)
-
-#endif /* ! CONFIG_KERNEL_MODE_NEON */
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index a792a599b9d68..1c13bfa2d38aa 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -428,6 +428,7 @@ enum {
 #define   KVM_DEV_ARM_ITS_RESTORE_TABLES        2
 #define   KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES	3
 #define   KVM_DEV_ARM_ITS_CTRL_RESET		4
+#define   KVM_DEV_ARM_VGIC_USERSPACE_PPIS	5
 
 /* Device Control API on vcpu fd */
 #define KVM_ARM_VCPU_PMU_V3_CTRL	0
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f8018b5c1f9ad..ab476ba060d10 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -273,7 +273,7 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
 alternative_else_nop_endif
 1:
 
-	scs_load_current
+	scs_load_current_base
 	.else
 	add	x21, sp, #PT_REGS_SIZE
 	get_current_task tsk
@@ -378,8 +378,6 @@ alternative_if ARM64_WORKAROUND_845719
 alternative_else_nop_endif
 #endif
 3:
-	scs_save tsk
-
 	/* Ignore asynchronous tag check faults in the uaccess routines */
 	ldr	x0, [tsk, THREAD_SCTLR_USER]
 	clear_mte_async_tcf x0
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index d7b0d12b10155..d4c7d45ae6bc8 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -138,6 +138,10 @@ KVM_NVHE_ALIAS(__hyp_data_start);
 KVM_NVHE_ALIAS(__hyp_data_end);
 KVM_NVHE_ALIAS(__hyp_rodata_start);
 KVM_NVHE_ALIAS(__hyp_rodata_end);
+#ifdef CONFIG_NVHE_EL2_TRACING
+KVM_NVHE_ALIAS(__hyp_event_ids_start);
+KVM_NVHE_ALIAS(__hyp_event_ids_end);
+#endif
 
 /* pKVM static key */
 KVM_NVHE_ALIAS(kvm_protected_mode_initialized);
diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c
index fba260ad87a96..e31fabed378a5 100644
--- a/arch/arm64/kernel/machine_kexec_file.c
+++ b/arch/arm64/kernel/machine_kexec_file.c
@@ -134,6 +134,10 @@ int load_other_segments(struct kimage *image,
 
 		kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
 			      image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
+
+		ret = crash_load_dm_crypt_keys(image);
+		if (ret)
+			goto out_err;
 	}
 #endif
 
diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c
index bbe7d30ed12b3..dac568e4a54f2 100644
--- a/arch/arm64/kernel/pi/patch-scs.c
+++ b/arch/arm64/kernel/pi/patch-scs.c
@@ -192,6 +192,14 @@ static int scs_handle_fde_frame(const struct eh_frame *frame,
 			size -= 2;
 			break;
 
+		case DW_CFA_advance_loc4:
+			loc += *opcode++ * code_alignment_factor;
+			loc += (*opcode++ << 8) * code_alignment_factor;
+			loc += (*opcode++ << 16) * code_alignment_factor;
+			loc += (*opcode++ << 24) * code_alignment_factor;
+			size -= 4;
+		break;
+
 		case DW_CFA_def_cfa:
 		case DW_CFA_offset_extended:
 			size = skip_xleb128(&opcode, size);
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 413f899e4ac63..6cb4209f5dab5 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -64,7 +64,8 @@ SYM_CODE_START(arm64_relocate_new_kernel)
 	mov	x19, x13
 	copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
 	add	x1, x19, #PAGE_SIZE
-	dcache_by_myline_op civac, sy, x19, x1, x15, x20
+	dcache_by_myline_op_nosync civac, x19, x1, x15, x20
+	dsb	sy
 	b	.Lnext
 .Ltest_indirection:
 	tbz	x16, IND_INDIRECTION_BIT, .Ltest_destination
diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
index c64a06f58c0bc..9e846ce4ef9ca 100644
--- a/arch/arm64/kernel/rsi.c
+++ b/arch/arm64/kernel/rsi.c
@@ -12,6 +12,7 @@
 
 #include <asm/io.h>
 #include <asm/mem_encrypt.h>
+#include <asm/pgtable.h>
 #include <asm/rsi.h>
 
 static struct realm_config config;
@@ -146,7 +147,7 @@ void __init arm64_rsi_init(void)
 		return;
 	if (WARN_ON(rsi_get_realm_config(&config)))
 		return;
-	prot_ns_shared = BIT(config.ipa_bits - 1);
+	prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1));
 
 	if (arm64_ioremap_prot_hook_register(realm_ioremap_hook))
 		return;
diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
index b9d4998c97efa..7e9860143add8 100644
--- a/arch/arm64/kernel/sys_compat.c
+++ b/arch/arm64/kernel/sys_compat.c
@@ -36,7 +36,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)
 			 * The workaround requires an inner-shareable tlbi.
 			 * We pick the reserved-ASID to minimise the impact.
 			 */
-			__tlbi(aside1is, __TLBI_VADDR(0, 0));
+			__tlbi(aside1is, 0UL);
 			__tlbi_sync_s1ish();
 		}
 
diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile
index 9d0efed91414c..bea3675fa668b 100644
--- a/arch/arm64/kernel/vdso32/Makefile
+++ b/arch/arm64/kernel/vdso32/Makefile
@@ -63,7 +63,7 @@ VDSO_CFLAGS += -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
                $(filter -Werror,$(KBUILD_CPPFLAGS)) \
                -Werror-implicit-function-declaration \
                -Wno-format-security \
-               -std=gnu11 -fms-extensions
+               $(CC_FLAGS_DIALECT)
 VDSO_CFLAGS  += -O2
 # Some useful compiler-dependent flags from top-level Makefile
 VDSO_CFLAGS += $(call cc32-option,-Wno-pointer-sign)
@@ -71,7 +71,6 @@ VDSO_CFLAGS += -fno-strict-overflow
 VDSO_CFLAGS += $(call cc32-option,-Werror=strict-prototypes)
 VDSO_CFLAGS += -Werror=date-time
 VDSO_CFLAGS += $(call cc32-option,-Werror=incompatible-pointer-types)
-VDSO_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag)
 
 # Compile as THUMB2 or ARM. Unwinding via frame-pointers in THUMB2 is
 # unreliable.
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 2964aad0362e4..4bf2a83c3448b 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -13,12 +13,23 @@
 	*(__kvm_ex_table)					\
 	__stop___kvm_ex_table = .;
 
+#ifdef CONFIG_NVHE_EL2_TRACING
+#define HYPERVISOR_EVENT_IDS 					\
+	. = ALIGN(PAGE_SIZE);					\
+	__hyp_event_ids_start = .;				\
+	*(HYP_SECTION_NAME(.event_ids))				\
+	__hyp_event_ids_end = .;
+#else
+#define HYPERVISOR_EVENT_IDS
+#endif
+
 #define HYPERVISOR_RODATA_SECTIONS				\
 	HYP_SECTION_NAME(.rodata) : {				\
 		. = ALIGN(PAGE_SIZE);				\
 		__hyp_rodata_start = .;				\
 		*(HYP_SECTION_NAME(.data..ro_after_init))	\
 		*(HYP_SECTION_NAME(.rodata))			\
+		HYPERVISOR_EVENT_IDS				\
 		. = ALIGN(PAGE_SIZE);				\
 		__hyp_rodata_end = .;				\
 	}
@@ -307,6 +318,13 @@ SECTIONS
 
 	HYPERVISOR_DATA_SECTION
 
+#ifdef CONFIG_NVHE_EL2_TRACING
+	.data.hyp_events : {
+		__hyp_events_start = .;
+		*(SORT(_hyp_events.*))
+		__hyp_events_end = .;
+	}
+#endif
 	/*
 	 * Data written with the MMU off but read with the MMU on requires
 	 * cache lines to be invalidated, discarding up to a Cache Writeback
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 7d1f22fd490b2..449154f9a4852 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -42,9 +42,27 @@ menuconfig KVM
 
 	  If unsure, say N.
 
+if KVM
+
+config PTDUMP_STAGE2_DEBUGFS
+	bool "Present the stage-2 pagetables to debugfs"
+	depends on DEBUG_KERNEL
+	depends on DEBUG_FS
+	depends on ARCH_HAS_PTDUMP
+	select PTDUMP
+	default n
+	help
+	  Say Y here if you want to show the stage-2 kernel pagetables
+	  layout in a debugfs file. This information is only useful for kernel developers
+	  who are working in architecture specific areas of the kernel.
+	  It is probably not a good idea to enable this feature in a production
+	  kernel.
+
+	  If in doubt, say N.
+
 config NVHE_EL2_DEBUG
 	bool "Debug mode for non-VHE EL2 object"
-	depends on KVM
+	default n
 	help
 	  Say Y here to enable the debug mode for the non-VHE KVM EL2 object.
 	  Failure reports will BUG() in the hypervisor. This is intended for
@@ -52,10 +70,29 @@ config NVHE_EL2_DEBUG
 
 	  If unsure, say N.
 
-config PROTECTED_NVHE_STACKTRACE
-	bool "Protected KVM hypervisor stacktraces"
-	depends on NVHE_EL2_DEBUG
+if NVHE_EL2_DEBUG
+
+config NVHE_EL2_TRACING
+	bool
+	depends on TRACING && FTRACE
+	select TRACE_REMOTE
+	default y
+
+config PKVM_DISABLE_STAGE2_ON_PANIC
+	bool "Disable the host stage-2 on panic"
 	default n
+	help
+	  Relax the host stage-2 on hypervisor panic to allow the kernel to
+	  unwind and symbolize the hypervisor stacktrace. This however tampers
+	  the system security. This is intended for local EL2 hypervisor
+	  development.
+
+	  If unsure, say N.
+
+config PKVM_STACKTRACE
+	bool "Protected KVM hypervisor stacktraces"
+	depends on PKVM_DISABLE_STAGE2_ON_PANIC
+	default y
 	help
 	  Say Y here to enable pKVM hypervisor stacktraces on hyp_panic()
 
@@ -65,21 +102,6 @@ config PROTECTED_NVHE_STACKTRACE
 
 	  If unsure, or not using protected nVHE (pKVM), say N.
 
-config PTDUMP_STAGE2_DEBUGFS
-	bool "Present the stage-2 pagetables to debugfs"
-	depends on KVM
-	depends on DEBUG_KERNEL
-	depends on DEBUG_FS
-	depends on ARCH_HAS_PTDUMP
-	select PTDUMP
-	default n
-	help
-	  Say Y here if you want to show the stage-2 kernel pagetables
-	  layout in a debugfs file. This information is only useful for kernel developers
-	  who are working in architecture specific areas of the kernel.
-	  It is probably not a good idea to enable this feature in a production
-	  kernel.
-
-	  If in doubt, say N.
-
+endif # NVHE_EL2_DEBUG
+endif # KVM
 endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 3ebc0570345cc..59612d2f277c1 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -30,6 +30,8 @@ kvm-$(CONFIG_HW_PERF_EVENTS)  += pmu-emul.o pmu.o
 kvm-$(CONFIG_ARM64_PTR_AUTH)  += pauth.o
 kvm-$(CONFIG_PTDUMP_STAGE2_DEBUGFS) += ptdump.o
 
+kvm-$(CONFIG_NVHE_EL2_TRACING) += hyp_trace.o
+
 always-y := hyp_constants.h hyp-constants.s
 
 define rule_gen_hyp_constants
diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c
index 600f250753b45..67b989671b410 100644
--- a/arch/arm64/kvm/arch_timer.c
+++ b/arch/arm64/kvm/arch_timer.c
@@ -56,6 +56,12 @@ static struct irq_ops arch_timer_irq_ops = {
 	.get_input_level = kvm_arch_timer_get_input_level,
 };
 
+static struct irq_ops arch_timer_irq_ops_vgic_v5 = {
+	.get_input_level = kvm_arch_timer_get_input_level,
+	.queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock,
+	.set_direct_injection = vgic_v5_set_ppi_dvi,
+};
+
 static int nr_timers(struct kvm_vcpu *vcpu)
 {
 	if (!vcpu_has_nv(vcpu))
@@ -177,6 +183,10 @@ void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
 		map->emul_ptimer = vcpu_ptimer(vcpu);
 	}
 
+	map->direct_vtimer->direct = true;
+	if (map->direct_ptimer)
+		map->direct_ptimer->direct = true;
+
 	trace_kvm_get_timer_map(vcpu->vcpu_id, map);
 }
 
@@ -396,7 +406,11 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
-	return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
+	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
+	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
+
+	return kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer) ||
+	       (vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0);
 }
 
 /*
@@ -447,6 +461,10 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 	if (userspace_irqchip(vcpu->kvm))
 		return;
 
+	/* Skip injecting on GICv5 for directly injected (DVI'd) timers */
+	if (vgic_is_v5(vcpu->kvm) && timer_ctx->direct)
+		return;
+
 	kvm_vgic_inject_irq(vcpu->kvm, vcpu,
 			    timer_irq(timer_ctx),
 			    timer_ctx->irq.level,
@@ -674,6 +692,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 		phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
 
 	phys_active |= ctx->irq.level;
+	phys_active |= vgic_is_v5(vcpu->kvm);
 
 	set_timer_irq_phys_active(ctx, phys_active);
 }
@@ -740,13 +759,11 @@ static void kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu *vcpu,
 
 		ret = kvm_vgic_map_phys_irq(vcpu,
 					    map->direct_vtimer->host_timer_irq,
-					    timer_irq(map->direct_vtimer),
-					    &arch_timer_irq_ops);
+					    timer_irq(map->direct_vtimer));
 		WARN_ON_ONCE(ret);
 		ret = kvm_vgic_map_phys_irq(vcpu,
 					    map->direct_ptimer->host_timer_irq,
-					    timer_irq(map->direct_ptimer),
-					    &arch_timer_irq_ops);
+					    timer_irq(map->direct_ptimer));
 		WARN_ON_ONCE(ret);
 	}
 }
@@ -864,7 +881,8 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 	get_timer_map(vcpu, &map);
 
 	if (static_branch_likely(&has_gic_active_state)) {
-		if (vcpu_has_nv(vcpu))
+		/* We don't do NV on GICv5, yet */
+		if (vcpu_has_nv(vcpu) && !vgic_is_v5(vcpu->kvm))
 			kvm_timer_vcpu_load_nested_switch(vcpu, &map);
 
 		kvm_timer_vcpu_load_gic(map.direct_vtimer);
@@ -934,6 +952,12 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 
 	if (kvm_vcpu_is_blocking(vcpu))
 		kvm_timer_blocking(vcpu);
+
+	if (vgic_is_v5(vcpu->kvm)) {
+		set_timer_irq_phys_active(map.direct_vtimer, false);
+		if (map.direct_ptimer)
+			set_timer_irq_phys_active(map.direct_ptimer, false);
+	}
 }
 
 void kvm_timer_sync_nested(struct kvm_vcpu *vcpu)
@@ -1097,10 +1121,19 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 		      HRTIMER_MODE_ABS_HARD);
 }
 
+/*
+ * This is always called during kvm_arch_init_vm, but will also be
+ * called from kvm_vgic_create if we have a vGICv5.
+ */
 void kvm_timer_init_vm(struct kvm *kvm)
 {
+	/*
+	 * Set up the default PPIs - note that we adjust them based on
+	 * the model of the GIC as GICv5 uses a different way to
+	 * describing interrupts.
+	 */
 	for (int i = 0; i < NR_KVM_TIMERS; i++)
-		kvm->arch.timer_data.ppi[i] = default_ppi[i];
+		kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);
 }
 
 void kvm_timer_cpu_up(void)
@@ -1269,7 +1302,15 @@ static int timer_irq_set_irqchip_state(struct irq_data *d,
 
 static void timer_irq_eoi(struct irq_data *d)
 {
-	if (!irqd_is_forwarded_to_vcpu(d))
+	/*
+	 * On a GICv5 host, we still need to call EOI on the parent for
+	 * PPIs. The host driver already handles irqs which are forwarded to
+	 * vcpus, and skips the GIC CDDI while still doing the GIC CDEOI. This
+	 * is required to emulate the EOIMode=1 on GICv5 hardware. Failure to
+	 * call EOI unsurprisingly results in *BAD* lock-ups.
+	 */
+	if (!irqd_is_forwarded_to_vcpu(d) ||
+	    kvm_vgic_global_state.type == VGIC_V5)
 		irq_chip_eoi_parent(d);
 }
 
@@ -1333,7 +1374,8 @@ static int kvm_irq_init(struct arch_timer_kvm_info *info)
 	host_vtimer_irq = info->virtual_irq;
 	kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags);
 
-	if (kvm_vgic_global_state.no_hw_deactivation) {
+	if (kvm_vgic_global_state.no_hw_deactivation ||
+	    kvm_vgic_global_state.type == VGIC_V5) {
 		struct fwnode_handle *fwnode;
 		struct irq_data *data;
 
@@ -1351,7 +1393,8 @@ static int kvm_irq_init(struct arch_timer_kvm_info *info)
 			return -ENOMEM;
 		}
 
-		arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE;
+		if (kvm_vgic_global_state.no_hw_deactivation)
+			arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE;
 		WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq,
 					    (void *)TIMER_VTIMER));
 	}
@@ -1502,10 +1545,13 @@ static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
 			break;
 
 		/*
-		 * We know by construction that we only have PPIs, so
-		 * all values are less than 32.
+		 * We know by construction that we only have PPIs, so all values
+		 * are less than 32 for non-GICv5 VGICs. On GICv5, they are
+		 * architecturally defined to be under 32 too. However, we mask
+		 * off most of the bits as we might be presented with a GICv5
+		 * style PPI where the type is encoded in the top-bits.
 		 */
-		ppis |= BIT(irq);
+		ppis |= BIT(irq & 0x1f);
 	}
 
 	valid = hweight32(ppis) == nr_timers(vcpu);
@@ -1543,6 +1589,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 	struct timer_map map;
+	struct irq_ops *ops;
 	int ret;
 
 	if (timer->enabled)
@@ -1563,20 +1610,22 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu)
 
 	get_timer_map(vcpu, &map);
 
+	ops = vgic_is_v5(vcpu->kvm) ? &arch_timer_irq_ops_vgic_v5 :
+				      &arch_timer_irq_ops;
+
+	for (int i = 0; i < nr_timers(vcpu); i++)
+		kvm_vgic_set_irq_ops(vcpu, timer_irq(vcpu_get_timer(vcpu, i)), ops);
+
 	ret = kvm_vgic_map_phys_irq(vcpu,
 				    map.direct_vtimer->host_timer_irq,
-				    timer_irq(map.direct_vtimer),
-				    &arch_timer_irq_ops);
+				    timer_irq(map.direct_vtimer));
 	if (ret)
 		return ret;
 
-	if (map.direct_ptimer) {
+	if (map.direct_ptimer)
 		ret = kvm_vgic_map_phys_irq(vcpu,
 					    map.direct_ptimer->host_timer_irq,
-					    timer_irq(map.direct_ptimer),
-					    &arch_timer_irq_ops);
-	}
-
+					    timer_irq(map.direct_ptimer));
 	if (ret)
 		return ret;
 
@@ -1603,15 +1652,14 @@ int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 	if (get_user(irq, uaddr))
 		return -EFAULT;
 
-	if (!(irq_is_ppi(irq)))
+	if (!(irq_is_ppi(vcpu->kvm, irq)))
 		return -EINVAL;
 
-	mutex_lock(&vcpu->kvm->arch.config_lock);
+	guard(mutex)(&vcpu->kvm->arch.config_lock);
 
 	if (test_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE,
 		     &vcpu->kvm->arch.flags)) {
-		ret = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 
 	switch (attr->attr) {
@@ -1628,10 +1676,16 @@ int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 		idx = TIMER_HPTIMER;
 		break;
 	default:
-		ret = -ENXIO;
-		goto out;
+		return -ENXIO;
 	}
 
+	/*
+	 * The PPIs for the Arch Timers are architecturally defined for
+	 * GICv5. Reject anything that changes them from the specified value.
+	 */
+	if (vgic_is_v5(vcpu->kvm) && vcpu->kvm->arch.timer_data.ppi[idx] != irq)
+		return -EINVAL;
+
 	/*
 	 * We cannot validate the IRQ unicity before we run, so take it at
 	 * face value. The verdict will be given on first vcpu run, for each
@@ -1639,8 +1693,6 @@ int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 	 */
 	vcpu->kvm->arch.timer_data.ppi[idx] = irq;
 
-out:
-	mutex_unlock(&vcpu->kvm->arch.config_lock);
 	return ret;
 }
 
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 410ffd41fd73a..c4c810b010122 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -24,6 +24,7 @@
 
 #define CREATE_TRACE_POINTS
 #include "trace_arm.h"
+#include "hyp_trace.h"
 
 #include <linux/uaccess.h>
 #include <asm/ptrace.h>
@@ -35,6 +36,7 @@
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
+#include <asm/kvm_hyp.h>
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_nested.h>
 #include <asm/kvm_pkvm.h>
@@ -45,6 +47,9 @@
 #include <kvm/arm_hypercalls.h>
 #include <kvm/arm_pmu.h>
 #include <kvm/arm_psci.h>
+#include <kvm/arm_vgic.h>
+
+#include <linux/irqchip/arm-gic-v5.h>
 
 #include "sys_regs.h"
 
@@ -613,6 +618,9 @@ static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
 	if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
 		return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
 
+	if (vgic_is_v5(vcpu->kvm))
+		return single_task_running();
+
 	return single_task_running() &&
 	       vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
 	       (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
@@ -705,6 +713,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
 	if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
 		vcpu_set_on_unsupported_cpu(vcpu);
+
+	vcpu->arch.pid = pid_nr(vcpu->pid);
 }
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
@@ -934,6 +944,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 			return ret;
 	}
 
+	ret = vgic_v5_finalize_ppi_state(kvm);
+	if (ret)
+		return ret;
+
 	if (is_protected_kvm_enabled()) {
 		ret = pkvm_create_hyp_vm(kvm);
 		if (ret)
@@ -1439,10 +1453,11 @@ static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 			  bool line_status)
 {
-	u32 irq = irq_level->irq;
 	unsigned int irq_type, vcpu_id, irq_num;
 	struct kvm_vcpu *vcpu = NULL;
 	bool level = irq_level->level;
+	u32 irq = irq_level->irq;
+	unsigned long *mask;
 
 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
 	vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
@@ -1472,16 +1487,37 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 		if (!vcpu)
 			return -EINVAL;
 
-		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
+		if (vgic_is_v5(kvm)) {
+			if (irq_num >= VGIC_V5_NR_PRIVATE_IRQS)
+				return -EINVAL;
+
+			/*
+			 * Only allow PPIs that are explicitly exposed to
+			 * usespace to be driven via KVM_IRQ_LINE
+			 */
+			mask = kvm->arch.vgic.gicv5_vm.userspace_ppis;
+			if (!test_bit(irq_num, mask))
+				return -EINVAL;
+
+			/* Build a GICv5-style IntID here */
+			irq_num = vgic_v5_make_ppi(irq_num);
+		} else if (irq_num < VGIC_NR_SGIS ||
+			   irq_num >= VGIC_NR_PRIVATE_IRQS) {
 			return -EINVAL;
+		}
 
 		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
 	case KVM_ARM_IRQ_TYPE_SPI:
 		if (!irqchip_in_kernel(kvm))
 			return -ENXIO;
 
-		if (irq_num < VGIC_NR_PRIVATE_IRQS)
-			return -EINVAL;
+		if (vgic_is_v5(kvm)) {
+			/* Build a GICv5-style IntID here */
+			irq_num = vgic_v5_make_spi(irq_num);
+		} else {
+			if (irq_num < VGIC_NR_PRIVATE_IRQS)
+				return -EINVAL;
+		}
 
 		return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
 	}
@@ -2414,6 +2450,10 @@ static int __init init_subsystems(void)
 
 	kvm_register_perf_callbacks();
 
+	err = kvm_hyp_trace_init();
+	if (err)
+		kvm_err("Failed to initialize Hyp tracing\n");
+
 out:
 	if (err)
 		hyp_cpu_pm_exit();
@@ -2465,7 +2505,7 @@ static int __init do_pkvm_init(u32 hyp_va_bits)
 	preempt_disable();
 	cpu_hyp_init_context();
 	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
-				num_possible_cpus(), kern_hyp_va(per_cpu_base),
+				kern_hyp_va(per_cpu_base),
 				hyp_va_bits);
 	cpu_hyp_init_features();
 
@@ -2507,6 +2547,7 @@ static void kvm_hyp_init_symbols(void)
 {
 	kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
 	kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
+	kvm_nvhe_sym(id_aa64pfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR2_EL1);
 	kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
 	kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
 	kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
@@ -2529,6 +2570,9 @@ static void kvm_hyp_init_symbols(void)
 	kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks;
 	kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks;
 	kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks;
+	kvm_nvhe_sym(ich_hfgrtr_masks) = ich_hfgrtr_masks;
+	kvm_nvhe_sym(ich_hfgwtr_masks) = ich_hfgwtr_masks;
+	kvm_nvhe_sym(ich_hfgitr_masks) = ich_hfgitr_masks;
 
 	/*
 	 * Flush entire BSS since part of its data containing init symbols is read
@@ -2674,6 +2718,8 @@ static int __init init_hyp_mode(void)
 		kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
 	}
 
+	kvm_nvhe_sym(hyp_nr_cpus) = num_possible_cpus();
+
 	/*
 	 * Map the Hyp-code called directly from the host
 	 */
diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c
index d9f553cbf9dfd..e14685343191b 100644
--- a/arch/arm64/kvm/config.c
+++ b/arch/arm64/kvm/config.c
@@ -225,6 +225,7 @@ struct reg_feat_map_desc {
 #define FEAT_MTPMU		ID_AA64DFR0_EL1, MTPMU, IMP
 #define FEAT_HCX		ID_AA64MMFR1_EL1, HCX, IMP
 #define FEAT_S2PIE		ID_AA64MMFR3_EL1, S2PIE, IMP
+#define FEAT_GCIE		ID_AA64PFR2_EL1, GCIE, IMP
 
 static bool not_feat_aa64el3(struct kvm *kvm)
 {
@@ -1277,6 +1278,58 @@ static const struct reg_bits_to_feat_map vtcr_el2_feat_map[] = {
 static const DECLARE_FEAT_MAP(vtcr_el2_desc, VTCR_EL2,
 			      vtcr_el2_feat_map, FEAT_AA64EL2);
 
+static const struct reg_bits_to_feat_map ich_hfgrtr_feat_map[] = {
+	NEEDS_FEAT(ICH_HFGRTR_EL2_ICC_APR_EL1 |
+		   ICH_HFGRTR_EL2_ICC_IDRn_EL1 |
+		   ICH_HFGRTR_EL2_ICC_CR0_EL1 |
+		   ICH_HFGRTR_EL2_ICC_HPPIR_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PCR_EL1 |
+		   ICH_HFGRTR_EL2_ICC_ICSR_EL1 |
+		   ICH_HFGRTR_EL2_ICC_IAFFIDR_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PPI_HMRn_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PPI_ENABLERn_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PPI_PENDRn_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PPI_PRIORITYRn_EL1 |
+		   ICH_HFGRTR_EL2_ICC_PPI_ACTIVERn_EL1,
+		   FEAT_GCIE),
+};
+
+static const DECLARE_FEAT_MAP_FGT(ich_hfgrtr_desc, ich_hfgrtr_masks,
+				  ich_hfgrtr_feat_map, FEAT_GCIE);
+
+static const struct reg_bits_to_feat_map ich_hfgwtr_feat_map[] = {
+	NEEDS_FEAT(ICH_HFGWTR_EL2_ICC_APR_EL1 |
+		   ICH_HFGWTR_EL2_ICC_CR0_EL1 |
+		   ICH_HFGWTR_EL2_ICC_PCR_EL1 |
+		   ICH_HFGWTR_EL2_ICC_ICSR_EL1 |
+		   ICH_HFGWTR_EL2_ICC_PPI_ENABLERn_EL1 |
+		   ICH_HFGWTR_EL2_ICC_PPI_PENDRn_EL1 |
+		   ICH_HFGWTR_EL2_ICC_PPI_PRIORITYRn_EL1 |
+		   ICH_HFGWTR_EL2_ICC_PPI_ACTIVERn_EL1,
+		   FEAT_GCIE),
+};
+
+static const DECLARE_FEAT_MAP_FGT(ich_hfgwtr_desc, ich_hfgwtr_masks,
+				  ich_hfgwtr_feat_map, FEAT_GCIE);
+
+static const struct reg_bits_to_feat_map ich_hfgitr_feat_map[] = {
+	NEEDS_FEAT(ICH_HFGITR_EL2_GICCDEN |
+		   ICH_HFGITR_EL2_GICCDDIS |
+		   ICH_HFGITR_EL2_GICCDPRI |
+		   ICH_HFGITR_EL2_GICCDAFF |
+		   ICH_HFGITR_EL2_GICCDPEND |
+		   ICH_HFGITR_EL2_GICCDRCFG |
+		   ICH_HFGITR_EL2_GICCDHM |
+		   ICH_HFGITR_EL2_GICCDEOI |
+		   ICH_HFGITR_EL2_GICCDDI |
+		   ICH_HFGITR_EL2_GICRCDIA |
+		   ICH_HFGITR_EL2_GICRCDNMIA,
+		   FEAT_GCIE),
+};
+
+static const DECLARE_FEAT_MAP_FGT(ich_hfgitr_desc, ich_hfgitr_masks,
+				  ich_hfgitr_feat_map, FEAT_GCIE);
+
 static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
 				  int map_size, u64 resx, const char *str)
 {
@@ -1328,6 +1381,9 @@ void __init check_feature_map(void)
 	check_reg_desc(&sctlr_el2_desc);
 	check_reg_desc(&mdcr_el2_desc);
 	check_reg_desc(&vtcr_el2_desc);
+	check_reg_desc(&ich_hfgrtr_desc);
+	check_reg_desc(&ich_hfgwtr_desc);
+	check_reg_desc(&ich_hfgitr_desc);
 }
 
 static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
@@ -1460,6 +1516,13 @@ void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt)
 		val |= compute_fgu_bits(kvm, &hdfgrtr2_desc);
 		val |= compute_fgu_bits(kvm, &hdfgwtr2_desc);
 		break;
+	case ICH_HFGRTR_GROUP:
+		val |= compute_fgu_bits(kvm, &ich_hfgrtr_desc);
+		val |= compute_fgu_bits(kvm, &ich_hfgwtr_desc);
+		break;
+	case ICH_HFGITR_GROUP:
+		val |= compute_fgu_bits(kvm, &ich_hfgitr_desc);
+		break;
 	default:
 		BUG();
 	}
@@ -1531,6 +1594,15 @@ struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg)
 	case VTCR_EL2:
 		resx = compute_reg_resx_bits(kvm, &vtcr_el2_desc, 0, 0);
 		break;
+	case ICH_HFGRTR_EL2:
+		resx = compute_reg_resx_bits(kvm, &ich_hfgrtr_desc, 0, 0);
+		break;
+	case ICH_HFGWTR_EL2:
+		resx = compute_reg_resx_bits(kvm, &ich_hfgwtr_desc, 0, 0);
+		break;
+	case ICH_HFGITR_EL2:
+		resx = compute_reg_resx_bits(kvm, &ich_hfgitr_desc, 0, 0);
+		break;
 	default:
 		WARN_ON_ONCE(1);
 		resx = (typeof(resx)){};
@@ -1565,6 +1637,12 @@ static __always_inline struct fgt_masks *__fgt_reg_to_masks(enum vcpu_sysreg reg
 		return &hdfgrtr2_masks;
 	case HDFGWTR2_EL2:
 		return &hdfgwtr2_masks;
+	case ICH_HFGRTR_EL2:
+		return &ich_hfgrtr_masks;
+	case ICH_HFGWTR_EL2:
+		return &ich_hfgwtr_masks;
+	case ICH_HFGITR_EL2:
+		return &ich_hfgitr_masks;
 	default:
 		BUILD_BUG_ON(1);
 	}
@@ -1606,6 +1684,32 @@ static void __compute_hdfgwtr(struct kvm_vcpu *vcpu)
 		*vcpu_fgt(vcpu, HDFGWTR_EL2) |= HDFGWTR_EL2_MDSCR_EL1;
 }
 
+static void __compute_ich_hfgrtr(struct kvm_vcpu *vcpu)
+{
+	__compute_fgt(vcpu, ICH_HFGRTR_EL2);
+
+	/*
+	 * ICC_IAFFIDR_EL1 *always* needs to be trapped when running a guest.
+	 *
+	 * We also trap accesses to ICC_IDR0_EL1 to allow us to completely hide
+	 * FEAT_GCIE_LEGACY from the guest, and to (potentially) present fewer
+	 * ID bits than the host supports.
+	 */
+	*vcpu_fgt(vcpu, ICH_HFGRTR_EL2) &= ~(ICH_HFGRTR_EL2_ICC_IAFFIDR_EL1 |
+					     ICH_HFGRTR_EL2_ICC_IDRn_EL1);
+}
+
+static void __compute_ich_hfgwtr(struct kvm_vcpu *vcpu)
+{
+	__compute_fgt(vcpu, ICH_HFGWTR_EL2);
+
+	/*
+	 * We present a different subset of PPIs the guest from what
+	 * exist in real hardware. We only trap writes, not reads.
+	 */
+	*vcpu_fgt(vcpu, ICH_HFGWTR_EL2) &= ~(ICH_HFGWTR_EL2_ICC_PPI_ENABLERn_EL1);
+}
+
 void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu)
 {
 	if (!cpus_have_final_cap(ARM64_HAS_FGT))
@@ -1618,12 +1722,17 @@ void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu)
 	__compute_hdfgwtr(vcpu);
 	__compute_fgt(vcpu, HAFGRTR_EL2);
 
-	if (!cpus_have_final_cap(ARM64_HAS_FGT2))
-		return;
+	if (cpus_have_final_cap(ARM64_HAS_FGT2)) {
+		__compute_fgt(vcpu, HFGRTR2_EL2);
+		__compute_fgt(vcpu, HFGWTR2_EL2);
+		__compute_fgt(vcpu, HFGITR2_EL2);
+		__compute_fgt(vcpu, HDFGRTR2_EL2);
+		__compute_fgt(vcpu, HDFGWTR2_EL2);
+	}
 
-	__compute_fgt(vcpu, HFGRTR2_EL2);
-	__compute_fgt(vcpu, HFGWTR2_EL2);
-	__compute_fgt(vcpu, HFGITR2_EL2);
-	__compute_fgt(vcpu, HDFGRTR2_EL2);
-	__compute_fgt(vcpu, HDFGWTR2_EL2);
+	if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
+		__compute_ich_hfgrtr(vcpu);
+		__compute_ich_hfgwtr(vcpu);
+		__compute_fgt(vcpu, ICH_HFGITR_EL2);
+	}
 }
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index 22d497554c949..dba7ced74ca5e 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -2053,6 +2053,60 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
 	SR_FGT(SYS_AMEVCNTR0_EL0(2),	HAFGRTR, AMEVCNTR02_EL0, 1),
 	SR_FGT(SYS_AMEVCNTR0_EL0(1),	HAFGRTR, AMEVCNTR01_EL0, 1),
 	SR_FGT(SYS_AMEVCNTR0_EL0(0),	HAFGRTR, AMEVCNTR00_EL0, 1),
+
+	/*
+	 * ICH_HFGRTR_EL2 & ICH_HFGWTR_EL2
+	 */
+	SR_FGT(SYS_ICC_APR_EL1,			ICH_HFGRTR, ICC_APR_EL1, 0),
+	SR_FGT(SYS_ICC_IDR0_EL1,		ICH_HFGRTR, ICC_IDRn_EL1, 0),
+	SR_FGT(SYS_ICC_CR0_EL1,			ICH_HFGRTR, ICC_CR0_EL1, 0),
+	SR_FGT(SYS_ICC_HPPIR_EL1,		ICH_HFGRTR, ICC_HPPIR_EL1, 0),
+	SR_FGT(SYS_ICC_PCR_EL1,			ICH_HFGRTR, ICC_PCR_EL1, 0),
+	SR_FGT(SYS_ICC_ICSR_EL1,		ICH_HFGRTR, ICC_ICSR_EL1, 0),
+	SR_FGT(SYS_ICC_IAFFIDR_EL1,		ICH_HFGRTR, ICC_IAFFIDR_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_HMR0_EL1,		ICH_HFGRTR, ICC_PPI_HMRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_HMR1_EL1,		ICH_HFGRTR, ICC_PPI_HMRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_ENABLER0_EL1,	ICH_HFGRTR, ICC_PPI_ENABLERn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_ENABLER1_EL1,	ICH_HFGRTR, ICC_PPI_ENABLERn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_CPENDR0_EL1,		ICH_HFGRTR, ICC_PPI_PENDRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_CPENDR1_EL1,		ICH_HFGRTR, ICC_PPI_PENDRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_SPENDR0_EL1,		ICH_HFGRTR, ICC_PPI_PENDRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_SPENDR1_EL1,		ICH_HFGRTR, ICC_PPI_PENDRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR0_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR1_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR2_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR3_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR4_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR5_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR6_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR7_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR8_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR9_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR10_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR11_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR12_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR13_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR14_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_PRIORITYR15_EL1,	ICH_HFGRTR, ICC_PPI_PRIORITYRn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_CACTIVER0_EL1,	ICH_HFGRTR, ICC_PPI_ACTIVERn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_CACTIVER1_EL1,	ICH_HFGRTR, ICC_PPI_ACTIVERn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_SACTIVER0_EL1,	ICH_HFGRTR, ICC_PPI_ACTIVERn_EL1, 0),
+	SR_FGT(SYS_ICC_PPI_SACTIVER1_EL1,	ICH_HFGRTR, ICC_PPI_ACTIVERn_EL1, 0),
+
+	/*
+	 * ICH_HFGITR_EL2
+	 */
+	SR_FGT(GICV5_OP_GIC_CDEN,	ICH_HFGITR, GICCDEN, 0),
+	SR_FGT(GICV5_OP_GIC_CDDIS,	ICH_HFGITR, GICCDDIS, 0),
+	SR_FGT(GICV5_OP_GIC_CDPRI,	ICH_HFGITR, GICCDPRI, 0),
+	SR_FGT(GICV5_OP_GIC_CDAFF,	ICH_HFGITR, GICCDAFF, 0),
+	SR_FGT(GICV5_OP_GIC_CDPEND,	ICH_HFGITR, GICCDPEND, 0),
+	SR_FGT(GICV5_OP_GIC_CDRCFG,	ICH_HFGITR, GICCDRCFG, 0),
+	SR_FGT(GICV5_OP_GIC_CDHM,	ICH_HFGITR, GICCDHM, 0),
+	SR_FGT(GICV5_OP_GIC_CDEOI,	ICH_HFGITR, GICCDEOI, 0),
+	SR_FGT(GICV5_OP_GIC_CDDI,	ICH_HFGITR, GICCDDI, 0),
+	SR_FGT(GICV5_OP_GICR_CDIA,	ICH_HFGITR, GICRCDIA, 0),
+	SR_FGT(GICV5_OP_GICR_CDNMIA,	ICH_HFGITR, GICRCDNMIA, 0),
 };
 
 /*
@@ -2127,6 +2181,9 @@ FGT_MASKS(hfgwtr2_masks, HFGWTR2_EL2);
 FGT_MASKS(hfgitr2_masks, HFGITR2_EL2);
 FGT_MASKS(hdfgrtr2_masks, HDFGRTR2_EL2);
 FGT_MASKS(hdfgwtr2_masks, HDFGWTR2_EL2);
+FGT_MASKS(ich_hfgrtr_masks, ICH_HFGRTR_EL2);
+FGT_MASKS(ich_hfgwtr_masks, ICH_HFGWTR_EL2);
+FGT_MASKS(ich_hfgitr_masks, ICH_HFGITR_EL2);
 
 static __init bool aggregate_fgt(union trap_config tc)
 {
@@ -2162,6 +2219,14 @@ static __init bool aggregate_fgt(union trap_config tc)
 		rmasks = &hfgitr2_masks;
 		wmasks = NULL;
 		break;
+	case ICH_HFGRTR_GROUP:
+		rmasks = &ich_hfgrtr_masks;
+		wmasks = &ich_hfgwtr_masks;
+		break;
+	case ICH_HFGITR_GROUP:
+		rmasks = &ich_hfgitr_masks;
+		wmasks = NULL;
+		break;
 	}
 
 	rresx = rmasks->res0 | rmasks->res1;
@@ -2232,6 +2297,9 @@ static __init int check_all_fgt_masks(int ret)
 		&hfgitr2_masks,
 		&hdfgrtr2_masks,
 		&hdfgwtr2_masks,
+		&ich_hfgrtr_masks,
+		&ich_hfgwtr_masks,
+		&ich_hfgitr_masks,
 	};
 	int err = 0;
 
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index cc7d5d1709cb8..54aedf93c78b6 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -539,7 +539,7 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr,
 
 		/* All hyp bugs, including warnings, are treated as fatal. */
 		if (!is_protected_kvm_enabled() ||
-		    IS_ENABLED(CONFIG_NVHE_EL2_DEBUG)) {
+		    IS_ENABLED(CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC)) {
 			struct bug_entry *bug = find_bug(elr_in_kimg);
 
 			if (bug)
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 2597e8bda8672..ae04fd680d1e2 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -233,6 +233,18 @@ static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu)
 	__activate_fgt(hctxt, vcpu, HDFGWTR2_EL2);
 }
 
+static inline void __activate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu)
+{
+	struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
+
+	if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
+		return;
+
+	__activate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2);
+	__activate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2);
+	__activate_fgt(hctxt, vcpu, ICH_HFGITR_EL2);
+}
+
 #define __deactivate_fgt(htcxt, vcpu, reg)				\
 	do {								\
 		write_sysreg_s(ctxt_sys_reg(hctxt, reg),		\
@@ -265,6 +277,19 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu)
 	__deactivate_fgt(hctxt, vcpu, HDFGWTR2_EL2);
 }
 
+static inline void __deactivate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu)
+{
+	struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
+
+	if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
+		return;
+
+	__deactivate_fgt(hctxt, vcpu, ICH_HFGRTR_EL2);
+	__deactivate_fgt(hctxt, vcpu, ICH_HFGWTR_EL2);
+	__deactivate_fgt(hctxt, vcpu, ICH_HFGITR_EL2);
+
+}
+
 static inline void  __activate_traps_mpam(struct kvm_vcpu *vcpu)
 {
 	u64 r = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1;
@@ -328,6 +353,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
 	}
 
 	__activate_traps_hfgxtr(vcpu);
+	__activate_traps_ich_hfgxtr(vcpu);
 	__activate_traps_mpam(vcpu);
 }
 
@@ -345,6 +371,7 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu)
 		write_sysreg_s(ctxt_sys_reg(hctxt, HCRX_EL2), SYS_HCRX_EL2);
 
 	__deactivate_traps_hfgxtr(vcpu);
+	__deactivate_traps_ich_hfgxtr(vcpu);
 	__deactivate_traps_mpam();
 }
 
diff --git a/arch/arm64/kvm/hyp/include/nvhe/arm-smccc.h b/arch/arm64/kvm/hyp/include/nvhe/arm-smccc.h
new file mode 100644
index 0000000000000..1258bc84477f8
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/arm-smccc.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ARM64_KVM_HYP_NVHE_ARM_SMCCC_H__
+#define __ARM64_KVM_HYP_NVHE_ARM_SMCCC_H__
+
+#include <asm/kvm_hypevents.h>
+
+#include <linux/arm-smccc.h>
+
+#define hyp_smccc_1_1_smc(...)					\
+	do {							\
+		trace_hyp_exit(NULL, HYP_REASON_SMC);		\
+		arm_smccc_1_1_smc(__VA_ARGS__);			\
+		trace_hyp_enter(NULL, HYP_REASON_SMC);		\
+	} while (0)
+
+#define hyp_smccc_1_2_smc(...)					\
+	do {							\
+		trace_hyp_exit(NULL, HYP_REASON_SMC);		\
+		arm_smccc_1_2_smc(__VA_ARGS__);			\
+		trace_hyp_enter(NULL, HYP_REASON_SMC);		\
+	} while (0)
+
+#endif /* __ARM64_KVM_HYP_NVHE_ARM_SMCCC_H__ */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/clock.h b/arch/arm64/kvm/hyp/include/nvhe/clock.h
new file mode 100644
index 0000000000000..9f429f5c06640
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/clock.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ARM64_KVM_HYP_NVHE_CLOCK_H
+#define __ARM64_KVM_HYP_NVHE_CLOCK_H
+#include <linux/types.h>
+
+#include <asm/kvm_hyp.h>
+
+#ifdef CONFIG_NVHE_EL2_TRACING
+void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
+u64 trace_clock(void);
+#else
+static inline void
+trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
+static inline u64 trace_clock(void) { return 0; }
+#endif
+#endif
diff --git a/arch/arm64/kvm/hyp/include/nvhe/define_events.h b/arch/arm64/kvm/hyp/include/nvhe/define_events.h
new file mode 100644
index 0000000000000..776d4c6cb702b
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/define_events.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef HYP_EVENT
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)	\
+	struct hyp_event_id hyp_event_id_##__name			\
+	__section(".hyp.event_ids."#__name) = {				\
+		.enabled = ATOMIC_INIT(0),				\
+	}
+
+#define HYP_EVENT_MULTI_READ
+#include <asm/kvm_hypevents.h>
+#undef HYP_EVENT_MULTI_READ
+
+#undef HYP_EVENT
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
index 5f9d56754e39f..f8a7b8c04c490 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
@@ -30,8 +30,6 @@ enum pkvm_component_id {
 	PKVM_ID_FFA,
 };
 
-extern unsigned long hyp_nr_cpus;
-
 int __pkvm_prot_finalize(void);
 int __pkvm_host_share_hyp(u64 pfn);
 int __pkvm_host_unshare_hyp(u64 pfn);
diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h
new file mode 100644
index 0000000000000..8813ff250f8e0
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ARM64_KVM_HYP_NVHE_TRACE_H
+#define __ARM64_KVM_HYP_NVHE_TRACE_H
+
+#include <linux/trace_remote_event.h>
+
+#include <asm/kvm_hyptrace.h>
+
+static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt)
+{
+	struct kvm_vcpu *vcpu;
+
+	if (!host_ctxt)
+		host_ctxt = host_data_ptr(host_ctxt);
+
+	vcpu = host_ctxt->__hyp_running_vcpu;
+
+	return vcpu ? vcpu->arch.pid : 0;
+}
+
+#define HE_PROTO(__args...)	__args
+#define HE_ASSIGN(__args...)	__args
+#define HE_STRUCT		RE_STRUCT
+#define he_field		re_field
+
+#ifdef CONFIG_NVHE_EL2_TRACING
+
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)		\
+	REMOTE_EVENT_FORMAT(__name, __struct);					\
+	extern struct hyp_event_id hyp_event_id_##__name;			\
+	static __always_inline void trace_##__name(__proto)			\
+	{									\
+		struct remote_event_format_##__name *__entry;			\
+		size_t length = sizeof(*__entry);				\
+										\
+		if (!atomic_read(&hyp_event_id_##__name.enabled))		\
+			return;							\
+		__entry = tracing_reserve_entry(length);			\
+		if (!__entry)							\
+			return;							\
+		__entry->hdr.id = hyp_event_id_##__name.id;			\
+		__assign							\
+		tracing_commit_entry();						\
+	}
+
+void *tracing_reserve_entry(unsigned long length);
+void tracing_commit_entry(void);
+
+int __tracing_load(unsigned long desc_va, size_t desc_size);
+void __tracing_unload(void);
+int __tracing_enable(bool enable);
+int __tracing_swap_reader(unsigned int cpu);
+void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
+int __tracing_reset(unsigned int cpu);
+int __tracing_enable_event(unsigned short id, bool enable);
+#else
+static inline void *tracing_reserve_entry(unsigned long length) { return NULL; }
+static inline void tracing_commit_entry(void) { }
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)      \
+	static inline void trace_##__name(__proto) {}
+
+static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; }
+static inline void __tracing_unload(void) { }
+static inline int __tracing_enable(bool enable) { return -ENODEV; }
+static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; }
+static inline void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
+static inline int __tracing_reset(unsigned int cpu) { return -ENODEV; }
+static inline int __tracing_enable_event(unsigned short id, bool enable)  { return -ENODEV; }
+#endif
+#endif
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index a244ec25f8c5b..62cdfbff75625 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -17,7 +17,7 @@ ccflags-y += -fno-stack-protector	\
 hostprogs := gen-hyprel
 HOST_EXTRACFLAGS += -I$(objtree)/include
 
-lib-objs := clear_page.o copy_page.o memcpy.o memset.o
+lib-objs := clear_page.o copy_page.o memcpy.o memset.o tishift.o
 lib-objs := $(addprefix ../../../lib/, $(lib-objs))
 
 CFLAGS_switch.nvhe.o += -Wno-override-init
@@ -26,11 +26,15 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
 	 hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \
 	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
 hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
-	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
+	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o
 hyp-obj-y += ../../../kernel/smccc-call.o
 hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
+hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o
 hyp-obj-y += $(lib-objs)
 
+# Path to simple_ring_buffer.c
+CFLAGS_trace.nvhe.o += -I$(srctree)/kernel/trace/
+
 ##
 ## Build rules for compiling nVHE hyp code
 ## Output of this folder is `kvm_nvhe.o`, a partially linked object
diff --git a/arch/arm64/kvm/hyp/nvhe/clock.c b/arch/arm64/kvm/hyp/nvhe/clock.c
new file mode 100644
index 0000000000000..32fc4313fe432
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/clock.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <nvhe/clock.h>
+
+#include <asm/arch_timer.h>
+#include <asm/div64.h>
+
+static struct clock_data {
+	struct {
+		u32 mult;
+		u32 shift;
+		u64 epoch_ns;
+		u64 epoch_cyc;
+		u64 cyc_overflow64;
+	} data[2];
+	u64 cur;
+} trace_clock_data;
+
+static u64 __clock_mult_uint128(u64 cyc, u32 mult, u32 shift)
+{
+	__uint128_t ns = (__uint128_t)cyc * mult;
+
+	ns >>= shift;
+
+	return (u64)ns;
+}
+
+/* Does not guarantee no reader on the modified bank. */
+void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
+{
+	struct clock_data *clock = &trace_clock_data;
+	u64 bank = clock->cur ^ 1;
+
+	clock->data[bank].mult			= mult;
+	clock->data[bank].shift			= shift;
+	clock->data[bank].epoch_ns		= epoch_ns;
+	clock->data[bank].epoch_cyc		= epoch_cyc;
+	clock->data[bank].cyc_overflow64	= ULONG_MAX / mult;
+
+	smp_store_release(&clock->cur, bank);
+}
+
+/* Use untrusted host data */
+u64 trace_clock(void)
+{
+	struct clock_data *clock = &trace_clock_data;
+	u64 bank = smp_load_acquire(&clock->cur);
+	u64 cyc, ns;
+
+	cyc = __arch_counter_get_cntvct() - clock->data[bank].epoch_cyc;
+
+	if (likely(cyc < clock->data[bank].cyc_overflow64)) {
+		ns = cyc * clock->data[bank].mult;
+		ns >>= clock->data[bank].shift;
+	} else {
+		ns = __clock_mult_uint128(cyc, clock->data[bank].mult,
+					  clock->data[bank].shift);
+	}
+
+	return (u64)ns + clock->data[bank].epoch_ns;
+}
diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c
new file mode 100644
index 0000000000000..add9383aadb5a
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/events.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <nvhe/mm.h>
+#include <nvhe/trace.h>
+
+#include <nvhe/define_events.h>
+
+int __tracing_enable_event(unsigned short id, bool enable)
+{
+	struct hyp_event_id *event_id = &__hyp_event_ids_start[id];
+	atomic_t *enabled;
+
+	if (event_id >= __hyp_event_ids_end)
+		return -EINVAL;
+
+	enabled = hyp_fixmap_map(__hyp_pa(&event_id->enabled));
+	atomic_set(enabled, enable);
+	hyp_fixmap_unmap();
+
+	return 0;
+}
diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c
index 94161ea1cd60b..1af722771178a 100644
--- a/arch/arm64/kvm/hyp/nvhe/ffa.c
+++ b/arch/arm64/kvm/hyp/nvhe/ffa.c
@@ -26,10 +26,10 @@
  * the duration and are therefore serialised.
  */
 
-#include <linux/arm-smccc.h>
 #include <linux/arm_ffa.h>
 #include <asm/kvm_pkvm.h>
 
+#include <nvhe/arm-smccc.h>
 #include <nvhe/ffa.h>
 #include <nvhe/mem_protect.h>
 #include <nvhe/memory.h>
@@ -147,7 +147,7 @@ static int ffa_map_hyp_buffers(u64 ffa_page_count)
 {
 	struct arm_smccc_1_2_regs res;
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_FN64_RXTX_MAP,
 		.a1 = hyp_virt_to_phys(hyp_buffers.tx),
 		.a2 = hyp_virt_to_phys(hyp_buffers.rx),
@@ -161,7 +161,7 @@ static int ffa_unmap_hyp_buffers(void)
 {
 	struct arm_smccc_1_2_regs res;
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_RXTX_UNMAP,
 		.a1 = HOST_FFA_ID,
 	}, &res);
@@ -172,7 +172,7 @@ static int ffa_unmap_hyp_buffers(void)
 static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 			     u32 handle_hi, u32 fraglen, u32 endpoint_id)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_MEM_FRAG_TX,
 		.a1 = handle_lo,
 		.a2 = handle_hi,
@@ -184,7 +184,7 @@ static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 			     u32 handle_hi, u32 fragoff)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_MEM_FRAG_RX,
 		.a1 = handle_lo,
 		.a2 = handle_hi,
@@ -196,7 +196,7 @@ static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len,
 			  u32 fraglen)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = func_id,
 		.a1 = len,
 		.a2 = fraglen,
@@ -206,7 +206,7 @@ static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len,
 static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 			     u32 handle_hi, u32 flags)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_MEM_RECLAIM,
 		.a1 = handle_lo,
 		.a2 = handle_hi,
@@ -216,7 +216,7 @@ static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo,
 
 static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_FN64_MEM_RETRIEVE_REQ,
 		.a1 = len,
 		.a2 = len,
@@ -225,7 +225,7 @@ static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len)
 
 static void ffa_rx_release(struct arm_smccc_1_2_regs *res)
 {
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_RX_RELEASE,
 	}, res);
 }
@@ -728,7 +728,7 @@ static int hyp_ffa_post_init(void)
 	size_t min_rxtx_sz;
 	struct arm_smccc_1_2_regs res;
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
 		.a0 = FFA_ID_GET,
 	}, &res);
 	if (res.a0 != FFA_SUCCESS)
@@ -737,7 +737,7 @@ static int hyp_ffa_post_init(void)
 	if (res.a2 != HOST_FFA_ID)
 		return -EINVAL;
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
 		.a0 = FFA_FEATURES,
 		.a1 = FFA_FN64_RXTX_MAP,
 	}, &res);
@@ -788,7 +788,7 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res,
 	 * first if TEE supports it.
 	 */
 	if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) {
-		arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+		hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 			.a0 = FFA_VERSION,
 			.a1 = ffa_req_version,
 		}, res);
@@ -824,7 +824,7 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
 		goto out_unlock;
 	}
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_PARTITION_INFO_GET,
 		.a1 = uuid0,
 		.a2 = uuid1,
@@ -939,7 +939,7 @@ int hyp_ffa_init(void *pages)
 	if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2)
 		return 0;
 
-	arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
+	hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
 		.a0 = FFA_VERSION,
 		.a1 = FFA_VERSION_1_2,
 	}, &res);
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index eef15b374abb0..3092befcd97ce 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -120,7 +120,7 @@ SYM_FUNC_START(__hyp_do_panic)
 
 	mov	x29, x0
 
-#ifdef CONFIG_NVHE_EL2_DEBUG
+#ifdef PKVM_DISABLE_STAGE2_ON_PANIC
 	/* Ensure host stage-2 is disabled */
 	mrs	x0, hcr_el2
 	bic	x0, x0, #HCR_VM
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index e7790097db93a..31f080307d959 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -12,12 +12,14 @@
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_hypevents.h>
 #include <asm/kvm_mmu.h>
 
 #include <nvhe/ffa.h>
 #include <nvhe/mem_protect.h>
 #include <nvhe/mm.h>
 #include <nvhe/pkvm.h>
+#include <nvhe/trace.h>
 #include <nvhe/trap_handler.h>
 
 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
@@ -136,6 +138,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	hyp_vcpu->vcpu.arch.vsesr_el2	= host_vcpu->arch.vsesr_el2;
 
 	hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3;
+
+	hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid;
 }
 
 static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
@@ -486,17 +490,15 @@ static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
 {
 	DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
 	DECLARE_REG(unsigned long, size, host_ctxt, 2);
-	DECLARE_REG(unsigned long, nr_cpus, host_ctxt, 3);
-	DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 4);
-	DECLARE_REG(u32, hyp_va_bits, host_ctxt, 5);
+	DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3);
+	DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4);
 
 	/*
 	 * __pkvm_init() will return only if an error occurred, otherwise it
 	 * will tail-call in __pkvm_init_finalise() which will have to deal
 	 * with the host context directly.
 	 */
-	cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, nr_cpus, per_cpu_base,
-					    hyp_va_bits);
+	cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits);
 }
 
 static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt)
@@ -589,6 +591,79 @@ static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt)
 	cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle);
 }
 
+static void handle___tracing_load(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1);
+	DECLARE_REG(size_t, desc_size, host_ctxt, 2);
+
+	cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size);
+}
+
+static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt)
+{
+	__tracing_unload();
+}
+
+static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(bool, enable, host_ctxt, 1);
+
+	cpu_reg(host_ctxt, 1) = __tracing_enable(enable);
+}
+
+static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned int, cpu, host_ctxt, 1);
+
+	cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu);
+}
+
+static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(u32, mult, host_ctxt, 1);
+	DECLARE_REG(u32, shift, host_ctxt, 2);
+	DECLARE_REG(u64, epoch_ns, host_ctxt, 3);
+	DECLARE_REG(u64, epoch_cyc, host_ctxt, 4);
+
+	__tracing_update_clock(mult, shift, epoch_ns, epoch_cyc);
+}
+
+static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned int, cpu, host_ctxt, 1);
+
+	cpu_reg(host_ctxt, 1) = __tracing_reset(cpu);
+}
+
+static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned short, id, host_ctxt, 1);
+	DECLARE_REG(bool, enable, host_ctxt, 2);
+
+	cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable);
+}
+
+static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(u64, id, host_ctxt, 1);
+
+	trace_selftest(id);
+}
+
+static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1);
+
+	__vgic_v5_save_apr(kern_hyp_va(cpu_if));
+}
+
+static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1);
+
+	__vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if));
+}
+
 typedef void (*hcall_t)(struct kvm_cpu_context *);
 
 #define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -622,6 +697,8 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__kvm_timer_set_cntvoff),
 	HANDLE_FUNC(__vgic_v3_save_aprs),
 	HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs),
+	HANDLE_FUNC(__vgic_v5_save_apr),
+	HANDLE_FUNC(__vgic_v5_restore_vmcr_apr),
 	HANDLE_FUNC(__pkvm_reserve_vm),
 	HANDLE_FUNC(__pkvm_unreserve_vm),
 	HANDLE_FUNC(__pkvm_init_vm),
@@ -630,6 +707,14 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__pkvm_vcpu_load),
 	HANDLE_FUNC(__pkvm_vcpu_put),
 	HANDLE_FUNC(__pkvm_tlb_flush_vmid),
+	HANDLE_FUNC(__tracing_load),
+	HANDLE_FUNC(__tracing_unload),
+	HANDLE_FUNC(__tracing_enable),
+	HANDLE_FUNC(__tracing_swap_reader),
+	HANDLE_FUNC(__tracing_update_clock),
+	HANDLE_FUNC(__tracing_reset),
+	HANDLE_FUNC(__tracing_enable_event),
+	HANDLE_FUNC(__tracing_write_event),
 };
 
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
@@ -670,7 +755,9 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
 
 static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt)
 {
+	trace_hyp_exit(host_ctxt, HYP_REASON_SMC);
 	__kvm_hyp_host_forward_smc(host_ctxt);
+	trace_hyp_enter(host_ctxt, HYP_REASON_SMC);
 }
 
 static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
@@ -757,15 +844,19 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
 {
 	u64 esr = read_sysreg_el2(SYS_ESR);
 
+
 	switch (ESR_ELx_EC(esr)) {
 	case ESR_ELx_EC_HVC64:
+		trace_hyp_enter(host_ctxt, HYP_REASON_HVC);
 		handle_host_hcall(host_ctxt);
 		break;
 	case ESR_ELx_EC_SMC64:
+		trace_hyp_enter(host_ctxt, HYP_REASON_SMC);
 		handle_host_smc(host_ctxt);
 		break;
 	case ESR_ELx_EC_IABT_LOW:
 	case ESR_ELx_EC_DABT_LOW:
+		trace_hyp_enter(host_ctxt, HYP_REASON_HOST_ABORT);
 		handle_host_mem_abort(host_ctxt);
 		break;
 	case ESR_ELx_EC_SYS64:
@@ -775,4 +866,6 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
 	default:
 		BUG();
 	}
+
+	trace_hyp_exit(host_ctxt, HYP_REASON_ERET_HOST);
 }
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index d724f6d69302a..7a02837203d15 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -16,6 +16,12 @@ SECTIONS {
 	HYP_SECTION(.text)
 	HYP_SECTION(.data..ro_after_init)
 	HYP_SECTION(.rodata)
+#ifdef CONFIG_NVHE_EL2_TRACING
+	. = ALIGN(PAGE_SIZE);
+	BEGIN_HYP_SECTION(.event_ids)
+		*(SORT(.hyp.event_ids.*))
+	END_HYP_SECTION
+#endif
 
 	/*
 	 * .hyp..data..percpu needs to be page aligned to maintain the same
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 218976287d3fe..3b0bee496bffb 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -244,7 +244,7 @@ static void *fixmap_map_slot(struct hyp_fixmap_slot *slot, phys_addr_t phys)
 
 void *hyp_fixmap_map(phys_addr_t phys)
 {
-	return fixmap_map_slot(this_cpu_ptr(&fixmap_slots), phys);
+	return fixmap_map_slot(this_cpu_ptr(&fixmap_slots), phys) + offset_in_page(phys);
 }
 
 static void fixmap_clear_slot(struct hyp_fixmap_slot *slot)
@@ -270,7 +270,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *slot)
 	 * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#mf10dfbaf1eaef9274c581b81c53758918c1d0f03
 	 */
 	dsb(ishst);
-	__tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level);
+	__tlbi_level(vale2is, addr, level);
 	__tlbi_sync_s1ish_hyp();
 	isb();
 }
@@ -366,7 +366,7 @@ void *hyp_fixblock_map(phys_addr_t phys, size_t *size)
 #ifdef HAS_FIXBLOCK
 	*size = PMD_SIZE;
 	hyp_spin_lock(&hyp_fixblock_lock);
-	return fixmap_map_slot(&hyp_fixblock_slot, phys);
+	return fixmap_map_slot(&hyp_fixblock_slot, phys) + offset_in_page(phys);
 #else
 	*size = PAGE_SIZE;
 	return hyp_fixmap_map(phys);
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index c3e196fb8b18f..ab4c7bddb1636 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -6,11 +6,12 @@
 
 #include <asm/kvm_asm.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_hypevents.h>
 #include <asm/kvm_mmu.h>
-#include <linux/arm-smccc.h>
 #include <linux/kvm_host.h>
 #include <uapi/linux/psci.h>
 
+#include <nvhe/arm-smccc.h>
 #include <nvhe/memory.h>
 #include <nvhe/trap_handler.h>
 
@@ -65,7 +66,7 @@ static unsigned long psci_call(unsigned long fn, unsigned long arg0,
 {
 	struct arm_smccc_res res;
 
-	arm_smccc_1_1_smc(fn, arg0, arg1, arg2, &res);
+	hyp_smccc_1_1_smc(fn, arg0, arg1, arg2, &res);
 	return res.a0;
 }
 
@@ -206,6 +207,7 @@ asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on)
 	struct kvm_cpu_context *host_ctxt;
 
 	host_ctxt = host_data_ptr(host_ctxt);
+	trace_hyp_enter(host_ctxt, HYP_REASON_PSCI);
 
 	if (is_cpu_on)
 		boot_args = this_cpu_ptr(&cpu_on_args);
@@ -221,6 +223,7 @@ asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on)
 	write_sysreg_el1(INIT_SCTLR_EL1_MMU_OFF, SYS_SCTLR);
 	write_sysreg(INIT_PSTATE_EL1, SPSR_EL2);
 
+	trace_hyp_exit(host_ctxt, HYP_REASON_PSCI);
 	__host_enter(host_ctxt);
 }
 
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 90bd014e952fb..d8e5b563fd3d9 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -341,8 +341,7 @@ void __noreturn __pkvm_init_finalise(void)
 	__host_enter(host_ctxt);
 }
 
-int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
-		unsigned long *per_cpu_base, u32 hyp_va_bits)
+int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_base, u32 hyp_va_bits)
 {
 	struct kvm_nvhe_init_params *params;
 	void *virt = hyp_phys_to_virt(phys);
@@ -355,7 +354,6 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
 		return -EINVAL;
 
 	hyp_spin_lock_init(&pkvm_pgd_lock);
-	hyp_nr_cpus = nr_cpus;
 
 	ret = divide_memory_pool(virt, size);
 	if (ret)
diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
index 5b6eeab1a7743..7c832d60d22bb 100644
--- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c
+++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
@@ -34,7 +34,7 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc)
 	stacktrace_info->pc = pc;
 }
 
-#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
+#ifdef CONFIG_PKVM_STACKTRACE
 #include <asm/stacktrace/nvhe.h>
 
 DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace);
@@ -134,11 +134,11 @@ static void pkvm_save_backtrace(unsigned long fp, unsigned long pc)
 
 	unwind(&state, pkvm_save_backtrace_entry, &idx);
 }
-#else /* !CONFIG_PROTECTED_NVHE_STACKTRACE */
+#else /* !CONFIG_PKVM_STACKTRACE */
 static void pkvm_save_backtrace(unsigned long fp, unsigned long pc)
 {
 }
-#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
+#endif /* CONFIG_PKVM_STACKTRACE */
 
 /*
  * kvm_nvhe_prepare_backtrace - prepare to dump the nVHE backtrace
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 779089e42681e..ed8830c707b0c 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -7,7 +7,6 @@
 #include <hyp/switch.h>
 #include <hyp/sysreg-sr.h>
 
-#include <linux/arm-smccc.h>
 #include <linux/kvm_host.h>
 #include <linux/types.h>
 #include <linux/jump_label.h>
@@ -21,6 +20,7 @@
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_hypevents.h>
 #include <asm/kvm_mmu.h>
 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
@@ -44,6 +44,9 @@ struct fgt_masks hfgwtr2_masks;
 struct fgt_masks hfgitr2_masks;
 struct fgt_masks hdfgrtr2_masks;
 struct fgt_masks hdfgwtr2_masks;
+struct fgt_masks ich_hfgrtr_masks;
+struct fgt_masks ich_hfgwtr_masks;
+struct fgt_masks ich_hfgitr_masks;
 
 extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc);
 
@@ -110,6 +113,12 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
 /* Save VGICv3 state on non-VHE systems */
 static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
 {
+	if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
+		__vgic_v5_save_state(&vcpu->arch.vgic_cpu.vgic_v5);
+		__vgic_v5_save_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5);
+		return;
+	}
+
 	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
 		__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
 		__vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
@@ -119,6 +128,12 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
 /* Restore VGICv3 state on non-VHE systems */
 static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
 {
+	if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
+		__vgic_v5_restore_state(&vcpu->arch.vgic_cpu.vgic_v5);
+		__vgic_v5_restore_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5);
+		return;
+	}
+
 	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
 		__vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
 		__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
@@ -308,10 +323,13 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 	__debug_switch_to_guest(vcpu);
 
 	do {
+		trace_hyp_exit(host_ctxt, HYP_REASON_ERET_GUEST);
+
 		/* Jump in the fire! */
 		exit_code = __guest_enter(vcpu);
 
 		/* And we're baaack! */
+		trace_hyp_enter(host_ctxt, HYP_REASON_GUEST_EXIT);
 	} while (fixup_guest_exit(vcpu, &exit_code));
 
 	__sysreg_save_state_nvhe(guest_ctxt);
diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
index 06d28621722ee..b40fd01ebf329 100644
--- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c
+++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
@@ -20,6 +20,7 @@
  */
 u64 id_aa64pfr0_el1_sys_val;
 u64 id_aa64pfr1_el1_sys_val;
+u64 id_aa64pfr2_el1_sys_val;
 u64 id_aa64isar0_el1_sys_val;
 u64 id_aa64isar1_el1_sys_val;
 u64 id_aa64isar2_el1_sys_val;
@@ -108,6 +109,11 @@ static const struct pvm_ftr_bits pvmid_aa64pfr1[] = {
 	FEAT_END
 };
 
+static const struct pvm_ftr_bits pvmid_aa64pfr2[] = {
+	MAX_FEAT(ID_AA64PFR2_EL1, GCIE, NI),
+	FEAT_END
+};
+
 static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = {
 	MAX_FEAT_ENUM(ID_AA64MMFR0_EL1, PARANGE, 40),
 	MAX_FEAT_ENUM(ID_AA64MMFR0_EL1, ASIDBITS, 16),
@@ -221,6 +227,8 @@ static u64 pvm_calc_id_reg(const struct kvm_vcpu *vcpu, u32 id)
 		return get_restricted_features(vcpu, id_aa64pfr0_el1_sys_val, pvmid_aa64pfr0);
 	case SYS_ID_AA64PFR1_EL1:
 		return get_restricted_features(vcpu, id_aa64pfr1_el1_sys_val, pvmid_aa64pfr1);
+	case SYS_ID_AA64PFR2_EL1:
+		return get_restricted_features(vcpu, id_aa64pfr2_el1_sys_val, pvmid_aa64pfr2);
 	case SYS_ID_AA64ISAR0_EL1:
 		return id_aa64isar0_el1_sys_val;
 	case SYS_ID_AA64ISAR1_EL1:
diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c
index 3dc1ce0d27fe6..b29140995d484 100644
--- a/arch/arm64/kvm/hyp/nvhe/tlb.c
+++ b/arch/arm64/kvm/hyp/nvhe/tlb.c
@@ -158,7 +158,6 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
 	 * Instead, we invalidate Stage-2 for this IPA, and the
 	 * whole of Stage-1. Weep...
 	 */
-	ipa >>= 12;
 	__tlbi_level(ipas2e1is, ipa, level);
 
 	/*
@@ -188,7 +187,6 @@ void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,
 	 * Instead, we invalidate Stage-2 for this IPA, and the
 	 * whole of Stage-1. Weep...
 	 */
-	ipa >>= 12;
 	__tlbi_level(ipas2e1, ipa, level);
 
 	/*
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c
new file mode 100644
index 0000000000000..a6ca27b18e154
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -0,0 +1,306 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <nvhe/clock.h>
+#include <nvhe/mem_protect.h>
+#include <nvhe/mm.h>
+#include <nvhe/trace.h>
+
+#include <asm/percpu.h>
+#include <asm/kvm_mmu.h>
+#include <asm/local.h>
+
+#include "simple_ring_buffer.c"
+
+static DEFINE_PER_CPU(struct simple_rb_per_cpu, __simple_rbs);
+
+static struct hyp_trace_buffer {
+	struct simple_rb_per_cpu __percpu	*simple_rbs;
+	void					*bpages_backing_start;
+	size_t					bpages_backing_size;
+	hyp_spinlock_t				lock;
+} trace_buffer = {
+	.simple_rbs = &__simple_rbs,
+	.lock = __HYP_SPIN_LOCK_UNLOCKED,
+};
+
+static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer)
+{
+	return trace_buffer->bpages_backing_size > 0;
+}
+
+void *tracing_reserve_entry(unsigned long length)
+{
+	return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length,
+					  trace_clock());
+}
+
+void tracing_commit_entry(void)
+{
+	simple_ring_buffer_commit(this_cpu_ptr(trace_buffer.simple_rbs));
+}
+
+static int __admit_host_mem(void *start, u64 size)
+{
+	if (!PAGE_ALIGNED(start) || !PAGE_ALIGNED(size) || !size)
+		return -EINVAL;
+
+	if (!is_protected_kvm_enabled())
+		return 0;
+
+	return __pkvm_host_donate_hyp(hyp_virt_to_pfn(start), size >> PAGE_SHIFT);
+}
+
+static void __release_host_mem(void *start, u64 size)
+{
+	if (!is_protected_kvm_enabled())
+		return;
+
+	WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT));
+}
+
+static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer,
+					       struct hyp_trace_desc *desc)
+{
+	void *start = (void *)kern_hyp_va(desc->bpages_backing_start);
+	size_t size = desc->bpages_backing_size;
+	int ret;
+
+	ret = __admit_host_mem(start, size);
+	if (ret)
+		return ret;
+
+	memset(start, 0, size);
+
+	trace_buffer->bpages_backing_start = start;
+	trace_buffer->bpages_backing_size = size;
+
+	return 0;
+}
+
+static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer)
+{
+	void *start = trace_buffer->bpages_backing_start;
+	size_t size = trace_buffer->bpages_backing_size;
+
+	if (!size)
+		return;
+
+	memset(start, 0, size);
+
+	__release_host_mem(start, size);
+
+	trace_buffer->bpages_backing_start = 0;
+	trace_buffer->bpages_backing_size = 0;
+}
+
+static void *__pin_shared_page(unsigned long kern_va)
+{
+	void *va = kern_hyp_va((void *)kern_va);
+
+	if (!is_protected_kvm_enabled())
+		return va;
+
+	return hyp_pin_shared_mem(va, va + PAGE_SIZE) ? NULL : va;
+}
+
+static void __unpin_shared_page(void *va)
+{
+	if (!is_protected_kvm_enabled())
+		return;
+
+	hyp_unpin_shared_mem(va, va + PAGE_SIZE);
+}
+
+static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer)
+{
+	int cpu;
+
+	hyp_assert_lock_held(&trace_buffer->lock);
+
+	if (!hyp_trace_buffer_loaded(trace_buffer))
+		return;
+
+	for (cpu = 0; cpu < hyp_nr_cpus; cpu++)
+		simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu),
+					     __unpin_shared_page);
+
+	hyp_trace_buffer_unload_bpage_backing(trace_buffer);
+}
+
+static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
+				 struct hyp_trace_desc *desc)
+{
+	struct simple_buffer_page *bpages;
+	struct ring_buffer_desc *rb_desc;
+	int ret, cpu;
+
+	hyp_assert_lock_held(&trace_buffer->lock);
+
+	if (hyp_trace_buffer_loaded(trace_buffer))
+		return -EINVAL;
+
+	ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc);
+	if (ret)
+		return ret;
+
+	bpages = trace_buffer->bpages_backing_start;
+	for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) {
+		ret = simple_ring_buffer_init_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu),
+						 bpages, rb_desc, __pin_shared_page,
+						__unpin_shared_page);
+		if (ret)
+			break;
+
+		bpages += rb_desc->nr_page_va;
+	}
+
+	if (ret)
+		hyp_trace_buffer_unload(trace_buffer);
+
+	return ret;
+}
+
+static bool hyp_trace_desc_validate(struct hyp_trace_desc *desc, size_t desc_size)
+{
+	struct ring_buffer_desc *rb_desc;
+	unsigned int cpu;
+	size_t nr_bpages;
+	void *desc_end;
+
+	/*
+	 * Both desc_size and bpages_backing_size are untrusted host-provided
+	 * values. We rely on __pkvm_host_donate_hyp() to enforce their validity.
+	 */
+	desc_end = (void *)desc + desc_size;
+	nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page);
+
+	for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) {
+		/* Can we read nr_page_va? */
+		if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end)
+			return false;
+
+		/* Overflow desc? */
+		if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end)
+			return false;
+
+		/* Overflow bpages backing memory? */
+		if (nr_bpages < rb_desc->nr_page_va)
+			return false;
+
+		if (cpu >= hyp_nr_cpus)
+			return false;
+
+		if (cpu != rb_desc->cpu)
+			return false;
+
+		nr_bpages -= rb_desc->nr_page_va;
+	}
+
+	return true;
+}
+
+int __tracing_load(unsigned long desc_hva, size_t desc_size)
+{
+	struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva);
+	int ret;
+
+	ret = __admit_host_mem(desc, desc_size);
+	if (ret)
+		return ret;
+
+	if (!hyp_trace_desc_validate(desc, desc_size))
+		goto err_release_desc;
+
+	hyp_spin_lock(&trace_buffer.lock);
+
+	ret = hyp_trace_buffer_load(&trace_buffer, desc);
+
+	hyp_spin_unlock(&trace_buffer.lock);
+
+err_release_desc:
+	__release_host_mem(desc, desc_size);
+	return ret;
+}
+
+void __tracing_unload(void)
+{
+	hyp_spin_lock(&trace_buffer.lock);
+	hyp_trace_buffer_unload(&trace_buffer);
+	hyp_spin_unlock(&trace_buffer.lock);
+}
+
+int __tracing_enable(bool enable)
+{
+	int cpu, ret = enable ? -EINVAL : 0;
+
+	hyp_spin_lock(&trace_buffer.lock);
+
+	if (!hyp_trace_buffer_loaded(&trace_buffer))
+		goto unlock;
+
+	for (cpu = 0; cpu < hyp_nr_cpus; cpu++)
+		simple_ring_buffer_enable_tracing(per_cpu_ptr(trace_buffer.simple_rbs, cpu),
+						  enable);
+
+	ret = 0;
+
+unlock:
+	hyp_spin_unlock(&trace_buffer.lock);
+
+	return ret;
+}
+
+int __tracing_swap_reader(unsigned int cpu)
+{
+	int ret = -ENODEV;
+
+	if (cpu >= hyp_nr_cpus)
+		return -EINVAL;
+
+	hyp_spin_lock(&trace_buffer.lock);
+
+	if (hyp_trace_buffer_loaded(&trace_buffer))
+		ret = simple_ring_buffer_swap_reader_page(
+				per_cpu_ptr(trace_buffer.simple_rbs, cpu));
+
+	hyp_spin_unlock(&trace_buffer.lock);
+
+	return ret;
+}
+
+void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
+{
+	int cpu;
+
+	/* After this loop, all CPUs are observing the new bank... */
+	for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
+		struct simple_rb_per_cpu *simple_rb = per_cpu_ptr(trace_buffer.simple_rbs, cpu);
+
+		while (READ_ONCE(simple_rb->status) == SIMPLE_RB_WRITING)
+			;
+	}
+
+	/* ...we can now override the old one and swap. */
+	trace_clock_update(mult, shift, epoch_ns, epoch_cyc);
+}
+
+int __tracing_reset(unsigned int cpu)
+{
+	int ret = -ENODEV;
+
+	if (cpu >= hyp_nr_cpus)
+		return -EINVAL;
+
+	hyp_spin_lock(&trace_buffer.lock);
+
+	if (hyp_trace_buffer_loaded(&trace_buffer))
+		ret = simple_ring_buffer_reset(per_cpu_ptr(trace_buffer.simple_rbs, cpu));
+
+	hyp_spin_unlock(&trace_buffer.lock);
+
+	return ret;
+}
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 9b480f947da26..30226f2d5564a 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -490,14 +490,14 @@ static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
 
 		kvm_clear_pte(ctx->ptep);
 		dsb(ishst);
-		__tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN);
+		__tlbi_level(vae2is, ctx->addr, TLBI_TTL_UNKNOWN);
 	} else {
 		if (ctx->end - ctx->addr < granule)
 			return -EINVAL;
 
 		kvm_clear_pte(ctx->ptep);
 		dsb(ishst);
-		__tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
+		__tlbi_level(vale2is, ctx->addr, ctx->level);
 		*unmapped += granule;
 	}
 
diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c
new file mode 100644
index 0000000000000..2c4304ffa9f33
--- /dev/null
+++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025, 2026 - Arm Ltd
+ */
+
+#include <linux/irqchip/arm-gic-v5.h>
+
+#include <asm/kvm_hyp.h>
+
+void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if)
+{
+	cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2);
+}
+
+static void  __vgic_v5_compat_mode_disable(void)
+{
+	sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, ICH_VCTLR_EL2_V3, 0);
+	isb();
+}
+
+void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if)
+{
+	__vgic_v5_compat_mode_disable();
+
+	write_sysreg_s(cpu_if->vgic_vmcr, SYS_ICH_VMCR_EL2);
+	write_sysreg_s(cpu_if->vgic_apr, SYS_ICH_APR_EL2);
+}
+
+void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if)
+{
+	/*
+	 * The following code assumes that the bitmap storage that we have for
+	 * PPIs is either 64 (architected PPIs, only) or 128 bits (architected &
+	 * impdef PPIs).
+	 */
+	BUILD_BUG_ON(VGIC_V5_NR_PRIVATE_IRQS % 64);
+
+	bitmap_write(host_data_ptr(vgic_v5_ppi_state)->activer_exit,
+		     read_sysreg_s(SYS_ICH_PPI_ACTIVER0_EL2), 0, 64);
+	bitmap_write(host_data_ptr(vgic_v5_ppi_state)->pendr_exit,
+		     read_sysreg_s(SYS_ICH_PPI_PENDR0_EL2), 0, 64);
+
+	cpu_if->vgic_ppi_priorityr[0] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR0_EL2);
+	cpu_if->vgic_ppi_priorityr[1] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR1_EL2);
+	cpu_if->vgic_ppi_priorityr[2] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR2_EL2);
+	cpu_if->vgic_ppi_priorityr[3] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR3_EL2);
+	cpu_if->vgic_ppi_priorityr[4] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR4_EL2);
+	cpu_if->vgic_ppi_priorityr[5] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR5_EL2);
+	cpu_if->vgic_ppi_priorityr[6] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR6_EL2);
+	cpu_if->vgic_ppi_priorityr[7] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR7_EL2);
+
+	if (VGIC_V5_NR_PRIVATE_IRQS == 128) {
+		bitmap_write(host_data_ptr(vgic_v5_ppi_state)->activer_exit,
+			     read_sysreg_s(SYS_ICH_PPI_ACTIVER1_EL2), 64, 64);
+		bitmap_write(host_data_ptr(vgic_v5_ppi_state)->pendr_exit,
+			     read_sysreg_s(SYS_ICH_PPI_PENDR1_EL2), 64, 64);
+
+		cpu_if->vgic_ppi_priorityr[8] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR8_EL2);
+		cpu_if->vgic_ppi_priorityr[9] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR9_EL2);
+		cpu_if->vgic_ppi_priorityr[10] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR10_EL2);
+		cpu_if->vgic_ppi_priorityr[11] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR11_EL2);
+		cpu_if->vgic_ppi_priorityr[12] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR12_EL2);
+		cpu_if->vgic_ppi_priorityr[13] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR13_EL2);
+		cpu_if->vgic_ppi_priorityr[14] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR14_EL2);
+		cpu_if->vgic_ppi_priorityr[15] = read_sysreg_s(SYS_ICH_PPI_PRIORITYR15_EL2);
+	}
+
+	/* Now that we are done, disable DVI */
+	write_sysreg_s(0, SYS_ICH_PPI_DVIR0_EL2);
+	write_sysreg_s(0, SYS_ICH_PPI_DVIR1_EL2);
+}
+
+void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if)
+{
+	DECLARE_BITMAP(pendr, VGIC_V5_NR_PRIVATE_IRQS);
+
+	/* We assume 64 or 128 PPIs - see above comment */
+	BUILD_BUG_ON(VGIC_V5_NR_PRIVATE_IRQS % 64);
+
+	/* Enable DVI so that the guest's interrupt config takes over */
+	write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_dvir, 0, 64),
+		       SYS_ICH_PPI_DVIR0_EL2);
+
+	write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_activer, 0, 64),
+		       SYS_ICH_PPI_ACTIVER0_EL2);
+	write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_enabler, 0, 64),
+		       SYS_ICH_PPI_ENABLER0_EL2);
+
+	/* Update the pending state of the NON-DVI'd PPIs, only */
+	bitmap_andnot(pendr, host_data_ptr(vgic_v5_ppi_state)->pendr_entry,
+		      cpu_if->vgic_ppi_dvir, VGIC_V5_NR_PRIVATE_IRQS);
+	write_sysreg_s(bitmap_read(pendr, 0, 64), SYS_ICH_PPI_PENDR0_EL2);
+
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[0],
+		       SYS_ICH_PPI_PRIORITYR0_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[1],
+		       SYS_ICH_PPI_PRIORITYR1_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[2],
+		       SYS_ICH_PPI_PRIORITYR2_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[3],
+		       SYS_ICH_PPI_PRIORITYR3_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[4],
+		       SYS_ICH_PPI_PRIORITYR4_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[5],
+		       SYS_ICH_PPI_PRIORITYR5_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[6],
+		       SYS_ICH_PPI_PRIORITYR6_EL2);
+	write_sysreg_s(cpu_if->vgic_ppi_priorityr[7],
+		       SYS_ICH_PPI_PRIORITYR7_EL2);
+
+	if (VGIC_V5_NR_PRIVATE_IRQS == 128) {
+		/* Enable DVI so that the guest's interrupt config takes over */
+		write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_dvir, 64, 64),
+			       SYS_ICH_PPI_DVIR1_EL2);
+
+		write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_activer, 64, 64),
+			       SYS_ICH_PPI_ACTIVER1_EL2);
+		write_sysreg_s(bitmap_read(cpu_if->vgic_ppi_enabler, 64, 64),
+			       SYS_ICH_PPI_ENABLER1_EL2);
+		write_sysreg_s(bitmap_read(pendr, 64, 64),
+			       SYS_ICH_PPI_PENDR1_EL2);
+
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[8],
+			       SYS_ICH_PPI_PRIORITYR8_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[9],
+			       SYS_ICH_PPI_PRIORITYR9_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[10],
+			       SYS_ICH_PPI_PRIORITYR10_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[11],
+			       SYS_ICH_PPI_PRIORITYR11_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[12],
+			       SYS_ICH_PPI_PRIORITYR12_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[13],
+			       SYS_ICH_PPI_PRIORITYR13_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[14],
+			       SYS_ICH_PPI_PRIORITYR14_EL2);
+		write_sysreg_s(cpu_if->vgic_ppi_priorityr[15],
+			       SYS_ICH_PPI_PRIORITYR15_EL2);
+	} else {
+		write_sysreg_s(0, SYS_ICH_PPI_DVIR1_EL2);
+
+		write_sysreg_s(0, SYS_ICH_PPI_ACTIVER1_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_ENABLER1_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PENDR1_EL2);
+
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR8_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR9_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR10_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR11_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR12_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR13_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR14_EL2);
+		write_sysreg_s(0, SYS_ICH_PPI_PRIORITYR15_EL2);
+	}
+}
+
+void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if)
+{
+	cpu_if->vgic_vmcr = read_sysreg_s(SYS_ICH_VMCR_EL2);
+	cpu_if->vgic_icsr = read_sysreg_s(SYS_ICC_ICSR_EL1);
+}
+
+void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if)
+{
+	write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1);
+}
diff --git a/arch/arm64/kvm/hyp/vhe/Makefile b/arch/arm64/kvm/hyp/vhe/Makefile
index afc4aed9231ac..9695328bbd96e 100644
--- a/arch/arm64/kvm/hyp/vhe/Makefile
+++ b/arch/arm64/kvm/hyp/vhe/Makefile
@@ -10,4 +10,4 @@ CFLAGS_switch.o += -Wno-override-init
 
 obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o
 obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
-	 ../fpsimd.o ../hyp-entry.o ../exception.o
+	 ../fpsimd.o ../hyp-entry.o ../exception.o ../vgic-v5-sr.o
diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c
index 35855dadfb1b3..f7b9dfe3f3a5a 100644
--- a/arch/arm64/kvm/hyp/vhe/tlb.c
+++ b/arch/arm64/kvm/hyp/vhe/tlb.c
@@ -104,7 +104,6 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
 	 * Instead, we invalidate Stage-2 for this IPA, and the
 	 * whole of Stage-1. Weep...
 	 */
-	ipa >>= 12;
 	__tlbi_level(ipas2e1is, ipa, level);
 
 	/*
@@ -136,7 +135,6 @@ void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,
 	 * Instead, we invalidate Stage-2 for this IPA, and the
 	 * whole of Stage-1. Weep...
 	 */
-	ipa >>= 12;
 	__tlbi_level(ipas2e1, ipa, level);
 
 	/*
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c
new file mode 100644
index 0000000000000..8b7f2bf2fba84
--- /dev/null
+++ b/arch/arm64/kvm/hyp_trace.c
@@ -0,0 +1,442 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <linux/cpumask.h>
+#include <linux/trace_remote.h>
+#include <linux/tracefs.h>
+#include <linux/simple_ring_buffer.h>
+
+#include <asm/arch_timer.h>
+#include <asm/kvm_host.h>
+#include <asm/kvm_hyptrace.h>
+#include <asm/kvm_mmu.h>
+
+#include "hyp_trace.h"
+
+/* Same 10min used by clocksource when width is more than 32-bits */
+#define CLOCK_MAX_CONVERSION_S	600
+/*
+ * Time to give for the clock init. Long enough to get a good mult/shift
+ * estimation. Short enough to not delay the tracing start too much.
+ */
+#define CLOCK_INIT_MS		100
+/*
+ * Time between clock checks. Must be small enough to catch clock deviation when
+ * it is still tiny.
+ */
+#define CLOCK_UPDATE_MS		500
+
+static struct hyp_trace_clock {
+	u64			cycles;
+	u64			cyc_overflow64;
+	u64			boot;
+	u32			mult;
+	u32			shift;
+	struct delayed_work	work;
+	struct completion	ready;
+	struct mutex		lock;
+	bool			running;
+} hyp_clock;
+
+static void __hyp_clock_work(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct hyp_trace_clock *hyp_clock;
+	struct system_time_snapshot snap;
+	u64 rate, delta_cycles;
+	u64 boot, delta_boot;
+
+	hyp_clock = container_of(dwork, struct hyp_trace_clock, work);
+
+	ktime_get_snapshot(&snap);
+	boot = ktime_to_ns(snap.boot);
+
+	delta_boot = boot - hyp_clock->boot;
+	delta_cycles = snap.cycles - hyp_clock->cycles;
+
+	/* Compare hyp clock with the kernel boot clock */
+	if (hyp_clock->mult) {
+		u64 err, cur = delta_cycles;
+
+		if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) {
+			__uint128_t tmp = (__uint128_t)cur * hyp_clock->mult;
+
+			cur = tmp >> hyp_clock->shift;
+		} else {
+			cur *= hyp_clock->mult;
+			cur >>= hyp_clock->shift;
+		}
+		cur += hyp_clock->boot;
+
+		err = abs_diff(cur, boot);
+		/* No deviation, only update epoch if necessary */
+		if (!err) {
+			if (delta_cycles >= (hyp_clock->cyc_overflow64 >> 1))
+				goto fast_forward;
+
+			goto resched;
+		}
+
+		/* Warn if the error is above tracing precision (1us) */
+		if (err > NSEC_PER_USEC)
+			pr_warn_ratelimited("hyp trace clock off by %lluus\n",
+					    err / NSEC_PER_USEC);
+	}
+
+	rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot);
+
+	clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift,
+			       rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S);
+
+	/* Add a comfortable 50% margin */
+	hyp_clock->cyc_overflow64 = (U64_MAX / hyp_clock->mult) >> 1;
+
+fast_forward:
+	hyp_clock->cycles = snap.cycles;
+	hyp_clock->boot = boot;
+	kvm_call_hyp_nvhe(__tracing_update_clock, hyp_clock->mult,
+			  hyp_clock->shift, hyp_clock->boot, hyp_clock->cycles);
+	complete(&hyp_clock->ready);
+
+resched:
+	schedule_delayed_work(&hyp_clock->work,
+			      msecs_to_jiffies(CLOCK_UPDATE_MS));
+}
+
+static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enable)
+{
+	struct system_time_snapshot snap;
+
+	if (hyp_clock->running == enable)
+		return;
+
+	if (!enable) {
+		cancel_delayed_work_sync(&hyp_clock->work);
+		hyp_clock->running = false;
+	}
+
+	ktime_get_snapshot(&snap);
+
+	hyp_clock->boot = ktime_to_ns(snap.boot);
+	hyp_clock->cycles = snap.cycles;
+	hyp_clock->mult = 0;
+
+	init_completion(&hyp_clock->ready);
+	INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work);
+	schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS));
+	wait_for_completion(&hyp_clock->ready);
+	hyp_clock->running = true;
+}
+
+/* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */
+static struct hyp_trace_buffer {
+	struct hyp_trace_desc	*desc;
+	size_t			desc_size;
+} trace_buffer;
+
+static int __map_hyp(void *start, size_t size)
+{
+	if (is_protected_kvm_enabled())
+		return 0;
+
+	return create_hyp_mappings(start, start + size, PAGE_HYP);
+}
+
+static int __share_page(unsigned long va)
+{
+	return kvm_share_hyp((void *)va, (void *)va + 1);
+}
+
+static void __unshare_page(unsigned long va)
+{
+	kvm_unshare_hyp((void *)va, (void *)va + 1);
+}
+
+static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size)
+{
+	int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1;
+	size_t backing_size;
+	void *start;
+
+	backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages *
+				  num_possible_cpus());
+
+	start = alloc_pages_exact(backing_size, GFP_KERNEL_ACCOUNT);
+	if (!start)
+		return -ENOMEM;
+
+	trace_buffer->desc->bpages_backing_start = (unsigned long)start;
+	trace_buffer->desc->bpages_backing_size = backing_size;
+
+	return __map_hyp(start, backing_size);
+}
+
+static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer)
+{
+	free_pages_exact((void *)trace_buffer->desc->bpages_backing_start,
+			 trace_buffer->desc->bpages_backing_size);
+}
+
+static void hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer *trace_buffer, int last_cpu)
+{
+	struct ring_buffer_desc *rb_desc;
+	int cpu, p;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) {
+		if (cpu > last_cpu)
+			break;
+
+		__share_page(rb_desc->meta_va);
+		for (p = 0; p < rb_desc->nr_page_va; p++)
+			__unshare_page(rb_desc->page_va[p]);
+	}
+}
+
+static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer)
+{
+	struct ring_buffer_desc *rb_desc;
+	int cpu, p, ret = 0;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, &trace_buffer->desc->trace_buffer_desc) {
+		ret = __share_page(rb_desc->meta_va);
+		if (ret)
+			break;
+
+		for (p = 0; p < rb_desc->nr_page_va; p++) {
+			ret = __share_page(rb_desc->page_va[p]);
+			if (ret)
+				break;
+		}
+
+		if (ret) {
+			for (p--; p >= 0; p--)
+				__unshare_page(rb_desc->page_va[p]);
+			break;
+		}
+	}
+
+	if (ret)
+		hyp_trace_buffer_unshare_hyp(trace_buffer, cpu--);
+
+	return ret;
+}
+
+static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv)
+{
+	struct hyp_trace_buffer *trace_buffer = priv;
+	struct hyp_trace_desc *desc;
+	size_t desc_size;
+	int ret;
+
+	if (WARN_ON(trace_buffer->desc))
+		return ERR_PTR(-EINVAL);
+
+	desc_size = trace_buffer_desc_size(size, num_possible_cpus());
+	if (desc_size == SIZE_MAX)
+		return ERR_PTR(-E2BIG);
+
+	desc_size = PAGE_ALIGN(desc_size);
+	desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+
+	ret = __map_hyp(desc, desc_size);
+	if (ret)
+		goto err_free_desc;
+
+	trace_buffer->desc = desc;
+
+	ret = hyp_trace_buffer_alloc_bpages_backing(trace_buffer, size);
+	if (ret)
+		goto err_free_desc;
+
+	ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, desc_size, size,
+					cpu_possible_mask);
+	if (ret)
+		goto err_free_backing;
+
+	ret = hyp_trace_buffer_share_hyp(trace_buffer);
+	if (ret)
+		goto err_free_buffer;
+
+	ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size);
+	if (ret)
+		goto err_unload_pages;
+
+	return &desc->trace_buffer_desc;
+
+err_unload_pages:
+	hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX);
+
+err_free_buffer:
+	trace_remote_free_buffer(&desc->trace_buffer_desc);
+
+err_free_backing:
+	hyp_trace_buffer_free_bpages_backing(trace_buffer);
+
+err_free_desc:
+	free_pages_exact(desc, desc_size);
+	trace_buffer->desc = NULL;
+
+	return ERR_PTR(ret);
+}
+
+static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv)
+{
+	struct hyp_trace_buffer *trace_buffer = priv;
+
+	if (WARN_ON(desc != &trace_buffer->desc->trace_buffer_desc))
+		return;
+
+	kvm_call_hyp_nvhe(__tracing_unload);
+	hyp_trace_buffer_unshare_hyp(trace_buffer, INT_MAX);
+	trace_remote_free_buffer(desc);
+	hyp_trace_buffer_free_bpages_backing(trace_buffer);
+	free_pages_exact(trace_buffer->desc, trace_buffer->desc_size);
+	trace_buffer->desc = NULL;
+}
+
+static int hyp_trace_enable_tracing(bool enable, void *priv)
+{
+	hyp_trace_clock_enable(&hyp_clock, enable);
+
+	return kvm_call_hyp_nvhe(__tracing_enable, enable);
+}
+
+static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv)
+{
+	return kvm_call_hyp_nvhe(__tracing_swap_reader, cpu);
+}
+
+static int hyp_trace_reset(unsigned int cpu, void *priv)
+{
+	return kvm_call_hyp_nvhe(__tracing_reset, cpu);
+}
+
+static int hyp_trace_enable_event(unsigned short id, bool enable, void *priv)
+{
+	struct hyp_event_id *event_id = lm_alias(&__hyp_event_ids_start[id]);
+	struct page *page;
+	atomic_t *enabled;
+	void *map;
+
+	if (is_protected_kvm_enabled())
+		return kvm_call_hyp_nvhe(__tracing_enable_event, id, enable);
+
+	enabled = &event_id->enabled;
+	page = virt_to_page(enabled);
+	map = vmap(&page, 1, VM_MAP, PAGE_KERNEL);
+	if (!map)
+		return -ENOMEM;
+
+	enabled = map + offset_in_page(enabled);
+	atomic_set(enabled, enable);
+
+	vunmap(map);
+
+	return 0;
+}
+
+static int hyp_trace_clock_show(struct seq_file *m, void *v)
+{
+	seq_puts(m, "[boot]\n");
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(hyp_trace_clock);
+
+static ssize_t hyp_trace_write_event_write(struct file *f, const char __user *ubuf,
+					   size_t cnt, loff_t *pos)
+{
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+	if (ret)
+		return ret;
+
+	kvm_call_hyp_nvhe(__tracing_write_event, val);
+
+	return cnt;
+}
+
+static const struct file_operations hyp_trace_write_event_fops = {
+	.write	= hyp_trace_write_event_write,
+};
+
+static int hyp_trace_init_tracefs(struct dentry *d, void *priv)
+{
+	if (!tracefs_create_file("write_event", 0200, d, NULL, &hyp_trace_write_event_fops))
+		return -ENOMEM;
+
+	return tracefs_create_file("trace_clock", 0440, d, NULL, &hyp_trace_clock_fops) ?
+		0 : -ENOMEM;
+}
+
+static struct trace_remote_callbacks trace_remote_callbacks = {
+	.init			= hyp_trace_init_tracefs,
+	.load_trace_buffer	= hyp_trace_load,
+	.unload_trace_buffer	= hyp_trace_unload,
+	.enable_tracing		= hyp_trace_enable_tracing,
+	.swap_reader_page	= hyp_trace_swap_reader_page,
+	.reset			= hyp_trace_reset,
+	.enable_event		= hyp_trace_enable_event,
+};
+
+static const char *__hyp_enter_exit_reason_str(u8 reason);
+
+#include <asm/kvm_define_hypevents.h>
+
+static const char *__hyp_enter_exit_reason_str(u8 reason)
+{
+	static const char strs[][12] = {
+		"smc",
+		"hvc",
+		"psci",
+		"host_abort",
+		"guest_exit",
+		"eret_host",
+		"eret_guest",
+		"unknown",
+	};
+
+	return strs[min(reason, HYP_REASON_UNKNOWN)];
+}
+
+static void __init hyp_trace_init_events(void)
+{
+	struct hyp_event_id *hyp_event_id = __hyp_event_ids_start;
+	struct remote_event *event = __hyp_events_start;
+	int id = 0;
+
+	/* Events on both sides hypervisor are sorted */
+	for (; event < __hyp_events_end; event++, hyp_event_id++, id++)
+		event->id = hyp_event_id->id = id;
+}
+
+int __init kvm_hyp_trace_init(void)
+{
+	int cpu;
+
+	if (is_kernel_in_hyp_mode())
+		return 0;
+
+	for_each_possible_cpu(cpu) {
+		const struct arch_timer_erratum_workaround *wa =
+			per_cpu(timer_unstable_counter_workaround, cpu);
+
+		if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) &&
+		    wa && wa->read_cntvct_el0) {
+			pr_warn("hyp trace can't handle CNTVCT workaround '%s'\n", wa->desc);
+			return -EOPNOTSUPP;
+		}
+	}
+
+	hyp_trace_init_events();
+
+	return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer,
+				     __hyp_events_start, __hyp_events_end - __hyp_events_start);
+}
diff --git a/arch/arm64/kvm/hyp_trace.h b/arch/arm64/kvm/hyp_trace.h
new file mode 100644
index 0000000000000..c991b1ec65f17
--- /dev/null
+++ b/arch/arm64/kvm/hyp_trace.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ARM64_KVM_HYP_TRACE_H__
+#define __ARM64_KVM_HYP_TRACE_H__
+
+#ifdef CONFIG_NVHE_EL2_TRACING
+int kvm_hyp_trace_init(void);
+#else
+static inline int kvm_hyp_trace_init(void) { return 0; }
+#endif
+#endif
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 2c43097248b21..efd5d21c7ac75 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -1558,6 +1558,11 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
 			 ID_AA64PFR1_EL1_MTE);
 		break;
 
+	case SYS_ID_AA64PFR2_EL1:
+		/* GICv5 is not yet supported for NV */
+		val &= ~ID_AA64PFR2_EL1_GCIE;
+		break;
+
 	case SYS_ID_AA64MMFR0_EL1:
 		/* Hide ExS, Secure Memory */
 		val &= ~(ID_AA64MMFR0_EL1_EXS		|
diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c
index 93cc9bbb5cecd..e1860acae641f 100644
--- a/arch/arm64/kvm/pmu-emul.c
+++ b/arch/arm64/kvm/pmu-emul.c
@@ -939,7 +939,8 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
 		 * number against the dimensions of the vgic and make sure
 		 * it's valid.
 		 */
-		if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))
+		if (!irq_is_ppi(vcpu->kvm, irq) &&
+		    !vgic_valid_spi(vcpu->kvm, irq))
 			return -EINVAL;
 	} else if (kvm_arm_pmu_irq_initialized(vcpu)) {
 		   return -EINVAL;
@@ -961,8 +962,13 @@ static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
 		if (!vgic_initialized(vcpu->kvm))
 			return -ENODEV;
 
-		if (!kvm_arm_pmu_irq_initialized(vcpu))
-			return -ENXIO;
+		if (!kvm_arm_pmu_irq_initialized(vcpu)) {
+			if (!vgic_is_v5(vcpu->kvm))
+				return -ENXIO;
+
+			/* Use the architected irq number for GICv5. */
+			vcpu->arch.pmu.irq_num = KVM_ARMV8_PMU_GICV5_IRQ;
+		}
 
 		ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,
 					 &vcpu->arch.pmu);
@@ -987,11 +993,15 @@ static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
 	unsigned long i;
 	struct kvm_vcpu *vcpu;
 
+	/* On GICv5, the PMUIRQ is architecturally mandated to be PPI 23 */
+	if (vgic_is_v5(kvm) && irq != KVM_ARMV8_PMU_GICV5_IRQ)
+		return false;
+
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (!kvm_arm_pmu_irq_initialized(vcpu))
 			continue;
 
-		if (irq_is_ppi(irq)) {
+		if (irq_is_ppi(vcpu->kvm, irq)) {
 			if (vcpu->arch.pmu.irq_num != irq)
 				return false;
 		} else {
@@ -1142,7 +1152,7 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 			return -EFAULT;
 
 		/* The PMU overflow interrupt can be a PPI or a valid SPI. */
-		if (!(irq_is_ppi(irq) || irq_is_spi(irq)))
+		if (!(irq_is_ppi(vcpu->kvm, irq) || irq_is_spi(vcpu->kvm, irq)))
 			return -EINVAL;
 
 		if (!pmu_irq_is_valid(kvm, irq))
diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c
index af5eec6811279..9724c320126b7 100644
--- a/arch/arm64/kvm/stacktrace.c
+++ b/arch/arm64/kvm/stacktrace.c
@@ -197,7 +197,7 @@ static void hyp_dump_backtrace(unsigned long hyp_offset)
 	kvm_nvhe_dump_backtrace_end();
 }
 
-#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
+#ifdef CONFIG_PKVM_STACKTRACE
 DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
 			 pkvm_stacktrace);
 
@@ -225,12 +225,12 @@ static void pkvm_dump_backtrace(unsigned long hyp_offset)
 		kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
 	kvm_nvhe_dump_backtrace_end();
 }
-#else	/* !CONFIG_PROTECTED_NVHE_STACKTRACE */
+#else	/* !CONFIG_PKVM_STACKTRACE */
 static void pkvm_dump_backtrace(unsigned long hyp_offset)
 {
-	kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
+	kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PKVM_STACKTRACE\n");
 }
-#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
+#endif /* CONFIG_PKVM_STACKTRACE */
 
 /*
  * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 1b4cacb6e918a..e1001544d4f40 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -681,6 +681,91 @@ static bool access_gic_dir(struct kvm_vcpu *vcpu,
 	return true;
 }
 
+static bool access_gicv5_idr0(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+			      const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		return undef_access(vcpu, p, r);
+
+	/*
+	 * Expose KVM's priority- and ID-bits to the guest, but not GCIE_LEGACY.
+	 *
+	 * Note: for GICv5 the mimic the way that the num_pri_bits and
+	 * num_id_bits fields are used with GICv3:
+	 * - num_pri_bits stores the actual number of priority bits, whereas the
+	 *   register field stores num_pri_bits - 1.
+	 * - num_id_bits stores the raw field value, which is 0b0000 for 16 bits
+	 *   and 0b0001 for 24 bits.
+	 */
+	p->regval = FIELD_PREP(ICC_IDR0_EL1_PRI_BITS, vcpu->arch.vgic_cpu.num_pri_bits - 1) |
+		    FIELD_PREP(ICC_IDR0_EL1_ID_BITS, vcpu->arch.vgic_cpu.num_id_bits);
+
+	return true;
+}
+
+static bool access_gicv5_iaffid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+				const struct sys_reg_desc *r)
+{
+	if (p->is_write)
+		return undef_access(vcpu, p, r);
+
+	/*
+	 * For GICv5 VMs, the IAFFID value is the same as the VPE ID. The VPE ID
+	 * is the same as the VCPU's ID.
+	 */
+	p->regval = FIELD_PREP(ICC_IAFFIDR_EL1_IAFFID, vcpu->vcpu_id);
+
+	return true;
+}
+
+static bool access_gicv5_ppi_enabler(struct kvm_vcpu *vcpu,
+				     struct sys_reg_params *p,
+				     const struct sys_reg_desc *r)
+{
+	unsigned long *mask = vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask;
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	int i;
+
+	/* We never expect to get here with a read! */
+	if (WARN_ON_ONCE(!p->is_write))
+		return undef_access(vcpu, p, r);
+
+	/*
+	 * If we're only handling architected PPIs and the guest writes to the
+	 * enable for the non-architected PPIs, we just return as there's
+	 * nothing to do at all. We don't even allocate the storage for them in
+	 * this case.
+	 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && p->Op2 % 2)
+		return true;
+
+	/*
+	 * Merge the raw guest write into out bitmap at an offset of either 0 or
+	 * 64, then and it with our PPI mask.
+	 */
+	bitmap_write(cpu_if->vgic_ppi_enabler, p->regval, 64 * (p->Op2 % 2), 64);
+	bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler, mask,
+		   VGIC_V5_NR_PRIVATE_IRQS);
+
+	/*
+	 * Sync the change in enable states to the vgic_irqs. We consider all
+	 * PPIs as we don't expose many to the guest.
+	 */
+	for_each_set_bit(i, mask, VGIC_V5_NR_PRIVATE_IRQS) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler);
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	return true;
+}
+
 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
 			struct sys_reg_params *p,
 			const struct sys_reg_desc *r)
@@ -1758,6 +1843,7 @@ static u8 pmuver_to_perfmon(u8 pmuver)
 
 static u64 sanitise_id_aa64pfr0_el1(const struct kvm_vcpu *vcpu, u64 val);
 static u64 sanitise_id_aa64pfr1_el1(const struct kvm_vcpu *vcpu, u64 val);
+static u64 sanitise_id_aa64pfr2_el1(const struct kvm_vcpu *vcpu, u64 val);
 static u64 sanitise_id_aa64dfr0_el1(const struct kvm_vcpu *vcpu, u64 val);
 
 /* Read a sanitised cpufeature ID register by sys_reg_desc */
@@ -1783,10 +1869,7 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
 		val = sanitise_id_aa64pfr1_el1(vcpu, val);
 		break;
 	case SYS_ID_AA64PFR2_EL1:
-		val &= ID_AA64PFR2_EL1_FPMR |
-			(kvm_has_mte(vcpu->kvm) ?
-			 ID_AA64PFR2_EL1_MTEFAR | ID_AA64PFR2_EL1_MTESTOREONLY :
-			 0);
+		val = sanitise_id_aa64pfr2_el1(vcpu, val);
 		break;
 	case SYS_ID_AA64ISAR1_EL1:
 		if (!vcpu_has_ptrauth(vcpu))
@@ -1985,7 +2068,7 @@ static u64 sanitise_id_aa64pfr0_el1(const struct kvm_vcpu *vcpu, u64 val)
 		val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
 	}
 
-	if (vgic_is_v3(vcpu->kvm)) {
+	if (vgic_host_has_gicv3()) {
 		val &= ~ID_AA64PFR0_EL1_GIC_MASK;
 		val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
 	}
@@ -2027,6 +2110,23 @@ static u64 sanitise_id_aa64pfr1_el1(const struct kvm_vcpu *vcpu, u64 val)
 	return val;
 }
 
+static u64 sanitise_id_aa64pfr2_el1(const struct kvm_vcpu *vcpu, u64 val)
+{
+	val &= ID_AA64PFR2_EL1_FPMR |
+	       ID_AA64PFR2_EL1_MTEFAR |
+	       ID_AA64PFR2_EL1_MTESTOREONLY;
+
+	if (!kvm_has_mte(vcpu->kvm)) {
+		val &= ~ID_AA64PFR2_EL1_MTEFAR;
+		val &= ~ID_AA64PFR2_EL1_MTESTOREONLY;
+	}
+
+	if (vgic_host_has_gicv5())
+		val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR2_EL1, GCIE, IMP);
+
+	return val;
+}
+
 static u64 sanitise_id_aa64dfr0_el1(const struct kvm_vcpu *vcpu, u64 val)
 {
 	val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
@@ -2177,14 +2277,6 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
 	    (vcpu_has_nv(vcpu) && !FIELD_GET(ID_AA64PFR0_EL1_EL2, user_val)))
 		return -EINVAL;
 
-	/*
-	 * If we are running on a GICv5 host and support FEAT_GCIE_LEGACY, then
-	 * we support GICv3. Fail attempts to do anything but set that to IMP.
-	 */
-	if (vgic_is_v3_compat(vcpu->kvm) &&
-	    FIELD_GET(ID_AA64PFR0_EL1_GIC_MASK, user_val) != ID_AA64PFR0_EL1_GIC_IMP)
-		return -EINVAL;
-
 	return set_id_reg(vcpu, rd, user_val);
 }
 
@@ -2224,6 +2316,12 @@ static int set_id_aa64pfr1_el1(struct kvm_vcpu *vcpu,
 	return set_id_reg(vcpu, rd, user_val);
 }
 
+static int set_id_aa64pfr2_el1(struct kvm_vcpu *vcpu,
+			       const struct sys_reg_desc *rd, u64 user_val)
+{
+	return set_id_reg(vcpu, rd, user_val);
+}
+
 /*
  * Allow userspace to de-feature a stage-2 translation granule but prevent it
  * from claiming the impossible.
@@ -3205,10 +3303,11 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 				       ID_AA64PFR1_EL1_RES0 |
 				       ID_AA64PFR1_EL1_MPAM_frac |
 				       ID_AA64PFR1_EL1_MTE)),
-	ID_WRITABLE(ID_AA64PFR2_EL1,
-		    ID_AA64PFR2_EL1_FPMR |
-		    ID_AA64PFR2_EL1_MTEFAR |
-		    ID_AA64PFR2_EL1_MTESTOREONLY),
+	ID_FILTERED(ID_AA64PFR2_EL1, id_aa64pfr2_el1,
+		    ~(ID_AA64PFR2_EL1_FPMR |
+		      ID_AA64PFR2_EL1_MTEFAR |
+		      ID_AA64PFR2_EL1_MTESTOREONLY |
+		      ID_AA64PFR2_EL1_GCIE)),
 	ID_UNALLOCATED(4,3),
 	ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0),
 	ID_HIDDEN(ID_AA64SMFR0_EL1),
@@ -3391,6 +3490,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	{ SYS_DESC(SYS_ICC_AP1R1_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_AP1R2_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_AP1R3_EL1), undef_access },
+	{ SYS_DESC(SYS_ICC_IDR0_EL1), access_gicv5_idr0 },
+	{ SYS_DESC(SYS_ICC_IAFFIDR_EL1), access_gicv5_iaffid },
+	{ SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1), access_gicv5_ppi_enabler },
+	{ SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1), access_gicv5_ppi_enabler },
 	{ SYS_DESC(SYS_ICC_DIR_EL1), access_gic_dir },
 	{ SYS_DESC(SYS_ICC_RPR_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
@@ -5647,6 +5750,8 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu)
 	compute_fgu(kvm, HFGRTR2_GROUP);
 	compute_fgu(kvm, HFGITR2_GROUP);
 	compute_fgu(kvm, HDFGRTR2_GROUP);
+	compute_fgu(kvm, ICH_HFGRTR_GROUP);
+	compute_fgu(kvm, ICH_HFGITR_GROUP);
 
 	set_bit(KVM_ARCH_FLAG_FGU_INITIALIZED, &kvm->arch.flags);
 out:
@@ -5667,17 +5772,52 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
 
 	guard(mutex)(&kvm->arch.config_lock);
 
+	if (kvm_vm_has_ran_once(kvm))
+		return 0;
+
 	/*
 	 * This hacks into the ID registers, so only perform it when the
 	 * first vcpu runs, or the kvm_set_vm_id_reg() helper will scream.
 	 */
-	if (!irqchip_in_kernel(kvm) && !kvm_vm_has_ran_once(kvm)) {
+	if (!irqchip_in_kernel(kvm)) {
 		u64 val;
 
 		val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
 		kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, val);
+		val = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1) & ~ID_AA64PFR2_EL1_GCIE;
+		kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1, val);
 		val = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
 		kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, val);
+	} else {
+		/*
+		 * Certain userspace software - QEMU - samples the system
+		 * register state without creating an irqchip, then blindly
+		 * restores the state prior to running the final guest. This
+		 * means that it restores the virtualization & emulation
+		 * capabilities of the host system, rather than something that
+		 * reflects the final guest state. Moreover, it checks that the
+		 * state was "correctly" restored (i.e., verbatim), bailing if
+		 * it isn't, so masking off invalid state isn't an option.
+		 *
+		 * On GICv5 hardware that supports FEAT_GCIE_LEGACY we can run
+		 * both GICv3- and GICv5-based guests. Therefore, we initially
+		 * present both ID_AA64PFR0.GIC and ID_AA64PFR2.GCIE as IMP to
+		 * reflect that userspace can create EITHER a vGICv3 or a
+		 * vGICv5. This is an architecturally invalid combination, of
+		 * course. Once an in-kernel GIC is created, the sysreg state is
+		 * updated to reflect the actual, valid configuration.
+		 *
+		 * Setting both the GIC and GCIE features to IMP unsurprisingly
+		 * results in guests falling over, and hence we need to fix up
+		 * this mess in KVM. Before running for the first time we yet
+		 * again ensure that the GIC and GCIE fields accurately reflect
+		 * the actual hardware the guest should see.
+		 *
+		 * This hack allows legacy QEMU-based GICv3 guests to run
+		 * unmodified on compatible GICv5 hosts, and avoids the inverse
+		 * problem for GICv5-based guests in the future.
+		 */
+		kvm_vgic_finalize_idregs(kvm);
 	}
 
 	if (vcpu_has_nv(vcpu)) {
diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c
index e9b8b5fc480c7..47169604100f2 100644
--- a/arch/arm64/kvm/vgic/vgic-init.c
+++ b/arch/arm64/kvm/vgic/vgic-init.c
@@ -66,12 +66,11 @@ static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type);
  * or through the generic KVM_CREATE_DEVICE API ioctl.
  * irqchip_in_kernel() tells you if this function succeeded or not.
  * @kvm: kvm struct pointer
- * @type: KVM_DEV_TYPE_ARM_VGIC_V[23]
+ * @type: KVM_DEV_TYPE_ARM_VGIC_V[235]
  */
 int kvm_vgic_create(struct kvm *kvm, u32 type)
 {
 	struct kvm_vcpu *vcpu;
-	u64 aa64pfr0, pfr1;
 	unsigned long i;
 	int ret;
 
@@ -132,8 +131,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 
 	if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
 		kvm->max_vcpus = VGIC_V2_MAX_CPUS;
-	else
+	else if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
 		kvm->max_vcpus = VGIC_V3_MAX_CPUS;
+	else if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
+		kvm->max_vcpus = min(VGIC_V5_MAX_CPUS,
+				     kvm_vgic_global_state.max_gic_vcpus);
 
 	if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
 		ret = -E2BIG;
@@ -145,19 +147,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 	kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
 	kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
 
-	aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
-	pfr1 = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
-
-	if (type == KVM_DEV_TYPE_ARM_VGIC_V2) {
-		kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
-	} else {
-		INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
-		aa64pfr0 |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
-		pfr1 |= SYS_FIELD_PREP_ENUM(ID_PFR1_EL1, GIC, GICv3);
-	}
-
-	kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0);
-	kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1);
+	/*
+	 * We've now created the GIC. Update the system register state
+	 * to accurately reflect what we've created.
+	 */
+	kvm_vgic_finalize_idregs(kvm);
 
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		ret = vgic_allocate_private_irqs_locked(vcpu, type);
@@ -179,6 +173,15 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 	if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
 		kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
 
+	/*
+	 * We now know that we have a GICv5. The Arch Timer PPI interrupts may
+	 * have been initialised at this stage, but will have done so assuming
+	 * that we have an older GIC, meaning that the IntIDs won't be
+	 * correct. We init them again, and this time they will be correct.
+	 */
+	if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
+		kvm_timer_init_vm(kvm);
+
 out_unlock:
 	mutex_unlock(&kvm->arch.config_lock);
 	kvm_unlock_all_vcpus(kvm);
@@ -259,9 +262,65 @@ int kvm_vgic_vcpu_nv_init(struct kvm_vcpu *vcpu)
 	return ret;
 }
 
+static void vgic_allocate_private_irq(struct kvm_vcpu *vcpu, int i, u32 type)
+{
+	struct vgic_irq *irq = &vcpu->arch.vgic_cpu.private_irqs[i];
+
+	INIT_LIST_HEAD(&irq->ap_list);
+	raw_spin_lock_init(&irq->irq_lock);
+	irq->vcpu = NULL;
+	irq->target_vcpu = vcpu;
+	refcount_set(&irq->refcount, 0);
+
+	irq->intid = i;
+	if (vgic_irq_is_sgi(i)) {
+		/* SGIs */
+		irq->enabled = 1;
+		irq->config = VGIC_CONFIG_EDGE;
+	} else {
+		/* PPIs */
+		irq->config = VGIC_CONFIG_LEVEL;
+	}
+
+	switch (type) {
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
+		irq->group = 1;
+		irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		irq->group = 0;
+		irq->targets = BIT(vcpu->vcpu_id);
+		break;
+	}
+}
+
+static void vgic_v5_allocate_private_irq(struct kvm_vcpu *vcpu, int i, u32 type)
+{
+	struct vgic_irq *irq = &vcpu->arch.vgic_cpu.private_irqs[i];
+	u32 intid = vgic_v5_make_ppi(i);
+
+	INIT_LIST_HEAD(&irq->ap_list);
+	raw_spin_lock_init(&irq->irq_lock);
+	irq->vcpu = NULL;
+	irq->target_vcpu = vcpu;
+	refcount_set(&irq->refcount, 0);
+
+	irq->intid = intid;
+
+	/* The only Edge architected PPI is the SW_PPI */
+	if (i == GICV5_ARCH_PPI_SW_PPI)
+		irq->config = VGIC_CONFIG_EDGE;
+	else
+		irq->config = VGIC_CONFIG_LEVEL;
+
+	/* Register the GICv5-specific PPI ops */
+	vgic_v5_set_ppi_ops(vcpu, intid);
+}
+
 static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type)
 {
 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+	u32 num_private_irqs;
 	int i;
 
 	lockdep_assert_held(&vcpu->kvm->arch.config_lock);
@@ -269,8 +328,13 @@ static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type)
 	if (vgic_cpu->private_irqs)
 		return 0;
 
+	if (vgic_is_v5(vcpu->kvm))
+		num_private_irqs = VGIC_V5_NR_PRIVATE_IRQS;
+	else
+		num_private_irqs = VGIC_NR_PRIVATE_IRQS;
+
 	vgic_cpu->private_irqs = kzalloc_objs(struct vgic_irq,
-					      VGIC_NR_PRIVATE_IRQS,
+					      num_private_irqs,
 					      GFP_KERNEL_ACCOUNT);
 
 	if (!vgic_cpu->private_irqs)
@@ -280,34 +344,11 @@ static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type)
 	 * Enable and configure all SGIs to be edge-triggered and
 	 * configure all PPIs as level-triggered.
 	 */
-	for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
-		struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
-
-		INIT_LIST_HEAD(&irq->ap_list);
-		raw_spin_lock_init(&irq->irq_lock);
-		irq->intid = i;
-		irq->vcpu = NULL;
-		irq->target_vcpu = vcpu;
-		refcount_set(&irq->refcount, 0);
-		if (vgic_irq_is_sgi(i)) {
-			/* SGIs */
-			irq->enabled = 1;
-			irq->config = VGIC_CONFIG_EDGE;
-		} else {
-			/* PPIs */
-			irq->config = VGIC_CONFIG_LEVEL;
-		}
-
-		switch (type) {
-		case KVM_DEV_TYPE_ARM_VGIC_V3:
-			irq->group = 1;
-			irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
-			break;
-		case KVM_DEV_TYPE_ARM_VGIC_V2:
-			irq->group = 0;
-			irq->targets = BIT(vcpu->vcpu_id);
-			break;
-		}
+	for (i = 0; i < num_private_irqs; i++) {
+		if (vgic_is_v5(vcpu->kvm))
+			vgic_v5_allocate_private_irq(vcpu, i, type);
+		else
+			vgic_allocate_private_irq(vcpu, i, type);
 	}
 
 	return 0;
@@ -366,7 +407,11 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 
 static void kvm_vgic_vcpu_reset(struct kvm_vcpu *vcpu)
 {
-	if (kvm_vgic_global_state.type == VGIC_V2)
+	const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
+	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5)
+		vgic_v5_reset(vcpu);
+	else if (kvm_vgic_global_state.type == VGIC_V2)
 		vgic_v2_reset(vcpu);
 	else
 		vgic_v3_reset(vcpu);
@@ -397,22 +442,28 @@ int vgic_init(struct kvm *kvm)
 	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
 		return -EBUSY;
 
-	/* freeze the number of spis */
-	if (!dist->nr_spis)
-		dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
+	if (!vgic_is_v5(kvm)) {
+		/* freeze the number of spis */
+		if (!dist->nr_spis)
+			dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
 
-	ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
-	if (ret)
-		goto out;
+		ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
+		if (ret)
+			return ret;
 
-	/*
-	 * Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
-	 * vLPIs) is supported.
-	 */
-	if (vgic_supports_direct_irqs(kvm)) {
-		ret = vgic_v4_init(kvm);
+		/*
+		 * Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
+		 * vLPIs) is supported.
+		 */
+		if (vgic_supports_direct_irqs(kvm)) {
+			ret = vgic_v4_init(kvm);
+			if (ret)
+				return ret;
+		}
+	} else {
+		ret = vgic_v5_init(kvm);
 		if (ret)
-			goto out;
+			return ret;
 	}
 
 	kvm_for_each_vcpu(idx, vcpu, kvm)
@@ -420,12 +471,12 @@ int vgic_init(struct kvm *kvm)
 
 	ret = kvm_vgic_setup_default_irq_routing(kvm);
 	if (ret)
-		goto out;
+		return ret;
 
 	vgic_debug_init(kvm);
 	dist->initialized = true;
-out:
-	return ret;
+
+	return 0;
 }
 
 static void kvm_vgic_dist_destroy(struct kvm *kvm)
@@ -569,6 +620,7 @@ int vgic_lazy_init(struct kvm *kvm)
 int kvm_vgic_map_resources(struct kvm *kvm)
 {
 	struct vgic_dist *dist = &kvm->arch.vgic;
+	bool needs_dist = true;
 	enum vgic_type type;
 	gpa_t dist_base;
 	int ret = 0;
@@ -587,12 +639,16 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
 		ret = vgic_v2_map_resources(kvm);
 		type = VGIC_V2;
-	} else {
+	} else if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
 		ret = vgic_v3_map_resources(kvm);
 		type = VGIC_V3;
+	} else {
+		ret = vgic_v5_map_resources(kvm);
+		type = VGIC_V5;
+		needs_dist = false;
 	}
 
-	if (ret)
+	if (ret || !needs_dist)
 		goto out;
 
 	dist_base = dist->vgic_dist_base;
@@ -617,6 +673,36 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	return ret;
 }
 
+void kvm_vgic_finalize_idregs(struct kvm *kvm)
+{
+	u32 type = kvm->arch.vgic.vgic_model;
+	u64 aa64pfr0, aa64pfr2, pfr1;
+
+	aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
+	aa64pfr2 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1) & ~ID_AA64PFR2_EL1_GCIE;
+	pfr1 = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
+
+	switch (type) {
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
+		INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
+		aa64pfr0 |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
+		pfr1 |= SYS_FIELD_PREP_ENUM(ID_PFR1_EL1, GIC, GICv3);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		aa64pfr2 |= SYS_FIELD_PREP_ENUM(ID_AA64PFR2_EL1, GCIE, IMP);
+		break;
+	default:
+		WARN_ONCE(1, "Unknown VGIC type!!!\n");
+	}
+
+	kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0);
+	kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1, aa64pfr2);
+	kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1);
+}
+
 /* GENERIC PROBE */
 
 void kvm_vgic_cpu_up(void)
diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c
index 3d1a776b716d7..a96c77dccf353 100644
--- a/arch/arm64/kvm/vgic/vgic-kvm-device.c
+++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c
@@ -336,6 +336,10 @@ int kvm_register_vgic_device(unsigned long type)
 			break;
 		ret = kvm_vgic_register_its_device();
 		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		ret = kvm_register_device_ops(&kvm_arm_vgic_v5_ops,
+					      KVM_DEV_TYPE_ARM_VGIC_V5);
+		break;
 	}
 
 	return ret;
@@ -639,7 +643,7 @@ static int vgic_v3_set_attr(struct kvm_device *dev,
 		if (vgic_initialized(dev->kvm))
 			return -EBUSY;
 
-		if (!irq_is_ppi(val))
+		if (!irq_is_ppi(dev->kvm, val))
 			return -EINVAL;
 
 		dev->kvm->arch.vgic.mi_intid = val;
@@ -715,3 +719,104 @@ struct kvm_device_ops kvm_arm_vgic_v3_ops = {
 	.get_attr = vgic_v3_get_attr,
 	.has_attr = vgic_v3_has_attr,
 };
+
+static int vgic_v5_get_userspace_ppis(struct kvm_device *dev,
+				      struct kvm_device_attr *attr)
+{
+	struct vgic_v5_vm *gicv5_vm = &dev->kvm->arch.vgic.gicv5_vm;
+	u64 __user *uaddr = (u64 __user *)(long)attr->addr;
+	int ret;
+
+	guard(mutex)(&dev->kvm->arch.config_lock);
+
+	/*
+	 * We either support 64 or 128 PPIs. In the former case, we need to
+	 * return 0s for the second 64 bits as we have no storage backing those.
+	 */
+	ret = put_user(bitmap_read(gicv5_vm->userspace_ppis, 0, 64), uaddr);
+	if (ret)
+		return ret;
+	uaddr++;
+
+	if (VGIC_V5_NR_PRIVATE_IRQS == 128)
+		ret = put_user(bitmap_read(gicv5_vm->userspace_ppis, 64, 128), uaddr);
+	else
+		ret = put_user(0, uaddr);
+
+	return ret;
+}
+
+static int vgic_v5_set_attr(struct kvm_device *dev,
+			    struct kvm_device_attr *attr)
+{
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
+		return -ENXIO;
+	case KVM_DEV_ARM_VGIC_GRP_CTRL:
+		switch (attr->attr) {
+		case KVM_DEV_ARM_VGIC_CTRL_INIT:
+			return vgic_set_common_attr(dev, attr);
+		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
+		default:
+			return -ENXIO;
+		}
+	default:
+		return -ENXIO;
+	}
+
+}
+
+static int vgic_v5_get_attr(struct kvm_device *dev,
+			    struct kvm_device_attr *attr)
+{
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
+		return -ENXIO;
+	case KVM_DEV_ARM_VGIC_GRP_CTRL:
+		switch (attr->attr) {
+		case KVM_DEV_ARM_VGIC_CTRL_INIT:
+			return vgic_get_common_attr(dev, attr);
+		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
+			return vgic_v5_get_userspace_ppis(dev, attr);
+		default:
+			return -ENXIO;
+		}
+	default:
+		return -ENXIO;
+	}
+}
+
+static int vgic_v5_has_attr(struct kvm_device *dev,
+			    struct kvm_device_attr *attr)
+{
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
+		return -ENXIO;
+	case KVM_DEV_ARM_VGIC_GRP_CTRL:
+		switch (attr->attr) {
+		case KVM_DEV_ARM_VGIC_CTRL_INIT:
+			return 0;
+		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
+			return 0;
+		default:
+			return -ENXIO;
+		}
+	default:
+		return -ENXIO;
+	}
+}
+
+struct kvm_device_ops kvm_arm_vgic_v5_ops = {
+	.name = "kvm-arm-vgic-v5",
+	.create = vgic_create,
+	.destroy = vgic_destroy,
+	.set_attr = vgic_v5_set_attr,
+	.get_attr = vgic_v5_get_attr,
+	.has_attr = vgic_v5_has_attr,
+};
diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c
index a573b1f0c6cbe..74d76dec97304 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio.c
+++ b/arch/arm64/kvm/vgic/vgic-mmio.c
@@ -842,18 +842,46 @@ vgic_find_mmio_region(const struct vgic_register_region *regions,
 
 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 {
-	if (kvm_vgic_global_state.type == VGIC_V2)
-		vgic_v2_set_vmcr(vcpu, vmcr);
-	else
+	const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		vgic_v5_set_vmcr(vcpu, vmcr);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		vgic_v3_set_vmcr(vcpu, vmcr);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+			vgic_v3_set_vmcr(vcpu, vmcr);
+		else
+			vgic_v2_set_vmcr(vcpu, vmcr);
+		break;
+	default:
+		BUG();
+	}
 }
 
 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 {
-	if (kvm_vgic_global_state.type == VGIC_V2)
-		vgic_v2_get_vmcr(vcpu, vmcr);
-	else
+	const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		vgic_v5_get_vmcr(vcpu, vmcr);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		vgic_v3_get_vmcr(vcpu, vmcr);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+			vgic_v3_get_vmcr(vcpu, vmcr);
+		else
+			vgic_v2_get_vmcr(vcpu, vmcr);
+		break;
+	default:
+		BUG();
+	}
 }
 
 /*
diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c
index 6a355eca19348..9e841e7afd4a7 100644
--- a/arch/arm64/kvm/vgic/vgic-v3.c
+++ b/arch/arm64/kvm/vgic/vgic-v3.c
@@ -499,7 +499,7 @@ void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu)
 {
 	struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
 
-	if (!vgic_is_v3(vcpu->kvm))
+	if (!vgic_host_has_gicv3())
 		return;
 
 	/* Hide GICv3 sysreg if necessary */
diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c
index 331651087e2c7..2b6cd5c3f9c2f 100644
--- a/arch/arm64/kvm/vgic/vgic-v5.c
+++ b/arch/arm64/kvm/vgic/vgic-v5.c
@@ -1,28 +1,52 @@
 // SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025, 2026 Arm Ltd.
+ */
 
 #include <kvm/arm_vgic.h>
+
+#include <linux/bitops.h>
 #include <linux/irqchip/arm-vgic-info.h>
 
 #include "vgic.h"
 
+static struct vgic_v5_ppi_caps ppi_caps;
+
+/*
+ * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which
+ * ones are, and generate a mask.
+ */
+static void vgic_v5_get_implemented_ppis(void)
+{
+	if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
+		return;
+
+	/*
+	 * If we have KVM, we have EL2, which means that we have support for the
+	 * EL1 and EL2 Physical & Virtual timers.
+	 */
+	__assign_bit(GICV5_ARCH_PPI_CNTHP, ppi_caps.impl_ppi_mask, 1);
+	__assign_bit(GICV5_ARCH_PPI_CNTV, ppi_caps.impl_ppi_mask, 1);
+	__assign_bit(GICV5_ARCH_PPI_CNTHV, ppi_caps.impl_ppi_mask, 1);
+	__assign_bit(GICV5_ARCH_PPI_CNTP, ppi_caps.impl_ppi_mask, 1);
+
+	/* The SW_PPI should be available */
+	__assign_bit(GICV5_ARCH_PPI_SW_PPI, ppi_caps.impl_ppi_mask, 1);
+
+	/* The PMUIRQ is available if we have the PMU */
+	__assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3());
+}
+
 /*
  * Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
- * Currently only supports GICv3-based VMs on a GICv5 host, and hence only
- * registers a VGIC_V3 device.
  */
 int vgic_v5_probe(const struct gic_kvm_info *info)
 {
+	bool v5_registered = false;
 	u64 ich_vtr_el2;
 	int ret;
 
-	if (!cpus_have_final_cap(ARM64_HAS_GICV5_LEGACY))
-		return -ENODEV;
-
 	kvm_vgic_global_state.type = VGIC_V5;
-	kvm_vgic_global_state.has_gcie_v3_compat = true;
-
-	/* We only support v3 compat mode - use vGICv3 limits */
-	kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS;
 
 	kvm_vgic_global_state.vcpu_base = 0;
 	kvm_vgic_global_state.vctrl_base = NULL;
@@ -30,6 +54,38 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 	kvm_vgic_global_state.has_gicv4 = false;
 	kvm_vgic_global_state.has_gicv4_1 = false;
 
+	/*
+	 * GICv5 is currently not supported in Protected mode. Skip the
+	 * registration of GICv5 completely to make sure no guests can create a
+	 * GICv5-based guest.
+	 */
+	if (is_protected_kvm_enabled()) {
+		kvm_info("GICv5-based guests are not supported with pKVM\n");
+		goto skip_v5;
+	}
+
+	kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS;
+
+	vgic_v5_get_implemented_ppis();
+
+	ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5);
+	if (ret) {
+		kvm_err("Cannot register GICv5 KVM device.\n");
+		goto skip_v5;
+	}
+
+	v5_registered = true;
+	kvm_info("GCIE system register CPU interface\n");
+
+skip_v5:
+	/* If we don't support the GICv3 compat mode we're done. */
+	if (!cpus_have_final_cap(ARM64_HAS_GICV5_LEGACY)) {
+		if (!v5_registered)
+			return -ENODEV;
+		return 0;
+	}
+
+	kvm_vgic_global_state.has_gcie_v3_compat = true;
 	ich_vtr_el2 =  kvm_call_hyp_ret(__vgic_v3_get_gic_config);
 	kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2;
 
@@ -45,6 +101,10 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 		return ret;
 	}
 
+	/* We potentially limit the max VCPUs further than we need to here */
+	kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS,
+						  VGIC_V5_MAX_CPUS);
+
 	static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
 	kvm_info("GCIE legacy system register CPU interface\n");
 
@@ -52,3 +112,428 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 
 	return 0;
 }
+
+void vgic_v5_reset(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * We always present 16-bits of ID space to the guest, irrespective of
+	 * the host allowing more.
+	 */
+	vcpu->arch.vgic_cpu.num_id_bits = ICC_IDR0_EL1_ID_BITS_16BITS;
+
+	/*
+	 * The GICv5 architeture only supports 5-bits of priority in the
+	 * CPUIF (but potentially fewer in the IRS).
+	 */
+	vcpu->arch.vgic_cpu.num_pri_bits = 5;
+}
+
+int vgic_v5_init(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu;
+	unsigned long idx;
+
+	if (vgic_initialized(kvm))
+		return 0;
+
+	kvm_for_each_vcpu(idx, vcpu, kvm) {
+		if (vcpu_has_nv(vcpu)) {
+			kvm_err("Nested GICv5 VMs are currently unsupported\n");
+			return -EINVAL;
+		}
+	}
+
+	/* We only allow userspace to drive the SW_PPI, if it is implemented. */
+	bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis,
+		    VGIC_V5_NR_PRIVATE_IRQS);
+	__assign_bit(GICV5_ARCH_PPI_SW_PPI,
+		     kvm->arch.vgic.gicv5_vm.userspace_ppis,
+		     VGIC_V5_NR_PRIVATE_IRQS);
+	bitmap_and(kvm->arch.vgic.gicv5_vm.userspace_ppis,
+		   kvm->arch.vgic.gicv5_vm.userspace_ppis,
+		   ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+
+	return 0;
+}
+
+int vgic_v5_map_resources(struct kvm *kvm)
+{
+	if (!vgic_initialized(kvm))
+		return -EBUSY;
+
+	return 0;
+}
+
+int vgic_v5_finalize_ppi_state(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu0;
+	int i;
+
+	if (!vgic_is_v5(kvm))
+		return 0;
+
+	/* The PPI state for all VCPUs should be the same. Pick the first. */
+	vcpu0 = kvm_get_vcpu(kvm, 0);
+
+	bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+	bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS);
+
+	for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
+		const u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu0, intid);
+
+		/* Expose PPIs with an owner or the SW_PPI, only */
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) {
+				__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, 1);
+				__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
+					     irq->config == VGIC_CONFIG_LEVEL);
+			}
+		}
+
+		vgic_put_irq(vcpu0->kvm, irq);
+	}
+
+	return 0;
+}
+
+static u32 vgic_v5_get_effective_priority_mask(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u32 highest_ap, priority_mask;
+
+	/*
+	 * If the guest's CPU has not opted to receive interrupts, then the
+	 * effective running priority is the highest priority. Just return 0
+	 * (the highest priority).
+	 */
+	if (!FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr))
+		return 0;
+
+	/*
+	 * Counting the number of trailing zeros gives the current active
+	 * priority. Explicitly use the 32-bit version here as we have 32
+	 * priorities. 32 then means that there are no active priorities.
+	 */
+	highest_ap = cpu_if->vgic_apr ? __builtin_ctz(cpu_if->vgic_apr) : 32;
+
+	/*
+	 * An interrupt is of sufficient priority if it is equal to or
+	 * greater than the priority mask. Add 1 to the priority mask
+	 * (i.e., lower priority) to match the APR logic before taking
+	 * the min. This gives us the lowest priority that is masked.
+	 */
+	priority_mask = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr);
+
+	return min(highest_ap, priority_mask + 1);
+}
+
+/*
+ * For GICv5, the PPIs are mostly directly managed by the hardware. We (the
+ * hypervisor) handle the pending, active, enable state save/restore, but don't
+ * need the PPIs to be queued on a per-VCPU AP list. Therefore, sanity check the
+ * state, unlock, and return.
+ */
+bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
+				  unsigned long flags)
+	__releases(&irq->irq_lock)
+{
+	struct kvm_vcpu *vcpu;
+
+	lockdep_assert_held(&irq->irq_lock);
+
+	if (WARN_ON_ONCE(!__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid)))
+		goto out_unlock_fail;
+
+	vcpu = irq->target_vcpu;
+	if (WARN_ON_ONCE(!vcpu))
+		goto out_unlock_fail;
+
+	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+
+	/* Directly kick the target VCPU to make sure it sees the IRQ */
+	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
+	kvm_vcpu_kick(vcpu);
+
+	return true;
+
+out_unlock_fail:
+	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+
+	return false;
+}
+
+/*
+ * Sets/clears the corresponding bit in the ICH_PPI_DVIR register.
+ */
+void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u32 ppi;
+
+	lockdep_assert_held(&irq->irq_lock);
+
+	ppi = vgic_v5_get_hwirq_id(irq->intid);
+	__assign_bit(ppi, cpu_if->vgic_ppi_dvir, dvi);
+}
+
+static struct irq_ops vgic_v5_ppi_irq_ops = {
+	.queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock,
+	.set_direct_injection = vgic_v5_set_ppi_dvi,
+};
+
+void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid)
+{
+	kvm_vgic_set_irq_ops(vcpu, vintid, &vgic_v5_ppi_irq_ops);
+}
+
+/*
+ * Sync back the PPI priorities to the vgic_irq shadow state for any interrupts
+ * exposed to the guest (skipping all others).
+ */
+static void vgic_v5_sync_ppi_priorities(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u64 priorityr;
+	int i;
+
+	/*
+	 * We have up to 16 PPI Priority regs, but only have a few interrupts
+	 * that the guest is allowed to use. Limit our sync of PPI priorities to
+	 * those actually exposed to the guest by first iterating over the mask
+	 * of exposed PPIs.
+	 */
+	for_each_set_bit(i, vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+		int pri_idx, pri_reg, pri_bit;
+		u8 priority;
+
+		/*
+		 * Determine which priority register and the field within it to
+		 * extract.
+		 */
+		pri_reg = i / 8;
+		pri_idx = i % 8;
+		pri_bit = pri_idx * 8;
+
+		priorityr = cpu_if->vgic_ppi_priorityr[pri_reg];
+		priority = field_get(GENMASK(pri_bit + 4, pri_bit), priorityr);
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			irq->priority = priority;
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+}
+
+bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu)
+{
+	unsigned int priority_mask;
+	int i;
+
+	priority_mask = vgic_v5_get_effective_priority_mask(vcpu);
+
+	/*
+	 * If the combined priority mask is 0, nothing can be signalled! In the
+	 * case where the guest has disabled interrupt delivery for the vcpu
+	 * (via ICV_CR0_EL1.EN->ICH_VMCR_EL2.EN), we calculate the priority mask
+	 * as 0 too (the highest possible priority).
+	 */
+	if (!priority_mask)
+		return false;
+
+	for_each_set_bit(i, vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
+		u32 intid = vgic_v5_make_ppi(i);
+		bool has_pending = false;
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			has_pending = (irq->enabled && irq_is_pending(irq) &&
+				       irq->priority <= priority_mask);
+
+		vgic_put_irq(vcpu->kvm, irq);
+
+		if (has_pending)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Detect any PPIs state changes, and propagate the state with KVM's
+ * shadow structures.
+ */
+void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	DECLARE_BITMAP(changed_active, VGIC_V5_NR_PRIVATE_IRQS);
+	DECLARE_BITMAP(changed_pending, VGIC_V5_NR_PRIVATE_IRQS);
+	DECLARE_BITMAP(changed_bits, VGIC_V5_NR_PRIVATE_IRQS);
+	unsigned long *activer, *pendr_entry, *pendr;
+	int i;
+
+	activer = host_data_ptr(vgic_v5_ppi_state)->activer_exit;
+	pendr_entry = host_data_ptr(vgic_v5_ppi_state)->pendr_entry;
+	pendr = host_data_ptr(vgic_v5_ppi_state)->pendr_exit;
+
+	bitmap_xor(changed_active, cpu_if->vgic_ppi_activer, activer,
+		   VGIC_V5_NR_PRIVATE_IRQS);
+	bitmap_xor(changed_pending, pendr_entry, pendr,
+		   VGIC_V5_NR_PRIVATE_IRQS);
+	bitmap_or(changed_bits, changed_active, changed_pending,
+		  VGIC_V5_NR_PRIVATE_IRQS);
+
+	for_each_set_bit(i, changed_bits, VGIC_V5_NR_PRIVATE_IRQS) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			irq->active = test_bit(i, activer);
+
+			/* This is an OR to avoid losing incoming edges! */
+			if (irq->config == VGIC_CONFIG_EDGE)
+				irq->pending_latch |= test_bit(i, pendr);
+		}
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	/*
+	 * Re-inject the exit state as entry state next time!
+	 *
+	 * Note that the write of the Enable state is trapped, and hence there
+	 * is nothing to explcitly sync back here as we already have the latest
+	 * copy by definition.
+	 */
+	bitmap_copy(cpu_if->vgic_ppi_activer, activer, VGIC_V5_NR_PRIVATE_IRQS);
+}
+
+void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu)
+{
+	DECLARE_BITMAP(pendr, VGIC_V5_NR_PRIVATE_IRQS);
+	int i;
+
+	/*
+	 * Time to enter the guest - we first need to build the guest's
+	 * ICC_PPI_PENDRx_EL1, however.
+	 */
+	bitmap_zero(pendr, VGIC_V5_NR_PRIVATE_IRQS);
+	for_each_set_bit(i, vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask,
+			 VGIC_V5_NR_PRIVATE_IRQS) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			__assign_bit(i, pendr, irq_is_pending(irq));
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	/*
+	 * Copy the shadow state to the pending reg that will be written to the
+	 * ICH_PPI_PENDRx_EL2 regs. While the guest is running we track any
+	 * incoming changes to the pending state in the vgic_irq structures. The
+	 * incoming changes are merged with the outgoing changes on the return
+	 * path.
+	 */
+	bitmap_copy(host_data_ptr(vgic_v5_ppi_state)->pendr_entry, pendr,
+		    VGIC_V5_NR_PRIVATE_IRQS);
+
+	/*
+	 * Make sure that we can correctly detect "edges" in the PPI
+	 * state. There's a path where we never actually enter the guest, and
+	 * failure to do this risks losing pending state
+	 */
+	bitmap_copy(host_data_ptr(vgic_v5_ppi_state)->pendr_exit, pendr,
+		    VGIC_V5_NR_PRIVATE_IRQS);
+}
+
+void vgic_v5_load(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/*
+	 * On the WFI path, vgic_load is called a second time. The first is when
+	 * scheduling in the vcpu thread again, and the second is when leaving
+	 * WFI. Skip the second instance as it serves no purpose and just
+	 * restores the same state again.
+	 */
+	if (cpu_if->gicv5_vpe.resident)
+		return;
+
+	kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if);
+
+	cpu_if->gicv5_vpe.resident = true;
+}
+
+void vgic_v5_put(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/*
+	 * Do nothing if we're not resident. This can happen in the WFI path
+	 * where we do a vgic_put in the WFI path and again later when
+	 * descheduling the thread. We risk losing VMCR state if we sync it
+	 * twice, so instead return early in this case.
+	 */
+	if (!cpu_if->gicv5_vpe.resident)
+		return;
+
+	kvm_call_hyp(__vgic_v5_save_apr, cpu_if);
+
+	cpu_if->gicv5_vpe.resident = false;
+
+	/* The shadow priority is only updated on entering WFI */
+	if (vcpu_get_flag(vcpu, IN_WFI))
+		vgic_v5_sync_ppi_priorities(vcpu);
+}
+
+void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u64 vmcr = cpu_if->vgic_vmcr;
+
+	vmcrp->en = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcr);
+	vmcrp->pmr = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcr);
+}
+
+void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u64 vmcr;
+
+	vmcr = FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcrp->pmr) |
+	       FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcrp->en);
+
+	cpu_if->vgic_vmcr = vmcr;
+}
+
+void vgic_v5_restore_state(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	__vgic_v5_restore_state(cpu_if);
+	__vgic_v5_restore_ppi_state(cpu_if);
+	dsb(sy);
+}
+
+void vgic_v5_save_state(struct kvm_vcpu *vcpu)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	__vgic_v5_save_state(cpu_if);
+	__vgic_v5_save_ppi_state(cpu_if);
+	dsb(sy);
+}
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index e22b79cfff965..1e9fe8764584d 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -86,6 +86,10 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
  */
 struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid)
 {
+	/* Non-private IRQs are not yet implemented for GICv5 */
+	if (vgic_is_v5(kvm))
+		return NULL;
+
 	/* SPIs */
 	if (intid >= VGIC_NR_PRIVATE_IRQS &&
 	    intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
@@ -94,7 +98,7 @@ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid)
 	}
 
 	/* LPIs */
-	if (intid >= VGIC_MIN_LPI)
+	if (irq_is_lpi(kvm, intid))
 		return vgic_get_lpi(kvm, intid);
 
 	return NULL;
@@ -105,6 +109,18 @@ struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid)
 	if (WARN_ON(!vcpu))
 		return NULL;
 
+	if (vgic_is_v5(vcpu->kvm)) {
+		u32 int_num, hwirq_id;
+
+		if (!__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, intid))
+			return NULL;
+
+		hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, intid);
+		int_num = array_index_nospec(hwirq_id, VGIC_V5_NR_PRIVATE_IRQS);
+
+		return &vcpu->arch.vgic_cpu.private_irqs[int_num];
+	}
+
 	/* SGIs and PPIs */
 	if (intid < VGIC_NR_PRIVATE_IRQS) {
 		intid = array_index_nospec(intid, VGIC_NR_PRIVATE_IRQS);
@@ -123,7 +139,7 @@ static void vgic_release_lpi_locked(struct vgic_dist *dist, struct vgic_irq *irq
 
 static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
 {
-	if (irq->intid < VGIC_MIN_LPI)
+	if (!irq_is_lpi(kvm, irq->intid))
 		return false;
 
 	return refcount_dec_and_test(&irq->refcount);
@@ -148,7 +164,7 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
 	 * Acquire/release it early on lockdep kernels to make locking issues
 	 * in rare release paths a bit more obvious.
 	 */
-	if (IS_ENABLED(CONFIG_LOCKDEP) && irq->intid >= VGIC_MIN_LPI) {
+	if (IS_ENABLED(CONFIG_LOCKDEP) && irq_is_lpi(kvm, irq->intid)) {
 		guard(spinlock_irqsave)(&dist->lpi_xa.xa_lock);
 	}
 
@@ -186,7 +202,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)
 	raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
 
 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
-		if (irq->intid >= VGIC_MIN_LPI) {
+		if (irq_is_lpi(vcpu->kvm, irq->intid)) {
 			raw_spin_lock(&irq->irq_lock);
 			list_del(&irq->ap_list);
 			irq->vcpu = NULL;
@@ -404,6 +420,9 @@ bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
 
 	lockdep_assert_held(&irq->irq_lock);
 
+	if (irq->ops && irq->ops->queue_irq_unlock)
+		return irq->ops->queue_irq_unlock(kvm, irq, flags);
+
 retry:
 	vcpu = vgic_target_oracle(irq);
 	if (irq->vcpu || !vcpu) {
@@ -521,12 +540,12 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 	if (ret)
 		return ret;
 
-	if (!vcpu && intid < VGIC_NR_PRIVATE_IRQS)
+	if (!vcpu && irq_is_private(kvm, intid))
 		return -EINVAL;
 
 	trace_vgic_update_irq_pending(vcpu ? vcpu->vcpu_idx : 0, intid, level);
 
-	if (intid < VGIC_NR_PRIVATE_IRQS)
+	if (irq_is_private(kvm, intid))
 		irq = vgic_get_vcpu_irq(vcpu, intid);
 	else
 		irq = vgic_get_irq(kvm, intid);
@@ -553,10 +572,27 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+void kvm_vgic_set_irq_ops(struct kvm_vcpu *vcpu, u32 vintid,
+			  struct irq_ops *ops)
+{
+	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);
+
+	BUG_ON(!irq);
+
+	scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+		irq->ops = ops;
+
+	vgic_put_irq(vcpu->kvm, irq);
+}
+
+void kvm_vgic_clear_irq_ops(struct kvm_vcpu *vcpu, u32 vintid)
+{
+	kvm_vgic_set_irq_ops(vcpu, vintid, NULL);
+}
+
 /* @irq->irq_lock must be held */
 static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
-			    unsigned int host_irq,
-			    struct irq_ops *ops)
+			    unsigned int host_irq)
 {
 	struct irq_desc *desc;
 	struct irq_data *data;
@@ -576,20 +612,25 @@ static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 	irq->hw = true;
 	irq->host_irq = host_irq;
 	irq->hwintid = data->hwirq;
-	irq->ops = ops;
+
+	if (irq->ops && irq->ops->set_direct_injection)
+		irq->ops->set_direct_injection(vcpu, irq, true);
+
 	return 0;
 }
 
 /* @irq->irq_lock must be held */
 static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq)
 {
+	if (irq->ops && irq->ops->set_direct_injection)
+		irq->ops->set_direct_injection(irq->target_vcpu, irq, false);
+
 	irq->hw = false;
 	irq->hwintid = 0;
-	irq->ops = NULL;
 }
 
 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
-			  u32 vintid, struct irq_ops *ops)
+			  u32 vintid)
 {
 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);
 	unsigned long flags;
@@ -598,7 +639,7 @@ int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
 	BUG_ON(!irq);
 
 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
-	ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);
+	ret = kvm_vgic_map_irq(vcpu, irq, host_irq);
 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 	vgic_put_irq(vcpu->kvm, irq);
 
@@ -685,7 +726,7 @@ int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner)
 		return -EAGAIN;
 
 	/* SGIs and LPIs cannot be wired up to any device */
-	if (!irq_is_ppi(intid) && !vgic_valid_spi(vcpu->kvm, intid))
+	if (!irq_is_ppi(vcpu->kvm, intid) && !vgic_valid_spi(vcpu->kvm, intid))
 		return -EINVAL;
 
 	irq = vgic_get_vcpu_irq(vcpu, intid);
@@ -812,8 +853,13 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 		vgic_release_deleted_lpis(vcpu->kvm);
 }
 
-static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
+static void vgic_fold_state(struct kvm_vcpu *vcpu)
 {
+	if (vgic_is_v5(vcpu->kvm)) {
+		vgic_v5_fold_ppi_state(vcpu);
+		return;
+	}
+
 	if (!*host_data_ptr(last_lr_irq))
 		return;
 
@@ -1002,7 +1048,10 @@ static inline bool can_access_vgic_from_kernel(void)
 
 static inline void vgic_save_state(struct kvm_vcpu *vcpu)
 {
-	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+	/* No switch statement here. See comment in vgic_restore_state() */
+	if (vgic_is_v5(vcpu->kvm))
+		vgic_v5_save_state(vcpu);
+	else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
 		vgic_v2_save_state(vcpu);
 	else
 		__vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3);
@@ -1011,20 +1060,24 @@ static inline void vgic_save_state(struct kvm_vcpu *vcpu)
 /* Sync back the hardware VGIC state into our emulation after a guest's run. */
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
 {
-	/* If nesting, emulate the HW effect from L0 to L1 */
-	if (vgic_state_is_nested(vcpu)) {
-		vgic_v3_sync_nested(vcpu);
-		return;
-	}
+	if (vgic_is_v3(vcpu->kvm)) {
+		/* If nesting, emulate the HW effect from L0 to L1 */
+		if (vgic_state_is_nested(vcpu)) {
+			vgic_v3_sync_nested(vcpu);
+			return;
+		}
 
-	if (vcpu_has_nv(vcpu))
-		vgic_v3_nested_update_mi(vcpu);
+		if (vcpu_has_nv(vcpu))
+			vgic_v3_nested_update_mi(vcpu);
+	}
 
 	if (can_access_vgic_from_kernel())
 		vgic_save_state(vcpu);
 
-	vgic_fold_lr_state(vcpu);
-	vgic_prune_ap_list(vcpu);
+	vgic_fold_state(vcpu);
+
+	if (!vgic_is_v5(vcpu->kvm))
+		vgic_prune_ap_list(vcpu);
 }
 
 /* Sync interrupts that were deactivated through a DIR trap */
@@ -1040,12 +1093,34 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu)
 
 static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
 {
-	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+	/*
+	 * As nice as it would be to restructure this code into a switch
+	 * statement as can be found elsewhere, the logic quickly gets ugly.
+	 *
+	 * __vgic_v3_restore_state() is doing a lot of heavy lifting here. It is
+	 * required for GICv3-on-GICv3, GICv2-on-GICv3, GICv3-on-GICv5, and the
+	 * no-in-kernel-irqchip case on GICv3 hardware. Hence, adding a switch
+	 * here results in much more complex code.
+	 */
+	if (vgic_is_v5(vcpu->kvm))
+		vgic_v5_restore_state(vcpu);
+	else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
 		vgic_v2_restore_state(vcpu);
 	else
 		__vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
 }
 
+static void vgic_flush_state(struct kvm_vcpu *vcpu)
+{
+	if (vgic_is_v5(vcpu->kvm)) {
+		vgic_v5_flush_ppi_state(vcpu);
+		return;
+	}
+
+	scoped_guard(raw_spinlock, &vcpu->arch.vgic_cpu.ap_list_lock)
+		vgic_flush_lr_state(vcpu);
+}
+
 /* Flush our emulation state into the GIC hardware before entering the guest. */
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 {
@@ -1082,42 +1157,69 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 
 	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
-	scoped_guard(raw_spinlock, &vcpu->arch.vgic_cpu.ap_list_lock)
-		vgic_flush_lr_state(vcpu);
+	vgic_flush_state(vcpu);
 
 	if (can_access_vgic_from_kernel())
 		vgic_restore_state(vcpu);
 
-	if (vgic_supports_direct_irqs(vcpu->kvm))
+	if (vgic_supports_direct_irqs(vcpu->kvm) && kvm_vgic_global_state.has_gicv4)
 		vgic_v4_commit(vcpu);
 }
 
 void kvm_vgic_load(struct kvm_vcpu *vcpu)
 {
+	const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
 	if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
 		if (has_vhe() && static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
 			__vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
 		return;
 	}
 
-	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
-		vgic_v2_load(vcpu);
-	else
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		vgic_v5_load(vcpu);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		vgic_v3_load(vcpu);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+			vgic_v3_load(vcpu);
+		else
+			vgic_v2_load(vcpu);
+		break;
+	default:
+		BUG();
+	}
 }
 
 void kvm_vgic_put(struct kvm_vcpu *vcpu)
 {
+	const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+
 	if (unlikely(!irqchip_in_kernel(vcpu->kvm) || !vgic_initialized(vcpu->kvm))) {
 		if (has_vhe() && static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
 			__vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3);
 		return;
 	}
 
-	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
-		vgic_v2_put(vcpu);
-	else
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		vgic_v5_put(vcpu);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		vgic_v3_put(vcpu);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))
+			vgic_v3_put(vcpu);
+		else
+			vgic_v2_put(vcpu);
+		break;
+	default:
+		BUG();
+	}
 }
 
 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
@@ -1128,6 +1230,9 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
 	unsigned long flags;
 	struct vgic_vmcr vmcr;
 
+	if (vgic_is_v5(vcpu->kvm))
+		return vgic_v5_has_pending_ppi(vcpu);
+
 	if (!vcpu->kvm->arch.vgic.enabled)
 		return false;
 
diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h
index c9b3bb07e483c..9d941241c8a2b 100644
--- a/arch/arm64/kvm/vgic/vgic.h
+++ b/arch/arm64/kvm/vgic/vgic.h
@@ -187,6 +187,7 @@ static inline u64 vgic_ich_hcr_trap_bits(void)
  * registers regardless of the hardware backed GIC used.
  */
 struct vgic_vmcr {
+	u32	en; /* GICv5-specific */
 	u32	grpen0;
 	u32	grpen1;
 
@@ -363,6 +364,19 @@ void vgic_debug_init(struct kvm *kvm);
 void vgic_debug_destroy(struct kvm *kvm);
 
 int vgic_v5_probe(const struct gic_kvm_info *info);
+void vgic_v5_reset(struct kvm_vcpu *vcpu);
+int vgic_v5_init(struct kvm *kvm);
+int vgic_v5_map_resources(struct kvm *kvm);
+void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid);
+bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu);
+void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu);
+void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu);
+void vgic_v5_load(struct kvm_vcpu *vcpu);
+void vgic_v5_put(struct kvm_vcpu *vcpu);
+void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
+void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
+void vgic_v5_restore_state(struct kvm_vcpu *vcpu);
+void vgic_v5_save_state(struct kvm_vcpu *vcpu);
 
 static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
 {
@@ -425,15 +439,6 @@ void vgic_its_invalidate_all_caches(struct kvm *kvm);
 int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
 int vgic_its_invall(struct kvm_vcpu *vcpu);
 
-bool system_supports_direct_sgis(void);
-bool vgic_supports_direct_msis(struct kvm *kvm);
-bool vgic_supports_direct_sgis(struct kvm *kvm);
-
-static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
-{
-	return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm);
-}
-
 int vgic_v4_init(struct kvm *kvm);
 void vgic_v4_teardown(struct kvm *kvm);
 void vgic_v4_configure_vsgis(struct kvm *kvm);
@@ -447,6 +452,11 @@ static inline bool kvm_has_gicv3(struct kvm *kvm)
 	return kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP);
 }
 
+static inline bool kvm_has_gicv5(struct kvm *kvm)
+{
+	return kvm_has_feat(kvm, ID_AA64PFR2_EL1, GCIE, IMP);
+}
+
 void vgic_v3_flush_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_sync_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_load_nested(struct kvm_vcpu *vcpu);
@@ -454,15 +464,32 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
 void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu);
 
-static inline bool vgic_is_v3_compat(struct kvm *kvm)
+static inline bool vgic_host_has_gicv3(void)
 {
-	return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) &&
+	/*
+	 * Either the host is a native GICv3, or it is GICv5 with
+	 * FEAT_GCIE_LEGACY.
+	 */
+	return kvm_vgic_global_state.type == VGIC_V3 ||
 		kvm_vgic_global_state.has_gcie_v3_compat;
 }
 
-static inline bool vgic_is_v3(struct kvm *kvm)
+static inline bool vgic_host_has_gicv5(void)
+{
+	return kvm_vgic_global_state.type == VGIC_V5;
+}
+
+bool system_supports_direct_sgis(void);
+bool vgic_supports_direct_msis(struct kvm *kvm);
+bool vgic_supports_direct_sgis(struct kvm *kvm);
+
+static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
 {
-	return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm);
+	/* GICv5 always supports direct IRQs */
+	if (vgic_is_v5(kvm))
+		return true;
+
+	return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm);
 }
 
 int vgic_its_debug_init(struct kvm_device *dev);
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 633e5223d944d..448c917494f30 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -5,12 +5,6 @@ lib-y		:= clear_user.o delay.o copy_from_user.o		\
 		   memset.o memcmp.o strcmp.o strncmp.o strlen.o	\
 		   strnlen.o strchr.o strrchr.o tishift.o
 
-ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
-obj-$(CONFIG_XOR_BLOCKS)	+= xor-neon.o
-CFLAGS_xor-neon.o		+= $(CC_FLAGS_FPU)
-CFLAGS_REMOVE_xor-neon.o	+= $(CC_FLAGS_NO_FPU)
-endif
-
 lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
 
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c
deleted file mode 100644
index 8fffebfa17b20..0000000000000
--- a/arch/arm64/lib/xor-neon.c
+++ /dev/null
@@ -1,338 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * arch/arm64/lib/xor-neon.c
- *
- * Authors: Jackie Liu <liuyun01@kylinos.cn>
- * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
- */
-
-#include <linux/raid/xor.h>
-#include <linux/module.h>
-#include <asm/neon-intrinsics.h>
-
-static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 */
-		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
-		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
-		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
-		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
-
-		/* store */
-		vst1q_u64(dp1 +  0, v0);
-		vst1q_u64(dp1 +  2, v1);
-		vst1q_u64(dp1 +  4, v2);
-		vst1q_u64(dp1 +  6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-	} while (--lines > 0);
-}
-
-static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 */
-		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
-		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
-		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
-		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
-
-		/* p1 ^= p3 */
-		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
-
-		/* store */
-		vst1q_u64(dp1 +  0, v0);
-		vst1q_u64(dp1 +  2, v1);
-		vst1q_u64(dp1 +  4, v2);
-		vst1q_u64(dp1 +  6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-	} while (--lines > 0);
-}
-
-static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-	uint64_t *dp4 = (uint64_t *)p4;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 */
-		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
-		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
-		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
-		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
-
-		/* p1 ^= p3 */
-		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
-
-		/* p1 ^= p4 */
-		v0 = veorq_u64(v0, vld1q_u64(dp4 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp4 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp4 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp4 +  6));
-
-		/* store */
-		vst1q_u64(dp1 +  0, v0);
-		vst1q_u64(dp1 +  2, v1);
-		vst1q_u64(dp1 +  4, v2);
-		vst1q_u64(dp1 +  6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-		dp4 += 8;
-	} while (--lines > 0);
-}
-
-static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4,
-	const unsigned long * __restrict p5)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-	uint64_t *dp4 = (uint64_t *)p4;
-	uint64_t *dp5 = (uint64_t *)p5;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 */
-		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
-		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
-		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
-		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
-
-		/* p1 ^= p3 */
-		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
-
-		/* p1 ^= p4 */
-		v0 = veorq_u64(v0, vld1q_u64(dp4 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp4 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp4 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp4 +  6));
-
-		/* p1 ^= p5 */
-		v0 = veorq_u64(v0, vld1q_u64(dp5 +  0));
-		v1 = veorq_u64(v1, vld1q_u64(dp5 +  2));
-		v2 = veorq_u64(v2, vld1q_u64(dp5 +  4));
-		v3 = veorq_u64(v3, vld1q_u64(dp5 +  6));
-
-		/* store */
-		vst1q_u64(dp1 +  0, v0);
-		vst1q_u64(dp1 +  2, v1);
-		vst1q_u64(dp1 +  4, v2);
-		vst1q_u64(dp1 +  6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-		dp4 += 8;
-		dp5 += 8;
-	} while (--lines > 0);
-}
-
-struct xor_block_template xor_block_inner_neon __ro_after_init = {
-	.name	= "__inner_neon__",
-	.do_2	= xor_arm64_neon_2,
-	.do_3	= xor_arm64_neon_3,
-	.do_4	= xor_arm64_neon_4,
-	.do_5	= xor_arm64_neon_5,
-};
-EXPORT_SYMBOL(xor_block_inner_neon);
-
-static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
-{
-	uint64x2_t res;
-
-	asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n"
-	    "eor3 %0.16b, %1.16b, %2.16b, %3.16b"
-	    : "=w"(res) : "w"(p), "w"(q), "w"(r));
-	return res;
-}
-
-static void xor_arm64_eor3_3(unsigned long bytes,
-	unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 ^ p3 */
-		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
-			  vld1q_u64(dp3 + 0));
-		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
-			  vld1q_u64(dp3 + 2));
-		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
-			  vld1q_u64(dp3 + 4));
-		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
-			  vld1q_u64(dp3 + 6));
-
-		/* store */
-		vst1q_u64(dp1 + 0, v0);
-		vst1q_u64(dp1 + 2, v1);
-		vst1q_u64(dp1 + 4, v2);
-		vst1q_u64(dp1 + 6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-	} while (--lines > 0);
-}
-
-static void xor_arm64_eor3_4(unsigned long bytes,
-	unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-	uint64_t *dp4 = (uint64_t *)p4;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 ^ p3 */
-		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
-			  vld1q_u64(dp3 + 0));
-		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
-			  vld1q_u64(dp3 + 2));
-		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
-			  vld1q_u64(dp3 + 4));
-		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
-			  vld1q_u64(dp3 + 6));
-
-		/* p1 ^= p4 */
-		v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
-		v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
-		v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
-		v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
-
-		/* store */
-		vst1q_u64(dp1 + 0, v0);
-		vst1q_u64(dp1 + 2, v1);
-		vst1q_u64(dp1 + 4, v2);
-		vst1q_u64(dp1 + 6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-		dp4 += 8;
-	} while (--lines > 0);
-}
-
-static void xor_arm64_eor3_5(unsigned long bytes,
-	unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4,
-	const unsigned long * __restrict p5)
-{
-	uint64_t *dp1 = (uint64_t *)p1;
-	uint64_t *dp2 = (uint64_t *)p2;
-	uint64_t *dp3 = (uint64_t *)p3;
-	uint64_t *dp4 = (uint64_t *)p4;
-	uint64_t *dp5 = (uint64_t *)p5;
-
-	register uint64x2_t v0, v1, v2, v3;
-	long lines = bytes / (sizeof(uint64x2_t) * 4);
-
-	do {
-		/* p1 ^= p2 ^ p3 */
-		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
-			  vld1q_u64(dp3 + 0));
-		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
-			  vld1q_u64(dp3 + 2));
-		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
-			  vld1q_u64(dp3 + 4));
-		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
-			  vld1q_u64(dp3 + 6));
-
-		/* p1 ^= p4 ^ p5 */
-		v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0));
-		v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2));
-		v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4));
-		v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6));
-
-		/* store */
-		vst1q_u64(dp1 + 0, v0);
-		vst1q_u64(dp1 + 2, v1);
-		vst1q_u64(dp1 + 4, v2);
-		vst1q_u64(dp1 + 6, v3);
-
-		dp1 += 8;
-		dp2 += 8;
-		dp3 += 8;
-		dp4 += 8;
-		dp5 += 8;
-	} while (--lines > 0);
-}
-
-static int __init xor_neon_init(void)
-{
-	if (cpu_have_named_feature(SHA3)) {
-		xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
-		xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
-		xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
-	}
-	return 0;
-}
-module_init(xor_neon_init);
-
-static void __exit xor_neon_exit(void)
-{
-}
-module_exit(xor_neon_exit);
-
-MODULE_AUTHOR("Jackie Liu <liuyun01@kylinos.cn>");
-MODULE_DESCRIPTION("ARMv8 XOR Extensions");
-MODULE_LICENSE("GPL");
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 503567c864fde..ab75c050f5590 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -132,17 +132,7 @@ alternative_else_nop_endif
 	ret
 SYM_FUNC_END(dcache_clean_pou)
 
-/*
- *	dcache_inval_poc(start, end)
- *
- * 	Ensure that any D-cache lines for the interval [start, end)
- * 	are invalidated. Any partial lines at the ends of the interval are
- *	also cleaned to PoC to prevent data loss.
- *
- *	- start   - kernel start address of region
- *	- end     - kernel end address of region
- */
-SYM_FUNC_START(__pi_dcache_inval_poc)
+.macro __dcache_inval_poc_nosync
 	dcache_line_size x2, x3
 	sub	x3, x2, #1
 	tst	x1, x3				// end cache line aligned?
@@ -158,11 +148,41 @@ SYM_FUNC_START(__pi_dcache_inval_poc)
 3:	add	x0, x0, x2
 	cmp	x0, x1
 	b.lo	2b
+.endm
+
+/*
+ *	dcache_inval_poc(start, end)
+ *
+ * 	Ensure that any D-cache lines for the interval [start, end)
+ * 	are invalidated. Any partial lines at the ends of the interval are
+ *	also cleaned to PoC to prevent data loss.
+ *
+ *	- start   - kernel start address of region
+ *	- end     - kernel end address of region
+ */
+SYM_FUNC_START(__pi_dcache_inval_poc)
+	__dcache_inval_poc_nosync
 	dsb	sy
 	ret
 SYM_FUNC_END(__pi_dcache_inval_poc)
 SYM_FUNC_ALIAS(dcache_inval_poc, __pi_dcache_inval_poc)
 
+/*
+ *	dcache_inval_poc_nosync(start, end)
+ *
+ * 	Issue the instructions of D-cache lines for the interval [start, end)
+ * 	for invalidation. Not necessarily cleaned to PoC till an explicit dsb
+ *	sy is issued later
+ *
+ *	- start   - kernel start address of region
+ *	- end     - kernel end address of region
+ */
+SYM_FUNC_START(__pi_dcache_inval_poc_nosync)
+	__dcache_inval_poc_nosync
+	ret
+SYM_FUNC_END(__pi_dcache_inval_poc_nosync)
+SYM_FUNC_ALIAS(dcache_inval_poc_nosync, __pi_dcache_inval_poc_nosync)
+
 /*
  *	dcache_clean_poc(start, end)
  *
@@ -178,6 +198,21 @@ SYM_FUNC_START(__pi_dcache_clean_poc)
 SYM_FUNC_END(__pi_dcache_clean_poc)
 SYM_FUNC_ALIAS(dcache_clean_poc, __pi_dcache_clean_poc)
 
+/*
+ *	dcache_clean_poc_nosync(start, end)
+ *
+ * 	Issue the instructions of D-cache lines for the interval [start, end).
+ * 	not necessarily cleaned to the PoC till an explicit dsb sy afterward.
+ *
+ *	- start   - virtual start address of region
+ *	- end     - virtual end address of region
+ */
+SYM_FUNC_START(__pi_dcache_clean_poc_nosync)
+	dcache_by_line_op_nosync cvac, x0, x1, x2, x3
+	ret
+SYM_FUNC_END(__pi_dcache_clean_poc_nosync)
+SYM_FUNC_ALIAS(dcache_clean_poc_nosync, __pi_dcache_clean_poc_nosync)
+
 /*
  *	dcache_clean_pop(start, end)
  *
diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c
index 1519d090d5ea7..3970392c43267 100644
--- a/arch/arm64/mm/contpte.c
+++ b/arch/arm64/mm/contpte.c
@@ -225,7 +225,8 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr,
 	 */
 
 	if (!system_supports_bbml2_noabort())
-		__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3);
+		__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3,
+				  TLBF_NOWALKCACHE);
 
 	__set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES);
 }
@@ -551,8 +552,8 @@ int contpte_clear_flush_young_ptes(struct vm_area_struct *vma,
 		 * See comment in __ptep_clear_flush_young(); same rationale for
 		 * eliding the trailing DSB applies here.
 		 */
-		__flush_tlb_range_nosync(vma->vm_mm, addr, end,
-					 PAGE_SIZE, true, 3);
+		__flush_tlb_range(vma, addr, end, PAGE_SIZE, 3,
+				  TLBF_NOWALKCACHE | TLBF_NOSYNC);
 	}
 
 	return young;
@@ -685,7 +686,10 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma,
 			__ptep_set_access_flags(vma, addr, ptep, entry, 0);
 
 		if (dirty)
-			local_flush_tlb_contpte(vma, start_addr);
+			__flush_tlb_range(vma, start_addr,
+					  start_addr + CONT_PTE_SIZE,
+					  PAGE_SIZE, 3,
+					  TLBF_NOWALKCACHE | TLBF_NOBROADCAST);
 	} else {
 		__contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte);
 		__ptep_set_access_flags(vma, addr, ptep, entry, dirty);
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index b2b5792b2caaf..ae1ae0280eef7 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -17,7 +17,7 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
 {
 	unsigned long start = (unsigned long)phys_to_virt(paddr);
 
-	dcache_clean_poc(start, start + size);
+	dcache_clean_poc_nosync(start, start + size);
 }
 
 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
@@ -28,7 +28,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 	if (dir == DMA_TO_DEVICE)
 		return;
 
-	dcache_inval_poc(start, start + size);
+	dcache_inval_poc_nosync(start, start + size);
 }
 
 void arch_dma_prep_coherent(struct page *page, size_t size)
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index be9dab2c7d6a8..f91aa686f1428 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -239,7 +239,7 @@ int __ptep_set_access_flags(struct vm_area_struct *vma,
 	 * flush_tlb_fix_spurious_fault().
 	 */
 	if (dirty)
-		local_flush_tlb_page(vma, address);
+		__flush_tlb_page(vma, address, TLBF_NOBROADCAST);
 	return 1;
 }
 
diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c
index 04a23a497f205..680749611a9a0 100644
--- a/arch/arm64/mm/gcs.c
+++ b/arch/arm64/mm/gcs.c
@@ -12,19 +12,7 @@
 
 static unsigned long alloc_gcs(unsigned long addr, unsigned long size)
 {
-	int flags = MAP_ANONYMOUS | MAP_PRIVATE;
-	struct mm_struct *mm = current->mm;
-	unsigned long mapped_addr, unused;
-
-	if (addr)
-		flags |= MAP_FIXED_NOREPLACE;
-
-	mmap_write_lock(mm);
-	mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags,
-			      VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL);
-	mmap_write_unlock(mm);
-
-	return mapped_addr;
+	return vm_mmap_shadow_stack(addr, size, 0);
 }
 
 static unsigned long gcs_size(unsigned long size)
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index a42c05cf56408..0b7ccd0cbb9ec 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -181,7 +181,7 @@ static pte_t get_clear_contig_flush(struct mm_struct *mm,
 	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
 	unsigned long end = addr + (pgsize * ncontig);
 
-	__flush_hugetlb_tlb_range(&vma, addr, end, pgsize, true);
+	__flush_hugetlb_tlb_range(&vma, addr, end, pgsize, TLBF_NOWALKCACHE);
 	return orig_pte;
 }
 
@@ -209,7 +209,7 @@ static void clear_flush(struct mm_struct *mm,
 	if (mm == &init_mm)
 		flush_tlb_kernel_range(saddr, addr);
 	else
-		__flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true);
+		__flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, TLBF_NOWALKCACHE);
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 96711b8578fd0..417ec7efe5692 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -328,6 +328,11 @@ void __init bootmem_init(void)
 	memblock_dump_all();
 }
 
+void __init arch_setup_zero_pages(void)
+{
+	__zero_page = phys_to_page(__pa_symbol(empty_zero_page));
+}
+
 void __init arch_mm_preinit(void)
 {
 	unsigned int flags = SWIOTLB_VERBOSE;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index a6a00accf4f93..b9c9f46107365 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -64,13 +64,6 @@ static bool rodata_is_rw __ro_after_init = true;
  */
 long __section(".mmuoff.data.write") __early_cpu_boot_status;
 
-/*
- * Empty_zero_page is a special page that is used for zero-initialized data
- * and COW.
- */
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 static DEFINE_SPINLOCK(swapper_pgdir_lock);
 static DEFINE_MUTEX(fixmap_lock);
 
@@ -2149,7 +2142,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
 		 */
 		if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte))
 			__flush_tlb_range(vma, addr, nr * PAGE_SIZE,
-					  PAGE_SIZE, true, 3);
+					  PAGE_SIZE, 3, TLBF_NOWALKCACHE);
 	}
 
 	return pte;
@@ -2206,7 +2199,7 @@ void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp)
 }
 
 #ifdef CONFIG_ARCH_HAS_PKEYS
-int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned long init_val)
+int arch_set_user_pkey_access(int pkey, unsigned long init_val)
 {
 	u64 new_por;
 	u64 old_por;
diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index bbea4f36f9f22..d13de4222cfb7 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -187,7 +187,9 @@
 /* Rn - imm12; set condition flags */
 #define A64_CMP_I(sf, Rn, imm12) A64_SUBS_I(sf, A64_ZR, Rn, imm12)
 /* Rd = Rn */
-#define A64_MOV(sf, Rd, Rn) A64_ADD_I(sf, Rd, Rn, 0)
+#define A64_MOV(sf, Rd, Rn) \
+	(((Rd) == A64_SP || (Rn) == A64_SP) ? A64_ADD_I(sf, Rd, Rn, 0) : \
+	 aarch64_insn_gen_move_reg(Rd, Rn, A64_VARIANT(sf)))
 
 /* Bitfield move */
 #define A64_BITFIELD(sf, Rd, Rn, immr, imms, type) \
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 9d1c211080571..51dcca5b2fa6e 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -3243,6 +3243,14 @@ UnsignedEnum	3:0	ID_BITS
 EndEnum
 EndSysreg
 
+Sysreg	ICC_HPPIR_EL1	3	0	12	10	3
+Res0	63:33
+Field	32	HPPIV
+Field	31:29	TYPE
+Res0	28:24
+Field	23:0	ID
+EndSysreg
+
 Sysreg	ICC_ICSR_EL1	3	0	12	10	4
 Res0	63:48
 Field	47:32	IAFFID
@@ -3257,6 +3265,11 @@ Field	1	Enabled
 Field	0	F
 EndSysreg
 
+Sysreg	ICC_IAFFIDR_EL1	3	0	12	10	5
+Res0	63:16
+Field	15:0	IAFFID
+EndSysreg
+
 SysregFields	ICC_PPI_ENABLERx_EL1
 Field	63	EN63
 Field	62	EN62
@@ -3663,6 +3676,42 @@ Res0	14:12
 Field	11:0	AFFINITY
 EndSysreg
 
+Sysreg	ICC_APR_EL1	3	1	12	0	0
+Res0	63:32
+Field	31	P31
+Field	30	P30
+Field	29	P29
+Field	28	P28
+Field	27	P27
+Field	26	P26
+Field	25	P25
+Field	24	P24
+Field	23	P23
+Field	22	P22
+Field	21	P21
+Field	20	P20
+Field	19	P19
+Field	18	P18
+Field	17	P17
+Field	16	P16
+Field	15	P15
+Field	14	P14
+Field	13	P13
+Field	12	P12
+Field	11	P11
+Field	10	P10
+Field	9	P9
+Field	8	P8
+Field	7	P7
+Field	6	P6
+Field	5	P5
+Field	4	P4
+Field	3	P3
+Field	2	P2
+Field	1	P1
+Field	0	P0
+EndSysreg
+
 Sysreg	ICC_CR0_EL1	3	1	12	0	1
 Res0	63:39
 Field	38	PID
@@ -4687,6 +4736,42 @@ Field	31:16	PhyPARTID29
 Field	15:0	PhyPARTID28
 EndSysreg
 
+Sysreg	ICH_APR_EL2	3	4	12	8	4
+Res0	63:32
+Field	31	P31
+Field	30	P30
+Field	29	P29
+Field	28	P28
+Field	27	P27
+Field	26	P26
+Field	25	P25
+Field	24	P24
+Field	23	P23
+Field	22	P22
+Field	21	P21
+Field	20	P20
+Field	19	P19
+Field	18	P18
+Field	17	P17
+Field	16	P16
+Field	15	P15
+Field	14	P14
+Field	13	P13
+Field	12	P12
+Field	11	P11
+Field	10	P10
+Field	9	P9
+Field	8	P8
+Field	7	P7
+Field	6	P6
+Field	5	P5
+Field	4	P4
+Field	3	P3
+Field	2	P2
+Field	1	P1
+Field	0	P0
+EndSysreg
+
 Sysreg	ICH_HFGRTR_EL2	3	4	12	9	4
 Res0	63:21
 Field	20	ICC_PPI_ACTIVERn_EL1
@@ -4735,6 +4820,306 @@ Field	1	GICCDDIS
 Field	0	GICCDEN
 EndSysreg
 
+SysregFields	ICH_PPI_DVIRx_EL2
+Field	63	DVI63
+Field	62	DVI62
+Field	61	DVI61
+Field	60	DVI60
+Field	59	DVI59
+Field	58	DVI58
+Field	57	DVI57
+Field	56	DVI56
+Field	55	DVI55
+Field	54	DVI54
+Field	53	DVI53
+Field	52	DVI52
+Field	51	DVI51
+Field	50	DVI50
+Field	49	DVI49
+Field	48	DVI48
+Field	47	DVI47
+Field	46	DVI46
+Field	45	DVI45
+Field	44	DVI44
+Field	43	DVI43
+Field	42	DVI42
+Field	41	DVI41
+Field	40	DVI40
+Field	39	DVI39
+Field	38	DVI38
+Field	37	DVI37
+Field	36	DVI36
+Field	35	DVI35
+Field	34	DVI34
+Field	33	DVI33
+Field	32	DVI32
+Field	31	DVI31
+Field	30	DVI30
+Field	29	DVI29
+Field	28	DVI28
+Field	27	DVI27
+Field	26	DVI26
+Field	25	DVI25
+Field	24	DVI24
+Field	23	DVI23
+Field	22	DVI22
+Field	21	DVI21
+Field	20	DVI20
+Field	19	DVI19
+Field	18	DVI18
+Field	17	DVI17
+Field	16	DVI16
+Field	15	DVI15
+Field	14	DVI14
+Field	13	DVI13
+Field	12	DVI12
+Field	11	DVI11
+Field	10	DVI10
+Field	9	DVI9
+Field	8	DVI8
+Field	7	DVI7
+Field	6	DVI6
+Field	5	DVI5
+Field	4	DVI4
+Field	3	DVI3
+Field	2	DVI2
+Field	1	DVI1
+Field	0	DVI0
+EndSysregFields
+
+Sysreg	ICH_PPI_DVIR0_EL2	3	4	12	10	0
+Fields ICH_PPI_DVIx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_DVIR1_EL2	3	4	12	10	1
+Fields ICH_PPI_DVIx_EL2
+EndSysreg
+
+SysregFields	ICH_PPI_ENABLERx_EL2
+Field	63	EN63
+Field	62	EN62
+Field	61	EN61
+Field	60	EN60
+Field	59	EN59
+Field	58	EN58
+Field	57	EN57
+Field	56	EN56
+Field	55	EN55
+Field	54	EN54
+Field	53	EN53
+Field	52	EN52
+Field	51	EN51
+Field	50	EN50
+Field	49	EN49
+Field	48	EN48
+Field	47	EN47
+Field	46	EN46
+Field	45	EN45
+Field	44	EN44
+Field	43	EN43
+Field	42	EN42
+Field	41	EN41
+Field	40	EN40
+Field	39	EN39
+Field	38	EN38
+Field	37	EN37
+Field	36	EN36
+Field	35	EN35
+Field	34	EN34
+Field	33	EN33
+Field	32	EN32
+Field	31	EN31
+Field	30	EN30
+Field	29	EN29
+Field	28	EN28
+Field	27	EN27
+Field	26	EN26
+Field	25	EN25
+Field	24	EN24
+Field	23	EN23
+Field	22	EN22
+Field	21	EN21
+Field	20	EN20
+Field	19	EN19
+Field	18	EN18
+Field	17	EN17
+Field	16	EN16
+Field	15	EN15
+Field	14	EN14
+Field	13	EN13
+Field	12	EN12
+Field	11	EN11
+Field	10	EN10
+Field	9	EN9
+Field	8	EN8
+Field	7	EN7
+Field	6	EN6
+Field	5	EN5
+Field	4	EN4
+Field	3	EN3
+Field	2	EN2
+Field	1	EN1
+Field	0	EN0
+EndSysregFields
+
+Sysreg	ICH_PPI_ENABLER0_EL2	3	4	12	10	2
+Fields ICH_PPI_ENABLERx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_ENABLER1_EL2	3	4	12	10	3
+Fields ICH_PPI_ENABLERx_EL2
+EndSysreg
+
+SysregFields	ICH_PPI_PENDRx_EL2
+Field	63	PEND63
+Field	62	PEND62
+Field	61	PEND61
+Field	60	PEND60
+Field	59	PEND59
+Field	58	PEND58
+Field	57	PEND57
+Field	56	PEND56
+Field	55	PEND55
+Field	54	PEND54
+Field	53	PEND53
+Field	52	PEND52
+Field	51	PEND51
+Field	50	PEND50
+Field	49	PEND49
+Field	48	PEND48
+Field	47	PEND47
+Field	46	PEND46
+Field	45	PEND45
+Field	44	PEND44
+Field	43	PEND43
+Field	42	PEND42
+Field	41	PEND41
+Field	40	PEND40
+Field	39	PEND39
+Field	38	PEND38
+Field	37	PEND37
+Field	36	PEND36
+Field	35	PEND35
+Field	34	PEND34
+Field	33	PEND33
+Field	32	PEND32
+Field	31	PEND31
+Field	30	PEND30
+Field	29	PEND29
+Field	28	PEND28
+Field	27	PEND27
+Field	26	PEND26
+Field	25	PEND25
+Field	24	PEND24
+Field	23	PEND23
+Field	22	PEND22
+Field	21	PEND21
+Field	20	PEND20
+Field	19	PEND19
+Field	18	PEND18
+Field	17	PEND17
+Field	16	PEND16
+Field	15	PEND15
+Field	14	PEND14
+Field	13	PEND13
+Field	12	PEND12
+Field	11	PEND11
+Field	10	PEND10
+Field	9	PEND9
+Field	8	PEND8
+Field	7	PEND7
+Field	6	PEND6
+Field	5	PEND5
+Field	4	PEND4
+Field	3	PEND3
+Field	2	PEND2
+Field	1	PEND1
+Field	0	PEND0
+EndSysregFields
+
+Sysreg	ICH_PPI_PENDR0_EL2	3	4	12	10	4
+Fields ICH_PPI_PENDRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PENDR1_EL2	3	4	12	10	5
+Fields ICH_PPI_PENDRx_EL2
+EndSysreg
+
+SysregFields	ICH_PPI_ACTIVERx_EL2
+Field	63	ACTIVE63
+Field	62	ACTIVE62
+Field	61	ACTIVE61
+Field	60	ACTIVE60
+Field	59	ACTIVE59
+Field	58	ACTIVE58
+Field	57	ACTIVE57
+Field	56	ACTIVE56
+Field	55	ACTIVE55
+Field	54	ACTIVE54
+Field	53	ACTIVE53
+Field	52	ACTIVE52
+Field	51	ACTIVE51
+Field	50	ACTIVE50
+Field	49	ACTIVE49
+Field	48	ACTIVE48
+Field	47	ACTIVE47
+Field	46	ACTIVE46
+Field	45	ACTIVE45
+Field	44	ACTIVE44
+Field	43	ACTIVE43
+Field	42	ACTIVE42
+Field	41	ACTIVE41
+Field	40	ACTIVE40
+Field	39	ACTIVE39
+Field	38	ACTIVE38
+Field	37	ACTIVE37
+Field	36	ACTIVE36
+Field	35	ACTIVE35
+Field	34	ACTIVE34
+Field	33	ACTIVE33
+Field	32	ACTIVE32
+Field	31	ACTIVE31
+Field	30	ACTIVE30
+Field	29	ACTIVE29
+Field	28	ACTIVE28
+Field	27	ACTIVE27
+Field	26	ACTIVE26
+Field	25	ACTIVE25
+Field	24	ACTIVE24
+Field	23	ACTIVE23
+Field	22	ACTIVE22
+Field	21	ACTIVE21
+Field	20	ACTIVE20
+Field	19	ACTIVE19
+Field	18	ACTIVE18
+Field	17	ACTIVE17
+Field	16	ACTIVE16
+Field	15	ACTIVE15
+Field	14	ACTIVE14
+Field	13	ACTIVE13
+Field	12	ACTIVE12
+Field	11	ACTIVE11
+Field	10	ACTIVE10
+Field	9	ACTIVE9
+Field	8	ACTIVE8
+Field	7	ACTIVE7
+Field	6	ACTIVE6
+Field	5	ACTIVE5
+Field	4	ACTIVE4
+Field	3	ACTIVE3
+Field	2	ACTIVE2
+Field	1	ACTIVE1
+Field	0	ACTIVE0
+EndSysregFields
+
+Sysreg	ICH_PPI_ACTIVER0_EL2	3	4	12	10	6
+Fields ICH_PPI_ACTIVERx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_ACTIVER1_EL2	3	4	12	10	7
+Fields ICH_PPI_ACTIVERx_EL2
+EndSysreg
+
 Sysreg	ICH_HCR_EL2	3	4	12	11	0
 Res0	63:32
 Field	31:27	EOIcount
@@ -4789,6 +5174,18 @@ Field	1	V3
 Field	0	En
 EndSysreg
 
+Sysreg	ICH_CONTEXTR_EL2	3	4	12	11	6
+Field	63	V
+Field	62	F
+Field	61	IRICHPPIDIS
+Field	60	DB
+Field	59:55	DBPM
+Res0	54:48
+Field	47:32	VPE
+Res0	31:16
+Field	15:0	VM
+EndSysreg
+
 Sysreg	ICH_VMCR_EL2	3	4	12	11	7
 Prefix	FEAT_GCIE
 Res0	63:32
@@ -4810,6 +5207,89 @@ Field	1	VENG1
 Field	0	VENG0
 EndSysreg
 
+SysregFields	ICH_PPI_PRIORITYRx_EL2
+Res0	63:61
+Field	60:56	Priority7
+Res0	55:53
+Field	52:48	Priority6
+Res0	47:45
+Field	44:40	Priority5
+Res0	39:37
+Field	36:32	Priority4
+Res0	31:29
+Field	28:24	Priority3
+Res0	23:21
+Field	20:16	Priority2
+Res0	15:13
+Field	12:8	Priority1
+Res0	7:5
+Field	4:0	Priority0
+EndSysregFields
+
+Sysreg	ICH_PPI_PRIORITYR0_EL2	3	4	12	14	0
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR1_EL2	3	4	12	14	1
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR2_EL2	3	4	12	14	2
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR3_EL2	3	4	12	14	3
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR4_EL2	3	4	12	14	4
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR5_EL2	3	4	12	14	5
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR6_EL2	3	4	12	14	6
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR7_EL2	3	4	12	14	7
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR8_EL2	3	4	12	15	0
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR9_EL2	3	4	12	15	1
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR10_EL2	3	4	12	15	2
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR11_EL2	3	4	12	15	3
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR12_EL2	3	4	12	15	4
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR13_EL2	3	4	12	15	5
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR14_EL2	3	4	12	15	6
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
+Sysreg	ICH_PPI_PRIORITYR15_EL2	3	4	12	15	7
+Fields	ICH_PPI_PRIORITYRx_EL2
+EndSysreg
+
 Sysreg	CONTEXTIDR_EL2	3	4	13	0	1
 Fields	CONTEXTIDR_ELx
 EndSysreg
diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h
index d606afbabce19..bafcd5823531a 100644
--- a/arch/csky/include/asm/pgtable.h
+++ b/arch/csky/include/asm/pgtable.h
@@ -76,9 +76,6 @@
 #define MAX_SWAPFILES_CHECK() \
 		BUILD_BUG_ON(MAX_SWAPFILES_SHIFT != 5)
 
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-
 extern void load_pgd(unsigned long pg_dir);
 extern pte_t invalid_pte_table[PTRS_PER_PTE];
 
diff --git a/arch/csky/mm/init.c b/arch/csky/mm/init.c
index 573da66b25438..fa16015ea1c04 100644
--- a/arch/csky/mm/init.c
+++ b/arch/csky/mm/init.c
@@ -38,9 +38,6 @@ pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
 pte_t kernel_pte_tables[PTRS_KERN_TABLE] __page_aligned_bss;
 
 EXPORT_SYMBOL(invalid_pte_table);
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
-						__page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
 
 void free_initmem(void)
 {
diff --git a/arch/hexagon/include/asm/page.h b/arch/hexagon/include/asm/page.h
index f0aed3ed812b9..6d82572a7f210 100644
--- a/arch/hexagon/include/asm/page.h
+++ b/arch/hexagon/include/asm/page.h
@@ -90,7 +90,7 @@ struct page;
 #define virt_to_page(kaddr) pfn_to_page(PFN_DOWN(__pa(kaddr)))
 
 /* Default vm area behavior is non-executable.  */
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_NON_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_NON_EXEC
 
 #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
 
diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h
index fbf24d1d1ca6a..27b269e2870d3 100644
--- a/arch/hexagon/include/asm/pgtable.h
+++ b/arch/hexagon/include/asm/pgtable.h
@@ -14,9 +14,6 @@
 #include <asm/page.h>
 #include <asm-generic/pgtable-nopmd.h>
 
-/* A handy thing to have if one has the RAM. Declared in head.S */
-extern unsigned long empty_zero_page;
-
 /*
  * The PTE model described here is that of the Hexagon Virtual Machine,
  * which autonomously walks 2-level page tables.  At a lower level, we
@@ -348,9 +345,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 	return (unsigned long)__va(pmd_val(pmd) & PAGE_MASK);
 }
 
-/* ZERO_PAGE - returns the globally shared zero page */
-#define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page))
-
 /*
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
  * are !pte_none() && !pte_present().
diff --git a/arch/hexagon/include/uapi/asm/sigcontext.h b/arch/hexagon/include/uapi/asm/sigcontext.h
index 7171edb1b8b71..179a97041b593 100644
--- a/arch/hexagon/include/uapi/asm/sigcontext.h
+++ b/arch/hexagon/include/uapi/asm/sigcontext.h
@@ -29,6 +29,6 @@
  */
 struct sigcontext {
 	struct user_regs_struct sc_regs;
-} __aligned(8);
+} __attribute__((aligned(8)));
 
 #endif
diff --git a/arch/hexagon/kernel/head.S b/arch/hexagon/kernel/head.S
index 0b016308cc79f..908ffece91324 100644
--- a/arch/hexagon/kernel/head.S
+++ b/arch/hexagon/kernel/head.S
@@ -216,8 +216,3 @@ __head_s_vaddr_target:
 .p2align PAGE_SHIFT
 ENTRY(external_cmdline_buffer)
         .fill _PAGE_SIZE,1,0
-
-.data
-.p2align PAGE_SHIFT
-ENTRY(empty_zero_page)
-        .fill _PAGE_SIZE,1,0
diff --git a/arch/hexagon/kernel/hexagon_ksyms.c b/arch/hexagon/kernel/hexagon_ksyms.c
index 36a80e31d1879..81bc6f81e2009 100644
--- a/arch/hexagon/kernel/hexagon_ksyms.c
+++ b/arch/hexagon/kernel/hexagon_ksyms.c
@@ -17,7 +17,6 @@ EXPORT_SYMBOL(raw_copy_to_user);
 EXPORT_SYMBOL(__vmgetie);
 EXPORT_SYMBOL(__vmsetie);
 EXPORT_SYMBOL(__vmyield);
-EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memset);
 
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index d211c6572b0a6..9d9a8fe0f7ad1 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -215,6 +215,7 @@ config LOONGARCH
 	select USE_PERCPU_NUMA_NODE_ID
 	select USER_STACKTRACE_SUPPORT
 	select VDSO_GETRANDOM
+	select XOR_BLOCKS_ARCH
 	select ZONE_DMA32
 
 config 32BIT
@@ -304,6 +305,9 @@ config AS_HAS_LBT_EXTENSION
 config AS_HAS_LVZ_EXTENSION
 	def_bool $(as-instr,hvcl 0)
 
+config AS_HAS_SCQ_EXTENSION
+	def_bool $(as-instr,sc.q \$t0$(comma)\$t1$(comma)\$t2)
+
 config CC_HAS_ANNOTATE_TABLEJUMP
 	def_bool $(cc-option,-mannotate-tablejump)
 
diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h
index 58cabab6d90d1..909f9274fe71a 100644
--- a/arch/loongarch/include/asm/cmpxchg.h
+++ b/arch/loongarch/include/asm/cmpxchg.h
@@ -238,6 +238,8 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int
 	arch_cmpxchg((ptr), (o), (n));					\
   })
 
+#ifdef CONFIG_AS_HAS_SCQ_EXTENSION
+
 union __u128_halves {
 	u128 full;
 	struct {
@@ -290,6 +292,9 @@ union __u128_halves {
 	BUILD_BUG_ON(sizeof(*(ptr)) != 16);				\
 	__arch_cmpxchg128(ptr, o, n, "");				\
 })
+
+#endif /* CONFIG_AS_HAS_SCQ_EXTENSION */
+
 #else
 #include <asm-generic/cmpxchg-local.h>
 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h
index 3943647503a95..537add26daf43 100644
--- a/arch/loongarch/include/asm/irq.h
+++ b/arch/loongarch/include/asm/irq.h
@@ -48,7 +48,6 @@ void spurious_interrupt(void);
  */
 #define NR_VECTORS		256
 #define NR_LEGACY_VECTORS	16
-#define IRQ_MATRIX_BITS		NR_VECTORS
 
 #define AVEC_IRQ_SHIFT		4
 #define AVEC_IRQ_BIT		8
diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h
index 327bf0bc92bf9..79235f4fc399f 100644
--- a/arch/loongarch/include/asm/page.h
+++ b/arch/loongarch/include/asm/page.h
@@ -104,7 +104,7 @@ struct page *tlb_virt_to_page(unsigned long kaddr);
 extern int __virt_addr_valid(volatile void *kaddr);
 #define virt_addr_valid(kaddr)	__virt_addr_valid((volatile void *)(kaddr))
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
 
 #include <asm-generic/memory_model.h>
 #include <asm-generic/getorder.h>
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index c33b3bcb733e3..155f70e934601 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -74,15 +74,6 @@
 struct mm_struct;
 struct vm_area_struct;
 
-/*
- * ZERO_PAGE is a global shared page that is always zero; used
- * for zero-mapped memory areas etc..
- */
-
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-
-#define ZERO_PAGE(vaddr)	virt_to_page(empty_zero_page)
-
 #ifdef CONFIG_32BIT
 
 #define VMALLOC_START	(vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
@@ -113,7 +104,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 	 min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits) / 2) - PMD_SIZE - VMEMMAP_SIZE - KFENCE_AREA_SIZE)
 #endif
 
-#define vmemmap		((struct page *)((VMALLOC_END + PMD_SIZE) & PMD_MASK))
+#define VMEMMAP_ALIGN	max(PMD_SIZE, MAX_FOLIO_VMEMMAP_ALIGN)
+#define vmemmap		((struct page *)(ALIGN(VMALLOC_END, VMEMMAP_ALIGN)))
 #define VMEMMAP_END	((unsigned long)vmemmap + VMEMMAP_SIZE - 1)
 
 #define KFENCE_AREA_START	(VMEMMAP_END + 1)
diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h
index 4e259d490e456..438269313e78c 100644
--- a/arch/loongarch/include/asm/uaccess.h
+++ b/arch/loongarch/include/asm/uaccess.h
@@ -253,8 +253,13 @@ do {									\
 									\
 	__get_kernel_common(*((type *)(dst)), sizeof(type),		\
 			    (__force type *)(src));			\
-	if (unlikely(__gu_err))						\
+	if (unlikely(__gu_err))	{					\
+		pr_info("%s: memory access failed, ecode 0x%x\n",	\
+			__func__, read_csr_excode());			\
+		pr_info("%s: the caller is %pS\n",			\
+			__func__, __builtin_return_address(0));		\
 		goto err_label;						\
+	}								\
 } while (0)
 
 #define __put_kernel_nofault(dst, src, type, err_label)			\
@@ -264,8 +269,13 @@ do {									\
 									\
 	__pu_val = *(__force type *)(src);				\
 	__put_kernel_common(((type *)(dst)), sizeof(type));		\
-	if (unlikely(__pu_err))						\
+	if (unlikely(__pu_err))	{					\
+		pr_info("%s: memory access failed, ecode 0x%x\n",	\
+			__func__, read_csr_excode());			\
+		pr_info("%s: the caller is %pS\n",			\
+			__func__, __builtin_return_address(0));		\
 		goto err_label;						\
+	}								\
 } while (0)
 
 extern unsigned long __copy_user(void *to, const void *from, __kernel_size_t n);
diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h
deleted file mode 100644
index 12467fffee468..0000000000000
--- a/arch/loongarch/include/asm/xor.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_H
-#define _ASM_LOONGARCH_XOR_H
-
-#include <asm/cpu-features.h>
-#include <asm/xor_simd.h>
-
-#ifdef CONFIG_CPU_HAS_LSX
-static struct xor_block_template xor_block_lsx = {
-	.name = "lsx",
-	.do_2 = xor_lsx_2,
-	.do_3 = xor_lsx_3,
-	.do_4 = xor_lsx_4,
-	.do_5 = xor_lsx_5,
-};
-
-#define XOR_SPEED_LSX()					\
-	do {						\
-		if (cpu_has_lsx)			\
-			xor_speed(&xor_block_lsx);	\
-	} while (0)
-#else /* CONFIG_CPU_HAS_LSX */
-#define XOR_SPEED_LSX()
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-static struct xor_block_template xor_block_lasx = {
-	.name = "lasx",
-	.do_2 = xor_lasx_2,
-	.do_3 = xor_lasx_3,
-	.do_4 = xor_lasx_4,
-	.do_5 = xor_lasx_5,
-};
-
-#define XOR_SPEED_LASX()					\
-	do {							\
-		if (cpu_has_lasx)				\
-			xor_speed(&xor_block_lasx);		\
-	} while (0)
-#else /* CONFIG_CPU_HAS_LASX */
-#define XOR_SPEED_LASX()
-#endif /* CONFIG_CPU_HAS_LASX */
-
-/*
- * For grins, also test the generic routines.
- *
- * More importantly: it cannot be ruled out at this point of time, that some
- * future (maybe reduced) models could run the vector algorithms slower than
- * the scalar ones, maybe for errata or micro-op reasons. It may be
- * appropriate to revisit this after one or two more uarch generations.
- */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-do {							\
-	xor_speed(&xor_block_8regs);			\
-	xor_speed(&xor_block_8regs_p);			\
-	xor_speed(&xor_block_32regs);			\
-	xor_speed(&xor_block_32regs_p);			\
-	XOR_SPEED_LSX();				\
-	XOR_SPEED_LASX();				\
-} while (0)
-
-#endif /* _ASM_LOONGARCH_XOR_H */
diff --git a/arch/loongarch/include/asm/xor_simd.h b/arch/loongarch/include/asm/xor_simd.h
deleted file mode 100644
index 471b96332f381..0000000000000
--- a/arch/loongarch/include/asm/xor_simd.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_SIMD_H
-#define _ASM_LOONGARCH_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2);
-void xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4);
-void xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
-	        const unsigned long * __restrict p2);
-void xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
-	        const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
-	        const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-	        const unsigned long * __restrict p4);
-void xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
-	        const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-	        const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* _ASM_LOONGARCH_XOR_SIMD_H */
diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c
index bf037f0c6b26c..1a728082944cb 100644
--- a/arch/loongarch/kernel/inst.c
+++ b/arch/loongarch/kernel/inst.c
@@ -246,32 +246,51 @@ static int text_copy_cb(void *data)
 
 	if (smp_processor_id() == copy->cpu) {
 		ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len);
-		if (ret)
+		if (ret) {
 			pr_err("%s: operation failed\n", __func__);
+			return ret;
+		}
 	}
 
 	flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len);
 
-	return ret;
+	return 0;
 }
 
 int larch_insn_text_copy(void *dst, void *src, size_t len)
 {
 	int ret = 0;
+	int err = 0;
 	size_t start, end;
 	struct insn_copy copy = {
 		.dst = dst,
 		.src = src,
 		.len = len,
-		.cpu = smp_processor_id(),
+		.cpu = raw_smp_processor_id(),
 	};
 
+	/*
+	 * Ensure copy.cpu won't be hot removed before stop_machine.
+	 * If it is removed nobody will really update the text.
+	 */
+	lockdep_assert_cpus_held();
+
 	start = round_down((size_t)dst, PAGE_SIZE);
 	end   = round_up((size_t)dst + len, PAGE_SIZE);
 
-	set_memory_rw(start, (end - start) / PAGE_SIZE);
-	ret = stop_machine(text_copy_cb, &copy, cpu_online_mask);
-	set_memory_rox(start, (end - start) / PAGE_SIZE);
+	err = set_memory_rw(start, (end - start) / PAGE_SIZE);
+	if (err) {
+		pr_info("%s: set_memory_rw() failed\n", __func__);
+		return err;
+	}
+
+	ret = stop_machine_cpuslocked(text_copy_cb, &copy, cpu_online_mask);
+
+	err = set_memory_rox(start, (end - start) / PAGE_SIZE);
+	if (err) {
+		pr_info("%s: set_memory_rox() failed\n", __func__);
+		return err;
+	}
 
 	return ret;
 }
diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c
index 4ac1c30861526..ac3a0baa5d00a 100644
--- a/arch/loongarch/kernel/process.c
+++ b/arch/loongarch/kernel/process.c
@@ -52,6 +52,7 @@
 #include <asm/switch_to.h>
 #include <asm/unwind.h>
 #include <asm/vdso.h>
+#include <asm/vdso/vdso.h>
 
 #ifdef CONFIG_STACKPROTECTOR
 #include <linux/stackprotector.h>
diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c
index 0aa10cadb9591..8ce8159c10b93 100644
--- a/arch/loongarch/kernel/vdso.c
+++ b/arch/loongarch/kernel/vdso.c
@@ -18,6 +18,7 @@
 
 #include <asm/page.h>
 #include <asm/vdso.h>
+#include <asm/vdso/vdso.h>
 #include <vdso/helpers.h>
 #include <vdso/vsyscall.h>
 #include <vdso/datapage.h>
diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c
index 17b3d5b36cfc0..8cc5ee1c53efb 100644
--- a/arch/loongarch/kvm/vm.c
+++ b/arch/loongarch/kvm/vm.c
@@ -49,8 +49,8 @@ static void kvm_vm_init_features(struct kvm *kvm)
 		kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PMU);
 
 	/* Enable all PV features by default */
-	kvm->arch.pv_features = BIT(KVM_FEATURE_IPI);
-	kvm->arch.kvm_features = BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI);
+	kvm->arch.pv_features |= BIT(KVM_FEATURE_IPI);
+	kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI);
 	if (kvm_pvtime_supported()) {
 		kvm->arch.pv_features |= BIT(KVM_FEATURE_PREEMPT);
 		kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME);
diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile
index ccea3bbd43531..827a88529a425 100644
--- a/arch/loongarch/lib/Makefile
+++ b/arch/loongarch/lib/Makefile
@@ -8,6 +8,4 @@ lib-y	+= delay.o memset.o memcpy.o memmove.o \
 
 obj-$(CONFIG_ARCH_SUPPORTS_INT128) += tishift.o
 
-obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o
-
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/loongarch/lib/xor_simd.c b/arch/loongarch/lib/xor_simd.c
deleted file mode 100644
index 84cd24b728c47..0000000000000
--- a/arch/loongarch/lib/xor_simd.c
+++ /dev/null
@@ -1,93 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * LoongArch SIMD XOR operations
- *
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-
-#include "xor_simd.h"
-
-/*
- * Process one cache line (64 bytes) per loop. This is assuming all future
- * popular LoongArch cores are similar performance-characteristics-wise to the
- * current models.
- */
-#define LINE_WIDTH 64
-
-#ifdef CONFIG_CPU_HAS_LSX
-
-#define LD(reg, base, offset)	\
-	"vld $vr" #reg ", %[" #base "], " #offset "\n\t"
-#define ST(reg, base, offset)	\
-	"vst $vr" #reg ", %[" #base "], " #offset "\n\t"
-#define XOR(dj, k)	"vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t"
-
-#define LD_INOUT_LINE(base)	\
-	LD(0, base, 0)		\
-	LD(1, base, 16)		\
-	LD(2, base, 32)		\
-	LD(3, base, 48)
-
-#define LD_AND_XOR_LINE(base)	\
-	LD(4, base, 0)		\
-	LD(5, base, 16)		\
-	LD(6, base, 32)		\
-	LD(7, base, 48)		\
-	XOR(0, 4)		\
-	XOR(1, 5)		\
-	XOR(2, 6)		\
-	XOR(3, 7)
-
-#define ST_LINE(base)		\
-	ST(0, base, 0)		\
-	ST(1, base, 16)		\
-	ST(2, base, 32)		\
-	ST(3, base, 48)
-
-#define XOR_FUNC_NAME(nr) __xor_lsx_##nr
-#include "xor_template.c"
-
-#undef LD
-#undef ST
-#undef XOR
-#undef LD_INOUT_LINE
-#undef LD_AND_XOR_LINE
-#undef ST_LINE
-#undef XOR_FUNC_NAME
-
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-
-#define LD(reg, base, offset)	\
-	"xvld $xr" #reg ", %[" #base "], " #offset "\n\t"
-#define ST(reg, base, offset)	\
-	"xvst $xr" #reg ", %[" #base "], " #offset "\n\t"
-#define XOR(dj, k)	"xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t"
-
-#define LD_INOUT_LINE(base)	\
-	LD(0, base, 0)		\
-	LD(1, base, 32)
-
-#define LD_AND_XOR_LINE(base)	\
-	LD(2, base, 0)		\
-	LD(3, base, 32)		\
-	XOR(0, 2)		\
-	XOR(1, 3)
-
-#define ST_LINE(base)		\
-	ST(0, base, 0)		\
-	ST(1, base, 32)
-
-#define XOR_FUNC_NAME(nr) __xor_lasx_##nr
-#include "xor_template.c"
-
-#undef LD
-#undef ST
-#undef XOR
-#undef LD_INOUT_LINE
-#undef LD_AND_XOR_LINE
-#undef ST_LINE
-#undef XOR_FUNC_NAME
-
-#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/arch/loongarch/lib/xor_simd.h b/arch/loongarch/lib/xor_simd.h
deleted file mode 100644
index f50f32514d804..0000000000000
--- a/arch/loongarch/lib/xor_simd.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Simple interface to link xor_simd.c and xor_simd_glue.c
- *
- * Separating these files ensures that no SIMD instructions are run outside of
- * the kfpu critical section.
- */
-
-#ifndef __LOONGARCH_LIB_XOR_SIMD_H
-#define __LOONGARCH_LIB_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void __xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
-		 const unsigned long * __restrict p2);
-void __xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
-		 const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void __xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
-		 const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-		 const unsigned long * __restrict p4);
-void __xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
-		 const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-		 const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void __xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
-		  const unsigned long * __restrict p2);
-void __xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
-		  const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void __xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
-		  const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-		  const unsigned long * __restrict p4);
-void __xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
-		  const unsigned long * __restrict p2, const unsigned long * __restrict p3,
-		  const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* __LOONGARCH_LIB_XOR_SIMD_H */
diff --git a/arch/loongarch/lib/xor_simd_glue.c b/arch/loongarch/lib/xor_simd_glue.c
deleted file mode 100644
index 393f689dbcf67..0000000000000
--- a/arch/loongarch/lib/xor_simd_glue.c
+++ /dev/null
@@ -1,72 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * LoongArch SIMD XOR operations
- *
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-
-#include <linux/export.h>
-#include <linux/sched.h>
-#include <asm/fpu.h>
-#include <asm/xor_simd.h>
-#include "xor_simd.h"
-
-#define MAKE_XOR_GLUE_2(flavor)							\
-void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1,	\
-		      const unsigned long * __restrict p2)			\
-{										\
-	kernel_fpu_begin();							\
-	__xor_##flavor##_2(bytes, p1, p2);					\
-	kernel_fpu_end();							\
-}										\
-EXPORT_SYMBOL_GPL(xor_##flavor##_2)
-
-#define MAKE_XOR_GLUE_3(flavor)							\
-void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1,	\
-		      const unsigned long * __restrict p2,			\
-		      const unsigned long * __restrict p3)			\
-{										\
-	kernel_fpu_begin();							\
-	__xor_##flavor##_3(bytes, p1, p2, p3);					\
-	kernel_fpu_end();							\
-}										\
-EXPORT_SYMBOL_GPL(xor_##flavor##_3)
-
-#define MAKE_XOR_GLUE_4(flavor)							\
-void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1,	\
-		      const unsigned long * __restrict p2,			\
-		      const unsigned long * __restrict p3,			\
-		      const unsigned long * __restrict p4)			\
-{										\
-	kernel_fpu_begin();							\
-	__xor_##flavor##_4(bytes, p1, p2, p3, p4);				\
-	kernel_fpu_end();							\
-}										\
-EXPORT_SYMBOL_GPL(xor_##flavor##_4)
-
-#define MAKE_XOR_GLUE_5(flavor)							\
-void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1,	\
-		      const unsigned long * __restrict p2,			\
-		      const unsigned long * __restrict p3,			\
-		      const unsigned long * __restrict p4,			\
-		      const unsigned long * __restrict p5)			\
-{										\
-	kernel_fpu_begin();							\
-	__xor_##flavor##_5(bytes, p1, p2, p3, p4, p5);				\
-	kernel_fpu_end();							\
-}										\
-EXPORT_SYMBOL_GPL(xor_##flavor##_5)
-
-#define MAKE_XOR_GLUES(flavor)		\
-	MAKE_XOR_GLUE_2(flavor);	\
-	MAKE_XOR_GLUE_3(flavor);	\
-	MAKE_XOR_GLUE_4(flavor);	\
-	MAKE_XOR_GLUE_5(flavor)
-
-#ifdef CONFIG_CPU_HAS_LSX
-MAKE_XOR_GLUES(lsx);
-#endif
-
-#ifdef CONFIG_CPU_HAS_LASX
-MAKE_XOR_GLUES(lasx);
-#endif
diff --git a/arch/loongarch/lib/xor_template.c b/arch/loongarch/lib/xor_template.c
deleted file mode 100644
index 0358ced7fe333..0000000000000
--- a/arch/loongarch/lib/xor_template.c
+++ /dev/null
@@ -1,110 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- *
- * Template for XOR operations, instantiated in xor_simd.c.
- *
- * Expected preprocessor definitions:
- *
- * - LINE_WIDTH
- * - XOR_FUNC_NAME(nr)
- * - LD_INOUT_LINE(buf)
- * - LD_AND_XOR_LINE(buf)
- * - ST_LINE(buf)
- */
-
-void XOR_FUNC_NAME(2)(unsigned long bytes,
-		      unsigned long * __restrict v1,
-		      const unsigned long * __restrict v2)
-{
-	unsigned long lines = bytes / LINE_WIDTH;
-
-	do {
-		__asm__ __volatile__ (
-			LD_INOUT_LINE(v1)
-			LD_AND_XOR_LINE(v2)
-			ST_LINE(v1)
-		: : [v1] "r"(v1), [v2] "r"(v2) : "memory"
-		);
-
-		v1 += LINE_WIDTH / sizeof(unsigned long);
-		v2 += LINE_WIDTH / sizeof(unsigned long);
-	} while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(3)(unsigned long bytes,
-		      unsigned long * __restrict v1,
-		      const unsigned long * __restrict v2,
-		      const unsigned long * __restrict v3)
-{
-	unsigned long lines = bytes / LINE_WIDTH;
-
-	do {
-		__asm__ __volatile__ (
-			LD_INOUT_LINE(v1)
-			LD_AND_XOR_LINE(v2)
-			LD_AND_XOR_LINE(v3)
-			ST_LINE(v1)
-		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory"
-		);
-
-		v1 += LINE_WIDTH / sizeof(unsigned long);
-		v2 += LINE_WIDTH / sizeof(unsigned long);
-		v3 += LINE_WIDTH / sizeof(unsigned long);
-	} while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(4)(unsigned long bytes,
-		      unsigned long * __restrict v1,
-		      const unsigned long * __restrict v2,
-		      const unsigned long * __restrict v3,
-		      const unsigned long * __restrict v4)
-{
-	unsigned long lines = bytes / LINE_WIDTH;
-
-	do {
-		__asm__ __volatile__ (
-			LD_INOUT_LINE(v1)
-			LD_AND_XOR_LINE(v2)
-			LD_AND_XOR_LINE(v3)
-			LD_AND_XOR_LINE(v4)
-			ST_LINE(v1)
-		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4)
-		: "memory"
-		);
-
-		v1 += LINE_WIDTH / sizeof(unsigned long);
-		v2 += LINE_WIDTH / sizeof(unsigned long);
-		v3 += LINE_WIDTH / sizeof(unsigned long);
-		v4 += LINE_WIDTH / sizeof(unsigned long);
-	} while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(5)(unsigned long bytes,
-		      unsigned long * __restrict v1,
-		      const unsigned long * __restrict v2,
-		      const unsigned long * __restrict v3,
-		      const unsigned long * __restrict v4,
-		      const unsigned long * __restrict v5)
-{
-	unsigned long lines = bytes / LINE_WIDTH;
-
-	do {
-		__asm__ __volatile__ (
-			LD_INOUT_LINE(v1)
-			LD_AND_XOR_LINE(v2)
-			LD_AND_XOR_LINE(v3)
-			LD_AND_XOR_LINE(v4)
-			LD_AND_XOR_LINE(v5)
-			ST_LINE(v1)
-		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4),
-		    [v5] "r"(v5) : "memory"
-		);
-
-		v1 += LINE_WIDTH / sizeof(unsigned long);
-		v2 += LINE_WIDTH / sizeof(unsigned long);
-		v3 += LINE_WIDTH / sizeof(unsigned long);
-		v4 += LINE_WIDTH / sizeof(unsigned long);
-		v5 += LINE_WIDTH / sizeof(unsigned long);
-	} while (--lines > 0);
-}
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index c331bf69d2ec1..00f3822b6e47c 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -36,9 +36,6 @@
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 void copy_user_highpage(struct page *to, struct page *from,
 	unsigned long vaddr, struct vm_area_struct *vma)
 {
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 3bd89f55960d9..9cb796e163794 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1379,9 +1379,11 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 {
 	int ret;
 
+	cpus_read_lock();
 	mutex_lock(&text_mutex);
 	ret = larch_insn_text_copy(dst, src, len);
 	mutex_unlock(&text_mutex);
+	cpus_read_unlock();
 
 	return ret ? ERR_PTR(-EINVAL) : dst;
 }
@@ -1429,10 +1431,12 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 	if (ret)
 		return ret;
 
+	cpus_read_lock();
 	mutex_lock(&text_mutex);
 	if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
 		ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
 	mutex_unlock(&text_mutex);
+	cpus_read_unlock();
 
 	return ret;
 }
@@ -1450,10 +1454,12 @@ int bpf_arch_text_invalidate(void *dst, size_t len)
 	for (i = 0; i < (len / sizeof(u32)); i++)
 		inst[i] = INSN_BREAK;
 
+	cpus_read_lock();
 	mutex_lock(&text_mutex);
 	if (larch_insn_text_copy(dst, inst, len))
 		ret = -EINVAL;
 	mutex_unlock(&text_mutex);
+	cpus_read_unlock();
 
 	kvfree(inst);
 
@@ -1568,6 +1574,11 @@ void arch_free_bpf_trampoline(void *image, unsigned int size)
 	bpf_prog_pack_free(image, size);
 }
 
+int arch_protect_bpf_trampoline(void *image, unsigned int size)
+{
+	return 0;
+}
+
 /*
  * Sign-extend the register if necessary
  */
diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile
index 520f1513f07dd..904ef4a0d8266 100644
--- a/arch/loongarch/vdso/Makefile
+++ b/arch/loongarch/vdso/Makefile
@@ -24,7 +24,7 @@ endif
 cflags-vdso := $(ccflags-vdso) \
 	-isystem $(shell $(CC) -print-file-name=include) \
 	$(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \
-	-std=gnu11 -fms-extensions -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \
+	$(CC_FLAGS_DIALECT) -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \
 	-fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \
 	$(call cc-option, -fno-asynchronous-unwind-tables) \
 	$(call cc-option, -fno-stack-protector)
diff --git a/arch/m68k/include/asm/pgtable_mm.h b/arch/m68k/include/asm/pgtable_mm.h
index bba64a9c49acd..7501ff030c633 100644
--- a/arch/m68k/include/asm/pgtable_mm.h
+++ b/arch/m68k/include/asm/pgtable_mm.h
@@ -110,15 +110,6 @@ extern unsigned long m68k_vmalloc_end;
 #define VMALLOC_END KMAP_START
 #endif
 
-/* zero page used for uninitialized stuff */
-extern void *empty_zero_page;
-
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-
 extern void kernel_set_cachemode(void *addr, unsigned long size, int cmode);
 
 /*
diff --git a/arch/m68k/include/asm/pgtable_no.h b/arch/m68k/include/asm/pgtable_no.h
index 1a86c15b9008f..11751807a3f3b 100644
--- a/arch/m68k/include/asm/pgtable_no.h
+++ b/arch/m68k/include/asm/pgtable_no.h
@@ -30,13 +30,6 @@
 
 #define swapper_pg_dir ((pgd_t *) 0)
 
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern void *empty_zero_page;
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-
 /*
  * All 32bit addresses are effectively valid for vmalloc...
  * Sort of meaningless for non-VM targets.
diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c
index 53b71f786c27e..3b88c0dd1616d 100644
--- a/arch/m68k/mm/init.c
+++ b/arch/m68k/mm/init.c
@@ -33,13 +33,6 @@
 #include <asm/sections.h>
 #include <asm/tlb.h>
 
-/*
- * ZERO_PAGE is a special page that is used for zero-initialized
- * data and COW.
- */
-void *empty_zero_page;
-EXPORT_SYMBOL(empty_zero_page);
-
 void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
 {
 	max_zone_pfns[ZONE_DMA] = PFN_DOWN(memblock_end_of_DRAM());
@@ -71,8 +64,6 @@ void __init paging_init(void)
 	unsigned long end_mem = memory_end & PAGE_MASK;
 
 	high_memory = (void *) end_mem;
-
-	empty_zero_page = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
 }
 
 #endif /* CONFIG_MMU */
diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c
index 3418fd8642377..4924f2ff8ef87 100644
--- a/arch/m68k/mm/mcfmmu.c
+++ b/arch/m68k/mm/mcfmmu.c
@@ -41,8 +41,6 @@ void __init paging_init(void)
 	unsigned long next_pgtable;
 	int i;
 
-	empty_zero_page = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
-
 	pg_dir = swapper_pg_dir;
 	memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
 
diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c
index 127a3fa69f4c1..b30aa69a73a6a 100644
--- a/arch/m68k/mm/motorola.c
+++ b/arch/m68k/mm/motorola.c
@@ -498,12 +498,6 @@ void __init paging_init(void)
 
 	early_memtest(min_addr, max_addr);
 
-	/*
-	 * initialize the bad page table and bad page to point
-	 * to a couple of allocated pages
-	 */
-	empty_zero_page = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
-
 	/*
 	 * Set up SFC/DFC registers
 	 */
diff --git a/arch/m68k/mm/sun3mmu.c b/arch/m68k/mm/sun3mmu.c
index c801677f7df87..f139cc15753a7 100644
--- a/arch/m68k/mm/sun3mmu.c
+++ b/arch/m68k/mm/sun3mmu.c
@@ -43,8 +43,6 @@ void __init paging_init(void)
 	unsigned long bootmem_end;
 	unsigned long size;
 
-	empty_zero_page = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
-
 	address = PAGE_OFFSET;
 	pg_dir = swapper_pg_dir;
 	memset (swapper_pg_dir, 0, sizeof (swapper_pg_dir));
diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h
index 4eb76de6be4ac..ea72291de5531 100644
--- a/arch/microblaze/include/asm/pgtable.h
+++ b/arch/microblaze/include/asm/pgtable.h
@@ -207,16 +207,6 @@ extern pte_t *va_to_pte(unsigned long address);
  * Also, write permissions imply read permissions.
  */
 
-#ifndef __ASSEMBLER__
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[1024];
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
-#endif /* __ASSEMBLER__ */
-
 #define pte_none(pte)		((pte_val(pte) & ~_PTE_NONE_MASK) == 0)
 #define pte_present(pte)	(pte_val(pte) & _PAGE_PRESENT)
 #define pte_clear(mm, addr, ptep) \
diff --git a/arch/microblaze/kernel/head.S b/arch/microblaze/kernel/head.S
index ec2fcb545e646..808019c3b7ac2 100644
--- a/arch/microblaze/kernel/head.S
+++ b/arch/microblaze/kernel/head.S
@@ -39,10 +39,6 @@
 #include <asm/processor.h>
 
 .section .data
-.global empty_zero_page
-.align 12
-empty_zero_page:
-	.space	PAGE_SIZE
 .global swapper_pg_dir
 swapper_pg_dir:
 	.space	PAGE_SIZE
diff --git a/arch/microblaze/kernel/microblaze_ksyms.c b/arch/microblaze/kernel/microblaze_ksyms.c
index a8553f54152b7..ad7596d7ba076 100644
--- a/arch/microblaze/kernel/microblaze_ksyms.c
+++ b/arch/microblaze/kernel/microblaze_ksyms.c
@@ -33,8 +33,6 @@ EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
 #endif
 
-EXPORT_SYMBOL(empty_zero_page);
-
 EXPORT_SYMBOL(mbc);
 
 extern void __divsi3(void);
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index e48b62b4dc489..4364f3dba688f 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1131,7 +1131,6 @@ config CSRC_IOASIC
 	bool
 
 config CSRC_R4K
-	select CLOCKSOURCE_WATCHDOG if CPU_FREQ
 	bool
 
 config CSRC_SB1250
diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h
index 5ec428fcc8877..50a382a0d8f64 100644
--- a/arch/mips/include/asm/page.h
+++ b/arch/mips/include/asm/page.h
@@ -213,7 +213,7 @@ extern bool __virt_addr_valid(const volatile void *kaddr);
 #define virt_addr_valid(kaddr)						\
 	__virt_addr_valid((const volatile void *) (kaddr))
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
 
 extern unsigned long __kaslr_offset;
 static inline unsigned long kaslr_offset(void)
diff --git a/arch/mips/include/asm/vdso/vdso.h b/arch/mips/include/asm/vdso/vdso.h
index 6889e0f2e5db9..ef50d33f3439a 100644
--- a/arch/mips/include/asm/vdso/vdso.h
+++ b/arch/mips/include/asm/vdso/vdso.h
@@ -4,6 +4,9 @@
  * Author: Alex Smith <alex.smith@imgtec.com>
  */
 
+#ifndef __ASM_VDSO_VDSO_H
+#define __ASM_VDSO_VDSO_H
+
 #include <asm/sgidefs.h>
 #include <vdso/page.h>
 
@@ -70,3 +73,5 @@ static inline void __iomem *get_gic(const struct vdso_time_data *data)
 #endif /* CONFIG_CLKSRC_MIPS_GIC */
 
 #endif /* __ASSEMBLER__ */
+
+#endif /* __ASM_VDSO_VDSO_H */
diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c
index de096777172f0..2fa4df3e46e47 100644
--- a/arch/mips/kernel/vdso.c
+++ b/arch/mips/kernel/vdso.c
@@ -21,6 +21,7 @@
 #include <asm/mips-cps.h>
 #include <asm/page.h>
 #include <asm/vdso.h>
+#include <asm/vdso/vdso.h>
 #include <vdso/helpers.h>
 #include <vdso/vsyscall.h>
 
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 4f6449ad02cae..55b25e85122a3 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -56,10 +56,7 @@ unsigned long empty_zero_page, zero_page_mask;
 EXPORT_SYMBOL_GPL(empty_zero_page);
 EXPORT_SYMBOL(zero_page_mask);
 
-/*
- * Not static inline because used by IP27 special magic initialization code
- */
-static void __init setup_zero_pages(void)
+void __init arch_setup_zero_pages(void)
 {
 	unsigned int order;
 
@@ -450,7 +447,6 @@ void __init arch_mm_preinit(void)
 	BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT));
 
 	maar_init();
-	setup_zero_pages();	/* Setup zeroed pages.  */
 	highmem_init();
 
 #ifdef CONFIG_64BIT
@@ -461,11 +457,6 @@ void __init arch_mm_preinit(void)
 				0x80000000 - 4, KCORE_TEXT);
 #endif
 }
-#else  /* CONFIG_NUMA */
-void __init arch_mm_preinit(void)
-{
-	setup_zero_pages();	/* This comes from node 0 */
-}
 #endif /* !CONFIG_NUMA */
 
 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
diff --git a/arch/mips/pic32/Kconfig b/arch/mips/pic32/Kconfig
index bb6ab1f3e80dc..cd14a071e631c 100644
--- a/arch/mips/pic32/Kconfig
+++ b/arch/mips/pic32/Kconfig
@@ -20,7 +20,6 @@ config PIC32MZDA
 	select LIBFDT
 	select USE_OF
 	select PINCTRL
-	select PIC32_EVIC
 	help
 	  Support for the Microchip PIC32MZDA microcontroller.
 
diff --git a/arch/nios2/include/asm/page.h b/arch/nios2/include/asm/page.h
index 722956ac0bf87..71eb7c1b67d49 100644
--- a/arch/nios2/include/asm/page.h
+++ b/arch/nios2/include/asm/page.h
@@ -85,7 +85,7 @@ extern struct page *mem_map;
 # define virt_to_page(vaddr)	pfn_to_page(PFN_DOWN(virt_to_phys(vaddr)))
 # define virt_addr_valid(vaddr)	pfn_valid(PFN_DOWN(virt_to_phys(vaddr)))
 
-# define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_NON_EXEC
+# define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_NON_EXEC
 
 #include <asm-generic/memory_model.h>
 
diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h
index 844dce55569f3..d389aa9ca57ce 100644
--- a/arch/nios2/include/asm/pgtable.h
+++ b/arch/nios2/include/asm/pgtable.h
@@ -65,13 +65,6 @@ struct mm_struct;
 #define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
 #define PGDIR_MASK	(~(PGDIR_SIZE-1))
 
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr)	(virt_to_page(empty_zero_page))
-
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 extern pte_t invalid_pte_table[PAGE_SIZE/sizeof(pte_t)];
 
diff --git a/arch/nios2/kernel/head.S b/arch/nios2/kernel/head.S
index 372ce4a33018b..613212e1a63ac 100644
--- a/arch/nios2/kernel/head.S
+++ b/arch/nios2/kernel/head.S
@@ -23,16 +23,6 @@
 #include <asm/asm-offsets.h>
 #include <asm/asm-macros.h>
 
-/*
- * ZERO_PAGE is a special page that is used for zero-initialized
- * data and COW.
- */
-.data
-.global empty_zero_page
-.align 12
-empty_zero_page:
-	.space	PAGE_SIZE
-
 /*
  * This global variable is used as an extension to the nios'
  * STATUS register to emulate a user/supervisor mode.
diff --git a/arch/nios2/kernel/nios2_ksyms.c b/arch/nios2/kernel/nios2_ksyms.c
index 54f7b23df1bf5..c40aa39e86587 100644
--- a/arch/nios2/kernel/nios2_ksyms.c
+++ b/arch/nios2/kernel/nios2_ksyms.c
@@ -20,7 +20,6 @@ EXPORT_SYMBOL(memmove);
 
 /* memory management */
 
-EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(flush_icache_range);
 
 /*
diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h
index b218050e2f6dd..6b89996d0b628 100644
--- a/arch/openrisc/include/asm/pgtable.h
+++ b/arch/openrisc/include/asm/pgtable.h
@@ -179,10 +179,6 @@ extern void paging_init(void);
 	__pgprot(_PAGE_ALL | _PAGE_SRE | _PAGE_SWE \
 		 | _PAGE_SHARED | _PAGE_DIRTY | _PAGE_EXEC | _PAGE_CI)
 
-/* zero page used for uninitialized stuff */
-extern unsigned long empty_zero_page[2048];
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 #define pte_none(x)	(!pte_val(x))
 #define pte_present(x)	(pte_val(x) & _PAGE_PRESENT)
 #define pte_clear(mm, addr, xp)	do { pte_val(*(xp)) = 0; } while (0)
diff --git a/arch/openrisc/kernel/head.S b/arch/openrisc/kernel/head.S
index bd760066f1cdc..45890393947d8 100644
--- a/arch/openrisc/kernel/head.S
+++ b/arch/openrisc/kernel/head.S
@@ -1563,9 +1563,6 @@ _string_nl:
  */
 	.section .data,"aw"
 	.align	8192
-	.global  empty_zero_page
-empty_zero_page:
-	.space  8192
 
 	.global  swapper_pg_dir
 swapper_pg_dir:
diff --git a/arch/openrisc/kernel/or32_ksyms.c b/arch/openrisc/kernel/or32_ksyms.c
index 212e5f85004cc..84a937a64e2a7 100644
--- a/arch/openrisc/kernel/or32_ksyms.c
+++ b/arch/openrisc/kernel/or32_ksyms.c
@@ -40,7 +40,6 @@ DECLARE_EXPORT(__ashldi3);
 DECLARE_EXPORT(__lshrdi3);
 DECLARE_EXPORT(__ucmpdi2);
 
-EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(__copy_tofrom_user);
 EXPORT_SYMBOL(__clear_user);
 EXPORT_SYMBOL(memset);
diff --git a/arch/openrisc/mm/init.c b/arch/openrisc/mm/init.c
index 78fb0734cdbc2..89d8c6df88551 100644
--- a/arch/openrisc/mm/init.c
+++ b/arch/openrisc/mm/init.c
@@ -188,9 +188,6 @@ void __init mem_init(void)
 {
 	BUG_ON(!mem_map);
 
-	/* clear the zero-page */
-	memset((void *)empty_zero_page, 0, PAGE_SIZE);
-
 	printk("mem_init_done ...........................................\n");
 	mem_init_done = 1;
 	return;
diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile
index f8481e4e9d214..14eefb5ed5d1d 100644
--- a/arch/parisc/boot/compressed/Makefile
+++ b/arch/parisc/boot/compressed/Makefile
@@ -18,7 +18,7 @@ KBUILD_CFLAGS += -fno-PIE -mno-space-regs -mdisable-fpregs -Os
 ifndef CONFIG_64BIT
 KBUILD_CFLAGS += -mfast-indirect-calls
 endif
-KBUILD_CFLAGS += -std=gnu11 -fms-extensions
+KBUILD_CFLAGS += $(CC_FLAGS_DIALECT)
 
 LDFLAGS_vmlinux := -X -e startup --as-needed -T
 $(obj)/vmlinux: $(obj)/vmlinux.lds $(addprefix $(obj)/, $(OBJECTS)) $(LIBGCC) FORCE
diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h
index 17afe7a59edfd..f6fb99cb94d9e 100644
--- a/arch/parisc/include/asm/pgtable.h
+++ b/arch/parisc/include/asm/pgtable.h
@@ -262,17 +262,6 @@ extern pgd_t swapper_pg_dir[]; /* declared in init_task.c */
 
 extern pte_t pg0[];
 
-/* zero page used for uninitialized stuff */
-
-extern unsigned long *empty_zero_page;
-
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 #define pte_none(x)     (pte_val(x) == 0)
 #define pte_present(x)	(pte_val(x) & _PAGE_PRESENT)
 #define pte_user(x)	(pte_val(x) & _PAGE_USER)
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 4c5240d3a3c7c..b189265785dc7 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -953,7 +953,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
 #else
 			"1: cmpb,<<,n	%0,%2,1b\n"
 #endif
-			"   fic,m	%3(%4,%0)\n"
+			"   fdc,m	%3(%4,%0)\n"
 			"2: sync\n"
 			ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1")
 			: "+r" (start), "+r" (error)
@@ -968,7 +968,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
 #else
 			"1: cmpb,<<,n	%0,%2,1b\n"
 #endif
-			"   fdc,m	%3(%4,%0)\n"
+			"   fic,m	%3(%4,%0)\n"
 			"2: sync\n"
 			ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1")
 			: "+r" (start), "+r" (error)
diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c
index 94dc48455dc67..71c9d54269957 100644
--- a/arch/parisc/kernel/time.c
+++ b/arch/parisc/kernel/time.c
@@ -210,12 +210,9 @@ static struct clocksource clocksource_cr16 = {
 	.read			= read_cr16,
 	.mask			= CLOCKSOURCE_MASK(BITS_PER_LONG),
 	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
-					CLOCK_SOURCE_VALID_FOR_HRES |
-					CLOCK_SOURCE_MUST_VERIFY |
-					CLOCK_SOURCE_VERIFY_PERCPU,
+				  CLOCK_SOURCE_VALID_FOR_HRES,
 };
 
-
 /*
  * timer interrupt and sched_clock() initialization
  */
diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
index 6a39e031e5ff7..be3380c9bcdae 100644
--- a/arch/parisc/mm/init.c
+++ b/arch/parisc/mm/init.c
@@ -604,9 +604,6 @@ void __init mem_init(void)
 #endif
 }
 
-unsigned long *empty_zero_page __ro_after_init;
-EXPORT_SYMBOL(empty_zero_page);
-
 /*
  * pagetable_init() sets up the page tables
  *
@@ -639,9 +636,6 @@ static void __init pagetable_init(void)
 			  initrd_end - initrd_start, PAGE_KERNEL, 0);
 	}
 #endif
-
-	empty_zero_page = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
-
 }
 
 static void __init gateway_init(void)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 10240cb809043..5355c3dca1ffb 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -328,6 +328,7 @@ config PPC
 	select THREAD_INFO_IN_TASK
 	select TRACE_IRQFLAGS_SUPPORT
 	select VDSO_GETRANDOM
+	select XOR_BLOCKS_ARCH
 	#
 	# Please keep this list sorted alphabetically.
 	#
@@ -1061,6 +1062,7 @@ config PPC_SECURE_BOOT
 	depends on IMA_ARCH_POLICY
 	imply IMA_SECURE_AND_OR_TRUSTED_BOOT
 	select PSERIES_PLPKS if PPC_PSERIES
+	select HAVE_ARCH_GET_SECUREBOOT
 	help
 	  Systems with firmware secure boot enabled need to define security
 	  policies to extend secure boot to the OS. This config allows a user
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index f1a4761ebd44b..7ee0fa2d3371e 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -70,7 +70,7 @@ BOOTCPPFLAGS	:= -nostdinc $(LINUXINCLUDE)
 BOOTCPPFLAGS	+= -isystem $(shell $(BOOTCC) -print-file-name=include)
 
 BOOTCFLAGS	:= $(BOOTTARGETFLAGS) \
-		   -std=gnu11 -fms-extensions \
+		   $(CC_FLAGS_DIALECT) \
 		   -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
 		   -fno-strict-aliasing -O2 \
 		   -msoft-float -mno-altivec -mno-vsx \
@@ -86,7 +86,6 @@ BOOTARFLAGS	:= -crD
 
 ifdef CONFIG_CC_IS_CLANG
 BOOTCFLAGS += $(CLANG_FLAGS)
-BOOTCFLAGS += -Wno-microsoft-anon-tag
 BOOTAFLAGS += $(CLANG_FLAGS)
 endif
 
diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
index 001e28f9eabca..75195bb44d062 100644
--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
@@ -438,7 +438,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
 	return true;
 }
 
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return pte_present(pte) && !is_kernel_addr(addr);
 }
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 1a91762b455d9..a56df313b585b 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -549,7 +549,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
 	return arch_pte_access_permitted(pte_val(pte), write, 0);
 }
 
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return pte_present(pte) && pte_user(pte);
 }
@@ -925,9 +925,9 @@ static inline bool pud_access_permitted(pud_t pud, bool write)
 }
 
 #define pud_user_accessible_page pud_user_accessible_page
-static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
+static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
 {
-	return pud_leaf(pud) && pte_user_accessible_page(pud_pte(pud), addr);
+	return pud_leaf(pud) && pte_user_accessible_page(mm, addr, pud_pte(pud));
 }
 
 #define __p4d_raw(x)	((p4d_t) { __pgd_raw(x) })
@@ -1096,9 +1096,9 @@ static inline bool pmd_access_permitted(pmd_t pmd, bool write)
 }
 
 #define pmd_user_accessible_page pmd_user_accessible_page
-static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
+static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
 {
-	return pmd_leaf(pmd) && pte_user_accessible_page(pmd_pte(pmd), addr);
+	return pmd_leaf(pmd) && pte_user_accessible_page(mm, addr, pmd_pte(pmd));
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
index e6da5eaccff63..0665d0abe89f3 100644
--- a/arch/powerpc/include/asm/nohash/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/pgtable.h
@@ -249,7 +249,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
 	return true;
 }
 
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return pte_present(pte) && !is_kernel_addr(addr);
 }
diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h
index f2bb1f98eebed..281f25e071a34 100644
--- a/arch/powerpc/include/asm/page.h
+++ b/arch/powerpc/include/asm/page.h
@@ -240,8 +240,8 @@ static inline const void *pfn_to_kaddr(unsigned long pfn)
  * and needs to be executable.  This means the whole heap ends
  * up being executable.
  */
-#define VM_DATA_DEFAULT_FLAGS32	VM_DATA_FLAGS_TSK_EXEC
-#define VM_DATA_DEFAULT_FLAGS64	VM_DATA_FLAGS_NON_EXEC
+#define VMA_DATA_DEFAULT_FLAGS32	VMA_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS64	VMA_DATA_FLAGS_NON_EXEC
 
 #ifdef __powerpc64__
 #include <asm/page_64.h>
diff --git a/arch/powerpc/include/asm/page_32.h b/arch/powerpc/include/asm/page_32.h
index 25482405a8111..1fd8c21f0a42e 100644
--- a/arch/powerpc/include/asm/page_32.h
+++ b/arch/powerpc/include/asm/page_32.h
@@ -10,7 +10,7 @@
 #endif
 #endif
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_DEFAULT_FLAGS32
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_DEFAULT_FLAGS32
 
 #if defined(CONFIG_PPC_256K_PAGES) || \
     (defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES))
diff --git a/arch/powerpc/include/asm/page_64.h b/arch/powerpc/include/asm/page_64.h
index 0f564a06bf684..d96c984d023b2 100644
--- a/arch/powerpc/include/asm/page_64.h
+++ b/arch/powerpc/include/asm/page_64.h
@@ -84,9 +84,9 @@ extern u64 ppc64_pft_size;
 
 #endif /* __ASSEMBLER__ */
 
-#define VM_DATA_DEFAULT_FLAGS \
+#define VMA_DATA_DEFAULT_FLAGS \
 	(is_32bit_task() ? \
-	 VM_DATA_DEFAULT_FLAGS32 : VM_DATA_DEFAULT_FLAGS64)
+	 VMA_DATA_DEFAULT_FLAGS32 : VMA_DATA_DEFAULT_FLAGS64)
 
 /*
  * This is the default if a program doesn't have a PT_GNU_STACK
@@ -94,12 +94,12 @@ extern u64 ppc64_pft_size;
  * stack by default, so in the absence of a PT_GNU_STACK program header
  * we turn execute permission off.
  */
-#define VM_STACK_DEFAULT_FLAGS32	VM_DATA_FLAGS_EXEC
-#define VM_STACK_DEFAULT_FLAGS64	VM_DATA_FLAGS_NON_EXEC
+#define VMA_STACK_DEFAULT_FLAGS32	VMA_DATA_FLAGS_EXEC
+#define VMA_STACK_DEFAULT_FLAGS64	VMA_DATA_FLAGS_NON_EXEC
 
-#define VM_STACK_DEFAULT_FLAGS \
+#define VMA_STACK_DEFAULT_FLAGS \
 	(is_32bit_task() ? \
-	 VM_STACK_DEFAULT_FLAGS32 : VM_STACK_DEFAULT_FLAGS64)
+	 VMA_STACK_DEFAULT_FLAGS32 : VMA_STACK_DEFAULT_FLAGS64)
 
 #include <asm-generic/getorder.h>
 
diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index dcd3a88caaf63..742cf16fb6006 100644
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -76,12 +76,6 @@ static inline const void *pmd_page_vaddr(pmd_t pmd)
 }
 #define pmd_page_vaddr pmd_page_vaddr
 #endif
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[];
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
 
 extern pgd_t swapper_pg_dir[];
 
@@ -205,11 +199,11 @@ static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size)
 #endif /* CONFIG_PPC64 */
 
 #ifndef pmd_user_accessible_page
-#define pmd_user_accessible_page(pmd, addr)	false
+#define pmd_user_accessible_page(mm, addr, pmd)	false
 #endif
 
 #ifndef pud_user_accessible_page
-#define pud_user_accessible_page(pud, addr)	false
+#define pud_user_accessible_page(mm, addr, pud)	false
 #endif
 
 #endif /* __ASSEMBLER__ */
diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h
index 28e752138996e..bd20d4106471c 100644
--- a/arch/powerpc/include/asm/pkeys.h
+++ b/arch/powerpc/include/asm/pkeys.h
@@ -143,10 +143,8 @@ static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
 	return __arch_override_mprotect_pkey(vma, prot, pkey);
 }
 
-extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-				       unsigned long init_val);
-static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-					    unsigned long init_val)
+extern int __arch_set_user_pkey_access(int pkey, unsigned long init_val);
+static inline int arch_set_user_pkey_access(int pkey, unsigned long init_val)
 {
 	if (!mmu_has_feature(MMU_FTR_PKEY))
 		return -EINVAL;
@@ -160,7 +158,7 @@ static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
 	if (pkey == 0)
 		return init_val ? -EINVAL : 0;
 
-	return __arch_set_user_pkey_access(tsk, pkey, init_val);
+	return __arch_set_user_pkey_access(pkey, init_val);
 }
 
 static inline bool arch_pkeys_enabled(void)
diff --git a/arch/powerpc/include/asm/vdso/gettimeofday.h b/arch/powerpc/include/asm/vdso/gettimeofday.h
index 8ea397e26ad07..a853f853da6cf 100644
--- a/arch/powerpc/include/asm/vdso/gettimeofday.h
+++ b/arch/powerpc/include/asm/vdso/gettimeofday.h
@@ -8,6 +8,7 @@
 #include <asm/barrier.h>
 #include <asm/unistd.h>
 #include <uapi/linux/time.h>
+#include <vdso/time32.h>
 
 #define VDSO_HAS_CLOCK_GETRES		1
 
diff --git a/arch/powerpc/include/asm/vdso/processor.h b/arch/powerpc/include/asm/vdso/processor.h
index c1f3d7aaf3ee9..4c6802c3a5808 100644
--- a/arch/powerpc/include/asm/vdso/processor.h
+++ b/arch/powerpc/include/asm/vdso/processor.h
@@ -4,6 +4,9 @@
 
 #ifndef __ASSEMBLER__
 
+#include <asm/cputable.h>
+#include <asm/feature-fixups.h>
+
 /* Macros for adjusting thread priority (hardware multi-threading) */
 #ifdef CONFIG_PPC64
 #define HMT_very_low()		asm volatile("or 31, 31, 31	# very low priority")
diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h
deleted file mode 100644
index 37d05c11d09cd..0000000000000
--- a/arch/powerpc/include/asm/xor.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *
- * Copyright (C) IBM Corporation, 2012
- *
- * Author: Anton Blanchard <anton@au.ibm.com>
- */
-#ifndef _ASM_POWERPC_XOR_H
-#define _ASM_POWERPC_XOR_H
-
-#ifdef CONFIG_ALTIVEC
-
-#include <asm/cputable.h>
-#include <asm/cpu_has_feature.h>
-#include <asm/xor_altivec.h>
-
-static struct xor_block_template xor_block_altivec = {
-	.name = "altivec",
-	.do_2 = xor_altivec_2,
-	.do_3 = xor_altivec_3,
-	.do_4 = xor_altivec_4,
-	.do_5 = xor_altivec_5,
-};
-
-#define XOR_SPEED_ALTIVEC()				\
-	do {						\
-		if (cpu_has_feature(CPU_FTR_ALTIVEC))	\
-			xor_speed(&xor_block_altivec);	\
-	} while (0)
-#else
-#define XOR_SPEED_ALTIVEC()
-#endif
-
-/* Also try the generic routines. */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-do {							\
-	xor_speed(&xor_block_8regs);			\
-	xor_speed(&xor_block_8regs_p);			\
-	xor_speed(&xor_block_32regs);			\
-	xor_speed(&xor_block_32regs_p);			\
-	XOR_SPEED_ALTIVEC();				\
-} while (0)
-
-#endif /* _ASM_POWERPC_XOR_H */
diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h
deleted file mode 100644
index 294620a25f802..0000000000000
--- a/arch/powerpc/include/asm/xor_altivec.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_POWERPC_XOR_ALTIVEC_H
-#define _ASM_POWERPC_XOR_ALTIVEC_H
-
-#ifdef CONFIG_ALTIVEC
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2);
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3);
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4);
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4,
-		   const unsigned long * __restrict p5);
-
-#endif
-#endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
diff --git a/arch/powerpc/kernel/compat_audit.c b/arch/powerpc/kernel/compat_audit.c
index 57b38c592b9f3..b4d81a57b2d97 100644
--- a/arch/powerpc/kernel/compat_audit.c
+++ b/arch/powerpc/kernel/compat_audit.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
-#undef __powerpc64__
 #include <linux/audit_arch.h>
-#include <asm/unistd.h>
+#include <asm/unistd_32.h>
 
 #include "audit_32.h"
 
diff --git a/arch/powerpc/kernel/ima_arch.c b/arch/powerpc/kernel/ima_arch.c
index b7029beed847d..17f9304855e92 100644
--- a/arch/powerpc/kernel/ima_arch.c
+++ b/arch/powerpc/kernel/ima_arch.c
@@ -7,11 +7,6 @@
 #include <linux/ima.h>
 #include <asm/secure_boot.h>
 
-bool arch_ima_get_secureboot(void)
-{
-	return is_ppc_secureboot_enabled();
-}
-
 /*
  * The "secure_rules" are enabled only on "secureboot" enabled systems.
  * These rules verify the file signatures against known good values.
@@ -63,8 +58,7 @@ static const char *const secure_and_trusted_rules[] = {
 const char *const *arch_get_ima_policy(void)
 {
 	if (is_ppc_secureboot_enabled()) {
-		if (IS_ENABLED(CONFIG_MODULE_SIG))
-			set_module_sig_enforced();
+		set_module_sig_enforced();
 
 		if (is_ppc_trustedboot_enabled())
 			return secure_and_trusted_rules;
diff --git a/arch/powerpc/kernel/secure_boot.c b/arch/powerpc/kernel/secure_boot.c
index 3a28795b4ed82..28436c1599e08 100644
--- a/arch/powerpc/kernel/secure_boot.c
+++ b/arch/powerpc/kernel/secure_boot.c
@@ -5,6 +5,7 @@
  */
 #include <linux/types.h>
 #include <linux/of.h>
+#include <linux/secure_boot.h>
 #include <linux/string_choices.h>
 #include <asm/secure_boot.h>
 
@@ -44,6 +45,11 @@ bool is_ppc_secureboot_enabled(void)
 	return enabled;
 }
 
+bool arch_get_secureboot(void)
+{
+	return is_ppc_secureboot_enabled();
+}
+
 bool is_ppc_trustedboot_enabled(void)
 {
 	struct device_node *node;
diff --git a/arch/powerpc/kexec/elf_64.c b/arch/powerpc/kexec/elf_64.c
index 5d6d616404cf9..ea50a072debf6 100644
--- a/arch/powerpc/kexec/elf_64.c
+++ b/arch/powerpc/kexec/elf_64.c
@@ -79,6 +79,10 @@ static void *elf64_load(struct kimage *image, char *kernel_buf,
 			goto out;
 		}
 
+		ret = crash_load_dm_crypt_keys(image);
+		if (ret)
+			goto out;
+
 		/* Setup cmdline for kdump kernel case */
 		modified_cmdline = setup_kdump_cmdline(image, cmdline,
 						       cmdline_len);
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 08e5816fdd616..61dbeea317f3b 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -36,7 +36,6 @@
 #include <linux/gfp.h>
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
-#include <linux/hugetlb.h>
 #include <linux/kvm_irqfd.h>
 #include <linux/irqbypass.h>
 #include <linux/module.h>
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index f14ecab674a34..002edc3f01d5c 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -73,9 +73,4 @@ obj-$(CONFIG_PPC_LIB_RHEAP) += rheap.o
 
 obj-$(CONFIG_FTR_FIXUP_SELFTEST) += feature-fixups-test.o
 
-obj-$(CONFIG_ALTIVEC)	+= xor_vmx.o xor_vmx_glue.o
-CFLAGS_xor_vmx.o += -mhard-float -maltivec $(call cc-option,-mabi=altivec)
-# Enable <altivec.h>
-CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include)
-
 obj-$(CONFIG_PPC64) += $(obj64-y)
diff --git a/arch/powerpc/lib/xor_vmx.c b/arch/powerpc/lib/xor_vmx.c
deleted file mode 100644
index aab49d056d188..0000000000000
--- a/arch/powerpc/lib/xor_vmx.c
+++ /dev/null
@@ -1,156 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *
- * Copyright (C) IBM Corporation, 2012
- *
- * Author: Anton Blanchard <anton@au.ibm.com>
- */
-
-/*
- * Sparse (as at v0.5.0) gets very, very confused by this file.
- * Make it a bit simpler for it.
- */
-#if !defined(__CHECKER__)
-#include <altivec.h>
-#else
-#define vec_xor(a, b) a ^ b
-#define vector __attribute__((vector_size(16)))
-#endif
-
-#include "xor_vmx.h"
-
-typedef vector signed char unative_t;
-
-#define DEFINE(V)				\
-	unative_t *V = (unative_t *)V##_in;	\
-	unative_t V##_0, V##_1, V##_2, V##_3
-
-#define LOAD(V)			\
-	do {			\
-		V##_0 = V[0];	\
-		V##_1 = V[1];	\
-		V##_2 = V[2];	\
-		V##_3 = V[3];	\
-	} while (0)
-
-#define STORE(V)		\
-	do {			\
-		V[0] = V##_0;	\
-		V[1] = V##_1;	\
-		V[2] = V##_2;	\
-		V[3] = V##_3;	\
-	} while (0)
-
-#define XOR(V1, V2)					\
-	do {						\
-		V1##_0 = vec_xor(V1##_0, V2##_0);	\
-		V1##_1 = vec_xor(V1##_1, V2##_1);	\
-		V1##_2 = vec_xor(V1##_2, V2##_2);	\
-		V1##_3 = vec_xor(V1##_3, V2##_3);	\
-	} while (0)
-
-void __xor_altivec_2(unsigned long bytes,
-		     unsigned long * __restrict v1_in,
-		     const unsigned long * __restrict v2_in)
-{
-	DEFINE(v1);
-	DEFINE(v2);
-	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
-	do {
-		LOAD(v1);
-		LOAD(v2);
-		XOR(v1, v2);
-		STORE(v1);
-
-		v1 += 4;
-		v2 += 4;
-	} while (--lines > 0);
-}
-
-void __xor_altivec_3(unsigned long bytes,
-		     unsigned long * __restrict v1_in,
-		     const unsigned long * __restrict v2_in,
-		     const unsigned long * __restrict v3_in)
-{
-	DEFINE(v1);
-	DEFINE(v2);
-	DEFINE(v3);
-	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
-	do {
-		LOAD(v1);
-		LOAD(v2);
-		LOAD(v3);
-		XOR(v1, v2);
-		XOR(v1, v3);
-		STORE(v1);
-
-		v1 += 4;
-		v2 += 4;
-		v3 += 4;
-	} while (--lines > 0);
-}
-
-void __xor_altivec_4(unsigned long bytes,
-		     unsigned long * __restrict v1_in,
-		     const unsigned long * __restrict v2_in,
-		     const unsigned long * __restrict v3_in,
-		     const unsigned long * __restrict v4_in)
-{
-	DEFINE(v1);
-	DEFINE(v2);
-	DEFINE(v3);
-	DEFINE(v4);
-	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
-	do {
-		LOAD(v1);
-		LOAD(v2);
-		LOAD(v3);
-		LOAD(v4);
-		XOR(v1, v2);
-		XOR(v3, v4);
-		XOR(v1, v3);
-		STORE(v1);
-
-		v1 += 4;
-		v2 += 4;
-		v3 += 4;
-		v4 += 4;
-	} while (--lines > 0);
-}
-
-void __xor_altivec_5(unsigned long bytes,
-		     unsigned long * __restrict v1_in,
-		     const unsigned long * __restrict v2_in,
-		     const unsigned long * __restrict v3_in,
-		     const unsigned long * __restrict v4_in,
-		     const unsigned long * __restrict v5_in)
-{
-	DEFINE(v1);
-	DEFINE(v2);
-	DEFINE(v3);
-	DEFINE(v4);
-	DEFINE(v5);
-	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
-	do {
-		LOAD(v1);
-		LOAD(v2);
-		LOAD(v3);
-		LOAD(v4);
-		LOAD(v5);
-		XOR(v1, v2);
-		XOR(v3, v4);
-		XOR(v1, v5);
-		XOR(v1, v3);
-		STORE(v1);
-
-		v1 += 4;
-		v2 += 4;
-		v3 += 4;
-		v4 += 4;
-		v5 += 4;
-	} while (--lines > 0);
-}
diff --git a/arch/powerpc/lib/xor_vmx.h b/arch/powerpc/lib/xor_vmx.h
deleted file mode 100644
index 573c41d90dac5..0000000000000
--- a/arch/powerpc/lib/xor_vmx.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Simple interface to link xor_vmx.c and xor_vmx_glue.c
- *
- * Separating these file ensures that no altivec instructions are run
- * outside of the enable/disable altivec block.
- */
-
-void __xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2);
-void __xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3);
-void __xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4);
-void __xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4,
-		     const unsigned long * __restrict p5);
diff --git a/arch/powerpc/lib/xor_vmx_glue.c b/arch/powerpc/lib/xor_vmx_glue.c
deleted file mode 100644
index 35d917ece4d1e..0000000000000
--- a/arch/powerpc/lib/xor_vmx_glue.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Altivec XOR operations
- *
- * Copyright 2017 IBM Corp.
- */
-
-#include <linux/preempt.h>
-#include <linux/export.h>
-#include <linux/sched.h>
-#include <asm/switch_to.h>
-#include <asm/xor_altivec.h>
-#include "xor_vmx.h"
-
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-	__xor_altivec_2(bytes, p1, p2);
-	disable_kernel_altivec();
-	preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_2);
-
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-	__xor_altivec_3(bytes, p1, p2, p3);
-	disable_kernel_altivec();
-	preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_3);
-
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-	__xor_altivec_4(bytes, p1, p2, p3, p4);
-	disable_kernel_altivec();
-	preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_4);
-
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4,
-		   const unsigned long * __restrict p5)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-	__xor_altivec_5(bytes, p1, p2, p3, p4, p5);
-	disable_kernel_altivec();
-	preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_5);
diff --git a/arch/powerpc/mm/book3s64/pkeys.c b/arch/powerpc/mm/book3s64/pkeys.c
index a974baf8f3278..2002331b05bab 100644
--- a/arch/powerpc/mm/book3s64/pkeys.c
+++ b/arch/powerpc/mm/book3s64/pkeys.c
@@ -335,8 +335,7 @@ static inline void init_iamr(int pkey, u8 init_bits)
  * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
  * specified in @init_val.
  */
-int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-				unsigned long init_val)
+int __arch_set_user_pkey_access(int pkey, unsigned long init_val)
 {
 	u64 new_amr_bits = 0x0ul;
 	u64 new_iamr_bits = 0x0ul;
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index b7982d0243d48..648d0c5602ec8 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -38,9 +38,6 @@
 
 unsigned long long memory_limit __initdata;
 
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 pgprot_t __phys_mem_access_prot(unsigned long pfn, unsigned long size,
 				pgprot_t vma_prot)
 {
diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig
index 604c1b4b6d45c..6805c19ac8a91 100644
--- a/arch/powerpc/platforms/85xx/Kconfig
+++ b/arch/powerpc/platforms/85xx/Kconfig
@@ -217,7 +217,6 @@ config GE_IMP3A
 config SGY_CTS1000
 	tristate "Servergy CTS-1000 support"
 	select GPIOLIB
-	select OF_GPIO
 	depends on CORENET_GENERIC
 	help
 	  Enable this to support functionality in Servergy's CTS-1000 systems.
diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c
index ea4ffa63f043e..e96d79db69fe4 100644
--- a/arch/powerpc/platforms/book3s/vas-api.c
+++ b/arch/powerpc/platforms/book3s/vas-api.c
@@ -414,7 +414,7 @@ static vm_fault_t vas_mmap_fault(struct vm_fault *vmf)
 	/*
 	 * When the LPAR lost credits due to core removal or during
 	 * migration, invalidate the existing mapping for the current
-	 * paste addresses and set windows in-active (zap_vma_pages in
+	 * paste addresses and set windows in-active (zap_vma() in
 	 * reconfig_close_windows()).
 	 * New mapping will be done later after migration or new credits
 	 * available. So continue to receive faults if the user space
diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c
index ceb0a8788c0a7..fa05f04364fe8 100644
--- a/arch/powerpc/platforms/pseries/vas.c
+++ b/arch/powerpc/platforms/pseries/vas.c
@@ -807,7 +807,7 @@ static int reconfig_close_windows(struct vas_caps *vcap, int excess_creds,
 		 * is done before the original mmap() and after the ioctl.
 		 */
 		if (vma)
-			zap_vma_pages(vma);
+			zap_vma(vma);
 
 		mutex_unlock(&task_ref->mmap_mutex);
 		mmap_write_unlock(task_ref->mm);
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 90c531e6abf5c..03ac092adb41d 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -227,6 +227,7 @@ config RISCV
 	select UACCESS_MEMCPY if !MMU
 	select VDSO_GETRANDOM if HAVE_GENERIC_VDSO && 64BIT
 	select USER_STACKTRACE_SUPPORT
+	select XOR_BLOCKS_ARCH
 	select ZONE_DMA32 if 64BIT
 
 config RUSTC_SUPPORTS_RISCV
diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile
index 987d1f0c41f0e..7d54ea5c6f3d2 100644
--- a/arch/riscv/boot/dts/canaan/Makefile
+++ b/arch/riscv/boot/dts/canaan/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_CANAAN) += canaan_kd233.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += k210_generic.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-canmv.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-evb.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_bit.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_dock.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_go.dtb
diff --git a/arch/riscv/boot/dts/canaan/k230-canmv.dts b/arch/riscv/boot/dts/canaan/k230-canmv.dts
new file mode 100644
index 0000000000000..32ba53d17c000
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-canmv.dts
@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Canaan CanMV-K230";
+	compatible = "canaan,canmv-k230", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&pinctrl {
+	jtag_pins: jtag-pins {
+		jtag-tck-cfg {
+			pinmux = <K230_PINMUX(2, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-down;
+			input-schmitt-enable;
+		};
+
+		jtag-tdi-cfg {
+			pinmux = <K230_PINMUX(3, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		jtag-tdo-cfg {
+			pinmux = <K230_PINMUX(4, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		jtag-tms-cfg {
+			pinmux = <K230_PINMUX(5, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-up;
+		};
+	};
+
+	uart2_pins: uart2-pins {
+		uart2-pins-cfg {
+			pinmux = <K230_PINMUX(5, 3)>, /* uart2 txd */
+				 <K230_PINMUX(6, 3)>; /* uart2 rxd */
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	pwm2_pins: pwm2-pins {
+		pwm2-pin-cfg {
+			pinmux = <K230_PINMUX(7, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm3_pins: pwm3-pins {
+		pwm3-pin-cfg {
+			pinmux = <K230_PINMUX(8, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm4_pins: pwm4-pins {
+		pwm4-pin-cfg {
+			pinmux = <K230_PINMUX(9, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iis_pins: iis-pins {
+		iis-clk-cfg {
+			pinmux = <K230_PINMUX(32, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-ws-cfg {
+			pinmux = <K230_PINMUX(33, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-din0-cfg {
+			pinmux = <K230_PINMUX(34, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		iis-dout0-cfg {
+			pinmux = <K230_PINMUX(35, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	uart4_pins: uart4-pins {
+		uart4-txd-cfg {
+			pinmux = <K230_PINMUX(36, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart4-rxd-cfg {
+			pinmux = <K230_PINMUX(37, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	uart0_pins: uart0-pins {
+		uart0-txd-cfg {
+			pinmux = <K230_PINMUX(38, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart0-rxd-cfg {
+			pinmux = <K230_PINMUX(39, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iic1_pins: iic1-pins {
+		iic1-pins-cfg {
+			pinmux = <K230_PINMUX(40, 2)>, /* iic1 scl */
+				 <K230_PINMUX(41, 2)>; /* iic1 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic3_pins: iic3-pins {
+		iic3-pins-cfg {
+			pinmux = <K230_PINMUX(44, 2)>, /* iic3 scl */
+				 <K230_PINMUX(45, 2)>; /* iic3 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic4_pins: iic4-pins {
+		iic4-pins-cfg {
+			pinmux = <K230_PINMUX(46, 3)>, /* iic4 scl */
+				 <K230_PINMUX(47, 3)>; /* iic4 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic0_pins: iic0-pins {
+		iic0-pins-cfg {
+			pinmux = <K230_PINMUX(48, 3)>, /* iic0 scl */
+				 <K230_PINMUX(49, 3)>; /* iic0 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	uart3_pins: uart3-pins {
+		uart3-txd-cfg {
+			pinmux = <K230_PINMUX(50, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart3-rxd-cfg {
+			pinmux = <K230_PINMUX(51, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	key_pins: key-pins {
+		key-pins-cfg {
+			pinmux = <K230_PINMUX(52, 0)>, /* key0 */
+				 <K230_PINMUX(53, 0)>; /* key1 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	mmc1_pins: mmc1-pins {
+		mmc1-cmd-cfg {
+			pinmux = <K230_PINMUX(54, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+
+		mmc1-clk-cfg {
+			pinmux = <K230_PINMUX(55, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		mmc1-data-cfg {
+			pinmux = <K230_PINMUX(56, 2)>, /* mmc1 data0 */
+				 <K230_PINMUX(57, 2)>, /* mmc1 data1 */
+				 <K230_PINMUX(58, 2)>, /* mmc1 data2 */
+				 <K230_PINMUX(59, 2)>; /* mmc1 data3 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-evb.dts b/arch/riscv/boot/dts/canaan/k230-evb.dts
new file mode 100644
index 0000000000000..bfa53f2e24033
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-evb.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Kendryte K230 EVB";
+	compatible = "canaan,k230-usip-lp3-evb", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-pinctrl.h b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
new file mode 100644
index 0000000000000..63dd999ca55b6
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/*
+ * Copyright (C) 2024 Canaan Bright Sight Co. Ltd
+ * Copyright (C) 2024 Ze Huang <18771902331@163.com>
+ */
+
+#ifndef _K230_PINCTRL_H
+#define _K230_PINCTRL_H
+
+#define K230_MSC_3V3 0
+#define K230_MSC_1V8 1
+
+#define BANK_VOLTAGE_DEFAULT       K230_MSC_1V8
+#define BANK_VOLTAGE_IO50_IO61     K230_MSC_3V3
+
+#define K230_PINMUX(pin, mode) (((pin) << 8) | (mode))
+
+#endif /* _K230_PINCTRL_H */
diff --git a/arch/riscv/boot/dts/canaan/k230.dtsi b/arch/riscv/boot/dts/canaan/k230.dtsi
new file mode 100644
index 0000000000000..a73b1926ab934
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230.dtsi
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/reset/canaan,k230-rst.h>
+#include "k230-pinctrl.h"
+
+/dts-v1/;
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "canaan,kendryte-k230";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <27000000>;
+
+		cpu@0 {
+			compatible = "thead,c908", "riscv";
+			device_type = "cpu";
+			reg = <0>;
+			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicsr_zifencei_zihpm_zfh_zba_zbb_zbc_zbs_zvfh_svpbmt";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
+					       "zicbop", "zicboz", "zicntr", "zicsr", "zifencei",
+					       "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zvfh",
+					       "svpbmt";
+			riscv,cbom-block-size = <64>;
+			riscv,cbop-block-size = <64>;
+			riscv,cboz-block-size = <64>;
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			next-level-cache = <&l2_cache>;
+			mmu-type = "riscv,sv39";
+
+			cpu0_intc: interrupt-controller {
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+		};
+
+		l2_cache: l2-cache {
+			compatible = "cache";
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-size = <262144>;
+			cache-sets = <256>;
+			cache-unified;
+		};
+	};
+
+	apb_clk: apb-clk-clock {
+		compatible = "fixed-clock";
+		clock-frequency = <50000000>;
+		clock-output-names = "apb_clk";
+		#clock-cells = <0>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&plic>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-noncoherent;
+		ranges;
+
+		plic: interrupt-controller@f00000000 {
+			compatible = "canaan,k230-plic", "thead,c900-plic";
+			reg = <0xf 0x00000000 0x0 0x04000000>;
+			interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>;
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <2>;
+			riscv,ndev = <208>;
+		};
+
+		clint: timer@f04000000 {
+			compatible = "canaan,k230-clint", "thead,c900-clint";
+			reg = <0xf 0x04000000 0x0 0x00010000>;
+			interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>;
+		};
+
+		rst: reset-controller@91101000 {
+			compatible = "canaan,k230-rst";
+			reg = <0x0 0x91101000 0x0 0x1000>;
+			#reset-cells = <1>;
+		};
+
+		pinctrl: pinctrl@91105000 {
+			compatible = "canaan,k230-pinctrl";
+			reg = <0x0 0x91105000 0x0 0x100>;
+		};
+
+		uart0: serial@91400000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91400000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <16 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART0>;
+			status = "disabled";
+		};
+
+		uart1: serial@91401000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91401000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART1>;
+			status = "disabled";
+		};
+
+		uart2: serial@91402000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91402000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <18 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART2>;
+			status = "disabled";
+		};
+
+		uart3: serial@91403000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91403000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART3>;
+			status = "disabled";
+		};
+
+		uart4: serial@91404000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91404000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART4>;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile
index 345ed7a48cc10..43f91cbf338bb 100644
--- a/arch/riscv/boot/dts/microchip/Makefile
+++ b/arch/riscv/boot/dts/microchip/Makefile
@@ -1,9 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-beaglev-fire.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-disco-kit.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit-prod.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb
-dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-tysom-m.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-beaglev-fire.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-disco-kit.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit-prod.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-m100pfsevp.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-polarberry.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-sev-kit.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-tysom-m.dtb
+dtb-$(CONFIG_ARCH_MICROCHIP) += pic64gx-curiosity-kit.dtb
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
index e01a216e6c3a8..888ecbb3b2759 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
@@ -3,7 +3,6 @@
 
 /dts-v1/;
 
-#include "mpfs.dtsi"
 #include "mpfs-icicle-kit-fabric.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/leds/common.h>
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
index 71f724325578f..785176dabcf12 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /* Copyright (c) 2020-2021 Microchip Technology Inc */
 
+#include "mpfs.dtsi"
+#include "mpfs-pinctrl.dtsi"
+
 / {
 	core_pwm0: pwm@40000000 {
 		compatible = "microchip,corepwm-rtl-v4";
@@ -80,6 +83,16 @@ refclk_ccc: clock-cccref {
 	};
 };
 
+&can0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&can0_fabric>;
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ikrd_can1_cfg>;
+};
+
 &ccc_nw {
 	clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>,
 		 <&refclk_ccc>, <&refclk_ccc>;
@@ -87,3 +100,53 @@ &ccc_nw {
 		      "dll0_ref", "dll1_ref";
 	status = "okay";
 };
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_fabric>;
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_fabric>;
+};
+
+&mmuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_fabric>;
+};
+
+&mmuart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_fabric>;
+};
+
+&mmuart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_fabric>;
+};
+
+&mmuart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_fabric>;
+};
+
+&mssio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_mssio>, <&can1_mssio>, <&mdio0_mssio>, <&mdio1_mssio>;
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_fabric>;
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_fabric>;
+};
+
+&spi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ikrd_spi1_cfg>;
+};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi
new file mode 100644
index 0000000000000..d37cabc6438a1
--- /dev/null
+++ b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+
+&iomux0 {
+	spi0_fabric: mux-spi0-fabric {
+		function = "spi0";
+		groups = "spi0_fabric";
+	};
+
+	spi0_mssio: mux-spi0-mssio {
+		function = "spi0";
+		groups = "spi0_mssio";
+	};
+
+	spi1_fabric: mux-spi1-fabric {
+		function = "spi1";
+		groups = "spi1_fabric";
+	};
+
+	spi1_mssio: mux-spi1-mssio {
+		function = "spi1";
+		groups = "spi1_mssio";
+	};
+
+	i2c0_fabric: mux-i2c0-fabric {
+		function = "i2c0";
+		groups = "i2c0_fabric";
+	};
+
+	i2c0_mssio: mux-i2c0-mssio {
+		function = "i2c0";
+		groups = "i2c0_mssio";
+	};
+
+	i2c1_fabric: mux-i2c1-fabric {
+		function = "i2c1";
+		groups = "i2c1_fabric";
+	};
+
+	i2c1_mssio: mux-i2c1-mssio {
+		function = "i2c1";
+		groups = "i2c1_mssio";
+	};
+
+	can0_fabric: mux-can0-fabric {
+		function = "can0";
+		groups = "can0_fabric";
+	};
+
+	can0_mssio: mux-can0-mssio {
+		function = "can0";
+		groups = "can0_mssio";
+	};
+
+	can1_fabric: mux-can1-fabric {
+		function = "can1";
+		groups = "can1_fabric";
+	};
+
+	can1_mssio: mux-can1-mssio {
+		function = "can1";
+		groups = "can1_mssio";
+	};
+
+	qspi_fabric: mux-qspi-fabric {
+		function = "qspi";
+		groups = "qspi_fabric";
+	};
+
+	qspi_mssio: mux-qspi-mssio {
+		function = "qspi";
+		groups = "qspi_mssio";
+	};
+
+	uart0_fabric: mux-uart0-fabric {
+		function = "uart0";
+		groups = "uart0_fabric";
+	};
+
+	uart0_mssio: mux-uart0-mssio {
+		function = "uart0";
+		groups = "uart0_mssio";
+	};
+
+	uart1_fabric: mux-uart1-fabric {
+		function = "uart1";
+		groups = "uart1_fabric";
+	};
+
+	uart1_mssio: mux-uart1-mssio {
+		function = "uart1";
+		groups = "uart1_mssio";
+	};
+
+	uart2_fabric: mux-uart2-fabric {
+		function = "uart2";
+		groups = "uart2_fabric";
+	};
+
+	uart2_mssio: mux-uart2-mssio {
+		function = "uart2";
+		groups = "uart2_mssio";
+	};
+
+	uart3_fabric: mux-uart3-fabric {
+		function = "uart3";
+		groups = "uart3_fabric";
+	};
+
+	uart3_mssio: mux-uart3-mssio {
+		function = "uart3";
+		groups = "uart3_mssio";
+	};
+
+	uart4_fabric: mux-uart4-fabric {
+		function = "uart4";
+		groups = "uart4_fabric";
+	};
+
+	uart4_mssio: mux-uart4-mssio {
+		function = "uart4";
+		groups = "uart4_mssio";
+	};
+
+	mdio0_fabric: mux-mdio0-fabric {
+		function = "mdio0";
+		groups = "mdio0_fabric";
+	};
+
+	mdio0_mssio: mux-mdio0-mssio {
+		function = "mdio0";
+		groups = "mdio0_mssio";
+	};
+
+	mdio1_fabric: mux-mdio1-fabric {
+		function = "mdio1";
+		groups = "mdio1_fabric";
+	};
+
+	mdio1_mssio: mux-mdio1-mssio {
+		function = "mdio1";
+		groups = "mdio1_mssio";
+	};
+};
+
+&mssio {
+	ikrd_can1_cfg: ikrd-can1-cfg {
+		can1-pins {
+			pins = <34>, <35>, <36>;
+			function = "can";
+			bias-pull-up;
+			drive-strength = <8>;
+			power-source = <3300000>;
+			microchip,ibufmd = <0x1>;
+		};
+	};
+
+	ikrd_spi1_cfg: ikrd-spi1-cfg {
+		spi1-pins {
+			pins = <30>, <31>, <32>, <33>;
+			function = "spi";
+			bias-pull-up;
+			drive-strength = <8>;
+			power-source = <3300000>;
+			microchip,ibufmd = <0x1>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi
index 5c2963e269b83..0e7d99ec4b266 100644
--- a/arch/riscv/boot/dts/microchip/mpfs.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi
@@ -254,7 +254,23 @@ pdma: dma-controller@3000000 {
 		mss_top_sysreg: syscon@20002000 {
 			compatible = "microchip,mpfs-mss-top-sysreg", "syscon", "simple-mfd";
 			reg = <0x0 0x20002000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			#reset-cells = <1>;
+
+			iomux0: pinctrl@200 {
+				compatible = "microchip,mpfs-pinctrl-iomux0";
+				reg = <0x200 0x4>;
+				pinctrl-use-default;
+
+			};
+
+			mssio: pinctrl@204 {
+				compatible = "microchip,mpfs-pinctrl-mssio";
+				reg = <0x204 0x7c>;
+				/* on icicle ref design at least */
+				pinctrl-use-default;
+			};
 		};
 
 		sysreg_scb: syscon@20003000 {
@@ -428,6 +444,7 @@ can0: can@2010c000 {
 			clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>;
 			interrupt-parent = <&plic>;
 			interrupts = <56>;
+			resets = <&mss_top_sysreg CLK_CAN0>;
 			status = "disabled";
 		};
 
@@ -437,6 +454,7 @@ can1: can@2010d000 {
 			clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>;
 			interrupt-parent = <&plic>;
 			interrupts = <57>;
+			resets = <&mss_top_sysreg CLK_CAN1>;
 			status = "disabled";
 		};
 
@@ -448,8 +466,8 @@ mac0: ethernet@20110000 {
 			interrupt-parent = <&plic>;
 			interrupts = <64>, <65>, <66>, <67>, <68>, <69>;
 			local-mac-address = [00 00 00 00 00 00];
-			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>;
-			clock-names = "pclk", "hclk";
+			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>;
+			clock-names = "pclk", "hclk", "tsu_clk";
 			resets = <&mss_top_sysreg CLK_MAC0>;
 			status = "disabled";
 		};
@@ -462,8 +480,8 @@ mac1: ethernet@20112000 {
 			interrupt-parent = <&plic>;
 			interrupts = <70>, <71>, <72>, <73>, <74>, <75>;
 			local-mac-address = [00 00 00 00 00 00];
-			clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>;
-			clock-names = "pclk", "hclk";
+			clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>, <&refclk>;
+			clock-names = "pclk", "hclk", "tsu_clk";
 			resets = <&mss_top_sysreg CLK_MAC1>;
 			status = "disabled";
 		};
diff --git a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
new file mode 100644
index 0000000000000..2f2ccd77af30a
--- /dev/null
+++ b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device Tree Source for the PIC64GX Curiosity Kit
+ *
+ * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com>
+ *
+ * The Curiosity-GX10000 (PIC64GX SoC Curiosity Kit) is a compact SoC
+ * prototyping board featuring a Microchip PIC64GX SoC
+ * PIC64GX-1000. Features include:
+ *  - 1 GB DDR4 SDRAM
+ *  - Gigabit Ethernet
+ *  - microSD-card slot
+ *
+ * https://www.microchip.com/en-us/development-tool/curiosity-pic64gx1000-kit-es
+ */
+
+/dts-v1/;
+
+#include "pic64gx.dtsi"
+#include "pic64gx-pinctrl.dtsi"
+
+/* Clock frequency (in Hz) of the rtcclk */
+#define RTCCLK_FREQ	1000000
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Microchip PIC64GX Curiosity Kit";
+	compatible = "microchip,pic64gx-curiosity-kit", "microchip,pic64gx";
+
+	aliases {
+		ethernet0 = &mac0;
+		serial1 = &mmuart1;
+		serial2 = &mmuart2;
+	};
+
+	chosen {
+		stdout-path = "serial1:115200n8";
+	};
+
+	cpus {
+		timebase-frequency = <RTCCLK_FREQ>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		hss: hss-buffer@bfc00000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0xbfc00000 0x0 0x400000>;
+			no-map;
+		};
+	};
+};
+
+&gpio0 {
+	interrupts = <13>, <14>, <15>, <16>,
+		     <17>, <18>, <19>, <20>,
+		     <21>, <22>, <23>, <24>,
+		     <25>, <26>;
+	status ="okay";
+	gpio-line-names =
+		"", "", "", "", "", "", "", "",
+		"", "", "", "", "MIPI_CAM_RESET", "MIPI_CAM_STANDBY";
+};
+
+&gpio1 {
+	interrupts = <27>, <28>, <29>, <30>,
+		     <31>, <32>, <33>, <34>,
+		     <35>, <36>, <37>, <38>,
+		     <39>, <40>, <41>, <42>,
+		     <43>, <44>, <45>, <46>,
+		     <47>, <48>, <49>, <50>;
+	status ="okay";
+	gpio-line-names =
+		"", "", "LED1", "LED2", "LED3", "LED4", "LED5", "LED6",
+		"LED7", "LED8", "", "", "", "", "", "",
+		"", "", "", "", "HDMI_HPD", "", "", "GPIO_1_23";
+};
+
+&gpio2 {
+	interrupts = <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>,
+		     <53>, <53>, <53>, <53>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mdio1_gpio>, <&spi0_gpio>, <&can0_gpio>, <&pcie_gpio>,
+		    <&qspi_gpio>, <&uart3_gpio>, <&uart4_gpio>, <&can1_gpio>;
+	status ="okay";
+	gpio-line-names =
+		"", "", "", "", "", "", "SWITCH2", "USR_IO12",
+		"DIP1", "DIP2", "", "DIP3", "USR_IO1", "USR_IO2", "USR_IO7", "USR_IO8",
+		"USR_IO3", "USR_IO4", "USR_IO5", "USR_IO6", "", "", "USR_IO9", "USR_IO10",
+		"DIP4", "USR_IO11", "", "", "SWITCH1", "", "", "";
+};
+
+&mac0 {
+	status = "okay";
+	phy-mode = "sgmii";
+	phy-handle = <&phy0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mdio0_default>;
+
+	phy0: ethernet-phy@b {
+		reg = <0xb>;
+	};
+};
+
+&mbox {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&mmc {
+	bus-width = <4>;
+	disable-wp;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	sdhci-caps-mask = <0x00000007 0x00000000>;
+	status = "okay";
+};
+
+&mmuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_fio>;
+	status = "okay";
+};
+
+&mmuart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_default>;
+	status = "okay";
+};
+
+&refclk {
+	clock-frequency = <125000000>;
+};
+
+&rtc {
+	status = "okay";
+};
+
+&syscontroller {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi
new file mode 100644
index 0000000000000..cf73b0556633d
--- /dev/null
+++ b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+
+&iomux0 {
+	spi0_fio: mux-spi0-fabric {
+		function = "spi0";
+		groups = "spi0_fabric";
+	};
+
+	spi1_mssio: mux-spi1-mssio {
+		function = "spi1";
+		groups = "spi1_mssio";
+	};
+
+	i2c0_mssio: mux-i2c0-mssio {
+		function = "i2c0";
+		groups = "i2c0_mssio";
+	};
+
+	i2c1_mssio: mux-i2c1-mssio {
+		function = "i2c1";
+		groups = "i2c1_mssio";
+	};
+
+	can0_fio: mux-can0-fabric {
+		function = "can0";
+		groups = "can0_fabric";
+	};
+
+	can1_fio: mux-can1-fabric {
+		function = "can1";
+		groups = "can1_fabric";
+	};
+
+	qspi_fio: mux-qspi-fabric {
+		function = "qspi";
+		groups = "qspi_fabric";
+	};
+
+	uart0_mssio: mux-uart0-mssio {
+		function = "uart0";
+		groups = "uart0_mssio";
+	};
+
+	uart1_fio: mux-uart1-fabric {
+		function = "uart1";
+		groups = "uart1_fabric";
+	};
+
+	uart2_fio: mux-uart2-fabric {
+		function = "uart2";
+		groups = "uart2_fabric";
+	};
+
+	uart3_fio: mux-uart3-fabric {
+		function = "uart3";
+		groups = "uart3_fabric";
+	};
+
+	uart4_fio: mux-uart4-fabric {
+		function = "uart4";
+		groups = "uart4_fabric";
+	};
+
+	mdio0_fio: mux-mdio0-fabric {
+		function = "mdio0";
+		groups = "mdio0_fabric";
+	};
+
+	mdio1_fio: mux-mdio1-fabric {
+		function = "mdio1";
+		groups = "mdio1_fabric";
+	};
+};
+
+&gpio2_pinctrl {
+	//TODO rethink the labels, since a bunch of these are not defaults or
+	//just outright remove the non-default groups
+	mdio0_default: mux-mac0 {
+		function = "mdio0";
+		groups = "mdio0";
+	};
+
+	mdio0_gpio: mux-mac0-gpio2 {
+		function = "gpio";
+		groups = "gpio_mdio0";
+	};
+
+	mdio1_default: mux-mac1 {
+		function = "mdio1";
+		groups = "mdio1";
+	};
+
+	mdio1_gpio: mux-mac1-gpio2 {
+		function = "gpio";
+		groups = "gpio_mdio1";
+	};
+
+	spi0_default: mux-spi0 {
+		function = "spi0";
+		groups = "spi0";
+	};
+
+	spi0_gpio: mux-spi0-gpio2 {
+		function = "gpio";
+		groups = "gpio_spi0";
+	};
+
+	can0_default: mux-can0 {
+		function = "can0";
+		groups = "can0";
+	};
+
+	can0_gpio: mux-can0-gpio2 {
+		function = "gpio";
+		groups = "gpio_can0";
+	};
+
+	pcie_default: mux-pcie {
+		function = "pcie";
+		groups = "pcie";
+	};
+
+	pcie_gpio: mux-pcie-gpio2 {
+		function = "gpio";
+		groups = "gpio_pcie";
+	};
+
+	qspi_default: mux-qspi {
+		function = "qspi";
+		groups = "qspi";
+	};
+
+	qspi_gpio: mux-qspi-gpio2 {
+		function = "gpio";
+		groups = "gpio_qspi";
+	};
+
+	uart3_default: mux-uart3 {
+		function = "uart3";
+		groups = "uart3";
+	};
+
+	uart3_gpio: mux-uart3-gpio2 {
+		function = "gpio";
+		groups = "gpio_uart3";
+	};
+
+	uart4_default: mux-uart4 {
+		function = "uart4";
+		groups = "uart4";
+	};
+
+	uart4_gpio: mux-uart4-gpio2 {
+		function = "gpio";
+		groups = "gpio_uart4";
+	};
+
+	can1_default: mux-can1 {
+		function = "can1";
+		groups = "can1";
+	};
+
+	can1_gpio: mux-can1-gpio2 {
+		function = "gpio";
+		groups = "gpio_can1";
+	};
+
+	uart2_default: mux-uart2 {
+		function = "uart2";
+		groups = "uart2";
+	};
+
+	uart2_gpio: mux-uart2-gpio2 {
+		function = "gpio";
+		groups = "gpio_uart2";
+	};
+};
diff --git a/arch/riscv/boot/dts/microchip/pic64gx.dtsi b/arch/riscv/boot/dts/microchip/pic64gx.dtsi
new file mode 100644
index 0000000000000..c164d7bc270a2
--- /dev/null
+++ b/arch/riscv/boot/dts/microchip/pic64gx.dtsi
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device Tree Source for the PIC64GX SoCs
+ *
+ * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com>
+ *
+ * PIC64GX is a series RISC-V multicore SoCs:
+ * https://www.microchip.com/en-us/products/microprocessors/64-bit-mpus/pic64gx
+ */
+
+/dts-v1/;
+#include "dt-bindings/clock/microchip,mpfs-clock.h"
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	model = "Microchip PIC64GX SoC";
+	compatible = "microchip,pic64gx";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "sifive,e51", "sifive,rocket0", "riscv";
+			device_type = "cpu";
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <16384>;
+			reg = <0>;
+			riscv,isa = "rv64imac";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "c", "zicntr", "zicsr",
+					       "zifencei", "zihpm";
+			clocks = <&clkcfg CLK_CPU>;
+			status = "disabled";
+
+			cpu0_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+
+		cpu1: cpu@1 {
+			compatible = "sifive,u54-mc", "sifive,rocket0", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <64>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			i-cache-block-size = <64>;
+			i-cache-sets = <64>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv39";
+			reg = <1>;
+			riscv,isa = "rv64imafdc";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr",
+					       "zicsr", "zifencei", "zihpm";
+			clocks = <&clkcfg CLK_CPU>;
+			tlb-split;
+			next-level-cache = <&cctrllr>;
+			status = "okay";
+
+			cpu1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+
+		cpu2: cpu@2 {
+			compatible = "sifive,u54-mc", "sifive,rocket0", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <64>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			i-cache-block-size = <64>;
+			i-cache-sets = <64>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv39";
+			reg = <2>;
+			riscv,isa = "rv64imafdc";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr",
+					       "zicsr", "zifencei", "zihpm";
+			clocks = <&clkcfg CLK_CPU>;
+			tlb-split;
+			next-level-cache = <&cctrllr>;
+			status = "okay";
+
+			cpu2_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+
+		cpu3: cpu@3 {
+			compatible = "sifive,u54-mc", "sifive,rocket0", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <64>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			i-cache-block-size = <64>;
+			i-cache-sets = <64>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv39";
+			reg = <3>;
+			riscv,isa = "rv64imafdc";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr",
+					       "zicsr", "zifencei", "zihpm";
+			clocks = <&clkcfg CLK_CPU>;
+			tlb-split;
+			next-level-cache = <&cctrllr>;
+			status = "okay";
+
+			cpu3_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+
+		cpu4: cpu@4 {
+			compatible = "sifive,u54-mc", "sifive,rocket0", "riscv";
+			d-cache-block-size = <64>;
+			d-cache-sets = <64>;
+			d-cache-size = <32768>;
+			d-tlb-sets = <1>;
+			d-tlb-size = <32>;
+			device_type = "cpu";
+			i-cache-block-size = <64>;
+			i-cache-sets = <64>;
+			i-cache-size = <32768>;
+			i-tlb-sets = <1>;
+			i-tlb-size = <32>;
+			mmu-type = "riscv,sv39";
+			reg = <4>;
+			riscv,isa = "rv64imafdc";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr",
+					       "zicsr", "zifencei", "zihpm";
+			clocks = <&clkcfg CLK_CPU>;
+			tlb-split;
+			next-level-cache = <&cctrllr>;
+			status = "okay";
+			cpu4_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+			};
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+			};
+		};
+	};
+
+	scbclk: clock-80000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <80000000>;
+	};
+
+	refclk: mssrefclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+	};
+
+	axiclk: axi-aclk0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <125000000>;
+	};
+
+	videoclk: video-aclk0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <125000000>;
+	};
+
+	syscontroller: syscontroller {
+		compatible = "microchip,pic64gx-sys-controller";
+		mboxes = <&mbox 0>;
+	};
+
+	soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		compatible = "simple-bus";
+		ranges;
+
+		clint: clint@2000000 {
+			compatible = "microchip,pic64gx-clint", "sifive,clint0";
+			reg = <0x0 0x2000000 0x0 0xC000>;
+			interrupts-extended = <&cpu0_intc 0xffffffff>, <&cpu0_intc 0xffffffff>,
+					      <&cpu1_intc 3>, <&cpu1_intc 7>,
+					      <&cpu2_intc 3>, <&cpu2_intc 7>,
+					      <&cpu3_intc 3>, <&cpu3_intc 7>,
+					      <&cpu4_intc 3>, <&cpu4_intc 7>;
+		};
+
+		cctrllr: cache-controller@2010000 {
+			compatible = "microchip,pic64gx-ccache", "microchip,mpfs-ccache",
+				     "sifive,fu540-c000-ccache", "cache";
+			reg = <0x0 0x2010000 0x0 0x1000>;
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <1024>;
+			cache-size = <2097152>;
+			cache-unified;
+			interrupt-parent = <&plic>;
+			interrupts = <1>, <3>, <4>, <2>;
+		};
+
+		pdma: dma-controller@3000000 {
+			compatible = "microchip,pic64gx-pdma", "microchip,mpfs-pdma",
+				     "sifive,pdma0";
+			reg = <0x0 0x3000000 0x0 0x8000>;
+			interrupt-parent = <&plic>;
+			interrupts = <5 6>, <7 8>, <9 10>, <11 12>;
+			#dma-cells = <1>;
+		};
+
+		plic: interrupt-controller@c000000 {
+			compatible = "microchip,pic64gx-plic", "sifive,plic-1.0.0";
+			reg = <0x0 0xc000000 0x0 0x4000000>;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupts-extended = <&cpu0_intc 0xffffffff>,
+					      <&cpu1_intc 0xffffffff>, <&cpu1_intc 9>,
+					      <&cpu2_intc 0xffffffff>, <&cpu2_intc 9>,
+					      <&cpu3_intc 0xffffffff>, <&cpu3_intc 9>,
+					      <&cpu4_intc 0xffffffff>, <&cpu4_intc 9>;
+			riscv,ndev = <186>;
+		};
+
+		mmuart0: serial@20000000 {
+			compatible = "ns16550a";
+			reg = <0x0 0x20000000 0x0 0x400>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			interrupt-parent = <&plic>;
+			interrupts = <90>;
+			current-speed = <115200>;
+			clocks = <&clkcfg CLK_MMUART0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&uart0_mssio>;
+			status = "disabled"; /* Reserved for the HSS */
+		};
+
+		mss_top_sysreg: syscon@20002000 {
+			compatible = "microchip,pic64gx-mss-top-sysreg",
+				     "microchip,mpfs-mss-top-sysreg",
+				     "syscon", "simple-mfd";
+			reg = <0x0 0x20002000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			#reset-cells = <1>;
+
+			iomux0: pinctrl@200 {
+				compatible = "microchip,pic64gx-pinctrl-iomux0",
+					     "microchip,mpfs-pinctrl-iomux0";
+				reg = <0x200 0x4>;
+				pinctrl-use-default;
+			};
+
+			mssio: pinctrl@204 {
+				compatible = "microchip,pic64gx-pinctrl-mssio",
+					     "microchip,mpfs-pinctrl-mssio";
+				reg = <0x204 0x7c>;
+				/* on icicle ref design at least */
+				pinctrl-use-default;
+			};
+		};
+
+		sysreg_scb: syscon@20003000 {
+			compatible = "microchip,pic64gx-sysreg-scb",
+				     "microchip,mpfs-sysreg-scb",
+				     "syscon";
+			reg = <0x0 0x20003000 0x0 0x1000>;
+		};
+
+		/* Common node entry for emmc/sd */
+		mmc: mmc@20008000 {
+			compatible = "microchip,pic64gx-sd4hc", "cdns,sd4hc";
+			reg = <0x0 0x20008000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <88>;
+			clocks = <&clkcfg CLK_MMC>;
+			max-frequency = <200000000>;
+			status = "disabled";
+		};
+
+		mmuart1: serial@20100000 {
+			compatible = "ns16550a";
+			reg = <0x0 0x20100000 0x0 0x400>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			interrupt-parent = <&plic>;
+			interrupts = <91>;
+			current-speed = <115200>;
+			clocks = <&clkcfg CLK_MMUART1>;
+			status = "disabled";
+		};
+
+		mmuart2: serial@20102000 {
+			compatible = "ns16550a";
+			reg = <0x0 0x20102000 0x0 0x400>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			interrupt-parent = <&plic>;
+			interrupts = <92>;
+			current-speed = <115200>;
+			clocks = <&clkcfg CLK_MMUART2>;
+			status = "disabled";
+		};
+
+		mmuart3: serial@20104000 {
+			compatible = "ns16550a";
+			reg = <0x0 0x20104000 0x0 0x400>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			interrupt-parent = <&plic>;
+			interrupts = <93>;
+			current-speed = <115200>;
+			clocks = <&clkcfg CLK_MMUART3>;
+			status = "disabled";
+		};
+
+		mmuart4: serial@20106000 {
+			compatible = "ns16550a";
+			reg = <0x0 0x20106000 0x0 0x400>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			interrupt-parent = <&plic>;
+			interrupts = <94>;
+			clocks = <&clkcfg CLK_MMUART4>;
+			current-speed = <115200>;
+			status = "disabled";
+		};
+
+		spi0: spi@20108000 {
+			compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x20108000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <54>;
+			clocks = <&clkcfg CLK_SPI0>;
+			status = "disabled";
+		};
+
+		spi1: spi@20109000 {
+			compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x20109000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <55>;
+			clocks = <&clkcfg CLK_SPI1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi1_mssio>;
+			status = "disabled";
+		};
+
+		i2c0: i2c@2010a000 {
+			compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7";
+			reg = <0x0 0x2010a000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&plic>;
+			interrupts = <58>;
+			clocks = <&clkcfg CLK_I2C0>;
+			clock-frequency = <100000>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&i2c0_mssio>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@2010b000 {
+			compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7";
+			reg = <0x0 0x2010b000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&plic>;
+			interrupts = <61>;
+			clocks = <&clkcfg CLK_I2C1>;
+			clock-frequency = <100000>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&i2c1_mssio>;
+			status = "disabled";
+		};
+
+		can0: can@2010c000 {
+			compatible = "microchip,pic64gx-can", "microchip,mpfs-can";
+			reg = <0x0 0x2010c000 0x0 0x1000>;
+			clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>;
+			interrupt-parent = <&plic>;
+			interrupts = <56>;
+			resets = <&mss_top_sysreg CLK_CAN0>;
+			status = "disabled";
+		};
+
+		can1: can@2010d000 {
+			compatible = "microchip,pic64gx-can", "microchip,mpfs-can";
+			reg = <0x0 0x2010d000 0x0 0x1000>;
+			clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>;
+			interrupt-parent = <&plic>;
+			interrupts = <57>;
+			resets = <&mss_top_sysreg CLK_CAN1>;
+			status = "disabled";
+		};
+
+		mac0: ethernet@20110000 {
+			compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb",
+				     "cdns,macb";
+			reg = <0x0 0x20110000 0x0 0x2000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&plic>;
+			interrupts = <64>, <65>, <66>, <67>, <68>, <69>;
+			/* Filled in by a bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>;
+			clock-names = "pclk", "hclk";
+			resets = <&mss_top_sysreg CLK_MAC0>;
+			status = "disabled";
+		};
+
+		mac1: ethernet@20112000 {
+			compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb",
+				     "cdns,macb";
+			reg = <0x0 0x20112000 0x0 0x2000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&plic>;
+			interrupts = <70>, <71>, <72>, <73>, <74>, <75>;
+			/* Filled in by a bootloader */
+			local-mac-address = [00 00 00 00 00 00];
+			clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>;
+			clock-names = "pclk", "hclk";
+			resets = <&mss_top_sysreg CLK_MAC1>;
+			status = "disabled";
+		};
+
+		gpio0: gpio@20120000 {
+			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
+			reg = <0x0 0x20120000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupt-controller;
+			#interrupt-cells = <1>;
+			clocks = <&clkcfg CLK_GPIO0>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <14>;
+			status = "disabled";
+		};
+
+		gpio1: gpio@20121000 {
+			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
+			reg = <0x0 0x20121000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupt-controller;
+			#interrupt-cells = <1>;
+			clocks = <&clkcfg CLK_GPIO1>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <24>;
+			status = "disabled";
+		};
+
+		gpio2: gpio@20122000 {
+			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
+			reg = <0x0 0x20122000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupt-controller;
+			#interrupt-cells = <1>;
+			clocks = <&clkcfg CLK_GPIO2>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			ngpios = <32>;
+			status = "disabled";
+		};
+
+		rtc: rtc@20124000 {
+			compatible = "microchip,pic64gx-rtc", "microchip,mpfs-rtc";
+			reg = <0x0 0x20124000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <80>, <81>;
+			clocks = <&clkcfg CLK_RTC>, <&clkcfg CLK_RTCREF>;
+			clock-names = "rtc", "rtcref";
+			status = "disabled";
+		};
+
+		usb: usb@20201000 {
+			compatible = "microchip,pic64gx-musb", "microchip,mpfs-musb";
+			reg = <0x0 0x20201000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <86>, <87>;
+			clocks = <&clkcfg CLK_USB>;
+			interrupt-names = "dma", "mc";
+			status = "disabled";
+		};
+
+		qspi: spi@21000000 {
+			compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x21000000 0x0 0x1000>;
+			interrupt-parent = <&plic>;
+			interrupts = <85>;
+			clocks = <&clkcfg CLK_QSPI>;
+			status = "disabled";
+		};
+
+		control_scb: syscon@37020000 {
+			compatible = "microchip,pic64gx-control-scb",
+				     "microchip,mpfs-control-scb",
+				     "syscon";
+			reg = <0x0 0x37020000 0x0 0x100>;
+		};
+
+		syscontroller_qspi: spi@37020100 {
+			compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x37020100 0x0 0x100>;
+			interrupt-parent = <&plic>;
+			interrupts = <110>;
+			clocks = <&scbclk>;
+			status = "disabled";
+		};
+
+		mbox: mailbox@37020800 {
+			compatible = "microchip,pic64gx-mailbox", "microchip,mpfs-mailbox";
+			reg = <0x0 0x37020800 0x0 0x100>;
+			interrupt-parent = <&plic>;
+			interrupts = <96>;
+			#mbox-cells = <1>;
+			status = "disabled";
+		};
+
+		ccc_se: clock-controller@38010000 {
+			compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc";
+			reg = <0x0 0x38010000 0x0 0x1000>, <0x0 0x38020000 0x0 0x1000>,
+			      <0x0 0x39010000 0x0 0x1000>, <0x0 0x39020000 0x0 0x1000>;
+			#clock-cells = <1>;
+			status = "disabled";
+		};
+
+		ccc_ne: clock-controller@38040000 {
+			compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc";
+			reg = <0x0 0x38040000 0x0 0x1000>, <0x0 0x38080000 0x0 0x1000>,
+			      <0x0 0x39040000 0x0 0x1000>, <0x0 0x39080000 0x0 0x1000>;
+			#clock-cells = <1>;
+			status = "disabled";
+		};
+
+		ccc_nw: clock-controller@38100000 {
+			compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc";
+			reg = <0x0 0x38100000 0x0 0x1000>, <0x0 0x38200000 0x0 0x1000>,
+			      <0x0 0x39100000 0x0 0x1000>, <0x0 0x39200000 0x0 0x1000>;
+			#clock-cells = <1>;
+			status = "disabled";
+		};
+
+		ccc_sw: clock-controller@38400000 {
+			compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc";
+			reg = <0x0 0x38400000 0x0 0x1000>, <0x0 0x38800000 0x0 0x1000>,
+			      <0x0 0x39400000 0x0 0x1000>, <0x0 0x39800000 0x0 0x1000>;
+			#clock-cells = <1>;
+			status = "disabled";
+		};
+
+		clkcfg: clkcfg@3e001000 {
+			compatible = "microchip,pic64gx-clkcfg", "microchip,mpfs-clkcfg";
+			reg = <0x0 0x3e001000 0x0 0x1000>;
+			clocks = <&refclk>;
+			#clock-cells = <1>;
+		};
+
+		gpio2_pinctrl: pinctrl@41000000 {
+			compatible = "microchip,pic64gx-pinctrl-gpio2";
+			reg = <0x0 0x41000000 0x0 0x4>;
+			pinctrl-use-default;
+			pinctrl-names = "default";
+			pinctrl-0 = <&mdio0_fio>, <&mdio1_fio>, <&spi0_fio>, <&qspi_fio>,
+				    <&uart3_fio>, <&uart4_fio>, <&can1_fio>, <&can0_fio>,
+				    <&uart2_fio>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
index 06b0ce5a2db7a..8446b4dfe1a1d 100644
--- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi
+++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
@@ -417,7 +417,7 @@ sdhci1: mmc@4320000 {
 		};
 
 		dmac: dma-controller@4330000 {
-			compatible = "snps,axi-dma-1.01a";
+			compatible = "sophgo,cv1800b-axi-dma";
 			reg = <0x04330000 0x1000>;
 			interrupts = <SOC_PERIPHERAL_IRQ(13) IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>;
diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
index 5971605754b35..404b69c47b91f 100644
--- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
@@ -33,7 +33,7 @@ led1 {
 		};
 	};
 
-	pcie_vcc_3v3: pcie-vcc3v3 {
+	pcie_vcc_3v3: regulator-pcie-vcc3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "PCIE_VCC3V3";
 		regulator-min-microvolt = <3300000>;
@@ -41,7 +41,7 @@ pcie_vcc_3v3: pcie-vcc3v3 {
 		regulator-always-on;
 	};
 
-	reg_dc_in: dc-in-12v {
+	reg_dc_in: regulator-dc-in-12v {
 		compatible = "regulator-fixed";
 		regulator-name = "dc_in_12v";
 		regulator-min-microvolt = <12000000>;
@@ -50,7 +50,7 @@ reg_dc_in: dc-in-12v {
 		regulator-always-on;
 	};
 
-	reg_vcc_4v: vcc-4v {
+	reg_vcc_4v: regulator-vcc-4v {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc_4v";
 		regulator-min-microvolt = <4000000>;
@@ -60,7 +60,7 @@ reg_vcc_4v: vcc-4v {
 		vin-supply = <&reg_dc_in>;
 	};
 
-	usb3-vbus-5v {
+	regulator-usb3-vbus-5v {
 		compatible = "regulator-fixed";
 		regulator-name = "USB30_VBUS";
 		regulator-min-microvolt = <5000000>;
@@ -70,7 +70,7 @@ usb3-vbus-5v {
 		enable-active-high;
 	};
 
-	usb3_hub_5v: usb3-hub-5v {
+	usb3_hub_5v: regulator-usb3-hub-5v {
 		compatible = "regulator-fixed";
 		regulator-name = "USB30_HUB";
 		regulator-min-microvolt = <5000000>;
@@ -190,7 +190,15 @@ pmic@41 {
 		compatible = "spacemit,p1";
 		reg = <0x41>;
 		interrupts = <64>;
-		vin-supply = <&reg_vcc_4v>;
+		vin1-supply = <&reg_vcc_4v>;
+		vin2-supply = <&reg_vcc_4v>;
+		vin3-supply = <&reg_vcc_4v>;
+		vin4-supply = <&reg_vcc_4v>;
+		vin5-supply = <&reg_vcc_4v>;
+		vin6-supply = <&reg_vcc_4v>;
+		aldoin-supply = <&reg_vcc_4v>;
+		dldoin1-supply = <&buck5>;
+		dldoin2-supply = <&buck5>;
 
 		regulators {
 			buck1 {
@@ -221,7 +229,7 @@ buck4 {
 				regulator-always-on;
 			};
 
-			buck5 {
+			buck5: buck5 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <3450000>;
 				regulator-ramp-delay = <5000>;
@@ -305,6 +313,7 @@ &pcie1_phy {
 
 &pcie1_port {
 	phys = <&pcie1_phy>;
+	vpcie3v3-supply = <&pcie_vcc_3v3>;
 };
 
 &pcie1 {
@@ -320,6 +329,7 @@ &pcie2_phy {
 
 &pcie2_port {
 	phys = <&pcie2_phy>;
+	vpcie3v3-supply = <&pcie_vcc_3v3>;
 };
 
 &pcie2 {
diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts
index 800a112d5d705..9959c8023ece0 100644
--- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts
@@ -21,7 +21,7 @@ chosen {
 		stdout-path = "serial0";
 	};
 
-	reg_dc_in: dc-in-12v {
+	reg_dc_in: regulator-dc-in-12v {
 		compatible = "regulator-fixed";
 		regulator-name = "dc_in_12v";
 		regulator-min-microvolt = <12000000>;
@@ -30,7 +30,7 @@ reg_dc_in: dc-in-12v {
 		regulator-always-on;
 	};
 
-	reg_vcc_4v: vcc-4v {
+	reg_vcc_4v: regulator-vcc-4v {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc_4v";
 		regulator-min-microvolt = <4000000>;
@@ -100,7 +100,15 @@ pmic@41 {
 		compatible = "spacemit,p1";
 		reg = <0x41>;
 		interrupts = <64>;
-		vin-supply = <&reg_vcc_4v>;
+		vin1-supply = <&reg_vcc_4v>;
+		vin2-supply = <&reg_vcc_4v>;
+		vin3-supply = <&reg_vcc_4v>;
+		vin4-supply = <&reg_vcc_4v>;
+		vin5-supply = <&reg_vcc_4v>;
+		vin6-supply = <&reg_vcc_4v>;
+		aldoin-supply = <&reg_vcc_4v>;
+		dldoin1-supply = <&buck5>;
+		dldoin2-supply = <&buck5>;
 
 		regulators {
 			buck1 {
@@ -131,7 +139,7 @@ buck4 {
 				regulator-always-on;
 			};
 
-			buck5 {
+			buck5: buck5 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <3450000>;
 				regulator-ramp-delay = <5000>;
diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi
index 529ec68e9c23e..d2015201f8e5a 100644
--- a/arch/riscv/boot/dts/spacemit/k1.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1.dtsi
@@ -1033,6 +1033,7 @@ pcie-bus {
 			#size-cells = <2>;
 			dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>,
 				     <0x0 0xb8000000 0x1 0x38000000 0x3 0x48000000>;
+
 			pcie0: pcie@ca000000 {
 				device_type = "pci";
 				compatible = "spacemit,k1-pcie";
@@ -1044,6 +1045,7 @@ pcie0: pcie@ca000000 {
 					    "atu",
 					    "config",
 					    "link";
+				linux,pci-domain = <0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0 0x8f002000 0x0 0x00100000>,
@@ -1087,6 +1089,7 @@ pcie1: pcie@ca400000 {
 					    "atu",
 					    "config",
 					    "link";
+				linux,pci-domain = <1>;
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0 0x9f002000 0x0 0x00100000>,
@@ -1130,6 +1133,7 @@ pcie2: pcie@ca800000 {
 					    "atu",
 					    "config",
 					    "link";
+				linux,pci-domain = <2>;
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0 0xb7002000 0x0 0x00100000>,
diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
index b691304d4b746..b098dbd0e7a15 100644
--- a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
+++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
@@ -5,6 +5,7 @@
  */
 
 #include "k3.dtsi"
+#include "k3-pinctrl.dtsi"
 
 / {
 	model = "SpacemiT K3 Pico-ITX";
@@ -25,5 +26,7 @@ memory@100000000 {
 };
 
 &uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
 	status = "okay";
 };
diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
new file mode 100644
index 0000000000000..efb0f15721887
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (c) 2026 Yixun Lan <dlan@kernel.org>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+
+#define K3_PADCONF(pin, func) (((pin) << 16) | (func))
+
+/* Map GPIO pin to each bank's <index, offset> */
+#define K3_GPIO(x)	(x / 32) (x % 32)
+
+&pinctrl {
+	/omit-if-no-ref/
+	uart0_0_cfg: uart0-0-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(149, 2)>,	/* uart0 tx */
+				 <K3_PADCONF(150, 2)>;	/* uart0 rx */
+
+			bias-pull-up = <0>;
+			drive-strength = <25>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi
index b69cf81b5d553..a3a8ceddabec4 100644
--- a/arch/riscv/boot/dts/spacemit/k3.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3.dtsi
@@ -4,6 +4,8 @@
  * Copyright (c) 2026 Guodong Xu <guodong@riscstar.com>
  */
 
+#include <dt-bindings/clock/spacemit,k3-clocks.h>
+#include <dt-bindings/reset/spacemit,k3-resets.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 
 /dts-v1/;
@@ -398,6 +400,36 @@ core3 {
 		};
 	};
 
+	clocks {
+		vctcxo_1m: clock-1m {
+			compatible = "fixed-clock";
+			clock-frequency = <1000000>;
+			clock-output-names = "vctcxo_1m";
+			#clock-cells = <0>;
+		};
+
+		vctcxo_24m: clock-24m {
+			compatible = "fixed-clock";
+			clock-frequency = <24000000>;
+			clock-output-names = "vctcxo_24m";
+			#clock-cells = <0>;
+		};
+
+		vctcxo_3m: clock-3m {
+			compatible = "fixed-clock";
+			clock-frequency = <3000000>;
+			clock-output-names = "vctcxo_3m";
+			#clock-cells = <0>;
+		};
+
+		osc_32k: clock-32k {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			clock-output-names = "osc_32k";
+			#clock-cells = <0>;
+		};
+	};
+
 	soc: soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&saplic>;
@@ -406,12 +438,24 @@ soc: soc {
 		dma-noncoherent;
 		ranges;
 
+		syscon_apbc: system-controller@d4015000 {
+			compatible = "spacemit,k3-syscon-apbc";
+			reg = <0x0 0xd4015000 0x0 0x1000>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>;
+			clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
 		uart0: serial@d4017000 {
 			compatible = "spacemit,k3-uart", "intel,xscale-uart";
 			reg = <0x0 0xd4017000 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART0>,
+				 <&syscon_apbc CLK_APBC_UART0_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART0>;
 			interrupts = <42 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -421,7 +465,10 @@ uart2: serial@d4017100 {
 			reg = <0x0 0xd4017100 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART2>,
+				 <&syscon_apbc CLK_APBC_UART2_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART2>;
 			interrupts = <44 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -431,7 +478,10 @@ uart3: serial@d4017200 {
 			reg = <0x0 0xd4017200 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART3>,
+				 <&syscon_apbc CLK_APBC_UART3_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART3>;
 			interrupts = <45 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -441,7 +491,10 @@ uart4: serial@d4017300 {
 			reg = <0x0 0xd4017300 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART4>,
+				 <&syscon_apbc CLK_APBC_UART4_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART4>;
 			interrupts = <46 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -451,7 +504,10 @@ uart5: serial@d4017400 {
 			reg = <0x0 0xd4017400 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART5>,
+				 <&syscon_apbc CLK_APBC_UART5_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART5>;
 			interrupts = <47 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -461,7 +517,10 @@ uart6: serial@d4017500 {
 			reg = <0x0 0xd4017500 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART6>,
+				 <&syscon_apbc CLK_APBC_UART6_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART6>;
 			interrupts = <48 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -471,7 +530,10 @@ uart7: serial@d4017600 {
 			reg = <0x0 0xd4017600 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART7>,
+				 <&syscon_apbc CLK_APBC_UART7_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART7>;
 			interrupts = <49 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -481,7 +543,10 @@ uart8: serial@d4017700 {
 			reg = <0x0 0xd4017700 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART8>,
+				 <&syscon_apbc CLK_APBC_UART8_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART8>;
 			interrupts = <50 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
@@ -491,21 +556,88 @@ uart9: serial@d4017800 {
 			reg = <0x0 0xd4017800 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART9>,
+				 <&syscon_apbc CLK_APBC_UART9_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART9>;
 			interrupts = <51 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
 
+		gpio: gpio@d4019000 {
+			compatible = "spacemit,k3-gpio";
+			reg = <0x0 0xd4019000 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_GPIO>,
+				 <&syscon_apbc CLK_APBC_GPIO_BUS>;
+			clock-names = "core", "bus";
+			gpio-controller;
+			#gpio-cells = <3>;
+			interrupts = <58 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			gpio-ranges = <&pinctrl 0 0 0 32>,
+				      <&pinctrl 1 0 32 32>,
+				      <&pinctrl 2 0 64 32>,
+				      <&pinctrl 3 0 96 32>;
+		};
+
+		pinctrl: pinctrl@d401e000 {
+			compatible = "spacemit,k3-pinctrl";
+			reg = <0x0 0xd401e000 0x0 0x1000>;
+			clocks = <&syscon_apbc CLK_APBC_AIB>,
+				 <&syscon_apbc CLK_APBC_AIB_BUS>;
+			clock-names = "func", "bus";
+		};
+
 		uart10: serial@d401f000 {
 			compatible = "spacemit,k3-uart", "intel,xscale-uart";
 			reg = <0x0 0xd401f000 0x0 0x100>;
 			reg-shift = <2>;
 			reg-io-width = <4>;
-			clock-frequency = <14700000>;
+			clocks = <&syscon_apbc CLK_APBC_UART10>,
+				 <&syscon_apbc CLK_APBC_UART10_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_UART10>;
 			interrupts = <281 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
 
+		syscon_mpmu: system-controller@d4050000 {
+			compatible = "spacemit,k3-syscon-mpmu";
+			reg = <0x0 0xd4050000 0x0 0x10000>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>;
+			clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m";
+			#clock-cells = <1>;
+			#power-domain-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		pll: clock-controller@d4090000 {
+			compatible = "spacemit,k3-pll";
+			reg = <0x0 0xd4090000 0x0 0x10000>;
+			clocks = <&vctcxo_24m>;
+			spacemit,mpmu = <&syscon_mpmu>;
+			#clock-cells = <1>;
+		};
+
+		syscon_apmu: system-controller@d4282800 {
+			compatible = "spacemit,k3-syscon-apmu";
+			reg = <0x0 0xd4282800 0x0 0x400>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>;
+			clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m";
+			#clock-cells = <1>;
+			#power-domain-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_dciu: system-controller@d8440000 {
+			compatible = "spacemit,k3-syscon-dciu";
+			reg = <0x0 0xd8440000 0x0 0xc000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
 		simsic: interrupt-controller@e0400000 {
 			compatible = "spacemit,k3-imsics", "riscv,imsics";
 			reg = <0x0 0xe0400000 0x0 0x200000>;
diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
index 8cfe8033305d8..11ab9fcee38af 100644
--- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
@@ -463,6 +463,7 @@ GPOEN_ENABLE,
 			input-disable;
 			input-schmitt-disable;
 			slew-rate = <0>;
+			bootph-pre-ram;
 		};
 
 		mmc-pins {
diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts
index 63aa94d65ab55..566b0bdc2800d 100644
--- a/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts
+++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts
@@ -13,7 +13,7 @@ / {
 
 &mmc0 {
 	bus-width = <4>;
-	cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>;
+	broken-cd;
 	disable-wp;
 };
 
diff --git a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts
index c58c2085ca92a..7cb7d28683bce 100644
--- a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts
+++ b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts
@@ -29,6 +29,17 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
 	thermal-zones {
 		cpu-thermal {
 			polling-delay = <1000>;
@@ -121,6 +132,20 @@ rx-pins {
 	};
 };
 
+&dpu {
+	status = "okay";
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&hdmi_out_port {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins>;
diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi
index bd5d33840884e..b663077428940 100644
--- a/arch/riscv/boot/dts/thead/th1520.dtsi
+++ b/arch/riscv/boot/dts/thead/th1520.dtsi
@@ -585,6 +585,72 @@ clk_vo: clock-controller@ffef528050 {
 			#clock-cells = <1>;
 		};
 
+		hdmi: hdmi@ffef540000 {
+			compatible = "thead,th1520-dw-hdmi";
+			reg = <0xff 0xef540000 0x0 0x40000>;
+			reg-io-width = <4>;
+			interrupts = <111 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clk_vo CLK_HDMI_PCLK>,
+				 <&clk_vo CLK_HDMI_SFR>,
+				 <&clk_vo CLK_HDMI_CEC>,
+				 <&clk_vo CLK_HDMI_PIXCLK>;
+			clock-names = "iahb", "isfr", "cec", "pix";
+			resets = <&rst TH1520_RESET_ID_HDMI>,
+				 <&rst TH1520_RESET_ID_HDMI_APB>;
+			reset-names = "main", "apb";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					hdmi_in: endpoint {
+						remote-endpoint = <&dpu_out_dp1>;
+					};
+				};
+
+				hdmi_out_port: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
+		dpu: display@ffef600000 {
+			compatible = "thead,th1520-dc8200", "verisilicon,dc";
+			reg = <0xff 0xef600000 0x0 0x100000>;
+			interrupts = <93 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clk_vo CLK_DPU_CCLK>,
+				 <&clk_vo CLK_DPU_ACLK>,
+				 <&clk_vo CLK_DPU_HCLK>,
+				 <&clk_vo CLK_DPU_PIXELCLK0>,
+				 <&clk_vo CLK_DPU_PIXELCLK1>;
+			clock-names = "core", "axi", "ahb", "pix0", "pix1";
+			resets = <&rst TH1520_RESET_ID_DPU_CORE>,
+				 <&rst TH1520_RESET_ID_DPU_AXI>,
+				 <&rst TH1520_RESET_ID_DPU_AHB>;
+			reset-names = "core", "axi", "ahb";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				dpu_port1: port@1 {
+					reg = <1>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					dpu_out_dp1: endpoint@1 {
+						reg = <1>;
+						remote-endpoint = <&hdmi_in>;
+					};
+				};
+			};
+		};
+
 		dmac0: dma-controller@ffefc00000 {
 			compatible = "snps,axi-dma-1.01a";
 			reg = <0xff 0xefc00000 0x0 0x1000>;
diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c
index 0b942183f708f..6f1c683f0ec03 100644
--- a/arch/riscv/errata/thead/errata.c
+++ b/arch/riscv/errata/thead/errata.c
@@ -153,7 +153,7 @@ static bool errata_probe_ghostwrite(unsigned int stage,
 	 * target-c9xx cores report arch_id and impid as 0
 	 *
 	 * While ghostwrite may not affect all c9xx cores that implement
-	 * xtheadvector, there is no futher granularity than c9xx. Assume
+	 * xtheadvector, there is no further granularity than c9xx. Assume
 	 * vulnerable for this entire class of processors when xtheadvector is
 	 * enabled.
 	 */
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index 3f33dc54f94b2..616b8b332ac56 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -46,7 +46,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 #endif
 
 /*
- * First, the atomic ops that have no ordering constraints and therefor don't
+ * First, the atomic ops that have no ordering constraints and therefore don't
  * have the AQ or RL bits set.  These don't return anything, so there's only
  * one version to worry about.
  */
@@ -81,7 +81,7 @@ ATOMIC_OPS(xor, xor,  i)
 
 /*
  * Atomic ops that have ordered, relaxed, acquire, and release variants.
- * There's two flavors of these: the arithmatic ops have both fetch and return
+ * There's two flavors of these: the arithmetic ops have both fetch and return
  * versions, while the logical ops only have fetch versions.
  */
 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix)	\
diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h
index c7aea7886d22a..aa4961b0e2082 100644
--- a/arch/riscv/include/asm/elf.h
+++ b/arch/riscv/include/asm/elf.h
@@ -59,8 +59,8 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr);
 #endif
 
 /*
- * Provides information on the availiable set of ISA extensions to userspace,
- * via a bitmap that coorespends to each single-letter ISA extension.  This is
+ * Provides information on the available set of ISA extensions to userspace,
+ * via a bitmap that corresponds to each single-letter ISA extension.  This is
  * essentially defunct, but will remain for compatibility with userspace.
  */
 #define ELF_HWCAP	riscv_get_elf_hwcap()
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 187aad0a7b03e..46dbf86198057 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -190,7 +190,10 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x);
 
 #define sym_to_pfn(x)           __phys_to_pfn(__pa_symbol(x))
 
-unsigned long kaslr_offset(void);
+static inline unsigned long kaslr_offset(void)
+{
+	return kernel_map.virt_offset;
+}
 
 static __always_inline void *pfn_to_kaddr(unsigned long pfn)
 {
@@ -204,7 +207,7 @@ static __always_inline void *pfn_to_kaddr(unsigned long pfn)
 	(unsigned long)(_addr) >= PAGE_OFFSET && pfn_valid(virt_to_pfn(_addr));	\
 })
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_NON_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_NON_EXEC
 
 #include <asm-generic/memory_model.h>
 #include <asm-generic/getorder.h>
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 08d1ca0471046..a75e93852f212 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -984,17 +984,17 @@ static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
 }
 
 #ifdef CONFIG_PAGE_TABLE_CHECK
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return pte_present(pte) && pte_user(pte);
 }
 
-static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
+static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
 {
 	return pmd_leaf(pmd) && pmd_user(pmd);
 }
 
-static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
+static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
 {
 	return pud_leaf(pud) && pud_user(pud);
 }
@@ -1284,13 +1284,6 @@ extern u64 satp_mode;
 void paging_init(void);
 void misc_mem_init(void);
 
-/*
- * ZERO_PAGE is a global shared page that is always zero,
- * used for zero-mapped memory areas, etc.
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 /*
  * Use set_p*_safe(), and elide TLB flushing, when confident that *no*
  * TLB flush will be required as a result of the "set". For example, use
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index 4c3dd94d0f638..812517b2cec13 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -86,7 +86,7 @@ struct pt_regs;
  *    preempt_v. All preempt_v context should be dropped in such case because
  *    V-regs are caller-saved. Only sstatus.VS=ON is persisted across a
  *    schedule() call.
- *  - bit 30: The in-kernel preempt_v context is saved, and requries to be
+ *  - bit 30: The in-kernel preempt_v context is saved, and is required to be
  *    restored when returning to the context that owns the preempt_v.
  *  - bit 31: The in-kernel preempt_v context is dirty, as signaled by the
  *    trap entry code. Any context switches out-of current task need to save
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 36918c9200c92..55019fdfa9eca 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -120,7 +120,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #include <asm-generic/thread_info_tif.h>
 
 #define TIF_32BIT			16	/* compat-mode 32bit process */
-#define TIF_RISCV_V_DEFER_RESTORE	17	/* restore Vector before returing to user */
+#define TIF_RISCV_V_DEFER_RESTORE	17	/* restore Vector before returning to user */
 
 #define _TIF_RISCV_V_DEFER_RESTORE	BIT(TIF_RISCV_V_DEFER_RESTORE)
 
diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h
deleted file mode 100644
index 96011861e46b4..0000000000000
--- a/arch/riscv/include/asm/xor.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2021 SiFive
- */
-
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#ifdef CONFIG_RISCV_ISA_V
-#include <asm/vector.h>
-#include <asm/switch_to.h>
-#include <asm/asm-prototypes.h>
-
-static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
-			 const unsigned long *__restrict p2)
-{
-	kernel_vector_begin();
-	xor_regs_2_(bytes, p1, p2);
-	kernel_vector_end();
-}
-
-static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1,
-			 const unsigned long *__restrict p2,
-			 const unsigned long *__restrict p3)
-{
-	kernel_vector_begin();
-	xor_regs_3_(bytes, p1, p2, p3);
-	kernel_vector_end();
-}
-
-static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1,
-			 const unsigned long *__restrict p2,
-			 const unsigned long *__restrict p3,
-			 const unsigned long *__restrict p4)
-{
-	kernel_vector_begin();
-	xor_regs_4_(bytes, p1, p2, p3, p4);
-	kernel_vector_end();
-}
-
-static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1,
-			 const unsigned long *__restrict p2,
-			 const unsigned long *__restrict p3,
-			 const unsigned long *__restrict p4,
-			 const unsigned long *__restrict p5)
-{
-	kernel_vector_begin();
-	xor_regs_5_(bytes, p1, p2, p3, p4, p5);
-	kernel_vector_end();
-}
-
-static struct xor_block_template xor_block_rvv = {
-	.name = "rvv",
-	.do_2 = xor_vector_2,
-	.do_3 = xor_vector_3,
-	.do_4 = xor_vector_4,
-	.do_5 = xor_vector_5
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES           \
-	do {        \
-		xor_speed(&xor_block_8regs);    \
-		xor_speed(&xor_block_32regs);    \
-		if (has_vector()) { \
-			xor_speed(&xor_block_rvv);\
-		} \
-	} while (0)
-#endif
diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c
index 71698ee11621a..26c018ebce03c 100644
--- a/arch/riscv/kernel/acpi.c
+++ b/arch/riscv/kernel/acpi.c
@@ -69,7 +69,7 @@ static int __init acpi_fadt_sanity_check(void)
 
 	/*
 	 * FADT is required on riscv; retrieve it to check its presence
-	 * and carry out revision and ACPI HW reduced compliancy tests
+	 * and carry out revision and ACPI HW reduced compliance tests
 	 */
 	status = acpi_get_table(ACPI_SIG_FADT, 0, &table);
 	if (ACPI_FAILURE(status)) {
diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c
index 8d18d6727f0fc..b430edfb83f4c 100644
--- a/arch/riscv/kernel/ftrace.c
+++ b/arch/riscv/kernel/ftrace.c
@@ -148,7 +148,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad
 
 /*
  * This is called early on, and isn't wrapped by
- * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
+ * ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold
  * text_mutex, which triggers a lockdep failure.  SMP isn't running so we could
  * just directly poke the text, but it's simpler to just take the lock
  * ourselves.
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 9c99c5ad6fe8a..e7ea7eb532a8f 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -81,7 +81,7 @@ relocate_enable_mmu:
 	sub a1, a1, a2
 	add ra, ra, a1
 
-	/* Point stvec to virtual address of intruction after satp write */
+	/* Point stvec to virtual address of instruction after satp write */
 	la a2, 1f
 	add a2, a2, a1
 	csrw CSR_TVEC, a2
diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c
index 15fec5d1e6dec..0bf629204c76a 100644
--- a/arch/riscv/kernel/kgdb.c
+++ b/arch/riscv/kernel/kgdb.c
@@ -175,7 +175,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = {
 	{DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)},
 	{DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)},
 	{DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)},
-	{DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)},
+	{DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, s1)},
 	{DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)},
 	{DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)},
 	{DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)},
@@ -244,8 +244,9 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task)
 	gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6];
 	gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7];
 	gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8];
-	gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10];
-	gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11];
+	gdb_regs[DBG_REG_S9_OFF] = task->thread.s[9];
+	gdb_regs[DBG_REG_S10_OFF] = task->thread.s[10];
+	gdb_regs[DBG_REG_S11_OFF] = task->thread.s[11];
 	gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra;
 }
 
diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S
index 48f6c4f7dca0c..082fe0b0e3c08 100644
--- a/arch/riscv/kernel/mcount-dyn.S
+++ b/arch/riscv/kernel/mcount-dyn.S
@@ -66,7 +66,7 @@
 * 8(sp) stores the function return address (i.e. parent IP) that
 * can be accessed by &(fregs)->ra in tracing function.
 *
-* The other regs are saved at the respective localtion and accessed
+* The other regs are saved at the respective location and accessed
 * by the respective ftrace_regs member.
 *
 * Here is the layout of stack for your reference.
diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c
index 1675cbad8619d..98eaac6f66060 100644
--- a/arch/riscv/kernel/module-sections.c
+++ b/arch/riscv/kernel/module-sections.c
@@ -148,7 +148,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
 		return -ENOEXEC;
 	}
 
-	/* Calculate the maxinum number of entries */
+	/* Calculate the maximum number of entries */
 	for (i = 0; i < ehdr->e_shnum; i++) {
 		size_t num_relas = sechdrs[i].sh_size / sizeof(Elf_Rela);
 		Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset;
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index db13c9ddf9e3d..16b243376f362 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -42,19 +42,20 @@ static inline bool is_kernel_exittext(uintptr_t addr)
 static __always_inline void *patch_map(void *addr, const unsigned int fixmap)
 {
 	uintptr_t uintaddr = (uintptr_t) addr;
-	struct page *page;
+	phys_addr_t phys;
 
-	if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr))
-		page = phys_to_page(__pa_symbol(addr));
-	else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
-		page = vmalloc_to_page(addr);
-	else
-		return addr;
+	if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) {
+		phys = __pa_symbol(addr);
+	} else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) {
+		struct page *page = vmalloc_to_page(addr);
 
-	BUG_ON(!page);
+		BUG_ON(!page);
+		phys = page_to_phys(page) + offset_in_page(addr);
+	} else {
+		return addr;
+	}
 
-	return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
-					 offset_in_page(addr));
+	return (void *)set_fixmap_offset(fixmap, phys);
 }
 
 static void patch_unmap(int fixmap)
diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c
index 8723390c7cad5..9e2afabf94e80 100644
--- a/arch/riscv/kernel/probes/kprobes.c
+++ b/arch/riscv/kernel/probes/kprobes.c
@@ -149,7 +149,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p)
 
 /*
  * Interrupts need to be disabled before single-step mode is set, and not
- * reenabled until after single-step mode ends.
+ * re-enabled until after single-step mode ends.
  * Without disabling interrupt on local CPU, there is a chance of
  * interrupt occurrence in the period of exception return and  start of
  * out-of-line single-step, that result in wrongly single stepping
diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c
index f0d0691a86887..eb177d0ce8ab0 100644
--- a/arch/riscv/kernel/probes/uprobes.c
+++ b/arch/riscv/kernel/probes/uprobes.c
@@ -111,7 +111,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
 
 	current->thread.bad_cause = utask->autask.saved_cause;
 	/*
-	 * Task has received a fatal signal, so reset back to probbed
+	 * Task has received a fatal signal, so reset back to probed
 	 * address.
 	 */
 	instruction_pointer_set(regs, utask->vaddr);
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index d85916a3660c3..8b628580fe118 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -251,18 +251,14 @@ asmlinkage __visible void smp_callin(void)
 	set_cpu_online(curr_cpuid, true);
 
 	/*
-	 * Remote cache and TLB flushes are ignored while the CPU is offline,
-	 * so flush them both right now just in case.
+	 * Remote instruction cache and TLB flushes are ignored while the CPU
+	 * is offline, so flush them both right now just in case.
 	 */
 	local_flush_icache_all();
 	local_flush_tlb_all();
 #ifndef CONFIG_HOTPLUG_PARALLEL
 	complete(&cpu_running);
 #endif
-	/*
-	 * Disable preemption before enabling interrupts, so we don't try to
-	 * schedule a CPU that hasn't actually started yet.
-	 */
 	local_irq_enable();
 	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 }
diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c
index a0516172a33c8..7d69e3b6bab43 100644
--- a/arch/riscv/kernel/soc.c
+++ b/arch/riscv/kernel/soc.c
@@ -8,7 +8,7 @@
 #include <asm/soc.h>
 
 /*
- * This is called extremly early, before parse_dtb(), to allow initializing
+ * This is called extremely early, before parse_dtb(), to allow initializing
  * SoC hardware before memory or any device driver initialization.
  */
 void __init soc_early_init(void)
diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c
index aff93090c4efc..3efbf7874f3ba 100644
--- a/arch/riscv/kernel/suspend.c
+++ b/arch/riscv/kernel/suspend.c
@@ -78,7 +78,7 @@ int cpu_suspend(unsigned long arg,
 	suspend_save_csrs(&context);
 
 	/*
-	 * Function graph tracer state gets incosistent when the kernel
+	 * Function graph tracer state gets inconsistent when the kernel
 	 * calls functions that never return (aka finishers) hence disable
 	 * graph tracing during their execution.
 	 */
diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c
index 1adba746f164e..7e57f54dc5b21 100644
--- a/arch/riscv/kernel/usercfi.c
+++ b/arch/riscv/kernel/usercfi.c
@@ -230,17 +230,7 @@ int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr)
 static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size,
 					   unsigned long token_offset, bool set_tok)
 {
-	int flags = MAP_ANONYMOUS | MAP_PRIVATE;
-	struct mm_struct *mm = current->mm;
-	unsigned long populate;
-
-	if (addr)
-		flags |= MAP_FIXED_NOREPLACE;
-
-	mmap_write_lock(mm);
-	addr = do_mmap(NULL, addr, size, PROT_READ, flags,
-		       VM_SHADOW_STACK | VM_WRITE, 0, &populate, NULL);
-	mmap_write_unlock(mm);
+	addr = vm_mmap_shadow_stack(addr, size, 0);
 
 	if (!set_tok || IS_ERR_VALUE(addr))
 		goto out;
diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c
index d5e448aa90e74..682ba423cf203 100644
--- a/arch/riscv/kernel/vmcore_info.c
+++ b/arch/riscv/kernel/vmcore_info.c
@@ -3,6 +3,11 @@
 #include <linux/vmcore_info.h>
 #include <linux/pagemap.h>
 
+static inline u64 get_satp_value(void)
+{
+	return csr_read(CSR_SATP);
+}
+
 void arch_crash_save_vmcoreinfo(void)
 {
 	VMCOREINFO_NUMBER(phys_ram_base);
@@ -27,5 +32,7 @@ void arch_crash_save_vmcoreinfo(void)
 #else
 	vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n",
 						kernel_map.va_kernel_pa_offset);
+	vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset());
+	vmcoreinfo_append_str("NUMBER(satp)=0x%llx\n", get_satp_value());
 #endif
 }
diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c
index ff1aeac4eb8eb..e44f3ebe38440 100644
--- a/arch/riscv/kvm/tlb.c
+++ b/arch/riscv/kvm/tlb.c
@@ -182,7 +182,7 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
 
 	/*
 	 * Flush VS-stage TLB entries for implementation where VS-stage
-	 * TLB does not cahce guest physical address and VMID.
+	 * TLB does not cache guest physical address and VMID.
 	 */
 	if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa))
 		kvm_riscv_local_hfence_vvma_all(vmid);
diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c
index e873430e596b2..4efbf0a65b1de 100644
--- a/arch/riscv/kvm/vcpu_pmu.c
+++ b/arch/riscv/kvm/vcpu_pmu.c
@@ -675,7 +675,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 				pmc->counter_val += perf_event_read_value(pmc->perf_event,
 									  &enabled, &running);
 			/*
-			 * The counter and overflow indicies in the snapshot region are w.r.to
+			 * The counter and overflow indices in the snapshot region are w.r.to
 			 * cbase. Modify the set bit in the counter mask instead of the pmc_index
 			 * which indicates the absolute counter index.
 			 */
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index bbc031124974e..e220c35764ebd 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -16,5 +16,4 @@ lib-$(CONFIG_MMU)	+= uaccess.o
 lib-$(CONFIG_64BIT)	+= tishift.o
 lib-$(CONFIG_RISCV_ISA_ZICBOZ)	+= clear_page.o
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
-lib-$(CONFIG_RISCV_ISA_V)	+= xor.o
 lib-$(CONFIG_RISCV_ISA_V)	+= riscv_v_helpers.o
diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c
index 75bd0abffd631..5d4379768e5d0 100644
--- a/arch/riscv/lib/csum.c
+++ b/arch/riscv/lib/csum.c
@@ -269,7 +269,7 @@ unsigned int do_csum(const unsigned char *buff, int len)
 	 * on machines with fast misaligned accesses.
 	 *
 	 * There is some duplicate code between the "with_alignment" and
-	 * "no_alignment" implmentations, but the overlap is too awkward to be
+	 * "no_alignment" implementations, but the overlap is too awkward to be
 	 * able to fit in one function without introducing multiple static
 	 * branches. The largest chunk of overlap was delegated into the
 	 * do_csum_common function.
diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S
index cb3e2e7ef0baa..410e598d46773 100644
--- a/arch/riscv/lib/memmove.S
+++ b/arch/riscv/lib/memmove.S
@@ -40,8 +40,8 @@ SYM_FUNC_START(__memmove)
 	 *   Both Copy Modes: t1 - Temporary for load-store
 	 *   Both Copy Modes: t2 - Temporary for load-store
 	 *   Both Copy Modes: a5 - dest to src alignment offset
-	 *   Both Copy Modes: a6 - Shift ammount
-	 *   Both Copy Modes: a7 - Inverse Shift ammount
+	 *   Both Copy Modes: a6 - Shift amount
+	 *   Both Copy Modes: a7 - Inverse Shift amount
 	 *   Both Copy Modes: a2 - Alternate breakpoint for unrolled loops
 	 */
 
diff --git a/arch/riscv/lib/xor.S b/arch/riscv/lib/xor.S
deleted file mode 100644
index b28f2430e52fa..0000000000000
--- a/arch/riscv/lib/xor.S
+++ /dev/null
@@ -1,81 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2021 SiFive
- */
-#include <linux/linkage.h>
-#include <linux/export.h>
-#include <asm/asm.h>
-
-SYM_FUNC_START(xor_regs_2_)
-	vsetvli a3, a0, e8, m8, ta, ma
-	vle8.v v0, (a1)
-	vle8.v v8, (a2)
-	sub a0, a0, a3
-	vxor.vv v16, v0, v8
-	add a2, a2, a3
-	vse8.v v16, (a1)
-	add a1, a1, a3
-	bnez a0, xor_regs_2_
-	ret
-SYM_FUNC_END(xor_regs_2_)
-EXPORT_SYMBOL(xor_regs_2_)
-
-SYM_FUNC_START(xor_regs_3_)
-	vsetvli a4, a0, e8, m8, ta, ma
-	vle8.v v0, (a1)
-	vle8.v v8, (a2)
-	sub a0, a0, a4
-	vxor.vv v0, v0, v8
-	vle8.v v16, (a3)
-	add a2, a2, a4
-	vxor.vv v16, v0, v16
-	add a3, a3, a4
-	vse8.v v16, (a1)
-	add a1, a1, a4
-	bnez a0, xor_regs_3_
-	ret
-SYM_FUNC_END(xor_regs_3_)
-EXPORT_SYMBOL(xor_regs_3_)
-
-SYM_FUNC_START(xor_regs_4_)
-	vsetvli a5, a0, e8, m8, ta, ma
-	vle8.v v0, (a1)
-	vle8.v v8, (a2)
-	sub a0, a0, a5
-	vxor.vv v0, v0, v8
-	vle8.v v16, (a3)
-	add a2, a2, a5
-	vxor.vv v0, v0, v16
-	vle8.v v24, (a4)
-	add a3, a3, a5
-	vxor.vv v16, v0, v24
-	add a4, a4, a5
-	vse8.v v16, (a1)
-	add a1, a1, a5
-	bnez a0, xor_regs_4_
-	ret
-SYM_FUNC_END(xor_regs_4_)
-EXPORT_SYMBOL(xor_regs_4_)
-
-SYM_FUNC_START(xor_regs_5_)
-	vsetvli a6, a0, e8, m8, ta, ma
-	vle8.v v0, (a1)
-	vle8.v v8, (a2)
-	sub a0, a0, a6
-	vxor.vv v0, v0, v8
-	vle8.v v16, (a3)
-	add a2, a2, a6
-	vxor.vv v0, v0, v16
-	vle8.v v24, (a4)
-	add a3, a3, a6
-	vxor.vv v0, v0, v24
-	vle8.v v8, (a5)
-	add a4, a4, a6
-	vxor.vv v16, v0, v8
-	add a5, a5, a6
-	vse8.v v16, (a1)
-	add a1, a1, a6
-	bnez a0, xor_regs_5_
-	ret
-SYM_FUNC_END(xor_regs_5_)
-EXPORT_SYMBOL(xor_regs_5_)
diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
index d83a612464f6c..f8ead7cb7c7d6 100644
--- a/arch/riscv/mm/cacheflush.c
+++ b/arch/riscv/mm/cacheflush.c
@@ -29,7 +29,7 @@ void flush_icache_all(void)
 	 * Make sure all previous writes to the D$ are ordered before making
 	 * the IPI. The RISC-V spec states that a hart must execute a data fence
 	 * before triggering a remote fence.i in order to make the modification
-	 * visable for remote harts.
+	 * visible for remote harts.
 	 *
 	 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or
 	 * S-IMSIC, so the fence ensures previous data writes "happen before"
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 811e03786c560..5142ca80be6fa 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -63,16 +63,13 @@ phys_addr_t phys_ram_base __ro_after_init;
 EXPORT_SYMBOL(phys_ram_base);
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-#define VMEMMAP_ADDR_ALIGN	(1ULL << SECTION_SIZE_BITS)
+#define VMEMMAP_ADDR_ALIGN	max(1ULL << SECTION_SIZE_BITS, \
+				    MAX_FOLIO_VMEMMAP_ALIGN)
 
 unsigned long vmemmap_start_pfn __ro_after_init;
 EXPORT_SYMBOL(vmemmap_start_pfn);
 #endif
 
-unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
-							__page_aligned_bss;
-EXPORT_SYMBOL(empty_zero_page);
-
 extern char _start[];
 void *_dtb_early_va __initdata;
 uintptr_t _dtb_early_pa __initdata;
@@ -1078,11 +1075,6 @@ static int __init print_nokaslr(char *p)
 	return 0;
 }
 early_param("nokaslr", print_nokaslr);
-
-unsigned long kaslr_offset(void)
-{
-	return kernel_map.virt_offset;
-}
 #endif
 
 asmlinkage void __init setup_vm(uintptr_t dtb_pa)
diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c
index 559d291fac5c6..b1836e9481cee 100644
--- a/arch/riscv/mm/physaddr.c
+++ b/arch/riscv/mm/physaddr.c
@@ -9,7 +9,7 @@
 phys_addr_t __virt_to_phys(unsigned long x)
 {
 	/*
-	 * Boundary checking aginst the kernel linear mapping space.
+	 * Boundary checking against the kernel linear mapping space.
 	 */
 	WARN(!is_linear_mapping(x) && !is_kernel_mapping(x),
 	     "virt_to_phys used for non-linear address: %p (%pS)\n",
@@ -25,7 +25,7 @@ phys_addr_t __phys_addr_symbol(unsigned long x)
 	unsigned long kernel_end = kernel_start + kernel_map.size;
 
 	/*
-	 * Boundary checking aginst the kernel image mapping.
+	 * Boundary checking against the kernel image mapping.
 	 * __pa_symbol should only be used on kernel symbol addresses.
 	 */
 	VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end);
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index edc927d9e85a5..9116a921df798 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -80,7 +80,6 @@ config S390
 	#
 	# Note: keep this list sorted alphabetically
 	#
-	imply IMA_SECURE_AND_OR_TRUSTED_BOOT
 	select ALTERNATE_USER_ADDRESS_SPACE
 	select ARCH_32BIT_USTAT_F_TINODE
 	select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE
@@ -154,6 +153,7 @@ config S390
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
 	select ARCH_SUPPORTS_NUMA_BALANCING
+	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
 	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF
@@ -182,6 +182,7 @@ config S390
 	select GENERIC_IOREMAP if PCI
 	select HAVE_ALIGNED_STRUCT_PAGE
 	select HAVE_ARCH_AUDITSYSCALL
+	select HAVE_ARCH_GET_SECUREBOOT
 	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_JUMP_LABEL_RELATIVE
 	select HAVE_ARCH_KASAN
@@ -287,6 +288,7 @@ config S390
 	select VDSO_GETRANDOM
 	select VIRT_CPU_ACCOUNTING
 	select VMAP_STACK
+	select XOR_BLOCKS_ARCH
 	select ZONE_DMA
 	# Note: keep the above list sorted alphabetically
 
@@ -1024,7 +1026,7 @@ config S390_KPROBES_SANITY_TEST
 
 config S390_MODULES_SANITY_TEST
 	def_tristate n
-	depends on KUNIT
+	depends on KUNIT && m
 	default KUNIT_ALL_TESTS
 	prompt "Enable s390 specific modules tests"
 	select S390_MODULES_SANITY_TEST_HELPERS
diff --git a/arch/s390/Makefile b/arch/s390/Makefile
index d78ad6885ca2a..297976b410886 100644
--- a/arch/s390/Makefile
+++ b/arch/s390/Makefile
@@ -12,8 +12,7 @@ LD_BFD		:= elf64-s390
 KBUILD_LDFLAGS	:= -m elf64_s390
 KBUILD_AFLAGS_MODULE += -fPIC
 KBUILD_CFLAGS_MODULE += -fPIC
-KBUILD_AFLAGS	+= -m64
-KBUILD_CFLAGS	+= -m64
+KBUILD_CPPFLAGS	+= -m64
 KBUILD_CFLAGS	+= -fPIC
 LDFLAGS_vmlinux	:= $(call ld-option,-no-pie)
 extra_tools	:= relocs
@@ -22,7 +21,7 @@ KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__
 ifndef CONFIG_AS_IS_LLVM
 KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf))
 endif
-KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 -mpacked-stack -std=gnu11 -fms-extensions
+KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 -mpacked-stack $(CC_FLAGS_DIALECT)
 KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY
 KBUILD_CFLAGS_DECOMPRESSOR += -D__DECOMPRESSOR
 KBUILD_CFLAGS_DECOMPRESSOR += -Wno-pointer-sign
@@ -35,7 +34,6 @@ KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, address-of-packed-membe
 KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g)
 KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,))
 KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_NO_ARRAY_BOUNDS),-Wno-array-bounds)
-KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag)
 
 UTS_MACHINE	:= s390x
 STACK_SIZE	:= $(if $(CONFIG_KASAN),65536,$(if $(CONFIG_KMSAN),65536,16384))
diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
index 7f3343493ab98..c1dcf8b1b5794 100644
--- a/arch/s390/boot/startup.c
+++ b/arch/s390/boot/startup.c
@@ -440,7 +440,8 @@ static unsigned long setup_kernel_memory_layout(unsigned long kernel_size)
 	max_mappable = max(ident_map_size, MAX_DCSS_ADDR);
 	max_mappable = min(max_mappable, vmemmap_start);
 #ifdef CONFIG_RANDOMIZE_IDENTITY_BASE
-	__identity_base = round_down(vmemmap_start - max_mappable, rte_size);
+	if (kaslr_enabled())
+		__identity_base = round_down(vmemmap_start - max_mappable, rte_size);
 #endif
 	boot_debug("identity map:        0x%016lx-0x%016lx\n", __identity_base,
 		   __identity_base + ident_map_size);
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 98fd0a2f51c6a..12cdaaefb6db8 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -929,3 +929,5 @@ CONFIG_PERCPU_TEST=m
 CONFIG_ATOMIC64_SELFTEST=y
 CONFIG_TEST_BITOPS=m
 CONFIG_TEST_BPF=m
+CONFIG_PAGE_TABLE_CHECK=y
+CONFIG_PAGE_TABLE_CHECK_ENFORCED=y
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 64a50f0862aab..3039c88daa633 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -710,6 +710,9 @@ void kvm_arch_crypto_clear_masks(struct kvm *kvm);
 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
 			       unsigned long *aqm, unsigned long *adm);
 
+#define SIE64_RETURN_NORMAL	0
+#define SIE64_RETURN_MCCK	1
+
 int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa,
 	     unsigned long gasce);
 
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h
index f339258135f7a..56da819a79e6d 100644
--- a/arch/s390/include/asm/page.h
+++ b/arch/s390/include/asm/page.h
@@ -277,7 +277,7 @@ static inline unsigned long virt_to_pfn(const void *kaddr)
 
 #define virt_addr_valid(kaddr)	pfn_valid(phys_to_pfn(__pa_nodebug((unsigned long)(kaddr))))
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_NON_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_NON_EXEC
 
 #endif /* !__ASSEMBLER__ */
 
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 1c3c3be93be9c..67f5df20a57e9 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -16,8 +16,10 @@
 #include <linux/mm_types.h>
 #include <linux/cpufeature.h>
 #include <linux/page-flags.h>
+#include <linux/page_table_check.h>
 #include <linux/radix-tree.h>
 #include <linux/atomic.h>
+#include <linux/mmap_lock.h>
 #include <asm/ctlreg.h>
 #include <asm/bug.h>
 #include <asm/page.h>
@@ -1190,6 +1192,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 	/* At this point the reference through the mapping is still present */
 	if (mm_is_protected(mm) && pte_present(res))
 		WARN_ON_ONCE(uv_convert_from_secure_pte(res));
+	page_table_check_pte_clear(mm, addr, res);
 	return res;
 }
 
@@ -1208,6 +1211,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
 	/* At this point the reference through the mapping is still present */
 	if (mm_is_protected(vma->vm_mm) && pte_present(res))
 		WARN_ON_ONCE(uv_convert_from_secure_pte(res));
+	page_table_check_pte_clear(vma->vm_mm, addr, res);
 	return res;
 }
 
@@ -1231,6 +1235,9 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 	} else {
 		res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
 	}
+
+	page_table_check_pte_clear(mm, addr, res);
+
 	/* Nothing to do */
 	if (!mm_is_protected(mm) || !pte_present(res))
 		return res;
@@ -1327,6 +1334,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
 {
 	if (pte_present(entry))
 		entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
+	page_table_check_ptes_set(mm, addr, ptep, entry, nr);
 	for (;;) {
 		set_pte(ptep, entry);
 		if (--nr == 0)
@@ -1703,6 +1711,7 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 			      pmd_t *pmdp, pmd_t entry)
 {
+	page_table_check_pmd_set(mm, addr, pmdp, entry);
 	set_pmd(pmdp, entry);
 }
 
@@ -1717,7 +1726,11 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
 					    unsigned long addr, pmd_t *pmdp)
 {
-	return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+	pmd_t pmd;
+
+	pmd = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+	page_table_check_pmd_clear(mm, addr, pmd);
+	return pmd;
 }
 
 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
@@ -1725,12 +1738,17 @@ static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
 						 unsigned long addr,
 						 pmd_t *pmdp, int full)
 {
+	pmd_t pmd;
+
 	if (full) {
-		pmd_t pmd = *pmdp;
+		pmd = *pmdp;
 		set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+		page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
 		return pmd;
 	}
-	return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+	pmd = pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
+	page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
+	return pmd;
 }
 
 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
@@ -1744,11 +1762,16 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
 static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
 				   unsigned long addr, pmd_t *pmdp)
 {
-	pmd_t pmd;
+	pmd_t pmd = *pmdp;
 
-	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
-	pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
-	return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
+	VM_WARN_ON_ONCE(!pmd_present(pmd));
+	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
+#ifdef CONFIG_PAGE_TABLE_CHECK
+	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_READ));
+#endif
+	page_table_check_pmd_set(vma->vm_mm, addr, pmdp, pmd);
+	pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
+	return pmd;
 }
 
 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
@@ -1783,6 +1806,29 @@ static inline int has_transparent_hugepage(void)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
+#ifdef CONFIG_PAGE_TABLE_CHECK
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
+{
+	VM_BUG_ON(mm == &init_mm);
+
+	return pte_present(pte);
+}
+
+static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
+{
+	VM_BUG_ON(mm == &init_mm);
+
+	return pmd_leaf(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_READ);
+}
+
+static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
+{
+	VM_BUG_ON(mm == &init_mm);
+
+	return pud_leaf(pud);
+}
+#endif
+
 /*
  * 64 bit swap entry format:
  * A page-table entry has some bits we have to treat in a special way.
diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h
index 7c57ac968bf62..cbf60ade741d5 100644
--- a/arch/s390/include/asm/setup.h
+++ b/arch/s390/include/asm/setup.h
@@ -52,7 +52,6 @@ extern unsigned int zlib_dfltcc_support;
 #define ZLIB_DFLTCC_INFLATE_ONLY	3
 #define ZLIB_DFLTCC_FULL_DEBUG		4
 
-extern unsigned long ident_map_size;
 extern unsigned long max_mappable;
 
 /* The Write Back bit position in the physaddr is given by the SLPC PCI */
diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h
index c9ae680a28af9..ac3606c3babe4 100644
--- a/arch/s390/include/asm/stacktrace.h
+++ b/arch/s390/include/asm/stacktrace.h
@@ -62,7 +62,7 @@ struct stack_frame {
 		struct {
 			unsigned long sie_control_block;
 			unsigned long sie_savearea;
-			unsigned long sie_reason;
+			unsigned long sie_return;
 			unsigned long sie_flags;
 			unsigned long sie_control_block_phys;
 			unsigned long sie_guest_asce;
diff --git a/arch/s390/include/asm/xor.h b/arch/s390/include/asm/xor.h
deleted file mode 100644
index 857d6759b67f0..0000000000000
--- a/arch/s390/include/asm/xor.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Optimited xor routines
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- */
-#ifndef _ASM_S390_XOR_H
-#define _ASM_S390_XOR_H
-
-extern struct xor_block_template xor_block_xc;
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-do {							\
-	xor_speed(&xor_block_xc);			\
-} while (0)
-
-#define XOR_SELECT_TEMPLATE(FASTEST)	(&xor_block_xc)
-
-#endif /* _ASM_S390_XOR_H */
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 42c83d60d6fae..137e4793ec117 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -71,7 +71,6 @@ obj-$(CONFIG_STACKPROTECTOR)	+= stackprotector.o
 obj-$(CONFIG_KEXEC_FILE)	+= machine_kexec_file.o kexec_image.o
 obj-$(CONFIG_KEXEC_FILE)	+= kexec_elf.o
 obj-$(CONFIG_CERT_STORE)	+= cert_store.o
-obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT)	+= ima_arch.o
 
 obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o
 obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf.o perf_cpum_sf.o
@@ -80,5 +79,7 @@ obj-$(CONFIG_PERF_EVENTS)	+= perf_pai.o
 
 obj-$(CONFIG_TRACEPOINTS)	+= trace.o
 
+obj-$(CONFIG_BPF_SYSCALL)	+= bpf.o
+
 # vdso
 obj-y				+= vdso/
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index e1a5b5b54e4f8..fbd26f3e9f96b 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -63,7 +63,7 @@ int main(void)
 	OFFSET(__SF_EMPTY, stack_frame, empty[0]);
 	OFFSET(__SF_SIE_CONTROL, stack_frame, sie_control_block);
 	OFFSET(__SF_SIE_SAVEAREA, stack_frame, sie_savearea);
-	OFFSET(__SF_SIE_REASON, stack_frame, sie_reason);
+	OFFSET(__SF_SIE_RETURN, stack_frame, sie_return);
 	OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags);
 	OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys);
 	OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce);
diff --git a/arch/s390/kernel/bpf.c b/arch/s390/kernel/bpf.c
new file mode 100644
index 0000000000000..713337fae6263
--- /dev/null
+++ b/arch/s390/kernel/bpf.c
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <asm/lowcore.h>
+#include <linux/btf.h>
+
+__bpf_kfunc_start_defs();
+
+__bpf_kfunc struct lowcore *bpf_get_lowcore(void)
+{
+	return get_lowcore();
+}
+
+__bpf_kfunc_end_defs();
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 4873fe9d891ba..5817cb47b2d0b 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -200,7 +200,7 @@ SYM_FUNC_START(__sie64a)
 	stg	%r3,__SF_SIE_CONTROL(%r15)	# ...and virtual addresses
 	stg	%r4,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
 	stg	%r5,__SF_SIE_GUEST_ASCE(%r15)	# save guest asce
-	xc	__SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
+	xc	__SF_SIE_RETURN(8,%r15),__SF_SIE_RETURN(%r15) # return code = 0
 	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags
 	lmg	%r0,%r13,0(%r4)			# load guest gprs 0-13
 	mvi	__TI_sie(%r14),1
@@ -237,7 +237,7 @@ SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
 	xgr	%r4,%r4
 	xgr	%r5,%r5
 	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
-	lg	%r2,__SF_SIE_REASON(%r15)	# return exit reason code
+	lg	%r2,__SF_SIE_RETURN(%r15)	# return sie return code
 	BR_EX	%r14
 SYM_FUNC_END(__sie64a)
 EXPORT_SYMBOL(__sie64a)
diff --git a/arch/s390/kernel/ima_arch.c b/arch/s390/kernel/ima_arch.c
deleted file mode 100644
index f3c3e6e1c5d38..0000000000000
--- a/arch/s390/kernel/ima_arch.c
+++ /dev/null
@@ -1,14 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include <linux/ima.h>
-#include <asm/boot_data.h>
-
-bool arch_ima_get_secureboot(void)
-{
-	return ipl_secure_flag;
-}
-
-const char * const *arch_get_ima_policy(void)
-{
-	return NULL;
-}
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 049c557c452ff..3c346b02ceb95 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -21,6 +21,7 @@
 #include <linux/crash_dump.h>
 #include <linux/debug_locks.h>
 #include <linux/vmalloc.h>
+#include <linux/secure_boot.h>
 #include <asm/asm-extable.h>
 #include <asm/machine.h>
 #include <asm/diag.h>
@@ -2387,6 +2388,11 @@ void __no_stack_protector s390_reset_system(void)
 	diag_amode31_ops.diag308_reset();
 }
 
+bool arch_get_secureboot(void)
+{
+	return ipl_secure_flag;
+}
+
 #ifdef CONFIG_KEXEC_FILE
 
 int ipl_report_add_component(struct ipl_report *report, struct kexec_buf *kbuf,
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index a55abbf65333a..94fbfad49f620 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -487,8 +487,8 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
 	mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK);
 	if (test_cpu_flag(CIF_MCCK_GUEST) &&
 	(mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) {
-		/* Set exit reason code for host's later handling */
-		*((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR;
+		/* Set sie return code for host's later handling */
+		((struct stack_frame *)regs->gprs[15])->sie_return = SIE64_RETURN_MCCK;
 	}
 	clear_cpu_flag(CIF_MCCK_GUEST);
 
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 4630b2a067ea6..a9da9390867d3 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -1434,7 +1434,8 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
 	if (rc)
 		return rc;
 
-	pgste = pgste_get_lock(ptep_h);
+	if (!pgste_get_trylock(ptep_h, &pgste))
+		return -EAGAIN;
 	newpte = _pte(f->pfn, f->writable, !p, 0);
 	newpte.s.d |= ptep->s.d;
 	newpte.s.sd |= ptep->s.sd;
@@ -1444,7 +1445,8 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
 	pgste_set_unlock(ptep_h, pgste);
 
 	newpte = _pte(f->pfn, 0, !p, 0);
-	pgste = pgste_get_lock(ptep);
+	if (!pgste_get_trylock(ptep, &pgste))
+		return -EAGAIN;
 	pgste = __dat_ptep_xchg(ptep, pgste, newpte, gpa_to_gfn(raddr), sg->asce, uses_skeys(sg));
 	pgste_set_unlock(ptep, pgste);
 
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 18932a65ca683..7cb8ce833b625 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -2724,6 +2724,9 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
 
 	bit = bit_nr + (addr % PAGE_SIZE) * 8;
 
+	/* kvm_set_routing_entry() should never allow this to happen */
+	WARN_ON_ONCE(bit > (PAGE_SIZE * BITS_PER_BYTE - 1));
+
 	return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
 }
 
@@ -2824,6 +2827,12 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
 	int rc;
 
 	mci.val = mcck_info->mcic;
+
+	/* log machine checks being reinjected on all debugs */
+	VCPU_EVENT(vcpu, 2, "guest machine check %lx", mci.val);
+	KVM_EVENT(2, "guest machine check %lx", mci.val);
+	pr_info("guest machine check pid %d: %lx", current->pid, mci.val);
+
 	if (mci.sr)
 		cr14 |= CR14_RECOVERY_SUBMASK;
 	if (mci.dg)
@@ -2852,6 +2861,7 @@ int kvm_set_routing_entry(struct kvm *kvm,
 			  struct kvm_kernel_irq_routing_entry *e,
 			  const struct kvm_irq_routing_entry *ue)
 {
+	const struct kvm_irq_routing_s390_adapter *adapter;
 	u64 uaddr_s, uaddr_i;
 	int idx;
 
@@ -2862,6 +2872,14 @@ int kvm_set_routing_entry(struct kvm *kvm,
 			return -EINVAL;
 		e->set = set_adapter_int;
 
+		adapter = &ue->u.adapter;
+		if (adapter->summary_addr + (adapter->summary_offset / 8) >=
+		    (adapter->summary_addr & PAGE_MASK) + PAGE_SIZE)
+			return -EINVAL;
+		if (adapter->ind_addr + (adapter->ind_offset / 8) >=
+		    (adapter->ind_addr & PAGE_MASK) + PAGE_SIZE)
+			return -EINVAL;
+
 		idx = srcu_read_lock(&kvm->srcu);
 		uaddr_s = gpa_to_hva(kvm, ue->u.adapter.summary_addr);
 		uaddr_i = gpa_to_hva(kvm, ue->u.adapter.ind_addr);
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 3eb60aa932ecc..b2c01fa7b852c 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -4617,7 +4617,7 @@ static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
-static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
+static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return)
 {
 	struct mcck_volatile_info *mcck_info;
 	struct sie_page *sie_page;
@@ -4633,14 +4633,14 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
 	vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
 	vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
 
-	if (exit_reason == -EINTR) {
-		VCPU_EVENT(vcpu, 3, "%s", "machine check");
+	if (sie_return == SIE64_RETURN_MCCK) {
 		sie_page = container_of(vcpu->arch.sie_block,
 					struct sie_page, sie_block);
 		mcck_info = &sie_page->mcck_info;
 		kvm_s390_reinject_machine_check(vcpu, mcck_info);
 		return 0;
 	}
+	WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL);
 
 	if (vcpu->arch.sie_block->icptcode > 0) {
 		rc = kvm_handle_sie_intercept(vcpu);
@@ -4679,7 +4679,7 @@ int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
 static int __vcpu_run(struct kvm_vcpu *vcpu)
 {
-	int rc, exit_reason;
+	int rc, sie_return;
 	struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
 
 	/*
@@ -4719,9 +4719,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 		guest_timing_enter_irqoff();
 		__disable_cpu_timer_accounting(vcpu);
 
-		exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
-						      vcpu->run->s.regs.gprs,
-						      vcpu->arch.gmap->asce.val);
+		sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
+						     vcpu->run->s.regs.gprs,
+						     vcpu->arch.gmap->asce.val);
 
 		__enable_cpu_timer_accounting(vcpu);
 		guest_timing_exit_irqoff();
@@ -4744,7 +4744,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 		}
 		kvm_vcpu_srcu_read_lock(vcpu);
 
-		rc = vcpu_post_run(vcpu, exit_reason);
+		rc = vcpu_post_run(vcpu, sie_return);
 		if (rc || guestdbg_exit_pending(vcpu)) {
 			kvm_vcpu_srcu_read_unlock(vcpu);
 			break;
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index d249b10044eb7..0330829b4046c 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -1122,6 +1122,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+	unsigned long sie_return = SIE64_RETURN_NORMAL;
 	int guest_bp_isolation;
 	int rc = 0;
 
@@ -1163,7 +1164,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc
 			goto xfer_to_guest_mode_check;
 		}
 		guest_timing_enter_irqoff();
-		rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val);
+		sie_return = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val);
 		guest_timing_exit_irqoff();
 		local_irq_enable();
 	}
@@ -1178,12 +1179,13 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc
 
 	kvm_vcpu_srcu_read_lock(vcpu);
 
-	if (rc == -EINTR) {
-		VCPU_EVENT(vcpu, 3, "%s", "machine check");
+	if (sie_return == SIE64_RETURN_MCCK) {
 		kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info);
 		return 0;
 	}
 
+	WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL);
+
 	if (rc > 0)
 		rc = 0; /* we could still have an icpt */
 	else if (current->thread.gmap_int_code)
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index f43f897d3fc02..2bf47204f6abd 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -5,7 +5,7 @@
 
 lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o
 lib-y += csum-partial.o
-obj-y += mem.o xor.o
+obj-y += mem.o
 lib-$(CONFIG_KPROBES) += probes.o
 lib-$(CONFIG_UPROBES) += probes.o
 obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c
deleted file mode 100644
index 5363e4c2462d1..0000000000000
--- a/arch/s390/lib/xor.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Optimized xor_block operation for RAID4/5
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- */
-
-#include <linux/types.h>
-#include <linux/export.h>
-#include <linux/raid/xor.h>
-#include <asm/xor.h>
-
-static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2)
-{
-	asm volatile(
-		"	aghi	%0,-1\n"
-		"	jm	3f\n"
-		"	srlg	0,%0,8\n"
-		"	ltgr	0,0\n"
-		"	jz	1f\n"
-		"0:	xc	0(256,%1),0(%2)\n"
-		"	la	%1,256(%1)\n"
-		"	la	%2,256(%2)\n"
-		"	brctg	0,0b\n"
-		"1:	exrl	%0,2f\n"
-		"	j	3f\n"
-		"2:	xc	0(1,%1),0(%2)\n"
-		"3:"
-		: "+a" (bytes), "+a" (p1), "+a" (p2)
-		: : "0", "cc", "memory");
-}
-
-static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3)
-{
-	asm volatile(
-		"	aghi	%0,-1\n"
-		"	jm	4f\n"
-		"	srlg	0,%0,8\n"
-		"	ltgr	0,0\n"
-		"	jz	1f\n"
-		"0:	xc	0(256,%1),0(%2)\n"
-		"	xc	0(256,%1),0(%3)\n"
-		"	la	%1,256(%1)\n"
-		"	la	%2,256(%2)\n"
-		"	la	%3,256(%3)\n"
-		"	brctg	0,0b\n"
-		"1:	exrl	%0,2f\n"
-		"	exrl	%0,3f\n"
-		"	j	4f\n"
-		"2:	xc	0(1,%1),0(%2)\n"
-		"3:	xc	0(1,%1),0(%3)\n"
-		"4:"
-		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
-		: : "0", "cc", "memory");
-}
-
-static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4)
-{
-	asm volatile(
-		"	aghi	%0,-1\n"
-		"	jm	5f\n"
-		"	srlg	0,%0,8\n"
-		"	ltgr	0,0\n"
-		"	jz	1f\n"
-		"0:	xc	0(256,%1),0(%2)\n"
-		"	xc	0(256,%1),0(%3)\n"
-		"	xc	0(256,%1),0(%4)\n"
-		"	la	%1,256(%1)\n"
-		"	la	%2,256(%2)\n"
-		"	la	%3,256(%3)\n"
-		"	la	%4,256(%4)\n"
-		"	brctg	0,0b\n"
-		"1:	exrl	%0,2f\n"
-		"	exrl	%0,3f\n"
-		"	exrl	%0,4f\n"
-		"	j	5f\n"
-		"2:	xc	0(1,%1),0(%2)\n"
-		"3:	xc	0(1,%1),0(%3)\n"
-		"4:	xc	0(1,%1),0(%4)\n"
-		"5:"
-		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
-		: : "0", "cc", "memory");
-}
-
-static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
-		     const unsigned long * __restrict p2,
-		     const unsigned long * __restrict p3,
-		     const unsigned long * __restrict p4,
-		     const unsigned long * __restrict p5)
-{
-	asm volatile(
-		"	aghi	%0,-1\n"
-		"	jm	6f\n"
-		"	srlg	0,%0,8\n"
-		"	ltgr	0,0\n"
-		"	jz	1f\n"
-		"0:	xc	0(256,%1),0(%2)\n"
-		"	xc	0(256,%1),0(%3)\n"
-		"	xc	0(256,%1),0(%4)\n"
-		"	xc	0(256,%1),0(%5)\n"
-		"	la	%1,256(%1)\n"
-		"	la	%2,256(%2)\n"
-		"	la	%3,256(%3)\n"
-		"	la	%4,256(%4)\n"
-		"	la	%5,256(%5)\n"
-		"	brctg	0,0b\n"
-		"1:	exrl	%0,2f\n"
-		"	exrl	%0,3f\n"
-		"	exrl	%0,4f\n"
-		"	exrl	%0,5f\n"
-		"	j	6f\n"
-		"2:	xc	0(1,%1),0(%2)\n"
-		"3:	xc	0(1,%1),0(%3)\n"
-		"4:	xc	0(1,%1),0(%4)\n"
-		"5:	xc	0(1,%1),0(%5)\n"
-		"6:"
-		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
-		  "+a" (p5)
-		: : "0", "cc", "memory");
-}
-
-struct xor_block_template xor_block_xc = {
-	.name = "xc",
-	.do_2 = xor_xc_2,
-	.do_3 = xor_xc_3,
-	.do_4 = xor_xc_4,
-	.do_5 = xor_xc_5,
-};
-EXPORT_SYMBOL(xor_block_xc);
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index a52aa7a99b6bf..191cc53caead3 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -441,10 +441,17 @@ void do_secure_storage_access(struct pt_regs *regs)
 		folio = phys_to_folio(addr);
 		if (unlikely(!folio_try_get(folio)))
 			return;
-		rc = arch_make_folio_accessible(folio);
+		rc = uv_convert_from_secure(folio_to_phys(folio));
+		if (!rc)
+			clear_bit(PG_arch_1, &folio->flags.f);
 		folio_put(folio);
+		/*
+		 * There are some valid fixup types for kernel
+		 * accesses to donated secure memory. zeropad is one
+		 * of them.
+		 */
 		if (rc)
-			BUG();
+			return handle_fault_error_nolock(regs, 0);
 	} else {
 		if (faulthandler_disabled())
 			return handle_fault_error_nolock(regs, 0);
diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c
index dea83e3103e5d..f8789ffcc05ca 100644
--- a/arch/s390/mm/gmap_helpers.c
+++ b/arch/s390/mm/gmap_helpers.c
@@ -89,7 +89,7 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo
 		if (!vma)
 			return;
 		if (!is_vm_hugetlb_page(vma))
-			zap_page_range_single(vma, vmaddr, min(end, vma->vm_end) - vmaddr, NULL);
+			zap_vma_range(vma, vmaddr, min(end, vma->vm_end) - vmaddr);
 		vmaddr = vma->vm_end;
 	}
 }
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 3c20475cbee22..1f72efc2a579f 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -69,7 +69,7 @@ unsigned long empty_zero_page, zero_page_mask;
 EXPORT_SYMBOL(empty_zero_page);
 EXPORT_SYMBOL(zero_page_mask);
 
-static void __init setup_zero_pages(void)
+void __init arch_setup_zero_pages(void)
 {
 	unsigned long total_pages = memblock_estimated_nr_free_pages();
 	unsigned int order;
@@ -159,8 +159,6 @@ void __init arch_mm_preinit(void)
 	cpumask_set_cpu(0, mm_cpumask(&init_mm));
 
 	pv_init();
-
-	setup_zero_pages();	/* Setup zeroed pages. */
 }
 
 unsigned long memory_block_size_bytes(void)
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index bf92964246eb1..d08d159b63193 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -830,25 +830,34 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp,
 }
 
 /*
- * Sign-extend the register if necessary
+ * Sign- or zero-extend the register if necessary
  */
-static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
+static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
 {
-	if (!(flags & BTF_FMODEL_SIGNED_ARG))
-		return 0;
-
 	switch (size) {
 	case 1:
-		/* lgbr %r,%r */
-		EMIT4(0xb9060000, r, r);
+		if (flags & BTF_FMODEL_SIGNED_ARG)
+			/* lgbr %r,%r */
+			EMIT4(0xb9060000, r, r);
+		else
+			/* llgcr %r,%r */
+			EMIT4(0xb9840000, r, r);
 		return 0;
 	case 2:
-		/* lghr %r,%r */
-		EMIT4(0xb9070000, r, r);
+		if (flags & BTF_FMODEL_SIGNED_ARG)
+			/* lghr %r,%r */
+			EMIT4(0xb9070000, r, r);
+		else
+			/* llghr %r,%r */
+			EMIT4(0xb9850000, r, r);
 		return 0;
 	case 4:
-		/* lgfr %r,%r */
-		EMIT4(0xb9140000, r, r);
+		if (flags & BTF_FMODEL_SIGNED_ARG)
+			/* lgfr %r,%r */
+			EMIT4(0xb9140000, r, r);
+		else
+			/* llgfr %r,%r */
+			EMIT4(0xb9160000, r, r);
 		return 0;
 	case 8:
 		return 0;
@@ -1798,9 +1807,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 				return -1;
 
 			for (j = 0; j < m->nr_args; j++) {
-				if (sign_extend(jit, BPF_REG_1 + j,
-						m->arg_size[j],
-						m->arg_flags[j]))
+				if (sign_zero_extend(jit, BPF_REG_1 + j,
+						     m->arg_size[j],
+						     m->arg_flags[j]))
 					return -1;
 			}
 		}
@@ -1862,20 +1871,21 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 				 jit->prg);
 
 		/*
-		 * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT)
+		 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
 		 *         goto out;
+		 *
+		 * tail_call_cnt is read into %w0, which needs to be preserved
+		 * until it's incremented and flushed.
 		 */
 
 		off = jit->frame_off +
 		      offsetof(struct prog_frame, tail_call_cnt);
-		/* lhi %w0,1 */
-		EMIT4_IMM(0xa7080000, REG_W0, 1);
-		/* laal %w1,%w0,off(%r15) */
-		EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
-		/* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */
+		/* ly %w0,off(%r15) */
+		EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off);
+		/* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */
 		patch_2_clij = jit->prg;
-		EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1,
-				 2, jit->prg);
+		EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT,
+				 0xa, jit->prg);
 
 		/*
 		 * prog = array->ptrs[index];
@@ -1894,6 +1904,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 		patch_3_brc = jit->prg;
 		EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
 
+		/* tail_call_cnt++; */
+		/* ahi %w0,1 */
+		EMIT4_IMM(0xa70a0000, REG_W0, 1);
+		/* sty %w0,off(%r15) */
+		EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off);
+
 		/*
 		 * Restore registers before calling function
 		 */
@@ -2480,8 +2496,8 @@ struct bpf_tramp_jit {
 	int ip_off;		/* For bpf_get_func_ip(), has to be at
 				 * (ctx - 16)
 				 */
-	int arg_cnt_off;	/* For bpf_get_func_arg_cnt(), has to be at
-				 * (ctx - 8)
+	int func_meta_off;	/* For bpf_get_func_arg_cnt()/fsession, has
+				 * to be at (ctx - 8)
 				 */
 	int bpf_args_off;	/* Offset of BPF_PROG context, which consists
 				 * of BPF arguments followed by return value
@@ -2506,6 +2522,13 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
 	EMIT6_IMM(0xc00d0000, dst_reg, val);
 }
 
+static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm)
+{
+	load_imm64(jit, tmp_reg, imm);
+	/* stg %tmp_reg,stack_off(%r15) */
+	EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off);
+}
+
 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 			   const struct btf_func_model *m,
 			   struct bpf_tramp_link *tlink, bool save_ret)
@@ -2520,10 +2543,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 	 * run_ctx.cookie = tlink->cookie;
 	 */
 
-	/* %r0 = tlink->cookie */
-	load_imm64(jit, REG_W0, tlink->cookie);
-	/* stg %r0,cookie_off(%r15) */
-	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off);
+	emit_store_stack_imm64(jit, REG_W0, cookie_off, tlink->cookie);
 
 	/*
 	 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0)
@@ -2555,7 +2575,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 	EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func);
 	/* stg %r2,retval_off(%r15) */
 	if (save_ret) {
-		if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags))
+		if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags))
 			return -1;
 		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15,
 			      tjit->retval_off);
@@ -2581,6 +2601,28 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 	return 0;
 }
 
+static int invoke_bpf(struct bpf_tramp_jit *tjit,
+		      const struct btf_func_model *m,
+		      struct bpf_tramp_links *tl, bool save_ret,
+		      u64 func_meta, int cookie_off)
+{
+	int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64);
+	struct bpf_jit *jit = &tjit->common;
+
+	for (i = 0; i < tl->nr_links; i++) {
+		if (bpf_prog_calls_session_cookie(tl->links[i])) {
+			u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
+
+			emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta);
+			cur_cookie--;
+		}
+		if (invoke_bpf_prog(tjit, m, tl->links[i], save_ret))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size)
 {
 	int stack_offset = tjit->stack_size;
@@ -2610,8 +2652,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
 	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
 	int nr_bpf_args, nr_reg_args, nr_stack_args;
+	int cookie_cnt, cookie_off, fsession_cnt;
 	struct bpf_jit *jit = &tjit->common;
 	int arg, bpf_arg_off;
+	u64 func_meta;
 	int i, j;
 
 	/* Support as many stack arguments as "mvc" instruction can handle. */
@@ -2643,6 +2687,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 			return -ENOTSUPP;
 	}
 
+	cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
+	fsession_cnt = bpf_fsession_cnt(tlinks);
+
 	/*
 	 * Calculate the stack layout.
 	 */
@@ -2655,8 +2702,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	tjit->backchain_off = tjit->stack_size - sizeof(u64);
 	tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64));
 	tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64));
+	cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64));
 	tjit->ip_off = alloc_stack(tjit, sizeof(u64));
-	tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64));
+	tjit->func_meta_off = alloc_stack(tjit, sizeof(u64));
 	tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64));
 	tjit->retval_off = alloc_stack(tjit, sizeof(u64));
 	tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64));
@@ -2743,18 +2791,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	 * arg_cnt = m->nr_args;
 	 */
 
-	if (flags & BPF_TRAMP_F_IP_ARG) {
-		/* %r0 = func_addr */
-		load_imm64(jit, REG_0, (u64)func_addr);
-		/* stg %r0,ip_off(%r15) */
-		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
-			      tjit->ip_off);
-	}
-	/* lghi %r0,nr_bpf_args */
-	EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args);
-	/* stg %r0,arg_cnt_off(%r15) */
+	if (flags & BPF_TRAMP_F_IP_ARG)
+		emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr);
+	func_meta = nr_bpf_args;
+	/* lghi %r0,func_meta */
+	EMIT4_IMM(0xa7090000, REG_0, func_meta);
+	/* stg %r0,func_meta_off(%r15) */
 	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
-		      tjit->arg_cnt_off);
+		      tjit->func_meta_off);
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/*
@@ -2767,10 +2811,17 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter);
 	}
 
-	for (i = 0; i < fentry->nr_links; i++)
-		if (invoke_bpf_prog(tjit, m, fentry->links[i],
-				    flags & BPF_TRAMP_F_RET_FENTRY_RET))
-			return -EINVAL;
+	if (fsession_cnt) {
+		/* Clear all the session cookies' value. */
+		for (i = 0; i < cookie_cnt; i++)
+			emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0);
+		/* Clear the return value to make sure fentry always gets 0. */
+		emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0);
+	}
+
+	if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET,
+		       func_meta, cookie_off))
+		return -EINVAL;
 
 	if (fmod_ret->nr_links) {
 		/*
@@ -2847,11 +2898,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue);
 	}
 
+	/* Set the "is_return" flag for fsession. */
+	func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
+	if (fsession_cnt)
+		emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off,
+				       func_meta);
+
 	/* do_fexit: */
 	tjit->do_fexit = jit->prg;
-	for (i = 0; i < fexit->nr_links; i++)
-		if (invoke_bpf_prog(tjit, m, fexit->links[i], false))
-			return -EINVAL;
+	if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off))
+		return -EINVAL;
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		im->ip_epilogue = jit->prg_buf + jit->prg;
@@ -2956,6 +3012,11 @@ bool bpf_jit_supports_arena(void)
 	return true;
 }
 
+bool bpf_jit_supports_fsession(void)
+{
+	return true;
+}
+
 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 {
 	if (!in_arena)
diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile
index 61d240a37633d..95a8ac45b67e6 100644
--- a/arch/s390/purgatory/Makefile
+++ b/arch/s390/purgatory/Makefile
@@ -13,7 +13,7 @@ CFLAGS_sha256.o := -D__NO_FORTIFY
 $(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE
 	$(call if_changed_rule,as_o_S)
 
-KBUILD_CFLAGS := -std=gnu11 -fms-extensions -fno-strict-aliasing -Wall -Wstrict-prototypes
+KBUILD_CFLAGS := $(CC_FLAGS_DIALECT) -fno-strict-aliasing -Wall -Wstrict-prototypes
 KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare
 KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding
 KBUILD_CFLAGS += -Os -m64 -msoft-float -fno-common
@@ -21,7 +21,6 @@ KBUILD_CFLAGS += -fno-stack-protector
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
 KBUILD_CFLAGS += -D__DISABLE_EXPORTS
 KBUILD_CFLAGS += $(CLANG_FLAGS)
-KBUILD_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag)
 KBUILD_CFLAGS += $(call cc-option,-fno-PIE)
 KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe)
 KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS))
diff --git a/arch/sh/drivers/platform_early.c b/arch/sh/drivers/platform_early.c
index 143747c45206f..48ddbc547bd9a 100644
--- a/arch/sh/drivers/platform_early.c
+++ b/arch/sh/drivers/platform_early.c
@@ -26,10 +26,6 @@ static int platform_match(struct device *dev, struct device_driver *drv)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct platform_driver *pdrv = to_platform_driver(drv);
 
-	/* When driver_override is set, only bind to the matching driver */
-	if (pdev->driver_override)
-		return !strcmp(pdev->driver_override, drv->name);
-
 	/* Then try to match against the id table */
 	if (pdrv->id_table)
 		return platform_match_id(pdrv->id_table, pdev) != NULL;
diff --git a/arch/sh/include/asm/pgtable.h b/arch/sh/include/asm/pgtable.h
index 10fa8f2bb8d1f..d5ce0950a3231 100644
--- a/arch/sh/include/asm/pgtable.h
+++ b/arch/sh/include/asm/pgtable.h
@@ -20,14 +20,6 @@
 #ifndef __ASSEMBLER__
 #include <asm/addrspace.h>
 #include <asm/fixmap.h>
-
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 #endif /* !__ASSEMBLER__ */
 
 /*
diff --git a/arch/sh/include/asm/setup.h b/arch/sh/include/asm/setup.h
index 84bb23a771f35..63c9efc063486 100644
--- a/arch/sh/include/asm/setup.h
+++ b/arch/sh/include/asm/setup.h
@@ -7,7 +7,8 @@
 /*
  * This is set up by the setup-routine at boot-time
  */
-#define PARAM	((unsigned char *)empty_zero_page)
+extern unsigned char *boot_params_page;
+#define PARAM boot_params_page
 
 #define MOUNT_ROOT_RDONLY (*(unsigned long *) (PARAM+0x000))
 #define RAMDISK_FLAGS (*(unsigned long *) (PARAM+0x004))
diff --git a/arch/sh/kernel/head_32.S b/arch/sh/kernel/head_32.S
index b603b7968b388..0b91bb85d40a0 100644
--- a/arch/sh/kernel/head_32.S
+++ b/arch/sh/kernel/head_32.S
@@ -26,7 +26,7 @@
 #endif
 
 	.section	.empty_zero_page, "aw"
-ENTRY(empty_zero_page)
+ENTRY(boot_params_page)
 	.long	1		/* MOUNT_ROOT_RDONLY */
 	.long	0		/* RAMDISK_FLAGS */
 	.long	0x0200		/* ORIG_ROOT_DEV */
@@ -39,7 +39,7 @@ ENTRY(empty_zero_page)
 	.long	0x53453f00 + 29	/* "SE?" = 29 bit */
 #endif
 1:
-	.skip	PAGE_SIZE - empty_zero_page - 1b
+	.skip	PAGE_SIZE - boot_params_page - 1b
 
 	__HEAD
 
diff --git a/arch/sh/kernel/sh_ksyms_32.c b/arch/sh/kernel/sh_ksyms_32.c
index 5858936cb4311..041191002e2e2 100644
--- a/arch/sh/kernel/sh_ksyms_32.c
+++ b/arch/sh/kernel/sh_ksyms_32.c
@@ -20,7 +20,6 @@ EXPORT_SYMBOL(csum_partial);
 EXPORT_SYMBOL(csum_partial_copy_generic);
 EXPORT_SYMBOL(copy_page);
 EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(empty_zero_page);
 #ifdef CONFIG_FLATMEM
 /* need in pfn_valid macro */
 EXPORT_SYMBOL(min_low_pfn);
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index 464a3a63e2fa5..4e40d5e96be9d 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -332,7 +332,6 @@ void __init mem_init(void)
 	cpu_cache_init();
 
 	/* clear the zero-page */
-	memset(empty_zero_page, 0, PAGE_SIZE);
 	__flush_wback_region(empty_zero_page, PAGE_SIZE);
 
 	vsyscall_init();
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 8699be91fca9b..8793a99ab09e4 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -50,6 +50,7 @@ config SPARC
 	select NEED_DMA_MAP_STATE
 	select NEED_SG_DMA_LENGTH
 	select TRACE_IRQFLAGS_SUPPORT
+	select XOR_BLOCKS_ARCH
 
 config SPARC32
 	def_bool !64BIT
@@ -104,7 +105,6 @@ config SPARC64
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS
 	select GENERIC_TIME_VSYSCALL
-	select ARCH_CLOCKSOURCE_DATA
 	select ARCH_HAS_PTE_SPECIAL
 	select PCI_DOMAINS if PCI
 	select ARCH_HAS_GIGANTIC_PAGE
@@ -115,6 +115,8 @@ config SPARC64
 	select ARCH_SUPPORTS_SCHED_SMT if SMP
 	select ARCH_SUPPORTS_SCHED_MC  if SMP
 	select ARCH_HAS_LAZY_MMU_MODE
+	select HAVE_GENERIC_VDSO
+	select GENERIC_GETTIMEOFDAY
 
 config ARCH_PROC_KCORE_TEXT
 	def_bool y
diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h
index 08810808ca6de..bbd1a8afaabf8 100644
--- a/arch/sparc/include/asm/asm-prototypes.h
+++ b/arch/sparc/include/asm/asm-prototypes.h
@@ -14,7 +14,6 @@
 #include <asm/oplib.h>
 #include <asm/pgtable.h>
 #include <asm/trap_block.h>
-#include <asm/xor.h>
 
 void *__memscan_zero(void *, size_t);
 void *__memscan_generic(void *, int, size_t);
diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h
index d63ef224befeb..68303ad26eb26 100644
--- a/arch/sparc/include/asm/clocksource.h
+++ b/arch/sparc/include/asm/clocksource.h
@@ -5,13 +5,4 @@
 #ifndef _ASM_SPARC_CLOCKSOURCE_H
 #define _ASM_SPARC_CLOCKSOURCE_H
 
-/* VDSO clocksources */
-#define VCLOCK_NONE   0  /* Nothing userspace can do. */
-#define VCLOCK_TICK   1  /* Use %tick.  */
-#define VCLOCK_STICK  2  /* Use %stick. */
-
-struct arch_clocksource_data {
-	int vclock_mode;
-};
-
 #endif /* _ASM_SPARC_CLOCKSOURCE_H */
diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h
index a9f802d1dd64f..f89b1250661df 100644
--- a/arch/sparc/include/asm/pgtable_32.h
+++ b/arch/sparc/include/asm/pgtable_32.h
@@ -71,14 +71,6 @@ extern unsigned long ptr_in_current_pgd;
 extern unsigned long phys_base;
 extern unsigned long pfn_base;
 
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 /*
  * In general all page table modifications should use the V8 atomic
  * swap instruction.  This insures the mmu and the cpu are in sync
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 615f460c50af4..74ede706fb325 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -210,9 +210,6 @@ extern unsigned long _PAGE_CACHE;
 extern unsigned long pg_iobits;
 extern unsigned long _PAGE_ALL_SZ_BITS;
 
-extern struct page *mem_map_zero;
-#define ZERO_PAGE(vaddr)	(mem_map_zero)
-
 /* PFNs are real physical page numbers.  However, mem_map only begins to record
  * per-page information starting at pfn_base.  This is to handle systems where
  * the first physical page in the machine is at some huge physical address,
diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h
index 18295ea625dd7..e34de956519aa 100644
--- a/arch/sparc/include/asm/processor.h
+++ b/arch/sparc/include/asm/processor.h
@@ -1,6 +1,9 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef ___ASM_SPARC_PROCESSOR_H
 #define ___ASM_SPARC_PROCESSOR_H
+
+#include <asm/vdso/processor.h>
+
 #if defined(__sparc__) && defined(__arch64__)
 #include <asm/processor_64.h>
 #else
diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h
index ba8b70ffec085..a074d313f4f80 100644
--- a/arch/sparc/include/asm/processor_32.h
+++ b/arch/sparc/include/asm/processor_32.h
@@ -91,8 +91,6 @@ unsigned long __get_wchan(struct task_struct *);
 extern struct task_struct *last_task_used_math;
 int do_mathemu(struct pt_regs *regs, struct task_struct *fpt);
 
-#define cpu_relax()	barrier()
-
 extern void (*sparc_idle)(void);
 
 #endif
diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h
index 321859454ca4c..4850704952634 100644
--- a/arch/sparc/include/asm/processor_64.h
+++ b/arch/sparc/include/asm/processor_64.h
@@ -182,31 +182,6 @@ unsigned long __get_wchan(struct task_struct *task);
 #define KSTK_EIP(tsk)  (task_pt_regs(tsk)->tpc)
 #define KSTK_ESP(tsk)  (task_pt_regs(tsk)->u_regs[UREG_FP])
 
-/* Please see the commentary in asm/backoff.h for a description of
- * what these instructions are doing and how they have been chosen.
- * To make a long story short, we are trying to yield the current cpu
- * strand during busy loops.
- */
-#ifdef	BUILD_VDSO
-#define	cpu_relax()	asm volatile("\n99:\n\t"			\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     ::: "memory")
-#else /* ! BUILD_VDSO */
-#define cpu_relax()	asm volatile("\n99:\n\t"			\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     "rd	%%ccr, %%g0\n\t"	\
-				     ".section	.pause_3insn_patch,\"ax\"\n\t"\
-				     ".word	99b\n\t"		\
-				     "wr	%%g0, 128, %%asr27\n\t"	\
-				     "nop\n\t"				\
-				     "nop\n\t"				\
-				     ".previous"			\
-				     ::: "memory")
-#endif
-
 /* Prefetch support.  This is tuned for UltraSPARC-III and later.
  * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
  * a shallower prefetch queue than later chips.
diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h
index 72205684e51ea..21bed55140288 100644
--- a/arch/sparc/include/asm/setup.h
+++ b/arch/sparc/include/asm/setup.h
@@ -17,8 +17,6 @@ extern char reboot_command[];
  */
 extern unsigned char boot_cpu_id;
 
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
-
 extern int serial_console;
 static inline int con_is_present(void)
 {
diff --git a/arch/sparc/include/asm/vdso.h b/arch/sparc/include/asm/vdso.h
index 59e79d35cd738..f08562d10215b 100644
--- a/arch/sparc/include/asm/vdso.h
+++ b/arch/sparc/include/asm/vdso.h
@@ -8,8 +8,6 @@
 struct vdso_image {
 	void *data;
 	unsigned long size;   /* Always a multiple of PAGE_SIZE */
-
-	long sym_vvar_start;  /* Negative offset to the vvar area */
 };
 
 #ifdef CONFIG_SPARC64
diff --git a/arch/sparc/include/asm/vdso/clocksource.h b/arch/sparc/include/asm/vdso/clocksource.h
new file mode 100644
index 0000000000000..007aa8ceaf522
--- /dev/null
+++ b/arch/sparc/include/asm/vdso/clocksource.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_VDSO_CLOCKSOURCE_H
+#define __ASM_VDSO_CLOCKSOURCE_H
+
+/* VDSO clocksources */
+#define VDSO_ARCH_CLOCKMODES	\
+	VDSO_CLOCKMODE_TICK,	\
+	VDSO_CLOCKMODE_STICK
+
+#endif /* __ASM_VDSO_CLOCKSOURCE_H */
diff --git a/arch/sparc/include/asm/vdso/gettimeofday.h b/arch/sparc/include/asm/vdso/gettimeofday.h
new file mode 100644
index 0000000000000..b0c80c8a28bb7
--- /dev/null
+++ b/arch/sparc/include/asm/vdso/gettimeofday.h
@@ -0,0 +1,184 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2006 Andi Kleen, SUSE Labs.
+ */
+
+#ifndef _ASM_SPARC_VDSO_GETTIMEOFDAY_H
+#define _ASM_SPARC_VDSO_GETTIMEOFDAY_H
+
+#include <uapi/linux/time.h>
+#include <uapi/linux/unistd.h>
+
+#include <vdso/align.h>
+#include <vdso/clocksource.h>
+#include <vdso/datapage.h>
+#include <vdso/page.h>
+
+#include <linux/types.h>
+
+#ifdef	CONFIG_SPARC64
+static __always_inline u64 vread_tick(void)
+{
+	u64	ret;
+
+	__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
+	return ret;
+}
+
+static __always_inline u64 vread_tick_stick(void)
+{
+	u64	ret;
+
+	__asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
+	return ret;
+}
+#else
+static __always_inline u64 vdso_shift_ns(u64 val, u32 amt)
+{
+	u64 ret;
+
+	__asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
+			     "srl %L1, 0, %L1\n\t"
+			     "or %%g1, %L1, %%g1\n\t"
+			     "srlx %%g1, %2, %L0\n\t"
+			     "srlx %L0, 32, %H0"
+			     : "=r" (ret)
+			     : "r" (val), "r" (amt)
+			     : "g1");
+	return ret;
+}
+#define vdso_shift_ns vdso_shift_ns
+
+static __always_inline u64 vread_tick(void)
+{
+	register unsigned long long ret asm("o4");
+
+	__asm__ __volatile__("rd %%tick, %L0\n\t"
+			     "srlx %L0, 32, %H0"
+			     : "=r" (ret));
+	return ret;
+}
+
+static __always_inline u64 vread_tick_stick(void)
+{
+	register unsigned long long ret asm("o4");
+
+	__asm__ __volatile__("rd %%asr24, %L0\n\t"
+			     "srlx %L0, 32, %H0"
+			     : "=r" (ret));
+	return ret;
+}
+#endif
+
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd)
+{
+	if (likely(clock_mode == VDSO_CLOCKMODE_STICK))
+		return vread_tick_stick();
+	else
+		return vread_tick();
+}
+
+#ifdef	CONFIG_SPARC64
+#define SYSCALL_STRING							\
+	"ta	0x6d;"							\
+	"bcs,a	1f;"							\
+	" sub	%%g0, %%o0, %%o0;"					\
+	"1:"
+#else
+#define SYSCALL_STRING							\
+	"ta	0x10;"							\
+	"bcs,a	1f;"							\
+	" sub	%%g0, %%o0, %%o0;"					\
+	"1:"
+#endif
+
+#define SYSCALL_CLOBBERS						\
+	"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",			\
+	"f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",		\
+	"f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",		\
+	"f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",		\
+	"f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",		\
+	"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",		\
+	"cc", "memory"
+
+#ifdef CONFIG_SPARC64
+
+static __always_inline
+long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts)
+{
+	register long num __asm__("g1") = __NR_clock_gettime;
+	register long o0 __asm__("o0") = clock;
+	register long o1 __asm__("o1") = (long) ts;
+
+	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+	return o0;
+}
+
+#else /* !CONFIG_SPARC64 */
+
+static __always_inline
+long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts)
+{
+	register long num __asm__("g1") = __NR_clock_gettime64;
+	register long o0 __asm__("o0") = clock;
+	register long o1 __asm__("o1") = (long) ts;
+
+	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+	return o0;
+}
+
+static __always_inline
+long clock_gettime32_fallback(clockid_t clock, struct old_timespec32 *ts)
+{
+	register long num __asm__("g1") = __NR_clock_gettime;
+	register long o0 __asm__("o0") = clock;
+	register long o1 __asm__("o1") = (long) ts;
+
+	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+	return o0;
+}
+
+#endif /* CONFIG_SPARC64 */
+
+static __always_inline
+long gettimeofday_fallback(struct __kernel_old_timeval *tv, struct timezone *tz)
+{
+	register long num __asm__("g1") = __NR_gettimeofday;
+	register long o0 __asm__("o0") = (long) tv;
+	register long o1 __asm__("o1") = (long) tz;
+
+	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+	return o0;
+}
+
+static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
+{
+	unsigned long ret;
+
+	/*
+	 * SPARC does not support native PC-relative code relocations.
+	 * Calculate the address manually, works for 32 and 64 bit code.
+	 */
+	__asm__ __volatile__(
+		"1:\n"
+		"call 3f\n"                     // Jump over the embedded data and set up %o7
+		"nop\n"                         // Delay slot
+		"2:\n"
+		".word vdso_u_time_data - .\n"  // Embedded offset to external symbol
+		"3:\n"
+		"add %%o7, 2b - 1b, %%o7\n"     // Point %o7 to the embedded offset
+		"ldsw [%%o7], %0\n"             // Load the offset
+		"add %0, %%o7, %0\n"            // Calculate the absolute address
+		: "=r" (ret)
+		:
+		: "o7");
+
+	return (const struct vdso_time_data *)ret;
+}
+#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data
+
+#endif /* _ASM_SPARC_VDSO_GETTIMEOFDAY_H */
diff --git a/arch/sparc/include/asm/vdso/processor.h b/arch/sparc/include/asm/vdso/processor.h
new file mode 100644
index 0000000000000..f7a9adc807f7c
--- /dev/null
+++ b/arch/sparc/include/asm/vdso/processor.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_SPARC_VDSO_PROCESSOR_H
+#define _ASM_SPARC_VDSO_PROCESSOR_H
+
+#include <linux/compiler.h>
+
+#if defined(__arch64__)
+
+/* Please see the commentary in asm/backoff.h for a description of
+ * what these instructions are doing and how they have been chosen.
+ * To make a long story short, we are trying to yield the current cpu
+ * strand during busy loops.
+ */
+#ifdef	BUILD_VDSO
+#define	cpu_relax()	asm volatile("\n99:\n\t"			\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     ::: "memory")
+#else /* ! BUILD_VDSO */
+#define cpu_relax()	asm volatile("\n99:\n\t"			\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     "rd	%%ccr, %%g0\n\t"	\
+				     ".section	.pause_3insn_patch,\"ax\"\n\t"\
+				     ".word	99b\n\t"		\
+				     "wr	%%g0, 128, %%asr27\n\t"	\
+				     "nop\n\t"				\
+				     "nop\n\t"				\
+				     ".previous"			\
+				     ::: "memory")
+#endif /* BUILD_VDSO */
+
+#else /* ! __arch64__ */
+
+#define cpu_relax()	barrier()
+
+#endif /* __arch64__ */
+
+#endif /* _ASM_SPARC_VDSO_PROCESSOR_H */
diff --git a/arch/sparc/include/asm/vdso/vsyscall.h b/arch/sparc/include/asm/vdso/vsyscall.h
new file mode 100644
index 0000000000000..8bfe703fedc56
--- /dev/null
+++ b/arch/sparc/include/asm/vdso/vsyscall.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_SPARC_VDSO_VSYSCALL_H
+#define _ASM_SPARC_VDSO_VSYSCALL_H
+
+#define __VDSO_PAGES 4
+
+#include <asm-generic/vdso/vsyscall.h>
+
+#endif /* _ASM_SPARC_VDSO_VSYSCALL_H */
diff --git a/arch/sparc/include/asm/vvar.h b/arch/sparc/include/asm/vvar.h
deleted file mode 100644
index 6eaf5cfcaae13..0000000000000
--- a/arch/sparc/include/asm/vvar.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
- */
-
-#ifndef _ASM_SPARC_VVAR_DATA_H
-#define _ASM_SPARC_VVAR_DATA_H
-
-#include <asm/clocksource.h>
-#include <asm/processor.h>
-#include <asm/barrier.h>
-#include <linux/time.h>
-#include <linux/types.h>
-
-struct vvar_data {
-	unsigned int seq;
-
-	int vclock_mode;
-	struct { /* extract of a clocksource struct */
-		u64	cycle_last;
-		u64	mask;
-		int	mult;
-		int	shift;
-	} clock;
-	/* open coded 'struct timespec' */
-	u64		wall_time_sec;
-	u64		wall_time_snsec;
-	u64		monotonic_time_snsec;
-	u64		monotonic_time_sec;
-	u64		monotonic_time_coarse_sec;
-	u64		monotonic_time_coarse_nsec;
-	u64		wall_time_coarse_sec;
-	u64		wall_time_coarse_nsec;
-
-	int		tz_minuteswest;
-	int		tz_dsttime;
-};
-
-extern struct vvar_data *vvar_data;
-extern int vdso_fix_stick;
-
-static inline unsigned int vvar_read_begin(const struct vvar_data *s)
-{
-	unsigned int ret;
-
-repeat:
-	ret = READ_ONCE(s->seq);
-	if (unlikely(ret & 1)) {
-		cpu_relax();
-		goto repeat;
-	}
-	smp_rmb(); /* Finish all reads before we return seq */
-	return ret;
-}
-
-static inline int vvar_read_retry(const struct vvar_data *s,
-					unsigned int start)
-{
-	smp_rmb(); /* Finish all reads before checking the value of seq */
-	return unlikely(s->seq != start);
-}
-
-static inline void vvar_write_begin(struct vvar_data *s)
-{
-	++s->seq;
-	smp_wmb(); /* Makes sure that increment of seq is reflected */
-}
-
-static inline void vvar_write_end(struct vvar_data *s)
-{
-	smp_wmb(); /* Makes the value of seq current before we increment */
-	++s->seq;
-}
-
-
-#endif /* _ASM_SPARC_VVAR_DATA_H */
diff --git a/arch/sparc/include/asm/xor.h b/arch/sparc/include/asm/xor.h
deleted file mode 100644
index f4c651e203c43..0000000000000
--- a/arch/sparc/include/asm/xor.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef ___ASM_SPARC_XOR_H
-#define ___ASM_SPARC_XOR_H
-#if defined(__sparc__) && defined(__arch64__)
-#include <asm/xor_64.h>
-#else
-#include <asm/xor_32.h>
-#endif
-#endif
diff --git a/arch/sparc/include/asm/xor_32.h b/arch/sparc/include/asm/xor_32.h
deleted file mode 100644
index 0351813cf3af5..0000000000000
--- a/arch/sparc/include/asm/xor_32.h
+++ /dev/null
@@ -1,268 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm/xor.h
- *
- * Optimized RAID-5 checksumming functions for 32-bit Sparc.
- */
-
-/*
- * High speed xor_block operation for RAID4/5 utilizing the
- * ldd/std SPARC instructions.
- *
- * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
- */
-
-static void
-sparc_2(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2)
-{
-	int lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		__asm__ __volatile__(
-		  "ldd [%0 + 0x00], %%g2\n\t"
-		  "ldd [%0 + 0x08], %%g4\n\t"
-		  "ldd [%0 + 0x10], %%o0\n\t"
-		  "ldd [%0 + 0x18], %%o2\n\t"
-		  "ldd [%1 + 0x00], %%o4\n\t"
-		  "ldd [%1 + 0x08], %%l0\n\t"
-		  "ldd [%1 + 0x10], %%l2\n\t"
-		  "ldd [%1 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "std %%g2, [%0 + 0x00]\n\t"
-		  "std %%g4, [%0 + 0x08]\n\t"
-		  "std %%o0, [%0 + 0x10]\n\t"
-		  "std %%o2, [%0 + 0x18]\n"
-		:
-		: "r" (p1), "r" (p2)
-		: "g2", "g3", "g4", "g5",
-		  "o0", "o1", "o2", "o3", "o4", "o5",
-		  "l0", "l1", "l2", "l3", "l4", "l5");
-		p1 += 8;
-		p2 += 8;
-	} while (--lines > 0);
-}
-
-static void
-sparc_3(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3)
-{
-	int lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		__asm__ __volatile__(
-		  "ldd [%0 + 0x00], %%g2\n\t"
-		  "ldd [%0 + 0x08], %%g4\n\t"
-		  "ldd [%0 + 0x10], %%o0\n\t"
-		  "ldd [%0 + 0x18], %%o2\n\t"
-		  "ldd [%1 + 0x00], %%o4\n\t"
-		  "ldd [%1 + 0x08], %%l0\n\t"
-		  "ldd [%1 + 0x10], %%l2\n\t"
-		  "ldd [%1 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%2 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%2 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%2 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%2 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "std %%g2, [%0 + 0x00]\n\t"
-		  "std %%g4, [%0 + 0x08]\n\t"
-		  "std %%o0, [%0 + 0x10]\n\t"
-		  "std %%o2, [%0 + 0x18]\n"
-		:
-		: "r" (p1), "r" (p2), "r" (p3)
-		: "g2", "g3", "g4", "g5",
-		  "o0", "o1", "o2", "o3", "o4", "o5",
-		  "l0", "l1", "l2", "l3", "l4", "l5");
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-	} while (--lines > 0);
-}
-
-static void
-sparc_4(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4)
-{
-	int lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		__asm__ __volatile__(
-		  "ldd [%0 + 0x00], %%g2\n\t"
-		  "ldd [%0 + 0x08], %%g4\n\t"
-		  "ldd [%0 + 0x10], %%o0\n\t"
-		  "ldd [%0 + 0x18], %%o2\n\t"
-		  "ldd [%1 + 0x00], %%o4\n\t"
-		  "ldd [%1 + 0x08], %%l0\n\t"
-		  "ldd [%1 + 0x10], %%l2\n\t"
-		  "ldd [%1 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%2 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%2 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%2 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%2 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%3 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%3 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%3 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%3 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "std %%g2, [%0 + 0x00]\n\t"
-		  "std %%g4, [%0 + 0x08]\n\t"
-		  "std %%o0, [%0 + 0x10]\n\t"
-		  "std %%o2, [%0 + 0x18]\n"
-		:
-		: "r" (p1), "r" (p2), "r" (p3), "r" (p4)
-		: "g2", "g3", "g4", "g5",
-		  "o0", "o1", "o2", "o3", "o4", "o5",
-		  "l0", "l1", "l2", "l3", "l4", "l5");
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-	} while (--lines > 0);
-}
-
-static void
-sparc_5(unsigned long bytes, unsigned long * __restrict p1,
-	const unsigned long * __restrict p2,
-	const unsigned long * __restrict p3,
-	const unsigned long * __restrict p4,
-	const unsigned long * __restrict p5)
-{
-	int lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		__asm__ __volatile__(
-		  "ldd [%0 + 0x00], %%g2\n\t"
-		  "ldd [%0 + 0x08], %%g4\n\t"
-		  "ldd [%0 + 0x10], %%o0\n\t"
-		  "ldd [%0 + 0x18], %%o2\n\t"
-		  "ldd [%1 + 0x00], %%o4\n\t"
-		  "ldd [%1 + 0x08], %%l0\n\t"
-		  "ldd [%1 + 0x10], %%l2\n\t"
-		  "ldd [%1 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%2 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%2 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%2 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%2 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%3 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%3 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%3 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%3 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "ldd [%4 + 0x00], %%o4\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "ldd [%4 + 0x08], %%l0\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "ldd [%4 + 0x10], %%l2\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "ldd [%4 + 0x18], %%l4\n\t"
-		  "xor %%g2, %%o4, %%g2\n\t"
-		  "xor %%g3, %%o5, %%g3\n\t"
-		  "xor %%g4, %%l0, %%g4\n\t"
-		  "xor %%g5, %%l1, %%g5\n\t"
-		  "xor %%o0, %%l2, %%o0\n\t"
-		  "xor %%o1, %%l3, %%o1\n\t"
-		  "xor %%o2, %%l4, %%o2\n\t"
-		  "xor %%o3, %%l5, %%o3\n\t"
-		  "std %%g2, [%0 + 0x00]\n\t"
-		  "std %%g4, [%0 + 0x08]\n\t"
-		  "std %%o0, [%0 + 0x10]\n\t"
-		  "std %%o2, [%0 + 0x18]\n"
-		:
-		: "r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
-		: "g2", "g3", "g4", "g5",
-		  "o0", "o1", "o2", "o3", "o4", "o5",
-		  "l0", "l1", "l2", "l3", "l4", "l5");
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-		p5 += 8;
-	} while (--lines > 0);
-}
-
-static struct xor_block_template xor_block_SPARC = {
-	.name	= "SPARC",
-	.do_2	= sparc_2,
-	.do_3	= sparc_3,
-	.do_4	= sparc_4,
-	.do_5	= sparc_5,
-};
-
-/* For grins, also test the generic routines.  */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-	do {						\
-		xor_speed(&xor_block_8regs);		\
-		xor_speed(&xor_block_32regs);		\
-		xor_speed(&xor_block_SPARC);		\
-	} while (0)
diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h
deleted file mode 100644
index caaddea8ad79d..0000000000000
--- a/arch/sparc/include/asm/xor_64.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm/xor.h
- *
- * High speed xor_block operation for RAID4/5 utilizing the
- * UltraSparc Visual Instruction Set and Niagara block-init
- * twin-load instructions.
- *
- * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
- */
-
-#include <asm/spitfire.h>
-
-void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2);
-void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3);
-void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4);
-void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4,
-	       const unsigned long * __restrict p5);
-
-/* XXX Ugh, write cheetah versions... -DaveM */
-
-static struct xor_block_template xor_block_VIS = {
-        .name	= "VIS",
-        .do_2	= xor_vis_2,
-        .do_3	= xor_vis_3,
-        .do_4	= xor_vis_4,
-        .do_5	= xor_vis_5,
-};
-
-void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2);
-void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3);
-void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4);
-void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
-		   const unsigned long * __restrict p2,
-		   const unsigned long * __restrict p3,
-		   const unsigned long * __restrict p4,
-		   const unsigned long * __restrict p5);
-
-static struct xor_block_template xor_block_niagara = {
-        .name	= "Niagara",
-        .do_2	= xor_niagara_2,
-        .do_3	= xor_niagara_3,
-        .do_4	= xor_niagara_4,
-        .do_5	= xor_niagara_5,
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-	do {						\
-		xor_speed(&xor_block_VIS);		\
-		xor_speed(&xor_block_niagara);		\
-	} while (0)
-
-/* For VIS for everything except Niagara.  */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
-	((tlb_type == hypervisor && \
-	  (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || \
-	   sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || \
-	   sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || \
-	   sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || \
-	   sun4v_chip_type == SUN4V_CHIP_NIAGARA5)) ? \
-	 &xor_block_niagara : \
-	 &xor_block_VIS)
diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
index 22170d4f8e060..497b5714fa8ff 100644
--- a/arch/sparc/kernel/Makefile
+++ b/arch/sparc/kernel/Makefile
@@ -41,7 +41,6 @@ obj-$(CONFIG_SPARC32)   += systbls_32.o
 obj-y                   += time_$(BITS).o
 obj-$(CONFIG_SPARC32)   += windows.o
 obj-y                   += cpu.o
-obj-$(CONFIG_SPARC64)	+= vdso.o
 obj-$(CONFIG_SPARC32)   += devices.o
 obj-y                   += ptrace_$(BITS).o
 obj-y                   += unaligned_$(BITS).o
diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S
index 38345460d5420..8c320fa25a670 100644
--- a/arch/sparc/kernel/head_32.S
+++ b/arch/sparc/kernel/head_32.S
@@ -57,13 +57,6 @@ sun4e_notsup:
 
 	.align PAGE_SIZE
 
-/* This was the only reasonable way I could think of to properly align
- * these page-table data structures.
- */
-	.globl empty_zero_page
-empty_zero_page:	.skip PAGE_SIZE
-EXPORT_SYMBOL(empty_zero_page)
-
 	.global root_flags
 	.global ram_flags
 	.global root_dev
diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c
index b32f27f929d1a..87b267043ccd2 100644
--- a/arch/sparc/kernel/time_64.c
+++ b/arch/sparc/kernel/time_64.c
@@ -838,14 +838,14 @@ void __init time_init_early(void)
 	if (tlb_type == spitfire) {
 		if (is_hummingbird()) {
 			init_tick_ops(&hbtick_operations);
-			clocksource_tick.archdata.vclock_mode = VCLOCK_NONE;
+			clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 		} else {
 			init_tick_ops(&tick_operations);
-			clocksource_tick.archdata.vclock_mode = VCLOCK_TICK;
+			clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_TICK;
 		}
 	} else {
 		init_tick_ops(&stick_operations);
-		clocksource_tick.archdata.vclock_mode = VCLOCK_STICK;
+		clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_STICK;
 	}
 }
 
diff --git a/arch/sparc/kernel/vdso.c b/arch/sparc/kernel/vdso.c
deleted file mode 100644
index 0e27437eb97bf..0000000000000
--- a/arch/sparc/kernel/vdso.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
- *  Copyright 2003 Andi Kleen, SuSE Labs.
- *
- *  Thanks to hpa@transmeta.com for some useful hint.
- *  Special thanks to Ingo Molnar for his early experience with
- *  a different vsyscall implementation for Linux/IA32 and for the name.
- */
-
-#include <linux/time.h>
-#include <linux/timekeeper_internal.h>
-
-#include <asm/vvar.h>
-
-void update_vsyscall_tz(void)
-{
-	if (unlikely(vvar_data == NULL))
-		return;
-
-	vvar_data->tz_minuteswest = sys_tz.tz_minuteswest;
-	vvar_data->tz_dsttime = sys_tz.tz_dsttime;
-}
-
-void update_vsyscall(struct timekeeper *tk)
-{
-	struct vvar_data *vdata = vvar_data;
-
-	if (unlikely(vdata == NULL))
-		return;
-
-	vvar_write_begin(vdata);
-	vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode;
-	vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
-	vdata->clock.mask = tk->tkr_mono.mask;
-	vdata->clock.mult = tk->tkr_mono.mult;
-	vdata->clock.shift = tk->tkr_mono.shift;
-
-	vdata->wall_time_sec = tk->xtime_sec;
-	vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
-
-	vdata->monotonic_time_sec = tk->xtime_sec +
-				    tk->wall_to_monotonic.tv_sec;
-	vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec +
-				      (tk->wall_to_monotonic.tv_nsec <<
-				       tk->tkr_mono.shift);
-
-	while (vdata->monotonic_time_snsec >=
-	       (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
-		vdata->monotonic_time_snsec -=
-				((u64)NSEC_PER_SEC) << tk->tkr_mono.shift;
-		vdata->monotonic_time_sec++;
-	}
-
-	vdata->wall_time_coarse_sec = tk->xtime_sec;
-	vdata->wall_time_coarse_nsec =
-			(long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
-
-	vdata->monotonic_time_coarse_sec =
-		vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec;
-	vdata->monotonic_time_coarse_nsec =
-		vdata->wall_time_coarse_nsec + tk->wall_to_monotonic.tv_nsec;
-
-	while (vdata->monotonic_time_coarse_nsec >= NSEC_PER_SEC) {
-		vdata->monotonic_time_coarse_nsec -= NSEC_PER_SEC;
-		vdata->monotonic_time_coarse_sec++;
-	}
-
-	vvar_write_end(vdata);
-}
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 783bdec0d7be0..dd10cdd6f0623 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -48,7 +48,7 @@ lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
 lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
 
 lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
-lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
+lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o hweight.o ffs.o
 
 obj-$(CONFIG_SPARC64) += iomap.o
 obj-$(CONFIG_SPARC32) += atomic32.o
diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S
deleted file mode 100644
index 35461e3b2a9b1..0000000000000
--- a/arch/sparc/lib/xor.S
+++ /dev/null
@@ -1,646 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * arch/sparc64/lib/xor.S
- *
- * High speed xor_block operation for RAID4/5 utilizing the
- * UltraSparc Visual Instruction Set and Niagara store-init/twin-load.
- *
- * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
- */
-
-#include <linux/export.h>
-#include <linux/linkage.h>
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
-
-/*
- *	Requirements:
- *	!(((long)dest | (long)sourceN) & (64 - 1)) &&
- *	!(len & 127) && len >= 256
- */
-	.text
-
-	/* VIS versions. */
-ENTRY(xor_vis_2)
-	rd	%fprs, %o5
-	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
-	be,pt	%icc, 0f
-	 sethi	%hi(VISenter), %g1
-	jmpl	%g1 + %lo(VISenter), %g7
-	 add	%g7, 8, %g7
-0:	wr	%g0, FPRS_FEF, %fprs
-	rd	%asi, %g1
-	wr	%g0, ASI_BLK_P, %asi
-	membar	#LoadStore|#StoreLoad|#StoreStore
-	sub	%o0, 128, %o0
-	ldda	[%o1] %asi, %f0
-	ldda	[%o2] %asi, %f16
-
-2:	ldda	[%o1 + 64] %asi, %f32
-	fxor	%f0, %f16, %f16
-	fxor	%f2, %f18, %f18
-	fxor	%f4, %f20, %f20
-	fxor	%f6, %f22, %f22
-	fxor	%f8, %f24, %f24
-	fxor	%f10, %f26, %f26
-	fxor	%f12, %f28, %f28
-	fxor	%f14, %f30, %f30
-	stda	%f16, [%o1] %asi
-	ldda	[%o2 + 64] %asi, %f48
-	ldda	[%o1 + 128] %asi, %f0
-	fxor	%f32, %f48, %f48
-	fxor	%f34, %f50, %f50
-	add	%o1, 128, %o1
-	fxor	%f36, %f52, %f52
-	add	%o2, 128, %o2
-	fxor	%f38, %f54, %f54
-	subcc	%o0, 128, %o0
-	fxor	%f40, %f56, %f56
-	fxor	%f42, %f58, %f58
-	fxor	%f44, %f60, %f60
-	fxor	%f46, %f62, %f62
-	stda	%f48, [%o1 - 64] %asi
-	bne,pt	%xcc, 2b
-	 ldda	[%o2] %asi, %f16
-
-	ldda	[%o1 + 64] %asi, %f32
-	fxor	%f0, %f16, %f16
-	fxor	%f2, %f18, %f18
-	fxor	%f4, %f20, %f20
-	fxor	%f6, %f22, %f22
-	fxor	%f8, %f24, %f24
-	fxor	%f10, %f26, %f26
-	fxor	%f12, %f28, %f28
-	fxor	%f14, %f30, %f30
-	stda	%f16, [%o1] %asi
-	ldda	[%o2 + 64] %asi, %f48
-	membar	#Sync
-	fxor	%f32, %f48, %f48
-	fxor	%f34, %f50, %f50
-	fxor	%f36, %f52, %f52
-	fxor	%f38, %f54, %f54
-	fxor	%f40, %f56, %f56
-	fxor	%f42, %f58, %f58
-	fxor	%f44, %f60, %f60
-	fxor	%f46, %f62, %f62
-	stda	%f48, [%o1 + 64] %asi
-	membar	#Sync|#StoreStore|#StoreLoad
-	wr	%g1, %g0, %asi
-	retl
-	  wr	%g0, 0, %fprs
-ENDPROC(xor_vis_2)
-EXPORT_SYMBOL(xor_vis_2)
-
-ENTRY(xor_vis_3)
-	rd	%fprs, %o5
-	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
-	be,pt	%icc, 0f
-	 sethi	%hi(VISenter), %g1
-	jmpl	%g1 + %lo(VISenter), %g7
-	 add	%g7, 8, %g7
-0:	wr	%g0, FPRS_FEF, %fprs
-	rd	%asi, %g1
-	wr	%g0, ASI_BLK_P, %asi
-	membar	#LoadStore|#StoreLoad|#StoreStore
-	sub	%o0, 64, %o0
-	ldda	[%o1] %asi, %f0
-	ldda	[%o2] %asi, %f16
-
-3:	ldda	[%o3] %asi, %f32
-	fxor	%f0, %f16, %f48
-	fxor	%f2, %f18, %f50
-	add	%o1, 64, %o1
-	fxor	%f4, %f20, %f52
-	fxor	%f6, %f22, %f54
-	add	%o2, 64, %o2
-	fxor	%f8, %f24, %f56
-	fxor	%f10, %f26, %f58
-	fxor	%f12, %f28, %f60
-	fxor	%f14, %f30, %f62
-	ldda	[%o1] %asi, %f0
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	add	%o3, 64, %o3
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	subcc	%o0, 64, %o0
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	stda	%f48, [%o1 - 64] %asi
-	bne,pt	%xcc, 3b
-	 ldda	[%o2] %asi, %f16
-
-	ldda	[%o3] %asi, %f32
-	fxor	%f0, %f16, %f48
-	fxor	%f2, %f18, %f50
-	fxor	%f4, %f20, %f52
-	fxor	%f6, %f22, %f54
-	fxor	%f8, %f24, %f56
-	fxor	%f10, %f26, %f58
-	fxor	%f12, %f28, %f60
-	fxor	%f14, %f30, %f62
-	membar	#Sync
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	stda	%f48, [%o1] %asi
-	membar	#Sync|#StoreStore|#StoreLoad
-	wr	%g1, %g0, %asi
-	retl
-	 wr	%g0, 0, %fprs
-ENDPROC(xor_vis_3)
-EXPORT_SYMBOL(xor_vis_3)
-
-ENTRY(xor_vis_4)
-	rd	%fprs, %o5
-	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
-	be,pt	%icc, 0f
-	 sethi	%hi(VISenter), %g1
-	jmpl	%g1 + %lo(VISenter), %g7
-	 add	%g7, 8, %g7
-0:	wr	%g0, FPRS_FEF, %fprs
-	rd	%asi, %g1
-	wr	%g0, ASI_BLK_P, %asi
-	membar	#LoadStore|#StoreLoad|#StoreStore
-	sub	%o0, 64, %o0
-	ldda	[%o1] %asi, %f0
-	ldda	[%o2] %asi, %f16
-
-4:	ldda	[%o3] %asi, %f32
-	fxor	%f0, %f16, %f16
-	fxor	%f2, %f18, %f18
-	add	%o1, 64, %o1
-	fxor	%f4, %f20, %f20
-	fxor	%f6, %f22, %f22
-	add	%o2, 64, %o2
-	fxor	%f8, %f24, %f24
-	fxor	%f10, %f26, %f26
-	fxor	%f12, %f28, %f28
-	fxor	%f14, %f30, %f30
-	ldda	[%o4] %asi, %f48
-	fxor	%f16, %f32, %f32
-	fxor	%f18, %f34, %f34
-	fxor	%f20, %f36, %f36
-	fxor	%f22, %f38, %f38
-	add	%o3, 64, %o3
-	fxor	%f24, %f40, %f40
-	fxor	%f26, %f42, %f42
-	fxor	%f28, %f44, %f44
-	fxor	%f30, %f46, %f46
-	ldda	[%o1] %asi, %f0
-	fxor	%f32, %f48, %f48
-	fxor	%f34, %f50, %f50
-	fxor	%f36, %f52, %f52
-	add	%o4, 64, %o4
-	fxor	%f38, %f54, %f54
-	fxor	%f40, %f56, %f56
-	fxor	%f42, %f58, %f58
-	subcc	%o0, 64, %o0
-	fxor	%f44, %f60, %f60
-	fxor	%f46, %f62, %f62
-	stda	%f48, [%o1 - 64] %asi
-	bne,pt	%xcc, 4b
-	 ldda	[%o2] %asi, %f16
-
-	ldda	[%o3] %asi, %f32
-	fxor	%f0, %f16, %f16
-	fxor	%f2, %f18, %f18
-	fxor	%f4, %f20, %f20
-	fxor	%f6, %f22, %f22
-	fxor	%f8, %f24, %f24
-	fxor	%f10, %f26, %f26
-	fxor	%f12, %f28, %f28
-	fxor	%f14, %f30, %f30
-	ldda	[%o4] %asi, %f48
-	fxor	%f16, %f32, %f32
-	fxor	%f18, %f34, %f34
-	fxor	%f20, %f36, %f36
-	fxor	%f22, %f38, %f38
-	fxor	%f24, %f40, %f40
-	fxor	%f26, %f42, %f42
-	fxor	%f28, %f44, %f44
-	fxor	%f30, %f46, %f46
-	membar	#Sync
-	fxor	%f32, %f48, %f48
-	fxor	%f34, %f50, %f50
-	fxor	%f36, %f52, %f52
-	fxor	%f38, %f54, %f54
-	fxor	%f40, %f56, %f56
-	fxor	%f42, %f58, %f58
-	fxor	%f44, %f60, %f60
-	fxor	%f46, %f62, %f62
-	stda	%f48, [%o1] %asi
-	membar	#Sync|#StoreStore|#StoreLoad
-	wr	%g1, %g0, %asi
-	retl
-	 wr	%g0, 0, %fprs
-ENDPROC(xor_vis_4)
-EXPORT_SYMBOL(xor_vis_4)
-
-ENTRY(xor_vis_5)
-	save	%sp, -192, %sp
-	rd	%fprs, %o5
-	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
-	be,pt	%icc, 0f
-	 sethi	%hi(VISenter), %g1
-	jmpl	%g1 + %lo(VISenter), %g7
-	 add	%g7, 8, %g7
-0:	wr	%g0, FPRS_FEF, %fprs
-	rd	%asi, %g1
-	wr	%g0, ASI_BLK_P, %asi
-	membar	#LoadStore|#StoreLoad|#StoreStore
-	sub	%i0, 64, %i0
-	ldda	[%i1] %asi, %f0
-	ldda	[%i2] %asi, %f16
-
-5:	ldda	[%i3] %asi, %f32
-	fxor	%f0, %f16, %f48
-	fxor	%f2, %f18, %f50
-	add	%i1, 64, %i1
-	fxor	%f4, %f20, %f52
-	fxor	%f6, %f22, %f54
-	add	%i2, 64, %i2
-	fxor	%f8, %f24, %f56
-	fxor	%f10, %f26, %f58
-	fxor	%f12, %f28, %f60
-	fxor	%f14, %f30, %f62
-	ldda	[%i4] %asi, %f16
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	add	%i3, 64, %i3
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	ldda	[%i5] %asi, %f32
-	fxor	%f48, %f16, %f48
-	fxor	%f50, %f18, %f50
-	add	%i4, 64, %i4
-	fxor	%f52, %f20, %f52
-	fxor	%f54, %f22, %f54
-	add	%i5, 64, %i5
-	fxor	%f56, %f24, %f56
-	fxor	%f58, %f26, %f58
-	fxor	%f60, %f28, %f60
-	fxor	%f62, %f30, %f62
-	ldda	[%i1] %asi, %f0
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	subcc	%i0, 64, %i0
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	stda	%f48, [%i1 - 64] %asi
-	bne,pt	%xcc, 5b
-	 ldda	[%i2] %asi, %f16
-
-	ldda	[%i3] %asi, %f32
-	fxor	%f0, %f16, %f48
-	fxor	%f2, %f18, %f50
-	fxor	%f4, %f20, %f52
-	fxor	%f6, %f22, %f54
-	fxor	%f8, %f24, %f56
-	fxor	%f10, %f26, %f58
-	fxor	%f12, %f28, %f60
-	fxor	%f14, %f30, %f62
-	ldda	[%i4] %asi, %f16
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	ldda	[%i5] %asi, %f32
-	fxor	%f48, %f16, %f48
-	fxor	%f50, %f18, %f50
-	fxor	%f52, %f20, %f52
-	fxor	%f54, %f22, %f54
-	fxor	%f56, %f24, %f56
-	fxor	%f58, %f26, %f58
-	fxor	%f60, %f28, %f60
-	fxor	%f62, %f30, %f62
-	membar	#Sync
-	fxor	%f48, %f32, %f48
-	fxor	%f50, %f34, %f50
-	fxor	%f52, %f36, %f52
-	fxor	%f54, %f38, %f54
-	fxor	%f56, %f40, %f56
-	fxor	%f58, %f42, %f58
-	fxor	%f60, %f44, %f60
-	fxor	%f62, %f46, %f62
-	stda	%f48, [%i1] %asi
-	membar	#Sync|#StoreStore|#StoreLoad
-	wr	%g1, %g0, %asi
-	wr	%g0, 0, %fprs
-	ret
-	 restore
-ENDPROC(xor_vis_5)
-EXPORT_SYMBOL(xor_vis_5)
-
-	/* Niagara versions. */
-ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
-	save		%sp, -192, %sp
-	prefetch	[%i1], #n_writes
-	prefetch	[%i2], #one_read
-	rd		%asi, %g7
-	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
-	srlx		%i0, 6, %g1
-	mov		%i1, %i0
-	mov		%i2, %i1
-1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src  + 0x00 */
-	ldda		[%i1 + 0x10] %asi, %i4	/* %i4/%i5 = src  + 0x10 */
-	ldda		[%i1 + 0x20] %asi, %g2	/* %g2/%g3 = src  + 0x20 */
-	ldda		[%i1 + 0x30] %asi, %l0	/* %l0/%l1 = src  + 0x30 */
-	prefetch	[%i1 + 0x40], #one_read
-	ldda		[%i0 + 0x00] %asi, %o0  /* %o0/%o1 = dest + 0x00 */
-	ldda		[%i0 + 0x10] %asi, %o2  /* %o2/%o3 = dest + 0x10 */
-	ldda		[%i0 + 0x20] %asi, %o4  /* %o4/%o5 = dest + 0x20 */
-	ldda		[%i0 + 0x30] %asi, %l2  /* %l2/%l3 = dest + 0x30 */
-	prefetch	[%i0 + 0x40], #n_writes
-	xor		%o0, %i2, %o0
-	xor		%o1, %i3, %o1
-	stxa		%o0, [%i0 + 0x00] %asi
-	stxa		%o1, [%i0 + 0x08] %asi
-	xor		%o2, %i4, %o2
-	xor		%o3, %i5, %o3
-	stxa		%o2, [%i0 + 0x10] %asi
-	stxa		%o3, [%i0 + 0x18] %asi
-	xor		%o4, %g2, %o4
-	xor		%o5, %g3, %o5
-	stxa		%o4, [%i0 + 0x20] %asi
-	stxa		%o5, [%i0 + 0x28] %asi
-	xor		%l2, %l0, %l2
-	xor		%l3, %l1, %l3
-	stxa		%l2, [%i0 + 0x30] %asi
-	stxa		%l3, [%i0 + 0x38] %asi
-	add		%i0, 0x40, %i0
-	subcc		%g1, 1, %g1
-	bne,pt		%xcc, 1b
-	 add		%i1, 0x40, %i1
-	membar		#Sync
-	wr		%g7, 0x0, %asi
-	ret
-	 restore
-ENDPROC(xor_niagara_2)
-EXPORT_SYMBOL(xor_niagara_2)
-
-ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
-	save		%sp, -192, %sp
-	prefetch	[%i1], #n_writes
-	prefetch	[%i2], #one_read
-	prefetch	[%i3], #one_read
-	rd		%asi, %g7
-	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
-	srlx		%i0, 6, %g1
-	mov		%i1, %i0
-	mov		%i2, %i1
-	mov		%i3, %l7
-1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
-	ldda		[%i1 + 0x10] %asi, %i4	/* %i4/%i5 = src1 + 0x10 */
-	ldda		[%l7 + 0x00] %asi, %g2	/* %g2/%g3 = src2 + 0x00 */
-	ldda		[%l7 + 0x10] %asi, %l0	/* %l0/%l1 = src2 + 0x10 */
-	ldda		[%i0 + 0x00] %asi, %o0  /* %o0/%o1 = dest + 0x00 */
-	ldda		[%i0 + 0x10] %asi, %o2  /* %o2/%o3 = dest + 0x10 */
-	xor		%g2, %i2, %g2
-	xor		%g3, %i3, %g3
-	xor		%o0, %g2, %o0
-	xor		%o1, %g3, %o1
-	stxa		%o0, [%i0 + 0x00] %asi
-	stxa		%o1, [%i0 + 0x08] %asi
-	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
-	ldda		[%l7 + 0x20] %asi, %g2	/* %g2/%g3 = src2 + 0x20 */
-	ldda		[%i0 + 0x20] %asi, %o0	/* %o0/%o1 = dest + 0x20 */
-	xor		%l0, %i4, %l0
-	xor		%l1, %i5, %l1
-	xor		%o2, %l0, %o2
-	xor		%o3, %l1, %o3
-	stxa		%o2, [%i0 + 0x10] %asi
-	stxa		%o3, [%i0 + 0x18] %asi
-	ldda		[%i1 + 0x30] %asi, %i4	/* %i4/%i5 = src1 + 0x30 */
-	ldda		[%l7 + 0x30] %asi, %l0	/* %l0/%l1 = src2 + 0x30 */
-	ldda		[%i0 + 0x30] %asi, %o2	/* %o2/%o3 = dest + 0x30 */
-	prefetch	[%i1 + 0x40], #one_read
-	prefetch	[%l7 + 0x40], #one_read
-	prefetch	[%i0 + 0x40], #n_writes
-	xor		%g2, %i2, %g2
-	xor		%g3, %i3, %g3
-	xor		%o0, %g2, %o0
-	xor		%o1, %g3, %o1
-	stxa		%o0, [%i0 + 0x20] %asi
-	stxa		%o1, [%i0 + 0x28] %asi
-	xor		%l0, %i4, %l0
-	xor		%l1, %i5, %l1
-	xor		%o2, %l0, %o2
-	xor		%o3, %l1, %o3
-	stxa		%o2, [%i0 + 0x30] %asi
-	stxa		%o3, [%i0 + 0x38] %asi
-	add		%i0, 0x40, %i0
-	add		%i1, 0x40, %i1
-	subcc		%g1, 1, %g1
-	bne,pt		%xcc, 1b
-	 add		%l7, 0x40, %l7
-	membar		#Sync
-	wr		%g7, 0x0, %asi
-	ret
-	 restore
-ENDPROC(xor_niagara_3)
-EXPORT_SYMBOL(xor_niagara_3)
-
-ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
-	save		%sp, -192, %sp
-	prefetch	[%i1], #n_writes
-	prefetch	[%i2], #one_read
-	prefetch	[%i3], #one_read
-	prefetch	[%i4], #one_read
-	rd		%asi, %g7
-	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
-	srlx		%i0, 6, %g1
-	mov		%i1, %i0
-	mov		%i2, %i1
-	mov		%i3, %l7
-	mov		%i4, %l6
-1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
-	ldda		[%l7 + 0x00] %asi, %i4	/* %i4/%i5 = src2 + 0x00 */
-	ldda		[%l6 + 0x00] %asi, %g2	/* %g2/%g3 = src3 + 0x00 */
-	ldda		[%i0 + 0x00] %asi, %l0	/* %l0/%l1 = dest + 0x00 */
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x10] %asi, %i2	/* %i2/%i3 = src1 + 0x10 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x10] %asi, %i4	/* %i4/%i5 = src2 + 0x10 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	stxa		%l0, [%i0 + 0x00] %asi
-	stxa		%l1, [%i0 + 0x08] %asi
-	ldda		[%l6 + 0x10] %asi, %g2	/* %g2/%g3 = src3 + 0x10 */
-	ldda		[%i0 + 0x10] %asi, %l0	/* %l0/%l1 = dest + 0x10 */
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x20] %asi, %i4	/* %i4/%i5 = src2 + 0x20 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	stxa		%l0, [%i0 + 0x10] %asi
-	stxa		%l1, [%i0 + 0x18] %asi
-	ldda		[%l6 + 0x20] %asi, %g2	/* %g2/%g3 = src3 + 0x20 */
-	ldda		[%i0 + 0x20] %asi, %l0	/* %l0/%l1 = dest + 0x20 */
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x30] %asi, %i2	/* %i2/%i3 = src1 + 0x30 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x30] %asi, %i4	/* %i4/%i5 = src2 + 0x30 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	stxa		%l0, [%i0 + 0x20] %asi
-	stxa		%l1, [%i0 + 0x28] %asi
-	ldda		[%l6 + 0x30] %asi, %g2	/* %g2/%g3 = src3 + 0x30 */
-	ldda		[%i0 + 0x30] %asi, %l0	/* %l0/%l1 = dest + 0x30 */
-
-	prefetch	[%i1 + 0x40], #one_read
-	prefetch	[%l7 + 0x40], #one_read
-	prefetch	[%l6 + 0x40], #one_read
-	prefetch	[%i0 + 0x40], #n_writes
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	stxa		%l0, [%i0 + 0x30] %asi
-	stxa		%l1, [%i0 + 0x38] %asi
-
-	add		%i0, 0x40, %i0
-	add		%i1, 0x40, %i1
-	add		%l7, 0x40, %l7
-	subcc		%g1, 1, %g1
-	bne,pt		%xcc, 1b
-	 add		%l6, 0x40, %l6
-	membar		#Sync
-	wr		%g7, 0x0, %asi
-	ret
-	 restore
-ENDPROC(xor_niagara_4)
-EXPORT_SYMBOL(xor_niagara_4)
-
-ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */
-	save		%sp, -192, %sp
-	prefetch	[%i1], #n_writes
-	prefetch	[%i2], #one_read
-	prefetch	[%i3], #one_read
-	prefetch	[%i4], #one_read
-	prefetch	[%i5], #one_read
-	rd		%asi, %g7
-	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
-	srlx		%i0, 6, %g1
-	mov		%i1, %i0
-	mov		%i2, %i1
-	mov		%i3, %l7
-	mov		%i4, %l6
-	mov		%i5, %l5
-1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
-	ldda		[%l7 + 0x00] %asi, %i4	/* %i4/%i5 = src2 + 0x00 */
-	ldda		[%l6 + 0x00] %asi, %g2	/* %g2/%g3 = src3 + 0x00 */
-	ldda		[%l5 + 0x00] %asi, %l0	/* %l0/%l1 = src4 + 0x00 */
-	ldda		[%i0 + 0x00] %asi, %l2	/* %l2/%l3 = dest + 0x00 */
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x10] %asi, %i2	/* %i2/%i3 = src1 + 0x10 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x10] %asi, %i4	/* %i4/%i5 = src2 + 0x10 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	ldda		[%l6 + 0x10] %asi, %g2	/* %g2/%g3 = src3 + 0x10 */
-	xor		%l2, %l0, %l2
-	xor		%l3, %l1, %l3
-	stxa		%l2, [%i0 + 0x00] %asi
-	stxa		%l3, [%i0 + 0x08] %asi
-	ldda		[%l5 + 0x10] %asi, %l0	/* %l0/%l1 = src4 + 0x10 */
-	ldda		[%i0 + 0x10] %asi, %l2	/* %l2/%l3 = dest + 0x10 */
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x20] %asi, %i4	/* %i4/%i5 = src2 + 0x20 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	ldda		[%l6 + 0x20] %asi, %g2	/* %g2/%g3 = src3 + 0x20 */
-	xor		%l2, %l0, %l2
-	xor		%l3, %l1, %l3
-	stxa		%l2, [%i0 + 0x10] %asi
-	stxa		%l3, [%i0 + 0x18] %asi
-	ldda		[%l5 + 0x20] %asi, %l0	/* %l0/%l1 = src4 + 0x20 */
-	ldda		[%i0 + 0x20] %asi, %l2	/* %l2/%l3 = dest + 0x20 */
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	ldda		[%i1 + 0x30] %asi, %i2	/* %i2/%i3 = src1 + 0x30 */
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	ldda		[%l7 + 0x30] %asi, %i4	/* %i4/%i5 = src2 + 0x30 */
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	ldda		[%l6 + 0x30] %asi, %g2	/* %g2/%g3 = src3 + 0x30 */
-	xor		%l2, %l0, %l2
-	xor		%l3, %l1, %l3
-	stxa		%l2, [%i0 + 0x20] %asi
-	stxa		%l3, [%i0 + 0x28] %asi
-	ldda		[%l5 + 0x30] %asi, %l0	/* %l0/%l1 = src4 + 0x30 */
-	ldda		[%i0 + 0x30] %asi, %l2	/* %l2/%l3 = dest + 0x30 */
-
-	prefetch	[%i1 + 0x40], #one_read
-	prefetch	[%l7 + 0x40], #one_read
-	prefetch	[%l6 + 0x40], #one_read
-	prefetch	[%l5 + 0x40], #one_read
-	prefetch	[%i0 + 0x40], #n_writes
-
-	xor		%i4, %i2, %i4
-	xor		%i5, %i3, %i5
-	xor		%g2, %i4, %g2
-	xor		%g3, %i5, %g3
-	xor		%l0, %g2, %l0
-	xor		%l1, %g3, %l1
-	xor		%l2, %l0, %l2
-	xor		%l3, %l1, %l3
-	stxa		%l2, [%i0 + 0x30] %asi
-	stxa		%l3, [%i0 + 0x38] %asi
-
-	add		%i0, 0x40, %i0
-	add		%i1, 0x40, %i1
-	add		%l7, 0x40, %l7
-	add		%l6, 0x40, %l6
-	subcc		%g1, 1, %g1
-	bne,pt		%xcc, 1b
-	 add		%l5, 0x40, %l5
-	membar		#Sync
-	wr		%g7, 0x0, %asi
-	ret
-	 restore
-ENDPROC(xor_niagara_5)
-EXPORT_SYMBOL(xor_niagara_5)
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index fdc93dd12c3e3..e0e66f91ceebc 100644
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -246,10 +246,6 @@ void __init arch_mm_preinit(void)
 		prom_halt();
 	}
 
-
-	/* Saves us work later. */
-	memset((void *)empty_zero_page, 0, PAGE_SIZE);
-
 	i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5);
 	i += 1;
 	sparc_valid_addr_bitmap = (unsigned long *)
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index f46394c46a764..367c269305e51 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -177,9 +177,6 @@ extern unsigned long sparc_ramdisk_image64;
 extern unsigned int sparc_ramdisk_image;
 extern unsigned int sparc_ramdisk_size;
 
-struct page *mem_map_zero __read_mostly;
-EXPORT_SYMBOL(mem_map_zero);
-
 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
 
 unsigned long sparc64_kern_pri_context __read_mostly;
@@ -2490,6 +2487,15 @@ static void __init register_page_bootmem_info(void)
 			register_page_bootmem_info_node(NODE_DATA(i));
 #endif
 }
+
+void __init arch_setup_zero_pages(void)
+{
+	phys_addr_t zero_page_pa = kern_base +
+		((unsigned long)&empty_zero_page[0] - KERNBASE);
+
+	__zero_page = phys_to_page(zero_page_pa);
+}
+
 void __init mem_init(void)
 {
 	/*
@@ -2500,18 +2506,6 @@ void __init mem_init(void)
 	 */
 	register_page_bootmem_info();
 
-	/*
-	 * Set up the zero page, mark it reserved, so that page count
-	 * is not manipulated when freeing the page from user ptes.
-	 */
-	mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
-	if (mem_map_zero == NULL) {
-		prom_printf("paging_init: Cannot alloc zero page.\n");
-		prom_halt();
-	}
-	mark_page_reserved(mem_map_zero);
-
-
 	if (tlb_type == cheetah || tlb_type == cheetah_plus)
 		cheetah_ecache_flush_init();
 }
@@ -2568,8 +2562,8 @@ unsigned long _PAGE_CACHE __read_mostly;
 EXPORT_SYMBOL(_PAGE_CACHE);
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
-			       int node, struct vmem_altmap *altmap)
+void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+			       unsigned long addr, unsigned long next)
 {
 	unsigned long pte_base;
 
@@ -2582,39 +2576,24 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
 
 	pte_base |= _PAGE_PMD_HUGE;
 
-	vstart = vstart & PMD_MASK;
-	vend = ALIGN(vend, PMD_SIZE);
-	for (; vstart < vend; vstart += PMD_SIZE) {
-		pgd_t *pgd = vmemmap_pgd_populate(vstart, node);
-		unsigned long pte;
-		p4d_t *p4d;
-		pud_t *pud;
-		pmd_t *pmd;
-
-		if (!pgd)
-			return -ENOMEM;
-
-		p4d = vmemmap_p4d_populate(pgd, vstart, node);
-		if (!p4d)
-			return -ENOMEM;
-
-		pud = vmemmap_pud_populate(p4d, vstart, node);
-		if (!pud)
-			return -ENOMEM;
+	pmd_val(*pmd) = pte_base | __pa(p);
+}
 
-		pmd = pmd_offset(pud, vstart);
-		pte = pmd_val(*pmd);
-		if (!(pte & _PAGE_VALID)) {
-			void *block = vmemmap_alloc_block(PMD_SIZE, node);
+int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
+				unsigned long addr, unsigned long next)
+{
+	int large = pmd_leaf(*pmdp);
 
-			if (!block)
-				return -ENOMEM;
+	if (large)
+		vmemmap_verify((pte_t *)pmdp, node, addr, next);
 
-			pmd_val(*pmd) = pte_base | __pa(block);
-		}
-	}
+	return large;
+}
 
-	return 0;
+int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
+			       int node, struct vmem_altmap *altmap)
+{
+	return vmemmap_populate_hugepages(vstart, vend, node, NULL);
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 
diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile
index 683b2d4082244..83fb2aca59cb9 100644
--- a/arch/sparc/vdso/Makefile
+++ b/arch/sparc/vdso/Makefile
@@ -3,6 +3,9 @@
 # Building vDSO images for sparc.
 #
 
+# Include the generic Makefile to check the built vDSO:
+include $(srctree)/lib/vdso/Makefile.include
+
 # files to link into the vdso
 vobjs-y := vdso-note.o vclock_gettime.o
 
@@ -90,6 +93,9 @@ KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING
 KBUILD_CFLAGS_32 += -mv8plus
 $(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32)
 
+CHECKFLAGS_32 := $(filter-out -m64 -D__sparc_v9__ -D__arch64__, $(CHECKFLAGS)) -m32
+$(obj)/vdso32.so.dbg: CHECKFLAGS = $(CHECKFLAGS_32)
+
 $(obj)/vdso32.so.dbg: FORCE \
 			$(obj)/vdso32/vdso32.lds \
 			$(obj)/vdso32/vclock_gettime.o \
@@ -102,6 +108,7 @@ $(obj)/vdso32.so.dbg: FORCE \
 quiet_cmd_vdso = VDSO    $@
       cmd_vdso = $(LD) -nostdlib -o $@ \
 		       $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
-		       -T $(filter %.lds,$^) $(filter %.o,$^)
+		       -T $(filter %.lds,$^) $(filter %.o,$^); \
+		       $(cmd_vdso_check)
 
-VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined
+VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined -z noexecstack
diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c
index 79607804ea1b0..1d9859392e4cf 100644
--- a/arch/sparc/vdso/vclock_gettime.c
+++ b/arch/sparc/vdso/vclock_gettime.c
@@ -12,382 +12,48 @@
  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
  */
 
-#include <linux/kernel.h>
-#include <linux/time.h>
-#include <linux/string.h>
-#include <asm/io.h>
-#include <asm/unistd.h>
-#include <asm/timex.h>
-#include <asm/clocksource.h>
-#include <asm/vvar.h>
+#include <linux/compiler.h>
+#include <linux/types.h>
 
-#ifdef	CONFIG_SPARC64
-#define SYSCALL_STRING							\
-	"ta	0x6d;"							\
-	"bcs,a	1f;"							\
-	" sub	%%g0, %%o0, %%o0;"					\
-	"1:"
-#else
-#define SYSCALL_STRING							\
-	"ta	0x10;"							\
-	"bcs,a	1f;"							\
-	" sub	%%g0, %%o0, %%o0;"					\
-	"1:"
-#endif
-
-#define SYSCALL_CLOBBERS						\
-	"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",			\
-	"f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",		\
-	"f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",		\
-	"f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",		\
-	"f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",		\
-	"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",		\
-	"cc", "memory"
-
-/*
- * Compute the vvar page's address in the process address space, and return it
- * as a pointer to the vvar_data.
- */
-notrace static __always_inline struct vvar_data *get_vvar_data(void)
-{
-	unsigned long ret;
-
-	/*
-	 * vdso data page is the first vDSO page so grab the PC
-	 * and move up a page to get to the data page.
-	 */
-	__asm__("rd %%pc, %0" : "=r" (ret));
-	ret &= ~(8192 - 1);
-	ret -= 8192;
-
-	return (struct vvar_data *) ret;
-}
+#include <vdso/gettime.h>
 
-notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
-{
-	register long num __asm__("g1") = __NR_clock_gettime;
-	register long o0 __asm__("o0") = clock;
-	register long o1 __asm__("o1") = (long) ts;
+#include <asm/vdso/gettimeofday.h>
 
-	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
-			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
-	return o0;
-}
+#include "../../../../lib/vdso/gettimeofday.c"
 
-notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
+int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
-	register long num __asm__("g1") = __NR_gettimeofday;
-	register long o0 __asm__("o0") = (long) tv;
-	register long o1 __asm__("o1") = (long) tz;
-
-	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
-			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
-	return o0;
+	return __cvdso_gettimeofday(tv, tz);
 }
 
-#ifdef	CONFIG_SPARC64
-notrace static __always_inline u64 __shr64(u64 val, int amt)
-{
-	return val >> amt;
-}
+int gettimeofday(struct __kernel_old_timeval *, struct timezone *)
+	__weak __alias(__vdso_gettimeofday);
 
-notrace static __always_inline u64 vread_tick(void)
+#if defined(CONFIG_SPARC64)
+int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
 {
-	u64	ret;
-
-	__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
-	return ret;
+	return __cvdso_clock_gettime(clock, ts);
 }
 
-notrace static __always_inline u64 vread_tick_stick(void)
-{
-	u64	ret;
+int clock_gettime(clockid_t, struct __kernel_timespec *)
+	__weak __alias(__vdso_clock_gettime);
 
-	__asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
-	return ret;
-}
 #else
-notrace static __always_inline u64 __shr64(u64 val, int amt)
-{
-	u64 ret;
-
-	__asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
-			     "srl %L1, 0, %L1\n\t"
-			     "or %%g1, %L1, %%g1\n\t"
-			     "srlx %%g1, %2, %L0\n\t"
-			     "srlx %L0, 32, %H0"
-			     : "=r" (ret)
-			     : "r" (val), "r" (amt)
-			     : "g1");
-	return ret;
-}
-
-notrace static __always_inline u64 vread_tick(void)
-{
-	register unsigned long long ret asm("o4");
-
-	__asm__ __volatile__("rd %%tick, %L0\n\t"
-			     "srlx %L0, 32, %H0"
-			     : "=r" (ret));
-	return ret;
-}
-
-notrace static __always_inline u64 vread_tick_stick(void)
-{
-	register unsigned long long ret asm("o4");
-
-	__asm__ __volatile__("rd %%asr24, %L0\n\t"
-			     "srlx %L0, 32, %H0"
-			     : "=r" (ret));
-	return ret;
-}
-#endif
 
-notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
+int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
 {
-	u64 v;
-	u64 cycles;
-
-	cycles = vread_tick();
-	v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
-	return v * vvar->clock.mult;
+	return __cvdso_clock_gettime32(clock, ts);
 }
 
-notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
-{
-	u64 v;
-	u64 cycles;
+int clock_gettime(clockid_t, struct old_timespec32 *)
+	__weak __alias(__vdso_clock_gettime);
 
-	cycles = vread_tick_stick();
-	v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
-	return v * vvar->clock.mult;
-}
-
-notrace static __always_inline int do_realtime(struct vvar_data *vvar,
-					       struct __kernel_old_timespec *ts)
+int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
 {
-	unsigned long seq;
-	u64 ns;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->wall_time_sec;
-		ns = vvar->wall_time_snsec;
-		ns += vgetsns(vvar);
-		ns = __shr64(ns, vvar->clock.shift);
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
-
-	return 0;
+	return __cvdso_clock_gettime(clock, ts);
 }
 
-notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
-						     struct __kernel_old_timespec *ts)
-{
-	unsigned long seq;
-	u64 ns;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->wall_time_sec;
-		ns = vvar->wall_time_snsec;
-		ns += vgetsns_stick(vvar);
-		ns = __shr64(ns, vvar->clock.shift);
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
+int clock_gettime64(clockid_t, struct __kernel_timespec *)
+	__weak __alias(__vdso_clock_gettime64);
 
-	return 0;
-}
-
-notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
-						struct __kernel_old_timespec *ts)
-{
-	unsigned long seq;
-	u64 ns;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->monotonic_time_sec;
-		ns = vvar->monotonic_time_snsec;
-		ns += vgetsns(vvar);
-		ns = __shr64(ns, vvar->clock.shift);
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
-
-	return 0;
-}
-
-notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
-						      struct __kernel_old_timespec *ts)
-{
-	unsigned long seq;
-	u64 ns;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->monotonic_time_sec;
-		ns = vvar->monotonic_time_snsec;
-		ns += vgetsns_stick(vvar);
-		ns = __shr64(ns, vvar->clock.shift);
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
-
-	return 0;
-}
-
-notrace static int do_realtime_coarse(struct vvar_data *vvar,
-				      struct __kernel_old_timespec *ts)
-{
-	unsigned long seq;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->wall_time_coarse_sec;
-		ts->tv_nsec = vvar->wall_time_coarse_nsec;
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-	return 0;
-}
-
-notrace static int do_monotonic_coarse(struct vvar_data *vvar,
-				       struct __kernel_old_timespec *ts)
-{
-	unsigned long seq;
-
-	do {
-		seq = vvar_read_begin(vvar);
-		ts->tv_sec = vvar->monotonic_time_coarse_sec;
-		ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
-	} while (unlikely(vvar_read_retry(vvar, seq)));
-
-	return 0;
-}
-
-notrace int
-__vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
-{
-	struct vvar_data *vvd = get_vvar_data();
-
-	switch (clock) {
-	case CLOCK_REALTIME:
-		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
-			break;
-		return do_realtime(vvd, ts);
-	case CLOCK_MONOTONIC:
-		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
-			break;
-		return do_monotonic(vvd, ts);
-	case CLOCK_REALTIME_COARSE:
-		return do_realtime_coarse(vvd, ts);
-	case CLOCK_MONOTONIC_COARSE:
-		return do_monotonic_coarse(vvd, ts);
-	}
-	/*
-	 * Unknown clock ID ? Fall back to the syscall.
-	 */
-	return vdso_fallback_gettime(clock, ts);
-}
-int
-clock_gettime(clockid_t, struct __kernel_old_timespec *)
-	__attribute__((weak, alias("__vdso_clock_gettime")));
-
-notrace int
-__vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
-{
-	struct vvar_data *vvd = get_vvar_data();
-
-	switch (clock) {
-	case CLOCK_REALTIME:
-		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
-			break;
-		return do_realtime_stick(vvd, ts);
-	case CLOCK_MONOTONIC:
-		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
-			break;
-		return do_monotonic_stick(vvd, ts);
-	case CLOCK_REALTIME_COARSE:
-		return do_realtime_coarse(vvd, ts);
-	case CLOCK_MONOTONIC_COARSE:
-		return do_monotonic_coarse(vvd, ts);
-	}
-	/*
-	 * Unknown clock ID ? Fall back to the syscall.
-	 */
-	return vdso_fallback_gettime(clock, ts);
-}
-
-notrace int
-__vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
-{
-	struct vvar_data *vvd = get_vvar_data();
-
-	if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
-		if (likely(tv != NULL)) {
-			union tstv_t {
-				struct __kernel_old_timespec ts;
-				struct __kernel_old_timeval tv;
-			} *tstv = (union tstv_t *) tv;
-			do_realtime(vvd, &tstv->ts);
-			/*
-			 * Assign before dividing to ensure that the division is
-			 * done in the type of tv_usec, not tv_nsec.
-			 *
-			 * There cannot be > 1 billion usec in a second:
-			 * do_realtime() has already distributed such overflow
-			 * into tv_sec.  So we can assign it to an int safely.
-			 */
-			tstv->tv.tv_usec = tstv->ts.tv_nsec;
-			tstv->tv.tv_usec /= 1000;
-		}
-		if (unlikely(tz != NULL)) {
-			/* Avoid memcpy. Some old compilers fail to inline it */
-			tz->tz_minuteswest = vvd->tz_minuteswest;
-			tz->tz_dsttime = vvd->tz_dsttime;
-		}
-		return 0;
-	}
-	return vdso_fallback_gettimeofday(tv, tz);
-}
-int
-gettimeofday(struct __kernel_old_timeval *, struct timezone *)
-	__attribute__((weak, alias("__vdso_gettimeofday")));
-
-notrace int
-__vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
-{
-	struct vvar_data *vvd = get_vvar_data();
-
-	if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
-		if (likely(tv != NULL)) {
-			union tstv_t {
-				struct __kernel_old_timespec ts;
-				struct __kernel_old_timeval tv;
-			} *tstv = (union tstv_t *) tv;
-			do_realtime_stick(vvd, &tstv->ts);
-			/*
-			 * Assign before dividing to ensure that the division is
-			 * done in the type of tv_usec, not tv_nsec.
-			 *
-			 * There cannot be > 1 billion usec in a second:
-			 * do_realtime() has already distributed such overflow
-			 * into tv_sec.  So we can assign it to an int safely.
-			 */
-			tstv->tv.tv_usec = tstv->ts.tv_nsec;
-			tstv->tv.tv_usec /= 1000;
-		}
-		if (unlikely(tz != NULL)) {
-			/* Avoid memcpy. Some old compilers fail to inline it */
-			tz->tz_minuteswest = vvd->tz_minuteswest;
-			tz->tz_dsttime = vvd->tz_dsttime;
-		}
-		return 0;
-	}
-	return vdso_fallback_gettimeofday(tv, tz);
-}
+#endif
diff --git a/arch/sparc/vdso/vdso-layout.lds.S b/arch/sparc/vdso/vdso-layout.lds.S
index d31e57e8a3bbf..180e5d0ee0717 100644
--- a/arch/sparc/vdso/vdso-layout.lds.S
+++ b/arch/sparc/vdso/vdso-layout.lds.S
@@ -4,15 +4,9 @@
  * This script controls its layout.
  */
 
-#if defined(BUILD_VDSO64)
-# define SHDR_SIZE 64
-#elif defined(BUILD_VDSO32)
-# define SHDR_SIZE 40
-#else
-# error unknown VDSO target
-#endif
-
-#define NUM_FAKE_SHDRS 7
+#include <vdso/datapage.h>
+#include <vdso/page.h>
+#include <asm/vdso/vsyscall.h>
 
 SECTIONS
 {
@@ -23,8 +17,7 @@ SECTIONS
 	 * segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries
 	 */
 
-	vvar_start = . -8192;
-	vvar_data = vvar_start;
+	VDSO_VVAR_SYMS
 
 	. = SIZEOF_HEADERS;
 
@@ -47,19 +40,8 @@ SECTIONS
 		*(.bss*)
 		*(.dynbss*)
 		*(.gnu.linkonce.b.*)
-
-		/*
-		 * Ideally this would live in a C file: kept in here for
-		 * compatibility with x86-64.
-		 */
-		VDSO_FAKE_SECTION_TABLE_START = .;
-		. = . + NUM_FAKE_SHDRS * SHDR_SIZE;
-		VDSO_FAKE_SECTION_TABLE_END = .;
 	}						:text
 
-	.fake_shstrtab	: { *(.fake_shstrtab) }		:text
-
-
 	.note		: { *(.note.*) }		:text	:note
 
 	.eh_frame_hdr	: { *(.eh_frame_hdr) }		:text	:eh_frame_hdr
diff --git a/arch/sparc/vdso/vdso.lds.S b/arch/sparc/vdso/vdso.lds.S
index 629ab6900df71..f3caa29a331c5 100644
--- a/arch/sparc/vdso/vdso.lds.S
+++ b/arch/sparc/vdso/vdso.lds.S
@@ -18,10 +18,8 @@ VERSION {
 	global:
 		clock_gettime;
 		__vdso_clock_gettime;
-		__vdso_clock_gettime_stick;
 		gettimeofday;
 		__vdso_gettimeofday;
-		__vdso_gettimeofday_stick;
 	local: *;
 	};
 }
diff --git a/arch/sparc/vdso/vdso2c.c b/arch/sparc/vdso/vdso2c.c
index dc81240aab6fd..e5c61214a0e28 100644
--- a/arch/sparc/vdso/vdso2c.c
+++ b/arch/sparc/vdso/vdso2c.c
@@ -58,28 +58,6 @@
 
 const char *outfilename;
 
-/* Symbols that we need in vdso2c. */
-enum {
-	sym_vvar_start,
-	sym_VDSO_FAKE_SECTION_TABLE_START,
-	sym_VDSO_FAKE_SECTION_TABLE_END,
-};
-
-struct vdso_sym {
-	const char *name;
-	int export;
-};
-
-struct vdso_sym required_syms[] = {
-	[sym_vvar_start] = {"vvar_start", 1},
-	[sym_VDSO_FAKE_SECTION_TABLE_START] = {
-		"VDSO_FAKE_SECTION_TABLE_START", 0
-	},
-	[sym_VDSO_FAKE_SECTION_TABLE_END] = {
-		"VDSO_FAKE_SECTION_TABLE_END", 0
-	},
-};
-
 __attribute__((format(printf, 1, 2))) __attribute__((noreturn))
 static void fail(const char *format, ...)
 {
@@ -119,8 +97,6 @@ static void fail(const char *format, ...)
 #define PUT_BE(x, val)					\
 	PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val))))
 
-#define NSYMS ARRAY_SIZE(required_syms)
-
 #define BITSFUNC3(name, bits, suffix) name##bits##suffix
 #define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix)
 #define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, )
diff --git a/arch/sparc/vdso/vdso2c.h b/arch/sparc/vdso/vdso2c.h
index 60d69acc748f2..bad6a0593f4ca 100644
--- a/arch/sparc/vdso/vdso2c.h
+++ b/arch/sparc/vdso/vdso2c.h
@@ -17,11 +17,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
 	unsigned long mapping_size;
 	int i;
 	unsigned long j;
-	ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr;
+	ELF(Shdr) *symtab_hdr = NULL;
 	ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr;
 	ELF(Dyn) *dyn = 0, *dyn_end = 0;
-	INT_BITS syms[NSYMS] = {};
-
 	ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff));
 
 	/* Walk the segment table. */
@@ -72,42 +70,6 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
 	if (!symtab_hdr)
 		fail("no symbol table\n");
 
-	strtab_hdr = raw_addr + GET_BE(&hdr->e_shoff) +
-		GET_BE(&hdr->e_shentsize) * GET_BE(&symtab_hdr->sh_link);
-
-	/* Walk the symbol table */
-	for (i = 0;
-	     i < GET_BE(&symtab_hdr->sh_size) / GET_BE(&symtab_hdr->sh_entsize);
-	     i++) {
-		int k;
-
-		ELF(Sym) *sym = raw_addr + GET_BE(&symtab_hdr->sh_offset) +
-			GET_BE(&symtab_hdr->sh_entsize) * i;
-		const char *name = raw_addr + GET_BE(&strtab_hdr->sh_offset) +
-			GET_BE(&sym->st_name);
-
-		for (k = 0; k < NSYMS; k++) {
-			if (!strcmp(name, required_syms[k].name)) {
-				if (syms[k]) {
-					fail("duplicate symbol %s\n",
-					     required_syms[k].name);
-				}
-
-				/*
-				 * Careful: we use negative addresses, but
-				 * st_value is unsigned, so we rely
-				 * on syms[k] being a signed type of the
-				 * correct width.
-				 */
-				syms[k] = GET_BE(&sym->st_value);
-			}
-		}
-	}
-
-	/* Validate mapping addresses. */
-	if (syms[sym_vvar_start] % 8192)
-		fail("vvar_begin must be a multiple of 8192\n");
-
 	if (!name) {
 		fwrite(stripped_addr, stripped_len, 1, outfile);
 		return;
@@ -133,10 +95,5 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
 	fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name);
 	fprintf(outfile, "\t.data = raw_data,\n");
 	fprintf(outfile, "\t.size = %lu,\n", mapping_size);
-	for (i = 0; i < NSYMS; i++) {
-		if (required_syms[i].export && syms[i])
-			fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n",
-				required_syms[i].name, (int64_t)syms[i]);
-	}
 	fprintf(outfile, "};\n");
 }
diff --git a/arch/sparc/vdso/vdso32/vdso32.lds.S b/arch/sparc/vdso/vdso32/vdso32.lds.S
index 218930fdff03d..a14e4f77e6f22 100644
--- a/arch/sparc/vdso/vdso32/vdso32.lds.S
+++ b/arch/sparc/vdso/vdso32/vdso32.lds.S
@@ -17,10 +17,10 @@ VERSION {
 	global:
 		clock_gettime;
 		__vdso_clock_gettime;
-		__vdso_clock_gettime_stick;
+		clock_gettime64;
+		__vdso_clock_gettime64;
 		gettimeofday;
 		__vdso_gettimeofday;
-		__vdso_gettimeofday_stick;
 	local: *;
 	};
 }
diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c
index c454689ce5fa0..60029d60f4d3a 100644
--- a/arch/sparc/vdso/vma.c
+++ b/arch/sparc/vdso/vma.c
@@ -16,17 +16,16 @@
 #include <linux/linkage.h>
 #include <linux/random.h>
 #include <linux/elf.h>
+#include <linux/vdso_datastore.h>
 #include <asm/cacheflush.h>
 #include <asm/spitfire.h>
 #include <asm/vdso.h>
-#include <asm/vvar.h>
 #include <asm/page.h>
 
-unsigned int __read_mostly vdso_enabled = 1;
+#include <vdso/datapage.h>
+#include <asm/vdso/vsyscall.h>
 
-static struct vm_special_mapping vvar_mapping = {
-	.name = "[vvar]"
-};
+unsigned int __read_mostly vdso_enabled = 1;
 
 #ifdef	CONFIG_SPARC64
 static struct vm_special_mapping vdso_mapping64 = {
@@ -40,207 +39,8 @@ static struct vm_special_mapping vdso_mapping32 = {
 };
 #endif
 
-struct vvar_data *vvar_data;
-
-struct vdso_elfinfo32 {
-	Elf32_Ehdr	*hdr;
-	Elf32_Sym	*dynsym;
-	unsigned long	dynsymsize;
-	const char	*dynstr;
-	unsigned long	text;
-};
-
-struct vdso_elfinfo64 {
-	Elf64_Ehdr	*hdr;
-	Elf64_Sym	*dynsym;
-	unsigned long	dynsymsize;
-	const char	*dynstr;
-	unsigned long	text;
-};
-
-struct vdso_elfinfo {
-	union {
-		struct vdso_elfinfo32 elf32;
-		struct vdso_elfinfo64 elf64;
-	} u;
-};
-
-static void *one_section64(struct vdso_elfinfo64 *e, const char *name,
-			   unsigned long *size)
-{
-	const char *snames;
-	Elf64_Shdr *shdrs;
-	unsigned int i;
-
-	shdrs = (void *)e->hdr + e->hdr->e_shoff;
-	snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
-	for (i = 1; i < e->hdr->e_shnum; i++) {
-		if (!strcmp(snames+shdrs[i].sh_name, name)) {
-			if (size)
-				*size = shdrs[i].sh_size;
-			return (void *)e->hdr + shdrs[i].sh_offset;
-		}
-	}
-	return NULL;
-}
-
-static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e)
-{
-	struct vdso_elfinfo64 *e = &_e->u.elf64;
-
-	e->hdr = image->data;
-	e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize);
-	e->dynstr = one_section64(e, ".dynstr", NULL);
-
-	if (!e->dynsym || !e->dynstr) {
-		pr_err("VDSO64: Missing symbol sections.\n");
-		return -ENODEV;
-	}
-	return 0;
-}
-
-static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name)
-{
-	unsigned int i;
-
-	for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) {
-		Elf64_Sym *s = &e->dynsym[i];
-		if (s->st_name == 0)
-			continue;
-		if (!strcmp(e->dynstr + s->st_name, name))
-			return s;
-	}
-	return NULL;
-}
-
-static int patchsym64(struct vdso_elfinfo *_e, const char *orig,
-		      const char *new)
-{
-	struct vdso_elfinfo64 *e = &_e->u.elf64;
-	Elf64_Sym *osym = find_sym64(e, orig);
-	Elf64_Sym *nsym = find_sym64(e, new);
-
-	if (!nsym || !osym) {
-		pr_err("VDSO64: Missing symbols.\n");
-		return -ENODEV;
-	}
-	osym->st_value = nsym->st_value;
-	osym->st_size = nsym->st_size;
-	osym->st_info = nsym->st_info;
-	osym->st_other = nsym->st_other;
-	osym->st_shndx = nsym->st_shndx;
-
-	return 0;
-}
-
-static void *one_section32(struct vdso_elfinfo32 *e, const char *name,
-			   unsigned long *size)
-{
-	const char *snames;
-	Elf32_Shdr *shdrs;
-	unsigned int i;
-
-	shdrs = (void *)e->hdr + e->hdr->e_shoff;
-	snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
-	for (i = 1; i < e->hdr->e_shnum; i++) {
-		if (!strcmp(snames+shdrs[i].sh_name, name)) {
-			if (size)
-				*size = shdrs[i].sh_size;
-			return (void *)e->hdr + shdrs[i].sh_offset;
-		}
-	}
-	return NULL;
-}
-
-static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e)
-{
-	struct vdso_elfinfo32 *e = &_e->u.elf32;
-
-	e->hdr = image->data;
-	e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize);
-	e->dynstr = one_section32(e, ".dynstr", NULL);
-
-	if (!e->dynsym || !e->dynstr) {
-		pr_err("VDSO32: Missing symbol sections.\n");
-		return -ENODEV;
-	}
-	return 0;
-}
-
-static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name)
-{
-	unsigned int i;
-
-	for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) {
-		Elf32_Sym *s = &e->dynsym[i];
-		if (s->st_name == 0)
-			continue;
-		if (!strcmp(e->dynstr + s->st_name, name))
-			return s;
-	}
-	return NULL;
-}
-
-static int patchsym32(struct vdso_elfinfo *_e, const char *orig,
-		      const char *new)
-{
-	struct vdso_elfinfo32 *e = &_e->u.elf32;
-	Elf32_Sym *osym = find_sym32(e, orig);
-	Elf32_Sym *nsym = find_sym32(e, new);
-
-	if (!nsym || !osym) {
-		pr_err("VDSO32: Missing symbols.\n");
-		return -ENODEV;
-	}
-	osym->st_value = nsym->st_value;
-	osym->st_size = nsym->st_size;
-	osym->st_info = nsym->st_info;
-	osym->st_other = nsym->st_other;
-	osym->st_shndx = nsym->st_shndx;
-
-	return 0;
-}
-
-static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e,
-			 bool elf64)
-{
-	if (elf64)
-		return find_sections64(image, e);
-	else
-		return find_sections32(image, e);
-}
-
-static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig,
-			    const char *new_target, bool elf64)
-{
-	if (elf64)
-		return patchsym64(e, orig, new_target);
-	else
-		return patchsym32(e, orig, new_target);
-}
-
-static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64)
-{
-	int err;
-
-	err = find_sections(image, e, elf64);
-	if (err)
-		return err;
-
-	err = patch_one_symbol(e,
-			       "__vdso_gettimeofday",
-			       "__vdso_gettimeofday_stick", elf64);
-	if (err)
-		return err;
-
-	return patch_one_symbol(e,
-				"__vdso_clock_gettime",
-				"__vdso_clock_gettime_stick", elf64);
-	return 0;
-}
-
 /*
- * Allocate pages for the vdso and vvar, and copy in the vdso text from the
+ * Allocate pages for the vdso and copy in the vdso text from the
  * kernel image.
  */
 static int __init init_vdso_image(const struct vdso_image *image,
@@ -248,16 +48,8 @@ static int __init init_vdso_image(const struct vdso_image *image,
 				  bool elf64)
 {
 	int cnpages = (image->size) / PAGE_SIZE;
-	struct page *dp, **dpp = NULL;
 	struct page *cp, **cpp = NULL;
-	struct vdso_elfinfo ei;
-	int i, dnpages = 0;
-
-	if (tlb_type != spitfire) {
-		int err = stick_patch(image, &ei, elf64);
-		if (err)
-			return err;
-	}
+	int i;
 
 	/*
 	 * First, the vdso text.  This is initialied data, an integral number of
@@ -280,31 +72,6 @@ static int __init init_vdso_image(const struct vdso_image *image,
 		copy_page(page_address(cp), image->data + i * PAGE_SIZE);
 	}
 
-	/*
-	 * Now the vvar page.  This is uninitialized data.
-	 */
-
-	if (vvar_data == NULL) {
-		dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1;
-		if (WARN_ON(dnpages != 1))
-			goto oom;
-		dpp = kzalloc_objs(struct page *, dnpages);
-		vvar_mapping.pages = dpp;
-
-		if (!dpp)
-			goto oom;
-
-		dp = alloc_page(GFP_KERNEL);
-		if (!dp)
-			goto oom;
-
-		dpp[0] = dp;
-		vvar_data = page_address(dp);
-		memset(vvar_data, 0, PAGE_SIZE);
-
-		vvar_data->seq = 0;
-	}
-
 	return 0;
  oom:
 	if (cpp != NULL) {
@@ -316,15 +83,6 @@ static int __init init_vdso_image(const struct vdso_image *image,
 		vdso_mapping->pages = NULL;
 	}
 
-	if (dpp != NULL) {
-		for (i = 0; i < dnpages; i++) {
-			if (dpp[i] != NULL)
-				__free_page(dpp[i]);
-		}
-		kfree(dpp);
-		vvar_mapping.pages = NULL;
-	}
-
 	pr_warn("Cannot allocate vdso\n");
 	vdso_enabled = 0;
 	return -ENOMEM;
@@ -359,9 +117,12 @@ static unsigned long vdso_addr(unsigned long start, unsigned int len)
 	return start + (offset << PAGE_SHIFT);
 }
 
+static_assert(VDSO_NR_PAGES == __VDSO_PAGES);
+
 static int map_vdso(const struct vdso_image *image,
 		struct vm_special_mapping *vdso_mapping)
 {
+	const size_t area_size = image->size + VDSO_NR_PAGES * PAGE_SIZE;
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long text_start, addr = 0;
@@ -374,23 +135,20 @@ static int map_vdso(const struct vdso_image *image,
 	 * region is free.
 	 */
 	if (current->flags & PF_RANDOMIZE) {
-		addr = get_unmapped_area(NULL, 0,
-					 image->size - image->sym_vvar_start,
-					 0, 0);
+		addr = get_unmapped_area(NULL, 0, area_size, 0, 0);
 		if (IS_ERR_VALUE(addr)) {
 			ret = addr;
 			goto up_fail;
 		}
-		addr = vdso_addr(addr, image->size - image->sym_vvar_start);
+		addr = vdso_addr(addr, area_size);
 	}
-	addr = get_unmapped_area(NULL, addr,
-				 image->size - image->sym_vvar_start, 0, 0);
+	addr = get_unmapped_area(NULL, addr, area_size, 0, 0);
 	if (IS_ERR_VALUE(addr)) {
 		ret = addr;
 		goto up_fail;
 	}
 
-	text_start = addr - image->sym_vvar_start;
+	text_start = addr + VDSO_NR_PAGES * PAGE_SIZE;
 	current->mm->context.vdso = (void __user *)text_start;
 
 	/*
@@ -408,11 +166,7 @@ static int map_vdso(const struct vdso_image *image,
 		goto up_fail;
 	}
 
-	vma = _install_special_mapping(mm,
-				       addr,
-				       -image->sym_vvar_start,
-				       VM_READ|VM_MAYREAD,
-				       &vvar_mapping);
+	vma = vdso_install_vvar_mapping(mm, addr);
 
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index 098cda44db225..77f752fc72d55 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -43,6 +43,7 @@ config UML
 	select THREAD_INFO_IN_TASK
 	select SPARSE_IRQ
 	select MMU_GATHER_RCU_TABLE_FREE
+	select XOR_BLOCKS_ARCH
 
 config MMU
 	bool
diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h
index 3b42b0f45bf6c..19e0608fb6497 100644
--- a/arch/um/include/asm/pgtable.h
+++ b/arch/um/include/asm/pgtable.h
@@ -34,9 +34,6 @@
 
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 
-/* zero page used for uninitialized stuff */
-extern unsigned long *empty_zero_page;
-
 /* Just any arbitrary offset to the start of the vmalloc VM area: the
  * current 8MB value just means that there will be a 8MB "hole" after the
  * physical memory until the kernel virtual memory starts.  That means that
@@ -74,12 +71,6 @@ extern unsigned long *empty_zero_page;
  * get..
  */
 
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-#define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page)
-
 #define pte_clear(mm, addr, xp) pte_set_val(*(xp), (phys_t) 0, __pgprot(_PAGE_NEEDSYNC))
 
 #define pmd_none(x)	(!((unsigned long)pmd_val(x) & ~_PAGE_NEEDSYNC))
diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h
deleted file mode 100644
index 647fae200c5d3..0000000000000
--- a/arch/um/include/asm/xor.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_UM_XOR_H
-#define _ASM_UM_XOR_H
-
-#ifdef CONFIG_64BIT
-#undef CONFIG_X86_32
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_sse_pf64))
-#else
-#define CONFIG_X86_32 1
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_8regs))
-#endif
-
-#include <asm/cpufeature.h>
-#include <../../x86/include/asm/xor.h>
-#include <linux/time-internal.h>
-
-#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
-#undef XOR_SELECT_TEMPLATE
-/* pick an arbitrary one - measuring isn't possible with inf-cpu */
-#define XOR_SELECT_TEMPLATE(x)	\
-	(time_travel_mode == TT_MODE_INFCPU ? TT_CPU_INF_XOR_DEFAULT : x)
-#endif
-
-#endif
diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h
index 38321188c04c4..9812efd14ec0e 100644
--- a/arch/um/include/shared/kern_util.h
+++ b/arch/um/include/shared/kern_util.h
@@ -38,7 +38,6 @@ extern void timer_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs
 extern void uml_pm_wake(void);
 
 extern int start_uml(void);
-extern void paging_init(void);
 
 extern void uml_cleanup(void);
 extern void do_uml_exitcalls(void);
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 89c8c8b94a796..1eef0e42ef5db 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -44,10 +44,6 @@ __section(".kasan_init") __used
 = kasan_init;
 #endif
 
-/* allocated in paging_init, zeroed in mem_init, and unchanged thereafter */
-unsigned long *empty_zero_page = NULL;
-EXPORT_SYMBOL(empty_zero_page);
-
 /*
  * Initialized during boot, and readonly for initializing page tables
  * afterwards
@@ -65,9 +61,6 @@ void __init arch_mm_preinit(void)
 	/* Safe to call after jump_label_init(). Enables KASAN. */
 	kasan_init_generic();
 
-	/* clear the zero-page */
-	memset(empty_zero_page, 0, PAGE_SIZE);
-
 	/* Map in the area just after the brk now that kmalloc is about
 	 * to be turned on.
 	 */
@@ -89,15 +82,6 @@ void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
 	max_zone_pfns[ZONE_NORMAL] = high_physmem >> PAGE_SHIFT;
 }
 
-void __init paging_init(void)
-{
-	empty_zero_page = (unsigned long *) memblock_alloc_low(PAGE_SIZE,
-							       PAGE_SIZE);
-	if (!empty_zero_page)
-		panic("%s: Failed to allocate %lu bytes align=%lx\n",
-		      __func__, PAGE_SIZE, PAGE_SIZE);
-}
-
 /*
  * This can't do anything because nothing in the kernel image can be freed
  * since it's not in kernel physical memory.
diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c
index e2b24e1ecfa64..2141f5f1f5a20 100644
--- a/arch/um/kernel/um_arch.c
+++ b/arch/um/kernel/um_arch.c
@@ -413,7 +413,6 @@ void __init setup_arch(char **cmdline_p)
 	uml_dtb_init();
 	read_initrd();
 
-	paging_init();
 	strscpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
 	*cmdline_p = command_line;
 	setup_hostinfo(host_info, sizeof host_info);
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e2df1b147184a..8038b26ae99e0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -141,6 +141,7 @@ config X86
 	select ARCH_USE_SYM_ANNOTATIONS
 	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
 	select ARCH_WANT_DEFAULT_BPF_JIT	if X86_64
+	select ARCH_WANTS_CLOCKSOURCE_READ_INLINE	if X86_64
 	select ARCH_WANTS_DYNAMIC_TASK_STRUCT
 	select ARCH_WANTS_NO_INSTR
 	select ARCH_WANT_GENERAL_HUGETLB
@@ -163,6 +164,7 @@ config X86
 	select EDAC_SUPPORT
 	select GENERIC_CLOCKEVENTS_BROADCAST	if X86_64 || (X86_32 && X86_LOCAL_APIC)
 	select GENERIC_CLOCKEVENTS_BROADCAST_IDLE	if GENERIC_CLOCKEVENTS_BROADCAST
+	select GENERIC_CLOCKEVENTS_COUPLED_INLINE	if X86_64
 	select GENERIC_CLOCKEVENTS_MIN_ADJUST
 	select GENERIC_CMOS_UPDATE
 	select GENERIC_CPU_AUTOPROBE
@@ -336,6 +338,7 @@ config X86
 	select ARCH_SUPPORTS_SCHED_CLUSTER	if SMP
 	select ARCH_SUPPORTS_SCHED_MC		if SMP
 	select HAVE_SINGLE_FTRACE_DIRECT_OPS	if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	select XOR_BLOCKS_ARCH
 
 config INSTRUCTION_DECODER
 	def_bool y
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 5f881460a8b5c..46fec0b08487a 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -48,7 +48,7 @@ endif
 
 # How to compile the 16-bit code.  Note we always compile for -march=i386;
 # that way we can complain to the user if the CPU is insufficient.
-REALMODE_CFLAGS	:= -std=gnu11 -fms-extensions -m16 -g -Os \
+REALMODE_CFLAGS	:= $(CC_FLAGS_DIALECT) -m16 -g -Os \
 		   -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
 		   -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \
 		   -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
@@ -59,10 +59,6 @@ REALMODE_CFLAGS += -fno-stack-protector
 REALMODE_CFLAGS += -Wno-address-of-packed-member
 REALMODE_CFLAGS += $(cc_stack_align4)
 REALMODE_CFLAGS += $(CLANG_FLAGS)
-ifdef CONFIG_CC_IS_CLANG
-REALMODE_CFLAGS += -Wno-gnu
-REALMODE_CFLAGS += -Wno-microsoft-anon-tag
-endif
 export REALMODE_CFLAGS
 
 # BITS is used as extension for files which are available in a 32 bit
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index b8b2b7bea1d31..07e0e64b9a986 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -25,7 +25,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
 # avoid errors with '-march=i386', and future flags may depend on the target to
 # be valid.
 KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS)
-KBUILD_CFLAGS += -std=gnu11 -fms-extensions
+KBUILD_CFLAGS += $(CC_FLAGS_DIALECT)
 KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
 KBUILD_CFLAGS += -Wundef
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
@@ -36,10 +36,6 @@ KBUILD_CFLAGS += -mno-mmx -mno-sse
 KBUILD_CFLAGS += -ffreestanding -fshort-wchar
 KBUILD_CFLAGS += -fno-stack-protector
 KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member)
-ifdef CONFIG_CC_IS_CLANG
-KBUILD_CFLAGS += -Wno-gnu
-KBUILD_CFLAGS += -Wno-microsoft-anon-tag
-endif
 KBUILD_CFLAGS += -Wno-pointer-sign
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 KBUILD_CFLAGS += -D__DISABLE_EXPORTS
diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
index e468476e9e4a0..c6512f2ea31e8 100644
--- a/arch/x86/boot/compressed/sev.c
+++ b/arch/x86/boot/compressed/sev.c
@@ -198,11 +198,11 @@ bool sev_es_check_ghcb_fault(unsigned long address)
 #endif
 
 /*
- * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
+ * SNP_FEATURES_IMPL is the mask of SNP features that are implemented
  * by the guest kernel. As and when a new feature is implemented in the
  * guest kernel, a corresponding bit should be added to the mask.
  */
-#define SNP_FEATURES_PRESENT	(MSR_AMD64_SNP_DEBUG_SWAP |	\
+#define SNP_FEATURES_IMPL	(MSR_AMD64_SNP_DEBUG_SWAP |	\
 				 MSR_AMD64_SNP_SECURE_TSC |	\
 				 SNP_FEATURE_SECURE_AVIC)
 
@@ -211,7 +211,7 @@ u64 snp_get_unsupported_features(u64 status)
 	if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
 		return 0;
 
-	return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
+	return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_IMPL;
 }
 
 void snp_check_features(void)
diff --git a/arch/x86/coco/tdx/debug.c b/arch/x86/coco/tdx/debug.c
index cef847c8bb67f..8e477db4ce0a0 100644
--- a/arch/x86/coco/tdx/debug.c
+++ b/arch/x86/coco/tdx/debug.c
@@ -7,21 +7,21 @@
 #include <linux/printk.h>
 #include <asm/tdx.h>
 
-#define DEF_TDX_ATTR_NAME(_name) [TDX_ATTR_##_name##_BIT] = __stringify(_name)
+#define DEF_TDX_TD_ATTR_NAME(_name) [TDX_TD_ATTR_##_name##_BIT] = __stringify(_name)
 
 static __initdata const char *tdx_attributes[] = {
-	DEF_TDX_ATTR_NAME(DEBUG),
-	DEF_TDX_ATTR_NAME(HGS_PLUS_PROF),
-	DEF_TDX_ATTR_NAME(PERF_PROF),
-	DEF_TDX_ATTR_NAME(PMT_PROF),
-	DEF_TDX_ATTR_NAME(ICSSD),
-	DEF_TDX_ATTR_NAME(LASS),
-	DEF_TDX_ATTR_NAME(SEPT_VE_DISABLE),
-	DEF_TDX_ATTR_NAME(MIGRTABLE),
-	DEF_TDX_ATTR_NAME(PKS),
-	DEF_TDX_ATTR_NAME(KL),
-	DEF_TDX_ATTR_NAME(TPA),
-	DEF_TDX_ATTR_NAME(PERFMON),
+	DEF_TDX_TD_ATTR_NAME(DEBUG),
+	DEF_TDX_TD_ATTR_NAME(HGS_PLUS_PROF),
+	DEF_TDX_TD_ATTR_NAME(PERF_PROF),
+	DEF_TDX_TD_ATTR_NAME(PMT_PROF),
+	DEF_TDX_TD_ATTR_NAME(ICSSD),
+	DEF_TDX_TD_ATTR_NAME(LASS),
+	DEF_TDX_TD_ATTR_NAME(SEPT_VE_DISABLE),
+	DEF_TDX_TD_ATTR_NAME(MIGRATABLE),
+	DEF_TDX_TD_ATTR_NAME(PKS),
+	DEF_TDX_TD_ATTR_NAME(KL),
+	DEF_TDX_TD_ATTR_NAME(TPA),
+	DEF_TDX_TD_ATTR_NAME(PERFMON),
 };
 
 #define DEF_TD_CTLS_NAME(_name) [TD_CTLS_##_name##_BIT] = __stringify(_name)
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 7b2833705d475..186915a17c50f 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -238,14 +238,14 @@ static void __noreturn tdx_panic(const char *msg)
  *
  * TDX 1.0 does not allow the guest to disable SEPT #VE on its own. The VMM
  * controls if the guest will receive such #VE with TD attribute
- * TDX_ATTR_SEPT_VE_DISABLE.
+ * TDX_TD_ATTR_SEPT_VE_DISABLE.
  *
  * Newer TDX modules allow the guest to control if it wants to receive SEPT
  * violation #VEs.
  *
  * Check if the feature is available and disable SEPT #VE if possible.
  *
- * If the TD is allowed to disable/enable SEPT #VEs, the TDX_ATTR_SEPT_VE_DISABLE
+ * If the TD is allowed to disable/enable SEPT #VEs, the TDX_TD_ATTR_SEPT_VE_DISABLE
  * attribute is no longer reliable. It reflects the initial state of the
  * control for the TD, but it will not be updated if someone (e.g. bootloader)
  * changes it before the kernel starts. Kernel must check TDCS_TD_CTLS bit to
@@ -254,14 +254,14 @@ static void __noreturn tdx_panic(const char *msg)
 static void disable_sept_ve(u64 td_attr)
 {
 	const char *msg = "TD misconfiguration: SEPT #VE has to be disabled";
-	bool debug = td_attr & TDX_ATTR_DEBUG;
+	bool debug = td_attr & TDX_TD_ATTR_DEBUG;
 	u64 config, controls;
 
 	/* Is this TD allowed to disable SEPT #VE */
 	tdg_vm_rd(TDCS_CONFIG_FLAGS, &config);
 	if (!(config & TDCS_CONFIG_FLEXIBLE_PENDING_VE)) {
 		/* No SEPT #VE controls for the guest: check the attribute */
-		if (td_attr & TDX_ATTR_SEPT_VE_DISABLE)
+		if (td_attr & TDX_TD_ATTR_SEPT_VE_DISABLE)
 			return;
 
 		/* Relax SEPT_VE_DISABLE check for debug TD for backtraces */
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 7d7310cdf8b0a..269f7d808be4e 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -230,6 +230,7 @@ CONFIG_EEEPC_LAPTOP=y
 CONFIG_AMD_IOMMU=y
 CONFIG_INTEL_IOMMU=y
 # CONFIG_INTEL_IOMMU_DEFAULT_ON is not set
+CONFIG_IRQ_REMAP=y
 CONFIG_EXT4_FS=y
 CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
diff --git a/arch/x86/entry/vdso/vdso32/Makefile b/arch/x86/entry/vdso/vdso32/Makefile
index add6afb484ba4..ded4fc6a48cdc 100644
--- a/arch/x86/entry/vdso/vdso32/Makefile
+++ b/arch/x86/entry/vdso/vdso32/Makefile
@@ -15,6 +15,10 @@ flags-y			:= -DBUILD_VDSO32 -m32 -mregparm=0
 flags-$(CONFIG_X86_64)	+= -include $(src)/fake_32bit_build.h
 flags-remove-y          := -m64
 
+# Checker flags
+CHECKFLAGS := $(subst -m64,-m32,$(CHECKFLAGS))
+CHECKFLAGS := $(subst -D__x86_64__,-D__i386__,$(CHECKFLAGS))
+
 # The location of this include matters!
 include $(src)/../common/Makefile.include
 
diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
index db1b15f686e32..bc3e549795c3f 100644
--- a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
+++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
@@ -11,6 +11,7 @@
 #undef CONFIG_PGTABLE_LEVELS
 #undef CONFIG_ILLEGAL_POINTER_VALUE
 #undef CONFIG_SPARSEMEM_VMEMMAP
+#undef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
 #undef CONFIG_NR_CPUS
 #undef CONFIG_PARAVIRT_XXL
 
diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
index 4bd1e271bb22b..ea36de9fa8641 100644
--- a/arch/x86/entry/vsyscall/vsyscall_64.c
+++ b/arch/x86/entry/vsyscall/vsyscall_64.c
@@ -23,7 +23,7 @@
  * soon be no new userspace code that will ever use a vsyscall.
  *
  * The code in this file emulates vsyscalls when notified of a page
- * fault to a vsyscall address.
+ * fault or a general protection fault to a vsyscall address.
  */
 
 #include <linux/kernel.h>
@@ -62,6 +62,11 @@ static int __init vsyscall_setup(char *str)
 		else
 			return -EINVAL;
 
+		if (cpu_feature_enabled(X86_FEATURE_LASS) && vsyscall_mode == EMULATE) {
+			setup_clear_cpu_cap(X86_FEATURE_LASS);
+			pr_warn_once("x86/cpu: Disabling LASS due to vsyscall=emulate\n");
+		}
+
 		return 0;
 	}
 
@@ -111,48 +116,17 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size)
 	}
 }
 
-bool emulate_vsyscall(unsigned long error_code,
-		      struct pt_regs *regs, unsigned long address)
+static bool __emulate_vsyscall(struct pt_regs *regs, unsigned long address)
 {
 	unsigned long caller;
 	int vsyscall_nr, syscall_nr, tmp;
 	long ret;
 	unsigned long orig_dx;
 
-	/* Write faults or kernel-privilege faults never get fixed up. */
-	if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER)
+	/* Confirm that the fault happened in 64-bit user mode */
+	if (!user_64bit_mode(regs))
 		return false;
 
-	/*
-	 * Assume that faults at regs->ip are because of an
-	 * instruction fetch. Return early and avoid
-	 * emulation for faults during data accesses:
-	 */
-	if (address != regs->ip) {
-		/* Failed vsyscall read */
-		if (vsyscall_mode == EMULATE)
-			return false;
-
-		/*
-		 * User code tried and failed to read the vsyscall page.
-		 */
-		warn_bad_vsyscall(KERN_INFO, regs, "vsyscall read attempt denied -- look up the vsyscall kernel parameter if you need a workaround");
-		return false;
-	}
-
-	/*
-	 * X86_PF_INSTR is only set when NX is supported.  When
-	 * available, use it to double-check that the emulation code
-	 * is only being used for instruction fetches:
-	 */
-	if (cpu_feature_enabled(X86_FEATURE_NX))
-		WARN_ON_ONCE(!(error_code & X86_PF_INSTR));
-
-	/*
-	 * No point in checking CS -- the only way to get here is a user mode
-	 * trap to a high address, which means that we're in 64-bit user code.
-	 */
-
 	if (vsyscall_mode == NONE) {
 		warn_bad_vsyscall(KERN_INFO, regs,
 				  "vsyscall attempted with vsyscall=none");
@@ -280,6 +254,53 @@ bool emulate_vsyscall(unsigned long error_code,
 	return true;
 }
 
+bool emulate_vsyscall_pf(unsigned long error_code, struct pt_regs *regs,
+			 unsigned long address)
+{
+	/* Write faults or kernel-privilege faults never get fixed up. */
+	if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER)
+		return false;
+
+	/*
+	 * Assume that faults at regs->ip are because of an instruction
+	 * fetch. Return early and avoid emulation for faults during
+	 * data accesses:
+	 */
+	if (address != regs->ip) {
+		/* Failed vsyscall read */
+		if (vsyscall_mode == EMULATE)
+			return false;
+
+		/* User code tried and failed to read the vsyscall page. */
+		warn_bad_vsyscall(KERN_INFO, regs,
+				  "vsyscall read attempt denied -- look up the vsyscall kernel parameter if you need a workaround");
+		return false;
+	}
+
+	/*
+	 * X86_PF_INSTR is only set when NX is supported.  When
+	 * available, use it to double-check that the emulation code
+	 * is only being used for instruction fetches:
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_NX))
+		WARN_ON_ONCE(!(error_code & X86_PF_INSTR));
+
+	return __emulate_vsyscall(regs, address);
+}
+
+bool emulate_vsyscall_gp(struct pt_regs *regs)
+{
+	/* Without LASS, vsyscall accesses are expected to generate a #PF */
+	if (!cpu_feature_enabled(X86_FEATURE_LASS))
+		return false;
+
+	/* Emulate only if the RIP points to the vsyscall address */
+	if (!is_vsyscall_vaddr(regs->ip))
+		return false;
+
+	return __emulate_vsyscall(regs, regs->ip);
+}
+
 /*
  * A pseudo VMA to allow ptrace access for the vsyscall page.  This only
  * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index aca89f23d2e00..eeb607b84ddae 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -32,6 +32,13 @@ static u32 ibs_caps;
 /* attr.config2 */
 #define IBS_SW_FILTER_MASK	1
 
+/* attr.config1 */
+#define IBS_OP_CONFIG1_LDLAT_MASK		(0xFFFULL <<  0)
+#define IBS_OP_CONFIG1_STRMST_MASK		    (1ULL << 12)
+#define IBS_OP_CONFIG1_STRMST_SHIFT			    (12)
+
+#define IBS_FETCH_CONFIG1_FETCHLAT_MASK		(0x7FFULL <<  0)
+
 /*
  * IBS states:
  *
@@ -83,9 +90,11 @@ struct cpu_perf_ibs {
 struct perf_ibs {
 	struct pmu			pmu;
 	unsigned int			msr;
+	unsigned int			msr2;
 	u64				config_mask;
 	u64				cnt_mask;
 	u64				enable_mask;
+	u64				disable_mask;
 	u64				valid_mask;
 	u16				min_period;
 	u64				max_period;
@@ -274,7 +283,23 @@ static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs,
 {
 	return perf_ibs == &perf_ibs_op &&
 	       (ibs_caps & IBS_CAPS_OPLDLAT) &&
-	       (event->attr.config1 & 0xFFF);
+	       (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK);
+}
+
+static bool perf_ibs_fetch_lat_event(struct perf_ibs *perf_ibs,
+				     struct perf_event *event)
+{
+	return perf_ibs == &perf_ibs_fetch &&
+	       (ibs_caps & IBS_CAPS_FETCHLAT) &&
+	       (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK);
+}
+
+static bool perf_ibs_strmst_event(struct perf_ibs *perf_ibs,
+				  struct perf_event *event)
+{
+	return perf_ibs == &perf_ibs_op &&
+	       (ibs_caps & IBS_CAPS_STRMST_RMTSOCKET) &&
+	       (event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK);
 }
 
 static int perf_ibs_init(struct perf_event *event)
@@ -289,6 +314,8 @@ static int perf_ibs_init(struct perf_event *event)
 		return -ENOENT;
 
 	config = event->attr.config;
+	hwc->extra_reg.config = 0;
+	hwc->extra_reg.reg = 0;
 
 	if (event->pmu != &perf_ibs->pmu)
 		return -ENOENT;
@@ -304,15 +331,44 @@ static int perf_ibs_init(struct perf_event *event)
 	    event->attr.exclude_idle)
 		return -EINVAL;
 
-	if (!(event->attr.config2 & IBS_SW_FILTER_MASK) &&
-	    (event->attr.exclude_kernel || event->attr.exclude_user ||
-	     event->attr.exclude_hv))
-		return -EINVAL;
-
 	ret = validate_group(event);
 	if (ret)
 		return ret;
 
+	if (perf_allow_kernel())
+		hwc->flags |= PERF_X86_EVENT_UNPRIVILEGED;
+
+	if (ibs_caps & IBS_CAPS_DIS) {
+		hwc->extra_reg.config &= ~perf_ibs->disable_mask;
+		hwc->extra_reg.reg = perf_ibs->msr2;
+	}
+
+	if (ibs_caps & IBS_CAPS_BIT63_FILTER) {
+		if (perf_ibs == &perf_ibs_fetch) {
+			if (event->attr.exclude_kernel) {
+				hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_1;
+				hwc->extra_reg.reg = perf_ibs->msr2;
+			}
+			if (event->attr.exclude_user) {
+				hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_0;
+				hwc->extra_reg.reg = perf_ibs->msr2;
+			}
+		} else {
+			if (event->attr.exclude_kernel) {
+				hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_1;
+				hwc->extra_reg.reg = perf_ibs->msr2;
+			}
+			if (event->attr.exclude_user) {
+				hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_0;
+				hwc->extra_reg.reg = perf_ibs->msr2;
+			}
+		}
+	} else if (!(event->attr.config2 & IBS_SW_FILTER_MASK) &&
+		   (event->attr.exclude_kernel || event->attr.exclude_user ||
+		    event->attr.exclude_hv)) {
+		return -EINVAL;
+	}
+
 	if (hwc->sample_period) {
 		if (config & perf_ibs->cnt_mask)
 			/* raw max_cnt may not be set */
@@ -349,14 +405,37 @@ static int perf_ibs_init(struct perf_event *event)
 	}
 
 	if (perf_ibs_ldlat_event(perf_ibs, event)) {
-		u64 ldlat = event->attr.config1 & 0xFFF;
+		u64 ldlat = event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK;
 
 		if (ldlat < 128 || ldlat > 2048)
 			return -EINVAL;
 		ldlat >>= 7;
 
-		config |= (ldlat - 1) << 59;
-		config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN;
+		config |= (ldlat - 1) << IBS_OP_LDLAT_THRSH_SHIFT;
+
+		config |= IBS_OP_LDLAT_EN;
+		if (cpu_feature_enabled(X86_FEATURE_ZEN5))
+			config |= IBS_OP_L3MISSONLY;
+	}
+
+	if (perf_ibs_fetch_lat_event(perf_ibs, event)) {
+		u64 fetchlat = event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK;
+
+		if (fetchlat < 128 || fetchlat > 1920)
+			return -EINVAL;
+		fetchlat >>= 7;
+
+		hwc->extra_reg.reg = perf_ibs->msr2;
+		hwc->extra_reg.config |= fetchlat << IBS_FETCH_2_FETCHLAT_FILTER_SHIFT;
+	}
+
+	if (perf_ibs_strmst_event(perf_ibs, event)) {
+		u64 strmst = event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK;
+
+		strmst >>= IBS_OP_CONFIG1_STRMST_SHIFT;
+
+		hwc->extra_reg.reg = perf_ibs->msr2;
+		hwc->extra_reg.config |= strmst << IBS_OP_2_STRM_ST_FILTER_SHIFT;
 	}
 
 	/*
@@ -439,6 +518,9 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
 		wrmsrq(hwc->config_base, tmp & ~perf_ibs->enable_mask);
 
 	wrmsrq(hwc->config_base, tmp | perf_ibs->enable_mask);
+
+	if (hwc->extra_reg.reg)
+		wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config);
 }
 
 /*
@@ -451,6 +533,11 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
 					  struct hw_perf_event *hwc, u64 config)
 {
+	if (ibs_caps & IBS_CAPS_DIS) {
+		wrmsrq(hwc->extra_reg.reg, perf_ibs->disable_mask);
+		return;
+	}
+
 	config &= ~perf_ibs->cnt_mask;
 	if (boot_cpu_data.x86 == 0x10)
 		wrmsrq(hwc->config_base, config);
@@ -487,6 +574,14 @@ static void perf_ibs_start(struct perf_event *event, int flags)
 	}
 	config |= period >> 4;
 
+	/*
+	 * Reset the IBS_{FETCH|OP}_CTL MSR before updating pcpu->state.
+	 * Doing so prevents a race condition in which an NMI due to other
+	 * source might accidentally activate the event before we enable
+	 * it ourselves.
+	 */
+	perf_ibs_disable_event(perf_ibs, hwc, 0);
+
 	/*
 	 * Set STARTED before enabling the hardware, such that a subsequent NMI
 	 * must observe it.
@@ -631,6 +726,11 @@ PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11");
 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1");
 PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1");
 PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1");
+PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_format, "config1:0-10");
+PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_cap, "1");
+PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_format, "config1:12");
+PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_cap, "1");
+PMU_EVENT_ATTR_STRING(rmtsocket, ibs_op_rmtsocket_cap, "1");
 
 static umode_t
 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i)
@@ -638,6 +738,24 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int
 	return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0;
 }
 
+static umode_t
+ibs_fetch_lat_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	return ibs_caps & IBS_CAPS_FETCHLAT ? attr->mode : 0;
+}
+
+static umode_t
+ibs_op_strmst_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0;
+}
+
+static umode_t
+ibs_op_rmtsocket_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0;
+}
+
 static umode_t
 ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
@@ -666,6 +784,16 @@ static struct attribute *zen4_ibs_extensions_attrs[] = {
 	NULL,
 };
 
+static struct attribute *ibs_fetch_lat_format_attrs[] = {
+	&ibs_fetch_lat_format.attr.attr,
+	NULL,
+};
+
+static struct attribute *ibs_fetch_lat_cap_attrs[] = {
+	&ibs_fetch_lat_cap.attr.attr,
+	NULL,
+};
+
 static struct attribute *ibs_op_ldlat_cap_attrs[] = {
 	&ibs_op_ldlat_cap.attr.attr,
 	NULL,
@@ -676,6 +804,16 @@ static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = {
 	NULL,
 };
 
+static struct attribute *ibs_op_strmst_cap_attrs[] = {
+	&ibs_op_strmst_cap.attr.attr,
+	NULL,
+};
+
+static struct attribute *ibs_op_rmtsocket_cap_attrs[] = {
+	&ibs_op_rmtsocket_cap.attr.attr,
+	NULL,
+};
+
 static struct attribute_group group_fetch_formats = {
 	.name = "format",
 	.attrs = fetch_attrs,
@@ -693,6 +831,18 @@ static struct attribute_group group_zen4_ibs_extensions = {
 	.is_visible = zen4_ibs_extensions_is_visible,
 };
 
+static struct attribute_group group_ibs_fetch_lat_cap = {
+	.name = "caps",
+	.attrs = ibs_fetch_lat_cap_attrs,
+	.is_visible = ibs_fetch_lat_is_visible,
+};
+
+static struct attribute_group group_ibs_fetch_lat_format = {
+	.name = "format",
+	.attrs = ibs_fetch_lat_format_attrs,
+	.is_visible = ibs_fetch_lat_is_visible,
+};
+
 static struct attribute_group group_ibs_op_ldlat_cap = {
 	.name = "caps",
 	.attrs = ibs_op_ldlat_cap_attrs,
@@ -705,6 +855,18 @@ static struct attribute_group group_ibs_op_dtlb_pgsize_cap = {
 	.is_visible = ibs_op_dtlb_pgsize_is_visible,
 };
 
+static struct attribute_group group_ibs_op_strmst_cap = {
+	.name = "caps",
+	.attrs = ibs_op_strmst_cap_attrs,
+	.is_visible = ibs_op_strmst_is_visible,
+};
+
+static struct attribute_group group_ibs_op_rmtsocket_cap = {
+	.name = "caps",
+	.attrs = ibs_op_rmtsocket_cap_attrs,
+	.is_visible = ibs_op_rmtsocket_is_visible,
+};
+
 static const struct attribute_group *fetch_attr_groups[] = {
 	&group_fetch_formats,
 	&empty_caps_group,
@@ -714,6 +876,8 @@ static const struct attribute_group *fetch_attr_groups[] = {
 static const struct attribute_group *fetch_attr_update[] = {
 	&group_fetch_l3missonly,
 	&group_zen4_ibs_extensions,
+	&group_ibs_fetch_lat_cap,
+	&group_ibs_fetch_lat_format,
 	NULL,
 };
 
@@ -748,6 +912,11 @@ static struct attribute *ibs_op_ldlat_format_attrs[] = {
 	NULL,
 };
 
+static struct attribute *ibs_op_strmst_format_attrs[] = {
+	&ibs_op_strmst_format.attr.attr,
+	NULL,
+};
+
 static struct attribute_group group_cnt_ctl = {
 	.name = "format",
 	.attrs = cnt_ctl_attrs,
@@ -772,6 +941,12 @@ static struct attribute_group group_ibs_op_ldlat_format = {
 	.is_visible = ibs_op_ldlat_is_visible,
 };
 
+static struct attribute_group group_ibs_op_strmst_format = {
+	.name = "format",
+	.attrs = ibs_op_strmst_format_attrs,
+	.is_visible = ibs_op_strmst_is_visible,
+};
+
 static const struct attribute_group *op_attr_update[] = {
 	&group_cnt_ctl,
 	&group_op_l3missonly,
@@ -779,6 +954,9 @@ static const struct attribute_group *op_attr_update[] = {
 	&group_ibs_op_ldlat_cap,
 	&group_ibs_op_ldlat_format,
 	&group_ibs_op_dtlb_pgsize_cap,
+	&group_ibs_op_strmst_cap,
+	&group_ibs_op_strmst_format,
+	&group_ibs_op_rmtsocket_cap,
 	NULL,
 };
 
@@ -795,6 +973,7 @@ static struct perf_ibs perf_ibs_fetch = {
 		.check_period	= perf_ibs_check_period,
 	},
 	.msr			= MSR_AMD64_IBSFETCHCTL,
+	.msr2			= MSR_AMD64_IBSFETCHCTL2,
 	.config_mask		= IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN,
 	.cnt_mask		= IBS_FETCH_MAX_CNT,
 	.enable_mask		= IBS_FETCH_ENABLE,
@@ -820,6 +999,7 @@ static struct perf_ibs perf_ibs_op = {
 		.check_period	= perf_ibs_check_period,
 	},
 	.msr			= MSR_AMD64_IBSOPCTL,
+	.msr2			= MSR_AMD64_IBSOPCTL2,
 	.config_mask		= IBS_OP_MAX_CNT,
 	.cnt_mask		= IBS_OP_MAX_CNT | IBS_OP_CUR_CNT |
 				  IBS_OP_CUR_CNT_RAND,
@@ -1155,7 +1335,8 @@ static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs,
 {
 	if (event->attr.sample_type & PERF_SAMPLE_RAW ||
 	    perf_ibs_is_mem_sample_type(perf_ibs, event) ||
-	    perf_ibs_ldlat_event(perf_ibs, event))
+	    perf_ibs_ldlat_event(perf_ibs, event) ||
+	    perf_ibs_fetch_lat_event(perf_ibs, event))
 		return perf_ibs->offset_max;
 	else if (check_rip)
 		return 3;
@@ -1190,7 +1371,7 @@ static bool perf_ibs_is_kernel_br_target(struct perf_event *event,
 			op_data.op_brn_ret && kernel_ip(br_target));
 }
 
-static bool perf_ibs_swfilt_discard(struct perf_ibs *perf_ibs, struct perf_event *event,
+static bool perf_ibs_discard_sample(struct perf_ibs *perf_ibs, struct perf_event *event,
 				    struct pt_regs *regs, struct perf_ibs_data *ibs_data,
 				    int br_target_idx)
 {
@@ -1214,12 +1395,10 @@ static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs,
 				   struct perf_ibs_data *ibs_data)
 {
 	if (perf_ibs == &perf_ibs_op) {
-		ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18);
 		ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0;
 		return;
 	}
 
-	ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52);
 	ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0;
 }
 
@@ -1293,8 +1472,20 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 		 * within [128, 2048] range.
 		 */
 		if (!op_data3.ld_op || !op_data3.dc_miss ||
-		    op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF))
+		    op_data3.dc_miss_lat <= (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK)) {
+			throttle = perf_event_account_interrupt(event);
 			goto out;
+		}
+	}
+
+	if (perf_ibs_fetch_lat_event(perf_ibs, event)) {
+		union ibs_fetch_ctl fetch_ctl;
+
+		fetch_ctl.val = ibs_data.regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)];
+		if (fetch_ctl.fetch_lat < (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK)) {
+			throttle = perf_event_account_interrupt(event);
+			goto out;
+		}
 	}
 
 	/*
@@ -1326,15 +1517,18 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 		regs.flags &= ~PERF_EFLAGS_EXACT;
 	} else {
 		/* Workaround for erratum #1197 */
-		if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1]))
+		if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) {
+			throttle = perf_event_account_interrupt(event);
 			goto out;
+		}
 
 		set_linear_ip(&regs, ibs_data.regs[1]);
 		regs.flags |= PERF_EFLAGS_EXACT;
 	}
 
-	if ((event->attr.config2 & IBS_SW_FILTER_MASK) &&
-	    perf_ibs_swfilt_discard(perf_ibs, event, &regs, &ibs_data, br_target_idx)) {
+	if (((ibs_caps & IBS_CAPS_BIT63_FILTER) ||
+	     (event->attr.config2 & IBS_SW_FILTER_MASK)) &&
+	    perf_ibs_discard_sample(perf_ibs, event, &regs, &ibs_data, br_target_idx)) {
 		throttle = perf_event_account_interrupt(event);
 		goto out;
 	}
@@ -1344,7 +1538,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 	 * unprivileged users.
 	 */
 	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
-	    perf_allow_kernel()) {
+	    (hwc->flags & PERF_X86_EVENT_UNPRIVILEGED)) {
 		perf_ibs_phyaddr_clear(perf_ibs, &ibs_data);
 	}
 
@@ -1375,6 +1569,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 
 out:
 	if (!throttle) {
+		if (ibs_caps & IBS_CAPS_DIS)
+			wrmsrq(hwc->extra_reg.reg, perf_ibs->disable_mask);
+
 		if (perf_ibs == &perf_ibs_op) {
 			if (ibs_caps & IBS_CAPS_OPCNTEXT) {
 				new_config = period & IBS_OP_MAX_CNT_EXT_MASK;
@@ -1446,6 +1643,9 @@ static __init int perf_ibs_fetch_init(void)
 	if (ibs_caps & IBS_CAPS_ZEN4)
 		perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY;
 
+	if (ibs_caps & IBS_CAPS_DIS)
+		perf_ibs_fetch.disable_mask = IBS_FETCH_2_DIS;
+
 	perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups;
 	perf_ibs_fetch.pmu.attr_update = fetch_attr_update;
 
@@ -1467,6 +1667,9 @@ static __init int perf_ibs_op_init(void)
 	if (ibs_caps & IBS_CAPS_ZEN4)
 		perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY;
 
+	if (ibs_caps & IBS_CAPS_DIS)
+		perf_ibs_op.disable_mask = IBS_OP_2_DIS;
+
 	perf_ibs_op.pmu.attr_groups = op_attr_groups;
 	perf_ibs_op.pmu.attr_update = op_attr_update;
 
@@ -1713,6 +1916,23 @@ static void clear_APIC_ibs(void)
 static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu)
 {
 	setup_APIC_ibs();
+
+	if (ibs_caps & IBS_CAPS_DIS) {
+		/*
+		 * IBS enable sequence:
+		 *   CTL[En] = 1;
+		 *   CTL2[Dis] = 0;
+		 *
+		 * IBS disable sequence:
+		 *   CTL2[Dis] = 1;
+		 *
+		 * Set CTL2[Dis] when CPU comes up. This is needed to make
+		 * enable sequence effective.
+		 */
+		wrmsrq(MSR_AMD64_IBSFETCHCTL2, IBS_FETCH_2_DIS);
+		wrmsrq(MSR_AMD64_IBSOPCTL2, IBS_OP_2_DIS);
+	}
+
 	return 0;
 }
 
@@ -1771,6 +1991,14 @@ static __init int amd_ibs_init(void)
 
 	perf_ibs_pm_init();
 
+#ifdef CONFIG_X86_32
+	/*
+	 * IBS_CAPS_BIT63_FILTER is used for exclude_kernel/user filtering,
+	 * which obviously won't work for 32 bit kernel.
+	 */
+	caps &= ~IBS_CAPS_BIT63_FILTER;
+#endif
+
 	ibs_caps = caps;
 	/* make ibs_caps visible to other cpus: */
 	smp_mb();
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 03ce1bc7ef2ea..810ab21ffd991 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -1372,14 +1372,17 @@ static void x86_pmu_enable(struct pmu *pmu)
 			else if (i < n_running)
 				continue;
 
-			if (hwc->state & PERF_HES_ARCH)
+			cpuc->events[hwc->idx] = event;
+
+			if (hwc->state & PERF_HES_ARCH) {
+				static_call(x86_pmu_set_period)(event);
 				continue;
+			}
 
 			/*
 			 * if cpuc->enabled = 0, then no wrmsr as
 			 * per x86_pmu_enable_event()
 			 */
-			cpuc->events[hwc->idx] = event;
 			x86_pmu_start(event, PERF_EF_RELOAD);
 		}
 		cpuc->n_added = 0;
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index cf3a4fe06ff26..2c1671ccf8622 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -4628,6 +4628,19 @@ static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event,
 		event->hw.dyn_constraint &= hybrid(event->pmu, acr_cause_mask64);
 }
 
+static inline int intel_set_branch_counter_constr(struct perf_event *event,
+						  int *num)
+{
+	if (branch_sample_call_stack(event))
+		return -EINVAL;
+	if (branch_sample_counters(event)) {
+		(*num)++;
+		event->hw.dyn_constraint &= x86_pmu.lbr_counters;
+	}
+
+	return 0;
+}
+
 static int intel_pmu_hw_config(struct perf_event *event)
 {
 	int ret = x86_pmu_hw_config(event);
@@ -4698,21 +4711,19 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		 * group, which requires the extra space to store the counters.
 		 */
 		leader = event->group_leader;
-		if (branch_sample_call_stack(leader))
+		if (intel_set_branch_counter_constr(leader, &num))
 			return -EINVAL;
-		if (branch_sample_counters(leader)) {
-			num++;
-			leader->hw.dyn_constraint &= x86_pmu.lbr_counters;
-		}
 		leader->hw.flags |= PERF_X86_EVENT_BRANCH_COUNTERS;
 
 		for_each_sibling_event(sibling, leader) {
-			if (branch_sample_call_stack(sibling))
+			if (intel_set_branch_counter_constr(sibling, &num))
+				return -EINVAL;
+		}
+
+		/* event isn't installed as a sibling yet. */
+		if (event != leader) {
+			if (intel_set_branch_counter_constr(event, &num))
 				return -EINVAL;
-			if (branch_sample_counters(sibling)) {
-				num++;
-				sibling->hw.dyn_constraint &= x86_pmu.lbr_counters;
-			}
 		}
 
 		if (num > fls(x86_pmu.lbr_counters))
@@ -5770,7 +5781,7 @@ static void __intel_pmu_check_dyn_constr(struct event_constraint *constr,
 			}
 
 			if (check_fail) {
-				pr_info("The two events 0x%llx and 0x%llx may not be "
+				pr_warn("The two events 0x%llx and 0x%llx may not be "
 					"fully scheduled under some circumstances as "
 					"%s.\n",
 					c1->code, c2->code, dyn_constr_type_name[type]);
@@ -5783,6 +5794,7 @@ static void intel_pmu_check_dyn_constr(struct pmu *pmu,
 				       struct event_constraint *constr,
 				       u64 cntr_mask)
 {
+	u64 gp_mask = GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
 	enum dyn_constr_type i;
 	u64 mask;
 
@@ -5797,20 +5809,25 @@ static void intel_pmu_check_dyn_constr(struct pmu *pmu,
 				mask = x86_pmu.lbr_counters;
 			break;
 		case DYN_CONSTR_ACR_CNTR:
-			mask = hybrid(pmu, acr_cntr_mask64) & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
+			mask = hybrid(pmu, acr_cntr_mask64) & gp_mask;
 			break;
 		case DYN_CONSTR_ACR_CAUSE:
-			if (hybrid(pmu, acr_cntr_mask64) == hybrid(pmu, acr_cause_mask64))
+			if (hybrid(pmu, acr_cntr_mask64) ==
+					hybrid(pmu, acr_cause_mask64))
 				continue;
-			mask = hybrid(pmu, acr_cause_mask64) & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
+			mask = hybrid(pmu, acr_cause_mask64) & gp_mask;
 			break;
 		case DYN_CONSTR_PEBS:
-			if (x86_pmu.arch_pebs)
-				mask = hybrid(pmu, arch_pebs_cap).counters;
+			if (x86_pmu.arch_pebs) {
+				mask = hybrid(pmu, arch_pebs_cap).counters &
+				       gp_mask;
+			}
 			break;
 		case DYN_CONSTR_PDIST:
-			if (x86_pmu.arch_pebs)
-				mask = hybrid(pmu, arch_pebs_cap).pdists;
+			if (x86_pmu.arch_pebs) {
+				mask = hybrid(pmu, arch_pebs_cap).pdists &
+				       gp_mask;
+			}
 			break;
 		default:
 			pr_warn("Unsupported dynamic constraint type %d\n", i);
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 5027afc97b655..7f0d515c07c54 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -345,12 +345,12 @@ static u64 parse_omr_data_source(u8 dse)
 	if (omr.omr_remote)
 		val |= REM;
 
-	val |= omr.omr_hitm ? P(SNOOP, HITM) : P(SNOOP, HIT);
-
 	if (omr.omr_source == 0x2) {
-		u8 snoop = omr.omr_snoop | omr.omr_promoted;
+		u8 snoop = omr.omr_snoop | (omr.omr_promoted << 1);
 
-		if (snoop == 0x0)
+		if (omr.omr_hitm)
+			val |= P(SNOOP, HITM);
+		else if (snoop == 0x0)
 			val |= P(SNOOP, NA);
 		else if (snoop == 0x1)
 			val |= P(SNOOP, MISS);
@@ -359,7 +359,10 @@ static u64 parse_omr_data_source(u8 dse)
 		else if (snoop == 0x3)
 			val |= P(SNOOP, NONE);
 	} else if (omr.omr_source > 0x2 && omr.omr_source < 0x7) {
+		val |= omr.omr_hitm ? P(SNOOP, HITM) : P(SNOOP, HIT);
 		val |= omr.omr_snoop ? P(SNOOPX, FWD) : 0;
+	} else {
+		val |= P(SNOOP, NONE);
 	}
 
 	return val;
diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c
index 44524a387c58b..b5726b50e77db 100644
--- a/arch/x86/events/intel/pt.c
+++ b/arch/x86/events/intel/pt.c
@@ -1591,7 +1591,6 @@ void intel_pt_handle_vmx(int on)
 
 	local_irq_restore(flags);
 }
-EXPORT_SYMBOL_FOR_KVM(intel_pt_handle_vmx);
 
 /*
  * PMU callbacks
diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h
index 70078334e4a33..47f84ee8f5409 100644
--- a/arch/x86/events/perf_event_flags.h
+++ b/arch/x86/events/perf_event_flags.h
@@ -23,3 +23,4 @@ PERF_ARCH(PEBS_LAT_HYBRID,	0x0020000) /* ld and st lat for hybrid */
 PERF_ARCH(NEEDS_BRANCH_STACK,	0x0040000) /* require branch stack setup */
 PERF_ARCH(BRANCH_COUNTERS,	0x0080000) /* logs the counters in the extra space of each branch */
 PERF_ARCH(ACR,			0x0100000) /* Auto counter reload */
+PERF_ARCH(UNPRIVILEGED,		0x0200000) /* Unprivileged event (wrt perf_allow_kernel()) */
diff --git a/arch/x86/hyperv/hv_crash.c b/arch/x86/hyperv/hv_crash.c
index 92da1b4f2e730..5ffcc23255de8 100644
--- a/arch/x86/hyperv/hv_crash.c
+++ b/arch/x86/hyperv/hv_crash.c
@@ -107,14 +107,12 @@ static void __noreturn hv_panic_timeout_reboot(void)
 		cpu_relax();
 }
 
-/* This cannot be inlined as it needs stack */
-static noinline __noclone void hv_crash_restore_tss(void)
+static void hv_crash_restore_tss(void)
 {
 	load_TR_desc();
 }
 
-/* This cannot be inlined as it needs stack */
-static noinline void hv_crash_clear_kernpt(void)
+static void hv_crash_clear_kernpt(void)
 {
 	pgd_t *pgd;
 	p4d_t *p4d;
@@ -125,6 +123,25 @@ static noinline void hv_crash_clear_kernpt(void)
 	native_p4d_clear(p4d);
 }
 
+
+static void __noreturn hv_crash_handle(void)
+{
+	hv_crash_restore_tss();
+	hv_crash_clear_kernpt();
+
+	/* we are now fully in devirtualized normal kernel mode */
+	__crash_kexec(NULL);
+
+	hv_panic_timeout_reboot();
+}
+
+/*
+ * __naked functions do not permit function calls, not even to __always_inline
+ * functions that only contain asm() blocks themselves. So use a macro instead.
+ */
+#define hv_wrmsr(msr, val) \
+	asm volatile("wrmsr" :: "c"(msr), "a"((u32)val), "d"((u32)(val >> 32)) : "memory")
+
 /*
  * This is the C entry point from the asm glue code after the disable hypercall.
  * We enter here in IA32-e long mode, ie, full 64bit mode running on kernel
@@ -133,51 +150,38 @@ static noinline void hv_crash_clear_kernpt(void)
  * available. We restore kernel GDT, and rest of the context, and continue
  * to kexec.
  */
-static asmlinkage void __noreturn hv_crash_c_entry(void)
+static void __naked hv_crash_c_entry(void)
 {
-	struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
-
 	/* first thing, restore kernel gdt */
-	native_load_gdt(&ctxt->gdtr);
+	asm volatile("lgdt %0" : : "m" (hv_crash_ctxt.gdtr));
 
-	asm volatile("movw %%ax, %%ss" : : "a"(ctxt->ss));
-	asm volatile("movq %0, %%rsp" : : "m"(ctxt->rsp));
+	asm volatile("movw %0, %%ss\n\t"
+		     "movq %1, %%rsp"
+		     :: "m"(hv_crash_ctxt.ss), "m"(hv_crash_ctxt.rsp));
 
-	asm volatile("movw %%ax, %%ds" : : "a"(ctxt->ds));
-	asm volatile("movw %%ax, %%es" : : "a"(ctxt->es));
-	asm volatile("movw %%ax, %%fs" : : "a"(ctxt->fs));
-	asm volatile("movw %%ax, %%gs" : : "a"(ctxt->gs));
+	asm volatile("movw %0, %%ds" : : "m"(hv_crash_ctxt.ds));
+	asm volatile("movw %0, %%es" : : "m"(hv_crash_ctxt.es));
+	asm volatile("movw %0, %%fs" : : "m"(hv_crash_ctxt.fs));
+	asm volatile("movw %0, %%gs" : : "m"(hv_crash_ctxt.gs));
 
-	native_wrmsrq(MSR_IA32_CR_PAT, ctxt->pat);
-	asm volatile("movq %0, %%cr0" : : "r"(ctxt->cr0));
+	hv_wrmsr(MSR_IA32_CR_PAT, hv_crash_ctxt.pat);
+	asm volatile("movq %0, %%cr0" : : "r"(hv_crash_ctxt.cr0));
 
-	asm volatile("movq %0, %%cr8" : : "r"(ctxt->cr8));
-	asm volatile("movq %0, %%cr4" : : "r"(ctxt->cr4));
-	asm volatile("movq %0, %%cr2" : : "r"(ctxt->cr4));
+	asm volatile("movq %0, %%cr8" : : "r"(hv_crash_ctxt.cr8));
+	asm volatile("movq %0, %%cr4" : : "r"(hv_crash_ctxt.cr4));
+	asm volatile("movq %0, %%cr2" : : "r"(hv_crash_ctxt.cr2));
 
-	native_load_idt(&ctxt->idtr);
-	native_wrmsrq(MSR_GS_BASE, ctxt->gsbase);
-	native_wrmsrq(MSR_EFER, ctxt->efer);
+	asm volatile("lidt %0" : : "m" (hv_crash_ctxt.idtr));
+	hv_wrmsr(MSR_GS_BASE, hv_crash_ctxt.gsbase);
+	hv_wrmsr(MSR_EFER, hv_crash_ctxt.efer);
 
 	/* restore the original kernel CS now via far return */
-	asm volatile("movzwq %0, %%rax\n\t"
-		     "pushq %%rax\n\t"
-		     "pushq $1f\n\t"
-		     "lretq\n\t"
-		     "1:nop\n\t" : : "m"(ctxt->cs) : "rax");
-
-	/* We are in asmlinkage without stack frame, hence make C function
-	 * calls which will buy stack frames.
-	 */
-	hv_crash_restore_tss();
-	hv_crash_clear_kernpt();
-
-	/* we are now fully in devirtualized normal kernel mode */
-	__crash_kexec(NULL);
-
-	hv_panic_timeout_reboot();
+	asm volatile("pushq %q0\n\t"
+		     "pushq %q1\n\t"
+		     "lretq"
+		     :: "r"(hv_crash_ctxt.cs), "r"(hv_crash_handle));
 }
-/* Tell gcc we are using lretq long jump in the above function intentionally */
+/* Tell objtool we are using lretq long jump in the above function intentionally */
 STACK_FRAME_NON_STANDARD(hv_crash_c_entry);
 
 static void hv_mark_tss_not_busy(void)
@@ -195,20 +199,20 @@ static void hv_hvcrash_ctxt_save(void)
 {
 	struct hv_crash_ctxt *ctxt = &hv_crash_ctxt;
 
-	asm volatile("movq %%rsp,%0" : "=m"(ctxt->rsp));
+	ctxt->rsp = current_stack_pointer;
 
 	ctxt->cr0 = native_read_cr0();
 	ctxt->cr4 = native_read_cr4();
 
-	asm volatile("movq %%cr2, %0" : "=a"(ctxt->cr2));
-	asm volatile("movq %%cr8, %0" : "=a"(ctxt->cr8));
+	asm volatile("movq %%cr2, %0" : "=r"(ctxt->cr2));
+	asm volatile("movq %%cr8, %0" : "=r"(ctxt->cr8));
 
-	asm volatile("movl %%cs, %%eax" : "=a"(ctxt->cs));
-	asm volatile("movl %%ss, %%eax" : "=a"(ctxt->ss));
-	asm volatile("movl %%ds, %%eax" : "=a"(ctxt->ds));
-	asm volatile("movl %%es, %%eax" : "=a"(ctxt->es));
-	asm volatile("movl %%fs, %%eax" : "=a"(ctxt->fs));
-	asm volatile("movl %%gs, %%eax" : "=a"(ctxt->gs));
+	asm volatile("movw %%cs, %0" : "=m"(ctxt->cs));
+	asm volatile("movw %%ss, %0" : "=m"(ctxt->ss));
+	asm volatile("movw %%ds, %0" : "=m"(ctxt->ds));
+	asm volatile("movw %%es, %0" : "=m"(ctxt->es));
+	asm volatile("movw %%fs, %0" : "=m"(ctxt->fs));
+	asm volatile("movw %%gs, %0" : "=m"(ctxt->gs));
 
 	native_store_gdt(&ctxt->gdtr);
 	store_idt(&ctxt->idtr);
diff --git a/arch/x86/include/asm/amd/ibs.h b/arch/x86/include/asm/amd/ibs.h
index fcc8a5abe54e6..68e24a1736d0d 100644
--- a/arch/x86/include/asm/amd/ibs.h
+++ b/arch/x86/include/asm/amd/ibs.h
@@ -77,7 +77,7 @@ union ibs_op_data {
 	__u64 val;
 	struct {
 		__u64	comp_to_ret_ctr:16,	/* 0-15: op completion to retire count */
-			tag_to_ret_ctr:16,	/* 15-31: op tag to retire count */
+			tag_to_ret_ctr:16,	/* 16-31: op tag to retire count */
 			reserved1:2,		/* 32-33: reserved */
 			op_return:1,		/* 34: return op */
 			op_brn_taken:1,		/* 35: taken branch op */
@@ -99,7 +99,9 @@ union ibs_op_data2 {
 			rmt_node:1,	/* 4: destination node */
 			cache_hit_st:1,	/* 5: cache hit state */
 			data_src_hi:2,	/* 6-7: data source high */
-			reserved1:56;	/* 8-63: reserved */
+			strm_st:1,	/* 8: streaming store */
+			rmt_socket:1,   /* 9: remote socket */
+			reserved1:54;   /* 10-63: reserved */
 	};
 };
 
diff --git a/arch/x86/include/asm/clock_inlined.h b/arch/x86/include/asm/clock_inlined.h
new file mode 100644
index 0000000000000..b2dee8db2fb9c
--- /dev/null
+++ b/arch/x86/include/asm/clock_inlined.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_CLOCK_INLINED_H
+#define _ASM_X86_CLOCK_INLINED_H
+
+#include <asm/tsc.h>
+
+struct clocksource;
+
+static __always_inline u64 arch_inlined_clocksource_read(struct clocksource *cs)
+{
+	return (u64)rdtsc_ordered();
+}
+
+struct clock_event_device;
+
+static __always_inline void
+arch_inlined_clockevent_set_next_coupled(u64 cycles, struct clock_event_device *evt)
+{
+	native_wrmsrq(MSR_IA32_TSC_DEADLINE, cycles);
+}
+
+#endif
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index dbe104df339b8..de7bd88e539db 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -473,6 +473,7 @@
 #define X86_FEATURE_GP_ON_USER_CPUID	(20*32+17) /* User CPUID faulting */
 
 #define X86_FEATURE_PREFETCHI		(20*32+20) /* Prefetch Data/Instruction to Cache Level */
+#define X86_FEATURE_AVX512_BMM		(20*32+23) /* AVX512 Bit Matrix Multiply instructions */
 #define X86_FEATURE_ERAPS		(20*32+24) /* Enhanced Return Address Predictor Security */
 #define X86_FEATURE_SBPB		(20*32+27) /* Selective Branch Prediction Barrier */
 #define X86_FEATURE_IBPB_BRTYPE		(20*32+28) /* MSR_PRED_CMD[IBPB] flushes all branch type predictions */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 51b4cdbea061a..dc8fe1361c18e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -401,9 +401,9 @@ extern int __init efi_memmap_split_count(efi_memory_desc_t *md,
 extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap,
 				     void *buf, struct efi_mem_range *mem);
 
-extern enum efi_secureboot_mode __x86_ima_efi_boot_mode(void);
+enum efi_secureboot_mode __x86_efi_boot_mode(void);
 
-#define arch_ima_efi_boot_mode	__x86_ima_efi_boot_mode()
+#define arch_efi_boot_mode __x86_efi_boot_mode()
 
 #ifdef CONFIG_EFI_RUNTIME_MAP
 int efi_get_runtime_map_size(void);
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index ca309a3227c78..2ea25745e0591 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -218,9 +218,8 @@ static inline void __iowrite32_copy(void __iomem *to, const void *from,
 				    size_t count)
 {
 	asm volatile("rep movsl"
-		     : "=&c"(count), "=&D"(to), "=&S"(from)
-		     : "0"(count), "1"(to), "2"(from)
-		     : "memory");
+		     : "+D"(to), "+S"(from), "+c"(count)
+		     : : "memory");
 }
 #define __iowrite32_copy __iowrite32_copy
 #endif
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index de709fb5bd76a..3776cf5382a26 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -1,8 +1,9 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#if !defined(KVM_X86_OP) || !defined(KVM_X86_OP_OPTIONAL)
-BUILD_BUG_ON(1)
-#endif
-
+#if !defined(KVM_X86_OP) || \
+    !defined(KVM_X86_OP_OPTIONAL) || \
+    !defined(KVM_X86_OP_OPTIONAL_RET0)
+#error Missing one or more KVM_X86_OP #defines
+#else
 /*
  * KVM_X86_OP() and KVM_X86_OP_OPTIONAL() are used to help generate
  * both DECLARE/DEFINE_STATIC_CALL() invocations and
@@ -148,6 +149,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
 KVM_X86_OP_OPTIONAL_RET0(gmem_prepare)
 KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
 KVM_X86_OP_OPTIONAL(gmem_invalidate)
+#endif
 
 #undef KVM_X86_OP
 #undef KVM_X86_OP_OPTIONAL
diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/kvm-x86-pmu-ops.h
index f0aa6996811f9..d5452b3433b7d 100644
--- a/arch/x86/include/asm/kvm-x86-pmu-ops.h
+++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h
@@ -1,7 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#if !defined(KVM_X86_PMU_OP) || !defined(KVM_X86_PMU_OP_OPTIONAL)
-BUILD_BUG_ON(1)
-#endif
+#if !defined(KVM_X86_PMU_OP) || \
+    !defined(KVM_X86_PMU_OP_OPTIONAL)
+#error Missing one or more KVM_X86_PMU_OP #defines
+#else
 
 /*
  * KVM_X86_PMU_OP() and KVM_X86_PMU_OP_OPTIONAL() are used to help generate
@@ -26,6 +27,7 @@ KVM_X86_PMU_OP_OPTIONAL(cleanup)
 KVM_X86_PMU_OP_OPTIONAL(write_global_ctrl)
 KVM_X86_PMU_OP(mediated_load)
 KVM_X86_PMU_OP(mediated_put)
+#endif
 
 #undef KVM_X86_PMU_OP
 #undef KVM_X86_PMU_OP_OPTIONAL
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 6e4e3ef9b8c72..d3bdc98281339 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -40,7 +40,8 @@
 #include <asm/irq_remapping.h>
 #include <asm/kvm_page_track.h>
 #include <asm/kvm_vcpu_regs.h>
-#include <asm/reboot.h>
+#include <asm/virt.h>
+
 #include <hyperv/hvhdk.h>
 
 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS
@@ -1261,7 +1262,7 @@ struct kvm_x86_pmu_event_filter {
 	__u32 nr_excludes;
 	__u64 *includes;
 	__u64 *excludes;
-	__u64 events[];
+	__u64 events[] __counted_by(nevents);
 };
 
 enum kvm_apicv_inhibit {
@@ -1433,6 +1434,7 @@ struct kvm_arch {
 	struct kvm_pit *vpit;
 #endif
 	atomic_t vapics_in_nmi_mode;
+
 	struct mutex apic_map_lock;
 	struct kvm_apic_map __rcu *apic_map;
 	atomic_t apic_map_dirty;
@@ -1440,8 +1442,22 @@ struct kvm_arch {
 	bool apic_access_memslot_enabled;
 	bool apic_access_memslot_inhibited;
 
-	/* Protects apicv_inhibit_reasons */
+	/*
+	 * Force apicv_update_lock and apicv_nr_irq_window_req to reside in a
+	 * dedicated cacheline.  They are write-mostly, whereas most everything
+	 * else in kvm_arch is read-mostly.  Note that apicv_inhibit_reasons is
+	 * read-mostly: toggling VM-wide inhibits is rare; _checking_ for
+	 * inhibits is common.
+	 */
+	____cacheline_aligned
+	/*
+	 * Protects apicv_inhibit_reasons and apicv_nr_irq_window_req (with an
+	 * asterisk, see kvm_inc_or_dec_irq_window_inhibit() for details).
+	 */
 	struct rw_semaphore apicv_update_lock;
+	atomic_t apicv_nr_irq_window_req;
+	____cacheline_aligned
+
 	unsigned long apicv_inhibit_reasons;
 
 	gpa_t wall_clock;
@@ -2097,9 +2113,6 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
 
 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
 
-int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
-			  const void *val, int bytes);
-
 extern bool tdp_enabled;
 
 u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
@@ -2316,6 +2329,18 @@ static inline void kvm_clear_apicv_inhibit(struct kvm *kvm,
 	kvm_set_or_clear_apicv_inhibit(kvm, reason, false);
 }
 
+void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc);
+
+static inline void kvm_inc_apicv_irq_window_req(struct kvm *kvm)
+{
+	kvm_inc_or_dec_irq_window_inhibit(kvm, true);
+}
+
+static inline void kvm_dec_apicv_irq_window_req(struct kvm *kvm)
+{
+	kvm_inc_or_dec_irq_window_inhibit(kvm, false);
+}
+
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 59aa966dc2127..4957018fef3ed 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -106,7 +106,7 @@ static inline bool local_add_negative(long i, local_t *l)
 static inline long local_add_return(long i, local_t *l)
 {
 	long __i = i;
-	asm volatile(_ASM_XADD "%0, %1;"
+	asm volatile(_ASM_XADD "%0, %1"
 		     : "+r" (i), "+m" (l->a.counter)
 		     : : "memory");
 	return i + __i;
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 2d98886de09ad..0175d39a58561 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -343,44 +343,60 @@ extern void apei_mce_report_mem_error(int corrected,
  */
 #ifdef CONFIG_X86_MCE_AMD
 
-/* These may be used by multiple smca_hwid_mcatypes */
+/*
+ * These may be used by multiple smca_hwid_mcatypes.
+ *
+ * Keep in alphanumeric order, numerals before letters.
+ * Exception: Keep "V2, etc." with their originals.
+ */
 enum smca_bank_types {
-	SMCA_LS = 0,	/* Load Store */
-	SMCA_LS_V2,
-	SMCA_IF,	/* Instruction Fetch */
-	SMCA_L2_CACHE,	/* L2 Cache */
+	SMCA_CS,	/* Coherent Station */
+	SMCA_CS_V2,
+	SMCA_DACC_BE,	/* Data Acceleration Back-end */
+	SMCA_DACC_FE,	/* Data Acceleration Front-end */
 	SMCA_DE,	/* Decoder Unit */
-	SMCA_RESERVED,	/* Reserved */
+	SMCA_EDDR5CMN,	/* eDDR5 CMN */
 	SMCA_EX,	/* Execution Unit */
 	SMCA_FP,	/* Floating Point */
+	SMCA_GMI_PCS,	/* GMI PCS Unit */
+	SMCA_GMI_PHY,	/* GMI PHY Unit */
+	SMCA_IF,	/* Instruction Fetch */
+	SMCA_L2_CACHE,	/* L2 Cache */
 	SMCA_L3_CACHE,	/* L3 Cache */
-	SMCA_CS,	/* Coherent Slave */
-	SMCA_CS_V2,
-	SMCA_PIE,	/* Power, Interrupts, etc. */
-	SMCA_UMC,	/* Unified Memory Controller */
-	SMCA_UMC_V2,
+	SMCA_LS,	/* Load Store */
+	SMCA_LS_V2,
 	SMCA_MA_LLC,	/* Memory Attached Last Level Cache */
-	SMCA_PB,	/* Parameter Block */
-	SMCA_PSP,	/* Platform Security Processor */
-	SMCA_PSP_V2,
-	SMCA_SMU,	/* System Management Unit */
-	SMCA_SMU_V2,
 	SMCA_MP5,	/* Microprocessor 5 Unit */
+	SMCA_MPART,	/* AMD Root of Trust Microprocessor */
+	SMCA_MPASP,	/* AMD Secure Processor */
+	SMCA_MPASP_V2,
+	SMCA_MPDACC,	/* MP for Data Acceleration */
 	SMCA_MPDMA,	/* MPDMA Unit */
+	SMCA_MPM,	/* Microprocessor Manageability Core */
+	SMCA_MPRAS,	/* MP for RAS */
+	SMCA_NBIF,	/* NBIF Unit */
 	SMCA_NBIO,	/* Northbridge IO Unit */
+	SMCA_PB,	/* Parameter Block */
 	SMCA_PCIE,	/* PCI Express Unit */
 	SMCA_PCIE_V2,
-	SMCA_XGMI_PCS,	/* xGMI PCS Unit */
-	SMCA_NBIF,	/* NBIF Unit */
-	SMCA_SHUB,	/* System HUB Unit */
+	SMCA_PCIE_PL,	/* PCIe Link */
+	SMCA_PIE,	/* Power, Interrupts, etc. */
+	SMCA_PSP,	/* Platform Security Processor */
+	SMCA_PSP_V2,
+	SMCA_RESERVED,	/* Reserved */
 	SMCA_SATA,	/* SATA Unit */
+	SMCA_SHUB,	/* System HUB Unit */
+	SMCA_SMU,	/* System Management Unit */
+	SMCA_SMU_V2,
+	SMCA_SSBDCI,	/* Die to Die Interconnect */
+	SMCA_UMC,	/* Unified Memory Controller */
+	SMCA_UMC_V2,
 	SMCA_USB,	/* USB Unit */
-	SMCA_USR_DP,	/* Ultra Short Reach Data Plane Controller */
 	SMCA_USR_CP,	/* Ultra Short Reach Control Plane Controller */
-	SMCA_GMI_PCS,	/* GMI PCS Unit */
-	SMCA_XGMI_PHY,	/* xGMI PHY Unit */
+	SMCA_USR_DP,	/* Ultra Short Reach Data Plane Controller */
 	SMCA_WAFL_PHY,	/* WAFL PHY Unit */
-	SMCA_GMI_PHY,	/* GMI PHY Unit */
+	SMCA_XGMI_PCS,	/* xGMI PCS Unit */
+	SMCA_XGMI_PHY,	/* xGMI PHY Unit */
 	N_SMCA_BANK_TYPES
 };
 
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 8b41f26f003b0..3c317d1557716 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -61,6 +61,8 @@ static inline int intel_microcode_get_datasize(struct microcode_header_intel *hd
 	return hdr->datasize ? : DEFAULT_UCODE_DATASIZE;
 }
 
+extern u32 intel_get_platform_id(void);
+
 static inline u32 intel_get_microcode_revision(void)
 {
 	u32 rev, dummy;
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 1acafb1c6a932..ef5b507de34e2 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -136,9 +136,6 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm)
 }
 #endif
 
-#define enter_lazy_tlb enter_lazy_tlb
-extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);
-
 extern void mm_init_global_asid(struct mm_struct *mm);
 extern void mm_free_global_asid(struct mm_struct *mm);
 
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 6673601246b38..be3e3cc963b2a 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -698,6 +698,8 @@
 #define MSR_AMD64_IBSBRTARGET		0xc001103b
 #define MSR_AMD64_ICIBSEXTDCTL		0xc001103c
 #define MSR_AMD64_IBSOPDATA4		0xc001103d
+#define MSR_AMD64_IBSOPCTL2		0xc001103e
+#define MSR_AMD64_IBSFETCHCTL2		0xc001103f
 #define MSR_AMD64_IBS_REG_COUNT_MAX	8 /* includes MSR_AMD64_IBSBRTARGET */
 #define MSR_AMD64_SVM_AVIC_DOORBELL	0xc001011b
 #define MSR_AMD64_VM_PAGE_FLUSH		0xc001011e
diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h
index e0125afa53fb9..5837c2bb277fd 100644
--- a/arch/x86/include/asm/orc_types.h
+++ b/arch/x86/include/asm/orc_types.h
@@ -28,15 +28,16 @@
  * and GCC realigned stacks.
  */
 #define ORC_REG_UNDEFINED		0
-#define ORC_REG_PREV_SP			1
+#define ORC_REG_AX			1
 #define ORC_REG_DX			2
-#define ORC_REG_DI			3
+#define ORC_REG_SP			3
 #define ORC_REG_BP			4
-#define ORC_REG_SP			5
+#define ORC_REG_DI			5
 #define ORC_REG_R10			6
 #define ORC_REG_R13			7
-#define ORC_REG_BP_INDIRECT		8
+#define ORC_REG_PREV_SP			8
 #define ORC_REG_SP_INDIRECT		9
+#define ORC_REG_BP_INDIRECT		10
 #define ORC_REG_MAX			15
 
 #define ORC_TYPE_UNDEFINED		0
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index 018a8d906ca35..3e0801a0f7823 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -26,7 +26,7 @@
 
 #define PAGE_OFFSET		((unsigned long)__PAGE_OFFSET)
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
 
 /* Physical address where kernel should be loaded. */
 #define LOAD_PHYSICAL_ADDR	__ALIGN_KERNEL_MASK(CONFIG_PHYSICAL_START, CONFIG_PHYSICAL_ALIGN - 1)
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index ff5acb8b199b0..752cb319d5eab 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -643,6 +643,10 @@ struct arch_pebs_cntr_header {
 #define IBS_CAPS_OPDATA4		(1U<<10)
 #define IBS_CAPS_ZEN4			(1U<<11)
 #define IBS_CAPS_OPLDLAT		(1U<<12)
+#define IBS_CAPS_DIS			(1U<<13)
+#define IBS_CAPS_FETCHLAT		(1U<<14)
+#define IBS_CAPS_BIT63_FILTER		(1U<<15)
+#define IBS_CAPS_STRMST_RMTSOCKET	(1U<<16)
 #define IBS_CAPS_OPDTLBPGSIZE		(1U<<19)
 
 #define IBS_CAPS_DEFAULT		(IBS_CAPS_AVAIL		\
@@ -657,31 +661,44 @@ struct arch_pebs_cntr_header {
 #define IBSCTL_LVT_OFFSET_MASK		0x0F
 
 /* IBS fetch bits/masks */
-#define IBS_FETCH_L3MISSONLY	(1ULL<<59)
-#define IBS_FETCH_RAND_EN	(1ULL<<57)
-#define IBS_FETCH_VAL		(1ULL<<49)
-#define IBS_FETCH_ENABLE	(1ULL<<48)
-#define IBS_FETCH_CNT		0xFFFF0000ULL
-#define IBS_FETCH_MAX_CNT	0x0000FFFFULL
+#define IBS_FETCH_L3MISSONLY		      (1ULL << 59)
+#define IBS_FETCH_RAND_EN		      (1ULL << 57)
+#define IBS_FETCH_VAL			      (1ULL << 49)
+#define IBS_FETCH_ENABLE		      (1ULL << 48)
+#define IBS_FETCH_CNT			     0xFFFF0000ULL
+#define IBS_FETCH_MAX_CNT		     0x0000FFFFULL
+
+#define IBS_FETCH_2_DIS			      (1ULL <<  0)
+#define IBS_FETCH_2_FETCHLAT_FILTER	    (0xFULL <<  1)
+#define IBS_FETCH_2_FETCHLAT_FILTER_SHIFT	       (1)
+#define IBS_FETCH_2_EXCL_RIP_63_EQ_1	      (1ULL <<  5)
+#define IBS_FETCH_2_EXCL_RIP_63_EQ_0	      (1ULL <<  6)
 
 /*
  * IBS op bits/masks
  * The lower 7 bits of the current count are random bits
  * preloaded by hardware and ignored in software
  */
-#define IBS_OP_LDLAT_EN		(1ULL<<63)
-#define IBS_OP_LDLAT_THRSH	(0xFULL<<59)
-#define IBS_OP_CUR_CNT		(0xFFF80ULL<<32)
-#define IBS_OP_CUR_CNT_RAND	(0x0007FULL<<32)
-#define IBS_OP_CUR_CNT_EXT_MASK	(0x7FULL<<52)
-#define IBS_OP_CNT_CTL		(1ULL<<19)
-#define IBS_OP_VAL		(1ULL<<18)
-#define IBS_OP_ENABLE		(1ULL<<17)
-#define IBS_OP_L3MISSONLY	(1ULL<<16)
-#define IBS_OP_MAX_CNT		0x0000FFFFULL
-#define IBS_OP_MAX_CNT_EXT	0x007FFFFFULL	/* not a register bit mask */
-#define IBS_OP_MAX_CNT_EXT_MASK	(0x7FULL<<20)	/* separate upper 7 bits */
-#define IBS_RIP_INVALID		(1ULL<<38)
+#define IBS_OP_LDLAT_EN			      (1ULL << 63)
+#define IBS_OP_LDLAT_THRSH		    (0xFULL << 59)
+#define IBS_OP_LDLAT_THRSH_SHIFT		      (59)
+#define IBS_OP_CUR_CNT			(0xFFF80ULL << 32)
+#define IBS_OP_CUR_CNT_RAND		(0x0007FULL << 32)
+#define IBS_OP_CUR_CNT_EXT_MASK		   (0x7FULL << 52)
+#define IBS_OP_CNT_CTL			      (1ULL << 19)
+#define IBS_OP_VAL			      (1ULL << 18)
+#define IBS_OP_ENABLE			      (1ULL << 17)
+#define IBS_OP_L3MISSONLY		      (1ULL << 16)
+#define IBS_OP_MAX_CNT			     0x0000FFFFULL
+#define IBS_OP_MAX_CNT_EXT		     0x007FFFFFULL	/* not a register bit mask */
+#define IBS_OP_MAX_CNT_EXT_MASK		   (0x7FULL << 20)	/* separate upper 7 bits */
+#define IBS_RIP_INVALID			      (1ULL << 38)
+
+#define IBS_OP_2_DIS			      (1ULL <<  0)
+#define IBS_OP_2_EXCL_RIP_63_EQ_0	      (1ULL <<  1)
+#define IBS_OP_2_EXCL_RIP_63_EQ_1	      (1ULL <<  2)
+#define IBS_OP_2_STRM_ST_FILTER		      (1ULL <<  3)
+#define IBS_OP_2_STRM_ST_FILTER_SHIFT		       (3)
 
 #ifdef CONFIG_X86_LOCAL_APIC
 extern u32 get_ibs_caps(void);
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 1662c5a8f445f..6be887694621c 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -47,14 +47,6 @@ void ptdump_walk_user_pgd_level_checkwx(void);
 #define debug_checkwx_user()	do { } while (0)
 #endif
 
-/*
- * ZERO_PAGE is a global shared page that is always zero: used
- * for zero-mapped memory areas etc..
- */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
-	__visible;
-#define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page))
-
 extern spinlock_t pgd_lock;
 extern struct list_head pgd_list;
 
@@ -1680,17 +1672,17 @@ static inline bool arch_has_hw_nonleaf_pmd_young(void)
 #endif
 
 #ifdef CONFIG_PAGE_TABLE_CHECK
-static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
+static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
 {
 	return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
 }
 
-static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
+static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
 {
 	return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && (pmd_val(pmd) & _PAGE_USER);
 }
 
-static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
+static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
 {
 	return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && (pud_val(pud) & _PAGE_USER);
 }
diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h
index 2e6c04d8a45b4..06ed2cd2592e8 100644
--- a/arch/x86/include/asm/pkeys.h
+++ b/arch/x86/include/asm/pkeys.h
@@ -9,8 +9,7 @@
  */
 #define arch_max_pkey() (cpu_feature_enabled(X86_FEATURE_OSPKE) ? 16 : 1)
 
-extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-		unsigned long init_val);
+extern int arch_set_user_pkey_access(int pkey, unsigned long init_val);
 
 static inline bool arch_pkeys_enabled(void)
 {
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index a24c7805acdb5..10b5355b323e2 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -140,6 +140,11 @@ struct cpuinfo_x86 {
 		__u32		x86_vfm;
 	};
 	__u8			x86_stepping;
+	union {
+		// MSR_IA32_PLATFORM_ID[52-50]
+		__u8			intel_platform_id;
+		__u8			amd_unused;
+	};
 #ifdef CONFIG_X86_64
 	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
 	int			x86_tlbsize;
diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h
index ecd58ea9a837b..a671a1145906d 100644
--- a/arch/x86/include/asm/reboot.h
+++ b/arch/x86/include/asm/reboot.h
@@ -25,17 +25,6 @@ void __noreturn machine_real_restart(unsigned int type);
 #define MRR_BIOS	0
 #define MRR_APM		1
 
-typedef void (cpu_emergency_virt_cb)(void);
-#if IS_ENABLED(CONFIG_KVM_X86)
-void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback);
-void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback);
-void cpu_emergency_disable_virtualization(void);
-#else
-static inline void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback) {}
-static inline void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback) {}
-static inline void cpu_emergency_disable_virtualization(void) {}
-#endif /* CONFIG_KVM_X86 */
-
 typedef void (*nmi_shootdown_cb)(int, struct pt_regs*);
 void nmi_shootdown_cpus(nmi_shootdown_cb callback);
 void run_crash_ipi_callback(struct pt_regs *regs);
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index 8bc074c8d7c6a..049638e3da743 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -20,31 +20,31 @@
 #define TDG_VM_RD			7
 #define TDG_VM_WR			8
 
-/* TDX attributes */
-#define TDX_ATTR_DEBUG_BIT		0
-#define TDX_ATTR_DEBUG			BIT_ULL(TDX_ATTR_DEBUG_BIT)
-#define TDX_ATTR_HGS_PLUS_PROF_BIT	4
-#define TDX_ATTR_HGS_PLUS_PROF		BIT_ULL(TDX_ATTR_HGS_PLUS_PROF_BIT)
-#define TDX_ATTR_PERF_PROF_BIT		5
-#define TDX_ATTR_PERF_PROF		BIT_ULL(TDX_ATTR_PERF_PROF_BIT)
-#define TDX_ATTR_PMT_PROF_BIT		6
-#define TDX_ATTR_PMT_PROF		BIT_ULL(TDX_ATTR_PMT_PROF_BIT)
-#define TDX_ATTR_ICSSD_BIT		16
-#define TDX_ATTR_ICSSD			BIT_ULL(TDX_ATTR_ICSSD_BIT)
-#define TDX_ATTR_LASS_BIT		27
-#define TDX_ATTR_LASS			BIT_ULL(TDX_ATTR_LASS_BIT)
-#define TDX_ATTR_SEPT_VE_DISABLE_BIT	28
-#define TDX_ATTR_SEPT_VE_DISABLE	BIT_ULL(TDX_ATTR_SEPT_VE_DISABLE_BIT)
-#define TDX_ATTR_MIGRTABLE_BIT		29
-#define TDX_ATTR_MIGRTABLE		BIT_ULL(TDX_ATTR_MIGRTABLE_BIT)
-#define TDX_ATTR_PKS_BIT		30
-#define TDX_ATTR_PKS			BIT_ULL(TDX_ATTR_PKS_BIT)
-#define TDX_ATTR_KL_BIT			31
-#define TDX_ATTR_KL			BIT_ULL(TDX_ATTR_KL_BIT)
-#define TDX_ATTR_TPA_BIT		62
-#define TDX_ATTR_TPA			BIT_ULL(TDX_ATTR_TPA_BIT)
-#define TDX_ATTR_PERFMON_BIT		63
-#define TDX_ATTR_PERFMON		BIT_ULL(TDX_ATTR_PERFMON_BIT)
+/* TDX TD attributes */
+#define TDX_TD_ATTR_DEBUG_BIT		0
+#define TDX_TD_ATTR_DEBUG		BIT_ULL(TDX_TD_ATTR_DEBUG_BIT)
+#define TDX_TD_ATTR_HGS_PLUS_PROF_BIT	4
+#define TDX_TD_ATTR_HGS_PLUS_PROF	BIT_ULL(TDX_TD_ATTR_HGS_PLUS_PROF_BIT)
+#define TDX_TD_ATTR_PERF_PROF_BIT	5
+#define TDX_TD_ATTR_PERF_PROF		BIT_ULL(TDX_TD_ATTR_PERF_PROF_BIT)
+#define TDX_TD_ATTR_PMT_PROF_BIT	6
+#define TDX_TD_ATTR_PMT_PROF		BIT_ULL(TDX_TD_ATTR_PMT_PROF_BIT)
+#define TDX_TD_ATTR_ICSSD_BIT		16
+#define TDX_TD_ATTR_ICSSD		BIT_ULL(TDX_TD_ATTR_ICSSD_BIT)
+#define TDX_TD_ATTR_LASS_BIT		27
+#define TDX_TD_ATTR_LASS		BIT_ULL(TDX_TD_ATTR_LASS_BIT)
+#define TDX_TD_ATTR_SEPT_VE_DISABLE_BIT	28
+#define TDX_TD_ATTR_SEPT_VE_DISABLE	BIT_ULL(TDX_TD_ATTR_SEPT_VE_DISABLE_BIT)
+#define TDX_TD_ATTR_MIGRATABLE_BIT	29
+#define TDX_TD_ATTR_MIGRATABLE		BIT_ULL(TDX_TD_ATTR_MIGRATABLE_BIT)
+#define TDX_TD_ATTR_PKS_BIT		30
+#define TDX_TD_ATTR_PKS			BIT_ULL(TDX_TD_ATTR_PKS_BIT)
+#define TDX_TD_ATTR_KL_BIT		31
+#define TDX_TD_ATTR_KL			BIT_ULL(TDX_TD_ATTR_KL_BIT)
+#define TDX_TD_ATTR_TPA_BIT		62
+#define TDX_TD_ATTR_TPA			BIT_ULL(TDX_TD_ATTR_TPA_BIT)
+#define TDX_TD_ATTR_PERFMON_BIT		63
+#define TDX_TD_ATTR_PERFMON		BIT_ULL(TDX_TD_ATTR_PERFMON_BIT)
 
 /* TDX TD-Scope Metadata. To be used by TDG.VM.WR and TDG.VM.RD */
 #define TDCS_CONFIG_FLAGS		0x1110000300000016
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index edde36097ddc3..bcfeb5e7c0edf 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -142,13 +142,13 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 	u64 exit_info_2;
 	u32 exit_int_info;
 	u32 exit_int_info_err;
-	u64 nested_ctl;
+	u64 misc_ctl;
 	u64 avic_vapic_bar;
 	u64 ghcb_gpa;
 	u32 event_inj;
 	u32 event_inj_err;
 	u64 nested_cr3;
-	u64 virt_ext;
+	u64 misc_ctl2;
 	u32 clean;
 	u32 reserved_5;
 	u64 next_rip;
@@ -182,6 +182,8 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define TLB_CONTROL_FLUSH_ASID 3
 #define TLB_CONTROL_FLUSH_ASID_LOCAL 7
 
+#define TLB_CONTROL_MASK GENMASK(2, 0)
+
 #define ERAP_CONTROL_ALLOW_LARGER_RAP BIT(0)
 #define ERAP_CONTROL_CLEAR_RAP BIT(1)
 
@@ -222,8 +224,7 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define X2APIC_MODE_SHIFT 30
 #define X2APIC_MODE_MASK (1 << X2APIC_MODE_SHIFT)
 
-#define LBR_CTL_ENABLE_MASK BIT_ULL(0)
-#define VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK BIT_ULL(1)
+#define SVM_INT_VECTOR_MASK GENMASK(7, 0)
 
 #define SVM_INTERRUPT_SHADOW_MASK	BIT_ULL(0)
 #define SVM_GUEST_INTERRUPT_MASK	BIT_ULL(1)
@@ -239,10 +240,12 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT)
 #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT)
 
-#define SVM_NESTED_CTL_NP_ENABLE	BIT(0)
-#define SVM_NESTED_CTL_SEV_ENABLE	BIT(1)
-#define SVM_NESTED_CTL_SEV_ES_ENABLE	BIT(2)
+#define SVM_MISC_ENABLE_NP		BIT(0)
+#define SVM_MISC_ENABLE_SEV		BIT(1)
+#define SVM_MISC_ENABLE_SEV_ES	BIT(2)
 
+#define SVM_MISC2_ENABLE_V_LBR	BIT_ULL(0)
+#define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE	BIT_ULL(1)
 
 #define SVM_TSC_RATIO_RSVD	0xffffff0000000000ULL
 #define SVM_TSC_RATIO_MIN	0x0000000000000001ULL
@@ -636,6 +639,9 @@ static inline void __unused_size_checks(void)
 #define SVM_EVTINJ_VALID (1 << 31)
 #define SVM_EVTINJ_VALID_ERR (1 << 11)
 
+#define SVM_EVTINJ_RESERVED_BITS ~(SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | \
+				   SVM_EVTINJ_VALID_ERR | SVM_EVTINJ_VALID)
+
 #define SVM_EXITINTINFO_VEC_MASK SVM_EVTINJ_VEC_MASK
 #define SVM_EXITINTINFO_TYPE_MASK SVM_EVTINJ_TYPE_MASK
 
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 6b338d7f01b7d..a149740b24e8b 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -145,8 +145,6 @@ static __always_inline u64 sc_retry(sc_func_t func, u64 fn,
 #define seamcall(_fn, _args)		sc_retry(__seamcall, (_fn), (_args))
 #define seamcall_ret(_fn, _args)	sc_retry(__seamcall_ret, (_fn), (_args))
 #define seamcall_saved_ret(_fn, _args)	sc_retry(__seamcall_saved_ret, (_fn), (_args))
-int tdx_cpu_enable(void);
-int tdx_enable(void);
 const char *tdx_dump_mce_info(struct mce *m);
 const struct tdx_sys_info *tdx_get_sysinfo(void);
 
@@ -223,8 +221,6 @@ u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td);
 u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page);
 #else
 static inline void tdx_init(void) { }
-static inline int tdx_cpu_enable(void) { return -ENODEV; }
-static inline int tdx_enable(void)  { return -ENODEV; }
 static inline u32 tdx_get_nr_guest_keyids(void) { return 0; }
 static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; }
 static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; }
diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h
index 060a2ad744bff..40689c8dc67eb 100644
--- a/arch/x86/include/asm/tdx_global_metadata.h
+++ b/arch/x86/include/asm/tdx_global_metadata.h
@@ -5,6 +5,12 @@
 
 #include <linux/types.h>
 
+struct tdx_sys_info_version {
+	u16 minor_version;
+	u16 major_version;
+	u16 update_version;
+};
+
 struct tdx_sys_info_features {
 	u64 tdx_features0;
 };
@@ -35,6 +41,7 @@ struct tdx_sys_info_td_conf {
 };
 
 struct tdx_sys_info {
+	struct tdx_sys_info_version version;
 	struct tdx_sys_info_features features;
 	struct tdx_sys_info_tdmr tdmr;
 	struct tdx_sys_info_td_ctrl td_ctrl;
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 5a3cdc439e38d..0545fe75c3fa1 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -172,6 +172,28 @@ struct tlb_state_shared {
 };
 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared);
 
+/*
+ * Please ignore the name of this function.  It should be called
+ * switch_to_kernel_thread().
+ *
+ * enter_lazy_tlb() is a hint from the scheduler that we are entering a
+ * kernel thread or other context without an mm.  Acceptable implementations
+ * include doing nothing whatsoever, switching to init_mm, or various clever
+ * lazy tricks to try to minimize TLB flushes.
+ *
+ * The scheduler reserves the right to call enter_lazy_tlb() several times
+ * in a row.  It will notify us that we're going back to a real mm by
+ * calling switch_mm_irqs_off().
+ */
+#define enter_lazy_tlb enter_lazy_tlb
+static __always_inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
+{
+	if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)
+		return;
+
+	this_cpu_write(cpu_tlbstate_shared.is_lazy, true);
+}
+
 bool nmi_uaccess_okay(void);
 #define nmi_uaccess_okay nmi_uaccess_okay
 
@@ -480,6 +502,10 @@ static inline void cpu_tlbstate_update_lam(unsigned long lam, u64 untag_mask)
 {
 }
 #endif
+#else /* !MODULE */
+#define enter_lazy_tlb enter_lazy_tlb
+extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
+	__compiletime_error("enter_lazy_tlb() should not be used in modules");
 #endif /* !MODULE */
 
 static inline void __native_tlb_flush_global(unsigned long cr4)
diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h
new file mode 100644
index 0000000000000..1558a0673d068
--- /dev/null
+++ b/arch/x86/include/asm/virt.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_X86_VIRT_H
+#define _ASM_X86_VIRT_H
+
+#include <asm/reboot.h>
+
+typedef void (cpu_emergency_virt_cb)(void);
+
+#if IS_ENABLED(CONFIG_KVM_X86)
+extern bool virt_rebooting;
+
+void __init x86_virt_init(void);
+
+int x86_virt_get_ref(int feat);
+void x86_virt_put_ref(int feat);
+
+int x86_virt_emergency_disable_virtualization_cpu(void);
+
+void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback);
+void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback);
+#else
+static __always_inline void x86_virt_init(void) {}
+static inline int x86_virt_emergency_disable_virtualization_cpu(void) { return -ENOENT; }
+#endif
+
+#endif /* _ASM_X86_VIRT_H */
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index b92ff87e35605..37080382df548 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -20,6 +20,17 @@
 #include <asm/trapnr.h>
 #include <asm/vmxfeatures.h>
 
+struct vmcs_hdr {
+	u32 revision_id:31;
+	u32 shadow_vmcs:1;
+};
+
+struct vmcs {
+	struct vmcs_hdr hdr;
+	u32 abort;
+	char data[];
+};
+
 #define VMCS_CONTROL_BIT(x)	BIT(VMX_FEATURE_##x & 0x1f)
 
 /*
diff --git a/arch/x86/include/asm/vsyscall.h b/arch/x86/include/asm/vsyscall.h
index 472f0263dbc61..538053b1656ac 100644
--- a/arch/x86/include/asm/vsyscall.h
+++ b/arch/x86/include/asm/vsyscall.h
@@ -14,12 +14,17 @@ extern void set_vsyscall_pgtable_user_bits(pgd_t *root);
  * Called on instruction fetch fault in vsyscall page.
  * Returns true if handled.
  */
-extern bool emulate_vsyscall(unsigned long error_code,
-			     struct pt_regs *regs, unsigned long address);
+bool emulate_vsyscall_pf(unsigned long error_code, struct pt_regs *regs, unsigned long address);
+bool emulate_vsyscall_gp(struct pt_regs *regs);
 #else
 static inline void map_vsyscall(void) {}
-static inline bool emulate_vsyscall(unsigned long error_code,
-				    struct pt_regs *regs, unsigned long address)
+static inline bool emulate_vsyscall_pf(unsigned long error_code,
+				       struct pt_regs *regs, unsigned long address)
+{
+	return false;
+}
+
+static inline bool emulate_vsyscall_gp(struct pt_regs *regs)
 {
 	return false;
 }
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
deleted file mode 100644
index 7b0307acc4103..0000000000000
--- a/arch/x86/include/asm/xor.h
+++ /dev/null
@@ -1,502 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_H
-#define _ASM_X86_XOR_H
-
-/*
- * Optimized RAID-5 checksumming functions for SSE.
- */
-
-/*
- * Cache avoiding checksumming functions utilizing KNI instructions
- * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
- */
-
-/*
- * Based on
- * High-speed RAID5 checksumming functions utilizing SSE instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
-
-/*
- * x86-64 changes / gcc fixes from Andi Kleen.
- * Copyright 2002 Andi Kleen, SuSE Labs.
- *
- * This hasn't been optimized for the hammer yet, but there are likely
- * no advantages to be gotten from x86-64 here anyways.
- */
-
-#include <asm/fpu/api.h>
-
-#ifdef CONFIG_X86_32
-/* reduce register pressure */
-# define XOR_CONSTANT_CONSTRAINT "i"
-#else
-# define XOR_CONSTANT_CONSTRAINT "re"
-#endif
-
-#define OFFS(x)		"16*("#x")"
-#define PF_OFFS(x)	"256+16*("#x")"
-#define PF0(x)		"	prefetchnta "PF_OFFS(x)"(%[p1])		;\n"
-#define LD(x, y)	"	movaps "OFFS(x)"(%[p1]), %%xmm"#y"	;\n"
-#define ST(x, y)	"	movaps %%xmm"#y", "OFFS(x)"(%[p1])	;\n"
-#define PF1(x)		"	prefetchnta "PF_OFFS(x)"(%[p2])		;\n"
-#define PF2(x)		"	prefetchnta "PF_OFFS(x)"(%[p3])		;\n"
-#define PF3(x)		"	prefetchnta "PF_OFFS(x)"(%[p4])		;\n"
-#define PF4(x)		"	prefetchnta "PF_OFFS(x)"(%[p5])		;\n"
-#define XO1(x, y)	"	xorps "OFFS(x)"(%[p2]), %%xmm"#y"	;\n"
-#define XO2(x, y)	"	xorps "OFFS(x)"(%[p3]), %%xmm"#y"	;\n"
-#define XO3(x, y)	"	xorps "OFFS(x)"(%[p4]), %%xmm"#y"	;\n"
-#define XO4(x, y)	"	xorps "OFFS(x)"(%[p5]), %%xmm"#y"	;\n"
-#define NOP(x)
-
-#define BLK64(pf, op, i)				\
-		pf(i)					\
-		op(i, 0)				\
-			op(i + 1, 1)			\
-				op(i + 2, 2)		\
-					op(i + 3, 3)
-
-static void
-xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
-	  const unsigned long * __restrict p2)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)					\
-		LD(i, 0)				\
-			LD(i + 1, 1)			\
-		PF1(i)					\
-				PF1(i + 2)		\
-				LD(i + 2, 2)		\
-					LD(i + 3, 3)	\
-		PF0(i + 4)				\
-				PF0(i + 6)		\
-		XO1(i, 0)				\
-			XO1(i + 1, 1)			\
-				XO1(i + 2, 2)		\
-					XO1(i + 3, 3)	\
-		ST(i, 0)				\
-			ST(i + 1, 1)			\
-				ST(i + 2, 2)		\
-					ST(i + 3, 3)	\
-
-
-		PF0(0)
-				PF0(2)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines),
-	  [p1] "+r" (p1), [p2] "+r" (p2)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)			\
-		BLK64(PF0, LD, i)	\
-		BLK64(PF1, XO1, i)	\
-		BLK64(NOP, ST, i)	\
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines),
-	  [p1] "+r" (p1), [p2] "+r" (p2)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
-	  const unsigned long * __restrict p2,
-	  const unsigned long * __restrict p3)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-		PF1(i)					\
-				PF1(i + 2)		\
-		LD(i, 0)				\
-			LD(i + 1, 1)			\
-				LD(i + 2, 2)		\
-					LD(i + 3, 3)	\
-		PF2(i)					\
-				PF2(i + 2)		\
-		PF0(i + 4)				\
-				PF0(i + 6)		\
-		XO1(i, 0)				\
-			XO1(i + 1, 1)			\
-				XO1(i + 2, 2)		\
-					XO1(i + 3, 3)	\
-		XO2(i, 0)				\
-			XO2(i + 1, 1)			\
-				XO2(i + 2, 2)		\
-					XO2(i + 3, 3)	\
-		ST(i, 0)				\
-			ST(i + 1, 1)			\
-				ST(i + 2, 2)		\
-					ST(i + 3, 3)	\
-
-
-		PF0(0)
-				PF0(2)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines),
-	  [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)			\
-		BLK64(PF0, LD, i)	\
-		BLK64(PF1, XO1, i)	\
-		BLK64(PF2, XO2, i)	\
-		BLK64(NOP, ST, i)	\
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines),
-	  [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
-	  const unsigned long * __restrict p2,
-	  const unsigned long * __restrict p3,
-	  const unsigned long * __restrict p4)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-		PF1(i)					\
-				PF1(i + 2)		\
-		LD(i, 0)				\
-			LD(i + 1, 1)			\
-				LD(i + 2, 2)		\
-					LD(i + 3, 3)	\
-		PF2(i)					\
-				PF2(i + 2)		\
-		XO1(i, 0)				\
-			XO1(i + 1, 1)			\
-				XO1(i + 2, 2)		\
-					XO1(i + 3, 3)	\
-		PF3(i)					\
-				PF3(i + 2)		\
-		PF0(i + 4)				\
-				PF0(i + 6)		\
-		XO2(i, 0)				\
-			XO2(i + 1, 1)			\
-				XO2(i + 2, 2)		\
-					XO2(i + 3, 3)	\
-		XO3(i, 0)				\
-			XO3(i + 1, 1)			\
-				XO3(i + 2, 2)		\
-					XO3(i + 3, 3)	\
-		ST(i, 0)				\
-			ST(i + 1, 1)			\
-				ST(i + 2, 2)		\
-					ST(i + 3, 3)	\
-
-
-		PF0(0)
-				PF0(2)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       add %[inc], %[p4]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines), [p1] "+r" (p1),
-	  [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)			\
-		BLK64(PF0, LD, i)	\
-		BLK64(PF1, XO1, i)	\
-		BLK64(PF2, XO2, i)	\
-		BLK64(PF3, XO3, i)	\
-		BLK64(NOP, ST, i)	\
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       add %[inc], %[p4]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines), [p1] "+r" (p1),
-	  [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
-	  const unsigned long * __restrict p2,
-	  const unsigned long * __restrict p3,
-	  const unsigned long * __restrict p4,
-	  const unsigned long * __restrict p5)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
-		PF1(i)					\
-				PF1(i + 2)		\
-		LD(i, 0)				\
-			LD(i + 1, 1)			\
-				LD(i + 2, 2)		\
-					LD(i + 3, 3)	\
-		PF2(i)					\
-				PF2(i + 2)		\
-		XO1(i, 0)				\
-			XO1(i + 1, 1)			\
-				XO1(i + 2, 2)		\
-					XO1(i + 3, 3)	\
-		PF3(i)					\
-				PF3(i + 2)		\
-		XO2(i, 0)				\
-			XO2(i + 1, 1)			\
-				XO2(i + 2, 2)		\
-					XO2(i + 3, 3)	\
-		PF4(i)					\
-				PF4(i + 2)		\
-		PF0(i + 4)				\
-				PF0(i + 6)		\
-		XO3(i, 0)				\
-			XO3(i + 1, 1)			\
-				XO3(i + 2, 2)		\
-					XO3(i + 3, 3)	\
-		XO4(i, 0)				\
-			XO4(i + 1, 1)			\
-				XO4(i + 2, 2)		\
-					XO4(i + 3, 3)	\
-		ST(i, 0)				\
-			ST(i + 1, 1)			\
-				ST(i + 2, 2)		\
-					ST(i + 3, 3)	\
-
-
-		PF0(0)
-				PF0(2)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       add %[inc], %[p4]       ;\n"
-	"       add %[inc], %[p5]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
-	  [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4,
-	       const unsigned long * __restrict p5)
-{
-	unsigned long lines = bytes >> 8;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)			\
-		BLK64(PF0, LD, i)	\
-		BLK64(PF1, XO1, i)	\
-		BLK64(PF2, XO2, i)	\
-		BLK64(PF3, XO3, i)	\
-		BLK64(PF4, XO4, i)	\
-		BLK64(NOP, ST, i)	\
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-		BLOCK(0)
-		BLOCK(4)
-		BLOCK(8)
-		BLOCK(12)
-
-	"       add %[inc], %[p1]       ;\n"
-	"       add %[inc], %[p2]       ;\n"
-	"       add %[inc], %[p3]       ;\n"
-	"       add %[inc], %[p4]       ;\n"
-	"       add %[inc], %[p5]       ;\n"
-	"       dec %[cnt]              ;\n"
-	"       jnz 1b                  ;\n"
-	: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
-	  [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
-	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_sse_pf64 = {
-	.name = "prefetch64-sse",
-	.do_2 = xor_sse_2_pf64,
-	.do_3 = xor_sse_3_pf64,
-	.do_4 = xor_sse_4_pf64,
-	.do_5 = xor_sse_5_pf64,
-};
-
-#undef LD
-#undef XO1
-#undef XO2
-#undef XO3
-#undef XO4
-#undef ST
-#undef NOP
-#undef BLK64
-#undef BLOCK
-
-#undef XOR_CONSTANT_CONSTRAINT
-
-#ifdef CONFIG_X86_32
-# include <asm/xor_32.h>
-#else
-# include <asm/xor_64.h>
-#endif
-
-#define XOR_SELECT_TEMPLATE(FASTEST) \
-	AVX_SELECT(FASTEST)
-
-#endif /* _ASM_X86_XOR_H */
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
deleted file mode 100644
index 7a6b9474591e7..0000000000000
--- a/arch/x86/include/asm/xor_32.h
+++ /dev/null
@@ -1,573 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_32_H
-#define _ASM_X86_XOR_32_H
-
-/*
- * Optimized RAID-5 checksumming functions for MMX.
- */
-
-/*
- * High-speed RAID5 checksumming functions utilizing MMX instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
-
-#define LD(x, y)	"       movq   8*("#x")(%1), %%mm"#y"   ;\n"
-#define ST(x, y)	"       movq %%mm"#y",   8*("#x")(%1)   ;\n"
-#define XO1(x, y)	"       pxor   8*("#x")(%2), %%mm"#y"   ;\n"
-#define XO2(x, y)	"       pxor   8*("#x")(%3), %%mm"#y"   ;\n"
-#define XO3(x, y)	"       pxor   8*("#x")(%4), %%mm"#y"   ;\n"
-#define XO4(x, y)	"       pxor   8*("#x")(%5), %%mm"#y"   ;\n"
-
-#include <asm/fpu/api.h>
-
-static void
-xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2)
-{
-	unsigned long lines = bytes >> 7;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)				\
-	LD(i, 0)				\
-		LD(i + 1, 1)			\
-			LD(i + 2, 2)		\
-				LD(i + 3, 3)	\
-	XO1(i, 0)				\
-	ST(i, 0)				\
-		XO1(i+1, 1)			\
-		ST(i+1, 1)			\
-			XO1(i + 2, 2)		\
-			ST(i + 2, 2)		\
-				XO1(i + 3, 3)	\
-				ST(i + 3, 3)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-	BLOCK(0)
-	BLOCK(4)
-	BLOCK(8)
-	BLOCK(12)
-
-	"       addl $128, %1         ;\n"
-	"       addl $128, %2         ;\n"
-	"       decl %0               ;\n"
-	"       jnz 1b                ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2)
-	:
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3)
-{
-	unsigned long lines = bytes >> 7;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)				\
-	LD(i, 0)				\
-		LD(i + 1, 1)			\
-			LD(i + 2, 2)		\
-				LD(i + 3, 3)	\
-	XO1(i, 0)				\
-		XO1(i + 1, 1)			\
-			XO1(i + 2, 2)		\
-				XO1(i + 3, 3)	\
-	XO2(i, 0)				\
-	ST(i, 0)				\
-		XO2(i + 1, 1)			\
-		ST(i + 1, 1)			\
-			XO2(i + 2, 2)		\
-			ST(i + 2, 2)		\
-				XO2(i + 3, 3)	\
-				ST(i + 3, 3)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-	BLOCK(0)
-	BLOCK(4)
-	BLOCK(8)
-	BLOCK(12)
-
-	"       addl $128, %1         ;\n"
-	"       addl $128, %2         ;\n"
-	"       addl $128, %3         ;\n"
-	"       decl %0               ;\n"
-	"       jnz 1b                ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3)
-	:
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3,
-	      const unsigned long * __restrict p4)
-{
-	unsigned long lines = bytes >> 7;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)				\
-	LD(i, 0)				\
-		LD(i + 1, 1)			\
-			LD(i + 2, 2)		\
-				LD(i + 3, 3)	\
-	XO1(i, 0)				\
-		XO1(i + 1, 1)			\
-			XO1(i + 2, 2)		\
-				XO1(i + 3, 3)	\
-	XO2(i, 0)				\
-		XO2(i + 1, 1)			\
-			XO2(i + 2, 2)		\
-				XO2(i + 3, 3)	\
-	XO3(i, 0)				\
-	ST(i, 0)				\
-		XO3(i + 1, 1)			\
-		ST(i + 1, 1)			\
-			XO3(i + 2, 2)		\
-			ST(i + 2, 2)		\
-				XO3(i + 3, 3)	\
-				ST(i + 3, 3)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-	BLOCK(0)
-	BLOCK(4)
-	BLOCK(8)
-	BLOCK(12)
-
-	"       addl $128, %1         ;\n"
-	"       addl $128, %2         ;\n"
-	"       addl $128, %3         ;\n"
-	"       addl $128, %4         ;\n"
-	"       decl %0               ;\n"
-	"       jnz 1b                ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
-	:
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-
-static void
-xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3,
-	      const unsigned long * __restrict p4,
-	      const unsigned long * __restrict p5)
-{
-	unsigned long lines = bytes >> 7;
-
-	kernel_fpu_begin();
-
-	/* Make sure GCC forgets anything it knows about p4 or p5,
-	   such that it won't pass to the asm volatile below a
-	   register that is shared with any other variable.  That's
-	   because we modify p4 and p5 there, but we can't mark them
-	   as read/write, otherwise we'd overflow the 10-asm-operands
-	   limit of GCC < 3.1.  */
-	asm("" : "+r" (p4), "+r" (p5));
-
-	asm volatile(
-#undef BLOCK
-#define BLOCK(i)				\
-	LD(i, 0)				\
-		LD(i + 1, 1)			\
-			LD(i + 2, 2)		\
-				LD(i + 3, 3)	\
-	XO1(i, 0)				\
-		XO1(i + 1, 1)			\
-			XO1(i + 2, 2)		\
-				XO1(i + 3, 3)	\
-	XO2(i, 0)				\
-		XO2(i + 1, 1)			\
-			XO2(i + 2, 2)		\
-				XO2(i + 3, 3)	\
-	XO3(i, 0)				\
-		XO3(i + 1, 1)			\
-			XO3(i + 2, 2)		\
-				XO3(i + 3, 3)	\
-	XO4(i, 0)				\
-	ST(i, 0)				\
-		XO4(i + 1, 1)			\
-		ST(i + 1, 1)			\
-			XO4(i + 2, 2)		\
-			ST(i + 2, 2)		\
-				XO4(i + 3, 3)	\
-				ST(i + 3, 3)
-
-	" .align 32			;\n"
-	" 1:                            ;\n"
-
-	BLOCK(0)
-	BLOCK(4)
-	BLOCK(8)
-	BLOCK(12)
-
-	"       addl $128, %1         ;\n"
-	"       addl $128, %2         ;\n"
-	"       addl $128, %3         ;\n"
-	"       addl $128, %4         ;\n"
-	"       addl $128, %5         ;\n"
-	"       decl %0               ;\n"
-	"       jnz 1b                ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3)
-	: "r" (p4), "r" (p5)
-	: "memory");
-
-	/* p4 and p5 were modified, and now the variables are dead.
-	   Clobber them just to be sure nobody does something stupid
-	   like assuming they have some legal value.  */
-	asm("" : "=r" (p4), "=r" (p5));
-
-	kernel_fpu_end();
-}
-
-#undef LD
-#undef XO1
-#undef XO2
-#undef XO3
-#undef XO4
-#undef ST
-#undef BLOCK
-
-static void
-xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2)
-{
-	unsigned long lines = bytes >> 6;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-	" .align 32	             ;\n"
-	" 1:                         ;\n"
-	"       movq   (%1), %%mm0   ;\n"
-	"       movq  8(%1), %%mm1   ;\n"
-	"       pxor   (%2), %%mm0   ;\n"
-	"       movq 16(%1), %%mm2   ;\n"
-	"       movq %%mm0,   (%1)   ;\n"
-	"       pxor  8(%2), %%mm1   ;\n"
-	"       movq 24(%1), %%mm3   ;\n"
-	"       movq %%mm1,  8(%1)   ;\n"
-	"       pxor 16(%2), %%mm2   ;\n"
-	"       movq 32(%1), %%mm4   ;\n"
-	"       movq %%mm2, 16(%1)   ;\n"
-	"       pxor 24(%2), %%mm3   ;\n"
-	"       movq 40(%1), %%mm5   ;\n"
-	"       movq %%mm3, 24(%1)   ;\n"
-	"       pxor 32(%2), %%mm4   ;\n"
-	"       movq 48(%1), %%mm6   ;\n"
-	"       movq %%mm4, 32(%1)   ;\n"
-	"       pxor 40(%2), %%mm5   ;\n"
-	"       movq 56(%1), %%mm7   ;\n"
-	"       movq %%mm5, 40(%1)   ;\n"
-	"       pxor 48(%2), %%mm6   ;\n"
-	"       pxor 56(%2), %%mm7   ;\n"
-	"       movq %%mm6, 48(%1)   ;\n"
-	"       movq %%mm7, 56(%1)   ;\n"
-
-	"       addl $64, %1         ;\n"
-	"       addl $64, %2         ;\n"
-	"       decl %0              ;\n"
-	"       jnz 1b               ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2)
-	:
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3)
-{
-	unsigned long lines = bytes >> 6;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-	" .align 32,0x90             ;\n"
-	" 1:                         ;\n"
-	"       movq   (%1), %%mm0   ;\n"
-	"       movq  8(%1), %%mm1   ;\n"
-	"       pxor   (%2), %%mm0   ;\n"
-	"       movq 16(%1), %%mm2   ;\n"
-	"       pxor  8(%2), %%mm1   ;\n"
-	"       pxor   (%3), %%mm0   ;\n"
-	"       pxor 16(%2), %%mm2   ;\n"
-	"       movq %%mm0,   (%1)   ;\n"
-	"       pxor  8(%3), %%mm1   ;\n"
-	"       pxor 16(%3), %%mm2   ;\n"
-	"       movq 24(%1), %%mm3   ;\n"
-	"       movq %%mm1,  8(%1)   ;\n"
-	"       movq 32(%1), %%mm4   ;\n"
-	"       movq 40(%1), %%mm5   ;\n"
-	"       pxor 24(%2), %%mm3   ;\n"
-	"       movq %%mm2, 16(%1)   ;\n"
-	"       pxor 32(%2), %%mm4   ;\n"
-	"       pxor 24(%3), %%mm3   ;\n"
-	"       pxor 40(%2), %%mm5   ;\n"
-	"       movq %%mm3, 24(%1)   ;\n"
-	"       pxor 32(%3), %%mm4   ;\n"
-	"       pxor 40(%3), %%mm5   ;\n"
-	"       movq 48(%1), %%mm6   ;\n"
-	"       movq %%mm4, 32(%1)   ;\n"
-	"       movq 56(%1), %%mm7   ;\n"
-	"       pxor 48(%2), %%mm6   ;\n"
-	"       movq %%mm5, 40(%1)   ;\n"
-	"       pxor 56(%2), %%mm7   ;\n"
-	"       pxor 48(%3), %%mm6   ;\n"
-	"       pxor 56(%3), %%mm7   ;\n"
-	"       movq %%mm6, 48(%1)   ;\n"
-	"       movq %%mm7, 56(%1)   ;\n"
-
-	"       addl $64, %1         ;\n"
-	"       addl $64, %2         ;\n"
-	"       addl $64, %3         ;\n"
-	"       decl %0              ;\n"
-	"       jnz 1b               ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3)
-	:
-	: "memory" );
-
-	kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3,
-	     const unsigned long * __restrict p4)
-{
-	unsigned long lines = bytes >> 6;
-
-	kernel_fpu_begin();
-
-	asm volatile(
-	" .align 32,0x90             ;\n"
-	" 1:                         ;\n"
-	"       movq   (%1), %%mm0   ;\n"
-	"       movq  8(%1), %%mm1   ;\n"
-	"       pxor   (%2), %%mm0   ;\n"
-	"       movq 16(%1), %%mm2   ;\n"
-	"       pxor  8(%2), %%mm1   ;\n"
-	"       pxor   (%3), %%mm0   ;\n"
-	"       pxor 16(%2), %%mm2   ;\n"
-	"       pxor  8(%3), %%mm1   ;\n"
-	"       pxor   (%4), %%mm0   ;\n"
-	"       movq 24(%1), %%mm3   ;\n"
-	"       pxor 16(%3), %%mm2   ;\n"
-	"       pxor  8(%4), %%mm1   ;\n"
-	"       movq %%mm0,   (%1)   ;\n"
-	"       movq 32(%1), %%mm4   ;\n"
-	"       pxor 24(%2), %%mm3   ;\n"
-	"       pxor 16(%4), %%mm2   ;\n"
-	"       movq %%mm1,  8(%1)   ;\n"
-	"       movq 40(%1), %%mm5   ;\n"
-	"       pxor 32(%2), %%mm4   ;\n"
-	"       pxor 24(%3), %%mm3   ;\n"
-	"       movq %%mm2, 16(%1)   ;\n"
-	"       pxor 40(%2), %%mm5   ;\n"
-	"       pxor 32(%3), %%mm4   ;\n"
-	"       pxor 24(%4), %%mm3   ;\n"
-	"       movq %%mm3, 24(%1)   ;\n"
-	"       movq 56(%1), %%mm7   ;\n"
-	"       movq 48(%1), %%mm6   ;\n"
-	"       pxor 40(%3), %%mm5   ;\n"
-	"       pxor 32(%4), %%mm4   ;\n"
-	"       pxor 48(%2), %%mm6   ;\n"
-	"       movq %%mm4, 32(%1)   ;\n"
-	"       pxor 56(%2), %%mm7   ;\n"
-	"       pxor 40(%4), %%mm5   ;\n"
-	"       pxor 48(%3), %%mm6   ;\n"
-	"       pxor 56(%3), %%mm7   ;\n"
-	"       movq %%mm5, 40(%1)   ;\n"
-	"       pxor 48(%4), %%mm6   ;\n"
-	"       pxor 56(%4), %%mm7   ;\n"
-	"       movq %%mm6, 48(%1)   ;\n"
-	"       movq %%mm7, 56(%1)   ;\n"
-
-	"       addl $64, %1         ;\n"
-	"       addl $64, %2         ;\n"
-	"       addl $64, %3         ;\n"
-	"       addl $64, %4         ;\n"
-	"       decl %0              ;\n"
-	"       jnz 1b               ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
-	:
-	: "memory");
-
-	kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3,
-	     const unsigned long * __restrict p4,
-	     const unsigned long * __restrict p5)
-{
-	unsigned long lines = bytes >> 6;
-
-	kernel_fpu_begin();
-
-	/* Make sure GCC forgets anything it knows about p4 or p5,
-	   such that it won't pass to the asm volatile below a
-	   register that is shared with any other variable.  That's
-	   because we modify p4 and p5 there, but we can't mark them
-	   as read/write, otherwise we'd overflow the 10-asm-operands
-	   limit of GCC < 3.1.  */
-	asm("" : "+r" (p4), "+r" (p5));
-
-	asm volatile(
-	" .align 32,0x90             ;\n"
-	" 1:                         ;\n"
-	"       movq   (%1), %%mm0   ;\n"
-	"       movq  8(%1), %%mm1   ;\n"
-	"       pxor   (%2), %%mm0   ;\n"
-	"       pxor  8(%2), %%mm1   ;\n"
-	"       movq 16(%1), %%mm2   ;\n"
-	"       pxor   (%3), %%mm0   ;\n"
-	"       pxor  8(%3), %%mm1   ;\n"
-	"       pxor 16(%2), %%mm2   ;\n"
-	"       pxor   (%4), %%mm0   ;\n"
-	"       pxor  8(%4), %%mm1   ;\n"
-	"       pxor 16(%3), %%mm2   ;\n"
-	"       movq 24(%1), %%mm3   ;\n"
-	"       pxor   (%5), %%mm0   ;\n"
-	"       pxor  8(%5), %%mm1   ;\n"
-	"       movq %%mm0,   (%1)   ;\n"
-	"       pxor 16(%4), %%mm2   ;\n"
-	"       pxor 24(%2), %%mm3   ;\n"
-	"       movq %%mm1,  8(%1)   ;\n"
-	"       pxor 16(%5), %%mm2   ;\n"
-	"       pxor 24(%3), %%mm3   ;\n"
-	"       movq 32(%1), %%mm4   ;\n"
-	"       movq %%mm2, 16(%1)   ;\n"
-	"       pxor 24(%4), %%mm3   ;\n"
-	"       pxor 32(%2), %%mm4   ;\n"
-	"       movq 40(%1), %%mm5   ;\n"
-	"       pxor 24(%5), %%mm3   ;\n"
-	"       pxor 32(%3), %%mm4   ;\n"
-	"       pxor 40(%2), %%mm5   ;\n"
-	"       movq %%mm3, 24(%1)   ;\n"
-	"       pxor 32(%4), %%mm4   ;\n"
-	"       pxor 40(%3), %%mm5   ;\n"
-	"       movq 48(%1), %%mm6   ;\n"
-	"       movq 56(%1), %%mm7   ;\n"
-	"       pxor 32(%5), %%mm4   ;\n"
-	"       pxor 40(%4), %%mm5   ;\n"
-	"       pxor 48(%2), %%mm6   ;\n"
-	"       pxor 56(%2), %%mm7   ;\n"
-	"       movq %%mm4, 32(%1)   ;\n"
-	"       pxor 48(%3), %%mm6   ;\n"
-	"       pxor 56(%3), %%mm7   ;\n"
-	"       pxor 40(%5), %%mm5   ;\n"
-	"       pxor 48(%4), %%mm6   ;\n"
-	"       pxor 56(%4), %%mm7   ;\n"
-	"       movq %%mm5, 40(%1)   ;\n"
-	"       pxor 48(%5), %%mm6   ;\n"
-	"       pxor 56(%5), %%mm7   ;\n"
-	"       movq %%mm6, 48(%1)   ;\n"
-	"       movq %%mm7, 56(%1)   ;\n"
-
-	"       addl $64, %1         ;\n"
-	"       addl $64, %2         ;\n"
-	"       addl $64, %3         ;\n"
-	"       addl $64, %4         ;\n"
-	"       addl $64, %5         ;\n"
-	"       decl %0              ;\n"
-	"       jnz 1b               ;\n"
-	: "+r" (lines),
-	  "+r" (p1), "+r" (p2), "+r" (p3)
-	: "r" (p4), "r" (p5)
-	: "memory");
-
-	/* p4 and p5 were modified, and now the variables are dead.
-	   Clobber them just to be sure nobody does something stupid
-	   like assuming they have some legal value.  */
-	asm("" : "=r" (p4), "=r" (p5));
-
-	kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_pII_mmx = {
-	.name = "pII_mmx",
-	.do_2 = xor_pII_mmx_2,
-	.do_3 = xor_pII_mmx_3,
-	.do_4 = xor_pII_mmx_4,
-	.do_5 = xor_pII_mmx_5,
-};
-
-static struct xor_block_template xor_block_p5_mmx = {
-	.name = "p5_mmx",
-	.do_2 = xor_p5_mmx_2,
-	.do_3 = xor_p5_mmx_3,
-	.do_4 = xor_p5_mmx_4,
-	.do_5 = xor_p5_mmx_5,
-};
-
-static struct xor_block_template xor_block_pIII_sse = {
-	.name = "pIII_sse",
-	.do_2 = xor_sse_2,
-	.do_3 = xor_sse_3,
-	.do_4 = xor_sse_4,
-	.do_5 = xor_sse_5,
-};
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* Also try the generic routines.  */
-#include <asm-generic/xor.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
-   We may also be able to load into the L1 only depending on how the cpu
-   deals with a load to a line that is being prefetched.  */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES				\
-do {							\
-	AVX_XOR_SPEED;					\
-	if (boot_cpu_has(X86_FEATURE_XMM)) {				\
-		xor_speed(&xor_block_pIII_sse);		\
-		xor_speed(&xor_block_sse_pf64);		\
-	} else if (boot_cpu_has(X86_FEATURE_MMX)) {	\
-		xor_speed(&xor_block_pII_mmx);		\
-		xor_speed(&xor_block_p5_mmx);		\
-	} else {					\
-		xor_speed(&xor_block_8regs);		\
-		xor_speed(&xor_block_8regs_p);		\
-		xor_speed(&xor_block_32regs);		\
-		xor_speed(&xor_block_32regs_p);		\
-	}						\
-} while (0)
-
-#endif /* _ASM_X86_XOR_32_H */
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
deleted file mode 100644
index 0307e4ec50440..0000000000000
--- a/arch/x86/include/asm/xor_64.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_XOR_64_H
-#define _ASM_X86_XOR_64_H
-
-static struct xor_block_template xor_block_sse = {
-	.name = "generic_sse",
-	.do_2 = xor_sse_2,
-	.do_3 = xor_sse_3,
-	.do_4 = xor_sse_4,
-	.do_5 = xor_sse_5,
-};
-
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
-   We may also be able to load into the L1 only depending on how the cpu
-   deals with a load to a line that is being prefetched.  */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES			\
-do {						\
-	AVX_XOR_SPEED;				\
-	xor_speed(&xor_block_sse_pf64);		\
-	xor_speed(&xor_block_sse);		\
-} while (0)
-
-#endif /* _ASM_X86_XOR_64_H */
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
deleted file mode 100644
index 7f81dd5897f41..0000000000000
--- a/arch/x86/include/asm/xor_avx.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef _ASM_X86_XOR_AVX_H
-#define _ASM_X86_XOR_AVX_H
-
-/*
- * Optimized RAID-5 checksumming functions for AVX
- *
- * Copyright (C) 2012 Intel Corporation
- * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
- *
- * Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
- */
-
-#include <linux/compiler.h>
-#include <asm/fpu/api.h>
-
-#define BLOCK4(i) \
-		BLOCK(32 * i, 0) \
-		BLOCK(32 * (i + 1), 1) \
-		BLOCK(32 * (i + 2), 2) \
-		BLOCK(32 * (i + 3), 3)
-
-#define BLOCK16() \
-		BLOCK4(0) \
-		BLOCK4(4) \
-		BLOCK4(8) \
-		BLOCK4(12)
-
-static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0,
-		      const unsigned long * __restrict p1)
-{
-	unsigned long lines = bytes >> 9;
-
-	kernel_fpu_begin();
-
-	while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
-	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p1[i / sizeof(*p1)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm"  #reg : : \
-		"m" (p0[i / sizeof(*p0)])); \
-	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
-		"=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
-		BLOCK16()
-
-		p0 = (unsigned long *)((uintptr_t)p0 + 512);
-		p1 = (unsigned long *)((uintptr_t)p1 + 512);
-	}
-
-	kernel_fpu_end();
-}
-
-static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
-		      const unsigned long * __restrict p1,
-		      const unsigned long * __restrict p2)
-{
-	unsigned long lines = bytes >> 9;
-
-	kernel_fpu_begin();
-
-	while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
-	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p2[i / sizeof(*p2)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p1[i / sizeof(*p1)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p0[i / sizeof(*p0)])); \
-	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
-		"=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
-		BLOCK16()
-
-		p0 = (unsigned long *)((uintptr_t)p0 + 512);
-		p1 = (unsigned long *)((uintptr_t)p1 + 512);
-		p2 = (unsigned long *)((uintptr_t)p2 + 512);
-	}
-
-	kernel_fpu_end();
-}
-
-static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
-		      const unsigned long * __restrict p1,
-		      const unsigned long * __restrict p2,
-		      const unsigned long * __restrict p3)
-{
-	unsigned long lines = bytes >> 9;
-
-	kernel_fpu_begin();
-
-	while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
-	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p3[i / sizeof(*p3)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p2[i / sizeof(*p2)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p1[i / sizeof(*p1)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p0[i / sizeof(*p0)])); \
-	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
-		"=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
-		BLOCK16();
-
-		p0 = (unsigned long *)((uintptr_t)p0 + 512);
-		p1 = (unsigned long *)((uintptr_t)p1 + 512);
-		p2 = (unsigned long *)((uintptr_t)p2 + 512);
-		p3 = (unsigned long *)((uintptr_t)p3 + 512);
-	}
-
-	kernel_fpu_end();
-}
-
-static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
-	     const unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3,
-	     const unsigned long * __restrict p4)
-{
-	unsigned long lines = bytes >> 9;
-
-	kernel_fpu_begin();
-
-	while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
-	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p4[i / sizeof(*p4)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p3[i / sizeof(*p3)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p2[i / sizeof(*p2)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p1[i / sizeof(*p1)])); \
-	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
-		"m" (p0[i / sizeof(*p0)])); \
-	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
-		"=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
-		BLOCK16()
-
-		p0 = (unsigned long *)((uintptr_t)p0 + 512);
-		p1 = (unsigned long *)((uintptr_t)p1 + 512);
-		p2 = (unsigned long *)((uintptr_t)p2 + 512);
-		p3 = (unsigned long *)((uintptr_t)p3 + 512);
-		p4 = (unsigned long *)((uintptr_t)p4 + 512);
-	}
-
-	kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_avx = {
-	.name = "avx",
-	.do_2 = xor_avx_2,
-	.do_3 = xor_avx_3,
-	.do_4 = xor_avx_4,
-	.do_5 = xor_avx_5,
-};
-
-#define AVX_XOR_SPEED \
-do { \
-	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE)) \
-		xor_speed(&xor_block_avx); \
-} while (0)
-
-#define AVX_SELECT(FASTEST) \
-	(boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE) ? &xor_block_avx : FASTEST)
-
-#endif
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index 0d4538fa6c31a..5f2b30d0405c8 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -197,13 +197,13 @@ struct kvm_msrs {
 	__u32 nmsrs; /* number of msrs in entries */
 	__u32 pad;
 
-	struct kvm_msr_entry entries[];
+	__DECLARE_FLEX_ARRAY(struct kvm_msr_entry, entries);
 };
 
 /* for KVM_GET_MSR_INDEX_LIST */
 struct kvm_msr_list {
 	__u32 nmsrs; /* number of msrs in entries */
-	__u32 indices[];
+	__DECLARE_FLEX_ARRAY(__u32, indices);
 };
 
 /* Maximum size of any access bitmap in bytes */
@@ -245,7 +245,7 @@ struct kvm_cpuid_entry {
 struct kvm_cpuid {
 	__u32 nent;
 	__u32 padding;
-	struct kvm_cpuid_entry entries[];
+	__DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry, entries);
 };
 
 struct kvm_cpuid_entry2 {
@@ -267,7 +267,7 @@ struct kvm_cpuid_entry2 {
 struct kvm_cpuid2 {
 	__u32 nent;
 	__u32 padding;
-	struct kvm_cpuid_entry2 entries[];
+	__DECLARE_FLEX_ARRAY(struct kvm_cpuid_entry2, entries);
 };
 
 /* for KVM_GET_PIT and KVM_SET_PIT */
@@ -398,7 +398,7 @@ struct kvm_xsave {
 	 * the contents of CPUID leaf 0xD on the host.
 	 */
 	__u32 region[1024];
-	__u32 extra[];
+	__DECLARE_FLEX_ARRAY(__u32, extra);
 };
 
 #define KVM_MAX_XCRS	16
@@ -566,7 +566,7 @@ struct kvm_pmu_event_filter {
 	__u32 fixed_counter_bitmap;
 	__u32 flags;
 	__u32 pad[4];
-	__u64 events[];
+	__DECLARE_FLEX_ARRAY(__u64, events);
 };
 
 #define KVM_PMU_EVENT_ALLOW 0
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 961714e6adae1..0c8970c4c3e33 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -412,23 +412,21 @@ EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
 /*
  * Program the next event, relative to now
  */
-static int lapic_next_event(unsigned long delta,
-			    struct clock_event_device *evt)
+static int lapic_next_event(unsigned long delta, struct clock_event_device *evt)
 {
 	apic_write(APIC_TMICT, delta);
 	return 0;
 }
 
-static int lapic_next_deadline(unsigned long delta,
-			       struct clock_event_device *evt)
+static int lapic_next_deadline(unsigned long delta, struct clock_event_device *evt)
 {
-	u64 tsc;
-
-	/* This MSR is special and need a special fence: */
-	weak_wrmsr_fence();
+	/*
+	 * There is no weak_wrmsr_fence() required here as all of this is purely
+	 * CPU local. Avoid the [ml]fence overhead.
+	 */
+	u64 tsc = rdtsc();
 
-	tsc = rdtsc();
-	wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
+	native_wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
 	return 0;
 }
 
@@ -452,7 +450,7 @@ static int lapic_timer_shutdown(struct clock_event_device *evt)
 	 * the timer _and_ zero the counter registers:
 	 */
 	if (v & APIC_LVT_TIMER_TSCDEADLINE)
-		wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
+		native_wrmsrq(MSR_IA32_TSC_DEADLINE, 0);
 	else
 		apic_write(APIC_TMICT, 0);
 
@@ -549,6 +547,11 @@ static __init bool apic_validate_deadline_timer(void)
 
 	if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
 		return false;
+
+	/* XEN_PV does not support it, but be paranoia about it */
+	if (boot_cpu_has(X86_FEATURE_XENPV))
+		goto clear;
+
 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 		return true;
 
@@ -561,9 +564,11 @@ static __init bool apic_validate_deadline_timer(void)
 	if (boot_cpu_data.microcode >= rev)
 		return true;
 
-	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
 	pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
 	       "please update microcode to version: 0x%x (or later)\n", rev);
+
+clear:
+	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
 	return false;
 }
 
@@ -586,14 +591,14 @@ static void setup_APIC_timer(void)
 
 	if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
 		levt->name = "lapic-deadline";
-		levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
-				    CLOCK_EVT_FEAT_DUMMY);
+		levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_DUMMY);
+		levt->features |= CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED;
+		levt->cs_id = CSID_X86_TSC;
 		levt->set_next_event = lapic_next_deadline;
-		clockevents_config_and_register(levt,
-						tsc_khz * (1000 / TSC_DIVISOR),
-						0xF, ~0UL);
-	} else
+		clockevents_config_and_register(levt, tsc_khz * (1000 / TSC_DIVISOR), 0xF, ~0UL);
+	} else {
 		clockevents_register_device(levt);
+	}
 
 	apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, true);
 }
diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c
index fb166662bc0d6..660aa9aa8bec2 100644
--- a/arch/x86/kernel/cpu/bus_lock.c
+++ b/arch/x86/kernel/cpu/bus_lock.c
@@ -391,34 +391,35 @@ static void __init split_lock_setup(struct cpuinfo_x86 *c)
 
 static void sld_state_show(void)
 {
-	if (!boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) &&
-	    !boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
+	const char *action = "warning";
+
+	if ((!boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) &&
+	     !boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) ||
+	    (sld_state == sld_off))
 		return;
 
-	switch (sld_state) {
-	case sld_off:
-		pr_info("disabled\n");
-		break;
-	case sld_warn:
-		if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) {
-			pr_info("#AC: crashing the kernel on kernel split_locks and warning on user-space split_locks\n");
-			if (cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
-					      "x86/splitlock", NULL, splitlock_cpu_offline) < 0)
-				pr_warn("No splitlock CPU offline handler\n");
-		} else if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) {
-			pr_info("#DB: warning on user-space bus_locks\n");
-		}
-		break;
-	case sld_fatal:
-		if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
-			pr_info("#AC: crashing the kernel on kernel split_locks and sending SIGBUS on user-space split_locks\n");
-		else if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT))
-			pr_info("#DB: sending SIGBUS on user-space bus_locks\n");
-		break;
-	case sld_ratelimit:
+	if (sld_state == sld_ratelimit) {
 		if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT))
 			pr_info("#DB: setting system wide bus lock rate limit to %u/sec\n", bld_ratelimit.burst);
-		break;
+		return;
+	} else if (sld_state == sld_fatal) {
+		action = "sending SIGBUS";
+	}
+
+	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) {
+		pr_info("#AC: crashing the kernel on kernel split_locks and %s on user-space split_locks\n", action);
+
+		/*
+		 * This is handling the case where a CPU goes offline at the
+		 * moment where split lock detection is disabled in the warn
+		 * setting, see split_lock_warn(). It doesn't have any effect
+		 * in the fatal case.
+		 */
+		if (cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/splitlock", NULL, splitlock_cpu_offline) < 0)
+			pr_warn("No splitlock CPU offline handler\n");
+
+	} else if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) {
+		pr_info("#DB: %s on user-space bus_locks\n", action);
 	}
 }
 
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a8ff4376c2860..c5abdd96e238f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -71,6 +71,7 @@
 #include <asm/traps.h>
 #include <asm/sev.h>
 #include <asm/tdx.h>
+#include <asm/virt.h>
 #include <asm/posted_intr.h>
 #include <asm/runtime-const.h>
 
@@ -409,27 +410,29 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c)
 	cr4_clear_bits(X86_CR4_UMIP);
 }
 
-static __always_inline void setup_lass(struct cpuinfo_x86 *c)
+static int enable_lass(unsigned int cpu)
+{
+	cr4_set_bits(X86_CR4_LASS);
+
+	return 0;
+}
+
+/*
+ * Finalize features that need to be enabled just before entering
+ * userspace. Note that this only runs on a single CPU. Use appropriate
+ * callbacks if all the CPUs need to reflect the same change.
+ */
+static int cpu_finalize_pre_userspace(void)
 {
 	if (!cpu_feature_enabled(X86_FEATURE_LASS))
-		return;
+		return 0;
 
-	/*
-	 * Legacy vsyscall page access causes a #GP when LASS is active.
-	 * Disable LASS because the #GP handler doesn't support vsyscall
-	 * emulation.
-	 *
-	 * Also disable LASS when running under EFI, as some runtime and
-	 * boot services rely on 1:1 mappings in the lower half.
-	 */
-	if (IS_ENABLED(CONFIG_X86_VSYSCALL_EMULATION) ||
-	    IS_ENABLED(CONFIG_EFI)) {
-		setup_clear_cpu_cap(X86_FEATURE_LASS);
-		return;
-	}
+	/* Runs on all online CPUs and future CPUs that come online. */
+	cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/lass:enable", enable_lass, NULL);
 
-	cr4_set_bits(X86_CR4_LASS);
+	return 0;
 }
+late_initcall(cpu_finalize_pre_userspace);
 
 /* These bits should not change their value after CPU init is finished. */
 static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
@@ -2048,7 +2051,6 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 	setup_smep(c);
 	setup_smap(c);
 	setup_umip(c);
-	setup_lass(c);
 
 	/* Enable FSGSBASE instructions if available. */
 	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
@@ -2154,6 +2156,7 @@ static __init void identify_boot_cpu(void)
 	cpu_detect_tlb(&boot_cpu_data);
 	setup_cr_pinning();
 
+	x86_virt_init();
 	tsx_init();
 	tdx_init();
 	lkgs_init();
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 646ff33c4651d..f28c0efb7c8f1 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -205,6 +205,7 @@ static void early_init_intel(struct cpuinfo_x86 *c)
 
 	if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
 		c->microcode = intel_get_microcode_revision();
+	c->intel_platform_id = intel_get_platform_id();
 
 	/* Now if any of them are set, check the blacklist and clear the lot */
 	if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c
index 6af1e8baeb0f2..4604802692dae 100644
--- a/arch/x86/kernel/cpu/match.c
+++ b/arch/x86/kernel/cpu/match.c
@@ -76,6 +76,9 @@ const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match)
 		if (m->steppings != X86_STEPPING_ANY &&
 		    !(BIT(c->x86_stepping) & m->steppings))
 			continue;
+		if (m->platform_mask != X86_PLATFORM_ANY &&
+		    !(BIT(c->intel_platform_id) & m->platform_mask))
+			continue;
 		if (m->feature != X86_FEATURE_ANY && !cpu_has(c, m->feature))
 			continue;
 		if (!x86_match_vendor_cpu_type(c, m))
diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c
index da13c1e37f87a..146f4207a863a 100644
--- a/arch/x86/kernel/cpu/mce/amd.c
+++ b/arch/x86/kernel/cpu/mce/amd.c
@@ -95,39 +95,49 @@ static DEFINE_PER_CPU_READ_MOSTLY(struct smca_bank[MAX_NR_BANKS], smca_banks);
 static DEFINE_PER_CPU_READ_MOSTLY(u8[N_SMCA_BANK_TYPES], smca_bank_counts);
 
 static const char * const smca_names[] = {
-	[SMCA_LS ... SMCA_LS_V2]	= "load_store",
-	[SMCA_IF]			= "insn_fetch",
-	[SMCA_L2_CACHE]			= "l2_cache",
+	[SMCA_CS ... SMCA_CS_V2]	= "coherent_station",
+	[SMCA_DACC_BE]			= "dacc_be",
+	[SMCA_DACC_FE]			= "dacc_fe",
 	[SMCA_DE]			= "decode_unit",
-	[SMCA_RESERVED]			= "reserved",
+	[SMCA_EDDR5CMN]			= "eddr5_cmn",
 	[SMCA_EX]			= "execution_unit",
 	[SMCA_FP]			= "floating_point",
+	[SMCA_GMI_PCS]			= "gmi_pcs",
+	[SMCA_GMI_PHY]			= "gmi_phy",
+	[SMCA_IF]			= "insn_fetch",
+	[SMCA_L2_CACHE]			= "l2_cache",
 	[SMCA_L3_CACHE]			= "l3_cache",
-	[SMCA_CS ... SMCA_CS_V2]	= "coherent_slave",
-	[SMCA_PIE]			= "pie",
-
-	/* UMC v2 is separate because both of them can exist in a single system. */
-	[SMCA_UMC]			= "umc",
-	[SMCA_UMC_V2]			= "umc_v2",
+	[SMCA_LS ... SMCA_LS_V2]	= "load_store",
 	[SMCA_MA_LLC]			= "ma_llc",
-	[SMCA_PB]			= "param_block",
-	[SMCA_PSP ... SMCA_PSP_V2]	= "psp",
-	[SMCA_SMU ... SMCA_SMU_V2]	= "smu",
 	[SMCA_MP5]			= "mp5",
+	[SMCA_MPART]			= "mpart",
+	[SMCA_MPASP ... SMCA_MPASP_V2]	= "mpasp",
+	[SMCA_MPDACC]			= "mpdacc",
 	[SMCA_MPDMA]			= "mpdma",
+	[SMCA_MPM]			= "mpm",
+	[SMCA_MPRAS]			= "mpras",
+	[SMCA_NBIF]			= "nbif",
 	[SMCA_NBIO]			= "nbio",
+	[SMCA_PB]			= "param_block",
 	[SMCA_PCIE ... SMCA_PCIE_V2]	= "pcie",
-	[SMCA_XGMI_PCS]			= "xgmi_pcs",
-	[SMCA_NBIF]			= "nbif",
-	[SMCA_SHUB]			= "shub",
+	[SMCA_PCIE_PL]			= "pcie_pl",
+	[SMCA_PIE]			= "pie",
+	[SMCA_PSP ... SMCA_PSP_V2]	= "psp",
+	[SMCA_RESERVED]			= "reserved",
 	[SMCA_SATA]			= "sata",
+	[SMCA_SHUB]			= "shub",
+	[SMCA_SMU ... SMCA_SMU_V2]	= "smu",
+	[SMCA_SSBDCI]			= "ssbdci",
+
+	/* UMC v2 is separate because both of them can exist in a single system. */
+	[SMCA_UMC]			= "umc",
+	[SMCA_UMC_V2]			= "umc_v2",
 	[SMCA_USB]			= "usb",
-	[SMCA_USR_DP]			= "usr_dp",
 	[SMCA_USR_CP]			= "usr_cp",
-	[SMCA_GMI_PCS]			= "gmi_pcs",
-	[SMCA_XGMI_PHY]			= "xgmi_phy",
+	[SMCA_USR_DP]			= "usr_dp",
 	[SMCA_WAFL_PHY]			= "wafl_phy",
-	[SMCA_GMI_PHY]			= "gmi_phy",
+	[SMCA_XGMI_PCS]			= "xgmi_pcs",
+	[SMCA_XGMI_PHY]			= "xgmi_phy",
 };
 
 static const char *smca_get_name(enum smca_bank_types t)
@@ -153,68 +163,60 @@ enum smca_bank_types smca_get_bank_type(unsigned int cpu, unsigned int bank)
 }
 EXPORT_SYMBOL_GPL(smca_get_bank_type);
 
+/*
+ * Format:
+ * { bank_type, hwid_mcatype }
+ *
+ * alphanumerically sorted by bank type.
+ */
 static const struct smca_hwid smca_hwid_mcatypes[] = {
-	/* { bank_type, hwid_mcatype } */
-
-	/* Reserved type */
-	{ SMCA_RESERVED, HWID_MCATYPE(0x00, 0x0)	},
-
-	/* ZN Core (HWID=0xB0) MCA types */
-	{ SMCA_LS,	 HWID_MCATYPE(0xB0, 0x0)	},
-	{ SMCA_LS_V2,	 HWID_MCATYPE(0xB0, 0x10)	},
-	{ SMCA_IF,	 HWID_MCATYPE(0xB0, 0x1)	},
-	{ SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2)	},
+	{ SMCA_CS,	 HWID_MCATYPE(0x2E, 0x0)	},
+	{ SMCA_CS_V2,	 HWID_MCATYPE(0x2E, 0x2)	},
+	{ SMCA_DACC_BE,	 HWID_MCATYPE(0x164, 0x0)	},
+	{ SMCA_DACC_FE,	 HWID_MCATYPE(0x157, 0x0)	},
 	{ SMCA_DE,	 HWID_MCATYPE(0xB0, 0x3)	},
-	/* HWID 0xB0 MCATYPE 0x4 is Reserved */
+	{ SMCA_EDDR5CMN, HWID_MCATYPE(0x1E0, 0x0)	},
 	{ SMCA_EX,	 HWID_MCATYPE(0xB0, 0x5)	},
 	{ SMCA_FP,	 HWID_MCATYPE(0xB0, 0x6)	},
+	{ SMCA_GMI_PCS,  HWID_MCATYPE(0x241, 0x0)	},
+	{ SMCA_GMI_PHY,	 HWID_MCATYPE(0x269, 0x0)	},
+	{ SMCA_IF,	 HWID_MCATYPE(0xB0, 0x1)	},
+	{ SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2)	},
 	{ SMCA_L3_CACHE, HWID_MCATYPE(0xB0, 0x7)	},
-
-	/* Data Fabric MCA types */
-	{ SMCA_CS,	 HWID_MCATYPE(0x2E, 0x0)	},
-	{ SMCA_PIE,	 HWID_MCATYPE(0x2E, 0x1)	},
-	{ SMCA_CS_V2,	 HWID_MCATYPE(0x2E, 0x2)	},
+	{ SMCA_LS,	 HWID_MCATYPE(0xB0, 0x0)	},
+	{ SMCA_LS_V2,	 HWID_MCATYPE(0xB0, 0x10)	},
 	{ SMCA_MA_LLC,	 HWID_MCATYPE(0x2E, 0x4)	},
-
-	/* Unified Memory Controller MCA type */
-	{ SMCA_UMC,	 HWID_MCATYPE(0x96, 0x0)	},
-	{ SMCA_UMC_V2,	 HWID_MCATYPE(0x96, 0x1)	},
-
-	/* Parameter Block MCA type */
-	{ SMCA_PB,	 HWID_MCATYPE(0x05, 0x0)	},
-
-	/* Platform Security Processor MCA type */
-	{ SMCA_PSP,	 HWID_MCATYPE(0xFF, 0x0)	},
-	{ SMCA_PSP_V2,	 HWID_MCATYPE(0xFF, 0x1)	},
-
-	/* System Management Unit MCA type */
-	{ SMCA_SMU,	 HWID_MCATYPE(0x01, 0x0)	},
-	{ SMCA_SMU_V2,	 HWID_MCATYPE(0x01, 0x1)	},
-
-	/* Microprocessor 5 Unit MCA type */
 	{ SMCA_MP5,	 HWID_MCATYPE(0x01, 0x2)	},
-
-	/* MPDMA MCA type */
+	{ SMCA_MPART,	 HWID_MCATYPE(0xFF, 0x2)	},
+	{ SMCA_MPASP,	 HWID_MCATYPE(0xFD, 0x0)	},
+	{ SMCA_MPASP_V2, HWID_MCATYPE(0xFD, 0x1)	},
+	{ SMCA_MPDACC,	 HWID_MCATYPE(0xBE, 0x0)	},
 	{ SMCA_MPDMA,	 HWID_MCATYPE(0x01, 0x3)	},
-
-	/* Northbridge IO Unit MCA type */
+	{ SMCA_MPM,	 HWID_MCATYPE(0xF9, 0x0)	},
+	{ SMCA_MPRAS,	 HWID_MCATYPE(0x12, 0x0)	},
+	{ SMCA_NBIF,	 HWID_MCATYPE(0x6C, 0x0)	},
 	{ SMCA_NBIO,	 HWID_MCATYPE(0x18, 0x0)	},
-
-	/* PCI Express Unit MCA type */
+	{ SMCA_PB,	 HWID_MCATYPE(0x05, 0x0)	},
 	{ SMCA_PCIE,	 HWID_MCATYPE(0x46, 0x0)	},
 	{ SMCA_PCIE_V2,	 HWID_MCATYPE(0x46, 0x1)	},
-
-	{ SMCA_XGMI_PCS, HWID_MCATYPE(0x50, 0x0)	},
-	{ SMCA_NBIF,	 HWID_MCATYPE(0x6C, 0x0)	},
-	{ SMCA_SHUB,	 HWID_MCATYPE(0x80, 0x0)	},
+	{ SMCA_PCIE_PL,	 HWID_MCATYPE(0x1E1, 0x0)	},
+	{ SMCA_PIE,	 HWID_MCATYPE(0x2E, 0x1)	},
+	{ SMCA_PSP,	 HWID_MCATYPE(0xFF, 0x0)	},
+	{ SMCA_PSP_V2,	 HWID_MCATYPE(0xFF, 0x1)	},
+	{ SMCA_RESERVED, HWID_MCATYPE(0x00, 0x0)	},
 	{ SMCA_SATA,	 HWID_MCATYPE(0xA8, 0x0)	},
+	{ SMCA_SHUB,	 HWID_MCATYPE(0x80, 0x0)	},
+	{ SMCA_SMU,	 HWID_MCATYPE(0x01, 0x0)	},
+	{ SMCA_SMU_V2,	 HWID_MCATYPE(0x01, 0x1)	},
+	{ SMCA_SSBDCI,	 HWID_MCATYPE(0x5C, 0x0)	},
+	{ SMCA_UMC,	 HWID_MCATYPE(0x96, 0x0)	},
+	{ SMCA_UMC_V2,	 HWID_MCATYPE(0x96, 0x1)	},
 	{ SMCA_USB,	 HWID_MCATYPE(0xAA, 0x0)	},
-	{ SMCA_USR_DP,	 HWID_MCATYPE(0x170, 0x0)	},
 	{ SMCA_USR_CP,	 HWID_MCATYPE(0x180, 0x0)	},
-	{ SMCA_GMI_PCS,  HWID_MCATYPE(0x241, 0x0)	},
-	{ SMCA_XGMI_PHY, HWID_MCATYPE(0x259, 0x0)	},
+	{ SMCA_USR_DP,	 HWID_MCATYPE(0x170, 0x0)	},
 	{ SMCA_WAFL_PHY, HWID_MCATYPE(0x267, 0x0)	},
-	{ SMCA_GMI_PHY,	 HWID_MCATYPE(0x269, 0x0)	},
+	{ SMCA_XGMI_PCS, HWID_MCATYPE(0x50, 0x0)	},
+	{ SMCA_XGMI_PHY, HWID_MCATYPE(0x259, 0x0)	},
 };
 
 /*
@@ -875,13 +877,18 @@ void amd_clear_bank(struct mce *m)
 {
 	amd_reset_thr_limit(m->bank);
 
-	/* Clear MCA_DESTAT for all deferred errors even those logged in MCA_STATUS. */
-	if (m->status & MCI_STATUS_DEFERRED)
-		mce_wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank), 0);
+	if (mce_flags.smca) {
+		/*
+		 * Clear MCA_DESTAT for all deferred errors even those
+		 * logged in MCA_STATUS.
+		 */
+		if (m->status & MCI_STATUS_DEFERRED)
+			mce_wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank), 0);
 
-	/* Don't clear MCA_STATUS if MCA_DESTAT was used exclusively. */
-	if (m->kflags & MCE_CHECK_DFR_REGS)
-		return;
+		/* Don't clear MCA_STATUS if MCA_DESTAT was used exclusively. */
+		if (m->kflags & MCE_CHECK_DFR_REGS)
+			return;
+	}
 
 	mce_wrmsrq(mca_msr_reg(m->bank, MCA_STATUS), 0);
 }
diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
index 2d48e6593540f..72c8809f88e52 100644
--- a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
+++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
@@ -1,160 +1,238 @@
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x03, .steppings = 0x0004, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .driver_data = 0x45 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0002, .driver_data = 0x40 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0001, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0020, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0002, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0004, .driver_data = 0x38 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0008, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0008, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .driver_data = 0x47 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0001, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0002, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0002, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0010, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0d, .steppings = 0x0040, .driver_data = 0x18 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x0100, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x1000, .driver_data = 0x59 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0004, .driver_data = 0x5d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .driver_data = 0xd2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0080, .driver_data = 0x6b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0400, .driver_data = 0x95 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .driver_data = 0x44 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0080, .driver_data = 0x70a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0010, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0020, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .driver_data = 0x219 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1d, .steppings = 0x0002, .driver_data = 0x29 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1e, .steppings = 0x0020, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0004, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0020, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x26, .steppings = 0x0002, .driver_data = 0x105 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2a, .steppings = 0x0080, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2c, .steppings = 0x0004, .driver_data = 0x1f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0040, .driver_data = 0x621 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0080, .driver_data = 0x71a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2e, .steppings = 0x0040, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2f, .steppings = 0x0004, .driver_data = 0x3b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0100, .driver_data = 0x838 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0200, .driver_data = 0x90d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3a, .steppings = 0x0200, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3c, .steppings = 0x0008, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3d, .steppings = 0x0010, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0010, .driver_data = 0x42e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0040, .driver_data = 0x600 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0080, .driver_data = 0x715 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0004, .driver_data = 0x49 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0010, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x45, .steppings = 0x0002, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x46, .steppings = 0x0002, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x47, .steppings = 0x0002, .driver_data = 0x22 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0008, .driver_data = 0x368 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0010, .driver_data = 0x411 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4d, .steppings = 0x0100, .driver_data = 0x12d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4e, .steppings = 0x0008, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0008, .driver_data = 0x1000191 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0010, .driver_data = 0x2007006 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0020, .driver_data = 0x3000010 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0080, .driver_data = 0x5003901 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0800, .driver_data = 0x7002b01 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0004, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0008, .driver_data = 0x700001c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0010, .driver_data = 0xf00001a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0020, .driver_data = 0xe000015 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0004, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0200, .driver_data = 0x48 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0400, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5e, .steppings = 0x0008, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5f, .steppings = 0x0002, .driver_data = 0x3e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x66, .steppings = 0x0008, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0020, .driver_data = 0xc0002f0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0040, .driver_data = 0xd000404 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6c, .steppings = 0x0002, .driver_data = 0x10002d0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0002, .driver_data = 0x42 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0100, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7e, .steppings = 0x0020, .driver_data = 0xca },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8a, .steppings = 0x0002, .driver_data = 0x33 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0002, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0004, .driver_data = 0x3c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8d, .steppings = 0x0002, .driver_data = 0x56 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0200, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0400, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0800, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x1000, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0010, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0020, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0040, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0080, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0100, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x96, .steppings = 0x0002, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0004, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0020, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0008, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0010, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9c, .steppings = 0x0001, .driver_data = 0x24000026 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0200, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0400, .driver_data = 0xfa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0800, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x1000, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x2000, .driver_data = 0x104 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0004, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0008, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0020, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0001, .driver_data = 0x102 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0002, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa7, .steppings = 0x0002, .driver_data = 0x64 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaa, .steppings = 0x0010, .driver_data = 0x24 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xad, .steppings = 0x0002, .driver_data = 0xa0000d1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaf, .steppings = 0x0008, .driver_data = 0x3000341 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb5, .steppings = 0x0001, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0002, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0010, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0004, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0008, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0100, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbd, .steppings = 0x0002, .driver_data = 0x11f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbe, .steppings = 0x0001, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0004, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0020, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0040, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0080, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc5, .steppings = 0x0004, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0004, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0010, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xca, .steppings = 0x0004, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0002, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0004, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0080, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .driver_data = 0x15 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x01, .steppings = 0x0004, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0040, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0004, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0008, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0010, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0002, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0008, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0010, .driver_data = 0x6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0080, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .driver_data = 0xe },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0200, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0400, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0004, .driver_data = 0xf },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0010, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0020, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0100, .driver_data = 0x9 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000404 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002d0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000639 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000639 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000639 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000639 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000639 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x437 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x437 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x24 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa0000d1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003a2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000341 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x12f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x12f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x4128 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x4128 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x4128 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x11f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x118 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002a9 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002a9 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 },
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 8744f3adc2a0e..37ac4afe09724 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -120,19 +120,44 @@ static inline unsigned int exttable_size(struct extended_sigtable *et)
 	return et->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE;
 }
 
+
+/*
+ * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86
+ * structures are populated.
+ */
+static u32 intel_cpuid_vfm(void)
+{
+	u32 eax   = cpuid_eax(1);
+	u32 fam   = x86_family(eax);
+	u32 model = x86_model(eax);
+
+	return IFM(fam, model);
+}
+
+u32 intel_get_platform_id(void)
+{
+	unsigned int val[2];
+
+	/*
+	 * This can be called early. Use CPUID directly instead of
+	 * relying on cpuinfo_x86 which may not be fully initialized.
+	 * The PII does not have MSR_IA32_PLATFORM_ID. Everything
+	 * before _it_ has no microcode (for Linux at least).
+	 */
+	if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH)
+		return 0;
+
+	/* get processor flags from MSR 0x17 */
+	native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]);
+
+	return (val[1] >> 18) & 7;
+}
+
 void intel_collect_cpu_info(struct cpu_signature *sig)
 {
 	sig->sig = cpuid_eax(1);
-	sig->pf = 0;
 	sig->rev = intel_get_microcode_revision();
-
-	if (IFM(x86_family(sig->sig), x86_model(sig->sig)) >= INTEL_PENTIUM_III_DESCHUTES) {
-		unsigned int val[2];
-
-		/* get processor flags from MSR 0x17 */
-		native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]);
-		sig->pf = 1 << ((val[1] >> 18) & 7);
-	}
+	sig->pf  = 1 << intel_get_platform_id();
 }
 EXPORT_SYMBOL_GPL(intel_collect_cpu_info);
 
@@ -142,8 +167,15 @@ static inline bool cpu_signatures_match(struct cpu_signature *s1, unsigned int s
 	if (s1->sig != sig2)
 		return false;
 
-	/* Processor flags are either both 0 or they intersect. */
-	return ((!s1->pf && !pf2) || (s1->pf & pf2));
+	/*
+	 * Consider an empty mask to match everything. This
+	 * should only occur for one CPU model, the PII.
+	 */
+	if (!pf2)
+		return true;
+
+	/* Is the CPU's platform ID in the signature mask? */
+	return s1->pf & pf2;
 }
 
 bool intel_find_matching_signature(void *mc, struct cpu_signature *sig)
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 89a2eb8a07221..9befdc557d9e5 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -496,8 +496,9 @@ static void hv_reserve_irq_vectors(void)
 	    test_and_set_bit(HYPERV_DBG_FASTFAIL_VECTOR, system_vectors))
 		BUG();
 
-	pr_info("Hyper-V: reserve vectors: %d %d %d\n", HYPERV_DBG_ASSERT_VECTOR,
-		HYPERV_DBG_SERVICE_VECTOR, HYPERV_DBG_FASTFAIL_VECTOR);
+	pr_info("Hyper-V: reserve vectors: 0x%x 0x%x 0x%x\n",
+		HYPERV_DBG_ASSERT_VECTOR, HYPERV_DBG_SERVICE_VECTOR,
+		HYPERV_DBG_FASTFAIL_VECTOR);
 }
 
 static void __init ms_hyperv_init_platform(void)
diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c
index 763534d77f593..e3eee9ae41414 100644
--- a/arch/x86/kernel/cpu/mtrr/cleanup.c
+++ b/arch/x86/kernel/cpu/mtrr/cleanup.c
@@ -437,9 +437,10 @@ static unsigned long nr_mtrr_spare_reg __initdata =
 
 static int __init parse_mtrr_spare_reg(char *arg)
 {
-	if (arg)
-		nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
-	return 0;
+	if (!arg)
+		return -EINVAL;
+
+	return kstrtoul(arg, 0, &nr_mtrr_spare_reg);
 }
 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
 
diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c
index ac60ebde5d9bc..3f0222d10f6e6 100644
--- a/arch/x86/kernel/cpu/sgx/encl.c
+++ b/arch/x86/kernel/cpu/sgx/encl.c
@@ -1220,7 +1220,7 @@ void sgx_zap_enclave_ptes(struct sgx_encl *encl, unsigned long addr)
 
 			ret = sgx_encl_find(encl_mm->mm, addr, &vma);
 			if (!ret && encl == vma->vm_private_data)
-				zap_vma_ptes(vma, addr, PAGE_SIZE);
+				zap_special_vma_range(vma, addr, PAGE_SIZE);
 
 			mmap_read_unlock(encl_mm->mm);
 
diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c
index eafcb1fc185ad..4913b64ec592d 100644
--- a/arch/x86/kernel/cpu/topology.c
+++ b/arch/x86/kernel/cpu/topology.c
@@ -204,15 +204,11 @@ static __init bool check_for_real_bsp(u32 apic_id)
 static unsigned int topo_unit_count(u32 lvlid, enum x86_topology_domains at_level,
 				    unsigned long *map)
 {
-	unsigned int id, end, cnt = 0;
+	unsigned int end;
 
 	/* Calculate the exclusive end */
 	end = lvlid + (1U << x86_topo_system.dom_shifts[at_level]);
-
-	/* Unfortunately there is no bitmap_weight_range() */
-	for (id = find_next_bit(map, end, lvlid); id < end; id = find_next_bit(map, end, ++id))
-		cnt++;
-	return cnt;
+	return bitmap_weight_from(map, lvlid, end);
 }
 
 static __init void topo_register_apic(u32 apic_id, u32 acpi_id, bool present)
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 335fd2ee9766d..cd796818d94d9 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -42,6 +42,7 @@
 #include <asm/crash.h>
 #include <asm/cmdline.h>
 #include <asm/sev.h>
+#include <asm/virt.h>
 
 /* Used while preparing memory map entries for second kernel */
 struct crash_memmap_data {
@@ -111,7 +112,7 @@ void native_machine_crash_shutdown(struct pt_regs *regs)
 
 	crash_smp_send_stop();
 
-	cpu_emergency_disable_virtualization();
+	x86_virt_emergency_disable_virtualization_cpu();
 
 	/*
 	 * Disable Intel PT to stop its logging
diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index 76153dfb58c9d..a7b6524a9dea2 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -1080,8 +1080,7 @@ void __user *get_xsave_addr_user(struct xregs_state __user *xsave, int xfeature_
  * This will go out and modify PKRU register to set the access
  * rights for @pkey to @init_val.
  */
-int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
-			      unsigned long init_val)
+int arch_set_user_pkey_access(int pkey, unsigned long init_val)
 {
 	u32 old_pkru, new_pkru_bits = 0;
 	int pkey_shift;
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 80ef5d386b03d..5171cb746444e 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -441,10 +441,6 @@ initial_pg_fixmap:
 swapper_pg_dir:
 	.fill 1024,4,0
 	.fill PTI_USER_PGD_FILL,4,0
-.globl empty_zero_page
-empty_zero_page:
-	.fill 4096,1,0
-EXPORT_SYMBOL(empty_zero_page)
 
 /*
  * This starts the data section.
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 85d4a5094f6bc..7ed5520dd52eb 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -684,10 +684,3 @@ SYM_PIC_ALIAS(phys_base);
 EXPORT_SYMBOL(phys_base)
 
 #include "../xen/xen-head.S"
-
-	__PAGE_ALIGNED_BSS
-SYM_DATA_START_PAGE_ALIGNED(empty_zero_page)
-	.skip PAGE_SIZE
-SYM_DATA_END(empty_zero_page)
-EXPORT_SYMBOL(empty_zero_page)
-
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index 5630c7dca1f3d..7e980ea49d8d6 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -525,12 +525,8 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 		if (ret)
 			return ERR_PTR(ret);
 		ret = crash_load_dm_crypt_keys(image);
-		if (ret == -ENOENT) {
-			kexec_dprintk("No dm crypt key to load\n");
-		} else if (ret) {
-			pr_err("Failed to load dm crypt keys\n");
+		if (ret)
 			return ERR_PTR(ret);
-		}
 		if (image->dm_crypt_keys_addr &&
 		    cmdline_len + MAX_ELFCOREHDR_STR_LEN + MAX_DMCRYPTKEYS_STR_LEN >
 			    header->cmdline_size) {
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index 6032fa9ec753c..0fed6d0d7e322 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -27,6 +27,7 @@
 #include <asm/cpu.h>
 #include <asm/nmi.h>
 #include <asm/smp.h>
+#include <asm/virt.h>
 
 #include <linux/ctype.h>
 #include <linux/mc146818rtc.h>
@@ -532,51 +533,6 @@ static inline void kb_wait(void)
 static inline void nmi_shootdown_cpus_on_restart(void);
 
 #if IS_ENABLED(CONFIG_KVM_X86)
-/* RCU-protected callback to disable virtualization prior to reboot. */
-static cpu_emergency_virt_cb __rcu *cpu_emergency_virt_callback;
-
-void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback)
-{
-	if (WARN_ON_ONCE(rcu_access_pointer(cpu_emergency_virt_callback)))
-		return;
-
-	rcu_assign_pointer(cpu_emergency_virt_callback, callback);
-}
-EXPORT_SYMBOL_FOR_KVM(cpu_emergency_register_virt_callback);
-
-void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback)
-{
-	if (WARN_ON_ONCE(rcu_access_pointer(cpu_emergency_virt_callback) != callback))
-		return;
-
-	rcu_assign_pointer(cpu_emergency_virt_callback, NULL);
-	synchronize_rcu();
-}
-EXPORT_SYMBOL_FOR_KVM(cpu_emergency_unregister_virt_callback);
-
-/*
- * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during
- * reboot.  VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if
- * GIF=0, i.e. if the crash occurred between CLGI and STGI.
- */
-void cpu_emergency_disable_virtualization(void)
-{
-	cpu_emergency_virt_cb *callback;
-
-	/*
-	 * IRQs must be disabled as KVM enables virtualization in hardware via
-	 * function call IPIs, i.e. IRQs need to be disabled to guarantee
-	 * virtualization stays disabled.
-	 */
-	lockdep_assert_irqs_disabled();
-
-	rcu_read_lock();
-	callback = rcu_dereference(cpu_emergency_virt_callback);
-	if (callback)
-		callback();
-	rcu_read_unlock();
-}
-
 static void emergency_reboot_disable_virtualization(void)
 {
 	local_irq_disable();
@@ -588,16 +544,11 @@ static void emergency_reboot_disable_virtualization(void)
 	 * We can't take any locks and we may be on an inconsistent state, so
 	 * use NMIs as IPIs to tell the other CPUs to disable VMX/SVM and halt.
 	 *
-	 * Do the NMI shootdown even if virtualization is off on _this_ CPU, as
-	 * other CPUs may have virtualization enabled.
+	 * Safely force _this_ CPU out of VMX/SVM operation, and if necessary,
+	 * blast NMIs to force other CPUs out of VMX/SVM as well.k
 	 */
-	if (rcu_access_pointer(cpu_emergency_virt_callback)) {
-		/* Safely force _this_ CPU out of VMX/SVM operation. */
-		cpu_emergency_disable_virtualization();
-
-		/* Disable VMX/SVM and halt on other CPUs. */
+	if (!x86_virt_emergency_disable_virtualization_cpu())
 		nmi_shootdown_cpus_on_restart();
-	}
 }
 #else
 static void emergency_reboot_disable_virtualization(void) { }
@@ -773,12 +724,15 @@ static void __machine_emergency_restart(int emergency)
 	machine_ops.emergency_restart();
 }
 
-static void native_machine_restart(char *__unused)
+static void native_machine_restart(char *command)
 {
 	pr_notice("machine restart\n");
 
 	if (!reboot_force)
 		machine_shutdown();
+
+	do_kernel_restart(command);
+
 	__machine_emergency_restart(0);
 }
 
@@ -875,10 +829,10 @@ static int crash_nmi_callback(unsigned int val, struct pt_regs *regs)
 		shootdown_callback(cpu, regs);
 
 	/*
-	 * Prepare the CPU for reboot _after_ invoking the callback so that the
-	 * callback can safely use virtualization instructions, e.g. VMCLEAR.
+	 * Disable virtualization, as both VMX and SVM can block INIT and thus
+	 * prevent AP bringup, e.g. in a kdump kernel or in firmware.
 	 */
-	cpu_emergency_disable_virtualization();
+	x86_virt_emergency_disable_virtualization_cpu();
 
 	atomic_dec(&waiting_for_crash_ipi);
 
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 51a849a79c985..314b062a15de9 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -2,10 +2,10 @@
 /*
  * RTC related functions
  */
+#include <linux/acpi.h>
 #include <linux/platform_device.h>
 #include <linux/mc146818rtc.h>
 #include <linux/export.h>
-#include <linux/pnp.h>
 
 #include <asm/vsyscall.h>
 #include <asm/x86_init.h>
@@ -133,25 +133,14 @@ static struct platform_device rtc_device = {
 
 static __init int add_rtc_cmos(void)
 {
-#ifdef CONFIG_PNP
-	static const char * const ids[] __initconst =
-	    { "PNP0b00", "PNP0b01", "PNP0b02", };
-	struct pnp_dev *dev;
-	int i;
-
-	pnp_for_each_dev(dev) {
-		for (i = 0; i < ARRAY_SIZE(ids); i++) {
-			if (compare_pnp_id(dev->id, ids[i]) != 0)
-				return 0;
-		}
-	}
-#endif
+	if (cmos_rtc_platform_device_present)
+		return 0;
+
 	if (!x86_platform.legacy.rtc)
 		return -ENODEV;
 
 	platform_device_register(&rtc_device);
-	dev_info(&rtc_device.dev,
-		 "registered platform RTC device (no PNP device found)\n");
+	dev_info(&rtc_device.dev, "registered fallback platform RTC device\n");
 
 	return 0;
 }
diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c
index 978232b6d48d7..9725e7d89b1ef 100644
--- a/arch/x86/kernel/shstk.c
+++ b/arch/x86/kernel/shstk.c
@@ -100,17 +100,9 @@ static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)
 static unsigned long alloc_shstk(unsigned long addr, unsigned long size,
 				 unsigned long token_offset, bool set_res_tok)
 {
-	int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_ABOVE4G;
-	struct mm_struct *mm = current->mm;
-	unsigned long mapped_addr, unused;
+	unsigned long mapped_addr;
 
-	if (addr)
-		flags |= MAP_FIXED_NOREPLACE;
-
-	mmap_write_lock(mm);
-	mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags,
-			      VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL);
-	mmap_write_unlock(mm);
+	mapped_addr = vm_mmap_shadow_stack(addr, size, MAP_ABOVE4G);
 
 	if (!set_res_tok || IS_ERR_VALUE(mapped_addr))
 		goto out;
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index b014e6d229f95..cbf95fe2b207a 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -35,6 +35,7 @@
 #include <asm/trace/irq_vectors.h>
 #include <asm/kexec.h>
 #include <asm/reboot.h>
+#include <asm/virt.h>
 
 /*
  *	Some notes on x86 processor bugs affecting SMP operation:
@@ -124,7 +125,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
 	if (raw_smp_processor_id() == atomic_read(&stopping_cpu))
 		return NMI_HANDLED;
 
-	cpu_emergency_disable_virtualization();
+	x86_virt_emergency_disable_virtualization_cpu();
 	stop_this_cpu(NULL);
 
 	return NMI_HANDLED;
@@ -136,7 +137,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
 DEFINE_IDTENTRY_SYSVEC(sysvec_reboot)
 {
 	apic_eoi();
-	cpu_emergency_disable_virtualization();
+	x86_virt_emergency_disable_virtualization_cpu();
 	stop_this_cpu(NULL);
 }
 
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 4dbff8ef9b1cd..0ca3912ecb7f4 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -70,6 +70,7 @@
 #include <asm/tdx.h>
 #include <asm/cfi.h>
 #include <asm/msr.h>
+#include <asm/vsyscall.h>
 
 #ifdef CONFIG_X86_64
 #include <asm/x86_init.h>
@@ -921,11 +922,6 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 
 	cond_local_irq_enable(regs);
 
-	if (static_cpu_has(X86_FEATURE_UMIP)) {
-		if (user_mode(regs) && fixup_umip_exception(regs))
-			goto exit;
-	}
-
 	if (v8086_mode(regs)) {
 		local_irq_enable();
 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
@@ -940,6 +936,12 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 		if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))
 			goto exit;
 
+		if (fixup_umip_exception(regs))
+			goto exit;
+
+		if (emulate_vsyscall_gp(regs))
+			goto exit;
+
 		gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);
 		goto exit;
 	}
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index d9aa694e43f3c..9ccd58c29409b 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -1200,9 +1200,10 @@ static struct clocksource clocksource_tsc = {
 	.read			= read_tsc,
 	.mask			= CLOCKSOURCE_MASK(64),
 	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
-				  CLOCK_SOURCE_VALID_FOR_HRES |
+				  CLOCK_SOURCE_CAN_INLINE_READ |
 				  CLOCK_SOURCE_MUST_VERIFY |
-				  CLOCK_SOURCE_VERIFY_PERCPU,
+				  CLOCK_SOURCE_VERIFY_PERCPU |
+				  CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT,
 	.id			= CSID_X86_TSC,
 	.vdso_clock_mode	= VDSO_CLOCKMODE_TSC,
 	.enable			= tsc_cs_enable,
@@ -1409,6 +1410,15 @@ static void tsc_refine_calibration_work(struct work_struct *work)
 		have_art = true;
 		clocksource_tsc.base = &art_base_clk;
 	}
+
+	/*
+	 * Transfer the valid for high resolution flag if it was set on the
+	 * early TSC already. That guarantees that there is no intermediate
+	 * clocksource selected once the early TSC is unregistered.
+	 */
+	if (clocksource_tsc_early.flags & CLOCK_SOURCE_VALID_FOR_HRES)
+		clocksource_tsc.flags |= CLOCK_SOURCE_VALID_FOR_HRES;
+
 	clocksource_register_khz(&clocksource_tsc, tsc_khz);
 unreg:
 	clocksource_unregister(&clocksource_tsc_early);
diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c
index d432f3824f0c2..3ce99cbcf1874 100644
--- a/arch/x86/kernel/umip.c
+++ b/arch/x86/kernel/umip.c
@@ -354,6 +354,9 @@ bool fixup_umip_exception(struct pt_regs *regs)
 	void __user *uaddr;
 	struct insn insn;
 
+	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
+		return false;
+
 	if (!regs)
 		return false;
 
diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c
index f610fde2d5c4b..6407bc9256bfb 100644
--- a/arch/x86/kernel/unwind_orc.c
+++ b/arch/x86/kernel/unwind_orc.c
@@ -546,17 +546,23 @@ bool unwind_next_frame(struct unwind_state *state)
 		indirect = true;
 		break;
 
-	case ORC_REG_R10:
-		if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
-			orc_warn_current("missing R10 value at %pB\n",
+	/*
+	 * Any of the below registers may temporarily hold the stack pointer,
+	 * typically during a DRAP stack realignment sequence or some other
+	 * stack swizzle.
+	 */
+
+	case ORC_REG_AX:
+		if (!get_reg(state, offsetof(struct pt_regs, ax), &sp)) {
+			orc_warn_current("missing AX value at %pB\n",
 					 (void *)state->ip);
 			goto err;
 		}
 		break;
 
-	case ORC_REG_R13:
-		if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
-			orc_warn_current("missing R13 value at %pB\n",
+	case ORC_REG_DX:
+		if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
+			orc_warn_current("missing DX value at %pB\n",
 					 (void *)state->ip);
 			goto err;
 		}
@@ -570,9 +576,17 @@ bool unwind_next_frame(struct unwind_state *state)
 		}
 		break;
 
-	case ORC_REG_DX:
-		if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
-			orc_warn_current("missing DX value at %pB\n",
+	case ORC_REG_R10:
+		if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
+			orc_warn_current("missing R10 value at %pB\n",
+					 (void *)state->ip);
+			goto err;
+		}
+		break;
+
+	case ORC_REG_R13:
+		if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
+			orc_warn_current("missing R13 value at %pB\n",
 					 (void *)state->ip);
 			goto err;
 		}
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 8137927e7387e..e69156b54cfff 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -1246,11 +1246,12 @@ void kvm_initialize_cpu_caps(void)
 		F(NULL_SEL_CLR_BASE),
 		/* UpperAddressIgnore */
 		F(AUTOIBRS),
-		F(PREFETCHI),
 		EMULATED_F(NO_SMM_CTL_MSR),
 		/* PrefetchCtlMsr */
 		/* GpOnUserCpuid */
 		/* EPSF */
+		F(PREFETCHI),
+		F(AVX512_BMM),
 		F(ERAPS),
 		SYNTHESIZED_F(SBPB),
 		SYNTHESIZED_F(IBPB_BRTYPE),
@@ -2160,7 +2161,8 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
 {
 	u32 eax, ebx, ecx, edx;
 
-	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
+	if (!is_smm(vcpu) && cpuid_fault_enabled(vcpu) &&
+	    !kvm_require_cpl(vcpu, 0))
 		return 1;
 
 	eax = kvm_rax_read(vcpu);
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index c8e292e9a24df..6145dac4a605a 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -1297,12 +1297,25 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
 	int rc;
 	struct read_cache *mc = &ctxt->mem_read;
 
+	/*
+	 * If the read gets a cache hit, simply copy the value from the cache.
+	 * A "hit" here means that there is unused data in the cache, i.e. when
+	 * re-emulating an instruction to complete a userspace exit, KVM relies
+	 * on "no decode" to ensure the instruction is re-emulated in the same
+	 * sequence, so that multiple reads are fulfilled in the correct order.
+	 */
 	if (mc->pos < mc->end)
 		goto read_cached;
 
 	if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))
 		return X86EMUL_UNHANDLEABLE;
 
+	/*
+	 * Route all reads to the cache.  This allows @dest to be an on-stack
+	 * variable without triggering use-after-free if KVM needs to exit to
+	 * userspace to handle an MMIO read (the MMIO fragment will point at
+	 * the current location in the cache).
+	 */
 	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
 				      &ctxt->exception);
 	if (rc != X86EMUL_CONTINUE)
@@ -3583,10 +3596,10 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt)
 	u64 msr = 0;
 
 	ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
-	if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
-	    ctxt->ops->cpl(ctxt)) {
+	if (!ctxt->ops->is_smm(ctxt) &&
+	    (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT) &&
+	    ctxt->ops->cpl(ctxt))
 		return emulate_gp(ctxt, 0);
-	}
 
 	eax = reg_read(ctxt, VCPU_REGS_RAX);
 	ecx = reg_read(ctxt, VCPU_REGS_RCX);
@@ -3708,7 +3721,7 @@ static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
  */
 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
 {
-	struct fxregs_state fx_state;
+	struct fxregs_state fx_state = {};
 	int rc;
 
 	rc = check_fxsr(ctxt);
@@ -3738,7 +3751,7 @@ static int em_fxsave(struct x86_emulate_ctxt *ctxt)
 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
 				 const size_t used_size)
 {
-	struct fxregs_state fx_tmp;
+	struct fxregs_state fx_tmp = {};
 	int rc;
 
 	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h
index 6ce160ffa6786..6301f79fcbae7 100644
--- a/arch/x86/kvm/hyperv.h
+++ b/arch/x86/kvm/hyperv.h
@@ -305,14 +305,6 @@ static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu)
 {
 	return false;
 }
-static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu)
-{
-	return false;
-}
-static inline bool guest_hv_cpuid_has_l2_tlb_flush(struct kvm_vcpu *vcpu)
-{
-	return false;
-}
 static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu)
 {
 	return 0;
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 9381c58d4c85c..e3ec4d8607c19 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -840,16 +840,16 @@ static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
 {
 	int i, count = 0;
 	struct kvm_vcpu *vcpu;
+	size_t map_index;
 
 	if (min > map->max_apic_id)
 		return 0;
 
-	min = array_index_nospec(min, map->max_apic_id + 1);
-
 	for_each_set_bit(i, ipi_bitmap,
-		min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
-		if (map->phys_map[min + i]) {
-			vcpu = map->phys_map[min + i]->vcpu;
+			 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
+		map_index = array_index_nospec(min + i, map->max_apic_id + 1);
+		if (map->phys_map[map_index]) {
+			vcpu = map->phys_map[map_index]->vcpu;
 			count += kvm_apic_set_irq(vcpu, irq, NULL);
 		}
 	}
@@ -2657,6 +2657,9 @@ void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
+	if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
+		return;
+
 	/*
 	 * ICR is a single 64-bit register when x2APIC is enabled, all others
 	 * registers hold 32-bit values.  For legacy xAPIC, ICR writes need to
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index b922a8b000577..3dce38ffee764 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -2940,6 +2940,15 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
 	if (kvm_gfn_is_write_tracked(kvm, slot, gfn))
 		return -EPERM;
 
+	/*
+	 * Only 4KiB mappings can become unsync, and KVM disallows hugepages
+	 * when accounting 4KiB shadow pages.  Upper-level gPTEs are always
+	 * write-protected (see above), thus if the gfn can be mapped with a
+	 * hugepage and isn't write-tracked, it can't have a shadow page.
+	 */
+	if (!lpage_info_slot(gfn, slot, PG_LEVEL_2M)->disallow_lpage)
+		return 0;
+
 	/*
 	 * The page is not write-tracked, mark existing shadow pages unsync
 	 * unless KVM is synchronizing an unsync SP.  In that case, KVM must
@@ -7487,9 +7496,14 @@ static void kvm_wake_nx_recovery_thread(struct kvm *kvm)
 
 static int get_nx_huge_pages(char *buffer, const struct kernel_param *kp)
 {
+	int val = *(int *)kp->arg;
+
 	if (nx_hugepage_mitigation_hard_disabled)
 		return sysfs_emit(buffer, "never\n");
 
+	if (val == -1)
+		return sysfs_emit(buffer, "auto\n");
+
 	return param_get_bool(buffer, kp);
 }
 
diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
index 91ce29fd6f1b2..8c0ffa2cded69 100644
--- a/arch/x86/kvm/mmu/spte.h
+++ b/arch/x86/kvm/mmu/spte.h
@@ -248,7 +248,7 @@ extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
 
 static inline hpa_t kvm_mmu_get_dummy_root(void)
 {
-	return my_zero_pfn(0) << PAGE_SHIFT;
+	return zero_pfn(0) << PAGE_SHIFT;
 }
 
 static inline bool kvm_mmu_is_dummy_root(hpa_t shadow_page)
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 9c26038f6b77e..7b1102d26f9cd 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -1507,7 +1507,7 @@ static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(void)
 	if (!sp)
 		return NULL;
 
-	sp->spt = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+	sp->spt = (void *)__get_free_page(GFP_KERNEL_ACCOUNT);
 	if (!sp->spt) {
 		kmem_cache_free(mmu_page_header_cache, sp);
 		return NULL;
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index bd6b785cf2612..e218352e34231 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -1256,7 +1256,7 @@ int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
 
 	r = -EFAULT;
 	if (copy_from_user(filter->events, user_filter->events,
-			   sizeof(filter->events[0]) * filter->nevents))
+			   flex_array_size(filter, events, filter->nevents)))
 		goto cleanup;
 
 	r = prepare_filter_lists(filter);
diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c
index f7ec7914e3c47..b6072872b7858 100644
--- a/arch/x86/kvm/svm/avic.c
+++ b/arch/x86/kvm/svm/avic.c
@@ -19,6 +19,7 @@
 #include <linux/amd-iommu.h>
 #include <linux/kvm_host.h>
 #include <linux/kvm_irqfd.h>
+#include <linux/sysfs.h>
 
 #include <asm/irq_remapping.h>
 #include <asm/msr.h>
@@ -76,23 +77,33 @@ static int avic_param_set(const char *val, const struct kernel_param *kp)
 	return param_set_bint(val, kp);
 }
 
+static int avic_param_get(char *buffer, const struct kernel_param *kp)
+{
+	int val = *(int *)kp->arg;
+
+	if (val == AVIC_AUTO_MODE)
+		return sysfs_emit(buffer, "N\n");
+
+	return param_get_bool(buffer, kp);
+}
+
 static const struct kernel_param_ops avic_ops = {
 	.flags = KERNEL_PARAM_OPS_FL_NOARG,
 	.set = avic_param_set,
-	.get = param_get_bool,
+	.get = avic_param_get,
 };
 
 /*
  * Enable / disable AVIC.  In "auto" mode (default behavior), AVIC is enabled
  * for Zen4+ CPUs with x2AVIC (and all other criteria for enablement are met).
  */
-static int avic = AVIC_AUTO_MODE;
+static int __ro_after_init avic = AVIC_AUTO_MODE;
 module_param_cb(avic, &avic_ops, &avic, 0444);
 __MODULE_PARM_TYPE(avic, "bool");
 
 module_param(enable_ipiv, bool, 0444);
 
-static bool force_avic;
+static bool __ro_after_init force_avic;
 module_param_unsafe(force_avic, bool, 0444);
 
 /* Note:
diff --git a/arch/x86/kvm/svm/hyperv.h b/arch/x86/kvm/svm/hyperv.h
index d3f8bfc05832e..f70d076911a63 100644
--- a/arch/x86/kvm/svm/hyperv.h
+++ b/arch/x86/kvm/svm/hyperv.h
@@ -41,10 +41,17 @@ static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
 	return hv_vcpu->vp_assist_page.nested_control.features.directhypercall;
 }
 
+static inline bool nested_svm_is_l2_tlb_flush_hcall(struct kvm_vcpu *vcpu)
+{
+	return guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
+	       nested_svm_l2_tlb_flush_enabled(vcpu) &&
+	       kvm_hv_is_tlb_flush_hcall(vcpu);
+}
+
 void svm_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu);
 #else /* CONFIG_KVM_HYPERV */
 static inline void nested_svm_hv_update_vm_vp_ids(struct kvm_vcpu *vcpu) {}
-static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
+static inline bool nested_svm_is_l2_tlb_flush_hcall(struct kvm_vcpu *vcpu)
 {
 	return false;
 }
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index b36c33255bed6..5ff01d2ac85e4 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -116,31 +116,28 @@ static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
 	if (!nested_npt_enabled(svm))
 		return true;
 
-	if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK))
+	if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE))
 		return true;
 
 	return false;
 }
 
-void recalc_intercepts(struct vcpu_svm *svm)
+void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm)
 {
-	struct vmcb_control_area *c, *h;
-	struct vmcb_ctrl_area_cached *g;
+	struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl;
+	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
+	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 	unsigned int i;
 
-	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
-
-	if (!is_guest_mode(&svm->vcpu))
+	if (WARN_ON_ONCE(svm->vmcb != vmcb02))
 		return;
 
-	c = &svm->vmcb->control;
-	h = &svm->vmcb01.ptr->control;
-	g = &svm->nested.ctl;
+	vmcb_mark_dirty(vmcb02, VMCB_INTERCEPTS);
 
 	for (i = 0; i < MAX_INTERCEPT; i++)
-		c->intercepts[i] = h->intercepts[i];
+		vmcb02->control.intercepts[i] = vmcb01->control.intercepts[i];
 
-	if (g->int_ctl & V_INTR_MASKING_MASK) {
+	if (vmcb12_ctrl->int_ctl & V_INTR_MASKING_MASK) {
 		/*
 		 * If L2 is active and V_INTR_MASKING is enabled in vmcb12,
 		 * disable intercept of CR8 writes as L2's CR8 does not affect
@@ -151,24 +148,17 @@ void recalc_intercepts(struct vcpu_svm *svm)
 		 * the effective RFLAGS.IF for L1 interrupts will never be set
 		 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs).
 		 */
-		vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE);
-		if (!(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF))
-			vmcb_clr_intercept(c, INTERCEPT_VINTR);
+		vmcb_clr_intercept(&vmcb02->control, INTERCEPT_CR8_WRITE);
+		if (!(vmcb01->save.rflags & X86_EFLAGS_IF))
+			vmcb_clr_intercept(&vmcb02->control, INTERCEPT_VINTR);
 	}
 
-	/*
-	 * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB
-	 * flush feature is enabled.
-	 */
-	if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu))
-		vmcb_clr_intercept(c, INTERCEPT_VMMCALL);
-
 	for (i = 0; i < MAX_INTERCEPT; i++)
-		c->intercepts[i] |= g->intercepts[i];
+		vmcb02->control.intercepts[i] |= vmcb12_ctrl->intercepts[i];
 
 	/* If SMI is not intercepted, ignore guest SMI intercept as well  */
 	if (!intercept_smi)
-		vmcb_clr_intercept(c, INTERCEPT_SMI);
+		vmcb_clr_intercept(&vmcb02->control, INTERCEPT_SMI);
 
 	if (nested_vmcb_needs_vls_intercept(svm)) {
 		/*
@@ -176,10 +166,10 @@ void recalc_intercepts(struct vcpu_svm *svm)
 		 * we must intercept these instructions to correctly
 		 * emulate them in case L1 doesn't intercept them.
 		 */
-		vmcb_set_intercept(c, INTERCEPT_VMLOAD);
-		vmcb_set_intercept(c, INTERCEPT_VMSAVE);
+		vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMLOAD);
+		vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMSAVE);
 	} else {
-		WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK));
+		WARN_ON_ONCE(!(vmcb02->control.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE));
 	}
 }
 
@@ -339,8 +329,56 @@ static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
 	    kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
 }
 
-static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
-					 struct vmcb_ctrl_area_cached *control)
+static bool nested_svm_event_inj_valid_exept(struct kvm_vcpu *vcpu, u8 vector)
+{
+	/*
+	 * Vectors that do not correspond to a defined exception are invalid
+	 * (including #NMI and reserved vectors). In a best effort to define
+	 * valid exceptions based on the virtual CPU, make all exceptions always
+	 * valid except those obviously tied to a CPU feature.
+	 */
+	switch (vector) {
+	case DE_VECTOR: case DB_VECTOR: case BP_VECTOR: case OF_VECTOR:
+	case BR_VECTOR: case UD_VECTOR: case NM_VECTOR: case DF_VECTOR:
+	case TS_VECTOR: case NP_VECTOR: case SS_VECTOR: case GP_VECTOR:
+	case PF_VECTOR: case MF_VECTOR: case AC_VECTOR: case MC_VECTOR:
+	case XM_VECTOR: case HV_VECTOR: case SX_VECTOR:
+		return true;
+	case CP_VECTOR:
+		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);
+	case VC_VECTOR:
+		return guest_cpu_cap_has(vcpu, X86_FEATURE_SEV_ES);
+	}
+	return false;
+}
+
+/*
+ * According to the APM, VMRUN exits with SVM_EXIT_ERR if SVM_EVTINJ_VALID is
+ * set and:
+ * - The type of event_inj is not one of the defined values.
+ * - The type is SVM_EVTINJ_TYPE_EXEPT, but the vector is not a valid exception.
+ */
+static bool nested_svm_check_event_inj(struct kvm_vcpu *vcpu, u32 event_inj)
+{
+	u32 type = event_inj & SVM_EVTINJ_TYPE_MASK;
+	u8 vector = event_inj & SVM_EVTINJ_VEC_MASK;
+
+	if (!(event_inj & SVM_EVTINJ_VALID))
+		return true;
+
+	if (type != SVM_EVTINJ_TYPE_INTR && type != SVM_EVTINJ_TYPE_NMI &&
+	    type != SVM_EVTINJ_TYPE_EXEPT && type != SVM_EVTINJ_TYPE_SOFT)
+		return false;
+
+	if (type == SVM_EVTINJ_TYPE_EXEPT &&
+	    !nested_svm_event_inj_valid_exept(vcpu, vector))
+		return false;
+
+	return true;
+}
+
+static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
+				       struct vmcb_ctrl_area_cached *control)
 {
 	if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN)))
 		return false;
@@ -348,7 +386,8 @@ static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 	if (CC(control->asid == 0))
 		return false;
 
-	if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled))
+	if (CC((control->misc_ctl & SVM_MISC_ENABLE_NP) &&
+	       !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3)))
 		return false;
 
 	if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa,
@@ -363,12 +402,15 @@ static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 		return false;
 	}
 
+	if (CC(!nested_svm_check_event_inj(vcpu, control->event_inj)))
+		return false;
+
 	return true;
 }
 
 /* Common checks that apply to both L1 and L2 state.  */
-static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
-				     struct vmcb_save_area_cached *save)
+static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu,
+				   struct vmcb_save_area_cached *save)
 {
 	if (CC(!(save->efer & EFER_SVME)))
 		return false;
@@ -390,6 +432,10 @@ static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
 		    CC(!(save->cr0 & X86_CR0_PE)) ||
 		    CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3)))
 			return false;
+
+		if (CC((save->cs.attrib & SVM_SELECTOR_L_MASK) &&
+		       (save->cs.attrib & SVM_SELECTOR_DB_MASK)))
+			return false;
 	}
 
 	/* Note, SVM doesn't have any additional restrictions on CR4. */
@@ -402,26 +448,12 @@ static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
 	return true;
 }
 
-static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu)
-{
-	struct vcpu_svm *svm = to_svm(vcpu);
-	struct vmcb_save_area_cached *save = &svm->nested.save;
-
-	return __nested_vmcb_check_save(vcpu, save);
-}
-
-static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu)
+int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
-	struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl;
 
-	return __nested_vmcb_check_controls(vcpu, ctl);
-}
-
-int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu)
-{
-	if (!nested_vmcb_check_save(vcpu) ||
-	    !nested_vmcb_check_controls(vcpu))
+	if (!nested_vmcb_check_save(vcpu, &svm->nested.save) ||
+	    !nested_vmcb_check_controls(vcpu, &svm->nested.ctl))
 		return -EINVAL;
 
 	return 0;
@@ -456,37 +488,39 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
 	nested_svm_sanitize_intercept(vcpu, to, SKINIT);
 	nested_svm_sanitize_intercept(vcpu, to, RDPRU);
 
-	to->iopm_base_pa        = from->iopm_base_pa;
-	to->msrpm_base_pa       = from->msrpm_base_pa;
+	/* Always clear SVM_MISC_ENABLE_NP if the guest cannot use NPTs */
+	to->misc_ctl = from->misc_ctl;
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT))
+		to->misc_ctl &= ~SVM_MISC_ENABLE_NP;
+
+	to->iopm_base_pa        = from->iopm_base_pa & PAGE_MASK;
+	to->msrpm_base_pa       = from->msrpm_base_pa & PAGE_MASK;
 	to->tsc_offset          = from->tsc_offset;
-	to->tlb_ctl             = from->tlb_ctl;
+	to->tlb_ctl             = from->tlb_ctl & TLB_CONTROL_MASK;
 	to->erap_ctl            = from->erap_ctl;
 	to->int_ctl             = from->int_ctl;
-	to->int_vector          = from->int_vector;
-	to->int_state           = from->int_state;
+	to->int_vector          = from->int_vector & SVM_INT_VECTOR_MASK;
+	to->int_state           = from->int_state & SVM_INTERRUPT_SHADOW_MASK;
 	to->exit_code           = from->exit_code;
 	to->exit_info_1         = from->exit_info_1;
 	to->exit_info_2         = from->exit_info_2;
 	to->exit_int_info       = from->exit_int_info;
 	to->exit_int_info_err   = from->exit_int_info_err;
-	to->nested_ctl          = from->nested_ctl;
-	to->event_inj           = from->event_inj;
+	to->event_inj           = from->event_inj & ~SVM_EVTINJ_RESERVED_BITS;
 	to->event_inj_err       = from->event_inj_err;
 	to->next_rip            = from->next_rip;
 	to->nested_cr3          = from->nested_cr3;
-	to->virt_ext            = from->virt_ext;
+	to->misc_ctl2		= from->misc_ctl2;
 	to->pause_filter_count  = from->pause_filter_count;
 	to->pause_filter_thresh = from->pause_filter_thresh;
 
-	/* Copy asid here because nested_vmcb_check_controls will check it.  */
+	/* Copy asid here because nested_vmcb_check_controls() will check it */
 	to->asid           = from->asid;
-	to->msrpm_base_pa &= ~0x0fffULL;
-	to->iopm_base_pa  &= ~0x0fffULL;
+	to->clean = from->clean;
 
 #ifdef CONFIG_KVM_HYPERV
 	/* Hyper-V extensions (Enlightened VMCB) */
 	if (kvm_hv_hypercall_enabled(vcpu)) {
-		to->clean = from->clean;
 		memcpy(&to->hv_enlightenments, &from->hv_enlightenments,
 		       sizeof(to->hv_enlightenments));
 	}
@@ -502,17 +536,34 @@ void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to,
 					     struct vmcb_save_area *from)
 {
-	/*
-	 * Copy only fields that are validated, as we need them
-	 * to avoid TOC/TOU races.
-	 */
+	to->es = from->es;
+	to->cs = from->cs;
+	to->ss = from->ss;
+	to->ds = from->ds;
+	to->gdtr = from->gdtr;
+	to->idtr = from->idtr;
+
+	to->cpl = from->cpl;
+
 	to->efer = from->efer;
-	to->cr0 = from->cr0;
-	to->cr3 = from->cr3;
 	to->cr4 = from->cr4;
-
-	to->dr6 = from->dr6;
+	to->cr3 = from->cr3;
+	to->cr0 = from->cr0;
 	to->dr7 = from->dr7;
+	to->dr6 = from->dr6;
+
+	to->rflags = from->rflags;
+	to->rip = from->rip;
+	to->rsp = from->rsp;
+
+	to->s_cet = from->s_cet;
+	to->ssp = from->ssp;
+	to->isst_addr = from->isst_addr;
+
+	to->rax = from->rax;
+	to->cr2 = from->cr2;
+
+	svm_copy_lbrs(to, from);
 }
 
 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
@@ -530,6 +581,7 @@ void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)
 	u32 mask;
 	svm->nested.ctl.event_inj      = svm->vmcb->control.event_inj;
 	svm->nested.ctl.event_inj_err  = svm->vmcb->control.event_inj_err;
+	svm->nested.ctl.int_state	= svm->vmcb->control.int_state;
 
 	/* Only a few fields of int_ctl are written by the processor.  */
 	mask = V_IRQ_MASK | V_TPR_MASK;
@@ -542,7 +594,7 @@ void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)
 	 * int_ctl (because it was never recognized while L2 was running).
 	 */
 	if (svm_is_intercept(svm, INTERCEPT_VINTR) &&
-	    !test_bit(INTERCEPT_VINTR, (unsigned long *)svm->nested.ctl.intercepts))
+	    !vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_VINTR))
 		mask &= ~V_IRQ_MASK;
 
 	if (nested_vgif_enabled(svm))
@@ -648,8 +700,16 @@ void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm)
 	svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;
 }
 
-static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12)
+static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu)
 {
+	return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
+		(to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR);
+}
+
+static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
+{
+	struct vmcb_ctrl_area_cached *control = &svm->nested.ctl;
+	struct vmcb_save_area_cached *save = &svm->nested.save;
 	bool new_vmcb12 = false;
 	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
@@ -665,64 +725,64 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12
 		svm->nested.force_msr_bitmap_recalc = true;
 	}
 
-	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) {
-		vmcb02->save.es = vmcb12->save.es;
-		vmcb02->save.cs = vmcb12->save.cs;
-		vmcb02->save.ss = vmcb12->save.ss;
-		vmcb02->save.ds = vmcb12->save.ds;
-		vmcb02->save.cpl = vmcb12->save.cpl;
+	if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_SEG))) {
+		vmcb02->save.es = save->es;
+		vmcb02->save.cs = save->cs;
+		vmcb02->save.ss = save->ss;
+		vmcb02->save.ds = save->ds;
+		vmcb02->save.cpl = save->cpl;
 		vmcb_mark_dirty(vmcb02, VMCB_SEG);
 	}
 
-	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) {
-		vmcb02->save.gdtr = vmcb12->save.gdtr;
-		vmcb02->save.idtr = vmcb12->save.idtr;
+	if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DT))) {
+		vmcb02->save.gdtr = save->gdtr;
+		vmcb02->save.idtr = save->idtr;
 		vmcb_mark_dirty(vmcb02, VMCB_DT);
 	}
 
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
-	    (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_CET)))) {
-		vmcb02->save.s_cet  = vmcb12->save.s_cet;
-		vmcb02->save.isst_addr = vmcb12->save.isst_addr;
-		vmcb02->save.ssp = vmcb12->save.ssp;
+	    (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_CET)))) {
+		vmcb02->save.s_cet  = save->s_cet;
+		vmcb02->save.isst_addr = save->isst_addr;
+		vmcb02->save.ssp = save->ssp;
 		vmcb_mark_dirty(vmcb02, VMCB_CET);
 	}
 
-	kvm_set_rflags(vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED);
+	kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED);
 
 	svm_set_efer(vcpu, svm->nested.save.efer);
 
 	svm_set_cr0(vcpu, svm->nested.save.cr0);
 	svm_set_cr4(vcpu, svm->nested.save.cr4);
 
-	svm->vcpu.arch.cr2 = vmcb12->save.cr2;
+	svm->vcpu.arch.cr2 = save->cr2;
 
-	kvm_rax_write(vcpu, vmcb12->save.rax);
-	kvm_rsp_write(vcpu, vmcb12->save.rsp);
-	kvm_rip_write(vcpu, vmcb12->save.rip);
+	kvm_rax_write(vcpu, save->rax);
+	kvm_rsp_write(vcpu, save->rsp);
+	kvm_rip_write(vcpu, save->rip);
 
 	/* In case we don't even reach vcpu_run, the fields are not updated */
-	vmcb02->save.rax = vmcb12->save.rax;
-	vmcb02->save.rsp = vmcb12->save.rsp;
-	vmcb02->save.rip = vmcb12->save.rip;
+	vmcb02->save.rax = save->rax;
+	vmcb02->save.rsp = save->rsp;
+	vmcb02->save.rip = save->rip;
 
-	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) {
+	if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) {
 		vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1;
 		svm->vcpu.arch.dr6  = svm->nested.save.dr6 | DR6_ACTIVE_LOW;
 		vmcb_mark_dirty(vmcb02, VMCB_DR);
 	}
 
-	if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
-		     (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) {
+	if (nested_vmcb12_has_lbrv(vcpu)) {
 		/*
 		 * Reserved bits of DEBUGCTL are ignored.  Be consistent with
 		 * svm_set_msr's definition of reserved bits.
 		 */
-		svm_copy_lbrs(vmcb02, vmcb12);
+		svm_copy_lbrs(&vmcb02->save, save);
 		vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS;
 	} else {
-		svm_copy_lbrs(vmcb02, vmcb01);
+		svm_copy_lbrs(&vmcb02->save, &vmcb01->save);
 	}
+	vmcb_mark_dirty(vmcb02, VMCB_LBR);
 	svm_update_lbrv(&svm->vcpu);
 }
 
@@ -750,18 +810,16 @@ static bool is_evtinj_nmi(u32 evtinj)
 	return type == SVM_EVTINJ_TYPE_NMI;
 }
 
-static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
-					  unsigned long vmcb12_rip,
-					  unsigned long vmcb12_csbase)
+static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 {
 	u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK;
 	u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK;
 
-	struct kvm_vcpu *vcpu = &svm->vcpu;
-	struct vmcb *vmcb01 = svm->vmcb01.ptr;
+	struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl;
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
-	u32 pause_count12;
-	u32 pause_thresh12;
+	struct vmcb *vmcb01 = svm->vmcb01.ptr;
+	struct kvm_vcpu *vcpu = &svm->vcpu;
+	u32 pause_count12, pause_thresh12;
 
 	nested_svm_transition_tlb_flush(vcpu);
 
@@ -774,7 +832,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 	 */
 
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) &&
-	    (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK))
+	    (vmcb12_ctrl->int_ctl & V_GIF_ENABLE_MASK))
 		int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
 	else
 		int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
@@ -790,8 +848,16 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 						V_NMI_BLOCKING_MASK);
 	}
 
-	/* Copied from vmcb01.  msrpm_base can be overwritten later.  */
-	vmcb02->control.nested_ctl = vmcb01->control.nested_ctl;
+	/*
+	 * Copied from vmcb01.  msrpm_base can be overwritten later.
+	 *
+	 * SVM_MISC_ENABLE_NP in vmcb12 is only used for consistency checks.  If
+	 * L1 enables NPTs, KVM shadows L1's NPTs and uses those to run L2. If
+	 * L1 disables NPT, KVM runs L2 with the same NPTs used to run L1. For
+	 * the latter, L1 runs L2 with shadow page tables that translate L2 GVAs
+	 * to L1 GPAs, so the same NPTs can be used for L1 and L2.
+	 */
+	vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & SVM_MISC_ENABLE_NP;
 	vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
 	vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
 	vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
@@ -818,7 +884,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 	 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the
 	 * entire cycle start over.
 	 */
-	if (vmcb02->save.rip && (svm->nested.ctl.bus_lock_rip == vmcb02->save.rip))
+	if (vmcb02->save.rip && (svm->nested.last_bus_lock_rip == vmcb02->save.rip))
 		vmcb02->control.bus_lock_counter = 1;
 	else
 		vmcb02->control.bus_lock_counter = 0;
@@ -832,10 +898,9 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 	if (nested_npt_enabled(svm))
 		nested_svm_init_mmu_context(vcpu);
 
-	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
-			vcpu->arch.l1_tsc_offset,
-			svm->nested.ctl.tsc_offset,
-			svm->tsc_ratio_msr);
+	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(vcpu->arch.l1_tsc_offset,
+							   vmcb12_ctrl->tsc_offset,
+							   svm->tsc_ratio_msr);
 
 	vmcb02->control.tsc_offset = vcpu->arch.tsc_offset;
 
@@ -844,49 +909,49 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 		nested_svm_update_tsc_ratio_msr(vcpu);
 
 	vmcb02->control.int_ctl             =
-		(svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) |
+		(vmcb12_ctrl->int_ctl & int_ctl_vmcb12_bits) |
 		(vmcb01->control.int_ctl & int_ctl_vmcb01_bits);
 
-	vmcb02->control.int_vector          = svm->nested.ctl.int_vector;
-	vmcb02->control.int_state           = svm->nested.ctl.int_state;
-	vmcb02->control.event_inj           = svm->nested.ctl.event_inj;
-	vmcb02->control.event_inj_err       = svm->nested.ctl.event_inj_err;
+	vmcb02->control.int_vector          = vmcb12_ctrl->int_vector;
+	vmcb02->control.int_state           = vmcb12_ctrl->int_state;
+	vmcb02->control.event_inj           = vmcb12_ctrl->event_inj;
+	vmcb02->control.event_inj_err       = vmcb12_ctrl->event_inj_err;
 
 	/*
-	 * next_rip is consumed on VMRUN as the return address pushed on the
-	 * stack for injected soft exceptions/interrupts.  If nrips is exposed
-	 * to L1, take it verbatim from vmcb12.  If nrips is supported in
-	 * hardware but not exposed to L1, stuff the actual L2 RIP to emulate
-	 * what a nrips=0 CPU would do (L1 is responsible for advancing RIP
-	 * prior to injecting the event).
+	 * If nrips is exposed to L1, take NextRIP as-is.  Otherwise, L1
+	 * advances L2's RIP before VMRUN instead of using NextRIP. KVM will
+	 * stuff the current RIP as vmcb02's NextRIP before L2 is run.  After
+	 * the first run of L2 (e.g. after save+restore), NextRIP is updated by
+	 * the CPU and/or KVM and should be used regardless of L1's support.
 	 */
-	if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
-		vmcb02->control.next_rip    = svm->nested.ctl.next_rip;
-	else if (boot_cpu_has(X86_FEATURE_NRIPS))
-		vmcb02->control.next_rip    = vmcb12_rip;
+	if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) ||
+	    !svm->nested.nested_run_pending)
+		vmcb02->control.next_rip = vmcb12_ctrl->next_rip;
 
 	svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj);
+
+	/*
+	 * soft_int_csbase, soft_int_old_rip, and soft_int_next_rip (if L1
+	 * doesn't have NRIPS) are initialized later, before the vCPU is run.
+	 */
 	if (is_evtinj_soft(vmcb02->control.event_inj)) {
 		svm->soft_int_injected = true;
-		svm->soft_int_csbase = vmcb12_csbase;
-		svm->soft_int_old_rip = vmcb12_rip;
-		if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
-			svm->soft_int_next_rip = svm->nested.ctl.next_rip;
-		else
-			svm->soft_int_next_rip = vmcb12_rip;
+		if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) ||
+		    !svm->nested.nested_run_pending)
+			svm->soft_int_next_rip = vmcb12_ctrl->next_rip;
 	}
 
-	/* LBR_CTL_ENABLE_MASK is controlled by svm_update_lbrv() */
+	/* SVM_MISC2_ENABLE_V_LBR is controlled by svm_update_lbrv() */
 
 	if (!nested_vmcb_needs_vls_intercept(svm))
-		vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
+		vmcb02->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE;
 
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER))
-		pause_count12 = svm->nested.ctl.pause_filter_count;
+		pause_count12 = vmcb12_ctrl->pause_filter_count;
 	else
 		pause_count12 = 0;
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD))
-		pause_thresh12 = svm->nested.ctl.pause_filter_thresh;
+		pause_thresh12 = vmcb12_ctrl->pause_filter_thresh;
 	else
 		pause_thresh12 = 0;
 	if (kvm_pause_in_guest(svm->vcpu.kvm)) {
@@ -900,7 +965,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 		vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh;
 
 		/* ... but ensure filtering is disabled if so requested.  */
-		if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) {
+		if (vmcb12_is_intercept(vmcb12_ctrl, INTERCEPT_PAUSE)) {
 			if (!pause_count12)
 				vmcb02->control.pause_filter_count = 0;
 			if (!pause_thresh12)
@@ -917,7 +982,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 	 * L2 is the "guest").
 	 */
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
-		vmcb02->control.erap_ctl = (svm->nested.ctl.erap_ctl &
+		vmcb02->control.erap_ctl = (vmcb12_ctrl->erap_ctl &
 					    ERAP_CONTROL_ALLOW_LARGER_RAP) |
 					   ERAP_CONTROL_CLEAR_RAP;
 
@@ -925,7 +990,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 	 * Merge guest and host intercepts - must be called with vcpu in
 	 * guest-mode to take effect.
 	 */
-	recalc_intercepts(svm);
+	nested_vmcb02_recalc_intercepts(svm);
 }
 
 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
@@ -940,28 +1005,29 @@ static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to
 	to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl;
 }
 
-int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa,
-			 struct vmcb *vmcb12, bool from_vmrun)
+int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
+	struct vmcb_ctrl_area_cached *control = &svm->nested.ctl;
+	struct vmcb_save_area_cached *save = &svm->nested.save;
 	int ret;
 
 	trace_kvm_nested_vmenter(svm->vmcb->save.rip,
 				 vmcb12_gpa,
-				 vmcb12->save.rip,
-				 vmcb12->control.int_ctl,
-				 vmcb12->control.event_inj,
-				 vmcb12->control.nested_ctl,
-				 vmcb12->control.nested_cr3,
-				 vmcb12->save.cr3,
+				 save->rip,
+				 control->int_ctl,
+				 control->event_inj,
+				 control->misc_ctl,
+				 control->nested_cr3,
+				 save->cr3,
 				 KVM_ISA_SVM);
 
-	trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff,
-				    vmcb12->control.intercepts[INTERCEPT_CR] >> 16,
-				    vmcb12->control.intercepts[INTERCEPT_EXCEPTION],
-				    vmcb12->control.intercepts[INTERCEPT_WORD3],
-				    vmcb12->control.intercepts[INTERCEPT_WORD4],
-				    vmcb12->control.intercepts[INTERCEPT_WORD5]);
+	trace_kvm_nested_intercepts(control->intercepts[INTERCEPT_CR] & 0xffff,
+				    control->intercepts[INTERCEPT_CR] >> 16,
+				    control->intercepts[INTERCEPT_EXCEPTION],
+				    control->intercepts[INTERCEPT_WORD3],
+				    control->intercepts[INTERCEPT_WORD4],
+				    control->intercepts[INTERCEPT_WORD5]);
 
 
 	svm->nested.vmcb12_gpa = vmcb12_gpa;
@@ -971,8 +1037,8 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa,
 	nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);
 
 	svm_switch_vmcb(svm, &svm->nested.vmcb02);
-	nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base);
-	nested_vmcb02_prepare_save(svm, vmcb12);
+	nested_vmcb02_prepare_control(svm);
+	nested_vmcb02_prepare_save(svm);
 
 	ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3,
 				  nested_npt_enabled(svm), from_vmrun);
@@ -992,12 +1058,38 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa,
 	return 0;
 }
 
+static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	struct kvm_host_map map;
+	struct vmcb *vmcb12;
+	int r = 0;
+
+	if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map))
+		return -EFAULT;
+
+	vmcb12 = map.hva;
+	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
+	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
+
+	if (nested_svm_check_cached_vmcb12(vcpu) < 0) {
+		vmcb12->control.exit_code = SVM_EXIT_ERR;
+		vmcb12->control.exit_info_1 = 0;
+		vmcb12->control.exit_info_2 = 0;
+		vmcb12->control.event_inj = 0;
+		vmcb12->control.event_inj_err = 0;
+		svm_set_gif(svm, false);
+		r = -EINVAL;
+	}
+
+	kvm_vcpu_unmap(vcpu, &map);
+	return r;
+}
+
 int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 	int ret;
-	struct vmcb *vmcb12;
-	struct kvm_host_map map;
 	u64 vmcb12_gpa;
 	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 
@@ -1018,32 +1110,25 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 		return ret;
 	}
 
-	vmcb12_gpa = svm->vmcb->save.rax;
-	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map);
-	if (ret == -EINVAL) {
-		kvm_inject_gp(vcpu, 0);
-		return 1;
-	} else if (ret) {
-		return kvm_skip_emulated_instruction(vcpu);
-	}
-
-	ret = kvm_skip_emulated_instruction(vcpu);
-
-	vmcb12 = map.hva;
-
 	if (WARN_ON_ONCE(!svm->nested.initialized))
 		return -EINVAL;
 
-	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
-	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
+	vmcb12_gpa = svm->vmcb->save.rax;
 
-	if (nested_svm_check_cached_vmcb12(vcpu) < 0) {
-		vmcb12->control.exit_code    = SVM_EXIT_ERR;
-		vmcb12->control.exit_info_1  = 0;
-		vmcb12->control.exit_info_2  = 0;
-		goto out;
+	ret = nested_svm_copy_vmcb12_to_cache(vcpu, vmcb12_gpa);
+	if (ret) {
+		if (ret == -EFAULT) {
+			kvm_inject_gp(vcpu, 0);
+			return 1;
+		}
+
+		/* Advance RIP past VMRUN as part of the nested #VMEXIT. */
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
+	/* At this point, VMRUN is guaranteed to not fault; advance RIP. */
+	ret = kvm_skip_emulated_instruction(vcpu);
+
 	/*
 	 * Since vmcb01 is not in use, we can use it to store some of the L1
 	 * state.
@@ -1059,25 +1144,18 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 
 	svm->nested.nested_run_pending = 1;
 
-	if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true))
-		goto out_exit_err;
-
-	if (nested_svm_merge_msrpm(vcpu))
-		goto out;
-
-out_exit_err:
-	svm->nested.nested_run_pending = 0;
-	svm->nmi_l1_to_l2 = false;
-	svm->soft_int_injected = false;
-
-	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
-	svm->vmcb->control.exit_info_1  = 0;
-	svm->vmcb->control.exit_info_2  = 0;
+	if (enter_svm_guest_mode(vcpu, vmcb12_gpa, true) ||
+	    !nested_svm_merge_msrpm(vcpu)) {
+		svm->nested.nested_run_pending = 0;
+		svm->nmi_l1_to_l2 = false;
+		svm->soft_int_injected = false;
 
-	nested_svm_vmexit(svm);
+		svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
+		svm->vmcb->control.exit_info_1  = 0;
+		svm->vmcb->control.exit_info_2  = 0;
 
-out:
-	kvm_vcpu_unmap(vcpu, &map);
+		nested_svm_vmexit(svm);
+	}
 
 	return ret;
 }
@@ -1107,6 +1185,11 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save,
 		to_save->isst_addr = from_save->isst_addr;
 		to_save->ssp = from_save->ssp;
 	}
+
+	if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) {
+		svm_copy_lbrs(to_save, from_save);
+		to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS;
+	}
 }
 
 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
@@ -1125,36 +1208,20 @@ void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
 	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
 }
 
-int nested_svm_vmexit(struct vcpu_svm *svm)
+static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 {
-	struct kvm_vcpu *vcpu = &svm->vcpu;
-	struct vmcb *vmcb01 = svm->vmcb01.ptr;
+	struct vcpu_svm *svm = to_svm(vcpu);
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
-	struct vmcb *vmcb12;
 	struct kvm_host_map map;
+	struct vmcb *vmcb12;
 	int rc;
 
 	rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);
-	if (rc) {
-		if (rc == -EINVAL)
-			kvm_inject_gp(vcpu, 0);
-		return 1;
-	}
+	if (rc)
+		return rc;
 
 	vmcb12 = map.hva;
 
-	/* Exit Guest-Mode */
-	leave_guest_mode(vcpu);
-	svm->nested.vmcb12_gpa = 0;
-	WARN_ON_ONCE(svm->nested.nested_run_pending);
-
-	kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
-
-	/* in case we halted in L2 */
-	kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
-
-	/* Give the current vmcb to the guest */
-
 	vmcb12->save.es     = vmcb02->save.es;
 	vmcb12->save.cs     = vmcb02->save.cs;
 	vmcb12->save.ss     = vmcb02->save.ss;
@@ -1164,7 +1231,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	vmcb12->save.efer   = svm->vcpu.arch.efer;
 	vmcb12->save.cr0    = kvm_read_cr0(vcpu);
 	vmcb12->save.cr3    = kvm_read_cr3(vcpu);
-	vmcb12->save.cr2    = vmcb02->save.cr2;
+	vmcb12->save.cr2    = vcpu->arch.cr2;
 	vmcb12->save.cr4    = svm->vcpu.arch.cr4;
 	vmcb12->save.rflags = kvm_get_rflags(vcpu);
 	vmcb12->save.rip    = kvm_rip_read(vcpu);
@@ -1191,9 +1258,42 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
 		vmcb12->control.next_rip  = vmcb02->control.next_rip;
 
+	if (nested_vmcb12_has_lbrv(vcpu))
+		svm_copy_lbrs(&vmcb12->save, &vmcb02->save);
+
+	vmcb12->control.event_inj	  = 0;
+	vmcb12->control.event_inj_err	  = 0;
 	vmcb12->control.int_ctl           = svm->nested.ctl.int_ctl;
-	vmcb12->control.event_inj         = svm->nested.ctl.event_inj;
-	vmcb12->control.event_inj_err     = svm->nested.ctl.event_inj_err;
+
+	trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
+				       vmcb12->control.exit_info_1,
+				       vmcb12->control.exit_info_2,
+				       vmcb12->control.exit_int_info,
+				       vmcb12->control.exit_int_info_err,
+				       KVM_ISA_SVM);
+
+	kvm_vcpu_unmap(vcpu, &map);
+	return 0;
+}
+
+void nested_svm_vmexit(struct vcpu_svm *svm)
+{
+	struct kvm_vcpu *vcpu = &svm->vcpu;
+	struct vmcb *vmcb01 = svm->vmcb01.ptr;
+	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
+
+	if (nested_svm_vmexit_update_vmcb12(vcpu))
+		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
+
+	/* Exit Guest-Mode */
+	leave_guest_mode(vcpu);
+	svm->nested.vmcb12_gpa = 0;
+	WARN_ON_ONCE(svm->nested.nested_run_pending);
+
+	kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
+
+	/* in case we halted in L2 */
+	kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
 
 	if (!kvm_pause_in_guest(vcpu->kvm)) {
 		vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count;
@@ -1202,11 +1302,11 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	}
 
 	/*
-	 * Invalidate bus_lock_rip unless KVM is still waiting for the guest
-	 * to make forward progress before re-enabling bus lock detection.
+	 * Invalidate last_bus_lock_rip unless KVM is still waiting for the
+	 * guest to make forward progress before re-enabling bus lock detection.
 	 */
 	if (!vmcb02->control.bus_lock_counter)
-		svm->nested.ctl.bus_lock_rip = INVALID_GPA;
+		svm->nested.last_bus_lock_rip = INVALID_GPA;
 
 	nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);
 
@@ -1239,11 +1339,10 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	if (!nested_exit_on_intr(svm))
 		kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
 
-	if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
-		     (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)))
-		svm_copy_lbrs(vmcb12, vmcb02);
-	else
-		svm_copy_lbrs(vmcb01, vmcb02);
+	if (!nested_vmcb12_has_lbrv(vcpu)) {
+		svm_copy_lbrs(&vmcb01->save, &vmcb02->save);
+		vmcb_mark_dirty(vmcb01, VMCB_LBR);
+	}
 
 	svm_update_lbrv(vcpu);
 
@@ -1296,22 +1395,16 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	svm->vcpu.arch.dr7 = DR7_FIXED_1;
 	kvm_update_dr7(&svm->vcpu);
 
-	trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
-				       vmcb12->control.exit_info_1,
-				       vmcb12->control.exit_info_2,
-				       vmcb12->control.exit_int_info,
-				       vmcb12->control.exit_int_info_err,
-				       KVM_ISA_SVM);
-
-	kvm_vcpu_unmap(vcpu, &map);
-
 	nested_svm_transition_tlb_flush(vcpu);
 
 	nested_svm_uninit_mmu_context(vcpu);
 
-	rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true);
-	if (rc)
-		return 1;
+	if (nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true))
+		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
+
+	/* Drop tracking for L1->L2 injected NMIs and soft IRQs */
+	svm->nmi_l1_to_l2 = false;
+	svm->soft_int_injected = false;
 
 	/*
 	 * Drop what we picked up for L2 via svm_complete_interrupts() so it
@@ -1336,8 +1429,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm)
 	 */
 	if (kvm_apicv_activated(vcpu->kvm))
 		__kvm_vcpu_update_apicv(vcpu);
-
-	return 0;
 }
 
 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu)
@@ -1682,9 +1773,7 @@ int nested_svm_exit_special(struct vcpu_svm *svm)
 	}
 	case SVM_EXIT_VMMCALL:
 		/* Hyper-V L2 TLB flush hypercall is handled by L0 */
-		if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
-		    nested_svm_l2_tlb_flush_enabled(vcpu) &&
-		    kvm_hv_is_tlb_flush_hcall(vcpu))
+		if (nested_svm_is_l2_tlb_flush_hcall(vcpu))
 			return NESTED_EXIT_HOST;
 		break;
 	default:
@@ -1729,12 +1818,12 @@ static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
 	dst->exit_info_2          = from->exit_info_2;
 	dst->exit_int_info        = from->exit_int_info;
 	dst->exit_int_info_err    = from->exit_int_info_err;
-	dst->nested_ctl           = from->nested_ctl;
+	dst->misc_ctl		  = from->misc_ctl;
 	dst->event_inj            = from->event_inj;
 	dst->event_inj_err        = from->event_inj_err;
 	dst->next_rip             = from->next_rip;
-	dst->nested_cr3           = from->nested_cr3;
-	dst->virt_ext              = from->virt_ext;
+	dst->nested_cr3		  = from->nested_cr3;
+	dst->misc_ctl2		  = from->misc_ctl2;
 	dst->pause_filter_count   = from->pause_filter_count;
 	dst->pause_filter_thresh  = from->pause_filter_thresh;
 	/* 'clean' and 'hv_enlightenments' are not changed by KVM */
@@ -1869,12 +1958,12 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 
 	ret = -EINVAL;
 	__nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl);
-	if (!__nested_vmcb_check_controls(vcpu, &ctl_cached))
+	if (!nested_vmcb_check_controls(vcpu, &ctl_cached))
 		goto out_free;
 
 	/*
 	 * Processor state contains L2 state.  Check that it is
-	 * valid for guest mode (see nested_vmcb_check_save).
+	 * valid for guest mode (see nested_vmcb_check_save()).
 	 */
 	cr0 = kvm_read_cr0(vcpu);
         if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW))
@@ -1888,7 +1977,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	if (!(save->cr0 & X86_CR0_PG) ||
 	    !(save->cr0 & X86_CR0_PE) ||
 	    (save->rflags & X86_EFLAGS_VM) ||
-	    !__nested_vmcb_check_save(vcpu, &save_cached))
+	    !nested_vmcb_check_save(vcpu, &save_cached))
 		goto out_free;
 
 
@@ -1915,7 +2004,13 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	nested_copy_vmcb_control_to_cache(svm, ctl);
 
 	svm_switch_vmcb(svm, &svm->nested.vmcb02);
-	nested_vmcb02_prepare_control(svm, svm->vmcb->save.rip, svm->vmcb->save.cs.base);
+	nested_vmcb02_prepare_control(svm);
+
+	/*
+	 * Any previously restored state (e.g. KVM_SET_SREGS) would mark fields
+	 * dirty in vmcb01 instead of vmcb02, so mark all of vmcb02 dirty here.
+	 */
+	vmcb_mark_all_dirty(svm->vmcb);
 
 	/*
 	 * While the nested guest CR3 is already checked and set by
@@ -1930,6 +2025,9 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 
 	svm->nested.force_msr_bitmap_recalc = true;
 
+	if (kvm_vcpu_apicv_active(vcpu))
+		kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
+
 	kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 	ret = 0;
 out_free:
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 3f9c1aa39a0af..b1aa85a6ca5af 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -52,18 +52,18 @@
 #define SNP_GUEST_VMM_ERR_GENERIC       (~0U)
 
 /* enable/disable SEV support */
-static bool sev_enabled = true;
+static bool __ro_after_init sev_enabled = true;
 module_param_named(sev, sev_enabled, bool, 0444);
 
 /* enable/disable SEV-ES support */
-static bool sev_es_enabled = true;
+static bool __ro_after_init sev_es_enabled = true;
 module_param_named(sev_es, sev_es_enabled, bool, 0444);
 
 /* enable/disable SEV-SNP support */
-static bool sev_snp_enabled = true;
+static bool __ro_after_init sev_snp_enabled = true;
 module_param_named(sev_snp, sev_snp_enabled, bool, 0444);
 
-static unsigned int nr_ciphertext_hiding_asids;
+static unsigned int __ro_after_init nr_ciphertext_hiding_asids;
 module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 0444);
 
 #define AP_RESET_HOLD_NONE		0
@@ -4434,25 +4434,17 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
 
 	switch (control->exit_code) {
 	case SVM_VMGEXIT_MMIO_READ:
-		ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
-		if (ret)
-			break;
+	case SVM_VMGEXIT_MMIO_WRITE: {
+		bool is_write = control->exit_code == SVM_VMGEXIT_MMIO_WRITE;
 
-		ret = kvm_sev_es_mmio_read(vcpu,
-					   control->exit_info_1,
-					   control->exit_info_2,
-					   svm->sev_es.ghcb_sa);
-		break;
-	case SVM_VMGEXIT_MMIO_WRITE:
-		ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);
+		ret = setup_vmgexit_scratch(svm, !is_write, control->exit_info_2);
 		if (ret)
 			break;
 
-		ret = kvm_sev_es_mmio_write(vcpu,
-					    control->exit_info_1,
-					    control->exit_info_2,
-					    svm->sev_es.ghcb_sa);
+		ret = kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1,
+				      control->exit_info_2, svm->sev_es.ghcb_sa);
 		break;
+	}
 	case SVM_VMGEXIT_NMI_COMPLETE:
 		++vcpu->stat.nmi_window_exits;
 		svm->nmi_masked = false;
@@ -4599,7 +4591,7 @@ static void sev_es_init_vmcb(struct vcpu_svm *svm, bool init_event)
 	struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);
 	struct vmcb *vmcb = svm->vmcb01.ptr;
 
-	svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE;
+	svm->vmcb->control.misc_ctl |= SVM_MISC_ENABLE_SEV_ES;
 
 	/*
 	 * An SEV-ES guest requires a VMSA area that is a separate from the
@@ -4639,7 +4631,7 @@ static void sev_es_init_vmcb(struct vcpu_svm *svm, bool init_event)
 	if (!sev_vcpu_has_debug_swap(svm)) {
 		vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
 		vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
-		recalc_intercepts(svm);
+		svm_mark_intercepts_dirty(svm);
 	} else {
 		/*
 		 * Disable #DB intercept iff DebugSwap is enabled.  KVM doesn't
@@ -4670,7 +4662,7 @@ void sev_init_vmcb(struct vcpu_svm *svm, bool init_event)
 {
 	struct kvm_vcpu *vcpu = &svm->vcpu;
 
-	svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
+	svm->vmcb->control.misc_ctl |= SVM_MISC_ENABLE_SEV;
 	clr_exception_intercept(svm, UD_VECTOR);
 
 	/*
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index e6477affac9a0..d2ca226871c2f 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -44,6 +44,7 @@
 #include <asm/traps.h>
 #include <asm/reboot.h>
 #include <asm/fpu/api.h>
+#include <asm/virt.h>
 
 #include <trace/events/ipi.h>
 
@@ -52,6 +53,7 @@
 #include "svm.h"
 #include "svm_ops.h"
 
+#include "hyperv.h"
 #include "kvm_onhyperv.h"
 #include "svm_onhyperv.h"
 
@@ -77,6 +79,7 @@ static bool erratum_383_found __read_mostly;
  * are published and we know what the new status bits are
  */
 static uint64_t osvw_len = 4, osvw_status;
+static DEFINE_SPINLOCK(osvw_lock);
 
 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 
@@ -110,52 +113,52 @@ static DEFINE_PER_CPU(u64, current_tsc_ratio);
  *	count only mode.
  */
 
-static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
+static unsigned short __ro_after_init pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 module_param(pause_filter_thresh, ushort, 0444);
 
-static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
+static unsigned short __ro_after_init pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 module_param(pause_filter_count, ushort, 0444);
 
 /* Default doubles per-vcpu window every exit. */
-static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
+static unsigned short __ro_after_init pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 module_param(pause_filter_count_grow, ushort, 0444);
 
 /* Default resets per-vcpu window every exit to pause_filter_count. */
-static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
+static unsigned short __ro_after_init pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 module_param(pause_filter_count_shrink, ushort, 0444);
 
 /* Default is to compute the maximum so we can never overflow. */
-static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
+static unsigned short __ro_after_init pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 module_param(pause_filter_count_max, ushort, 0444);
 
 /*
  * Use nested page tables by default.  Note, NPT may get forced off by
  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
  */
-bool npt_enabled = true;
+bool __ro_after_init npt_enabled = true;
 module_param_named(npt, npt_enabled, bool, 0444);
 
 /* allow nested virtualization in KVM/SVM */
-static int nested = true;
+static int __ro_after_init nested = true;
 module_param(nested, int, 0444);
 
 /* enable/disable Next RIP Save */
-int nrips = true;
+int __ro_after_init nrips = true;
 module_param(nrips, int, 0444);
 
 /* enable/disable Virtual VMLOAD VMSAVE */
-static int vls = true;
+static int __ro_after_init vls = true;
 module_param(vls, int, 0444);
 
 /* enable/disable Virtual GIF */
-int vgif = true;
+int __ro_after_init vgif = true;
 module_param(vgif, int, 0444);
 
 /* enable/disable LBR virtualization */
-int lbrv = true;
+int __ro_after_init lbrv = true;
 module_param(lbrv, int, 0444);
 
-static int tsc_scaling = true;
+static int __ro_after_init tsc_scaling = true;
 module_param(tsc_scaling, int, 0444);
 
 module_param(enable_device_posted_irqs, bool, 0444);
@@ -164,19 +167,19 @@ bool __read_mostly dump_invalid_vmcb;
 module_param(dump_invalid_vmcb, bool, 0644);
 
 
-bool intercept_smi = true;
+bool __ro_after_init intercept_smi = true;
 module_param(intercept_smi, bool, 0444);
 
-bool vnmi = true;
+bool __ro_after_init vnmi = true;
 module_param(vnmi, bool, 0444);
 
 module_param(enable_mediated_pmu, bool, 0444);
 
-static bool svm_gp_erratum_intercept = true;
+static bool __ro_after_init svm_gp_erratum_intercept = true;
 
 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 
-static unsigned long iopm_base;
+static unsigned long __read_mostly iopm_base;
 
 DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
 
@@ -216,6 +219,19 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 
 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
 		if (!(efer & EFER_SVME)) {
+			/*
+			 * Architecturally, clearing EFER.SVME while a guest is
+			 * running yields undefined behavior, i.e. KVM can do
+			 * literally anything.  Force the vCPU back into L1 as
+			 * that is the safest option for KVM, but synthesize a
+			 * triple fault (for L1!) so that KVM at least doesn't
+			 * run random L2 code in the context of L1.  Do so if
+			 * and only if the vCPU is actively running, e.g. to
+			 * avoid positives if userspace is stuffing state.
+			 */
+			if (is_guest_mode(vcpu) && vcpu->wants_to_run)
+				kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
+
 			svm_leave_nested(vcpu);
 			/* #GP intercept is still needed for vmware backdoor */
 			if (!enable_vmware_backdoor)
@@ -244,6 +260,8 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
 				set_exception_intercept(svm, GP_VECTOR);
 		}
+
+		kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
 	}
 
 	svm->vmcb->save.efer = efer | EFER_SVME;
@@ -420,6 +438,48 @@ static void svm_init_osvw(struct kvm_vcpu *vcpu)
 		vcpu->arch.osvw.status |= 1;
 }
 
+static void svm_init_os_visible_workarounds(void)
+{
+	u64 len, status;
+
+	/*
+	 * Get OS-Visible Workarounds (OSVW) bits.
+	 *
+	 * Note that it is possible to have a system with mixed processor
+	 * revisions and therefore different OSVW bits. If bits are not the same
+	 * on different processors then choose the worst case (i.e. if erratum
+	 * is present on one processor and not on another then assume that the
+	 * erratum is present everywhere).
+	 *
+	 * Note #2!  The OSVW MSRs are used to communciate that an erratum is
+	 * NOT present!  Software must assume erratum as present if its bit is
+	 * set in OSVW_STATUS *or* the bit number exceeds OSVW_ID_LENGTH.  If
+	 * either RDMSR fails, simply zero out the length to treat all errata
+	 * as being present.  Similarly, use the *minimum* length across all
+	 * CPUs, not the maximum length.
+	 *
+	 * If the length is zero, then is KVM already treating all errata as
+	 * being present and there's nothing left to do.
+	 */
+	if (!osvw_len)
+		return;
+
+	if (!this_cpu_has(X86_FEATURE_OSVW) ||
+	    native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &len) ||
+	    native_read_msr_safe(MSR_AMD64_OSVW_STATUS, &status))
+		len = status = 0;
+
+	if (status == READ_ONCE(osvw_status) && len >= READ_ONCE(osvw_len))
+		return;
+
+	guard(spinlock)(&osvw_lock);
+
+	if (len < osvw_len)
+		osvw_len = len;
+	osvw_status |= status;
+	osvw_status &= (1ULL << osvw_len) - 1;
+}
+
 static bool __kvm_is_svm_supported(void)
 {
 	int cpu = smp_processor_id();
@@ -477,27 +537,9 @@ static __always_inline struct sev_es_save_area *sev_es_host_save_area(struct svm
 	return &sd->save_area->host_sev_es_save;
 }
 
-static inline void kvm_cpu_svm_disable(void)
-{
-	uint64_t efer;
-
-	wrmsrq(MSR_VM_HSAVE_PA, 0);
-	rdmsrq(MSR_EFER, efer);
-	if (efer & EFER_SVME) {
-		/*
-		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
-		 * NMI aren't blocked.
-		 */
-		stgi();
-		wrmsrq(MSR_EFER, efer & ~EFER_SVME);
-	}
-}
-
 static void svm_emergency_disable_virtualization_cpu(void)
 {
-	kvm_rebooting = true;
-
-	kvm_cpu_svm_disable();
+	wrmsrq(MSR_VM_HSAVE_PA, 0);
 }
 
 static void svm_disable_virtualization_cpu(void)
@@ -506,7 +548,8 @@ static void svm_disable_virtualization_cpu(void)
 	if (tsc_scaling)
 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 
-	kvm_cpu_svm_disable();
+	x86_virt_put_ref(X86_FEATURE_SVM);
+	wrmsrq(MSR_VM_HSAVE_PA, 0);
 
 	amd_pmu_disable_virt();
 }
@@ -515,12 +558,12 @@ static int svm_enable_virtualization_cpu(void)
 {
 
 	struct svm_cpu_data *sd;
-	uint64_t efer;
 	int me = raw_smp_processor_id();
+	int r;
 
-	rdmsrq(MSR_EFER, efer);
-	if (efer & EFER_SVME)
-		return -EBUSY;
+	r = x86_virt_get_ref(X86_FEATURE_SVM);
+	if (r)
+		return r;
 
 	sd = per_cpu_ptr(&svm_data, me);
 	sd->asid_generation = 1;
@@ -528,8 +571,6 @@ static int svm_enable_virtualization_cpu(void)
 	sd->next_asid = sd->max_asid + 1;
 	sd->min_asid = max_sev_asid + 1;
 
-	wrmsrq(MSR_EFER, efer | EFER_SVME);
-
 	wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa);
 
 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
@@ -540,34 +581,7 @@ static int svm_enable_virtualization_cpu(void)
 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 	}
 
-
-	/*
-	 * Get OSVW bits.
-	 *
-	 * Note that it is possible to have a system with mixed processor
-	 * revisions and therefore different OSVW bits. If bits are not the same
-	 * on different processors then choose the worst case (i.e. if erratum
-	 * is present on one processor and not on another then assume that the
-	 * erratum is present everywhere).
-	 */
-	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
-		u64 len, status = 0;
-		int err;
-
-		err = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &len);
-		if (!err)
-			err = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, &status);
-
-		if (err)
-			osvw_status = osvw_len = 0;
-		else {
-			if (len < osvw_len)
-				osvw_len = len;
-			osvw_status |= status;
-			osvw_status &= (1ULL << osvw_len) - 1;
-		}
-	} else
-		osvw_status = osvw_len = 0;
+	svm_init_os_visible_workarounds();
 
 	svm_init_erratum_383();
 
@@ -635,7 +649,7 @@ static void set_dr_intercepts(struct vcpu_svm *svm)
 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static void clr_dr_intercepts(struct vcpu_svm *svm)
@@ -644,7 +658,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm)
 
 	vmcb->control.intercepts[INTERCEPT_DR] = 0;
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
@@ -710,7 +724,7 @@ void *svm_alloc_permissions_map(unsigned long size, gfp_t gfp_mask)
 static void svm_recalc_lbr_msr_intercepts(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
-	bool intercept = !(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK);
+	bool intercept = !(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR);
 
 	if (intercept == svm->lbr_msrs_intercepted)
 		return;
@@ -841,20 +855,9 @@ static void svm_recalc_msr_intercepts(struct kvm_vcpu *vcpu)
 	 */
 }
 
-void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
-{
-	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
-	to_vmcb->save.br_from		= from_vmcb->save.br_from;
-	to_vmcb->save.br_to		= from_vmcb->save.br_to;
-	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
-	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
-
-	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
-}
-
 static void __svm_enable_lbrv(struct kvm_vcpu *vcpu)
 {
-	to_svm(vcpu)->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
+	to_svm(vcpu)->vmcb->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_LBR;
 }
 
 void svm_enable_lbrv(struct kvm_vcpu *vcpu)
@@ -866,16 +869,16 @@ void svm_enable_lbrv(struct kvm_vcpu *vcpu)
 static void __svm_disable_lbrv(struct kvm_vcpu *vcpu)
 {
 	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
-	to_svm(vcpu)->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
+	to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR;
 }
 
 void svm_update_lbrv(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
-	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
+	bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR;
 	bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) ||
 			    (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
-			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
+			    (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR));
 
 	if (enable_lbrv && !current_enable_lbrv)
 		__svm_enable_lbrv(vcpu);
@@ -1009,6 +1012,14 @@ void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
 	preempt_enable();
 }
 
+static bool svm_has_pending_gif_event(struct vcpu_svm *svm)
+{
+	return svm->vcpu.arch.smi_pending ||
+	       svm->vcpu.arch.nmi_pending ||
+	       kvm_cpu_has_injectable_intr(&svm->vcpu) ||
+	       kvm_apic_has_pending_init_or_sipi(&svm->vcpu);
+}
+
 /* Evaluate instruction intercepts that depend on guest CPUID features. */
 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu)
 {
@@ -1034,22 +1045,45 @@ static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu)
 	}
 
 	/*
-	 * No need to toggle VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK here, it is
-	 * always set if vls is enabled. If the intercepts are set, the bit is
-	 * meaningless anyway.
+	 * Intercept instructions that #UD if EFER.SVME=0, as SVME must be set
+	 * even when running the guest, i.e. hardware will only ever see
+	 * EFER.SVME=1.
+	 *
+	 * No need to toggle any of the vgif/vls/etc. enable bits here, as they
+	 * are set when the VMCB is initialized and never cleared (if the
+	 * relevant intercepts are set, the enablements are meaningless anyway).
 	 */
-	if (guest_cpuid_is_intel_compatible(vcpu)) {
+	if (!(vcpu->arch.efer & EFER_SVME)) {
 		svm_set_intercept(svm, INTERCEPT_VMLOAD);
 		svm_set_intercept(svm, INTERCEPT_VMSAVE);
+		svm_set_intercept(svm, INTERCEPT_CLGI);
+		svm_set_intercept(svm, INTERCEPT_STGI);
 	} else {
 		/*
 		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
 		 * in VMCB and clear intercepts to avoid #VMEXIT.
 		 */
-		if (vls) {
+		if (guest_cpuid_is_intel_compatible(vcpu)) {
+			svm_set_intercept(svm, INTERCEPT_VMLOAD);
+			svm_set_intercept(svm, INTERCEPT_VMSAVE);
+		} else if (vls) {
 			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
 			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
 		}
+
+		/*
+		 * Process pending events when clearing STGI/CLGI intercepts if
+		 * there's at least one pending event that is masked by GIF, so
+		 * that KVM re-evaluates if the intercept needs to be set again
+		 * to track when GIF is re-enabled (e.g. for NMI injection).
+		 */
+		if (vgif) {
+			svm_clr_intercept(svm, INTERCEPT_CLGI);
+			svm_clr_intercept(svm, INTERCEPT_STGI);
+
+			if (svm_has_pending_gif_event(svm))
+				kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
+		}
 	}
 
 	if (kvm_need_rdpmc_intercept(vcpu))
@@ -1162,7 +1196,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event)
 
 	if (npt_enabled) {
 		/* Setup VMCB for Nested Paging */
-		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
+		control->misc_ctl |= SVM_MISC_ENABLE_NP;
 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
 		clr_exception_intercept(svm, PF_VECTOR);
 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
@@ -1194,14 +1228,11 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event)
 	if (vnmi)
 		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
 
-	if (vgif) {
-		svm_clr_intercept(svm, INTERCEPT_STGI);
-		svm_clr_intercept(svm, INTERCEPT_CLGI);
+	if (vgif)
 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
-	}
 
 	if (vls)
-		svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
+		svm->vmcb->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE;
 
 	if (vcpu->kvm->arch.bus_lock_detection_enabled)
 		svm_set_intercept(svm, INTERCEPT_BUSLOCK);
@@ -2244,13 +2275,9 @@ static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
 		[SVM_INSTR_VMSAVE] = vmsave_interception,
 	};
 	struct vcpu_svm *svm = to_svm(vcpu);
-	int ret;
 
 	if (is_guest_mode(vcpu)) {
-		/* Returns '1' or -errno on failure, '0' on success. */
-		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
-		if (ret)
-			return ret;
+		nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
 		return 1;
 	}
 	return svm_instr_handlers[opcode](vcpu);
@@ -2319,10 +2346,7 @@ void svm_set_gif(struct vcpu_svm *svm, bool value)
 			svm_clear_vintr(svm);
 
 		enable_gif(svm);
-		if (svm->vcpu.arch.smi_pending ||
-		    svm->vcpu.arch.nmi_pending ||
-		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
-		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
+		if (svm_has_pending_gif_event(svm))
 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
 	} else {
 		disable_gif(svm);
@@ -2366,6 +2390,9 @@ static int invlpga_interception(struct kvm_vcpu *vcpu)
 	gva_t gva = kvm_rax_read(vcpu);
 	u32 asid = kvm_rcx_read(vcpu);
 
+	if (nested_svm_check_permissions(vcpu))
+		return 1;
+
 	/* FIXME: Handle an address size prefix. */
 	if (!is_long_mode(vcpu))
 		gva = (u32)gva;
@@ -2723,6 +2750,24 @@ static int svm_get_feature_msr(u32 msr, u64 *data)
 	return 0;
 }
 
+static u64 *svm_vmcb_lbr(struct vcpu_svm *svm, u32 msr)
+{
+	switch (msr) {
+	case MSR_IA32_LASTBRANCHFROMIP:
+		return &svm->vmcb->save.br_from;
+	case MSR_IA32_LASTBRANCHTOIP:
+		return &svm->vmcb->save.br_to;
+	case MSR_IA32_LASTINTFROMIP:
+		return &svm->vmcb->save.last_excp_from;
+	case MSR_IA32_LASTINTTOIP:
+		return &svm->vmcb->save.last_excp_to;
+	default:
+		break;
+	}
+	KVM_BUG_ON(1, svm->vcpu.kvm);
+	return &svm->vmcb->save.br_from;
+}
+
 static bool sev_es_prevent_msr_access(struct kvm_vcpu *vcpu,
 				      struct msr_data *msr_info)
 {
@@ -2796,19 +2841,13 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		msr_info->data = svm->tsc_aux;
 		break;
 	case MSR_IA32_DEBUGCTLMSR:
-		msr_info->data = svm->vmcb->save.dbgctl;
+		msr_info->data = lbrv ? svm->vmcb->save.dbgctl : 0;
 		break;
 	case MSR_IA32_LASTBRANCHFROMIP:
-		msr_info->data = svm->vmcb->save.br_from;
-		break;
 	case MSR_IA32_LASTBRANCHTOIP:
-		msr_info->data = svm->vmcb->save.br_to;
-		break;
 	case MSR_IA32_LASTINTFROMIP:
-		msr_info->data = svm->vmcb->save.last_excp_from;
-		break;
 	case MSR_IA32_LASTINTTOIP:
-		msr_info->data = svm->vmcb->save.last_excp_to;
+		msr_info->data = lbrv ? *svm_vmcb_lbr(svm, msr_info->index) : 0;
 		break;
 	case MSR_VM_HSAVE_PA:
 		msr_info->data = svm->nested.hsave_msr;
@@ -3083,6 +3122,17 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
 		vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
 		svm_update_lbrv(vcpu);
 		break;
+	case MSR_IA32_LASTBRANCHFROMIP:
+	case MSR_IA32_LASTBRANCHTOIP:
+	case MSR_IA32_LASTINTFROMIP:
+	case MSR_IA32_LASTINTTOIP:
+		if (!lbrv)
+			return KVM_MSR_RET_UNSUPPORTED;
+		if (!msr->host_initiated)
+			return 1;
+		*svm_vmcb_lbr(svm, ecx) = data;
+		vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
+		break;
 	case MSR_VM_HSAVE_PA:
 		/*
 		 * Old kernels did not validate the value written to
@@ -3131,20 +3181,6 @@ static int interrupt_window_interception(struct kvm_vcpu *vcpu)
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 	svm_clear_vintr(to_svm(vcpu));
 
-	/*
-	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
-	 * In this case AVIC was temporarily disabled for
-	 * requesting the IRQ window and we have to re-enable it.
-	 *
-	 * If running nested, still remove the VM wide AVIC inhibit to
-	 * support case in which the interrupt window was requested when the
-	 * vCPU was not running nested.
-
-	 * All vCPUs which run still run nested, will remain to have their
-	 * AVIC still inhibited due to per-cpu AVIC inhibition.
-	 */
-	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
-
 	++vcpu->stat.irq_window_exits;
 	return 1;
 }
@@ -3224,11 +3260,27 @@ static int bus_lock_exit(struct kvm_vcpu *vcpu)
 	vcpu->arch.complete_userspace_io = complete_userspace_buslock;
 
 	if (is_guest_mode(vcpu))
-		svm->nested.ctl.bus_lock_rip = vcpu->arch.cui_linear_rip;
+		svm->nested.last_bus_lock_rip = vcpu->arch.cui_linear_rip;
 
 	return 0;
 }
 
+static int vmmcall_interception(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Inject a #UD if L2 is active and the VMMCALL isn't a Hyper-V TLB
+	 * hypercall, as VMMCALL #UDs if it's not intercepted, and this path is
+	 * reachable if and only if L1 doesn't want to intercept VMMCALL or has
+	 * enabled L0 (KVM) handling of Hyper-V L2 TLB flush hypercalls.
+	 */
+	if (is_guest_mode(vcpu) && !nested_svm_is_l2_tlb_flush_hcall(vcpu)) {
+		kvm_queue_exception(vcpu, UD_VECTOR);
+		return 1;
+	}
+
+	return kvm_emulate_hypercall(vcpu);
+}
+
 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[SVM_EXIT_READ_CR0]			= cr_interception,
 	[SVM_EXIT_READ_CR3]			= cr_interception,
@@ -3279,7 +3331,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
 	[SVM_EXIT_VMRUN]			= vmrun_interception,
-	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
+	[SVM_EXIT_VMMCALL]			= vmmcall_interception,
 	[SVM_EXIT_VMLOAD]			= vmload_interception,
 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
 	[SVM_EXIT_STGI]				= stgi_interception,
@@ -3354,13 +3406,13 @@ static void dump_vmcb(struct kvm_vcpu *vcpu)
 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
-	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
+	pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl);
 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
-	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
+	pr_err("%-20s%lld\n", "misc_ctl2:", control->misc_ctl2);
 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
@@ -3638,6 +3690,16 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 	return svm_invoke_exit_handler(vcpu, svm->vmcb->control.exit_code);
 }
 
+static void svm_set_nested_run_soft_int_state(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+
+	svm->soft_int_csbase = svm->vmcb->save.cs.base;
+	svm->soft_int_old_rip = kvm_rip_read(vcpu);
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
+		svm->soft_int_next_rip = kvm_rip_read(vcpu);
+}
+
 static int pre_svm_run(struct kvm_vcpu *vcpu)
 {
 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
@@ -3733,12 +3795,59 @@ static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
 		type = SVM_EVTINJ_TYPE_INTR;
 	}
 
+	/*
+	 * If AVIC was inhibited in order to detect an IRQ window, and there's
+	 * no other injectable interrupts pending or L2 is active (see below),
+	 * then drop the inhibit as the window has served its purpose.
+	 *
+	 * If L2 is active, this path is reachable if L1 is not intercepting
+	 * IRQs, i.e. if KVM is injecting L1 IRQs into L2.  AVIC is locally
+	 * inhibited while L2 is active; drop the VM-wide inhibit to optimize
+	 * the case in which the interrupt window was requested while L1 was
+	 * active (the vCPU was not running nested).
+	 */
+	if (svm->avic_irq_window &&
+	    (!kvm_cpu_has_injectable_intr(vcpu) || is_guest_mode(vcpu))) {
+		svm->avic_irq_window = false;
+		kvm_dec_apicv_irq_window_req(svm->vcpu.kvm);
+	}
+
 	trace_kvm_inj_virq(intr->nr, intr->soft, reinjected);
 	++vcpu->stat.irq_injections;
 
 	svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type;
 }
 
+static void svm_fixup_nested_rips(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+
+	if (!is_guest_mode(vcpu) || !svm->nested.nested_run_pending)
+		return;
+
+	/*
+	 * If nrips is supported in hardware but not exposed to L1, stuff the
+	 * actual L2 RIP to emulate what a nrips=0 CPU would do (L1 is
+	 * responsible for advancing RIP prior to injecting the event). Once L2
+	 * runs after L1 executes VMRUN, NextRIP is updated by the CPU and/or
+	 * KVM, and this is no longer needed.
+	 *
+	 * This is done here (as opposed to when preparing vmcb02) to use the
+	 * most up-to-date value of RIP regardless of the order of restoring
+	 * registers and nested state in the vCPU save+restore path.
+	 */
+	if (boot_cpu_has(X86_FEATURE_NRIPS) &&
+	    !guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
+		svm->vmcb->control.next_rip = kvm_rip_read(vcpu);
+
+	/*
+	 * Simiarly, initialize the soft int metadata here to use the most
+	 * up-to-date values of RIP and CS base, regardless of restore order.
+	 */
+	if (svm->soft_int_injected)
+		svm_set_nested_run_soft_int_state(vcpu);
+}
+
 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
 				     int trig_mode, int vector)
 {
@@ -3933,17 +4042,28 @@ static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
 	 */
 	if (vgif || gif_set(svm)) {
 		/*
-		 * IRQ window is not needed when AVIC is enabled,
-		 * unless we have pending ExtINT since it cannot be injected
-		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
-		 * and fallback to injecting IRQ via V_IRQ.
+		 * KVM only enables IRQ windows when AVIC is enabled if there's
+		 * pending ExtINT since it cannot be injected via AVIC (ExtINT
+		 * bypasses the local APIC).  V_IRQ is ignored by hardware when
+		 * AVIC is enabled, and so KVM needs to temporarily disable
+		 * AVIC in order to detect when it's ok to inject the ExtINT.
+		 *
+		 * If running nested, AVIC is already locally inhibited on this
+		 * vCPU (L2 vCPUs use a different MMU that never maps the AVIC
+		 * backing page), therefore there is no need to increment the
+		 * VM-wide AVIC inhibit.  KVM will re-evaluate events when the
+		 * vCPU exits to L1 and enable an IRQ window if the ExtINT is
+		 * still pending.
 		 *
-		 * If running nested, AVIC is already locally inhibited
-		 * on this vCPU, therefore there is no need to request
-		 * the VM wide AVIC inhibition.
+		 * Note, the IRQ window inhibit needs to be updated even if
+		 * AVIC is inhibited for a different reason, as KVM needs to
+		 * keep AVIC inhibited if the other reason is cleared and there
+		 * is still an injectable interrupt pending.
 		 */
-		if (!is_guest_mode(vcpu))
-			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
+		if (enable_apicv && !svm->avic_irq_window && !is_guest_mode(vcpu)) {
+			svm->avic_irq_window = true;
+			kvm_inc_apicv_irq_window_req(vcpu->kvm);
+		}
 
 		svm_set_vintr(svm);
 	}
@@ -4106,6 +4226,18 @@ static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
 	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
 	struct vcpu_svm *svm = to_svm(vcpu);
 
+	/*
+	 * Initialize the soft int fields *before* reading them below if KVM
+	 * aborted entry to the guest with a nested VMRUN pending.  To ensure
+	 * KVM uses up-to-date values for RIP and CS base across save/restore,
+	 * regardless of restore order, KVM waits to set the soft int fields
+	 * until VMRUN is imminent.  But when canceling injection, KVM requeues
+	 * the soft int and will reinject it via the standard injection flow,
+	 * and so KVM needs to grab the state from the pending nested VMRUN.
+	 */
+	if (is_guest_mode(vcpu) && svm->nested.nested_run_pending)
+		svm_set_nested_run_soft_int_state(vcpu);
+
 	/*
 	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
 	 * associated with the original soft exception/interrupt.  next_rip is
@@ -4335,6 +4467,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	    kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS))
 		svm->vmcb->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP;
 
+	svm_fixup_nested_rips(vcpu);
+
 	svm_hv_update_vp_id(svm->vmcb, vcpu);
 
 	/*
@@ -4355,7 +4489,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	 * VM-Exit), as running with the host's DEBUGCTL can negatively affect
 	 * guest state and can even be fatal, e.g. due to Bus Lock Detect.
 	 */
-	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
+	if (!(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) &&
 	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
 		update_debugctlmsr(svm->vmcb->save.dbgctl);
 
@@ -4386,7 +4520,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
 
-	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
+	if (!(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) &&
 	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
 		update_debugctlmsr(vcpu->arch.host_debugctl);
 
@@ -4436,6 +4570,16 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	svm_complete_interrupts(vcpu);
 
+	/*
+	 * Update the cache after completing interrupts to get an accurate
+	 * NextRIP, e.g. when re-injecting a soft interrupt.
+	 *
+	 * FIXME: Rework svm_get_nested_state() to not pull data from the
+	 *        cache (except for maybe int_ctl).
+	 */
+	if (is_guest_mode(vcpu))
+		svm->nested.ctl.next_rip = svm->vmcb->control.next_rip;
+
 	return svm_exit_handlers_fastpath(vcpu);
 }
 
@@ -4784,7 +4928,6 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct kvm_host_map map_save;
-	int ret;
 
 	if (!is_guest_mode(vcpu))
 		return 0;
@@ -4804,9 +4947,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
 
-	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
-	if (ret)
-		return ret;
+	nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
 
 	/*
 	 * KVM uses VMCB01 to store L1 host state while L2 runs but
@@ -4884,8 +5025,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
 	if (nested_svm_check_cached_vmcb12(vcpu) < 0)
 		goto unmap_save;
 
-	if (enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa,
-				 vmcb12, false) != 0)
+	if (enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, false) != 0)
 		goto unmap_save;
 
 	ret = 0;
@@ -5410,14 +5550,10 @@ static __init int svm_hardware_setup(void)
 		pr_err_ratelimited("NX (Execute Disable) not supported\n");
 		return -EOPNOTSUPP;
 	}
-	kvm_enable_efer_bits(EFER_NX);
 
 	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
 				     XFEATURE_MASK_BNDCSR);
 
-	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
-		kvm_enable_efer_bits(EFER_FFXSR);
-
 	if (tsc_scaling) {
 		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 			tsc_scaling = false;
@@ -5431,9 +5567,6 @@ static __init int svm_hardware_setup(void)
 
 	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
 
-	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
-		kvm_enable_efer_bits(EFER_AUTOIBRS);
-
 	/* Check for pause filtering support */
 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
 		pause_filter_count = 0;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 6942e6b0eda67..ff1e4b4dc9986 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -140,12 +140,32 @@ struct kvm_vmcb_info {
 };
 
 struct vmcb_save_area_cached {
+	struct vmcb_seg es;
+	struct vmcb_seg cs;
+	struct vmcb_seg ss;
+	struct vmcb_seg ds;
+	struct vmcb_seg gdtr;
+	struct vmcb_seg idtr;
+	u8 cpl;
 	u64 efer;
 	u64 cr4;
 	u64 cr3;
 	u64 cr0;
 	u64 dr7;
 	u64 dr6;
+	u64 rflags;
+	u64 rip;
+	u64 rsp;
+	u64 s_cet;
+	u64 ssp;
+	u64 isst_addr;
+	u64 rax;
+	u64 cr2;
+	u64 dbgctl;
+	u64 br_from;
+	u64 br_to;
+	u64 last_excp_from;
+	u64 last_excp_to;
 };
 
 struct vmcb_ctrl_area_cached {
@@ -166,14 +186,13 @@ struct vmcb_ctrl_area_cached {
 	u64 exit_info_2;
 	u32 exit_int_info;
 	u32 exit_int_info_err;
-	u64 nested_ctl;
+	u64 misc_ctl;
 	u32 event_inj;
 	u32 event_inj_err;
 	u64 next_rip;
 	u64 nested_cr3;
-	u64 virt_ext;
+	u64 misc_ctl2;
 	u32 clean;
-	u64 bus_lock_rip;
 	union {
 #if IS_ENABLED(CONFIG_HYPERV) || IS_ENABLED(CONFIG_KVM_HYPERV)
 		struct hv_vmcb_enlightenments hv_enlightenments;
@@ -188,6 +207,7 @@ struct svm_nested_state {
 	u64 vm_cr_msr;
 	u64 vmcb12_gpa;
 	u64 last_vmcb12_gpa;
+	u64 last_bus_lock_rip;
 
 	/*
 	 * The MSR permissions map used for vmcb02, which is the merge result
@@ -333,6 +353,7 @@ struct vcpu_svm {
 
 	bool guest_state_loaded;
 
+	bool avic_irq_window;
 	bool x2avic_msrs_intercepted;
 	bool lbr_msrs_intercepted;
 
@@ -357,8 +378,6 @@ struct svm_cpu_data {
 
 DECLARE_PER_CPU(struct svm_cpu_data, svm_data);
 
-void recalc_intercepts(struct vcpu_svm *svm);
-
 static __always_inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
 {
 	return container_of(kvm, struct kvm_svm, kvm);
@@ -415,9 +434,9 @@ static inline void vmcb_mark_dirty(struct vmcb *vmcb, int bit)
 	vmcb->control.clean &= ~(1 << bit);
 }
 
-static inline bool vmcb_is_dirty(struct vmcb *vmcb, int bit)
+static inline bool vmcb12_is_dirty(struct vmcb_ctrl_area_cached *control, int bit)
 {
-        return !test_bit(bit, (unsigned long *)&vmcb->control.clean);
+	return !test_bit(bit, (unsigned long *)&control->clean);
 }
 
 static __always_inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
@@ -486,6 +505,22 @@ static inline bool vmcb12_is_intercept(struct vmcb_ctrl_area_cached *control, u3
 	return __vmcb_is_intercept((unsigned long *)&control->intercepts, bit);
 }
 
+void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm);
+
+static inline void svm_mark_intercepts_dirty(struct vcpu_svm *svm)
+{
+	vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_INTERCEPTS);
+
+	/*
+	 * If L2 is active, recalculate the intercepts for vmcb02 to account
+	 * for the changes made to vmcb01.  All intercept configuration is done
+	 * for vmcb01 and then propagated to vmcb02 to combine KVM's intercepts
+	 * with L1's intercepts (from the vmcb12 snapshot).
+	 */
+	if (is_guest_mode(&svm->vcpu))
+		nested_vmcb02_recalc_intercepts(svm);
+}
+
 static inline void set_exception_intercept(struct vcpu_svm *svm, u32 bit)
 {
 	struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -493,7 +528,7 @@ static inline void set_exception_intercept(struct vcpu_svm *svm, u32 bit)
 	WARN_ON_ONCE(bit >= 32);
 	vmcb_set_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static inline void clr_exception_intercept(struct vcpu_svm *svm, u32 bit)
@@ -503,7 +538,7 @@ static inline void clr_exception_intercept(struct vcpu_svm *svm, u32 bit)
 	WARN_ON_ONCE(bit >= 32);
 	vmcb_clr_intercept(&vmcb->control, INTERCEPT_EXCEPTION_OFFSET + bit);
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static inline void svm_set_intercept(struct vcpu_svm *svm, int bit)
@@ -512,7 +547,7 @@ static inline void svm_set_intercept(struct vcpu_svm *svm, int bit)
 
 	vmcb_set_intercept(&vmcb->control, bit);
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static inline void svm_clr_intercept(struct vcpu_svm *svm, int bit)
@@ -521,7 +556,7 @@ static inline void svm_clr_intercept(struct vcpu_svm *svm, int bit)
 
 	vmcb_clr_intercept(&vmcb->control, bit);
 
-	recalc_intercepts(svm);
+	svm_mark_intercepts_dirty(svm);
 }
 
 static inline bool svm_is_intercept(struct vcpu_svm *svm, int bit)
@@ -578,7 +613,7 @@ static inline bool gif_set(struct vcpu_svm *svm)
 
 static inline bool nested_npt_enabled(struct vcpu_svm *svm)
 {
-	return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE;
+	return svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_NP;
 }
 
 static inline bool nested_vnmi_enabled(struct vcpu_svm *svm)
@@ -713,8 +748,16 @@ static inline void *svm_vcpu_alloc_msrpm(void)
 	return svm_alloc_permissions_map(MSRPM_SIZE, GFP_KERNEL_ACCOUNT);
 }
 
+#define svm_copy_lbrs(to, from)					\
+do {								\
+	(to)->dbgctl		= (from)->dbgctl;		\
+	(to)->br_from		= (from)->br_from;		\
+	(to)->br_to		= (from)->br_to;		\
+	(to)->last_excp_from	= (from)->last_excp_from;	\
+	(to)->last_excp_to	= (from)->last_excp_to;		\
+} while (0)
+
 void svm_vcpu_free_msrpm(void *msrpm);
-void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb);
 void svm_enable_lbrv(struct kvm_vcpu *vcpu);
 void svm_update_lbrv(struct kvm_vcpu *vcpu);
 
@@ -776,8 +819,7 @@ static inline bool nested_exit_on_nmi(struct vcpu_svm *svm)
 
 int __init nested_svm_init_msrpm_merge_offsets(void);
 
-int enter_svm_guest_mode(struct kvm_vcpu *vcpu,
-			 u64 vmcb_gpa, struct vmcb *vmcb12, bool from_vmrun);
+int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb_gpa, bool from_vmrun);
 void svm_leave_nested(struct kvm_vcpu *vcpu);
 void svm_free_nested(struct vcpu_svm *svm);
 int svm_allocate_nested(struct vcpu_svm *svm);
@@ -785,14 +827,14 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu);
 void svm_copy_vmrun_state(struct vmcb_save_area *to_save,
 			  struct vmcb_save_area *from_save);
 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb);
-int nested_svm_vmexit(struct vcpu_svm *svm);
+void nested_svm_vmexit(struct vcpu_svm *svm);
 
-static inline int nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code)
+static inline void nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code)
 {
 	svm->vmcb->control.exit_code	= exit_code;
 	svm->vmcb->control.exit_info_1	= 0;
 	svm->vmcb->control.exit_info_2	= 0;
-	return nested_svm_vmexit(svm);
+	nested_svm_vmexit(svm);
 }
 
 int nested_svm_exit_handled(struct vcpu_svm *svm);
diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
index 3392bcadfb896..d47c5c93c9913 100644
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
@@ -298,16 +298,16 @@ SYM_FUNC_START(__svm_vcpu_run)
 	RESTORE_GUEST_SPEC_CTRL_BODY
 	RESTORE_HOST_SPEC_CTRL_BODY (%_ASM_SP)
 
-10:	cmpb $0, _ASM_RIP(kvm_rebooting)
+10:	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne 2b
 	ud2
-30:	cmpb $0, _ASM_RIP(kvm_rebooting)
+30:	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne 4b
 	ud2
-50:	cmpb $0, _ASM_RIP(kvm_rebooting)
+50:	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne 6b
 	ud2
-70:	cmpb $0, _ASM_RIP(kvm_rebooting)
+70:	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne 8b
 	ud2
 
@@ -394,7 +394,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	RESTORE_GUEST_SPEC_CTRL_BODY
 	RESTORE_HOST_SPEC_CTRL_BODY %sil
 
-3:	cmpb $0, kvm_rebooting(%rip)
+3:	cmpb $0, virt_rebooting(%rip)
 	jne 2b
 	ud2
 
diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h
index 4e371c93ae168..56cacc06225ec 100644
--- a/arch/x86/kvm/vmx/capabilities.h
+++ b/arch/x86/kvm/vmx/capabilities.h
@@ -107,7 +107,7 @@ static inline bool cpu_has_load_perf_global_ctrl(void)
 
 static inline bool cpu_has_load_cet_ctrl(void)
 {
-	return (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_CET_STATE);
+	return vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_CET_STATE;
 }
 
 static inline bool cpu_has_save_perf_global_ctrl(void)
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index a46ccd6707859..dbebddf648be7 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -29,10 +29,15 @@ static __init int vt_hardware_setup(void)
 	if (ret)
 		return ret;
 
+	return enable_tdx ? tdx_hardware_setup() : 0;
+}
+
+static void vt_hardware_unsetup(void)
+{
 	if (enable_tdx)
-		tdx_hardware_setup();
+		tdx_hardware_unsetup();
 
-	return 0;
+	vmx_hardware_unsetup();
 }
 
 static int vt_vm_init(struct kvm *kvm)
@@ -869,7 +874,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.check_processor_compatibility = vmx_check_processor_compat,
 
-	.hardware_unsetup = vmx_hardware_unsetup,
+	.hardware_unsetup = vt_op(hardware_unsetup),
 
 	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
 	.disable_virtualization_cpu = vt_op(disable_virtualization_cpu),
@@ -1029,7 +1034,6 @@ struct kvm_x86_init_ops vt_init_ops __initdata = {
 static void __exit vt_exit(void)
 {
 	kvm_exit();
-	tdx_cleanup();
 	vmx_exit();
 }
 module_exit(vt_exit);
@@ -1043,11 +1047,6 @@ static int __init vt_init(void)
 	if (r)
 		return r;
 
-	/* tdx_init() has been taken */
-	r = tdx_bringup();
-	if (r)
-		goto err_tdx_bringup;
-
 	/*
 	 * TDX and VMX have different vCPU structures.  Calculate the
 	 * maximum size/align so that kvm_init() can use the larger
@@ -1074,8 +1073,6 @@ static int __init vt_init(void)
 	return 0;
 
 err_kvm_init:
-	tdx_cleanup();
-err_tdx_bringup:
 	vmx_exit();
 	return r;
 }
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index c5065f84b78be..04ce321ebdf39 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -6,6 +6,7 @@
 #include <linux/misc_cgroup.h>
 #include <linux/mmu_context.h>
 #include <asm/tdx.h>
+#include <asm/virt.h>
 #include "capabilities.h"
 #include "mmu.h"
 #include "x86_ops.h"
@@ -58,8 +59,6 @@ module_param_named(tdx, enable_tdx, bool, 0444);
 #define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51))
 #define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47))
 
-static enum cpuhp_state tdx_cpuhp_state;
-
 static const struct tdx_sys_info *tdx_sysinfo;
 
 void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err)
@@ -75,7 +74,7 @@ void tdh_vp_wr_failed(struct vcpu_tdx *tdx, char *uclass, char *op, u32 field,
 	pr_err("TDH_VP_WR[%s.0x%x]%s0x%llx failed: 0x%llx\n", uclass, field, op, val, err);
 }
 
-#define KVM_SUPPORTED_TD_ATTRS (TDX_TD_ATTR_SEPT_VE_DISABLE)
+#define KVM_SUPPORTED_TDX_TD_ATTRS (TDX_TD_ATTR_SEPT_VE_DISABLE)
 
 static __always_inline struct kvm_tdx *to_kvm_tdx(struct kvm *kvm)
 {
@@ -89,7 +88,7 @@ static __always_inline struct vcpu_tdx *to_tdx(struct kvm_vcpu *vcpu)
 
 static u64 tdx_get_supported_attrs(const struct tdx_sys_info_td_conf *td_conf)
 {
-	u64 val = KVM_SUPPORTED_TD_ATTRS;
+	u64 val = KVM_SUPPORTED_TDX_TD_ATTRS;
 
 	if ((val & td_conf->attributes_fixed1) != td_conf->attributes_fixed1)
 		return 0;
@@ -218,8 +217,6 @@ static int init_kvm_tdx_caps(const struct tdx_sys_info_td_conf *td_conf,
  */
 static DEFINE_MUTEX(tdx_lock);
 
-static atomic_t nr_configured_hkid;
-
 static bool tdx_operand_busy(u64 err)
 {
 	return (err & TDX_SEAMCALL_STATUS_MASK) == TDX_OPERAND_BUSY;
@@ -267,7 +264,6 @@ static inline void tdx_hkid_free(struct kvm_tdx *kvm_tdx)
 {
 	tdx_guest_keyid_free(kvm_tdx->hkid);
 	kvm_tdx->hkid = -1;
-	atomic_dec(&nr_configured_hkid);
 	misc_cg_uncharge(MISC_CG_RES_TDX, kvm_tdx->misc_cg, 1);
 	put_misc_cg(kvm_tdx->misc_cg);
 	kvm_tdx->misc_cg = NULL;
@@ -1467,17 +1463,11 @@ static int tdx_emulate_mmio(struct kvm_vcpu *vcpu)
 
 	/* Request the device emulation to userspace device model. */
 	vcpu->mmio_is_write = write;
-	if (!write)
-		vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;
 
-	vcpu->run->mmio.phys_addr = gpa;
-	vcpu->run->mmio.len = size;
-	vcpu->run->mmio.is_write = write;
-	vcpu->run->exit_reason = KVM_EXIT_MMIO;
+	__kvm_prepare_emulated_mmio_exit(vcpu, gpa, size, &val, write);
 
-	if (write) {
-		memcpy(vcpu->run->mmio.data, &val, size);
-	} else {
+	if (!write) {
+		vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;
 		vcpu->mmio_fragments[0].gpa = gpa;
 		vcpu->mmio_fragments[0].len = size;
 		trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, size, gpa, NULL);
@@ -1994,7 +1984,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
 	 * TDX_SEAMCALL_VMFAILINVALID.
 	 */
 	if (unlikely((vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR)) {
-		KVM_BUG_ON(!kvm_rebooting, vcpu->kvm);
+		KVM_BUG_ON(!virt_rebooting, vcpu->kvm);
 		goto unhandled_exit;
 	}
 
@@ -2397,8 +2387,6 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 
 	ret = -ENOMEM;
 
-	atomic_inc(&nr_configured_hkid);
-
 	tdr_page = alloc_page(GFP_KERNEL);
 	if (!tdr_page)
 		goto free_hkid;
@@ -3290,106 +3278,15 @@ int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
 	return PG_LEVEL_4K;
 }
 
-static int tdx_online_cpu(unsigned int cpu)
-{
-	unsigned long flags;
-	int r;
-
-	/* Sanity check CPU is already in post-VMXON */
-	WARN_ON_ONCE(!(cr4_read_shadow() & X86_CR4_VMXE));
-
-	local_irq_save(flags);
-	r = tdx_cpu_enable();
-	local_irq_restore(flags);
-
-	return r;
-}
-
-static int tdx_offline_cpu(unsigned int cpu)
-{
-	int i;
-
-	/* No TD is running.  Allow any cpu to be offline. */
-	if (!atomic_read(&nr_configured_hkid))
-		return 0;
-
-	/*
-	 * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to
-	 * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory
-	 * controller with pconfig.  If we have active TDX HKID, refuse to
-	 * offline the last online cpu.
-	 */
-	for_each_online_cpu(i) {
-		/*
-		 * Found another online cpu on the same package.
-		 * Allow to offline.
-		 */
-		if (i != cpu && topology_physical_package_id(i) ==
-				topology_physical_package_id(cpu))
-			return 0;
-	}
-
-	/*
-	 * This is the last cpu of this package.  Don't offline it.
-	 *
-	 * Because it's hard for human operator to understand the
-	 * reason, warn it.
-	 */
-#define MSG_ALLPKG_ONLINE \
-	"TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n"
-	pr_warn_ratelimited(MSG_ALLPKG_ONLINE);
-	return -EBUSY;
-}
-
-static void __do_tdx_cleanup(void)
+void tdx_hardware_unsetup(void)
 {
-	/*
-	 * Once TDX module is initialized, it cannot be disabled and
-	 * re-initialized again w/o runtime update (which isn't
-	 * supported by kernel).  Only need to remove the cpuhp here.
-	 * The TDX host core code tracks TDX status and can handle
-	 * 'multiple enabling' scenario.
-	 */
-	WARN_ON_ONCE(!tdx_cpuhp_state);
-	cpuhp_remove_state_nocalls_cpuslocked(tdx_cpuhp_state);
-	tdx_cpuhp_state = 0;
+	misc_cg_set_capacity(MISC_CG_RES_TDX, 0);
 }
 
-static void __tdx_cleanup(void)
-{
-	cpus_read_lock();
-	__do_tdx_cleanup();
-	cpus_read_unlock();
-}
-
-static int __init __do_tdx_bringup(void)
-{
-	int r;
-
-	/*
-	 * TDX-specific cpuhp callback to call tdx_cpu_enable() on all
-	 * online CPUs before calling tdx_enable(), and on any new
-	 * going-online CPU to make sure it is ready for TDX guest.
-	 */
-	r = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
-					 "kvm/cpu/tdx:online",
-					 tdx_online_cpu, tdx_offline_cpu);
-	if (r < 0)
-		return r;
-
-	tdx_cpuhp_state = r;
-
-	r = tdx_enable();
-	if (r)
-		__do_tdx_cleanup();
-
-	return r;
-}
-
-static int __init __tdx_bringup(void)
+static int __init __tdx_hardware_setup(void)
 {
 	const struct tdx_sys_info_td_conf *td_conf;
-	int r, i;
+	int i;
 
 	for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++) {
 		/*
@@ -3405,34 +3302,18 @@ static int __init __tdx_bringup(void)
 		}
 	}
 
-	/*
-	 * Enabling TDX requires enabling hardware virtualization first,
-	 * as making SEAMCALLs requires CPU being in post-VMXON state.
-	 */
-	r = kvm_enable_virtualization();
-	if (r)
-		return r;
-
-	cpus_read_lock();
-	r = __do_tdx_bringup();
-	cpus_read_unlock();
-
-	if (r)
-		goto tdx_bringup_err;
-
-	r = -EINVAL;
 	/* Get TDX global information for later use */
 	tdx_sysinfo = tdx_get_sysinfo();
-	if (WARN_ON_ONCE(!tdx_sysinfo))
-		goto get_sysinfo_err;
+	if (!tdx_sysinfo)
+		return -ENODEV;
 
 	/* Check TDX module and KVM capabilities */
 	if (!tdx_get_supported_attrs(&tdx_sysinfo->td_conf) ||
 	    !tdx_get_supported_xfam(&tdx_sysinfo->td_conf))
-		goto get_sysinfo_err;
+		return -EINVAL;
 
 	if (!(tdx_sysinfo->features.tdx_features0 & MD_FIELD_ID_FEATURES0_TOPOLOGY_ENUM))
-		goto get_sysinfo_err;
+		return -EINVAL;
 
 	/*
 	 * TDX has its own limit of maximum vCPUs it can support for all
@@ -3467,35 +3348,16 @@ static int __init __tdx_bringup(void)
 	if (td_conf->max_vcpus_per_td < num_present_cpus()) {
 		pr_err("Disable TDX: MAX_VCPU_PER_TD (%u) smaller than number of logical CPUs (%u).\n",
 				td_conf->max_vcpus_per_td, num_present_cpus());
-		goto get_sysinfo_err;
+		return -EINVAL;
 	}
 
 	if (misc_cg_set_capacity(MISC_CG_RES_TDX, tdx_get_nr_guest_keyids()))
-		goto get_sysinfo_err;
+		return -EINVAL;
 
-	/*
-	 * Leave hardware virtualization enabled after TDX is enabled
-	 * successfully.  TDX CPU hotplug depends on this.
-	 */
 	return 0;
-
-get_sysinfo_err:
-	__tdx_cleanup();
-tdx_bringup_err:
-	kvm_disable_virtualization();
-	return r;
 }
 
-void tdx_cleanup(void)
-{
-	if (enable_tdx) {
-		misc_cg_set_capacity(MISC_CG_RES_TDX, 0);
-		__tdx_cleanup();
-		kvm_disable_virtualization();
-	}
-}
-
-int __init tdx_bringup(void)
+int __init tdx_hardware_setup(void)
 {
 	int r, i;
 
@@ -3526,40 +3388,12 @@ int __init tdx_bringup(void)
 		goto success_disable_tdx;
 	}
 
-	if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
-		pr_err("tdx: MOVDIR64B is required for TDX\n");
-		goto success_disable_tdx;
-	}
-
-	if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) {
-		pr_err("Self-snoop is required for TDX\n");
-		goto success_disable_tdx;
-	}
-
 	if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) {
-		pr_err("tdx: no TDX private KeyIDs available\n");
-		goto success_disable_tdx;
-	}
-
-	if (!enable_virt_at_load) {
-		pr_err("tdx: tdx requires kvm.enable_virt_at_load=1\n");
+		pr_err("TDX not supported by the host platform\n");
 		goto success_disable_tdx;
 	}
 
-	/*
-	 * Ideally KVM should probe whether TDX module has been loaded
-	 * first and then try to bring it up.  But TDX needs to use SEAMCALL
-	 * to probe whether the module is loaded (there is no CPUID or MSR
-	 * for that), and making SEAMCALL requires enabling virtualization
-	 * first, just like the rest steps of bringing up TDX module.
-	 *
-	 * So, for simplicity do everything in __tdx_bringup(); the first
-	 * SEAMCALL will return -ENODEV when the module is not loaded.  The
-	 * only complication is having to make sure that initialization
-	 * SEAMCALLs don't return TDX_SEAMCALL_VMFAILINVALID in other
-	 * cases.
-	 */
-	r = __tdx_bringup();
+	r = __tdx_hardware_setup();
 	if (r) {
 		/*
 		 * Disable TDX only but don't fail to load module if the TDX
@@ -3574,24 +3408,11 @@ int __init tdx_bringup(void)
 		if (r == -ENODEV)
 			goto success_disable_tdx;
 
-		enable_tdx = 0;
+		return r;
 	}
 
-	return r;
-
-success_disable_tdx:
-	enable_tdx = 0;
-	return 0;
-}
-
-void __init tdx_hardware_setup(void)
-{
 	KVM_SANITY_CHECK_VM_STRUCT_SIZE(kvm_tdx);
 
-	/*
-	 * Note, if the TDX module can't be loaded, KVM TDX support will be
-	 * disabled but KVM will continue loading (see tdx_bringup()).
-	 */
 	vt_x86_ops.vm_size = max_t(unsigned int, vt_x86_ops.vm_size, sizeof(struct kvm_tdx));
 
 	vt_x86_ops.link_external_spt = tdx_sept_link_private_spt;
@@ -3599,4 +3420,9 @@ void __init tdx_hardware_setup(void)
 	vt_x86_ops.free_external_spt = tdx_sept_free_private_spt;
 	vt_x86_ops.remove_external_spte = tdx_sept_remove_private_spte;
 	vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;
+	return 0;
+
+success_disable_tdx:
+	enable_tdx = 0;
+	return 0;
 }
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index 45b5183ccb36c..b5cd2ffb303e5 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -8,9 +8,8 @@
 #ifdef CONFIG_KVM_INTEL_TDX
 #include "common.h"
 
-void tdx_hardware_setup(void);
-int tdx_bringup(void);
-void tdx_cleanup(void);
+int tdx_hardware_setup(void);
+void tdx_hardware_unsetup(void);
 
 extern bool enable_tdx;
 
@@ -187,9 +186,6 @@ TDX_BUILD_TDVPS_ACCESSORS(8, MANAGEMENT, management);
 TDX_BUILD_TDVPS_ACCESSORS(64, STATE_NON_ARCH, state_non_arch);
 
 #else
-static inline int tdx_bringup(void) { return 0; }
-static inline void tdx_cleanup(void) {}
-
 #define enable_tdx	0
 
 struct kvm_tdx {
diff --git a/arch/x86/kvm/vmx/tdx_arch.h b/arch/x86/kvm/vmx/tdx_arch.h
index a30e880849e33..350143b9b1451 100644
--- a/arch/x86/kvm/vmx/tdx_arch.h
+++ b/arch/x86/kvm/vmx/tdx_arch.h
@@ -75,12 +75,6 @@ struct tdx_cpuid_value {
 	u32 edx;
 } __packed;
 
-#define TDX_TD_ATTR_DEBUG		BIT_ULL(0)
-#define TDX_TD_ATTR_SEPT_VE_DISABLE	BIT_ULL(28)
-#define TDX_TD_ATTR_PKS			BIT_ULL(30)
-#define TDX_TD_ATTR_KL			BIT_ULL(31)
-#define TDX_TD_ATTR_PERFMON		BIT_ULL(63)
-
 #define TDX_EXT_EXIT_QUAL_TYPE_MASK	GENMASK(3, 0)
 #define TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION  6
 /*
diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h
index 66d747e265b12..1f16ddeae9cb1 100644
--- a/arch/x86/kvm/vmx/vmcs.h
+++ b/arch/x86/kvm/vmx/vmcs.h
@@ -22,17 +22,6 @@
 #define VMCS12_IDX_TO_ENC(idx) ROL16(idx, 10)
 #define ENC_TO_VMCS12_IDX(enc) ROL16(enc, 6)
 
-struct vmcs_hdr {
-	u32 revision_id:31;
-	u32 shadow_vmcs:1;
-};
-
-struct vmcs {
-	struct vmcs_hdr hdr;
-	u32 abort;
-	char data[];
-};
-
 DECLARE_PER_CPU(struct vmcs *, current_vmcs);
 
 /*
diff --git a/arch/x86/kvm/vmx/vmcs_shadow_fields.h b/arch/x86/kvm/vmx/vmcs_shadow_fields.h
index cad128d1657be..67e821c2be6d5 100644
--- a/arch/x86/kvm/vmx/vmcs_shadow_fields.h
+++ b/arch/x86/kvm/vmx/vmcs_shadow_fields.h
@@ -1,6 +1,6 @@
 #if !defined(SHADOW_FIELD_RO) && !defined(SHADOW_FIELD_RW)
-BUILD_BUG_ON(1)
-#endif
+#error Must #define at least one of SHADOW_FIELD_RO or SHADOW_FIELD_RW
+#else
 
 #ifndef SHADOW_FIELD_RO
 #define SHADOW_FIELD_RO(x, y)
@@ -74,6 +74,7 @@ SHADOW_FIELD_RW(HOST_GS_BASE, host_gs_base)
 /* 64-bit */
 SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS, guest_physical_address)
 SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS_HIGH, guest_physical_address)
+#endif
 
 #undef SHADOW_FIELD_RO
 #undef SHADOW_FIELD_RW
diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S
index 4426d34811fce..8a481dae9cae2 100644
--- a/arch/x86/kvm/vmx/vmenter.S
+++ b/arch/x86/kvm/vmx/vmenter.S
@@ -310,7 +310,7 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL)
 	RET
 
 .Lfixup:
-	cmpb $0, _ASM_RIP(kvm_rebooting)
+	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne .Lvmfail
 	ud2
 .Lvmfail:
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 8b24e682535bf..b15b4662b653e 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -48,6 +48,7 @@
 #include <asm/msr.h>
 #include <asm/mwait.h>
 #include <asm/spec-ctrl.h>
+#include <asm/virt.h>
 #include <asm/vmx.h>
 
 #include <trace/events/ipi.h>
@@ -579,7 +580,6 @@ noinline void invept_error(unsigned long ext, u64 eptp)
 	vmx_insn_failed("invept failed: ext=0x%lx eptp=%llx\n", ext, eptp);
 }
 
-static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 /*
  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
@@ -786,53 +786,17 @@ static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx,
 	return ret;
 }
 
-/*
- * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
- *
- * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
- * atomically track post-VMXON state, e.g. this may be called in NMI context.
- * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
- * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
- * magically in RM, VM86, compat mode, or at CPL>0.
- */
-static int kvm_cpu_vmxoff(void)
-{
-	asm goto("1: vmxoff\n\t"
-			  _ASM_EXTABLE(1b, %l[fault])
-			  ::: "cc", "memory" : fault);
-
-	cr4_clear_bits(X86_CR4_VMXE);
-	return 0;
-
-fault:
-	cr4_clear_bits(X86_CR4_VMXE);
-	return -EIO;
-}
-
 void vmx_emergency_disable_virtualization_cpu(void)
 {
 	int cpu = raw_smp_processor_id();
 	struct loaded_vmcs *v;
 
-	kvm_rebooting = true;
-
-	/*
-	 * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be
-	 * set in task context.  If this races with VMX is disabled by an NMI,
-	 * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to
-	 * kvm_rebooting set.
-	 */
-	if (!(__read_cr4() & X86_CR4_VMXE))
-		return;
-
 	list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
 			    loaded_vmcss_on_cpu_link) {
 		vmcs_clear(v->vmcs);
 		if (v->shadow_vmcs)
 			vmcs_clear(v->shadow_vmcs);
 	}
-
-	kvm_cpu_vmxoff();
 }
 
 static void __loaded_vmcs_clear(void *arg)
@@ -2927,12 +2891,16 @@ static bool __kvm_is_vmx_supported(void)
 		return false;
 	}
 
-	if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
-	    !this_cpu_has(X86_FEATURE_VMX)) {
+	if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL)) {
 		pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu);
 		return false;
 	}
 
+	if (!this_cpu_has(X86_FEATURE_VMX)) {
+		pr_err("VMX not fully enabled on CPU %d.  Check kernel logs and/or BIOS\n", cpu);
+		return false;
+	}
+
 	return true;
 }
 
@@ -2984,34 +2952,9 @@ int vmx_check_processor_compat(void)
 	return 0;
 }
 
-static int kvm_cpu_vmxon(u64 vmxon_pointer)
-{
-	u64 msr;
-
-	cr4_set_bits(X86_CR4_VMXE);
-
-	asm goto("1: vmxon %[vmxon_pointer]\n\t"
-			  _ASM_EXTABLE(1b, %l[fault])
-			  : : [vmxon_pointer] "m"(vmxon_pointer)
-			  : : fault);
-	return 0;
-
-fault:
-	WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
-		  rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
-	cr4_clear_bits(X86_CR4_VMXE);
-
-	return -EFAULT;
-}
-
 int vmx_enable_virtualization_cpu(void)
 {
 	int cpu = raw_smp_processor_id();
-	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
-	int r;
-
-	if (cr4_read_shadow() & X86_CR4_VMXE)
-		return -EBUSY;
 
 	/*
 	 * This can happen if we hot-added a CPU but failed to allocate
@@ -3020,15 +2963,7 @@ int vmx_enable_virtualization_cpu(void)
 	if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu))
 		return -EFAULT;
 
-	intel_pt_handle_vmx(1);
-
-	r = kvm_cpu_vmxon(phys_addr);
-	if (r) {
-		intel_pt_handle_vmx(0);
-		return r;
-	}
-
-	return 0;
+	return x86_virt_get_ref(X86_FEATURE_VMX);
 }
 
 static void vmclear_local_loaded_vmcss(void)
@@ -3045,12 +2980,9 @@ void vmx_disable_virtualization_cpu(void)
 {
 	vmclear_local_loaded_vmcss();
 
-	if (kvm_cpu_vmxoff())
-		kvm_spurious_fault();
+	x86_virt_put_ref(X86_FEATURE_VMX);
 
 	hv_reset_evmcs();
-
-	intel_pt_handle_vmx(0);
 }
 
 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
@@ -3128,47 +3060,6 @@ int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
 	return -ENOMEM;
 }
 
-static void free_kvm_area(void)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		free_vmcs(per_cpu(vmxarea, cpu));
-		per_cpu(vmxarea, cpu) = NULL;
-	}
-}
-
-static __init int alloc_kvm_area(void)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		struct vmcs *vmcs;
-
-		vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
-		if (!vmcs) {
-			free_kvm_area();
-			return -ENOMEM;
-		}
-
-		/*
-		 * When eVMCS is enabled, alloc_vmcs_cpu() sets
-		 * vmcs->revision_id to KVM_EVMCS_VERSION instead of
-		 * revision_id reported by MSR_IA32_VMX_BASIC.
-		 *
-		 * However, even though not explicitly documented by
-		 * TLFS, VMXArea passed as VMXON argument should
-		 * still be marked with revision_id reported by
-		 * physical CPU.
-		 */
-		if (kvm_is_using_evmcs())
-			vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic);
-
-		per_cpu(vmxarea, cpu) = vmcs;
-	}
-	return 0;
-}
-
 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
 		struct kvm_segment *save)
 {
@@ -8569,8 +8460,6 @@ void vmx_hardware_unsetup(void)
 
 	if (nested)
 		nested_vmx_hardware_unsetup();
-
-	free_kvm_area();
 }
 
 void vmx_vm_destroy(struct kvm *kvm)
@@ -8698,10 +8587,6 @@ __init int vmx_hardware_setup(void)
 
 	vmx_setup_user_return_msrs();
 
-
-	if (boot_cpu_has(X86_FEATURE_NX))
-		kvm_enable_efer_bits(EFER_NX);
-
 	if (boot_cpu_has(X86_FEATURE_MPX)) {
 		rdmsrq(MSR_IA32_BNDCFGS, host_bndcfgs);
 		WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost");
@@ -8873,10 +8758,6 @@ __init int vmx_hardware_setup(void)
 			return r;
 	}
 
-	r = alloc_kvm_area();
-	if (r)
-		goto err_kvm_area;
-
 	kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
 
 	/*
@@ -8903,11 +8784,6 @@ __init int vmx_hardware_setup(void)
 	kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
 
 	return 0;
-
-err_kvm_area:
-	if (nested)
-		nested_vmx_hardware_unsetup();
-	return r;
 }
 
 void vmx_exit(void)
diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h
index 96677576c836d..81784befaaf49 100644
--- a/arch/x86/kvm/vmx/vmx_ops.h
+++ b/arch/x86/kvm/vmx/vmx_ops.h
@@ -119,7 +119,6 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field)
 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
 
 	asm volatile("1: vmread %[field], %[output]\n\t"
-		     ".byte 0x3e\n\t" /* branch taken hint */
 		     "ja 3f\n\t"
 
 		     /*
@@ -191,7 +190,6 @@ static __always_inline unsigned long vmcs_readl(unsigned long field)
 #define vmx_asm1(insn, op1, error_args...)				\
 do {									\
 	asm goto("1: " __stringify(insn) " %0\n\t"			\
-			  ".byte 0x2e\n\t" /* branch not taken hint */	\
 			  "jna %l[error]\n\t"				\
 			  _ASM_EXTABLE(1b, %l[fault])			\
 			  : : op1 : "cc" : error, fault);		\
@@ -208,7 +206,6 @@ fault:									\
 #define vmx_asm2(insn, op1, op2, error_args...)				\
 do {									\
 	asm goto("1: "  __stringify(insn) " %1, %0\n\t"			\
-			  ".byte 0x2e\n\t" /* branch not taken hint */	\
 			  "jna %l[error]\n\t"				\
 			  _ASM_EXTABLE(1b, %l[fault])			\
 			  : : op1, op2 : "cc" : error, fault);		\
@@ -224,7 +221,7 @@ fault:									\
 
 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
 {
-	vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value);
+	vmx_asm2(vmwrite, "r" (field), ASM_INPUT_RM (value), field, value);
 }
 
 static __always_inline void vmcs_write16(unsigned long field, u16 value)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index fd1c4a36b5936..0b5d48e75b657 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -83,6 +83,8 @@
 #include <asm/intel_pt.h>
 #include <asm/emulate_prefix.h>
 #include <asm/sgx.h>
+#include <asm/virt.h>
+
 #include <clocksource/hyperv_timer.h>
 
 #define CREATE_TRACE_POINTS
@@ -351,6 +353,9 @@ static const u32 msrs_to_save_base[] = {
 	MSR_IA32_U_CET, MSR_IA32_S_CET,
 	MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP,
 	MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB,
+	MSR_IA32_DEBUGCTLMSR,
+	MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP,
+	MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP,
 };
 
 static const u32 msrs_to_save_pmu[] = {
@@ -710,7 +715,7 @@ static void drop_user_return_notifiers(void)
 noinstr void kvm_spurious_fault(void)
 {
 	/* Fault while not rebooting.  We want the trace. */
-	BUG_ON(!kvm_rebooting);
+	BUG_ON(!virt_rebooting);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spurious_fault);
 
@@ -864,9 +869,6 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr,
 		vcpu->arch.exception.error_code = error_code;
 		vcpu->arch.exception.has_payload = has_payload;
 		vcpu->arch.exception.payload = payload;
-		if (!is_guest_mode(vcpu))
-			kvm_deliver_exception_payload(vcpu,
-						      &vcpu->arch.exception);
 		return;
 	}
 
@@ -5531,18 +5533,8 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
-static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
-					       struct kvm_vcpu_events *events)
+static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *vcpu)
 {
-	struct kvm_queued_exception *ex;
-
-	process_nmi(vcpu);
-
-#ifdef CONFIG_KVM_SMM
-	if (kvm_check_request(KVM_REQ_SMI, vcpu))
-		process_smi(vcpu);
-#endif
-
 	/*
 	 * KVM's ABI only allows for one exception to be migrated.  Luckily,
 	 * the only time there can be two queued exceptions is if there's a
@@ -5553,21 +5545,46 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
 	if (vcpu->arch.exception_vmexit.pending &&
 	    !vcpu->arch.exception.pending &&
 	    !vcpu->arch.exception.injected)
-		ex = &vcpu->arch.exception_vmexit;
-	else
-		ex = &vcpu->arch.exception;
+		return &vcpu->arch.exception_vmexit;
+
+	return &vcpu->arch.exception;
+}
+
+static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu)
+{
+	struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu);
 
 	/*
-	 * In guest mode, payload delivery should be deferred if the exception
-	 * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1
-	 * intercepts #PF, ditto for DR6 and #DBs.  If the per-VM capability,
-	 * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not
-	 * propagate the payload and so it cannot be safely deferred.  Deliver
-	 * the payload if the capability hasn't been requested.
+	 * If KVM_CAP_EXCEPTION_PAYLOAD is disabled, then (prematurely) deliver
+	 * the pending exception payload when userspace saves *any* vCPU state
+	 * that interacts with exception payloads to avoid breaking userspace.
+	 *
+	 * Architecturally, KVM must not deliver an exception payload until the
+	 * exception is actually injected, e.g. to avoid losing pending #DB
+	 * information (which VMX tracks in the VMCS), and to avoid clobbering
+	 * state if the exception is never injected for whatever reason.  But
+	 * if KVM_CAP_EXCEPTION_PAYLOAD isn't enabled, then userspace may or
+	 * may not propagate the payload across save+restore, and so KVM can't
+	 * safely defer delivery of the payload.
 	 */
 	if (!vcpu->kvm->arch.exception_payload_enabled &&
 	    ex->pending && ex->has_payload)
 		kvm_deliver_exception_payload(vcpu, ex);
+}
+
+static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
+					       struct kvm_vcpu_events *events)
+{
+	struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu);
+
+	process_nmi(vcpu);
+
+#ifdef CONFIG_KVM_SMM
+	if (kvm_check_request(KVM_REQ_SMI, vcpu))
+		process_smi(vcpu);
+#endif
+
+	kvm_handle_exception_payload_quirk(vcpu);
 
 	memset(events, 0, sizeof(*events));
 
@@ -5746,6 +5763,8 @@ static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
 	    vcpu->arch.guest_state_protected)
 		return -EINVAL;
 
+	kvm_handle_exception_payload_quirk(vcpu);
+
 	memset(dbgregs, 0, sizeof(*dbgregs));
 
 	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db));
@@ -7768,11 +7787,14 @@ static void kvm_init_msr_lists(void)
 }
 
 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
-			   const void *v)
+			   void *__v)
 {
+	const void *v = __v;
 	int handled = 0;
 	int n;
 
+	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, addr, __v);
+
 	do {
 		n = min(len, 8);
 		if (!(lapic_in_kernel(vcpu) &&
@@ -7807,6 +7829,9 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
 		v += n;
 	} while (len);
 
+	if (len)
+		trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len, addr, NULL);
+
 	return handled;
 }
 
@@ -8095,90 +8120,32 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
 }
 
-int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
-			const void *val, int bytes)
-{
-	int ret;
-
-	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
-	if (ret < 0)
-		return 0;
-	kvm_page_track_write(vcpu, gpa, val, bytes);
-	return 1;
-}
-
 struct read_write_emulator_ops {
-	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
-				  int bytes);
-	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
-				  void *val, int bytes);
+	int (*read_write_guest)(struct kvm_vcpu *vcpu, gpa_t gpa,
+				void *val, int bytes);
 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
 			       int bytes, void *val);
-	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
-				    void *val, int bytes);
 	bool write;
 };
 
-static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
-{
-	if (vcpu->mmio_read_completed) {
-		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
-			       vcpu->mmio_fragments[0].gpa, val);
-		vcpu->mmio_read_completed = 0;
-		return 1;
-	}
-
-	return 0;
-}
-
-static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
-			void *val, int bytes)
+static int emulator_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa,
+			       void *val, int bytes)
 {
 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
 }
 
-static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
-			 void *val, int bytes)
-{
-	return emulator_write_phys(vcpu, gpa, val, bytes);
-}
-
-static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
-{
-	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
-	return vcpu_mmio_write(vcpu, gpa, bytes, val);
-}
-
-static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
-			  void *val, int bytes)
+static int emulator_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa,
+				void *val, int bytes)
 {
-	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
-	return X86EMUL_IO_NEEDED;
-}
-
-static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
-			   void *val, int bytes)
-{
-	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
+	int ret;
 
-	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
-	return X86EMUL_CONTINUE;
+	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
+	if (ret < 0)
+		return 0;
+	kvm_page_track_write(vcpu, gpa, val, bytes);
+	return 1;
 }
 
-static const struct read_write_emulator_ops read_emultor = {
-	.read_write_prepare = read_prepare,
-	.read_write_emulate = read_emulate,
-	.read_write_mmio = vcpu_mmio_read,
-	.read_write_exit_mmio = read_exit_mmio,
-};
-
-static const struct read_write_emulator_ops write_emultor = {
-	.read_write_emulate = write_emulate,
-	.read_write_mmio = write_mmio,
-	.read_write_exit_mmio = write_exit_mmio,
-	.write = true,
-};
-
 static int emulator_read_write_onepage(unsigned long addr, void *val,
 				       unsigned int bytes,
 				       struct x86_exception *exception,
@@ -8208,11 +8175,22 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
 			return X86EMUL_PROPAGATE_FAULT;
 	}
 
-	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
+	/*
+	 * If the memory is not _known_ to be emulated MMIO, attempt to access
+	 * guest memory.  If accessing guest memory fails, e.g. because there's
+	 * no memslot, then handle the access as MMIO.  Note, treating the
+	 * access as emulated MMIO is technically wrong if there is a memslot,
+	 * i.e. if accessing host user memory failed, but this has been KVM's
+	 * historical ABI for decades.
+	 */
+	if (!ret && ops->read_write_guest(vcpu, gpa, val, bytes))
 		return X86EMUL_CONTINUE;
 
 	/*
-	 * Is this MMIO handled locally?
+	 * Attempt to handle emulated MMIO within the kernel, e.g. for accesses
+	 * to an in-kernel local or I/O APIC, or to an ioeventfd range attached
+	 * to MMIO bus.  If the access isn't fully resolved, insert an MMIO
+	 * fragment with the relevant details.
 	 */
 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
 	if (handled == bytes)
@@ -8225,8 +8203,21 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
 	frag->gpa = gpa;
-	frag->data = val;
+	if (write && bytes <= 8u) {
+		frag->val = 0;
+		frag->data = &frag->val;
+		memcpy(&frag->val, val, bytes);
+	} else {
+		frag->data = val;
+	}
 	frag->len = bytes;
+
+	/*
+	 * Continue emulating, even though KVM needs to (eventually) do an MMIO
+	 * exit to userspace.  If the access splits multiple pages, then KVM
+	 * needs to exit to userspace only after emulating both parts of the
+	 * access.
+	 */
 	return X86EMUL_CONTINUE;
 }
 
@@ -8237,12 +8228,33 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
 			const struct read_write_emulator_ops *ops)
 {
 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
-	gpa_t gpa;
 	int rc;
 
-	if (ops->read_write_prepare &&
-		  ops->read_write_prepare(vcpu, val, bytes))
+	if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val)))
+		return X86EMUL_UNHANDLEABLE;
+
+	/*
+	 * If the read was already completed via a userspace MMIO exit, there's
+	 * nothing left to do except trace the MMIO read.  When completing MMIO
+	 * reads, KVM re-emulates the instruction to propagate the value into
+	 * the correct destination, e.g. into the correct register, but the
+	 * value itself has already been copied to the read cache.
+	 *
+	 * Note!  This is *tightly* coupled to read_emulated() satisfying reads
+	 * from the emulator's mem_read cache, so that the MMIO fragment data
+	 * is copied to the correct chunk of the correct operand.
+	 */
+	if (!ops->write && vcpu->mmio_read_completed) {
+		/*
+		 * For simplicity, trace the entire MMIO read in one shot, even
+		 * though the GPA might be incorrect if there are two fragments
+		 * that aren't contiguous in the GPA space.
+		 */
+		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
+			       vcpu->mmio_fragments[0].gpa, val);
+		vcpu->mmio_read_completed = 0;
 		return X86EMUL_CONTINUE;
+	}
 
 	vcpu->mmio_nr_fragments = 0;
 
@@ -8271,17 +8283,21 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
 	if (!vcpu->mmio_nr_fragments)
 		return X86EMUL_CONTINUE;
 
-	gpa = vcpu->mmio_fragments[0].gpa;
-
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_cur_fragment = 0;
+	vcpu->mmio_is_write = ops->write;
 
-	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
-	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
-	vcpu->run->exit_reason = KVM_EXIT_MMIO;
-	vcpu->run->mmio.phys_addr = gpa;
+	kvm_prepare_emulated_mmio_exit(vcpu, &vcpu->mmio_fragments[0]);
 
-	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
+	/*
+	 * For MMIO reads, stop emulating and immediately exit to userspace, as
+	 * KVM needs the value to correctly emulate the instruction.  For MMIO
+	 * writes, continue emulating as the write to MMIO is a side effect for
+	 * all intents and purposes.  KVM will still exit to userspace, but
+	 * after completing emulation (see the check on vcpu->mmio_needed in
+	 * x86_emulate_instruction()).
+	 */
+	return ops->write ? X86EMUL_CONTINUE : X86EMUL_IO_NEEDED;
 }
 
 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
@@ -8290,8 +8306,13 @@ static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
 				  unsigned int bytes,
 				  struct x86_exception *exception)
 {
-	return emulator_read_write(ctxt, addr, val, bytes,
-				   exception, &read_emultor);
+	static const struct read_write_emulator_ops ops = {
+		.read_write_guest = emulator_read_guest,
+		.read_write_mmio = vcpu_mmio_read,
+		.write = false,
+	};
+
+	return emulator_read_write(ctxt, addr, val, bytes, exception, &ops);
 }
 
 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
@@ -8300,8 +8321,13 @@ static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
 			    unsigned int bytes,
 			    struct x86_exception *exception)
 {
-	return emulator_read_write(ctxt, addr, (void *)val, bytes,
-				   exception, &write_emultor);
+	static const struct read_write_emulator_ops ops = {
+		.read_write_guest = emulator_write_guest,
+		.read_write_mmio = vcpu_mmio_write,
+		.write = true,
+	};
+
+	return emulator_read_write(ctxt, addr, (void *)val, bytes, exception, &ops);
 }
 
 #define emulator_try_cmpxchg_user(t, ptr, old, new) \
@@ -9694,7 +9720,8 @@ static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
 	unsigned long val;
 
 	/* We should only ever be called with arch.pio.count equal to 1 */
-	BUG_ON(vcpu->arch.pio.count != 1);
+	if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm))
+		return -EIO;
 
 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) {
 		vcpu->arch.pio.count = 0;
@@ -9998,6 +10025,18 @@ void kvm_setup_xss_caps(void)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps);
 
+static void kvm_setup_efer_caps(void)
+{
+	if (kvm_cpu_cap_has(X86_FEATURE_NX))
+		kvm_enable_efer_bits(EFER_NX);
+
+	if (kvm_cpu_cap_has(X86_FEATURE_FXSR_OPT))
+		kvm_enable_efer_bits(EFER_FFXSR);
+
+	if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS))
+		kvm_enable_efer_bits(EFER_AUTOIBRS);
+}
+
 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
 {
 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
@@ -10134,6 +10173,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
 	if (r != 0)
 		goto out_mmu_exit;
 
+	kvm_setup_efer_caps();
+
 	enable_device_posted_irqs &= enable_apicv &&
 				     irq_remapping_cap(IRQ_POSTING_CAP);
 
@@ -10736,12 +10777,10 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
 					     X86_EFLAGS_RF);
 
-		if (vcpu->arch.exception.vector == DB_VECTOR) {
-			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);
-			if (vcpu->arch.dr7 & DR7_GD) {
-				vcpu->arch.dr7 &= ~DR7_GD;
-				kvm_update_dr7(vcpu);
-			}
+		if (vcpu->arch.exception.vector == DB_VECTOR &&
+		    vcpu->arch.dr7 & DR7_GD) {
+			vcpu->arch.dr7 &= ~DR7_GD;
+			kvm_update_dr7(vcpu);
 		}
 
 		kvm_inject_exception(vcpu);
@@ -10973,7 +11012,11 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
 
 	old = new = kvm->arch.apicv_inhibit_reasons;
 
-	set_or_clear_apicv_inhibit(&new, reason, set);
+	if (reason != APICV_INHIBIT_REASON_IRQWIN)
+		set_or_clear_apicv_inhibit(&new, reason, set);
+
+	set_or_clear_apicv_inhibit(&new, APICV_INHIBIT_REASON_IRQWIN,
+				   atomic_read(&kvm->arch.apicv_nr_irq_window_req));
 
 	if (!!old != !!new) {
 		/*
@@ -11014,6 +11057,45 @@ void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_or_clear_apicv_inhibit);
 
+void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc)
+{
+	int add = inc ? 1 : -1;
+
+	if (!enable_apicv)
+		return;
+
+	/*
+	 * IRQ windows are requested either because of ExtINT injections, or
+	 * because APICv is already disabled/inhibited for another reason.
+	 * While ExtINT injections are rare and should not happen while the
+	 * vCPU is running its actual workload, it's worth avoiding thrashing
+	 * if the IRQ window is being requested because APICv is already
+	 * inhibited.  So, toggle the actual inhibit (which requires taking
+	 * the lock for write) if and only if there's no other inhibit.
+	 * kvm_set_or_clear_apicv_inhibit() always evaluates the IRQ window
+	 * count; thus the IRQ window inhibit call _will_ be lazily updated on
+	 * the next call, if it ever happens.
+	 */
+	if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) {
+		guard(rwsem_read)(&kvm->arch.apicv_update_lock);
+		if (READ_ONCE(kvm->arch.apicv_inhibit_reasons) & ~BIT(APICV_INHIBIT_REASON_IRQWIN)) {
+			atomic_add(add, &kvm->arch.apicv_nr_irq_window_req);
+			return;
+		}
+	}
+
+	/*
+	 * Strictly speaking, the lock is only needed if going 0->1 or 1->0,
+	 * a la atomic_dec_and_mutex_lock.  However, ExtINTs are rare and
+	 * only target a single CPU, so that is the common case; do not
+	 * bother eliding the down_write()/up_write() pair.
+	 */
+	guard(rwsem_write)(&kvm->arch.apicv_update_lock);
+	if (atomic_add_return(add, &kvm->arch.apicv_nr_irq_window_req) == inc)
+		__kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_IRQWIN, inc);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inc_or_dec_irq_window_inhibit);
+
 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
 {
 	if (!kvm_apic_present(vcpu))
@@ -11804,7 +11886,8 @@ static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
 
 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
 {
-	BUG_ON(!vcpu->arch.pio.count);
+	if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm))
+		return -EIO;
 
 	return complete_emulated_io(vcpu);
 }
@@ -11833,7 +11916,8 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
 	struct kvm_mmio_fragment *frag;
 	unsigned len;
 
-	BUG_ON(!vcpu->mmio_needed);
+	if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
+		return -EIO;
 
 	/* Complete previous fragment */
 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
@@ -11846,6 +11930,9 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
 		frag++;
 		vcpu->mmio_cur_fragment++;
 	} else {
+		if (WARN_ON_ONCE(frag->data == &frag->val))
+			return -EIO;
+
 		/* Go forward to the next mmio piece. */
 		frag->data += len;
 		frag->gpa += len;
@@ -11862,12 +11949,7 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
 		return complete_emulated_io(vcpu);
 	}
 
-	run->exit_reason = KVM_EXIT_MMIO;
-	run->mmio.phys_addr = frag->gpa;
-	if (vcpu->mmio_is_write)
-		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
-	run->mmio.len = min(8u, frag->len);
-	run->mmio.is_write = vcpu->mmio_is_write;
+	kvm_prepare_emulated_mmio_exit(vcpu, frag);
 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
 	return 0;
 }
@@ -12136,6 +12218,8 @@ static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 	if (vcpu->arch.guest_state_protected)
 		goto skip_protected_regs;
 
+	kvm_handle_exception_payload_quirk(vcpu);
+
 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
@@ -12529,7 +12613,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 
 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
 		r = -EBUSY;
-		if (kvm_is_exception_pending(vcpu))
+		if (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected)
 			goto out;
 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
 			kvm_queue_exception(vcpu, DB_VECTOR);
@@ -13073,12 +13157,12 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_deliver_sipi_vector);
 
 void kvm_arch_enable_virtualization(void)
 {
-	cpu_emergency_register_virt_callback(kvm_x86_ops.emergency_disable_virtualization_cpu);
+	x86_virt_register_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu);
 }
 
 void kvm_arch_disable_virtualization(void)
 {
-	cpu_emergency_unregister_virt_callback(kvm_x86_ops.emergency_disable_virtualization_cpu);
+	x86_virt_unregister_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu);
 }
 
 int kvm_arch_enable_virtualization_cpu(void)
@@ -13177,6 +13261,25 @@ int kvm_arch_enable_virtualization_cpu(void)
 	return 0;
 }
 
+void kvm_arch_shutdown(void)
+{
+	/*
+	 * Set virt_rebooting to indicate that KVM has asynchronously disabled
+	 * hardware virtualization, i.e. that errors and/or exceptions on SVM
+	 * and VMX instructions are expected and should be ignored.
+	 */
+	virt_rebooting = true;
+
+	/*
+	 * Ensure virt_rebooting is visible before IPIs are sent to other CPUs
+	 * to disable virtualization.  Effectively pairs with the reception of
+	 * the IPI (virt_rebooting is read in task/exception context, but only
+	 * _needs_ to be read as %true after the IPI function callback disables
+	 * virtualization).
+	 */
+	smp_wmb();
+}
+
 void kvm_arch_disable_virtualization_cpu(void)
 {
 	kvm_x86_call(disable_virtualization_cpu)();
@@ -13191,7 +13294,7 @@ void kvm_arch_disable_virtualization_cpu(void)
 	 * disable virtualization arrives.  Handle the extreme edge case here
 	 * instead of trying to account for it in the normal flows.
 	 */
-	if (in_task() || WARN_ON_ONCE(!kvm_rebooting))
+	if (in_task() || WARN_ON_ONCE(!virt_rebooting))
 		drop_user_return_notifiers();
 	else
 		__module_get(THIS_MODULE);
@@ -14243,7 +14346,8 @@ static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
 	struct kvm_mmio_fragment *frag;
 	unsigned int len;
 
-	BUG_ON(!vcpu->mmio_needed);
+	if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
+		return -EIO;
 
 	/* Complete previous fragment */
 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
@@ -14265,73 +14369,32 @@ static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
 		vcpu->mmio_needed = 0;
 
-		// VMG change, at this point, we're always done
-		// RIP has already been advanced
+		/*
+		 * All done, as frag->data always points at the GHCB scratch
+		 * area and VMGEXIT is trap-like (RIP is advanced by hardware).
+		 */
 		return 1;
 	}
 
 	// More MMIO is needed
-	run->mmio.phys_addr = frag->gpa;
-	run->mmio.len = min(8u, frag->len);
-	run->mmio.is_write = vcpu->mmio_is_write;
-	if (run->mmio.is_write)
-		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
-	run->exit_reason = KVM_EXIT_MMIO;
-
+	kvm_prepare_emulated_mmio_exit(vcpu, frag);
 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
-
 	return 0;
 }
 
-int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
-			  void *data)
+int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
+		    unsigned int bytes, void *data)
 {
-	int handled;
 	struct kvm_mmio_fragment *frag;
-
-	if (!data)
-		return -EINVAL;
-
-	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
-	if (handled == bytes)
-		return 1;
-
-	bytes -= handled;
-	gpa += handled;
-	data += handled;
-
-	/*TODO: Check if need to increment number of frags */
-	frag = vcpu->mmio_fragments;
-	vcpu->mmio_nr_fragments = 1;
-	frag->len = bytes;
-	frag->gpa = gpa;
-	frag->data = data;
-
-	vcpu->mmio_needed = 1;
-	vcpu->mmio_cur_fragment = 0;
-
-	vcpu->run->mmio.phys_addr = gpa;
-	vcpu->run->mmio.len = min(8u, frag->len);
-	vcpu->run->mmio.is_write = 1;
-	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
-	vcpu->run->exit_reason = KVM_EXIT_MMIO;
-
-	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
-
-	return 0;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_write);
-
-int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
-			 void *data)
-{
 	int handled;
-	struct kvm_mmio_fragment *frag;
 
-	if (!data)
+	if (!data || WARN_ON_ONCE(object_is_on_stack(data)))
 		return -EINVAL;
 
-	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
+	if (is_write)
+		handled = vcpu_mmio_write(vcpu, gpa, bytes, data);
+	else
+		handled = vcpu_mmio_read(vcpu, gpa, bytes, data);
 	if (handled == bytes)
 		return 1;
 
@@ -14339,26 +14402,25 @@ int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
 	gpa += handled;
 	data += handled;
 
-	/*TODO: Check if need to increment number of frags */
+	/*
+	 * TODO: Determine whether or not userspace plays nice with MMIO
+	 *       requests that split a page boundary.
+	 */
 	frag = vcpu->mmio_fragments;
-	vcpu->mmio_nr_fragments = 1;
 	frag->len = bytes;
 	frag->gpa = gpa;
 	frag->data = data;
 
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_cur_fragment = 0;
+	vcpu->mmio_nr_fragments = 1;
+	vcpu->mmio_is_write = is_write;
 
-	vcpu->run->mmio.phys_addr = gpa;
-	vcpu->run->mmio.len = min(8u, frag->len);
-	vcpu->run->mmio.is_write = 0;
-	vcpu->run->exit_reason = KVM_EXIT_MMIO;
-
+	kvm_prepare_emulated_mmio_exit(vcpu, frag);
 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
-
 	return 0;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_read);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio);
 
 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
 {
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 94d4f07aaaa09..44a28d343d407 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -712,14 +712,38 @@ static inline bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 	__reserved_bits;                                \
 })
 
-int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes,
-			  void *dst);
-int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes,
-			 void *dst);
+int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
+		    unsigned int bytes, void *data);
 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
 			 unsigned int port, void *data,  unsigned int count,
 			 int in);
 
+static inline void __kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu,
+						    gpa_t gpa, unsigned int len,
+						    const void *data,
+						    bool is_write)
+{
+	struct kvm_run *run = vcpu->run;
+
+	KVM_BUG_ON(len > 8, vcpu->kvm);
+
+	run->mmio.len = len;
+	run->mmio.is_write = is_write;
+	run->exit_reason = KVM_EXIT_MMIO;
+	run->mmio.phys_addr = gpa;
+	if (is_write)
+		memcpy(run->mmio.data, data, len);
+}
+
+static inline void kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu,
+						  struct kvm_mmio_fragment *frag)
+{
+	WARN_ON_ONCE(!vcpu->mmio_needed || !vcpu->mmio_nr_fragments);
+
+	__kvm_prepare_emulated_mmio_exit(vcpu, frag->gpa, min(8u, frag->len),
+					 frag->data, vcpu->mmio_is_write);
+}
+
 static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr)
 {
 	return kvm->arch.hypercall_exit_enabled & BIT(hc_nr);
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index b83a06739b511..f0e77e0844820 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1314,7 +1314,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 	 * to consider the PF_PK bit.
 	 */
 	if (is_vsyscall_vaddr(address)) {
-		if (emulate_vsyscall(error_code, regs, address))
+		if (emulate_vsyscall_pf(error_code, regs, address))
 			return;
 	}
 #endif
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index 40581a720fe82..cba907c397186 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -1119,9 +1119,10 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 
 static int
 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
-		   struct page *base)
+		   struct ptdesc *ptdesc)
 {
 	unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
+	struct page *base = ptdesc_page(ptdesc);
 	pte_t *pbase = (pte_t *)page_address(base);
 	unsigned int i, level;
 	pgprot_t ref_prot;
@@ -1226,18 +1227,18 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
 			    unsigned long address)
 {
-	struct page *base;
+	struct ptdesc *ptdesc;
 
 	if (!debug_pagealloc_enabled())
 		spin_unlock(&cpa_lock);
-	base = alloc_pages(GFP_KERNEL, 0);
+	ptdesc = pagetable_alloc(GFP_KERNEL, 0);
 	if (!debug_pagealloc_enabled())
 		spin_lock(&cpa_lock);
-	if (!base)
+	if (!ptdesc)
 		return -ENOMEM;
 
-	if (__split_large_page(cpa, kpte, address, base))
-		__free_page(base);
+	if (__split_large_page(cpa, kpte, address, ptdesc))
+		pagetable_free(ptdesc);
 
 	return 0;
 }
@@ -1408,7 +1409,7 @@ static bool try_to_free_pte_page(pte_t *pte)
 		if (!pte_none(pte[i]))
 			return false;
 
-	free_page((unsigned long)pte);
+	pte_free_kernel(&init_mm, pte);
 	return true;
 }
 
@@ -1420,7 +1421,7 @@ static bool try_to_free_pmd_page(pmd_t *pmd)
 		if (!pmd_none(pmd[i]))
 			return false;
 
-	free_page((unsigned long)pmd);
+	pmd_free(&init_mm, pmd);
 	return true;
 }
 
@@ -1539,7 +1540,7 @@ static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
 
 static int alloc_pte_page(pmd_t *pmd)
 {
-	pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
+	pte_t *pte = pte_alloc_one_kernel(&init_mm);
 	if (!pte)
 		return -1;
 
@@ -1549,7 +1550,11 @@ static int alloc_pte_page(pmd_t *pmd)
 
 static int alloc_pmd_page(pud_t *pud)
 {
-	pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
+	/*
+	 * Pass 0 as a placeholder for the second argument, since the
+	 * generic implementation of pmd_alloc_one() does not use it.
+	 */
+	pmd_t *pmd = pmd_alloc_one(&init_mm, 0);
 	if (!pmd)
 		return -1;
 
@@ -1743,7 +1748,11 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
 	pgd_entry = cpa->pgd + pgd_index(addr);
 
 	if (pgd_none(*pgd_entry)) {
-		p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
+		/*
+		 * Pass 0 as a placeholder for the second argument, since the
+		 * generic implementation of p4d_alloc_one() does not use it.
+		 */
+		p4d = p4d_alloc_one(&init_mm, 0);
 		if (!p4d)
 			return -1;
 
@@ -1755,7 +1764,11 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
 	 */
 	p4d = p4d_offset(pgd_entry, addr);
 	if (p4d_none(*p4d)) {
-		pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
+		/*
+		 * Pass 0 as a placeholder for the second argument, since the
+		 * generic implementation of pud_alloc_one() does not use it.
+		 */
+		pud = pud_alloc_one(&init_mm, 0);
 		if (!pud)
 			return -1;
 
diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c
index 7418c367e3281..1dfcfaf77e23c 100644
--- a/arch/x86/mm/pkeys.c
+++ b/arch/x86/mm/pkeys.c
@@ -42,8 +42,7 @@ int __execute_only_pkey(struct mm_struct *mm)
 	 * Set up PKRU so that it denies access for everything
 	 * other than execution.
 	 */
-	ret = arch_set_user_pkey_access(current, execute_only_pkey,
-			PKEY_DISABLE_ACCESS);
+	ret = arch_set_user_pkey_access(execute_only_pkey, PKEY_DISABLE_ACCESS);
 	/*
 	 * If the PKRU-set operation failed somehow, just return
 	 * 0 and effectively disable execute-only support.
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 621e09d049cb9..af43d177087e7 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -971,27 +971,6 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
 	}
 }
 
-/*
- * Please ignore the name of this function.  It should be called
- * switch_to_kernel_thread().
- *
- * enter_lazy_tlb() is a hint from the scheduler that we are entering a
- * kernel thread or other context without an mm.  Acceptable implementations
- * include doing nothing whatsoever, switching to init_mm, or various clever
- * lazy tricks to try to minimize TLB flushes.
- *
- * The scheduler reserves the right to call enter_lazy_tlb() several times
- * in a row.  It will notify us that we're going back to a real mm by
- * calling switch_mm_irqs_off().
- */
-void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
-{
-	if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)
-		return;
-
-	this_cpu_write(cpu_tlbstate_shared.is_lazy, true);
-}
-
 /*
  * Using a temporary mm allows to set temporary mappings that are not accessible
  * by other CPUs. Such mappings are needed to perform sensitive memory writes
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index d84c6020dda12..0c39adb96b912 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -920,7 +920,7 @@ umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n)
 	return attr->mode;
 }
 
-enum efi_secureboot_mode __x86_ima_efi_boot_mode(void)
+enum efi_secureboot_mode __x86_efi_boot_mode(void)
 {
 	return boot_params.secure_boot;
 }
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index b4409df2105a6..5861008eab222 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -55,6 +55,7 @@
  */
 static u64 efi_va = EFI_VA_START;
 static struct mm_struct *efi_prev_mm;
+static unsigned long efi_cr4_lass;
 
 /*
  * We need our own copy of the higher levels of the page tables
@@ -443,16 +444,50 @@ static void efi_leave_mm(void)
 	unuse_temporary_mm(efi_prev_mm);
 }
 
+/*
+ * Toggle LASS to allow EFI to access any 1:1 mapped region in the lower
+ * half.
+ *
+ * Disable LASS only after switching to EFI-mm, as userspace is not
+ * mapped in it. Similar to EFI-mm, these rely on preemption being
+ * disabled and the calls being serialized.
+ */
+
+static void efi_disable_lass(void)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_LASS))
+		return;
+
+	lockdep_assert_preemption_disabled();
+
+	/* Save current CR4.LASS state */
+	efi_cr4_lass = cr4_read_shadow() & X86_CR4_LASS;
+	cr4_clear_bits(efi_cr4_lass);
+}
+
+static void efi_enable_lass(void)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_LASS))
+		return;
+
+	lockdep_assert_preemption_disabled();
+
+	/* Reprogram CR4.LASS only if it was set earlier */
+	cr4_set_bits(efi_cr4_lass);
+}
+
 void arch_efi_call_virt_setup(void)
 {
 	efi_sync_low_kernel_mappings();
 	efi_fpu_begin();
 	firmware_restrict_branch_speculation_start();
 	efi_enter_mm();
+	efi_disable_lass();
 }
 
 void arch_efi_call_virt_teardown(void)
 {
+	efi_enable_lass();
 	efi_leave_mm();
 	firmware_restrict_branch_speculation_end();
 	efi_fpu_end();
diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c
index a595953f1d6d8..e72d26acae797 100644
--- a/arch/x86/power/hibernate_64.c
+++ b/arch/x86/power/hibernate_64.c
@@ -14,8 +14,6 @@
 #include <linux/kdebug.h>
 #include <linux/pgtable.h>
 
-#include <crypto/hash.h>
-
 #include <asm/e820/api.h>
 #include <asm/init.h>
 #include <asm/proto.h>
diff --git a/arch/x86/um/asm/vm-flags.h b/arch/x86/um/asm/vm-flags.h
index df7a3896f5dd7..622d36d6ddff6 100644
--- a/arch/x86/um/asm/vm-flags.h
+++ b/arch/x86/um/asm/vm-flags.h
@@ -9,11 +9,11 @@
 
 #ifdef CONFIG_X86_32
 
-#define VM_DATA_DEFAULT_FLAGS	VM_DATA_FLAGS_TSK_EXEC
+#define VMA_DATA_DEFAULT_FLAGS	VMA_DATA_FLAGS_TSK_EXEC
 
 #else
 
-#define VM_STACK_DEFAULT_FLAGS (VM_GROWSDOWN | VM_DATA_FLAGS_EXEC)
+#define VMA_STACK_DEFAULT_FLAGS append_vma_flags(VMA_DATA_FLAGS_EXEC, VMA_GROWSDOWN_BIT)
 
 #endif
 #endif
diff --git a/arch/x86/virt/Makefile b/arch/x86/virt/Makefile
index ea343fc392dcc..6e485751650cb 100644
--- a/arch/x86/virt/Makefile
+++ b/arch/x86/virt/Makefile
@@ -1,2 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-y	+= svm/ vmx/
+
+obj-$(subst m,y,$(CONFIG_KVM_X86)) += hw.o
\ No newline at end of file
diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
new file mode 100644
index 0000000000000..c898f16fe6126
--- /dev/null
+++ b/arch/x86/virt/hw.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/errno.h>
+#include <linux/kvm_types.h>
+#include <linux/list.h>
+#include <linux/percpu.h>
+
+#include <asm/perf_event.h>
+#include <asm/processor.h>
+#include <asm/virt.h>
+#include <asm/vmx.h>
+
+struct x86_virt_ops {
+	int feature;
+	int (*enable_virtualization_cpu)(void);
+	int (*disable_virtualization_cpu)(void);
+	void (*emergency_disable_virtualization_cpu)(void);
+};
+static struct x86_virt_ops virt_ops __ro_after_init;
+
+__visible bool virt_rebooting;
+EXPORT_SYMBOL_FOR_KVM(virt_rebooting);
+
+static DEFINE_PER_CPU(int, virtualization_nr_users);
+
+static cpu_emergency_virt_cb __rcu *kvm_emergency_callback;
+
+void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback)
+{
+	if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback)))
+		return;
+
+	rcu_assign_pointer(kvm_emergency_callback, callback);
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_register_emergency_callback);
+
+void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback)
+{
+	if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback) != callback))
+		return;
+
+	rcu_assign_pointer(kvm_emergency_callback, NULL);
+	synchronize_rcu();
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_unregister_emergency_callback);
+
+static void x86_virt_invoke_kvm_emergency_callback(void)
+{
+	cpu_emergency_virt_cb *kvm_callback;
+
+	kvm_callback = rcu_dereference(kvm_emergency_callback);
+	if (kvm_callback)
+		kvm_callback();
+}
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+static DEFINE_PER_CPU(struct vmcs *, root_vmcs);
+
+static int x86_virt_cpu_vmxon(void)
+{
+	u64 vmxon_pointer = __pa(per_cpu(root_vmcs, raw_smp_processor_id()));
+	u64 msr;
+
+	cr4_set_bits(X86_CR4_VMXE);
+
+	asm goto("1: vmxon %[vmxon_pointer]\n\t"
+			  _ASM_EXTABLE(1b, %l[fault])
+			  : : [vmxon_pointer] "m"(vmxon_pointer)
+			  : : fault);
+	return 0;
+
+fault:
+	WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
+		  rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
+	cr4_clear_bits(X86_CR4_VMXE);
+
+	return -EFAULT;
+}
+
+static int x86_vmx_enable_virtualization_cpu(void)
+{
+	int r;
+
+	if (cr4_read_shadow() & X86_CR4_VMXE)
+		return -EBUSY;
+
+	intel_pt_handle_vmx(1);
+
+	r = x86_virt_cpu_vmxon();
+	if (r) {
+		intel_pt_handle_vmx(0);
+		return r;
+	}
+
+	return 0;
+}
+
+/*
+ * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
+ *
+ * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
+ * atomically track post-VMXON state, e.g. this may be called in NMI context.
+ * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
+ * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
+ * magically in RM, VM86, compat mode, or at CPL>0.
+ */
+static int x86_vmx_disable_virtualization_cpu(void)
+{
+	int r = -EIO;
+
+	asm goto("1: vmxoff\n\t"
+		 _ASM_EXTABLE(1b, %l[fault])
+		 ::: "cc", "memory" : fault);
+	r = 0;
+
+fault:
+	cr4_clear_bits(X86_CR4_VMXE);
+	intel_pt_handle_vmx(0);
+	return r;
+}
+
+static void x86_vmx_emergency_disable_virtualization_cpu(void)
+{
+	virt_rebooting = true;
+
+	/*
+	 * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be
+	 * set in task context.  If this races with _another_ emergency call
+	 * from NMI context, VMCLEAR (in KVM) and VMXOFF may #UD, but KVM and
+	 * the kernel will eat those faults due to virt_rebooting being set by
+	 * the interrupting NMI callback.
+	 */
+	if (!(__read_cr4() & X86_CR4_VMXE))
+		return;
+
+	x86_virt_invoke_kvm_emergency_callback();
+
+	x86_vmx_disable_virtualization_cpu();
+}
+
+static __init void x86_vmx_exit(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		free_page((unsigned long)per_cpu(root_vmcs, cpu));
+		per_cpu(root_vmcs, cpu) = NULL;
+	}
+}
+
+static __init int __x86_vmx_init(void)
+{
+	const struct x86_virt_ops vmx_ops = {
+		.feature = X86_FEATURE_VMX,
+		.enable_virtualization_cpu = x86_vmx_enable_virtualization_cpu,
+		.disable_virtualization_cpu = x86_vmx_disable_virtualization_cpu,
+		.emergency_disable_virtualization_cpu = x86_vmx_emergency_disable_virtualization_cpu,
+	};
+
+	u64 basic_msr;
+	u32 rev_id;
+	int cpu;
+
+	if (!cpu_feature_enabled(X86_FEATURE_VMX))
+		return -EOPNOTSUPP;
+
+	rdmsrq(MSR_IA32_VMX_BASIC, basic_msr);
+
+	/* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
+	if (WARN_ON_ONCE(vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE))
+		return -EIO;
+
+	/*
+	 * Even if eVMCS is enabled (or will be enabled?), and even though not
+	 * explicitly documented by TLFS, the root VMCS  passed to VMXON should
+	 * still be marked with the revision_id reported by the physical CPU.
+	 */
+	rev_id = vmx_basic_vmcs_revision_id(basic_msr);
+
+	for_each_possible_cpu(cpu) {
+		int node = cpu_to_node(cpu);
+		struct page *page;
+		struct vmcs *vmcs;
+
+		page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
+		if (WARN_ON_ONCE(!page)) {
+			x86_vmx_exit();
+			return -ENOMEM;
+		}
+
+		vmcs = page_address(page);
+		vmcs->hdr.revision_id = rev_id;
+		per_cpu(root_vmcs, cpu) = vmcs;
+	}
+
+	memcpy(&virt_ops, &vmx_ops, sizeof(virt_ops));
+	return 0;
+}
+
+static __init int x86_vmx_init(void)
+{
+	int r;
+
+	r = __x86_vmx_init();
+	if (r)
+		setup_clear_cpu_cap(X86_FEATURE_VMX);
+	return r;
+}
+#else
+static __init int x86_vmx_init(void) { return -EOPNOTSUPP; }
+static __init void x86_vmx_exit(void) { }
+#endif
+
+#if IS_ENABLED(CONFIG_KVM_AMD)
+static int x86_svm_enable_virtualization_cpu(void)
+{
+	u64 efer;
+
+	rdmsrq(MSR_EFER, efer);
+	if (efer & EFER_SVME)
+		return -EBUSY;
+
+	wrmsrq(MSR_EFER, efer | EFER_SVME);
+	return 0;
+}
+
+static int x86_svm_disable_virtualization_cpu(void)
+{
+	int r = -EIO;
+	u64 efer;
+
+	/*
+	 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
+	 * NMI aren't blocked.
+	 */
+	asm goto("1: stgi\n\t"
+		 _ASM_EXTABLE(1b, %l[fault])
+		 ::: "memory" : fault);
+	r = 0;
+
+fault:
+	rdmsrq(MSR_EFER, efer);
+	wrmsrq(MSR_EFER, efer & ~EFER_SVME);
+	return r;
+}
+
+static void x86_svm_emergency_disable_virtualization_cpu(void)
+{
+	u64 efer;
+
+	virt_rebooting = true;
+
+	rdmsrq(MSR_EFER, efer);
+	if (!(efer & EFER_SVME))
+		return;
+
+	x86_virt_invoke_kvm_emergency_callback();
+
+	x86_svm_disable_virtualization_cpu();
+}
+
+static __init int x86_svm_init(void)
+{
+	const struct x86_virt_ops svm_ops = {
+		.feature = X86_FEATURE_SVM,
+		.enable_virtualization_cpu = x86_svm_enable_virtualization_cpu,
+		.disable_virtualization_cpu = x86_svm_disable_virtualization_cpu,
+		.emergency_disable_virtualization_cpu = x86_svm_emergency_disable_virtualization_cpu,
+	};
+
+	if (!cpu_feature_enabled(X86_FEATURE_SVM))
+		return -EOPNOTSUPP;
+
+	memcpy(&virt_ops, &svm_ops, sizeof(virt_ops));
+	return 0;
+}
+#else
+static __init int x86_svm_init(void) { return -EOPNOTSUPP; }
+#endif
+
+int x86_virt_get_ref(int feat)
+{
+	int r;
+
+	/* Ensure the !feature check can't get false positives. */
+	BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX);
+
+	if (!virt_ops.feature || virt_ops.feature != feat)
+		return -EOPNOTSUPP;
+
+	guard(preempt)();
+
+	if (this_cpu_inc_return(virtualization_nr_users) > 1)
+		return 0;
+
+	r = virt_ops.enable_virtualization_cpu();
+	if (r)
+		WARN_ON_ONCE(this_cpu_dec_return(virtualization_nr_users));
+
+	return r;
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_get_ref);
+
+void x86_virt_put_ref(int feat)
+{
+	guard(preempt)();
+
+	if (WARN_ON_ONCE(!this_cpu_read(virtualization_nr_users)) ||
+	    this_cpu_dec_return(virtualization_nr_users))
+		return;
+
+	BUG_ON(virt_ops.disable_virtualization_cpu() && !virt_rebooting);
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_put_ref);
+
+/*
+ * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during
+ * reboot.  VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if
+ * GIF=0, i.e. if the crash occurred between CLGI and STGI.
+ */
+int x86_virt_emergency_disable_virtualization_cpu(void)
+{
+	if (!virt_ops.feature)
+		return -EOPNOTSUPP;
+
+	/*
+	 * IRQs must be disabled as virtualization is enabled in hardware via
+	 * function call IPIs, i.e. IRQs need to be disabled to guarantee
+	 * virtualization stays disabled.
+	 */
+	lockdep_assert_irqs_disabled();
+
+	/*
+	 * Do the NMI shootdown even if virtualization is off on _this_ CPU, as
+	 * other CPUs may have virtualization enabled.
+	 *
+	 * TODO: Track whether or not virtualization might be enabled on other
+	 *	 CPUs?  May not be worth avoiding the NMI shootdown...
+	 */
+	virt_ops.emergency_disable_virtualization_cpu();
+	return 0;
+}
+
+void __init x86_virt_init(void)
+{
+	/*
+	 * Attempt to initialize both SVM and VMX, and simply use whichever one
+	 * is present.  Rsefuse to enable/use SVM or VMX if both are somehow
+	 * supported.  No known CPU supports both SVM and VMX.
+	 */
+	bool has_vmx = !x86_vmx_init();
+	bool has_svm = !x86_svm_init();
+
+	if (WARN_ON_ONCE(has_vmx && has_svm)) {
+		x86_vmx_exit();
+		memset(&virt_ops, 0, sizeof(virt_ops));
+	}
+}
diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index a4f3a364fb65d..e35fac0a8a3d8 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -117,6 +117,8 @@ static u64 rmp_segment_mask;
 
 static u64 rmp_cfg;
 
+static void *rmp_bookkeeping __ro_after_init;
+
 /* Mask to apply to a PFN to get the first PFN of a 2MB page */
 #define PFN_PMD_MASK	GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT)
 
@@ -130,33 +132,20 @@ static unsigned long snp_nr_leaked_pages;
 #undef pr_fmt
 #define pr_fmt(fmt)	"SEV-SNP: " fmt
 
-static int __mfd_enable(unsigned int cpu)
+static __init void mfd_enable(void *arg)
 {
-	u64 val;
-
 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
-		return 0;
-
-	rdmsrq(MSR_AMD64_SYSCFG, val);
-
-	val |= MSR_AMD64_SYSCFG_MFDM;
-
-	wrmsrq(MSR_AMD64_SYSCFG, val);
-
-	return 0;
-}
+		return;
 
-static __init void mfd_enable(void *arg)
-{
-	__mfd_enable(smp_processor_id());
+	msr_set_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT);
 }
 
-static int __snp_enable(unsigned int cpu)
+static __init void snp_enable(void *arg)
 {
 	u64 val;
 
 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
-		return 0;
+		return;
 
 	rdmsrq(MSR_AMD64_SYSCFG, val);
 
@@ -164,13 +153,6 @@ static int __snp_enable(unsigned int cpu)
 	val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN;
 
 	wrmsrq(MSR_AMD64_SYSCFG, val);
-
-	return 0;
-}
-
-static __init void snp_enable(void *arg)
-{
-	__snp_enable(smp_processor_id());
 }
 
 static void __init __snp_fixup_e820_tables(u64 pa)
@@ -260,23 +242,6 @@ void __init snp_fixup_e820_tables(void)
 	}
 }
 
-static bool __init clear_rmptable_bookkeeping(void)
-{
-	void *bk;
-
-	bk = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB);
-	if (!bk) {
-		pr_err("Failed to map RMP bookkeeping area\n");
-		return false;
-	}
-
-	memset(bk, 0, RMPTABLE_CPU_BOOKKEEPING_SZ);
-
-	memunmap(bk);
-
-	return true;
-}
-
 static bool __init alloc_rmp_segment_desc(u64 segment_pa, u64 segment_size, u64 pa)
 {
 	u64 rst_index, rmp_segment_size_max;
@@ -494,10 +459,22 @@ static bool __init setup_segmented_rmptable(void)
 static bool __init setup_rmptable(void)
 {
 	if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) {
-		return setup_segmented_rmptable();
+		if (!setup_segmented_rmptable())
+			return false;
 	} else {
-		return setup_contiguous_rmptable();
+		if (!setup_contiguous_rmptable())
+			return false;
+	}
+
+	rmp_bookkeeping = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB);
+	if (!rmp_bookkeeping) {
+		pr_err("Failed to map RMP bookkeeping area\n");
+		free_rmp_segment_table();
+
+		return false;
 	}
+
+	return true;
 }
 
 /*
@@ -528,10 +505,7 @@ int __init snp_rmptable_init(void)
 		goto skip_enable;
 
 	/* Zero out the RMP bookkeeping area */
-	if (!clear_rmptable_bookkeeping()) {
-		free_rmp_segment_table();
-		return -ENOSYS;
-	}
+	memset(rmp_bookkeeping, 0, RMPTABLE_CPU_BOOKKEEPING_SZ);
 
 	/* Zero out the RMP entries */
 	for (i = 0; i < rst_max_index; i++) {
@@ -544,17 +518,12 @@ int __init snp_rmptable_init(void)
 		memset(desc->rmp_entry, 0, desc->size);
 	}
 
-	/* Flush the caches to ensure that data is written before SNP is enabled. */
-	wbinvd_on_all_cpus();
-
 	/* MtrrFixDramModEn must be enabled on all the CPUs prior to enabling SNP. */
 	on_each_cpu(mfd_enable, NULL, 1);
 
 	on_each_cpu(snp_enable, NULL, 1);
 
 skip_enable:
-	cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/rmptable_init:online", __snp_enable, NULL);
-
 	/*
 	 * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic
 	 * notifier is invoked to do SNP IOMMU shutdown before kdump.
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 8b8e165a2001a..cb9b3210ab710 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -28,6 +28,7 @@
 #include <linux/log2.h>
 #include <linux/acpi.h>
 #include <linux/suspend.h>
+#include <linux/syscore_ops.h>
 #include <linux/idr.h>
 #include <linux/kvm_types.h>
 #include <asm/page.h>
@@ -39,6 +40,7 @@
 #include <asm/cpu_device_id.h>
 #include <asm/processor.h>
 #include <asm/mce.h>
+#include <asm/virt.h>
 #include "tdx.h"
 
 static u32 tdx_global_keyid __ro_after_init;
@@ -51,13 +53,11 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);
 
 static struct tdmr_info_list tdx_tdmr_list;
 
-static enum tdx_module_status_t tdx_module_status;
-static DEFINE_MUTEX(tdx_module_lock);
-
 /* All TDX-usable memory regions.  Protected by mem_hotplug_lock. */
 static LIST_HEAD(tdx_memlist);
 
-static struct tdx_sys_info tdx_sysinfo;
+static struct tdx_sys_info tdx_sysinfo __ro_after_init;
+static bool tdx_module_initialized __ro_after_init;
 
 typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args);
 
@@ -106,8 +106,7 @@ static __always_inline int sc_retry_prerr(sc_func_t func,
 
 /*
  * Do the module global initialization once and return its result.
- * It can be done on any cpu.  It's always called with interrupts
- * disabled.
+ * It can be done on any cpu, and from task or IRQ context.
  */
 static int try_init_module_global(void)
 {
@@ -116,8 +115,6 @@ static int try_init_module_global(void)
 	static bool sysinit_done;
 	static int sysinit_ret;
 
-	lockdep_assert_irqs_disabled();
-
 	raw_spin_lock(&sysinit_lock);
 
 	if (sysinit_done)
@@ -142,26 +139,15 @@ static int try_init_module_global(void)
 }
 
 /**
- * tdx_cpu_enable - Enable TDX on local cpu
- *
- * Do one-time TDX module per-cpu initialization SEAMCALL (and TDX module
- * global initialization SEAMCALL if not done) on local cpu to make this
- * cpu be ready to run any other SEAMCALLs.
- *
- * Always call this function via IPI function calls.
- *
- * Return 0 on success, otherwise errors.
+ * Enable VMXON and then do one-time TDX module per-cpu initialization SEAMCALL
+ * (and TDX module global initialization SEAMCALL if not done) on local cpu to
+ * make this cpu be ready to run any other SEAMCALLs.
  */
-int tdx_cpu_enable(void)
+static int tdx_cpu_enable(void)
 {
 	struct tdx_module_args args = {};
 	int ret;
 
-	if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM))
-		return -ENODEV;
-
-	lockdep_assert_irqs_disabled();
-
 	if (__this_cpu_read(tdx_lp_initialized))
 		return 0;
 
@@ -182,15 +168,101 @@ int tdx_cpu_enable(void)
 
 	return 0;
 }
-EXPORT_SYMBOL_FOR_KVM(tdx_cpu_enable);
+
+static int tdx_online_cpu(unsigned int cpu)
+{
+	int ret;
+
+	ret = x86_virt_get_ref(X86_FEATURE_VMX);
+	if (ret)
+		return ret;
+
+	ret = tdx_cpu_enable();
+	if (ret)
+		x86_virt_put_ref(X86_FEATURE_VMX);
+
+	return ret;
+}
+
+static int tdx_offline_cpu(unsigned int cpu)
+{
+	int i;
+
+	/* No TD is running.  Allow any cpu to be offline. */
+	if (ida_is_empty(&tdx_guest_keyid_pool))
+		goto done;
+
+	/*
+	 * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to
+	 * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory
+	 * controller with pconfig.  If we have active TDX HKID, refuse to
+	 * offline the last online cpu.
+	 */
+	for_each_online_cpu(i) {
+		/*
+		 * Found another online cpu on the same package.
+		 * Allow to offline.
+		 */
+		if (i != cpu && topology_physical_package_id(i) ==
+				topology_physical_package_id(cpu))
+			goto done;
+	}
+
+	/*
+	 * This is the last cpu of this package.  Don't offline it.
+	 *
+	 * Because it's hard for human operator to understand the
+	 * reason, warn it.
+	 */
+#define MSG_ALLPKG_ONLINE \
+	"TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n"
+	pr_warn_ratelimited(MSG_ALLPKG_ONLINE);
+	return -EBUSY;
+
+done:
+	x86_virt_put_ref(X86_FEATURE_VMX);
+	return 0;
+}
+
+static void tdx_shutdown_cpu(void *ign)
+{
+	x86_virt_put_ref(X86_FEATURE_VMX);
+}
+
+static void tdx_shutdown(void *ign)
+{
+	on_each_cpu(tdx_shutdown_cpu, NULL, 1);
+}
+
+static int tdx_suspend(void *ign)
+{
+	x86_virt_put_ref(X86_FEATURE_VMX);
+	return 0;
+}
+
+static void tdx_resume(void *ign)
+{
+	WARN_ON_ONCE(x86_virt_get_ref(X86_FEATURE_VMX));
+}
+
+static const struct syscore_ops tdx_syscore_ops = {
+	.suspend = tdx_suspend,
+	.resume = tdx_resume,
+	.shutdown = tdx_shutdown,
+};
+
+static struct syscore tdx_syscore = {
+	.ops = &tdx_syscore_ops,
+};
 
 /*
  * Add a memory region as a TDX memory block.  The caller must make sure
  * all memory regions are added in address ascending order and don't
  * overlap.
  */
-static int add_tdx_memblock(struct list_head *tmb_list, unsigned long start_pfn,
-			    unsigned long end_pfn, int nid)
+static __init int add_tdx_memblock(struct list_head *tmb_list,
+				   unsigned long start_pfn,
+				   unsigned long end_pfn, int nid)
 {
 	struct tdx_memblock *tmb;
 
@@ -208,7 +280,7 @@ static int add_tdx_memblock(struct list_head *tmb_list, unsigned long start_pfn,
 	return 0;
 }
 
-static void free_tdx_memlist(struct list_head *tmb_list)
+static __init void free_tdx_memlist(struct list_head *tmb_list)
 {
 	/* @tmb_list is protected by mem_hotplug_lock */
 	while (!list_empty(tmb_list)) {
@@ -226,7 +298,7 @@ static void free_tdx_memlist(struct list_head *tmb_list)
  * ranges off in a secondary structure because memblock is modified
  * in memory hotplug while TDX memory regions are fixed.
  */
-static int build_tdx_memlist(struct list_head *tmb_list)
+static __init int build_tdx_memlist(struct list_head *tmb_list)
 {
 	unsigned long start_pfn, end_pfn;
 	int i, nid, ret;
@@ -258,7 +330,7 @@ static int build_tdx_memlist(struct list_head *tmb_list)
 	return ret;
 }
 
-static int read_sys_metadata_field(u64 field_id, u64 *data)
+static __init int read_sys_metadata_field(u64 field_id, u64 *data)
 {
 	struct tdx_module_args args = {};
 	int ret;
@@ -280,7 +352,7 @@ static int read_sys_metadata_field(u64 field_id, u64 *data)
 
 #include "tdx_global_metadata.c"
 
-static int check_features(struct tdx_sys_info *sysinfo)
+static __init int check_features(struct tdx_sys_info *sysinfo)
 {
 	u64 tdx_features0 = sysinfo->features.tdx_features0;
 
@@ -293,7 +365,7 @@ static int check_features(struct tdx_sys_info *sysinfo)
 }
 
 /* Calculate the actual TDMR size */
-static int tdmr_size_single(u16 max_reserved_per_tdmr)
+static __init int tdmr_size_single(u16 max_reserved_per_tdmr)
 {
 	int tdmr_sz;
 
@@ -307,8 +379,8 @@ static int tdmr_size_single(u16 max_reserved_per_tdmr)
 	return ALIGN(tdmr_sz, TDMR_INFO_ALIGNMENT);
 }
 
-static int alloc_tdmr_list(struct tdmr_info_list *tdmr_list,
-			   struct tdx_sys_info_tdmr *sysinfo_tdmr)
+static __init int alloc_tdmr_list(struct tdmr_info_list *tdmr_list,
+				  struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
 	size_t tdmr_sz, tdmr_array_sz;
 	void *tdmr_array;
@@ -339,7 +411,7 @@ static int alloc_tdmr_list(struct tdmr_info_list *tdmr_list,
 	return 0;
 }
 
-static void free_tdmr_list(struct tdmr_info_list *tdmr_list)
+static __init void free_tdmr_list(struct tdmr_info_list *tdmr_list)
 {
 	free_pages_exact(tdmr_list->tdmrs,
 			tdmr_list->max_tdmrs * tdmr_list->tdmr_sz);
@@ -368,8 +440,8 @@ static inline u64 tdmr_end(struct tdmr_info *tdmr)
  * preallocated @tdmr_list, following all the special alignment
  * and size rules for TDMR.
  */
-static int fill_out_tdmrs(struct list_head *tmb_list,
-			  struct tdmr_info_list *tdmr_list)
+static __init int fill_out_tdmrs(struct list_head *tmb_list,
+				 struct tdmr_info_list *tdmr_list)
 {
 	struct tdx_memblock *tmb;
 	int tdmr_idx = 0;
@@ -445,8 +517,8 @@ static int fill_out_tdmrs(struct list_head *tmb_list,
  * Calculate PAMT size given a TDMR and a page size.  The returned
  * PAMT size is always aligned up to 4K page boundary.
  */
-static unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
-				      u16 pamt_entry_size)
+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
+					     u16 pamt_entry_size)
 {
 	unsigned long pamt_sz, nr_pamt_entries;
 
@@ -477,7 +549,7 @@ static unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
  * PAMT.  This node will have some memory covered by the TDMR.  The
  * relative amount of memory covered is not considered.
  */
-static int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list)
+static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list)
 {
 	struct tdx_memblock *tmb;
 
@@ -506,9 +578,9 @@ static int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list)
  * Allocate PAMTs from the local NUMA node of some memory in @tmb_list
  * within @tdmr, and set up PAMTs for @tdmr.
  */
-static int tdmr_set_up_pamt(struct tdmr_info *tdmr,
-			    struct list_head *tmb_list,
-			    u16 pamt_entry_size[])
+static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
+				   struct list_head *tmb_list,
+				   u16 pamt_entry_size[])
 {
 	unsigned long pamt_base[TDX_PS_NR];
 	unsigned long pamt_size[TDX_PS_NR];
@@ -578,7 +650,7 @@ static void tdmr_get_pamt(struct tdmr_info *tdmr, unsigned long *pamt_base,
 	*pamt_size = pamt_sz;
 }
 
-static void tdmr_do_pamt_func(struct tdmr_info *tdmr,
+static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr,
 		void (*pamt_func)(unsigned long base, unsigned long size))
 {
 	unsigned long pamt_base, pamt_size;
@@ -595,17 +667,17 @@ static void tdmr_do_pamt_func(struct tdmr_info *tdmr,
 	pamt_func(pamt_base, pamt_size);
 }
 
-static void free_pamt(unsigned long pamt_base, unsigned long pamt_size)
+static __init void free_pamt(unsigned long pamt_base, unsigned long pamt_size)
 {
 	free_contig_range(pamt_base >> PAGE_SHIFT, pamt_size >> PAGE_SHIFT);
 }
 
-static void tdmr_free_pamt(struct tdmr_info *tdmr)
+static __init void tdmr_free_pamt(struct tdmr_info *tdmr)
 {
 	tdmr_do_pamt_func(tdmr, free_pamt);
 }
 
-static void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
+static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
 {
 	int i;
 
@@ -614,9 +686,9 @@ static void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
 }
 
 /* Allocate and set up PAMTs for all TDMRs */
-static int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,
-				 struct list_head *tmb_list,
-				 u16 pamt_entry_size[])
+static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,
+					struct list_head *tmb_list,
+					u16 pamt_entry_size[])
 {
 	int i, ret = 0;
 
@@ -665,12 +737,13 @@ void tdx_quirk_reset_page(struct page *page)
 }
 EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_page);
 
-static void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr)
+static __init void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr)
+
 {
 	tdmr_do_pamt_func(tdmr, tdx_quirk_reset_paddr);
 }
 
-static void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list)
+static __init void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list)
 {
 	int i;
 
@@ -678,7 +751,7 @@ static void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list)
 		tdmr_quirk_reset_pamt(tdmr_entry(tdmr_list, i));
 }
 
-static unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list)
+static __init unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list)
 {
 	unsigned long pamt_size = 0;
 	int i;
@@ -693,8 +766,8 @@ static unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list)
 	return pamt_size / 1024;
 }
 
-static int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, u64 addr,
-			      u64 size, u16 max_reserved_per_tdmr)
+static __init int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx,
+				     u64 addr, u64 size, u16 max_reserved_per_tdmr)
 {
 	struct tdmr_reserved_area *rsvd_areas = tdmr->reserved_areas;
 	int idx = *p_idx;
@@ -727,10 +800,10 @@ static int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, u64 addr,
  * those holes fall within @tdmr, set up a TDMR reserved area to cover
  * the hole.
  */
-static int tdmr_populate_rsvd_holes(struct list_head *tmb_list,
-				    struct tdmr_info *tdmr,
-				    int *rsvd_idx,
-				    u16 max_reserved_per_tdmr)
+static __init int tdmr_populate_rsvd_holes(struct list_head *tmb_list,
+					   struct tdmr_info *tdmr,
+					   int *rsvd_idx,
+					   u16 max_reserved_per_tdmr)
 {
 	struct tdx_memblock *tmb;
 	u64 prev_end;
@@ -791,10 +864,10 @@ static int tdmr_populate_rsvd_holes(struct list_head *tmb_list,
  * overlaps with @tdmr, set up a TDMR reserved area to cover the
  * overlapping part.
  */
-static int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list,
-				    struct tdmr_info *tdmr,
-				    int *rsvd_idx,
-				    u16 max_reserved_per_tdmr)
+static __init int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list,
+					   struct tdmr_info *tdmr,
+					   int *rsvd_idx,
+					   u16 max_reserved_per_tdmr)
 {
 	int i, ret;
 
@@ -829,7 +902,7 @@ static int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list,
 }
 
 /* Compare function called by sort() for TDMR reserved areas */
-static int rsvd_area_cmp_func(const void *a, const void *b)
+static __init int rsvd_area_cmp_func(const void *a, const void *b)
 {
 	struct tdmr_reserved_area *r1 = (struct tdmr_reserved_area *)a;
 	struct tdmr_reserved_area *r2 = (struct tdmr_reserved_area *)b;
@@ -848,10 +921,10 @@ static int rsvd_area_cmp_func(const void *a, const void *b)
  * Populate reserved areas for the given @tdmr, including memory holes
  * (via @tmb_list) and PAMTs (via @tdmr_list).
  */
-static int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr,
-				    struct list_head *tmb_list,
-				    struct tdmr_info_list *tdmr_list,
-				    u16 max_reserved_per_tdmr)
+static __init int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr,
+					   struct list_head *tmb_list,
+					   struct tdmr_info_list *tdmr_list,
+					   u16 max_reserved_per_tdmr)
 {
 	int ret, rsvd_idx = 0;
 
@@ -876,9 +949,9 @@ static int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr,
  * Populate reserved areas for all TDMRs in @tdmr_list, including memory
  * holes (via @tmb_list) and PAMTs.
  */
-static int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list,
-					 struct list_head *tmb_list,
-					 u16 max_reserved_per_tdmr)
+static __init int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list,
+						struct list_head *tmb_list,
+						u16 max_reserved_per_tdmr)
 {
 	int i;
 
@@ -899,9 +972,9 @@ static int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list,
  * to cover all TDX memory regions in @tmb_list based on the TDX module
  * TDMR global information in @sysinfo_tdmr.
  */
-static int construct_tdmrs(struct list_head *tmb_list,
-			   struct tdmr_info_list *tdmr_list,
-			   struct tdx_sys_info_tdmr *sysinfo_tdmr)
+static __init int construct_tdmrs(struct list_head *tmb_list,
+				  struct tdmr_info_list *tdmr_list,
+				  struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
 	u16 pamt_entry_size[TDX_PS_NR] = {
 		sysinfo_tdmr->pamt_4k_entry_size,
@@ -933,7 +1006,8 @@ static int construct_tdmrs(struct list_head *tmb_list,
 	return ret;
 }
 
-static int config_tdx_module(struct tdmr_info_list *tdmr_list, u64 global_keyid)
+static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
+				    u64 global_keyid)
 {
 	struct tdx_module_args args = {};
 	u64 *tdmr_pa_array;
@@ -968,7 +1042,7 @@ static int config_tdx_module(struct tdmr_info_list *tdmr_list, u64 global_keyid)
 	return ret;
 }
 
-static int do_global_key_config(void *unused)
+static __init int do_global_key_config(void *unused)
 {
 	struct tdx_module_args args = {};
 
@@ -986,7 +1060,7 @@ static int do_global_key_config(void *unused)
  * KVM) can ensure success by ensuring sufficient CPUs are online and
  * can run SEAMCALLs.
  */
-static int config_global_keyid(void)
+static __init int config_global_keyid(void)
 {
 	cpumask_var_t packages;
 	int cpu, ret = -EINVAL;
@@ -1026,7 +1100,7 @@ static int config_global_keyid(void)
 	return ret;
 }
 
-static int init_tdmr(struct tdmr_info *tdmr)
+static __init int init_tdmr(struct tdmr_info *tdmr)
 {
 	u64 next;
 
@@ -1057,7 +1131,7 @@ static int init_tdmr(struct tdmr_info *tdmr)
 	return 0;
 }
 
-static int init_tdmrs(struct tdmr_info_list *tdmr_list)
+static __init int init_tdmrs(struct tdmr_info_list *tdmr_list)
 {
 	int i;
 
@@ -1076,7 +1150,7 @@ static int init_tdmrs(struct tdmr_info_list *tdmr_list)
 	return 0;
 }
 
-static int init_tdx_module(void)
+static __init int init_tdx_module(void)
 {
 	int ret;
 
@@ -1157,67 +1231,50 @@ static int init_tdx_module(void)
 	goto out_put_tdxmem;
 }
 
-static int __tdx_enable(void)
+static __init int tdx_enable(void)
 {
+	enum cpuhp_state state;
 	int ret;
 
-	ret = init_tdx_module();
-	if (ret) {
-		pr_err("module initialization failed (%d)\n", ret);
-		tdx_module_status = TDX_MODULE_ERROR;
-		return ret;
+	if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) {
+		pr_err("TDX not supported by the host platform\n");
+		return -ENODEV;
 	}
 
-	pr_info("module initialized\n");
-	tdx_module_status = TDX_MODULE_INITIALIZED;
-
-	return 0;
-}
+	if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) {
+		pr_err("XSAVE is required for TDX\n");
+		return -EINVAL;
+	}
 
-/**
- * tdx_enable - Enable TDX module to make it ready to run TDX guests
- *
- * This function assumes the caller has: 1) held read lock of CPU hotplug
- * lock to prevent any new cpu from becoming online; 2) done both VMXON
- * and tdx_cpu_enable() on all online cpus.
- *
- * This function requires there's at least one online cpu for each CPU
- * package to succeed.
- *
- * This function can be called in parallel by multiple callers.
- *
- * Return 0 if TDX is enabled successfully, otherwise error.
- */
-int tdx_enable(void)
-{
-	int ret;
+	if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
+		pr_err("MOVDIR64B is required for TDX\n");
+		return -EINVAL;
+	}
 
-	if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM))
+	if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) {
+		pr_err("Self-snoop is required for TDX\n");
 		return -ENODEV;
+	}
 
-	lockdep_assert_cpus_held();
-
-	mutex_lock(&tdx_module_lock);
+	state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "virt/tdx:online",
+				  tdx_online_cpu, tdx_offline_cpu);
+	if (state < 0)
+		return state;
 
-	switch (tdx_module_status) {
-	case TDX_MODULE_UNINITIALIZED:
-		ret = __tdx_enable();
-		break;
-	case TDX_MODULE_INITIALIZED:
-		/* Already initialized, great, tell the caller. */
-		ret = 0;
-		break;
-	default:
-		/* Failed to initialize in the previous attempts */
-		ret = -EINVAL;
-		break;
+	ret = init_tdx_module();
+	if (ret) {
+		pr_err("TDX-Module initialization failed (%d)\n", ret);
+		cpuhp_remove_state(state);
+		return ret;
 	}
 
-	mutex_unlock(&tdx_module_lock);
+	register_syscore(&tdx_syscore);
 
-	return ret;
+	tdx_module_initialized = true;
+	pr_info("TDX-Module initialized\n");
+	return 0;
 }
-EXPORT_SYMBOL_FOR_KVM(tdx_enable);
+subsys_initcall(tdx_enable);
 
 static bool is_pamt_page(unsigned long phys)
 {
@@ -1468,15 +1525,10 @@ void __init tdx_init(void)
 
 const struct tdx_sys_info *tdx_get_sysinfo(void)
 {
-	const struct tdx_sys_info *p = NULL;
-
-	/* Make sure all fields in @tdx_sysinfo have been populated */
-	mutex_lock(&tdx_module_lock);
-	if (tdx_module_status == TDX_MODULE_INITIALIZED)
-		p = (const struct tdx_sys_info *)&tdx_sysinfo;
-	mutex_unlock(&tdx_module_lock);
+	if (!tdx_module_initialized)
+		return NULL;
 
-	return p;
+	return (const struct tdx_sys_info *)&tdx_sysinfo;
 }
 EXPORT_SYMBOL_FOR_KVM(tdx_get_sysinfo);
 
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index 82bb82be8567f..dde219c823b41 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -91,14 +91,6 @@ struct tdmr_info {
  * Do not put any hardware-defined TDX structure representations below
  * this comment!
  */
-
-/* Kernel defined TDX module status during module initialization. */
-enum tdx_module_status_t {
-	TDX_MODULE_UNINITIALIZED,
-	TDX_MODULE_INITIALIZED,
-	TDX_MODULE_ERROR
-};
-
 struct tdx_memblock {
 	struct list_head list;
 	unsigned long start_pfn;
diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
index 13ad2663488b1..c7db393a9cfb1 100644
--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
@@ -7,7 +7,22 @@
  * Include this file to other C file instead.
  */
 
-static int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_features)
+static __init int get_tdx_sys_info_version(struct tdx_sys_info_version *sysinfo_version)
+{
+	int ret = 0;
+	u64 val;
+
+	if (!ret && !(ret = read_sys_metadata_field(0x0800000100000003, &val)))
+		sysinfo_version->minor_version = val;
+	if (!ret && !(ret = read_sys_metadata_field(0x0800000100000004, &val)))
+		sysinfo_version->major_version = val;
+	if (!ret && !(ret = read_sys_metadata_field(0x0800000100000005, &val)))
+		sysinfo_version->update_version = val;
+
+	return ret;
+}
+
+static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_features)
 {
 	int ret = 0;
 	u64 val;
@@ -18,7 +33,7 @@ static int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_featu
 	return ret;
 }
 
-static int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
+static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
 	int ret = 0;
 	u64 val;
@@ -37,7 +52,7 @@ static int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
 	return ret;
 }
 
-static int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl)
+static __init int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl)
 {
 	int ret = 0;
 	u64 val;
@@ -52,7 +67,7 @@ static int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl
 	return ret;
 }
 
-static int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf)
+static __init int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf)
 {
 	int ret = 0;
 	u64 val;
@@ -85,10 +100,17 @@ static int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf
 	return ret;
 }
 
-static int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
+static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
 {
 	int ret = 0;
 
+	ret = ret ?: get_tdx_sys_info_version(&sysinfo->version);
+
+	pr_info("Module version: %u.%u.%02u\n",
+		sysinfo->version.major_version,
+		sysinfo->version.minor_version,
+		sysinfo->version.update_version);
+
 	ret = ret ?: get_tdx_sys_info_features(&sysinfo->features);
 	ret = ret ?: get_tdx_sys_info_tdmr(&sysinfo->tdmr);
 	ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl);
diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
index 50a136213b2bf..61f07d981a94d 100644
--- a/arch/xtensa/include/asm/pgtable.h
+++ b/arch/xtensa/include/asm/pgtable.h
@@ -209,10 +209,6 @@
 #define pgd_ERROR(e) \
 	printk("%s:%d: bad pgd entry %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 
-extern unsigned long empty_zero_page[1024];
-
-#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
-
 #ifdef CONFIG_MMU
 extern pgd_t swapper_pg_dir[PAGE_SIZE/sizeof(pgd_t)];
 extern void paging_init(void);
diff --git a/arch/xtensa/include/asm/ucontext.h b/arch/xtensa/include/asm/ucontext.h
deleted file mode 100644
index 94c94ed3e00a1..0000000000000
--- a/arch/xtensa/include/asm/ucontext.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * include/asm-xtensa/ucontext.h
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2001 - 2005 Tensilica Inc.
- */
-
-#ifndef _XTENSA_UCONTEXT_H
-#define _XTENSA_UCONTEXT_H
-
-struct ucontext {
-	unsigned long	  uc_flags;
-	struct ucontext  *uc_link;
-	stack_t		  uc_stack;
-	struct sigcontext uc_mcontext;
-	sigset_t	  uc_sigmask;	/* mask last for extensibility */
-};
-
-#endif /* _XTENSA_UCONTEXT_H */
diff --git a/arch/xtensa/include/uapi/asm/Kbuild b/arch/xtensa/include/uapi/asm/Kbuild
index b97c552db3c5a..da95c5a0301f8 100644
--- a/arch/xtensa/include/uapi/asm/Kbuild
+++ b/arch/xtensa/include/uapi/asm/Kbuild
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 generated-y += unistd_32.h
+generic-y += ucontext.h
diff --git a/arch/xtensa/kernel/head.S b/arch/xtensa/kernel/head.S
index 8484294bc623f..4b0c5c5e685af 100644
--- a/arch/xtensa/kernel/head.S
+++ b/arch/xtensa/kernel/head.S
@@ -381,6 +381,3 @@ ENTRY(swapper_pg_dir)
 	.fill	PAGE_SIZE, 1, 0
 END(swapper_pg_dir)
 #endif
-ENTRY(empty_zero_page)
-	.fill	PAGE_SIZE, 1, 0
-END(empty_zero_page)
diff --git a/arch/xtensa/kernel/xtensa_ksyms.c b/arch/xtensa/kernel/xtensa_ksyms.c
index 62d81e76e18e2..ced335b4df5fa 100644
--- a/arch/xtensa/kernel/xtensa_ksyms.c
+++ b/arch/xtensa/kernel/xtensa_ksyms.c
@@ -15,8 +15,6 @@
 #include <linux/module.h>
 #include <asm/pgtable.h>
 
-EXPORT_SYMBOL(empty_zero_page);
-
 unsigned int __sync_fetch_and_and_4(volatile void *p, unsigned int v)
 {
 	BUG();
diff --git a/arch/xtensa/platforms/iss/setup.c b/arch/xtensa/platforms/iss/setup.c
index 0f1fe132691e2..21283acab1a89 100644
--- a/arch/xtensa/platforms/iss/setup.c
+++ b/arch/xtensa/platforms/iss/setup.c
@@ -32,8 +32,7 @@ static int iss_power_off(struct sys_off_data *unused)
 	return NOTIFY_DONE;
 }
 
-static int iss_restart(struct notifier_block *this,
-		       unsigned long event, void *ptr)
+static int iss_restart(struct sys_off_data *unused)
 {
 	/* Flush and reset the mmu, simulate a processor reset, and
 	 * jump to the reset vector. */
@@ -42,10 +41,6 @@ static int iss_restart(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block iss_restart_block = {
-	.notifier_call = iss_restart,
-};
-
 static int
 iss_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
 {
@@ -84,7 +79,9 @@ void __init platform_setup(char **p_cmdline)
 	}
 
 	atomic_notifier_chain_register(&panic_notifier_list, &iss_panic_block);
-	register_restart_handler(&iss_restart_block);
+	register_sys_off_handler(SYS_OFF_MODE_RESTART,
+				 SYS_OFF_PRIO_PLATFORM,
+				 iss_restart, NULL);
 	register_sys_off_handler(SYS_OFF_MODE_POWER_OFF,
 				 SYS_OFF_PRIO_PLATFORM,
 				 iss_power_off, NULL);
diff --git a/arch/xtensa/platforms/xt2000/setup.c b/arch/xtensa/platforms/xt2000/setup.c
index 258e01a51fd88..eda4c15c68264 100644
--- a/arch/xtensa/platforms/xt2000/setup.c
+++ b/arch/xtensa/platforms/xt2000/setup.c
@@ -50,8 +50,7 @@ static int xt2000_power_off(struct sys_off_data *unused)
 	return NOTIFY_DONE;
 }
 
-static int xt2000_restart(struct notifier_block *this,
-			  unsigned long event, void *ptr)
+static int xt2000_restart(struct sys_off_data *unused)
 {
 	/* Flush and reset the mmu, simulate a processor reset, and
 	 * jump to the reset vector. */
@@ -60,10 +59,6 @@ static int xt2000_restart(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block xt2000_restart_block = {
-	.notifier_call = xt2000_restart,
-};
-
 void __init platform_setup(char** cmdline)
 {
 	led_print (0, "LINUX   ");
@@ -140,7 +135,9 @@ static int __init xt2000_setup_devinit(void)
 	platform_device_register(&xt2000_serial8250_device);
 	platform_device_register(&xt2000_sonic_device);
 	mod_timer(&heartbeat_timer, jiffies + HZ / 2);
-	register_restart_handler(&xt2000_restart_block);
+	register_sys_off_handler(SYS_OFF_MODE_RESTART,
+				 SYS_OFF_PRIO_PLATFORM,
+				 xt2000_restart, NULL);
 	register_sys_off_handler(SYS_OFF_MODE_POWER_OFF,
 				 SYS_OFF_PRIO_DEFAULT,
 				 xt2000_power_off, NULL);
diff --git a/arch/xtensa/platforms/xtfpga/setup.c b/arch/xtensa/platforms/xtfpga/setup.c
index a2432f0817101..67d7f37f1802e 100644
--- a/arch/xtensa/platforms/xtfpga/setup.c
+++ b/arch/xtensa/platforms/xtfpga/setup.c
@@ -42,8 +42,7 @@ static int xtfpga_power_off(struct sys_off_data *unused)
 	return NOTIFY_DONE;
 }
 
-static int xtfpga_restart(struct notifier_block *this,
-			  unsigned long event, void *ptr)
+static int xtfpga_restart(struct sys_off_data *unused)
 {
 	/* Try software reset first. */
 	WRITE_ONCE(*(u32 *)XTFPGA_SWRST_VADDR, 0xdead);
@@ -56,10 +55,6 @@ static int xtfpga_restart(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block xtfpga_restart_block = {
-	.notifier_call = xtfpga_restart,
-};
-
 #ifdef CONFIG_XTENSA_CALIBRATE_CCOUNT
 
 void __init platform_calibrate_ccount(void)
@@ -71,7 +66,9 @@ void __init platform_calibrate_ccount(void)
 
 static void __init xtfpga_register_handlers(void)
 {
-	register_restart_handler(&xtfpga_restart_block);
+	register_sys_off_handler(SYS_OFF_MODE_RESTART,
+				 SYS_OFF_PRIO_PLATFORM,
+				 xtfpga_restart, NULL);
 	register_sys_off_handler(SYS_OFF_MODE_POWER_OFF,
 				 SYS_OFF_PRIO_DEFAULT,
 				 xtfpga_power_off, NULL);
diff --git a/block/Makefile b/block/Makefile
index c65f4da937026..7dce2e44276c4 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -26,7 +26,7 @@ bfq-y				:= bfq-iosched.o bfq-wf2q.o bfq-cgroup.o
 obj-$(CONFIG_IOSCHED_BFQ)	+= bfq.o
 
 obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o \
-				   bio-integrity-auto.o
+				   bio-integrity-auto.o bio-integrity-fs.o
 obj-$(CONFIG_BLK_DEV_ZONED)	+= blk-zoned.o
 obj-$(CONFIG_BLK_WBT)		+= blk-wbt.o
 obj-$(CONFIG_BLK_DEBUG_FS)	+= blk-mq-debugfs.o
diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c
index 44dcdf7520c56..ebd17f47e0f9e 100644
--- a/block/bio-integrity-auto.c
+++ b/block/bio-integrity-auto.c
@@ -39,7 +39,7 @@ static void bio_integrity_verify_fn(struct work_struct *work)
 		container_of(work, struct bio_integrity_data, work);
 	struct bio *bio = bid->bio;
 
-	blk_integrity_verify_iter(bio, &bid->saved_bio_iter);
+	bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter);
 	bio_integrity_finish(bid);
 	bio_endio(bio);
 }
@@ -50,11 +50,6 @@ static bool bip_should_check(struct bio_integrity_payload *bip)
 	return bip->bip_flags & BIP_CHECK_FLAGS;
 }
 
-static bool bi_offload_capable(struct blk_integrity *bi)
-{
-	return bi->metadata_size == bi->pi_tuple_size;
-}
-
 /**
  * __bio_integrity_endio - Integrity I/O completion function
  * @bio:	Protected bio
@@ -84,83 +79,30 @@ bool __bio_integrity_endio(struct bio *bio)
 /**
  * bio_integrity_prep - Prepare bio for integrity I/O
  * @bio:	bio to prepare
+ * @action:	preparation action needed (BI_ACT_*)
  *
- * Checks if the bio already has an integrity payload attached.  If it does, the
- * payload has been generated by another kernel subsystem, and we just pass it
- * through.
- * Otherwise allocates integrity payload and for writes the integrity metadata
- * will be generated.  For reads, the completion handler will verify the
- * metadata.
+ * Allocate the integrity payload.  For writes, generate the integrity metadata
+ * and for reads, setup the completion handler to verify the metadata.
+ *
+ * This is used for bios that do not have user integrity payloads attached.
  */
-bool bio_integrity_prep(struct bio *bio)
+void bio_integrity_prep(struct bio *bio, unsigned int action)
 {
-	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 	struct bio_integrity_data *bid;
-	bool set_flags = true;
-	gfp_t gfp = GFP_NOIO;
-
-	if (!bi)
-		return true;
-
-	if (!bio_sectors(bio))
-		return true;
-
-	/* Already protected? */
-	if (bio_integrity(bio))
-		return true;
-
-	switch (bio_op(bio)) {
-	case REQ_OP_READ:
-		if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
-			if (bi_offload_capable(bi))
-				return true;
-			set_flags = false;
-		}
-		break;
-	case REQ_OP_WRITE:
-		/*
-		 * Zero the memory allocated to not leak uninitialized kernel
-		 * memory to disk for non-integrity metadata where nothing else
-		 * initializes the memory.
-		 */
-		if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
-			if (bi_offload_capable(bi))
-				return true;
-			set_flags = false;
-			gfp |= __GFP_ZERO;
-		} else if (bi->metadata_size > bi->pi_tuple_size)
-			gfp |= __GFP_ZERO;
-		break;
-	default:
-		return true;
-	}
-
-	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
-		return true;
 
 	bid = mempool_alloc(&bid_pool, GFP_NOIO);
 	bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
 	bid->bio = bio;
 	bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
-	bio_integrity_alloc_buf(bio, gfp & __GFP_ZERO);
-
-	bip_set_seed(&bid->bip, bio->bi_iter.bi_sector);
-
-	if (set_flags) {
-		if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
-			bid->bip.bip_flags |= BIP_IP_CHECKSUM;
-		if (bi->csum_type)
-			bid->bip.bip_flags |= BIP_CHECK_GUARD;
-		if (bi->flags & BLK_INTEGRITY_REF_TAG)
-			bid->bip.bip_flags |= BIP_CHECK_REFTAG;
-	}
+	bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO);
+	if (action & BI_ACT_CHECK)
+		bio_integrity_setup_default(bio);
 
 	/* Auto-generate integrity metadata if this is a write */
 	if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip))
-		blk_integrity_generate(bio);
+		bio_integrity_generate(bio);
 	else
 		bid->saved_bio_iter = bio->bi_iter;
-	return true;
 }
 EXPORT_SYMBOL(bio_integrity_prep);
 
diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c
new file mode 100644
index 0000000000000..acb1e5f270d2b
--- /dev/null
+++ b/block/bio-integrity-fs.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2025 Christoph Hellwig.
+ */
+#include <linux/blk-integrity.h>
+#include <linux/bio-integrity.h>
+#include "blk.h"
+
+struct fs_bio_integrity_buf {
+	struct bio_integrity_payload	bip;
+	struct bio_vec			bvec;
+};
+
+static struct kmem_cache *fs_bio_integrity_cache;
+static mempool_t fs_bio_integrity_pool;
+
+unsigned int fs_bio_integrity_alloc(struct bio *bio)
+{
+	struct fs_bio_integrity_buf *iib;
+	unsigned int action;
+
+	action = bio_integrity_action(bio);
+	if (!action)
+		return 0;
+
+	iib = mempool_alloc(&fs_bio_integrity_pool, GFP_NOIO);
+	bio_integrity_init(bio, &iib->bip, &iib->bvec, 1);
+
+	bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO);
+	if (action & BI_ACT_CHECK)
+		bio_integrity_setup_default(bio);
+	return action;
+}
+
+void fs_bio_integrity_free(struct bio *bio)
+{
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+
+	bio_integrity_free_buf(bip);
+	mempool_free(container_of(bip, struct fs_bio_integrity_buf, bip),
+			&fs_bio_integrity_pool);
+
+	bio->bi_integrity = NULL;
+	bio->bi_opf &= ~REQ_INTEGRITY;
+}
+
+void fs_bio_integrity_generate(struct bio *bio)
+{
+	if (fs_bio_integrity_alloc(bio))
+		bio_integrity_generate(bio);
+}
+EXPORT_SYMBOL_GPL(fs_bio_integrity_generate);
+
+int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
+{
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+
+	/*
+	 * Reinitialize bip->bip_iter.
+	 *
+	 * This is for use in the submitter after the driver is done with the
+	 * bio.  Requires the submitter to remember the sector and the size.
+	 */
+	memset(&bip->bip_iter, 0, sizeof(bip->bip_iter));
+	bip->bip_iter.bi_sector = sector;
+	bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT);
+	return blk_status_to_errno(bio_integrity_verify(bio, &bip->bip_iter));
+}
+
+static int __init fs_bio_integrity_init(void)
+{
+	fs_bio_integrity_cache = kmem_cache_create("fs_bio_integrity",
+			sizeof(struct fs_bio_integrity_buf), 0,
+			SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
+	if (mempool_init_slab_pool(&fs_bio_integrity_pool, BIO_POOL_SIZE,
+			fs_bio_integrity_cache))
+		panic("fs_bio_integrity: can't create pool\n");
+	return 0;
+}
+fs_initcall(fs_bio_integrity_init);
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 20f5d301d32dd..e79eaf0477943 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/blk-integrity.h>
+#include <linux/t10-pi.h>
 #include "blk.h"
 
 struct bio_integrity_alloc {
@@ -16,6 +17,53 @@ struct bio_integrity_alloc {
 
 static mempool_t integrity_buf_pool;
 
+static bool bi_offload_capable(struct blk_integrity *bi)
+{
+	return bi->metadata_size == bi->pi_tuple_size;
+}
+
+unsigned int __bio_integrity_action(struct bio *bio)
+{
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
+
+	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
+		return 0;
+
+	switch (bio_op(bio)) {
+	case REQ_OP_READ:
+		if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
+			if (bi_offload_capable(bi))
+				return 0;
+			return BI_ACT_BUFFER;
+		}
+		return BI_ACT_BUFFER | BI_ACT_CHECK;
+	case REQ_OP_WRITE:
+		/*
+		 * Flush masquerading as write?
+		 */
+		if (!bio_sectors(bio))
+			return 0;
+
+		/*
+		 * Zero the memory allocated to not leak uninitialized kernel
+		 * memory to disk for non-integrity metadata where nothing else
+		 * initializes the memory.
+		 */
+		if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
+			if (bi_offload_capable(bi))
+				return 0;
+			return BI_ACT_BUFFER | BI_ACT_ZERO;
+		}
+
+		if (bi->metadata_size > bi->pi_tuple_size)
+			return BI_ACT_BUFFER | BI_ACT_CHECK | BI_ACT_ZERO;
+		return BI_ACT_BUFFER | BI_ACT_CHECK;
+	default:
+		return 0;
+	}
+}
+EXPORT_SYMBOL_GPL(__bio_integrity_action);
+
 void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer)
 {
 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
@@ -53,6 +101,22 @@ void bio_integrity_free_buf(struct bio_integrity_payload *bip)
 		kfree(bvec_virt(bv));
 }
 
+void bio_integrity_setup_default(struct bio *bio)
+{
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+
+	bip_set_seed(bip, bio->bi_iter.bi_sector);
+
+	if (bi->csum_type) {
+		bip->bip_flags |= BIP_CHECK_GUARD;
+		if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
+			bip->bip_flags |= BIP_IP_CHECKSUM;
+	}
+	if (bi->flags & BLK_INTEGRITY_REF_TAG)
+		bip->bip_flags |= BIP_CHECK_REFTAG;
+}
+
 /**
  * bio_integrity_free - Free bio integrity payload
  * @bio:	bio containing bip to be freed
diff --git a/block/bio.c b/block/bio.c
index d80d5d26804e3..5057047194c49 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -34,6 +34,8 @@ struct bio_alloc_cache {
 	unsigned int		nr_irq;
 };
 
+#define BIO_INLINE_VECS 4
+
 static struct biovec_slab {
 	int nr_vecs;
 	char *name;
@@ -159,57 +161,16 @@ static void bio_put_slab(struct bio_set *bs)
 	mutex_unlock(&bio_slab_lock);
 }
 
-void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
-{
-	BUG_ON(nr_vecs > BIO_MAX_VECS);
-
-	if (nr_vecs == BIO_MAX_VECS)
-		mempool_free(bv, pool);
-	else if (nr_vecs > BIO_INLINE_VECS)
-		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
-}
-
 /*
  * Make the first allocation restricted and don't dump info on allocation
  * failures, since we'll fall back to the mempool in case of failure.
  */
-static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
+static inline gfp_t try_alloc_gfp(gfp_t gfp)
 {
 	return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
 		__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
 }
 
-struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
-		gfp_t gfp_mask)
-{
-	struct biovec_slab *bvs = biovec_slab(*nr_vecs);
-
-	if (WARN_ON_ONCE(!bvs))
-		return NULL;
-
-	/*
-	 * Upgrade the nr_vecs request to take full advantage of the allocation.
-	 * We also rely on this in the bvec_free path.
-	 */
-	*nr_vecs = bvs->nr_vecs;
-
-	/*
-	 * Try a slab allocation first for all smaller allocations.  If that
-	 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
-	 * The mempool is sized to handle up to BIO_MAX_VECS entries.
-	 */
-	if (*nr_vecs < BIO_MAX_VECS) {
-		struct bio_vec *bvl;
-
-		bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
-		if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
-			return bvl;
-		*nr_vecs = BIO_MAX_VECS;
-	}
-
-	return mempool_alloc(pool, gfp_mask);
-}
-
 void bio_uninit(struct bio *bio)
 {
 #ifdef CONFIG_BLK_CGROUP
@@ -231,9 +192,14 @@ static void bio_free(struct bio *bio)
 	void *p = bio;
 
 	WARN_ON_ONCE(!bs);
+	WARN_ON_ONCE(bio->bi_max_vecs > BIO_MAX_VECS);
 
 	bio_uninit(bio);
-	bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
+	if (bio->bi_max_vecs == BIO_MAX_VECS)
+		mempool_free(bio->bi_io_vec, &bs->bvec_pool);
+	else if (bio->bi_max_vecs > BIO_INLINE_VECS)
+		kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab,
+				bio->bi_io_vec);
 	mempool_free(p - bs->front_pad, &bs->bio_pool);
 }
 
@@ -430,13 +396,31 @@ static void bio_alloc_rescue(struct work_struct *work)
 	}
 }
 
+/*
+ * submit_bio_noacct() converts recursion to iteration; this means if we're
+ * running beneath it, any bios we allocate and submit will not be submitted
+ * (and thus freed) until after we return.
+ *
+ * This exposes us to a potential deadlock if we allocate multiple bios from the
+ * same bio_set while running underneath submit_bio_noacct().  If we were to
+ * allocate multiple bios (say a stacking block driver that was splitting bios),
+ * we would deadlock if we exhausted the mempool's reserve.
+ *
+ * We solve this, and guarantee forward progress by punting the bios on
+ * current->bio_list to a per bio_set rescuer workqueue before blocking to wait
+ * for elements being returned to the mempool.
+ */
 static void punt_bios_to_rescuer(struct bio_set *bs)
 {
 	struct bio_list punt, nopunt;
 	struct bio *bio;
 
-	if (WARN_ON_ONCE(!bs->rescue_workqueue))
+	if (!current->bio_list || !bs->rescue_workqueue)
+		return;
+	if (bio_list_empty(&current->bio_list[0]) &&
+	    bio_list_empty(&current->bio_list[1]))
 		return;
+
 	/*
 	 * In order to guarantee forward progress we must punt only bios that
 	 * were allocated from this bio_set; otherwise, if there was a bio on
@@ -483,9 +467,7 @@ static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache)
 	local_irq_restore(flags);
 }
 
-static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
-		unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
-		struct bio_set *bs)
+static struct bio *bio_alloc_percpu_cache(struct bio_set *bs)
 {
 	struct bio_alloc_cache *cache;
 	struct bio *bio;
@@ -503,11 +485,6 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
 	cache->free_list = bio->bi_next;
 	cache->nr--;
 	put_cpu();
-
-	if (nr_vecs)
-		bio_init_inline(bio, bdev, nr_vecs, opf);
-	else
-		bio_init(bio, bdev, NULL, nr_vecs, opf);
 	bio->bi_pool = bs;
 	return bio;
 }
@@ -517,7 +494,7 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
  * @bdev:	block device to allocate the bio for (can be %NULL)
  * @nr_vecs:	number of bvecs to pre-allocate
  * @opf:	operation and flags for bio
- * @gfp_mask:   the GFP_* mask given to the slab allocator
+ * @gfp:	the GFP_* mask given to the slab allocator
  * @bs:		the bio_set to allocate from.
  *
  * Allocate a bio from the mempools in @bs.
@@ -547,91 +524,77 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
  * Returns: Pointer to new bio on success, NULL on failure.
  */
 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
-			     blk_opf_t opf, gfp_t gfp_mask,
-			     struct bio_set *bs)
+			     blk_opf_t opf, gfp_t gfp, struct bio_set *bs)
 {
-	gfp_t saved_gfp = gfp_mask;
-	struct bio *bio;
+	struct bio_vec *bvecs = NULL;
+	struct bio *bio = NULL;
+	gfp_t saved_gfp = gfp;
 	void *p;
 
 	/* should not use nobvec bioset for nr_vecs > 0 */
 	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
 		return NULL;
 
+	gfp = try_alloc_gfp(gfp);
 	if (bs->cache && nr_vecs <= BIO_INLINE_VECS) {
-		opf |= REQ_ALLOC_CACHE;
-		bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf,
-					     gfp_mask, bs);
-		if (bio)
-			return bio;
 		/*
-		 * No cached bio available, bio returned below marked with
-		 * REQ_ALLOC_CACHE to participate in per-cpu alloc cache.
+		 * Set REQ_ALLOC_CACHE even if no cached bio is available to
+		 * return the allocated bio to the percpu cache when done.
 		 */
-	} else
+		opf |= REQ_ALLOC_CACHE;
+		bio = bio_alloc_percpu_cache(bs);
+	} else {
 		opf &= ~REQ_ALLOC_CACHE;
-
-	/*
-	 * submit_bio_noacct() converts recursion to iteration; this means if
-	 * we're running beneath it, any bios we allocate and submit will not be
-	 * submitted (and thus freed) until after we return.
-	 *
-	 * This exposes us to a potential deadlock if we allocate multiple bios
-	 * from the same bio_set() while running underneath submit_bio_noacct().
-	 * If we were to allocate multiple bios (say a stacking block driver
-	 * that was splitting bios), we would deadlock if we exhausted the
-	 * mempool's reserve.
-	 *
-	 * We solve this, and guarantee forward progress, with a rescuer
-	 * workqueue per bio_set. If we go to allocate and there are bios on
-	 * current->bio_list, we first try the allocation without
-	 * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
-	 * blocking to the rescuer workqueue before we retry with the original
-	 * gfp_flags.
-	 */
-	if (current->bio_list &&
-	    (!bio_list_empty(&current->bio_list[0]) ||
-	     !bio_list_empty(&current->bio_list[1])) &&
-	    bs->rescue_workqueue)
-		gfp_mask &= ~__GFP_DIRECT_RECLAIM;
-
-	p = mempool_alloc(&bs->bio_pool, gfp_mask);
-	if (!p && gfp_mask != saved_gfp) {
-		punt_bios_to_rescuer(bs);
-		gfp_mask = saved_gfp;
-		p = mempool_alloc(&bs->bio_pool, gfp_mask);
+		p = kmem_cache_alloc(bs->bio_slab, gfp);
+		if (p)
+			bio = p + bs->front_pad;
 	}
-	if (unlikely(!p))
-		return NULL;
-	if (!mempool_is_saturated(&bs->bio_pool))
-		opf &= ~REQ_ALLOC_CACHE;
 
-	bio = p + bs->front_pad;
-	if (nr_vecs > BIO_INLINE_VECS) {
-		struct bio_vec *bvl = NULL;
+	if (bio && nr_vecs > BIO_INLINE_VECS) {
+		struct biovec_slab *bvs = biovec_slab(nr_vecs);
 
-		bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
-		if (!bvl && gfp_mask != saved_gfp) {
-			punt_bios_to_rescuer(bs);
-			gfp_mask = saved_gfp;
-			bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask);
+		/*
+		 * Upgrade nr_vecs to take full advantage of the allocation.
+		 * We also rely on this in bio_free().
+		 */
+		nr_vecs = bvs->nr_vecs;
+		bvecs = kmem_cache_alloc(bvs->slab, gfp);
+		if (unlikely(!bvecs)) {
+			kmem_cache_free(bs->bio_slab, p);
+			bio = NULL;
 		}
-		if (unlikely(!bvl))
-			goto err_free;
+	}
 
-		bio_init(bio, bdev, bvl, nr_vecs, opf);
-	} else if (nr_vecs) {
-		bio_init_inline(bio, bdev, BIO_INLINE_VECS, opf);
-	} else {
-		bio_init(bio, bdev, NULL, 0, opf);
+	if (unlikely(!bio)) {
+		/*
+		 * Give up if we are not allow to sleep as non-blocking mempool
+		 * allocations just go back to the slab allocation.
+		 */
+		if (!(saved_gfp & __GFP_DIRECT_RECLAIM))
+			return NULL;
+
+		punt_bios_to_rescuer(bs);
+
+		/*
+		 * Don't rob the mempools by returning to the per-CPU cache if
+		 * we're tight on memory.
+		 */
+		opf &= ~REQ_ALLOC_CACHE;
+
+		p = mempool_alloc(&bs->bio_pool, gfp);
+		bio = p + bs->front_pad;
+		if (nr_vecs > BIO_INLINE_VECS) {
+			nr_vecs = BIO_MAX_VECS;
+			bvecs = mempool_alloc(&bs->bvec_pool, gfp);
+		}
 	}
 
+	if (nr_vecs && nr_vecs <= BIO_INLINE_VECS)
+		bio_init_inline(bio, bdev, nr_vecs, opf);
+	else
+		bio_init(bio, bdev, bvecs, nr_vecs, opf);
 	bio->bi_pool = bs;
 	return bio;
-
-err_free:
-	mempool_free(p, &bs->bio_pool);
-	return NULL;
 }
 EXPORT_SYMBOL(bio_alloc_bioset);
 
@@ -897,10 +860,11 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
  * @gfp: allocation priority
  * @bs: bio_set to allocate from
  *
- * Allocate a new bio that is a clone of @bio_src. The caller owns the returned
- * bio, but not the actual data it points to.
- *
- * The caller must ensure that the return bio is not freed before @bio_src.
+ * Allocate a new bio that is a clone of @bio_src. This reuses the bio_vecs
+ * pointed to by @bio_src->bi_io_vec, and clones the iterator pointing to
+ * the current position in it.  The caller owns the returned bio, but not
+ * the bio_vecs, and must ensure the bio is freed before the memory
+ * pointed to by @bio_Src->bi_io_vecs.
  */
 struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
 		gfp_t gfp, struct bio_set *bs)
@@ -929,9 +893,7 @@ EXPORT_SYMBOL(bio_alloc_clone);
  * @gfp: allocation priority
  *
  * Initialize a new bio in caller provided memory that is a clone of @bio_src.
- * The caller owns the returned bio, but not the actual data it points to.
- *
- * The caller must ensure that @bio_src is not freed before @bio.
+ * The same bio_vecs reuse and bio lifetime rules as bio_alloc_clone() apply.
  */
 int bio_init_clone(struct block_device *bdev, struct bio *bio,
 		struct bio *bio_src, gfp_t gfp)
@@ -1064,6 +1026,8 @@ int bio_add_page(struct bio *bio, struct page *page,
 {
 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 		return 0;
+	if (WARN_ON_ONCE(len == 0))
+		return 0;
 	if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len)
 		return 0;
 
@@ -1327,9 +1291,10 @@ static void bio_free_folios(struct bio *bio)
 	}
 }
 
-static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter)
+static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
+		size_t maxlen)
 {
-	size_t total_len = iov_iter_count(iter);
+	size_t total_len = min(maxlen, iov_iter_count(iter));
 
 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 		return -EINVAL;
@@ -1367,9 +1332,10 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter)
 	return 0;
 }
 
-static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
+static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
+		size_t maxlen)
 {
-	size_t len = min(iov_iter_count(iter), SZ_1M);
+	size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
 	struct folio *folio;
 
 	folio = folio_alloc_greedy(GFP_KERNEL, &len);
@@ -1408,6 +1374,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
  * bio_iov_iter_bounce - bounce buffer data from an iter into a bio
  * @bio:	bio to send
  * @iter:	iter to read from / write into
+ * @maxlen:	maximum size to bounce
  *
  * Helper for direct I/O implementations that need to bounce buffer because
  * we need to checksum the data or perform other operations that require
@@ -1415,11 +1382,11 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
  * copies the data into it.  Needs to be paired with bio_iov_iter_unbounce()
  * called on completion.
  */
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter)
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen)
 {
 	if (op_is_write(bio_op(bio)))
-		return bio_iov_iter_bounce_write(bio, iter);
-	return bio_iov_iter_bounce_read(bio, iter);
+		return bio_iov_iter_bounce_write(bio, iter, maxlen);
+	return bio_iov_iter_bounce_read(bio, iter, maxlen);
 }
 
 static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b70096497d389..2d7b18eb72915 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -24,6 +24,7 @@
 #include <linux/backing-dev.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
+#include <linux/wait_bit.h>
 #include <linux/atomic.h>
 #include <linux/ctype.h>
 #include <linux/resume_user_mode.h>
@@ -611,6 +612,8 @@ static void blkg_destroy_all(struct gendisk *disk)
 
 	q->root_blkg = NULL;
 	spin_unlock_irq(&q->queue_lock);
+
+	wake_up_var(&q->root_blkg);
 }
 
 static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
@@ -1498,6 +1501,18 @@ int blkcg_init_disk(struct gendisk *disk)
 	struct blkcg_gq *new_blkg, *blkg;
 	bool preloaded;
 
+	/*
+	 * If the queue is shared across disk rebind (e.g., SCSI), the
+	 * previous disk's blkcg state is cleaned up asynchronously via
+	 * disk_release() -> blkcg_exit_disk(). Wait for that cleanup to
+	 * finish (indicated by root_blkg becoming NULL) before setting up
+	 * new blkcg state. Otherwise, we may overwrite q->root_blkg while
+	 * the old one is still alive, and radix_tree_insert() in
+	 * blkg_create() will fail with -EEXIST because the old entries
+	 * still occupy the same queue id slot in blkcg->blkg_tree.
+	 */
+	wait_var_event(&q->root_blkg, !READ_ONCE(q->root_blkg));
+
 	new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL);
 	if (!new_blkg)
 		return -ENOMEM;
diff --git a/block/blk-crypto-sysfs.c b/block/blk-crypto-sysfs.c
index ea7a0b85a46f5..b069c418b6ccc 100644
--- a/block/blk-crypto-sysfs.c
+++ b/block/blk-crypto-sysfs.c
@@ -18,7 +18,7 @@ struct blk_crypto_kobj {
 struct blk_crypto_attr {
 	struct attribute attr;
 	ssize_t (*show)(struct blk_crypto_profile *profile,
-			struct blk_crypto_attr *attr, char *page);
+			const struct blk_crypto_attr *attr, char *page);
 };
 
 static struct blk_crypto_profile *kobj_to_crypto_profile(struct kobject *kobj)
@@ -26,39 +26,39 @@ static struct blk_crypto_profile *kobj_to_crypto_profile(struct kobject *kobj)
 	return container_of(kobj, struct blk_crypto_kobj, kobj)->profile;
 }
 
-static struct blk_crypto_attr *attr_to_crypto_attr(struct attribute *attr)
+static const struct blk_crypto_attr *attr_to_crypto_attr(const struct attribute *attr)
 {
-	return container_of(attr, struct blk_crypto_attr, attr);
+	return container_of_const(attr, struct blk_crypto_attr, attr);
 }
 
 static ssize_t hw_wrapped_keys_show(struct blk_crypto_profile *profile,
-				    struct blk_crypto_attr *attr, char *page)
+				    const struct blk_crypto_attr *attr, char *page)
 {
 	/* Always show supported, since the file doesn't exist otherwise. */
 	return sysfs_emit(page, "supported\n");
 }
 
 static ssize_t max_dun_bits_show(struct blk_crypto_profile *profile,
-				 struct blk_crypto_attr *attr, char *page)
+				 const struct blk_crypto_attr *attr, char *page)
 {
 	return sysfs_emit(page, "%u\n", 8 * profile->max_dun_bytes_supported);
 }
 
 static ssize_t num_keyslots_show(struct blk_crypto_profile *profile,
-				 struct blk_crypto_attr *attr, char *page)
+				 const struct blk_crypto_attr *attr, char *page)
 {
 	return sysfs_emit(page, "%u\n", profile->num_slots);
 }
 
 static ssize_t raw_keys_show(struct blk_crypto_profile *profile,
-			     struct blk_crypto_attr *attr, char *page)
+			     const struct blk_crypto_attr *attr, char *page)
 {
 	/* Always show supported, since the file doesn't exist otherwise. */
 	return sysfs_emit(page, "supported\n");
 }
 
 #define BLK_CRYPTO_RO_ATTR(_name) \
-	static struct blk_crypto_attr _name##_attr = __ATTR_RO(_name)
+	static const struct blk_crypto_attr _name##_attr = __ATTR_RO(_name)
 
 BLK_CRYPTO_RO_ATTR(hw_wrapped_keys);
 BLK_CRYPTO_RO_ATTR(max_dun_bits);
@@ -66,10 +66,10 @@ BLK_CRYPTO_RO_ATTR(num_keyslots);
 BLK_CRYPTO_RO_ATTR(raw_keys);
 
 static umode_t blk_crypto_is_visible(struct kobject *kobj,
-				     struct attribute *attr, int n)
+				     const struct attribute *attr, int n)
 {
 	struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj);
-	struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
+	const struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
 
 	if (a == &hw_wrapped_keys_attr &&
 	    !(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED))
@@ -81,7 +81,7 @@ static umode_t blk_crypto_is_visible(struct kobject *kobj,
 	return 0444;
 }
 
-static struct attribute *blk_crypto_attrs[] = {
+static const struct attribute *const blk_crypto_attrs[] = {
 	&hw_wrapped_keys_attr.attr,
 	&max_dun_bits_attr.attr,
 	&num_keyslots_attr.attr,
@@ -90,8 +90,8 @@ static struct attribute *blk_crypto_attrs[] = {
 };
 
 static const struct attribute_group blk_crypto_attr_group = {
-	.attrs = blk_crypto_attrs,
-	.is_visible = blk_crypto_is_visible,
+	.attrs_const = blk_crypto_attrs,
+	.is_visible_const = blk_crypto_is_visible,
 };
 
 /*
@@ -99,13 +99,13 @@ static const struct attribute_group blk_crypto_attr_group = {
  * modes, these are initialized at boot time by blk_crypto_sysfs_init().
  */
 static struct blk_crypto_attr __blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX];
-static struct attribute *blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX + 1];
+static const struct attribute *blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX + 1];
 
 static umode_t blk_crypto_mode_is_visible(struct kobject *kobj,
-					  struct attribute *attr, int n)
+					  const struct attribute *attr, int n)
 {
 	struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj);
-	struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
+	const struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
 	int mode_num = a - __blk_crypto_mode_attrs;
 
 	if (profile->modes_supported[mode_num])
@@ -114,7 +114,7 @@ static umode_t blk_crypto_mode_is_visible(struct kobject *kobj,
 }
 
 static ssize_t blk_crypto_mode_show(struct blk_crypto_profile *profile,
-				    struct blk_crypto_attr *attr, char *page)
+				    const struct blk_crypto_attr *attr, char *page)
 {
 	int mode_num = attr - __blk_crypto_mode_attrs;
 
@@ -123,8 +123,8 @@ static ssize_t blk_crypto_mode_show(struct blk_crypto_profile *profile,
 
 static const struct attribute_group blk_crypto_modes_attr_group = {
 	.name = "modes",
-	.attrs = blk_crypto_mode_attrs,
-	.is_visible = blk_crypto_mode_is_visible,
+	.attrs_const = blk_crypto_mode_attrs,
+	.is_visible_const = blk_crypto_mode_is_visible,
 };
 
 static const struct attribute_group *blk_crypto_attr_groups[] = {
@@ -137,7 +137,7 @@ static ssize_t blk_crypto_attr_show(struct kobject *kobj,
 				    struct attribute *attr, char *page)
 {
 	struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj);
-	struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
+	const struct blk_crypto_attr *a = attr_to_crypto_attr(attr);
 
 	return a->show(profile, a, page);
 }
diff --git a/block/blk-ia-ranges.c b/block/blk-ia-ranges.c
index d479f5481b66a..7be8b58893c90 100644
--- a/block/blk-ia-ranges.c
+++ b/block/blk-ia-ranges.c
@@ -30,17 +30,17 @@ struct blk_ia_range_sysfs_entry {
 	ssize_t (*show)(struct blk_independent_access_range *iar, char *buf);
 };
 
-static struct blk_ia_range_sysfs_entry blk_ia_range_sector_entry = {
+static const struct blk_ia_range_sysfs_entry blk_ia_range_sector_entry = {
 	.attr = { .name = "sector", .mode = 0444 },
 	.show = blk_ia_range_sector_show,
 };
 
-static struct blk_ia_range_sysfs_entry blk_ia_range_nr_sectors_entry = {
+static const struct blk_ia_range_sysfs_entry blk_ia_range_nr_sectors_entry = {
 	.attr = { .name = "nr_sectors", .mode = 0444 },
 	.show = blk_ia_range_nr_sectors_show,
 };
 
-static struct attribute *blk_ia_range_attrs[] = {
+static const struct attribute *const blk_ia_range_attrs[] = {
 	&blk_ia_range_sector_entry.attr,
 	&blk_ia_range_nr_sectors_entry.attr,
 	NULL,
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index 28167c9baa559..047ec887456b6 100644
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -97,6 +97,7 @@ static const char *const blk_queue_flag_name[] = {
 	QUEUE_FLAG_NAME(NO_ELV_SWITCH),
 	QUEUE_FLAG_NAME(QOS_ENABLED),
 	QUEUE_FLAG_NAME(BIO_ISSUE_TIME),
+	QUEUE_FLAG_NAME(ZONED_QD1_WRITES),
 };
 #undef QUEUE_FLAG_NAME
 
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 58ec293373c6f..895397831eccc 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -53,7 +53,7 @@ static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
 	struct request_queue *q;
 	ssize_t res;
 
-	entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
+	entry = container_of_const(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
 	hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
 	q = hctx->queue;
 
@@ -101,20 +101,20 @@ static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
 	return pos + ret;
 }
 
-static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
+static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
 	.attr = {.name = "nr_tags", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_nr_tags_show,
 };
-static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
+static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
 	.attr = {.name = "nr_reserved_tags", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_nr_reserved_tags_show,
 };
-static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
+static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
 	.attr = {.name = "cpu_list", .mode = 0444 },
 	.show = blk_mq_hw_sysfs_cpus_show,
 };
 
-static struct attribute *default_hw_ctx_attrs[] = {
+static const struct attribute *const default_hw_ctx_attrs[] = {
 	&blk_mq_hw_sysfs_nr_tags.attr,
 	&blk_mq_hw_sysfs_nr_reserved_tags.attr,
 	&blk_mq_hw_sysfs_cpus.attr,
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 3da2215b29125..4c5c16cce4f8f 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -3143,6 +3143,7 @@ void blk_mq_submit_bio(struct bio *bio)
 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 	struct blk_plug *plug = current->plug;
 	const int is_sync = op_is_sync(bio->bi_opf);
+	unsigned int integrity_action;
 	struct blk_mq_hw_ctx *hctx;
 	unsigned int nr_segs;
 	struct request *rq;
@@ -3195,8 +3196,9 @@ void blk_mq_submit_bio(struct bio *bio)
 	if (!bio)
 		goto queue_exit;
 
-	if (!bio_integrity_prep(bio))
-		goto queue_exit;
+	integrity_action = bio_integrity_action(bio);
+	if (integrity_action)
+		bio_integrity_prep(bio, integrity_action);
 
 	blk_mq_bio_issue_init(q, bio);
 	if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
@@ -3422,6 +3424,25 @@ EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
  */
 void blk_steal_bios(struct bio_list *list, struct request *rq)
 {
+	struct bio *bio;
+
+	for (bio = rq->bio; bio; bio = bio->bi_next) {
+		if (bio->bi_opf & REQ_POLLED) {
+			bio->bi_opf &= ~REQ_POLLED;
+			bio->bi_cookie = BLK_QC_T_NONE;
+		}
+		/*
+		 * The alternate request queue that we may end up submitting
+		 * the bio to may be frozen temporarily, in this case REQ_NOWAIT
+		 * will fail the I/O immediately with EAGAIN to the issuer.
+		 * We are not in the issuer context which cannot block. Clear
+		 * the flag to avoid spurious EAGAIN I/O failures.
+		 */
+		bio->bi_opf &= ~REQ_NOWAIT;
+		bio_clear_flag(bio, BIO_QOS_THROTTLED);
+		bio_clear_flag(bio, BIO_QOS_MERGED);
+	}
+
 	if (rq->bio) {
 		if (list->tail)
 			list->tail->bi_next = rq->bio;
diff --git a/block/blk-settings.c b/block/blk-settings.c
index a9e65dc090dae..78c83817b9d36 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -123,19 +123,6 @@ static int blk_validate_zoned_limits(struct queue_limits *lim)
 	return 0;
 }
 
-/*
- * Maximum size of I/O that needs a block layer integrity buffer.  Limited
- * by the number of intervals for which we can fit the integrity buffer into
- * the buffer size.  Because the buffer is a single segment it is also limited
- * by the maximum segment size.
- */
-static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
-{
-	return min_t(unsigned int, lim->max_segment_size,
-		(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
-			lim->integrity.interval_exp);
-}
-
 static int blk_validate_integrity_limits(struct queue_limits *lim)
 {
 	struct blk_integrity *bi = &lim->integrity;
@@ -202,11 +189,11 @@ static int blk_validate_integrity_limits(struct queue_limits *lim)
 	}
 
 	/*
-	 * The PI generation / validation helpers do not expect intervals to
-	 * straddle multiple bio_vecs.  Enforce alignment so that those are
+	 * Some IO controllers can not handle data intervals straddling
+	 * multiple bio_vecs.  For those, enforce alignment so that those are
 	 * never generated, and that each buffer is aligned as expected.
 	 */
-	if (bi->csum_type) {
+	if (!(bi->flags & BLK_SPLIT_INTERVAL_CAPABLE) && bi->csum_type) {
 		lim->dma_alignment = max(lim->dma_alignment,
 					(1U << bi->interval_exp) - 1);
 	}
@@ -1005,10 +992,14 @@ bool queue_limits_stack_integrity(struct queue_limits *t,
 		if ((ti->flags & BLK_INTEGRITY_REF_TAG) !=
 		    (bi->flags & BLK_INTEGRITY_REF_TAG))
 			goto incompatible;
+		if ((ti->flags & BLK_SPLIT_INTERVAL_CAPABLE) &&
+		    !(bi->flags & BLK_SPLIT_INTERVAL_CAPABLE))
+			ti->flags &= ~BLK_SPLIT_INTERVAL_CAPABLE;
 	} else {
 		ti->flags = BLK_INTEGRITY_STACKED;
 		ti->flags |= (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) |
-			     (bi->flags & BLK_INTEGRITY_REF_TAG);
+			     (bi->flags & BLK_INTEGRITY_REF_TAG) |
+			     (bi->flags & BLK_SPLIT_INTERVAL_CAPABLE);
 		ti->csum_type = bi->csum_type;
 		ti->pi_tuple_size = bi->pi_tuple_size;
 		ti->metadata_size = bi->metadata_size;
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 55a1bbfef7d45..f22c1f253eb3a 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -390,6 +390,36 @@ static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)
 	return queue_var_show(disk_nr_zones(disk), page);
 }
 
+static ssize_t queue_zoned_qd1_writes_show(struct gendisk *disk, char *page)
+{
+	return queue_var_show(!!blk_queue_zoned_qd1_writes(disk->queue),
+			      page);
+}
+
+static ssize_t queue_zoned_qd1_writes_store(struct gendisk *disk,
+					    const char *page, size_t count)
+{
+	struct request_queue *q = disk->queue;
+	unsigned long qd1_writes;
+	unsigned int memflags;
+	ssize_t ret;
+
+	ret = queue_var_store(&qd1_writes, page, count);
+	if (ret < 0)
+		return ret;
+
+	memflags = blk_mq_freeze_queue(q);
+	blk_mq_quiesce_queue(q);
+	if (qd1_writes)
+		blk_queue_flag_set(QUEUE_FLAG_ZONED_QD1_WRITES, q);
+	else
+		blk_queue_flag_clear(QUEUE_FLAG_ZONED_QD1_WRITES, q);
+	blk_mq_unquiesce_queue(q);
+	blk_mq_unfreeze_queue(q, memflags);
+
+	return count;
+}
+
 static ssize_t queue_iostats_passthrough_show(struct gendisk *disk, char *page)
 {
 	return queue_var_show(!!blk_queue_passthrough_stat(disk->queue), page);
@@ -551,27 +581,27 @@ static int queue_wc_store(struct gendisk *disk, const char *page,
 	return 0;
 }
 
-#define QUEUE_RO_ENTRY(_prefix, _name)			\
-static struct queue_sysfs_entry _prefix##_entry = {	\
-	.attr	= { .name = _name, .mode = 0444 },	\
-	.show	= _prefix##_show,			\
+#define QUEUE_RO_ENTRY(_prefix, _name)				\
+static const struct queue_sysfs_entry _prefix##_entry = {	\
+	.attr	= { .name = _name, .mode = 0444 },		\
+	.show	= _prefix##_show,				\
 };
 
-#define QUEUE_RW_ENTRY(_prefix, _name)			\
-static struct queue_sysfs_entry _prefix##_entry = {	\
-	.attr	= { .name = _name, .mode = 0644 },	\
-	.show	= _prefix##_show,			\
-	.store	= _prefix##_store,			\
+#define QUEUE_RW_ENTRY(_prefix, _name)				\
+static const struct queue_sysfs_entry _prefix##_entry = {	\
+	.attr	= { .name = _name, .mode = 0644 },		\
+	.show	= _prefix##_show,				\
+	.store	= _prefix##_store,				\
 };
 
 #define QUEUE_LIM_RO_ENTRY(_prefix, _name)			\
-static struct queue_sysfs_entry _prefix##_entry = {	\
+static const struct queue_sysfs_entry _prefix##_entry = {	\
 	.attr		= { .name = _name, .mode = 0444 },	\
 	.show_limit	= _prefix##_show,			\
 }
 
 #define QUEUE_LIM_RW_ENTRY(_prefix, _name)			\
-static struct queue_sysfs_entry _prefix##_entry = {	\
+static const struct queue_sysfs_entry _prefix##_entry = {	\
 	.attr		= { .name = _name, .mode = 0644 },	\
 	.show_limit	= _prefix##_show,			\
 	.store_limit	= _prefix##_store,			\
@@ -617,6 +647,7 @@ QUEUE_LIM_RO_ENTRY(queue_max_zone_append_sectors, "zone_append_max_bytes");
 QUEUE_LIM_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
 
 QUEUE_LIM_RO_ENTRY(queue_zoned, "zoned");
+QUEUE_RW_ENTRY(queue_zoned_qd1_writes, "zoned_qd1_writes");
 QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
 QUEUE_LIM_RO_ENTRY(queue_max_open_zones, "max_open_zones");
 QUEUE_LIM_RO_ENTRY(queue_max_active_zones, "max_active_zones");
@@ -634,7 +665,7 @@ QUEUE_LIM_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
 QUEUE_LIM_RO_ENTRY(queue_dma_alignment, "dma_alignment");
 
 /* legacy alias for logical_block_size: */
-static struct queue_sysfs_entry queue_hw_sector_size_entry = {
+static const struct queue_sysfs_entry queue_hw_sector_size_entry = {
 	.attr		= {.name = "hw_sector_size", .mode = 0444 },
 	.show_limit	= queue_logical_block_size_show,
 };
@@ -700,7 +731,7 @@ QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
 #endif
 
 /* Common attributes for bio-based and request-based queues. */
-static struct attribute *queue_attrs[] = {
+static const struct attribute *const queue_attrs[] = {
 	/*
 	 * Attributes which are protected with q->limits_lock.
 	 */
@@ -754,12 +785,13 @@ static struct attribute *queue_attrs[] = {
 	&queue_nomerges_entry.attr,
 	&queue_poll_entry.attr,
 	&queue_poll_delay_entry.attr,
+	&queue_zoned_qd1_writes_entry.attr,
 
 	NULL,
 };
 
 /* Request-based queue attributes that are not relevant for bio-based queues. */
-static struct attribute *blk_mq_queue_attrs[] = {
+static const struct attribute *const blk_mq_queue_attrs[] = {
 	/*
 	 * Attributes which require some form of locking other than
 	 * q->sysfs_lock.
@@ -779,14 +811,15 @@ static struct attribute *blk_mq_queue_attrs[] = {
 	NULL,
 };
 
-static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
+static umode_t queue_attr_visible(struct kobject *kobj, const struct attribute *attr,
 				int n)
 {
 	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 	struct request_queue *q = disk->queue;
 
 	if ((attr == &queue_max_open_zones_entry.attr ||
-	     attr == &queue_max_active_zones_entry.attr) &&
+	     attr == &queue_max_active_zones_entry.attr ||
+	     attr == &queue_zoned_qd1_writes_entry.attr) &&
 	    !blk_queue_is_zoned(q))
 		return 0;
 
@@ -794,7 +827,7 @@ static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
 }
 
 static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
-					 struct attribute *attr, int n)
+					 const struct attribute *attr, int n)
 {
 	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 	struct request_queue *q = disk->queue;
@@ -808,17 +841,17 @@ static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group queue_attr_group = {
-	.attrs = queue_attrs,
-	.is_visible = queue_attr_visible,
+static const struct attribute_group queue_attr_group = {
+	.attrs_const = queue_attrs,
+	.is_visible_const = queue_attr_visible,
 };
 
-static struct attribute_group blk_mq_queue_attr_group = {
-	.attrs = blk_mq_queue_attrs,
-	.is_visible = blk_mq_queue_attr_visible,
+static const struct attribute_group blk_mq_queue_attr_group = {
+	.attrs_const = blk_mq_queue_attrs,
+	.is_visible_const = blk_mq_queue_attr_visible,
 };
 
-#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
+#define to_queue(atr) container_of_const((atr), struct queue_sysfs_entry, attr)
 
 static ssize_t
 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
@@ -934,6 +967,14 @@ int blk_register_queue(struct gendisk *disk)
 		blk_mq_debugfs_register(q);
 	blk_debugfs_unlock(q, memflags);
 
+	/*
+	 * For blk-mq rotational zoned devices, default to using QD=1
+	 * writes. For non-mq rotational zoned devices, the device driver can
+	 * set an appropriate default.
+	 */
+	if (queue_is_mq(q) && blk_queue_rot(q) && blk_queue_is_zoned(q))
+		blk_queue_flag_set(QUEUE_FLAG_ZONED_QD1_WRITES, q);
+
 	ret = disk_register_independent_access_ranges(disk);
 	if (ret)
 		goto out_debugfs_remove;
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index 9d1dd6ccfad72..e1a23c8b676d5 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -16,6 +16,8 @@
 #include <linux/spinlock.h>
 #include <linux/refcount.h>
 #include <linux/mempool.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
 
 #include <trace/events/block.h>
 
@@ -40,6 +42,8 @@ static const char *const zone_cond_name[] = {
 /*
  * Per-zone write plug.
  * @node: hlist_node structure for managing the plug using a hash table.
+ * @entry: list_head structure for listing the plug in the disk list of active
+ *         zone write plugs.
  * @bio_list: The list of BIOs that are currently plugged.
  * @bio_work: Work struct to handle issuing of plugged BIOs
  * @rcu_head: RCU head to free zone write plugs with an RCU grace period.
@@ -62,6 +66,7 @@ static const char *const zone_cond_name[] = {
  */
 struct blk_zone_wplug {
 	struct hlist_node	node;
+	struct list_head	entry;
 	struct bio_list		bio_list;
 	struct work_struct	bio_work;
 	struct rcu_head		rcu_head;
@@ -99,17 +104,17 @@ static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk)
  *    being executed or the zone write plug bio list is not empty.
  *  - BLK_ZONE_WPLUG_NEED_WP_UPDATE: Indicates that we lost track of a zone
  *    write pointer offset and need to update it.
- *  - BLK_ZONE_WPLUG_UNHASHED: Indicates that the zone write plug was removed
- *    from the disk hash table and that the initial reference to the zone
- *    write plug set when the plug was first added to the hash table has been
- *    dropped. This flag is set when a zone is reset, finished or become full,
- *    to prevent new references to the zone write plug to be taken for
- *    newly incoming BIOs. A zone write plug flagged with this flag will be
- *    freed once all remaining references from BIOs or functions are dropped.
+ *  - BLK_ZONE_WPLUG_DEAD: Indicates that the zone write plug will be
+ *    removed from the disk hash table of zone write plugs when the last
+ *    reference on the zone write plug is dropped. If set, this flag also
+ *    indicates that the initial extra reference on the zone write plug was
+ *    dropped, meaning that the reference count indicates the current number of
+ *    active users (code context or BIOs and requests in flight). This flag is
+ *    set when a zone is reset, finished or becomes full.
  */
 #define BLK_ZONE_WPLUG_PLUGGED		(1U << 0)
 #define BLK_ZONE_WPLUG_NEED_WP_UPDATE	(1U << 1)
-#define BLK_ZONE_WPLUG_UNHASHED		(1U << 2)
+#define BLK_ZONE_WPLUG_DEAD		(1U << 2)
 
 /**
  * blk_zone_cond_str - Return a zone condition name string
@@ -492,18 +497,12 @@ static bool disk_zone_is_last(struct gendisk *disk, struct blk_zone *zone)
 	return zone->start + zone->len >= get_capacity(disk);
 }
 
-static bool disk_zone_is_full(struct gendisk *disk,
-			      unsigned int zno, unsigned int offset_in_zone)
-{
-	if (zno < disk->nr_zones - 1)
-		return offset_in_zone >= disk->zone_capacity;
-	return offset_in_zone >= disk->last_zone_capacity;
-}
-
 static bool disk_zone_wplug_is_full(struct gendisk *disk,
 				    struct blk_zone_wplug *zwplug)
 {
-	return disk_zone_is_full(disk, zwplug->zone_no, zwplug->wp_offset);
+	if (zwplug->zone_no < disk->nr_zones - 1)
+		return zwplug->wp_offset >= disk->zone_capacity;
+	return zwplug->wp_offset >= disk->last_zone_capacity;
 }
 
 static bool disk_insert_zone_wplug(struct gendisk *disk,
@@ -520,10 +519,11 @@ static bool disk_insert_zone_wplug(struct gendisk *disk,
 	 * are racing with other submission context, so we may already have a
 	 * zone write plug for the same zone.
 	 */
-	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);
+	spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags);
 	hlist_for_each_entry_rcu(zwplg, &disk->zone_wplugs_hash[idx], node) {
 		if (zwplg->zone_no == zwplug->zone_no) {
-			spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);
+			spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock,
+					       flags);
 			return false;
 		}
 	}
@@ -535,7 +535,7 @@ static bool disk_insert_zone_wplug(struct gendisk *disk,
 	 * necessarilly in the active condition.
 	 */
 	zones_cond = rcu_dereference_check(disk->zones_cond,
-				lockdep_is_held(&disk->zone_wplugs_lock));
+				lockdep_is_held(&disk->zone_wplugs_hash_lock));
 	if (zones_cond)
 		zwplug->cond = zones_cond[zwplug->zone_no];
 	else
@@ -543,7 +543,7 @@ static bool disk_insert_zone_wplug(struct gendisk *disk,
 
 	hlist_add_head_rcu(&zwplug->node, &disk->zone_wplugs_hash[idx]);
 	atomic_inc(&disk->nr_zone_wplugs);
-	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);
+	spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags);
 
 	return true;
 }
@@ -587,105 +587,76 @@ static void disk_free_zone_wplug_rcu(struct rcu_head *rcu_head)
 	mempool_free(zwplug, zwplug->disk->zone_wplugs_pool);
 }
 
-static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug)
+static void disk_free_zone_wplug(struct blk_zone_wplug *zwplug)
 {
-	if (refcount_dec_and_test(&zwplug->ref)) {
-		WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list));
-		WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED);
-		WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_UNHASHED));
+	struct gendisk *disk = zwplug->disk;
+	unsigned long flags;
 
-		call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu);
-	}
-}
+	WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_DEAD));
+	WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED);
+	WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list));
 
-static inline bool disk_should_remove_zone_wplug(struct gendisk *disk,
-						 struct blk_zone_wplug *zwplug)
-{
-	lockdep_assert_held(&zwplug->lock);
+	spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags);
+	blk_zone_set_cond(rcu_dereference_check(disk->zones_cond,
+				lockdep_is_held(&disk->zone_wplugs_hash_lock)),
+			  zwplug->zone_no, zwplug->cond);
+	hlist_del_init_rcu(&zwplug->node);
+	atomic_dec(&disk->nr_zone_wplugs);
+	spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags);
 
-	/* If the zone write plug was already removed, we are done. */
-	if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED)
-		return false;
+	call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu);
+}
 
-	/* If the zone write plug is still plugged, it cannot be removed. */
-	if (zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)
-		return false;
+static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug)
+{
+	if (refcount_dec_and_test(&zwplug->ref))
+		disk_free_zone_wplug(zwplug);
+}
 
-	/*
-	 * Completions of BIOs with blk_zone_write_plug_bio_endio() may
-	 * happen after handling a request completion with
-	 * blk_zone_write_plug_finish_request() (e.g. with split BIOs
-	 * that are chained). In such case, disk_zone_wplug_unplug_bio()
-	 * should not attempt to remove the zone write plug until all BIO
-	 * completions are seen. Check by looking at the zone write plug
-	 * reference count, which is 2 when the plug is unused (one reference
-	 * taken when the plug was allocated and another reference taken by the
-	 * caller context).
-	 */
-	if (refcount_read(&zwplug->ref) > 2)
-		return false;
+/*
+ * Flag the zone write plug as dead and drop the initial reference we got when
+ * the zone write plug was added to the hash table. The zone write plug will be
+ * unhashed when its last reference is dropped.
+ */
+static void disk_mark_zone_wplug_dead(struct blk_zone_wplug *zwplug)
+{
+	lockdep_assert_held(&zwplug->lock);
 
-	/* We can remove zone write plugs for zones that are empty or full. */
-	return !zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug);
+	if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)) {
+		zwplug->flags |= BLK_ZONE_WPLUG_DEAD;
+		disk_put_zone_wplug(zwplug);
+	}
 }
 
-static void disk_remove_zone_wplug(struct gendisk *disk,
-				   struct blk_zone_wplug *zwplug)
+static bool disk_zone_wplug_submit_bio(struct gendisk *disk,
+				       struct blk_zone_wplug *zwplug);
+
+static void blk_zone_wplug_bio_work(struct work_struct *work)
 {
-	unsigned long flags;
+	struct blk_zone_wplug *zwplug =
+		container_of(work, struct blk_zone_wplug, bio_work);
 
-	/* If the zone write plug was already removed, we have nothing to do. */
-	if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED)
-		return;
+	disk_zone_wplug_submit_bio(zwplug->disk, zwplug);
 
-	/*
-	 * Mark the zone write plug as unhashed and drop the extra reference we
-	 * took when the plug was inserted in the hash table. Also update the
-	 * disk zone condition array with the current condition of the zone
-	 * write plug.
-	 */
-	zwplug->flags |= BLK_ZONE_WPLUG_UNHASHED;
-	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);
-	blk_zone_set_cond(rcu_dereference_check(disk->zones_cond,
-				lockdep_is_held(&disk->zone_wplugs_lock)),
-			  zwplug->zone_no, zwplug->cond);
-	hlist_del_init_rcu(&zwplug->node);
-	atomic_dec(&disk->nr_zone_wplugs);
-	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);
+	/* Drop the reference we took in disk_zone_wplug_schedule_work(). */
 	disk_put_zone_wplug(zwplug);
 }
 
-static void blk_zone_wplug_bio_work(struct work_struct *work);
-
 /*
- * Get a reference on the write plug for the zone containing @sector.
- * If the plug does not exist, it is allocated and hashed.
- * Return a pointer to the zone write plug with the plug spinlock held.
+ * Get a zone write plug for the zone containing @sector.
+ * If the plug does not exist, it is allocated and inserted in the disk hash
+ * table.
  */
-static struct blk_zone_wplug *disk_get_and_lock_zone_wplug(struct gendisk *disk,
-					sector_t sector, gfp_t gfp_mask,
-					unsigned long *flags)
+static struct blk_zone_wplug *disk_get_or_alloc_zone_wplug(struct gendisk *disk,
+					sector_t sector, gfp_t gfp_mask)
 {
 	unsigned int zno = disk_zone_no(disk, sector);
 	struct blk_zone_wplug *zwplug;
 
 again:
 	zwplug = disk_get_zone_wplug(disk, sector);
-	if (zwplug) {
-		/*
-		 * Check that a BIO completion or a zone reset or finish
-		 * operation has not already removed the zone write plug from
-		 * the hash table and dropped its reference count. In such case,
-		 * we need to get a new plug so start over from the beginning.
-		 */
-		spin_lock_irqsave(&zwplug->lock, *flags);
-		if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) {
-			spin_unlock_irqrestore(&zwplug->lock, *flags);
-			disk_put_zone_wplug(zwplug);
-			goto again;
-		}
+	if (zwplug)
 		return zwplug;
-	}
 
 	/*
 	 * Allocate and initialize a zone write plug with an extra reference
@@ -704,17 +675,15 @@ static struct blk_zone_wplug *disk_get_and_lock_zone_wplug(struct gendisk *disk,
 	zwplug->wp_offset = bdev_offset_from_zone_start(disk->part0, sector);
 	bio_list_init(&zwplug->bio_list);
 	INIT_WORK(&zwplug->bio_work, blk_zone_wplug_bio_work);
+	INIT_LIST_HEAD(&zwplug->entry);
 	zwplug->disk = disk;
 
-	spin_lock_irqsave(&zwplug->lock, *flags);
-
 	/*
 	 * Insert the new zone write plug in the hash table. This can fail only
 	 * if another context already inserted a plug. Retry from the beginning
 	 * in such case.
 	 */
 	if (!disk_insert_zone_wplug(disk, zwplug)) {
-		spin_unlock_irqrestore(&zwplug->lock, *flags);
 		mempool_free(zwplug, disk->zone_wplugs_pool);
 		goto again;
 	}
@@ -739,6 +708,7 @@ static inline void blk_zone_wplug_bio_io_error(struct blk_zone_wplug *zwplug,
  */
 static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug)
 {
+	struct gendisk *disk = zwplug->disk;
 	struct bio *bio;
 
 	lockdep_assert_held(&zwplug->lock);
@@ -752,6 +722,20 @@ static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug)
 		blk_zone_wplug_bio_io_error(zwplug, bio);
 
 	zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
+
+	/*
+	 * If we are using the per disk zone write plugs worker thread, remove
+	 * the zone write plug from the work list and drop the reference we
+	 * took when the zone write plug was added to that list.
+	 */
+	if (blk_queue_zoned_qd1_writes(disk->queue)) {
+		spin_lock(&disk->zone_wplugs_list_lock);
+		if (!list_empty(&zwplug->entry)) {
+			list_del_init(&zwplug->entry);
+			disk_put_zone_wplug(zwplug);
+		}
+		spin_unlock(&disk->zone_wplugs_list_lock);
+	}
 }
 
 /*
@@ -788,14 +772,8 @@ static void disk_zone_wplug_set_wp_offset(struct gendisk *disk,
 	disk_zone_wplug_update_cond(disk, zwplug);
 
 	disk_zone_wplug_abort(zwplug);
-
-	/*
-	 * The zone write plug now has no BIO plugged: remove it from the
-	 * hash table so that it cannot be seen. The plug will be freed
-	 * when the last reference is dropped.
-	 */
-	if (disk_should_remove_zone_wplug(disk, zwplug))
-		disk_remove_zone_wplug(disk, zwplug);
+	if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug))
+		disk_mark_zone_wplug_dead(zwplug);
 }
 
 static unsigned int blk_zone_wp_offset(struct blk_zone *zone)
@@ -1192,19 +1170,24 @@ void blk_zone_mgmt_bio_endio(struct bio *bio)
 	}
 }
 
-static void disk_zone_wplug_schedule_bio_work(struct gendisk *disk,
-					      struct blk_zone_wplug *zwplug)
+static void disk_zone_wplug_schedule_work(struct gendisk *disk,
+					  struct blk_zone_wplug *zwplug)
 {
 	lockdep_assert_held(&zwplug->lock);
 
 	/*
-	 * Take a reference on the zone write plug and schedule the submission
-	 * of the next plugged BIO. blk_zone_wplug_bio_work() will release the
-	 * reference we take here.
+	 * Schedule the submission of the next plugged BIO. Taking a reference
+	 * to the zone write plug is required as the bio_work belongs to the
+	 * plug, and thus we must ensure that the write plug does not go away
+	 * while the work is being scheduled but has not run yet.
+	 * blk_zone_wplug_bio_work() will release the reference we take here,
+	 * and we also drop this reference if the work is already scheduled.
 	 */
 	WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED));
+	WARN_ON_ONCE(blk_queue_zoned_qd1_writes(disk->queue));
 	refcount_inc(&zwplug->ref);
-	queue_work(disk->zone_wplugs_wq, &zwplug->bio_work);
+	if (!queue_work(disk->zone_wplugs_wq, &zwplug->bio_work))
+		disk_put_zone_wplug(zwplug);
 }
 
 static inline void disk_zone_wplug_add_bio(struct gendisk *disk,
@@ -1241,6 +1224,22 @@ static inline void disk_zone_wplug_add_bio(struct gendisk *disk,
 	bio_list_add(&zwplug->bio_list, bio);
 	trace_disk_zone_wplug_add_bio(zwplug->disk->queue, zwplug->zone_no,
 				      bio->bi_iter.bi_sector, bio_sectors(bio));
+
+	/*
+	 * If we are using the disk zone write plugs worker instead of the per
+	 * zone write plug BIO work, add the zone write plug to the work list
+	 * if it is not already there. Make sure to also get an extra reference
+	 * on the zone write plug so that it does not go away until it is
+	 * removed from the work list.
+	 */
+	if (blk_queue_zoned_qd1_writes(disk->queue)) {
+		spin_lock(&disk->zone_wplugs_list_lock);
+		if (list_empty(&zwplug->entry)) {
+			list_add_tail(&zwplug->entry, &disk->zone_wplugs_list);
+			refcount_inc(&zwplug->ref);
+		}
+		spin_unlock(&disk->zone_wplugs_list_lock);
+	}
 }
 
 /*
@@ -1438,7 +1437,7 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
 	if (bio->bi_opf & REQ_NOWAIT)
 		gfp_mask = GFP_NOWAIT;
 
-	zwplug = disk_get_and_lock_zone_wplug(disk, sector, gfp_mask, &flags);
+	zwplug = disk_get_or_alloc_zone_wplug(disk, sector, gfp_mask);
 	if (!zwplug) {
 		if (bio->bi_opf & REQ_NOWAIT)
 			bio_wouldblock_error(bio);
@@ -1447,6 +1446,21 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
 		return true;
 	}
 
+	spin_lock_irqsave(&zwplug->lock, flags);
+
+	/*
+	 * If we got a zone write plug marked as dead, then the user is issuing
+	 * writes to a full zone, or without synchronizing with zone reset or
+	 * zone finish operations. In such case, fail the BIO to signal this
+	 * invalid usage.
+	 */
+	if (zwplug->flags & BLK_ZONE_WPLUG_DEAD) {
+		spin_unlock_irqrestore(&zwplug->lock, flags);
+		disk_put_zone_wplug(zwplug);
+		bio_io_error(bio);
+		return true;
+	}
+
 	/* Indicate that this BIO is being handled using zone write plugging. */
 	bio_set_flag(bio, BIO_ZONE_WRITE_PLUGGING);
 
@@ -1459,6 +1473,13 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
 		goto queue_bio;
 	}
 
+	/*
+	 * For rotational devices, we will use the gendisk zone write plugs
+	 * work instead of the per zone write plug BIO work, so queue the BIO.
+	 */
+	if (blk_queue_zoned_qd1_writes(disk->queue))
+		goto queue_bio;
+
 	/* If the zone is already plugged, add the BIO to the BIO plug list. */
 	if (zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)
 		goto queue_bio;
@@ -1481,7 +1502,10 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
 
 	if (!(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)) {
 		zwplug->flags |= BLK_ZONE_WPLUG_PLUGGED;
-		disk_zone_wplug_schedule_bio_work(disk, zwplug);
+		if (blk_queue_zoned_qd1_writes(disk->queue))
+			wake_up_process(disk->zone_wplugs_worker);
+		else
+			disk_zone_wplug_schedule_work(disk, zwplug);
 	}
 
 	spin_unlock_irqrestore(&zwplug->lock, flags);
@@ -1527,7 +1551,7 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio)
 				    disk->disk_name, zwplug->zone_no);
 		disk_zone_wplug_abort(zwplug);
 	}
-	disk_remove_zone_wplug(disk, zwplug);
+	disk_mark_zone_wplug_dead(zwplug);
 	spin_unlock_irqrestore(&zwplug->lock, flags);
 
 	disk_put_zone_wplug(zwplug);
@@ -1622,21 +1646,21 @@ static void disk_zone_wplug_unplug_bio(struct gendisk *disk,
 
 	spin_lock_irqsave(&zwplug->lock, flags);
 
-	/* Schedule submission of the next plugged BIO if we have one. */
-	if (!bio_list_empty(&zwplug->bio_list)) {
-		disk_zone_wplug_schedule_bio_work(disk, zwplug);
-		spin_unlock_irqrestore(&zwplug->lock, flags);
-		return;
-	}
-
-	zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
-
 	/*
-	 * If the zone is full (it was fully written or finished, or empty
-	 * (it was reset), remove its zone write plug from the hash table.
+	 * For rotational devices, signal the BIO completion to the zone write
+	 * plug work. Otherwise, schedule submission of the next plugged BIO
+	 * if we have one.
 	 */
-	if (disk_should_remove_zone_wplug(disk, zwplug))
-		disk_remove_zone_wplug(disk, zwplug);
+	if (bio_list_empty(&zwplug->bio_list))
+		zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
+
+	if (blk_queue_zoned_qd1_writes(disk->queue))
+		complete(&disk->zone_wplugs_worker_bio_done);
+	else if (!bio_list_empty(&zwplug->bio_list))
+		disk_zone_wplug_schedule_work(disk, zwplug);
+
+	if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug))
+		disk_mark_zone_wplug_dead(zwplug);
 
 	spin_unlock_irqrestore(&zwplug->lock, flags);
 }
@@ -1727,10 +1751,9 @@ void blk_zone_write_plug_finish_request(struct request *req)
 	disk_put_zone_wplug(zwplug);
 }
 
-static void blk_zone_wplug_bio_work(struct work_struct *work)
+static bool disk_zone_wplug_submit_bio(struct gendisk *disk,
+				       struct blk_zone_wplug *zwplug)
 {
-	struct blk_zone_wplug *zwplug =
-		container_of(work, struct blk_zone_wplug, bio_work);
 	struct block_device *bdev;
 	unsigned long flags;
 	struct bio *bio;
@@ -1746,7 +1769,7 @@ static void blk_zone_wplug_bio_work(struct work_struct *work)
 	if (!bio) {
 		zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
 		spin_unlock_irqrestore(&zwplug->lock, flags);
-		goto put_zwplug;
+		return false;
 	}
 
 	trace_blk_zone_wplug_bio(zwplug->disk->queue, zwplug->zone_no,
@@ -1760,14 +1783,15 @@ static void blk_zone_wplug_bio_work(struct work_struct *work)
 		goto again;
 	}
 
-	bdev = bio->bi_bdev;
-
 	/*
 	 * blk-mq devices will reuse the extra reference on the request queue
 	 * usage counter we took when the BIO was plugged, but the submission
 	 * path for BIO-based devices will not do that. So drop this extra
 	 * reference here.
 	 */
+	if (blk_queue_zoned_qd1_writes(disk->queue))
+		reinit_completion(&disk->zone_wplugs_worker_bio_done);
+	bdev = bio->bi_bdev;
 	if (bdev_test_flag(bdev, BD_HAS_SUBMIT_BIO)) {
 		bdev->bd_disk->fops->submit_bio(bio);
 		blk_queue_exit(bdev->bd_disk->queue);
@@ -1775,14 +1799,78 @@ static void blk_zone_wplug_bio_work(struct work_struct *work)
 		blk_mq_submit_bio(bio);
 	}
 
-put_zwplug:
-	/* Drop the reference we took in disk_zone_wplug_schedule_bio_work(). */
-	disk_put_zone_wplug(zwplug);
+	return true;
+}
+
+static struct blk_zone_wplug *disk_get_zone_wplugs_work(struct gendisk *disk)
+{
+	struct blk_zone_wplug *zwplug;
+
+	spin_lock_irq(&disk->zone_wplugs_list_lock);
+	zwplug = list_first_entry_or_null(&disk->zone_wplugs_list,
+					  struct blk_zone_wplug, entry);
+	if (zwplug)
+		list_del_init(&zwplug->entry);
+	spin_unlock_irq(&disk->zone_wplugs_list_lock);
+
+	return zwplug;
+}
+
+static int disk_zone_wplugs_worker(void *data)
+{
+	struct gendisk *disk = data;
+	struct blk_zone_wplug *zwplug;
+	unsigned int noio_flag;
+
+	noio_flag = memalloc_noio_save();
+	set_user_nice(current, MIN_NICE);
+	set_freezable();
+
+	for (;;) {
+		set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE);
+
+		zwplug = disk_get_zone_wplugs_work(disk);
+		if (zwplug) {
+			/*
+			 * Process all BIOs of this zone write plug and then
+			 * drop the reference we took when adding the zone write
+			 * plug to the active list.
+			 */
+			set_current_state(TASK_RUNNING);
+			while (disk_zone_wplug_submit_bio(disk, zwplug))
+				blk_wait_io(&disk->zone_wplugs_worker_bio_done);
+			disk_put_zone_wplug(zwplug);
+			continue;
+		}
+
+		/*
+		 * Only sleep if nothing sets the state to running. Else check
+		 * for zone write plugs work again as a newly submitted BIO
+		 * might have added a zone write plug to the work list.
+		 */
+		if (get_current_state() == TASK_RUNNING) {
+			try_to_freeze();
+		} else {
+			if (kthread_should_stop()) {
+				set_current_state(TASK_RUNNING);
+				break;
+			}
+			schedule();
+		}
+	}
+
+	WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list));
+	memalloc_noio_restore(noio_flag);
+
+	return 0;
 }
 
 void disk_init_zone_resources(struct gendisk *disk)
 {
-	spin_lock_init(&disk->zone_wplugs_lock);
+	spin_lock_init(&disk->zone_wplugs_hash_lock);
+	spin_lock_init(&disk->zone_wplugs_list_lock);
+	INIT_LIST_HEAD(&disk->zone_wplugs_list);
+	init_completion(&disk->zone_wplugs_worker_bio_done);
 }
 
 /*
@@ -1798,6 +1886,7 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
 				     unsigned int pool_size)
 {
 	unsigned int i;
+	int ret = -ENOMEM;
 
 	atomic_set(&disk->nr_zone_wplugs, 0);
 	disk->zone_wplugs_hash_bits =
@@ -1823,8 +1912,21 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
 	if (!disk->zone_wplugs_wq)
 		goto destroy_pool;
 
+	disk->zone_wplugs_worker =
+		kthread_create(disk_zone_wplugs_worker, disk,
+			       "%s_zwplugs_worker", disk->disk_name);
+	if (IS_ERR(disk->zone_wplugs_worker)) {
+		ret = PTR_ERR(disk->zone_wplugs_worker);
+		disk->zone_wplugs_worker = NULL;
+		goto destroy_wq;
+	}
+	wake_up_process(disk->zone_wplugs_worker);
+
 	return 0;
 
+destroy_wq:
+	destroy_workqueue(disk->zone_wplugs_wq);
+	disk->zone_wplugs_wq = NULL;
 destroy_pool:
 	mempool_destroy(disk->zone_wplugs_pool);
 	disk->zone_wplugs_pool = NULL;
@@ -1832,7 +1934,7 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
 	kfree(disk->zone_wplugs_hash);
 	disk->zone_wplugs_hash = NULL;
 	disk->zone_wplugs_hash_bits = 0;
-	return -ENOMEM;
+	return ret;
 }
 
 static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk)
@@ -1848,9 +1950,9 @@ static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk)
 		while (!hlist_empty(&disk->zone_wplugs_hash[i])) {
 			zwplug = hlist_entry(disk->zone_wplugs_hash[i].first,
 					     struct blk_zone_wplug, node);
-			refcount_inc(&zwplug->ref);
-			disk_remove_zone_wplug(disk, zwplug);
-			disk_put_zone_wplug(zwplug);
+			spin_lock_irq(&zwplug->lock);
+			disk_mark_zone_wplug_dead(zwplug);
+			spin_unlock_irq(&zwplug->lock);
 		}
 	}
 
@@ -1872,16 +1974,20 @@ static void disk_set_zones_cond_array(struct gendisk *disk, u8 *zones_cond)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);
+	spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags);
 	zones_cond = rcu_replace_pointer(disk->zones_cond, zones_cond,
-				lockdep_is_held(&disk->zone_wplugs_lock));
-	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);
+				lockdep_is_held(&disk->zone_wplugs_hash_lock));
+	spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags);
 
 	kfree_rcu_mightsleep(zones_cond);
 }
 
 void disk_free_zone_resources(struct gendisk *disk)
 {
+	if (disk->zone_wplugs_worker)
+		kthread_stop(disk->zone_wplugs_worker);
+	WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list));
+
 	if (disk->zone_wplugs_wq) {
 		destroy_workqueue(disk->zone_wplugs_wq);
 		disk->zone_wplugs_wq = NULL;
@@ -2078,7 +2184,6 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx,
 	struct gendisk *disk = args->disk;
 	struct blk_zone_wplug *zwplug;
 	unsigned int wp_offset;
-	unsigned long flags;
 
 	/*
 	 * Remember the capacity of the first sequential zone and check
@@ -2108,10 +2213,9 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx,
 	if (!wp_offset || wp_offset >= zone->capacity)
 		return 0;
 
-	zwplug = disk_get_and_lock_zone_wplug(disk, zone->wp, GFP_NOIO, &flags);
+	zwplug = disk_get_or_alloc_zone_wplug(disk, zone->wp, GFP_NOIO);
 	if (!zwplug)
 		return -ENOMEM;
-	spin_unlock_irqrestore(&zwplug->lock, flags);
 	disk_put_zone_wplug(zwplug);
 
 	return 0;
diff --git a/block/blk.h b/block/blk.h
index f6053e9dd2aac..103cb1d0b9cb3 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -108,11 +108,6 @@ static inline void blk_wait_io(struct completion *done)
 struct block_device *blkdev_get_no_open(dev_t dev, bool autoload);
 void blkdev_put_no_open(struct block_device *bdev);
 
-#define BIO_INLINE_VECS 4
-struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
-		gfp_t gfp_mask);
-void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
-
 bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
 		struct page *page, unsigned len, unsigned offset);
 
@@ -699,8 +694,10 @@ int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
 	      const struct blk_holder_ops *hops, struct file *bdev_file);
 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
 
-void blk_integrity_generate(struct bio *bio);
-void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter);
+void bio_integrity_generate(struct bio *bio);
+blk_status_t bio_integrity_verify(struct bio *bio,
+		struct bvec_iter *saved_iter);
+
 void blk_integrity_prepare(struct request *rq);
 void blk_integrity_complete(struct request *rq, unsigned int nr_bytes);
 
diff --git a/block/bsg-lib.c b/block/bsg-lib.c
index 20cd0ef3c394b..fdb4b290ca689 100644
--- a/block/bsg-lib.c
+++ b/block/bsg-lib.c
@@ -393,7 +393,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
 
 	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
 
-	bset->bd = bsg_register_queue(q, dev, name, bsg_transport_sg_io_fn);
+	bset->bd = bsg_register_queue(q, dev, name, bsg_transport_sg_io_fn, NULL);
 	if (IS_ERR(bset->bd)) {
 		ret = PTR_ERR(bset->bd);
 		goto out_cleanup_queue;
diff --git a/block/bsg.c b/block/bsg.c
index e0af6206ed287..82aaf3cee5825 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -12,6 +12,7 @@
 #include <linux/idr.h>
 #include <linux/bsg.h>
 #include <linux/slab.h>
+#include <linux/io_uring/cmd.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_ioctl.h>
@@ -28,6 +29,7 @@ struct bsg_device {
 	unsigned int timeout;
 	unsigned int reserved_size;
 	bsg_sg_io_fn *sg_io_fn;
+	bsg_uring_cmd_fn *uring_cmd_fn;
 };
 
 static inline struct bsg_device *to_bsg_device(struct inode *inode)
@@ -158,11 +160,38 @@ static long bsg_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	}
 }
 
+static int bsg_check_uring_features(unsigned int issue_flags)
+{
+	/* BSG passthrough requires big SQE/CQE support */
+	if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=
+	    (IO_URING_F_SQE128|IO_URING_F_CQE32))
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static int bsg_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
+{
+	struct bsg_device *bd = to_bsg_device(file_inode(ioucmd->file));
+	bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
+	struct request_queue *q = bd->queue;
+	int ret;
+
+	ret = bsg_check_uring_features(issue_flags);
+	if (ret)
+		return ret;
+
+	if (!bd->uring_cmd_fn)
+		return -EOPNOTSUPP;
+
+	return bd->uring_cmd_fn(q, ioucmd, issue_flags, open_for_write);
+}
+
 static const struct file_operations bsg_fops = {
 	.open		=	bsg_open,
 	.release	=	bsg_release,
 	.unlocked_ioctl	=	bsg_ioctl,
 	.compat_ioctl	=	compat_ptr_ioctl,
+	.uring_cmd	=	bsg_uring_cmd,
 	.owner		=	THIS_MODULE,
 	.llseek		=	default_llseek,
 };
@@ -187,7 +216,8 @@ void bsg_unregister_queue(struct bsg_device *bd)
 EXPORT_SYMBOL_GPL(bsg_unregister_queue);
 
 struct bsg_device *bsg_register_queue(struct request_queue *q,
-		struct device *parent, const char *name, bsg_sg_io_fn *sg_io_fn)
+		struct device *parent, const char *name, bsg_sg_io_fn *sg_io_fn,
+		bsg_uring_cmd_fn *uring_cmd_fn)
 {
 	struct bsg_device *bd;
 	int ret;
@@ -199,6 +229,7 @@ struct bsg_device *bsg_register_queue(struct request_queue *q,
 	bd->reserved_size = INT_MAX;
 	bd->queue = q;
 	bd->sg_io_fn = sg_io_fn;
+	bd->uring_cmd_fn = uring_cmd_fn;
 
 	ret = ida_alloc_max(&bsg_minor_ida, BSG_MAX_DEVS - 1, GFP_KERNEL);
 	if (ret < 0) {
diff --git a/block/opal_proto.h b/block/opal_proto.h
index d247a457bf6e3..d138785b81982 100644
--- a/block/opal_proto.h
+++ b/block/opal_proto.h
@@ -125,6 +125,7 @@ enum opal_uid {
 	OPAL_LOCKING_INFO_TABLE,
 	OPAL_ENTERPRISE_LOCKING_INFO_TABLE,
 	OPAL_DATASTORE,
+	OPAL_LOCKING_TABLE,
 	/* C_PIN_TABLE object ID's */
 	OPAL_C_PIN_MSID,
 	OPAL_C_PIN_SID,
@@ -154,6 +155,7 @@ enum opal_method {
 	OPAL_AUTHENTICATE,
 	OPAL_RANDOM,
 	OPAL_ERASE,
+	OPAL_REACTIVATE,
 };
 
 enum opal_token {
@@ -224,6 +226,8 @@ enum opal_lockingstate {
 
 enum opal_parameter {
 	OPAL_SUM_SET_LIST = 0x060000,
+	OPAL_SUM_RANGE_POLICY = 0x060001,
+	OPAL_SUM_ADMIN1_PIN = 0x060002,
 };
 
 enum opal_revertlsp {
diff --git a/block/sed-opal.c b/block/sed-opal.c
index 3ded1ca723caf..c34d19e912018 100644
--- a/block/sed-opal.c
+++ b/block/sed-opal.c
@@ -160,6 +160,8 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = {
 		{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00 },
 	[OPAL_DATASTORE] =
 		{ 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00 },
+	[OPAL_LOCKING_TABLE] =
+		{ 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x00 },
 
 	/* C_PIN_TABLE object ID's */
 	[OPAL_C_PIN_MSID] =
@@ -218,6 +220,8 @@ static const u8 opalmethod[][OPAL_METHOD_LENGTH] = {
 		{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x06, 0x01 },
 	[OPAL_ERASE] =
 		{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x08, 0x03 },
+	[OPAL_REACTIVATE] =
+		{ 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x08, 0x01 },
 };
 
 static int end_opal_session_error(struct opal_dev *dev);
@@ -1514,7 +1518,7 @@ static inline int enable_global_lr(struct opal_dev *dev, u8 *uid,
 	return err;
 }
 
-static int setup_locking_range(struct opal_dev *dev, void *data)
+static int setup_enable_range(struct opal_dev *dev, void *data)
 {
 	u8 uid[OPAL_UID_LENGTH];
 	struct opal_user_lr_setup *setup = data;
@@ -1528,38 +1532,47 @@ static int setup_locking_range(struct opal_dev *dev, void *data)
 
 	if (lr == 0)
 		err = enable_global_lr(dev, uid, setup);
-	else {
-		err = cmd_start(dev, uid, opalmethod[OPAL_SET]);
+	else
+		err = generic_lr_enable_disable(dev, uid, !!setup->RLE, !!setup->WLE, 0, 0);
+	if (err) {
+		pr_debug("Failed to create enable lr command.\n");
+		return err;
+	}
 
-		add_token_u8(&err, dev, OPAL_STARTNAME);
-		add_token_u8(&err, dev, OPAL_VALUES);
-		add_token_u8(&err, dev, OPAL_STARTLIST);
+	return finalize_and_send(dev, parse_and_check_status);
+}
 
-		add_token_u8(&err, dev, OPAL_STARTNAME);
-		add_token_u8(&err, dev, OPAL_RANGESTART);
-		add_token_u64(&err, dev, setup->range_start);
-		add_token_u8(&err, dev, OPAL_ENDNAME);
+static int setup_locking_range_start_length(struct opal_dev *dev, void *data)
+{
+	int err;
+	u8 uid[OPAL_UID_LENGTH];
+	struct opal_user_lr_setup *setup = data;
 
-		add_token_u8(&err, dev, OPAL_STARTNAME);
-		add_token_u8(&err, dev, OPAL_RANGELENGTH);
-		add_token_u64(&err, dev, setup->range_length);
-		add_token_u8(&err, dev, OPAL_ENDNAME);
+	err = build_locking_range(uid, sizeof(uid), setup->session.opal_key.lr);
+	if (err)
+		return err;
 
-		add_token_u8(&err, dev, OPAL_STARTNAME);
-		add_token_u8(&err, dev, OPAL_READLOCKENABLED);
-		add_token_u64(&err, dev, !!setup->RLE);
-		add_token_u8(&err, dev, OPAL_ENDNAME);
+	err = cmd_start(dev, uid, opalmethod[OPAL_SET]);
 
-		add_token_u8(&err, dev, OPAL_STARTNAME);
-		add_token_u8(&err, dev, OPAL_WRITELOCKENABLED);
-		add_token_u64(&err, dev, !!setup->WLE);
-		add_token_u8(&err, dev, OPAL_ENDNAME);
+	add_token_u8(&err, dev, OPAL_STARTNAME);
+	add_token_u8(&err, dev, OPAL_VALUES);
+	add_token_u8(&err, dev, OPAL_STARTLIST);
+
+	add_token_u8(&err, dev, OPAL_STARTNAME);
+	add_token_u8(&err, dev, OPAL_RANGESTART);
+	add_token_u64(&err, dev, setup->range_start);
+	add_token_u8(&err, dev, OPAL_ENDNAME);
+
+	add_token_u8(&err, dev, OPAL_STARTNAME);
+	add_token_u8(&err, dev, OPAL_RANGELENGTH);
+	add_token_u64(&err, dev, setup->range_length);
+	add_token_u8(&err, dev, OPAL_ENDNAME);
+
+	add_token_u8(&err, dev, OPAL_ENDLIST);
+	add_token_u8(&err, dev, OPAL_ENDNAME);
 
-		add_token_u8(&err, dev, OPAL_ENDLIST);
-		add_token_u8(&err, dev, OPAL_ENDNAME);
-	}
 	if (err) {
-		pr_debug("Error building Setup Locking range command.\n");
+		pr_debug("Error building Setup Locking RangeStartLength command.\n");
 		return err;
 	}
 
@@ -1568,7 +1581,7 @@ static int setup_locking_range(struct opal_dev *dev, void *data)
 
 static int response_get_column(const struct parsed_resp *resp,
 			       int *iter,
-			       u8 column,
+			       u64 column,
 			       u64 *value)
 {
 	const struct opal_resp_tok *tok;
@@ -1586,7 +1599,7 @@ static int response_get_column(const struct parsed_resp *resp,
 	n++;
 
 	if (response_get_u64(resp, n) != column) {
-		pr_debug("Token %d does not match expected column %u.\n",
+		pr_debug("Token %d does not match expected column %llu.\n",
 			 n, column);
 		return OPAL_INVAL_PARAM;
 	}
@@ -1744,6 +1757,12 @@ static int start_anybodyASP_opal_session(struct opal_dev *dev, void *data)
 					  OPAL_ADMINSP_UID, NULL, 0);
 }
 
+static int start_anybodyLSP_opal_session(struct opal_dev *dev, void *data)
+{
+	return start_generic_opal_session(dev, OPAL_ANYBODY_UID,
+					  OPAL_LOCKINGSP_UID, NULL, 0);
+}
+
 static int start_SIDASP_opal_session(struct opal_dev *dev, void *data)
 {
 	int ret;
@@ -2285,6 +2304,74 @@ static int activate_lsp(struct opal_dev *dev, void *data)
 	return finalize_and_send(dev, parse_and_check_status);
 }
 
+static int reactivate_lsp(struct opal_dev *dev, void *data)
+{
+	struct opal_lr_react *opal_react = data;
+	u8 user_lr[OPAL_UID_LENGTH];
+	int err, i;
+
+	err = cmd_start(dev, opaluid[OPAL_THISSP_UID],
+			opalmethod[OPAL_REACTIVATE]);
+
+	if (err) {
+		pr_debug("Error building Reactivate LockingSP command.\n");
+		return err;
+	}
+
+	/*
+	 * If neither 'entire_table' nor 'num_lrs' is set, the device
+	 * gets reactivated with SUM disabled. Only Admin1PIN will change
+	 * if set.
+	 */
+	if (opal_react->entire_table) {
+		/* Entire Locking table (all locking ranges) will be put in SUM. */
+		add_token_u8(&err, dev, OPAL_STARTNAME);
+		add_token_u64(&err, dev, OPAL_SUM_SET_LIST);
+		add_token_bytestring(&err, dev, opaluid[OPAL_LOCKING_TABLE], OPAL_UID_LENGTH);
+		add_token_u8(&err, dev, OPAL_ENDNAME);
+	} else if (opal_react->num_lrs) {
+		/* Subset of Locking table (selected locking range(s)) to be put in SUM */
+		err = build_locking_range(user_lr, sizeof(user_lr),
+					  opal_react->lr[0]);
+		if (err)
+			return err;
+
+		add_token_u8(&err, dev, OPAL_STARTNAME);
+		add_token_u64(&err, dev, OPAL_SUM_SET_LIST);
+
+		add_token_u8(&err, dev, OPAL_STARTLIST);
+		add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH);
+		for (i = 1; i < opal_react->num_lrs; i++) {
+			user_lr[7] = opal_react->lr[i];
+			add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH);
+		}
+		add_token_u8(&err, dev, OPAL_ENDLIST);
+		add_token_u8(&err, dev, OPAL_ENDNAME);
+	}
+
+	/* Skipping the rangle policy parameter is same as setting its value to zero */
+	if (opal_react->range_policy && (opal_react->num_lrs || opal_react->entire_table)) {
+		add_token_u8(&err, dev, OPAL_STARTNAME);
+		add_token_u64(&err, dev, OPAL_SUM_RANGE_POLICY);
+		add_token_u8(&err, dev, 1);
+		add_token_u8(&err, dev, OPAL_ENDNAME);
+	}
+
+	/*
+	 * Optional parameter. If set, it changes the Admin1 PIN even when SUM
+	 * is being disabled.
+	 */
+	if (opal_react->new_admin_key.key_len) {
+		add_token_u8(&err, dev, OPAL_STARTNAME);
+		add_token_u64(&err, dev, OPAL_SUM_ADMIN1_PIN);
+		add_token_bytestring(&err, dev, opal_react->new_admin_key.key,
+				     opal_react->new_admin_key.key_len);
+		add_token_u8(&err, dev, OPAL_ENDNAME);
+	}
+
+	return finalize_and_send(dev, parse_and_check_status);
+}
+
 /* Determine if we're in the Manufactured Inactive or Active state */
 static int get_lsp_lifecycle(struct opal_dev *dev, void *data)
 {
@@ -2955,12 +3042,92 @@ static int opal_activate_lsp(struct opal_dev *dev,
 	return ret;
 }
 
+static int opal_reactivate_lsp(struct opal_dev *dev,
+			       struct opal_lr_react *opal_lr_react)
+{
+	const struct opal_step active_steps[] = {
+		{ start_admin1LSP_opal_session, &opal_lr_react->key },
+		{ reactivate_lsp, opal_lr_react },
+		/* No end_opal_session. The controller terminates the session */
+	};
+	int ret;
+
+	/* use either 'entire_table' parameter or set of locking ranges */
+	if (opal_lr_react->num_lrs > OPAL_MAX_LRS ||
+	    (opal_lr_react->num_lrs && opal_lr_react->entire_table))
+		return -EINVAL;
+
+	ret = opal_get_key(dev, &opal_lr_react->key);
+	if (ret)
+		return ret;
+	mutex_lock(&dev->dev_lock);
+	setup_opal_dev(dev);
+	ret = execute_steps(dev, active_steps, ARRAY_SIZE(active_steps));
+	mutex_unlock(&dev->dev_lock);
+
+	return ret;
+}
+
 static int opal_setup_locking_range(struct opal_dev *dev,
 				    struct opal_user_lr_setup *opal_lrs)
 {
 	const struct opal_step lr_steps[] = {
 		{ start_auth_opal_session, &opal_lrs->session },
-		{ setup_locking_range, opal_lrs },
+		{ setup_locking_range_start_length, opal_lrs },
+		{ setup_enable_range, opal_lrs },
+		{ end_opal_session, }
+	}, lr_global_steps[] = {
+		{ start_auth_opal_session, &opal_lrs->session },
+		{ setup_enable_range, opal_lrs },
+		{ end_opal_session, }
+	};
+	int ret;
+
+	ret = opal_get_key(dev, &opal_lrs->session.opal_key);
+	if (ret)
+		return ret;
+	mutex_lock(&dev->dev_lock);
+	setup_opal_dev(dev);
+	if (opal_lrs->session.opal_key.lr == 0)
+		ret = execute_steps(dev, lr_global_steps, ARRAY_SIZE(lr_global_steps));
+	else
+		ret = execute_steps(dev, lr_steps, ARRAY_SIZE(lr_steps));
+	mutex_unlock(&dev->dev_lock);
+
+	return ret;
+}
+
+static int opal_setup_locking_range_start_length(struct opal_dev *dev,
+				    struct opal_user_lr_setup *opal_lrs)
+{
+	const struct opal_step lr_steps[] = {
+		{ start_auth_opal_session, &opal_lrs->session },
+		{ setup_locking_range_start_length, opal_lrs },
+		{ end_opal_session, }
+	};
+	int ret;
+
+	/* we can not set global locking range offset or length */
+	if (opal_lrs->session.opal_key.lr == 0)
+		return -EINVAL;
+
+	ret = opal_get_key(dev, &opal_lrs->session.opal_key);
+	if (ret)
+		return ret;
+	mutex_lock(&dev->dev_lock);
+	setup_opal_dev(dev);
+	ret = execute_steps(dev, lr_steps, ARRAY_SIZE(lr_steps));
+	mutex_unlock(&dev->dev_lock);
+
+	return ret;
+}
+
+static int opal_enable_disable_range(struct opal_dev *dev,
+			     struct opal_user_lr_setup *opal_lrs)
+{
+	const struct opal_step lr_steps[] = {
+		{ start_auth_opal_session, &opal_lrs->session },
+		{ setup_enable_range, opal_lrs },
 		{ end_opal_session, }
 	};
 	int ret;
@@ -3228,6 +3395,156 @@ static int opal_get_geometry(struct opal_dev *dev, void __user *data)
 	return 0;
 }
 
+static int get_sum_ranges(struct opal_dev *dev, void *data)
+{
+	const char *lr_uid;
+	size_t lr_uid_len;
+	u64 val;
+	const struct opal_resp_tok *tok;
+	int err, tok_n = 2;
+	struct opal_sum_ranges *sranges = data;
+	const __u8 lr_all[OPAL_MAX_LRS] = { 0, 1, 2, 3, 4, 5, 6, 7, 8 };
+
+	err = generic_get_columns(dev, opaluid[OPAL_LOCKING_INFO_TABLE], OPAL_SUM_SET_LIST,
+				  OPAL_SUM_RANGE_POLICY);
+	if (err) {
+		pr_debug("Couldn't get locking info table columns %d to %d.\n",
+			 OPAL_SUM_SET_LIST, OPAL_SUM_RANGE_POLICY);
+		return err;
+	}
+
+	tok = response_get_token(&dev->parsed, tok_n);
+	if (IS_ERR(tok))
+		return PTR_ERR(tok);
+
+	if (!response_token_matches(tok, OPAL_STARTNAME)) {
+		pr_debug("Unexpected response token type %d.\n", tok_n);
+		return OPAL_INVAL_PARAM;
+	}
+	tok_n++;
+
+	if (response_get_u64(&dev->parsed, tok_n) != OPAL_SUM_SET_LIST) {
+		pr_debug("Token %d does not match expected column %u.\n",
+			 tok_n, OPAL_SUM_SET_LIST);
+		return OPAL_INVAL_PARAM;
+	}
+	tok_n++;
+
+	tok = response_get_token(&dev->parsed, tok_n);
+	if (IS_ERR(tok))
+		return PTR_ERR(tok);
+
+	/*
+	 * The OPAL_SUM_SET_LIST response contains two distinct values:
+	 *
+	 *  - the list of individual locking ranges (UIDs) put in SUM. The list
+	 *    may also be empty signaling the SUM is disabled.
+	 *
+	 *  - the Locking table UID if the entire Locking table is put in SUM.
+	 */
+	if (response_token_matches(tok, OPAL_STARTLIST)) {
+		sranges->num_lrs = 0;
+
+		tok_n++;
+		tok = response_get_token(&dev->parsed, tok_n);
+		if (IS_ERR(tok))
+			return PTR_ERR(tok);
+
+		while (!response_token_matches(tok, OPAL_ENDLIST)) {
+			lr_uid_len = response_get_string(&dev->parsed, tok_n, &lr_uid);
+			if (lr_uid_len != OPAL_UID_LENGTH) {
+				pr_debug("Unexpected response token type %d.\n", tok_n);
+				return OPAL_INVAL_PARAM;
+			}
+
+			if (memcmp(lr_uid, opaluid[OPAL_LOCKINGRANGE_GLOBAL], OPAL_UID_LENGTH)) {
+				if (lr_uid[5] != LOCKING_RANGE_NON_GLOBAL) {
+					pr_debug("Unexpected byte %d at LR UUID position 5.\n",
+						 lr_uid[5]);
+					return OPAL_INVAL_PARAM;
+				}
+				sranges->lr[sranges->num_lrs++] = lr_uid[7];
+			} else
+				sranges->lr[sranges->num_lrs++] = 0;
+
+			tok_n++;
+			tok = response_get_token(&dev->parsed, tok_n);
+			if (IS_ERR(tok))
+				return PTR_ERR(tok);
+		}
+	} else {
+		/* Only OPAL_LOCKING_TABLE UID is an alternative to OPAL_STARTLIST here. */
+		lr_uid_len = response_get_string(&dev->parsed, tok_n, &lr_uid);
+		if (lr_uid_len != OPAL_UID_LENGTH) {
+			pr_debug("Unexpected response token type %d.\n", tok_n);
+			return OPAL_INVAL_PARAM;
+		}
+
+		if (memcmp(lr_uid, opaluid[OPAL_LOCKING_TABLE], OPAL_UID_LENGTH)) {
+			pr_debug("Unexpected response UID.\n");
+			return OPAL_INVAL_PARAM;
+		}
+
+		/* sed-opal kernel API already provides following limit in Activate command */
+		sranges->num_lrs = OPAL_MAX_LRS;
+		memcpy(sranges->lr, lr_all, OPAL_MAX_LRS);
+	}
+	tok_n++;
+
+	tok = response_get_token(&dev->parsed, tok_n);
+	if (IS_ERR(tok))
+		return PTR_ERR(tok);
+
+	if (!response_token_matches(tok, OPAL_ENDNAME)) {
+		pr_debug("Unexpected response token type %d.\n", tok_n);
+		return OPAL_INVAL_PARAM;
+	}
+	tok_n++;
+
+	err = response_get_column(&dev->parsed, &tok_n, OPAL_SUM_RANGE_POLICY, &val);
+	if (err)
+		return err;
+
+	sranges->range_policy = val ? 1 : 0;
+
+	return 0;
+}
+
+static int opal_get_sum_ranges(struct opal_dev *dev, struct opal_sum_ranges *opal_sum_rngs,
+			       void __user *data)
+{
+	const struct opal_step admin_steps[] = {
+		{ start_admin1LSP_opal_session, &opal_sum_rngs->key },
+		{ get_sum_ranges, opal_sum_rngs },
+		{ end_opal_session, }
+	}, anybody_steps[] = {
+		{ start_anybodyLSP_opal_session, NULL },
+		{ get_sum_ranges, opal_sum_rngs },
+		{ end_opal_session, }
+	};
+	int ret;
+
+	mutex_lock(&dev->dev_lock);
+	setup_opal_dev(dev);
+	if (opal_sum_rngs->key.key_len)
+		/* Use Admin1 session (authenticated by PIN) to retrieve LockingInfo columns */
+		ret = execute_steps(dev, admin_steps, ARRAY_SIZE(admin_steps));
+	else
+		/* Use Anybody session (no key) to retrieve LockingInfo columns */
+		ret = execute_steps(dev, anybody_steps, ARRAY_SIZE(anybody_steps));
+	mutex_unlock(&dev->dev_lock);
+
+	/* skip session info when copying back to uspace */
+	if (!ret && copy_to_user(data + offsetof(struct opal_sum_ranges, num_lrs),
+				(void *)opal_sum_rngs + offsetof(struct opal_sum_ranges, num_lrs),
+				sizeof(*opal_sum_rngs) - offsetof(struct opal_sum_ranges, num_lrs))) {
+		pr_debug("Error copying SUM ranges info to userspace\n");
+		return -EFAULT;
+	}
+
+	return ret;
+}
+
 int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg)
 {
 	void *p;
@@ -3313,6 +3630,18 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg)
 	case IOC_OPAL_SET_SID_PW:
 		ret = opal_set_new_sid_pw(dev, p);
 		break;
+	case IOC_OPAL_REACTIVATE_LSP:
+		ret = opal_reactivate_lsp(dev, p);
+		break;
+	case IOC_OPAL_LR_SET_START_LEN:
+		ret = opal_setup_locking_range_start_length(dev, p);
+		break;
+	case IOC_OPAL_ENABLE_DISABLE_LR:
+		ret = opal_enable_disable_range(dev, p);
+		break;
+	case IOC_OPAL_GET_SUM_STATUS:
+		ret = opal_get_sum_ranges(dev, p, arg);
+		break;
 
 	default:
 		break;
diff --git a/block/t10-pi.c b/block/t10-pi.c
index 0c4ed97021460..a19b4e102a837 100644
--- a/block/t10-pi.c
+++ b/block/t10-pi.c
@@ -12,462 +12,556 @@
 #include <linux/unaligned.h>
 #include "blk.h"
 
+#define APP_TAG_ESCAPE 0xffff
+#define REF_TAG_ESCAPE 0xffffffff
+
+/*
+ * This union is used for onstack allocations when the pi field is split across
+ * segments. blk_validate_integrity_limits() guarantees pi_tuple_size matches
+ * the sizeof one of these two types.
+ */
+union pi_tuple {
+	struct crc64_pi_tuple	crc64_pi;
+	struct t10_pi_tuple	t10_pi;
+};
+
 struct blk_integrity_iter {
-	void			*prot_buf;
-	void			*data_buf;
-	sector_t		seed;
-	unsigned int		data_size;
-	unsigned short		interval;
-	const char		*disk_name;
+	struct bio			*bio;
+	struct bio_integrity_payload	*bip;
+	struct blk_integrity		*bi;
+	struct bvec_iter		data_iter;
+	struct bvec_iter		prot_iter;
+	unsigned int			interval_remaining;
+	u64				seed;
+	u64				csum;
 };
 
-static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,
-		unsigned char csum_type)
+static void blk_calculate_guard(struct blk_integrity_iter *iter, void *data,
+				unsigned int len)
 {
-	if (csum_type == BLK_INTEGRITY_CSUM_IP)
-		return (__force __be16)ip_compute_csum(data, len);
-	return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));
+	switch (iter->bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+		iter->csum = crc64_nvme(iter->csum, data, len);
+		break;
+	case BLK_INTEGRITY_CSUM_CRC:
+		iter->csum = crc_t10dif_update(iter->csum, data, len);
+		break;
+	case BLK_INTEGRITY_CSUM_IP:
+		iter->csum = (__force u32)csum_partial(data, len,
+						(__force __wsum)iter->csum);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		iter->csum = U64_MAX;
+		break;
+	}
+}
+
+static void blk_integrity_csum_finish(struct blk_integrity_iter *iter)
+{
+	switch (iter->bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_IP:
+		iter->csum = (__force u16)csum_fold((__force __wsum)iter->csum);
+		break;
+	default:
+		break;
+	}
 }
 
 /*
- * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
- * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
- * tag.
+ * Update the csum for formats that have metadata padding in front of the data
+ * integrity field
  */
-static void t10_pi_generate(struct blk_integrity_iter *iter,
-		struct blk_integrity *bi)
+static void blk_integrity_csum_offset(struct blk_integrity_iter *iter)
 {
-	u8 offset = bi->pi_offset;
-	unsigned int i;
+	unsigned int offset = iter->bi->pi_offset;
+	struct bio_vec *bvec = iter->bip->bip_vec;
+
+	while (offset > 0) {
+		struct bio_vec pbv = bvec_iter_bvec(bvec, iter->prot_iter);
+		unsigned int len = min(pbv.bv_len, offset);
+		void *prot_buf = bvec_kmap_local(&pbv);
+
+		blk_calculate_guard(iter, prot_buf, len);
+		kunmap_local(prot_buf);
+		offset -= len;
+		bvec_iter_advance_single(bvec, &iter->prot_iter, len);
+	}
+	blk_integrity_csum_finish(iter);
+}
 
-	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
-		struct t10_pi_tuple *pi = iter->prot_buf + offset;
+static void blk_integrity_copy_from_tuple(struct bio_integrity_payload *bip,
+					  struct bvec_iter *iter, void *tuple,
+					  unsigned int tuple_size)
+{
+	while (tuple_size) {
+		struct bio_vec pbv = bvec_iter_bvec(bip->bip_vec, *iter);
+		unsigned int len = min(tuple_size, pbv.bv_len);
+		void *prot_buf = bvec_kmap_local(&pbv);
+
+		memcpy(prot_buf, tuple, len);
+		kunmap_local(prot_buf);
+		bvec_iter_advance_single(bip->bip_vec, iter, len);
+		tuple_size -= len;
+		tuple += len;
+	}
+}
 
-		pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval,
-				bi->csum_type);
-		if (offset)
-			pi->guard_tag = t10_pi_csum(pi->guard_tag,
-					iter->prot_buf, offset, bi->csum_type);
-		pi->app_tag = 0;
+static void blk_integrity_copy_to_tuple(struct bio_integrity_payload *bip,
+					struct bvec_iter *iter, void *tuple,
+					unsigned int tuple_size)
+{
+	while (tuple_size) {
+		struct bio_vec pbv = bvec_iter_bvec(bip->bip_vec, *iter);
+		unsigned int len = min(tuple_size, pbv.bv_len);
+		void *prot_buf = bvec_kmap_local(&pbv);
+
+		memcpy(tuple, prot_buf, len);
+		kunmap_local(prot_buf);
+		bvec_iter_advance_single(bip->bip_vec, iter, len);
+		tuple_size -= len;
+		tuple += len;
+	}
+}
 
-		if (bi->flags & BLK_INTEGRITY_REF_TAG)
-			pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
-		else
-			pi->ref_tag = 0;
+static bool ext_pi_ref_escape(const u8 ref_tag[6])
+{
+	static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
-		iter->data_buf += iter->interval;
-		iter->prot_buf += bi->metadata_size;
-		iter->seed++;
-	}
+	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
 }
 
-static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
-		struct blk_integrity *bi)
-{
-	u8 offset = bi->pi_offset;
-	unsigned int i;
-
-	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
-		struct t10_pi_tuple *pi = iter->prot_buf + offset;
-		__be16 csum;
-
-		if (bi->flags & BLK_INTEGRITY_REF_TAG) {
-			if (pi->app_tag == T10_PI_APP_ESCAPE)
-				goto next;
-
-			if (be32_to_cpu(pi->ref_tag) !=
-			    lower_32_bits(iter->seed)) {
-				pr_err("%s: ref tag error at location %llu " \
-				       "(rcvd %u)\n", iter->disk_name,
-				       (unsigned long long)
-				       iter->seed, be32_to_cpu(pi->ref_tag));
-				return BLK_STS_PROTECTION;
-			}
-		} else {
-			if (pi->app_tag == T10_PI_APP_ESCAPE &&
-			    pi->ref_tag == T10_PI_REF_ESCAPE)
-				goto next;
+static blk_status_t blk_verify_ext_pi(struct blk_integrity_iter *iter,
+				      struct crc64_pi_tuple *pi)
+{
+	u64 seed = lower_48_bits(iter->seed);
+	u64 guard = get_unaligned_be64(&pi->guard_tag);
+	u64 ref = get_unaligned_be48(pi->ref_tag);
+	u16 app = get_unaligned_be16(&pi->app_tag);
+
+	if (iter->bi->flags & BLK_INTEGRITY_REF_TAG) {
+		if (app == APP_TAG_ESCAPE)
+			return BLK_STS_OK;
+		if (ref != seed) {
+			pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
+				iter->bio->bi_bdev->bd_disk->disk_name, seed,
+				ref);
+			return BLK_STS_PROTECTION;
 		}
+	} else if (app == APP_TAG_ESCAPE && ext_pi_ref_escape(pi->ref_tag)) {
+		return BLK_STS_OK;
+	}
+
+	if (guard != iter->csum) {
+		pr_err("%s: guard tag error at sector %llu (rcvd %016llx, want %016llx)\n",
+			iter->bio->bi_bdev->bd_disk->disk_name, iter->seed,
+			guard, iter->csum);
+		return BLK_STS_PROTECTION;
+	}
+
+	return BLK_STS_OK;
+}
 
-		csum = t10_pi_csum(0, iter->data_buf, iter->interval,
-				bi->csum_type);
-		if (offset)
-			csum = t10_pi_csum(csum, iter->prot_buf, offset,
-					bi->csum_type);
-
-		if (pi->guard_tag != csum) {
-			pr_err("%s: guard tag error at sector %llu " \
-			       "(rcvd %04x, want %04x)\n", iter->disk_name,
-			       (unsigned long long)iter->seed,
-			       be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
+static blk_status_t blk_verify_pi(struct blk_integrity_iter *iter,
+				      struct t10_pi_tuple *pi, u16 guard)
+{
+	u32 seed = lower_32_bits(iter->seed);
+	u32 ref = get_unaligned_be32(&pi->ref_tag);
+	u16 app = get_unaligned_be16(&pi->app_tag);
+
+	if (iter->bi->flags & BLK_INTEGRITY_REF_TAG) {
+		if (app == APP_TAG_ESCAPE)
+			return BLK_STS_OK;
+		if (ref != seed) {
+			pr_err("%s: ref tag error at location %u (rcvd %u)\n",
+				iter->bio->bi_bdev->bd_disk->disk_name, seed,
+				ref);
 			return BLK_STS_PROTECTION;
 		}
+	} else if (app == APP_TAG_ESCAPE && ref == REF_TAG_ESCAPE) {
+		return BLK_STS_OK;
+	}
 
-next:
-		iter->data_buf += iter->interval;
-		iter->prot_buf += bi->metadata_size;
-		iter->seed++;
+	if (guard != (u16)iter->csum) {
+		pr_err("%s: guard tag error at sector %llu (rcvd %04x, want %04x)\n",
+			iter->bio->bi_bdev->bd_disk->disk_name, iter->seed,
+			guard, (u16)iter->csum);
+		return BLK_STS_PROTECTION;
 	}
 
 	return BLK_STS_OK;
 }
 
-/**
- * t10_pi_type1_prepare - prepare PI prior submitting request to device
- * @rq:              request with PI that should be prepared
- *
- * For Type 1/Type 2, the virtual start sector is the one that was
- * originally submitted by the block layer for the ref_tag usage. Due to
- * partitioning, MD/DM cloning, etc. the actual physical start sector is
- * likely to be different. Remap protection information to match the
- * physical LBA.
- */
-static void t10_pi_type1_prepare(struct request *rq)
+static blk_status_t blk_verify_t10_pi(struct blk_integrity_iter *iter,
+				      struct t10_pi_tuple *pi)
 {
-	struct blk_integrity *bi = &rq->q->limits.integrity;
-	const int tuple_sz = bi->metadata_size;
-	u32 ref_tag = t10_pi_ref_tag(rq);
-	u8 offset = bi->pi_offset;
-	struct bio *bio;
+	u16 guard = get_unaligned_be16(&pi->guard_tag);
 
-	__rq_for_each_bio(bio, rq) {
-		struct bio_integrity_payload *bip = bio_integrity(bio);
-		u32 virt = bip_get_seed(bip) & 0xffffffff;
-		struct bio_vec iv;
-		struct bvec_iter iter;
+	return blk_verify_pi(iter, pi, guard);
+}
 
-		/* Already remapped? */
-		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
-			break;
+static blk_status_t blk_verify_ip_pi(struct blk_integrity_iter *iter,
+				     struct t10_pi_tuple *pi)
+{
+	u16 guard = get_unaligned((u16 *)&pi->guard_tag);
 
-		bip_for_each_vec(iv, bip, iter) {
-			unsigned int j;
-			void *p;
-
-			p = bvec_kmap_local(&iv);
-			for (j = 0; j < iv.bv_len; j += tuple_sz) {
-				struct t10_pi_tuple *pi = p + offset;
-
-				if (be32_to_cpu(pi->ref_tag) == virt)
-					pi->ref_tag = cpu_to_be32(ref_tag);
-				virt++;
-				ref_tag++;
-				p += tuple_sz;
-			}
-			kunmap_local(p);
-		}
+	return blk_verify_pi(iter, pi, guard);
+}
 
-		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
+static blk_status_t blk_integrity_verify(struct blk_integrity_iter *iter,
+					 union pi_tuple *tuple)
+{
+	switch (iter->bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+		return blk_verify_ext_pi(iter, &tuple->crc64_pi);
+	case BLK_INTEGRITY_CSUM_CRC:
+		return blk_verify_t10_pi(iter, &tuple->t10_pi);
+	case BLK_INTEGRITY_CSUM_IP:
+		return blk_verify_ip_pi(iter, &tuple->t10_pi);
+	default:
+		return BLK_STS_OK;
 	}
 }
 
-/**
- * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
- * @rq:              request with PI that should be prepared
- * @nr_bytes:        total bytes to prepare
- *
- * For Type 1/Type 2, the virtual start sector is the one that was
- * originally submitted by the block layer for the ref_tag usage. Due to
- * partitioning, MD/DM cloning, etc. the actual physical start sector is
- * likely to be different. Since the physical start sector was submitted
- * to the device, we should remap it back to virtual values expected by the
- * block layer.
- */
-static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
+static void blk_set_ext_pi(struct blk_integrity_iter *iter,
+			   struct crc64_pi_tuple *pi)
 {
-	struct blk_integrity *bi = &rq->q->limits.integrity;
-	unsigned intervals = nr_bytes >> bi->interval_exp;
-	const int tuple_sz = bi->metadata_size;
-	u32 ref_tag = t10_pi_ref_tag(rq);
-	u8 offset = bi->pi_offset;
-	struct bio *bio;
+	put_unaligned_be64(iter->csum, &pi->guard_tag);
+	put_unaligned_be16(0, &pi->app_tag);
+	put_unaligned_be48(iter->seed, &pi->ref_tag);
+}
 
-	__rq_for_each_bio(bio, rq) {
-		struct bio_integrity_payload *bip = bio_integrity(bio);
-		u32 virt = bip_get_seed(bip) & 0xffffffff;
-		struct bio_vec iv;
-		struct bvec_iter iter;
-
-		bip_for_each_vec(iv, bip, iter) {
-			unsigned int j;
-			void *p;
-
-			p = bvec_kmap_local(&iv);
-			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
-				struct t10_pi_tuple *pi = p + offset;
-
-				if (be32_to_cpu(pi->ref_tag) == ref_tag)
-					pi->ref_tag = cpu_to_be32(virt);
-				virt++;
-				ref_tag++;
-				intervals--;
-				p += tuple_sz;
-			}
-			kunmap_local(p);
-		}
-	}
+static void blk_set_pi(struct blk_integrity_iter *iter,
+		       struct t10_pi_tuple *pi, __be16 csum)
+{
+	put_unaligned(csum, &pi->guard_tag);
+	put_unaligned_be16(0, &pi->app_tag);
+	put_unaligned_be32(iter->seed, &pi->ref_tag);
 }
 
-static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
+static void blk_set_t10_pi(struct blk_integrity_iter *iter,
+			   struct t10_pi_tuple *pi)
 {
-	return cpu_to_be64(crc64_nvme(crc, data, len));
+	blk_set_pi(iter, pi, cpu_to_be16((u16)iter->csum));
 }
 
-static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
-		struct blk_integrity *bi)
+static void blk_set_ip_pi(struct blk_integrity_iter *iter,
+			  struct t10_pi_tuple *pi)
 {
-	u8 offset = bi->pi_offset;
-	unsigned int i;
+	blk_set_pi(iter, pi, (__force __be16)(u16)iter->csum);
+}
 
-	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
-		struct crc64_pi_tuple *pi = iter->prot_buf + offset;
+static void blk_integrity_set(struct blk_integrity_iter *iter,
+			      union pi_tuple *tuple)
+{
+	switch (iter->bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+		return blk_set_ext_pi(iter, &tuple->crc64_pi);
+	case BLK_INTEGRITY_CSUM_CRC:
+		return blk_set_t10_pi(iter, &tuple->t10_pi);
+	case BLK_INTEGRITY_CSUM_IP:
+		return blk_set_ip_pi(iter, &tuple->t10_pi);
+	default:
+		WARN_ON_ONCE(1);
+		return;
+	}
+}
 
-		pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval);
-		if (offset)
-			pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),
-					iter->prot_buf, offset);
-		pi->app_tag = 0;
+static blk_status_t blk_integrity_interval(struct blk_integrity_iter *iter,
+					   bool verify)
+{
+	blk_status_t ret = BLK_STS_OK;
+	union pi_tuple tuple;
+	void *ptuple = &tuple;
+	struct bio_vec pbv;
+
+	blk_integrity_csum_offset(iter);
+	pbv = bvec_iter_bvec(iter->bip->bip_vec, iter->prot_iter);
+	if (pbv.bv_len >= iter->bi->pi_tuple_size) {
+		ptuple = bvec_kmap_local(&pbv);
+		bvec_iter_advance_single(iter->bip->bip_vec, &iter->prot_iter,
+				iter->bi->metadata_size - iter->bi->pi_offset);
+	} else if (verify) {
+		blk_integrity_copy_to_tuple(iter->bip, &iter->prot_iter,
+				ptuple, iter->bi->pi_tuple_size);
+	}
 
-		if (bi->flags & BLK_INTEGRITY_REF_TAG)
-			put_unaligned_be48(iter->seed, pi->ref_tag);
-		else
-			put_unaligned_be48(0ULL, pi->ref_tag);
+	if (verify)
+		ret = blk_integrity_verify(iter, ptuple);
+	else
+		blk_integrity_set(iter, ptuple);
 
-		iter->data_buf += iter->interval;
-		iter->prot_buf += bi->metadata_size;
-		iter->seed++;
+	if (likely(ptuple != &tuple)) {
+		kunmap_local(ptuple);
+	} else if (!verify) {
+		blk_integrity_copy_from_tuple(iter->bip, &iter->prot_iter,
+				ptuple, iter->bi->pi_tuple_size);
 	}
+
+	iter->interval_remaining = 1 << iter->bi->interval_exp;
+	iter->csum = 0;
+	iter->seed++;
+	return ret;
 }
 
-static bool ext_pi_ref_escape(const u8 ref_tag[6])
+static blk_status_t blk_integrity_iterate(struct bio *bio,
+					  struct bvec_iter *data_iter,
+					  bool verify)
 {
-	static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+	struct blk_integrity_iter iter = {
+		.bio = bio,
+		.bip = bip,
+		.bi = bi,
+		.data_iter = *data_iter,
+		.prot_iter = bip->bip_iter,
+		.interval_remaining = 1 << bi->interval_exp,
+		.seed = data_iter->bi_sector,
+		.csum = 0,
+	};
+	blk_status_t ret = BLK_STS_OK;
+
+	while (iter.data_iter.bi_size && ret == BLK_STS_OK) {
+		struct bio_vec bv = bvec_iter_bvec(iter.bio->bi_io_vec,
+						   iter.data_iter);
+		void *kaddr = bvec_kmap_local(&bv);
+		void *data = kaddr;
+		unsigned int len;
+
+		bvec_iter_advance_single(iter.bio->bi_io_vec, &iter.data_iter,
+					 bv.bv_len);
+		while (bv.bv_len && ret == BLK_STS_OK) {
+			len = min(iter.interval_remaining, bv.bv_len);
+			blk_calculate_guard(&iter, data, len);
+			bv.bv_len -= len;
+			data += len;
+			iter.interval_remaining -= len;
+			if (!iter.interval_remaining)
+				ret = blk_integrity_interval(&iter, verify);
+		}
+		kunmap_local(kaddr);
+	}
 
-	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
+	return ret;
 }
 
-static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
-		struct blk_integrity *bi)
-{
-	u8 offset = bi->pi_offset;
-	unsigned int i;
-
-	for (i = 0; i < iter->data_size; i += iter->interval) {
-		struct crc64_pi_tuple *pi = iter->prot_buf + offset;
-		u64 ref, seed;
-		__be64 csum;
-
-		if (bi->flags & BLK_INTEGRITY_REF_TAG) {
-			if (pi->app_tag == T10_PI_APP_ESCAPE)
-				goto next;
-
-			ref = get_unaligned_be48(pi->ref_tag);
-			seed = lower_48_bits(iter->seed);
-			if (ref != seed) {
-				pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
-					iter->disk_name, seed, ref);
-				return BLK_STS_PROTECTION;
-			}
-		} else {
-			if (pi->app_tag == T10_PI_APP_ESCAPE &&
-			    ext_pi_ref_escape(pi->ref_tag))
-				goto next;
-		}
+void bio_integrity_generate(struct bio *bio)
+{
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 
-		csum = ext_pi_crc64(0, iter->data_buf, iter->interval);
-		if (offset)
-			csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf,
-					    offset);
+	switch (bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+	case BLK_INTEGRITY_CSUM_CRC:
+	case BLK_INTEGRITY_CSUM_IP:
+		blk_integrity_iterate(bio, &bio->bi_iter, false);
+		break;
+	default:
+		break;
+	}
+}
 
-		if (pi->guard_tag != csum) {
-			pr_err("%s: guard tag error at sector %llu " \
-			       "(rcvd %016llx, want %016llx)\n",
-				iter->disk_name, (unsigned long long)iter->seed,
-				be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
-			return BLK_STS_PROTECTION;
-		}
+blk_status_t bio_integrity_verify(struct bio *bio, struct bvec_iter *saved_iter)
+{
+	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 
-next:
-		iter->data_buf += iter->interval;
-		iter->prot_buf += bi->metadata_size;
-		iter->seed++;
+	switch (bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+	case BLK_INTEGRITY_CSUM_CRC:
+	case BLK_INTEGRITY_CSUM_IP:
+		return blk_integrity_iterate(bio, saved_iter, true);
+	default:
+		break;
 	}
 
 	return BLK_STS_OK;
 }
 
-static void ext_pi_type1_prepare(struct request *rq)
+/*
+ * Advance @iter past the protection offset for protection formats that
+ * contain front padding on the metadata region.
+ */
+static void blk_pi_advance_offset(struct blk_integrity *bi,
+				  struct bio_integrity_payload *bip,
+				  struct bvec_iter *iter)
 {
-	struct blk_integrity *bi = &rq->q->limits.integrity;
-	const int tuple_sz = bi->metadata_size;
-	u64 ref_tag = ext_pi_ref_tag(rq);
-	u8 offset = bi->pi_offset;
-	struct bio *bio;
+	unsigned int offset = bi->pi_offset;
 
-	__rq_for_each_bio(bio, rq) {
-		struct bio_integrity_payload *bip = bio_integrity(bio);
-		u64 virt = lower_48_bits(bip_get_seed(bip));
-		struct bio_vec iv;
-		struct bvec_iter iter;
+	while (offset > 0) {
+		struct bio_vec bv = mp_bvec_iter_bvec(bip->bip_vec, *iter);
+		unsigned int len = min(bv.bv_len, offset);
 
-		/* Already remapped? */
-		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
-			break;
+		bvec_iter_advance_single(bip->bip_vec, iter, len);
+		offset -= len;
+	}
+}
 
-		bip_for_each_vec(iv, bip, iter) {
-			unsigned int j;
-			void *p;
-
-			p = bvec_kmap_local(&iv);
-			for (j = 0; j < iv.bv_len; j += tuple_sz) {
-				struct crc64_pi_tuple *pi = p +  offset;
-				u64 ref = get_unaligned_be48(pi->ref_tag);
-
-				if (ref == virt)
-					put_unaligned_be48(ref_tag, pi->ref_tag);
-				virt++;
-				ref_tag++;
-				p += tuple_sz;
-			}
-			kunmap_local(p);
-		}
+static void *blk_tuple_remap_begin(union pi_tuple *tuple,
+				   struct blk_integrity *bi,
+				   struct bio_integrity_payload *bip,
+				   struct bvec_iter *iter)
+{
+	struct bvec_iter titer;
+	struct bio_vec pbv;
 
-		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
+	blk_pi_advance_offset(bi, bip, iter);
+	pbv = bvec_iter_bvec(bip->bip_vec, *iter);
+	if (likely(pbv.bv_len >= bi->pi_tuple_size))
+		return bvec_kmap_local(&pbv);
+
+	/*
+	 * We need to preserve the state of the original iter for the
+	 * copy_from_tuple at the end, so make a temp iter for here.
+	 */
+	titer = *iter;
+	blk_integrity_copy_to_tuple(bip, &titer, tuple, bi->pi_tuple_size);
+	return tuple;
+}
+
+static void blk_tuple_remap_end(union pi_tuple *tuple, void *ptuple,
+				struct blk_integrity *bi,
+				struct bio_integrity_payload *bip,
+				struct bvec_iter *iter)
+{
+	unsigned int len = bi->metadata_size - bi->pi_offset;
+
+	if (likely(ptuple != tuple)) {
+		kunmap_local(ptuple);
+	} else {
+		blk_integrity_copy_from_tuple(bip, iter, ptuple,
+				bi->pi_tuple_size);
+		len -= bi->pi_tuple_size;
 	}
+
+	bvec_iter_advance(bip->bip_vec, iter, len);
 }
 
-static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
+static void blk_set_ext_unmap_ref(struct crc64_pi_tuple *pi, u64 virt,
+				  u64 ref_tag)
 {
-	struct blk_integrity *bi = &rq->q->limits.integrity;
-	unsigned intervals = nr_bytes >> bi->interval_exp;
-	const int tuple_sz = bi->metadata_size;
-	u64 ref_tag = ext_pi_ref_tag(rq);
-	u8 offset = bi->pi_offset;
-	struct bio *bio;
+	u64 ref = get_unaligned_be48(&pi->ref_tag);
 
-	__rq_for_each_bio(bio, rq) {
-		struct bio_integrity_payload *bip = bio_integrity(bio);
-		u64 virt = lower_48_bits(bip_get_seed(bip));
-		struct bio_vec iv;
-		struct bvec_iter iter;
-
-		bip_for_each_vec(iv, bip, iter) {
-			unsigned int j;
-			void *p;
-
-			p = bvec_kmap_local(&iv);
-			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
-				struct crc64_pi_tuple *pi = p + offset;
-				u64 ref = get_unaligned_be48(pi->ref_tag);
-
-				if (ref == ref_tag)
-					put_unaligned_be48(virt, pi->ref_tag);
-				virt++;
-				ref_tag++;
-				intervals--;
-				p += tuple_sz;
-			}
-			kunmap_local(p);
-		}
-	}
+	if (ref == lower_48_bits(ref_tag) && ref != lower_48_bits(virt))
+		put_unaligned_be48(virt, pi->ref_tag);
 }
 
-void blk_integrity_generate(struct bio *bio)
+static void blk_set_t10_unmap_ref(struct t10_pi_tuple *pi, u32 virt,
+				  u32 ref_tag)
 {
-	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
-	struct bio_integrity_payload *bip = bio_integrity(bio);
-	struct blk_integrity_iter iter;
-	struct bvec_iter bviter;
-	struct bio_vec bv;
-
-	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
-	iter.interval = 1 << bi->interval_exp;
-	iter.seed = bio->bi_iter.bi_sector;
-	iter.prot_buf = bvec_virt(bip->bip_vec);
-	bio_for_each_segment(bv, bio, bviter) {
-		void *kaddr = bvec_kmap_local(&bv);
+	u32 ref = get_unaligned_be32(&pi->ref_tag);
 
-		iter.data_buf = kaddr;
-		iter.data_size = bv.bv_len;
-		switch (bi->csum_type) {
-		case BLK_INTEGRITY_CSUM_CRC64:
-			ext_pi_crc64_generate(&iter, bi);
-			break;
-		case BLK_INTEGRITY_CSUM_CRC:
-		case BLK_INTEGRITY_CSUM_IP:
-			t10_pi_generate(&iter, bi);
-			break;
-		default:
-			break;
-		}
-		kunmap_local(kaddr);
+	if (ref == ref_tag && ref != virt)
+		put_unaligned_be32(virt, &pi->ref_tag);
+}
+
+static void blk_reftag_remap_complete(struct blk_integrity *bi,
+				      union pi_tuple *tuple, u64 virt, u64 ref)
+{
+	switch (bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+		blk_set_ext_unmap_ref(&tuple->crc64_pi, virt, ref);
+		break;
+	case BLK_INTEGRITY_CSUM_CRC:
+	case BLK_INTEGRITY_CSUM_IP:
+		blk_set_t10_unmap_ref(&tuple->t10_pi, virt, ref);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		break;
 	}
 }
 
-void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter)
+static void blk_set_ext_map_ref(struct crc64_pi_tuple *pi, u64 virt,
+				u64 ref_tag)
 {
-	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
-	struct bio_integrity_payload *bip = bio_integrity(bio);
-	struct blk_integrity_iter iter;
-	struct bvec_iter bviter;
-	struct bio_vec bv;
+	u64 ref = get_unaligned_be48(&pi->ref_tag);
 
-	/*
-	 * At the moment verify is called bi_iter has been advanced during split
-	 * and completion, so use the copy created during submission here.
-	 */
-	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
-	iter.interval = 1 << bi->interval_exp;
-	iter.seed = saved_iter->bi_sector;
-	iter.prot_buf = bvec_virt(bip->bip_vec);
-	__bio_for_each_segment(bv, bio, bviter, *saved_iter) {
-		void *kaddr = bvec_kmap_local(&bv);
-		blk_status_t ret = BLK_STS_OK;
+	if (ref == lower_48_bits(virt) && ref != ref_tag)
+		put_unaligned_be48(ref_tag, pi->ref_tag);
+}
 
-		iter.data_buf = kaddr;
-		iter.data_size = bv.bv_len;
-		switch (bi->csum_type) {
-		case BLK_INTEGRITY_CSUM_CRC64:
-			ret = ext_pi_crc64_verify(&iter, bi);
-			break;
-		case BLK_INTEGRITY_CSUM_CRC:
-		case BLK_INTEGRITY_CSUM_IP:
-			ret = t10_pi_verify(&iter, bi);
-			break;
-		default:
-			break;
-		}
-		kunmap_local(kaddr);
+static void blk_set_t10_map_ref(struct t10_pi_tuple *pi, u32 virt, u32 ref_tag)
+{
+	u32 ref = get_unaligned_be32(&pi->ref_tag);
 
-		if (ret) {
-			bio->bi_status = ret;
-			return;
-		}
+	if (ref == virt && ref != ref_tag)
+		put_unaligned_be32(ref_tag, &pi->ref_tag);
+}
+
+static void blk_reftag_remap_prepare(struct blk_integrity *bi,
+				     union pi_tuple *tuple,
+				     u64 virt, u64 ref)
+{
+	switch (bi->csum_type) {
+	case BLK_INTEGRITY_CSUM_CRC64:
+		blk_set_ext_map_ref(&tuple->crc64_pi, virt, ref);
+		break;
+	case BLK_INTEGRITY_CSUM_CRC:
+	case BLK_INTEGRITY_CSUM_IP:
+		blk_set_t10_map_ref(&tuple->t10_pi, virt, ref);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		break;
 	}
 }
 
-void blk_integrity_prepare(struct request *rq)
+static void __blk_reftag_remap(struct bio *bio, struct blk_integrity *bi,
+			       unsigned *intervals, u64 *ref, bool prep)
 {
-	struct blk_integrity *bi = &rq->q->limits.integrity;
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+	struct bvec_iter iter = bip->bip_iter;
+	u64 virt = bip_get_seed(bip);
+	union pi_tuple *ptuple;
+	union pi_tuple tuple;
 
-	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
+	if (prep && bip->bip_flags & BIP_MAPPED_INTEGRITY) {
+		*ref += bio->bi_iter.bi_size >> bi->interval_exp;
 		return;
+	}
 
-	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
-		ext_pi_type1_prepare(rq);
-	else
-		t10_pi_type1_prepare(rq);
+	while (iter.bi_size && *intervals) {
+		ptuple = blk_tuple_remap_begin(&tuple, bi, bip, &iter);
+
+		if (prep)
+			blk_reftag_remap_prepare(bi, ptuple, virt, *ref);
+		else
+			blk_reftag_remap_complete(bi, ptuple, virt, *ref);
+
+		blk_tuple_remap_end(&tuple, ptuple, bi, bip, &iter);
+		(*intervals)--;
+		(*ref)++;
+		virt++;
+	}
+
+	if (prep)
+		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
 }
 
-void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
+static void blk_integrity_remap(struct request *rq, unsigned int nr_bytes,
+				bool prep)
 {
 	struct blk_integrity *bi = &rq->q->limits.integrity;
+	u64 ref = blk_rq_pos(rq) >> (bi->interval_exp - SECTOR_SHIFT);
+	unsigned intervals = nr_bytes >> bi->interval_exp;
+	struct bio *bio;
 
 	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
 		return;
 
-	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
-		ext_pi_type1_complete(rq, nr_bytes);
-	else
-		t10_pi_type1_complete(rq, nr_bytes);
+	__rq_for_each_bio(bio, rq) {
+		__blk_reftag_remap(bio, bi, &intervals, &ref, prep);
+		if (!intervals)
+			break;
+	}
+}
+
+void blk_integrity_prepare(struct request *rq)
+{
+	blk_integrity_remap(rq, blk_rq_bytes(rq), true);
+}
+
+void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
+{
+	blk_integrity_remap(rq, nr_bytes, false);
 }
diff --git a/crypto/Kconfig b/crypto/Kconfig
index b4bb85e8e2261..6a8c9e967020b 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -2,8 +2,6 @@
 #
 # Generic algorithms support
 #
-config XOR_BLOCKS
-	tristate
 
 #
 # async_tx api: hardware offloaded memory transfer/transform support
@@ -361,6 +359,8 @@ config CRYPTO_AES
 	tristate "AES (Advanced Encryption Standard)"
 	select CRYPTO_ALGAPI
 	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM
+	select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM
 	help
 	  AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3)
 
@@ -1175,8 +1175,8 @@ endif	# if CRYPTO_DRBG_MENU
 
 config CRYPTO_JITTERENTROPY
 	tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)"
+	select CRYPTO_LIB_SHA3
 	select CRYPTO_RNG
-	select CRYPTO_SHA3
 	help
 	  CPU Jitter RNG (Random Number Generator) from the Jitterentropy library
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 04e269117589a..795c2eea51fe6 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -196,7 +196,6 @@ obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o
 #
 # generic algorithms and the async_tx api
 #
-obj-$(CONFIG_XOR_BLOCKS) += xor.o
 obj-$(CONFIG_ASYNC_CORE) += async_tx/
 obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
 crypto_simd-y := simd.o
diff --git a/crypto/aes.c b/crypto/aes.c
index ae8385df0ce51..6bf23eb0503f1 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -5,8 +5,10 @@
  * Copyright 2026 Google LLC
  */
 
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/aes.h>
 #include <crypto/algapi.h>
+#include <crypto/internal/hash.h>
 #include <linux/module.h>
 
 static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN);
@@ -33,6 +35,98 @@ static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 	aes_decrypt(key, out, in);
 }
 
+static_assert(__alignof__(struct aes_cmac_key) <= CRYPTO_MINALIGN);
+#define AES_CMAC_KEY(tfm) ((struct aes_cmac_key *)crypto_shash_ctx(tfm))
+#define AES_CMAC_CTX(desc) ((struct aes_cmac_ctx *)shash_desc_ctx(desc))
+
+static int __maybe_unused crypto_aes_cmac_setkey(struct crypto_shash *tfm,
+						 const u8 *in_key,
+						 unsigned int key_len)
+{
+	return aes_cmac_preparekey(AES_CMAC_KEY(tfm), in_key, key_len);
+}
+
+static int __maybe_unused crypto_aes_xcbc_setkey(struct crypto_shash *tfm,
+						 const u8 *in_key,
+						 unsigned int key_len)
+{
+	if (key_len != AES_KEYSIZE_128)
+		return -EINVAL;
+	aes_xcbcmac_preparekey(AES_CMAC_KEY(tfm), in_key);
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cmac_init(struct shash_desc *desc)
+{
+	aes_cmac_init(AES_CMAC_CTX(desc), AES_CMAC_KEY(desc->tfm));
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cmac_update(struct shash_desc *desc,
+						 const u8 *data,
+						 unsigned int len)
+{
+	aes_cmac_update(AES_CMAC_CTX(desc), data, len);
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cmac_final(struct shash_desc *desc,
+						u8 *out)
+{
+	aes_cmac_final(AES_CMAC_CTX(desc), out);
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc,
+						 const u8 *data,
+						 unsigned int len, u8 *out)
+{
+	aes_cmac(AES_CMAC_KEY(desc->tfm), data, len, out);
+	return 0;
+}
+
+static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN);
+#define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm))
+#define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc))
+
+static int __maybe_unused crypto_aes_cbcmac_setkey(struct crypto_shash *tfm,
+						   const u8 *in_key,
+						   unsigned int key_len)
+{
+	return aes_prepareenckey(AES_CBCMAC_KEY(tfm), in_key, key_len);
+}
+
+static int __maybe_unused crypto_aes_cbcmac_init(struct shash_desc *desc)
+{
+	aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm));
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cbcmac_update(struct shash_desc *desc,
+						   const u8 *data,
+						   unsigned int len)
+{
+	aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len);
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cbcmac_final(struct shash_desc *desc,
+						  u8 *out)
+{
+	aes_cbcmac_final(AES_CBCMAC_CTX(desc), out);
+	return 0;
+}
+
+static int __maybe_unused crypto_aes_cbcmac_digest(struct shash_desc *desc,
+						   const u8 *data,
+						   unsigned int len, u8 *out)
+{
+	aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm));
+	aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len);
+	aes_cbcmac_final(AES_CBCMAC_CTX(desc), out);
+	return 0;
+}
+
 static struct crypto_alg alg = {
 	.cra_name = "aes",
 	.cra_driver_name = "aes-lib",
@@ -48,19 +142,106 @@ static struct crypto_alg alg = {
 			       .cia_decrypt = crypto_aes_decrypt } }
 };
 
+static struct shash_alg mac_algs[] = {
+#if IS_ENABLED(CONFIG_CRYPTO_CMAC)
+	{
+		.base.cra_name = "cmac(aes)",
+		.base.cra_driver_name = "cmac-aes-lib",
+		.base.cra_priority = 300,
+		.base.cra_blocksize = AES_BLOCK_SIZE,
+		.base.cra_ctxsize = sizeof(struct aes_cmac_key),
+		.base.cra_module = THIS_MODULE,
+		.digestsize = AES_BLOCK_SIZE,
+		.setkey = crypto_aes_cmac_setkey,
+		.init = crypto_aes_cmac_init,
+		.update = crypto_aes_cmac_update,
+		.final = crypto_aes_cmac_final,
+		.digest = crypto_aes_cmac_digest,
+		.descsize = sizeof(struct aes_cmac_ctx),
+	},
+#endif
+#if IS_ENABLED(CONFIG_CRYPTO_XCBC)
+	{
+		/*
+		 * Note that the only difference between xcbc(aes) and cmac(aes)
+		 * is the preparekey function.
+		 */
+		.base.cra_name = "xcbc(aes)",
+		.base.cra_driver_name = "xcbc-aes-lib",
+		.base.cra_priority = 300,
+		.base.cra_blocksize = AES_BLOCK_SIZE,
+		.base.cra_ctxsize = sizeof(struct aes_cmac_key),
+		.base.cra_module = THIS_MODULE,
+		.digestsize = AES_BLOCK_SIZE,
+		.setkey = crypto_aes_xcbc_setkey,
+		.init = crypto_aes_cmac_init,
+		.update = crypto_aes_cmac_update,
+		.final = crypto_aes_cmac_final,
+		.digest = crypto_aes_cmac_digest,
+		.descsize = sizeof(struct aes_cmac_ctx),
+	},
+#endif
+#if IS_ENABLED(CONFIG_CRYPTO_CCM)
+	{
+		.base.cra_name = "cbcmac(aes)",
+		.base.cra_driver_name = "cbcmac-aes-lib",
+		.base.cra_priority = 300,
+		.base.cra_blocksize = AES_BLOCK_SIZE,
+		.base.cra_ctxsize = sizeof(struct aes_enckey),
+		.base.cra_module = THIS_MODULE,
+		.digestsize = AES_BLOCK_SIZE,
+		.setkey = crypto_aes_cbcmac_setkey,
+		.init = crypto_aes_cbcmac_init,
+		.update = crypto_aes_cbcmac_update,
+		.final = crypto_aes_cbcmac_final,
+		.digest = crypto_aes_cbcmac_digest,
+		.descsize = sizeof(struct aes_cbcmac_ctx),
+	},
+#endif
+};
+
 static int __init crypto_aes_mod_init(void)
 {
-	return crypto_register_alg(&alg);
+	int err = crypto_register_alg(&alg);
+
+	if (err)
+		return err;
+
+	if (ARRAY_SIZE(mac_algs) > 0) {
+		err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
+		if (err)
+			goto err_unregister_alg;
+	} /* Else, CONFIG_CRYPTO_HASH might not be enabled. */
+	return 0;
+
+err_unregister_alg:
+	crypto_unregister_alg(&alg);
+	return err;
 }
 module_init(crypto_aes_mod_init);
 
 static void __exit crypto_aes_mod_exit(void)
 {
+	if (ARRAY_SIZE(mac_algs) > 0)
+		crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
 	crypto_unregister_alg(&alg);
 }
 module_exit(crypto_aes_mod_exit);
 
 MODULE_DESCRIPTION("Crypto API support for AES block cipher");
+MODULE_IMPORT_NS("CRYPTO_INTERNAL");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS_CRYPTO("aes");
 MODULE_ALIAS_CRYPTO("aes-lib");
+#if IS_ENABLED(CONFIG_CRYPTO_CMAC)
+MODULE_ALIAS_CRYPTO("cmac(aes)");
+MODULE_ALIAS_CRYPTO("cmac-aes-lib");
+#endif
+#if IS_ENABLED(CONFIG_CRYPTO_XCBC)
+MODULE_ALIAS_CRYPTO("xcbc(aes)");
+MODULE_ALIAS_CRYPTO("xcbc-aes-lib");
+#endif
+#if IS_ENABLED(CONFIG_CRYPTO_CCM)
+MODULE_ALIAS_CRYPTO("cbcmac(aes)");
+MODULE_ALIAS_CRYPTO("cbcmac-aes-lib");
+#endif
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 2c499654a36c8..84458375b202b 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -103,7 +103,6 @@ do_sync_xor_offs(struct page *dest, unsigned int offset,
 {
 	int i;
 	int xor_src_cnt = 0;
-	int src_off = 0;
 	void *dest_buf;
 	void **srcs;
 
@@ -117,23 +116,12 @@ do_sync_xor_offs(struct page *dest, unsigned int offset,
 		if (src_list[i])
 			srcs[xor_src_cnt++] = page_address(src_list[i]) +
 				(src_offs ? src_offs[i] : offset);
-	src_cnt = xor_src_cnt;
+
 	/* set destination address */
 	dest_buf = page_address(dest) + offset;
-
 	if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
 		memset(dest_buf, 0, len);
-
-	while (src_cnt > 0) {
-		/* process up to 'MAX_XOR_BLOCKS' sources */
-		xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
-		xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);
-
-		/* drop completed sources */
-		src_cnt -= xor_src_cnt;
-		src_off += xor_src_cnt;
-	}
-
+	xor_gen(dest_buf, srcs, xor_src_cnt, len);
 	async_tx_sync_epilog(submit);
 }
 
@@ -168,11 +156,10 @@ dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset,
  *
  * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
  *
- * xor_blocks always uses the dest as a source so the
- * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
- * the calculation.  The assumption with dma engines is that they only
- * use the destination buffer as a source when it is explicitly specified
- * in the source list.
+ * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
+ * must be set to not include dest data in the calculation.  The assumption with
+ * dma engines is that they only use the destination buffer as a source when it
+ * is explicitly specified in the source list.
  *
  * src_list note: if the dest is also a source it must be at index zero.
  * The contents of this array will be overwritten if a scribble region
@@ -259,11 +246,10 @@ EXPORT_SYMBOL_GPL(async_xor_offs);
  *
  * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
  *
- * xor_blocks always uses the dest as a source so the
- * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
- * the calculation.  The assumption with dma engines is that they only
- * use the destination buffer as a source when it is explicitly specified
- * in the source list.
+ * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
+ * must be set to not include dest data in the calculation.  The assumption with
+ * dma engines is that they only use the destination buffer as a source when it
+ * is explicitly specified in the source list.
  *
  * src_list note: if the dest is also a source it must be at index zero.
  * The contents of this array will be overwritten if a scribble region
diff --git a/crypto/crc32c.c b/crypto/crc32c.c
index 1eff54dde2f74..9c9d0ae21101e 100644
--- a/crypto/crc32c.c
+++ b/crypto/crc32c.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Cryptographic API.
- *
- * CRC32C chksum
+ * crypto_shash support for CRC-32C
  *
  *@Article{castagnoli-crc,
  * author =       { Guy Castagnoli and Stefan Braeuer and Martin Herrman},
@@ -15,16 +13,6 @@
  * pages =        {},
  * month =        {June},
  *}
- * Used by the iSCSI driver, possibly others, and derived from
- * the iscsi-crc.c module of the linux-iscsi driver at
- * http://linux-iscsi.sourceforge.net.
- *
- * Following the example of lib/crc32, this function is intended to be
- * flexible and useful for all users.  Modules that currently have their
- * own crc32c, but hopefully may be able to use this one are:
- *  net/sctp (please add all your doco to here if you change to
- *            use this one!)
- *  <endoflist>
  *
  * Copyright (c) 2004 Cisco Systems, Inc.
  * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
@@ -49,11 +37,6 @@ struct chksum_desc_ctx {
 	u32 crc;
 };
 
-/*
- * Steps through buffer one byte at a time, calculates reflected
- * crc using table.
- */
-
 static int chksum_init(struct shash_desc *desc)
 {
 	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c
index 7c880cf34c523..4ad7293574410 100644
--- a/crypto/jitterentropy-kcapi.c
+++ b/crypto/jitterentropy-kcapi.c
@@ -37,7 +37,6 @@
  * DAMAGE.
  */
 
-#include <crypto/hash.h>
 #include <crypto/sha3.h>
 #include <linux/fips.h>
 #include <linux/kernel.h>
@@ -48,8 +47,6 @@
 
 #include "jitterentropy.h"
 
-#define JENT_CONDITIONING_HASH	"sha3-256"
-
 /***************************************************************************
  * Helper function
  ***************************************************************************/
@@ -101,22 +98,14 @@ void jent_get_nstime(__u64 *out)
 	jent_raw_hires_entropy_store(tmp);
 }
 
-int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
-		   unsigned int addtl_len, __u64 hash_loop_cnt,
-		   unsigned int stuck)
+void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl,
+		    unsigned int addtl_len, __u64 hash_loop_cnt,
+		    unsigned int stuck)
 {
-	struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state;
-	SHASH_DESC_ON_STACK(desc, hash_state_desc->tfm);
+	struct sha3_ctx tmp_state; /* zeroized by sha3_final() */
 	u8 intermediary[SHA3_256_DIGEST_SIZE];
 	__u64 j = 0;
-	int ret;
-
-	desc->tfm = hash_state_desc->tfm;
 
-	if (sizeof(intermediary) != crypto_shash_digestsize(desc->tfm)) {
-		pr_warn_ratelimited("Unexpected digest size\n");
-		return -EINVAL;
-	}
 	kmsan_unpoison_memory(intermediary, sizeof(intermediary));
 
 	/*
@@ -130,24 +119,20 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
 	 *
 	 * Note, it does not matter which or how much data you inject, we are
 	 * interested in one Keccack1600 compression operation performed with
-	 * the crypto_shash_final.
+	 * the sha3_final.
 	 */
 	for (j = 0; j < hash_loop_cnt; j++) {
-		ret = crypto_shash_init(desc) ?:
-		      crypto_shash_update(desc, intermediary,
-					  sizeof(intermediary)) ?:
-		      crypto_shash_finup(desc, addtl, addtl_len, intermediary);
-		if (ret)
-			goto err;
+		sha3_256_init(&tmp_state);
+		sha3_update(&tmp_state, intermediary, sizeof(intermediary));
+		sha3_update(&tmp_state, addtl, addtl_len);
+		sha3_final(&tmp_state, intermediary);
 	}
 
 	/*
 	 * Inject the data from the previous loop into the pool. This data is
 	 * not considered to contain any entropy, but it stirs the pool a bit.
 	 */
-	ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary));
-	if (ret)
-		goto err;
+	sha3_update(hash_state, intermediary, sizeof(intermediary));
 
 	/*
 	 * Insert the time stamp into the hash context representing the pool.
@@ -162,30 +147,24 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
 		time = 0;
 	}
 
-	ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64));
-
-err:
-	shash_desc_zero(desc);
+	sha3_update(hash_state, (u8 *)&time, sizeof(__u64));
 	memzero_explicit(intermediary, sizeof(intermediary));
-
-	return ret;
 }
 
-int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len)
+void jent_read_random_block(struct sha3_ctx *hash_state, char *dst,
+			    unsigned int dst_len)
 {
-	struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state;
 	u8 jent_block[SHA3_256_DIGEST_SIZE];
+
 	/* Obtain data from entropy pool and re-initialize it */
-	int ret = crypto_shash_final(hash_state_desc, jent_block) ?:
-		  crypto_shash_init(hash_state_desc) ?:
-		  crypto_shash_update(hash_state_desc, jent_block,
-				      sizeof(jent_block));
+	sha3_final(hash_state, jent_block);
+	sha3_256_init(hash_state);
+	sha3_update(hash_state, jent_block, sizeof(jent_block));
 
-	if (!ret && dst_len)
+	if (dst_len)
 		memcpy(dst, jent_block, dst_len);
 
 	memzero_explicit(jent_block, sizeof(jent_block));
-	return ret;
 }
 
 /***************************************************************************
@@ -195,8 +174,7 @@ int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len)
 struct jitterentropy {
 	spinlock_t jent_lock;
 	struct rand_data *entropy_collector;
-	struct crypto_shash *tfm;
-	struct shash_desc *sdesc;
+	struct sha3_ctx hash_state;
 };
 
 static void jent_kcapi_cleanup(struct crypto_tfm *tfm)
@@ -205,15 +183,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm)
 
 	spin_lock(&rng->jent_lock);
 
-	if (rng->sdesc) {
-		shash_desc_zero(rng->sdesc);
-		kfree(rng->sdesc);
-	}
-	rng->sdesc = NULL;
-
-	if (rng->tfm)
-		crypto_free_shash(rng->tfm);
-	rng->tfm = NULL;
+	memzero_explicit(&rng->hash_state, sizeof(rng->hash_state));
 
 	if (rng->entropy_collector)
 		jent_entropy_collector_free(rng->entropy_collector);
@@ -224,34 +194,15 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm)
 static int jent_kcapi_init(struct crypto_tfm *tfm)
 {
 	struct jitterentropy *rng = crypto_tfm_ctx(tfm);
-	struct crypto_shash *hash;
-	struct shash_desc *sdesc;
-	int size, ret = 0;
+	int ret = 0;
 
 	spin_lock_init(&rng->jent_lock);
 
 	/* Use SHA3-256 as conditioner */
-	hash = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0);
-	if (IS_ERR(hash)) {
-		pr_err("Cannot allocate conditioning digest\n");
-		return PTR_ERR(hash);
-	}
-	rng->tfm = hash;
-
-	size = sizeof(struct shash_desc) + crypto_shash_descsize(hash);
-	sdesc = kmalloc(size, GFP_KERNEL);
-	if (!sdesc) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	sdesc->tfm = hash;
-	crypto_shash_init(sdesc);
-	rng->sdesc = sdesc;
+	sha3_256_init(&rng->hash_state);
 
-	rng->entropy_collector =
-		jent_entropy_collector_alloc(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0,
-					     sdesc);
+	rng->entropy_collector = jent_entropy_collector_alloc(
+		CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &rng->hash_state);
 	if (!rng->entropy_collector) {
 		ret = -ENOMEM;
 		goto err;
@@ -326,23 +277,16 @@ static struct rng_alg jent_alg = {
 
 static int __init jent_mod_init(void)
 {
-	SHASH_DESC_ON_STACK(desc, tfm);
-	struct crypto_shash *tfm;
+	struct sha3_ctx hash_state;
 	int ret = 0;
 
 	jent_testing_init();
 
-	tfm = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0);
-	if (IS_ERR(tfm)) {
-		jent_testing_exit();
-		return PTR_ERR(tfm);
-	}
+	sha3_256_init(&hash_state);
 
-	desc->tfm = tfm;
-	crypto_shash_init(desc);
-	ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, desc, NULL);
-	shash_desc_zero(desc);
-	crypto_free_shash(tfm);
+	ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &hash_state,
+				NULL);
+	memzero_explicit(&hash_state, sizeof(hash_state));
 	if (ret) {
 		/* Handle permanent health test error */
 		if (fips_enabled)
diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
index 3f93cdc9a7afb..d5832caa8ab3c 100644
--- a/crypto/jitterentropy.c
+++ b/crypto/jitterentropy.c
@@ -68,7 +68,7 @@ struct rand_data {
 	 * of the RNG are marked as SENSITIVE. A user must not
 	 * access that information while the RNG executes its loops to
 	 * calculate the next random value. */
-	void *hash_state;		/* SENSITIVE hash state entropy pool */
+	struct sha3_ctx *hash_state;	/* SENSITIVE hash state entropy pool */
 	__u64 prev_time;		/* SENSITIVE Previous time stamp */
 	__u64 last_delta;		/* SENSITIVE stuck test */
 	__s64 last_delta2;		/* SENSITIVE stuck test */
@@ -417,10 +417,9 @@ static __u64 jent_loop_shuffle(unsigned int bits, unsigned int min)
  * time [in] time stamp to be injected
  * stuck [in] Is the time stamp identified as stuck?
  *
- * Output:
- * updated hash context in the entropy collector or error code
+ * Output: updated hash context in the entropy collector
  */
-static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck)
+static void jent_condition_data(struct rand_data *ec, __u64 time, int stuck)
 {
 #define SHA3_HASH_LOOP (1<<3)
 	struct {
@@ -435,8 +434,8 @@ static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck)
 		ec->apt_base
 	};
 
-	return jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl),
-			      SHA3_HASH_LOOP, stuck);
+	jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl),
+		       SHA3_HASH_LOOP, stuck);
 }
 
 /*
@@ -538,8 +537,7 @@ static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta)
 	stuck = jent_stuck(ec, current_delta);
 
 	/* Now call the next noise sources which also injects the data */
-	if (jent_condition_data(ec, current_delta, stuck))
-		stuck = 1;
+	jent_condition_data(ec, current_delta, stuck);
 
 	/* return the raw entropy value */
 	if (ret_current_delta)
@@ -597,7 +595,7 @@ static void jent_gen_entropy(struct rand_data *ec)
  * @return 0 when request is fulfilled or an error
  *
  * The following error codes can occur:
- *	-1	entropy_collector is NULL or the generation failed
+ *	-1	entropy_collector is NULL
  *	-2	Intermittent health failure
  *	-3	Permanent health failure
  */
@@ -640,8 +638,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data,
 		}
 
 		tocopy = min(DATA_SIZE_BITS / 8, len);
-		if (jent_read_random_block(ec->hash_state, p, tocopy))
-			return -1;
+		jent_read_random_block(ec->hash_state, p, tocopy);
 
 		len -= tocopy;
 		p += tocopy;
@@ -656,7 +653,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data,
 
 struct rand_data *jent_entropy_collector_alloc(unsigned int osr,
 					       unsigned int flags,
-					       void *hash_state)
+					       struct sha3_ctx *hash_state)
 {
 	struct rand_data *entropy_collector;
 
@@ -704,8 +701,8 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector)
 	jent_zfree(entropy_collector);
 }
 
-int jent_entropy_init(unsigned int osr, unsigned int flags, void *hash_state,
-		      struct rand_data *p_ec)
+int jent_entropy_init(unsigned int osr, unsigned int flags,
+		      struct sha3_ctx *hash_state, struct rand_data *p_ec)
 {
 	/*
 	 * If caller provides an allocated ec, reuse it which implies that the
diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h
index 4c5dbf2a8d8f9..5bb15cb330009 100644
--- a/crypto/jitterentropy.h
+++ b/crypto/jitterentropy.h
@@ -1,24 +1,27 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+struct sha3_ctx;
 extern void *jent_kvzalloc(unsigned int len);
 extern void jent_kvzfree(void *ptr, unsigned int len);
 extern void *jent_zalloc(unsigned int len);
 extern void jent_zfree(void *ptr);
 extern void jent_get_nstime(__u64 *out);
-extern int jent_hash_time(void *hash_state, __u64 time, u8 *addtl,
-			  unsigned int addtl_len, __u64 hash_loop_cnt,
-			  unsigned int stuck);
-int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len);
+void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl,
+		    unsigned int addtl_len, __u64 hash_loop_cnt,
+		    unsigned int stuck);
+void jent_read_random_block(struct sha3_ctx *hash_state, char *dst,
+			    unsigned int dst_len);
 
 struct rand_data;
 extern int jent_entropy_init(unsigned int osr, unsigned int flags,
-			     void *hash_state, struct rand_data *p_ec);
+			     struct sha3_ctx *hash_state,
+			     struct rand_data *p_ec);
 extern int jent_read_entropy(struct rand_data *ec, unsigned char *data,
 			     unsigned int len);
 
-extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr,
-						      unsigned int flags,
-						      void *hash_state);
+extern struct rand_data *
+jent_entropy_collector_alloc(unsigned int osr, unsigned int flags,
+			     struct sha3_ctx *hash_state);
 extern void jent_entropy_collector_free(struct rand_data *entropy_collector);
 
 #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index aded375461374..db860f45765fb 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -911,8 +911,14 @@ static void test_ahash_speed_common(const char *algo, unsigned int secs,
 			break;
 		}
 
-		if (klen)
-			crypto_ahash_setkey(tfm, tvmem[0], klen);
+		if (klen) {
+			ret = crypto_ahash_setkey(tfm, tvmem[0], klen);
+			if (ret) {
+				pr_err("setkey() failed flags=%x: %d\n",
+				       crypto_ahash_get_flags(tfm), ret);
+				break;
+			}
+		}
 
 		pr_info("test%3u "
 			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
@@ -2808,6 +2814,11 @@ static int __init tcrypt_mod_init(void)
 			goto err_free_tv;
 	}
 
+	if (!num_mb) {
+		pr_warn("num_mb must be at least 1; forcing to 1\n");
+		num_mb = 1;
+	}
+
 	err = do_test(alg, type, mask, mode, num_mb);
 
 	if (err) {
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 4985411dedaec..46adefd9de241 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -4079,6 +4079,13 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.aead = __VECS(aegis128_tv_template)
 		}
+	}, {
+		.alg = "authenc(hmac(md5),cbc(des))",
+		.generic_driver = "authenc(hmac-md5-lib,cbc(des-generic))",
+		.test = alg_test_aead,
+		.suite = {
+			.aead = __VECS(hmac_md5_des_cbc_tv_temp)
+		}
 	}, {
 		.alg = "authenc(hmac(md5),cbc(des3_ede))",
 		.generic_driver = "authenc(hmac-md5-lib,cbc(des3_ede-generic))",
@@ -4128,12 +4135,17 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
-		.test = alg_test_null,
+		.generic_driver = "authenc(hmac-sha1-lib,rfc3686(ctr(aes-lib)))",
+		.test = alg_test_aead,
 		.fips_allowed = 1,
+		.suite = {
+			.aead = __VECS(hmac_sha1_aes_ctr_rfc3686_tv_temp)
+		}
 	}, {
 		.alg = "authenc(hmac(sha224),cbc(aes))",
 		.generic_driver = "authenc(hmac-sha224-lib,cbc(aes-lib))",
 		.test = alg_test_aead,
+		.fips_allowed = 1,
 		.suite = {
 			.aead = __VECS(hmac_sha224_aes_cbc_tv_temp)
 		}
@@ -4153,8 +4165,12 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
-		.test = alg_test_null,
+		.generic_driver = "authenc(hmac-sha224-lib,rfc3686(ctr(aes-lib)))",
+		.test = alg_test_aead,
 		.fips_allowed = 1,
+		.suite = {
+			.aead = __VECS(hmac_sha224_aes_ctr_rfc3686_tv_temp)
+		}
 	}, {
 		.alg = "authenc(hmac(sha256),cbc(aes))",
 		.generic_driver = "authenc(hmac-sha256-lib,cbc(aes-lib))",
@@ -4190,12 +4206,17 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
-		.test = alg_test_null,
+		.generic_driver = "authenc(hmac-sha256-lib,rfc3686(ctr(aes-lib)))",
+		.test = alg_test_aead,
 		.fips_allowed = 1,
+		.suite = {
+			.aead = __VECS(hmac_sha256_aes_ctr_rfc3686_tv_temp)
+		}
 	}, {
 		.alg = "authenc(hmac(sha384),cbc(aes))",
 		.generic_driver = "authenc(hmac-sha384-lib,cbc(aes-lib))",
 		.test = alg_test_aead,
+		.fips_allowed = 1,
 		.suite = {
 			.aead = __VECS(hmac_sha384_aes_cbc_tv_temp)
 		}
@@ -4226,8 +4247,12 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
-		.test = alg_test_null,
+		.generic_driver = "authenc(hmac-sha384-lib,rfc3686(ctr(aes-lib)))",
+		.test = alg_test_aead,
 		.fips_allowed = 1,
+		.suite = {
+			.aead = __VECS(hmac_sha384_aes_ctr_rfc3686_tv_temp)
+		}
 	}, {
 		.alg = "authenc(hmac(sha512),cbc(aes))",
 		.generic_driver = "authenc(hmac-sha512-lib,cbc(aes-lib))",
@@ -4256,8 +4281,12 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.fips_allowed = 1,
 	}, {
 		.alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
-		.test = alg_test_null,
+		.generic_driver = "authenc(hmac-sha512-lib,rfc3686(ctr(aes-lib)))",
+		.test = alg_test_aead,
 		.fips_allowed = 1,
+		.suite = {
+			.aead = __VECS(hmac_sha512_aes_ctr_rfc3686_tv_temp)
+		}
 	}, {
 		.alg = "blake2b-160",
 		.generic_driver = "blake2b-160-lib",
@@ -4388,7 +4417,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 	}, {
 #endif
 		.alg = "cbcmac(aes)",
-		.generic_driver = "cbcmac(aes-lib)",
+		.generic_driver = "cbcmac-aes-lib",
 		.test = alg_test_hash,
 		.suite = {
 			.hash = __VECS(aes_cbcmac_tv_template)
@@ -4401,7 +4430,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "ccm(aes)",
-		.generic_driver = "ccm_base(ctr(aes-lib),cbcmac(aes-lib))",
+		.generic_driver = "ccm_base(ctr(aes-lib),cbcmac-aes-lib)",
 		.test = alg_test_aead,
 		.fips_allowed = 1,
 		.suite = {
@@ -4429,7 +4458,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		},
 	}, {
 		.alg = "cmac(aes)",
-		.generic_driver = "cmac(aes-lib)",
+		.generic_driver = "cmac-aes-lib",
 		.fips_allowed = 1,
 		.test = alg_test_hash,
 		.suite = {
@@ -5326,7 +5355,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "rfc4309(ccm(aes))",
-		.generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac(aes-lib)))",
+		.generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac-aes-lib))",
 		.test = alg_test_aead,
 		.fips_allowed = 1,
 		.suite = {
@@ -5515,7 +5544,7 @@ static const struct alg_test_desc alg_test_descs[] = {
 		}
 	}, {
 		.alg = "xcbc(aes)",
-		.generic_driver = "xcbc(aes-lib)",
+		.generic_driver = "xcbc-aes-lib",
 		.test = alg_test_hash,
 		.suite = {
 			.hash = __VECS(aes_xcbc128_tv_template)
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 1c69c11c0cdb4..458fc8e87673f 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -14873,6 +14873,227 @@ static const struct aead_testvec hmac_sha1_aes_cbc_tv_temp[] = {
 	},
 };
 
+static const struct aead_testvec hmac_sha1_aes_ctr_rfc3686_tv_temp[] = {
+	{ /* RFC 3686 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 8 + 20 + 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8"
+			  "\x70\xdc\x6b\x62\x43\xa1\x2f\x08"
+			  "\xf1\xec\x93\x7d\x69\xb2\x8e\x1f"
+			  "\x0a\x97\x39\x86",
+		.clen	= 16 + 20,
+	}, { /* RFC 3686 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33"
+			  "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 8 + 20 + 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.assoc	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28"
+			  "\x6b\x7b\x4d\x39\x36\x1c\x12\x5f"
+			  "\x72\xd2\x88\xb2\x26\xa6\xa6\xb5"
+			  "\x1d\x3a\x49\xa6",
+		.clen	= 32 + 20,
+	}, { /* RFC 3686 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55"
+			  "\x76\x91\xbe\x03\x5e\x50\x20\xa8"
+			  "\xac\x6e\x61\x85\x29\xf9\xa0\xdc"
+			  "\x00\xe0\x01\x7b",
+		.klen	= 8 + 20 + 20,
+		.iv	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.assoc	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23",
+		.plen	= 36,
+		.ctext	= "\xc1\xcf\x48\xa8\x9f\x2f\xfd\xd9"
+			  "\xcf\x46\x52\xe9\xef\xdb\x72\xd7"
+			  "\x45\x40\xa4\x2b\xde\x6d\x78\x36"
+			  "\xd5\x9a\x5c\xea\xae\xf3\x10\x53"
+			  "\x25\xb2\x07\x2f"
+			  "\x2c\x86\xa0\x90\x8e\xc1\x02\x1d"
+			  "\x51\xdc\xd6\x21\xc7\x30\xcc\x32"
+			  "\x38\x55\x47\x64",
+		.clen	= 36 + 20,
+	}, { /* RFC 3686 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 8 + 20 + 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.assoc	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28"
+			  "\xe9\x4e\x49\xf0\x6b\x8d\x58\x2b"
+			  "\x26\x7f\xf3\xab\xeb\x2f\x74\x2f"
+			  "\x45\x43\x64\xc1",
+		.clen	= 16 + 20,
+	}, { /* RFC 3686 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33"
+			  "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 8 + 20 + 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.assoc	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00"
+			  "\xab\xc4\xfa\x6d\x20\xe1\xce\x72"
+			  "\x0e\x92\x4e\x97\xaa\x4d\x30\x84"
+			  "\xb6\xd8\x4d\x3b",
+		.clen	= 32 + 20,
+	}, { /* RFC 3686 Case 7 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 8 + 20 + 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.assoc	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0"
+			  "\x3d\x6c\x23\x27\xda\x0e\x7f\x29"
+			  "\xfd\x8d\x3c\x1b\xf7\x7a\x63\xd9"
+			  "\x7e\x0f\xe9\xf6",
+		.clen	= 16 + 20,
+	}, { /* RFC 3686 Case 8 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33"
+			  "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 8 + 20 + 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.assoc	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c"
+			  "\xe7\xee\x22\xa4\xdd\xbf\x5d\x44"
+			  "\x3b\x43\x1c\x69\x55\x11\xd5\xad"
+			  "\x14\x5f\x44\xa6",
+		.clen	= 32 + 20,
+	},
+};
+
 static const struct aead_testvec hmac_sha1_ecb_cipher_null_tv_temp[] = {
 	{ /* Input data from RFC 2410 Case 1 */
 #ifdef __LITTLE_ENDIAN
@@ -15204,8 +15425,8 @@ static const struct aead_testvec hmac_sha224_aes_cbc_tv_temp[] = {
 	},
 };
 
-static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = {
-	{ /* RFC 3602 Case 1 */
+static const struct aead_testvec hmac_sha224_aes_ctr_rfc3686_tv_temp[] = {
+	{ /* RFC 3686 Case 1 */
 #ifdef __LITTLE_ENDIAN
 		.key    = "\x08\x00"		/* rta length */
 			  "\x01\x00"		/* rta type */
@@ -15213,29 +15434,28 @@ static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = {
 		.key    = "\x00\x08"		/* rta length */
 			  "\x00\x01"		/* rta type */
 #endif
-			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x14"	/* enc key length */
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
 			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
-			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
-			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
-		.alen	= 16,
+			  "\x00\x00\x00\x00"
+			  "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 8 + 28 + 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 8,
 		.ptext	= "Single block msg",
 		.plen	= 16,
-		.ctext	= "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
-			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
-			  "\xcc\xde\x2d\x6a\xae\xf1\x0b\xcc"
-			  "\x38\x06\x38\x51\xb4\xb8\xf3\x5b"
-			  "\x5c\x34\xa6\xa3\x6e\x0b\x05\xe5"
-			  "\x6a\x6d\x44\xaa\x26\xa8\x44\xa5",
-		.clen	= 16 + 32,
-	}, { /* RFC 3602 Case 2 */
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8"
+			  "\x36\xb4\x3b\x9c\x62\xed\xcf\x77"
+			  "\xdc\x19\x27\x3f\x92\x80\x52\xce"
+			  "\x8f\xad\x01\x0b\x79\xda\x04\x83"
+			  "\xcb\x45\x1a\x52",
+		.clen	= 16 + 28,
+	}, { /* RFC 3686 Case 2 */
 #ifdef __LITTLE_ENDIAN
 		.key    = "\x08\x00"		/* rta length */
 			  "\x01\x00"		/* rta type */
@@ -15243,94 +15463,330 @@ static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = {
 		.key    = "\x00\x08"		/* rta length */
 			  "\x00\x01"		/* rta type */
 #endif
-			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x14"	/* enc key length */
 			  "\x20\x21\x22\x23\x24\x25\x26\x27"
 			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
 			  "\x30\x31\x32\x33\x34\x35\x36\x37"
-			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
-			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
-			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
-			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
-		.alen	= 16,
+			  "\x38\x39\x3a\x3b"
+			  "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 8 + 28 + 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.assoc	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.alen	= 8,
 		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
 			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 			  "\x10\x11\x12\x13\x14\x15\x16\x17"
 			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
 		.plen	= 32,
-		.ctext	= "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
-			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
-			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
-			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
-			  "\xf5\x33\x53\xf3\x68\x85\x2a\x99"
-			  "\x0e\x06\x58\x8f\xba\xf6\x06\xda"
-			  "\x49\x69\x0d\x5b\xd4\x36\x06\x62"
-			  "\x35\x5e\x54\x58\x53\x4d\xdf\xbf",
-		.clen	= 32 + 32,
-	}, { /* RFC 3602 Case 3 */
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28"
+			  "\x7f\xe4\x8f\xa7\x06\x71\xe9\xe5"
+			  "\x16\x79\xef\xf9\x7e\x5c\x93\x4d"
+			  "\xa0\xf8\x3b\x3a\xaa\x1c\xc0\xd9"
+			  "\x6b\x48\x49\x01",
+		.clen	= 32 + 28,
+	}, { /* RFC 3686 Case 3 */
 #ifdef __LITTLE_ENDIAN
 		.key    = "\x08\x00"		/* rta length */
-			  "\x01\x00"            /* rta type */
+			  "\x01\x00"		/* rta type */
 #else
 		.key    = "\x00\x08"		/* rta length */
 			  "\x00\x01"		/* rta type */
 #endif
-			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x14"	/* enc key length */
 			  "\x11\x22\x33\x44\x55\x66\x77\x88"
 			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
 			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
-			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
-			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
-		.alen	= 16,
-		.ptext	= "This is a 48-byte message (exactly 3 AES blocks)",
-		.plen	= 48,
-		.ctext	= "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
-			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
-			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
-			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
-			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
-			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
-			  "\x68\xb9\x3e\x90\x38\xa0\x88\x01"
-			  "\xe7\xc6\xce\x10\x31\x2f\x9b\x1d"
-			  "\x24\x78\xfb\xbe\x02\xe0\x4f\x40"
-			  "\x10\xbd\xaa\xc6\xa7\x79\xe0\x1a",
-		.clen	= 48 + 32,
-	}, { /* RFC 3602 Case 4 */
+			  "\xaa\xbb\xcc\xdd"
+			  "\x76\x91\xbe\x03\x5e\x50\x20\xa8"
+			  "\xac\x6e\x61\x85\x29\xf9\xa0\xdc"
+			  "\x00\xe0\x01\x7b",
+		.klen	= 8 + 28 + 20,
+		.iv	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.assoc	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23",
+		.plen	= 36,
+		.ctext	= "\xc1\xcf\x48\xa8\x9f\x2f\xfd\xd9"
+			  "\xcf\x46\x52\xe9\xef\xdb\x72\xd7"
+			  "\x45\x40\xa4\x2b\xde\x6d\x78\x36"
+			  "\xd5\x9a\x5c\xea\xae\xf3\x10\x53"
+			  "\x25\xb2\x07\x2f"
+			  "\xb0\x19\x45\xee\xa7\x31\xd9\xd0"
+			  "\x74\x6b\xb8\xb1\x67\x61\x2f\x8c"
+			  "\x68\xde\xe3\xc9\x3b\x0c\x72\xda"
+			  "\x48\xba\x1b\x51",
+		.clen	= 36 + 28,
+	}, { /* RFC 3686 Case 4 */
 #ifdef __LITTLE_ENDIAN
 		.key    = "\x08\x00"		/* rta length */
 			  "\x01\x00"		/* rta type */
 #else
 		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
+			  "\x00\x01"		/* rta type */
 #endif
-			  "\x00\x00\x00\x10"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
-			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
-		.klen   = 8 + 32 + 16,
-		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
-			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
-		.alen	= 16,
-		.ptext	= "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
-			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
-			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
-			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
-			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
-			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 8 + 28 + 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.assoc	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28"
+			  "\xfd\xf5\x35\x26\x50\x3d\xdf\x80"
+			  "\x6e\xbe\xba\x8d\x56\xf3\x03\xb7"
+			  "\x27\xb8\x13\xe8\x72\x8f\xc9\x52"
+			  "\x4a\xb7\xc3\x3a",
+		.clen	= 16 + 28,
+	}, { /* RFC 3686 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b"
+			  "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 8 + 28 + 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.assoc	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00"
+			  "\x72\x89\xa8\x04\xa5\xac\x8f\x29"
+			  "\xe6\xb8\x58\xe8\xcf\x6a\x91\x89"
+			  "\xd3\x66\x3b\xdc\xce\x43\x23\xb7"
+			  "\x6a\xdd\x9d\xbd",
+		.clen	= 32 + 28,
+	}, { /* RFC 3686 Case 7 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00"
+			  "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 8 + 28 + 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.assoc	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0"
+			  "\xfe\xdf\x6f\x62\x8a\x79\xb5\x34"
+			  "\xd0\x6f\x32\xaf\x31\x50\x5b\x1f"
+			  "\xe0\x6d\x0b\xbc\x02\x25\xee\x74"
+			  "\x7a\xdf\x97\x3c",
+		.clen	= 16 + 28,
+	}, { /* RFC 3686 Case 8 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b"
+			  "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 8 + 28 + 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.assoc	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c"
+			  "\x19\x1e\x9c\x2c\x6d\x4e\x21\xda"
+			  "\x6c\x4d\x88\x90\xf8\x5f\xa5\x9d"
+			  "\xb4\xd4\x40\xad\xfa\x67\x3f\x0e"
+			  "\x11\x12\xd6\x10",
+		.clen	= 32 + 28,
+	},
+};
+
+static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = {
+	{ /* RFC 3602 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x06\xa9\x21\x40\x36\xb8\xa1\x5b"
+			  "\x51\x2e\x03\xd5\x34\x12\x00\x06",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.assoc	= "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30"
+			  "\xb4\x22\xda\x80\x2c\x9f\xac\x41",
+		.alen	= 16,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\xe3\x53\x77\x9c\x10\x79\xae\xb8"
+			  "\x27\x08\x94\x2d\xbe\x77\x18\x1a"
+			  "\xcc\xde\x2d\x6a\xae\xf1\x0b\xcc"
+			  "\x38\x06\x38\x51\xb4\xb8\xf3\x5b"
+			  "\x5c\x34\xa6\xa3\x6e\x0b\x05\xe5"
+			  "\x6a\x6d\x44\xaa\x26\xa8\x44\xa5",
+		.clen	= 16 + 32,
+	}, { /* RFC 3602 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0"
+			  "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.assoc	= "\x56\x2e\x17\x99\x6d\x09\x3d\x28"
+			  "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58",
+		.alen	= 16,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a"
+			  "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a"
+			  "\x75\x86\x60\x2d\x25\x3c\xff\xf9"
+			  "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1"
+			  "\xf5\x33\x53\xf3\x68\x85\x2a\x99"
+			  "\x0e\x06\x58\x8f\xba\xf6\x06\xda"
+			  "\x49\x69\x0d\x5b\xd4\x36\x06\x62"
+			  "\x35\x5e\x54\x58\x53\x4d\xdf\xbf",
+		.clen	= 32 + 32,
+	}, { /* RFC 3602 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"            /* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x6c\x3e\xa0\x47\x76\x30\xce\x21"
+			  "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.assoc	= "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb"
+			  "\xd9\xcd\x27\xd8\x25\x68\x2c\x81",
+		.alen	= 16,
+		.ptext	= "This is a 48-byte message (exactly 3 AES blocks)",
+		.plen	= 48,
+		.ctext	= "\xd0\xa0\x2b\x38\x36\x45\x17\x53"
+			  "\xd4\x93\x66\x5d\x33\xf0\xe8\x86"
+			  "\x2d\xea\x54\xcd\xb2\x93\xab\xc7"
+			  "\x50\x69\x39\x27\x67\x72\xf8\xd5"
+			  "\x02\x1c\x19\x21\x6b\xad\x52\x5c"
+			  "\x85\x79\x69\x5d\x83\xba\x26\x84"
+			  "\x68\xb9\x3e\x90\x38\xa0\x88\x01"
+			  "\xe7\xc6\xce\x10\x31\x2f\x9b\x1d"
+			  "\x24\x78\xfb\xbe\x02\xe0\x4f\x40"
+			  "\x10\xbd\xaa\xc6\xa7\x79\xe0\x1a",
+		.clen	= 48 + 32,
+	}, { /* RFC 3602 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x10"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x56\xe4\x7a\x38\xc5\x59\x89\x74"
+			  "\xbc\x46\x90\x3d\xba\x29\x03\x49",
+		.klen   = 8 + 32 + 16,
+		.iv     = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.assoc	= "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c"
+			  "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9",
+		.alen	= 16,
+		.ptext	= "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
 			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
 			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf",
 		.plen	= 64,
@@ -15403,87 +15859,322 @@ static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = {
 		.key    = "\x00\x08"		/* rta length */
 			  "\x00\x01"            /* rta type */
 #endif
-			  "\x00\x00\x00\x18"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
-			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
-			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
-		.klen   = 8 + 32 + 24,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.plen	= 64,
-		.ctext	= "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
-			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
-			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
-			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
-			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
-			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
-			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
-			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
-			  "\x2f\xee\x5f\xdb\x66\xfe\x79\x09"
-			  "\x61\x81\x31\xea\x5b\x3d\x8e\xfb"
-			  "\xca\x71\x85\x93\xf7\x85\x55\x8b"
-			  "\x7a\xe4\x94\xca\x8b\xba\x19\x33",
-		.clen	= 64 + 32,
-	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
+			  "\x00\x00\x00\x18"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x8e\x73\xb0\xf7\xda\x0e\x64\x52"
+			  "\xc8\x10\xf3\x2b\x80\x90\x79\xe5"
+			  "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b",
+		.klen   = 8 + 32 + 24,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.plen	= 64,
+		.ctext	= "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d"
+			  "\x71\x78\x18\x3a\x9f\xa0\x71\xe8"
+			  "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4"
+			  "\xe5\xe7\x38\x76\x3f\x69\x14\x5a"
+			  "\x57\x1b\x24\x20\x12\xfb\x7a\xe0"
+			  "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0"
+			  "\x08\xb0\xe2\x79\x88\x59\x88\x81"
+			  "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd"
+			  "\x2f\xee\x5f\xdb\x66\xfe\x79\x09"
+			  "\x61\x81\x31\xea\x5b\x3d\x8e\xfb"
+			  "\xca\x71\x85\x93\xf7\x85\x55\x8b"
+			  "\x7a\xe4\x94\xca\x8b\xba\x19\x33",
+		.clen	= 64 + 32,
+	}, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"            /* rta type */
+#endif
+			  "\x00\x00\x00\x20"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
+			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
+			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
+			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
+		.klen   = 8 + 32 + 32,
+		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+		.alen   = 16,
+		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
+			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
+			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
+			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
+			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
+			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
+			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
+			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
+		.plen	= 64,
+		.ctext	= "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
+			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
+			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
+			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
+			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
+			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
+			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
+			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
+			  "\x24\x29\xed\xc2\x31\x49\xdb\xb1"
+			  "\x8f\x74\xbd\x17\x92\x03\xbe\x8f"
+			  "\xf3\x61\xde\x1c\xe9\xdb\xcd\xd0"
+			  "\xcc\xce\xe9\x85\x57\xcf\x6f\x5f",
+		.clen	= 64 + 32,
+	},
+};
+
+static const struct aead_testvec hmac_sha256_aes_ctr_rfc3686_tv_temp[] = {
+	{ /* RFC 3686 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 8 + 32 + 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8"
+			  "\x9b\xa2\x34\x62\xe5\xb3\xe8\x2d"
+			  "\x6d\xdb\x93\x64\xa5\x08\x2e\x77"
+			  "\x72\x1f\x21\x94\xc7\xbe\x14\xa6"
+			  "\xcd\xea\x96\xa1\x29\x8f\x30\xc3",
+		.clen	= 16 + 32,
+	}, { /* RFC 3686 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 8 + 32 + 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.assoc	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28"
+			  "\x22\xf7\x95\xa8\xbb\xcd\x19\xf4"
+			  "\x58\x16\x54\x28\x2b\xf4\x52\xe7"
+			  "\x5c\x6c\xe1\x44\x0b\xd5\x10\x6e"
+			  "\xe1\xf7\x04\xc4\x2c\xab\x93\xdd",
+		.clen	= 32 + 32,
+	}, { /* RFC 3686 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x76\x91\xbe\x03\x5e\x50\x20\xa8"
+			  "\xac\x6e\x61\x85\x29\xf9\xa0\xdc"
+			  "\x00\xe0\x01\x7b",
+		.klen	= 8 + 32 + 20,
+		.iv	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.assoc	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23",
+		.plen	= 36,
+		.ctext	= "\xc1\xcf\x48\xa8\x9f\x2f\xfd\xd9"
+			  "\xcf\x46\x52\xe9\xef\xdb\x72\xd7"
+			  "\x45\x40\xa4\x2b\xde\x6d\x78\x36"
+			  "\xd5\x9a\x5c\xea\xae\xf3\x10\x53"
+			  "\x25\xb2\x07\x2f"
+			  "\x1d\x05\x5f\x77\x3b\x4f\x5c\x21"
+			  "\x29\xea\xf1\xa8\x71\x49\x7b\x0b"
+			  "\x66\x0d\xff\x18\x81\x63\xfc\xc3"
+			  "\x91\xb6\x38\xc8\xcd\x2d\x39\x83",
+		.clen	= 36 + 32,
+	}, { /* RFC 3686 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 8 + 32 + 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.assoc	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28"
+			  "\x8d\x03\x77\xb2\x1c\xc9\xe0\xac"
+			  "\xde\x69\xbe\x8a\xef\x5b\x13\x74"
+			  "\x1d\x39\xbc\xdc\x95\xa4\xbf\xc3"
+			  "\xd5\xc6\xd1\xda\xda\x3b\xca\x78",
+		.clen	= 16 + 32,
+	}, { /* RFC 3686 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 8 + 32 + 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.assoc	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00"
+			  "\x34\x06\x2b\x3d\xf1\xa8\x3d\xf1"
+			  "\xa6\x5e\x5c\x1a\xdb\x0c\xb5\x1e"
+			  "\x8f\xdb\xf4\xca\x7d\x09\x5e\x81"
+			  "\xdb\x32\x07\x4a\x1d\x1c\x6d\x83",
+		.clen	= 32 + 32,
+	}, { /* RFC 3686 Case 7 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 8 + 32 + 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.assoc	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0"
+			  "\xc3\xb4\x5f\xb0\xbf\xf5\x1b\xff"
+			  "\x7c\xf1\x79\x00\x63\x50\xdd\x77"
+			  "\xc0\x4a\xba\xcd\xdc\x47\x05\x2a"
+			  "\x5d\x85\x2d\x83\x44\xca\x79\x2c",
+		.clen	= 16 + 32,
+	}, { /* RFC 3686 Case 8 */
 #ifdef __LITTLE_ENDIAN
 		.key    = "\x08\x00"		/* rta length */
 			  "\x01\x00"		/* rta type */
 #else
 		.key    = "\x00\x08"		/* rta length */
-			  "\x00\x01"            /* rta type */
+			  "\x00\x01"		/* rta type */
 #endif
-			  "\x00\x00\x00\x20"	/* enc key length */
-			  "\x11\x22\x33\x44\x55\x66\x77\x88"
-			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
-			  "\x22\x33\x44\x55\x66\x77\x88\x99"
-			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
-			  "\x60\x3d\xeb\x10\x15\xca\x71\xbe"
-			  "\x2b\x73\xae\xf0\x85\x7d\x77\x81"
-			  "\x1f\x35\x2c\x07\x3b\x61\x08\xd7"
-			  "\x2d\x98\x10\xa3\x09\x14\xdf\xf4",
-		.klen   = 8 + 32 + 32,
-		.iv     = "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.assoc	= "\x00\x01\x02\x03\x04\x05\x06\x07"
-			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-		.alen   = 16,
-		.ptext	= "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96"
-			  "\xe9\x3d\x7e\x11\x73\x93\x17\x2a"
-			  "\xae\x2d\x8a\x57\x1e\x03\xac\x9c"
-			  "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51"
-			  "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11"
-			  "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef"
-			  "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17"
-			  "\xad\x2b\x41\x7b\xe6\x6c\x37\x10",
-		.plen	= 64,
-		.ctext	= "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba"
-			  "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6"
-			  "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d"
-			  "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d"
-			  "\x39\xf2\x33\x69\xa9\xd9\xba\xcf"
-			  "\xa5\x30\xe2\x63\x04\x23\x14\x61"
-			  "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc"
-			  "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b"
-			  "\x24\x29\xed\xc2\x31\x49\xdb\xb1"
-			  "\x8f\x74\xbd\x17\x92\x03\xbe\x8f"
-			  "\xf3\x61\xde\x1c\xe9\xdb\xcd\xd0"
-			  "\xcc\xce\xe9\x85\x57\xcf\x6f\x5f",
-		.clen	= 64 + 32,
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 8 + 32 + 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.assoc	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c"
+			  "\xc8\x59\x5d\xe1\xba\xac\x13\x82"
+			  "\xfd\x21\x7c\x8c\x23\x31\x04\x02"
+			  "\x9e\x69\x5b\x57\xa8\x13\xe7\x21"
+			  "\x60\x0c\x24\xc2\x80\x4a\x93\x6e",
+		.clen	= 32 + 32,
 	},
 };
 
@@ -15798,6 +16489,269 @@ static const struct aead_testvec hmac_sha384_aes_cbc_tv_temp[] = {
 	},
 };
 
+static const struct aead_testvec hmac_sha384_aes_ctr_rfc3686_tv_temp[] = {
+	{ /* RFC 3686 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 8 + 48 + 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8"
+			  "\x45\x51\x59\x72\x16\xd3\xc6\x15"
+			  "\x25\x1e\xe8\x92\x2e\x47\x52\xcc"
+			  "\x91\x9c\x24\xef\x11\xb2\x53\x00"
+			  "\x10\x20\x43\x06\xe2\x35\x88\x9e"
+			  "\x18\x32\x5a\x79\x7d\x73\x7e\x89"
+			  "\xfe\xa1\xda\xa4\x86\xc4\x2a\x04",
+		.clen	= 16 + 48,
+	}, { /* RFC 3686 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 8 + 48 + 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.assoc	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28"
+			  "\x83\x65\x32\x1e\x6b\x60\xe6\x4a"
+			  "\xe2\xab\x52\x2b\xa6\x70\x3a\xfa"
+			  "\xd2\xec\x83\xe4\x31\x0c\x28\x40"
+			  "\x9b\x5e\x18\xa4\xdc\x48\xb8\x56"
+			  "\x33\xab\x7f\x2b\xaf\xe4\x3a\xe3"
+			  "\x8a\x61\xf6\x22\xb4\x6b\xfe\x7d",
+		.clen	= 32 + 48,
+	}, { /* RFC 3686 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x76\x91\xbe\x03\x5e\x50\x20\xa8"
+			  "\xac\x6e\x61\x85\x29\xf9\xa0\xdc"
+			  "\x00\xe0\x01\x7b",
+		.klen	= 8 + 48 + 20,
+		.iv	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.assoc	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23",
+		.plen	= 36,
+		.ctext	= "\xc1\xcf\x48\xa8\x9f\x2f\xfd\xd9"
+			  "\xcf\x46\x52\xe9\xef\xdb\x72\xd7"
+			  "\x45\x40\xa4\x2b\xde\x6d\x78\x36"
+			  "\xd5\x9a\x5c\xea\xae\xf3\x10\x53"
+			  "\x25\xb2\x07\x2f"
+			  "\x4a\xaa\xad\x3b\x3b\xb6\x9a\xba"
+			  "\xa1\x7b\xc6\xce\x96\xc3\xff\x67"
+			  "\xf3\x0c\x33\x57\xf0\x51\x24\x08"
+			  "\xed\x4f\x6a\x9c\x22\x42\xbd\x18"
+			  "\x97\x74\x68\x36\x00\xf1\x69\x3a"
+			  "\x18\x77\x40\xf0\x56\xba\xba\xe0",
+		.clen	= 36 + 48,
+	}, { /* RFC 3686 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 8 + 48 + 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.assoc	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28"
+			  "\x36\xd6\xc7\x55\xac\xb6\x0b\x14"
+			  "\x95\x71\xf9\x86\x30\xe3\x96\xc3"
+			  "\x76\x85\x6d\xa5\x06\xed\x6f\x34"
+			  "\xcc\x1f\xcc\x2d\x88\x06\xb0\x1d"
+			  "\xbe\xd9\xa2\xd3\x64\xf1\x33\x03"
+			  "\x13\x50\x8f\xae\x61\x2d\x82\xb8",
+		.clen	= 16 + 48,
+	}, { /* RFC 3686 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 8 + 48 + 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.assoc	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00"
+			  "\x80\x12\x67\x22\xf2\x4d\x9b\xbf"
+			  "\xdc\x38\xd3\xaa\x12\xc0\x58\x1a"
+			  "\x9a\x62\x6e\x42\x3d\x44\x63\xdd"
+			  "\xee\x7e\xe3\xa3\xdf\x2a\x65\x05"
+			  "\xd0\xc1\xd2\x54\x55\x35\x5c\xc7"
+			  "\xb0\xb5\xb1\x36\xe0\x0b\xaf\x72",
+		.clen	= 32 + 48,
+	}, { /* RFC 3686 Case 7 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 8 + 48 + 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.assoc	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0"
+			  "\xb1\x7b\xb1\xec\xca\x94\x55\xc4"
+			  "\x3f\x2b\xb1\x70\x04\x91\xf5\x9d"
+			  "\x1a\xc0\xe1\x2a\x93\x5f\x96\x2a"
+			  "\x12\x85\x38\x36\xe1\xb2\xe9\xf0"
+			  "\xf2\x6e\x5d\x81\xcc\x49\x07\x9c"
+			  "\x5b\x88\xc8\xcc\xc4\x21\x4f\x32",
+		.clen	= 16 + 48,
+	}, { /* RFC 3686 Case 8 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 8 + 48 + 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.assoc	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c"
+			  "\xd6\x96\xbb\x12\x39\xc4\x4d\xe2"
+			  "\x4c\x02\xe7\x1f\xdc\xb2\xb1\x57"
+			  "\x38\x0d\xdd\x13\xb3\x89\x57\x9e"
+			  "\x1f\xb5\x48\x32\xc4\xd3\x9d\x1f"
+			  "\x68\xab\x8d\xc6\xa8\x05\x3a\xc2"
+			  "\x87\xaf\x23\xb3\xe4\x1b\xde\xb3",
+		.clen	= 32 + 48,
+	},
+};
+
 static const struct aead_testvec hmac_sha512_aes_cbc_tv_temp[] = {
 	{ /* RFC 3602 Case 1 */
 #ifdef __LITTLE_ENDIAN
@@ -16137,6 +17091,354 @@ static const struct aead_testvec hmac_sha512_aes_cbc_tv_temp[] = {
 	},
 };
 
+static const struct aead_testvec hmac_md5_des_cbc_tv_temp[] = {
+	{ /*Generated with cryptopp*/
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x08"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24",
+		.klen	= 8 + 16 + 8,
+		.iv	= "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.assoc  = "\x00\x00\x43\x21\x00\x00\x00\x01"
+			  "\x7D\x33\x88\x93\x0F\x93\xB2\x42",
+		.alen   = 16,
+		.ptext	= "\x6f\x54\x20\x6f\x61\x4d\x79\x6e"
+			  "\x53\x20\x63\x65\x65\x72\x73\x74"
+			  "\x54\x20\x6f\x6f\x4d\x20\x6e\x61"
+			  "\x20\x79\x65\x53\x72\x63\x74\x65"
+			  "\x20\x73\x6f\x54\x20\x6f\x61\x4d"
+			  "\x79\x6e\x53\x20\x63\x65\x65\x72"
+			  "\x73\x74\x54\x20\x6f\x6f\x4d\x20"
+			  "\x6e\x61\x20\x79\x65\x53\x72\x63"
+			  "\x74\x65\x20\x73\x6f\x54\x20\x6f"
+			  "\x61\x4d\x79\x6e\x53\x20\x63\x65"
+			  "\x65\x72\x73\x74\x54\x20\x6f\x6f"
+			  "\x4d\x20\x6e\x61\x20\x79\x65\x53"
+			  "\x72\x63\x74\x65\x20\x73\x6f\x54"
+			  "\x20\x6f\x61\x4d\x79\x6e\x53\x20"
+			  "\x63\x65\x65\x72\x73\x74\x54\x20"
+			  "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79",
+		.plen	= 128,
+		.ctext	= "\x70\xd6\xde\x64\x87\x17\xf1\xe8"
+			  "\x54\x31\x85\x37\xed\x6b\x01\x8d"
+			  "\xe3\xcc\xe0\x1d\x5e\xf3\xfe\xf1"
+			  "\x41\xaa\x33\x91\xa7\x7d\x99\x88"
+			  "\x4d\x85\x6e\x2f\xa3\x69\xf5\x82"
+			  "\x3a\x6f\x25\xcb\x7d\x58\x1f\x9b"
+			  "\xaa\x9c\x11\xd5\x76\x67\xce\xde"
+			  "\x56\xd7\x5a\x80\x69\xea\x3a\x02"
+			  "\xf0\xc7\x7c\xe3\xcb\x40\xe5\x52"
+			  "\xd1\x10\x92\x78\x0b\x8e\x5b\xf1"
+			  "\xe3\x26\x1f\xe1\x15\x41\xc7\xba"
+			  "\x99\xdb\x08\x51\x1c\xd3\x01\xf4"
+			  "\x87\x47\x39\xb8\xd2\xdd\xbd\xfb"
+			  "\x66\x13\xdf\x1c\x01\x44\xf0\x7a"
+			  "\x1a\x6b\x13\xf5\xd5\x0b\xb8\xba"
+			  "\x53\xba\xe1\x76\xe3\x82\x07\x86"
+			  "\x95\x9e\x7d\x37\x1e\x60\xaf\x7c"
+			  "\x53\x12\x61\x68\xef\xb4\x47\xa6",
+			  .clen	= 128 + 16,
+	},
+};
+
+static const struct aead_testvec hmac_sha512_aes_ctr_rfc3686_tv_temp[] = {
+	{ /* RFC 3686 Case 1 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\xae\x68\x52\xf8\x12\x10\x67\xcc"
+			  "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e"
+			  "\x00\x00\x00\x30",
+		.klen	= 8 + 64 + 20,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.assoc	= "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\xe4\x09\x5d\x4f\xb7\xa7\xb3\x79"
+			  "\x2d\x61\x75\xa3\x26\x13\x11\xb8"
+			  "\xa4\x45\x3a\x44\x9c\xe5\x1c\xd9"
+			  "\x10\x43\x51\x2e\x76\x5e\xf8\x9d"
+			  "\x03\x12\x1a\x31\x00\x33\x10\xb4"
+			  "\x94\x4b\x70\x84\x6c\xda\xb1\x46"
+			  "\x24\xb6\x3b\x2a\xec\xd5\x67\xb8"
+			  "\x65\xa2\xbd\xac\x18\xe2\xf8\x55"
+			  "\xc6\x91\xb0\x92\x84\x2d\x74\x44"
+			  "\xa7\xee\xc3\x44\xa0\x07\x0e\x62",
+		.clen	= 16 + 64,
+	}, { /* RFC 3686 Case 2 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x7e\x24\x06\x78\x17\xfa\xe0\xd7"
+			  "\x43\xd6\xce\x1f\x32\x53\x91\x63"
+			  "\x00\x6c\xb6\xdb",
+		.klen	= 8 + 64 + 20,
+		.iv	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.assoc	= "\xc0\x54\x3b\x59\xda\x48\xd9\x0b",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x51\x04\xa1\x06\x16\x8a\x72\xd9"
+			  "\x79\x0d\x41\xee\x8e\xda\xd3\x88"
+			  "\xeb\x2e\x1e\xfc\x46\xda\x57\xc8"
+			  "\xfc\xe6\x30\xdf\x91\x41\xbe\x28"
+			  "\xec\x67\x0d\xb3\xbd\x98\x13\x01"
+			  "\x2b\x04\x9b\xe6\x06\x67\x3c\x76"
+			  "\xcd\x41\xb7\xcc\x70\x6c\x7f\xc8"
+			  "\x67\xbd\x22\x39\xb2\xaa\xe8\x88"
+			  "\xe0\x4f\x81\x52\xdf\xc9\xc3\xd6"
+			  "\x44\xf4\x66\x33\x87\x64\x61\x02"
+			  "\x02\xa2\x64\x15\x2b\xe9\x0b\x3d"
+			  "\x4c\xea\xa1\xa5\xa7\xc9\xd3\x1b",
+		.clen	= 32 + 64,
+	}, { /* RFC 3686 Case 3 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x14"	/* enc key length */
+			  "\x11\x22\x33\x44\x55\x66\x77\x88"
+			  "\x99\xaa\xbb\xcc\xdd\xee\xff\x11"
+			  "\x22\x33\x44\x55\x66\x77\x88\x99"
+			  "\xaa\xbb\xcc\xdd\xee\xff\x11\x22"
+			  "\x33\x44\x55\x66\x77\x88\x99\xaa"
+			  "\xbb\xcc\xdd\xee\xff\x11\x22\x33"
+			  "\x44\x55\x66\x77\x88\x99\xaa\xbb"
+			  "\xcc\xdd\xee\xff\x11\x22\x33\x44"
+			  "\x76\x91\xbe\x03\x5e\x50\x20\xa8"
+			  "\xac\x6e\x61\x85\x29\xf9\xa0\xdc"
+			  "\x00\xe0\x01\x7b",
+		.klen	= 8 + 64 + 20,
+		.iv	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.assoc	= "\x27\x77\x7f\x3f\x4a\x17\x86\xf0",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23",
+		.plen	= 36,
+		.ctext	= "\xc1\xcf\x48\xa8\x9f\x2f\xfd\xd9"
+			  "\xcf\x46\x52\xe9\xef\xdb\x72\xd7"
+			  "\x45\x40\xa4\x2b\xde\x6d\x78\x36"
+			  "\xd5\x9a\x5c\xea\xae\xf3\x10\x53"
+			  "\x25\xb2\x07\x2f"
+			  "\x6f\x90\xb6\xa3\x35\x43\x59\xff"
+			  "\x1e\x32\xd6\xfe\xfa\x33\xf9\xf0"
+			  "\x31\x2f\x03\x2d\x88\x1d\xab\xbf"
+			  "\x0e\x19\x16\xd9\xf3\x98\x3e\xdd"
+			  "\x0c\xec\xfe\xe8\x89\x13\x91\x15"
+			  "\xf6\x61\x65\x5c\x1b\x7d\xde\xc0"
+			  "\xe4\xba\x6d\x27\xe2\x89\x23\x24"
+			  "\x15\x82\x37\x3d\x48\xd3\xc9\x32",
+		.clen	= 36 + 64,
+	}, { /* RFC 3686 Case 4 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x16\xaf\x5b\x14\x5f\xc9\xf5\x79"
+			  "\xc1\x75\xf9\x3e\x3b\xfb\x0e\xed"
+			  "\x86\x3d\x06\xcc\xfd\xb7\x85\x15"
+			  "\x00\x00\x00\x48",
+		.klen	= 8 + 64 + 28,
+		.iv	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.assoc	= "\x36\x73\x3c\x14\x7d\x6d\x93\xcb",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x4b\x55\x38\x4f\xe2\x59\xc9\xc8"
+			  "\x4e\x79\x35\xa0\x03\xcb\xe9\x28"
+			  "\x25\xea\xdc\xad\x52\xb8\x0f\x70"
+			  "\xe7\x39\x83\x80\x10\x3f\x18\xc4"
+			  "\xf8\x59\x14\x25\x5f\xba\x20\x87"
+			  "\x0b\x04\x5e\xf7\xde\x41\x39\xff"
+			  "\xa2\xee\x84\x3f\x9d\x38\xfd\x17"
+			  "\xc0\x66\x5e\x74\x39\xe3\xd3\xd7"
+			  "\x3d\xbc\xe3\x99\x2f\xe7\xef\x37"
+			  "\x61\x03\xf3\x9e\x01\xaf\xba\x9d",
+		.clen	= 16 + 64,
+	}, { /* RFC 3686 Case 5 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x1c"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x7c\x5c\xb2\x40\x1b\x3d\xc3\x3c"
+			  "\x19\xe7\x34\x08\x19\xe0\xf6\x9c"
+			  "\x67\x8c\x3d\xb8\xe6\xf6\xa9\x1a"
+			  "\x00\x96\xb0\x3b",
+		.klen	= 8 + 64 + 28,
+		.iv	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.assoc	= "\x02\x0c\x6e\xad\xc2\xcb\x50\x0d",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\x45\x32\x43\xfc\x60\x9b\x23\x32"
+			  "\x7e\xdf\xaa\xfa\x71\x31\xcd\x9f"
+			  "\x84\x90\x70\x1c\x5a\xd4\xa7\x9c"
+			  "\xfc\x1f\xe0\xff\x42\xf4\xfb\x00"
+			  "\x51\xa3\xe6\x1d\x23\x7d\xd1\x18"
+			  "\x55\x9c\x1c\x92\x2b\xc2\xcd\xfe"
+			  "\x8a\xa8\xa5\x96\x65\x2e\x9d\xdb"
+			  "\x06\xd2\x1c\x57\x2b\x76\xb5\x9c"
+			  "\xd4\x3e\x8b\x61\x54\x2d\x08\xe5"
+			  "\xb2\xf8\x88\x20\x0c\xad\xe8\x85"
+			  "\x61\x8e\x5c\xa4\x96\x2c\xe2\x7d"
+			  "\x4f\xb6\x1d\xb2\x8c\xd7\xe3\x38",
+		.clen	= 32 + 64,
+	}, { /* RFC 3686 Case 7 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x77\x6b\xef\xf2\x85\x1d\xb0\x6f"
+			  "\x4c\x8a\x05\x42\xc8\x69\x6f\x6c"
+			  "\x6a\x81\xaf\x1e\xec\x96\xb4\xd3"
+			  "\x7f\xc1\xd6\x89\xe6\xc1\xc1\x04"
+			  "\x00\x00\x00\x60",
+		.klen	= 8 + 64 + 36,
+		.iv	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.assoc	= "\xdb\x56\x72\xc9\x7a\xa8\xf0\xb2",
+		.alen	= 8,
+		.ptext	= "Single block msg",
+		.plen	= 16,
+		.ctext	= "\x14\x5a\xd0\x1d\xbf\x82\x4e\xc7"
+			  "\x56\x08\x63\xdc\x71\xe3\xe0\xc0"
+			  "\x6b\x68\x0b\x99\x9a\x4d\xc8\xb9"
+			  "\x35\xea\xcd\x56\x3f\x40\xa2\xb6"
+			  "\x68\xda\x59\xd8\xa0\x89\xcd\x52"
+			  "\xb1\x6e\xed\xc1\x42\x10\xa5\x0f"
+			  "\x88\x0b\x80\xce\xc4\x67\xf0\x45"
+			  "\x5d\xb2\x9e\xde\x1c\x79\x52\x0d"
+			  "\xff\x75\x36\xd5\x0f\x52\x8e\xe5"
+			  "\x31\x85\xcf\x1d\x31\xf8\x62\x67",
+		.clen	= 16 + 64,
+	}, { /* RFC 3686 Case 8 */
+#ifdef __LITTLE_ENDIAN
+		.key    = "\x08\x00"		/* rta length */
+			  "\x01\x00"		/* rta type */
+#else
+		.key    = "\x00\x08"		/* rta length */
+			  "\x00\x01"		/* rta type */
+#endif
+			  "\x00\x00\x00\x24"	/* enc key length */
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\xf6\xd6\x6d\x6b\xd5\x2d\x59\xbb"
+			  "\x07\x96\x36\x58\x79\xef\xf8\x86"
+			  "\xc6\x6d\xd5\x1a\x5b\x6a\x99\x74"
+			  "\x4b\x50\x59\x0c\x87\xa2\x38\x84"
+			  "\x00\xfa\xac\x24",
+		.klen	= 8 + 64 + 36,
+		.iv	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.assoc	= "\xc1\x58\x5e\xf1\x5a\x43\xd8\x75",
+		.alen	= 8,
+		.ptext	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f",
+		.plen	= 32,
+		.ctext	= "\xf0\x5e\x23\x1b\x38\x94\x61\x2c"
+			  "\x49\xee\x00\x0b\x80\x4e\xb2\xa9"
+			  "\xb8\x30\x6b\x50\x8f\x83\x9d\x6a"
+			  "\x55\x30\x83\x1d\x93\x44\xaf\x1c"
+			  "\x9a\xac\x38\xbd\xf3\xcf\xd5\xd0"
+			  "\x09\x07\xa6\xe1\x7f\xd6\x79\x98"
+			  "\x4e\x90\x0e\xc0\x3d\xa0\xf2\x12"
+			  "\x52\x79\x9c\x17\xff\xb9\xb8\xe3"
+			  "\x2f\x31\xcb\xbd\x63\x70\x72\x7b"
+			  "\x4e\x1e\xd1\xde\xb5\x6b\x7d\x54"
+			  "\x68\x56\xdd\xe5\x53\xee\x29\xd2"
+			  "\x85\xa1\x73\x61\x00\xa9\x26\x8f",
+		.clen	= 32 + 64,
+	},
+};
+
 static const struct aead_testvec hmac_sha1_des_cbc_tv_temp[] = {
 	{ /*Generated with cryptopp*/
 #ifdef __LITTLE_ENDIAN
diff --git a/crypto/xor.c b/crypto/xor.c
deleted file mode 100644
index f39621a57bb33..0000000000000
--- a/crypto/xor.c
+++ /dev/null
@@ -1,174 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * xor.c : Multiple Devices driver for Linux
- *
- * Copyright (C) 1996, 1997, 1998, 1999, 2000,
- * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
- *
- * Dispatch optimized RAID-5 checksumming functions.
- */
-
-#define BH_TRACE 0
-#include <linux/module.h>
-#include <linux/gfp.h>
-#include <linux/raid/xor.h>
-#include <linux/jiffies.h>
-#include <linux/preempt.h>
-#include <asm/xor.h>
-
-#ifndef XOR_SELECT_TEMPLATE
-#define XOR_SELECT_TEMPLATE(x) (x)
-#endif
-
-/* The xor routines to use.  */
-static struct xor_block_template *active_template;
-
-void
-xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs)
-{
-	unsigned long *p1, *p2, *p3, *p4;
-
-	p1 = (unsigned long *) srcs[0];
-	if (src_count == 1) {
-		active_template->do_2(bytes, dest, p1);
-		return;
-	}
-
-	p2 = (unsigned long *) srcs[1];
-	if (src_count == 2) {
-		active_template->do_3(bytes, dest, p1, p2);
-		return;
-	}
-
-	p3 = (unsigned long *) srcs[2];
-	if (src_count == 3) {
-		active_template->do_4(bytes, dest, p1, p2, p3);
-		return;
-	}
-
-	p4 = (unsigned long *) srcs[3];
-	active_template->do_5(bytes, dest, p1, p2, p3, p4);
-}
-EXPORT_SYMBOL(xor_blocks);
-
-/* Set of all registered templates.  */
-static struct xor_block_template *__initdata template_list;
-
-#ifndef MODULE
-static void __init do_xor_register(struct xor_block_template *tmpl)
-{
-	tmpl->next = template_list;
-	template_list = tmpl;
-}
-
-static int __init register_xor_blocks(void)
-{
-	active_template = XOR_SELECT_TEMPLATE(NULL);
-
-	if (!active_template) {
-#define xor_speed	do_xor_register
-		// register all the templates and pick the first as the default
-		XOR_TRY_TEMPLATES;
-#undef xor_speed
-		active_template = template_list;
-	}
-	return 0;
-}
-#endif
-
-#define BENCH_SIZE	4096
-#define REPS		800U
-
-static void __init
-do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
-{
-	int speed;
-	unsigned long reps;
-	ktime_t min, start, t0;
-
-	tmpl->next = template_list;
-	template_list = tmpl;
-
-	preempt_disable();
-
-	reps = 0;
-	t0 = ktime_get();
-	/* delay start until time has advanced */
-	while ((start = ktime_get()) == t0)
-		cpu_relax();
-	do {
-		mb(); /* prevent loop optimization */
-		tmpl->do_2(BENCH_SIZE, b1, b2);
-		mb();
-	} while (reps++ < REPS || (t0 = ktime_get()) == start);
-	min = ktime_sub(t0, start);
-
-	preempt_enable();
-
-	// bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s]
-	speed = (1000 * reps * BENCH_SIZE) / (unsigned int)ktime_to_ns(min);
-	tmpl->speed = speed;
-
-	pr_info("   %-16s: %5d MB/sec\n", tmpl->name, speed);
-}
-
-static int __init
-calibrate_xor_blocks(void)
-{
-	void *b1, *b2;
-	struct xor_block_template *f, *fastest;
-
-	fastest = XOR_SELECT_TEMPLATE(NULL);
-
-	if (fastest) {
-		printk(KERN_INFO "xor: automatically using best "
-				 "checksumming function   %-10s\n",
-		       fastest->name);
-		goto out;
-	}
-
-	b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
-	if (!b1) {
-		printk(KERN_WARNING "xor: Yikes!  No memory available.\n");
-		return -ENOMEM;
-	}
-	b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
-
-	/*
-	 * If this arch/cpu has a short-circuited selection, don't loop through
-	 * all the possible functions, just test the best one
-	 */
-
-#define xor_speed(templ)	do_xor_speed((templ), b1, b2)
-
-	printk(KERN_INFO "xor: measuring software checksum speed\n");
-	template_list = NULL;
-	XOR_TRY_TEMPLATES;
-	fastest = template_list;
-	for (f = fastest; f; f = f->next)
-		if (f->speed > fastest->speed)
-			fastest = f;
-
-	pr_info("xor: using function: %s (%d MB/sec)\n",
-	       fastest->name, fastest->speed);
-
-#undef xor_speed
-
-	free_pages((unsigned long)b1, 2);
-out:
-	active_template = fastest;
-	return 0;
-}
-
-static __exit void xor_exit(void) { }
-
-MODULE_DESCRIPTION("RAID-5 checksumming functions");
-MODULE_LICENSE("GPL");
-
-#ifndef MODULE
-/* when built-in xor.o must initialize before drivers/md/md.o */
-core_initcall(register_xor_blocks);
-#endif
-
-module_init(calibrate_xor_blocks);
-module_exit(xor_exit);
diff --git a/drivers/accel/amdxdna/Makefile b/drivers/accel/amdxdna/Makefile
index 3fa0e74fd8f5d..cf9bf19dedb9e 100644
--- a/drivers/accel/amdxdna/Makefile
+++ b/drivers/accel/amdxdna/Makefile
@@ -11,6 +11,7 @@ amdxdna-y := \
 	aie2_solver.o \
 	amdxdna_ctx.o \
 	amdxdna_gem.o \
+	amdxdna_iommu.o \
 	amdxdna_mailbox.o \
 	amdxdna_mailbox_helper.o \
 	amdxdna_pci_drv.o \
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index c0d348884f749..1fa8e92d1449f 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -29,6 +29,16 @@ MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)");
 
 #define HWCTX_MAX_TIMEOUT	60000 /* milliseconds */
 
+struct aie2_ctx_health {
+	struct amdxdna_ctx_health header;
+	u32 txn_op_idx;
+	u32 ctx_pc;
+	u32 fatal_error_type;
+	u32 fatal_error_exception_type;
+	u32 fatal_error_exception_pc;
+	u32 fatal_error_app_module;
+};
+
 static void aie2_job_release(struct kref *ref)
 {
 	struct amdxdna_sched_job *job;
@@ -39,6 +49,7 @@ static void aie2_job_release(struct kref *ref)
 	wake_up(&job->hwctx->priv->job_free_wq);
 	if (job->out_fence)
 		dma_fence_put(job->out_fence);
+	kfree(job->aie2_job_health);
 	kfree(job);
 }
 
@@ -68,7 +79,8 @@ static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hw
 	}
 
 	ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
-				heap->mem.userptr, heap->mem.size);
+				amdxdna_obj_dma_addr(hwctx->client, heap),
+				heap->mem.size);
 	if (ret) {
 		XDNA_ERR(xdna, "Map host buf failed, ret %d", ret);
 		goto out;
@@ -174,6 +186,50 @@ aie2_sched_notify(struct amdxdna_sched_job *job)
 	aie2_job_put(job);
 }
 
+static void aie2_set_cmd_timeout(struct amdxdna_sched_job *job)
+{
+	struct aie2_ctx_health *aie2_health __free(kfree) = NULL;
+	struct amdxdna_dev *xdna = job->hwctx->client->xdna;
+	struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
+	struct app_health_report *report = job->aie2_job_health;
+	u32 fail_cmd_idx = 0;
+
+	if (!report)
+		goto set_timeout;
+
+	XDNA_ERR(xdna, "Firmware timeout state capture:");
+	XDNA_ERR(xdna, "\tVersion: %d.%d", report->major, report->minor);
+	XDNA_ERR(xdna, "\tReport size: 0x%x", report->size);
+	XDNA_ERR(xdna, "\tContext ID: %d", report->context_id);
+	XDNA_ERR(xdna, "\tDPU PC: 0x%x", report->dpu_pc);
+	XDNA_ERR(xdna, "\tTXN OP ID: 0x%x", report->txn_op_id);
+	XDNA_ERR(xdna, "\tContext PC: 0x%x", report->ctx_pc);
+	XDNA_ERR(xdna, "\tFatal error type: 0x%x", report->fatal_info.fatal_type);
+	XDNA_ERR(xdna, "\tFatal error exception type: 0x%x", report->fatal_info.exception_type);
+	XDNA_ERR(xdna, "\tFatal error exception PC: 0x%x", report->fatal_info.exception_pc);
+	XDNA_ERR(xdna, "\tFatal error app module: 0x%x", report->fatal_info.app_module);
+	XDNA_ERR(xdna, "\tFatal error task ID: %d", report->fatal_info.task_index);
+	XDNA_ERR(xdna, "\tTimed out sub command ID: %d", report->run_list_id);
+
+	fail_cmd_idx = report->run_list_id;
+	aie2_health = kzalloc_obj(*aie2_health);
+	if (!aie2_health)
+		goto set_timeout;
+
+	aie2_health->header.version = AMDXDNA_CMD_CTX_HEALTH_V1;
+	aie2_health->header.npu_gen = AMDXDNA_CMD_CTX_HEALTH_AIE2;
+	aie2_health->txn_op_idx = report->txn_op_id;
+	aie2_health->ctx_pc = report->ctx_pc;
+	aie2_health->fatal_error_type = report->fatal_info.fatal_type;
+	aie2_health->fatal_error_exception_type = report->fatal_info.exception_type;
+	aie2_health->fatal_error_exception_pc = report->fatal_info.exception_pc;
+	aie2_health->fatal_error_app_module = report->fatal_info.app_module;
+
+set_timeout:
+	amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_TIMEOUT,
+			      aie2_health, sizeof(*aie2_health));
+}
+
 static int
 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size)
 {
@@ -185,13 +241,13 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size)
 	cmd_abo = job->cmd_bo;
 
 	if (unlikely(job->job_timeout)) {
-		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_TIMEOUT);
+		aie2_set_cmd_timeout(job);
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (unlikely(!data) || unlikely(size != sizeof(u32))) {
-		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT);
+		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -201,7 +257,7 @@ aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size)
 	if (status == AIE2_STATUS_SUCCESS)
 		amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED);
 	else
-		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR);
+		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR, NULL, 0);
 
 out:
 	aie2_sched_notify(job);
@@ -235,21 +291,21 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size)
 	struct amdxdna_sched_job *job = handle;
 	struct amdxdna_gem_obj *cmd_abo;
 	struct amdxdna_dev *xdna;
+	u32 fail_cmd_idx = 0;
 	u32 fail_cmd_status;
-	u32 fail_cmd_idx;
 	u32 cmd_status;
 	int ret = 0;
 
 	cmd_abo = job->cmd_bo;
 
 	if (unlikely(job->job_timeout)) {
-		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_TIMEOUT);
+		aie2_set_cmd_timeout(job);
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) {
-		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT);
+		amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -269,10 +325,10 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size)
 		 fail_cmd_idx, fail_cmd_status);
 
 	if (fail_cmd_status == AIE2_STATUS_SUCCESS) {
-		amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT);
+		amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT, NULL, 0);
 		ret = -EINVAL;
 	} else {
-		amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR);
+		amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR, NULL, 0);
 	}
 
 out:
@@ -352,12 +408,26 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job)
 {
 	struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
 	struct amdxdna_hwctx *hwctx = job->hwctx;
+	struct app_health_report *report;
 	struct amdxdna_dev *xdna;
+	int ret;
 
 	xdna = hwctx->client->xdna;
 	trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq);
 	job->job_timeout = true;
+
 	mutex_lock(&xdna->dev_lock);
+	report = kzalloc_obj(*report);
+	if (!report)
+		goto reset_hwctx;
+
+	ret = aie2_query_app_health(xdna->dev_handle, hwctx->fw_ctx_id, report);
+	if (ret)
+		kfree(report);
+	else
+		job->aie2_job_health = report;
+
+reset_hwctx:
 	aie2_hwctx_stop(xdna, hwctx, sched_job);
 
 	aie2_hwctx_restart(xdna, hwctx);
@@ -627,7 +697,8 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 	}
 
 	ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
-				heap->mem.userptr, heap->mem.size);
+				amdxdna_obj_dma_addr(hwctx->client, heap),
+				heap->mem.size);
 	if (ret) {
 		XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret);
 		goto release_resource;
diff --git a/drivers/accel/amdxdna/aie2_error.c b/drivers/accel/amdxdna/aie2_error.c
index 9f0ea1d9c05f1..58abb59b61535 100644
--- a/drivers/accel/amdxdna/aie2_error.c
+++ b/drivers/accel/amdxdna/aie2_error.c
@@ -355,9 +355,8 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 		return -ENOMEM;
 
 	events->buf = aie2_alloc_msg_buffer(ndev, &total_size, &events->addr);
-
-	if (!events->buf) {
-		ret = -ENOMEM;
+	if (IS_ERR(events->buf)) {
+		ret = PTR_ERR(events->buf);
 		goto free_events;
 	}
 	events->size = total_size;
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index ffcf3be79e236..4ec5913068549 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -61,11 +61,27 @@ void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
 	*size = max(*size, SZ_8K);
 	order = get_order(*size);
 	if (order > MAX_PAGE_ORDER)
-		return NULL;
+		return ERR_PTR(-EINVAL);
 	*size = PAGE_SIZE << order;
 
+	if (amdxdna_iova_on(xdna))
+		return amdxdna_iommu_alloc(xdna, *size, dma_addr);
+
 	return dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
-				     DMA_FROM_DEVICE, GFP_KERNEL);
+				      DMA_FROM_DEVICE, GFP_KERNEL);
+}
+
+void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
+			  void *cpu_addr, dma_addr_t dma_addr)
+{
+	struct amdxdna_dev *xdna = ndev->xdna;
+
+	if (amdxdna_iova_on(xdna)) {
+		amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
+		return;
+	}
+
+	dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
 }
 
 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
@@ -256,7 +272,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 	req.num_col = hwctx->num_col;
 	req.num_unused_col = hwctx->num_unused_col;
 	req.num_cq_pairs_requested = 1;
-	req.pasid = hwctx->client->pasid;
+	req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0;
 	req.context_priority = aie2_get_context_priority(ndev, hwctx);
 
 	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
@@ -380,8 +396,8 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 	int ret;
 
 	buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
-	if (!buff_addr)
-		return -ENOMEM;
+	if (IS_ERR(buff_addr))
+		return PTR_ERR(buff_addr);
 
 	/* Go through each hardware context and mark the AIE columns that are active */
 	list_for_each_entry(client, &xdna->client_list, node)
@@ -436,8 +452,8 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 		return -EINVAL;
 
 	addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
-	if (!addr)
-		return -ENOMEM;
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
 
 	req.buf_addr = dma_addr;
 	req.buf_size = buf_sz;
@@ -986,7 +1002,7 @@ int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
 	struct amdxdna_cmd_chain *payload;
 	struct xdna_mailbox_msg msg;
 	union exec_chain_req req;
-	u32 payload_len;
+	u32 payload_len, ccnt;
 	u32 offset = 0;
 	size_t size;
 	int ret;
@@ -995,12 +1011,24 @@ int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
 
 	op = amdxdna_cmd_get_op(cmd_abo);
 	payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len);
-	if (op != ERT_CMD_CHAIN || !payload ||
-	    payload_len < struct_size(payload, data, payload->command_count))
+	if (op != ERT_CMD_CHAIN) {
+		XDNA_DBG(xdna, "Invalid op code %d", op);
 		return -EINVAL;
+	}
+
+	if (!payload) {
+		XDNA_DBG(xdna, "Failed to get command payload");
+		return -EINVAL;
+	}
+
+	ccnt = payload->command_count;
+	if (payload_len < struct_size(payload, data, ccnt)) {
+		XDNA_DBG(xdna, "Invalid command count %d", ccnt);
+		return -EINVAL;
+	}
 
 	op = ERT_INVALID_CMD;
-	for (i = 0; i < payload->command_count; i++) {
+	for (i = 0; i < ccnt; i++) {
 		u32 boh = (u32)(payload->data[i]);
 		struct amdxdna_gem_obj *abo;
 
@@ -1019,19 +1047,26 @@ int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
 
 		offset += size;
 	}
+
+	XDNA_DBG(xdna, "Total %d commands:", ccnt);
+	print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4,
+			     cmdbuf_abo->mem.kva, offset, false);
+
 	msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
 	if (msg.opcode == MSG_OP_MAX_OPCODE)
 		return -EOPNOTSUPP;
 
 	/* The offset is the accumulated total size of the cmd buffer */
 	EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr,
-					   offset, payload->command_count);
+					   offset, ccnt);
 	drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset);
 
 	msg.handle = job;
 	msg.notify_cb = notify_cb;
 	msg.send_data = (u8 *)&req;
 	msg.send_size = sizeof(req);
+	print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
+			     &req, msg.send_size, false);
 	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 	if (ret) {
 		XDNA_ERR(xdna, "Send message failed");
@@ -1060,6 +1095,9 @@ int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
 	if (ret)
 		return ret;
 
+	print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4,
+			     cmdbuf_abo->mem.kva, size, false);
+
 	msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
 	if (msg.opcode == MSG_OP_MAX_OPCODE)
 		return -EOPNOTSUPP;
@@ -1072,6 +1110,8 @@ int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
 	msg.notify_cb = notify_cb;
 	msg.send_data = (u8 *)&req;
 	msg.send_size = sizeof(req);
+	print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
+			     &req, msg.send_size, false);
 	ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 	if (ret) {
 		XDNA_ERR(hwctx->client->xdna, "Send message failed");
@@ -1145,3 +1185,44 @@ int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *
 
 	return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
 }
+
+int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
+			  struct app_health_report *report)
+{
+	DECLARE_AIE2_MSG(get_app_health, MSG_OP_GET_APP_HEALTH);
+	struct amdxdna_dev *xdna = ndev->xdna;
+	struct app_health_report *buf;
+	dma_addr_t dma_addr;
+	u32 buf_size;
+	int ret;
+
+	if (!AIE2_FEATURE_ON(ndev, AIE2_APP_HEALTH)) {
+		XDNA_DBG(xdna, "App health feature not supported");
+		return -EOPNOTSUPP;
+	}
+
+	buf_size = sizeof(*report);
+	buf = aie2_alloc_msg_buffer(ndev, &buf_size, &dma_addr);
+	if (IS_ERR(buf)) {
+		XDNA_ERR(xdna, "Failed to allocate buffer for app health");
+		return PTR_ERR(buf);
+	}
+
+	req.buf_addr = dma_addr;
+	req.context_id = context_id;
+	req.buf_size = buf_size;
+
+	drm_clflush_virt_range(buf, sizeof(*report));
+	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	if (ret) {
+		XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status);
+		goto free_buf;
+	}
+
+	/* Copy the report to caller's buffer */
+	memcpy(report, buf, sizeof(*report));
+
+free_buf:
+	aie2_free_msg_buffer(ndev, buf_size, buf, dma_addr);
+	return ret;
+}
diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h
index 728ef56f7f0ad..f18e89a39e357 100644
--- a/drivers/accel/amdxdna/aie2_msg_priv.h
+++ b/drivers/accel/amdxdna/aie2_msg_priv.h
@@ -31,6 +31,7 @@ enum aie2_msg_opcode {
 	MSG_OP_SET_RUNTIME_CONFIG          = 0x10A,
 	MSG_OP_GET_RUNTIME_CONFIG          = 0x10B,
 	MSG_OP_REGISTER_ASYNC_EVENT_MSG    = 0x10C,
+	MSG_OP_GET_APP_HEALTH              = 0x114,
 	MSG_OP_MAX_DRV_OPCODE,
 	MSG_OP_GET_PROTOCOL_VERSION        = 0x301,
 	MSG_OP_MAX_OPCODE
@@ -451,4 +452,55 @@ struct config_debug_bo_req {
 struct config_debug_bo_resp {
 	enum aie2_msg_status	status;
 } __packed;
+
+struct fatal_error_info {
+	__u32 fatal_type;         /* Fatal error type */
+	__u32 exception_type;     /* Only valid if fatal_type is a specific value */
+	__u32 exception_argument; /* Argument based on exception type */
+	__u32 exception_pc;       /* Program Counter at the time of the exception */
+	__u32 app_module;         /* Error module name */
+	__u32 task_index;         /* Index of the task in which the error occurred */
+	__u32 reserved[128];
+};
+
+struct app_health_report {
+	__u16 major;
+	__u16 minor;
+	__u32 size;
+	__u32 context_id;
+	/*
+	 * Program Counter (PC) of the last initiated DPU opcode, as reported by the ERT
+	 * application. Before execution begins or after successful completion, the value is set
+	 * to UINT_MAX. If execution halts prematurely due to an error, this field retains the
+	 * opcode's PC value.
+	 * Note: To optimize performance, the ERT may simplify certain aspects of reporting.
+	 * Proper interpretation requires familiarity with the implementation details.
+	 */
+	__u32 dpu_pc;
+	/*
+	 * Index of the last initiated TXN opcode.
+	 * Before execution starts or after successful completion, the value is set to UINT_MAX.
+	 * If execution halts prematurely due to an error, this field retains the opcode's ID.
+	 * Note: To optimize performance, the ERT may simplify certain aspects of reporting.
+	 * Proper interpretation requires familiarity with the implementation details.
+	 */
+	__u32 txn_op_id;
+	/* The PC of the context at the time of the report */
+	__u32 ctx_pc;
+	struct fatal_error_info		fatal_info;
+	/* Index of the most recently executed run list entry. */
+	__u32 run_list_id;
+};
+
+struct get_app_health_req {
+	__u32 context_id;
+	__u32 buf_size;
+	__u64 buf_addr;
+} __packed;
+
+struct get_app_health_resp {
+	enum aie2_msg_status status;
+	__u32 required_buffer_size;
+	__u32 reserved[7];
+} __packed;
 #endif /* _AIE2_MSG_PRIV_H_ */
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index 4924a9da55b64..9e39bfe75971b 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -10,6 +10,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 #include <drm/gpu_scheduler.h>
+#include <linux/amd-pmf-io.h>
 #include <linux/cleanup.h>
 #include <linux/errno.h>
 #include <linux/firmware.h>
@@ -791,12 +792,64 @@ static int aie2_get_clock_metadata(struct amdxdna_client *client,
 	return ret;
 }
 
+static int aie2_get_sensors(struct amdxdna_client *client,
+			    struct amdxdna_drm_get_info *args)
+{
+	struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
+	struct amdxdna_drm_query_sensor sensor = {};
+	struct amd_pmf_npu_metrics npu_metrics;
+	u32 sensors_count = 0, i;
+	int ret;
+
+	ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics);
+	if (ret)
+		return ret;
+
+	sensor.type = AMDXDNA_SENSOR_TYPE_POWER;
+	sensor.input = npu_metrics.npu_power;
+	sensor.unitm = -3;
+	scnprintf(sensor.label, sizeof(sensor.label), "Total Power");
+	scnprintf(sensor.units, sizeof(sensor.units), "mW");
+
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor)))
+		return -EFAULT;
+
+	sensors_count++;
+	if (args->buffer_size <= sensors_count * sizeof(sensor))
+		goto out;
+
+	for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) {
+		memset(&sensor, 0, sizeof(sensor));
+		sensor.input = npu_metrics.npu_busy[i];
+		sensor.type = AMDXDNA_SENSOR_TYPE_COLUMN_UTILIZATION;
+		sensor.unitm = 0;
+		scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i);
+		scnprintf(sensor.units, sizeof(sensor.units), "%%");
+
+		if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor),
+				 &sensor, sizeof(sensor)))
+			return -EFAULT;
+
+		sensors_count++;
+		if (args->buffer_size <= sensors_count * sizeof(sensor))
+			goto out;
+	}
+
+out:
+	args->buffer_size = sensors_count * sizeof(sensor);
+
+	return 0;
+}
+
 static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
 {
 	struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL;
 	struct amdxdna_drm_get_array *array_args = arg;
 	struct amdxdna_drm_hwctx_entry __user *buf;
+	struct app_health_report report;
+	struct amdxdna_dev_hdl *ndev;
 	u32 size;
+	int ret;
 
 	if (!array_args->num_element)
 		return -EINVAL;
@@ -819,6 +872,17 @@ static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
 	tmp->latency = hwctx->qos.latency;
 	tmp->frame_exec_time = hwctx->qos.frame_exec_time;
 	tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE;
+	ndev = hwctx->client->xdna->dev_handle;
+	ret = aie2_query_app_health(ndev, hwctx->fw_ctx_id, &report);
+	if (!ret) {
+		/* Fill in app health report fields */
+		tmp->txn_op_idx = report.txn_op_id;
+		tmp->ctx_pc = report.ctx_pc;
+		tmp->fatal_error_type = report.fatal_info.fatal_type;
+		tmp->fatal_error_exception_type = report.fatal_info.exception_type;
+		tmp->fatal_error_exception_pc = report.fatal_info.exception_pc;
+		tmp->fatal_error_app_module = report.fatal_info.app_module;
+	}
 
 	buf = u64_to_user_ptr(array_args->buffer);
 	size = min(sizeof(*tmp), array_args->element_size);
@@ -994,6 +1058,9 @@ static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_i
 	case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
 		ret = aie2_get_clock_metadata(client, args);
 		break;
+	case DRM_AMDXDNA_QUERY_SENSORS:
+		ret = aie2_get_sensors(client, args);
+		break;
 	case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
 		ret = aie2_get_hwctx_status(client, args);
 		break;
diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index e72311c77996a..efcf4be035f04 100644
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -7,8 +7,10 @@
 #define _AIE2_PCI_H_
 
 #include <drm/amdxdna_accel.h>
+#include <linux/limits.h>
 #include <linux/semaphore.h>
 
+#include "aie2_msg_priv.h"
 #include "amdxdna_mailbox.h"
 
 #define AIE2_INTERVAL	20000	/* us */
@@ -46,6 +48,33 @@
 	pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->xdna->dev_info->mbox_bar); \
 })
 
+#if IS_ENABLED(CONFIG_AMD_PMF)
+#define AIE2_GET_PMF_NPU_METRICS(metrics) amd_pmf_get_npu_data(metrics)
+#define AIE2_GET_PMF_NPU_DATA(field, val)				\
+({									\
+	struct amd_pmf_npu_metrics _npu_metrics;			\
+	int _ret;							\
+									\
+	_ret = amd_pmf_get_npu_data(&_npu_metrics);			\
+	val = _ret ? U32_MAX : _npu_metrics.field;			\
+	(_ret);								\
+})
+#else
+#define AIE2_GET_PMF_NPU_METRICS(metrics)				\
+({									\
+	typeof(metrics) _m = metrics;					\
+	memset(_m, 0xff, sizeof(*_m));					\
+	(-EOPNOTSUPP);							\
+})
+
+#define SENSOR_DEFAULT_npu_power	U32_MAX
+#define AIE2_GET_PMF_NPU_DATA(field, val)				\
+({									\
+	val = SENSOR_DEFAULT_##field;					\
+	(-EOPNOTSUPP);							\
+})
+#endif
+
 enum aie2_smu_reg_idx {
 	SMU_CMD_REG = 0,
 	SMU_ARG_REG,
@@ -233,6 +262,7 @@ enum aie2_fw_feature {
 	AIE2_NPU_COMMAND,
 	AIE2_PREEMPT,
 	AIE2_TEMPORAL_ONLY,
+	AIE2_APP_HEALTH,
 	AIE2_FEATURE_MAX
 };
 
@@ -243,6 +273,7 @@ struct aie2_fw_feature_tbl {
 	u32 min_minor;
 };
 
+#define AIE2_ALL_FEATURES	GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND)
 #define AIE2_FEATURE_ON(ndev, feature)	test_bit(feature, &(ndev)->feature_mask)
 
 struct amdxdna_dev_priv {
@@ -313,6 +344,8 @@ int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *ver
 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata);
 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 				struct amdxdna_fw_ver *fw_ver);
+int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
+			  struct app_health_report *report);
 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size);
@@ -338,9 +371,8 @@ int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *
 			 int (*notify_cb)(void *, void __iomem *, size_t));
 void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
 			    dma_addr_t *dma_addr);
-#define aie2_free_msg_buffer(ndev, size, buff_addr, dma_addr)		\
-	dma_free_noncoherent((ndev)->xdna->ddev.dev, size, buff_addr,	\
-			     dma_addr, DMA_FROM_DEVICE)
+void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
+			  void *cpu_addr, dma_addr_t dma_addr);
 
 /* aie2_hwctx.c */
 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c
index 838430903a3ea..2abe27dbd44d1 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.c
+++ b/drivers/accel/amdxdna/amdxdna_ctx.c
@@ -138,7 +138,8 @@ u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
 
 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
 			  struct amdxdna_sched_job *job, u32 cmd_idx,
-			  enum ert_cmd_state error_state)
+			  enum ert_cmd_state error_state,
+			  void *err_data, size_t size)
 {
 	struct amdxdna_client *client = job->hwctx->client;
 	struct amdxdna_cmd *cmd = abo->mem.kva;
@@ -157,6 +158,9 @@ int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
 	}
 
 	memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd));
+	if (err_data)
+		memcpy(cmd->data, err_data, min(size, abo->mem.size - sizeof(*cmd)));
+
 	if (cc)
 		amdxdna_gem_put_obj(abo);
 
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h
index fbdf9d0008713..57db1527a93b5 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.h
+++ b/drivers/accel/amdxdna/amdxdna_ctx.h
@@ -72,6 +72,13 @@ struct amdxdna_cmd_preempt_data {
 	u32 prop_args[];    /* properties and regular kernel arguments */
 };
 
+#define AMDXDNA_CMD_CTX_HEALTH_V1	1
+#define AMDXDNA_CMD_CTX_HEALTH_AIE2	0
+struct amdxdna_ctx_health {
+	u32 version;
+	u32 npu_gen;
+};
+
 /* Exec buffer command header format */
 #define AMDXDNA_CMD_STATE		GENMASK(3, 0)
 #define AMDXDNA_CMD_EXTRA_CU_MASK	GENMASK(11, 10)
@@ -122,6 +129,11 @@ struct amdxdna_drv_cmd {
 	u32			result;
 };
 
+struct app_health_report;
+union amdxdna_job_priv {
+	struct app_health_report *aie2_health;
+};
+
 struct amdxdna_sched_job {
 	struct drm_sched_job	base;
 	struct kref		refcnt;
@@ -136,10 +148,13 @@ struct amdxdna_sched_job {
 	u64			seq;
 	struct amdxdna_drv_cmd	*drv_cmd;
 	struct amdxdna_gem_obj	*cmd_bo;
+	union amdxdna_job_priv	priv;
 	size_t			bo_cnt;
 	struct drm_gem_object	*bos[] __counted_by(bo_cnt);
 };
 
+#define aie2_job_health priv.aie2_health
+
 static inline u32
 amdxdna_cmd_get_op(struct amdxdna_gem_obj *abo)
 {
@@ -169,7 +184,8 @@ void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size);
 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo);
 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
 			  struct amdxdna_sched_job *job, u32 cmd_idx,
-			  enum ert_cmd_state error_state);
+			  enum ert_cmd_state error_state,
+			  void *err_data, size_t size);
 
 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job);
 void amdxdna_hwctx_remove_all(struct amdxdna_client *client);
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index d60db49ead714..d80cf164740c3 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -474,12 +474,50 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
 	drm_gem_shmem_free(&abo->base);
 }
 
+static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
+	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+	int ret;
+
+	guard(mutex)(&abo->lock);
+	if (abo->ref) {
+		abo->ref++;
+		return 0;
+	}
+
+	if (amdxdna_iova_on(xdna)) {
+		ret = amdxdna_iommu_map_bo(xdna, abo);
+		if (ret)
+			return ret;
+	}
+	abo->ref++;
+
+	return 0;
+}
+
+static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
+	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+
+	guard(mutex)(&abo->lock);
+	abo->ref--;
+	if (abo->ref)
+		return;
+
+	if (amdxdna_iova_on(xdna))
+		amdxdna_iommu_unmap_bo(xdna, abo);
+}
+
 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
 	.free = amdxdna_gem_dev_obj_free,
 };
 
 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
 	.free = amdxdna_gem_obj_free,
+	.open = amdxdna_gem_obj_open,
+	.close = amdxdna_gem_obj_close,
 	.print_info = drm_gem_shmem_object_print_info,
 	.pin = drm_gem_shmem_object_pin,
 	.unpin = drm_gem_shmem_object_unpin,
@@ -506,6 +544,7 @@ amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
 
 	abo->mem.userptr = AMDXDNA_INVALID_ADDR;
 	abo->mem.dev_addr = AMDXDNA_INVALID_ADDR;
+	abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
 	abo->mem.size = size;
 	INIT_LIST_HEAD(&abo->mem.umap_list);
 
@@ -621,6 +660,7 @@ amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
 	abo = to_xdna_obj(gobj);
 	abo->attach = attach;
 	abo->dma_buf = dma_buf;
+	abo->type = AMDXDNA_BO_SHMEM;
 
 	return gobj;
 
@@ -905,7 +945,10 @@ int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm
 
 	abo = to_xdna_obj(gobj);
 	args->vaddr = abo->mem.userptr;
-	args->xdna_addr = abo->mem.dev_addr;
+	if (abo->mem.dev_addr != AMDXDNA_INVALID_ADDR)
+		args->xdna_addr = abo->mem.dev_addr;
+	else
+		args->xdna_addr = abo->mem.dma_addr;
 
 	if (abo->type != AMDXDNA_BO_DEV)
 		args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node);
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index f79fc7f3c93bc..fbeb622e7cf96 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -6,7 +6,9 @@
 #ifndef _AMDXDNA_GEM_H_
 #define _AMDXDNA_GEM_H_
 
+#include <drm/drm_gem_shmem_helper.h>
 #include <linux/hmm.h>
+#include <linux/iova.h>
 #include "amdxdna_pci_drv.h"
 
 struct amdxdna_umap {
@@ -25,6 +27,7 @@ struct amdxdna_mem {
 	u64				userptr;
 	void				*kva;
 	u64				dev_addr;
+	u64				dma_addr;
 	size_t				size;
 	struct page			**pages;
 	u32				nr_pages;
@@ -39,6 +42,7 @@ struct amdxdna_gem_obj {
 	bool				pinned;
 	struct mutex			lock; /* Protects: pinned */
 	struct amdxdna_mem		mem;
+	u32				ref;
 
 	/* Below members is uninitialized when needed */
 	struct drm_mm			mm; /* For AMDXDNA_BO_DEV_HEAP */
@@ -68,6 +72,12 @@ static inline u64 amdxdna_dev_bo_offset(struct amdxdna_gem_obj *abo)
 	return abo->mem.dev_addr - abo->client->dev_heap->mem.dev_addr;
 }
 
+static inline u64 amdxdna_obj_dma_addr(struct amdxdna_client *client,
+				       struct amdxdna_gem_obj *abo)
+{
+	return amdxdna_pasid_on(client) ? abo->mem.userptr : abo->mem.dma_addr;
+}
+
 void amdxdna_umap_put(struct amdxdna_umap *mapp);
 
 struct drm_gem_object *
diff --git a/drivers/accel/amdxdna/amdxdna_iommu.c b/drivers/accel/amdxdna/amdxdna_iommu.c
new file mode 100644
index 0000000000000..4626434d4180c
--- /dev/null
+++ b/drivers/accel/amdxdna/amdxdna_iommu.c
@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <linux/iommu.h>
+#include <linux/iova.h>
+
+#include "amdxdna_gem.h"
+#include "amdxdna_pci_drv.h"
+
+static bool force_iova;
+module_param(force_iova, bool, 0600);
+MODULE_PARM_DESC(force_iova, "Force use IOVA (Default false)");
+
+static struct iova *amdxdna_iommu_alloc_iova(struct amdxdna_dev *xdna,
+					     size_t size,
+					     dma_addr_t *dma_addr,
+					     bool size_aligned)
+{
+	unsigned long shift, end;
+	struct iova *iova;
+
+	end = xdna->domain->geometry.aperture_end;
+	shift = iova_shift(&xdna->iovad);
+	size = iova_align(&xdna->iovad, size);
+
+	iova = alloc_iova(&xdna->iovad, size >> shift, end >> shift, size_aligned);
+	if (!iova)
+		return ERR_PTR(-ENOMEM);
+
+	*dma_addr = iova_dma_addr(&xdna->iovad, iova);
+
+	return iova;
+}
+
+int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
+{
+	struct sg_table *sgt;
+	dma_addr_t dma_addr;
+	struct iova *iova;
+	size_t size;
+
+	if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHMEM)
+		return 0;
+
+	sgt = drm_gem_shmem_get_pages_sgt(&abo->base);
+	if (IS_ERR(sgt)) {
+		XDNA_ERR(xdna, "Get sgt failed, ret %ld", PTR_ERR(sgt));
+		return PTR_ERR(sgt);
+	}
+
+	if (!sgt->orig_nents || !sg_page(sgt->sgl)) {
+		XDNA_ERR(xdna, "sgl is zero length or not page backed");
+		return -EOPNOTSUPP;
+	}
+
+	iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr,
+					(abo->type == AMDXDNA_BO_DEV_HEAP));
+	if (IS_ERR(iova)) {
+		XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
+		return PTR_ERR(iova);
+	}
+
+	size = iommu_map_sgtable(xdna->domain, dma_addr, sgt,
+				 IOMMU_READ | IOMMU_WRITE);
+	if (size < abo->mem.size) {
+		__free_iova(&xdna->iovad, iova);
+		return -ENXIO;
+	}
+
+	abo->mem.dma_addr = dma_addr;
+
+	return 0;
+}
+
+void amdxdna_iommu_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
+{
+	size_t size;
+
+	if (abo->mem.dma_addr == AMDXDNA_INVALID_ADDR)
+		return;
+
+	size = iova_align(&xdna->iovad, abo->mem.size);
+	iommu_unmap(xdna->domain, abo->mem.dma_addr, size);
+	free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr));
+	abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
+}
+
+void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr)
+{
+	struct iova *iova;
+	void *cpu_addr;
+	int ret;
+
+	iova = amdxdna_iommu_alloc_iova(xdna, size, dma_addr, true);
+	if (IS_ERR(iova)) {
+		XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
+		return iova;
+	}
+
+	cpu_addr = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
+	if (!cpu_addr) {
+		ret = -ENOMEM;
+		goto free_iova;
+	}
+
+	ret = iommu_map(xdna->domain, *dma_addr, virt_to_phys(cpu_addr),
+			iova_align(&xdna->iovad, size),
+			IOMMU_READ | IOMMU_WRITE, GFP_KERNEL);
+	if (ret)
+		goto free_iova;
+
+	return cpu_addr;
+
+free_iova:
+	__free_iova(&xdna->iovad, iova);
+	return ERR_PTR(ret);
+}
+
+void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
+			void *cpu_addr, dma_addr_t dma_addr)
+{
+	iommu_unmap(xdna->domain, dma_addr, iova_align(&xdna->iovad, size));
+	free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, dma_addr));
+	free_pages((unsigned long)cpu_addr, get_order(size));
+}
+
+int amdxdna_iommu_init(struct amdxdna_dev *xdna)
+{
+	unsigned long order;
+	int ret;
+
+	xdna->group = iommu_group_get(xdna->ddev.dev);
+	if (!xdna->group || !force_iova)
+		return 0;
+
+	XDNA_WARN(xdna, "Enabled force_iova mode.");
+	xdna->domain = iommu_paging_domain_alloc_flags(xdna->ddev.dev,
+						       IOMMU_HWPT_ALLOC_PASID);
+	if (IS_ERR(xdna->domain)) {
+		XDNA_ERR(xdna, "Failed to alloc iommu domain");
+		ret = PTR_ERR(xdna->domain);
+		goto put_group;
+	}
+
+	ret = iova_cache_get();
+	if (ret)
+		goto free_domain;
+
+	order = __ffs(xdna->domain->pgsize_bitmap);
+	init_iova_domain(&xdna->iovad, 1UL << order, 0);
+
+	ret = iommu_attach_group(xdna->domain, xdna->group);
+	if (ret)
+		goto put_iova;
+
+	return 0;
+
+put_iova:
+	put_iova_domain(&xdna->iovad);
+	iova_cache_put();
+free_domain:
+	iommu_domain_free(xdna->domain);
+put_group:
+	iommu_group_put(xdna->group);
+	xdna->domain = NULL;
+
+	return ret;
+}
+
+void amdxdna_iommu_fini(struct amdxdna_dev *xdna)
+{
+	if (xdna->domain) {
+		iommu_detach_group(xdna->domain, xdna->group);
+		put_iova_domain(&xdna->iovad);
+		iova_cache_put();
+		iommu_domain_free(xdna->domain);
+	}
+
+	if (xdna->group)
+		iommu_group_put(xdna->group);
+}
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c
index a4384593bdccb..5143b8c9b92b3 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.c
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c
@@ -35,9 +35,10 @@ MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin");
  * 0.4: Support getting resource information
  * 0.5: Support getting telemetry data
  * 0.6: Support preemption
+ * 0.7: Support getting power and utilization data
  */
 #define AMDXDNA_DRIVER_MAJOR		0
-#define AMDXDNA_DRIVER_MINOR		6
+#define AMDXDNA_DRIVER_MINOR		7
 
 /*
  * Bind the driver base on (vendor_id, device_id) pair and later use the
@@ -72,18 +73,21 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 
 	client->pid = pid_nr(rcu_access_pointer(filp->pid));
 	client->xdna = xdna;
-
-	client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm);
-	if (IS_ERR(client->sva)) {
-		ret = PTR_ERR(client->sva);
-		XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret);
-		goto failed;
-	}
-	client->pasid = iommu_sva_get_pasid(client->sva);
-	if (client->pasid == IOMMU_PASID_INVALID) {
-		XDNA_ERR(xdna, "SVA get pasid failed");
-		ret = -ENODEV;
-		goto unbind_sva;
+	client->pasid = IOMMU_PASID_INVALID;
+
+	if (!amdxdna_iova_on(xdna)) {
+		client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm);
+		if (IS_ERR(client->sva)) {
+			ret = PTR_ERR(client->sva);
+			XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret);
+			goto failed;
+		}
+		client->pasid = iommu_sva_get_pasid(client->sva);
+		if (client->pasid == IOMMU_PASID_INVALID) {
+			XDNA_ERR(xdna, "SVA get pasid failed");
+			ret = -ENODEV;
+			goto unbind_sva;
+		}
 	}
 	client->mm = current->mm;
 	mmgrab(client->mm);
@@ -102,7 +106,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 	return 0;
 
 unbind_sva:
-	iommu_sva_unbind_device(client->sva);
+	if (!IS_ERR_OR_NULL(client->sva))
+		iommu_sva_unbind_device(client->sva);
 failed:
 	kfree(client);
 
@@ -120,7 +125,8 @@ static void amdxdna_client_cleanup(struct amdxdna_client *client)
 	if (client->dev_heap)
 		drm_gem_object_put(to_gobj(client->dev_heap));
 
-	iommu_sva_unbind_device(client->sva);
+	if (!IS_ERR_OR_NULL(client->sva))
+		iommu_sva_unbind_device(client->sva);
 	mmdrop(client->mm);
 
 	kfree(client);
@@ -281,9 +287,15 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		fs_reclaim_release(GFP_KERNEL);
 	}
 
+	ret = amdxdna_iommu_init(xdna);
+	if (ret)
+		return ret;
+
 	xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM);
-	if (!xdna->notifier_wq)
-		return -ENOMEM;
+	if (!xdna->notifier_wq) {
+		ret = -ENOMEM;
+		goto iommu_fini;
+	}
 
 	mutex_lock(&xdna->dev_lock);
 	ret = xdna->dev_info->ops->init(xdna);
@@ -315,6 +327,8 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	mutex_unlock(&xdna->dev_lock);
 destroy_notifier_wq:
 	destroy_workqueue(xdna->notifier_wq);
+iommu_fini:
+	amdxdna_iommu_fini(xdna);
 	return ret;
 }
 
@@ -340,6 +354,8 @@ static void amdxdna_remove(struct pci_dev *pdev)
 
 	xdna->dev_info->ops->fini(xdna);
 	mutex_unlock(&xdna->dev_lock);
+
+	amdxdna_iommu_fini(xdna);
 }
 
 static const struct dev_pm_ops amdxdna_pm_ops = {
@@ -358,5 +374,6 @@ static struct pci_driver amdxdna_pci_driver = {
 module_pci_driver(amdxdna_pci_driver);
 
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("AMD_PMF");
 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
 MODULE_DESCRIPTION("amdxdna driver");
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h
index f08406b8fdf93..e91d14ae5190e 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.h
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h
@@ -6,7 +6,10 @@
 #ifndef _AMDXDNA_PCI_DRV_H_
 #define _AMDXDNA_PCI_DRV_H_
 
+#include <drm/amdxdna_accel.h>
 #include <drm/drm_print.h>
+#include <linux/iommu.h>
+#include <linux/iova.h>
 #include <linux/workqueue.h>
 #include <linux/xarray.h>
 
@@ -101,6 +104,10 @@ struct amdxdna_dev {
 	struct amdxdna_fw_ver		fw_ver;
 	struct rw_semaphore		notifier_lock; /* for mmu notifier*/
 	struct workqueue_struct		*notifier_wq;
+
+	struct iommu_group		*group;
+	struct iommu_domain		*domain;
+	struct iova_domain		iovad;
 };
 
 /*
@@ -145,4 +152,21 @@ extern const struct amdxdna_dev_info dev_npu6_info;
 int amdxdna_sysfs_init(struct amdxdna_dev *xdna);
 void amdxdna_sysfs_fini(struct amdxdna_dev *xdna);
 
+int amdxdna_iommu_init(struct amdxdna_dev *xdna);
+void amdxdna_iommu_fini(struct amdxdna_dev *xdna);
+int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
+void amdxdna_iommu_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
+void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr);
+void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
+			void *cpu_addr, dma_addr_t dma_addr);
+
+static inline bool amdxdna_iova_on(struct amdxdna_dev *xdna)
+{
+	return !!xdna->domain;
+}
+
+static inline bool amdxdna_pasid_on(struct amdxdna_client *client)
+{
+	return client->pasid != IOMMU_PASID_INVALID;
+}
 #endif /* _AMDXDNA_PCI_DRV_H_ */
diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c
index ce25eef5fc34f..619bff042e52c 100644
--- a/drivers/accel/amdxdna/npu4_regs.c
+++ b/drivers/accel/amdxdna/npu4_regs.c
@@ -93,7 +93,8 @@ const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = {
 	{ .features = BIT_U64(AIE2_NPU_COMMAND), .major = 6, .min_minor = 15 },
 	{ .features = BIT_U64(AIE2_PREEMPT), .major = 6, .min_minor = 12 },
 	{ .features = BIT_U64(AIE2_TEMPORAL_ONLY), .major = 6, .min_minor = 12 },
-	{ .features = GENMASK_ULL(AIE2_TEMPORAL_ONLY, AIE2_NPU_COMMAND), .major = 7 },
+	{ .features = BIT_U64(AIE2_APP_HEALTH), .major = 6, .min_minor = 18 },
+	{ .features = AIE2_ALL_FEATURES, .major = 7 },
 	{ 0 }
 };
 
diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c
index a09f54fc43020..189dbe94cf141 100644
--- a/drivers/accel/ivpu/ivpu_debugfs.c
+++ b/drivers/accel/ivpu/ivpu_debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2024 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/debugfs.h>
@@ -127,6 +127,14 @@ static int firewall_irq_counter_show(struct seq_file *s, void *v)
 	return 0;
 }
 
+static int engine_reset_counter_show(struct seq_file *s, void *v)
+{
+	struct ivpu_device *vdev = seq_to_ivpu(s);
+
+	seq_printf(s, "%d\n", atomic_read(&vdev->pm->engine_reset_counter));
+	return 0;
+}
+
 static const struct drm_debugfs_info vdev_debugfs_list[] = {
 	{"bo_list", bo_list_show, 0},
 	{"fw_name", fw_name_show, 0},
@@ -137,6 +145,7 @@ static const struct drm_debugfs_info vdev_debugfs_list[] = {
 	{"reset_counter", reset_counter_show, 0},
 	{"reset_pending", reset_pending_show, 0},
 	{"firewall_irq_counter", firewall_irq_counter_show, 0},
+	{"engine_reset_counter", engine_reset_counter_show, 0},
 };
 
 static int dvfs_mode_get(void *data, u64 *dvfs_mode)
@@ -352,8 +361,9 @@ static const struct file_operations ivpu_force_recovery_fops = {
 static int ivpu_reset_engine_fn(void *data, u64 val)
 {
 	struct ivpu_device *vdev = (struct ivpu_device *)data;
+	struct vpu_jsm_msg resp;
 
-	return ivpu_jsm_reset_engine(vdev, (u32)val);
+	return ivpu_jsm_reset_engine(vdev, (u32)val, &resp);
 }
 
 DEFINE_DEBUGFS_ATTRIBUTE(ivpu_reset_engine_fops, NULL, ivpu_reset_engine_fn, "0x%02llx\n");
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 5900a40c7a786..2801378e3e192 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -67,6 +67,73 @@ bool ivpu_force_snoop;
 module_param_named(force_snoop, ivpu_force_snoop, bool, 0444);
 MODULE_PARM_DESC(force_snoop, "Force snooping for NPU host memory access");
 
+static struct ivpu_user_limits *ivpu_user_limits_alloc(struct ivpu_device *vdev, uid_t uid)
+{
+	struct ivpu_user_limits *limits;
+
+	limits = kzalloc_obj(*limits);
+	if (!limits)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&limits->ref);
+	atomic_set(&limits->db_count, 0);
+	limits->vdev = vdev;
+	limits->uid = uid;
+
+	/* Allow root user to allocate all contexts */
+	if (uid == 0) {
+		limits->max_ctx_count = ivpu_get_context_count(vdev);
+		limits->max_db_count = ivpu_get_doorbell_count(vdev);
+	} else {
+		limits->max_ctx_count = ivpu_get_context_count(vdev) / 2;
+		limits->max_db_count = ivpu_get_doorbell_count(vdev) / 2;
+	}
+
+	hash_add(vdev->user_limits, &limits->hash_node, uid);
+
+	return limits;
+}
+
+static struct ivpu_user_limits *ivpu_user_limits_get(struct ivpu_device *vdev)
+{
+	struct ivpu_user_limits *limits;
+	uid_t uid = current_uid().val;
+
+	guard(mutex)(&vdev->user_limits_lock);
+
+	hash_for_each_possible(vdev->user_limits, limits, hash_node, uid) {
+		if (limits->uid == uid) {
+			if (kref_read(&limits->ref) >= limits->max_ctx_count) {
+				ivpu_dbg(vdev, IOCTL, "User %u exceeded max ctx count %u\n", uid,
+					 limits->max_ctx_count);
+				return ERR_PTR(-EMFILE);
+			}
+
+			kref_get(&limits->ref);
+			return limits;
+		}
+	}
+
+	return ivpu_user_limits_alloc(vdev, uid);
+}
+
+static void ivpu_user_limits_release(struct kref *ref)
+{
+	struct ivpu_user_limits *limits = container_of(ref, struct ivpu_user_limits, ref);
+	struct ivpu_device *vdev = limits->vdev;
+
+	lockdep_assert_held(&vdev->user_limits_lock);
+	drm_WARN_ON(&vdev->drm, atomic_read(&limits->db_count));
+	hash_del(&limits->hash_node);
+	kfree(limits);
+}
+
+static void ivpu_user_limits_put(struct ivpu_device *vdev, struct ivpu_user_limits *limits)
+{
+	guard(mutex)(&vdev->user_limits_lock);
+	kref_put(&limits->ref, ivpu_user_limits_release);
+}
+
 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv)
 {
 	struct ivpu_device *vdev = file_priv->vdev;
@@ -110,6 +177,7 @@ static void file_priv_release(struct kref *ref)
 	mutex_unlock(&vdev->context_list_lock);
 	pm_runtime_put_autosuspend(vdev->drm.dev);
 
+	ivpu_user_limits_put(vdev, file_priv->user_limits);
 	mutex_destroy(&file_priv->ms_lock);
 	mutex_destroy(&file_priv->lock);
 	kfree(file_priv);
@@ -169,7 +237,7 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
 		args->value = ivpu_hw_dpu_max_freq_get(vdev);
 		break;
 	case DRM_IVPU_PARAM_NUM_CONTEXTS:
-		args->value = ivpu_get_context_count(vdev);
+		args->value = file_priv->user_limits->max_ctx_count;
 		break;
 	case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS:
 		args->value = vdev->hw->ranges.user.start;
@@ -231,22 +299,30 @@ static int ivpu_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct ivpu_device *vdev = to_ivpu_device(dev);
 	struct ivpu_file_priv *file_priv;
+	struct ivpu_user_limits *limits;
 	u32 ctx_id;
 	int idx, ret;
 
 	if (!drm_dev_enter(dev, &idx))
 		return -ENODEV;
 
+	limits = ivpu_user_limits_get(vdev);
+	if (IS_ERR(limits)) {
+		ret = PTR_ERR(limits);
+		goto err_dev_exit;
+	}
+
 	file_priv = kzalloc_obj(*file_priv);
 	if (!file_priv) {
 		ret = -ENOMEM;
-		goto err_dev_exit;
+		goto err_user_limits_put;
 	}
 
 	INIT_LIST_HEAD(&file_priv->ms_instance_list);
 
 	file_priv->vdev = vdev;
 	file_priv->bound = true;
+	file_priv->user_limits = limits;
 	kref_init(&file_priv->ref);
 	mutex_init(&file_priv->lock);
 	mutex_init(&file_priv->ms_lock);
@@ -284,6 +360,8 @@ static int ivpu_open(struct drm_device *dev, struct drm_file *file)
 	mutex_destroy(&file_priv->ms_lock);
 	mutex_destroy(&file_priv->lock);
 	kfree(file_priv);
+err_user_limits_put:
+	ivpu_user_limits_put(vdev, limits);
 err_dev_exit:
 	drm_dev_exit(idx);
 	return ret;
@@ -343,8 +421,7 @@ static int ivpu_wait_for_ready(struct ivpu_device *vdev)
 	ivpu_ipc_consumer_del(vdev, &cons);
 
 	if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) {
-		ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n",
-			 ipc_hdr.data_addr);
+		ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n", ipc_hdr.data_addr);
 		return -EIO;
 	}
 
@@ -453,7 +530,7 @@ int ivpu_shutdown(struct ivpu_device *vdev)
 }
 
 static const struct file_operations ivpu_fops = {
-	.owner		= THIS_MODULE,
+	.owner = THIS_MODULE,
 	DRM_ACCEL_FOPS,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo = drm_show_fdinfo,
@@ -588,10 +665,12 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 	vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;
 	atomic64_set(&vdev->unique_id_counter, 0);
 	atomic_set(&vdev->job_timeout_counter, 0);
+	atomic_set(&vdev->faults_detected, 0);
 	xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
 	xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1);
 	INIT_LIST_HEAD(&vdev->bo_list);
+	hash_init(vdev->user_limits);
 
 	vdev->db_limit.min = IVPU_MIN_DB;
 	vdev->db_limit.max = IVPU_MAX_DB;
@@ -600,6 +679,10 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 	if (ret)
 		goto err_xa_destroy;
 
+	ret = drmm_mutex_init(&vdev->drm, &vdev->user_limits_lock);
+	if (ret)
+		goto err_xa_destroy;
+
 	ret = drmm_mutex_init(&vdev->drm, &vdev->submitted_jobs_lock);
 	if (ret)
 		goto err_xa_destroy;
@@ -717,7 +800,7 @@ static struct pci_device_id ivpu_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PTL_P) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_WCL) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_NVL) },
-	{ }
+	{}
 };
 MODULE_DEVICE_TABLE(pci, ivpu_pci_ids);
 
diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index 5b34b6f50e696..b739738c45666 100644
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __IVPU_DRV_H__
@@ -12,6 +12,7 @@
 #include <drm/drm_mm.h>
 #include <drm/drm_print.h>
 
+#include <linux/hashtable.h>
 #include <linux/pci.h>
 #include <linux/xarray.h>
 #include <uapi/drm/ivpu_accel.h>
@@ -43,7 +44,7 @@
 /* SSID 1 is used by the VPU to represent reserved context */
 #define IVPU_RESERVED_CONTEXT_MMU_SSID 1
 #define IVPU_USER_CONTEXT_MIN_SSID     2
-#define IVPU_USER_CONTEXT_MAX_SSID     (IVPU_USER_CONTEXT_MIN_SSID + 63)
+#define IVPU_USER_CONTEXT_MAX_SSID     (IVPU_USER_CONTEXT_MIN_SSID + 128)
 
 #define IVPU_MIN_DB 1
 #define IVPU_MAX_DB 255
@@ -51,9 +52,6 @@
 #define IVPU_JOB_ID_JOB_MASK		GENMASK(7, 0)
 #define IVPU_JOB_ID_CONTEXT_MASK	GENMASK(31, 8)
 
-#define IVPU_NUM_PRIORITIES    4
-#define IVPU_NUM_CMDQS_PER_CTX (IVPU_NUM_PRIORITIES)
-
 #define IVPU_CMDQ_MIN_ID 1
 #define IVPU_CMDQ_MAX_ID 255
 
@@ -123,6 +121,16 @@ struct ivpu_fw_info;
 struct ivpu_ipc_info;
 struct ivpu_pm_info;
 
+struct ivpu_user_limits {
+	struct hlist_node hash_node;
+	struct ivpu_device *vdev;
+	struct kref ref;
+	u32 max_ctx_count;
+	u32 max_db_count;
+	u32 uid;
+	atomic_t db_count;
+};
+
 struct ivpu_device {
 	struct drm_device drm;
 	void __iomem *regb;
@@ -142,6 +150,8 @@ struct ivpu_device {
 	struct mutex context_list_lock; /* Protects user context addition/removal */
 	struct xarray context_xa;
 	struct xa_limit context_xa_limit;
+	DECLARE_HASHTABLE(user_limits, 8);
+	struct mutex user_limits_lock; /* Protects user_limits */
 
 	struct xarray db_xa;
 	struct xa_limit db_limit;
@@ -158,6 +168,7 @@ struct ivpu_device {
 	struct xarray submitted_jobs_xa;
 	struct ivpu_ipc_consumer job_done_consumer;
 	atomic_t job_timeout_counter;
+	atomic_t faults_detected;
 
 	atomic64_t unique_id_counter;
 
@@ -189,6 +200,7 @@ struct ivpu_file_priv {
 	struct list_head ms_instance_list;
 	struct ivpu_bo *ms_info_bo;
 	struct xa_limit job_limit;
+	struct ivpu_user_limits *user_limits;
 	u32 job_id_next;
 	struct xa_limit cmdq_limit;
 	u32 cmdq_id_next;
@@ -286,6 +298,13 @@ static inline u32 ivpu_get_context_count(struct ivpu_device *vdev)
 	return (ctx_limit.max - ctx_limit.min + 1);
 }
 
+static inline u32 ivpu_get_doorbell_count(struct ivpu_device *vdev)
+{
+	struct xa_limit db_limit = vdev->db_limit;
+
+	return (db_limit.max - db_limit.min + 1);
+}
+
 static inline u32 ivpu_get_platform(struct ivpu_device *vdev)
 {
 	WARN_ON_ONCE(vdev->platform == IVPU_PLATFORM_INVALID);
diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c
index 98b9ce26962b1..4f2005a8d4966 100644
--- a/drivers/accel/ivpu/ivpu_gem.c
+++ b/drivers/accel/ivpu/ivpu_gem.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/dma-buf.h>
@@ -31,7 +31,7 @@ static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, con
 		 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n",
 		 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr,
 		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
-		 (bool)drm_gem_is_imported(&bo->base.base));
+		 drm_gem_is_imported(&bo->base.base));
 }
 
 static inline int ivpu_bo_lock(struct ivpu_bo *bo)
@@ -48,7 +48,7 @@ static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct
 {
 	struct sg_table *sgt;
 
-	drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach);
+	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base));
 
 	ivpu_bo_lock(bo);
 
@@ -81,7 +81,7 @@ int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
 
 	ivpu_dbg_bo(vdev, bo, "bind");
 
-	if (bo->base.base.import_attach)
+	if (drm_gem_is_imported(&bo->base.base))
 		sgt = ivpu_bo_map_attachment(vdev, bo);
 	else
 		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
@@ -157,7 +157,7 @@ static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
 	}
 
 	if (bo->base.sgt) {
-		if (bo->base.base.import_attach) {
+		if (drm_gem_is_imported(&bo->base.base)) {
 			dma_buf_unmap_attachment(bo->base.base.import_attach,
 						 bo->base.sgt, DMA_BIDIRECTIONAL);
 		} else {
@@ -195,7 +195,7 @@ struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t siz
 	if (size == 0 || !PAGE_ALIGNED(size))
 		return ERR_PTR(-EINVAL);
 
-	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	bo = kzalloc_obj(*bo);
 	if (!bo)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index fe02b7bd465b3..521931d1f7fca 100644
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <drm/drm_file.h>
@@ -173,7 +173,7 @@ static struct ivpu_cmdq *ivpu_cmdq_create(struct ivpu_file_priv *file_priv, u8 p
 	ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit,
 			      &file_priv->cmdq_id_next, GFP_KERNEL);
 	if (ret < 0) {
-		ivpu_err(vdev, "Failed to allocate command queue ID: %d\n", ret);
+		ivpu_dbg(vdev, IOCTL, "Failed to allocate command queue ID: %d\n", ret);
 		goto err_free_cmdq;
 	}
 
@@ -215,14 +215,22 @@ static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq
 
 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 {
+	struct ivpu_user_limits *limits = file_priv->user_limits;
 	struct ivpu_device *vdev = file_priv->vdev;
 	int ret;
 
+	if (atomic_inc_return(&limits->db_count) > limits->max_db_count) {
+		ivpu_dbg(vdev, IOCTL, "Maximum number of %u doorbells for uid %u reached\n",
+			 limits->max_db_count, limits->uid);
+		ret = -EBUSY;
+		goto err_dec_db_count;
+	}
+
 	ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next,
 			      GFP_KERNEL);
 	if (ret < 0) {
-		ivpu_err(vdev, "Failed to allocate doorbell ID: %d\n", ret);
-		return ret;
+		ivpu_dbg(vdev, IOCTL, "Failed to allocate doorbell ID: %d\n", ret);
+		goto err_dec_db_count;
 	}
 
 	if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW)
@@ -231,15 +239,18 @@ static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *
 	else
 		ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
 					   cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
-
-	if (!ret) {
-		ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n",
-			 cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority);
-	} else {
+	if (ret) {
 		xa_erase(&vdev->db_xa, cmdq->db_id);
 		cmdq->db_id = 0;
+		goto err_dec_db_count;
 	}
 
+	ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n",
+		 cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority);
+	return 0;
+
+err_dec_db_count:
+	atomic_dec(&limits->db_count);
 	return ret;
 }
 
@@ -298,6 +309,7 @@ static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cm
 	}
 
 	xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
+	atomic_dec(&file_priv->user_limits->db_count);
 	cmdq->db_id = 0;
 
 	return 0;
@@ -313,6 +325,7 @@ static inline u8 ivpu_job_to_jsm_priority(u8 priority)
 
 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 {
+	lockdep_assert_held(&file_priv->lock);
 	ivpu_cmdq_unregister(file_priv, cmdq);
 	xa_erase(&file_priv->cmdq_xa, cmdq->id);
 	ivpu_cmdq_free(file_priv, cmdq);
@@ -380,8 +393,11 @@ static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)
 	mutex_lock(&file_priv->lock);
 
 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) {
-		xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
-		cmdq->db_id = 0;
+		if (cmdq->db_id) {
+			xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
+			atomic_dec(&file_priv->user_limits->db_count);
+			cmdq->db_id = 0;
+		}
 	}
 
 	mutex_unlock(&file_priv->lock);
@@ -591,6 +607,7 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_
 		 * status and ensure both are handled in the same way
 		 */
 		job->file_priv->has_mmu_faults = true;
+		atomic_set(&vdev->faults_detected, 1);
 		queue_work(system_percpu_wq, &vdev->context_abort_work);
 		return true;
 	}
@@ -1099,6 +1116,51 @@ void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 	ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 }
 
+static int reset_engine_and_mark_faulty_contexts(struct ivpu_device *vdev)
+{
+	u32 num_impacted_contexts;
+	struct vpu_jsm_msg resp;
+	int ret;
+	u32 i;
+
+	ret = ivpu_jsm_reset_engine(vdev, 0, &resp);
+	if (ret)
+		return ret;
+
+	/*
+	 * If faults are detected, ignore guilty contexts from engine reset as NPU may not be stuck
+	 * and could return currently running good context and faulty contexts are already marked
+	 */
+	if (atomic_cmpxchg(&vdev->faults_detected, 1, 0) == 1)
+		return 0;
+
+	num_impacted_contexts = resp.payload.engine_reset_done.num_impacted_contexts;
+
+	ivpu_warn_ratelimited(vdev, "Engine reset performed, impacted contexts: %u\n",
+			      num_impacted_contexts);
+
+	if (!in_range(num_impacted_contexts, 1, VPU_MAX_ENGINE_RESET_IMPACTED_CONTEXTS - 1)) {
+		ivpu_pm_trigger_recovery(vdev, "Cannot determine guilty contexts");
+		return -EIO;
+	}
+
+	/* No faults detected, NPU likely got stuck. Mark returned contexts as guilty */
+	guard(mutex)(&vdev->context_list_lock);
+
+	for (i = 0; i < num_impacted_contexts; i++) {
+		u32 ssid = resp.payload.engine_reset_done.impacted_contexts[i].host_ssid;
+		struct ivpu_file_priv *file_priv = xa_load(&vdev->context_xa, ssid);
+
+		if (file_priv) {
+			mutex_lock(&file_priv->lock);
+			file_priv->has_mmu_faults = true;
+			mutex_unlock(&file_priv->lock);
+		}
+	}
+
+	return 0;
+}
+
 void ivpu_context_abort_work_fn(struct work_struct *work)
 {
 	struct ivpu_device *vdev = container_of(work, struct ivpu_device, context_abort_work);
@@ -1111,7 +1173,7 @@ void ivpu_context_abort_work_fn(struct work_struct *work)
 		return;
 
 	if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW)
-		if (ivpu_jsm_reset_engine(vdev, 0))
+		if (reset_engine_and_mark_faulty_contexts(vdev))
 			goto runtime_put;
 
 	mutex_lock(&vdev->context_list_lock);
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c
index 0256b2dfefc10..07b1d6f615a91 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.c
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.c
@@ -151,10 +151,9 @@ int ivpu_jsm_get_heartbeat(struct ivpu_device *vdev, u32 engine, u64 *heartbeat)
 	return ret;
 }
 
-int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine)
+int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine, struct vpu_jsm_msg *resp)
 {
 	struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_ENGINE_RESET };
-	struct vpu_jsm_msg resp;
 	int ret;
 
 	if (engine != VPU_ENGINE_COMPUTE)
@@ -162,14 +161,17 @@ int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine)
 
 	req.payload.engine_reset.engine_idx = engine;
 
-	ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_ENGINE_RESET_DONE, &resp,
+	ret = ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_ENGINE_RESET_DONE, resp,
 				    VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
 	if (ret) {
 		ivpu_err_ratelimited(vdev, "Failed to reset engine %d: %d\n", engine, ret);
 		ivpu_pm_trigger_recovery(vdev, "Engine reset failed");
+		return ret;
 	}
 
-	return ret;
+	atomic_inc(&vdev->pm->engine_reset_counter);
+
+	return 0;
 }
 
 int ivpu_jsm_preempt_engine(struct ivpu_device *vdev, u32 engine, u32 preempt_id)
@@ -554,6 +556,15 @@ int ivpu_jsm_dct_disable(struct ivpu_device *vdev)
 }
 
 int ivpu_jsm_state_dump(struct ivpu_device *vdev)
+{
+	struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_STATE_DUMP };
+	struct vpu_jsm_msg resp;
+
+	return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_STATE_DUMP_RSP, &resp,
+					      VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
+}
+
+int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev)
 {
 	struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_STATE_DUMP };
 
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h
index 9e84d3526a146..a74f5a0b0d935 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.h
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.h
@@ -14,7 +14,7 @@ int ivpu_jsm_register_db(struct ivpu_device *vdev, u32 ctx_id, u32 db_id,
 			 u64 jobq_base, u32 jobq_size);
 int ivpu_jsm_unregister_db(struct ivpu_device *vdev, u32 db_id);
 int ivpu_jsm_get_heartbeat(struct ivpu_device *vdev, u32 engine, u64 *heartbeat);
-int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine);
+int ivpu_jsm_reset_engine(struct ivpu_device *vdev, u32 engine, struct vpu_jsm_msg *response);
 int ivpu_jsm_preempt_engine(struct ivpu_device *vdev, u32 engine, u32 preempt_id);
 int ivpu_jsm_dyndbg_control(struct ivpu_device *vdev, char *command, size_t size);
 int ivpu_jsm_trace_get_capability(struct ivpu_device *vdev, u32 *trace_destination_mask,
@@ -44,5 +44,6 @@ int ivpu_jsm_metric_streamer_info(struct ivpu_device *vdev, u64 metric_group_mas
 int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us);
 int ivpu_jsm_dct_disable(struct ivpu_device *vdev);
 int ivpu_jsm_state_dump(struct ivpu_device *vdev);
+int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev);
 
 #endif
diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c
index e1baf6b649351..41efd8985fa67 100644
--- a/drivers/accel/ivpu/ivpu_mmu.c
+++ b/drivers/accel/ivpu/ivpu_mmu.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2024 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/circ_buf.h>
@@ -964,6 +964,7 @@ void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
 		file_priv = xa_load(&vdev->context_xa, ssid);
 		if (file_priv) {
 			if (!READ_ONCE(file_priv->has_mmu_faults)) {
+				atomic_set(&vdev->faults_detected, 1);
 				ivpu_mmu_dump_event(vdev, event);
 				WRITE_ONCE(file_priv->has_mmu_faults, true);
 			}
diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index d20144a21e090..83da9b297f378 100644
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2024 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/highmem.h>
@@ -166,7 +166,7 @@ static void ivpu_pm_recovery_work(struct work_struct *work)
 	ivpu_pm_reset_begin(vdev);
 
 	if (!pm_runtime_status_suspended(vdev->drm.dev)) {
-		ivpu_jsm_state_dump(vdev);
+		ivpu_jsm_state_dump_no_reply(vdev);
 		ivpu_dev_coredump(vdev);
 		ivpu_suspend(vdev);
 	}
@@ -205,23 +205,25 @@ static void ivpu_job_timeout_work(struct work_struct *work)
 
 	if (ivpu_jsm_get_heartbeat(vdev, 0, &heartbeat) || heartbeat <= vdev->fw->last_heartbeat) {
 		ivpu_err(vdev, "Job timeout detected, heartbeat not progressed\n");
-		goto recovery;
+		goto abort;
 	}
 
 	inference_max_retries = DIV_ROUND_UP(inference_timeout_ms, timeout_ms);
 	if (atomic_fetch_inc(&vdev->job_timeout_counter) >= inference_max_retries) {
 		ivpu_err(vdev, "Job timeout detected, heartbeat limit (%lld) exceeded\n",
 			 inference_max_retries);
-		goto recovery;
+		goto abort;
 	}
 
 	vdev->fw->last_heartbeat = heartbeat;
 	ivpu_start_job_timeout_detection(vdev);
 	return;
 
-recovery:
+abort:
 	atomic_set(&vdev->job_timeout_counter, 0);
-	ivpu_pm_trigger_recovery(vdev, "TDR");
+	ivpu_jsm_state_dump(vdev);
+	ivpu_dev_coredump(vdev);
+	queue_work(system_percpu_wq, &vdev->context_abort_work);
 }
 
 void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)
@@ -404,6 +406,7 @@ void ivpu_pm_init(struct ivpu_device *vdev)
 	init_rwsem(&pm->reset_lock);
 	atomic_set(&pm->reset_pending, 0);
 	atomic_set(&pm->reset_counter, 0);
+	atomic_set(&pm->engine_reset_counter, 0);
 
 	INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);
 	INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work);
diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h
index 00f2a01e3df66..2f07bb0b43be3 100644
--- a/drivers/accel/ivpu/ivpu_pm.h
+++ b/drivers/accel/ivpu/ivpu_pm.h
@@ -18,6 +18,7 @@ struct ivpu_pm_info {
 	struct rw_semaphore reset_lock;
 	atomic_t reset_counter;
 	atomic_t reset_pending;
+	atomic_t engine_reset_counter;
 	u8 dct_active_percent;
 };
 
diff --git a/drivers/accel/ivpu/vpu_boot_api.h b/drivers/accel/ivpu/vpu_boot_api.h
index 218468bbbcada..a41170bbc6b7b 100644
--- a/drivers/accel/ivpu/vpu_boot_api.h
+++ b/drivers/accel/ivpu/vpu_boot_api.h
@@ -1,12 +1,22 @@
 /* SPDX-License-Identifier: MIT */
 /*
- * Copyright (c) 2020-2024, Intel Corporation.
+ * Copyright (c) 2020-2025, Intel Corporation.
+ */
+
+/**
+ * @addtogroup Boot
+ * @{
+ */
+
+/**
+ * @file
+ * @brief Boot API public header file.
  */
 
 #ifndef VPU_BOOT_API_H
 #define VPU_BOOT_API_H
 
-/*
+/**
  *  The below values will be used to construct the version info this way:
  *  fw_bin_header->api_version[VPU_BOOT_API_VER_ID] = (VPU_BOOT_API_VER_MAJOR << 16) |
  *  VPU_BOOT_API_VER_MINOR;
@@ -16,24 +26,24 @@
  *  partial info a build error will be generated.
  */
 
-/*
+/**
  * Major version changes that break backward compatibility.
  * Major version must start from 1 and can only be incremented.
  */
 #define VPU_BOOT_API_VER_MAJOR 3
 
-/*
+/**
  * Minor version changes when API backward compatibility is preserved.
  * Resets to 0 if Major version is incremented.
  */
-#define VPU_BOOT_API_VER_MINOR 28
+#define VPU_BOOT_API_VER_MINOR 29
 
-/*
+/**
  * API header changed (field names, documentation, formatting) but API itself has not been changed
  */
-#define VPU_BOOT_API_VER_PATCH 3
+#define VPU_BOOT_API_VER_PATCH 4
 
-/*
+/**
  * Index in the API version table
  * Must be unique for each API
  */
@@ -41,7 +51,7 @@
 
 #pragma pack(push, 4)
 
-/*
+/**
  * Firmware image header format
  */
 #define VPU_FW_HEADER_SIZE    4096
@@ -61,44 +71,41 @@ struct vpu_firmware_header {
 	u32 firmware_version_size;
 	u64 boot_params_load_address;
 	u32 api_version[VPU_FW_API_VER_NUM];
-	/* Size of memory require for firmware execution */
+	/** Size of memory require for firmware execution */
 	u32 runtime_size;
 	u32 shave_nn_fw_size;
-	/*
+	/**
 	 * Size of primary preemption buffer, assuming a 2-job submission queue.
 	 * NOTE: host driver is expected to adapt size accordingly to actual
 	 * submission queue size and device capabilities.
 	 */
 	u32 preemption_buffer_1_size;
-	/*
+	/**
 	 * Size of secondary preemption buffer, assuming a 2-job submission queue.
 	 * NOTE: host driver is expected to adapt size accordingly to actual
 	 * submission queue size and device capabilities.
 	 */
 	u32 preemption_buffer_2_size;
-	/*
+	/**
 	 * Maximum preemption buffer size that the FW can use: no need for the host
 	 * driver to allocate more space than that specified by these fields.
 	 * A value of 0 means no declared limit.
 	 */
 	u32 preemption_buffer_1_max_size;
 	u32 preemption_buffer_2_max_size;
-	/* Space reserved for future preemption-related fields. */
+	/** Space reserved for future preemption-related fields. */
 	u32 preemption_reserved[4];
-	/* FW image read only section start address, 4KB aligned */
+	/** FW image read only section start address, 4KB aligned */
 	u64 ro_section_start_address;
-	/* FW image read only section size, 4KB aligned */
+	/** FW image read only section size, 4KB aligned */
 	u32 ro_section_size;
 	u32 reserved;
 };
 
-/*
+/**
  * Firmware boot parameters format
  */
 
-#define VPU_BOOT_PLL_COUNT     3
-#define VPU_BOOT_PLL_OUT_COUNT 4
-
 /** Values for boot_type field */
 #define VPU_BOOT_TYPE_COLDBOOT 0
 #define VPU_BOOT_TYPE_WARMBOOT 1
@@ -166,7 +173,7 @@ enum vpu_trace_destination {
 #define VPU_TRACE_PROC_BIT_ACT_SHV_3 22
 #define VPU_TRACE_PROC_NO_OF_HW_DEVS 23
 
-/* VPU 30xx HW component IDs are sequential, so define first and last IDs. */
+/** VPU 30xx HW component IDs are sequential, so define first and last IDs. */
 #define VPU_TRACE_PROC_BIT_30XX_FIRST VPU_TRACE_PROC_BIT_LRT
 #define VPU_TRACE_PROC_BIT_30XX_LAST  VPU_TRACE_PROC_BIT_SHV_15
 
@@ -175,15 +182,7 @@ struct vpu_boot_l2_cache_config {
 	u8 cfg;
 };
 
-struct vpu_warm_boot_section {
-	u32 src;
-	u32 dst;
-	u32 size;
-	u32 core_id;
-	u32 is_clear_op;
-};
-
-/*
+/**
  * When HW scheduling mode is enabled, a present period is defined.
  * It will be used by VPU to swap between normal and focus priorities
  * to prevent starving of normal priority band (when implemented).
@@ -206,24 +205,24 @@ struct vpu_warm_boot_section {
  * Enum for dvfs_mode boot param.
  */
 enum vpu_governor {
-	VPU_GOV_DEFAULT = 0, /* Default Governor for the system */
-	VPU_GOV_MAX_PERFORMANCE = 1, /* Maximum performance governor */
-	VPU_GOV_ON_DEMAND = 2, /* On Demand frequency control governor */
-	VPU_GOV_POWER_SAVE = 3, /* Power save governor */
-	VPU_GOV_ON_DEMAND_PRIORITY_AWARE = 4 /* On Demand priority based governor */
+	VPU_GOV_DEFAULT = 0, /** Default Governor for the system */
+	VPU_GOV_MAX_PERFORMANCE = 1, /** Maximum performance governor */
+	VPU_GOV_ON_DEMAND = 2, /** On Demand frequency control governor */
+	VPU_GOV_POWER_SAVE = 3, /** Power save governor */
+	VPU_GOV_ON_DEMAND_PRIORITY_AWARE = 4 /** On Demand priority based governor */
 };
 
 struct vpu_boot_params {
 	u32 magic;
 	u32 vpu_id;
 	u32 vpu_count;
-	u32 pad0[5];
-	/* Clock frequencies: 0x20 - 0xFF */
+	u32 reserved_0[5];
+	/** Clock frequencies: 0x20 - 0xFF */
 	u32 frequency;
-	u32 pll[VPU_BOOT_PLL_COUNT][VPU_BOOT_PLL_OUT_COUNT];
+	u32 reserved_1[12];
 	u32 perf_clk_frequency;
-	u32 pad1[42];
-	/* Memory regions: 0x100 - 0x1FF */
+	u32 reserved_2[42];
+	/** Memory regions: 0x100 - 0x1FF */
 	u64 ipc_header_area_start;
 	u32 ipc_header_area_size;
 	u64 shared_region_base;
@@ -234,41 +233,24 @@ struct vpu_boot_params {
 	u32 global_aliased_pio_size;
 	u32 autoconfig;
 	struct vpu_boot_l2_cache_config cache_defaults[VPU_BOOT_L2_CACHE_CFG_NUM];
-	u64 global_memory_allocator_base;
-	u32 global_memory_allocator_size;
+	u32 reserved_3[3];
 	/**
 	 * ShaveNN FW section VPU base address
 	 * On VPU2.7 HW this address must be within 2GB range starting from L2C_PAGE_TABLE base
 	 */
 	u64 shave_nn_fw_base;
-	u64 save_restore_ret_address; /* stores the address of FW's restore entry point */
-	u32 pad2[43];
-	/* IRQ re-direct numbers: 0x200 - 0x2FF */
+	u64 save_restore_ret_address; /** stores the address of FW's restore entry point */
+	u32 reserved_4[43];
+	/** IRQ re-direct numbers: 0x200 - 0x2FF */
 	s32 watchdog_irq_mss;
 	s32 watchdog_irq_nce;
-	/* ARM -> VPU doorbell interrupt. ARM is notifying VPU of async command or compute job. */
+	/** ARM -> VPU doorbell interrupt. ARM is notifying VPU of async command or compute job. */
 	u32 host_to_vpu_irq;
-	/* VPU -> ARM job done interrupt. VPU is notifying ARM of compute job completion. */
+	/** VPU -> ARM job done interrupt. VPU is notifying ARM of compute job completion. */
 	u32 job_done_irq;
-	/* VPU -> ARM IRQ line to use to request MMU update. */
-	u32 mmu_update_request_irq;
-	/* ARM -> VPU IRQ line to use to notify of MMU update completion. */
-	u32 mmu_update_done_irq;
-	/* ARM -> VPU IRQ line to use to request power level change. */
-	u32 set_power_level_irq;
-	/* VPU -> ARM IRQ line to use to notify of power level change completion. */
-	u32 set_power_level_done_irq;
-	/* VPU -> ARM IRQ line to use to notify of VPU idle state change */
-	u32 set_vpu_idle_update_irq;
-	/* VPU -> ARM IRQ line to use to request counter reset. */
-	u32 metric_query_event_irq;
-	/* ARM -> VPU IRQ line to use to notify of counter reset completion. */
-	u32 metric_query_event_done_irq;
-	/* VPU -> ARM IRQ line to use to notify of preemption completion. */
-	u32 preemption_done_irq;
-	/* Padding. */
-	u32 pad3[52];
-	/* Silicon information: 0x300 - 0x3FF */
+	/** Padding. */
+	u32 reserved_5[60];
+	/** Silicon information: 0x300 - 0x3FF */
 	u32 host_version_id;
 	u32 si_stepping;
 	u64 device_id;
@@ -294,7 +276,7 @@ struct vpu_boot_params {
 	u32 crit_tracing_buff_size;
 	u64 verbose_tracing_buff_addr;
 	u32 verbose_tracing_buff_size;
-	u64 verbose_tracing_sw_component_mask; /* TO BE REMOVED */
+	u64 verbose_tracing_sw_component_mask; /** TO BE REMOVED */
 	/**
 	 * Mask of destinations to which logging messages are delivered; bitwise OR
 	 * of values defined in vpu_trace_destination enum.
@@ -308,11 +290,7 @@ struct vpu_boot_params {
 	/** Mask of trace message formats supported by the driver */
 	u64 tracing_buff_message_format_mask;
 	u64 trace_reserved_1[2];
-	/**
-	 * Period at which the VPU reads the temp sensor values into MMIO, on
-	 * platforms where that is necessary (in ms). 0 to disable reads.
-	 */
-	u32 temp_sensor_period_ms;
+	u32 reserved_6;
 	/** PLL ratio for efficient clock frequency */
 	u32 pn_freq_pll_ratio;
 	/**
@@ -347,11 +325,11 @@ struct vpu_boot_params {
 	 *       1: IPC message required to save state on D0i3 entry flow.
 	 */
 	u32 d0i3_delayed_entry;
-	/* Time spent by VPU in D0i3 state */
+	/** Time spent by VPU in D0i3 state */
 	u64 d0i3_residency_time_us;
-	/* Value of VPU perf counter at the time of entering D0i3 state . */
+	/** Value of VPU perf counter at the time of entering D0i3 state . */
 	u64 d0i3_entry_vpu_ts;
-	/*
+	/**
 	 * The system time of the host operating system in microseconds.
 	 * E.g the number of microseconds since 1st of January 1970, or whatever
 	 * date the host operating system uses to maintain system time.
@@ -359,57 +337,52 @@ struct vpu_boot_params {
 	 * The KMD is required to update this value on every VPU reset.
 	 */
 	u64 system_time_us;
-	u32 pad4[2];
-	/*
+	u32 reserved_7[2];
+	/**
 	 * The delta between device monotonic time and the current value of the
 	 * HW timestamp register, in ticks. Written by the firmware during boot.
 	 * Can be used by the KMD to calculate device time.
 	 */
 	u64 device_time_delta_ticks;
-	u32 pad7[14];
-	/* Warm boot information: 0x400 - 0x43F */
-	u32 warm_boot_sections_count;
-	u32 warm_boot_start_address_reference;
-	u32 warm_boot_section_info_address_offset;
-	u32 pad5[13];
-	/* Power States transitions timestamps: 0x440 - 0x46F*/
-	struct {
-		/* VPU_IDLE -> VPU_ACTIVE transition initiated timestamp */
+	u32 reserved_8[30];
+	/** Power States transitions timestamps: 0x440 - 0x46F*/
+	struct power_states_timestamps {
+		/** VPU_IDLE -> VPU_ACTIVE transition initiated timestamp */
 		u64 vpu_active_state_requested;
-		/* VPU_IDLE -> VPU_ACTIVE transition completed timestamp */
+		/** VPU_IDLE -> VPU_ACTIVE transition completed timestamp */
 		u64 vpu_active_state_achieved;
-		/* VPU_ACTIVE -> VPU_IDLE transition initiated timestamp */
+		/** VPU_ACTIVE -> VPU_IDLE transition initiated timestamp */
 		u64 vpu_idle_state_requested;
-		/* VPU_ACTIVE -> VPU_IDLE transition completed timestamp */
+		/** VPU_ACTIVE -> VPU_IDLE transition completed timestamp */
 		u64 vpu_idle_state_achieved;
-		/* VPU_IDLE -> VPU_STANDBY transition initiated timestamp */
+		/** VPU_IDLE -> VPU_STANDBY transition initiated timestamp */
 		u64 vpu_standby_state_requested;
-		/* VPU_IDLE -> VPU_STANDBY transition completed timestamp */
+		/** VPU_IDLE -> VPU_STANDBY transition completed timestamp */
 		u64 vpu_standby_state_achieved;
 	} power_states_timestamps;
-	/* VPU scheduling mode. Values defined by VPU_SCHEDULING_MODE_* macros. */
+	/** VPU scheduling mode. Values defined by VPU_SCHEDULING_MODE_* macros. */
 	u32 vpu_scheduling_mode;
-	/* Present call period in milliseconds. */
+	/** Present call period in milliseconds. */
 	u32 vpu_focus_present_timer_ms;
-	/* VPU ECC Signaling */
+	/** VPU ECC Signaling */
 	u32 vpu_uses_ecc_mca_signal;
-	/* Values defined by POWER_PROFILE* macros */
+	/** Values defined by POWER_PROFILE* macros */
 	u32 power_profile;
-	/* Microsecond value for DCT active cycle */
+	/** Microsecond value for DCT active cycle */
 	u32 dct_active_us;
-	/* Microsecond value for DCT inactive cycle */
+	/** Microsecond value for DCT inactive cycle */
 	u32 dct_inactive_us;
-	/* Unused/reserved: 0x488 - 0xFFF */
-	u32 pad6[734];
+	/** Unused/reserved: 0x488 - 0xFFF */
+	u32 reserved_9[734];
 };
 
-/* Magic numbers set between host and vpu to detect corruption of tracing init */
+/** Magic numbers set between host and vpu to detect corruption of tracing init */
 #define VPU_TRACING_BUFFER_CANARY (0xCAFECAFE)
 
-/* Tracing buffer message format definitions */
+/** Tracing buffer message format definitions */
 #define VPU_TRACING_FORMAT_STRING 0
 #define VPU_TRACING_FORMAT_MIPI	  2
-/*
+/**
  * Header of the tracing buffer.
  * The below defined header will be stored at the beginning of
  * each allocated tracing buffer, followed by a series of 256b
@@ -421,53 +394,55 @@ struct vpu_tracing_buffer_header {
 	 * @see VPU_TRACING_BUFFER_CANARY
 	 */
 	u32 host_canary_start;
-	/* offset from start of buffer for trace entries */
+	/** offset from start of buffer for trace entries */
 	u32 read_index;
-	/* keeps track of wrapping on the reader side */
+	/** keeps track of wrapping on the reader side */
 	u32 read_wrap_count;
 	u32 pad_to_cache_line_size_0[13];
-	/* End of first cache line */
+	/** End of first cache line */
 
 	/**
 	 * Magic number set by host to detect corruption
 	 * @see VPU_TRACING_BUFFER_CANARY
 	 */
 	u32 vpu_canary_start;
-	/* offset from start of buffer from write start */
+	/** offset from start of buffer from write start */
 	u32 write_index;
-	/* counter for buffer wrapping */
+	/** counter for buffer wrapping */
 	u32 wrap_count;
-	/* legacy field - do not use */
+	/** legacy field - do not use */
 	u32 reserved_0;
 	/**
-	 * Size of the log buffer include this header (@header_size) and space
-	 * reserved for all messages. If @alignment` is greater that 0 the @Size
-	 * must be multiple of @Alignment.
+	 * Size of the log buffer including this header (`header_size`) and space
+	 * reserved for all messages. If `alignment` is greater than 0, the `size`
+	 * must be a multiple of `alignment`.
 	 */
 	u32 size;
-	/* Header version */
+	/** Header version */
 	u16 header_version;
-	/* Header size */
+	/** Header size */
 	u16 header_size;
-	/*
+	/**
 	 * Format of the messages in the trace buffer
 	 * 0 - null terminated string
 	 * 1 - size + null terminated string
 	 * 2 - MIPI-SysT encoding
 	 */
 	u32 format;
-	/*
+	/**
 	 * Message alignment
 	 * 0 - messages are place 1 after another
 	 * n - every message starts and multiple on offset
 	 */
-	u32 alignment; /* 64, 128, 256 */
-	/* Name of the logging entity, i.e "LRT", "LNN", "SHV0", etc */
+	u32 alignment; /** 64, 128, 256 */
+	/** Name of the logging entity, i.e "LRT", "LNN", "SHV0", etc */
 	char name[16];
 	u32 pad_to_cache_line_size_1[4];
-	/* End of second cache line */
+	/** End of second cache line */
 };
 
 #pragma pack(pop)
 
 #endif
+
+///@}
diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c
index c5d77c3cb4bce..e9e970fd8f33e 100644
--- a/drivers/acpi/ac.c
+++ b/drivers/acpi/ac.c
@@ -21,8 +21,6 @@
 #include <linux/acpi.h>
 #include <acpi/battery.h>
 
-#define ACPI_AC_CLASS			"ac_adapter"
-#define ACPI_AC_DEVICE_NAME		"AC Adapter"
 #define ACPI_AC_FILE_STATE		"state"
 #define ACPI_AC_NOTIFY_STATUS		0x80
 #define ACPI_AC_STATUS_OFFLINE		0x00
@@ -33,22 +31,12 @@ MODULE_AUTHOR("Paul Diefenbaugh");
 MODULE_DESCRIPTION("ACPI AC Adapter Driver");
 MODULE_LICENSE("GPL");
 
-static int acpi_ac_probe(struct platform_device *pdev);
-static void acpi_ac_remove(struct platform_device *pdev);
-
-static void acpi_ac_notify(acpi_handle handle, u32 event, void *data);
-
 static const struct acpi_device_id ac_device_ids[] = {
 	{"ACPI0003", 0},
 	{"", 0},
 };
 MODULE_DEVICE_TABLE(acpi, ac_device_ids);
 
-#ifdef CONFIG_PM_SLEEP
-static int acpi_ac_resume(struct device *dev);
-#endif
-static SIMPLE_DEV_PM_OPS(acpi_ac_pm, NULL, acpi_ac_resume);
-
 static int ac_sleep_before_get_state_ms;
 static int ac_only;
 
@@ -141,10 +129,11 @@ static void acpi_ac_notify(acpi_handle handle, u32 event, void *data)
 			msleep(ac_sleep_before_get_state_ms);
 
 		acpi_ac_get_state(ac);
-		acpi_bus_generate_netlink_event(adev->pnp.device_class,
-						  dev_name(&adev->dev), event,
-						  (u32) ac->state);
-		acpi_notifier_call_chain(adev, event, (u32) ac->state);
+		acpi_bus_generate_netlink_event(ACPI_AC_CLASS,
+						dev_name(&adev->dev), event,
+						ac->state);
+		acpi_notifier_call_chain(ACPI_AC_CLASS, acpi_device_bid(adev),
+					 event, ac->state);
 		power_supply_changed(ac->charger);
 	}
 }
@@ -213,8 +202,6 @@ static int acpi_ac_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ac->device = adev;
-	strscpy(acpi_device_name(adev), ACPI_AC_DEVICE_NAME);
-	strscpy(acpi_device_class(adev), ACPI_AC_CLASS);
 
 	platform_set_drvdata(pdev, ac);
 
@@ -236,8 +223,8 @@ static int acpi_ac_probe(struct platform_device *pdev)
 		goto err_release_ac;
 	}
 
-	pr_info("%s [%s] (%s-line)\n", acpi_device_name(adev),
-		acpi_device_bid(adev), str_on_off(ac->state));
+	pr_info("AC Adapter [%s] (%s-line)\n", acpi_device_bid(adev),
+		str_on_off(ac->state));
 
 	ac->battery_nb.notifier_call = acpi_ac_battery_notify;
 	register_acpi_notifier(&ac->battery_nb);
@@ -272,10 +259,10 @@ static int acpi_ac_resume(struct device *dev)
 
 	return 0;
 }
-#else
-#define acpi_ac_resume NULL
 #endif
 
+static SIMPLE_DEV_PM_OPS(acpi_ac_pm, NULL, acpi_ac_resume);
+
 static void acpi_ac_remove(struct platform_device *pdev)
 {
 	struct acpi_ac *ac = platform_get_drvdata(pdev);
diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c
index 02a472fa85fc3..1d7dfe4ee9a61 100644
--- a/drivers/acpi/acpi_memhotplug.c
+++ b/drivers/acpi/acpi_memhotplug.c
@@ -18,9 +18,7 @@
 
 #include "internal.h"
 
-#define ACPI_MEMORY_DEVICE_CLASS		"memory"
 #define ACPI_MEMORY_DEVICE_HID			"PNP0C80"
-#define ACPI_MEMORY_DEVICE_NAME			"Hotplug Mem Device"
 
 static const struct acpi_device_id memory_device_ids[] = {
 	{ACPI_MEMORY_DEVICE_HID, 0},
@@ -297,8 +295,6 @@ static int acpi_memory_device_add(struct acpi_device *device,
 	INIT_LIST_HEAD(&mem_device->res_list);
 	mem_device->device = device;
 	mem_device->mgid = -1;
-	sprintf(acpi_device_name(device), "%s", ACPI_MEMORY_DEVICE_NAME);
-	sprintf(acpi_device_class(device), "%s", ACPI_MEMORY_DEVICE_CLASS);
 	device->driver_data = mem_device;
 
 	/* Get the range from the _CRS */
diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c
index c9a0bcaba2e4c..0a8e02bc8c8b6 100644
--- a/drivers/acpi/acpi_pad.c
+++ b/drivers/acpi/acpi_pad.c
@@ -23,8 +23,6 @@
 #include <asm/mwait.h>
 #include <xen/xen.h>
 
-#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
-#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 #define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 
 #define ACPI_PROCESSOR_AGGREGATOR_STATUS_SUCCESS	0
@@ -407,16 +405,15 @@ static void acpi_pad_handle_notify(acpi_handle handle)
 	mutex_unlock(&isolated_cpus_lock);
 }
 
-static void acpi_pad_notify(acpi_handle handle, u32 event,
-	void *data)
+static void acpi_pad_notify(acpi_handle handle, u32 event, void *data)
 {
 	struct acpi_device *adev = data;
 
 	switch (event) {
 	case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
 		acpi_pad_handle_notify(handle);
-		acpi_bus_generate_netlink_event(adev->pnp.device_class,
-			dev_name(&adev->dev), event, 0);
+		acpi_bus_generate_netlink_event("acpi_pad",
+						dev_name(&adev->dev), event, 0);
 		break;
 	default:
 		pr_warn("Unsupported event [0x%x]\n", event);
@@ -427,30 +424,19 @@ static void acpi_pad_notify(acpi_handle handle, u32 event,
 static int acpi_pad_probe(struct platform_device *pdev)
 {
 	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
-	acpi_status status;
 
-	strscpy(acpi_device_name(adev), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
-	strscpy(acpi_device_class(adev), ACPI_PROCESSOR_AGGREGATOR_CLASS);
-
-	status = acpi_install_notify_handler(adev->handle,
-		ACPI_DEVICE_NOTIFY, acpi_pad_notify, adev);
-
-	if (ACPI_FAILURE(status))
-		return -ENODEV;
-
-	return 0;
+	return acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
+					       acpi_pad_notify, adev);
 }
 
 static void acpi_pad_remove(struct platform_device *pdev)
 {
-	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
-
 	mutex_lock(&isolated_cpus_lock);
 	acpi_pad_idle_cpus(0);
 	mutex_unlock(&isolated_cpus_lock);
 
-	acpi_remove_notify_handler(adev->handle,
-		ACPI_DEVICE_NOTIFY, acpi_pad_notify);
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, acpi_pad_notify);
 }
 
 static const struct acpi_device_id pad_device_ids[] = {
diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c
index 85d9f78619a21..da886923b0082 100644
--- a/drivers/acpi/acpi_pnp.c
+++ b/drivers/acpi/acpi_pnp.c
@@ -125,10 +125,6 @@ static const struct acpi_device_id acpi_pnp_device_ids[] = {
 	{"PNP0401"},		/* ECP Printer Port */
 	/* apple-gmux */
 	{"APP000B"},
-	/* rtc_cmos */
-	{"PNP0b00"},
-	{"PNP0b01"},
-	{"PNP0b02"},
 	/* c6xdigio */
 	{"PNP0400"},		/* Standard LPT Printer Port */
 	{"PNP0401"},		/* ECP Printer Port */
@@ -355,25 +351,9 @@ static struct acpi_scan_handler acpi_pnp_handler = {
 	.attach = acpi_pnp_attach,
 };
 
-/*
- * For CMOS RTC devices, the PNP ACPI scan handler does not work, because
- * there is a CMOS RTC ACPI scan handler installed already, so we need to
- * check those devices and enumerate them to the PNP bus directly.
- */
-static int is_cmos_rtc_device(struct acpi_device *adev)
-{
-	static const struct acpi_device_id ids[] = {
-		{ "PNP0B00" },
-		{ "PNP0B01" },
-		{ "PNP0B02" },
-		{""},
-	};
-	return !acpi_match_device_ids(adev, ids);
-}
-
 bool acpi_is_pnp_device(struct acpi_device *adev)
 {
-	return adev->handler == &acpi_pnp_handler || is_cmos_rtc_device(adev);
+	return adev->handler == &acpi_pnp_handler;
 }
 EXPORT_SYMBOL_GPL(acpi_is_pnp_device);
 
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index b34a48068a8d1..7c3e5607b797f 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -113,6 +113,10 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev)
 				     PCI_ANY_ID, PCI_ANY_ID, NULL);
 		if (ide_dev) {
 			errata.piix4.bmisx = pci_resource_start(ide_dev, 4);
+			if (errata.piix4.bmisx)
+				dev_dbg(&ide_dev->dev,
+					"Bus master activity detection (BM-IDE) erratum enabled\n");
+
 			pci_dev_put(ide_dev);
 		}
 
@@ -131,20 +135,17 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev)
 		if (isa_dev) {
 			pci_read_config_byte(isa_dev, 0x76, &value1);
 			pci_read_config_byte(isa_dev, 0x77, &value2);
-			if ((value1 & 0x80) || (value2 & 0x80))
+			if ((value1 & 0x80) || (value2 & 0x80)) {
 				errata.piix4.fdma = 1;
+				dev_dbg(&isa_dev->dev,
+					"Type-F DMA livelock erratum (C3 disabled)\n");
+			}
 			pci_dev_put(isa_dev);
 		}
 
 		break;
 	}
 
-	if (ide_dev)
-		dev_dbg(&ide_dev->dev, "Bus master activity detection (BM-IDE) erratum enabled\n");
-
-	if (isa_dev)
-		dev_dbg(&isa_dev->dev, "Type-F DMA livelock erratum (C3 disabled)\n");
-
 	return 0;
 }
 
@@ -438,8 +439,6 @@ static int acpi_processor_add(struct acpi_device *device,
 	}
 
 	pr->handle = device->handle;
-	strscpy(acpi_device_name(device), ACPI_PROCESSOR_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_PROCESSOR_CLASS);
 	device->driver_data = pr;
 
 	result = acpi_processor_get_info(device);
diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c
index 6d870d97ada65..d67c43edf8558 100644
--- a/drivers/acpi/acpi_tad.c
+++ b/drivers/acpi/acpi_tad.c
@@ -2,12 +2,10 @@
 /*
  * ACPI Time and Alarm (TAD) Device Driver
  *
- * Copyright (C) 2018 Intel Corporation
+ * Copyright (C) 2018 - 2026 Intel Corporation
  * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
  *
- * This driver is based on Section 9.18 of the ACPI 6.2 specification revision.
- *
- * It only supports the system wakeup capabilities of the TAD.
+ * This driver is based on ACPI 6.6, Section 9.17.
  *
  * Provided are sysfs attributes, available under the TAD platform device,
  * allowing user space to manage the AC and DC wakeup timers of the TAD:
@@ -18,6 +16,11 @@
  *
  * The wakeup events handling and power management of the TAD is expected to
  * be taken care of by the ACPI PM domain attached to its platform device.
+ *
+ * If the TAD supports the get/set real time features, as indicated by the
+ * capability mask returned by _GCP under the TAD object, additional sysfs
+ * attributes are created allowing the real time to be set and read and an RTC
+ * class device is registered under the TAD platform device.
  */
 
 #include <linux/acpi.h>
@@ -25,13 +28,14 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/rtc.h>
 #include <linux/suspend.h>
 
 MODULE_DESCRIPTION("ACPI Time and Alarm (TAD) Device Driver");
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Rafael J. Wysocki");
 
-/* ACPI TAD capability flags (ACPI 6.2, Section 9.18.2) */
+/* ACPI TAD capability flags (ACPI 6.6, Section 9.17.2) */
 #define ACPI_TAD_AC_WAKE	BIT(0)
 #define ACPI_TAD_DC_WAKE	BIT(1)
 #define ACPI_TAD_RT		BIT(2)
@@ -49,6 +53,10 @@ MODULE_AUTHOR("Rafael J. Wysocki");
 /* Special value for disabled timer or expired timer wake policy. */
 #define ACPI_TAD_WAKE_DISABLED	(~(u32)0)
 
+/* ACPI TAD RTC */
+#define ACPI_TAD_TZ_UNSPEC	2047
+#define ACPI_TAD_TIME_ISDST	3
+
 struct acpi_tad_driver_data {
 	u32 capabilities;
 };
@@ -67,6 +75,16 @@ struct acpi_tad_rt {
 	u8 padding[3]; /* must be 0 */
 } __packed;
 
+static bool acpi_tad_rt_is_invalid(struct acpi_tad_rt *rt)
+{
+	return rt->year < 1900 || rt->year > 9999 ||
+	    rt->month < 1 || rt->month > 12 ||
+	    rt->hour > 23 || rt->minute > 59 || rt->second > 59 ||
+	    rt->tz < -1440 ||
+	    (rt->tz > 1440 && rt->tz != ACPI_TAD_TZ_UNSPEC) ||
+	    rt->daylight > 3;
+}
+
 static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt)
 {
 	acpi_handle handle = ACPI_HANDLE(dev);
@@ -80,12 +98,12 @@ static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt)
 	unsigned long long retval;
 	acpi_status status;
 
-	if (rt->year < 1900 || rt->year > 9999 ||
-	    rt->month < 1 || rt->month > 12 ||
-	    rt->hour > 23 || rt->minute > 59 || rt->second > 59 ||
-	    rt->tz < -1440 || (rt->tz > 1440 && rt->tz != 2047) ||
-	    rt->daylight > 3)
-		return -ERANGE;
+	if (acpi_tad_rt_is_invalid(rt))
+		return -EINVAL;
+
+	rt->valid = 0;
+	rt->msec = 0;
+	memset(rt->padding, 0, 3);
 
 	args[0].buffer.pointer = (u8 *)rt;
 	args[0].buffer.length = sizeof(*rt);
@@ -145,9 +163,14 @@ static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt)
 	if (ret)
 		return ret;
 
+	if (acpi_tad_rt_is_invalid(rt))
+		return -ENODATA;
+
 	return 0;
 }
 
+/* sysfs interface */
+
 static char *acpi_tad_rt_next_field(char *s, int *val)
 {
 	char *p;
@@ -167,69 +190,65 @@ static ssize_t time_store(struct device *dev, struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
 	struct acpi_tad_rt rt;
-	char *str, *s;
-	int val, ret = -ENODATA;
+	int val, ret;
+	char *s;
 
-	str = kmemdup_nul(buf, count, GFP_KERNEL);
+	char *str __free(kfree) = kmemdup_nul(buf, count, GFP_KERNEL);
 	if (!str)
 		return -ENOMEM;
 
 	s = acpi_tad_rt_next_field(str, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.year = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.month = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.day = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.hour = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.minute = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.second = val;
 
 	s = acpi_tad_rt_next_field(s, &val);
 	if (!s)
-		goto out_free;
+		return -ENODATA;
 
 	rt.tz = val;
 
 	if (kstrtoint(s, 10, &val))
-		goto out_free;
+		return -ENODATA;
 
 	rt.daylight = val;
 
-	rt.valid = 0;
-	rt.msec = 0;
-	memset(rt.padding, 0, 3);
-
 	ret = acpi_tad_set_real_time(dev, &rt);
+	if (ret)
+		return ret;
 
-out_free:
-	kfree(str);
-	return ret ? ret : count;
+	return count;
 }
 
 static ssize_t time_show(struct device *dev, struct device_attribute *attr,
@@ -249,14 +268,6 @@ static ssize_t time_show(struct device *dev, struct device_attribute *attr,
 
 static DEVICE_ATTR_RW(time);
 
-static struct attribute *acpi_tad_time_attrs[] = {
-	&dev_attr_time.attr,
-	NULL,
-};
-static const struct attribute_group acpi_tad_time_attr_group = {
-	.attrs	= acpi_tad_time_attrs,
-};
-
 static int acpi_tad_wake_set(struct device *dev, char *method, u32 timer_id,
 			     u32 value)
 {
@@ -486,17 +497,6 @@ static ssize_t ac_status_show(struct device *dev, struct device_attribute *attr,
 
 static DEVICE_ATTR_RW(ac_status);
 
-static struct attribute *acpi_tad_attrs[] = {
-	&dev_attr_caps.attr,
-	&dev_attr_ac_alarm.attr,
-	&dev_attr_ac_policy.attr,
-	&dev_attr_ac_status.attr,
-	NULL,
-};
-static const struct attribute_group acpi_tad_attr_group = {
-	.attrs	= acpi_tad_attrs,
-};
-
 static ssize_t dc_alarm_store(struct device *dev, struct device_attribute *attr,
 			      const char *buf, size_t count)
 {
@@ -545,16 +545,117 @@ static ssize_t dc_status_show(struct device *dev, struct device_attribute *attr,
 
 static DEVICE_ATTR_RW(dc_status);
 
-static struct attribute *acpi_tad_dc_attrs[] = {
+static struct attribute *acpi_tad_attrs[] = {
+	&dev_attr_caps.attr,
+	&dev_attr_ac_alarm.attr,
+	&dev_attr_ac_policy.attr,
+	&dev_attr_ac_status.attr,
 	&dev_attr_dc_alarm.attr,
 	&dev_attr_dc_policy.attr,
 	&dev_attr_dc_status.attr,
+	&dev_attr_time.attr,
 	NULL,
 };
-static const struct attribute_group acpi_tad_dc_attr_group = {
-	.attrs	= acpi_tad_dc_attrs,
+
+static umode_t acpi_tad_attr_is_visible(struct kobject *kobj,
+					struct attribute *a, int n)
+{
+	struct acpi_tad_driver_data *dd = dev_get_drvdata(kobj_to_dev(kobj));
+
+	if (a == &dev_attr_caps.attr)
+		return a->mode;
+
+	if ((dd->capabilities & ACPI_TAD_AC_WAKE) &&
+	    (a == &dev_attr_ac_alarm.attr || a == &dev_attr_ac_policy.attr ||
+	     a == &dev_attr_ac_status.attr))
+		return a->mode;
+
+	if ((dd->capabilities & ACPI_TAD_DC_WAKE) &&
+	    (a == &dev_attr_dc_alarm.attr || a == &dev_attr_dc_policy.attr ||
+	     a == &dev_attr_dc_status.attr))
+		return a->mode;
+
+	if ((dd->capabilities & ACPI_TAD_RT) && a == &dev_attr_time.attr)
+		return a->mode;
+
+	return 0;
+}
+
+static const struct attribute_group acpi_tad_attr_group = {
+	.attrs	= acpi_tad_attrs,
+	.is_visible = acpi_tad_attr_is_visible,
+};
+
+static const struct attribute_group *acpi_tad_attr_groups[] = {
+	&acpi_tad_attr_group,
+	NULL,
+};
+
+#ifdef CONFIG_RTC_CLASS
+/* RTC class device interface */
+
+static int acpi_tad_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct acpi_tad_rt rt;
+
+	rt.year = tm->tm_year + 1900;
+	rt.month = tm->tm_mon + 1;
+	rt.day = tm->tm_mday;
+	rt.hour = tm->tm_hour;
+	rt.minute = tm->tm_min;
+	rt.second = tm->tm_sec;
+	rt.tz = ACPI_TAD_TZ_UNSPEC;
+	rt.daylight = ACPI_TAD_TIME_ISDST * !!tm->tm_isdst;
+
+	return acpi_tad_set_real_time(dev, &rt);
+}
+
+static int acpi_tad_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct acpi_tad_rt rt;
+	int ret;
+
+	ret = acpi_tad_get_real_time(dev, &rt);
+	if (ret)
+		return ret;
+
+	tm->tm_year = rt.year - 1900;
+	tm->tm_mon = rt.month - 1;
+	tm->tm_mday = rt.day;
+	tm->tm_hour = rt.hour;
+	tm->tm_min = rt.minute;
+	tm->tm_sec = rt.second;
+	tm->tm_isdst = rt.daylight == ACPI_TAD_TIME_ISDST;
+
+	return 0;
+}
+
+static const struct rtc_class_ops acpi_tad_rtc_ops = {
+	.read_time = acpi_tad_rtc_read_time,
+	.set_time = acpi_tad_rtc_set_time,
 };
 
+static void acpi_tad_register_rtc(struct device *dev)
+{
+	struct rtc_device *rtc;
+
+	rtc = devm_rtc_allocate_device(dev);
+	if (IS_ERR(rtc))
+		return;
+
+	rtc->range_min = mktime64(1900,  1,  1,  0,  0,  0);
+	rtc->range_max = mktime64(9999, 12, 31, 23, 59, 59);
+
+	rtc->ops = &acpi_tad_rtc_ops;
+
+	devm_rtc_register_device(rtc);
+}
+#else /* !CONFIG_RTC_CLASS */
+static inline void acpi_tad_register_rtc(struct device *dev) {}
+#endif /* !CONFIG_RTC_CLASS */
+
+/* Platform driver interface */
+
 static int acpi_tad_disable_timer(struct device *dev, u32 timer_id)
 {
 	return acpi_tad_wake_set(dev, "_STV", timer_id, ACPI_TAD_WAKE_DISABLED);
@@ -563,22 +664,15 @@ static int acpi_tad_disable_timer(struct device *dev, u32 timer_id)
 static void acpi_tad_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	acpi_handle handle = ACPI_HANDLE(dev);
 	struct acpi_tad_driver_data *dd = dev_get_drvdata(dev);
 
 	device_init_wakeup(dev, false);
 
-	if (dd->capabilities & ACPI_TAD_RT)
-		sysfs_remove_group(&dev->kobj, &acpi_tad_time_attr_group);
-
-	if (dd->capabilities & ACPI_TAD_DC_WAKE)
-		sysfs_remove_group(&dev->kobj, &acpi_tad_dc_attr_group);
-
-	sysfs_remove_group(&dev->kobj, &acpi_tad_attr_group);
-
 	scoped_guard(pm_runtime_noresume, dev) {
-		acpi_tad_disable_timer(dev, ACPI_TAD_AC_TIMER);
-		acpi_tad_clear_status(dev, ACPI_TAD_AC_TIMER);
+		if (dd->capabilities & ACPI_TAD_AC_WAKE) {
+			acpi_tad_disable_timer(dev, ACPI_TAD_AC_TIMER);
+			acpi_tad_clear_status(dev, ACPI_TAD_AC_TIMER);
+		}
 		if (dd->capabilities & ACPI_TAD_DC_WAKE) {
 			acpi_tad_disable_timer(dev, ACPI_TAD_DC_TIMER);
 			acpi_tad_clear_status(dev, ACPI_TAD_DC_TIMER);
@@ -587,7 +681,6 @@ static void acpi_tad_remove(struct platform_device *pdev)
 
 	pm_runtime_suspend(dev);
 	pm_runtime_disable(dev);
-	acpi_remove_cmos_rtc_space_handler(handle);
 }
 
 static int acpi_tad_probe(struct platform_device *pdev)
@@ -597,13 +690,7 @@ static int acpi_tad_probe(struct platform_device *pdev)
 	struct acpi_tad_driver_data *dd;
 	acpi_status status;
 	unsigned long long caps;
-	int ret;
 
-	ret = acpi_install_cmos_rtc_space_handler(handle);
-	if (ret < 0) {
-		dev_info(dev, "Unable to install space handler\n");
-		return -ENODEV;
-	}
 	/*
 	 * Initialization failure messages are mostly about firmware issues, so
 	 * print them at the "info" level.
@@ -611,27 +698,17 @@ static int acpi_tad_probe(struct platform_device *pdev)
 	status = acpi_evaluate_integer(handle, "_GCP", NULL, &caps);
 	if (ACPI_FAILURE(status)) {
 		dev_info(dev, "Unable to get capabilities\n");
-		ret = -ENODEV;
-		goto remove_handler;
-	}
-
-	if (!(caps & ACPI_TAD_AC_WAKE)) {
-		dev_info(dev, "Unsupported capabilities\n");
-		ret = -ENODEV;
-		goto remove_handler;
+		return -ENODEV;
 	}
 
 	if (!acpi_has_method(handle, "_PRW")) {
 		dev_info(dev, "Missing _PRW\n");
-		ret = -ENODEV;
-		goto remove_handler;
+		caps &= ~(ACPI_TAD_AC_WAKE | ACPI_TAD_DC_WAKE);
 	}
 
 	dd = devm_kzalloc(dev, sizeof(*dd), GFP_KERNEL);
-	if (!dd) {
-		ret = -ENOMEM;
-		goto remove_handler;
-	}
+	if (!dd)
+		return -ENOMEM;
 
 	dd->capabilities = caps;
 	dev_set_drvdata(dev, dd);
@@ -642,9 +719,12 @@ static int acpi_tad_probe(struct platform_device *pdev)
 	 * runtime suspend.  Everything else should be taken care of by the ACPI
 	 * PM domain callbacks.
 	 */
-	device_init_wakeup(dev, true);
-	dev_pm_set_driver_flags(dev, DPM_FLAG_SMART_SUSPEND |
-				     DPM_FLAG_MAY_SKIP_RESUME);
+	if (ACPI_TAD_AC_WAKE | ACPI_TAD_DC_WAKE) {
+		device_init_wakeup(dev, true);
+		dev_pm_set_driver_flags(dev, DPM_FLAG_SMART_SUSPEND |
+					     DPM_FLAG_MAY_SKIP_RESUME);
+	}
+
 	/*
 	 * The platform bus type layer tells the ACPI PM domain powers up the
 	 * device, so set the runtime PM status of it to "active".
@@ -653,32 +733,10 @@ static int acpi_tad_probe(struct platform_device *pdev)
 	pm_runtime_enable(dev);
 	pm_runtime_suspend(dev);
 
-	ret = sysfs_create_group(&dev->kobj, &acpi_tad_attr_group);
-	if (ret)
-		goto fail;
-
-	if (caps & ACPI_TAD_DC_WAKE) {
-		ret = sysfs_create_group(&dev->kobj, &acpi_tad_dc_attr_group);
-		if (ret)
-			goto fail;
-	}
-
-	if (caps & ACPI_TAD_RT) {
-		ret = sysfs_create_group(&dev->kobj, &acpi_tad_time_attr_group);
-		if (ret)
-			goto fail;
-	}
+	if (caps & ACPI_TAD_RT)
+		acpi_tad_register_rtc(dev);
 
 	return 0;
-
-fail:
-	acpi_tad_remove(pdev);
-	/* Don't fallthrough because cmos rtc space handler is removed in acpi_tad_remove() */
-	return ret;
-
-remove_handler:
-	acpi_remove_cmos_rtc_space_handler(handle);
-	return ret;
 }
 
 static const struct acpi_device_id acpi_tad_ids[] = {
@@ -690,6 +748,7 @@ static struct platform_driver acpi_tad_driver = {
 	.driver = {
 		.name = "acpi-tad",
 		.acpi_match_table = acpi_tad_ids,
+		.dev_groups = acpi_tad_attr_groups,
 	},
 	.probe = acpi_tad_probe,
 	.remove = acpi_tad_remove,
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index adbaf0226c905..05793ddef7875 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -30,9 +30,6 @@
 #include <linux/uaccess.h>
 #include <linux/string_choices.h>
 
-#define ACPI_VIDEO_BUS_NAME		"Video Bus"
-#define ACPI_VIDEO_DEVICE_NAME		"Video Device"
-
 #define MAX_NAME_LEN	20
 
 MODULE_AUTHOR("Bruno Ducrot");
@@ -1144,9 +1141,6 @@ static int acpi_video_bus_get_one_device(struct acpi_device *device, void *arg)
 		return -ENOMEM;
 	}
 
-	strscpy(acpi_device_name(device), ACPI_VIDEO_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_VIDEO_CLASS);
-
 	data->device_id = device_id;
 	data->video = video;
 	data->dev = device;
@@ -1570,7 +1564,8 @@ static void acpi_video_bus_notify(acpi_handle handle, u32 event, void *data)
 		break;
 	}
 
-	if (acpi_notifier_call_chain(device, event, 0))
+	if (acpi_notifier_call_chain(ACPI_VIDEO_CLASS, acpi_device_bid(device),
+				     event, 0))
 		/* Something vetoed the keypress. */
 		keycode = 0;
 
@@ -1611,7 +1606,8 @@ static void acpi_video_device_notify(acpi_handle handle, u32 event, void *data)
 		if (video_device->backlight)
 			backlight_force_update(video_device->backlight,
 					       BACKLIGHT_UPDATE_HOTKEY);
-		acpi_notifier_call_chain(device, event, 0);
+		acpi_notifier_call_chain(ACPI_VIDEO_CLASS, acpi_device_bid(device),
+					 event, 0);
 		return;
 	}
 
@@ -1644,7 +1640,8 @@ static void acpi_video_device_notify(acpi_handle handle, u32 event, void *data)
 	if (keycode)
 		may_report_brightness_keys = true;
 
-	acpi_notifier_call_chain(device, event, 0);
+	acpi_notifier_call_chain(ACPI_VIDEO_CLASS, acpi_device_bid(device),
+				 event, 0);
 
 	if (keycode && (report_key_events & REPORT_BRIGHTNESS_KEY_EVENTS)) {
 		input_report_key(input, keycode, 1);
@@ -1681,26 +1678,6 @@ static int acpi_video_resume(struct notifier_block *nb,
 	return NOTIFY_DONE;
 }
 
-static acpi_status
-acpi_video_bus_match(acpi_handle handle, u32 level, void *context,
-			void **return_value)
-{
-	struct acpi_device *device = context;
-	struct acpi_device *sibling;
-
-	if (handle == device->handle)
-		return AE_CTRL_TERMINATE;
-
-	sibling = acpi_fetch_acpi_dev(handle);
-	if (!sibling)
-		return AE_OK;
-
-	if (!strcmp(acpi_device_name(sibling), ACPI_VIDEO_BUS_NAME))
-			return AE_ALREADY_EXISTS;
-
-	return AE_OK;
-}
-
 static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 {
 	struct backlight_properties props;
@@ -1902,7 +1879,7 @@ static int acpi_video_bus_add_notify_handler(struct acpi_video_bus *video,
 	snprintf(video->phys, sizeof(video->phys),
 			"%s/video/input0", acpi_device_hid(video->device));
 
-	input->name = acpi_device_name(video->device);
+	input->name = "Video Bus";
 	input->phys = video->phys;
 	input->id.bustype = BUS_HOST;
 	input->id.product = 0x06;
@@ -1976,53 +1953,69 @@ static int acpi_video_bus_put_devices(struct acpi_video_bus *video)
 	return 0;
 }
 
-static int instance;
+static int duplicate_dev_check(struct device *sibling, void *data)
+{
+	struct acpi_video_bus *video;
+
+	if (sibling == data || !dev_is_auxiliary(sibling))
+		return 0;
+
+	guard(mutex)(&video_list_lock);
+
+	list_for_each_entry(video, &video_bus_head, entry) {
+		if (video == dev_get_drvdata(sibling))
+			return -EEXIST;
+	}
+
+	return 0;
+}
+
+static bool acpi_video_bus_dev_is_duplicate(struct device *dev)
+{
+	return device_for_each_child(dev->parent, dev, duplicate_dev_check);
+}
 
 static int acpi_video_bus_probe(struct auxiliary_device *aux_dev,
 				const struct auxiliary_device_id *id_unused)
 {
 	struct acpi_device *device = ACPI_COMPANION(&aux_dev->dev);
+	static DEFINE_MUTEX(probe_lock);
 	struct acpi_video_bus *video;
+	static int instance;
 	bool auto_detect;
 	int error;
-	acpi_status status;
 
-	status = acpi_walk_namespace(ACPI_TYPE_DEVICE,
-				acpi_dev_parent(device)->handle, 1,
-				acpi_video_bus_match, NULL,
-				device, NULL);
-	if (status == AE_ALREADY_EXISTS) {
+	/* Probe one video bus device at a time in case there are duplicates. */
+	guard(mutex)(&probe_lock);
+
+	if (!allow_duplicates && acpi_video_bus_dev_is_duplicate(&aux_dev->dev)) {
 		pr_info(FW_BUG
 			"Duplicate ACPI video bus devices for the"
 			" same VGA controller, please try module "
 			"parameter \"video.allow_duplicates=1\""
 			"if the current driver doesn't work.\n");
-		if (!allow_duplicates)
-			return -ENODEV;
+		return -ENODEV;
 	}
 
 	video = kzalloc_obj(struct acpi_video_bus);
 	if (!video)
 		return -ENOMEM;
 
-	/* a hack to fix the duplicate name "VID" problem on T61 */
-	if (!strcmp(device->pnp.bus_id, "VID")) {
-		if (instance)
-			device->pnp.bus_id[3] = '0' + instance;
-		instance++;
-	}
-	/* a hack to fix the duplicate name "VGA" problem on Pa 3553 */
-	if (!strcmp(device->pnp.bus_id, "VGA")) {
+	/*
+	 * A hack to fix the duplicate name "VID" problem on T61 and the
+	 * duplicate name "VGA" problem on Pa 3553.
+	 */
+	if (!strcmp(device->pnp.bus_id, "VID") ||
+	    !strcmp(device->pnp.bus_id, "VGA")) {
 		if (instance)
 			device->pnp.bus_id[3] = '0' + instance;
+
 		instance++;
 	}
 
 	auxiliary_set_drvdata(aux_dev, video);
 
 	video->device = device;
-	strscpy(acpi_device_name(device), ACPI_VIDEO_BUS_NAME);
-	strscpy(acpi_device_class(device), ACPI_VIDEO_CLASS);
 	device->driver_data = video;
 
 	acpi_video_bus_find_cap(video);
@@ -2043,11 +2036,10 @@ static int acpi_video_bus_probe(struct auxiliary_device *aux_dev,
 	 */
 	acpi_device_fix_up_power_children(device);
 
-	pr_info("%s [%s] (multi-head: %s  rom: %s  post: %s)\n",
-	       ACPI_VIDEO_DEVICE_NAME, acpi_device_bid(device),
-	       str_yes_no(video->flags.multihead),
-	       str_yes_no(video->flags.rom),
-	       str_yes_no(video->flags.post));
+	pr_info("Video Device [%s] (multi-head: %s  rom: %s  post: %s)\n",
+		acpi_device_bid(device), str_yes_no(video->flags.multihead),
+		str_yes_no(video->flags.rom), str_yes_no(video->flags.post));
+
 	mutex_lock(&video_list_lock);
 	list_add_tail(&video->entry, &video_bus_head);
 	mutex_unlock(&video_list_lock);
diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h
index 6c9b5bf7d392d..07d5790d09f8c 100644
--- a/drivers/acpi/acpica/acpredef.h
+++ b/drivers/acpi/acpica/acpredef.h
@@ -451,7 +451,7 @@ const union acpi_predefined_info acpi_gbl_predefined_methods[] = {
 
 	{{"_DSM",
 	  METHOD_4ARGS(ACPI_TYPE_BUFFER, ACPI_TYPE_INTEGER, ACPI_TYPE_INTEGER,
-		       ACPI_TYPE_ANY | ACPI_TYPE_PACKAGE) |
+		       ACPI_TYPE_PACKAGE | ACPI_TYPE_ANY) |
 		       ARG_COUNT_IS_MINIMUM,
 	  METHOD_RETURNS(ACPI_RTYPE_ALL)}},	/* Must return a value, but it can be of any type */
 
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index 8fbad8bc46503..b4c25474f42f4 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -33,8 +33,6 @@
 #define ACPI_BATTERY_CAPACITY_VALID(capacity) \
 	((capacity) != 0 && (capacity) != ACPI_BATTERY_VALUE_UNKNOWN)
 
-#define ACPI_BATTERY_DEVICE_NAME	"Battery"
-
 /* Battery power unit: 0 means mW, 1 means mA */
 #define ACPI_BATTERY_POWER_UNIT_MA	1
 
@@ -1080,10 +1078,11 @@ static void acpi_battery_notify(acpi_handle handle, u32 event, void *data)
 	if (event == ACPI_BATTERY_NOTIFY_INFO)
 		acpi_battery_refresh(battery);
 	acpi_battery_update(battery, false);
-	acpi_bus_generate_netlink_event(device->pnp.device_class,
+	acpi_bus_generate_netlink_event(ACPI_BATTERY_CLASS,
 					dev_name(&device->dev), event,
 					acpi_battery_present(battery));
-	acpi_notifier_call_chain(device, event, acpi_battery_present(battery));
+	acpi_notifier_call_chain(ACPI_BATTERY_CLASS, acpi_device_bid(device),
+				 event, acpi_battery_present(battery));
 	/* acpi_battery_update could remove power_supply object */
 	if (old && battery->bat)
 		power_supply_changed(battery->bat);
@@ -1229,8 +1228,6 @@ static int acpi_battery_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, battery);
 
 	battery->device = device;
-	strscpy(acpi_device_name(device), ACPI_BATTERY_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_BATTERY_CLASS);
 
 	result = devm_mutex_init(&pdev->dev, &battery->update_lock);
 	if (result)
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index f6707325f5821..2ec095e2009e4 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -818,9 +818,6 @@ const struct acpi_device *acpi_companion_match(const struct device *dev)
 	if (list_empty(&adev->pnp.ids))
 		return NULL;
 
-	if (adev->pnp.type.backlight)
-		return adev;
-
 	return acpi_primary_dev_companion(adev, dev);
 }
 
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index 97b05246efab6..dc064a388c23e 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -468,7 +468,7 @@ static void acpi_button_notify(acpi_handle handle, u32 event, void *data)
 	input_report_key(input, keycode, 0);
 	input_sync(input);
 
-	acpi_bus_generate_netlink_event(device->pnp.device_class,
+	acpi_bus_generate_netlink_event(acpi_device_class(device),
 					dev_name(&device->dev),
 					event, ++button->pushed);
 }
@@ -558,27 +558,26 @@ static int acpi_button_probe(struct platform_device *pdev)
 		goto err_free_button;
 	}
 
-	name = acpi_device_name(device);
 	class = acpi_device_class(device);
 
 	if (!strcmp(hid, ACPI_BUTTON_HID_POWER) ||
 	    !strcmp(hid, ACPI_BUTTON_HID_POWERF)) {
 		button->type = ACPI_BUTTON_TYPE_POWER;
 		handler = acpi_button_notify;
-		strscpy(name, ACPI_BUTTON_DEVICE_NAME_POWER, MAX_ACPI_DEVICE_NAME_LEN);
+		name = ACPI_BUTTON_DEVICE_NAME_POWER;
 		sprintf(class, "%s/%s",
 			ACPI_BUTTON_CLASS, ACPI_BUTTON_SUBCLASS_POWER);
 	} else if (!strcmp(hid, ACPI_BUTTON_HID_SLEEP) ||
 		   !strcmp(hid, ACPI_BUTTON_HID_SLEEPF)) {
 		button->type = ACPI_BUTTON_TYPE_SLEEP;
 		handler = acpi_button_notify;
-		strscpy(name, ACPI_BUTTON_DEVICE_NAME_SLEEP, MAX_ACPI_DEVICE_NAME_LEN);
+		name = ACPI_BUTTON_DEVICE_NAME_SLEEP;
 		sprintf(class, "%s/%s",
 			ACPI_BUTTON_CLASS, ACPI_BUTTON_SUBCLASS_SLEEP);
 	} else if (!strcmp(hid, ACPI_BUTTON_HID_LID)) {
 		button->type = ACPI_BUTTON_TYPE_LID;
 		handler = acpi_lid_notify;
-		strscpy(name, ACPI_BUTTON_DEVICE_NAME_LID, MAX_ACPI_DEVICE_NAME_LEN);
+		name = ACPI_BUTTON_DEVICE_NAME_LID;
 		sprintf(class, "%s/%s",
 			ACPI_BUTTON_CLASS, ACPI_BUTTON_SUBCLASS_LID);
 		input->open = acpi_lid_input_open;
@@ -698,6 +697,8 @@ static void acpi_button_remove(struct platform_device *pdev)
 	acpi_button_remove_fs(button);
 	input_unregister_device(button->input);
 	kfree(button);
+
+	memset(acpi_device_class(adev), 0, sizeof(acpi_device_class));
 }
 
 static int param_set_lid_init_state(const char *val,
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index f0e513e9ed5d3..053fc6765a592 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -177,12 +177,12 @@ __ATTR(_name, 0444, show_##_name, NULL)
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, highest_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_perf);
+show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, reference_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_nonlinear_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, guaranteed_perf);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_freq);
 show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq);
 
-show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf);
 show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time);
 
 /* Check for valid access_width, otherwise, fallback to using bit_width */
@@ -853,6 +853,16 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	}
 	per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_subspace_id;
 
+	/*
+	 * In CPPC v1, DESIRED_PERF is mandatory. In CPPC v2, it is optional
+	 * only when AUTO_SEL_ENABLE is supported.
+	 */
+	if (!CPC_SUPPORTED(&cpc_ptr->cpc_regs[DESIRED_PERF]) &&
+	    (!osc_sb_cppc2_support_acked ||
+	     !CPC_SUPPORTED(&cpc_ptr->cpc_regs[AUTO_SEL_ENABLE])))
+		pr_warn("Desired perf. register is mandatory if CPPC v2 is not supported "
+			"or autonomous selection is disabled\n");
+
 	/*
 	 * Initialize the remaining cpc_regs as unsupported.
 	 * Example: In case FW exposes CPPC v2, the below loop will initialize
@@ -1342,9 +1352,10 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
 	struct cpc_register_resource *highest_reg, *lowest_reg,
-		*lowest_non_linear_reg, *nominal_reg, *guaranteed_reg,
-		*low_freq_reg = NULL, *nom_freq_reg = NULL;
-	u64 high, low, guaranteed, nom, min_nonlinear, low_f = 0, nom_f = 0;
+		*lowest_non_linear_reg, *nominal_reg, *reference_reg,
+		*guaranteed_reg, *low_freq_reg = NULL, *nom_freq_reg = NULL;
+	u64 high, low, guaranteed, nom, ref, min_nonlinear,
+	    low_f = 0, nom_f = 0;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
 	int ret = 0, regs_in_pcc = 0;
@@ -1358,6 +1369,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	lowest_reg = &cpc_desc->cpc_regs[LOWEST_PERF];
 	lowest_non_linear_reg = &cpc_desc->cpc_regs[LOW_NON_LINEAR_PERF];
 	nominal_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
+	reference_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
 	low_freq_reg = &cpc_desc->cpc_regs[LOWEST_FREQ];
 	nom_freq_reg = &cpc_desc->cpc_regs[NOMINAL_FREQ];
 	guaranteed_reg = &cpc_desc->cpc_regs[GUARANTEED_PERF];
@@ -1365,6 +1377,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) ||
 		CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) ||
+		(CPC_SUPPORTED(reference_reg) && CPC_IN_PCC(reference_reg)) ||
 		CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg) ||
 		CPC_IN_PCC(guaranteed_reg)) {
 		if (pcc_ss_id < 0) {
@@ -1381,35 +1394,66 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 		}
 	}
 
-	cpc_read(cpunum, highest_reg, &high);
+	ret = cpc_read(cpunum, highest_reg, &high);
+	if (ret)
+		goto out_err;
 	perf_caps->highest_perf = high;
 
-	cpc_read(cpunum, lowest_reg, &low);
+	ret = cpc_read(cpunum, lowest_reg, &low);
+	if (ret)
+		goto out_err;
 	perf_caps->lowest_perf = low;
 
-	cpc_read(cpunum, nominal_reg, &nom);
+	ret = cpc_read(cpunum, nominal_reg, &nom);
+	if (ret)
+		goto out_err;
 	perf_caps->nominal_perf = nom;
 
+	/*
+	 * If reference perf register is not supported then we should
+	 * use the nominal perf value
+	 */
+	if (CPC_SUPPORTED(reference_reg)) {
+		ret = cpc_read(cpunum, reference_reg, &ref);
+		if (ret)
+			goto out_err;
+	} else {
+		ref = nom;
+	}
+	perf_caps->reference_perf = ref;
+
 	if (guaranteed_reg->type != ACPI_TYPE_BUFFER  ||
 	    IS_NULL_REG(&guaranteed_reg->cpc_entry.reg)) {
 		perf_caps->guaranteed_perf = 0;
 	} else {
-		cpc_read(cpunum, guaranteed_reg, &guaranteed);
+		ret = cpc_read(cpunum, guaranteed_reg, &guaranteed);
+		if (ret)
+			goto out_err;
 		perf_caps->guaranteed_perf = guaranteed;
 	}
 
-	cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear);
+	ret = cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear);
+	if (ret)
+		goto out_err;
 	perf_caps->lowest_nonlinear_perf = min_nonlinear;
 
-	if (!high || !low || !nom || !min_nonlinear)
+	if (!high || !low || !nom || !ref || !min_nonlinear) {
 		ret = -EFAULT;
+		goto out_err;
+	}
 
 	/* Read optional lowest and nominal frequencies if present */
-	if (CPC_SUPPORTED(low_freq_reg))
-		cpc_read(cpunum, low_freq_reg, &low_f);
+	if (CPC_SUPPORTED(low_freq_reg)) {
+		ret = cpc_read(cpunum, low_freq_reg, &low_f);
+		if (ret)
+			goto out_err;
+	}
 
-	if (CPC_SUPPORTED(nom_freq_reg))
-		cpc_read(cpunum, nom_freq_reg, &nom_f);
+	if (CPC_SUPPORTED(nom_freq_reg)) {
+		ret = cpc_read(cpunum, nom_freq_reg, &nom_f);
+		if (ret)
+			goto out_err;
+	}
 
 	perf_caps->lowest_freq = low_f;
 	perf_caps->nominal_freq = nom_f;
@@ -1431,20 +1475,10 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_caps);
 bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
-	struct cpc_register_resource *ref_perf_reg;
-
-	/*
-	 * If reference perf register is not supported then we should use the
-	 * nominal perf value
-	 */
-	ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
-	if (!CPC_SUPPORTED(ref_perf_reg))
-		ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
 
 	return CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) ||
 		CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) ||
-		CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]) ||
-		CPC_IN_PCC(ref_perf_reg);
+		CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]);
 }
 EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu);
 
@@ -1481,10 +1515,10 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
 	struct cpc_register_resource *delivered_reg, *reference_reg,
-		*ref_perf_reg, *ctr_wrap_reg;
+		*ctr_wrap_reg;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
-	u64 delivered, reference, ref_perf, ctr_wrap_time;
+	u64 delivered, reference, ctr_wrap_time;
 	int ret = 0, regs_in_pcc = 0;
 
 	if (!cpc_desc) {
@@ -1494,19 +1528,11 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 
 	delivered_reg = &cpc_desc->cpc_regs[DELIVERED_CTR];
 	reference_reg = &cpc_desc->cpc_regs[REFERENCE_CTR];
-	ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
 	ctr_wrap_reg = &cpc_desc->cpc_regs[CTR_WRAP_TIME];
 
-	/*
-	 * If reference perf register is not supported then we should
-	 * use the nominal perf value
-	 */
-	if (!CPC_SUPPORTED(ref_perf_reg))
-		ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
-
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(delivered_reg) || CPC_IN_PCC(reference_reg) ||
-		CPC_IN_PCC(ctr_wrap_reg) || CPC_IN_PCC(ref_perf_reg)) {
+		CPC_IN_PCC(ctr_wrap_reg)) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id\n");
 			return -ENODEV;
@@ -1521,9 +1547,13 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 		}
 	}
 
-	cpc_read(cpunum, delivered_reg, &delivered);
-	cpc_read(cpunum, reference_reg, &reference);
-	cpc_read(cpunum, ref_perf_reg, &ref_perf);
+	ret = cpc_read(cpunum, delivered_reg, &delivered);
+	if (ret)
+		goto out_err;
+
+	ret = cpc_read(cpunum, reference_reg, &reference);
+	if (ret)
+		goto out_err;
 
 	/*
 	 * Per spec, if ctr_wrap_time optional register is unsupported, then the
@@ -1531,17 +1561,19 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 	 * platform
 	 */
 	ctr_wrap_time = (u64)(~((u64)0));
-	if (CPC_SUPPORTED(ctr_wrap_reg))
-		cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time);
+	if (CPC_SUPPORTED(ctr_wrap_reg)) {
+		ret = cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time);
+		if (ret)
+			goto out_err;
+	}
 
-	if (!delivered || !reference ||	!ref_perf) {
+	if (!delivered || !reference) {
 		ret = -EFAULT;
 		goto out_err;
 	}
 
 	perf_fb_ctrs->delivered = delivered;
 	perf_fb_ctrs->reference = reference;
-	perf_fb_ctrs->reference_perf = ref_perf;
 	perf_fb_ctrs->wraparound_time = ctr_wrap_time;
 out_err:
 	if (regs_in_pcc)
@@ -1561,6 +1593,8 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 	struct cpc_register_resource *auto_sel_reg;
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
+	bool autosel_ffh_sysmem;
+	bool epp_ffh_sysmem;
 	int ret;
 
 	if (!cpc_desc) {
@@ -1571,6 +1605,11 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
 	epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF];
 
+	epp_ffh_sysmem = CPC_SUPPORTED(epp_set_reg) &&
+		(CPC_IN_FFH(epp_set_reg) || CPC_IN_SYSTEM_MEMORY(epp_set_reg));
+	autosel_ffh_sysmem = CPC_SUPPORTED(auto_sel_reg) &&
+		(CPC_IN_FFH(auto_sel_reg) || CPC_IN_SYSTEM_MEMORY(auto_sel_reg));
+
 	if (CPC_IN_PCC(epp_set_reg) || CPC_IN_PCC(auto_sel_reg)) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu);
@@ -1596,11 +1635,22 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 		ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
 		up_write(&pcc_ss_data->pcc_lock);
 	} else if (osc_cpc_flexible_adr_space_confirmed &&
-		   CPC_SUPPORTED(epp_set_reg) && CPC_IN_FFH(epp_set_reg)) {
-		ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf);
+		   (epp_ffh_sysmem || autosel_ffh_sysmem)) {
+		if (autosel_ffh_sysmem) {
+			ret = cpc_write(cpu, auto_sel_reg, enable);
+			if (ret)
+				return ret;
+		}
+
+		if (epp_ffh_sysmem) {
+			ret = cpc_write(cpu, epp_set_reg,
+					perf_ctrls->energy_perf);
+			if (ret)
+				return ret;
+		}
 	} else {
 		ret = -ENOTSUPP;
-		pr_debug("_CPC in PCC and _CPC in FFH are not supported\n");
+		pr_debug("_CPC in PCC/FFH/SystemMemory are not supported\n");
 	}
 
 	return ret;
@@ -1738,6 +1788,101 @@ int cppc_set_enable(int cpu, bool enable)
 }
 EXPORT_SYMBOL_GPL(cppc_set_enable);
 
+/**
+ * cppc_get_perf - Get a CPU's performance controls.
+ * @cpu: CPU for which to get performance controls.
+ * @perf_ctrls: ptr to cppc_perf_ctrls. See cppc_acpi.h
+ *
+ * Return: 0 for success with perf_ctrls, -ERRNO otherwise.
+ */
+int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
+{
+	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
+	struct cpc_register_resource *desired_perf_reg,
+				     *min_perf_reg, *max_perf_reg,
+				     *energy_perf_reg, *auto_sel_reg;
+	u64 desired_perf = 0, min = 0, max = 0, energy_perf = 0, auto_sel = 0;
+	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
+	struct cppc_pcc_data *pcc_ss_data = NULL;
+	int ret = 0, regs_in_pcc = 0;
+
+	if (!cpc_desc) {
+		pr_debug("No CPC descriptor for CPU:%d\n", cpu);
+		return -ENODEV;
+	}
+
+	if (!perf_ctrls) {
+		pr_debug("Invalid perf_ctrls pointer\n");
+		return -EINVAL;
+	}
+
+	desired_perf_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
+	min_perf_reg = &cpc_desc->cpc_regs[MIN_PERF];
+	max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF];
+	energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF];
+	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+
+	/* Are any of the regs PCC ?*/
+	if (CPC_IN_PCC(desired_perf_reg) || CPC_IN_PCC(min_perf_reg) ||
+	    CPC_IN_PCC(max_perf_reg) || CPC_IN_PCC(energy_perf_reg) ||
+	    CPC_IN_PCC(auto_sel_reg)) {
+		if (pcc_ss_id < 0) {
+			pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu);
+			return -ENODEV;
+		}
+		pcc_ss_data = pcc_data[pcc_ss_id];
+		regs_in_pcc = 1;
+		down_write(&pcc_ss_data->pcc_lock);
+		/* Ring doorbell once to update PCC subspace */
+		if (send_pcc_cmd(pcc_ss_id, CMD_READ) < 0) {
+			ret = -EIO;
+			goto out_err;
+		}
+	}
+
+	/* Read optional elements if present */
+	if (CPC_SUPPORTED(max_perf_reg)) {
+		ret = cpc_read(cpu, max_perf_reg, &max);
+		if (ret)
+			goto out_err;
+	}
+	perf_ctrls->max_perf = max;
+
+	if (CPC_SUPPORTED(min_perf_reg)) {
+		ret = cpc_read(cpu, min_perf_reg, &min);
+		if (ret)
+			goto out_err;
+	}
+	perf_ctrls->min_perf = min;
+
+	if (CPC_SUPPORTED(desired_perf_reg)) {
+		ret = cpc_read(cpu, desired_perf_reg, &desired_perf);
+		if (ret)
+			goto out_err;
+	}
+	perf_ctrls->desired_perf = desired_perf;
+
+	if (CPC_SUPPORTED(energy_perf_reg)) {
+		ret = cpc_read(cpu, energy_perf_reg, &energy_perf);
+		if (ret)
+			goto out_err;
+	}
+	perf_ctrls->energy_perf = energy_perf;
+
+	if (CPC_SUPPORTED(auto_sel_reg)) {
+		ret = cpc_read(cpu, auto_sel_reg, &auto_sel);
+		if (ret)
+			goto out_err;
+	}
+	perf_ctrls->auto_sel = (bool)auto_sel;
+
+out_err:
+	if (regs_in_pcc)
+		up_write(&pcc_ss_data->pcc_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cppc_get_perf);
+
 /**
  * cppc_set_perf - Set a CPU's performance controls.
  * @cpu: CPU for which to set performance controls.
@@ -1870,6 +2015,62 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 }
 EXPORT_SYMBOL_GPL(cppc_set_perf);
 
+/**
+ * cppc_get_perf_limited - Get the Performance Limited register value.
+ * @cpu: CPU from which to get Performance Limited register.
+ * @perf_limited: Pointer to store the Performance Limited value.
+ *
+ * The returned value contains sticky status bits indicating platform-imposed
+ * performance limitations.
+ *
+ * Return: 0 for success, -EIO on failure, -EOPNOTSUPP if not supported.
+ */
+int cppc_get_perf_limited(int cpu, u64 *perf_limited)
+{
+	return cppc_get_reg_val(cpu, PERF_LIMITED, perf_limited);
+}
+EXPORT_SYMBOL_GPL(cppc_get_perf_limited);
+
+/**
+ * cppc_set_perf_limited() - Clear bits in the Performance Limited register.
+ * @cpu: CPU on which to write register.
+ * @bits_to_clear: Bitmask of bits to clear in the perf_limited register.
+ *
+ * The Performance Limited register contains two sticky bits set by platform:
+ *   - Bit 0 (Desired_Excursion): Set when delivered performance is constrained
+ *     below desired performance. Not used when Autonomous Selection is enabled.
+ *   - Bit 1 (Minimum_Excursion): Set when delivered performance is constrained
+ *     below minimum performance.
+ *
+ * These bits are sticky and remain set until OSPM explicitly clears them.
+ * This function only allows clearing bits (the platform sets them).
+ *
+ * Return: 0 for success, -EINVAL for invalid bits, -EIO on register
+ *         access failure, -EOPNOTSUPP if not supported.
+ */
+int cppc_set_perf_limited(int cpu, u64 bits_to_clear)
+{
+	u64 current_val, new_val;
+	int ret;
+
+	/* Only bits 0 and 1 are valid */
+	if (bits_to_clear & ~CPPC_PERF_LIMITED_MASK)
+		return -EINVAL;
+
+	if (!bits_to_clear)
+		return 0;
+
+	ret = cppc_get_perf_limited(cpu, &current_val);
+	if (ret)
+		return ret;
+
+	/* Clear the specified bits */
+	new_val = current_val & ~bits_to_clear;
+
+	return cppc_set_reg_val(cpu, PERF_LIMITED, new_val);
+}
+EXPORT_SYMBOL_GPL(cppc_set_perf_limited);
+
 /**
  * cppc_get_transition_latency - returns frequency transition latency in ns
  * @cpu_num: CPU number for per_cpu().
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 5f63ed120a2c7..0624d86736793 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -35,9 +35,6 @@
 
 #include "internal.h"
 
-#define ACPI_EC_CLASS			"embedded_controller"
-#define ACPI_EC_DEVICE_NAME		"Embedded Controller"
-
 /* EC status register */
 #define ACPI_EC_FLAG_OBF	0x01	/* Output buffer full */
 #define ACPI_EC_FLAG_IBF	0x02	/* Input buffer full */
@@ -1681,9 +1678,6 @@ static int acpi_ec_probe(struct platform_device *pdev)
 	struct acpi_ec *ec;
 	int ret;
 
-	strscpy(acpi_device_name(device), ACPI_EC_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_EC_CLASS);
-
 	if (boot_ec && (boot_ec->handle == device->handle ||
 	    !strcmp(acpi_device_hid(device), ACPI_ECDT_HID))) {
 		/* Fast path: this device corresponds to the boot EC. */
diff --git a/drivers/acpi/event.c b/drivers/acpi/event.c
index 96a9aaaaf9f70..4d840d2e7b98b 100644
--- a/drivers/acpi/event.c
+++ b/drivers/acpi/event.c
@@ -24,12 +24,13 @@
 /* ACPI notifier chain */
 static BLOCKING_NOTIFIER_HEAD(acpi_chain_head);
 
-int acpi_notifier_call_chain(struct acpi_device *dev, u32 type, u32 data)
+int acpi_notifier_call_chain(const char *device_class,
+			     const char *bus_id, u32 type, u32 data)
 {
 	struct acpi_bus_event event;
 
-	strscpy(event.device_class, dev->pnp.device_class);
-	strscpy(event.bus_id, dev->pnp.bus_id);
+	strscpy(event.device_class, device_class);
+	strscpy(event.bus_id, bus_id);
 	event.type = type;
 	event.data = data;
 	return (blocking_notifier_call_chain(&acpi_chain_head, 0, (void *)&event)
diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c
index aa87ee1583a4e..62d4a8df0b8c1 100644
--- a/drivers/acpi/numa/srat.c
+++ b/drivers/acpi/numa/srat.c
@@ -654,8 +654,11 @@ int __init acpi_numa_init(void)
 	}
 	last_real_pxm = fake_pxm;
 	fake_pxm++;
-	acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws,
-			      &fake_pxm);
+
+	/* No need to expand numa nodes if CXL is disabled */
+	if (IS_ENABLED(CONFIG_CXL_ACPI))
+		acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws,
+				      &fake_pxm);
 
 	if (cnt < 0)
 		return cnt;
diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c
index 62b9c83d4f20b..ed2162a97bd2a 100644
--- a/drivers/acpi/osl.c
+++ b/drivers/acpi/osl.c
@@ -13,6 +13,8 @@
 
 #include <linux/module.h>
 #include <linux/kernel.h>
+#include <linux/panic.h>
+#include <linux/reboot.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
 #include <linux/highmem.h>
@@ -70,6 +72,10 @@ static bool acpi_os_initialized;
 unsigned int acpi_sci_irq = INVALID_ACPI_IRQ;
 bool acpi_permanent_mmap = false;
 
+static bool poweroff_on_fatal = true;
+module_param(poweroff_on_fatal, bool, 0);
+MODULE_PARM_DESC(poweroff_on_fatal, "Poweroff when encountering a fatal ACPI error");
+
 /*
  * This list of permanent mappings is for memory that may be accessed from
  * interrupt context, where we can't do the ioremap().
@@ -1257,7 +1263,7 @@ acpi_status acpi_os_delete_semaphore(acpi_handle handle)
 
 	ACPI_DEBUG_PRINT((ACPI_DB_MUTEX, "Deleting semaphore[%p].\n", handle));
 
-	BUG_ON(!list_empty(&sem->wait_list));
+	BUG_ON(sem->first_waiter);
 	kfree(sem);
 	sem = NULL;
 
@@ -1381,9 +1387,20 @@ acpi_status acpi_os_notify_command_complete(void)
 
 acpi_status acpi_os_signal(u32 function, void *info)
 {
+	struct acpi_signal_fatal_info *fatal_info;
+
 	switch (function) {
 	case ACPI_SIGNAL_FATAL:
-		pr_err("Fatal opcode executed\n");
+		fatal_info = info;
+		pr_emerg("Fatal error while evaluating ACPI control method\n");
+		pr_emerg("Type 0x%X Code 0x%X Argument 0x%X\n",
+			 fatal_info->type, fatal_info->code, fatal_info->argument);
+
+		if (poweroff_on_fatal)
+			orderly_poweroff(true);
+		else
+			add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
+
 		break;
 	case ACPI_SIGNAL_BREAKPOINT:
 		/*
diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c
index 45bdfd06bd213..e6ed13aee48d0 100644
--- a/drivers/acpi/pci_link.c
+++ b/drivers/acpi/pci_link.c
@@ -29,8 +29,6 @@
 
 #include "internal.h"
 
-#define ACPI_PCI_LINK_CLASS		"pci_irq_routing"
-#define ACPI_PCI_LINK_DEVICE_NAME	"PCI Interrupt Link"
 #define ACPI_PCI_LINK_MAX_POSSIBLE	16
 
 static int acpi_pci_link_add(struct acpi_device *device,
@@ -725,8 +723,6 @@ static int acpi_pci_link_add(struct acpi_device *device,
 		return -ENOMEM;
 
 	link->device = device;
-	strscpy(acpi_device_name(device), ACPI_PCI_LINK_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_PCI_LINK_CLASS);
 	device->driver_data = link;
 
 	mutex_lock(&acpi_link_lock);
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index 4a882e9395256..a0ba64e45e8ab 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -24,8 +24,6 @@
 #include <linux/platform_data/x86/apple.h>
 #include "internal.h"
 
-#define ACPI_PCI_ROOT_CLASS		"pci_bridge"
-#define ACPI_PCI_ROOT_DEVICE_NAME	"PCI Root Bridge"
 static int acpi_pci_root_add(struct acpi_device *device,
 			     const struct acpi_device_id *not_used);
 static void acpi_pci_root_remove(struct acpi_device *device);
@@ -689,8 +687,6 @@ static int acpi_pci_root_add(struct acpi_device *device,
 
 	root->device = device;
 	root->segment = segment & 0xFFFF;
-	strscpy(acpi_device_name(device), ACPI_PCI_ROOT_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_PCI_ROOT_CLASS);
 	device->driver_data = root;
 
 	if (hotadd && dmar_device_add(handle)) {
@@ -698,9 +694,8 @@ static int acpi_pci_root_add(struct acpi_device *device,
 		goto end;
 	}
 
-	pr_info("%s [%s] (domain %04x %pR)\n",
-	       acpi_device_name(device), acpi_device_bid(device),
-	       root->segment, &root->secondary);
+	pr_info("PCI Root Bridge [%s] (domain %04x %pR)\n",
+		acpi_device_bid(device), root->segment, &root->secondary);
 
 	root->mcfg_addr = acpi_pci_root_get_mcfg_addr(handle);
 
diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
index 4611159ee734d..6b1680ec36943 100644
--- a/drivers/acpi/power.c
+++ b/drivers/acpi/power.c
@@ -37,8 +37,6 @@
 #include "sleep.h"
 #include "internal.h"
 
-#define ACPI_POWER_CLASS		"power_resource"
-#define ACPI_POWER_DEVICE_NAME		"Power Resource"
 #define ACPI_POWER_RESOURCE_STATE_OFF	0x00
 #define ACPI_POWER_RESOURCE_STATE_ON	0x01
 #define ACPI_POWER_RESOURCE_STATE_UNKNOWN 0xFF
@@ -955,8 +953,6 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle)
 	mutex_init(&resource->resource_lock);
 	INIT_LIST_HEAD(&resource->list_node);
 	INIT_LIST_HEAD(&resource->dependents);
-	strscpy(acpi_device_name(device), ACPI_POWER_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_POWER_CLASS);
 	device->power.state = ACPI_STATE_UNKNOWN;
 	device->flags.match_driver = true;
 
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index 882709796b4f7..cda8fd7200004 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -53,7 +53,7 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data)
 {
 	struct acpi_device *device = data;
 	struct acpi_processor *pr;
-	int saved;
+	int saved, ev_data = 0;
 
 	if (device->handle != handle)
 		return;
@@ -66,33 +66,27 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data)
 	case ACPI_PROCESSOR_NOTIFY_PERFORMANCE:
 		saved = pr->performance_platform_limit;
 		acpi_processor_ppc_has_changed(pr, 1);
-		if (saved == pr->performance_platform_limit)
-			break;
-		acpi_bus_generate_netlink_event(device->pnp.device_class,
-						  dev_name(&device->dev), event,
-						  pr->performance_platform_limit);
+		ev_data = pr->performance_platform_limit;
+		if (saved == ev_data)
+			return;
+
 		break;
 	case ACPI_PROCESSOR_NOTIFY_POWER:
 		acpi_processor_power_state_has_changed(pr);
-		acpi_bus_generate_netlink_event(device->pnp.device_class,
-						  dev_name(&device->dev), event, 0);
 		break;
 	case ACPI_PROCESSOR_NOTIFY_THROTTLING:
 		acpi_processor_tstate_has_changed(pr);
-		acpi_bus_generate_netlink_event(device->pnp.device_class,
-						  dev_name(&device->dev), event, 0);
 		break;
 	case ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED:
 		cpufreq_update_limits(pr->id);
-		acpi_bus_generate_netlink_event(device->pnp.device_class,
-						  dev_name(&device->dev), event, 0);
 		break;
 	default:
 		acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event);
-		break;
+		return;
 	}
 
-	return;
+	acpi_bus_generate_netlink_event("processor", dev_name(&device->dev),
+					event, ev_data);
 }
 
 static int __acpi_processor_start(struct acpi_device *device);
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index f6c72e3a2be1b..479995a4c48ab 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -819,19 +819,13 @@ static void acpi_processor_setup_cstates(struct acpi_processor *pr)
 	drv->state_count = count;
 }
 
-static inline void acpi_processor_cstate_first_run_checks(void)
+static inline void acpi_processor_update_max_cstate(void)
 {
-	static int first_run;
-
-	if (first_run)
-		return;
 	dmi_check_system(processor_power_dmi_table);
 	max_cstate = acpi_processor_cstate_check(max_cstate);
 	if (max_cstate < ACPI_C_STATES_MAX)
 		pr_notice("processor limited to max C-state %d\n", max_cstate);
 
-	first_run++;
-
 	if (nocst)
 		return;
 
@@ -840,7 +834,7 @@ static inline void acpi_processor_cstate_first_run_checks(void)
 #else
 
 static inline int disabled_by_idle_boot_param(void) { return 0; }
-static inline void acpi_processor_cstate_first_run_checks(void) { }
+static inline void acpi_processor_update_max_cstate(void) { }
 static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
 {
 	return -ENODEV;
@@ -1016,9 +1010,7 @@ static bool combine_lpi_states(struct acpi_lpi_state *local,
 	result->arch_flags = parent->arch_flags;
 	result->index = parent->index;
 
-	strscpy(result->desc, local->desc, ACPI_CX_DESC_LEN);
-	strlcat(result->desc, "+", ACPI_CX_DESC_LEN);
-	strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN);
+	scnprintf(result->desc, ACPI_CX_DESC_LEN, "%s+%s", local->desc, parent->desc);
 	return true;
 }
 
@@ -1068,6 +1060,8 @@ static unsigned int flatten_lpi_states(struct acpi_processor *pr,
 				stash_composite_state(curr_level, flpi);
 				flat_state_cnt++;
 				flpi++;
+				if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER)
+					break;
 			}
 		}
 	}
@@ -1357,6 +1351,8 @@ void acpi_processor_register_idle_driver(void)
 	int ret = -ENODEV;
 	int cpu;
 
+	acpi_processor_update_max_cstate();
+
 	/*
 	 * ACPI idle driver is used by all possible CPUs.
 	 * Use the processor power info of one in them to set up idle states.
@@ -1368,7 +1364,6 @@ void acpi_processor_register_idle_driver(void)
 		if (!pr)
 			continue;
 
-		acpi_processor_cstate_first_run_checks();
 		ret = acpi_processor_get_power_info(pr);
 		if (!ret) {
 			pr->flags.power_setup_done = 1;
@@ -1409,8 +1404,6 @@ void acpi_processor_power_init(struct acpi_processor *pr)
 	if (disabled_by_idle_boot_param())
 		return;
 
-	acpi_processor_cstate_first_run_checks();
-
 	if (!acpi_processor_get_power_info(pr))
 		pr->flags.power_setup_done = 1;
 
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index bbd3938f7b524..440f1d69aca86 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -26,8 +26,6 @@
 
 #include "sbshc.h"
 
-#define ACPI_SBS_CLASS			"sbs"
-#define ACPI_AC_CLASS			"ac_adapter"
 #define ACPI_SBS_DEVICE_NAME		"Smart Battery System"
 #define ACPI_BATTERY_DIR_NAME		"BAT%i"
 #define ACPI_AC_DIR_NAME		"AC0"
@@ -648,8 +646,6 @@ static int acpi_sbs_probe(struct platform_device *pdev)
 
 	sbs->hc = dev_get_drvdata(pdev->dev.parent);
 	sbs->device = device;
-	strscpy(acpi_device_name(device), ACPI_SBS_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_SBS_CLASS);
 
 	result = acpi_charger_add(sbs);
 	if (result && result != -ENODEV)
diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c
index 36850831910bc..f413270415b68 100644
--- a/drivers/acpi/sbshc.c
+++ b/drivers/acpi/sbshc.c
@@ -18,9 +18,6 @@
 #include "sbshc.h"
 #include "internal.h"
 
-#define ACPI_SMB_HC_CLASS	"smbus_host_ctl"
-#define ACPI_SMB_HC_DEVICE_NAME	"ACPI SMBus HC"
-
 struct acpi_smb_hc {
 	struct acpi_ec *ec;
 	struct mutex lock;
@@ -251,9 +248,6 @@ static int acpi_smbus_hc_probe(struct platform_device *pdev)
 		return -EIO;
 	}
 
-	strscpy(acpi_device_name(device), ACPI_SMB_HC_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_SMB_HC_CLASS);
-
 	hc = kzalloc_obj(struct acpi_smb_hc);
 	if (!hc)
 		return -ENOMEM;
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index 64356b004a57d..b8b487d89d25b 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -35,7 +35,6 @@
 #include "internal.h"
 
 #define ACPI_THERMAL_CLASS		"thermal_zone"
-#define ACPI_THERMAL_DEVICE_NAME	"Thermal Zone"
 #define ACPI_THERMAL_NOTIFY_TEMPERATURE	0x80
 #define ACPI_THERMAL_NOTIFY_THRESHOLDS	0x81
 #define ACPI_THERMAL_NOTIFY_DEVICES	0x82
@@ -341,7 +340,7 @@ static void acpi_thermal_trips_update(struct acpi_thermal *tz, u32 event)
 	thermal_zone_for_each_trip(tz->thermal_zone,
 				   acpi_thermal_adjust_trip, &atd);
 	acpi_queue_thermal_check(tz);
-	acpi_bus_generate_netlink_event(adev->pnp.device_class,
+	acpi_bus_generate_netlink_event(ACPI_THERMAL_CLASS,
 					dev_name(&adev->dev), event, 0);
 }
 
@@ -543,7 +542,7 @@ static void acpi_thermal_zone_device_hot(struct thermal_zone_device *thermal)
 {
 	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 
-	acpi_bus_generate_netlink_event(tz->device->pnp.device_class,
+	acpi_bus_generate_netlink_event(ACPI_THERMAL_CLASS,
 					dev_name(&tz->device->dev),
 					ACPI_THERMAL_NOTIFY_HOT, 1);
 }
@@ -552,7 +551,7 @@ static void acpi_thermal_zone_device_critical(struct thermal_zone_device *therma
 {
 	struct acpi_thermal *tz = thermal_zone_device_priv(thermal);
 
-	acpi_bus_generate_netlink_event(tz->device->pnp.device_class,
+	acpi_bus_generate_netlink_event(ACPI_THERMAL_CLASS,
 					dev_name(&tz->device->dev),
 					ACPI_THERMAL_NOTIFY_CRITICAL, 1);
 
@@ -800,8 +799,6 @@ static int acpi_thermal_probe(struct platform_device *pdev)
 
 	tz->device = device;
 	strscpy(tz->name, device->pnp.bus_id);
-	strscpy(acpi_device_name(device), ACPI_THERMAL_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_THERMAL_CLASS);
 
 	acpi_thermal_aml_dependency_fix(tz);
 
@@ -879,8 +876,8 @@ static int acpi_thermal_probe(struct platform_device *pdev)
 	mutex_init(&tz->thermal_check_lock);
 	INIT_WORK(&tz->thermal_check_work, acpi_thermal_check_fn);
 
-	pr_info("%s [%s] (%ld C)\n", acpi_device_name(device),
-		acpi_device_bid(device), deci_kelvin_to_celsius(tz->temp_dk));
+	pr_info("Thermal Zone [%s] (%ld C)\n", acpi_device_bid(device),
+		deci_kelvin_to_celsius(tz->temp_dk));
 
 	result = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
 						 acpi_thermal_notify, tz);
diff --git a/drivers/acpi/x86/cmos_rtc.c b/drivers/acpi/x86/cmos_rtc.c
index 51643ff6fe5fc..ced334e198961 100644
--- a/drivers/acpi/x86/cmos_rtc.c
+++ b/drivers/acpi/x86/cmos_rtc.c
@@ -18,78 +18,86 @@
 #include "../internal.h"
 
 static const struct acpi_device_id acpi_cmos_rtc_ids[] = {
-	{ "PNP0B00" },
-	{ "PNP0B01" },
-	{ "PNP0B02" },
-	{}
+	{ "ACPI000E", 1 }, /* ACPI Time and Alarm Device (TAD) */
+	ACPI_CMOS_RTC_IDS
 };
 
-static acpi_status
-acpi_cmos_rtc_space_handler(u32 function, acpi_physical_address address,
-		      u32 bits, u64 *value64,
-		      void *handler_context, void *region_context)
+bool cmos_rtc_platform_device_present;
+
+static acpi_status acpi_cmos_rtc_space_handler(u32 function,
+					       acpi_physical_address address,
+					       u32 bits, u64 *value64,
+					       void *handler_context,
+					       void *region_context)
 {
-	int i;
+	unsigned int i, bytes = DIV_ROUND_UP(bits, 8);
 	u8 *value = (u8 *)value64;
 
 	if (address > 0xff || !value64)
 		return AE_BAD_PARAMETER;
 
-	if (function != ACPI_WRITE && function != ACPI_READ)
-		return AE_BAD_PARAMETER;
+	guard(spinlock_irq)(&rtc_lock);
+
+	if (function == ACPI_WRITE) {
+		for (i = 0; i < bytes; i++, address++, value++)
+			CMOS_WRITE(*value, address);
 
-	spin_lock_irq(&rtc_lock);
+		return AE_OK;
+	}
 
-	for (i = 0; i < DIV_ROUND_UP(bits, 8); ++i, ++address, ++value)
-		if (function == ACPI_READ)
+	if (function == ACPI_READ) {
+		for (i = 0; i < bytes; i++, address++, value++)
 			*value = CMOS_READ(address);
-		else
-			CMOS_WRITE(*value, address);
 
-	spin_unlock_irq(&rtc_lock);
+		return AE_OK;
+	}
 
-	return AE_OK;
+	return AE_BAD_PARAMETER;
 }
 
-int acpi_install_cmos_rtc_space_handler(acpi_handle handle)
+static int acpi_install_cmos_rtc_space_handler(acpi_handle handle)
 {
+	static bool cmos_rtc_space_handler_present __read_mostly;
 	acpi_status status;
 
+	if (cmos_rtc_space_handler_present)
+		return 0;
+
 	status = acpi_install_address_space_handler(handle,
-			ACPI_ADR_SPACE_CMOS,
-			&acpi_cmos_rtc_space_handler,
-			NULL, NULL);
+						    ACPI_ADR_SPACE_CMOS,
+						    acpi_cmos_rtc_space_handler,
+						    NULL, NULL);
 	if (ACPI_FAILURE(status)) {
-		pr_err("Error installing CMOS-RTC region handler\n");
+		pr_err("Failed to install CMOS-RTC address space handler\n");
 		return -ENODEV;
 	}
 
+	cmos_rtc_space_handler_present = true;
+
 	return 1;
 }
-EXPORT_SYMBOL_GPL(acpi_install_cmos_rtc_space_handler);
 
-void acpi_remove_cmos_rtc_space_handler(acpi_handle handle)
+static int acpi_cmos_rtc_attach(struct acpi_device *adev,
+				const struct acpi_device_id *id)
 {
-	if (ACPI_FAILURE(acpi_remove_address_space_handler(handle,
-			ACPI_ADR_SPACE_CMOS, &acpi_cmos_rtc_space_handler)))
-		pr_err("Error removing CMOS-RTC region handler\n");
-}
-EXPORT_SYMBOL_GPL(acpi_remove_cmos_rtc_space_handler);
+	int ret;
 
-static int acpi_cmos_rtc_attach_handler(struct acpi_device *adev, const struct acpi_device_id *id)
-{
-	return acpi_install_cmos_rtc_space_handler(adev->handle);
-}
+	ret = acpi_install_cmos_rtc_space_handler(adev->handle);
+	if (ret < 0)
+		return ret;
 
-static void acpi_cmos_rtc_detach_handler(struct acpi_device *adev)
-{
-	acpi_remove_cmos_rtc_space_handler(adev->handle);
+	if (IS_ERR_OR_NULL(acpi_create_platform_device(adev, NULL))) {
+		pr_err("Failed to create a platform device for %s\n", (char *)id->id);
+		return 0;
+	} else if (!id->driver_data) {
+		cmos_rtc_platform_device_present = true;
+	}
+	return 1;
 }
 
 static struct acpi_scan_handler cmos_rtc_handler = {
 	.ids = acpi_cmos_rtc_ids,
-	.attach = acpi_cmos_rtc_attach_handler,
-	.detach = acpi_cmos_rtc_detach_handler,
+	.attach = acpi_cmos_rtc_attach,
 };
 
 void __init acpi_cmos_rtc_init(void)
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index 21f91d9f2fbcf..9e61942245936 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -3117,7 +3117,7 @@ static void binder_transaction(struct binder_proc *proc,
 	t->start_time = t_start_time;
 	t->from_pid = proc->pid;
 	t->from_tid = thread->pid;
-	t->sender_euid = task_euid(proc->tsk);
+	t->sender_euid = current_euid();
 	t->code = tr->code;
 	t->flags = tr->flags;
 	t->priority = task_nice(current);
diff --git a/drivers/android/binder/context.rs b/drivers/android/binder/context.rs
index 9cf437c025a20..ddddb66b35571 100644
--- a/drivers/android/binder/context.rs
+++ b/drivers/android/binder/context.rs
@@ -94,6 +94,17 @@ pub(crate) fn deregister_process(self: &Arc<Self>, proc: &Arc<Process>) {
         }
         let mut manager = self.manager.lock();
         manager.all_procs.retain(|p| !Arc::ptr_eq(p, proc));
+
+        // Shrink the vector if it has significant unused capacity to avoid memory waste,
+        // but use a conservative strategy to prevent shrink-then-regrow oscillation.
+        // Only shrink when length drops below 1/4 of capacity, and shrink to twice the length.
+        let len = manager.all_procs.len();
+        let cap = manager.all_procs.capacity();
+        if len < cap / 4 {
+            // Shrink to twice the current length. Ignore allocation failures since this
+            // is just an optimization; the vector remains valid even if shrinking fails.
+            let _ = manager.all_procs.shrink_to(len * 2, GFP_KERNEL);
+        }
     }
 
     pub(crate) fn set_manager_node(&self, node_ref: NodeRef) -> Result {
diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs
index 9dfc154e5dd4e..8882fd18d9f3f 100644
--- a/drivers/android/binder/page_range.rs
+++ b/drivers/android/binder/page_range.rs
@@ -130,7 +130,7 @@ pub(crate) struct ShrinkablePageRange {
     pid: Pid,
     /// The mm for the relevant process.
     mm: ARef<Mm>,
-    /// Used to synchronize calls to `vm_insert_page` and `zap_page_range_single`.
+    /// Used to synchronize calls to `vm_insert_page` and `zap_vma_range`.
     #[pin]
     mm_lock: Mutex<()>,
     /// Spinlock protecting changes to pages.
@@ -762,7 +762,7 @@ fn drop(self: Pin<&mut Self>) {
     if let Some(unchecked_vma) = mmap_read.vma_lookup(vma_addr) {
         if let Some(vma) = check_vma(unchecked_vma, range_ptr) {
             let user_page_addr = vma_addr + (page_index << PAGE_SHIFT);
-            vma.zap_page_range_single(user_page_addr, PAGE_SIZE);
+            vma.zap_vma_range(user_page_addr, PAGE_SIZE);
         }
     }
 
diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs
index f06498129aa97..da81a9569365e 100644
--- a/drivers/android/binder/process.rs
+++ b/drivers/android/binder/process.rs
@@ -1443,6 +1443,9 @@ pub(crate) fn drop_outstanding_txn(&self) {
         }
     }
 
+    // #[export_name] is a temporary workaround so that ps output does not become unreadable from
+    // mangled symbol names.
+    #[export_name = "rust_binder_freeze"]
     pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result {
         if info.enable == 0 {
             let msgs = self.prepare_freeze_messages()?;
diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs
index c004214b16629..6f283de53213f 100644
--- a/drivers/android/binder/thread.rs
+++ b/drivers/android/binder/thread.rs
@@ -513,6 +513,9 @@ pub(crate) fn has_current_transaction(&self) -> bool {
     /// Attempts to fetch a work item from the thread-local queue. The behaviour if the queue is
     /// empty depends on `wait`: if it is true, the function waits for some work to be queued (or a
     /// signal); otherwise it returns indicating that none is available.
+    // #[export_name] is a temporary workaround so that ps output does not become unreadable from
+    // mangled symbol names.
+    #[export_name = "rust_binder_waitlcl"]
     fn get_work_local(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> {
         {
             let mut inner = self.inner.lock();
@@ -551,6 +554,9 @@ fn get_work_local(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn Deliv
     ///
     /// This must only be called when the thread is not participating in a transaction chain. If it
     /// is, the local version (`get_work_local`) should be used instead.
+    // #[export_name] is a temporary workaround so that ps output does not become unreadable from
+    // mangled symbol names.
+    #[export_name = "rust_binder_wait"]
     fn get_work(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> {
         // Try to get work from the thread's work queue, using only a local lock.
         {
diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs
index 75e6f5fbaaae5..10af40527ca7c 100644
--- a/drivers/android/binder/transaction.rs
+++ b/drivers/android/binder/transaction.rs
@@ -107,7 +107,7 @@ pub(crate) fn new(
             debug_id,
             target_node: Some(target_node),
             from_parent,
-            sender_euid: from.process.task.euid(),
+            sender_euid: Kuid::current_euid(),
             from: from.clone(),
             to,
             code: trd.code,
@@ -147,7 +147,7 @@ pub(crate) fn new_reply(
             debug_id,
             target_node: None,
             from_parent: None,
-            sender_euid: from.process.task.euid(),
+            sender_euid: Kuid::current_euid(),
             from: from.clone(),
             to,
             code: trd.code,
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
index 241f16a9b63d1..e4488ad86a655 100644
--- a/drivers/android/binder_alloc.c
+++ b/drivers/android/binder_alloc.c
@@ -1185,7 +1185,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item,
 	if (vma) {
 		trace_binder_unmap_user_start(alloc, index);
 
-		zap_page_range_single(vma, page_addr, PAGE_SIZE, NULL);
+		zap_vma_range(vma, page_addr, PAGE_SIZE);
 
 		trace_binder_unmap_user_end(alloc, index);
 	}
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 2349bca136e05..fff305ec1e788 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -194,7 +194,6 @@ config AHCI_DM816
 config AHCI_DWC
 	tristate "Synopsys DWC AHCI SATA support"
 	select SATA_HOST
-	select MFD_SYSCON if (MIPS_BAIKAL_T1 || COMPILE_TEST)
 	help
 	  This option enables support for the Synopsys DWC AHCI SATA
 	  controller implementation.
diff --git a/drivers/ata/ahci_dwc.c b/drivers/ata/ahci_dwc.c
index 64abf865bb674..feb8a8539b8ef 100644
--- a/drivers/ata/ahci_dwc.c
+++ b/drivers/ata/ahci_dwc.c
@@ -13,12 +13,10 @@
 #include <linux/kernel.h>
 #include <linux/libata.h>
 #include <linux/log2.h>
-#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
-#include <linux/regmap.h>
 
 #include "ahci.h"
 
@@ -92,20 +90,6 @@
 #define AHCI_DWC_PORT_PHYCR		0x74
 #define AHCI_DWC_PORT_PHYSR		0x78
 
-/* Baikal-T1 AHCI SATA specific registers */
-#define AHCI_BT1_HOST_PHYCR		AHCI_DWC_HOST_GPCR
-#define AHCI_BT1_HOST_MPLM_MASK		GENMASK(29, 23)
-#define AHCI_BT1_HOST_LOSDT_MASK	GENMASK(22, 20)
-#define AHCI_BT1_HOST_CRR		BIT(19)
-#define AHCI_BT1_HOST_CRW		BIT(18)
-#define AHCI_BT1_HOST_CRCD		BIT(17)
-#define AHCI_BT1_HOST_CRCA		BIT(16)
-#define AHCI_BT1_HOST_CRDI_MASK		GENMASK(15, 0)
-
-#define AHCI_BT1_HOST_PHYSR		AHCI_DWC_HOST_GPSR
-#define AHCI_BT1_HOST_CRA		BIT(16)
-#define AHCI_BT1_HOST_CRDO_MASK		GENMASK(15, 0)
-
 struct ahci_dwc_plat_data {
 	unsigned int pflags;
 	unsigned int hflags;
@@ -122,39 +106,6 @@ struct ahci_dwc_host_priv {
 	u32 dmacr[AHCI_MAX_PORTS];
 };
 
-static int ahci_bt1_init(struct ahci_host_priv *hpriv)
-{
-	struct ahci_dwc_host_priv *dpriv = hpriv->plat_data;
-	int ret;
-
-	/* APB, application and reference clocks are required */
-	if (!ahci_platform_find_clk(hpriv, "pclk") ||
-	    !ahci_platform_find_clk(hpriv, "aclk") ||
-	    !ahci_platform_find_clk(hpriv, "ref")) {
-		dev_err(&dpriv->pdev->dev, "No system clocks specified\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Fully reset the SATA AXI and ref clocks domain to ensure the state
-	 * machine is working from scratch especially if the reference clocks
-	 * source has been changed.
-	 */
-	ret = ahci_platform_assert_rsts(hpriv);
-	if (ret) {
-		dev_err(&dpriv->pdev->dev, "Couldn't assert the resets\n");
-		return ret;
-	}
-
-	ret = ahci_platform_deassert_rsts(hpriv);
-	if (ret) {
-		dev_err(&dpriv->pdev->dev, "Couldn't de-assert the resets\n");
-		return ret;
-	}
-
-	return 0;
-}
-
 static struct ahci_host_priv *ahci_dwc_get_resources(struct platform_device *pdev)
 {
 	struct ahci_dwc_host_priv *dpriv;
@@ -457,15 +408,9 @@ static struct ahci_dwc_plat_data ahci_dwc_plat = {
 	.pflags = AHCI_PLATFORM_GET_RESETS,
 };
 
-static struct ahci_dwc_plat_data ahci_bt1_plat = {
-	.pflags = AHCI_PLATFORM_GET_RESETS | AHCI_PLATFORM_RST_TRIGGER,
-	.init = ahci_bt1_init,
-};
-
 static const struct of_device_id ahci_dwc_of_match[] = {
 	{ .compatible = "snps,dwc-ahci", &ahci_dwc_plat },
 	{ .compatible = "snps,spear-ahci", &ahci_dwc_plat },
-	{ .compatible = "baikal,bt1-ahci", &ahci_bt1_plat },
 	{},
 };
 MODULE_DEVICE_TABLE(of, ahci_dwc_of_match);
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 6c4e567b6582f..81479ddb83b18 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -4188,6 +4188,9 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
 	{ "ST3320[68]13AS",	"SD1[5-9]",	ATA_QUIRK_NONCQ |
 						ATA_QUIRK_FIRMWARE_WARN },
 
+	/* ADATA devices with LPM issues. */
+	{ "ADATA SU680",	NULL,		ATA_QUIRK_NOLPM },
+
 	/* Seagate disks with LPM issues */
 	{ "ST1000DM010-2EP102",	NULL,		ATA_QUIRK_NOLPM },
 	{ "ST2000DM008-2FR102",	NULL,		ATA_QUIRK_NOLPM },
@@ -5148,8 +5151,13 @@ void ata_qc_issue(struct ata_queued_cmd *qc)
 	struct ata_link *link = qc->dev->link;
 	u8 prot = qc->tf.protocol;
 
-	/* Make sure only one non-NCQ command is outstanding. */
-	WARN_ON_ONCE(ata_tag_valid(link->active_tag));
+	/*
+	 * Make sure we have a valid tag and that only one non-NCQ command is
+	 * outstanding.
+	 */
+	if (WARN_ON_ONCE(!ata_tag_valid(qc->tag)) ||
+	    WARN_ON_ONCE(ata_tag_valid(link->active_tag)))
+		goto sys_err;
 
 	if (ata_is_ncq(prot)) {
 		WARN_ON_ONCE(link->sactive & (1 << qc->hw_tag));
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 23be85418b3b1..bb0813ea8b180 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -560,21 +560,27 @@ void ata_scsi_error(struct Scsi_Host *host)
 {
 	struct ata_port *ap = ata_shost_to_port(host);
 	unsigned long flags;
+	int nr_timedout;
 	LIST_HEAD(eh_work_q);
 
 	spin_lock_irqsave(host->host_lock, flags);
 	list_splice_init(&host->eh_cmd_q, &eh_work_q);
 	spin_unlock_irqrestore(host->host_lock, flags);
 
-	ata_scsi_cmd_error_handler(host, ap, &eh_work_q);
-
-	/* If we timed raced normal completion and there is nothing to
-	   recover nr_timedout == 0 why exactly are we doing error recovery ? */
-	ata_scsi_port_error_handler(host, ap);
+	/*
+	 * First check what errors we got with ata_scsi_cmd_error_handler().
+	 * If we had no command timeouts and EH is not scheduled for this port,
+	 * meaning that we do not have any failed command, then there is no
+	 * need to go through the full port error handling. We only need to
+	 * flush the completed commands we have.
+	 */
+	nr_timedout = ata_scsi_cmd_error_handler(host, ap, &eh_work_q);
+	if (nr_timedout || ata_port_eh_scheduled(ap))
+		ata_scsi_port_error_handler(host, ap);
+	else
+		scsi_eh_flush_done_q(&ap->eh_done_q);
 
-	/* finish or retry handled scmd's and clean up */
 	WARN_ON(!list_empty(&eh_work_q));
-
 }
 
 /**
@@ -586,9 +592,11 @@ void ata_scsi_error(struct Scsi_Host *host)
  * process the given list of commands and return those finished to the
  * ap->eh_done_q.  This function is the first part of the libata error
  * handler which processes a given list of failed commands.
+ *
+ * Return the number of commands that timed out.
  */
-void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
-				struct list_head *eh_work_q)
+int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
+			       struct list_head *eh_work_q)
 {
 	int i;
 	unsigned long flags;
@@ -695,6 +703,8 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
 	ap->eh_tries = ATA_EH_MAX_TRIES;
 
 	spin_unlock_irqrestore(ap->lock, flags);
+
+	return nr_timedout;
 }
 EXPORT_SYMBOL(ata_scsi_cmd_error_handler);
 
diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c
index b9d635088f5fb..2ee54d60ea4b7 100644
--- a/drivers/ata/libata-sata.c
+++ b/drivers/ata/libata-sata.c
@@ -1378,15 +1378,13 @@ EXPORT_SYMBOL_GPL(ata_sas_sdev_configure);
 
 int ata_sas_queuecmd(struct scsi_cmnd *cmd, struct ata_port *ap)
 {
-	int rc = 0;
-
 	if (likely(ata_dev_enabled(ap->link.device)))
-		rc = __ata_scsi_queuecmd(cmd, ap->link.device);
-	else {
-		cmd->result = (DID_BAD_TARGET << 16);
-		scsi_done(cmd);
-	}
-	return rc;
+		return __ata_scsi_queuecmd(cmd, ap->link.device);
+
+	cmd->result = (DID_BAD_TARGET << 16);
+	scsi_done(cmd);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(ata_sas_queuecmd);
 
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index ad798e5246b49..00b3ffbfe169a 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1649,12 +1649,16 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 	return 1;
 }
 
-static void ata_qc_done(struct ata_queued_cmd *qc)
+static void ata_scsi_qc_done(struct ata_queued_cmd *qc, bool set_result,
+			     u32 scmd_result)
 {
 	struct scsi_cmnd *cmd = qc->scsicmd;
 	void (*done)(struct scsi_cmnd *) = qc->scsidone;
 
 	ata_qc_free(qc);
+
+	if (set_result)
+		cmd->result = scmd_result;
 	done(cmd);
 }
 
@@ -1685,7 +1689,6 @@ void ata_scsi_deferred_qc_work(struct work_struct *work)
 void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
 {
 	struct ata_queued_cmd *qc = ap->deferred_qc;
-	struct scsi_cmnd *scmd;
 
 	lockdep_assert_held(ap->lock);
 
@@ -1694,15 +1697,11 @@ void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
 	 * do not try to be smart about what to do with this deferred command
 	 * and simply retry it by completing it with DID_SOFT_ERROR.
 	 */
-	if (!qc)
-		return;
-
-	scmd = qc->scsicmd;
-	ap->deferred_qc = NULL;
-	cancel_work(&ap->deferred_qc_work);
-	ata_qc_free(qc);
-	scmd->result = (DID_SOFT_ERROR << 16);
-	scsi_done(scmd);
+	if (qc) {
+		ap->deferred_qc = NULL;
+		cancel_work(&ap->deferred_qc_work);
+		ata_scsi_qc_done(qc, true, DID_SOFT_ERROR << 16);
+	}
 }
 
 static void ata_scsi_schedule_deferred_qc(struct ata_port *ap)
@@ -1758,7 +1757,7 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc)
 		ata_scsi_set_sense_information(qc);
 	}
 
-	ata_qc_done(qc);
+	ata_scsi_qc_done(qc, false, 0);
 
 	ata_scsi_schedule_deferred_qc(ap);
 }
@@ -2917,17 +2916,15 @@ static void atapi_qc_complete(struct ata_queued_cmd *qc)
 		if (qc->cdb[0] == ALLOW_MEDIUM_REMOVAL && qc->dev->sdev)
 			qc->dev->sdev->locked = 0;
 
-		qc->scsicmd->result = SAM_STAT_CHECK_CONDITION;
-		ata_qc_done(qc);
+		ata_scsi_qc_done(qc, true, SAM_STAT_CHECK_CONDITION);
 		return;
 	}
 
 	/* successful completion path */
 	if (cmd->cmnd[0] == INQUIRY && (cmd->cmnd[1] & 0x03) == 0)
 		atapi_fixup_inquiry(cmd);
-	cmd->result = SAM_STAT_GOOD;
 
-	ata_qc_done(qc);
+	ata_scsi_qc_done(qc, true, SAM_STAT_GOOD);
 }
 /**
  *	atapi_xlat - Initialize PACKET taskfile
@@ -3600,7 +3597,7 @@ static unsigned int ata_scsiop_maint_in(struct ata_device *dev,
 
 	if (cdb[2] != 1 && cdb[2] != 3) {
 		ata_dev_warn(dev, "invalid command format %d\n", cdb[2]);
-		ata_scsi_set_invalid_field(dev, cmd, 1, 0xff);
+		ata_scsi_set_invalid_field(dev, cmd, 2, 0xff);
 		return 0;
 	}
 
@@ -4424,6 +4421,79 @@ static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev, u8 cmd)
 	return NULL;
 }
 
+/**
+ *	ata_scsi_simulate - simulate SCSI command on ATA device
+ *	@dev: the target device
+ *	@cmd: SCSI command being sent to device.
+ *
+ *	Interprets and directly executes a select list of SCSI commands
+ *	that can be handled internally.
+ *
+ *	LOCKING:
+ *	spin_lock_irqsave(host lock)
+ */
+static void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
+{
+	const u8 *scsicmd = cmd->cmnd;
+	u8 tmp8;
+
+	switch (scsicmd[0]) {
+	case INQUIRY:
+		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_inquiry);
+		break;
+
+	case MODE_SENSE:
+	case MODE_SENSE_10:
+		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_mode_sense);
+		break;
+
+	case READ_CAPACITY:
+	case SERVICE_ACTION_IN_16:
+		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_read_cap);
+		break;
+
+	case REPORT_LUNS:
+		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_report_luns);
+		break;
+
+	case REQUEST_SENSE:
+		ata_scsi_set_sense(dev, cmd, 0, 0, 0);
+		break;
+
+	/* if we reach this, then writeback caching is disabled,
+	 * turning this into a no-op.
+	 */
+	case SYNCHRONIZE_CACHE:
+	case SYNCHRONIZE_CACHE_16:
+		fallthrough;
+
+	/* no-op's, complete with success */
+	case REZERO_UNIT:
+	case SEEK_6:
+	case SEEK_10:
+	case TEST_UNIT_READY:
+		break;
+
+	case SEND_DIAGNOSTIC:
+		tmp8 = scsicmd[1] & ~(1 << 3);
+		if (tmp8 != 0x4 || scsicmd[3] || scsicmd[4])
+			ata_scsi_set_invalid_field(dev, cmd, 1, 0xff);
+		break;
+
+	case MAINTENANCE_IN:
+		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_maint_in);
+		break;
+
+	/* all other commands */
+	default:
+		ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		/* "Invalid command operation code" */
+		break;
+	}
+
+	scsi_done(cmd);
+}
+
 enum scsi_qc_status __ata_scsi_queuecmd(struct scsi_cmnd *scmd,
 					struct ata_device *dev)
 {
@@ -4526,80 +4596,6 @@ enum scsi_qc_status ata_scsi_queuecmd(struct Scsi_Host *shost,
 }
 EXPORT_SYMBOL_GPL(ata_scsi_queuecmd);
 
-/**
- *	ata_scsi_simulate - simulate SCSI command on ATA device
- *	@dev: the target device
- *	@cmd: SCSI command being sent to device.
- *
- *	Interprets and directly executes a select list of SCSI commands
- *	that can be handled internally.
- *
- *	LOCKING:
- *	spin_lock_irqsave(host lock)
- */
-
-void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
-{
-	const u8 *scsicmd = cmd->cmnd;
-	u8 tmp8;
-
-	switch(scsicmd[0]) {
-	case INQUIRY:
-		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_inquiry);
-		break;
-
-	case MODE_SENSE:
-	case MODE_SENSE_10:
-		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_mode_sense);
-		break;
-
-	case READ_CAPACITY:
-	case SERVICE_ACTION_IN_16:
-		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_read_cap);
-		break;
-
-	case REPORT_LUNS:
-		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_report_luns);
-		break;
-
-	case REQUEST_SENSE:
-		ata_scsi_set_sense(dev, cmd, 0, 0, 0);
-		break;
-
-	/* if we reach this, then writeback caching is disabled,
-	 * turning this into a no-op.
-	 */
-	case SYNCHRONIZE_CACHE:
-	case SYNCHRONIZE_CACHE_16:
-		fallthrough;
-
-	/* no-op's, complete with success */
-	case REZERO_UNIT:
-	case SEEK_6:
-	case SEEK_10:
-	case TEST_UNIT_READY:
-		break;
-
-	case SEND_DIAGNOSTIC:
-		tmp8 = scsicmd[1] & ~(1 << 3);
-		if (tmp8 != 0x4 || scsicmd[3] || scsicmd[4])
-			ata_scsi_set_invalid_field(dev, cmd, 1, 0xff);
-		break;
-
-	case MAINTENANCE_IN:
-		ata_scsi_rbuf_fill(dev, cmd, ata_scsiop_maint_in);
-		break;
-
-	/* all other commands */
-	default:
-		ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x20, 0x0);
-		/* "Invalid command operation code" */
-		break;
-	}
-
-	scsi_done(cmd);
-}
-
 int ata_scsi_add_hosts(struct ata_host *host, const struct scsi_host_template *sht)
 {
 	int i, rc;
diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c
index defb0573e43c6..c7a9627287524 100644
--- a/drivers/auxdisplay/lcd2s.c
+++ b/drivers/auxdisplay/lcd2s.c
@@ -99,8 +99,13 @@ static int lcd2s_print(struct charlcd *lcd, int c)
 {
 	struct lcd2s_data *lcd2s = lcd->drvdata;
 	u8 buf[2] = { LCD2S_CMD_WRITE, c };
+	int ret;
 
-	lcd2s_i2c_master_send(lcd2s->i2c, buf, sizeof(buf));
+	ret = lcd2s_i2c_master_send(lcd2s->i2c, buf, sizeof(buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(buf))
+		return -EIO;
 	return 0;
 }
 
@@ -108,9 +113,13 @@ static int lcd2s_gotoxy(struct charlcd *lcd, unsigned int x, unsigned int y)
 {
 	struct lcd2s_data *lcd2s = lcd->drvdata;
 	u8 buf[3] = { LCD2S_CMD_CUR_POS, y + 1, x + 1 };
+	int ret;
 
-	lcd2s_i2c_master_send(lcd2s->i2c, buf, sizeof(buf));
-
+	ret = lcd2s_i2c_master_send(lcd2s->i2c, buf, sizeof(buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(buf))
+		return -EIO;
 	return 0;
 }
 
diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c
index 9fd3820d1f8a2..11949d6bcda46 100644
--- a/drivers/base/auxiliary.c
+++ b/drivers/base/auxiliary.c
@@ -502,6 +502,16 @@ struct auxiliary_device *__devm_auxiliary_device_create(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(__devm_auxiliary_device_create);
 
+/**
+ * dev_is_auxiliary - check if the device is an auxiliary one
+ * @dev: device to check
+ */
+bool dev_is_auxiliary(struct device *dev)
+{
+	return dev->bus == &auxiliary_bus_type;
+}
+EXPORT_SYMBOL_GPL(dev_is_auxiliary);
+
 void __init auxiliary_bus_init(void)
 {
 	WARN_ON(bus_register(&auxiliary_bus_type));
diff --git a/drivers/base/base.h b/drivers/base/base.h
index 1af95ac68b778..afeda89fd4710 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -13,27 +13,28 @@
 #include <linux/notifier.h>
 
 /**
- * struct subsys_private - structure to hold the private to the driver core portions of the bus_type/class structure.
- *
- * @subsys - the struct kset that defines this subsystem
- * @devices_kset - the subsystem's 'devices' directory
- * @interfaces - list of subsystem interfaces associated
- * @mutex - protect the devices, and interfaces lists.
- *
- * @drivers_kset - the list of drivers associated
- * @klist_devices - the klist to iterate over the @devices_kset
- * @klist_drivers - the klist to iterate over the @drivers_kset
- * @bus_notifier - the bus notifier list for anything that cares about things
- *                 on this bus.
- * @bus - pointer back to the struct bus_type that this structure is associated
- *        with.
+ * struct subsys_private - structure to hold the private to the driver core
+ *			   portions of the bus_type/class structure.
+ * @subsys: the struct kset that defines this subsystem
+ * @devices_kset: the subsystem's 'devices' directory
+ * @interfaces: list of subsystem interfaces associated
+ * @mutex: protect the devices, and interfaces lists.
+ * @drivers_kset: the list of drivers associated
+ * @klist_devices: the klist to iterate over the @devices_kset
+ * @klist_drivers: the klist to iterate over the @drivers_kset
+ * @bus_notifier: the bus notifier list for anything that cares about things
+ *		  on this bus.
+ * @drivers_autoprobe: gate whether new devices are automatically attached to
+ *		       registered drivers, or new drivers automatically attach
+ *		       to existing devices.
+ * @bus: pointer back to the struct bus_type that this structure is associated
+ *	 with.
  * @dev_root: Default device to use as the parent.
- *
- * @glue_dirs - "glue" directory to put in-between the parent device to
- *              avoid namespace conflicts
- * @class - pointer back to the struct class that this structure is associated
- *          with.
- * @lock_key:	Lock class key for use by the lock validator
+ * @glue_dirs: "glue" directory to put in-between the parent device to
+ *	       avoid namespace conflicts
+ * @class: pointer back to the struct class that this structure is associated
+ *	   with.
+ * @lock_key: Lock class key for use by the lock validator
  *
  * This structure is the one that is the actual kobject allowing struct
  * bus_type/class to be statically allocated safely.  Nothing outside of the
@@ -98,24 +99,26 @@ struct driver_type {
 #endif
 
 /**
- * struct device_private - structure to hold the private to the driver core portions of the device structure.
- *
- * @klist_children - klist containing all children of this device
- * @knode_parent - node in sibling list
- * @knode_driver - node in driver list
- * @knode_bus - node in bus list
- * @knode_class - node in class list
- * @deferred_probe - entry in deferred_probe_list which is used to retry the
- *	binding of drivers which were unable to get all the resources needed by
- *	the device; typically because it depends on another driver getting
- *	probed first.
- * @async_driver - pointer to device driver awaiting probe via async_probe
- * @device - pointer back to the struct device that this structure is
- * associated with.
- * @driver_type - The type of the bound Rust driver.
- * @dead - This device is currently either in the process of or has been
- *	removed from the system. Any asynchronous events scheduled for this
- *	device should exit without taking any action.
+ * struct device_private - structure to hold the private to the driver core
+ *			   portions of the device structure.
+ * @klist_children: klist containing all children of this device
+ * @knode_parent: node in sibling list
+ * @knode_driver: node in driver list
+ * @knode_bus: node in bus list
+ * @knode_class: node in class list
+ * @deferred_probe: entry in deferred_probe_list which is used to retry the
+ *		    binding of drivers which were unable to get all the
+ *		    resources needed by the device; typically because it depends
+ *		    on another driver getting probed first.
+ * @async_driver: pointer to device driver awaiting probe via async_probe
+ * @deferred_probe_reason: capture the -EPROBE_DEFER message emitted with
+ *			   dev_err_probe() for later retrieval via debugfs
+ * @device: pointer back to the struct device that this structure is
+ *	    associated with.
+ * @driver_type: The type of the bound Rust driver.
+ * @dead: This device is currently either in the process of or has been
+ *	  removed from the system. Any asynchronous events scheduled for this
+ *	  device should exit without taking any action.
  *
  * Nothing outside of the driver core should ever touch these fields.
  */
@@ -213,6 +216,24 @@ static inline void device_set_driver(struct device *dev, const struct device_dri
 	WRITE_ONCE(dev->driver, (struct device_driver *)drv);
 }
 
+struct devres_node;
+typedef void (*dr_node_release_t)(struct device *dev, struct devres_node *node);
+typedef void (*dr_node_free_t)(struct devres_node *node);
+
+struct devres_node {
+	struct list_head		entry;
+	dr_node_release_t		release;
+	dr_node_free_t			free_node;
+	const char			*name;
+	size_t				size;
+};
+
+void devres_node_init(struct devres_node *node, dr_node_release_t release,
+		      dr_node_free_t free_node);
+void devres_node_add(struct device *dev, struct devres_node *node);
+bool devres_node_remove(struct device *dev, struct devres_node *node);
+void devres_set_node_dbginfo(struct devres_node *node, const char *name,
+			     size_t size);
 void devres_for_each_res(struct device *dev, dr_release_t release,
 			 dr_match_t match, void *match_data,
 			 void (*fn)(struct device *, void *, void *),
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index bb61d8adbab12..8b6722ff8590d 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -504,6 +504,36 @@ int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start,
 }
 EXPORT_SYMBOL_GPL(bus_for_each_drv);
 
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	int ret;
+
+	ret = __device_set_driver_override(dev, buf, count);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	guard(spinlock)(&dev->driver_override.lock);
+	return sysfs_emit(buf, "%s\n", dev->driver_override.name);
+}
+static DEVICE_ATTR_RW(driver_override);
+
+static struct attribute *driver_override_dev_attrs[] = {
+	&dev_attr_driver_override.attr,
+	NULL,
+};
+
+static const struct attribute_group driver_override_dev_group = {
+	.attrs = driver_override_dev_attrs,
+};
+
 /**
  * bus_add_device - add device to bus
  * @dev: device being added
@@ -537,9 +567,15 @@ int bus_add_device(struct device *dev)
 	if (error)
 		goto out_put;
 
+	if (dev->bus->driver_override) {
+		error = device_add_group(dev, &driver_override_dev_group);
+		if (error)
+			goto out_groups;
+	}
+
 	error = sysfs_create_link(&sp->devices_kset->kobj, &dev->kobj, dev_name(dev));
 	if (error)
-		goto out_groups;
+		goto out_override;
 
 	error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem");
 	if (error)
@@ -550,6 +586,9 @@ int bus_add_device(struct device *dev)
 
 out_subsys:
 	sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev));
+out_override:
+	if (dev->bus->driver_override)
+		device_remove_group(dev, &driver_override_dev_group);
 out_groups:
 	device_remove_groups(dev, sp->bus->dev_groups);
 out_put:
@@ -607,6 +646,8 @@ void bus_remove_device(struct device *dev)
 
 	sysfs_remove_link(&dev->kobj, "subsystem");
 	sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev));
+	if (dev->bus->driver_override)
+		device_remove_group(dev, &driver_override_dev_group);
 	device_remove_groups(dev, dev->bus->dev_groups);
 	if (klist_node_attached(&dev->p->knode_bus))
 		klist_del(&dev->p->knode_bus);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 791f9e444df8d..5933852df8374 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -2556,6 +2556,7 @@ static void device_release(struct kobject *kobj)
 	devres_release_all(dev);
 
 	kfree(dev->dma_range_map);
+	kfree(dev->driver_override.name);
 
 	if (dev->release)
 		dev->release(dev);
@@ -2831,14 +2832,15 @@ static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RO(removable);
 
-int device_add_groups(struct device *dev, const struct attribute_group **groups)
+int device_add_groups(struct device *dev,
+		      const struct attribute_group *const *groups)
 {
 	return sysfs_create_groups(&dev->kobj, groups);
 }
 EXPORT_SYMBOL_GPL(device_add_groups);
 
 void device_remove_groups(struct device *dev,
-			  const struct attribute_group **groups)
+			  const struct attribute_group *const *groups)
 {
 	sysfs_remove_groups(&dev->kobj, groups);
 }
@@ -3159,6 +3161,7 @@ void device_initialize(struct device *dev)
 	kobject_init(&dev->kobj, &device_ktype);
 	INIT_LIST_HEAD(&dev->dma_pools);
 	mutex_init(&dev->mutex);
+	spin_lock_init(&dev->driver_override.lock);
 	lockdep_set_novalidate_class(&dev->mutex);
 	spin_lock_init(&dev->devres_lock);
 	INIT_LIST_HEAD(&dev->devres_head);
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index bea8da5f8a3a9..37c7e54e0e4c7 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -381,6 +381,66 @@ static void __exit deferred_probe_exit(void)
 }
 __exitcall(deferred_probe_exit);
 
+int __device_set_driver_override(struct device *dev, const char *s, size_t len)
+{
+	const char *new, *old;
+	char *cp;
+
+	if (!s)
+		return -EINVAL;
+
+	/*
+	 * The stored value will be used in sysfs show callback (sysfs_emit()),
+	 * which has a length limit of PAGE_SIZE and adds a trailing newline.
+	 * Thus we can store one character less to avoid truncation during sysfs
+	 * show.
+	 */
+	if (len >= (PAGE_SIZE - 1))
+		return -EINVAL;
+
+	/*
+	 * Compute the real length of the string in case userspace sends us a
+	 * bunch of \0 characters like python likes to do.
+	 */
+	len = strlen(s);
+
+	if (!len) {
+		/* Empty string passed - clear override */
+		spin_lock(&dev->driver_override.lock);
+		old = dev->driver_override.name;
+		dev->driver_override.name = NULL;
+		spin_unlock(&dev->driver_override.lock);
+		kfree(old);
+
+		return 0;
+	}
+
+	cp = strnchr(s, len, '\n');
+	if (cp)
+		len = cp - s;
+
+	new = kstrndup(s, len, GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	spin_lock(&dev->driver_override.lock);
+	old = dev->driver_override.name;
+	if (cp != s) {
+		dev->driver_override.name = new;
+		spin_unlock(&dev->driver_override.lock);
+	} else {
+		/* "\n" passed - clear override */
+		dev->driver_override.name = NULL;
+		spin_unlock(&dev->driver_override.lock);
+
+		kfree(new);
+	}
+	kfree(old);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(__device_set_driver_override);
+
 /**
  * device_is_bound() - Check if device is bound to a driver
  * @dev: device to check
diff --git a/drivers/base/devres.c b/drivers/base/devres.c
index 171750c1f6918..9d9842fc5a19f 100644
--- a/drivers/base/devres.c
+++ b/drivers/base/devres.c
@@ -16,15 +16,9 @@
 #include "base.h"
 #include "trace.h"
 
-struct devres_node {
-	struct list_head		entry;
-	dr_release_t			release;
-	const char			*name;
-	size_t				size;
-};
-
 struct devres {
 	struct devres_node		node;
+	dr_release_t			release;
 	/*
 	 * Some archs want to perform DMA into kmalloc caches
 	 * and need a guaranteed alignment larger than
@@ -42,7 +36,21 @@ struct devres_group {
 	/* -- 8 pointers */
 };
 
-static void set_node_dbginfo(struct devres_node *node, const char *name,
+void devres_node_init(struct devres_node *node,
+		      dr_node_release_t release,
+		      dr_node_free_t free_node)
+{
+	INIT_LIST_HEAD(&node->entry);
+	node->release = release;
+	node->free_node = free_node;
+}
+
+static inline void free_node(struct devres_node *node)
+{
+	node->free_node(node);
+}
+
+void devres_set_node_dbginfo(struct devres_node *node, const char *name,
 			     size_t size)
 {
 	node->name = name;
@@ -75,12 +83,12 @@ static void devres_log(struct device *dev, struct devres_node *node,
  * Release functions for devres group.  These callbacks are used only
  * for identification.
  */
-static void group_open_release(struct device *dev, void *res)
+static void group_open_release(struct device *dev, struct devres_node *node)
 {
 	/* noop */
 }
 
-static void group_close_release(struct device *dev, void *res)
+static void group_close_release(struct device *dev, struct devres_node *node)
 {
 	/* noop */
 }
@@ -107,6 +115,20 @@ static bool check_dr_size(size_t size, size_t *tot_size)
 	return true;
 }
 
+static void dr_node_release(struct device *dev, struct devres_node *node)
+{
+	struct devres *dr = container_of(node, struct devres, node);
+
+	dr->release(dev, dr->data);
+}
+
+static void dr_node_free(struct devres_node *node)
+{
+	struct devres *dr = container_of(node, struct devres, node);
+
+	kfree(dr);
+}
+
 static __always_inline struct devres *alloc_dr(dr_release_t release,
 					       size_t size, gfp_t gfp, int nid)
 {
@@ -124,8 +146,8 @@ static __always_inline struct devres *alloc_dr(dr_release_t release,
 	if (!(gfp & __GFP_ZERO))
 		memset(dr, 0, offsetof(struct devres, data));
 
-	INIT_LIST_HEAD(&dr->node.entry);
-	dr->node.release = release;
+	devres_node_init(&dr->node, dr_node_release, dr_node_free);
+	dr->release = release;
 	return dr;
 }
 
@@ -167,7 +189,7 @@ void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp, int nid,
 	dr = alloc_dr(release, size, gfp | __GFP_ZERO, nid);
 	if (unlikely(!dr))
 		return NULL;
-	set_node_dbginfo(&dr->node, name, size);
+	devres_set_node_dbginfo(&dr->node, name, size);
 	return dr->data;
 }
 EXPORT_SYMBOL_GPL(__devres_alloc_node);
@@ -194,26 +216,31 @@ void devres_for_each_res(struct device *dev, dr_release_t release,
 {
 	struct devres_node *node;
 	struct devres_node *tmp;
-	unsigned long flags;
 
 	if (!fn)
 		return;
 
-	spin_lock_irqsave(&dev->devres_lock, flags);
+	guard(spinlock_irqsave)(&dev->devres_lock);
 	list_for_each_entry_safe_reverse(node, tmp,
 			&dev->devres_head, entry) {
 		struct devres *dr = container_of(node, struct devres, node);
 
-		if (node->release != release)
+		if (node->release != dr_node_release)
+			continue;
+		if (dr->release != release)
 			continue;
 		if (match && !match(dev, dr->data, match_data))
 			continue;
 		fn(dev, dr->data, data);
 	}
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
 }
 EXPORT_SYMBOL_GPL(devres_for_each_res);
 
+static inline void free_dr(struct devres *dr)
+{
+	free_node(&dr->node);
+}
+
 /**
  * devres_free - Free device resource data
  * @res: Pointer to devres data to free
@@ -226,11 +253,18 @@ void devres_free(void *res)
 		struct devres *dr = container_of(res, struct devres, data);
 
 		BUG_ON(!list_empty(&dr->node.entry));
-		kfree(dr);
+		free_dr(dr);
 	}
 }
 EXPORT_SYMBOL_GPL(devres_free);
 
+void devres_node_add(struct device *dev, struct devres_node *node)
+{
+	guard(spinlock_irqsave)(&dev->devres_lock);
+
+	add_dr(dev, node);
+}
+
 /**
  * devres_add - Register device resource
  * @dev: Device to add resource to
@@ -243,11 +277,8 @@ EXPORT_SYMBOL_GPL(devres_free);
 void devres_add(struct device *dev, void *res)
 {
 	struct devres *dr = container_of(res, struct devres, data);
-	unsigned long flags;
 
-	spin_lock_irqsave(&dev->devres_lock, flags);
-	add_dr(dev, &dr->node);
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
+	devres_node_add(dev, &dr->node);
 }
 EXPORT_SYMBOL_GPL(devres_add);
 
@@ -259,7 +290,9 @@ static struct devres *find_dr(struct device *dev, dr_release_t release,
 	list_for_each_entry_reverse(node, &dev->devres_head, entry) {
 		struct devres *dr = container_of(node, struct devres, node);
 
-		if (node->release != release)
+		if (node->release != dr_node_release)
+			continue;
+		if (dr->release != release)
 			continue;
 		if (match && !match(dev, dr->data, match_data))
 			continue;
@@ -287,14 +320,12 @@ void *devres_find(struct device *dev, dr_release_t release,
 		  dr_match_t match, void *match_data)
 {
 	struct devres *dr;
-	unsigned long flags;
 
-	spin_lock_irqsave(&dev->devres_lock, flags);
+	guard(spinlock_irqsave)(&dev->devres_lock);
 	dr = find_dr(dev, release, match, match_data);
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
-
 	if (dr)
 		return dr->data;
+
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(devres_find);
@@ -321,7 +352,7 @@ void *devres_get(struct device *dev, void *new_res,
 	unsigned long flags;
 
 	spin_lock_irqsave(&dev->devres_lock, flags);
-	dr = find_dr(dev, new_dr->node.release, match, match_data);
+	dr = find_dr(dev, new_dr->release, match, match_data);
 	if (!dr) {
 		add_dr(dev, &new_dr->node);
 		dr = new_dr;
@@ -334,6 +365,22 @@ void *devres_get(struct device *dev, void *new_res,
 }
 EXPORT_SYMBOL_GPL(devres_get);
 
+bool devres_node_remove(struct device *dev, struct devres_node *node)
+{
+	struct devres_node *__node;
+
+	guard(spinlock_irqsave)(&dev->devres_lock);
+	list_for_each_entry_reverse(__node, &dev->devres_head, entry) {
+		if (__node == node) {
+			list_del_init(&node->entry);
+			devres_log(dev, node, "REM");
+			return true;
+		}
+	}
+
+	return false;
+}
+
 /**
  * devres_remove - Find a device resource and remove it
  * @dev: Device to find resource from
@@ -353,18 +400,15 @@ void *devres_remove(struct device *dev, dr_release_t release,
 		    dr_match_t match, void *match_data)
 {
 	struct devres *dr;
-	unsigned long flags;
 
-	spin_lock_irqsave(&dev->devres_lock, flags);
+	guard(spinlock_irqsave)(&dev->devres_lock);
 	dr = find_dr(dev, release, match, match_data);
 	if (dr) {
 		list_del_init(&dr->node.entry);
 		devres_log(dev, &dr->node, "REM");
+		return dr->data;
 	}
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
 
-	if (dr)
-		return dr->data;
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(devres_remove);
@@ -495,15 +539,12 @@ static int remove_nodes(struct device *dev,
 
 static void release_nodes(struct device *dev, struct list_head *todo)
 {
-	struct devres *dr, *tmp;
+	struct devres_node *node, *tmp;
 
-	/* Release.  Note that both devres and devres_group are
-	 * handled as devres in the following loop.  This is safe.
-	 */
-	list_for_each_entry_safe_reverse(dr, tmp, todo, node.entry) {
-		devres_log(dev, &dr->node, "REL");
-		dr->node.release(dev, dr->data);
-		kfree(dr);
+	list_for_each_entry_safe_reverse(node, tmp, todo, entry) {
+		devres_log(dev, node, "REL");
+		node->release(dev, node);
+		free_node(node);
 	}
 }
 
@@ -536,6 +577,13 @@ int devres_release_all(struct device *dev)
 	return cnt;
 }
 
+static void devres_group_free(struct devres_node *node)
+{
+	struct devres_group *grp = container_of(node, struct devres_group, node[0]);
+
+	kfree(grp);
+}
+
 /**
  * devres_open_group - Open a new devres group
  * @dev: Device to open devres group for
@@ -552,26 +600,21 @@ int devres_release_all(struct device *dev)
 void *devres_open_group(struct device *dev, void *id, gfp_t gfp)
 {
 	struct devres_group *grp;
-	unsigned long flags;
 
 	grp = kmalloc_obj(*grp, gfp);
 	if (unlikely(!grp))
 		return NULL;
 
-	grp->node[0].release = &group_open_release;
-	grp->node[1].release = &group_close_release;
-	INIT_LIST_HEAD(&grp->node[0].entry);
-	INIT_LIST_HEAD(&grp->node[1].entry);
-	set_node_dbginfo(&grp->node[0], "grp<", 0);
-	set_node_dbginfo(&grp->node[1], "grp>", 0);
+	devres_node_init(&grp->node[0], &group_open_release, devres_group_free);
+	devres_node_init(&grp->node[1], &group_close_release, NULL);
+	devres_set_node_dbginfo(&grp->node[0], "grp<", 0);
+	devres_set_node_dbginfo(&grp->node[1], "grp>", 0);
 	grp->id = grp;
 	if (id)
 		grp->id = id;
 	grp->color = 0;
 
-	spin_lock_irqsave(&dev->devres_lock, flags);
-	add_dr(dev, &grp->node[0]);
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
+	devres_node_add(dev, &grp->node[0]);
 	return grp->id;
 }
 EXPORT_SYMBOL_GPL(devres_open_group);
@@ -613,17 +656,13 @@ static struct devres_group *find_group(struct device *dev, void *id)
 void devres_close_group(struct device *dev, void *id)
 {
 	struct devres_group *grp;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev->devres_lock, flags);
 
+	guard(spinlock_irqsave)(&dev->devres_lock);
 	grp = find_group(dev, id);
 	if (grp)
 		add_dr(dev, &grp->node[1]);
 	else
 		WARN_ON(1);
-
-	spin_unlock_irqrestore(&dev->devres_lock, flags);
 }
 EXPORT_SYMBOL_GPL(devres_close_group);
 
@@ -677,7 +716,6 @@ int devres_release_group(struct device *dev, void *id)
 	int cnt = 0;
 
 	spin_lock_irqsave(&dev->devres_lock, flags);
-
 	grp = find_group(dev, id);
 	if (grp) {
 		struct list_head *first = &grp->node[0].entry;
@@ -687,20 +725,18 @@ int devres_release_group(struct device *dev, void *id)
 			end = grp->node[1].entry.next;
 
 		cnt = remove_nodes(dev, first, end, &todo);
-		spin_unlock_irqrestore(&dev->devres_lock, flags);
-
-		release_nodes(dev, &todo);
 	} else if (list_empty(&dev->devres_head)) {
 		/*
 		 * dev is probably dying via devres_release_all(): groups
 		 * have already been removed and are on the process of
 		 * being released - don't touch and don't warn.
 		 */
-		spin_unlock_irqrestore(&dev->devres_lock, flags);
 	} else {
 		WARN_ON(1);
-		spin_unlock_irqrestore(&dev->devres_lock, flags);
 	}
+	spin_unlock_irqrestore(&dev->devres_lock, flags);
+
+	release_nodes(dev, &todo);
 
 	return cnt;
 }
@@ -716,20 +752,29 @@ struct action_devres {
 	void (*action)(void *);
 };
 
-static int devm_action_match(struct device *dev, void *res, void *p)
+struct devres_action {
+	struct devres_node node;
+	struct action_devres action;
+};
+
+static int devm_action_match(struct devres_action *devres, struct action_devres *target)
 {
-	struct action_devres *devres = res;
-	struct action_devres *target = p;
+	return devres->action.action == target->action &&
+	       devres->action.data == target->data;
+}
 
-	return devres->action == target->action &&
-	       devres->data == target->data;
+static void devm_action_release(struct device *dev, struct devres_node *node)
+{
+	struct devres_action *devres = container_of(node, struct devres_action, node);
+
+	devres->action.action(devres->action.data);
 }
 
-static void devm_action_release(struct device *dev, void *res)
+static void devm_action_free(struct devres_node *node)
 {
-	struct action_devres *devres = res;
+	struct devres_action *action = container_of(node, struct devres_action, node);
 
-	devres->action(devres->data);
+	kfree(action);
 }
 
 /**
@@ -744,32 +789,71 @@ static void devm_action_release(struct device *dev, void *res)
  */
 int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name)
 {
-	struct action_devres *devres;
+	struct devres_action *devres;
 
-	devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres),
-				     GFP_KERNEL, NUMA_NO_NODE, name);
+	devres = kzalloc_obj(*devres);
 	if (!devres)
 		return -ENOMEM;
 
-	devres->data = data;
-	devres->action = action;
+	devres_node_init(&devres->node, devm_action_release, devm_action_free);
+	devres_set_node_dbginfo(&devres->node, name, sizeof(*devres));
 
-	devres_add(dev, devres);
+	devres->action.data = data;
+	devres->action.action = action;
+
+	devres_node_add(dev, &devres->node);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__devm_add_action);
 
-bool devm_is_action_added(struct device *dev, void (*action)(void *), void *data)
+static struct devres_action *devres_action_find(struct device *dev,
+						void (*action)(void *),
+						void *data)
 {
-	struct action_devres devres = {
+	struct devres_node *node;
+	struct action_devres target = {
 		.data = data,
 		.action = action,
 	};
 
-	return devres_find(dev, devm_action_release, devm_action_match, &devres);
+	list_for_each_entry_reverse(node, &dev->devres_head, entry) {
+		struct devres_action *dr = container_of(node, struct devres_action, node);
+
+		if (node->release != devm_action_release)
+			continue;
+		if (devm_action_match(dr, &target))
+			return dr;
+	}
+
+	return NULL;
+}
+
+bool devm_is_action_added(struct device *dev, void (*action)(void *), void *data)
+{
+	guard(spinlock_irqsave)(&dev->devres_lock);
+
+	return !!devres_action_find(dev, action, data);
 }
 EXPORT_SYMBOL_GPL(devm_is_action_added);
 
+static struct devres_action *remove_action(struct device *dev,
+					   void (*action)(void *),
+					   void *data)
+{
+	struct devres_action *dr;
+
+	guard(spinlock_irqsave)(&dev->devres_lock);
+
+	dr = devres_action_find(dev, action, data);
+	if (!dr)
+		return ERR_PTR(-ENOENT);
+
+	list_del_init(&dr->node.entry);
+	devres_log(dev, &dr->node, "REM");
+
+	return dr;
+}
+
 /**
  * devm_remove_action_nowarn() - removes previously added custom action
  * @dev: Device that owns the action
@@ -794,13 +878,15 @@ int devm_remove_action_nowarn(struct device *dev,
 			      void (*action)(void *),
 			      void *data)
 {
-	struct action_devres devres = {
-		.data = data,
-		.action = action,
-	};
+	struct devres_action *dr;
 
-	return devres_destroy(dev, devm_action_release, devm_action_match,
-			      &devres);
+	dr = remove_action(dev, action, data);
+	if (IS_ERR(dr))
+		return PTR_ERR(dr);
+
+	kfree(dr);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(devm_remove_action_nowarn);
 
@@ -816,14 +902,15 @@ EXPORT_SYMBOL_GPL(devm_remove_action_nowarn);
  */
 void devm_release_action(struct device *dev, void (*action)(void *), void *data)
 {
-	struct action_devres devres = {
-		.data = data,
-		.action = action,
-	};
+	struct devres_action *dr;
 
-	WARN_ON(devres_release(dev, devm_action_release, devm_action_match,
-			       &devres));
+	dr = remove_action(dev, action, data);
+	if (WARN_ON(IS_ERR(dr)))
+		return;
+
+	dr->action.action(dr->action.data);
 
+	kfree(dr);
 }
 EXPORT_SYMBOL_GPL(devm_release_action);
 
@@ -869,7 +956,7 @@ void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp)
 	 * This is named devm_kzalloc_release for historical reasons
 	 * The initial implementation did not support kmalloc, only kzalloc
 	 */
-	set_node_dbginfo(&dr->node, "devm_kzalloc_release", size);
+	devres_set_node_dbginfo(&dr->node, "devm_kzalloc_release", size);
 	devres_add(dev, dr->data);
 	return dr->data;
 }
@@ -940,6 +1027,8 @@ void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp)
 	if (!new_dr)
 		return NULL;
 
+	devres_set_node_dbginfo(&new_dr->node, "devm_krealloc_release", new_size);
+
 	/*
 	 * The spinlock protects the linked list against concurrent
 	 * modifications but not the resource itself.
@@ -949,7 +1038,7 @@ void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp)
 	old_dr = find_dr(dev, devm_kmalloc_release, devm_kmalloc_match, ptr);
 	if (!old_dr) {
 		spin_unlock_irqrestore(&dev->devres_lock, flags);
-		kfree(new_dr);
+		free_dr(new_dr);
 		WARN(1, "Memory chunk not managed or managed by a different device.");
 		return NULL;
 	}
@@ -969,7 +1058,7 @@ void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp)
 	 * list. This is also the reason why we must not use devm_kfree() - the
 	 * links are no longer valid.
 	 */
-	kfree(old_dr);
+	free_dr(old_dr);
 
 	return new_dr->data;
 }
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index a3091924918b0..5380050b16b7a 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -452,7 +452,7 @@ static ssize_t phys_device_show(struct device *dev,
 static int print_allowed_zone(char *buf, int len, int nid,
 			      struct memory_group *group,
 			      unsigned long start_pfn, unsigned long nr_pages,
-			      int online_type, struct zone *default_zone)
+			      enum mmop online_type, struct zone *default_zone)
 {
 	struct zone *zone;
 
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 70db08f3ac6fa..69eed058eb208 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -61,10 +61,6 @@ static const struct msi_domain_template platform_msi_template = {
  * parent. The parent domain sets up the new domain. The domain has
  * a fixed size of @nvec. The domain is managed by devres and will
  * be removed when the device is removed.
- *
- * Note: For migration purposes this falls back to the original platform_msi code
- *	 up to the point where all platforms have been converted to the MSI
- *	 parent model.
  */
 int platform_device_msi_init_and_alloc_irqs(struct device *dev, unsigned int nvec,
 					    irq_write_msi_msg_t write_msi_msg)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index b45d41b018ca6..75b4698d0e582 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -75,7 +75,7 @@ struct resource *platform_get_mem_or_io(struct platform_device *dev,
 	for (i = 0; i < dev->num_resources; i++) {
 		struct resource *r = &dev->resource[i];
 
-		if ((resource_type(r) & (IORESOURCE_MEM|IORESOURCE_IO)) && num-- == 0)
+		if ((resource_type(r) & (IORESOURCE_MEM | IORESOURCE_IO)) && num-- == 0)
 			return r;
 	}
 	return NULL;
@@ -97,7 +97,7 @@ EXPORT_SYMBOL_GPL(platform_get_mem_or_io);
  */
 void __iomem *
 devm_platform_get_and_ioremap_resource(struct platform_device *pdev,
-				unsigned int index, struct resource **res)
+				       unsigned int index, struct resource **res)
 {
 	struct resource *r;
 
@@ -172,7 +172,7 @@ static const struct cpumask *get_irq_affinity(struct platform_device *dev,
  * @num:	interrupt number index
  * @affinity:	optional cpumask pointer to get the affinity of a per-cpu interrupt
  *
- * Gets an interupt for a platform device. Device drivers should check the
+ * Gets an interrupt for a platform device. Device drivers should check the
  * return value for errors so as to not pass a negative integer value to
  * the request_irq() APIs. Optional affinity information is provided in the
  * affinity pointer if available, and NULL otherwise.
@@ -603,7 +603,6 @@ static void platform_device_release(struct device *dev)
 	kfree(pa->pdev.dev.platform_data);
 	kfree(pa->pdev.mfd_cell);
 	kfree(pa->pdev.resource);
-	kfree(pa->pdev.driver_override);
 	kfree(pa);
 }
 
@@ -844,12 +843,14 @@ EXPORT_SYMBOL_GPL(platform_device_unregister);
  *
  * Returns &struct platform_device pointer on success, or ERR_PTR() on error.
  */
-struct platform_device *platform_device_register_full(
-		const struct platform_device_info *pdevinfo)
+struct platform_device *platform_device_register_full(const struct platform_device_info *pdevinfo)
 {
 	int ret;
 	struct platform_device *pdev;
 
+	if (pdevinfo->swnode && pdevinfo->properties)
+		return ERR_PTR(-EINVAL);
+
 	pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id);
 	if (!pdev)
 		return ERR_PTR(-ENOMEM);
@@ -865,17 +866,19 @@ struct platform_device *platform_device_register_full(
 		pdev->dev.coherent_dma_mask = pdevinfo->dma_mask;
 	}
 
-	ret = platform_device_add_resources(pdev,
-			pdevinfo->res, pdevinfo->num_res);
+	ret = platform_device_add_resources(pdev, pdevinfo->res, pdevinfo->num_res);
 	if (ret)
 		goto err;
 
-	ret = platform_device_add_data(pdev,
-			pdevinfo->data, pdevinfo->size_data);
+	ret = platform_device_add_data(pdev, pdevinfo->data, pdevinfo->size_data);
 	if (ret)
 		goto err;
 
-	if (pdevinfo->properties) {
+	if (pdevinfo->swnode) {
+		ret = device_add_software_node(&pdev->dev, pdevinfo->swnode);
+		if (ret)
+			goto err;
+	} else if (pdevinfo->properties) {
 		ret = device_create_managed_software_node(&pdev->dev,
 							  pdevinfo->properties, NULL);
 		if (ret)
@@ -899,8 +902,7 @@ EXPORT_SYMBOL_GPL(platform_device_register_full);
  * @drv: platform driver structure
  * @owner: owning module/driver
  */
-int __platform_driver_register(struct platform_driver *drv,
-				struct module *owner)
+int __platform_driver_register(struct platform_driver *drv, struct module *owner)
 {
 	drv->driver.owner = owner;
 	drv->driver.bus = &platform_bus_type;
@@ -952,13 +954,14 @@ static int is_bound_to_driver(struct device *dev, void *driver)
  * a negative error code and with the driver not registered.
  */
 int __init_or_module __platform_driver_probe(struct platform_driver *drv,
-		int (*probe)(struct platform_device *), struct module *module)
+					     int (*probe)(struct platform_device *),
+					     struct module *module)
 {
 	int retval;
 
 	if (drv->driver.probe_type == PROBE_PREFER_ASYNCHRONOUS) {
 		pr_err("%s: drivers registered with %s can not be probed asynchronously\n",
-			 drv->driver.name, __func__);
+		       drv->driver.name, __func__);
 		return -EINVAL;
 	}
 
@@ -1014,11 +1017,11 @@ EXPORT_SYMBOL_GPL(__platform_driver_probe);
  *
  * Returns &struct platform_device pointer on success, or ERR_PTR() on error.
  */
-struct platform_device * __init_or_module __platform_create_bundle(
-			struct platform_driver *driver,
-			int (*probe)(struct platform_device *),
-			struct resource *res, unsigned int n_res,
-			const void *data, size_t size, struct module *module)
+struct platform_device * __init_or_module
+__platform_create_bundle(struct platform_driver *driver,
+			 int (*probe)(struct platform_device *),
+			 struct resource *res, unsigned int n_res,
+			 const void *data, size_t size, struct module *module)
 {
 	struct platform_device *pdev;
 	int error;
@@ -1117,9 +1120,8 @@ void platform_unregister_drivers(struct platform_driver * const *drivers,
 }
 EXPORT_SYMBOL_GPL(platform_unregister_drivers);
 
-static const struct platform_device_id *platform_match_id(
-			const struct platform_device_id *id,
-			struct platform_device *pdev)
+static const struct platform_device_id *
+platform_match_id(const struct platform_device_id *id, struct platform_device *pdev)
 {
 	while (id->name[0]) {
 		if (strcmp(pdev->name, id->name) == 0) {
@@ -1306,48 +1308,18 @@ static ssize_t numa_node_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(numa_node);
 
-static ssize_t driver_override_show(struct device *dev,
-				    struct device_attribute *attr, char *buf)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	ssize_t len;
-
-	device_lock(dev);
-	len = sysfs_emit(buf, "%s\n", pdev->driver_override);
-	device_unlock(dev);
-
-	return len;
-}
-
-static ssize_t driver_override_store(struct device *dev,
-				     struct device_attribute *attr,
-				     const char *buf, size_t count)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	int ret;
-
-	ret = driver_set_override(dev, &pdev->driver_override, buf, count);
-	if (ret)
-		return ret;
-
-	return count;
-}
-static DEVICE_ATTR_RW(driver_override);
-
 static struct attribute *platform_dev_attrs[] = {
 	&dev_attr_modalias.attr,
 	&dev_attr_numa_node.attr,
-	&dev_attr_driver_override.attr,
 	NULL,
 };
 
-static umode_t platform_dev_attrs_visible(struct kobject *kobj, struct attribute *a,
-		int n)
+static umode_t platform_dev_attrs_visible(struct kobject *kobj,
+					  struct attribute *a, int n)
 {
 	struct device *dev = container_of(kobj, typeof(*dev), kobj);
 
-	if (a == &dev_attr_numa_node.attr &&
-			dev_to_node(dev) == NUMA_NO_NODE)
+	if (a == &dev_attr_numa_node.attr && dev_to_node(dev) == NUMA_NO_NODE)
 		return 0;
 
 	return a->mode;
@@ -1359,7 +1331,6 @@ static const struct attribute_group platform_dev_group = {
 };
 __ATTRIBUTE_GROUPS(platform_dev);
 
-
 /**
  * platform_match - bind platform device to platform driver.
  * @dev: device.
@@ -1377,10 +1348,12 @@ static int platform_match(struct device *dev, const struct device_driver *drv)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct platform_driver *pdrv = to_platform_driver(drv);
+	int ret;
 
 	/* When driver_override is set, only bind to the matching driver */
-	if (pdev->driver_override)
-		return !strcmp(pdev->driver_override, drv->name);
+	ret = device_match_driver_override(dev, drv);
+	if (ret >= 0)
+		return ret;
 
 	/* Attempt an OF style match first */
 	if (of_driver_match_device(dev, drv))
@@ -1412,8 +1385,7 @@ static int platform_uevent(const struct device *dev, struct kobj_uevent_env *env
 	if (rc != -ENODEV)
 		return rc;
 
-	add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX,
-			pdev->name);
+	add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX, pdev->name);
 	return 0;
 }
 
@@ -1516,6 +1488,7 @@ static const struct dev_pm_ops platform_dev_pm_ops = {
 const struct bus_type platform_bus_type = {
 	.name		= "platform",
 	.dev_groups	= platform_dev_groups,
+	.driver_override = true,
 	.match		= platform_match,
 	.uevent		= platform_uevent,
 	.probe		= platform_probe,
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 0ee8ea971aa46..335288e8b5b31 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -1895,6 +1895,7 @@ void pm_runtime_reinit(struct device *dev)
 void pm_runtime_remove(struct device *dev)
 {
 	__pm_runtime_disable(dev, false);
+	flush_work(&dev->power.work);
 	pm_runtime_reinit(dev);
 }
 
diff --git a/drivers/base/regmap/Kconfig b/drivers/base/regmap/Kconfig
index 06d7eb2aac145..3c76a5bf67b7e 100644
--- a/drivers/base/regmap/Kconfig
+++ b/drivers/base/regmap/Kconfig
@@ -56,7 +56,7 @@ config REGMAP_W1
 
 config REGMAP_MDIO
 	tristate
-	select MDIO_BUS
+	depends on PHYLIB
 
 config REGMAP_MMIO
 	tristate
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index 5bf9931654387..c77f3a49a89e2 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -162,7 +162,7 @@ struct regmap {
 	bool no_sync_defaults;
 
 	struct reg_sequence *patch;
-	int patch_regs;
+	unsigned int patch_regs;
 
 	/* if set, the regmap core can sleep */
 	bool can_sleep;
diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c
index a35f2b20298b2..27616b05111cc 100644
--- a/drivers/base/regmap/regcache.c
+++ b/drivers/base/regmap/regcache.c
@@ -42,33 +42,25 @@ void regcache_sort_defaults(struct reg_default *defaults, unsigned int ndefaults
 }
 EXPORT_SYMBOL_GPL(regcache_sort_defaults);
 
-static int regcache_hw_init(struct regmap *map)
+static int regcache_count_cacheable_registers(struct regmap *map)
 {
-	int i, j;
-	int ret;
-	int count;
-	unsigned int reg, val;
-	void *tmp_buf;
-
-	if (!map->num_reg_defaults_raw)
-		return -EINVAL;
+	unsigned int count;
 
 	/* calculate the size of reg_defaults */
-	for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++)
+	count = 0;
+	for (unsigned int i = 0; i < map->num_reg_defaults_raw; i++)
 		if (regmap_readable(map, i * map->reg_stride) &&
 		    !regmap_volatile(map, i * map->reg_stride))
 			count++;
 
-	/* all registers are unreadable or volatile, so just bypass */
-	if (!count) {
-		map->cache_bypass = true;
-		return 0;
-	}
+	return count;
+}
 
-	map->num_reg_defaults = count;
-	map->reg_defaults = kmalloc_objs(struct reg_default, count);
-	if (!map->reg_defaults)
-		return -ENOMEM;
+static int regcache_hw_init(struct regmap *map)
+{
+	int ret;
+	unsigned int reg, val;
+	void *tmp_buf;
 
 	if (!map->reg_defaults_raw) {
 		bool cache_bypass = map->cache_bypass;
@@ -77,10 +69,8 @@ static int regcache_hw_init(struct regmap *map)
 		/* Bypass the cache access till data read from HW */
 		map->cache_bypass = true;
 		tmp_buf = kmalloc(map->cache_size_raw, GFP_KERNEL);
-		if (!tmp_buf) {
-			ret = -ENOMEM;
-			goto err_free;
-		}
+		if (!tmp_buf)
+			return -ENOMEM;
 		ret = regmap_raw_read(map, 0, tmp_buf,
 				      map->cache_size_raw);
 		map->cache_bypass = cache_bypass;
@@ -93,7 +83,7 @@ static int regcache_hw_init(struct regmap *map)
 	}
 
 	/* fill the reg_defaults */
-	for (i = 0, j = 0; i < map->num_reg_defaults_raw; i++) {
+	for (unsigned int i = 0, j = 0; i < map->num_reg_defaults_raw; i++) {
 		reg = i * map->reg_stride;
 
 		if (!regmap_readable(map, reg))
@@ -111,9 +101,9 @@ static int regcache_hw_init(struct regmap *map)
 			ret = regmap_read(map, reg, &val);
 			map->cache_bypass = cache_bypass;
 			if (ret != 0) {
-				dev_err(map->dev, "Failed to read %d: %d\n",
+				dev_err(map->dev, "Failed to read %x: %d\n",
 					reg, ret);
-				goto err_free;
+				return ret;
 			}
 		}
 
@@ -123,15 +113,17 @@ static int regcache_hw_init(struct regmap *map)
 	}
 
 	return 0;
+}
 
-err_free:
-	kfree(map->reg_defaults);
-
-	return ret;
+static void regcache_hw_exit(struct regmap *map)
+{
+	if (map->cache_free)
+		kfree(map->reg_defaults_raw);
 }
 
 int regcache_init(struct regmap *map, const struct regmap_config *config)
 {
+	int count = 0;
 	int ret;
 	int i;
 	void *tmp_buf;
@@ -196,15 +188,18 @@ int regcache_init(struct regmap *map, const struct regmap_config *config)
 			return -ENOMEM;
 		map->reg_defaults = tmp_buf;
 	} else if (map->num_reg_defaults_raw) {
-		/* Some devices such as PMICs don't have cache defaults,
-		 * we cope with this by reading back the HW registers and
-		 * crafting the cache defaults by hand.
-		 */
-		ret = regcache_hw_init(map);
-		if (ret < 0)
-			return ret;
+		count = regcache_count_cacheable_registers(map);
+		if (!count)
+			map->cache_bypass = true;
+
+		/* All registers are unreadable or volatile, so just bypass */
 		if (map->cache_bypass)
 			return 0;
+
+		map->num_reg_defaults = count;
+		map->reg_defaults = kmalloc_objs(struct reg_default, count);
+		if (!map->reg_defaults)
+			return -ENOMEM;
 	}
 
 	if (!map->max_register_is_set && map->num_reg_defaults_raw) {
@@ -219,7 +214,18 @@ int regcache_init(struct regmap *map, const struct regmap_config *config)
 		ret = map->cache_ops->init(map);
 		map->unlock(map->lock_arg);
 		if (ret)
-			goto err_free;
+			goto err_free_reg_defaults;
+	}
+
+	/*
+	 * Some devices such as PMICs don't have cache defaults,
+	 * we cope with this by reading back the HW registers and
+	 * crafting the cache defaults by hand.
+	 */
+	if (count) {
+		ret = regcache_hw_init(map);
+		if (ret)
+			goto err_exit;
 	}
 
 	if (map->cache_ops->populate &&
@@ -229,10 +235,12 @@ int regcache_init(struct regmap *map, const struct regmap_config *config)
 		ret = map->cache_ops->populate(map);
 		map->unlock(map->lock_arg);
 		if (ret)
-			goto err_exit;
+			goto err_free;
 	}
 	return 0;
 
+err_free:
+	regcache_hw_exit(map);
 err_exit:
 	if (map->cache_ops->exit) {
 		dev_dbg(map->dev, "Destroying %s cache\n", map->cache_ops->name);
@@ -240,10 +248,8 @@ int regcache_init(struct regmap *map, const struct regmap_config *config)
 		ret = map->cache_ops->exit(map);
 		map->unlock(map->lock_arg);
 	}
-err_free:
+err_free_reg_defaults:
 	kfree(map->reg_defaults);
-	if (map->cache_free)
-		kfree(map->reg_defaults_raw);
 
 	return ret;
 }
@@ -255,9 +261,7 @@ void regcache_exit(struct regmap *map)
 
 	BUG_ON(!map->cache_ops);
 
-	kfree(map->reg_defaults);
-	if (map->cache_free)
-		kfree(map->reg_defaults_raw);
+	regcache_hw_exit(map);
 
 	if (map->cache_ops->exit) {
 		dev_dbg(map->dev, "Destroying %s cache\n",
@@ -266,6 +270,8 @@ void regcache_exit(struct regmap *map)
 		map->cache_ops->exit(map);
 		map->unlock(map->lock_arg);
 	}
+
+	kfree(map->reg_defaults);
 }
 
 /**
@@ -504,7 +510,7 @@ int regcache_sync_region(struct regmap *map, unsigned int min,
 	bypass = map->cache_bypass;
 
 	name = map->cache_ops->name;
-	dev_dbg(map->dev, "Syncing %s cache from %d-%d\n", name, min, max);
+	dev_dbg(map->dev, "Syncing %s cache from %#x-%#x\n", name, min, max);
 
 	trace_regcache_sync(map, name, "start region");
 
@@ -835,13 +841,13 @@ static int regcache_sync_block_raw(struct regmap *map, void *block,
 			    unsigned int block_base, unsigned int start,
 			    unsigned int end)
 {
-	unsigned int i, val;
 	unsigned int regtmp = 0;
 	unsigned int base = 0;
 	const void *data = NULL;
+	unsigned int val;
 	int ret;
 
-	for (i = start; i < end; i++) {
+	for (unsigned int i = start; i < end; i++) {
 		regtmp = block_base + (i * map->reg_stride);
 
 		if (!regcache_reg_present(cache_present, i) ||
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index 607c1246d994a..b2b26f07f4e3f 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -1182,9 +1182,9 @@ struct regmap *__regmap_init(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(__regmap_init);
 
-static void devm_regmap_release(struct device *dev, void *res)
+static void devm_regmap_release(void *regmap)
 {
-	regmap_exit(*(struct regmap **)res);
+	regmap_exit(regmap);
 }
 
 struct regmap *__devm_regmap_init(struct device *dev,
@@ -1194,20 +1194,17 @@ struct regmap *__devm_regmap_init(struct device *dev,
 				  struct lock_class_key *lock_key,
 				  const char *lock_name)
 {
-	struct regmap **ptr, *regmap;
-
-	ptr = devres_alloc(devm_regmap_release, sizeof(*ptr), GFP_KERNEL);
-	if (!ptr)
-		return ERR_PTR(-ENOMEM);
+	struct regmap *regmap;
+	int ret;
 
 	regmap = __regmap_init(dev, bus, bus_context, config,
 			       lock_key, lock_name);
-	if (!IS_ERR(regmap)) {
-		*ptr = regmap;
-		devres_add(dev, ptr);
-	} else {
-		devres_free(ptr);
-	}
+	if (IS_ERR(regmap))
+		return regmap;
+
+	ret = devm_add_action_or_reset(dev, devm_regmap_release, regmap);
+	if (ret)
+		return ERR_PTR(ret);
 
 	return regmap;
 }
@@ -1545,6 +1542,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg,
 			       unsigned int val_num)
 {
 	void *orig_work_buf;
+	unsigned int selector_reg;
 	unsigned int win_offset;
 	unsigned int win_page;
 	bool page_chg;
@@ -1563,10 +1561,31 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg,
 			return -EINVAL;
 	}
 
-	/* It is possible to have selector register inside data window.
-	   In that case, selector register is located on every page and
-	   it needs no page switching, when accessed alone. */
+	/*
+	 * Calculate the address of the selector register in the corresponding
+	 * data window if it is located on every page.
+	 */
+	page_chg = in_range(range->selector_reg, range->window_start, range->window_len);
+	if (page_chg)
+		selector_reg = range->range_min + win_page * range->window_len +
+			       range->selector_reg - range->window_start;
+
+	/*
+	 * It is possible to have selector register inside data window.
+	 * In that case, selector register is located on every page and it
+	 * needs no page switching, when accessed alone.
+	 *
+	 * Nevertheless we should synchronize the cache values for it.
+	 * This can't be properly achieved if the selector register is
+	 * the first and the only one to be read inside the data window.
+	 * That's why we update it in that case as well.
+	 *
+	 * However, we specifically avoid updating it for the default page,
+	 * when it's overlapped with the real data window, to prevent from
+	 * infinite looping.
+	 */
 	if (val_num > 1 ||
+	    (page_chg && selector_reg != range->selector_reg) ||
 	    range->window_start + win_offset != range->selector_reg) {
 		/* Use separate work_buf during page switching */
 		orig_work_buf = map->work_buf;
@@ -1575,7 +1594,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg,
 		ret = _regmap_update_bits(map, range->selector_reg,
 					  range->selector_mask,
 					  win_page << range->selector_shift,
-					  &page_chg, false);
+					  NULL, false);
 
 		map->work_buf = orig_work_buf;
 
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index c8d3db9daa2f0..65ce72d492303 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -5,16 +5,16 @@
  * Author: Lee Jones <lee.jones@linaro.org> for ST-Ericsson.
  */
 
-#include <linux/sysfs.h>
+#include <linux/err.h>
+#include <linux/glob.h>
+#include <linux/idr.h>
 #include <linux/init.h>
 #include <linux/of.h>
-#include <linux/stat.h>
 #include <linux/slab.h>
-#include <linux/idr.h>
 #include <linux/spinlock.h>
+#include <linux/stat.h>
+#include <linux/sysfs.h>
 #include <linux/sys_soc.h>
-#include <linux/err.h>
-#include <linux/glob.h>
 
 static DEFINE_IDA(soc_ida);
 
@@ -111,17 +111,14 @@ static void soc_release(struct device *dev)
 	kfree(soc_dev);
 }
 
-static void soc_device_get_machine(struct soc_device_attribute *soc_dev_attr)
+int soc_attr_read_machine(struct soc_device_attribute *soc_dev_attr)
 {
-	struct device_node *np;
-
 	if (soc_dev_attr->machine)
-		return;
+		return -EBUSY;
 
-	np = of_find_node_by_path("/");
-	of_property_read_string(np, "model", &soc_dev_attr->machine);
-	of_node_put(np);
+	return of_machine_read_model(&soc_dev_attr->machine);
 }
+EXPORT_SYMBOL_GPL(soc_attr_read_machine);
 
 static struct soc_device_attribute *early_soc_dev_attr;
 
@@ -131,7 +128,7 @@ struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr
 	const struct attribute_group **soc_attr_groups;
 	int ret;
 
-	soc_device_get_machine(soc_dev_attr);
+	soc_attr_read_machine(soc_dev_attr);
 
 	if (!soc_bus_registered) {
 		if (early_soc_dev_attr)
diff --git a/drivers/block/drbd/Kconfig b/drivers/block/drbd/Kconfig
index 495a72da04c6c..4c72421d65641 100644
--- a/drivers/block/drbd/Kconfig
+++ b/drivers/block/drbd/Kconfig
@@ -11,6 +11,7 @@ config BLK_DEV_DRBD
 	depends on PROC_FS && INET
 	select LRU_CACHE
 	select CRC32
+	select NET_HANDSHAKE
 	help
 
 	  NOTE: In order to authenticate connections you have to select
@@ -71,3 +72,61 @@ config DRBD_FAULT_INJECTION
 		echo 5 > /sys/module/drbd/parameters/fault_rate
 
 	  If unsure, say N.
+
+config BLK_DEV_DRBD_TCP
+	tristate "DRBD TCP transport"
+	depends on BLK_DEV_DRBD
+	default BLK_DEV_DRBD
+	help
+
+	  TCP transport support for DRBD. This is the standard transport
+	  for DRBD replication over TCP/IP networks.
+
+	  If unsure, say Y.
+
+config BLK_DEV_DRBD_LB_TCP
+	tristate "DRBD load-balanced TCP transport"
+	depends on BLK_DEV_DRBD
+	help
+
+	  Load-balanced TCP transport support for DRBD. This transport
+	  distributes DRBD replication traffic across multiple TCP
+	  connections for improved throughput.
+
+	  If unsure, say N.
+
+config BLK_DEV_DRBD_RDMA
+	tristate "DRBD RDMA transport"
+	depends on BLK_DEV_DRBD && INFINIBAND && INFINIBAND_ADDR_TRANS
+	help
+
+	  RDMA transport support for DRBD. This enables DRBD replication
+	  over RDMA-capable networks for lower latency and higher throughput.
+
+	  If unsure, say N.
+
+config DRBD_COMPAT_84
+	bool "Enable legacy (8.4) /proc/drbd and metadata compatibility"
+	depends on BLK_DEV_DRBD
+	default y
+	help
+
+	  This option enables the DRBD-8.4 style representation of
+	  DRBD devices in /proc/drbd. With DRBD 9.0, released in 2015,
+	  we deprecated this interface. The replacement is `drbdsetup
+	  status [--json] <resname>`. The new interface is optionally
+	  machine-readable, extensible, and scales to 1000s of
+	  resources. The deprecated interface had issues with all
+	  three areas.
+	  At the same time, this option also enables the DRBD driver
+	  to read and write the deprecated 8.4 version of the
+	  metadata.
+	  Only when you create a DRBD resource in the legacy way
+	  `drbdsetup new-resource <resname>` (no node-id positional
+	  argument), then the resource is put into legacy mode, and
+	  shows up in /proc/drbd, is capable of reading and writing
+	  the 8.4 metadata, and can have only a single peer. Without
+	  this compile-time option, creating a resource without a node
+	  ID is not possible.
+
+	  If unsure, say Y.
diff --git a/drivers/block/drbd/Makefile b/drivers/block/drbd/Makefile
index 67a8b352a1d5d..af482bea1af1c 100644
--- a/drivers/block/drbd/Makefile
+++ b/drivers/block/drbd/Makefile
@@ -1,9 +1,16 @@
 # SPDX-License-Identifier: GPL-2.0-only
 drbd-y := drbd_buildtag.o drbd_bitmap.o drbd_proc.o
-drbd-y += drbd_worker.o drbd_receiver.o drbd_req.o drbd_actlog.o
+drbd-y += drbd_sender.o drbd_receiver.o drbd_req.o drbd_actlog.o
 drbd-y += drbd_main.o drbd_strings.o drbd_nl.o
 drbd-y += drbd_interval.o drbd_state.o
 drbd-y += drbd_nla.o
+drbd-y += drbd_transport.o
+drbd-$(CONFIG_DEV_DAX_PMEM) += drbd_dax_pmem.o
 drbd-$(CONFIG_DEBUG_FS) += drbd_debugfs.o
+drbd-$(CONFIG_DRBD_COMPAT_84) += drbd_legacy_84.o
 
 obj-$(CONFIG_BLK_DEV_DRBD)     += drbd.o
+
+obj-$(CONFIG_BLK_DEV_DRBD_TCP) += drbd_transport_tcp.o
+obj-$(CONFIG_BLK_DEV_DRBD_LB_TCP) += drbd_transport_lb-tcp.o
+obj-$(CONFIG_BLK_DEV_DRBD_RDMA) += drbd_transport_rdma.o
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index b3dbf6c76e98f..7a69d643560d7 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -14,81 +14,41 @@
 #include <linux/slab.h>
 #include <linux/crc32c.h>
 #include <linux/drbd.h>
-#include <linux/drbd_limits.h>
+#include <linux/dynamic_debug.h>
 #include "drbd_int.h"
-
-
-enum al_transaction_types {
-	AL_TR_UPDATE = 0,
-	AL_TR_INITIALIZED = 0xffff
-};
-/* all fields on disc in big endian */
-struct __packed al_transaction_on_disk {
-	/* don't we all like magic */
-	__be32	magic;
-
-	/* to identify the most recent transaction block
-	 * in the on disk ring buffer */
-	__be32	tr_number;
-
-	/* checksum on the full 4k block, with this field set to 0. */
-	__be32	crc32c;
-
-	/* type of transaction, special transaction types like:
-	 * purge-all, set-all-idle, set-all-active, ... to-be-defined
-	 * see also enum al_transaction_types */
-	__be16	transaction_type;
-
-	/* we currently allow only a few thousand extents,
-	 * so 16bit will be enough for the slot number. */
-
-	/* how many updates in this transaction */
-	__be16	n_updates;
-
-	/* maximum slot number, "al-extents" in drbd.conf speak.
-	 * Having this in each transaction should make reconfiguration
-	 * of that parameter easier. */
-	__be16	context_size;
-
-	/* slot number the context starts with */
-	__be16	context_start_slot_nr;
-
-	/* Some reserved bytes.  Expected usage is a 64bit counter of
-	 * sectors-written since device creation, and other data generation tag
-	 * supporting usage */
-	__be32	__reserved[4];
-
-	/* --- 36 byte used --- */
-
-	/* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes
-	 * in one transaction, then use the remaining byte in the 4k block for
-	 * context information.  "Flexible" number of updates per transaction
-	 * does not help, as we have to account for the case when all update
-	 * slots are used anyways, so it would only complicate code without
-	 * additional benefit.
-	 */
-	__be16	update_slot_nr[AL_UPDATES_PER_TRANSACTION];
-
-	/* but the extent number is 32bit, which at an extent size of 4 MiB
-	 * allows to cover device sizes of up to 2**54 Byte (16 PiB) */
-	__be32	update_extent_nr[AL_UPDATES_PER_TRANSACTION];
-
-	/* --- 420 bytes used (36 + 64*6) --- */
-
-	/* 4096 - 420 = 3676 = 919 * 4 */
-	__be32	context[AL_CONTEXT_PER_TRANSACTION];
-};
+#include "drbd_meta_data.h"
+#include "drbd_dax_pmem.h"
 
 void *drbd_md_get_buffer(struct drbd_device *device, const char *intent)
 {
 	int r;
+	long t;
+	unsigned long t0 = jiffies;
+	unsigned int warn_s = 10;
+
+	for (;;) {
+		t = wait_event_timeout(device->misc_wait,
+				(r = atomic_cmpxchg(&device->md_io.in_use, 0, 1)) == 0 ||
+				device->disk_state[NOW] <= D_FAILED,
+				HZ * warn_s);
 
-	wait_event(device->misc_wait,
-		   (r = atomic_cmpxchg(&device->md_io.in_use, 0, 1)) == 0 ||
-		   device->state.disk <= D_FAILED);
+		if (r == 0)
+			break;
+
+		if (t != 0) {
+			drbd_err(device, "Failed to get md_buffer for %s: disk state %s\n",
+				 intent, drbd_disk_str(device->disk_state[NOW]));
+			return NULL;
+		}
 
-	if (r)
-		return NULL;
+		/* r != 0, t == 0: still in use, hit the timeout above.
+		 * Warn, but keep trying.
+		 */
+		drbd_err(device, "Waited %lds on md_buffer for %s; in use by %s; still trying...\n",
+			 (jiffies - t0 + HZ-1)/HZ, intent, device->md_io.current_use);
+		/* reduce warn frequency */
+		warn_s = max(30U, warn_s + 10);
+	}
 
 	device->md_io.current_use = intent;
 	device->md_io.start_jif = jiffies;
@@ -103,7 +63,7 @@ void drbd_md_put_buffer(struct drbd_device *device)
 }
 
 void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev,
-				     unsigned int *done)
+				       unsigned int *done)
 {
 	long dt;
 
@@ -115,10 +75,14 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b
 		dt = MAX_SCHEDULE_TIMEOUT;
 
 	dt = wait_event_timeout(device->misc_wait,
-			*done || test_bit(FORCE_DETACH, &device->flags), dt);
+			*done ||
+			test_bit(FORCE_DETACH, &device->flags) ||
+			test_bit(ABORT_MDIO, &device->flags),
+			dt);
+
 	if (dt == 0) {
 		drbd_err(device, "meta-data IO operation timed out\n");
-		drbd_chk_io_error(device, 1, DRBD_FORCE_DETACH);
+		drbd_handle_io_error(device, DRBD_FORCE_DETACH);
 	}
 }
 
@@ -132,15 +96,15 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
 	int err;
 	blk_opf_t op_flags = 0;
 
-	device->md_io.done = 0;
-	device->md_io.error = -ENODEV;
-
 	if ((op == REQ_OP_WRITE) && !test_bit(MD_NO_FUA, &device->flags))
 		op_flags |= REQ_FUA | REQ_PREFLUSH;
-	op_flags |= REQ_SYNC;
+	op_flags |= REQ_META | REQ_SYNC;
+
+	device->md_io.done = 0;
+	device->md_io.error = -ENODEV;
 
-	bio = bio_alloc_bioset(bdev->md_bdev, 1, op | op_flags, GFP_NOIO,
-			       &drbd_md_io_bio_set);
+	bio = bio_alloc_bioset(bdev->md_bdev, 1, op | op_flags,
+		GFP_NOIO, &drbd_md_io_bio_set);
 	bio->bi_iter.bi_sector = sector;
 	err = -EIO;
 	if (bio_add_page(bio, device->md_io.page, size, 0) != size)
@@ -148,7 +112,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
 	bio->bi_private = device;
 	bio->bi_end_io = drbd_md_endio;
 
-	if (op != REQ_OP_WRITE && device->state.disk == D_DISKLESS && device->ldev == NULL)
+	if (op != REQ_OP_WRITE && device->disk_state[NOW] == D_DISKLESS && device->ldev == NULL)
 		/* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */
 		;
 	else if (!get_ldev_if_state(device, D_ATTACHING)) {
@@ -161,14 +125,14 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
 	bio_get(bio); /* one bio_put() is in the completion handler */
 	atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */
 	device->md_io.submit_jif = jiffies;
-	if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD))
-		bio_io_error(bio);
-	else
+	if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {
+		bio->bi_status = BLK_STS_IOERR;
+		bio_endio(bio);
+	} else {
 		submit_bio(bio);
+	}
 	wait_until_done_or_force_detached(device, bdev, &device->md_io.done);
-	if (!bio->bi_status)
-		err = device->md_io.error;
-
+	err = device->md_io.error;
  out:
 	bio_put(bio);
 	return err;
@@ -180,7 +144,10 @@ int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bd
 	int err;
 	D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1);
 
-	BUG_ON(!bdev->md_bdev);
+	if (!bdev->md_bdev) {
+		drbd_err_ratelimit(device, "bdev->md_bdev==NULL\n");
+		return -EIO;
+	}
 
 	dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n",
 	     current->comm, current->pid, __func__,
@@ -203,96 +170,142 @@ int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bd
 	return err;
 }
 
-static struct bm_extent *find_active_resync_extent(struct drbd_device *device, unsigned int enr)
+bool drbd_al_active(struct drbd_device *device, sector_t sector, unsigned int size)
 {
-	struct lc_element *tmp;
-	tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
-	if (unlikely(tmp != NULL)) {
-		struct bm_extent  *bm_ext = lc_entry(tmp, struct bm_extent, lce);
-		if (test_bit(BME_NO_WRITES, &bm_ext->flags))
-			return bm_ext;
+	unsigned first = sector >> (AL_EXTENT_SHIFT-9);
+	unsigned last = size == 0 ? first : (sector + (size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
+	unsigned enr;
+	bool active = false;
+
+	spin_lock_irq(&device->al_lock);
+	for (enr = first; enr <= last; enr++) {
+		struct lc_element *al_ext;
+		al_ext = lc_find(device->act_log, enr);
+		if (al_ext && al_ext->refcnt > 0) {
+			active = true;
+			break;
+		}
 	}
-	return NULL;
+	spin_unlock_irq(&device->al_lock);
+
+	return active;
 }
 
-static struct lc_element *_al_get(struct drbd_device *device, unsigned int enr, bool nonblock)
+static
+struct lc_element *_al_get_nonblock(struct drbd_device *device, unsigned int enr)
 {
 	struct lc_element *al_ext;
-	struct bm_extent *bm_ext;
-	int wake;
 
 	spin_lock_irq(&device->al_lock);
-	bm_ext = find_active_resync_extent(device, enr);
-	if (bm_ext) {
-		wake = !test_and_set_bit(BME_PRIORITY, &bm_ext->flags);
-		spin_unlock_irq(&device->al_lock);
-		if (wake)
-			wake_up(&device->al_wait);
-		return NULL;
-	}
-	if (nonblock)
-		al_ext = lc_try_get(device->act_log, enr);
-	else
-		al_ext = lc_get(device->act_log, enr);
+	al_ext = lc_try_get(device->act_log, enr);
 	spin_unlock_irq(&device->al_lock);
+
 	return al_ext;
 }
 
-bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i)
+#if IS_ENABLED(CONFIG_DEV_DAX_PMEM)
+static
+struct lc_element *_al_get(struct drbd_device *device, unsigned int enr)
 {
-	/* for bios crossing activity log extent boundaries,
-	 * we may need to activate two extents in one go */
-	unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
-	unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
+	struct lc_element *al_ext;
 
-	D_ASSERT(device, first <= last);
-	D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
+	spin_lock_irq(&device->al_lock);
+	al_ext = lc_get(device->act_log, enr);
+	spin_unlock_irq(&device->al_lock);
 
-	/* FIXME figure out a fast path for bios crossing AL extent boundaries */
-	if (first != last)
-		return false;
+	return al_ext;
+}
+
+static bool
+drbd_dax_begin_io_fp(struct drbd_device *device, unsigned int first, unsigned int last)
+{
+	struct lc_element *al_ext;
+	unsigned long flags;
+	unsigned int enr;
+	unsigned int abort_enr;
+	bool wake = 0;
 
-	return _al_get(device, first, true);
+	for (enr = first; enr <= last; enr++) {
+		al_ext = _al_get(device, enr);
+		if (!al_ext)
+			goto abort;
+
+		if (al_ext->lc_number != enr) {
+			spin_lock_irqsave(&device->al_lock, flags);
+			drbd_dax_al_update(device, al_ext);
+			lc_committed(device->act_log);
+			spin_unlock_irqrestore(&device->al_lock, flags);
+		}
+	}
+	return true;
+abort:
+	abort_enr = enr;
+	for (enr = first; enr < abort_enr; enr++) {
+		spin_lock_irqsave(&device->al_lock, flags);
+		al_ext = lc_find(device->act_log, enr);
+		wake |= lc_put(device->act_log, al_ext) == 0;
+		spin_unlock_irqrestore(&device->al_lock, flags);
+	}
+	if (wake)
+		wake_up(&device->al_wait);
+	return false;
 }
+#else
+static bool
+drbd_dax_begin_io_fp(struct drbd_device *device, unsigned int first, unsigned int last)
+{
+	return false;
+}
+#endif
 
-bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i)
+bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i)
 {
 	/* for bios crossing activity log extent boundaries,
 	 * we may need to activate two extents in one go */
 	unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
 	unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
-	unsigned enr;
-	bool need_transaction = false;
 
 	D_ASSERT(device, first <= last);
 	D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
 
-	for (enr = first; enr <= last; enr++) {
-		struct lc_element *al_ext;
-		wait_event(device->al_wait,
-				(al_ext = _al_get(device, enr, false)) != NULL);
-		if (al_ext->lc_number != enr)
-			need_transaction = true;
-	}
-	return need_transaction;
+	if (drbd_md_dax_active(device->ldev))
+		return drbd_dax_begin_io_fp(device, first, last);
+
+	/* FIXME figure out a fast path for bios crossing AL extent boundaries */
+	if (first != last)
+		return false;
+
+	return _al_get_nonblock(device, first) != NULL;
 }
 
-#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
-/* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
+#if AL_EXTENT_SHIFT > 27
+/* Condition used to be:
+ * (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
+ * """
+ * Currently BM_BLOCK_SHIFT and AL_EXTENT_SHIFT
  * are still coupled, or assume too much about their relation.
  * Code below will not work if this is violated.
- * Will be cleaned up with some followup patch.
+ * """
+ *
+ * We want configurable bitmap granularity now.
+ * We only allow bytes per bit >= 4k, though: BM_BLOCK_SHIFT >= 12,
+ * increasing it will only make the right side smaller,
+ * not changing the boolean result.
+ * PAGE_SHIFT is 12 or larger (may be 14,16,18 for some architectures).
+ * That will increase the left side, so won't change the boolean result.
+ *
+ * Unfortunately I don't remember the specifics of which simplifications
+ * below this is supposed to protect.
+ * But assuming it is still relevant,
+ * we keep AL_EXTENT_SHIFT at 22, and must not increase it above 27
+ * without proving the code below to still work.
  */
 # error FIXME
 #endif
 
-static unsigned int al_extent_to_bm_page(unsigned int al_enr)
+static unsigned long al_extent_to_bm_bit(const struct drbd_bitmap *bm, unsigned int al_enr)
 {
-	return al_enr >>
-		/* bit to page */
-		((PAGE_SHIFT + 3) -
-		/* al extent number to bit */
-		 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
+	return (unsigned long)al_enr << (AL_EXTENT_SHIFT - bm->bm_block_shift);
 }
 
 static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
@@ -315,12 +328,14 @@ static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
 
 static int __al_write_transaction(struct drbd_device *device, struct al_transaction_on_disk *buffer)
 {
+	struct drbd_bitmap *bm = device->bitmap;
 	struct lc_element *e;
 	sector_t sector;
 	int i, mx;
 	unsigned extent_nr;
 	unsigned crc = 0;
 	int err = 0;
+	ktime_var_for_accounting(start_kt);
 
 	memset(buffer, 0, sizeof(*buffer));
 	buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
@@ -342,9 +357,13 @@ static int __al_write_transaction(struct drbd_device *device, struct al_transact
 		}
 		buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
 		buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
-		if (e->lc_number != LC_FREE)
-			drbd_bm_mark_for_writeout(device,
-					al_extent_to_bm_page(e->lc_number));
+		if (e->lc_number != LC_FREE) {
+			unsigned long start, end;
+
+			start = al_extent_to_bm_bit(bm, e->lc_number);
+			end = al_extent_to_bm_bit(bm, e->lc_number + 1) - 1;
+			drbd_bm_mark_range_for_writeout(device, start, end);
+		}
 		i++;
 	}
 	spin_unlock_irq(&device->al_lock);
@@ -378,22 +397,21 @@ static int __al_write_transaction(struct drbd_device *device, struct al_transact
 	crc = crc32c(0, buffer, 4096);
 	buffer->crc32c = cpu_to_be32(crc);
 
-	if (drbd_bm_write_hinted(device))
-		err = -EIO;
-	else {
-		bool write_al_updates;
-		rcu_read_lock();
-		write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
-		rcu_read_unlock();
-		if (write_al_updates) {
-			if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
-				err = -EIO;
-				drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
-			} else {
-				device->al_tr_number++;
-				device->al_writ_cnt++;
-			}
+	ktime_aggregate_delta(device, start_kt, al_before_bm_write_hinted_kt);
+	err = drbd_bm_write_hinted(device);
+	if (!err) {
+		ktime_aggregate_delta(device, start_kt, al_mid_kt);
+		if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
+			err = -EIO;
+			drbd_handle_io_error(device, DRBD_META_IO_ERROR);
+		} else {
+			device->al_tr_number++;
+			device->al_writ_cnt++;
+			device->al_histogram[min_t(unsigned int,
+					device->act_log->pending_changes,
+					AL_UPDATES_PER_TRANSACTION)]++;
 		}
+		ktime_aggregate_delta(device, start_kt, al_after_sync_page_kt);
 	}
 
 	return err;
@@ -406,15 +424,15 @@ static int al_write_transaction(struct drbd_device *device)
 
 	if (!get_ldev(device)) {
 		drbd_err(device, "disk is %s, cannot start al transaction\n",
-			drbd_disk_str(device->state.disk));
+			drbd_disk_str(device->disk_state[NOW]));
 		return -EIO;
 	}
 
 	/* The bitmap write may have failed, causing a state change. */
-	if (device->state.disk < D_INCONSISTENT) {
+	if (device->disk_state[NOW] < D_INCONSISTENT) {
 		drbd_err(device,
 			"disk is %s, cannot write al transaction\n",
-			drbd_disk_str(device->state.disk));
+			drbd_disk_str(device->disk_state[NOW]));
 		put_ldev(device);
 		return -EIO;
 	}
@@ -435,27 +453,47 @@ static int al_write_transaction(struct drbd_device *device)
 	return err;
 }
 
+bool drbd_al_try_lock(struct drbd_device *device)
+{
+	bool locked;
+
+	spin_lock_irq(&device->al_lock);
+	locked = lc_try_lock(device->act_log);
+	spin_unlock_irq(&device->al_lock);
+
+	return locked;
+}
+
+bool drbd_al_try_lock_for_transaction(struct drbd_device *device)
+{
+	bool locked;
+
+	spin_lock_irq(&device->al_lock);
+	locked = lc_try_lock_for_transaction(device->act_log);
+	spin_unlock_irq(&device->al_lock);
+
+	return locked;
+}
 
 void drbd_al_begin_io_commit(struct drbd_device *device)
 {
 	bool locked = false;
 
-	/* Serialize multiple transactions.
-	 * This uses test_and_set_bit, memory barrier is implicit.
-	 */
+
+	if (drbd_md_dax_active(device->ldev)) {
+		drbd_dax_al_begin_io_commit(device);
+		return;
+	}
+
 	wait_event(device->al_wait,
 			device->act_log->pending_changes == 0 ||
-			(locked = lc_try_lock_for_transaction(device->act_log)));
+			(locked = drbd_al_try_lock_for_transaction(device)));
 
 	if (locked) {
-		/* Double check: it may have been committed by someone else,
-		 * while we have been waiting for the lock. */
+		/* Double check: it may have been committed by someone else
+		 * while we were waiting for the lock. */
 		if (device->act_log->pending_changes) {
-			bool write_al_updates;
-
-			rcu_read_lock();
-			write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
-			rcu_read_unlock();
+			bool write_al_updates = !(device->ldev->md.flags & MDF_AL_DISABLED);
 
 			if (write_al_updates)
 				al_write_transaction(device);
@@ -472,13 +510,32 @@ void drbd_al_begin_io_commit(struct drbd_device *device)
 	}
 }
 
-/*
- * @delegate:   delegate activity log I/O to the worker thread
- */
-void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i)
+static bool put_actlog(struct drbd_device *device, unsigned int first, unsigned int last)
 {
-	if (drbd_al_begin_io_prepare(device, i))
-		drbd_al_begin_io_commit(device);
+	struct lc_element *extent;
+	unsigned long flags;
+	unsigned int enr;
+	bool wake = false;
+
+	D_ASSERT(device, first <= last);
+	spin_lock_irqsave(&device->al_lock, flags);
+	for (enr = first; enr <= last; enr++) {
+		extent = lc_find(device->act_log, enr);
+		/* Yes, this masks a bug elsewhere.  However, during normal
+		 * operation this is harmless, so no need to crash the kernel
+		 * by the BUG_ON(refcount == 0) in lc_put().
+		 */
+		if (!extent || extent->refcnt == 0) {
+			drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr);
+			continue;
+		}
+		if (lc_put(device->act_log, extent) == 0)
+			wake = true;
+	}
+	spin_unlock_irqrestore(&device->al_lock, flags);
+	if (wake)
+		wake_up(&device->al_wait);
+	return wake;
 }
 
 int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i)
@@ -497,20 +554,6 @@ int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *
 
 	D_ASSERT(device, first <= last);
 
-	/* Is resync active in this area? */
-	for (enr = first; enr <= last; enr++) {
-		struct lc_element *tmp;
-		tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
-		if (unlikely(tmp != NULL)) {
-			struct bm_extent  *bm_ext = lc_entry(tmp, struct bm_extent, lce);
-			if (test_bit(BME_NO_WRITES, &bm_ext->flags)) {
-				if (!test_and_set_bit(BME_PRIORITY, &bm_ext->flags))
-					return -EBUSY;
-				return -EWOULDBLOCK;
-			}
-		}
-	}
-
 	/* Try to checkout the refcounts. */
 	for (enr = first; enr <= last; enr++) {
 		struct lc_element *al_ext;
@@ -530,33 +573,18 @@ int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *
 	return 0;
 }
 
-void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i)
+/* put activity log extent references corresponding to interval i, return true
+ * if at least one extent is now unreferenced. */
+bool drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i)
 {
 	/* for bios crossing activity log extent boundaries,
 	 * we may need to activate two extents in one go */
 	unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
 	unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
-	unsigned enr;
-	struct lc_element *extent;
-	unsigned long flags;
 
 	D_ASSERT(device, first <= last);
-	spin_lock_irqsave(&device->al_lock, flags);
 
-	for (enr = first; enr <= last; enr++) {
-		extent = lc_find(device->act_log, enr);
-		/* Yes, this masks a bug elsewhere.  However, during normal
-		 * operation this is harmless, so no need to crash the kernel
-		 * by the BUG_ON(refcount == 0) in lc_put().
-		 */
-		if (!extent || extent->refcnt == 0) {
-			drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr);
-			continue;
-		}
-		lc_put(device->act_log, extent);
-	}
-	spin_unlock_irqrestore(&device->al_lock, flags);
-	wake_up(&device->al_wait);
+	return put_actlog(device, first, last);
 }
 
 static int _try_lc_del(struct drbd_device *device, struct lc_element *al_ext)
@@ -605,6 +633,9 @@ int drbd_al_initialize(struct drbd_device *device, void *buffer)
 	int al_size_4k = md->al_stripes * md->al_stripe_size_4k;
 	int i;
 
+	if (drbd_md_dax_active(device->ldev))
+		return drbd_dax_al_initialize(device);
+
 	__al_write_transaction(device, al);
 	/* There may or may not have been a pending transaction. */
 	spin_lock_irq(&device->al_lock);
@@ -622,219 +653,91 @@ int drbd_al_initialize(struct drbd_device *device, void *buffer)
 	return 0;
 }
 
-static const char *drbd_change_sync_fname[] = {
-	[RECORD_RS_FAILED] = "drbd_rs_failed_io",
-	[SET_IN_SYNC] = "drbd_set_in_sync",
-	[SET_OUT_OF_SYNC] = "drbd_set_out_of_sync"
-};
-
-/* ATTENTION. The AL's extents are 4MB each, while the extents in the
- * resync LRU-cache are 16MB each.
- * The caller of this function has to hold an get_ldev() reference.
- *
- * Adjusts the caching members ->rs_left (success) or ->rs_failed (!success),
- * potentially pulling in (and recounting the corresponding bits)
- * this resync extent into the resync extent lru cache.
- *
- * Returns whether all bits have been cleared for this resync extent,
- * precisely: (rs_left <= rs_failed)
- *
- * TODO will be obsoleted once we have a caching lru of the on disk bitmap
- */
-static bool update_rs_extent(struct drbd_device *device,
-		unsigned int enr, int count,
-		enum update_sync_bits_mode mode)
+void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go)
 {
-	struct lc_element *e;
+	unsigned long now;
+	int next;
 
-	D_ASSERT(device, atomic_read(&device->local_cnt));
-
-	/* When setting out-of-sync bits,
-	 * we don't need it cached (lc_find).
-	 * But if it is present in the cache,
-	 * we should update the cached bit count.
-	 * Otherwise, that extent should be in the resync extent lru cache
-	 * already -- or we want to pull it in if necessary -- (lc_get),
-	 * then update and check rs_left and rs_failed. */
-	if (mode == SET_OUT_OF_SYNC)
-		e = lc_find(device->resync, enr);
-	else
-		e = lc_get(device->resync, enr);
-	if (e) {
-		struct bm_extent *ext = lc_entry(e, struct bm_extent, lce);
-		if (ext->lce.lc_number == enr) {
-			if (mode == SET_IN_SYNC)
-				ext->rs_left -= count;
-			else if (mode == SET_OUT_OF_SYNC)
-				ext->rs_left += count;
-			else
-				ext->rs_failed += count;
-			if (ext->rs_left < ext->rs_failed) {
-				drbd_warn(device, "BAD! enr=%u rs_left=%d "
-				    "rs_failed=%d count=%d cstate=%s\n",
-				     ext->lce.lc_number, ext->rs_left,
-				     ext->rs_failed, count,
-				     drbd_conn_str(device->state.conn));
-
-				/* We don't expect to be able to clear more bits
-				 * than have been set when we originally counted
-				 * the set bits to cache that value in ext->rs_left.
-				 * Whatever the reason (disconnect during resync,
-				 * delayed local completion of an application write),
-				 * try to fix it up by recounting here. */
-				ext->rs_left = drbd_bm_e_weight(device, enr);
-			}
-		} else {
-			/* Normally this element should be in the cache,
-			 * since drbd_rs_begin_io() pulled it already in.
-			 *
-			 * But maybe an application write finished, and we set
-			 * something outside the resync lru_cache in sync.
-			 */
-			int rs_left = drbd_bm_e_weight(device, enr);
-			if (ext->flags != 0) {
-				drbd_warn(device, "changing resync lce: %d[%u;%02lx]"
-				     " -> %d[%u;00]\n",
-				     ext->lce.lc_number, ext->rs_left,
-				     ext->flags, enr, rs_left);
-				ext->flags = 0;
-			}
-			if (ext->rs_failed) {
-				drbd_warn(device, "Kicking resync_lru element enr=%u "
-				     "out with rs_failed=%d\n",
-				     ext->lce.lc_number, ext->rs_failed);
-			}
-			ext->rs_left = rs_left;
-			ext->rs_failed = (mode == RECORD_RS_FAILED) ? count : 0;
-			/* we don't keep a persistent log of the resync lru,
-			 * we can commit any change right away. */
-			lc_committed(device->resync);
-		}
-		if (mode != SET_OUT_OF_SYNC)
-			lc_put(device->resync, &ext->lce);
-		/* no race, we are within the al_lock! */
+	/* report progress and advance marks only if we made progress */
+	if (peer_device->rs_mark_left[peer_device->rs_last_mark] == still_to_go)
+		return;
 
-		if (ext->rs_left <= ext->rs_failed) {
-			ext->rs_failed = 0;
-			return true;
-		}
-	} else if (mode != SET_OUT_OF_SYNC) {
-		/* be quiet if lc_find() did not find it. */
-		drbd_err(device, "lc_get() failed! locked=%d/%d flags=%lu\n",
-		    device->resync_locked,
-		    device->resync->nr_elements,
-		    device->resync->flags);
-	}
-	return false;
-}
+	/* report progress and advance marks at most once every DRBD_SYNC_MARK_STEP (3 seconds) */
+	now = jiffies;
+	if (!time_after_eq(now, peer_device->rs_last_progress_report_ts + DRBD_SYNC_MARK_STEP))
+		return;
 
-void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go)
-{
-	struct drbd_device *device = peer_device->device;
-	unsigned long now = jiffies;
-	unsigned long last = device->rs_mark_time[device->rs_last_mark];
-	int next = (device->rs_last_mark + 1) % DRBD_SYNC_MARKS;
-	if (time_after_eq(now, last + DRBD_SYNC_MARK_STEP)) {
-		if (device->rs_mark_left[device->rs_last_mark] != still_to_go &&
-		    device->state.conn != C_PAUSED_SYNC_T &&
-		    device->state.conn != C_PAUSED_SYNC_S) {
-			device->rs_mark_time[next] = now;
-			device->rs_mark_left[next] = still_to_go;
-			device->rs_last_mark = next;
-		}
+	/* Do not advance marks if we are "paused" */
+	if (peer_device->repl_state[NOW] != L_PAUSED_SYNC_T &&
+	    peer_device->repl_state[NOW] != L_PAUSED_SYNC_S) {
+		next = (peer_device->rs_last_mark + 1) % DRBD_SYNC_MARKS;
+		peer_device->rs_mark_time[next] = now;
+		peer_device->rs_mark_left[next] = still_to_go;
+		peer_device->rs_last_mark = next;
 	}
-}
 
-/* It is called lazy update, so don't do write-out too often. */
-static bool lazy_bitmap_update_due(struct drbd_device *device)
-{
-	return time_after(jiffies, device->rs_last_bcast + 2*HZ);
+	/* But still report progress even if paused. */
+	peer_device->rs_last_progress_report_ts = now;
+	drbd_peer_device_post_work(peer_device, RS_PROGRESS);
 }
 
-static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done)
+/* It is called lazy update, so don't do write-out too often. */
+bool drbd_lazy_bitmap_update_due(struct drbd_peer_device *peer_device)
 {
-	if (rs_done) {
-		struct drbd_connection *connection = first_peer_device(device)->connection;
-		if (connection->agreed_pro_version <= 95 ||
-		    is_sync_target_state(device->state.conn))
-			set_bit(RS_DONE, &device->flags);
-			/* and also set RS_PROGRESS below */
-
-		/* Else: rather wait for explicit notification via receive_state,
-		 * to avoid uuids-rotated-too-fast causing full resync
-		 * in next handshake, in case the replication link breaks
-		 * at the most unfortunate time... */
-	} else if (!lazy_bitmap_update_due(device))
-		return;
-
-	drbd_device_post_work(device, RS_PROGRESS);
+	return time_after(jiffies, peer_device->rs_last_writeout + 2*HZ);
 }
 
-static int update_sync_bits(struct drbd_device *device,
+static int update_sync_bits(struct drbd_peer_device *peer_device,
 		unsigned long sbnr, unsigned long ebnr,
 		enum update_sync_bits_mode mode)
 {
-	/*
-	 * We keep a count of set bits per resync-extent in the ->rs_left
-	 * caching member, so we need to loop and work within the resync extent
-	 * alignment. Typically this loop will execute exactly once.
-	 */
-	unsigned long flags;
+	struct drbd_device *device = peer_device->device;
 	unsigned long count = 0;
-	unsigned int cleared = 0;
-	while (sbnr <= ebnr) {
-		/* set temporary boundary bit number to last bit number within
-		 * the resync extent of the current start bit number,
-		 * but cap at provided end bit number */
-		unsigned long tbnr = min(ebnr, sbnr | BM_BLOCKS_PER_BM_EXT_MASK);
-		unsigned long c;
-
-		if (mode == RECORD_RS_FAILED)
-			/* Only called from drbd_rs_failed_io(), bits
-			 * supposedly still set.  Recount, maybe some
-			 * of the bits have been successfully cleared
-			 * by application IO meanwhile.
-			 */
-			c = drbd_bm_count_bits(device, sbnr, tbnr);
-		else if (mode == SET_IN_SYNC)
-			c = drbd_bm_clear_bits(device, sbnr, tbnr);
-		else /* if (mode == SET_OUT_OF_SYNC) */
-			c = drbd_bm_set_bits(device, sbnr, tbnr);
+	int bmi = peer_device->bitmap_index;
+
+	if (mode == RECORD_RS_FAILED)
+		/* Only called from drbd_rs_failed_io(), bits
+		 * supposedly still set.  Recount, maybe some
+		 * of the bits have been successfully cleared
+		 * by application IO meanwhile.
+		 */
+		count = drbd_bm_count_bits(device, bmi, sbnr, ebnr);
+	else if (mode == SET_IN_SYNC)
+		count = drbd_bm_clear_bits(device, bmi, sbnr, ebnr);
+	else /* if (mode == SET_OUT_OF_SYNC) */
+		count = drbd_bm_set_bits(device, bmi, sbnr, ebnr);
 
-		if (c) {
-			spin_lock_irqsave(&device->al_lock, flags);
-			cleared += update_rs_extent(device, BM_BIT_TO_EXT(sbnr), c, mode);
-			spin_unlock_irqrestore(&device->al_lock, flags);
-			count += c;
-		}
-		sbnr = tbnr + 1;
-	}
 	if (count) {
 		if (mode == SET_IN_SYNC) {
-			unsigned long still_to_go = drbd_bm_total_weight(device);
-			bool rs_is_done = (still_to_go <= device->rs_failed);
-			drbd_advance_rs_marks(first_peer_device(device), still_to_go);
-			if (cleared || rs_is_done)
-				maybe_schedule_on_disk_bitmap_update(device, rs_is_done);
-		} else if (mode == RECORD_RS_FAILED)
-			device->rs_failed += count;
+			unsigned long still_to_go = drbd_bm_total_weight(peer_device);
+
+			drbd_advance_rs_marks(peer_device, still_to_go);
+
+			if (drbd_lazy_bitmap_update_due(peer_device))
+				drbd_peer_device_post_work(peer_device, RS_LAZY_BM_WRITE);
+
+			if (peer_device->connection->agreed_pro_version <= 95 &&
+					still_to_go <= peer_device->rs_failed &&
+					is_sync_source_state(peer_device, NOW))
+				drbd_peer_device_post_work(peer_device, RS_DONE);
+		} else if (mode == RECORD_RS_FAILED) {
+			peer_device->rs_failed += count;
+		} else /* if (mode == SET_OUT_OF_SYNC) */ {
+			enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+			if (repl_state >= L_SYNC_SOURCE && repl_state <= L_PAUSED_SYNC_T)
+				peer_device->rs_total += count;
+		}
 		wake_up(&device->al_wait);
 	}
 	return count;
 }
 
-static bool plausible_request_size(int size)
-{
-	return size > 0
-		&& size <= DRBD_MAX_BATCH_BIO_SIZE
-		&& IS_ALIGNED(size, 512);
-}
-
-/* clear the bit corresponding to the piece of storage in question:
- * size byte of data starting from sector.  Only clear bits of the affected
- * one or more _aligned_ BM_BLOCK_SIZE blocks.
+/* Change bits corresponding to the piece of storage in question:
+ * size byte of data starting from sector.
+ * Only clear bits for fully affected _aligned_ BM_BLOCK_SIZE blocks.
+ * Set bits even for partially affected blocks.
  *
- * called by worker on C_SYNC_TARGET and receiver on SyncSource.
+ * called by worker on L_SYNC_TARGET and receiver on SyncSource.
  *
  */
 int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, int size,
@@ -842,395 +745,152 @@ int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, in
 {
 	/* Is called from worker and receiver context _only_ */
 	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm;
 	unsigned long sbnr, ebnr, lbnr;
 	unsigned long count = 0;
 	sector_t esector, nr_sectors;
 
-	/* This would be an empty REQ_PREFLUSH, be silent. */
+	/* This would be an empty REQ_OP_FLUSH, be silent. */
 	if ((mode == SET_OUT_OF_SYNC) && size == 0)
 		return 0;
 
-	if (!plausible_request_size(size)) {
-		drbd_err(device, "%s: sector=%llus size=%d nonsense!\n",
-				drbd_change_sync_fname[mode],
-				(unsigned long long)sector, size);
+	if (peer_device->bitmap_index == -1) /* no bitmap... */
 		return 0;
-	}
 
 	if (!get_ldev(device))
 		return 0; /* no disk, no metadata, no bitmap to manipulate bits in */
 
+	bm = device->bitmap;
+
 	nr_sectors = get_capacity(device->vdisk);
 	esector = sector + (size >> 9) - 1;
 
-	if (!expect(device, sector < nr_sectors))
+	if (!expect(peer_device, sector < nr_sectors))
 		goto out;
-	if (!expect(device, esector < nr_sectors))
+	if (!expect(peer_device, esector < nr_sectors))
 		esector = nr_sectors - 1;
 
-	lbnr = BM_SECT_TO_BIT(nr_sectors-1);
+	lbnr = bm_sect_to_bit(bm, nr_sectors-1);
 
 	if (mode == SET_IN_SYNC) {
 		/* Round up start sector, round down end sector.  We make sure
 		 * we only clear full, aligned, BM_BLOCK_SIZE blocks. */
-		if (unlikely(esector < BM_SECT_PER_BIT-1))
+		if (unlikely(esector < bm_sect_per_bit(bm)-1))
 			goto out;
 		if (unlikely(esector == (nr_sectors-1)))
 			ebnr = lbnr;
 		else
-			ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1));
-		sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1);
+			ebnr = bm_sect_to_bit(bm, esector - (bm_sect_per_bit(bm)-1));
+		sbnr = bm_sect_to_bit(bm, sector + bm_sect_per_bit(bm)-1);
 	} else {
 		/* We set it out of sync, or record resync failure.
 		 * Should not round anything here. */
-		sbnr = BM_SECT_TO_BIT(sector);
-		ebnr = BM_SECT_TO_BIT(esector);
+		sbnr = bm_sect_to_bit(bm, sector);
+		ebnr = bm_sect_to_bit(bm, esector);
 	}
 
-	count = update_sync_bits(device, sbnr, ebnr, mode);
+	count = update_sync_bits(peer_device, sbnr, ebnr, mode);
 out:
 	put_ldev(device);
 	return count;
 }
 
-static
-struct bm_extent *_bme_get(struct drbd_device *device, unsigned int enr)
-{
-	struct lc_element *e;
-	struct bm_extent *bm_ext;
-	int wakeup = 0;
-	unsigned long rs_flags;
-
-	spin_lock_irq(&device->al_lock);
-	if (device->resync_locked > device->resync->nr_elements/2) {
-		spin_unlock_irq(&device->al_lock);
-		return NULL;
-	}
-	e = lc_get(device->resync, enr);
-	bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
-	if (bm_ext) {
-		if (bm_ext->lce.lc_number != enr) {
-			bm_ext->rs_left = drbd_bm_e_weight(device, enr);
-			bm_ext->rs_failed = 0;
-			lc_committed(device->resync);
-			wakeup = 1;
-		}
-		if (bm_ext->lce.refcnt == 1)
-			device->resync_locked++;
-		set_bit(BME_NO_WRITES, &bm_ext->flags);
-	}
-	rs_flags = device->resync->flags;
-	spin_unlock_irq(&device->al_lock);
-	if (wakeup)
-		wake_up(&device->al_wait);
-
-	if (!bm_ext) {
-		if (rs_flags & LC_STARVING)
-			drbd_warn(device, "Have to wait for element"
-			     " (resync LRU too small?)\n");
-		BUG_ON(rs_flags & LC_LOCKED);
-	}
-
-	return bm_ext;
-}
-
-static int _is_in_al(struct drbd_device *device, unsigned int enr)
+unsigned long drbd_set_all_out_of_sync(struct drbd_device *device, sector_t sector, int size)
 {
-	int rv;
-
-	spin_lock_irq(&device->al_lock);
-	rv = lc_is_used(device->act_log, enr);
-	spin_unlock_irq(&device->al_lock);
-
-	return rv;
+	return drbd_set_sync(device, sector, size, -1, -1);
 }
 
 /**
- * drbd_rs_begin_io() - Gets an extent in the resync LRU cache and sets it to BME_LOCKED
- * @device:	DRBD device.
- * @sector:	The sector number.
+ * drbd_set_sync  -  Set a disk range in or out of sync
+ * @device:	DRBD device
+ * @sector:	start sector of disk range
+ * @size:	size of disk range in bytes
+ * @bits:	bit values to use by bitmap index
+ * @mask:	bitmap indexes to modify (mask set)
  *
- * This functions sleeps on al_wait.
- *
- * Returns: %0 on success, -EINTR if interrupted.
+ * Returns a mask of the bitmap indexes which were modified.
  */
-int drbd_rs_begin_io(struct drbd_device *device, sector_t sector)
+unsigned long drbd_set_sync(struct drbd_device *device, sector_t sector, int size,
+		   unsigned long bits, unsigned long mask)
 {
-	unsigned int enr = BM_SECT_TO_EXT(sector);
-	struct bm_extent *bm_ext;
-	int i, sig;
-	bool sa;
-
-retry:
-	sig = wait_event_interruptible(device->al_wait,
-			(bm_ext = _bme_get(device, enr)));
-	if (sig)
-		return -EINTR;
-
-	if (test_bit(BME_LOCKED, &bm_ext->flags))
-		return 0;
-
-	/* step aside only while we are above c-min-rate; unless disabled. */
-	sa = drbd_rs_c_min_rate_throttle(device);
-
-	for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
-		sig = wait_event_interruptible(device->al_wait,
-					       !_is_in_al(device, enr * AL_EXT_PER_BM_SECT + i) ||
-					       (sa && test_bit(BME_PRIORITY, &bm_ext->flags)));
+	long set_start, set_end, clear_start, clear_end;
+	struct drbd_peer_device *peer_device;
+	struct drbd_bitmap *bm;
+	sector_t esector, nr_sectors;
+	unsigned long irq_flags;
+	unsigned long modified = 0;
 
-		if (sig || (sa && test_bit(BME_PRIORITY, &bm_ext->flags))) {
-			spin_lock_irq(&device->al_lock);
-			if (lc_put(device->resync, &bm_ext->lce) == 0) {
-				bm_ext->flags = 0; /* clears BME_NO_WRITES and eventually BME_PRIORITY */
-				device->resync_locked--;
-				wake_up(&device->al_wait);
-			}
-			spin_unlock_irq(&device->al_lock);
-			if (sig)
-				return -EINTR;
-			if (schedule_timeout_interruptible(HZ/10))
-				return -EINTR;
-			goto retry;
-		}
+	if (size <= 0 || !IS_ALIGNED(size, 512)) {
+		drbd_err(device, "%s sector: %llus, size: %d\n",
+			 __func__, (unsigned long long)sector, size);
+		return 0;
 	}
-	set_bit(BME_LOCKED, &bm_ext->flags);
-	return 0;
-}
-
-/**
- * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep
- * @peer_device: DRBD device.
- * @sector:	The sector number.
- *
- * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then
- * tries to set it to BME_LOCKED.
- *
- * Returns: %0 upon success, and -EAGAIN
- * if there is still application IO going on in this area.
- */
-int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector)
-{
-	struct drbd_device *device = peer_device->device;
-	unsigned int enr = BM_SECT_TO_EXT(sector);
-	const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT;
-	struct lc_element *e;
-	struct bm_extent *bm_ext;
-	int i;
-	bool throttle = drbd_rs_should_slow_down(peer_device, sector, true);
 
-	/* If we need to throttle, a half-locked (only marked BME_NO_WRITES,
-	 * not yet BME_LOCKED) extent needs to be kicked out explicitly if we
-	 * need to throttle. There is at most one such half-locked extent,
-	 * which is remembered in resync_wenr. */
-
-	if (throttle && device->resync_wenr != enr)
-		return -EAGAIN;
-
-	spin_lock_irq(&device->al_lock);
-	if (device->resync_wenr != LC_FREE && device->resync_wenr != enr) {
-		/* in case you have very heavy scattered io, it may
-		 * stall the syncer undefined if we give up the ref count
-		 * when we try again and requeue.
-		 *
-		 * if we don't give up the refcount, but the next time
-		 * we are scheduled this extent has been "synced" by new
-		 * application writes, we'd miss the lc_put on the
-		 * extent we keep the refcount on.
-		 * so we remembered which extent we had to try again, and
-		 * if the next requested one is something else, we do
-		 * the lc_put here...
-		 * we also have to wake_up
-		 */
-		e = lc_find(device->resync, device->resync_wenr);
-		bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
-		if (bm_ext) {
-			D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
-			D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
-			clear_bit(BME_NO_WRITES, &bm_ext->flags);
-			device->resync_wenr = LC_FREE;
-			if (lc_put(device->resync, &bm_ext->lce) == 0) {
-				bm_ext->flags = 0;
-				device->resync_locked--;
-			}
-			wake_up(&device->al_wait);
-		} else {
-			drbd_alert(device, "LOGIC BUG\n");
-		}
-	}
-	/* TRY. */
-	e = lc_try_get(device->resync, enr);
-	bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
-	if (bm_ext) {
-		if (test_bit(BME_LOCKED, &bm_ext->flags))
-			goto proceed;
-		if (!test_and_set_bit(BME_NO_WRITES, &bm_ext->flags)) {
-			device->resync_locked++;
-		} else {
-			/* we did set the BME_NO_WRITES,
-			 * but then could not set BME_LOCKED,
-			 * so we tried again.
-			 * drop the extra reference. */
-			bm_ext->lce.refcnt--;
-			D_ASSERT(device, bm_ext->lce.refcnt > 0);
-		}
-		goto check_al;
-	} else {
-		/* do we rather want to try later? */
-		if (device->resync_locked > device->resync->nr_elements-3)
-			goto try_again;
-		/* Do or do not. There is no try. -- Yoda */
-		e = lc_get(device->resync, enr);
-		bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
-		if (!bm_ext) {
-			const unsigned long rs_flags = device->resync->flags;
-			if (rs_flags & LC_STARVING)
-				drbd_warn(device, "Have to wait for element"
-				     " (resync LRU too small?)\n");
-			BUG_ON(rs_flags & LC_LOCKED);
-			goto try_again;
-		}
-		if (bm_ext->lce.lc_number != enr) {
-			bm_ext->rs_left = drbd_bm_e_weight(device, enr);
-			bm_ext->rs_failed = 0;
-			lc_committed(device->resync);
-			wake_up(&device->al_wait);
-			D_ASSERT(device, test_bit(BME_LOCKED, &bm_ext->flags) == 0);
-		}
-		set_bit(BME_NO_WRITES, &bm_ext->flags);
-		D_ASSERT(device, bm_ext->lce.refcnt == 1);
-		device->resync_locked++;
-		goto check_al;
-	}
-check_al:
-	for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
-		if (lc_is_used(device->act_log, al_enr+i))
-			goto try_again;
-	}
-	set_bit(BME_LOCKED, &bm_ext->flags);
-proceed:
-	device->resync_wenr = LC_FREE;
-	spin_unlock_irq(&device->al_lock);
-	return 0;
+	if (!get_ldev(device))
+		return 0; /* no disk, no metadata, no bitmap to set bits in */
 
-try_again:
-	if (bm_ext) {
-		if (throttle) {
-			D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
-			D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
-			clear_bit(BME_NO_WRITES, &bm_ext->flags);
-			device->resync_wenr = LC_FREE;
-			if (lc_put(device->resync, &bm_ext->lce) == 0) {
-				bm_ext->flags = 0;
-				device->resync_locked--;
-			}
-			wake_up(&device->al_wait);
-		} else
-			device->resync_wenr = enr;
-	}
-	spin_unlock_irq(&device->al_lock);
-	return -EAGAIN;
-}
+	bm = device->bitmap;
+	mask &= (1 << bm->bm_max_peers) - 1;
 
-void drbd_rs_complete_io(struct drbd_device *device, sector_t sector)
-{
-	unsigned int enr = BM_SECT_TO_EXT(sector);
-	struct lc_element *e;
-	struct bm_extent *bm_ext;
-	unsigned long flags;
+	nr_sectors = get_capacity(device->vdisk);
+	esector = sector + (size >> 9) - 1;
 
-	spin_lock_irqsave(&device->al_lock, flags);
-	e = lc_find(device->resync, enr);
-	bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
-	if (!bm_ext) {
-		spin_unlock_irqrestore(&device->al_lock, flags);
-		if (drbd_ratelimit())
-			drbd_err(device, "drbd_rs_complete_io() called, but extent not found\n");
-		return;
-	}
+	if (!expect(device, sector < nr_sectors))
+		goto out;
+	if (!expect(device, esector < nr_sectors))
+		esector = nr_sectors - 1;
 
-	if (bm_ext->lce.refcnt == 0) {
-		spin_unlock_irqrestore(&device->al_lock, flags);
-		drbd_err(device, "drbd_rs_complete_io(,%llu [=%u]) called, "
-		    "but refcnt is 0!?\n",
-		    (unsigned long long)sector, enr);
-		return;
-	}
+	/* For marking sectors as out of sync, we need to round up. */
+	set_start = bm_sect_to_bit(bm, sector);
+	set_end = bm_sect_to_bit(bm, esector);
+
+	/* For marking sectors as in sync, we need to round down except when we
+	 * reach the end of the device: The last bit in the bitmap does not
+	 * account for sectors past the end of the device.
+	 * CLEAR_END can become negative here. */
+	clear_start = bm_sect_to_bit(bm, sector + bm_sect_per_bit(bm) - 1);
+	if (esector == nr_sectors - 1)
+		clear_end = bm_sect_to_bit(bm, esector);
+	else
+		clear_end = bm_sect_to_bit(bm, esector + 1) - 1;
 
-	if (lc_put(device->resync, &bm_ext->lce) == 0) {
-		bm_ext->flags = 0; /* clear BME_LOCKED, BME_NO_WRITES and BME_PRIORITY */
-		device->resync_locked--;
-		wake_up(&device->al_wait);
-	}
+	spin_lock_irqsave(&bm->bm_all_slots_lock, irq_flags);
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		int bitmap_index = peer_device->bitmap_index;
 
-	spin_unlock_irqrestore(&device->al_lock, flags);
-}
+		if (bitmap_index == -1)
+			continue;
 
-/**
- * drbd_rs_cancel_all() - Removes all extents from the resync LRU (even BME_LOCKED)
- * @device:	DRBD device.
- */
-void drbd_rs_cancel_all(struct drbd_device *device)
-{
-	spin_lock_irq(&device->al_lock);
+		if (!test_and_clear_bit(bitmap_index, &mask))
+			continue;
 
-	if (get_ldev_if_state(device, D_FAILED)) { /* Makes sure ->resync is there. */
-		lc_reset(device->resync);
-		put_ldev(device);
+		if (test_bit(bitmap_index, &bits)) {
+			if (update_sync_bits(peer_device, set_start, set_end, SET_OUT_OF_SYNC))
+				__set_bit(bitmap_index, &modified);
+		} else if (clear_start <= clear_end) {
+			if (update_sync_bits(peer_device, clear_start, clear_end, SET_IN_SYNC))
+				__set_bit(bitmap_index, &modified);
+		}
 	}
-	device->resync_locked = 0;
-	device->resync_wenr = LC_FREE;
-	spin_unlock_irq(&device->al_lock);
-	wake_up(&device->al_wait);
-}
-
-/**
- * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU
- * @device:	DRBD device.
- *
- * Returns: %0 upon success, -EAGAIN if at least one reference count was
- * not zero.
- */
-int drbd_rs_del_all(struct drbd_device *device)
-{
-	struct lc_element *e;
-	struct bm_extent *bm_ext;
-	int i;
-
-	spin_lock_irq(&device->al_lock);
-
-	if (get_ldev_if_state(device, D_FAILED)) {
-		/* ok, ->resync is there. */
-		for (i = 0; i < device->resync->nr_elements; i++) {
-			e = lc_element_by_index(device->resync, i);
-			bm_ext = lc_entry(e, struct bm_extent, lce);
-			if (bm_ext->lce.lc_number == LC_FREE)
-				continue;
-			if (bm_ext->lce.lc_number == device->resync_wenr) {
-				drbd_info(device, "dropping %u in drbd_rs_del_all, apparently"
-				     " got 'synced' by application io\n",
-				     device->resync_wenr);
-				D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
-				D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
-				clear_bit(BME_NO_WRITES, &bm_ext->flags);
-				device->resync_wenr = LC_FREE;
-				lc_put(device->resync, &bm_ext->lce);
-			}
-			if (bm_ext->lce.refcnt != 0) {
-				drbd_info(device, "Retrying drbd_rs_del_all() later. "
-				     "refcnt=%d\n", bm_ext->lce.refcnt);
-				put_ldev(device);
-				spin_unlock_irq(&device->al_lock);
-				return -EAGAIN;
+	rcu_read_unlock();
+	if (mask) {
+		int bitmap_index;
+
+		for_each_set_bit(bitmap_index, &mask, BITS_PER_LONG) {
+			if (test_bit(bitmap_index, &bits)) {
+				if (drbd_bm_set_bits(device, bitmap_index, set_start, set_end))
+					__set_bit(bitmap_index, &modified);
+			} else if (clear_start <= clear_end) {
+				if (drbd_bm_clear_bits(device, bitmap_index,
+							clear_start, clear_end))
+					__set_bit(bitmap_index, &modified);
 			}
-			D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
-			D_ASSERT(device, !test_bit(BME_NO_WRITES, &bm_ext->flags));
-			lc_del(device->resync, &bm_ext->lce);
 		}
-		D_ASSERT(device, device->resync->used == 0);
-		put_ldev(device);
 	}
-	spin_unlock_irq(&device->al_lock);
-	wake_up(&device->al_wait);
+	spin_unlock_irqrestore(&bm->bm_all_slots_lock, irq_flags);
+out:
+	put_ldev(device);
 
-	return 0;
+	return modified;
 }
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 65ea6ec66bfd5..b0a0e1e70c853 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -12,15 +12,25 @@
 
 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
-#include <linux/bitmap.h>
+#include <linux/bitops.h>
 #include <linux/vmalloc.h>
 #include <linux/string.h>
 #include <linux/drbd.h>
 #include <linux/slab.h>
-#include <linux/highmem.h>
+#include <linux/dynamic_debug.h>
+#include <linux/libnvdimm.h>
 
 #include "drbd_int.h"
+#include "drbd_meta_data.h"
+#include "drbd_dax_pmem.h"
 
+#ifndef BITS_PER_PAGE
+#define BITS_PER_PAGE		(1UL << (PAGE_SHIFT + 3))
+#else
+# if BITS_PER_PAGE != (1UL << (PAGE_SHIFT + 3))
+#  error "ambiguous BITS_PER_PAGE"
+# endif
+#endif
 
 /* OPAQUE outside this file!
  * interface defined in drbd_int.h
@@ -80,48 +90,57 @@
  *  so we need spin_lock_irqsave().
  *  And we need the kmap_atomic.
  */
-struct drbd_bitmap {
-	struct page **bm_pages;
-	spinlock_t bm_lock;
 
-	/* exclusively to be used by __al_write_transaction(),
-	 * drbd_bm_mark_for_writeout() and
-	 * and drbd_bm_write_hinted() -> bm_rw() called from there.
-	 */
-	unsigned int n_bitmap_hints;
-	unsigned int al_bitmap_hints[AL_UPDATES_PER_TRANSACTION];
-
-	/* see LIMITATIONS: above */
-
-	unsigned long bm_set;       /* nr of set bits; THINK maybe atomic_t? */
-	unsigned long bm_bits;
-	size_t   bm_words;
-	size_t   bm_number_of_pages;
-	sector_t bm_dev_capacity;
-	struct mutex bm_change; /* serializes resize operations */
-
-	wait_queue_head_t bm_io_wait; /* used to serialize IO of single pages */
-
-	enum bm_flag bm_flags;
-
-	/* debugging aid, in case we are still racy somewhere */
-	char          *bm_why;
-	struct task_struct *bm_task;
+enum bitmap_operations {
+	BM_OP_CLEAR,
+	BM_OP_SET,
+	BM_OP_TEST,
+	BM_OP_COUNT,
+	BM_OP_MERGE,
+	BM_OP_EXTRACT,
+	BM_OP_FIND_BIT,
+	BM_OP_FIND_ZERO_BIT,
 };
 
-#define bm_print_lock_info(m) __bm_print_lock_info(m, __func__)
-static void __bm_print_lock_info(struct drbd_device *device, const char *func)
-{
+static void
+bm_print_lock_info(struct drbd_device *device, unsigned int bitmap_index, enum bitmap_operations op)
+{
+	static const char *op_names[] = {
+		[BM_OP_CLEAR] = "clear",
+		[BM_OP_SET] = "set",
+		[BM_OP_TEST] = "test",
+		[BM_OP_COUNT] = "count",
+		[BM_OP_MERGE] = "merge",
+		[BM_OP_EXTRACT] = "extract",
+		[BM_OP_FIND_BIT] = "find_bit",
+		[BM_OP_FIND_ZERO_BIT] = "find_zero_bit",
+	};
+
 	struct drbd_bitmap *b = device->bitmap;
-	if (!drbd_ratelimit())
+	if (!drbd_device_ratelimit(device, GENERIC))
 		return;
-	drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n",
+	drbd_err(device, "FIXME %s[%d] op %s, bitmap locked for '%s' by %s[%d]\n",
 		 current->comm, task_pid_nr(current),
-		 func, b->bm_why ?: "?",
-		 b->bm_task->comm, task_pid_nr(b->bm_task));
+		 op_names[op], b->bm_why ?: "?",
+		 b->bm_task_comm, b->bm_task_pid);
 }
 
-void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags)
+/* drbd_bm_lock() was introduced before drbd-9.0 to ensure that access to
+   bitmap is locked out by other means (states, etc..). If a needed lock was
+   not acquired or already taken a warning gets logged, and the critical
+   sections get serialized on a mutex.
+
+   Since drbd-9.0 actions on the bitmap could happen in parallel (e.g. "receive
+   bitmap").
+   The cheap solution taken right now, is to completely serialize bitmap
+   operations but do not warn if they operate on different bitmap slots.
+
+   The real solution is to make the locking more fine grained (one lock per
+   bitmap slot) and to allow those operations to happen parallel.
+ */
+static void
+_drbd_bm_lock(struct drbd_device *device, struct drbd_peer_device *peer_device,
+	      const char *why, enum bm_flag flags)
 {
 	struct drbd_bitmap *b = device->bitmap;
 	int trylock_failed;
@@ -133,19 +152,36 @@ void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags)
 
 	trylock_failed = !mutex_trylock(&b->bm_change);
 
+	if (trylock_failed && peer_device && b->bm_locked_peer != peer_device) {
+		mutex_lock(&b->bm_change);
+		trylock_failed = 0;
+	}
+
 	if (trylock_failed) {
 		drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n",
 			  current->comm, task_pid_nr(current),
 			  why, b->bm_why ?: "?",
-			  b->bm_task->comm, task_pid_nr(b->bm_task));
+			  b->bm_task_comm, b->bm_task_pid);
 		mutex_lock(&b->bm_change);
 	}
-	if (BM_LOCKED_MASK & b->bm_flags)
+	if (b->bm_flags & BM_LOCK_ALL)
 		drbd_err(device, "FIXME bitmap already locked in bm_lock\n");
-	b->bm_flags |= flags & BM_LOCKED_MASK;
+	b->bm_flags |= flags & BM_LOCK_ALL;
 
 	b->bm_why  = why;
-	b->bm_task = current;
+	strscpy(b->bm_task_comm, current->comm);
+	b->bm_task_pid = task_pid_nr(current);
+	b->bm_locked_peer = peer_device;
+}
+
+void drbd_bm_lock(struct drbd_device *device, const char *why, enum bm_flag flags)
+{
+	_drbd_bm_lock(device, NULL, why, flags);
+}
+
+void drbd_bm_slot_lock(struct drbd_peer_device *peer_device, char *why, enum bm_flag flags)
+{
+	_drbd_bm_lock(peer_device->device, peer_device, why, flags);
 }
 
 void drbd_bm_unlock(struct drbd_device *device)
@@ -156,15 +192,22 @@ void drbd_bm_unlock(struct drbd_device *device)
 		return;
 	}
 
-	if (!(BM_LOCKED_MASK & device->bitmap->bm_flags))
+	if (!(b->bm_flags & BM_LOCK_ALL))
 		drbd_err(device, "FIXME bitmap not locked in bm_unlock\n");
 
-	b->bm_flags &= ~BM_LOCKED_MASK;
+	b->bm_flags &= ~BM_LOCK_ALL;
 	b->bm_why  = NULL;
-	b->bm_task = NULL;
+	b->bm_task_comm[0] = 0;
+	b->bm_task_pid = 0;
+	b->bm_locked_peer = NULL;
 	mutex_unlock(&b->bm_change);
 }
 
+void drbd_bm_slot_unlock(struct drbd_peer_device *peer_device)
+{
+	drbd_bm_unlock(peer_device->device);
+}
+
 /* we store some "meta" info about our pages in page->private */
 /* at a granularity of 4k storage per bitmap bit:
  * one peta byte storage: 1<<50 byte, 1<<38 * 4k storage blocks
@@ -220,7 +263,7 @@ static void bm_page_unlock_io(struct drbd_device *device, int page_nr)
 	struct drbd_bitmap *b = device->bitmap;
 	void *addr = &page_private(b->bm_pages[page_nr]);
 	clear_bit_unlock(BM_PAGE_IO_LOCK, addr);
-	wake_up(&device->bitmap->bm_io_wait);
+	wake_up(&b->bm_io_wait);
 }
 
 /* set _before_ submit_io, so it may be reset due to being changed
@@ -232,9 +275,12 @@ static void bm_set_page_unchanged(struct page *page)
 	clear_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
 }
 
-static void bm_set_page_need_writeout(struct page *page)
+static void bm_set_page_need_writeout(struct drbd_bitmap *bitmap, unsigned int page_nr)
 {
-	set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));
+	if (!(bitmap->bm_flags & BM_ON_DAX_PMEM)) {
+		struct page *page = bitmap->bm_pages[page_nr];
+		set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));
+	}
 }
 
 void drbd_bm_reset_al_hints(struct drbd_device *device)
@@ -242,30 +288,6 @@ void drbd_bm_reset_al_hints(struct drbd_device *device)
 	device->bitmap->n_bitmap_hints = 0;
 }
 
-/**
- * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout
- * @device:	DRBD device.
- * @page_nr:	the bitmap page to mark with the "hint" flag
- *
- * From within an activity log transaction, we mark a few pages with these
- * hints, then call drbd_bm_write_hinted(), which will only write out changed
- * pages which are flagged with this mark.
- */
-void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	struct page *page;
-	if (page_nr >= device->bitmap->bm_number_of_pages) {
-		drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n",
-			 page_nr, (int)device->bitmap->bm_number_of_pages);
-		return;
-	}
-	page = device->bitmap->bm_pages[page_nr];
-	BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints));
-	if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)))
-		b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr;
-}
-
 static int bm_test_page_unchanged(struct page *page)
 {
 	volatile const unsigned long *addr = &page_private(page);
@@ -282,9 +304,12 @@ static void bm_clear_page_io_err(struct page *page)
 	clear_bit(BM_PAGE_IO_ERROR, &page_private(page));
 }
 
-static void bm_set_page_lazy_writeout(struct page *page)
+static void bm_set_page_lazy_writeout(struct drbd_bitmap *bitmap, unsigned int page_nr)
 {
-	set_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
+	if (!(bitmap->bm_flags & BM_ON_DAX_PMEM)) {
+		struct page *page = bitmap->bm_pages[page_nr];
+		set_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
+	}
 }
 
 static int bm_test_page_lazy_writeout(struct page *page)
@@ -292,57 +317,6 @@ static int bm_test_page_lazy_writeout(struct page *page)
 	return test_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));
 }
 
-/* on a 32bit box, this would allow for exactly (2<<38) bits. */
-static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr)
-{
-	/* page_nr = (word*sizeof(long)) >> PAGE_SHIFT; */
-	unsigned int page_nr = long_nr >> (PAGE_SHIFT - LN2_BPL + 3);
-	BUG_ON(page_nr >= b->bm_number_of_pages);
-	return page_nr;
-}
-
-static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr)
-{
-	/* page_nr = (bitnr/8) >> PAGE_SHIFT; */
-	unsigned int page_nr = bitnr >> (PAGE_SHIFT + 3);
-	BUG_ON(page_nr >= b->bm_number_of_pages);
-	return page_nr;
-}
-
-static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)
-{
-	struct page *page = b->bm_pages[idx];
-	return (unsigned long *) kmap_atomic(page);
-}
-
-static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)
-{
-	return __bm_map_pidx(b, idx);
-}
-
-static void __bm_unmap(unsigned long *p_addr)
-{
-	kunmap_atomic(p_addr);
-};
-
-static void bm_unmap(unsigned long *p_addr)
-{
-	return __bm_unmap(p_addr);
-}
-
-/* long word offset of _bitmap_ sector */
-#define S2W(s)	((s)<<(BM_EXT_SHIFT-BM_BLOCK_SHIFT-LN2_BPL))
-/* word offset from start of bitmap to word number _in_page_
- * modulo longs per page
-#define MLPP(X) ((X) % (PAGE_SIZE/sizeof(long))
- hm, well, Philipp thinks gcc might not optimize the % into & (... - 1)
- so do it explicitly:
- */
-#define MLPP(X) ((X) & ((PAGE_SIZE/sizeof(long))-1))
-
-/* Long words per page */
-#define LWPP (PAGE_SIZE/sizeof(long))
-
 /*
  * actually most functions herein should take a struct drbd_bitmap*, not a
  * struct drbd_device*, but for the debug macros I like to have the device around
@@ -367,16 +341,12 @@ static void bm_free_pages(struct page **pages, unsigned long number)
 	}
 }
 
-static inline void bm_vk_free(void *ptr)
-{
-	kvfree(ptr);
-}
-
 /*
  * "have" and "want" are NUMBER OF PAGES.
  */
-static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
+static struct page **bm_realloc_pages(struct drbd_device *device, unsigned long want)
 {
+	struct drbd_bitmap *b = device->bitmap;
 	struct page **old_pages = b->bm_pages;
 	struct page **new_pages, *page;
 	unsigned int i, bytes;
@@ -388,15 +358,18 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 	if (have == want)
 		return old_pages;
 
-	/* Trying kmalloc first, falling back to vmalloc.
+	/*
+	 * Trying kmalloc first, falling back to vmalloc.
 	 * GFP_NOIO, as this is called while drbd IO is "suspended",
 	 * and during resize or attach on diskless Primary,
 	 * we must not block on IO to ourselves.
-	 * Context is receiver thread or dmsetup. */
+	 * Context is receiver thread or drbdsetup.
+	 */
 	bytes = sizeof(struct page *)*want;
 	new_pages = kzalloc(bytes, GFP_NOIO | __GFP_NOWARN);
 	if (!new_pages) {
-		new_pages = __vmalloc(bytes, GFP_NOIO | __GFP_ZERO);
+		new_pages = __vmalloc(bytes,
+				GFP_NOIO | __GFP_HIGHMEM | __GFP_ZERO);
 		if (!new_pages)
 			return NULL;
 	}
@@ -405,10 +378,14 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 		for (i = 0; i < have; i++)
 			new_pages[i] = old_pages[i];
 		for (; i < want; i++) {
-			page = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
+			page = alloc_page(GFP_NOIO | __GFP_HIGHMEM | __GFP_RETRY_MAYFAIL |
+					__GFP_NOWARN | __GFP_ZERO);
 			if (!page) {
 				bm_free_pages(new_pages + have, i - have);
-				bm_vk_free(new_pages);
+				kvfree(new_pages);
+				drbd_err(device, "Failed to allocate bitmap; allocated %lu KiB / %lu KiB\n",
+						(unsigned long) i << (PAGE_SHIFT - 10),
+						want << (PAGE_SHIFT - 10));
 				return NULL;
 			}
 			/* we want to know which page it is
@@ -423,27 +400,32 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
 		bm_free_pages(old_pages + want, have - want);
 		*/
 	}
-
 	return new_pages;
 }
 
-/*
- * allocates the drbd_bitmap and stores it in device->bitmap.
- */
-int drbd_bm_init(struct drbd_device *device)
+struct drbd_bitmap *drbd_bm_alloc(unsigned int max_peers, unsigned int bm_block_shift)
 {
-	struct drbd_bitmap *b = device->bitmap;
-	WARN_ON(b != NULL);
+	struct drbd_bitmap *b;
+
+	if (bm_block_shift < BM_BLOCK_SHIFT_MIN
+	||  bm_block_shift > BM_BLOCK_SIZE_MAX)
+		return NULL;
+	if (max_peers < 1 || max_peers > DRBD_PEERS_MAX)
+		return NULL;
+
 	b = kzalloc_obj(struct drbd_bitmap);
 	if (!b)
-		return -ENOMEM;
+		return NULL;
+
 	spin_lock_init(&b->bm_lock);
+	spin_lock_init(&b->bm_all_slots_lock);
 	mutex_init(&b->bm_change);
 	init_waitqueue_head(&b->bm_io_wait);
 
-	device->bitmap = b;
+	b->bm_max_peers = max_peers;
+	b->bm_block_shift = bm_block_shift;
 
-	return 0;
+	return b;
 }
 
 sector_t drbd_bm_capacity(struct drbd_device *device)
@@ -453,170 +435,450 @@ sector_t drbd_bm_capacity(struct drbd_device *device)
 	return device->bitmap->bm_dev_capacity;
 }
 
-/* called on driver unload. TODO: call when a device is destroyed.
- */
-void drbd_bm_cleanup(struct drbd_device *device)
+void drbd_bm_free(struct drbd_device *device)
 {
-	if (!expect(device, device->bitmap))
+	struct drbd_bitmap *bitmap = device->bitmap;
+
+	if (bitmap == NULL)
 		return;
-	bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);
-	bm_vk_free(device->bitmap->bm_pages);
-	kfree(device->bitmap);
+
+	drbd_bm_resize(device, 0, 0);
+
+	kfree(bitmap);
 	device->bitmap = NULL;
 }
 
+static inline unsigned long interleaved_word32(struct drbd_bitmap *bitmap,
+					       unsigned int bitmap_index,
+					       unsigned long bit)
+{
+	return (bit >> 5) * bitmap->bm_max_peers + bitmap_index;
+}
+
+static inline unsigned long word32_to_page(unsigned long word)
+{
+	return word >> (PAGE_SHIFT - 2);
+}
+
+static inline unsigned int word32_in_page(unsigned long word)
+{
+	return word & ((1 << (PAGE_SHIFT - 2)) - 1);
+}
+
+static inline unsigned long last_bit_on_page(struct drbd_bitmap *bitmap,
+					     unsigned int bitmap_index,
+					     unsigned long bit)
+{
+	unsigned long word = interleaved_word32(bitmap, bitmap_index, bit);
+
+	return (bit | 31) + ((word32_in_page(-(word + 1)) / bitmap->bm_max_peers) << 5);
+}
+
+static inline unsigned long bit_to_page_interleaved(struct drbd_bitmap *bitmap,
+						    unsigned int bitmap_index,
+						    unsigned long bit)
+{
+	return word32_to_page(interleaved_word32(bitmap, bitmap_index, bit));
+}
+
+static void *bm_map(struct drbd_bitmap *bitmap, unsigned int page)
+{
+	if (!(bitmap->bm_flags & BM_ON_DAX_PMEM))
+		return kmap_atomic(bitmap->bm_pages[page]);
+
+	return ((unsigned char *)bitmap->bm_on_pmem) + (unsigned long)page * PAGE_SIZE;
+}
+
+static void bm_unmap(struct drbd_bitmap *bitmap, void *addr)
+{
+	if (!(bitmap->bm_flags & BM_ON_DAX_PMEM))
+		kunmap_atomic(addr);
+}
+
+
 /*
- * since (b->bm_bits % BITS_PER_LONG) != 0,
- * this masks out the remaining bits.
- * Returns the number of bits cleared.
+ * find_next_bit() and find_next_zero_bit() expect an (unsigned long *),
+ * and will dereference it.
+ * When scanning our bitmap, we are interested in 32bit words of it.
+ * The "current 32 bit word pointer" may point to the last 32 bits in a page.
+ * For 64bit long, if the page after the current page is not mapped,
+ * this causes "page fault - not-present page".
+ * Duplicate the "fast path" of these functions,
+ * simplified for "size: 32, offset: 0".
+ * Little endian arch: le32_to_cpu is a no-op.
+ * Big endian arch: le32_to_cpu moves the least significant 32 bits around.
+ * __ffs / ffz do an implicit cast to (unsignd long). On 64bit, that fills up
+ * the most significant bits with 0; we are not interested in those anyways.
  */
-#ifndef BITS_PER_PAGE
-#define BITS_PER_PAGE		(1UL << (PAGE_SHIFT + 3))
-#define BITS_PER_PAGE_MASK	(BITS_PER_PAGE - 1)
-#else
-# if BITS_PER_PAGE != (1UL << (PAGE_SHIFT + 3))
-#  error "ambiguous BITS_PER_PAGE"
-# endif
-#endif
-#define BITS_PER_LONG_MASK	(BITS_PER_LONG - 1)
-static int bm_clear_surplus(struct drbd_bitmap *b)
-{
-	unsigned long mask;
-	unsigned long *p_addr, *bm;
-	int tmp;
-	int cleared = 0;
-
-	/* number of bits modulo bits per page */
-	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);
-	/* mask the used bits of the word containing the last bit */
-	mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1;
-	/* bitmap is always stored little endian,
-	 * on disk and in core memory alike */
-	mask = cpu_to_lel(mask);
-
-	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);
-	bm = p_addr + (tmp/BITS_PER_LONG);
-	if (mask) {
-		/* If mask != 0, we are not exactly aligned, so bm now points
-		 * to the long containing the last bit.
-		 * If mask == 0, bm already points to the word immediately
-		 * after the last (long word aligned) bit. */
-		cleared = hweight_long(*bm & ~mask);
-		*bm &= mask;
-		bm++;
-	}
+static inline unsigned long find_next_bit_le32(const __le32 *addr)
+{
+	uint32_t val = *addr;
+
+	return val ? __ffs(le32_to_cpu(val)) : 32;
+}
+
+static inline unsigned long find_next_zero_bit_le32(const __le32 *addr)
+{
+	uint32_t val = *addr;
+
+	return val == ~0U ? 32 : ffz(le32_to_cpu(val));
+}
+
+
+static __always_inline unsigned long
+____bm_op(struct drbd_device *device, unsigned int bitmap_index, unsigned long start, unsigned long end,
+	 enum bitmap_operations op, __le32 *buffer)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned int word32_skip = 32 * bitmap->bm_max_peers;
+	unsigned long total = 0;
+	unsigned long word;
+	unsigned int page, bit_in_page;
+
+	if (end >= bitmap->bm_bits)
+		end = bitmap->bm_bits - 1;
+
+	word = interleaved_word32(bitmap, bitmap_index, start);
+	page = word32_to_page(word);
+	bit_in_page = (word32_in_page(word) << 5) | (start & 31);
+
+	for (; start <= end; page++) {
+		unsigned int count = 0;
+		void *addr;
+
+		addr = bm_map(bitmap, page);
+		if (((start & 31) && (start | 31) <= end) || op == BM_OP_TEST) {
+			unsigned int last = bit_in_page | 31;
+
+			switch (op) {
+			default:
+				do {
+					switch (op) {
+					case BM_OP_CLEAR:
+						if (__test_and_clear_bit_le(bit_in_page, addr))
+							count++;
+						break;
+					case BM_OP_SET:
+						if (!__test_and_set_bit_le(bit_in_page, addr))
+							count++;
+						break;
+					case BM_OP_COUNT:
+						if (test_bit_le(bit_in_page, addr))
+							total++;
+						break;
+					case BM_OP_TEST:
+						total = !!test_bit_le(bit_in_page, addr);
+						bm_unmap(bitmap, addr);
+						return total;
+					default:
+						break;
+					}
+					bit_in_page++;
+				} while (bit_in_page <= last);
+				break;
+			case BM_OP_MERGE:
+			case BM_OP_EXTRACT:
+				BUG();
+				break;
+			case BM_OP_FIND_BIT:
+				count = find_next_bit_le(addr, last + 1, bit_in_page);
+				if (count < last + 1)
+					goto found;
+				bit_in_page = last + 1;
+				break;
+			case BM_OP_FIND_ZERO_BIT:
+				count = find_next_zero_bit_le(addr, last + 1, bit_in_page);
+				if (count < last + 1)
+					goto found;
+				bit_in_page = last + 1;
+				break;
+			}
+			start = (start | 31) + 1;
+			bit_in_page += word32_skip - 32;
+			if (bit_in_page >= BITS_PER_PAGE)
+				goto next_page;
+		}
 
-	if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) {
-		/* on a 32bit arch, we may need to zero out
-		 * a padding long to align with a 64bit remote */
-		cleared += hweight_long(*bm);
-		*bm = 0;
+		while (start + 31 <= end) {
+			__le32 *p = (__le32 *)addr + (bit_in_page >> 5);
+
+			switch (op) {
+			case BM_OP_CLEAR:
+				count += hweight32(*p);
+				*p = 0;
+				break;
+			case BM_OP_SET:
+				count += hweight32(~*p);
+				*p = -1;
+				break;
+			case BM_OP_TEST:
+				BUG();
+				break;
+			case BM_OP_COUNT:
+				total += hweight32(*p);
+				break;
+			case BM_OP_MERGE:
+				count += hweight32(~*p & *buffer);
+				*p |= *buffer++;
+				break;
+			case BM_OP_EXTRACT:
+				*buffer++ = *p;
+				break;
+			case BM_OP_FIND_BIT:
+				count = find_next_bit_le32(p);
+				if (count < 32) {
+					count += bit_in_page;
+					goto found;
+				}
+				break;
+			case BM_OP_FIND_ZERO_BIT:
+				count = find_next_zero_bit_le32(p);
+				if (count < 32) {
+					count += bit_in_page;
+					goto found;
+				}
+				break;
+			}
+			start += 32;
+			bit_in_page += word32_skip;
+			if (bit_in_page >= BITS_PER_PAGE)
+				goto next_page;
+		}
+
+		/* don't overrun buffers with MERGE or EXTRACT,
+		 * jump to the kunmap and then out... */
+		if (start > end)
+			goto next_page;
+
+		switch (op) {
+		default:
+			while (start <= end) {
+				switch (op) {
+				case BM_OP_CLEAR:
+					if (__test_and_clear_bit_le(bit_in_page, addr))
+						count++;
+					break;
+				case BM_OP_SET:
+					if (!__test_and_set_bit_le(bit_in_page, addr))
+						count++;
+					break;
+				case BM_OP_COUNT:
+					if (test_bit_le(bit_in_page, addr))
+						total++;
+					break;
+				default:
+					break;
+				}
+				start++;
+				bit_in_page++;
+			}
+			break;
+		case BM_OP_MERGE:
+			{
+				__le32 *p = (__le32 *)addr + (bit_in_page >> 5);
+				__le32 b = *buffer++ & cpu_to_le32((1 << (end - start + 1)) - 1);
+
+				count += hweight32(~*p & b);
+				*p |= b;
+
+				start = end + 1;
+			}
+			break;
+		case BM_OP_EXTRACT:
+			{
+				__le32 *p = (__le32 *)addr + (bit_in_page >> 5);
+
+				*buffer++ = *p & cpu_to_le32((1 << (end - start + 1)) - 1);
+				start = end + 1;
+			}
+			break;
+		case BM_OP_FIND_BIT:
+			{
+				unsigned int last = bit_in_page + (end - start);
+
+				count = find_next_bit_le(addr, last + 1, bit_in_page);
+				if (count < last + 1)
+					goto found;
+				start = end + 1;
+			}
+			break;
+		case BM_OP_FIND_ZERO_BIT:
+			{
+				unsigned int last = bit_in_page + (end - start);
+				count = find_next_zero_bit_le(addr, last + 1, bit_in_page);
+				if (count < last + 1)
+					goto found;
+				start = end + 1;
+			}
+			break;
+		}
+
+	    next_page:
+		bm_unmap(bitmap, addr);
+		bit_in_page -= BITS_PER_PAGE;
+		switch (op) {
+		case BM_OP_CLEAR:
+			if (count) {
+				bm_set_page_lazy_writeout(bitmap, page);
+				total += count;
+			}
+			break;
+		case BM_OP_SET:
+		case BM_OP_MERGE:
+			if (count) {
+				bm_set_page_need_writeout(bitmap, page);
+				total += count;
+			}
+			break;
+		default:
+			break;
+		}
+		continue;
+
+	    found:
+		bm_unmap(bitmap, addr);
+		return start + count - bit_in_page;
 	}
-	bm_unmap(p_addr);
-	return cleared;
-}
-
-static void bm_set_surplus(struct drbd_bitmap *b)
-{
-	unsigned long mask;
-	unsigned long *p_addr, *bm;
-	int tmp;
-
-	/* number of bits modulo bits per page */
-	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);
-	/* mask the used bits of the word containing the last bit */
-	mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1;
-	/* bitmap is always stored little endian,
-	 * on disk and in core memory alike */
-	mask = cpu_to_lel(mask);
-
-	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);
-	bm = p_addr + (tmp/BITS_PER_LONG);
-	if (mask) {
-		/* If mask != 0, we are not exactly aligned, so bm now points
-		 * to the long containing the last bit.
-		 * If mask == 0, bm already points to the word immediately
-		 * after the last (long word aligned) bit. */
-		*bm |= ~mask;
-		bm++;
+	switch (op) {
+	case BM_OP_CLEAR:
+		if (total)
+			bitmap->bm_set[bitmap_index] -= total;
+		break;
+	case BM_OP_SET:
+	case BM_OP_MERGE:
+		if (total)
+			bitmap->bm_set[bitmap_index] += total;
+		break;
+	case BM_OP_FIND_BIT:
+	case BM_OP_FIND_ZERO_BIT:
+		total = DRBD_END_OF_BITMAP;
+		break;
+	default:
+		break;
 	}
+	return total;
+}
+
+/* Returns the number of bits changed.  */
+static __always_inline unsigned long
+__bm_op(struct drbd_device *device, unsigned int bitmap_index, unsigned long start, unsigned long end,
+	enum bitmap_operations op, __le32 *buffer)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+
+	if (!expect(device, bitmap))
+		return 1;
+	if (!expect(device, bitmap->bm_pages))
+		return 0;
+
+	if (!bitmap->bm_bits)
+		return 0;
 
-	if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) {
-		/* on a 32bit arch, we may need to zero out
-		 * a padding long to align with a 64bit remote */
-		*bm = ~0UL;
+	if (bitmap->bm_task_pid != task_pid_nr(current)) {
+		switch (op) {
+		case BM_OP_CLEAR:
+			if (bitmap->bm_flags & BM_LOCK_CLEAR)
+				bm_print_lock_info(device, bitmap_index, op);
+			break;
+		case BM_OP_SET:
+		case BM_OP_MERGE:
+			if (bitmap->bm_flags & BM_LOCK_SET)
+				bm_print_lock_info(device, bitmap_index, op);
+			break;
+		case BM_OP_TEST:
+		case BM_OP_COUNT:
+		case BM_OP_EXTRACT:
+		case BM_OP_FIND_BIT:
+		case BM_OP_FIND_ZERO_BIT:
+			if (bitmap->bm_flags & BM_LOCK_TEST)
+				bm_print_lock_info(device, bitmap_index, op);
+			break;
+		}
 	}
-	bm_unmap(p_addr);
+	return ____bm_op(device, bitmap_index, start, end, op, buffer);
 }
 
+static __always_inline unsigned long
+bm_op(struct drbd_device *device, unsigned int bitmap_index, unsigned long start, unsigned long end,
+      enum bitmap_operations op, __le32 *buffer)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned long irq_flags;
+	unsigned long count;
+
+	spin_lock_irqsave(&bitmap->bm_lock, irq_flags);
+	count = __bm_op(device, bitmap_index, start, end, op, buffer);
+	spin_unlock_irqrestore(&bitmap->bm_lock, irq_flags);
+	return count;
+}
+
+#ifdef BITMAP_DEBUG
+#define bm_op(device, bitmap_index, start, end, op, buffer) \
+	({ unsigned long ret; \
+	   drbd_info(device, "%s: bm_op(..., %u, %lu, %lu, %u, %p)\n", \
+		     __func__, bitmap_index, start, end, op, buffer); \
+	   ret = bm_op(device, bitmap_index, start, end, op, buffer); \
+	   drbd_info(device, "= %lu\n", ret); \
+	   ret; })
+
+#define __bm_op(device, bitmap_index, start, end, op, buffer) \
+	({ unsigned long ret; \
+	   drbd_info(device, "%s: __bm_op(..., %u, %lu, %lu, %u, %p)\n", \
+		     __func__, bitmap_index, start, end, op, buffer); \
+	   ret = __bm_op(device, bitmap_index, start, end, op, buffer); \
+	   drbd_info(device, "= %lu\n", ret); \
+	   ret; })
+#endif
+
+#ifdef BITMAP_DEBUG
+#define ___bm_op(device, bitmap_index, start, end, op, buffer) \
+	({ unsigned long ret; \
+	   drbd_info(device, "%s: ___bm_op(..., %u, %lu, %lu, %u, %p)\n", \
+		     __func__, bitmap_index, start, end, op, buffer); \
+	   ret = ____bm_op(device, bitmap_index, start, end, op, buffer); \
+	   drbd_info(device, "= %lu\n", ret); \
+	   ret; })
+#else
+#define ___bm_op(device, bitmap_index, start, end, op, buffer) \
+	____bm_op(device, bitmap_index, start, end, op, buffer)
+#endif
+
 /* you better not modify the bitmap while this is running,
  * or its results will be stale */
-static unsigned long bm_count_bits(struct drbd_bitmap *b)
-{
-	unsigned long *p_addr;
-	unsigned long bits = 0;
-	unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;
-	int idx, last_word;
-
-	/* all but last page */
-	for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) {
-		p_addr = __bm_map_pidx(b, idx);
-		bits += bitmap_weight(p_addr, BITS_PER_PAGE);
-		__bm_unmap(p_addr);
-		cond_resched();
-	}
-	/* last (or only) page */
-	last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;
-	p_addr = __bm_map_pidx(b, idx);
-	bits += bitmap_weight(p_addr, last_word * BITS_PER_LONG);
-	p_addr[last_word] &= cpu_to_lel(mask);
-	bits += hweight_long(p_addr[last_word]);
-	/* 32bit arch, may have an unused padding long */
-	if (BITS_PER_LONG == 32 && (last_word & 1) == 0)
-		p_addr[last_word+1] = 0;
-	__bm_unmap(p_addr);
-	return bits;
-}
-
-/* offset and len in long words.*/
-static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)
-{
-	unsigned long *p_addr, *bm;
-	unsigned int idx;
-	size_t do_now, end;
-
-	end = offset + len;
-
-	if (end > b->bm_words) {
-		pr_alert("bm_memset end > bm_words\n");
-		return;
-	}
+static void bm_count_bits(struct drbd_device *device)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned int bitmap_index;
+
+	for (bitmap_index = 0; bitmap_index < bitmap->bm_max_peers; bitmap_index++) {
+		unsigned long bit = 0, bits_set = 0;
+
+		while (bit < bitmap->bm_bits) {
+			unsigned long last_bit = last_bit_on_page(bitmap, bitmap_index, bit);
 
-	while (offset < end) {
-		do_now = min_t(size_t, ALIGN(offset + 1, LWPP), end) - offset;
-		idx = bm_word_to_page_idx(b, offset);
-		p_addr = bm_map_pidx(b, idx);
-		bm = p_addr + MLPP(offset);
-		if (bm+do_now > p_addr + LWPP) {
-			pr_alert("BUG BUG BUG! p_addr:%p bm:%p do_now:%d\n",
-			       p_addr, bm, (int)do_now);
-		} else
-			memset(bm, c, do_now * sizeof(long));
-		bm_unmap(p_addr);
-		bm_set_page_need_writeout(b->bm_pages[idx]);
-		offset += do_now;
+			bits_set += ___bm_op(device, bitmap_index, bit, last_bit, BM_OP_COUNT, NULL);
+			bit = last_bit + 1;
+			cond_resched();
+		}
+		bitmap->bm_set[bitmap_index] = bits_set;
 	}
 }
 
 /* For the layout, see comment above drbd_md_set_sector_offsets(). */
-static u64 drbd_md_on_disk_bits(struct drbd_backing_dev *ldev)
+static u64 drbd_md_on_disk_bits(struct drbd_device *device)
 {
-	u64 bitmap_sectors;
+	struct drbd_backing_dev *ldev = device->ldev;
+	u64 bitmap_sectors, word64_on_disk;
 	if (ldev->md.al_offset == 8)
 		bitmap_sectors = ldev->md.md_size_sect - ldev->md.bm_offset;
 	else
 		bitmap_sectors = ldev->md.al_offset - ldev->md.bm_offset;
-	return bitmap_sectors << (9 + 3);
+
+	/* for interoperability between 32bit and 64bit architectures,
+	 * we round on 64bit words.  FIXME do we still need this? */
+	word64_on_disk = bitmap_sectors << (9 - 3); /* x * (512/8) */
+	do_div(word64_on_disk, ldev->md.max_peers);
+	return word64_on_disk << 6; /* x * 64 */;
 }
 
 /*
@@ -627,19 +889,20 @@ static u64 drbd_md_on_disk_bits(struct drbd_backing_dev *ldev)
  * In case this is actually a resize, we copy the old bitmap into the new one.
  * Otherwise, the bitmap is initialized to all bits set.
  */
-int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bits)
+int drbd_bm_resize(struct drbd_device *device, sector_t capacity, bool set_new_bits)
 {
 	struct drbd_bitmap *b = device->bitmap;
-	unsigned long bits, words, owords, obits;
+	unsigned long bits, words, obits;
 	unsigned long want, have, onpages; /* number of pages */
-	struct page **npages, **opages = NULL;
+	struct page **npages = NULL, **opages = NULL;
+	void *bm_on_pmem = NULL;
 	int err = 0;
 	bool growing;
 
 	if (!expect(device, b))
 		return -ENOMEM;
 
-	drbd_bm_lock(device, "resize", BM_LOCKED_MASK);
+	drbd_bm_lock(device, "resize", BM_LOCK_ALL);
 
 	drbd_info(device, "drbd_bm_resize called with capacity == %llu\n",
 			(unsigned long long)capacity);
@@ -648,95 +911,119 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
 		goto out;
 
 	if (capacity == 0) {
+		unsigned int bitmap_index;
+
 		spin_lock_irq(&b->bm_lock);
 		opages = b->bm_pages;
 		onpages = b->bm_number_of_pages;
-		owords = b->bm_words;
 		b->bm_pages = NULL;
-		b->bm_number_of_pages =
-		b->bm_set   =
-		b->bm_bits  =
-		b->bm_words =
+		b->bm_number_of_pages = 0;
+		for (bitmap_index = 0; bitmap_index < b->bm_max_peers; bitmap_index++)
+			b->bm_set[bitmap_index] = 0;
+		b->bm_bits = 0;
+		b->bm_bits_4k = 0;
+		b->bm_words = 0;
 		b->bm_dev_capacity = 0;
 		spin_unlock_irq(&b->bm_lock);
-		bm_free_pages(opages, onpages);
-		bm_vk_free(opages);
+		if (!(b->bm_flags & BM_ON_DAX_PMEM)) {
+			bm_free_pages(opages, onpages);
+			kvfree(opages);
+		}
 		goto out;
 	}
-	bits  = BM_SECT_TO_BIT(ALIGN(capacity, BM_SECT_PER_BIT));
-
-	/* if we would use
-	   words = ALIGN(bits,BITS_PER_LONG) >> LN2_BPL;
-	   a 32bit host could present the wrong number of words
-	   to a 64bit host.
-	*/
-	words = ALIGN(bits, 64) >> LN2_BPL;
+	bits  = bm_sect_to_bit(b, ALIGN(capacity, bm_sect_per_bit(b)));
+	words = (ALIGN(bits, 64) * b->bm_max_peers) / BITS_PER_LONG;
 
 	if (get_ldev(device)) {
-		u64 bits_on_disk = drbd_md_on_disk_bits(device->ldev);
+		u64 bits_on_disk = drbd_md_on_disk_bits(device);
 		put_ldev(device);
 		if (bits > bits_on_disk) {
-			drbd_info(device, "bits = %lu\n", bits);
-			drbd_info(device, "bits_on_disk = %llu\n", bits_on_disk);
+			drbd_err(device, "Not enough space for bitmap: %lu > %lu\n",
+				(unsigned long)bits, (unsigned long)bits_on_disk);
 			err = -ENOSPC;
 			goto out;
 		}
 	}
 
-	want = PFN_UP(words*sizeof(long));
+	want = PFN_UP(words * sizeof(long));
 	have = b->bm_number_of_pages;
-	if (want == have) {
-		D_ASSERT(device, b->bm_pages != NULL);
-		npages = b->bm_pages;
+	if (drbd_md_dax_active(device->ldev)) {
+		bm_on_pmem = drbd_dax_bitmap(device, want);
 	} else {
-		if (drbd_insert_fault(device, DRBD_FAULT_BM_ALLOC))
-			npages = NULL;
-		else
-			npages = bm_realloc_pages(b, want);
-	}
+		if (want == have) {
+			D_ASSERT(device, b->bm_pages != NULL);
+			npages = b->bm_pages;
+		} else {
+			if (drbd_insert_fault(device, DRBD_FAULT_BM_ALLOC))
+				npages = NULL;
+			else
+				npages = bm_realloc_pages(device, want);
+		}
 
-	if (!npages) {
-		err = -ENOMEM;
-		goto out;
+		if (!npages) {
+			err = -ENOMEM;
+			goto out;
+		}
 	}
 
 	spin_lock_irq(&b->bm_lock);
-	opages = b->bm_pages;
-	owords = b->bm_words;
 	obits  = b->bm_bits;
 
 	growing = bits > obits;
-	if (opages && growing && set_new_bits)
-		bm_set_surplus(b);
 
-	b->bm_pages = npages;
+	if (bm_on_pmem) {
+		if (b->bm_on_pmem) {
+			void *src = b->bm_on_pmem;
+			memmove(bm_on_pmem, src, b->bm_words * sizeof(long));
+			arch_wb_cache_pmem(bm_on_pmem, b->bm_words * sizeof(long));
+		} else {
+			/* We are attaching a bitmap on PMEM. Since the memory
+			 * is persistent, the bitmap is still valid. Do not
+			 * overwrite it. */
+			growing = false;
+		}
+		b->bm_on_pmem = bm_on_pmem;
+		b->bm_flags |= BM_ON_DAX_PMEM;
+	} else {
+		opages = b->bm_pages;
+		b->bm_pages = npages;
+	}
 	b->bm_number_of_pages = want;
-	b->bm_bits  = bits;
+	b->bm_bits = bits;
+	b->bm_bits_4k = sect_to_bit(ALIGN(capacity, sect_per_bit(BM_BLOCK_SHIFT_4k)),
+				BM_BLOCK_SHIFT_4k);
 	b->bm_words = words;
 	b->bm_dev_capacity = capacity;
 
 	if (growing) {
-		if (set_new_bits) {
-			bm_memset(b, owords, 0xff, words-owords);
-			b->bm_set += bits - obits;
-		} else
-			bm_memset(b, owords, 0x00, words-owords);
+		unsigned int bitmap_index;
+
+		for (bitmap_index = 0; bitmap_index < b->bm_max_peers; bitmap_index++) {
+			unsigned long bm_set = b->bm_set[bitmap_index];
 
+			if (set_new_bits) {
+				___bm_op(device, bitmap_index, obits, -1UL, BM_OP_SET, NULL);
+				bm_set += bits - obits;
+			} else {
+				___bm_op(device, bitmap_index, obits, -1UL, BM_OP_CLEAR, NULL);
+			}
+
+			b->bm_set[bitmap_index] = bm_set;
+		}
 	}
 
-	if (want < have) {
+	if (want < have && !(b->bm_flags & BM_ON_DAX_PMEM)) {
 		/* implicit: (opages != NULL) && (opages != npages) */
 		bm_free_pages(opages + want, have - want);
 	}
 
-	(void)bm_clear_surplus(b);
-
 	spin_unlock_irq(&b->bm_lock);
 	if (opages != npages)
-		bm_vk_free(opages);
+		kvfree(opages);
 	if (!growing)
-		b->bm_set = bm_count_bits(b);
-	drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want);
+		bm_count_bits(device);
+	drbd_info(device, "resync bitmap: bits=%lu bits_4k=%lu words=%lu pages=%lu\n",
+			bits, b->bm_bits_4k, words, want);
 
  out:
 	drbd_bm_unlock(device);
@@ -748,10 +1035,8 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi
  * leaving this function...
  * we still need to lock it, since it is important that this returns
  * bm_set == 0 precisely.
- *
- * maybe bm_set should be atomic_t ?
  */
-unsigned long _drbd_bm_total_weight(struct drbd_device *device)
+unsigned long _drbd_bm_total_weight(struct drbd_device *device, int bitmap_index)
 {
 	struct drbd_bitmap *b = device->bitmap;
 	unsigned long s;
@@ -763,172 +1048,98 @@ unsigned long _drbd_bm_total_weight(struct drbd_device *device)
 		return 0;
 
 	spin_lock_irqsave(&b->bm_lock, flags);
-	s = b->bm_set;
+	s = b->bm_set[bitmap_index];
 	spin_unlock_irqrestore(&b->bm_lock, flags);
 
 	return s;
 }
 
-unsigned long drbd_bm_total_weight(struct drbd_device *device)
+unsigned long drbd_bm_total_weight(struct drbd_peer_device *peer_device)
 {
+	struct drbd_device *device = peer_device->device;
 	unsigned long s;
+
+	if (peer_device->bitmap_index == -1)
+		return 0;
+
 	/* if I don't have a disk, I don't know about out-of-sync status */
 	if (!get_ldev_if_state(device, D_NEGOTIATING))
 		return 0;
-	s = _drbd_bm_total_weight(device);
+	s = _drbd_bm_total_weight(device, peer_device->bitmap_index);
 	put_ldev(device);
 	return s;
 }
 
+/* Returns the number of unsigned long words per peer */
 size_t drbd_bm_words(struct drbd_device *device)
 {
 	struct drbd_bitmap *b = device->bitmap;
+
 	if (!expect(device, b))
 		return 0;
 	if (!expect(device, b->bm_pages))
 		return 0;
 
-	return b->bm_words;
+	return b->bm_words / b->bm_max_peers;
 }
 
 unsigned long drbd_bm_bits(struct drbd_device *device)
 {
 	struct drbd_bitmap *b = device->bitmap;
+
 	if (!expect(device, b))
 		return 0;
 
 	return b->bm_bits;
 }
 
+unsigned long drbd_bm_bits_4k(struct drbd_device *device)
+{
+	struct drbd_bitmap *b = device->bitmap;
+
+	if (!expect(device, b))
+		return 0;
+
+	return b->bm_bits_4k;
+}
+
 /* merge number words from buffer into the bitmap starting at offset.
  * buffer[i] is expected to be little endian unsigned long.
  * bitmap must be locked by drbd_bm_lock.
  * currently only used from receive_bitmap.
  */
-void drbd_bm_merge_lel(struct drbd_device *device, size_t offset, size_t number,
+void drbd_bm_merge_lel(struct drbd_peer_device *peer_device, size_t offset, size_t number,
 			unsigned long *buffer)
 {
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr, *bm;
-	unsigned long word, bits;
-	unsigned int idx;
-	size_t end, do_now;
+	unsigned long start, end;
 
-	end = offset + number;
-
-	if (!expect(device, b))
-		return;
-	if (!expect(device, b->bm_pages))
-		return;
-	if (number == 0)
-		return;
-	WARN_ON(offset >= b->bm_words);
-	WARN_ON(end    >  b->bm_words);
-
-	spin_lock_irq(&b->bm_lock);
-	while (offset < end) {
-		do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
-		idx = bm_word_to_page_idx(b, offset);
-		p_addr = bm_map_pidx(b, idx);
-		bm = p_addr + MLPP(offset);
-		offset += do_now;
-		while (do_now--) {
-			bits = hweight_long(*bm);
-			word = *bm | *buffer++;
-			*bm++ = word;
-			b->bm_set += hweight_long(word) - bits;
-		}
-		bm_unmap(p_addr);
-		bm_set_page_need_writeout(b->bm_pages[idx]);
-	}
-	/* with 32bit <-> 64bit cross-platform connect
-	 * this is only correct for current usage,
-	 * where we _know_ that we are 64 bit aligned,
-	 * and know that this function is used in this way, too...
-	 */
-	if (end == b->bm_words)
-		b->bm_set -= bm_clear_surplus(b);
-	spin_unlock_irq(&b->bm_lock);
+	start = offset * BITS_PER_LONG;
+	end = start + number * BITS_PER_LONG - 1;
+	bm_op(peer_device->device, peer_device->bitmap_index, start, end, BM_OP_MERGE, (__le32 *)buffer);
 }
 
 /* copy number words from the bitmap starting at offset into the buffer.
  * buffer[i] will be little endian unsigned long.
  */
-void drbd_bm_get_lel(struct drbd_device *device, size_t offset, size_t number,
+void drbd_bm_get_lel(struct drbd_peer_device *peer_device, size_t offset, size_t number,
 		     unsigned long *buffer)
 {
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr, *bm;
-	size_t end, do_now;
+	unsigned long start, end;
 
-	end = offset + number;
-
-	if (!expect(device, b))
-		return;
-	if (!expect(device, b->bm_pages))
-		return;
-
-	spin_lock_irq(&b->bm_lock);
-	if ((offset >= b->bm_words) ||
-	    (end    >  b->bm_words) ||
-	    (number <= 0))
-		drbd_err(device, "offset=%lu number=%lu bm_words=%lu\n",
-			(unsigned long)	offset,
-			(unsigned long)	number,
-			(unsigned long) b->bm_words);
-	else {
-		while (offset < end) {
-			do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
-			p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, offset));
-			bm = p_addr + MLPP(offset);
-			offset += do_now;
-			while (do_now--)
-				*buffer++ = *bm++;
-			bm_unmap(p_addr);
-		}
-	}
-	spin_unlock_irq(&b->bm_lock);
+	start = offset * BITS_PER_LONG;
+	end = start + number * BITS_PER_LONG - 1;
+	bm_op(peer_device->device, peer_device->bitmap_index, start, end, BM_OP_EXTRACT, (__le32 *)buffer);
 }
 
-/* set all bits in the bitmap */
-void drbd_bm_set_all(struct drbd_device *device)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	if (!expect(device, b))
-		return;
-	if (!expect(device, b->bm_pages))
-		return;
-
-	spin_lock_irq(&b->bm_lock);
-	bm_memset(b, 0, 0xff, b->bm_words);
-	(void)bm_clear_surplus(b);
-	b->bm_set = b->bm_bits;
-	spin_unlock_irq(&b->bm_lock);
-}
-
-/* clear all bits in the bitmap */
-void drbd_bm_clear_all(struct drbd_device *device)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	if (!expect(device, b))
-		return;
-	if (!expect(device, b->bm_pages))
-		return;
-
-	spin_lock_irq(&b->bm_lock);
-	bm_memset(b, 0, 0, b->bm_words);
-	b->bm_set = 0;
-	spin_unlock_irq(&b->bm_lock);
-}
 
 static void drbd_bm_aio_ctx_destroy(struct kref *kref)
 {
 	struct drbd_bm_aio_ctx *ctx = container_of(kref, struct drbd_bm_aio_ctx, kref);
 	unsigned long flags;
 
-	spin_lock_irqsave(&ctx->device->resource->req_lock, flags);
+	spin_lock_irqsave(&ctx->device->pending_bmio_lock, flags);
 	list_del(&ctx->list);
-	spin_unlock_irqrestore(&ctx->device->resource->req_lock, flags);
+	spin_unlock_irqrestore(&ctx->device->pending_bmio_lock, flags);
 	put_ldev(ctx->device);
 	kfree(ctx);
 }
@@ -939,22 +1150,24 @@ static void drbd_bm_endio(struct bio *bio)
 	struct drbd_bm_aio_ctx *ctx = bio->bi_private;
 	struct drbd_device *device = ctx->device;
 	struct drbd_bitmap *b = device->bitmap;
-	unsigned int idx = bm_page_to_idx(bio_first_page_all(bio));
+	unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page);
+
+	blk_status_t status = bio->bi_status;
 
 	if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 &&
 	    !bm_test_page_unchanged(b->bm_pages[idx]))
 		drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx);
 
-	if (bio->bi_status) {
+	if (status) {
 		/* ctx error will hold the completed-last non-zero error code,
 		 * in case error codes differ. */
-		ctx->error = blk_status_to_errno(bio->bi_status);
+		ctx->error = blk_status_to_errno(status);
 		bm_set_page_io_err(b->bm_pages[idx]);
 		/* Not identical to on disk version of it.
 		 * Is BM_PAGE_IO_ERROR enough? */
-		if (drbd_ratelimit())
+		if (drbd_device_ratelimit(device, BACKEND))
 			drbd_err(device, "IO ERROR %d on bitmap page idx %u\n",
-					bio->bi_status, idx);
+				 status, idx);
 	} else {
 		bm_clear_page_io_err(b->bm_pages[idx]);
 		dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx);
@@ -989,15 +1202,15 @@ static inline sector_t drbd_md_last_bitmap_sector(struct drbd_backing_dev *bdev)
 
 static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)
 {
+	struct bio *bio;
 	struct drbd_device *device = ctx->device;
-	enum req_op op = ctx->flags & BM_AIO_READ ? REQ_OP_READ : REQ_OP_WRITE;
 	struct drbd_bitmap *b = device->bitmap;
-	struct bio *bio;
 	struct page *page;
 	sector_t last_bm_sect;
 	sector_t first_bm_sect;
 	sector_t on_disk_sector;
 	unsigned int len;
+	enum req_op op = ctx->flags & BM_AIO_READ ? REQ_OP_READ : REQ_OP_WRITE;
 
 	first_bm_sect = device->ldev->md.md_offset + device->ldev->md.bm_offset;
 	on_disk_sector = first_bm_sect + (((sector_t)page_nr) << (PAGE_SHIFT-SECTOR_SHIFT));
@@ -1013,9 +1226,9 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
 		else
 			len = PAGE_SIZE;
 	} else {
-		if (drbd_ratelimit()) {
+		if (drbd_device_ratelimit(device, METADATA)) {
 			drbd_err(device, "Invalid offset during on-disk bitmap access: "
-				 "page idx %u, sector %llu\n", page_nr, on_disk_sector);
+				 "page idx %u, sector %llu\n", page_nr, (unsigned long long) on_disk_sector);
 		}
 		ctx->error = -EIO;
 		bm_set_page_io_err(b->bm_pages[page_nr]);
@@ -1040,35 +1253,58 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
 		bm_store_page_idx(page, page_nr);
 	} else
 		page = b->bm_pages[page_nr];
+
 	bio = bio_alloc_bioset(device->ldev->md_bdev, 1, op, GFP_NOIO,
-			&drbd_md_io_bio_set);
+		&drbd_md_io_bio_set);
 	bio->bi_iter.bi_sector = on_disk_sector;
 	__bio_add_page(bio, page, len, 0);
 	bio->bi_private = ctx;
 	bio->bi_end_io = drbd_bm_endio;
 
 	if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {
-		bio_io_error(bio);
+		bio->bi_status = BLK_STS_IOERR;
+		bio_endio(bio);
 	} else {
 		submit_bio(bio);
+		if (op == REQ_OP_WRITE)
+			device->bm_writ_cnt++;
 		/* this should not count as user activity and cause the
 		 * resync to throttle -- see drbd_rs_should_slow_down(). */
 		atomic_add(len >> 9, &device->rs_sect_ev);
 	}
 }
 
-/*
- * bm_rw: read/write the whole bitmap from/to its on disk location.
+/**
+ * bm_rw_range() - read/write the specified range of bitmap pages
+ * @device: drbd device this bitmap is associated with
+ * @start_page: start of bitmap page indices to process
+ * @end_page: end of bitmap page indices to process
+ * @flags: BM_AIO_*, see struct bm_aio_ctx.
+ *
+ * Silently limits end_page to the current bitmap size.
+ *
+ * We don't want to special case on logical_block_size of the backend device,
+ * so we submit PAGE_SIZE aligned pieces.
+ * Note that on "most" systems, PAGE_SIZE is 4k.
+ *
+ * In case this becomes an issue on systems with larger PAGE_SIZE,
+ * we may want to change this again to do 4k aligned 4k pieces.
  */
-static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned lazy_writeout_upper_idx) __must_hold(local)
+static int bm_rw_range(struct drbd_device *device,
+	unsigned int start_page, unsigned int end_page,
+	unsigned flags) __must_hold(local)
 {
 	struct drbd_bm_aio_ctx *ctx;
 	struct drbd_bitmap *b = device->bitmap;
-	unsigned int num_pages, i, count = 0;
+	unsigned int i, count = 0;
 	unsigned long now;
-	char ppb[10];
 	int err = 0;
 
+	if (b->bm_flags & BM_ON_DAX_PMEM) {
+		if (flags & (BM_AIO_WRITE_HINTED | BM_AIO_WRITE_ALL_PAGES | BM_AIO_WRITE_LAZY))
+			arch_wb_cache_pmem(b->bm_on_pmem, b->bm_words * sizeof(long));
+		return 0;
+	}
 	/*
 	 * We are protected against bitmap disappearing/resizing by holding an
 	 * ldev reference (caller must have called get_ldev()).
@@ -1078,6 +1314,10 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 	 * as we submit copies of pages anyways.
 	 */
 
+	/* if we reach this, we should have at least *some* bitmap pages. */
+	if (!expect(device, b->bm_number_of_pages))
+		return -ENODEV;
+
 	ctx = kmalloc_obj(struct drbd_bm_aio_ctx, GFP_NOIO);
 	if (!ctx)
 		return -ENOMEM;
@@ -1092,29 +1332,33 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 		.kref = KREF_INIT(2),
 	};
 
-	if (!get_ldev_if_state(device, D_ATTACHING)) {  /* put is in drbd_bm_aio_ctx_destroy() */
-		drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in bm_rw()\n");
+	if (!expect(device, get_ldev_if_state(device, D_ATTACHING))) {  /* put is in drbd_bm_aio_ctx_destroy() */
 		kfree(ctx);
 		return -ENODEV;
 	}
-	/* Here D_ATTACHING is sufficient since drbd_bm_read() is called only from
-	   drbd_adm_attach(), after device->ldev was assigned. */
+	/* Here, D_ATTACHING is sufficient because drbd_bm_read() is only
+	 * called from drbd_adm_attach(), after device->ldev has been assigned.
+	 *
+	 * The corresponding put_ldev() happens in bm_aio_ctx_destroy().
+	 */
 
 	if (0 == (ctx->flags & ~BM_AIO_READ))
-		WARN_ON(!(BM_LOCKED_MASK & b->bm_flags));
+		WARN_ON(!(b->bm_flags & BM_LOCK_ALL));
 
-	spin_lock_irq(&device->resource->req_lock);
-	list_add_tail(&ctx->list, &device->pending_bitmap_io);
-	spin_unlock_irq(&device->resource->req_lock);
+	if (end_page >= b->bm_number_of_pages)
+		end_page = b->bm_number_of_pages - 1;
 
-	num_pages = b->bm_number_of_pages;
+	spin_lock_irq(&device->pending_bmio_lock);
+	list_add_tail(&ctx->list, &device->pending_bitmap_io);
+	spin_unlock_irq(&device->pending_bmio_lock);
 
 	now = jiffies;
 
-	/* let the layers below us try to merge these bios... */
+	blk_start_plug(&ctx->bm_aio_plug);
+	/* implicit unplug if scheduled for whatever reason */
 
 	if (flags & BM_AIO_READ) {
-		for (i = 0; i < num_pages; i++) {
+		for (i = start_page; i <= end_page; i++) {
 			atomic_inc(&ctx->in_flight);
 			bm_page_io_async(ctx, i);
 			++count;
@@ -1125,7 +1369,7 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 		unsigned int hint;
 		for (hint = 0; hint < b->n_bitmap_hints; hint++) {
 			i = b->al_bitmap_hints[hint];
-			if (i >= num_pages) /* == -1U: no hint here. */
+			if (i > end_page)
 				continue;
 			/* Several AL-extents may point to the same page. */
 			if (!test_and_clear_bit(BM_PAGE_HINT_WRITEOUT,
@@ -1139,10 +1383,9 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 			++count;
 		}
 	} else {
-		for (i = 0; i < num_pages; i++) {
-			/* ignore completely unchanged pages */
-			if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx)
-				break;
+		for (i = start_page; i <= end_page; i++) {
+			/* ignore completely unchanged pages,
+			 * unless specifically requested to write ALL pages */
 			if (!(flags & BM_AIO_WRITE_ALL_PAGES) &&
 			    bm_test_page_unchanged(b->bm_pages[i])) {
 				dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i);
@@ -1150,7 +1393,7 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 			}
 			/* during lazy writeout,
 			 * ignore those pages not marked for lazy writeout. */
-			if (lazy_writeout_upper_idx &&
+			if ((flags & BM_AIO_WRITE_LAZY) &&
 			    !bm_test_page_lazy_writeout(b->bm_pages[i])) {
 				dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i);
 				continue;
@@ -1161,6 +1404,8 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 			cond_resched();
 		}
 	}
+	/* explicit unplug, we are done submitting */
+	blk_finish_plug(&ctx->bm_aio_plug);
 
 	/*
 	 * We initialize ctx->in_flight to one to make sure drbd_bm_endio
@@ -1170,13 +1415,13 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 	 * no need to wait.  Still, we need to put the kref associated with the
 	 * "in_flight reached zero, all done" event.
 	 */
-	if (!atomic_dec_and_test(&ctx->in_flight))
+	if (!atomic_dec_and_test(&ctx->in_flight)) {
 		wait_until_done_or_force_detached(device, device->ldev, &ctx->done);
-	else
+	} else
 		kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
 
-	/* summary for global bitmap IO */
-	if (flags == 0) {
+	/* summary stats for global bitmap IO */
+	if ((flags & BM_AIO_NO_STATS) == 0 && count) {
 		unsigned int ms = jiffies_to_msecs(jiffies - now);
 		if (ms > 5) {
 			drbd_info(device, "bitmap %s of %u pages took %u ms\n",
@@ -1186,63 +1431,106 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned
 	}
 
 	if (ctx->error) {
-		drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n");
-		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
+		drbd_err(device, "we had at least one MD IO ERROR during bitmap IO\n");
+		drbd_handle_io_error(device, DRBD_META_IO_ERROR);
 		err = -EIO; /* ctx->error ? */
 	}
 
 	if (atomic_read(&ctx->in_flight))
 		err = -EIO; /* Disk timeout/force-detach during IO... */
 
-	now = jiffies;
 	if (flags & BM_AIO_READ) {
-		b->bm_set = bm_count_bits(b);
-		drbd_info(device, "recounting of set bits took additional %lu jiffies\n",
-		     jiffies - now);
+		unsigned int ms;
+		now = jiffies;
+		bm_count_bits(device);
+		ms = jiffies_to_msecs(jiffies - now);
+		/* If we can count quickly, there is no need to report this either */
+		if (ms > 3)
+			drbd_info(device, "recounting of set bits took additional %ums\n", ms);
 	}
-	now = b->bm_set;
-
-	if ((flags & ~BM_AIO_READ) == 0)
-		drbd_info(device, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n",
-		     ppsize(ppb, now << (BM_BLOCK_SHIFT-10)), now);
 
 	kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy);
 	return err;
 }
 
-/*
+static int bm_rw(struct drbd_device *device, unsigned flags)
+{
+	return bm_rw_range(device, 0, -1U, flags);
+}
+
+/**
  * drbd_bm_read() - Read the whole bitmap from its on disk location.
  * @device:	DRBD device.
+ * @peer_device: parameter ignored
  */
 int drbd_bm_read(struct drbd_device *device,
 		 struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	return bm_rw(device, BM_AIO_READ);
+}
 
+static void push_al_bitmap_hint(struct drbd_device *device, unsigned int page_nr)
 {
-	return bm_rw(device, BM_AIO_READ, 0);
+	struct drbd_bitmap *b = device->bitmap;
+	struct page *page = b->bm_pages[page_nr];
+	BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints));
+	if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)))
+		b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr;
 }
 
-/*
+/**
+ * drbd_bm_mark_range_for_writeout() - mark with a "hint" to be considered for writeout
+ * @device:	DRBD device.
+ * @start: 	Start index of the range to mark.
+ * @end:	End index of the range to mark.
+ *
+ * From within an activity log transaction, we mark a few pages with these
+ * hints, then call drbd_bm_write_hinted(), which will only write out changed
+ * pages which are flagged with this mark.
+ */
+void drbd_bm_mark_range_for_writeout(struct drbd_device *device, unsigned long start, unsigned long end)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned int page_nr, last_page;
+
+	if (bitmap->bm_flags & BM_ON_DAX_PMEM)
+		return;
+
+	if (end >= bitmap->bm_bits)
+		end = bitmap->bm_bits - 1;
+
+	page_nr = bit_to_page_interleaved(bitmap, 0, start);
+	last_page = bit_to_page_interleaved(bitmap, bitmap->bm_max_peers - 1, end);
+	for (; page_nr <= last_page; page_nr++)
+		push_al_bitmap_hint(device, page_nr);
+}
+
+
+/**
  * drbd_bm_write() - Write the whole bitmap to its on disk location.
  * @device:	DRBD device.
+ * @peer_device: parameter ignored
  *
  * Will only write pages that have changed since last IO.
  */
 int drbd_bm_write(struct drbd_device *device,
-		 struct drbd_peer_device *peer_device) __must_hold(local)
+		  struct drbd_peer_device *peer_device) __must_hold(local)
 {
-	return bm_rw(device, 0, 0);
+	return bm_rw(device, 0);
 }
 
-/*
+/**
  * drbd_bm_write_all() - Write the whole bitmap to its on disk location.
- * @device:	DRBD device.
+ * @device:	 DRBD device.
+ * @peer_device: parameter ignored
  *
- * Will write all pages.
+ * Will write all pages. Is used for online resize operations. The
+ * whole bitmap should be written into its new position.
  */
 int drbd_bm_write_all(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local)
+		      struct drbd_peer_device *peer_device) __must_hold(local)
 {
-	return bm_rw(device, BM_AIO_WRITE_ALL_PAGES, 0);
+	return bm_rw(device, BM_AIO_WRITE_ALL_PAGES);
 }
 
 /**
@@ -1252,12 +1540,13 @@ int drbd_bm_write_all(struct drbd_device *device,
  */
 int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local)
 {
-	return bm_rw(device, BM_AIO_COPY_PAGES, upper_idx);
+	return bm_rw_range(device, 0, upper_idx - 1, BM_AIO_COPY_PAGES | BM_AIO_WRITE_LAZY);
 }
 
-/*
+/**
  * drbd_bm_write_copy_pages() - Write the whole bitmap to its on disk location.
  * @device:	DRBD device.
+ * @peer_device: parameter ignored
  *
  * Will only write pages that have changed since last IO.
  * In contrast to drbd_bm_write(), this will copy the bitmap pages
@@ -1267,308 +1556,126 @@ int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_ho
  * pending resync acks are still being processed.
  */
 int drbd_bm_write_copy_pages(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local)
+			     struct drbd_peer_device *peer_device) __must_hold(local)
 {
-	return bm_rw(device, BM_AIO_COPY_PAGES, 0);
+	return bm_rw(device, BM_AIO_COPY_PAGES);
 }
 
-/*
+/**
  * drbd_bm_write_hinted() - Write bitmap pages with "hint" marks, if they have changed.
  * @device:	DRBD device.
  */
 int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local)
 {
-	return bm_rw(device, BM_AIO_WRITE_HINTED | BM_AIO_COPY_PAGES, 0);
+	return bm_rw(device, BM_AIO_WRITE_HINTED | BM_AIO_COPY_PAGES);
 }
 
-/* NOTE
- * find_first_bit returns int, we return unsigned long.
- * For this to work on 32bit arch with bitnumbers > (1<<32),
- * we'd need to return u64, and get a whole lot of other places
- * fixed where we still use unsigned long.
- *
- * this returns a bit number, NOT a sector!
- */
-static unsigned long __bm_find_next(struct drbd_device *device, unsigned long bm_fo,
-	const int find_zero_bit)
+unsigned long drbd_bm_find_next(struct drbd_peer_device *peer_device, unsigned long start)
 {
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr;
-	unsigned long bit_offset;
-	unsigned i;
-
-
-	if (bm_fo > b->bm_bits) {
-		drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);
-		bm_fo = DRBD_END_OF_BITMAP;
-	} else {
-		while (bm_fo < b->bm_bits) {
-			/* bit offset of the first bit in the page */
-			bit_offset = bm_fo & ~BITS_PER_PAGE_MASK;
-			p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo));
-
-			if (find_zero_bit)
-				i = find_next_zero_bit_le(p_addr,
-						PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
-			else
-				i = find_next_bit_le(p_addr,
-						PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
-
-			__bm_unmap(p_addr);
-			if (i < PAGE_SIZE*8) {
-				bm_fo = bit_offset + i;
-				if (bm_fo >= b->bm_bits)
-					break;
-				goto found;
-			}
-			bm_fo = bit_offset + PAGE_SIZE*8;
-		}
-		bm_fo = DRBD_END_OF_BITMAP;
-	}
- found:
-	return bm_fo;
-}
-
-static unsigned long bm_find_next(struct drbd_device *device,
-	unsigned long bm_fo, const int find_zero_bit)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long i = DRBD_END_OF_BITMAP;
-
-	if (!expect(device, b))
-		return i;
-	if (!expect(device, b->bm_pages))
-		return i;
-
-	spin_lock_irq(&b->bm_lock);
-	if (BM_DONT_TEST & b->bm_flags)
-		bm_print_lock_info(device);
-
-	i = __bm_find_next(device, bm_fo, find_zero_bit);
-
-	spin_unlock_irq(&b->bm_lock);
-	return i;
+	return bm_op(peer_device->device, peer_device->bitmap_index, start, -1UL,
+		     BM_OP_FIND_BIT, NULL);
 }
 
-unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)
+/* does not spin_lock_irqsave.
+ * you must take drbd_bm_lock() first */
+unsigned long _drbd_bm_find_next(struct drbd_peer_device *peer_device, unsigned long start)
 {
-	return bm_find_next(device, bm_fo, 0);
+	/* WARN_ON(!(device->b->bm_flags & BM_LOCK_SET)); */
+	return ____bm_op(peer_device->device, peer_device->bitmap_index, start, -1UL,
+		    BM_OP_FIND_BIT, NULL);
 }
 
-#if 0
-/* not yet needed for anything. */
-unsigned long drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)
+unsigned long _drbd_bm_find_next_zero(struct drbd_peer_device *peer_device, unsigned long start)
 {
-	return bm_find_next(device, bm_fo, 1);
+	/* WARN_ON(!(device->b->bm_flags & BM_LOCK_SET)); */
+	return ____bm_op(peer_device->device, peer_device->bitmap_index, start, -1UL,
+		    BM_OP_FIND_ZERO_BIT, NULL);
 }
-#endif
 
-/* does not spin_lock_irqsave.
- * you must take drbd_bm_lock() first */
-unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo)
+unsigned int drbd_bm_set_bits(struct drbd_device *device, unsigned int bitmap_index,
+			      unsigned long start, unsigned long end)
 {
-	/* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */
-	return __bm_find_next(device, bm_fo, 0);
+	return bm_op(device, bitmap_index, start, end, BM_OP_SET, NULL);
 }
 
-unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo)
+static __always_inline void
+__bm_many_bits_op(struct drbd_device *device, unsigned int bitmap_index, unsigned long start, unsigned long end,
+		  enum bitmap_operations op)
 {
-	/* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */
-	return __bm_find_next(device, bm_fo, 1);
-}
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned long bit = start;
 
-/* returns number of bits actually changed.
- * for val != 0, we change 0 -> 1, return code positive
- * for val == 0, we change 1 -> 0, return code negative
- * wants bitnr, not sector.
- * expected to be called for only a few bits (e - s about BITS_PER_LONG).
- * Must hold bitmap lock already. */
-static int __bm_change_bits_to(struct drbd_device *device, const unsigned long s,
-	unsigned long e, int val)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr = NULL;
-	unsigned long bitnr;
-	unsigned int last_page_nr = -1U;
-	int c = 0;
-	int changed_total = 0;
-
-	if (e >= b->bm_bits) {
-		drbd_err(device, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n",
-				s, e, b->bm_bits);
-		e = b->bm_bits ? b->bm_bits -1 : 0;
-	}
-	for (bitnr = s; bitnr <= e; bitnr++) {
-		unsigned int page_nr = bm_bit_to_page_idx(b, bitnr);
-		if (page_nr != last_page_nr) {
-			if (p_addr)
-				__bm_unmap(p_addr);
-			if (c < 0)
-				bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);
-			else if (c > 0)
-				bm_set_page_need_writeout(b->bm_pages[last_page_nr]);
-			changed_total += c;
-			c = 0;
-			p_addr = __bm_map_pidx(b, page_nr);
-			last_page_nr = page_nr;
-		}
-		if (val)
-			c += (0 == __test_and_set_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr));
-		else
-			c -= (0 != __test_and_clear_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr));
-	}
-	if (p_addr)
-		__bm_unmap(p_addr);
-	if (c < 0)
-		bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);
-	else if (c > 0)
-		bm_set_page_need_writeout(b->bm_pages[last_page_nr]);
-	changed_total += c;
-	b->bm_set += changed_total;
-	return changed_total;
-}
-
-/* returns number of bits actually changed.
- * for val != 0, we change 0 -> 1, return code positive
- * for val == 0, we change 1 -> 0, return code negative
- * wants bitnr, not sector */
-static int bm_change_bits_to(struct drbd_device *device, const unsigned long s,
-	const unsigned long e, int val)
-{
-	unsigned long flags;
-	struct drbd_bitmap *b = device->bitmap;
-	int c = 0;
+	spin_lock_irq(&bitmap->bm_lock);
 
-	if (!expect(device, b))
-		return 1;
-	if (!expect(device, b->bm_pages))
-		return 0;
+	if (end >= bitmap->bm_bits)
+		end = bitmap->bm_bits - 1;
 
-	spin_lock_irqsave(&b->bm_lock, flags);
-	if ((val ? BM_DONT_SET : BM_DONT_CLEAR) & b->bm_flags)
-		bm_print_lock_info(device);
+	while (bit <= end) {
+		unsigned long last_bit = last_bit_on_page(bitmap, bitmap_index, bit);
 
-	c = __bm_change_bits_to(device, s, e, val);
+		if (end < last_bit)
+			last_bit = end;
 
-	spin_unlock_irqrestore(&b->bm_lock, flags);
-	return c;
+		__bm_op(device, bitmap_index, bit, last_bit, op, NULL);
+		bit = last_bit + 1;
+		spin_unlock_irq(&bitmap->bm_lock);
+		if (need_resched())
+			cond_resched();
+		spin_lock_irq(&bitmap->bm_lock);
+	}
+	spin_unlock_irq(&bitmap->bm_lock);
 }
 
-/* returns number of bits changed 0 -> 1 */
-int drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
+void drbd_bm_set_many_bits(struct drbd_peer_device *peer_device, unsigned long start, unsigned long end)
 {
-	return bm_change_bits_to(device, s, e, 1);
+	if (peer_device->bitmap_index == -1)
+		return;
+	__bm_many_bits_op(peer_device->device, peer_device->bitmap_index, start, end, BM_OP_SET);
 }
 
-/* returns number of bits changed 1 -> 0 */
-int drbd_bm_clear_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
+void drbd_bm_clear_many_bits(struct drbd_peer_device *peer_device, unsigned long start, unsigned long end)
 {
-	return -bm_change_bits_to(device, s, e, 0);
+	if (peer_device->bitmap_index == -1)
+		return;
+	__bm_many_bits_op(peer_device->device, peer_device->bitmap_index, start, end, BM_OP_CLEAR);
 }
 
-/* sets all bits in full words,
- * from first_word up to, but not including, last_word */
-static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,
-		int page_nr, int first_word, int last_word)
+void
+_drbd_bm_clear_many_bits(struct drbd_device *device, int bitmap_index, unsigned long start, unsigned long end)
 {
-	int i;
-	int bits;
-	int changed = 0;
-	unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]);
-
-	/* I think it is more cache line friendly to hweight_long then set to ~0UL,
-	 * than to first bitmap_weight() all words, then bitmap_fill() all words */
-	for (i = first_word; i < last_word; i++) {
-		bits = hweight_long(paddr[i]);
-		paddr[i] = ~0UL;
-		changed += BITS_PER_LONG - bits;
-	}
-	kunmap_atomic(paddr);
-	if (changed) {
-		/* We only need lazy writeout, the information is still in the
-		 * remote bitmap as well, and is reconstructed during the next
-		 * bitmap exchange, if lost locally due to a crash. */
-		bm_set_page_lazy_writeout(b->bm_pages[page_nr]);
-		b->bm_set += changed;
-	}
+	__bm_many_bits_op(device, bitmap_index, start, end, BM_OP_CLEAR);
 }
 
-/* Same thing as drbd_bm_set_bits,
- * but more efficient for a large bit range.
- * You must first drbd_bm_lock().
- * Can be called to set the whole bitmap in one go.
- * Sets bits from s to e _inclusive_. */
-void _drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
+void
+_drbd_bm_set_many_bits(struct drbd_device *device, int bitmap_index, unsigned long start, unsigned long end)
 {
-	/* First set_bit from the first bit (s)
-	 * up to the next long boundary (sl),
-	 * then assign full words up to the last long boundary (el),
-	 * then set_bit up to and including the last bit (e).
-	 *
-	 * Do not use memset, because we must account for changes,
-	 * so we need to loop over the words with hweight() anyways.
-	 */
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long sl = ALIGN(s,BITS_PER_LONG);
-	unsigned long el = (e+1) & ~((unsigned long)BITS_PER_LONG-1);
-	int first_page;
-	int last_page;
-	int page_nr;
-	int first_word;
-	int last_word;
-
-	if (e - s <= 3*BITS_PER_LONG) {
-		/* don't bother; el and sl may even be wrong. */
-		spin_lock_irq(&b->bm_lock);
-		__bm_change_bits_to(device, s, e, 1);
-		spin_unlock_irq(&b->bm_lock);
-		return;
-	}
-
-	/* difference is large enough that we can trust sl and el */
-
-	spin_lock_irq(&b->bm_lock);
+	__bm_many_bits_op(device, bitmap_index, start, end, BM_OP_SET);
+}
 
-	/* bits filling the current long */
-	if (sl)
-		__bm_change_bits_to(device, s, sl-1, 1);
+/* set all bits in the bitmap */
+void drbd_bm_set_all(struct drbd_device *device)
+{
+       struct drbd_bitmap *bitmap = device->bitmap;
+       unsigned int bitmap_index;
 
-	first_page = sl >> (3 + PAGE_SHIFT);
-	last_page = el >> (3 + PAGE_SHIFT);
+       for (bitmap_index = 0; bitmap_index < bitmap->bm_max_peers; bitmap_index++)
+	       __bm_many_bits_op(device, bitmap_index, 0, -1, BM_OP_SET);
+}
 
-	/* MLPP: modulo longs per page */
-	/* LWPP: long words per page */
-	first_word = MLPP(sl >> LN2_BPL);
-	last_word = LWPP;
+/* clear all bits in the bitmap */
+void drbd_bm_clear_all(struct drbd_device *device)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned int bitmap_index;
 
-	/* first and full pages, unless first page == last page */
-	for (page_nr = first_page; page_nr < last_page; page_nr++) {
-		bm_set_full_words_within_one_page(device->bitmap, page_nr, first_word, last_word);
-		spin_unlock_irq(&b->bm_lock);
-		cond_resched();
-		first_word = 0;
-		spin_lock_irq(&b->bm_lock);
-	}
-	/* last page (respectively only page, for first page == last page) */
-	last_word = MLPP(el >> LN2_BPL);
-
-	/* consider bitmap->bm_bits = 32768, bitmap->bm_number_of_pages = 1. (or multiples).
-	 * ==> e = 32767, el = 32768, last_page = 2,
-	 * and now last_word = 0.
-	 * We do not want to touch last_page in this case,
-	 * as we did not allocate it, it is not present in bitmap->bm_pages.
-	 */
-	if (last_word)
-		bm_set_full_words_within_one_page(device->bitmap, last_page, first_word, last_word);
+	for (bitmap_index = 0; bitmap_index < bitmap->bm_max_peers; bitmap_index++)
+		__bm_many_bits_op(device, bitmap_index, 0, -1, BM_OP_CLEAR);
+}
 
-	/* possibly trailing bits.
-	 * example: (e & 63) == 63, el will be e+1.
-	 * if that even was the very last bit,
-	 * it would trigger an assert in __bm_change_bits_to()
-	 */
-	if (el <= e)
-		__bm_change_bits_to(device, el, e, 1);
-	spin_unlock_irq(&b->bm_lock);
+unsigned int drbd_bm_clear_bits(struct drbd_device *device, unsigned int bitmap_index,
+				unsigned long start, unsigned long end)
+{
+	return bm_op(device, bitmap_index, start, end, BM_OP_CLEAR, NULL);
 }
 
 /* returns bit state
@@ -1578,120 +1685,74 @@ void _drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const
  *  0 ... bit not set
  * -1 ... first out of bounds access, stop testing for bits!
  */
-int drbd_bm_test_bit(struct drbd_device *device, const unsigned long bitnr)
+int drbd_bm_test_bit(struct drbd_peer_device *peer_device, const unsigned long bitnr)
 {
-	unsigned long flags;
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr;
-	int i;
+	struct drbd_bitmap *bitmap = peer_device->device->bitmap;
+	unsigned long irq_flags;
+	int ret;
 
-	if (!expect(device, b))
-		return 0;
-	if (!expect(device, b->bm_pages))
-		return 0;
-
-	spin_lock_irqsave(&b->bm_lock, flags);
-	if (BM_DONT_TEST & b->bm_flags)
-		bm_print_lock_info(device);
-	if (bitnr < b->bm_bits) {
-		p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr));
-		i = test_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr) ? 1 : 0;
-		bm_unmap(p_addr);
-	} else if (bitnr == b->bm_bits) {
-		i = -1;
-	} else { /* (bitnr > b->bm_bits) */
-		drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);
-		i = 0;
-	}
-
-	spin_unlock_irqrestore(&b->bm_lock, flags);
-	return i;
+	spin_lock_irqsave(&bitmap->bm_lock, irq_flags);
+	if (bitnr >= bitmap->bm_bits)
+		ret = -1;
+	else
+		ret = __bm_op(peer_device->device, peer_device->bitmap_index, bitnr, bitnr,
+			      BM_OP_COUNT, NULL);
+	spin_unlock_irqrestore(&bitmap->bm_lock, irq_flags);
+	return ret;
 }
 
 /* returns number of bits set in the range [s, e] */
-int drbd_bm_count_bits(struct drbd_device *device, const unsigned long s, const unsigned long e)
-{
-	unsigned long flags;
-	struct drbd_bitmap *b = device->bitmap;
-	unsigned long *p_addr = NULL;
-	unsigned long bitnr;
-	unsigned int page_nr = -1U;
-	int c = 0;
-
-	/* If this is called without a bitmap, that is a bug.  But just to be
-	 * robust in case we screwed up elsewhere, in that case pretend there
-	 * was one dirty bit in the requested area, so we won't try to do a
-	 * local read there (no bitmap probably implies no disk) */
-	if (!expect(device, b))
-		return 1;
-	if (!expect(device, b->bm_pages))
-		return 1;
-
-	spin_lock_irqsave(&b->bm_lock, flags);
-	if (BM_DONT_TEST & b->bm_flags)
-		bm_print_lock_info(device);
-	for (bitnr = s; bitnr <= e; bitnr++) {
-		unsigned int idx = bm_bit_to_page_idx(b, bitnr);
-		if (page_nr != idx) {
-			page_nr = idx;
-			if (p_addr)
-				bm_unmap(p_addr);
-			p_addr = bm_map_pidx(b, idx);
+int drbd_bm_count_bits(struct drbd_device *device, unsigned int bitmap_index, unsigned long s, unsigned long e)
+{
+	return bm_op(device, bitmap_index, s, e, BM_OP_COUNT, NULL);
+}
+
+void drbd_bm_copy_slot(struct drbd_device *device, unsigned int from_index, unsigned int to_index)
+{
+	struct drbd_bitmap *bitmap = device->bitmap;
+	unsigned long word_nr, from_word_nr, to_word_nr, words32_total;
+	unsigned int from_page_nr, to_page_nr, current_page_nr;
+	u32 data_word, *addr;
+
+	words32_total = bitmap->bm_words * sizeof(unsigned long) / sizeof(u32);
+	spin_lock_irq(&bitmap->bm_all_slots_lock);
+	spin_lock(&bitmap->bm_lock);
+
+	bitmap->bm_set[to_index] = 0;
+	current_page_nr = 0;
+	addr = bm_map(bitmap, current_page_nr);
+	for (word_nr = 0; word_nr < words32_total; word_nr += bitmap->bm_max_peers) {
+		from_word_nr = word_nr + from_index;
+		from_page_nr = word32_to_page(from_word_nr);
+		to_word_nr = word_nr + to_index;
+		to_page_nr = word32_to_page(to_word_nr);
+
+		if (current_page_nr != from_page_nr) {
+			bm_unmap(bitmap, addr);
+			spin_unlock(&bitmap->bm_lock);
+			spin_unlock_irq(&bitmap->bm_all_slots_lock);
+			if (need_resched())
+				cond_resched();
+			spin_lock_irq(&bitmap->bm_all_slots_lock);
+			spin_lock(&bitmap->bm_lock);
+			current_page_nr = from_page_nr;
+			addr = bm_map(bitmap, current_page_nr);
 		}
-		if (expect(device, bitnr < b->bm_bits))
-			c += (0 != test_bit_le(bitnr - (page_nr << (PAGE_SHIFT+3)), p_addr));
-		else
-			drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);
-	}
-	if (p_addr)
-		bm_unmap(p_addr);
-	spin_unlock_irqrestore(&b->bm_lock, flags);
-	return c;
-}
-
-
-/* inherently racy...
- * return value may be already out-of-date when this function returns.
- * but the general usage is that this is only use during a cstate when bits are
- * only cleared, not set, and typically only care for the case when the return
- * value is zero, or we already "locked" this "bitmap extent" by other means.
- *
- * enr is bm-extent number, since we chose to name one sector (512 bytes)
- * worth of the bitmap a "bitmap extent".
- *
- * TODO
- * I think since we use it like a reference count, we should use the real
- * reference count of some bitmap extent element from some lru instead...
- *
- */
-int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr)
-{
-	struct drbd_bitmap *b = device->bitmap;
-	int count, s, e;
-	unsigned long flags;
-	unsigned long *p_addr, *bm;
+		data_word = addr[word32_in_page(from_word_nr)];
 
-	if (!expect(device, b))
-		return 0;
-	if (!expect(device, b->bm_pages))
-		return 0;
+		if (current_page_nr != to_page_nr) {
+			bm_unmap(bitmap, addr);
+			current_page_nr = to_page_nr;
+			addr = bm_map(bitmap, current_page_nr);
+		}
 
-	spin_lock_irqsave(&b->bm_lock, flags);
-	if (BM_DONT_TEST & b->bm_flags)
-		bm_print_lock_info(device);
-
-	s = S2W(enr);
-	e = min((size_t)S2W(enr+1), b->bm_words);
-	count = 0;
-	if (s < b->bm_words) {
-		int n = e-s;
-		p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s));
-		bm = p_addr + MLPP(s);
-		count += bitmap_weight(bm, n * BITS_PER_LONG);
-		bm_unmap(p_addr);
-	} else {
-		drbd_err(device, "start offset (%d) too large in drbd_bm_e_weight\n", s);
+		if (addr[word32_in_page(to_word_nr)] != data_word)
+			bm_set_page_need_writeout(bitmap, current_page_nr);
+		addr[word32_in_page(to_word_nr)] = data_word;
+		bitmap->bm_set[to_index] += hweight32(data_word);
 	}
-	spin_unlock_irqrestore(&b->bm_lock, flags);
-	return count;
+	bm_unmap(bitmap, addr);
+
+	spin_unlock(&bitmap->bm_lock);
+	spin_unlock_irq(&bitmap->bm_all_slots_lock);
 }
diff --git a/drivers/block/drbd/drbd_dax_pmem.c b/drivers/block/drbd/drbd_dax_pmem.c
new file mode 100644
index 0000000000000..6f29dfd763a3c
--- /dev/null
+++ b/drivers/block/drbd/drbd_dax_pmem.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+   drbd_dax.c
+
+   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
+
+   Copyright (C) 2017, LINBIT HA-Solutions GmbH.
+
+
+ */
+
+/*
+  In case DRBD's meta-data resides in persistent memory do a few things
+   different.
+
+   1 direct access the bitmap in place. Do not load it into DRAM, do not
+     write it back from DRAM.
+   2 Use a better fitting format for the on-disk activity log. Instead of
+     writing transactions, the unmangled LRU-cache hash table is there.
+*/
+
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/dax.h>
+#include <linux/libnvdimm.h>
+#include <linux/blkdev.h>
+#include "drbd_int.h"
+#include "drbd_dax_pmem.h"
+#include "drbd_meta_data.h"
+
+static int map_superblock_for_dax(struct drbd_backing_dev *bdev, struct dax_device *dax_dev)
+{
+	long want = 1;
+	pgoff_t pgoff = bdev->md.md_offset >> (PAGE_SHIFT - SECTOR_SHIFT);
+	void *kaddr;
+	long len;
+	int id;
+
+	id = dax_read_lock();
+	len = dax_direct_access(dax_dev, pgoff, want, DAX_ACCESS, &kaddr, NULL);
+	dax_read_unlock(id);
+
+	if (len < want)
+		return -EIO;
+
+	bdev->md_on_pmem = kaddr;
+
+	return 0;
+}
+
+/**
+ * drbd_dax_open() - Open device for dax and map metadata superblock
+ * @bdev: backing device to be opened
+ */
+int drbd_dax_open(struct drbd_backing_dev *bdev)
+{
+	struct dax_device *dax_dev;
+	int err;
+	u64 part_off;
+
+	dax_dev = fs_dax_get_by_bdev(bdev->md_bdev, &part_off, NULL, NULL);
+	if (!dax_dev)
+		return -ENODEV;
+
+	err = map_superblock_for_dax(bdev, dax_dev);
+	if (!err)
+		bdev->dax_dev = dax_dev;
+	else
+		put_dax(dax_dev);
+
+	return err;
+}
+
+void drbd_dax_close(struct drbd_backing_dev *bdev)
+{
+	put_dax(bdev->dax_dev);
+}
+
+/**
+ * drbd_dax_map() - Map metadata for dax
+ * @bdev: backing device whose metadata is to be mapped
+ */
+int drbd_dax_map(struct drbd_backing_dev *bdev)
+{
+	struct dax_device *dax_dev = bdev->dax_dev;
+	sector_t first_sector = drbd_md_first_sector(bdev);
+	sector_t al_sector = bdev->md.md_offset + bdev->md.al_offset;
+	long want = (drbd_md_last_sector(bdev) + 1 - first_sector) >> (PAGE_SHIFT - SECTOR_SHIFT);
+	pgoff_t pgoff = first_sector >> (PAGE_SHIFT - SECTOR_SHIFT);
+	long md_offset_byte = (bdev->md.md_offset - first_sector) << SECTOR_SHIFT;
+	long al_offset_byte = (al_sector - first_sector) << SECTOR_SHIFT;
+	void *kaddr;
+	long len;
+	int id;
+
+	id = dax_read_lock();
+	len = dax_direct_access(dax_dev, pgoff, want, DAX_ACCESS, &kaddr, NULL);
+	dax_read_unlock(id);
+
+	if (len < want)
+		return -EIO;
+
+	bdev->md_on_pmem = kaddr + md_offset_byte;
+	bdev->al_on_pmem = kaddr + al_offset_byte;
+
+	return 0;
+}
+
+void drbd_dax_al_update(struct drbd_device *device, struct lc_element *al_ext)
+{
+	struct al_on_pmem *al_on_pmem = device->ldev->al_on_pmem;
+	__be32 *slot = &al_on_pmem->slots[al_ext->lc_index];
+
+	*slot = cpu_to_be32(al_ext->lc_new_number);
+	arch_wb_cache_pmem(slot, sizeof(*slot));
+}
+
+
+void drbd_dax_al_begin_io_commit(struct drbd_device *device)
+{
+	struct lc_element *e;
+
+	spin_lock_irq(&device->al_lock);
+
+	list_for_each_entry(e, &device->act_log->to_be_changed, list)
+		drbd_dax_al_update(device, e);
+
+	lc_committed(device->act_log);
+
+	spin_unlock_irq(&device->al_lock);
+}
+
+int drbd_dax_al_initialize(struct drbd_device *device)
+{
+	struct al_on_pmem *al_on_pmem = device->ldev->al_on_pmem;
+	__be32 *slots = al_on_pmem->slots;
+	int i, al_slots = (device->ldev->md.al_size_4k << (12 - 2)) - 1;
+
+	al_on_pmem->magic = cpu_to_be32(DRBD_AL_PMEM_MAGIC);
+	/* initialize all slots rather than just the configured number in case
+	 * the configuration is later changed */
+	for (i = 0; i < al_slots; i++) {
+		unsigned int extent_nr = i < device->act_log->nr_elements ?
+			lc_element_by_index(device->act_log, i)->lc_number :
+			LC_FREE;
+		slots[i] = cpu_to_be32(extent_nr);
+	}
+
+	return 0;
+}
+
+void *drbd_dax_bitmap(struct drbd_device *device, unsigned long want)
+{
+	struct drbd_backing_dev *bdev = device->ldev;
+	unsigned char *md_on_pmem = (unsigned char *)bdev->md_on_pmem;
+
+	return md_on_pmem + (long)bdev->md.bm_offset * SECTOR_SIZE;
+}
diff --git a/drivers/block/drbd/drbd_dax_pmem.h b/drivers/block/drbd/drbd_dax_pmem.h
new file mode 100644
index 0000000000000..9a929969ff27c
--- /dev/null
+++ b/drivers/block/drbd/drbd_dax_pmem.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef DRBD_DAX_H
+#define DRBD_DAX_H
+
+#include <linux/kconfig.h>
+
+#if IS_ENABLED(CONFIG_DEV_DAX_PMEM)
+
+int drbd_dax_open(struct drbd_backing_dev *bdev);
+void drbd_dax_close(struct drbd_backing_dev *bdev);
+int drbd_dax_map(struct drbd_backing_dev *bdev);
+void drbd_dax_al_update(struct drbd_device *device, struct lc_element *al_ext);
+void drbd_dax_al_begin_io_commit(struct drbd_device *device);
+int drbd_dax_al_initialize(struct drbd_device *device);
+void *drbd_dax_bitmap(struct drbd_device *device, unsigned long want);
+
+static inline bool drbd_md_dax_active(struct drbd_backing_dev *bdev)
+{
+	return bdev->dax_dev != NULL;
+}
+static inline struct meta_data_on_disk_9 *drbd_dax_md_addr(struct drbd_backing_dev *bdev)
+{
+	return bdev->md_on_pmem;
+}
+#else
+
+#define drbd_dax_open(B) do { } while (0)
+#define drbd_dax_close(B) do { } while (0)
+#define drbd_dax_map(B) (-ENOTSUPP)
+#define drbd_dax_al_begin_io_commit(D) do { } while (0)
+#define drbd_dax_al_initialize(D) (-EIO)
+#define drbd_dax_bitmap(D, L) (NULL)
+#define drbd_md_dax_active(B) (false)
+#define drbd_dax_md_addr(B) (NULL)
+
+#define arch_wb_cache_pmem(A, L) do { } while (0)
+
+#endif /* IS_ENABLED(CONFIG_DEV_DAX_PMEM) */
+
+#endif /* DRBD_DAX_H */
diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c
index 12460b584bcb9..a0e025d2bfd66 100644
--- a/drivers/block/drbd/drbd_debugfs.c
+++ b/drivers/block/drbd/drbd_debugfs.c
@@ -1,17 +1,18 @@
 // SPDX-License-Identifier: GPL-2.0-only
-#define pr_fmt(fmt) "drbd debugfs: " fmt
+#define pr_fmt(fmt)	KBUILD_MODNAME " debugfs: " fmt
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
-#include <linux/stat.h>
 #include <linux/jiffies.h>
 #include <linux/list.h>
+#include <generated/utsrelease.h>
 
 #include "drbd_int.h"
 #include "drbd_req.h"
 #include "drbd_debugfs.h"
-
+#include "drbd_transport.h"
+#include "drbd_dax_pmem.h"
 
 /**********************************************************************
  * Whenever you change the file format, remember to bump the version. *
@@ -19,26 +20,48 @@
 
 static struct dentry *drbd_debugfs_root;
 static struct dentry *drbd_debugfs_version;
+static struct dentry *drbd_debugfs_refcounts;
 static struct dentry *drbd_debugfs_resources;
 static struct dentry *drbd_debugfs_minors;
+static struct dentry *drbd_debugfs_compat;
+
+static void seq_print_node_mask(struct seq_file *m, struct drbd_resource *resource, u64 nodes)
+{
+	struct drbd_connection *connection;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (NODE_MASK(connection->peer_node_id) & nodes) {
+			char *name = rcu_dereference((connection)->transport.net_conf)->name;
 
-static void seq_print_age_or_dash(struct seq_file *m, bool valid, unsigned long dt)
+			seq_printf(m, "%s, ", name);
+		}
+	}
+	rcu_read_unlock();
+	seq_puts(m, "\n");
+}
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+static void seq_print_age_or_dash(struct seq_file *m, bool valid, ktime_t dt)
 {
 	if (valid)
-		seq_printf(m, "\t%d", jiffies_to_msecs(dt));
+		seq_printf(m, "\t%d", (int)ktime_to_ms(dt));
 	else
-		seq_printf(m, "\t-");
+		seq_puts(m, "\t-");
 }
+#endif
 
 static void __seq_print_rq_state_bit(struct seq_file *m,
 	bool is_set, char *sep, const char *set_name, const char *unset_name)
 {
 	if (is_set && set_name) {
-		seq_putc(m, *sep);
+		if (*sep)
+			seq_putc(m, *sep);
 		seq_puts(m, set_name);
 		*sep = '|';
 	} else if (!is_set && unset_name) {
-		seq_putc(m, *sep);
+		if (*sep)
+			seq_putc(m, *sep);
 		seq_puts(m, unset_name);
 		*sep = '|';
 	}
@@ -53,17 +76,20 @@ static void seq_print_rq_state_bit(struct seq_file *m,
 /* pretty print enum drbd_req_state_bits req->rq_state */
 static void seq_print_request_state(struct seq_file *m, struct drbd_request *req)
 {
-	unsigned int s = req->rq_state;
+	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device;
+	unsigned int s = req->local_rq_state;
 	char sep = ' ';
 	seq_printf(m, "\t0x%08x", s);
-	seq_printf(m, "\tmaster: %s", req->master_bio ? "pending" : "completed");
+	seq_puts(m, "\tmaster:");
+	__seq_print_rq_state_bit(m, req->master_bio, &sep, "pending", "completed");
+	seq_print_rq_state_bit(m, s & RQ_POSTPONED, &sep, "postponed");
+	seq_print_rq_state_bit(m, s & RQ_COMPLETION_SUSP, &sep, "suspended");
 
 	/* RQ_WRITE ignored, already reported */
 	seq_puts(m, "\tlocal:");
-	seq_print_rq_state_bit(m, s & RQ_IN_ACT_LOG, &sep, "in-AL");
-	seq_print_rq_state_bit(m, s & RQ_POSTPONED, &sep, "postponed");
-	seq_print_rq_state_bit(m, s & RQ_COMPLETION_SUSP, &sep, "suspended");
 	sep = ' ';
+	seq_print_rq_state_bit(m, s & RQ_IN_ACT_LOG, &sep, "in-AL");
 	seq_print_rq_state_bit(m, s & RQ_LOCAL_PENDING, &sep, "pending");
 	seq_print_rq_state_bit(m, s & RQ_LOCAL_COMPLETED, &sep, "completed");
 	seq_print_rq_state_bit(m, s & RQ_LOCAL_ABORTED, &sep, "aborted");
@@ -71,64 +97,99 @@ static void seq_print_request_state(struct seq_file *m, struct drbd_request *req
 	if (sep == ' ')
 		seq_puts(m, " -");
 
-	/* for_each_connection ... */
-	seq_printf(m, "\tnet:");
-	sep = ' ';
-	seq_print_rq_state_bit(m, s & RQ_NET_PENDING, &sep, "pending");
-	seq_print_rq_state_bit(m, s & RQ_NET_QUEUED, &sep, "queued");
-	seq_print_rq_state_bit(m, s & RQ_NET_SENT, &sep, "sent");
-	seq_print_rq_state_bit(m, s & RQ_NET_DONE, &sep, "done");
-	seq_print_rq_state_bit(m, s & RQ_NET_SIS, &sep, "sis");
-	seq_print_rq_state_bit(m, s & RQ_NET_OK, &sep, "ok");
-	if (sep == ' ')
-		seq_puts(m, " -");
+	for_each_peer_device(peer_device, device) {
+		s = req->net_rq_state[peer_device->node_id];
+		seq_printf(m, "\tnet[%d]:", peer_device->node_id);
+		sep = ' ';
+		seq_print_rq_state_bit(m, s & RQ_NET_PENDING, &sep, "pending");
+		seq_print_rq_state_bit(m, s & RQ_NET_PENDING_OOS, &sep, "pending-oos");
+		seq_print_rq_state_bit(m, s & RQ_NET_QUEUED, &sep, "queued");
+		seq_print_rq_state_bit(m, s & RQ_NET_READY, &sep, "ready");
+		seq_print_rq_state_bit(m, s & RQ_NET_SENT, &sep, "sent");
+		seq_print_rq_state_bit(m, s & RQ_NET_DONE, &sep, "done");
+		seq_print_rq_state_bit(m, s & RQ_NET_SIS, &sep, "sis");
+		seq_print_rq_state_bit(m, s & RQ_NET_OK, &sep, "ok");
+		if (sep == ' ')
+			seq_puts(m, " -");
+
+		seq_puts(m, " :");
+		sep = ' ';
+		seq_print_rq_state_bit(m, s & RQ_EXP_RECEIVE_ACK, &sep, "B");
+		seq_print_rq_state_bit(m, s & RQ_EXP_WRITE_ACK, &sep, "C");
+		seq_print_rq_state_bit(m, s & RQ_EXP_BARR_ACK, &sep, "barr");
+		if (sep == ' ')
+			seq_puts(m, " -");
+	}
+	seq_putc(m, '\n');
+}
 
-	seq_printf(m, " :");
-	sep = ' ';
-	seq_print_rq_state_bit(m, s & RQ_EXP_RECEIVE_ACK, &sep, "B");
-	seq_print_rq_state_bit(m, s & RQ_EXP_WRITE_ACK, &sep, "C");
-	seq_print_rq_state_bit(m, s & RQ_EXP_BARR_ACK, &sep, "barr");
-	if (sep == ' ')
-		seq_puts(m, " -");
-	seq_printf(m, "\n");
+#define memberat(PTR, TYPE, OFFSET) (*(TYPE *)((char *)PTR + OFFSET))
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+static void print_one_age_or_dash(struct seq_file *m, struct drbd_request *req,
+				  unsigned int set_mask, unsigned int clear_mask,
+				  ktime_t now, size_t offset)
+{
+	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device) {
+		unsigned int s = req->net_rq_state[peer_device->node_id];
+
+		if (s & set_mask && !(s & clear_mask)) {
+			ktime_t ktime = ktime_sub(now, memberat(req, ktime_t, offset));
+			seq_printf(m, "\t[%d]%d", peer_device->node_id, (int)ktime_to_ms(ktime));
+			return;
+		}
+	}
+	seq_puts(m, "\t-");
 }
+#endif
 
-static void seq_print_one_request(struct seq_file *m, struct drbd_request *req, unsigned long now)
+static void seq_print_one_request(struct seq_file *m, struct drbd_request *req, ktime_t now, unsigned long jif)
 {
 	/* change anything here, fixup header below! */
-	unsigned int s = req->rq_state;
+	unsigned int s = req->local_rq_state;
+	unsigned long flags;
 
+	spin_lock_irqsave(&req->rq_lock, flags);
 #define RQ_HDR_1 "epoch\tsector\tsize\trw"
 	seq_printf(m, "0x%x\t%llu\t%u\t%s",
 		req->epoch,
 		(unsigned long long)req->i.sector, req->i.size >> 9,
 		(s & RQ_WRITE) ? "W" : "R");
 
+#ifdef CONFIG_DRBD_TIMING_STATS
 #define RQ_HDR_2 "\tstart\tin AL\tsubmit"
-	seq_printf(m, "\t%d", jiffies_to_msecs(now - req->start_jif));
-	seq_print_age_or_dash(m, s & RQ_IN_ACT_LOG, now - req->in_actlog_jif);
-	seq_print_age_or_dash(m, s & RQ_LOCAL_PENDING, now - req->pre_submit_jif);
+	seq_printf(m, "\t%d", (int)ktime_to_ms(ktime_sub(now, req->start_kt)));
+	seq_print_age_or_dash(m, s & RQ_IN_ACT_LOG, ktime_sub(now, req->in_actlog_kt));
+	seq_print_age_or_dash(m, s & RQ_LOCAL_PENDING, ktime_sub(now, req->pre_submit_kt));
 
 #define RQ_HDR_3 "\tsent\tacked\tdone"
-	seq_print_age_or_dash(m, s & RQ_NET_SENT, now - req->pre_send_jif);
-	seq_print_age_or_dash(m, (s & RQ_NET_SENT) && !(s & RQ_NET_PENDING), now - req->acked_jif);
-	seq_print_age_or_dash(m, s & RQ_NET_DONE, now - req->net_done_jif);
-
+	print_one_age_or_dash(m, req, RQ_NET_SENT, 0, now, offsetof(typeof(*req), pre_send_kt));
+	print_one_age_or_dash(m, req, RQ_NET_SENT, RQ_NET_PENDING, now, offsetof(typeof(*req), acked_kt));
+	print_one_age_or_dash(m, req, RQ_NET_DONE, 0, now, offsetof(typeof(*req), net_done_kt));
+#else
+#define RQ_HDR_2 "\tstart"
+#define RQ_HDR_3 ""
+	seq_printf(m, "\t%d", (int)jiffies_to_msecs(jif - req->start_jif));
+#endif
 #define RQ_HDR_4 "\tstate\n"
 	seq_print_request_state(m, req);
+	spin_unlock_irqrestore(&req->rq_lock, flags);
 }
 #define RQ_HDR RQ_HDR_1 RQ_HDR_2 RQ_HDR_3 RQ_HDR_4
 
-static void seq_print_minor_vnr_req(struct seq_file *m, struct drbd_request *req, unsigned long now)
+static void seq_print_minor_vnr_req(struct seq_file *m, struct drbd_request *req, ktime_t now, unsigned long jif)
 {
 	seq_printf(m, "%u\t%u\t", req->device->minor, req->device->vnr);
-	seq_print_one_request(m, req, now);
+	seq_print_one_request(m, req, now, jif);
 }
 
-static void seq_print_resource_pending_meta_io(struct seq_file *m, struct drbd_resource *resource, unsigned long now)
+static void seq_print_resource_pending_meta_io(struct seq_file *m, struct drbd_resource *resource, unsigned long jif)
 {
 	struct drbd_device *device;
-	unsigned int i;
+	int i;
 
 	seq_puts(m, "minor\tvnr\tstart\tsubmit\tintent\n");
 	rcu_read_lock();
@@ -142,45 +203,46 @@ static void seq_print_resource_pending_meta_io(struct seq_file *m, struct drbd_r
 		if (atomic_read(&tmp.in_use)) {
 			seq_printf(m, "%u\t%u\t%d\t",
 				device->minor, device->vnr,
-				jiffies_to_msecs(now - tmp.start_jif));
+				jiffies_to_msecs(jif - tmp.start_jif));
 			if (time_before(tmp.submit_jif, tmp.start_jif))
 				seq_puts(m, "-\t");
 			else
-				seq_printf(m, "%d\t", jiffies_to_msecs(now - tmp.submit_jif));
+				seq_printf(m, "%d\t", jiffies_to_msecs(jif - tmp.submit_jif));
 			seq_printf(m, "%s\n", tmp.current_use);
 		}
 	}
 	rcu_read_unlock();
 }
 
-static void seq_print_waiting_for_AL(struct seq_file *m, struct drbd_resource *resource, unsigned long now)
+static void seq_print_waiting_for_AL(struct seq_file *m, struct drbd_resource *resource, ktime_t now, unsigned long jif)
 {
 	struct drbd_device *device;
-	unsigned int i;
+	int i;
 
 	seq_puts(m, "minor\tvnr\tage\t#waiting\n");
 	rcu_read_lock();
 	idr_for_each_entry(&resource->devices, device, i) {
-		unsigned long jif;
 		struct drbd_request *req;
 		int n = atomic_read(&device->ap_actlog_cnt);
 		if (n) {
-			spin_lock_irq(&device->resource->req_lock);
+			spin_lock_irq(&device->pending_completion_lock);
 			req = list_first_entry_or_null(&device->pending_master_completion[1],
 				struct drbd_request, req_pending_master_completion);
 			/* if the oldest request does not wait for the activity log
 			 * it is not interesting for us here */
-			if (req && !(req->rq_state & RQ_IN_ACT_LOG))
-				jif = req->start_jif;
-			else
+			if (req && (req->local_rq_state & RQ_IN_ACT_LOG))
 				req = NULL;
-			spin_unlock_irq(&device->resource->req_lock);
+			spin_unlock_irq(&device->pending_completion_lock);
 		}
 		if (n) {
 			seq_printf(m, "%u\t%u\t", device->minor, device->vnr);
-			if (req)
-				seq_printf(m, "%u\t", jiffies_to_msecs(now - jif));
-			else
+			if (req) {
+#ifdef CONFIG_DRBD_TIMING_STATS
+				seq_printf(m, "%d\t", (int)ktime_to_ms(ktime_sub(now, req->start_kt)));
+#else
+				seq_printf(m, "%d\t", (int)jiffies_to_msecs(jif - req->start_jif));
+#endif
+			} else
 				seq_puts(m, "-\t");
 			seq_printf(m, "%u\n", n);
 		}
@@ -188,13 +250,13 @@ static void seq_print_waiting_for_AL(struct seq_file *m, struct drbd_resource *r
 	rcu_read_unlock();
 }
 
-static void seq_print_device_bitmap_io(struct seq_file *m, struct drbd_device *device, unsigned long now)
+static void seq_print_device_bitmap_io(struct seq_file *m, struct drbd_device *device, unsigned long jif)
 {
 	struct drbd_bm_aio_ctx *ctx;
 	unsigned long start_jif;
 	unsigned int in_flight;
 	unsigned int flags;
-	spin_lock_irq(&device->resource->req_lock);
+	spin_lock_irq(&device->pending_bmio_lock);
 	ctx = list_first_entry_or_null(&device->pending_bitmap_io, struct drbd_bm_aio_ctx, list);
 	if (ctx && ctx->done)
 		ctx = NULL;
@@ -203,25 +265,25 @@ static void seq_print_device_bitmap_io(struct seq_file *m, struct drbd_device *d
 		in_flight = atomic_read(&ctx->in_flight);
 		flags = ctx->flags;
 	}
-	spin_unlock_irq(&device->resource->req_lock);
+	spin_unlock_irq(&device->pending_bmio_lock);
 	if (ctx) {
 		seq_printf(m, "%u\t%u\t%c\t%u\t%u\n",
 			device->minor, device->vnr,
 			(flags & BM_AIO_READ) ? 'R' : 'W',
-			jiffies_to_msecs(now - start_jif),
+			jiffies_to_msecs(jif - start_jif),
 			in_flight);
 	}
 }
 
-static void seq_print_resource_pending_bitmap_io(struct seq_file *m, struct drbd_resource *resource, unsigned long now)
+static void seq_print_resource_pending_bitmap_io(struct seq_file *m, struct drbd_resource *resource, unsigned long jif)
 {
 	struct drbd_device *device;
-	unsigned int i;
+	int i;
 
 	seq_puts(m, "minor\tvnr\trw\tage\t#in-flight\n");
 	rcu_read_lock();
 	idr_for_each_entry(&resource->devices, device, i) {
-		seq_print_device_bitmap_io(m, device, now);
+		seq_print_device_bitmap_io(m, device, jif);
 	}
 	rcu_read_unlock();
 }
@@ -230,104 +292,195 @@ static void seq_print_resource_pending_bitmap_io(struct seq_file *m, struct drbd
 static void seq_print_peer_request_flags(struct seq_file *m, struct drbd_peer_request *peer_req)
 {
 	unsigned long f = peer_req->flags;
-	char sep = ' ';
-
-	__seq_print_rq_state_bit(m, f & EE_SUBMITTED, &sep, "submitted", "preparing");
-	__seq_print_rq_state_bit(m, f & EE_APPLICATION, &sep, "application", "internal");
-	seq_print_rq_state_bit(m, f & EE_CALL_AL_COMPLETE_IO, &sep, "in-AL");
+	char sep = 0;
+
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &peer_req->i.flags),
+			&sep, "submit-conflict-queued");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_SUBMITTED, &peer_req->i.flags),
+			&sep, "submitted");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_CONFLICT, &peer_req->i.flags),
+			&sep, "conflict");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_SENT, &peer_req->i.flags),
+			&sep, "sent");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_RECEIVED, &peer_req->i.flags),
+			&sep, "received");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_BACKING_COMPLETED, &peer_req->i.flags),
+			&sep, "backing-completed");
+	seq_print_rq_state_bit(m, test_bit(INTERVAL_COMPLETED, &peer_req->i.flags),
+			&sep, "completed");
+	seq_print_rq_state_bit(m, f & EE_IS_BARRIER, &sep, "barr");
 	seq_print_rq_state_bit(m, f & EE_SEND_WRITE_ACK, &sep, "C");
 	seq_print_rq_state_bit(m, f & EE_MAY_SET_IN_SYNC, &sep, "set-in-sync");
+	seq_print_rq_state_bit(m, f & EE_SET_OUT_OF_SYNC, &sep, "set-out-of-sync");
+	seq_print_rq_state_bit(m, peer_req->i.type == INTERVAL_PEER_WRITE && !(f & EE_IN_ACTLOG), &sep, "blocked-on-al");
 	seq_print_rq_state_bit(m, f & EE_TRIM, &sep, "trim");
 	seq_print_rq_state_bit(m, f & EE_ZEROOUT, &sep, "zero-out");
 	seq_print_rq_state_bit(m, f & EE_WRITE_SAME, &sep, "write-same");
 	seq_putc(m, '\n');
 }
 
-static void seq_print_peer_request(struct seq_file *m,
-	struct drbd_device *device, struct list_head *lh,
-	unsigned long now)
+enum drbd_peer_request_state {
+	PRS_NEW,
+	PRS_SENT,
+	PRS_SUBMITTED,
+	PRS_LAST,
+};
+
+static enum drbd_peer_request_state drbd_get_peer_request_state(struct drbd_peer_request *peer_req)
 {
-	bool reported_preparing = false;
+	unsigned long interval_flags = peer_req->i.flags;
+
+	if (interval_flags & INTERVAL_SUBMITTED)
+		return PRS_SUBMITTED;
+
+	if (interval_flags & INTERVAL_SENT)
+		return PRS_SENT;
+
+	return PRS_NEW;
+}
+
+static void seq_print_peer_request_one(struct seq_file *m,
+	struct drbd_peer_request *peer_req,
+	const char *list_name, unsigned long jif)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device ? peer_device->device : NULL;
+
+	seq_printf(m, "%s\t", list_name);
+
+	if (device)
+		seq_printf(m, "%u\t%u\t", device->minor, device->vnr);
+
+	seq_printf(m, "%llu\t%u\t%s\t%u\t",
+			(unsigned long long)peer_req->i.sector, peer_req->i.size >> 9,
+			drbd_interval_type_str(&peer_req->i),
+			jiffies_to_msecs(jif - peer_req->submit_jif));
+	seq_print_peer_request_flags(m, peer_req);
+}
+
+static void seq_print_peer_request_w(struct seq_file *m,
+	struct drbd_connection *connection, struct list_head *lh,
+	const char *list_name, unsigned long jif)
+{
+	int count[PRS_LAST] = {0};
 	struct drbd_peer_request *peer_req;
+
 	list_for_each_entry(peer_req, lh, w.list) {
-		if (reported_preparing && !(peer_req->flags & EE_SUBMITTED))
-			continue;
+		enum drbd_peer_request_state state = drbd_get_peer_request_state(peer_req);
 
-		if (device)
-			seq_printf(m, "%u\t%u\t", device->minor, device->vnr);
+		count[state]++;
+		if (count[state] <= 16)
+			seq_print_peer_request_one(m, peer_req, list_name, jif);
+	}
+}
 
-		seq_printf(m, "%llu\t%u\t%c\t%u\t",
-			(unsigned long long)peer_req->i.sector, peer_req->i.size >> 9,
-			(peer_req->flags & EE_WRITE) ? 'W' : 'R',
-			jiffies_to_msecs(now - peer_req->submit_jif));
-		seq_print_peer_request_flags(m, peer_req);
-		if (peer_req->flags & EE_SUBMITTED)
-			break;
-		else
-			reported_preparing = true;
+static void seq_print_peer_request(struct seq_file *m,
+	struct drbd_connection *connection, struct list_head *lh,
+	const char *list_name, unsigned long jif)
+{
+	int count = 0;
+	struct drbd_peer_request *peer_req;
+
+	list_for_each_entry(peer_req, lh, recv_order) {
+		count++;
+		if (count <= 16)
+			seq_print_peer_request_one(m, peer_req, list_name, jif);
 	}
 }
 
-static void seq_print_device_peer_requests(struct seq_file *m,
-	struct drbd_device *device, unsigned long now)
+static void seq_print_connection_peer_requests(struct seq_file *m,
+	struct drbd_connection *connection, unsigned long jif)
+{
+	struct drbd_peer_device *peer_device;
+	int i;
+
+	seq_printf(m, "list\t\tminor\tvnr\tsector\tsize\ttype\t\tage\tflags\n");
+	spin_lock_irq(&connection->peer_reqs_lock);
+	seq_print_peer_request_w(m, connection, &connection->done_ee, "done\t", jif);
+	seq_print_peer_request_w(m, connection, &connection->dagtag_wait_ee, "dagtag_wait", jif);
+	seq_print_peer_request_w(m, connection, &connection->resync_ack_ee, "resync_ack", jif);
+	seq_print_peer_request(m, connection, &connection->peer_requests, "peer_requests", jif);
+	seq_print_peer_request(m, connection, &connection->peer_reads, "peer_reads", jif);
+	idr_for_each_entry(&connection->peer_devices, peer_device, i)
+		seq_print_peer_request(m, connection, &peer_device->resync_requests,
+				"resync_requests", jif);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+}
+
+static void seq_print_device_peer_flushes(struct seq_file *m,
+	struct drbd_device *device, unsigned long jif)
 {
-	seq_puts(m, "minor\tvnr\tsector\tsize\trw\tage\tflags\n");
-	spin_lock_irq(&device->resource->req_lock);
-	seq_print_peer_request(m, device, &device->active_ee, now);
-	seq_print_peer_request(m, device, &device->read_ee, now);
-	seq_print_peer_request(m, device, &device->sync_ee, now);
-	spin_unlock_irq(&device->resource->req_lock);
 	if (test_bit(FLUSH_PENDING, &device->flags)) {
 		seq_printf(m, "%u\t%u\t-\t-\tF\t%u\tflush\n",
 			device->minor, device->vnr,
-			jiffies_to_msecs(now - device->flush_jif));
+			jiffies_to_msecs(jif - device->flush_jif));
 	}
 }
 
 static void seq_print_resource_pending_peer_requests(struct seq_file *m,
-	struct drbd_resource *resource, unsigned long now)
+	struct drbd_resource *resource, unsigned long jif)
 {
+	struct drbd_connection *connection;
 	struct drbd_device *device;
-	unsigned int i;
+	int i;
 
 	rcu_read_lock();
+
+	for_each_connection_rcu(connection, resource) {
+		seq_printf(m, "oldest peer requests (peer: %s)\n",
+			rcu_dereference(connection->transport.net_conf)->name);
+		seq_print_connection_peer_requests(m, connection, jif);
+		seq_putc(m, '\n');
+	}
+
+	seq_puts(m, "flushes\n");
 	idr_for_each_entry(&resource->devices, device, i) {
-		seq_print_device_peer_requests(m, device, now);
+		seq_print_device_peer_flushes(m, device, jif);
 	}
+	seq_putc(m, '\n');
+
 	rcu_read_unlock();
 }
 
 static void seq_print_resource_transfer_log_summary(struct seq_file *m,
 	struct drbd_resource *resource,
-	struct drbd_connection *connection,
-	unsigned long now)
+	ktime_t now, unsigned long jif)
 {
 	struct drbd_request *req;
 	unsigned int count = 0;
 	unsigned int show_state = 0;
 
 	seq_puts(m, "n\tdevice\tvnr\t" RQ_HDR);
-	spin_lock_irq(&resource->req_lock);
-	list_for_each_entry(req, &connection->transfer_log, tl_requests) {
+	rcu_read_lock();
+	list_for_each_entry_rcu(req, &resource->transfer_log, tl_requests) {
+		struct drbd_device *device = req->device;
+		struct drbd_peer_device *peer_device;
 		unsigned int tmp = 0;
 		unsigned int s;
-		++count;
 
-		/* don't disable irq "forever" */
-		if (!(count & 0x1ff)) {
-			struct drbd_request *req_next;
-			kref_get(&req->kref);
-			spin_unlock_irq(&resource->req_lock);
+		/* don't disable preemption "forever" */
+		if ((count & 0x1ff) == 0x1ff) {
+			struct list_head *next_hdr;
+			/* Only get if the request hasn't already been removed from transfer_log. */
+			if (!refcount_inc_not_zero(&req->oos_send_ref))
+				continue;
+			rcu_read_unlock();
 			cond_resched();
-			spin_lock_irq(&resource->req_lock);
-			req_next = list_next_entry(req, tl_requests);
-			if (kref_put(&req->kref, drbd_req_destroy))
-				req = req_next;
-			if (&req->tl_requests == &connection->transfer_log)
-				break;
+			rcu_read_lock();
+			next_hdr = rcu_dereference(list_next_rcu(&req->tl_requests));
+			drbd_put_ref_tl_walk(req, 0, 1);
+			if (!refcount_read(&req->done_ref)) {
+				if (next_hdr == &resource->transfer_log)
+					break;
+				req = list_entry_rcu(next_hdr,
+						     struct drbd_request,
+						     tl_requests);
+			}
 		}
+		++count;
 
-		s = req->rq_state;
+		spin_lock_irq(&req->rq_lock);
+		s = req->local_rq_state;
 
 		/* This is meant to summarize timing issues, to be able to tell
 		 * local disk problems from network problems.
@@ -337,40 +490,43 @@ static void seq_print_resource_transfer_log_summary(struct seq_file *m,
 			tmp |= 1;
 		if ((s & RQ_LOCAL_MASK) && (s & RQ_LOCAL_PENDING))
 			tmp |= 2;
-		if (s & RQ_NET_MASK) {
-			if (!(s & RQ_NET_SENT))
-				tmp |= 4;
-			if (s & RQ_NET_PENDING)
-				tmp |= 8;
-			if (!(s & RQ_NET_DONE))
-				tmp |= 16;
+
+		for_each_peer_device_rcu(peer_device, device) {
+			s = READ_ONCE(req->net_rq_state[peer_device->node_id]);
+			if (s & RQ_NET_MASK) {
+				if (!(s & RQ_NET_SENT))
+					tmp |= 4;
+				if (s & RQ_NET_PENDING)
+					tmp |= 8;
+				if (!(s & RQ_NET_DONE))
+					tmp |= 16;
+			}
 		}
+		spin_unlock_irq(&req->rq_lock);
+
 		if ((tmp & show_state) == tmp)
 			continue;
 		show_state |= tmp;
 		seq_printf(m, "%u\t", count);
-		seq_print_minor_vnr_req(m, req, now);
+		seq_print_minor_vnr_req(m, req, now, jif);
 		if (show_state == 0x1f)
 			break;
 	}
-	spin_unlock_irq(&resource->req_lock);
+	rcu_read_unlock();
+	seq_printf(m, "%u total\n", count);
 }
 
-/* TODO: transfer_log and friends should be moved to resource */
-static int in_flight_summary_show(struct seq_file *m, void *pos)
+static int resource_in_flight_summary_show(struct seq_file *m, void *pos)
 {
 	struct drbd_resource *resource = m->private;
 	struct drbd_connection *connection;
+	struct drbd_transport *transport;
+	struct drbd_transport_stats transport_stats;
+	ktime_t now = ktime_get();
 	unsigned long jif = jiffies;
 
-	connection = first_connection(resource);
-	/* This does not happen, actually.
-	 * But be robust and prepare for future code changes. */
-	if (!connection || !kref_get_unless_zero(&connection->kref))
-		return -ESTALE;
-
 	/* BUMP me if you change the file format/content/presentation */
-	seq_printf(m, "v: %u\n\n", 0);
+	seq_printf(m, "v: %u\n\n", 1);
 
 	seq_puts(m, "oldest bitmap IO\n");
 	seq_print_resource_pending_bitmap_io(m, resource, jif);
@@ -380,37 +536,125 @@ static int in_flight_summary_show(struct seq_file *m, void *pos)
 	seq_print_resource_pending_meta_io(m, resource, jif);
 	seq_putc(m, '\n');
 
-	seq_puts(m, "socket buffer stats\n");
-	/* for each connection ... once we have more than one */
+	seq_puts(m, "transport buffer stats\n");
+	seq_puts(m, "peer\ttransport class\tunread receive buffer\tunacked send buffer\n");
 	rcu_read_lock();
-	if (connection->data.socket) {
-		/* open coded SIOCINQ, the "relevant" part */
-		struct tcp_sock *tp = tcp_sk(connection->data.socket->sk);
-		int answ = tp->rcv_nxt - tp->copied_seq;
-		seq_printf(m, "unread receive buffer: %u Byte\n", answ);
-		/* open coded SIOCOUTQ, the "relevant" part */
-		answ = tp->write_seq - tp->snd_una;
-		seq_printf(m, "unacked send buffer: %u Byte\n", answ);
+	for_each_connection_rcu(connection, resource) {
+		char *name;
+
+		transport = &connection->transport;
+		name = rcu_dereference(transport->net_conf)->name;
+		seq_printf(m, "%s\t%s\t", name, transport->class->name);
+
+		if (transport->class->ops.stream_ok(transport, DATA_STREAM)) {
+			transport->class->ops.stats(transport, &transport_stats);
+			seq_printf(m, "%u\t%u\n",
+				transport_stats.unread_received,
+				transport_stats.unacked_send);
+		} else {
+			seq_printf(m, "-\t-\n");
+		}
 	}
 	rcu_read_unlock();
 	seq_putc(m, '\n');
 
-	seq_puts(m, "oldest peer requests\n");
 	seq_print_resource_pending_peer_requests(m, resource, jif);
-	seq_putc(m, '\n');
 
 	seq_puts(m, "application requests waiting for activity log\n");
-	seq_print_waiting_for_AL(m, resource, jif);
+	seq_print_waiting_for_AL(m, resource, now, jif);
 	seq_putc(m, '\n');
 
 	seq_puts(m, "oldest application requests\n");
-	seq_print_resource_transfer_log_summary(m, resource, connection, jif);
+	seq_print_resource_transfer_log_summary(m, resource, now, jif);
 	seq_putc(m, '\n');
 
 	jif = jiffies - jif;
 	if (jif)
 		seq_printf(m, "generated in %d ms\n", jiffies_to_msecs(jif));
-	kref_put(&connection->kref, drbd_destroy_connection);
+	return 0;
+}
+
+static int resource_state_twopc_show(struct seq_file *m, void *pos)
+{
+	struct drbd_resource *resource = m->private;
+	struct twopc_reply twopc;
+	bool active = false;
+	unsigned long jif;
+
+	read_lock_irq(&resource->state_rwlock);
+	if (resource->remote_state_change) {
+		twopc = resource->twopc_reply;
+		active = true;
+	}
+	read_unlock_irq(&resource->state_rwlock);
+
+	seq_printf(m, "v: %u\n\n", 1);
+	if (active) {
+		struct drbd_connection *connection;
+
+		seq_printf(m,
+			   "Executing tid: %u\n"
+			   "  initiator_node_id: %d\n"
+			   "  target_node_id: %d\n",
+			   twopc.tid, twopc.initiator_node_id,
+			   twopc.target_node_id);
+
+		if (twopc.initiator_node_id != resource->res_opts.node_id) {
+			seq_puts(m, "  parent node mask: ");
+			seq_print_node_mask(m, resource, resource->twopc_parent_nodes);
+
+			if (resource->twopc_prepare_reply_cmd)
+				seq_printf(m,
+					   "  Reply sent: %s\n",
+					   resource->twopc_prepare_reply_cmd == P_TWOPC_YES ? "yes" :
+					   resource->twopc_prepare_reply_cmd == P_TWOPC_NO ? "no" :
+					   resource->twopc_prepare_reply_cmd == P_TWOPC_RETRY ? "retry" :
+					   "else!?!");
+		}
+
+		seq_puts(m, "  received replies: ");
+		rcu_read_lock();
+		for_each_connection_rcu(connection, resource) {
+			char *name = rcu_dereference((connection)->transport.net_conf)->name;
+
+			if (!test_bit(TWOPC_PREPARED, &connection->flags))
+				/* seq_printf(m, "%s n.p., ", name) * print nothing! */;
+			else if (test_bit(TWOPC_NO, &connection->flags))
+				seq_printf(m, "%s no, ", name);
+			else if (test_bit(TWOPC_RETRY, &connection->flags))
+				seq_printf(m, "%s ret, ", name);
+			else if (test_bit(TWOPC_YES, &connection->flags))
+				seq_printf(m, "%s yes, ", name);
+			else
+				seq_printf(m, "%s ___, ", name);
+		}
+		rcu_read_unlock();
+		seq_puts(m, "\n");
+		if (twopc.initiator_node_id != resource->res_opts.node_id) {
+			/* The timer is only relevant for twopcs initiated by other nodes */
+			jif = resource->twopc_timer.expires - jiffies;
+			seq_printf(m, "  timer expires in: %d ms\n", jiffies_to_msecs(jif));
+		}
+	} else {
+		seq_puts(m, "No ongoing two phase state transaction\n");
+	}
+
+	return 0;
+}
+
+static int resource_worker_pid_show(struct seq_file *m, void *pos)
+{
+	struct drbd_resource *resource = m->private;
+	if (resource->worker.task)
+		seq_printf(m, "%d\n", resource->worker.task->pid);
+	return 0;
+}
+
+static int resource_members_show(struct seq_file *m, void *pos)
+{
+	struct drbd_resource *resource = m->private;
+
+	seq_printf(m, "0x%016llX\n", resource->members);
 	return 0;
 }
 
@@ -425,6 +669,9 @@ static int drbd_single_open(struct file *file, int (*show)(struct seq_file *, vo
 	/* Are we still linked,
 	 * or has debugfs_remove() already been called? */
 	parent = file->f_path.dentry->d_parent;
+	/* not sure if this can happen: */
+	if (!parent || !parent->d_inode)
+		goto out;
 	/* serialize with d_delete() */
 	inode_lock(d_inode(parent));
 	/* Make sure the object is still alive */
@@ -437,31 +684,55 @@ static int drbd_single_open(struct file *file, int (*show)(struct seq_file *, vo
 		if (ret)
 			kref_put(kref, release);
 	}
+out:
 	return ret;
 }
 
-static int in_flight_summary_open(struct inode *inode, struct file *file)
-{
-	struct drbd_resource *resource = inode->i_private;
-	return drbd_single_open(file, in_flight_summary_show, resource,
-				&resource->kref, drbd_destroy_resource);
-}
-
-static int in_flight_summary_release(struct inode *inode, struct file *file)
+static int resource_attr_release(struct inode *inode, struct file *file)
 {
 	struct drbd_resource *resource = inode->i_private;
 	kref_put(&resource->kref, drbd_destroy_resource);
 	return single_release(inode, file);
 }
 
-static const struct file_operations in_flight_summary_fops = {
-	.owner		= THIS_MODULE,
-	.open		= in_flight_summary_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= in_flight_summary_release,
+#define drbd_debugfs_resource_attr(name)				\
+static int resource_ ## name ## _open(struct inode *inode, struct file *file) \
+{									\
+	struct drbd_resource *resource = inode->i_private;		\
+	return drbd_single_open(file, resource_ ## name ## _show, resource, \
+				&resource->kref, drbd_destroy_resource); \
+}									\
+static const struct file_operations resource_ ## name ## _fops = {	\
+	.owner		= THIS_MODULE,					\
+	.open		= resource_ ## name ## _open,			\
+	.read		= seq_read,					\
+	.llseek		= seq_lseek,					\
+	.release	= resource_attr_release,			\
 };
 
+drbd_debugfs_resource_attr(in_flight_summary)
+drbd_debugfs_resource_attr(state_twopc)
+drbd_debugfs_resource_attr(worker_pid)
+drbd_debugfs_resource_attr(members)
+
+#define drbd_dcf(top, obj, attr, perm) do {			\
+	dentry = debugfs_create_file(#attr, perm,		\
+			top, obj, &obj ## _ ## attr ## _fops);	\
+	top ## _ ## attr = dentry;				\
+	} while (0)
+
+#define res_dcf(attr) \
+	drbd_dcf(resource->debugfs_res, resource, attr, 0400)
+
+#define conn_dcf(attr) \
+	drbd_dcf(connection->debugfs_conn, connection, attr, 0400)
+
+#define vol_dcf(attr) \
+	drbd_dcf(device->debugfs_vol, device, attr, 0400)
+
+#define peer_dev_dcf(attr) \
+	drbd_dcf(peer_device->debugfs_peer_dev, peer_device, attr, 0400)
+
 void drbd_debugfs_resource_add(struct drbd_resource *resource)
 {
 	struct dentry *dentry;
@@ -475,10 +746,11 @@ void drbd_debugfs_resource_add(struct drbd_resource *resource)
 	dentry = debugfs_create_dir("connections", resource->debugfs_res);
 	resource->debugfs_res_connections = dentry;
 
-	dentry = debugfs_create_file("in_flight_summary", 0440,
-				     resource->debugfs_res, resource,
-				     &in_flight_summary_fops);
-	resource->debugfs_res_in_flight_summary = dentry;
+	/* debugfs create file */
+	res_dcf(in_flight_summary);
+	res_dcf(state_twopc);
+	res_dcf(worker_pid);
+	res_dcf(members);
 }
 
 static void drbd_debugfs_remove(struct dentry **dp)
@@ -489,16 +761,35 @@ static void drbd_debugfs_remove(struct dentry **dp)
 
 void drbd_debugfs_resource_cleanup(struct drbd_resource *resource)
 {
+	/* Older kernels have a broken implementation of
+	 * debugfs_remove_recursive (prior to upstream commit 776164c1f)
+	 * That unfortunately includes a number of "enterprise" kernels.
+	 * Even older kernels do not even have the _recursive() helper at all.
+	 * For now, remember all debugfs nodes we created,
+	 * and call debugfs_remove on all of them separately.
+	 */
 	/* it is ok to call debugfs_remove(NULL) */
+	drbd_debugfs_remove(&resource->debugfs_res_members);
+	drbd_debugfs_remove(&resource->debugfs_res_worker_pid);
+	drbd_debugfs_remove(&resource->debugfs_res_state_twopc);
 	drbd_debugfs_remove(&resource->debugfs_res_in_flight_summary);
 	drbd_debugfs_remove(&resource->debugfs_res_connections);
 	drbd_debugfs_remove(&resource->debugfs_res_volumes);
 	drbd_debugfs_remove(&resource->debugfs_res);
 }
 
+void drbd_debugfs_resource_rename(struct drbd_resource *resource, const char *new_name)
+{
+	int err;
+
+	err = debugfs_change_name(resource->debugfs_res, "%s", new_name);
+	if (err)
+		drbd_err(resource, "failed to rename debugfs entry for resource\n");
+}
+
 static void seq_print_one_timing_detail(struct seq_file *m,
 	const struct drbd_thread_timing_details *tdp,
-	unsigned long now)
+	unsigned long jif)
 {
 	struct drbd_thread_timing_details td;
 	/* No locking...
@@ -510,14 +801,14 @@ static void seq_print_one_timing_detail(struct seq_file *m,
 		return;
 	seq_printf(m, "%u\t%d\t%s:%u\t%ps\n",
 			td.cb_nr,
-			jiffies_to_msecs(now - td.start_jif),
+			jiffies_to_msecs(jif - td.start_jif),
 			td.caller_fn, td.line,
 			td.cb_addr);
 }
 
 static void seq_print_timing_details(struct seq_file *m,
 		const char *title,
-		unsigned int cb_nr, struct drbd_thread_timing_details *tdp, unsigned long now)
+		unsigned int cb_nr, struct drbd_thread_timing_details *tdp, unsigned long jif)
 {
 	unsigned int start_idx;
 	unsigned int i;
@@ -529,135 +820,301 @@ static void seq_print_timing_details(struct seq_file *m,
 	 */
 	start_idx = cb_nr % DRBD_THREAD_DETAILS_HIST;
 	for (i = start_idx; i < DRBD_THREAD_DETAILS_HIST; i++)
-		seq_print_one_timing_detail(m, tdp+i, now);
+		seq_print_one_timing_detail(m, tdp+i, jif);
 	for (i = 0; i < start_idx; i++)
-		seq_print_one_timing_detail(m, tdp+i, now);
+		seq_print_one_timing_detail(m, tdp+i, jif);
 }
 
-static int callback_history_show(struct seq_file *m, void *ignored)
+static int connection_callback_history_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_connection *connection = m->private;
+	struct drbd_resource *resource = connection->resource;
 	unsigned long jif = jiffies;
 
 	/* BUMP me if you change the file format/content/presentation */
 	seq_printf(m, "v: %u\n\n", 0);
 
 	seq_puts(m, "n\tage\tcallsite\tfn\n");
-	seq_print_timing_details(m, "worker", connection->w_cb_nr, connection->w_timing_details, jif);
+	seq_print_timing_details(m, "sender", connection->s_cb_nr, connection->s_timing_details, jif);
 	seq_print_timing_details(m, "receiver", connection->r_cb_nr, connection->r_timing_details, jif);
+	seq_print_timing_details(m, "worker", resource->w_cb_nr, resource->w_timing_details, jif);
 	return 0;
 }
 
-static int callback_history_open(struct inode *inode, struct file *file)
-{
-	struct drbd_connection *connection = inode->i_private;
-	return drbd_single_open(file, callback_history_show, connection,
-				&connection->kref, drbd_destroy_connection);
-}
-
-static int callback_history_release(struct inode *inode, struct file *file)
-{
-	struct drbd_connection *connection = inode->i_private;
-	kref_put(&connection->kref, drbd_destroy_connection);
-	return single_release(inode, file);
-}
-
-static const struct file_operations connection_callback_history_fops = {
-	.owner		= THIS_MODULE,
-	.open		= callback_history_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= callback_history_release,
-};
-
 static int connection_oldest_requests_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_connection *connection = m->private;
-	unsigned long now = jiffies;
+	ktime_t now = ktime_get();
+	unsigned long jif = jiffies;
 	struct drbd_request *r1, *r2;
 
 	/* BUMP me if you change the file format/content/presentation */
 	seq_printf(m, "v: %u\n\n", 0);
 
-	spin_lock_irq(&connection->resource->req_lock);
-	r1 = connection->req_next;
+	rcu_read_lock();
+	r1 = READ_ONCE(connection->todo.req_next);
 	if (r1)
-		seq_print_minor_vnr_req(m, r1, now);
-	r2 = connection->req_ack_pending;
+		seq_print_minor_vnr_req(m, r1, now, jif);
+	r2 = READ_ONCE(connection->req_ack_pending);
 	if (r2 && r2 != r1) {
 		r1 = r2;
-		seq_print_minor_vnr_req(m, r1, now);
+		seq_print_minor_vnr_req(m, r1, now, jif);
 	}
-	r2 = connection->req_not_net_done;
+	r2 = READ_ONCE(connection->req_not_net_done);
 	if (r2 && r2 != r1)
-		seq_print_minor_vnr_req(m, r2, now);
-	spin_unlock_irq(&connection->resource->req_lock);
+		seq_print_minor_vnr_req(m, r2, now, jif);
+	rcu_read_unlock();
 	return 0;
 }
 
-static int connection_oldest_requests_open(struct inode *inode, struct file *file)
+static int connection_transport_show(struct seq_file *m, void *ignored)
 {
-	struct drbd_connection *connection = inode->i_private;
-	return drbd_single_open(file, connection_oldest_requests_show, connection,
-				&connection->kref, drbd_destroy_connection);
+	struct drbd_connection *connection = m->private;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+	enum drbd_stream i;
+
+	seq_printf(m, "v: %u\n\n", 0);
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+		struct drbd_send_buffer *sbuf = &connection->send_buffer[i];
+		seq_printf(m, "%s stream\n", i == DATA_STREAM ? "data" : "control");
+		seq_printf(m, "  corked: %d\n", test_bit(CORKED + i, &connection->flags));
+		seq_printf(m, "  unsent: %ld bytes\n", (long)(sbuf->pos - sbuf->unsent));
+		seq_printf(m, "  allocated: %d bytes\n", sbuf->allocated_size);
+	}
+
+	seq_printf(m, "\ntransport_type: %s\n", transport->class->name);
+
+	tr_ops->debugfs_show(transport, m);
+
+	return 0;
 }
 
-static int connection_oldest_requests_release(struct inode *inode, struct file *file)
+static int connection_debug_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_connection *connection = m->private;
+	struct drbd_resource *resource = connection->resource;
+	unsigned long flags = connection->flags;
+	unsigned int u1, u2;
+	unsigned long long ull1, ull2;
+	int in_flight;
+	char sep = ' ';
+
+	seq_puts(m, "content and format of this will change without notice\n");
+
+	seq_printf(m, "flags: 0x%04lx :", flags);
+#define pretty_print_bit(n) \
+	seq_print_rq_state_bit(m, test_bit(n, &flags), &sep, #n);
+	pretty_print_bit(PING_PENDING);
+	pretty_print_bit(TWOPC_PREPARED);
+	pretty_print_bit(TWOPC_YES);
+	pretty_print_bit(TWOPC_NO);
+	pretty_print_bit(TWOPC_RETRY);
+	pretty_print_bit(CONN_DRY_RUN);
+	pretty_print_bit(DISCONNECT_EXPECTED);
+	pretty_print_bit(BARRIER_ACK_PENDING);
+	pretty_print_bit(DATA_CORKED);
+	pretty_print_bit(CONTROL_CORKED);
+	pretty_print_bit(C_UNREGISTERED);
+	pretty_print_bit(RECONNECT);
+	pretty_print_bit(CONN_DISCARD_MY_DATA);
+	pretty_print_bit(SEND_STATE_AFTER_AHEAD_C);
+	pretty_print_bit(NOTIFY_PEERS_LOST_PRIMARY);
+	pretty_print_bit(CHECKING_PEER);
+	pretty_print_bit(CONN_CONGESTED);
+	pretty_print_bit(CONN_HANDSHAKE_DISCONNECT);
+	pretty_print_bit(CONN_HANDSHAKE_RETRY);
+	pretty_print_bit(CONN_HANDSHAKE_READY);
+#undef pretty_print_bit
+	seq_putc(m, '\n');
+
+	u1 = atomic_read(&resource->current_tle_nr);
+	u2 = connection->send.current_epoch_nr;
+	seq_printf(m, "resource->current_tle_nr: %u\n", u1);
+	seq_printf(m, "   send.current_epoch_nr: %u (%d)\n", u2, (int)(u2 - u1));
+
+	ull1 = resource->dagtag_sector;
+	ull2 = resource->last_peer_acked_dagtag;
+	seq_printf(m, " resource->dagtag_sector: %llu\n", ull1);
+	seq_printf(m, "  last_peer_acked_dagtag: %llu (%lld)\n", ull2, (long long)(ull2 - ull1));
+	ull2 = connection->send.current_dagtag_sector;
+	seq_printf(m, " send.current_dagtag_sec: %llu (%lld)\n", ull2, (long long)(ull2 - ull1));
+	ull2 = atomic64_read(&connection->last_dagtag_sector);
+	seq_printf(m, "      last_dagtag_sector: %llu\n", ull2);
+	seq_printf(m, "last_peer_ack_dagtag_seen: %llu\n",
+			(unsigned long long) connection->last_peer_ack_dagtag_seen);
+
+	spin_lock_irq(&resource->initiator_flush_lock);
+	seq_printf(m, "resource->current_flush_sequence: %llu\n",
+			(unsigned long long) resource->current_flush_sequence);
+	seq_puts(m, "      pending_flush_mask: ");
+	seq_print_node_mask(m, resource, connection->pending_flush_mask);
+	spin_unlock_irq(&resource->initiator_flush_lock);
+
+	spin_lock_irq(&connection->primary_flush_lock);
+	seq_printf(m, "   flush_requests_dagtag: %llu\n",
+			(unsigned long long) connection->flush_requests_dagtag);
+	seq_printf(m, "          flush_sequence: %llu\n",
+			(unsigned long long) connection->flush_sequence);
+	seq_puts(m, " flush_forward_sent_mask: ");
+	seq_print_node_mask(m, resource, connection->flush_forward_sent_mask);
+	spin_unlock_irq(&connection->primary_flush_lock);
+
+	spin_lock_irq(&connection->flush_ack_lock);
+	for (u1 = 0; u1 < DRBD_PEERS_MAX; u1++) {
+		if (connection->flush_ack_sequence[u1])
+			seq_printf(m, "      flush_ack_sequence[%u]: %llu\n", u1,
+					(unsigned long long) connection->flush_ack_sequence[u1]);
+	}
+	spin_unlock_irq(&connection->flush_ack_lock);
+
+	in_flight = atomic_read(&connection->ap_in_flight);
+	seq_printf(m, "            ap_in_flight: %d KiB (%d sectors)\n", in_flight / 2, in_flight);
+
+	in_flight = atomic_read(&connection->rs_in_flight);
+	seq_printf(m, "            rs_in_flight: %d KiB (%d sectors)\n", in_flight / 2, in_flight);
+
+	seq_printf(m, "             done_ee_cnt: %d\n"
+			"          backing_ee_cnt: %d\n"
+			"           active_ee_cnt: %d\n",
+			atomic_read(&connection->done_ee_cnt),
+			atomic_read(&connection->backing_ee_cnt),
+			atomic_read(&connection->active_ee_cnt));
+	seq_printf(m, "      agreed_pro_version: %d\n", connection->agreed_pro_version);
+	seq_printf(m, "            send control: %u bytes/pckt (%u bytes, %u pckts)\n",
+		   connection->ctl_bytes / (connection->ctl_packets ?: 1),
+		   connection->ctl_bytes, connection->ctl_packets);
+	return 0;
+}
+
+static void pid_show(struct seq_file *m, struct drbd_thread *thi)
+{
+	struct task_struct *task = NULL;
+	pid_t pid;
+
+	spin_lock_irq(&thi->t_lock);
+	task = thi->task;
+	if (task)
+		pid = task->pid;
+	spin_unlock_irq(&thi->t_lock);
+	if (task)
+		seq_printf(m, "%d\n", pid);
+}
+
+static int connection_receiver_pid_show(struct seq_file *m, void *pos)
+{
+	struct drbd_connection *connection = m->private;
+	pid_show(m, &connection->receiver);
+	return 0;
+}
+
+static int connection_sender_pid_show(struct seq_file *m, void *pos)
+{
+	struct drbd_connection *connection = m->private;
+	pid_show(m, &connection->sender);
+	return 0;
+}
+
+static int connection_attr_release(struct inode *inode, struct file *file)
 {
 	struct drbd_connection *connection = inode->i_private;
 	kref_put(&connection->kref, drbd_destroy_connection);
 	return single_release(inode, file);
 }
 
-static const struct file_operations connection_oldest_requests_fops = {
-	.owner		= THIS_MODULE,
-	.open		= connection_oldest_requests_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= connection_oldest_requests_release,
+#define drbd_debugfs_connection_attr(name)				\
+static int connection_ ## name ## _open(struct inode *inode, struct file *file) \
+{									\
+	struct drbd_connection *connection = inode->i_private;		\
+	return drbd_single_open(file, connection_ ## name ## _show,	\
+				connection, &connection->kref,		\
+				drbd_destroy_connection);		\
+}									\
+static const struct file_operations connection_ ## name ## _fops = {	\
+	.owner		= THIS_MODULE,				      	\
+	.open		= connection_ ## name ##_open,			\
+	.read		= seq_read,					\
+	.llseek		= seq_lseek,					\
+	.release	= connection_attr_release,			\
 };
 
+drbd_debugfs_connection_attr(oldest_requests)
+drbd_debugfs_connection_attr(callback_history)
+drbd_debugfs_connection_attr(transport)
+drbd_debugfs_connection_attr(debug)
+drbd_debugfs_connection_attr(receiver_pid)
+drbd_debugfs_connection_attr(sender_pid)
+
 void drbd_debugfs_connection_add(struct drbd_connection *connection)
 {
 	struct dentry *conns_dir = connection->resource->debugfs_res_connections;
+	struct drbd_peer_device *peer_device;
+	char conn_name[SHARED_SECRET_MAX];
 	struct dentry *dentry;
+	int vnr;
 
-	/* Once we enable mutliple peers,
-	 * these connections will have descriptive names.
-	 * For now, it is just the one connection to the (only) "peer". */
-	dentry = debugfs_create_dir("peer", conns_dir);
-	connection->debugfs_conn = dentry;
+	rcu_read_lock();
+	strscpy(conn_name, rcu_dereference(connection->transport.net_conf)->name);
+	rcu_read_unlock();
 
-	dentry = debugfs_create_file("callback_history", 0440,
-				     connection->debugfs_conn, connection,
-				     &connection_callback_history_fops);
-	connection->debugfs_conn_callback_history = dentry;
+	dentry = debugfs_create_dir(conn_name, conns_dir);
+	connection->debugfs_conn = dentry;
 
-	dentry = debugfs_create_file("oldest_requests", 0440,
-				     connection->debugfs_conn, connection,
-				     &connection_oldest_requests_fops);
-	connection->debugfs_conn_oldest_requests = dentry;
+	/* debugfs create file */
+	conn_dcf(callback_history);
+	conn_dcf(oldest_requests);
+	conn_dcf(transport);
+	conn_dcf(debug);
+	conn_dcf(receiver_pid);
+	conn_dcf(sender_pid);
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (!peer_device->debugfs_peer_dev)
+			drbd_debugfs_peer_device_add(peer_device);
+	}
 }
 
 void drbd_debugfs_connection_cleanup(struct drbd_connection *connection)
 {
+	drbd_debugfs_remove(&connection->debugfs_conn_sender_pid);
+	drbd_debugfs_remove(&connection->debugfs_conn_receiver_pid);
+	drbd_debugfs_remove(&connection->debugfs_conn_debug);
+	drbd_debugfs_remove(&connection->debugfs_conn_transport);
 	drbd_debugfs_remove(&connection->debugfs_conn_callback_history);
 	drbd_debugfs_remove(&connection->debugfs_conn_oldest_requests);
 	drbd_debugfs_remove(&connection->debugfs_conn);
 }
 
-static void resync_dump_detail(struct seq_file *m, struct lc_element *e)
+static void seq_printf_nice_histogram(struct seq_file *m, unsigned *hist, unsigned const n)
 {
-       struct bm_extent *bme = lc_entry(e, struct bm_extent, lce);
+	unsigned i;
+	unsigned max = 0;
+	unsigned n_transactions = 0;
+	unsigned long n_updates = 0;
+
+	for (i = 1; i <= n; i++) {
+		if (hist[i] > max)
+			max = hist[i];
+		n_updates += i * hist[i];
+		n_transactions += hist[i];
+	}
+
+	seq_puts(m, "updates per activity log transaction\n");
+	seq_printf(m, "avg: %lu\n", n_transactions == 0 ? 0 : n_updates / n_transactions);
 
-       seq_printf(m, "%5d %s %s %s", bme->rs_left,
-		  test_bit(BME_NO_WRITES, &bme->flags) ? "NO_WRITES" : "---------",
-		  test_bit(BME_LOCKED, &bme->flags) ? "LOCKED" : "------",
-		  test_bit(BME_PRIORITY, &bme->flags) ? "PRIORITY" : "--------"
-		  );
+	if (!max)
+		return;
+
+	for (i = 0; i <= n; i++) {
+		unsigned v = (hist[i] * 60UL + max-1) / max;
+		seq_printf(m, "%2u : %10u : %-60.*s\n", i, hist[i], v,
+			"############################################################");
+	}
 }
 
-static int device_resync_extents_show(struct seq_file *m, void *ignored)
+
+static int device_act_log_histogram_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_device *device = m->private;
 
@@ -665,8 +1122,7 @@ static int device_resync_extents_show(struct seq_file *m, void *ignored)
 	seq_printf(m, "v: %u\n\n", 0);
 
 	if (get_ldev_if_state(device, D_FAILED)) {
-		lc_seq_printf_stats(m, device->resync);
-		lc_seq_dump_details(m, device->resync, "rs_left flags", resync_dump_detail);
+		seq_printf_nice_histogram(m, device->al_histogram, AL_UPDATES_PER_TRANSACTION);
 		put_ldev(device);
 	}
 	return 0;
@@ -690,8 +1146,8 @@ static int device_act_log_extents_show(struct seq_file *m, void *ignored)
 static int device_oldest_requests_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_device *device = m->private;
-	struct drbd_resource *resource = device->resource;
-	unsigned long now = jiffies;
+	ktime_t now = ktime_get();
+	unsigned long jif = jiffies;
 	struct drbd_request *r1, *r2;
 	int i;
 
@@ -699,7 +1155,7 @@ static int device_oldest_requests_show(struct seq_file *m, void *ignored)
 	seq_printf(m, "v: %u\n\n", 0);
 
 	seq_puts(m, RQ_HDR);
-	spin_lock_irq(&resource->req_lock);
+	spin_lock_irq(&device->pending_completion_lock);
 	/* WRITE, then READ */
 	for (i = 1; i >= 0; --i) {
 		r1 = list_first_entry_or_null(&device->pending_master_completion[i],
@@ -707,11 +1163,87 @@ static int device_oldest_requests_show(struct seq_file *m, void *ignored)
 		r2 = list_first_entry_or_null(&device->pending_completion[i],
 			struct drbd_request, req_pending_local);
 		if (r1)
-			seq_print_one_request(m, r1, now);
+			seq_print_one_request(m, r1, now, jif);
 		if (r2 && r2 != r1)
-			seq_print_one_request(m, r2, now);
+			seq_print_one_request(m, r2, now, jif);
+	}
+	spin_unlock_irq(&device->pending_completion_lock);
+	return 0;
+}
+
+static int device_openers_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+	struct drbd_resource *resource = device->resource;
+	ktime_t now = ktime_get_real();
+	struct opener *tmp;
+
+	spin_lock(&device->openers_lock);
+	list_for_each_entry(tmp, &device->openers, list)
+		seq_printf(m, "%s\t%d\t%lld\n", tmp->comm, tmp->pid,
+			ktime_to_ms(ktime_sub(now, tmp->opened)));
+	spin_unlock(&device->openers_lock);
+	if (mutex_trylock(&resource->open_release)) {
+		if (resource->auto_promoted_by.pid != 0
+		&&  device->minor == resource->auto_promoted_by.minor) {
+			seq_printf(m, "+%s\t%d\t%lld\n",
+				resource->auto_promoted_by.comm,
+				resource->auto_promoted_by.pid,
+				ktime_to_ms(ktime_sub(now, resource->auto_promoted_by.opened)));
+		}
+		mutex_unlock(&resource->open_release);
+	}
+
+	return 0;
+}
+
+static int device_md_io_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+
+	if (get_ldev_if_state(device, D_FAILED)) {
+		seq_puts(m, drbd_md_dax_active(device->ldev) ? "dax-pmem\n" : "blk-bio\n");
+		put_ldev(device);
 	}
-	spin_unlock_irq(&resource->req_lock);
+
+	return 0;
+}
+
+static void seq_printf_interval_tree(struct seq_file *m, struct rb_root *root)
+{
+	struct rb_node *node;
+
+	node = rb_first(root);
+	while (node) {
+		struct drbd_interval *i = rb_entry(node, struct drbd_interval, rb);
+		char sep = ' ';
+
+		seq_printf(m, "%llus+%u %s", (unsigned long long) i->sector, i->size, drbd_interval_type_str(i));
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_SENT, &i->flags), &sep, "sent");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_RECEIVED, &i->flags), &sep, "received");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags), &sep, "submit-conflict-queued");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_SUBMITTED, &i->flags), &sep, "submitted");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_BACKING_COMPLETED, &i->flags), &sep, "backing-completed");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_COMPLETED, &i->flags), &sep, "completed");
+		seq_print_rq_state_bit(m, test_bit(INTERVAL_CANCELED, &i->flags), &sep, "canceled");
+		seq_putc(m, '\n');
+
+		node = rb_next(node);
+	}
+}
+
+static int device_interval_tree_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+
+	spin_lock_irq(&device->interval_lock);
+	seq_puts(m, "Write requests:\n");
+	seq_printf_interval_tree(m, &device->requests);
+	seq_putc(m, '\n');
+	seq_puts(m, "Read requests:\n");
+	seq_printf_interval_tree(m, &device->read_requests);
+	spin_unlock_irq(&device->interval_lock);
+
 	return 0;
 }
 
@@ -719,58 +1251,221 @@ static int device_data_gen_id_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_device *device = m->private;
 	struct drbd_md *md;
-	enum drbd_uuid_index idx;
+	int node_id, i = 0;
 
 	if (!get_ldev_if_state(device, D_FAILED))
 		return -ENODEV;
 
 	md = &device->ldev->md;
+
 	spin_lock_irq(&md->uuid_lock);
-	for (idx = UI_CURRENT; idx <= UI_HISTORY_END; idx++) {
-		seq_printf(m, "0x%016llX\n", md->uuid[idx]);
+	seq_printf(m, "0x%016llX\n", drbd_current_uuid(device));
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (!(md->peers[node_id].flags & MDF_HAVE_BITMAP))
+			continue;
+		seq_printf(m, "%s[%d]0x%016llX", i++ ? " " : "", node_id,
+			   md->peers[node_id].bitmap_uuid);
 	}
+	seq_putc(m, '\n');
+
+	for (i = 0; i < HISTORY_UUIDS; i++)
+		seq_printf(m, "0x%016llX\n", drbd_history_uuid(device, i));
 	spin_unlock_irq(&md->uuid_lock);
 	put_ldev(device);
 	return 0;
 }
 
+static int device_io_frozen_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+	unsigned long flags = device->flags;
+	char sep = ' ';
+
+	if (!get_ldev_if_state(device, D_FAILED))
+		return -ENODEV;
+
+	/* BUMP me if you change the file format/content/presentation */
+	seq_printf(m, "v: %u\n\n", 0);
+
+	seq_printf(m, "drbd_suspended(): %d\n", drbd_suspended(device));
+	seq_printf(m, "suspend_cnt: %d\n", atomic_read(&device->suspend_cnt));
+	seq_printf(m, "!drbd_state_is_stable(): %d\n", device->cached_state_unstable);
+	seq_printf(m, "ap_bio_cnt[READ]: %d\n", atomic_read(&device->ap_bio_cnt[READ]));
+	seq_printf(m, "ap_bio_cnt[WRITE]: %d\n", atomic_read(&device->ap_bio_cnt[WRITE]));
+	seq_printf(m, "device->pending_bitmap_work.n: %d\n", atomic_read(&device->pending_bitmap_work.n));
+	seq_printf(m, "may_inc_ap_bio(): %d\n", may_inc_ap_bio(device));
+	seq_printf(m, "flags: 0x%04lx :", flags);
+#define pretty_print_bit(n) \
+	seq_print_rq_state_bit(m, test_bit(n, &flags), &sep, #n)
+	pretty_print_bit(NEW_CUR_UUID);
+	pretty_print_bit(WRITING_NEW_CUR_UUID);
+	pretty_print_bit(MAKE_NEW_CUR_UUID);
+#undef pretty_print_bit
+	seq_putc(m, '\n');
+	put_ldev(device);
+
+	return 0;
+}
+
+static int device_al_updates_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+	bool al_updates, cfg_al_updates;
+
+	if (!get_ldev_if_state(device, D_FAILED))
+		return -ENODEV;
+
+	al_updates = !(device->ldev->md.flags & MDF_AL_DISABLED);
+	rcu_read_lock();
+	cfg_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
+	rcu_read_unlock();
+	put_ldev(device);
+
+	seq_printf(m, "%s\n",
+		    al_updates &&  cfg_al_updates ? "yes" :
+		   !al_updates &&  cfg_al_updates ? "no (optimized)" :
+		   !al_updates && !cfg_al_updates ? "no" :
+		   "?");
+	return 0;
+}
+
 static int device_ed_gen_id_show(struct seq_file *m, void *ignored)
 {
 	struct drbd_device *device = m->private;
-	seq_printf(m, "0x%016llX\n", (unsigned long long)device->ed_uuid);
+	seq_printf(m, "0x%016llX\n", (unsigned long long)device->exposed_data_uuid);
 	return 0;
 }
 
-#define drbd_debugfs_device_attr(name)						\
+#define show_per_peer(M) do {							\
+		seq_printf(m, "%-16s", #M ":");					\
+		for_each_peer_device(peer_device, device)			\
+			seq_printf(m, " %12lld", ktime_to_ns(peer_device->M));	\
+		seq_printf(m, "\n");						\
+	} while (0);
+
+#define PRId64 "lld"
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+static int device_req_timing_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_device *device = m->private;
+	struct drbd_peer_device *peer_device;
+
+	seq_printf(m,
+		   "timing values are nanoseconds; write an 'r' to reset all to 0\n\n"
+		   "requests:        %12lu\n"
+		   "before_queue:    %12" PRId64 "\n"
+		   "before_al_begin  %12" PRId64 "\n"
+		   "in_actlog:       %12" PRId64 "\n"
+		   "pre_submit:      %12" PRId64 "\n\n"
+		   "al_updates:      %12u\n"
+		   "before_bm_write  %12" PRId64 "\n"
+		   "mid              %12" PRId64 "\n"
+		   "after_sync_page  %12" PRId64 "\n",
+		   device->reqs,
+		   ktime_to_ns(device->before_queue_kt),
+		   ktime_to_ns(device->before_al_begin_io_kt),
+		   ktime_to_ns(device->in_actlog_kt),
+		   ktime_to_ns(device->pre_submit_kt),
+		   device->al_writ_cnt,
+		   ktime_to_ns(device->al_before_bm_write_hinted_kt),
+		   ktime_to_ns(device->al_mid_kt),
+		   ktime_to_ns(device->al_after_sync_page_kt));
+
+	seq_puts(m, "\npeer:           ");
+	for_each_peer_device(peer_device, device) {
+		struct drbd_connection *connection = peer_device->connection;
+		seq_printf(m, " %12.12s", rcu_dereference(connection->transport.net_conf)->name);
+	}
+	seq_puts(m, "\n");
+	show_per_peer(pre_send_kt);
+	show_per_peer(acked_kt);
+	show_per_peer(net_done_kt);
+
+	return 0;
+}
+
+static ssize_t device_req_timing_write(struct file *file, const char __user *ubuf,
+				       size_t cnt, loff_t *ppos)
+{
+	struct drbd_device *device = file_inode(file)->i_private;
+	char buffer;
+
+	if (copy_from_user(&buffer, ubuf, 1))
+		return -EFAULT;
+
+	if (buffer == 'r' || buffer == 'R') {
+		struct drbd_peer_device *peer_device;
+		unsigned long flags;
+
+		spin_lock_irqsave(&device->timing_lock, flags);
+		device->reqs = 0;
+		device->in_actlog_kt = ns_to_ktime(0);
+		device->pre_submit_kt = ns_to_ktime(0);
+
+		device->before_queue_kt = ns_to_ktime(0);
+		device->before_al_begin_io_kt = ns_to_ktime(0);
+		device->al_writ_cnt = 0;
+		device->al_before_bm_write_hinted_kt = ns_to_ktime(0);
+		device->al_mid_kt = ns_to_ktime(0);
+		device->al_after_sync_page_kt = ns_to_ktime(0);
+
+		for_each_peer_device(peer_device, device) {
+			peer_device->pre_send_kt = ns_to_ktime(0);
+			peer_device->acked_kt = ns_to_ktime(0);
+			peer_device->net_done_kt = ns_to_ktime(0);
+		}
+		spin_unlock_irqrestore(&device->timing_lock, flags);
+	}
+
+	*ppos += cnt;
+	return cnt;
+}
+#endif
+
+static int device_attr_release(struct inode *inode, struct file *file)
+{
+	struct drbd_device *device = inode->i_private;
+	kref_put(&device->kref, drbd_destroy_device);
+	return single_release(inode, file);
+}
+
+#define __drbd_debugfs_device_attr(name, write_fn)				\
 static int device_ ## name ## _open(struct inode *inode, struct file *file)	\
 {										\
 	struct drbd_device *device = inode->i_private;				\
 	return drbd_single_open(file, device_ ## name ## _show, device,		\
 				&device->kref, drbd_destroy_device);		\
 }										\
-static int device_ ## name ## _release(struct inode *inode, struct file *file)	\
-{										\
-	struct drbd_device *device = inode->i_private;				\
-	kref_put(&device->kref, drbd_destroy_device);				\
-	return single_release(inode, file);					\
-}										\
 static const struct file_operations device_ ## name ## _fops = {		\
 	.owner		= THIS_MODULE,						\
 	.open		= device_ ## name ## _open,				\
+	.write          = write_fn,						\
 	.read		= seq_read,						\
 	.llseek		= seq_lseek,						\
-	.release	= device_ ## name ## _release,				\
+	.release	= device_attr_release,					\
 };
+#define drbd_debugfs_device_attr(name) __drbd_debugfs_device_attr(name, NULL)
 
 drbd_debugfs_device_attr(oldest_requests)
 drbd_debugfs_device_attr(act_log_extents)
-drbd_debugfs_device_attr(resync_extents)
+drbd_debugfs_device_attr(act_log_histogram)
 drbd_debugfs_device_attr(data_gen_id)
+drbd_debugfs_device_attr(io_frozen)
 drbd_debugfs_device_attr(ed_gen_id)
+drbd_debugfs_device_attr(openers)
+drbd_debugfs_device_attr(md_io)
+drbd_debugfs_device_attr(interval_tree)
+drbd_debugfs_device_attr(al_updates)
+#ifdef CONFIG_DRBD_TIMING_STATS
+__drbd_debugfs_device_attr(req_timing, device_req_timing_write)
+#endif
 
 void drbd_debugfs_device_add(struct drbd_device *device)
 {
 	struct dentry *vols_dir = device->resource->debugfs_res_volumes;
+	struct drbd_peer_device *peer_device;
 	char minor_buf[8]; /* MINORMASK, MINORBITS == 20; */
 	char vnr_buf[8];   /* volume number vnr is even 16 bit only; */
 	char *slink_name = NULL;
@@ -793,19 +1488,27 @@ void drbd_debugfs_device_add(struct drbd_device *device)
 	kfree(slink_name);
 	slink_name = NULL;
 
-#define DCF(name)	do {					\
-	dentry = debugfs_create_file(#name, 0440,	\
-			device->debugfs_vol, device,		\
-			&device_ ## name ## _fops);		\
-	device->debugfs_vol_ ## name = dentry;			\
-	} while (0)
+	/* debugfs create file */
+	vol_dcf(oldest_requests);
+	vol_dcf(act_log_extents);
+	vol_dcf(act_log_histogram);
+	vol_dcf(data_gen_id);
+	vol_dcf(io_frozen);
+	vol_dcf(ed_gen_id);
+	vol_dcf(openers);
+	vol_dcf(md_io);
+	vol_dcf(interval_tree);
+	vol_dcf(al_updates);
+#ifdef CONFIG_DRBD_TIMING_STATS
+	drbd_dcf(device->debugfs_vol, device, req_timing, 0600);
+#endif
+
+	/* Caller holds conf_update */
+	for_each_peer_device(peer_device, device) {
+		if (!peer_device->debugfs_peer_dev)
+			drbd_debugfs_peer_device_add(peer_device);
+	}
 
-	DCF(oldest_requests);
-	DCF(act_log_extents);
-	DCF(resync_extents);
-	DCF(data_gen_id);
-	DCF(ed_gen_id);
-#undef DCF
 	return;
 
 fail:
@@ -818,12 +1521,351 @@ void drbd_debugfs_device_cleanup(struct drbd_device *device)
 	drbd_debugfs_remove(&device->debugfs_minor);
 	drbd_debugfs_remove(&device->debugfs_vol_oldest_requests);
 	drbd_debugfs_remove(&device->debugfs_vol_act_log_extents);
-	drbd_debugfs_remove(&device->debugfs_vol_resync_extents);
+	drbd_debugfs_remove(&device->debugfs_vol_act_log_histogram);
 	drbd_debugfs_remove(&device->debugfs_vol_data_gen_id);
+	drbd_debugfs_remove(&device->debugfs_vol_io_frozen);
 	drbd_debugfs_remove(&device->debugfs_vol_ed_gen_id);
+	drbd_debugfs_remove(&device->debugfs_vol_openers);
+	drbd_debugfs_remove(&device->debugfs_vol_md_io);
+	drbd_debugfs_remove(&device->debugfs_vol_interval_tree);
+	drbd_debugfs_remove(&device->debugfs_vol_al_updates);
+#ifdef CONFIG_DRBD_TIMING_STATS
+	drbd_debugfs_remove(&device->debugfs_vol_req_timing);
+#endif
 	drbd_debugfs_remove(&device->debugfs_vol);
 }
 
+static int drbd_single_open_peer_device(struct file *file,
+					int (*show)(struct seq_file *, void *),
+					struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	bool got_connection, got_device;
+	struct dentry *parent;
+
+	parent = file->f_path.dentry->d_parent;
+	if (!parent || !parent->d_inode)
+		goto out;
+	inode_lock(d_inode(parent));
+	if (!simple_positive(file->f_path.dentry))
+		goto out_unlock;
+
+	got_connection = kref_get_unless_zero(&connection->kref);
+	got_device = kref_get_unless_zero(&device->kref);
+
+	if (got_connection && got_device) {
+		int ret;
+		inode_unlock(d_inode(parent));
+		ret = single_open(file, show, peer_device);
+		if (ret) {
+			kref_put(&connection->kref, drbd_destroy_connection);
+			kref_put(&device->kref, drbd_destroy_device);
+		}
+		return ret;
+	}
+
+	if (got_connection)
+		kref_put(&connection->kref, drbd_destroy_connection);
+	if (got_device)
+		kref_put(&device->kref, drbd_destroy_device);
+out_unlock:
+	inode_unlock(d_inode(parent));
+out:
+	return -ESTALE;
+}
+
+static void seq_printf_with_thousands_grouping(struct seq_file *seq, long v)
+{
+	/* v is in kB/sec. We don't expect TiByte/sec yet. */
+	if (unlikely(v >= 1000000)) {
+		/* cool: > GiByte/s */
+		seq_printf(seq, "%ld,", v / 1000000);
+		v %= 1000000;
+		seq_printf(seq, "%03ld,%03ld", v/1000, v % 1000);
+	} else if (likely(v >= 1000))
+		seq_printf(seq, "%ld,%03ld", v/1000, v % 1000);
+	else
+		seq_printf(seq, "%ld", v);
+}
+
+static void drbd_get_syncer_progress(struct drbd_peer_device *pd,
+		enum drbd_repl_state repl_state, unsigned long *rs_total,
+		unsigned long *bits_left, unsigned int *per_mil_done)
+{
+	/* this is to break it at compile time when we change that, in case we
+	 * want to support more than (1<<32) bits on a 32bit arch. */
+	typecheck(unsigned long, pd->rs_total);
+	*rs_total = pd->rs_total;
+
+	/* note: both rs_total and rs_left are in bits, i.e. in
+	 * units of BM_BLOCK_SIZE.
+	 * for the percentage, we don't care. */
+
+	if (repl_state == L_VERIFY_S || repl_state == L_VERIFY_T)
+		*bits_left = atomic64_read(&pd->ov_left);
+	else
+		*bits_left = drbd_bm_total_weight(pd) - pd->rs_failed;
+	/* >> 10 to prevent overflow,
+	 * +1 to prevent division by zero */
+	if (*bits_left > *rs_total) {
+		/* D'oh. Maybe a logic bug somewhere.  More likely just a race
+		 * between state change and reset of rs_total.
+		 */
+		*bits_left = *rs_total;
+		*per_mil_done = *rs_total ? 0 : 1000;
+	} else {
+		/* Make sure the division happens in long context.
+		 * We allow up to one petabyte storage right now,
+		 * at a granularity of 4k per bit that is 2**38 bits.
+		 * After shift right and multiplication by 1000,
+		 * this should still fit easily into a 32bit long,
+		 * so we don't need a 64bit division on 32bit arch.
+		 * Note: currently we don't support such large bitmaps on 32bit
+		 * arch anyways, but no harm done to be prepared for it here.
+		 */
+		unsigned int shift = *rs_total > UINT_MAX ? 16 : 10;
+		unsigned long left = *bits_left >> shift;
+		unsigned long total = 1UL + (*rs_total >> shift);
+		unsigned long tmp = 1000UL - left * 1000UL/total;
+		*per_mil_done = tmp;
+	}
+}
+
+static void drbd_syncer_progress(struct drbd_peer_device *pd, struct seq_file *seq,
+		enum drbd_repl_state repl_state)
+{
+	unsigned long db, dt, dbdt, rt, rs_total, rs_left;
+	unsigned int res;
+	int i, x, y;
+	int stalled = 0;
+	unsigned int bm_block_shift = pd->device->last_bm_block_shift;
+
+	drbd_get_syncer_progress(pd, repl_state, &rs_total, &rs_left, &res);
+
+	x = res/50;
+	y = 20-x;
+	seq_puts(seq, "\t[");
+	for (i = 1; i < x; i++)
+		seq_putc(seq, '=');
+	seq_putc(seq, '>');
+	for (i = 0; i < y; i++)
+		seq_putc(seq, '.');
+	seq_puts(seq, "] ");
+
+	if (repl_state == L_VERIFY_S || repl_state == L_VERIFY_T)
+		seq_puts(seq, "verified:");
+	else
+		seq_puts(seq, "sync'ed:");
+	seq_printf(seq, "%3u.%u%% ", res / 10, res % 10);
+
+	/* if more than a few GB, display in MB */
+	if (rs_total > (4UL << (30 - bm_block_shift)))
+		seq_printf(seq, "(%llu/%llu)M",
+			    bit_to_kb(rs_left >> 10, bm_block_shift),
+			    bit_to_kb(rs_total >> 10, bm_block_shift));
+	else
+		seq_printf(seq, "(%llu/%llu)K",
+			    bit_to_kb(rs_left, bm_block_shift),
+			    bit_to_kb(rs_total, bm_block_shift));
+
+	seq_puts(seq, "\n\t");
+
+	/* see drivers/md/md.c
+	 * We do not want to overflow, so the order of operands and
+	 * the * 100 / 100 trick are important. We do a +1 to be
+	 * safe against division by zero. We only estimate anyway.
+	 *
+	 * dt: time from mark until now
+	 * db: blocks written from mark until now
+	 * rt: remaining time
+	 */
+	/* Rolling marks. last_mark+1 may just now be modified.  last_mark+2 is
+	 * at least (DRBD_SYNC_MARKS-2)*DRBD_SYNC_MARK_STEP old, and has at
+	 * least DRBD_SYNC_MARK_STEP time before it will be modified. */
+	/* ------------------------ ~18s average ------------------------ */
+	i = (pd->rs_last_mark + 2) % DRBD_SYNC_MARKS;
+	dt = (jiffies - pd->rs_mark_time[i]) / HZ;
+	if (dt > 180)
+		stalled = 1;
+
+	if (!dt)
+		dt++;
+	db = pd->rs_mark_left[i] - rs_left;
+	rt = (dt * (rs_left / (db/100+1)))/100; /* seconds */
+
+	seq_printf(seq, "finish: %lu:%02lu:%02lu",
+		rt / 3600, (rt % 3600) / 60, rt % 60);
+
+	dbdt = bit_to_kb(db/dt, bm_block_shift);
+	seq_puts(seq, " speed: ");
+	seq_printf_with_thousands_grouping(seq, dbdt);
+	seq_puts(seq, " (");
+	/* ------------------------- ~3s average ------------------------ */
+	if (1) {
+		/* this is what drbd_rs_should_slow_down() uses */
+		i = (pd->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
+		dt = (jiffies - pd->rs_mark_time[i]) / HZ;
+		if (!dt)
+			dt++;
+		db = pd->rs_mark_left[i] - rs_left;
+		dbdt = bit_to_kb(db/dt, bm_block_shift);
+		seq_printf_with_thousands_grouping(seq, dbdt);
+		seq_puts(seq, " -- ");
+	}
+
+	/* --------------------- long term average ---------------------- */
+	/* mean speed since syncer started
+	 * we do account for PausedSync periods */
+	dt = (jiffies - pd->rs_start - pd->rs_paused) / HZ;
+	if (dt == 0)
+		dt = 1;
+	db = rs_total - rs_left;
+	dbdt = bit_to_kb(db/dt, bm_block_shift);
+	seq_printf_with_thousands_grouping(seq, dbdt);
+	seq_putc(seq, ')');
+
+	if (repl_state == L_SYNC_TARGET ||
+	    repl_state == L_VERIFY_S) {
+		seq_puts(seq, " want: ");
+		seq_printf_with_thousands_grouping(seq, pd->c_sync_rate);
+	}
+	seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : "");
+
+	{
+		/* 64 bit:
+		 * we convert to sectors in the display below. */
+		unsigned long bm_bits = drbd_bm_bits(pd->device);
+		unsigned long bit_pos;
+		unsigned long long stop_sector = 0;
+		if (repl_state == L_VERIFY_S ||
+		    repl_state == L_VERIFY_T) {
+			bit_pos = bm_bits - (unsigned long)atomic64_read(&pd->ov_left);
+			if (verify_can_do_stop_sector(pd))
+				stop_sector = pd->ov_stop_sector;
+		} else
+			bit_pos = pd->resync_next_bit;
+		/* Total sectors may be slightly off for oddly
+		 * sized devices. So what. */
+		seq_printf(seq,
+			"\t%3d%% sector pos: %llu/%llu",
+			(int)(bit_pos / (bm_bits/100+1)),
+			(unsigned long long)bit_pos * sect_per_bit(bm_block_shift),
+			(unsigned long long)bm_bits * sect_per_bit(bm_block_shift));
+		if (stop_sector != 0 && stop_sector != ULLONG_MAX)
+			seq_printf(seq, " stop sector: %llu", stop_sector);
+		seq_putc(seq, '\n');
+	}
+}
+
+static int peer_device_proc_drbd_show(struct seq_file *m, void *ignored)
+{
+	struct drbd_peer_device *peer_device = m->private;
+	struct drbd_device *device = peer_device->device;
+	union drbd_state state;
+	const char *sn;
+	struct net_conf *nc;
+	char wp;
+
+	state.disk = device->disk_state[NOW];
+	state.pdsk = peer_device->disk_state[NOW];
+	state.conn = peer_device->repl_state[NOW];
+	state.role = device->resource->role[NOW];
+	state.peer = peer_device->connection->peer_role[NOW];
+
+	state.user_isp = peer_device->resync_susp_user[NOW];
+	state.peer_isp = peer_device->resync_susp_peer[NOW];
+	state.aftr_isp = peer_device->resync_susp_dependency[NOW];
+
+	sn = drbd_repl_str(state.conn);
+
+	rcu_read_lock();
+	{
+		/* reset device->congestion_reason */
+
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		wp = nc ? nc->wire_protocol - DRBD_PROT_A + 'A' : ' ';
+		seq_printf(m,
+		   "%2d: cs:%s ro:%s/%s ds:%s/%s %c %c%c%c%c%c%c\n"
+		   "    ns:%u nr:%u dw:%u dr:%u al:%u bm:%u "
+		   "lo:%d pe:[%d;%d] ua:%d ap:[%d;%d] ep:%d wo:%d",
+		   device->minor, sn,
+		   drbd_role_str(state.role),
+		   drbd_role_str(state.peer),
+		   drbd_disk_str(state.disk),
+		   drbd_disk_str(state.pdsk),
+		   wp,
+		   drbd_suspended(device) ? 's' : 'r',
+		   state.aftr_isp ? 'a' : '-',
+		   state.peer_isp ? 'p' : '-',
+		   state.user_isp ? 'u' : '-',
+		   '-' /* congestion reason... FIXME */,
+		   test_bit(AL_SUSPENDED, &device->flags) ? 's' : '-',
+		   peer_device->send_cnt/2,
+		   peer_device->recv_cnt/2,
+		   device->writ_cnt/2,
+		   device->read_cnt/2,
+		   device->al_writ_cnt,
+		   device->bm_writ_cnt,
+		   atomic_read(&device->local_cnt),
+		   atomic_read(&peer_device->ap_pending_cnt),
+		   atomic_read(&peer_device->rs_pending_cnt),
+		   atomic_read(&peer_device->unacked_cnt),
+		   atomic_read(&device->ap_bio_cnt[WRITE]),
+		   atomic_read(&device->ap_bio_cnt[READ]),
+		   peer_device->connection->epochs,
+		   device->resource->write_ordering
+		);
+		seq_printf(m, " oos:%llu\n",
+			   device_bit_to_kb(device, drbd_bm_total_weight(peer_device)));
+	}
+	if (state.conn == L_SYNC_SOURCE ||
+	    state.conn == L_SYNC_TARGET ||
+	    state.conn == L_VERIFY_S ||
+	    state.conn == L_VERIFY_T)
+		drbd_syncer_progress(peer_device, m, state.conn);
+
+	if (get_ldev_if_state(device, D_FAILED)) {
+		lc_seq_printf_stats(m, device->act_log);
+		put_ldev(device);
+	}
+
+	seq_printf(m, "\tblocked on activity log: %d/%d/%d\n",
+		atomic_read(&device->ap_actlog_cnt),	/* requests */
+		atomic_read(&device->wait_for_actlog),	/* peer_requests */
+		/* nr extents needed to satisfy the above in the worst case */
+		atomic_read(&device->wait_for_actlog_ecnt));
+
+	rcu_read_unlock();
+
+	return 0;
+}
+
+#define drbd_debugfs_peer_device_attr(name)					\
+static int peer_device_ ## name ## _open(struct inode *inode, struct file *file)\
+{										\
+	struct drbd_peer_device *peer_device = inode->i_private;		\
+	return drbd_single_open_peer_device(file,				\
+					    peer_device_ ## name ## _show,	\
+					    peer_device);			\
+}										\
+static int peer_device_ ## name ## _release(struct inode *inode, struct file *file)\
+{										\
+	struct drbd_peer_device *peer_device = inode->i_private;		\
+	kref_put(&peer_device->connection->kref, drbd_destroy_connection);	\
+	kref_put(&peer_device->device->kref, drbd_destroy_device);		\
+	return single_release(inode, file);					\
+}										\
+static const struct file_operations peer_device_ ## name ## _fops = {		\
+	.owner		= THIS_MODULE,						\
+	.open		= peer_device_ ## name ## _open,			\
+	.read		= seq_read,						\
+	.llseek		= seq_lseek,						\
+	.release	= peer_device_ ## name ## _release,			\
+};
+
+drbd_debugfs_peer_device_attr(proc_drbd)
+
 void drbd_debugfs_peer_device_add(struct drbd_peer_device *peer_device)
 {
 	struct dentry *conn_dir = peer_device->connection->debugfs_conn;
@@ -833,10 +1875,14 @@ void drbd_debugfs_peer_device_add(struct drbd_peer_device *peer_device)
 	snprintf(vnr_buf, sizeof(vnr_buf), "%u", peer_device->device->vnr);
 	dentry = debugfs_create_dir(vnr_buf, conn_dir);
 	peer_device->debugfs_peer_dev = dentry;
+
+	/* debugfs create file */
+	peer_dev_dcf(proc_drbd);
 }
 
 void drbd_debugfs_peer_device_cleanup(struct drbd_peer_device *peer_device)
 {
+	drbd_debugfs_remove(&peer_device->debugfs_peer_dev_proc_drbd);
 	drbd_debugfs_remove(&peer_device->debugfs_peer_dev);
 }
 
@@ -847,6 +1893,11 @@ static int drbd_version_show(struct seq_file *m, void *ignored)
 	seq_printf(m, "API_VERSION=%u\n", GENL_MAGIC_VERSION);
 	seq_printf(m, "PRO_VERSION_MIN=%u\n", PRO_VERSION_MIN);
 	seq_printf(m, "PRO_VERSION_MAX=%u\n", PRO_VERSION_MAX);
+#ifdef UTS_RELEASE
+	/* the UTS_RELEASE string of the prepared kernel source tree this
+	 * module was built against */
+	seq_printf(m, "UTS_RELEASE=%s\n", UTS_RELEASE);
+#endif
 	return 0;
 }
 
@@ -863,13 +1914,53 @@ static const struct file_operations drbd_version_fops = {
 	.release = single_release,
 };
 
+static int drbd_refcounts_show(struct seq_file *m, void *ignored)
+{
+	seq_printf(m, "v: %u\n\n", 0);
+
+	return 0;
+}
+
+static int drbd_refcounts_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, drbd_refcounts_show, NULL);
+}
+
+static const struct file_operations drbd_refcounts_fops = {
+	.owner = THIS_MODULE,
+	.open = drbd_refcounts_open,
+	.llseek = seq_lseek,
+	.read = seq_read,
+	.release = single_release,
+};
+
+static int drbd_compat_show(struct seq_file *m, void *ignored)
+{
+	return 0;
+}
+
+static int drbd_compat_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, drbd_compat_show, NULL);
+}
+
+static const struct file_operations drbd_compat_fops = {
+	.owner = THIS_MODULE,
+	.open = drbd_compat_open,
+	.llseek = seq_lseek,
+	.read = seq_read,
+	.release = single_release,
+};
+
 /* not __exit, may be indirectly called
  * from the module-load-failure path as well. */
 void drbd_debugfs_cleanup(void)
 {
+	drbd_debugfs_remove(&drbd_debugfs_compat);
 	drbd_debugfs_remove(&drbd_debugfs_resources);
 	drbd_debugfs_remove(&drbd_debugfs_minors);
 	drbd_debugfs_remove(&drbd_debugfs_version);
+	drbd_debugfs_remove(&drbd_debugfs_refcounts);
 	drbd_debugfs_remove(&drbd_debugfs_root);
 }
 
@@ -883,9 +1974,15 @@ void __init drbd_debugfs_init(void)
 	dentry = debugfs_create_file("version", 0444, drbd_debugfs_root, NULL, &drbd_version_fops);
 	drbd_debugfs_version = dentry;
 
+	dentry = debugfs_create_file("reference_counts", 0444, drbd_debugfs_root, NULL, &drbd_refcounts_fops);
+	drbd_debugfs_refcounts = dentry;
+
 	dentry = debugfs_create_dir("resources", drbd_debugfs_root);
 	drbd_debugfs_resources = dentry;
 
 	dentry = debugfs_create_dir("minors", drbd_debugfs_root);
 	drbd_debugfs_minors = dentry;
+
+	dentry = debugfs_create_file("compat", 0444, drbd_debugfs_root, NULL, &drbd_compat_fops);
+	drbd_debugfs_compat = dentry;
 }
diff --git a/drivers/block/drbd/drbd_debugfs.h b/drivers/block/drbd/drbd_debugfs.h
index ee3d66eb40c68..37037b196e4ac 100644
--- a/drivers/block/drbd/drbd_debugfs.h
+++ b/drivers/block/drbd/drbd_debugfs.h
@@ -11,6 +11,7 @@ void drbd_debugfs_cleanup(void);
 
 void drbd_debugfs_resource_add(struct drbd_resource *resource);
 void drbd_debugfs_resource_cleanup(struct drbd_resource *resource);
+void drbd_debugfs_resource_rename(struct drbd_resource *resource, const char *new_name);
 
 void drbd_debugfs_connection_add(struct drbd_connection *connection);
 void drbd_debugfs_connection_cleanup(struct drbd_connection *connection);
@@ -27,6 +28,7 @@ static inline void drbd_debugfs_cleanup(void) { }
 
 static inline void drbd_debugfs_resource_add(struct drbd_resource *resource) { }
 static inline void drbd_debugfs_resource_cleanup(struct drbd_resource *resource) { }
+static inline void drbd_debugfs_resource_rename(struct drbd_resource *resource, const char *new_name) { }
 
 static inline void drbd_debugfs_connection_add(struct drbd_connection *connection) { }
 static inline void drbd_debugfs_connection_cleanup(struct drbd_connection *connection) { }
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index f6d6276974eef..724cc13123c92 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -18,55 +18,101 @@
 #include <linux/compiler.h>
 #include <linux/types.h>
 #include <linux/list.h>
+#include <linux/sched.h>
 #include <linux/sched/signal.h>
 #include <linux/bitops.h>
 #include <linux/slab.h>
 #include <linux/ratelimit.h>
-#include <linux/tcp.h>
 #include <linux/mutex.h>
 #include <linux/major.h>
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
 #include <linux/idr.h>
-#include <linux/dynamic_debug.h>
-#include <net/tcp.h>
 #include <linux/lru_cache.h>
 #include <linux/prefetch.h>
 #include <linux/drbd_genl_api.h>
 #include <linux/drbd.h>
 #include <linux/drbd_config.h>
+
 #include "drbd_strings.h"
 #include "drbd_state.h"
-#include "drbd_protocol.h"
+#include "drbd_state_change.h"
+#include <linux/drbd_protocol.h>
+#include "drbd_transport.h"
 #include "drbd_polymorph_printk.h"
 
-/* shared module parameters, defined in drbd_main.c */
+/* module parameter, defined in drbd_main.c */
+extern unsigned int drbd_minor_count;
+extern unsigned int drbd_protocol_version_min;
+extern bool drbd_strict_names;
+
+static inline bool drbd_protocol_version_acceptable(unsigned int pv)
+{
+	return	/* DRBD 9 */ (pv >= PRO_VERSION_MIN && pv <= PRO_VERSION_MAX) ||
+		/* DRBD 8 */ (pv >= PRO_VERSION_8_MIN && pv <= PRO_VERSION_8_MAX);
+}
+
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 extern int drbd_enable_faults;
 extern int drbd_fault_rate;
 #endif
 
-extern unsigned int drbd_minor_count;
 extern char drbd_usermode_helper[];
-extern int drbd_proc_details;
+enum {
+	/* drbd_khelper returns >= 0, we can use negative values as flags for drbd_maybe_khelper */
+	DRBD_UMH_DISABLED = INT_MIN,
+};
 
+#ifndef DRBD_MAJOR
+# define DRBD_MAJOR 147
+#endif
 
 /* This is used to stop/restart our threads.
  * Cannot use SIGTERM nor SIGKILL, since these
  * are sent out by init on runlevel changes
  * I choose SIGHUP for now.
+ *
+ * FIXME btw, we should register some reboot notifier.
  */
 #define DRBD_SIGKILL SIGHUP
 
+/* For compatibility with protocol < 122 */
+#define ID_SKIP         (4710ULL)
 #define ID_IN_SYNC      (4711ULL)
 #define ID_OUT_OF_SYNC  (4712ULL)
 #define ID_SYNCER (-1ULL)
 
+static inline enum ov_result drbd_block_id_to_ov_result(u64 block_id)
+{
+	switch (block_id) {
+	case ID_IN_SYNC:
+		return OV_RESULT_IN_SYNC;
+	case ID_OUT_OF_SYNC:
+		return OV_RESULT_OUT_OF_SYNC;
+	default:
+		return OV_RESULT_SKIP;
+	}
+}
+
+static inline u64 drbd_ov_result_to_block_id(enum ov_result result)
+{
+	switch (result) {
+	case OV_RESULT_IN_SYNC:
+		return ID_IN_SYNC;
+	case OV_RESULT_OUT_OF_SYNC:
+		return ID_OUT_OF_SYNC;
+	default:
+		return ID_SKIP;
+	}
+}
+
 #define UUID_NEW_BM_OFFSET ((u64)0x0001000000000000ULL)
 
 struct drbd_device;
 struct drbd_connection;
-struct drbd_peer_device;
+
+/* I want to be able to grep for "drbd $resource_name"
+ * and get all relevant log lines. */
 
 /* Defines to control fault insertion */
 enum {
@@ -84,7 +130,7 @@ enum {
 	DRBD_FAULT_MAX,
 };
 
-extern unsigned int
+unsigned int
 _drbd_insert_fault(struct drbd_device *device, unsigned int type);
 
 static inline int
@@ -98,28 +144,31 @@ drbd_insert_fault(struct drbd_device *device, unsigned int type) {
 #endif
 }
 
-/* integer division, round _UP_ to the next integer */
-#define div_ceil(A, B) ((A)/(B) + ((A)%(B) ? 1 : 0))
-/* usual integer division */
-#define div_floor(A, B) ((A)/(B))
-
-extern struct ratelimit_state drbd_ratelimit_state;
-extern struct idr drbd_devices; /* RCU, updates: genl_lock() */
-extern struct list_head drbd_resources; /* RCU, updates: genl_lock() */
+/*
+ * our structs
+ *************************/
 
-extern const char *cmdname(enum drbd_packet cmd);
+extern struct idr drbd_devices; /* RCU, updates: drbd_devices_lock */
+extern struct list_head drbd_resources; /* RCU, updates: resources_mutex */
+extern struct mutex resources_mutex;
 
 /* for sending/receiving the bitmap,
- * possibly in some encoding scheme */
+ * possibly in some encoding scheme.
+ * For compatibility, we transfer as if bm_block_size was 4k.
+ */
 struct bm_xfer_ctx {
 	/* "const"
 	 * stores total bits and long words
 	 * of the bitmap, so we don't need to
 	 * call the accessor functions over and again. */
+	unsigned long bm_bits_4k; /* unused on sending side */
 	unsigned long bm_bits;
 	unsigned long bm_words;
+	unsigned int scale; /* against BM_BLOCK_SHIFT_4k */
 	/* during xfer, current position within the bitmap */
 	unsigned long bit_offset;
+	/* receiving "partial" bits; unused on sending side. */
+	unsigned long bit_offset_4k;
 	unsigned long word_offset;
 
 	/* statistics; index: (h->command == P_BITMAP) */
@@ -127,8 +176,8 @@ struct bm_xfer_ctx {
 	unsigned bytes[2];
 };
 
-extern void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
-			       const char *direction, struct bm_xfer_ctx *c);
+void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
+			const char *direction, struct bm_xfer_ctx *c);
 
 static inline void bm_xfer_ctx_bit_to_word_offset(struct bm_xfer_ctx *c)
 {
@@ -149,7 +198,7 @@ static inline void bm_xfer_ctx_bit_to_word_offset(struct bm_xfer_ctx *c)
 #endif
 }
 
-extern unsigned int drbd_header_size(struct drbd_connection *connection);
+unsigned int drbd_header_size(struct drbd_connection *connection);
 
 /**********************************************************************/
 enum drbd_thread_state {
@@ -164,7 +213,7 @@ struct drbd_thread {
 	struct task_struct *task;
 	struct completion stop;
 	enum drbd_thread_state t_state;
-	int (*function) (struct drbd_thread *);
+	int (*function)(struct drbd_thread *thi);
 	struct drbd_resource *resource;
 	struct drbd_connection *connection;
 	int reset_cpu_mask;
@@ -183,31 +232,61 @@ static inline enum drbd_thread_state get_t_state(struct drbd_thread *thi)
 
 struct drbd_work {
 	struct list_head list;
-	int (*cb)(struct drbd_work *, int cancel);
+	int (*cb)(struct drbd_work *w, int cancel);
 };
 
-struct drbd_device_work {
+struct drbd_peer_device_work {
 	struct drbd_work w;
-	struct drbd_device *device;
+	struct drbd_peer_device *peer_device;
 };
 
-#include "drbd_interval.h"
-
-extern int drbd_wait_misc(struct drbd_device *, struct drbd_interval *);
+enum drbd_stream;
 
-extern void lock_all_resources(void);
-extern void unlock_all_resources(void);
+#include "drbd_interval.h"
 
+void lock_all_resources(void);
+void unlock_all_resources(void);
+
+enum drbd_disk_state disk_state_from_md(struct drbd_device *device);
+bool want_bitmap(struct drbd_peer_device *peer_device);
+long twopc_timeout(struct drbd_resource *resource);
+long twopc_retry_timeout(struct drbd_resource *resource, int retries);
+void twopc_connection_down(struct drbd_connection *connection);
+u64 directly_connected_nodes(struct drbd_resource *resource,
+			     enum which_state which);
+
+/* sequence arithmetic for dagtag (data generation tag) sector numbers.
+ * dagtag_newer_eq: true, if a is newer than b */
+#define dagtag_newer_eq(a, b)      \
+	(typecheck(u64, a) && \
+	 typecheck(u64, b) && \
+	((s64)(a) - (s64)(b) >= 0))
+
+#define dagtag_newer(a, b)      \
+	(typecheck(u64, a) && \
+	 typecheck(u64, b) && \
+	((s64)(a) - (s64)(b) > 0))
+
+/* An application I/O request.
+ *
+ * Fields marked as "immutable" may only be modified when the request is
+ * exclusively owned, e.g. when the request is created or is being retried.
+ */
 struct drbd_request {
-	struct drbd_work w;
+	/* "immutable" */
 	struct drbd_device *device;
 
 	/* if local IO is not allowed, will be NULL.
 	 * if local IO _is_ allowed, holds the locally submitted bio clone,
 	 * or, after local IO completion, the ERR_PTR(error).
-	 * see drbd_request_endio(). */
+	 * see drbd_request_endio().
+	 *
+	 * Only accessed by app/submitter/endio - strictly sequential,
+	 * no serialization required. */
 	struct bio *private_bio;
 
+	/* Fields sector and size are "immutable". Other fields protected
+	 * by interval_lock. */
 	struct drbd_interval i;
 
 	/* epoch: used to check on "completion" whether this req was in
@@ -217,96 +296,152 @@ struct drbd_request {
 	 * This corresponds to "barrier" in struct p_barrier[_ack],
 	 * and to "barrier_nr" in struct drbd_epoch (and various
 	 * comments/function parameters/local variable names).
+	 *
+	 * "immutable"
 	 */
 	unsigned int epoch;
 
-	struct list_head tl_requests; /* ring list in the transfer log */
-	struct bio *master_bio;       /* master bio pointer */
+	/* Position of this request in the serialized per-resource change
+	 * stream. Can be used to serialize with other events when
+	 * communicating the change stream via multiple connections.
+	 * Assigned from device->resource->dagtag_sector.
+	 *
+	 * Given that some IO backends write several GB per second meanwhile,
+	 * lets just use a 64bit sequence space.
+	 *
+	 * "immutable"
+	 */
+	u64 dagtag_sector;
+
+	/* list entry in transfer log (protected by RCU) */
+	struct list_head tl_requests;
+
+	/* list entry in submitter lists, peer ack list, or retry lists;
+	 * protected by the locks for those lists */
+	struct list_head list;
+
+	/* master bio pointer; "immutable" */
+	struct bio *master_bio;
 
 	/* see struct drbd_device */
 	struct list_head req_pending_master_completion;
 	struct list_head req_pending_local;
 
-	/* for generic IO accounting */
+	/* for generic IO accounting; "immutable" */
 	unsigned long start_jif;
 
-	/* for DRBD internal statistics */
+	/* for request_timer_fn() */
+	unsigned long pre_submit_jif;
+	unsigned long pre_send_jif[DRBD_PEERS_MAX];
 
-	/* Minimal set of time stamps to determine if we wait for activity log
-	 * transactions, local disk or peer.  32 bit "jiffies" are good enough,
-	 * we don't expect a DRBD request to be stalled for several month.
-	 */
+#ifdef CONFIG_DRBD_TIMING_STATS
+	/* for DRBD internal statistics */
+	ktime_t start_kt;
 
 	/* before actual request processing */
-	unsigned long in_actlog_jif;
+	ktime_t in_actlog_kt;
 
 	/* local disk */
-	unsigned long pre_submit_jif;
+	ktime_t pre_submit_kt;
 
 	/* per connection */
-	unsigned long pre_send_jif;
-	unsigned long acked_jif;
-	unsigned long net_done_jif;
-
+	ktime_t pre_send_kt[DRBD_PEERS_MAX];
+	ktime_t acked_kt[DRBD_PEERS_MAX];
+	ktime_t net_done_kt[DRBD_PEERS_MAX];
+#endif
 	/* Possibly even more detail to track each phase:
-	 *  master_completion_jif
+	 *  master_completion_kt
 	 *      how long did it take to complete the master bio
 	 *      (application visible latency)
-	 *  allocated_jif
+	 *  allocated_kt
 	 *      how long the master bio was blocked until we finally allocated
 	 *      a tracking struct
-	 *  in_actlog_jif
+	 *  in_actlog_kt
 	 *      how long did we wait for activity log transactions
 	 *
-	 *  net_queued_jif
+	 *  net_queued_kt
 	 *      when did we finally queue it for sending
-	 *  pre_send_jif
+	 *  pre_send_kt
 	 *      when did we start sending it
-	 *  post_send_jif
+	 *  post_send_kt
 	 *      how long did we block in the network stack trying to send it
-	 *  acked_jif
+	 *  acked_kt
 	 *      when did we receive (or fake, in protocol A) a remote ACK
-	 *  net_done_jif
+	 *  net_done_kt
 	 *      when did we receive final acknowledgement (P_BARRIER_ACK),
 	 *      or decide, e.g. on connection loss, that we do no longer expect
 	 *      anything from this peer for this request.
 	 *
-	 *  pre_submit_jif
-	 *  post_sub_jif
+	 *  pre_submit_kt
+	 *  post_sub_kt
 	 *      when did we start submiting to the lower level device,
 	 *      and how long did we block in that submit function
-	 *  local_completion_jif
+	 *  local_completion_kt
 	 *      how long did it take the lower level device to complete this request
 	 */
 
 
 	/* once it hits 0, we may complete the master_bio */
 	atomic_t completion_ref;
+	/* once it hits 0, we may remove the request from the interval tree and activity log */
+	refcount_t done_ref;
+	/* once it hits 0, we may remove from transfer log and send a corresponding peer ack */
+	refcount_t oos_send_ref;
 	/* once it hits 0, we may destroy this drbd_request object */
 	struct kref kref;
 
-	unsigned rq_state; /* see comments above _req_mod() */
+	/* Creates a dependency chain between writes so that we know that a
+	 * peer ack can be sent when done_ref reaches zero.
+	 *
+	 * If not NULL, when this drbd_request is done, one done_ref reference
+	 * of ->done_next will be put.
+	 *
+	 * "immutable" */
+	struct drbd_request *next_write;
+
+	/* lock to protect state flags */
+	spinlock_t rq_lock;
+	unsigned int local_rq_state;
+	u16 net_rq_state[DRBD_NODE_ID_MAX];
+
+	/* for reclaim from transfer log */
+	struct rcu_head rcu;
+};
+
+/* Used to multicast peer acks. */
+struct drbd_peer_ack {
+	struct drbd_resource *resource;
+	struct list_head list;
+	/*
+	 * Keeps track of which connections have not yet processed this peer
+	 * ack. Peer acks are queued for connections on which they are not sent
+	 * so that last_peer_ack_dagtag_seen is updated at the correct moment.
+	 */
+	u64 queued_mask;
+	u64 pending_mask; /* Peer ack is sent to these nodes */
+	u64 mask; /* Nodes which successfully wrote the requests covered by this peer ack */
+	u64 dagtag_sector;
 };
 
+/* Tracks received writes grouped in epochs. Protected by epoch_lock. */
 struct drbd_epoch {
 	struct drbd_connection *connection;
+	struct drbd_peer_request *oldest_unconfirmed_peer_req;
 	struct list_head list;
 	unsigned int barrier_nr;
 	atomic_t epoch_size; /* increased on every request added. */
 	atomic_t active;     /* increased on every req. added, and dec on every finished. */
+	atomic_t confirmed;  /* adjusted for every P_CONFIRM_STABLE */
 	unsigned long flags;
 };
 
 /* drbd_epoch flag bits */
 enum {
+	DE_BARRIER_IN_NEXT_EPOCH_ISSUED,
+	DE_BARRIER_IN_NEXT_EPOCH_DONE,
+	DE_CONTAINS_A_BARRIER,
 	DE_HAVE_BARRIER_NUMBER,
-};
-
-enum epoch_event {
-	EV_PUT,
-	EV_GOT_BARRIER_NR,
-	EV_BECAME_LAST,
-	EV_CLEANUP = 32, /* used as flag */
+	DE_IS_FINISHING,
 };
 
 struct digest_info {
@@ -317,17 +452,34 @@ struct digest_info {
 struct drbd_peer_request {
 	struct drbd_work w;
 	struct drbd_peer_device *peer_device;
-	struct drbd_epoch *epoch; /* for writes */
-	struct page *pages;
-	blk_opf_t opf;
+	struct list_head recv_order; /* see peer_requests, peer_reads, resync_requests */
+
+	union {
+		struct { /* read requests */
+			unsigned int depend_dagtag_node_id;
+			u64 depend_dagtag;
+		};
+		struct { /* resync target requests */
+			unsigned int requested_size;
+		};
+	};
+
+	struct drbd_page_chain_head page_chain;
+	blk_opf_t opf; /* to be used as bi_opf */
 	atomic_t pending_bios;
 	struct drbd_interval i;
-	/* see comments on ee flag bits below */
-	unsigned long flags;
-	unsigned long submit_jif;
+	unsigned long flags;	/* see comments on ee flag bits below */
 	union {
-		u64 block_id;
-		struct digest_info *digest;
+		struct { /* regular peer_request */
+			struct drbd_epoch *epoch; /* for writes */
+			unsigned long submit_jif;
+			u64 block_id;
+			struct digest_info *digest;
+			u64 dagtag_sector;
+		};
+		struct { /* reused object for sending OOS to other nodes */
+			u64 send_oos_pending;
+		};
 	};
 };
 
@@ -342,9 +494,19 @@ struct drbd_peer_request {
  * non-atomic modification to ee->flags is ok.
  */
 enum {
-	__EE_CALL_AL_COMPLETE_IO,
+	/* If successfully written,
+	 * we may clear the corresponding out-of-sync bits */
 	__EE_MAY_SET_IN_SYNC,
 
+	/* Peer did not write this one, we must set-out-of-sync
+	 * before actually submitting ourselves */
+	__EE_SET_OUT_OF_SYNC,
+
+	/* This peer request closes an epoch using a barrier.
+	 * On successful completion, the epoch is released,
+	 * and the P_BARRIER_ACK send. */
+	__EE_IS_BARRIER,
+
 	/* is this a TRIM aka REQ_OP_DISCARD? */
 	__EE_TRIM,
 	/* explicit zero-out requested, or
@@ -364,125 +526,199 @@ enum {
 	/* This ee has a pointer to a digest instead of a block id */
 	__EE_HAS_DIGEST,
 
-	/* Conflicting local requests need to be restarted after this request */
-	__EE_RESTART_REQUESTS,
-
 	/* The peer wants a write ACK for this (wire proto C) */
 	__EE_SEND_WRITE_ACK,
 
-	/* Is set when net_conf had two_primaries set while creating this peer_req */
-	__EE_IN_INTERVAL_TREE,
-
-	/* for debugfs: */
-	/* has this been submitted, or does it still wait for something else? */
-	__EE_SUBMITTED,
-
-	/* this is/was a write request */
-	__EE_WRITE,
-
 	/* hand back using mempool_free(e, drbd_buffer_page_pool) */
 	__EE_RELEASE_TO_MEMPOOL,
 
 	/* this is/was a write same request */
 	__EE_WRITE_SAME,
 
-	/* this originates from application on peer
-	 * (not some resync or verify or other DRBD internal request) */
-	__EE_APPLICATION,
-
-	/* If it contains only 0 bytes, send back P_RS_DEALLOCATED */
+	/* On target: Send P_RS_THIN_REQ.
+	 * On source: If it contains only 0 bytes, send back P_RS_DEALLOCATED. */
 	__EE_RS_THIN_REQ,
+
+	/* Hold reference in activity log */
+	__EE_IN_ACTLOG,
+
+	/* SyncTarget: This is the last resync request. */
+	__EE_LAST_RESYNC_REQUEST,
+
+	/* This peer_req->recv_order is on some list */
+	__EE_ON_RECV_ORDER,
 };
-#define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO)
 #define EE_MAY_SET_IN_SYNC     (1<<__EE_MAY_SET_IN_SYNC)
+#define EE_SET_OUT_OF_SYNC     (1<<__EE_SET_OUT_OF_SYNC)
+#define EE_IS_BARRIER          (1<<__EE_IS_BARRIER)
 #define EE_TRIM                (1<<__EE_TRIM)
 #define EE_ZEROOUT             (1<<__EE_ZEROOUT)
 #define EE_RESUBMITTED         (1<<__EE_RESUBMITTED)
 #define EE_WAS_ERROR           (1<<__EE_WAS_ERROR)
 #define EE_HAS_DIGEST          (1<<__EE_HAS_DIGEST)
-#define EE_RESTART_REQUESTS	(1<<__EE_RESTART_REQUESTS)
 #define EE_SEND_WRITE_ACK	(1<<__EE_SEND_WRITE_ACK)
-#define EE_IN_INTERVAL_TREE	(1<<__EE_IN_INTERVAL_TREE)
-#define EE_SUBMITTED		(1<<__EE_SUBMITTED)
-#define EE_WRITE		(1<<__EE_WRITE)
 #define EE_RELEASE_TO_MEMPOOL	(1<<__EE_RELEASE_TO_MEMPOOL)
 #define EE_WRITE_SAME		(1<<__EE_WRITE_SAME)
-#define EE_APPLICATION		(1<<__EE_APPLICATION)
 #define EE_RS_THIN_REQ		(1<<__EE_RS_THIN_REQ)
+#define EE_IN_ACTLOG		(1<<__EE_IN_ACTLOG)
+#define EE_LAST_RESYNC_REQUEST	(1<<__EE_LAST_RESYNC_REQUEST)
+#define EE_ON_RECV_ORDER	(1<<__EE_ON_RECV_ORDER)
 
 /* flag bits per device */
-enum {
-	UNPLUG_REMOTE,		/* sending a "UnplugRemote" could help */
+enum device_flag {
 	MD_DIRTY,		/* current uuids and flags not yet on disk */
-	USE_DEGR_WFC_T,		/* degr-wfc-timeout instead of wfc-timeout. */
-	CL_ST_CHG_SUCCESS,
-	CL_ST_CHG_FAIL,
 	CRASHED_PRIMARY,	/* This node was a crashed primary.
 				 * Gets cleared when the state.conn
-				 * goes into C_CONNECTED state. */
-	CONSIDER_RESYNC,
-
-	MD_NO_FUA,		/* Users wants us to not use FUA/FLUSH on meta data dev */
-
-	BITMAP_IO,		/* suspend application io;
-				   once no more io in flight, start bitmap io */
-	BITMAP_IO_QUEUED,       /* Started bitmap IO */
-	WAS_IO_ERROR,		/* Local disk failed, returned IO error */
-	WAS_READ_ERROR,		/* Local disk READ failed (set additionally to the above) */
+				 * goes into L_ESTABLISHED state. */
+	MD_NO_FUA,		/* meta data device does not support barriers,
+				   so don't even try */
 	FORCE_DETACH,		/* Force-detach from local disk, aborting any pending local IO */
-	RESYNC_AFTER_NEG,       /* Resync after online grow after the attach&negotiate finished. */
-	RESIZE_PENDING,		/* Size change detected locally, waiting for the response from
-				 * the peer, if it changed there as well. */
-	NEW_CUR_UUID,		/* Create new current UUID when thawing IO */
+	ABORT_MDIO,		/* Interrupt ongoing meta-data I/O */
+	NEW_CUR_UUID,		/* Create new current UUID when thawing IO or issuing local IO */
+	__NEW_CUR_UUID,		/* Set NEW_CUR_UUID as soon as state change visible */
+	WRITING_NEW_CUR_UUID,	/* Set while the new current ID gets generated. */
 	AL_SUSPENDED,		/* Activity logging is currently suspended. */
-	AHEAD_TO_SYNC_SOURCE,   /* Ahead -> SyncSource queued */
-	B_RS_H_DONE,		/* Before resync handler done (already executed) */
-	DISCARD_MY_DATA,	/* discard_my_data flag per volume */
-	READ_BALANCE_RR,
-
+	UNREGISTERED,
 	FLUSH_PENDING,		/* if set, device->flush_jif is when we submitted that flush
 				 * from drbd_flush_after_epoch() */
 
 	/* cleared only after backing device related structures have been destroyed. */
-	GOING_DISKLESS,		/* Disk is being detached, because of io-error, or admin request. */
+	GOING_DISKLESS,         /* Disk is being detached, because of io-error, or admin request. */
 
 	/* to be used in drbd_device_post_work() */
-	GO_DISKLESS,		/* tell worker to schedule cleanup before detach */
-	DESTROY_DISK,		/* tell worker to close backing devices and destroy related structures. */
+	GO_DISKLESS,            /* tell worker to schedule cleanup before detach */
 	MD_SYNC,		/* tell worker to call drbd_md_sync() */
+	MAKE_NEW_CUR_UUID,	/* tell worker to ping peers and eventually write new current uuid */
+
+	STABLE_RESYNC,		/* One peer_device finished the resync stable! */
+	READ_BALANCE_RR,
+	PRIMARY_LOST_QUORUM,
+	TIEBREAKER_QUORUM,	/* Tiebreaker keeps quorum; used to avoid too verbose logging */
+	DESTROYING_DEV,
+	TRY_TO_GET_RESYNC,
+	OUTDATE_ON_2PC_COMMIT,
+	RESTORE_QUORUM,		/* Restore quorum when we have the same members as before */
+	RESTORING_QUORUM,	/* sanitize_state() -> finish_state_change() */
+	LEGACY_84_MD,
+	BDEV_FROZEN,		/* called bdev_freeze(), needs bdev_thaw() on resume-io */
+};
+
+/* flag bits per peer device */
+enum peer_device_flag {
+	CONSIDER_RESYNC,
+	RESYNC_AFTER_NEG,       /* Resync after online grow after the attach&negotiate finished. */
+	RESIZE_PENDING,		/* Size change detected locally, waiting for the response from
+				 * the peer, if it changed there as well. */
 	RS_START,		/* tell worker to start resync/OV */
 	RS_PROGRESS,		/* tell worker that resync made significant progress */
+	RS_LAZY_BM_WRITE,	/*  -"- and bitmap writeout should be efficient now */
 	RS_DONE,		/* tell worker that resync is done */
+	B_RS_H_DONE,		/* Before resync handler done (already executed) */
+	DISCARD_MY_DATA,	/* discard_my_data flag per volume */
+	USE_DEGR_WFC_T,		/* degr-wfc-timeout instead of wfc-timeout. */
+	INITIAL_STATE_SENT,
+	INITIAL_STATE_RECEIVED,
+	RECONCILIATION_RESYNC,
+	UNSTABLE_RESYNC,	/* Sync source went unstable during resync. */
+	SEND_STATE_AFTER_AHEAD,
+	GOT_NEG_ACK,		/* got a neg_ack while primary, wait until peer_disk is lower than
+				   D_UP_TO_DATE before becoming secondary! */
+	AHEAD_TO_SYNC_SOURCE,   /* Ahead -> SyncSource queued */
+	SYNC_TARGET_TO_BEHIND,  /* SyncTarget, wait for Behind */
+	HANDLING_CONGESTION,    /* Set while testing for congestion and handling it */
+	HANDLE_CONGESTION,      /* tell worker to change state due to congestion */
+	HOLDING_UUID_READ_LOCK, /* did a down_read(&device->uuid_sem) */
+	RS_SOURCE_MISSED_END,   /* SyncSource did not got P_UUIDS110 */
+	RS_PEER_MISSED_END,     /* Peer (which was SyncSource) did not got P_UUIDS110 after resync */
+	SYNC_SRC_CRASHED_PRI,   /* Source of this resync was a crashed primary */
+	HAVE_SIZES,		/* Cleared when connection gets lost; set when sizes received */
+	UUIDS_RECEIVED,		/* Have recent UUIDs from the peer */
+	CURRENT_UUID_RECEIVED,	/* Got a p_current_uuid packet */
+	PEER_QUORATE,		/* Peer has quorum */
+	RS_REQUEST_UNSUCCESSFUL, /* Some resync request was unsuccessful in current cycle */
+	REPLICATION_NEXT, /* If unset, do not replicate writes when next Inconsistent */
+	PEER_REPLICATION_NEXT, /* We have instructed peer not to replicate writes */
 };
 
-struct drbd_bitmap; /* opaque for drbd_device */
+/* We could make these currently hardcoded constants configurable
+ * variables at create-md time (or even re-configurable at runtime?).
+ * Which will require some more changes to the DRBD "super block"
+ * and attach code.
+ *
+ * updates per transaction:
+ *   This many changes to the active set can be logged with one transaction.
+ *   This number is arbitrary.
+ * context per transaction:
+ *   This many context extent numbers are logged with each transaction.
+ *   This number is resulting from the transaction block size (4k), the layout
+ *   of the transaction header, and the number of updates per transaction.
+ *   See drbd_actlog.c:struct al_transaction_on_disk
+ * */
+#define AL_UPDATES_PER_TRANSACTION	 64	// arbitrary
+#define AL_CONTEXT_PER_TRANSACTION	919	// (4096 - 36 - 6*64)/4
 
 /* definition of bits in bm_flags to be used in drbd_bm_lock
  * and drbd_bitmap_io and friends. */
 enum bm_flag {
-	/* currently locked for bulk operation */
-	BM_LOCKED_MASK = 0xf,
-
-	/* in detail, that is: */
-	BM_DONT_CLEAR = 0x1,
-	BM_DONT_SET   = 0x2,
-	BM_DONT_TEST  = 0x4,
+	/*
+	 * The bitmap can be locked to prevent others from clearing, setting,
+	 * and/or testing bits.  The following combinations of lock flags make
+	 * sense:
+	 *
+	 *   BM_LOCK_CLEAR,
+	 *   BM_LOCK_SET, | BM_LOCK_CLEAR,
+	 *   BM_LOCK_TEST | BM_LOCK_SET | BM_LOCK_CLEAR.
+	 */
 
-	/* so we can mark it locked for bulk operation,
-	 * and still allow all non-bulk operations */
-	BM_IS_LOCKED  = 0x8,
+	BM_LOCK_TEST = 0x1,
+	BM_LOCK_SET = 0x2,
+	BM_LOCK_CLEAR = 0x4,
+	BM_LOCK_BULK = 0x8, /* locked for bulk operation, allow all non-bulk operations */
 
-	/* (test bit, count bit) allowed (common case) */
-	BM_LOCKED_TEST_ALLOWED = BM_DONT_CLEAR | BM_DONT_SET | BM_IS_LOCKED,
+	BM_LOCK_ALL = BM_LOCK_TEST | BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK,
 
-	/* testing bits, as well as setting new bits allowed, but clearing bits
-	 * would be unexpected.  Used during bitmap receive.  Setting new bits
-	 * requires sending of "out-of-sync" information, though. */
-	BM_LOCKED_SET_ALLOWED = BM_DONT_CLEAR | BM_IS_LOCKED,
+	BM_LOCK_SINGLE_SLOT = 0x10,
+	BM_ON_DAX_PMEM = 0x10000,
+};
 
-	/* for drbd_bm_write_copy_pages, everything is allowed,
-	 * only concurrent bulk operations are locked out. */
-	BM_LOCKED_CHANGE_ALLOWED = BM_IS_LOCKED,
+struct drbd_bitmap {
+	union {
+		struct page **bm_pages;
+		void *bm_on_pmem;
+	};
+	spinlock_t bm_lock;		/* fine-grain lock (TODO: per slot) */
+	spinlock_t bm_all_slots_lock;	/* all bitmap slots lock */
+
+	unsigned long bm_set[DRBD_PEERS_MAX]; /* number of bits set */
+	unsigned long bm_bits;  /* bits per peer */
+	unsigned long bm_bits_4k;  /* bits per peer, if we had bm_block_size of 4k */
+	size_t   bm_words; /* platform specitif word size; not 32bit!! */
+	size_t   bm_number_of_pages;
+	sector_t bm_dev_capacity;
+	struct mutex bm_change; /* serializes resize operations */
+
+	wait_queue_head_t bm_io_wait; /* used to serialize IO of single pages */
+
+	enum bm_flag bm_flags;
+	unsigned int bm_max_peers;
+	unsigned int bm_block_shift; /* ln2 of bytes per bit for this bitmap */
+
+	/* exclusively to be used by __al_write_transaction(),
+	 * and drbd_bm_write_hinted() -> bm_rw() called from there.
+	 * One activity log extent represents 4MB of storage, which are 1024
+	 * bits (at 4k per bit), times at most DRBD_PEERS_MAX (currently 32).
+	 * The bitmap is created interleaved, with a potentially odd number
+	 * of peer slots determined at create-md time.  Which means that one
+	 * AL-extent may be associated with one or two bitmap pages.
+	 */
+	unsigned int n_bitmap_hints;
+	unsigned int al_bitmap_hints[2*AL_UPDATES_PER_TRANSACTION];
+
+	/* debugging aid, in case we are still racy somewhere */
+	const char    *bm_why;
+	char          bm_task_comm[TASK_COMM_LEN];
+	pid_t         bm_task_pid;
+	struct drbd_peer_device *bm_locked_peer;
 };
 
 struct drbd_work_queue {
@@ -491,29 +727,37 @@ struct drbd_work_queue {
 	wait_queue_head_t q_wait;
 };
 
-struct drbd_socket {
-	struct mutex mutex;
-	struct socket    *socket;
-	/* this way we get our
-	 * send/receive buffers off the stack */
-	void *sbuf;
-	void *rbuf;
+struct drbd_peer_md {
+	u64 bitmap_uuid;
+	u64 bitmap_dagtag;
+	u32 flags;
+	s32 bitmap_index;
 };
 
 struct drbd_md {
 	u64 md_offset;		/* sector offset to 'super' block */
 
-	u64 la_size_sect;	/* last agreed size, unit sectors */
+	u64 effective_size;	/* last agreed size (sectors) */
+	u64 prev_members;	/* read from the meta-data */
+	u64 members;		/* current member mask for writing meta-data */
 	spinlock_t uuid_lock;
-	u64 uuid[UI_SIZE];
+	u64 current_uuid;
 	u64 device_uuid;
 	u32 flags;
+	s32 node_id;
 	u32 md_size_sect;
 
 	s32 al_offset;	/* signed relative sector offset to activity log */
 	s32 bm_offset;	/* signed relative sector offset to bitmap */
 
-	/* cached value of bdev->disk_conf->meta_dev_idx (see below) */
+	u32 max_peers;
+	u32 bm_block_size;
+	u32 bm_block_shift; /* ilog2(bm_block_size) */
+
+	struct drbd_peer_md peers[DRBD_NODE_ID_MAX];
+	u64 history_uuids[HISTORY_UUIDS];
+
+	/* cached value of bdev->disk_conf->meta_dev_idx */
 	s32 meta_dev_idx;
 
 	/* see al_tr_number_to_on_disk_sector() */
@@ -528,8 +772,13 @@ struct drbd_backing_dev {
 	struct block_device *md_bdev;
 	struct file *f_md_bdev;
 	struct drbd_md md;
-	struct disk_conf *disk_conf; /* RCU, for updates: resource->conf_update */
+	struct disk_conf __rcu *disk_conf; /* RCU, for updates: resource->conf_update */
 	sector_t known_size; /* last known size of that backing device */
+#if IS_ENABLED(CONFIG_DEV_DAX_PMEM)
+	struct dax_device *dax_dev;
+	struct meta_data_on_disk_9 *md_on_pmem; /* address of md_offset */
+	struct al_on_pmem *al_on_pmem;
+#endif
 };
 
 struct drbd_md_io {
@@ -544,43 +793,151 @@ struct drbd_md_io {
 
 struct bm_io_work {
 	struct drbd_work w;
+	struct drbd_device *device;
 	struct drbd_peer_device *peer_device;
 	char *why;
 	enum bm_flag flags;
-	int (*io_fn)(struct drbd_device *device, struct drbd_peer_device *peer_device);
-	void (*done)(struct drbd_device *device, int rv);
+	int (*io_fn)(struct drbd_device *device,
+		     struct drbd_peer_device *peer_device);
+	void (*done)(struct drbd_device *device,
+		     struct drbd_peer_device *peer_device,
+		     int rv);
 };
 
 struct fifo_buffer {
+	/* singly linked list to accumulate multiple such struct fifo_buffers,
+	 * to be freed after a single syncronize_rcu(),
+	 * outside a critical section. */
+	struct fifo_buffer *next;
 	unsigned int head_index;
 	unsigned int size;
 	int total; /* sum of all values */
 	int values[] __counted_by(size);
 };
-extern struct fifo_buffer *fifo_alloc(unsigned int fifo_size);
+struct fifo_buffer *fifo_alloc(unsigned int fifo_size);
 
 /* flag bits per connection */
-enum {
-	NET_CONGESTED,		/* The data socket is congested */
-	RESOLVE_CONFLICTS,	/* Set on one node, cleared on the peer! */
-	SEND_PING,
-	GOT_PING_ACK,		/* set when we receive a ping_ack packet, ping_wait gets woken */
-	CONN_WD_ST_CHG_REQ,	/* A cluster wide state change on the connection is active */
-	CONN_WD_ST_CHG_OKAY,
-	CONN_WD_ST_CHG_FAIL,
+enum connection_flag {
+	PING_PENDING,		/* cleared upon receiveing a ping_ack packet, wakes state_wait */
+	TWOPC_PREPARED,
+	TWOPC_YES,
+	TWOPC_NO,
+	TWOPC_RETRY,
 	CONN_DRY_RUN,		/* Expect disconnect after resync handshake. */
-	CREATE_BARRIER,		/* next P_DATA is preceded by a P_BARRIER */
-	STATE_SENT,		/* Do not change state/UUIDs while this is set */
+	DISCONNECT_EXPECTED,
+	BARRIER_ACK_PENDING,
+	CORKED,
+	DATA_CORKED = CORKED,	/* used as computed value CORKED + DATA_STREAM */
+	CONTROL_CORKED,		/* used as computed value CORKED + CONTROL_STREAM */
+	C_UNREGISTERED,
+	RECONNECT,
+	CONN_DISCARD_MY_DATA,
+	SEND_STATE_AFTER_AHEAD_C,
+	NOTIFY_PEERS_LOST_PRIMARY,
+	CHECKING_PEER,		/* used by make_new_urrent_uuid() to check liveliness */
+	CONN_CONGESTED,
+	CONN_HANDSHAKE_DISCONNECT,
+	CONN_HANDSHAKE_RETRY,
+	CONN_HANDSHAKE_READY,
+	RECEIVED_DAGTAG, /* Whether we received any write or dagtag since connecting. */
+	PING_TIMEOUT_ACTIVE,
+};
+
+/* flag bits per resource */
+enum resource_flag {
+	EXPLICIT_PRIMARY,
 	CALLBACK_PENDING,	/* Whether we have a call_usermodehelper(, UMH_WAIT_PROC)
 				 * pending, from drbd worker context.
 				 */
-	DISCONNECT_SENT,
+	TWOPC_ABORT_LOCAL,
+	TWOPC_WORK_PENDING,     /* Set while work for sending reply is scheduled */
+	TWOPC_EXECUTED,         /* Commited or aborted */
+	TWOPC_STATE_CHANGE_PENDING, /* set between sending commit and changing local state */
+
+	TRY_BECOME_UP_TO_DATE_PENDING,
 
 	DEVICE_WORK_PENDING,	/* tell worker that some device has pending work */
+	PEER_DEVICE_WORK_PENDING,/* tell worker that some peer_device has pending work */
+
+	/* to be used in drbd_post_work() */
+	R_UNREGISTERED,
+	DOWN_IN_PROGRESS,
+	CHECKING_PEERS,
+	WRONG_MDF_EXISTS,	/* Warned about MDF_EXISTS flag on all peer slots */
+	TWOPC_RECV_SIZES_ERR,	/* Error processing sizes packet during 2PC connect */
 };
 
 enum which_state { NOW, OLD = NOW, NEW };
 
+enum twopc_type {
+	TWOPC_STATE_CHANGE,
+	TWOPC_RESIZE,
+};
+
+struct twopc_reply {
+	int vnr;
+	unsigned int tid;  /* transaction identifier */
+	int initiator_node_id;  /* initiator of the transaction */
+	int target_node_id;  /* target of the transaction (or -1) */
+	u64 target_reachable_nodes;  /* behind the target node */
+	u64 reachable_nodes;  /* behind other nodes */
+	union {
+		struct { /* type == TWOPC_STATE_CHANGE */
+			u64 primary_nodes;
+			u64 weak_nodes;
+		};
+		struct { /* type == TWOPC_RESIZE */
+			u64 diskful_primary_nodes;
+			u64 max_possible_size;
+		};
+	};
+	unsigned int is_disconnect:1;
+	unsigned int is_connect:1;
+	unsigned int is_aborted:1;
+	/* Whether the state change on receiving the twopc failed. When this is
+	 * a twopc for transitioning to C_CONNECTED, we cannot immediately
+	 * reply with P_TWOPC_NO. The state handshake must complete first to
+	 * decide the appropriate reply. */
+	unsigned int state_change_failed:1;
+};
+
+struct twopc_request {
+	u64 nodes_to_reach;
+	enum drbd_packet cmd;
+	unsigned int tid;
+	int initiator_node_id;
+	int target_node_id;
+	int vnr;
+	u32 flags;
+};
+
+struct drbd_thread_timing_details {
+	unsigned long start_jif;
+	void *cb_addr;
+	const char *caller_fn;
+	unsigned int line;
+	unsigned int cb_nr;
+};
+#define DRBD_THREAD_DETAILS_HIST	16
+
+struct drbd_send_buffer {
+	struct page *page;  /* current buffer page for sending data */
+	char *unsent;  /* start of unsent area != pos if corked... */
+	char *pos; /* position within that page */
+	int allocated_size; /* currently allocated space */
+	int additional_size;  /* additional space to be added to next packet's size */
+};
+
+struct drbd_mutable_buffer {
+	u8 *buffer;
+	unsigned int avail;
+};
+
+enum drbd_per_resource_ratelimit {
+	D_RL_R_NOLIMIT = -1,
+	D_RL_R_GENERIC,
+};
+
 struct drbd_resource {
 	char *name;
 #ifdef CONFIG_DEBUG_FS
@@ -588,32 +945,141 @@ struct drbd_resource {
 	struct dentry *debugfs_res_volumes;
 	struct dentry *debugfs_res_connections;
 	struct dentry *debugfs_res_in_flight_summary;
+	struct dentry *debugfs_res_state_twopc;
+	struct dentry *debugfs_res_worker_pid;
+	struct dentry *debugfs_res_members;
 #endif
 	struct kref kref;
-	struct idr devices;		/* volume number to device mapping */
+
+	/* Volume number to device mapping. Updates protected by conf_update. */
+	struct idr devices;
+
+	struct ratelimit_state ratelimit[1];
+
+	/* RCU list. Updates protected by adm_mutex, conf_update and state_rwlock. */
 	struct list_head connections;
-	struct list_head resources;
+
+	struct list_head resources;     /* list entry in global resources list */
 	struct res_opts res_opts;
-	struct mutex conf_update;	/* mutex for ready-copy-update of net_conf and disk_conf */
+	int max_node_id;
+	/*
+	 * For read-copy-update of net_conf and disk_conf and devices,
+	 * connection, peer_devices and paths lists.
+	 */
+	struct mutex conf_update;
 	struct mutex adm_mutex;		/* mutex to serialize administrative requests */
-	spinlock_t req_lock;
+	struct mutex open_release;	/* serialize open/release */
+	struct {
+		char comm[TASK_COMM_LEN];
+		unsigned int minor;
+		pid_t pid;
+		ktime_t opened;
+	} auto_promoted_by;
+
+	rwlock_t state_rwlock;          /* serialize state changes */
+	u64 dagtag_sector;		/* Protected by tl_update_lock.
+					 * See also dagtag_sector in
+					 * &drbd_request */
+	u64 dagtag_from_backing_dev;
+	u64 dagtag_before_attach;
+	u64 members;			/* mask of online nodes */
+	unsigned long flags;
+
+	/* Protects updates to the transfer log and related counters. */
+	spinlock_t tl_update_lock;
+	struct list_head transfer_log;	/* all requests not yet fully processed */
+	struct drbd_request *tl_previous_write;
 
-	unsigned susp:1;		/* IO suspended by user */
-	unsigned susp_nod:1;		/* IO suspended because no data */
-	unsigned susp_fen:1;		/* IO suspended because fence peer handler runs */
+	spinlock_t peer_ack_lock;
+	struct list_head peer_ack_req_list;  /* requests to send peer acks for */
+	struct list_head peer_ack_list;  /* peer acks to send */
+	struct drbd_work peer_ack_work;
+	u64 last_peer_acked_dagtag;  /* dagtag of last PEER_ACK'ed request */
+	struct drbd_request *peer_ack_req;  /* last request not yet PEER_ACK'ed */
+
+	/* Protects current_flush_sequence and pending_flush_mask (connection) */
+	spinlock_t initiator_flush_lock;
+	u64 current_flush_sequence;
+
+	struct semaphore state_sem;
+	wait_queue_head_t state_wait;  /* upon each state change. */
+	enum chg_state_flags state_change_flags;
+	const char **state_change_err_str;
+	bool remote_state_change;  /* remote state change in progress */
+	enum drbd_packet twopc_prepare_reply_cmd; /* this node's answer to the prepare phase or 0 */
+	u64 twopc_parent_nodes;
+	struct twopc_reply twopc_reply;
+	struct timer_list twopc_timer;
+	struct work_struct twopc_work;
+	wait_queue_head_t twopc_wait;
+	struct {
+		enum twopc_type type;
+		union {
+			struct twopc_resize {
+				int dds_flags;		   /* from prepare phase */
+				sector_t user_size;	   /* from prepare phase */
+				u64 diskful_primary_nodes; /* added in commit phase */
+				u64 new_size;		   /* added in commit phase */
+			} resize;
+			struct twopc_state_change {
+				union drbd_state mask;	/* from prepare phase */
+				union drbd_state val;	/* from prepare phase */
+				u64 primary_nodes;	/* added in commit phase */
+				u64 reachable_nodes;	/* added in commit phase */
+			} state_change;
+		};
+	} twopc;
+	enum drbd_role role[2];
+	bool susp_user[2];			/* IO suspended by user */
+	bool susp_nod[2];		/* IO suspended because no data */
+	bool susp_quorum[2];		/* IO suspended because no quorum */
+	bool susp_uuid[2];		/* IO suspended because waiting new current UUID */
+	bool fail_io[2];		/* Fail all IO requests because forced a demote */
+	bool cached_susp;		/* cached result of looking at all different suspend bits */
+	bool cached_all_devices_have_quorum;
 
 	enum write_ordering_e write_ordering;
 
+	/* Protects the current transfer log (tle) fields. */
+	spinlock_t current_tle_lock;
+	atomic_t current_tle_nr;	/* transfer log epoch number */
+	unsigned current_tle_writes;	/* writes seen within this tl epoch */
+
+	unsigned cached_min_aggreed_protocol_version;
+
 	cpumask_var_t cpu_mask;
+
+	struct drbd_work_queue work;
+	struct drbd_thread worker;
+
+	struct list_head listeners;
+	spinlock_t listeners_lock;
+
+	struct timer_list peer_ack_timer; /* send a P_PEER_ACK after last completion */
+
+	unsigned int w_cb_nr; /* keeps counting up */
+	struct drbd_thread_timing_details w_timing_details[DRBD_THREAD_DETAILS_HIST];
+	wait_queue_head_t barrier_wait;  /* upon each state change. */
+	struct rcu_head rcu;
+
+	struct list_head suspended_reqs;
+	/*
+	 * The side effects of an empty state change two-phase commit are:
+	 *
+	 * * A local consistent disk can upgrade to up-to-date when no primary is reachable
+	 *   (or become outdated if the prepare packets reach a primary).
+	 *
+	 * * resource->members are updates
+	 *
+	 * * Faraway nodes might outdate themselves if they learn about the existence of a primary
+	 *   (with access to data) node.
+	 */
+	struct work_struct empty_twopc;
 };
 
-struct drbd_thread_timing_details
-{
-	unsigned long start_jif;
-	void *cb_addr;
-	const char *caller_fn;
-	unsigned int line;
-	unsigned int cb_nr;
+enum drbd_per_connection_ratelimit {
+	D_RL_C_NOLIMIT = -1,
+	D_RL_C_GENERIC,
 };
 
 struct drbd_connection {
@@ -623,36 +1089,49 @@ struct drbd_connection {
 	struct dentry *debugfs_conn;
 	struct dentry *debugfs_conn_callback_history;
 	struct dentry *debugfs_conn_oldest_requests;
+	struct dentry *debugfs_conn_transport;
+	struct dentry *debugfs_conn_debug;
+	struct dentry *debugfs_conn_receiver_pid;
+	struct dentry *debugfs_conn_sender_pid;
 #endif
 	struct kref kref;
 	struct idr peer_devices;	/* volume number to peer device mapping */
-	enum drbd_conns cstate;		/* Only C_STANDALONE to C_WF_REPORT_PARAMS */
-	struct mutex cstate_mutex;	/* Protects graceful disconnects */
-	unsigned int connect_cnt;	/* Inc each time a connection is established */
+	enum drbd_conn_state cstate[2];
+	enum drbd_role peer_role[2];
+	bool susp_fen[2];		/* IO suspended because fence peer handler runs */
+
+	struct ratelimit_state ratelimit[1];
 
 	unsigned long flags;
-	struct net_conf *net_conf;	/* content protected by rcu */
-	wait_queue_head_t ping_wait;	/* Woken upon reception of a ping, and a state change */
+	enum drbd_fencing_policy fencing_policy;
 
-	struct sockaddr_storage my_addr;
-	int my_addr_len;
-	struct sockaddr_storage peer_addr;
-	int peer_addr_len;
+	struct drbd_send_buffer send_buffer[2];
+	struct mutex mutex[2]; /* Protect assembling of new packet until sending it (in send_buffer) */
+	/* scratch buffers for use while "owning" the DATA_STREAM send_buffer,
+	 * to avoid larger on-stack temporary variables,
+	 * introduced for holding digests in drbd_send_dblock() */
+	union {
+		/* MAX_DIGEST_SIZE in the linux kernel at this point is 64 byte, afaik */
+		struct {
+			char before[64];
+			char after[64];
+		} d;
+	} scratch_buffer;
 
-	struct drbd_socket data;	/* data/barrier/cstate/parameter packets */
-	struct drbd_socket meta;	/* ping/ack (metadata) packets */
 	int agreed_pro_version;		/* actually used protocol version */
 	u32 agreed_features;
-	unsigned long last_received;	/* in jiffies, either socket */
-	unsigned int ko_count;
+	atomic_t ap_in_flight; /* App sectors in flight (waiting for ack) */
+	atomic_t rs_in_flight; /* Resync sectors in flight */
 
-	struct list_head transfer_log;	/* all requests not yet fully processed */
+	struct drbd_work connect_timer_work;
+	struct timer_list connect_timer;
 
 	struct crypto_shash *cram_hmac_tfm;
-	struct crypto_shash *integrity_tfm;  /* checksums we compute, updates protected by connection->data->mutex */
+	struct crypto_shash *integrity_tfm;  /* checksums we compute, updates protected by connection->mutex[DATA_STREAM] */
 	struct crypto_shash *peer_integrity_tfm;  /* checksums we verify, only accessed from receiver thread  */
 	struct crypto_shash *csums_tfm;
 	struct crypto_shash *verify_tfm;
+
 	void *int_dig_in;
 	void *int_dig_vv;
 
@@ -660,35 +1139,138 @@ struct drbd_connection {
 	struct drbd_epoch *current_epoch;
 	spinlock_t epoch_lock;
 	unsigned int epochs;
-	atomic_t current_tle_nr;	/* transfer log epoch number */
-	unsigned current_tle_writes;	/* writes seen within this tl epoch */
 
 	unsigned long last_reconnect_jif;
 	/* empty member on older kernels without blk_start_plug() */
 	struct blk_plug receiver_plug;
 	struct drbd_thread receiver;
-	struct drbd_thread worker;
-	struct drbd_thread ack_receiver;
+	struct drbd_thread sender;
 	struct workqueue_struct *ack_sender;
+	struct work_struct peer_ack_work;
+
+	/* Work for sending P_OUT_OF_SYNC due to P_PEER_ACK */
+	struct drbd_work send_oos_work;
+	/*
+	 * These peers have sent us a P_PEER_ACK for which we need to send
+	 * P_OUT_OF_SYNC on this connection.
+	 */
+	unsigned long send_oos_from_mask;
+
+	atomic64_t last_dagtag_sector;
+	/* Record of last peer ack to determine whether we can ack flush */
+	u64 last_peer_ack_dagtag_seen;
+
+	/* Mask of nodes from which we are waiting for a flush ack corresponding to this Primary */
+	u64 pending_flush_mask;
+
+	/* Protects the flush members below for this connection */
+	spinlock_t primary_flush_lock;
+	/* For handling P_FLUSH_REQUESTS from this peer */
+	u64 flush_requests_dagtag;
+	u64 flush_sequence;
+	u64 flush_forward_sent_mask;
+
+	/* For handling forwarded flushes. On connection to initiator node. */
+	spinlock_t flush_ack_lock;
+	struct drbd_work flush_ack_work;
+	/* For forwarded flushes. On connection to initiator node. Indexed by primary node ID */
+	u64 flush_ack_sequence[DRBD_PEERS_MAX];
+
+	atomic_t active_ee_cnt; /* Peer write requests waiting for activity log or backing disk. */
+	atomic_t backing_ee_cnt; /* Other peer requests waiting for conflicts or backing disk. */
+	atomic_t done_ee_cnt;
+	spinlock_t peer_reqs_lock;
+	spinlock_t send_oos_lock; /* Protects send_oos list */
+
+	/* Lists using drbd_peer_request.recv_order (see also drbd_peer_device.resync_requests) */
+	struct list_head peer_requests; /* All peer writes in the order we received them */
+	struct list_head peer_reads; /* All reads in the order we received them */
+	/*
+	 * Peer writes for which we need to send some P_OUT_OF_SYNC. These peer
+	 * writes continue to be stored on the connection over which the writes
+	 * and the P_PEER_ACK are received. They are accessed by the sender for
+	 * each relevant peer. Protected by send_oos_lock on this connection.
+	 */
+	struct list_head send_oos;
+
+	/* Lists using drbd_peer_request.w.list */
+	struct list_head done_ee;   /* Need to send P_WRITE_ACK/P_RS_WRITE_ACK */
+	struct list_head dagtag_wait_ee; /* Resync read waiting for dagtag to be reached */
+	struct list_head resync_ack_ee;   /* P_RS_DATA_REPLY sent, waiting for P_RS_WRITE_ACK */
+
+	struct work_struct send_acks_work;
+	struct work_struct send_ping_ack_work;
+	struct work_struct send_ping_work;
+	wait_queue_head_t ee_wait;
+
+	atomic_t pp_in_use;		/* allocated from page pool */
+	atomic_t pp_in_use_by_net;	/* sendpage()d, still referenced by transport */
+	/* sender side */
+	struct drbd_work_queue sender_work;
+
+	struct drbd_work send_dagtag_work;
+	u64 send_dagtag;
+
+	struct sender_todo {
+		struct list_head work_list;
+
+		/* If upper layers trigger an unplug on this side, we want to
+		 * send and unplug hint over to the peer.  Sending it too
+		 * early, or missing it completely, causes a potential latency
+		 * penalty (requests idling too long in the remote queue).
+		 * There is no harm done if we occasionally send one too many
+		 * such unplug hints.
+		 *
+		 * We have two slots, which are used in an alternating fashion:
+		 * If a new unplug event happens while the current pending one
+		 * has not even been processed yet, we overwrite the next
+		 * pending slot: there is not much point in unplugging on the
+		 * remote side, if we have a full request queue to be send on
+		 * this side still, and not even reached the position in the
+		 * change stream when the previous local unplug happened.
+		 */
+		u64 unplug_dagtag_sector[2];
+		unsigned int unplug_slot; /* 0 or 1 */
+
+		/* the currently (or last) processed request,
+		 * see process_sender_todo() */
+		struct drbd_request *req;
+
+		/* Points to the next request on the resource->transfer_log,
+		 * which is RQ_NET_QUEUED for this connection, and so can
+		 * safely be used as next starting point for the list walk
+		 * in tl_next_request_for_connection().
+		 *
+		 * If it is NULL (we walked off the tail last time), it will be
+		 * set by __req_mod( QUEUE_FOR.* ), so fast connections don't
+		 * need to walk the full transfer_log list every time, even if
+		 * the list is kept long by some slow connections.
+		 *
+		 * req_next is only accessed by drbd_sender thread, in
+		 * case of a resend from some worker, but then regular IO
+		 * is suspended.
+		 */
+		struct drbd_request *req_next;
+	} todo;
 
 	/* cached pointers,
 	 * so we can look up the oldest pending requests more quickly.
-	 * protected by resource->req_lock */
-	struct drbd_request *req_next; /* DRBD 9: todo.req_next */
+	 * TODO: RCU */
 	struct drbd_request *req_ack_pending;
+	/* The oldest request that is or was queued for this peer, but is not
+	 * done towards it. */
 	struct drbd_request *req_not_net_done;
+	/* Protects the caching pointers from being advanced concurrently. */
+	spinlock_t advance_cache_ptr_lock;
 
-	/* sender side */
-	struct drbd_work_queue sender_work;
-
-#define DRBD_THREAD_DETAILS_HIST	16
-	unsigned int w_cb_nr; /* keeps counting up */
+	unsigned int s_cb_nr; /* keeps counting up */
 	unsigned int r_cb_nr; /* keeps counting up */
-	struct drbd_thread_timing_details w_timing_details[DRBD_THREAD_DETAILS_HIST];
+	struct drbd_thread_timing_details s_timing_details[DRBD_THREAD_DETAILS_HIST];
 	struct drbd_thread_timing_details r_timing_details[DRBD_THREAD_DETAILS_HIST];
 
 	struct {
 		unsigned long last_sent_barrier_jif;
+		int last_sent_epoch_nr;
 
 		/* whether this sender thread
 		 * has processed a single write yet. */
@@ -701,52 +1283,245 @@ struct drbd_connection {
 		 * with req->epoch == current_epoch_nr.
 		 * If none, no P_BARRIER will be sent. */
 		unsigned current_epoch_writes;
-	} send;
-};
 
-static inline bool has_net_conf(struct drbd_connection *connection)
-{
-	bool has_net_conf;
+		/* Position in change stream of last write sent. */
+		u64 current_dagtag_sector;
 
-	rcu_read_lock();
-	has_net_conf = rcu_dereference(connection->net_conf);
-	rcu_read_unlock();
+		/* Position in change stream of last ready request seen. */
+		u64 seen_dagtag_sector;
+	} send;
 
-	return has_net_conf;
-}
+	struct {
+		u64 dagtag_sector;
+		int lost_node_id;
+	} after_reconciliation;
 
-void __update_timing_details(
-		struct drbd_thread_timing_details *tdp,
-		unsigned int *cb_nr,
-		void *cb,
-		const char *fn, const unsigned int line);
+	unsigned int peer_node_id;
 
-#define update_worker_timing_details(c, cb) \
-	__update_timing_details(c->w_timing_details, &c->w_cb_nr, cb, __func__ , __LINE__ )
-#define update_receiver_timing_details(c, cb) \
-	__update_timing_details(c->r_timing_details, &c->r_cb_nr, cb, __func__ , __LINE__ )
+	struct drbd_mutable_buffer reassemble_buffer;
+	union {
+		u8 bytes[8];
+		struct p_block_ack block_ack;
+		struct p_barrier_ack barrier_ack;
+		struct p_confirm_stable confirm_stable;
+		struct p_peer_ack peer_ack;
+		struct p_peer_block_desc peer_block_desc;
+		struct p_twopc_reply twopc_reply;
+	} reassemble_buffer_bytes;
+
+	/* Used when a network namespace is removed to track all connections
+	 * that need disconnecting. */
+	struct list_head remove_net_list;
+
+	struct rcu_head rcu;
+
+	unsigned int ctl_packets;
+	unsigned int ctl_bytes;
+
+	struct drbd_transport transport; /* The transport needs to be the last member. The acutal
+					    implementation might have more members than the
+					    abstract one. */
+};
 
-struct submit_worker {
-	struct workqueue_struct *wq;
-	struct work_struct worker;
+/* used to get the next lower or next higher peer_device depending on device node-id */
+enum drbd_neighbor {
+	NEXT_LOWER,
+	NEXT_HIGHER
+};
 
-	/* protected by ..->resource->req_lock */
-	struct list_head writes;
+enum drbd_per_peer_device_ratelimit {
+	D_RL_PD_NOLIMIT = -1,
+	D_RL_PD_GENERIC,
 };
 
 struct drbd_peer_device {
 	struct list_head peer_devices;
 	struct drbd_device *device;
 	struct drbd_connection *connection;
-	struct work_struct send_acks_work;
+	struct peer_device_conf __rcu *conf; /* RCU, for updates: resource->conf_update */
+	enum drbd_disk_state disk_state[2];
+	enum drbd_repl_state repl_state[2];
+	bool resync_susp_user[2];
+	bool resync_susp_peer[2];
+	bool resync_susp_dependency[2];
+	bool resync_susp_other_c[2];
+	bool resync_active[2];
+	bool replication[2]; /* Only while peer is Inconsistent: Is replication enabled? */
+	bool peer_replication[2]; /* Whether we have instructed peer to replicate to us */
+	enum drbd_repl_state negotiation_result; /* To find disk state after attach */
+	unsigned int send_cnt;
+	unsigned int recv_cnt;
+	atomic_t packet_seq;
+	unsigned int peer_seq;
+	spinlock_t peer_seq_lock;
+	uint64_t d_size;  /* size of disk */
+	uint64_t u_size;  /* user requested size */
+	uint64_t c_size;  /* current exported size */
+	uint64_t max_size;
+	int bitmap_index;
+	int node_id;
+
+	struct ratelimit_state ratelimit[1];
+
+	unsigned long flags;
+
+	enum drbd_repl_state start_resync_side;
+	enum drbd_repl_state last_repl_state; /* What we received from the peer */
+	struct timer_list start_resync_timer;
+	struct drbd_work resync_work;
+	struct timer_list resync_timer;
+	struct drbd_work propagate_uuids_work;
+
+	enum drbd_disk_state resync_finished_pdsk; /* Finished while starting resync */
+	int resync_again; /* decided to resync again while resync running */
+	sector_t last_peers_in_sync_end; /* sector after end of last scheduled peers-in-sync */
+	unsigned long resync_next_bit; /* bitmap bit to search from for next resync request */
+	unsigned long last_resync_pass_bits; /* bitmap weight at end of previous pass */
+
+	atomic_t ap_pending_cnt; /* AP data packets on the wire, ack expected (RQ_NET_PENDING set) */
+	atomic_t unacked_cnt;	 /* Need to send replies for */
+	atomic_t rs_pending_cnt; /* RS request/data packets on the wire */
+
+	/* Protected by connection->peer_reqs_lock */
+	struct list_head resync_requests; /* Resync requests in the order we sent them */
+	/*
+	 * If not NULL, all requests in resync_requests until this one have
+	 * been received. Discards are only counted as "received" once merging
+	 * is complete.
+	 */
+	struct drbd_peer_request *received_last;
+	/*
+	 * If not NULL, all requests in resync_requests after received_last
+	 * until this one are discards.
+	 */
+	struct drbd_peer_request *discard_last;
+
+	/* use checksums for *this* resync */
+	bool use_csums;
+	/* blocks to resync in this run [unit BM_BLOCK_SIZE] */
+	unsigned long rs_total;
+	/* number of resync blocks that failed in this run */
+	unsigned long rs_failed;
+	/* Syncer's start time [unit jiffies] */
+	unsigned long rs_start;
+	/* cumulated time in PausedSyncX state [unit jiffies] */
+	unsigned long rs_paused;
+	/* skipped because csum was equal [unit BM_BLOCK_SIZE] */
+	unsigned long rs_same_csum;
+	unsigned long rs_last_progress_report_ts;
+#define DRBD_SYNC_MARKS 8
+#define DRBD_SYNC_MARK_STEP (3*HZ)
+	/* block not up-to-date at mark [unit BM_BLOCK_SIZE] */
+	unsigned long rs_mark_left[DRBD_SYNC_MARKS];
+	/* marks's time [unit jiffies] */
+	unsigned long rs_mark_time[DRBD_SYNC_MARKS];
+	/* current index into rs_mark_{left,time} */
+	int rs_last_mark;
+	unsigned long rs_last_writeout;
+
+	/* where does the admin want us to start? (sector) */
+	sector_t ov_start_sector;
+	sector_t ov_stop_sector;
+	/* where are we now? (sector) */
+	sector_t ov_position;
+	/* Start sector of out of sync range (to merge printk reporting). */
+	sector_t ov_last_oos_start;
+	/* size of out-of-sync range in sectors. */
+	sector_t ov_last_oos_size;
+	/* Start sector of skipped range (to merge printk reporting). */
+	sector_t ov_last_skipped_start;
+	/* size of skipped range in sectors. */
+	sector_t ov_last_skipped_size;
+	int c_sync_rate; /* current resync rate after syncer throttle magic */
+	struct fifo_buffer __rcu *rs_plan_s; /* correction values of resync planer (RCU, connection->conn_update) */
+	atomic_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
+	int rs_last_events;  /* counter of read or write "events" (unit sectors)
+			      * on the lower level device when we last looked. */
+	int rs_in_flight; /* resync sectors in flight (to proxy, in proxy and from proxy) */
+	ktime_t rs_last_mk_req_kt;
+	atomic64_t ov_left; /* in bits */
+	unsigned long ov_skipped; /* in bits */
+	u64 rs_start_uuid;
+
+	u64 current_uuid;
+	u64 bitmap_uuids[DRBD_PEERS_MAX];
+	u64 history_uuids[HISTORY_UUIDS];
+	u64 dirty_bits;
+	u64 uuid_flags;
+	u64 uuid_node_mask; /* might be authoritative_nodes or weak_nodes */
+
+	unsigned long comm_bm_set; /* communicated number of set bits. */
+	u64 comm_current_uuid; /* communicated current UUID */
+	u64 comm_uuid_flags; /* communicated UUID flags */
+	u64 comm_bitmap_uuid;
+	union drbd_state comm_state;
+
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *debugfs_peer_dev;
+	struct dentry *debugfs_peer_dev_proc_drbd;
 #endif
+	ktime_t pre_send_kt;
+	ktime_t acked_kt;
+	ktime_t net_done_kt;
+
+	struct {/* sender todo per peer_device */
+		bool was_sending_out_of_sync;
+	} todo;
+	union drbd_state connect_state;
+	struct {
+		unsigned int	physical_block_size;
+		unsigned int	logical_block_size;
+		unsigned int	alignment_offset;
+		unsigned int	io_min;
+		unsigned int	io_opt;
+		unsigned int	max_bio_size;
+	} q_limits;
+	/* communicated as part of o_qlim, if agreed on DRBD_FF_BM_BLOCK_SHIFT */
+	unsigned int bm_block_shift;
+};
+
+struct conflict_worker {
+	struct workqueue_struct *wq;
+	struct work_struct worker;
+
+	spinlock_t lock;
+	struct list_head resync_writes;
+	struct list_head resync_reads;
+	struct list_head writes;
+	struct list_head peer_writes;
+};
+
+struct submit_worker {
+	struct workqueue_struct *wq;
+	struct work_struct worker;
+
+	spinlock_t lock;
+	struct list_head writes;
+	struct list_head peer_writes;
+};
+
+struct opener {
+	struct list_head list;
+	char comm[TASK_COMM_LEN];
+	pid_t pid;
+	ktime_t opened;
+};
+
+enum drbd_per_device_ratelimit {
+	D_RL_D_NOLIMIT = -1,
+	D_RL_D_GENERIC,
+	D_RL_D_METADATA,
+	D_RL_D_BACKEND,
+	__D_RL_D_N
 };
 
 struct drbd_device {
 	struct drbd_resource *resource;
+
+	/* RCU list. Updates protected by adm_mutex, conf_update and state_rwlock. */
 	struct list_head peer_devices;
+
+	spinlock_t pending_bmio_lock;
 	struct list_head pending_bitmap_io;
 
 	unsigned long flush_jif;
@@ -755,12 +1530,21 @@ struct drbd_device {
 	struct dentry *debugfs_vol;
 	struct dentry *debugfs_vol_oldest_requests;
 	struct dentry *debugfs_vol_act_log_extents;
-	struct dentry *debugfs_vol_resync_extents;
+	struct dentry *debugfs_vol_act_log_histogram;
 	struct dentry *debugfs_vol_data_gen_id;
+	struct dentry *debugfs_vol_io_frozen;
 	struct dentry *debugfs_vol_ed_gen_id;
+	struct dentry *debugfs_vol_openers;
+	struct dentry *debugfs_vol_md_io;
+	struct dentry *debugfs_vol_interval_tree;
+	struct dentry *debugfs_vol_al_updates;
+#ifdef CONFIG_DRBD_TIMING_STATS
+	struct dentry *debugfs_vol_req_timing;
+#endif
 #endif
+	struct ratelimit_state ratelimit[__D_RL_D_N];
 
-	unsigned int vnr;	/* volume number within the connection */
+	unsigned int vnr;	/* volume number within the resource */
 	unsigned int minor;	/* device minor number */
 
 	struct kref kref;
@@ -771,146 +1555,122 @@ struct drbd_device {
 	/* configured by drbdsetup */
 	struct drbd_backing_dev *ldev;
 
-	sector_t p_size;     /* partner's disk size */
+	/* Used to close backing devices and destroy related structures. */
+	struct work_struct ldev_destroy_work;
+
 	struct request_queue *rq_queue;
 	struct gendisk	    *vdisk;
 
 	unsigned long last_reattach_jif;
-	struct drbd_work resync_work;
-	struct drbd_work unplug_work;
-	struct timer_list resync_timer;
 	struct timer_list md_sync_timer;
-	struct timer_list start_resync_timer;
 	struct timer_list request_timer;
 
-	/* Used after attach while negotiating new disk state. */
-	union drbd_state new_state_tmp;
-
-	union drbd_dev_state state;
+	enum drbd_disk_state disk_state[2];
 	wait_queue_head_t misc_wait;
-	wait_queue_head_t state_wait;  /* upon each state change. */
-	unsigned int send_cnt;
-	unsigned int recv_cnt;
 	unsigned int read_cnt;
 	unsigned int writ_cnt;
 	unsigned int al_writ_cnt;
 	unsigned int bm_writ_cnt;
-	atomic_t ap_bio_cnt;	 /* Requests we need to complete */
-	atomic_t ap_actlog_cnt;  /* Requests waiting for activity log */
-	atomic_t ap_pending_cnt; /* AP data packets on the wire, ack expected */
-	atomic_t rs_pending_cnt; /* RS request/data packets on the wire */
-	atomic_t unacked_cnt;	 /* Need to send replies for */
+	atomic_t ap_bio_cnt[2];	 /* Requests we need to complete. [READ] and [WRITE] */
 	atomic_t local_cnt;	 /* Waiting for local completion */
-	atomic_t suspend_cnt;
+	atomic_t ap_actlog_cnt;  /* Requests waiting for activity log */
+	atomic_t wait_for_actlog; /* Peer requests waiting for activity log */
+	/* worst case extent count needed to satisfy both requests and peer requests
+	 * currently waiting for the activity log */
+	atomic_t wait_for_actlog_ecnt;
+
+	atomic_t suspend_cnt;	/* recursive suspend counter, if non-zero, IO will be blocked. */
 
-	/* Interval tree of pending local requests */
-	struct rb_root read_requests;
-	struct rb_root write_requests;
+	/* Interval trees of pending requests */
+	spinlock_t interval_lock;
+	struct rb_root read_requests; /* Local reads */
+	struct rb_root requests; /* Local and peer writes, resync operations etc. */
 
 	/* for statistics and timeouts */
 	/* [0] read, [1] write */
+	spinlock_t pending_completion_lock;
 	struct list_head pending_master_completion[2];
 	struct list_head pending_completion[2];
 
-	/* use checksums for *this* resync */
-	bool use_csums;
-	/* blocks to resync in this run [unit BM_BLOCK_SIZE] */
-	unsigned long rs_total;
-	/* number of resync blocks that failed in this run */
-	unsigned long rs_failed;
-	/* Syncer's start time [unit jiffies] */
-	unsigned long rs_start;
-	/* cumulated time in PausedSyncX state [unit jiffies] */
-	unsigned long rs_paused;
-	/* skipped because csum was equal [unit BM_BLOCK_SIZE] */
-	unsigned long rs_same_csum;
-#define DRBD_SYNC_MARKS 8
-#define DRBD_SYNC_MARK_STEP (3*HZ)
-	/* block not up-to-date at mark [unit BM_BLOCK_SIZE] */
-	unsigned long rs_mark_left[DRBD_SYNC_MARKS];
-	/* marks's time [unit jiffies] */
-	unsigned long rs_mark_time[DRBD_SYNC_MARKS];
-	/* current index into rs_mark_{left,time} */
-	int rs_last_mark;
-	unsigned long rs_last_bcast; /* [unit jiffies] */
-
-	/* where does the admin want us to start? (sector) */
-	sector_t ov_start_sector;
-	sector_t ov_stop_sector;
-	/* where are we now? (sector) */
-	sector_t ov_position;
-	/* Start sector of out of sync range (to merge printk reporting). */
-	sector_t ov_last_oos_start;
-	/* size of out-of-sync range in sectors. */
-	sector_t ov_last_oos_size;
-	unsigned long ov_left; /* in bits */
-
 	struct drbd_bitmap *bitmap;
-	unsigned long bm_resync_fo; /* bit offset for drbd_bm_find_next */
-
-	/* Used to track operations of resync... */
-	struct lru_cache *resync;
-	/* Number of locked elements in resync LRU */
-	unsigned int resync_locked;
-	/* resync extent number waiting for application requests */
-	unsigned int resync_wenr;
+	/* We may want to report on resync progress
+	 * even after we detached again (bitmap == NULL).
+	 * Cache the last bitmap block size here.
+	 */
+	unsigned int last_bm_block_shift;
 
 	int open_cnt;
-	u64 *p_uuid;
+	bool writable;
+	/* FIXME clean comments, restructure so it is more obvious which
+	 * members are protected by what */
 
-	struct list_head active_ee; /* IO in progress (P_DATA gets written to disk) */
-	struct list_head sync_ee;   /* IO in progress (P_RS_DATA_REPLY gets written to disk) */
-	struct list_head done_ee;   /* need to send P_WRITE_ACK */
-	struct list_head read_ee;   /* [RS]P_DATA_REQUEST being read */
-
-	struct list_head resync_reads;
-	atomic_t pp_in_use;		/* allocated from page pool */
-	atomic_t pp_in_use_by_net;	/* sendpage()d, still referenced by tcp */
-	wait_queue_head_t ee_wait;
 	struct drbd_md_io md_io;
 	spinlock_t al_lock;
 	wait_queue_head_t al_wait;
 	struct lru_cache *act_log;	/* activity log */
+	unsigned al_histogram[AL_UPDATES_PER_TRANSACTION+1];
 	unsigned int al_tr_number;
 	int al_tr_cycle;
 	wait_queue_head_t seq_wait;
-	atomic_t packet_seq;
-	unsigned int peer_seq;
-	spinlock_t peer_seq_lock;
-	unsigned long comm_bm_set; /* communicated number of set bits. */
-	struct bm_io_work bm_io_work;
-	u64 ed_uuid; /* UUID of the exposed data */
-	struct mutex own_state_mutex;
-	struct mutex *state_mutex; /* either own_state_mutex or first_peer_device(device)->connection->cstate_mutex */
-	char congestion_reason;  /* Why we where congested... */
-	atomic_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
+	u64 exposed_data_uuid; /* UUID of the exposed data */
+	u64 next_exposed_data_uuid;
+	struct rw_semaphore uuid_sem;
 	atomic_t rs_sect_ev; /* for submitted resync data rate, both */
-	int rs_last_sect_ev; /* counter to compare with */
-	int rs_last_events;  /* counter of read or write "events" (unit sectors)
-			      * on the lower level device when we last looked. */
-	int c_sync_rate; /* current resync rate after syncer throttle magic */
-	struct fifo_buffer *rs_plan_s; /* correction values of resync planer (RCU, connection->conn_update) */
-	int rs_in_flight; /* resync sectors in flight (to proxy, in proxy and from proxy) */
-	atomic_t ap_in_flight; /* App sectors in flight (waiting for ack) */
-	unsigned int peer_max_bio_size;
-	unsigned int local_max_bio_size;
-
-	/* any requests that would block in drbd_make_request()
-	 * are deferred to this single-threaded work queue */
+	struct pending_bitmap_work_s {
+		atomic_t n;		/* inc when queued here, */
+		spinlock_t q_lock;	/* dec only once finished. */
+		struct list_head q;	/* n > 0 even if q already empty */
+	} pending_bitmap_work;
+	struct device_conf device_conf;
+
+	/* any requests that were blocked due to conflicts with other requests
+	 * or resync are submitted on this ordered work queue */
+	struct conflict_worker submit_conflict;
+	/* any requests that would block due to the activity log
+	 * are deferred to this ordered work queue */
 	struct submit_worker submit;
+	u64 read_nodes; /* used for balancing read requests among peers */
+	bool have_quorum[2];	/* no quorum -> suspend IO or error IO */
+	bool cached_state_unstable; /* updates with each state change */
+	bool cached_err_io; /* complete all IOs with error */
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+	spinlock_t timing_lock;
+	unsigned long reqs;
+	ktime_t in_actlog_kt;
+	ktime_t pre_submit_kt; /* sum over over all reqs */
+
+	ktime_t before_queue_kt; /* sum over all al_misses */
+	ktime_t before_al_begin_io_kt;
+
+	ktime_t al_before_bm_write_hinted_kt; /* sum over all al_writ_cnt */
+	ktime_t al_mid_kt;
+	ktime_t al_after_sync_page_kt;
+#endif
+	struct list_head openers;
+	spinlock_t openers_lock;
+
+	struct rcu_head rcu;
+	struct work_struct finalize_work;
 };
 
 struct drbd_bm_aio_ctx {
 	struct drbd_device *device;
-	struct list_head list; /* on device->pending_bitmap_io */;
+	struct list_head list; /* on device->pending_bitmap_io */
 	unsigned long start_jif;
+	struct blk_plug bm_aio_plug;
 	atomic_t in_flight;
 	unsigned int done;
 	unsigned flags;
 #define BM_AIO_COPY_PAGES	1
 #define BM_AIO_WRITE_HINTED	2
 #define BM_AIO_WRITE_ALL_PAGES	4
-#define BM_AIO_READ		8
+#define BM_AIO_READ	        8
+#define BM_AIO_WRITE_LAZY      16
+	/* only report stats for global read, write, write all */
+#define BM_AIO_NO_STATS (BM_AIO_COPY_PAGES\
+			|BM_AIO_WRITE_HINTED\
+			|BM_AIO_WRITE_LAZY)
 	int error;
 	struct kref kref;
 };
@@ -921,12 +1681,14 @@ struct drbd_config_context {
 	/* assigned from request attributes, if present */
 	unsigned int volume;
 #define VOLUME_UNSPECIFIED		(-1U)
+	unsigned int peer_node_id;
+#define PEER_NODE_ID_UNSPECIFIED	(-1U)
 	/* pointer into the request skb,
 	 * limited lifetime! */
 	char *resource_name;
-	struct nlattr *my_addr;
-	struct nlattr *peer_addr;
 
+	/* network namespace of the sending socket */
+	struct net *net;
 	/* reply buffer */
 	struct sk_buff *reply_skb;
 	/* pointer into reply buffer */
@@ -935,6 +1697,7 @@ struct drbd_config_context {
 	struct drbd_device *device;
 	struct drbd_resource *resource;
 	struct drbd_connection *connection;
+	struct drbd_peer_device *peer_device;
 };
 
 static inline struct drbd_device *minor_to_device(unsigned int minor)
@@ -942,10 +1705,6 @@ static inline struct drbd_device *minor_to_device(unsigned int minor)
 	return (struct drbd_device *)idr_find(&drbd_devices, minor);
 }
 
-static inline struct drbd_peer_device *first_peer_device(struct drbd_device *device)
-{
-	return list_first_entry_or_null(&device->peer_devices, struct drbd_peer_device, peer_devices);
-}
 
 static inline struct drbd_peer_device *
 conn_peer_device(struct drbd_connection *connection, int volume_number)
@@ -959,18 +1718,19 @@ conn_peer_device(struct drbd_connection *connection, int volume_number)
 #define for_each_resource_rcu(resource, _resources) \
 	list_for_each_entry_rcu(resource, _resources, resources)
 
-#define for_each_resource_safe(resource, tmp, _resources) \
-	list_for_each_entry_safe(resource, tmp, _resources, resources)
-
+/* see drbd_resource.connections for locking requirements */
 #define for_each_connection(connection, resource) \
 	list_for_each_entry(connection, &resource->connections, connections)
 
 #define for_each_connection_rcu(connection, resource) \
 	list_for_each_entry_rcu(connection, &resource->connections, connections)
 
-#define for_each_connection_safe(connection, tmp, resource) \
-	list_for_each_entry_safe(connection, tmp, &resource->connections, connections)
+#define for_each_connection_ref(connection, m, resource)		\
+	for (connection = __drbd_next_connection_ref(&m, NULL, resource); \
+	     connection;						\
+	     connection = __drbd_next_connection_ref(&m, connection, resource))
 
+/* see drbd_device.peer_devices for locking requirements */
 #define for_each_peer_device(peer_device, device) \
 	list_for_each_entry(peer_device, &device->peer_devices, peer_devices)
 
@@ -980,10 +1740,10 @@ conn_peer_device(struct drbd_connection *connection, int volume_number)
 #define for_each_peer_device_safe(peer_device, tmp, device) \
 	list_for_each_entry_safe(peer_device, tmp, &device->peer_devices, peer_devices)
 
-static inline unsigned int device_to_minor(struct drbd_device *device)
-{
-	return device->minor;
-}
+#define for_each_peer_device_ref(peer_device, m, device)		\
+	for (peer_device = __drbd_next_peer_device_ref(&m, NULL, device); \
+	     peer_device;						\
+	     peer_device = __drbd_next_peer_device_ref(&m, peer_device, device))
 
 /*
  * function declarations
@@ -992,97 +1752,163 @@ static inline unsigned int device_to_minor(struct drbd_device *device)
 /* drbd_main.c */
 
 enum dds_flags {
-	DDSF_FORCED    = 1,
+	/* This enum is part of the wire protocol!
+	 * See P_SIZES, struct p_sizes; */
+	DDSF_ASSUME_UNCONNECTED_PEER_HAS_SPACE    = 1,
 	DDSF_NO_RESYNC = 2, /* Do not run a resync for the new space */
+	DDSF_IGNORE_PEER_CONSTRAINTS = 4, /* no longer used */
+	DDSF_2PC = 8, /* local only, not on the wire */
 };
+struct meta_data_on_disk_9;
 
-extern void drbd_init_set_defaults(struct drbd_device *device);
-extern int  drbd_thread_start(struct drbd_thread *thi);
-extern void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait);
+int drbd_thread_start(struct drbd_thread *thi);
+void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait);
 #ifdef CONFIG_SMP
-extern void drbd_thread_current_set_cpu(struct drbd_thread *thi);
+void drbd_thread_current_set_cpu(struct drbd_thread *thi);
 #else
 #define drbd_thread_current_set_cpu(A) ({})
 #endif
-extern void tl_release(struct drbd_connection *, unsigned int barrier_nr,
-		       unsigned int set_size);
-extern void tl_clear(struct drbd_connection *);
-extern void drbd_free_sock(struct drbd_connection *connection);
-extern int drbd_send(struct drbd_connection *connection, struct socket *sock,
-		     void *buf, size_t size, unsigned msg_flags);
-extern int drbd_send_all(struct drbd_connection *, struct socket *, void *, size_t,
-			 unsigned);
-
-extern int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd);
-extern int drbd_send_protocol(struct drbd_connection *connection);
-extern int drbd_send_uuids(struct drbd_peer_device *);
-extern int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *);
-extern void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *);
-extern int drbd_send_sizes(struct drbd_peer_device *, int trigger_reply, enum dds_flags flags);
-extern int drbd_send_state(struct drbd_peer_device *, union drbd_state s);
-extern int drbd_send_current_state(struct drbd_peer_device *);
-extern int drbd_send_sync_param(struct drbd_peer_device *);
-extern void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr,
-			    u32 set_size);
-extern int drbd_send_ack(struct drbd_peer_device *, enum drbd_packet,
-			 struct drbd_peer_request *);
-extern void drbd_send_ack_rp(struct drbd_peer_device *, enum drbd_packet,
-			     struct p_block_req *rp);
-extern void drbd_send_ack_dp(struct drbd_peer_device *, enum drbd_packet,
-			     struct p_data *dp, int data_size);
-extern int drbd_send_ack_ex(struct drbd_peer_device *, enum drbd_packet,
-			    sector_t sector, int blksize, u64 block_id);
-extern int drbd_send_out_of_sync(struct drbd_peer_device *, struct drbd_request *);
-extern int drbd_send_block(struct drbd_peer_device *, enum drbd_packet,
-			   struct drbd_peer_request *);
-extern int drbd_send_dblock(struct drbd_peer_device *, struct drbd_request *req);
-extern int drbd_send_drequest(struct drbd_peer_device *, int cmd,
-			      sector_t sector, int size, u64 block_id);
-extern int drbd_send_drequest_csum(struct drbd_peer_device *, sector_t sector,
-				   int size, void *digest, int digest_size,
-				   enum drbd_packet cmd);
-extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int size);
-
-extern int drbd_send_bitmap(struct drbd_device *device, struct drbd_peer_device *peer_device);
-extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode);
-extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode);
-extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *);
-extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev);
-extern void drbd_device_cleanup(struct drbd_device *device);
-extern void drbd_print_uuids(struct drbd_device *device, const char *text);
-extern void drbd_queue_unplug(struct drbd_device *device);
-
-extern void conn_md_sync(struct drbd_connection *connection);
-extern void drbd_md_write(struct drbd_device *device, void *buffer);
-extern void drbd_md_sync(struct drbd_device *device);
-extern int  drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev);
-extern void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
-extern void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
-extern void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local);
-extern void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local);
-extern void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local);
-extern void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local);
-extern void drbd_md_set_flag(struct drbd_device *device, int flags) __must_hold(local);
-extern void drbd_md_clear_flag(struct drbd_device *device, int flags)__must_hold(local);
-extern int drbd_md_test_flag(struct drbd_backing_dev *, int);
-extern void drbd_md_mark_dirty(struct drbd_device *device);
-extern void drbd_queue_bitmap_io(struct drbd_device *device,
-				 int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
-				 void (*done)(struct drbd_device *, int),
-				 char *why, enum bm_flag flags,
-				 struct drbd_peer_device *peer_device);
-extern int drbd_bitmap_io(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
-		char *why, enum bm_flag flags,
-		struct drbd_peer_device *peer_device);
-extern int drbd_bitmap_io_from_worker(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
-		char *why, enum bm_flag flags,
-		struct drbd_peer_device *peer_device);
-extern int drbd_bmio_set_n_write(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
-extern int drbd_bmio_clear_n_write(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
+int tl_release(struct drbd_connection *connection, uint64_t o_block_id,
+	       uint64_t y_block_id, unsigned int barrier_nr,
+	       unsigned int set_size);
+
+int __drbd_send_protocol(struct drbd_connection *connection,
+			 enum drbd_packet cmd);
+u64 drbd_collect_local_uuid_flags(struct drbd_peer_device *peer_device,
+				  u64 *authoritative_mask);
+u64 drbd_resolved_uuid(struct drbd_peer_device *peer_device_base,
+		       u64 *uuid_flags);
+int drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags,
+		    u64 node_mask);
+void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device);
+int drbd_send_sizes(struct drbd_peer_device *peer_device,
+		    uint64_t u_size_diskless, enum dds_flags flags);
+int conn_send_state(struct drbd_connection *connection,
+		    union drbd_state state);
+int drbd_send_state(struct drbd_peer_device *peer_device,
+		    union drbd_state state);
+int drbd_send_current_state(struct drbd_peer_device *peer_device);
+int drbd_send_sync_param(struct drbd_peer_device *peer_device);
+int drbd_send_out_of_sync(struct drbd_peer_device *peer_device,
+			  sector_t sector, unsigned int size);
+int drbd_send_block(struct drbd_peer_device *peer_device,
+		    enum drbd_packet cmd, struct drbd_peer_request *peer_req);
+int drbd_send_dblock(struct drbd_peer_device *peer_device,
+		     struct drbd_request *req);
+int drbd_send_drequest(struct drbd_peer_device *peer_device, sector_t sector,
+		       int size, u64 block_id);
+int drbd_send_rs_request(struct drbd_peer_device *peer_device,
+			 enum drbd_packet cmd, sector_t sector, int size,
+			 u64 block_id, unsigned int dagtag_node_id,
+			 u64 dagtag);
+void *drbd_prepare_drequest_csum(struct drbd_peer_request *peer_req,
+				 enum drbd_packet cmd, int digest_size,
+				 unsigned int dagtag_node_id, u64 dagtag);
+
+int drbd_send_bitmap(struct drbd_device *device,
+		     struct drbd_peer_device *peer_device);
+int drbd_send_dagtag(struct drbd_connection *connection, u64 dagtag);
+void drbd_send_sr_reply(struct drbd_connection *connection, int vnr,
+			enum drbd_state_rv retcode);
+int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
+			     struct drbd_peer_request *peer_req);
+void drbd_send_twopc_reply(struct drbd_connection *connection,
+			   enum drbd_packet cmd, struct twopc_reply *reply);
+void drbd_send_peers_in_sync(struct drbd_peer_device *peer_device, u64 mask,
+			     sector_t sector, int size);
+int drbd_send_peer_dagtag(struct drbd_connection *connection,
+			  struct drbd_connection *lost_peer);
+int drbd_send_flush_requests(struct drbd_connection *connection,
+			     u64 flush_sequence);
+int drbd_send_flush_forward(struct drbd_connection *connection,
+			    u64 flush_sequence, int initiator_node_id);
+int drbd_send_flush_requests_ack(struct drbd_connection *connection,
+				 u64 flush_sequence, int primary_node_id);
+int drbd_send_enable_replication_next(struct drbd_peer_device *peer_device);
+int drbd_send_enable_replication(struct drbd_peer_device *peer_device, bool enable);
+int drbd_send_current_uuid(struct drbd_peer_device *peer_device,
+			   u64 current_uuid, u64 weak_nodes);
+void drbd_backing_dev_free(struct drbd_device *device,
+			   struct drbd_backing_dev *ldev);
+void drbd_cleanup_device(struct drbd_device *device);
+void drbd_print_uuids(struct drbd_peer_device *peer_device, const char *text);
+void drbd_queue_unplug(struct drbd_device *device);
+
+u64 drbd_capacity_to_on_disk_bm_sect(u64 capacity_sect, const struct drbd_md *md);
+void drbd_md_set_sector_offsets(struct drbd_backing_dev *bdev);
+int drbd_md_write(struct drbd_device *device,
+		  struct meta_data_on_disk_9 *buffer);
+int drbd_md_sync(struct drbd_device *device);
+int drbd_md_sync_if_dirty(struct drbd_device *device);
+void drbd_uuid_received_new_current(struct drbd_peer_device *from_pd, u64 val,
+				    u64 weak_nodes) __must_hold(local);
+void drbd_uuid_set_bitmap(struct drbd_peer_device *peer_device, u64 uuid) __must_hold(local);
+void _drbd_uuid_set_bitmap(struct drbd_peer_device *peer_device, u64 val) __must_hold(local);
+void _drbd_uuid_set_current(struct drbd_device *device, u64 val) __must_hold(local);
+void drbd_uuid_new_current(struct drbd_device *device, bool forced);
+void drbd_uuid_new_current_by_user(struct drbd_device *device);
+void _drbd_uuid_push_history(struct drbd_device *device, u64 val) __must_hold(local);
+u64 _drbd_uuid_pull_history(struct drbd_peer_device *peer_device) __must_hold(local);
+void drbd_uuid_resync_starting(struct drbd_peer_device *peer_device); __must_hold(local);
+u64 drbd_uuid_resync_finished(struct drbd_peer_device *peer_device) __must_hold(local);
+void drbd_uuid_detect_finished_resyncs(struct drbd_peer_device *peer_device) __must_hold(local);
+bool drbd_uuid_set_exposed(struct drbd_device *device, u64 val, bool log);
+u64 drbd_weak_nodes_device(struct drbd_device *device);
+int drbd_md_test_flag(struct drbd_backing_dev *bdev, enum mdf_flag flag);
+void drbd_md_set_peer_flag(struct drbd_peer_device *peer_device,
+			   enum mdf_peer_flag flag);
+void drbd_md_clear_peer_flag(struct drbd_peer_device *peer_device,
+			     enum mdf_peer_flag flag);
+bool drbd_md_test_peer_flag(struct drbd_peer_device *peer_device,
+			    enum mdf_peer_flag flag);
+void drbd_md_mark_dirty(struct drbd_device *device);
+void drbd_queue_bitmap_io(struct drbd_device *device,
+			  int (*io_fn)(struct drbd_device *device,
+				       struct drbd_peer_device *peer_device),
+			  void (*done)(struct drbd_device *device,
+				       struct drbd_peer_device *peer_device,
+				       int rv),
+			  char *why, enum bm_flag flags,
+			  struct drbd_peer_device *peer_device);
+int drbd_bitmap_io(struct drbd_device *device,
+		   int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		   char *why, enum bm_flag flags,
+		   struct drbd_peer_device *peer_device);
+int drbd_bitmap_io_from_worker(struct drbd_device *device,
+			       int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+			       char *why, enum bm_flag flags,
+			       struct drbd_peer_device *peer_device);
+int drbd_bmio_set_n_write(struct drbd_device *device,
+			  struct drbd_peer_device *peer_device) __must_hold(local);
+int drbd_bmio_clear_all_n_write(struct drbd_device *device,
+				struct drbd_peer_device *peer_device) __must_hold(local);
+int drbd_bmio_set_all_n_write(struct drbd_device *device,
+			      struct drbd_peer_device *peer_device) __must_hold(local);
+int drbd_bmio_set_allocated_n_write(struct drbd_device *device,
+				    struct drbd_peer_device *peer_device) __must_hold(local);
+int drbd_bmio_clear_one_peer(struct drbd_device *device,
+			     struct drbd_peer_device *peer_device) __must_hold(local);
+bool drbd_device_stable(struct drbd_device *device, u64 *authoritative_ptr);
+void drbd_flush_peer_acks(struct drbd_resource *resource);
+void drbd_cork(struct drbd_connection *connection, enum drbd_stream stream);
+void drbd_uncork(struct drbd_connection *connection, enum drbd_stream stream);
+void drbd_open_counts(struct drbd_resource *resource, int *rw_count_ptr,
+		      int *ro_count_ptr);
+
+struct drbd_connection *
+__drbd_next_connection_ref(u64 *visited, struct drbd_connection *connection,
+			   struct drbd_resource *resource);
+
+struct drbd_peer_device *
+__drbd_next_peer_device_ref(u64 *visited,
+			    struct drbd_peer_device *peer_device,
+			    struct drbd_device *device);
+
+void tl_abort_disk_io(struct drbd_device *device);
+
+sector_t drbd_get_max_capacity(struct drbd_device *device,
+			       struct drbd_backing_dev *bdev, bool warn);
+sector_t drbd_partition_data_capacity(struct drbd_device *device);
 
 /* Meta data layout
  *
@@ -1114,59 +1940,10 @@ extern int drbd_bmio_clear_n_write(struct drbd_device *device,
  *  but is about to become configurable.
  */
 
-/* Our old fixed size meta data layout
- * allows up to about 3.8TB, so if you want more,
- * you need to use the "flexible" meta data format. */
-#define MD_128MB_SECT (128LLU << 11)  /* 128 MB, unit sectors */
-#define MD_4kB_SECT	 8
-#define MD_32kB_SECT	64
-
 /* One activity log extent represents 4M of storage */
 #define AL_EXTENT_SHIFT 22
 #define AL_EXTENT_SIZE (1<<AL_EXTENT_SHIFT)
 
-/* We could make these currently hardcoded constants configurable
- * variables at create-md time (or even re-configurable at runtime?).
- * Which will require some more changes to the DRBD "super block"
- * and attach code.
- *
- * updates per transaction:
- *   This many changes to the active set can be logged with one transaction.
- *   This number is arbitrary.
- * context per transaction:
- *   This many context extent numbers are logged with each transaction.
- *   This number is resulting from the transaction block size (4k), the layout
- *   of the transaction header, and the number of updates per transaction.
- *   See drbd_actlog.c:struct al_transaction_on_disk
- * */
-#define AL_UPDATES_PER_TRANSACTION	 64	// arbitrary
-#define AL_CONTEXT_PER_TRANSACTION	919	// (4096 - 36 - 6*64)/4
-
-#if BITS_PER_LONG == 32
-#define LN2_BPL 5
-#define cpu_to_lel(A) cpu_to_le32(A)
-#define lel_to_cpu(A) le32_to_cpu(A)
-#elif BITS_PER_LONG == 64
-#define LN2_BPL 6
-#define cpu_to_lel(A) cpu_to_le64(A)
-#define lel_to_cpu(A) le64_to_cpu(A)
-#else
-#error "LN2 of BITS_PER_LONG unknown!"
-#endif
-
-/* resync bitmap */
-/* 16MB sized 'bitmap extent' to track syncer usage */
-struct bm_extent {
-	int rs_left; /* number of bits set (out of sync) in this extent. */
-	int rs_failed; /* number of failed resync requests in this extent. */
-	unsigned long flags;
-	struct lc_element lce;
-};
-
-#define BME_NO_WRITES  0  /* bm_extent.flags: no more requests on this one! */
-#define BME_LOCKED     1  /* bm_extent.flags: syncer active on this one. */
-#define BME_PRIORITY   2  /* finish resync IO on this extent ASAP! App IO waiting! */
-
 /* drbd_bitmap.c */
 /*
  * We need to store one bit for a block.
@@ -1175,94 +1952,86 @@ struct bm_extent {
  * Bit 1 ==> local node thinks this block needs to be synced.
  */
 
-#define SLEEP_TIME (HZ/10)
-
-/* We do bitmap IO in units of 4k blocks.
- * We also still have a hardcoded 4k per bit relation. */
-#define BM_BLOCK_SHIFT	12			 /* 4k per bit */
-#define BM_BLOCK_SIZE	 (1<<BM_BLOCK_SHIFT)
-/* mostly arbitrarily set the represented size of one bitmap extent,
- * aka resync extent, to 16 MiB (which is also 512 Byte worth of bitmap
- * at 4k per bit resolution) */
-#define BM_EXT_SHIFT	 24	/* 16 MiB per resync extent */
-#define BM_EXT_SIZE	 (1<<BM_EXT_SHIFT)
-
-#if (BM_EXT_SHIFT != 24) || (BM_BLOCK_SHIFT != 12)
-#error "HAVE YOU FIXED drbdmeta AS WELL??"
-#endif
-
-/* thus many _storage_ sectors are described by one bit */
-#define BM_SECT_TO_BIT(x)   ((x)>>(BM_BLOCK_SHIFT-9))
-#define BM_BIT_TO_SECT(x)   ((sector_t)(x)<<(BM_BLOCK_SHIFT-9))
-#define BM_SECT_PER_BIT     BM_BIT_TO_SECT(1)
-
-/* bit to represented kilo byte conversion */
-#define Bit2KB(bits) ((bits)<<(BM_BLOCK_SHIFT-10))
+#define RS_MAKE_REQS_INTV    (HZ/10)
+#define RS_MAKE_REQS_INTV_NS (NSEC_PER_SEC/10)
 
-/* in which _bitmap_ extent (resp. sector) the bit for a certain
- * _storage_ sector is located in */
-#define BM_SECT_TO_EXT(x)   ((x)>>(BM_EXT_SHIFT-9))
-#define BM_BIT_TO_EXT(x)    ((x) >> (BM_EXT_SHIFT - BM_BLOCK_SHIFT))
-
-/* first storage sector a bitmap extent corresponds to */
-#define BM_EXT_TO_SECT(x)   ((sector_t)(x) << (BM_EXT_SHIFT-9))
-/* how much _storage_ sectors we have per bitmap extent */
-#define BM_SECT_PER_EXT     BM_EXT_TO_SECT(1)
-/* how many bits are covered by one bitmap extent (resync extent) */
-#define BM_BITS_PER_EXT     (1UL << (BM_EXT_SHIFT - BM_BLOCK_SHIFT))
-
-#define BM_BLOCKS_PER_BM_EXT_MASK  (BM_BITS_PER_EXT - 1)
+#define LEGACY_BM_EXT_SHIFT	 27	/* 128 MiB per resync extent */
+#define LEGACY_BM_EXT_SECT_MASK ((1UL << (LEGACY_BM_EXT_SHIFT - SECTOR_SHIFT)) - 1)
 
+static inline unsigned int bm_block_size(const struct drbd_bitmap *bm)
+{
+	return 1 << bm->bm_block_shift;
+}
+static inline sector_t bm_bit_to_kb(const struct drbd_bitmap *bm, unsigned long bit)
+{
+	return (sector_t)bit << (bm->bm_block_shift - 10);
+}
+static inline unsigned long bm_sect_to_bit(const struct drbd_bitmap *bm, sector_t s)
+{
+	return s >> (bm->bm_block_shift - 9);
+}
+static inline sector_t bm_bit_to_sect(const struct drbd_bitmap *bm, unsigned long bit)
+{
+	return (sector_t)bit << (bm->bm_block_shift - 9);
+}
+static inline sector_t bm_sect_per_bit(const struct drbd_bitmap *bm)
+{
+	return (sector_t)1 << (bm->bm_block_shift - 9);
+}
 
-/* in one sector of the bitmap, we have this many activity_log extents. */
-#define AL_EXT_PER_BM_SECT  (1 << (BM_EXT_SHIFT - AL_EXTENT_SHIFT))
+static inline sector_t bit_to_kb(unsigned long bit, unsigned int bm_block_shift)
+{
+	return (sector_t)bit << (bm_block_shift - 10);
+}
+static inline unsigned long sect_to_bit(sector_t s, unsigned int bm_block_shift)
+{
+	return s >> (bm_block_shift - 9);
+}
+static inline sector_t bit_to_sect(unsigned long bit, unsigned int bm_block_shift)
+{
+	return (sector_t)bit << (bm_block_shift - 9);
+}
+static inline sector_t sect_per_bit(unsigned int bm_block_shift)
+{
+	return (sector_t)1 << (bm_block_shift - 9);
+}
 
-/* the extent in "PER_EXTENT" below is an activity log extent
- * we need that many (long words/bytes) to store the bitmap
- *		     of one AL_EXTENT_SIZE chunk of storage.
- * we can store the bitmap for that many AL_EXTENTS within
- * one sector of the _on_disk_ bitmap:
- * bit	 0	  bit 37   bit 38	     bit (512*8)-1
- *	     ...|........|........|.. // ..|........|
- * sect. 0	 `296	  `304			   ^(512*8*8)-1
- *
-#define BM_WORDS_PER_EXT    ( (AL_EXT_SIZE/BM_BLOCK_SIZE) / BITS_PER_LONG )
-#define BM_BYTES_PER_EXT    ( (AL_EXT_SIZE/BM_BLOCK_SIZE) / 8 )  // 128
-#define BM_EXT_PER_SECT	    ( 512 / BM_BYTES_PER_EXTENT )	 //   4
+/* We may have just lost our backing device, and with it ->ldev and ->bitmap.
+ * But we can still report sync progress and similar based on our last known
+ * bitmap block size.
  */
+static inline sector_t device_bit_to_kb(struct drbd_device *device, unsigned long bit)
+{
+	return bit_to_kb(bit, device->last_bm_block_shift);
+}
 
-#define DRBD_MAX_SECTORS_32 (0xffffffffLU)
-/* we have a certain meta data variant that has a fixed on-disk size of 128
- * MiB, of which 4k are our "superblock", and 32k are the fixed size activity
+/* Send P_PEERS_IN_SYNC in steps defined by this shift. Set to the activity log
+ * extent shift since the P_PEERS_IN_SYNC intervals are broken up based on
+ * activity log extents anyway. */
+#define PEERS_IN_SYNC_STEP_SHIFT AL_EXTENT_SHIFT
+#define PEERS_IN_SYNC_STEP_SECT_MASK ((1UL << (PEERS_IN_SYNC_STEP_SHIFT - SECTOR_SHIFT)) - 1)
+
+/* Indexed external meta data has a fixed on-disk size of 128MiB, of which
+ * 4KiB are our "superblock", and 32KiB are the fixed size activity
  * log, leaving this many sectors for the bitmap.
  */
+#define DRBD_BM_SECTORS_INDEXED \
+	  (((128 << 20) - (32 << 10) - (4 << 10)) >> SECTOR_SHIFT)
 
-#define DRBD_MAX_SECTORS_FIXED_BM \
-	  ((MD_128MB_SECT - MD_32kB_SECT - MD_4kB_SECT) * (1LL<<(BM_EXT_SHIFT-9)))
-#define DRBD_MAX_SECTORS      DRBD_MAX_SECTORS_FIXED_BM
-/* 16 TB in units of sectors */
 #if BITS_PER_LONG == 32
-/* adjust by one page worth of bitmap,
- * so we won't wrap around in drbd_bm_find_next_bit.
- * you should use 64bit OS for that much storage, anyways. */
-#define DRBD_MAX_SECTORS_FLEX BM_BIT_TO_SECT(0xffff7fff)
+#if !defined(CONFIG_LBDAF) && !defined(CONFIG_LBD)
+#define DRBD_MAX_SECTORS (0xffffffffLU)
 #else
-/* we allow up to 1 PiB now on 64bit architecture with "flexible" meta data */
-#define DRBD_MAX_SECTORS_FLEX (1UL << 51)
-/* corresponds to (1UL << 38) bits right now. */
+/* With large block device support, the size is limited by the fact that we
+ * want to be able to address bitmap bits with a long. Additionally adjust by
+ * one page worth of bitmap, so we don't wrap around when iterating. */
+#define DRBD_MAX_SECTORS BM_BIT_TO_SECT(0xffff7fff)
 #endif
-
-/* Estimate max bio size as 256 * PAGE_SIZE,
- * so for typical PAGE_SIZE of 4k, that is (1<<20) Byte.
- * Since we may live in a mixed-platform cluster,
- * we limit us to a platform agnostic constant here for now.
- * A followup commit may allow even bigger BIO sizes,
- * once we thought that through. */
-#define DRBD_MAX_BIO_SIZE (1U << 20)
-#if DRBD_MAX_BIO_SIZE > (BIO_MAX_VECS << PAGE_SHIFT)
-#error Architecture not supported: DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE
+#else
+/* We allow up to 1 PiB on 64 bit architectures as long as our meta data
+ * is large enough. */
+#define DRBD_MAX_SECTORS (1UL << (50 - SECTOR_SHIFT))
 #endif
-#define DRBD_MAX_BIO_SIZE_SAFE (1U << 12)       /* Works always = 4k */
 
 #define DRBD_MAX_SIZE_H80_PACKET (1U << 15) /* Header 80 only allows packets up to 32KiB data */
 #define DRBD_MAX_BIO_SIZE_P95    (1U << 17) /* Protocol 95 to 99 allows bios up to 128KiB */
@@ -1273,61 +2042,93 @@ struct bm_extent {
 #define DRBD_MAX_BATCH_BIO_SIZE	 (AL_UPDATES_PER_TRANSACTION/2*AL_EXTENT_SIZE)
 #define DRBD_MAX_BBIO_SECTORS    (DRBD_MAX_BATCH_BIO_SIZE >> 9)
 
-extern int  drbd_bm_init(struct drbd_device *device);
-extern int  drbd_bm_resize(struct drbd_device *device, sector_t sectors, int set_new_bits);
-extern void drbd_bm_cleanup(struct drbd_device *device);
-extern void drbd_bm_set_all(struct drbd_device *device);
-extern void drbd_bm_clear_all(struct drbd_device *device);
+/* This gets ignored if the backing device has a larger discard granularity */
+#define DRBD_MAX_RS_DISCARD_SIZE (1U << 27) /* 128MiB; arbitrary */
+
+/* how many activity log extents are touched by this interval? */
+static inline int interval_to_al_extents(struct drbd_interval *i)
+{
+	unsigned int first = i->sector >> (AL_EXTENT_SHIFT-9);
+	unsigned int last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
+	return 1 + last - first; /* worst case: all touched extends are cold. */
+}
+
+struct drbd_bitmap *drbd_bm_alloc(unsigned int max_peers, unsigned int bm_block_shift);
+int  drbd_bm_resize(struct drbd_device *device, sector_t capacity,
+		    bool set_new_bits);
+void drbd_bm_free(struct drbd_device *device);
+void drbd_bm_set_all(struct drbd_device *device);
+void drbd_bm_clear_all(struct drbd_device *device);
 /* set/clear/test only a few bits at a time */
-extern int  drbd_bm_set_bits(
-		struct drbd_device *device, unsigned long s, unsigned long e);
-extern int  drbd_bm_clear_bits(
-		struct drbd_device *device, unsigned long s, unsigned long e);
-extern int drbd_bm_count_bits(
-	struct drbd_device *device, const unsigned long s, const unsigned long e);
+unsigned int drbd_bm_set_bits(struct drbd_device *device,
+			      unsigned int bitmap_index, unsigned long start,
+			      unsigned long end);
+unsigned int drbd_bm_clear_bits(struct drbd_device *device,
+				unsigned int bitmap_index,
+				unsigned long start, unsigned long end);
+int drbd_bm_count_bits(struct drbd_device *device, unsigned int bitmap_index,
+		       unsigned long s, unsigned long e);
 /* bm_set_bits variant for use while holding drbd_bm_lock,
  * may process the whole bitmap in one go */
-extern void _drbd_bm_set_bits(struct drbd_device *device,
-		const unsigned long s, const unsigned long e);
-extern int  drbd_bm_test_bit(struct drbd_device *device, unsigned long bitnr);
-extern int  drbd_bm_e_weight(struct drbd_device *device, unsigned long enr);
-extern int  drbd_bm_read(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
-extern void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr);
-extern int  drbd_bm_write(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
-extern void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local);
-extern int  drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local);
-extern int  drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local);
-extern int drbd_bm_write_all(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
-extern int  drbd_bm_write_copy_pages(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local);
-extern size_t	     drbd_bm_words(struct drbd_device *device);
-extern unsigned long drbd_bm_bits(struct drbd_device *device);
-extern sector_t      drbd_bm_capacity(struct drbd_device *device);
+void drbd_bm_set_many_bits(struct drbd_peer_device *peer_device,
+			   unsigned long start, unsigned long end);
+void drbd_bm_clear_many_bits(struct drbd_peer_device *peer_device,
+			     unsigned long start, unsigned long end);
+void _drbd_bm_clear_many_bits(struct drbd_device *device, int bitmap_index,
+			      unsigned long start, unsigned long end);
+void _drbd_bm_set_many_bits(struct drbd_device *device, int bitmap_index,
+			    unsigned long start, unsigned long end);
+int drbd_bm_test_bit(struct drbd_peer_device *peer_device,
+		     const unsigned long bitnr);
+int  drbd_bm_read(struct drbd_device *device,
+		  struct drbd_peer_device *peer_device) __must_hold(local);
+void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local);
+void drbd_bm_mark_range_for_writeout(struct drbd_device *device,
+				     unsigned long start, unsigned long end);
+int  drbd_bm_write(struct drbd_device *device,
+		   struct drbd_peer_device *peer_device) __must_hold(local);
+int  drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local);
+int  drbd_bm_write_lazy(struct drbd_device *device, unsigned int upper_idx) __must_hold(local);
+int drbd_bm_write_all(struct drbd_device *device,
+		      struct drbd_peer_device *peer_device) __must_hold(local);
+int drbd_bm_write_copy_pages(struct drbd_device *device,
+			     struct drbd_peer_device *peer_device) __must_hold(local);
+size_t	     drbd_bm_words(struct drbd_device *device);
+unsigned long drbd_bm_bits(struct drbd_device *device);
+unsigned long drbd_bm_bits_4k(struct drbd_device *device);
+sector_t      drbd_bm_capacity(struct drbd_device *device);
 
 #define DRBD_END_OF_BITMAP	(~(unsigned long)0)
-extern unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo);
+unsigned long drbd_bm_find_next(struct drbd_peer_device *peer_device,
+				unsigned long start);
 /* bm_find_next variants for use while you hold drbd_bm_lock() */
-extern unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo);
-extern unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo);
-extern unsigned long _drbd_bm_total_weight(struct drbd_device *device);
-extern unsigned long drbd_bm_total_weight(struct drbd_device *device);
+unsigned long _drbd_bm_find_next(struct drbd_peer_device *peer_device,
+				 unsigned long start);
+unsigned long _drbd_bm_find_next_zero(struct drbd_peer_device *peer_device,
+				      unsigned long start);
+unsigned long _drbd_bm_total_weight(struct drbd_device *device,
+				    int bitmap_index);
+unsigned long drbd_bm_total_weight(struct drbd_peer_device *peer_device);
 /* for receive_bitmap */
-extern void drbd_bm_merge_lel(struct drbd_device *device, size_t offset,
-		size_t number, unsigned long *buffer);
+void drbd_bm_merge_lel(struct drbd_peer_device *peer_device, size_t offset,
+		       size_t number, unsigned long *buffer);
 /* for _drbd_send_bitmap */
-extern void drbd_bm_get_lel(struct drbd_device *device, size_t offset,
-		size_t number, unsigned long *buffer);
-
-extern void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags);
-extern void drbd_bm_unlock(struct drbd_device *device);
+void drbd_bm_get_lel(struct drbd_peer_device *peer_device, size_t offset,
+		     size_t number, unsigned long *buffer);
+
+void drbd_bm_lock(struct drbd_device *device, const char *why,
+		  enum bm_flag flags);
+void drbd_bm_unlock(struct drbd_device *device);
+void drbd_bm_slot_lock(struct drbd_peer_device *peer_device, char *why,
+		       enum bm_flag flags);
+void drbd_bm_slot_unlock(struct drbd_peer_device *peer_device);
+void drbd_bm_copy_slot(struct drbd_device *device, unsigned int from_index,
+		       unsigned int to_index);
 /* drbd_main.c */
 
+extern struct workqueue_struct *ping_ack_sender;
 extern struct kmem_cache *drbd_request_cache;
 extern struct kmem_cache *drbd_ee_cache;	/* peer requests */
-extern struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
 extern struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
 extern mempool_t drbd_request_mempool;
 extern mempool_t drbd_ee_mempool;
@@ -1348,38 +2149,69 @@ extern struct bio_set drbd_md_io_bio_set;
 /* And a bio_set for cloning */
 extern struct bio_set drbd_io_bio_set;
 
-extern struct mutex resources_mutex;
-
-extern enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor);
-extern void drbd_destroy_device(struct kref *kref);
-extern void drbd_delete_device(struct drbd_device *device);
-
-extern struct drbd_resource *drbd_create_resource(const char *name);
-extern void drbd_free_resource(struct drbd_resource *resource);
-
-extern int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts);
-extern struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts);
-extern void drbd_destroy_connection(struct kref *kref);
-extern struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
-					    void *peer_addr, int peer_addr_len);
-extern struct drbd_resource *drbd_find_resource(const char *name);
-extern void drbd_destroy_resource(struct kref *kref);
-extern void conn_free_crypto(struct drbd_connection *connection);
+struct drbd_peer_device *create_peer_device(struct drbd_device *device,
+					    struct drbd_connection *connection);
+enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx,
+				      unsigned int minor,
+				      struct device_conf *device_conf,
+				      struct drbd_device **p_device);
+void drbd_unregister_device(struct drbd_device *device);
+void drbd_reclaim_device(struct rcu_head *rp);
+void drbd_unregister_connection(struct drbd_connection *connection);
+void drbd_reclaim_connection(struct rcu_head *rp);
+void drbd_reclaim_path(struct rcu_head *rp);
+void del_connect_timer(struct drbd_connection *connection);
+
+struct drbd_resource *drbd_create_resource(const char *name,
+					   struct res_opts *res_opts);
+void drbd_reclaim_resource(struct rcu_head *rp);
+struct drbd_resource *drbd_find_resource(const char *name);
+void drbd_destroy_resource(struct kref *kref);
+
+void drbd_destroy_device(struct kref *kref);
+
+int set_resource_options(struct drbd_resource *resource,
+			 struct res_opts *res_opts, const char *tag);
+struct drbd_connection *drbd_create_connection(struct drbd_resource *resource,
+					       struct drbd_transport_class *tc);
+void drbd_transport_shutdown(struct drbd_connection *connection,
+			     enum drbd_tr_free_op op);
+void drbd_destroy_connection(struct kref *kref);
+void conn_free_crypto(struct drbd_connection *connection);
 
 /* drbd_req */
-extern void do_submit(struct work_struct *ws);
-extern void __drbd_make_request(struct drbd_device *, struct bio *);
+void drbd_do_submit_conflict(struct work_struct *ws);
+void do_submit(struct work_struct *ws);
+#ifndef CONFIG_DRBD_TIMING_STATS
+#define __drbd_make_request(d, b, k, j) __drbd_make_request(d, b, j)
+#endif
+void __drbd_make_request(struct drbd_device *device, struct bio *bio,
+			 ktime_t start_kt, unsigned long start_jif);
 void drbd_submit_bio(struct bio *bio);
 
-/* drbd_nl.c */
-
-extern struct mutex notification_mutex;
+enum drbd_force_detach_flags {
+	DRBD_READ_ERROR,
+	DRBD_WRITE_ERROR,
+	DRBD_META_IO_ERROR,
+	DRBD_FORCE_DETACH,
+};
+#define drbd_handle_io_error(m, f) drbd_handle_io_error_(m, f,  __func__)
+void drbd_handle_io_error_(struct drbd_device *device,
+			   enum drbd_force_detach_flags df, const char *where);
 
-extern void drbd_suspend_io(struct drbd_device *device);
-extern void drbd_resume_io(struct drbd_device *device);
-extern char *ppsize(char *buf, unsigned long long size);
-extern sector_t drbd_new_dev_size(struct drbd_device *, struct drbd_backing_dev *, sector_t, int);
+/* drbd_nl.c */
+enum suspend_scope {
+	READ_AND_WRITE,
+	WRITE_ONLY
+};
+void drbd_suspend_io(struct drbd_device *device, enum suspend_scope ss);
+void drbd_resume_io(struct drbd_device *device);
+char *ppsize(char *buf, unsigned long long size);
+sector_t drbd_new_dev_size(struct drbd_device *device, sector_t current_size,
+			   sector_t user_capped_size, enum dds_flags flags) __must_hold(local);
 enum determine_dev_size {
+	DS_2PC_ERR = -5,
+	DS_2PC_NOT_SUPPORTED = -4,
 	DS_ERROR_SHRINK = -3,
 	DS_ERROR_SPACE_MD = -2,
 	DS_ERROR = -1,
@@ -1388,96 +2220,225 @@ enum determine_dev_size {
 	DS_GREW = 2,
 	DS_GREW_FROM_ZERO = 3,
 };
-extern enum determine_dev_size
-drbd_determine_dev_size(struct drbd_device *, enum dds_flags, struct resize_parms *) __must_hold(local);
-extern void resync_after_online_grow(struct drbd_device *);
-extern void drbd_reconsider_queue_parameters(struct drbd_device *device,
-			struct drbd_backing_dev *bdev, struct o_qlim *o);
-extern enum drbd_state_rv drbd_set_role(struct drbd_device *device,
-					enum drbd_role new_role,
-					int force);
-extern bool conn_try_outdate_peer(struct drbd_connection *connection);
-extern void conn_try_outdate_peer_async(struct drbd_connection *connection);
-extern enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd);
-extern int drbd_khelper(struct drbd_device *device, char *cmd);
-
-/* drbd_worker.c */
-/* bi_end_io handlers */
-extern void drbd_md_endio(struct bio *bio);
-extern void drbd_peer_request_endio(struct bio *bio);
-extern void drbd_request_endio(struct bio *bio);
-extern int drbd_worker(struct drbd_thread *thi);
+enum determine_dev_size
+drbd_determine_dev_size(struct drbd_device *device,
+			sector_t peer_current_size, enum dds_flags flags,
+			struct resize_parms *rs) __must_hold(local);
+void resync_after_online_grow(struct drbd_peer_device *peer_device);
+void drbd_reconsider_queue_parameters(struct drbd_device *device,
+				      struct drbd_backing_dev *bdev);
+bool barrier_pending(struct drbd_resource *resource);
+enum drbd_state_rv
+drbd_set_role(struct drbd_resource *resource, enum drbd_role role, bool force,
+	      const char *tag, struct sk_buff *reply_skb);
+void conn_try_outdate_peer_async(struct drbd_connection *connection);
+int drbd_maybe_khelper(struct drbd_device *device,
+		       struct drbd_connection *connection, char *cmd);
+int drbd_create_peer_device_default_config(struct drbd_peer_device *peer_device);
+int drbd_unallocated_index(struct drbd_backing_dev *bdev);
+void youngest_and_oldest_opener_to_str(struct drbd_device *device, char *buf,
+				       size_t len);
+int param_set_drbd_strict_names(const char *val,
+				const struct kernel_param *kp);
+void drbd_enable_netns(void);
+
+/* drbd_sender.c */
+int drbd_sender(struct drbd_thread *thi);
+int drbd_worker(struct drbd_thread *thi);
 enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor);
 void drbd_resync_after_changed(struct drbd_device *device);
-extern void drbd_start_resync(struct drbd_device *device, enum drbd_conns side);
-extern void resume_next_sg(struct drbd_device *device);
-extern void suspend_other_sg(struct drbd_device *device);
-extern int drbd_resync_finished(struct drbd_peer_device *peer_device);
+bool drbd_stable_sync_source_present(struct drbd_peer_device *except_peer_device,
+				     enum which_state which);
+void drbd_start_resync(struct drbd_peer_device *peer_device,
+		       enum drbd_repl_state side, const char *tag);
+void resume_next_sg(struct drbd_device *device);
+void suspend_other_sg(struct drbd_device *device);
+void drbd_resync_finished(struct drbd_peer_device *peer_device,
+			  enum drbd_disk_state new_peer_disk_state);
+void verify_progress(struct drbd_peer_device *peer_device,
+		     const sector_t sector, const unsigned int size);
 /* maybe rather drbd_main.c ? */
-extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent);
-extern void drbd_md_put_buffer(struct drbd_device *device);
-extern int drbd_md_sync_page_io(struct drbd_device *device,
-		struct drbd_backing_dev *bdev, sector_t sector, enum req_op op);
-extern void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device,
-		sector_t sector, int size);
-extern void wait_until_done_or_force_detached(struct drbd_device *device,
-		struct drbd_backing_dev *bdev, unsigned int *done);
-extern void drbd_rs_controller_reset(struct drbd_peer_device *peer_device);
+void *drbd_md_get_buffer(struct drbd_device *device, const char *intent);
+void drbd_md_put_buffer(struct drbd_device *device);
+int drbd_md_sync_page_io(struct drbd_device *device,
+			 struct drbd_backing_dev *bdev, sector_t sector,
+			 enum req_op op);
+bool drbd_al_active(struct drbd_device *device, sector_t sector,
+		    unsigned int size);
+void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device,
+			       sector_t sector, int size);
+void wait_until_done_or_force_detached(struct drbd_device *device,
+				       struct drbd_backing_dev *bdev,
+				       unsigned int *done);
+void drbd_rs_controller_reset(struct drbd_peer_device *peer_device);
+void drbd_rs_all_in_flight_came_back(struct drbd_peer_device *peer_device,
+				     int rs_sect_in);
+void drbd_check_peers(struct drbd_resource *resource);
+void drbd_check_peers_new_current_uuid(struct drbd_device *device);
+void drbd_conflict_send_resync_request(struct drbd_peer_request *peer_req);
+void drbd_ping_peer(struct drbd_connection *connection);
+struct drbd_peer_device *peer_device_by_node_id(struct drbd_device *device,
+						int node_id);
+void drbd_update_mdf_al_disabled(struct drbd_device *device,
+				 enum which_state which);
 
 static inline void ov_out_of_sync_print(struct drbd_peer_device *peer_device)
 {
-	struct drbd_device *device = peer_device->device;
-
-	if (device->ov_last_oos_size) {
+	if (peer_device->ov_last_oos_size) {
 		drbd_err(peer_device, "Out of sync: start=%llu, size=%lu (sectors)\n",
-		     (unsigned long long)device->ov_last_oos_start,
-		     (unsigned long)device->ov_last_oos_size);
+		     (unsigned long long)peer_device->ov_last_oos_start,
+		     (unsigned long)peer_device->ov_last_oos_size);
 	}
-	device->ov_last_oos_size = 0;
+	peer_device->ov_last_oos_size = 0;
 }
 
+static inline void ov_skipped_print(struct drbd_peer_device *peer_device)
+{
+	if (peer_device->ov_last_skipped_size) {
+		drbd_info(peer_device, "Skipped verify, too busy: start=%llu, size=%lu (sectors)\n",
+		     (unsigned long long)peer_device->ov_last_skipped_start,
+		     (unsigned long)peer_device->ov_last_skipped_size);
+	}
+	peer_device->ov_last_skipped_size = 0;
+}
+
+void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest);
+void drbd_csum_pages(struct crypto_shash *tfm, struct page *page,
+		     void *digest);
+void drbd_resync_read_req_mod(struct drbd_peer_request *peer_req,
+			      enum drbd_interval_flags bit_to_set);
 
-extern void drbd_csum_bio(struct crypto_shash *, struct bio *, void *);
-extern void drbd_csum_ee(struct crypto_shash *, struct drbd_peer_request *,
-			 void *);
 /* worker callbacks */
-extern int w_e_end_data_req(struct drbd_work *, int);
-extern int w_e_end_rsdata_req(struct drbd_work *, int);
-extern int w_e_end_csum_rs_req(struct drbd_work *, int);
-extern int w_e_end_ov_reply(struct drbd_work *, int);
-extern int w_e_end_ov_req(struct drbd_work *, int);
-extern int w_ov_finished(struct drbd_work *, int);
-extern int w_resync_timer(struct drbd_work *, int);
-extern int w_send_write_hint(struct drbd_work *, int);
-extern int w_send_dblock(struct drbd_work *, int);
-extern int w_send_read_req(struct drbd_work *, int);
-extern int w_restart_disk_io(struct drbd_work *, int);
-extern int w_send_out_of_sync(struct drbd_work *, int);
-
-extern void resync_timer_fn(struct timer_list *t);
-extern void start_resync_timer_fn(struct timer_list *t);
-
-extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req);
+int w_e_end_data_req(struct drbd_work *w, int cancel);
+int w_e_end_rsdata_req(struct drbd_work *w, int cancel);
+int w_e_end_ov_reply(struct drbd_work *w, int cancel);
+int w_e_end_ov_req(struct drbd_work *w, int cancel);
+int w_resync_timer(struct drbd_work *w, int cancel);
+int w_e_reissue(struct drbd_work *w, int cancel);
+int w_send_dagtag(struct drbd_work *w, int cancel);
+int w_send_uuids(struct drbd_work *w, int cancel);
+
+bool drbd_any_flush_pending(struct drbd_resource *resource);
+void resync_timer_fn(struct timer_list *t);
+void start_resync_timer_fn(struct timer_list *t);
+
+int drbd_unmerge_discard(struct drbd_peer_request *peer_req_main,
+			 struct list_head *list);
+void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req);
+
+/* bi_end_io handlers */
+void drbd_md_endio(struct bio *bio);
+void drbd_peer_request_endio(struct bio *bio);
+void drbd_request_endio(struct bio *bio);
+
+void __update_timing_details(
+		struct drbd_thread_timing_details *tdp,
+		unsigned int *cb_nr,
+		void *cb,
+		const char *fn, const unsigned int line);
+
+#define update_sender_timing_details(c, cb) \
+	__update_timing_details(c->s_timing_details, &c->s_cb_nr, cb, __func__, __LINE__)
+#define update_receiver_timing_details(c, cb) \
+	__update_timing_details(c->r_timing_details, &c->r_cb_nr, cb, __func__, __LINE__)
+#define update_worker_timing_details(r, cb) \
+	__update_timing_details(r->w_timing_details, &r->w_cb_nr, cb, __func__, __LINE__)
 
 /* drbd_receiver.c */
-extern int drbd_issue_discard_or_zero_out(struct drbd_device *device,
-		sector_t start, unsigned int nr_sectors, int flags);
-extern int drbd_receiver(struct drbd_thread *thi);
-extern int drbd_ack_receiver(struct drbd_thread *thi);
-extern void drbd_send_acks_wf(struct work_struct *ws);
-extern bool drbd_rs_c_min_rate_throttle(struct drbd_device *device);
-extern bool drbd_rs_should_slow_down(struct drbd_peer_device *peer_device, sector_t sector,
-		bool throttle_if_app_is_waiting);
-extern int drbd_submit_peer_request(struct drbd_peer_request *peer_req);
-extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *);
-extern struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *, u64,
-						     sector_t, unsigned int,
-						     unsigned int,
-						     gfp_t) __must_hold(local);
-extern void drbd_free_peer_req(struct drbd_device *device, struct drbd_peer_request *req);
-extern struct page *drbd_alloc_pages(struct drbd_peer_device *, unsigned int, bool);
-extern void _drbd_clear_done_ee(struct drbd_device *device, struct list_head *to_be_freed);
-extern int drbd_connected(struct drbd_peer_device *);
+struct packet_info {
+	enum drbd_packet cmd;
+	unsigned int size;
+	int vnr;
+	void *data;
+};
+
+/* packet_info->data is just a pointer into some temporary buffer
+ * owned by the transport. As soon as we call into the transport for
+ * any further receive operation, the data it points to is undefined.
+ * The buffer may be freed/recycled/re-used already.
+ * Convert and store the relevant information for any incoming data
+ * in drbd_peer_request_detail.
+ */
+
+struct drbd_peer_request_details {
+	uint64_t sector;	/* be64_to_cpu(p_data.sector) */
+	uint64_t block_id;	/* unmodified p_data.block_id */
+	uint32_t peer_seq;	/* be32_to_cpu(p_data.seq_num) */
+	uint32_t dp_flags;	/* be32_to_cpu(p_data.dp_flags) */
+	uint32_t length;	/* endian converted p_head*.length */
+	uint32_t bi_size;	/* resulting bio size */
+	/* for non-discards: bi_size = length - digest_size */
+	uint32_t digest_size;
+};
+
+
+void drbd_queue_update_peers(struct drbd_peer_device *peer_device,
+			     sector_t sector_start, sector_t sector_end);
+int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start,
+				   unsigned int nr_sectors, int flags);
+int drbd_send_ack_be(struct drbd_peer_device *peer_device,
+		     enum drbd_packet cmd, sector_t sector, int size,
+		     u64 block_id);
+int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+		  struct drbd_peer_request *peer_req);
+int drbd_send_ov_result(struct drbd_peer_device *peer_device, sector_t sector,
+			int blksize, u64 block_id, enum ov_result result);
+int drbd_receiver(struct drbd_thread *thi);
+void drbd_unsuccessful_resync_request(struct drbd_peer_request *peer_req,
+				      bool failed);
+int drbd_send_out_of_sync_wf(struct drbd_work *w, int cancel);
+int drbd_flush_ack_wf(struct drbd_work *w, int unused);
+void drbd_send_ping_wf(struct work_struct *ws);
+void drbd_send_acks_wf(struct work_struct *ws);
+void drbd_send_peer_ack_wf(struct work_struct *ws);
+bool drbd_rs_c_min_rate_throttle(struct drbd_peer_device *peer_device);
+void drbd_verify_skipped_block(struct drbd_peer_device *peer_device,
+			       const sector_t sector, const unsigned int size);
+void drbd_conflict_submit_resync_request(struct drbd_peer_request *peer_req);
+void drbd_conflict_submit_peer_read(struct drbd_peer_request *peer_req);
+void drbd_conflict_submit_peer_write(struct drbd_peer_request *peer_req);
+int drbd_submit_peer_request(struct drbd_peer_request *peer_req);
+void drbd_cleanup_after_failed_submit_peer_write(struct drbd_peer_request *peer_req);
+void drbd_cleanup_peer_requests_wfa(struct drbd_device *device,
+				    struct list_head *cleanup);
+void drbd_remove_peer_req_interval(struct drbd_peer_request *peer_req);
+int drbd_free_peer_reqs(struct drbd_connection *connection,
+			struct list_head *list);
+struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *peer_device,
+					      gfp_t gfp_mask) __must_hold(local);
+void drbd_free_peer_req(struct drbd_peer_request *peer_req);
+int drbd_connected(struct drbd_peer_device *peer_device);
+void conn_connect2(struct drbd_connection *connection);
+void wait_initial_states_received(struct drbd_connection *connection);
+void abort_connect(struct drbd_connection *connection);
+void drbd_print_cluster_wide_state_change(struct drbd_resource *resource,
+					  const char *message,
+					  unsigned int tid,
+					  unsigned int initiator_node_id,
+					  int target_node_id,
+					  union drbd_state mask,
+					  union drbd_state val);
+void apply_unacked_peer_requests(struct drbd_connection *connection);
+struct drbd_connection *drbd_connection_by_node_id(struct drbd_resource *resource,
+						   int node_id);
+struct drbd_connection *drbd_get_connection_by_node_id(struct drbd_resource *resource,
+						       int node_id);
+bool drbd_have_local_disk(struct drbd_resource *resource);
+enum drbd_state_rv drbd_support_2pc_resize(struct drbd_resource *resource);
+enum determine_dev_size
+drbd_commit_size_change(struct drbd_device *device, struct resize_parms *rs,
+			u64 nodes_to_reach);
+void drbd_try_to_get_resynced(struct drbd_device *device);
+void drbd_process_rs_discards(struct drbd_peer_device *peer_device,
+			      bool submit_all);
+void drbd_last_resync_request(struct drbd_peer_device *peer_device,
+			      bool submit_all);
+void drbd_init_connect_state(struct drbd_connection *connection);
+
+static inline sector_t drbd_get_capacity(struct block_device *bdev)
+{
+	return bdev ? bdev_nr_sectors(bdev) : 0;
+}
 
 /* sets the number of 512 byte sectors of our virtual device */
 void drbd_set_my_capacity(struct drbd_device *device, sector_t size);
@@ -1489,206 +2450,109 @@ static inline void drbd_submit_bio_noacct(struct drbd_device *device,
 					     int fault_type, struct bio *bio)
 {
 	__release(local);
-	if (!bio->bi_bdev) {
-		drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n");
+
+	if (drbd_insert_fault(device, fault_type)) {
 		bio->bi_status = BLK_STS_IOERR;
 		bio_endio(bio);
-		return;
-	}
-
-	if (drbd_insert_fault(device, fault_type))
-		bio_io_error(bio);
-	else
+	} else {
 		submit_bio_noacct(bio);
+	}
 }
 
 void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backing_dev *bdev,
 			      enum write_ordering_e wo);
 
+void twopc_timer_fn(struct timer_list *t);
+void connect_timer_fn(struct timer_list *t);
+
 /* drbd_proc.c */
 extern struct proc_dir_entry *drbd_proc;
 int drbd_seq_show(struct seq_file *seq, void *v);
 
 /* drbd_actlog.c */
-extern bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i);
-extern int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i);
-extern void drbd_al_begin_io_commit(struct drbd_device *device);
-extern bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i);
-extern void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i);
-extern void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i);
-extern void drbd_rs_complete_io(struct drbd_device *device, sector_t sector);
-extern int drbd_rs_begin_io(struct drbd_device *device, sector_t sector);
-extern int drbd_try_rs_begin_io(struct drbd_peer_device *peer_device, sector_t sector);
-extern void drbd_rs_cancel_all(struct drbd_device *device);
-extern int drbd_rs_del_all(struct drbd_device *device);
-extern void drbd_rs_failed_io(struct drbd_peer_device *peer_device,
-		sector_t sector, int size);
-extern void drbd_advance_rs_marks(struct drbd_peer_device *peer_device, unsigned long still_to_go);
-
+bool drbd_al_try_lock(struct drbd_device *device);
+bool drbd_al_try_lock_for_transaction(struct drbd_device *device);
+int drbd_al_begin_io_nonblock(struct drbd_device *device,
+			      struct drbd_interval *i);
+void drbd_al_begin_io_commit(struct drbd_device *device);
+bool drbd_al_begin_io_fastpath(struct drbd_device *device,
+			       struct drbd_interval *i);
+bool drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i);
+void drbd_advance_rs_marks(struct drbd_peer_device *peer_device,
+			   unsigned long still_to_go);
+bool drbd_lazy_bitmap_update_due(struct drbd_peer_device *peer_device);
+unsigned long drbd_set_all_out_of_sync(struct drbd_device *device, sector_t sector,
+			     int size);
+unsigned long drbd_set_sync(struct drbd_device *device, sector_t sector, int size,
+		  unsigned long bits, unsigned long mask);
 enum update_sync_bits_mode { RECORD_RS_FAILED, SET_OUT_OF_SYNC, SET_IN_SYNC };
-extern int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector, int size,
-		enum update_sync_bits_mode mode);
+int __drbd_change_sync(struct drbd_peer_device *peer_device, sector_t sector,
+		       int size, enum update_sync_bits_mode mode);
 #define drbd_set_in_sync(peer_device, sector, size) \
 	__drbd_change_sync(peer_device, sector, size, SET_IN_SYNC)
 #define drbd_set_out_of_sync(peer_device, sector, size) \
 	__drbd_change_sync(peer_device, sector, size, SET_OUT_OF_SYNC)
 #define drbd_rs_failed_io(peer_device, sector, size) \
 	__drbd_change_sync(peer_device, sector, size, RECORD_RS_FAILED)
-extern void drbd_al_shrink(struct drbd_device *device);
-extern int drbd_al_initialize(struct drbd_device *, void *);
+void drbd_al_shrink(struct drbd_device *device);
+int drbd_al_initialize(struct drbd_device *device, void *buffer);
 
 /* drbd_nl.c */
-/* state info broadcast */
-struct sib_info {
-	enum drbd_state_info_bcast_reason sib_reason;
-	union {
-		struct {
-			char *helper_name;
-			unsigned helper_exit_code;
-		};
-		struct {
-			union drbd_state os;
-			union drbd_state ns;
-		};
-	};
-};
-void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib);
-
-extern int notify_resource_state(struct sk_buff *,
-				  unsigned int,
-				  struct drbd_resource *,
-				  struct resource_info *,
-				  enum drbd_notification_type);
-extern int notify_device_state(struct sk_buff *,
-				unsigned int,
-				struct drbd_device *,
-				struct device_info *,
-				enum drbd_notification_type);
-extern int notify_connection_state(struct sk_buff *,
-				    unsigned int,
-				    struct drbd_connection *,
-				    struct connection_info *,
-				    enum drbd_notification_type);
-extern int notify_peer_device_state(struct sk_buff *,
-				     unsigned int,
-				     struct drbd_peer_device *,
-				     struct peer_device_info *,
-				     enum drbd_notification_type);
-extern void notify_helper(enum drbd_notification_type, struct drbd_device *,
-			  struct drbd_connection *, const char *, int);
 
+extern struct mutex notification_mutex;
+extern atomic_t drbd_genl_seq;
+
+int notify_resource_state(struct sk_buff *skb, unsigned int seq,
+			  struct drbd_resource *resource,
+			  struct resource_info *resource_info,
+			  struct rename_resource_info *rename_resource_info,
+			  enum drbd_notification_type type);
+int notify_device_state(struct sk_buff *skb, unsigned int seq,
+			struct drbd_device *device,
+			struct device_info *device_info,
+			enum drbd_notification_type type);
+int notify_connection_state(struct sk_buff *skb, unsigned int seq,
+			    struct drbd_connection *connection,
+			    struct connection_info *connection_info,
+			    enum drbd_notification_type type);
+int notify_peer_device_state(struct sk_buff *skb, unsigned int seq,
+			     struct drbd_peer_device *peer_device,
+			     struct peer_device_info *peer_device_info,
+			     enum drbd_notification_type type);
+void notify_helper(enum drbd_notification_type type,
+		   struct drbd_device *device,
+		   struct drbd_connection *connection, const char *name,
+		   int status);
+int notify_path(struct drbd_connection *connection, struct drbd_path *path,
+		enum drbd_notification_type type);
+void drbd_broadcast_peer_device_state(struct drbd_peer_device *peer_device);
+
+sector_t drbd_local_max_size(struct drbd_device *device) __must_hold(local);
+int drbd_open_ro_count(struct drbd_resource *resource);
+
+void device_to_info(struct device_info *info, struct drbd_device *device);
+void device_state_change_to_info(struct device_info *info,
+				 struct drbd_device_state_change *state_change);
+void peer_device_state_change_to_info(struct peer_device_info *info,
+				      struct drbd_peer_device_state_change *state_change);
 /*
  * inline helper functions
  *************************/
 
-/* see also page_chain_add and friends in drbd_receiver.c */
-static inline struct page *page_chain_next(struct page *page)
-{
-	return (struct page *)page_private(page);
-}
-#define page_chain_for_each(page) \
-	for (; page && ({ prefetch(page_chain_next(page)); 1; }); \
-			page = page_chain_next(page))
-#define page_chain_for_each_safe(page, n) \
-	for (; page && ({ n = page_chain_next(page); 1; }); page = n)
-
-
-static inline union drbd_state drbd_read_state(struct drbd_device *device)
-{
-	struct drbd_resource *resource = device->resource;
-	union drbd_state rv;
-
-	rv.i = device->state.i;
-	rv.susp = resource->susp;
-	rv.susp_nod = resource->susp_nod;
-	rv.susp_fen = resource->susp_fen;
-
-	return rv;
-}
-
-enum drbd_force_detach_flags {
-	DRBD_READ_ERROR,
-	DRBD_WRITE_ERROR,
-	DRBD_META_IO_ERROR,
-	DRBD_FORCE_DETACH,
-};
-
-#define __drbd_chk_io_error(m,f) __drbd_chk_io_error_(m,f, __func__)
-static inline void __drbd_chk_io_error_(struct drbd_device *device,
-		enum drbd_force_detach_flags df,
-		const char *where)
-{
-	enum drbd_io_error_p ep;
-
-	rcu_read_lock();
-	ep = rcu_dereference(device->ldev->disk_conf)->on_io_error;
-	rcu_read_unlock();
-	switch (ep) {
-	case EP_PASS_ON: /* FIXME would this be better named "Ignore"? */
-		if (df == DRBD_READ_ERROR || df == DRBD_WRITE_ERROR) {
-			if (drbd_ratelimit())
-				drbd_err(device, "Local IO failed in %s.\n", where);
-			if (device->state.disk > D_INCONSISTENT)
-				_drbd_set_state(_NS(device, disk, D_INCONSISTENT), CS_HARD, NULL);
-			break;
-		}
-		fallthrough;	/* for DRBD_META_IO_ERROR or DRBD_FORCE_DETACH */
-	case EP_DETACH:
-	case EP_CALL_HELPER:
-		/* Remember whether we saw a READ or WRITE error.
-		 *
-		 * Recovery of the affected area for WRITE failure is covered
-		 * by the activity log.
-		 * READ errors may fall outside that area though. Certain READ
-		 * errors can be "healed" by writing good data to the affected
-		 * blocks, which triggers block re-allocation in lower layers.
-		 *
-		 * If we can not write the bitmap after a READ error,
-		 * we may need to trigger a full sync (see w_go_diskless()).
-		 *
-		 * Force-detach is not really an IO error, but rather a
-		 * desperate measure to try to deal with a completely
-		 * unresponsive lower level IO stack.
-		 * Still it should be treated as a WRITE error.
-		 *
-		 * Meta IO error is always WRITE error:
-		 * we read meta data only once during attach,
-		 * which will fail in case of errors.
-		 */
-		set_bit(WAS_IO_ERROR, &device->flags);
-		if (df == DRBD_READ_ERROR)
-			set_bit(WAS_READ_ERROR, &device->flags);
-		if (df == DRBD_FORCE_DETACH)
-			set_bit(FORCE_DETACH, &device->flags);
-		if (device->state.disk > D_FAILED) {
-			_drbd_set_state(_NS(device, disk, D_FAILED), CS_HARD, NULL);
-			drbd_err(device,
-				"Local IO failed in %s. Detaching...\n", where);
-		}
-		break;
-	}
-}
-
-/**
- * drbd_chk_io_error: Handle the on_io_error setting, should be called from all io completion handlers
- * @device:	 DRBD device.
- * @error:	 Error code passed to the IO completion callback
- * @forcedetach: Force detach. I.e. the error happened while accessing the meta data
- *
- * See also drbd_main.c:after_state_ch() if (os.disk > D_FAILED && ns.disk == D_FAILED)
+/*
+ * When a device has a replication state above L_OFF, it must be
+ * connected.  Otherwise, we report the connection state, which has values up
+ * to C_CONNECTED == L_OFF.
  */
-#define drbd_chk_io_error(m,e,f) drbd_chk_io_error_(m,e,f, __func__)
-static inline void drbd_chk_io_error_(struct drbd_device *device,
-	int error, enum drbd_force_detach_flags forcedetach, const char *where)
+static inline int combined_conn_state(struct drbd_peer_device *peer_device, enum which_state which)
 {
-	if (error) {
-		unsigned long flags;
-		spin_lock_irqsave(&device->resource->req_lock, flags);
-		__drbd_chk_io_error_(device, forcedetach, where);
-		spin_unlock_irqrestore(&device->resource->req_lock, flags);
-	}
-}
+	enum drbd_repl_state repl_state = peer_device->repl_state[which];
 
+	if (repl_state > L_OFF)
+		return repl_state;
+	else
+		return peer_device->connection->cstate[which];
+}
 
 /**
  * drbd_md_first_sector() - Returns the first sector number of the meta data area
@@ -1718,52 +2582,11 @@ static inline sector_t drbd_md_last_sector(struct drbd_backing_dev *bdev)
 	switch (bdev->md.meta_dev_idx) {
 	case DRBD_MD_INDEX_INTERNAL:
 	case DRBD_MD_INDEX_FLEX_INT:
-		return bdev->md.md_offset + MD_4kB_SECT -1;
-	case DRBD_MD_INDEX_FLEX_EXT:
-	default:
-		return bdev->md.md_offset + bdev->md.md_size_sect -1;
-	}
-}
-
-/* Returns the number of 512 byte sectors of the device */
-static inline sector_t drbd_get_capacity(struct block_device *bdev)
-{
-	return bdev ? bdev_nr_sectors(bdev) : 0;
-}
-
-/**
- * drbd_get_max_capacity() - Returns the capacity we announce to out peer
- * @bdev:	Meta data block device.
- *
- * returns the capacity we announce to out peer.  we clip ourselves at the
- * various MAX_SECTORS, because if we don't, current implementation will
- * oops sooner or later
- */
-static inline sector_t drbd_get_max_capacity(struct drbd_backing_dev *bdev)
-{
-	sector_t s;
-
-	switch (bdev->md.meta_dev_idx) {
-	case DRBD_MD_INDEX_INTERNAL:
-	case DRBD_MD_INDEX_FLEX_INT:
-		s = drbd_get_capacity(bdev->backing_bdev)
-			? min_t(sector_t, DRBD_MAX_SECTORS_FLEX,
-				drbd_md_first_sector(bdev))
-			: 0;
-		break;
+		return bdev->md.md_offset + (4096 >> 9) - 1;
 	case DRBD_MD_INDEX_FLEX_EXT:
-		s = min_t(sector_t, DRBD_MAX_SECTORS_FLEX,
-				drbd_get_capacity(bdev->backing_bdev));
-		/* clip at maximum size the meta device can support */
-		s = min_t(sector_t, s,
-			BM_EXT_TO_SECT(bdev->md.md_size_sect
-				     - bdev->md.bm_offset));
-		break;
 	default:
-		s = min_t(sector_t, DRBD_MAX_SECTORS,
-				drbd_get_capacity(bdev->backing_bdev));
+		return bdev->md.md_offset + bdev->md.md_size_sect - 1;
 	}
-	return s;
 }
 
 /**
@@ -1784,18 +2607,10 @@ static inline sector_t drbd_md_ss(struct drbd_backing_dev *bdev)
 		return (drbd_get_capacity(bdev->backing_bdev) & ~7ULL) - 8;
 
 	/* external, some index; this is the old fixed size layout */
-	return MD_128MB_SECT * bdev->md.meta_dev_idx;
+	return (128 << 20 >> 9) * bdev->md.meta_dev_idx;
 }
 
-static inline void
-drbd_queue_work(struct drbd_work_queue *q, struct drbd_work *w)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&q->q_lock, flags);
-	list_add_tail(&w->list, &q->q);
-	spin_unlock_irqrestore(&q->q_lock, flags);
-	wake_up(&q->q_wait);
-}
+void drbd_queue_work(struct drbd_work_queue *, struct drbd_work *);
 
 static inline void
 drbd_queue_work_if_unqueued(struct drbd_work_queue *q, struct drbd_work *w)
@@ -1812,46 +2627,48 @@ static inline void
 drbd_device_post_work(struct drbd_device *device, int work_bit)
 {
 	if (!test_and_set_bit(work_bit, &device->flags)) {
-		struct drbd_connection *connection =
-			first_peer_device(device)->connection;
-		struct drbd_work_queue *q = &connection->sender_work;
-		if (!test_and_set_bit(DEVICE_WORK_PENDING, &connection->flags))
+		struct drbd_resource *resource = device->resource;
+		struct drbd_work_queue *q = &resource->work;
+		if (!test_and_set_bit(DEVICE_WORK_PENDING, &resource->flags))
 			wake_up(&q->q_wait);
 	}
 }
 
-extern void drbd_flush_workqueue(struct drbd_work_queue *work_queue);
-
-/* To get the ack_receiver out of the blocking network stack,
- * so it can change its sk_rcvtimeo from idle- to ping-timeout,
- * and send a ping, we need to send a signal.
- * Which signal we send is irrelevant. */
-static inline void wake_ack_receiver(struct drbd_connection *connection)
-{
-	struct task_struct *task = connection->ack_receiver.task;
-	if (task && get_t_state(&connection->ack_receiver) == RUNNING)
-		send_sig(SIGXCPU, task, 1);
-}
-
-static inline void request_ping(struct drbd_connection *connection)
+static inline void
+drbd_peer_device_post_work(struct drbd_peer_device *peer_device, int work_bit)
 {
-	set_bit(SEND_PING, &connection->flags);
-	wake_ack_receiver(connection);
+	if (!test_and_set_bit(work_bit, &peer_device->flags)) {
+		struct drbd_resource *resource = peer_device->device->resource;
+		struct drbd_work_queue *q = &resource->work;
+		if (!test_and_set_bit(PEER_DEVICE_WORK_PENDING, &resource->flags))
+			wake_up(&q->q_wait);
+	}
 }
 
-extern void *conn_prepare_command(struct drbd_connection *, struct drbd_socket *);
-extern void *drbd_prepare_command(struct drbd_peer_device *, struct drbd_socket *);
-extern int conn_send_command(struct drbd_connection *, struct drbd_socket *,
-			     enum drbd_packet, unsigned int, void *,
-			     unsigned int);
-extern int drbd_send_command(struct drbd_peer_device *, struct drbd_socket *,
-			     enum drbd_packet, unsigned int, void *,
-			     unsigned int);
-
-extern int drbd_send_ping(struct drbd_connection *connection);
-extern int drbd_send_ping_ack(struct drbd_connection *connection);
-extern int drbd_send_state_req(struct drbd_peer_device *, union drbd_state, union drbd_state);
-extern int conn_send_state_req(struct drbd_connection *, union drbd_state, union drbd_state);
+void drbd_flush_workqueue(struct drbd_work_queue *work_queue);
+void drbd_flush_workqueue_interruptible(struct drbd_device *device);
+
+void *__conn_prepare_command(struct drbd_connection *connection, int size,
+			     enum drbd_stream drbd_stream);
+void *conn_prepare_command(struct drbd_connection *connection, int size,
+			   enum drbd_stream drbd_stream);
+void *drbd_prepare_command(struct drbd_peer_device *peer_device, int size,
+			   enum drbd_stream drbd_stream);
+int __send_command(struct drbd_connection *connection, int vnr,
+		   enum drbd_packet cmd, int stream_and_flags);
+int send_command(struct drbd_connection *connection, int vnr,
+		 enum drbd_packet cmd, int stream_and_flags);
+int drbd_send_command(struct drbd_peer_device *peer_device,
+		      enum drbd_packet cmd, enum drbd_stream drbd_stream);
+
+int drbd_send_ping(struct drbd_connection *connection);
+int conn_send_state_req(struct drbd_connection *connection, int vnr,
+			enum drbd_packet cmd, union drbd_state mask,
+			union drbd_state val);
+int conn_send_twopc_request(struct drbd_connection *connection,
+			    struct twopc_request *request);
+int drbd_send_peer_ack(struct drbd_connection *connection, u64 mask,
+		       u64 dagtag_sector);
 
 static inline void drbd_thread_stop(struct drbd_thread *thi)
 {
@@ -1868,59 +2685,37 @@ static inline void drbd_thread_restart_nowait(struct drbd_thread *thi)
 	_drbd_thread_stop(thi, true, false);
 }
 
-/* counts how many answer packets packets we expect from our peer,
- * for either explicit application requests,
- * or implicit barrier packets as necessary.
- * increased:
- *  w_send_barrier
- *  _req_mod(req, QUEUE_FOR_NET_WRITE or QUEUE_FOR_NET_READ);
- *    it is much easier and equally valid to count what we queue for the
- *    worker, even before it actually was queued or send.
- *    (drbd_make_request_common; recovery path on read io-error)
- * decreased:
- *  got_BarrierAck (respective tl_clear, tl_clear_barrier)
- *  _req_mod(req, DATA_RECEIVED)
- *     [from receive_DataReply]
- *  _req_mod(req, WRITE_ACKED_BY_PEER or RECV_ACKED_BY_PEER or NEG_ACKED)
- *     [from got_BlockAck (P_WRITE_ACK, P_RECV_ACK)]
- *     for some reason it is NOT decreased in got_NegAck,
- *     but in the resulting cleanup code from report_params.
- *     we should try to remember the reason for that...
- *  _req_mod(req, SEND_FAILED or SEND_CANCELED)
- *  _req_mod(req, CONNECTION_LOST_WHILE_PENDING)
- *     [from tl_clear_barrier]
- */
-static inline void inc_ap_pending(struct drbd_device *device)
+static inline void inc_ap_pending(struct drbd_peer_device *peer_device)
 {
-	atomic_inc(&device->ap_pending_cnt);
+	atomic_inc(&peer_device->ap_pending_cnt);
 }
 
-#define dec_ap_pending(device) ((void)expect((device), __dec_ap_pending(device) >= 0))
-static inline int __dec_ap_pending(struct drbd_device *device)
+#define dec_ap_pending(peer_device) \
+	((void)expect((peer_device), __dec_ap_pending(peer_device) >= 0))
+static inline int __dec_ap_pending(struct drbd_peer_device *peer_device)
 {
-	int ap_pending_cnt = atomic_dec_return(&device->ap_pending_cnt);
-
+	int ap_pending_cnt = atomic_dec_return(&peer_device->ap_pending_cnt);
 	if (ap_pending_cnt == 0)
-		wake_up(&device->misc_wait);
+		wake_up(&peer_device->device->misc_wait);
 	return ap_pending_cnt;
 }
 
 /* counts how many resync-related answers we still expect from the peer
  *		     increase			decrease
- * C_SYNC_TARGET sends P_RS_DATA_REQUEST (and expects P_RS_DATA_REPLY)
- * C_SYNC_SOURCE sends P_RS_DATA_REPLY   (and expects P_WRITE_ACK with ID_SYNCER)
+ * L_SYNC_TARGET sends P_RS_DATA_REQUEST (and expects P_RS_DATA_REPLY)
+ * L_SYNC_SOURCE sends P_RS_DATA_REPLY   (and expects P_WRITE_ACK with ID_SYNCER)
  *					   (or P_NEG_ACK with ID_SYNCER)
  */
 static inline void inc_rs_pending(struct drbd_peer_device *peer_device)
 {
-	atomic_inc(&peer_device->device->rs_pending_cnt);
+	atomic_inc(&peer_device->rs_pending_cnt);
 }
 
 #define dec_rs_pending(peer_device) \
 	((void)expect((peer_device), __dec_rs_pending(peer_device) >= 0))
 static inline int __dec_rs_pending(struct drbd_peer_device *peer_device)
 {
-	return atomic_dec_return(&peer_device->device->rs_pending_cnt);
+	return atomic_dec_return(&peer_device->rs_pending_cnt);
 }
 
 /* counts how many answers we still need to send to the peer.
@@ -1929,42 +2724,68 @@ static inline int __dec_rs_pending(struct drbd_peer_device *peer_device)
  *			we need to send a P_RECV_ACK (proto B)
  *			or P_WRITE_ACK (proto C)
  *  receive_RSDataReply (recv_resync_read) we need to send a P_WRITE_ACK
- *  receive_DataRequest (receive_RSDataRequest) we need to send back P_DATA
+ *  receive_data_request etc we need to send back P_DATA
  *  receive_Barrier_*	we need to send a P_BARRIER_ACK
  */
-static inline void inc_unacked(struct drbd_device *device)
+static inline void inc_unacked(struct drbd_peer_device *peer_device)
+{
+	atomic_inc(&peer_device->unacked_cnt);
+}
+
+#define dec_unacked(peer_device) \
+	((void)expect(peer_device, __dec_unacked(peer_device) >= 0))
+static inline int __dec_unacked(struct drbd_peer_device *peer_device)
 {
-	atomic_inc(&device->unacked_cnt);
+	return atomic_dec_return(&peer_device->unacked_cnt);
 }
 
-#define dec_unacked(device) ((void)expect(device, __dec_unacked(device) >= 0))
-static inline int __dec_unacked(struct drbd_device *device)
+static inline bool repl_is_sync_target(enum drbd_repl_state repl_state)
 {
-	return atomic_dec_return(&device->unacked_cnt);
+	return repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T;
 }
 
-#define sub_unacked(device, n) ((void)expect(device, __sub_unacked(device) >= 0))
-static inline int __sub_unacked(struct drbd_device *device, int n)
+static inline bool repl_is_sync_source(enum drbd_repl_state repl_state)
 {
-	return atomic_sub_return(n, &device->unacked_cnt);
+	return repl_state == L_SYNC_SOURCE || repl_state == L_PAUSED_SYNC_S;
 }
 
-static inline bool is_sync_target_state(enum drbd_conns connection_state)
+static inline bool repl_is_sync(enum drbd_repl_state repl_state)
 {
-	return	connection_state == C_SYNC_TARGET ||
-		connection_state == C_PAUSED_SYNC_T;
+	return repl_is_sync_source(repl_state) ||
+		repl_is_sync_target(repl_state);
 }
 
-static inline bool is_sync_source_state(enum drbd_conns connection_state)
+static inline bool is_sync_target_state(struct drbd_peer_device *peer_device,
+					enum which_state which)
 {
-	return	connection_state == C_SYNC_SOURCE ||
-		connection_state == C_PAUSED_SYNC_S;
+	return repl_is_sync_target(peer_device->repl_state[which]);
 }
 
-static inline bool is_sync_state(enum drbd_conns connection_state)
+static inline bool is_sync_source_state(struct drbd_peer_device *peer_device,
+					enum which_state which)
 {
-	return	is_sync_source_state(connection_state) ||
-		is_sync_target_state(connection_state);
+	return repl_is_sync_source(peer_device->repl_state[which]);
+}
+
+static inline bool is_sync_state(struct drbd_peer_device *peer_device,
+				 enum which_state which)
+{
+	return repl_is_sync(peer_device->repl_state[which]);
+}
+
+static inline bool is_verify_state(struct drbd_peer_device *peer_device,
+				   enum which_state which)
+{
+	enum drbd_repl_state repl_state = peer_device->repl_state[which];
+	return repl_state == L_VERIFY_S || repl_state == L_VERIFY_T;
+}
+
+static inline bool resync_susp_comb_dep(struct drbd_peer_device *peer_device, enum which_state which)
+{
+	struct drbd_device *device = peer_device->device;
+
+	return peer_device->resync_susp_dependency[which] || peer_device->resync_susp_other_c[which] ||
+		(is_sync_source_state(peer_device, which) && device->disk_state[which] <= D_INCONSISTENT);
 }
 
 /**
@@ -1981,7 +2802,7 @@ static inline bool is_sync_state(enum drbd_conns connection_state)
 
 static inline void put_ldev(struct drbd_device *device)
 {
-	enum drbd_disk_state disk_state = device->state.disk;
+	enum drbd_disk_state disk_state = device->disk_state[NOW];
 	/* We must check the state *before* the atomic_dec becomes visible,
 	 * or we have a theoretical race where someone hitting zero,
 	 * while state still D_FAILED, will then see D_DISKLESS in the
@@ -1994,10 +2815,12 @@ static inline void put_ldev(struct drbd_device *device)
 	__release(local);
 	D_ASSERT(device, i >= 0);
 	if (i == 0) {
-		if (disk_state == D_DISKLESS)
+		if (disk_state == D_DISKLESS) {
 			/* even internal references gone, safe to destroy */
-			drbd_device_post_work(device, DESTROY_DISK);
-		if (disk_state == D_FAILED)
+			kref_get(&device->kref);
+			schedule_work(&device->ldev_destroy_work);
+		}
+		if (disk_state == D_FAILED || disk_state == D_DETACHING)
 			/* all application IO references gone. */
 			if (!test_and_set_bit(GOING_DISKLESS, &device->flags))
 				drbd_device_post_work(device, GO_DISKLESS);
@@ -2011,116 +2834,51 @@ static inline int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_
 	int io_allowed;
 
 	/* never get a reference while D_DISKLESS */
-	if (device->state.disk == D_DISKLESS)
+	if (device->disk_state[NOW] == D_DISKLESS)
 		return 0;
 
 	atomic_inc(&device->local_cnt);
-	io_allowed = (device->state.disk >= mins);
+	io_allowed = (device->disk_state[NOW] >= mins);
 	if (!io_allowed)
 		put_ldev(device);
 	return io_allowed;
 }
 #else
-extern int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins);
+int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins);
 #endif
 
-/* this throttles on-the-fly application requests
- * according to max_buffers settings;
- * maybe re-implement using semaphores? */
-static inline int drbd_get_max_buffers(struct drbd_device *device)
-{
-	struct net_conf *nc;
-	int mxb;
-
-	rcu_read_lock();
-	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-	mxb = nc ? nc->max_buffers : 1000000;  /* arbitrary limit on open requests */
-	rcu_read_unlock();
-
-	return mxb;
-}
+void drbd_queue_pending_bitmap_work(struct drbd_device *device);
 
-static inline int drbd_state_is_stable(struct drbd_device *device)
+/* rw = READ or WRITE (0 or 1); nothing else. */
+static inline void dec_ap_bio(struct drbd_device *device, int rw)
 {
-	union drbd_dev_state s = device->state;
-
-	/* DO NOT add a default clause, we want the compiler to warn us
-	 * for any newly introduced state we may have forgotten to add here */
-
-	switch ((enum drbd_conns)s.conn) {
-	/* new io only accepted when there is no connection, ... */
-	case C_STANDALONE:
-	case C_WF_CONNECTION:
-	/* ... or there is a well established connection. */
-	case C_CONNECTED:
-	case C_SYNC_SOURCE:
-	case C_SYNC_TARGET:
-	case C_VERIFY_S:
-	case C_VERIFY_T:
-	case C_PAUSED_SYNC_S:
-	case C_PAUSED_SYNC_T:
-	case C_AHEAD:
-	case C_BEHIND:
-		/* transitional states, IO allowed */
-	case C_DISCONNECTING:
-	case C_UNCONNECTED:
-	case C_TIMEOUT:
-	case C_BROKEN_PIPE:
-	case C_NETWORK_FAILURE:
-	case C_PROTOCOL_ERROR:
-	case C_TEAR_DOWN:
-	case C_WF_REPORT_PARAMS:
-	case C_STARTING_SYNC_S:
-	case C_STARTING_SYNC_T:
-		break;
-
-		/* Allow IO in BM exchange states with new protocols */
-	case C_WF_BITMAP_S:
-		if (first_peer_device(device)->connection->agreed_pro_version < 96)
-			return 0;
-		break;
-
-		/* no new io accepted in these states */
-	case C_WF_BITMAP_T:
-	case C_WF_SYNC_UUID:
-	case C_MASK:
-		/* not "stable" */
-		return 0;
-	}
+	unsigned int nr_requests = device->resource->res_opts.nr_requests;
+	int ap_bio = atomic_dec_return(&device->ap_bio_cnt[rw]);
 
-	switch ((enum drbd_disk_state)s.disk) {
-	case D_DISKLESS:
-	case D_INCONSISTENT:
-	case D_OUTDATED:
-	case D_CONSISTENT:
-	case D_UP_TO_DATE:
-	case D_FAILED:
-		/* disk state is stable as well. */
-		break;
+	D_ASSERT(device, ap_bio >= 0);
 
-	/* no new io accepted during transitional states */
-	case D_ATTACHING:
-	case D_NEGOTIATING:
-	case D_UNKNOWN:
-	case D_MASK:
-		/* not "stable" */
-		return 0;
-	}
+	/* Check for list_empty outside the lock is ok.  Worst case it queues
+	 * nothing because someone else just now did.  During list_add, a
+	 * refcount on ap_bio_cnt[WRITE] is held, so the bitmap work will be
+	 * queued when that is released if we miss it here.
+	 * Checking pending_bitmap_work.n is not correct,
+	 * it has a different lifetime. */
+	if (ap_bio == 0 && rw == WRITE && !list_empty(&device->pending_bitmap_work.q))
+		drbd_queue_pending_bitmap_work(device);
 
-	return 1;
+	if (ap_bio == 0 || ap_bio == nr_requests-1)
+		wake_up(&device->misc_wait);
 }
 
-static inline int drbd_suspended(struct drbd_device *device)
+static inline bool drbd_suspended(struct drbd_device *device)
 {
-	struct drbd_resource *resource = device->resource;
-
-	return resource->susp || resource->susp_fen || resource->susp_nod;
+	return device->resource->cached_susp;
 }
 
 static inline bool may_inc_ap_bio(struct drbd_device *device)
 {
-	int mxb = drbd_get_max_buffers(device);
-
+	if (device->cached_err_io)
+		return true;
 	if (drbd_suspended(device))
 		return false;
 	if (atomic_read(&device->suspend_cnt))
@@ -2131,76 +2889,45 @@ static inline bool may_inc_ap_bio(struct drbd_device *device)
 	 * to start during "stable" states. */
 
 	/* no new io accepted when attaching or detaching the disk */
-	if (!drbd_state_is_stable(device))
+	if (device->cached_state_unstable)
 		return false;
 
-	/* since some older kernels don't have atomic_add_unless,
-	 * and we are within the spinlock anyways, we have this workaround.  */
-	if (atomic_read(&device->ap_bio_cnt) > mxb)
-		return false;
-	if (test_bit(BITMAP_IO, &device->flags))
+	if (atomic_read(&device->pending_bitmap_work.n))
 		return false;
 	return true;
 }
 
-static inline bool inc_ap_bio_cond(struct drbd_device *device)
+static inline u64 drbd_current_uuid(struct drbd_device *device)
 {
-	bool rv = false;
-
-	spin_lock_irq(&device->resource->req_lock);
-	rv = may_inc_ap_bio(device);
-	if (rv)
-		atomic_inc(&device->ap_bio_cnt);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	return rv;
+	if (!device->ldev)
+		return 0;
+	return device->ldev->md.current_uuid;
 }
 
-static inline void inc_ap_bio(struct drbd_device *device)
+static inline bool verify_can_do_stop_sector(struct drbd_peer_device *peer_device)
 {
-	/* we wait here
-	 *    as long as the device is suspended
-	 *    until the bitmap is no longer on the fly during connection
-	 *    handshake as long as we would exceed the max_buffer limit.
-	 *
-	 * to avoid races with the reconnect code,
-	 * we need to atomic_inc within the spinlock. */
-
-	wait_event(device->misc_wait, inc_ap_bio_cond(device));
+	return peer_device->connection->agreed_pro_version >= 97 &&
+		peer_device->connection->agreed_pro_version != 100;
 }
 
-static inline void dec_ap_bio(struct drbd_device *device)
+static inline u64 drbd_bitmap_uuid(struct drbd_peer_device *peer_device)
 {
-	int mxb = drbd_get_max_buffers(device);
-	int ap_bio = atomic_dec_return(&device->ap_bio_cnt);
-
-	D_ASSERT(device, ap_bio >= 0);
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_md *peer_md;
 
-	if (ap_bio == 0 && test_bit(BITMAP_IO, &device->flags)) {
-		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
-			drbd_queue_work(&first_peer_device(device)->
-				connection->sender_work,
-				&device->bm_io_work.w);
-	}
+	if (!device->ldev)
+		return 0;
 
-	/* this currently does wake_up for every dec_ap_bio!
-	 * maybe rather introduce some type of hysteresis?
-	 * e.g. (ap_bio == mxb/2 || ap_bio == 0) ? */
-	if (ap_bio < mxb)
-		wake_up(&device->misc_wait);
+	peer_md = &device->ldev->md.peers[peer_device->node_id];
+	return peer_md->bitmap_uuid;
 }
 
-static inline bool verify_can_do_stop_sector(struct drbd_device *device)
+static inline u64 drbd_history_uuid(struct drbd_device *device, int i)
 {
-	return first_peer_device(device)->connection->agreed_pro_version >= 97 &&
-		first_peer_device(device)->connection->agreed_pro_version != 100;
-}
+	if (!device->ldev || i >= ARRAY_SIZE(device->ldev->md.history_uuids))
+		return 0;
 
-static inline int drbd_set_ed_uuid(struct drbd_device *device, u64 val)
-{
-	int changed = device->ed_uuid != val;
-	device->ed_uuid = val;
-	return changed;
+	return device->ldev->md.history_uuids[i];
 }
 
 static inline int drbd_queue_order_type(struct drbd_device *device)
@@ -2219,4 +2946,214 @@ static inline struct drbd_connection *first_connection(struct drbd_resource *res
 				struct drbd_connection, connections);
 }
 
+static inline struct net *drbd_net_assigned_to_connection(struct drbd_connection *connection)
+{
+	struct drbd_path *path;
+	struct net *net;
+
+	rcu_read_lock();
+	path = list_first_or_null_rcu(&connection->transport.paths, struct drbd_path, list);
+	net = path ? path->net : NULL;
+	rcu_read_unlock();
+
+	return net;
+}
+
+#define NODE_MASK(id) ((u64)1 << (id))
+
+static inline void drbd_list_del_resync_request(struct drbd_peer_request *peer_req)
+{
+	peer_req->flags &= ~EE_ON_RECV_ORDER;
+	list_del(&peer_req->recv_order);
+
+	if (peer_req == peer_req->peer_device->received_last)
+		peer_req->peer_device->received_last = NULL;
+
+	if (peer_req == peer_req->peer_device->discard_last)
+		peer_req->peer_device->discard_last = NULL;
+}
+
+/*
+ * drbd_interval_same_peer - determine whether "interval" is for the same peer as "i"
+ *
+ * "i" must be an interval corresponding to a drbd_peer_request.
+ */
+static inline bool drbd_interval_same_peer(struct drbd_interval *interval, struct drbd_interval *i)
+{
+	struct drbd_peer_request *interval_peer_req, *i_peer_req;
+
+	/* Ensure we only call "container_of" if it is actually a peer request. */
+	if (interval->type == INTERVAL_LOCAL_WRITE ||
+			interval->type == INTERVAL_LOCAL_READ ||
+			interval->type == INTERVAL_PEERS_IN_SYNC_LOCK)
+		return false;
+
+	interval_peer_req = container_of(interval, struct drbd_peer_request, i);
+	i_peer_req = container_of(i, struct drbd_peer_request, i);
+	return interval_peer_req->peer_device == i_peer_req->peer_device;
+}
+
+/*
+ * drbd_should_defer_to_resync - determine whether "interval" should defer to
+ * "i" in order to ensure that resync makes progress
+ */
+static inline bool drbd_should_defer_to_resync(struct drbd_interval *interval, struct drbd_interval *i)
+{
+	if (!drbd_interval_is_resync(i))
+		return false;
+
+	/* Always defer to resync requests once the reply has been received.
+	 * These just need to wait for conflicting local I/O to complete. This
+	 * is necessary to ensure that resync replies received before
+	 * application writes are submitted first, so that the resync writes do
+	 * not overwrite newer data. */
+	if (test_bit(INTERVAL_RECEIVED, &i->flags))
+		return true;
+
+	/* If we are still waiting for a reply from the peer, only defer to the
+	 * request if it is towards a different peer. The exclusivity between
+	 * resync requests and application writes from another peer is
+	 * necessary to avoid overwriting newer data with older in the resync.
+	 * When the data in both cases is coming from the same peer, this is
+	 * not necessary. The peer ensures that the data stream is correctly
+	 * ordered. */
+	return !drbd_interval_same_peer(interval, i);
+}
+
+/*
+ * drbd_should_defer_to_interval - determine whether "interval" should defer to "i"
+ */
+static inline bool drbd_should_defer_to_interval(struct drbd_interval *interval,
+		struct drbd_interval *i, bool defer_to_resync)
+{
+	if (test_bit(INTERVAL_SUBMITTED, &i->flags))
+		return true;
+
+	if (defer_to_resync && drbd_should_defer_to_resync(interval, i))
+		return true;
+
+	/*
+	 * We do not send conflicting resync requests because that causes
+	 * difficulties associating the replies to the requests.
+	 */
+	if (interval->type == INTERVAL_RESYNC_WRITE &&
+			i->type == INTERVAL_RESYNC_WRITE && test_bit(INTERVAL_SENT, &i->flags))
+		return true;
+
+	return false;
+}
+
+/* Find conflicts at application level instead of at disk level. */
+#define CONFLICT_FLAG_APPLICATION_ONLY (1 << 0)
+
+/*
+ * Ignore peer writes from the peer that this request relates to. This is only
+ * used for determining whether to send a request. It must not be used for
+ * determining whether to submit a request, because that would allow concurrent
+ * writes to the backing disk.
+ */
+#define CONFLICT_FLAG_IGNORE_SAME_PEER (1 << 1)
+
+/*
+ * drbd_find_conflict - find conflicting interval, if any
+ */
+static inline struct drbd_interval *drbd_find_conflict(struct drbd_device *device,
+		struct drbd_interval *interval, unsigned long flags)
+{
+	struct drbd_interval *i;
+	sector_t sector = interval->sector;
+	int size = interval->size;
+	bool application_only = flags & CONFLICT_FLAG_APPLICATION_ONLY;
+	bool defer_to_resync =
+		(interval->type == INTERVAL_LOCAL_WRITE || interval->type == INTERVAL_PEER_WRITE) &&
+		!application_only;
+	bool exclusive_until_completed = interval->type == INTERVAL_LOCAL_WRITE || application_only;
+	bool ignore_same_peer = flags & CONFLICT_FLAG_IGNORE_SAME_PEER;
+
+	lockdep_assert_held(&device->interval_lock);
+
+	drbd_for_each_overlap(i, &device->requests, sector, size) {
+		/* Ignore the interval itself. */
+		if (i == interval)
+			continue;
+
+		if (exclusive_until_completed) {
+			/* Ignore, if already completed to upper layers. */
+			if (test_bit(INTERVAL_COMPLETED, &i->flags))
+				continue;
+		} else {
+			/* Ignore, if already completed by the backing disk. */
+			if (test_bit(INTERVAL_BACKING_COMPLETED, &i->flags))
+				continue;
+		}
+
+		/* Ignore, if there is no need to defer to it. */
+		if (!drbd_should_defer_to_interval(interval, i, defer_to_resync))
+			continue;
+
+		/*
+		 * Ignore peer writes from the peer that this request relates
+		 * to, if requested.
+		 */
+		if (ignore_same_peer && i->type == INTERVAL_PEER_WRITE && drbd_interval_same_peer(interval, i))
+			continue;
+
+		if (unlikely(application_only)) {
+			/* Ignore, if not an application request. */
+			if (!drbd_interval_is_application(i))
+				continue;
+		}
+
+		if (drbd_interval_is_write(interval)) {
+			/*
+			 * Mark verify requests as conflicting rather than
+			 * treating them as conflicts for us.
+			 */
+			if (drbd_interval_is_verify(i)) {
+				set_bit(INTERVAL_CONFLICT, &i->flags);
+				continue;
+			}
+		} else {
+			/* Ignore other resync reads. */
+			if (i->type == INTERVAL_RESYNC_READ)
+				continue;
+
+			/* Ignore verify requests, since they are always reads. */
+			if (drbd_interval_is_verify(i))
+				continue;
+
+			/* Ignore peers-in-sync intervals, since they are always reads. */
+			if (i->type == INTERVAL_PEERS_IN_SYNC_LOCK)
+				continue;
+		}
+
+		dynamic_drbd_dbg(device,
+				"%s at %llus+%u conflicts with %s at %llus+%u\n",
+				drbd_interval_type_str(interval),
+				(unsigned long long) sector, size,
+				drbd_interval_type_str(i),
+				(unsigned long long) i->sector, i->size);
+
+		break;
+	}
+
+	return i;
+}
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+#define ktime_aggregate_delta(D, ST, M) (D->M = ktime_add(D->M, ktime_sub(ktime_get(), ST)))
+#define ktime_aggregate(D, R, M) (D->M = ktime_add(D->M, ktime_sub(R->M, R->start_kt)))
+#define ktime_aggregate_pd(P, N, R, M) (P->M = ktime_add(P->M, ktime_sub(R->M[N], R->start_kt)))
+#define ktime_get_accounting(V) (V = ktime_get())
+#define ktime_get_accounting_assign(V, T) (V = T)
+#define ktime_var_for_accounting(V) ktime_t V = ktime_get()
+#else
+#define ktime_aggregate_delta(D, ST, M)
+#define ktime_aggregate(D, R, M)
+#define ktime_aggregate_pd(P, N, R, M)
+#define ktime_get_accounting(V)
+#define ktime_get_accounting_assign(V, T)
+#define ktime_var_for_accounting(V)
+#endif
+
 #endif
diff --git a/drivers/block/drbd/drbd_interval.c b/drivers/block/drbd/drbd_interval.c
index 873beda6de241..b16eeeaa27d30 100644
--- a/drivers/block/drbd/drbd_interval.c
+++ b/drivers/block/drbd/drbd_interval.c
@@ -14,9 +14,28 @@ sector_t interval_end(struct rb_node *node)
 }
 
 #define NODE_END(node) ((node)->sector + ((node)->size >> 9))
+RB_DECLARE_CALLBACKS_MAX(static, augment_callbacks, struct drbd_interval, rb,
+		sector_t, end, NODE_END);
+
+static const char * const drbd_interval_type_names[] = {
+	[INTERVAL_LOCAL_WRITE]    = "LocalWrite",
+	[INTERVAL_PEER_WRITE]     = "PeerWrite",
+	[INTERVAL_RESYNC_WRITE]   = "ResyncWrite",
+	[INTERVAL_RESYNC_READ]    = "ResyncRead",
+	[INTERVAL_OV_READ_SOURCE] = "VerifySource",
+	[INTERVAL_OV_READ_TARGET] = "VerifyTarget",
+	[INTERVAL_PEERS_IN_SYNC_LOCK] = "PeersInSync",
+};
+
+const char *drbd_interval_type_str(struct drbd_interval *i)
+{
+	enum drbd_interval_type type = i->type;
+	unsigned int size = sizeof drbd_interval_type_names / sizeof drbd_interval_type_names[0];
 
-RB_DECLARE_CALLBACKS_MAX(static, augment_callbacks,
-			 struct drbd_interval, rb, sector_t, end, NODE_END);
+	return (type < 0 || type >= size ||
+		!drbd_interval_type_names[type]) ?
+		       "?" : drbd_interval_type_names[type];
+}
 
 /*
  * drbd_insert_interval  -  insert a new interval into a tree
@@ -102,6 +121,18 @@ drbd_remove_interval(struct rb_root *root, struct drbd_interval *this)
 	rb_erase_augmented(&this->rb, root, &augment_callbacks);
 }
 
+void drbd_update_interval_size(struct drbd_interval *this, unsigned int new_size)
+{
+	this->size = new_size;
+
+	/* The size is one of the inputs to calculate the tree node's
+	 * augmented value. When we change it we need to update the augmented
+	 * value in this node and maybe in some parent nodes. That might be
+	 * all the way up to the root. As this function is used for joining
+	 * intervals, usually it will propagate only to the parent node. */
+	augment_callbacks_propagate(&this->rb, NULL);
+}
+
 /**
  * drbd_find_overlap  - search for an interval overlapping with [sector, sector + size)
  * @root:	red black tree root
diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h
index 5d3213b81eede..e875392dd003b 100644
--- a/drivers/block/drbd/drbd_interval.h
+++ b/drivers/block/drbd/drbd_interval.h
@@ -5,20 +5,150 @@
 #include <linux/types.h>
 #include <linux/rbtree.h>
 
+/* Interval types stored directly in drbd_interval so that we can handle
+ * conflicts without having to inspect the containing object. The value 0 is
+ * reserved for uninitialized intervals. */
+enum drbd_interval_type {
+	INTERVAL_LOCAL_WRITE = 1,
+	INTERVAL_PEER_WRITE,
+	INTERVAL_LOCAL_READ,
+	INTERVAL_PEER_READ,
+	INTERVAL_RESYNC_WRITE, /* L_SYNC_TARGET */
+	INTERVAL_RESYNC_READ, /* L_SYNC_SOURCE */
+	INTERVAL_OV_READ_SOURCE, /* L_VERIFY_S */
+	INTERVAL_OV_READ_TARGET, /* L_VERIFY_T */
+	INTERVAL_PEERS_IN_SYNC_LOCK,
+};
+
+#define INTERVAL_TYPE_MASK(type) (1 << (type))
+
+enum drbd_interval_flags {
+	/*
+	 * Whether this peer request has been sent yet. For resync writes, the
+	 * flag is set before sending.
+	 *
+	 * For resync reads, this flag is set after sending and is used to
+	 * manage the lifetime of the request. When INTERVAL_SENT is not set,
+	 * the sending path still has a reference to the request. This
+	 * reference counting is protected by peer_reqs_lock.
+	 */
+	INTERVAL_SENT,
+
+	/*
+	 * Whether this peer request has been received yet.
+	 *
+	 * For resync reads, this flag is set when the corresponding ack has
+	 * been received and is used to manage the lifetime of the request.
+	 * When INTERVAL_RECEIVED is not set, the receiving path has a
+	 * reference to the request. This reference counting is protected by
+	 * peer_reqs_lock.
+	 */
+	INTERVAL_RECEIVED,
+
+	/* Whether this has been queued after conflict. */
+	INTERVAL_SUBMIT_CONFLICT_QUEUED,
+
+	/* Whether this has been submitted already. */
+	INTERVAL_SUBMITTED,
+
+	/* Whether the local backing device bio is complete. */
+	INTERVAL_BACKING_COMPLETED,
+
+	/* This has been completed already; ignore for conflict detection. */
+	INTERVAL_COMPLETED,
+
+	/* For verify requests: whether this has conflicts. */
+	INTERVAL_CONFLICT,
+
+	/* For resync requests: whether this was canceled while waiting for conflict resolution. */
+	INTERVAL_CANCELED,
+
+	/*
+	 * For local requests: whether this is done.
+	 *
+	 * Included here instead of in local_rq_state to allow access with
+	 * atomic bit operations instead of taking rq_lock.
+	 */
+	INTERVAL_DONE,
+
+	/*
+	 * For local requests: when we put the AL extent for this request, it
+	 * was the last in that extent.
+	 *
+	 * Included here instead of in local_rq_state to allow access with
+	 * atomic bit operations instead of taking rq_lock.
+	 */
+	INTERVAL_AL_EXTENT_LAST,
+};
+
+/* Intervals used to manage conflicts between application requests and various
+ * internal requests, so that the disk content is deterministic.
+ *
+ * The requests progress through states indicated by successively setting the
+ * flags "INTERVAL_SUBMITTED", "INTERVAL_BACKING_COMPLETED" and
+ * "INTERVAL_COMPLETED".
+ *
+ * Application and resync requests wait to be submitted until any conflicts
+ * that are "INTERVAL_SUBMITTED" have reached "INTERVAL_BACKING_COMPLETED"
+ * state. Application requests also wait for conflicting application requests
+ * to ensure consistency between the replicated copies. In addition,
+ * application requests wait for resync requests that have not yet been
+ * submitted. Resync takes priority over application writes in this way because
+ * a resync locks each block at most once, so it will finish at some point,
+ * whereas the application may repeatedly write the same blocks, which would
+ * potentially lock out resync indefinitely.
+ *
+ * Resync read requests do not conflict with each other, but they are
+ * nevertheless mutually exclusive with writes, so that the bitmap can be
+ * updated reliably.
+ *
+ * Verify requests do not wait for other requests. If there are conflicts, they
+ * are simply cancelled. Futhermore, they do not lock out other requests;
+ * instead they are simply marked as having conflicts and ignored.
+ *
+ * Application write request intervals are retained even when they are
+ * "INTERVAL_COMPLETED", so that they can be used to look up remote replies
+ * that are still pending.
+ */
 struct drbd_interval {
 	struct rb_node rb;
 	sector_t sector;		/* start sector of the interval */
 	sector_t end;			/* highest interval end in subtree */
 	unsigned int size;		/* size in bytes */
-	unsigned int local:1		/* local or remote request? */;
-	unsigned int waiting:1;		/* someone is waiting for completion */
-	unsigned int completed:1;	/* this has been completed already;
-					 * ignore for conflict detection */
+	enum drbd_interval_type type;	/* what type of interval this is */
+	unsigned long flags;
 
 	/* to resume a partially successful drbd_al_begin_io_nonblock(); */
 	unsigned int partially_in_al_next_enr;
 };
 
+static inline bool drbd_interval_is_application(struct drbd_interval *i)
+{
+	return i->type == INTERVAL_LOCAL_WRITE || i->type == INTERVAL_PEER_WRITE ||
+		i->type == INTERVAL_LOCAL_READ || i->type == INTERVAL_PEER_READ;
+}
+
+static inline bool drbd_interval_is_write(struct drbd_interval *i)
+{
+	return i->type == INTERVAL_LOCAL_WRITE || i->type == INTERVAL_PEER_WRITE ||
+		i->type == INTERVAL_RESYNC_WRITE;
+}
+
+static inline bool drbd_interval_is_resync(struct drbd_interval *i)
+{
+	return i->type == INTERVAL_RESYNC_WRITE || i->type == INTERVAL_RESYNC_READ;
+}
+
+static inline bool drbd_interval_is_verify(struct drbd_interval *i)
+{
+	return i->type == INTERVAL_OV_READ_SOURCE || i->type == INTERVAL_OV_READ_TARGET;
+}
+
+static inline bool drbd_interval_is_local(struct drbd_interval *i)
+{
+	return i->type == INTERVAL_LOCAL_READ || i->type == INTERVAL_LOCAL_WRITE;
+}
+
 static inline void drbd_clear_interval(struct drbd_interval *i)
 {
 	RB_CLEAR_NODE(&i->rb);
@@ -29,14 +159,17 @@ static inline bool drbd_interval_empty(struct drbd_interval *i)
 	return RB_EMPTY_NODE(&i->rb);
 }
 
-extern bool drbd_insert_interval(struct rb_root *, struct drbd_interval *);
-extern bool drbd_contains_interval(struct rb_root *, sector_t,
-				   struct drbd_interval *);
-extern void drbd_remove_interval(struct rb_root *, struct drbd_interval *);
-extern struct drbd_interval *drbd_find_overlap(struct rb_root *, sector_t,
-					unsigned int);
-extern struct drbd_interval *drbd_next_overlap(struct drbd_interval *, sector_t,
-					unsigned int);
+const char *drbd_interval_type_str(struct drbd_interval *i);
+bool drbd_insert_interval(struct rb_root *root, struct drbd_interval *this);
+bool drbd_contains_interval(struct rb_root *root, sector_t sector,
+			    struct drbd_interval *interval);
+void drbd_remove_interval(struct rb_root *root, struct drbd_interval *this);
+struct drbd_interval *drbd_find_overlap(struct rb_root *root, sector_t sector,
+					unsigned int size);
+struct drbd_interval *drbd_next_overlap(struct drbd_interval *i,
+					sector_t sector, unsigned int size);
+void drbd_update_interval_size(struct drbd_interval *this,
+			       unsigned int new_size);
 
 #define drbd_for_each_overlap(i, root, sector, size)		\
 	for (i = drbd_find_overlap(root, sector, size);		\
diff --git a/drivers/block/drbd/drbd_legacy_84.c b/drivers/block/drbd/drbd_legacy_84.c
new file mode 100644
index 0000000000000..5363dab319181
--- /dev/null
+++ b/drivers/block/drbd/drbd_legacy_84.c
@@ -0,0 +1,559 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "drbd_legacy_84.h"
+#include "drbd_meta_data.h"
+
+/*
+ *   drbd-8.4                      drbd-9 md.flags                   drbd-9 peer-md.flags
+ * MDF_CONSISTENT      1 << 0  MDF_CONSISTENT =        1 << 0,   MDF_PEER_CONNECTED =    1 << 0,
+ * MDF_PRIMARY_IND     1 << 1  MDF_PRIMARY_IND =       1 << 1,   MDF_PEER_OUTDATED =     1 << 1,
+ * MDF_CONNECTED_IND   1 << 2                                    MDF_PEER_FENCING =      1 << 2,
+ * MDF_FULL_SYNC       1 << 3                                    MDF_PEER_FULL_SYNC =    1 << 3,
+ * MDF_WAS_UP_TO_DATE  1 << 4  MDF_WAS_UP_TO_DATE =    1 << 4,   MDF_PEER_DEVICE_SEEN =  1 << 4,
+ * MDF_PEER_OUT_DATED  1 << 5
+ * MDF_CRASHED_PRIMARY 1 << 6  MDF_CRASHED_PRIMARY =   1 << 6,
+ * MDF_AL_CLEAN        1 << 7  MDF_AL_CLEAN =          1 << 7,
+ * MDF_AL_DISABLED     1 << 8  MDF_AL_DISABLED =       1 << 8,
+ *                             MDF_PRIMARY_LOST_QUORUM = 1 << 9,
+ *                             MDF_HAVE_QUORUM =       1 << 10,
+ *                                                                MDF_NODE_EXISTS =      1 << 16,
+ */
+
+#define MDF_84_MASK (MDF_CONSISTENT | MDF_PRIMARY_IND | MDF_WAS_UP_TO_DATE | \
+		     MDF_CRASHED_PRIMARY | MDF_AL_CLEAN | MDF_AL_DISABLED)
+#define MDF_84_PEER_MASK (MDF_PEER_FULL_SYNC)
+#define MDF_84_CONNECTED_IND (1<<2)
+#define MDF_84_PEER_OUTDATED (1<<5)
+
+struct meta_data_on_disk_84 {
+	u64 la_size_sect;      /* last agreed size. */
+	u64 uuid[UI_SIZE];   /* UUIDs. */
+	u64 device_uuid;
+	u64 reserved_u64_1;
+	u32 flags;             /* MDF */
+	u32 magic;
+	u32 md_size_sect;
+	u32 al_offset;         /* offset to this block */
+	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
+	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
+	u32 bm_offset;         /* offset to the bitmap, from here */
+	u32 bm_bytes_per_bit;  /* 4k. Treat as magic number, must keep it compatible. */
+	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
+
+	/* see al_tr_number_to_on_disk_sector() */
+	u32 al_stripes;
+	u32 al_stripe_size_4k;
+
+	u8 reserved_u8[4096 - (7*8 + 10*4)];
+} __packed;
+
+
+static const char * const drbd_conn_s_names[] = {
+	[C_STANDALONE]       = "StandAlone",
+	[C_DISCONNECTING]    = "Disconnecting",
+	[C_UNCONNECTED]      = "Unconnected",
+	[C_TIMEOUT]          = "Timeout",
+	[C_BROKEN_PIPE]      = "BrokenPipe",
+	[C_NETWORK_FAILURE]  = "NetworkFailure",
+	[C_PROTOCOL_ERROR]   = "ProtocolError",
+	[C_CONNECTING]       = "WFConnection",
+	/* [C_WF_REPORT_PARAMS] = "WFReportParams", */
+	[C_TEAR_DOWN]        = "TearDown",
+	[C_CONNECTED]        = "Connected",
+	[L_STARTING_SYNC_S]  = "StartingSyncS",
+	[L_STARTING_SYNC_T]  = "StartingSyncT",
+	[L_WF_BITMAP_S]      = "WFBitMapS",
+	[L_WF_BITMAP_T]      = "WFBitMapT",
+	[L_WF_SYNC_UUID]     = "WFSyncUUID",
+	[L_SYNC_SOURCE]      = "SyncSource",
+	[L_SYNC_TARGET]      = "SyncTarget",
+	[L_PAUSED_SYNC_S]    = "PausedSyncS",
+	[L_PAUSED_SYNC_T]    = "PausedSyncT",
+	[L_VERIFY_S]         = "VerifyS",
+	[L_VERIFY_T]         = "VerifyT",
+	[L_AHEAD]            = "Ahead",
+	[L_BEHIND]           = "Behind",
+};
+
+static const char write_ordering_chars[] = {
+	[WO_NONE] = 'n',
+	[WO_DRAIN_IO] = 'd',
+	[WO_BDEV_FLUSH] = 'f',
+	[WO_BIO_BARRIER] = 'b',
+};
+
+
+static int seq_print_device_proc_drbd(struct seq_file *m, struct drbd_device *device);
+
+atomic_t nr_drbd8_devices;
+
+void drbd_md_decode_84(struct meta_data_on_disk_84 *on_disk, struct drbd_md *md)
+{
+	struct drbd_peer_md *peer_md;
+	const int peer_node_id = 0; /* setup_node_ids_84() moves it later */
+	u32 on_disk_flags;
+	int i;
+
+	md->effective_size = be64_to_cpu(on_disk->la_size_sect);
+	md->current_uuid = be64_to_cpu(on_disk->uuid[UI_CURRENT]);
+	md->prev_members = 0;
+	md->device_uuid = be64_to_cpu(on_disk->device_uuid);
+	md->md_size_sect = be32_to_cpu(on_disk->md_size_sect);
+	md->al_offset = be32_to_cpu(on_disk->al_offset);
+
+	md->bm_offset = be32_to_cpu(on_disk->bm_offset);
+
+	on_disk_flags = be32_to_cpu(on_disk->flags);
+	md->flags = on_disk_flags & MDF_84_MASK;
+
+	md->max_peers = 1;
+	md->bm_block_size = be32_to_cpu(on_disk->bm_bytes_per_bit);
+	md->node_id = -1; /* no node_id in the drbd-8.4 meta-data */
+	md->al_stripes = be32_to_cpu(on_disk->al_stripes);
+	md->al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
+
+
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		peer_md = &md->peers[i];
+
+		peer_md->bitmap_uuid = 0;
+		peer_md->bitmap_dagtag = 0;
+		peer_md->flags = 0;
+		peer_md->bitmap_index = -1;
+	}
+	peer_md = &md->peers[peer_node_id];
+	peer_md->bitmap_uuid = be64_to_cpu(on_disk->uuid[UI_BITMAP]);
+	peer_md->bitmap_index = 0;
+	peer_md->flags = on_disk_flags & MDF_84_PEER_MASK;
+	peer_md->flags |= MDF_HAVE_BITMAP;
+	peer_md->flags |= on_disk_flags & MDF_84_PEER_OUTDATED ? MDF_PEER_OUTDATED : 0;
+	peer_md->flags |= on_disk_flags & MDF_84_CONNECTED_IND ? MDF_PEER_CONNECTED : 0;
+
+
+	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
+		md->history_uuids[i - UI_HISTORY_START] = be64_to_cpu(on_disk->uuid[i]);
+}
+
+void drbd_md_encode_84(struct drbd_device *device, struct meta_data_on_disk_84 *buffer)
+{
+	struct drbd_md *md = &device->ldev->md;
+	int peer_node_id = !md->node_id;
+	struct drbd_peer_md *peer_md = &md->peers[peer_node_id];
+	u32 flags = (md->flags & MDF_84_MASK) | (peer_md->flags & MDF_84_PEER_MASK);
+	int i;
+
+	if (device->bitmap == NULL)
+		flags |= MDF_PEER_FULL_SYNC;
+
+	flags |= peer_md->flags & MDF_PEER_OUTDATED ? MDF_84_PEER_OUTDATED : 0;
+	flags |= peer_md->flags & MDF_PEER_CONNECTED ? MDF_84_CONNECTED_IND : 0;
+	buffer->la_size_sect = cpu_to_be64(md->effective_size);
+	buffer->device_uuid = cpu_to_be64(md->device_uuid);
+	buffer->uuid[UI_CURRENT] = cpu_to_be64(md->current_uuid);
+	buffer->uuid[UI_BITMAP] = cpu_to_be64(peer_md->bitmap_uuid);
+	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
+		buffer->uuid[i] = cpu_to_be64(md->history_uuids[i - UI_HISTORY_START]);
+	buffer->reserved_u64_1 = 0;
+	buffer->flags = cpu_to_be32(flags);
+	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
+	buffer->md_size_sect = cpu_to_be32(md->md_size_sect);
+	buffer->al_offset = cpu_to_be32(md->al_offset);
+	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
+	buffer->bm_offset = cpu_to_be32(md->bm_offset);
+	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE_4k); /* treat as magic number */
+	buffer->la_peer_max_bio_size = cpu_to_be32(device->device_conf.max_bio_size);
+
+	buffer->al_stripes = cpu_to_be32(md->al_stripes);
+	buffer->al_stripe_size_4k = cpu_to_be32(md->al_stripe_size_4k);
+}
+
+
+/*
+ * This is DRBD 8 userspace compatibility mode, so we do not have a node ID
+ * yet. We derive our own node ID from the peer node ID. drbdsetup gives us the
+ * peer-node-id, which it determines by comparing the IP addresses.
+ */
+int drbd_setup_node_ids_84(struct drbd_connection *connection, struct drbd_path *path,
+			    unsigned int peer_node_id)
+{
+	int vnr, my_node_id, nr_legacy = 0, nr_v9 = 0;
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+
+	my_node_id = !peer_node_id;
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (test_bit(LEGACY_84_MD, &device->flags)) {
+			nr_legacy++;
+		} else {
+			nr_v9++;
+			if (get_ldev(device)) {
+				int md_my_node_id = device->ldev->md.node_id;
+
+				put_ldev(device);
+				if (my_node_id != md_my_node_id) {
+					drbd_err(connection, "inconsistent node_ids %d %d\n",
+						 my_node_id, md_my_node_id);
+					return -ENOTUNIQ;
+				}
+			}
+		}
+	}
+
+	if (nr_legacy && nr_v9)
+		drbd_warn(connection, "legacy-84 and drbd-9 metadata in one resource\n");
+
+	drbd_info(connection, "drbd8 userspace compat mode: setting my node id to %d\n",
+		  my_node_id);
+
+	/* setting up all node_ids*/
+	resource->res_opts.node_id = my_node_id;
+	connection->peer_node_id = peer_node_id;
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		peer_device = list_first_entry_or_null(&device->peer_devices,
+						       struct drbd_peer_device, peer_devices);
+		peer_device->node_id = peer_node_id;
+		peer_device->bitmap_index = 0;
+
+		if (get_ldev(device)) {
+			const struct drbd_peer_md clear = { .bitmap_index = -1 };
+			struct drbd_md *md = &device->ldev->md;
+			struct drbd_peer_md *to = &md->peers[peer_node_id];
+			int i;
+
+			md->node_id = my_node_id;
+
+			for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+				struct drbd_peer_md *from = &md->peers[i];
+
+				if (from->bitmap_index != -1) {
+					if (from != to) {
+						*to = *from;
+						*from = clear;
+					}
+					break;
+				}
+			}
+			put_ldev(device);
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+ * Some resources may be operating in "DRBD 8 compatibility mode", where the
+ * user created the resource using the old drbd8-style drbdsetup command line
+ * syntax.
+ * This implies that the user probably also expects the old drbd8-style
+ * /proc/drbd output showing the device state.
+ * If the flag is set for a resource, we show the old-style output for that
+ * resource.
+ * If any resource is in DRBD 8 compatibility mode, this function returns true.
+ */
+bool drbd_show_legacy_device(struct seq_file *seq, void *v)
+{
+	struct drbd_device *device;
+	int i, prev_i = -1;
+
+	if (!atomic_read(&nr_drbd8_devices))
+		return false;
+
+	rcu_read_lock();
+	idr_for_each_entry(&drbd_devices, device, i) {
+		if (!device->resource->res_opts.drbd8_compat_mode)
+			continue;
+
+		if (prev_i != i - 1)
+			seq_putc(seq, '\n');
+		prev_i = i;
+
+		seq_print_device_proc_drbd(seq, device);
+	}
+	rcu_read_unlock();
+	return true;
+}
+
+static void seq_printf_with_thousands_grouping(struct seq_file *seq, long v)
+{
+	/* v is in kB/sec. We don't expect TiByte/sec yet. */
+	if (unlikely(v >= 1000000)) {
+		/* cool: > GiByte/s */
+		seq_printf(seq, "%ld,", v / 1000000);
+		v %= 1000000;
+		seq_printf(seq, "%03ld,%03ld", v/1000, v % 1000);
+	} else if (likely(v >= 1000))
+		seq_printf(seq, "%ld,%03ld", v/1000, v % 1000);
+	else
+		seq_printf(seq, "%ld", v);
+}
+
+static void drbd_get_syncer_progress(struct drbd_peer_device *pd,
+		enum drbd_repl_state repl_state, unsigned long *rs_total,
+		unsigned long *bits_left, unsigned int *per_mil_done)
+{
+	/* this is to break it at compile time when we change that, in case we
+	 * want to support more than (1<<32) bits on a 32bit arch.
+	 */
+	typecheck(unsigned long, pd->rs_total);
+	*rs_total = pd->rs_total;
+
+	/* note: both rs_total and rs_left are in bits, i.e. in
+	 * units of BM_BLOCK_SIZE.
+	 * for the percentage, we don't care.
+	 */
+
+	if (repl_state == L_VERIFY_S || repl_state == L_VERIFY_T)
+		*bits_left = atomic64_read(&pd->ov_left);
+	else
+		*bits_left = drbd_bm_total_weight(pd) - pd->rs_failed;
+	/* >> 10 to prevent overflow,
+	 * +1 to prevent division by zero
+	 */
+	if (*bits_left > *rs_total) {
+		/* D'oh. Maybe a logic bug somewhere.  More likely just a race
+		 * between state change and reset of rs_total.
+		 */
+		*bits_left = *rs_total;
+		*per_mil_done = *rs_total ? 0 : 1000;
+	} else {
+		/* Make sure the division happens in long context.
+		 * We allow up to one petabyte storage right now,
+		 * at a granularity of 4k per bit that is 2**38 bits.
+		 * After shift right and multiplication by 1000,
+		 * this should still fit easily into a 32bit long,
+		 * so we don't need a 64bit division on 32bit arch.
+		 * Note: currently we don't support such large bitmaps on 32bit
+		 * arch anyways, but no harm done to be prepared for it here.
+		 */
+		unsigned int shift = *rs_total > UINT_MAX ? 16 : 10;
+		unsigned long left = *bits_left >> shift;
+		unsigned long total = 1UL + (*rs_total >> shift);
+		unsigned long tmp = 1000UL - left * 1000UL/total;
+		*per_mil_done = tmp;
+	}
+}
+
+static void drbd_syncer_progress(struct drbd_peer_device *pd, struct seq_file *seq,
+		enum drbd_repl_state repl_state)
+{
+	unsigned long db, dt, dbdt, rt, rs_total, rs_left;
+	unsigned int res;
+	int i, x, y;
+	int stalled = 0;
+	unsigned int bm_block_shift = pd->device->last_bm_block_shift;
+
+	drbd_get_syncer_progress(pd, repl_state, &rs_total, &rs_left, &res);
+
+	x = res/50;
+	y = 20-x;
+	seq_puts(seq, "\t[");
+	for (i = 1; i < x; i++)
+		seq_putc(seq, '=');
+	seq_putc(seq, '>');
+	for (i = 0; i < y; i++)
+		seq_putc(seq, '.');
+	seq_puts(seq, "] ");
+
+	if (repl_state == L_VERIFY_S || repl_state == L_VERIFY_T)
+		seq_puts(seq, "verified:");
+	else
+		seq_puts(seq, "sync'ed:");
+	seq_printf(seq, "%3u.%u%% ", res / 10, res % 10);
+
+	/* if more than a few GB, display in MB */
+	if (rs_total > (4UL << (30 - bm_block_shift)))
+		seq_printf(seq, "(%llu/%llu)M",
+			    bit_to_kb(rs_left >> 10, bm_block_shift),
+			    bit_to_kb(rs_total >> 10, bm_block_shift));
+	else
+		seq_printf(seq, "(%llu/%llu)K",
+			    bit_to_kb(rs_left, bm_block_shift),
+			    bit_to_kb(rs_total, bm_block_shift));
+
+	seq_puts(seq, "\n\t");
+
+	/* see drivers/md/md.c
+	 * We do not want to overflow, so the order of operands and
+	 * the * 100 / 100 trick are important. We do a +1 to be
+	 * safe against division by zero. We only estimate anyway.
+	 *
+	 * dt: time from mark until now
+	 * db: blocks written from mark until now
+	 * rt: remaining time
+	 */
+	/* Rolling marks. last_mark+1 may just now be modified.  last_mark+2 is
+	 * at least (DRBD_SYNC_MARKS-2)*DRBD_SYNC_MARK_STEP old, and has at
+	 * least DRBD_SYNC_MARK_STEP time before it will be modified.
+	 */
+	/* ------------------------ ~18s average ------------------------ */
+	i = (pd->rs_last_mark + 2) % DRBD_SYNC_MARKS;
+	dt = (jiffies - pd->rs_mark_time[i]) / HZ;
+	if (dt > 180)
+		stalled = 1;
+
+	if (!dt)
+		dt++;
+	db = pd->rs_mark_left[i] - rs_left;
+	rt = (dt * (rs_left / (db/100+1)))/100; /* seconds */
+
+	seq_printf(seq, "finish: %lu:%02lu:%02lu",
+		rt / 3600, (rt % 3600) / 60, rt % 60);
+
+	dbdt = bit_to_kb(db/dt, bm_block_shift);
+	seq_puts(seq, " speed: ");
+	seq_printf_with_thousands_grouping(seq, dbdt);
+	seq_puts(seq, " (");
+	/* ------------------------- ~3s average ------------------------ */
+	if (1) {
+		/* this is what drbd_rs_should_slow_down() uses */
+		i = (pd->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
+		dt = (jiffies - pd->rs_mark_time[i]) / HZ;
+		if (!dt)
+			dt++;
+		db = pd->rs_mark_left[i] - rs_left;
+		dbdt = bit_to_kb(db/dt, bm_block_shift);
+		seq_printf_with_thousands_grouping(seq, dbdt);
+		seq_puts(seq, " -- ");
+	}
+
+	/* --------------------- long term average ---------------------- */
+	/* mean speed since syncer started we do account for PausedSync periods */
+	dt = (jiffies - pd->rs_start - pd->rs_paused) / HZ;
+	if (dt == 0)
+		dt = 1;
+	db = rs_total - rs_left;
+	dbdt = bit_to_kb(db/dt, bm_block_shift);
+	seq_printf_with_thousands_grouping(seq, dbdt);
+	seq_putc(seq, ')');
+
+	if (repl_state == L_SYNC_TARGET ||
+	    repl_state == L_VERIFY_S) {
+		seq_puts(seq, " want: ");
+		seq_printf_with_thousands_grouping(seq, pd->c_sync_rate);
+	}
+	seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : "");
+
+	{
+		/* 64 bit: we convert to sectors in the display below. */
+		unsigned long bm_bits = drbd_bm_bits(pd->device);
+		unsigned long bit_pos;
+		unsigned long long stop_sector = 0;
+
+		if (repl_state == L_VERIFY_S ||
+		    repl_state == L_VERIFY_T) {
+			bit_pos = bm_bits - (unsigned long)atomic64_read(&pd->ov_left);
+			if (verify_can_do_stop_sector(pd))
+				stop_sector = pd->ov_stop_sector;
+		} else
+			bit_pos = pd->resync_next_bit;
+		/* Total sectors may be slightly off for oddly sized devices. So what. */
+		seq_printf(seq,
+			"\t%3d%% sector pos: %llu/%llu",
+			(int)(bit_pos / (bm_bits/100+1)),
+			(unsigned long long)bit_pos * sect_per_bit(bm_block_shift),
+			(unsigned long long)bm_bits * sect_per_bit(bm_block_shift));
+		if (stop_sector != 0 && stop_sector != ULLONG_MAX)
+			seq_printf(seq, " stop sector: %llu", stop_sector);
+		seq_putc(seq, '\n');
+	}
+}
+
+static const char *drbd_conn_str_84(enum drbd_conn_state s)
+{
+	/* enums are unsigned... */
+	return (int)s > (int)L_BEHIND ? "TOO_LARGE" : drbd_conn_s_names[s];
+}
+
+
+static int seq_print_device_proc_drbd(struct seq_file *m, struct drbd_device *device)
+{
+	unsigned int send_kb, recv_kb, pending_cnt, unacked_cnt, epochs;
+	struct drbd_connection *connection = NULL;
+	struct drbd_peer_device *peer_device;
+	union drbd_state state;
+	const char *sn;
+	char wp;
+
+	peer_device = list_first_or_null_rcu(&device->peer_devices, struct drbd_peer_device,
+					     peer_devices);
+
+	if (peer_device) {
+		state = drbd_get_peer_device_state(peer_device, NOW);
+		connection = peer_device->connection;
+		send_kb = peer_device->send_cnt/2;
+		recv_kb = peer_device->recv_cnt/2;
+		pending_cnt = atomic_read(&peer_device->ap_pending_cnt) +
+			atomic_read(&peer_device->rs_pending_cnt);
+		unacked_cnt = atomic_read(&peer_device->unacked_cnt);
+	} else {
+		state = drbd_get_device_state(device, NOW);
+		connection = list_first_or_null_rcu(&device->resource->connections,
+						    struct drbd_connection, connections);
+		send_kb = 0;
+		recv_kb = 0;
+		pending_cnt = 0;
+		unacked_cnt = 0;
+	}
+	if (connection) {
+		struct net_conf *nc = rcu_dereference(connection->transport.net_conf);
+
+		wp = nc ? nc->wire_protocol - DRBD_PROT_A + 'A' : ' ';
+		epochs = connection->epochs;
+	} else {
+		wp = 'C';
+		epochs = 0;
+	}
+
+	sn = drbd_conn_str_84(state.conn);
+
+	if (state.conn == C_STANDALONE &&
+	    state.disk == D_DISKLESS &&
+	    state.role == R_SECONDARY) {
+		seq_printf(m, "%2d: cs:Unconfigured\n", device->minor);
+	} else {
+		seq_printf(m,
+			   "%2d: cs:%s ro:%s/%s ds:%s/%s %c %c%c%c%c%c%c\n"
+			   "    ns:%u nr:%u dw:%u dr:%u al:%u bm:%u "
+			   "lo:%d pe:%d ua:%d ap:%d ep:%d wo:%c",
+			   device->minor, sn,
+			   drbd_role_str(state.role),
+			   drbd_role_str(state.peer),
+			   drbd_disk_str(state.disk),
+			   drbd_disk_str(state.pdsk),
+			   wp,
+			   drbd_suspended(device) ? 's' : 'r',
+			   state.aftr_isp ? 'a' : '-',
+			   state.peer_isp ? 'p' : '-',
+			   state.user_isp ? 'u' : '-',
+			   '-' /* congestion reason... FIXME */,
+			   test_bit(AL_SUSPENDED, &device->flags) ? 's' : '-',
+			   send_kb,
+			   recv_kb,
+			   device->writ_cnt/2,
+			   device->read_cnt/2,
+			   device->al_writ_cnt,
+			   device->bm_writ_cnt,
+			   atomic_read(&device->local_cnt),
+			   pending_cnt,
+			   unacked_cnt,
+			   atomic_read(&device->ap_bio_cnt[WRITE]) +
+			   atomic_read(&device->ap_bio_cnt[READ]),
+			   epochs,
+			   write_ordering_chars[device->resource->write_ordering]
+			);
+		seq_printf(m, " oos:%llu\n",
+			   peer_device ?
+				device_bit_to_kb(device, drbd_bm_total_weight(peer_device)) : 0);
+	}
+	if (state.conn == L_SYNC_SOURCE ||
+	    state.conn == L_SYNC_TARGET ||
+	    state.conn == L_VERIFY_S ||
+	    state.conn == L_VERIFY_T)
+		drbd_syncer_progress(peer_device, m, state.conn);
+
+	/* drbd_proc_details 1 or 2 missing */
+
+	return 0;
+}
diff --git a/drivers/block/drbd/drbd_legacy_84.h b/drivers/block/drbd/drbd_legacy_84.h
new file mode 100644
index 0000000000000..5c11a43c0bfaa
--- /dev/null
+++ b/drivers/block/drbd/drbd_legacy_84.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#ifndef __DRBD_LEGACY_84_H
+#define __DRBD_LEGACY_84_H
+
+#include "drbd_int.h"
+
+struct meta_data_on_disk_84;
+
+#ifdef CONFIG_DRBD_COMPAT_84
+extern atomic_t nr_drbd8_devices;
+
+void drbd_md_decode_84(struct meta_data_on_disk_84 *on_disk, struct drbd_md *md);
+void drbd_md_encode_84(struct drbd_device *device, struct meta_data_on_disk_84 *buffer);
+int drbd_setup_node_ids_84(struct drbd_connection *connection, struct drbd_path *path,
+			   unsigned int peer_node_id);
+bool drbd_show_legacy_device(struct seq_file *seq, void *v);
+#else
+static inline void drbd_md_decode_84(struct meta_data_on_disk_84 *on_disk, struct drbd_md *md) {};
+static inline void drbd_md_encode_84(struct drbd_device *device,
+	struct meta_data_on_disk_84 *buffer) {};
+static inline int drbd_setup_node_ids_84(struct drbd_connection *connection, struct drbd_path *path,
+			   unsigned int peer_node_id) { return 0; };
+static inline bool drbd_show_legacy_device(struct seq_file *seq, void *v) { return false; };
+#endif  /* CONFIG_DRBD_COMPAT_84 */
+
+#endif  /* __DRBD_LEGACY_84_H */
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 200d464e984bb..e2c3530cfa13e 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
-   drbd.c
+   drbd_main.c
 
    This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
 
@@ -19,41 +19,47 @@
 #include <linux/module.h>
 #include <linux/jiffies.h>
 #include <linux/drbd.h>
-#include <linux/uaccess.h>
 #include <asm/types.h>
+#include <net/net_namespace.h>
 #include <net/sock.h>
 #include <linux/ctype.h>
-#include <linux/mutex.h>
 #include <linux/fs.h>
 #include <linux/file.h>
 #include <linux/proc_fs.h>
 #include <linux/init.h>
 #include <linux/mm.h>
-#include <linux/memcontrol.h>
+#include <linux/memcontrol.h> /* needed on kernels <4.3 */
 #include <linux/mm_inline.h>
 #include <linux/slab.h>
 #include <linux/string.h>
-#include <linux/random.h>
-#include <linux/reboot.h>
 #include <linux/notifier.h>
-#include <linux/kthread.h>
 #include <linux/workqueue.h>
-#include <linux/unistd.h>
+#include <linux/kthread.h>
 #include <linux/vmalloc.h>
-#include <linux/sched/signal.h>
+#include <linux/dynamic_debug.h>
+#include <linux/libnvdimm.h>
+#include <linux/swab.h>
+#include <linux/overflow.h>
 
 #include <linux/drbd_limits.h>
 #include "drbd_int.h"
-#include "drbd_protocol.h"
-#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
+#include <linux/drbd_protocol.h>
+#include "drbd_req.h"
 #include "drbd_vli.h"
 #include "drbd_debugfs.h"
+#include "drbd_meta_data.h"
+#include "drbd_legacy_84.h"
+#include "drbd_dax_pmem.h"
 
-static DEFINE_MUTEX(drbd_main_mutex);
-static int drbd_open(struct gendisk *disk, blk_mode_t mode);
+static int drbd_open(struct gendisk *gd, blk_mode_t mode);
 static void drbd_release(struct gendisk *gd);
 static void md_sync_timer_fn(struct timer_list *t);
 static int w_bitmap_io(struct drbd_work *w, int unused);
+static int flush_send_buffer(struct drbd_connection *connection, enum drbd_stream drbd_stream);
+static u64 __set_bitmap_slots(struct drbd_device *device, u64 bitmap_uuid, u64 do_nodes) __must_hold(local);
+static u64 __test_bitmap_slots(struct drbd_device *device) __must_hold(local);
+static void drbd_send_ping_ack_wf(struct work_struct *ws);
+static void __net_exit __drbd_net_exit(struct net *net);
 
 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
 	      "Lars Ellenberg <lars@linbit.com>");
@@ -63,16 +69,16 @@ MODULE_LICENSE("GPL");
 MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
 		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);
+MODULE_SOFTDEP("post: handshake");
 
 #include <linux/moduleparam.h>
-/* thanks to these macros, if compiled into the kernel (not-module),
- * these become boot parameters (e.g., drbd.minor_count) */
 
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 int drbd_enable_faults;
 int drbd_fault_rate;
 static int drbd_fault_count;
 static int drbd_fault_devs;
+
 /* bitmap of enabled faults */
 module_param_named(enable_faults, drbd_enable_faults, int, 0664);
 /* fault rate % value - applies to all enabled faults */
@@ -84,15 +90,12 @@ module_param_named(fault_devs, drbd_fault_devs, int, 0644);
 #endif
 
 /* module parameters we can keep static */
-static bool drbd_allow_oos; /* allow_open_on_secondary */
 static bool drbd_disable_sendpage;
+static bool drbd_allow_oos; /* allow_open_on_secondary */
 MODULE_PARM_DESC(allow_oos, "DONT USE!");
-module_param_named(allow_oos, drbd_allow_oos, bool, 0);
 module_param_named(disable_sendpage, drbd_disable_sendpage, bool, 0644);
+module_param_named(allow_oos, drbd_allow_oos, bool, 0);
 
-/* module parameters we share */
-int drbd_proc_details; /* Detail level in proc drbd*/
-module_param_named(proc_details, drbd_proc_details, int, 0644);
 /* module parameters shared with defaults */
 unsigned int drbd_minor_count = DRBD_MINOR_COUNT_DEF;
 /* Module parameter for setting the user mode helper program
@@ -101,16 +104,60 @@ char drbd_usermode_helper[80] = "/sbin/drbdadm";
 module_param_named(minor_count, drbd_minor_count, uint, 0444);
 module_param_string(usermode_helper, drbd_usermode_helper, sizeof(drbd_usermode_helper), 0644);
 
+static int param_set_drbd_protocol_version(const char *s, const struct kernel_param *kp)
+{
+	unsigned long long tmp;
+	unsigned int *res = kp->arg;
+	int rv;
+
+	rv = kstrtoull(s, 0, &tmp);
+	if (rv < 0)
+		return rv;
+	if (!drbd_protocol_version_acceptable(tmp))
+		return -ERANGE;
+	*res = tmp;
+	return 0;
+}
+
+#define param_check_drbd_protocol_version	param_check_uint
+#define param_get_drbd_protocol_version		param_get_uint
+
+static const struct kernel_param_ops param_ops_drbd_protocol_version = {
+	.set = param_set_drbd_protocol_version,
+	.get = param_get_drbd_protocol_version,
+};
+
+unsigned int drbd_protocol_version_min = PRO_VERSION_8_MIN;
+module_param_named(protocol_version_min, drbd_protocol_version_min, drbd_protocol_version, 0644);
+#define protocol_version_min_desc								\
+	"\n\t\tReject DRBD dialects older than this.\n\t\t"					\
+	"Supported: "										\
+	"DRBD 8 [" __stringify(PRO_VERSION_8_MIN) "-" __stringify(PRO_VERSION_8_MAX) "]; "	\
+	"DRBD 9 [" __stringify(PRO_VERSION_MIN) "-" __stringify(PRO_VERSION_MAX) "].\n\t\t"	\
+	"Default: " __stringify(PRO_VERSION_8_MIN)
+MODULE_PARM_DESC(protocol_version_min, protocol_version_min_desc);
+
+#define param_check_drbd_strict_names		param_check_bool
+#define param_get_drbd_strict_names		param_get_bool
+const struct kernel_param_ops param_ops_drbd_strict_names = {
+	.set = param_set_drbd_strict_names,
+	.get = param_get_drbd_strict_names,
+};
+bool drbd_strict_names = true;
+MODULE_PARM_DESC(strict_names, "restrict resource and connection names to ascii alnum and a subset of punct");
+module_param_named(strict_names, drbd_strict_names, drbd_strict_names, 0644);
+
 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
  * as member "struct gendisk *vdisk;"
  */
 struct idr drbd_devices;
 struct list_head drbd_resources;
-struct mutex resources_mutex;
+static DEFINE_SPINLOCK(drbd_devices_lock);
+DEFINE_MUTEX(resources_mutex);
 
+struct workqueue_struct *ping_ack_sender;
 struct kmem_cache *drbd_request_cache;
 struct kmem_cache *drbd_ee_cache;	/* peer requests */
-struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
 struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
 mempool_t drbd_request_mempool;
 mempool_t drbd_ee_mempool;
@@ -119,8 +166,6 @@ mempool_t drbd_buffer_page_pool;
 struct bio_set drbd_md_io_bio_set;
 struct bio_set drbd_io_bio_set;
 
-DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);
-
 static const struct block_device_operations drbd_ops = {
 	.owner		= THIS_MODULE,
 	.submit_bio	= drbd_submit_bio,
@@ -128,6 +173,10 @@ static const struct block_device_operations drbd_ops = {
 	.release	= drbd_release,
 };
 
+static struct pernet_operations drbd_pernet_ops = {
+	.exit = __drbd_net_exit,
+};
+
 #ifdef __CHECKER__
 /* When checking with sparse, and this is an inline function, sparse will
    give tons of false positives. When this is a real functions sparse works.
@@ -137,7 +186,7 @@ int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
 	int io_allowed;
 
 	atomic_inc(&device->local_cnt);
-	io_allowed = (device->state.disk >= mins);
+	io_allowed = (device->disk_state[NOW] >= mins);
 	if (!io_allowed) {
 		if (atomic_dec_and_test(&device->local_cnt))
 			wake_up(&device->misc_wait);
@@ -147,52 +196,237 @@ int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
 
 #endif
 
+struct drbd_connection *__drbd_next_connection_ref(u64 *visited,
+						   struct drbd_connection *connection,
+						   struct drbd_resource *resource)
+{
+	int node_id;
+
+	rcu_read_lock();
+	if (!connection) {
+		connection = list_first_or_null_rcu(&resource->connections,
+						    struct drbd_connection,
+						    connections);
+		*visited = 0;
+	} else {
+		struct list_head *pos;
+		bool previous_visible; /* on the resources connections list */
+
+		pos = list_next_rcu(&connection->connections);
+		/* follow the pointer first, then check if the previous element was
+		   still an element on the list of visible connections. */
+		smp_rmb();
+		previous_visible = !test_bit(C_UNREGISTERED, &connection->flags);
+
+		kref_put(&connection->kref, drbd_destroy_connection);
+
+		if (pos == &resource->connections) {
+			connection = NULL;
+		} else if (previous_visible) {	/* visible -> we are now on a vital element */
+			connection = list_entry_rcu(pos, struct drbd_connection, connections);
+		} else { /* not visible -> pos might point to a dead element now */
+			for_each_connection_rcu(connection, resource) {
+				node_id = connection->peer_node_id;
+				if (!(*visited & NODE_MASK(node_id)))
+					goto found;
+			}
+			connection = NULL;
+		}
+	}
+
+	if (connection) {
+	found:
+		node_id = connection->peer_node_id;
+		*visited |= NODE_MASK(node_id);
+
+		kref_get(&connection->kref);
+	}
+
+	rcu_read_unlock();
+	return connection;
+}
+
+
+struct drbd_peer_device *__drbd_next_peer_device_ref(u64 *visited,
+						     struct drbd_peer_device *peer_device,
+						     struct drbd_device *device)
+{
+	rcu_read_lock();
+	if (!peer_device) {
+		peer_device = list_first_or_null_rcu(&device->peer_devices,
+						    struct drbd_peer_device,
+						    peer_devices);
+		*visited = 0;
+	} else {
+		struct list_head *pos;
+		bool previous_visible;
+
+		pos = list_next_rcu(&peer_device->peer_devices);
+		smp_rmb();
+		previous_visible = !test_bit(C_UNREGISTERED, &peer_device->connection->flags);
+
+		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
+
+		if (pos == &device->peer_devices) {
+			peer_device = NULL;
+		} else if (previous_visible) {
+			peer_device = list_entry_rcu(pos, struct drbd_peer_device, peer_devices);
+		} else {
+			for_each_peer_device_rcu(peer_device, device) {
+				if (!(*visited & NODE_MASK(peer_device->node_id)))
+					goto found;
+			}
+			peer_device = NULL;
+		}
+	}
+
+	if (peer_device) {
+	found:
+		*visited |= NODE_MASK(peer_device->node_id);
+
+		kref_get(&peer_device->connection->kref);
+	}
+
+	rcu_read_unlock();
+	return peer_device;
+}
+
+static void dump_epoch(struct drbd_resource *resource, int node_id, int epoch)
+{
+	struct drbd_request *req;
+	bool found_epoch = false;
+
+	list_for_each_entry_rcu(req, &resource->transfer_log, tl_requests) {
+		if (!found_epoch && req->epoch == epoch)
+			found_epoch = true;
+
+		if (found_epoch) {
+			if (req->epoch != epoch)
+				break;
+			drbd_info(req->device, "XXX %u %llu+%u 0x%x 0x%x\n",
+					req->epoch,
+					(unsigned long long)req->i.sector, req->i.size >> 9,
+					req->local_rq_state, req->net_rq_state[node_id]
+				 );
+		}
+	}
+}
+
 /**
  * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
  * @connection:	DRBD connection.
+ * @o_block_id: "block id" aka expected pointer address of the oldest request
+ * @y_block_id: "block id" aka expected pointer address of the youngest request
+ *		confirmed to be on stable storage.
  * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
- * @set_size:	Expected number of requests before that barrier.
+ * @set_size:	Expected number of requests before that barrier, respectively
+ *		number of requests in the interval [o_block_id;y_block_id]
+ *
+ * Called for both P_BARRIER_ACK and P_CONFIRM_STABLE,
+ * which is similar to an unsolicited partial barrier ack.
+ *
+ * Either barrier_nr (for barrier acks) or both o_block_id and y_blockid (for
+ * confirm stable) are given.  For barrier acks, all requests in the epoch
+ * designated by "barrier_nr" are confirmed to be on stable storage.
+ *
+ * For confirm stable, both o_block_id and y_block_id are given, barrier_nr is
+ * ignored, and all requests from "o_block_id" up to and including y_block_id
+ * are confirmed to be on stable storage on the reporting peer.
  *
  * In case the passed barrier_nr or set_size does not match the oldest
  * epoch of not yet barrier-acked requests, this function will cause a
  * termination of the connection.
  */
-void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
+int tl_release(struct drbd_connection *connection,
+		uint64_t o_block_id,
+		uint64_t y_block_id,
+		unsigned int barrier_nr,
 		unsigned int set_size)
 {
+	struct drbd_resource *resource = connection->resource;
+	const int idx = connection->peer_node_id;
 	struct drbd_request *r;
-	struct drbd_request *req = NULL, *tmp = NULL;
+	struct drbd_request *req = NULL;
+	struct drbd_request *req_y = NULL;
 	int expect_epoch = 0;
 	int expect_size = 0;
 
-	spin_lock_irq(&connection->resource->req_lock);
-
+	rcu_read_lock();
 	/* find oldest not yet barrier-acked write request,
 	 * count writes in its epoch. */
-	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
-		const unsigned s = r->rq_state;
+	r = READ_ONCE(connection->req_not_net_done);
+	if (r == NULL) {
+		drbd_err(connection, "BarrierAck #%u received, but req_not_net_done = NULL\n",
+			 barrier_nr);
+		goto bail;
+	}
+	smp_rmb(); /* paired with smp_wmb() in set_cache_ptr_if_null() */
+	list_for_each_entry_from_rcu(r, &resource->transfer_log, tl_requests) {
+		unsigned int local_rq_state, net_rq_state;
+
+		spin_lock_irq(&r->rq_lock);
+		local_rq_state = r->local_rq_state;
+		net_rq_state = r->net_rq_state[idx];
+		spin_unlock_irq(&r->rq_lock);
+
 		if (!req) {
-			if (!(s & RQ_WRITE))
+			if (!(local_rq_state & RQ_WRITE))
 				continue;
-			if (!(s & RQ_NET_MASK))
+			if (!(net_rq_state & RQ_NET_MASK))
 				continue;
-			if (s & RQ_NET_DONE)
+			if (net_rq_state & RQ_NET_DONE)
 				continue;
 			req = r;
 			expect_epoch = req->epoch;
-			expect_size ++;
+			expect_size++;
 		} else {
+			const u16 s = r->net_rq_state[idx];
 			if (r->epoch != expect_epoch)
 				break;
-			if (!(s & RQ_WRITE))
+			if (!(local_rq_state & RQ_WRITE))
 				continue;
-			/* if (s & RQ_DONE): not expected */
-			/* if (!(s & RQ_NET_MASK)): not expected */
+			/* probably a "send_out_of_sync", during Ahead/Behind mode,
+			 * while at least one volume already started to resync again.
+			 * Or a write that was not replicated during a resync, and
+			 * replication has been enabled since it was submitted.
+			 */
+			if ((s & RQ_NET_MASK) && !(s & RQ_EXP_BARR_ACK))
+				continue;
+			if (s & RQ_NET_DONE || (s & RQ_NET_MASK) == 0) {
+				drbd_warn(connection, "unexpected state flags: 0x%x during BarrierAck #%u\n",
+					s, barrier_nr);
+			}
 			expect_size++;
 		}
+		if (y_block_id && (struct drbd_request *)(unsigned long)y_block_id == r) {
+			req_y = r;
+			break;
+		}
 	}
 
 	/* first some paranoia code */
+	if (o_block_id) {
+		if ((struct drbd_request *)(unsigned long)o_block_id != req) {
+			drbd_err(connection, "BAD! ConfirmedStable: expected %p, found %p\n",
+				(struct drbd_request *)(unsigned long)o_block_id, req);
+			goto bail;
+		}
+		if (!req_y) {
+			drbd_err(connection, "BAD! ConfirmedStable: expected youngest request %p NOT found\n",
+				(struct drbd_req *)(unsigned long)y_block_id);
+			goto bail;
+		}
+		/* A P_CONFIRM_STABLE cannot tell me the to-be-expected barrier nr,
+		 * it does not know it yet. But we just confirmed it knew the
+		 * expected request, so just use that one. */
+		barrier_nr = expect_epoch;
+		/* Both requests referenced must be in the same epoch. */
+		if (req_y->epoch != expect_epoch) {
+			drbd_err(connection, "BAD! ConfirmedStable: reported requests not in the same epoch (%u != %u)\n",
+				req->epoch, req_y->epoch);
+			goto bail;
+		}
+	}
 	if (req == NULL) {
 		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
 			 barrier_nr);
@@ -205,111 +439,135 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
 	}
 
 	if (expect_size != set_size) {
-		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
-			 barrier_nr, set_size, expect_size);
+		if (!o_block_id) {
+			DEFINE_DYNAMIC_DEBUG_METADATA(ddm, "Bad barrier ack dump");
+
+			drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
+				 barrier_nr, set_size, expect_size);
+
+			if (DYNAMIC_DEBUG_BRANCH(ddm))
+				dump_epoch(resource, connection->peer_node_id, expect_epoch);
+		} else
+			drbd_err(connection, "BAD! ConfirmedStable [%p,%p] received with n_writes=%u, expected n_writes=%u!\n",
+				 req, req_y, set_size, expect_size);
 		goto bail;
 	}
 
 	/* Clean up list of requests processed during current epoch. */
-	/* this extra list walk restart is paranoia,
-	 * to catch requests being barrier-acked "unexpectedly".
-	 * It usually should find the same req again, or some READ preceding it. */
-	list_for_each_entry(req, &connection->transfer_log, tl_requests)
-		if (req->epoch == expect_epoch) {
-			tmp = req;
-			break;
-		}
-	req = list_prepare_entry(tmp, &connection->transfer_log, tl_requests);
-	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
+	list_for_each_entry_from_rcu(req, &resource->transfer_log, tl_requests) {
 		struct drbd_peer_device *peer_device;
+
 		if (req->epoch != expect_epoch)
 			break;
 		peer_device = conn_peer_device(connection, req->device->vnr);
-		_req_mod(req, BARRIER_ACKED, peer_device);
+		req_mod(req, BARRIER_ACKED, peer_device);
+		if (req == req_y)
+			break;
+	}
+	rcu_read_unlock();
+
+	/* urgently flush out peer acks for P_CONFIRM_STABLE */
+	if (req_y) {
+		drbd_flush_peer_acks(resource);
+	} else if (barrier_nr == connection->send.last_sent_epoch_nr) {
+		clear_bit(BARRIER_ACK_PENDING, &connection->flags);
+		wake_up(&resource->barrier_wait);
 	}
-	spin_unlock_irq(&connection->resource->req_lock);
 
-	return;
+	return 0;
 
 bail:
-	spin_unlock_irq(&connection->resource->req_lock);
-	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
+	rcu_read_unlock();
+	return -EPROTO;
 }
 
 
 /**
- * _tl_restart() - Walks the transfer log, and applies an action to all requests
- * @connection:	DRBD connection to operate on.
+ * __tl_walk() - Walks the transfer log, and applies an action to all requests
+ * @resource:	DRBD resource to opterate on
+ * @connection: DRBD connection to operate on
+ * @from_req:    If set, the walk starts from the request that this points to
  * @what:       The action/event to perform with all request objects
  *
- * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
- * RESTART_FROZEN_DISK_IO.
+ * @what might be one of CONNECTION_LOST, CONNECTION_LOST_WHILE_SUSPENDED,
+ * RESEND, CANCEL_SUSPENDED_IO, COMPLETION_RESUMED.
  */
-/* must hold resource->req_lock */
-void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
+void __tl_walk(struct drbd_resource *const resource,
+		struct drbd_connection *const connection,
+		struct drbd_request **from_req,
+		const enum drbd_req_event what)
 {
 	struct drbd_peer_device *peer_device;
-	struct drbd_request *req, *r;
+	struct drbd_request *req = NULL;
 
-	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
-		peer_device = conn_peer_device(connection, req->device->vnr);
+	rcu_read_lock();
+	if (from_req)
+		req = READ_ONCE(*from_req);
+	if (!req)
+		req = list_entry_rcu(resource->transfer_log.next, struct drbd_request, tl_requests);
+	smp_rmb(); /* paired with smp_wmb() in set_cache_ptr_if_null() */
+	list_for_each_entry_from_rcu(req, &resource->transfer_log, tl_requests) {
+		/* Skip if the request has already been destroyed. */
+		if (!kref_get_unless_zero(&req->kref))
+			continue;
+
+		peer_device = connection == NULL ? NULL :
+			conn_peer_device(connection, req->device->vnr);
 		_req_mod(req, what, peer_device);
+		kref_put(&req->kref, drbd_req_destroy);
 	}
+	rcu_read_unlock();
 }
 
-void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
+void tl_walk(struct drbd_connection *connection, struct drbd_request **from_req, enum drbd_req_event what)
 {
-	spin_lock_irq(&connection->resource->req_lock);
-	_tl_restart(connection, what);
-	spin_unlock_irq(&connection->resource->req_lock);
-}
+	struct drbd_resource *resource = connection->resource;
 
-/**
- * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
- * @connection:	DRBD connection.
- *
- * This is called after the connection to the peer was lost. The storage covered
- * by the requests on the transfer gets marked as our of sync. Called from the
- * receiver thread and the worker thread.
- */
-void tl_clear(struct drbd_connection *connection)
-{
-	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
+	read_lock_irq(&resource->state_rwlock);
+	__tl_walk(connection->resource, connection, from_req, what);
+	read_unlock_irq(&resource->state_rwlock);
 }
 
 /**
  * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
- * @device:	DRBD device.
+ * @device:     DRBD device.
  */
 void tl_abort_disk_io(struct drbd_device *device)
 {
-	struct drbd_connection *connection = first_peer_device(device)->connection;
-	struct drbd_request *req, *r;
+	struct drbd_resource *resource = device->resource;
+	struct drbd_request *req;
 
-	spin_lock_irq(&connection->resource->req_lock);
-	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
-		if (!(req->rq_state & RQ_LOCAL_PENDING))
+	rcu_read_lock();
+	list_for_each_entry_rcu(req, &resource->transfer_log, tl_requests) {
+		if (!(READ_ONCE(req->local_rq_state) & RQ_LOCAL_PENDING))
 			continue;
 		if (req->device != device)
 			continue;
-		_req_mod(req, ABORT_DISK_IO, NULL);
+		/* Skip if the request has already been destroyed. */
+		if (!kref_get_unless_zero(&req->kref))
+			continue;
+
+		req_mod(req, ABORT_DISK_IO, NULL);
+		kref_put(&req->kref, drbd_req_destroy);
 	}
-	spin_unlock_irq(&connection->resource->req_lock);
+	rcu_read_unlock();
 }
 
 static int drbd_thread_setup(void *arg)
 {
 	struct drbd_thread *thi = (struct drbd_thread *) arg;
 	struct drbd_resource *resource = thi->resource;
+	struct drbd_connection *connection = thi->connection;
 	unsigned long flags;
 	int retval;
 
-	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
-		 thi->name[0],
-		 resource->name);
-
 	allow_kernel_signal(DRBD_SIGKILL);
 	allow_kernel_signal(SIGXCPU);
+
+	if (connection)
+		kref_get(&connection->kref);
+	else
+		kref_get(&resource->kref);
 restart:
 	retval = thi->function(thi);
 
@@ -326,26 +584,33 @@ static int drbd_thread_setup(void *arg)
 	 */
 
 	if (thi->t_state == RESTARTING) {
-		drbd_info(resource, "Restarting %s thread\n", thi->name);
+		if (connection)
+			drbd_info(connection, "Restarting %s thread\n", thi->name);
+		else
+			drbd_info(resource, "Restarting %s thread\n", thi->name);
 		thi->t_state = RUNNING;
 		spin_unlock_irqrestore(&thi->t_lock, flags);
+		flush_signals(current); /* likely it got a signal to look at t_state... */
 		goto restart;
 	}
 
 	thi->task = NULL;
 	thi->t_state = NONE;
 	smp_mb();
-	complete_all(&thi->stop);
-	spin_unlock_irqrestore(&thi->t_lock, flags);
 
-	drbd_info(resource, "Terminating %s\n", current->comm);
+	if (connection)
+		drbd_info(connection, "Terminating %s thread\n", thi->name);
+	else
+		drbd_info(resource, "Terminating %s thread\n", thi->name);
 
-	/* Release mod reference taken when thread was started */
+	complete(&thi->stop);
+	spin_unlock_irqrestore(&thi->t_lock, flags);
+
+	if (connection)
+		kref_put(&connection->kref, drbd_destroy_connection);
+	else
+		kref_put(&resource->kref, drbd_destroy_resource);
 
-	if (thi->connection)
-		kref_put(&thi->connection->kref, drbd_destroy_connection);
-	kref_put(&resource->kref, drbd_destroy_resource);
-	module_put(THIS_MODULE);
 	return retval;
 }
 
@@ -364,6 +629,7 @@ static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread
 int drbd_thread_start(struct drbd_thread *thi)
 {
 	struct drbd_resource *resource = thi->resource;
+	struct drbd_connection *connection = thi->connection;
 	struct task_struct *nt;
 	unsigned long flags;
 
@@ -373,36 +639,29 @@ int drbd_thread_start(struct drbd_thread *thi)
 
 	switch (thi->t_state) {
 	case NONE:
-		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
-			 thi->name, current->comm, current->pid);
-
-		/* Get ref on module for thread - this is released when thread exits */
-		if (!try_module_get(THIS_MODULE)) {
-			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
-			spin_unlock_irqrestore(&thi->t_lock, flags);
-			return false;
-		}
-
-		kref_get(&resource->kref);
-		if (thi->connection)
-			kref_get(&thi->connection->kref);
+		if (connection)
+			drbd_info(connection, "Starting %s thread (peer-node-id %d)\n",
+				 thi->name, connection->peer_node_id);
+		else
+			drbd_info(resource, "Starting %s thread (node-id %d)\n",
+				 thi->name, resource->res_opts.node_id);
 
 		init_completion(&thi->stop);
+		D_ASSERT(resource, thi->task == NULL);
 		thi->reset_cpu_mask = 1;
 		thi->t_state = RUNNING;
 		spin_unlock_irqrestore(&thi->t_lock, flags);
 		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */
 
 		nt = kthread_create(drbd_thread_setup, (void *) thi,
-				    "drbd_%c_%s", thi->name[0], thi->resource->name);
+				    "drbd_%c_%s", thi->name[0], resource->name);
 
 		if (IS_ERR(nt)) {
-			drbd_err(resource, "Couldn't start thread\n");
+			if (connection)
+				drbd_err(connection, "Couldn't start thread: %ld\n", PTR_ERR(nt));
+			else
+				drbd_err(resource, "Couldn't start thread: %ld\n", PTR_ERR(nt));
 
-			if (thi->connection)
-				kref_put(&thi->connection->kref, drbd_destroy_connection);
-			kref_put(&resource->kref, drbd_destroy_resource);
-			module_put(THIS_MODULE);
 			return false;
 		}
 		spin_lock_irqsave(&thi->t_lock, flags);
@@ -413,8 +672,10 @@ int drbd_thread_start(struct drbd_thread *thi)
 		break;
 	case EXITING:
 		thi->t_state = RESTARTING;
-		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
-				thi->name, current->comm, current->pid);
+		if (connection)
+			drbd_info(connection, "Restarting %s thread\n", thi->name);
+		else
+			drbd_info(resource, "Restarting %s thread\n", thi->name);
 		fallthrough;
 	case RUNNING:
 	case RESTARTING:
@@ -443,6 +704,12 @@ void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
 		return;
 	}
 
+	if (thi->t_state == EXITING && ns == RESTARTING) {
+		/* Do not abort a stop request, otherwise a waiter might never wake up */
+		spin_unlock_irqrestore(&thi->t_lock, flags);
+		return;
+	}
+
 	if (thi->t_state != ns) {
 		if (thi->task == NULL) {
 			spin_unlock_irqrestore(&thi->t_lock, flags);
@@ -455,7 +722,6 @@ void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
 		if (thi->task != current)
 			send_sig(DRBD_SIGKILL, thi->task, 1);
 	}
-
 	spin_unlock_irqrestore(&thi->t_lock, flags);
 
 	if (wait)
@@ -473,8 +739,7 @@ static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
 {
 	unsigned int *resources_per_cpu, min_index = ~0;
 
-	resources_per_cpu = kcalloc(nr_cpu_ids, sizeof(*resources_per_cpu),
-				    GFP_KERNEL);
+	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
 	if (resources_per_cpu) {
 		struct drbd_resource *resource;
 		unsigned int cpu, min = ~0;
@@ -521,6 +786,46 @@ void drbd_thread_current_set_cpu(struct drbd_thread *thi)
 #define drbd_calc_cpu_mask(A) ({})
 #endif
 
+static bool drbd_all_neighbor_secondary(struct drbd_device *device, u64 *authoritative_ptr)
+{
+	struct drbd_peer_device *peer_device;
+	bool all_secondary = true;
+	u64 authoritative = 0;
+	int id;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED &&
+		    peer_device->connection->peer_role[NOW] == R_PRIMARY) {
+			all_secondary = false;
+			id = peer_device->node_id;
+			authoritative |= NODE_MASK(id);
+		}
+	}
+	rcu_read_unlock();
+	if (authoritative_ptr)
+		*authoritative_ptr = authoritative;
+	return all_secondary;
+}
+
+/* This function is supposed to have the same semantics as calc_device_stable() in drbd_state.c
+   A primary is stable since it is authoritative.
+   Unstable are neighbors of a primary and resync target nodes.
+   Nodes further away from a primary are stable! */
+bool drbd_device_stable(struct drbd_device *device, u64 *authoritative_ptr)
+{
+	struct drbd_resource *resource = device->resource;
+	bool device_stable = true;
+
+	if (resource->role[NOW] == R_PRIMARY)
+		return true;
+
+	if (!drbd_all_neighbor_secondary(device, authoritative_ptr))
+		return false;
+
+	return device_stable;
+}
+
 /*
  * drbd_header_size  -  size of a packet header
  *
@@ -532,177 +837,358 @@ unsigned int drbd_header_size(struct drbd_connection *connection)
 {
 	if (connection->agreed_pro_version >= 100) {
 		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
-		return sizeof(struct p_header100);
+		return sizeof(struct p_header100); /* 16 */
 	} else {
 		BUILD_BUG_ON(sizeof(struct p_header80) !=
 			     sizeof(struct p_header95));
 		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
-		return sizeof(struct p_header80);
+		return sizeof(struct p_header80); /* 8 */
 	}
 }
 
-static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
+static void prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
 {
 	h->magic   = cpu_to_be32(DRBD_MAGIC);
 	h->command = cpu_to_be16(cmd);
-	h->length  = cpu_to_be16(size);
-	return sizeof(struct p_header80);
+	h->length  = cpu_to_be16(size - sizeof(struct p_header80));
 }
 
-static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
+static void prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
 {
 	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
 	h->command = cpu_to_be16(cmd);
-	h->length = cpu_to_be32(size);
-	return sizeof(struct p_header95);
+	h->length = cpu_to_be32(size - sizeof(struct p_header95));
 }
 
-static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
+static void prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
 				      int size, int vnr)
 {
 	h->magic = cpu_to_be32(DRBD_MAGIC_100);
 	h->volume = cpu_to_be16(vnr);
 	h->command = cpu_to_be16(cmd);
-	h->length = cpu_to_be32(size);
+	h->length = cpu_to_be32(size - sizeof(struct p_header100));
 	h->pad = 0;
-	return sizeof(struct p_header100);
 }
 
-static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
-				   void *buffer, enum drbd_packet cmd, int size)
+static void prepare_header(struct drbd_connection *connection, int vnr,
+			   void *buffer, enum drbd_packet cmd, int size)
 {
 	if (connection->agreed_pro_version >= 100)
-		return prepare_header100(buffer, cmd, size, vnr);
+		prepare_header100(buffer, cmd, size, vnr);
 	else if (connection->agreed_pro_version >= 95 &&
 		 size > DRBD_MAX_SIZE_H80_PACKET)
-		return prepare_header95(buffer, cmd, size);
+		prepare_header95(buffer, cmd, size);
 	else
-		return prepare_header80(buffer, cmd, size);
+		prepare_header80(buffer, cmd, size);
+}
+
+static void new_or_recycle_send_buffer_page(struct drbd_send_buffer *sbuf)
+{
+	while (1) {
+		struct page *page;
+		int count = page_count(sbuf->page);
+
+		BUG_ON(count == 0);
+		if (count == 1)
+			goto have_page;
+
+		page = alloc_page(GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
+		if (page) {
+			put_page(sbuf->page);
+			sbuf->page = page;
+			goto have_page;
+		}
+
+		schedule_timeout_uninterruptible(HZ / 10);
+	}
+have_page:
+	sbuf->unsent =
+	sbuf->pos = page_address(sbuf->page);
+}
+
+static char *alloc_send_buffer(struct drbd_connection *connection, int size,
+			       enum drbd_stream drbd_stream)
+{
+	struct drbd_send_buffer *sbuf = &connection->send_buffer[drbd_stream];
+	char *page_start = page_address(sbuf->page);
+
+	if (sbuf->pos - page_start + size > PAGE_SIZE) {
+		flush_send_buffer(connection, drbd_stream);
+		new_or_recycle_send_buffer_page(sbuf);
+	}
+
+	sbuf->allocated_size = size;
+	sbuf->additional_size = 0;
+
+	return sbuf->pos;
+}
+
+/* If we called alloc_send_buffer(), possibly indirectly via __conn_prepare_command(),
+ * but then decide that we actually don't want to use it.
+ */
+static void cancel_send_buffer(struct drbd_connection *connection,
+		enum drbd_stream drbd_stream)
+{
+	connection->send_buffer[drbd_stream].allocated_size = 0;
+}
+
+/* Only used the shrink the previously allocated size. */
+static void resize_prepared_command(struct drbd_connection *connection,
+				    enum drbd_stream drbd_stream,
+				    int size)
+{
+	connection->send_buffer[drbd_stream].allocated_size =
+		size + drbd_header_size(connection);
+}
+
+static void additional_size_command(struct drbd_connection *connection,
+				    enum drbd_stream drbd_stream,
+				    int additional_size)
+{
+	connection->send_buffer[drbd_stream].additional_size = additional_size;
 }
 
-static void *__conn_prepare_command(struct drbd_connection *connection,
-				    struct drbd_socket *sock)
+void *__conn_prepare_command(struct drbd_connection *connection, int size,
+				    enum drbd_stream drbd_stream)
 {
-	if (!sock->socket)
+	struct drbd_transport *transport = &connection->transport;
+	int header_size;
+
+	if (connection->cstate[NOW] < C_CONNECTING)
+		return NULL;
+
+	if (!transport->class->ops.stream_ok(transport, drbd_stream))
 		return NULL;
-	return sock->sbuf + drbd_header_size(connection);
+
+	header_size = drbd_header_size(connection);
+	return alloc_send_buffer(connection, header_size + size, drbd_stream) + header_size;
 }
 
-void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
+/**
+ * conn_prepare_command() - Allocate a send buffer for a packet/command
+ * @connection: the connections the packet will be sent through
+ * @size:	number of bytes to allocate
+ * @drbd_stream: DATA_STREAM or CONTROL_STREAM
+ *
+ * This allocates a buffer with capacity to hold the header, and
+ * the requested size. Upon success is return a pointer that points
+ * to the first byte behind the header. The caller is expected to
+ * call xxx_send_command() soon.
+ */
+void *conn_prepare_command(struct drbd_connection *connection, int size,
+			   enum drbd_stream drbd_stream)
 {
 	void *p;
 
-	mutex_lock(&sock->mutex);
-	p = __conn_prepare_command(connection, sock);
+	mutex_lock(&connection->mutex[drbd_stream]);
+	p = __conn_prepare_command(connection, size, drbd_stream);
 	if (!p)
-		mutex_unlock(&sock->mutex);
+		mutex_unlock(&connection->mutex[drbd_stream]);
 
 	return p;
 }
 
-void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
+/**
+ * drbd_prepare_command() - Allocate a send buffer for a packet/command
+ * @peer_device: the DRBD peer device the packet will be sent to
+ * @size: number of bytes to allocate
+ * @drbd_stream: DATA_STREAM or CONTROL_STREAM
+ *
+ * This allocates a buffer with capacity to hold the header, and
+ * the requested size. Upon success is return a pointer that points
+ * to the first byte behind the header. The caller is expected to
+ * call xxx_send_command() soon.
+ */
+void *drbd_prepare_command(struct drbd_peer_device *peer_device, int size, enum drbd_stream drbd_stream)
 {
-	return conn_prepare_command(peer_device->connection, sock);
+	return conn_prepare_command(peer_device->connection, size, drbd_stream);
 }
 
-static int __send_command(struct drbd_connection *connection, int vnr,
-			  struct drbd_socket *sock, enum drbd_packet cmd,
-			  unsigned int header_size, void *data,
-			  unsigned int size)
+static int flush_send_buffer(struct drbd_connection *connection, enum drbd_stream drbd_stream)
 {
-	int msg_flags;
+	struct drbd_send_buffer *sbuf = &connection->send_buffer[drbd_stream];
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+	unsigned int flags, offset, size;
 	int err;
 
-	/*
-	 * Called with @data == NULL and the size of the data blocks in @size
-	 * for commands that send data blocks.  For those commands, omit the
-	 * MSG_MORE flag: this will increase the likelihood that data blocks
-	 * which are page aligned on the sender will end up page aligned on the
-	 * receiver.
-	 */
-	msg_flags = data ? MSG_MORE : 0;
-
-	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
-				      header_size + size);
-	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
-			    msg_flags);
-	if (data && !err)
-		err = drbd_send_all(connection, sock->socket, data, size, 0);
-	/* DRBD protocol "pings" are latency critical.
-	 * This is supposed to trigger tcp_push_pending_frames() */
-	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
-		tcp_sock_set_nodelay(sock->socket->sk);
+	size = sbuf->pos - sbuf->unsent + sbuf->allocated_size;
+	if (size == 0)
+		return 0;
+
+	if (drbd_stream == CONTROL_STREAM) {
+		connection->ctl_packets++;
+		if (check_add_overflow(connection->ctl_bytes, size, &connection->ctl_bytes)) {
+			connection->ctl_bytes = size;
+			connection->ctl_packets = 1;
+		}
+	}
+
+	if (drbd_stream == DATA_STREAM) {
+		rcu_read_lock();
+		connection->transport.ko_count = rcu_dereference(connection->transport.net_conf)->ko_count;
+		rcu_read_unlock();
+	}
+
+	flags = (connection->cstate[NOW] < C_CONNECTING ? MSG_DONTWAIT : 0) |
+		(sbuf->additional_size ? MSG_MORE : 0);
+	offset = sbuf->unsent - (char *)page_address(sbuf->page);
+	err = tr_ops->send_page(transport, drbd_stream, sbuf->page, offset, size, flags);
+	if (!err) {
+		sbuf->unsent =
+		sbuf->pos += sbuf->allocated_size;      /* send buffer submitted! */
+	}
+
+	sbuf->allocated_size = 0;
 
 	return err;
 }
 
-static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
-			       enum drbd_packet cmd, unsigned int header_size,
-			       void *data, unsigned int size)
+/*
+ * SFLAG_FLUSH makes sure the packet (and everything queued in front
+ * of it) gets sent immediately independently if it is currently
+ * corked.
+ *
+ * This is used for P_PING, P_PING_ACK, P_TWOPC_PREPARE, P_TWOPC_ABORT,
+ * P_TWOPC_YES, P_TWOPC_NO, P_TWOPC_RETRY and P_TWOPC_COMMIT.
+ *
+ * This quirk is necessary because it is corked while the worker
+ * thread processes work items. When it stops processing items, it
+ * uncorks. That works perfectly to coalesce ack packets etc..
+ * A work item doing two-phase commits needs to override that behavior.
+ */
+#define SFLAG_FLUSH 0x10
+#define DRBD_STREAM_FLAGS (SFLAG_FLUSH)
+
+static inline enum drbd_stream extract_stream(int stream_and_flags)
 {
-	return __send_command(connection, 0, sock, cmd, header_size, data, size);
+	return stream_and_flags & ~DRBD_STREAM_FLAGS;
 }
 
-int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
-		      enum drbd_packet cmd, unsigned int header_size,
-		      void *data, unsigned int size)
+int __send_command(struct drbd_connection *connection, int vnr,
+		   enum drbd_packet cmd, int stream_and_flags)
 {
+	enum drbd_stream drbd_stream = extract_stream(stream_and_flags);
+	struct drbd_send_buffer *sbuf = &connection->send_buffer[drbd_stream];
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+	/* CORKED + drbd_stream is either DATA_CORKED or CONTROL_CORKED */
+	bool corked = test_bit(CORKED + drbd_stream, &connection->flags);
+	bool flush = stream_and_flags & SFLAG_FLUSH;
 	int err;
 
-	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
-	mutex_unlock(&sock->mutex);
+	if (connection->cstate[NOW] < C_CONNECTING)
+		return -EIO;
+	prepare_header(connection, vnr, sbuf->pos, cmd,
+		       sbuf->allocated_size + sbuf->additional_size);
+
+	if (corked && !flush) {
+		sbuf->pos += sbuf->allocated_size;
+		sbuf->allocated_size = 0;
+		err = 0;
+	} else {
+		err = flush_send_buffer(connection, drbd_stream);
+
+		/* DRBD protocol "pings" are latency critical.
+		 * This is supposed to trigger tcp_push_pending_frames() */
+		if (!err && flush)
+			tr_ops->hint(transport, drbd_stream, NODELAY);
+
+	}
+
 	return err;
 }
 
-int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
-		      enum drbd_packet cmd, unsigned int header_size,
-		      void *data, unsigned int size)
+void drbd_cork(struct drbd_connection *connection, enum drbd_stream stream)
+{
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+
+	mutex_lock(&connection->mutex[stream]);
+	set_bit(CORKED + stream, &connection->flags);
+	/* only call into transport, if we expect it to work */
+	if (connection->cstate[NOW] >= C_CONNECTING)
+		tr_ops->hint(transport, stream, CORK);
+	mutex_unlock(&connection->mutex[stream]);
+}
+
+void drbd_uncork(struct drbd_connection *connection, enum drbd_stream stream)
+{
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+
+	mutex_lock(&connection->mutex[stream]);
+	flush_send_buffer(connection, stream);
+
+	clear_bit(CORKED + stream, &connection->flags);
+	/* only call into transport, if we expect it to work */
+	if (connection->cstate[NOW] >= C_CONNECTING)
+		tr_ops->hint(transport, stream, UNCORK);
+	mutex_unlock(&connection->mutex[stream]);
+}
+
+int send_command(struct drbd_connection *connection, int vnr,
+		 enum drbd_packet cmd, int stream_and_flags)
 {
+	enum drbd_stream drbd_stream = extract_stream(stream_and_flags);
 	int err;
 
-	err = __send_command(peer_device->connection, peer_device->device->vnr,
-			     sock, cmd, header_size, data, size);
-	mutex_unlock(&sock->mutex);
+	err = __send_command(connection, vnr, cmd, stream_and_flags);
+	mutex_unlock(&connection->mutex[drbd_stream]);
 	return err;
 }
 
-int drbd_send_ping(struct drbd_connection *connection)
+int drbd_send_command(struct drbd_peer_device *peer_device,
+		      enum drbd_packet cmd, enum drbd_stream drbd_stream)
 {
-	struct drbd_socket *sock;
+	return send_command(peer_device->connection, peer_device->device->vnr,
+			    cmd, drbd_stream);
+}
 
-	sock = &connection->meta;
-	if (!conn_prepare_command(connection, sock))
+int drbd_send_ping(struct drbd_connection *connection)
+{
+	if (!conn_prepare_command(connection, 0, CONTROL_STREAM))
 		return -EIO;
-	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
+	return send_command(connection, -1, P_PING, CONTROL_STREAM | SFLAG_FLUSH);
+}
+
+void drbd_send_ping_ack_wf(struct work_struct *ws)
+{
+	struct drbd_connection *connection =
+		container_of(ws, struct drbd_connection, send_ping_ack_work);
+	int err;
+
+	err = conn_prepare_command(connection, 0, CONTROL_STREAM) ? 0 : -EIO;
+	if (!err)
+		err = send_command(connection, -1, P_PING_ACK, CONTROL_STREAM | SFLAG_FLUSH);
+	if (err)
+		change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
 }
 
-int drbd_send_ping_ack(struct drbd_connection *connection)
+int drbd_send_peer_ack(struct drbd_connection *connection, u64 mask, u64 dagtag_sector)
 {
-	struct drbd_socket *sock;
+	struct p_peer_ack *p;
 
-	sock = &connection->meta;
-	if (!conn_prepare_command(connection, sock))
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
+	if (!p)
 		return -EIO;
-	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
+	p->mask = cpu_to_be64(mask);
+	p->dagtag = cpu_to_be64(dagtag_sector);
+
+	return send_command(connection, -1, P_PEER_ACK, CONTROL_STREAM);
 }
 
 int drbd_send_sync_param(struct drbd_peer_device *peer_device)
 {
-	struct drbd_socket *sock;
 	struct p_rs_param_95 *p;
 	int size;
 	const int apv = peer_device->connection->agreed_pro_version;
 	enum drbd_packet cmd;
 	struct net_conf *nc;
-	struct disk_conf *dc;
-
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
+	struct peer_device_conf *pdc;
 
 	rcu_read_lock();
-	nc = rcu_dereference(peer_device->connection->net_conf);
+	nc = rcu_dereference(peer_device->connection->transport.net_conf);
 
 	size = apv <= 87 ? sizeof(struct p_rs_param)
 		: apv == 88 ? sizeof(struct p_rs_param)
@@ -711,18 +1197,30 @@ int drbd_send_sync_param(struct drbd_peer_device *peer_device)
 		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
 
 	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
+	rcu_read_unlock();
+
+	p = drbd_prepare_command(peer_device, size, DATA_STREAM);
+	if (!p)
+		return -EIO;
 
 	/* initialize verify_alg and csums_alg */
-	BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
-	memset(&p->algs, 0, sizeof(p->algs));
+	memset(p->verify_alg, 0, sizeof(p->verify_alg));
+	memset(p->csums_alg, 0, sizeof(p->csums_alg));
+
+	rcu_read_lock();
+	nc = rcu_dereference(peer_device->connection->transport.net_conf);
 
 	if (get_ldev(peer_device->device)) {
-		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
-		p->resync_rate = cpu_to_be32(dc->resync_rate);
-		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
-		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
-		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
-		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
+		pdc = rcu_dereference(peer_device->conf);
+		/* These values will be ignored by peers running DRBD 9.2+, but
+		 * we have to send something, so send the real values. We
+		 * cannot omit the entire packet because we must verify that
+		 * the algorithms match. */
+		p->resync_rate = cpu_to_be32(pdc->resync_rate);
+		p->c_plan_ahead = cpu_to_be32(pdc->c_plan_ahead);
+		p->c_delay_target = cpu_to_be32(pdc->c_delay_target);
+		p->c_fill_target = cpu_to_be32(pdc->c_fill_target);
+		p->c_max_rate = cpu_to_be32(pdc->c_max_rate);
 		put_ldev(peer_device->device);
 	} else {
 		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
@@ -738,36 +1236,37 @@ int drbd_send_sync_param(struct drbd_peer_device *peer_device)
 		strscpy(p->csums_alg, nc->csums_alg);
 	rcu_read_unlock();
 
-	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
+	return drbd_send_command(peer_device, cmd, DATA_STREAM);
 }
 
 int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
 {
-	struct drbd_socket *sock;
 	struct p_protocol *p;
 	struct net_conf *nc;
 	size_t integrity_alg_len;
 	int size, cf;
 
-	sock = &connection->data;
-	p = __conn_prepare_command(connection, sock);
-	if (!p)
-		return -EIO;
-
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-
-	if (nc->tentative && connection->agreed_pro_version < 92) {
-		rcu_read_unlock();
+	if (test_bit(CONN_DRY_RUN, &connection->flags) && connection->agreed_pro_version < 92) {
+		clear_bit(CONN_DRY_RUN, &connection->flags);
 		drbd_err(connection, "--dry-run is not supported by peer");
 		return -EOPNOTSUPP;
 	}
 
 	size = sizeof(*p);
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
 	if (connection->agreed_pro_version >= 87) {
 		integrity_alg_len = strlen(nc->integrity_alg) + 1;
 		size += integrity_alg_len;
 	}
+	rcu_read_unlock();
+
+	p = __conn_prepare_command(connection, size, DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
 
 	p->protocol      = cpu_to_be32(nc->wire_protocol);
 	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
@@ -775,9 +1274,9 @@ int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cm
 	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
 	p->two_primaries = cpu_to_be32(nc->two_primaries);
 	cf = 0;
-	if (nc->discard_my_data)
+	if (test_bit(CONN_DISCARD_MY_DATA, &connection->flags))
 		cf |= CF_DISCARD_MY_DATA;
-	if (nc->tentative)
+	if (test_bit(CONN_DRY_RUN, &connection->flags))
 		cf |= CF_DRY_RUN;
 	p->conn_flags    = cpu_to_be32(cf);
 
@@ -785,133 +1284,301 @@ int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cm
 		strscpy(p->integrity_alg, nc->integrity_alg, integrity_alg_len);
 	rcu_read_unlock();
 
-	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
-}
-
-int drbd_send_protocol(struct drbd_connection *connection)
-{
-	int err;
-
-	mutex_lock(&connection->data.mutex);
-	err = __drbd_send_protocol(connection, P_PROTOCOL);
-	mutex_unlock(&connection->data.mutex);
-
-	return err;
+	return __send_command(connection, -1, cmd, DATA_STREAM);
 }
 
 static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
 {
 	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock;
 	struct p_uuids *p;
 	int i;
 
 	if (!get_ldev_if_state(device, D_NEGOTIATING))
 		return 0;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
 	if (!p) {
 		put_ldev(device);
 		return -EIO;
 	}
+
 	spin_lock_irq(&device->ldev->md.uuid_lock);
-	for (i = UI_CURRENT; i < UI_SIZE; i++)
-		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
+	p->current_uuid = cpu_to_be64(drbd_current_uuid(device));
+	p->bitmap_uuid = cpu_to_be64(drbd_bitmap_uuid(peer_device));
+	for (i = 0; i < ARRAY_SIZE(p->history_uuids); i++)
+		p->history_uuids[i] = cpu_to_be64(drbd_history_uuid(device, i));
 	spin_unlock_irq(&device->ldev->md.uuid_lock);
 
-	device->comm_bm_set = drbd_bm_total_weight(device);
-	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
+	peer_device->comm_bm_set = drbd_bm_total_weight(peer_device);
+	p->dirty_bits = cpu_to_be64(peer_device->comm_bm_set);
+
+	if (test_bit(DISCARD_MY_DATA, &peer_device->flags))
+		uuid_flags |= UUID_FLAG_DISCARD_MY_DATA;
+	if (test_bit(CRASHED_PRIMARY, &device->flags))
+		uuid_flags |= UUID_FLAG_CRASHED_PRIMARY;
+	if (!drbd_md_test_flag(device->ldev, MDF_CONSISTENT))
+		uuid_flags |= UUID_FLAG_INCONSISTENT;
+
+	/* Silently mask out any "too recent" flags,
+	 * we cannot communicate those in old DRBD
+	 * protocol versions. */
+	uuid_flags &= UUID_FLAG_MASK_COMPAT_84;
+
+	peer_device->comm_uuid_flags = uuid_flags;
+	p->uuid_flags = cpu_to_be64(uuid_flags);
+
+	put_ldev(device);
+
+	return drbd_send_command(peer_device, P_UUIDS, DATA_STREAM);
+}
+
+static u64 __bitmap_uuid(struct drbd_device *device, int node_id) __must_hold(local)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	u64 bitmap_uuid = peer_md[node_id].bitmap_uuid;
+
+	/* Sending a bitmap_uuid of 0 means that we are in sync with that peer.
+	   The recipient of this message might use this assumption to throw away it's
+	   bitmap to that peer.
+
+	   Send -1 instead if we are (resync target from that peer) not at the same
+	   current uuid.
+	   This corner case is relevant if we finish resync from an UpToDate peer first,
+	   and the second resync (which was paused first) is from an Outdated node.
+	   And that second resync gets canceled by the resync target due to the first
+	   resync finished successfully.
+
+	   Exceptions to the above are when the peer's UUID is not known yet
+	 */
+
 	rcu_read_lock();
-	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
+	peer_device = peer_device_by_node_id(device, node_id);
+	if (peer_device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+		if (bitmap_uuid == 0 &&
+		    (repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) &&
+		    peer_device->current_uuid != 0 &&
+		    (peer_device->current_uuid & ~UUID_PRIMARY) !=
+		    (drbd_current_uuid(device) & ~UUID_PRIMARY))
+			bitmap_uuid = -1;
+	}
 	rcu_read_unlock();
-	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
-	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
-	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
 
-	put_ldev(device);
-	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
+	return bitmap_uuid;
+}
+
+u64 drbd_collect_local_uuid_flags(struct drbd_peer_device *peer_device, u64 *authoritative_mask)
+{
+	struct drbd_device *device = peer_device->device;
+	u64 uuid_flags = 0;
+
+	if (test_bit(DISCARD_MY_DATA, &peer_device->flags))
+		uuid_flags |= UUID_FLAG_DISCARD_MY_DATA;
+	if (test_bit(CRASHED_PRIMARY, &device->flags))
+		uuid_flags |= UUID_FLAG_CRASHED_PRIMARY;
+	if (!drbd_md_test_flag(device->ldev, MDF_CONSISTENT))
+		uuid_flags |= UUID_FLAG_INCONSISTENT;
+	if (test_bit(RECONNECT, &peer_device->connection->flags))
+		uuid_flags |= UUID_FLAG_RECONNECT;
+	if (test_bit(PRIMARY_LOST_QUORUM, &device->flags))
+		uuid_flags |= UUID_FLAG_PRIMARY_LOST_QUORUM;
+	if (drbd_device_stable(device, authoritative_mask))
+		uuid_flags |= UUID_FLAG_STABLE;
+
+	return uuid_flags;
+}
+
+/* sets UUID_FLAG_SYNC_TARGET on uuid_flags as appropriate (may be NULL) */
+u64 drbd_resolved_uuid(struct drbd_peer_device *peer_device_base, u64 *uuid_flags)
+{
+	struct drbd_device *device = peer_device_base->device;
+	struct drbd_peer_device *peer_device;
+	u64 uuid = drbd_current_uuid(device);
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->node_id == peer_device_base->node_id)
+			continue;
+		if (peer_device->repl_state[NOW] == L_SYNC_TARGET) {
+			uuid = peer_device->current_uuid;
+			if (uuid_flags)
+				*uuid_flags |= UUID_FLAG_SYNC_TARGET;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return uuid;
 }
 
-int drbd_send_uuids(struct drbd_peer_device *peer_device)
+static int _drbd_send_uuids110(struct drbd_peer_device *peer_device, u64 uuid_flags, u64 node_mask)
 {
-	return _drbd_send_uuids(peer_device, 0);
+	struct drbd_device *device = peer_device->device;
+	const int my_node_id = device->resource->res_opts.node_id;
+	struct drbd_peer_md *peer_md;
+	struct p_uuids110 *p;
+	bool sent_one_unallocated;
+	int i, pos = 0;
+	u64 local_uuid_flags = 0, authoritative_mask, bitmap_uuids_mask = 0;
+	int p_size = sizeof(*p);
+
+	if (!get_ldev_if_state(device, D_NEGOTIATING))
+		return drbd_send_current_uuid(peer_device, device->exposed_data_uuid,
+					      drbd_weak_nodes_device(device));
+
+	peer_md = device->ldev->md.peers;
+
+	p_size += (DRBD_PEERS_MAX + HISTORY_UUIDS) * sizeof(p->other_uuids[0]);
+	p = drbd_prepare_command(peer_device, p_size, DATA_STREAM);
+	if (!p) {
+		put_ldev(device);
+		return -EIO;
+	}
+
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+	peer_device->comm_current_uuid = drbd_resolved_uuid(peer_device, &local_uuid_flags);
+	p->current_uuid = cpu_to_be64(peer_device->comm_current_uuid);
+
+	sent_one_unallocated = peer_device->connection->agreed_pro_version < 116;
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		u64 val = __bitmap_uuid(device, i);
+		bool send_this = peer_md[i].flags & (MDF_HAVE_BITMAP | MDF_NODE_EXISTS);
+		if (!send_this && !sent_one_unallocated &&
+		    i != my_node_id && i != peer_device->node_id && val) {
+			send_this = true;
+			sent_one_unallocated = true;
+			uuid_flags |= (u64)i << UUID_FLAG_UNALLOC_SHIFT;
+			uuid_flags |= UUID_FLAG_HAS_UNALLOC;
+		}
+		if (send_this) {
+			bitmap_uuids_mask |= NODE_MASK(i);
+			p->other_uuids[pos++] = cpu_to_be64(val);
+		}
+	}
+	peer_device->comm_bitmap_uuid = drbd_bitmap_uuid(peer_device);
+
+	for (i = 0; i < HISTORY_UUIDS; i++)
+		p->other_uuids[pos++] = cpu_to_be64(drbd_history_uuid(device, i));
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
+
+	p->bitmap_uuids_mask = cpu_to_be64(bitmap_uuids_mask);
+
+	peer_device->comm_bm_set = drbd_bm_total_weight(peer_device);
+	p->dirty_bits = cpu_to_be64(peer_device->comm_bm_set);
+	local_uuid_flags |= drbd_collect_local_uuid_flags(peer_device, &authoritative_mask);
+	peer_device->comm_uuid_flags = local_uuid_flags;
+	uuid_flags |= local_uuid_flags;
+	if (uuid_flags & UUID_FLAG_STABLE) {
+		p->node_mask = cpu_to_be64(node_mask);
+	} else {
+		D_ASSERT(peer_device, node_mask == 0);
+		p->node_mask = cpu_to_be64(authoritative_mask);
+	}
+
+	p->uuid_flags = cpu_to_be64(uuid_flags);
+
+	put_ldev(device);
+
+	p_size = sizeof(*p) +
+		(hweight64(bitmap_uuids_mask) + HISTORY_UUIDS) * sizeof(p->other_uuids[0]);
+	resize_prepared_command(peer_device->connection, DATA_STREAM, p_size);
+	return drbd_send_command(peer_device, P_UUIDS110, DATA_STREAM);
 }
 
-int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
+int drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags, u64 node_mask)
 {
-	return _drbd_send_uuids(peer_device, 8);
+	if (peer_device->connection->agreed_pro_version >= 110)
+		return _drbd_send_uuids110(peer_device, uuid_flags, node_mask);
+	else
+		return _drbd_send_uuids(peer_device, uuid_flags);
 }
 
-void drbd_print_uuids(struct drbd_device *device, const char *text)
+void drbd_print_uuids(struct drbd_peer_device *peer_device, const char *text)
 {
+	struct drbd_device *device = peer_device->device;
+
 	if (get_ldev_if_state(device, D_NEGOTIATING)) {
-		u64 *uuid = device->ldev->md.uuid;
-		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
-		     text,
-		     (unsigned long long)uuid[UI_CURRENT],
-		     (unsigned long long)uuid[UI_BITMAP],
-		     (unsigned long long)uuid[UI_HISTORY_START],
-		     (unsigned long long)uuid[UI_HISTORY_END]);
+		drbd_info(peer_device, "%s %016llX:%016llX:%016llX:%016llX\n",
+			  text,
+			  (unsigned long long)drbd_current_uuid(device),
+			  (unsigned long long)drbd_bitmap_uuid(peer_device),
+			  (unsigned long long)drbd_history_uuid(device, 0),
+			  (unsigned long long)drbd_history_uuid(device, 1));
 		put_ldev(device);
 	} else {
-		drbd_info(device, "%s effective data uuid: %016llX\n",
-				text,
-				(unsigned long long)device->ed_uuid);
+		drbd_info(peer_device, "%s exposed data uuid: %016llX\n",
+			  text,
+			  (unsigned long long)device->exposed_data_uuid);
 	}
 }
 
+int drbd_send_current_uuid(struct drbd_peer_device *peer_device, u64 current_uuid, u64 weak_nodes)
+{
+	struct p_current_uuid *p;
+
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	peer_device->comm_current_uuid = current_uuid;
+	p->uuid = cpu_to_be64(current_uuid);
+	p->weak_nodes = cpu_to_be64(weak_nodes);
+	return drbd_send_command(peer_device, P_CURRENT_UUID, DATA_STREAM);
+}
+
 void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
 {
 	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock;
-	struct p_rs_uuid *p;
+	struct p_uuid *p;
 	u64 uuid;
 
-	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
+	D_ASSERT(device, device->disk_state[NOW] == D_UP_TO_DATE);
 
-	uuid = device->ldev->md.uuid[UI_BITMAP];
+	down_write(&device->uuid_sem);
+	uuid = drbd_bitmap_uuid(peer_device);
 	if (uuid && uuid != UUID_JUST_CREATED)
 		uuid = uuid + UUID_NEW_BM_OFFSET;
 	else
 		get_random_bytes(&uuid, sizeof(u64));
-	drbd_uuid_set(device, UI_BITMAP, uuid);
-	drbd_print_uuids(device, "updated sync UUID");
+	drbd_uuid_set_bitmap(peer_device, uuid);
+	drbd_print_uuids(peer_device, "updated sync UUID");
 	drbd_md_sync(device);
+	downgrade_write(&device->uuid_sem);
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
 	if (p) {
 		p->uuid = cpu_to_be64(uuid);
-		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
+		drbd_send_command(peer_device, P_SYNC_UUID, DATA_STREAM);
 	}
+	up_read(&device->uuid_sem);
 }
 
-int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
+int drbd_send_sizes(struct drbd_peer_device *peer_device,
+		    uint64_t u_size_diskless, enum dds_flags flags)
 {
+	struct drbd_connection *connection = peer_device->connection;
 	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock;
 	struct p_sizes *p;
 	sector_t d_size, u_size;
 	int q_order_type;
 	unsigned int max_bio_size;
 	unsigned int packet_size;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
-
 	packet_size = sizeof(*p);
-	if (peer_device->connection->agreed_features & DRBD_FF_WSAME)
+	if (connection->agreed_features & DRBD_FF_WSAME)
 		packet_size += sizeof(p->qlim[0]);
 
+	p = drbd_prepare_command(peer_device, packet_size, DATA_STREAM);
+	if (!p)
+		return -EIO;
+
 	memset(p, 0, packet_size);
 	if (get_ldev_if_state(device, D_NEGOTIATING)) {
 		struct block_device *bdev = device->ldev->backing_bdev;
 		struct request_queue *q = bdev_get_queue(bdev);
 
-		d_size = drbd_get_max_capacity(device->ldev);
+		d_size = drbd_get_max_capacity(device, device->ldev, false);
 		rcu_read_lock();
 		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
 		rcu_read_unlock();
@@ -927,6 +1594,10 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu
 		p->qlim->io_min = cpu_to_be32(bdev_io_min(bdev));
 		p->qlim->io_opt = cpu_to_be32(bdev_io_opt(bdev));
 		p->qlim->discard_enabled = !!bdev_max_discard_sectors(bdev);
+		p->qlim->write_same_capable = 0;
+		if (connection->agreed_features & DRBD_FF_BM_BLOCK_SHIFT)
+			p->qlim->bm_block_shift_minus_12 =
+				device->bitmap->bm_block_shift - BM_BLOCK_SHIFT_4k;
 		put_ldev(device);
 	} else {
 		struct request_queue *q = device->rq_queue;
@@ -939,128 +1610,307 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu
 		p->qlim->io_min = cpu_to_be32(queue_io_min(q));
 		p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
 		p->qlim->discard_enabled = 0;
+		p->qlim->write_same_capable = 0;
 
 		d_size = 0;
-		u_size = 0;
+		u_size = u_size_diskless;
 		q_order_type = QUEUE_ORDERED_NONE;
 		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
 	}
 
-	if (peer_device->connection->agreed_pro_version <= 94)
+	if (connection->agreed_pro_version <= 94)
 		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
-	else if (peer_device->connection->agreed_pro_version < 100)
+	else if (connection->agreed_pro_version < 100)
 		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
 
+	/* 9.0.4 bumped pro_version to 112 and introduced 2PC resizes */
+	if (connection->agreed_pro_version >= 112)
+		d_size = drbd_partition_data_capacity(device);
+
 	p->d_size = cpu_to_be64(d_size);
 	p->u_size = cpu_to_be64(u_size);
-	if (trigger_reply)
-		p->c_size = 0;
-	else
-		p->c_size = cpu_to_be64(get_capacity(device->vdisk));
+	/*
+	TODO verify: this may be needed for v8 compatibility still.
+	p->c_size = cpu_to_be64(trigger_reply ? 0 : get_capacity(device->vdisk));
+	*/
+	p->c_size = cpu_to_be64(get_capacity(device->vdisk));
 	p->max_bio_size = cpu_to_be32(max_bio_size);
 	p->queue_order_type = cpu_to_be16(q_order_type);
 	p->dds_flags = cpu_to_be16(flags);
 
-	return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0);
+	return drbd_send_command(peer_device, P_SIZES, DATA_STREAM);
 }
 
-/**
- * drbd_send_current_state() - Sends the drbd state to the peer
- * @peer_device:	DRBD peer device.
- */
 int drbd_send_current_state(struct drbd_peer_device *peer_device)
 {
-	struct drbd_socket *sock;
+	return drbd_send_state(peer_device, drbd_get_peer_device_state(peer_device, NOW));
+}
+
+static int send_state(struct drbd_connection *connection, int vnr, union drbd_state state)
+{
 	struct p_state *p;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
-	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
-	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
+
+	if (connection->agreed_pro_version < 110) {
+		/* D_DETACHING was introduced with drbd-9.0 */
+		if (state.disk > D_DETACHING)
+			state.disk--;
+		if (state.pdsk > D_DETACHING)
+			state.pdsk--;
+	}
+
+	p->state = cpu_to_be32(state.i); /* Within the send mutex */
+	return send_command(connection, vnr, P_STATE, DATA_STREAM);
+}
+
+int conn_send_state(struct drbd_connection *connection, union drbd_state state)
+{
+	BUG_ON(connection->agreed_pro_version < 100);
+	return send_state(connection, -1, state);
 }
 
 /**
- * drbd_send_state() - After a state change, sends the new state to the peer
- * @peer_device:      DRBD peer device.
- * @state:     the state to send, not necessarily the current state.
- *
- * Each state change queues an "after_state_ch" work, which will eventually
- * send the resulting new state to the peer. If more state changes happen
- * between queuing and processing of the after_state_ch work, we still
- * want to send each intermediary state in the order it occurred.
+ * drbd_send_state() - Sends the drbd state to the peer
+ * @peer_device: Peer DRBD device to send the state to.
+ * @state: state to send
  */
 int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
 {
-	struct drbd_socket *sock;
-	struct p_state *p;
-
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
-	p->state = cpu_to_be32(state.i); /* Within the send mutex */
-	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
+	peer_device->comm_state = state;
+	return send_state(peer_device->connection, peer_device->device->vnr, state);
 }
 
-int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
+int conn_send_state_req(struct drbd_connection *connection, int vnr, enum drbd_packet cmd,
+			union drbd_state mask, union drbd_state val)
 {
-	struct drbd_socket *sock;
 	struct p_req_state *p;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
+	/* Protocols before version 100 only support one volume and connection.
+	 * All state change requests are via P_STATE_CHG_REQ. */
+	if (connection->agreed_pro_version < 100)
+		cmd = P_STATE_CHG_REQ;
+
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
 	p->mask = cpu_to_be32(mask.i);
 	p->val = cpu_to_be32(val.i);
-	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
+
+	return send_command(connection, vnr, cmd, DATA_STREAM);
 }
 
-int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
+int conn_send_twopc_request(struct drbd_connection *connection, struct twopc_request *request)
 {
-	enum drbd_packet cmd;
-	struct drbd_socket *sock;
-	struct p_req_state *p;
+	struct drbd_resource *resource = connection->resource;
+	struct p_twopc_request *p;
+
+	dynamic_drbd_dbg(connection, "Sending %s request for state change %u\n",
+			 drbd_packet_name(request->cmd),
+			 request->tid);
 
-	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
-	sock = &connection->data;
-	p = conn_prepare_command(connection, sock);
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
-	p->mask = cpu_to_be32(mask.i);
-	p->val = cpu_to_be32(val.i);
-	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
+	p->tid = cpu_to_be32(request->tid);
+	if (connection->agreed_features & DRBD_FF_2PC_V2) {
+		p->flags = cpu_to_be32(TWOPC_HAS_FLAGS | request->flags);
+		p->_pad = 0;
+		p->s8_initiator_node_id = request->initiator_node_id;
+		p->s8_target_node_id = request->target_node_id;
+	} else {
+		p->u32_initiator_node_id = cpu_to_be32(request->initiator_node_id);
+		p->u32_target_node_id = cpu_to_be32(request->target_node_id);
+	}
+	p->nodes_to_reach = cpu_to_be64(request->nodes_to_reach);
+	switch (resource->twopc.type) {
+	case TWOPC_STATE_CHANGE:
+		if (request->cmd == P_TWOPC_PREPARE) {
+			p->_compat_pad = 0;
+			p->mask = cpu_to_be32(resource->twopc.state_change.mask.i);
+			p->val = cpu_to_be32(resource->twopc.state_change.val.i);
+		} else { /* P_TWOPC_COMMIT */
+			p->primary_nodes = cpu_to_be64(resource->twopc.state_change.primary_nodes);
+			if (request->flags & TWOPC_HAS_REACHABLE &&
+			    connection->agreed_features & DRBD_FF_2PC_V2) {
+				p->reachable_nodes = cpu_to_be64(
+					resource->twopc.state_change.reachable_nodes);
+			} else {
+				p->mask = cpu_to_be32(resource->twopc.state_change.mask.i);
+				p->val = cpu_to_be32(resource->twopc.state_change.val.i);
+			}
+		}
+		break;
+	case TWOPC_RESIZE:
+		if (request->cmd == P_TWOPC_PREP_RSZ) {
+			p->user_size = cpu_to_be64(resource->twopc.resize.user_size);
+			p->dds_flags = cpu_to_be16(resource->twopc.resize.dds_flags);
+		} else { /* P_TWOPC_COMMIT */
+			p->diskful_primary_nodes =
+				cpu_to_be64(resource->twopc.resize.diskful_primary_nodes);
+			p->exposed_size = cpu_to_be64(resource->twopc.resize.new_size);
+		}
+	}
+	return send_command(connection, request->vnr, request->cmd, DATA_STREAM | SFLAG_FLUSH);
 }
 
-void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
+void drbd_send_sr_reply(struct drbd_connection *connection, int vnr, enum drbd_state_rv retcode)
 {
-	struct drbd_socket *sock;
 	struct p_req_state_reply *p;
 
-	sock = &peer_device->connection->meta;
-	p = drbd_prepare_command(peer_device, sock);
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
 	if (p) {
+		enum drbd_packet cmd = P_STATE_CHG_REPLY;
+
+		if (connection->agreed_pro_version >= 100 && vnr < 0)
+			cmd = P_CONN_ST_CHG_REPLY;
+
 		p->retcode = cpu_to_be32(retcode);
-		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
+		send_command(connection, vnr, cmd, CONTROL_STREAM);
 	}
 }
 
-void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
+void drbd_send_twopc_reply(struct drbd_connection *connection,
+			   enum drbd_packet cmd, struct twopc_reply *reply)
 {
-	struct drbd_socket *sock;
-	struct p_req_state_reply *p;
-	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
+	struct p_twopc_reply *p;
 
-	sock = &connection->meta;
-	p = conn_prepare_command(connection, sock);
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
 	if (p) {
-		p->retcode = cpu_to_be32(retcode);
-		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
+		p->tid = cpu_to_be32(reply->tid);
+		p->initiator_node_id = cpu_to_be32(reply->initiator_node_id);
+		p->reachable_nodes = cpu_to_be64(reply->reachable_nodes);
+		switch (connection->resource->twopc.type) {
+		case TWOPC_STATE_CHANGE:
+			p->primary_nodes = cpu_to_be64(reply->primary_nodes);
+			p->weak_nodes = cpu_to_be64(reply->weak_nodes);
+			break;
+		case TWOPC_RESIZE:
+			p->diskful_primary_nodes = cpu_to_be64(reply->diskful_primary_nodes);
+			p->max_possible_size = cpu_to_be64(reply->max_possible_size);
+			break;
+		}
+		send_command(connection, reply->vnr, cmd, CONTROL_STREAM | SFLAG_FLUSH);
+	}
+}
+
+void drbd_send_peers_in_sync(struct drbd_peer_device *peer_device, u64 mask, sector_t sector, int size)
+{
+	struct p_peer_block_desc *p;
+
+	p = drbd_prepare_command(peer_device, sizeof(*p), CONTROL_STREAM);
+	if (p) {
+		p->sector = cpu_to_be64(sector);
+		p->mask = cpu_to_be64(mask);
+		p->size = cpu_to_be32(size);
+		p->pad = 0;
+		drbd_send_command(peer_device, P_PEERS_IN_SYNC, CONTROL_STREAM);
 	}
 }
 
+int drbd_send_peer_dagtag(struct drbd_connection *connection, struct drbd_connection *lost_peer)
+{
+	struct p_peer_dagtag *p;
+
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->dagtag = cpu_to_be64(atomic64_read(&lost_peer->last_dagtag_sector));
+	p->node_id = cpu_to_be32(lost_peer->peer_node_id);
+
+	return send_command(connection, -1, P_PEER_DAGTAG, DATA_STREAM);
+}
+
+int drbd_send_flush_requests(struct drbd_connection *connection, u64 flush_sequence)
+{
+	struct p_flush_requests *p;
+
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->flush_sequence = cpu_to_be64(flush_sequence);
+
+	return send_command(connection, -1, P_FLUSH_REQUESTS, DATA_STREAM);
+}
+
+int drbd_send_flush_forward(struct drbd_connection *connection, u64 flush_sequence,
+		int initiator_node_id)
+{
+	struct p_flush_forward *p;
+
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->flush_sequence = cpu_to_be64(flush_sequence);
+	p->initiator_node_id = cpu_to_be32(initiator_node_id);
+
+	return send_command(connection, -1, P_FLUSH_FORWARD, CONTROL_STREAM);
+}
+
+int drbd_send_flush_requests_ack(struct drbd_connection *connection, u64 flush_sequence,
+		int primary_node_id)
+{
+	struct p_flush_ack *p;
+
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->flush_sequence = cpu_to_be64(flush_sequence);
+	p->primary_node_id = cpu_to_be32(primary_node_id);
+
+	return send_command(connection, -1, P_FLUSH_REQUESTS_ACK, DATA_STREAM);
+}
+
+int drbd_send_enable_replication_next(struct drbd_peer_device *peer_device)
+{
+	struct p_enable_replication *p;
+	struct peer_device_conf *pdc;
+	bool resync_without_replication;
+
+	set_bit(PEER_REPLICATION_NEXT, &peer_device->flags);
+	if (!(peer_device->connection->agreed_features & DRBD_FF_RESYNC_WITHOUT_REPLICATION))
+		return 0;
+
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	rcu_read_lock();
+	pdc = rcu_dereference(peer_device->conf);
+	resync_without_replication = pdc->resync_without_replication;
+	rcu_read_unlock();
+
+	if (resync_without_replication)
+		clear_bit(PEER_REPLICATION_NEXT, &peer_device->flags);
+
+	p->enable = !resync_without_replication;
+	p->_pad1 = 0;
+	p->_pad2 = 0;
+
+	return drbd_send_command(peer_device, P_ENABLE_REPLICATION_NEXT, DATA_STREAM);
+}
+
+int drbd_send_enable_replication(struct drbd_peer_device *peer_device, bool enable)
+{
+	struct p_enable_replication *p;
+
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->enable = enable;
+	p->_pad1 = 0;
+	p->_pad2 = 0;
+
+	return drbd_send_command(peer_device, P_ENABLE_REPLICATION, DATA_STREAM);
+}
+
 static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
 {
 	BUG_ON(code & ~0xf);
@@ -1078,24 +1928,28 @@ static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
 	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
 }
 
-static int fill_bitmap_rle_bits(struct drbd_device *device,
-			 struct p_compressed_bm *p,
-			 unsigned int size,
-			 struct bm_xfer_ctx *c)
+/* For compat reasons, encode bitmap as if it was 4k per bit!
+ * Easy: just scale the run length.
+ */
+static int fill_bitmap_rle_bits(struct drbd_peer_device *peer_device,
+				struct p_compressed_bm *p,
+				unsigned int size,
+				struct bm_xfer_ctx *c)
 {
 	struct bitstream bs;
 	unsigned long plain_bits;
 	unsigned long tmp;
 	unsigned long rl;
+	unsigned long rl_4k;
 	unsigned len;
 	unsigned toggle;
 	int bits, use_rle;
 
 	/* may we use this feature? */
 	rcu_read_lock();
-	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
+	use_rle = rcu_dereference(peer_device->connection->transport.net_conf)->use_rle;
 	rcu_read_unlock();
-	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
+	if (!use_rle || peer_device->connection->agreed_pro_version < 90)
 		return 0;
 
 	if (c->bit_offset >= c->bm_bits)
@@ -1115,11 +1969,16 @@ static int fill_bitmap_rle_bits(struct drbd_device *device,
 	/* see how much plain bits we can stuff into one packet
 	 * using RLE and VLI. */
 	do {
-		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
-				    : _drbd_bm_find_next(device, c->bit_offset);
-		if (tmp == -1UL)
+		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(peer_device, c->bit_offset)
+				    : _drbd_bm_find_next(peer_device, c->bit_offset);
+		if (tmp == -1UL) {
 			tmp = c->bm_bits;
-		rl = tmp - c->bit_offset;
+			rl = tmp - c->bit_offset;
+			rl_4k = c->bm_bits_4k - (c->bit_offset << c->scale);
+		} else {
+			rl = tmp - c->bit_offset;
+			rl_4k = rl << c->scale;
+		}
 
 		if (toggle == 2) { /* first iteration */
 			if (rl == 0) {
@@ -1136,16 +1995,16 @@ static int fill_bitmap_rle_bits(struct drbd_device *device,
 		/* paranoia: catch zero runlength.
 		 * can only happen if bitmap is modified while we scan it. */
 		if (rl == 0) {
-			drbd_err(device, "unexpected zero runlength while encoding bitmap "
+			drbd_err(peer_device, "unexpected zero runlength while encoding bitmap "
 			    "t:%u bo:%lu\n", toggle, c->bit_offset);
 			return -1;
 		}
 
-		bits = vli_encode_bits(&bs, rl);
+		bits = vli_encode_bits(&bs, rl_4k);
 		if (bits == -ENOBUFS) /* buffer full */
 			break;
 		if (bits <= 0) {
-			drbd_err(device, "error while encoding bitmap: %d\n", bits);
+			drbd_err(peer_device, "error while encoding bitmap: %d\n", bits);
 			return 0;
 		}
 
@@ -1156,7 +2015,7 @@ static int fill_bitmap_rle_bits(struct drbd_device *device,
 
 	len = bs.cur.b - p->code + !!bs.cur.bit;
 
-	if (plain_bits < (len << 3)) {
+	if (plain_bits << c->scale < (len << 3)) {
 		/* incompressible with this method.
 		 * we need to rewind both word and bit position. */
 		c->bit_offset -= plain_bits;
@@ -1175,33 +2034,65 @@ static int fill_bitmap_rle_bits(struct drbd_device *device,
 	return len;
 }
 
+/* Repeat extracted bits by "peeling off" words from the end.
+ * scale != 0 implies that repeat >= 2.
+ * Feel free to optimize ...
+ */
+static void repeat_bits(unsigned long *base, unsigned long num, unsigned int scale)
+{
+	unsigned long *src, *dst;
+	unsigned int repeat = 1 << scale;
+	unsigned int n;
+	int sbit, dbit, i;
+
+	for (n = num - 1; n > 0; n--) {
+		src = &base[n];
+		for (i = 0; i < repeat; i++) {
+			dst = &base[n*repeat + i];
+			*dst = 0;
+			for (dbit = 0; dbit < BITS_PER_LONG; dbit++) {
+				sbit = (i * BITS_PER_LONG + dbit) >> scale;
+				if (test_bit(sbit, src))
+					*dst |= 1UL << dbit;
+			}
+		}
+	}
+}
+
 /*
  * send_bitmap_rle_or_plain
  *
  * Return 0 when done, 1 when another iteration is needed, and a negative error
  * code upon failure.
+ *
+ * For compat reasons, send bitmap as if it was 4k per bit!
+ * Good thing that a "scaled" bitmap will always "compress".
  */
 static int
 send_bitmap_rle_or_plain(struct drbd_peer_device *peer_device, struct bm_xfer_ctx *c)
 {
 	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock = &peer_device->connection->data;
 	unsigned int header_size = drbd_header_size(peer_device->connection);
-	struct p_compressed_bm *p = sock->sbuf + header_size;
+	struct p_compressed_bm *pc;
 	int len, err;
 
-	len = fill_bitmap_rle_bits(device, p,
-			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
-	if (len < 0)
+	pc = (struct p_compressed_bm *)
+		(alloc_send_buffer(peer_device->connection, DRBD_SOCKET_BUFFER_SIZE, DATA_STREAM) + header_size);
+
+	len = fill_bitmap_rle_bits(peer_device, pc,
+			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*pc), c);
+	if (len < 0) {
+		cancel_send_buffer(peer_device->connection, DATA_STREAM);
 		return -EIO;
+	}
 
 	if (len) {
-		dcbp_set_code(p, RLE_VLI_Bits);
-		err = __send_command(peer_device->connection, device->vnr, sock,
-				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
-				     NULL, 0);
+		dcbp_set_code(pc, RLE_VLI_Bits);
+		resize_prepared_command(peer_device->connection, DATA_STREAM, sizeof(*pc) + len);
+		err = __send_command(peer_device->connection, device->vnr,
+				     P_COMPRESSED_BITMAP, DATA_STREAM);
 		c->packets[0]++;
-		c->bytes[0] += header_size + sizeof(*p) + len;
+		c->bytes[0] += header_size + sizeof(*pc) + len;
 
 		if (c->bit_offset >= c->bm_bits)
 			len = 0; /* DONE */
@@ -1210,16 +2101,40 @@ send_bitmap_rle_or_plain(struct drbd_peer_device *peer_device, struct bm_xfer_ct
 		 * send a buffer full of plain text bits instead. */
 		unsigned int data_size;
 		unsigned long num_words;
-		unsigned long *p = sock->sbuf + header_size;
-
+		unsigned long words_left = c->bm_words - c->word_offset;
+		unsigned long *pu = (unsigned long *)pc;
+
+		/* Only send full native bitmap words (actual granularity),
+		 * scaled to what they would look like at 4k granularity.
+		 * At maximum scale, which is (20 - 12), factor 256,
+		 * to transfer at least one word of unscaled bitmap,
+		 * we need data_size >= 256 (unsigned long) words,
+		 * that is >= 2048 byte. Which we always have.
+		 */
 		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
-		num_words = min_t(size_t, data_size / sizeof(*p),
-				  c->bm_words - c->word_offset);
-		len = num_words * sizeof(*p);
-		if (len)
-			drbd_bm_get_lel(device, c->word_offset, num_words, p);
-		err = __send_command(peer_device->connection, device->vnr, sock, P_BITMAP,
-				     len, NULL, 0);
+		data_size = ALIGN_DOWN(data_size, sizeof(*pu) * (1UL << c->scale));
+		num_words = (data_size / sizeof(*pu)) >> c->scale;
+		num_words = min_t(size_t, num_words, words_left);
+
+		len = num_words * sizeof(*pu);
+		if (len) {
+			drbd_bm_get_lel(peer_device, c->word_offset, num_words, pu);
+
+			if (c->scale) {
+				repeat_bits(pu, num_words, c->scale);
+				len <<= c->scale;
+			}
+		} else if (words_left != 0) {
+			drbd_err(peer_device,
+				"failed to scale %lu words by %u while sending bitmap\n",
+				words_left, c->scale);
+			cancel_send_buffer(peer_device->connection, DATA_STREAM);
+			return -ERANGE;
+		}
+
+		resize_prepared_command(peer_device->connection, DATA_STREAM, len);
+		err = __send_command(peer_device->connection, device->vnr, P_BITMAP, DATA_STREAM);
+
 		c->word_offset += num_words;
 		c->bit_offset = c->word_offset * BITS_PER_LONG;
 
@@ -1240,242 +2155,169 @@ send_bitmap_rle_or_plain(struct drbd_peer_device *peer_device, struct bm_xfer_ct
 }
 
 /* See the comment at receive_bitmap() */
-static int _drbd_send_bitmap(struct drbd_device *device,
-			    struct drbd_peer_device *peer_device)
+static bool _drbd_send_bitmap(struct drbd_device *device,
+			     struct drbd_peer_device *peer_device)
 {
 	struct bm_xfer_ctx c;
-	int err;
+	int res;
 
 	if (!expect(device, device->bitmap))
 		return false;
 
 	if (get_ldev(device)) {
-		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
+		if (drbd_md_test_peer_flag(peer_device, MDF_PEER_FULL_SYNC)) {
 			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
-			drbd_bm_set_all(device);
-			if (drbd_bm_write(device, peer_device)) {
+			drbd_bm_set_many_bits(peer_device, 0, -1UL);
+			if (drbd_bm_write(device, NULL)) {
 				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
 				 * but otherwise process as per normal - need to tell other
 				 * side that a full resync is required! */
 				drbd_err(device, "Failed to write bitmap to disk!\n");
 			} else {
-				drbd_md_clear_flag(device, MDF_FULL_SYNC);
+				drbd_md_clear_peer_flag(peer_device, MDF_PEER_FULL_SYNC);
 				drbd_md_sync(device);
 			}
 		}
+		c = (struct bm_xfer_ctx) {
+			.bm_bits_4k = drbd_bm_bits_4k(device),
+			.bm_bits = drbd_bm_bits(device),
+			.bm_words = drbd_bm_words(device),
+			.scale = device->bitmap->bm_block_shift - BM_BLOCK_SHIFT_4k,
+		};
+
 		put_ldev(device);
+	} else {
+		return false;
 	}
 
-	c = (struct bm_xfer_ctx) {
-		.bm_bits = drbd_bm_bits(device),
-		.bm_words = drbd_bm_words(device),
-	};
-
 	do {
-		err = send_bitmap_rle_or_plain(peer_device, &c);
-	} while (err > 0);
+		if (get_ldev(device)) {
+			res = send_bitmap_rle_or_plain(peer_device, &c);
+			put_ldev(device);
+		} else {
+			return false;
+		}
+	} while (res > 0);
 
-	return err == 0;
+	return res == 0;
 }
 
 int drbd_send_bitmap(struct drbd_device *device, struct drbd_peer_device *peer_device)
 {
-	struct drbd_socket *sock = &peer_device->connection->data;
+	struct drbd_transport *peer_transport = &peer_device->connection->transport;
 	int err = -1;
 
-	mutex_lock(&sock->mutex);
-	if (sock->socket)
+	if (peer_device->bitmap_index == -1) {
+		drbd_err(peer_device, "No bitmap allocated in drbd_send_bitmap()!\n");
+		return -EIO;
+	}
+
+	mutex_lock(&peer_device->connection->mutex[DATA_STREAM]);
+	if (peer_transport->class->ops.stream_ok(peer_transport, DATA_STREAM))
 		err = !_drbd_send_bitmap(device, peer_device);
-	mutex_unlock(&sock->mutex);
+	mutex_unlock(&peer_device->connection->mutex[DATA_STREAM]);
+
 	return err;
 }
 
-void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
+int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
+			     struct drbd_peer_request *peer_req)
 {
-	struct drbd_socket *sock;
-	struct p_barrier_ack *p;
+	struct p_block_ack *p_id;
 
-	if (connection->cstate < C_WF_REPORT_PARAMS)
-		return;
+	if (peer_device->connection->agreed_pro_version < 122) {
+		struct p_block_desc *p;
 
-	sock = &connection->meta;
-	p = conn_prepare_command(connection, sock);
-	if (!p)
-		return;
-	p->barrier = barrier_nr;
-	p->set_size = cpu_to_be32(set_size);
-	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
+		p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
+		if (!p)
+			return -EIO;
+		p->sector = cpu_to_be64(peer_req->i.sector);
+		p->blksize = cpu_to_be32(peer_req->i.size);
+		p->pad = 0;
+		return drbd_send_command(peer_device, P_RS_DEALLOCATED, DATA_STREAM);
+	}
+
+	p_id = drbd_prepare_command(peer_device, sizeof(*p_id), DATA_STREAM);
+	if (!p_id)
+		return -EIO;
+	p_id->sector = cpu_to_be64(peer_req->i.sector);
+	p_id->blksize = cpu_to_be32(peer_req->i.size);
+	p_id->block_id = peer_req->block_id;
+	p_id->seq_num = 0;
+	return drbd_send_command(peer_device, P_RS_DEALLOCATED_ID, DATA_STREAM);
 }
 
-/**
- * _drbd_send_ack() - Sends an ack packet
- * @peer_device:	DRBD peer device.
- * @cmd:		Packet command code.
- * @sector:		sector, needs to be in big endian byte order
- * @blksize:		size in byte, needs to be in big endian byte order
- * @block_id:		Id, big endian byte order
- */
-static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
-			  u64 sector, u32 blksize, u64 block_id)
+int drbd_send_drequest(struct drbd_peer_device *peer_device,
+		       sector_t sector, int size, u64 block_id)
 {
-	struct drbd_socket *sock;
-	struct p_block_ack *p;
-
-	if (peer_device->device->state.conn < C_CONNECTED)
-		return -EIO;
+	struct p_block_req *p;
 
-	sock = &peer_device->connection->meta;
-	p = drbd_prepare_command(peer_device, sock);
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
-	p->sector = sector;
-	p->block_id = block_id;
-	p->blksize = blksize;
-	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
-	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
-}
-
-/* dp->sector and dp->block_id already/still in network byte order,
- * data_size is payload size according to dp->head,
- * and may need to be corrected for digest size. */
-void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
-		      struct p_data *dp, int data_size)
-{
-	if (peer_device->connection->peer_integrity_tfm)
-		data_size -= crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm);
-	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
-		       dp->block_id);
-}
-
-void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
-		      struct p_block_req *rp)
-{
-	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
-}
-
-/**
- * drbd_send_ack() - Sends an ack packet
- * @peer_device:	DRBD peer device
- * @cmd:		packet command code
- * @peer_req:		peer request
- */
-int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
-		  struct drbd_peer_request *peer_req)
-{
-	return _drbd_send_ack(peer_device, cmd,
-			      cpu_to_be64(peer_req->i.sector),
-			      cpu_to_be32(peer_req->i.size),
-			      peer_req->block_id);
-}
-
-/* This function misuses the block_id field to signal if the blocks
- * are is sync or not. */
-int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
-		     sector_t sector, int blksize, u64 block_id)
-{
-	return _drbd_send_ack(peer_device, cmd,
-			      cpu_to_be64(sector),
-			      cpu_to_be32(blksize),
-			      cpu_to_be64(block_id));
-}
-
-int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
-			     struct drbd_peer_request *peer_req)
-{
-	struct drbd_socket *sock;
-	struct p_block_desc *p;
-
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
-	p->sector = cpu_to_be64(peer_req->i.sector);
-	p->blksize = cpu_to_be32(peer_req->i.size);
-	p->pad = 0;
-	return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0);
-}
-
-int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
-		       sector_t sector, int size, u64 block_id)
-{
-	struct drbd_socket *sock;
-	struct p_block_req *p;
-
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
-	p->sector = cpu_to_be64(sector);
+	p->sector = cpu_to_be64(sector);
 	p->block_id = block_id;
 	p->blksize = cpu_to_be32(size);
-	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
-}
-
-int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
-			    void *digest, int digest_size, enum drbd_packet cmd)
-{
-	struct drbd_socket *sock;
-	struct p_block_req *p;
+	p->pad = 0;
+	return drbd_send_command(peer_device, P_DATA_REQUEST, DATA_STREAM);
+}
+
+static void *drbd_prepare_rs_req(struct drbd_peer_device *peer_device, enum drbd_packet cmd, int payload_size,
+		sector_t sector, int blksize, u64 block_id, unsigned int dagtag_node_id, u64 dagtag)
+{
+	void *payload;
+	struct p_block_req_common *req_common;
+
+	if (cmd == P_RS_DAGTAG_REQ || cmd == P_RS_CSUM_DAGTAG_REQ || cmd == P_RS_THIN_DAGTAG_REQ ||
+			cmd == P_OV_DAGTAG_REQ || cmd == P_OV_DAGTAG_REPLY) {
+		struct p_rs_req *p;
+		/* Due to the slightly complicated nested struct definition,
+		 * verify that the packet size is as expected. */
+		BUILD_BUG_ON(sizeof(struct p_rs_req) != 32);
+		p = drbd_prepare_command(peer_device, sizeof(*p) + payload_size, DATA_STREAM);
+		if (!p)
+			return NULL;
+		payload = p + 1;
+		req_common = &p->req_common;
+		p->dagtag_node_id = cpu_to_be32(dagtag_node_id);
+		p->dagtag = cpu_to_be64(dagtag);
+	} else {
+		struct p_block_req *p;
+		/* Due to the slightly complicated nested struct definition,
+		 * verify that the packet size is as expected. */
+		BUILD_BUG_ON(sizeof(struct p_block_req) != 24);
+		p = drbd_prepare_command(peer_device, sizeof(*p) + payload_size, DATA_STREAM);
+		if (!p)
+			return NULL;
+		payload = p + 1;
+		req_common = &p->req_common;
+		p->pad = 0;
+	}
 
-	/* FIXME: Put the digest into the preallocated socket buffer.  */
+	req_common->sector = cpu_to_be64(sector);
+	req_common->block_id = block_id;
+	req_common->blksize = cpu_to_be32(blksize);
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
-		return -EIO;
-	p->sector = cpu_to_be64(sector);
-	p->block_id = ID_SYNCER /* unused */;
-	p->blksize = cpu_to_be32(size);
-	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
+	return payload;
 }
 
-int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
+int drbd_send_rs_request(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+		       sector_t sector, int size, u64 block_id,
+		       unsigned int dagtag_node_id, u64 dagtag)
 {
-	struct drbd_socket *sock;
-	struct p_block_req *p;
-
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	if (!p)
+	if (!drbd_prepare_rs_req(peer_device, cmd, 0,
+				sector, size, block_id, dagtag_node_id, dagtag))
 		return -EIO;
-	p->sector = cpu_to_be64(sector);
-	p->block_id = ID_SYNCER /* unused */;
-	p->blksize = cpu_to_be32(size);
-	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
+	return drbd_send_command(peer_device, cmd, DATA_STREAM);
 }
 
-/* called on sndtimeo
- * returns false if we should retry,
- * true if we think connection is dead
- */
-static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
-{
-	int drop_it;
-	/* long elapsed = (long)(jiffies - device->last_received); */
-
-	drop_it =   connection->meta.socket == sock
-		|| !connection->ack_receiver.task
-		|| get_t_state(&connection->ack_receiver) != RUNNING
-		|| connection->cstate < C_WF_REPORT_PARAMS;
-
-	if (drop_it)
-		return true;
-
-	drop_it = !--connection->ko_count;
-	if (!drop_it) {
-		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
-			 current->comm, current->pid, connection->ko_count);
-		request_ping(connection);
-	}
-
-	return drop_it; /* && (device->state == R_PRIMARY) */;
-}
-
-static void drbd_update_congested(struct drbd_connection *connection)
+void *drbd_prepare_drequest_csum(struct drbd_peer_request *peer_req, enum drbd_packet cmd,
+		int digest_size, unsigned int dagtag_node_id, u64 dagtag)
 {
-	struct sock *sk = connection->data.socket->sk;
-	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
-		set_bit(NET_CONGESTED, &connection->flags);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	return drbd_prepare_rs_req(peer_device, cmd, digest_size,
+			peer_req->i.sector, peer_req->i.size, peer_req->block_id,
+			dagtag_node_id, dagtag);
 }
 
 /* The idea of sendpage seems to be to put some kind of reference
@@ -1499,87 +2341,67 @@ static void drbd_update_congested(struct drbd_connection *connection)
  * As a workaround, we disable sendpage on pages
  * with page_count == 0 or PageSlab.
  */
-static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
-			      int offset, size_t size, unsigned msg_flags)
+static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
+			    int offset, size_t size, unsigned msg_flags)
 {
-	struct socket *socket;
-	void *addr;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
 	int err;
 
-	socket = peer_device->connection->data.socket;
-	addr = kmap(page) + offset;
-	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
-	kunmap(page);
+	err = tr_ops->send_page(transport, DATA_STREAM, page, offset, size, msg_flags);
 	if (!err)
-		peer_device->device->send_cnt += size >> 9;
+		peer_device->send_cnt += size >> 9;
+
 	return err;
 }
 
-static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
-		    int offset, size_t size, unsigned msg_flags)
+static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
+			      int offset, size_t size, unsigned msg_flags)
 {
-	struct socket *socket = peer_device->connection->data.socket;
-	struct msghdr msg = { .msg_flags = msg_flags, };
-	struct bio_vec bvec;
-	int len = size;
-	int err = -EIO;
-
-	/* e.g. XFS meta- & log-data is in slab pages, which have a
-	 * page_count of 0 and/or have PageSlab() set.
-	 * we cannot use send_page for those, as that does get_page();
-	 * put_page(); and would cause either a VM_BUG directly, or
-	 * __page_cache_release a page that would actually still be referenced
-	 * by someone, leading to some obscure delayed Oops somewhere else. */
-	if (!drbd_disable_sendpage && sendpages_ok(page, len, offset))
-		msg.msg_flags |= MSG_NOSIGNAL | MSG_SPLICE_PAGES;
-
-	drbd_update_congested(peer_device->connection);
-	do {
-		int sent;
-
-		bvec_set_page(&bvec, page, len, offset);
-		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_send_buffer *sbuf = &connection->send_buffer[DATA_STREAM];
+	char *from_base;
+	void *buffer2;
+	int err;
 
-		sent = sock_sendmsg(socket, &msg);
-		if (sent <= 0) {
-			if (sent == -EAGAIN) {
-				if (we_should_drop_the_connection(peer_device->connection, socket))
-					break;
-				continue;
-			}
-			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
-			     __func__, (int)size, len, sent);
-			if (sent < 0)
-				err = sent;
-			break;
-		}
-		len    -= sent;
-		offset += sent;
-	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
-	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
+	buffer2 = alloc_send_buffer(connection, size, DATA_STREAM);
+	from_base = kmap_atomic(page);
+	memcpy(buffer2, from_base + offset, size);
+	kunmap_atomic(from_base);
 
-	if (len == 0) {
+	if (msg_flags & MSG_MORE) {
+		sbuf->pos += sbuf->allocated_size;
+		sbuf->allocated_size = 0;
 		err = 0;
-		peer_device->device->send_cnt += size >> 9;
+	} else {
+		err = flush_send_buffer(connection, DATA_STREAM);
 	}
+
 	return err;
 }
 
 static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
 {
+	struct drbd_connection *connection = peer_device->connection;
 	struct bio_vec bvec;
 	struct bvec_iter iter;
 
+	/* Flush send buffer and make sure PAGE_SIZE is available... */
+	alloc_send_buffer(connection, PAGE_SIZE, DATA_STREAM);
+	connection->send_buffer[DATA_STREAM].allocated_size = 0;
+
 	/* hint all but last page with MSG_MORE */
 	bio_for_each_segment(bvec, bio, iter) {
 		int err;
 
 		err = _drbd_no_send_page(peer_device, bvec.bv_page,
 					 bvec.bv_offset, bvec.bv_len,
-					 bio_iter_last(bvec, iter)
-					 ? 0 : MSG_MORE);
+					 bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
 		if (err)
 			return err;
+
+		peer_device->send_cnt += bvec.bv_len >> 9;
 	}
 	return 0;
 }
@@ -1588,31 +2410,59 @@ static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *b
 {
 	struct bio_vec bvec;
 	struct bvec_iter iter;
+	bool no_zc = drbd_disable_sendpage;
 
-	/* hint all but last page with MSG_MORE */
-	bio_for_each_segment(bvec, bio, iter) {
+	/* e.g. XFS meta- & log-data is in slab pages, which have a
+	 * page_count of 0 and/or have PageSlab() set.
+	 * we cannot use send_page for those, as that does get_page();
+	 * put_page(); and would cause either a VM_BUG directly, or
+	 * __page_cache_release a page that would actually still be referenced
+	 * by someone, leading to some obscure delayed Oops somewhere else. */
+	if (!no_zc)
+		bio_for_each_segment(bvec, bio, iter) {
+			struct page *page = bvec.bv_page;
+
+			if (!sendpage_ok(page)) {
+				no_zc = true;
+				break;
+			}
+		}
+
+	if (no_zc) {
+		return _drbd_send_bio(peer_device, bio);
+	} else {
+		struct drbd_connection *connection = peer_device->connection;
+		struct drbd_transport *transport = &connection->transport;
+		struct drbd_transport_ops *tr_ops = &transport->class->ops;
 		int err;
 
-		err = _drbd_send_page(peer_device, bvec.bv_page,
-				      bvec.bv_offset, bvec.bv_len,
-				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
-		if (err)
-			return err;
+		flush_send_buffer(connection, DATA_STREAM);
+
+		err = tr_ops->send_zc_bio(transport, bio);
+		if (!err)
+			peer_device->send_cnt += bio->bi_iter.bi_size >> 9;
+
+		return err;
 	}
-	return 0;
 }
 
 static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
 			    struct drbd_peer_request *peer_req)
 {
 	bool use_sendpage = !(peer_req->flags & EE_RELEASE_TO_MEMPOOL);
-	struct page *page = peer_req->pages;
+	struct page *page = peer_req->page_chain.head;
 	unsigned len = peer_req->i.size;
 	int err;
 
+	flush_send_buffer(peer_device->connection, DATA_STREAM);
 	/* hint all but last page with MSG_MORE */
 	page_chain_for_each(page) {
 		unsigned l = min_t(unsigned, len, PAGE_SIZE);
+		if (page_chain_offset(page) != 0 ||
+		    page_chain_size(page) != l) {
+			drbd_err(peer_device, "FIXME page %p offset %u len %u\n",
+				page, page_chain_offset(page), page_chain_size(page));
+		}
 
 		if (likely(use_sendpage))
 			err = _drbd_send_page(peer_device, page, 0, l,
@@ -1620,7 +2470,6 @@ static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
 		else
 			err = _drbd_no_send_page(peer_device, page, 0, l,
 						 page_chain_next(page) ? MSG_MORE : 0);
-
 		if (err)
 			return err;
 		len -= l;
@@ -1628,8 +2477,8 @@ static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
 	return 0;
 }
 
-static u32 bio_flags_to_wire(struct drbd_connection *connection,
-			     struct bio *bio)
+/* see also wire_flags_to_bio() */
+static u32 bio_flags_to_wire(struct drbd_connection *connection, struct bio *bio)
 {
 	if (connection->agreed_pro_version >= 95)
 		return  (bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0) |
@@ -1637,12 +2486,13 @@ static u32 bio_flags_to_wire(struct drbd_connection *connection,
 			(bio->bi_opf & REQ_PREFLUSH ? DP_FLUSH : 0) |
 			(bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0) |
 			(bio_op(bio) == REQ_OP_WRITE_ZEROES ?
-			  ((connection->agreed_features & DRBD_FF_WZEROES) ?
-			   (DP_ZEROES |(!(bio->bi_opf & REQ_NOUNMAP) ? DP_DISCARD : 0))
-			   : DP_DISCARD)
-			: 0);
-	else
-		return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0;
+			 ((connection->agreed_features & DRBD_FF_WZEROES) ?
+			  (DP_ZEROES | (!(bio->bi_opf & REQ_NOUNMAP) ? DP_DISCARD : 0))
+			  : DP_DISCARD)
+			 : 0);
+
+	/* else: we used to communicate one bit only in older DRBD */
+	return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0;
 }
 
 /* Used to send write or TRIM aka REQ_OP_DISCARD requests
@@ -1651,53 +2501,61 @@ static u32 bio_flags_to_wire(struct drbd_connection *connection,
 int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
 {
 	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock;
+	char *const before = peer_device->connection->scratch_buffer.d.before;
+	char *const after = peer_device->connection->scratch_buffer.d.after;
+	struct p_trim *trim = NULL;
 	struct p_data *p;
-	void *digest_out;
+	void *digest_out = NULL;
 	unsigned int dp_flags = 0;
-	int digest_size;
+	int digest_size = 0;
 	int err;
+	const unsigned s = req->net_rq_state[peer_device->node_id];
+	const enum req_op op = bio_op(req->master_bio);
+
+	if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
+		trim = drbd_prepare_command(peer_device, sizeof(*trim), DATA_STREAM);
+		if (!trim)
+			return -EIO;
+		p = &trim->p_data;
+		trim->size = cpu_to_be32(req->i.size);
+	} else {
+		if (peer_device->connection->integrity_tfm)
+			digest_size = crypto_shash_digestsize(peer_device->connection->integrity_tfm);
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-	digest_size = peer_device->connection->integrity_tfm ?
-		      crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0;
+		p = drbd_prepare_command(peer_device, sizeof(*p) + digest_size, DATA_STREAM);
+		if (!p)
+			return -EIO;
+		digest_out = p + 1;
+	}
 
-	if (!p)
-		return -EIO;
 	p->sector = cpu_to_be64(req->i.sector);
 	p->block_id = (unsigned long)req;
-	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
+	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->packet_seq));
 	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio);
-	if (device->state.conn >= C_SYNC_SOURCE &&
-	    device->state.conn <= C_PAUSED_SYNC_T)
+	if (peer_device->repl_state[NOW] >= L_SYNC_SOURCE && peer_device->repl_state[NOW] <= L_PAUSED_SYNC_T)
 		dp_flags |= DP_MAY_SET_IN_SYNC;
 	if (peer_device->connection->agreed_pro_version >= 100) {
-		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
+		if (s & RQ_EXP_RECEIVE_ACK)
 			dp_flags |= DP_SEND_RECEIVE_ACK;
-		/* During resync, request an explicit write ack,
-		 * even in protocol != C */
-		if (req->rq_state & RQ_EXP_WRITE_ACK
-		|| (dp_flags & DP_MAY_SET_IN_SYNC))
+		if (s & RQ_EXP_WRITE_ACK || dp_flags & DP_MAY_SET_IN_SYNC)
 			dp_flags |= DP_SEND_WRITE_ACK;
 	}
 	p->dp_flags = cpu_to_be32(dp_flags);
 
-	if (dp_flags & (DP_DISCARD|DP_ZEROES)) {
-		enum drbd_packet cmd = (dp_flags & DP_ZEROES) ? P_ZEROES : P_TRIM;
-		struct p_trim *t = (struct p_trim*)p;
-		t->size = cpu_to_be32(req->i.size);
-		err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*t), NULL, 0);
+	if (trim) {
+		err = __send_command(peer_device->connection, device->vnr,
+				(dp_flags & DP_ZEROES) ? P_ZEROES : P_TRIM, DATA_STREAM);
 		goto out;
 	}
-	digest_out = p + 1;
 
-	/* our digest is still only over the payload.
-	 * TRIM does not carry any payload. */
-	if (digest_size)
-		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out);
-	err = __send_command(peer_device->connection, device->vnr, sock, P_DATA,
-			     sizeof(*p) + digest_size, NULL, req->i.size);
+	if (digest_size && digest_out) {
+		BUG_ON(digest_size > sizeof(peer_device->connection->scratch_buffer.d.before));
+		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, before);
+		memcpy(digest_out, before, digest_size);
+	}
+
+	additional_size_command(peer_device->connection, DATA_STREAM, req->i.size);
+	err = __send_command(peer_device->connection, device->vnr, P_DATA, DATA_STREAM);
 	if (!err) {
 		/* For protocol A, we have to memcpy the payload into
 		 * socket buffers, as we may complete right away
@@ -1710,365 +2568,778 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *
 		 * out ok after sending on this side, but does not fit on the
 		 * receiving side, we sure have detected corruption elsewhere.
 		 */
-		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || digest_size)
+		if (!(s & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || digest_size)
 			err = _drbd_send_bio(peer_device, req->master_bio);
 		else
 			err = _drbd_send_zc_bio(peer_device, req->master_bio);
 
 		/* double check digest, sometimes buffers have been modified in flight. */
-		if (digest_size > 0 && digest_size <= 64) {
-			/* 64 byte, 512 bit, is the largest digest size
-			 * currently supported in kernel crypto. */
-			unsigned char digest[64];
-			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
-			if (memcmp(p + 1, digest, digest_size)) {
+		if (digest_size > 0) {
+			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, after);
+			if (memcmp(before, after, digest_size)) {
 				drbd_warn(device,
 					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
 					(unsigned long long)req->i.sector, req->i.size);
 			}
-		} /* else if (digest_size > 64) {
-		     ... Be noisy about digest too large ...
-		} */
+		}
 	}
 out:
-	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
+	mutex_unlock(&peer_device->connection->mutex[DATA_STREAM]);
 
 	return err;
 }
 
 /* answer packet, used to send data back for read requests:
  *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
- *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
+ *  L_SYNC_SOURCE -> L_SYNC_TARGET         (P_RS_DATA_REPLY)
  */
 int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
 		    struct drbd_peer_request *peer_req)
 {
-	struct drbd_device *device = peer_device->device;
-	struct drbd_socket *sock;
 	struct p_data *p;
 	int err;
 	int digest_size;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
-
 	digest_size = peer_device->connection->integrity_tfm ?
 		      crypto_shash_digestsize(peer_device->connection->integrity_tfm) : 0;
 
+	p = drbd_prepare_command(peer_device, sizeof(*p) + digest_size, DATA_STREAM);
+
 	if (!p)
 		return -EIO;
 	p->sector = cpu_to_be64(peer_req->i.sector);
 	p->block_id = peer_req->block_id;
 	p->seq_num = 0;  /* unused */
 	p->dp_flags = 0;
+
+	/* Older peers expect block_id for P_RS_DATA_REPLY to be ID_SYNCER. */
+	if (peer_device->connection->agreed_pro_version < 122 && cmd == P_RS_DATA_REPLY)
+		p->block_id = ID_SYNCER;
+
 	if (digest_size)
-		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
-	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + digest_size, NULL, peer_req->i.size);
+		drbd_csum_pages(peer_device->connection->integrity_tfm, peer_req->page_chain.head, p + 1);
+	additional_size_command(peer_device->connection, DATA_STREAM, peer_req->i.size);
+	err = __send_command(peer_device->connection,
+			     peer_device->device->vnr, cmd, DATA_STREAM);
 	if (!err)
 		err = _drbd_send_zc_ee(peer_device, peer_req);
-	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
+	mutex_unlock(&peer_device->connection->mutex[DATA_STREAM]);
 
 	return err;
 }
 
-int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
+int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, sector_t sector, unsigned int size)
 {
-	struct drbd_socket *sock;
 	struct p_block_desc *p;
 
-	sock = &peer_device->connection->data;
-	p = drbd_prepare_command(peer_device, sock);
+	p = drbd_prepare_command(peer_device, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
-	p->sector = cpu_to_be64(req->i.sector);
-	p->blksize = cpu_to_be32(req->i.size);
-	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
+	p->sector = cpu_to_be64(sector);
+	p->blksize = cpu_to_be32(size);
+	return drbd_send_command(peer_device, P_OUT_OF_SYNC, DATA_STREAM);
 }
 
-/*
-  drbd_send distinguishes two cases:
+int drbd_send_dagtag(struct drbd_connection *connection, u64 dagtag)
+{
+	struct p_dagtag *p;
 
-  Packets sent via the data socket "sock"
-  and packets sent via the meta data socket "msock"
+	if (connection->agreed_pro_version < 110)
+		return 0;
 
-		    sock                      msock
-  -----------------+-------------------------+------------------------------
-  timeout           conf.timeout / 2          conf.timeout / 2
-  timeout action    send a ping via msock     Abort communication
-					      and close all sockets
-*/
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+	p->dagtag = cpu_to_be64(dagtag);
+	return send_command(connection, -1, P_DAGTAG, DATA_STREAM);
+}
 
-/*
- * you must have down()ed the appropriate [m]sock_mutex elsewhere!
- */
-int drbd_send(struct drbd_connection *connection, struct socket *sock,
-	      void *buf, size_t size, unsigned msg_flags)
+/* primary_peer_present_and_not_two_primaries_allowed() */
+static bool primary_peer_present(struct drbd_resource *resource)
 {
-	struct kvec iov = {.iov_base = buf, .iov_len = size};
-	struct msghdr msg = {.msg_flags = msg_flags | MSG_NOSIGNAL};
-	int rv, sent = 0;
+	struct drbd_connection *connection;
+	struct net_conf *nc;
+	bool two_primaries, rv = false;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		nc = rcu_dereference(connection->transport.net_conf);
+		two_primaries = nc ? nc->two_primaries : false;
+
+		if (connection->peer_role[NOW] == R_PRIMARY && !two_primaries) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
 
-	if (!sock)
-		return -EBADR;
+	return rv;
+}
 
-	/* THINK  if (signal_pending) return ... ? */
+static bool any_disk_is_uptodate(struct drbd_device *device)
+{
+	bool ret = false;
 
-	iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &iov, 1, size);
+	rcu_read_lock();
+	if (device->disk_state[NOW] == D_UP_TO_DATE)
+		ret = true;
+	else {
+		struct drbd_peer_device *peer_device;
 
-	if (sock == connection->data.socket) {
-		rcu_read_lock();
-		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
-		rcu_read_unlock();
-		drbd_update_congested(connection);
-	}
-	do {
-		rv = sock_sendmsg(sock, &msg);
-		if (rv == -EAGAIN) {
-			if (we_should_drop_the_connection(connection, sock))
+		for_each_peer_device_rcu(peer_device, device) {
+			if (peer_device->disk_state[NOW] == D_UP_TO_DATE) {
+				ret = true;
 				break;
-			else
-				continue;
-		}
-		if (rv == -EINTR) {
-			flush_signals(current);
-			rv = 0;
+			}
 		}
-		if (rv < 0)
-			break;
-		sent += rv;
-	} while (sent < size);
-
-	if (sock == connection->data.socket)
-		clear_bit(NET_CONGESTED, &connection->flags);
-
-	if (rv <= 0) {
-		if (rv != -EAGAIN) {
-			drbd_err(connection, "%s_sendmsg returned %d\n",
-				 sock == connection->meta.socket ? "msock" : "sock",
-				 rv);
-			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
-		} else
-			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
 	}
+	rcu_read_unlock();
 
-	return sent;
+	return ret;
 }
 
-/*
- * drbd_send_all  -  Send an entire buffer
- *
- * Returns 0 upon success and a negative error value otherwise.
+/* If we are trying to (re-)establish some connection,
+ * it may be useful to re-try the conditions in drbd_open().
+ * But if we have no connection at all (yet/anymore),
+ * or are disconnected and not trying to (re-)establish,
+ * or are established already, retrying won't help at all.
+ * Asking the same peer(s) the same question
+ * is unlikely to change their answer.
+ * Almost always triggered by udev (and the configured probes) while bringing
+ * the resource "up", just after "new-minor", even before "attach" or any
+ * "peers"/"paths" are configured.
  */
-int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
-		  size_t size, unsigned msg_flags)
+static bool connection_state_may_improve_soon(struct drbd_resource *resource)
 {
-	int err;
-
-	err = drbd_send(connection, sock, buffer, size, msg_flags);
-	if (err < 0)
-		return err;
-	if (err != size)
-		return -EIO;
-	return 0;
+	struct drbd_connection *connection;
+	bool ret = false;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		enum drbd_conn_state cstate = connection->cstate[NOW];
+		if (C_DISCONNECTING < cstate && cstate < C_CONNECTED) {
+			ret = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+	return ret;
 }
 
-static int drbd_open(struct gendisk *disk, blk_mode_t mode)
+/* TASK_COMM_LEN reserves one '\0', sizeof("") both include '\0',
+ * that's room enough for ':' and ' ' separators and the EOS.
+ */
+union comm_pid_tag_buf {
+	char comm[TASK_COMM_LEN];
+	char buf[TASK_COMM_LEN + sizeof("2147483647") + sizeof("auto-promote")];
+};
+
+static void snprintf_current_comm_pid_tag(union comm_pid_tag_buf *s, const char *tag)
 {
-	struct drbd_device *device = disk->private_data;
-	unsigned long flags;
-	int rv = 0;
+	int len;
 
-	mutex_lock(&drbd_main_mutex);
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	/* to have a stable device->state.role
-	 * and no race with updating open_cnt */
+	get_task_comm(s->comm, current);
+	len = strlen(s->buf);
+	snprintf(s->buf + len, sizeof(s->buf)-len, ":%d %s", task_pid_nr(current), tag);
+}
 
-	if (device->state.role != R_PRIMARY) {
-		if (mode & BLK_OPEN_WRITE)
-			rv = -EROFS;
-		else if (!drbd_allow_oos)
-			rv = -EMEDIUMTYPE;
-	}
+static int try_to_promote(struct drbd_device *device, long timeout, bool ndelay)
+{
+	struct drbd_resource *resource = device->resource;
+	int rv;
 
-	if (!rv)
-		device->open_cnt++;
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-	mutex_unlock(&drbd_main_mutex);
+	do {
+		union comm_pid_tag_buf tag;
+		unsigned long start = jiffies;
+		long t;
 
+		snprintf_current_comm_pid_tag(&tag, "auto-promote");
+		rv = drbd_set_role(resource, R_PRIMARY, false, tag.buf, NULL);
+		timeout -= jiffies - start;
+
+		if (ndelay || rv >= SS_SUCCESS || timeout <= 0) {
+			break;
+		} else if (rv == SS_CW_FAILED_BY_PEER) {
+			/* Probably udev has it open read-only on one of the peers;
+			   since commit cbcbb50a65 from 2017 it waits on the peer;
+			   retry only if the timeout permits */
+			if (jiffies - start < HZ / 10) {
+				t = schedule_timeout_interruptible(HZ / 10);
+				if (t)
+					break;
+				timeout -= HZ / 10;
+			}
+		} else if (rv == SS_TWO_PRIMARIES) {
+			/* Wait till the peer demoted itself */
+			t = wait_event_interruptible_timeout(resource->state_wait,
+				resource->role[NOW] == R_PRIMARY ||
+				(!primary_peer_present(resource) && any_disk_is_uptodate(device)),
+				timeout);
+			if (t <= 0)
+				break;
+			timeout -= t;
+		} else if (rv == SS_NO_UP_TO_DATE_DISK && connection_state_may_improve_soon(resource)) {
+			/* Wait until we get a connection established */
+			t = wait_event_interruptible_timeout(resource->state_wait,
+				any_disk_is_uptodate(device), timeout);
+			if (t <= 0)
+				break;
+			timeout -= t;
+		} else {
+			break;
+		}
+	} while (timeout > 0);
 	return rv;
 }
 
-static void drbd_release(struct gendisk *gd)
+static int ro_open_cond(struct drbd_device *device)
 {
-	struct drbd_device *device = gd->private_data;
+	struct drbd_resource *resource = device->resource;
 
-	mutex_lock(&drbd_main_mutex);
-	device->open_cnt--;
-	mutex_unlock(&drbd_main_mutex);
+	if (!device->have_quorum[NOW])
+		return -ENODATA;
+	else if (resource->role[NOW] != R_PRIMARY &&
+		primary_peer_present(resource) && !drbd_allow_oos)
+		return -EMEDIUMTYPE;
+	else if (any_disk_is_uptodate(device))
+		return 0;
+	else if (connection_state_may_improve_soon(resource))
+		return -EAGAIN;
+	else
+		return -ENODATA;
 }
 
-/* need to hold resource->req_lock */
-void drbd_queue_unplug(struct drbd_device *device)
+enum ioc_rv {
+	IOC_SLEEP = 0,
+	IOC_OK = 1,
+	IOC_ABORT = 2,
+};
+
+/* If we are in the middle of a cluster wide state change, we don't want
+ * to change (open_cnt == 0), as that then could cause a failure to commit
+ * some already promised peer auto-promote locally.
+ * So we wait until the pending remote_state_change is finalized,
+ * or give up when the timeout is reached.
+ *
+ * But we don't want to fail an open on a Primary just because it happens
+ * during some unrelated remote state change.
+ * If we are already Primary, or already have an open count != 0,
+ * we don't need to wait, it won't change anything.
+ */
+static enum ioc_rv inc_open_count(struct drbd_device *device, blk_mode_t mode)
 {
-	if (device->state.pdsk >= D_INCONSISTENT && device->state.conn >= C_CONNECTED) {
-		D_ASSERT(device, device->state.role == R_PRIMARY);
-		if (test_and_clear_bit(UNPLUG_REMOTE, &device->flags)) {
-			drbd_queue_work_if_unqueued(
-				&first_peer_device(device)->connection->sender_work,
-				&device->unplug_work);
-		}
+	struct drbd_resource *resource = device->resource;
+	enum ioc_rv r;
+
+	if (test_bit(DOWN_IN_PROGRESS, &resource->flags))
+		return IOC_ABORT;
+
+	read_lock_irq(&resource->state_rwlock);
+	if (test_bit(UNREGISTERED, &device->flags))
+		r = IOC_ABORT;
+	else if (resource->remote_state_change &&
+		resource->role[NOW] != R_PRIMARY &&
+		(device->open_cnt == 0 || mode & BLK_OPEN_WRITE)) {
+		if (mode & BLK_OPEN_NDELAY)
+			r = IOC_ABORT;
+		else
+			r = IOC_SLEEP;
+	} else {
+		r = IOC_OK;
+		device->open_cnt++;
+		if (mode & BLK_OPEN_WRITE)
+			device->writable = true;
 	}
-}
+	read_unlock_irq(&resource->state_rwlock);
 
-static void drbd_set_defaults(struct drbd_device *device)
-{
-	/* Beware! The actual layout differs
-	 * between big endian and little endian */
-	device->state = (union drbd_dev_state) {
-		{ .role = R_SECONDARY,
-		  .peer = R_UNKNOWN,
-		  .conn = C_STANDALONE,
-		  .disk = D_DISKLESS,
-		  .pdsk = D_UNKNOWN,
-		} };
+	return r;
 }
 
-void drbd_init_set_defaults(struct drbd_device *device)
+static void __prune_or_free_openers(struct drbd_device *device, pid_t pid)
 {
-	/* the memset(,0,) did most of this.
-	 * note: only assignments, no allocation in here */
-
-	drbd_set_defaults(device);
-
-	atomic_set(&device->ap_bio_cnt, 0);
-	atomic_set(&device->ap_actlog_cnt, 0);
-	atomic_set(&device->ap_pending_cnt, 0);
-	atomic_set(&device->rs_pending_cnt, 0);
-	atomic_set(&device->unacked_cnt, 0);
-	atomic_set(&device->local_cnt, 0);
-	atomic_set(&device->pp_in_use_by_net, 0);
-	atomic_set(&device->rs_sect_in, 0);
-	atomic_set(&device->rs_sect_ev, 0);
-	atomic_set(&device->ap_in_flight, 0);
-	atomic_set(&device->md_io.in_use, 0);
-
-	mutex_init(&device->own_state_mutex);
-	device->state_mutex = &device->own_state_mutex;
-
-	spin_lock_init(&device->al_lock);
-	spin_lock_init(&device->peer_seq_lock);
-
-	INIT_LIST_HEAD(&device->active_ee);
-	INIT_LIST_HEAD(&device->sync_ee);
-	INIT_LIST_HEAD(&device->done_ee);
-	INIT_LIST_HEAD(&device->read_ee);
-	INIT_LIST_HEAD(&device->resync_reads);
-	INIT_LIST_HEAD(&device->resync_work.list);
-	INIT_LIST_HEAD(&device->unplug_work.list);
-	INIT_LIST_HEAD(&device->bm_io_work.w.list);
-	INIT_LIST_HEAD(&device->pending_master_completion[0]);
-	INIT_LIST_HEAD(&device->pending_master_completion[1]);
-	INIT_LIST_HEAD(&device->pending_completion[0]);
-	INIT_LIST_HEAD(&device->pending_completion[1]);
+	struct opener *pos, *tmp;
 
-	device->resync_work.cb  = w_resync_timer;
-	device->unplug_work.cb  = w_send_write_hint;
-	device->bm_io_work.w.cb = w_bitmap_io;
+	list_for_each_entry_safe(pos, tmp, &device->openers, list) {
+		// if pid == 0, i.e., counts were 0, delete all entries, else the matching one
+		if (pid == 0 || pid == pos->pid) {
+			dynamic_drbd_dbg(device, "%sopeners del: %s(%d)\n", pid == 0 ? "" : "all ",
+					pos->comm, pos->pid);
+			list_del(&pos->list);
+			kfree(pos);
 
-	timer_setup(&device->resync_timer, resync_timer_fn, 0);
-	timer_setup(&device->md_sync_timer, md_sync_timer_fn, 0);
-	timer_setup(&device->start_resync_timer, start_resync_timer_fn, 0);
-	timer_setup(&device->request_timer, request_timer_fn, 0);
+			/* in case we remove a real process, stop here, there might be multiple openers with the same pid */
+			/* this assumes that the oldest opener with the same pid releases first. "as good as it gets" */
+			if (pid != 0)
+				break;
+		}
+	}
+}
 
-	init_waitqueue_head(&device->misc_wait);
-	init_waitqueue_head(&device->state_wait);
-	init_waitqueue_head(&device->ee_wait);
-	init_waitqueue_head(&device->al_wait);
-	init_waitqueue_head(&device->seq_wait);
+static void free_openers(struct drbd_device *device)
+{
+	__prune_or_free_openers(device, 0);
+}
 
-	device->resync_wenr = LC_FREE;
-	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
-	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
+static void prune_or_free_openers(struct drbd_device *device, pid_t pid)
+{
+	spin_lock(&device->openers_lock);
+	__prune_or_free_openers(device, pid);
+	spin_unlock(&device->openers_lock);
 }
 
-void drbd_set_my_capacity(struct drbd_device *device, sector_t size)
+static void add_opener(struct drbd_device *device, bool did_auto_promote)
 {
-	char ppb[10];
+	struct opener *opener, *tmp;
+	ktime_t now = ktime_get_real();
+	int len = 0;
+
+	if (did_auto_promote) {
+		struct drbd_resource *resource = device->resource;
 
-	set_capacity_and_notify(device->vdisk, size);
+		resource->auto_promoted_by.minor = device->minor;
+		resource->auto_promoted_by.pid = task_pid_nr(current);
+		resource->auto_promoted_by.opened = now;
+		get_task_comm(resource->auto_promoted_by.comm, current);
+	}
+	opener = kmalloc_obj(*opener, GFP_NOIO);
+	if (!opener)
+		return;
+	get_task_comm(opener->comm, current);
+	opener->pid = task_pid_nr(current);
+	opener->opened = now;
+
+	spin_lock(&device->openers_lock);
+	list_for_each_entry(tmp, &device->openers, list)
+		if (++len > 100) { /* 100 ought to be enough for everybody */
+			dynamic_drbd_dbg(device, "openers: list full, do not add new opener\n");
+			kfree(opener);
+			goto out;
+		}
 
-	drbd_info(device, "size = %s (%llu KB)\n",
-		ppsize(ppb, size>>1), (unsigned long long)size>>1);
+	list_add(&opener->list, &device->openers);
+	dynamic_drbd_dbg(device, "openers add: %s(%d)\n", opener->comm, opener->pid);
+out:
+	spin_unlock(&device->openers_lock);
 }
 
-void drbd_device_cleanup(struct drbd_device *device)
+static int drbd_open(struct gendisk *gd, blk_mode_t mode)
 {
-	int i;
-	if (first_peer_device(device)->connection->receiver.t_state != NONE)
-		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
-				first_peer_device(device)->connection->receiver.t_state);
-
-	device->al_writ_cnt  =
-	device->bm_writ_cnt  =
-	device->read_cnt     =
-	device->recv_cnt     =
-	device->send_cnt     =
-	device->writ_cnt     =
-	device->p_size       =
-	device->rs_start     =
-	device->rs_total     =
-	device->rs_failed    = 0;
-	device->rs_last_events = 0;
-	device->rs_last_sect_ev = 0;
-	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
-		device->rs_mark_left[i] = 0;
-		device->rs_mark_time[i] = 0;
-	}
-	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
-
-	set_capacity_and_notify(device->vdisk, 0);
-	if (device->bitmap) {
-		/* maybe never allocated. */
-		drbd_bm_resize(device, 0, 1);
-		drbd_bm_cleanup(device);
+	struct drbd_device *device = gd->private_data;
+	struct drbd_resource *resource = device->resource;
+	long timeout = resource->res_opts.auto_promote_timeout * HZ / 10;
+	enum drbd_state_rv rv = SS_UNKNOWN_ERROR;
+	bool was_writable;
+	enum ioc_rv r;
+	int err = 0;
+
+	/* Fail read-only open from systemd-udev (version <= 238) */
+	if (!(mode & BLK_OPEN_WRITE) && !drbd_allow_oos) {
+		char comm[TASK_COMM_LEN];
+		get_task_comm(comm, current);
+		if (!strcmp("systemd-udevd", comm))
+			return -EACCES;
 	}
 
-	drbd_backing_dev_free(device, device->ldev);
-	device->ldev = NULL;
+	/* Fail read-write open early,
+	 * in case someone explicitly set us read-only (blockdev --setro) */
+	if (bdev_read_only(gd->part0) && (mode & BLK_OPEN_WRITE))
+		return -EACCES;
 
-	clear_bit(AL_SUSPENDED, &device->flags);
+	if (resource->fail_io[NOW])
+		return -ENOTRECOVERABLE;
 
-	D_ASSERT(device, list_empty(&device->active_ee));
-	D_ASSERT(device, list_empty(&device->sync_ee));
-	D_ASSERT(device, list_empty(&device->done_ee));
-	D_ASSERT(device, list_empty(&device->read_ee));
-	D_ASSERT(device, list_empty(&device->resync_reads));
-	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
-	D_ASSERT(device, list_empty(&device->resync_work.list));
-	D_ASSERT(device, list_empty(&device->unplug_work.list));
+	kref_get(&device->kref);
+
+	mutex_lock(&resource->open_release);
+	was_writable = device->writable;
+
+	timeout = wait_event_interruptible_timeout(resource->twopc_wait,
+						   (r = inc_open_count(device, mode)),
+						   timeout);
+
+	if (r == IOC_ABORT || (r == IOC_SLEEP && timeout <= 0)) {
+		mutex_unlock(&resource->open_release);
+
+		kref_put(&device->kref, drbd_destroy_device);
+		return -EAGAIN;
+	}
+
+	if (resource->res_opts.auto_promote) {
+		/* Allow opening in read-only mode on an unconnected secondary.
+		   This avoids split brain when the drbd volume gets opened
+		   temporarily by udev while it scans for PV signatures. */
+
+		if (mode & BLK_OPEN_WRITE) {
+			if (resource->role[NOW] == R_SECONDARY) {
+				rv = try_to_promote(device, timeout, (mode & BLK_OPEN_NDELAY));
+				if (rv < SS_SUCCESS)
+					drbd_info(resource, "Auto-promote failed: %s (%d)\n",
+						  drbd_set_st_err_str(rv), rv);
+			}
+		} else if ((mode & BLK_OPEN_NDELAY) == 0) {
+			/* Double check peers
+			 *
+			 * Some services may try to first open ro, and only if that
+			 * works open rw.  An attempt to failover immediately after
+			 * primary crash, before DRBD has noticed that the primary peer
+			 * is gone, would result in open failure, thus failure to take
+			 * over services. */
+			err = ro_open_cond(device);
+			if (err == -EMEDIUMTYPE) {
+				drbd_check_peers(resource);
+				err = -EAGAIN;
+			}
+			if (err == -EAGAIN) {
+				wait_event_interruptible_timeout(resource->state_wait,
+					ro_open_cond(device) != -EAGAIN,
+					resource->res_opts.auto_promote_timeout * HZ / 10);
+			}
+		}
+	} else if (resource->role[NOW] != R_PRIMARY &&
+			!(mode & BLK_OPEN_WRITE) && !drbd_allow_oos) {
+		err = -EMEDIUMTYPE;
+		goto out;
+	}
+
+	if (test_bit(UNREGISTERED, &device->flags)) {
+		err = -ENODEV;
+	} else if (mode & BLK_OPEN_WRITE) {
+		if (resource->role[NOW] != R_PRIMARY)
+			err = rv == SS_INTERRUPTED ? -ERESTARTSYS : -EROFS;
+	} else /* READ access only */ {
+		err = ro_open_cond(device);
+	}
+out:
+	/* still keep mutex, but release ASAP */
+	if (!err) {
+		add_opener(device, rv >= SS_SUCCESS);
+		/* Only interested in first open and last close. */
+		if (device->open_cnt == 1) {
+			struct device_info info;
+
+			device_to_info(&info, device);
+			mutex_lock(&notification_mutex);
+			notify_device_state(NULL, 0, device, &info, NOTIFY_CHANGE);
+			mutex_unlock(&notification_mutex);
+		}
+	} else
+		device->writable = was_writable;
+
+	mutex_unlock(&resource->open_release);
+	if (err) {
+		drbd_release(gd);
+		if (err == -EAGAIN && !(mode & BLK_OPEN_NDELAY))
+			err = -EMEDIUMTYPE;
+	}
+
+	return err;
+}
+
+void drbd_open_counts(struct drbd_resource *resource, int *rw_count_ptr, int *ro_count_ptr)
+{
+	struct drbd_device *device;
+	int vnr, rw_count = 0, ro_count = 0;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (device->writable)
+			rw_count += device->open_cnt;
+		else
+			ro_count += device->open_cnt;
+	}
+	rcu_read_unlock();
+	*rw_count_ptr = rw_count;
+	*ro_count_ptr = ro_count;
+}
+
+static void wait_for_peer_disk_updates(struct drbd_resource *resource)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	int vnr;
+
+restart:
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		for_each_peer_device_rcu(peer_device, device) {
+			if (test_bit(GOT_NEG_ACK, &peer_device->flags)) {
+				clear_bit(GOT_NEG_ACK, &peer_device->flags);
+				rcu_read_unlock();
+				wait_event(resource->state_wait, peer_device->disk_state[NOW] < D_UP_TO_DATE);
+				goto restart;
+			}
+		}
+	}
+	rcu_read_unlock();
+}
+
+static void drbd_fsync_device(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+
+	sync_blockdev(device->vdisk->part0);
+	/* Prevent writes occurring after demotion, at least
+	 * the writes already submitted in this context. This
+	 * covers the case where DRBD auto-demotes on release,
+	 * which is important because it often occurs
+	 * immediately after a write. */
+	wait_event(device->misc_wait, !atomic_read(&device->ap_bio_cnt[WRITE]));
+
+	if (start_new_tl_epoch(resource)) {
+		struct drbd_connection *connection;
+		u64 im;
+
+		for_each_connection_ref(connection, im, resource)
+			drbd_flush_workqueue(&connection->sender_work);
+	}
+	wait_event(resource->barrier_wait, !barrier_pending(resource));
+	/* After waiting for pending barriers, we got any possible NEG_ACKs,
+	   and see them in wait_for_peer_disk_updates() */
+	wait_for_peer_disk_updates(resource);
+
+	/* In case switching from R_PRIMARY to R_SECONDARY works
+	   out, there is no rw opener at this point. Thus, no new
+	   writes can come in. -> Flushing queued peer acks is
+	   necessary and sufficient.
+	   The cluster wide role change required packets to be
+	   received by the sender. -> We can be sure that the
+	   peer_acks queued on a sender's TODO list go out before
+	   we send the two phase commit packet.
+	*/
+	drbd_flush_peer_acks(resource);
+}
+
+static void drbd_release(struct gendisk *gd)
+{
+	struct drbd_device *device = gd->private_data;
+	struct drbd_resource *resource = device->resource;
+	int open_rw_cnt, open_ro_cnt;
+
+	mutex_lock(&resource->open_release);
+	/* The last one to close already called sync_blockdevice(), generic
+	 * bdev_release() respectively blkdev_put_whole() takes care of that.
+	 * We still want our side effects of drbd_fsync_device():
+	 * wait until all peers confirmed they have all the data, regardless of
+	 * replication protocol, even if that is asynchronous.
+	 * Still, do it before decreasing the open_cnt, just in case, so we
+	 * won't confuse drbd_reject_write_early() or other code paths that may
+	 * check for open_cnt != 0 when they see write requests.
+	 */
+	if (device->writable && device->open_cnt == 1) {
+		drbd_fsync_device(device);
+		device->writable = false;
+	}
+	device->open_cnt--;
+	drbd_open_counts(resource, &open_rw_cnt, &open_ro_cnt);
+
+	if (open_ro_cnt == 0)
+		wake_up_all(&resource->state_wait);
+
+	if (test_bit(UNREGISTERED, &device->flags) && device->open_cnt == 0 &&
+	    !test_and_set_bit(DESTROYING_DEV, &device->flags))
+		call_rcu(&device->rcu, drbd_reclaim_device);
+
+	if (resource->res_opts.auto_promote &&
+			open_rw_cnt == 0 &&
+			resource->role[NOW] == R_PRIMARY &&
+			!test_bit(EXPLICIT_PRIMARY, &resource->flags)) {
+		union comm_pid_tag_buf tag;
+		sigset_t mask, oldmask;
+		int rv;
+
+		snprintf_current_comm_pid_tag(&tag, "auto-demote");
+
+		/*
+		 * Auto-demote is triggered by the last opener releasing the
+		 * DRBD device. However, it is an implicit action, so it should
+		 * not be affected by the state of the process. In particular,
+		 * it should ignore any pending signals. It may be the case
+		 * that the process is releasing DRBD because it is being
+		 * terminated using a signal.
+		 */
+		sigfillset(&mask);
+		sigprocmask(SIG_BLOCK, &mask, &oldmask);
+
+		rv = drbd_set_role(resource, R_SECONDARY, false, tag.buf, NULL);
+		if (rv < SS_SUCCESS)
+			drbd_warn(resource, "Auto-demote failed: %s (%d)\n",
+					drbd_set_st_err_str(rv), rv);
+
+		sigprocmask(SIG_SETMASK, &oldmask, NULL);
+	}
+
+	if (open_ro_cnt == 0 && open_rw_cnt == 0 && resource->fail_io[NOW]) {
+		unsigned long irq_flags;
+
+		begin_state_change(resource, &irq_flags, CS_VERBOSE);
+		resource->fail_io[NEW] = false;
+		end_state_change(resource, &irq_flags, "release");
+	}
+
+	/* if the open count is 0, we free the whole list, otherwise we remove the specific pid */
+	prune_or_free_openers(device, (device->open_cnt == 0) ? 0 : task_pid_nr(current));
+	if (open_rw_cnt == 0 && open_ro_cnt == 0 && resource->auto_promoted_by.pid != 0)
+		memset(&resource->auto_promoted_by, 0, sizeof(resource->auto_promoted_by));
+	if (device->open_cnt == 0) {
+		struct device_info info;
+
+		device_to_info(&info, device);
+		mutex_lock(&notification_mutex);
+		notify_device_state(NULL, 0, device, &info, NOTIFY_CHANGE);
+		mutex_unlock(&notification_mutex);
+	}
+	mutex_unlock(&resource->open_release);
+
+	kref_put(&device->kref, drbd_destroy_device);  /* might destroy the resource as well */
+}
+
+static void drbd_remove_all_paths(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_path *path, *tmp;
+
+	lockdep_assert_held(&resource->conf_update);
+
+	list_for_each_entry(path, &transport->paths, list)
+		set_bit(TR_UNREGISTERED, &path->flags);
+
+	/* Ensure flag visible before list manipulation. */
+	smp_wmb();
+
+	list_for_each_entry_safe(path, tmp, &transport->paths, list) {
+		/* Exclusive with reading state, in particular remember_state_change() */
+		write_lock_irq(&resource->state_rwlock);
+		list_del_rcu(&path->list);
+		write_unlock_irq(&resource->state_rwlock);
+
+		transport->class->ops.remove_path(path);
+		notify_path(connection, path, NOTIFY_DESTROY);
+		call_rcu(&path->rcu, drbd_reclaim_path);
+	}
+}
+
+/** __drbd_net_exit is called when a network namespace is removed.
+ *
+ * For DRBD this means it needs to remove any sockets assigned to that namespace,
+ * i.e. it needs to disconnect some connections. It also needs to remove those
+ * paths associated with the to be removed namespace, so the connection can be
+ * reconfigured from a new namespace.
+ */
+static void __net_exit __drbd_net_exit(struct net *net)
+{
+	struct drbd_resource *resource;
+	struct drbd_connection *connection, *n;
+	enum drbd_state_rv rv;
+	LIST_HEAD(connections_wait_list);
+
+	/* Disconnect and removal of paths works in 3 steps:
+	 * 1. Find all connections associated with the namespace, add it to a separate list.
+	 * 2. Iterate over all connections in the new list and start the disconnect.
+	 * 3. Iterate again over all connections, waiting for them to disconnect and remove the path configuration.*/
+
+	/* Step 1 */
+	rcu_read_lock();
+	for_each_resource_rcu(resource, &drbd_resources) {
+		for_each_connection_rcu(connection, resource) {
+			/* We don't have to worry about any races here:
+			 * For a connection to be "missed", it would need to be configured
+			 * from the namespace to be removed. Since netlink does keep the
+			 * namespace alive for the duration of it's connection, we can
+			 * assume the namespace assignment can no longer be changed. */
+			if (net_eq(net, drbd_net_assigned_to_connection(connection))) {
+				drbd_info(connection, "Disconnect because network namespace is exiting\n");
+
+				kref_get(&connection->kref);
+
+				list_add(&connection->remove_net_list, &connections_wait_list);
+			}
+		}
+	}
+	rcu_read_unlock();
+
+	/* Step 2 */
+	list_for_each_entry(connection, &connections_wait_list, remove_net_list) {
+		/* We just start the disconnect here.  We have to use force=true here,
+		 * otherwise the disconnect might fail waiting for some openers to disappear.
+		 *
+		 * Actually waiting for the disconnect is relegated to step 3, so we disconnect
+		 * in parallel. */
+		rv = change_cstate(connection, C_DISCONNECTING, CS_HARD);
+		if (rv < SS_SUCCESS && rv != SS_ALREADY_STANDALONE)
+			drbd_err(connection, "Failed to disconnect: %s\n", drbd_set_st_err_str(rv));
+	}
+
+	/* Step 3 */
+	list_for_each_entry_safe(connection, n, &connections_wait_list, remove_net_list) {
+		list_del_init(&connection->remove_net_list);
+
+		/* Wait here for StandAlone: a path can only be removed if it's not established */
+		wait_event(connection->resource->state_wait, connection->cstate[NOW] == C_STANDALONE);
+
+		mutex_lock(&connection->resource->adm_mutex);
+		mutex_lock(&connection->resource->conf_update);
+		drbd_remove_all_paths(connection);
+		mutex_unlock(&connection->resource->conf_update);
+		mutex_unlock(&connection->resource->adm_mutex);
+
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
+}
+
+void drbd_queue_unplug(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	struct drbd_connection *connection;
+	u64 dagtag_sector;
+
+	dagtag_sector = resource->dagtag_sector;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		/* use the "next" slot */
+		unsigned int i = !connection->todo.unplug_slot;
+		connection->todo.unplug_dagtag_sector[i] = dagtag_sector;
+		wake_up(&connection->sender_work.q_wait);
+	}
+	rcu_read_unlock();
+}
+
+static void drbd_set_defaults(struct drbd_device *device)
+{
+	device->disk_state[NOW] = D_DISKLESS;
+}
+
+void drbd_cleanup_device(struct drbd_device *device)
+{
+	device->al_writ_cnt = 0;
+	device->bm_writ_cnt = 0;
+	device->read_cnt = 0;
+	device->writ_cnt = 0;
+
+	drbd_bm_free(device);
 
+	clear_bit(AL_SUSPENDED, &device->flags);
 	drbd_set_defaults(device);
 }
 
 
 static void drbd_destroy_mempools(void)
 {
-	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
-
 	bioset_exit(&drbd_io_bio_set);
 	bioset_exit(&drbd_md_io_bio_set);
 	mempool_exit(&drbd_buffer_page_pool);
 	mempool_exit(&drbd_md_io_page_pool);
 	mempool_exit(&drbd_ee_mempool);
 	mempool_exit(&drbd_request_mempool);
-	kmem_cache_destroy(drbd_ee_cache);
-	kmem_cache_destroy(drbd_request_cache);
-	kmem_cache_destroy(drbd_bm_ext_cache);
-	kmem_cache_destroy(drbd_al_ext_cache);
+	if (drbd_ee_cache)
+		kmem_cache_destroy(drbd_ee_cache);
+	if (drbd_request_cache)
+		kmem_cache_destroy(drbd_request_cache);
+	if (drbd_al_ext_cache)
+		kmem_cache_destroy(drbd_al_ext_cache);
 
 	drbd_ee_cache        = NULL;
 	drbd_request_cache   = NULL;
-	drbd_bm_ext_cache    = NULL;
 	drbd_al_ext_cache    = NULL;
 
 	return;
@@ -2090,11 +3361,6 @@ static int drbd_create_mempools(void)
 	if (drbd_ee_cache == NULL)
 		goto Enomem;
 
-	drbd_bm_ext_cache = kmem_cache_create(
-		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
-	if (drbd_bm_ext_cache == NULL)
-		goto Enomem;
-
 	drbd_al_ext_cache = kmem_cache_create(
 		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
 	if (drbd_al_ext_cache == NULL)
@@ -2113,7 +3379,6 @@ static int drbd_create_mempools(void)
 	ret = mempool_init_page_pool(&drbd_md_io_page_pool, DRBD_MIN_POOL_PAGES, 0);
 	if (ret)
 		goto Enomem;
-
 	ret = mempool_init_page_pool(&drbd_buffer_page_pool, number, 0);
 	if (ret)
 		goto Enomem;
@@ -2134,70 +3399,76 @@ static int drbd_create_mempools(void)
 	return -ENOMEM;
 }
 
-static void drbd_release_all_peer_reqs(struct drbd_device *device)
+static void free_peer_device(struct drbd_peer_device *peer_device)
 {
-	int rr;
+	if (test_and_clear_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags))
+		up_read_non_owner(&peer_device->device->uuid_sem);
 
-	rr = drbd_free_peer_reqs(device, &device->active_ee);
-	if (rr)
-		drbd_err(device, "%d EEs in active list found!\n", rr);
+	kfree(peer_device->rs_plan_s);
+	kfree(peer_device->conf);
+	kfree(peer_device);
+}
 
-	rr = drbd_free_peer_reqs(device, &device->sync_ee);
-	if (rr)
-		drbd_err(device, "%d EEs in sync list found!\n", rr);
+static void drbd_device_finalize_work_fn(struct work_struct *work)
+{
+	struct drbd_device *device = container_of(work, struct drbd_device, finalize_work);
+	struct drbd_resource *resource = device->resource;
 
-	rr = drbd_free_peer_reqs(device, &device->read_ee);
-	if (rr)
-		drbd_err(device, "%d EEs in read list found!\n", rr);
+	drbd_bm_free(device);
 
-	rr = drbd_free_peer_reqs(device, &device->done_ee);
-	if (rr)
-		drbd_err(device, "%d EEs in done list found!\n", rr);
+	put_disk(device->vdisk);
+
+	kfree(device);
+
+	kref_put(&resource->kref, drbd_destroy_resource);
 }
 
-/* caution. no locking. */
+/* may not sleep, called from call_rcu. */
 void drbd_destroy_device(struct kref *kref)
 {
 	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
-	struct drbd_resource *resource = device->resource;
-	struct drbd_peer_device *peer_device, *tmp_peer_device;
-
-	timer_shutdown_sync(&device->request_timer);
-
-	/* paranoia asserts */
-	D_ASSERT(device, device->open_cnt == 0);
-	/* end paranoia asserts */
+	struct drbd_peer_device *peer_device, *tmp;
 
 	/* cleanup stuff that may have been allocated during
 	 * device (re-)configuration or state changes */
 
-	drbd_backing_dev_free(device, device->ldev);
-	device->ldev = NULL;
+#ifdef CONFIG_DRBD_COMPAT_84
+	if (device->resource->res_opts.drbd8_compat_mode)
+		atomic_dec(&nr_drbd8_devices);
+#endif
 
-	drbd_release_all_peer_reqs(device);
+	free_openers(device);
 
 	lc_destroy(device->act_log);
-	lc_destroy(device->resync);
-
-	kfree(device->p_uuid);
-	/* device->p_uuid = NULL; */
+	for_each_peer_device_safe(peer_device, tmp, device) {
+		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
+		free_peer_device(peer_device);
+	}
 
-	if (device->bitmap) /* should no longer be there. */
-		drbd_bm_cleanup(device);
 	__free_page(device->md_io.page);
-	put_disk(device->vdisk);
-	kfree(device->rs_plan_s);
 
-	/* not for_each_connection(connection, resource):
-	 * those may have been cleaned up and disassociated already.
-	 */
-	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
-		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
-		kfree(peer_device);
-	}
-	if (device->submit.wq)
-		destroy_workqueue(device->submit.wq);
-	kfree(device);
+	INIT_WORK(&device->finalize_work, drbd_device_finalize_work_fn);
+	schedule_work(&device->finalize_work);
+}
+
+void drbd_destroy_resource(struct kref *kref)
+{
+	struct drbd_resource *resource = container_of(kref, struct drbd_resource, kref);
+
+	idr_destroy(&resource->devices);
+	free_cpumask_var(resource->cpu_mask);
+	kfree(resource->name);
+	kfree(resource);
+	module_put(THIS_MODULE);
+}
+
+void drbd_reclaim_resource(struct rcu_head *rp)
+{
+	struct drbd_resource *resource = container_of(rp, struct drbd_resource, rcu);
+
+	drbd_thread_stop_nowait(&resource->worker);
+
+	mempool_free(resource->peer_ack_req, &drbd_request_mempool);
 	kref_put(&resource->kref, drbd_destroy_resource);
 }
 
@@ -2222,96 +3493,88 @@ static void do_retry(struct work_struct *ws)
 	list_splice_init(&retry->writes, &writes);
 	spin_unlock_irq(&retry->lock);
 
-	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
+	list_for_each_entry_safe(req, tmp, &writes, list) {
 		struct drbd_device *device = req->device;
+		struct drbd_resource *resource = device->resource;
 		struct bio *bio = req->master_bio;
+		unsigned long start_jif = req->start_jif;
 		bool expected;
+		ktime_get_accounting_assign(ktime_t start_kt, req->start_kt);
+
 
+		/* No locking when accessing local_rq_state & net_rq_state, since
+		 * this request is not active at the moment. */
 		expected =
 			expect(device, atomic_read(&req->completion_ref) == 0) &&
-			expect(device, req->rq_state & RQ_POSTPONED) &&
-			expect(device, (req->rq_state & RQ_LOCAL_PENDING) == 0 ||
-				(req->rq_state & RQ_LOCAL_ABORTED) != 0);
+			expect(device, req->local_rq_state & RQ_POSTPONED) &&
+			expect(device, (req->local_rq_state & RQ_LOCAL_PENDING) == 0 ||
+			       (req->local_rq_state & RQ_LOCAL_ABORTED) != 0);
 
 		if (!expected)
 			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
 				req, atomic_read(&req->completion_ref),
-				req->rq_state);
+				req->local_rq_state);
 
-		/* We still need to put one kref associated with the
+		/* We still need to put one done reference associated with the
 		 * "completion_ref" going zero in the code path that queued it
 		 * here.  The request object may still be referenced by a
 		 * frozen local req->private_bio, in case we force-detached.
 		 */
-		kref_put(&req->kref, drbd_req_destroy);
+		read_lock_irq(&resource->state_rwlock);
+		drbd_put_ref_tl_walk(req, 1, 0);
+		read_unlock_irq(&resource->state_rwlock);
 
 		/* A single suspended or otherwise blocking device may stall
-		 * all others as well.  Fortunately, this code path is to
-		 * recover from a situation that "should not happen":
-		 * concurrent writes in multi-primary setup.
-		 * In a "normal" lifecycle, this workqueue is supposed to be
-		 * destroyed without ever doing anything.
-		 * If it turns out to be an issue anyways, we can do per
+		 * all others as well. This code path is to recover from a
+		 * situation that "should not happen": concurrent writes in
+		 * multi-primary setup. It is also used for retrying failed
+		 * reads. If it turns out to be an issue, we can do per
 		 * resource (replication group) or per device (minor) retry
 		 * workqueues instead.
 		 */
 
 		/* We are not just doing submit_bio_noacct(),
 		 * as we want to keep the start_time information. */
-		inc_ap_bio(device);
-		__drbd_make_request(device, bio);
+		__drbd_make_request(device, bio, start_kt, start_jif);
 	}
 }
 
-/* called via drbd_req_put_completion_ref(),
- * holds resource->req_lock */
+/* called via drbd_req_put_completion_ref() */
 void drbd_restart_request(struct drbd_request *req)
 {
+	struct drbd_device *device = req->device;
+	struct drbd_resource *resource = device->resource;
+	bool susp = drbd_suspended(device);
 	unsigned long flags;
+
 	spin_lock_irqsave(&retry.lock, flags);
-	list_move_tail(&req->tl_requests, &retry.writes);
+	list_move_tail(&req->list, susp ? &resource->suspended_reqs : &retry.writes);
 	spin_unlock_irqrestore(&retry.lock, flags);
 
 	/* Drop the extra reference that would otherwise
 	 * have been dropped by complete_master_bio.
 	 * do_retry() needs to grab a new one. */
-	dec_ap_bio(req->device);
+	dec_ap_bio(req->device, bio_data_dir(req->master_bio));
 
-	queue_work(retry.wq, &retry.worker);
+	if (!susp)
+		queue_work(retry.wq, &retry.worker);
 }
 
-void drbd_destroy_resource(struct kref *kref)
+void drbd_restart_suspended_reqs(struct drbd_resource *resource)
 {
-	struct drbd_resource *resource =
-		container_of(kref, struct drbd_resource, kref);
-
-	idr_destroy(&resource->devices);
-	free_cpumask_var(resource->cpu_mask);
-	kfree(resource->name);
-	kfree(resource);
-}
+	unsigned long flags;
 
-void drbd_free_resource(struct drbd_resource *resource)
-{
-	struct drbd_connection *connection, *tmp;
+	spin_lock_irqsave(&retry.lock, flags);
+	list_splice_init(&resource->suspended_reqs, &retry.writes);
+	spin_unlock_irqrestore(&retry.lock, flags);
 
-	for_each_connection_safe(connection, tmp, resource) {
-		list_del(&connection->connections);
-		drbd_debugfs_connection_cleanup(connection);
-		kref_put(&connection->kref, drbd_destroy_connection);
-	}
-	drbd_debugfs_resource_cleanup(resource);
-	kref_put(&resource->kref, drbd_destroy_resource);
+	queue_work(retry.wq, &retry.worker);
 }
 
 static void drbd_cleanup(void)
 {
-	unsigned int i;
-	struct drbd_device *device;
-	struct drbd_resource *resource, *tmp;
-
 	/* first remove proc,
-	 * drbdsetup uses it's presence to detect
+	 * drbdsetup uses its presence to detect
 	 * whether DRBD is loaded.
 	 * If we would get stuck in proc removal,
 	 * but have netlink already deregistered,
@@ -2325,19 +3588,13 @@ static void drbd_cleanup(void)
 		destroy_workqueue(retry.wq);
 
 	drbd_genl_unregister();
-
-	idr_for_each_entry(&drbd_devices, device, i)
-		drbd_delete_device(device);
-
-	/* not _rcu since, no other updater anymore. Genl already unregistered */
-	for_each_resource_safe(resource, tmp, &drbd_resources) {
-		list_del(&resource->resources);
-		drbd_free_resource(resource);
-	}
-
 	drbd_debugfs_cleanup();
 
+	unregister_pernet_device(&drbd_pernet_ops);
+
 	drbd_destroy_mempools();
+	if (ping_ack_sender)
+		destroy_workqueue(ping_ack_sender);
 	unregister_blkdev(DRBD_MAJOR, "drbd");
 
 	idr_destroy(&drbd_devices);
@@ -2366,6 +3623,16 @@ static int w_complete(struct drbd_work *w, int cancel)
 	return 0;
 }
 
+void drbd_queue_work(struct drbd_work_queue *q, struct drbd_work *w)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&q->q_lock, flags);
+	list_add_tail(&w->list, &q->q);
+	spin_unlock_irqrestore(&q->q_lock, flags);
+	wake_up(&q->q_wait);
+}
+
 void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
 {
 	struct completion_work completion_work;
@@ -2376,6 +3643,23 @@ void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
 	wait_for_completion(&completion_work.done);
 }
 
+void drbd_flush_workqueue_interruptible(struct drbd_device *device)
+{
+	struct completion_work completion_work;
+	int err;
+
+	completion_work.w.cb = w_complete;
+	init_completion(&completion_work.done);
+	drbd_queue_work(&device->resource->work, &completion_work.w);
+	err = wait_for_completion_interruptible(&completion_work.done);
+	if (err == -ERESTARTSYS) {
+		set_bit(ABORT_MDIO, &device->flags);
+		wake_up_all(&device->misc_wait);
+		wait_for_completion(&completion_work.done);
+		clear_bit(ABORT_MDIO, &device->flags);
+	}
+}
+
 struct drbd_resource *drbd_find_resource(const char *name)
 {
 	struct drbd_resource *resource;
@@ -2396,51 +3680,58 @@ struct drbd_resource *drbd_find_resource(const char *name)
 	return resource;
 }
 
-struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
-				     void *peer_addr, int peer_addr_len)
+static void drbd_put_send_buffers(struct drbd_connection *connection)
 {
-	struct drbd_resource *resource;
-	struct drbd_connection *connection;
+	unsigned int i;
 
-	rcu_read_lock();
-	for_each_resource_rcu(resource, &drbd_resources) {
-		for_each_connection_rcu(connection, resource) {
-			if (connection->my_addr_len == my_addr_len &&
-			    connection->peer_addr_len == peer_addr_len &&
-			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
-			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
-				kref_get(&connection->kref);
-				goto found;
-			}
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		if (connection->send_buffer[i].page) {
+			put_page(connection->send_buffer[i].page);
+			connection->send_buffer[i].page = NULL;
 		}
 	}
-	connection = NULL;
-found:
-	rcu_read_unlock();
-	return connection;
 }
 
-static int drbd_alloc_socket(struct drbd_socket *socket)
+static int drbd_alloc_send_buffers(struct drbd_connection *connection)
 {
-	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
-	if (!socket->rbuf)
-		return -ENOMEM;
-	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
-	if (!socket->sbuf)
-		return -ENOMEM;
+	unsigned int i;
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		struct page *page;
+
+		page = alloc_page(GFP_KERNEL);
+		if (!page) {
+			drbd_put_send_buffers(connection);
+			return -ENOMEM;
+		}
+		connection->send_buffer[i].page = page;
+		connection->send_buffer[i].unsent =
+		connection->send_buffer[i].pos = page_address(page);
+	}
+
 	return 0;
 }
 
-static void drbd_free_socket(struct drbd_socket *socket)
+void drbd_flush_peer_acks(struct drbd_resource *resource)
+{
+	spin_lock_irq(&resource->peer_ack_lock);
+	if (resource->peer_ack_req) {
+		resource->last_peer_acked_dagtag = resource->peer_ack_req->dagtag_sector;
+		drbd_queue_peer_ack(resource, resource->peer_ack_req);
+		resource->peer_ack_req = NULL;
+	}
+	spin_unlock_irq(&resource->peer_ack_lock);
+}
+
+static void peer_ack_timer_fn(struct timer_list *t)
 {
-	free_page((unsigned long) socket->sbuf);
-	free_page((unsigned long) socket->rbuf);
+	struct drbd_resource *resource = timer_container_of(resource, t, peer_ack_timer);
+
+	drbd_flush_peer_acks(resource);
 }
 
 void conn_free_crypto(struct drbd_connection *connection)
 {
-	drbd_free_sock(connection);
-
 	crypto_free_shash(connection->csums_tfm);
 	crypto_free_shash(connection->verify_tfm);
 	crypto_free_shash(connection->cram_hmac_tfm);
@@ -2458,11 +3749,25 @@ void conn_free_crypto(struct drbd_connection *connection)
 	connection->int_dig_vv = NULL;
 }
 
-int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
+static void wake_all_device_misc(struct drbd_resource *resource)
 {
-	struct drbd_connection *connection;
-	cpumask_var_t new_cpu_mask;
-	int err;
+	struct drbd_device *device;
+	int vnr;
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr)
+		wake_up(&device->misc_wait);
+	rcu_read_unlock();
+}
+
+int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts, const char *tag)
+{
+	struct drbd_connection *connection;
+	cpumask_var_t new_cpu_mask;
+	int err;
+	bool wake_device_misc = false;
+	bool force_state_recalc = false;
+	unsigned long irq_flags;
+	struct res_opts *old_opts = &resource->res_opts;
 
 	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
 		return -ENOMEM;
@@ -2491,26 +3796,47 @@ int set_resource_options(struct drbd_resource *resource, struct res_opts *res_op
 			goto fail;
 		}
 	}
+	if (res_opts->nr_requests < DRBD_NR_REQUESTS_MIN)
+		res_opts->nr_requests = DRBD_NR_REQUESTS_MIN;
+
+	if (old_opts->quorum != res_opts->quorum ||
+	    old_opts->on_no_quorum != res_opts->on_no_quorum)
+		force_state_recalc = true;
+
+	if (resource->res_opts.nr_requests < res_opts->nr_requests)
+		wake_device_misc = true;
+
 	resource->res_opts = *res_opts;
 	if (cpumask_empty(new_cpu_mask))
 		drbd_calc_cpu_mask(&new_cpu_mask);
 	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
 		cpumask_copy(resource->cpu_mask, new_cpu_mask);
+		resource->worker.reset_cpu_mask = 1;
+		rcu_read_lock();
 		for_each_connection_rcu(connection, resource) {
 			connection->receiver.reset_cpu_mask = 1;
-			connection->ack_receiver.reset_cpu_mask = 1;
-			connection->worker.reset_cpu_mask = 1;
+			connection->sender.reset_cpu_mask = 1;
 		}
+		rcu_read_unlock();
 	}
 	err = 0;
 
+	if (force_state_recalc) {
+		begin_state_change(resource, &irq_flags, CS_VERBOSE | CS_FORCE_RECALC);
+		end_state_change(resource, &irq_flags, tag);
+	}
+
+	if (wake_device_misc)
+		wake_all_device_misc(resource);
+
 fail:
 	free_cpumask_var(new_cpu_mask);
 	return err;
 
 }
 
-struct drbd_resource *drbd_create_resource(const char *name)
+struct drbd_resource *drbd_create_resource(const char *name,
+					   struct res_opts *res_opts)
 {
 	struct drbd_resource *resource;
 
@@ -2525,12 +3851,52 @@ struct drbd_resource *drbd_create_resource(const char *name)
 	kref_init(&resource->kref);
 	idr_init(&resource->devices);
 	INIT_LIST_HEAD(&resource->connections);
-	resource->write_ordering = WO_BDEV_FLUSH;
-	list_add_tail_rcu(&resource->resources, &drbd_resources);
+	spin_lock_init(&resource->tl_update_lock);
+	INIT_LIST_HEAD(&resource->transfer_log);
+	spin_lock_init(&resource->peer_ack_lock);
+	INIT_LIST_HEAD(&resource->peer_ack_req_list);
+	INIT_LIST_HEAD(&resource->peer_ack_list);
+	INIT_LIST_HEAD(&resource->peer_ack_work.list);
+	resource->peer_ack_work.cb = w_queue_peer_ack;
+	timer_setup(&resource->peer_ack_timer, peer_ack_timer_fn, 0);
+	spin_lock_init(&resource->initiator_flush_lock);
+	sema_init(&resource->state_sem, 1);
+	resource->role[NOW] = R_SECONDARY;
+	resource->max_node_id = res_opts->drbd8_compat_mode ? 1 : res_opts->node_id;
+	resource->twopc_reply.initiator_node_id = -1;
 	mutex_init(&resource->conf_update);
 	mutex_init(&resource->adm_mutex);
-	spin_lock_init(&resource->req_lock);
+	mutex_init(&resource->open_release);
+	rwlock_init(&resource->state_rwlock);
+	INIT_LIST_HEAD(&resource->listeners);
+	spin_lock_init(&resource->listeners_lock);
+	init_waitqueue_head(&resource->state_wait);
+	init_waitqueue_head(&resource->twopc_wait);
+	init_waitqueue_head(&resource->barrier_wait);
+	timer_setup(&resource->twopc_timer, twopc_timer_fn, 0);
+	INIT_WORK(&resource->twopc_work, nested_twopc_work);
+	drbd_init_workqueue(&resource->work);
+	drbd_thread_init(resource, &resource->worker, drbd_worker, "worker");
+	spin_lock_init(&resource->current_tle_lock);
 	drbd_debugfs_resource_add(resource);
+	resource->cached_min_aggreed_protocol_version = drbd_protocol_version_min;
+	/* members is a bit mask of the "seen" nodes in this resource.
+	 * In drbd8 compatibility mode, we only have one peer, so we can
+	 * set this to 1. */
+	resource->members = res_opts->drbd8_compat_mode ? 1 : NODE_MASK(res_opts->node_id);
+	INIT_WORK(&resource->empty_twopc, drbd_empty_twopc_work_fn);
+	INIT_LIST_HEAD(&resource->suspended_reqs);
+
+	ratelimit_state_init(&resource->ratelimit[D_RL_R_GENERIC], 5*HZ, 10);
+
+
+	if (set_resource_options(resource, res_opts, "create-resource"))
+		goto fail_free_name;
+
+	drbd_thread_start(&resource->worker);
+
+	list_add_tail_rcu(&resource->resources, &drbd_resources);
+
 	return resource;
 
 fail_free_name:
@@ -2542,128 +3908,289 @@ struct drbd_resource *drbd_create_resource(const char *name)
 }
 
 /* caller must be under adm_mutex */
-struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
+struct drbd_connection *drbd_create_connection(struct drbd_resource *resource,
+					       struct drbd_transport_class *tc)
 {
-	struct drbd_resource *resource;
 	struct drbd_connection *connection;
+	int size;
 
-	connection = kzalloc_obj(struct drbd_connection);
+	size = sizeof(*connection) - sizeof(connection->transport) + tc->instance_size;
+	connection = kzalloc(size, GFP_KERNEL);
 	if (!connection)
 		return NULL;
 
-	if (drbd_alloc_socket(&connection->data))
-		goto fail;
-	if (drbd_alloc_socket(&connection->meta))
+	ratelimit_state_init(&connection->ratelimit[D_RL_C_GENERIC], 5*HZ, /* no burst */ 1);
+
+	if (drbd_alloc_send_buffers(connection))
 		goto fail;
 
 	connection->current_epoch = kzalloc_obj(struct drbd_epoch);
 	if (!connection->current_epoch)
 		goto fail;
 
-	INIT_LIST_HEAD(&connection->transfer_log);
-
 	INIT_LIST_HEAD(&connection->current_epoch->list);
 	connection->epochs = 1;
 	spin_lock_init(&connection->epoch_lock);
 
+	INIT_LIST_HEAD(&connection->todo.work_list);
+	connection->todo.req = NULL;
+
+	atomic_set(&connection->ap_in_flight, 0);
+	atomic_set(&connection->rs_in_flight, 0);
 	connection->send.seen_any_write_yet = false;
 	connection->send.current_epoch_nr = 0;
 	connection->send.current_epoch_writes = 0;
+	connection->send.current_dagtag_sector =
+		resource->dagtag_sector - ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT) - 1;
 
-	resource = drbd_create_resource(name);
-	if (!resource)
-		goto fail;
-
-	connection->cstate = C_STANDALONE;
-	mutex_init(&connection->cstate_mutex);
-	init_waitqueue_head(&connection->ping_wait);
+	connection->cstate[NOW] = C_STANDALONE;
+	connection->peer_role[NOW] = R_UNKNOWN;
 	idr_init(&connection->peer_devices);
 
 	drbd_init_workqueue(&connection->sender_work);
-	mutex_init(&connection->data.mutex);
-	mutex_init(&connection->meta.mutex);
+	mutex_init(&connection->mutex[DATA_STREAM]);
+	mutex_init(&connection->mutex[CONTROL_STREAM]);
+
+	INIT_LIST_HEAD(&connection->connect_timer_work.list);
+	timer_setup(&connection->connect_timer, connect_timer_fn, 0);
 
 	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
 	connection->receiver.connection = connection;
-	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
-	connection->worker.connection = connection;
-	drbd_thread_init(resource, &connection->ack_receiver, drbd_ack_receiver, "ack_recv");
-	connection->ack_receiver.connection = connection;
+	drbd_thread_init(resource, &connection->sender, drbd_sender, "sender");
+	connection->sender.connection = connection;
+	spin_lock_init(&connection->primary_flush_lock);
+	spin_lock_init(&connection->flush_ack_lock);
+	spin_lock_init(&connection->peer_reqs_lock);
+	spin_lock_init(&connection->send_oos_lock);
+	INIT_LIST_HEAD(&connection->peer_requests);
+	INIT_LIST_HEAD(&connection->peer_reads);
+	INIT_LIST_HEAD(&connection->send_oos);
+	INIT_LIST_HEAD(&connection->connections);
+	INIT_LIST_HEAD(&connection->done_ee);
+	INIT_LIST_HEAD(&connection->dagtag_wait_ee);
+	INIT_LIST_HEAD(&connection->resync_ack_ee);
+	INIT_LIST_HEAD(&connection->remove_net_list);
+	init_waitqueue_head(&connection->ee_wait);
 
 	kref_init(&connection->kref);
 
-	connection->resource = resource;
+	INIT_WORK(&connection->peer_ack_work, drbd_send_peer_ack_wf);
+	INIT_LIST_HEAD(&connection->send_oos_work.list);
+	connection->send_oos_work.cb = drbd_send_out_of_sync_wf;
+	INIT_LIST_HEAD(&connection->flush_ack_work.list);
+	connection->flush_ack_work.cb = drbd_flush_ack_wf;
+	INIT_WORK(&connection->send_acks_work, drbd_send_acks_wf);
+	INIT_WORK(&connection->send_ping_ack_work, drbd_send_ping_ack_wf);
+	INIT_WORK(&connection->send_ping_work, drbd_send_ping_wf);
 
-	if (set_resource_options(resource, res_opts))
-		goto fail_resource;
+	INIT_LIST_HEAD(&connection->send_dagtag_work.list);
+	connection->send_dagtag_work.cb = w_send_dagtag;
+
+	spin_lock_init(&connection->advance_cache_ptr_lock);
 
 	kref_get(&resource->kref);
-	list_add_tail_rcu(&connection->connections, &resource->connections);
-	drbd_debugfs_connection_add(connection);
+	connection->resource = resource;
+	connection->after_reconciliation.lost_node_id = -1;
+
+	connection->reassemble_buffer.buffer = connection->reassemble_buffer_bytes.bytes;
+
+	INIT_LIST_HEAD(&connection->transport.paths);
+	connection->transport.log_prefix = resource->name;
+	if (tc->ops.init(&connection->transport))
+		goto fail;
+
 	return connection;
 
-fail_resource:
-	list_del(&resource->resources);
-	drbd_free_resource(resource);
 fail:
+	drbd_put_send_buffers(connection);
 	kfree(connection->current_epoch);
-	drbd_free_socket(&connection->meta);
-	drbd_free_socket(&connection->data);
 	kfree(connection);
+
 	return NULL;
 }
 
+/**
+ * drbd_transport_shutdown() - Free the transport specific members (e.g., sockets) of a connection
+ * @connection: The connection to shut down
+ * @op: The operation. Only close the connection or destroy the whole transport
+ *
+ * Must be called with conf_update held.
+ */
+void drbd_transport_shutdown(struct drbd_connection *connection, enum drbd_tr_free_op op)
+{
+	struct drbd_transport *transport = &connection->transport;
+
+	lockdep_assert_held(&connection->resource->conf_update);
+
+	mutex_lock(&connection->mutex[DATA_STREAM]);
+	mutex_lock(&connection->mutex[CONTROL_STREAM]);
+
+	flush_send_buffer(connection, DATA_STREAM);
+	flush_send_buffer(connection, CONTROL_STREAM);
+
+	/* Holding conf_update ensures that paths list is not modified concurrently. */
+	transport->class->ops.free(transport, op);
+	if (op == DESTROY_TRANSPORT) {
+		drbd_remove_all_paths(connection);
+
+		/* Wait for the delayed drbd_reclaim_path() calls. */
+		rcu_barrier();
+		drbd_put_transport_class(transport->class);
+	}
+
+	mutex_unlock(&connection->mutex[CONTROL_STREAM]);
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
+}
+
+void drbd_destroy_path(struct kref *kref)
+{
+	struct drbd_path *path = container_of(kref, struct drbd_path, kref);
+	struct drbd_connection *connection =
+		container_of(path->transport, struct drbd_connection, transport);
+
+	kref_put(&connection->kref, drbd_destroy_connection);
+	kfree(path);
+}
+
 void drbd_destroy_connection(struct kref *kref)
 {
 	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
 	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
 	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
 		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
 	kfree(connection->current_epoch);
 
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		free_peer_device(peer_device);
+		kref_put(&device->kref, drbd_destroy_device);
+	}
 	idr_destroy(&connection->peer_devices);
 
-	drbd_free_socket(&connection->meta);
-	drbd_free_socket(&connection->data);
-	kfree(connection->int_dig_in);
-	kfree(connection->int_dig_vv);
+	kfree(connection->transport.net_conf);
 	kfree(connection);
 	kref_put(&resource->kref, drbd_destroy_resource);
 }
 
+struct drbd_peer_device *create_peer_device(struct drbd_device *device, struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int err;
+
+	peer_device = kzalloc_obj(struct drbd_peer_device);
+	if (!peer_device)
+		return NULL;
+
+	peer_device->connection = connection;
+	peer_device->device = device;
+	peer_device->disk_state[NOW] = D_UNKNOWN;
+	peer_device->repl_state[NOW] = L_OFF;
+	peer_device->replication[NOW] = true;
+	peer_device->peer_replication[NOW] = true;
+	spin_lock_init(&peer_device->peer_seq_lock);
+
+	ratelimit_state_init(&peer_device->ratelimit[D_RL_PD_GENERIC], 5*HZ, /* no burst */ 1);
+
+	err = drbd_create_peer_device_default_config(peer_device);
+	if (err) {
+		kfree(peer_device);
+		return NULL;
+	}
+
+	timer_setup(&peer_device->start_resync_timer, start_resync_timer_fn, 0);
+
+	INIT_LIST_HEAD(&peer_device->resync_work.list);
+	peer_device->resync_work.cb  = w_resync_timer;
+	timer_setup(&peer_device->resync_timer, resync_timer_fn, 0);
+
+	INIT_LIST_HEAD(&peer_device->propagate_uuids_work.list);
+	peer_device->propagate_uuids_work.cb = w_send_uuids;
+
+	atomic_set(&peer_device->ap_pending_cnt, 0);
+	atomic_set(&peer_device->unacked_cnt, 0);
+	atomic_set(&peer_device->rs_pending_cnt, 0);
+
+	INIT_LIST_HEAD(&peer_device->resync_requests);
+
+	atomic_set(&peer_device->rs_sect_in, 0);
+
+	peer_device->bitmap_index = -1;
+	peer_device->resync_finished_pdsk = D_UNKNOWN;
+
+	peer_device->q_limits.physical_block_size = SECTOR_SIZE;
+	peer_device->q_limits.logical_block_size = SECTOR_SIZE;
+	peer_device->q_limits.alignment_offset = 0;
+	peer_device->q_limits.io_min = SECTOR_SIZE;
+	peer_device->q_limits.io_opt = PAGE_SIZE;
+	peer_device->q_limits.max_bio_size = DRBD_MAX_BIO_SIZE;
+
+	return peer_device;
+}
+
+static void drbd_ldev_destroy(struct work_struct *ws)
+{
+	struct drbd_device *device = container_of(ws, struct drbd_device, ldev_destroy_work);
+
+	lc_destroy(device->act_log);
+	device->act_log = NULL;
+	__acquire(local);
+	drbd_backing_dev_free(device, device->ldev);
+	device->ldev = NULL;
+	__release(local);
+
+	clear_bit(GOING_DISKLESS, &device->flags);
+	wake_up(&device->misc_wait);
+	kref_put(&device->kref, drbd_destroy_device);
+}
+
+static int init_conflict_submitter(struct drbd_device *device)
+{
+	/* Short name so that it is recognizable from the first 15 characters. */
+	device->submit_conflict.wq =
+		alloc_ordered_workqueue("drbd%u_sc", WQ_MEM_RECLAIM, device->minor);
+	if (!device->submit_conflict.wq)
+		return -ENOMEM;
+	INIT_WORK(&device->submit_conflict.worker, drbd_do_submit_conflict);
+	INIT_LIST_HEAD(&device->submit_conflict.resync_writes);
+	INIT_LIST_HEAD(&device->submit_conflict.resync_reads);
+	INIT_LIST_HEAD(&device->submit_conflict.writes);
+	INIT_LIST_HEAD(&device->submit_conflict.peer_writes);
+	spin_lock_init(&device->submit_conflict.lock);
+	return 0;
+}
+
 static int init_submitter(struct drbd_device *device)
 {
-	/* opencoded create_singlethread_workqueue(),
-	 * to be able to say "drbd%d", ..., minor */
 	device->submit.wq =
 		alloc_ordered_workqueue("drbd%u_submit", WQ_MEM_RECLAIM, device->minor);
 	if (!device->submit.wq)
 		return -ENOMEM;
-
 	INIT_WORK(&device->submit.worker, do_submit);
 	INIT_LIST_HEAD(&device->submit.writes);
+	INIT_LIST_HEAD(&device->submit.peer_writes);
+	spin_lock_init(&device->submit.lock);
 	return 0;
 }
 
-enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
+enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor,
+				      struct device_conf *device_conf, struct drbd_device **p_device)
 {
 	struct drbd_resource *resource = adm_ctx->resource;
-	struct drbd_connection *connection, *n;
+	struct drbd_connection *connection;
 	struct drbd_device *device;
 	struct drbd_peer_device *peer_device, *tmp_peer_device;
 	struct gendisk *disk;
+	LIST_HEAD(peer_devices);
+	LIST_HEAD(tmp);
 	int id;
 	int vnr = adm_ctx->volume;
 	enum drbd_ret_code err = ERR_NOMEM;
-	struct queue_limits lim = {
-		/*
-		 * Setting the max_hw_sectors to an odd value of 8kibyte here.
-		 * This triggers a max_bio_size message upon first attach or
-		 * connect.
-		 */
-		.max_hw_sectors		= DRBD_MAX_BIO_SIZE_SAFE >> 8,
-	};
+	bool locked = false;
+
+	lockdep_assert_held(&resource->conf_update);
 
 	device = minor_to_device(minor);
 	if (device)
@@ -2675,24 +4202,64 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
 		return ERR_NOMEM;
 	kref_init(&device->kref);
 
+	ratelimit_state_init(&device->ratelimit[D_RL_D_GENERIC], 5*HZ, /* no burst */ 1);
+	ratelimit_state_init(&device->ratelimit[D_RL_D_METADATA], 5*HZ, 10);
+	ratelimit_state_init(&device->ratelimit[D_RL_D_BACKEND], 5*HZ, 10);
+
 	kref_get(&resource->kref);
 	device->resource = resource;
 	device->minor = minor;
 	device->vnr = vnr;
+	device->device_conf = *device_conf;
+
+	drbd_set_defaults(device);
+
+	atomic_set(&device->ap_bio_cnt[READ], 0);
+	atomic_set(&device->ap_bio_cnt[WRITE], 0);
+	atomic_set(&device->ap_actlog_cnt, 0);
+	atomic_set(&device->wait_for_actlog, 0);
+	atomic_set(&device->wait_for_actlog_ecnt, 0);
+	atomic_set(&device->local_cnt, 0);
+	atomic_set(&device->rs_sect_ev, 0);
+	atomic_set(&device->md_io.in_use, 0);
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+	spin_lock_init(&device->timing_lock);
+#endif
+	spin_lock_init(&device->al_lock);
+
+	spin_lock_init(&device->pending_completion_lock);
+	INIT_LIST_HEAD(&device->pending_master_completion[0]);
+	INIT_LIST_HEAD(&device->pending_master_completion[1]);
+	INIT_LIST_HEAD(&device->pending_completion[0]);
+	INIT_LIST_HEAD(&device->pending_completion[1]);
+	INIT_LIST_HEAD(&device->openers);
+	spin_lock_init(&device->openers_lock);
+
+	atomic_set(&device->pending_bitmap_work.n, 0);
+	spin_lock_init(&device->pending_bitmap_work.q_lock);
+	INIT_LIST_HEAD(&device->pending_bitmap_work.q);
+
+	timer_setup(&device->md_sync_timer, md_sync_timer_fn, 0);
+	timer_setup(&device->request_timer, request_timer_fn, 0);
+
+	init_waitqueue_head(&device->misc_wait);
+	init_waitqueue_head(&device->al_wait);
+	init_waitqueue_head(&device->seq_wait);
 
-	drbd_init_set_defaults(device);
+	init_rwsem(&device->uuid_sem);
 
-	disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
+	disk = blk_alloc_disk(NULL, NUMA_NO_NODE);
 	if (IS_ERR(disk)) {
 		err = PTR_ERR(disk);
 		goto out_no_disk;
 	}
 
+	INIT_WORK(&device->ldev_destroy_work, drbd_ldev_destroy);
+
 	device->vdisk = disk;
 	device->rq_queue = disk->queue;
 
-	set_disk_ro(disk, true);
-
 	disk->major = DRBD_MAJOR;
 	disk->first_minor = minor;
 	disk->minors = 1;
@@ -2705,12 +4272,39 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
 	if (!device->md_io.page)
 		goto out_no_io_page;
 
-	if (drbd_bm_init(device))
-		goto out_no_bitmap;
+	/* Just put in some sane default; should never be used. */
+	device->last_bm_block_shift = BM_BLOCK_SHIFT_MIN;
+
+	spin_lock_init(&device->interval_lock);
 	device->read_requests = RB_ROOT;
-	device->write_requests = RB_ROOT;
+	device->requests = RB_ROOT;
+
+	BUG_ON(!mutex_is_locked(&resource->conf_update));
+	for_each_connection(connection, resource) {
+		peer_device = create_peer_device(device, connection);
+		if (!peer_device)
+			goto out_no_peer_device;
+		list_add(&peer_device->peer_devices, &peer_devices);
+	}
+
+	/* Insert the new device into all idrs under state_rwlock write lock
+	   to guarantee a consistent object model. idr_preload() doesn't help
+	   because it can only guarantee that a single idr_alloc() will
+	   succeed. This fails (and will be retried) if no memory is
+	   immediately available.
+	   Keep in mid that RCU readers might find the device in the moment
+	   we add it to the resources->devices IDR!
+	*/
+
+	INIT_LIST_HEAD(&device->peer_devices);
+	spin_lock_init(&device->pending_bmio_lock);
+	INIT_LIST_HEAD(&device->pending_bitmap_io);
 
-	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
+	locked = true;
+	write_lock_irq(&resource->state_rwlock);
+	spin_lock(&drbd_devices_lock);
+	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_NOWAIT);
+	spin_unlock(&drbd_devices_lock);
 	if (id < 0) {
 		if (id == -ENOSPC)
 			err = ERR_MINOR_OR_VOLUME_EXISTS;
@@ -2718,7 +4312,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
 	}
 	kref_get(&device->kref);
 
-	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
+	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_NOWAIT);
 	if (id < 0) {
 		if (id == -ENOSPC)
 			err = ERR_MINOR_OR_VOLUME_EXISTS;
@@ -2726,105 +4320,219 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
 	}
 	kref_get(&device->kref);
 
-	INIT_LIST_HEAD(&device->peer_devices);
-	INIT_LIST_HEAD(&device->pending_bitmap_io);
-	for_each_connection(connection, resource) {
-		peer_device = kzalloc_obj(struct drbd_peer_device);
-		if (!peer_device)
-			goto out_idr_remove_from_resource;
-		peer_device->connection = connection;
-		peer_device->device = device;
-
-		list_add(&peer_device->peer_devices, &device->peer_devices);
+	list_for_each_entry_safe(peer_device, tmp_peer_device, &peer_devices, peer_devices) {
+		connection = peer_device->connection;
+		id = idr_alloc(&connection->peer_devices, peer_device,
+			       device->vnr, device->vnr + 1, GFP_NOWAIT);
+		if (id < 0)
+			goto out_remove_peer_device;
+		list_del(&peer_device->peer_devices);
+		list_add_rcu(&peer_device->peer_devices, &device->peer_devices);
+		kref_get(&connection->kref);
 		kref_get(&device->kref);
+	}
+	write_unlock_irq(&resource->state_rwlock);
+	locked = false;
 
-		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
-		if (id < 0) {
-			if (id == -ENOSPC)
-				err = ERR_INVALID_REQUEST;
-			goto out_idr_remove_from_resource;
-		}
-		kref_get(&connection->kref);
-		INIT_WORK(&peer_device->send_acks_work, drbd_send_acks_wf);
+	if (init_conflict_submitter(device)) {
+		err = ERR_NOMEM;
+		goto out_remove_peer_device;
 	}
 
 	if (init_submitter(device)) {
 		err = ERR_NOMEM;
-		goto out_idr_remove_from_resource;
+		goto out_remove_peer_device;
 	}
 
 	err = add_disk(disk);
 	if (err)
-		goto out_destroy_workqueue;
+		goto out_destroy_submitter;
+	device->have_quorum[OLD] =
+	device->have_quorum[NEW] =
+		(resource->res_opts.quorum == QOU_OFF);
+
+	for_each_peer_device(peer_device, device) {
+		connection = peer_device->connection;
+		peer_device->node_id = connection->peer_node_id;
 
-	/* inherit the connection state */
-	device->state.conn = first_connection(resource)->cstate;
-	if (device->state.conn == C_WF_REPORT_PARAMS) {
-		for_each_peer_device(peer_device, device)
+		if (connection->cstate[NOW] >= C_CONNECTED)
 			drbd_connected(peer_device);
 	}
-	/* move to create_peer_device() */
-	for_each_peer_device(peer_device, device)
-		drbd_debugfs_peer_device_add(peer_device);
+
 	drbd_debugfs_device_add(device);
+	*p_device = device;
 	return NO_ERROR;
 
-out_destroy_workqueue:
+out_destroy_submitter:
 	destroy_workqueue(device->submit.wq);
-out_idr_remove_from_resource:
-	for_each_connection_safe(connection, n, resource) {
-		peer_device = idr_remove(&connection->peer_devices, vnr);
-		if (peer_device)
-			kref_put(&connection->kref, drbd_destroy_connection);
-	}
-	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
+	device->submit.wq = NULL;
+out_remove_peer_device:
+	list_splice_init_rcu(&device->peer_devices, &tmp, synchronize_rcu);
+	list_for_each_entry_safe(peer_device, tmp_peer_device, &tmp, peer_devices) {
+		struct drbd_connection *connection = peer_device->connection;
+
+		idr_remove(&connection->peer_devices, device->vnr);
 		list_del(&peer_device->peer_devices);
 		kfree(peer_device);
+		kref_put(&connection->kref, drbd_destroy_connection);
 	}
 	idr_remove(&resource->devices, vnr);
+
 out_idr_remove_minor:
+	spin_lock(&drbd_devices_lock);
 	idr_remove(&drbd_devices, minor);
-	synchronize_rcu();
+	spin_unlock(&drbd_devices_lock);
 out_no_minor_idr:
-	drbd_bm_cleanup(device);
-out_no_bitmap:
+	if (locked)
+		write_unlock_irq(&resource->state_rwlock);
+	synchronize_rcu();
+
+out_no_peer_device:
+	list_for_each_entry_safe(peer_device, tmp_peer_device, &peer_devices, peer_devices) {
+		list_del(&peer_device->peer_devices);
+		kfree(peer_device);
+	}
+
 	__free_page(device->md_io.page);
 out_no_io_page:
 	put_disk(disk);
 out_no_disk:
 	kref_put(&resource->kref, drbd_destroy_resource);
+		/* kref debugging wants an extra put, see has_refs() */
 	kfree(device);
 	return err;
 }
 
-void drbd_delete_device(struct drbd_device *device)
+/**
+ * drbd_unregister_device()  -  make a device "invisible"
+ * @device: DRBD device to unregister
+ *
+ * Remove the device from the drbd object model and unregister it in the
+ * kernel.  Keep reference counts on device->kref; they are dropped in
+ * drbd_reclaim_device().
+ */
+void drbd_unregister_device(struct drbd_device *device)
 {
 	struct drbd_resource *resource = device->resource;
 	struct drbd_connection *connection;
 	struct drbd_peer_device *peer_device;
 
-	/* move to free_peer_device() */
-	for_each_peer_device(peer_device, device)
-		drbd_debugfs_peer_device_cleanup(peer_device);
-	drbd_debugfs_device_cleanup(device);
+	write_lock_irq(&resource->state_rwlock);
 	for_each_connection(connection, resource) {
 		idr_remove(&connection->peer_devices, device->vnr);
-		kref_put(&device->kref, drbd_destroy_device);
 	}
 	idr_remove(&resource->devices, device->vnr);
-	kref_put(&device->kref, drbd_destroy_device);
-	idr_remove(&drbd_devices, device_to_minor(device));
-	kref_put(&device->kref, drbd_destroy_device);
+	spin_lock(&drbd_devices_lock);
+	idr_remove(&drbd_devices, device->minor);
+	spin_unlock(&drbd_devices_lock);
+	write_unlock_irq(&resource->state_rwlock);
+
+	for_each_peer_device(peer_device, device)
+		drbd_debugfs_peer_device_cleanup(peer_device);
+	drbd_debugfs_device_cleanup(device);
 	del_gendisk(device->vdisk);
-	synchronize_rcu();
-	kref_put(&device->kref, drbd_destroy_device);
+
+	destroy_workqueue(device->submit_conflict.wq);
+	device->submit_conflict.wq = NULL;
+	destroy_workqueue(device->submit.wq);
+	device->submit.wq = NULL;
+	timer_shutdown_sync(&device->request_timer);
+}
+
+void drbd_reclaim_device(struct rcu_head *rp)
+{
+	struct drbd_device *device = container_of(rp, struct drbd_device, rcu);
+	struct drbd_peer_device *peer_device;
+	int i;
+
+	for_each_peer_device(peer_device, device) {
+		kref_put(&device->kref, drbd_destroy_device);
+	}
+
+	for (i = 0; i < 3; i++) {
+		kref_put(&device->kref, drbd_destroy_device);
+	}
+}
+
+static void shutdown_connect_timer(struct drbd_connection *connection)
+{
+	if (timer_shutdown_sync(&connection->connect_timer)) {
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
+}
+
+void del_connect_timer(struct drbd_connection *connection)
+{
+	if (timer_delete_sync(&connection->connect_timer)) {
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
+}
+
+/**
+ * drbd_unregister_connection()  -  make a connection "invisible"
+ * @connection: DRBD connection to unregister
+ *
+ * Remove the connection from the drbd object model.  Keep reference counts on
+ * connection->kref; they are dropped in drbd_reclaim_connection().
+ */
+void drbd_unregister_connection(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	int vnr, rr;
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+		drbd_debugfs_peer_device_cleanup(peer_device);
+
+	write_lock_irq(&resource->state_rwlock);
+	set_bit(C_UNREGISTERED, &connection->flags);
+	smp_wmb();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+		list_del_rcu(&peer_device->peer_devices);
+	list_del_rcu(&connection->connections);
+	write_unlock_irq(&resource->state_rwlock);
+
+	drbd_debugfs_connection_cleanup(connection);
+
+	shutdown_connect_timer(connection);
+
+	rr = drbd_free_peer_reqs(connection, &connection->done_ee);
+	if (rr)
+		drbd_err(connection, "%d EEs in done list found!\n", rr);
+
+	drbd_transport_shutdown(connection, DESTROY_TRANSPORT);
+	drbd_put_send_buffers(connection);
+	conn_free_crypto(connection);
+}
+
+void drbd_reclaim_connection(struct rcu_head *rp)
+{
+	struct drbd_connection *connection =
+		container_of(rp, struct drbd_connection, rcu);
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
+	kref_put(&connection->kref, drbd_destroy_connection);
+}
+
+void drbd_reclaim_path(struct rcu_head *rp)
+{
+	struct drbd_path *path = container_of(rp, struct drbd_path, rcu);
+
+	INIT_LIST_HEAD(&path->list);
+	kref_put(&path->kref, drbd_destroy_path);
 }
 
 static int __init drbd_init(void)
 {
 	int err;
 
-	if (drbd_minor_count < DRBD_MINOR_COUNT_MIN || drbd_minor_count > DRBD_MINOR_COUNT_MAX) {
+
+	if (drbd_minor_count < DRBD_MINOR_COUNT_MIN
+	||  drbd_minor_count > DRBD_MINOR_COUNT_MAX) {
 		pr_err("invalid minor_count (%d)\n", drbd_minor_count);
 #ifdef MODULE
 		return -EINVAL;
@@ -2840,24 +4548,40 @@ static int __init drbd_init(void)
 		return err;
 	}
 
+	/*
+	 * allocate all necessary structs
+	 */
 	drbd_proc = NULL; /* play safe for drbd_cleanup */
 	idr_init(&drbd_devices);
 
-	mutex_init(&resources_mutex);
 	INIT_LIST_HEAD(&drbd_resources);
 
+	err = register_pernet_device(&drbd_pernet_ops);
+	if (err) {
+		pr_err("unable to register net namespace handlers\n");
+		goto fail;
+	}
+
+	drbd_enable_netns();
 	err = drbd_genl_register();
 	if (err) {
 		pr_err("unable to register generic netlink family\n");
 		goto fail;
 	}
 
+	ping_ack_sender = alloc_workqueue("drbd_pas",
+			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
+	if (!ping_ack_sender)
+		goto fail;
+
 	err = drbd_create_mempools();
 	if (err)
 		goto fail;
 
 	err = -ENOMEM;
-	drbd_proc = proc_create_single("drbd", S_IFREG | 0444 , NULL, drbd_seq_show);
+	drbd_proc = proc_create_single("drbd", S_IFREG | 0444, NULL,
+			drbd_seq_show);
+
 	if (!drbd_proc)	{
 		pr_err("unable to register proc file\n");
 		goto fail;
@@ -2879,9 +4603,14 @@ static int __init drbd_init(void)
 	       GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
 	pr_info("%s\n", drbd_buildtag());
 	pr_info("registered as block device major %d\n", DRBD_MAJOR);
-	return 0; /* Success! */
 
-fail:
+#ifdef CONFIG_DRBD_COMPAT_84
+	atomic_set(&nr_drbd8_devices, 0);
+#endif
+
+	return 0; /* Success! */
+
+fail:
 	drbd_cleanup();
 	if (err == -ENOMEM)
 		pr_err("ran out of memory\n");
@@ -2890,493 +4619,1050 @@ static int __init drbd_init(void)
 	return err;
 }
 
-static void drbd_free_one_sock(struct drbd_socket *ds)
+/* meta data management */
+
+static
+void drbd_md_encode_9(struct drbd_device *device, struct meta_data_on_disk_9 *buffer)
 {
-	struct socket *s;
-	mutex_lock(&ds->mutex);
-	s = ds->socket;
-	ds->socket = NULL;
-	mutex_unlock(&ds->mutex);
-	if (s) {
-		/* so debugfs does not need to mutex_lock() */
-		synchronize_rcu();
-		kernel_sock_shutdown(s, SHUT_RDWR);
-		sock_release(s);
+	int i;
+
+	buffer->effective_size = cpu_to_be64(device->ldev->md.effective_size);
+	buffer->current_uuid = cpu_to_be64(device->ldev->md.current_uuid);
+	buffer->members = cpu_to_be64(device->ldev->md.members);
+	buffer->flags = cpu_to_be32(device->ldev->md.flags);
+	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_09);
+
+	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
+	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
+	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
+	buffer->bm_bytes_per_bit = cpu_to_be32(device->ldev->md.bm_block_size);
+	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
+
+	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
+	buffer->la_peer_max_bio_size = cpu_to_be32(device->device_conf.max_bio_size);
+	buffer->bm_max_peers = cpu_to_be32(device->ldev->md.max_peers);
+	buffer->node_id = cpu_to_be32(device->ldev->md.node_id);
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[i];
+
+		buffer->peers[i].bitmap_uuid = cpu_to_be64(peer_md->bitmap_uuid);
+		buffer->peers[i].bitmap_dagtag = cpu_to_be64(peer_md->bitmap_dagtag);
+		buffer->peers[i].flags = cpu_to_be32(peer_md->flags & ~MDF_HAVE_BITMAP);
+		buffer->peers[i].bitmap_index = cpu_to_be32(peer_md->bitmap_index);
 	}
+	BUILD_BUG_ON(ARRAY_SIZE(device->ldev->md.history_uuids) != ARRAY_SIZE(buffer->history_uuids));
+	for (i = 0; i < ARRAY_SIZE(buffer->history_uuids); i++)
+		buffer->history_uuids[i] = cpu_to_be64(device->ldev->md.history_uuids[i]);
+
+	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
+	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
+
+	if (device->bitmap == NULL)
+		for (i = 0; i < DRBD_PEERS_MAX; i++)
+			buffer->peers[i].flags |= cpu_to_be32(MDF_PEER_FULL_SYNC);
 }
 
-void drbd_free_sock(struct drbd_connection *connection)
+static void drbd_md_encode(struct drbd_device *device, void *buffer)
 {
-	if (connection->data.socket)
-		drbd_free_one_sock(&connection->data);
-	if (connection->meta.socket)
-		drbd_free_one_sock(&connection->meta);
+	if (test_bit(LEGACY_84_MD, &device->flags))
+		drbd_md_encode_84(device, buffer);
+	else
+		drbd_md_encode_9(device, buffer);
 }
 
-/* meta data management */
+int drbd_md_write(struct drbd_device *device, struct meta_data_on_disk_9 *buffer)
+{
+	sector_t sector;
+	int err;
+
+	if (drbd_md_dax_active(device->ldev)) {
+		drbd_md_encode(device, drbd_dax_md_addr(device->ldev));
+		arch_wb_cache_pmem(drbd_dax_md_addr(device->ldev),
+				   sizeof(struct meta_data_on_disk_9));
+		return 0;
+	}
+
+	memset(buffer, 0, sizeof(*buffer));
+
+	drbd_md_encode(device, buffer);
+
+	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
+	sector = device->ldev->md.md_offset;
+
+	err = drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE);
+	if (err) {
+		drbd_err(device, "meta data update failed!\n");
+		drbd_handle_io_error(device, DRBD_META_IO_ERROR);
+	}
+
+	return err;
+}
+
+/**
+ * __drbd_md_sync() - Writes the meta data super block (conditionally) if the MD_DIRTY flag bit is set
+ * @device:	DRBD device.
+ * @maybe:	meta data may in fact be "clean", the actual write may be skipped.
+ */
+static int __drbd_md_sync(struct drbd_device *device, bool maybe)
+{
+	struct meta_data_on_disk_9 *buffer;
+	int err = -EIO;
+
+	/* Don't accidentally change the DRBD meta data layout. */
+	BUILD_BUG_ON(DRBD_PEERS_MAX != 32);
+	BUILD_BUG_ON(HISTORY_UUIDS != 32);
+	BUILD_BUG_ON(sizeof(struct meta_data_on_disk_9) != 4096);
+
+	if (!get_ldev_if_state(device, D_DETACHING))
+		return -EIO;
+
+	buffer = drbd_md_get_buffer(device, __func__);
+	if (!buffer)
+		goto out;
+
+	timer_delete(&device->md_sync_timer);
+	/* timer may be rearmed by drbd_md_mark_dirty() now. */
+
+	if (test_and_clear_bit(MD_DIRTY, &device->flags) || !maybe) {
+		err = drbd_md_write(device, buffer);
+		if (err)
+			set_bit(MD_DIRTY, &device->flags);
+	}
+
+	drbd_md_put_buffer(device);
+out:
+	put_ldev(device);
+
+	return err;
+}
+
+int drbd_md_sync(struct drbd_device *device)
+{
+	return __drbd_md_sync(device, false);
+}
+
+int drbd_md_sync_if_dirty(struct drbd_device *device)
+{
+	return __drbd_md_sync(device, true);
+}
+
+/**
+ * drbd_md_mark_dirty() - Mark meta data super block as dirty
+ * @device:	DRBD device.
+ *
+ * Call this function if you change anything that should be written to
+ * the meta-data super block. This function sets MD_DIRTY, and starts a
+ * timer that ensures that within five seconds you have to call drbd_md_sync().
+ */
+void drbd_md_mark_dirty(struct drbd_device *device)
+{
+	if (!test_and_set_bit(MD_DIRTY, &device->flags))
+		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
+}
+
+void _drbd_uuid_push_history(struct drbd_device *device, u64 val) __must_hold(local)
+{
+	struct drbd_md *md = &device->ldev->md;
+	int node_id, i;
+
+	if (val == UUID_JUST_CREATED || val == 0)
+		return;
+
+	val &= ~UUID_PRIMARY;
 
-void conn_md_sync(struct drbd_connection *connection)
+	if (val == (md->current_uuid & ~UUID_PRIMARY))
+		return;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == md->node_id)
+			continue;
+		if (val == (md->peers[node_id].bitmap_uuid & ~UUID_PRIMARY))
+			return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(md->history_uuids); i++) {
+		if (md->history_uuids[i] == val)
+			return;
+	}
+
+	for (i = ARRAY_SIZE(md->history_uuids) - 1; i > 0; i--)
+		md->history_uuids[i] = md->history_uuids[i - 1];
+	md->history_uuids[i] = val;
+}
+
+u64 _drbd_uuid_pull_history(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_md *md = &device->ldev->md;
+	u64 first_history_uuid;
+	int i;
+
+	first_history_uuid = md->history_uuids[0];
+	for (i = 0; i < ARRAY_SIZE(md->history_uuids) - 1; i++)
+		md->history_uuids[i] = md->history_uuids[i + 1];
+	md->history_uuids[i] = 0;
+
+	return first_history_uuid;
+}
+
+static void __drbd_uuid_set_current(struct drbd_device *device, u64 val)
+{
+	drbd_md_mark_dirty(device);
+	if (device->resource->role[NOW] == R_PRIMARY)
+		val |= UUID_PRIMARY;
+	else
+		val &= ~UUID_PRIMARY;
+
+	device->ldev->md.current_uuid = val;
+	drbd_uuid_set_exposed(device, val, false);
+}
+
+static void __drbd_uuid_set_bitmap(struct drbd_peer_device *peer_device, u64 val)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_md *peer_md = &device->ldev->md.peers[peer_device->node_id];
+
+	drbd_md_mark_dirty(device);
+	peer_md->bitmap_uuid = val;
+	peer_md->bitmap_dagtag = val ? device->resource->dagtag_sector : 0;
+}
+
+void _drbd_uuid_set_current(struct drbd_device *device, u64 val) __must_hold(local)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
+	__drbd_uuid_set_current(device, val);
+	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
+}
+
+void _drbd_uuid_set_bitmap(struct drbd_peer_device *peer_device, u64 val) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	unsigned long flags;
+
+	down_write(&device->uuid_sem);
+	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
+	__drbd_uuid_set_bitmap(peer_device, val);
+	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
+	up_write(&device->uuid_sem);
+}
+
+/* call holding down_write(uuid_sem) */
+void drbd_uuid_set_bitmap(struct drbd_peer_device *peer_device, u64 uuid) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	unsigned long flags;
+	u64 previous_uuid;
+
+	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
+	previous_uuid = drbd_bitmap_uuid(peer_device);
+	__drbd_uuid_set_bitmap(peer_device, uuid);
+	if (previous_uuid)
+		_drbd_uuid_push_history(device, previous_uuid);
+	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
+}
+
+static u64 rotate_current_into_bitmap(struct drbd_device *device, u64 weak_nodes, u64 dagtag) __must_hold(local)
 {
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
 	struct drbd_peer_device *peer_device;
-	int vnr;
+	int node_id;
+	u64 bm_uuid, prev_c_uuid;
+	u64 node_mask = 0;  /* bit mask of node-ids processed */
+	u64 slot_mask = 0;  /* bit mask of on-disk bitmap slots processed */
+	/* return value, bit mask of node-ids for which we
+	 * actually set a new bitmap uuid */
+	u64 got_new_bitmap_uuid = 0;
+
+	if (device->ldev->md.current_uuid != UUID_JUST_CREATED)
+		prev_c_uuid = device->ldev->md.current_uuid;
+	else
+		get_random_bytes(&prev_c_uuid, sizeof(u64));
 
 	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_disk_state pdsk;
+		node_id = peer_device->node_id;
+		node_mask |= NODE_MASK(node_id);
+		if (peer_device->bitmap_index != -1)
+			__set_bit(peer_device->bitmap_index, (unsigned long *)&slot_mask);
+		bm_uuid = peer_md[node_id].bitmap_uuid;
+		if (bm_uuid && bm_uuid != prev_c_uuid)
+			continue;
 
-		kref_get(&device->kref);
-		rcu_read_unlock();
-		drbd_md_sync(device);
-		kref_put(&device->kref, drbd_destroy_device);
-		rcu_read_lock();
+		pdsk = peer_device->disk_state[NOW];
+
+		/* Create a new current UUID for a peer that is diskless but usually has a backing disk.
+		 * Do not create a new current UUID for a CONNECTED intentional diskless peer.
+		 * Create one for an intentional diskless peer that is currently away. */
+		if (pdsk == D_DISKLESS && !(peer_md[node_id].flags & MDF_HAVE_BITMAP))
+			continue;
+
+		if ((pdsk <= D_UNKNOWN && pdsk != D_NEGOTIATING) ||
+		    (NODE_MASK(node_id) & weak_nodes)) {
+			peer_md[node_id].bitmap_uuid = prev_c_uuid;
+			peer_md[node_id].bitmap_dagtag = dagtag;
+			drbd_md_mark_dirty(device);
+			got_new_bitmap_uuid |= NODE_MASK(node_id);
+		}
+	}
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		int slot_nr;
+		if (node_id == device->ldev->md.node_id)
+			continue;
+		if (node_mask & NODE_MASK(node_id))
+			continue;
+		slot_nr = peer_md[node_id].bitmap_index;
+		if (slot_nr != -1) {
+			if (test_bit(slot_nr, (unsigned long *)&slot_mask))
+				continue;
+			__set_bit(slot_nr, (unsigned long *)&slot_mask);
+		}
+		bm_uuid = peer_md[node_id].bitmap_uuid;
+		if (bm_uuid && bm_uuid != prev_c_uuid)
+			continue;
+		if (slot_nr == -1) {
+			slot_nr = find_first_zero_bit((unsigned long *)&slot_mask, sizeof(slot_mask) * BITS_PER_BYTE);
+			__set_bit(slot_nr, (unsigned long *)&slot_mask);
+		}
+		peer_md[node_id].bitmap_uuid = prev_c_uuid;
+		peer_md[node_id].bitmap_dagtag = dagtag;
+		drbd_md_mark_dirty(device);
+		/* count, but only if that bitmap index exists. */
+		if (slot_nr < device->ldev->md.max_peers)
+			got_new_bitmap_uuid |= NODE_MASK(node_id);
 	}
 	rcu_read_unlock();
+
+	return got_new_bitmap_uuid;
 }
 
-/* aligned 4kByte */
-struct meta_data_on_disk {
-	u64 la_size_sect;      /* last agreed size. */
-	u64 uuid[UI_SIZE];   /* UUIDs. */
-	u64 device_uuid;
-	u64 reserved_u64_1;
-	u32 flags;             /* MDF */
-	u32 magic;
-	u32 md_size_sect;
-	u32 al_offset;         /* offset to this block */
-	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
-	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
-	u32 bm_offset;         /* offset to the bitmap, from here */
-	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
-	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
+static u64 initial_resync_nodes(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	u64 nodes = 0;
 
-	/* see al_tr_number_to_on_disk_sector() */
-	u32 al_stripes;
-	u32 al_stripe_size_4k;
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->disk_state[NOW] == D_INCONSISTENT &&
+		    peer_device->repl_state[NOW] == L_ESTABLISHED)
+			nodes |= NODE_MASK(peer_device->node_id);
+	}
 
-	u8 reserved_u8[4096 - (7*8 + 10*4)];
-} __packed;
+	return nodes;
+}
 
+u64 drbd_weak_nodes_device(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	u64 not_weak = 0;
 
+	if (device->disk_state[NOW] == D_UP_TO_DATE)
+		not_weak = NODE_MASK(device->resource->res_opts.node_id);
 
-void drbd_md_write(struct drbd_device *device, void *b)
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_disk_state pdsk = peer_device->disk_state[NOW];
+		if (!(pdsk <= D_FAILED || pdsk == D_UNKNOWN || pdsk == D_OUTDATED))
+			not_weak |= NODE_MASK(peer_device->node_id);
+
+	}
+	rcu_read_unlock();
+
+	return ~not_weak;
+}
+
+
+static bool __new_current_uuid_prepare(struct drbd_device *device, bool forced) __must_hold(local)
 {
-	struct meta_data_on_disk *buffer = b;
-	sector_t sector;
-	int i;
+	u64 got_new_bitmap_uuid, val, old_current_uuid;
+	int err;
 
-	memset(buffer, 0, sizeof(*buffer));
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+	got_new_bitmap_uuid = rotate_current_into_bitmap(device,
+					forced ? initial_resync_nodes(device) : 0,
+					device->resource->dagtag_sector);
 
-	buffer->la_size_sect = cpu_to_be64(get_capacity(device->vdisk));
-	for (i = UI_CURRENT; i < UI_SIZE; i++)
-		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
-	buffer->flags = cpu_to_be32(device->ldev->md.flags);
-	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
+	if (!got_new_bitmap_uuid) {
+		spin_unlock_irq(&device->ldev->md.uuid_lock);
+		return false;
+	}
 
-	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
-	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
-	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
-	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
-	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
+	old_current_uuid = device->ldev->md.current_uuid;
+	get_random_bytes(&val, sizeof(u64));
+	__drbd_uuid_set_current(device, val);
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
 
-	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
-	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
+	/* get it to stable storage _now_ */
+	err = drbd_md_sync(device);
+	if (err) {
+		_drbd_uuid_set_current(device, old_current_uuid);
+		return false;
+	}
 
-	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
-	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
+	return true;
+}
 
-	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
-	sector = device->ldev->md.md_offset;
+static void __new_current_uuid_info(struct drbd_device *device, u64 weak_nodes)
+{
+	drbd_info(device, "new current UUID: %016llX weak: %016llX\n",
+		  device->ldev->md.current_uuid, weak_nodes);
+}
 
-	if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
-		/* this was a try anyways ... */
-		drbd_err(device, "meta data update failed!\n");
-		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
+static void __new_current_uuid_send(struct drbd_device *device, u64 weak_nodes, bool forced) __must_hold(local)
+{
+	struct drbd_peer_device *peer_device;
+	u64 im;
+
+	for_each_peer_device_ref(peer_device, im, device) {
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED)
+			drbd_send_uuids(peer_device, forced ? 0 : UUID_FLAG_NEW_DATAGEN, weak_nodes);
+	}
+}
+
+static void __drbd_uuid_new_current_send(struct drbd_device *device, bool forced) __must_hold(local)
+{
+	u64 weak_nodes;
+
+	down_write(&device->uuid_sem);
+	if (!__new_current_uuid_prepare(device, forced)) {
+		up_write(&device->uuid_sem);
+		return;
+	}
+	downgrade_write(&device->uuid_sem);
+	weak_nodes = drbd_weak_nodes_device(device);
+	__new_current_uuid_info(device, weak_nodes);
+	__new_current_uuid_send(device, weak_nodes, forced);
+	up_read(&device->uuid_sem);
+}
+
+static void __drbd_uuid_new_current_holding_uuid_sem(struct drbd_device *device) __must_hold(local)
+{
+	u64 weak_nodes;
+
+	if (!__new_current_uuid_prepare(device, false))
+		return;
+	weak_nodes = drbd_weak_nodes_device(device);
+	__new_current_uuid_info(device, weak_nodes);
+}
+
+static bool peer_can_fill_a_bitmap_slot(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	const bool intentional_diskless = device->device_conf.intentional_diskless;
+	const int my_node_id = device->resource->res_opts.node_id;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == peer_device->node_id)
+			continue;
+		if (peer_device->bitmap_uuids[node_id] == 0) {
+			struct drbd_peer_device *p2;
+			p2 = peer_device_by_node_id(peer_device->device, node_id);
+			if (p2 && !want_bitmap(p2))
+				continue;
+
+			if (node_id == my_node_id && intentional_diskless)
+				continue;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static bool diskfull_peers_need_new_cur_uuid(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->connection->agreed_pro_version < 110)
+			continue;
+
+		/* Only an up-to-date peer persists a new current uuid! */
+		if (peer_device->disk_state[NOW] < D_UP_TO_DATE)
+			continue;
+		if (peer_can_fill_a_bitmap_slot(peer_device)) {
+			rv = true;
+			break;
+		}
 	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+static bool a_lost_peer_is_on_same_cur_uuid(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_disk_state pdsk = peer_device->disk_state[NOW];
+
+		if (pdsk >= D_INCONSISTENT && pdsk <= D_UNKNOWN &&
+		    (device->exposed_data_uuid & ~UUID_PRIMARY) ==
+		    (peer_device->current_uuid & ~UUID_PRIMARY) &&
+		    !(peer_device->uuid_flags & UUID_FLAG_SYNC_TARGET)) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
 }
 
 /**
- * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
+ * drbd_uuid_new_current() - Creates a new current UUID
  * @device:	DRBD device.
+ * @forced:	Force UUID creation
+ *
+ * Creates a new current UUID, and rotates the old current UUID into
+ * the bitmap slot. Causes an incremental resync upon next connect.
  */
-void drbd_md_sync(struct drbd_device *device)
+void drbd_uuid_new_current(struct drbd_device *device, bool forced)
 {
-	struct meta_data_on_disk *buffer;
+	if (get_ldev_if_state(device, D_UP_TO_DATE)) {
+		__drbd_uuid_new_current_send(device, forced);
+		put_ldev(device);
+	} else if (diskfull_peers_need_new_cur_uuid(device) ||
+		   a_lost_peer_is_on_same_cur_uuid(device)) {
+		struct drbd_peer_device *peer_device;
+		/* The peers will store the new current UUID... */
+		u64 current_uuid, weak_nodes;
+		get_random_bytes(&current_uuid, sizeof(u64));
+		if (device->resource->role[NOW] == R_PRIMARY)
+			current_uuid |= UUID_PRIMARY;
+		else
+			current_uuid &= ~UUID_PRIMARY;
+
+		down_write(&device->uuid_sem);
+		drbd_uuid_set_exposed(device, current_uuid, false);
+		downgrade_write(&device->uuid_sem);
+		drbd_info(device, "sending new current UUID: %016llX\n", current_uuid);
+
+		weak_nodes = drbd_weak_nodes_device(device);
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->repl_state[NOW] >= L_ESTABLISHED) {
+				drbd_send_current_uuid(peer_device, current_uuid, weak_nodes);
+				peer_device->current_uuid = current_uuid;
+			}
+		}
+		up_read(&device->uuid_sem);
+	}
+}
 
-	/* Don't accidentally change the DRBD meta data layout. */
-	BUILD_BUG_ON(UI_SIZE != 4);
-	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);
+void drbd_uuid_new_current_by_user(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
 
-	timer_delete(&device->md_sync_timer);
-	/* timer may be rearmed by drbd_md_mark_dirty() now. */
-	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
-		return;
+	down_write(&device->uuid_sem);
+	for_each_peer_device(peer_device, device)
+		drbd_uuid_set_bitmap(peer_device, 0); /* Rotate UI_BITMAP to History 1, etc... */
 
-	/* We use here D_FAILED and not D_ATTACHING because we try to write
-	 * metadata even if we detach due to a disk failure! */
-	if (!get_ldev_if_state(device, D_FAILED))
-		return;
+	if (get_ldev(device)) {
+		__drbd_uuid_new_current_holding_uuid_sem(device);
+		put_ldev(device);
+	}
+	up_write(&device->uuid_sem);
+}
 
-	buffer = drbd_md_get_buffer(device, __func__);
-	if (!buffer)
-		goto out;
+static void drbd_propagate_uuids(struct drbd_device *device, u64 nodes)
+{
+	struct drbd_peer_device *peer_device;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (!(nodes & NODE_MASK(peer_device->node_id)))
+			continue;
+
+		if (peer_device->repl_state[NOW] < L_ESTABLISHED)
+			continue;
+
+		if (list_empty(&peer_device->propagate_uuids_work.list))
+			drbd_queue_work(&peer_device->connection->sender_work,
+					&peer_device->propagate_uuids_work);
+	}
+	rcu_read_unlock();
+}
+
+void drbd_uuid_received_new_current(struct drbd_peer_device *from_pd, u64 val, u64 weak_nodes) __must_hold(local)
+{
+	struct drbd_device *device = from_pd->device;
+	u64 dagtag = atomic64_read(&from_pd->connection->last_dagtag_sector);
+	struct drbd_peer_device *peer_device;
+	u64 recipients = 0;
+	bool set_current = true;
+
+	down_write(&device->uuid_sem);
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->repl_state[NOW] == L_SYNC_TARGET ||
+		    peer_device->repl_state[NOW] == L_BEHIND      ||
+		    peer_device->repl_state[NOW] == L_PAUSED_SYNC_T) {
+			peer_device->current_uuid = val;
+			set_current = false;
+		}
+		if (peer_device->repl_state[NOW] == L_WF_BITMAP_S ||
+		    peer_device->repl_state[NOW] == L_SYNC_SOURCE ||
+		    peer_device->repl_state[NOW] == L_PAUSED_SYNC_S ||
+		    peer_device->repl_state[NOW] == L_ESTABLISHED)
+			recipients |= NODE_MASK(peer_device->node_id);
+
+		if (peer_device->disk_state[NOW] == D_DISKLESS)
+			recipients |= NODE_MASK(peer_device->node_id);
+	}
+	rcu_read_unlock();
+
+	if (set_current) {
+		u64 old_current = device->ldev->md.current_uuid;
+		u64 upd;
+
+		if (device->disk_state[NOW] == D_UP_TO_DATE)
+			recipients |= rotate_current_into_bitmap(device, weak_nodes, dagtag);
+
+		upd = ~weak_nodes; /* These nodes are connected to the primary */
+		upd &= __test_bitmap_slots(device); /* of those, I have a bitmap for */
+		__set_bitmap_slots(device, val, upd);
+		/* Setting bitmap to the (new) current-UUID, means, at this moment
+		   we know that we are at the same data as this not connected peer. */
+
+		__drbd_uuid_set_current(device, val);
+
+		/* Even when the old current UUID was not used as any bitmap
+		 * UUID, we still add it to the history. This is relevant, in
+		 * particular, when we afterwards perform a sync handshake with
+		 * a peer which is not one of the "weak_nodes", but hasn't
+		 * received the new current UUID. If we do not add the current
+		 * UUID to the history, we will end up with a spurious
+		 * unrelated data or split-brain decision. */
+		_drbd_uuid_push_history(device, old_current);
+	}
+
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
+	downgrade_write(&device->uuid_sem);
+	if (set_current)
+		drbd_propagate_uuids(device, recipients);
+	up_read(&device->uuid_sem);
+}
+
+static u64 __set_bitmap_slots(struct drbd_device *device, u64 bitmap_uuid, u64 do_nodes) __must_hold(local)
+{
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	u64 modified = 0;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == device->ldev->md.node_id)
+			continue;
+		if (!(do_nodes & NODE_MASK(node_id)))
+			continue;
+		if (!(peer_md[node_id].flags & MDF_HAVE_BITMAP))
+			continue;
+		if (peer_md[node_id].bitmap_uuid != bitmap_uuid) {
+			u64 previous_bitmap_uuid = peer_md[node_id].bitmap_uuid;
+			/* drbd_info(device, "XXX bitmap[node_id=%d] = %llX\n", node_id, bitmap_uuid); */
+			peer_md[node_id].bitmap_uuid = bitmap_uuid;
+			peer_md[node_id].bitmap_dagtag =
+				bitmap_uuid ? device->resource->dagtag_sector : 0;
+			_drbd_uuid_push_history(device, previous_bitmap_uuid);
+			drbd_md_mark_dirty(device);
+			modified |= NODE_MASK(node_id);
+		}
+	}
+
+	return modified;
+}
 
-	drbd_md_write(device, buffer);
+static u64 __test_bitmap_slots(struct drbd_device *device) __must_hold(local)
+{
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	int node_id;
+	u64 rv = 0;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (peer_md[node_id].bitmap_uuid)
+			rv |= NODE_MASK(node_id);
+	}
+
+	return rv;
+}
+
+/* __test_bitmap_slots_of_peer() operates on view of the world I know the
+   SyncSource had. It might be that in the mean time some peers sent more
+   recent UUIDs to me. Remove all peers that are on the same UUID as I am
+   now from the set of nodes */
+static u64 __test_bitmap_slots_of_peer(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	u64 set_bitmap_slots = 0;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		u64 bitmap_uuid = peer_device->bitmap_uuids[node_id];
+
+		if (bitmap_uuid != 0 && bitmap_uuid != -1)
+			set_bitmap_slots |= NODE_MASK(node_id);
+	}
+
+	return set_bitmap_slots;
+}
+
+static u64
+peers_with_current_uuid(struct drbd_device *device, u64 current_uuid)
+{
+	struct drbd_peer_device *peer_device;
+	u64 nodes = 0;
+
+	current_uuid &= ~UUID_PRIMARY;
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_disk_state peer_disk_state = peer_device->disk_state[NOW];
+		if (peer_disk_state < D_INCONSISTENT || peer_disk_state == D_UNKNOWN)
+			continue;
+		if (current_uuid == (peer_device->current_uuid & ~UUID_PRIMARY))
+			nodes |= NODE_MASK(peer_device->node_id);
+	}
+	rcu_read_unlock();
+
+	return nodes;
+}
+
+void drbd_uuid_resync_starting(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+
+	peer_device->rs_start_uuid = drbd_current_uuid(device);
+	if (peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY)
+		set_bit(SYNC_SRC_CRASHED_PRI, &peer_device->flags);
+	rotate_current_into_bitmap(device, 0, device->resource->dagtag_sector);
+}
+
+u64 drbd_uuid_resync_finished(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	unsigned long flags;
+	int i;
+	u64 ss_nz_bm; /* sync_source has non zero bitmap for. expressed as nodemask */
+	u64 pwcu; /* peers with current uuid */
+	u64 newer;
+
+	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
+	// Inherit history from the sync source
+	for (i = 0; i < ARRAY_SIZE(peer_device->history_uuids); i++)
+		_drbd_uuid_push_history(device, peer_device->history_uuids[i] & ~UUID_PRIMARY);
+
+	// Inherit history in bitmap UUIDs from the sync source
+	for (i = 0; i < DRBD_PEERS_MAX; i++)
+		if (peer_device->bitmap_uuids[i] != -1)
+			_drbd_uuid_push_history(device,
+					peer_device->bitmap_uuids[i] & ~UUID_PRIMARY);
+
+	ss_nz_bm = __test_bitmap_slots_of_peer(peer_device);
+	pwcu = peers_with_current_uuid(device, peer_device->current_uuid);
+
+	newer = __set_bitmap_slots(device, peer_device->rs_start_uuid, ss_nz_bm & ~pwcu);
+	__set_bitmap_slots(device, 0, ~ss_nz_bm | pwcu);
+	_drbd_uuid_push_history(device, drbd_current_uuid(device));
+	__drbd_uuid_set_current(device, peer_device->current_uuid);
+	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
+
+	return newer;
+}
+
+bool drbd_uuid_set_exposed(struct drbd_device *device, u64 val, bool log)
+{
+	if ((device->exposed_data_uuid & ~UUID_PRIMARY) == (val & ~UUID_PRIMARY) ||
+	    val == UUID_JUST_CREATED)
+		return false;
+
+	if (device->resource->role[NOW] == R_PRIMARY)
+		val |= UUID_PRIMARY;
+	else
+		val &= ~UUID_PRIMARY;
+
+	device->exposed_data_uuid = val;
+
+	if (log)
+		drbd_info(device, "Setting exposed data uuid: %016llX\n", (unsigned long long)val);
+
+	return true;
+}
 
-	/* Update device->ldev->md.la_size_sect,
-	 * since we updated it on metadata. */
-	device->ldev->md.la_size_sect = get_capacity(device->vdisk);
+static const char *name_of_node_id(struct drbd_resource *resource, int node_id)
+{
+	/* Caller need to hold rcu_read_lock */
+	struct drbd_connection *connection = drbd_connection_by_node_id(resource, node_id);
 
-	drbd_md_put_buffer(device);
-out:
-	put_ldev(device);
+	return connection ? rcu_dereference(connection->transport.net_conf)->name : "";
 }
 
-static int check_activity_log_stripe_size(struct drbd_device *device,
-		struct meta_data_on_disk *on_disk,
-		struct drbd_md *in_core)
+static void forget_bitmap(struct drbd_device *device, int node_id) __must_hold(local)
 {
-	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
-	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
-	u64 al_size_4k;
+	int bitmap_index = device->ldev->md.peers[node_id].bitmap_index;
+	const char *name;
 
-	/* both not set: default to old fixed size activity log */
-	if (al_stripes == 0 && al_stripe_size_4k == 0) {
-		al_stripes = 1;
-		al_stripe_size_4k = MD_32kB_SECT/8;
-	}
+	if (_drbd_bm_total_weight(device, bitmap_index) == 0)
+		return;
 
-	/* some paranoia plausibility checks */
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
+	rcu_read_lock();
+	name = name_of_node_id(device->resource, node_id);
+	drbd_info(device, "clearing bitmap UUID and content (%lu bits) for node %d (%s)(slot %d)\n",
+		  _drbd_bm_total_weight(device, bitmap_index), node_id, name, bitmap_index);
+	rcu_read_unlock();
+	drbd_suspend_io(device, WRITE_ONLY);
+	drbd_bm_lock(device, "forget_bitmap()", BM_LOCK_TEST | BM_LOCK_SET);
+	_drbd_bm_clear_many_bits(device, bitmap_index, 0, -1UL);
+	drbd_bm_unlock(device);
+	drbd_resume_io(device);
+	drbd_md_mark_dirty(device);
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+}
 
-	/* we need both values to be set */
-	if (al_stripes == 0 || al_stripe_size_4k == 0)
-		goto err;
+static void copy_bitmap(struct drbd_device *device, int from_id, int to_id) __must_hold(local)
+{
+	struct drbd_peer_device *peer_device = peer_device_by_node_id(device, to_id);
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	u64 previous_bitmap_uuid = peer_md[to_id].bitmap_uuid;
+	int from_index = peer_md[from_id].bitmap_index;
+	int to_index = peer_md[to_id].bitmap_index;
+	const char *from_name, *to_name;
 
-	al_size_4k = (u64)al_stripes * al_stripe_size_4k;
+	peer_md[to_id].bitmap_uuid = peer_md[from_id].bitmap_uuid;
+	peer_md[to_id].bitmap_dagtag = peer_md[from_id].bitmap_dagtag;
+	_drbd_uuid_push_history(device, previous_bitmap_uuid);
 
-	/* Upper limit of activity log area, to avoid potential overflow
-	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
-	 * than 72 * 4k blocks total only increases the amount of history,
-	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
-	if (al_size_4k > (16 * 1024 * 1024/4))
-		goto err;
+	/* Pretending that the updated UUID was sent is a hack.
+	   Unfortunately Necessary to not interrupt the handshake */
+	if (peer_device && peer_device->comm_bitmap_uuid == previous_bitmap_uuid)
+		peer_device->comm_bitmap_uuid = peer_md[from_id].bitmap_uuid;
 
-	/* Lower limit: we need at least 8 transaction slots (32kB)
-	 * to not break existing setups */
-	if (al_size_4k < MD_32kB_SECT/8)
-		goto err;
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
+	rcu_read_lock();
+	from_name = name_of_node_id(device->resource, from_id);
+	to_name = name_of_node_id(device->resource, to_id);
+	drbd_info(device, "Node %d (%s) synced up to node %d (%s). copying bitmap slot %d to %d.\n",
+		  to_id, to_name, from_id, from_name, from_index, to_index);
+	rcu_read_unlock();
+	drbd_suspend_io(device, WRITE_ONLY);
+	drbd_bm_lock(device, "copy_bitmap()", BM_LOCK_ALL);
+	drbd_bm_copy_slot(device, from_index, to_index);
+	drbd_bm_unlock(device);
+	drbd_resume_io(device);
+	drbd_md_mark_dirty(device);
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+}
 
-	in_core->al_stripe_size_4k = al_stripe_size_4k;
-	in_core->al_stripes = al_stripes;
-	in_core->al_size_4k = al_size_4k;
+static int find_node_id_by_bitmap_uuid(struct drbd_device *device, u64 bm_uuid) __must_hold(local)
+{
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	int node_id;
 
-	return 0;
-err:
-	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
-			al_stripes, al_stripe_size_4k);
-	return -EINVAL;
-}
-
-static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
-{
-	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
-	struct drbd_md *in_core = &bdev->md;
-	s32 on_disk_al_sect;
-	s32 on_disk_bm_sect;
-
-	/* The on-disk size of the activity log, calculated from offsets, and
-	 * the size of the activity log calculated from the stripe settings,
-	 * should match.
-	 * Though we could relax this a bit: it is ok, if the striped activity log
-	 * fits in the available on-disk activity log size.
-	 * Right now, that would break how resize is implemented.
-	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
-	 * of possible unused padding space in the on disk layout. */
-	if (in_core->al_offset < 0) {
-		if (in_core->bm_offset > in_core->al_offset)
-			goto err;
-		on_disk_al_sect = -in_core->al_offset;
-		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
-	} else {
-		if (in_core->al_offset != MD_4kB_SECT)
-			goto err;
-		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
-			goto err;
+	bm_uuid &= ~UUID_PRIMARY;
 
-		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
-		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if ((peer_md[node_id].bitmap_uuid & ~UUID_PRIMARY) == bm_uuid &&
+		    peer_md[node_id].flags & MDF_HAVE_BITMAP)
+			return node_id;
 	}
 
-	/* old fixed size meta data is exactly that: fixed. */
-	if (in_core->meta_dev_idx >= 0) {
-		if (in_core->md_size_sect != MD_128MB_SECT
-		||  in_core->al_offset != MD_4kB_SECT
-		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
-		||  in_core->al_stripes != 1
-		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
-			goto err;
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if ((peer_md[node_id].bitmap_uuid & ~UUID_PRIMARY) == bm_uuid)
+			return node_id;
 	}
 
-	if (capacity < in_core->md_size_sect)
-		goto err;
-	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
-		goto err;
-
-	/* should be aligned, and at least 32k */
-	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
-		goto err;
-
-	/* should fit (for now: exactly) into the available on-disk space;
-	 * overflow prevention is in check_activity_log_stripe_size() above. */
-	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
-		goto err;
-
-	/* again, should be aligned */
-	if (in_core->bm_offset & 7)
-		goto err;
+	return -1;
+}
 
-	/* FIXME check for device grow with flex external meta data? */
+static bool node_connected(struct drbd_resource *resource, int node_id)
+{
+	struct drbd_connection *connection;
+	bool r = false;
 
-	/* can the available bitmap space cover the last agreed device size? */
-	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
-		goto err;
+	rcu_read_lock();
+	connection = drbd_connection_by_node_id(resource, node_id);
+	if (connection)
+		r = connection->cstate[NOW] == C_CONNECTED;
+	rcu_read_unlock();
 
-	return 0;
+	return r;
+}
 
-err:
-	drbd_err(device, "meta data offsets don't make sense: idx=%d "
-			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
-			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
-			in_core->meta_dev_idx,
-			in_core->al_stripes, in_core->al_stripe_size_4k,
-			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
-			(unsigned long long)in_core->la_size_sect,
-			(unsigned long long)capacity);
+static bool detect_copy_ops_on_peer(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	struct drbd_resource *resource = device->resource;
+	int node_id1, node_id2, from_id;
+	u64 peer_bm_uuid;
+	bool modified = false;
 
-	return -EINVAL;
-}
+	for (node_id1 = 0; node_id1 < DRBD_NODE_ID_MAX; node_id1++) {
+		if (device->ldev->md.peers[node_id1].bitmap_index == -1)
+			continue;
 
+		if (node_connected(resource, node_id1))
+			continue;
 
-/**
- * drbd_md_read() - Reads in the meta data super block
- * @device:	DRBD device.
- * @bdev:	Device from which the meta data should be read in.
- *
- * Return NO_ERROR on success, and an enum drbd_ret_code in case
- * something goes wrong.
- *
- * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
- * even before @bdev is assigned to @device->ldev.
- */
-int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
-{
-	struct meta_data_on_disk *buffer;
-	u32 magic, flags;
-	int i, rv = NO_ERROR;
+		peer_bm_uuid = peer_device->bitmap_uuids[node_id1];
+		if (peer_bm_uuid == 0 || peer_bm_uuid == -1ULL)
+			continue;
 
-	if (device->state.disk != D_DISKLESS)
-		return ERR_DISK_CONFIGURED;
+		peer_bm_uuid &= ~UUID_PRIMARY;
+		for (node_id2 = node_id1 + 1; node_id2 < DRBD_NODE_ID_MAX; node_id2++) {
+			if (device->ldev->md.peers[node_id2].bitmap_index == -1)
+				continue;
 
-	buffer = drbd_md_get_buffer(device, __func__);
-	if (!buffer)
-		return ERR_NOMEM;
+			if (node_connected(resource, node_id2))
+				continue;
 
-	/* First, figure out where our meta data superblock is located,
-	 * and read it. */
-	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
-	bdev->md.md_offset = drbd_md_ss(bdev);
-	/* Even for (flexible or indexed) external meta data,
-	 * initially restrict us to the 4k superblock for now.
-	 * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */
-	bdev->md.md_size_sect = 8;
-
-	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset,
-				 REQ_OP_READ)) {
-		/* NOTE: can't do normal error processing here as this is
-		   called BEFORE disk is attached */
-		drbd_err(device, "Error while reading metadata.\n");
-		rv = ERR_IO_MD_DISK;
-		goto err;
-	}
-
-	magic = be32_to_cpu(buffer->magic);
-	flags = be32_to_cpu(buffer->flags);
-	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
-	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
-			/* btw: that's Activity Log clean, not "all" clean. */
-		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
-		rv = ERR_MD_UNCLEAN;
-		goto err;
-	}
-
-	rv = ERR_MD_INVALID;
-	if (magic != DRBD_MD_MAGIC_08) {
-		if (magic == DRBD_MD_MAGIC_07)
-			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
-		else
-			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
-		goto err;
+			if (peer_bm_uuid == (peer_device->bitmap_uuids[node_id2] & ~UUID_PRIMARY))
+				goto found;
+		}
 	}
+	return false;
 
-	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
-		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
-		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
-		goto err;
+found:
+	from_id = find_node_id_by_bitmap_uuid(device, peer_bm_uuid);
+	if (from_id == -1) {
+		if (peer_md[node_id1].bitmap_uuid == 0 && peer_md[node_id2].bitmap_uuid == 0)
+			return false;
+		drbd_err(peer_device, "unexpected\n");
+		drbd_err(peer_device, "In UUIDs from node %d found equal UUID (%llX) for nodes %d %d\n",
+			 peer_device->node_id, peer_bm_uuid, node_id1, node_id2);
+		drbd_err(peer_device, "I have %llX for node_id=%d\n",
+			 peer_md[node_id1].bitmap_uuid, node_id1);
+		drbd_err(peer_device, "I have %llX for node_id=%d\n",
+			 peer_md[node_id2].bitmap_uuid, node_id2);
+		return false;
 	}
 
+	if (!(peer_md[from_id].flags & MDF_HAVE_BITMAP))
+		return false;
 
-	/* convert to in_core endian */
-	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
-	for (i = UI_CURRENT; i < UI_SIZE; i++)
-		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
-	bdev->md.flags = be32_to_cpu(buffer->flags);
-	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);
-
-	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
-	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
-	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);
-
-	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
-		goto err;
-	if (check_offsets_and_sizes(device, bdev))
-		goto err;
+	if (from_id != node_id1 &&
+	    peer_md[node_id1].bitmap_uuid != peer_bm_uuid) {
+		copy_bitmap(device, from_id, node_id1);
+		modified = true;
 
-	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
-		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
-		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
-		goto err;
 	}
-	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
-		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
-		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
-		goto err;
-	}
-
-	rv = NO_ERROR;
-
-	spin_lock_irq(&device->resource->req_lock);
-	if (device->state.conn < C_CONNECTED) {
-		unsigned int peer;
-		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
-		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
-		device->peer_max_bio_size = peer;
+	if (from_id != node_id2 &&
+	    peer_md[node_id2].bitmap_uuid != peer_bm_uuid) {
+		copy_bitmap(device, from_id, node_id2);
+		modified = true;
 	}
-	spin_unlock_irq(&device->resource->req_lock);
-
- err:
-	drbd_md_put_buffer(device);
 
-	return rv;
+	return modified;
 }
 
-/**
- * drbd_md_mark_dirty() - Mark meta data super block as dirty
- * @device:	DRBD device.
- *
- * Call this function if you change anything that should be written to
- * the meta-data super block. This function sets MD_DIRTY, and starts a
- * timer that ensures that within five seconds you have to call drbd_md_sync().
- */
-void drbd_md_mark_dirty(struct drbd_device *device)
+void drbd_uuid_detect_finished_resyncs(struct drbd_peer_device *peer_device) __must_hold(local)
 {
-	if (!test_and_set_bit(MD_DIRTY, &device->flags))
-		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
-}
+	u64 peer_current_uuid = peer_device->current_uuid & ~UUID_PRIMARY;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	const int my_node_id = device->resource->res_opts.node_id;
+	bool write_bm = false;
+	bool filled = false;
+	bool current_equal;
+	int node_id;
 
-void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
-{
-	int i;
+	current_equal = peer_current_uuid == (drbd_resolved_uuid(peer_device, NULL) & ~UUID_PRIMARY) &&
+		!(peer_device->uuid_flags & UUID_FLAG_SYNC_TARGET) &&
+		!(peer_device->comm_uuid_flags & UUID_FLAG_SYNC_TARGET);
 
-	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
-		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
-}
+	spin_lock_irq(&device->ldev->md.uuid_lock);
 
-void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
-{
-	if (idx == UI_CURRENT) {
-		if (device->state.role == R_PRIMARY)
-			val |= 1;
-		else
-			val &= ~((u64)1);
+	if (peer_device->repl_state[NOW] == L_OFF && current_equal) {
+		u64 bm_to_peer = peer_device->comm_bitmap_uuid & ~UUID_PRIMARY;
+		u64 bm_towards_me = peer_device->bitmap_uuids[my_node_id] & ~UUID_PRIMARY;
 
-		drbd_set_ed_uuid(device, val);
+		if (bm_towards_me != 0 && bm_to_peer == 0 &&
+		    bm_towards_me != peer_current_uuid) {
+			drbd_info(peer_device, "Peer missed end of resync\n");
+			set_bit(RS_PEER_MISSED_END, &peer_device->flags);
+		}
+		if (bm_towards_me == 0 && bm_to_peer != 0 &&
+		    bm_to_peer != peer_current_uuid) {
+			drbd_info(peer_device, "Missed end of resync as sync-source\n");
+			set_bit(RS_SOURCE_MISSED_END, &peer_device->flags);
+		}
+		spin_unlock_irq(&device->ldev->md.uuid_lock);
+		return;
 	}
 
-	device->ldev->md.uuid[idx] = val;
-	drbd_md_mark_dirty(device);
-}
-
-void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
-	__drbd_uuid_set(device, idx, val);
-	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
-}
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_device *pd2;
 
-void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
-{
-	unsigned long flags;
-	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
-	if (device->ldev->md.uuid[idx]) {
-		drbd_uuid_move_history(device);
-		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
-	}
-	__drbd_uuid_set(device, idx, val);
-	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
-}
+		if (node_id == device->ldev->md.node_id)
+			continue;
 
-/**
- * drbd_uuid_new_current() - Creates a new current UUID
- * @device:	DRBD device.
- *
- * Creates a new current UUID, and rotates the old current UUID into
- * the bitmap slot. Causes an incremental resync upon next connect.
- */
-void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
-{
-	u64 val;
-	unsigned long long bm_uuid;
+		if (!(peer_md[node_id].flags & MDF_HAVE_BITMAP) && !(peer_md[node_id].flags & MDF_NODE_EXISTS))
+			continue;
 
-	get_random_bytes(&val, sizeof(u64));
+		pd2 = peer_device_by_node_id(device, node_id);
+		if (pd2 && pd2 != peer_device && pd2->repl_state[NOW] > L_ESTABLISHED)
+			continue;
 
-	spin_lock_irq(&device->ldev->md.uuid_lock);
-	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
+		if (peer_device->bitmap_uuids[node_id] == 0 && peer_md[node_id].bitmap_uuid != 0) {
+			int from_node_id;
+
+			if (current_equal) {
+				u64 previous_bitmap_uuid = peer_md[node_id].bitmap_uuid;
+				peer_md[node_id].bitmap_uuid = 0;
+				_drbd_uuid_push_history(device, previous_bitmap_uuid);
+				if (node_id == peer_device->node_id)
+					drbd_print_uuids(peer_device, "updated UUIDs");
+				else if (peer_md[node_id].flags & MDF_HAVE_BITMAP)
+					forget_bitmap(device, node_id);
+				else
+					drbd_info(device, "Clearing bitmap UUID for node %d\n",
+						  node_id);
+				drbd_md_mark_dirty(device);
+				write_bm = true;
+			}
 
-	if (bm_uuid)
-		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
+			from_node_id = find_node_id_by_bitmap_uuid(device, peer_current_uuid);
+			if (from_node_id != -1 && node_id != from_node_id &&
+			    dagtag_newer(peer_md[from_node_id].bitmap_dagtag,
+					 peer_md[node_id].bitmap_dagtag)) {
+				if (peer_md[node_id].flags & MDF_HAVE_BITMAP &&
+				    peer_md[from_node_id].flags & MDF_HAVE_BITMAP)
+					copy_bitmap(device, from_node_id, node_id);
+				else
+					drbd_info(device, "Node %d synced up to node %d.\n",
+						  node_id, from_node_id);
+				drbd_md_mark_dirty(device);
+				filled = true;
+			}
+		}
+	}
 
-	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
-	__drbd_uuid_set(device, UI_CURRENT, val);
+	write_bm |= detect_copy_ops_on_peer(peer_device);
 	spin_unlock_irq(&device->ldev->md.uuid_lock);
 
-	drbd_print_uuids(device, "new current UUID");
-	/* get it to stable storage _now_ */
-	drbd_md_sync(device);
+	if (write_bm || filled) {
+		u64 to_nodes = filled ? -1 : ~NODE_MASK(peer_device->node_id);
+		drbd_propagate_uuids(device, to_nodes);
+		drbd_suspend_io(device, WRITE_ONLY);
+		drbd_bm_lock(device, "detect_finished_resyncs()", BM_LOCK_BULK);
+		drbd_bm_write(device, NULL);
+		drbd_bm_unlock(device);
+		drbd_resume_io(device);
+	}
 }
 
-void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
+int drbd_bmio_set_all_n_write(struct drbd_device *device,
+			      struct drbd_peer_device *peer_device) __must_hold(local)
 {
-	unsigned long flags;
-	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
-	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0) {
-		spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
-		return;
-	}
-
-	if (val == 0) {
-		drbd_uuid_move_history(device);
-		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
-		device->ldev->md.uuid[UI_BITMAP] = 0;
-	} else {
-		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
-		if (bm_uuid)
-			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
-
-		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
-	}
-	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
-
-	drbd_md_mark_dirty(device);
+	drbd_bm_set_all(device);
+	return drbd_bm_write(device, NULL);
 }
 
 /**
@@ -3384,22 +5670,21 @@ void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
  * @device:	DRBD device.
  * @peer_device: Peer DRBD device.
  *
- * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
+ * Sets all bits in the bitmap towards one peer and writes the whole bitmap to stable storage.
  */
 int drbd_bmio_set_n_write(struct drbd_device *device,
 			  struct drbd_peer_device *peer_device) __must_hold(local)
-
 {
 	int rv = -EIO;
 
-	drbd_md_set_flag(device, MDF_FULL_SYNC);
+	drbd_md_set_peer_flag(peer_device, MDF_PEER_FULL_SYNC);
 	drbd_md_sync(device);
-	drbd_bm_set_all(device);
+	drbd_bm_set_many_bits(peer_device, 0, -1UL);
 
-	rv = drbd_bm_write(device, peer_device);
+	rv = drbd_bm_write(device, NULL);
 
 	if (!rv) {
-		drbd_md_clear_flag(device, MDF_FULL_SYNC);
+		drbd_md_clear_peer_flag(peer_device, MDF_PEER_FULL_SYNC);
 		drbd_md_sync(device);
 	}
 
@@ -3407,67 +5692,109 @@ int drbd_bmio_set_n_write(struct drbd_device *device,
 }
 
 /**
- * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
+ * drbd_bmio_set_allocated_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
+ * @device:	DRBD device.
+ * @peer_device: parameter ignored
+ *
+ * Sets all bits in all allocated bitmap slots and writes it to stable storage.
+ */
+int drbd_bmio_set_allocated_n_write(struct drbd_device *device,
+				    struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	const int my_node_id = device->resource->res_opts.node_id;
+	struct drbd_md *md = &device->ldev->md;
+	int rv = -EIO;
+	int node_id, bitmap_index;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == my_node_id)
+			continue;
+		bitmap_index = md->peers[node_id].bitmap_index;
+		if (bitmap_index == -1)
+			continue;
+		_drbd_bm_set_many_bits(device, bitmap_index, 0, -1UL);
+	}
+	rv = drbd_bm_write(device, NULL);
+
+	return rv;
+}
+
+/**
+ * drbd_bmio_clear_all_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
  * @device:	DRBD device.
  * @peer_device: Peer DRBD device.
  *
  * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
  */
-int drbd_bmio_clear_n_write(struct drbd_device *device,
-			  struct drbd_peer_device *peer_device) __must_hold(local)
-
+int drbd_bmio_clear_all_n_write(struct drbd_device *device,
+			    struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	drbd_resume_al(device);
 	drbd_bm_clear_all(device);
-	return drbd_bm_write(device, peer_device);
+	return drbd_bm_write(device, NULL);
+}
+
+int drbd_bmio_clear_one_peer(struct drbd_device *device,
+			     struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	drbd_bm_clear_many_bits(peer_device, 0, -1UL);
+	return drbd_bm_write(device, NULL);
 }
 
 static int w_bitmap_io(struct drbd_work *w, int unused)
 {
-	struct drbd_device *device =
-		container_of(w, struct drbd_device, bm_io_work.w);
-	struct bm_io_work *work = &device->bm_io_work;
+	struct bm_io_work *work =
+		container_of(w, struct bm_io_work, w);
+	struct drbd_device *device = work->device;
 	int rv = -EIO;
 
-	if (work->flags != BM_LOCKED_CHANGE_ALLOWED) {
-		int cnt = atomic_read(&device->ap_bio_cnt);
-		if (cnt)
-			drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n",
-					cnt, work->why);
-	}
-
 	if (get_ldev(device)) {
-		drbd_bm_lock(device, work->why, work->flags);
+		if (work->flags & BM_LOCK_SINGLE_SLOT)
+			drbd_bm_slot_lock(work->peer_device, work->why, work->flags);
+		else
+			drbd_bm_lock(device, work->why, work->flags);
 		rv = work->io_fn(device, work->peer_device);
-		drbd_bm_unlock(device);
+		if (work->flags & BM_LOCK_SINGLE_SLOT)
+			drbd_bm_slot_unlock(work->peer_device);
+		else
+			drbd_bm_unlock(device);
 		put_ldev(device);
 	}
 
-	clear_bit_unlock(BITMAP_IO, &device->flags);
-	wake_up(&device->misc_wait);
-
 	if (work->done)
-		work->done(device, rv);
+		work->done(device, work->peer_device, rv);
 
-	clear_bit(BITMAP_IO_QUEUED, &device->flags);
-	work->why = NULL;
-	work->flags = 0;
+	if (atomic_dec_and_test(&device->pending_bitmap_work.n))
+		wake_up(&device->misc_wait);
+	kfree(work);
 
 	return 0;
 }
 
+void drbd_queue_pending_bitmap_work(struct drbd_device *device)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&device->pending_bitmap_work.q_lock, flags);
+	spin_lock(&device->resource->work.q_lock);
+	list_splice_tail_init(&device->pending_bitmap_work.q, &device->resource->work.q);
+	spin_unlock(&device->resource->work.q_lock);
+	spin_unlock_irqrestore(&device->pending_bitmap_work.q_lock, flags);
+	wake_up(&device->resource->work.q_wait);
+}
+
 /**
  * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
  * @device:	DRBD device.
  * @io_fn:	IO callback to be called when bitmap IO is possible
  * @done:	callback to be called after the bitmap IO was performed
  * @why:	Descriptive text of the reason for doing the IO
- * @flags:	Bitmap flags
+ * @flags:	Bitmap operation flags
  * @peer_device: Peer DRBD device.
  *
  * While IO on the bitmap happens we freeze application IO thus we ensure
  * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
- * called from worker context. It MUST NOT be used while a previous such
+ * called from sender context. It MUST NOT be used while a previous such
  * work is still pending!
  *
  * Its worker function encloses the call of io_fn() by get_ldev() and
@@ -3475,35 +5802,63 @@ static int w_bitmap_io(struct drbd_work *w, int unused)
  */
 void drbd_queue_bitmap_io(struct drbd_device *device,
 			  int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
-			  void (*done)(struct drbd_device *, int),
+			  void (*done)(struct drbd_device *, struct drbd_peer_device *, int),
 			  char *why, enum bm_flag flags,
 			  struct drbd_peer_device *peer_device)
 {
-	D_ASSERT(device, current == peer_device->connection->worker.task);
-
-	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
-	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
-	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
-	if (device->bm_io_work.why)
-		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
-			why, device->bm_io_work.why);
-
-	device->bm_io_work.peer_device = peer_device;
-	device->bm_io_work.io_fn = io_fn;
-	device->bm_io_work.done = done;
-	device->bm_io_work.why = why;
-	device->bm_io_work.flags = flags;
-
-	spin_lock_irq(&device->resource->req_lock);
-	set_bit(BITMAP_IO, &device->flags);
-	/* don't wait for pending application IO if the caller indicates that
-	 * application IO does not conflict anyways. */
-	if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
-		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
-			drbd_queue_work(&peer_device->connection->sender_work,
-					&device->bm_io_work.w);
+	struct bm_io_work *bm_io_work;
+
+	D_ASSERT(device, current == device->resource->worker.task);
+
+	bm_io_work = kmalloc_obj(*bm_io_work, GFP_NOIO);
+	if (!bm_io_work) {
+		if (done)
+			done(device, peer_device, -ENOMEM);
+		return;
 	}
-	spin_unlock_irq(&device->resource->req_lock);
+	bm_io_work->w.cb = w_bitmap_io;
+	bm_io_work->device = device;
+	bm_io_work->peer_device = peer_device;
+	bm_io_work->io_fn = io_fn;
+	bm_io_work->done = done;
+	bm_io_work->why = why;
+	bm_io_work->flags = flags;
+
+	/*
+	 * Whole-bitmap operations can only take place when there is no
+	 * concurrent application I/O.  We ensure exclusion between the two
+	 * types of I/O  with the following mechanism:
+	 *
+	 *  - device->ap_bio_cnt keeps track of the number of application I/O
+	 *    requests in progress.
+	 *
+	 *  - A non-empty device->pending_bitmap_work list indicates that
+	 *    whole-bitmap I/O operations are pending, and no new application
+	 *    I/O should be started.  We make sure that the list doesn't appear
+	 *    empty system wide before trying to queue the whole-bitmap I/O.
+	 *
+	 *  - In dec_ap_bio(), we decrement device->ap_bio_cnt.  If it reaches
+	 *    zero and the device->pending_bitmap_work list is non-empty, we
+	 *    queue the whole-bitmap operations.
+	 *
+	 *  - In inc_ap_bio(), we increment device->ap_bio_cnt before checking
+	 *    if the device->pending_bitmap_work list is non-empty.  If
+	 *    device->pending_bitmap_work is non-empty, we immediately call
+	 *    dec_ap_bio().
+	 *
+	 * This ensures that whenever there is pending whole-bitmap I/O, we
+	 * realize in dec_ap_bio().
+	 *
+	 */
+
+	/* no one should accidentally schedule the next bitmap IO
+	 * when it is only half-queued yet */
+	atomic_inc(&device->ap_bio_cnt[WRITE]);
+	atomic_inc(&device->pending_bitmap_work.n);
+	spin_lock_irq(&device->pending_bitmap_work.q_lock);
+	list_add_tail(&bm_io_work->w.list, &device->pending_bitmap_work.q);
+	spin_unlock_irq(&device->pending_bitmap_work.q_lock);
+	dec_ap_bio(device, WRITE);  /* may move to actual work queue */
 }
 
 /**
@@ -3511,11 +5866,11 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
  * @device:	DRBD device.
  * @io_fn:	IO callback to be called when bitmap IO is possible
  * @why:	Descriptive text of the reason for doing the IO
- * @flags:	Bitmap flags
+ * @flags:	Bitmap operation flags
  * @peer_device: Peer DRBD device.
  *
  * freezes application IO while that the actual IO operations runs. This
- * functions MAY NOT be called from worker context.
+ * functions MAY NOT be called from sender context.
  */
 int drbd_bitmap_io(struct drbd_device *device,
 		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
@@ -3523,17 +5878,28 @@ int drbd_bitmap_io(struct drbd_device *device,
 		struct drbd_peer_device *peer_device)
 {
 	/* Only suspend io, if some operation is supposed to be locked out */
-	const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST);
+	const bool do_suspend_io = flags & (BM_LOCK_CLEAR|BM_LOCK_SET|BM_LOCK_TEST);
 	int rv;
 
-	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
+	D_ASSERT(device, current != device->resource->worker.task);
+
+	if (!device->bitmap)
+		return 0;
 
 	if (do_suspend_io)
-		drbd_suspend_io(device);
+		drbd_suspend_io(device, WRITE_ONLY);
+
+	if (flags & BM_LOCK_SINGLE_SLOT)
+		drbd_bm_slot_lock(peer_device, why, flags);
+	else
+		drbd_bm_lock(device, why, flags);
 
-	drbd_bm_lock(device, why, flags);
 	rv = io_fn(device, peer_device);
-	drbd_bm_unlock(device);
+
+	if (flags & BM_LOCK_SINGLE_SLOT)
+		drbd_bm_slot_unlock(peer_device);
+	else
+		drbd_bm_unlock(device);
 
 	if (do_suspend_io)
 		drbd_resume_io(device);
@@ -3541,142 +5907,52 @@ int drbd_bitmap_io(struct drbd_device *device,
 	return rv;
 }
 
-void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
+void drbd_md_set_peer_flag(struct drbd_peer_device *peer_device,
+			   enum mdf_peer_flag flag) __must_hold(local)
 {
-	if ((device->ldev->md.flags & flag) != flag) {
+	struct drbd_device *device = peer_device->device;
+	struct drbd_md *md = &device->ldev->md;
+
+	if (!(md->peers[peer_device->node_id].flags & flag)) {
 		drbd_md_mark_dirty(device);
-		device->ldev->md.flags |= flag;
+		md->peers[peer_device->node_id].flags |= flag;
 	}
 }
 
-void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
+void drbd_md_clear_peer_flag(struct drbd_peer_device *peer_device,
+			     enum mdf_peer_flag flag) __must_hold(local)
 {
-	if ((device->ldev->md.flags & flag) != 0) {
+	struct drbd_device *device = peer_device->device;
+	struct drbd_md *md = &device->ldev->md;
+
+	if (md->peers[peer_device->node_id].flags & flag) {
 		drbd_md_mark_dirty(device);
-		device->ldev->md.flags &= ~flag;
+		md->peers[peer_device->node_id].flags &= ~flag;
 	}
 }
-int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
+
+int drbd_md_test_flag(struct drbd_backing_dev *bdev, enum mdf_flag flag)
 {
 	return (bdev->md.flags & flag) != 0;
 }
 
-static void md_sync_timer_fn(struct timer_list *t)
+bool drbd_md_test_peer_flag(struct drbd_peer_device *peer_device, enum mdf_peer_flag flag)
 {
-	struct drbd_device *device = timer_container_of(device, t,
-							md_sync_timer);
-	drbd_device_post_work(device, MD_SYNC);
-}
+	struct drbd_md *md = &peer_device->device->ldev->md;
 
-const char *cmdname(enum drbd_packet cmd)
-{
-	/* THINK may need to become several global tables
-	 * when we want to support more than
-	 * one PRO_VERSION */
-	static const char *cmdnames[] = {
-
-		[P_DATA]	        = "Data",
-		[P_DATA_REPLY]	        = "DataReply",
-		[P_RS_DATA_REPLY]	= "RSDataReply",
-		[P_BARRIER]	        = "Barrier",
-		[P_BITMAP]	        = "ReportBitMap",
-		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
-		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
-		[P_UNPLUG_REMOTE]	= "UnplugRemote",
-		[P_DATA_REQUEST]	= "DataRequest",
-		[P_RS_DATA_REQUEST]     = "RSDataRequest",
-		[P_SYNC_PARAM]	        = "SyncParam",
-		[P_PROTOCOL]            = "ReportProtocol",
-		[P_UUIDS]	        = "ReportUUIDs",
-		[P_SIZES]	        = "ReportSizes",
-		[P_STATE]	        = "ReportState",
-		[P_SYNC_UUID]           = "ReportSyncUUID",
-		[P_AUTH_CHALLENGE]      = "AuthChallenge",
-		[P_AUTH_RESPONSE]	= "AuthResponse",
-		[P_STATE_CHG_REQ]       = "StateChgRequest",
-		[P_PING]		= "Ping",
-		[P_PING_ACK]	        = "PingAck",
-		[P_RECV_ACK]	        = "RecvAck",
-		[P_WRITE_ACK]	        = "WriteAck",
-		[P_RS_WRITE_ACK]	= "RSWriteAck",
-		[P_SUPERSEDED]          = "Superseded",
-		[P_NEG_ACK]	        = "NegAck",
-		[P_NEG_DREPLY]	        = "NegDReply",
-		[P_NEG_RS_DREPLY]	= "NegRSDReply",
-		[P_BARRIER_ACK]	        = "BarrierAck",
-		[P_STATE_CHG_REPLY]     = "StateChgReply",
-		[P_OV_REQUEST]          = "OVRequest",
-		[P_OV_REPLY]            = "OVReply",
-		[P_OV_RESULT]           = "OVResult",
-		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
-		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
-		[P_SYNC_PARAM89]	= "SyncParam89",
-		[P_COMPRESSED_BITMAP]   = "CBitmap",
-		[P_DELAY_PROBE]         = "DelayProbe",
-		[P_OUT_OF_SYNC]		= "OutOfSync",
-		[P_RS_CANCEL]		= "RSCancel",
-		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
-		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
-		[P_PROTOCOL_UPDATE]	= "protocol_update",
-		[P_TRIM]	        = "Trim",
-		[P_RS_THIN_REQ]         = "rs_thin_req",
-		[P_RS_DEALLOCATED]      = "rs_deallocated",
-		[P_WSAME]	        = "WriteSame",
-		[P_ZEROES]		= "Zeroes",
-
-		/* enum drbd_packet, but not commands - obsoleted flags:
-		 *	P_MAY_IGNORE
-		 *	P_MAX_OPT_CMD
-		 */
-	};
+	if (peer_device->bitmap_index == -1)
+		return false;
 
-	/* too big for the array: 0xfffX */
-	if (cmd == P_INITIAL_META)
-		return "InitialMeta";
-	if (cmd == P_INITIAL_DATA)
-		return "InitialData";
-	if (cmd == P_CONNECTION_FEATURES)
-		return "ConnectionFeatures";
-	if (cmd >= ARRAY_SIZE(cmdnames))
-		return "Unknown";
-	return cmdnames[cmd];
+	return md->peers[peer_device->node_id].flags & flag;
 }
 
-/**
- * drbd_wait_misc  -  wait for a request to make progress
- * @device:	device associated with the request
- * @i:		the struct drbd_interval embedded in struct drbd_request or
- *		struct drbd_peer_request
- */
-int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
+static void md_sync_timer_fn(struct timer_list *t)
 {
-	struct net_conf *nc;
-	DEFINE_WAIT(wait);
-	long timeout;
-
-	rcu_read_lock();
-	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-	if (!nc) {
-		rcu_read_unlock();
-		return -ETIMEDOUT;
-	}
-	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
-	rcu_read_unlock();
-
-	/* Indicate to wake up device->misc_wait on progress.  */
-	i->waiting = true;
-	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
-	spin_unlock_irq(&device->resource->req_lock);
-	timeout = schedule_timeout(timeout);
-	finish_wait(&device->misc_wait, &wait);
-	spin_lock_irq(&device->resource->req_lock);
-	if (!timeout || device->state.conn < C_CONNECTED)
-		return -ETIMEDOUT;
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-	return 0;
+	struct drbd_device *device = timer_container_of(device, t, md_sync_timer);
+	drbd_device_post_work(device, MD_SYNC);
 }
 
+
 void lock_all_resources(void)
 {
 	struct drbd_resource *resource;
@@ -3685,7 +5961,7 @@ void lock_all_resources(void)
 	mutex_lock(&resources_mutex);
 	local_irq_disable();
 	for_each_resource(resource, &drbd_resources)
-		spin_lock_nested(&resource->req_lock, i++);
+		read_lock(&resource->state_rwlock);
 }
 
 void unlock_all_resources(void)
@@ -3693,11 +5969,141 @@ void unlock_all_resources(void)
 	struct drbd_resource *resource;
 
 	for_each_resource(resource, &drbd_resources)
-		spin_unlock(&resource->req_lock);
+		read_unlock(&resource->state_rwlock);
 	local_irq_enable();
 	mutex_unlock(&resources_mutex);
 }
 
+long twopc_timeout(struct drbd_resource *resource)
+{
+	return resource->res_opts.twopc_timeout * HZ/10;
+}
+
+u64 directly_connected_nodes(struct drbd_resource *resource, enum which_state which)
+{
+	u64 directly_connected = 0;
+	struct drbd_connection *connection;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[which] < C_CONNECTED)
+			continue;
+		directly_connected |= NODE_MASK(connection->peer_node_id);
+	}
+	rcu_read_unlock();
+
+	return directly_connected;
+}
+
+static sector_t bm_sect_to_max_capacity(const struct drbd_md *md, sector_t bm_sect)
+{
+	/* we do our meta data IO in 4k units */
+	u64 bm_bytes = ALIGN_DOWN(bm_sect << SECTOR_SHIFT, 4096);
+	u64 bm_bytes_per_peer = div_u64(bm_bytes, md->max_peers);
+	u64 bm_bits_per_peer = bm_bytes_per_peer * BITS_PER_BYTE;
+	return bm_bits_per_peer << (md->bm_block_shift - SECTOR_SHIFT);
+}
+
+
+/**
+ * drbd_get_max_capacity() - Returns the capacity for user-data on the local backing device
+ * @device: The DRBD device.
+ * @bdev: Meta data block device.
+ * @warn: Whether to warn when size is clipped.
+ *
+ * This function returns the capacity for user-data on the local backing
+ * device. In the case of internal meta-data, this is the backing disk size
+ * reduced by the meta-data size. In the case of external meta-data, this is
+ * the size of the backing disk.
+ */
+sector_t drbd_get_max_capacity(
+		struct drbd_device *device, struct drbd_backing_dev *bdev, bool warn)
+{
+	unsigned int bm_max_peers = bdev->md.max_peers;
+	unsigned int bm_block_size = bdev->md.bm_block_size;
+	sector_t backing_bdev_capacity = drbd_get_capacity(bdev->backing_bdev);
+	sector_t bm_sect;
+	sector_t backing_capacity_remaining;
+	sector_t metadata_limit;
+	sector_t max_capacity;
+
+	switch (bdev->md.meta_dev_idx) {
+	case DRBD_MD_INDEX_INTERNAL:
+	case DRBD_MD_INDEX_FLEX_INT:
+		bm_sect = bdev->md.al_offset - bdev->md.bm_offset;
+		backing_capacity_remaining = drbd_md_first_sector(bdev);
+		break;
+	case DRBD_MD_INDEX_FLEX_EXT:
+		bm_sect = bdev->md.md_size_sect - bdev->md.bm_offset;
+		backing_capacity_remaining = backing_bdev_capacity;
+		break;
+	default:
+		bm_sect = DRBD_BM_SECTORS_INDEXED;
+		backing_capacity_remaining = backing_bdev_capacity;
+	}
+
+	metadata_limit = bm_sect_to_max_capacity(&bdev->md, bm_sect);
+
+	dynamic_drbd_dbg(device,
+			"Backing device capacity: %llus, remaining: %llus, bitmap sectors: %llus\n",
+			(unsigned long long) backing_bdev_capacity,
+			(unsigned long long) backing_capacity_remaining,
+			(unsigned long long) bm_sect);
+	dynamic_drbd_dbg(device,
+			"Max peers: %u, bytes_per_bit: %u, metadata limit: %llus, hard limit: %llus\n",
+			bm_max_peers, bm_block_size,
+			(unsigned long long) metadata_limit,
+			(unsigned long long) DRBD_MAX_SECTORS);
+
+	max_capacity = backing_capacity_remaining;
+	if (max_capacity > DRBD_MAX_SECTORS) {
+		if (warn)
+			drbd_warn(device, "Device size clipped from %llus to %llus due to DRBD limitations\n",
+					(unsigned long long) max_capacity,
+					(unsigned long long) DRBD_MAX_SECTORS);
+		max_capacity = DRBD_MAX_SECTORS;
+	}
+	if (max_capacity > metadata_limit) {
+		if (warn)
+			drbd_warn(device, "Device size clipped from %llus to %llus due to metadata size\n",
+					(unsigned long long) max_capacity,
+					(unsigned long long) metadata_limit);
+		max_capacity = metadata_limit;
+	}
+	return max_capacity;
+}
+
+/* this is about cluster partitions, not block device partitions */
+sector_t drbd_partition_data_capacity(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	sector_t capacity = (sector_t)(-1);
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (test_bit(HAVE_SIZES, &peer_device->flags)) {
+			dynamic_drbd_dbg(peer_device, "d_size: %llus\n",
+					(unsigned long long)peer_device->d_size);
+			capacity = min_not_zero(capacity, peer_device->d_size);
+		}
+	}
+	rcu_read_unlock();
+
+	if (get_ldev_if_state(device, D_ATTACHING)) {
+		/* In case we somehow end up here while attaching, but before
+		 * we even assigned the ldev, pretend to still be diskless.
+		 */
+		if (device->ldev != NULL) {
+			sector_t local_capacity = drbd_local_max_size(device);
+
+			capacity = min_not_zero(capacity, local_capacity);
+		}
+		put_ldev(device);
+	}
+
+	return capacity != (sector_t)(-1) ? capacity : 0;
+}
+
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 /* Fault insertion support including random number generator shamelessly
  * stolen from kernel/rcutorture.c */
@@ -3753,14 +6159,13 @@ _drbd_insert_fault(struct drbd_device *device, unsigned int type)
 
 	unsigned int ret = (
 		(drbd_fault_devs == 0 ||
-			((1 << device_to_minor(device)) & drbd_fault_devs) != 0) &&
+			((1 << device->minor) & drbd_fault_devs) != 0) &&
 		(((_drbd_fault_random(&rrs) % 100) + 1) <= drbd_fault_rate));
 
 	if (ret) {
 		drbd_fault_count++;
 
-		if (drbd_ratelimit())
-			drbd_warn(device, "***Simulating %s failure\n",
+		drbd_warn_ratelimit(device, "***Simulating %s failure\n",
 				_drbd_fault_str(type));
 	}
 
@@ -3771,7 +6176,6 @@ _drbd_insert_fault(struct drbd_device *device, unsigned int type)
 module_init(drbd_init)
 module_exit(drbd_cleanup)
 
-EXPORT_SYMBOL(drbd_conn_str);
-EXPORT_SYMBOL(drbd_role_str);
-EXPORT_SYMBOL(drbd_disk_str);
-EXPORT_SYMBOL(drbd_set_st_err_str);
+/* For transport layer */
+EXPORT_SYMBOL(drbd_destroy_connection);
+EXPORT_SYMBOL(drbd_destroy_path);
diff --git a/drivers/block/drbd/drbd_meta_data.h b/drivers/block/drbd/drbd_meta_data.h
new file mode 100644
index 0000000000000..ddbf2a0a1a6c8
--- /dev/null
+++ b/drivers/block/drbd/drbd_meta_data.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef DRBD_META_DATA_H
+#define DRBD_META_DATA_H
+
+#ifdef __KERNEL__
+#define be_u64 __be64
+#define be_u32 __be32
+#define be_s32 __be32
+#define be_u16 __be16
+#else
+#define be_u64 struct { uint64_t be; }
+#define be_u32 struct { uint32_t be; }
+#define be_s32 struct { int32_t be; }
+#define be_u16 struct { uint16_t be; }
+#endif
+
+/* how I came up with this magic?
+ * base64 decode "actlog==" ;) */
+#define DRBD_AL_MAGIC 0x69cb65a2
+
+#define BM_BLOCK_SHIFT_4k	12			 /* 4k per bit */
+#define BM_BLOCK_SHIFT_MIN	BM_BLOCK_SHIFT_4k
+#define BM_BLOCK_SHIFT_MAX	20
+#define BM_BLOCK_SIZE_4k	4096
+#define BM_BLOCK_SIZE_MIN	(1<<BM_BLOCK_SHIFT_MIN)
+#define BM_BLOCK_SIZE_MAX	(1<<BM_BLOCK_SHIFT_MAX)
+
+struct peer_dev_md_on_disk_9 {
+	be_u64 bitmap_uuid;
+	be_u64 bitmap_dagtag;
+	be_u32 flags;
+	be_s32 bitmap_index;
+	be_u32 reserved_u32[2];
+} __packed;
+
+struct meta_data_on_disk_9 {
+	be_u64 effective_size;    /* last agreed size */
+	be_u64 current_uuid;
+	be_u64 members;		  /* only if MDF_HAVE_MEMBERS_MASK is in the flags */
+	be_u64 reserved_u64[3];   /* to have the magic at the same position as in v07, and v08 */
+	be_u64 device_uuid;
+	be_u32 flags;             /* MDF */
+	be_u32 magic;
+	be_u32 md_size_sect;
+	be_u32 al_offset;         /* offset to this block */
+	be_u32 al_nr_extents;     /* important for restoring the AL */
+	be_u32 bm_offset;         /* offset to the bitmap, from here */
+	be_u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
+	be_u32 la_peer_max_bio_size;   /* last peer max_bio_size */
+	be_u32 bm_max_peers;
+	be_s32 node_id;
+
+	/* see al_tr_number_to_on_disk_sector() */
+	be_u32 al_stripes;
+	be_u32 al_stripe_size_4k;
+
+	be_u32 reserved_u32[2];
+
+	struct peer_dev_md_on_disk_9 peers[DRBD_PEERS_MAX];
+	be_u64 history_uuids[HISTORY_UUIDS];
+
+	unsigned char padding_start[0];
+	unsigned char padding_end[0] __aligned(4096);
+} __packed;
+
+/* Attention, these two are defined in drbd_int.h as well! */
+#define AL_UPDATES_PER_TRANSACTION 64
+#define AL_CONTEXT_PER_TRANSACTION 919
+
+enum al_transaction_types {
+	AL_TR_UPDATE = 0,
+	AL_TR_INITIALIZED = 0xffff
+};
+/* all fields on disc in big endian */
+struct __packed al_transaction_on_disk {
+	/* don't we all like magic */
+	be_u32	magic;
+
+	/* to identify the most recent transaction block
+	 * in the on disk ring buffer */
+	be_u32	tr_number;
+
+	/* checksum on the full 4k block, with this field set to 0. */
+	be_u32	crc32c;
+
+	/* type of transaction, special transaction types like:
+	 * purge-all, set-all-idle, set-all-active, ... to-be-defined
+	 * see also enum al_transaction_types */
+	be_u16	transaction_type;
+
+	/* we currently allow only a few thousand extents,
+	 * so 16bit will be enough for the slot number. */
+
+	/* how many updates in this transaction */
+	be_u16	n_updates;
+
+	/* maximum slot number, "al-extents" in drbd.conf speak.
+	 * Having this in each transaction should make reconfiguration
+	 * of that parameter easier. */
+	be_u16	context_size;
+
+	/* slot number the context starts with */
+	be_u16	context_start_slot_nr;
+
+	/* Some reserved bytes.  Expected usage is a 64bit counter of
+	 * sectors-written since device creation, and other data generation tag
+	 * supporting usage */
+	be_u32	__reserved[4];
+
+	/* --- 36 byte used --- */
+
+	/* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes
+	 * in one transaction, then use the remaining byte in the 4k block for
+	 * context information.  "Flexible" number of updates per transaction
+	 * does not help, as we have to account for the case when all update
+	 * slots are used anyways, so it would only complicate code without
+	 * additional benefit.
+	 */
+	be_u16	update_slot_nr[AL_UPDATES_PER_TRANSACTION];
+
+	/* but the extent number is 32bit, which at an extent size of 4 MiB
+	 * allows to cover device sizes of up to 2**54 Byte (16 PiB) */
+	be_u32	update_extent_nr[AL_UPDATES_PER_TRANSACTION];
+
+	/* --- 420 bytes used (36 + 64*6) --- */
+
+	/* 4096 - 420 = 3676 = 919 * 4 */
+	be_u32	context[AL_CONTEXT_PER_TRANSACTION];
+};
+
+#define DRBD_AL_PMEM_MAGIC 0x6aa667a6 /* "al==pmem" */
+
+struct __packed al_on_pmem {
+	be_u32 magic;
+	be_u32 slots[];
+};
+
+#undef be_u64
+#undef be_u32
+#undef be_s32
+#undef be_u16
+
+#endif
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index e201f0087a0f7..c5e253820ccf3 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -19,66 +19,80 @@
 #include <linux/fs.h>
 #include <linux/file.h>
 #include <linux/slab.h>
-#include <linux/blkpg.h>
 #include <linux/cpumask.h>
+#include <linux/random.h>
 #include "drbd_int.h"
-#include "drbd_protocol.h"
-#include "drbd_req.h"
+#include <linux/drbd_protocol.h>
 #include "drbd_state_change.h"
-#include <linux/unaligned.h>
+#include "drbd_debugfs.h"
+#include "drbd_transport.h"
+#include "drbd_dax_pmem.h"
 #include <linux/drbd_limits.h>
 #include <linux/kthread.h>
-
+#include <linux/security.h>
 #include <net/genetlink.h>
+#include <net/sock.h>
+
+#include "drbd_meta_data.h"
+#include "drbd_legacy_84.h"
 
 /* .doit */
-// int drbd_adm_create_resource(struct sk_buff *skb, struct genl_info *info);
-// int drbd_adm_delete_resource(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_down(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info);
+
+static int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_down(struct sk_buff *skb, struct genl_info *info);
+
+static int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_new_peer(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_del_peer(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_new_path(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_del_path(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_peer_device_opts(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_forget_peer(struct sk_buff *skb, struct genl_info *info);
+static int drbd_adm_rename_resource(struct sk_buff *skb, struct genl_info *info);
 /* .dumpit */
-int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_devices_done(struct netlink_callback *cb);
-int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_connections_done(struct netlink_callback *cb);
-int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb);
-int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_devices_done(struct netlink_callback *cb);
+static int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_connections_done(struct netlink_callback *cb);
+static int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb);
+static int drbd_adm_dump_paths(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_dump_paths_done(struct netlink_callback *cb);
+static int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb);
+static int drbd_adm_get_initial_state_done(struct netlink_callback *cb);
 
 #include <linux/drbd_genl_api.h>
 #include "drbd_nla.h"
 #include <linux/genl_magic_func.h>
 
-static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
-static atomic_t notify_genl_seq = ATOMIC_INIT(2); /* two. */
+void drbd_enable_netns(void)
+{
+	drbd_genl_family.netnsok = true;
+}
+
+atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
 
 DEFINE_MUTEX(notification_mutex);
 
@@ -110,11 +124,15 @@ static int drbd_msg_put_info(struct sk_buff *skb, const char *info)
 	if (err) {
 		nla_nest_cancel(skb, nla);
 		return err;
-	} else
-		nla_nest_end(skb, nla);
+	}
+	nla_nest_end(skb, nla);
 	return 0;
 }
 
+static int drbd_adm_finish(struct drbd_config_context *, struct genl_info *, int);
+
+extern struct genl_ops drbd_genl_ops[];
+
 __printf(2, 3)
 static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...)
 {
@@ -122,6 +140,8 @@ static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...)
 	struct nlattr *nla, *txt;
 	int err = -EMSGSIZE;
 	int len;
+	int aligned_len;
+	char *msg_buf;
 
 	nla = nla_nest_start_noflag(skb, DRBD_NLA_CFG_REPLY);
 	if (!nla)
@@ -132,30 +152,56 @@ static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...)
 		nla_nest_cancel(skb, nla);
 		return err;
 	}
+	msg_buf = nla_data(txt);
 	va_start(args, fmt);
-	len = vscnprintf(nla_data(txt), 256, fmt, args);
+	len = vscnprintf(msg_buf, 256, fmt, args);
 	va_end(args);
 
 	/* maybe: retry with larger reserve, if truncated */
-	txt->nla_len = nla_attr_size(len+1);
-	nlmsg_trim(skb, (char*)txt + NLA_ALIGN(txt->nla_len));
+
+	/* zero-out padding bytes to avoid transmitting uninitialized bytes */
+	++len;
+	txt->nla_len = nla_attr_size(len);
+	aligned_len = NLA_ALIGN(len);
+	while (len < aligned_len) {
+		msg_buf[len] = '\0';
+		++len;
+	}
+	nlmsg_trim(skb, (char *) txt + NLA_ALIGN(txt->nla_len));
 	nla_nest_end(skb, nla);
 
 	return 0;
 }
 
+static bool need_sys_admin(u8 cmd)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(drbd_genl_ops); i++)
+		if (drbd_genl_ops[i].cmd == cmd)
+			return 0 != (drbd_genl_ops[i].flags & GENL_ADMIN_PERM);
+	return true;
+}
+
+static struct drbd_path *first_path(struct drbd_connection *connection)
+{
+	/* Ideally this function is removed at a later point in time.
+	   It was introduced when replacing the single address pair
+	   with a list of address pairs (or paths). */
+
+	return list_first_or_null_rcu(&connection->transport.paths, struct drbd_path, list);
+}
+
 /* This would be a good candidate for a "pre_doit" hook,
  * and per-family private info->pointers.
  * But we need to stay compatible with older kernels.
  * If it returns successfully, adm_ctx members are valid.
- *
- * At this point, we still rely on the global genl_lock().
- * If we want to avoid that, and allow "genl_family.parallel_ops", we may need
- * to add additional synchronization against object destruction/modification.
  */
-#define DRBD_ADM_NEED_MINOR	1
-#define DRBD_ADM_NEED_RESOURCE	2
-#define DRBD_ADM_NEED_CONNECTION 4
+#define DRBD_ADM_NEED_MINOR        (1 << 0)
+#define DRBD_ADM_NEED_RESOURCE     (1 << 1)
+#define DRBD_ADM_NEED_CONNECTION   (1 << 2)
+#define DRBD_ADM_NEED_PEER_DEVICE  (1 << 3)
+#define DRBD_ADM_NEED_PEER_NODE    (1 << 4)
+#define DRBD_ADM_IGNORE_VERSION    (1 << 5)
 static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 	struct sk_buff *skb, struct genl_info *info, unsigned flags)
 {
@@ -165,9 +211,15 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 
 	memset(adm_ctx, 0, sizeof(*adm_ctx));
 
-	/* genl_rcv_msg only checks for CAP_NET_ADMIN on "GENL_ADMIN_PERM" :( */
-	if (cmd != DRBD_ADM_GET_STATUS && !capable(CAP_NET_ADMIN))
-	       return -EPERM;
+	adm_ctx->net = sock_net(skb->sk);
+
+	/*
+	 * genl_rcv_msg() only checks if commands with the GENL_ADMIN_PERM flag
+	 * set have CAP_NET_ADMIN; we also require CAP_SYS_ADMIN for
+	 * administrative commands.
+	 */
+	if (need_sys_admin(cmd) && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
 
 	adm_ctx->reply_skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 	if (!adm_ctx->reply_skb) {
@@ -184,14 +236,29 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 		goto fail;
 	}
 
+	if (info->genlhdr->version != GENL_MAGIC_VERSION && (flags & DRBD_ADM_IGNORE_VERSION) == 0) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "Wrong API version, upgrade your drbd utils.");
+		err = -EINVAL;
+		goto fail;
+	}
+
+	if (flags & DRBD_ADM_NEED_PEER_DEVICE)
+		flags |= DRBD_ADM_NEED_CONNECTION;
+	if (flags & DRBD_ADM_NEED_CONNECTION)
+		flags |= DRBD_ADM_NEED_PEER_NODE;
+	if (flags & DRBD_ADM_NEED_PEER_NODE)
+		flags |= DRBD_ADM_NEED_RESOURCE;
+
 	adm_ctx->reply_dh->minor = d_in->minor;
 	adm_ctx->reply_dh->ret_code = NO_ERROR;
 
 	adm_ctx->volume = VOLUME_UNSPECIFIED;
+	adm_ctx->peer_node_id = PEER_NODE_ID_UNSPECIFIED;
 	if (info->attrs[DRBD_NLA_CFG_CONTEXT]) {
 		struct nlattr *nla;
+		struct nlattr **nested_attr_tb;
 		/* parse and validate only */
-		err = drbd_cfg_context_from_attrs(NULL, info);
+		err = drbd_cfg_context_ntb_from_attrs(&nested_attr_tb, info);
 		if (err)
 			goto fail;
 
@@ -207,108 +274,142 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 		nla = nested_attr_tb[__nla_type(T_ctx_volume)];
 		if (nla)
 			adm_ctx->volume = nla_get_u32(nla);
+		nla = nested_attr_tb[__nla_type(T_ctx_peer_node_id)];
+		if (nla)
+			adm_ctx->peer_node_id = nla_get_u32(nla);
 		nla = nested_attr_tb[__nla_type(T_ctx_resource_name)];
 		if (nla)
 			adm_ctx->resource_name = nla_data(nla);
-		adm_ctx->my_addr = nested_attr_tb[__nla_type(T_ctx_my_addr)];
-		adm_ctx->peer_addr = nested_attr_tb[__nla_type(T_ctx_peer_addr)];
-		if ((adm_ctx->my_addr &&
-		     nla_len(adm_ctx->my_addr) > sizeof(adm_ctx->connection->my_addr)) ||
-		    (adm_ctx->peer_addr &&
-		     nla_len(adm_ctx->peer_addr) > sizeof(adm_ctx->connection->peer_addr))) {
-			err = -EINVAL;
-			goto fail;
-		}
+		kfree(nested_attr_tb);
+	}
+
+	if (adm_ctx->resource_name) {
+		adm_ctx->resource = drbd_find_resource(adm_ctx->resource_name);
 	}
 
 	adm_ctx->minor = d_in->minor;
+	rcu_read_lock();
 	adm_ctx->device = minor_to_device(d_in->minor);
-
-	/* We are protected by the global genl_lock().
-	 * But we may explicitly drop it/retake it in drbd_adm_set_role(),
-	 * so make sure this object stays around. */
-	if (adm_ctx->device)
+	if (adm_ctx->device) {
 		kref_get(&adm_ctx->device->kref);
-
-	if (adm_ctx->resource_name) {
-		adm_ctx->resource = drbd_find_resource(adm_ctx->resource_name);
 	}
+	rcu_read_unlock();
 
 	if (!adm_ctx->device && (flags & DRBD_ADM_NEED_MINOR)) {
 		drbd_msg_put_info(adm_ctx->reply_skb, "unknown minor");
-		return ERR_MINOR_INVALID;
+		err = ERR_MINOR_INVALID;
+		goto finish;
 	}
 	if (!adm_ctx->resource && (flags & DRBD_ADM_NEED_RESOURCE)) {
 		drbd_msg_put_info(adm_ctx->reply_skb, "unknown resource");
+		err = ERR_INVALID_REQUEST;
 		if (adm_ctx->resource_name)
-			return ERR_RES_NOT_KNOWN;
-		return ERR_INVALID_REQUEST;
+			err = ERR_RES_NOT_KNOWN;
+		goto finish;
 	}
-
-	if (flags & DRBD_ADM_NEED_CONNECTION) {
-		if (adm_ctx->resource) {
-			drbd_msg_put_info(adm_ctx->reply_skb, "no resource name expected");
-			return ERR_INVALID_REQUEST;
+	if (adm_ctx->peer_node_id != PEER_NODE_ID_UNSPECIFIED) {
+		/* peer_node_id is unsigned int */
+		if (adm_ctx->peer_node_id >= DRBD_NODE_ID_MAX) {
+			drbd_msg_put_info(adm_ctx->reply_skb, "peer node id out of range");
+			err = ERR_INVALID_REQUEST;
+			goto finish;
 		}
-		if (adm_ctx->device) {
-			drbd_msg_put_info(adm_ctx->reply_skb, "no minor number expected");
-			return ERR_INVALID_REQUEST;
+		if (adm_ctx->resource && adm_ctx->peer_node_id == adm_ctx->resource->res_opts.node_id) {
+			drbd_msg_put_info(adm_ctx->reply_skb, "peer node id cannot be my own node id");
+			err = ERR_INVALID_REQUEST;
+			goto finish;
 		}
-		if (adm_ctx->my_addr && adm_ctx->peer_addr)
-			adm_ctx->connection = conn_get_by_addrs(nla_data(adm_ctx->my_addr),
-							  nla_len(adm_ctx->my_addr),
-							  nla_data(adm_ctx->peer_addr),
-							  nla_len(adm_ctx->peer_addr));
+		adm_ctx->connection = drbd_get_connection_by_node_id(adm_ctx->resource, adm_ctx->peer_node_id);
+	} else if (flags & DRBD_ADM_NEED_PEER_NODE) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "peer node id missing");
+		err = ERR_INVALID_REQUEST;
+		goto finish;
+	}
+	if (flags & DRBD_ADM_NEED_CONNECTION) {
 		if (!adm_ctx->connection) {
 			drbd_msg_put_info(adm_ctx->reply_skb, "unknown connection");
-			return ERR_INVALID_REQUEST;
+			err = ERR_INVALID_REQUEST;
+			goto finish;
+		}
+	}
+	if (flags & DRBD_ADM_NEED_PEER_DEVICE) {
+		rcu_read_lock();
+		if (adm_ctx->volume != VOLUME_UNSPECIFIED)
+			adm_ctx->peer_device =
+				idr_find(&adm_ctx->connection->peer_devices,
+					 adm_ctx->volume);
+		if (!adm_ctx->peer_device) {
+			drbd_msg_put_info(adm_ctx->reply_skb, "unknown volume");
+			err = ERR_INVALID_REQUEST;
+			rcu_read_unlock();
+			goto finish;
+		}
+		if (!adm_ctx->device) {
+			adm_ctx->device = adm_ctx->peer_device->device;
+			kref_get(&adm_ctx->device->kref);
 		}
+		rcu_read_unlock();
 	}
 
 	/* some more paranoia, if the request was over-determined */
 	if (adm_ctx->device && adm_ctx->resource &&
 	    adm_ctx->device->resource != adm_ctx->resource) {
 		pr_warn("request: minor=%u, resource=%s; but that minor belongs to resource %s\n",
-			adm_ctx->minor, adm_ctx->resource->name,
-			adm_ctx->device->resource->name);
+				adm_ctx->minor, adm_ctx->resource->name,
+				adm_ctx->device->resource->name);
 		drbd_msg_put_info(adm_ctx->reply_skb, "minor exists in different resource");
-		return ERR_INVALID_REQUEST;
+		err = ERR_INVALID_REQUEST;
+		goto finish;
 	}
 	if (adm_ctx->device &&
 	    adm_ctx->volume != VOLUME_UNSPECIFIED &&
 	    adm_ctx->volume != adm_ctx->device->vnr) {
 		pr_warn("request: minor=%u, volume=%u; but that minor is volume %u in %s\n",
-			adm_ctx->minor, adm_ctx->volume,
-			adm_ctx->device->vnr, adm_ctx->device->resource->name);
+				adm_ctx->minor, adm_ctx->volume,
+				adm_ctx->device->vnr,
+				adm_ctx->device->resource->name);
 		drbd_msg_put_info(adm_ctx->reply_skb, "minor exists as different volume");
-		return ERR_INVALID_REQUEST;
+		err = ERR_INVALID_REQUEST;
+		goto finish;
+	}
+	if (adm_ctx->device && adm_ctx->peer_device &&
+	    adm_ctx->resource && adm_ctx->resource->name &&
+	    adm_ctx->peer_device->device != adm_ctx->device) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "peer_device->device != device");
+		pr_warn("request: minor=%u, resource=%s, volume=%u, peer_node=%u; device != peer_device->device\n",
+				adm_ctx->minor, adm_ctx->resource->name,
+				adm_ctx->device->vnr, adm_ctx->peer_node_id);
+		err = ERR_INVALID_REQUEST;
+		goto finish;
 	}
 
 	/* still, provide adm_ctx->resource always, if possible. */
 	if (!adm_ctx->resource) {
 		adm_ctx->resource = adm_ctx->device ? adm_ctx->device->resource
 			: adm_ctx->connection ? adm_ctx->connection->resource : NULL;
-		if (adm_ctx->resource)
+		if (adm_ctx->resource) {
 			kref_get(&adm_ctx->resource->kref);
+		}
 	}
-
 	return NO_ERROR;
 
 fail:
 	nlmsg_free(adm_ctx->reply_skb);
 	adm_ctx->reply_skb = NULL;
 	return err;
+
+finish:
+	return drbd_adm_finish(adm_ctx, info, err);
 }
 
-static int drbd_adm_finish(struct drbd_config_context *adm_ctx,
-	struct genl_info *info, int retcode)
+static int drbd_adm_finish(struct drbd_config_context *adm_ctx, struct genl_info *info, int retcode)
 {
 	if (adm_ctx->device) {
 		kref_put(&adm_ctx->device->kref, drbd_destroy_device);
 		adm_ctx->device = NULL;
 	}
 	if (adm_ctx->connection) {
-		kref_put(&adm_ctx->connection->kref, &drbd_destroy_connection);
+		kref_put(&adm_ctx->connection->kref, drbd_destroy_connection);
 		adm_ctx->connection = NULL;
 	}
 	if (adm_ctx->resource) {
@@ -321,220 +422,404 @@ static int drbd_adm_finish(struct drbd_config_context *adm_ctx,
 
 	adm_ctx->reply_dh->ret_code = retcode;
 	drbd_adm_send_reply(adm_ctx->reply_skb, info);
+	adm_ctx->reply_skb = NULL;
 	return 0;
 }
 
-static void setup_khelper_env(struct drbd_connection *connection, char **envp)
+static void conn_md_sync(struct drbd_connection *connection)
 {
-	char *afs;
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	/* FIXME: A future version will not allow this case. */
-	if (connection->my_addr_len == 0 || connection->peer_addr_len == 0)
-		return;
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		drbd_md_sync_if_dirty(device);
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+}
+
+/* Try to figure out where we are happy to become primary.
+   This is unsed by the crm-fence-peer mechanism
+*/
+static u64 up_to_date_nodes(struct drbd_device *device, bool op_is_fence)
+{
+	struct drbd_resource *resource = device->resource;
+	const int my_node_id = resource->res_opts.node_id;
+	u64 mask = NODE_MASK(my_node_id);
+
+	if (resource->role[NOW] == R_PRIMARY || op_is_fence) {
+		struct drbd_peer_device *peer_device;
+
+		rcu_read_lock();
+		for_each_peer_device_rcu(peer_device, device) {
+			enum drbd_disk_state pdsk = peer_device->disk_state[NOW];
+			if (pdsk == D_UP_TO_DATE)
+				mask |= NODE_MASK(peer_device->node_id);
+		}
+		rcu_read_unlock();
+	} else if (device->disk_state[NOW] == D_UP_TO_DATE) {
+		struct drbd_peer_md *peer_md = device->ldev->md.peers;
+		int node_id;
+
+		for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+			struct drbd_peer_device *peer_device;
+			if (node_id == my_node_id)
+				continue;
+
+			peer_device = peer_device_by_node_id(device, node_id);
 
-	switch (((struct sockaddr *)&connection->peer_addr)->sa_family) {
+			if ((peer_device && peer_device->disk_state[NOW] == D_UP_TO_DATE) ||
+			    (peer_md[node_id].flags & MDF_NODE_EXISTS &&
+			     peer_md[node_id].bitmap_uuid == 0))
+				mask |= NODE_MASK(node_id);
+		}
+	} else
+		  mask = 0;
+
+	return mask;
+}
+
+/* Buffer to construct the environment of a user-space helper in. */
+struct env {
+	char *buffer;
+	int size, pos;
+};
+
+/* Print into an env buffer. */
+static __printf(2, 3) int env_print(struct env *env, const char *fmt, ...)
+{
+	va_list args;
+	int pos, ret;
+
+	pos = env->pos;
+	if (pos < 0)
+		return pos;
+	va_start(args, fmt);
+	ret = vsnprintf(env->buffer + pos, env->size - pos, fmt, args);
+	va_end(args);
+	if (ret < 0) {
+		env->pos = ret;
+		goto out;
+	}
+	if (ret >= env->size - pos) {
+		ret = env->pos = -ENOMEM;
+		goto out;
+	}
+	env->pos += ret + 1;
+    out:
+	return ret;
+}
+
+/* Put env variables for an address into an env buffer. */
+static void env_print_address(struct env *env, const char *prefix,
+			      struct sockaddr_storage *storage)
+{
+	const char *afs;
+
+	switch (storage->ss_family) {
 	case AF_INET6:
 		afs = "ipv6";
-		snprintf(envp[4], 60, "DRBD_PEER_ADDRESS=%pI6",
-			 &((struct sockaddr_in6 *)&connection->peer_addr)->sin6_addr);
+		env_print(env, "%sADDRESS=%pI6", prefix,
+			  &((struct sockaddr_in6 *)storage)->sin6_addr);
 		break;
 	case AF_INET:
 		afs = "ipv4";
-		snprintf(envp[4], 60, "DRBD_PEER_ADDRESS=%pI4",
-			 &((struct sockaddr_in *)&connection->peer_addr)->sin_addr);
+		env_print(env, "%sADDRESS=%pI4", prefix,
+			  &((struct sockaddr_in *)storage)->sin_addr);
 		break;
 	default:
 		afs = "ssocks";
-		snprintf(envp[4], 60, "DRBD_PEER_ADDRESS=%pI4",
-			 &((struct sockaddr_in *)&connection->peer_addr)->sin_addr);
+		env_print(env, "%sADDRESS=%pI4", prefix,
+			  &((struct sockaddr_in *)storage)->sin_addr);
 	}
-	snprintf(envp[3], 20, "DRBD_PEER_AF=%s", afs);
+	env_print(env, "%sAF=%s", prefix, afs);
+}
+
+/* Construct char **envp inside an env buffer. */
+static char **make_envp(struct env *env)
+{
+	char **envp, *b;
+	unsigned int n;
+
+	if (env->pos < 0)
+		return NULL;
+	if (env->pos >= env->size)
+		goto out_nomem;
+	env->buffer[env->pos++] = 0;
+	for (b = env->buffer, n = 1; *b; n++)
+		b = strchr(b, 0) + 1;
+	if (env->size - env->pos < sizeof(envp) * n)
+		goto out_nomem;
+	envp = (char **)(env->buffer + env->size) - n;
+
+	for (b = env->buffer; *b; ) {
+		*envp++ = b;
+		b = strchr(b, 0) + 1;
+	}
+	*envp++ = NULL;
+	return envp - n;
+
+    out_nomem:
+	env->pos = -ENOMEM;
+	return NULL;
 }
 
-int drbd_khelper(struct drbd_device *device, char *cmd)
+/* Macro refers to local variables peer_device, device and connection! */
+#define magic_printk(level, fmt, args...)				\
+	do {								\
+		if (peer_device)					\
+			drbd_printk(NOLIMIT, level, peer_device, fmt, args);	\
+		else if (device)					\
+			drbd_printk(NOLIMIT, level, device, fmt, args);	\
+		else							\
+			drbd_printk(NOLIMIT, level, connection, fmt, args);	\
+	} while (0)
+
+static int drbd_khelper(struct drbd_device *device, struct drbd_connection *connection, char *cmd)
 {
-	char *envp[] = { "HOME=/",
-			"TERM=linux",
-			"PATH=/sbin:/usr/sbin:/bin:/usr/bin",
-			 (char[20]) { }, /* address family */
-			 (char[60]) { }, /* address */
-			NULL };
-	char mb[14];
-	char *argv[] = {drbd_usermode_helper, cmd, mb, NULL };
-	struct drbd_connection *connection = first_peer_device(device)->connection;
-	struct sib_info sib;
+	struct drbd_resource *resource = device ? device->resource : connection->resource;
+	char *argv[] = { drbd_usermode_helper, cmd, resource->name, NULL };
+	struct drbd_peer_device *peer_device = NULL;
+	struct env env = { .size = PAGE_SIZE };
+	char **envp;
 	int ret;
 
-	if (current == connection->worker.task)
-		set_bit(CALLBACK_PENDING, &connection->flags);
+    enlarge_buffer:
+	env.buffer = (char *)__get_free_pages(GFP_NOIO, get_order(env.size));
+	if (!env.buffer) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+	env.pos = 0;
+
+	rcu_read_lock();
+	env_print(&env, "HOME=/");
+	env_print(&env, "TERM=linux");
+	env_print(&env, "PATH=/sbin:/usr/sbin:/bin:/usr/bin");
+	if (device) {
+		env_print(&env, "DRBD_MINOR=%u", device->minor);
+		env_print(&env, "DRBD_VOLUME=%u", device->vnr);
+		if (get_ldev(device)) {
+			struct disk_conf *disk_conf =
+				rcu_dereference(device->ldev->disk_conf);
+			env_print(&env, "DRBD_BACKING_DEV=%s",
+				  disk_conf->backing_dev);
+			put_ldev(device);
+		}
+	}
+	if (connection) {
+		struct drbd_path *path;
+
+		rcu_read_lock();
+		path = first_path(connection);
+		if (path) {
+			/* TO BE DELETED */
+			env_print_address(&env, "DRBD_MY_", &path->my_addr);
+			env_print_address(&env, "DRBD_PEER_", &path->peer_addr);
+		}
+		rcu_read_unlock();
+
+		env_print(&env, "DRBD_PEER_NODE_ID=%u", connection->peer_node_id);
+		env_print(&env, "DRBD_CSTATE=%s", drbd_conn_str(connection->cstate[NOW]));
+	}
+	if (connection && !device) {
+		struct drbd_peer_device *peer_device;
+		int vnr;
+
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			struct drbd_device *device = peer_device->device;
+
+			env_print(&env, "DRBD_MINOR_%u=%u",
+				  vnr, peer_device->device->minor);
+			if (get_ldev(device)) {
+				struct disk_conf *disk_conf =
+					rcu_dereference(device->ldev->disk_conf);
+				env_print(&env, "DRBD_BACKING_DEV_%u=%s",
+					  vnr, disk_conf->backing_dev);
+				put_ldev(device);
+			}
+		}
+	}
+	rcu_read_unlock();
+
+	if (strstr(cmd, "fence")) {
+		bool op_is_fence = strcmp(cmd, "fence-peer") == 0;
+		struct drbd_peer_device *peer_device;
+		u64 mask = -1ULL;
+		int vnr;
+
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			struct drbd_device *device = peer_device->device;
 
-	snprintf(mb, 14, "minor-%d", device_to_minor(device));
-	setup_khelper_env(connection, envp);
+			if (get_ldev(device)) {
+				u64 m = up_to_date_nodes(device, op_is_fence);
+				if (m)
+					mask &= m;
+				put_ldev(device);
+				/* Yes we outright ignore volumes that are not up-to-date
+				   on a single node. */
+			}
+		}
+		env_print(&env, "UP_TO_DATE_NODES=0x%08llX", mask);
+	}
+
+	envp = make_envp(&env);
+	if (!envp) {
+		if (env.pos == -ENOMEM) {
+			free_pages((unsigned long)env.buffer, get_order(env.size));
+			env.size += PAGE_SIZE;
+			goto enlarge_buffer;
+		}
+		ret = env.pos;
+		goto out_err;
+	}
+
+	if (current == resource->worker.task)
+		set_bit(CALLBACK_PENDING, &resource->flags);
 
 	/* The helper may take some time.
 	 * write out any unsynced meta data changes now */
-	drbd_md_sync(device);
+	if (device)
+		drbd_md_sync_if_dirty(device);
+	else if (connection)
+		conn_md_sync(connection);
 
-	drbd_info(device, "helper command: %s %s %s\n", drbd_usermode_helper, cmd, mb);
-	sib.sib_reason = SIB_HELPER_PRE;
-	sib.helper_name = cmd;
-	drbd_bcast_event(device, &sib);
+	if (connection && device)
+		peer_device = conn_peer_device(connection, device->vnr);
+
+	magic_printk(KERN_INFO, "helper command: %s %s\n", drbd_usermode_helper, cmd);
 	notify_helper(NOTIFY_CALL, device, connection, cmd, 0);
 	ret = call_usermodehelper(drbd_usermode_helper, argv, envp, UMH_WAIT_PROC);
 	if (ret)
-		drbd_warn(device, "helper command: %s %s %s exit code %u (0x%x)\n",
-				drbd_usermode_helper, cmd, mb,
-				(ret >> 8) & 0xff, ret);
+		magic_printk(KERN_WARNING,
+			     "helper command: %s %s exit code %u (0x%x)\n",
+			     drbd_usermode_helper, cmd,
+			     (ret >> 8) & 0xff, ret);
 	else
-		drbd_info(device, "helper command: %s %s %s exit code %u (0x%x)\n",
-				drbd_usermode_helper, cmd, mb,
-				(ret >> 8) & 0xff, ret);
-	sib.sib_reason = SIB_HELPER_POST;
-	sib.helper_exit_code = ret;
-	drbd_bcast_event(device, &sib);
+		magic_printk(KERN_INFO,
+			     "helper command: %s %s exit code 0\n",
+			     drbd_usermode_helper, cmd);
 	notify_helper(NOTIFY_RESPONSE, device, connection, cmd, ret);
 
-	if (current == connection->worker.task)
-		clear_bit(CALLBACK_PENDING, &connection->flags);
+	if (current == resource->worker.task)
+		clear_bit(CALLBACK_PENDING, &resource->flags);
 
 	if (ret < 0) /* Ignore any ERRNOs we got. */
 		ret = 0;
 
+	free_pages((unsigned long)env.buffer, get_order(env.size));
 	return ret;
-}
-
-enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd)
-{
-	char *envp[] = { "HOME=/",
-			"TERM=linux",
-			"PATH=/sbin:/usr/sbin:/bin:/usr/bin",
-			 (char[20]) { }, /* address family */
-			 (char[60]) { }, /* address */
-			NULL };
-	char *resource_name = connection->resource->name;
-	char *argv[] = {drbd_usermode_helper, cmd, resource_name, NULL };
-	int ret;
-
-	setup_khelper_env(connection, envp);
-	conn_md_sync(connection);
 
-	drbd_info(connection, "helper command: %s %s %s\n", drbd_usermode_helper, cmd, resource_name);
-	/* TODO: conn_bcast_event() ?? */
-	notify_helper(NOTIFY_CALL, NULL, connection, cmd, 0);
+    out_err:
+	drbd_err(resource, "Could not call %s user-space helper: error %d"
+		 "out of memory\n", cmd, ret);
+	return 0;
+}
 
-	ret = call_usermodehelper(drbd_usermode_helper, argv, envp, UMH_WAIT_PROC);
-	if (ret)
-		drbd_warn(connection, "helper command: %s %s %s exit code %u (0x%x)\n",
-			  drbd_usermode_helper, cmd, resource_name,
-			  (ret >> 8) & 0xff, ret);
-	else
-		drbd_info(connection, "helper command: %s %s %s exit code %u (0x%x)\n",
-			  drbd_usermode_helper, cmd, resource_name,
-			  (ret >> 8) & 0xff, ret);
-	/* TODO: conn_bcast_event() ?? */
-	notify_helper(NOTIFY_RESPONSE, NULL, connection, cmd, ret);
+#undef magic_printk
 
-	if (ret < 0) /* Ignore any ERRNOs we got. */
-		ret = 0;
+int drbd_maybe_khelper(struct drbd_device *device, struct drbd_connection *connection, char *cmd)
+{
+	if (strcmp(drbd_usermode_helper, "disabled") == 0)
+		return DRBD_UMH_DISABLED;
 
-	return ret;
+	return drbd_khelper(device, connection, cmd);
 }
 
-static enum drbd_fencing_p highest_fencing_policy(struct drbd_connection *connection)
+static bool initial_states_pending(struct drbd_connection *connection)
 {
-	enum drbd_fencing_p fp = FP_NOT_AVAIL;
 	struct drbd_peer_device *peer_device;
 	int vnr;
+	bool pending = false;
 
 	rcu_read_lock();
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		if (get_ldev_if_state(device, D_CONSISTENT)) {
-			struct disk_conf *disk_conf =
-				rcu_dereference(peer_device->device->ldev->disk_conf);
-			fp = max_t(enum drbd_fencing_p, fp, disk_conf->fencing);
-			put_ldev(device);
+		if (test_bit(INITIAL_STATE_SENT, &peer_device->flags) &&
+		    peer_device->repl_state[NOW] == L_OFF) {
+			pending = true;
+			break;
 		}
 	}
 	rcu_read_unlock();
-
-	return fp;
+	return pending;
 }
 
-static bool resource_is_supended(struct drbd_resource *resource)
+static bool intentional_diskless(struct drbd_resource *resource)
 {
-	return resource->susp || resource->susp_fen || resource->susp_nod;
+	bool intentional_diskless = true;
+	struct drbd_device *device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (!device->device_conf.intentional_diskless) {
+			intentional_diskless = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return intentional_diskless;
 }
 
-bool conn_try_outdate_peer(struct drbd_connection *connection)
+static bool conn_try_outdate_peer(struct drbd_connection *connection, const char *tag)
 {
-	struct drbd_resource * const resource = connection->resource;
-	unsigned int connect_cnt;
-	union drbd_state mask = { };
-	union drbd_state val = { };
-	enum drbd_fencing_p fp;
+	struct drbd_resource *resource = connection->resource;
+	unsigned long last_reconnect_jif;
+	enum drbd_fencing_policy fencing_policy;
+	enum drbd_disk_state disk_state;
 	char *ex_to_string;
 	int r;
+	unsigned long irq_flags;
 
-	spin_lock_irq(&resource->req_lock);
-	if (connection->cstate >= C_WF_REPORT_PARAMS) {
-		drbd_err(connection, "Expected cstate < C_WF_REPORT_PARAMS\n");
-		spin_unlock_irq(&resource->req_lock);
+	read_lock_irq(&resource->state_rwlock);
+	if (connection->cstate[NOW] >= C_CONNECTED) {
+		drbd_err(connection, "Expected cstate < C_CONNECTED\n");
+		read_unlock_irq(&resource->state_rwlock);
 		return false;
 	}
 
-	connect_cnt = connection->connect_cnt;
-	spin_unlock_irq(&resource->req_lock);
-
-	fp = highest_fencing_policy(connection);
-	switch (fp) {
-	case FP_NOT_AVAIL:
-		drbd_warn(connection, "Not fencing peer, I'm not even Consistent myself.\n");
-		spin_lock_irq(&resource->req_lock);
-		if (connection->cstate < C_WF_REPORT_PARAMS) {
-			_conn_request_state(connection,
-					    (union drbd_state) { { .susp_fen = 1 } },
-					    (union drbd_state) { { .susp_fen = 0 } },
-					    CS_VERBOSE | CS_HARD | CS_DC_SUSP);
-			/* We are no longer suspended due to the fencing policy.
-			 * We may still be suspended due to the on-no-data-accessible policy.
-			 * If that was OND_IO_ERROR, fail pending requests. */
-			if (!resource_is_supended(resource))
-				_tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
-		}
-		/* Else: in case we raced with a connection handshake,
-		 * let the handshake figure out if we maybe can RESEND,
-		 * and do not resume/fail pending requests here.
-		 * Worst case is we stay suspended for now, which may be
-		 * resolved by either re-establishing the replication link, or
-		 * the next link failure, or eventually the administrator.  */
-		spin_unlock_irq(&resource->req_lock);
+	last_reconnect_jif = connection->last_reconnect_jif;
+
+	disk_state = conn_highest_disk(connection);
+	if (disk_state < D_CONSISTENT &&
+	    !(disk_state == D_DISKLESS && intentional_diskless(resource))) {
+		begin_state_change_locked(resource, CS_VERBOSE | CS_HARD);
+		__change_io_susp_fencing(connection, false);
+		end_state_change_locked(resource, tag);
+		read_unlock_irq(&resource->state_rwlock);
 		return false;
+	}
+	read_unlock_irq(&resource->state_rwlock);
 
-	case FP_DONT_CARE:
+	fencing_policy = connection->fencing_policy;
+	if (fencing_policy == FP_DONT_CARE)
 		return true;
-	default: ;
-	}
 
-	r = conn_khelper(connection, "fence-peer");
+	r = drbd_maybe_khelper(NULL, connection, "fence-peer");
+	if (r == DRBD_UMH_DISABLED)
+		return true;
 
+	begin_state_change(resource, &irq_flags, CS_VERBOSE);
 	switch ((r>>8) & 0xff) {
 	case P_INCONSISTENT: /* peer is inconsistent */
 		ex_to_string = "peer is inconsistent or worse";
-		mask.pdsk = D_MASK;
-		val.pdsk = D_INCONSISTENT;
+		__downgrade_peer_disk_states(connection, D_INCONSISTENT);
 		break;
 	case P_OUTDATED: /* peer got outdated, or was already outdated */
 		ex_to_string = "peer was fenced";
-		mask.pdsk = D_MASK;
-		val.pdsk = D_OUTDATED;
+		__downgrade_peer_disk_states(connection, D_OUTDATED);
 		break;
 	case P_DOWN: /* peer was down */
 		if (conn_highest_disk(connection) == D_UP_TO_DATE) {
 			/* we will(have) create(d) a new UUID anyways... */
 			ex_to_string = "peer is unreachable, assumed to be dead";
-			mask.pdsk = D_MASK;
-			val.pdsk = D_OUTDATED;
+			__downgrade_peer_disk_states(connection, D_OUTDATED);
 		} else {
 			ex_to_string = "peer unreachable, doing nothing since disk != UpToDate";
 		}
@@ -544,42 +829,44 @@ bool conn_try_outdate_peer(struct drbd_connection *connection)
 		 * become R_PRIMARY, but finds the other peer being active. */
 		ex_to_string = "peer is active";
 		drbd_warn(connection, "Peer is primary, outdating myself.\n");
-		mask.disk = D_MASK;
-		val.disk = D_OUTDATED;
+		__downgrade_disk_states(resource, D_OUTDATED);
 		break;
 	case P_FENCING:
 		/* THINK: do we need to handle this
-		 * like case 4, or more like case 5? */
-		if (fp != FP_STONITH)
+		 * like case 4 P_OUTDATED, or more like case 5 P_DOWN? */
+		if (fencing_policy != FP_STONITH)
 			drbd_err(connection, "fence-peer() = 7 && fencing != Stonith !!!\n");
 		ex_to_string = "peer was stonithed";
-		mask.pdsk = D_MASK;
-		val.pdsk = D_OUTDATED;
+		__downgrade_peer_disk_states(connection, D_OUTDATED);
 		break;
 	default:
 		/* The script is broken ... */
 		drbd_err(connection, "fence-peer helper broken, returned %d\n", (r>>8)&0xff);
+		abort_state_change(resource, &irq_flags);
 		return false; /* Eventually leave IO frozen */
 	}
 
 	drbd_info(connection, "fence-peer helper returned %d (%s)\n",
 		  (r>>8) & 0xff, ex_to_string);
 
-	/* Not using
-	   conn_request_state(connection, mask, val, CS_VERBOSE);
-	   here, because we might were able to re-establish the connection in the
-	   meantime. */
-	spin_lock_irq(&resource->req_lock);
-	if (connection->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &connection->flags)) {
-		if (connection->connect_cnt != connect_cnt)
-			/* In case the connection was established and droped
-			   while the fence-peer handler was running, ignore it */
-			drbd_info(connection, "Ignoring fence-peer exit code\n");
-		else
-			_conn_request_state(connection, mask, val, CS_VERBOSE);
+	if (connection->cstate[NOW] >= C_CONNECTED ||
+	    initial_states_pending(connection)) {
+		/* connection re-established; do not fence */
+		goto abort;
 	}
-	spin_unlock_irq(&resource->req_lock);
+	if (connection->last_reconnect_jif != last_reconnect_jif) {
+		/* In case the connection was established and dropped
+		   while the fence-peer handler was running, ignore it */
+		drbd_info(connection, "Ignoring fence-peer exit code\n");
+		goto abort;
+	}
+
+	end_state_change(resource, &irq_flags, tag);
 
+	goto out;
+ abort:
+	abort_state_change(resource, &irq_flags);
+ out:
 	return conn_highest_pdsk(connection) <= D_OUTDATED;
 }
 
@@ -587,7 +874,7 @@ static int _try_outdate_peer_async(void *data)
 {
 	struct drbd_connection *connection = (struct drbd_connection *)data;
 
-	conn_try_outdate_peer(connection);
+	conn_try_outdate_peer(connection, "outdate-async");
 
 	kref_put(&connection->kref, drbd_destroy_connection);
 	return 0;
@@ -611,151 +898,451 @@ void conn_try_outdate_peer_async(struct drbd_connection *connection)
 	}
 }
 
-enum drbd_state_rv
-drbd_set_role(struct drbd_device *const device, enum drbd_role new_role, int force)
+bool barrier_pending(struct drbd_resource *resource)
 {
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
-	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
-	const int max_tries = 4;
-	enum drbd_state_rv rv = SS_UNKNOWN_ERROR;
-	struct net_conf *nc;
-	int try = 0;
-	int forced = 0;
-	union drbd_state mask, val;
+	struct drbd_connection *connection;
+	bool rv = false;
 
-	if (new_role == R_PRIMARY) {
-		struct drbd_connection *connection;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (test_bit(BARRIER_ACK_PENDING, &connection->flags)) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
 
-		/* Detect dead peers as soon as possible.  */
+	return rv;
+}
 
-		rcu_read_lock();
-		for_each_connection(connection, device->resource)
-			request_ping(connection);
-		rcu_read_unlock();
+static int count_up_to_date(struct drbd_resource *resource)
+{
+	struct drbd_device *device;
+	int vnr, nr_up_to_date = 0;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		enum drbd_disk_state disk_state = device->disk_state[NOW];
+		if (disk_state == D_UP_TO_DATE)
+			nr_up_to_date++;
 	}
+	rcu_read_unlock();
+	return nr_up_to_date;
+}
 
-	mutex_lock(device->state_mutex);
+static bool reconciliation_ongoing(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device_rcu(peer_device, device) {
+		if (test_bit(RECONCILIATION_RESYNC, &peer_device->flags))
+			return true;
+	}
+	return false;
+}
 
-	mask.i = 0; mask.role = R_MASK;
-	val.i  = 0; val.role  = new_role;
+static bool any_peer_is_consistent(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
 
-	while (try++ < max_tries) {
-		rv = _drbd_request_state_holding_state_mutex(device, mask, val, CS_WAIT_COMPLETE);
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->disk_state[NOW] == D_CONSISTENT)
+			return true;
+	}
+	return false;
+}
+/* reconciliation resyncs finished and I know if I am D_UP_TO_DATE or D_OUTDATED */
+static bool after_primary_lost_events_settled(struct drbd_resource *resource)
+{
+	struct drbd_device *device;
+	int vnr;
 
-		/* in case we first succeeded to outdate,
-		 * but now suddenly could establish a connection */
-		if (rv == SS_CW_FAILED_BY_PEER && mask.pdsk != 0) {
-			val.pdsk = 0;
-			mask.pdsk = 0;
-			continue;
-		}
+	if (test_bit(TRY_BECOME_UP_TO_DATE_PENDING, &resource->flags))
+		return false;
 
-		if (rv == SS_NO_UP_TO_DATE_DISK && force &&
-		    (device->state.disk < D_UP_TO_DATE &&
-		     device->state.disk >= D_INCONSISTENT)) {
-			mask.disk = D_MASK;
-			val.disk  = D_UP_TO_DATE;
-			forced = 1;
-			continue;
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		enum drbd_disk_state disk_state = device->disk_state[NOW];
+		if (disk_state == D_CONSISTENT ||
+		    any_peer_is_consistent(device) ||
+		    (reconciliation_ongoing(device) &&
+		     (disk_state == D_OUTDATED || disk_state == D_INCONSISTENT))) {
+			rcu_read_unlock();
+			return false;
 		}
+	}
+	rcu_read_unlock();
+	return true;
+}
 
-		if (rv == SS_NO_UP_TO_DATE_DISK &&
-		    device->state.disk == D_CONSISTENT && mask.pdsk == 0) {
-			D_ASSERT(device, device->state.pdsk == D_UNKNOWN);
+static long drbd_max_ping_timeout(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	long ping_timeout = 0;
 
-			if (conn_try_outdate_peer(connection)) {
-				val.disk = D_UP_TO_DATE;
-				mask.disk = D_MASK;
-			}
-			continue;
-		}
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource)
+		ping_timeout = max(ping_timeout, (long) connection->transport.net_conf->ping_timeo);
+	rcu_read_unlock();
 
-		if (rv == SS_NOTHING_TO_DO)
-			goto out;
-		if (rv == SS_PRIMARY_NOP && mask.pdsk == 0) {
-			if (!conn_try_outdate_peer(connection) && force) {
-				drbd_warn(device, "Forced into split brain situation!\n");
-				mask.pdsk = D_MASK;
-				val.pdsk  = D_OUTDATED;
+	return ping_timeout;
+}
 
-			}
+static bool wait_up_to_date(struct drbd_resource *resource)
+{
+	/*
+	 * Adding ping-timeout is necessary to ensure that we do not proceed
+	 * while the loss of some connection has not yet been detected. Ideally
+	 * we would use the maximum ping timeout from the entire cluster. Since
+	 * we do not have that, use the maximum from our connections on a
+	 * best-effort basis.
+	 */
+	long timeout = (resource->res_opts.auto_promote_timeout +
+			drbd_max_ping_timeout(resource)) * HZ / 10;
+	int initial_up_to_date, up_to_date;
+
+	initial_up_to_date = count_up_to_date(resource);
+	wait_event_interruptible_timeout(resource->state_wait,
+					 after_primary_lost_events_settled(resource),
+					 timeout);
+	up_to_date = count_up_to_date(resource);
+	return up_to_date > initial_up_to_date;
+}
+
+enum drbd_state_rv
+drbd_set_role(struct drbd_resource *resource, enum drbd_role role, bool force, const char *tag,
+		struct sk_buff *reply_skb)
+{
+	struct drbd_device *device;
+	int vnr, try = 0;
+	const int max_tries = 4;
+	enum drbd_state_rv rv = SS_UNKNOWN_ERROR;
+	bool retried_ss_two_primaries = false, retried_ss_primary_nop = false;
+	const char *err_str = NULL;
+	enum chg_state_flags flags = CS_ALREADY_SERIALIZED | CS_DONT_RETRY | CS_WAIT_COMPLETE;
+	bool fenced_peers = false;
+
+retry:
+
+	if (role == R_PRIMARY) {
+		drbd_check_peers(resource);
+		wait_up_to_date(resource);
+	}
+	down(&resource->state_sem);
+
+	while (try++ < max_tries) {
+		if (try == max_tries - 1)
+			flags |= CS_VERBOSE;
+
+		if (err_str) {
+			kfree(err_str);
+			err_str = NULL;
+		}
+		rv = stable_state_change(resource,
+			change_role(resource, role, flags, tag, &err_str));
+
+		if (rv == SS_TIMEOUT || rv == SS_CONCURRENT_ST_CHG) {
+			long timeout = twopc_retry_timeout(resource, try);
+			/* It might be that the receiver tries to start resync, and
+			   sleeps on state_sem. Give it up, and retry in a short
+			   while */
+			up(&resource->state_sem);
+			schedule_timeout_interruptible(timeout);
+			goto retry;
+		}
+		/* in case we first succeeded to outdate,
+		 * but now suddenly could establish a connection */
+		if (rv == SS_CW_FAILED_BY_PEER && fenced_peers) {
+			flags &= ~CS_FP_LOCAL_UP_TO_DATE;
+			continue;
+		}
+
+		if (rv == SS_NO_UP_TO_DATE_DISK && force && !(flags & CS_FP_LOCAL_UP_TO_DATE)) {
+			flags |= CS_FP_LOCAL_UP_TO_DATE;
+			continue;
+		}
+
+		if (rv == SS_DEVICE_IN_USE && force && !(flags & CS_FS_IGN_OPENERS)) {
+			drbd_warn(resource, "forced demotion\n");
+			flags |= CS_FS_IGN_OPENERS; /* this sets resource->fail_io[NOW] */
+			continue;
+		}
+
+		if (rv == SS_NO_UP_TO_DATE_DISK) {
+			bool a_disk_became_up_to_date;
+
+			/* need to give up state_sem, see try_become_up_to_date(); */
+			up(&resource->state_sem);
+			drbd_flush_workqueue(&resource->work);
+			a_disk_became_up_to_date = wait_up_to_date(resource);
+			down(&resource->state_sem);
+			if (a_disk_became_up_to_date)
+				continue;
+			/* fall through into possible fence-peer or even force cases */
+		}
+
+		if (rv == SS_NO_UP_TO_DATE_DISK && !(flags & CS_FP_LOCAL_UP_TO_DATE)) {
+			struct drbd_connection *connection;
+			bool any_fencing_failed = false;
+			u64 im;
+
+			fenced_peers = false;
+			up(&resource->state_sem); /* Allow connect while fencing */
+			for_each_connection_ref(connection, im, resource) {
+				struct drbd_peer_device *peer_device;
+				int vnr;
+
+				if (conn_highest_pdsk(connection) != D_UNKNOWN)
+					continue;
+
+				idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+					struct drbd_device *device = peer_device->device;
+
+					if (device->disk_state[NOW] != D_CONSISTENT)
+						continue;
+
+					if (conn_try_outdate_peer(connection, tag))
+						fenced_peers = true;
+					else
+						any_fencing_failed = true;
+				}
+			}
+			down(&resource->state_sem);
+			if (fenced_peers && !any_fencing_failed) {
+				flags |= CS_FP_LOCAL_UP_TO_DATE;
+				continue;
+			}
+		}
+
+		/* In case the disk is Consistent and fencing is enabled, and fencing did not work
+		 * but the user forces promote..., try it pretending we fenced the peers */
+		if (rv == SS_PRIMARY_NOP && force &&
+		    (flags & CS_FP_LOCAL_UP_TO_DATE) && !(flags & CS_FP_OUTDATE_PEERS)) {
+			flags |= CS_FP_OUTDATE_PEERS;
 			continue;
 		}
-		if (rv == SS_TWO_PRIMARIES) {
-			/* Maybe the peer is detected as dead very soon...
-			   retry at most once more in this case. */
-			if (try < max_tries) {
-				int timeo;
-				try = max_tries - 1;
-				rcu_read_lock();
-				nc = rcu_dereference(connection->net_conf);
-				timeo = nc ? (nc->ping_timeo + 1) * HZ / 10 : 1;
-				rcu_read_unlock();
-				schedule_timeout_interruptible(timeo);
+
+		if (rv == SS_NO_QUORUM && force && !(flags & CS_FP_OUTDATE_PEERS)) {
+			flags |= CS_FP_OUTDATE_PEERS;
+			continue;
+		}
+
+		if (rv == SS_NOTHING_TO_DO)
+			goto out;
+		if (rv == SS_PRIMARY_NOP && !retried_ss_primary_nop) {
+			struct drbd_connection *connection;
+			u64 im;
+
+			retried_ss_primary_nop = true;
+
+			up(&resource->state_sem); /* Allow connect while fencing */
+			for_each_connection_ref(connection, im, resource) {
+				bool outdated_peer = conn_try_outdate_peer(connection, tag);
+				if (!outdated_peer && force) {
+					drbd_warn(connection, "Forced into split brain situation!\n");
+					flags |= CS_FP_LOCAL_UP_TO_DATE;
+				}
 			}
+			down(&resource->state_sem);
 			continue;
 		}
-		if (rv < SS_SUCCESS) {
-			rv = _drbd_request_state(device, mask, val,
-						CS_VERBOSE + CS_WAIT_COMPLETE);
-			if (rv < SS_SUCCESS)
-				goto out;
+
+		if (rv == SS_TWO_PRIMARIES && !retried_ss_two_primaries) {
+			struct drbd_connection *connection;
+			struct net_conf *nc;
+			int timeout = 0;
+
+			retried_ss_two_primaries = true;
+
+			/*
+			 * Catch the case where we discover that the other
+			 * primary has died soon after the state change
+			 * failure: retry once after a short timeout.
+			 */
+
+			rcu_read_lock();
+			for_each_connection_rcu(connection, resource) {
+				nc = rcu_dereference(connection->transport.net_conf);
+				if (nc && nc->ping_timeo > timeout)
+					timeout = nc->ping_timeo;
+			}
+			rcu_read_unlock();
+			timeout = timeout * HZ / 10;
+			if (timeout == 0)
+				timeout = 1;
+
+			up(&resource->state_sem);
+			schedule_timeout_interruptible(timeout);
+			goto retry;
 		}
+
 		break;
 	}
 
 	if (rv < SS_SUCCESS)
 		goto out;
 
-	if (forced)
-		drbd_warn(device, "Forced to consider local data as UpToDate!\n");
-
-	/* Wait until nothing is on the fly :) */
-	wait_event(device->misc_wait, atomic_read(&device->ap_pending_cnt) == 0);
-
-	/* FIXME also wait for all pending P_BARRIER_ACK? */
+	if (force) {
+		if (flags & CS_FP_LOCAL_UP_TO_DATE)
+			drbd_warn(resource, "Forced to consider local data as UpToDate!\n");
+		if (flags & CS_FP_OUTDATE_PEERS)
+			drbd_warn(resource, "Forced to consider peers as Outdated!\n");
+	}
 
-	if (new_role == R_SECONDARY) {
-		if (get_ldev(device)) {
-			device->ldev->md.uuid[UI_CURRENT] &= ~(u64)1;
-			put_ldev(device);
+	if (role == R_SECONDARY) {
+		idr_for_each_entry(&resource->devices, device, vnr) {
+			if (get_ldev(device)) {
+				device->ldev->md.current_uuid &= ~UUID_PRIMARY;
+				put_ldev(device);
+			}
 		}
 	} else {
-		mutex_lock(&device->resource->conf_update);
-		nc = connection->net_conf;
-		if (nc)
-			nc->discard_my_data = 0; /* without copy; single bit op is atomic */
-		mutex_unlock(&device->resource->conf_update);
+		struct drbd_connection *connection;
 
-		if (get_ldev(device)) {
-			if (((device->state.conn < C_CONNECTED ||
-			       device->state.pdsk <= D_FAILED)
-			      && device->ldev->md.uuid[UI_BITMAP] == 0) || forced)
-				drbd_uuid_new_current(device);
+		rcu_read_lock();
+		for_each_connection_rcu(connection, resource)
+			clear_bit(CONN_DISCARD_MY_DATA, &connection->flags);
+		rcu_read_unlock();
 
-			device->ldev->md.uuid[UI_CURRENT] |=  (u64)1;
-			put_ldev(device);
+		idr_for_each_entry(&resource->devices, device, vnr) {
+			if (flags & CS_FP_LOCAL_UP_TO_DATE) {
+				drbd_uuid_new_current(device, true);
+				clear_bit(NEW_CUR_UUID, &device->flags);
+			}
 		}
 	}
 
-	/* writeout of activity log covered areas of the bitmap
-	 * to stable storage done in after state change already */
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		 struct drbd_peer_device *peer_device;
+		 u64 im;
 
-	if (device->state.conn >= C_WF_REPORT_PARAMS) {
-		/* if this was forced, we should consider sync */
-		if (forced)
-			drbd_send_uuids(peer_device);
-		drbd_send_current_state(peer_device);
+		 for_each_peer_device_ref(peer_device, im, device) {
+			/* writeout of activity log covered areas of the bitmap
+			 * to stable storage done in after state change already */
+
+			if (peer_device->connection->cstate[NOW] == C_CONNECTED) {
+				/* if this was forced, we should consider sync */
+				if (flags & CS_FP_LOCAL_UP_TO_DATE) {
+					drbd_send_uuids(peer_device, 0, 0);
+					set_bit(CONSIDER_RESYNC, &peer_device->flags);
+				}
+				drbd_send_current_state(peer_device);
+			}
+		}
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		drbd_md_sync_if_dirty(device);
+		if (!resource->res_opts.auto_promote && role == R_PRIMARY)
+			kobject_uevent(&disk_to_dev(device->vdisk)->kobj, KOBJ_CHANGE);
 	}
 
-	drbd_md_sync(device);
-	set_disk_ro(device->vdisk, new_role == R_SECONDARY);
-	kobject_uevent(&disk_to_dev(device->vdisk)->kobj, KOBJ_CHANGE);
 out:
-	mutex_unlock(device->state_mutex);
+	up(&resource->state_sem);
+	if (err_str) {
+		drbd_err(resource, "%s", err_str);
+		if (reply_skb)
+			drbd_msg_put_info(reply_skb, err_str);
+		kfree(err_str);
+	}
 	return rv;
 }
 
+/* suggested buffer size: 128 byte */
+void youngest_and_oldest_opener_to_str(struct drbd_device *device, char *buf, size_t len)
+{
+	struct timespec64 ts;
+	struct tm tm;
+	struct opener *first;
+	struct opener *last;
+	int cnt;
+
+	buf[0] = '\0';
+	/* Do we have opener information? */
+	if (!device->open_cnt)
+		return;
+	cnt = snprintf(buf, len, " open_cnt:%d", device->open_cnt);
+	if (cnt > 0 && cnt < len) {
+		buf += cnt;
+		len -= cnt;
+	} else
+		return;
+	spin_lock(&device->openers_lock);
+	if (!list_empty(&device->openers)) {
+		first = list_first_entry(&device->openers, struct opener, list);
+		ts = ktime_to_timespec64(first->opened);
+		time64_to_tm(ts.tv_sec, -sys_tz.tz_minuteswest * 60, &tm);
+		cnt = snprintf(buf, len, " [%s:%d:%04ld-%02d-%02d_%02d:%02d:%02d.%03ld]",
+			      first->comm, first->pid,
+			      tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+			      tm.tm_hour, tm.tm_min, tm.tm_sec, ts.tv_nsec / NSEC_PER_MSEC);
+		last = list_last_entry(&device->openers, struct opener, list);
+		if (cnt > 0 && cnt < len && last != first) {
+			/* append, overwriting the previously added ']' */
+			buf += cnt-1;
+			len -= cnt-1;
+			ts = ktime_to_timespec64(last->opened);
+			time64_to_tm(ts.tv_sec, -sys_tz.tz_minuteswest * 60, &tm);
+			snprintf(buf, len, "%s%s:%d:%04ld-%02d-%02d_%02d:%02d:%02d.%03ld]",
+			      device->open_cnt > 2 ? ", ..., " : ", ",
+			      last->comm, last->pid,
+			      tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+			      tm.tm_hour, tm.tm_min, tm.tm_sec, ts.tv_nsec / NSEC_PER_MSEC);
+		}
+	}
+	spin_unlock(&device->openers_lock);
+}
+
+static int put_device_opener_info(struct drbd_device *device, struct sk_buff *reply_skb)
+{
+	struct timespec64 ts;
+	struct opener *o;
+	struct tm tm;
+	int cnt = 0;
+	char *dotdotdot = "";
+
+	spin_lock(&device->openers_lock);
+	if (!device->open_cnt) {
+		spin_unlock(&device->openers_lock);
+		return cnt;
+	}
+	drbd_msg_sprintf_info(reply_skb,
+		"/dev/drbd%d open_cnt:%d, writable:%d; list of openers follows",
+		device->minor, device->open_cnt, device->writable);
+	list_for_each_entry(o, &device->openers, list) {
+		ts = ktime_to_timespec64(o->opened);
+		time64_to_tm(ts.tv_sec, -sys_tz.tz_minuteswest * 60, &tm);
+
+		if (++cnt >= 10 && !list_is_last(&o->list, &device->openers)) {
+			o = list_last_entry(&device->openers, struct opener, list);
+			dotdotdot = "[...]\n";
+		}
+		drbd_msg_sprintf_info(reply_skb,
+			"%sdrbd%d opened by %s (pid %d) at %04ld-%02d-%02d %02d:%02d:%02d.%03ld",
+			dotdotdot,
+			device->minor, o->comm, o->pid,
+			tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+			tm.tm_hour, tm.tm_min, tm.tm_sec,
+			ts.tv_nsec / NSEC_PER_MSEC);
+	}
+	spin_unlock(&device->openers_lock);
+	return cnt;
+}
+
+static void opener_info(struct drbd_resource *resource,
+			struct sk_buff *reply_skb,
+			enum drbd_state_rv rv)
+{
+	struct drbd_device *device;
+	int i;
+
+	if (rv != SS_DEVICE_IN_USE && rv != SS_NO_UP_TO_DATE_DISK)
+		return;
+
+	idr_for_each_entry(&resource->devices, device, i)
+		put_device_opener_info(device, reply_skb);
+}
+
 static const char *from_attrs_err_to_txt(int err)
 {
 	return	err == -ENOMSG ? "required attribute missing" :
@@ -764,20 +1351,21 @@ static const char *from_attrs_err_to_txt(int err)
 		"invalid attribute value";
 }
 
-int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
+	struct drbd_resource *resource;
 	struct set_role_parms parms;
-	int err;
-	enum drbd_ret_code retcode;
 	enum drbd_state_rv rv;
+	enum drbd_ret_code retcode;
+	enum drbd_role new_role;
+	int err;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	rv = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
 	if (!adm_ctx.reply_skb)
-		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
+		return rv;
 
+	resource = adm_ctx.resource;
 	memset(&parms, 0, sizeof(parms));
 	if (info->attrs[DRBD_NLA_SET_ROLE_PARMS]) {
 		err = set_role_parms_from_attrs(&parms, info);
@@ -787,16 +1375,28 @@ int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 	}
-	genl_unlock();
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
 
-	if (info->genlhdr->cmd == DRBD_ADM_PRIMARY)
-		rv = drbd_set_role(adm_ctx.device, R_PRIMARY, parms.assume_uptodate);
-	else
-		rv = drbd_set_role(adm_ctx.device, R_SECONDARY, 0);
+	new_role = info->genlhdr->cmd == DRBD_ADM_PRIMARY ? R_PRIMARY : R_SECONDARY;
+	if (new_role == R_PRIMARY)
+		set_bit(EXPLICIT_PRIMARY, &resource->flags);
 
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-	genl_lock();
+	rv = drbd_set_role(resource,
+				new_role,
+				parms.force,
+				new_role == R_PRIMARY ? "primary" : "secondary",
+				adm_ctx.reply_skb);
+
+	if (resource->role[NOW] != R_PRIMARY)
+		clear_bit(EXPLICIT_PRIMARY, &resource->flags);
+
+	if (rv == SS_DEVICE_IN_USE)
+		opener_info(resource, adm_ctx.reply_skb, rv);
+
+	mutex_unlock(&resource->adm_mutex);
 	drbd_adm_finish(&adm_ctx, info, rv);
 	return 0;
 out:
@@ -804,6 +1404,28 @@ int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
+u64 drbd_capacity_to_on_disk_bm_sect(u64 capacity_sect, const struct drbd_md *md)
+{
+	u64 bits, bytes;
+
+	/* round up storage sectors to full "bitmap sectors per bit", then
+	 * convert to number of bits needed, and round that up to 64bit words
+	 * to ease interoperability between 32bit and 64bit architectures.
+	 */
+	bits = ALIGN(sect_to_bit(
+			ALIGN(capacity_sect, sect_per_bit(md->bm_block_shift)),
+			md->bm_block_shift), 64);
+
+	/* convert to bytes, multiply by number of peers,
+	 * and, because we do all our meta data IO in 4k blocks,
+	 * round up to full 4k
+	 */
+	bytes = ALIGN(bits / 8 * md->max_peers, 4096);
+
+	/* convert to number of sectors */
+	return bytes >> 9;
+}
+
 /* Initializes the md.*_offset members, so we are able to find
  * the on disk meta data.
  *
@@ -823,10 +1445,9 @@ int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
  *  ==> bitmap sectors = Y = al_offset - bm_offset
  *
  *  Activity log size used to be fixed 32kB,
- *  but is about to become configurable.
+ *  but is actually al_stripes * al_stripe_size_4k.
  */
-static void drbd_md_set_sector_offsets(struct drbd_device *device,
-				       struct drbd_backing_dev *bdev)
+void drbd_md_set_sector_offsets(struct drbd_backing_dev *bdev)
 {
 	sector_t md_size_sect = 0;
 	unsigned int al_size_sect = bdev->md.al_size_4k * 8;
@@ -836,33 +1457,32 @@ static void drbd_md_set_sector_offsets(struct drbd_device *device,
 	switch (bdev->md.meta_dev_idx) {
 	default:
 		/* v07 style fixed size indexed meta data */
-		bdev->md.md_size_sect = MD_128MB_SECT;
-		bdev->md.al_offset = MD_4kB_SECT;
-		bdev->md.bm_offset = MD_4kB_SECT + al_size_sect;
+		/* FIXME we should drop support for this! */
+		bdev->md.md_size_sect = (128 << 20 >> 9);
+		bdev->md.al_offset = (4096 >> 9);
+		bdev->md.bm_offset = (4096 >> 9) + al_size_sect;
 		break;
 	case DRBD_MD_INDEX_FLEX_EXT:
 		/* just occupy the full device; unit: sectors */
 		bdev->md.md_size_sect = drbd_get_capacity(bdev->md_bdev);
-		bdev->md.al_offset = MD_4kB_SECT;
-		bdev->md.bm_offset = MD_4kB_SECT + al_size_sect;
+		bdev->md.al_offset = (4096 >> 9);
+		bdev->md.bm_offset = (4096 >> 9) + al_size_sect;
 		break;
 	case DRBD_MD_INDEX_INTERNAL:
 	case DRBD_MD_INDEX_FLEX_INT:
-		/* al size is still fixed */
 		bdev->md.al_offset = -al_size_sect;
-		/* we need (slightly less than) ~ this much bitmap sectors: */
-		md_size_sect = drbd_get_capacity(bdev->backing_bdev);
-		md_size_sect = ALIGN(md_size_sect, BM_SECT_PER_EXT);
-		md_size_sect = BM_SECT_TO_EXT(md_size_sect);
-		md_size_sect = ALIGN(md_size_sect, 8);
 
-		/* plus the "drbd meta data super block",
+		/* enough bitmap to cover the storage,
+		 * plus the "drbd meta data super block",
 		 * and the activity log; */
-		md_size_sect += MD_4kB_SECT + al_size_sect;
+		md_size_sect = drbd_capacity_to_on_disk_bm_sect(
+				drbd_get_capacity(bdev->backing_bdev),
+				&bdev->md)
+			+ (4096 >> 9) + al_size_sect;
 
 		bdev->md.md_size_sect = md_size_sect;
 		/* bitmap offset is adjusted by 'super' block size */
-		bdev->md.bm_offset   = -md_size_sect + MD_4kB_SECT;
+		bdev->md.bm_offset   = -md_size_sect + (4096 >> 9);
 		break;
 	}
 }
@@ -884,18 +1504,11 @@ char *ppsize(char *buf, unsigned long long size)
 	return buf;
 }
 
-/* there is still a theoretical deadlock when called from receiver
- * on an D_INCONSISTENT R_PRIMARY:
- *  remote READ does inc_ap_bio, receiver would need to receive answer
- *  packet from remote to dec_ap_bio again.
- *  receiver receive_sizes(), comes here,
- *  waits for ap_bio_cnt == 0. -> deadlock.
- * but this cannot happen, actually, because:
- *  R_PRIMARY D_INCONSISTENT, and peer's disk is unreachable
- *  (not connected, or bad/no disk on peer):
- *  see drbd_fail_request_early, ap_bio_cnt is zero.
- *  R_PRIMARY D_INCONSISTENT, and C_SYNC_TARGET:
- *  peer may not initiate a resize.
+/* The receiver may call drbd_suspend_io(device, WRITE_ONLY).
+ * It should not call drbd_suspend_io(device, READ_AND_WRITE) since
+ * if the node is an D_INCONSISTENT R_PRIMARY (L_SYNC_TARGET) it
+ * may need to issue remote READs. Those is turn need the receiver
+ * to complete. -> calling drbd_suspend_io(device, READ_AND_WRITE) deadlocks.
  */
 /* Note these are not to be confused with
  * drbd_adm_suspend_io/drbd_adm_resume_io,
@@ -905,12 +1518,12 @@ char *ppsize(char *buf, unsigned long long size)
  * and should be short-lived. */
 /* It needs to be a counter, since multiple threads might
    independently suspend and resume IO. */
-void drbd_suspend_io(struct drbd_device *device)
+void drbd_suspend_io(struct drbd_device *device, enum suspend_scope ss)
 {
 	atomic_inc(&device->suspend_cnt);
-	if (drbd_suspended(device))
-		return;
-	wait_event(device->misc_wait, !atomic_read(&device->ap_bio_cnt));
+	wait_event(device->misc_wait, drbd_suspended(device) ||
+		   (atomic_read(&device->ap_bio_cnt[WRITE]) +
+		    ss == READ_AND_WRITE ? atomic_read(&device->ap_bio_cnt[READ]) : 0) == 0);
 }
 
 void drbd_resume_io(struct drbd_device *device)
@@ -919,18 +1532,64 @@ void drbd_resume_io(struct drbd_device *device)
 		wake_up(&device->misc_wait);
 }
 
+/**
+ * effective_disk_size_determined()  -  is the effective disk size "fixed" already?
+ * @device: DRBD device.
+ *
+ * When a device is configured in a cluster, the size of the replicated disk is
+ * determined by the minimum size of the disks on all nodes.  Additional nodes
+ * can be added, and this can still change the effective size of the replicated
+ * disk.
+ *
+ * When the disk on any node becomes D_UP_TO_DATE, the effective disk size
+ * becomes "fixed".  It is written to the metadata so that it will not be
+ * forgotten across node restarts.  Further nodes can only be added if their
+ * disks are big enough.
+ */
+static bool effective_disk_size_determined(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	if (device->ldev->md.effective_size != 0)
+		return true;
+	if (device->disk_state[NOW] == D_UP_TO_DATE)
+		return true;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->disk_state[NOW] == D_UP_TO_DATE) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+void drbd_set_my_capacity(struct drbd_device *device, sector_t size)
+{
+	char ppb[10];
+
+	set_capacity_and_notify(device->vdisk, size);
+
+	drbd_info(device, "size = %s (%llu KB)\n",
+		ppsize(ppb, size>>1), (unsigned long long)size>>1);
+}
+
 /*
  * drbd_determine_dev_size() -  Sets the right device size obeying all constraints
  * @device:	DRBD device.
  *
- * Returns 0 on success, negative return values indicate errors.
  * You should call drbd_md_sync() after calling this function.
  */
 enum determine_dev_size
-drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct resize_parms *rs) __must_hold(local)
+drbd_determine_dev_size(struct drbd_device *device, sector_t peer_current_size,
+			enum dds_flags flags, struct resize_parms *rs) __must_hold(local)
 {
 	struct md_offsets_and_sizes {
-		u64 last_agreed_sect;
+		u64 effective_size;
 		u64 md_offset;
 		s32 al_offset;
 		s32 bm_offset;
@@ -939,7 +1598,7 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		u32 al_stripes;
 		u32 al_stripe_size_4k;
 	} prev;
-	sector_t u_size, size;
+	sector_t u_size, size, prev_size;
 	struct drbd_md *md = &device->ldev->md;
 	void *buffer;
 
@@ -954,7 +1613,7 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 	 * Move is not exactly correct, btw, currently we have all our meta
 	 * data in core memory, to "move" it we just write it all out, there
 	 * are no reads. */
-	drbd_suspend_io(device);
+	drbd_suspend_io(device, READ_AND_WRITE);
 	buffer = drbd_md_get_buffer(device, __func__); /* Lock meta-data IO */
 	if (!buffer) {
 		drbd_resume_io(device);
@@ -962,29 +1621,31 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 	}
 
 	/* remember current offset and sizes */
-	prev.last_agreed_sect = md->la_size_sect;
+	prev.effective_size = md->effective_size;
 	prev.md_offset = md->md_offset;
 	prev.al_offset = md->al_offset;
 	prev.bm_offset = md->bm_offset;
 	prev.md_size_sect = md->md_size_sect;
 	prev.al_stripes = md->al_stripes;
 	prev.al_stripe_size_4k = md->al_stripe_size_4k;
+	prev_size = get_capacity(device->vdisk);
 
 	if (rs) {
+		/* FIXME race with peer requests that want to do an AL transaction */
 		/* rs is non NULL if we should change the AL layout only */
 		md->al_stripes = rs->al_stripes;
 		md->al_stripe_size_4k = rs->al_stripe_size / 4;
 		md->al_size_4k = (u64)rs->al_stripes * rs->al_stripe_size / 4;
 	}
 
-	drbd_md_set_sector_offsets(device, device->ldev);
+	drbd_md_set_sector_offsets(device->ldev);
 
 	rcu_read_lock();
 	u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
 	rcu_read_unlock();
-	size = drbd_new_dev_size(device, device->ldev, u_size, flags & DDSF_FORCED);
+	size = drbd_new_dev_size(device, peer_current_size, u_size, flags);
 
-	if (size < prev.last_agreed_sect) {
+	if (size < prev.effective_size) {
 		if (rs && u_size == 0) {
 			/* Remove "rs &&" later. This check should always be active, but
 			   right now the receiver expects the permissive behavior */
@@ -1000,9 +1661,11 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 	}
 
 	if (get_capacity(device->vdisk) != size ||
-	    drbd_bm_capacity(device) != size) {
-		int err;
-		err = drbd_bm_resize(device, size, !(flags & DDSF_NO_RESYNC));
+	    (device->bitmap && drbd_bm_capacity(device) != size)) {
+		int err = 0;
+
+		if (device->bitmap)
+			err = drbd_bm_resize(device, size, !(flags & DDSF_NO_RESYNC));
 		if (unlikely(err)) {
 			/* currently there is only one error: ENOMEM! */
 			size = drbd_bm_capacity(device);
@@ -1014,21 +1677,32 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 				    "Leaving size unchanged\n");
 			}
 			rv = DS_ERROR;
+		} else {
+			/* racy, see comments above. */
+			drbd_set_my_capacity(device, size);
+			if (effective_disk_size_determined(device)
+			&& md->effective_size != size) {
+				char ppb[10];
+
+				drbd_info(device, "persisting effective size = %s (%llu KB)\n",
+					ppsize(ppb, size >> 1),
+					(unsigned long long)size >> 1);
+				md->effective_size = size;
+			}
 		}
-		/* racy, see comments above. */
-		drbd_set_my_capacity(device, size);
-		md->la_size_sect = size;
 	}
 	if (rv <= DS_ERROR)
 		goto err_out;
 
-	la_size_changed = (prev.last_agreed_sect != md->la_size_sect);
+	la_size_changed = (prev.effective_size != md->effective_size);
 
 	md_moved = prev.md_offset    != md->md_offset
 		|| prev.md_size_sect != md->md_size_sect;
 
 	if (la_size_changed || md_moved || rs) {
-		u32 prev_flags;
+		int i;
+		bool prev_al_disabled = 0;
+		u32 prev_peer_full_sync = 0;
 
 		/* We do some synchronous IO below, which may take some time.
 		 * Clear the timer, to avoid scary "timer expired!" messages,
@@ -1039,11 +1713,25 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		 * to move the on-disk location of the activity log ringbuffer.
 		 * Lock for transaction is good enough, it may well be "dirty"
 		 * or even "starving". */
-		wait_event(device->al_wait, lc_try_lock_for_transaction(device->act_log));
+		wait_event(device->al_wait, drbd_al_try_lock_for_transaction(device));
+
+		if (drbd_md_dax_active(device->ldev)) {
+			if (drbd_dax_map(device->ldev)) {
+				drbd_err(device, "Could not remap DAX; aborting resize\n");
+				lc_unlock(device->act_log);
+				goto err_out;
+			}
+		}
 
 		/* mark current on-disk bitmap and activity log as unreliable */
-		prev_flags = md->flags;
-		md->flags |= MDF_FULL_SYNC | MDF_AL_DISABLED;
+		prev_al_disabled = !!(md->flags & MDF_AL_DISABLED);
+		md->flags |= MDF_AL_DISABLED;
+		for (i = 0; i < DRBD_PEERS_MAX; i++) {
+			if (md->peers[i].flags & MDF_PEER_FULL_SYNC)
+				prev_peer_full_sync |= 1 << i;
+			else
+				md->peers[i].flags |= MDF_PEER_FULL_SYNC;
+		}
 		drbd_md_write(device, buffer);
 
 		drbd_al_initialize(device, buffer);
@@ -1053,27 +1741,35 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 			 la_size_changed ? "size changed" : "md moved");
 		/* next line implicitly does drbd_suspend_io()+drbd_resume_io() */
 		drbd_bitmap_io(device, md_moved ? &drbd_bm_write_all : &drbd_bm_write,
-			       "size changed", BM_LOCKED_MASK, NULL);
+			       "size changed", BM_LOCK_ALL, NULL);
 
 		/* on-disk bitmap and activity log is authoritative again
 		 * (unless there was an IO error meanwhile...) */
-		md->flags = prev_flags;
+		if (!prev_al_disabled)
+			md->flags &= ~MDF_AL_DISABLED;
+		for (i = 0; i < DRBD_PEERS_MAX; i++) {
+			if (0 == (prev_peer_full_sync & (1 << i)))
+				md->peers[i].flags &= ~MDF_PEER_FULL_SYNC;
+		}
 		drbd_md_write(device, buffer);
 
 		if (rs)
 			drbd_info(device, "Changed AL layout to al-stripes = %d, al-stripe-size-kB = %d\n",
-				  md->al_stripes, md->al_stripe_size_4k * 4);
+				 md->al_stripes, md->al_stripe_size_4k * 4);
+
+		lc_unlock(device->act_log);
+		wake_up(&device->al_wait);
 	}
 
-	if (size > prev.last_agreed_sect)
-		rv = prev.last_agreed_sect ? DS_GREW : DS_GREW_FROM_ZERO;
-	if (size < prev.last_agreed_sect)
+	if (size > prev_size)
+		rv = prev_size ? DS_GREW : DS_GREW_FROM_ZERO;
+	if (size < prev_size)
 		rv = DS_SHRUNK;
 
 	if (0) {
 	err_out:
 		/* restore previous offset and sizes */
-		md->la_size_sect = prev.last_agreed_sect;
+		md->effective_size = prev.effective_size;
 		md->md_offset = prev.md_offset;
 		md->al_offset = prev.al_offset;
 		md->bm_offset = prev.bm_offset;
@@ -1082,57 +1778,167 @@ drbd_determine_dev_size(struct drbd_device *device, enum dds_flags flags, struct
 		md->al_stripe_size_4k = prev.al_stripe_size_4k;
 		md->al_size_4k = (u64)prev.al_stripes * prev.al_stripe_size_4k;
 	}
-	lc_unlock(device->act_log);
-	wake_up(&device->al_wait);
 	drbd_md_put_buffer(device);
 	drbd_resume_io(device);
 
 	return rv;
 }
 
-sector_t
-drbd_new_dev_size(struct drbd_device *device, struct drbd_backing_dev *bdev,
-		  sector_t u_size, int assume_peer_has_space)
+/**
+ * get_max_agreeable_size()
+ * @device: DRBD device
+ * @max: Pointer to store the maximum agreeable size in
+ * @twopc_reachable_nodes: Bitmap of reachable nodes from two-phase-commit reply
+ *
+ * Check if all peer devices that have bitmap slots assigned in the metadata
+ * are connected.
+ */
+static bool get_max_agreeable_size(struct drbd_device *device, uint64_t *max,
+		uint64_t twopc_reachable_nodes) __must_hold(local)
 {
-	sector_t p_size = device->p_size;   /* partner's disk size. */
-	sector_t la_size_sect = bdev->md.la_size_sect; /* last agreed size. */
-	sector_t m_size; /* my size */
-	sector_t size = 0;
+	int node_id;
+	bool all_known;
 
-	m_size = drbd_get_max_capacity(bdev);
-
-	if (device->state.conn < C_CONNECTED && assume_peer_has_space) {
-		drbd_warn(device, "Resize while not connected was forced by the user!\n");
-		p_size = m_size;
-	}
+	all_known = true;
+	rcu_read_lock();
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
+		struct drbd_peer_device *peer_device;
 
-	if (p_size && m_size) {
-		size = min_t(sector_t, p_size, m_size);
-	} else {
-		if (la_size_sect) {
-			size = la_size_sect;
-			if (m_size && m_size < size)
-				size = m_size;
-			if (p_size && p_size < size)
-				size = p_size;
+		if (device->ldev->md.node_id == node_id) {
+			dynamic_drbd_dbg(device, "my node_id: %u\n", node_id);
+			continue; /* skip myself... */
+		}
+		/* peer_device may be NULL if we don't have a connection to that node. */
+		peer_device = peer_device_by_node_id(device, node_id);
+		if (twopc_reachable_nodes & NODE_MASK(node_id)) {
+			uint64_t size = device->resource->twopc_reply.max_possible_size;
+
+			dynamic_drbd_dbg(device, "node_id: %u, twopc YES for max_size: %llu\n",
+					node_id, (unsigned long long)size);
+
+			/* Update our cached information, they said "yes".
+			 * Note:
+			 * d_size == 0 indicates diskless peer, or not directly
+			 * connected.  It will be ignored by the min_not_zero()
+			 * aggregation elsewhere.  Only reset if size > d_size
+			 * here.  Once we really commit the change, this will
+			 * also be assigned if it was a shrinkage.
+			 */
+			if (peer_device) {
+				if (peer_device->d_size && size > peer_device->d_size)
+					peer_device->d_size = size;
+				if (size > peer_device->max_size)
+					peer_device->max_size = size;
+			}
+			continue;
+		}
+		if (peer_device) {
+			enum drbd_disk_state pdsk = peer_device->disk_state[NOW];
+			dynamic_drbd_dbg(peer_device, "node_id: %u idx: %u bm-uuid: 0x%llx flags: 0x%x max_size: %llu (%s)\n",
+					node_id,
+					peer_md->bitmap_index,
+					peer_md->bitmap_uuid,
+					peer_md->flags,
+					peer_device->max_size,
+					drbd_disk_str(pdsk));
+
+			if (test_bit(HAVE_SIZES, &peer_device->flags)) {
+				/* If we still can see it, consider its last
+				 * known size, even if it may have meanwhile
+				 * detached from its disk.
+				 * If we no longer see it, we may want to
+				 * ignore the size we last knew, and
+				 * "assume_peer_has_space".  */
+				*max = min_not_zero(*max, peer_device->max_size);
+				continue;
+			}
 		} else {
-			if (m_size)
-				size = m_size;
-			if (p_size)
-				size = p_size;
+			dynamic_drbd_dbg(device, "node_id: %u idx: %u bm-uuid: 0x%llx flags: 0x%x (not currently reachable)\n",
+					node_id,
+					peer_md->bitmap_index,
+					peer_md->bitmap_uuid,
+					peer_md->flags);
 		}
+		/* Even the currently diskless peer does not really know if it
+		 * is diskless on purpose (a "DRBD client") or if it just was
+		 * not possible to attach (backend device gone for some
+		 * reason).  But we remember in our meta data if we have ever
+		 * seen a peer disk for this peer.  If we did not ever see a
+		 * peer disk, assume that's intentional. */
+		if ((peer_md->flags & MDF_PEER_DEVICE_SEEN) == 0)
+			continue;
+
+		all_known = false;
+		/* don't break yet, min aggregation may still find a peer */
 	}
+	rcu_read_unlock();
+	return all_known;
+}
 
+#define DDUMP_LLU(d, x) do { dynamic_drbd_dbg(d, "%u: " #x ": %llu\n", __LINE__, (unsigned long long)x); } while (0)
+
+/* MUST hold a reference on ldev. */
+sector_t
+drbd_new_dev_size(struct drbd_device *device,
+		sector_t current_size, /* need at least this much */
+		sector_t user_capped_size, /* want (at most) this much */
+		enum dds_flags flags) __must_hold(local)
+{
+	struct drbd_resource *resource = device->resource;
+	uint64_t p_size = 0;
+	uint64_t la_size = device->ldev->md.effective_size; /* last agreed size */
+	uint64_t m_size; /* my size */
+	uint64_t size = 0;
+	bool all_known_connected;
+
+	/* If there are reachable_nodes, get_max_agreeable_size() will
+	 * also aggregate the twopc.resize.new_size into their d_size
+	 * and max_size.  Do that first, so drbd_partition_data_capacity()
+	 * can use that new knowledge.
+	 */
+
+	all_known_connected = get_max_agreeable_size(device, &p_size,
+		flags & DDSF_2PC ? resource->twopc_reply.reachable_nodes : 0);
+	m_size = drbd_partition_data_capacity(device);
+
+	if (all_known_connected) {
+		/* If we currently can see all peer devices,
+		 * and p_size is still 0, apparently all our peers have been
+		 * diskless, always.  If we have the only persistent backend,
+		 * only our size counts. */
+		DDUMP_LLU(device, p_size);
+		DDUMP_LLU(device, m_size);
+		p_size = min_not_zero(p_size, m_size);
+	} else if (flags & DDSF_ASSUME_UNCONNECTED_PEER_HAS_SPACE) {
+		DDUMP_LLU(device, p_size);
+		DDUMP_LLU(device, m_size);
+		DDUMP_LLU(device, la_size);
+		p_size = min_not_zero(p_size, m_size);
+		if (p_size > la_size)
+			drbd_warn(device, "Resize forced while not fully connected!\n");
+	} else {
+		DDUMP_LLU(device, p_size);
+		DDUMP_LLU(device, m_size);
+		DDUMP_LLU(device, la_size);
+		/* We currently cannot see all peer devices,
+		 * fall back to what we last agreed upon. */
+		p_size = min_not_zero(p_size, la_size);
+	}
+
+	DDUMP_LLU(device, p_size);
+	DDUMP_LLU(device, m_size);
+	size = min_not_zero(p_size, m_size);
+	DDUMP_LLU(device, size);
 	if (size == 0)
-		drbd_err(device, "Both nodes diskless!\n");
+		drbd_err(device, "All nodes diskless!\n");
 
-	if (u_size) {
-		if (u_size > size)
-			drbd_err(device, "Requested disk size is too big (%lu > %lu)\n",
-			    (unsigned long)u_size>>1, (unsigned long)size>>1);
-		else
-			size = u_size;
-	}
+	if (user_capped_size > size)
+		drbd_err(device, "Requested disk size is too big (%llu > %llu)kiB\n",
+		    (unsigned long long)user_capped_size>>1,
+		    (unsigned long long)size>>1);
+	else if (user_capped_size)
+		size = user_capped_size;
 
 	return size;
 }
@@ -1184,57 +1990,58 @@ static int drbd_check_al_size(struct drbd_device *device, struct disk_conf *dc)
 		return -EBUSY;
 	} else {
 		lc_destroy(t);
+		device->al_writ_cnt = 0;
+		memset(device->al_histogram, 0, sizeof(device->al_histogram));
 	}
 	drbd_md_mark_dirty(device); /* we changed device->act_log->nr_elemens */
 	return 0;
 }
 
-static unsigned int drbd_max_peer_bio_size(struct drbd_device *device)
+static u32 common_connection_features(struct drbd_resource *resource)
 {
-	/*
-	 * We may ignore peer limits if the peer is modern enough.  From 8.3.8
-	 * onwards the peer can use multiple BIOs for a single peer_request.
-	 */
-	if (device->state.conn < C_WF_REPORT_PARAMS)
-		return device->peer_max_bio_size;
-
-	if (first_peer_device(device)->connection->agreed_pro_version < 94)
-		return min(device->peer_max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
+	struct drbd_connection *connection;
+	u32 features = -1;
 
-	/*
-	 * Correct old drbd (up to 8.3.7) if it believes it can do more than
-	 * 32KiB.
-	 */
-	if (first_peer_device(device)->connection->agreed_pro_version == 94)
-		return DRBD_MAX_SIZE_H80_PACKET;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[NOW] < C_CONNECTED)
+			continue;
+		features &= connection->agreed_features;
+	}
+	rcu_read_unlock();
 
-	/*
-	 * drbd 8.3.8 onwards, before 8.4.0
-	 */
-	if (first_peer_device(device)->connection->agreed_pro_version < 100)
-		return DRBD_MAX_BIO_SIZE_P95;
-	return DRBD_MAX_BIO_SIZE;
+	return features;
 }
 
-static unsigned int drbd_max_discard_sectors(struct drbd_connection *connection)
+static unsigned int drbd_max_discard_sectors(struct drbd_resource *resource)
 {
-	/* when we introduced REQ_WRITE_SAME support, we also bumped
+	struct drbd_connection *connection;
+	unsigned int s = DRBD_MAX_BBIO_SECTORS;
+
+	/* when we introduced WRITE_SAME support, we also bumped
 	 * our maximum supported batch bio size used for discards. */
-	if (connection->agreed_features & DRBD_FF_WSAME)
-		return DRBD_MAX_BBIO_SECTORS;
-	/* before, with DRBD <= 8.4.6, we only allowed up to one AL_EXTENT_SIZE. */
-	return AL_EXTENT_SIZE >> 9;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[NOW] == C_CONNECTED &&
+		    !(connection->agreed_features & DRBD_FF_WSAME)) {
+			/* before, with DRBD <= 8.4.6, we only allowed up to one AL_EXTENT_SIZE. */
+			s = AL_EXTENT_SIZE >> SECTOR_SHIFT;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return s;
 }
 
-static bool drbd_discard_supported(struct drbd_connection *connection,
+static bool drbd_discard_supported(struct drbd_device *device,
 		struct drbd_backing_dev *bdev)
 {
 	if (bdev && !bdev_max_discard_sectors(bdev->backing_bdev))
 		return false;
 
-	if (connection->cstate >= C_CONNECTED &&
-	    !(connection->agreed_features & DRBD_FF_TRIM)) {
-		drbd_info(connection,
+	if (!(common_connection_features(device->resource) & DRBD_FF_TRIM)) {
+		drbd_info(device,
 			"peer DRBD too old, does not support TRIM: disabling discards\n");
 		return false;
 	}
@@ -1242,85 +2049,66 @@ static bool drbd_discard_supported(struct drbd_connection *connection,
 	return true;
 }
 
-/* This is the workaround for "bio would need to, but cannot, be split" */
-static unsigned int drbd_backing_dev_max_segments(struct drbd_device *device)
+static void get_common_queue_limits(struct queue_limits *common_limits,
+		struct drbd_device *device)
 {
-	unsigned int max_segments;
+	struct drbd_peer_device *peer_device;
+	struct queue_limits peer_limits = { 0 };
+
+	blk_set_stacking_limits(common_limits);
+	/* This is the workaround for "bio would need to, but cannot, be split" */
+	common_limits->seg_boundary_mask = PAGE_SIZE - 1;
+	common_limits->max_hw_sectors = device->device_conf.max_bio_size >> SECTOR_SHIFT;
+	common_limits->max_sectors = device->device_conf.max_bio_size >> SECTOR_SHIFT;
+	common_limits->physical_block_size = device->device_conf.block_size;
+	common_limits->logical_block_size = device->device_conf.block_size;
+	common_limits->io_min = device->device_conf.block_size;
+	common_limits->max_hw_zone_append_sectors = 0;
 
 	rcu_read_lock();
-	max_segments = rcu_dereference(device->ldev->disk_conf)->max_bio_bvecs;
+	for_each_peer_device_rcu(peer_device, device) {
+		if (!test_bit(HAVE_SIZES, &peer_device->flags) &&
+		    peer_device->repl_state[NOW] < L_ESTABLISHED)
+			continue;
+		blk_set_stacking_limits(&peer_limits);
+		peer_limits.logical_block_size = peer_device->q_limits.logical_block_size;
+		peer_limits.physical_block_size = peer_device->q_limits.physical_block_size;
+		peer_limits.alignment_offset = peer_device->q_limits.alignment_offset;
+		peer_limits.io_min = peer_device->q_limits.io_min;
+		peer_limits.io_opt = peer_device->q_limits.io_opt;
+		peer_limits.max_hw_sectors = peer_device->q_limits.max_bio_size >> SECTOR_SHIFT;
+		peer_limits.max_sectors = peer_device->q_limits.max_bio_size >> SECTOR_SHIFT;
+		blk_stack_limits(common_limits, &peer_limits, 0);
+	}
 	rcu_read_unlock();
-
-	if (!max_segments)
-		return BLK_MAX_SEGMENTS;
-	return max_segments;
 }
 
-void drbd_reconsider_queue_parameters(struct drbd_device *device,
-		struct drbd_backing_dev *bdev, struct o_qlim *o)
+void drbd_reconsider_queue_parameters(struct drbd_device *device, struct drbd_backing_dev *bdev)
 {
-	struct drbd_connection *connection =
-		first_peer_device(device)->connection;
 	struct request_queue * const q = device->rq_queue;
-	unsigned int now = queue_max_hw_sectors(q) << 9;
 	struct queue_limits lim;
 	struct request_queue *b = NULL;
-	unsigned int new;
-
-	if (bdev) {
-		b = bdev->backing_bdev->bd_disk->queue;
-
-		device->local_max_bio_size =
-			queue_max_hw_sectors(b) << SECTOR_SHIFT;
-	}
-
-	/*
-	 * We may later detach and re-attach on a disconnected Primary.  Avoid
-	 * decreasing the value in this case.
-	 *
-	 * We want to store what we know the peer DRBD can handle, not what the
-	 * peer IO backend can handle.
-	 */
-	new = min3(DRBD_MAX_BIO_SIZE, device->local_max_bio_size,
-		max(drbd_max_peer_bio_size(device), device->peer_max_bio_size));
-	if (new != now) {
-		if (device->state.role == R_PRIMARY && new < now)
-			drbd_err(device, "ASSERT FAILED new < now; (%u < %u)\n",
-					new, now);
-		drbd_info(device, "max BIO size = %u\n", new);
-	}
 
 	lim = queue_limits_start_update(q);
-	if (bdev) {
-		blk_set_stacking_limits(&lim);
-		lim.max_segments = drbd_backing_dev_max_segments(device);
-	} else {
-		lim.max_segments = BLK_MAX_SEGMENTS;
-		lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA |
-			       BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES;
-	}
-
-	lim.max_hw_sectors = new >> SECTOR_SHIFT;
-	lim.seg_boundary_mask = PAGE_SIZE - 1;
+	get_common_queue_limits(&lim, device);
 
 	/*
 	 * We don't care for the granularity, really.
-	 *
-	 * Stacking limits below should fix it for the local device.  Whether or
+	 * Stacking limits below should fix it for the local device. Whether or
 	 * not it is a suitable granularity on the remote device is not our
 	 * problem, really. If you care, you need to use devices with similar
 	 * topology on all peers.
 	 */
-	if (drbd_discard_supported(connection, bdev)) {
+	if (drbd_discard_supported(device, bdev)) {
 		lim.discard_granularity = 512;
-		lim.max_hw_discard_sectors =
-			drbd_max_discard_sectors(connection);
+		lim.max_hw_discard_sectors = drbd_max_discard_sectors(device->resource);
 	} else {
 		lim.discard_granularity = 0;
 		lim.max_hw_discard_sectors = 0;
 	}
 
 	if (bdev) {
+		b = bdev->backing_bdev->bd_disk->queue;
 		blk_stack_limits(&lim, &b->limits, 0);
 		/*
 		 * blk_set_stacking_limits() cleared the features, and
@@ -1337,14 +2125,18 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device,
 		 *    receiver will detect a checksum mismatch.
 		 */
 		lim.features |= BLK_FEAT_STABLE_WRITES;
+	} else {
+		lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA |
+			       BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES;
+
 	}
 
 	/*
-	 * If we can handle "zeroes" efficiently on the protocol, we want to do
-	 * that, even if our backend does not announce max_write_zeroes_sectors
-	 * itself.
+	 * If we can handle "zeroes" efficiently on the protocol,
+	 * we want to do that, even if our backend does not announce
+	 * max_write_zeroes_sectors itself.
 	 */
-	if (connection->agreed_features & DRBD_FF_WZEROES)
+	if (common_connection_features(device->resource) & DRBD_FF_WZEROES)
 		lim.max_write_zeroes_sectors = DRBD_MAX_BBIO_SECTORS;
 	else
 		lim.max_write_zeroes_sectors = 0;
@@ -1352,6 +2144,11 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device,
 
 	if ((lim.discard_granularity >> SECTOR_SHIFT) >
 	    lim.max_hw_discard_sectors) {
+		/*
+		 * discard_granularity is the smallest supported unit of a
+		 * discard. If that is larger than the maximum supported discard
+		 * size, we need to disable discards altogether.
+		 */
 		lim.discard_granularity = 0;
 		lim.max_hw_discard_sectors = 0;
 	}
@@ -1360,56 +2157,48 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device,
 		drbd_err(device, "setting new queue limits failed\n");
 }
 
-/* Starts the worker thread */
-static void conn_reconfig_start(struct drbd_connection *connection)
+/* Make sure IO is suspended before calling this function(). */
+static void drbd_try_suspend_al(struct drbd_device *device)
 {
-	drbd_thread_start(&connection->worker);
-	drbd_flush_workqueue(&connection->sender_work);
-}
+	struct drbd_peer_device *peer_device;
+	bool suspend = true;
+	int max_peers = device->ldev->md.max_peers, bitmap_index;
 
-/* if still unconfigured, stops worker again. */
-static void conn_reconfig_done(struct drbd_connection *connection)
-{
-	bool stop_threads;
-	spin_lock_irq(&connection->resource->req_lock);
-	stop_threads = conn_all_vols_unconf(connection) &&
-		connection->cstate == C_STANDALONE;
-	spin_unlock_irq(&connection->resource->req_lock);
-	if (stop_threads) {
-		/* ack_receiver thread and ack_sender workqueue are implicitly
-		 * stopped by receiver in conn_disconnect() */
-		drbd_thread_stop(&connection->receiver);
-		drbd_thread_stop(&connection->worker);
+	if (device->bitmap) {
+		for (bitmap_index = 0; bitmap_index < max_peers; bitmap_index++) {
+			if (_drbd_bm_total_weight(device, bitmap_index) != drbd_bm_bits(device))
+				return;
+		}
 	}
-}
-
-/* Make sure IO is suspended before calling this function(). */
-static void drbd_suspend_al(struct drbd_device *device)
-{
-	int s = 0;
 
-	if (!lc_try_lock(device->act_log)) {
-		drbd_warn(device, "Failed to lock al in drbd_suspend_al()\n");
+	if (!drbd_al_try_lock(device)) {
+		drbd_warn(device, "Failed to lock al in %s()", __func__);
 		return;
 	}
 
 	drbd_al_shrink(device);
-	spin_lock_irq(&device->resource->req_lock);
-	if (device->state.conn < C_CONNECTED)
-		s = !test_and_set_bit(AL_SUSPENDED, &device->flags);
-	spin_unlock_irq(&device->resource->req_lock);
+	read_lock_irq(&device->resource->state_rwlock);
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED) {
+			suspend = false;
+			break;
+		}
+	}
+	if (suspend)
+		suspend = !test_and_set_bit(AL_SUSPENDED, &device->flags);
+	read_unlock_irq(&device->resource->state_rwlock);
 	lc_unlock(device->act_log);
+	wake_up(&device->al_wait);
 
-	if (s)
+	if (suspend)
 		drbd_info(device, "Suspended AL updates\n");
 }
 
 
 static bool should_set_defaults(struct genl_info *info)
 {
-	struct drbd_genlmsghdr *dh = genl_info_userhdr(info);
-
-	return 0 != (dh->flags & DRBD_GENL_F_SET_DEFAULTS);
+	unsigned int flags = ((struct drbd_genlmsghdr *)genl_info_userhdr(info))->flags;
+	return 0 != (flags & DRBD_GENL_F_SET_DEFAULTS);
 }
 
 static unsigned int drbd_al_extents_max(struct drbd_backing_dev *bdev)
@@ -1464,25 +2253,47 @@ static void sanitize_disk_conf(struct drbd_device *device, struct disk_conf *dis
 		}
 	}
 
+	/* To be effective, rs_discard_granularity must not be larger than the
+	 * maximum resync request size, and multiple of 4k
+	 * (preferably a power-of-two multiple 4k).
+	 * See also make_resync_request().
+	 * That also means that if q->limits.discard_granularity or
+	 * q->limits.discard_alignment are "odd", rs_discard_granularity won't
+	 * be particularly effective, or not effective at all.
+	 */
 	if (disk_conf->rs_discard_granularity) {
-		int orig_value = disk_conf->rs_discard_granularity;
-		sector_t discard_size = bdev_max_discard_sectors(bdev) << 9;
+		unsigned int new_discard_granularity =
+			disk_conf->rs_discard_granularity;
+		unsigned int discard_sectors = bdev_max_discard_sectors(bdev);
 		unsigned int discard_granularity = bdev_discard_granularity(bdev);
-		int remainder;
-
-		if (discard_granularity > disk_conf->rs_discard_granularity)
-			disk_conf->rs_discard_granularity = discard_granularity;
 
-		remainder = disk_conf->rs_discard_granularity %
-				discard_granularity;
-		disk_conf->rs_discard_granularity += remainder;
-
-		if (disk_conf->rs_discard_granularity > discard_size)
-			disk_conf->rs_discard_granularity = discard_size;
-
-		if (disk_conf->rs_discard_granularity != orig_value)
+		/* should be at least the discard_granularity of the bdev,
+		 * and preferably a multiple (or the backend won't be able to
+		 * discard some of the "cuttings").
+		 * This also sanitizes nonsensical settings like "77 byte".
+		 */
+		new_discard_granularity = roundup(new_discard_granularity,
+				discard_granularity);
+
+		/* more than the max resync request size won't work anyways */
+		discard_sectors = min(discard_sectors,
+				DRBD_RS_DISCARD_GRANULARITY_MAX >> SECTOR_SHIFT);
+		/* Avoid compiler warning about truncated integer.
+		 * The min() above made sure the result fits even after left shift. */
+		new_discard_granularity = min(
+				new_discard_granularity >> SECTOR_SHIFT,
+				discard_sectors) << SECTOR_SHIFT;
+		/* less than the backend discard granularity is allowed if
+		   the backend granularity is a multiple of the configured value */
+		if (new_discard_granularity < discard_granularity &&
+		    discard_granularity % new_discard_granularity != 0)
+			new_discard_granularity = 0;
+
+		if (disk_conf->rs_discard_granularity != new_discard_granularity) {
 			drbd_info(device, "rs_discard_granularity changed to %d\n",
-				  disk_conf->rs_discard_granularity);
+					new_discard_granularity);
+			disk_conf->rs_discard_granularity = new_discard_granularity;
+		}
 	}
 }
 
@@ -1494,13 +2305,13 @@ static int disk_opts_check_al_size(struct drbd_device *device, struct disk_conf
 	    device->act_log->nr_elements == dc->al_extents)
 		return 0;
 
-	drbd_suspend_io(device);
+	drbd_suspend_io(device, READ_AND_WRITE);
 	/* If IO completion is currently blocked, we would likely wait
 	 * "forever" for the activity log to become unused. So we don't. */
-	if (atomic_read(&device->ap_bio_cnt))
+	if (atomic_read(&device->ap_bio_cnt[WRITE]) || atomic_read(&device->ap_bio_cnt[READ]))
 		goto out;
 
-	wait_event(device->al_wait, lc_try_lock(device->act_log));
+	wait_event(device->al_wait, drbd_al_try_lock(device));
 	drbd_al_shrink(device);
 	err = drbd_check_al_size(device, dc);
 	lc_unlock(device->act_log);
@@ -1510,24 +2321,113 @@ static int disk_opts_check_al_size(struct drbd_device *device, struct disk_conf
 	return err;
 }
 
-int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
+static struct drbd_connection *the_only_peer_with_disk(struct drbd_device *device,
+						       enum which_state which)
+{
+	const int my_node_id = device->resource->res_opts.node_id;
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	struct drbd_connection *connection = NULL;
+	struct drbd_peer_device *peer_device;
+	int node_id, peer_disks = 0;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == my_node_id)
+			continue;
+
+		if (peer_md[node_id].flags & MDF_PEER_DEVICE_SEEN)
+			peer_disks++;
+
+		if (peer_disks > 1)
+			return NULL;
+
+		peer_device = peer_device_by_node_id(device, node_id);
+		if (peer_device) {
+			enum drbd_disk_state pdsk = peer_device->disk_state[which];
+
+			if (pdsk >= D_INCONSISTENT && pdsk != D_UNKNOWN)
+				connection = peer_device->connection;
+		}
+	}
+	return connection;
+}
+
+static void __update_mdf_al_disabled(struct drbd_device *device, bool al_updates,
+				     enum which_state which)
+{
+	struct drbd_md *md = &device->ldev->md;
+	struct drbd_connection *peer = NULL;
+	bool al_updates_old = !(md->flags & MDF_AL_DISABLED);
+	bool optimized = false;
+
+	if (al_updates)
+		peer = the_only_peer_with_disk(device, which);
+
+	if (device->bitmap == NULL ||
+	    (al_updates && device->ldev->md.max_peers == 1 &&
+	    peer && peer->peer_role[which] == R_PRIMARY &&
+	    device->resource->role[which] == R_SECONDARY)) {
+		al_updates = false;
+		optimized = true;
+	}
+
+	if (al_updates_old == al_updates)
+		return;
+
+	if (al_updates) {
+		drbd_info(device, "Enabling local AL-updates\n");
+		md->flags &= ~MDF_AL_DISABLED;
+	} else {
+		drbd_info(device, "Disabling local AL-updates %s\n",
+			  optimized ? "(optimization)" : "(config)");
+		md->flags |= MDF_AL_DISABLED;
+	}
+	drbd_md_mark_dirty(device);
+}
+
+/**
+ * drbd_update_mdf_al_disabled() - update the MDF_AL_DISABLED bit in md.flags
+ * @device: DRBD device
+ * @which: OLD or NEW
+ *
+ * This function also optimizes performance by turning off al-updates when:
+ * - the cluster has only two nodes with backing disk
+ * - the other node with a backing disk is the primary
+ */
+void drbd_update_mdf_al_disabled(struct drbd_device *device, enum which_state which)
+{
+	bool al_updates;
+
+	if (!get_ldev(device))
+		return;
+
+	rcu_read_lock();
+	al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
+	rcu_read_unlock();
+	__update_mdf_al_disabled(device, al_updates, which);
+
+	put_ldev(device);
+}
+
+static int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
 	struct drbd_device *device;
+	struct drbd_resource *resource;
 	struct disk_conf *new_disk_conf, *old_disk_conf;
-	struct fifo_buffer *old_plan = NULL, *new_plan = NULL;
+	struct drbd_peer_device *peer_device;
 	int err;
-	unsigned int fifo_size;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
 
 	device = adm_ctx.device;
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	resource = device->resource;
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
 
 	/* we also need a disk
 	 * to change the options on */
@@ -1542,7 +2442,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		goto fail;
 	}
 
-	mutex_lock(&device->resource->conf_update);
+	mutex_lock(&resource->conf_update);
 	old_disk_conf = device->ldev->disk_conf;
 	*new_disk_conf = *old_disk_conf;
 	if (should_set_defaults(info))
@@ -1555,24 +2455,8 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		goto fail_unlock;
 	}
 
-	if (!expect(device, new_disk_conf->resync_rate >= 1))
-		new_disk_conf->resync_rate = 1;
-
 	sanitize_disk_conf(device, new_disk_conf, device->ldev);
 
-	if (new_disk_conf->c_plan_ahead > DRBD_C_PLAN_AHEAD_MAX)
-		new_disk_conf->c_plan_ahead = DRBD_C_PLAN_AHEAD_MAX;
-
-	fifo_size = (new_disk_conf->c_plan_ahead * 10 * SLEEP_TIME) / HZ;
-	if (fifo_size != device->rs_plan_s->size) {
-		new_plan = fifo_alloc(fifo_size);
-		if (!new_plan) {
-			drbd_err(device, "kmalloc of fifo_buffer failed");
-			retcode = ERR_NOMEM;
-			goto fail_unlock;
-		}
-	}
-
 	err = disk_opts_check_al_size(device, new_disk_conf);
 	if (err) {
 		/* Could be just "busy". Ignore?
@@ -1583,6 +2467,30 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		goto fail_unlock;
 	}
 
+	if (!old_disk_conf->d_bitmap && new_disk_conf->d_bitmap) {
+		struct drbd_md *md = &device->ldev->md;
+
+		device->bitmap = drbd_bm_alloc(md->max_peers, md->bm_block_shift);
+		if (!device->bitmap) {
+			drbd_msg_put_info(adm_ctx.reply_skb, "Failed to allocate bitmap");
+			retcode = ERR_NOMEM;
+			goto fail_unlock;
+		}
+		err = drbd_bm_resize(device, get_capacity(device->vdisk), true);
+		if (err) {
+			drbd_msg_put_info(adm_ctx.reply_skb, "Failed to allocate bitmap pages");
+			retcode = ERR_NOMEM;
+			goto fail_unlock;
+		}
+
+		drbd_bitmap_io(device, &drbd_bm_write, "write from disk_opts", BM_LOCK_ALL, NULL);
+	} else if (old_disk_conf->d_bitmap && !new_disk_conf->d_bitmap) {
+		/* That would be quite some effort, and there is no use case for this */
+		drbd_msg_put_info(adm_ctx.reply_skb, "Online freeing of the bitmap not supported");
+		retcode = ERR_INVALID_REQUEST;
+		goto fail_unlock;
+	}
+
 	lock_all_resources();
 	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
 	if (retcode == NO_ERROR) {
@@ -1594,17 +2502,9 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 	if (retcode != NO_ERROR)
 		goto fail_unlock;
 
-	if (new_plan) {
-		old_plan = device->rs_plan_s;
-		rcu_assign_pointer(device->rs_plan_s, new_plan);
-	}
-
-	mutex_unlock(&device->resource->conf_update);
+	mutex_unlock(&resource->conf_update);
 
-	if (new_disk_conf->al_updates)
-		device->ldev->md.flags &= ~MDF_AL_DISABLED;
-	else
-		device->ldev->md.flags |= MDF_AL_DISABLED;
+	__update_mdf_al_disabled(device, new_disk_conf->al_updates, NOW);
 
 	if (new_disk_conf->md_flushes)
 		clear_bit(MD_NO_FUA, &device->flags);
@@ -1612,65 +2512,298 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 		set_bit(MD_NO_FUA, &device->flags);
 
 	if (write_ordering_changed(old_disk_conf, new_disk_conf))
-		drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH);
+		drbd_bump_write_ordering(device->resource, NULL, WO_BIO_BARRIER);
 
 	if (old_disk_conf->discard_zeroes_if_aligned !=
 	    new_disk_conf->discard_zeroes_if_aligned)
-		drbd_reconsider_queue_parameters(device, device->ldev, NULL);
-
-	drbd_md_sync(device);
+		drbd_reconsider_queue_parameters(device, device->ldev);
 
-	if (device->state.conn >= C_CONNECTED) {
-		struct drbd_peer_device *peer_device;
+	drbd_md_sync_if_dirty(device);
 
-		for_each_peer_device(peer_device, device)
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED)
 			drbd_send_sync_param(peer_device);
 	}
 
 	kvfree_rcu_mightsleep(old_disk_conf);
-	kfree(old_plan);
 	mod_timer(&device->request_timer, jiffies + HZ);
 	goto success;
 
 fail_unlock:
-	mutex_unlock(&device->resource->conf_update);
+	mutex_unlock(&resource->conf_update);
  fail:
 	kfree(new_disk_conf);
-	kfree(new_plan);
 success:
+	if (retcode != NO_ERROR)
+		synchronize_rcu();
 	put_ldev(device);
  out:
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
- finish:
+out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static struct file *open_backing_dev(struct drbd_device *device,
-		const char *bdev_path, void *claim_ptr, bool do_bd_link)
+static void mutex_unlock_cond(struct mutex *mutex, bool *have_mutex)
 {
-	struct file *file;
-	int err = 0;
+	if (*have_mutex) {
+		mutex_unlock(mutex);
+		*have_mutex = false;
+	}
+}
+
+static void update_resource_dagtag(struct drbd_resource *resource, struct drbd_backing_dev *bdev)
+{
+	u64 dagtag = 0;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md;
+		if (bdev->md.node_id == node_id)
+			continue;
+
+		peer_md = &bdev->md.peers[node_id];
+
+		if (peer_md->bitmap_uuid)
+			dagtag = max(peer_md->bitmap_dagtag, dagtag);
+	}
+
+	spin_lock_irq(&resource->tl_update_lock);
+	if (dagtag > resource->dagtag_sector) {
+		resource->dagtag_before_attach = resource->dagtag_sector;
+		resource->dagtag_from_backing_dev = dagtag;
+		WRITE_ONCE(resource->dagtag_sector, dagtag);
+	}
+	spin_unlock_irq(&resource->tl_update_lock);
+}
+
+static int used_bitmap_slots(struct drbd_backing_dev *bdev)
+{
+	int node_id;
+	int used = 0;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &bdev->md.peers[node_id];
+
+		if (peer_md->flags & MDF_HAVE_BITMAP)
+			used++;
+	}
+
+	return used;
+}
+
+static bool bitmap_index_vacant(struct drbd_backing_dev *bdev, int bitmap_index)
+{
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &bdev->md.peers[node_id];
+
+		if (peer_md->bitmap_index == bitmap_index)
+			return false;
+	}
+	return true;
+}
+
+int drbd_unallocated_index(struct drbd_backing_dev *bdev)
+{
+	int bitmap_index;
+	int bm_max_peers = bdev->md.max_peers;
+
+	for (bitmap_index = 0; bitmap_index < bm_max_peers; bitmap_index++) {
+		if (bitmap_index_vacant(bdev, bitmap_index))
+			return bitmap_index;
+	}
+
+	return -1;
+}
 
-	file = bdev_file_open_by_path(bdev_path, BLK_OPEN_READ | BLK_OPEN_WRITE,
-				      claim_ptr, NULL);
+static int
+allocate_bitmap_index(struct drbd_peer_device *peer_device,
+		      struct drbd_backing_dev *nbc)
+{
+	const int peer_node_id = peer_device->connection->peer_node_id;
+	struct drbd_peer_md *peer_md = &nbc->md.peers[peer_node_id];
+	int bitmap_index;
+
+	bitmap_index = drbd_unallocated_index(nbc);
+	if (bitmap_index == -1) {
+		drbd_err(peer_device, "Not enough free bitmap slots\n");
+		return -ENOSPC;
+	}
+
+	peer_md->bitmap_index = bitmap_index;
+	peer_device->bitmap_index = bitmap_index;
+	peer_md->flags |= MDF_HAVE_BITMAP;
+
+	return 0;
+}
+
+static struct drbd_peer_md *day0_peer_md(struct drbd_device *device)
+{
+	const int my_node_id = device->resource->res_opts.node_id;
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (node_id == my_node_id)
+			continue;
+		/* Only totally unused slots definitely contain the day0 UUID. */
+		if (peer_md[node_id].bitmap_index == -1 && !peer_md[node_id].flags)
+			return &peer_md[node_id];
+	}
+	return NULL;
+}
+
+/*
+ * Clear the slot for this peer in the metadata. If md_flags is empty, clear
+ * the slot completely. Otherwise make it a slot for a diskless peer. Also
+ * clear any bitmap associated with this peer.
+ */
+static int clear_peer_slot(struct drbd_device *device, int peer_node_id, u32 md_flags)
+{
+	struct drbd_peer_md *peer_md, *day0_md;
+	struct meta_data_on_disk_9 *buffer;
+	int from_index, freed_index;
+	bool free_bitmap_slot;
+
+	if (!get_ldev(device))
+		return -ENODEV;
+
+	peer_md = &device->ldev->md.peers[peer_node_id];
+	free_bitmap_slot = peer_md->flags & MDF_HAVE_BITMAP;
+	if (free_bitmap_slot) {
+		drbd_suspend_io(device, WRITE_ONLY);
+
+		/*
+		 * Unallocated slots are considered to track writes to the
+		 * device since day 0. In order to keep that promise, copy the
+		 * bitmap from an unallocated slot to this one, or set it to
+		 * all out-of-sync.
+		 */
+
+		from_index = drbd_unallocated_index(device->ldev);
+		freed_index = peer_md->bitmap_index;
+	}
+	buffer = drbd_md_get_buffer(device, __func__); /* lock meta-data IO to superblock */
+	if (buffer == NULL)
+		goto out_no_buffer;
+
+	/* Look for day0 UUID before changing this peer slot to a day0 slot. */
+	day0_md = day0_peer_md(device);
+
+	peer_md->flags &= md_flags & ~MDF_HAVE_BITMAP;
+	peer_md->bitmap_index = -1;
+
+	if (free_bitmap_slot) {
+		drbd_bm_lock(device, __func__, BM_LOCK_BULK);
+		/*
+		 * Regular bitmap OPs (calling into bm_op()) can run in parallel to
+		 * drbd_bm_copy_slot() and interleave with it as drbd_bm_copy_slot()
+		 * gives up its locks when it moves on to the next source page.
+		 * The bitmap->bm_all_slots_lock ensures that drbd_set_sync()
+		 * (which iterates over multiple slots) does not interleave with
+		 * drbd_bm_copy_slot() while it copies data from one slot to another
+		 * one.
+		 */
+		if (from_index != -1)
+			drbd_bm_copy_slot(device, from_index, freed_index);
+		else
+			_drbd_bm_set_many_bits(device, freed_index, 0, -1UL);
+
+		drbd_bm_write(device, NULL);
+		drbd_bm_unlock(device);
+	}
+
+	/*
+	 * When we forget a peer, we clear the flags. In this case, reset the
+	 * bitmap UUID to the day0 UUID. Peer slots without any bitmap index or
+	 * any flags set should always contain the day0 UUID.
+	 */
+	if (!peer_md->flags && day0_md) {
+		peer_md->bitmap_uuid = day0_md->bitmap_uuid;
+		peer_md->bitmap_dagtag = day0_md->bitmap_dagtag;
+	} else {
+		peer_md->bitmap_uuid = 0;
+		peer_md->bitmap_dagtag = 0;
+	}
+
+	clear_bit(MD_DIRTY, &device->flags);
+	drbd_md_write(device, buffer);
+	drbd_md_put_buffer(device);
+
+ out_no_buffer:
+	if (free_bitmap_slot)
+		drbd_resume_io(device);
+
+	put_ldev(device);
+
+	return 0;
+}
+
+bool want_bitmap(struct drbd_peer_device *peer_device)
+{
+	struct peer_device_conf *pdc;
+	bool want_bitmap = false;
+
+	rcu_read_lock();
+	pdc = rcu_dereference(peer_device->conf);
+	if (pdc)
+		want_bitmap |= pdc->bitmap;
+	rcu_read_unlock();
+
+	return want_bitmap;
+}
+
+static void close_backing_dev(struct drbd_device *device,
+		struct file *bdev_file, bool do_bd_unlink)
+{
+	if (!bdev_file)
+		return;
+	if (do_bd_unlink)
+		bd_unlink_disk_holder(file_bdev(bdev_file), device->vdisk);
+	fput(bdev_file);
+}
+
+void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev)
+{
+	if (ldev == NULL)
+		return;
+
+	drbd_dax_close(ldev);
+
+	close_backing_dev(device,
+			  ldev->f_md_bdev,
+			  ldev->md_bdev != ldev->backing_bdev);
+	close_backing_dev(device, ldev->backing_bdev_file, true);
+
+	kfree(ldev->disk_conf);
+	kfree(ldev);
+}
+
+static struct file *open_backing_dev(struct drbd_device *device,
+		const char *bdev_path, void *claim_ptr)
+{
+	struct file *file = bdev_file_open_by_path(bdev_path,
+				  BLK_OPEN_READ | BLK_OPEN_WRITE,
+				  claim_ptr, NULL);
 	if (IS_ERR(file)) {
 		drbd_err(device, "open(\"%s\") failed with %ld\n",
 				bdev_path, PTR_ERR(file));
-		return file;
 	}
+	return file;
+}
 
-	if (!do_bd_link)
-		return file;
-
-	err = bd_link_disk_holder(file_bdev(file), device->vdisk);
+static int link_backing_dev(struct drbd_device *device,
+		const char *bdev_path, struct file *file)
+{
+	int err = bd_link_disk_holder(file_bdev(file), device->vdisk);
 	if (err) {
 		fput(file);
 		drbd_err(device, "bd_link_disk_holder(\"%s\", ...) failed with %d\n",
 				bdev_path, err);
-		file = ERR_PTR(err);
 	}
-	return file;
+	return err;
 }
 
 static int open_backing_devices(struct drbd_device *device,
@@ -1678,14 +2811,27 @@ static int open_backing_devices(struct drbd_device *device,
 		struct drbd_backing_dev *nbc)
 {
 	struct file *file;
+	void *meta_claim_ptr;
+	int err;
 
-	file = open_backing_dev(device, new_disk_conf->backing_dev, device,
-				  true);
+	file = open_backing_dev(device, new_disk_conf->backing_dev, device);
 	if (IS_ERR(file))
 		return ERR_OPEN_DISK;
+
+	err = link_backing_dev(device, new_disk_conf->backing_dev, file);
+	if (err) {
+		/* close without unlinking; otherwise error path will try to unlink */
+		close_backing_dev(device, file, false);
+		return ERR_OPEN_DISK;
+	}
 	nbc->backing_bdev = file_bdev(file);
 	nbc->backing_bdev_file = file;
 
+	/* meta_claim_ptr: device, if claimed exclusively; shared drbd_m_holder,
+	 * if potentially shared with other drbd minors
+	 */
+	meta_claim_ptr = (new_disk_conf->meta_dev_idx < 0) ?
+		(void *)device : (void *)drbd_m_holder;
 	/*
 	 * meta_dev_idx >= 0: external fixed size, possibly multiple
 	 * drbd sharing one meta device.  TODO in that case, paranoia
@@ -1694,95 +2840,402 @@ static int open_backing_devices(struct drbd_device *device,
 	 * should check it for you already; but if you don't, or
 	 * someone fooled it, we need to double check here)
 	 */
-	file = open_backing_dev(device, new_disk_conf->meta_dev,
-		/* claim ptr: device, if claimed exclusively; shared drbd_m_holder,
-		 * if potentially shared with other drbd minors */
-			(new_disk_conf->meta_dev_idx < 0) ? (void*)device : (void*)drbd_m_holder,
-		/* avoid double bd_claim_by_disk() for the same (source,target) tuple,
-		 * as would happen with internal metadata. */
-			(new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_FLEX_INT &&
-			 new_disk_conf->meta_dev_idx != DRBD_MD_INDEX_INTERNAL));
+	file = open_backing_dev(device, new_disk_conf->meta_dev, meta_claim_ptr);
 	if (IS_ERR(file))
 		return ERR_OPEN_MD_DISK;
+
+	/* avoid double bd_claim_by_disk() for the same (source,target) tuple,
+	 * as would happen with internal metadata. */
+	if (file_bdev(file) != nbc->backing_bdev) {
+		err = link_backing_dev(device, new_disk_conf->meta_dev, file);
+		if (err) {
+			/* close without unlinking; otherwise error path will try to unlink */
+			close_backing_dev(device, file, false);
+			return ERR_OPEN_MD_DISK;
+		}
+	}
+
 	nbc->md_bdev = file_bdev(file);
 	nbc->f_md_bdev = file;
 	return NO_ERROR;
 }
 
-static void close_backing_dev(struct drbd_device *device,
-		struct file *bdev_file, bool do_bd_unlink)
+static int check_activity_log_stripe_size(struct drbd_device *device, struct drbd_md *md)
 {
-	if (!bdev_file)
-		return;
-	if (do_bd_unlink)
-		bd_unlink_disk_holder(file_bdev(bdev_file), device->vdisk);
-	fput(bdev_file);
-}
+	u32 al_stripes = md->al_stripes;
+	u32 al_stripe_size_4k = md->al_stripe_size_4k;
+	u64 al_size_4k;
 
-void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev)
-{
-	if (ldev == NULL)
-		return;
+	/* both not set: default to old fixed size activity log */
+	if (al_stripes == 0 && al_stripe_size_4k == 0) {
+		al_stripes = 1;
+		al_stripe_size_4k = (32768 >> 9)/8;
+	}
 
-	close_backing_dev(device, ldev->f_md_bdev,
-			  ldev->md_bdev != ldev->backing_bdev);
-	close_backing_dev(device, ldev->backing_bdev_file, true);
+	/* some paranoia plausibility checks */
 
-	kfree(ldev->disk_conf);
-	kfree(ldev);
+	/* we need both values to be set */
+	if (al_stripes == 0 || al_stripe_size_4k == 0)
+		goto err;
+
+	al_size_4k = (u64)al_stripes * al_stripe_size_4k;
+
+	/* Upper limit of activity log area, to avoid potential overflow
+	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
+	 * than 72 * 4k blocks total only increases the amount of history,
+	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
+	if (al_size_4k > (16 * 1024 * 1024/4))
+		goto err;
+
+	/* Lower limit: we need at least 8 transaction slots (32kB)
+	 * to not break existing setups */
+	if (al_size_4k < (32768 >> 9)/8)
+		goto err;
+
+	md->al_size_4k = al_size_4k;
+
+	return 0;
+err:
+	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
+			al_stripes, al_stripe_size_4k);
+	return -EINVAL;
 }
 
-int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
+static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
 {
-	struct drbd_config_context adm_ctx;
-	struct drbd_device *device;
-	struct drbd_peer_device *peer_device;
-	struct drbd_connection *connection;
-	int err;
-	enum drbd_ret_code retcode;
-	enum determine_dev_size dd;
-	sector_t max_possible_sectors;
+	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
+	struct drbd_md *md = &bdev->md;
+	s32 on_disk_al_sect;
+	s32 on_disk_bm_sect;
+
+	if (md->max_peers > DRBD_PEERS_MAX) {
+		drbd_err(device, "bm_max_peers too high\n");
+		goto err;
+	}
+
+	/* The on-disk size of the activity log, calculated from offsets, and
+	 * the size of the activity log calculated from the stripe settings,
+	 * should match.
+	 * Though we could relax this a bit: it is ok, if the striped activity log
+	 * fits in the available on-disk activity log size.
+	 * Right now, that would break how resize is implemented.
+	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
+	 * of possible unused padding space in the on disk layout. */
+	if (md->al_offset < 0) {
+		if (md->bm_offset > md->al_offset)
+			goto err;
+		on_disk_al_sect = -md->al_offset;
+		on_disk_bm_sect = md->al_offset - md->bm_offset;
+	} else {
+		if (md->al_offset != (4096 >> 9))
+			goto err;
+		if (md->bm_offset < md->al_offset + md->al_size_4k * (4096 >> 9))
+			goto err;
+
+		on_disk_al_sect = md->bm_offset - (4096 >> 9);
+		on_disk_bm_sect = md->md_size_sect - md->bm_offset;
+	}
+
+	/* old fixed size meta data is exactly that: fixed. */
+	if (md->meta_dev_idx >= 0) {
+		if (md->bm_block_size != BM_BLOCK_SIZE_4k
+		||  md->md_size_sect != (128 << 20 >> 9)
+		||  md->al_offset != (4096 >> 9)
+		||  md->bm_offset != (4096 >> 9) + (32768 >> 9)
+		||  md->al_stripes != 1
+		||  md->al_stripe_size_4k != (32768 >> 12))
+			goto err;
+	}
+
+	if (capacity < md->md_size_sect)
+		goto err;
+	if (capacity - md->md_size_sect < drbd_md_first_sector(bdev))
+		goto err;
+
+	/* should be aligned, and at least 32k */
+	if ((on_disk_al_sect & 7) || (on_disk_al_sect < (32768 >> 9)))
+		goto err;
+
+	/* should fit (for now: exactly) into the available on-disk space;
+	 * overflow prevention is in check_activity_log_stripe_size() above. */
+	if (on_disk_al_sect != md->al_size_4k * (4096 >> 9))
+		goto err;
+
+	/* again, should be aligned */
+	if (md->bm_offset & 7)
+		goto err;
+
+	/* FIXME check for device grow with flex external meta data? */
+
+	/* can the available bitmap space cover the last agreed device size? */
+	if (on_disk_bm_sect < drbd_capacity_to_on_disk_bm_sect(
+				md->effective_size, md))
+		goto err;
+
+	return 0;
+
+err:
+	drbd_err(device, "meta data offsets don't make sense: idx=%d bm_block_size=%d al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
+			md->meta_dev_idx, md->bm_block_size,
+			md->al_stripes, md->al_stripe_size_4k,
+			md->al_offset, md->bm_offset, md->md_size_sect,
+			(unsigned long long)md->effective_size,
+			(unsigned long long)capacity);
+
+	return -EINVAL;
+}
+
+__printf(2, 3)
+static void drbd_err_and_skb_info(struct drbd_config_context *adm_ctx, const char *format, ...)
+{
+	struct drbd_device *device = adm_ctx->device;
+	va_list args;
+	char *text;
+
+	va_start(args, format);
+	text = kvasprintf(GFP_ATOMIC, format, args);
+	va_end(args);
+
+	if (!text)
+		return;
+
+	drbd_err(device, "%s", text);
+	drbd_msg_put_info(adm_ctx->reply_skb, text);
+
+	kfree(text);
+}
+
+static void decode_md_9(struct meta_data_on_disk_9 *on_disk, struct drbd_md *md)
+{
+	int i;
+
+	md->effective_size = be64_to_cpu(on_disk->effective_size);
+	md->current_uuid = be64_to_cpu(on_disk->current_uuid);
+	md->prev_members = be64_to_cpu(on_disk->members);
+	md->device_uuid = be64_to_cpu(on_disk->device_uuid);
+	md->md_size_sect = be32_to_cpu(on_disk->md_size_sect);
+	md->al_offset = be32_to_cpu(on_disk->al_offset);
+
+	md->bm_offset = be32_to_cpu(on_disk->bm_offset);
+
+	md->flags = be32_to_cpu(on_disk->flags);
+
+	md->max_peers = be32_to_cpu(on_disk->bm_max_peers);
+	md->bm_block_size = be32_to_cpu(on_disk->bm_bytes_per_bit);
+	md->node_id = be32_to_cpu(on_disk->node_id);
+	md->al_stripes = be32_to_cpu(on_disk->al_stripes);
+	md->al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
+
+
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		struct drbd_peer_md *peer_md = &md->peers[i];
+
+		peer_md->bitmap_uuid = be64_to_cpu(on_disk->peers[i].bitmap_uuid);
+		peer_md->bitmap_dagtag = be64_to_cpu(on_disk->peers[i].bitmap_dagtag);
+		peer_md->flags = be32_to_cpu(on_disk->peers[i].flags);
+		peer_md->bitmap_index = be32_to_cpu(on_disk->peers[i].bitmap_index);
+
+		if (peer_md->bitmap_index == -1)
+			continue;
+		peer_md->flags |= MDF_HAVE_BITMAP;
+	}
+	for (i = 0; i < ARRAY_SIZE(on_disk->history_uuids); i++)
+		md->history_uuids[i] = be64_to_cpu(on_disk->history_uuids[i]);
+
+	BUILD_BUG_ON(ARRAY_SIZE(md->history_uuids) != ARRAY_SIZE(on_disk->history_uuids));
+}
+
+
+static void decode_magic(struct meta_data_on_disk_9 *on_disk, u32 *magic, u32 *flags)
+{
+	/* magic and flags are in at the same offsets in 8.4 and 9 */
+	*magic = be32_to_cpu(on_disk->magic);
+	*flags = be32_to_cpu(on_disk->flags);
+}
+
+static
+int drbd_md_decode(struct drbd_config_context *adm_ctx,
+		   struct drbd_backing_dev *bdev,
+		   void *buffer)
+{
+	struct drbd_device *device = adm_ctx->device;
+	u32 magic, flags;
+	int i, rv = NO_ERROR;
+	int my_node_id = device->resource->res_opts.node_id;
+
+	decode_magic(buffer, &magic, &flags);
+	if ((magic == DRBD_MD_MAGIC_09 && !(flags & MDF_AL_CLEAN)) ||
+	    magic == DRBD_MD_MAGIC_84_UNCLEAN ||
+	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
+		/* btw: that's Activity Log clean, not "all" clean. */
+		drbd_err_and_skb_info(adm_ctx,
+				"Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
+		rv = ERR_MD_UNCLEAN;
+		goto err;
+	}
+	rv = ERR_MD_INVALID;
+	if (magic != DRBD_MD_MAGIC_09 && magic !=
+	    DRBD_MD_MAGIC_84_UNCLEAN && magic !=  DRBD_MD_MAGIC_08) {
+		if (magic == DRBD_MD_MAGIC_07)
+			drbd_err_and_skb_info(adm_ctx,
+				"Found old meta data magic. Did you \"drbdadm create-md\"?\n");
+		else
+			drbd_err_and_skb_info(adm_ctx,
+				"Meta data magic not found. Did you \"drbdadm create-md\"?\n");
+		goto err;
+	}
+
+	if (magic == DRBD_MD_MAGIC_09) {
+		clear_bit(LEGACY_84_MD, &device->flags);
+		decode_md_9(buffer, &bdev->md);
+	} else {
+		if (!device->resource->res_opts.drbd8_compat_mode) {
+			drbd_err_and_skb_info(adm_ctx,
+				"Found old meta data magic. Did you \"drbdadm create-md\"?\n");
+			goto err;
+		}
+		set_bit(LEGACY_84_MD, &device->flags);
+		drbd_md_decode_84(buffer, &bdev->md);
+		if (bdev->md.bm_block_size != BM_BLOCK_SIZE_4k) {
+			drbd_err_and_skb_info(adm_ctx,
+				"unexpected bm_bytes_per_bit: %u (expected %u)\n",
+				bdev->md.bm_block_size, BM_BLOCK_SIZE_4k);
+			goto err;
+		}
+	}
+
+	if (!is_power_of_2(bdev->md.bm_block_size)
+	|| bdev->md.bm_block_size < BM_BLOCK_SIZE_MIN
+	|| bdev->md.bm_block_size > BM_BLOCK_SIZE_MAX) {
+		drbd_err_and_skb_info(adm_ctx,
+			"unexpected bm_bytes_per_bit: %u (expected power of 2 in [%u..%u])\n",
+			bdev->md.bm_block_size, BM_BLOCK_SIZE_MIN, BM_BLOCK_SIZE_MAX);
+		goto err;
+	}
+	bdev->md.bm_block_shift = ilog2(bdev->md.bm_block_size);
+
+	if (check_activity_log_stripe_size(device, &bdev->md))
+		goto err;
+	if (check_offsets_and_sizes(device, bdev))
+		goto err;
+
+	if (bdev->md.node_id != -1 && bdev->md.node_id != my_node_id) {
+		drbd_err_and_skb_info(adm_ctx, "ambiguous node id: meta-data: %d, config: %d\n",
+			bdev->md.node_id, my_node_id);
+		goto err;
+	}
+
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		struct drbd_peer_md *peer_md = &bdev->md.peers[i];
+
+		if (peer_md->bitmap_index == -1)
+			continue;
+		if (i == my_node_id) {
+			drbd_err_and_skb_info(adm_ctx, "my own node id (%d) should not have a bitmap index (%d)\n",
+				my_node_id, peer_md->bitmap_index);
+			goto err;
+		}
+		if (peer_md->bitmap_index < -1 || peer_md->bitmap_index >= bdev->md.max_peers) {
+			drbd_err_and_skb_info(adm_ctx, "peer node id %d: bitmap index (%d) exceeds allocated bitmap slots (%d)\n",
+				i, peer_md->bitmap_index, bdev->md.max_peers);
+			goto err;
+		}
+		/* maybe: for each bitmap_index != -1, create a connection object
+		 * with peer_node_id = i, unless already present. */
+	}
+
+	rv = NO_ERROR;
+
+err:
+	return rv;
+}
+
+/**
+ * drbd_md_read() - Reads in the meta data super block
+ * @adm_ctx:	DRBD config context.
+ * @bdev:	Device from which the meta data should be read in.
+ *
+ * Return NO_ERROR on success, and an enum drbd_ret_code in case
+ * something goes wrong.
+ *
+ * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
+ * even before @bdev is assigned to @device->ldev.
+ */
+static int drbd_md_read(struct drbd_config_context *adm_ctx, struct drbd_backing_dev *bdev)
+{
+	struct drbd_device *device = adm_ctx->device;
+	void *buffer;
+	int rv;
+
+	if (device->disk_state[NOW] != D_DISKLESS)
+		return ERR_DISK_CONFIGURED;
+
+	/* First, figure out where our meta data superblock is located,
+	 * and read it. */
+	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
+	bdev->md.md_offset = drbd_md_ss(bdev);
+	/* Even for (flexible or indexed) external meta data,
+	 * initially restrict us to the 4k superblock for now.
+	 * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */
+	bdev->md.md_size_sect = 8;
+
+	drbd_dax_open(bdev);
+	if (drbd_md_dax_active(bdev)) {
+		drbd_info(device, "meta-data IO uses: dax-pmem\n");
+		rv = drbd_md_decode(adm_ctx, bdev, drbd_dax_md_addr(bdev));
+		if (rv != NO_ERROR)
+			return rv;
+		if (drbd_dax_map(bdev))
+			return ERR_IO_MD_DISK;
+		return NO_ERROR;
+	}
+	drbd_info(device, "meta-data IO uses: blk-bio\n");
+
+	buffer = drbd_md_get_buffer(device, __func__);
+	if (!buffer)
+		return ERR_NOMEM;
+
+	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset,
+				 REQ_OP_READ)) {
+		/* NOTE: can't do normal error processing here as this is
+		   called BEFORE disk is attached */
+		drbd_err_and_skb_info(adm_ctx, "Error while reading metadata.\n");
+		rv = ERR_IO_MD_DISK;
+		goto err;
+	}
+
+	rv = drbd_md_decode(adm_ctx, bdev, buffer);
+ err:
+	drbd_md_put_buffer(device);
+
+	return rv;
+}
+
+static int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct drbd_device *device;
+	struct drbd_resource *resource;
+	int err, retcode;
+	enum determine_dev_size dd;
 	sector_t min_md_device_sectors;
-	struct drbd_backing_dev *nbc = NULL; /* new_backing_conf */
+	struct drbd_backing_dev *nbc; /* new_backing_conf */
+	sector_t backing_disk_max_sectors;
 	struct disk_conf *new_disk_conf = NULL;
-	struct lru_cache *resync_lru = NULL;
-	struct fifo_buffer *new_plan = NULL;
-	union drbd_state ns, os;
 	enum drbd_state_rv rv;
-	struct net_conf *nc;
+	struct drbd_peer_device *peer_device;
+	unsigned int slots_needed = 0;
+	bool have_conf_update = false;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
-
 	device = adm_ctx.device;
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	peer_device = first_peer_device(device);
-	connection = peer_device->connection;
-	conn_reconfig_start(connection);
-
-	/* if you want to reconfigure, please tear down first */
-	if (device->state.disk > D_DISKLESS) {
-		retcode = ERR_DISK_CONFIGURED;
-		goto fail;
+	resource = device->resource;
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
 	}
-	/* It may just now have detached because of IO error.  Make sure
-	 * drbd_ldev_destroy is done already, we may end up here very fast,
-	 * e.g. if someone calls attach from the on-io-error handler,
-	 * to realize a "hot spare" feature (not that I'd recommend that) */
-	wait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags));
-
-	/* make sure there is no leftover from previous force-detach attempts */
-	clear_bit(FORCE_DETACH, &device->flags);
-	clear_bit(WAS_IO_ERROR, &device->flags);
-	clear_bit(WAS_READ_ERROR, &device->flags);
-
-	/* and no leftover from previously aborted resync or verify, either */
-	device->rs_total = 0;
-	device->rs_failed = 0;
-	atomic_set(&device->rs_pending_cnt, 0);
 
 	/* allocation not in the IO path, drbdsetup context */
 	nbc = kzalloc_obj(struct drbd_backing_dev);
@@ -1807,30 +3260,16 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		goto fail;
 	}
 
-	if (new_disk_conf->c_plan_ahead > DRBD_C_PLAN_AHEAD_MAX)
-		new_disk_conf->c_plan_ahead = DRBD_C_PLAN_AHEAD_MAX;
-
-	new_plan = fifo_alloc((new_disk_conf->c_plan_ahead * 10 * SLEEP_TIME) / HZ);
-	if (!new_plan) {
-		retcode = ERR_NOMEM;
-		goto fail;
-	}
-
 	if (new_disk_conf->meta_dev_idx < DRBD_MD_INDEX_FLEX_INT) {
 		retcode = ERR_MD_IDX_INVALID;
 		goto fail;
 	}
 
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (nc) {
-		if (new_disk_conf->fencing == FP_STONITH && nc->wire_protocol == DRBD_PROT_A) {
-			rcu_read_unlock();
-			retcode = ERR_STONITH_AND_PROT_A;
-			goto fail;
-		}
-	}
-	rcu_read_unlock();
+	lock_all_resources();
+	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
+	unlock_all_resources();
+	if (retcode != NO_ERROR)
+		goto fail;
 
 	retcode = open_backing_devices(device, new_disk_conf, nbc);
 	if (retcode != NO_ERROR)
@@ -1843,37 +3282,79 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		goto fail;
 	}
 
-	resync_lru = lc_create("resync", drbd_bm_ext_cache,
-			1, 61, sizeof(struct bm_extent),
-			offsetof(struct bm_extent, lce));
-	if (!resync_lru) {
-		retcode = ERR_NOMEM;
+	/* if you want to reconfigure, please tear down first */
+	if (device->disk_state[NOW] > D_DISKLESS) {
+		retcode = ERR_DISK_CONFIGURED;
 		goto fail;
 	}
+	/* It may just now have detached because of IO error.  Make sure
+	 * drbd_ldev_destroy is done already, we may end up here very fast,
+	 * e.g. if someone calls attach from the on-io-error handler,
+	 * to realize a "hot spare" feature (not that I'd recommend that) */
+	wait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags));
+
+	/* make sure there is no leftover from previous force-detach attempts */
+	clear_bit(FORCE_DETACH, &device->flags);
+
+	/* and no leftover from previously aborted resync or verify, either */
+	for_each_peer_device(peer_device, device) {
+		while (atomic_read(&peer_device->rs_pending_cnt)) {
+			drbd_info_ratelimit(peer_device, "wait for rs_pending_cnt to clear\n");
+			if (schedule_timeout_interruptible(HZ / 10)) {
+				retcode = ERR_INTR;
+				goto fail;
+			}
+		}
+
+		peer_device->rs_total = 0;
+		peer_device->rs_failed = 0;
+	}
 
 	/* Read our meta data super block early.
-	 * This also sets other on-disk offsets. */
-	retcode = drbd_md_read(device, nbc);
+	 * This also sets other on-disk offsets.
+	 */
+	retcode = drbd_md_read(&adm_ctx, nbc);
 	if (retcode != NO_ERROR)
 		goto fail;
 
+	if (device->bitmap) {
+		drbd_err_and_skb_info(&adm_ctx, "already has a bitmap, this should not happen\n");
+		retcode = ERR_INVALID_REQUEST;
+		goto fail;
+	}
+
+	if (new_disk_conf->d_bitmap) {
+		device->bitmap = drbd_bm_alloc(nbc->md.max_peers, nbc->md.bm_block_shift);
+		if (!device->bitmap) {
+			retcode = ERR_NOMEM;
+			goto fail;
+		}
+	} else {
+		if (!list_empty(&resource->connections)) {
+			drbd_err_and_skb_info(&adm_ctx,
+				"Disabling bitmap allocation with peers defined is not allowed");
+			retcode = ERR_INVALID_REQUEST;
+			goto fail;
+		}
+	}
+	device->last_bm_block_shift = nbc->md.bm_block_shift;
+
 	sanitize_disk_conf(device, new_disk_conf, nbc);
 
-	if (drbd_get_max_capacity(nbc) < new_disk_conf->disk_size) {
-		drbd_err(device, "max capacity %llu smaller than disk size %llu\n",
-			(unsigned long long) drbd_get_max_capacity(nbc),
+	backing_disk_max_sectors = drbd_get_max_capacity(device, nbc, true);
+	if (backing_disk_max_sectors < new_disk_conf->disk_size) {
+		drbd_err_and_skb_info(&adm_ctx, "max capacity %llu smaller than disk size %llu\n",
+			(unsigned long long) backing_disk_max_sectors,
 			(unsigned long long) new_disk_conf->disk_size);
 		retcode = ERR_DISK_TOO_SMALL;
 		goto fail;
 	}
 
 	if (new_disk_conf->meta_dev_idx < 0) {
-		max_possible_sectors = DRBD_MAX_SECTORS_FLEX;
 		/* at least one MB, otherwise it does not make sense */
 		min_md_device_sectors = (2<<10);
 	} else {
-		max_possible_sectors = DRBD_MAX_SECTORS;
-		min_md_device_sectors = MD_128MB_SECT * (new_disk_conf->meta_dev_idx + 1);
+		min_md_device_sectors = (128 << 20 >> 9) * (new_disk_conf->meta_dev_idx + 1);
 	}
 
 	if (drbd_get_capacity(nbc->md_bdev) < min_md_device_sectors) {
@@ -1886,36 +3367,32 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 
 	/* Make sure the new disk is big enough
 	 * (we may currently be R_PRIMARY with no local disk...) */
-	if (drbd_get_max_capacity(nbc) < get_capacity(device->vdisk)) {
+	if (backing_disk_max_sectors <
+	    get_capacity(device->vdisk)) {
+		drbd_err_and_skb_info(&adm_ctx,
+			"Current (diskless) capacity %llu, cannot attach smaller (%llu) disk\n",
+			(unsigned long long)get_capacity(device->vdisk),
+			(unsigned long long)backing_disk_max_sectors);
 		retcode = ERR_DISK_TOO_SMALL;
 		goto fail;
 	}
 
 	nbc->known_size = drbd_get_capacity(nbc->backing_bdev);
 
-	if (nbc->known_size > max_possible_sectors) {
-		drbd_warn(device, "==> truncating very big lower level device "
-			"to currently maximum possible %llu sectors <==\n",
-			(unsigned long long) max_possible_sectors);
-		if (new_disk_conf->meta_dev_idx >= 0)
-			drbd_warn(device, "==>> using internal or flexible "
-				      "meta data may help <<==\n");
-	}
-
-	drbd_suspend_io(device);
-	/* also wait for the last barrier ack. */
-	/* FIXME see also https://daiquiri.linbit/cgi-bin/bugzilla/show_bug.cgi?id=171
-	 * We need a way to either ignore barrier acks for barriers sent before a device
-	 * was attached, or a way to wait for all pending barrier acks to come in.
-	 * As barriers are counted per resource,
-	 * we'd need to suspend io on all devices of a resource.
-	 */
-	wait_event(device->misc_wait, !atomic_read(&device->ap_pending_cnt) || drbd_suspended(device));
-	/* and for any other previously queued work */
-	drbd_flush_workqueue(&connection->sender_work);
-
-	rv = _drbd_request_state(device, NS(disk, D_ATTACHING), CS_VERBOSE);
+	drbd_suspend_io(device, READ_AND_WRITE);
+	wait_event(resource->barrier_wait, !barrier_pending(resource));
+	for_each_peer_device(peer_device, device)
+		wait_event(device->misc_wait,
+			   (!atomic_read(&peer_device->ap_pending_cnt) ||
+			    drbd_suspended(device)));
+	/* and for other previously queued resource work */
+	drbd_flush_workqueue(&resource->work);
+
+	rv = stable_state_change(resource,
+		change_disk_state(device, D_ATTACHING, CS_VERBOSE | CS_SERIALIZE, "attach", NULL));
 	retcode = (enum drbd_ret_code)rv;
+	if (rv >= SS_SUCCESS)
+		update_resource_dagtag(resource, nbc);
 	drbd_resume_io(device);
 	if (rv < SS_SUCCESS)
 		goto fail;
@@ -1923,20 +3400,97 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	if (!get_ldev_if_state(device, D_ATTACHING))
 		goto force_diskless;
 
-	if (!device->bitmap) {
-		if (drbd_bm_init(device)) {
-			retcode = ERR_NOMEM;
+	drbd_info(device, "Maximum number of peer devices = %u\n", nbc->md.max_peers);
+
+	mutex_lock(&resource->conf_update);
+	have_conf_update = true;
+
+	/* Make sure the local node id matches or is unassigned */
+	if (nbc->md.node_id != -1 && nbc->md.node_id != resource->res_opts.node_id) {
+		drbd_err_and_skb_info(&adm_ctx, "Local node id %d differs from local "
+			 "node id %d on device\n",
+			 resource->res_opts.node_id,
+			 nbc->md.node_id);
+		retcode = ERR_INVALID_REQUEST;
+		goto force_diskless_dec;
+	}
+
+	/* Make sure no bitmap slot has our own node id.
+	 * If we are operating in "drbd 8 compatibility mode", the node ID is
+	 * not yet initialized at this point, so just ignore this check.
+	 */
+	if (resource->res_opts.node_id != -1 &&
+	    nbc->md.peers[resource->res_opts.node_id].bitmap_index != -1) {
+		drbd_err_and_skb_info(&adm_ctx, "There is a bitmap for my own node id (%d)\n",
+			 resource->res_opts.node_id);
+		retcode = ERR_INVALID_REQUEST;
+		goto force_diskless_dec;
+	}
+
+	/* Make sure we have a bitmap slot for each peer id */
+	for_each_peer_device(peer_device, device) {
+		struct drbd_connection *connection = peer_device->connection;
+		int bitmap_index;
+
+		if (peer_device->bitmap_index != -1) {
+			drbd_err_and_skb_info(&adm_ctx,
+					"ASSERTION FAILED bitmap_index %d during attach, expected -1\n",
+					peer_device->bitmap_index);
+		}
+
+		bitmap_index = nbc->md.peers[connection->peer_node_id].bitmap_index;
+		if (want_bitmap(peer_device)) {
+			if (bitmap_index != -1)
+				peer_device->bitmap_index = bitmap_index;
+			else
+				slots_needed++;
+		} else if (bitmap_index != -1) {
+			/* Pretend in core that there is not bitmap for that peer,
+			   in the on disk meta-data we keep it until it is de-allocated
+			   with forget-peer */
+			nbc->md.peers[connection->peer_node_id].flags &= ~MDF_HAVE_BITMAP;
+		}
+	}
+	if (slots_needed) {
+		int slots_available = nbc->md.max_peers - used_bitmap_slots(nbc);
+
+		if (slots_needed > slots_available) {
+			drbd_err_and_skb_info(&adm_ctx, "Not enough free bitmap "
+				 "slots (available=%d, needed=%d)\n",
+				 slots_available,
+				 slots_needed);
+			retcode = ERR_INVALID_REQUEST;
 			goto force_diskless_dec;
 		}
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->bitmap_index != -1 || !want_bitmap(peer_device))
+				continue;
+
+			err = allocate_bitmap_index(peer_device, nbc);
+			if (err) {
+				retcode = ERR_INVALID_REQUEST;
+				goto force_diskless_dec;
+			}
+		}
 	}
 
-	if (device->state.pdsk != D_UP_TO_DATE && device->ed_uuid &&
-	    (device->state.role == R_PRIMARY || device->state.peer == R_PRIMARY) &&
-            (device->ed_uuid & ~((u64)1)) != (nbc->md.uuid[UI_CURRENT] & ~((u64)1))) {
-		drbd_err(device, "Can only attach to data with current UUID=%016llX\n",
-		    (unsigned long long)device->ed_uuid);
-		retcode = ERR_DATA_NOT_CURRENT;
-		goto force_diskless_dec;
+	/* Assign the local node id (if not assigned already) */
+	nbc->md.node_id = resource->res_opts.node_id;
+
+	if (resource->role[NOW] == R_PRIMARY && device->exposed_data_uuid &&
+	    (device->exposed_data_uuid & ~UUID_PRIMARY) !=
+	    (nbc->md.current_uuid & ~UUID_PRIMARY)) {
+		int data_present = false;
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->disk_state[NOW] == D_UP_TO_DATE)
+				data_present = true;
+		}
+		if (!data_present) {
+			drbd_err_and_skb_info(&adm_ctx, "Can only attach to data with current UUID=%016llX\n",
+				 (unsigned long long)device->exposed_data_uuid);
+			retcode = ERR_DATA_NOT_CURRENT;
+			goto force_diskless_dec;
+		}
 	}
 
 	/* Since we are diskless, fix the activity log first... */
@@ -1945,26 +3499,30 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		goto force_diskless_dec;
 	}
 
-	/* Prevent shrinking of consistent devices ! */
-	{
-	unsigned long long nsz = drbd_new_dev_size(device, nbc, nbc->disk_conf->disk_size, 0);
-	unsigned long long eff = nbc->md.la_size_sect;
-	if (drbd_md_test_flag(nbc, MDF_CONSISTENT) && nsz < eff) {
-		if (nsz == nbc->disk_conf->disk_size) {
-			drbd_warn(device, "truncating a consistent device during attach (%llu < %llu)\n", nsz, eff);
-		} else {
-			drbd_warn(device, "refusing to truncate a consistent device (%llu < %llu)\n", nsz, eff);
-			drbd_msg_sprintf_info(adm_ctx.reply_skb,
-				"To-be-attached device has last effective > current size, and is consistent\n"
-				"(%llu > %llu sectors). Refusing to attach.", eff, nsz);
-			retcode = ERR_IMPLICIT_SHRINK;
+	/* Point of no return reached.
+	 * Devices and memory are no longer released by error cleanup below.
+	 * now device takes over responsibility, and the state engine should
+	 * clean it up somewhere.  */
+	D_ASSERT(device, device->ldev == NULL);
+	device->ldev = nbc;
+	nbc = NULL;
+	new_disk_conf = NULL;
+
+	if (drbd_md_dax_active(device->ldev)) {
+		/* The on-disk activity log is always initialized with the
+		 * non-pmem format. We have now decided to access it using
+		 * dax, so re-initialize it appropriately. */
+		if (drbd_dax_al_initialize(device)) {
+			retcode = ERR_IO_MD_DISK;
 			goto force_diskless_dec;
 		}
 	}
-	}
+
+	mutex_unlock(&resource->conf_update);
+	have_conf_update = false;
 
 	lock_all_resources();
-	retcode = drbd_resync_after_valid(device, new_disk_conf->resync_after);
+	retcode = drbd_resync_after_valid(device, device->ldev->disk_conf->resync_after);
 	if (retcode != NO_ERROR) {
 		unlock_all_resources();
 		goto force_diskless_dec;
@@ -1972,43 +3530,53 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 
 	/* Reset the "barriers don't work" bits here, then force meta data to
 	 * be written, to ensure we determine if barriers are supported. */
-	if (new_disk_conf->md_flushes)
+	if (device->ldev->disk_conf->md_flushes)
 		clear_bit(MD_NO_FUA, &device->flags);
 	else
 		set_bit(MD_NO_FUA, &device->flags);
 
-	/* Point of no return reached.
-	 * Devices and memory are no longer released by error cleanup below.
-	 * now device takes over responsibility, and the state engine should
-	 * clean it up somewhere.  */
-	D_ASSERT(device, device->ldev == NULL);
-	device->ldev = nbc;
-	device->resync = resync_lru;
-	device->rs_plan_s = new_plan;
-	nbc = NULL;
-	resync_lru = NULL;
-	new_disk_conf = NULL;
-	new_plan = NULL;
-
 	drbd_resync_after_changed(device);
-	drbd_bump_write_ordering(device->resource, device->ldev, WO_BDEV_FLUSH);
+	drbd_bump_write_ordering(resource, device->ldev, WO_BIO_BARRIER);
 	unlock_all_resources();
 
-	if (drbd_md_test_flag(device->ldev, MDF_CRASHED_PRIMARY))
+	/* Prevent shrinking of consistent devices ! */
+	{
+	unsigned long long nsz = drbd_new_dev_size(device, 0, device->ldev->disk_conf->disk_size, 0);
+	unsigned long long eff = device->ldev->md.effective_size;
+	if (drbd_md_test_flag(device->ldev, MDF_CONSISTENT) && nsz < eff) {
+		if (nsz == device->ldev->disk_conf->disk_size) {
+			drbd_warn(device, "truncating a consistent device during attach (%llu < %llu)\n", nsz, eff);
+		} else {
+			drbd_warn(device, "refusing to truncate a consistent device (%llu < %llu)\n", nsz, eff);
+			drbd_msg_sprintf_info(adm_ctx.reply_skb,
+				"To-be-attached device has last effective > current size, and is consistent\n"
+				"(%llu > %llu sectors). Refusing to attach.", eff, nsz);
+			retcode = ERR_IMPLICIT_SHRINK;
+			goto force_diskless_dec;
+		}
+	}
+	}
+
+	if (drbd_md_test_flag(device->ldev, MDF_HAVE_QUORUM) &&
+	    drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE) &&
+	    device->ldev->md.prev_members == NODE_MASK(resource->res_opts.node_id))
+		set_bit(RESTORE_QUORUM, &device->flags);
+
+	if (drbd_md_test_flag(device->ldev, MDF_CRASHED_PRIMARY) &&
+	    !(resource->role[NOW] == R_PRIMARY && resource->susp_nod[NOW]) &&
+	    !device->exposed_data_uuid && !test_bit(NEW_CUR_UUID, &device->flags))
 		set_bit(CRASHED_PRIMARY, &device->flags);
 	else
 		clear_bit(CRASHED_PRIMARY, &device->flags);
 
-	if (drbd_md_test_flag(device->ldev, MDF_PRIMARY_IND) &&
-	    !(device->state.role == R_PRIMARY && device->resource->susp_nod))
-		set_bit(CRASHED_PRIMARY, &device->flags);
+	if (drbd_md_test_flag(device->ldev, MDF_PRIMARY_LOST_QUORUM) &&
+	    !device->have_quorum[NOW])
+		set_bit(PRIMARY_LOST_QUORUM, &device->flags);
 
-	device->send_cnt = 0;
-	device->recv_cnt = 0;
 	device->read_cnt = 0;
 	device->writ_cnt = 0;
 
-	drbd_reconsider_queue_parameters(device, device->ldev, NULL);
+	drbd_reconsider_queue_parameters(device, device->ldev);
 
 	/* If I am currently not R_PRIMARY,
 	 * but meta data primary indicator is set,
@@ -2024,147 +3592,159 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	 * so we can automatically recover from a crash of a
 	 * degraded but active "cluster" after a certain timeout.
 	 */
-	clear_bit(USE_DEGR_WFC_T, &device->flags);
-	if (device->state.role != R_PRIMARY &&
-	     drbd_md_test_flag(device->ldev, MDF_PRIMARY_IND) &&
-	    !drbd_md_test_flag(device->ldev, MDF_CONNECTED_IND))
-		set_bit(USE_DEGR_WFC_T, &device->flags);
-
-	dd = drbd_determine_dev_size(device, 0, NULL);
-	if (dd <= DS_ERROR) {
+	for_each_peer_device(peer_device, device) {
+		clear_bit(USE_DEGR_WFC_T, &peer_device->flags);
+		if (resource->role[NOW] != R_PRIMARY &&
+		    drbd_md_test_flag(device->ldev, MDF_PRIMARY_IND) &&
+		    !drbd_md_test_peer_flag(peer_device, MDF_PEER_CONNECTED))
+			set_bit(USE_DEGR_WFC_T, &peer_device->flags);
+	}
+
+	/*
+	 * If we are attaching to a disk that is marked as being up-to-date,
+	 * then we do not need to set the bitmap bits.
+	 */
+	dd = drbd_determine_dev_size(device, 0,
+			disk_state_from_md(device) == D_UP_TO_DATE ? DDSF_NO_RESYNC : 0,
+			NULL);
+	if (dd == DS_ERROR) {
 		retcode = ERR_NOMEM_BITMAP;
 		goto force_diskless_dec;
-	} else if (dd == DS_GREW)
-		set_bit(RESYNC_AFTER_NEG, &device->flags);
-
-	if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC) ||
-	    (test_bit(CRASHED_PRIMARY, &device->flags) &&
-	     drbd_md_test_flag(device->ldev, MDF_AL_DISABLED))) {
-		drbd_info(device, "Assuming that all blocks are out of sync "
-		     "(aka FullSync)\n");
-		if (drbd_bitmap_io(device, &drbd_bmio_set_n_write,
-			"set_n_write from attaching", BM_LOCKED_MASK,
-			NULL)) {
-			retcode = ERR_IO_MD_DISK;
-			goto force_diskless_dec;
-		}
-	} else {
-		if (drbd_bitmap_io(device, &drbd_bm_read,
-			"read from attaching", BM_LOCKED_MASK,
-			NULL)) {
-			retcode = ERR_IO_MD_DISK;
-			goto force_diskless_dec;
-		}
+	} else if (dd == DS_GREW) {
+		for_each_peer_device(peer_device, device)
+			set_bit(RESYNC_AFTER_NEG, &peer_device->flags);
 	}
 
-	if (_drbd_bm_total_weight(device) == drbd_bm_bits(device))
-		drbd_suspend_al(device); /* IO is still suspended here... */
-
-	spin_lock_irq(&device->resource->req_lock);
-	os = drbd_read_state(device);
-	ns = os;
-	/* If MDF_CONSISTENT is not set go into inconsistent state,
-	   otherwise investigate MDF_WasUpToDate...
-	   If MDF_WAS_UP_TO_DATE is not set go into D_OUTDATED disk state,
-	   otherwise into D_CONSISTENT state.
-	*/
-	if (drbd_md_test_flag(device->ldev, MDF_CONSISTENT)) {
-		if (drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE))
-			ns.disk = D_CONSISTENT;
-		else
-			ns.disk = D_OUTDATED;
-	} else {
-		ns.disk = D_INCONSISTENT;
+	err = drbd_bitmap_io(device, &drbd_bm_read,
+			     "read from attaching", BM_LOCK_ALL,
+			     NULL);
+	if (err) {
+		retcode = ERR_IO_MD_DISK;
+		goto force_diskless_dec;
 	}
 
-	if (drbd_md_test_flag(device->ldev, MDF_PEER_OUT_DATED))
-		ns.pdsk = D_OUTDATED;
-
-	rcu_read_lock();
-	if (ns.disk == D_CONSISTENT &&
-	    (ns.pdsk == D_OUTDATED || rcu_dereference(device->ldev->disk_conf)->fencing == FP_DONT_CARE))
-		ns.disk = D_UP_TO_DATE;
-
-	/* All tests on MDF_PRIMARY_IND, MDF_CONNECTED_IND,
-	   MDF_CONSISTENT and MDF_WAS_UP_TO_DATE must happen before
-	   this point, because drbd_request_state() modifies these
-	   flags. */
+	for_each_peer_device(peer_device, device) {
+		if ((test_bit(CRASHED_PRIMARY, &device->flags) &&
+		     drbd_md_test_flag(device->ldev, MDF_AL_DISABLED)) ||
+		    drbd_md_test_peer_flag(peer_device, MDF_PEER_FULL_SYNC)) {
+			drbd_info(peer_device, "Assuming that all blocks are out of sync "
+				  "(aka FullSync)\n");
+			if (drbd_bitmap_io(device, &drbd_bmio_set_n_write,
+				"set_n_write from attaching", BM_LOCK_ALL,
+				peer_device)) {
+				retcode = ERR_IO_MD_DISK;
+				goto force_diskless_dec;
+			}
+		}
+	}
 
-	if (rcu_dereference(device->ldev->disk_conf)->al_updates)
-		device->ldev->md.flags &= ~MDF_AL_DISABLED;
-	else
-		device->ldev->md.flags |= MDF_AL_DISABLED;
+	drbd_try_suspend_al(device); /* IO is still suspended here... */
 
-	rcu_read_unlock();
+	drbd_update_mdf_al_disabled(device, NOW);
 
-	/* In case we are C_CONNECTED postpone any decision on the new disk
-	   state after the negotiation phase. */
-	if (device->state.conn == C_CONNECTED) {
-		device->new_state_tmp.i = ns.i;
-		ns.i = os.i;
-		ns.disk = D_NEGOTIATING;
+	/* change_disk_state uses disk_state_from_md(device); in case D_NEGOTIATING not
+	   necessary, and falls back to a local state change */
+	rv = stable_state_change(resource, change_disk_state(device,
+				D_NEGOTIATING, CS_VERBOSE | CS_SERIALIZE, "attach", NULL));
 
-		/* We expect to receive up-to-date UUIDs soon.
-		   To avoid a race in receive_state, free p_uuid while
-		   holding req_lock. I.e. atomic with the state change */
-		kfree(device->p_uuid);
-		device->p_uuid = NULL;
+	if (rv < SS_SUCCESS) {
+		if (rv == SS_CW_FAILED_BY_PEER)
+			drbd_msg_put_info(adm_ctx.reply_skb,
+				"Probably this node is marked as intentional diskless on a peer");
+		retcode = rv;
+		goto force_diskless_dec;
 	}
 
-	rv = _drbd_set_state(device, ns, CS_VERBOSE, NULL);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	if (rv < SS_SUCCESS)
-		goto force_diskless_dec;
+	device->device_conf.intentional_diskless = false; /* just in case... */
 
 	mod_timer(&device->request_timer, jiffies + HZ);
 
-	if (device->state.role == R_PRIMARY)
-		device->ldev->md.uuid[UI_CURRENT] |=  (u64)1;
+	if (resource->role[NOW] == R_PRIMARY
+	&&  device->ldev->md.current_uuid != UUID_JUST_CREATED)
+		device->ldev->md.current_uuid |= UUID_PRIMARY;
 	else
-		device->ldev->md.uuid[UI_CURRENT] &= ~(u64)1;
+		device->ldev->md.current_uuid &= ~UUID_PRIMARY;
 
-	drbd_md_mark_dirty(device);
 	drbd_md_sync(device);
 
 	kobject_uevent(&disk_to_dev(device->vdisk)->kobj, KOBJ_CHANGE);
 	put_ldev(device);
-	conn_reconfig_done(connection);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
+	mutex_unlock(&resource->adm_mutex);
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 
  force_diskless_dec:
 	put_ldev(device);
  force_diskless:
-	drbd_force_state(device, NS(disk, D_DISKLESS));
-	drbd_md_sync(device);
+	change_disk_state(device, D_DISKLESS, CS_HARD, "attach", NULL);
  fail:
-	conn_reconfig_done(connection);
-	if (nbc) {
-		close_backing_dev(device, nbc->f_md_bdev,
-			  nbc->md_bdev != nbc->backing_bdev);
-		close_backing_dev(device, nbc->backing_bdev_file, true);
-		kfree(nbc);
-	}
-	kfree(new_disk_conf);
-	lc_destroy(resync_lru);
-	kfree(new_plan);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
- finish:
+	drbd_bm_free(device);
+	mutex_unlock_cond(&resource->conf_update, &have_conf_update);
+	drbd_backing_dev_free(device, nbc);
+	mutex_unlock(&resource->adm_mutex);
+ out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static int adm_detach(struct drbd_device *device, int force)
+static enum drbd_disk_state get_disk_state(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	enum drbd_disk_state disk_state;
+
+	read_lock_irq(&resource->state_rwlock);
+	disk_state = device->disk_state[NOW];
+	read_unlock_irq(&resource->state_rwlock);
+	return disk_state;
+}
+
+static int adm_detach(struct drbd_device *device, bool force, bool intentional_diskless,
+		      const char *tag, struct sk_buff *reply_skb)
 {
+	const char *err_str = NULL;
+	int ret, retcode;
+
+	device->device_conf.intentional_diskless = intentional_diskless;
 	if (force) {
 		set_bit(FORCE_DETACH, &device->flags);
-		drbd_force_state(device, NS(disk, D_FAILED));
-		return SS_SUCCESS;
+		change_disk_state(device, D_DETACHING, CS_HARD, tag, NULL);
+		retcode = SS_SUCCESS;
+		goto out;
 	}
 
-	return drbd_request_detach_interruptible(device);
+	drbd_suspend_io(device, READ_AND_WRITE); /* so no-one is stuck in drbd_al_begin_io */
+	retcode = stable_state_change(device->resource,
+		change_disk_state(device, D_DETACHING,
+			CS_VERBOSE | CS_SERIALIZE, tag, &err_str));
+	/*
+	 * D_DETACHING will transition to DISKLESS.
+	 * I did not use CS_WAIT_COMPLETE above since that would deadlock on a backing device that
+	 * does not finish the I/O requests from writing to internal meta-data.  Instead, I
+	 * explicitly flush the worker queue here to ensure w_after_state_change() is completed.
+	 */
+	drbd_flush_workqueue_interruptible(device);
+
+	drbd_resume_io(device);
+	ret = wait_event_interruptible(device->misc_wait,
+			get_disk_state(device) != D_DETACHING);
+	if (retcode >= SS_SUCCESS) {
+		/* wait for completion of drbd_ldev_destroy() */
+		wait_event_interruptible(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags));
+		drbd_cleanup_device(device);
+	} else {
+		device->device_conf.intentional_diskless = false;
+	}
+	if (retcode == SS_IS_DISKLESS)
+		retcode = SS_NOTHING_TO_DO;
+	if (ret)
+		retcode = ERR_INTR;
+out:
+	if (err_str) {
+		drbd_msg_put_info(reply_skb, err_str);
+		kfree(err_str);
+	} else if (retcode == SS_NO_UP_TO_DATE_DISK)
+		put_device_opener_info(device, reply_skb);
+	return retcode;
 }
 
 /* Detaching the disk is a process in multiple stages.  First we need to lock
@@ -2172,7 +3752,7 @@ static int adm_detach(struct drbd_device *device, int force)
  * Then we transition to D_DISKLESS, and wait for put_ldev() to return all
  * internal references as well.
  * Only then we have finally detached. */
-int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
@@ -2182,8 +3762,6 @@ int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info)
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
 
 	if (info->attrs[DRBD_NLA_DETACH_PARMS]) {
 		err = detach_parms_from_attrs(&parms, info);
@@ -2194,9 +3772,14 @@ int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	retcode = adm_detach(adm_ctx.device, parms.force_detach);
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
+	retcode = (enum drbd_ret_code)adm_detach(adm_ctx.device, parms.force_detach,
+			parms.intentional_diskless_detach, "detach", adm_ctx.reply_skb);
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
+
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
@@ -2210,11 +3793,10 @@ static bool conn_resync_running(struct drbd_connection *connection)
 
 	rcu_read_lock();
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		if (device->state.conn == C_SYNC_SOURCE ||
-		    device->state.conn == C_SYNC_TARGET ||
-		    device->state.conn == C_PAUSED_SYNC_S ||
-		    device->state.conn == C_PAUSED_SYNC_T) {
+		if (peer_device->repl_state[NOW] == L_SYNC_SOURCE ||
+		    peer_device->repl_state[NOW] == L_SYNC_TARGET ||
+		    peer_device->repl_state[NOW] == L_PAUSED_SYNC_S ||
+		    peer_device->repl_state[NOW] == L_PAUSED_SYNC_T) {
 			rv = true;
 			break;
 		}
@@ -2232,9 +3814,8 @@ static bool conn_ov_running(struct drbd_connection *connection)
 
 	rcu_read_lock();
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		if (device->state.conn == C_VERIFY_S ||
-		    device->state.conn == C_VERIFY_T) {
+		if (peer_device->repl_state[NOW] == L_VERIFY_S ||
+		    peer_device->repl_state[NOW] == L_VERIFY_T) {
 			rv = true;
 			break;
 		}
@@ -2247,10 +3828,7 @@ static bool conn_ov_running(struct drbd_connection *connection)
 static enum drbd_ret_code
 _check_net_options(struct drbd_connection *connection, struct net_conf *old_net_conf, struct net_conf *new_net_conf)
 {
-	struct drbd_peer_device *peer_device;
-	int i;
-
-	if (old_net_conf && connection->cstate == C_WF_REPORT_PARAMS && connection->agreed_pro_version < 100) {
+	if (old_net_conf && connection->cstate[NOW] == C_CONNECTED && connection->agreed_pro_version < 100) {
 		if (new_net_conf->wire_protocol != old_net_conf->wire_protocol)
 			return ERR_NEED_APV_100;
 
@@ -2262,27 +3840,20 @@ _check_net_options(struct drbd_connection *connection, struct net_conf *old_net_
 	}
 
 	if (!new_net_conf->two_primaries &&
-	    conn_highest_role(connection) == R_PRIMARY &&
-	    conn_highest_peer(connection) == R_PRIMARY)
+	    connection->resource->role[NOW] == R_PRIMARY &&
+	    connection->peer_role[NOW] == R_PRIMARY)
 		return ERR_NEED_ALLOW_TWO_PRI;
 
 	if (new_net_conf->two_primaries &&
 	    (new_net_conf->wire_protocol != DRBD_PROT_C))
 		return ERR_NOT_PROTO_C;
 
-	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-		struct drbd_device *device = peer_device->device;
-		if (get_ldev(device)) {
-			enum drbd_fencing_p fp = rcu_dereference(device->ldev->disk_conf)->fencing;
-			put_ldev(device);
-			if (new_net_conf->wire_protocol == DRBD_PROT_A && fp == FP_STONITH)
-				return ERR_STONITH_AND_PROT_A;
-		}
-		if (device->state.role == R_PRIMARY && new_net_conf->discard_my_data)
-			return ERR_DISCARD_IMPOSSIBLE;
-	}
+	if (new_net_conf->wire_protocol == DRBD_PROT_A &&
+	    new_net_conf->fencing_policy == FP_STONITH)
+		return ERR_STONITH_AND_PROT_A;
 
-	if (new_net_conf->on_congestion != OC_BLOCK && new_net_conf->wire_protocol != DRBD_PROT_A)
+	if (new_net_conf->on_congestion != OC_BLOCK &&
+	    new_net_conf->wire_protocol != DRBD_PROT_A)
 		return ERR_CONG_NOT_PROTO_A;
 
 	return NO_ERROR;
@@ -2292,22 +3863,11 @@ static enum drbd_ret_code
 check_net_options(struct drbd_connection *connection, struct net_conf *new_net_conf)
 {
 	enum drbd_ret_code rv;
-	struct drbd_peer_device *peer_device;
-	int i;
 
 	rcu_read_lock();
-	rv = _check_net_options(connection, rcu_dereference(connection->net_conf), new_net_conf);
+	rv = _check_net_options(connection, rcu_dereference(connection->transport.net_conf), new_net_conf);
 	rcu_read_unlock();
 
-	/* connection->peer_devices protected by genl_lock() here */
-	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-		struct drbd_device *device = peer_device->device;
-		if (!device->bitmap) {
-			if (drbd_bm_init(device))
-				return ERR_NOMEM;
-		}
-	}
-
 	return rv;
 }
 
@@ -2318,48 +3878,88 @@ struct crypto {
 	struct crypto_shash *integrity_tfm;
 };
 
+static bool needs_key(struct crypto_shash *h)
+{
+	return h && (crypto_shash_get_flags(h) & CRYPTO_TFM_NEED_KEY);
+}
+
+/**
+ * alloc_shash() - Allocate a keyed or unkeyed shash algorithm
+ * @tfm: Destination crypto_shash
+ * @tfm_name: Which algorithm to use
+ * @type: The functionality that the hash is used for
+ * @must_unkeyed: If set, a check is included which ensures that the algorithm
+ * 	     does not require a key
+ * @reply_skb: for sending detailed error description to user-space
+ */
 static int
-alloc_shash(struct crypto_shash **tfm, char *tfm_name, int err_alg)
+alloc_shash(struct crypto_shash **tfm, char *tfm_name, const char *type, bool must_unkeyed,
+	    struct sk_buff *reply_skb)
 {
 	if (!tfm_name[0])
-		return NO_ERROR;
+		return 0;
 
 	*tfm = crypto_alloc_shash(tfm_name, 0, 0);
 	if (IS_ERR(*tfm)) {
+		drbd_msg_sprintf_info(reply_skb, "failed to allocate %s for %s\n", tfm_name, type);
 		*tfm = NULL;
-		return err_alg;
+		return -EINVAL;
 	}
 
-	return NO_ERROR;
+	if (must_unkeyed && needs_key(*tfm)) {
+		drbd_msg_sprintf_info(reply_skb,
+				      "may not use %s for %s. It requires an unkeyed algorithm\n",
+				      tfm_name, type);
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static enum drbd_ret_code
-alloc_crypto(struct crypto *crypto, struct net_conf *new_net_conf)
+alloc_crypto(struct crypto *crypto, struct net_conf *new_net_conf, struct sk_buff *reply_skb)
 {
 	char hmac_name[CRYPTO_MAX_ALG_NAME];
-	enum drbd_ret_code rv;
+	int digest_size = 0;
+	int err;
+
+	err = alloc_shash(&crypto->csums_tfm, new_net_conf->csums_alg,
+			  "csums", true, reply_skb);
+	if (err)
+		return ERR_CSUMS_ALG;
+
+	err = alloc_shash(&crypto->verify_tfm, new_net_conf->verify_alg,
+			  "verify", true, reply_skb);
+	if (err)
+		return ERR_VERIFY_ALG;
+
+	err = alloc_shash(&crypto->integrity_tfm, new_net_conf->integrity_alg,
+			  "integrity", true, reply_skb);
+	if (err)
+		return ERR_INTEGRITY_ALG;
+
+	if (crypto->integrity_tfm) {
+		const int max_digest_size = sizeof(((struct drbd_connection *)0)->scratch_buffer.d.before);
+		digest_size = crypto_shash_digestsize(crypto->integrity_tfm);
+		if (digest_size > max_digest_size) {
+			drbd_msg_sprintf_info(reply_skb,
+				"we currently support only digest sizes <= %d bits, but digest size of %s is %d bits\n",
+				max_digest_size * 8, new_net_conf->integrity_alg, digest_size * 8);
+			return ERR_INTEGRITY_ALG;
+		}
+	}
 
-	rv = alloc_shash(&crypto->csums_tfm, new_net_conf->csums_alg,
-			 ERR_CSUMS_ALG);
-	if (rv != NO_ERROR)
-		return rv;
-	rv = alloc_shash(&crypto->verify_tfm, new_net_conf->verify_alg,
-			 ERR_VERIFY_ALG);
-	if (rv != NO_ERROR)
-		return rv;
-	rv = alloc_shash(&crypto->integrity_tfm, new_net_conf->integrity_alg,
-			 ERR_INTEGRITY_ALG);
-	if (rv != NO_ERROR)
-		return rv;
 	if (new_net_conf->cram_hmac_alg[0] != 0) {
 		snprintf(hmac_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)",
 			 new_net_conf->cram_hmac_alg);
 
-		rv = alloc_shash(&crypto->cram_hmac_tfm, hmac_name,
-				 ERR_AUTH_ALG);
+		err = alloc_shash(&crypto->cram_hmac_tfm, hmac_name,
+				  "hmac", false, reply_skb);
+		if (err)
+			return ERR_AUTH_ALG;
 	}
 
-	return rv;
+	return NO_ERROR;
 }
 
 static void free_crypto(struct crypto *crypto)
@@ -2370,11 +3970,12 @@ static void free_crypto(struct crypto *crypto)
 	crypto_free_shash(crypto->verify_tfm);
 }
 
-int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
 	struct drbd_connection *connection;
+	struct drbd_transport *transport;
 	struct net_conf *old_net_conf, *new_net_conf = NULL;
 	int err;
 	int ovr; /* online verify running */
@@ -2384,11 +3985,12 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
 
 	connection = adm_ctx.connection;
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
 
 	new_net_conf = kzalloc_obj(struct net_conf);
 	if (!new_net_conf) {
@@ -2396,11 +3998,12 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 		goto out;
 	}
 
-	conn_reconfig_start(connection);
+	drbd_flush_workqueue(&connection->sender_work);
 
-	mutex_lock(&connection->data.mutex);
 	mutex_lock(&connection->resource->conf_update);
-	old_net_conf = connection->net_conf;
+	mutex_lock(&connection->mutex[DATA_STREAM]);
+	transport = &connection->transport;
+	old_net_conf = transport->net_conf;
 
 	if (!old_net_conf) {
 		drbd_msg_put_info(adm_ctx.reply_skb, "net conf missing, try connect");
@@ -2412,6 +4015,12 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 	if (should_set_defaults(info))
 		set_net_conf_defaults(new_net_conf);
 
+	/* The transport_name is immutable taking precedence over set_net_conf_defaults() */
+	memcpy(new_net_conf->transport_name, old_net_conf->transport_name,
+	       old_net_conf->transport_name_len);
+	new_net_conf->transport_name_len = old_net_conf->transport_name_len;
+	new_net_conf->load_balance_paths = old_net_conf->load_balance_paths;
+
 	err = net_conf_from_attrs_for_change(new_net_conf, info);
 	if (err && err != -ENOMSG) {
 		retcode = ERR_MANDATORY_TAG;
@@ -2437,11 +4046,22 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 		goto fail;
 	}
 
-	retcode = alloc_crypto(&crypto, new_net_conf);
+	retcode = alloc_crypto(&crypto, new_net_conf, adm_ctx.reply_skb);
 	if (retcode != NO_ERROR)
 		goto fail;
 
-	rcu_assign_pointer(connection->net_conf, new_net_conf);
+	/* Call before updating net_conf in case the transport needs to compare
+	 * old and new configurations. */
+	err = transport->class->ops.net_conf_change(transport, new_net_conf);
+	if (err) {
+		drbd_msg_sprintf_info(adm_ctx.reply_skb, "transport net_conf_change failed: %d",
+				      err);
+		retcode = ERR_INVALID_REQUEST;
+		goto fail;
+	}
+
+	rcu_assign_pointer(transport->net_conf, new_net_conf);
+	connection->fencing_policy = new_net_conf->fencing_policy;
 
 	if (!rsr) {
 		crypto_free_shash(connection->csums_tfm);
@@ -2456,18 +4076,18 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 
 	crypto_free_shash(connection->integrity_tfm);
 	connection->integrity_tfm = crypto.integrity_tfm;
-	if (connection->cstate >= C_WF_REPORT_PARAMS && connection->agreed_pro_version >= 100)
+	if (connection->cstate[NOW] >= C_CONNECTED && connection->agreed_pro_version >= 100)
 		/* Do this without trying to take connection->data.mutex again.  */
 		__drbd_send_protocol(connection, P_PROTOCOL_UPDATE);
 
 	crypto_free_shash(connection->cram_hmac_tfm);
 	connection->cram_hmac_tfm = crypto.cram_hmac_tfm;
 
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
 	mutex_unlock(&connection->resource->conf_update);
-	mutex_unlock(&connection->data.mutex);
 	kvfree_rcu_mightsleep(old_net_conf);
 
-	if (connection->cstate >= C_WF_REPORT_PARAMS) {
+	if (connection->cstate[NOW] >= C_CONNECTED) {
 		struct drbd_peer_device *peer_device;
 		int vnr;
 
@@ -2475,277 +4095,1037 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 			drbd_send_sync_param(peer_device);
 	}
 
-	goto done;
+	goto out;
 
  fail:
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
 	mutex_unlock(&connection->resource->conf_update);
-	mutex_unlock(&connection->data.mutex);
 	free_crypto(&crypto);
 	kfree(new_net_conf);
- done:
-	conn_reconfig_done(connection);
  out:
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
- finish:
+ out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static void connection_to_info(struct connection_info *info,
-			       struct drbd_connection *connection)
+static int adjust_resync_fifo(struct drbd_peer_device *peer_device,
+			      struct peer_device_conf *conf,
+			      struct fifo_buffer **pp_old_plan)
 {
-	info->conn_connection_state = connection->cstate;
-	info->conn_role = conn_highest_peer(connection);
-}
+	struct fifo_buffer *old_plan, *new_plan = NULL;
+	unsigned int fifo_size;
 
-static void peer_device_to_info(struct peer_device_info *info,
-				struct drbd_peer_device *peer_device)
-{
-	struct drbd_device *device = peer_device->device;
+	fifo_size = (conf->c_plan_ahead * 10 * RS_MAKE_REQS_INTV) / HZ;
+
+	old_plan = rcu_dereference_protected(peer_device->rs_plan_s,
+			     lockdep_is_held(&peer_device->connection->resource->conf_update));
+	if (!old_plan || fifo_size != old_plan->size) {
+		new_plan = fifo_alloc(fifo_size);
+		if (!new_plan) {
+			drbd_err(peer_device, "kmalloc of fifo_buffer failed");
+			return -ENOMEM;
+		}
+		rcu_assign_pointer(peer_device->rs_plan_s, new_plan);
+		if (pp_old_plan)
+			*pp_old_plan = old_plan;
+	}
 
-	info->peer_repl_state =
-		max_t(enum drbd_conns, C_WF_REPORT_PARAMS, device->state.conn);
-	info->peer_disk_state = device->state.pdsk;
-	info->peer_resync_susp_user = device->state.user_isp;
-	info->peer_resync_susp_peer = device->state.peer_isp;
-	info->peer_resync_susp_dependency = device->state.aftr_isp;
+	return 0;
 }
 
-int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_peer_device_opts(struct sk_buff *skb, struct genl_info *info)
 {
-	struct connection_info connection_info;
-	enum drbd_notification_type flags;
-	unsigned int peer_devices = 0;
 	struct drbd_config_context adm_ctx;
-	struct drbd_peer_device *peer_device;
-	struct net_conf *old_net_conf, *new_net_conf = NULL;
-	struct crypto crypto = { };
-	struct drbd_resource *resource;
-	struct drbd_connection *connection;
 	enum drbd_ret_code retcode;
-	enum drbd_state_rv rv;
-	int i;
+	struct drbd_peer_device *peer_device;
+	struct peer_device_conf *old_peer_device_conf, *new_peer_device_conf = NULL;
+	struct fifo_buffer *old_plan = NULL;
+	struct drbd_device *device;
+	bool notify = false;
 	int err;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
-
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
-	if (!(adm_ctx.my_addr && adm_ctx.peer_addr)) {
-		drbd_msg_put_info(adm_ctx.reply_skb, "connection endpoint(s) missing");
-		retcode = ERR_INVALID_REQUEST;
-		goto out;
+
+	peer_device = adm_ctx.peer_device;
+	device = peer_device->device;
+
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
 	}
+	mutex_lock(&adm_ctx.resource->conf_update);
 
-	/* No need for _rcu here. All reconfiguration is
-	 * strictly serialized on genl_lock(). We are protected against
-	 * concurrent reconfiguration/addition/deletion */
-	for_each_resource(resource, &drbd_resources) {
-		for_each_connection(connection, resource) {
-			if (nla_len(adm_ctx.my_addr) == connection->my_addr_len &&
-			    !memcmp(nla_data(adm_ctx.my_addr), &connection->my_addr,
-				    connection->my_addr_len)) {
-				retcode = ERR_LOCAL_ADDR;
-				goto out;
-			}
+	new_peer_device_conf = kzalloc_obj(struct peer_device_conf);
+	if (!new_peer_device_conf)
+		goto fail;
 
-			if (nla_len(adm_ctx.peer_addr) == connection->peer_addr_len &&
-			    !memcmp(nla_data(adm_ctx.peer_addr), &connection->peer_addr,
-				    connection->peer_addr_len)) {
-				retcode = ERR_PEER_ADDR;
-				goto out;
+	old_peer_device_conf = peer_device->conf;
+	*new_peer_device_conf = *old_peer_device_conf;
+	if (should_set_defaults(info))
+		set_peer_device_conf_defaults(new_peer_device_conf);
+
+	err = peer_device_conf_from_attrs_for_change(new_peer_device_conf, info);
+	if (err && err != -ENOMSG) {
+		retcode = ERR_MANDATORY_TAG;
+		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+		goto fail_ret_set;
+	}
+
+	if (!old_peer_device_conf->bitmap && new_peer_device_conf->bitmap &&
+	    peer_device->bitmap_index == -1) {
+		if (get_ldev(device)) {
+			err = allocate_bitmap_index(peer_device, device->ldev);
+			put_ldev(device);
+			if (err) {
+				drbd_msg_put_info(adm_ctx.reply_skb,
+						  "No bitmap slot available in meta-data");
+				retcode = ERR_INVALID_REQUEST;
+				goto fail_ret_set;
 			}
+			drbd_info(peer_device,
+				  "Former intentional diskless peer got bitmap slot %d\n",
+				  peer_device->bitmap_index);
+			drbd_md_sync(device);
+			notify = true;
+		}
+	}
+
+	if (old_peer_device_conf->bitmap && !new_peer_device_conf->bitmap) {
+		enum drbd_disk_state pdsk = peer_device->disk_state[NOW];
+		enum drbd_disk_state disk = device->disk_state[NOW];
+		if (!(disk == D_DISKLESS || pdsk == D_DISKLESS || pdsk == D_UNKNOWN)) {
+			drbd_msg_put_info(adm_ctx.reply_skb,
+					  "Can not drop the bitmap when both sides have a disk");
+			retcode = ERR_INVALID_REQUEST;
+			goto fail_ret_set;
+		}
+		err = clear_peer_slot(device, peer_device->node_id, MDF_NODE_EXISTS);
+		if (!err) {
+			peer_device->bitmap_index = -1;
+			notify = true;
 		}
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	connection = first_connection(adm_ctx.resource);
-	conn_reconfig_start(connection);
+	if (!expect(peer_device, new_peer_device_conf->resync_rate >= 1))
+		new_peer_device_conf->resync_rate = 1;
 
-	if (connection->cstate > C_STANDALONE) {
-		retcode = ERR_NET_CONFIGURED;
+	if (new_peer_device_conf->c_plan_ahead > DRBD_C_PLAN_AHEAD_MAX)
+		new_peer_device_conf->c_plan_ahead = DRBD_C_PLAN_AHEAD_MAX;
+
+	err = adjust_resync_fifo(peer_device, new_peer_device_conf, &old_plan);
+	if (err)
 		goto fail;
-	}
 
-	/* allocation not in the IO path, drbdsetup / netlink process context */
-	new_net_conf = kzalloc_obj(*new_net_conf);
-	if (!new_net_conf) {
+	rcu_assign_pointer(peer_device->conf, new_peer_device_conf);
+
+	kvfree_rcu_mightsleep(old_peer_device_conf);
+	kfree(old_plan);
+
+	/* No need to call drbd_send_sync_param() here. The values in
+	 * peer_device->conf that we send are ignored by recent peers anyway. */
+
+	if (0) {
+fail:
 		retcode = ERR_NOMEM;
-		goto fail;
+fail_ret_set:
+		kfree(new_peer_device_conf);
 	}
 
-	set_net_conf_defaults(new_net_conf);
+	mutex_unlock(&adm_ctx.resource->conf_update);
+	mutex_unlock(&adm_ctx.resource->adm_mutex);
+out_no_adm_mutex:
+	if (notify)
+		drbd_broadcast_peer_device_state(peer_device);
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
 
-	err = net_conf_from_attrs(new_net_conf, info);
-	if (err && err != -ENOMSG) {
-		retcode = ERR_MANDATORY_TAG;
-		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
-		goto fail;
-	}
+}
 
-	retcode = check_net_options(connection, new_net_conf);
-	if (retcode != NO_ERROR)
-		goto fail;
+int drbd_create_peer_device_default_config(struct drbd_peer_device *peer_device)
+{
+	struct peer_device_conf *conf;
+	int err;
 
-	retcode = alloc_crypto(&crypto, new_net_conf);
-	if (retcode != NO_ERROR)
-		goto fail;
+	conf = kzalloc_obj(*conf);
+	if (!conf)
+		return -ENOMEM;
 
-	((char *)new_net_conf->shared_secret)[SHARED_SECRET_MAX-1] = 0;
+	set_peer_device_conf_defaults(conf);
+	err = adjust_resync_fifo(peer_device, conf, NULL);
+	if (err)
+		return err;
 
-	drbd_flush_workqueue(&connection->sender_work);
+	peer_device->conf = conf;
 
-	mutex_lock(&adm_ctx.resource->conf_update);
-	old_net_conf = connection->net_conf;
-	if (old_net_conf) {
-		retcode = ERR_NET_CONFIGURED;
-		mutex_unlock(&adm_ctx.resource->conf_update);
-		goto fail;
+	return 0;
+}
+
+static void connection_to_info(struct connection_info *info,
+			       struct drbd_connection *connection)
+{
+	info->conn_connection_state = connection->cstate[NOW];
+	info->conn_role = connection->peer_role[NOW];
+}
+
+#define str_to_info(info, field, str) ({ \
+	strscpy(info->field, str, sizeof(info->field)); \
+	info->field ## _len = min(strlen(str), sizeof(info->field)); \
+})
+
+/* shared logic between peer_device_to_info and peer_device_state_change_to_info */
+static void __peer_device_to_info(struct peer_device_info *info,
+				  struct drbd_peer_device *peer_device,
+				  enum which_state which)
+{
+	info->peer_resync_susp_dependency = resync_susp_comb_dep(peer_device, which);
+	info->peer_is_intentional_diskless = !want_bitmap(peer_device);
+}
+
+static void peer_device_to_info(struct peer_device_info *info,
+				struct drbd_peer_device *peer_device)
+{
+	info->peer_repl_state = peer_device->repl_state[NOW];
+	info->peer_disk_state = peer_device->disk_state[NOW];
+	info->peer_resync_susp_user = peer_device->resync_susp_user[NOW];
+	info->peer_resync_susp_peer = peer_device->resync_susp_peer[NOW];
+	__peer_device_to_info(info, peer_device, NOW);
+}
+
+void peer_device_state_change_to_info(struct peer_device_info *info,
+				      struct drbd_peer_device_state_change *state_change)
+{
+	info->peer_repl_state = state_change->repl_state[NEW];
+	info->peer_disk_state = state_change->disk_state[NEW];
+	info->peer_resync_susp_user = state_change->resync_susp_user[NEW];
+	info->peer_resync_susp_peer = state_change->resync_susp_peer[NEW];
+	__peer_device_to_info(info, state_change->peer_device, NEW);
+}
+
+/* shared logic between device_to_info and device_state_change_to_info */
+static void __device_to_info(struct device_info *info,
+			     struct drbd_device *device)
+{
+	info->is_intentional_diskless = device->device_conf.intentional_diskless;
+	info->dev_is_open = device->open_cnt != 0;
+
+	rcu_read_lock();
+	if (get_ldev(device)) {
+		struct disk_conf *disk_conf =
+			rcu_dereference(device->ldev->disk_conf);
+		str_to_info(info, backing_dev_path, disk_conf->backing_dev);
+		put_ldev(device);
+	} else {
+		info->backing_dev_path[0] = '\0';
+		info->backing_dev_path_len = 0;
 	}
-	rcu_assign_pointer(connection->net_conf, new_net_conf);
+	rcu_read_unlock();
+}
 
-	conn_free_crypto(connection);
-	connection->cram_hmac_tfm = crypto.cram_hmac_tfm;
-	connection->integrity_tfm = crypto.integrity_tfm;
-	connection->csums_tfm = crypto.csums_tfm;
-	connection->verify_tfm = crypto.verify_tfm;
+void device_to_info(struct device_info *info,
+			   struct drbd_device *device)
+{
+	info->dev_disk_state = device->disk_state[NOW];
+	info->dev_has_quorum = device->have_quorum[NOW];
+	__device_to_info(info, device);
+}
+
+void device_state_change_to_info(struct device_info *info,
+				 struct drbd_device_state_change *state_change)
+{
+	info->dev_disk_state = state_change->disk_state[NEW];
+	info->dev_has_quorum = state_change->have_quorum[NEW];
+	__device_to_info(info, state_change->device);
+}
+
+static bool is_resync_target_in_other_connection(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_device *p;
 
-	connection->my_addr_len = nla_len(adm_ctx.my_addr);
-	memcpy(&connection->my_addr, nla_data(adm_ctx.my_addr), connection->my_addr_len);
-	connection->peer_addr_len = nla_len(adm_ctx.peer_addr);
-	memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len);
+	for_each_peer_device(p, device) {
+		if (p == peer_device)
+			continue;
 
-	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-		peer_devices++;
+		if (p->repl_state[NOW] == L_SYNC_TARGET)
+			return true;
 	}
 
-	connection_to_info(&connection_info, connection);
-	flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
-	mutex_lock(&notification_mutex);
-	notify_connection_state(NULL, 0, connection, &connection_info, NOTIFY_CREATE | flags);
-	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-		struct peer_device_info peer_device_info;
+	return false;
+}
+
+static enum drbd_ret_code drbd_check_name_str(const char *name, const bool strict);
+static void drbd_msg_put_name_error(struct sk_buff *reply_skb, enum drbd_ret_code ret_code);
+
+static enum drbd_ret_code drbd_check_conn_name(struct drbd_resource *resource, const char *new_name)
+{
+	struct drbd_connection *connection;
+	enum drbd_ret_code retcode;
+	const char *tmp_name;
+
+	retcode = drbd_check_name_str(new_name, drbd_strict_names);
+	if (retcode != NO_ERROR)
+		return retcode;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		/* is this even possible? */
+		if (!connection->transport.net_conf)
+			continue;
+		tmp_name = connection->transport.net_conf->name;
+		if (!tmp_name)
+			continue;
+		if (strcmp(tmp_name, new_name))
+			continue;
+		retcode = ERR_ALREADY_EXISTS;
+		break;
+	}
+	rcu_read_unlock();
+	return retcode;
+}
+
+static int adm_new_connection(struct drbd_config_context *adm_ctx, struct genl_info *info)
+{
+	struct connection_info connection_info;
+	enum drbd_notification_type flags;
+	unsigned int peer_devices = 0;
+	struct drbd_device *device;
+	struct drbd_peer_device *peer_device;
+	struct net_conf *old_net_conf, *new_net_conf = NULL;
+	struct crypto crypto = { NULL, };
+	struct drbd_connection *connection;
+	enum drbd_ret_code retcode;
+	int i, err;
+	char *transport_name;
+	struct drbd_transport_class *tr_class;
+	struct drbd_transport *transport;
+
+	/* allocation not in the IO path, drbdsetup / netlink process context */
+	new_net_conf = kzalloc_obj(*new_net_conf);
+	if (!new_net_conf)
+		return ERR_NOMEM;
+
+	set_net_conf_defaults(new_net_conf);
+
+	err = net_conf_from_attrs(new_net_conf, info);
+	if (err) {
+		retcode = ERR_MANDATORY_TAG;
+		drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err));
+		goto fail;
+	}
+
+	retcode = drbd_check_conn_name(adm_ctx->resource, new_net_conf->name);
+	if (retcode != NO_ERROR) {
+		drbd_msg_put_name_error(adm_ctx->reply_skb, retcode);
+		goto fail;
+	}
+
+	transport_name = new_net_conf->transport_name_len ? new_net_conf->transport_name :
+		new_net_conf->load_balance_paths ? "lb-tcp" : "tcp";
+	tr_class = drbd_get_transport_class(transport_name);
+	if (!tr_class) {
+		retcode = ERR_CREATE_TRANSPORT;
+		goto fail;
+	}
+
+	connection = drbd_create_connection(adm_ctx->resource, tr_class);
+	if (!connection) {
+		retcode = ERR_NOMEM;
+		goto fail_put_transport;
+	}
+	connection->peer_node_id = adm_ctx->peer_node_id;
+	/* transport class reference now owned by connection,
+	 * prevent double cleanup. */
+	tr_class = NULL;
+
+	mutex_lock(&adm_ctx->resource->conf_update);
+	retcode = check_net_options(connection, new_net_conf);
+	if (retcode != NO_ERROR)
+		goto unlock_fail_free_connection;
+
+	retcode = alloc_crypto(&crypto, new_net_conf, adm_ctx->reply_skb);
+	if (retcode != NO_ERROR)
+		goto unlock_fail_free_connection;
+
+	((char *)new_net_conf->shared_secret)[SHARED_SECRET_MAX-1] = 0;
+
+	idr_for_each_entry(&adm_ctx->resource->devices, device, i) {
+		int id;
+
+		retcode = ERR_NOMEM;
+		peer_device = create_peer_device(device, connection);
+		if (!peer_device)
+			goto unlock_fail_free_connection;
+		id = idr_alloc(&connection->peer_devices, peer_device,
+			       device->vnr, device->vnr + 1, GFP_KERNEL);
+		if (id < 0)
+			goto unlock_fail_free_connection;
+
+		if (get_ldev(device)) {
+			struct drbd_peer_md *peer_md =
+				&device->ldev->md.peers[adm_ctx->peer_node_id];
+			if (peer_md->flags & MDF_PEER_OUTDATED)
+				peer_device->disk_state[NOW] = D_OUTDATED;
+			put_ldev(device);
+		}
+	}
+
+	/* Set bitmap_index if it was allocated previously */
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		unsigned int bitmap_index;
+
+		device = peer_device->device;
+		if (!get_ldev(device))
+			continue;
+
+		bitmap_index = device->ldev->md.peers[adm_ctx->peer_node_id].bitmap_index;
+		if (bitmap_index != -1) {
+			if (want_bitmap(peer_device))
+				peer_device->bitmap_index = bitmap_index;
+			else
+				device->ldev->md.peers[adm_ctx->peer_node_id].flags &= ~MDF_HAVE_BITMAP;
+		}
+		put_ldev(device);
+	}
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		peer_device->send_cnt = 0;
+		peer_device->recv_cnt = 0;
+	}
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		struct drbd_device *device = peer_device->device;
+
+		peer_device->resync_susp_other_c[NOW] =
+			is_resync_target_in_other_connection(peer_device);
+		list_add_rcu(&peer_device->peer_devices, &device->peer_devices);
+		kref_get(&connection->kref);
+		kref_get(&device->kref);
+		peer_devices++;
+		peer_device->node_id = connection->peer_node_id;
+	}
+
+	write_lock_irq(&adm_ctx->resource->state_rwlock);
+
+	/*
+	 * Initialize to the current dagtag so that flushes can be acked even
+	 * if no further writes occur.
+	 */
+	connection->last_peer_ack_dagtag_seen = READ_ONCE(adm_ctx->resource->dagtag_sector);
+
+	list_add_tail_rcu(&connection->connections, &adm_ctx->resource->connections);
+	write_unlock_irq(&adm_ctx->resource->state_rwlock);
+
+	transport = &connection->transport;
+	old_net_conf = transport->net_conf;
+	if (old_net_conf) {
+		retcode = ERR_NET_CONFIGURED;
+		goto unlock_fail_free_connection;
+	}
+
+	err = transport->class->ops.net_conf_change(transport, new_net_conf);
+	if (err) {
+		drbd_msg_sprintf_info(adm_ctx->reply_skb, "transport net_conf_change failed: %d",
+				      err);
+		retcode = ERR_INVALID_REQUEST;
+		goto unlock_fail_free_connection;
+	}
+
+	rcu_assign_pointer(transport->net_conf, new_net_conf);
+	connection->fencing_policy = new_net_conf->fencing_policy;
+
+	connection->cram_hmac_tfm = crypto.cram_hmac_tfm;
+	connection->integrity_tfm = crypto.integrity_tfm;
+	connection->csums_tfm = crypto.csums_tfm;
+	connection->verify_tfm = crypto.verify_tfm;
+
+	/* transferred ownership. prevent double cleanup. */
+	new_net_conf = NULL;
+	memset(&crypto, 0, sizeof(crypto));
+
+	if (connection->peer_node_id > adm_ctx->resource->max_node_id)
+		adm_ctx->resource->max_node_id = connection->peer_node_id;
+
+	connection_to_info(&connection_info, connection);
+	flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+	mutex_lock(&notification_mutex);
+	notify_connection_state(NULL, 0, connection, &connection_info, NOTIFY_CREATE | flags);
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		struct peer_device_info peer_device_info;
+
+		peer_device_to_info(&peer_device_info, peer_device);
+		flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
+		notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CREATE | flags);
+	}
+	mutex_unlock(&notification_mutex);
+
+	mutex_unlock(&adm_ctx->resource->conf_update);
+
+	drbd_debugfs_connection_add(connection); /* after ->net_conf was assigned */
+	drbd_thread_start(&connection->sender);
+	return NO_ERROR;
+
+unlock_fail_free_connection:
+	drbd_unregister_connection(connection);
+	mutex_unlock(&adm_ctx->resource->conf_update);
+	synchronize_rcu();
+	drbd_reclaim_connection(&connection->rcu);
+fail_put_transport:
+	drbd_put_transport_class(tr_class);
+fail:
+	free_crypto(&crypto);
+	kfree(new_net_conf);
+
+	return retcode;
+}
+
+static bool addr_eq_nla(const struct sockaddr_storage *addr, const int addr_len, const struct nlattr *nla)
+{
+	return	nla_len(nla) == addr_len && memcmp(nla_data(nla), addr, addr_len) == 0;
+}
+
+static enum drbd_ret_code
+check_path_against_nla(const struct drbd_path *path,
+		       const struct nlattr *my_addr, const struct nlattr *peer_addr)
+{
+	enum drbd_ret_code ret = NO_ERROR;
+
+	if (addr_eq_nla(&path->my_addr, path->my_addr_len, my_addr))
+		ret = ERR_LOCAL_ADDR;
+	if (addr_eq_nla(&path->peer_addr, path->peer_addr_len, peer_addr))
+		ret = (ret == ERR_LOCAL_ADDR ? ERR_LOCAL_AND_PEER_ADDR : ERR_PEER_ADDR);
+	return ret;
+}
+
+static enum drbd_ret_code
+check_path_usable(const struct drbd_config_context *adm_ctx,
+		  const struct nlattr *my_addr, const struct nlattr *peer_addr)
+{
+	struct drbd_resource *resource;
+	struct drbd_connection *connection;
+	enum drbd_ret_code retcode;
+
+	if (!(my_addr && peer_addr)) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "connection endpoint(s) missing");
+		return ERR_INVALID_REQUEST;
+	}
+
+	for_each_resource_rcu(resource, &drbd_resources) {
+		for_each_connection_rcu(connection, resource) {
+			struct drbd_path *path;
+			list_for_each_entry_rcu(path, &connection->transport.paths, list) {
+				retcode = check_path_against_nla(path, my_addr, peer_addr);
+				if (retcode == NO_ERROR)
+					continue;
+				/* Within the same resource, it is ok to use
+				 * the same endpoint several times */
+				if (retcode != ERR_LOCAL_AND_PEER_ADDR &&
+				    resource == adm_ctx->resource)
+					continue;
+				return retcode;
+			}
+		}
+	}
+	return NO_ERROR;
+}
+
+
+static enum drbd_ret_code
+adm_add_path(struct drbd_config_context *adm_ctx,  struct genl_info *info)
+{
+	struct drbd_transport *transport = &adm_ctx->connection->transport;
+	struct drbd_resource *resource = adm_ctx->resource;
+	struct drbd_connection *connection = adm_ctx->connection;
+	struct nlattr **nested_attr_tb;
+	struct nlattr *my_addr, *peer_addr;
+	struct drbd_path *path;
+	struct net *existing_net;
+	enum drbd_ret_code retcode;
+	int err;
+
+	/* parse and validate only */
+	existing_net = drbd_net_assigned_to_connection(adm_ctx->connection);
+	if (existing_net && !net_eq(adm_ctx->net, existing_net)) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "connection already assigned to a different network namespace");
+		return ERR_INVALID_REQUEST;
+	}
+
+	err = path_parms_ntb_from_attrs(&nested_attr_tb, info);
+	if (err) {
+		drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err));
+		return ERR_MANDATORY_TAG;
+	}
+	my_addr = nested_attr_tb[__nla_type(T_my_addr)];
+	peer_addr = nested_attr_tb[__nla_type(T_peer_addr)];
+	kfree(nested_attr_tb);
+	nested_attr_tb = NULL;
+
+	rcu_read_lock();
+	retcode = check_path_usable(adm_ctx, my_addr, peer_addr);
+	rcu_read_unlock();
+	if (retcode != NO_ERROR)
+		return retcode;
+
+	path = kzalloc(transport->class->path_instance_size, GFP_KERNEL);
+	if (!path)
+		return ERR_NOMEM;
+
+	path->net = adm_ctx->net;
+	path->my_addr_len = nla_len(my_addr);
+	memcpy(&path->my_addr, nla_data(my_addr), path->my_addr_len);
+	path->peer_addr_len = nla_len(peer_addr);
+	memcpy(&path->peer_addr, nla_data(peer_addr), path->peer_addr_len);
+
+	kref_get(&adm_ctx->connection->kref);
+	path->transport = transport;
+
+	kref_init(&path->kref);
+
+	if (connection->resource->res_opts.drbd8_compat_mode && resource->res_opts.node_id == -1) {
+		err = drbd_setup_node_ids_84(connection, path, adm_ctx->peer_node_id);
+		if (err) {
+			drbd_msg_put_info(adm_ctx->reply_skb,
+				err == -ENOTUNIQ ? "node-id from drbdsetup and meta-data differ" :
+				"error setting up node IDs");
+			kref_put(&path->kref, drbd_destroy_path);
+			return ERR_INVALID_REQUEST;
+		}
+	}
+
+	/* Exclusive with transport op "prepare_connect()" */
+	mutex_lock(&resource->conf_update);
+
+	err = transport->class->ops.add_path(path);
+
+	if (err) {
+		kref_put(&path->kref, drbd_destroy_path);
+		drbd_err(connection, "add_path() failed with %d\n", err);
+		drbd_msg_put_info(adm_ctx->reply_skb, "add_path on transport failed");
+		mutex_unlock(&resource->conf_update);
+		return ERR_INVALID_REQUEST;
+	}
+
+	/* Exclusive with reading state, in particular remember_state_change() */
+	write_lock_irq(&resource->state_rwlock);
+	list_add_tail_rcu(&path->list, &transport->paths);
+	write_unlock_irq(&resource->state_rwlock);
+
+	mutex_unlock(&resource->conf_update);
+
+	notify_path(adm_ctx->connection, path, NOTIFY_CREATE);
+	return NO_ERROR;
+}
+
+static int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct connect_parms parms = { 0, };
+	struct drbd_peer_device *peer_device;
+	struct drbd_connection *connection;
+	enum drbd_ret_code retcode;
+	enum drbd_state_rv rv;
+	enum drbd_conn_state cstate;
+	int i, err;
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+
+	connection = adm_ctx.connection;
+	cstate = connection->cstate[NOW];
+	if (cstate != C_STANDALONE) {
+		retcode = ERR_NET_CONFIGURED;
+		goto out;
+	}
+
+	if (first_path(connection) == NULL) {
+		drbd_msg_put_info(adm_ctx.reply_skb, "connection endpoint(s) missing");
+		retcode = ERR_INVALID_REQUEST;
+		goto out;
+	}
+
+	if (!net_eq(adm_ctx.net, drbd_net_assigned_to_connection(connection))) {
+		drbd_msg_put_info(adm_ctx.reply_skb, "connection assigned to a different network namespace");
+		retcode = ERR_INVALID_REQUEST;
+		goto out;
+	}
+
+	if (info->attrs[DRBD_NLA_CONNECT_PARMS]) {
+		err = connect_parms_from_attrs(&parms, info);
+		if (err) {
+			retcode = ERR_MANDATORY_TAG;
+			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+			goto out;
+		}
+	}
+	if (parms.discard_my_data) {
+		if (adm_ctx.resource->role[NOW] == R_PRIMARY) {
+			retcode = ERR_DISCARD_IMPOSSIBLE;
+			goto out;
+		}
+		set_bit(CONN_DISCARD_MY_DATA, &connection->flags);
+	}
+	if (parms.tentative)
+		set_bit(CONN_DRY_RUN, &connection->flags);
+
+	/* Eventually allocate bitmap indexes for the peer_devices here */
+	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
+		struct drbd_device *device;
+
+		if (peer_device->bitmap_index != -1 || !want_bitmap(peer_device))
+			continue;
+
+		device = peer_device->device;
+		if (!get_ldev(device))
+			continue;
+
+		err = allocate_bitmap_index(peer_device, device->ldev);
+		put_ldev(device);
+		if (err) {
+			retcode = ERR_INVALID_REQUEST;
+			goto out;
+		}
+		drbd_md_mark_dirty(device);
+	}
+
+	rv = change_cstate_tag(connection, C_UNCONNECTED, CS_VERBOSE, "connect", NULL);
+	drbd_adm_finish(&adm_ctx, info, rv);
+	return 0;
+out:
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
+}
+
+static int drbd_adm_new_peer(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct drbd_connection *connection;
+	struct drbd_resource *resource;
+	enum drbd_ret_code retcode;
+	struct drbd_device *device;
+	int vnr, n_connections = 0;
+
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_NODE);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+
+	resource = adm_ctx.resource;
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		bool fail = false;
+
+		if (get_ldev_if_state(device, D_FAILED)) {
+			fail = !device->ldev->disk_conf->d_bitmap;
+			put_ldev(device);
+		}
+		if (fail) {
+			rcu_read_unlock();
+			retcode = ERR_INVALID_REQUEST;
+			drbd_msg_sprintf_info(adm_ctx.reply_skb,
+			      "Cannot add a peer while having a disk without an allocated bitmap");
+			goto out_unlock;
+		}
+	}
+	rcu_read_unlock();
+
+	for_each_connection(connection, resource)
+		n_connections++;
+	if (resource->res_opts.drbd8_compat_mode && n_connections >= 1) {
+		retcode = ERR_INVALID_REQUEST;
+		drbd_msg_sprintf_info(adm_ctx.reply_skb,
+				      "drbd8 compat mode allows one peer at max");
+		goto out_unlock;
+	}
+
+	/* ensure uniqueness of peer_node_id by checking with adm_mutex */
+	connection = drbd_connection_by_node_id(resource, adm_ctx.peer_node_id);
+	if (adm_ctx.connection || connection) {
+		retcode = ERR_INVALID_REQUEST;
+		drbd_msg_sprintf_info(adm_ctx.reply_skb,
+				      "Connection for peer node id %d already exists",
+				      adm_ctx.peer_node_id);
+	} else {
+		retcode = adm_new_connection(&adm_ctx, info);
+	}
+
+out_unlock:
+	mutex_unlock(&resource->adm_mutex);
+out:
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
+}
+
+static int drbd_adm_new_path(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	enum drbd_ret_code retcode;
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+
+	/* remote transport endpoints need to be globally unique */
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+	} else {
+		retcode = adm_add_path(&adm_ctx, info);
+		mutex_unlock(&adm_ctx.resource->adm_mutex);
+	}
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
+}
+
+static enum drbd_ret_code
+adm_del_path(struct drbd_config_context *adm_ctx,  struct genl_info *info)
+{
+	struct drbd_resource *resource = adm_ctx->resource;
+	struct drbd_connection *connection = adm_ctx->connection;
+	struct drbd_transport *transport = &connection->transport;
+	struct nlattr **nested_attr_tb;
+	struct nlattr *my_addr, *peer_addr;
+	struct drbd_path *path;
+	int nr_paths = 0;
+	int err;
+
+	/* parse and validate only */
+	if (!net_eq(adm_ctx->net, drbd_net_assigned_to_connection(connection))) {
+		drbd_msg_put_info(adm_ctx->reply_skb, "connection assigned to a different network namespace");
+		return ERR_INVALID_REQUEST;
+	}
+
+	err = path_parms_ntb_from_attrs(&nested_attr_tb, info);
+	if (err) {
+		drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err));
+		return ERR_MANDATORY_TAG;
+	}
+	my_addr = nested_attr_tb[__nla_type(T_my_addr)];
+	peer_addr = nested_attr_tb[__nla_type(T_peer_addr)];
+	kfree(nested_attr_tb);
+	nested_attr_tb = NULL;
+
+	list_for_each_entry(path, &transport->paths, list)
+		nr_paths++;
+
+	if (nr_paths == 1 && connection->cstate[NOW] >= C_CONNECTING) {
+		drbd_msg_put_info(adm_ctx->reply_skb,
+				  "Can not delete last path, use disconnect first!");
+		return ERR_INVALID_REQUEST;
+	}
+
+	err = -ENOENT;
+	list_for_each_entry(path, &transport->paths, list) {
+		if (!addr_eq_nla(&path->my_addr, path->my_addr_len, my_addr))
+			continue;
+		if (!addr_eq_nla(&path->peer_addr, path->peer_addr_len, peer_addr))
+			continue;
+
+		/* Exclusive with transport op "prepare_connect()" */
+		mutex_lock(&resource->conf_update);
+
+		if (!transport->class->ops.may_remove_path(path)) {
+			err = -EBUSY;
+			mutex_unlock(&resource->conf_update);
+			break;
+		}
 
-		peer_device_to_info(&peer_device_info, peer_device);
-		flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
-		notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CREATE | flags);
-	}
-	mutex_unlock(&notification_mutex);
-	mutex_unlock(&adm_ctx.resource->conf_update);
+		set_bit(TR_UNREGISTERED, &path->flags);
+		/* Ensure flag visible before list manipulation. */
+		smp_wmb();
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-		struct drbd_device *device = peer_device->device;
-		device->send_cnt = 0;
-		device->recv_cnt = 0;
+		/* Exclusive with reading state, in particular remember_state_change() */
+		write_lock_irq(&resource->state_rwlock);
+		list_del_rcu(&path->list);
+		write_unlock_irq(&resource->state_rwlock);
+
+		mutex_unlock(&resource->conf_update);
+
+		transport->class->ops.remove_path(path);
+		notify_path(connection, path, NOTIFY_DESTROY);
+		/* Transport modules might use RCU on the path list. */
+		call_rcu(&path->rcu, drbd_reclaim_path);
+
+		return NO_ERROR;
 	}
-	rcu_read_unlock();
 
-	rv = conn_request_state(connection, NS(conn, C_UNCONNECTED), CS_VERBOSE);
+	drbd_err(connection, "del_path() failed with %d\n", err);
+	drbd_msg_put_info(adm_ctx->reply_skb,
+			  err == -ENOENT ? "no such path" : "del_path on transport failed");
+	return ERR_INVALID_REQUEST;
+}
 
-	conn_reconfig_done(connection);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-	drbd_adm_finish(&adm_ctx, info, rv);
-	return 0;
+static int drbd_adm_del_path(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	enum drbd_ret_code retcode;
 
-fail:
-	free_crypto(&crypto);
-	kfree(new_net_conf);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION);
+	if (!adm_ctx.reply_skb)
+		return retcode;
 
-	conn_reconfig_done(connection);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out:
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+	} else {
+		retcode = adm_del_path(&adm_ctx, info);
+		mutex_unlock(&adm_ctx.resource->adm_mutex);
+	}
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static enum drbd_state_rv conn_try_disconnect(struct drbd_connection *connection, bool force)
+int drbd_open_ro_count(struct drbd_resource *resource)
 {
-	enum drbd_conns cstate;
-	enum drbd_state_rv rv;
+	struct drbd_device *device;
+	int vnr, open_ro_cnt = 0;
+
+	read_lock_irq(&resource->state_rwlock);
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (!device->writable)
+			open_ro_cnt += device->open_cnt;
+	}
+	read_unlock_irq(&resource->state_rwlock);
 
-repeat:
-	rv = conn_request_state(connection, NS(conn, C_DISCONNECTING),
-			force ? CS_HARD : 0);
+	return open_ro_cnt;
+}
+
+static enum drbd_state_rv conn_try_disconnect(struct drbd_connection *connection, bool force,
+					      const char *tag, struct sk_buff *reply_skb)
+{
+	struct drbd_resource *resource = connection->resource;
+	enum drbd_conn_state cstate;
+	enum drbd_state_rv rv;
+	enum chg_state_flags flags = (force ? CS_HARD : 0) | CS_VERBOSE;
+	const char *err_str = NULL;
+	long t;
 
+    repeat:
+	rv = change_cstate_tag(connection, C_DISCONNECTING, flags, tag, &err_str);
 	switch (rv) {
-	case SS_NOTHING_TO_DO:
+	case SS_CW_FAILED_BY_PEER:
+	case SS_NEED_CONNECTION:
+		read_lock_irq(&resource->state_rwlock);
+		cstate = connection->cstate[NOW];
+		read_unlock_irq(&resource->state_rwlock);
+		if (cstate < C_CONNECTED)
+			goto repeat;
 		break;
+	case SS_NO_UP_TO_DATE_DISK:
+		if (resource->role[NOW] == R_PRIMARY)
+			break;
+		/* Most probably udev opened it read-only. That might happen
+		   if it was demoted very recently. Wait up to one second. */
+		t = wait_event_interruptible_timeout(resource->state_wait,
+						     drbd_open_ro_count(resource) == 0,
+						     HZ);
+		if (t <= 0)
+			break;
+		goto repeat;
 	case SS_ALREADY_STANDALONE:
-		return SS_SUCCESS;
-	case SS_PRIMARY_NOP:
-		/* Our state checking code wants to see the peer outdated. */
-		rv = conn_request_state(connection, NS2(conn, C_DISCONNECTING, pdsk, D_OUTDATED), 0);
-
-		if (rv == SS_OUTDATE_WO_CONN) /* lost connection before graceful disconnect succeeded */
-			rv = conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_VERBOSE);
-
+		rv = SS_SUCCESS;
 		break;
-	case SS_CW_FAILED_BY_PEER:
-		spin_lock_irq(&connection->resource->req_lock);
-		cstate = connection->cstate;
-		spin_unlock_irq(&connection->resource->req_lock);
-		if (cstate <= C_WF_CONNECTION)
+	case SS_IS_DISKLESS:
+	case SS_LOWER_THAN_OUTDATED:
+		rv = change_cstate_tag(connection, C_DISCONNECTING, CS_HARD, tag, NULL);
+		break;
+	case SS_NO_QUORUM:
+		if (!(flags & CS_VERBOSE)) {
+			flags |= CS_VERBOSE;
 			goto repeat;
-		/* The peer probably wants to see us outdated. */
-		rv = conn_request_state(connection, NS2(conn, C_DISCONNECTING,
-							disk, D_OUTDATED), 0);
-		if (rv == SS_IS_DISKLESS || rv == SS_LOWER_THAN_OUTDATED) {
-			rv = conn_request_state(connection, NS(conn, C_DISCONNECTING),
-					CS_HARD);
 		}
 		break;
 	default:;
 		/* no special handling necessary */
 	}
 
-	if (rv >= SS_SUCCESS) {
-		enum drbd_state_rv rv2;
-		/* No one else can reconfigure the network while I am here.
-		 * The state handling only uses drbd_thread_stop_nowait(),
-		 * we want to really wait here until the receiver is no more.
-		 */
-		drbd_thread_stop(&connection->receiver);
-
-		/* Race breaker.  This additional state change request may be
-		 * necessary, if this was a forced disconnect during a receiver
-		 * restart.  We may have "killed" the receiver thread just
-		 * after drbd_receiver() returned.  Typically, we should be
-		 * C_STANDALONE already, now, and this becomes a no-op.
-		 */
-		rv2 = conn_request_state(connection, NS(conn, C_STANDALONE),
-				CS_VERBOSE | CS_HARD);
-		if (rv2 < SS_SUCCESS)
-			drbd_err(connection,
-				"unexpected rv2=%d in conn_try_disconnect()\n",
-				rv2);
-		/* Unlike in DRBD 9, the state engine has generated
-		 * NOTIFY_DESTROY events before clearing connection->net_conf. */
+	if (rv >= SS_SUCCESS)
+		wait_event_interruptible_timeout(resource->state_wait,
+						 connection->cstate[NOW] == C_STANDALONE,
+						 HZ);
+	if (err_str) {
+		drbd_msg_put_info(reply_skb, err_str);
+		kfree(err_str);
 	}
+
 	return rv;
 }
 
-int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info)
+/* this can only be called immediately after a successful
+ * peer_try_disconnect, within the same resource->adm_mutex */
+static void del_connection(struct drbd_connection *connection, const char *tag)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	enum drbd_state_rv rv2;
+	int vnr;
+
+	if (test_bit(C_UNREGISTERED, &connection->flags))
+		return;
+
+	/* No one else can reconfigure the network while I am here.
+	 * The state handling only uses drbd_thread_stop_nowait(),
+	 * we want to really wait here until the receiver is no more.
+	 */
+	drbd_thread_stop(&connection->receiver);
+
+	/* Race breaker.  This additional state change request may be
+	 * necessary, if this was a forced disconnect during a receiver
+	 * restart.  We may have "killed" the receiver thread just
+	 * after drbd_receiver() returned.  Typically, we should be
+	 * C_STANDALONE already, now, and this becomes a no-op.
+	 */
+	rv2 = change_cstate_tag(connection, C_STANDALONE, CS_VERBOSE | CS_HARD, tag, NULL);
+	if (rv2 < SS_SUCCESS)
+		drbd_err(connection,
+			"unexpected rv2=%d in del_connection()\n",
+			rv2);
+	/* Make sure the sender thread has actually stopped: state
+	 * handling only does drbd_thread_stop_nowait().
+	 */
+	drbd_thread_stop(&connection->sender);
+
+	mutex_lock(&resource->conf_update);
+	drbd_unregister_connection(connection);
+	mutex_unlock(&resource->conf_update);
+
+	/*
+	 * Flush the resource work queue to make sure that no more
+	 * events like state change notifications for this connection
+	 * are queued: we want the "destroy" event to come last.
+	 */
+	drbd_flush_workqueue(&resource->work);
+
+	mutex_lock(&notification_mutex);
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+		notify_peer_device_state(NULL, 0, peer_device, NULL,
+					 NOTIFY_DESTROY | NOTIFY_CONTINUES);
+	notify_connection_state(NULL, 0, connection, NULL, NOTIFY_DESTROY);
+	mutex_unlock(&notification_mutex);
+	call_rcu(&connection->rcu, drbd_reclaim_connection);
+}
+
+static int adm_disconnect(struct sk_buff *skb, struct genl_info *info, bool destroy)
 {
 	struct drbd_config_context adm_ctx;
 	struct disconnect_parms parms;
 	struct drbd_connection *connection;
+	struct net *existing_net;
 	enum drbd_state_rv rv;
 	enum drbd_ret_code retcode;
-	int err;
+	const char *tag = destroy ? "del-peer" : "disconnect";
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto fail;
 
-	connection = adm_ctx.connection;
 	memset(&parms, 0, sizeof(parms));
 	if (info->attrs[DRBD_NLA_DISCONNECT_PARMS]) {
-		err = disconnect_parms_from_attrs(&parms, info);
+		int err = disconnect_parms_from_attrs(&parms, info);
 		if (err) {
 			retcode = ERR_MANDATORY_TAG;
 			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
@@ -2753,55 +5133,114 @@ int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	rv = conn_try_disconnect(connection, parms.force_disconnect);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-	if (rv < SS_SUCCESS) {
-		drbd_adm_finish(&adm_ctx, info, rv);
-		return 0;
+	existing_net = drbd_net_assigned_to_connection(adm_ctx.connection);
+	if (existing_net && !net_eq(adm_ctx.net, existing_net)) {
+		drbd_msg_put_info(adm_ctx.reply_skb, "connection assigned to a different network namespace");
+		retcode =  ERR_INVALID_REQUEST;
+		goto fail;
+	}
+
+	connection = adm_ctx.connection;
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto fail;
+	}
+	rv = conn_try_disconnect(connection, parms.force_disconnect, tag, adm_ctx.reply_skb);
+	if (rv >= SS_SUCCESS && destroy) {
+		del_connection(connection, tag);
 	}
-	retcode = NO_ERROR;
+	if (rv < SS_SUCCESS)
+		retcode = (enum drbd_ret_code)rv;
+	else
+		retcode = NO_ERROR;
+	mutex_unlock(&adm_ctx.resource->adm_mutex);
  fail:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-void resync_after_online_grow(struct drbd_device *device)
+static int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info)
 {
-	int iass; /* I am sync source */
+	return adm_disconnect(skb, info, 0);
+}
 
-	drbd_info(device, "Resync of new storage after online grow\n");
-	if (device->state.role != device->state.peer)
-		iass = (device->state.role == R_PRIMARY);
-	else
-		iass = test_bit(RESOLVE_CONFLICTS, &first_peer_device(device)->connection->flags);
+static int drbd_adm_del_peer(struct sk_buff *skb, struct genl_info *info)
+{
+	return adm_disconnect(skb, info, 1);
+}
 
-	if (iass)
-		drbd_start_resync(device, C_SYNC_SOURCE);
-	else
-		_drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE + CS_SERIALIZE);
+void resync_after_online_grow(struct drbd_peer_device *peer_device)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	bool sync_source = false;
+	s32 peer_id;
+
+	drbd_info(peer_device, "Resync of new storage after online grow\n");
+	if (device->resource->role[NOW] != connection->peer_role[NOW])
+		sync_source = (device->resource->role[NOW] == R_PRIMARY);
+	else if (connection->agreed_pro_version < 111)
+		sync_source = test_bit(RESOLVE_CONFLICTS,
+				&peer_device->connection->transport.flags);
+	else if (get_ldev(device)) {
+		/* multiple or no primaries, proto new enough, resolve by node-id */
+		s32 self_id = device->ldev->md.node_id;
+		put_ldev(device);
+		peer_id = peer_device->node_id;
+
+		sync_source = self_id < peer_id ? 1 : 0;
+	}
+
+	if (!sync_source && connection->agreed_pro_version < 110) {
+		stable_change_repl_state(peer_device, L_WF_SYNC_UUID,
+					 CS_VERBOSE | CS_SERIALIZE, "online-grow");
+		return;
+	}
+	drbd_start_resync(peer_device, sync_source ? L_SYNC_SOURCE : L_SYNC_TARGET, "online-grow");
+}
+
+sector_t drbd_local_max_size(struct drbd_device *device) __must_hold(local)
+{
+	struct drbd_backing_dev *tmp_bdev;
+	sector_t s;
+
+	tmp_bdev = kmalloc_obj(struct drbd_backing_dev, GFP_ATOMIC);
+	if (!tmp_bdev)
+		return 0;
+
+	*tmp_bdev = *device->ldev;
+	drbd_md_set_sector_offsets(tmp_bdev);
+	s = drbd_get_max_capacity(device, tmp_bdev, false);
+	kfree(tmp_bdev);
+
+	return s;
 }
 
-int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	struct disk_conf *old_disk_conf, *new_disk_conf = NULL;
 	struct resize_parms rs;
 	struct drbd_device *device;
-	enum drbd_ret_code retcode;
 	enum determine_dev_size dd;
 	bool change_al_layout = false;
 	enum dds_flags ddsf;
 	sector_t u_size;
-	int err;
+	int err, retcode;
+	struct drbd_peer_device *peer_device;
+	bool resolve_by_node_id = true;
+	bool has_up_to_date_primary;
+	bool traditional_resize = false;
+	sector_t local_max_size;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
 	device = adm_ctx.device;
 	if (!get_ldev(device)) {
 		retcode = ERR_NO_DISK;
@@ -2820,20 +5259,58 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
-	if (device->state.conn > C_CONNECTED) {
-		retcode = ERR_RESIZE_RESYNC;
+	device = adm_ctx.device;
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->repl_state[NOW] > L_ESTABLISHED) {
+			retcode = ERR_RESIZE_RESYNC;
+			goto fail_ldev;
+		}
+	}
+
+
+	local_max_size = drbd_local_max_size(device);
+	if (rs.resize_size && local_max_size < (sector_t)rs.resize_size) {
+		drbd_err(device, "requested %llu sectors, backend seems only able to support %llu\n",
+			 (unsigned long long)(sector_t)rs.resize_size,
+			 (unsigned long long)local_max_size);
+		retcode = ERR_DISK_TOO_SMALL;
 		goto fail_ldev;
 	}
 
-	if (device->state.role == R_SECONDARY &&
-	    device->state.peer == R_SECONDARY) {
+	/* Maybe I could serve as sync source myself? */
+	has_up_to_date_primary =
+		device->resource->role[NOW] == R_PRIMARY &&
+		device->disk_state[NOW] == D_UP_TO_DATE;
+
+	if (!has_up_to_date_primary) {
+		for_each_peer_device(peer_device, device) {
+			/* ignore unless connection is fully established */
+			if (peer_device->repl_state[NOW] < L_ESTABLISHED)
+				continue;
+			if (peer_device->connection->agreed_pro_version < 111) {
+				resolve_by_node_id = false;
+				if (peer_device->connection->peer_role[NOW] == R_PRIMARY
+				&&  peer_device->disk_state[NOW] == D_UP_TO_DATE) {
+					has_up_to_date_primary = true;
+					break;
+				}
+			}
+		}
+	}
+
+	if (!has_up_to_date_primary && !resolve_by_node_id) {
 		retcode = ERR_NO_PRIMARY;
 		goto fail_ldev;
 	}
 
-	if (rs.no_resync && first_peer_device(device)->connection->agreed_pro_version < 93) {
-		retcode = ERR_NEED_APV_93;
-		goto fail_ldev;
+	for_each_peer_device(peer_device, device) {
+		struct drbd_connection *connection = peer_device->connection;
+		if (rs.no_resync &&
+		    connection->cstate[NOW] == C_CONNECTED &&
+		    connection->agreed_pro_version < 93) {
+			retcode = ERR_NEED_APV_93;
+			goto fail_ldev;
+		}
 	}
 
 	rcu_read_lock();
@@ -2856,21 +5333,21 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 			goto fail_ldev;
 		}
 
-		if (al_size_k < MD_32kB_SECT/2) {
+		if (al_size_k < (32768 >> 10)) {
 			retcode = ERR_MD_LAYOUT_TOO_SMALL;
 			goto fail_ldev;
 		}
 
+		/* Removed this pre-condition while merging from 8.4 to 9.0
 		if (device->state.conn != C_CONNECTED && !rs.resize_force) {
 			retcode = ERR_MD_LAYOUT_CONNECTED;
 			goto fail_ldev;
-		}
+		} */
 
 		change_al_layout = true;
 	}
 
-	if (device->ldev->known_size != drbd_get_capacity(device->ldev->backing_bdev))
-		device->ldev->known_size = drbd_get_capacity(device->ldev->backing_bdev);
+	device->ldev->known_size = drbd_get_capacity(device->ldev->backing_bdev);
 
 	if (new_disk_conf) {
 		mutex_lock(&device->resource->conf_update);
@@ -2883,9 +5360,17 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 		new_disk_conf = NULL;
 	}
 
-	ddsf = (rs.resize_force ? DDSF_FORCED : 0) | (rs.no_resync ? DDSF_NO_RESYNC : 0);
-	dd = drbd_determine_dev_size(device, ddsf, change_al_layout ? &rs : NULL);
-	drbd_md_sync(device);
+	ddsf = (rs.resize_force ? DDSF_ASSUME_UNCONNECTED_PEER_HAS_SPACE : 0)
+		| (rs.no_resync ? DDSF_NO_RESYNC : 0);
+
+	dd = change_cluster_wide_device_size(device, local_max_size, rs.resize_size, ddsf,
+					     change_al_layout ? &rs : NULL);
+	if (dd == DS_2PC_NOT_SUPPORTED) {
+		traditional_resize = true;
+		dd = drbd_determine_dev_size(device, 0, ddsf, change_al_layout ? &rs : NULL);
+	}
+
+	drbd_md_sync_if_dirty(device);
 	put_ldev(device);
 	if (dd == DS_ERROR) {
 		retcode = ERR_NOMEM_BITMAP;
@@ -2896,19 +5381,25 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 	} else if (dd == DS_ERROR_SHRINK) {
 		retcode = ERR_IMPLICIT_SHRINK;
 		goto fail;
+	} else if (dd == DS_2PC_ERR) {
+		retcode = SS_INTERRUPTED;
+		goto fail;
 	}
 
-	if (device->state.conn == C_CONNECTED) {
-		if (dd == DS_GREW)
-			set_bit(RESIZE_PENDING, &device->flags);
-
-		drbd_send_uuids(first_peer_device(device));
-		drbd_send_sizes(first_peer_device(device), 1, ddsf);
+	if (traditional_resize) {
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->repl_state[NOW] == L_ESTABLISHED) {
+				if (dd == DS_GREW)
+					set_bit(RESIZE_PENDING, &peer_device->flags);
+				drbd_send_uuids(peer_device, 0, 0);
+				drbd_send_sizes(peer_device, rs.resize_size, ddsf);
+			}
+		}
 	}
 
  fail:
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
- finish:
+ out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 
@@ -2918,7 +5409,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 	goto fail;
 }
 
-int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
@@ -2928,298 +5419,558 @@ int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info)
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto fail;
 
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
 	res_opts = adm_ctx.resource->res_opts;
 	if (should_set_defaults(info))
 		set_res_opts_defaults(&res_opts);
 
-	err = res_opts_from_attrs(&res_opts, info);
+	err = res_opts_from_attrs_for_change(&res_opts, info);
 	if (err && err != -ENOMSG) {
 		retcode = ERR_MANDATORY_TAG;
 		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
 		goto fail;
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	err = set_resource_options(adm_ctx.resource, &res_opts);
+	if (res_opts.explicit_drbd8_compat) {
+		struct drbd_connection *connection;
+		int n_connections = 0;
+
+		for_each_connection(connection, adm_ctx.resource)
+			n_connections++;
+
+		if (n_connections > 1) {
+			drbd_msg_sprintf_info(adm_ctx.reply_skb,
+					      "drbd8 compat mode allows one peer at max");
+			goto fail;
+		}
+	}
+
+	if (res_opts.node_id != -1) {
+#ifdef CONFIG_DRBD_COMPAT_84
+		if (!res_opts.drbd8_compat_mode && res_opts.explicit_drbd8_compat)
+			atomic_inc(&nr_drbd8_devices);
+		else if (res_opts.drbd8_compat_mode && !res_opts.explicit_drbd8_compat)
+			atomic_dec(&nr_drbd8_devices);
+#endif
+		res_opts.drbd8_compat_mode = res_opts.explicit_drbd8_compat;
+	}
+
+	err = set_resource_options(adm_ctx.resource, &res_opts, "resource-options");
 	if (err) {
 		retcode = ERR_INVALID_REQUEST;
 		if (err == -ENOMEM)
 			retcode = ERR_NOMEM;
 	}
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
 
 fail:
+	mutex_unlock(&adm_ctx.resource->adm_mutex);
+out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info)
+static enum drbd_state_rv invalidate_resync(struct drbd_peer_device *peer_device)
+{
+	struct drbd_resource *resource = peer_device->connection->resource;
+	enum drbd_state_rv rv;
+
+	drbd_flush_workqueue(&peer_device->connection->sender_work);
+
+	rv = change_repl_state(peer_device, L_STARTING_SYNC_T, CS_SERIALIZE, "invalidate");
+
+	if (rv < SS_SUCCESS && rv != SS_NEED_CONNECTION)
+		rv = stable_change_repl_state(peer_device, L_STARTING_SYNC_T,
+			CS_VERBOSE | CS_SERIALIZE, "invalidate");
+
+	wait_event_interruptible(resource->state_wait,
+				 peer_device->repl_state[NOW] != L_STARTING_SYNC_T);
+
+	return rv;
+}
+
+static enum drbd_state_rv invalidate_no_resync(struct drbd_device *device) __must_hold(local)
+{
+	struct drbd_resource *resource = device->resource;
+	struct drbd_peer_device *peer_device;
+	struct drbd_connection *connection;
+	unsigned long irq_flags;
+	enum drbd_state_rv rv;
+
+	begin_state_change(resource, &irq_flags, CS_VERBOSE);
+	for_each_connection(connection, resource) {
+		peer_device = conn_peer_device(connection, device->vnr);
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED) {
+			abort_state_change(resource, &irq_flags);
+			return SS_UNKNOWN_ERROR;
+		}
+	}
+	__change_disk_state(device, D_INCONSISTENT);
+	rv = end_state_change(resource, &irq_flags, "invalidate");
+
+	if (rv >= SS_SUCCESS) {
+		drbd_bitmap_io(device, &drbd_bmio_set_all_n_write,
+			       "set_n_write from invalidate",
+			       BM_LOCK_CLEAR | BM_LOCK_BULK,
+			       NULL);
+	}
+
+	return rv;
+}
+
+static int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
+	struct drbd_peer_device *sync_from_peer_device = NULL;
+	struct drbd_resource *resource;
 	struct drbd_device *device;
-	int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+	int retcode = 0; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+	struct invalidate_parms inv = {
+		.sync_from_peer_node_id = -1,
+		.reset_bitmap = DRBD_INVALIDATE_RESET_BITMAP_DEF,
+	};
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
 
 	device = adm_ctx.device;
+
 	if (!get_ldev(device)) {
 		retcode = ERR_NO_DISK;
-		goto out;
+		goto out_no_ldev;
+	}
+
+	resource = device->resource;
+
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (info->attrs[DRBD_NLA_INVALIDATE_PARMS]) {
+		int err;
+
+		err = invalidate_parms_from_attrs(&inv, info);
+		if (err) {
+			retcode = ERR_MANDATORY_TAG;
+			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+			goto out_no_resume;
+		}
+
+		if (inv.sync_from_peer_node_id != -1) {
+			struct drbd_connection *connection =
+				drbd_connection_by_node_id(resource, inv.sync_from_peer_node_id);
+			sync_from_peer_device = conn_peer_device(connection, device->vnr);
+		}
+
+		if (!inv.reset_bitmap && sync_from_peer_device &&
+		    sync_from_peer_device->connection->agreed_pro_version < 120) {
+			retcode = ERR_APV_TOO_LOW;
+			drbd_msg_put_info(adm_ctx.reply_skb,
+					  "Need protocol level 120 to initiate bitmap based resync");
+			goto out_no_resume;
+		}
+	}
 
 	/* If there is still bitmap IO pending, probably because of a previous
 	 * resync just being finished, wait for it before requesting a new resync.
-	 * Also wait for it's after_state_ch(). */
-	drbd_suspend_io(device);
-	wait_event(device->misc_wait, !test_bit(BITMAP_IO, &device->flags));
-	drbd_flush_workqueue(&first_peer_device(device)->connection->sender_work);
-
-	/* If we happen to be C_STANDALONE R_SECONDARY, just change to
-	 * D_INCONSISTENT, and set all bits in the bitmap.  Otherwise,
-	 * try to start a resync handshake as sync target for full sync.
-	 */
-	if (device->state.conn == C_STANDALONE && device->state.role == R_SECONDARY) {
-		retcode = drbd_request_state(device, NS(disk, D_INCONSISTENT));
-		if (retcode >= SS_SUCCESS) {
-			if (drbd_bitmap_io(device, &drbd_bmio_set_n_write,
-				"set_n_write from invalidate", BM_LOCKED_MASK, NULL))
-				retcode = ERR_IO_MD_DISK;
+	 * Also wait for its after_state_ch(). */
+	drbd_suspend_io(device, READ_AND_WRITE);
+	wait_event(device->misc_wait, !atomic_read(&device->pending_bitmap_work.n));
+
+	if (sync_from_peer_device) {
+		if (inv.reset_bitmap) {
+			retcode = invalidate_resync(sync_from_peer_device);
+		} else {
+			retcode = change_repl_state(sync_from_peer_device, L_WF_BITMAP_T,
+					CS_VERBOSE | CS_CLUSTER_WIDE | CS_WAIT_COMPLETE |
+					CS_SERIALIZE, "invalidate");
 		}
-	} else
-		retcode = drbd_request_state(device, NS(conn, C_STARTING_SYNC_T));
-	drbd_resume_io(device);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-	put_ldev(device);
-out:
-	drbd_adm_finish(&adm_ctx, info, retcode);
-	return 0;
-}
+	} else {
+		int retry = 3;
+		do {
+			struct drbd_connection *connection;
 
-static int drbd_adm_simple_request_state(struct sk_buff *skb, struct genl_info *info,
-		union drbd_state mask, union drbd_state val)
-{
-	struct drbd_config_context adm_ctx;
-	enum drbd_ret_code retcode;
+			for_each_connection(connection, resource) {
+				struct drbd_peer_device *peer_device;
+
+				peer_device = conn_peer_device(connection, device->vnr);
+				if (!peer_device)
+					continue;
+
+				if (inv.reset_bitmap) {
+					retcode = invalidate_resync(peer_device);
+				} else {
+					if (connection->agreed_pro_version < 120) {
+						retcode = ERR_APV_TOO_LOW;
+						continue;
+					}
+					retcode = change_repl_state(peer_device, L_WF_BITMAP_T,
+								CS_VERBOSE | CS_CLUSTER_WIDE |
+								CS_WAIT_COMPLETE | CS_SERIALIZE,
+								"invalidate");
+				}
+				if (retcode >= SS_SUCCESS)
+					goto out;
+			}
+			if (retcode != SS_NEED_CONNECTION)
+				break;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
-	if (!adm_ctx.reply_skb)
-		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
+			retcode = invalidate_no_resync(device);
+		} while (retcode == SS_UNKNOWN_ERROR && retry--);
+	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	retcode = drbd_request_state(adm_ctx.device, mask, val);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
 out:
+	drbd_resume_io(device);
+out_no_resume:
+	mutex_unlock(&resource->adm_mutex);
+out_no_adm_mutex:
+	put_ldev(device);
+out_no_ldev:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static int drbd_bmio_set_susp_al(struct drbd_device *device,
-		struct drbd_peer_device *peer_device) __must_hold(local)
+static int drbd_bmio_set_susp_al(struct drbd_device *device, struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	int rv;
 
 	rv = drbd_bmio_set_n_write(device, peer_device);
-	drbd_suspend_al(device);
+	drbd_try_suspend_al(device);
 	return rv;
 }
 
-int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info)
+static int full_sync_from_peer(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_resource *resource = device->resource;
+	int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+
+	retcode = stable_change_repl_state(peer_device, L_STARTING_SYNC_S, CS_SERIALIZE,
+			"invalidate-remote");
+	if (retcode < SS_SUCCESS) {
+		if (retcode == SS_NEED_CONNECTION && resource->role[NOW] == R_PRIMARY) {
+			/* The peer will get a resync upon connect anyways.
+			 * Just make that into a full resync. */
+			retcode = change_peer_disk_state(peer_device, D_INCONSISTENT,
+					CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE,
+					"invalidate-remote");
+			if (retcode >= SS_SUCCESS) {
+				if (drbd_bitmap_io(device, &drbd_bmio_set_susp_al,
+						   "set_n_write from invalidate_peer",
+						   BM_LOCK_CLEAR | BM_LOCK_BULK, peer_device))
+					retcode = ERR_IO_MD_DISK;
+			}
+		} else {
+			retcode = stable_change_repl_state(peer_device, L_STARTING_SYNC_S,
+					CS_VERBOSE | CS_SERIALIZE, "invalidate-remote");
+		}
+	}
+
+	return retcode;
+}
+
+
+static int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
-	int retcode; /* drbd_ret_code, drbd_state_rv */
+	struct drbd_peer_device *peer_device;
+	struct drbd_resource *resource;
 	struct drbd_device *device;
+	int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+	struct invalidate_peer_parms inv = {
+		.p_reset_bitmap = DRBD_INVALIDATE_RESET_BITMAP_DEF,
+	};
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
 
-	device = adm_ctx.device;
+	peer_device = adm_ctx.peer_device;
+	device = peer_device->device;
+	resource = device->resource;
+
 	if (!get_ldev(device)) {
 		retcode = ERR_NO_DISK;
 		goto out;
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
 
-	/* If there is still bitmap IO pending, probably because of a previous
-	 * resync just being finished, wait for it before requesting a new resync.
-	 * Also wait for it's after_state_ch(). */
-	drbd_suspend_io(device);
-	wait_event(device->misc_wait, !test_bit(BITMAP_IO, &device->flags));
-	drbd_flush_workqueue(&first_peer_device(device)->connection->sender_work);
-
-	/* If we happen to be C_STANDALONE R_PRIMARY, just set all bits
-	 * in the bitmap.  Otherwise, try to start a resync handshake
-	 * as sync source for full sync.
-	 */
-	if (device->state.conn == C_STANDALONE && device->state.role == R_PRIMARY) {
-		/* The peer will get a resync upon connect anyways. Just make that
-		   into a full resync. */
-		retcode = drbd_request_state(device, NS(pdsk, D_INCONSISTENT));
-		if (retcode >= SS_SUCCESS) {
-			if (drbd_bitmap_io(device, &drbd_bmio_set_susp_al,
-				"set_n_write from invalidate_peer",
-				BM_LOCKED_SET_ALLOWED, NULL))
-				retcode = ERR_IO_MD_DISK;
+	if (info->attrs[DRBD_NLA_INVAL_PEER_PARAMS]) {
+		int err;
+
+		err = invalidate_peer_parms_from_attrs(&inv, info);
+		if (err) {
+			retcode = ERR_MANDATORY_TAG;
+			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+			goto out_unlock;
 		}
-	} else
-		retcode = drbd_request_state(device, NS(conn, C_STARTING_SYNC_S));
+		if (!inv.p_reset_bitmap && peer_device->connection->agreed_pro_version < 120) {
+			retcode = ERR_APV_TOO_LOW;
+			drbd_msg_put_info(adm_ctx.reply_skb,
+					  "Need protocol level 120 to initiate bitmap based resync");
+			goto out_unlock;
+		}
+	}
+
+	drbd_suspend_io(device, READ_AND_WRITE);
+	wait_event(device->misc_wait, !atomic_read(&device->pending_bitmap_work.n));
+	drbd_flush_workqueue(&peer_device->connection->sender_work);
+
+	if (inv.p_reset_bitmap) {
+		retcode = full_sync_from_peer(peer_device);
+	} else {
+		retcode = change_repl_state(peer_device, L_WF_BITMAP_S,
+				CS_VERBOSE | CS_CLUSTER_WIDE | CS_WAIT_COMPLETE | CS_SERIALIZE,
+				"invalidate-remote");
+	}
 	drbd_resume_io(device);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
+
+out_unlock:
+	mutex_unlock(&resource->adm_mutex);
+out_no_adm_mutex:
 	put_ldev(device);
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
+	struct drbd_peer_device *peer_device;
 	enum drbd_ret_code retcode;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
+
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
 		goto out;
+	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	if (drbd_request_state(adm_ctx.device, NS(user_isp, 1)) == SS_NOTHING_TO_DO)
+	peer_device = adm_ctx.peer_device;
+	if (change_resync_susp_user(peer_device, true,
+			CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE) == SS_NOTHING_TO_DO)
 		retcode = ERR_PAUSE_IS_SET;
+
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out:
+ out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
-	union drbd_dev_state s;
+	struct drbd_peer_device *peer_device;
 	enum drbd_ret_code retcode;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
+
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
 		goto out;
+	}
+
+	peer_device = adm_ctx.peer_device;
+	if (change_resync_susp_user(peer_device, false,
+			CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE) == SS_NOTHING_TO_DO) {
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	if (drbd_request_state(adm_ctx.device, NS(user_isp, 0)) == SS_NOTHING_TO_DO) {
-		s = adm_ctx.device->state;
-		if (s.conn == C_PAUSED_SYNC_S || s.conn == C_PAUSED_SYNC_T) {
-			retcode = s.aftr_isp ? ERR_PIC_AFTER_DEP :
-				  s.peer_isp ? ERR_PIC_PEER_DEP : ERR_PAUSE_IS_CLEAR;
+		if (peer_device->repl_state[NOW] == L_PAUSED_SYNC_S ||
+		    peer_device->repl_state[NOW] == L_PAUSED_SYNC_T) {
+			if (peer_device->resync_susp_dependency[NOW])
+				retcode = ERR_PIC_AFTER_DEP;
+			else if (peer_device->resync_susp_peer[NOW])
+				retcode = ERR_PIC_PEER_DEP;
+			else
+				retcode = ERR_PAUSE_IS_CLEAR;
 		} else {
 			retcode = ERR_PAUSE_IS_CLEAR;
 		}
 	}
+
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out:
+ out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info)
+static bool io_drained(struct drbd_device *device)
 {
-	return drbd_adm_simple_request_state(skb, info, NS(susp, 1));
+	struct drbd_peer_device *peer_device;
+	bool drained = true;
+
+	if (atomic_read(&device->local_cnt))
+		return false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (atomic_read(&peer_device->ap_pending_cnt)) {
+			drained = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return drained;
 }
 
-int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
+	struct drbd_resource *resource;
 	struct drbd_device *device;
-	int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+	int retcode, vnr, err = 0;
+	struct suspend_io_parms params = {
+		.bdev_freeze = true,
+	};
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
+	resource = adm_ctx.device->resource;
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	device = adm_ctx.device;
-	if (test_bit(NEW_CUR_UUID, &device->flags)) {
-		if (get_ldev_if_state(device, D_ATTACHING)) {
-			drbd_uuid_new_current(device);
-			put_ldev(device);
-		} else {
-			/* This is effectively a multi-stage "forced down".
-			 * The NEW_CUR_UUID bit is supposedly only set, if we
-			 * lost the replication connection, and are configured
-			 * to freeze IO and wait for some fence-peer handler.
-			 * So we still don't have a replication connection.
-			 * And now we don't have a local disk either.  After
-			 * resume, we will fail all pending and new IO, because
-			 * we don't have any data anymore.  Which means we will
-			 * eventually be able to terminate all users of this
-			 * device, and then take it down.  By bumping the
-			 * "effective" data uuid, we make sure that you really
-			 * need to tear down before you reconfigure, we will
-			 * the refuse to re-connect or re-attach (because no
-			 * matching real data uuid exists).
-			 */
-			u64 val;
-			get_random_bytes(&val, sizeof(u64));
-			drbd_set_ed_uuid(device, val);
-			drbd_warn(device, "Resumed without access to data; please tear down before attempting to re-configure.\n");
+	if (info->attrs[DRBD_NLA_SUSPEND_IO_PARAMS]) {
+		err = suspend_io_parms_from_attrs(&params, info);
+		if (err) {
+			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+			return err;
 		}
-		clear_bit(NEW_CUR_UUID, &device->flags);
 	}
-	drbd_suspend_io(device);
-	retcode = drbd_request_state(device, NS3(susp, 0, susp_nod, 0, susp_fen, 0));
-	if (retcode == SS_SUCCESS) {
-		if (device->state.conn < C_CONNECTED)
-			tl_clear(first_peer_device(device)->connection);
-		if (device->state.disk == D_DISKLESS || device->state.disk == D_FAILED)
-			tl_restart(first_peer_device(device)->connection, FAIL_FROZEN_DISK_IO);
+
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr)
+		if (params.bdev_freeze && !test_bit(BDEV_FROZEN, &device->flags)) {
+			err = bdev_freeze(device->vdisk->part0);
+			if (err)
+				goto out_thaw;
+
+			set_bit(BDEV_FROZEN, &device->flags);
+		}
+
+	retcode = stable_state_change(resource, change_io_susp_user(resource, true,
+						CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE));
+	mutex_unlock(&resource->adm_mutex);
+	if (retcode < SS_SUCCESS)
+		goto out;
+
+	idr_for_each_entry(&resource->devices, device, vnr)
+		wait_event_interruptible(device->misc_wait, io_drained(device));
+out:
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
+out_thaw:
+	idr_for_each_entry(&resource->devices, device, vnr)
+		if (test_and_clear_bit(BDEV_FROZEN, &device->flags))
+			bdev_thaw(device->vdisk->part0);
+
+	mutex_unlock(&resource->adm_mutex);
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return err;
+}
+
+static int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct drbd_connection *connection;
+	struct drbd_resource *resource;
+	struct drbd_device *device;
+	unsigned long irq_flags;
+	int vnr, retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
 	}
+	device = adm_ctx.device;
+	resource = device->resource;
+	if (test_and_clear_bit(NEW_CUR_UUID, &device->flags))
+		drbd_uuid_new_current(device, false);
+	drbd_suspend_io(device, READ_AND_WRITE);
+	begin_state_change(resource, &irq_flags, CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE);
+	__change_io_susp_user(resource, false);
+	__change_io_susp_no_data(resource, false);
+	for_each_connection(connection, resource)
+		__change_io_susp_fencing(connection, false);
+
+	__change_io_susp_quorum(resource, false);
+	retcode = end_state_change(resource, &irq_flags, "resume-io");
 	drbd_resume_io(device);
+
+	idr_for_each_entry(&resource->devices, device, vnr)
+		if (test_and_clear_bit(BDEV_FROZEN, &device->flags))
+			bdev_thaw(device->vdisk->part0);
+
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out:
+ out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info)
 {
-	return drbd_adm_simple_request_state(skb, info, NS(disk, D_OUTDATED));
+	struct drbd_config_context adm_ctx;
+	enum drbd_ret_code retcode;
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+	} else {
+		retcode = stable_state_change(adm_ctx.device->resource,
+			change_disk_state(adm_ctx.device, D_OUTDATED,
+				  CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE, "outdate", NULL));
+		mutex_unlock(&adm_ctx.resource->adm_mutex);
+	}
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
 }
 
 static int nla_put_drbd_cfg_context(struct sk_buff *skb,
 				    struct drbd_resource *resource,
 				    struct drbd_connection *connection,
-				    struct drbd_device *device)
+				    struct drbd_device *device,
+				    struct drbd_path *path)
 {
 	struct nlattr *nla;
 	nla = nla_nest_start_noflag(skb, DRBD_NLA_CFG_CONTEXT);
 	if (!nla)
 		goto nla_put_failure;
-	if (device &&
-	    nla_put_u32(skb, T_ctx_volume, device->vnr))
-		goto nla_put_failure;
-	if (nla_put_string(skb, T_ctx_resource_name, resource->name))
-		goto nla_put_failure;
+	if (device)
+		nla_put_u32(skb, T_ctx_volume, device->vnr);
+	if (resource)
+		nla_put_string(skb, T_ctx_resource_name, resource->name);
 	if (connection) {
-		if (connection->my_addr_len &&
-		    nla_put(skb, T_ctx_my_addr, connection->my_addr_len, &connection->my_addr))
-			goto nla_put_failure;
-		if (connection->peer_addr_len &&
-		    nla_put(skb, T_ctx_peer_addr, connection->peer_addr_len, &connection->peer_addr))
-			goto nla_put_failure;
+		nla_put_u32(skb, T_ctx_peer_node_id, connection->peer_node_id);
+		rcu_read_lock();
+		if (connection->transport.net_conf)
+			nla_put_string(skb, T_ctx_conn_name, connection->transport.net_conf->name);
+		rcu_read_unlock();
+	}
+	if (path) {
+		nla_put(skb, T_ctx_my_addr, path->my_addr_len, &path->my_addr);
+		nla_put(skb, T_ctx_peer_addr, path->peer_addr_len, &path->peer_addr);
 	}
 	nla_nest_end(skb, nla);
 	return 0;
@@ -3250,7 +6001,7 @@ static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr
 
 static void resource_to_info(struct resource_info *, struct drbd_resource *);
 
-int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct drbd_genlmsghdr *dh;
 	struct drbd_resource *resource;
@@ -3285,7 +6036,7 @@ int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
 		goto out;
 	dh->minor = -1U;
 	dh->ret_code = NO_ERROR;
-	err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL);
+	err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL, NULL);
 	if (err)
 		goto out;
 	err = res_opts_to_skb(skb, &resource->res_opts, !capable(CAP_SYS_ADMIN));
@@ -3321,16 +6072,18 @@ static void device_to_statistics(struct device_statistics *s,
 		int n;
 
 		spin_lock_irq(&md->uuid_lock);
-		s->dev_current_uuid = md->uuid[UI_CURRENT];
-		BUILD_BUG_ON(sizeof(s->history_uuids) < UI_HISTORY_END - UI_HISTORY_START + 1);
-		for (n = 0; n < UI_HISTORY_END - UI_HISTORY_START + 1; n++)
-			history_uuids[n] = md->uuid[UI_HISTORY_START + n];
-		for (; n < HISTORY_UUIDS; n++)
-			history_uuids[n] = 0;
-		s->history_uuids_len = HISTORY_UUIDS;
+		s->dev_current_uuid = md->current_uuid;
+		BUILD_BUG_ON(sizeof(s->history_uuids) != sizeof(md->history_uuids));
+		for (n = 0; n < ARRAY_SIZE(md->history_uuids); n++)
+			history_uuids[n] = md->history_uuids[n];
+		s->history_uuids_len = sizeof(s->history_uuids);
 		spin_unlock_irq(&md->uuid_lock);
 
 		s->dev_disk_flags = md->flags;
+		/* originally, this used the bdi congestion framework,
+		 * but that was removed in linux 5.18.
+		 * so just never report the lower device as congested. */
+		s->dev_lower_blocked = false;
 		put_ldev(device);
 	}
 	s->dev_size = get_capacity(device->vdisk);
@@ -3338,10 +6091,11 @@ static void device_to_statistics(struct device_statistics *s,
 	s->dev_write = device->writ_cnt;
 	s->dev_al_writes = device->al_writ_cnt;
 	s->dev_bm_writes = device->bm_writ_cnt;
-	s->dev_upper_pending = atomic_read(&device->ap_bio_cnt);
+	s->dev_upper_pending = atomic_read(&device->ap_bio_cnt[READ]) +
+		atomic_read(&device->ap_bio_cnt[WRITE]);
 	s->dev_lower_pending = atomic_read(&device->local_cnt);
 	s->dev_al_suspended = test_bit(AL_SUSPENDED, &device->flags);
-	s->dev_exposed_data_uuid = device->ed_uuid;
+	s->dev_exposed_data_uuid = device->exposed_data_uuid;
 }
 
 static int put_resource_in_arg0(struct netlink_callback *cb, int holder_nr)
@@ -3355,13 +6109,12 @@ static int put_resource_in_arg0(struct netlink_callback *cb, int holder_nr)
 	return 0;
 }
 
-int drbd_adm_dump_devices_done(struct netlink_callback *cb) {
+static int drbd_adm_dump_devices_done(struct netlink_callback *cb)
+{
 	return put_resource_in_arg0(cb, 7);
 }
 
-static void device_to_info(struct device_info *, struct drbd_device *);
-
-int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource;
@@ -3373,6 +6126,8 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 	struct idr *idr_to_search;
 
 	resource = (struct drbd_resource *)cb->args[0];
+
+	rcu_read_lock();
 	if (!cb->args[0] && !cb->args[1]) {
 		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
 		if (resource_filter) {
@@ -3384,7 +6139,6 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		}
 	}
 
-	rcu_read_lock();
 	minor = cb->args[1];
 	idr_to_search = resource ? &resource->devices : &drbd_devices;
 	device = idr_get_next(idr_to_search, &minor);
@@ -3410,7 +6164,7 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 	dh->minor = -1U;
 	if (retcode == NO_ERROR) {
 		dh->minor = device->minor;
-		err = nla_put_drbd_cfg_context(skb, device->resource, NULL, device);
+		err = nla_put_drbd_cfg_context(skb, device->resource, NULL, device, NULL);
 		if (err)
 			goto out;
 		if (get_ldev(device)) {
@@ -3422,6 +6176,9 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 			if (err)
 				goto out;
 		}
+		err = device_conf_to_skb(skb, &device->device_conf, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
 		device_to_info(&device_info, device);
 		err = device_info_to_skb(skb, &device_info, !capable(CAP_SYS_ADMIN));
 		if (err)
@@ -3443,14 +6200,47 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 	return skb->len;
 }
 
-int drbd_adm_dump_connections_done(struct netlink_callback *cb)
+static int drbd_adm_dump_connections_done(struct netlink_callback *cb)
 {
 	return put_resource_in_arg0(cb, 6);
 }
 
+static int connection_paths_to_skb(struct sk_buff *skb, struct drbd_connection *connection)
+{
+	struct drbd_path *path;
+	struct nlattr *tla = nla_nest_start_noflag(skb, DRBD_NLA_PATH_PARMS);
+	if (!tla)
+		goto nla_put_failure;
+
+	/* array of such paths. */
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &connection->transport.paths, list) {
+		if (nla_put(skb, T_my_addr, path->my_addr_len, &path->my_addr) ||
+				nla_put(skb, T_peer_addr, path->peer_addr_len, &path->peer_addr)) {
+			rcu_read_unlock();
+			goto nla_put_failure;
+		}
+	}
+	rcu_read_unlock();
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+static void connection_to_statistics(struct connection_statistics *s, struct drbd_connection *connection)
+{
+	s->conn_congested = test_bit(NET_CONGESTED, &connection->transport.flags);
+	s->ap_in_flight = atomic_read(&connection->ap_in_flight);
+	s->rs_in_flight = atomic_read(&connection->rs_in_flight);
+}
+
 enum { SINGLE_RESOURCE, ITERATE_RESOURCES };
 
-int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource = NULL, *next_resource;
@@ -3484,7 +6274,13 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 
     next_resource:
 	rcu_read_unlock();
-	mutex_lock(&resource->conf_update);
+	if (mutex_lock_interruptible(&resource->conf_update)) {
+		kref_put(&resource->kref, drbd_destroy_resource);
+		resource = NULL;
+		retcode = ERR_INTR;
+		rcu_read_lock();
+		goto put_result;
+	}
 	rcu_read_lock();
 	if (cb->args[2]) {
 		for_each_connection_rcu(connection, resource)
@@ -3497,8 +6293,6 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 
 found_connection:
 	list_for_each_entry_continue_rcu(connection, &resource->connections, connections) {
-		if (!has_net_conf(connection))
-			continue;
 		retcode = NO_ERROR;
 		goto put_result;  /* only one iteration */
 	}
@@ -3537,20 +6331,21 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 	if (retcode == NO_ERROR) {
 		struct net_conf *net_conf;
 
-		err = nla_put_drbd_cfg_context(skb, resource, connection, NULL);
+		err = nla_put_drbd_cfg_context(skb, resource, connection, NULL, NULL);
 		if (err)
 			goto out;
-		net_conf = rcu_dereference(connection->net_conf);
+		net_conf = rcu_dereference(connection->transport.net_conf);
 		if (net_conf) {
 			err = net_conf_to_skb(skb, net_conf, !capable(CAP_SYS_ADMIN));
 			if (err)
 				goto out;
 		}
 		connection_to_info(&connection_info, connection);
+		connection_paths_to_skb(skb, connection);
 		err = connection_info_to_skb(skb, &connection_info, !capable(CAP_SYS_ADMIN));
 		if (err)
 			goto out;
-		connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
+		connection_to_statistics(&connection_statistics, connection);
 		err = connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
 		if (err)
 			goto out;
@@ -3568,51 +6363,92 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 	return skb->len;
 }
 
-enum mdf_peer_flag {
-	MDF_PEER_CONNECTED =	1 << 0,
-	MDF_PEER_OUTDATED =	1 << 1,
-	MDF_PEER_FENCING =	1 << 2,
-	MDF_PEER_FULL_SYNC =	1 << 3,
-};
-
 static void peer_device_to_statistics(struct peer_device_statistics *s,
-				      struct drbd_peer_device *peer_device)
+				      struct drbd_peer_device *pd)
 {
-	struct drbd_device *device = peer_device->device;
+	struct drbd_device *device = pd->device;
+	struct drbd_md *md;
+	struct drbd_peer_md *peer_md;
+	struct drbd_bitmap *bm;
+	unsigned long now = jiffies;
+	unsigned long rs_left = 0;
+	int i;
+
+	/* userspace should get "future proof" units,
+	 * convert to sectors or milli seconds as appropriate */
 
 	memset(s, 0, sizeof(*s));
-	s->peer_dev_received = device->recv_cnt;
-	s->peer_dev_sent = device->send_cnt;
-	s->peer_dev_pending = atomic_read(&device->ap_pending_cnt) +
-			      atomic_read(&device->rs_pending_cnt);
-	s->peer_dev_unacked = atomic_read(&device->unacked_cnt);
-	s->peer_dev_out_of_sync = drbd_bm_total_weight(device) << (BM_BLOCK_SHIFT - 9);
-	s->peer_dev_resync_failed = device->rs_failed << (BM_BLOCK_SHIFT - 9);
-	if (get_ldev(device)) {
-		struct drbd_md *md = &device->ldev->md;
+	s->peer_dev_received = pd->recv_cnt;
+	s->peer_dev_sent = pd->send_cnt;
+	s->peer_dev_pending = atomic_read(&pd->ap_pending_cnt) +
+			      atomic_read(&pd->rs_pending_cnt);
+	s->peer_dev_unacked = atomic_read(&pd->unacked_cnt);
+	s->peer_dev_uuid_flags = pd->uuid_flags;
+
+	/* Below are resync / verify / bitmap / meta data stats.
+	 * Without disk, we don't have those.
+	 */
+	if (!get_ldev(device))
+		return;
 
-		spin_lock_irq(&md->uuid_lock);
-		s->peer_dev_bitmap_uuid = md->uuid[UI_BITMAP];
-		spin_unlock_irq(&md->uuid_lock);
-		s->peer_dev_flags =
-			(drbd_md_test_flag(device->ldev, MDF_CONNECTED_IND) ?
-				MDF_PEER_CONNECTED : 0) +
-			(drbd_md_test_flag(device->ldev, MDF_CONSISTENT) &&
-			 !drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE) ?
-				MDF_PEER_OUTDATED : 0) +
-			/* FIXME: MDF_PEER_FENCING? */
-			(drbd_md_test_flag(device->ldev, MDF_FULL_SYNC) ?
-				MDF_PEER_FULL_SYNC : 0);
-		put_ldev(device);
+	bm = device->bitmap;
+	s->peer_dev_out_of_sync = bm_bit_to_sect(bm, drbd_bm_total_weight(pd));
+
+	if (is_verify_state(pd, NOW)) {
+		rs_left = bm_bit_to_sect(bm, atomic64_read(&pd->ov_left));
+		s->peer_dev_ov_start_sector = pd->ov_start_sector;
+		s->peer_dev_ov_stop_sector = pd->ov_stop_sector;
+		s->peer_dev_ov_position = pd->ov_position;
+		s->peer_dev_ov_left = bm_bit_to_sect(bm, atomic64_read(&pd->ov_left));
+		s->peer_dev_ov_skipped = bm_bit_to_sect(bm, pd->ov_skipped);
+	} else if (is_sync_state(pd, NOW)) {
+		rs_left = s->peer_dev_out_of_sync - bm_bit_to_sect(bm, pd->rs_failed);
+		s->peer_dev_resync_failed = bm_bit_to_sect(bm, pd->rs_failed);
+		s->peer_dev_rs_same_csum = bm_bit_to_sect(bm, pd->rs_same_csum);
+	}
+
+	if (rs_left) {
+		enum drbd_repl_state repl_state = pd->repl_state[NOW];
+		if (repl_state == L_SYNC_TARGET || repl_state == L_VERIFY_S)
+			s->peer_dev_rs_c_sync_rate = pd->c_sync_rate;
+
+		s->peer_dev_rs_total = bm_bit_to_sect(bm, pd->rs_total);
+
+		s->peer_dev_rs_dt_start_ms = jiffies_to_msecs(now - pd->rs_start);
+		s->peer_dev_rs_paused_ms = jiffies_to_msecs(pd->rs_paused);
+
+		i = (pd->rs_last_mark + 2) % DRBD_SYNC_MARKS;
+		s->peer_dev_rs_dt0_ms = jiffies_to_msecs(now - pd->rs_mark_time[i]);
+		s->peer_dev_rs_db0_sectors = bm_bit_to_sect(bm, pd->rs_mark_left[i]) - rs_left;
+
+		i = (pd->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
+		s->peer_dev_rs_dt1_ms = jiffies_to_msecs(now - pd->rs_mark_time[i]);
+		s->peer_dev_rs_db1_sectors = bm_bit_to_sect(bm, pd->rs_mark_left[i]) - rs_left;
+
+		/* long term average:
+		 * dt = rs_dt_start_ms - rs_paused_ms;
+		 * db = rs_total - rs_left, which is
+		 *   rs_total - (ov_left? ov_left : out_of_sync - rs_failed)
+		 */
 	}
+
+	md = &device->ldev->md;
+	peer_md = &md->peers[pd->node_id];
+
+	spin_lock_irq(&md->uuid_lock);
+	s->peer_dev_bitmap_uuid = peer_md->bitmap_uuid;
+	spin_unlock_irq(&md->uuid_lock);
+	s->peer_dev_flags = peer_md->flags;
+
+	put_ldev(device);
 }
 
-int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb)
+static int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb)
 {
 	return put_resource_in_arg0(cb, 9);
 }
 
-int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource;
@@ -3623,6 +6459,8 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 	struct idr *idr_to_search;
 
 	resource = (struct drbd_resource *)cb->args[0];
+
+	rcu_read_lock();
 	if (!cb->args[0] && !cb->args[1]) {
 		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
 		if (resource_filter) {
@@ -3634,7 +6472,6 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		cb->args[0] = (long)resource;
 	}
 
-	rcu_read_lock();
 	minor = cb->args[1];
 	idr_to_search = resource ? &resource->devices : &drbd_devices;
 	device = idr_find(idr_to_search, minor);
@@ -3649,7 +6486,7 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		}
 	}
 	if (cb->args[2]) {
-		for_each_peer_device(peer_device, device)
+		for_each_peer_device_rcu(peer_device, device)
 			if (peer_device == (struct drbd_peer_device *)cb->args[2])
 				goto found_peer_device;
 		/* peer device was probably deleted */
@@ -3660,8 +6497,6 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 
 found_peer_device:
 	list_for_each_entry_continue_rcu(peer_device, &device->peer_devices, peer_devices) {
-		if (!has_net_conf(peer_device->connection))
-			continue;
 		retcode = NO_ERROR;
 		goto put_result;  /* only one iteration */
 	}
@@ -3679,9 +6514,10 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 	if (retcode == NO_ERROR) {
 		struct peer_device_info peer_device_info;
 		struct peer_device_statistics peer_device_statistics;
+		struct peer_device_conf *peer_device_conf;
 
 		dh->minor = minor;
-		err = nla_put_drbd_cfg_context(skb, device->resource, peer_device->connection, device);
+		err = nla_put_drbd_cfg_context(skb, device->resource, peer_device->connection, device, NULL);
 		if (err)
 			goto out;
 		peer_device_to_info(&peer_device_info, peer_device);
@@ -3692,6 +6528,13 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		err = peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
 		if (err)
 			goto out;
+		peer_device_conf = rcu_dereference(peer_device->conf);
+		if (peer_device_conf) {
+			err = peer_device_conf_to_skb(skb, peer_device_conf, !capable(CAP_SYS_ADMIN));
+			if (err)
+				goto out;
+		}
+
 		cb->args[1] = minor;
 		cb->args[2] = (long)peer_device;
 	}
@@ -3704,362 +6547,150 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		return err;
 	return skb->len;
 }
-/*
- * Return the connection of @resource if @resource has exactly one connection.
- */
-static struct drbd_connection *the_only_connection(struct drbd_resource *resource)
-{
-	struct list_head *connections = &resource->connections;
-
-	if (list_empty(connections) || connections->next->next != connections)
-		return NULL;
-	return list_first_entry(&resource->connections, struct drbd_connection, connections);
-}
-
-static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
-		const struct sib_info *sib)
-{
-	struct drbd_resource *resource = device->resource;
-	struct state_info *si = NULL; /* for sizeof(si->member); */
-	struct nlattr *nla;
-	int got_ldev;
-	int err = 0;
-	int exclude_sensitive;
-
-	/* If sib != NULL, this is drbd_bcast_event, which anyone can listen
-	 * to.  So we better exclude_sensitive information.
-	 *
-	 * If sib == NULL, this is drbd_adm_get_status, executed synchronously
-	 * in the context of the requesting user process. Exclude sensitive
-	 * information, unless current has superuser.
-	 *
-	 * NOTE: for drbd_adm_get_status_all(), this is a netlink dump, and
-	 * relies on the current implementation of netlink_dump(), which
-	 * executes the dump callback successively from netlink_recvmsg(),
-	 * always in the context of the receiving process */
-	exclude_sensitive = sib || !capable(CAP_SYS_ADMIN);
-
-	got_ldev = get_ldev(device);
-
-	/* We need to add connection name and volume number information still.
-	 * Minor number is in drbd_genlmsghdr. */
-	if (nla_put_drbd_cfg_context(skb, resource, the_only_connection(resource), device))
-		goto nla_put_failure;
-
-	if (res_opts_to_skb(skb, &device->resource->res_opts, exclude_sensitive))
-		goto nla_put_failure;
-
-	rcu_read_lock();
-	if (got_ldev) {
-		struct disk_conf *disk_conf;
-
-		disk_conf = rcu_dereference(device->ldev->disk_conf);
-		err = disk_conf_to_skb(skb, disk_conf, exclude_sensitive);
-	}
-	if (!err) {
-		struct net_conf *nc;
-
-		nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-		if (nc)
-			err = net_conf_to_skb(skb, nc, exclude_sensitive);
-	}
-	rcu_read_unlock();
-	if (err)
-		goto nla_put_failure;
-
-	nla = nla_nest_start_noflag(skb, DRBD_NLA_STATE_INFO);
-	if (!nla)
-		goto nla_put_failure;
-	if (nla_put_u32(skb, T_sib_reason, sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) ||
-	    nla_put_u32(skb, T_current_state, device->state.i) ||
-	    nla_put_u64_0pad(skb, T_ed_uuid, device->ed_uuid) ||
-	    nla_put_u64_0pad(skb, T_capacity, get_capacity(device->vdisk)) ||
-	    nla_put_u64_0pad(skb, T_send_cnt, device->send_cnt) ||
-	    nla_put_u64_0pad(skb, T_recv_cnt, device->recv_cnt) ||
-	    nla_put_u64_0pad(skb, T_read_cnt, device->read_cnt) ||
-	    nla_put_u64_0pad(skb, T_writ_cnt, device->writ_cnt) ||
-	    nla_put_u64_0pad(skb, T_al_writ_cnt, device->al_writ_cnt) ||
-	    nla_put_u64_0pad(skb, T_bm_writ_cnt, device->bm_writ_cnt) ||
-	    nla_put_u32(skb, T_ap_bio_cnt, atomic_read(&device->ap_bio_cnt)) ||
-	    nla_put_u32(skb, T_ap_pending_cnt, atomic_read(&device->ap_pending_cnt)) ||
-	    nla_put_u32(skb, T_rs_pending_cnt, atomic_read(&device->rs_pending_cnt)))
-		goto nla_put_failure;
-
-	if (got_ldev) {
-		int err;
-
-		spin_lock_irq(&device->ldev->md.uuid_lock);
-		err = nla_put(skb, T_uuids, sizeof(si->uuids), device->ldev->md.uuid);
-		spin_unlock_irq(&device->ldev->md.uuid_lock);
-
-		if (err)
-			goto nla_put_failure;
-
-		if (nla_put_u32(skb, T_disk_flags, device->ldev->md.flags) ||
-		    nla_put_u64_0pad(skb, T_bits_total, drbd_bm_bits(device)) ||
-		    nla_put_u64_0pad(skb, T_bits_oos,
-				     drbd_bm_total_weight(device)))
-			goto nla_put_failure;
-		if (C_SYNC_SOURCE <= device->state.conn &&
-		    C_PAUSED_SYNC_T >= device->state.conn) {
-			if (nla_put_u64_0pad(skb, T_bits_rs_total,
-					     device->rs_total) ||
-			    nla_put_u64_0pad(skb, T_bits_rs_failed,
-					     device->rs_failed))
-				goto nla_put_failure;
-		}
-	}
-
-	if (sib) {
-		switch(sib->sib_reason) {
-		case SIB_SYNC_PROGRESS:
-		case SIB_GET_STATUS_REPLY:
-			break;
-		case SIB_STATE_CHANGE:
-			if (nla_put_u32(skb, T_prev_state, sib->os.i) ||
-			    nla_put_u32(skb, T_new_state, sib->ns.i))
-				goto nla_put_failure;
-			break;
-		case SIB_HELPER_POST:
-			if (nla_put_u32(skb, T_helper_exit_code,
-					sib->helper_exit_code))
-				goto nla_put_failure;
-			fallthrough;
-		case SIB_HELPER_PRE:
-			if (nla_put_string(skb, T_helper, sib->helper_name))
-				goto nla_put_failure;
-			break;
-		}
-	}
-	nla_nest_end(skb, nla);
-
-	if (0)
-nla_put_failure:
-		err = -EMSGSIZE;
-	if (got_ldev)
-		put_ldev(device);
-	return err;
-}
 
-int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_dump_paths_done(struct netlink_callback *cb)
 {
-	struct drbd_config_context adm_ctx;
-	enum drbd_ret_code retcode;
-	int err;
-
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
-	if (!adm_ctx.reply_skb)
-		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
-
-	err = nla_put_status_info(adm_ctx.reply_skb, adm_ctx.device, NULL);
-	if (err) {
-		nlmsg_free(adm_ctx.reply_skb);
-		return err;
-	}
-out:
-	drbd_adm_finish(&adm_ctx, info, retcode);
-	return 0;
+	return put_resource_in_arg0(cb, 10);
 }
 
-static int get_one_status(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_dump_paths(struct sk_buff *skb, struct netlink_callback *cb)
 {
-	struct drbd_device *device;
+	struct nlattr *resource_filter;
+	struct drbd_resource *resource = NULL, *next_resource;
+	struct drbd_connection *connection = NULL;
+	struct drbd_path *path = NULL;
+	int err = 0, retcode;
 	struct drbd_genlmsghdr *dh;
-	struct drbd_resource *pos = (struct drbd_resource *)cb->args[0];
-	struct drbd_resource *resource = NULL;
-	struct drbd_resource *tmp;
-	unsigned volume = cb->args[1];
-
-	/* Open coded, deferred, iteration:
-	 * for_each_resource_safe(resource, tmp, &drbd_resources) {
-	 *      connection = "first connection of resource or undefined";
-	 *	idr_for_each_entry(&resource->devices, device, i) {
-	 *	  ...
-	 *	}
-	 * }
-	 * where resource is cb->args[0];
-	 * and i is cb->args[1];
-	 *
-	 * cb->args[2] indicates if we shall loop over all resources,
-	 * or just dump all volumes of a single resource.
-	 *
-	 * This may miss entries inserted after this dump started,
-	 * or entries deleted before they are reached.
-	 *
-	 * We need to make sure the device won't disappear while
-	 * we are looking at it, and revalidate our iterators
-	 * on each iteration.
-	 */
 
-	/* synchronize with conn_create()/drbd_destroy_connection() */
 	rcu_read_lock();
-	/* revalidate iterator position */
-	for_each_resource_rcu(tmp, &drbd_resources) {
-		if (pos == NULL) {
-			/* first iteration */
-			pos = tmp;
-			resource = pos;
-			break;
-		}
-		if (tmp == pos) {
-			resource = pos;
-			break;
+	resource = (struct drbd_resource *)cb->args[0];
+	if (!cb->args[0]) {
+		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		if (resource_filter) {
+			retcode = ERR_RES_NOT_KNOWN;
+			resource = drbd_find_resource(nla_data(resource_filter));
+			if (!resource)
+				goto put_result;
+			cb->args[0] = (long)resource;
+			cb->args[1] = SINGLE_RESOURCE;
 		}
 	}
-	if (resource) {
-next_resource:
-		device = idr_get_next(&resource->devices, &volume);
-		if (!device) {
-			/* No more volumes to dump on this resource.
-			 * Advance resource iterator. */
-			pos = list_entry_rcu(resource->resources.next,
-					     struct drbd_resource, resources);
-			/* Did we dump any volume of this resource yet? */
-			if (volume != 0) {
-				/* If we reached the end of the list,
-				 * or only a single resource dump was requested,
-				 * we are done. */
-				if (&pos->resources == &drbd_resources || cb->args[2])
-					goto out;
-				volume = 0;
-				resource = pos;
-				goto next_resource;
-			}
-		}
-
-		dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-				cb->nlh->nlmsg_seq, &drbd_genl_family,
-				NLM_F_MULTI, DRBD_ADM_GET_STATUS);
-		if (!dh)
+	if (!resource) {
+		if (list_empty(&drbd_resources))
 			goto out;
+		resource = list_first_entry(&drbd_resources, struct drbd_resource, resources);
+		kref_get(&resource->kref);
+		cb->args[0] = (long)resource;
+		cb->args[1] = ITERATE_RESOURCES;
+	}
 
-		if (!device) {
-			/* This is a connection without a single volume.
-			 * Suprisingly enough, it may have a network
-			 * configuration. */
-			struct drbd_connection *connection;
-
-			dh->minor = -1U;
-			dh->ret_code = NO_ERROR;
-			connection = the_only_connection(resource);
-			if (nla_put_drbd_cfg_context(skb, resource, connection, NULL))
-				goto cancel;
-			if (connection) {
-				struct net_conf *nc;
-
-				nc = rcu_dereference(connection->net_conf);
-				if (nc && net_conf_to_skb(skb, nc, 1) != 0)
-					goto cancel;
-			}
-			goto done;
-		}
-
-		D_ASSERT(device, device->vnr == volume);
-		D_ASSERT(device, device->resource == resource);
-
-		dh->minor = device_to_minor(device);
-		dh->ret_code = NO_ERROR;
-
-		if (nla_put_status_info(skb, device, NULL)) {
-cancel:
-			genlmsg_cancel(skb, dh);
-			goto out;
+next_resource:
+	rcu_read_unlock();
+	mutex_lock(&resource->conf_update);
+	rcu_read_lock();
+	if (cb->args[2]) {
+		for_each_connection_rcu(connection, resource) {
+			list_for_each_entry_rcu(path, &connection->transport.paths, list)
+				if (path == (struct drbd_path *)cb->args[2])
+					goto found_path;
 		}
-done:
-		genlmsg_end(skb, dh);
+		/* path was probably deleted */
+		goto no_more_paths;
 	}
 
-out:
-	rcu_read_unlock();
-	/* where to start the next iteration */
-	cb->args[0] = (long)pos;
-	cb->args[1] = (pos == resource) ? volume + 1 : 0;
+	connection = first_connection(resource);
+	if (!connection)
+		goto no_more_paths;
 
-	/* No more resources/volumes/minors found results in an empty skb.
-	 * Which will terminate the dump. */
-        return skb->len;
-}
+	path = list_entry(&connection->transport.paths, struct drbd_path, list);
 
-/*
- * Request status of all resources, or of all volumes within a single resource.
- *
- * This is a dump, as the answer may not fit in a single reply skb otherwise.
- * Which means we cannot use the family->attrbuf or other such members, because
- * dump is NOT protected by the genl_lock().  During dump, we only have access
- * to the incoming skb, and need to opencode "parsing" of the nlattr payload.
- *
- * Once things are setup properly, we call into get_one_status().
- */
-int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb)
-{
-	const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ;
-	struct nlattr *nla;
-	const char *resource_name;
-	struct drbd_resource *resource;
-	int maxtype;
+found_path:
+	/* Advance to next path in connection. */
+	list_for_each_entry_continue_rcu(path, &connection->transport.paths, list) {
+		retcode = NO_ERROR;
+		goto put_result;  /* only one iteration */
+	}
 
-	/* Is this a followup call? */
-	if (cb->args[0]) {
-		/* ... of a single resource dump,
-		 * and the resource iterator has been advanced already? */
-		if (cb->args[2] && cb->args[2] != cb->args[0])
-			return 0; /* DONE. */
-		goto dump;
+	/* Advance to next connection. */
+	list_for_each_entry_continue_rcu(connection, &resource->connections, connections) {
+		path = first_path(connection);
+		if (!path)
+			continue;
+		retcode = NO_ERROR;
+		goto put_result;
 	}
 
-	/* First call (from netlink_dump_start).  We need to figure out
-	 * which resource(s) the user wants us to dump. */
-	nla = nla_find(nlmsg_attrdata(cb->nlh, hdrlen),
-			nlmsg_attrlen(cb->nlh, hdrlen),
-			DRBD_NLA_CFG_CONTEXT);
+no_more_paths:
+	if (cb->args[1] == ITERATE_RESOURCES) {
+		for_each_resource_rcu(next_resource, &drbd_resources) {
+			if (next_resource == resource)
+				goto found_resource;
+		}
+		/* resource was probably deleted */
+	}
+	goto out;
 
-	/* No explicit context given.  Dump all. */
-	if (!nla)
-		goto dump;
-	maxtype = ARRAY_SIZE(drbd_cfg_context_nl_policy) - 1;
-	nla = drbd_nla_find_nested(maxtype, nla, __nla_type(T_ctx_resource_name));
-	if (IS_ERR(nla))
-		return PTR_ERR(nla);
-	/* context given, but no name present? */
-	if (!nla)
-		return -EINVAL;
-	resource_name = nla_data(nla);
-	if (!*resource_name)
-		return -ENODEV;
-	resource = drbd_find_resource(resource_name);
-	if (!resource)
-		return -ENODEV;
+found_resource:
+	list_for_each_entry_continue_rcu(next_resource, &drbd_resources, resources) {
+		mutex_unlock(&resource->conf_update);
+		kref_put(&resource->kref, drbd_destroy_resource);
+		resource = next_resource;
+		kref_get(&resource->kref);
+		cb->args[0] = (long)resource;
+		cb->args[2] = 0;
+		goto next_resource;
+	}
+	goto out;  /* no more resources */
 
-	kref_put(&resource->kref, drbd_destroy_resource); /* get_one_status() revalidates the resource */
+put_result:
+	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			NLM_F_MULTI, DRBD_ADM_GET_PATHS);
+	err = -ENOMEM;
+	if (!dh)
+		goto out;
+	dh->ret_code = retcode;
+	dh->minor = -1U;
+	if (retcode == NO_ERROR && connection && path) {
+		struct drbd_path_info path_info;
 
-	/* prime iterators, and set "filter" mode mark:
-	 * only dump this connection. */
-	cb->args[0] = (long)resource;
-	/* cb->args[1] = 0; passed in this way. */
-	cb->args[2] = (long)resource;
+		err = nla_put_drbd_cfg_context(skb, resource, connection, NULL, path);
+		if (err)
+			goto out;
+		path_info.path_established = test_bit(TR_ESTABLISHED, &path->flags);
+		err = drbd_path_info_to_skb(skb, &path_info, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto out;
+		cb->args[2] = (long)path;
+	}
+	genlmsg_end(skb, dh);
+	err = 0;
 
-dump:
-	return get_one_status(skb, cb);
+out:
+	rcu_read_unlock();
+	if (resource)
+		mutex_unlock(&resource->conf_update);
+	if (err)
+		return err;
+	return skb->len;
 }
 
-int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
+	struct drbd_peer_device *peer_device;
 	enum drbd_ret_code retcode;
 	struct timeout_parms tp;
 	int err;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
+	peer_device = adm_ctx.peer_device;
 
 	tp.timeout_type =
-		adm_ctx.device->state.pdsk == D_OUTDATED ? UT_PEER_OUTDATED :
-		test_bit(USE_DEGR_WFC_T, &adm_ctx.device->flags) ? UT_DEGRADED :
+		peer_device->disk_state[NOW] == D_OUTDATED ? UT_PEER_OUTDATED :
+		test_bit(USE_DEGR_WFC_T, &peer_device->flags) ? UT_DEGRADED :
 		UT_DEFAULT;
 
 	err = timeout_parms_to_priv_skb(adm_ctx.reply_skb, &tp);
@@ -4067,28 +6698,29 @@ int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
 		nlmsg_free(adm_ctx.reply_skb);
 		return err;
 	}
-out:
+
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	struct drbd_device *device;
+	struct drbd_peer_device *peer_device;
 	enum drbd_ret_code retcode;
+	enum drbd_state_rv rv;
 	struct start_ov_parms parms;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_PEER_DEVICE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
 
-	device = adm_ctx.device;
+	peer_device = adm_ctx.peer_device;
+	device = peer_device->device;
 
 	/* resume from last known position, if possible */
-	parms.ov_start_sector = device->ov_start_sector;
+	parms.ov_start_sector = peer_device->ov_start_sector;
 	parms.ov_stop_sector = ULLONG_MAX;
 	if (info->attrs[DRBD_NLA_START_OV_PARMS]) {
 		int err = start_ov_parms_from_attrs(&parms, info);
@@ -4098,40 +6730,51 @@ int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 	}
-	mutex_lock(&adm_ctx.resource->adm_mutex);
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out;
+	}
 
 	/* w_make_ov_request expects position to be aligned */
-	device->ov_start_sector = parms.ov_start_sector & ~(BM_SECT_PER_BIT-1);
-	device->ov_stop_sector = parms.ov_stop_sector;
+	peer_device->ov_start_sector = parms.ov_start_sector & ~(bm_sect_per_bit(device->bitmap)-1);
+	peer_device->ov_stop_sector = parms.ov_stop_sector;
 
 	/* If there is still bitmap IO pending, e.g. previous resync or verify
 	 * just being finished, wait for it before requesting a new resync. */
-	drbd_suspend_io(device);
-	wait_event(device->misc_wait, !test_bit(BITMAP_IO, &device->flags));
-	retcode = drbd_request_state(device, NS(conn, C_VERIFY_S));
+	drbd_suspend_io(device, READ_AND_WRITE);
+	wait_event(device->misc_wait, !atomic_read(&device->pending_bitmap_work.n));
+	rv = stable_change_repl_state(peer_device,
+		L_VERIFY_S, CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE, "verify");
 	drbd_resume_io(device);
 
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
+	drbd_adm_finish(&adm_ctx, info, rv);
+	return 0;
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
+static bool should_skip_initial_sync(struct drbd_peer_device *peer_device)
+{
+	return peer_device->repl_state[NOW] == L_ESTABLISHED &&
+	       peer_device->connection->agreed_pro_version >= 90 &&
+	       drbd_current_uuid(peer_device->device) == UUID_JUST_CREATED;
+}
 
-int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	struct drbd_device *device;
+	struct drbd_peer_device *peer_device;
 	enum drbd_ret_code retcode;
-	int skip_initial_sync = 0;
 	int err;
 	struct new_c_uuid_parms args;
+	u64 nodes = 0, diskful = 0;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out_nolock;
 
 	device = adm_ctx.device;
 	memset(&args, 0, sizeof(args));
@@ -4140,12 +6783,18 @@ int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
 		if (err) {
 			retcode = ERR_MANDATORY_TAG;
 			drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
-			goto out_nolock;
+			goto out_no_adm_mutex;
 		}
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	mutex_lock(device->state_mutex); /* Protects us against serialized state changes. */
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
+	if (down_interruptible(&device->resource->state_sem)) {
+		retcode = ERR_INTR;
+		goto out_no_state_sem;
+	}
 
 	if (!get_ldev(device)) {
 		retcode = ERR_NO_DISK;
@@ -4153,148 +6802,323 @@ int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	/* this is "skip initial sync", assume to be clean */
-	if (device->state.conn == C_CONNECTED &&
-	    first_peer_device(device)->connection->agreed_pro_version >= 90 &&
-	    device->ldev->md.uuid[UI_CURRENT] == UUID_JUST_CREATED && args.clear_bm) {
-		drbd_info(device, "Preparing to skip initial sync\n");
-		skip_initial_sync = 1;
-	} else if (device->state.conn != C_STANDALONE) {
-		retcode = ERR_CONNECTED;
-		goto out_dec;
+	for_each_peer_device(peer_device, device) {
+		if ((args.clear_bm || args.force_resync) && should_skip_initial_sync(peer_device)) {
+			if (peer_device->disk_state[NOW] >= D_INCONSISTENT) {
+				drbd_info(peer_device, "Preparing to %s initial sync\n",
+					  args.clear_bm ? "skip" : "force");
+				diskful |= NODE_MASK(peer_device->node_id);
+			}
+			nodes |= NODE_MASK(peer_device->node_id);
+		} else if (peer_device->repl_state[NOW] != L_OFF) {
+			retcode = ERR_CONNECTED;
+			goto out_dec;
+		}
 	}
 
-	drbd_uuid_set(device, UI_BITMAP, 0); /* Rotate UI_BITMAP to History 1, etc... */
-	drbd_uuid_new_current(device); /* New current, previous to UI_BITMAP */
+	drbd_uuid_new_current_by_user(device); /* New current, previous to UI_BITMAP */
+
+	if (args.force_resync) {
+		unsigned long irq_flags;
+		begin_state_change(device->resource, &irq_flags, CS_VERBOSE);
+		__change_disk_state(device, D_UP_TO_DATE);
+		end_state_change(device->resource, &irq_flags, "new-c-uuid");
+
+		for_each_peer_device(peer_device, device) {
+			if (NODE_MASK(peer_device->node_id) & nodes) {
+				if (NODE_MASK(peer_device->node_id) & diskful) {
+					drbd_info(peer_device, "Forcing resync");
+					set_bit(CONSIDER_RESYNC, &peer_device->flags);
+					drbd_send_uuids(peer_device, UUID_FLAG_RESYNC, 0);
+					drbd_send_current_state(peer_device);
+				} else {
+					drbd_send_uuids(peer_device, 0, 0);
+				}
+
+				drbd_print_uuids(peer_device, "forced resync UUID");
+			}
+		}
+	}
 
 	if (args.clear_bm) {
-		err = drbd_bitmap_io(device, &drbd_bmio_clear_n_write,
-			"clear_n_write from new_c_uuid", BM_LOCKED_MASK, NULL);
+		unsigned long irq_flags;
+
+		err = drbd_bitmap_io(device, &drbd_bmio_clear_all_n_write,
+			"clear_n_write from new_c_uuid", BM_LOCK_ALL, NULL);
 		if (err) {
 			drbd_err(device, "Writing bitmap failed with %d\n", err);
 			retcode = ERR_IO_MD_DISK;
 		}
-		if (skip_initial_sync) {
-			drbd_send_uuids_skip_initial_sync(first_peer_device(device));
-			_drbd_uuid_set(device, UI_BITMAP, 0);
-			drbd_print_uuids(device, "cleared bitmap UUID");
-			spin_lock_irq(&device->resource->req_lock);
-			_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),
-					CS_VERBOSE, NULL);
-			spin_unlock_irq(&device->resource->req_lock);
+		for_each_peer_device(peer_device, device) {
+			if (NODE_MASK(peer_device->node_id) & nodes) {
+				_drbd_uuid_set_bitmap(peer_device, 0);
+				drbd_send_uuids(peer_device, UUID_FLAG_SKIP_INITIAL_SYNC, 0);
+				drbd_print_uuids(peer_device, "cleared bitmap UUID");
+			}
+		}
+		begin_state_change(device->resource, &irq_flags, CS_VERBOSE);
+		__change_disk_state(device, D_UP_TO_DATE);
+		for_each_peer_device(peer_device, device) {
+			if (NODE_MASK(peer_device->node_id) & diskful)
+				__change_peer_disk_state(peer_device, D_UP_TO_DATE);
 		}
+		end_state_change(device->resource, &irq_flags, "new-c-uuid");
 	}
 
-	drbd_md_sync(device);
+	drbd_md_sync_if_dirty(device);
 out_dec:
 	put_ldev(device);
 out:
-	mutex_unlock(device->state_mutex);
+	up(&device->resource->state_sem);
+out_no_state_sem:
 	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out_nolock:
+out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static enum drbd_ret_code
-drbd_check_resource_name(struct drbd_config_context *adm_ctx)
+/* name: a resource or connection name
+ * Comes from a NLA_NUL_STRING, and already passed validate_nla().
+ * It is known to be NUL-terminated within the bounds of our defined netlink
+ * attribute policy.
+ *
+ * It must not be empty.
+ * It must not be the literal "all".
+ *
+ * If strict:
+ * Only allow strict ascii alnum [0-9A-Za-z]
+ * and some hand selected punctuation characters
+ *
+ * If non strict:
+ * It must not contain '/', we use it as directory name in debugfs.
+ * It shall not contain "control characters" or space, as those may confuse
+ * utils when trying to parse the output of "drbdsetup events2" or similar.
+ * Otherwise, we don't care, it may be any tag that makes sense to userland,
+ * we do not enforce strict ascii or any other "encoding".
+ */
+static enum drbd_ret_code drbd_check_name_str(const char *name, const bool strict)
 {
-	const char *name = adm_ctx->resource_name;
-	if (!name || !name[0]) {
-		drbd_msg_put_info(adm_ctx->reply_skb, "resource name missing");
+	unsigned char c;
+	if (name == NULL || name[0] == 0)
 		return ERR_MANDATORY_TAG;
-	}
-	/* if we want to use these in sysfs/configfs/debugfs some day,
-	 * we must not allow slashes */
-	if (strchr(name, '/')) {
-		drbd_msg_put_info(adm_ctx->reply_skb, "invalid resource name");
+
+	/* Tools reserve the literal "all" to mean what you would expect. */
+	/* If we want to get really paranoid,
+	 * we could add a number of "reserved" names,
+	 * like the *_state_names defined in drbd_strings.c */
+	if (memcmp("all", name, 4) == 0)
 		return ERR_INVALID_REQUEST;
+
+	while ((c = *name++)) {
+		if (c == '/' || c <= ' ' || c == '\x7f')
+			return ERR_INVALID_REQUEST;
+		if (strict) {
+			switch (c) {
+			case '0' ... '9':
+			case 'A' ... 'Z':
+			case 'a' ... 'z':
+				/* if you change this, also change "strict_pattern" below */
+			case '+': case '-': case '.': case '_':
+				break;
+			default:
+				return ERR_INVALID_REQUEST;
+			}
+		}
 	}
 	return NO_ERROR;
 }
 
+int param_set_drbd_strict_names(const char *val, const struct kernel_param *kp)
+{
+	int err = 0;
+	bool new_value;
+	bool orig_value = *(bool *)kp->arg;
+	struct kernel_param dummy_kp = *kp;
+
+	dummy_kp.arg = &new_value;
+
+	err = param_set_bool(val, &dummy_kp);
+	if (err || new_value == orig_value)
+		return err;
+
+	if (new_value) {
+		struct drbd_resource *resource;
+		struct drbd_connection *connection;
+		int non_strict_cnt = 0;
+
+		/* If we transition from "not enforced" to "enforcing strict names",
+		 * we complain about all "non-strict names" that still exist,
+		 * but intentionally still enable the enforcing.
+		 *
+		 * That way we can prevent new "non-strict" from being created,
+		 * while allowing us to clean up the existing ones at some
+		 * "convenient time" later.
+		 */
+		rcu_read_lock();
+		for_each_resource_rcu(resource, &drbd_resources) {
+			for_each_connection_rcu(connection, resource) {
+				char *name = connection->transport.net_conf->name;
+				if (drbd_check_name_str(name, true) == NO_ERROR)
+					continue;
+				drbd_info(connection, "non-strict name still in use\n");
+				++non_strict_cnt;
+			}
+			if (drbd_check_name_str(resource->name, true) == NO_ERROR)
+				continue;
+			drbd_info(resource, "non-strict name still in use\n");
+			++non_strict_cnt;
+		}
+		rcu_read_unlock();
+		if (non_strict_cnt)
+			pr_notice("%u non-strict names still in use\n", non_strict_cnt);
+	}
+	if (!err) {
+		*(bool *)kp->arg = new_value;
+		pr_info("%s strict name checks\n", new_value ? "enabled" : "disabled");
+	}
+	return err;
+}
+
+static void drbd_msg_put_name_error(struct sk_buff *reply_skb, enum drbd_ret_code ret_code)
+{
+	char *strict_pattern = " (strict_names=1 allows only [0-9A-Za-z+._-])";
+	char *non_strict_pat = " (disallowed: ascii control, space, slash)";
+	if (ret_code == NO_ERROR)
+		return;
+	if (ret_code == ERR_INVALID_REQUEST) {
+		drbd_msg_sprintf_info(reply_skb, "invalid name%s",
+			drbd_strict_names ? strict_pattern : non_strict_pat);
+	} else if (ret_code == ERR_MANDATORY_TAG) {
+		drbd_msg_put_info(reply_skb, "name missing");
+	} else if (ret_code == ERR_ALREADY_EXISTS) {
+		drbd_msg_put_info(reply_skb, "name already exists");
+	} else {
+		drbd_msg_put_info(reply_skb, "unhandled error in drbd_check_name_str");
+	}
+}
+
+static enum drbd_ret_code drbd_check_resource_name(struct drbd_config_context *const adm_ctx)
+{
+	enum drbd_ret_code ret_code = drbd_check_name_str(adm_ctx->resource_name, drbd_strict_names);
+	drbd_msg_put_name_error(adm_ctx->reply_skb, ret_code);
+	return ret_code;
+}
+
 static void resource_to_info(struct resource_info *info,
 			     struct drbd_resource *resource)
 {
-	info->res_role = conn_highest_role(first_connection(resource));
-	info->res_susp = resource->susp;
-	info->res_susp_nod = resource->susp_nod;
-	info->res_susp_fen = resource->susp_fen;
+	info->res_role = resource->role[NOW];
+	info->res_susp = resource->susp_user[NOW];
+	info->res_susp_nod = resource->susp_nod[NOW];
+	info->res_susp_fen = is_suspended_fen(resource, NOW);
+	info->res_susp_quorum = resource->susp_quorum[NOW];
+	info->res_fail_io = resource->fail_io[NOW];
 }
 
-int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info)
 {
-	struct drbd_connection *connection;
 	struct drbd_config_context adm_ctx;
+	struct drbd_resource *resource;
 	enum drbd_ret_code retcode;
 	struct res_opts res_opts;
 	int err;
 
+	mutex_lock(&resources_mutex);
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, 0);
-	if (!adm_ctx.reply_skb)
+	if (!adm_ctx.reply_skb) {
+		mutex_unlock(&resources_mutex);
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
+	}
 
 	set_res_opts_defaults(&res_opts);
+	res_opts.node_id = -1;
 	err = res_opts_from_attrs(&res_opts, info);
-	if (err && err != -ENOMSG) {
+	if (err) {
 		retcode = ERR_MANDATORY_TAG;
 		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
 		goto out;
 	}
 
+	/* ERR_ALREADY_EXISTS? */
+	if (adm_ctx.resource)
+		goto out;
+
 	retcode = drbd_check_resource_name(&adm_ctx);
 	if (retcode != NO_ERROR)
 		goto out;
 
-	if (adm_ctx.resource) {
-		if (info->nlhdr->nlmsg_flags & NLM_F_EXCL) {
-			retcode = ERR_INVALID_REQUEST;
-			drbd_msg_put_info(adm_ctx.reply_skb, "resource exists");
-		}
-		/* else: still NO_ERROR */
+	if (res_opts.explicit_drbd8_compat)
+		res_opts.drbd8_compat_mode = true;
+
+	if (res_opts.drbd8_compat_mode) {
+#ifdef CONFIG_DRBD_COMPAT_84
+		pr_info("drbd: running in DRBD 8 compatibility mode.\n");
+		/*
+		 * That means we ignore the value of node_id for now. That
+		 * will be set to an actual value when the resource is
+		 * connected later.
+		 */
+		atomic_inc(&nr_drbd8_devices);
+		res_opts.auto_promote = false;
+#else
+		drbd_msg_put_info(adm_ctx.reply_skb, "CONFIG_DRBD_COMPAT_84 not enabled");
+		goto out;
+#endif
+	} else if (res_opts.node_id >= DRBD_NODE_ID_MAX) {
+		pr_err("drbd: invalid node id (%d)\n", res_opts.node_id);
+		retcode = ERR_INVALID_REQUEST;
 		goto out;
 	}
 
-	/* not yet safe for genl_family.parallel_ops */
-	mutex_lock(&resources_mutex);
-	connection = conn_create(adm_ctx.resource_name, &res_opts);
+	if (!try_module_get(THIS_MODULE)) {
+		pr_err("drbd: Could not get a module reference\n");
+		retcode = ERR_INVALID_REQUEST;
+		goto out;
+	}
+
+	resource = drbd_create_resource(adm_ctx.resource_name, &res_opts);
 	mutex_unlock(&resources_mutex);
 
-	if (connection) {
+	if (resource) {
 		struct resource_info resource_info;
 
 		mutex_lock(&notification_mutex);
-		resource_to_info(&resource_info, connection->resource);
-		notify_resource_state(NULL, 0, connection->resource,
-				      &resource_info, NOTIFY_CREATE);
+		resource_to_info(&resource_info, resource);
+		notify_resource_state(NULL, 0, resource, &resource_info, NULL, NOTIFY_CREATE);
 		mutex_unlock(&notification_mutex);
-	} else
+	} else {
+		module_put(THIS_MODULE);
 		retcode = ERR_NOMEM;
-
+	}
+	goto out_no_unlock;
 out:
+	mutex_unlock(&resources_mutex);
+out_no_unlock:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-static void device_to_info(struct device_info *info,
-			   struct drbd_device *device)
-{
-	info->dev_disk_state = device->state.disk;
-}
-
-
-int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	struct drbd_genlmsghdr *dh = genl_info_userhdr(info);
+	struct device_conf device_conf;
+	struct drbd_resource *resource;
+	struct drbd_device *device;
 	enum drbd_ret_code retcode;
+	int err;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
+
+	set_device_conf_defaults(&device_conf);
+	err = device_conf_from_attrs(&device_conf, info);
+	if (err && err != -ENOMSG) {
+		retcode = ERR_MANDATORY_TAG;
+		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
 		goto out;
+	}
 
 	if (dh->minor > MINORMASK) {
 		drbd_msg_put_info(adm_ctx.reply_skb, "requested minor out of range");
@@ -4306,31 +7130,33 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 		retcode = ERR_INVALID_REQUEST;
 		goto out;
 	}
-
-	/* drbd_adm_prepare made sure already
-	 * that first_peer_device(device)->connection and device->vnr match the request. */
-	if (adm_ctx.device) {
-		if (info->nlhdr->nlmsg_flags & NLM_F_EXCL)
-			retcode = ERR_MINOR_OR_VOLUME_EXISTS;
-		/* else: still NO_ERROR */
+	if (device_conf.block_size != 512 && device_conf.block_size != 1024 &&
+	    device_conf.block_size != 2048 && device_conf.block_size != 4096) {
+		drbd_msg_put_info(adm_ctx.reply_skb, "block_size not 512, 1024, 2048, or 4096");
+		retcode = ERR_INVALID_REQUEST;
 		goto out;
 	}
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	retcode = drbd_create_device(&adm_ctx, dh->minor);
+	if (adm_ctx.device)
+		goto out;
+
+	resource = adm_ctx.resource;
+	mutex_lock(&resource->conf_update);
+	for (;;) {
+		retcode = drbd_create_device(&adm_ctx, dh->minor, &device_conf, &device);
+		if (retcode != ERR_NOMEM ||
+		    schedule_timeout_interruptible(HZ / 10))
+			break;
+		/* Keep retrying until the memory allocations eventually succeed. */
+	}
 	if (retcode == NO_ERROR) {
-		struct drbd_device *device;
 		struct drbd_peer_device *peer_device;
 		struct device_info info;
 		unsigned int peer_devices = 0;
 		enum drbd_notification_type flags;
 
-		device = minor_to_device(dh->minor);
-		for_each_peer_device(peer_device, device) {
-			if (!has_net_conf(peer_device->connection))
-				continue;
+		for_each_peer_device(peer_device, device)
 			peer_devices++;
-		}
 
 		device_to_info(&info, device);
 		mutex_lock(&notification_mutex);
@@ -4339,8 +7165,6 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 		for_each_peer_device(peer_device, device) {
 			struct peer_device_info peer_device_info;
 
-			if (!has_net_conf(peer_device->connection))
-				continue;
 			peer_device_to_info(&peer_device_info, peer_device);
 			flags = (peer_devices--) ? NOTIFY_CONTINUES : 0;
 			notify_peer_device_state(NULL, 0, peer_device, &peer_device_info,
@@ -4348,7 +7172,7 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 		}
 		mutex_unlock(&notification_mutex);
 	}
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
+	mutex_unlock(&resource->conf_update);
 out:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
@@ -4356,42 +7180,51 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
 
 static enum drbd_ret_code adm_del_minor(struct drbd_device *device)
 {
+	struct drbd_resource *resource = device->resource;
 	struct drbd_peer_device *peer_device;
+	enum drbd_ret_code ret;
+	u64 im;
+
+	read_lock_irq(&resource->state_rwlock);
+	if (device->disk_state[NOW] == D_DISKLESS)
+		ret = test_and_set_bit(UNREGISTERED, &device->flags) ? ERR_MINOR_INVALID : NO_ERROR;
+	else
+		ret = ERR_MINOR_CONFIGURED;
+	read_unlock_irq(&resource->state_rwlock);
+
+	if (ret != NO_ERROR)
+		return ret;
 
-	if (device->state.disk == D_DISKLESS &&
-	    /* no need to be device->state.conn == C_STANDALONE &&
-	     * we may want to delete a minor from a live replication group.
-	     */
-	    device->state.role == R_SECONDARY) {
-		struct drbd_connection *connection =
-			first_connection(device->resource);
+	for_each_peer_device_ref(peer_device, im, device)
+		stable_change_repl_state(peer_device, L_OFF,
+					 CS_VERBOSE | CS_WAIT_COMPLETE, "del-minor");
+
+	/* If drbd_ldev_destroy() is pending, wait for it to run before
+	 * unregistering the device. */
+	wait_event(device->misc_wait, !test_bit(GOING_DISKLESS, &device->flags));
+	/*
+	 * Flush the resource work queue to make sure that no more events like
+	 * state change notifications for this device are queued: we want the
+	 * "destroy" event to come last.
+	 */
+	drbd_flush_workqueue(&resource->work);
 
-		_drbd_request_state(device, NS(conn, C_WF_REPORT_PARAMS),
-				    CS_VERBOSE + CS_WAIT_COMPLETE);
+	drbd_unregister_device(device);
 
-		/* If the state engine hasn't stopped the sender thread yet, we
-		 * need to flush the sender work queue before generating the
-		 * DESTROY events here. */
-		if (get_t_state(&connection->worker) == RUNNING)
-			drbd_flush_workqueue(&connection->sender_work);
+	mutex_lock(&notification_mutex);
+	for_each_peer_device_ref(peer_device, im, device)
+		notify_peer_device_state(NULL, 0, peer_device, NULL,
+					 NOTIFY_DESTROY | NOTIFY_CONTINUES);
+	notify_device_state(NULL, 0, device, NULL, NOTIFY_DESTROY);
+	mutex_unlock(&notification_mutex);
 
-		mutex_lock(&notification_mutex);
-		for_each_peer_device(peer_device, device) {
-			if (!has_net_conf(peer_device->connection))
-				continue;
-			notify_peer_device_state(NULL, 0, peer_device, NULL,
-						 NOTIFY_DESTROY | NOTIFY_CONTINUES);
-		}
-		notify_device_state(NULL, 0, device, NULL, NOTIFY_DESTROY);
-		mutex_unlock(&notification_mutex);
+	if (device->open_cnt == 0 && !test_and_set_bit(DESTROYING_DEV, &device->flags))
+		call_rcu(&device->rcu, drbd_reclaim_device);
 
-		drbd_delete_device(device);
-		return NO_ERROR;
-	} else
-		return ERR_MINOR_CONFIGURED;
+	return ret;
 }
 
-int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	enum drbd_ret_code retcode;
@@ -4399,168 +7232,159 @@ int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info)
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto out;
 
-	mutex_lock(&adm_ctx.resource->adm_mutex);
-	retcode = adm_del_minor(adm_ctx.device);
-	mutex_unlock(&adm_ctx.resource->adm_mutex);
-out:
+	if (mutex_lock_interruptible(&adm_ctx.resource->adm_mutex)) {
+		retcode = ERR_INTR;
+	} else {
+		retcode = adm_del_minor(adm_ctx.device);
+		mutex_unlock(&adm_ctx.resource->adm_mutex);
+	}
+
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
 static int adm_del_resource(struct drbd_resource *resource)
 {
-	struct drbd_connection *connection;
+	int err;
 
-	for_each_connection(connection, resource) {
-		if (connection->cstate > C_STANDALONE)
-			return ERR_NET_CONFIGURED;
-	}
+	/*
+	 * Flush the resource work queue to make sure that no more events like
+	 * state change notifications are queued: we want the "destroy" event
+	 * to come last.
+	 */
+	drbd_flush_workqueue(&resource->work);
+
+	mutex_lock(&resources_mutex);
+	err = ERR_RES_NOT_KNOWN;
+	if (test_bit(R_UNREGISTERED, &resource->flags))
+		goto out;
+	err = ERR_NET_CONFIGURED;
+	if (!list_empty(&resource->connections))
+		goto out;
+	err = ERR_RES_IN_USE;
 	if (!idr_is_empty(&resource->devices))
-		return ERR_RES_IN_USE;
+		goto out;
+
+	set_bit(R_UNREGISTERED, &resource->flags);
+	list_del_rcu(&resource->resources);
+	drbd_debugfs_resource_cleanup(resource);
+	mutex_unlock(&resources_mutex);
+
+	if (cancel_work_sync(&resource->empty_twopc)) {
+		kref_put(&resource->kref, drbd_destroy_resource);
+	}
+	timer_shutdown_sync(&resource->twopc_timer);
+	timer_shutdown_sync(&resource->peer_ack_timer);
+	call_rcu(&resource->rcu, drbd_reclaim_resource);
 
-	/* The state engine has stopped the sender thread, so we don't
-	 * need to flush the sender work queue before generating the
-	 * DESTROY event here. */
 	mutex_lock(&notification_mutex);
-	notify_resource_state(NULL, 0, resource, NULL, NOTIFY_DESTROY);
+	notify_resource_state(NULL, 0, resource, NULL, NULL, NOTIFY_DESTROY);
 	mutex_unlock(&notification_mutex);
 
-	mutex_lock(&resources_mutex);
-	list_del_rcu(&resource->resources);
-	mutex_unlock(&resources_mutex);
-	/* Make sure all threads have actually stopped: state handling only
-	 * does drbd_thread_stop_nowait(). */
-	list_for_each_entry(connection, &resource->connections, connections)
-		drbd_thread_stop(&connection->worker);
-	synchronize_rcu();
-	drbd_free_resource(resource);
+	/* When the last resource was removed do an explicit synchronize RCU.
+	   Without this a immediately following rmmod would fail, since the
+	   resource's worker thread still has a reference count to the module. */
+	if (list_empty(&drbd_resources))
+		synchronize_rcu();
 	return NO_ERROR;
+out:
+	mutex_unlock(&resources_mutex);
+	return err;
 }
 
-int drbd_adm_down(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_down(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
 	struct drbd_resource *resource;
 	struct drbd_connection *connection;
 	struct drbd_device *device;
 	int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */
-	unsigned i;
+	enum drbd_ret_code ret;
+	int i;
+	u64 im;
 
-	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info,
+			DRBD_ADM_NEED_RESOURCE | DRBD_ADM_IGNORE_VERSION);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
 
 	resource = adm_ctx.resource;
-	mutex_lock(&resource->adm_mutex);
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
+	set_bit(DOWN_IN_PROGRESS, &resource->flags);
 	/* demote */
-	for_each_connection(connection, resource) {
-		struct drbd_peer_device *peer_device;
-
-		idr_for_each_entry(&connection->peer_devices, peer_device, i) {
-			retcode = drbd_set_role(peer_device->device, R_SECONDARY, 0);
-			if (retcode < SS_SUCCESS) {
-				drbd_msg_put_info(adm_ctx.reply_skb, "failed to demote");
-				goto out;
-			}
-		}
+	retcode = drbd_set_role(resource, R_SECONDARY, false, "down", adm_ctx.reply_skb);
+	if (retcode < SS_SUCCESS) {
+		drbd_msg_put_info(adm_ctx.reply_skb, "failed to demote");
+		goto out;
+	}
 
-		retcode = conn_try_disconnect(connection, 0);
-		if (retcode < SS_SUCCESS) {
-			drbd_msg_put_info(adm_ctx.reply_skb, "failed to disconnect");
+	for_each_connection_ref(connection, im, resource) {
+		retcode = SS_SUCCESS;
+		if (connection->cstate[NOW] > C_STANDALONE)
+			retcode = conn_try_disconnect(connection, 0, "down", adm_ctx.reply_skb);
+		if (retcode >= SS_SUCCESS) {
+			del_connection(connection, "down");
+		} else {
+			kref_put(&connection->kref, drbd_destroy_connection);
 			goto out;
 		}
 	}
 
-	/* detach */
+	/* detach and delete minor */
+	rcu_read_lock();
 	idr_for_each_entry(&resource->devices, device, i) {
-		retcode = adm_detach(device, 0);
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		retcode = adm_detach(device, 0, 0, "down", adm_ctx.reply_skb);
+		mutex_lock(&resource->conf_update);
+		ret = adm_del_minor(device);
+		mutex_unlock(&resource->conf_update);
+		kref_put(&device->kref, drbd_destroy_device);
 		if (retcode < SS_SUCCESS || retcode > NO_ERROR) {
 			drbd_msg_put_info(adm_ctx.reply_skb, "failed to detach");
 			goto out;
 		}
-	}
-
-	/* delete volumes */
-	idr_for_each_entry(&resource->devices, device, i) {
-		retcode = adm_del_minor(device);
-		if (retcode != NO_ERROR) {
+		if (ret != NO_ERROR) {
 			/* "can not happen" */
 			drbd_msg_put_info(adm_ctx.reply_skb, "failed to delete volume");
 			goto out;
 		}
+		rcu_read_lock();
 	}
+	rcu_read_unlock();
 
+	mutex_lock(&resource->conf_update);
 	retcode = adm_del_resource(resource);
+	/* holding a reference to resource in adm_crx until drbd_adm_finish() */
+	mutex_unlock(&resource->conf_update);
 out:
+	opener_info(adm_ctx.resource, adm_ctx.reply_skb, (enum drbd_state_rv)retcode);
+	clear_bit(DOWN_IN_PROGRESS, &resource->flags);
 	mutex_unlock(&resource->adm_mutex);
-finish:
+out_no_adm_mutex:
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info)
+static int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context adm_ctx;
-	struct drbd_resource *resource;
 	enum drbd_ret_code retcode;
 
 	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
 	if (!adm_ctx.reply_skb)
 		return retcode;
-	if (retcode != NO_ERROR)
-		goto finish;
-	resource = adm_ctx.resource;
 
-	mutex_lock(&resource->adm_mutex);
-	retcode = adm_del_resource(resource);
-	mutex_unlock(&resource->adm_mutex);
-finish:
+	retcode = adm_del_resource(adm_ctx.resource);
+
 	drbd_adm_finish(&adm_ctx, info, retcode);
 	return 0;
 }
 
-void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
-{
-	struct sk_buff *msg;
-	struct drbd_genlmsghdr *d_out;
-	unsigned seq;
-	int err = -ENOMEM;
-
-	seq = atomic_inc_return(&drbd_genl_seq);
-	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
-	if (!msg)
-		goto failed;
-
-	err = -EMSGSIZE;
-	d_out = genlmsg_put(msg, 0, seq, &drbd_genl_family, 0, DRBD_EVENT);
-	if (!d_out) /* cannot happen, but anyways. */
-		goto nla_put_failure;
-	d_out->minor = device_to_minor(device);
-	d_out->ret_code = NO_ERROR;
-
-	if (nla_put_status_info(msg, device, sib))
-		goto nla_put_failure;
-	genlmsg_end(msg, d_out);
-	err = drbd_genl_multicast_events(msg, GFP_NOWAIT);
-	/* msg has been consumed or freed in netlink_broadcast() */
-	if (err && err != -ESRCH)
-		goto failed;
-
-	return;
-
-nla_put_failure:
-	nlmsg_free(msg);
-failed:
-	drbd_err(device, "Error %d while broadcasting event. "
-			"Event seq:%u sib_reason:%u\n",
-			err, seq, sib->sib_reason);
-}
-
 static int nla_put_notification_header(struct sk_buff *msg,
 				       enum drbd_notification_type type)
 {
@@ -4575,6 +7399,7 @@ int notify_resource_state(struct sk_buff *skb,
 			   unsigned int seq,
 			   struct drbd_resource *resource,
 			   struct resource_info *resource_info,
+			   struct rename_resource_info *rename_resource_info,
 			   enum drbd_notification_type type)
 {
 	struct resource_statistics resource_statistics;
@@ -4583,7 +7408,7 @@ int notify_resource_state(struct sk_buff *skb,
 	int err;
 
 	if (!skb) {
-		seq = atomic_inc_return(&notify_genl_seq);
+		seq = atomic_inc_return(&drbd_genl_seq);
 		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
 		err = -ENOMEM;
 		if (!skb)
@@ -4597,18 +7422,29 @@ int notify_resource_state(struct sk_buff *skb,
 		goto nla_put_failure;
 	dh->minor = -1U;
 	dh->ret_code = NO_ERROR;
-	if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL) ||
-	    nla_put_notification_header(skb, type) ||
-	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
-	     resource_info_to_skb(skb, resource_info, true)))
+	if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL, NULL) ||
+	    nla_put_notification_header(skb, type))
 		goto nla_put_failure;
+
+	if (resource_info) {
+		err = resource_info_to_skb(skb, resource_info, true);
+		if (err)
+			goto nla_put_failure;
+	}
+
 	resource_statistics.res_stat_write_ordering = resource->write_ordering;
 	err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN));
 	if (err)
 		goto nla_put_failure;
+
+	if (rename_resource_info) {
+		err = rename_resource_info_to_skb(skb, rename_resource_info, !capable(CAP_SYS_ADMIN));
+		if (err)
+			goto nla_put_failure;
+	}
 	genlmsg_end(skb, dh);
 	if (multicast) {
-		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		err = drbd_genl_multicast_events(skb);
 		/* skb has been consumed or freed in netlink_broadcast() */
 		if (err && err != -ESRCH)
 			goto failed;
@@ -4635,7 +7471,7 @@ int notify_device_state(struct sk_buff *skb,
 	int err;
 
 	if (!skb) {
-		seq = atomic_inc_return(&notify_genl_seq);
+		seq = atomic_inc_return(&drbd_genl_seq);
 		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
 		err = -ENOMEM;
 		if (!skb)
@@ -4649,7 +7485,7 @@ int notify_device_state(struct sk_buff *skb,
 		goto nla_put_failure;
 	dh->minor = device->minor;
 	dh->ret_code = NO_ERROR;
-	if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device) ||
+	if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
 	     device_info_to_skb(skb, device_info, true)))
@@ -4658,7 +7494,7 @@ int notify_device_state(struct sk_buff *skb,
 	device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN));
 	genlmsg_end(skb, dh);
 	if (multicast) {
-		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		err = drbd_genl_multicast_events(skb);
 		/* skb has been consumed or freed in netlink_broadcast() */
 		if (err && err != -ESRCH)
 			goto failed;
@@ -4673,6 +7509,7 @@ int notify_device_state(struct sk_buff *skb,
 	return err;
 }
 
+/* open coded path_parms_to_skb() iterating of the list */
 int notify_connection_state(struct sk_buff *skb,
 			     unsigned int seq,
 			     struct drbd_connection *connection,
@@ -4685,7 +7522,7 @@ int notify_connection_state(struct sk_buff *skb,
 	int err;
 
 	if (!skb) {
-		seq = atomic_inc_return(&notify_genl_seq);
+		seq = atomic_inc_return(&drbd_genl_seq);
 		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
 		err = -ENOMEM;
 		if (!skb)
@@ -4699,16 +7536,17 @@ int notify_connection_state(struct sk_buff *skb,
 		goto nla_put_failure;
 	dh->minor = -1U;
 	dh->ret_code = NO_ERROR;
-	if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL) ||
+	if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
 	     connection_info_to_skb(skb, connection_info, true)))
 		goto nla_put_failure;
-	connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
+	connection_paths_to_skb(skb, connection);
+	connection_to_statistics(&connection_statistics, connection);
 	connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
 	genlmsg_end(skb, dh);
 	if (multicast) {
-		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		err = drbd_genl_multicast_events(skb);
 		/* skb has been consumed or freed in netlink_broadcast() */
 		if (err && err != -ESRCH)
 			goto failed;
@@ -4736,7 +7574,7 @@ int notify_peer_device_state(struct sk_buff *skb,
 	int err;
 
 	if (!skb) {
-		seq = atomic_inc_return(&notify_genl_seq);
+		seq = atomic_inc_return(&drbd_genl_seq);
 		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
 		err = -ENOMEM;
 		if (!skb)
@@ -4750,7 +7588,7 @@ int notify_peer_device_state(struct sk_buff *skb,
 		goto nla_put_failure;
 	dh->minor = -1U;
 	dh->ret_code = NO_ERROR;
-	if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device) ||
+	if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
 	     peer_device_info_to_skb(skb, peer_device_info, true)))
@@ -4759,7 +7597,7 @@ int notify_peer_device_state(struct sk_buff *skb,
 	peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
 	genlmsg_end(skb, dh);
 	if (multicast) {
-		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+		err = drbd_genl_multicast_events(skb);
 		/* skb has been consumed or freed in netlink_broadcast() */
 		if (err && err != -ESRCH)
 			goto failed;
@@ -4774,13 +7612,86 @@ int notify_peer_device_state(struct sk_buff *skb,
 	return err;
 }
 
+void drbd_broadcast_peer_device_state(struct drbd_peer_device *peer_device)
+{
+	struct peer_device_info peer_device_info;
+	mutex_lock(&notification_mutex);
+	peer_device_to_info(&peer_device_info, peer_device);
+	notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CHANGE);
+	mutex_unlock(&notification_mutex);
+}
+
+static int notify_path_state(struct sk_buff *skb,
+		       unsigned int seq,
+		       /* until we have a backpointer in drbd_path, we need an explicit connection: */
+		       struct drbd_connection *connection,
+		       struct drbd_path *path,
+		       struct drbd_path_info *path_info,
+		       enum drbd_notification_type type)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_genlmsghdr *dh;
+	bool multicast = false;
+	int err;
+
+	if (!skb) {
+		seq = atomic_inc_return(&drbd_genl_seq);
+		skb = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
+		err = -ENOMEM;
+		if (!skb)
+			goto failed;
+		multicast = true;
+	}
+
+	err = -EMSGSIZE;
+	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_PATH_STATE);
+	if (!dh)
+		goto nla_put_failure;
+
+	dh->minor = -1U;
+	dh->ret_code = NO_ERROR;
+	if (nla_put_drbd_cfg_context(skb, resource, connection, NULL, path) ||
+	    nla_put_notification_header(skb, type) ||
+	    drbd_path_info_to_skb(skb, path_info, true))
+		goto nla_put_failure;
+	genlmsg_end(skb, dh);
+	if (multicast) {
+		err = drbd_genl_multicast_events(skb);
+		/* skb has been consumed or freed in netlink_broadcast() */
+		if (err && err != -ESRCH)
+			goto failed;
+	}
+	return 0;
+
+nla_put_failure:
+	nlmsg_free(skb);
+failed:
+	/* FIXME add path specifics to our drbd_polymorph_printk.h */
+	drbd_err(connection, "path: Error %d while broadcasting event. Event seq:%u\n",
+		 err, seq);
+	return err;
+}
+
+int notify_path(struct drbd_connection *connection, struct drbd_path *path, enum drbd_notification_type type)
+{
+	struct drbd_path_info path_info;
+	int err;
+
+	path_info.path_established = test_bit(TR_ESTABLISHED, &path->flags);
+	mutex_lock(&notification_mutex);
+	err = notify_path_state(NULL, 0, connection, path, &path_info, type);
+	mutex_unlock(&notification_mutex);
+	return err;
+
+}
+
 void notify_helper(enum drbd_notification_type type,
 		   struct drbd_device *device, struct drbd_connection *connection,
 		   const char *name, int status)
 {
 	struct drbd_resource *resource = device ? device->resource : connection->resource;
 	struct drbd_helper_info helper_info;
-	unsigned int seq = atomic_inc_return(&notify_genl_seq);
+	unsigned int seq = atomic_inc_return(&drbd_genl_seq);
 	struct sk_buff *skb = NULL;
 	struct drbd_genlmsghdr *dh;
 	int err;
@@ -4801,12 +7712,12 @@ void notify_helper(enum drbd_notification_type type,
 	dh->minor = device ? device->minor : -1;
 	dh->ret_code = NO_ERROR;
 	mutex_lock(&notification_mutex);
-	if (nla_put_drbd_cfg_context(skb, resource, connection, device) ||
+	if (nla_put_drbd_cfg_context(skb, resource, connection, device, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    drbd_helper_info_to_skb(skb, &helper_info, true))
 		goto unlock_fail;
 	genlmsg_end(skb, dh);
-	err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
+	err = drbd_genl_multicast_events(skb);
 	skb = NULL;
 	/* skb has been consumed or freed in netlink_broadcast() */
 	if (err && err != -ESRCH)
@@ -4859,7 +7770,8 @@ static unsigned int notifications_for_state_change(struct drbd_state_change *sta
 	return 1 +
 	       state_change->n_connections +
 	       state_change->n_devices +
-	       state_change->n_devices * state_change->n_connections;
+	       state_change->n_devices * state_change->n_connections +
+	       state_change->n_paths;
 }
 
 static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
@@ -4871,7 +7783,7 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	int err = 0;
 
 	/* There is no need for taking notification_mutex here: it doesn't
-	   matter if the initial state events mix with later state chage
+	   matter if the initial state events mix with later state change
 	   events; we can always tell the events apart by the NOTIFY_EXISTS
 	   flag. */
 
@@ -4884,7 +7796,7 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	if (cb->args[4] < cb->args[3])
 		flags |= NOTIFY_CONTINUES;
 	if (n < 1) {
-		err = notify_resource_state_change(skb, seq, state_change->resource,
+		err = notify_resource_state_change(skb, seq, state_change,
 					     NOTIFY_EXISTS | flags);
 		goto next;
 	}
@@ -4895,6 +7807,18 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 		goto next;
 	}
 	n -= state_change->n_connections;
+	if (n < state_change->n_paths) {
+		struct drbd_path_state *path_state = &state_change->paths[n];
+		struct drbd_path_info path_info;
+
+		path_info.path_established = path_state->path_established;
+		err = notify_path_state(skb, seq,
+				path_state->connection,
+				path_state->path,
+				&path_info, NOTIFY_EXISTS | flags);
+		goto next;
+	}
+	n -= state_change->n_paths;
 	if (n < state_change->n_devices) {
 		err = notify_device_state_change(skb, seq, &state_change->devices[n],
 					   NOTIFY_EXISTS | flags);
@@ -4906,6 +7830,7 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 						NOTIFY_EXISTS | flags);
 		goto next;
 	}
+	n -= state_change->n_devices * state_change->n_connections;
 
 next:
 	if (cb->args[4] == cb->args[3]) {
@@ -4919,11 +7844,25 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 out:
 	if (err)
 		return err;
-	else
-		return skb->len;
+	return skb->len;
+}
+
+static int drbd_adm_get_initial_state_done(struct netlink_callback *cb)
+{
+	LIST_HEAD(head);
+	if (cb->args[0]) {
+		struct drbd_state_change *state_change =
+			(struct drbd_state_change *)cb->args[0];
+		cb->args[0] = 0;
+
+		/* connect list to head */
+		list_add(&head, &state_change->list);
+		free_state_changes(&head);
+	}
+	return 0;
 }
 
-int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+static int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct drbd_resource *resource;
 	LIST_HEAD(head);
@@ -4931,14 +7870,6 @@ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	if (cb->args[5] >= 1) {
 		if (cb->args[5] > 1)
 			return get_initial_state(skb, cb);
-		if (cb->args[0]) {
-			struct drbd_state_change *state_change =
-				(struct drbd_state_change *)cb->args[0];
-
-			/* connect list to head */
-			list_add(&head, &state_change->list);
-			free_state_changes(&head);
-		}
 		return 0;
 	}
 
@@ -4947,7 +7878,9 @@ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	for_each_resource(resource, &drbd_resources) {
 		struct drbd_state_change *state_change;
 
-		state_change = remember_old_state(resource, GFP_KERNEL);
+		read_lock_irq(&resource->state_rwlock);
+		state_change = remember_state_change(resource, GFP_ATOMIC);
+		read_unlock_irq(&resource->state_rwlock);
 		if (!state_change) {
 			if (!list_empty(&head))
 				free_state_changes(&head);
@@ -4971,3 +7904,144 @@ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	cb->args[2] = cb->nlh->nlmsg_seq;
 	return get_initial_state(skb, cb);
 }
+
+static int drbd_adm_forget_peer(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct drbd_resource *resource;
+	struct drbd_device *device;
+	struct forget_peer_parms parms = { };
+	enum drbd_ret_code retcode;
+	int vnr, peer_node_id, err;
+
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
+	if (!adm_ctx.reply_skb)
+		return retcode;
+
+	resource = adm_ctx.resource;
+
+	err = forget_peer_parms_from_attrs(&parms, info);
+	if (err) {
+		retcode = ERR_MANDATORY_TAG;
+		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+		goto out_no_adm_mutex;
+	}
+
+	if (mutex_lock_interruptible(&resource->adm_mutex)) {
+		retcode = ERR_INTR;
+		goto out_no_adm_mutex;
+	}
+
+	peer_node_id = parms.forget_peer_node_id;
+	if (drbd_connection_by_node_id(resource, peer_node_id)) {
+		retcode = ERR_NET_CONFIGURED;
+		goto out;
+	}
+
+	if (peer_node_id < 0 || peer_node_id >= DRBD_NODE_ID_MAX) {
+		retcode = ERR_INVALID_PEER_NODE_ID;
+		goto out;
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr)
+		clear_peer_slot(device, peer_node_id, 0);
+out:
+	mutex_unlock(&resource->adm_mutex);
+out_no_adm_mutex:
+	idr_for_each_entry(&resource->devices, device, vnr)
+		drbd_md_sync_if_dirty(device);
+
+	drbd_adm_finish(&adm_ctx, info, (enum drbd_ret_code)retcode);
+	return 0;
+
+}
+
+static enum drbd_ret_code validate_new_resource_name(const struct drbd_resource *resource, const char *new_name)
+{
+	enum drbd_ret_code retcode = drbd_check_name_str(new_name, drbd_strict_names);
+
+	if (retcode == NO_ERROR) {
+		struct drbd_resource *next_resource;
+		rcu_read_lock();
+		for_each_resource_rcu(next_resource, &drbd_resources) {
+			if (strcmp(next_resource->name, new_name) == 0) {
+				retcode = ERR_ALREADY_EXISTS;
+				break;
+			}
+		}
+		rcu_read_unlock();
+	}
+	return retcode;
+}
+
+static int drbd_adm_rename_resource(struct sk_buff *skb, struct genl_info *info)
+{
+	struct drbd_config_context adm_ctx;
+	struct drbd_resource *resource;
+	struct drbd_device *device;
+	struct rename_resource_info rename_resource_info;
+	struct rename_resource_parms parms = { };
+	char *old_res_name, *new_res_name;
+	enum drbd_ret_code retcode;
+	enum drbd_ret_code validate_err;
+	int err;
+	int vnr;
+
+	mutex_lock(&resources_mutex);
+	retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE);
+	if (!adm_ctx.reply_skb) {
+		mutex_unlock(&resources_mutex);
+		return retcode;
+	}
+
+	resource = adm_ctx.resource;
+
+	err = rename_resource_parms_from_attrs(&parms, info);
+	if (err) {
+		retcode = ERR_MANDATORY_TAG;
+		drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err));
+		goto out;
+	}
+
+	validate_err = validate_new_resource_name(resource, parms.new_resource_name);
+	if (validate_err != NO_ERROR) {
+		if (ERR_ALREADY_EXISTS) {
+			drbd_msg_sprintf_info(adm_ctx.reply_skb,
+				"Cannot rename to %s: a resource with that name already exists\n",
+				 parms.new_resource_name);
+		} else {
+			drbd_msg_put_name_error(adm_ctx.reply_skb, validate_err);
+		}
+		retcode = validate_err;
+		goto out;
+	}
+
+	drbd_info(resource, "Renaming to %s\n", parms.new_resource_name);
+
+	strscpy(rename_resource_info.res_new_name, parms.new_resource_name, sizeof(rename_resource_info.res_new_name));
+	rename_resource_info.res_new_name_len = min(strlen(parms.new_resource_name), sizeof(rename_resource_info.res_new_name));
+
+	mutex_lock(&notification_mutex);
+	notify_resource_state(NULL, 0, resource, NULL, &rename_resource_info, NOTIFY_RENAME);
+	mutex_unlock(&notification_mutex);
+
+	new_res_name = kstrdup(parms.new_resource_name, GFP_KERNEL);
+	if (!new_res_name) {
+		retcode = ERR_NOMEM;
+		goto out;
+	}
+	old_res_name = resource->name;
+	resource->name = new_res_name;
+	kvfree_rcu_mightsleep(old_res_name);
+
+	drbd_debugfs_resource_rename(resource, new_res_name);
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		kobject_uevent(&disk_to_dev(device->vdisk)->kobj, KOBJ_CHANGE);
+	}
+
+out:
+	mutex_unlock(&resources_mutex);
+	drbd_adm_finish(&adm_ctx, info, retcode);
+	return 0;
+}
diff --git a/drivers/block/drbd/drbd_nla.h b/drivers/block/drbd/drbd_nla.h
index d3555df0d3537..4463657c020de 100644
--- a/drivers/block/drbd/drbd_nla.h
+++ b/drivers/block/drbd/drbd_nla.h
@@ -2,8 +2,9 @@
 #ifndef __DRBD_NLA_H
 #define __DRBD_NLA_H
 
-extern int drbd_nla_parse_nested(struct nlattr *tb[], int maxtype, struct nlattr *nla,
-				 const struct nla_policy *policy);
-extern struct nlattr *drbd_nla_find_nested(int maxtype, struct nlattr *nla, int attrtype);
+int drbd_nla_parse_nested(struct nlattr *tb[], int maxtype,
+			  struct nlattr *nla, const struct nla_policy *policy);
+struct nlattr *drbd_nla_find_nested(int maxtype, struct nlattr *nla,
+				    int attrtype);
 
 #endif  /* __DRBD_NLA_H */
diff --git a/drivers/block/drbd/drbd_polymorph_printk.h b/drivers/block/drbd/drbd_polymorph_printk.h
index 8e0082d139bac..7b0873d2980ea 100644
--- a/drivers/block/drbd/drbd_polymorph_printk.h
+++ b/drivers/block/drbd/drbd_polymorph_printk.h
@@ -11,104 +11,188 @@
 #define DYNAMIC_DEBUG_BRANCH(D) false
 #endif
 
+#define __drbd_printk(level, fmt, args...)				\
+	printk(level fmt, ## args)
+#define __drbd_dyn_dbg(descriptor, fmt, args...)			\
+	__dynamic_pr_debug(descriptor, fmt, ## args)
+
+#define ___drbd_printk_device(prmacro, rlt, device, lvl_or_desc, fmt, args...)\
+({									\
+	const struct drbd_device *__d =					\
+		(const struct drbd_device *)(device);			\
+	const struct drbd_resource *__r = __d->resource;		\
+	const char *__unregistered = "";				\
+	if (test_bit(UNREGISTERED, &__d->flags))			\
+		__unregistered = "/unregistered/";			\
+	if (drbd_device_ratelimit(__d, rlt))				\
+		prmacro(lvl_or_desc, "drbd %s%s/%u drbd%u: " fmt,	\
+			__unregistered, __r->name, __d->vnr, __d->minor,\
+			## args);					\
+})
+
+#define ___drbd_printk_resource(prmacro, rlt, resource, lvl_or_desc, fmt, args...)\
+({									\
+	const struct drbd_resource *__r =				\
+		(const struct drbd_resource *)(resource);		\
+	const char *__unregistered = "";				\
+	if (test_bit(R_UNREGISTERED, &__r->flags))			\
+		__unregistered = "/unregistered/";			\
+	if (drbd_resource_ratelimit(__r, rlt))				\
+		prmacro(lvl_or_desc, "drbd %s%s: " fmt,			\
+			__unregistered, __r->name, ## args);		\
+})
+
+// As long as the connection is still "registered", the resource
+// can not yet be "unregistered", no need to test R_UNREGISTERED
+#define ___drbd_printk_peer_device(prmacro, rlt, peer_device, lvl_or_desc, fmt, args...)\
+({									\
+	const struct drbd_peer_device *__pd;				\
+	const struct drbd_device *__d;					\
+	const struct drbd_connection *__c;				\
+	const struct drbd_resource *__r;				\
+	const char *__cn;						\
+	const char *__unregistered = "";				\
+	rcu_read_lock();						\
+	__pd = (const struct drbd_peer_device *)(peer_device);		\
+	__d = __pd->device;						\
+	__c = __pd->connection;						\
+	__r = __d->resource;						\
+	__cn = rcu_dereference(__c->transport.net_conf)->name;		\
+	if (test_bit(C_UNREGISTERED, &__c->flags))			\
+		__unregistered = "/unregistered/";			\
+	if (drbd_peer_device_ratelimit(__pd, rlt))			\
+		prmacro(lvl_or_desc, "drbd %s%s/%u drbd%u %s: " fmt,		\
+			__unregistered, __r->name, __d->vnr, __d->minor, __cn,	\
+			 ## args);					\
+	rcu_read_unlock();						\
+})
+
+#define ___drbd_printk_connection(prmacro, rlt, connection, lvl_or_desc, fmt, args...)	\
+({									\
+	const struct drbd_connection *__c =				\
+		(const struct drbd_connection *)(connection);		\
+	const struct drbd_resource *__r = __c->resource;		\
+	const char *__cn;						\
+	const char *__unregistered = "";				\
+	rcu_read_lock();						\
+	__cn = rcu_dereference(__c->transport.net_conf)->name;		\
+	if (test_bit(C_UNREGISTERED, &__c->flags))			\
+		__unregistered = "/unregistered/";			\
+	if (drbd_connection_ratelimit(__c, rlt))			\
+		prmacro(lvl_or_desc, "drbd %s%s %s: " fmt,		\
+			__unregistered, __r->name, __cn, ## args);	\
+	rcu_read_unlock();						\
+})
 
-#define __drbd_printk_drbd_device_prep(device)			\
-	const struct drbd_device *__d = (device);		\
-	const struct drbd_resource *__r = __d->resource
-#define __drbd_printk_drbd_device_fmt(fmt)	"drbd %s/%u drbd%u: " fmt
-#define __drbd_printk_drbd_device_args()	__r->name, __d->vnr, __d->minor
-#define __drbd_printk_drbd_device_unprep()
-
-#define __drbd_printk_drbd_peer_device_prep(peer_device)	\
-	const struct drbd_device *__d;				\
-	const struct drbd_resource *__r;			\
-	__d = (peer_device)->device;				\
-	__r = __d->resource
-#define __drbd_printk_drbd_peer_device_fmt(fmt) \
-	"drbd %s/%u drbd%u: " fmt
-#define __drbd_printk_drbd_peer_device_args() \
-	__r->name, __d->vnr, __d->minor
-#define __drbd_printk_drbd_peer_device_unprep()
-
-#define __drbd_printk_drbd_resource_prep(resource) \
-	const struct drbd_resource *__r = resource
-#define __drbd_printk_drbd_resource_fmt(fmt) "drbd %s: " fmt
-#define __drbd_printk_drbd_resource_args()	__r->name
-#define __drbd_printk_drbd_resource_unprep(resource)
-
-#define __drbd_printk_drbd_connection_prep(connection)		\
-	const struct drbd_connection *__c = (connection);	\
-	const struct drbd_resource *__r = __c->resource
-#define __drbd_printk_drbd_connection_fmt(fmt)			\
-	"drbd %s: " fmt
-#define __drbd_printk_drbd_connection_args()			\
-	__r->name
-#define __drbd_printk_drbd_connection_unprep()
+#define __drbd_printk_device(rlt, device, level, fmt, args...)\
+	___drbd_printk_device(__drbd_printk, rlt, device, level, fmt, ## args)
+#define __drbd_printk_resource(rlt, resource, level, fmt, args...)\
+	 ___drbd_printk_resource(__drbd_printk, rlt, resource, level, fmt, ## args)
+#define __drbd_printk_peer_device(rlt, peer_device, level, fmt, args...)\
+	 ___drbd_printk_peer_device(__drbd_printk, rlt, peer_device, level, fmt, ## args)
+#define __drbd_printk_connection(rlt, connection, level, fmt, args...)\
+	 ___drbd_printk_connection(__drbd_printk, rlt, connection, level, fmt, ## args)
 
 void drbd_printk_with_wrong_object_type(void);
 void drbd_dyn_dbg_with_wrong_object_type(void);
 
 #define __drbd_printk_choose_cond(obj, struct_name) \
-	(__builtin_types_compatible_p(typeof(obj), struct struct_name *) || \
-	 __builtin_types_compatible_p(typeof(obj), const struct struct_name *))
-#define __drbd_printk_if_same_type(obj, struct_name, level, fmt, args...) \
-	__drbd_printk_choose_cond(obj, struct_name), \
-({ \
-	__drbd_printk_ ## struct_name ## _prep((const struct struct_name *)(obj)); \
-	printk(level __drbd_printk_ ## struct_name ## _fmt(fmt), \
-		__drbd_printk_ ## struct_name ## _args(), ## args); \
-	__drbd_printk_ ## struct_name ## _unprep(); \
-})
-
-#define drbd_printk(level, obj, fmt, args...) \
-	__builtin_choose_expr( \
-	  __drbd_printk_if_same_type(obj, drbd_device, level, fmt, ## args), \
-	  __builtin_choose_expr( \
-	    __drbd_printk_if_same_type(obj, drbd_resource, level, fmt, ## args), \
-	    __builtin_choose_expr( \
-	      __drbd_printk_if_same_type(obj, drbd_connection, level, fmt, ## args), \
-	      __builtin_choose_expr( \
-		__drbd_printk_if_same_type(obj, drbd_peer_device, level, fmt, ## args), \
-		drbd_printk_with_wrong_object_type()))))
+	(__builtin_types_compatible_p(typeof(obj), struct drbd_ ## struct_name *) || \
+	 __builtin_types_compatible_p(typeof(obj), const struct drbd_ ## struct_name *))
+
+#define __drbd_obj_ratelimit(struct_name, obj, rlt)		\
+	({							\
+	int __rlt = (rlt);					\
+	BUILD_BUG_ON(!__drbd_printk_choose_cond(obj, struct_name)); \
+	BUILD_BUG_ON(__rlt < -1);				\
+	BUILD_BUG_ON(__rlt >= (int)ARRAY_SIZE(obj->ratelimit)); \
+	__rlt == -1 ? 1						\
+	: __ratelimit(/* unconst cast ratelimit state */	\
+		(struct ratelimit_state *)(unsigned long)	\
+		&obj->ratelimit[__rlt]);			\
+	})
+
+#define drbd_device_ratelimit(obj, rlt)		\
+	__drbd_obj_ratelimit(device, obj, D_RL_D_ ## rlt)
+#define drbd_resource_ratelimit(obj, rlt)	\
+	__drbd_obj_ratelimit(resource, obj, D_RL_R_ ## rlt)
+#define drbd_connection_ratelimit(obj, rlt)	\
+	__drbd_obj_ratelimit(connection, obj, D_RL_C_ ## rlt)
+#define drbd_peer_device_ratelimit(obj, rlt)	\
+	__drbd_obj_ratelimit(peer_device, obj, D_RL_PD_ ## rlt)
+
+#define drbd_printk(ratelimit_type, level, obj, fmt, args...) \
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, device), \
+	__drbd_printk_device(ratelimit_type, obj, level, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, resource), \
+	__drbd_printk_resource(ratelimit_type, obj, level, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, connection), \
+	__drbd_printk_connection(ratelimit_type, obj, level, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, peer_device), \
+	__drbd_printk_peer_device(ratelimit_type, obj, level, fmt, ## args), \
+	\
+	drbd_printk_with_wrong_object_type() \
+	))))
 
 #define __drbd_dyn_dbg_if_same_type(obj, struct_name, fmt, args...) \
-	__drbd_printk_choose_cond(obj, struct_name), \
 ({ \
 	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt);		\
 	if (DYNAMIC_DEBUG_BRANCH(descriptor)) {			\
-		__drbd_printk_ ## struct_name ## _prep((const struct struct_name *)(obj)); \
-		__dynamic_pr_debug(&descriptor, __drbd_printk_ ## struct_name ## _fmt(fmt), \
-			__drbd_printk_ ## struct_name ## _args(), ## args); \
-		__drbd_printk_ ## struct_name ## _unprep();	\
+		___drbd_printk_ ## struct_name(			\
+			__drbd_dyn_dbg,				\
+				NOLIMIT, obj,			\
+				&descriptor, fmt, ## args);	\
 	}							\
 })
 
 #define dynamic_drbd_dbg(obj, fmt, args...) \
-	__builtin_choose_expr( \
-	  __drbd_dyn_dbg_if_same_type(obj, drbd_device, fmt, ## args), \
-	  __builtin_choose_expr( \
-	    __drbd_dyn_dbg_if_same_type(obj, drbd_resource, fmt, ## args), \
-	    __builtin_choose_expr( \
-	      __drbd_dyn_dbg_if_same_type(obj, drbd_connection, fmt, ## args), \
-	      __builtin_choose_expr( \
-		__drbd_dyn_dbg_if_same_type(obj, drbd_peer_device, fmt, ## args), \
-		drbd_dyn_dbg_with_wrong_object_type()))))
-
-#define drbd_emerg(device, fmt, args...) \
-	drbd_printk(KERN_EMERG, device, fmt, ## args)
-#define drbd_alert(device, fmt, args...) \
-	drbd_printk(KERN_ALERT, device, fmt, ## args)
-#define drbd_crit(device, fmt, args...) \
-	drbd_printk(KERN_CRIT, device, fmt, ## args)
-#define drbd_err(device, fmt, args...) \
-	drbd_printk(KERN_ERR, device, fmt, ## args)
-#define drbd_warn(device, fmt, args...) \
-	drbd_printk(KERN_WARNING, device, fmt, ## args)
-#define drbd_notice(device, fmt, args...) \
-	drbd_printk(KERN_NOTICE, device, fmt, ## args)
-#define drbd_info(device, fmt, args...) \
-	drbd_printk(KERN_INFO, device, fmt, ## args)
-
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, device), \
+	__drbd_dyn_dbg_if_same_type(obj, device, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, resource), \
+	__drbd_dyn_dbg_if_same_type(obj, resource, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, connection), \
+	__drbd_dyn_dbg_if_same_type(obj, connection, fmt, ## args), \
+	\
+	__builtin_choose_expr(__drbd_printk_choose_cond(obj, peer_device), \
+	__drbd_dyn_dbg_if_same_type(obj, peer_device, fmt, ## args), \
+	\
+	drbd_dyn_dbg_with_wrong_object_type() \
+	))))
+
+#define drbd_emerg_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_EMERG, obj, fmt, ## args)
+#define drbd_alert_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_ALERT, obj, fmt, ## args)
+#define drbd_crit_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_CRIT, obj, fmt, ## args)
+#define drbd_err_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_ERR, obj, fmt, ## args)
+#define drbd_warn_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_WARNING, obj, fmt, ## args)
+#define drbd_notice_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_NOTICE, obj, fmt, ## args)
+#define drbd_info_ratelimit(obj, fmt, args...) \
+	drbd_printk(GENERIC, KERN_INFO, obj, fmt, ## args)
+
+#define drbd_emerg(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_EMERG, obj, fmt,  ## args)
+#define drbd_alert(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_ALERT, obj, fmt,  ## args)
+#define drbd_crit(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_CRIT, obj, fmt,  ## args)
+#define drbd_err(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_ERR, obj, fmt,  ## args)
+#define drbd_warn(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_WARNING, obj, fmt,  ## args)
+#define drbd_notice(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_NOTICE, obj, fmt,  ## args)
+#define drbd_info(obj, fmt, args...) \
+	drbd_printk(NOLIMIT, KERN_INFO, obj, fmt,  ## args)
 
 #define drbd_ratelimit() \
 ({						\
@@ -122,7 +206,7 @@ void drbd_dyn_dbg_with_wrong_object_type(void);
 	do {									\
 		if (!(exp))							\
 			drbd_err(x, "ASSERTION %s FAILED in %s\n",		\
-				#exp, __func__);				\
+				 #exp, __func__);				\
 	} while (0)
 
 /**
@@ -130,12 +214,13 @@ void drbd_dyn_dbg_with_wrong_object_type(void);
  *
  * Unlike the assert macro, this macro returns a boolean result.
  */
-#define expect(x, exp) ({							\
-		bool _bool = (exp);						\
-		if (!_bool && drbd_ratelimit())					\
-			drbd_err(x, "ASSERTION %s FAILED in %s\n",		\
-				#exp, __func__);				\
-		_bool;								\
+#define expect(x, exp) ({					\
+		bool _bool = (exp);				\
+		if (!_bool)					\
+			drbd_err_ratelimit(x,			\
+				"ASSERTION %s FAILED in %s\n",	\
+				#exp, __func__);		\
+		_bool;						\
 		})
 
 #endif
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 1d0feafceadcb..0d741108ce0c5 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -11,313 +11,35 @@
 
  */
 
-#include <linux/module.h>
-
-#include <linux/uaccess.h>
-#include <linux/fs.h>
-#include <linux/file.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
-#include <linux/drbd.h>
 #include "drbd_int.h"
+#include "drbd_transport.h"
+#include "drbd_legacy_84.h"
 
 struct proc_dir_entry *drbd_proc;
 
-static void seq_printf_with_thousands_grouping(struct seq_file *seq, long v)
-{
-	/* v is in kB/sec. We don't expect TiByte/sec yet. */
-	if (unlikely(v >= 1000000)) {
-		/* cool: > GiByte/s */
-		seq_printf(seq, "%ld,", v / 1000000);
-		v %= 1000000;
-		seq_printf(seq, "%03ld,%03ld", v/1000, v % 1000);
-	} else if (likely(v >= 1000))
-		seq_printf(seq, "%ld,%03ld", v/1000, v % 1000);
-	else
-		seq_printf(seq, "%ld", v);
-}
-
-static void drbd_get_syncer_progress(struct drbd_device *device,
-		union drbd_dev_state state, unsigned long *rs_total,
-		unsigned long *bits_left, unsigned int *per_mil_done)
-{
-	/* this is to break it at compile time when we change that, in case we
-	 * want to support more than (1<<32) bits on a 32bit arch. */
-	typecheck(unsigned long, device->rs_total);
-	*rs_total = device->rs_total;
-
-	/* note: both rs_total and rs_left are in bits, i.e. in
-	 * units of BM_BLOCK_SIZE.
-	 * for the percentage, we don't care. */
-
-	if (state.conn == C_VERIFY_S || state.conn == C_VERIFY_T)
-		*bits_left = device->ov_left;
-	else
-		*bits_left = drbd_bm_total_weight(device) - device->rs_failed;
-	/* >> 10 to prevent overflow,
-	 * +1 to prevent division by zero */
-	if (*bits_left > *rs_total) {
-		/* D'oh. Maybe a logic bug somewhere.  More likely just a race
-		 * between state change and reset of rs_total.
-		 */
-		*bits_left = *rs_total;
-		*per_mil_done = *rs_total ? 0 : 1000;
-	} else {
-		/* Make sure the division happens in long context.
-		 * We allow up to one petabyte storage right now,
-		 * at a granularity of 4k per bit that is 2**38 bits.
-		 * After shift right and multiplication by 1000,
-		 * this should still fit easily into a 32bit long,
-		 * so we don't need a 64bit division on 32bit arch.
-		 * Note: currently we don't support such large bitmaps on 32bit
-		 * arch anyways, but no harm done to be prepared for it here.
-		 */
-		unsigned int shift = *rs_total > UINT_MAX ? 16 : 10;
-		unsigned long left = *bits_left >> shift;
-		unsigned long total = 1UL + (*rs_total >> shift);
-		unsigned long tmp = 1000UL - left * 1000UL/total;
-		*per_mil_done = tmp;
-	}
-}
-
-
-/*lge
- * progress bars shamelessly adapted from driver/md/md.c
- * output looks like
- *	[=====>..............] 33.5% (23456/123456)
- *	finish: 2:20:20 speed: 6,345 (6,456) K/sec
- */
-static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *seq,
-		union drbd_dev_state state)
-{
-	unsigned long db, dt, dbdt, rt, rs_total, rs_left;
-	unsigned int res;
-	int i, x, y;
-	int stalled = 0;
-
-	drbd_get_syncer_progress(device, state, &rs_total, &rs_left, &res);
-
-	x = res/50;
-	y = 20-x;
-	seq_puts(seq, "\t[");
-	for (i = 1; i < x; i++)
-		seq_putc(seq, '=');
-	seq_putc(seq, '>');
-	for (i = 0; i < y; i++)
-		seq_putc(seq, '.');
-	seq_puts(seq, "] ");
-
-	if (state.conn == C_VERIFY_S || state.conn == C_VERIFY_T)
-		seq_puts(seq, "verified:");
-	else
-		seq_puts(seq, "sync'ed:");
-	seq_printf(seq, "%3u.%u%% ", res / 10, res % 10);
-
-	/* if more than a few GB, display in MB */
-	if (rs_total > (4UL << (30 - BM_BLOCK_SHIFT)))
-		seq_printf(seq, "(%lu/%lu)M",
-			    (unsigned long) Bit2KB(rs_left >> 10),
-			    (unsigned long) Bit2KB(rs_total >> 10));
-	else
-		seq_printf(seq, "(%lu/%lu)K",
-			    (unsigned long) Bit2KB(rs_left),
-			    (unsigned long) Bit2KB(rs_total));
-
-	seq_puts(seq, "\n\t");
-
-	/* see drivers/md/md.c
-	 * We do not want to overflow, so the order of operands and
-	 * the * 100 / 100 trick are important. We do a +1 to be
-	 * safe against division by zero. We only estimate anyway.
-	 *
-	 * dt: time from mark until now
-	 * db: blocks written from mark until now
-	 * rt: remaining time
-	 */
-	/* Rolling marks. last_mark+1 may just now be modified.  last_mark+2 is
-	 * at least (DRBD_SYNC_MARKS-2)*DRBD_SYNC_MARK_STEP old, and has at
-	 * least DRBD_SYNC_MARK_STEP time before it will be modified. */
-	/* ------------------------ ~18s average ------------------------ */
-	i = (device->rs_last_mark + 2) % DRBD_SYNC_MARKS;
-	dt = (jiffies - device->rs_mark_time[i]) / HZ;
-	if (dt > 180)
-		stalled = 1;
-
-	if (!dt)
-		dt++;
-	db = device->rs_mark_left[i] - rs_left;
-	rt = (dt * (rs_left / (db/100+1)))/100; /* seconds */
-
-	seq_printf(seq, "finish: %lu:%02lu:%02lu",
-		rt / 3600, (rt % 3600) / 60, rt % 60);
-
-	dbdt = Bit2KB(db/dt);
-	seq_puts(seq, " speed: ");
-	seq_printf_with_thousands_grouping(seq, dbdt);
-	seq_puts(seq, " (");
-	/* ------------------------- ~3s average ------------------------ */
-	if (drbd_proc_details >= 1) {
-		/* this is what drbd_rs_should_slow_down() uses */
-		i = (device->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
-		dt = (jiffies - device->rs_mark_time[i]) / HZ;
-		if (!dt)
-			dt++;
-		db = device->rs_mark_left[i] - rs_left;
-		dbdt = Bit2KB(db/dt);
-		seq_printf_with_thousands_grouping(seq, dbdt);
-		seq_puts(seq, " -- ");
-	}
-
-	/* --------------------- long term average ---------------------- */
-	/* mean speed since syncer started
-	 * we do account for PausedSync periods */
-	dt = (jiffies - device->rs_start - device->rs_paused) / HZ;
-	if (dt == 0)
-		dt = 1;
-	db = rs_total - rs_left;
-	dbdt = Bit2KB(db/dt);
-	seq_printf_with_thousands_grouping(seq, dbdt);
-	seq_putc(seq, ')');
-
-	if (state.conn == C_SYNC_TARGET ||
-	    state.conn == C_VERIFY_S) {
-		seq_puts(seq, " want: ");
-		seq_printf_with_thousands_grouping(seq, device->c_sync_rate);
-	}
-	seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : "");
-
-	if (drbd_proc_details >= 1) {
-		/* 64 bit:
-		 * we convert to sectors in the display below. */
-		unsigned long bm_bits = drbd_bm_bits(device);
-		unsigned long bit_pos;
-		unsigned long long stop_sector = 0;
-		if (state.conn == C_VERIFY_S ||
-		    state.conn == C_VERIFY_T) {
-			bit_pos = bm_bits - device->ov_left;
-			if (verify_can_do_stop_sector(device))
-				stop_sector = device->ov_stop_sector;
-		} else
-			bit_pos = device->bm_resync_fo;
-		/* Total sectors may be slightly off for oddly
-		 * sized devices. So what. */
-		seq_printf(seq,
-			"\t%3d%% sector pos: %llu/%llu",
-			(int)(bit_pos / (bm_bits/100+1)),
-			(unsigned long long)bit_pos * BM_SECT_PER_BIT,
-			(unsigned long long)bm_bits * BM_SECT_PER_BIT);
-		if (stop_sector != 0 && stop_sector != ULLONG_MAX)
-			seq_printf(seq, " stop sector: %llu", stop_sector);
-		seq_putc(seq, '\n');
-	}
-}
-
 int drbd_seq_show(struct seq_file *seq, void *v)
 {
-	int i, prev_i = -1;
-	const char *sn;
-	struct drbd_device *device;
-	struct net_conf *nc;
-	union drbd_dev_state state;
-	char wp;
-
-	static char write_ordering_chars[] = {
-		[WO_NONE] = 'n',
-		[WO_DRAIN_IO] = 'd',
-		[WO_BDEV_FLUSH] = 'f',
-	};
-
-	seq_printf(seq, "version: " REL_VERSION " (api:%d/proto:%d-%d)\n%s\n",
-		   GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, drbd_buildtag());
-
-	/*
-	  cs .. connection state
-	  ro .. node role (local/remote)
-	  ds .. disk state (local/remote)
-	     protocol
-	     various flags
-	  ns .. network send
-	  nr .. network receive
-	  dw .. disk write
-	  dr .. disk read
-	  al .. activity log write count
-	  bm .. bitmap update write count
-	  pe .. pending (waiting for ack or data reply)
-	  ua .. unack'd (still need to send ack or data reply)
-	  ap .. application requests accepted, but not yet completed
-	  ep .. number of epochs currently "on the fly", P_BARRIER_ACK pending
-	  wo .. write ordering mode currently in use
-	 oos .. known out-of-sync kB
-	*/
-
-	rcu_read_lock();
-	idr_for_each_entry(&drbd_devices, device, i) {
-		if (prev_i != i - 1)
-			seq_putc(seq, '\n');
-		prev_i = i;
-
-		state = device->state;
-		sn = drbd_conn_str(state.conn);
-
-		if (state.conn == C_STANDALONE &&
-		    state.disk == D_DISKLESS &&
-		    state.role == R_SECONDARY) {
-			seq_printf(seq, "%2d: cs:Unconfigured\n", i);
-		} else {
-			/* reset device->congestion_reason */
-
-			nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-			wp = nc ? nc->wire_protocol - DRBD_PROT_A + 'A' : ' ';
-			seq_printf(seq,
-			   "%2d: cs:%s ro:%s/%s ds:%s/%s %c %c%c%c%c%c%c\n"
-			   "    ns:%u nr:%u dw:%u dr:%u al:%u bm:%u "
-			   "lo:%d pe:%d ua:%d ap:%d ep:%d wo:%c",
-			   i, sn,
-			   drbd_role_str(state.role),
-			   drbd_role_str(state.peer),
-			   drbd_disk_str(state.disk),
-			   drbd_disk_str(state.pdsk),
-			   wp,
-			   drbd_suspended(device) ? 's' : 'r',
-			   state.aftr_isp ? 'a' : '-',
-			   state.peer_isp ? 'p' : '-',
-			   state.user_isp ? 'u' : '-',
-			   device->congestion_reason ?: '-',
-			   test_bit(AL_SUSPENDED, &device->flags) ? 's' : '-',
-			   device->send_cnt/2,
-			   device->recv_cnt/2,
-			   device->writ_cnt/2,
-			   device->read_cnt/2,
-			   device->al_writ_cnt,
-			   device->bm_writ_cnt,
-			   atomic_read(&device->local_cnt),
-			   atomic_read(&device->ap_pending_cnt) +
-			   atomic_read(&device->rs_pending_cnt),
-			   atomic_read(&device->unacked_cnt),
-			   atomic_read(&device->ap_bio_cnt),
-			   first_peer_device(device)->connection->epochs,
-			   write_ordering_chars[device->resource->write_ordering]
-			);
-			seq_printf(seq, " oos:%llu\n",
-				   Bit2KB((unsigned long long)
-					   drbd_bm_total_weight(device)));
-		}
-		if (state.conn == C_SYNC_SOURCE ||
-		    state.conn == C_SYNC_TARGET ||
-		    state.conn == C_VERIFY_S ||
-		    state.conn == C_VERIFY_T)
-			drbd_syncer_progress(device, seq, state);
-
-		if (drbd_proc_details >= 1 && get_ldev_if_state(device, D_FAILED)) {
-			lc_seq_printf_stats(seq, device->resync);
-			lc_seq_printf_stats(seq, device->act_log);
-			put_ldev(device);
-		}
-
-		if (drbd_proc_details >= 2)
-			seq_printf(seq, "\tblocked on activity log: %d\n", atomic_read(&device->ap_actlog_cnt));
+	bool any_legacy;
+	static const char legacy_info[] =
+#ifdef CONFIG_DRBD_COMPAT_84
+		" (compat 8.4)";
+#else
+		"";
+#endif
+
+	seq_printf(seq, "version: " REL_VERSION " (api:%d/proto:%d-%d)%s\n%s\n",
+		   GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, legacy_info,
+		   drbd_buildtag());
+
+	any_legacy = drbd_show_legacy_device(seq, v);
+	if (!any_legacy) {
+		/*
+		 * DRBD 8 did not output the transport information, so do not
+		 * display it if any resources are in DRBD 8 compatibility mode.
+		 */
+		drbd_print_transports_loaded(seq);
 	}
-	rcu_read_unlock();
-
 	return 0;
 }
diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h
deleted file mode 100644
index 56bbca9d7700d..0000000000000
--- a/drivers/block/drbd/drbd_protocol.h
+++ /dev/null
@@ -1,428 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef __DRBD_PROTOCOL_H
-#define __DRBD_PROTOCOL_H
-
-enum drbd_packet {
-	/* receiver (data socket) */
-	P_DATA		      = 0x00,
-	P_DATA_REPLY	      = 0x01, /* Response to P_DATA_REQUEST */
-	P_RS_DATA_REPLY	      = 0x02, /* Response to P_RS_DATA_REQUEST */
-	P_BARRIER	      = 0x03,
-	P_BITMAP	      = 0x04,
-	P_BECOME_SYNC_TARGET  = 0x05,
-	P_BECOME_SYNC_SOURCE  = 0x06,
-	P_UNPLUG_REMOTE	      = 0x07, /* Used at various times to hint the peer */
-	P_DATA_REQUEST	      = 0x08, /* Used to ask for a data block */
-	P_RS_DATA_REQUEST     = 0x09, /* Used to ask for a data block for resync */
-	P_SYNC_PARAM	      = 0x0a,
-	P_PROTOCOL	      = 0x0b,
-	P_UUIDS		      = 0x0c,
-	P_SIZES		      = 0x0d,
-	P_STATE		      = 0x0e,
-	P_SYNC_UUID	      = 0x0f,
-	P_AUTH_CHALLENGE      = 0x10,
-	P_AUTH_RESPONSE	      = 0x11,
-	P_STATE_CHG_REQ	      = 0x12,
-
-	/* (meta socket) */
-	P_PING		      = 0x13,
-	P_PING_ACK	      = 0x14,
-	P_RECV_ACK	      = 0x15, /* Used in protocol B */
-	P_WRITE_ACK	      = 0x16, /* Used in protocol C */
-	P_RS_WRITE_ACK	      = 0x17, /* Is a P_WRITE_ACK, additionally call set_in_sync(). */
-	P_SUPERSEDED	      = 0x18, /* Used in proto C, two-primaries conflict detection */
-	P_NEG_ACK	      = 0x19, /* Sent if local disk is unusable */
-	P_NEG_DREPLY	      = 0x1a, /* Local disk is broken... */
-	P_NEG_RS_DREPLY	      = 0x1b, /* Local disk is broken... */
-	P_BARRIER_ACK	      = 0x1c,
-	P_STATE_CHG_REPLY     = 0x1d,
-
-	/* "new" commands, no longer fitting into the ordering scheme above */
-
-	P_OV_REQUEST	      = 0x1e, /* data socket */
-	P_OV_REPLY	      = 0x1f,
-	P_OV_RESULT	      = 0x20, /* meta socket */
-	P_CSUM_RS_REQUEST     = 0x21, /* data socket */
-	P_RS_IS_IN_SYNC	      = 0x22, /* meta socket */
-	P_SYNC_PARAM89	      = 0x23, /* data socket, protocol version 89 replacement for P_SYNC_PARAM */
-	P_COMPRESSED_BITMAP   = 0x24, /* compressed or otherwise encoded bitmap transfer */
-	/* P_CKPT_FENCE_REQ      = 0x25, * currently reserved for protocol D */
-	/* P_CKPT_DISABLE_REQ    = 0x26, * currently reserved for protocol D */
-	P_DELAY_PROBE         = 0x27, /* is used on BOTH sockets */
-	P_OUT_OF_SYNC         = 0x28, /* Mark as out of sync (Outrunning), data socket */
-	P_RS_CANCEL           = 0x29, /* meta: Used to cancel RS_DATA_REQUEST packet by SyncSource */
-	P_CONN_ST_CHG_REQ     = 0x2a, /* data sock: Connection wide state request */
-	P_CONN_ST_CHG_REPLY   = 0x2b, /* meta sock: Connection side state req reply */
-	P_RETRY_WRITE	      = 0x2c, /* Protocol C: retry conflicting write request */
-	P_PROTOCOL_UPDATE     = 0x2d, /* data sock: is used in established connections */
-        /* 0x2e to 0x30 reserved, used in drbd 9 */
-
-	/* REQ_OP_DISCARD. We used "discard" in different contexts before,
-	 * which is why I chose TRIM here, to disambiguate. */
-	P_TRIM                = 0x31,
-
-	/* Only use these two if both support FF_THIN_RESYNC */
-	P_RS_THIN_REQ         = 0x32, /* Request a block for resync or reply P_RS_DEALLOCATED */
-	P_RS_DEALLOCATED      = 0x33, /* Contains only zeros on sync source node */
-
-	/* REQ_WRITE_SAME.
-	 * On a receiving side without REQ_WRITE_SAME,
-	 * we may fall back to an opencoded loop instead. */
-	P_WSAME               = 0x34,
-
-	/* 0x35 already claimed in DRBD 9 */
-	P_ZEROES              = 0x36, /* data sock: zero-out, WRITE_ZEROES */
-
-	/* 0x40 .. 0x48 already claimed in DRBD 9 */
-
-	P_MAY_IGNORE	      = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */
-	P_MAX_OPT_CMD	      = 0x101,
-
-	/* special command ids for handshake */
-
-	P_INITIAL_META	      = 0xfff1, /* First Packet on the MetaSock */
-	P_INITIAL_DATA	      = 0xfff2, /* First Packet on the Socket */
-
-	P_CONNECTION_FEATURES = 0xfffe	/* FIXED for the next century! */
-};
-
-#ifndef __packed
-#define __packed __attribute__((packed))
-#endif
-
-/* This is the layout for a packet on the wire.
- * The byteorder is the network byte order.
- *     (except block_id and barrier fields.
- *	these are pointers to local structs
- *	and have no relevance for the partner,
- *	which just echoes them as received.)
- *
- * NOTE that the payload starts at a long aligned offset,
- * regardless of 32 or 64 bit arch!
- */
-struct p_header80 {
-	u32	  magic;
-	u16	  command;
-	u16	  length;	/* bytes of data after this header */
-} __packed;
-
-/* Header for big packets, Used for data packets exceeding 64kB */
-struct p_header95 {
-	u16	  magic;	/* use DRBD_MAGIC_BIG here */
-	u16	  command;
-	u32	  length;
-} __packed;
-
-struct p_header100 {
-	u32	  magic;
-	u16	  volume;
-	u16	  command;
-	u32	  length;
-	u32	  pad;
-} __packed;
-
-/* These defines must not be changed without changing the protocol version.
- * New defines may only be introduced together with protocol version bump or
- * new protocol feature flags.
- */
-#define DP_HARDBARRIER	      1 /* no longer used */
-#define DP_RW_SYNC	      2 /* equals REQ_SYNC    */
-#define DP_MAY_SET_IN_SYNC    4
-#define DP_UNPLUG             8 /* not used anymore   */
-#define DP_FUA               16 /* equals REQ_FUA     */
-#define DP_FLUSH             32 /* equals REQ_PREFLUSH   */
-#define DP_DISCARD           64 /* equals REQ_OP_DISCARD */
-#define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */
-#define DP_SEND_WRITE_ACK   256 /* This is a proto C write request */
-#define DP_WSAME            512 /* equiv. REQ_WRITE_SAME */
-#define DP_ZEROES          1024 /* equiv. REQ_OP_WRITE_ZEROES */
-
-/* possible combinations:
- * REQ_OP_WRITE_ZEROES:  DP_DISCARD | DP_ZEROES
- * REQ_OP_WRITE_ZEROES + REQ_NOUNMAP: DP_ZEROES
- */
-
-struct p_data {
-	u64	    sector;    /* 64 bits sector number */
-	u64	    block_id;  /* to identify the request in protocol B&C */
-	u32	    seq_num;
-	u32	    dp_flags;
-} __packed;
-
-struct p_trim {
-	struct p_data p_data;
-	u32	    size;	/* == bio->bi_size */
-} __packed;
-
-struct p_wsame {
-	struct p_data p_data;
-	u32           size;     /* == bio->bi_size */
-} __packed;
-
-/*
- * commands which share a struct:
- *  p_block_ack:
- *   P_RECV_ACK (proto B), P_WRITE_ACK (proto C),
- *   P_SUPERSEDED (proto C, two-primaries conflict detection)
- *  p_block_req:
- *   P_DATA_REQUEST, P_RS_DATA_REQUEST
- */
-struct p_block_ack {
-	u64	    sector;
-	u64	    block_id;
-	u32	    blksize;
-	u32	    seq_num;
-} __packed;
-
-struct p_block_req {
-	u64 sector;
-	u64 block_id;
-	u32 blksize;
-	u32 pad;	/* to multiple of 8 Byte */
-} __packed;
-
-/*
- * commands with their own struct for additional fields:
- *   P_CONNECTION_FEATURES
- *   P_BARRIER
- *   P_BARRIER_ACK
- *   P_SYNC_PARAM
- *   ReportParams
- */
-
-/* supports TRIM/DISCARD on the "wire" protocol */
-#define DRBD_FF_TRIM 1
-
-/* Detect all-zeros during resync, and rather TRIM/UNMAP/DISCARD those blocks
- * instead of fully allocate a supposedly thin volume on initial resync */
-#define DRBD_FF_THIN_RESYNC 2
-
-/* supports REQ_WRITE_SAME on the "wire" protocol.
- * Note: this flag is overloaded,
- * its presence also
- *   - indicates support for 128 MiB "batch bios",
- *     max discard size of 128 MiB
- *     instead of 4M before that.
- *   - indicates that we exchange additional settings in p_sizes
- *     drbd_send_sizes()/receive_sizes()
- */
-#define DRBD_FF_WSAME 4
-
-/* supports REQ_OP_WRITE_ZEROES on the "wire" protocol.
- *
- * We used to map that to "discard" on the sending side, and if we cannot
- * guarantee that discard zeroes data, the receiving side would map discard
- * back to zero-out.
- *
- * With the introduction of REQ_OP_WRITE_ZEROES,
- * we started to use that for both WRITE_ZEROES and DISCARDS,
- * hoping that WRITE_ZEROES would "do what we want",
- * UNMAP if possible, zero-out the rest.
- *
- * The example scenario is some LVM "thin" backend.
- *
- * While an un-allocated block on dm-thin reads as zeroes, on a dm-thin
- * with "skip_block_zeroing=true", after a partial block write allocated
- * that block, that same block may well map "undefined old garbage" from
- * the backends on LBAs that have not yet been written to.
- *
- * If we cannot distinguish between zero-out and discard on the receiving
- * side, to avoid "undefined old garbage" to pop up randomly at later times
- * on supposedly zero-initialized blocks, we'd need to map all discards to
- * zero-out on the receiving side.  But that would potentially do a full
- * alloc on thinly provisioned backends, even when the expectation was to
- * unmap/trim/discard/de-allocate.
- *
- * We need to distinguish on the protocol level, whether we need to guarantee
- * zeroes (and thus use zero-out, potentially doing the mentioned full-alloc),
- * or if we want to put the emphasis on discard, and only do a "best effort
- * zeroing" (by "discarding" blocks aligned to discard-granularity, and zeroing
- * only potential unaligned head and tail clippings), to at least *try* to
- * avoid "false positives" in an online-verify later, hoping that someone
- * set skip_block_zeroing=false.
- */
-#define DRBD_FF_WZEROES 8
-
-
-struct p_connection_features {
-	u32 protocol_min;
-	u32 feature_flags;
-	u32 protocol_max;
-
-	/* should be more than enough for future enhancements
-	 * for now, feature_flags and the reserved array shall be zero.
-	 */
-
-	u32 _pad;
-	u64 reserved[7];
-} __packed;
-
-struct p_barrier {
-	u32 barrier;	/* barrier number _handle_ only */
-	u32 pad;	/* to multiple of 8 Byte */
-} __packed;
-
-struct p_barrier_ack {
-	u32 barrier;
-	u32 set_size;
-} __packed;
-
-struct p_rs_param {
-	u32 resync_rate;
-
-	      /* Since protocol version 88 and higher. */
-	char verify_alg[];
-} __packed;
-
-struct p_rs_param_89 {
-	u32 resync_rate;
-	/* protocol version 89: */
-	char verify_alg[SHARED_SECRET_MAX];
-	char csums_alg[SHARED_SECRET_MAX];
-} __packed;
-
-struct p_rs_param_95 {
-	u32 resync_rate;
-	struct_group(algs,
-		char verify_alg[SHARED_SECRET_MAX];
-		char csums_alg[SHARED_SECRET_MAX];
-	);
-	u32 c_plan_ahead;
-	u32 c_delay_target;
-	u32 c_fill_target;
-	u32 c_max_rate;
-} __packed;
-
-enum drbd_conn_flags {
-	CF_DISCARD_MY_DATA = 1,
-	CF_DRY_RUN = 2,
-};
-
-struct p_protocol {
-	u32 protocol;
-	u32 after_sb_0p;
-	u32 after_sb_1p;
-	u32 after_sb_2p;
-	u32 conn_flags;
-	u32 two_primaries;
-
-	/* Since protocol version 87 and higher. */
-	char integrity_alg[];
-
-} __packed;
-
-struct p_uuids {
-	u64 uuid[UI_EXTENDED_SIZE];
-} __packed;
-
-struct p_rs_uuid {
-	u64	    uuid;
-} __packed;
-
-/* optional queue_limits if (agreed_features & DRBD_FF_WSAME)
- * see also struct queue_limits, as of late 2015 */
-struct o_qlim {
-	/* we don't need it yet, but we may as well communicate it now */
-	u32 physical_block_size;
-
-	/* so the original in struct queue_limits is unsigned short,
-	 * but I'd have to put in padding anyways. */
-	u32 logical_block_size;
-
-	/* One incoming bio becomes one DRBD request,
-	 * which may be translated to several bio on the receiving side.
-	 * We don't need to communicate chunk/boundary/segment ... limits.
-	 */
-
-	/* various IO hints may be useful with "diskless client" setups */
-	u32 alignment_offset;
-	u32 io_min;
-	u32 io_opt;
-
-	/* We may need to communicate integrity stuff at some point,
-	 * but let's not get ahead of ourselves. */
-
-	/* Backend discard capabilities.
-	 * Receiving side uses "blkdev_issue_discard()", no need to communicate
-	 * more specifics.  If the backend cannot do discards, the DRBD peer
-	 * may fall back to blkdev_issue_zeroout().
-	 */
-	u8 discard_enabled;
-	u8 discard_zeroes_data;
-	u8 write_same_capable;
-	u8 _pad;
-} __packed;
-
-struct p_sizes {
-	u64	    d_size;  /* size of disk */
-	u64	    u_size;  /* user requested size */
-	u64	    c_size;  /* current exported size */
-	u32	    max_bio_size;  /* Maximal size of a BIO */
-	u16	    queue_order_type;  /* not yet implemented in DRBD*/
-	u16	    dds_flags; /* use enum dds_flags here. */
-
-	/* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */
-	struct o_qlim qlim[];
-} __packed;
-
-struct p_state {
-	u32	    state;
-} __packed;
-
-struct p_req_state {
-	u32	    mask;
-	u32	    val;
-} __packed;
-
-struct p_req_state_reply {
-	u32	    retcode;
-} __packed;
-
-struct p_drbd06_param {
-	u64	  size;
-	u32	  state;
-	u32	  blksize;
-	u32	  protocol;
-	u32	  version;
-	u32	  gen_cnt[5];
-	u32	  bit_map_gen[5];
-} __packed;
-
-struct p_block_desc {
-	u64 sector;
-	u32 blksize;
-	u32 pad;	/* to multiple of 8 Byte */
-} __packed;
-
-/* Valid values for the encoding field.
- * Bump proto version when changing this. */
-enum drbd_bitmap_code {
-	/* RLE_VLI_Bytes = 0,
-	 * and other bit variants had been defined during
-	 * algorithm evaluation. */
-	RLE_VLI_Bits = 2,
-};
-
-struct p_compressed_bm {
-	/* (encoding & 0x0f): actual encoding, see enum drbd_bitmap_code
-	 * (encoding & 0x80): polarity (set/unset) of first runlength
-	 * ((encoding >> 4) & 0x07): pad_bits, number of trailing zero bits
-	 * used to pad up to head.length bytes
-	 */
-	u8 encoding;
-
-	u8 code[];
-} __packed;
-
-struct p_delay_probe93 {
-	u32     seq_num; /* sequence number to match the two probe packets */
-	u32     offset;  /* usecs the probe got sent after the reference time point */
-} __packed;
-
-/*
- * Bitmap packets need to fit within a single page on the sender and receiver,
- * so we are limited to 4 KiB (and not to PAGE_SIZE, which can be bigger).
- */
-#define DRBD_SOCKET_BUFFER_SIZE 4096
-
-#endif  /* __DRBD_PROTOCOL_H */
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 58b95bf4bdca6..06d83b5ffafbc 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -10,42 +10,53 @@
 
  */
 
-
-#include <linux/module.h>
-
-#include <linux/uaccess.h>
 #include <net/sock.h>
 
+#include <linux/bio.h>
 #include <linux/drbd.h>
 #include <linux/fs.h>
 #include <linux/file.h>
 #include <linux/in.h>
 #include <linux/mm.h>
-#include <linux/memcontrol.h>
+#include <linux/memcontrol.h> /* needed on kernels <4.3 */
 #include <linux/mm_inline.h>
 #include <linux/slab.h>
-#include <uapi/linux/sched/types.h>
-#include <linux/sched/signal.h>
 #include <linux/pkt_sched.h>
-#include <linux/unistd.h>
+#include <uapi/linux/sched/types.h>
 #include <linux/vmalloc.h>
 #include <linux/random.h>
-#include <linux/string.h>
-#include <linux/scatterlist.h>
+#include <net/ipv6.h>
 #include <linux/part_stat.h>
-#include <linux/mempool.h>
+
 #include "drbd_int.h"
-#include "drbd_protocol.h"
+#include "drbd_meta_data.h"
+#include <linux/drbd_protocol.h>
 #include "drbd_req.h"
 #include "drbd_vli.h"
 
-#define PRO_FEATURES (DRBD_FF_TRIM|DRBD_FF_THIN_RESYNC|DRBD_FF_WSAME|DRBD_FF_WZEROES)
+enum ao_op {
+	OUTDATE_DISKS,
+	OUTDATE_DISKS_AND_DISCONNECT,
+};
 
-struct packet_info {
-	enum drbd_packet cmd;
-	unsigned int size;
-	unsigned int vnr;
-	void *data;
+struct flush_work {
+	struct drbd_work w;
+	struct drbd_epoch *epoch;
+};
+
+struct update_peers_work {
+       struct drbd_work w;
+       struct drbd_peer_device *peer_device;
+       sector_t sector_start;
+       sector_t sector_end;
+};
+
+enum epoch_event {
+	EV_PUT,
+	EV_GOT_BARRIER_NR,
+	EV_BARRIER_DONE,
+	EV_BECAME_LAST,
+	EV_CLEANUP = 32, /* used as flag */
 };
 
 enum finish_epoch {
@@ -54,46 +65,285 @@ enum finish_epoch {
 	FE_RECYCLED,
 };
 
-static int drbd_do_features(struct drbd_connection *connection);
-static int drbd_do_auth(struct drbd_connection *connection);
-static int drbd_disconnected(struct drbd_peer_device *);
-static void conn_wait_active_ee_empty(struct drbd_connection *connection);
+enum resync_reason {
+	AFTER_UNSTABLE,
+	DISKLESS_PRIMARY,
+};
+
+enum sync_rule {
+	RULE_SYNC_SOURCE_MISSED_FINISH,
+	RULE_SYNC_SOURCE_PEER_MISSED_FINISH,
+	RULE_SYNC_TARGET_MISSED_FINISH,
+	RULE_SYNC_TARGET_PEER_MISSED_FINISH,
+	RULE_SYNC_TARGET_MISSED_START,
+	RULE_SYNC_SOURCE_MISSED_START,
+	RULE_INITIAL_HANDSHAKE_CHANGED,
+	RULE_JUST_CREATED_PEER,
+	RULE_JUST_CREATED_SELF,
+	RULE_JUST_CREATED_BOTH,
+	RULE_CRASHED_PRIMARY,
+	RULE_LOST_QUORUM,
+	RULE_RECONNECTED,
+	RULE_BOTH_OFF,
+	RULE_BITMAP_PEER,
+	RULE_BITMAP_PEER_OTHER,
+	RULE_BITMAP_SELF,
+	RULE_BITMAP_SELF_OTHER,
+	RULE_BITMAP_BOTH,
+	RULE_HISTORY_PEER,
+	RULE_HISTORY_SELF,
+	RULE_HISTORY_BOTH,
+};
+
+static const char * const sync_rule_names[] = {
+	[RULE_SYNC_SOURCE_MISSED_FINISH] = "sync-source-missed-finish",
+	[RULE_SYNC_SOURCE_PEER_MISSED_FINISH] = "sync-source-peer-missed-finish",
+	[RULE_SYNC_TARGET_MISSED_FINISH] = "sync-target-missed-finish",
+	[RULE_SYNC_TARGET_PEER_MISSED_FINISH] = "sync-target-peer-missed-finish",
+	[RULE_SYNC_TARGET_MISSED_START] = "sync-target-missed-start",
+	[RULE_SYNC_SOURCE_MISSED_START] = "sync-source-missed-start",
+	[RULE_INITIAL_HANDSHAKE_CHANGED] = "initial-handshake-changed",
+	[RULE_JUST_CREATED_PEER] = "just-created-peer",
+	[RULE_JUST_CREATED_SELF] = "just-created-self",
+	[RULE_JUST_CREATED_BOTH] = "just-created-both",
+	[RULE_CRASHED_PRIMARY] = "crashed-primary",
+	[RULE_LOST_QUORUM] = "lost-quorum",
+	[RULE_RECONNECTED] = "reconnected",
+	[RULE_BOTH_OFF] = "both-off",
+	[RULE_BITMAP_PEER] = "bitmap-peer",
+	[RULE_BITMAP_PEER_OTHER] = "bitmap-peer-other",
+	[RULE_BITMAP_SELF] = "bitmap-self",
+	[RULE_BITMAP_SELF_OTHER] = "bitmap-self-other",
+	[RULE_BITMAP_BOTH] = "bitmap-both",
+	[RULE_HISTORY_PEER] = "history-peer",
+	[RULE_HISTORY_SELF] = "history-self",
+	[RULE_HISTORY_BOTH] = "history-both",
+};
+
+enum sync_strategy {
+	UNDETERMINED = 0,
+	NO_SYNC,
+	SYNC_SOURCE_IF_BOTH_FAILED,
+	SYNC_SOURCE_USE_BITMAP,
+	SYNC_SOURCE_SET_BITMAP,
+	SYNC_SOURCE_COPY_BITMAP,
+	SYNC_TARGET_IF_BOTH_FAILED,
+	SYNC_TARGET_USE_BITMAP,
+	SYNC_TARGET_SET_BITMAP,
+	SYNC_TARGET_CLEAR_BITMAP,
+	SPLIT_BRAIN_AUTO_RECOVER,
+	SPLIT_BRAIN_DISCONNECT,
+	UNRELATED_DATA,
+	RETRY_CONNECT,
+	REQUIRES_PROTO_91,
+	REQUIRES_PROTO_96,
+	SYNC_TARGET_PRIMARY_RECONNECT,
+	SYNC_TARGET_PRIMARY_DISCONNECT,
+};
+
+struct sync_descriptor {
+	char * const name;
+	int required_protocol;
+	bool is_split_brain;
+	bool is_sync_source;
+	bool is_sync_target;
+	bool reconnect;
+	bool disconnect;
+	int resync_peer_preference;
+	enum sync_strategy full_sync_equivalent;
+	enum sync_strategy reverse;
+};
+
+static const struct sync_descriptor sync_descriptors[] = {
+	[UNDETERMINED] = {
+		.name = "?",
+	},
+	[NO_SYNC] = {
+		.name = "no-sync",
+		.resync_peer_preference = 5,
+	},
+	[SYNC_SOURCE_IF_BOTH_FAILED] = {
+		.name = "source-if-both-failed",
+		.is_sync_source = true,
+		.reverse = SYNC_TARGET_IF_BOTH_FAILED,
+	},
+	[SYNC_SOURCE_USE_BITMAP] = {
+		.name = "source-use-bitmap",
+		.is_sync_source = true,
+		.full_sync_equivalent = SYNC_SOURCE_SET_BITMAP,
+		.reverse = SYNC_TARGET_USE_BITMAP,
+	},
+	[SYNC_SOURCE_SET_BITMAP] = {
+		.name = "source-set-bitmap",
+		.is_sync_source = true,
+		.reverse = SYNC_TARGET_SET_BITMAP,
+	},
+	[SYNC_SOURCE_COPY_BITMAP] = {
+		.name = "source-copy-other-bitmap",
+		.is_sync_source = true,
+	},
+	[SYNC_TARGET_IF_BOTH_FAILED] = {
+		.name = "target-if-both-failed",
+		.is_sync_target = true,
+		.resync_peer_preference = 4,
+		.reverse = SYNC_SOURCE_IF_BOTH_FAILED,
+	},
+	[SYNC_TARGET_USE_BITMAP] = {
+		.name = "target-use-bitmap",
+		.is_sync_target = true,
+		.full_sync_equivalent = SYNC_TARGET_SET_BITMAP,
+		.resync_peer_preference = 3,
+		.reverse = SYNC_SOURCE_USE_BITMAP,
+	},
+	[SYNC_TARGET_SET_BITMAP] = {
+		.name = "target-set-bitmap",
+		.is_sync_target = true,
+		.resync_peer_preference = 2,
+		.reverse = SYNC_SOURCE_SET_BITMAP,
+	},
+	[SYNC_TARGET_CLEAR_BITMAP] = {
+		.name = "target-clear-bitmap",
+		.is_sync_target = true,
+		.resync_peer_preference = 1,
+	},
+	[SPLIT_BRAIN_AUTO_RECOVER] = {
+		.name = "split-brain-auto-recover",
+		.is_split_brain = true,
+		.disconnect = true,
+	},
+	[SPLIT_BRAIN_DISCONNECT] = {
+		.name = "split-brain-disconnect",
+		.is_split_brain = true,
+		.disconnect = true,
+	},
+	[UNRELATED_DATA] = {
+		.name = "unrelated-data",
+		.disconnect = true,
+	},
+	[RETRY_CONNECT] = {
+		.name = "retry-connect",
+		.reconnect = true,
+	},
+	[REQUIRES_PROTO_91] = {
+		.name = "requires-proto-91",
+		.required_protocol = 91,
+		.disconnect = true,
+	},
+	[REQUIRES_PROTO_96] = {
+		.name = "requires-proto-96",
+		.required_protocol = 96,
+		.disconnect = true,
+	},
+	[SYNC_TARGET_PRIMARY_RECONNECT] = {
+		.name = "sync-target-primary-reconnect",
+		.is_sync_target = true,
+		.reconnect = true,
+	},
+	[SYNC_TARGET_PRIMARY_DISCONNECT] = {
+		.name = "sync-target-primary-disconnect",
+		.is_sync_target = true,
+		.disconnect = true,
+	},
+};
+
+enum rcv_timeou_kind {
+	PING_TIMEOUT,
+	REGULAR_TIMEOUT,
+};
+
+int drbd_do_features(struct drbd_connection *connection);
+int drbd_do_auth(struct drbd_connection *connection);
+static void conn_disconnect(struct drbd_connection *connection);
+
 static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *, struct drbd_epoch *, enum epoch_event);
 static int e_end_block(struct drbd_work *, int);
+static void cleanup_unacked_peer_requests(struct drbd_connection *connection);
+static void cleanup_peer_ack_list(struct drbd_connection *connection);
+static u64 node_ids_to_bitmap(struct drbd_device *device, u64 node_ids);
+static void process_twopc(struct drbd_connection *, struct twopc_reply *, struct packet_info *, unsigned long);
+static void drbd_resync(struct drbd_peer_device *, enum resync_reason) __must_hold(local);
+static void drbd_unplug_all_devices(struct drbd_connection *connection);
+static int decode_header(struct drbd_connection *, const void *, struct packet_info *);
+static void check_resync_source(struct drbd_device *device, u64 weak_nodes);
+static void set_rcvtimeo(struct drbd_connection *connection, enum rcv_timeou_kind kind);
+static bool disconnect_expected(struct drbd_connection *connection);
+static bool uuid_in_peer_history(struct drbd_peer_device *peer_device, u64 uuid);
+static bool uuid_in_my_history(struct drbd_device *device, u64 uuid);
+static void drbd_cancel_conflicting_resync_requests(struct drbd_peer_device *peer_device);
+
+static const char *drbd_sync_rule_str(enum sync_rule rule)
+{
+	if (rule < 0 || rule > ARRAY_SIZE(sync_rule_names)) {
+		WARN_ON(true);
+		return "?";
+	}
+	return sync_rule_names[rule];
+}
+
+static struct sync_descriptor strategy_descriptor(enum sync_strategy strategy)
+{
+	if (strategy < 0 || strategy > ARRAY_SIZE(sync_descriptors)) {
+		WARN_ON(true);
+		return sync_descriptors[UNDETERMINED];
+	}
+	return sync_descriptors[strategy];
+}
 
+static bool is_strategy_determined(enum sync_strategy strategy)
+{
+	return strategy == NO_SYNC ||
+			strategy_descriptor(strategy).is_sync_source ||
+			strategy_descriptor(strategy).is_sync_target;
+}
 
-#define GFP_TRY	(__GFP_HIGHMEM | __GFP_NOWARN)
+static struct drbd_epoch *previous_epoch(struct drbd_connection *connection, struct drbd_epoch *epoch)
+{
+	struct drbd_epoch *prev;
+	spin_lock(&connection->epoch_lock);
+	prev = list_entry(epoch->list.prev, struct drbd_epoch, list);
+	if (prev == epoch || prev == connection->current_epoch)
+		prev = NULL;
+	spin_unlock(&connection->epoch_lock);
+	return prev;
+}
 
-static struct page *__drbd_alloc_pages(unsigned int number)
+static struct page *__drbd_alloc_pages(struct drbd_resource *resource, unsigned int number, gfp_t gfp_mask)
 {
 	struct page *page = NULL;
 	struct page *tmp = NULL;
 	unsigned int i = 0;
 
-	/* GFP_TRY, because we must not cause arbitrary write-out: in a DRBD
-	 * "criss-cross" setup, that might cause write-out on some other DRBD,
-	 * which in turn might block on the other node at this very place.  */
 	for (i = 0; i < number; i++) {
-		tmp = mempool_alloc(&drbd_buffer_page_pool, GFP_TRY);
+		tmp = mempool_alloc(&drbd_buffer_page_pool, gfp_mask);
 		if (!tmp)
 			goto fail;
-		set_page_private(tmp, (unsigned long)page);
+		set_page_chain_next_offset_size(tmp, page, 0, 0);
 		page = tmp;
 	}
 	return page;
 fail:
 	page_chain_for_each_safe(page, tmp) {
-		set_page_private(page, 0);
+		set_page_chain_next_offset_size(page, NULL, 0, 0);
 		mempool_free(page, &drbd_buffer_page_pool);
 	}
 	return NULL;
 }
 
+static void rs_sectors_came_in(struct drbd_peer_device *peer_device, int size)
+{
+	int rs_sect_in = atomic_add_return(size >> 9, &peer_device->rs_sect_in);
+
+	/* When resync runs faster than anticipated, consider running the
+	 * resync_work early. */
+	if (rs_sect_in >= peer_device->rs_in_flight)
+		drbd_rs_all_in_flight_came_back(peer_device, rs_sect_in);
+}
+
 /**
  * drbd_alloc_pages() - Returns @number pages, retries forever (or until signalled)
- * @peer_device:	DRBD device.
- * @number:		number of pages requested
- * @retry:		whether to retry, if not enough pages are available right now
+ * @transport:	DRBD transport.
+ * @number:	number of pages requested
+ * @gfp_mask:	how to allocate and whether to loop until we succeed
  *
  * Tries to allocate number pages, first from our own page pool, then from
  * the kernel.
@@ -107,36 +357,37 @@ static struct page *__drbd_alloc_pages(unsigned int number)
  * (checksum based) resync, if the max-buffers, socket buffer sizes and
  * resync-rate settings are mis-configured.
  *
- * Returns a page chain linked via page->private.
+ * Returns a page chain linked via (struct drbd_page_chain*)&page->lru.
  */
-struct page *drbd_alloc_pages(struct drbd_peer_device *peer_device, unsigned int number,
-			      bool retry)
+struct page *drbd_alloc_pages(struct drbd_transport *transport, unsigned int number,
+			      gfp_t gfp_mask)
 {
-	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+	struct drbd_resource *resource = connection->resource;
 	struct page *page;
-	struct net_conf *nc;
 	unsigned int mxb;
 
 	rcu_read_lock();
-	nc = rcu_dereference(peer_device->connection->net_conf);
-	mxb = nc ? nc->max_buffers : 1000000;
+	mxb = rcu_dereference(transport->net_conf)->max_buffers;
 	rcu_read_unlock();
 
-	if (atomic_read(&device->pp_in_use) >= mxb)
+	if (atomic_read(&connection->pp_in_use) >= mxb)
 		schedule_timeout_interruptible(HZ / 10);
-	page = __drbd_alloc_pages(number);
+	page = __drbd_alloc_pages(resource, number, gfp_mask);
 
 	if (page)
-		atomic_add(number, &device->pp_in_use);
+		atomic_add(number, &connection->pp_in_use);
 	return page;
 }
 
 /* Must not be used from irq, as that may deadlock: see drbd_alloc_pages.
- * Is also used from inside an other spin_lock_irq(&resource->req_lock);
- * Either links the page chain back to the global pool,
+ * Either links the page chain back to the pool of free pages,
  * or returns all pages to the system. */
-static void drbd_free_pages(struct drbd_device *device, struct page *page)
+void drbd_free_pages(struct drbd_transport *transport, struct page *page)
 {
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
 	struct page *tmp;
 	int i = 0;
 
@@ -144,43 +395,26 @@ static void drbd_free_pages(struct drbd_device *device, struct page *page)
 		return;
 
 	page_chain_for_each_safe(page, tmp) {
-		set_page_private(page, 0);
+		set_page_chain_next_offset_size(page, NULL, 0, 0);
 		if (page_count(page) == 1)
 			mempool_free(page, &drbd_buffer_page_pool);
 		else
 			put_page(page);
 		i++;
 	}
-	i = atomic_sub_return(i, &device->pp_in_use);
+	i = atomic_sub_return(i, &connection->pp_in_use);
 	if (i < 0)
-		drbd_warn(device, "ASSERTION FAILED: pp_in_use: %d < 0\n", i);
+		drbd_warn(connection, "ASSERTION FAILED: pp_in_use: %d < 0\n", i);
 }
 
-/*
-You need to hold the req_lock:
- _drbd_wait_ee_list_empty()
-
-You must not have the req_lock:
- drbd_free_peer_req()
- drbd_alloc_peer_req()
- drbd_free_peer_reqs()
- drbd_ee_fix_bhs()
- drbd_finish_peer_reqs()
- drbd_clear_done_ee()
- drbd_wait_ee_list_empty()
-*/
-
 /* normal: payload_size == request size (bi_size)
  * w_same: payload_size == logical_block_size
  * trim: payload_size == 0 */
 struct drbd_peer_request *
-drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t sector,
-		    unsigned int request_size, unsigned int payload_size, gfp_t gfp_mask) __must_hold(local)
+drbd_alloc_peer_req(struct drbd_peer_device *peer_device, gfp_t gfp_mask) __must_hold(local)
 {
 	struct drbd_device *device = peer_device->device;
 	struct drbd_peer_request *peer_req;
-	struct page *page = NULL;
-	unsigned int nr_pages = PFN_UP(payload_size);
 
 	if (drbd_insert_fault(device, DRBD_FAULT_AL_EE))
 		return NULL;
@@ -192,63 +426,53 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto
 		return NULL;
 	}
 
-	if (nr_pages) {
-		page = drbd_alloc_pages(peer_device, nr_pages,
-					gfpflags_allow_blocking(gfp_mask));
-		if (!page)
-			goto fail;
-		if (!mempool_is_saturated(&drbd_buffer_page_pool))
-			peer_req->flags |= EE_RELEASE_TO_MEMPOOL;
-	}
-
 	memset(peer_req, 0, sizeof(*peer_req));
 	INIT_LIST_HEAD(&peer_req->w.list);
 	drbd_clear_interval(&peer_req->i);
-	peer_req->i.size = request_size;
-	peer_req->i.sector = sector;
+	INIT_LIST_HEAD(&peer_req->recv_order);
 	peer_req->submit_jif = jiffies;
+	kref_get(&device->kref); /* this kref holds the peer_req->peer_device object alive */
 	peer_req->peer_device = peer_device;
-	peer_req->pages = page;
-	/*
-	 * The block_id is opaque to the receiver.  It is not endianness
-	 * converted, and sent back to the sender unchanged.
-	 */
-	peer_req->block_id = id;
+	peer_req->block_id = (unsigned long) peer_req;
 
 	return peer_req;
-
- fail:
-	mempool_free(peer_req, &drbd_ee_mempool);
-	return NULL;
 }
 
-void drbd_free_peer_req(struct drbd_device *device, struct drbd_peer_request *peer_req)
+void drbd_free_peer_req(struct drbd_peer_request *peer_req)
 {
-	might_sleep();
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+
+	if (peer_req->flags & EE_ON_RECV_ORDER) {
+		spin_lock_irq(&connection->peer_reqs_lock);
+		if (peer_req->i.type == INTERVAL_RESYNC_WRITE)
+			drbd_list_del_resync_request(peer_req);
+		else
+			list_del(&peer_req->recv_order);
+		spin_unlock_irq(&connection->peer_reqs_lock);
+	}
+
 	if (peer_req->flags & EE_HAS_DIGEST)
 		kfree(peer_req->digest);
-	drbd_free_pages(device, peer_req->pages);
-	D_ASSERT(device, atomic_read(&peer_req->pending_bios) == 0);
-	D_ASSERT(device, drbd_interval_empty(&peer_req->i));
-	if (!expect(device, !(peer_req->flags & EE_CALL_AL_COMPLETE_IO))) {
-		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
-		drbd_al_complete_io(device, &peer_req->i);
-	}
+	D_ASSERT(peer_device, atomic_read(&peer_req->pending_bios) == 0);
+	D_ASSERT(peer_device, drbd_interval_empty(&peer_req->i));
+	drbd_free_page_chain(&peer_device->connection->transport, &peer_req->page_chain);
+	kref_put(&peer_device->device->kref, drbd_destroy_device);
 	mempool_free(peer_req, &drbd_ee_mempool);
 }
 
-int drbd_free_peer_reqs(struct drbd_device *device, struct list_head *list)
+int drbd_free_peer_reqs(struct drbd_connection *connection, struct list_head *list)
 {
 	LIST_HEAD(work_list);
 	struct drbd_peer_request *peer_req, *t;
 	int count = 0;
 
-	spin_lock_irq(&device->resource->req_lock);
+	spin_lock_irq(&connection->peer_reqs_lock);
 	list_splice_init(list, &work_list);
-	spin_unlock_irq(&device->resource->req_lock);
+	spin_unlock_irq(&connection->peer_reqs_lock);
 
 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {
-		drbd_free_peer_req(device, peer_req);
+		drbd_free_peer_req(peer_req);
 		count++;
 	}
 	return count;
@@ -257,90 +481,58 @@ int drbd_free_peer_reqs(struct drbd_device *device, struct list_head *list)
 /*
  * See also comments in _req_mod(,BARRIER_ACKED) and receive_Barrier.
  */
-static int drbd_finish_peer_reqs(struct drbd_device *device)
+static int drbd_finish_peer_reqs(struct drbd_connection *connection)
 {
 	LIST_HEAD(work_list);
 	struct drbd_peer_request *peer_req, *t;
 	int err = 0;
+	int n = 0;
 
-	spin_lock_irq(&device->resource->req_lock);
-	list_splice_init(&device->done_ee, &work_list);
-	spin_unlock_irq(&device->resource->req_lock);
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_splice_init(&connection->done_ee, &work_list);
+	spin_unlock_irq(&connection->peer_reqs_lock);
 
 	/* possible callbacks here:
-	 * e_end_block, and e_end_resync_block, e_send_superseded.
+	 * e_end_block, and e_end_resync_block.
 	 * all ignore the last argument.
 	 */
 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {
 		int err2;
 
+		++n;
 		/* list_del not necessary, next/prev members not touched */
+		/* The callback may free peer_req. */
 		err2 = peer_req->w.cb(&peer_req->w, !!err);
 		if (!err)
 			err = err2;
-		drbd_free_peer_req(device, peer_req);
 	}
-	wake_up(&device->ee_wait);
+	if (atomic_sub_and_test(n, &connection->done_ee_cnt))
+		wake_up(&connection->ee_wait);
 
 	return err;
 }
 
-static void _drbd_wait_ee_list_empty(struct drbd_device *device,
-				     struct list_head *head)
-{
-	DEFINE_WAIT(wait);
-
-	/* avoids spin_lock/unlock
-	 * and calling prepare_to_wait in the fast path */
-	while (!list_empty(head)) {
-		prepare_to_wait(&device->ee_wait, &wait, TASK_UNINTERRUPTIBLE);
-		spin_unlock_irq(&device->resource->req_lock);
-		io_schedule();
-		finish_wait(&device->ee_wait, &wait);
-		spin_lock_irq(&device->resource->req_lock);
-	}
-}
-
-static void drbd_wait_ee_list_empty(struct drbd_device *device,
-				    struct list_head *head)
-{
-	spin_lock_irq(&device->resource->req_lock);
-	_drbd_wait_ee_list_empty(device, head);
-	spin_unlock_irq(&device->resource->req_lock);
-}
-
-static int drbd_recv_short(struct socket *sock, void *buf, size_t size, int flags)
-{
-	struct kvec iov = {
-		.iov_base = buf,
-		.iov_len = size,
-	};
-	struct msghdr msg = {
-		.msg_flags = (flags ? flags : MSG_WAITALL | MSG_NOSIGNAL)
-	};
-	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, size);
-	return sock_recvmsg(sock, &msg, msg.msg_flags);
-}
-
-static int drbd_recv(struct drbd_connection *connection, void *buf, size_t size)
+static int drbd_recv(struct drbd_connection *connection, void **buf, size_t size, int flags)
 {
+	struct drbd_transport_ops *tr_ops = &connection->transport.class->ops;
 	int rv;
 
-	rv = drbd_recv_short(connection->data.socket, buf, size, 0);
+	rv = tr_ops->recv(&connection->transport, DATA_STREAM, buf, size, flags);
 
 	if (rv < 0) {
 		if (rv == -ECONNRESET)
 			drbd_info(connection, "sock was reset by peer\n");
 		else if (rv != -ERESTARTSYS)
-			drbd_err(connection, "sock_recvmsg returned %d\n", rv);
+			drbd_info(connection, "sock_recvmsg returned %d\n", rv);
 	} else if (rv == 0) {
-		if (test_bit(DISCONNECT_SENT, &connection->flags)) {
+		if (test_bit(DISCONNECT_EXPECTED, &connection->flags)) {
 			long t;
 			rcu_read_lock();
-			t = rcu_dereference(connection->net_conf)->ping_timeo * HZ/10;
+			t = rcu_dereference(connection->transport.net_conf)->ping_timeo * HZ/10;
 			rcu_read_unlock();
 
-			t = wait_event_timeout(connection->ping_wait, connection->cstate < C_WF_REPORT_PARAMS, t);
+			t = wait_event_timeout(connection->resource->state_wait,
+					       connection->cstate[NOW] < C_CONNECTED, t);
 
 			if (t)
 				goto out;
@@ -349,17 +541,32 @@ static int drbd_recv(struct drbd_connection *connection, void *buf, size_t size)
 	}
 
 	if (rv != size)
-		conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
+		change_cstate(connection, C_BROKEN_PIPE, CS_HARD);
 
 out:
 	return rv;
 }
 
-static int drbd_recv_all(struct drbd_connection *connection, void *buf, size_t size)
+static int drbd_recv_into(struct drbd_connection *connection, void *buf, size_t size)
+{
+	int err;
+
+	err = drbd_recv(connection, &buf, size, CALLER_BUFFER);
+
+	if (err != size) {
+		if (err >= 0)
+			err = -EIO;
+	} else
+		err = 0;
+	return err;
+}
+
+static int drbd_recv_all(struct drbd_connection *connection, void **buf, size_t size)
 {
 	int err;
 
-	err = drbd_recv(connection, buf, size);
+	err = drbd_recv(connection, buf, size, 0);
+
 	if (err != size) {
 		if (err >= 0)
 			err = -EIO;
@@ -368,7 +575,7 @@ static int drbd_recv_all(struct drbd_connection *connection, void *buf, size_t s
 	return err;
 }
 
-static int drbd_recv_all_warn(struct drbd_connection *connection, void *buf, size_t size)
+static int drbd_recv_all_warn(struct drbd_connection *connection, void **buf, size_t size)
 {
 	int err;
 
@@ -378,628 +585,545 @@ static int drbd_recv_all_warn(struct drbd_connection *connection, void *buf, siz
 	return err;
 }
 
-/* quoting tcp(7):
- *   On individual connections, the socket buffer size must be set prior to the
- *   listen(2) or connect(2) calls in order to have it take effect.
- * This is our wrapper to do so.
- */
-static void drbd_setbufsize(struct socket *sock, unsigned int snd,
-		unsigned int rcv)
+static int drbd_send_disconnect(struct drbd_connection *connection)
 {
-	/* open coded SO_SNDBUF, SO_RCVBUF */
-	if (snd) {
-		sock->sk->sk_sndbuf = snd;
-		sock->sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
-	}
-	if (rcv) {
-		sock->sk->sk_rcvbuf = rcv;
-		sock->sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
-	}
+	if (connection->agreed_pro_version < 118)
+		return 0;
+
+	if (!conn_prepare_command(connection, 0, DATA_STREAM))
+		return -EIO;
+	return send_command(connection, -1, P_DISCONNECT, DATA_STREAM);
 }
 
-static struct socket *drbd_try_connect(struct drbd_connection *connection)
+static void initialize_send_buffer(struct drbd_connection *connection, enum drbd_stream drbd_stream)
 {
-	const char *what;
-	struct socket *sock;
-	struct sockaddr_in6 src_in6;
-	struct sockaddr_in6 peer_in6;
-	struct net_conf *nc;
-	int err, peer_addr_len, my_addr_len;
-	int sndbuf_size, rcvbuf_size, connect_int;
-	int disconnect_on_error = 1;
-
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (!nc) {
-		rcu_read_unlock();
-		return NULL;
-	}
-	sndbuf_size = nc->sndbuf_size;
-	rcvbuf_size = nc->rcvbuf_size;
-	connect_int = nc->connect_int;
-	rcu_read_unlock();
-
-	my_addr_len = min_t(int, connection->my_addr_len, sizeof(src_in6));
-	memcpy(&src_in6, &connection->my_addr, my_addr_len);
+	struct drbd_send_buffer *sbuf = &connection->send_buffer[drbd_stream];
 
-	if (((struct sockaddr *)&connection->my_addr)->sa_family == AF_INET6)
-		src_in6.sin6_port = 0;
-	else
-		((struct sockaddr_in *)&src_in6)->sin_port = 0; /* AF_INET & AF_SCI */
+	sbuf->unsent =
+	sbuf->pos = page_address(sbuf->page);
+	sbuf->allocated_size = 0;
+	sbuf->additional_size = 0;
+}
 
-	peer_addr_len = min_t(int, connection->peer_addr_len, sizeof(src_in6));
-	memcpy(&peer_in6, &connection->peer_addr, peer_addr_len);
+/* Gets called if a connection is established, or if a new minor gets created
+   in a connection */
+int drbd_connected(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	u64 weak_nodes = 0;
+	int err;
 
-	what = "sock_create_kern";
-	err = sock_create_kern(&init_net, ((struct sockaddr *)&src_in6)->sa_family,
-			       SOCK_STREAM, IPPROTO_TCP, &sock);
-	if (err < 0) {
-		sock = NULL;
-		goto out;
-	}
+	atomic_set(&peer_device->packet_seq, 0);
+	peer_device->peer_seq = 0;
 
-	sock->sk->sk_rcvtimeo =
-	sock->sk->sk_sndtimeo = connect_int * HZ;
-	drbd_setbufsize(sock, sndbuf_size, rcvbuf_size);
-
-       /* explicitly bind to the configured IP as source IP
-	*  for the outgoing connections.
-	*  This is needed for multihomed hosts and to be
-	*  able to use lo: interfaces for drbd.
-	* Make sure to use 0 as port number, so linux selects
-	*  a free one dynamically.
-	*/
-	what = "bind before connect";
-	err = sock->ops->bind(sock, (struct sockaddr_unsized *) &src_in6, my_addr_len);
-	if (err < 0)
-		goto out;
+	if (device->resource->role[NOW] == R_PRIMARY)
+		weak_nodes = drbd_weak_nodes_device(device);
 
-	/* connect may fail, peer not yet available.
-	 * stay C_WF_CONNECTION, don't go Disconnecting! */
-	disconnect_on_error = 0;
-	what = "connect";
-	err = sock->ops->connect(sock, (struct sockaddr_unsized *) &peer_in6, peer_addr_len, 0);
+	err = drbd_send_sync_param(peer_device);
 
-out:
-	if (err < 0) {
-		if (sock) {
-			sock_release(sock);
-			sock = NULL;
-		}
-		switch (-err) {
-			/* timeout, busy, signal pending */
-		case ETIMEDOUT: case EAGAIN: case EINPROGRESS:
-		case EINTR: case ERESTARTSYS:
-			/* peer not (yet) available, network problem */
-		case ECONNREFUSED: case ENETUNREACH:
-		case EHOSTDOWN:    case EHOSTUNREACH:
-			disconnect_on_error = 0;
-			break;
-		default:
-			drbd_err(connection, "%s failed, err = %d\n", what, err);
-		}
-		if (disconnect_on_error)
-			conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
+	if (!err)
+		err = drbd_send_enable_replication_next(peer_device);
+	if (!err)
+		err = drbd_send_sizes(peer_device, 0, 0);
+	if (!err)
+		err = drbd_send_uuids(peer_device, 0, weak_nodes);
+	if (!err) {
+		set_bit(INITIAL_STATE_SENT, &peer_device->flags);
+		err = drbd_send_current_state(peer_device);
 	}
 
-	return sock;
+	clear_bit(USE_DEGR_WFC_T, &peer_device->flags);
+	clear_bit(RESIZE_PENDING, &peer_device->flags);
+	mod_timer(&device->request_timer, jiffies + HZ); /* just start it here. */
+	return err;
 }
 
-struct accept_wait_data {
-	struct drbd_connection *connection;
-	struct socket *s_listen;
-	struct completion door_bell;
-	void (*original_sk_state_change)(struct sock *sk);
-
-};
-
-static void drbd_incoming_connection(struct sock *sk)
+void conn_connect2(struct drbd_connection *connection)
 {
-	struct accept_wait_data *ad = sk->sk_user_data;
-	void (*state_change)(struct sock *sk);
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	state_change = ad->original_sk_state_change;
-	if (sk->sk_state == TCP_ESTABLISHED)
-		complete(&ad->door_bell);
-	state_change(sk);
-}
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
 
-static int prepare_listen_socket(struct drbd_connection *connection, struct accept_wait_data *ad)
-{
-	int err, sndbuf_size, rcvbuf_size, my_addr_len;
-	struct sockaddr_in6 my_addr;
-	struct socket *s_listen;
-	struct net_conf *nc;
-	const char *what;
+		kref_get(&device->kref);
 
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (!nc) {
+		/* connection cannot go away: caller holds a reference. */
 		rcu_read_unlock();
-		return -EIO;
-	}
-	sndbuf_size = nc->sndbuf_size;
-	rcvbuf_size = nc->rcvbuf_size;
-	rcu_read_unlock();
 
-	my_addr_len = min_t(int, connection->my_addr_len, sizeof(struct sockaddr_in6));
-	memcpy(&my_addr, &connection->my_addr, my_addr_len);
+		/* In the compatibility case with protocol version < 110, that
+		 * is DRBD 8.4, we do not hold uuid_sem while exchanging the
+		 * initial UUID and state packets. There is no need because
+		 * there are no other peers which could interfere. */
+		if (connection->agreed_pro_version >= 110) {
+			down_read_non_owner(&device->uuid_sem);
+			set_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags);
+			/* since drbd_connected() is also called from drbd_create_device()
+			   aquire lock here before calling drbd_connected(). */
+		}
+		drbd_connected(peer_device);
 
-	what = "sock_create_kern";
-	err = sock_create_kern(&init_net, ((struct sockaddr *)&my_addr)->sa_family,
-			       SOCK_STREAM, IPPROTO_TCP, &s_listen);
-	if (err) {
-		s_listen = NULL;
-		goto out;
+		rcu_read_lock();
+		kref_put(&device->kref, drbd_destroy_device);
 	}
+	rcu_read_unlock();
+	drbd_uncork(connection, DATA_STREAM);
+}
 
-	s_listen->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
-	drbd_setbufsize(s_listen, sndbuf_size, rcvbuf_size);
-
-	what = "bind before listen";
-	err = s_listen->ops->bind(s_listen, (struct sockaddr_unsized *)&my_addr, my_addr_len);
-	if (err < 0)
-		goto out;
-
-	ad->s_listen = s_listen;
-	write_lock_bh(&s_listen->sk->sk_callback_lock);
-	ad->original_sk_state_change = s_listen->sk->sk_state_change;
-	s_listen->sk->sk_state_change = drbd_incoming_connection;
-	s_listen->sk->sk_user_data = ad;
-	write_unlock_bh(&s_listen->sk->sk_callback_lock);
-
-	what = "listen";
-	err = s_listen->ops->listen(s_listen, 5);
-	if (err < 0)
-		goto out;
+static bool initial_states_received(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+	bool rv = true;
 
-	return 0;
-out:
-	if (s_listen)
-		sock_release(s_listen);
-	if (err < 0) {
-		if (err != -EAGAIN && err != -EINTR && err != -ERESTARTSYS) {
-			drbd_err(connection, "%s failed, err = %d\n", what, err);
-			conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (!test_bit(INITIAL_STATE_RECEIVED, &peer_device->flags)) {
+			rv = false;
+			break;
 		}
 	}
+	rcu_read_unlock();
 
-	return -EIO;
-}
-
-static void unregister_state_change(struct sock *sk, struct accept_wait_data *ad)
-{
-	write_lock_bh(&sk->sk_callback_lock);
-	sk->sk_state_change = ad->original_sk_state_change;
-	sk->sk_user_data = NULL;
-	write_unlock_bh(&sk->sk_callback_lock);
+	return rv;
 }
 
-static struct socket *drbd_wait_for_connect(struct drbd_connection *connection, struct accept_wait_data *ad)
+void wait_initial_states_received(struct drbd_connection *connection)
 {
-	int timeo, connect_int, err = 0;
-	struct socket *s_estab = NULL;
 	struct net_conf *nc;
+	long timeout;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (!nc) {
-		rcu_read_unlock();
-		return NULL;
-	}
-	connect_int = nc->connect_int;
+	nc = rcu_dereference(connection->transport.net_conf);
+	timeout = nc->ping_timeo * HZ/10;
 	rcu_read_unlock();
+	wait_event_interruptible_timeout(connection->ee_wait,
+					 initial_states_received(connection),
+					 timeout);
+}
 
-	timeo = connect_int * HZ;
-	/* 28.5% random jitter */
-	timeo += get_random_u32_below(2) ? timeo / 7 : -timeo / 7;
-
-	err = wait_for_completion_interruptible_timeout(&ad->door_bell, timeo);
-	if (err <= 0)
-		return NULL;
-
-	err = kernel_accept(ad->s_listen, &s_estab, 0);
-	if (err < 0) {
-		if (err != -EAGAIN && err != -EINTR && err != -ERESTARTSYS) {
-			drbd_err(connection, "accept failed, err = %d\n", err);
-			conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
-		}
-	}
-
-	if (s_estab)
-		unregister_state_change(s_estab->sk, ad);
+void connect_timer_fn(struct timer_list *t)
+{
+	struct drbd_connection *connection = timer_container_of(connection, t, connect_timer);
 
-	return s_estab;
+	drbd_queue_work(&connection->sender_work, &connection->connect_timer_work);
 }
 
-static int decode_header(struct drbd_connection *, void *, struct packet_info *);
-
-static int send_first_packet(struct drbd_connection *connection, struct drbd_socket *sock,
-			     enum drbd_packet cmd)
+static void arm_connect_timer(struct drbd_connection *connection, unsigned long expires)
 {
-	if (!conn_prepare_command(connection, sock))
-		return -EIO;
-	return conn_send_command(connection, sock, cmd, 0, NULL, 0);
+	bool was_pending = mod_timer(&connection->connect_timer, expires);
+
+	if (was_pending) {
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
 }
 
-static int receive_first_packet(struct drbd_connection *connection, struct socket *sock)
+static bool retry_by_rr_conflict(struct drbd_connection *connection)
 {
-	unsigned int header_size = drbd_header_size(connection);
-	struct packet_info pi;
+	enum drbd_after_sb_p rr_conflict;
 	struct net_conf *nc;
-	int err;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (!nc) {
-		rcu_read_unlock();
-		return -EIO;
-	}
-	sock->sk->sk_rcvtimeo = nc->ping_timeo * 4 * HZ / 10;
+	nc = rcu_dereference(connection->transport.net_conf);
+	rr_conflict = nc->rr_conflict;
 	rcu_read_unlock();
 
-	err = drbd_recv_short(sock, connection->data.rbuf, header_size, 0);
-	if (err != header_size) {
-		if (err >= 0)
-			err = -EIO;
-		return err;
-	}
-	err = decode_header(connection, connection->data.rbuf, &pi);
-	if (err)
-		return err;
-	return pi.cmd;
+	return rr_conflict == ASB_RETRY_CONNECT;
 }
 
-/**
- * drbd_socket_okay() - Free the socket if its connection is not okay
- * @sock:	pointer to the pointer to the socket.
- */
-static bool drbd_socket_okay(struct socket **sock)
+static void apply_local_state_change(struct drbd_connection *connection, enum ao_op ao_op, bool force_demote)
 {
-	int rr;
-	char tb[4];
+	/* Although the connect failed, outdate local disks if we learn from the
+	 * handshake that the peer has more recent data */
+	struct drbd_resource *resource = connection->resource;
+	unsigned long irq_flags;
+	int vnr;
 
-	if (!*sock)
-		return false;
+	mutex_lock(&resource->open_release);
+	begin_state_change(resource, &irq_flags, CS_HARD | (force_demote ? CS_FS_IGN_OPENERS : 0));
+	if (ao_op == OUTDATE_DISKS_AND_DISCONNECT)
+		__change_cstate(connection, C_DISCONNECTING);
+	if (resource->role[NOW] == R_SECONDARY ||
+	    (resource->cached_susp && (
+		    resource->res_opts.on_no_data == OND_IO_ERROR ||
+		    resource->res_opts.on_susp_primary_outdated == SPO_FORCE_SECONDARY))) {
+		/* One day we might relax the above condition to
+		 * resource->role[NOW] == R_SECONDARY || resource->cached_susp
+		 * Right now it is that way, because we do not offer a way to gracefully
+		 * get out of a Primary/Outdated state */
+		struct drbd_peer_device *peer_device;
+		bool set_fail_io = false;
 
-	rr = drbd_recv_short(*sock, tb, 4, MSG_DONTWAIT | MSG_PEEK);
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			enum drbd_repl_state r = peer_device->connect_state.conn;
+			struct drbd_device *device = peer_device->device;
 
-	if (rr > 0 || rr == -EAGAIN) {
-		return true;
-	} else {
-		sock_release(*sock);
-		*sock = NULL;
-		return false;
+			if (r == L_WF_BITMAP_T || r == L_SYNC_TARGET || r == L_PAUSED_SYNC_T)
+				__change_disk_state(device, D_OUTDATED);
+
+			if (device->open_cnt)
+				set_fail_io = true;
+		}
+		if (resource->role[NOW] == R_PRIMARY && force_demote) {
+			drbd_warn(connection, "Remote node has more recent data;"
+				  " force secondary!\n");
+			resource->role[NEW] = R_SECONDARY;
+			if (set_fail_io)
+				resource->fail_io[NEW] = true;
+		}
 	}
+	end_state_change(resource, &irq_flags, "connect-failed");
+	mutex_unlock(&resource->open_release);
 }
 
-static bool connection_established(struct drbd_connection *connection,
-				   struct socket **sock1,
-				   struct socket **sock2)
+static int connect_work(struct drbd_work *work, int cancel)
 {
-	struct net_conf *nc;
-	int timeout;
-	bool ok;
+	struct drbd_connection *connection =
+		container_of(work, struct drbd_connection, connect_timer_work);
+	struct drbd_resource *resource = connection->resource;
+	enum drbd_state_rv rv;
+	long t = resource->res_opts.auto_promote_timeout * HZ / 10;
+	bool retry = retry_by_rr_conflict(connection);
+	bool incompat_states, force_demote;
 
-	if (!*sock1 || !*sock2)
-		return false;
+	if (connection->cstate[NOW] != C_CONNECTING)
+		goto out_put;
 
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	timeout = (nc->sock_check_timeo ?: nc->ping_timeo) * HZ / 10;
-	rcu_read_unlock();
-	schedule_timeout_interruptible(timeout);
+	if (connection->agreed_pro_version == 117)
+		wait_initial_states_received(connection);
+
+	do {
+		/* Carefully check if it is okay to do a two_phase_commit from sender context */
+		if (down_trylock(&resource->state_sem)) {
+			rv = SS_CONCURRENT_ST_CHG;
+			break;
+		}
+		rv = change_cstate_tag(connection, C_CONNECTED, CS_SERIALIZE |
+				   CS_ALREADY_SERIALIZED | CS_VERBOSE | CS_DONT_RETRY,
+				   "connected", NULL);
+		up(&resource->state_sem);
+		if (rv != SS_PRIMARY_READER)
+			break;
 
-	ok = drbd_socket_okay(sock1);
-	ok = drbd_socket_okay(sock2) && ok;
+		/* We have a connection established, peer is primary. On my side is a
+		   read-only opener, probably udev or some other scanning after device creating.
+		   This short lived read-only open prevents now that we can continue.
+		   Better retry after the read-only opener goes away. */
+
+		t = wait_event_interruptible_timeout(resource->state_wait,
+						     !drbd_open_ro_count(resource),
+						     t);
+	} while (t > 0);
+
+	incompat_states = (rv == SS_CW_FAILED_BY_PEER || rv == SS_TWO_PRIMARIES);
+	force_demote = resource->role[NOW] == R_PRIMARY &&
+		resource->res_opts.on_susp_primary_outdated == SPO_FORCE_SECONDARY;
+	retry = retry || force_demote;
+
+	if (rv >= SS_SUCCESS) {
+		if (connection->agreed_pro_version < 117)
+			conn_connect2(connection);
+	} else if (rv == SS_TIMEOUT || rv == SS_CONCURRENT_ST_CHG) {
+		if (connection->cstate[NOW] != C_CONNECTING)
+			goto out_put;
+		arm_connect_timer(connection, jiffies + HZ/20);
+		return 0; /* Return early. Keep the reference on the connection! */
+	} else if (rv == SS_HANDSHAKE_RETRY || (incompat_states && retry)) {
+		arm_connect_timer(connection, jiffies + HZ);
+		apply_local_state_change(connection, OUTDATE_DISKS, force_demote);
+		return 0; /* Keep reference */
+	} else if (rv == SS_HANDSHAKE_DISCONNECT || (incompat_states && !retry)) {
+		drbd_send_disconnect(connection);
+		apply_local_state_change(connection, OUTDATE_DISKS_AND_DISCONNECT, force_demote);
+	} else {
+		drbd_info(connection, "Failure to connect: %s (%d); retrying\n",
+			  drbd_set_st_err_str(rv), rv);
+		change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+	}
 
-	return ok;
+ out_put:
+	kref_put(&connection->kref, drbd_destroy_connection);
+	return 0;
 }
 
-/* Gets called if a connection is established, or if a new minor gets created
-   in a connection */
-int drbd_connected(struct drbd_peer_device *peer_device)
+static int drbd_transport_connect(struct drbd_connection *connection)
 {
-	struct drbd_device *device = peer_device->device;
-	int err;
-
-	atomic_set(&device->packet_seq, 0);
-	device->peer_seq = 0;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_resource *resource = connection->resource;
+	int err = 0;
 
-	device->state_mutex = peer_device->connection->agreed_pro_version < 100 ?
-		&peer_device->connection->cstate_mutex :
-		&device->own_state_mutex;
+	mutex_lock(&resource->conf_update);
+	err = transport->class->ops.prepare_connect(transport);
+	mutex_unlock(&resource->conf_update);
 
-	err = drbd_send_sync_param(peer_device);
-	if (!err)
-		err = drbd_send_sizes(peer_device, 0, 0);
-	if (!err)
-		err = drbd_send_uuids(peer_device);
 	if (!err)
-		err = drbd_send_current_state(peer_device);
-	clear_bit(USE_DEGR_WFC_T, &device->flags);
-	clear_bit(RESIZE_PENDING, &device->flags);
-	atomic_set(&device->ap_in_flight, 0);
-	mod_timer(&device->request_timer, jiffies + HZ); /* just start it here. */
+		err = transport->class->ops.connect(transport);
+
+	mutex_lock(&resource->conf_update);
+	transport->class->ops.finish_connect(transport);
+	mutex_unlock(&resource->conf_update);
+
 	return err;
 }
 
 /*
- * return values:
- *   1 yes, we have a valid connection
- *   0 oops, did not work out, please try again
- *  -1 peer talks different language,
- *     no point in trying again, please go standalone.
- *  -2 We do not have a network config...
+ * Returns true if we have a valid connection.
  */
-static int conn_connect(struct drbd_connection *connection)
+static bool conn_connect(struct drbd_connection *connection)
 {
-	struct drbd_socket sock, msock;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_resource *resource = connection->resource;
+	int ping_timeo, ping_int, h, err, vnr;
 	struct drbd_peer_device *peer_device;
+	enum drbd_stream stream;
 	struct net_conf *nc;
-	int vnr, timeout, h;
-	bool discard_my_data, ok;
-	enum drbd_state_rv rv;
-	struct accept_wait_data ad = {
-		.connection = connection,
-		.door_bell = COMPLETION_INITIALIZER_ONSTACK(ad.door_bell),
-	};
-
-	clear_bit(DISCONNECT_SENT, &connection->flags);
-	if (conn_request_state(connection, NS(conn, C_WF_CONNECTION), CS_VERBOSE) < SS_SUCCESS)
-		return -2;
-
-	mutex_init(&sock.mutex);
-	sock.sbuf = connection->data.sbuf;
-	sock.rbuf = connection->data.rbuf;
-	sock.socket = NULL;
-	mutex_init(&msock.mutex);
-	msock.sbuf = connection->meta.sbuf;
-	msock.rbuf = connection->meta.rbuf;
-	msock.socket = NULL;
-
-	/* Assume that the peer only understands protocol 80 until we know better.  */
-	connection->agreed_pro_version = 80;
-
-	if (prepare_listen_socket(connection, &ad))
-		return 0;
+	bool discard_my_data;
+	bool have_mutex;
+	bool no_addr = false;
+
+start:
+	have_mutex = false;
+	clear_bit(PING_PENDING, &connection->flags);
+	clear_bit(DISCONNECT_EXPECTED, &connection->flags);
+	if (change_cstate_tag(connection, C_CONNECTING, CS_VERBOSE, "connecting", NULL)
+			< SS_SUCCESS) {
+		/* We do not have a network config. */
+		return false;
+	}
 
-	do {
-		struct socket *s;
-
-		s = drbd_try_connect(connection);
-		if (s) {
-			if (!sock.socket) {
-				sock.socket = s;
-				send_first_packet(connection, &sock, P_INITIAL_DATA);
-			} else if (!msock.socket) {
-				clear_bit(RESOLVE_CONFLICTS, &connection->flags);
-				msock.socket = s;
-				send_first_packet(connection, &msock, P_INITIAL_META);
-			} else {
-				drbd_err(connection, "Logic error in conn_connect()\n");
-				goto out_release_sockets;
-			}
-		}
+	/* Assume that the peer only understands our minimum supported
+	 * protocol version; until we know better. */
+	connection->agreed_pro_version = drbd_protocol_version_min;
 
-		if (connection_established(connection, &sock.socket, &msock.socket))
-			break;
+	err = drbd_transport_connect(connection);
+	if (err == -EAGAIN) {
+		enum drbd_conn_state cstate;
+		read_lock_irq(&resource->state_rwlock); /* See commit message */
+		cstate = connection->cstate[NOW];
+		read_unlock_irq(&resource->state_rwlock);
+		if (cstate == C_DISCONNECTING)
+			return false;
+		goto retry;
+	} else if (err == -EADDRNOTAVAIL) {
+		struct net_conf *nc;
+		int connect_int;
+		long t;
 
-retry:
-		s = drbd_wait_for_connect(connection, &ad);
-		if (s) {
-			int fp = receive_first_packet(connection, s);
-			drbd_socket_okay(&sock.socket);
-			drbd_socket_okay(&msock.socket);
-			switch (fp) {
-			case P_INITIAL_DATA:
-				if (sock.socket) {
-					drbd_warn(connection, "initial packet S crossed\n");
-					sock_release(sock.socket);
-					sock.socket = s;
-					goto randomize;
-				}
-				sock.socket = s;
-				break;
-			case P_INITIAL_META:
-				set_bit(RESOLVE_CONFLICTS, &connection->flags);
-				if (msock.socket) {
-					drbd_warn(connection, "initial packet M crossed\n");
-					sock_release(msock.socket);
-					msock.socket = s;
-					goto randomize;
-				}
-				msock.socket = s;
-				break;
-			default:
-				drbd_warn(connection, "Error receiving initial packet\n");
-				sock_release(s);
-randomize:
-				if (get_random_u32_below(2))
-					goto retry;
-			}
-		}
+		rcu_read_lock();
+		nc = rcu_dereference(transport->net_conf);
+		connect_int = nc ? nc->connect_int : 10;
+		rcu_read_unlock();
 
-		if (connection->cstate <= C_DISCONNECTING)
-			goto out_release_sockets;
-		if (signal_pending(current)) {
-			flush_signals(current);
-			smp_rmb();
-			if (get_t_state(&connection->receiver) == EXITING)
-				goto out_release_sockets;
+		if (!no_addr) {
+			drbd_warn(connection,
+				  "Configured local address not found, retrying every %d sec, "
+				  "err=%d\n", connect_int, err);
+			no_addr = true;
 		}
 
-		ok = connection_established(connection, &sock.socket, &msock.socket);
-	} while (!ok);
-
-	if (ad.s_listen)
-		sock_release(ad.s_listen);
-
-	sock.socket->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
-	msock.socket->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
-
-	sock.socket->sk->sk_allocation = GFP_NOIO;
-	msock.socket->sk->sk_allocation = GFP_NOIO;
-
-	sock.socket->sk->sk_use_task_frag = false;
-	msock.socket->sk->sk_use_task_frag = false;
+		t = schedule_timeout_interruptible(connect_int * HZ);
+		if (t || connection->cstate[NOW] == C_DISCONNECTING)
+			return false;
+		goto start;
+	} else if (err == -EDESTADDRREQ) {
+		/*
+		 * No destination address, we cannot possibly make a connection.
+		 * Maybe a resource was partially left over due to some other bug?
+		 * Either way, abort here and go StandAlone to prevent reconnection.
+		 */
+		drbd_err(connection, "No destination address, err=%d\n", err);
+		change_cstate_tag(connection, C_STANDALONE, CS_HARD, "no-dest-addr", NULL);
+		return false;
+	} else if (err < 0) {
+		drbd_warn(connection, "Failed to initiate connection, err=%d\n", err);
+		goto abort;
+	}
 
-	sock.socket->sk->sk_priority = TC_PRIO_INTERACTIVE_BULK;
-	msock.socket->sk->sk_priority = TC_PRIO_INTERACTIVE;
+	connection->reassemble_buffer.avail = 0;
 
-	/* NOT YET ...
-	 * sock.socket->sk->sk_sndtimeo = connection->net_conf->timeout*HZ/10;
-	 * sock.socket->sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
-	 * first set it to the P_CONNECTION_FEATURES timeout,
-	 * which we set to 4x the configured ping_timeout. */
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-
-	sock.socket->sk->sk_sndtimeo =
-	sock.socket->sk->sk_rcvtimeo = nc->ping_timeo*4*HZ/10;
-
-	msock.socket->sk->sk_rcvtimeo = nc->ping_int*HZ;
-	timeout = nc->timeout * HZ / 10;
-	discard_my_data = nc->discard_my_data;
+	nc = rcu_dereference(connection->transport.net_conf);
+	ping_timeo = nc->ping_timeo;
+	ping_int = nc->ping_int;
 	rcu_read_unlock();
 
-	msock.socket->sk->sk_sndtimeo = timeout;
-
-	/* we don't want delays.
-	 * we use TCP_CORK where appropriate, though */
-	tcp_sock_set_nodelay(sock.socket->sk);
-	tcp_sock_set_nodelay(msock.socket->sk);
+	/* Make sure we are "uncorked", otherwise we risk timeouts,
+	 * in case this is a reconnect and we had been corked before. */
+	for (stream = DATA_STREAM; stream <= CONTROL_STREAM; stream++) {
+		initialize_send_buffer(connection, stream);
+		drbd_uncork(connection, stream);
+	}
 
-	connection->data.socket = sock.socket;
-	connection->meta.socket = msock.socket;
-	connection->last_received = jiffies;
+	/* Make sure the handshake happens without interference from other threads,
+	 * or the challenge response authentication could be garbled. */
+	mutex_lock(&connection->mutex[DATA_STREAM]);
+	have_mutex = true;
+	transport->class->ops.set_rcvtimeo(transport, DATA_STREAM, ping_timeo * 4 * HZ/10);
+	transport->class->ops.set_rcvtimeo(transport, CONTROL_STREAM, ping_int * HZ);
 
 	h = drbd_do_features(connection);
-	if (h <= 0)
-		return h;
+	if (h < 0)
+		goto abort;
+	if (h == 0)
+		goto retry;
 
 	if (connection->cram_hmac_tfm) {
-		/* drbd_request_state(device, NS(conn, WFAuth)); */
 		switch (drbd_do_auth(connection)) {
 		case -1:
 			drbd_err(connection, "Authentication of peer failed\n");
-			return -1;
+			goto abort;
 		case 0:
 			drbd_err(connection, "Authentication of peer failed, trying again.\n");
-			return 0;
+			goto retry;
 		}
 	}
 
-	connection->data.socket->sk->sk_sndtimeo = timeout;
-	connection->data.socket->sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
-
-	if (drbd_send_protocol(connection) == -EOPNOTSUPP)
-		return -1;
-
-	/* Prevent a race between resync-handshake and
-	 * being promoted to Primary.
-	 *
-	 * Grab and release the state mutex, so we know that any current
-	 * drbd_set_role() is finished, and any incoming drbd_set_role
-	 * will see the STATE_SENT flag, and wait for it to be cleared.
-	 */
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-		mutex_lock(peer_device->device->state_mutex);
-
-	/* avoid a race with conn_request_state( C_DISCONNECTING ) */
-	spin_lock_irq(&connection->resource->req_lock);
-	set_bit(STATE_SENT, &connection->flags);
-	spin_unlock_irq(&connection->resource->req_lock);
+	discard_my_data = test_bit(CONN_DISCARD_MY_DATA, &connection->flags);
 
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-		mutex_unlock(peer_device->device->state_mutex);
+	if (__drbd_send_protocol(connection, P_PROTOCOL) == -EOPNOTSUPP)
+		goto abort;
 
 	rcu_read_lock();
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		kref_get(&device->kref);
-		rcu_read_unlock();
-
+		set_bit(REPLICATION_NEXT, &peer_device->flags);
 		if (discard_my_data)
-			set_bit(DISCARD_MY_DATA, &device->flags);
+			set_bit(DISCARD_MY_DATA, &peer_device->flags);
 		else
-			clear_bit(DISCARD_MY_DATA, &device->flags);
-
-		drbd_connected(peer_device);
-		kref_put(&device->kref, drbd_destroy_device);
-		rcu_read_lock();
+			clear_bit(DISCARD_MY_DATA, &peer_device->flags);
 	}
 	rcu_read_unlock();
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
+	have_mutex = false;
 
-	rv = conn_request_state(connection, NS(conn, C_WF_REPORT_PARAMS), CS_VERBOSE);
-	if (rv < SS_SUCCESS || connection->cstate != C_WF_REPORT_PARAMS) {
-		clear_bit(STATE_SENT, &connection->flags);
-		return 0;
-	}
-
-	drbd_thread_start(&connection->ack_receiver);
-	/* opencoded create_singlethread_workqueue(),
-	 * to be able to use format string arguments */
 	connection->ack_sender =
-		alloc_ordered_workqueue("drbd_as_%s", WQ_MEM_RECLAIM, connection->resource->name);
+		alloc_ordered_workqueue("drbd_as_%s", WQ_MEM_RECLAIM, resource->name);
 	if (!connection->ack_sender) {
 		drbd_err(connection, "Failed to create workqueue ack_sender\n");
-		return 0;
+		schedule_timeout_uninterruptible(HZ);
+		goto retry;
 	}
 
-	mutex_lock(&connection->resource->conf_update);
-	/* The discard_my_data flag is a single-shot modifier to the next
-	 * connection attempt, the handshake of which is now well underway.
-	 * No need for rcu style copying of the whole struct
-	 * just to clear a single value. */
-	connection->net_conf->discard_my_data = 0;
-	mutex_unlock(&connection->resource->conf_update);
+	atomic_set(&connection->ap_in_flight, 0);
+	atomic_set(&connection->rs_in_flight, 0);
 
-	return h;
+	if (connection->agreed_pro_version >= 110) {
+		/* Allow 10 times the ping_timeo for two-phase commits. That is
+		 * 5 seconds by default. The unit of ping_timeo is tenths of a
+		 * second. */
+		transport->class->ops.set_rcvtimeo(transport, DATA_STREAM, ping_timeo * HZ);
 
-out_release_sockets:
-	if (ad.s_listen)
-		sock_release(ad.s_listen);
-	if (sock.socket)
-		sock_release(sock.socket);
-	if (msock.socket)
-		sock_release(msock.socket);
-	return -1;
+		if (connection->agreed_pro_version == 117)
+			conn_connect2(connection);
+
+		if (resource->res_opts.node_id < connection->peer_node_id) {
+			kref_get(&connection->kref);
+			connection->connect_timer_work.cb = connect_work;
+			arm_connect_timer(connection, jiffies);
+		}
+	} else {
+		enum drbd_state_rv rv;
+		rv = change_cstate(connection, C_CONNECTED,
+				   CS_VERBOSE | CS_WAIT_COMPLETE | CS_SERIALIZE | CS_LOCAL_ONLY);
+		if (rv < SS_SUCCESS || connection->cstate[NOW] != C_CONNECTED)
+			goto retry;
+		conn_connect2(connection);
+	}
+
+	clear_bit(PING_TIMEOUT_ACTIVE, &connection->flags);
+	return true;
+
+retry:
+	if (have_mutex)
+		mutex_unlock(&connection->mutex[DATA_STREAM]);
+	conn_disconnect(connection);
+	schedule_timeout_interruptible(HZ);
+	goto start;
+
+abort:
+	if (have_mutex)
+		mutex_unlock(&connection->mutex[DATA_STREAM]);
+	change_cstate(connection, C_DISCONNECTING, CS_HARD);
+	return false;
 }
 
-static int decode_header(struct drbd_connection *connection, void *header, struct packet_info *pi)
+static unsigned int decode_header_size(const void *header)
 {
-	unsigned int header_size = drbd_header_size(connection);
+	const u32 first_dword = *(u32 *)header;
+	const u16 first_word = *(u16 *)header;
 
-	if (header_size == sizeof(struct p_header100) &&
-	    *(__be32 *)header == cpu_to_be32(DRBD_MAGIC_100)) {
-		struct p_header100 *h = header;
-		if (h->pad != 0) {
-			drbd_err(connection, "Header padding is not zero\n");
-			return -EINVAL;
-		}
-		pi->vnr = be16_to_cpu(h->volume);
+	return first_dword == cpu_to_be32(DRBD_MAGIC_100) ? sizeof(struct p_header100) :
+		first_word == cpu_to_be16(DRBD_MAGIC_BIG) ? sizeof(struct p_header95) :
+		sizeof(struct p_header80);
+}
+
+static int __decode_header(const void *header, struct packet_info *pi)
+{
+	const u32 first_dword = *(u32 *)header;
+	const u16 first_word = *(u16 *)header;
+	unsigned int header_size;
+	int header_version;
+
+	if (first_dword == cpu_to_be32(DRBD_MAGIC_100)) {
+		const struct p_header100 *h = header;
+		u16 vnr = be16_to_cpu(h->volume);
+
+		if (h->pad != 0)
+			return -ENOENT;
+
+		pi->vnr = vnr == ((u16) 0xFFFF) ? -1 : vnr;
 		pi->cmd = be16_to_cpu(h->command);
 		pi->size = be32_to_cpu(h->length);
-	} else if (header_size == sizeof(struct p_header95) &&
-		   *(__be16 *)header == cpu_to_be16(DRBD_MAGIC_BIG)) {
-		struct p_header95 *h = header;
+		header_size = sizeof(*h);
+		header_version = 100;
+	} else if (first_word == cpu_to_be16(DRBD_MAGIC_BIG)) {
+		const struct p_header95 *h = header;
+
 		pi->cmd = be16_to_cpu(h->command);
 		pi->size = be32_to_cpu(h->length);
 		pi->vnr = 0;
-	} else if (header_size == sizeof(struct p_header80) &&
-		   *(__be32 *)header == cpu_to_be32(DRBD_MAGIC)) {
-		struct p_header80 *h = header;
+		header_size = sizeof(*h);
+		header_version = 95;
+	} else if (first_dword == cpu_to_be32(DRBD_MAGIC)) {
+		const struct p_header80 *h = header;
+
 		pi->cmd = be16_to_cpu(h->command);
 		pi->size = be16_to_cpu(h->length);
 		pi->vnr = 0;
+		header_size = sizeof(*h);
+		header_version = 80;
 	} else {
-		drbd_err(connection, "Wrong magic value 0x%08x in protocol version %d\n",
-			 be32_to_cpu(*(__be32 *)header),
-			 connection->agreed_pro_version);
 		return -EINVAL;
 	}
-	pi->data = header + header_size;
+
+	pi->data = (void *)(header + header_size); /* casting away 'const'! */
+	return header_version;
+}
+
+static bool header_version_good(int header_version, int protocol_version)
+{
+	switch (header_version) {
+	case 100: return protocol_version >= 100;
+	case 95: return protocol_version < 100;
+	case 80: return protocol_version < 95;
+	default: return false;
+	}
+}
+
+static int decode_header(struct drbd_connection *connection, const void *header,
+			 struct packet_info *pi)
+{
+	const int agreed_pro_version = connection->agreed_pro_version;
+	int header_version = __decode_header(header, pi);
+
+	if (header_version == -ENOENT) {
+		drbd_err(connection, "Header padding is not zero\n");
+		return -EINVAL;
+	} else if (header_version < 0 || !header_version_good(header_version, agreed_pro_version)) {
+		drbd_err(connection, "Wrong magic value 0x%08x in protocol version %d, %d [data]\n",
+			 be32_to_cpu(*(__be32 *)header), agreed_pro_version, header_version);
+		return -EINVAL;
+	}
 	return 0;
 }
 
@@ -1013,49 +1137,58 @@ static void drbd_unplug_all_devices(struct drbd_connection *connection)
 
 static int drbd_recv_header(struct drbd_connection *connection, struct packet_info *pi)
 {
-	void *buffer = connection->data.rbuf;
+	void *buffer;
 	int err;
 
-	err = drbd_recv_all_warn(connection, buffer, drbd_header_size(connection));
+	err = drbd_recv_all_warn(connection, &buffer, drbd_header_size(connection));
 	if (err)
 		return err;
 
 	err = decode_header(connection, buffer, pi);
-	connection->last_received = jiffies;
 
 	return err;
 }
 
 static int drbd_recv_header_maybe_unplug(struct drbd_connection *connection, struct packet_info *pi)
 {
-	void *buffer = connection->data.rbuf;
+	struct drbd_transport_ops *tr_ops = &connection->transport.class->ops;
 	unsigned int size = drbd_header_size(connection);
+	void *buffer;
 	int err;
 
-	err = drbd_recv_short(connection->data.socket, buffer, size, MSG_NOSIGNAL|MSG_DONTWAIT);
+	err = tr_ops->recv(&connection->transport, DATA_STREAM, &buffer,
+			   size, MSG_NOSIGNAL | MSG_DONTWAIT);
 	if (err != size) {
+		int rflags = 0;
+
 		/* If we have nothing in the receive buffer now, to reduce
 		 * application latency, try to drain the backend queues as
 		 * quickly as possible, and let remote TCP know what we have
 		 * received so far. */
 		if (err == -EAGAIN) {
-			tcp_sock_set_quickack(connection->data.socket->sk, 2);
+			tr_ops->hint(&connection->transport, DATA_STREAM, QUICKACK);
 			drbd_unplug_all_devices(connection);
-		}
-		if (err > 0) {
-			buffer += err;
+		} else if (err > 0) {
 			size -= err;
+			rflags |= GROW_BUFFER;
 		}
-		err = drbd_recv_all_warn(connection, buffer, size);
+
+		err = drbd_recv(connection, &buffer, size, rflags);
+		if (err != size) {
+			if (err >= 0)
+				err = -EIO;
+		} else
+			err = 0;
+
 		if (err)
 			return err;
 	}
 
-	err = decode_header(connection, connection->data.rbuf, pi);
-	connection->last_received = jiffies;
+	err = decode_header(connection, buffer, pi);
 
 	return err;
 }
+
 /* This is blkdev_issue_flush, but asynchronous.
  * We want to submit to all component volumes in parallel,
  * then wait for all completions.
@@ -1076,9 +1209,11 @@ static void one_flush_endio(struct bio *bio)
 	struct drbd_device *device = octx->device;
 	struct issue_flush_context *ctx = octx->ctx;
 
-	if (bio->bi_status) {
-		ctx->error = blk_status_to_errno(bio->bi_status);
-		drbd_info(device, "local disk FLUSH FAILED with status %d\n", bio->bi_status);
+	blk_status_t status = bio->bi_status;
+
+	if (status) {
+		ctx->error = blk_status_to_errno(status);
+		drbd_info(device, "local disk FLUSH FAILED with status %d\n", status);
 	}
 	kfree(octx);
 	bio_put(bio);
@@ -1094,7 +1229,7 @@ static void one_flush_endio(struct bio *bio)
 static void submit_one_flush(struct drbd_device *device, struct issue_flush_context *ctx)
 {
 	struct bio *bio = bio_alloc(device->ldev->backing_bdev, 0,
-				    REQ_OP_WRITE | REQ_PREFLUSH, GFP_NOIO);
+			REQ_OP_WRITE | REQ_PREFLUSH, GFP_NOIO);
 	struct one_flush_context *octx = kmalloc_obj(*octx, GFP_NOIO);
 
 	if (!octx) {
@@ -1121,10 +1256,12 @@ static void submit_one_flush(struct drbd_device *device, struct issue_flush_cont
 	submit_bio(bio);
 }
 
-static void drbd_flush(struct drbd_connection *connection)
+static enum finish_epoch drbd_flush_after_epoch(struct drbd_connection *connection, struct drbd_epoch *epoch)
 {
-	if (connection->resource->write_ordering >= WO_BDEV_FLUSH) {
-		struct drbd_peer_device *peer_device;
+	struct drbd_resource *resource = connection->resource;
+
+	if (resource->write_ordering >= WO_BDEV_FLUSH) {
+		struct drbd_device *device;
 		struct issue_flush_context ctx;
 		int vnr;
 
@@ -1133,9 +1270,7 @@ static void drbd_flush(struct drbd_connection *connection)
 		init_completion(&ctx.done);
 
 		rcu_read_lock();
-		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-			struct drbd_device *device = peer_device->device;
-
+		idr_for_each_entry(&resource->devices, device, vnr) {
 			if (!get_ldev(device))
 				continue;
 			kref_get(&device->kref);
@@ -1160,6 +1295,88 @@ static void drbd_flush(struct drbd_connection *connection)
 			drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO);
 		}
 	}
+
+	/* If called before sending P_CONFIRM_STABLE, we don't have the epoch
+	 * (and must not finish it yet, anyways) */
+	if (epoch == NULL)
+		return FE_STILL_LIVE;
+	return drbd_may_finish_epoch(connection, epoch, EV_BARRIER_DONE);
+}
+
+static int w_flush(struct drbd_work *w, int cancel)
+{
+	struct flush_work *fw = container_of(w, struct flush_work, w);
+	struct drbd_epoch *epoch = fw->epoch;
+	struct drbd_connection *connection = epoch->connection;
+
+	kfree(fw);
+
+	if (!test_and_set_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &epoch->flags))
+		drbd_flush_after_epoch(connection, epoch);
+
+	drbd_may_finish_epoch(connection, epoch, EV_PUT |
+			      (connection->cstate[NOW] < C_CONNECTED ? EV_CLEANUP : 0));
+
+	return 0;
+}
+
+static void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
+{
+	struct p_barrier_ack *p;
+
+	if (connection->cstate[NOW] < C_CONNECTED)
+		return;
+
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
+	if (!p)
+		return;
+	p->barrier = barrier_nr;
+	p->set_size = cpu_to_be32(set_size);
+	send_command(connection, -1, P_BARRIER_ACK, CONTROL_STREAM);
+}
+
+static void drbd_send_confirm_stable(struct drbd_peer_request *peer_req)
+{
+	struct drbd_connection *connection = peer_req->peer_device->connection;
+	struct drbd_epoch *epoch = peer_req->epoch;
+	struct drbd_peer_request *oldest, *youngest;
+	struct p_confirm_stable *p;
+	int count;
+
+	if (connection->cstate[NOW] < C_CONNECTED)
+		return;
+
+	/* peer_req is not on stable storage yet, but the only one in this epoch.
+	 * Nothing to confirm, just wait for the normal barrier_ack and peer_ack
+	 * to do their work. */
+	oldest = epoch->oldest_unconfirmed_peer_req;
+	if (oldest == peer_req)
+		return;
+
+	p = conn_prepare_command(connection, sizeof(*p), CONTROL_STREAM);
+	if (!p)
+		return;
+
+	/* peer_req has not been added to connection->peer_requests yet, so
+	 * connection->peer_requests.prev is the youngest request that should
+	 * now be on stable storage. */
+	spin_lock_irq(&connection->peer_reqs_lock);
+	youngest = list_entry(connection->peer_requests.prev, struct drbd_peer_request, recv_order);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	count = atomic_read(&epoch->epoch_size) - atomic_read(&epoch->confirmed) - 1;
+	atomic_add(count, &epoch->confirmed);
+	epoch->oldest_unconfirmed_peer_req = peer_req;
+
+	D_ASSERT(connection, oldest->epoch == youngest->epoch);
+	D_ASSERT(connection, count > 0);
+
+	p->oldest_block_id = oldest->block_id;
+	p->youngest_block_id = youngest->block_id;
+	p->set_size = cpu_to_be32(count);
+	p->pad = 0;
+
+	send_command(connection, -1, P_CONFIRM_STABLE, CONTROL_STREAM);
 }
 
 /**
@@ -1172,13 +1389,16 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
 					       struct drbd_epoch *epoch,
 					       enum epoch_event ev)
 {
-	int epoch_size;
+	int finish, epoch_size;
 	struct drbd_epoch *next_epoch;
+	int schedule_flush = 0;
 	enum finish_epoch rv = FE_STILL_LIVE;
+	struct drbd_resource *resource = connection->resource;
 
 	spin_lock(&connection->epoch_lock);
 	do {
 		next_epoch = NULL;
+		finish = 0;
 
 		epoch_size = atomic_read(&epoch->epoch_size);
 
@@ -1188,6 +1408,16 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
 			break;
 		case EV_GOT_BARRIER_NR:
 			set_bit(DE_HAVE_BARRIER_NUMBER, &epoch->flags);
+
+			/* Special case: If we just switched from WO_BIO_BARRIER to
+			   WO_BDEV_FLUSH we should not finish the current epoch */
+			if (test_bit(DE_CONTAINS_A_BARRIER, &epoch->flags) && epoch_size == 1 &&
+			    resource->write_ordering != WO_BIO_BARRIER &&
+			    epoch == connection->current_epoch)
+				clear_bit(DE_CONTAINS_A_BARRIER, &epoch->flags);
+			break;
+		case EV_BARRIER_DONE:
+			set_bit(DE_BARRIER_IN_NEXT_EPOCH_DONE, &epoch->flags);
 			break;
 		case EV_BECAME_LAST:
 			/* nothing to do*/
@@ -1196,18 +1426,30 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
 
 		if (epoch_size != 0 &&
 		    atomic_read(&epoch->active) == 0 &&
-		    (test_bit(DE_HAVE_BARRIER_NUMBER, &epoch->flags) || ev & EV_CLEANUP)) {
+		    (test_bit(DE_HAVE_BARRIER_NUMBER, &epoch->flags) || ev & EV_CLEANUP) &&
+		    epoch->list.prev == &connection->current_epoch->list &&
+		    !test_bit(DE_IS_FINISHING, &epoch->flags)) {
+			/* Nearly all conditions are met to finish that epoch... */
+			if (test_bit(DE_BARRIER_IN_NEXT_EPOCH_DONE, &epoch->flags) ||
+			    resource->write_ordering == WO_NONE ||
+			    (epoch_size == 1 && test_bit(DE_CONTAINS_A_BARRIER, &epoch->flags)) ||
+			    ev & EV_CLEANUP) {
+				finish = 1;
+				set_bit(DE_IS_FINISHING, &epoch->flags);
+			} else if (!test_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &epoch->flags) &&
+				 resource->write_ordering == WO_BIO_BARRIER) {
+				atomic_inc(&epoch->active);
+				schedule_flush = 1;
+			}
+		}
+		if (finish) {
 			if (!(ev & EV_CLEANUP)) {
+				/* adjust for nr requests already confirmed via P_CONFIRM_STABLE, if any. */
+				epoch_size -= atomic_read(&epoch->confirmed);
 				spin_unlock(&connection->epoch_lock);
 				drbd_send_b_ack(epoch->connection, epoch->barrier_nr, epoch_size);
 				spin_lock(&connection->epoch_lock);
 			}
-#if 0
-			/* FIXME: dec unacked on connection, once we have
-			 * something to count pending connection packets in. */
-			if (test_bit(DE_HAVE_BARRIER_NUMBER, &epoch->flags))
-				dec_unacked(epoch->connection);
-#endif
 
 			if (connection->current_epoch != epoch) {
 				next_epoch = list_entry(epoch->list.next, struct drbd_epoch, list);
@@ -1219,9 +1461,11 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
 				if (rv == FE_STILL_LIVE)
 					rv = FE_DESTROYED;
 			} else {
+				epoch->oldest_unconfirmed_peer_req = NULL;
 				epoch->flags = 0;
 				atomic_set(&epoch->epoch_size, 0);
-				/* atomic_set(&epoch->active, 0); is already zero */
+				atomic_set(&epoch->confirmed, 0);
+				/* atomic_set(&epoch->active, 0); is alrady zero */
 				if (rv == FE_STILL_LIVE)
 					rv = FE_RECYCLED;
 			}
@@ -1235,6 +1479,22 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
 
 	spin_unlock(&connection->epoch_lock);
 
+	if (schedule_flush) {
+		struct flush_work *fw;
+		fw = kmalloc_obj(*fw, GFP_ATOMIC);
+		if (fw) {
+			fw->w.cb = w_flush;
+			fw->epoch = epoch;
+			drbd_queue_work(&resource->work, &fw->w);
+		} else {
+			drbd_warn(resource, "Could not kmalloc a flush_work obj\n");
+			set_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &epoch->flags);
+			/* That is not a recursion, only one level */
+			drbd_may_finish_epoch(connection, epoch, EV_BARRIER_DONE);
+			drbd_may_finish_epoch(connection, epoch, EV_PUT);
+		}
+	}
+
 	return rv;
 }
 
@@ -1245,6 +1505,8 @@ max_allowed_wo(struct drbd_backing_dev *bdev, enum write_ordering_e wo)
 
 	dc = rcu_dereference(bdev->disk_conf);
 
+	if (wo == WO_BIO_BARRIER && !dc->disk_barrier)
+		wo = WO_BDEV_FLUSH;
 	if (wo == WO_BDEV_FLUSH && !dc->disk_flushes)
 		wo = WO_DRAIN_IO;
 	if (wo == WO_DRAIN_IO && !dc->disk_drain)
@@ -1258,22 +1520,26 @@ max_allowed_wo(struct drbd_backing_dev *bdev, enum write_ordering_e wo)
  * @wo:		Write ordering method to try.
  */
 void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backing_dev *bdev,
-			      enum write_ordering_e wo)
+			      enum write_ordering_e wo) __must_hold(local)
 {
 	struct drbd_device *device;
 	enum write_ordering_e pwo;
-	int vnr;
+	int vnr, i = 0;
 	static char *write_ordering_str[] = {
 		[WO_NONE] = "none",
 		[WO_DRAIN_IO] = "drain",
 		[WO_BDEV_FLUSH] = "flush",
+		[WO_BIO_BARRIER] = "barrier",
 	};
 
 	pwo = resource->write_ordering;
-	if (wo != WO_BDEV_FLUSH)
+	if (wo != WO_BIO_BARRIER)
 		wo = min(pwo, wo);
 	rcu_read_lock();
 	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (i++ == 1 && wo == WO_BIO_BARRIER)
+			wo = WO_BDEV_FLUSH; /* WO = barrier does not handle multiple volumes */
+
 		if (get_ldev(device)) {
 			wo = max_allowed_wo(device->ldev, wo);
 			if (device->ldev == bdev)
@@ -1288,21 +1554,11 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin
 	rcu_read_unlock();
 
 	resource->write_ordering = wo;
-	if (pwo != resource->write_ordering || wo == WO_BDEV_FLUSH)
+	if (pwo != resource->write_ordering || wo == WO_BIO_BARRIER)
 		drbd_info(resource, "Method to ensure write ordering: %s\n", write_ordering_str[resource->write_ordering]);
 }
 
 /*
- * Mapping "discard" to ZEROOUT with UNMAP does not work for us:
- * Drivers have to "announce" q->limits.max_write_zeroes_sectors, or it
- * will directly go to fallback mode, submitting normal writes, and
- * never even try to UNMAP.
- *
- * And dm-thin does not do this (yet), mostly because in general it has
- * to assume that "skip_block_zeroing" is set.  See also:
- * https://www.mail-archive.com/dm-devel%40redhat.com/msg07965.html
- * https://www.redhat.com/archives/dm-devel/2018-January/msg00271.html
- *
  * We *may* ignore the discard-zeroes-data setting, if so configured.
  *
  * Assumption is that this "discard_zeroes_data=0" is only because the backend
@@ -1325,6 +1581,7 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin
 int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start, unsigned int nr_sectors, int flags)
 {
 	struct block_device *bdev = device->ldev->backing_bdev;
+	struct request_queue *q = bdev_get_queue(bdev);
 	sector_t tmp, nr;
 	unsigned int max_discard_sectors, granularity;
 	int alignment;
@@ -1334,7 +1591,7 @@ int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start, u
 		goto zero_out;
 
 	/* Zero-sector (unknown) and one-sector granularities are the same.  */
-	granularity = max(bdev_discard_granularity(bdev) >> 9, 1U);
+	granularity = max(q->limits.discard_granularity >> 9, 1U);
 	alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
 
 	max_discard_sectors = min(bdev_max_discard_sectors(bdev), (1U << 22));
@@ -1361,8 +1618,7 @@ int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start, u
 		start = tmp;
 	}
 	while (nr_sectors >= max_discard_sectors) {
-		err |= blkdev_issue_discard(bdev, start, max_discard_sectors,
-					    GFP_NOIO);
+		err |= blkdev_issue_discard(bdev, start, max_discard_sectors, GFP_NOIO);
 		nr_sectors -= max_discard_sectors;
 		start += max_discard_sectors;
 	}
@@ -1420,10 +1676,10 @@ static void drbd_issue_peer_discard_or_zero_out(struct drbd_device *device, stru
 static int peer_request_fault_type(struct drbd_peer_request *peer_req)
 {
 	if (peer_req_op(peer_req) == REQ_OP_READ) {
-		return peer_req->flags & EE_APPLICATION ?
+		return drbd_interval_is_application(&peer_req->i) ?
 			DRBD_FAULT_DT_RD : DRBD_FAULT_RS_RD;
 	} else {
-		return peer_req->flags & EE_APPLICATION ?
+		return drbd_interval_is_application(&peer_req->i) ?
 			DRBD_FAULT_DT_WR : DRBD_FAULT_RS_WR;
 	}
 }
@@ -1441,6 +1697,9 @@ static int peer_request_fault_type(struct drbd_peer_request *peer_req)
  *  single page to an empty bio (which should never happen and likely indicates
  *  that the lower level IO stack is in some way broken). This has been observed
  *  on certain Xen deployments.
+ *
+ *  When this function returns 0, it "consumes" an ldev reference; the
+ *  reference is released when the request completes.
  */
 /* TODO allocate from our own bio_set. */
 int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
@@ -1448,11 +1707,17 @@ int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
 	struct drbd_device *device = peer_req->peer_device->device;
 	struct bio *bios = NULL;
 	struct bio *bio;
-	struct page *page = peer_req->pages;
+	struct page *page = peer_req->page_chain.head;
 	sector_t sector = peer_req->i.sector;
-	unsigned int data_size = peer_req->i.size;
-	unsigned int n_bios = 0;
-	unsigned int nr_pages = PFN_UP(data_size);
+	unsigned data_size = peer_req->i.size;
+	unsigned n_bios = 0;
+	unsigned nr_pages = peer_req->page_chain.nr_pages;
+	unsigned int bio_nr_iovecs;
+	int err;
+
+	if (peer_req->flags & EE_SET_OUT_OF_SYNC)
+		drbd_set_out_of_sync(peer_req->peer_device,
+				peer_req->i.sector, peer_req->i.size);
 
 	/* TRIM/DISCARD: for now, always use the helper function
 	 * blkdev_issue_zeroout(..., discard=true).
@@ -1460,22 +1725,8 @@ int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
 	 * Correctness first, performance later.  Next step is to code an
 	 * asynchronous variant of the same.
 	 */
-	if (peer_req->flags & (EE_TRIM | EE_ZEROOUT)) {
-		/* wait for all pending IO completions, before we start
-		 * zeroing things out. */
-		conn_wait_active_ee_empty(peer_req->peer_device->connection);
-		/* add it to the active list now,
-		 * so we can find it to present it in debugfs */
+	if (peer_req->flags & (EE_TRIM|EE_ZEROOUT)) {
 		peer_req->submit_jif = jiffies;
-		peer_req->flags |= EE_SUBMITTED;
-
-		/* If this was a resync request from receive_rs_deallocated(),
-		 * it is already on the sync_ee list */
-		if (list_empty(&peer_req->w.list)) {
-			spin_lock_irq(&device->resource->req_lock);
-			list_add_tail(&peer_req->w.list, &device->active_ee);
-			spin_unlock_irq(&device->resource->req_lock);
-		}
 
 		drbd_issue_peer_discard_or_zero_out(device, peer_req);
 		return 0;
@@ -1498,10 +1749,22 @@ int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
 	if (!(peer_req_op(peer_req) == REQ_OP_WRITE ||
 				peer_req_op(peer_req) == REQ_OP_READ)) {
 		drbd_err(device, "Invalid bio op received: 0x%x\n", peer_req->opf);
-		return -EINVAL;
+		err = -EINVAL;
+		goto fail;
 	}
 
-	bio = bio_alloc(device->ldev->backing_bdev, nr_pages, peer_req->opf, GFP_NOIO);
+	/*
+	 * Peer may have sent more pages in one request than bio_alloc can
+	 * handle. Even when (nr_pages > bio_nr_iovecs), we may only need one
+	 * bio because bio_add_page may be able to merge consecutive pages into
+	 * one bio_vec.
+	 */
+	bio_nr_iovecs = bio_max_segs(nr_pages);
+
+	/* we special case some flags in the multi-bio case, see below
+	 * (REQ_PREFLUSH, or BIO_RW_BARRIER in older kernels) */
+	bio = bio_alloc(device->ldev->backing_bdev, bio_nr_iovecs, peer_req->opf,
+			GFP_NOIO);
 	/* > peer_req->i.sector, unless this is the first bio */
 	bio->bi_iter.bi_sector = sector;
 	bio->bi_private = peer_req;
@@ -1512,9 +1775,38 @@ int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
 	++n_bios;
 
 	page_chain_for_each(page) {
-		unsigned len = min_t(unsigned, data_size, PAGE_SIZE);
-		if (!bio_add_page(bio, page, len, 0))
+		unsigned off, len;
+		int res;
+
+		if (peer_req_op(peer_req) == REQ_OP_READ) {
+			set_page_chain_offset(page, 0);
+			set_page_chain_size(page, min_t(unsigned, data_size, PAGE_SIZE));
+		}
+		off = page_chain_offset(page);
+		len = page_chain_size(page);
+
+		if (off > PAGE_SIZE || len > PAGE_SIZE - off || len > data_size || len == 0) {
+			drbd_err(device, "invalid page chain: offset %u size %u remaining data_size %u\n",
+					off, len, data_size);
+			err = -EINVAL;
+			goto fail;
+		}
+
+		res = bio_add_page(bio, page, len, off);
+		if (res <= 0) {
+			/* A single page must always be possible!
+			 * But in case it fails anyways,
+			 * we deal with it, and complain (below). */
+			if (bio->bi_vcnt == 0) {
+				drbd_err(device,
+					"bio_add_page(%p, %p, %u, %u): %d (bi_vcnt %u bi_max_vecs %u bi_sector %llu, bi_flags 0x%lx)\n",
+					bio, page, len, off, res, bio->bi_vcnt, bio->bi_max_vecs, (uint64_t)bio->bi_iter.bi_sector,
+					 (unsigned long)bio->bi_flags);
+				err = -ENOSPC;
+				goto fail;
+			}
 			goto next_bio;
+		}
 		data_size -= len;
 		sector += len >> 9;
 		--nr_pages;
@@ -1525,54 +1817,130 @@ int drbd_submit_peer_request(struct drbd_peer_request *peer_req)
 	atomic_set(&peer_req->pending_bios, n_bios);
 	/* for debugfs: update timestamp, mark as submitted */
 	peer_req->submit_jif = jiffies;
-	peer_req->flags |= EE_SUBMITTED;
 	do {
 		bio = bios;
 		bios = bios->bi_next;
 		bio->bi_next = NULL;
 
 		drbd_submit_bio_noacct(device, peer_request_fault_type(peer_req), bio);
+
+		/* strip off REQ_PREFLUSH,
+		 * unless it is the first or last bio */
+		if (bios && bios->bi_next)
+			bios->bi_opf &= ~REQ_PREFLUSH;
 	} while (bios);
 	return 0;
+
+fail:
+	while (bios) {
+		bio = bios;
+		bios = bios->bi_next;
+		bio_put(bio);
+	}
+	return err;
 }
 
-static void drbd_remove_epoch_entry_interval(struct drbd_device *device,
-					     struct drbd_peer_request *peer_req)
+void drbd_remove_peer_req_interval(struct drbd_peer_request *peer_req)
 {
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
 	struct drbd_interval *i = &peer_req->i;
+	unsigned long flags;
 
-	drbd_remove_interval(&device->write_requests, i);
+	spin_lock_irqsave(&device->interval_lock, flags);
+	D_ASSERT(device, !drbd_interval_empty(i));
+	drbd_remove_interval(&device->requests, i);
 	drbd_clear_interval(i);
-
-	/* Wake up any processes waiting for this peer request to complete.  */
-	if (i->waiting)
-		wake_up(&device->misc_wait);
+	if (!drbd_interval_is_verify(&peer_req->i))
+		drbd_release_conflicts(device, i);
+	spin_unlock_irqrestore(&device->interval_lock, flags);
 }
 
-static void conn_wait_active_ee_empty(struct drbd_connection *connection)
+/**
+ * w_e_reissue() - Worker callback; Resubmit a bio
+ * @w:		work object.
+ * @cancel:	The connection will be closed anyways (unused in this callback)
+ */
+int w_e_reissue(struct drbd_work *w, int cancel) __releases(local)
 {
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	struct drbd_peer_request *peer_req =
+		container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	int err;
+	/* We leave DE_CONTAINS_A_BARRIER and EE_IS_BARRIER in place,
+	   (and DE_BARRIER_IN_NEXT_EPOCH_ISSUED in the previous Epoch)
+	   so that we can finish that epoch in drbd_may_finish_epoch().
+	   That is necessary if we already have a long chain of Epochs, before
+	   we realize that BARRIER is actually not supported */
+
+	/* As long as the -ENOTSUPP on the barrier is reported immediately
+	   that will never trigger. If it is reported late, we will just
+	   print that warning and continue correctly for all future requests
+	   with WO_BDEV_FLUSH */
+	if (previous_epoch(connection, peer_req->epoch))
+		drbd_warn(device, "Write ordering was not enforced (one time event)\n");
+
+	/* we still have a local reference,
+	 * get_ldev was done in receive_Data. */
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
+	peer_req->w.cb = e_end_block;
+	err = drbd_submit_peer_request(peer_req);
+	switch (err) {
+	case -ENOMEM:
+		peer_req->w.cb = w_e_reissue;
+		drbd_queue_work(&connection->sender_work,
+				&peer_req->w);
+		/* retry later */
+		fallthrough;
+	case 0:
+		/* keep worker happy and connection up */
+		return 0;
 
-		kref_get(&device->kref);
-		rcu_read_unlock();
-		drbd_wait_ee_list_empty(device, &device->active_ee);
-		kref_put(&device->kref, drbd_destroy_device);
-		rcu_read_lock();
+	case -ENOSPC:
+		/* no other error expected, but anyways: */
+	default:
+		/* forget the object,
+		 * and cause a "Network failure" */
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_al_complete_io(device, &peer_req->i);
+		drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP);
+		drbd_free_peer_req(peer_req);
+		drbd_err(device, "submit failed, triggering re-connect\n");
+		return err;
 	}
-	rcu_read_unlock();
+}
+
+static void conn_wait_done_ee_empty_or_disconnect(struct drbd_connection *connection)
+{
+	wait_event(connection->ee_wait,
+		atomic_read(&connection->done_ee_cnt) == 0
+		|| connection->cstate[NOW] < C_CONNECTED);
+}
+
+static void conn_wait_active_ee_empty_or_disconnect(struct drbd_connection *connection)
+{
+	if (atomic_read(&connection->active_ee_cnt) == 0)
+		return;
+
+	drbd_unplug_all_devices(connection);
+
+	wait_event(connection->ee_wait,
+		atomic_read(&connection->active_ee_cnt) == 0
+		|| connection->cstate[NOW] < C_CONNECTED);
 }
 
 static int receive_Barrier(struct drbd_connection *connection, struct packet_info *pi)
 {
-	int rv;
+	struct drbd_transport_ops *tr_ops = &connection->transport.class->ops;
+	int rv, issue_flush;
 	struct p_barrier *p = pi->data;
 	struct drbd_epoch *epoch;
 
+	tr_ops->hint(&connection->transport, DATA_STREAM, QUICKACK);
+	drbd_unplug_all_devices(connection);
+
 	/* FIXME these are unacked on connection,
 	 * not a specific (peer)device.
 	 */
@@ -1586,41 +1954,48 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
 	 * Therefore we must send the barrier_ack after the barrier request was
 	 * completed. */
 	switch (connection->resource->write_ordering) {
+	case WO_BIO_BARRIER:
 	case WO_NONE:
 		if (rv == FE_RECYCLED)
 			return 0;
-
-		/* receiver context, in the writeout path of the other node.
-		 * avoid potential distributed deadlock */
-		epoch = kmalloc_obj(struct drbd_epoch, GFP_NOIO);
-		if (epoch)
-			break;
-		else
-			drbd_warn(connection, "Allocation of an epoch failed, slowing down\n");
-		fallthrough;
+		break;
 
 	case WO_BDEV_FLUSH:
 	case WO_DRAIN_IO:
-		conn_wait_active_ee_empty(connection);
-		drbd_flush(connection);
+		if (rv == FE_STILL_LIVE) {
+			set_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &connection->current_epoch->flags);
+			conn_wait_active_ee_empty_or_disconnect(connection);
+			rv = drbd_flush_after_epoch(connection, connection->current_epoch);
+		}
+		if (rv == FE_RECYCLED)
+			return 0;
 
-		if (atomic_read(&connection->current_epoch->epoch_size)) {
-			epoch = kmalloc_obj(struct drbd_epoch, GFP_NOIO);
-			if (epoch)
-				break;
+		/*
+		 * The ack_sender will send all the ACKs and barrier ACKs out,
+		 * since all EEs added to done_ee. We need to provide a new
+		 * epoch object for the EEs that come in soon.
+		 */
+		break;
+	}
+
+	/* receiver context, in the writeout path of the other node.
+	 * avoid potential distributed deadlock */
+	epoch = kzalloc_obj(struct drbd_epoch, GFP_NOIO);
+	if (!epoch) {
+		drbd_warn(connection, "Allocation of an epoch failed, slowing down\n");
+		issue_flush = !test_and_set_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &connection->current_epoch->flags);
+		conn_wait_active_ee_empty_or_disconnect(connection);
+		if (issue_flush) {
+			rv = drbd_flush_after_epoch(connection, connection->current_epoch);
+			if (rv == FE_RECYCLED)
+				return 0;
 		}
 
+		conn_wait_done_ee_empty_or_disconnect(connection);
+
 		return 0;
-	default:
-		drbd_err(connection, "Strangeness in connection->write_ordering %d\n",
-			 connection->resource->write_ordering);
-		return -EIO;
 	}
 
-	epoch->flags = 0;
-	atomic_set(&epoch->epoch_size, 0);
-	atomic_set(&epoch->active, 0);
-
 	spin_lock(&connection->epoch_lock);
 	if (atomic_read(&connection->current_epoch->epoch_size)) {
 		list_add(&epoch->list, &connection->current_epoch->list);
@@ -1635,15 +2010,25 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
 	return 0;
 }
 
-/* quick wrapper in case payload size != request_size (write same) */
-static void drbd_csum_ee_size(struct crypto_shash *h,
-			      struct drbd_peer_request *r, void *d,
-			      unsigned int payload_size)
+/* pi->data points into some recv buffer, which may be
+ * re-used/recycled/overwritten by the next receive operation.
+ * (read_in_block via recv_resync_read) */
+static void p_req_detail_from_pi(struct drbd_connection *connection,
+		struct drbd_peer_request_details *d, struct packet_info *pi)
 {
-	unsigned int tmp = r->i.size;
-	r->i.size = payload_size;
-	drbd_csum_ee(h, r, d);
-	r->i.size = tmp;
+	struct p_trim *p = pi->data;
+	bool is_trim_or_zeroes = pi->cmd == P_TRIM || pi->cmd == P_ZEROES;
+	unsigned int digest_size =
+		pi->cmd != P_TRIM && connection->peer_integrity_tfm ?
+		crypto_shash_digestsize(connection->peer_integrity_tfm) : 0;
+
+	d->sector = be64_to_cpu(p->p_data.sector);
+	d->block_id = p->p_data.block_id;
+	d->peer_seq = be32_to_cpu(p->p_data.seq_num);
+	d->dp_flags = be32_to_cpu(p->p_data.dp_flags);
+	d->length = pi->size;
+	d->bi_size = is_trim_or_zeroes ? be32_to_cpu(p->size) : pi->size - digest_size;
+	d->digest_size = digest_size;
 }
 
 /* used from receive_RSDataReply (recv_resync_read)
@@ -1655,140 +2040,95 @@ static void drbd_csum_ee_size(struct crypto_shash *h,
  * both trim and write same have the bi_size ("data len to be affected")
  * as extra argument in the packet header.
  */
-static struct drbd_peer_request *
-read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector,
-	      struct packet_info *pi) __must_hold(local)
+static int
+read_in_block(struct drbd_peer_request *peer_req, struct drbd_peer_request_details *d) __must_hold(local)
 {
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
 	struct drbd_device *device = peer_device->device;
-	const sector_t capacity = get_capacity(device->vdisk);
-	struct drbd_peer_request *peer_req;
-	struct page *page;
-	int digest_size, err;
-	unsigned int data_size = pi->size, ds;
+	const uint64_t capacity = get_capacity(device->vdisk);
+	int err;
 	void *dig_in = peer_device->connection->int_dig_in;
 	void *dig_vv = peer_device->connection->int_dig_vv;
-	unsigned long *data;
-	struct p_trim *trim = (pi->cmd == P_TRIM) ? pi->data : NULL;
-	struct p_trim *zeroes = (pi->cmd == P_ZEROES) ? pi->data : NULL;
+	struct drbd_transport *transport = &peer_device->connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
 
-	digest_size = 0;
-	if (!trim && peer_device->connection->peer_integrity_tfm) {
-		digest_size = crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm);
-		/*
-		 * FIXME: Receive the incoming digest into the receive buffer
-		 *	  here, together with its struct p_data?
-		 */
-		err = drbd_recv_all_warn(peer_device->connection, dig_in, digest_size);
+	if (d->digest_size) {
+		err = drbd_recv_into(peer_device->connection, dig_in, d->digest_size);
 		if (err)
-			return NULL;
-		data_size -= digest_size;
-	}
-
-	/* assume request_size == data_size, but special case trim. */
-	ds = data_size;
-	if (trim) {
-		if (!expect(peer_device, data_size == 0))
-			return NULL;
-		ds = be32_to_cpu(trim->size);
-	} else if (zeroes) {
-		if (!expect(peer_device, data_size == 0))
-			return NULL;
-		ds = be32_to_cpu(zeroes->size);
+			return err;
 	}
 
-	if (!expect(peer_device, IS_ALIGNED(ds, 512)))
-		return NULL;
-	if (trim || zeroes) {
-		if (!expect(peer_device, ds <= (DRBD_MAX_BBIO_SECTORS << 9)))
-			return NULL;
-	} else if (!expect(peer_device, ds <= DRBD_MAX_BIO_SIZE))
-		return NULL;
+	if (!expect(peer_device, IS_ALIGNED(d->bi_size, 512)))
+		return -EINVAL;
+	/* The WSAME mechanism was removed in Linux 5.18,
+	 * and subsequently from drbd.
+	 * In theory, a "modern" drbd will never advertise support for
+	 * WRITE_SAME, so a compliant peer should never send a DP_WSAME
+	 * packet. If we receive one anyway, that's a protocol error.
+	 */
+	if (!expect(peer_device, (d->dp_flags & DP_WSAME) == 0))
+		return -EINVAL;
+	if (d->dp_flags & (DP_DISCARD|DP_ZEROES)) {
+		if (!expect(peer_device, d->bi_size <= (DRBD_MAX_BBIO_SECTORS << 9)))
+			return -EINVAL;
+	} else if (!expect(peer_device, d->bi_size <= DRBD_MAX_BIO_SIZE))
+		return -EINVAL;
 
-	/* even though we trust out peer,
+	/* even though we trust our peer,
 	 * we sometimes have to double check. */
-	if (sector + (ds>>9) > capacity) {
+	if (d->sector + (d->bi_size>>9) > capacity) {
 		drbd_err(device, "request from peer beyond end of local disk: "
 			"capacity: %llus < sector: %llus + size: %u\n",
-			(unsigned long long)capacity,
-			(unsigned long long)sector, ds);
-		return NULL;
+			capacity, d->sector, d->bi_size);
+		return -EINVAL;
 	}
 
-	/* GFP_NOIO, because we must not cause arbitrary write-out: in a DRBD
-	 * "criss-cross" setup, that might cause write-out on some other DRBD,
-	 * which in turn might block on the other node at this very place.  */
-	peer_req = drbd_alloc_peer_req(peer_device, id, sector, ds, data_size, GFP_NOIO);
-	if (!peer_req)
-		return NULL;
+	peer_req->block_id = d->block_id;
 
-	peer_req->flags |= EE_WRITE;
-	if (trim) {
-		peer_req->flags |= EE_TRIM;
-		return peer_req;
-	}
-	if (zeroes) {
-		peer_req->flags |= EE_ZEROOUT;
-		return peer_req;
-	}
+	if (d->length == 0)
+		return 0;
 
-	/* receive payload size bytes into page chain */
-	ds = data_size;
-	page = peer_req->pages;
-	page_chain_for_each(page) {
-		unsigned len = min_t(int, ds, PAGE_SIZE);
-		data = kmap_local_page(page);
-		err = drbd_recv_all_warn(peer_device->connection, data, len);
-		if (drbd_insert_fault(device, DRBD_FAULT_RECEIVE)) {
-			drbd_err(device, "Fault injection: Corrupting data on receive\n");
-			data[0] = data[0] ^ (unsigned long)-1;
-		}
-		kunmap_local(data);
-		if (err) {
-			drbd_free_peer_req(device, peer_req);
-			return NULL;
-		}
-		ds -= len;
+	err = tr_ops->recv_pages(transport, &peer_req->page_chain, d->length - d->digest_size);
+	if (err)
+		return err;
+
+	if (drbd_insert_fault(device, DRBD_FAULT_RECEIVE)) {
+		struct page *page;
+		unsigned long *data;
+		drbd_err(device, "Fault injection: Corrupting data on receive, sector %llu\n",
+				d->sector);
+		page = peer_req->page_chain.head;
+		data = kmap(page) + page_chain_offset(page);
+		data[0] = ~data[0];
+		kunmap(page);
 	}
 
-	if (digest_size) {
-		drbd_csum_ee_size(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv, data_size);
-		if (memcmp(dig_in, dig_vv, digest_size)) {
+	if (d->digest_size) {
+		drbd_csum_pages(peer_device->connection->peer_integrity_tfm, peer_req->page_chain.head, dig_vv);
+		if (memcmp(dig_in, dig_vv, d->digest_size)) {
 			drbd_err(device, "Digest integrity check FAILED: %llus +%u\n",
-				(unsigned long long)sector, data_size);
-			drbd_free_peer_req(device, peer_req);
-			return NULL;
+				d->sector, d->bi_size);
+			return -EINVAL;
 		}
 	}
-	device->recv_cnt += data_size >> 9;
-	return peer_req;
+	peer_device->recv_cnt += d->bi_size >> 9;
+	return 0;
 }
 
-/* drbd_drain_block() just takes a data block
- * out of the socket input buffer, and discards it.
- */
-static int drbd_drain_block(struct drbd_peer_device *peer_device, int data_size)
+static int ignore_remaining_packet(struct drbd_connection *connection, int size)
 {
-	struct page *page;
-	int err = 0;
-	void *data;
-
-	if (!data_size)
-		return 0;
+	void *data_to_ignore;
 
-	page = drbd_alloc_pages(peer_device, 1, 1);
-
-	data = kmap_local_page(page);
-	while (data_size) {
-		unsigned int len = min_t(int, data_size, PAGE_SIZE);
+	while (size) {
+		int s = min_t(int, size, DRBD_SOCKET_BUFFER_SIZE);
+		int rv = drbd_recv(connection, &data_to_ignore, s, 0);
+		if (rv < 0)
+			return rv;
 
-		err = drbd_recv_all_warn(peer_device->connection, data, len);
-		if (err)
-			break;
-		data_size -= len;
+		size -= rv;
 	}
-	kunmap_local(data);
-	drbd_free_pages(peer_device->device, page);
-	return err;
+
+	return 0;
 }
 
 static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_request *req,
@@ -1804,7 +2144,7 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req
 	digest_size = 0;
 	if (peer_device->connection->peer_integrity_tfm) {
 		digest_size = crypto_shash_digestsize(peer_device->connection->peer_integrity_tfm);
-		err = drbd_recv_all_warn(peer_device->connection, dig_in, digest_size);
+		err = drbd_recv_into(peer_device->connection, dig_in, digest_size);
 		if (err)
 			return err;
 		data_size -= digest_size;
@@ -1812,7 +2152,7 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req
 
 	/* optimistically update recv_cnt.  if receiving fails below,
 	 * we disconnect anyways, and counters will be reset. */
-	peer_device->device->recv_cnt += data_size>>9;
+	peer_device->recv_cnt += data_size >> 9;
 
 	bio = req->master_bio;
 	D_ASSERT(peer_device->device, sector == bio->bi_iter.bi_sector);
@@ -1820,7 +2160,7 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req
 	bio_for_each_segment(bvec, bio, iter) {
 		void *mapped = bvec_kmap_local(&bvec);
 		expect = min_t(int, data_size, bvec.bv_len);
-		err = drbd_recv_all_warn(peer_device->connection, mapped, expect);
+		err = drbd_recv_into(peer_device->connection, mapped, expect);
 		kunmap_local(mapped);
 		if (err)
 			return err;
@@ -1839,6 +2179,232 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req
 	return 0;
 }
 
+static bool bits_in_sync(struct drbd_peer_device *peer_device, sector_t sector_start, sector_t sector_end)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap;
+
+	if (peer_device->repl_state[NOW] == L_ESTABLISHED ||
+			peer_device->repl_state[NOW] == L_SYNC_SOURCE ||
+			peer_device->repl_state[NOW] == L_SYNC_TARGET ||
+			peer_device->repl_state[NOW] == L_PAUSED_SYNC_S ||
+			peer_device->repl_state[NOW] == L_PAUSED_SYNC_T) {
+		if (drbd_bm_total_weight(peer_device) == 0)
+			return true;
+		if (drbd_bm_count_bits(device, peer_device->bitmap_index,
+					bm_sect_to_bit(bm, sector_start),
+					bm_sect_to_bit(bm, sector_end - 1)) == 0)
+			return true;
+	}
+	return false;
+}
+
+static void update_peers_for_interval(struct drbd_peer_device *peer_device,
+		struct drbd_interval *interval)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap;
+	u64 mask = NODE_MASK(peer_device->node_id), im;
+	struct drbd_peer_device *p;
+	sector_t sector_end = interval->sector + (interval->size >> SECTOR_SHIFT);
+
+	/* Only send P_PEERS_IN_SYNC if we are actually in sync with this peer. */
+	if (drbd_bm_count_bits(device, peer_device->bitmap_index,
+				bm_sect_to_bit(bm, interval->sector),
+				bm_sect_to_bit(bm, sector_end - 1)))
+		return;
+
+	for_each_peer_device_ref(p, im, device) {
+		if (p == peer_device)
+			continue;
+
+		if (bits_in_sync(p, interval->sector, sector_end))
+			mask |= NODE_MASK(p->node_id);
+	}
+
+	for_each_peer_device_ref(p, im, device) {
+		/* Only send to the peer whose bitmap bits have been cleared if
+		 * we are connected to that peer. The bits may have been
+		 * cleared by a P_PEERS_IN_SYNC from another peer while we are
+		 * connecting to this one. We mustn't send P_PEERS_IN_SYNC
+		 * during the initial connection handshake. */
+		if (p == peer_device && p->connection->cstate[NOW] != C_CONNECTED)
+			continue;
+
+		if (mask & NODE_MASK(p->node_id))
+			drbd_send_peers_in_sync(p, mask, interval->sector, interval->size);
+	}
+}
+
+/* Potentially send P_PEERS_IN_SYNC for a range with size that fits in an int. */
+static void update_peers_for_small_range(struct drbd_peer_device *peer_device,
+		sector_t sector, int size)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_interval interval;
+
+	memset(&interval, 0, sizeof(interval));
+	drbd_clear_interval(&interval);
+	interval.sector = sector;
+	interval.size = size;
+	interval.type = INTERVAL_PEERS_IN_SYNC_LOCK;
+
+	spin_lock_irq(&device->interval_lock);
+	if (drbd_find_conflict(device, &interval, 0)) {
+		spin_unlock_irq(&device->interval_lock);
+		return;
+	}
+	drbd_insert_interval(&device->requests, &interval);
+	/* Interval is not waiting for conflicts to resolve so mark as "submitted". */
+	set_bit(INTERVAL_SUBMITTED, &interval.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	/* Check for activity in the activity log extent _after_ locking the
+	 * interval. Otherwise a write might occur between checking and
+	 * locking. */
+	if (!drbd_al_active(device, sector, size))
+		update_peers_for_interval(peer_device, &interval);
+
+	spin_lock_irq(&device->interval_lock);
+	drbd_remove_interval(&device->requests, &interval);
+	drbd_release_conflicts(device, &interval);
+	spin_unlock_irq(&device->interval_lock);
+}
+
+static void update_peers_for_range(struct drbd_peer_device *peer_device,
+		sector_t sector_start, sector_t sector_end)
+{
+	struct drbd_device *device = peer_device->device;
+	unsigned int enr_start = sector_start >> (AL_EXTENT_SHIFT - SECTOR_SHIFT);
+	unsigned int enr_end = ((sector_end - 1) >> (AL_EXTENT_SHIFT - SECTOR_SHIFT)) + 1;
+	unsigned int enr;
+
+	if (!get_ldev(device))
+		return;
+
+	for (enr = enr_start; enr < enr_end; enr++) {
+		sector_t enr_start_sector = max(sector_start,
+				((sector_t) enr) << (AL_EXTENT_SHIFT - SECTOR_SHIFT));
+		sector_t enr_end_sector = min(sector_end,
+				((sector_t) (enr + 1)) << (AL_EXTENT_SHIFT - SECTOR_SHIFT));
+
+		update_peers_for_small_range(peer_device,
+				enr_start_sector, (enr_end_sector - enr_start_sector) << SECTOR_SHIFT);
+	}
+
+	put_ldev(device);
+}
+
+static int w_update_peers(struct drbd_work *w, int unused)
+{
+	struct update_peers_work *upw = container_of(w, struct update_peers_work, w);
+	struct drbd_peer_device *peer_device = upw->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+
+	if (connection->agreed_pro_version >= 110)
+		update_peers_for_range(peer_device, upw->sector_start, upw->sector_end);
+
+	kfree(upw);
+
+	kref_put(&device->kref, drbd_destroy_device);
+
+	kref_put(&connection->kref, drbd_destroy_connection);
+
+	return 0;
+}
+
+void drbd_queue_update_peers(struct drbd_peer_device *peer_device,
+		sector_t sector_start, sector_t sector_end)
+{
+	struct drbd_device *device = peer_device->device;
+	struct update_peers_work *upw;
+
+	upw = kmalloc_obj(*upw, GFP_ATOMIC | __GFP_NOWARN);
+	if (upw) {
+		upw->sector_start = sector_start;
+		upw->sector_end = sector_end;
+		upw->w.cb = w_update_peers;
+
+		kref_get(&peer_device->device->kref);
+
+		kref_get(&peer_device->connection->kref);
+
+		upw->peer_device = peer_device;
+		drbd_queue_work(&device->resource->work, &upw->w);
+	} else {
+		if (drbd_ratelimit())
+			drbd_warn(peer_device, "kmalloc(upw) failed.\n");
+	}
+}
+
+static void drbd_peers_in_sync_progress(struct drbd_peer_device *peer_device, sector_t sector_end)
+{
+	/* Round down to the boundary defined by PEERS_IN_SYNC_STEP_SHIFT. */
+	sector_t peers_in_sync_end = sector_end & ~PEERS_IN_SYNC_STEP_SECT_MASK;
+
+	/* If we move backwards then move marker back. */
+	if (peers_in_sync_end < peer_device->last_peers_in_sync_end) {
+		peer_device->last_peers_in_sync_end = peers_in_sync_end;
+		return;
+	}
+
+	/* Only schedule when we cross a boundary as defined by PEERS_IN_SYNC_STEP_SHIFT. */
+	if (peers_in_sync_end == peer_device->last_peers_in_sync_end)
+		return;
+
+	drbd_queue_update_peers(peer_device, peer_device->last_peers_in_sync_end, peers_in_sync_end);
+	peer_device->last_peers_in_sync_end = peers_in_sync_end;
+
+	/* Also consider scheduling a bitmap update to reduce the size of the
+	 * next resync if this one is disrupted. */
+	if (drbd_lazy_bitmap_update_due(peer_device))
+		drbd_peer_device_post_work(peer_device, RS_LAZY_BM_WRITE);
+}
+
+static void drbd_check_peers_in_sync_progress(struct drbd_peer_device *peer_device)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	LIST_HEAD(completed);
+	struct drbd_peer_request *peer_req, *tmp;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_for_each_entry_safe(peer_req, tmp, &peer_device->resync_requests, recv_order) {
+		if (!test_bit(INTERVAL_COMPLETED, &peer_req->i.flags))
+			break;
+
+		drbd_list_del_resync_request(peer_req);
+		list_add_tail(&peer_req->recv_order, &completed);
+	}
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	list_for_each_entry_safe(peer_req, tmp, &completed, recv_order) {
+		drbd_peers_in_sync_progress(peer_device, peer_req->i.sector + (peer_req->i.size >> SECTOR_SHIFT));
+		drbd_free_peer_req(peer_req);
+	}
+}
+
+static void drbd_resync_request_complete(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+
+	/*
+	 * Free the pages now but leave the peer request until the
+	 * corresponding peers-in-sync has been scheduled.
+	 */
+	drbd_free_page_chain(&peer_device->connection->transport, &peer_req->page_chain);
+
+	/*
+	 * The interval is no longer in the tree, but use this flag anyway,
+	 * since it has an appropriate meaning. After setting the flag,
+	 * peer_req may be freed by another thread.
+	 */
+	set_bit(INTERVAL_COMPLETED, &peer_req->i.flags);
+	peer_req = NULL;
+
+	drbd_check_peers_in_sync_progress(peer_device);
+}
+
 /*
  * e_end_resync_block() is called in ack_sender context via
  * drbd_finish_peer_reqs().
@@ -1848,39 +2414,171 @@ static int e_end_resync_block(struct drbd_work *w, int unused)
 	struct drbd_peer_request *peer_req =
 		container_of(w, struct drbd_peer_request, w);
 	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
 	sector_t sector = peer_req->i.sector;
+	unsigned int size = peer_req->requested_size;
+	u64 block_id = peer_req->block_id;
 	int err;
 
-	D_ASSERT(device, drbd_interval_empty(&peer_req->i));
-
 	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		drbd_set_in_sync(peer_device, sector, peer_req->i.size);
-		err = drbd_send_ack(peer_device, P_RS_WRITE_ACK, peer_req);
+		drbd_set_in_sync(peer_device, sector, size);
+		err = drbd_send_ack_be(peer_device, P_RS_WRITE_ACK, sector, size, block_id);
 	} else {
 		/* Record failure to sync */
-		drbd_rs_failed_io(peer_device, sector, peer_req->i.size);
+		drbd_rs_failed_io(peer_device, sector, size);
 
-		err  = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);
+		err = drbd_send_ack_be(peer_device, P_RS_NEG_ACK, sector, size, block_id);
 	}
-	dec_unacked(device);
 
+	dec_unacked(peer_device);
+
+	/*
+	 * If INTERVAL_SUBMITTED is not set, this request was merged into
+	 * another discard. It has already been removed from the interval tree.
+	 */
+	if (test_bit(INTERVAL_SUBMITTED, &peer_req->i.flags))
+		drbd_remove_peer_req_interval(peer_req);
+
+	drbd_resync_request_complete(peer_req);
 	return err;
 }
 
-static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t sector,
-			    struct packet_info *pi) __releases(local)
+static struct drbd_peer_request *find_resync_request(struct drbd_peer_device *peer_device,
+		unsigned long type_mask, sector_t sector, unsigned int size, u64 block_id)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_interval *i;
+	struct drbd_peer_request *peer_req = NULL;
+
+	spin_lock_irq(&device->interval_lock);
+	drbd_for_each_overlap(i, &device->requests, sector, size) {
+		struct drbd_peer_request *pr;
+
+		if (!test_bit(INTERVAL_SENT, &i->flags))
+			continue;
+
+		if (!(INTERVAL_TYPE_MASK(i->type) & type_mask))
+			continue;
+
+		if (i->sector != sector || i->size != size)
+			continue;
+
+		pr = container_of(i, struct drbd_peer_request, i);
+		/* With agreed_pro_version < 122, block_id is always ID_SYNCER. */
+		if (pr->peer_device == peer_device &&
+				(block_id == ID_SYNCER || pr->block_id == block_id)) {
+			peer_req = pr;
+			break;
+		}
+	}
+	spin_unlock_irq(&device->interval_lock);
+
+	if (peer_req)
+		D_ASSERT(peer_device, peer_req->i.size == size);
+	else if (drbd_ratelimit())
+		drbd_err(peer_device, "Unexpected resync reply at %llus+%u\n",
+				(unsigned long long) sector, size);
+
+	return peer_req;
+}
+
+/* With agreed_pro_version < 122, one ack may correspond to multiple peer requests. */
+static void find_resync_requests(struct drbd_peer_device *peer_device,
+		struct list_head *matching, sector_t sector, unsigned int size, u64 block_id)
 {
+	struct drbd_connection *connection = peer_device->connection;
 	struct drbd_device *device = peer_device->device;
+	struct drbd_interval *i;
 	struct drbd_peer_request *peer_req;
 
-	peer_req = read_in_block(peer_device, ID_SYNCER, sector, pi);
-	if (!peer_req)
-		goto fail;
+	/* peer_reqs_lock required for using pr->w.list */
+	spin_lock_irq(&connection->peer_reqs_lock);
+	spin_lock(&device->interval_lock);
+	drbd_for_each_overlap(i, &device->requests, sector, size) {
+		if (!test_bit(INTERVAL_SUBMITTED, &i->flags))
+			continue;
+
+		if (i->type != INTERVAL_RESYNC_READ)
+			continue;
+
+		peer_req = container_of(i, struct drbd_peer_request, i);
+		/* With agreed_pro_version < 122, block_id is always ID_SYNCER. */
+		if (peer_req->peer_device == peer_device &&
+				(block_id == ID_SYNCER || peer_req->block_id == block_id))
+			list_move_tail(&peer_req->w.list, matching); /* from resync_ack_ee */
+	}
+	spin_unlock(&device->interval_lock);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+}
+
+static void drbd_cleanup_received_resync_write(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+
+	drbd_remove_peer_req_interval(peer_req);
+
+	atomic_sub(peer_req->i.size >> SECTOR_SHIFT, &device->rs_sect_ev);
+	dec_unacked(peer_device);
+
+	drbd_free_peer_req(peer_req);
+	put_ldev(device);
+
+	if (atomic_dec_and_test(&connection->backing_ee_cnt))
+		wake_up(&connection->ee_wait);
+}
+
+void drbd_conflict_submit_resync_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	bool conflict;
+	bool canceled;
+
+	spin_lock_irq(&device->interval_lock);
+	clear_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &peer_req->i.flags);
+	canceled = test_bit(INTERVAL_CANCELED, &peer_req->i.flags);
+	set_bit(INTERVAL_RECEIVED, &peer_req->i.flags);
+	conflict = drbd_find_conflict(device, &peer_req->i, 0);
+	if (!conflict)
+		set_bit(INTERVAL_SUBMITTED, &peer_req->i.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	if (!conflict) {
+		int err = drbd_submit_peer_request(peer_req);
+		if (err) {
+			if (drbd_ratelimit())
+				drbd_err(device, "submit failed, triggering re-connect\n");
+
+			drbd_cleanup_received_resync_write(peer_req);
+			change_cstate(peer_device->connection, C_PROTOCOL_ERROR, CS_HARD);
+		}
+	} else if (canceled) {
+		drbd_cleanup_received_resync_write(peer_req);
+	}
+}
+
+static int recv_resync_read(struct drbd_peer_device *peer_device,
+			    struct drbd_peer_request *peer_req,
+			    struct drbd_peer_request_details *d) __releases(local)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	unsigned int size;
+	sector_t sector;
+	int err;
+	u64 im;
+
+	err = read_in_block(peer_req, d);
+	if (err)
+		return -EIO;
+
+	if (test_bit(UNSTABLE_RESYNC, &peer_device->flags))
+		clear_bit(STABLE_RESYNC, &device->flags);
 
 	dec_rs_pending(peer_device);
 
-	inc_unacked(device);
+	inc_unacked(peer_device);
 	/* corresponding dec_unacked() in e_end_resync_block()
 	 * respective _drbd_clear_done_ee */
 
@@ -1888,35 +2586,43 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto
 	peer_req->opf = REQ_OP_WRITE;
 	peer_req->submit_jif = jiffies;
 
-	spin_lock_irq(&device->resource->req_lock);
-	list_add_tail(&peer_req->w.list, &device->sync_ee);
-	spin_unlock_irq(&device->resource->req_lock);
+	atomic_add(d->bi_size >> 9, &device->rs_sect_ev);
 
-	atomic_add(pi->size >> 9, &device->rs_sect_ev);
-	if (drbd_submit_peer_request(peer_req) == 0)
-		return 0;
+	sector = peer_req->i.sector;
+	size = peer_req->i.size;
 
-	/* don't care for the reason here */
-	drbd_err(device, "submit failed, triggering re-connect\n");
-	spin_lock_irq(&device->resource->req_lock);
-	list_del(&peer_req->w.list);
-	spin_unlock_irq(&device->resource->req_lock);
+	/* Setting all peers out of sync here. The sync source peer will be
+	 * set in sync when the write completes. The sync source will soon
+	 * set other peers in sync with a P_PEERS_IN_SYNC packet.
+	 */
+	drbd_set_all_out_of_sync(device, sector, size);
 
-	drbd_free_peer_req(device, peer_req);
-fail:
-	put_ldev(device);
-	return -EIO;
+	atomic_inc(&connection->backing_ee_cnt);
+	drbd_conflict_submit_resync_request(peer_req);
+	peer_req = NULL; /* since submitted, might be destroyed already */
+
+	drbd_process_rs_discards(peer_device, false);
+
+	for_each_peer_device_ref(peer_device, im, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+
+		if (repl_is_sync_source(repl_state) || repl_state == L_WF_BITMAP_S)
+			drbd_send_out_of_sync(peer_device, sector, size);
+	}
+	return 0;
 }
 
+/* caller must hold interval_lock */
 static struct drbd_request *
-find_request(struct drbd_device *device, struct rb_root *root, u64 id,
+find_request(struct drbd_device *device, enum drbd_interval_type type, u64 id,
 	     sector_t sector, bool missing_ok, const char *func)
 {
+	struct rb_root *root = type == INTERVAL_LOCAL_READ ? &device->read_requests : &device->requests;
 	struct drbd_request *req;
 
 	/* Request object according to our peer */
 	req = (struct drbd_request *)(unsigned long)id;
-	if (drbd_contains_interval(root, sector, &req->i) && req->i.local)
+	if (drbd_contains_interval(root, sector, &req->i) && req->i.type == type)
 		return req;
 	if (!missing_ok) {
 		drbd_err(device, "%s: failed to find request 0x%lx, sector %llus\n", func,
@@ -1941,9 +2647,9 @@ static int receive_DataReply(struct drbd_connection *connection, struct packet_i
 
 	sector = be64_to_cpu(p->sector);
 
-	spin_lock_irq(&device->resource->req_lock);
-	req = find_request(device, &device->read_requests, p->block_id, sector, false, __func__);
-	spin_unlock_irq(&device->resource->req_lock);
+	spin_lock_irq(&device->interval_lock);
+	req = find_request(device, INTERVAL_LOCAL_READ, p->block_id, sector, false, __func__);
+	spin_unlock_irq(&device->interval_lock);
 	if (unlikely(!req))
 		return -EIO;
 
@@ -1957,132 +2663,202 @@ static int receive_DataReply(struct drbd_connection *connection, struct packet_i
 	return err;
 }
 
-static int receive_RSDataReply(struct drbd_connection *connection, struct packet_info *pi)
+/**
+ * _drbd_send_ack() - Sends an ack packet
+ * @peer_device: DRBD peer device.
+ * @cmd:	Packet command code.
+ * @sector:	sector, needs to be in big endian byte order
+ * @blksize:	size in byte, needs to be in big endian byte order
+ * @block_id:	Id, big endian byte order
+ */
+static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+			  u64 sector, u32 blksize, u64 block_id)
 {
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	sector_t sector;
-	int err;
-	struct p_data *p = pi->data;
+	struct p_block_ack *p;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
+	if (peer_device->repl_state[NOW] < L_ESTABLISHED)
 		return -EIO;
-	device = peer_device->device;
-
-	sector = be64_to_cpu(p->sector);
-	D_ASSERT(device, p->block_id == ID_SYNCER);
-
-	if (get_ldev(device)) {
-		/* data is submitted to disk within recv_resync_read.
-		 * corresponding put_ldev done below on error,
-		 * or in drbd_peer_request_endio. */
-		err = recv_resync_read(peer_device, sector, pi);
-	} else {
-		if (drbd_ratelimit())
-			drbd_err(device, "Can not write resync data to local disk.\n");
-
-		err = drbd_drain_block(peer_device, pi->size);
 
-		drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
+	p = drbd_prepare_command(peer_device, sizeof(*p), CONTROL_STREAM);
+	if (!p)
+		return -EIO;
+	p->sector = sector;
+	p->block_id = block_id;
+	p->blksize = blksize;
+	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->packet_seq));
+
+	if (peer_device->connection->agreed_pro_version < 122) {
+		switch (cmd) {
+		case P_RS_NEG_ACK:
+			cmd = P_NEG_ACK;
+			p->block_id = ID_SYNCER;
+			break;
+		case P_WRITE_ACK_IN_SYNC:
+			cmd = P_RS_WRITE_ACK;
+			break;
+		case P_RS_WRITE_ACK:
+			p->block_id = ID_SYNCER;
+			break;
+		default:
+			break;
+		}
 	}
 
-	atomic_add(pi->size >> 9, &device->rs_sect_in);
-
-	return err;
+	return drbd_send_command(peer_device, cmd, CONTROL_STREAM);
 }
 
-static void restart_conflicting_writes(struct drbd_device *device,
-				       sector_t sector, int size)
+static int drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+		  struct drbd_peer_request_details *d)
 {
-	struct drbd_interval *i;
-	struct drbd_request *req;
+	return _drbd_send_ack(peer_device, cmd,
+			      cpu_to_be64(d->sector),
+			      cpu_to_be32(d->bi_size),
+			      d->block_id);
+}
 
-	drbd_for_each_overlap(i, &device->write_requests, sector, size) {
-		if (!i->local)
-			continue;
-		req = container_of(i, struct drbd_request, i);
-		if (req->rq_state & RQ_LOCAL_PENDING ||
-		    !(req->rq_state & RQ_POSTPONED))
-			continue;
-		/* as it is RQ_POSTPONED, this will cause it to
-		 * be queued on the retry workqueue. */
-		__req_mod(req, CONFLICT_RESOLVED, NULL, NULL);
-	}
+/* Send an ack packet wih a block ID that is already in big endian byte order. */
+int drbd_send_ack_be(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+		      sector_t sector, int size, u64 block_id)
+{
+	return _drbd_send_ack(peer_device, cmd, cpu_to_be64(sector), cpu_to_be32(size), block_id);
 }
 
-/*
- * e_end_block() is called in ack_sender context via drbd_finish_peer_reqs().
+/**
+ * drbd_send_ack() - Sends an ack packet
+ * @peer_device: DRBD peer device
+ * @cmd:	packet command code
+ * @peer_req:	peer request
  */
-static int e_end_block(struct drbd_work *w, int cancel)
+int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
+		  struct drbd_peer_request *peer_req)
 {
-	struct drbd_peer_request *peer_req =
-		container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	sector_t sector = peer_req->i.sector;
-	int err = 0, pcmd;
+	return _drbd_send_ack(peer_device, cmd,
+			      cpu_to_be64(peer_req->i.sector),
+			      cpu_to_be32(peer_req->i.size),
+			      peer_req->block_id);
+}
 
-	if (peer_req->flags & EE_SEND_WRITE_ACK) {
-		if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-			pcmd = (device->state.conn >= C_SYNC_SOURCE &&
-				device->state.conn <= C_PAUSED_SYNC_T &&
-				peer_req->flags & EE_MAY_SET_IN_SYNC) ?
-				P_RS_WRITE_ACK : P_WRITE_ACK;
-			err = drbd_send_ack(peer_device, pcmd, peer_req);
-			if (pcmd == P_RS_WRITE_ACK)
-				drbd_set_in_sync(peer_device, sector, peer_req->i.size);
-		} else {
-			err = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);
-			/* we expect it to be marked out of sync anyways...
-			 * maybe assert this?  */
-		}
-		dec_unacked(device);
-	}
+int drbd_send_ov_result(struct drbd_peer_device *peer_device, sector_t sector, int blksize,
+		u64 block_id, enum ov_result result)
+{
+	struct p_ov_result *p;
 
-	/* we delete from the conflict detection hash _after_ we sent out the
-	 * P_WRITE_ACK / P_NEG_ACK, to get the sequence number right.  */
-	if (peer_req->flags & EE_IN_INTERVAL_TREE) {
-		spin_lock_irq(&device->resource->req_lock);
-		D_ASSERT(device, !drbd_interval_empty(&peer_req->i));
-		drbd_remove_epoch_entry_interval(device, peer_req);
-		if (peer_req->flags & EE_RESTART_REQUESTS)
-			restart_conflicting_writes(device, sector, peer_req->i.size);
-		spin_unlock_irq(&device->resource->req_lock);
-	} else
-		D_ASSERT(device, drbd_interval_empty(&peer_req->i));
+	if (peer_device->connection->agreed_pro_version < 122)
+		/* Misuse the block_id field to signal if the blocks are is sync or not. */
+		return _drbd_send_ack(peer_device, P_OV_RESULT,
+				cpu_to_be64(sector),
+				cpu_to_be32(blksize),
+				cpu_to_be64(drbd_ov_result_to_block_id(result)));
+
+	if (peer_device->repl_state[NOW] < L_ESTABLISHED)
+		return -EIO;
 
-	drbd_may_finish_epoch(peer_device->connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0));
+	p = drbd_prepare_command(peer_device, sizeof(*p), CONTROL_STREAM);
+	if (!p)
+		return -EIO;
+	p->sector = cpu_to_be64(sector);
+	p->block_id = block_id;
+	p->blksize = cpu_to_be32(blksize);
+	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->packet_seq));
+	p->result = cpu_to_be32(result);
+	p->pad = 0;
 
-	return err;
+	return drbd_send_command(peer_device, P_OV_RESULT_ID, CONTROL_STREAM);
 }
 
-static int e_send_ack(struct drbd_work *w, enum drbd_packet ack)
+static int receive_RSDataReply(struct drbd_connection *connection, struct packet_info *pi)
 {
-	struct drbd_peer_request *peer_req =
-		container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_peer_request_details d;
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct drbd_peer_request *peer_req;
 	int err;
 
-	err = drbd_send_ack(peer_device, ack, peer_req);
-	dec_unacked(peer_device->device);
+	p_req_detail_from_pi(connection, &d, pi);
+	pi->data = NULL;
 
-	return err;
-}
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	device = peer_device->device;
 
-static int e_send_superseded(struct drbd_work *w, int unused)
-{
-	return e_send_ack(w, P_SUPERSEDED);
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_RESYNC_WRITE),
+			d.sector, d.bi_size, d.block_id);
+	if (!peer_req)
+		return -EIO;
+
+	if (get_ldev(device)) {
+		err = recv_resync_read(peer_device, peer_req, &d);
+		if (err)
+			put_ldev(device);
+	} else {
+		drbd_err_ratelimit(device, "Cannot write resync data to local disk.\n");
+
+		err = ignore_remaining_packet(connection, pi->size);
+
+		drbd_send_ack_dp(peer_device, P_RS_NEG_ACK, &d);
+
+		dec_rs_pending(peer_device);
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+
+	rs_sectors_came_in(peer_device, d.bi_size);
+
+	return err;
 }
 
-static int e_send_retry_write(struct drbd_work *w, int unused)
+/*
+ * e_end_block() is called in ack_sender context via drbd_finish_peer_reqs().
+ */
+static int e_end_block(struct drbd_work *w, int cancel)
 {
 	struct drbd_peer_request *peer_req =
 		container_of(w, struct drbd_peer_request, w);
-	struct drbd_connection *connection = peer_req->peer_device->connection;
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	sector_t sector = peer_req->i.sector;
+	struct drbd_epoch *epoch;
+	int err = 0, pcmd;
+
+	if (peer_req->flags & EE_IS_BARRIER) {
+		epoch = previous_epoch(connection, peer_req->epoch);
+		if (epoch)
+			drbd_may_finish_epoch(connection, epoch, EV_BARRIER_DONE + (cancel ? EV_CLEANUP : 0));
+	}
+
+	if (peer_req->flags & EE_SEND_WRITE_ACK) {
+		if (unlikely(peer_req->flags & EE_WAS_ERROR)) {
+			pcmd = P_NEG_ACK;
+			/* we expect it to be marked out of sync anyways...
+			 * maybe assert this?  */
+		} else if (peer_device->repl_state[NOW] >= L_SYNC_SOURCE &&
+			   peer_device->repl_state[NOW] <= L_PAUSED_SYNC_T &&
+			   peer_req->flags & EE_MAY_SET_IN_SYNC) {
+			pcmd = P_WRITE_ACK_IN_SYNC;
+			drbd_set_in_sync(peer_device, sector, peer_req->i.size);
+		} else
+			pcmd = P_WRITE_ACK;
+		err = drbd_send_ack(peer_device, pcmd, peer_req);
+		dec_unacked(peer_device);
+	}
+
+	drbd_remove_peer_req_interval(peer_req);
+
+	if (connection->agreed_pro_version < 110) {
+		drbd_al_complete_io(device, &peer_req->i);
+		drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0));
+		drbd_free_peer_req(peer_req);
+	} else {
+		drbd_free_page_chain(&connection->transport, &peer_req->page_chain);
+		drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0));
+		/* Do not use peer_req after this point. We may have sent the
+		 * corresponding barrier and received the corresponding peer ack. As a
+		 * result, peer_req may have been freed. */
+	}
 
-	return e_send_ack(w, connection->agreed_pro_version >= 100 ?
-			     P_RETRY_WRITE : P_SUPERSEDED);
+	return err;
 }
 
 static bool seq_greater(u32 a, u32 b)
@@ -2102,42 +2878,17 @@ static u32 seq_max(u32 a, u32 b)
 
 static void update_peer_seq(struct drbd_peer_device *peer_device, unsigned int peer_seq)
 {
-	struct drbd_device *device = peer_device->device;
 	unsigned int newest_peer_seq;
 
-	if (test_bit(RESOLVE_CONFLICTS, &peer_device->connection->flags)) {
-		spin_lock(&device->peer_seq_lock);
-		newest_peer_seq = seq_max(device->peer_seq, peer_seq);
-		device->peer_seq = newest_peer_seq;
-		spin_unlock(&device->peer_seq_lock);
-		/* wake up only if we actually changed device->peer_seq */
+	if (test_bit(RESOLVE_CONFLICTS, &peer_device->connection->transport.flags)) {
+		spin_lock_bh(&peer_device->peer_seq_lock);
+		newest_peer_seq = seq_max(peer_device->peer_seq, peer_seq);
+		peer_device->peer_seq = newest_peer_seq;
+		spin_unlock_bh(&peer_device->peer_seq_lock);
+		/* wake up only if we actually changed peer_device->peer_seq */
 		if (peer_seq == newest_peer_seq)
-			wake_up(&device->seq_wait);
-	}
-}
-
-static inline int overlaps(sector_t s1, int l1, sector_t s2, int l2)
-{
-	return !((s1 + (l1>>9) <= s2) || (s1 >= s2 + (l2>>9)));
-}
-
-/* maybe change sync_ee into interval trees as well? */
-static bool overlapping_resync_write(struct drbd_device *device, struct drbd_peer_request *peer_req)
-{
-	struct drbd_peer_request *rs_req;
-	bool rv = false;
-
-	spin_lock_irq(&device->resource->req_lock);
-	list_for_each_entry(rs_req, &device->sync_ee, w.list) {
-		if (overlaps(peer_req->i.sector, peer_req->i.size,
-			     rs_req->i.sector, rs_req->i.size)) {
-			rv = true;
-			break;
-		}
+			wake_up(&peer_device->device->seq_wait);
 	}
-	spin_unlock_irq(&device->resource->req_lock);
-
-	return rv;
 }
 
 /* Called from receive_Data.
@@ -2149,9 +2900,9 @@ static bool overlapping_resync_write(struct drbd_device *device, struct drbd_pee
  *
  * Note: we don't care for Ack packets overtaking P_DATA packets.
  *
- * In case packet_seq is larger than device->peer_seq number, there are
+ * In case packet_seq is larger than peer_device->peer_seq number, there are
  * outstanding packets on the msock. We wait for them to arrive.
- * In case we are the logically next packet, we update device->peer_seq
+ * In case we are the logically next packet, we update peer_device->peer_seq
  * ourselves. Correctly handles 32bit wrap around.
  *
  * Assume we have a 10 GBit connection, that is about 1<<30 byte per second,
@@ -2163,18 +2914,18 @@ static bool overlapping_resync_write(struct drbd_device *device, struct drbd_pee
  * -ERESTARTSYS if we were interrupted (by disconnect signal). */
 static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, const u32 peer_seq)
 {
-	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
 	DEFINE_WAIT(wait);
 	long timeout;
 	int ret = 0, tp;
 
-	if (!test_bit(RESOLVE_CONFLICTS, &peer_device->connection->flags))
+	if (!test_bit(RESOLVE_CONFLICTS, &connection->transport.flags))
 		return 0;
 
-	spin_lock(&device->peer_seq_lock);
+	spin_lock_bh(&peer_device->peer_seq_lock);
 	for (;;) {
-		if (!seq_greater(peer_seq - 1, device->peer_seq)) {
-			device->peer_seq = seq_max(device->peer_seq, peer_seq);
+		if (!seq_greater(peer_seq - 1, peer_device->peer_seq)) {
+			peer_device->peer_seq = seq_max(peer_device->peer_seq, peer_seq);
 			break;
 		}
 
@@ -2184,28 +2935,28 @@ static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, co
 		}
 
 		rcu_read_lock();
-		tp = rcu_dereference(peer_device->connection->net_conf)->two_primaries;
+		tp = rcu_dereference(connection->transport.net_conf)->two_primaries;
 		rcu_read_unlock();
 
 		if (!tp)
 			break;
 
 		/* Only need to wait if two_primaries is enabled */
-		prepare_to_wait(&device->seq_wait, &wait, TASK_INTERRUPTIBLE);
-		spin_unlock(&device->peer_seq_lock);
+		prepare_to_wait(&peer_device->device->seq_wait, &wait, TASK_INTERRUPTIBLE);
+		spin_unlock_bh(&peer_device->peer_seq_lock);
 		rcu_read_lock();
-		timeout = rcu_dereference(peer_device->connection->net_conf)->ping_timeo*HZ/10;
+		timeout = rcu_dereference(connection->transport.net_conf)->ping_timeo*HZ/10;
 		rcu_read_unlock();
 		timeout = schedule_timeout(timeout);
-		spin_lock(&device->peer_seq_lock);
+		spin_lock_bh(&peer_device->peer_seq_lock);
 		if (!timeout) {
 			ret = -ETIMEDOUT;
-			drbd_err(device, "Timed out waiting for missing ack packets; disconnecting\n");
+			drbd_err(peer_device, "Timed out waiting for missing ack packets; disconnecting\n");
 			break;
 		}
 	}
-	spin_unlock(&device->peer_seq_lock);
-	finish_wait(&device->seq_wait, &wait);
+	spin_unlock_bh(&peer_device->peer_seq_lock);
+	finish_wait(&peer_device->device->seq_wait, &wait);
 	return ret;
 }
 
@@ -2215,182 +2966,268 @@ static enum req_op wire_flags_to_bio_op(u32 dpf)
 		return REQ_OP_WRITE_ZEROES;
 	if (dpf & DP_DISCARD)
 		return REQ_OP_DISCARD;
-	else
-		return REQ_OP_WRITE;
+	return REQ_OP_WRITE;
 }
 
 /* see also bio_flags_to_wire() */
 static blk_opf_t wire_flags_to_bio(struct drbd_connection *connection, u32 dpf)
 {
-	return wire_flags_to_bio_op(dpf) |
-		(dpf & DP_RW_SYNC ? REQ_SYNC : 0) |
-		(dpf & DP_FUA ? REQ_FUA : 0) |
-		(dpf & DP_FLUSH ? REQ_PREFLUSH : 0);
+	blk_opf_t opf = wire_flags_to_bio_op(dpf) |
+		(dpf & DP_RW_SYNC ? REQ_SYNC : 0);
+
+	/* we used to communicate one bit only in older DRBD */
+	if (connection->agreed_pro_version >= 95)
+		opf |= (dpf & DP_FUA ? REQ_FUA : 0) |
+			    (dpf & DP_FLUSH ? REQ_PREFLUSH : 0);
+
+	return opf;
 }
 
-static void fail_postponed_requests(struct drbd_device *device, sector_t sector,
-				    unsigned int size)
+static void drbd_wait_for_activity_log_extents(struct drbd_peer_request *peer_req)
 {
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	struct drbd_interval *i;
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct lru_cache *al;
+	int nr_al_extents;
+	int nr, used, ecnt;
 
-    repeat:
-	drbd_for_each_overlap(i, &device->write_requests, sector, size) {
-		struct drbd_request *req;
-		struct bio_and_error m;
+	/* Let the activity log know we are about to use it.
+	 * See also drbd_request_prepare() for the "request" entry point. */
+	nr_al_extents = interval_to_al_extents(&peer_req->i);
+	ecnt = atomic_add_return(nr_al_extents, &device->wait_for_actlog_ecnt);
 
-		if (!i->local)
-			continue;
-		req = container_of(i, struct drbd_request, i);
-		if (!(req->rq_state & RQ_POSTPONED))
-			continue;
-		req->rq_state &= ~RQ_POSTPONED;
-		__req_mod(req, NEG_ACKED, peer_device, &m);
-		spin_unlock_irq(&device->resource->req_lock);
-		if (m.bio)
-			complete_master_bio(device, &m);
-		spin_lock_irq(&device->resource->req_lock);
-		goto repeat;
+	spin_lock_irq(&device->al_lock);
+	al = device->act_log;
+	nr = al->nr_elements;
+	used = al->used;
+	spin_unlock_irq(&device->al_lock);
+
+	/* note: due to the slight delay between being accounted in "used" after
+	 * being committed to the activity log with drbd_al_begin_io_commit(),
+	 * and being subtracted from "wait_for_actlog_ecnt" in __drbd_submit_peer_request(),
+	 * this can err, but only on the conservative side (overestimating ecnt).
+	 * ecnt also includes any requests which are held due to conflicts,
+	 * conservatively overestimating the number of activity log extents
+	 * required. */
+	if (ecnt > nr - used) {
+		conn_wait_active_ee_empty_or_disconnect(connection);
+		drbd_flush_after_epoch(connection, NULL);
+		conn_wait_done_ee_empty_or_disconnect(connection);
+
+		/* would this peer even understand me? */
+		if (connection->agreed_pro_version >= 114)
+			drbd_send_confirm_stable(peer_req);
 	}
 }
 
-static int handle_write_conflicts(struct drbd_device *device,
-				  struct drbd_peer_request *peer_req)
+static int drbd_peer_write_conflicts(struct drbd_peer_request *peer_req)
 {
-	struct drbd_connection *connection = peer_req->peer_device->connection;
-	bool resolve_conflicts = test_bit(RESOLVE_CONFLICTS, &connection->flags);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
 	sector_t sector = peer_req->i.sector;
 	const unsigned int size = peer_req->i.size;
 	struct drbd_interval *i;
-	bool equal;
-	int err;
 
-	/*
-	 * Inserting the peer request into the write_requests tree will prevent
-	 * new conflicting local requests from being added.
-	 */
-	drbd_insert_interval(&device->write_requests, &peer_req->i);
+	i = drbd_find_conflict(device, &peer_req->i, CONFLICT_FLAG_APPLICATION_ONLY);
 
-    repeat:
-	drbd_for_each_overlap(i, &device->write_requests, sector, size) {
-		if (i == &peer_req->i)
-			continue;
-		if (i->completed)
+	if (i) {
+		drbd_alert(device, "Concurrent writes detected: "
+				"local=%llus +%u, remote=%llus +%u\n",
+				(unsigned long long) i->sector, i->size,
+				(unsigned long long) sector, size);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static void drbd_queue_peer_request(struct drbd_device *device, struct drbd_peer_request *peer_req)
+{
+	atomic_inc(&device->wait_for_actlog);
+	spin_lock(&device->submit.lock);
+	list_add_tail(&peer_req->w.list, &device->submit.peer_writes);
+	spin_unlock(&device->submit.lock);
+	queue_work(device->submit.wq, &device->submit.worker);
+	/* do_submit() may sleep internally on al_wait, too */
+	wake_up(&device->al_wait);
+}
+
+static struct drbd_peer_request *find_released_peer_request(struct drbd_resource *resource, unsigned int node_id, u64 dagtag)
+{
+	struct drbd_connection *connection;
+	struct drbd_peer_request *released_peer_req = NULL;
+
+	read_lock_irq(&resource->state_rwlock);
+	for_each_connection(connection, resource) {
+		struct drbd_peer_request *peer_req;
+
+		/* Skip if we are not connected. If we are in the process of
+		 * disconnecting, the requests on dagtag_wait_ee will be
+		 * cleared up. Do not interfere with that process. */
+		if (connection->cstate[NOW] < C_CONNECTED)
 			continue;
 
-		if (!i->local) {
-			/*
-			 * Our peer has sent a conflicting remote request; this
-			 * should not happen in a two-node setup.  Wait for the
-			 * earlier peer request to complete.
-			 */
-			err = drbd_wait_misc(device, i);
-			if (err)
-				goto out;
-			goto repeat;
+		spin_lock(&connection->peer_reqs_lock);
+		list_for_each_entry(peer_req, &connection->dagtag_wait_ee, w.list) {
+			if (!peer_req->depend_dagtag ||
+					peer_req->depend_dagtag_node_id != node_id ||
+					peer_req->depend_dagtag > dagtag)
+				continue;
+
+			dynamic_drbd_dbg(peer_req->peer_device, "%s at %llus+%u: Wait for dagtag %llus from peer %u complete\n",
+					drbd_interval_type_str(&peer_req->i),
+					(unsigned long long) peer_req->i.sector, peer_req->i.size,
+					(unsigned long long) peer_req->depend_dagtag,
+					peer_req->depend_dagtag_node_id);
+
+			list_del(&peer_req->w.list);
+			released_peer_req = peer_req;
+			break;
 		}
+		spin_unlock(&connection->peer_reqs_lock);
 
-		equal = i->sector == sector && i->size == size;
-		if (resolve_conflicts) {
-			/*
-			 * If the peer request is fully contained within the
-			 * overlapping request, it can be considered overwritten
-			 * and thus superseded; otherwise, it will be retried
-			 * once all overlapping requests have completed.
-			 */
-			bool superseded = i->sector <= sector && i->sector +
-				       (i->size >> 9) >= sector + (size >> 9);
-
-			if (!equal)
-				drbd_alert(device, "Concurrent writes detected: "
-					       "local=%llus +%u, remote=%llus +%u, "
-					       "assuming %s came first\n",
-					  (unsigned long long)i->sector, i->size,
-					  (unsigned long long)sector, size,
-					  superseded ? "local" : "remote");
-
-			peer_req->w.cb = superseded ? e_send_superseded :
-						   e_send_retry_write;
-			list_add_tail(&peer_req->w.list, &device->done_ee);
-			/* put is in drbd_send_acks_wf() */
-			kref_get(&device->kref);
-			if (!queue_work(connection->ack_sender,
-					&peer_req->peer_device->send_acks_work))
-				kref_put(&device->kref, drbd_destroy_device);
+		if (released_peer_req)
+			break;
+	}
+	read_unlock_irq(&resource->state_rwlock);
 
-			err = -ENOENT;
-			goto out;
-		} else {
-			struct drbd_request *req =
-				container_of(i, struct drbd_request, i);
+	return released_peer_req;
+}
 
-			if (!equal)
-				drbd_alert(device, "Concurrent writes detected: "
-					       "local=%llus +%u, remote=%llus +%u\n",
-					  (unsigned long long)i->sector, i->size,
-					  (unsigned long long)sector, size);
+static void release_dagtag_wait(struct drbd_resource *resource, unsigned int node_id, u64 dagtag)
+{
+	struct drbd_peer_request *peer_req;
 
-			if (req->rq_state & RQ_LOCAL_PENDING ||
-			    !(req->rq_state & RQ_POSTPONED)) {
-				/*
-				 * Wait for the node with the discard flag to
-				 * decide if this request has been superseded
-				 * or needs to be retried.
-				 * Requests that have been superseded will
-				 * disappear from the write_requests tree.
-				 *
-				 * In addition, wait for the conflicting
-				 * request to finish locally before submitting
-				 * the conflicting peer request.
-				 */
-				err = drbd_wait_misc(device, &req->i);
-				if (err) {
-					_conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
-					fail_postponed_requests(device, sector, size);
-					goto out;
-				}
-				goto repeat;
-			}
-			/*
-			 * Remember to restart the conflicting requests after
-			 * the new peer request has completed.
-			 */
-			peer_req->flags |= EE_RESTART_REQUESTS;
-		}
+	while ((peer_req = find_released_peer_request(resource, node_id, dagtag))) {
+		atomic_inc(&peer_req->peer_device->connection->backing_ee_cnt);
+		drbd_conflict_submit_peer_read(peer_req);
 	}
-	err = 0;
+}
+
+static void set_connection_dagtag(struct drbd_connection *connection, u64 dagtag)
+{
+	atomic64_set(&connection->last_dagtag_sector, dagtag);
+	set_bit(RECEIVED_DAGTAG, &connection->flags);
+
+	release_dagtag_wait(connection->resource, connection->peer_node_id, dagtag);
+}
+
+static void submit_peer_request_activity_log(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	int err;
+	int nr_al_extents = interval_to_al_extents(&peer_req->i);
+
+	if (nr_al_extents != 1 || !drbd_al_begin_io_fastpath(device, &peer_req->i)) {
+		drbd_queue_peer_request(device, peer_req);
+		return;
+	}
+
+	peer_req->flags |= EE_IN_ACTLOG;
+	atomic_sub(nr_al_extents, &device->wait_for_actlog_ecnt);
 
-    out:
+	err = drbd_submit_peer_request(peer_req);
 	if (err)
-		drbd_remove_epoch_entry_interval(device, peer_req);
-	return err;
+		drbd_cleanup_after_failed_submit_peer_write(peer_req);
 }
 
-/* mirrored write */
+void drbd_conflict_submit_peer_write(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	bool conflict = false;
+
+	spin_lock_irq(&device->interval_lock);
+	clear_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &peer_req->i.flags);
+	conflict = drbd_find_conflict(device, &peer_req->i, 0);
+	if (!conflict)
+		set_bit(INTERVAL_SUBMITTED, &peer_req->i.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	if (!conflict)
+		submit_peer_request_activity_log(peer_req);
+}
+
+/* mirrored write
+ *
+ * Request handling flow:
+ *
+ *                      conflict
+ * receive_Data -----------------------+
+ *       |                             |
+ *       |                            ...
+ *       |                             |
+ *       |                             v
+ *       |                   drbd_do_submit_conflict
+ *       |                             |
+ *       |                             v
+ *       +------------------ drbd_conflict_submit_peer_write
+ *       |
+ *       v                         wait for AL
+ * submit_peer_request_activity_log --------> drbd_queue_peer_request
+ *       |                                         |
+ *       |                                        ...
+ *       |                                         |
+ *       |                                         v    AL extent active
+ *       |                                    do_submit ----------------+
+ *       |                                         |                    |
+ *       |                                         v                    v
+ *       |                              send_and_submit_pending   submit_fast_path
+ *       |                                         |                    |
+ *       v                                         v                    |
+ * drbd_submit_peer_request <-------- __drbd_submit_peer_request <------+
+ *       |
+ *      ... backing device
+ *       |
+ *       v
+ * drbd_peer_request_endio
+ *       |
+ *       v
+ * drbd_endio_write_sec_final
+ *       |
+ *      ... done_ee
+ *       |
+ *       v
+ * drbd_finish_peer_reqs
+ *       |
+ *       v
+ * e_end_block
+ *       |
+ *      ... via peer
+ *       |
+ *       v
+ * got_peer_ack
+ */
 static int receive_Data(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct drbd_peer_device *peer_device;
 	struct drbd_device *device;
 	struct net_conf *nc;
-	sector_t sector;
 	struct drbd_peer_request *peer_req;
-	struct p_data *p = pi->data;
-	u32 peer_seq = be32_to_cpu(p->seq_num);
-	u32 dp_flags;
+	struct drbd_peer_request_details d;
 	int err, tp;
+	bool conflict = false;
 
 	peer_device = conn_peer_device(connection, pi->vnr);
 	if (!peer_device)
 		return -EIO;
 	device = peer_device->device;
 
+	if (pi->cmd == P_TRIM)
+		D_ASSERT(peer_device, pi->size == 0);
+
+	p_req_detail_from_pi(connection, &d, pi);
+	pi->data = NULL;
+
 	if (!get_ldev(device)) {
 		int err2;
 
-		err = wait_for_and_update_peer_seq(peer_device, peer_seq);
-		drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
+		err = wait_for_and_update_peer_seq(peer_device, d.peer_seq);
+		drbd_send_ack_dp(peer_device, P_NEG_ACK, &d);
 		atomic_inc(&connection->current_epoch->epoch_size);
-		err2 = drbd_drain_block(peer_device, pi->size);
+		err2 = ignore_remaining_packet(connection, pi->size);
 		if (!err)
 			err = err2;
 		return err;
@@ -2402,71 +3239,118 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
 	 * end of this function.
 	 */
 
-	sector = be64_to_cpu(p->sector);
-	peer_req = read_in_block(peer_device, p->block_id, sector, pi);
+	peer_req = drbd_alloc_peer_req(peer_device, GFP_TRY);
 	if (!peer_req) {
 		put_ldev(device);
 		return -EIO;
 	}
+	peer_req->i.size = d.bi_size; /* storage size */
+	peer_req->i.sector = d.sector;
+	peer_req->i.type = INTERVAL_PEER_WRITE;
+
+	err = read_in_block(peer_req, &d);
+	if (err) {
+		drbd_free_peer_req(peer_req);
+		put_ldev(device);
+		return err;
+	}
+
+	if (pi->cmd == P_TRIM)
+		peer_req->flags |= EE_TRIM;
+	else if (pi->cmd == P_ZEROES)
+		peer_req->flags |= EE_ZEROOUT;
 
 	peer_req->w.cb = e_end_block;
 	peer_req->submit_jif = jiffies;
-	peer_req->flags |= EE_APPLICATION;
 
-	dp_flags = be32_to_cpu(p->dp_flags);
-	peer_req->opf = wire_flags_to_bio(connection, dp_flags);
+	peer_req->opf = wire_flags_to_bio(connection, d.dp_flags);
 	if (pi->cmd == P_TRIM) {
 		D_ASSERT(peer_device, peer_req->i.size > 0);
+		D_ASSERT(peer_device, d.dp_flags & DP_DISCARD);
 		D_ASSERT(peer_device, peer_req_op(peer_req) == REQ_OP_DISCARD);
-		D_ASSERT(peer_device, peer_req->pages == NULL);
+		D_ASSERT(peer_device, peer_req->page_chain.head == NULL);
+		D_ASSERT(peer_device, peer_req->page_chain.nr_pages == 0);
 		/* need to play safe: an older DRBD sender
 		 * may mean zero-out while sending P_TRIM. */
 		if (0 == (connection->agreed_features & DRBD_FF_WZEROES))
 			peer_req->flags |= EE_ZEROOUT;
 	} else if (pi->cmd == P_ZEROES) {
 		D_ASSERT(peer_device, peer_req->i.size > 0);
+		D_ASSERT(peer_device, d.dp_flags & DP_ZEROES);
 		D_ASSERT(peer_device, peer_req_op(peer_req) == REQ_OP_WRITE_ZEROES);
-		D_ASSERT(peer_device, peer_req->pages == NULL);
+		D_ASSERT(peer_device, peer_req->page_chain.head == NULL);
+		D_ASSERT(peer_device, peer_req->page_chain.nr_pages == 0);
 		/* Do (not) pass down BLKDEV_ZERO_NOUNMAP? */
-		if (dp_flags & DP_DISCARD)
+		if (d.dp_flags & DP_DISCARD)
 			peer_req->flags |= EE_TRIM;
-	} else if (peer_req->pages == NULL) {
+	} else if (peer_req->page_chain.head == NULL) {
+		/* Actually, this must not happen anymore,
+		 * "empty" flushes are mapped to P_BARRIER,
+		 * and should never end up here.
+		 * Compat with old DRBD? */
 		D_ASSERT(device, peer_req->i.size == 0);
-		D_ASSERT(device, dp_flags & DP_FLUSH);
+		D_ASSERT(device, d.dp_flags & DP_FLUSH);
+	} else {
+		D_ASSERT(peer_device, peer_req->i.size > 0);
+		D_ASSERT(peer_device, peer_req_op(peer_req) == REQ_OP_WRITE);
 	}
 
-	if (dp_flags & DP_MAY_SET_IN_SYNC)
+	if (d.dp_flags & DP_MAY_SET_IN_SYNC)
 		peer_req->flags |= EE_MAY_SET_IN_SYNC;
 
 	spin_lock(&connection->epoch_lock);
 	peer_req->epoch = connection->current_epoch;
 	atomic_inc(&peer_req->epoch->epoch_size);
 	atomic_inc(&peer_req->epoch->active);
+	if (peer_req->epoch->oldest_unconfirmed_peer_req == NULL)
+		peer_req->epoch->oldest_unconfirmed_peer_req = peer_req;
+
+	if (connection->resource->write_ordering == WO_BIO_BARRIER &&
+	    atomic_read(&peer_req->epoch->epoch_size) == 1) {
+		struct drbd_epoch *epoch;
+		/* Issue a barrier if we start a new epoch, and the previous epoch
+		   was not a epoch containing a single request which already was
+		   a Barrier. */
+		epoch = list_entry(peer_req->epoch->list.prev, struct drbd_epoch, list);
+		if (epoch == peer_req->epoch) {
+			set_bit(DE_CONTAINS_A_BARRIER, &peer_req->epoch->flags);
+			peer_req->opf |= REQ_PREFLUSH | REQ_FUA;
+			peer_req->flags |= EE_IS_BARRIER;
+		} else {
+			if (atomic_read(&epoch->epoch_size) > 1 ||
+			    !test_bit(DE_CONTAINS_A_BARRIER, &epoch->flags)) {
+				set_bit(DE_BARRIER_IN_NEXT_EPOCH_ISSUED, &epoch->flags);
+				set_bit(DE_CONTAINS_A_BARRIER, &peer_req->epoch->flags);
+				peer_req->opf |= REQ_PREFLUSH | REQ_FUA;
+				peer_req->flags |= EE_IS_BARRIER;
+			}
+		}
+	}
 	spin_unlock(&connection->epoch_lock);
 
 	rcu_read_lock();
-	nc = rcu_dereference(peer_device->connection->net_conf);
+	nc = rcu_dereference(connection->transport.net_conf);
 	tp = nc->two_primaries;
-	if (peer_device->connection->agreed_pro_version < 100) {
+	if (connection->agreed_pro_version < 100) {
 		switch (nc->wire_protocol) {
 		case DRBD_PROT_C:
-			dp_flags |= DP_SEND_WRITE_ACK;
+			d.dp_flags |= DP_SEND_WRITE_ACK;
 			break;
 		case DRBD_PROT_B:
-			dp_flags |= DP_SEND_RECEIVE_ACK;
+			d.dp_flags |= DP_SEND_RECEIVE_ACK;
 			break;
 		}
 	}
 	rcu_read_unlock();
 
-	if (dp_flags & DP_SEND_WRITE_ACK) {
+	if (d.dp_flags & DP_SEND_WRITE_ACK) {
 		peer_req->flags |= EE_SEND_WRITE_ACK;
-		inc_unacked(device);
+		inc_unacked(peer_device);
 		/* corresponding dec_unacked() in e_end_block()
 		 * respective _drbd_clear_done_ee */
 	}
 
-	if (dp_flags & DP_SEND_RECEIVE_ACK) {
+	if (d.dp_flags & DP_SEND_RECEIVE_ACK) {
 		/* I really don't like it that the receiver thread
 		 * sends on the msock, but anyways */
 		drbd_send_ack(peer_device, P_RECV_ACK, peer_req);
@@ -2474,66 +3358,136 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
 
 	if (tp) {
 		/* two primaries implies protocol C */
-		D_ASSERT(device, dp_flags & DP_SEND_WRITE_ACK);
-		peer_req->flags |= EE_IN_INTERVAL_TREE;
-		err = wait_for_and_update_peer_seq(peer_device, peer_seq);
+		D_ASSERT(device, d.dp_flags & DP_SEND_WRITE_ACK);
+		err = wait_for_and_update_peer_seq(peer_device, d.peer_seq);
 		if (err)
-			goto out_interrupted;
-		spin_lock_irq(&device->resource->req_lock);
-		err = handle_write_conflicts(device, peer_req);
-		if (err) {
-			spin_unlock_irq(&device->resource->req_lock);
-			if (err == -ENOENT) {
-				put_ldev(device);
-				return 0;
-			}
-			goto out_interrupted;
-		}
+			goto out;
 	} else {
-		update_peer_seq(peer_device, peer_seq);
-		spin_lock_irq(&device->resource->req_lock);
-	}
-	/* TRIM and is processed synchronously,
-	 * we wait for all pending requests, respectively wait for
-	 * active_ee to become empty in drbd_submit_peer_request();
-	 * better not add ourselves here. */
-	if ((peer_req->flags & (EE_TRIM | EE_ZEROOUT)) == 0)
-		list_add_tail(&peer_req->w.list, &device->active_ee);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	if (device->state.conn == C_SYNC_TARGET)
-		wait_event(device->ee_wait, !overlapping_resync_write(device, peer_req));
-
-	if (device->state.pdsk < D_INCONSISTENT) {
-		/* In case we have the only disk of the cluster, */
-		drbd_set_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
-		peer_req->flags &= ~EE_MAY_SET_IN_SYNC;
-		drbd_al_begin_io(device, &peer_req->i);
-		peer_req->flags |= EE_CALL_AL_COMPLETE_IO;
+		update_peer_seq(peer_device, d.peer_seq);
 	}
 
-	err = drbd_submit_peer_request(peer_req);
-	if (!err)
-		return 0;
+	peer_req->dagtag_sector = atomic64_read(&connection->last_dagtag_sector) + (peer_req->i.size >> 9);
 
-	/* don't care for the reason here */
-	drbd_err(device, "submit failed, triggering re-connect\n");
-	spin_lock_irq(&device->resource->req_lock);
-	list_del(&peer_req->w.list);
-	drbd_remove_epoch_entry_interval(device, peer_req);
-	spin_unlock_irq(&device->resource->req_lock);
-	if (peer_req->flags & EE_CALL_AL_COMPLETE_IO) {
-		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
-		drbd_al_complete_io(device, &peer_req->i);
+	drbd_wait_for_activity_log_extents(peer_req);
+
+	atomic_inc(&connection->active_ee_cnt);
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_add_tail(&peer_req->recv_order, &connection->peer_requests);
+	peer_req->flags |= EE_ON_RECV_ORDER;
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	/* Note: this now may or may not be "hot" in the activity log.
+	 * Still, it is the best time to record that we need to set the
+	 * out-of-sync bit, if we delay that until drbd_submit_peer_request(),
+	 * we may introduce a race with some re-attach on the peer.
+	 * Unless we want to guarantee that we drain all in-flight IO
+	 * whenever we receive a state change. Which I'm not sure about.
+	 * Use the EE_SET_OUT_OF_SYNC flag, to be acted on just before
+	 * the actual submit, when we can be sure it is "hot".
+	 */
+	if (peer_device->disk_state[NOW] < D_INCONSISTENT) {
+		peer_req->flags &= ~EE_MAY_SET_IN_SYNC;
+		peer_req->flags |= EE_SET_OUT_OF_SYNC;
+	}
+
+	spin_lock_irq(&device->interval_lock);
+	if (tp) {
+		err = drbd_peer_write_conflicts(peer_req);
+		if (err) {
+			spin_unlock_irq(&device->interval_lock);
+			goto out_del_list;
+		}
 	}
+	conflict = drbd_find_conflict(device, &peer_req->i, 0);
+	drbd_insert_interval(&device->requests, &peer_req->i);
+	if (!conflict)
+		set_bit(INTERVAL_SUBMITTED, &peer_req->i.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	/* The connection dagtag may only be set after inserting the interval
+	 * into the tree, so that requests that were waiting for the dagtag
+	 * enter the interval tree after the request with the dagtag itself. */
+	set_connection_dagtag(connection, peer_req->dagtag_sector);
+
+	if (!conflict)
+		submit_peer_request_activity_log(peer_req);
+	return 0;
+
+out_del_list:
+	spin_lock_irq(&connection->peer_reqs_lock);
+	peer_req->flags &= ~EE_ON_RECV_ORDER;
+	list_del(&peer_req->recv_order);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	atomic_dec(&connection->active_ee_cnt);
+	atomic_sub(interval_to_al_extents(&peer_req->i), &device->wait_for_actlog_ecnt);
 
-out_interrupted:
-	drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP);
+out:
+	if (peer_req->flags & EE_SEND_WRITE_ACK)
+		dec_unacked(peer_device);
+	drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + EV_CLEANUP);
 	put_ldev(device);
-	drbd_free_peer_req(device, peer_req);
+	drbd_free_peer_req(peer_req);
 	return err;
 }
 
+/*
+ * To be called when drbd_submit_peer_request() fails for a peer write request.
+ */
+void drbd_cleanup_after_failed_submit_peer_write(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+
+	drbd_err_ratelimit(peer_device, "submit failed, triggering re-connect\n");
+
+	if (peer_req->flags & EE_IN_ACTLOG)
+		drbd_al_complete_io(device, &peer_req->i);
+
+	drbd_remove_peer_req_interval(peer_req);
+
+	drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + EV_CLEANUP);
+	put_ldev(device);
+	drbd_free_peer_req(peer_req);
+	change_cstate(connection, C_PROTOCOL_ERROR, CS_HARD);
+}
+
+/* Possibly "cancel" and forget about all peer_requests that had still been
+ * waiting for the activity log (wfa) when the connection to their peer failed,
+ * and pretend we never received them.
+ */
+void drbd_cleanup_peer_requests_wfa(struct drbd_device *device, struct list_head *cleanup)
+{
+	struct drbd_connection *connection;
+	struct drbd_peer_request *peer_req, *pr_tmp;
+
+	write_lock_irq(&device->resource->state_rwlock);
+	list_for_each_entry(peer_req, cleanup, w.list) {
+		atomic_dec(&peer_req->peer_device->connection->active_ee_cnt);
+		drbd_remove_peer_req_interval(peer_req);
+	}
+	write_unlock_irq(&device->resource->state_rwlock);
+
+	list_for_each_entry_safe(peer_req, pr_tmp, cleanup, w.list) {
+		atomic_sub(interval_to_al_extents(&peer_req->i), &device->wait_for_actlog_ecnt);
+		atomic_dec(&device->wait_for_actlog);
+		if (peer_req->flags & EE_SEND_WRITE_ACK)
+			dec_unacked(peer_req->peer_device);
+		list_del_init(&peer_req->w.list);
+		drbd_may_finish_epoch(peer_req->peer_device->connection, peer_req->epoch, EV_PUT | EV_CLEANUP);
+		drbd_free_peer_req(peer_req);
+		put_ldev(device);
+	}
+	/*
+	 * We changed (likely: cleared) active_ee_cnt for "at least one" connection.
+	 * We should wake potential waiters, just in case.
+	 */
+	for_each_connection(connection, device->resource)
+		wake_up(&connection->ee_wait);
+}
+
 /* We may throttle resync, if the lower device seems to be busy,
  * and current sync rate is above c_min_rate.
  *
@@ -2545,69 +3499,45 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
  * The current sync rate used here uses only the most recent two step marks,
  * to have a short time average so we can react faster.
  */
-bool drbd_rs_should_slow_down(struct drbd_peer_device *peer_device, sector_t sector,
-		bool throttle_if_app_is_waiting)
+bool drbd_rs_c_min_rate_throttle(struct drbd_peer_device *peer_device)
 {
 	struct drbd_device *device = peer_device->device;
-	struct lc_element *tmp;
-	bool throttle = drbd_rs_c_min_rate_throttle(device);
+	struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
+	unsigned long db, dt, dbdt;
+	unsigned int c_min_rate;
+	int curr_events;
 
-	if (!throttle || throttle_if_app_is_waiting)
-		return throttle;
-
-	spin_lock_irq(&device->al_lock);
-	tmp = lc_find(device->resync, BM_SECT_TO_EXT(sector));
-	if (tmp) {
-		struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);
-		if (test_bit(BME_PRIORITY, &bm_ext->flags))
-			throttle = false;
-		/* Do not slow down if app IO is already waiting for this extent,
-		 * and our progress is necessary for application IO to complete. */
-	}
-	spin_unlock_irq(&device->al_lock);
-
-	return throttle;
-}
-
-bool drbd_rs_c_min_rate_throttle(struct drbd_device *device)
-{
-	struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
-	unsigned long db, dt, dbdt;
-	unsigned int c_min_rate;
-	int curr_events;
-
-	rcu_read_lock();
-	c_min_rate = rcu_dereference(device->ldev->disk_conf)->c_min_rate;
-	rcu_read_unlock();
+	rcu_read_lock();
+	c_min_rate = rcu_dereference(peer_device->conf)->c_min_rate;
+	rcu_read_unlock();
 
 	/* feature disabled? */
 	if (c_min_rate == 0)
 		return false;
 
-	curr_events = (int)part_stat_read_accum(disk->part0, sectors) -
-			atomic_read(&device->rs_sect_ev);
+	curr_events = (int)part_stat_read_accum(disk->part0, sectors)
+		- atomic_read(&device->rs_sect_ev);
 
-	if (atomic_read(&device->ap_actlog_cnt)
-	    || curr_events - device->rs_last_events > 64) {
+	if (atomic_read(&device->ap_actlog_cnt) || curr_events - peer_device->rs_last_events > 64) {
 		unsigned long rs_left;
 		int i;
 
-		device->rs_last_events = curr_events;
+		peer_device->rs_last_events = curr_events;
 
 		/* sync speed average over the last 2*DRBD_SYNC_MARK_STEP,
 		 * approx. */
-		i = (device->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
+		i = (peer_device->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
 
-		if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T)
-			rs_left = device->ov_left;
+		if (peer_device->repl_state[NOW] == L_VERIFY_S || peer_device->repl_state[NOW] == L_VERIFY_T)
+			rs_left = atomic64_read(&peer_device->ov_left);
 		else
-			rs_left = drbd_bm_total_weight(device) - device->rs_failed;
+			rs_left = drbd_bm_total_weight(peer_device) - peer_device->rs_failed;
 
-		dt = ((long)jiffies - (long)device->rs_mark_time[i]) / HZ;
+		dt = ((long)jiffies - (long)peer_device->rs_mark_time[i]) / HZ;
 		if (!dt)
 			dt++;
-		db = device->rs_mark_left[i] - rs_left;
-		dbdt = Bit2KB(db/dt);
+		db = peer_device->rs_mark_left[i] - rs_left;
+		dbdt = device_bit_to_kb(device, db/dt);
 
 		if (dbdt > c_min_rate)
 			return true;
@@ -2615,16 +3545,199 @@ bool drbd_rs_c_min_rate_throttle(struct drbd_device *device)
 	return false;
 }
 
-static int receive_DataRequest(struct drbd_connection *connection, struct packet_info *pi)
+void drbd_verify_skipped_block(struct drbd_peer_device *peer_device,
+		const sector_t sector, const unsigned int size)
+{
+	++peer_device->ov_skipped;
+	if (peer_device->ov_last_skipped_start + peer_device->ov_last_skipped_size == sector) {
+		peer_device->ov_last_skipped_size += size>>9;
+	} else {
+		ov_skipped_print(peer_device);
+		peer_device->ov_last_skipped_start = sector;
+		peer_device->ov_last_skipped_size = size>>9;
+	}
+}
+
+static void drbd_cleanup_peer_read(
+		struct drbd_peer_request *peer_req, bool in_interval_tree)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+
+	if (in_interval_tree)
+		drbd_remove_peer_req_interval(peer_req);
+
+	if (peer_req->i.type != INTERVAL_PEER_READ)
+		atomic_sub(peer_req->i.size >> SECTOR_SHIFT, &device->rs_sect_ev);
+	dec_unacked(peer_device);
+
+	drbd_free_peer_req(peer_req);
+	put_ldev(device);
+
+	if (atomic_dec_and_test(&connection->backing_ee_cnt))
+		wake_up(&connection->ee_wait);
+}
+
+void drbd_conflict_submit_peer_read(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	bool submit = true;
+	bool interval_tree = false;
+	bool canceled = false;
+
+	/* Hold resync reads until conflicts have cleared so that we know which
+	 * bitmap bits we can safely clear. Also add verify requests on the
+	 * target to the interval tree so that conflicts can be detected.
+	 * Verify requests on the source have already been added. */
+	if (drbd_interval_is_resync(&peer_req->i) || peer_req->i.type == INTERVAL_OV_READ_TARGET) {
+		bool conflict = false;
+		interval_tree = true;
+		spin_lock_irq(&device->interval_lock);
+		clear_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &peer_req->i.flags);
+		canceled = test_bit(INTERVAL_CANCELED, &peer_req->i.flags);
+		conflict = drbd_find_conflict(device, &peer_req->i, 0);
+		if (drbd_interval_empty(&peer_req->i)) {
+			if (conflict)
+				set_bit(INTERVAL_CONFLICT, &peer_req->i.flags);
+			drbd_insert_interval(&device->requests, &peer_req->i);
+		}
+		if (!conflict || drbd_interval_is_verify(&peer_req->i))
+			set_bit(INTERVAL_SUBMITTED, &peer_req->i.flags);
+		else
+			submit = false;
+		spin_unlock_irq(&device->interval_lock);
+	}
+
+	/* Wait if there are conflicts unless this is a verify request, in
+	 * which case we submit it anyway but skip the block if it conflicted. */
+	if (submit) {
+		int err = drbd_submit_peer_request(peer_req);
+		if (err) {
+			if (drbd_ratelimit())
+				drbd_err(peer_device, "submit failed, triggering re-connect\n");
+
+			drbd_cleanup_peer_read(peer_req, interval_tree);
+			change_cstate(peer_device->connection, C_PROTOCOL_ERROR, CS_HARD);
+		}
+	} else if (canceled) {
+		drbd_cleanup_peer_read(peer_req, interval_tree);
+	}
+}
+
+static bool need_to_wait_for_dagtag_of_peer_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_resource *resource = device->resource;
+	struct drbd_connection *connection;
+	bool ret = false;
+
+	rcu_read_lock();
+	connection = drbd_connection_by_node_id(resource, peer_req->depend_dagtag_node_id);
+	if (connection && connection->cstate[NOW] == C_CONNECTED) {
+		if (atomic64_read(&connection->last_dagtag_sector) < peer_req->depend_dagtag)
+			ret = true;
+	}
+	/*
+	 * I am a weak node if the resync source (myself) is not connected to the
+	 * depend_dagtag_node_id. The resync target will abort this resync soon.
+	 * See check_resync_source().
+	 */
+	rcu_read_unlock();
+	return ret;
+}
+
+static void drbd_peer_resync_read_cancel(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	sector_t sector = peer_req->i.sector;
+	int size = peer_req->i.size;
+	u64 block_id = peer_req->block_id;
+
+	if (peer_req->i.type == INTERVAL_OV_READ_SOURCE) {
+		/* P_OV_REPLY */
+		dec_rs_pending(peer_device);
+		drbd_send_ov_result(peer_device, sector, size, block_id, OV_RESULT_SKIP);
+	} else if (peer_req->i.type == INTERVAL_OV_READ_TARGET) {
+		/* P_OV_REQUEST */
+		drbd_verify_skipped_block(peer_device, sector, size);
+		verify_progress(peer_device, sector, size);
+		drbd_send_ack_be(peer_device, P_RS_CANCEL, sector, size, block_id);
+	} else {
+		/* P_RS_DATA_REQUEST etc */
+		drbd_send_ack_be(peer_device, P_RS_CANCEL, sector, size, block_id);
+	}
+}
+
+static void drbd_peer_resync_read(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	unsigned int size = peer_req->i.size;
+
+	if (connection->peer_role[NOW] != R_PRIMARY &&
+			drbd_rs_c_min_rate_throttle(peer_device))
+		schedule_timeout_uninterruptible(HZ/10);
+
+	atomic_add(size >> 9, &device->rs_sect_ev);
+
+	/* dagtag 0 means that there is no dependency to be fulfilled,
+	 * so we can ignore it.
+	 * If we are the dependent node, we can also ignore the dagtag
+	 * dependency, because the request with this dagtag must already be in
+	 * the interval tree, so the read will wait until the interval tree
+	 * conflict is resolved before being submitted. */
+	if (peer_req->depend_dagtag &&
+	    peer_req->depend_dagtag_node_id != device->resource->res_opts.node_id &&
+	    need_to_wait_for_dagtag_of_peer_request(peer_req)) {
+		dynamic_drbd_dbg(peer_device,
+				 "%s at %llus+%u: Waiting for dagtag %llus from peer %u\n",
+				 drbd_interval_type_str(&peer_req->i),
+				 (unsigned long long)peer_req->i.sector, size,
+				 (unsigned long long)peer_req->depend_dagtag,
+				 peer_req->depend_dagtag_node_id);
+		spin_lock_irq(&connection->peer_reqs_lock);
+		list_add_tail(&peer_req->w.list, &connection->dagtag_wait_ee);
+		spin_unlock_irq(&connection->peer_reqs_lock);
+		return;
+	}
+
+	atomic_inc(&connection->backing_ee_cnt);
+	drbd_conflict_submit_peer_read(peer_req);
+}
+
+static int receive_digest(struct drbd_peer_request *peer_req, int digest_size)
+{
+	struct digest_info *di = NULL;
+
+	di = kmalloc(sizeof(*di) + digest_size, GFP_NOIO);
+	if (!di)
+		return -ENOMEM;
+
+	di->digest_size = digest_size;
+	di->digest = (((char *)di)+sizeof(struct digest_info));
+
+	peer_req->digest = di;
+	peer_req->flags |= EE_HAS_DIGEST;
+
+	return drbd_recv_into(peer_req->peer_device->connection, di->digest, digest_size);
+}
+
+static int receive_common_data_request(struct drbd_connection *connection, struct packet_info *pi,
+		struct p_block_req_common *p,
+		unsigned int depend_dagtag_node_id, u64 depend_dagtag)
 {
 	struct drbd_peer_device *peer_device;
 	struct drbd_device *device;
-	sector_t sector;
+	sector_t sector = be64_to_cpu(p->sector);
 	sector_t capacity;
 	struct drbd_peer_request *peer_req;
-	struct digest_info *di = NULL;
-	int size, verb;
-	struct p_block_req *p =	pi->data;
+	int size = be32_to_cpu(p->blksize);
+	enum drbd_disk_state min_d_state;
+	int err;
 
 	peer_device = conn_peer_device(connection, pi->vnr);
 	if (!peer_device)
@@ -2632,67 +3745,144 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 	device = peer_device->device;
 	capacity = get_capacity(device->vdisk);
 
-	sector = be64_to_cpu(p->sector);
-	size   = be32_to_cpu(p->blksize);
-
 	if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_BIO_SIZE) {
-		drbd_err(device, "%s:%d: sector: %llus, size: %u\n", __FILE__, __LINE__,
+		drbd_err(peer_device, "%s:%d: sector: %llus, size: %u\n", __FILE__, __LINE__,
 				(unsigned long long)sector, size);
 		return -EINVAL;
 	}
 	if (sector + (size>>9) > capacity) {
-		drbd_err(device, "%s:%d: sector: %llus, size: %u\n", __FILE__, __LINE__,
+		drbd_err(peer_device, "%s:%d: sector: %llus, size: %u\n", __FILE__, __LINE__,
 				(unsigned long long)sector, size);
 		return -EINVAL;
 	}
 
-	if (!get_ldev_if_state(device, D_UP_TO_DATE)) {
-		verb = 1;
+	/* Tell target to have a retry, waiting for the rescheduled
+	 * drbd_start_resync to complete. Otherwise the concurrency
+	 * of send oos and resync may lead to data loss.
+	 */
+	if (peer_device->repl_state[NOW] == L_WF_BITMAP_S ||
+			peer_device->repl_state[NOW] == L_STARTING_SYNC_S) {
 		switch (pi->cmd) {
-		case P_DATA_REQUEST:
-			drbd_send_ack_rp(peer_device, P_NEG_DREPLY, p);
-			break;
-		case P_RS_THIN_REQ:
 		case P_RS_DATA_REQUEST:
+		case P_RS_DAGTAG_REQ:
 		case P_CSUM_RS_REQUEST:
+		case P_RS_CSUM_DAGTAG_REQ:
+		case P_RS_THIN_REQ:
+		case P_RS_THIN_DAGTAG_REQ:
+			drbd_send_ack_be(peer_device, P_RS_CANCEL, sector, size, p->block_id);
+			return ignore_remaining_packet(connection, pi->size);
+		default:
+			break;
+		}
+	}
+
+	min_d_state = pi->cmd == P_DATA_REQUEST ? D_UP_TO_DATE : D_OUTDATED;
+	if (!get_ldev_if_state(device, min_d_state)) {
+		switch (pi->cmd) {
+		case P_DATA_REQUEST:
+			drbd_send_ack_be(peer_device, P_NEG_DREPLY, sector, size, p->block_id);
+			break;
 		case P_OV_REQUEST:
-			drbd_send_ack_rp(peer_device, P_NEG_RS_DREPLY , p);
+		case P_OV_DAGTAG_REQ:
+			drbd_verify_skipped_block(peer_device, sector, size);
+			verify_progress(peer_device, sector, size);
+			drbd_send_ack_be(peer_device, P_RS_CANCEL, sector, size, p->block_id);
 			break;
-		case P_OV_REPLY:
-			verb = 0;
-			dec_rs_pending(peer_device);
-			drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size, ID_IN_SYNC);
+		case P_RS_DATA_REQUEST:
+		case P_RS_DAGTAG_REQ:
+		case P_CSUM_RS_REQUEST:
+		case P_RS_CSUM_DAGTAG_REQ:
+		case P_RS_THIN_REQ:
+		case P_RS_THIN_DAGTAG_REQ:
+			if (peer_device->repl_state[NOW] == L_PAUSED_SYNC_S)
+				drbd_send_ack_be(peer_device, P_RS_CANCEL, sector, size, p->block_id);
+			else
+				drbd_send_ack_be(peer_device, P_NEG_RS_DREPLY, sector, size, p->block_id);
 			break;
 		default:
 			BUG();
 		}
-		if (verb && drbd_ratelimit())
-			drbd_err(device, "Can not satisfy peer's read request, "
-			    "no local data.\n");
 
-		/* drain possibly payload */
-		return drbd_drain_block(peer_device, pi->size);
+		if (peer_device->repl_state[NOW] != L_PAUSED_SYNC_S)
+			drbd_err_ratelimit(device,
+				"Can not satisfy peer's read request, no local data.\n");
+
+		/* drain possible payload */
+		return ignore_remaining_packet(connection, pi->size);
 	}
 
-	/* GFP_NOIO, because we must not cause arbitrary write-out: in a DRBD
-	 * "criss-cross" setup, that might cause write-out on some other DRBD,
-	 * which in turn might block on the other node at this very place.  */
-	peer_req = drbd_alloc_peer_req(peer_device, p->block_id, sector, size,
-			size, GFP_NOIO);
-	if (!peer_req) {
-		put_ldev(device);
-		return -ENOMEM;
+	if (pi->cmd != P_DATA_REQUEST
+	&& !IS_ALIGNED(size, bm_block_size(device->bitmap))
+	&& (sector + (size >> 9) != device->bitmap->bm_dev_capacity)) {
+		drbd_warn_ratelimit(peer_device,
+			"Unaligned %s request (%u vs %u) at %llu; may lead to hung or repeating resync.\n",
+			drbd_packet_name(pi->cmd), size, bm_block_size(device->bitmap), sector);
+		/* For now, try to continue anyways */
+	}
+
+	inc_unacked(peer_device);
+
+	peer_req = drbd_alloc_peer_req(peer_device, GFP_TRY);
+	err = -ENOMEM;
+	if (!peer_req)
+		goto fail;
+	if (size) {
+		drbd_alloc_page_chain(&peer_device->connection->transport,
+			&peer_req->page_chain, DIV_ROUND_UP(size, PAGE_SIZE), GFP_TRY);
+		if (!peer_req->page_chain.head)
+			goto fail2;
+		if (!mempool_is_saturated(&drbd_buffer_page_pool))
+			peer_req->flags |= EE_RELEASE_TO_MEMPOOL;
 	}
+	peer_req->i.size = size;
+	peer_req->i.sector = sector;
+	peer_req->block_id = p->block_id;
+	peer_req->depend_dagtag_node_id = depend_dagtag_node_id;
+	peer_req->depend_dagtag = depend_dagtag;
 	peer_req->opf = REQ_OP_READ;
+	/* no longer valid, about to call drbd_recv again for the digest... */
+	p = NULL;
+	pi->data = NULL;
+
+	if (peer_device->repl_state[NOW] == L_AHEAD) {
+		if (pi->cmd == P_DATA_REQUEST) {
+			/* P_DATA_REQUEST originates from a Primary,
+			 * so if I am "Ahead", the Primary would be "Behind":
+			 * Can not happen. */
+			drbd_err_ratelimit(peer_device, "received P_DATA_REQUEST while L_AHEAD\n");
+			err = -EINVAL;
+			goto fail2;
+		}
+		if (connection->agreed_pro_version >= 115) {
+			switch (pi->cmd) {
+			/* case P_DATA_REQUEST: see above, not based on protocol version */
+			case P_OV_REQUEST:
+			case P_OV_DAGTAG_REQ:
+				drbd_verify_skipped_block(peer_device, sector, size);
+				verify_progress(peer_device, sector, size);
+				fallthrough;
+			case P_RS_DATA_REQUEST:
+			case P_RS_DAGTAG_REQ:
+			case P_CSUM_RS_REQUEST:
+			case P_RS_CSUM_DAGTAG_REQ:
+			case P_RS_THIN_REQ:
+			case P_RS_THIN_DAGTAG_REQ:
+				err = drbd_send_ack(peer_device, P_RS_CANCEL_AHEAD, peer_req);
+				goto fail2;
+			default:
+				BUG();
+			}
+		}
+	}
 
 	switch (pi->cmd) {
 	case P_DATA_REQUEST:
 		peer_req->w.cb = w_e_end_data_req;
-		/* application IO, don't drbd_rs_begin_io */
-		peer_req->flags |= EE_APPLICATION;
+		peer_req->i.type = INTERVAL_PEER_READ;
 		goto submit;
 
 	case P_RS_THIN_REQ:
+	case P_RS_THIN_DAGTAG_REQ:
 		/* If at some point in the future we have a smart way to
 		   find out if this data block is completely deallocated,
 		   then we would do something smarter here than reading
@@ -2700,56 +3890,44 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 		peer_req->flags |= EE_RS_THIN_REQ;
 		fallthrough;
 	case P_RS_DATA_REQUEST:
+	case P_RS_DAGTAG_REQ:
+		peer_req->i.type = INTERVAL_RESYNC_READ;
 		peer_req->w.cb = w_e_end_rsdata_req;
-		/* used in the sector offset progress display */
-		device->bm_resync_fo = BM_SECT_TO_BIT(sector);
 		break;
 
-	case P_OV_REPLY:
 	case P_CSUM_RS_REQUEST:
-		di = kmalloc(sizeof(*di) + pi->size, GFP_NOIO);
-		if (!di)
-			goto out_free_e;
-
-		di->digest_size = pi->size;
-		di->digest = (((char *)di)+sizeof(struct digest_info));
-
-		peer_req->digest = di;
-		peer_req->flags |= EE_HAS_DIGEST;
-
-		if (drbd_recv_all(peer_device->connection, di->digest, pi->size))
-			goto out_free_e;
-
-		if (pi->cmd == P_CSUM_RS_REQUEST) {
-			D_ASSERT(device, peer_device->connection->agreed_pro_version >= 89);
-			peer_req->w.cb = w_e_end_csum_rs_req;
-			/* used in the sector offset progress display */
-			device->bm_resync_fo = BM_SECT_TO_BIT(sector);
-			/* remember to report stats in drbd_resync_finished */
-			device->use_csums = true;
-		} else if (pi->cmd == P_OV_REPLY) {
-			/* track progress, we may need to throttle */
-			atomic_add(size >> 9, &device->rs_sect_in);
-			peer_req->w.cb = w_e_end_ov_reply;
-			dec_rs_pending(peer_device);
-			/* drbd_rs_begin_io done when we sent this request,
-			 * but accounting still needs to be done. */
-			goto submit_for_resync;
-		}
+	case P_RS_CSUM_DAGTAG_REQ:
+		D_ASSERT(device, connection->agreed_pro_version >= 89);
+		peer_req->i.type = INTERVAL_RESYNC_READ;
+
+		err = receive_digest(peer_req, pi->size);
+		if (err)
+			goto fail2;
+
+		peer_req->w.cb = w_e_end_rsdata_req;
+		/* remember to report stats in drbd_resync_finished */
+		peer_device->use_csums = true;
 		break;
 
 	case P_OV_REQUEST:
-		if (device->ov_start_sector == ~(sector_t)0 &&
-		    peer_device->connection->agreed_pro_version >= 90) {
+	case P_OV_DAGTAG_REQ:
+		peer_req->i.type = INTERVAL_OV_READ_TARGET;
+		peer_device->ov_position = sector;
+		if (peer_device->ov_start_sector == ~(sector_t)0 &&
+		    connection->agreed_pro_version >= 90) {
 			unsigned long now = jiffies;
 			int i;
-			device->ov_start_sector = sector;
-			device->ov_position = sector;
-			device->ov_left = drbd_bm_bits(device) - BM_SECT_TO_BIT(sector);
-			device->rs_total = device->ov_left;
+			unsigned long ov_left = drbd_bm_bits(device)
+					- bm_sect_to_bit(device->bitmap, sector);
+			atomic64_set(&peer_device->ov_left, ov_left);
+			peer_device->ov_start_sector = sector;
+			peer_device->ov_skipped = 0;
+			peer_device->rs_total = ov_left;
+			peer_device->rs_last_writeout = now;
+			peer_device->rs_last_progress_report_ts = now;
 			for (i = 0; i < DRBD_SYNC_MARKS; i++) {
-				device->rs_mark_left[i] = device->ov_left;
-				device->rs_mark_time[i] = now;
+				peer_device->rs_mark_left[i] = ov_left;
+				peer_device->rs_mark_time[i] = now;
 			}
 			drbd_info(device, "Online Verify start sector: %llu\n",
 					(unsigned long long)sector);
@@ -2761,162 +3939,394 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
 		BUG();
 	}
 
-	/* Throttle, drbd_rs_begin_io and submit should become asynchronous
-	 * wrt the receiver, but it is not as straightforward as it may seem.
-	 * Various places in the resync start and stop logic assume resync
-	 * requests are processed in order, requeuing this on the worker thread
-	 * introduces a bunch of new code for synchronization between threads.
-	 *
-	 * Unlimited throttling before drbd_rs_begin_io may stall the resync
-	 * "forever", throttling after drbd_rs_begin_io will lock that extent
-	 * for application writes for the same time.  For now, just throttle
-	 * here, where the rest of the code expects the receiver to sleep for
-	 * a while, anyways.
-	 */
+submit:
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_add_tail(&peer_req->recv_order, &connection->peer_reads);
+	peer_req->flags |= EE_ON_RECV_ORDER;
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	if (pi->cmd == P_DATA_REQUEST) {
+		atomic_inc(&connection->backing_ee_cnt);
+		drbd_conflict_submit_peer_read(peer_req);
+	} else {
+		drbd_peer_resync_read(peer_req);
+	}
+	return 0;
+fail2:
+	drbd_free_peer_req(peer_req);
+fail:
+	dec_unacked(peer_device);
+	put_ldev(device);
+	return err;
+}
 
-	/* Throttle before drbd_rs_begin_io, as that locks out application IO;
-	 * this defers syncer requests for some time, before letting at least
-	 * on request through.  The resync controller on the receiving side
-	 * will adapt to the incoming rate accordingly.
-	 *
-	 * We cannot throttle here if remote is Primary/SyncTarget:
-	 * we would also throttle its application reads.
-	 * In that case, throttling is done on the SyncTarget only.
-	 */
+static int receive_data_request(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_block_req *p_block_req = pi->data;
 
-	/* Even though this may be a resync request, we do add to "read_ee";
-	 * "sync_ee" is only used for resync WRITEs.
-	 * Add to list early, so debugfs can find this request
-	 * even if we have to sleep below. */
-	spin_lock_irq(&device->resource->req_lock);
-	list_add_tail(&peer_req->w.list, &device->read_ee);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	update_receiver_timing_details(connection, drbd_rs_should_slow_down);
-	if (device->state.peer != R_PRIMARY
-	&& drbd_rs_should_slow_down(peer_device, sector, false))
-		schedule_timeout_uninterruptible(HZ/10);
-	update_receiver_timing_details(connection, drbd_rs_begin_io);
-	if (drbd_rs_begin_io(device, sector))
-		goto out_free_e;
+	return receive_common_data_request(connection, pi,
+			&p_block_req->req_common,
+			0, 0);
+}
 
-submit_for_resync:
-	atomic_add(size >> 9, &device->rs_sect_ev);
+/* receive_dagtag_data_request() - handle a request for data with dagtag
+ * dependency initiated by the peer
+ *
+ * Request handling flow:
+ *
+ * receive_dagtag_data_request
+ *        |
+ *        V
+ * receive_common_data_request
+ *        |
+ *        v              dagtag waiting
+ * drbd_peer_resync_read --------------+
+ *        |                            |
+ *        |                           ... dagtag_wait_ee
+ *        |                            |
+ *        |                            v
+ *        +--------------- release_dagtag_wait
+ *        |
+ *        v                       conflict (resync only)
+ * drbd_conflict_submit_peer_read -----+
+ *        |          ^                 |
+ *        |          |                ...
+ *        |          |                 |
+ *        |          |                 v
+ *        |          +---- drbd_do_submit_conflict
+ *        v
+ * drbd_submit_peer_request
+ *        |
+ *       ... backing device
+ *        |
+ *        v
+ * drbd_peer_request_endio
+ *        |
+ *        v                  online verify request
+ * drbd_endio_read_sec_final ------------------+
+ *        |                                    |
+ *       ... sender_work                      ... sender_work
+ *        |                                    |
+ *        v                                    v
+ * w_e_end_rsdata_req                    w_e_end_ov_req
+ *        |                                    |
+ *       ... via peer (on resync_ack_ee)      ... via peer
+ *        |                                    |
+ *        v                                    v
+ * got_RSWriteAck                         got_OVResult
+ */
+static int receive_dagtag_data_request(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_rs_req *p_rs_req = pi->data;
 
-submit:
-	update_receiver_timing_details(connection, drbd_submit_peer_request);
-	inc_unacked(device);
-	if (drbd_submit_peer_request(peer_req) == 0)
-		return 0;
+	return receive_common_data_request(connection, pi,
+			&p_rs_req->req_common,
+			be32_to_cpu(p_rs_req->dagtag_node_id), be64_to_cpu(p_rs_req->dagtag));
+}
+
+static int receive_common_ov_reply(struct drbd_connection *connection, struct packet_info *pi,
+		struct p_block_req_common *p,
+		unsigned int depend_dagtag_node_id, u64 depend_dagtag)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	sector_t sector = be64_to_cpu(p->sector);
+	struct drbd_peer_request *peer_req;
+	int size = be32_to_cpu(p->blksize);
+	int err;
 
-	/* don't care for the reason here */
-	drbd_err(device, "submit failed, triggering re-connect\n");
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	device = peer_device->device;
 
-out_free_e:
-	spin_lock_irq(&device->resource->req_lock);
-	list_del(&peer_req->w.list);
-	spin_unlock_irq(&device->resource->req_lock);
-	/* no drbd_rs_complete_io(), we are dropping the connection anyways */
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_OV_READ_SOURCE),
+			sector, size, p->block_id);
+	if (!peer_req)
+		return -EIO;
+
+	dec_rs_pending(peer_device);
+
+	if (!get_ldev_if_state(device, D_OUTDATED)) {
+		drbd_peer_resync_read_cancel(peer_req);
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+
+		/* drain payload */
+		return ignore_remaining_packet(connection, pi->size);
+	}
+
+	err = receive_digest(peer_req, pi->size);
+	if (err)
+		goto fail;
+
+	set_bit(INTERVAL_RECEIVED, &peer_req->i.flags);
+
+	drbd_alloc_page_chain(&peer_device->connection->transport, &peer_req->page_chain,
+			      DIV_ROUND_UP(size, PAGE_SIZE), GFP_TRY);
+	if (!peer_req->page_chain.head) {
+		err = -ENOMEM;
+		goto fail;
+	}
+	if (!mempool_is_saturated(&drbd_buffer_page_pool))
+		peer_req->flags |= EE_RELEASE_TO_MEMPOOL;
+
+	inc_unacked(peer_device);
+
+	peer_req->depend_dagtag_node_id = depend_dagtag_node_id;
+	peer_req->depend_dagtag = depend_dagtag;
+	peer_req->opf = REQ_OP_READ;
+	peer_req->w.cb = w_e_end_ov_reply;
+
+	/* track progress, we may need to throttle */
+	rs_sectors_came_in(peer_device, size);
 
+	drbd_peer_resync_read(peer_req);
+	return 0;
+fail:
+	drbd_remove_peer_req_interval(peer_req);
+	drbd_free_peer_req(peer_req);
 	put_ldev(device);
-	drbd_free_peer_req(device, peer_req);
-	return -EIO;
+	return err;
 }
 
-/*
- * drbd_asb_recover_0p  -  Recover after split-brain with no remaining primaries
- */
-static int drbd_asb_recover_0p(struct drbd_peer_device *peer_device) __must_hold(local)
+static int receive_ov_reply(struct drbd_connection *connection, struct packet_info *pi)
 {
-	struct drbd_device *device = peer_device->device;
-	int self, peer, rv = -100;
-	unsigned long ch_self, ch_peer;
-	enum drbd_after_sb_p after_sb_0p;
+	struct p_block_req *p_block_req = pi->data;
+
+	return receive_common_ov_reply(connection, pi,
+			&p_block_req->req_common,
+			0, 0);
+}
+
+static int receive_dagtag_ov_reply(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_rs_req *p_rs_req = pi->data;
+
+	return receive_common_ov_reply(connection, pi,
+			&p_rs_req->req_common,
+			be32_to_cpu(p_rs_req->dagtag_node_id), be64_to_cpu(p_rs_req->dagtag));
+}
+
+static int receive_flush_requests(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_connection *other_connection;
+	struct p_flush_requests *p_flush_requests = pi->data;
+	u64 flush_requests_dagtag;
+
+	spin_lock_irq(&resource->tl_update_lock);
+	/*
+	 * If the current dagtag was read from the metadata then there is no
+	 * associated request. Hence there is nothing to flush. Flush up to the
+	 * preceding dagtag instead.
+	 */
+	if (resource->dagtag_sector == resource->dagtag_from_backing_dev)
+		flush_requests_dagtag = resource->dagtag_before_attach;
+	else
+		flush_requests_dagtag = resource->dagtag_sector;
+	spin_unlock_irq(&resource->tl_update_lock);
 
-	self = device->ldev->md.uuid[UI_BITMAP] & 1;
-	peer = device->p_uuid[UI_BITMAP] & 1;
+	spin_lock_irq(&connection->primary_flush_lock);
+	connection->flush_requests_dagtag = flush_requests_dagtag;
+	connection->flush_sequence = be64_to_cpu(p_flush_requests->flush_sequence);
+	connection->flush_forward_sent_mask = 0;
+	spin_unlock_irq(&connection->primary_flush_lock);
 
-	ch_peer = device->p_uuid[UI_SIZE];
-	ch_self = device->comm_bm_set;
+	/* Queue any request waiting for peer ack to be sent */
+	drbd_flush_peer_acks(resource);
 
+	/* For each peer, check if peer ack for this dagtag has already been sent */
 	rcu_read_lock();
-	after_sb_0p = rcu_dereference(peer_device->connection->net_conf)->after_sb_0p;
-	rcu_read_unlock();
-	switch (after_sb_0p) {
-	case ASB_CONSENSUS:
-	case ASB_DISCARD_SECONDARY:
-	case ASB_CALL_HELPER:
-	case ASB_VIOLENTLY:
-		drbd_err(device, "Configuration error.\n");
-		break;
-	case ASB_DISCONNECT:
-		break;
-	case ASB_DISCARD_YOUNGER_PRI:
-		if (self == 0 && peer == 1) {
-			rv = -1;
-			break;
-		}
-		if (self == 1 && peer == 0) {
-			rv =  1;
-			break;
-		}
-		fallthrough;	/* to one of the other strategies */
-	case ASB_DISCARD_OLDER_PRI:
-		if (self == 0 && peer == 1) {
-			rv = 1;
-			break;
-		}
-		if (self == 1 && peer == 0) {
-			rv = -1;
-			break;
-		}
-		/* Else fall through to one of the other strategies... */
-		drbd_warn(device, "Discard younger/older primary did not find a decision\n"
-		     "Using discard-least-changes instead\n");
-		fallthrough;
-	case ASB_DISCARD_ZERO_CHG:
-		if (ch_peer == 0 && ch_self == 0) {
-			rv = test_bit(RESOLVE_CONFLICTS, &peer_device->connection->flags)
-				? -1 : 1;
-			break;
-		} else {
-			if (ch_peer == 0) { rv =  1; break; }
-			if (ch_self == 0) { rv = -1; break; }
-		}
-		if (after_sb_0p == ASB_DISCARD_ZERO_CHG)
-			break;
-		fallthrough;
-	case ASB_DISCARD_LEAST_CHG:
-		if	(ch_self < ch_peer)
-			rv = -1;
-		else if (ch_self > ch_peer)
-			rv =  1;
-		else /* ( ch_self == ch_peer ) */
-		     /* Well, then use something else. */
-			rv = test_bit(RESOLVE_CONFLICTS, &peer_device->connection->flags)
-				? -1 : 1;
-		break;
-	case ASB_DISCARD_LOCAL:
-		rv = -1;
-		break;
-	case ASB_DISCARD_REMOTE:
-		rv =  1;
+	for_each_connection_rcu(other_connection, resource) {
+		if (other_connection->cstate[NOW] == C_CONNECTED)
+			queue_work(other_connection->ack_sender, &other_connection->peer_ack_work);
 	}
+	rcu_read_unlock();
 
-	return rv;
+	return 0;
 }
 
-/*
+static int receive_flush_requests_ack(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_connection *primary_connection;
+	struct p_flush_ack *p_flush_ack = pi->data;
+	u64 flush_sequence = be64_to_cpu(p_flush_ack->flush_sequence);
+	int primary_node_id = be32_to_cpu(p_flush_ack->primary_node_id);
+
+	spin_lock_irq(&resource->initiator_flush_lock);
+	if (flush_sequence < resource->current_flush_sequence) {
+		spin_unlock_irq(&resource->initiator_flush_lock);
+		return 0;
+	}
+
+	rcu_read_lock();
+	primary_connection = drbd_connection_by_node_id(resource, primary_node_id);
+	if (primary_connection)
+		primary_connection->pending_flush_mask &= ~NODE_MASK(connection->peer_node_id);
+	rcu_read_unlock();
+	spin_unlock_irq(&resource->initiator_flush_lock);
+	return 0;
+}
+
+/*
+ * config_unknown_volume  -  device configuration command for unknown volume
+ *
+ * When a device is added to an existing connection, the node on which the
+ * device is added first will send configuration commands to its peer but the
+ * peer will not know about the device yet.  It will warn and ignore these
+ * commands.  Once the device is added on the second node, the second node will
+ * send the same device configuration commands, but in the other direction.
+ *
+ * (We can also end up here if drbd is misconfigured.)
+ */
+static int config_unknown_volume(struct drbd_connection *connection, struct packet_info *pi)
+{
+	drbd_warn(connection, "%s packet received for volume %d, which is not configured locally\n",
+		  drbd_packet_name(pi->cmd), pi->vnr);
+	return ignore_remaining_packet(connection, pi->size);
+}
+
+static int receive_enable_replication_next(struct drbd_connection *connection,
+		struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct p_enable_replication *p_enable_replication = pi->data;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return config_unknown_volume(connection, pi);
+
+	if (p_enable_replication->enable)
+		set_bit(REPLICATION_NEXT, &peer_device->flags);
+	else
+		clear_bit(REPLICATION_NEXT, &peer_device->flags);
+
+	return 0;
+}
+
+static int receive_enable_replication(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	struct p_enable_replication *p_enable_replication = pi->data;
+	unsigned long irq_flags;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	begin_state_change(resource, &irq_flags, CS_VERBOSE);
+	peer_device->replication[NEW] = p_enable_replication->enable;
+	end_state_change(resource, &irq_flags, "enable-replication");
+	return 0;
+}
+
+/*
+ * drbd_asb_recover_0p  -  Recover after split-brain with no remaining primaries
+ */
+static enum sync_strategy drbd_asb_recover_0p(struct drbd_peer_device *peer_device) __must_hold(local)
+{
+	const int node_id = peer_device->device->resource->res_opts.node_id;
+	int self, peer;
+	enum sync_strategy rv = SPLIT_BRAIN_DISCONNECT;
+	unsigned long ch_self, ch_peer;
+	enum drbd_after_sb_p after_sb_0p;
+
+	self = drbd_bitmap_uuid(peer_device) & UUID_PRIMARY;
+	peer = peer_device->bitmap_uuids[node_id] & UUID_PRIMARY;
+
+	ch_peer = peer_device->dirty_bits;
+	ch_self = peer_device->comm_bm_set;
+
+	rcu_read_lock();
+	after_sb_0p = rcu_dereference(peer_device->connection->transport.net_conf)->after_sb_0p;
+	rcu_read_unlock();
+	switch (after_sb_0p) {
+	case ASB_CONSENSUS:
+	case ASB_DISCARD_SECONDARY:
+	case ASB_CALL_HELPER:
+	case ASB_VIOLENTLY:
+	case ASB_RETRY_CONNECT:
+	case ASB_AUTO_DISCARD:
+		drbd_err(peer_device, "Configuration error.\n");
+		break;
+	case ASB_DISCONNECT:
+		break;
+	case ASB_DISCARD_YOUNGER_PRI:
+		if (self == 0 && peer == 1) {
+			rv = SYNC_TARGET_USE_BITMAP;
+			break;
+		}
+		if (self == 1 && peer == 0) {
+			rv = SYNC_SOURCE_USE_BITMAP;
+			break;
+		}
+		fallthrough;	/* to one of the other strategies */
+	case ASB_DISCARD_OLDER_PRI:
+		if (self == 0 && peer == 1) {
+			rv = SYNC_SOURCE_USE_BITMAP;
+			break;
+		}
+		if (self == 1 && peer == 0) {
+			rv = SYNC_TARGET_USE_BITMAP;
+			break;
+		}
+		drbd_warn(peer_device, "Discard younger/older primary did not find a decision\n"
+			  "Using discard-least-changes instead\n");
+		fallthrough;
+	case ASB_DISCARD_ZERO_CHG:
+		if (ch_peer == 0 && ch_self == 0) {
+			rv = test_bit(RESOLVE_CONFLICTS, &peer_device->connection->transport.flags)
+				? SYNC_TARGET_USE_BITMAP : SYNC_SOURCE_USE_BITMAP;
+			break;
+		} else {
+			if (ch_peer == 0) {
+				rv = SYNC_SOURCE_USE_BITMAP;
+				break;
+			}
+			if (ch_self == 0) {
+				rv = SYNC_TARGET_USE_BITMAP;
+				break;
+			}
+		}
+		if (after_sb_0p == ASB_DISCARD_ZERO_CHG)
+			break;
+		fallthrough;
+	case ASB_DISCARD_LEAST_CHG:
+		if	(ch_self < ch_peer)
+			rv = SYNC_TARGET_USE_BITMAP;
+		else if (ch_self > ch_peer)
+			rv = SYNC_SOURCE_USE_BITMAP;
+		else /* ( ch_self == ch_peer ) */
+		     /* Well, then use something else. */
+			rv = test_bit(RESOLVE_CONFLICTS, &peer_device->connection->transport.flags)
+				? SYNC_TARGET_USE_BITMAP : SYNC_SOURCE_USE_BITMAP;
+		break;
+	case ASB_DISCARD_LOCAL:
+		rv = SYNC_TARGET_USE_BITMAP;
+		break;
+	case ASB_DISCARD_REMOTE:
+		rv = SYNC_SOURCE_USE_BITMAP;
+	}
+
+	return rv;
+}
+
+/*
  * drbd_asb_recover_1p  -  Recover after split-brain with one remaining primary
  */
-static int drbd_asb_recover_1p(struct drbd_peer_device *peer_device) __must_hold(local)
+static enum sync_strategy drbd_asb_recover_1p(struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	struct drbd_device *device = peer_device->device;
-	int hg, rv = -100;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_resource *resource = device->resource;
+	enum sync_strategy strategy, rv = SPLIT_BRAIN_DISCONNECT;
 	enum drbd_after_sb_p after_sb_1p;
 
 	rcu_read_lock();
-	after_sb_1p = rcu_dereference(peer_device->connection->net_conf)->after_sb_1p;
+	after_sb_1p = rcu_dereference(connection->transport.net_conf)->after_sb_1p;
 	rcu_read_unlock();
 	switch (after_sb_1p) {
 	case ASB_DISCARD_YOUNGER_PRI:
@@ -2925,39 +4335,42 @@ static int drbd_asb_recover_1p(struct drbd_peer_device *peer_device) __must_hold
 	case ASB_DISCARD_LOCAL:
 	case ASB_DISCARD_REMOTE:
 	case ASB_DISCARD_ZERO_CHG:
+	case ASB_RETRY_CONNECT:
+	case ASB_AUTO_DISCARD:
 		drbd_err(device, "Configuration error.\n");
 		break;
 	case ASB_DISCONNECT:
 		break;
 	case ASB_CONSENSUS:
-		hg = drbd_asb_recover_0p(peer_device);
-		if (hg == -1 && device->state.role == R_SECONDARY)
-			rv = hg;
-		if (hg == 1  && device->state.role == R_PRIMARY)
-			rv = hg;
+		strategy = drbd_asb_recover_0p(peer_device);
+		if (strategy == SYNC_TARGET_USE_BITMAP && resource->role[NOW] == R_SECONDARY)
+			rv = strategy;
+		if (strategy == SYNC_SOURCE_USE_BITMAP && resource->role[NOW] == R_PRIMARY)
+			rv = strategy;
 		break;
 	case ASB_VIOLENTLY:
 		rv = drbd_asb_recover_0p(peer_device);
 		break;
 	case ASB_DISCARD_SECONDARY:
-		return device->state.role == R_PRIMARY ? 1 : -1;
+		return resource->role[NOW] == R_PRIMARY ? SYNC_SOURCE_USE_BITMAP : SYNC_TARGET_USE_BITMAP;
 	case ASB_CALL_HELPER:
-		hg = drbd_asb_recover_0p(peer_device);
-		if (hg == -1 && device->state.role == R_PRIMARY) {
+		strategy = drbd_asb_recover_0p(peer_device);
+		if (strategy == SYNC_TARGET_USE_BITMAP && resource->role[NOW] == R_PRIMARY) {
 			enum drbd_state_rv rv2;
 
 			 /* drbd_change_state() does not sleep while in SS_IN_TRANSIENT_STATE,
-			  * we might be here in C_WF_REPORT_PARAMS which is transient.
+			  * we might be here in L_OFF which is transient.
 			  * we do not need to wait for the after state change work either. */
-			rv2 = drbd_change_state(device, CS_VERBOSE, NS(role, R_SECONDARY));
+			rv2 = change_role(resource, R_SECONDARY, CS_VERBOSE,
+					"after-sb-1pri", NULL);
 			if (rv2 != SS_SUCCESS) {
-				drbd_khelper(device, "pri-lost-after-sb");
+				drbd_maybe_khelper(device, connection, "pri-lost-after-sb");
 			} else {
 				drbd_warn(device, "Successfully gave up primary role.\n");
-				rv = hg;
+				rv = strategy;
 			}
 		} else
-			rv = hg;
+			rv = strategy;
 	}
 
 	return rv;
@@ -2966,14 +4379,15 @@ static int drbd_asb_recover_1p(struct drbd_peer_device *peer_device) __must_hold
 /*
  * drbd_asb_recover_2p  -  Recover after split-brain with two remaining primaries
  */
-static int drbd_asb_recover_2p(struct drbd_peer_device *peer_device) __must_hold(local)
+static enum sync_strategy drbd_asb_recover_2p(struct drbd_peer_device *peer_device) __must_hold(local)
 {
 	struct drbd_device *device = peer_device->device;
-	int hg, rv = -100;
+	struct drbd_connection *connection = peer_device->connection;
+	enum sync_strategy strategy, rv = SPLIT_BRAIN_DISCONNECT;
 	enum drbd_after_sb_p after_sb_2p;
 
 	rcu_read_lock();
-	after_sb_2p = rcu_dereference(peer_device->connection->net_conf)->after_sb_2p;
+	after_sb_2p = rcu_dereference(connection->transport.net_conf)->after_sb_2p;
 	rcu_read_unlock();
 	switch (after_sb_2p) {
 	case ASB_DISCARD_YOUNGER_PRI:
@@ -2984,6 +4398,8 @@ static int drbd_asb_recover_2p(struct drbd_peer_device *peer_device) __must_hold
 	case ASB_CONSENSUS:
 	case ASB_DISCARD_SECONDARY:
 	case ASB_DISCARD_ZERO_CHG:
+	case ASB_RETRY_CONNECT:
+	case ASB_AUTO_DISCARD:
 		drbd_err(device, "Configuration error.\n");
 		break;
 	case ASB_VIOLENTLY:
@@ -2992,457 +4408,983 @@ static int drbd_asb_recover_2p(struct drbd_peer_device *peer_device) __must_hold
 	case ASB_DISCONNECT:
 		break;
 	case ASB_CALL_HELPER:
-		hg = drbd_asb_recover_0p(peer_device);
-		if (hg == -1) {
+		strategy = drbd_asb_recover_0p(peer_device);
+		if (strategy == SYNC_TARGET_USE_BITMAP) {
 			enum drbd_state_rv rv2;
 
 			 /* drbd_change_state() does not sleep while in SS_IN_TRANSIENT_STATE,
-			  * we might be here in C_WF_REPORT_PARAMS which is transient.
+			  * we might be here in L_OFF which is transient.
 			  * we do not need to wait for the after state change work either. */
-			rv2 = drbd_change_state(device, CS_VERBOSE, NS(role, R_SECONDARY));
+			rv2 = change_role(device->resource, R_SECONDARY, CS_VERBOSE,
+					"after-sb-2pri", NULL);
 			if (rv2 != SS_SUCCESS) {
-				drbd_khelper(device, "pri-lost-after-sb");
+				drbd_maybe_khelper(device, connection, "pri-lost-after-sb");
 			} else {
 				drbd_warn(device, "Successfully gave up primary role.\n");
-				rv = hg;
+				rv = strategy;
 			}
 		} else
-			rv = hg;
+			rv = strategy;
 	}
 
 	return rv;
 }
 
-static void drbd_uuid_dump(struct drbd_device *device, char *text, u64 *uuid,
-			   u64 bits, u64 flags)
+static void drbd_uuid_dump_self(struct drbd_peer_device *peer_device, u64 bits, u64 flags)
 {
-	if (!uuid) {
-		drbd_info(device, "%s uuid info vanished while I was looking!\n", text);
-		return;
-	}
-	drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX bits:%llu flags:%llX\n",
-	     text,
-	     (unsigned long long)uuid[UI_CURRENT],
-	     (unsigned long long)uuid[UI_BITMAP],
-	     (unsigned long long)uuid[UI_HISTORY_START],
-	     (unsigned long long)uuid[UI_HISTORY_END],
-	     (unsigned long long)bits,
-	     (unsigned long long)flags);
+	struct drbd_device *device = peer_device->device;
+
+	drbd_info(peer_device, "self %016llX:%016llX:%016llX:%016llX bits:%llu flags:%llX\n",
+		  (unsigned long long)drbd_resolved_uuid(peer_device, NULL),
+		  (unsigned long long)drbd_bitmap_uuid(peer_device),
+		  (unsigned long long)drbd_history_uuid(device, 0),
+		  (unsigned long long)drbd_history_uuid(device, 1),
+		  (unsigned long long)bits,
+		  (unsigned long long)flags);
 }
 
-/*
-  100	after split brain try auto recover
-    2	C_SYNC_SOURCE set BitMap
-    1	C_SYNC_SOURCE use BitMap
-    0	no Sync
-   -1	C_SYNC_TARGET use BitMap
-   -2	C_SYNC_TARGET set BitMap
- -100	after split brain, disconnect
--1000	unrelated data
--1091   requires proto 91
--1096   requires proto 96
- */
 
-static int drbd_uuid_compare(struct drbd_peer_device *const peer_device,
-		enum drbd_role const peer_role, int *rule_nr) __must_hold(local)
+static void drbd_uuid_dump_peer(struct drbd_peer_device *peer_device, u64 bits, u64 flags)
 {
-	struct drbd_connection *const connection = peer_device->connection;
-	struct drbd_device *device = peer_device->device;
-	u64 self, peer;
-	int i, j;
+	const int node_id = peer_device->device->resource->res_opts.node_id;
 
-	self = device->ldev->md.uuid[UI_CURRENT] & ~((u64)1);
-	peer = device->p_uuid[UI_CURRENT] & ~((u64)1);
+	drbd_info(peer_device, "peer %016llX:%016llX:%016llX:%016llX bits:%llu flags:%llX\n",
+	     (unsigned long long)peer_device->current_uuid,
+	     (unsigned long long)peer_device->bitmap_uuids[node_id],
+	     (unsigned long long)peer_device->history_uuids[0],
+	     (unsigned long long)peer_device->history_uuids[1],
+	     (unsigned long long)bits,
+	     (unsigned long long)flags);
+}
 
-	*rule_nr = 10;
-	if (self == UUID_JUST_CREATED && peer == UUID_JUST_CREATED)
-		return 0;
+/* find the peer's bitmap slot for the given UUID, if they have one */
+static int drbd_find_peer_bitmap_by_uuid(struct drbd_peer_device *peer_device, u64 uuid)
+{
+	u64 peer;
+	int i;
 
-	*rule_nr = 20;
-	if ((self == UUID_JUST_CREATED || self == (u64)0) &&
-	     peer != UUID_JUST_CREATED)
-		return -2;
+	for (i = 0; i < DRBD_PEERS_MAX; i++) {
+		peer = peer_device->bitmap_uuids[i] & ~UUID_PRIMARY;
+		if (uuid == peer)
+			return i;
+	}
 
-	*rule_nr = 30;
-	if (self != UUID_JUST_CREATED &&
-	    (peer == UUID_JUST_CREATED || peer == (u64)0))
-		return 2;
+	return -1;
+}
 
-	if (self == peer) {
-		int rct, dc; /* roles at crash time */
+/* find our bitmap slot for the given UUID, if we have one */
+static int drbd_find_bitmap_by_uuid(struct drbd_peer_device *peer_device, u64 uuid)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	u64 self;
+	int i;
 
-		if (device->p_uuid[UI_BITMAP] == (u64)0 && device->ldev->md.uuid[UI_BITMAP] != (u64)0) {
+	for (i = 0; i < DRBD_NODE_ID_MAX; i++) {
+		if (i == device->ldev->md.node_id)
+			continue;
+		if (connection->agreed_pro_version < 116 &&
+		    device->ldev->md.peers[i].bitmap_index == -1)
+			continue;
+		self = device->ldev->md.peers[i].bitmap_uuid & ~UUID_PRIMARY;
+		if (self == uuid)
+			return i;
+	}
 
-			if (connection->agreed_pro_version < 91)
-				return -1091;
+	return -1;
+}
 
-			if ((device->ldev->md.uuid[UI_BITMAP] & ~((u64)1)) == (device->p_uuid[UI_HISTORY_START] & ~((u64)1)) &&
-			    (device->ldev->md.uuid[UI_HISTORY_START] & ~((u64)1)) == (device->p_uuid[UI_HISTORY_START + 1] & ~((u64)1))) {
-				drbd_info(device, "was SyncSource, missed the resync finished event, corrected myself:\n");
-				drbd_uuid_move_history(device);
-				device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
-				device->ldev->md.uuid[UI_BITMAP] = 0;
+static enum sync_strategy uuid_fixup_resync_end(struct drbd_peer_device *peer_device, enum sync_rule *rule) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	const int node_id = device->resource->res_opts.node_id;
 
-				drbd_uuid_dump(device, "self", device->ldev->md.uuid,
-					       device->state.disk >= D_NEGOTIATING ? drbd_bm_total_weight(device) : 0, 0);
-				*rule_nr = 34;
-			} else {
-				drbd_info(device, "was SyncSource (peer failed to write sync_uuid)\n");
-				*rule_nr = 36;
-			}
+	if (peer_device->bitmap_uuids[node_id] == (u64)0 && drbd_bitmap_uuid(peer_device) != (u64)0) {
 
-			return 1;
-		}
+		if (peer_device->connection->agreed_pro_version < 91)
+			return REQUIRES_PROTO_91;
 
-		if (device->ldev->md.uuid[UI_BITMAP] == (u64)0 && device->p_uuid[UI_BITMAP] != (u64)0) {
+		if ((drbd_bitmap_uuid(peer_device) & ~UUID_PRIMARY) ==
+		    (peer_device->history_uuids[0] & ~UUID_PRIMARY) &&
+		    (drbd_history_uuid(device, 0) & ~UUID_PRIMARY) ==
+		    (peer_device->history_uuids[0] & ~UUID_PRIMARY)) {
+			struct drbd_peer_md *peer_md = &device->ldev->md.peers[peer_device->node_id];
+			u64 previous_bitmap_uuid = peer_md->bitmap_uuid;
 
-			if (connection->agreed_pro_version < 91)
-				return -1091;
+			drbd_info(device, "was SyncSource, missed the resync finished event, corrected myself:\n");
+			peer_md->bitmap_uuid = 0;
+			_drbd_uuid_push_history(device, previous_bitmap_uuid);
 
-			if ((device->ldev->md.uuid[UI_HISTORY_START] & ~((u64)1)) == (device->p_uuid[UI_BITMAP] & ~((u64)1)) &&
-			    (device->ldev->md.uuid[UI_HISTORY_START + 1] & ~((u64)1)) == (device->p_uuid[UI_HISTORY_START] & ~((u64)1))) {
-				drbd_info(device, "was SyncTarget, peer missed the resync finished event, corrected peer:\n");
+			drbd_uuid_dump_self(peer_device,
+					    device->disk_state[NOW] >= D_NEGOTIATING ? drbd_bm_total_weight(peer_device) : 0, 0);
+			*rule = RULE_SYNC_SOURCE_MISSED_FINISH;
+		} else {
+			drbd_info(device, "was SyncSource (peer failed to write sync_uuid)\n");
+			*rule = RULE_SYNC_SOURCE_PEER_MISSED_FINISH;
+		}
 
-				device->p_uuid[UI_HISTORY_START + 1] = device->p_uuid[UI_HISTORY_START];
-				device->p_uuid[UI_HISTORY_START] = device->p_uuid[UI_BITMAP];
-				device->p_uuid[UI_BITMAP] = 0UL;
+		return SYNC_SOURCE_USE_BITMAP;
+	}
 
-				drbd_uuid_dump(device, "peer", device->p_uuid, device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]);
-				*rule_nr = 35;
-			} else {
-				drbd_info(device, "was SyncTarget (failed to write sync_uuid)\n");
-				*rule_nr = 37;
-			}
+	if (drbd_bitmap_uuid(peer_device) == (u64)0 && peer_device->bitmap_uuids[node_id] != (u64)0) {
 
-			return -1;
-		}
+		if (peer_device->connection->agreed_pro_version < 91)
+			return REQUIRES_PROTO_91;
 
-		/* Common power [off|failure] */
-		rct = (test_bit(CRASHED_PRIMARY, &device->flags) ? 1 : 0) +
-			(device->p_uuid[UI_FLAGS] & 2);
-		/* lowest bit is set when we were primary,
-		 * next bit (weight 2) is set when peer was primary */
-		*rule_nr = 40;
+		if ((drbd_history_uuid(device, 0) & ~UUID_PRIMARY) ==
+		    (peer_device->bitmap_uuids[node_id] & ~UUID_PRIMARY) &&
+		    (drbd_history_uuid(device, 1) & ~UUID_PRIMARY) ==
+		    (peer_device->history_uuids[0] & ~UUID_PRIMARY)) {
+			int i;
 
-		/* Neither has the "crashed primary" flag set,
-		 * only a replication link hickup. */
-		if (rct == 0)
-			return 0;
+			drbd_info(device, "was SyncTarget, peer missed the resync finished event, corrected peer:\n");
 
-		/* Current UUID equal and no bitmap uuid; does not necessarily
-		 * mean this was a "simultaneous hard crash", maybe IO was
-		 * frozen, so no UUID-bump happened.
-		 * This is a protocol change, overload DRBD_FF_WSAME as flag
-		 * for "new-enough" peer DRBD version. */
-		if (device->state.role == R_PRIMARY || peer_role == R_PRIMARY) {
-			*rule_nr = 41;
-			if (!(connection->agreed_features & DRBD_FF_WSAME)) {
-				drbd_warn(peer_device, "Equivalent unrotated UUIDs, but current primary present.\n");
-				return -(0x10000 | PRO_VERSION_MAX | (DRBD_FF_WSAME << 8));
-			}
-			if (device->state.role == R_PRIMARY && peer_role == R_PRIMARY) {
-				/* At least one has the "crashed primary" bit set,
-				 * both are primary now, but neither has rotated its UUIDs?
-				 * "Can not happen." */
-				drbd_err(peer_device, "Equivalent unrotated UUIDs, but both are primary. Can not resolve this.\n");
-				return -100;
-			}
-			if (device->state.role == R_PRIMARY)
-				return 1;
-			return -1;
-		}
+			for (i = ARRAY_SIZE(peer_device->history_uuids) - 1; i > 0; i--)
+				peer_device->history_uuids[i] = peer_device->history_uuids[i - 1];
+			peer_device->history_uuids[i] = peer_device->bitmap_uuids[node_id];
+			peer_device->bitmap_uuids[node_id] = 0;
 
-		/* Both are secondary.
-		 * Really looks like recovery from simultaneous hard crash.
-		 * Check which had been primary before, and arbitrate. */
-		switch (rct) {
-		case 0: /* !self_pri && !peer_pri */ return 0; /* already handled */
-		case 1: /*  self_pri && !peer_pri */ return 1;
-		case 2: /* !self_pri &&  peer_pri */ return -1;
-		case 3: /*  self_pri &&  peer_pri */
-			dc = test_bit(RESOLVE_CONFLICTS, &connection->flags);
-			return dc ? -1 : 1;
+			drbd_uuid_dump_peer(peer_device, peer_device->dirty_bits, peer_device->uuid_flags);
+			*rule = RULE_SYNC_TARGET_PEER_MISSED_FINISH;
+		} else {
+			drbd_info(device, "was SyncTarget (failed to write sync_uuid)\n");
+			*rule = RULE_SYNC_TARGET_MISSED_FINISH;
 		}
+
+		return SYNC_TARGET_USE_BITMAP;
 	}
 
-	*rule_nr = 50;
-	peer = device->p_uuid[UI_BITMAP] & ~((u64)1);
-	if (self == peer)
-		return -1;
+	return UNDETERMINED;
+}
+
+static enum sync_strategy uuid_fixup_resync_start1(struct drbd_peer_device *peer_device, enum sync_rule *rule) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	const int node_id = peer_device->device->resource->res_opts.node_id;
+	u64 self, peer;
+
+	self = drbd_current_uuid(device) & ~UUID_PRIMARY;
+	peer = peer_device->history_uuids[0] & ~UUID_PRIMARY;
 
-	*rule_nr = 51;
-	peer = device->p_uuid[UI_HISTORY_START] & ~((u64)1);
 	if (self == peer) {
-		if (connection->agreed_pro_version < 96 ?
-		    (device->ldev->md.uuid[UI_HISTORY_START] & ~((u64)1)) ==
-		    (device->p_uuid[UI_HISTORY_START + 1] & ~((u64)1)) :
-		    peer + UUID_NEW_BM_OFFSET == (device->p_uuid[UI_BITMAP] & ~((u64)1))) {
+		if (peer_device->connection->agreed_pro_version < 96 ?
+		    (drbd_history_uuid(device, 0) & ~UUID_PRIMARY) ==
+		    (peer_device->history_uuids[1] & ~UUID_PRIMARY) :
+		    peer + UUID_NEW_BM_OFFSET == (peer_device->bitmap_uuids[node_id] & ~UUID_PRIMARY)) {
+			int i;
+
 			/* The last P_SYNC_UUID did not get though. Undo the last start of
 			   resync as sync source modifications of the peer's UUIDs. */
+			*rule = RULE_SYNC_TARGET_MISSED_START;
 
-			if (connection->agreed_pro_version < 91)
-				return -1091;
+			if (peer_device->connection->agreed_pro_version < 91)
+				return REQUIRES_PROTO_91;
 
-			device->p_uuid[UI_BITMAP] = device->p_uuid[UI_HISTORY_START];
-			device->p_uuid[UI_HISTORY_START] = device->p_uuid[UI_HISTORY_START + 1];
+			peer_device->bitmap_uuids[node_id] = peer_device->history_uuids[0];
+			for (i = 0; i < ARRAY_SIZE(peer_device->history_uuids) - 1; i++)
+				peer_device->history_uuids[i] = peer_device->history_uuids[i + 1];
+			peer_device->history_uuids[i] = 0;
 
 			drbd_info(device, "Lost last syncUUID packet, corrected:\n");
-			drbd_uuid_dump(device, "peer", device->p_uuid, device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]);
+			drbd_uuid_dump_peer(peer_device, peer_device->dirty_bits, peer_device->uuid_flags);
 
-			return -1;
+			return SYNC_TARGET_USE_BITMAP;
 		}
 	}
 
-	*rule_nr = 60;
-	self = device->ldev->md.uuid[UI_CURRENT] & ~((u64)1);
-	for (i = UI_HISTORY_START; i <= UI_HISTORY_END; i++) {
-		peer = device->p_uuid[i] & ~((u64)1);
-		if (self == peer)
-			return -2;
-	}
+	return UNDETERMINED;
+}
 
-	*rule_nr = 70;
-	self = device->ldev->md.uuid[UI_BITMAP] & ~((u64)1);
-	peer = device->p_uuid[UI_CURRENT] & ~((u64)1);
-	if (self == peer)
-		return 1;
+static enum sync_strategy uuid_fixup_resync_start2(struct drbd_peer_device *peer_device, enum sync_rule *rule) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	u64 self, peer;
+
+	self = drbd_history_uuid(device, 0) & ~UUID_PRIMARY;
+	peer = peer_device->current_uuid & ~UUID_PRIMARY;
 
-	*rule_nr = 71;
-	self = device->ldev->md.uuid[UI_HISTORY_START] & ~((u64)1);
 	if (self == peer) {
-		if (connection->agreed_pro_version < 96 ?
-		    (device->ldev->md.uuid[UI_HISTORY_START + 1] & ~((u64)1)) ==
-		    (device->p_uuid[UI_HISTORY_START] & ~((u64)1)) :
-		    self + UUID_NEW_BM_OFFSET == (device->ldev->md.uuid[UI_BITMAP] & ~((u64)1))) {
+		if (peer_device->connection->agreed_pro_version < 96 ?
+		    (drbd_history_uuid(device, 1) & ~UUID_PRIMARY) ==
+		    (peer_device->history_uuids[0] & ~UUID_PRIMARY) :
+		    self + UUID_NEW_BM_OFFSET == (drbd_bitmap_uuid(peer_device) & ~UUID_PRIMARY)) {
+			u64 bitmap_uuid;
+
 			/* The last P_SYNC_UUID did not get though. Undo the last start of
 			   resync as sync source modifications of our UUIDs. */
+			*rule = RULE_SYNC_SOURCE_MISSED_START;
 
-			if (connection->agreed_pro_version < 91)
-				return -1091;
+			if (peer_device->connection->agreed_pro_version < 91)
+				return REQUIRES_PROTO_91;
 
-			__drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_HISTORY_START]);
-			__drbd_uuid_set(device, UI_HISTORY_START, device->ldev->md.uuid[UI_HISTORY_START + 1]);
+			bitmap_uuid = _drbd_uuid_pull_history(peer_device);
+			_drbd_uuid_set_bitmap(peer_device, bitmap_uuid);
 
 			drbd_info(device, "Last syncUUID did not get through, corrected:\n");
-			drbd_uuid_dump(device, "self", device->ldev->md.uuid,
-				       device->state.disk >= D_NEGOTIATING ? drbd_bm_total_weight(device) : 0, 0);
+			drbd_uuid_dump_self(peer_device,
+					    device->disk_state[NOW] >= D_NEGOTIATING ? drbd_bm_total_weight(peer_device) : 0, 0);
 
-			return 1;
+			return SYNC_SOURCE_USE_BITMAP;
 		}
 	}
 
+	return UNDETERMINED;
+}
 
-	*rule_nr = 80;
-	peer = device->p_uuid[UI_CURRENT] & ~((u64)1);
-	for (i = UI_HISTORY_START; i <= UI_HISTORY_END; i++) {
-		self = device->ldev->md.uuid[i] & ~((u64)1);
-		if (self == peer)
-			return 2;
-	}
-
-	*rule_nr = 90;
-	self = device->ldev->md.uuid[UI_BITMAP] & ~((u64)1);
-	peer = device->p_uuid[UI_BITMAP] & ~((u64)1);
-	if (self == peer && self != ((u64)0))
-		return 100;
+static enum sync_strategy drbd_uuid_compare(struct drbd_peer_device *peer_device,
+			     enum sync_rule *rule, int *peer_node_id) __must_hold(local)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	const int node_id = device->resource->res_opts.node_id;
+	bool my_current_in_peers_history, peers_current_in_my_history;
+	bool bitmap_matches, flags_matches, uuid_matches;
+	u64 resolved_uuid, bitmap_uuid;
+	u64 local_uuid_flags = 0;
+	u64 self, peer;
+	int i, j;
 
-	*rule_nr = 100;
-	for (i = UI_HISTORY_START; i <= UI_HISTORY_END; i++) {
-		self = device->ldev->md.uuid[i] & ~((u64)1);
-		for (j = UI_HISTORY_START; j <= UI_HISTORY_END; j++) {
-			peer = device->p_uuid[j] & ~((u64)1);
-			if (self == peer)
-				return -100;
+	resolved_uuid = drbd_resolved_uuid(peer_device, &local_uuid_flags) & ~UUID_PRIMARY;
+	bitmap_uuid = drbd_bitmap_uuid(peer_device);
+	local_uuid_flags |= drbd_collect_local_uuid_flags(peer_device, NULL);
+
+	uuid_matches = resolved_uuid == (peer_device->comm_current_uuid & ~UUID_PRIMARY);
+	bitmap_matches = bitmap_uuid == peer_device->comm_bitmap_uuid;
+	/* UUID_FLAG_INCONSISTENT is not relevant for the handshake, allow it to change */
+	flags_matches = !((local_uuid_flags ^ peer_device->comm_uuid_flags) & ~UUID_FLAG_INCONSISTENT);
+	if (!test_bit(INITIAL_STATE_SENT, &peer_device->flags)) {
+		drbd_warn(peer_device, "Initial UUIDs and state not sent yet. Not verifying\n");
+	} else if (!uuid_matches || !flags_matches || !bitmap_matches) {
+		if (!uuid_matches)
+			drbd_warn(peer_device, "My current UUID changed during handshake.\n");
+		if (!bitmap_matches)
+			drbd_warn(peer_device, "My bitmap UUID changed during "
+				  "handshake. 0x%llX to 0x%llX\n",
+				  (unsigned long long)peer_device->comm_bitmap_uuid,
+				  (unsigned long long)bitmap_uuid);
+		if (!flags_matches)
+			drbd_warn(peer_device,
+				  "My uuid_flags changed from 0x%llX to 0x%llX during handshake.\n",
+				  (unsigned long long)peer_device->comm_uuid_flags,
+				  (unsigned long long)local_uuid_flags);
+		if (connection->cstate[NOW] == C_CONNECTING) {
+			*rule = RULE_INITIAL_HANDSHAKE_CHANGED;
+			return RETRY_CONNECT;
 		}
 	}
 
-	return -1000;
-}
-
-/* drbd_sync_handshake() returns the new conn state on success, or
-   CONN_MASK (-1) on failure.
- */
-static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device,
-					   enum drbd_role peer_role,
-					   enum drbd_disk_state peer_disk) __must_hold(local)
-{
-	struct drbd_device *device = peer_device->device;
-	enum drbd_conns rv = C_MASK;
-	enum drbd_disk_state mydisk;
-	struct net_conf *nc;
-	int hg, rule_nr, rr_conflict, tentative, always_asbp;
+	self = resolved_uuid;
+	peer = peer_device->current_uuid & ~UUID_PRIMARY;
 
-	mydisk = device->state.disk;
-	if (mydisk == D_NEGOTIATING)
-		mydisk = device->new_state_tmp.disk;
+	/* Before DRBD 8.0.2 (from 2007), the uuid on sync targets was set to
+	 * zero during resyncs for no good reason. */
+	if (self == 0)
+		self = UUID_JUST_CREATED;
+	if (peer == 0)
+		peer = UUID_JUST_CREATED;
 
-	drbd_info(device, "drbd_sync_handshake:\n");
+	*rule = RULE_JUST_CREATED_BOTH;
+	if (self == UUID_JUST_CREATED && peer == UUID_JUST_CREATED)
+		return NO_SYNC;
 
-	spin_lock_irq(&device->ldev->md.uuid_lock);
-	drbd_uuid_dump(device, "self", device->ldev->md.uuid, device->comm_bm_set, 0);
-	drbd_uuid_dump(device, "peer", device->p_uuid,
-		       device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]);
+	*rule = RULE_JUST_CREATED_SELF;
+	if (self == UUID_JUST_CREATED)
+		return SYNC_TARGET_SET_BITMAP;
 
-	hg = drbd_uuid_compare(peer_device, peer_role, &rule_nr);
-	spin_unlock_irq(&device->ldev->md.uuid_lock);
+	*rule = RULE_JUST_CREATED_PEER;
+	if (peer == UUID_JUST_CREATED)
+		return SYNC_SOURCE_SET_BITMAP;
 
-	drbd_info(device, "uuid_compare()=%d by rule %d\n", hg, rule_nr);
+	if (self == peer) {
+		struct net_conf *nc;
+		int wire_protocol;
 
-	if (hg == -1000) {
-		drbd_alert(device, "Unrelated data, aborting!\n");
-		return C_MASK;
-	}
-	if (hg < -0x10000) {
-		int proto, fflags;
-		hg = -hg;
-		proto = hg & 0xff;
-		fflags = (hg >> 8) & 0xff;
-		drbd_alert(device, "To resolve this both sides have to support at least protocol %d and feature flags 0x%x\n",
-					proto, fflags);
-		return C_MASK;
-	}
-	if (hg < -1000) {
-		drbd_alert(device, "To resolve this both sides have to support at least protocol %d\n", -hg - 1000);
-		return C_MASK;
-	}
+		rcu_read_lock();
+		nc = rcu_dereference(connection->transport.net_conf);
+		wire_protocol = nc->wire_protocol;
+		rcu_read_unlock();
 
-	if    ((mydisk == D_INCONSISTENT && peer_disk > D_INCONSISTENT) ||
-	    (peer_disk == D_INCONSISTENT && mydisk    > D_INCONSISTENT)) {
-		int f = (hg == -100) || abs(hg) == 2;
-		hg = mydisk > D_INCONSISTENT ? 1 : -1;
-		if (f)
-			hg = hg*2;
-		drbd_info(device, "Becoming sync %s due to disk states.\n",
-		     hg > 0 ? "source" : "target");
-	}
+		if (connection->agreed_pro_version < 110) {
+			enum sync_strategy rv = uuid_fixup_resync_end(peer_device, rule);
+			if (rv != UNDETERMINED)
+				return rv;
+		}
 
-	if (abs(hg) == 100)
-		drbd_khelper(device, "initial-split-brain");
+		if (test_bit(RS_SOURCE_MISSED_END, &peer_device->flags)) {
+			*rule = RULE_SYNC_SOURCE_MISSED_FINISH;
+			return SYNC_SOURCE_USE_BITMAP;
+		}
+		if (test_bit(RS_PEER_MISSED_END, &peer_device->flags)) {
+			*rule = RULE_SYNC_TARGET_PEER_MISSED_FINISH;
+			return SYNC_TARGET_USE_BITMAP;
+		}
 
-	rcu_read_lock();
-	nc = rcu_dereference(peer_device->connection->net_conf);
-	always_asbp = nc->always_asbp;
-	rr_conflict = nc->rr_conflict;
-	tentative = nc->tentative;
-	rcu_read_unlock();
+		if (connection->agreed_pro_version >= 120) {
+			*rule = RULE_RECONNECTED;
+			if (peer_device->uuid_flags & UUID_FLAG_RECONNECT &&
+			    local_uuid_flags & UUID_FLAG_RECONNECT)
+				return NO_SYNC;
+		}
 
-	if (hg == 100 || (hg == -100 && always_asbp)) {
-		int pcount = (device->state.role == R_PRIMARY)
-			   + (peer_role == R_PRIMARY);
-		int forced = (hg == -100);
+		if (connection->agreed_pro_version >= 121 &&
+		    (wire_protocol == DRBD_PROT_A || wire_protocol == DRBD_PROT_B)) {
+			*rule = RULE_CRASHED_PRIMARY;
+			if (local_uuid_flags & UUID_FLAG_CRASHED_PRIMARY &&
+			    !(peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY))
+				return SYNC_SOURCE_USE_BITMAP;
 
-		switch (pcount) {
-		case 0:
-			hg = drbd_asb_recover_0p(peer_device);
-			break;
-		case 1:
-			hg = drbd_asb_recover_1p(peer_device);
-			break;
-		case 2:
-			hg = drbd_asb_recover_2p(peer_device);
-			break;
+			if (peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY &&
+			    !(local_uuid_flags & UUID_FLAG_CRASHED_PRIMARY))
+				return SYNC_TARGET_USE_BITMAP;
 		}
-		if (abs(hg) < 100) {
-			drbd_warn(device, "Split-Brain detected, %d primaries, "
-			     "automatically solved. Sync from %s node\n",
-			     pcount, (hg < 0) ? "peer" : "this");
-			if (forced) {
-				drbd_warn(device, "Doing a full sync, since"
-				     " UUIDs where ambiguous.\n");
-				hg = hg*2;
-			}
+
+		*rule = RULE_LOST_QUORUM;
+		if (peer_device->uuid_flags & UUID_FLAG_PRIMARY_LOST_QUORUM &&
+		    !test_bit(PRIMARY_LOST_QUORUM, &device->flags))
+			return SYNC_TARGET_IF_BOTH_FAILED;
+
+		if (!(peer_device->uuid_flags & UUID_FLAG_PRIMARY_LOST_QUORUM) &&
+		    test_bit(PRIMARY_LOST_QUORUM, &device->flags))
+			return SYNC_SOURCE_IF_BOTH_FAILED;
+
+		if (peer_device->uuid_flags & UUID_FLAG_PRIMARY_LOST_QUORUM &&
+		    test_bit(PRIMARY_LOST_QUORUM, &device->flags))
+			return test_bit(RESOLVE_CONFLICTS, &connection->transport.flags) ?
+				SYNC_SOURCE_IF_BOTH_FAILED :
+				SYNC_TARGET_IF_BOTH_FAILED;
+
+		if (connection->agreed_pro_version < 120) {
+			*rule = RULE_RECONNECTED;
+			if (peer_device->uuid_flags & UUID_FLAG_RECONNECT &&
+			    local_uuid_flags & UUID_FLAG_RECONNECT)
+				return NO_SYNC;
 		}
+
+		/* Peer crashed as primary, I survived, resync from me */
+		if (peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY &&
+		    local_uuid_flags & UUID_FLAG_RECONNECT)
+			return SYNC_SOURCE_IF_BOTH_FAILED;
+
+		/* I am a crashed primary, peer survived, resync to me */
+		if (local_uuid_flags & UUID_FLAG_CRASHED_PRIMARY &&
+		    peer_device->uuid_flags & UUID_FLAG_RECONNECT)
+			return SYNC_TARGET_IF_BOTH_FAILED;
+
+		/* One of us had a connection to the other node before.
+		   i.e. this is not a common power failure. */
+		if (peer_device->uuid_flags & UUID_FLAG_RECONNECT ||
+		    local_uuid_flags & UUID_FLAG_RECONNECT)
+			return NO_SYNC;
+
+		/* Common power [off|failure]? */
+		*rule = RULE_BOTH_OFF;
+		if (local_uuid_flags & UUID_FLAG_CRASHED_PRIMARY) {
+			if ((peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY) &&
+			    test_bit(RESOLVE_CONFLICTS, &connection->transport.flags))
+				return SYNC_TARGET_IF_BOTH_FAILED;
+			return SYNC_SOURCE_IF_BOTH_FAILED;
+		} else if (peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY)
+				return SYNC_TARGET_IF_BOTH_FAILED;
+		else
+			return NO_SYNC;
 	}
 
-	if (hg == -100) {
-		if (test_bit(DISCARD_MY_DATA, &device->flags) && !(device->p_uuid[UI_FLAGS]&1))
-			hg = -1;
-		if (!test_bit(DISCARD_MY_DATA, &device->flags) && (device->p_uuid[UI_FLAGS]&1))
-			hg = 1;
+	*rule = RULE_BITMAP_PEER;
+	peer = peer_device->bitmap_uuids[node_id] & ~UUID_PRIMARY;
+	if (self == peer)
+		return SYNC_TARGET_USE_BITMAP;
 
-		if (abs(hg) < 100)
-			drbd_warn(device, "Split-Brain detected, manually solved. "
-			     "Sync from %s node\n",
-			     (hg < 0) ? "peer" : "this");
+	*rule = RULE_BITMAP_PEER_OTHER;
+	i = drbd_find_peer_bitmap_by_uuid(peer_device, self);
+	if (i != -1) {
+		*peer_node_id = i;
+		return SYNC_TARGET_CLEAR_BITMAP;
 	}
 
-	if (hg == -100) {
-		/* FIXME this log message is not correct if we end up here
-		 * after an attempted attach on a diskless node.
-		 * We just refuse to attach -- well, we drop the "connection"
-		 * to that disk, in a way... */
-		drbd_alert(device, "Split-Brain detected but unresolved, dropping connection!\n");
-		drbd_khelper(device, "split-brain");
-		return C_MASK;
+	if (connection->agreed_pro_version < 110) {
+		enum sync_strategy rv = uuid_fixup_resync_start1(peer_device, rule);
+		if (rv != UNDETERMINED)
+			return rv;
 	}
 
-	if (hg > 0 && mydisk <= D_INCONSISTENT) {
-		drbd_err(device, "I shall become SyncSource, but I am inconsistent!\n");
-		return C_MASK;
-	}
+	*rule = RULE_BITMAP_SELF;
+	self = bitmap_uuid & ~UUID_PRIMARY;
+	peer = peer_device->current_uuid & ~UUID_PRIMARY;
+	if (self == peer)
+		return SYNC_SOURCE_USE_BITMAP;
 
-	if (hg < 0 && /* by intention we do not use mydisk here. */
-	    device->state.role == R_PRIMARY && device->state.disk >= D_CONSISTENT) {
-		switch (rr_conflict) {
-		case ASB_CALL_HELPER:
-			drbd_khelper(device, "pri-lost");
-			fallthrough;
-		case ASB_DISCONNECT:
-			drbd_err(device, "I shall become SyncTarget, but I am primary!\n");
-			return C_MASK;
-		case ASB_VIOLENTLY:
-			drbd_warn(device, "Becoming SyncTarget, violating the stable-data"
-			     "assumption\n");
-		}
+	*rule = RULE_BITMAP_SELF_OTHER;
+	i = drbd_find_bitmap_by_uuid(peer_device, peer);
+	if (i != -1) {
+		*peer_node_id = i;
+		return SYNC_SOURCE_COPY_BITMAP;
 	}
 
-	if (tentative || test_bit(CONN_DRY_RUN, &peer_device->connection->flags)) {
-		if (hg == 0)
-			drbd_info(device, "dry-run connect: No resync, would become Connected immediately.\n");
-		else
-			drbd_info(device, "dry-run connect: Would become %s, doing a %s resync.",
-				 drbd_conn_str(hg > 0 ? C_SYNC_SOURCE : C_SYNC_TARGET),
-				 abs(hg) >= 2 ? "full" : "bit-map based");
-		return C_MASK;
+	self = resolved_uuid;
+	my_current_in_peers_history = uuid_in_peer_history(peer_device, self);
+
+	if (connection->agreed_pro_version < 110) {
+		enum sync_strategy rv = uuid_fixup_resync_start2(peer_device, rule);
+		if (rv != UNDETERMINED)
+			return rv;
 	}
 
-	if (abs(hg) >= 2) {
-		drbd_info(device, "Writing the whole bitmap, full sync required after drbd_sync_handshake.\n");
-		if (drbd_bitmap_io(device, &drbd_bmio_set_n_write, "set_n_write from sync_handshake",
-					BM_LOCKED_SET_ALLOWED, NULL))
-			return C_MASK;
+	peer = peer_device->current_uuid & ~UUID_PRIMARY;
+	peers_current_in_my_history = uuid_in_my_history(device, peer);
+
+	if (my_current_in_peers_history && !peers_current_in_my_history) {
+		*rule = RULE_HISTORY_PEER;
+		return SYNC_TARGET_SET_BITMAP;
+	}
+	if (!my_current_in_peers_history && peers_current_in_my_history) {
+		*rule = RULE_HISTORY_SELF;
+		return SYNC_SOURCE_SET_BITMAP;
 	}
 
-	if (hg > 0) { /* become sync source. */
-		rv = C_WF_BITMAP_S;
-	} else if (hg < 0) { /* become sync target */
-		rv = C_WF_BITMAP_T;
-	} else {
-		rv = C_CONNECTED;
-		if (drbd_bm_total_weight(device)) {
-			drbd_info(device, "No resync, but %lu bits in bitmap!\n",
-			     drbd_bm_total_weight(device));
+	*rule = RULE_BITMAP_BOTH;
+	self = bitmap_uuid & ~UUID_PRIMARY;
+	peer = peer_device->bitmap_uuids[node_id] & ~UUID_PRIMARY;
+	if (self == peer && self != ((u64)0))
+		return SPLIT_BRAIN_AUTO_RECOVER;
+
+	*rule = RULE_HISTORY_BOTH;
+	for (i = 0; i < HISTORY_UUIDS; i++) {
+		self = drbd_history_uuid(device, i) & ~UUID_PRIMARY;
+		/* Don't conclude to have "data divergence" from a "common ancestor"
+		 * if that common ancestor is just a not used yet slot in the history,
+		 * which is still initialized to zero on both peers. */
+		if (self == 0)
+			break;
+		for (j = 0; j < ARRAY_SIZE(peer_device->history_uuids); j++) {
+			peer = peer_device->history_uuids[j] & ~UUID_PRIMARY;
+			if (peer == 0)
+				break;
+			if (self == peer)
+				return SPLIT_BRAIN_DISCONNECT;
 		}
 	}
 
-	return rv;
+	return UNRELATED_DATA;
 }
 
-static enum drbd_after_sb_p convert_after_sb(enum drbd_after_sb_p peer)
+static void log_handshake(struct drbd_peer_device *peer_device)
 {
-	/* ASB_DISCARD_REMOTE - ASB_DISCARD_LOCAL is valid */
-	if (peer == ASB_DISCARD_REMOTE)
-		return ASB_DISCARD_LOCAL;
-
-	/* any other things with ASB_DISCARD_REMOTE or ASB_DISCARD_LOCAL are invalid */
-	if (peer == ASB_DISCARD_LOCAL)
-		return ASB_DISCARD_REMOTE;
+	u64 uuid_flags = drbd_collect_local_uuid_flags(peer_device, NULL);
 
-	/* everything else is valid if they are equal on both sides. */
-	return peer;
+	drbd_info(peer_device, "drbd_sync_handshake:\n");
+	drbd_uuid_dump_self(peer_device, peer_device->comm_bm_set, uuid_flags);
+	drbd_uuid_dump_peer(peer_device, peer_device->dirty_bits, peer_device->uuid_flags);
 }
 
-static int receive_protocol(struct drbd_connection *connection, struct packet_info *pi)
+static enum sync_strategy drbd_handshake(struct drbd_peer_device *peer_device,
+			  enum sync_rule *rule,
+			  int *peer_node_id,
+			  bool always_verbose) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	enum sync_strategy strategy;
+
+	spin_lock_irq(&device->ldev->md.uuid_lock);
+	if (always_verbose)
+		log_handshake(peer_device);
+
+	strategy = drbd_uuid_compare(peer_device, rule, peer_node_id);
+	if (strategy != NO_SYNC && !always_verbose)
+		log_handshake(peer_device);
+	spin_unlock_irq(&device->ldev->md.uuid_lock);
+
+	if (strategy != NO_SYNC || always_verbose)
+		drbd_info(peer_device, "uuid_compare()=%s by rule=%s\n",
+				strategy_descriptor(strategy).name,
+				drbd_sync_rule_str(*rule));
+
+	return strategy;
+}
+
+static bool is_resync_running(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+		if (repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+static int bitmap_mod_after_handshake(struct drbd_peer_device *peer_device, enum sync_strategy strategy, int peer_node_id)
+{
+	struct drbd_device *device = peer_device->device;
+
+	if (strategy == SYNC_SOURCE_COPY_BITMAP) {
+		int from = device->ldev->md.peers[peer_node_id].bitmap_index;
+
+		if (from == -1)
+			from = drbd_unallocated_index(device->ldev);
+
+		if (peer_device->bitmap_index == -1)
+			return 0;
+
+		if (from == -1)
+			drbd_info(peer_device,
+				  "Setting all bitmap bits, day0 bm not available node_id=%d\n",
+				  peer_node_id);
+		else
+			drbd_info(peer_device,
+				  "Copying bitmap of peer node_id=%d (bitmap_index=%d)\n",
+				  peer_node_id, from);
+
+		drbd_suspend_io(device, WRITE_ONLY);
+		drbd_bm_slot_lock(peer_device, "copy_slot/set_many sync_handshake", BM_LOCK_BULK);
+		if (from == -1)
+			drbd_bm_set_many_bits(peer_device, 0, -1UL);
+		else
+			drbd_bm_copy_slot(device, from, peer_device->bitmap_index);
+		drbd_bm_write(device, NULL);
+		drbd_bm_slot_unlock(peer_device);
+		drbd_resume_io(device);
+	} else if (strategy == SYNC_TARGET_CLEAR_BITMAP) {
+		drbd_info(peer_device, "Resync source provides bitmap (node_id=%d)\n", peer_node_id);
+		drbd_suspend_io(device, WRITE_ONLY);
+		drbd_bm_slot_lock(peer_device, "bm_clear_many_bits sync_handshake", BM_LOCK_BULK);
+		drbd_bm_clear_many_bits(peer_device, 0, -1UL);
+		drbd_bm_write(device, NULL);
+		drbd_bm_slot_unlock(peer_device);
+		drbd_resume_io(device);
+	} else if (strategy == SYNC_SOURCE_SET_BITMAP || strategy == SYNC_TARGET_SET_BITMAP) {
+		int (*io_func)(struct drbd_device *, struct drbd_peer_device *);
+		int err;
+
+		if (strategy == SYNC_TARGET_SET_BITMAP &&
+		    drbd_current_uuid(device) == UUID_JUST_CREATED &&
+		    is_resync_running(device))
+			return 0;
+
+		if (drbd_current_uuid(device) == UUID_JUST_CREATED) {
+			drbd_info(peer_device, "Setting and writing the whole bitmap, fresh node\n");
+			io_func = &drbd_bmio_set_allocated_n_write;
+		} else {
+			drbd_info(peer_device, "Setting and writing one bitmap slot, after drbd_sync_handshake\n");
+			io_func = &drbd_bmio_set_n_write;
+		}
+		err = drbd_bitmap_io(device, io_func, "set_n_write sync_handshake",
+				     BM_LOCK_CLEAR | BM_LOCK_BULK, peer_device);
+		if (err)
+			return err;
+
+		if (drbd_current_uuid(device) != UUID_JUST_CREATED &&
+		    peer_device->current_uuid != UUID_JUST_CREATED &&
+		    strategy == SYNC_SOURCE_SET_BITMAP) {
+			/*
+			 * We have just written the bitmap slot. Update the
+			 * bitmap UUID so that the resync does not start from
+			 * the beginning again if we disconnect and reconnect.
+			 *
+			 * Initial resync continuation is handled in
+			 * drbd_start_resync() at comment:
+			 * prepare to continue an interrupted initial resync later
+			 */
+			drbd_uuid_set_bitmap(peer_device, peer_device->current_uuid);
+			drbd_print_uuids(peer_device, "updated bitmap UUID");
+			drbd_md_sync(device);
+		}
+	}
+	return 0;
+}
+
+static enum drbd_repl_state strategy_to_repl_state(struct drbd_peer_device *peer_device,
+						   enum drbd_role peer_role,
+						   enum sync_strategy strategy)
+{
+	enum drbd_role role = peer_device->device->resource->role[NOW];
+	enum drbd_repl_state rv;
+
+	if (strategy == SYNC_SOURCE_IF_BOTH_FAILED || strategy == SYNC_TARGET_IF_BOTH_FAILED) {
+		if (role == R_PRIMARY || peer_role == R_PRIMARY) {
+			/* We have at least one primary, follow that with the resync decision */
+			rv = peer_role == R_SECONDARY ? L_WF_BITMAP_S :
+				role == R_SECONDARY ? L_WF_BITMAP_T :
+				L_ESTABLISHED;
+			return rv;
+		}
+		/* No current primary. Handle it as a common power failure, consider the
+		   roles at crash time */
+	}
+
+	if (strategy_descriptor(strategy).is_sync_source) {
+		rv = L_WF_BITMAP_S;
+	} else if (strategy_descriptor(strategy).is_sync_target) {
+		rv = L_WF_BITMAP_T;
+	} else {
+		rv = L_ESTABLISHED;
+	}
+
+	return rv;
+}
+
+static enum sync_strategy drbd_disk_states_source_strategy(
+		struct drbd_peer_device *peer_device,
+		int *peer_node_id)
+{
+	const int node_id = peer_device->device->resource->res_opts.node_id;
+	u64 bitmap_uuid;
+	int i = -1;
+
+	if (!(peer_device->uuid_flags & UUID_FLAG_SYNC_TARGET))
+		return SYNC_SOURCE_USE_BITMAP;
+
+	/* A resync with identical current-UUIDs -> USE_BITMAP */
+	bitmap_uuid = peer_device->bitmap_uuids[node_id];
+	if (bitmap_uuid == peer_device->current_uuid &&
+	    bitmap_uuid == drbd_current_uuid(peer_device->device))
+		return SYNC_SOURCE_USE_BITMAP;
+
+	/* When the peer is already a sync target, we actually see its
+	 * current UUID in the bitmap UUID slot towards us. We may need
+	 * to pick a different bitmap as a result. */
+	if (bitmap_uuid)
+		i = drbd_find_bitmap_by_uuid(peer_device, bitmap_uuid);
+
+	if (i == -1)
+		return SYNC_SOURCE_SET_BITMAP;
+
+	if (i == peer_device->node_id)
+		return SYNC_SOURCE_USE_BITMAP;
+
+	*peer_node_id = i;
+	return SYNC_SOURCE_COPY_BITMAP;
+}
+
+static enum sync_strategy drbd_disk_states_target_strategy(
+		struct drbd_peer_device *peer_device,
+		int *peer_node_id)
+{
+	const int node_id = peer_device->device->resource->res_opts.node_id;
+	u64 bitmap_uuid;
+	int i;
+
+	if (!(peer_device->comm_uuid_flags & UUID_FLAG_SYNC_TARGET))
+		return SYNC_TARGET_USE_BITMAP;
+
+	bitmap_uuid = drbd_bitmap_uuid(peer_device);
+	if (bitmap_uuid == peer_device->current_uuid &&
+	    bitmap_uuid == drbd_current_uuid(peer_device->device))
+		return SYNC_TARGET_USE_BITMAP;
+
+	/* When we are already a sync target, we need to choose our
+	 * strategy to mirror the peer's choice (see
+	 * drbd_disk_states_source_strategy). */
+	i = drbd_find_peer_bitmap_by_uuid(peer_device, bitmap_uuid);
+
+	if (i == -1)
+		return SYNC_TARGET_SET_BITMAP;
+
+	if (i == node_id)
+		return SYNC_TARGET_USE_BITMAP;
+
+	*peer_node_id = i;
+	return SYNC_TARGET_CLEAR_BITMAP;
+}
+
+static void disk_states_to_strategy(struct drbd_peer_device *peer_device,
+				    enum drbd_disk_state peer_disk_state,
+				    enum sync_strategy *strategy, enum sync_rule rule,
+				    int *peer_node_id)
+{
+	enum drbd_disk_state disk_state = peer_device->comm_state.disk;
+	struct drbd_device *device = peer_device->device;
+	bool decide_based_on_dstates = false;
+	bool prefer_local, either_inconsistent;
+
+	if (disk_state == D_NEGOTIATING)
+		disk_state = disk_state_from_md(device);
+
+	either_inconsistent =
+		(disk_state == D_INCONSISTENT && peer_disk_state > D_INCONSISTENT) ||
+		(peer_disk_state == D_INCONSISTENT && disk_state > D_INCONSISTENT);
+
+	if (peer_device->connection->agreed_pro_version >= 119) {
+		bool dstates_want_resync =
+			disk_state != peer_disk_state && disk_state >= D_INCONSISTENT &&
+			peer_disk_state >= D_INCONSISTENT && peer_disk_state != D_UNKNOWN;
+		bool resync_direction_arbitrary =
+			*strategy == SYNC_TARGET_IF_BOTH_FAILED ||
+			*strategy == SYNC_SOURCE_IF_BOTH_FAILED;
+
+		decide_based_on_dstates =
+			dstates_want_resync &&
+			(((rule == RULE_RECONNECTED || rule == RULE_LOST_QUORUM || rule == RULE_BOTH_OFF) &&
+			  resync_direction_arbitrary) ||
+			 (*strategy == NO_SYNC && either_inconsistent));
+
+		prefer_local = disk_state > peer_disk_state;
+		/* RULE_BOTH_OFF means that the current UUIDs are equal. The decision
+		   was found by looking at the crashed_primary bits.
+		   The current disk states might give a better basis for decision-making! */
+
+		/* RULE_LOST_QUORUM means that the current UUIDs are equal. The resync direction
+		   was found by looking if a node lost quorum while being primary */
+	} else {
+		decide_based_on_dstates =
+			(rule == RULE_BOTH_OFF || *strategy == NO_SYNC) && either_inconsistent;
+
+		prefer_local = disk_state > D_INCONSISTENT;
+	}
+
+	if (decide_based_on_dstates) {
+		*strategy = prefer_local ?
+			drbd_disk_states_source_strategy(peer_device, peer_node_id) :
+			drbd_disk_states_target_strategy(peer_device, peer_node_id);
+		drbd_info(peer_device, "strategy = %s due to disk states. (%s/%s)\n",
+			  strategy_descriptor(*strategy).name,
+			  drbd_disk_str(disk_state), drbd_disk_str(peer_disk_state));
+	}
+}
+
+static enum sync_strategy drbd_attach_handshake(struct drbd_peer_device *peer_device,
+						  enum drbd_disk_state peer_disk_state) __must_hold(local)
+{
+	enum sync_strategy strategy;
+	enum sync_rule rule;
+	int peer_node_id, err;
+
+	strategy = drbd_handshake(peer_device, &rule, &peer_node_id, true);
+
+	if (!is_strategy_determined(strategy))
+		return strategy;
+
+	disk_states_to_strategy(peer_device, peer_disk_state, &strategy, rule, &peer_node_id);
+	err = bitmap_mod_after_handshake(peer_device, strategy, peer_node_id);
+	if (err)
+		return RETRY_CONNECT;
+
+	return strategy;
+}
+
+static enum sync_strategy discard_my_data_to_strategy(struct drbd_peer_device *peer_device)
+{
+	enum sync_strategy strategy = UNDETERMINED;
+
+	if (test_bit(DISCARD_MY_DATA, &peer_device->flags) &&
+	    !(peer_device->uuid_flags & UUID_FLAG_DISCARD_MY_DATA))
+		strategy = SYNC_TARGET_USE_BITMAP;
+
+	if (!test_bit(DISCARD_MY_DATA, &peer_device->flags) &&
+	    (peer_device->uuid_flags & UUID_FLAG_DISCARD_MY_DATA))
+		strategy = SYNC_SOURCE_USE_BITMAP;
+
+	return strategy;
+}
+
+/* drbd_sync_handshake() returns the new replication state on success, and -1
+ * on failure.
+ */
+static enum sync_strategy drbd_sync_handshake(struct drbd_peer_device *peer_device,
+					      union drbd_state peer_state) __must_hold(local)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct net_conf *nc;
+	enum sync_strategy strategy;
+	enum sync_rule rule;
+	int rr_conflict, always_asbp, peer_node_id = 0, err;
+	enum drbd_role peer_role = peer_state.role;
+	enum drbd_disk_state peer_disk_state = peer_state.disk;
+	int required_protocol;
+	enum sync_strategy strategy_from_user = discard_my_data_to_strategy(peer_device);
+	bool need_full_sync_after_split_brain;
+
+	strategy = drbd_handshake(peer_device, &rule, &peer_node_id, true);
+
+	if (strategy == RETRY_CONNECT)
+		return strategy;
+
+	if (strategy == UNRELATED_DATA) {
+		drbd_alert(peer_device, "Unrelated data, aborting!\n");
+		return strategy;
+	}
+	required_protocol = strategy_descriptor(strategy).required_protocol;
+	if (required_protocol) {
+		drbd_alert(peer_device, "To resolve this both sides have to support at least protocol %d\n", required_protocol);
+		return strategy;
+	}
+
+	disk_states_to_strategy(peer_device, peer_disk_state, &strategy, rule, &peer_node_id);
+
+	if (strategy == SPLIT_BRAIN_AUTO_RECOVER && (!drbd_device_stable(device, NULL) || !(peer_device->uuid_flags & UUID_FLAG_STABLE))) {
+		drbd_warn(peer_device, "Ignore Split-Brain, for now, at least one side unstable\n");
+		strategy = NO_SYNC;
+	}
+
+	if (strategy_descriptor(strategy).is_split_brain)
+		drbd_maybe_khelper(device, connection, "initial-split-brain");
+
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	always_asbp = nc->always_asbp;
+	rr_conflict = nc->rr_conflict;
+	rcu_read_unlock();
+
+	/* Evaluate the original strategy,
+	 * before it is re-mapped by additional configuration below.
+	 */
+	need_full_sync_after_split_brain = (strategy == SPLIT_BRAIN_DISCONNECT);
+
+	if (strategy == SPLIT_BRAIN_AUTO_RECOVER || (strategy == SPLIT_BRAIN_DISCONNECT && always_asbp)) {
+		int pcount = (device->resource->role[NOW] == R_PRIMARY)
+			   + (peer_role == R_PRIMARY);
+
+		if (device->resource->res_opts.quorum != QOU_OFF &&
+		    connection->agreed_pro_version >= 113) {
+			if (device->have_quorum[NOW] && !peer_state.quorum)
+				strategy = SYNC_SOURCE_USE_BITMAP;
+			else if (!device->have_quorum[NOW] && peer_state.quorum)
+				strategy = SYNC_TARGET_USE_BITMAP;
+		}
+		if (strategy_descriptor(strategy).is_split_brain) {
+			switch (pcount) {
+			case 0:
+				strategy = drbd_asb_recover_0p(peer_device);
+				break;
+			case 1:
+				strategy = drbd_asb_recover_1p(peer_device);
+				break;
+			case 2:
+				strategy = drbd_asb_recover_2p(peer_device);
+				break;
+			}
+		}
+		if (!strategy_descriptor(strategy).is_split_brain) {
+			drbd_warn(peer_device, "Split-Brain detected, %d primaries, "
+			     "automatically solved. Sync from %s node\n",
+			     pcount, strategy_descriptor(strategy).is_sync_target ? "peer" : "this");
+			if (need_full_sync_after_split_brain) {
+				if (!strategy_descriptor(strategy).full_sync_equivalent) {
+					drbd_alert(peer_device, "Want full sync but cannot decide direction, dropping connection!\n");
+					return SPLIT_BRAIN_DISCONNECT;
+				}
+				drbd_warn(peer_device, "Doing a full sync, since"
+				     " UUIDs where ambiguous.\n");
+				strategy = strategy_descriptor(strategy).full_sync_equivalent;
+			}
+		}
+	}
+
+	if (strategy == SPLIT_BRAIN_DISCONNECT && strategy_from_user != UNDETERMINED) {
+		/* strategy_from_user via "--discard-my-data" is either
+		 * SYNC_TARGET_USE_BITMAP or SYNC_SOURCE_USE_BITMAP.
+		 * But here we do no longer have a relevant bitmap anymore.
+		 * Map to their "full sync equivalent".
+		 */
+		if (need_full_sync_after_split_brain)
+			strategy = strategy_descriptor(strategy_from_user).full_sync_equivalent;
+		else
+			strategy = strategy_from_user;
+		drbd_warn(peer_device, "Split-Brain detected, manually solved. %s from %s node\n",
+			  need_full_sync_after_split_brain ? "Full sync" : "Sync",
+			  strategy_descriptor(strategy).is_sync_target ? "peer" : "this");
+	}
+
+	if (strategy_descriptor(strategy).is_split_brain) {
+		drbd_alert(peer_device, "Split-Brain detected but unresolved, dropping connection!\n");
+		drbd_maybe_khelper(device, connection, "split-brain");
+		return strategy;
+	}
+
+	if (!is_strategy_determined(strategy)) {
+		drbd_alert(peer_device, "Failed to fully determine sync strategy, dropping connection!\n");
+		return strategy;
+	}
+
+	if (connection->agreed_pro_version >= 121 && strategy != NO_SYNC &&
+	    strategy_from_user != UNDETERMINED &&
+	    strategy_descriptor(strategy).is_sync_source != strategy_descriptor(strategy_from_user).is_sync_source) {
+		if (strategy_descriptor(strategy).reverse != UNDETERMINED) {
+			enum sync_strategy reversed = strategy_descriptor(strategy).reverse;
+			enum drbd_disk_state resync_source_disk_state =
+				strategy_descriptor(reversed).is_sync_source ? device->disk_state[NOW] : peer_disk_state;
+			if (resync_source_disk_state > D_INCONSISTENT) {
+				strategy = reversed;
+				drbd_warn(peer_device, "Resync direction reversed by --discard-my-data. Reverting to older data!\n");
+			} else {
+				drbd_warn(peer_device, "Ignoring --discard-my-data\n");
+			}
+		} else {
+			drbd_warn(peer_device, "Can not reverse resync direction (requested via --discard-my-data)\n");
+		}
+	}
+
+	if (strategy_descriptor(strategy).is_sync_target &&
+	    strategy != SYNC_TARGET_IF_BOTH_FAILED &&
+	    device->resource->role[NOW] == R_PRIMARY && device->disk_state[NOW] >= D_CONSISTENT &&
+	    (peer_device->comm_bm_set > 0 || peer_device->dirty_bits > 0)) {
+		switch (rr_conflict) {
+		case ASB_CALL_HELPER:
+			drbd_maybe_khelper(device, connection, "pri-lost");
+			fallthrough;
+		case ASB_DISCONNECT:
+		case ASB_RETRY_CONNECT:
+			drbd_err(peer_device, "I shall become SyncTarget, but I am primary!\n");
+			strategy = rr_conflict == ASB_RETRY_CONNECT ?
+				SYNC_TARGET_PRIMARY_RECONNECT : SYNC_TARGET_PRIMARY_DISCONNECT;
+			break;
+		case ASB_VIOLENTLY:
+			drbd_warn(peer_device, "Becoming SyncTarget, violating the stable-data"
+			     "assumption\n");
+			break;
+		case ASB_AUTO_DISCARD:
+			if (strategy == SYNC_TARGET_USE_BITMAP && rule == RULE_CRASHED_PRIMARY) {
+				drbd_warn(peer_device, "reversing resync by auto-discard\n");
+				strategy = SYNC_SOURCE_USE_BITMAP;
+			}
+		}
+	}
+	if (strategy == SYNC_SOURCE_USE_BITMAP && rule == RULE_CRASHED_PRIMARY &&
+	    peer_role == R_PRIMARY && peer_disk_state >= D_CONSISTENT &&
+	    rr_conflict == ASB_AUTO_DISCARD) {
+		drbd_warn(peer_device, "reversing resync by auto-discard\n");
+		strategy = SYNC_TARGET_USE_BITMAP;
+	}
+
+	if (test_bit(CONN_DRY_RUN, &connection->flags)) {
+		if (strategy == NO_SYNC)
+			drbd_info(peer_device, "dry-run connect: No resync, would become Connected immediately.\n");
+		else
+			drbd_info(peer_device, "dry-run connect: Would become %s, doing a %s resync.",
+				 drbd_repl_str(strategy_descriptor(strategy).is_sync_target ? L_SYNC_TARGET : L_SYNC_SOURCE),
+				 strategy_descriptor(strategy).name);
+		return -2;
+	}
+
+	err = bitmap_mod_after_handshake(peer_device, strategy, peer_node_id);
+	if (err)
+		return RETRY_CONNECT;
+
+	return strategy;
+}
+
+static enum drbd_after_sb_p convert_after_sb(enum drbd_after_sb_p peer)
+{
+	/* ASB_DISCARD_REMOTE - ASB_DISCARD_LOCAL is valid */
+	if (peer == ASB_DISCARD_REMOTE)
+		return ASB_DISCARD_LOCAL;
+
+	/* any other things with ASB_DISCARD_REMOTE or ASB_DISCARD_LOCAL are invalid */
+	if (peer == ASB_DISCARD_LOCAL)
+		return ASB_DISCARD_REMOTE;
+
+	/* everything else is valid if they are equal on both sides. */
+	return peer;
+}
+
+static int receive_protocol(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct p_protocol *p = pi->data;
 	enum drbd_after_sb_p p_after_sb_0p, p_after_sb_1p, p_after_sb_2p;
@@ -3465,20 +5407,18 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 
 		if (pi->size > sizeof(integrity_alg))
 			return -EIO;
-		err = drbd_recv_all(connection, integrity_alg, pi->size);
+		err = drbd_recv_into(connection, integrity_alg, pi->size);
 		if (err)
 			return err;
 		integrity_alg[SHARED_SECRET_MAX - 1] = 0;
 	}
 
 	if (pi->cmd != P_PROTOCOL_UPDATE) {
-		clear_bit(CONN_DRY_RUN, &connection->flags);
-
 		if (cf & CF_DRY_RUN)
 			set_bit(CONN_DRY_RUN, &connection->flags);
 
 		rcu_read_lock();
-		nc = rcu_dereference(connection->net_conf);
+		nc = rcu_dereference(connection->transport.net_conf);
 
 		if (p_proto != nc->wire_protocol) {
 			drbd_err(connection, "incompatible %s settings\n", "protocol");
@@ -3500,7 +5440,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 			goto disconnect_rcu_unlock;
 		}
 
-		if (p_discard_my_data && nc->discard_my_data) {
+		if (p_discard_my_data && test_bit(CONN_DISCARD_MY_DATA, &connection->flags)) {
 			drbd_err(connection, "incompatible %s settings\n", "discard-my-data");
 			goto disconnect_rcu_unlock;
 		}
@@ -3551,9 +5491,13 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 	if (!new_net_conf)
 		goto disconnect;
 
-	mutex_lock(&connection->data.mutex);
-	mutex_lock(&connection->resource->conf_update);
-	old_net_conf = connection->net_conf;
+	if (mutex_lock_interruptible(&connection->resource->conf_update)) {
+		drbd_err(connection, "Interrupted while waiting for conf_update\n");
+		goto disconnect;
+	}
+
+	mutex_lock(&connection->mutex[DATA_STREAM]);
+	old_net_conf = connection->transport.net_conf;
 	*new_net_conf = *old_net_conf;
 
 	new_net_conf->wire_protocol = p_proto;
@@ -3562,9 +5506,9 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 	new_net_conf->after_sb_2p = convert_after_sb(p_after_sb_2p);
 	new_net_conf->two_primaries = p_two_primaries;
 
-	rcu_assign_pointer(connection->net_conf, new_net_conf);
+	rcu_assign_pointer(connection->transport.net_conf, new_net_conf);
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
 	mutex_unlock(&connection->resource->conf_update);
-	mutex_unlock(&connection->data.mutex);
 
 	crypto_free_shash(connection->peer_integrity_tfm);
 	kfree(connection->int_dig_in);
@@ -3583,10 +5527,11 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
 disconnect_rcu_unlock:
 	rcu_read_unlock();
 disconnect:
+	kfree(new_net_conf);
 	crypto_free_shash(peer_integrity_tfm);
 	kfree(int_dig_in);
 	kfree(int_dig_vv);
-	conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
+	change_cstate(connection, C_DISCONNECTING, CS_HARD);
 	return -EIO;
 }
 
@@ -3595,8 +5540,7 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in
  * return: NULL (alg name was "")
  *         ERR_PTR(error) if something goes wrong
  *         or the crypto hash ptr, if it worked out ok. */
-static struct crypto_shash *drbd_crypto_alloc_digest_safe(
-		const struct drbd_device *device,
+static struct crypto_shash *drbd_crypto_alloc_digest_safe(const struct drbd_device *device,
 		const char *alg, const char *name)
 {
 	struct crypto_shash *tfm;
@@ -3613,44 +5557,11 @@ static struct crypto_shash *drbd_crypto_alloc_digest_safe(
 	return tfm;
 }
 
-static int ignore_remaining_packet(struct drbd_connection *connection, struct packet_info *pi)
-{
-	void *buffer = connection->data.rbuf;
-	int size = pi->size;
-
-	while (size) {
-		int s = min_t(int, size, DRBD_SOCKET_BUFFER_SIZE);
-		s = drbd_recv(connection, buffer, s);
-		if (s <= 0) {
-			if (s < 0)
-				return s;
-			break;
-		}
-		size -= s;
-	}
-	if (size)
-		return -EIO;
-	return 0;
-}
-
-/*
- * config_unknown_volume  -  device configuration command for unknown volume
- *
- * When a device is added to an existing connection, the node on which the
- * device is added first will send configuration commands to its peer but the
- * peer will not know about the device yet.  It will warn and ignore these
- * commands.  Once the device is added on the second node, the second node will
- * send the same device configuration commands, but in the other direction.
- *
- * (We can also end up here if drbd is misconfigured.)
- */
-static int config_unknown_volume(struct drbd_connection *connection, struct packet_info *pi)
-{
-	drbd_warn(connection, "%s packet received for volume %u, which is not configured locally\n",
-		  cmdname(pi->cmd), pi->vnr);
-	return ignore_remaining_packet(connection, pi);
-}
-
+/* Receive P_SYNC_PARAM89 and the older P_SYNC_PARAM. The peer_device fields
+ * related to resync configuration are ignored. These include resync_rate,
+ * c_max_rate and the like. We ignore them because applying them to our own
+ * configuration would be confusing. It would cause us to swap configuration
+ * with our peer each time we connected. */
 static int receive_SyncParam(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct drbd_peer_device *peer_device;
@@ -3660,10 +5571,10 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 	struct crypto_shash *verify_tfm = NULL;
 	struct crypto_shash *csums_tfm = NULL;
 	struct net_conf *old_net_conf, *new_net_conf = NULL;
-	struct disk_conf *old_disk_conf = NULL, *new_disk_conf = NULL;
+	struct peer_device_conf *old_peer_device_conf = NULL;
 	const int apv = connection->agreed_pro_version;
 	struct fifo_buffer *old_plan = NULL, *new_plan = NULL;
-	unsigned int fifo_size = 0;
+	struct drbd_resource *resource = connection->resource;
 	int err;
 
 	peer_device = conn_peer_device(connection, pi->vnr);
@@ -3696,48 +5607,26 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 		D_ASSERT(device, data_size == 0);
 	}
 
-	/* initialize verify_alg and csums_alg */
-	p = pi->data;
-	BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
-	memset(&p->algs, 0, sizeof(p->algs));
-
-	err = drbd_recv_all(peer_device->connection, p, header_size);
+	err = drbd_recv_all(connection, (void **)&p, header_size + data_size);
 	if (err)
 		return err;
 
-	mutex_lock(&connection->resource->conf_update);
-	old_net_conf = peer_device->connection->net_conf;
-	if (get_ldev(device)) {
-		new_disk_conf = kzalloc_obj(struct disk_conf);
-		if (!new_disk_conf) {
-			put_ldev(device);
-			mutex_unlock(&connection->resource->conf_update);
-			drbd_err(device, "Allocation of new disk_conf failed\n");
-			return -ENOMEM;
-		}
-
-		old_disk_conf = device->ldev->disk_conf;
-		*new_disk_conf = *old_disk_conf;
-
-		new_disk_conf->resync_rate = be32_to_cpu(p->resync_rate);
+	err = mutex_lock_interruptible(&resource->conf_update);
+	if (err) {
+		drbd_err(connection, "Interrupted while waiting for conf_update\n");
+		return err;
 	}
+	old_net_conf = connection->transport.net_conf;
 
 	if (apv >= 88) {
 		if (apv == 88) {
 			if (data_size > SHARED_SECRET_MAX || data_size == 0) {
-				drbd_err(device, "verify-alg of wrong size, "
-					"peer wants %u, accepting only up to %u byte\n",
-					data_size, SHARED_SECRET_MAX);
+				drbd_err(device, "verify-alg too long, "
+					 "peer wants %u, accepting only %u byte\n",
+					 data_size, SHARED_SECRET_MAX);
 				goto reconnect;
 			}
-
-			err = drbd_recv_all(peer_device->connection, p->verify_alg, data_size);
-			if (err)
-				goto reconnect;
-			/* we expect NUL terminated string */
-			/* but just in case someone tries to be evil */
-			D_ASSERT(device, p->verify_alg[data_size-1] == 0);
-			p->verify_alg[data_size-1] = 0;
+			p->verify_alg[data_size] = 0;
 
 		} else /* apv >= 89 */ {
 			/* we still expect NUL terminated strings */
@@ -3749,7 +5638,7 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 		}
 
 		if (strcmp(old_net_conf->verify_alg, p->verify_alg)) {
-			if (device->state.conn == C_WF_REPORT_PARAMS) {
+			if (peer_device->repl_state[NOW] == L_OFF) {
 				drbd_err(device, "Different verify-alg settings. me=\"%s\" peer=\"%s\"\n",
 				    old_net_conf->verify_alg, p->verify_alg);
 				goto disconnect;
@@ -3763,7 +5652,7 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 		}
 
 		if (apv >= 89 && strcmp(old_net_conf->csums_alg, p->csums_alg)) {
-			if (device->state.conn == C_WF_REPORT_PARAMS) {
+			if (peer_device->repl_state[NOW] == L_OFF) {
 				drbd_err(device, "Different csums-alg settings. me=\"%s\" peer=\"%s\"\n",
 				    old_net_conf->csums_alg, p->csums_alg);
 				goto disconnect;
@@ -3776,23 +5665,6 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 			}
 		}
 
-		if (apv > 94 && new_disk_conf) {
-			new_disk_conf->c_plan_ahead = be32_to_cpu(p->c_plan_ahead);
-			new_disk_conf->c_delay_target = be32_to_cpu(p->c_delay_target);
-			new_disk_conf->c_fill_target = be32_to_cpu(p->c_fill_target);
-			new_disk_conf->c_max_rate = be32_to_cpu(p->c_max_rate);
-
-			fifo_size = (new_disk_conf->c_plan_ahead * 10 * SLEEP_TIME) / HZ;
-			if (fifo_size != device->rs_plan_s->size) {
-				new_plan = fifo_alloc(fifo_size);
-				if (!new_plan) {
-					drbd_err(device, "kmalloc of fifo_buffer failed");
-					put_ldev(device);
-					goto disconnect;
-				}
-			}
-		}
-
 		if (verify_tfm || csums_tfm) {
 			new_net_conf = kzalloc_obj(struct net_conf);
 			if (!new_net_conf)
@@ -3803,66 +5675,58 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i
 			if (verify_tfm) {
 				strscpy(new_net_conf->verify_alg, p->verify_alg);
 				new_net_conf->verify_alg_len = strlen(p->verify_alg) + 1;
-				crypto_free_shash(peer_device->connection->verify_tfm);
-				peer_device->connection->verify_tfm = verify_tfm;
+				crypto_free_shash(connection->verify_tfm);
+				connection->verify_tfm = verify_tfm;
 				drbd_info(device, "using verify-alg: \"%s\"\n", p->verify_alg);
 			}
 			if (csums_tfm) {
 				strscpy(new_net_conf->csums_alg, p->csums_alg);
 				new_net_conf->csums_alg_len = strlen(p->csums_alg) + 1;
-				crypto_free_shash(peer_device->connection->csums_tfm);
-				peer_device->connection->csums_tfm = csums_tfm;
+				crypto_free_shash(connection->csums_tfm);
+				connection->csums_tfm = csums_tfm;
 				drbd_info(device, "using csums-alg: \"%s\"\n", p->csums_alg);
 			}
-			rcu_assign_pointer(connection->net_conf, new_net_conf);
+			rcu_assign_pointer(connection->transport.net_conf, new_net_conf);
 		}
 	}
 
-	if (new_disk_conf) {
-		rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
-		put_ldev(device);
-	}
-
-	if (new_plan) {
-		old_plan = device->rs_plan_s;
-		rcu_assign_pointer(device->rs_plan_s, new_plan);
-	}
+	if (new_plan)
+		rcu_assign_pointer(peer_device->rs_plan_s, new_plan);
 
-	mutex_unlock(&connection->resource->conf_update);
+	mutex_unlock(&resource->conf_update);
 	synchronize_rcu();
 	if (new_net_conf)
 		kfree(old_net_conf);
-	kfree(old_disk_conf);
-	kfree(old_plan);
+	kfree(old_peer_device_conf);
+	if (new_plan)
+		kfree(old_plan);
 
 	return 0;
 
 reconnect:
-	if (new_disk_conf) {
-		put_ldev(device);
-		kfree(new_disk_conf);
-	}
-	mutex_unlock(&connection->resource->conf_update);
+	mutex_unlock(&resource->conf_update);
 	return -EIO;
 
 disconnect:
 	kfree(new_plan);
-	if (new_disk_conf) {
-		put_ldev(device);
-		kfree(new_disk_conf);
-	}
-	mutex_unlock(&connection->resource->conf_update);
+	mutex_unlock(&resource->conf_update);
 	/* just for completeness: actually not needed,
 	 * as this is not reached if csums_tfm was ok. */
 	crypto_free_shash(csums_tfm);
 	/* but free the verify_tfm again, if csums_tfm did not work out */
 	crypto_free_shash(verify_tfm);
-	conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
+	change_cstate(connection, C_DISCONNECTING, CS_HARD);
 	return -EIO;
 }
 
+static void drbd_setup_order_type(struct drbd_device *device, int peer)
+{
+	/* sorry, we currently have no working implementation
+	 * of distributed TCQ */
+}
+
 /* warn if the arguments differ by more than 12.5% */
-static void warn_if_differ_considerably(struct drbd_device *device,
+static void warn_if_differ_considerably(struct drbd_peer_device *peer_device,
 	const char *s, sector_t a, sector_t b)
 {
 	sector_t d;
@@ -3870,135 +5734,325 @@ static void warn_if_differ_considerably(struct drbd_device *device,
 		return;
 	d = (a > b) ? (a - b) : (b - a);
 	if (d > (a>>3) || d > (b>>3))
-		drbd_warn(device, "Considerable difference in %s: %llus vs. %llus\n", s,
+		drbd_warn(peer_device, "Considerable difference in %s: %llus vs. %llus\n", s,
 		     (unsigned long long)a, (unsigned long long)b);
 }
 
-static int receive_sizes(struct drbd_connection *connection, struct packet_info *pi)
+static bool drbd_other_peer_smaller(struct drbd_peer_device *reference_peer_device, uint64_t new_size)
 {
+	struct drbd_device *device = reference_peer_device->device;
 	struct drbd_peer_device *peer_device;
+	bool smaller = false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device == reference_peer_device)
+			continue;
+
+		/* Ignore peers without an attached disk. */
+		if (peer_device->disk_state[NOW] < D_INCONSISTENT)
+			continue;
+
+		if (peer_device->d_size != 0 && peer_device->d_size < new_size)
+			smaller = true;
+	}
+	rcu_read_unlock();
+
+	return smaller;
+}
+
+/* Maximum bio size that a protocol version supports. */
+static unsigned int conn_max_bio_size(struct drbd_connection *connection)
+{
+	if (connection->agreed_pro_version >= 100)
+		return DRBD_MAX_BIO_SIZE;
+	else if (connection->agreed_pro_version >= 95)
+		return DRBD_MAX_BIO_SIZE_P95;
+	else
+		return DRBD_MAX_SIZE_H80_PACKET;
+}
+
+static struct drbd_peer_device *get_neighbor_device(struct drbd_device *device,
+		enum drbd_neighbor neighbor)
+{
+	s32 self_id, peer_id, pivot;
+	struct drbd_peer_device *peer_device, *peer_device_ret = NULL;
+
+	if (!get_ldev(device))
+		return NULL;
+	self_id = device->ldev->md.node_id;
+	put_ldev(device);
+
+	pivot = neighbor == NEXT_LOWER ? 0 : neighbor == NEXT_HIGHER ? S32_MAX : -1;
+	if (pivot == -1)
+		return NULL;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		bool found_new = false;
+		peer_id = peer_device->node_id;
+
+		if (neighbor == NEXT_LOWER && peer_id < self_id && peer_id >= pivot)
+			found_new = true;
+		else if (neighbor == NEXT_HIGHER && peer_id > self_id && peer_id <= pivot)
+			found_new = true;
+
+		if (found_new && peer_device->disk_state[NOW] >= D_INCONSISTENT) {
+			pivot = peer_id;
+			peer_device_ret = peer_device;
+		}
+	}
+	rcu_read_unlock();
+
+	return peer_device_ret;
+}
+
+static void maybe_trigger_resync(struct drbd_device *device, struct drbd_peer_device *peer_device, bool grew, bool skip)
+{
+	if (!peer_device)
+		return;
+	if (peer_device->repl_state[NOW] <= L_OFF)
+		return;
+	if (test_and_clear_bit(RESIZE_PENDING, &peer_device->flags) ||
+	    (grew && peer_device->repl_state[NOW] == L_ESTABLISHED)) {
+		if (peer_device->disk_state[NOW] >= D_INCONSISTENT &&
+		    device->disk_state[NOW] >= D_INCONSISTENT) {
+			if (skip)
+				drbd_info(peer_device, "Resync of new storage suppressed with --assume-clean\n");
+			else
+				resync_after_online_grow(peer_device);
+		} else
+			set_bit(RESYNC_AFTER_NEG, &peer_device->flags);
+	}
+}
+
+static int receive_sizes(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device, *peer_device_it = NULL;
 	struct drbd_device *device;
 	struct p_sizes *p = pi->data;
-	struct o_qlim *o = (connection->agreed_features & DRBD_FF_WSAME) ? p->qlim : NULL;
+	uint64_t p_size, p_usize, p_csize;
+	uint64_t my_usize, my_max_size, cur_size;
 	enum determine_dev_size dd = DS_UNCHANGED;
-	sector_t p_size, p_usize, p_csize, my_usize;
-	sector_t new_size, cur_size;
-	int ldsc = 0; /* local disk size changed */
+	bool should_send_sizes = false;
 	enum dds_flags ddsf;
+	unsigned int protocol_max_bio_size;
+	bool have_ldev = false;
+	bool have_mutex = false;
+	bool is_handshake;
+	int err;
+	u64 im;
 
 	peer_device = conn_peer_device(connection, pi->vnr);
 	if (!peer_device)
 		return config_unknown_volume(connection, pi);
 	device = peer_device->device;
-	cur_size = get_capacity(device->vdisk);
 
+	err = mutex_lock_interruptible(&connection->resource->conf_update);
+	if (err) {
+		drbd_err(connection, "Interrupted while waiting for conf_update\n");
+		goto out;
+	}
+	have_mutex = true;
+
+	/* just store the peer's disk size for now.
+	 * we still need to figure out whether we accept that. */
 	p_size = be64_to_cpu(p->d_size);
 	p_usize = be64_to_cpu(p->u_size);
 	p_csize = be64_to_cpu(p->c_size);
 
-	/* just store the peer's disk size for now.
-	 * we still need to figure out whether we accept that. */
-	device->p_size = p_size;
+	peer_device->d_size = p_size;
+	peer_device->u_size = p_usize;
+	peer_device->c_size = p_csize;
+
+	/* Ignore "current" size for calculating "max" size. */
+	/* If it used to have a disk, but now is detached, don't revert back to zero. */
+	if (p_size)
+		peer_device->max_size = p_size;
+
+	cur_size = get_capacity(device->vdisk);
+	dynamic_drbd_dbg(device, "current_size: %llu\n", (unsigned long long)cur_size);
+	dynamic_drbd_dbg(peer_device, "c_size: %llu u_size: %llu d_size: %llu max_size: %llu\n",
+			(unsigned long long)p_csize,
+			(unsigned long long)p_usize,
+			(unsigned long long)p_size,
+			(unsigned long long)peer_device->max_size);
+
+	if ((p_size && p_csize > p_size) || (p_usize && p_csize > p_usize)) {
+		drbd_warn(peer_device, "Peer sent bogus sizes, disconnecting\n");
+		goto disconnect;
+	}
+
+	/* The protocol version limits how big requests can be.  In addition,
+	 * peers before protocol version 94 cannot split large requests into
+	 * multiple bios; their reported max_bio_size is a hard limit.
+	 */
+	protocol_max_bio_size = conn_max_bio_size(connection);
+	peer_device->q_limits.max_bio_size = min(be32_to_cpu(p->max_bio_size),
+						 protocol_max_bio_size);
+	ddsf = be16_to_cpu(p->dds_flags);
+	is_handshake = (peer_device->repl_state[NOW] == L_OFF);
+	set_bit(HAVE_SIZES, &peer_device->flags);
 
 	if (get_ldev(device)) {
+		sector_t new_size;
+
+		have_ldev = true;
+
 		rcu_read_lock();
 		my_usize = rcu_dereference(device->ldev->disk_conf)->disk_size;
 		rcu_read_unlock();
 
-		warn_if_differ_considerably(device, "lower level device sizes",
-			   p_size, drbd_get_max_capacity(device->ldev));
-		warn_if_differ_considerably(device, "user requested size",
+		my_max_size = drbd_get_max_capacity(device, device->ldev, false);
+		dynamic_drbd_dbg(peer_device, "la_size: %llu my_usize: %llu my_max_size: %llu\n",
+			(unsigned long long)device->ldev->md.effective_size,
+			(unsigned long long)my_usize,
+			(unsigned long long)my_max_size);
+
+		if (peer_device->disk_state[NOW] > D_DISKLESS)
+			warn_if_differ_considerably(peer_device, "lower level device sizes",
+				   p_size, my_max_size);
+		warn_if_differ_considerably(peer_device, "user requested size",
 					    p_usize, my_usize);
 
-		/* if this is the first connect, or an otherwise expected
-		 * param exchange, choose the minimum */
-		if (device->state.conn == C_WF_REPORT_PARAMS)
+		if (is_handshake)
 			p_usize = min_not_zero(my_usize, p_usize);
 
+		if (p_usize == 0) {
+			/* Peer may reset usize to zero only if it has a backend.
+			 * Because a diskless node has no disk config,
+			 * and always sends zero. */
+			if (p_size == 0)
+				p_usize = my_usize;
+		}
+
+		new_size = drbd_new_dev_size(device, p_csize, p_usize, ddsf);
+
 		/* Never shrink a device with usable data during connect,
 		 * or "attach" on the peer.
 		 * But allow online shrinking if we are connected. */
-		new_size = drbd_new_dev_size(device, device->ldev, p_usize, 0);
 		if (new_size < cur_size &&
-		    device->state.disk >= D_OUTDATED &&
-		    (device->state.conn < C_CONNECTED || device->state.pdsk == D_DISKLESS)) {
-			drbd_err(device, "The peer's disk size is too small! (%llu < %llu sectors)\n",
+		    device->disk_state[NOW] >= D_OUTDATED &&
+		    (peer_device->repl_state[NOW] < L_ESTABLISHED || peer_device->disk_state[NOW] == D_DISKLESS)) {
+			drbd_err(peer_device, "The peer's disk size is too small! (%llu < %llu sectors)\n",
 					(unsigned long long)new_size, (unsigned long long)cur_size);
-			conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
-			put_ldev(device);
-			return -EIO;
+			goto disconnect;
+		}
+
+		/* Disconnect, if we cannot grow to the peer's current size */
+		if (my_max_size < p_csize && !is_handshake) {
+			drbd_err(peer_device, "Peer's size larger than my maximum capacity (%llu < %llu sectors)\n",
+					(unsigned long long)my_max_size, (unsigned long long)p_csize);
+			goto disconnect;
 		}
 
 		if (my_usize != p_usize) {
-			struct disk_conf *old_disk_conf, *new_disk_conf = NULL;
+			struct disk_conf *old_disk_conf, *new_disk_conf;
 
 			new_disk_conf = kzalloc_obj(struct disk_conf);
 			if (!new_disk_conf) {
-				put_ldev(device);
-				return -ENOMEM;
+				err = -ENOMEM;
+				goto out;
 			}
 
-			mutex_lock(&connection->resource->conf_update);
 			old_disk_conf = device->ldev->disk_conf;
 			*new_disk_conf = *old_disk_conf;
 			new_disk_conf->disk_size = p_usize;
 
 			rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
-			mutex_unlock(&connection->resource->conf_update);
 			kvfree_rcu_mightsleep(old_disk_conf);
 
-			drbd_info(device, "Peer sets u_size to %lu sectors (old: %lu)\n",
-				 (unsigned long)p_usize, (unsigned long)my_usize);
+			drbd_info(peer_device, "Peer sets u_size to %llu sectors (old: %llu)\n",
+				 (unsigned long long)p_usize, (unsigned long long)my_usize);
+			/* Do not set should_send_sizes here. That might cause packet storms */
 		}
+	}
 
-		put_ldev(device);
+	if (connection->agreed_features & DRBD_FF_WSAME) {
+		struct o_qlim *qlim = p->qlim;
+
+		peer_device->q_limits.physical_block_size = be32_to_cpu(qlim->physical_block_size);
+		peer_device->q_limits.logical_block_size = be32_to_cpu(qlim->logical_block_size);
+		peer_device->q_limits.alignment_offset = be32_to_cpu(qlim->alignment_offset);
+		peer_device->q_limits.io_min = be32_to_cpu(qlim->io_min);
+		peer_device->q_limits.io_opt = be32_to_cpu(qlim->io_opt);
+	}
+
+	if (connection->agreed_features & DRBD_FF_BM_BLOCK_SHIFT)
+		peer_device->bm_block_shift =
+			p->qlim->bm_block_shift_minus_12 + BM_BLOCK_SHIFT_4k;
+	else {
+		int bbs = device->bitmap ? bm_block_size(device->bitmap) : BM_BLOCK_SIZE_4k;
+		/* May work as long as this one is SyncTarget. May result in
+		 * funny never ending / repeating resyncs if the other guy is
+		 * SyncTarget, but unaware of bitmap granularity issues.
+		 */
+		if (bbs != BM_BLOCK_SIZE_4k)
+			drbd_warn(peer_device,
+				"My bitmap granularity is %u. Upgrade this peer to make it aware.\n",
+				bbs);
+		peer_device->bm_block_shift = BM_BLOCK_SHIFT_4k;
 	}
 
-	device->peer_max_bio_size = be32_to_cpu(p->max_bio_size);
 	/* Leave drbd_reconsider_queue_parameters() before drbd_determine_dev_size().
 	   In case we cleared the QUEUE_FLAG_DISCARD from our queue in
 	   drbd_reconsider_queue_parameters(), we can be sure that after
-	   drbd_determine_dev_size() no REQ_DISCARDs are in the queue. */
+	   drbd_determine_dev_size() no REQ_OP_DISCARDs are in the queue. */
+	if (have_ldev) {
+		enum dds_flags local_ddsf = ddsf;
+		drbd_reconsider_queue_parameters(device, device->ldev);
 
-	ddsf = be16_to_cpu(p->dds_flags);
-	if (get_ldev(device)) {
-		drbd_reconsider_queue_parameters(device, device->ldev, o);
-		dd = drbd_determine_dev_size(device, ddsf, NULL);
-		put_ldev(device);
-		if (dd == DS_ERROR)
-			return -EIO;
-		drbd_md_sync(device);
+		/* To support thinly provisioned nodes (partial resync) joining later,
+		   clear all bitmap slots, including the unused ones. */
+		if (device->ldev->md.effective_size == 0)
+			local_ddsf |= DDSF_NO_RESYNC;
+
+		dd = drbd_determine_dev_size(device, p_csize, local_ddsf, NULL);
+
+		if (dd == DS_GREW || dd == DS_SHRUNK)
+			should_send_sizes = true;
+
+		if (dd == DS_ERROR) {
+			err = -EIO;
+			goto out;
+		}
+		drbd_md_sync_if_dirty(device);
 	} else {
-		/*
-		 * I am diskless, need to accept the peer's *current* size.
-		 * I must NOT accept the peers backing disk size,
-		 * it may have been larger than mine all along...
+		uint64_t new_size = 0;
+
+		drbd_reconsider_queue_parameters(device, NULL);
+		/* In case I am diskless, need to accept the peer's *current* size.
 		 *
 		 * At this point, the peer knows more about my disk, or at
 		 * least about what we last agreed upon, than myself.
 		 * So if his c_size is less than his d_size, the most likely
-		 * reason is that *my* d_size was smaller last time we checked.
-		 *
-		 * However, if he sends a zero current size,
-		 * take his (user-capped or) backing disk size anyways.
+		 * reason is that *my* d_size was smaller last time we checked,
+		 * or some other peer does not (yet) have enough room.
 		 *
 		 * Unless of course he does not have a disk himself.
 		 * In which case we ignore this completely.
 		 */
-		sector_t new_size = p_csize ?: p_usize ?: p_size;
-		drbd_reconsider_queue_parameters(device, NULL, o);
+		new_size = p_csize;
+		new_size = min_not_zero(new_size, p_usize);
+		new_size = min_not_zero(new_size, p_size);
+
 		if (new_size == 0) {
 			/* Ignore, peer does not know nothing. */
 		} else if (new_size == cur_size) {
 			/* nothing to do */
 		} else if (cur_size != 0 && p_size == 0) {
-			drbd_warn(device, "Ignored diskless peer device size (peer:%llu != me:%llu sectors)!\n",
+			dynamic_drbd_dbg(peer_device,
+					"Ignored diskless peer device size (peer:%llu != me:%llu sectors)!\n",
 					(unsigned long long)new_size, (unsigned long long)cur_size);
-		} else if (new_size < cur_size && device->state.role == R_PRIMARY) {
-			drbd_err(device, "The peer's device size is too small! (%llu < %llu sectors); demote me first!\n",
-					(unsigned long long)new_size, (unsigned long long)cur_size);
-			conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
-			return -EIO;
+		} else if (new_size < cur_size && device->resource->role[NOW] == R_PRIMARY) {
+			drbd_err(peer_device,
+				"The peer's device size is too small! (%llu < %llu sectors); demote me first!\n",
+				(unsigned long long)new_size, (unsigned long long)cur_size);
+			goto disconnect;
+		} else if (drbd_other_peer_smaller(peer_device, new_size)) {
+			dynamic_drbd_dbg(peer_device,
+					"Ignored peer device size (peer:%llu sectors); other peer smaller!\n",
+					(unsigned long long)new_size);
 		} else {
 			/* I believe the peer, if
 			 *  - I don't have a current size myself
@@ -4009,1071 +6063,3886 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info
 			 *    and he has the only disk,
 			 *    which is larger than my current size
 			 */
+			should_send_sizes = true;
 			drbd_set_my_capacity(device, new_size);
 		}
 	}
 
-	if (get_ldev(device)) {
+	if (device->device_conf.max_bio_size > protocol_max_bio_size ||
+	    (connection->agreed_pro_version < 94 &&
+	     device->device_conf.max_bio_size > peer_device->q_limits.max_bio_size)) {
+		drbd_err(device, "Peer cannot deal with requests bigger than %u. "
+			 "Please reduce max_bio_size in the configuration.\n",
+			 peer_device->q_limits.max_bio_size);
+		goto disconnect;
+	}
+
+	if (have_ldev) {
 		if (device->ldev->known_size != drbd_get_capacity(device->ldev->backing_bdev)) {
 			device->ldev->known_size = drbd_get_capacity(device->ldev->backing_bdev);
-			ldsc = 1;
+			should_send_sizes = true;
 		}
 
+		drbd_setup_order_type(device, be16_to_cpu(p->queue_order_type));
+	}
+
+	cur_size = get_capacity(device->vdisk);
+
+	for_each_peer_device_ref(peer_device_it, im, device) {
+		struct drbd_connection *con_it = peer_device_it->connection;
+
+		/* drop cached max_size, if we already grew beyond it */
+		if (peer_device_it->max_size < cur_size)
+			peer_device_it->max_size = 0;
+
+		if (con_it->cstate[NOW] < C_CONNECTED)
+			continue;
+
+		/* Send size updates only if something relevant has changed.
+		 * TODO: only tell the sender thread to do so,
+		 * or we may end up in a distributed deadlock on congestion. */
+
+		if (should_send_sizes)
+			drbd_send_sizes(peer_device_it, p_usize, ddsf);
+	}
+
+	maybe_trigger_resync(device, get_neighbor_device(device, NEXT_HIGHER),
+					dd == DS_GREW, ddsf & DDSF_NO_RESYNC);
+	maybe_trigger_resync(device, get_neighbor_device(device, NEXT_LOWER),
+					dd == DS_GREW, ddsf & DDSF_NO_RESYNC);
+	err = 0;
+
+out:
+	if (have_ldev)
 		put_ldev(device);
+	if (have_mutex)
+		mutex_unlock(&connection->resource->conf_update);
+	return err;
+
+disconnect:
+	/* don't let a rejected peer confuse future handshakes with different peers. */
+	peer_device->max_size = 0;
+
+	if (connection->resource->remote_state_change)
+		set_bit(TWOPC_RECV_SIZES_ERR, &connection->resource->flags);
+	else
+		err = -EIO;
+	goto out;
+}
+
+static enum sync_strategy resolve_splitbrain_from_disk_states(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	enum drbd_disk_state peer_disk_state = peer_device->disk_state[NOW];
+	enum drbd_disk_state disk_state = device->disk_state[NOW];
+
+	return  disk_state <= D_UP_TO_DATE && peer_disk_state == D_UP_TO_DATE ? SYNC_TARGET_USE_BITMAP :
+		disk_state == D_UP_TO_DATE && peer_disk_state <= D_UP_TO_DATE ? SYNC_SOURCE_USE_BITMAP :
+		SPLIT_BRAIN_AUTO_RECOVER;
+}
+
+static void drbd_resync(struct drbd_peer_device *peer_device,
+			enum resync_reason reason) __must_hold(local)
+{
+	enum drbd_role peer_role = peer_device->connection->peer_role[NOW];
+	enum drbd_repl_state new_repl_state;
+	enum drbd_disk_state peer_disk_state;
+	enum sync_strategy strategy;
+	enum sync_rule rule;
+	int peer_node_id;
+	enum drbd_state_rv rv;
+	const char *tag = reason == AFTER_UNSTABLE ? "after-unstable" : "diskless-primary";
+
+	strategy = drbd_handshake(peer_device, &rule, &peer_node_id, reason == DISKLESS_PRIMARY);
+	if (strategy == SPLIT_BRAIN_AUTO_RECOVER && reason == AFTER_UNSTABLE)
+		strategy = resolve_splitbrain_from_disk_states(peer_device);
+
+	if (!is_strategy_determined(strategy)) {
+		drbd_info(peer_device, "Unexpected result of handshake() %s!\n", strategy_descriptor(strategy).name);
+		return;
 	}
 
-	if (device->state.conn > C_WF_REPORT_PARAMS) {
-		if (be64_to_cpu(p->c_size) != get_capacity(device->vdisk) ||
-		    ldsc) {
-			/* we have different sizes, probably peer
-			 * needs to know my new size... */
-			drbd_send_sizes(peer_device, 0, ddsf);
-		}
-		if (test_and_clear_bit(RESIZE_PENDING, &device->flags) ||
-		    (dd == DS_GREW && device->state.conn == C_CONNECTED)) {
-			if (device->state.pdsk >= D_INCONSISTENT &&
-			    device->state.disk >= D_INCONSISTENT) {
-				if (ddsf & DDSF_NO_RESYNC)
-					drbd_info(device, "Resync of new storage suppressed with --assume-clean\n");
-				else
-					resync_after_online_grow(device);
-			} else
-				set_bit(RESYNC_AFTER_NEG, &device->flags);
+	peer_disk_state = peer_device->disk_state[NOW];
+	if (reason == DISKLESS_PRIMARY)
+		disk_states_to_strategy(peer_device, peer_disk_state, &strategy, rule, &peer_node_id);
+
+	new_repl_state = strategy_to_repl_state(peer_device, peer_role, strategy);
+	if (new_repl_state != L_ESTABLISHED) {
+		bitmap_mod_after_handshake(peer_device, strategy, peer_node_id);
+		drbd_info(peer_device, "Becoming %s %s\n", drbd_repl_str(new_repl_state),
+			  reason == AFTER_UNSTABLE ? "after unstable" : "because primary is diskless");
+	}
+
+	if (new_repl_state == L_ESTABLISHED && peer_disk_state >= D_CONSISTENT &&
+	    peer_device->device->disk_state[NOW] == D_OUTDATED) {
+		/* No resync with up-to-date peer -> I should be consistent or up-to-date as well.
+		   Note: Former unstable (but up-to-date) nodes become consistent for a short
+		   time after loosing their primary peer. Therefore consider consistent here
+		   as well. */
+		drbd_info(peer_device, "Upgrading local disk to %s after unstable/weak (and no resync).\n",
+			  drbd_disk_str(peer_disk_state));
+		change_disk_state(peer_device->device, peer_disk_state, CS_VERBOSE, tag, NULL);
+		return;
+	}
+
+	rv = change_repl_state(peer_device, new_repl_state, CS_VERBOSE, tag);
+	if ((rv == SS_NOTHING_TO_DO || rv == SS_RESYNC_RUNNING) &&
+	    (new_repl_state == L_WF_BITMAP_S || new_repl_state == L_WF_BITMAP_T)) {
+		/* Those events might happen very quickly. In case we are still processing
+		   the previous resync we need to re-enter that state. Schedule sending of
+		   the bitmap here explicitly */
+		peer_device->resync_again++;
+		drbd_info(peer_device, "...postponing this until current resync finished\n");
+	}
+}
+
+static void update_bitmap_slot_of_peer(struct drbd_peer_device *peer_device, int node_id, u64 bitmap_uuid)
+{
+	struct drbd_device *device = peer_device->device;
+
+	if (peer_device->bitmap_uuids[node_id] && bitmap_uuid == 0) {
+		/* If we learn from a neighbor that it no longer has a bitmap
+		   against a third node, we need to deduce from that knowledge
+		   that in the other direction the bitmap was cleared as well.
+		 */
+		struct drbd_peer_device *peer_device2;
+
+		rcu_read_lock();
+		peer_device2 = peer_device_by_node_id(peer_device->device, node_id);
+		if (peer_device2) {
+			int node_id2 = peer_device->connection->peer_node_id;
+			peer_device2->bitmap_uuids[node_id2] = 0;
 		}
+		rcu_read_unlock();
 	}
 
-	return 0;
+	if (node_id != device->resource->res_opts.node_id && bitmap_uuid != -1 && get_ldev(device)) {
+		_drbd_uuid_push_history(device, bitmap_uuid);
+		put_ldev(device);
+	}
+	peer_device->bitmap_uuids[node_id] = bitmap_uuid;
 }
 
-static int receive_uuids(struct drbd_connection *connection, struct packet_info *pi)
+static void propagate_skip_initial_to_diskless(struct drbd_device *device)
 {
 	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_uuids *p = pi->data;
-	u64 *p_uuid;
-	int i, updated_uuids = 0;
-
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return config_unknown_volume(connection, pi);
-	device = peer_device->device;
+	u64 im;
 
-	p_uuid = kmalloc_array(UI_EXTENDED_SIZE, sizeof(*p_uuid), GFP_NOIO);
-	if (!p_uuid)
-		return false;
+	for_each_peer_device_ref(peer_device, im, device) {
+		if (peer_device->disk_state[NOW] == D_DISKLESS)
+			drbd_send_uuids(peer_device, UUID_FLAG_SKIP_INITIAL_SYNC, 0);
+	}
+}
 
-	for (i = UI_CURRENT; i < UI_EXTENDED_SIZE; i++)
-		p_uuid[i] = be64_to_cpu(p->uuid[i]);
+static int __receive_uuids(struct drbd_peer_device *peer_device, u64 node_mask)
+{
+	enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+	struct drbd_device *device = peer_device->device;
+	struct drbd_resource *resource = device->resource;
+	int updated_uuids = 0, err = 0;
+	bool bad_server, uuid_match;
+	struct net_conf *nc;
+	bool two_primaries_allowed;
 
-	kfree(device->p_uuid);
-	device->p_uuid = p_uuid;
+	uuid_match =
+		(device->exposed_data_uuid & ~UUID_PRIMARY) ==
+		(peer_device->current_uuid & ~UUID_PRIMARY);
+	bad_server =
+		repl_state < L_ESTABLISHED &&
+		device->disk_state[NOW] < D_INCONSISTENT &&
+		device->resource->role[NOW] == R_PRIMARY && !uuid_match;
 
-	if ((device->state.conn < C_CONNECTED || device->state.pdsk == D_DISKLESS) &&
-	    device->state.disk < D_INCONSISTENT &&
-	    device->state.role == R_PRIMARY &&
-	    (device->ed_uuid & ~((u64)1)) != (p_uuid[UI_CURRENT] & ~((u64)1))) {
+	if (peer_device->connection->agreed_pro_version < 110 && bad_server) {
 		drbd_err(device, "Can only connect to data with current UUID=%016llX\n",
-		    (unsigned long long)device->ed_uuid);
-		conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
+		    (unsigned long long)device->exposed_data_uuid);
+		change_cstate(peer_device->connection, C_DISCONNECTING, CS_HARD);
 		return -EIO;
 	}
 
+	rcu_read_lock();
+	nc = rcu_dereference(peer_device->connection->transport.net_conf);
+	two_primaries_allowed = nc && nc->two_primaries;
+	rcu_read_unlock();
+
 	if (get_ldev(device)) {
-		int skip_initial_sync =
-			device->state.conn == C_CONNECTED &&
+		bool skip_initial_sync =
+			repl_state == L_ESTABLISHED &&
 			peer_device->connection->agreed_pro_version >= 90 &&
-			device->ldev->md.uuid[UI_CURRENT] == UUID_JUST_CREATED &&
-			(p_uuid[UI_FLAGS] & 8);
+			drbd_current_uuid(device) == UUID_JUST_CREATED &&
+			(peer_device->uuid_flags & UUID_FLAG_SKIP_INITIAL_SYNC);
 		if (skip_initial_sync) {
+			unsigned long irq_flags;
+
 			drbd_info(device, "Accepted new current UUID, preparing to skip initial sync\n");
-			drbd_bitmap_io(device, &drbd_bmio_clear_n_write,
+			drbd_bitmap_io(device, &drbd_bmio_clear_all_n_write,
 					"clear_n_write from receive_uuids",
-					BM_LOCKED_TEST_ALLOWED, NULL);
-			_drbd_uuid_set(device, UI_CURRENT, p_uuid[UI_CURRENT]);
-			_drbd_uuid_set(device, UI_BITMAP, 0);
-			_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),
-					CS_VERBOSE, NULL);
-			drbd_md_sync(device);
+					BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK, NULL);
+			_drbd_uuid_set_current(device, peer_device->current_uuid);
+			peer_device->comm_current_uuid = peer_device->current_uuid;
+			peer_device->comm_uuid_flags = peer_device->uuid_flags;
+			peer_device->comm_bitmap_uuid = 0;
+			_drbd_uuid_set_bitmap(peer_device, 0);
+			begin_state_change(device->resource, &irq_flags, CS_VERBOSE);
+			__change_disk_state(device, D_UP_TO_DATE);
+			__change_peer_disk_state(peer_device, D_UP_TO_DATE);
+			end_state_change(device->resource, &irq_flags, "skip-initial-sync");
 			updated_uuids = 1;
+			propagate_skip_initial_to_diskless(device);
+		}
+
+		if (peer_device->uuid_flags & UUID_FLAG_NEW_DATAGEN) {
+			drbd_warn(peer_device, "received new current UUID: %016llX "
+				  "weak_nodes=%016llX\n", peer_device->current_uuid, node_mask);
+			drbd_uuid_received_new_current(peer_device, peer_device->current_uuid, node_mask);
 		}
+
+		drbd_uuid_detect_finished_resyncs(peer_device);
+
+		drbd_md_sync_if_dirty(device);
 		put_ldev(device);
-	} else if (device->state.disk < D_INCONSISTENT &&
-		   device->state.role == R_PRIMARY) {
-		/* I am a diskless primary, the peer just created a new current UUID
-		   for me. */
-		updated_uuids = drbd_set_ed_uuid(device, p_uuid[UI_CURRENT]);
-	}
-
-	/* Before we test for the disk state, we should wait until an eventually
-	   ongoing cluster wide state change is finished. That is important if
-	   we are primary and are detaching from our disk. We need to see the
-	   new disk state... */
-	mutex_lock(device->state_mutex);
-	mutex_unlock(device->state_mutex);
-	if (device->state.conn >= C_CONNECTED && device->state.disk < D_INCONSISTENT)
-		updated_uuids |= drbd_set_ed_uuid(device, p_uuid[UI_CURRENT]);
+	} else if (device->disk_state[NOW] < D_INCONSISTENT && repl_state >= L_ESTABLISHED &&
+		   peer_device->disk_state[NOW] == D_UP_TO_DATE && !uuid_match &&
+		   (resource->role[NOW] == R_SECONDARY ||
+		    (two_primaries_allowed && test_and_clear_bit(NEW_CUR_UUID, &device->flags)))) {
+
+		write_lock_irq(&resource->state_rwlock);
+		if (resource->remote_state_change) {
+			drbd_info(peer_device, "Delaying update of exposed data uuid\n");
+			device->next_exposed_data_uuid = peer_device->current_uuid;
+		} else {
+			updated_uuids =
+				drbd_uuid_set_exposed(device, peer_device->current_uuid, false);
+		}
+		write_unlock_irq(&resource->state_rwlock);
+
+	}
+
+	if (device->disk_state[NOW] == D_DISKLESS && uuid_match &&
+	    peer_device->disk_state[NOW] == D_CONSISTENT) {
+		drbd_info(peer_device, "Peer is on same UUID now\n");
+		change_peer_disk_state(peer_device, D_UP_TO_DATE, CS_VERBOSE, "receive-uuids");
+	}
 
 	if (updated_uuids)
-		drbd_print_uuids(device, "receiver updated UUIDs to");
+		drbd_print_uuids(peer_device, "receiver updated UUIDs to");
 
-	return 0;
+	peer_device->uuid_node_mask = node_mask;
+
+	if ((repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) &&
+	    !(peer_device->uuid_flags & UUID_FLAG_STABLE) &&
+	    !drbd_stable_sync_source_present(peer_device, NOW))
+		set_bit(UNSTABLE_RESYNC, &peer_device->flags);
+
+	/* send notification in case UUID flags have changed */
+	drbd_broadcast_peer_device_state(peer_device);
+
+	return err;
 }
 
-/**
- * convert_state() - Converts the peer's view of the cluster state to our point of view
- * @ps:		The state as seen by the peer.
- */
-static union drbd_state convert_state(union drbd_state ps)
+/* drbd 8.4 compat */
+static int receive_uuids(struct drbd_connection *connection, struct packet_info *pi)
 {
-	union drbd_state ms;
-
-	static enum drbd_conns c_tab[] = {
-		[C_WF_REPORT_PARAMS] = C_WF_REPORT_PARAMS,
-		[C_CONNECTED] = C_CONNECTED,
+	const int node_id = connection->resource->res_opts.node_id;
+	struct drbd_peer_device *peer_device;
+	struct p_uuids *p = pi->data;
+	int history_uuids, i;
 
-		[C_STARTING_SYNC_S] = C_STARTING_SYNC_T,
-		[C_STARTING_SYNC_T] = C_STARTING_SYNC_S,
-		[C_DISCONNECTING] = C_TEAR_DOWN, /* C_NETWORK_FAILURE, */
-		[C_VERIFY_S]       = C_VERIFY_T,
-		[C_MASK]   = C_MASK,
-	};
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return config_unknown_volume(connection, pi);
 
-	ms.i = ps.i;
+	history_uuids = min_t(int, HISTORY_UUIDS_V08,
+			      ARRAY_SIZE(peer_device->history_uuids));
 
-	ms.conn = c_tab[ps.conn];
-	ms.peer = ps.role;
-	ms.role = ps.peer;
-	ms.pdsk = ps.disk;
-	ms.disk = ps.pdsk;
-	ms.peer_isp = (ps.aftr_isp | ps.user_isp);
+	peer_device->current_uuid = be64_to_cpu(p->current_uuid);
+	peer_device->bitmap_uuids[node_id] = be64_to_cpu(p->bitmap_uuid);
+	for (i = 0; i < history_uuids; i++)
+		peer_device->history_uuids[i] = be64_to_cpu(p->history_uuids[i]);
+	for (; i < ARRAY_SIZE(peer_device->history_uuids); i++)
+		peer_device->history_uuids[i] = 0;
+	peer_device->dirty_bits = be64_to_cpu(p->dirty_bits);
+	peer_device->uuid_flags = be64_to_cpu(p->uuid_flags) | UUID_FLAG_STABLE;
+	set_bit(UUIDS_RECEIVED, &peer_device->flags);
 
-	return ms;
+	return __receive_uuids(peer_device, 0);
 }
 
-static int receive_req_state(struct drbd_connection *connection, struct packet_info *pi)
+static int receive_uuids110(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct drbd_peer_device *peer_device;
+	struct p_uuids110 *p = pi->data;
+	int bitmap_uuids, history_uuids, rest, i, pos, err;
+	u64 bitmap_uuids_mask, node_mask;
+	struct drbd_peer_md *peer_md = NULL;
 	struct drbd_device *device;
-	struct p_req_state *p = pi->data;
-	union drbd_state mask, val;
-	enum drbd_state_rv rv;
+	int not_allocated = -1;
+
 
 	peer_device = conn_peer_device(connection, pi->vnr);
 	if (!peer_device)
-		return -EIO;
+		return config_unknown_volume(connection, pi);
+
 	device = peer_device->device;
+	bitmap_uuids_mask = be64_to_cpu(p->bitmap_uuids_mask);
+	if (bitmap_uuids_mask & ~(NODE_MASK(DRBD_PEERS_MAX) - 1))
+		return -EIO;
+	bitmap_uuids = hweight64(bitmap_uuids_mask);
 
-	mask.i = be32_to_cpu(p->mask);
-	val.i = be32_to_cpu(p->val);
+	if (pi->size / sizeof(p->other_uuids[0]) < bitmap_uuids)
+		return -EIO;
+	history_uuids = pi->size / sizeof(p->other_uuids[0]) - bitmap_uuids;
+	if (history_uuids > ARRAY_SIZE(peer_device->history_uuids))
+		history_uuids = ARRAY_SIZE(peer_device->history_uuids);
 
-	if (test_bit(RESOLVE_CONFLICTS, &peer_device->connection->flags) &&
-	    mutex_is_locked(device->state_mutex)) {
-		drbd_send_sr_reply(peer_device, SS_CONCURRENT_ST_CHG);
-		return 0;
+	err = drbd_recv_into(connection, p->other_uuids,
+			     (bitmap_uuids + history_uuids) *
+			     sizeof(p->other_uuids[0]));
+	if (err)
+		return err;
+
+	rest = pi->size - (bitmap_uuids + history_uuids) * sizeof(p->other_uuids[0]);
+	if (rest) {
+		err = ignore_remaining_packet(connection, rest);
+		if (err)
+			return err;
 	}
 
-	mask = convert_state(mask);
-	val = convert_state(val);
+	if (get_ldev(device)) {
+		peer_md = device->ldev->md.peers;
+		spin_lock_irq(&device->ldev->md.uuid_lock);
+	}
 
-	rv = drbd_change_state(device, CS_VERBOSE, mask, val);
-	drbd_send_sr_reply(peer_device, rv);
+	if (device->resource->role[NOW] != R_PRIMARY ||
+	    device->disk_state[NOW] != D_DISKLESS ||
+	    (peer_device->current_uuid & ~UUID_PRIMARY) !=
+						(device->exposed_data_uuid & ~UUID_PRIMARY) ||
+	    (peer_device->comm_current_uuid & ~UUID_PRIMARY) !=
+						(device->exposed_data_uuid & ~UUID_PRIMARY))
+		peer_device->current_uuid = be64_to_cpu(p->current_uuid);
 
-	drbd_md_sync(device);
+	peer_device->dirty_bits = be64_to_cpu(p->dirty_bits);
+	peer_device->uuid_flags = be64_to_cpu(p->uuid_flags);
+	if (peer_device->uuid_flags & UUID_FLAG_HAS_UNALLOC) {
+		not_allocated = peer_device->uuid_flags >> UUID_FLAG_UNALLOC_SHIFT;
+		peer_device->uuid_flags &= ~UUID_FLAG_UNALLOC_MASK;
+	}
 
-	return 0;
-}
+	pos = 0;
+	for (i = 0; i < ARRAY_SIZE(peer_device->bitmap_uuids); i++) {
+		u64 bitmap_uuid;
 
-static int receive_req_conn_state(struct drbd_connection *connection, struct packet_info *pi)
-{
-	struct p_req_state *p = pi->data;
-	union drbd_state mask, val;
-	enum drbd_state_rv rv;
+		if (bitmap_uuids_mask & NODE_MASK(i)) {
+			bitmap_uuid = be64_to_cpu(p->other_uuids[pos++]);
 
-	mask.i = be32_to_cpu(p->mask);
-	val.i = be32_to_cpu(p->val);
+			if (peer_md && !(peer_md[i].flags & MDF_HAVE_BITMAP) &&
+			    i != not_allocated)
+				peer_md[i].flags |= MDF_NODE_EXISTS;
+		} else {
+			bitmap_uuid = -1;
+		}
 
-	if (test_bit(RESOLVE_CONFLICTS, &connection->flags) &&
-	    mutex_is_locked(&connection->cstate_mutex)) {
-		conn_send_sr_reply(connection, SS_CONCURRENT_ST_CHG);
-		return 0;
+		update_bitmap_slot_of_peer(peer_device, i, bitmap_uuid);
 	}
 
-	mask = convert_state(mask);
-	val = convert_state(val);
+	for (i = 0; i < history_uuids; i++)
+		peer_device->history_uuids[i] = be64_to_cpu(p->other_uuids[pos++]);
+	while (i < ARRAY_SIZE(peer_device->history_uuids))
+		peer_device->history_uuids[i++] = 0;
+	set_bit(UUIDS_RECEIVED, &peer_device->flags);
+	if (peer_md) {
+		spin_unlock_irq(&device->ldev->md.uuid_lock);
+		put_ldev(device);
+	}
 
-	rv = conn_request_state(connection, mask, val, CS_VERBOSE | CS_LOCAL_ONLY | CS_IGN_OUTD_FAIL);
-	conn_send_sr_reply(connection, rv);
+	node_mask = be64_to_cpu(p->node_mask);
 
-	return 0;
-}
+	if (peer_device->connection->peer_role[NOW] == R_PRIMARY &&
+	    peer_device->uuid_flags & UUID_FLAG_STABLE)
+		check_resync_source(device, node_mask);
 
-static int receive_state(struct drbd_connection *connection, struct packet_info *pi)
-{
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_state *p = pi->data;
-	union drbd_state os, ns, peer_state;
-	enum drbd_disk_state real_peer_disk;
-	enum chg_state_flags cs_flags;
-	int rv;
+	err = __receive_uuids(peer_device, node_mask);
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return config_unknown_volume(connection, pi);
-	device = peer_device->device;
+	if (!test_bit(RECONCILIATION_RESYNC, &peer_device->flags)) {
+		if (peer_device->uuid_flags & UUID_FLAG_GOT_STABLE) {
+			struct drbd_device *device = peer_device->device;
 
-	peer_state.i = be32_to_cpu(p->state);
+			if (peer_device->repl_state[NOW] == L_ESTABLISHED &&
+			    drbd_device_stable(device, NULL) && get_ldev(device)) {
+				drbd_send_uuids(peer_device, UUID_FLAG_RESYNC, 0);
+				drbd_resync(peer_device, AFTER_UNSTABLE);
+				put_ldev(device);
+			}
+		}
 
-	real_peer_disk = peer_state.disk;
-	if (peer_state.disk == D_NEGOTIATING) {
-		real_peer_disk = device->p_uuid[UI_FLAGS] & 4 ? D_INCONSISTENT : D_CONSISTENT;
-		drbd_info(device, "real peer disk state = %s\n", drbd_disk_str(real_peer_disk));
+		if (peer_device->uuid_flags & UUID_FLAG_RESYNC) {
+			if (get_ldev(device)) {
+				bool dp = peer_device->uuid_flags & UUID_FLAG_DISKLESS_PRIMARY;
+				drbd_resync(peer_device, dp ? DISKLESS_PRIMARY : AFTER_UNSTABLE);
+				put_ldev(device);
+			}
+		}
 	}
 
-	spin_lock_irq(&device->resource->req_lock);
- retry:
-	os = ns = drbd_read_state(device);
-	spin_unlock_irq(&device->resource->req_lock);
+	return err;
+}
 
-	/* If some other part of the code (ack_receiver thread, timeout)
-	 * already decided to close the connection again,
-	 * we must not "re-establish" it here. */
-	if (os.conn <= C_TEAR_DOWN)
-		return -ECONNRESET;
-
-	/* If this is the "end of sync" confirmation, usually the peer disk
-	 * transitions from D_INCONSISTENT to D_UP_TO_DATE. For empty (0 bits
-	 * set) resync started in PausedSyncT, or if the timing of pause-/
-	 * unpause-sync events has been "just right", the peer disk may
-	 * transition from D_CONSISTENT to D_UP_TO_DATE as well.
-	 */
-	if ((os.pdsk == D_INCONSISTENT || os.pdsk == D_CONSISTENT) &&
-	    real_peer_disk == D_UP_TO_DATE &&
-	    os.conn > C_CONNECTED && os.disk == D_UP_TO_DATE) {
-		/* If we are (becoming) SyncSource, but peer is still in sync
-		 * preparation, ignore its uptodate-ness to avoid flapping, it
-		 * will change to inconsistent once the peer reaches active
-		 * syncing states.
-		 * It may have changed syncer-paused flags, however, so we
-		 * cannot ignore this completely. */
-		if (peer_state.conn > C_CONNECTED &&
-		    peer_state.conn < C_SYNC_SOURCE)
-			real_peer_disk = D_INCONSISTENT;
+/**
+ * check_resync_source() - Abort resync if the source is weak
+ * @device: The device to check
+ * @weak_nodes: Mask of currently weak nodes in the cluster
+ *
+ * If a primary loses connection to a SYNC_SOURCE node from us, then we
+ * need to abort that resync. Why?
+ *
+ * When the primary sends a write, we get that and write that as well. With
+ * the peer_ack packet, we will set that as out-of-sync towards the sync
+ * source node.
+ * When the resync process finds such bits, we request outdated
+ * data from the sync source!
+ * We are stopping the resync from such an outdated source here and waiting
+ * until all the resync activity has drained (P_RS_DATA_REPLY packets).
+ */
+static void check_resync_source(struct drbd_device *device, u64 weak_nodes)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_connection *connection;
 
-		/* if peer_state changes to connected at the same time,
-		 * it explicitly notifies us that it finished resync.
-		 * Maybe we should finish it up, too? */
-		else if (os.conn >= C_SYNC_SOURCE &&
-			 peer_state.conn == C_CONNECTED) {
-			if (drbd_bm_total_weight(device) <= device->rs_failed)
-				drbd_resync_finished(peer_device);
-			return 0;
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+		if ((repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) &&
+		    NODE_MASK(peer_device->node_id) & weak_nodes) {
+			rcu_read_unlock();
+			goto abort;
 		}
 	}
+	rcu_read_unlock();
+	return;
+abort:
+	connection = peer_device->connection;
+	drbd_info(peer_device, "My sync source became a weak node, aborting resync!\n");
+	change_repl_state(peer_device, L_ESTABLISHED, CS_VERBOSE, "abort-resync");
+	drbd_flush_workqueue(&connection->sender_work);
+	drbd_cancel_conflicting_resync_requests(peer_device);
 
-	/* explicit verify finished notification, stop sector reached. */
-	if (os.conn == C_VERIFY_T && os.disk == D_UP_TO_DATE &&
-	    peer_state.conn == C_CONNECTED && real_peer_disk == D_UP_TO_DATE) {
-		ov_out_of_sync_print(peer_device);
-		drbd_resync_finished(peer_device);
-		return 0;
-	}
+	wait_event_interruptible(connection->ee_wait,
+				 peer_device->repl_state[NOW] <= L_ESTABLISHED ||
+				 atomic_read(&connection->backing_ee_cnt) == 0);
+	wait_event_interruptible(device->misc_wait,
+				 peer_device->repl_state[NOW] <= L_ESTABLISHED ||
+				 atomic_read(&peer_device->rs_pending_cnt) == 0);
 
-	/* peer says his disk is inconsistent, while we think it is uptodate,
-	 * and this happens while the peer still thinks we have a sync going on,
-	 * but we think we are already done with the sync.
-	 * We ignore this to avoid flapping pdsk.
-	 * This should not happen, if the peer is a recent version of drbd. */
-	if (os.pdsk == D_UP_TO_DATE && real_peer_disk == D_INCONSISTENT &&
-	    os.conn == C_CONNECTED && peer_state.conn > C_SYNC_SOURCE)
-		real_peer_disk = D_UP_TO_DATE;
+	peer_device->rs_total  = 0;
+	peer_device->rs_failed = 0;
+	peer_device->rs_paused = 0;
+}
 
-	if (ns.conn == C_WF_REPORT_PARAMS)
-		ns.conn = C_CONNECTED;
+/**
+ * convert_state() - Converts the peer's view of the cluster state to our point of view
+ * @peer_state:	The state as seen by the peer.
+ */
+static union drbd_state convert_state(union drbd_state peer_state)
+{
+	union drbd_state state;
 
-	if (peer_state.conn == C_AHEAD)
-		ns.conn = C_BEHIND;
+	static unsigned int c_tab[] = {
+		[L_OFF] = L_OFF,
+		[L_ESTABLISHED] = L_ESTABLISHED,
 
-	/* TODO:
-	 * if (primary and diskless and peer uuid != effective uuid)
-	 *     abort attach on peer;
-	 *
-	 * If this node does not have good data, was already connected, but
-	 * the peer did a late attach only now, trying to "negotiate" with me,
-	 * AND I am currently Primary, possibly frozen, with some specific
-	 * "effective" uuid, this should never be reached, really, because
-	 * we first send the uuids, then the current state.
-	 *
-	 * In this scenario, we already dropped the connection hard
-	 * when we received the unsuitable uuids (receive_uuids().
-	 *
-	 * Should we want to change this, that is: not drop the connection in
-	 * receive_uuids() already, then we would need to add a branch here
-	 * that aborts the attach of "unsuitable uuids" on the peer in case
-	 * this node is currently Diskless Primary.
-	 */
+		[L_STARTING_SYNC_S] = L_STARTING_SYNC_T,
+		[L_STARTING_SYNC_T] = L_STARTING_SYNC_S,
+		[L_WF_BITMAP_S] = L_WF_BITMAP_T,
+		[L_WF_BITMAP_T] = L_WF_BITMAP_S,
+		[C_DISCONNECTING] = C_TEAR_DOWN, /* C_NETWORK_FAILURE, */
+		[C_CONNECTING] = C_CONNECTING,
+		[L_VERIFY_S] = L_VERIFY_T,
+		[C_MASK] = C_MASK,
+	};
 
-	if (device->p_uuid && peer_state.disk >= D_NEGOTIATING &&
-	    get_ldev_if_state(device, D_NEGOTIATING)) {
-		int cr; /* consider resync */
+	state.i = peer_state.i;
 
-		/* if we established a new connection */
-		cr  = (os.conn < C_CONNECTED);
-		/* if we had an established connection
-		 * and one of the nodes newly attaches a disk */
-		cr |= (os.conn == C_CONNECTED &&
-		       (peer_state.disk == D_NEGOTIATING ||
-			os.disk == D_NEGOTIATING));
-		/* if we have both been inconsistent, and the peer has been
-		 * forced to be UpToDate with --force */
-		cr |= test_bit(CONSIDER_RESYNC, &device->flags);
-		/* if we had been plain connected, and the admin requested to
-		 * start a sync by "invalidate" or "invalidate-remote" */
-		cr |= (os.conn == C_CONNECTED &&
-				(peer_state.conn >= C_STARTING_SYNC_S &&
-				 peer_state.conn <= C_WF_BITMAP_T));
+	state.conn = c_tab[peer_state.conn];
+	state.peer = peer_state.role;
+	state.role = peer_state.peer;
+	state.pdsk = peer_state.disk;
+	state.disk = peer_state.pdsk;
+	state.peer_isp = (peer_state.aftr_isp | peer_state.user_isp);
 
-		if (cr)
-			ns.conn = drbd_sync_handshake(peer_device, peer_state.role, real_peer_disk);
+	return state;
+}
 
-		put_ldev(device);
-		if (ns.conn == C_MASK) {
-			ns.conn = C_CONNECTED;
-			if (device->state.disk == D_NEGOTIATING) {
-				drbd_force_state(device, NS(disk, D_FAILED));
-			} else if (peer_state.disk == D_NEGOTIATING) {
-				drbd_err(device, "Disk attach process on the peer node was aborted.\n");
-				peer_state.disk = D_DISKLESS;
-				real_peer_disk = D_DISKLESS;
-			} else {
-				if (test_and_clear_bit(CONN_DRY_RUN, &peer_device->connection->flags))
-					return -EIO;
-				D_ASSERT(device, os.conn == C_WF_REPORT_PARAMS);
-				conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
-				return -EIO;
-			}
-		}
-	}
+static enum drbd_state_rv
+__change_connection_state(struct drbd_connection *connection,
+			  union drbd_state mask, union drbd_state val,
+			  enum chg_state_flags flags)
+{
+	struct drbd_resource *resource = connection->resource;
 
-	spin_lock_irq(&device->resource->req_lock);
-	if (os.i != drbd_read_state(device).i)
-		goto retry;
-	clear_bit(CONSIDER_RESYNC, &device->flags);
-	ns.peer = peer_state.role;
-	ns.pdsk = real_peer_disk;
-	ns.peer_isp = (peer_state.aftr_isp | peer_state.user_isp);
-	if ((ns.conn == C_CONNECTED || ns.conn == C_WF_BITMAP_S) && ns.disk == D_NEGOTIATING)
-		ns.disk = device->new_state_tmp.disk;
-	cs_flags = CS_VERBOSE + (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED ? 0 : CS_HARD);
-	if (ns.pdsk == D_CONSISTENT && drbd_suspended(device) && ns.conn == C_CONNECTED && os.conn < C_CONNECTED &&
-	    test_bit(NEW_CUR_UUID, &device->flags)) {
-		/* Do not allow tl_restart(RESEND) for a rebooted peer. We can only allow this
-		   for temporal network outages! */
-		spin_unlock_irq(&device->resource->req_lock);
-		drbd_err(device, "Aborting Connect, can not thaw IO with an only Consistent peer\n");
-		tl_clear(peer_device->connection);
-		drbd_uuid_new_current(device);
-		clear_bit(NEW_CUR_UUID, &device->flags);
-		conn_request_state(peer_device->connection, NS2(conn, C_PROTOCOL_ERROR, susp, 0), CS_HARD);
-		return -EIO;
+	if (mask.role) {
+		/* not allowed */
 	}
-	rv = _drbd_set_state(device, ns, cs_flags, NULL);
-	ns = drbd_read_state(device);
-	spin_unlock_irq(&device->resource->req_lock);
+	if (mask.susp) {
+		mask.susp ^= -1;
+		__change_io_susp_user(resource, val.susp);
+	}
+	if (mask.susp_nod) {
+		mask.susp_nod ^= -1;
+		__change_io_susp_no_data(resource, val.susp_nod);
+	}
+	if (mask.susp_fen) {
+		mask.susp_fen ^= -1;
+		__change_io_susp_fencing(connection, val.susp_fen);
+	}
+	if (mask.disk) {
+		/* Handled in __change_peer_device_state(). */
+		mask.disk ^= -1;
+	}
+	if (mask.conn) {
+		mask.conn ^= -1;
+		__change_cstate(connection,
+				min_t(enum drbd_conn_state, val.conn, C_CONNECTED));
+	}
+	if (mask.pdsk) {
+		/* Handled in __change_peer_device_state(). */
+		mask.pdsk ^= -1;
+	}
+	if (mask.peer) {
+		mask.peer ^= -1;
+		__change_peer_role(connection, val.peer);
+	}
+	if (mask.i) {
+		drbd_info(connection, "Remote state change: request %u/%u not "
+		"understood\n", mask.i, val.i & mask.i);
+		return SS_NOT_SUPPORTED;
+	}
+	return SS_SUCCESS;
+}
 
-	if (rv < SS_SUCCESS) {
-		conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD);
-		return -EIO;
+static enum drbd_state_rv
+__change_peer_device_state(struct drbd_peer_device *peer_device,
+			   union drbd_state mask, union drbd_state val)
+{
+	struct drbd_device *device = peer_device->device;
+
+	if (mask.peer) {
+		/* Handled in __change_connection_state(). */
+		mask.peer ^= -1;
+	}
+	if (mask.disk) {
+		mask.disk ^= -1;
+		__change_disk_state(device, val.disk);
 	}
 
-	if (os.conn > C_WF_REPORT_PARAMS) {
-		if (ns.conn > C_CONNECTED && peer_state.conn <= C_CONNECTED &&
-		    peer_state.disk != D_NEGOTIATING ) {
-			/* we want resync, peer has not yet decided to sync... */
-			/* Nowadays only used when forcing a node into primary role and
-			   setting its disk to UpToDate with that */
-			drbd_send_uuids(peer_device);
-			drbd_send_current_state(peer_device);
-		}
+	if (mask.conn) {
+		mask.conn ^= -1;
+		__change_repl_state(peer_device,
+				max_t(enum drbd_repl_state, val.conn, L_OFF));
+	}
+	if (mask.pdsk) {
+		mask.pdsk ^= -1;
+		__change_peer_disk_state(peer_device, val.pdsk);
+	}
+	if (mask.user_isp) {
+		mask.user_isp ^= -1;
+		__change_resync_susp_user(peer_device, val.user_isp);
+	}
+	if (mask.peer_isp) {
+		mask.peer_isp ^= -1;
+		__change_resync_susp_peer(peer_device, val.peer_isp);
+	}
+	if (mask.aftr_isp) {
+		mask.aftr_isp ^= -1;
+		__change_resync_susp_dependency(peer_device, val.aftr_isp);
 	}
+	if (mask.i) {
+		drbd_info(peer_device, "Remote state change: request %u/%u not "
+		"understood\n", mask.i, val.i & mask.i);
+		return SS_NOT_SUPPORTED;
+	}
+	return SS_SUCCESS;
+}
 
-	clear_bit(DISCARD_MY_DATA, &device->flags);
+static union drbd_state
+sanitize_outdate(struct drbd_peer_device *peer_device,
+		 union drbd_state mask,
+		 union drbd_state val)
+{
+	struct drbd_device *device = peer_device->device;
+	union drbd_state result_mask = mask;
 
-	drbd_md_sync(device); /* update connected indicator, la_size_sect, ... */
+	if (val.pdsk == D_OUTDATED && peer_device->disk_state[NEW] < D_OUTDATED)
+		result_mask.pdsk = 0;
+	if (val.disk == D_OUTDATED && device->disk_state[NEW] < D_OUTDATED)
+		result_mask.disk = 0;
 
-	return 0;
+	return result_mask;
 }
 
-static int receive_sync_uuid(struct drbd_connection *connection, struct packet_info *pi)
+static void log_openers(struct drbd_resource *resource)
 {
-	struct drbd_peer_device *peer_device;
 	struct drbd_device *device;
-	struct p_rs_uuid *p = pi->data;
+	int vnr;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct opener *opener;
 
-	wait_event(device->misc_wait,
-		   device->state.conn == C_WF_SYNC_UUID ||
-		   device->state.conn == C_BEHIND ||
-		   device->state.conn < C_CONNECTED ||
-		   device->state.disk < D_NEGOTIATING);
+		spin_lock(&device->openers_lock);
+		opener = list_first_entry_or_null(&device->openers, struct opener, list);
+		if (opener)
+			drbd_warn(device, "Held open by %s(%d)\n", opener->comm, opener->pid);
+		spin_unlock(&device->openers_lock);
+	}
+	rcu_read_unlock();
+}
 
-	/* D_ASSERT(device,  device->state.conn == C_WF_SYNC_UUID ); */
+/**
+ * change_connection_state()  -  change state of a connection and all its peer devices
+ * @connection: DRBD connection to operate on
+ * @state_change: Prepared state change
+ * @reply: Two phase commit reply
+ * @flags: State change flags
+ *
+ * Also changes the state of the peer devices' devices and of the resource.
+ * Cluster-wide state changes are not supported.
+ */
+static enum drbd_state_rv
+change_connection_state(struct drbd_connection *connection,
+			struct twopc_state_change *state_change,
+			struct twopc_reply *reply,
+			enum chg_state_flags flags)
+{
+	struct drbd_resource *resource = connection->resource;
+	long t = resource->res_opts.auto_promote_timeout * HZ / 10;
+	union drbd_state mask = state_change->mask;
+	union drbd_state val = state_change->val;
+	bool is_disconnect = false;
+	bool is_connect = false;
+	bool abort = flags & CS_ABORT;
+	struct drbd_peer_device *peer_device;
+	unsigned long irq_flags;
+	enum drbd_state_rv rv;
+	int vnr;
 
-	/* Here the _drbd_uuid_ functions are right, current should
-	   _not_ be rotated into the history */
-	if (get_ldev_if_state(device, D_NEGOTIATING)) {
-		_drbd_uuid_set(device, UI_CURRENT, be64_to_cpu(p->uuid));
-		_drbd_uuid_set(device, UI_BITMAP, 0UL);
+	if (reply) {
+		is_disconnect = reply->is_disconnect;
+		is_connect = reply->is_connect;
+	} else if (mask.conn == conn_MASK) {
+		is_connect = val.conn == C_CONNECTED;
+		is_disconnect = val.conn == C_DISCONNECTING;
+	}
 
-		drbd_print_uuids(device, "updated sync uuid");
-		drbd_start_resync(device, C_SYNC_TARGET);
+	mask = convert_state(mask);
+	val = convert_state(val);
 
-		put_ldev(device);
-	} else
-		drbd_err(device, "Ignoring SyncUUID packet!\n");
+	if (is_connect && connection->agreed_pro_version >= 118) {
+		if (flags & CS_PREPARE)
+			conn_connect2(connection);
+		if (abort)
+			abort_connect(connection);
+	}
+retry:
+	begin_state_change(resource, &irq_flags, flags & ~CS_VERBOSE);
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		union drbd_state l_mask;
+		l_mask = is_disconnect ? sanitize_outdate(peer_device, mask, val) : mask;
+		rv = __change_peer_device_state(peer_device, l_mask, val);
+		if (rv < SS_SUCCESS)
+			goto fail;
+	}
+	rv = __change_connection_state(connection, mask, val, flags);
+	if (rv < SS_SUCCESS)
+		goto fail;
 
-	return 0;
+	if (reply && !abort) {
+		u64 directly_reachable = directly_connected_nodes(resource, NEW) |
+			NODE_MASK(resource->res_opts.node_id);
+
+		if (reply->primary_nodes & ~directly_reachable)
+			__outdate_myself(resource);
+	}
+
+	if (is_connect && connection->agreed_pro_version >= 117)
+		apply_connect(connection, (flags & CS_PREPARED) && !abort);
+	rv = end_state_change(resource, &irq_flags, "remote");
+out:
+
+	if ((rv == SS_NO_UP_TO_DATE_DISK && resource->role[NOW] != R_PRIMARY) ||
+	    rv == SS_PRIMARY_READER) {
+		/* Most probably udev opened it read-only. That might happen
+		   if it was demoted very recently. Wait up to one second. */
+		t = wait_event_interruptible_timeout(resource->state_wait,
+						     drbd_open_ro_count(resource) == 0,
+						     t);
+		if (t > 0)
+			goto retry;
+	}
+
+	if (rv < SS_SUCCESS) {
+		drbd_err(resource, "State change failed: %s (%d)\n", drbd_set_st_err_str(rv), rv);
+		if (rv == SS_PRIMARY_READER)
+			log_openers(resource);
+	}
+
+	return rv;
+fail:
+	abort_state_change(resource, &irq_flags);
+	goto out;
 }
 
-/*
- * receive_bitmap_plain
+/**
+ * change_peer_device_state()  -  change state of a peer and its connection
+ * @peer_device: DRBD peer device
+ * @state_change: Prepared state change
+ * @flags: State change flags
  *
- * Return 0 when done, 1 when another iteration is needed, and a negative error
- * code upon failure.
+ * Also changes the state of the peer device's device and of the resource.
+ * Cluster-wide state changes are not supported.
  */
-static int
-receive_bitmap_plain(struct drbd_peer_device *peer_device, unsigned int size,
-		     unsigned long *p, struct bm_xfer_ctx *c)
+static enum drbd_state_rv
+change_peer_device_state(struct drbd_peer_device *peer_device,
+			 struct twopc_state_change *state_change,
+			 enum chg_state_flags flags)
 {
-	unsigned int data_size = DRBD_SOCKET_BUFFER_SIZE -
-				 drbd_header_size(peer_device->connection);
-	unsigned int num_words = min_t(size_t, data_size / sizeof(*p),
-				       c->bm_words - c->word_offset);
-	unsigned int want = num_words * sizeof(*p);
-	int err;
+	struct drbd_connection *connection = peer_device->connection;
+	union drbd_state mask = state_change->mask;
+	union drbd_state val = state_change->val;
+	unsigned long irq_flags;
+	enum drbd_state_rv rv;
 
-	if (want != size) {
-		drbd_err(peer_device, "%s:want (%u) != size (%u)\n", __func__, want, size);
+	mask = convert_state(mask);
+	val = convert_state(val);
+
+	begin_state_change(connection->resource, &irq_flags, flags);
+	rv = __change_peer_device_state(peer_device, mask, val);
+	if (rv < SS_SUCCESS)
+		goto fail;
+	rv = __change_connection_state(connection, mask, val, flags);
+	if (rv < SS_SUCCESS)
+		goto fail;
+	rv = end_state_change(connection->resource, &irq_flags, "remote");
+out:
+	return rv;
+fail:
+	abort_state_change(connection->resource, &irq_flags);
+	goto out;
+}
+
+static int receive_req_state(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct twopc_state_change *state_change = &resource->twopc.state_change;
+	struct drbd_peer_device *peer_device = NULL;
+	struct p_req_state *p = pi->data;
+	enum chg_state_flags flags = CS_VERBOSE | CS_LOCAL_ONLY | CS_TWOPC;
+	enum drbd_state_rv rv;
+	int vnr = -1;
+
+	if (!expect(connection, connection->agreed_pro_version < 110)) {
+		drbd_err(connection, "Packet %s not allowed in protocol version %d\n",
+			 drbd_packet_name(pi->cmd),
+			 connection->agreed_pro_version);
 		return -EIO;
 	}
-	if (want == 0)
+
+	state_change->mask.i = be32_to_cpu(p->mask);
+	state_change->val.i = be32_to_cpu(p->val);
+
+	/* P_STATE_CHG_REQ packets must have a valid vnr.  P_CONN_ST_CHG_REQ
+	 * packets have an undefined vnr. */
+	if (pi->cmd == P_STATE_CHG_REQ) {
+		peer_device = conn_peer_device(connection, pi->vnr);
+		if (!peer_device) {
+			const union drbd_state conn_mask = { .conn = conn_MASK };
+			const union drbd_state val_off = { .conn = L_OFF };
+
+			if (state_change->mask.i == conn_mask.i &&
+			    state_change->val.i == val_off.i) {
+				/* The peer removed this volume, we do not have it... */
+				drbd_send_sr_reply(connection, vnr, SS_NOTHING_TO_DO);
+				return 0;
+			}
+
+			return -EIO;
+		}
+		vnr = peer_device->device->vnr;
+	}
+
+	rv = SS_SUCCESS;
+	write_lock_irq(&resource->state_rwlock);
+	if (resource->remote_state_change)
+		rv = SS_CONCURRENT_ST_CHG;
+	else
+		resource->remote_state_change = true;
+	write_unlock_irq(&resource->state_rwlock);
+
+	if (rv != SS_SUCCESS) {
+		drbd_info(connection, "Rejecting concurrent remote state change\n");
+		drbd_send_sr_reply(connection, vnr, rv);
 		return 0;
-	err = drbd_recv_all(peer_device->connection, p, want);
-	if (err)
-		return err;
+	}
 
-	drbd_bm_merge_lel(peer_device->device, c->word_offset, num_words, p);
+	/* Send the reply before carrying out the state change: this is needed
+	 * for connection state changes which close the network connection.  */
+	if (peer_device) {
+		rv = change_peer_device_state(peer_device, state_change, flags | CS_PREPARE);
+		drbd_send_sr_reply(connection, vnr, rv);
+		rv = change_peer_device_state(peer_device, state_change, flags | CS_PREPARED);
+		if (rv >= SS_SUCCESS)
+			drbd_md_sync_if_dirty(peer_device->device);
+	} else {
+		flags |= CS_IGN_OUTD_FAIL;
+		rv = change_connection_state(connection, state_change, NULL, flags | CS_PREPARE);
+		drbd_send_sr_reply(connection, vnr, rv);
+		change_connection_state(connection, state_change, NULL, flags | CS_PREPARED);
+	}
 
-	c->word_offset += num_words;
-	c->bit_offset = c->word_offset * BITS_PER_LONG;
-	if (c->bit_offset > c->bm_bits)
-		c->bit_offset = c->bm_bits;
+	write_lock_irq(&resource->state_rwlock);
+	resource->remote_state_change = false;
+	write_unlock_irq(&resource->state_rwlock);
+	wake_up_all(&resource->twopc_wait);
 
-	return 1;
+	return 0;
 }
 
-static enum drbd_bitmap_code dcbp_get_code(struct p_compressed_bm *p)
+static void drbd_abort_twopc(struct drbd_resource *resource)
 {
-	return (enum drbd_bitmap_code)(p->encoding & 0x0f);
+	struct drbd_connection *connection;
+	int initiator_node_id;
+	bool is_connect;
+
+	initiator_node_id = resource->twopc_reply.initiator_node_id;
+	if (initiator_node_id != -1) {
+		connection = drbd_get_connection_by_node_id(resource, initiator_node_id);
+		is_connect = resource->twopc_reply.is_connect &&
+			resource->twopc_reply.target_node_id == resource->res_opts.node_id;
+		resource->remote_state_change = false;
+		resource->twopc_reply.initiator_node_id = -1;
+		resource->twopc_parent_nodes = 0;
+
+		if (connection) {
+			if (is_connect)
+				abort_connect(connection);
+			kref_put(&connection->kref, drbd_destroy_connection);
+			connection = NULL;
+		}
+
+		/* Aborting a prepared state change. Give up the state mutex! */
+		up(&resource->state_sem);
+	}
+
+	wake_up_all(&resource->twopc_wait);
 }
 
-static int dcbp_get_start(struct p_compressed_bm *p)
+void twopc_timer_fn(struct timer_list *t)
 {
-	return (p->encoding & 0x80) != 0;
+	struct drbd_resource *resource = timer_container_of(resource, t, twopc_timer);
+	unsigned long irq_flags;
+
+	write_lock_irqsave(&resource->state_rwlock, irq_flags);
+	if (!test_bit(TWOPC_WORK_PENDING, &resource->flags)) {
+		drbd_err(resource, "Two-phase commit %u timeout\n",
+			   resource->twopc_reply.tid);
+		drbd_abort_twopc(resource);
+	} else {
+		mod_timer(&resource->twopc_timer, jiffies + HZ/10);
+	}
+	write_unlock_irqrestore(&resource->state_rwlock, irq_flags);
 }
 
-static int dcbp_get_pad_bits(struct p_compressed_bm *p)
+bool drbd_have_local_disk(struct drbd_resource *resource)
 {
-	return (p->encoding >> 4) & 0x7;
+	struct drbd_device *device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (device->disk_state[NOW] > D_DISKLESS) {
+			rcu_read_unlock();
+			return true;
+		}
+	}
+	rcu_read_unlock();
+	return false;
 }
 
-/*
- * recv_bm_rle_bits
- *
- * Return 0 when done, 1 when another iteration is needed, and a negative error
- * code upon failure.
- */
-static int
-recv_bm_rle_bits(struct drbd_peer_device *peer_device,
-		struct p_compressed_bm *p,
-		 struct bm_xfer_ctx *c,
-		 unsigned int len)
+static enum drbd_state_rv
+far_away_change(struct drbd_connection *connection,
+		struct twopc_request *request,
+		struct twopc_reply *reply,
+		enum chg_state_flags flags)
 {
-	struct bitstream bs;
-	u64 look_ahead;
-	u64 rl;
-	u64 tmp;
-	unsigned long s = c->bit_offset;
-	unsigned long e;
-	int toggle = dcbp_get_start(p);
-	int have;
-	int bits;
+	struct drbd_resource *resource = connection->resource;
+	struct twopc_state_change *state_change = &resource->twopc.state_change;
+	u64 directly_reachable = directly_connected_nodes(resource, NOW) |
+		NODE_MASK(resource->res_opts.node_id);
+	union drbd_state mask = state_change->mask;
+	union drbd_state val = state_change->val;
+	int vnr = resource->twopc_reply.vnr;
+	struct drbd_device *device;
+	unsigned long irq_flags;
+	int iterate_vnr;
 
-	bitstream_init(&bs, p->code, len, dcbp_get_pad_bits(p));
 
-	bits = bitstream_get_bits(&bs, &look_ahead, 64);
-	if (bits < 0)
-		return -EIO;
+	if (flags & CS_PREPARE && mask.role == role_MASK && val.role == R_PRIMARY &&
+	    resource->role[NOW] == R_PRIMARY) {
+		struct net_conf *nc;
+		bool two_primaries_allowed = false;
 
-	for (have = bits; have > 0; s += rl, toggle = !toggle) {
-		bits = vli_decode_bits(&rl, look_ahead);
-		if (bits <= 0)
-			return -EIO;
+		rcu_read_lock();
+		nc = rcu_dereference(connection->transport.net_conf);
+		if (nc)
+			two_primaries_allowed = nc->two_primaries;
+		rcu_read_unlock();
+		if (!two_primaries_allowed)
+			return SS_TWO_PRIMARIES;
 
-		if (toggle) {
-			e = s + rl -1;
-			if (e >= c->bm_bits) {
-				drbd_err(peer_device, "bitmap overflow (e:%lu) while decoding bm RLE packet\n", e);
-				return -EIO;
+		/* A node further away wants to become primary. In case I am primary allow it only
+		 * when I am diskless. See also check_primaries_distances() in drbd_state.c
+		 */
+		if (drbd_have_local_disk(resource))
+			return SS_WEAKLY_CONNECTED;
+	}
+
+	begin_state_change(resource, &irq_flags, flags);
+	if (mask.i == 0 && val.i == 0 &&
+	    resource->role[NOW] == R_PRIMARY && vnr == -1) {
+		/* A node far away test if there are primaries. I am the guy he is concerned
+		 * about... He learned about me in the CS_PREPARE phase. Since he is committing it
+		 * I know that he is outdated now...
+		 */
+		struct drbd_connection *affected_connection;
+		int initiator_node_id = resource->twopc_reply.initiator_node_id;
+
+		affected_connection = drbd_get_connection_by_node_id(resource, initiator_node_id);
+		if (affected_connection) {
+			__downgrade_peer_disk_states(affected_connection, D_OUTDATED);
+			kref_put(&affected_connection->kref, drbd_destroy_connection);
+		} else if (flags & CS_PREPARED) {
+			idr_for_each_entry(&resource->devices, device, iterate_vnr) {
+				struct drbd_peer_md *peer_md;
+
+				if (!get_ldev(device))
+					continue;
+
+				peer_md = &device->ldev->md.peers[initiator_node_id];
+				peer_md->flags |= MDF_PEER_OUTDATED;
+				put_ldev(device);
+				drbd_md_mark_dirty(device);
 			}
-			_drbd_bm_set_bits(peer_device->device, s, e);
 		}
+	}
 
-		if (have < bits) {
-			drbd_err(peer_device, "bitmap decoding error: h:%d b:%d la:0x%08llx l:%u/%u\n",
-				have, bits, look_ahead,
-				(unsigned int)(bs.cur.b - p->code),
-				(unsigned int)bs.buf_len);
-			return -EIO;
-		}
-		/* if we consumed all 64 bits, assign 0; >> 64 is "undefined"; */
-		if (likely(bits < 64))
-			look_ahead >>= bits;
-		else
-			look_ahead = 0;
-		have -= bits;
+	if (state_change->primary_nodes & ~directly_reachable &&
+	    !(request->flags & TWOPC_PRI_INCAPABLE))
+		__outdate_myself(resource);
 
-		bits = bitstream_get_bits(&bs, &tmp, 64 - have);
-		if (bits < 0)
-			return -EIO;
-		look_ahead |= tmp << have;
-		have += bits;
+	idr_for_each_entry(&resource->devices, device, iterate_vnr) {
+		if (test_bit(OUTDATE_ON_2PC_COMMIT, &device->flags) &&
+		    device->disk_state[NEW] > D_OUTDATED)
+			__change_disk_state(device, D_OUTDATED);
 	}
 
-	c->bit_offset = s;
-	bm_xfer_ctx_bit_to_word_offset(c);
-
-	return (s != c->bm_bits);
+	/* even if no outdate happens, CS_FORCE_RECALC might be set here */
+	return end_state_change(resource, &irq_flags, "far-away");
 }
 
-/*
- * decode_bitmap_c
- *
- * Return 0 when done, 1 when another iteration is needed, and a negative error
- * code upon failure.
- */
-static int
-decode_bitmap_c(struct drbd_peer_device *peer_device,
-		struct p_compressed_bm *p,
-		struct bm_xfer_ctx *c,
-		unsigned int len)
+static void handle_neighbor_demotion(struct drbd_connection *connection,
+				     struct twopc_state_change *state_change,
+				     struct twopc_reply *reply)
 {
-	if (dcbp_get_code(p) == RLE_VLI_Bits)
-		return recv_bm_rle_bits(peer_device, p, c, len - sizeof(*p));
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_device *device;
+	int vnr;
 
-	/* other variants had been implemented for evaluation,
-	 * but have been dropped as this one turned out to be "best"
-	 * during all our tests. */
+	if (reply->initiator_node_id != connection->peer_node_id ||
+	    connection->peer_role[NOW] != R_PRIMARY ||
+	    state_change->mask.role != role_MASK || state_change->val.role != R_SECONDARY)
+		return;
 
-	drbd_err(peer_device, "receive_bitmap_c: unknown encoding %u\n", p->encoding);
-	conn_request_state(peer_device->connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
-	return -EIO;
+	/* A directly connected neighbor that was primary demotes to secondary */
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		if (get_ldev(device)) {
+			drbd_bitmap_io(device, &drbd_bm_write, "peer demote",
+				       BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK, NULL);
+			put_ldev(device);
+		}
+		rcu_read_lock();
+		kref_put(&device->kref, drbd_destroy_device);
+	}
+	rcu_read_unlock();
 }
 
-void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
-		const char *direction, struct bm_xfer_ctx *c)
+static void peer_device_init_connect_state(struct drbd_peer_device *peer_device)
 {
-	/* what would it take to transfer it "plaintext" */
-	unsigned int header_size = drbd_header_size(peer_device->connection);
-	unsigned int data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
-	unsigned int plain =
-		header_size * (DIV_ROUND_UP(c->bm_words, data_size) + 1) +
-		c->bm_words * sizeof(unsigned long);
-	unsigned int total = c->bytes[0] + c->bytes[1];
-	unsigned int r;
-
-	/* total can not be zero. but just in case: */
-	if (total == 0)
-		return;
-
-	/* don't report if not compressed */
-	if (total >= plain)
-		return;
+	clear_bit(INITIAL_STATE_SENT, &peer_device->flags);
+	clear_bit(INITIAL_STATE_RECEIVED, &peer_device->flags);
+	clear_bit(HAVE_SIZES, &peer_device->flags);
+	clear_bit(UUIDS_RECEIVED, &peer_device->flags);
+	clear_bit(CURRENT_UUID_RECEIVED, &peer_device->flags);
+	clear_bit(PEER_QUORATE, &peer_device->flags);
+	peer_device->connect_state = (union drbd_state) {{ .disk = D_MASK }};
+}
 
-	/* total < plain. check for overflow, still */
-	r = (total > UINT_MAX/1000) ? (total / (plain/1000))
-		                    : (1000 * total / plain);
 
-	if (r > 1000)
-		r = 1000;
+/**
+ * drbd_init_connect_state() - Prepare twopc that establishes the connection
+ * @connection:	The connection this is about
+ *
+ * After a transport implementation has established the lower-level aspects
+ * of a connection, DRBD executes a two-phase commit so that the membership
+ * information changes in a cluster-wide, consistent way. During that
+ * two-phase commit, DRBD exchanges the UUIDs, size information, and the
+ * initial state. A two-phase commit might be aborted, which needs to be
+ * retried. This function re-initializes the struct members for this. The
+ * callsites are at the beginning of a two-phase connect commit, active and
+ * passive side.
+ */
+void drbd_init_connect_state(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	r = 1000 - r;
-	drbd_info(peer_device, "%s bitmap stats [Bytes(packets)]: plain %u(%u), RLE %u(%u), "
-	     "total %u; compression: %u.%u%%\n",
-			direction,
-			c->bytes[1], c->packets[1],
-			c->bytes[0], c->packets[0],
-			total, r/10, r % 10);
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+		peer_device_init_connect_state(peer_device);
+	rcu_read_unlock();
+	clear_bit(CONN_HANDSHAKE_DISCONNECT, &connection->flags);
+	clear_bit(CONN_HANDSHAKE_RETRY, &connection->flags);
+	clear_bit(CONN_HANDSHAKE_READY, &connection->flags);
 }
 
-/* Since we are processing the bitfield from lower addresses to higher,
-   it does not matter if the process it in 32 bit chunks or 64 bit
-   chunks as long as it is little endian. (Understand it as byte stream,
-   beginning with the lowest byte...) If we would use big endian
-   we would need to process it from the highest address to the lowest,
-   in order to be agnostic to the 32 vs 64 bits issue.
+enum csc_rv {
+	CSC_CLEAR,
+	CSC_REJECT,
+	CSC_ABORT_LOCAL,
+	CSC_TID_MISS,
+	CSC_MATCH,
+};
 
-   returns 0 on failure, 1 if we successfully received it. */
-static int receive_bitmap(struct drbd_connection *connection, struct packet_info *pi)
+static enum csc_rv
+check_concurrent_transactions(struct drbd_resource *resource, struct twopc_reply *new_r)
 {
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct bm_xfer_ctx c;
-	int err;
+	struct twopc_reply *ongoing = &resource->twopc_reply;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	if (!resource->remote_state_change)
+		return CSC_CLEAR;
 
-	drbd_bm_lock(device, "receive bitmap", BM_LOCKED_SET_ALLOWED);
-	/* you are supposed to send additional out-of-sync information
-	 * if you actually set bits during this phase */
+	if (new_r->initiator_node_id < ongoing->initiator_node_id) {
+		if (ongoing->initiator_node_id == resource->res_opts.node_id)
+			return CSC_ABORT_LOCAL;
+		else
+			return CSC_REJECT;
+	} else if (new_r->initiator_node_id > ongoing->initiator_node_id) {
+		return CSC_REJECT;
+	}
+	if (new_r->tid != ongoing->tid)
+		return CSC_TID_MISS;
 
-	c = (struct bm_xfer_ctx) {
-		.bm_bits = drbd_bm_bits(device),
-		.bm_words = drbd_bm_words(device),
-	};
+	return CSC_MATCH;
+}
 
-	for(;;) {
-		if (pi->cmd == P_BITMAP)
-			err = receive_bitmap_plain(peer_device, pi->size, pi->data, &c);
-		else if (pi->cmd == P_COMPRESSED_BITMAP) {
-			/* MAYBE: sanity check that we speak proto >= 90,
-			 * and the feature is enabled! */
-			struct p_compressed_bm *p = pi->data;
 
-			if (pi->size > DRBD_SOCKET_BUFFER_SIZE - drbd_header_size(connection)) {
-				drbd_err(device, "ReportCBitmap packet too large\n");
-				err = -EIO;
-				goto out;
-			}
-			if (pi->size <= sizeof(*p)) {
-				drbd_err(device, "ReportCBitmap packet too small (l:%u)\n", pi->size);
-				err = -EIO;
-				goto out;
-			}
-			err = drbd_recv_all(peer_device->connection, p, pi->size);
-			if (err)
-			       goto out;
-			err = decode_bitmap_c(peer_device, p, &c, pi->size);
-		} else {
-			drbd_warn(device, "receive_bitmap: cmd neither ReportBitMap nor ReportCBitMap (is 0x%x)", pi->cmd);
-			err = -EIO;
-			goto out;
-		}
+enum alt_rv {
+	ALT_LOCKED,
+	ALT_MATCH,
+	ALT_TIMEOUT,
+};
 
-		c.packets[pi->cmd == P_BITMAP]++;
-		c.bytes[pi->cmd == P_BITMAP] += drbd_header_size(connection) + pi->size;
+static enum alt_rv when_done_lock(struct drbd_resource *resource, unsigned int for_tid)
+{
+	write_lock_irq(&resource->state_rwlock);
+	if (!resource->remote_state_change)
+		return ALT_LOCKED;
+	write_unlock_irq(&resource->state_rwlock);
+	if (resource->twopc_reply.tid == for_tid)
+		return ALT_MATCH;
 
-		if (err <= 0) {
-			if (err < 0)
-				goto out;
-			break;
-		}
-		err = drbd_recv_header(peer_device->connection, pi);
-		if (err)
-			goto out;
-	}
+	return ALT_TIMEOUT;
+}
+static enum alt_rv abort_local_transaction(struct drbd_connection *connection, unsigned int for_tid)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct net_conf *nc;
+	enum alt_rv rv;
+	long t;
 
-	INFO_bm_xfer_stats(peer_device, "receive", &c);
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	t = nc->ping_timeo * HZ/10 * 3 / 2;
+	rcu_read_unlock();
 
-	if (device->state.conn == C_WF_BITMAP_T) {
-		enum drbd_state_rv rv;
+	set_bit(TWOPC_ABORT_LOCAL, &resource->flags);
+	write_unlock_irq(&resource->state_rwlock);
+	wake_up_all(&resource->state_wait);
+	wait_event_timeout(resource->twopc_wait,
+			   (rv = when_done_lock(resource, for_tid)) != ALT_TIMEOUT, t);
+	clear_bit(TWOPC_ABORT_LOCAL, &resource->flags);
+	return rv;
+}
 
-		err = drbd_send_bitmap(device, peer_device);
-		if (err)
-			goto out;
-		/* Omit CS_ORDERED with this state transition to avoid deadlocks. */
-		rv = _drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE);
-		D_ASSERT(device, rv == SS_SUCCESS);
-	} else if (device->state.conn != C_WF_BITMAP_S) {
-		/* admin may have requested C_DISCONNECTING,
-		 * other threads may have noticed network errors */
-		drbd_info(device, "unexpected cstate (%s) in receive_bitmap\n",
-		    drbd_conn_str(device->state.conn));
+static int receive_twopc(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct p_twopc_request *p = pi->data;
+	struct twopc_reply reply = {0};
+
+	reply.vnr = pi->vnr;
+	reply.tid = be32_to_cpu(p->tid);
+	if (connection->agreed_features & DRBD_FF_2PC_V2) {
+		reply.initiator_node_id = p->s8_initiator_node_id;
+		reply.target_node_id = p->s8_target_node_id;
+	} else {
+		reply.initiator_node_id = be32_to_cpu(p->u32_initiator_node_id);
+		reply.target_node_id = be32_to_cpu(p->u32_target_node_id);
 	}
-	err = 0;
+	reply.reachable_nodes = directly_connected_nodes(resource, NOW) |
+				NODE_MASK(resource->res_opts.node_id);
 
- out:
-	drbd_bm_unlock(device);
-	if (!err && device->state.conn == C_WF_BITMAP_S)
-		drbd_start_resync(device, C_SYNC_SOURCE);
-	return err;
+	if (pi->cmd == P_TWOPC_PREPARE)
+		clear_bit(TWOPC_RECV_SIZES_ERR, &resource->flags);
+
+	process_twopc(connection, &reply, pi, jiffies);
+
+	return 0;
 }
 
-static int receive_skip(struct drbd_connection *connection, struct packet_info *pi)
+static void nested_twopc_abort(struct drbd_resource *resource, struct twopc_request *request)
 {
-	drbd_warn(connection, "skipping unknown optional packet type %d, l: %d!\n",
-		 pi->cmd, pi->size);
+	struct drbd_connection *connection;
+	u64 nodes_to_reach, reach_immediately, im;
+
+	read_lock_irq(&resource->state_rwlock);
+	nodes_to_reach = request->nodes_to_reach;
+	reach_immediately = directly_connected_nodes(resource, NOW) & nodes_to_reach;
+	nodes_to_reach &= ~(reach_immediately | NODE_MASK(resource->res_opts.node_id));
+	request->nodes_to_reach = nodes_to_reach;
+	read_unlock_irq(&resource->state_rwlock);
 
-	return ignore_remaining_packet(connection, pi);
+	for_each_connection_ref(connection, im, resource) {
+		u64 mask = NODE_MASK(connection->peer_node_id);
+		if (reach_immediately & mask)
+			conn_send_twopc_request(connection, request);
+	}
 }
 
-static int receive_UnplugRemote(struct drbd_connection *connection, struct packet_info *pi)
+static bool is_prepare(enum drbd_packet cmd)
 {
-	/* Make sure we've acked all the TCP data associated
-	 * with the data requests being unplugged */
-	tcp_sock_set_quickack(connection->data.socket->sk, 2);
-	return 0;
+	return cmd == P_TWOPC_PREP_RSZ || cmd == P_TWOPC_PREPARE;
 }
 
-static int receive_out_of_sync(struct drbd_connection *connection, struct packet_info *pi)
+
+enum determine_dev_size
+drbd_commit_size_change(struct drbd_device *device, struct resize_parms *rs, u64 nodes_to_reach)
 {
+	struct twopc_resize *tr = &device->resource->twopc.resize;
 	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_block_desc *p = pi->data;
+	enum determine_dev_size dd;
+	uint64_t my_usize;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		/* update cached sizes, relevant for the next handshake */
+		peer_device->c_size = tr->new_size;
+		peer_device->u_size = tr->user_size;
 
-	switch (device->state.conn) {
-	case C_WF_SYNC_UUID:
-	case C_WF_BITMAP_T:
-	case C_BEHIND:
-			break;
-	default:
-		drbd_err(device, "ASSERT FAILED cstate = %s, expected: WFSyncUUID|WFBitMapT|Behind\n",
-				drbd_conn_str(device->state.conn));
+		if (peer_device->d_size)
+			peer_device->d_size = tr->new_size;
+		peer_device->max_size = tr->new_size;
+	}
+	rcu_read_unlock();
+
+	if (!get_ldev(device)) {
+		drbd_set_my_capacity(device, tr->new_size);
+		return DS_UNCHANGED; /* Not entirely true, but we are diskless... */
 	}
 
-	drbd_set_out_of_sync(peer_device, be64_to_cpu(p->sector), be32_to_cpu(p->blksize));
+	rcu_read_lock();
+	my_usize = rcu_dereference(device->ldev->disk_conf)->disk_size;
+	rcu_read_unlock();
 
-	return 0;
-}
+	if (my_usize != tr->user_size) {
+		struct disk_conf *old_disk_conf, *new_disk_conf;
 
-static int receive_rs_deallocated(struct drbd_connection *connection, struct packet_info *pi)
-{
-	struct drbd_peer_device *peer_device;
-	struct p_block_desc *p = pi->data;
-	struct drbd_device *device;
-	sector_t sector;
-	int size, err = 0;
+		drbd_info(device, "New u_size %llu sectors\n",
+			  (unsigned long long)tr->user_size);
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+		new_disk_conf = kzalloc_obj(struct disk_conf);
+		if (!new_disk_conf) {
+			device->ldev->disk_conf->disk_size = tr->user_size;
+			goto cont;
+		}
 
-	sector = be64_to_cpu(p->sector);
-	size = be32_to_cpu(p->blksize);
+		old_disk_conf = device->ldev->disk_conf;
+		*new_disk_conf = *old_disk_conf;
+		new_disk_conf->disk_size = tr->user_size;
 
-	dec_rs_pending(peer_device);
+		rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
+		kvfree_rcu_mightsleep(old_disk_conf);
+	}
+cont:
+	dd = drbd_determine_dev_size(device, tr->new_size, tr->dds_flags | DDSF_2PC, rs);
 
-	if (get_ldev(device)) {
-		struct drbd_peer_request *peer_req;
+	if (dd == DS_GREW && !(tr->dds_flags & DDSF_NO_RESYNC)) {
+		struct drbd_resource *resource = device->resource;
+		const int my_node_id = resource->res_opts.node_id;
+		struct drbd_peer_device *peer_device;
+		u64 im;
 
-		peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector,
-					       size, 0, GFP_NOIO);
-		if (!peer_req) {
-			put_ldev(device);
-			return -ENOMEM;
+		for_each_peer_device_ref(peer_device, im, device) {
+			if (peer_device->repl_state[NOW] != L_ESTABLISHED ||
+			    peer_device->disk_state[NOW] < D_INCONSISTENT)
+				continue;
+
+			if (tr->diskful_primary_nodes) {
+				if (tr->diskful_primary_nodes & NODE_MASK(my_node_id)) {
+					enum drbd_repl_state resync;
+					if (tr->diskful_primary_nodes & NODE_MASK(peer_device->node_id)) {
+						/* peer is also primary */
+						resync = peer_device->node_id < my_node_id ?
+							L_SYNC_TARGET : L_SYNC_SOURCE;
+					} else {
+						/* peer is secondary */
+						resync = L_SYNC_SOURCE;
+					}
+					drbd_start_resync(peer_device, resync, "resize");
+				} else {
+					if (tr->diskful_primary_nodes & NODE_MASK(peer_device->node_id))
+						drbd_start_resync(peer_device, L_SYNC_TARGET,
+								"resize");
+					/* else  no resync */
+				}
+			} else {
+				if (resource->twopc_parent_nodes & NODE_MASK(peer_device->node_id))
+					drbd_start_resync(peer_device, L_SYNC_TARGET, "resize");
+				else if (nodes_to_reach & NODE_MASK(peer_device->node_id))
+					drbd_start_resync(peer_device, L_SYNC_SOURCE, "resize");
+				/* else  no resync */
+			}
 		}
+	}
 
-		peer_req->w.cb = e_end_resync_block;
-		peer_req->opf = REQ_OP_DISCARD;
-		peer_req->submit_jif = jiffies;
-		peer_req->flags |= EE_TRIM;
+	put_ldev(device);
+	return dd;
+}
+
+enum drbd_state_rv drbd_support_2pc_resize(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	enum drbd_state_rv rv = SS_SUCCESS;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[NOW] == C_CONNECTED &&
+		    connection->agreed_pro_version < 112) {
+			rv = SS_NOT_SUPPORTED;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
 
-		spin_lock_irq(&device->resource->req_lock);
-		list_add_tail(&peer_req->w.list, &device->sync_ee);
-		spin_unlock_irq(&device->resource->req_lock);
+static bool any_neighbor_quorate(struct drbd_resource *resource)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_connection *connection;
+	bool peer_with_quorum = false;
+	int vnr;
 
-		atomic_add(pi->size >> 9, &device->rs_sect_ev);
-		err = drbd_submit_peer_request(peer_req);
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		peer_with_quorum = true;
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			if (test_bit(PEER_QUORATE, &peer_device->flags))
+				continue;
+			peer_with_quorum = false;
+			break;
+		}
+
+		if (peer_with_quorum)
+			break;
+	}
+	rcu_read_unlock();
+
+	return peer_with_quorum;
+}
+
+static void process_twopc(struct drbd_connection *connection,
+			 struct twopc_reply *reply,
+			 struct packet_info *pi,
+			 unsigned long receive_jif)
+{
+	struct drbd_connection *affected_connection = connection;
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device = NULL;
+	struct p_twopc_request *p = pi->data;
+	struct twopc_state_change *state_change = &resource->twopc.state_change;
+	enum chg_state_flags flags = CS_VERBOSE | CS_LOCAL_ONLY;
+	enum drbd_state_rv rv = SS_SUCCESS;
+	struct twopc_request request;
+	bool waiting_allowed = true;
+	enum csc_rv csc_rv;
+
+	request.tid = be32_to_cpu(p->tid);
+	if (connection->agreed_features & DRBD_FF_2PC_V2) {
+		request.flags = be32_to_cpu(p->flags);
+		request.initiator_node_id = p->s8_initiator_node_id;
+		request.target_node_id = p->s8_target_node_id;
+	} else {
+		request.flags = 0;
+		request.initiator_node_id = be32_to_cpu(p->u32_initiator_node_id);
+		request.target_node_id = be32_to_cpu(p->u32_target_node_id);
+	}
+	request.nodes_to_reach = be64_to_cpu(p->nodes_to_reach);
+	request.cmd = pi->cmd;
+	request.vnr = pi->vnr;
+
+	/* Check for concurrent transactions and duplicate packets. */
+retry:
+	write_lock_irq(&resource->state_rwlock);
+
+	csc_rv = check_concurrent_transactions(resource, reply);
+
+	if (csc_rv == CSC_CLEAR && pi->cmd != P_TWOPC_ABORT) {
+		struct drbd_device *device;
+		int iterate_vnr;
+
+		if (!is_prepare(pi->cmd)) {
+			/* We have committed or aborted this transaction already. */
+			write_unlock_irq(&resource->state_rwlock);
+			dynamic_drbd_dbg(connection, "Ignoring %s packet %u\n",
+				   drbd_packet_name(pi->cmd),
+				   reply->tid);
+			return;
+		}
+		if (reply->is_aborted) {
+			write_unlock_irq(&resource->state_rwlock);
+			return;
+		}
+		resource->remote_state_change = true;
+		resource->twopc.type =
+			pi->cmd == P_TWOPC_PREPARE ? TWOPC_STATE_CHANGE : TWOPC_RESIZE;
+		resource->twopc_prepare_reply_cmd = 0;
+		resource->twopc_parent_nodes = NODE_MASK(connection->peer_node_id);
+		clear_bit(TWOPC_EXECUTED, &resource->flags);
+		idr_for_each_entry(&resource->devices, device, iterate_vnr)
+			clear_bit(OUTDATE_ON_2PC_COMMIT, &device->flags);
+	} else if (csc_rv == CSC_MATCH && !is_prepare(pi->cmd)) {
+		flags |= CS_PREPARED;
+
+		if (test_and_set_bit(TWOPC_EXECUTED, &resource->flags)) {
+			write_unlock_irq(&resource->state_rwlock);
+			drbd_info(connection, "Ignoring redundant %s packet %u.\n",
+				  drbd_packet_name(pi->cmd),
+				  reply->tid);
+			return;
+		}
+	} else if (csc_rv == CSC_ABORT_LOCAL && is_prepare(pi->cmd)) {
+		enum alt_rv alt_rv;
+
+		drbd_info(connection, "Aborting local state change %u to yield to remote "
+			  "state change %u.\n",
+			  resource->twopc_reply.tid,
+			  reply->tid);
+		alt_rv = abort_local_transaction(connection, reply->tid);
+		if (alt_rv == ALT_MATCH) {
+			/* abort_local_transaction() comes back unlocked in this case... */
+			goto match;
+		} else if (alt_rv == ALT_TIMEOUT) {
+			/* abort_local_transaction() comes back unlocked in this case... */
+			drbd_info(connection, "Aborting local state change %u "
+				  "failed. Rejecting remote state change %u.\n",
+				  resource->twopc_reply.tid,
+				  reply->tid);
+			drbd_send_twopc_reply(connection, P_TWOPC_RETRY, reply);
+			return;
+		}
+		/* abort_local_transaction() returned with the state_rwlock write lock */
+		if (reply->is_aborted) {
+			write_unlock_irq(&resource->state_rwlock);
+			return;
+		}
+		resource->remote_state_change = true;
+		resource->twopc.type =
+			pi->cmd == P_TWOPC_PREPARE ? TWOPC_STATE_CHANGE : TWOPC_RESIZE;
+		resource->twopc_parent_nodes = NODE_MASK(connection->peer_node_id);
+		resource->twopc_prepare_reply_cmd = 0;
+		clear_bit(TWOPC_EXECUTED, &resource->flags);
+	} else if (pi->cmd == P_TWOPC_ABORT) {
+		/* crc_rc != CRC_MATCH */
+		write_unlock_irq(&resource->state_rwlock);
+		nested_twopc_abort(resource, &request);
+		return;
+	} else {
+		write_unlock_irq(&resource->state_rwlock);
+
+		if (csc_rv == CSC_TID_MISS && is_prepare(pi->cmd) && waiting_allowed) {
+			/* CSC_TID_MISS implies the two transactions are from the same initiator */
+			if (!(resource->twopc_parent_nodes & NODE_MASK(connection->peer_node_id))) {
+				long timeout = twopc_timeout(resource) / 20; /* usually 1.5 sec */
+				/*
+				 * We are expecting the P_TWOPC_COMMIT or P_TWOPC_ABORT through
+				 * another connection. So we can wait without deadlocking.
+				 */
+				wait_event_interruptible_timeout(resource->twopc_wait,
+						!resource->remote_state_change, timeout);
+				waiting_allowed = false; /* retry only once */
+				goto retry;
+			}
+		}
+
+		if (csc_rv == CSC_REJECT ||
+		    (csc_rv == CSC_TID_MISS && is_prepare(pi->cmd))) {
+			drbd_info(connection, "Rejecting concurrent "
+				  "remote state change %u because of "
+				  "state change %u\n",
+				  reply->tid,
+				  resource->twopc_reply.tid);
+			drbd_send_twopc_reply(connection, P_TWOPC_RETRY, reply);
+			return;
+		}
+
+		if (is_prepare(pi->cmd)) {
+			if (csc_rv == CSC_MATCH) {
+				/* We have prepared this transaction already. */
+				enum drbd_packet reply_cmd;
+
+			match:
+				drbd_info(connection,
+						"Duplicate prepare for remote state change %u\n",
+						reply->tid);
+				write_lock_irq(&resource->state_rwlock);
+				resource->twopc_parent_nodes |= NODE_MASK(connection->peer_node_id);
+				reply_cmd = resource->twopc_prepare_reply_cmd;
+				write_unlock_irq(&resource->state_rwlock);
+
+				if (reply_cmd) {
+					drbd_send_twopc_reply(connection, reply_cmd,
+							      &resource->twopc_reply);
+				} else {
+					/* if a node sends us a prepare, that means he has
+					   prepared this himsilf successfully. */
+					write_lock_irq(&resource->state_rwlock);
+					set_bit(TWOPC_YES, &connection->flags);
+					drbd_maybe_cluster_wide_reply(resource);
+					write_unlock_irq(&resource->state_rwlock);
+				}
+			}
+		} else {
+			drbd_info(connection, "Ignoring %s packet %u "
+				  "current processing state change %u\n",
+				  drbd_packet_name(pi->cmd),
+				  reply->tid,
+				  resource->twopc_reply.tid);
+		}
+		return;
+	}
+
+	if (reply->initiator_node_id != connection->peer_node_id) {
+		/*
+		 * This is an indirect request.  Unless we are directly
+		 * connected to the initiator as well as indirectly, we don't
+		 * have connection or peer device objects for this peer.
+		 */
+		affected_connection = drbd_connection_by_node_id(resource, reply->initiator_node_id);
+	}
+
+	if (reply->target_node_id != -1 &&
+	    reply->target_node_id != resource->res_opts.node_id) {
+		affected_connection = NULL;
+	}
+
+	switch (resource->twopc.type) {
+	case TWOPC_STATE_CHANGE:
+		if (pi->cmd == P_TWOPC_PREPARE) {
+			state_change->mask.i = be32_to_cpu(p->mask);
+			state_change->val.i = be32_to_cpu(p->val);
+		} else { /* P_TWOPC_COMMIT */
+			state_change->primary_nodes = be64_to_cpu(p->primary_nodes);
+			state_change->reachable_nodes = be64_to_cpu(p->reachable_nodes);
+		}
+		break;
+	case TWOPC_RESIZE:
+		if (request.cmd == P_TWOPC_PREP_RSZ) {
+			resource->twopc.resize.user_size = be64_to_cpu(p->user_size);
+			resource->twopc.resize.dds_flags = be16_to_cpu(p->dds_flags);
+		} else { /* P_TWOPC_COMMIT */
+			resource->twopc.resize.diskful_primary_nodes =
+				be64_to_cpu(p->diskful_primary_nodes);
+			resource->twopc.resize.new_size = be64_to_cpu(p->exposed_size);
+		}
+	}
+
+	if (affected_connection && affected_connection->cstate[NOW] < C_CONNECTED &&
+	    state_change->mask.conn == 0)
+		affected_connection = NULL;
+
+	if (pi->vnr != -1 && affected_connection) {
+		peer_device = conn_peer_device(affected_connection, pi->vnr);
+		/* If we do not know the peer_device, then we are fine with
+		   whatever is going on in the cluster. E.g. detach and del-minor
+		   one each node, one after the other */
+
+		affected_connection = NULL; /* It is intended for a peer_device! */
+	}
+
+	if (state_change->mask.conn == conn_MASK) {
+		u64 m = NODE_MASK(reply->initiator_node_id);
+
+		if (state_change->val.conn == C_CONNECTED) {
+			reply->reachable_nodes |= m;
+			if (affected_connection) {
+				reply->is_connect = 1;
+
+				if (pi->cmd == P_TWOPC_PREPARE)
+					drbd_init_connect_state(connection);
+			}
+		}
+		if (state_change->val.conn == C_DISCONNECTING) {
+			reply->reachable_nodes &= ~m;
+			reply->is_disconnect = 1;
+		}
+	}
+
+	if (pi->cmd == P_TWOPC_PREPARE) {
+		reply->primary_nodes = be64_to_cpu(p->primary_nodes);
+		if (resource->role[NOW] == R_PRIMARY) {
+			reply->primary_nodes |= NODE_MASK(resource->res_opts.node_id);
+
+			if (drbd_res_data_accessible(resource))
+				reply->weak_nodes = ~reply->reachable_nodes;
+		}
+	}
+	if (pi->cmd == P_TWOPC_PREP_RSZ) {
+		struct drbd_device *device;
+
+		device = (peer_device ?: conn_peer_device(connection, pi->vnr))->device;
+		if (get_ldev(device)) {
+			if (resource->role[NOW] == R_PRIMARY)
+				reply->diskful_primary_nodes = NODE_MASK(resource->res_opts.node_id);
+			reply->max_possible_size = drbd_local_max_size(device);
+			put_ldev(device);
+		} else {
+			reply->max_possible_size = DRBD_MAX_SECTORS;
+			reply->diskful_primary_nodes = 0;
+		}
+	}
+
+	resource->twopc_reply = *reply;
+	write_unlock_irq(&resource->state_rwlock);
+
+	if (affected_connection && affected_connection != connection &&
+	    affected_connection->cstate[NOW] == C_CONNECTED) {
+		drbd_ping_peer(affected_connection);
+		if (affected_connection->cstate[NOW] < C_CONNECTED)
+			affected_connection = NULL;
+	}
+
+	switch (pi->cmd) {
+	case P_TWOPC_PREPARE:
+		drbd_print_cluster_wide_state_change(resource, "Preparing remote state change",
+				reply->tid, reply->initiator_node_id, reply->target_node_id,
+				state_change->mask, state_change->val);
+		flags |= CS_PREPARE;
+		break;
+	case P_TWOPC_PREP_RSZ:
+		drbd_info(connection, "Preparing remote state change %u "
+			  "(local_max_size = %llu KiB)\n",
+			  reply->tid, (unsigned long long)reply->max_possible_size >> 1);
+		flags |= CS_PREPARE;
+		break;
+	case P_TWOPC_ABORT:
+		drbd_info(connection, "Aborting remote state change %u\n",
+			  reply->tid);
+		flags |= CS_ABORT;
+		break;
+	case P_TWOPC_COMMIT:
+		drbd_info(connection, "Committing remote state change %u (primary_nodes=%llX)\n",
+			  reply->tid, be64_to_cpu(p->primary_nodes));
+		break;
+	default:
+		BUG();
+	}
+
+	switch (resource->twopc.type) {
+	case TWOPC_STATE_CHANGE:
+		if (flags & CS_PREPARED && !(flags & CS_ABORT)) {
+			reply->primary_nodes = state_change->primary_nodes;
+			handle_neighbor_demotion(connection, state_change, reply);
+
+			if ((resource->cached_all_devices_have_quorum ||
+			     any_neighbor_quorate(resource)) &&
+			    request.flags & TWOPC_HAS_REACHABLE) {
+				resource->members = state_change->reachable_nodes;
+				if (!resource->cached_all_devices_have_quorum)
+					flags |= CS_FORCE_RECALC;
+			}
+			if (state_change->mask.conn == conn_MASK &&
+			    state_change->val.conn == C_CONNECTED) {
+				/* Add nodes connecting "far away" to members */
+				u64 add_mask = NODE_MASK(reply->initiator_node_id) |
+					NODE_MASK(reply->target_node_id);
+
+				resource->members |= add_mask;
+			}
+		}
+
+		if (peer_device)
+			rv = change_peer_device_state(peer_device, state_change, flags);
+		else if (affected_connection)
+			rv = change_connection_state(affected_connection, state_change, reply,
+						     flags | CS_IGN_OUTD_FAIL);
+		else
+			rv = far_away_change(connection, &request, reply, flags);
+		break;
+	case TWOPC_RESIZE:
+		if (flags & CS_PREPARE)
+			rv = drbd_support_2pc_resize(resource);
+		break;
+	}
+
+	if (flags & CS_PREPARE) {
+		mod_timer(&resource->twopc_timer, receive_jif + twopc_timeout(resource));
+
+		/* Retry replies can be sent immediately. Otherwise use the
+		 * nested twopc path. This waits for the state handshake to
+		 * complete in the case of a twopc for transitioning to
+		 * C_CONNECTED. */
+		if (rv == SS_IN_TRANSIENT_STATE) {
+			resource->twopc_prepare_reply_cmd = P_TWOPC_RETRY;
+			drbd_send_twopc_reply(connection, P_TWOPC_RETRY, reply);
+		} else {
+			resource->twopc_reply.state_change_failed = rv < SS_SUCCESS;
+			nested_twopc_request(resource, &request);
+		}
+	} else {
+		if (flags & CS_PREPARED) {
+			if (rv < SS_SUCCESS)
+				drbd_err(resource, "FATAL: Local commit of prepared %u failed! \n",
+					 reply->tid);
+
+			timer_delete(&resource->twopc_timer);
+		}
+
+		nested_twopc_request(resource, &request);
+
+		if (resource->twopc.type == TWOPC_RESIZE && flags & CS_PREPARED &&
+		    !(flags & CS_ABORT)) {
+			struct drbd_device *device;
+
+			device = (peer_device ?: conn_peer_device(connection, pi->vnr))->device;
+
+			drbd_commit_size_change(device, NULL, request.nodes_to_reach);
+			rv = SS_SUCCESS;
+		}
+
+		clear_remote_state_change(resource);
+
+		if (peer_device && rv >= SS_SUCCESS && !(flags & CS_ABORT))
+			drbd_md_sync_if_dirty(peer_device->device);
+
+		if (connection->agreed_pro_version < 117 &&
+		    rv >= SS_SUCCESS && !(flags & CS_ABORT) &&
+		    affected_connection &&
+		    state_change->mask.conn == conn_MASK && state_change->val.conn == C_CONNECTED)
+			conn_connect2(connection);
+	}
+}
+
+void drbd_try_to_get_resynced(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device, *best_peer_device = NULL;
+	enum sync_strategy best_strategy = UNDETERMINED;
+	int best_preference = 0;
+
+	if (!get_ldev(device))
+		return;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum sync_strategy strategy;
+		enum sync_rule rule;
+		int peer_node_id;
+
+		if (peer_device->disk_state[NOW] != D_UP_TO_DATE)
+			continue;
+
+		strategy = drbd_uuid_compare(peer_device, &rule, &peer_node_id);
+		disk_states_to_strategy(peer_device, peer_device->disk_state[NOW], &strategy, rule,
+					&peer_node_id);
+		drbd_info(peer_device, "strategy = %s\n", strategy_descriptor(strategy).name);
+		if (strategy_descriptor(strategy).resync_peer_preference > best_preference) {
+			best_preference = strategy_descriptor(strategy).resync_peer_preference;
+			best_peer_device = peer_device;
+			best_strategy = strategy;
+		}
+	}
+	rcu_read_unlock();
+	peer_device = best_peer_device;
+
+	if (best_strategy == NO_SYNC) {
+		change_disk_state(device, D_UP_TO_DATE, CS_VERBOSE, "get-resync", NULL);
+	} else if (peer_device &&
+		   (!repl_is_sync_target(peer_device->repl_state[NOW]) ||
+		    test_bit(UNSTABLE_RESYNC, &peer_device->flags))) {
+		drbd_resync(peer_device, DISKLESS_PRIMARY);
+		drbd_send_uuids(peer_device, UUID_FLAG_RESYNC | UUID_FLAG_DISKLESS_PRIMARY, 0);
+	}
+	put_ldev(device);
+}
+
+static void finish_nested_twopc(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	int vnr = 0;
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (!test_bit(INITIAL_STATE_RECEIVED, &peer_device->flags))
+			return;
+	}
+
+	set_bit(CONN_HANDSHAKE_READY, &connection->flags);
+
+	wake_up_all(&resource->state_wait);
+
+	write_lock_irq(&resource->state_rwlock);
+	drbd_maybe_cluster_wide_reply(resource);
+	write_unlock_irq(&resource->state_rwlock);
+}
+
+static bool uuid_in_peer_history(struct drbd_peer_device *peer_device, u64 uuid)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(peer_device->history_uuids); i++)
+		if ((peer_device->history_uuids[i] & ~UUID_PRIMARY) == uuid)
+			return true;
+
+	return false;
+}
+
+static bool uuid_in_my_history(struct drbd_device *device, u64 uuid)
+{
+	int i;
+
+	for (i = 0; i < HISTORY_UUIDS; i++) {
+		if ((drbd_history_uuid(device, i) & ~UUID_PRIMARY) == uuid)
+			return true;
+	}
+
+	return false;
+}
+
+static bool peer_data_is_successor_of_mine(struct drbd_peer_device *peer_device)
+{
+	u64 exposed = peer_device->device->exposed_data_uuid & ~UUID_PRIMARY;
+	int i;
+
+	i = drbd_find_peer_bitmap_by_uuid(peer_device, exposed);
+	if (i != -1)
+		return true;
+
+	return uuid_in_peer_history(peer_device, exposed);
+}
+
+static bool peer_data_is_ancestor_of_mine(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	u64 peer_uuid = peer_device->current_uuid;
+	struct drbd_peer_device *p2;
+	bool rv = false;
+	int i;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(p2, device) {
+		if (peer_device == p2)
+			continue;
+		i = drbd_find_peer_bitmap_by_uuid(p2, peer_uuid);
+		if (i != -1 || uuid_in_peer_history(peer_device, peer_uuid)) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+static void propagate_exposed_uuid(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	u64 im;
+
+	for_each_peer_device_ref(peer_device, im, device) {
+		if (!test_bit(INITIAL_STATE_SENT, &peer_device->flags))
+			continue;
+		drbd_send_current_uuid(peer_device, device->exposed_data_uuid, 0);
+	}
+}
+
+static void maybe_force_secondary(struct drbd_peer_device *peer_device)
+{
+	struct drbd_resource *resource = peer_device->device->resource;
+	unsigned long irq_flags;
+
+	if (!resource->fail_io[NOW] && resource->cached_susp &&
+	    resource->res_opts.on_susp_primary_outdated == SPO_FORCE_SECONDARY) {
+		drbd_warn(peer_device, "force secondary!\n");
+		begin_state_change(resource, &irq_flags,
+				   CS_VERBOSE | CS_HARD | CS_FS_IGN_OPENERS);
+		resource->role[NEW] = R_SECONDARY;
+		/* resource->fail_io[NEW] gets set via CS_FS_IGN_OPENERS */
+		end_state_change(resource, &irq_flags, "peer-state");
+	}
+}
+
+static void diskless_with_peers_different_current_uuids(struct drbd_peer_device *peer_device,
+							enum drbd_disk_state *peer_disk_state)
+{
+	bool data_successor = peer_data_is_successor_of_mine(peer_device);
+	bool data_ancestor = peer_data_is_ancestor_of_mine(peer_device);
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_device *device = peer_device->device;
+
+	if (data_successor && resource->role[NOW] == R_PRIMARY) {
+		drbd_warn(peer_device, "Remote node has more recent data\n");
+		maybe_force_secondary(peer_device);
+		set_bit(CONN_HANDSHAKE_RETRY, &connection->flags);
+	} else if (data_successor && resource->role[NOW] == R_SECONDARY) {
+		drbd_uuid_set_exposed(device, peer_device->current_uuid, true);
+		propagate_exposed_uuid(device);
+	} else if (data_ancestor) {
+		drbd_warn(peer_device, "Downgrading joining peer's disk as its data is older\n");
+		if (*peer_disk_state > D_OUTDATED)
+			*peer_disk_state = D_OUTDATED;
+			/* See "Do not trust this guy!" in sanitize_state() */
+	} else {
+		drbd_warn(peer_device, "Current UUID of peer does not match my exposed UUID.");
+		set_bit(CONN_HANDSHAKE_DISCONNECT, &connection->flags);
+	}
+}
+
+static int receive_state(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device = NULL;
+	enum drbd_repl_state *repl_state;
+	struct drbd_device *device = NULL;
+	struct p_state *p = pi->data;
+	union drbd_state old_peer_state, peer_state;
+	enum drbd_disk_state peer_disk_state;
+	enum drbd_repl_state new_repl_state;
+	bool peer_was_resync_target, do_handshake = false;
+	enum chg_state_flags begin_state_chg_flags = CS_VERBOSE | CS_WAIT_COMPLETE;
+	unsigned long irq_flags;
+	int rv;
+
+	if (pi->vnr != -1) {
+		peer_device = conn_peer_device(connection, pi->vnr);
+		if (!peer_device)
+			return config_unknown_volume(connection, pi);
+		device = peer_device->device;
+	}
+
+	peer_state.i = be32_to_cpu(p->state);
+
+	if (connection->agreed_pro_version < 110) {
+		/* Before drbd-9.0 there was no D_DETACHING it was D_FAILED... */
+		if (peer_state.disk >= D_DETACHING)
+			peer_state.disk++;
+		if (peer_state.pdsk >= D_DETACHING)
+			peer_state.pdsk++;
+	}
+
+	if (pi->vnr == -1) {
+		if (peer_state.role == R_SECONDARY) {
+			begin_state_change(resource, &irq_flags, CS_HARD | CS_VERBOSE);
+			__change_peer_role(connection, R_SECONDARY);
+			rv = end_state_change(resource, &irq_flags, "peer-state");
+			if (rv < SS_SUCCESS)
+				goto fail;
+		}
+		return 0;
+	}
+
+	peer_disk_state = peer_state.disk;
+
+	if (peer_disk_state > D_DISKLESS && !want_bitmap(peer_device)) {
+		drbd_warn(peer_device, "The peer is configured to be diskless but presents %s\n",
+			  drbd_disk_str(peer_disk_state));
+		goto fail;
+	}
+
+	if (peer_state.disk == D_NEGOTIATING) {
+		peer_disk_state = peer_device->uuid_flags & UUID_FLAG_INCONSISTENT ?
+			D_INCONSISTENT : D_CONSISTENT;
+		drbd_info(peer_device, "real peer disk state = %s\n", drbd_disk_str(peer_disk_state));
+	}
+
+	read_lock_irq(&resource->state_rwlock);
+	old_peer_state = drbd_get_peer_device_state(peer_device, NOW);
+	read_unlock_irq(&resource->state_rwlock);
+ retry:
+	new_repl_state = max_t(enum drbd_repl_state, old_peer_state.conn, L_OFF);
+
+	/* If some other part of the code (ack_receiver thread, timeout)
+	 * already decided to close the connection again,
+	 * we must not "re-establish" it here. */
+	if (old_peer_state.conn <= C_TEAR_DOWN)
+		return -ECONNRESET;
+
+	if (!test_bit(INITIAL_STATE_RECEIVED, &peer_device->flags) &&
+	    peer_state.role == R_PRIMARY && peer_device->uuid_flags & UUID_FLAG_STABLE)
+		check_resync_source(device, peer_device->uuid_node_mask);
+
+	peer_was_resync_target =
+		connection->agreed_pro_version >= 110 ?
+		peer_device->last_repl_state == L_SYNC_TARGET ||
+		peer_device->last_repl_state == L_PAUSED_SYNC_T
+		:
+		true;
+	/* If this is the "end of sync" confirmation, usually the peer disk
+	 * was D_INCONSISTENT or D_CONSISTENT. (Since the peer might be
+	 * weak we do not know anything about its new disk state)
+	 */
+	if (peer_was_resync_target &&
+	    (old_peer_state.pdsk == D_INCONSISTENT || old_peer_state.pdsk == D_CONSISTENT) &&
+	    old_peer_state.conn > L_ESTABLISHED && old_peer_state.disk >= D_INCONSISTENT) {
+		/* If we are (becoming) SyncSource, but peer is still in sync
+		 * preparation, ignore its uptodate-ness to avoid flapping, it
+		 * will change to inconsistent once the peer reaches active
+		 * syncing states.
+		 * It may have changed syncer-paused flags, however, so we
+		 * cannot ignore this completely. */
+		if (peer_state.conn > L_ESTABLISHED &&
+		    peer_state.conn < L_SYNC_SOURCE)
+			peer_disk_state = D_INCONSISTENT;
+
+		/* if peer_state changes to connected at the same time,
+		 * it explicitly notifies us that it finished resync.
+		 * Maybe we should finish it up, too? */
+		else if (peer_state.conn == L_ESTABLISHED) {
+			bool finish_now = false;
+
+			if (old_peer_state.conn == L_WF_BITMAP_S) {
+				read_lock_irq(&resource->state_rwlock);
+				if (peer_device->repl_state[NOW] == L_WF_BITMAP_S)
+					peer_device->resync_finished_pdsk = peer_state.disk;
+				else if (peer_device->repl_state[NOW] == L_SYNC_SOURCE)
+					finish_now = true;
+				read_unlock_irq(&resource->state_rwlock);
+			}
+
+			if (finish_now || old_peer_state.conn == L_SYNC_SOURCE ||
+			    old_peer_state.conn == L_PAUSED_SYNC_S) {
+				drbd_resync_finished(peer_device, peer_state.disk);
+				peer_device->last_repl_state = peer_state.conn;
+			}
+			return 0;
+		}
+	}
+
+	/* explicit verify finished notification, stop sector reached. */
+	if (old_peer_state.conn == L_VERIFY_T && old_peer_state.disk == D_UP_TO_DATE &&
+	    peer_state.conn == L_ESTABLISHED && peer_disk_state == D_UP_TO_DATE) {
+		ov_out_of_sync_print(peer_device);
+		drbd_resync_finished(peer_device, D_MASK);
+		peer_device->last_repl_state = peer_state.conn;
+		return 0;
+	}
+
+	/* Start resync after AHEAD/BEHIND */
+	if (connection->agreed_pro_version >= 110 &&
+	    peer_state.conn == L_SYNC_SOURCE && old_peer_state.conn == L_BEHIND) {
+		/*
+		 * Become Inconsistent immediately because we may now receive
+		 * data. Delay the start of the resync itself until any
+		 * previous resync is no longer active.
+		 */
+		rv = change_disk_state(device, D_INCONSISTENT, CS_VERBOSE,
+				"resync-after-behind", NULL);
+		if (rv < SS_SUCCESS)
+			goto fail;
+
+		peer_device->start_resync_side = L_SYNC_TARGET;
+		drbd_peer_device_post_work(peer_device, RS_START);
+		return 0;
+	}
+
+	/* peer says his disk is inconsistent, while we think it is uptodate,
+	 * and this happens while the peer still thinks we have a sync going on,
+	 * but we think we are already done with the sync.
+	 * We ignore this to avoid flapping pdsk.
+	 * This should not happen, if the peer is a recent version of drbd. */
+	if (old_peer_state.pdsk == D_UP_TO_DATE && peer_disk_state == D_INCONSISTENT &&
+	    old_peer_state.conn == L_ESTABLISHED && peer_state.conn > L_SYNC_SOURCE)
+		peer_disk_state = D_UP_TO_DATE;
+
+	if (new_repl_state == L_OFF)
+		new_repl_state = L_ESTABLISHED;
+
+	if (peer_state.conn == L_AHEAD)
+		new_repl_state = L_BEHIND;
+
+	/* with protocol >= 118 uuid & state packets come after the 2PC prepare packet */
+	do_handshake =
+		(test_bit(UUIDS_RECEIVED, &peer_device->flags) ||
+			test_bit(CURRENT_UUID_RECEIVED, &peer_device->flags)) &&
+		(connection->agreed_pro_version < 118 ||
+			drbd_twopc_between_peer_and_me(connection)) &&
+		old_peer_state.conn < L_ESTABLISHED;
+
+	if (test_bit(UUIDS_RECEIVED, &peer_device->flags) &&
+	    peer_state.disk >= D_NEGOTIATING &&
+	    get_ldev_if_state(device, D_NEGOTIATING)) {
+		enum sync_strategy strategy = UNDETERMINED;
+		bool consider_resync;
+
+		/* clear CONN_DISCARD_MY_DATA so late, to not lose it if peer
+		   gets aborted before we are able to do the resync handshake. */
+		clear_bit(CONN_DISCARD_MY_DATA, &connection->flags);
+
+		/* if we established a new connection */
+		consider_resync = do_handshake &&
+					!test_bit(INITIAL_STATE_RECEIVED, &peer_device->flags);
+		/* if we have both been inconsistent, and the peer has been
+		 * forced to be UpToDate with --force */
+		consider_resync |= test_bit(CONSIDER_RESYNC, &peer_device->flags);
+		/* if we had been plain connected, and the admin requested to
+		 * start a sync by "invalidate" or "invalidate-remote" */
+		consider_resync |= (old_peer_state.conn == L_ESTABLISHED &&
+				    (peer_state.conn == L_STARTING_SYNC_S ||
+				     peer_state.conn == L_STARTING_SYNC_T));
+
+		consider_resync |= peer_state.conn == L_WF_BITMAP_T &&
+				   peer_device->flags & UUID_FLAG_CRASHED_PRIMARY;
+
+		if (consider_resync) {
+			strategy = drbd_sync_handshake(peer_device, peer_state);
+			new_repl_state = strategy_to_repl_state(peer_device, peer_state.role, strategy);
+		} else if (old_peer_state.conn == L_ESTABLISHED &&
+			   (peer_state.disk == D_NEGOTIATING ||
+			    old_peer_state.disk == D_NEGOTIATING)) {
+			strategy = drbd_attach_handshake(peer_device, peer_disk_state);
+			new_repl_state = strategy_to_repl_state(peer_device, peer_state.role, strategy);
+			if (new_repl_state == L_ESTABLISHED && device->disk_state[NOW] == D_UP_TO_DATE)
+				peer_disk_state = D_UP_TO_DATE;
+		}
+
+		put_ldev(device);
+		if (strategy_descriptor(strategy).reconnect) { /* retry connect */
+			maybe_force_secondary(peer_device);
+			if (connection->agreed_pro_version >= 118)
+				set_bit(CONN_HANDSHAKE_RETRY, &connection->flags);
+			else
+				return -EIO; /* retry connect */
+		} else if (strategy_descriptor(strategy).disconnect) {
+			if (device->disk_state[NOW] == D_NEGOTIATING) {
+				new_repl_state = L_NEG_NO_RESULT;
+			} else if (peer_state.disk == D_NEGOTIATING) {
+				if (connection->agreed_pro_version < 110) {
+					drbd_err(device, "Disk attach process on the peer node was aborted.\n");
+					peer_state.disk = D_DISKLESS;
+					peer_disk_state = D_DISKLESS;
+				} else {
+					/* The peer will decide later and let us know... */
+					peer_disk_state = D_NEGOTIATING;
+				}
+			} else {
+				if (test_and_clear_bit(CONN_DRY_RUN, &connection->flags))
+					return -EIO;
+				if (connection->agreed_pro_version >= 118)
+					set_bit(CONN_HANDSHAKE_DISCONNECT, &connection->flags);
+				else
+					goto fail;
+			}
+		}
+
+		if (device->disk_state[NOW] == D_NEGOTIATING) {
+			begin_state_chg_flags |= CS_FORCE_RECALC;
+			peer_device->negotiation_result = new_repl_state;
+		}
+	}
+
+	if (test_bit(UUIDS_RECEIVED, &peer_device->flags) &&
+	    peer_device->repl_state[NOW] == L_OFF && device->disk_state[NOW] == D_DISKLESS) {
+		u64 exposed_data_uuid = device->exposed_data_uuid;
+		u64 peer_current_uuid = peer_device->current_uuid;
+
+		drbd_info(peer_device, "my exposed UUID: %016llX\n", exposed_data_uuid);
+		drbd_uuid_dump_peer(peer_device, peer_device->dirty_bits, peer_device->uuid_flags);
+
+		/* I am diskless connecting to a peer with disk, check that UUID match
+		   We only check if the peer claims to have D_UP_TO_DATE data. Only then is the
+		   peer a source for my data anyways. */
+		if (exposed_data_uuid && peer_state.disk == D_UP_TO_DATE &&
+		    (exposed_data_uuid & ~UUID_PRIMARY) != (peer_current_uuid & ~UUID_PRIMARY))
+			diskless_with_peers_different_current_uuids(peer_device, &peer_disk_state);
+		if (!exposed_data_uuid && peer_state.disk == D_UP_TO_DATE) {
+			drbd_uuid_set_exposed(device, peer_current_uuid, true);
+			propagate_exposed_uuid(device);
+		}
+	}
+	if (peer_device->repl_state[NOW] == L_OFF && peer_state.disk == D_DISKLESS && get_ldev(device)) {
+		u64 uuid_flags = 0;
+
+		drbd_collect_local_uuid_flags(peer_device, NULL);
+		drbd_uuid_dump_self(peer_device, peer_device->comm_bm_set, uuid_flags);
+		drbd_info(peer_device, "peer's exposed UUID: %016llX\n", peer_device->current_uuid);
+
+		if (peer_state.role == R_PRIMARY &&
+		    (peer_device->current_uuid & ~UUID_PRIMARY) ==
+		    (drbd_current_uuid(device) & ~UUID_PRIMARY)) {
+			/* Connecting to diskless primary peer. When the state change is committed,
+			 * sanitize_state might set me D_UP_TO_DATE. Make sure the
+			 * effective_size is set. */
+			peer_device->max_size = peer_device->c_size;
+			drbd_determine_dev_size(device, peer_device->max_size, 0, NULL);
+		}
+
+		put_ldev(device);
+	}
+
+	if (test_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags) ||
+			connection->agreed_pro_version < 110) {
+		struct drbd_transport *transport = &connection->transport;
+		/* Last packet of handshake received, disarm receive timeout */
+		transport->class->ops.set_rcvtimeo(transport, DATA_STREAM, MAX_SCHEDULE_TIMEOUT);
+	}
+
+	if (new_repl_state == L_ESTABLISHED && peer_disk_state == D_CONSISTENT &&
+	    drbd_suspended(device) && peer_device->repl_state[NOW] < L_ESTABLISHED &&
+	    test_and_clear_bit(NEW_CUR_UUID, &device->flags)) {
+		/* Do not allow RESEND for a rebooted peer. We can only allow this
+	 * for temporary network outages! */
+		drbd_err(peer_device, "Aborting Connect, can not thaw IO with an only Consistent peer\n");
+		drbd_uuid_new_current(device, false);
+		begin_state_change(resource, &irq_flags, CS_HARD);
+		__change_cstate(connection, C_PROTOCOL_ERROR);
+		__change_io_susp_user(resource, false);
+		end_state_change(resource, &irq_flags, "abort-connect");
+		return -EIO;
+	}
+
+	clear_bit(RS_SOURCE_MISSED_END, &peer_device->flags);
+	clear_bit(RS_PEER_MISSED_END, &peer_device->flags);
+
+	if (peer_state.quorum)
+		set_bit(PEER_QUORATE, &peer_device->flags);
+	else
+		clear_bit(PEER_QUORATE, &peer_device->flags);
+
+	if (do_handshake) {
+		/* Ignoring state packets before the 2PC; they are from aborted 2PCs */
+		bool done = test_bit(INITIAL_STATE_RECEIVED, &peer_device->flags);
+
+		set_bit(INITIAL_STATE_RECEIVED, &peer_device->flags);
+		if (connection->cstate[NOW] == C_CONNECTING) {
+			peer_device->connect_state.peer_isp =
+				peer_state.aftr_isp | peer_state.user_isp;
+
+			if (!done) {
+				peer_device->connect_state.conn = new_repl_state;
+				peer_device->connect_state.peer = peer_state.role;
+				peer_device->connect_state.pdsk = peer_disk_state;
+			}
+			wake_up(&connection->ee_wait);
+			finish_nested_twopc(connection);
+		}
+	}
+
+	/* State change will be performed when the two-phase commit is committed. */
+	if (connection->cstate[NOW] == C_CONNECTING)
+		return 0;
+
+	if (peer_state.conn == L_OFF) {
+		/* device/minor hot add on the peer of a minor already locally known */
+		if (peer_device->repl_state[NOW] == L_NEGOTIATING) {
+			drbd_send_enable_replication_next(peer_device);
+			drbd_send_sizes(peer_device, 0, 0);
+			drbd_send_uuids(peer_device, 0, 0);
+		}
+		drbd_send_current_state(peer_device);
+	}
+
+	begin_state_change(resource, &irq_flags, begin_state_chg_flags);
+	if (old_peer_state.i != drbd_get_peer_device_state(peer_device, NOW).i) {
+		old_peer_state = drbd_get_peer_device_state(peer_device, NOW);
+		abort_state_change_locked(resource);
+		write_unlock_irq(&resource->state_rwlock);
+		goto retry;
+	}
+	clear_bit(CONSIDER_RESYNC, &peer_device->flags);
+	if (device->disk_state[NOW] != D_NEGOTIATING)
+		__change_repl_state(peer_device, new_repl_state);
+	__change_peer_role(connection, peer_state.role);
+	if (peer_state.disk != D_NEGOTIATING)
+		__change_peer_disk_state(peer_device, peer_disk_state);
+	__change_resync_susp_peer(peer_device, peer_state.aftr_isp | peer_state.user_isp);
+	repl_state = peer_device->repl_state;
+	if (repl_state[OLD] < L_ESTABLISHED && repl_state[NEW] >= L_ESTABLISHED)
+		resource->state_change_flags |= CS_HARD;
+
+	rv = end_state_change(resource, &irq_flags, "peer-state");
+	new_repl_state = peer_device->repl_state[NOW];
+
+	if (rv < SS_SUCCESS)
+		goto fail;
+
+	if (old_peer_state.conn > L_OFF) {
+		if (new_repl_state > L_ESTABLISHED && peer_state.conn <= L_ESTABLISHED &&
+		    peer_state.disk != D_NEGOTIATING) {
+			/* we want resync, peer has not yet decided to sync... */
+			/* Nowadays only used when forcing a node into primary role and
+			   setting its disk to UpToDate with that */
+			drbd_send_uuids(peer_device, 0, 0);
+			drbd_send_current_state(peer_device);
+		}
+	}
+
+	clear_bit(DISCARD_MY_DATA, &peer_device->flags); /* Only relevant for agreed_pro_version < 117 */
+
+	drbd_md_sync(device); /* update connected indicator, effective_size, ... */
+
+	peer_device->last_repl_state = peer_state.conn;
+	return 0;
+fail:
+	change_cstate(connection, C_DISCONNECTING, CS_HARD);
+	return -EIO;
+}
+
+static int receive_sync_uuid(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct p_uuid *p = pi->data;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	device = peer_device->device;
+
+	wait_event(device->misc_wait,
+		   peer_device->repl_state[NOW] == L_WF_SYNC_UUID ||
+		   peer_device->repl_state[NOW] == L_BEHIND ||
+		   peer_device->repl_state[NOW] < L_ESTABLISHED ||
+		   device->disk_state[NOW] < D_NEGOTIATING);
+
+	/* D_ASSERT(device,  peer_device->repl_state[NOW] == L_WF_SYNC_UUID ); */
+
+	/* Here the _drbd_uuid_ functions are right, current should
+	   _not_ be rotated into the history */
+	if (get_ldev_if_state(device, D_NEGOTIATING)) {
+		_drbd_uuid_set_current(device, be64_to_cpu(p->uuid));
+		_drbd_uuid_set_bitmap(peer_device, 0UL);
+
+		drbd_print_uuids(peer_device, "updated sync uuid");
+		drbd_start_resync(peer_device, L_SYNC_TARGET, "peer-sync-uuid");
+
+		put_ldev(device);
+	} else
+		drbd_err(device, "Ignoring SyncUUID packet!\n");
+
+	return 0;
+}
+
+static void scale_bits(unsigned long *base, unsigned int num_4k, unsigned int scale)
+{
+	unsigned int bits = num_4k * BITS_PER_LONG;
+	unsigned int sbit;
+
+	if (scale == 0)
+		return;
+
+	for (sbit = 0; sbit < bits; sbit++)
+		if (test_bit(sbit, base))
+			__set_bit(sbit >> scale, base);
+}
+
+/*
+ * receive_bitmap_plain
+ *
+ * Return 0 when done, 1 when another iteration is needed, and a negative error
+ * code upon failure.
+ *
+ * Received bitmap is 4k per bit, need to aggregate by c->scale.
+ */
+static int
+receive_bitmap_plain(struct drbd_peer_device *peer_device, unsigned int size,
+		     struct bm_xfer_ctx *c)
+{
+	unsigned long *p;
+	unsigned int data_size = DRBD_SOCKET_BUFFER_SIZE -
+				 drbd_header_size(peer_device->connection);
+	unsigned int num_words_4k = min_t(size_t, data_size / sizeof(*p),
+				       (c->bm_words - c->word_offset) << c->scale);
+	unsigned int want = num_words_4k * sizeof(*p);
+	int err;
+
+	if (want != size) {
+		drbd_err(peer_device, "%s:want (%u) != size (%u)\n", __func__, want, size);
+		return -EIO;
+	}
+	if (want == 0)
+		return 0;
+	err = drbd_recv_all(peer_device->connection, (void **)&p, want);
+	if (err)
+		return err;
+
+	if ((num_words_4k & ((1 << c->scale)-1)) != 0) {
+		drbd_err(peer_device,
+			"number of words %u not aligned to scale %u while receiving bitmap\n",
+			num_words_4k, c->scale);
+		return -ERANGE;
+	}
+
+	if (get_ldev(peer_device->device)) {
+		scale_bits(p, num_words_4k, c->scale);
+		drbd_bm_merge_lel(peer_device, c->word_offset, num_words_4k >> c->scale, p);
+		put_ldev(peer_device->device);
+	} else {
+		drbd_err(peer_device, "lost backend device while receiving bitmap\n");
+		return -EIO;
+	}
+
+	c->word_offset += num_words_4k >> c->scale;
+	c->bit_offset = c->word_offset * BITS_PER_LONG;
+	c->bit_offset_4k = (c->word_offset << c->scale) * BITS_PER_LONG;
+	if (c->bit_offset > c->bm_bits)
+		c->bit_offset = c->bm_bits;
+
+	return 1;
+}
+
+static enum drbd_bitmap_code dcbp_get_code(struct p_compressed_bm *p)
+{
+	return (enum drbd_bitmap_code)(p->encoding & 0x0f);
+}
+
+static int dcbp_get_start(struct p_compressed_bm *p)
+{
+	return (p->encoding & 0x80) != 0;
+}
+
+static int dcbp_get_pad_bits(struct p_compressed_bm *p)
+{
+	return (p->encoding >> 4) & 0x7;
+}
+
+/*
+ * recv_bm_rle_bits
+ *
+ * Return 0 when done, 1 when another iteration is needed, and a negative error
+ * code upon failure.
+ */
+static int
+recv_bm_rle_bits(struct drbd_peer_device *peer_device,
+		struct p_compressed_bm *p,
+		 struct bm_xfer_ctx *c,
+		 unsigned int len)
+{
+	struct bitstream bs;
+	u64 look_ahead;
+	u64 rl_4k;
+	u64 tmp;
+	unsigned long s_4k = c->bit_offset_4k;
+	int toggle = dcbp_get_start(p);
+	int have;
+	int bits;
+
+	bitstream_init(&bs, p->code, len, dcbp_get_pad_bits(p));
+
+	bits = bitstream_get_bits(&bs, &look_ahead, 64);
+	if (bits < 0)
+		return -EIO;
+
+	for (have = bits; have > 0; s_4k += rl_4k, toggle = !toggle) {
+		bits = vli_decode_bits(&rl_4k, look_ahead);
+		if (bits <= 0)
+			return -EIO;
+
+		if (toggle) {
+			/* If peers bm_block_size is smaller than ours,
+			 * this may be a "partially" set bit ;-)
+			 * there is no such thing. Round down s, round up e.
+			 */
+			unsigned long s = s_4k >> c->scale;
+			unsigned long e = ((s_4k + rl_4k + (1UL << c->scale)-1) >> c->scale) - 1;
+
+			if (e >= c->bm_bits) {
+				drbd_err(peer_device, "bitmap overflow (e:%lu) while decoding bm RLE packet\n", e);
+				return -EIO;
+			}
+			drbd_bm_set_many_bits(peer_device, s, e);
+		}
+
+		if (have < bits) {
+			drbd_err(peer_device, "bitmap decoding error: h:%d b:%d la:0x%08llx l:%u/%u\n",
+				have, bits, look_ahead,
+				(unsigned int)(bs.cur.b - p->code),
+				(unsigned int)bs.buf_len);
+			return -EIO;
+		}
+		/* if we consumed all 64 bits, assign 0; >> 64 is "undefined"; */
+		if (likely(bits < 64))
+			look_ahead >>= bits;
+		else
+			look_ahead = 0;
+		have -= bits;
+
+		bits = bitstream_get_bits(&bs, &tmp, 64 - have);
+		if (bits < 0)
+			return -EIO;
+		look_ahead |= tmp << have;
+		have += bits;
+	}
+
+	c->bit_offset_4k = s_4k;
+	c->bit_offset = s_4k >> c->scale;
+	bm_xfer_ctx_bit_to_word_offset(c);
+
+	return (c->bit_offset_4k != c->bm_bits_4k);
+}
+
+/*
+ * decode_bitmap_c
+ *
+ * Return 0 when done, 1 when another iteration is needed, and a negative error
+ * code upon failure.
+ */
+static int
+decode_bitmap_c(struct drbd_peer_device *peer_device,
+		struct p_compressed_bm *p,
+		struct bm_xfer_ctx *c,
+		unsigned int len)
+{
+	if (dcbp_get_code(p) == RLE_VLI_Bits) {
+		struct drbd_device *device = peer_device->device;
+		int res;
+
+		if (!get_ldev(device)) {
+			drbd_err(peer_device, "lost backend device while receiving bitmap\n");
+			return -EIO;
+		}
+
+		res = recv_bm_rle_bits(peer_device, p, c, len - sizeof(*p));
+		put_ldev(device);
+		return res;
+	}
+
+	/* other variants had been implemented for evaluation,
+	 * but have been dropped as this one turned out to be "best"
+	 * during all our tests.
+	 */
+
+	drbd_err(peer_device, "receive_bitmap_c: unknown encoding %u\n", p->encoding);
+	change_cstate(peer_device->connection, C_PROTOCOL_ERROR, CS_HARD);
+	return -EIO;
+}
+
+void INFO_bm_xfer_stats(struct drbd_peer_device *peer_device,
+		const char *direction, struct bm_xfer_ctx *c)
+{
+	/* what would it take to transfer it "plaintext" */
+	unsigned int header_size = drbd_header_size(peer_device->connection);
+	unsigned int data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
+	unsigned int plain =
+		header_size * (DIV_ROUND_UP(c->bm_words, data_size) + 1) +
+		c->bm_words * sizeof(unsigned long);
+	unsigned int total = c->bytes[0] + c->bytes[1];
+	unsigned int r;
+
+	/* total can not be zero. but just in case: */
+	if (total == 0)
+		return;
+
+	/* don't report if not compressed */
+	if (total >= plain)
+		return;
+
+	/* total < plain. check for overflow, still */
+	r = (total > UINT_MAX/1000) ? (total / (plain/1000))
+				    : (1000 * total / plain);
+
+	if (r > 1000)
+		r = 1000;
+
+	r = 1000 - r;
+	drbd_info(peer_device, "%s bitmap stats [Bytes(packets)]: plain %u(%u), RLE %u(%u), "
+	     "total %u; compression: %u.%u%%\n",
+			direction,
+			c->bytes[1], c->packets[1],
+			c->bytes[0], c->packets[0],
+			total, r/10, r % 10);
+}
+
+static bool ready_for_bitmap(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	bool ready = true;
+
+	read_lock_irq(&resource->state_rwlock);
+	if (device->disk_state[NOW] == D_NEGOTIATING)
+		ready = false;
+	if (test_bit(TWOPC_STATE_CHANGE_PENDING, &resource->flags))
+		ready = false;
+	read_unlock_irq(&resource->state_rwlock);
+
+	return ready;
+}
+
+/* Since we are processing the bitfield from lower addresses to higher,
+   it does not matter if the process it in 32 bit chunks or 64 bit
+   chunks as long as it is little endian. (Understand it as byte stream,
+   beginning with the lowest byte...) If we would use big endian
+   we would need to process it from the highest address to the lowest,
+   in order to be agnostic to the 32 vs 64 bits issue.
+
+   returns 0 on failure, 1 if we successfully received it. */
+static int receive_bitmap(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	enum drbd_repl_state repl_state;
+	struct drbd_device *device;
+	struct bm_xfer_ctx c;
+	int err = -EIO;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	if (peer_device->bitmap_index == -1) {
+		drbd_err(peer_device, "No bitmap allocated in receive_bitmap()!\n");
+		return -EIO;
+	}
+	device = peer_device->device;
+
+	/* Final repl_states become visible when the disk leaves NEGOTIATING state */
+	wait_event_interruptible(device->resource->state_wait,
+				 ready_for_bitmap(device));
+
+	drbd_bm_slot_lock(peer_device, "receive bitmap", BM_LOCK_CLEAR | BM_LOCK_BULK);
+	/* you are supposed to send additional out-of-sync information
+	 * if you actually set bits during this phase */
+
+	if (!get_ldev(device))
+		goto out;
+
+	c = (struct bm_xfer_ctx) {
+		.bm_bits_4k = drbd_bm_bits_4k(device),
+		.bm_bits = drbd_bm_bits(device),
+		.bm_words = drbd_bm_words(device),
+		.scale = device->bitmap->bm_block_shift - BM_BLOCK_SHIFT_4k,
+	};
+	put_ldev(device);
+
+	for (;;) {
+		if (pi->cmd == P_BITMAP)
+			err = receive_bitmap_plain(peer_device, pi->size, &c);
+		else if (pi->cmd == P_COMPRESSED_BITMAP) {
+			/* MAYBE: sanity check that we speak proto >= 90,
+			 * and the feature is enabled! */
+			struct p_compressed_bm *p;
+
+			if (pi->size > DRBD_SOCKET_BUFFER_SIZE - drbd_header_size(connection)) {
+				drbd_err(device, "ReportCBitmap packet too large\n");
+				err = -EIO;
+				goto out;
+			}
+			if (pi->size <= sizeof(*p)) {
+				drbd_err(device, "ReportCBitmap packet too small (l:%u)\n", pi->size);
+				err = -EIO;
+				goto out;
+			}
+			err = drbd_recv_all(connection, (void **)&p, pi->size);
+			if (err)
+			       goto out;
+			err = decode_bitmap_c(peer_device, p, &c, pi->size);
+		} else {
+			drbd_warn(device, "receive_bitmap: cmd neither ReportBitMap nor ReportCBitMap (is 0x%x)", pi->cmd);
+			err = -EIO;
+			goto out;
+		}
+
+		c.packets[pi->cmd == P_BITMAP]++;
+		c.bytes[pi->cmd == P_BITMAP] += drbd_header_size(connection) + pi->size;
+
+		if (err <= 0) {
+			if (err < 0)
+				goto out;
+			break;
+		}
+		err = drbd_recv_header(connection, pi);
+		if (err)
+			goto out;
+	}
+
+	INFO_bm_xfer_stats(peer_device, "receive", &c);
+
+	repl_state = peer_device->repl_state[NOW];
+	if (repl_state == L_WF_BITMAP_T) {
+		err = drbd_send_bitmap(device, peer_device);
+		if (err)
+			goto out;
+	}
+
+	drbd_bm_slot_unlock(peer_device);
+
+	if (test_bit(B_RS_H_DONE, &peer_device->flags)) {
+		/* We have entered drbd_start_resync() since starting the bitmap exchange. */
+		drbd_warn(peer_device, "Received bitmap more than once; ignoring\n");
+	} else if (repl_state == L_WF_BITMAP_S) {
+		drbd_start_resync(peer_device, L_SYNC_SOURCE, "receive-bitmap");
+	} else if (repl_state == L_WF_BITMAP_T) {
+		if (connection->agreed_pro_version < 110) {
+			enum drbd_state_rv rv;
+
+			/* Omit CS_WAIT_COMPLETE and CS_SERIALIZE with this state
+			 * transition to avoid deadlocks. */
+			rv = stable_change_repl_state(peer_device, L_WF_SYNC_UUID, CS_VERBOSE,
+					"receive-bitmap");
+			D_ASSERT(device, rv == SS_SUCCESS);
+		} else {
+			drbd_start_resync(peer_device, L_SYNC_TARGET, "receive-bitmap");
+		}
+	} else {
+		/* admin may have requested C_DISCONNECTING,
+		 * other threads may have noticed network errors */
+		drbd_info(peer_device, "unexpected repl_state (%s) in receive_bitmap\n",
+			  drbd_repl_str(repl_state));
+	}
+
+	return 0;
+ out:
+	drbd_bm_slot_unlock(peer_device);
+	return err;
+}
+
+static int receive_skip(struct drbd_connection *connection, struct packet_info *pi)
+{
+	drbd_warn(connection, "skipping unknown optional packet type %d, l: %d!\n",
+		 pi->cmd, pi->size);
+
+	return ignore_remaining_packet(connection, pi->size);
+}
+
+static int receive_UnplugRemote(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_transport *transport = &connection->transport;
+
+	/* Make sure we've acked all the data associated
+	 * with the data requests being unplugged */
+	transport->class->ops.hint(transport, DATA_STREAM, QUICKACK);
+
+	/* just unplug all devices always, regardless which volume number */
+	drbd_unplug_all_devices(connection);
+
+	return 0;
+}
+
+static int receive_out_of_sync(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct p_block_desc *p = pi->data;
+	sector_t sector;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	sector = be64_to_cpu(p->sector);
+
+	/* see also process_one_request(), before drbd_send_out_of_sync().
+	 * Make sure any pending write requests that potentially may
+	 * set in-sync have drained, before setting it out-of-sync.
+	 * That should be implicit, because of the "epoch" and P_BARRIER logic,
+	 * But let's just double-check.
+	 */
+	conn_wait_active_ee_empty_or_disconnect(connection);
+	conn_wait_done_ee_empty_or_disconnect(connection);
+
+	drbd_set_out_of_sync(peer_device, sector, be32_to_cpu(p->blksize));
+
+	return 0;
+}
+
+static int receive_dagtag(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_dagtag *p = pi->data;
+
+	set_connection_dagtag(connection, be64_to_cpu(p->dagtag));
+	return 0;
+}
+
+struct drbd_connection *drbd_connection_by_node_id(struct drbd_resource *resource, int node_id)
+{
+	/* Caller needs to hold rcu_read_lock(), conf_update */
+	struct drbd_connection *connection;
+
+	for_each_connection_rcu(connection, resource) {
+		if (connection->peer_node_id == node_id)
+			return connection;
+	}
+
+	return NULL;
+}
+
+struct drbd_connection *drbd_get_connection_by_node_id(struct drbd_resource *resource, int node_id)
+{
+	struct drbd_connection *connection;
+
+	rcu_read_lock();
+	connection = drbd_connection_by_node_id(resource, node_id);
+	if (connection)
+		kref_get(&connection->kref);
+	rcu_read_unlock();
+
+	return connection;
+}
+
+static int receive_peer_dagtag(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	enum drbd_repl_state new_repl_state;
+	struct p_peer_dagtag *p = pi->data;
+	struct drbd_connection *lost_peer;
+	enum sync_strategy strategy = NO_SYNC;
+	s64 dagtag_offset;
+	int vnr = 0;
+
+	lost_peer = drbd_get_connection_by_node_id(resource, be32_to_cpu(p->node_id));
+	if (!lost_peer)
+		return 0;
+
+
+	if (lost_peer->cstate[NOW] == C_CONNECTED) {
+		drbd_ping_peer(lost_peer);
+		if (lost_peer->cstate[NOW] == C_CONNECTED)
+			goto out;
+	}
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		enum sync_strategy ps;
+		enum sync_rule rule;
+		int unused;
+
+		if (peer_device->repl_state[NOW] > L_ESTABLISHED)
+			goto out;
+		if (peer_device->device->disk_state[NOW] != D_CONSISTENT &&
+		    peer_device->device->disk_state[NOW] != D_UP_TO_DATE)
+			goto out;
+		if (!get_ldev(peer_device->device))
+			continue;
+		ps = drbd_uuid_compare(peer_device, &rule, &unused);
+		put_ldev(peer_device->device);
+
+		if (strategy == NO_SYNC) {
+			strategy = ps;
+			if (strategy != NO_SYNC &&
+			    strategy != SYNC_SOURCE_USE_BITMAP &&
+			    strategy != SYNC_TARGET_USE_BITMAP) {
+				drbd_info(peer_device,
+					  "receive_peer_dagatg(): %s by rule=%s\n",
+					  strategy_descriptor(strategy).name,
+					  drbd_sync_rule_str(rule));
+				goto out;
+			}
+		} else if (ps != strategy) {
+			drbd_err(peer_device,
+				 "receive_peer_dagatg(): Inconsistent resync directions %s %s\n",
+				 strategy_descriptor(strategy).name, strategy_descriptor(ps).name);
+			goto out;
+		}
+	}
+
+	/* We must wait until the other receiver thread has called the
+	 * cleanup_unacked_peer_requests() and drbd_notify_peers_lost_primary() functions. If we
+	 * become a resync target, the peer would complain about being in the wrong state when he
+	 * gets the bitmap before the P_PEER_DAGTAG packet.
+	 */
+	wait_event(resource->state_wait,
+		   !test_bit(NOTIFY_PEERS_LOST_PRIMARY, &lost_peer->flags));
+
+	dagtag_offset = atomic64_read(&lost_peer->last_dagtag_sector) - (s64)be64_to_cpu(p->dagtag);
+	if (strategy == SYNC_SOURCE_USE_BITMAP)  {
+		new_repl_state = L_WF_BITMAP_S;
+	} else if (strategy == SYNC_TARGET_USE_BITMAP)  {
+		new_repl_state = L_WF_BITMAP_T;
+	} else {
+		if (dagtag_offset > 0)
+			new_repl_state = L_WF_BITMAP_S;
+		else if (dagtag_offset < 0)
+			new_repl_state = L_WF_BITMAP_T;
+		else
+			new_repl_state = L_ESTABLISHED;
+	}
+
+	if (new_repl_state != L_ESTABLISHED) {
+		unsigned long irq_flags;
+		enum drbd_state_rv rv;
+
+		if (new_repl_state == L_WF_BITMAP_T) {
+			connection->after_reconciliation.dagtag_sector = be64_to_cpu(p->dagtag);
+			connection->after_reconciliation.lost_node_id = be32_to_cpu(p->node_id);
+		}
+
+		begin_state_change(resource, &irq_flags, CS_VERBOSE);
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			__change_repl_state(peer_device, new_repl_state);
+			set_bit(RECONCILIATION_RESYNC, &peer_device->flags);
+		}
+		rv = end_state_change(resource, &irq_flags, "receive-peer-dagtag");
+		if (rv == SS_SUCCESS)
+			drbd_info(connection, "Reconciliation resync because \'%s\' disappeared. (o=%d)\n",
+				  lost_peer->transport.net_conf->name, (int)dagtag_offset);
+		else if (rv == SS_NOTHING_TO_DO)
+			drbd_info(connection, "\'%s\' disappeared (o=%d), no reconciliation since one diskless\n",
+				  lost_peer->transport.net_conf->name, (int)dagtag_offset);
+			/* sanitize_state() silently removes the resync and the RECONCILIATION_RESYNC bit */
+		else
+			drbd_info(connection, "rv = %d", rv);
+	} else {
+		drbd_info(connection, "No reconciliation resync even though \'%s\' disappeared. (o=%d)\n",
+			  lost_peer->transport.net_conf->name, (int)dagtag_offset);
+
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+			drbd_bm_clear_many_bits(peer_device, 0, -1UL);
+	}
+
+out:
+	kref_put(&lost_peer->kref, drbd_destroy_connection);
+	return 0;
+}
+
+static bool drbd_diskless_moved_on(struct drbd_peer_device *peer_device, u64 current_uuid)
+{
+	struct drbd_device *device = peer_device->device;
+	u64 previous = peer_device->current_uuid;
+	bool from_the_past = false;
+
+	/* No exposed UUID => did not move on. */
+	if (!current_uuid)
+		return false;
+
+	/* Same as last time => did not move on. */
+	if ((previous & ~UUID_PRIMARY) == (current_uuid & ~UUID_PRIMARY))
+		return false;
+
+	/* Only consider the peer to have moved on if we were on the same UUID. */
+	if ((previous & ~UUID_PRIMARY) != (drbd_current_uuid(device) & ~UUID_PRIMARY))
+		return false;
+
+	if (get_ldev(device)) {
+		from_the_past =
+			drbd_find_bitmap_by_uuid(peer_device, current_uuid & ~UUID_PRIMARY) != -1;
+		if (!from_the_past)
+			from_the_past = uuid_in_my_history(device, current_uuid & ~UUID_PRIMARY);
+		put_ldev(device);
+	}
+
+	/* It is an old UUID => did not move on. */
+	if (from_the_past)
+		return false;
+
+	return true;
+}
+
+/* Accept a new current UUID generated on a diskless node, that just became primary
+   (or during handshake) */
+static int receive_current_uuid(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct p_current_uuid *p = pi->data;
+	u64 current_uuid, weak_nodes;
+	bool moved_on;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return config_unknown_volume(connection, pi);
+	device = peer_device->device;
+
+	current_uuid = be64_to_cpu(p->uuid);
+	weak_nodes = be64_to_cpu(p->weak_nodes);
+	weak_nodes |= NODE_MASK(peer_device->node_id);
+	moved_on = drbd_diskless_moved_on(peer_device, current_uuid);
+
+	peer_device->current_uuid = current_uuid;
+
+	if (get_ldev(device)) {
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[peer_device->node_id];
+		peer_md->flags |= MDF_NODE_EXISTS;
+		put_ldev(device);
+	}
+	if (connection->peer_role[NOW] == R_PRIMARY)
+		check_resync_source(device, weak_nodes);
+
+	if (connection->peer_role[NOW] == R_UNKNOWN) {
+		set_bit(CURRENT_UUID_RECEIVED, &peer_device->flags);
+		if (moved_on && device->disk_state[NOW] > D_OUTDATED)
+			peer_device->connect_state.disk = D_OUTDATED;
+		return 0;
+	}
+
+	if (current_uuid == drbd_current_uuid(device))
+		return 0;
+
+	if (peer_device->repl_state[NOW] >= L_ESTABLISHED &&
+	    get_ldev_if_state(device, D_UP_TO_DATE)) {
+		if (connection->peer_role[NOW] == R_PRIMARY) {
+			drbd_warn(peer_device, "received new current UUID: %016llX "
+				  "weak_nodes=%016llX\n", current_uuid, weak_nodes);
+			drbd_uuid_received_new_current(peer_device, current_uuid, weak_nodes);
+			drbd_md_sync_if_dirty(device);
+		} else if (moved_on) {
+			if (resource->remote_state_change)
+				set_bit(OUTDATE_ON_2PC_COMMIT, &device->flags);
+			else
+				change_disk_state(device, D_OUTDATED, CS_VERBOSE,
+						"receive-current-uuid", NULL);
+		}
+		put_ldev(device);
+	} else if (device->disk_state[NOW] == D_DISKLESS && resource->role[NOW] == R_PRIMARY) {
+		drbd_uuid_set_exposed(device, peer_device->current_uuid, true);
+	}
+
+	return 0;
+}
+
+static bool interval_is_adjacent(const struct drbd_interval *i1, const struct drbd_interval *i2)
+{
+	return i1->sector + (i1->size >> SECTOR_SHIFT) == i2->sector;
+}
+
+/* Advance caching pointers received_last and discard_last. Return next discard to be submitted. */
+static struct drbd_peer_request *drbd_advance_to_next_rs_discard(
+		struct drbd_peer_device *peer_device, unsigned int align, bool submit_all)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_request *peer_req;
+	struct drbd_peer_request *discard_last = peer_device->discard_last;
+	bool discard_range_end = false;
+
+	/* Advance received_last. */
+	peer_req = list_prepare_entry(peer_device->received_last,
+			&peer_device->resync_requests, recv_order);
+	list_for_each_entry_continue(peer_req, &peer_device->resync_requests, recv_order) {
+		if (!test_bit(INTERVAL_RECEIVED, &peer_req->i.flags) && !submit_all)
+			break;
+
+		if (peer_req->flags & EE_TRIM)
+			break;
+
+		peer_device->received_last = peer_req;
+	}
+
+	/* Advance discard_last. */
+	peer_req = discard_last ? discard_last :
+		list_prepare_entry(peer_device->received_last,
+				&peer_device->resync_requests, recv_order);
+	list_for_each_entry_continue(peer_req, &peer_device->resync_requests, recv_order) {
+		/* Consider submitting previous discards. */
+		if (discard_last && !interval_is_adjacent(&discard_last->i, &peer_req->i)) {
+			discard_range_end = true;
+			break;
+		}
+
+		if (!(peer_req->flags & EE_TRIM)) {
+			discard_range_end =
+				test_bit(INTERVAL_RECEIVED, &peer_req->i.flags) || submit_all;
+			break;
+		}
+
+		discard_last = peer_req;
+
+		if (IS_ALIGNED(peer_req->i.sector + (peer_req->i.size >> SECTOR_SHIFT), align)) {
+			discard_range_end = true;
+			break;
+		}
+	}
+
+	/*
+	 * If we haven't found a discard range, or that range is not
+	 * finished, then there is nothing to submit.
+	 */
+	if (!discard_last || !(discard_range_end || discard_last->flags & EE_LAST_RESYNC_REQUEST)) {
+		peer_device->discard_last = discard_last;
+		return NULL;
+	}
+
+	/* Find start of discard range. */
+	peer_req = list_next_entry(list_prepare_entry(peer_device->received_last,
+				&peer_device->resync_requests, recv_order), recv_order);
+
+	spin_lock(&device->interval_lock); /* irqs already disabled */
+	if (peer_req != discard_last) {
+		struct drbd_peer_request *peer_req_merged = peer_req;
+
+		list_for_each_entry_continue(peer_req_merged,
+				&peer_device->resync_requests, recv_order) {
+			drbd_remove_interval(&device->requests, &peer_req_merged->i);
+			drbd_clear_interval(&peer_req_merged->i);
+			peer_req_merged->w.cb = e_end_resync_block;
+			if (peer_req_merged == discard_last)
+				break;
+		}
+	}
+	drbd_update_interval_size(&peer_req->i,
+			discard_last->i.size +
+			((discard_last->i.sector - peer_req->i.sector) << SECTOR_SHIFT));
+	spin_unlock(&device->interval_lock);
+
+	peer_device->received_last = discard_last;
+	peer_device->discard_last = NULL;
+
+	return peer_req;
+}
+
+static void drbd_submit_rs_discard(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+
+	if (get_ldev(device)) {
+		list_del(&peer_req->w.list);
+
+		peer_req->w.cb = e_end_resync_block;
+		peer_req->opf = REQ_OP_DISCARD;
+
+		atomic_inc(&connection->backing_ee_cnt);
+		drbd_conflict_submit_resync_request(peer_req);
+
+		/* No put_ldev() here. Gets called in drbd_endio_write_sec_final(). */
+	} else {
+		LIST_HEAD(free_list);
+		struct drbd_peer_request *t;
+
+		if (drbd_ratelimit())
+			drbd_err(device, "Cannot discard on local disk.\n");
+
+		drbd_send_ack(peer_device, P_RS_NEG_ACK, peer_req);
+
+		drbd_remove_peer_req_interval(peer_req);
+		list_move_tail(&peer_req->w.list, &free_list);
+
+		spin_lock_irq(&connection->peer_reqs_lock);
+		drbd_unmerge_discard(peer_req, &free_list);
+		spin_unlock_irq(&connection->peer_reqs_lock);
+
+		list_for_each_entry_safe(peer_req, t, &free_list, w.list)
+			drbd_free_peer_req(peer_req);
+	}
+}
+
+/* Find and submit discards in resync_requests which are ready. */
+void drbd_process_rs_discards(struct drbd_peer_device *peer_device, bool submit_all)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_request *peer_req;
+	struct drbd_peer_request *pr_tmp;
+	unsigned int align = DRBD_MAX_RS_DISCARD_SIZE;
+	LIST_HEAD(work_list);
+
+	if (get_ldev(device)) {
+		/*
+		 * Limit the size of the merged requests. We want to allow the size to
+		 * increase up to the backing discard granularity.  If that is smaller
+		 * than DRBD_MAX_RS_DISCARD_SIZE, then allow merging up to a size of
+		 * DRBD_MAX_RS_DISCARD_SIZE.
+		 */
+		align = max(DRBD_MAX_RS_DISCARD_SIZE, bdev_discard_granularity(
+					device->ldev->backing_bdev)) >> SECTOR_SHIFT;
+		put_ldev(device);
+	}
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	while (true) {
+		peer_req = drbd_advance_to_next_rs_discard(peer_device, align, submit_all);
+		if (!peer_req)
+			break;
+
+		list_add_tail(&peer_req->w.list, &work_list);
+	}
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	list_for_each_entry_safe(peer_req, pr_tmp, &work_list, w.list)
+		drbd_submit_rs_discard(peer_req); /* removes it from the work_list */
+}
+
+static int receive_rs_deallocated(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct drbd_peer_request *peer_req;
+	sector_t sector;
+	int size, err = 0;
+	u64 block_id;
+	u64 im;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	device = peer_device->device;
+
+	if (pi->cmd == P_RS_DEALLOCATED) {
+		struct p_block_desc *p = pi->data;
+
+		sector = be64_to_cpu(p->sector);
+		size = be32_to_cpu(p->blksize);
+		block_id = ID_SYNCER;
+	} else { /* P_RS_DEALLOCATED_ID */
+		struct p_block_ack *p = pi->data;
+
+		sector = be64_to_cpu(p->sector);
+		size = be32_to_cpu(p->blksize);
+		block_id = p->block_id;
+	}
+
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_RESYNC_WRITE),
+			sector, size, block_id);
+	if (!peer_req)
+		return -EIO;
+
+	dec_rs_pending(peer_device);
+	inc_unacked(peer_device);
+	atomic_add(size >> 9, &device->rs_sect_ev);
+	peer_req->flags |= EE_TRIM;
+
+	/* Setting all peers out of sync here. The sync source peer will be
+	 * set in sync when the discard completes. The sync source will soon
+	 * set other peers in sync with a P_PEERS_IN_SYNC packet.
+	 */
+	drbd_set_all_out_of_sync(device, sector, size);
+	drbd_process_rs_discards(peer_device, false);
+	rs_sectors_came_in(peer_device, size);
+
+	for_each_peer_device_ref(peer_device, im, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+
+		if (repl_is_sync_source(repl_state) || repl_state == L_WF_BITMAP_S)
+			drbd_send_out_of_sync(peer_device, sector, size);
+	}
+
+	return err;
+}
+
+void drbd_last_resync_request(struct drbd_peer_device *peer_device, bool submit_all)
+{
+	struct drbd_connection *connection = peer_device->connection;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	if (!list_empty(&peer_device->resync_requests)) {
+		struct drbd_peer_request *peer_req = list_last_entry(&peer_device->resync_requests,
+				struct drbd_peer_request, recv_order);
+		peer_req->flags |= EE_LAST_RESYNC_REQUEST;
+	}
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	drbd_process_rs_discards(peer_device, submit_all);
+}
+
+static int receive_disconnect(struct drbd_connection *connection, struct packet_info *pi)
+{
+	change_cstate_tag(connection, C_DISCONNECTING, CS_HARD, "receive-disconnect", NULL);
+	return 0;
+}
+
+struct data_cmd {
+	int expect_payload;
+	unsigned int pkt_size;
+	int (*fn)(struct drbd_connection *, struct packet_info *);
+};
+
+static struct data_cmd drbd_cmd_handler[] = {
+	[P_DATA]	    = { 1, sizeof(struct p_data), receive_Data },
+	[P_DATA_REPLY]	    = { 1, sizeof(struct p_data), receive_DataReply },
+	[P_RS_DATA_REPLY]   = { 1, sizeof(struct p_data), receive_RSDataReply },
+	[P_BARRIER]	    = { 0, sizeof(struct p_barrier), receive_Barrier },
+	[P_BITMAP]	    = { 1, 0, receive_bitmap },
+	[P_COMPRESSED_BITMAP] = { 1, 0, receive_bitmap },
+	[P_UNPLUG_REMOTE]   = { 0, 0, receive_UnplugRemote },
+	[P_DATA_REQUEST]    = { 0, sizeof(struct p_block_req), receive_data_request },
+	[P_RS_DATA_REQUEST] = { 0, sizeof(struct p_block_req), receive_data_request },
+	[P_SYNC_PARAM]	    = { 1, 0, receive_SyncParam },
+	[P_SYNC_PARAM89]    = { 1, 0, receive_SyncParam },
+	[P_PROTOCOL]        = { 1, sizeof(struct p_protocol), receive_protocol },
+	[P_UUIDS]	    = { 0, sizeof(struct p_uuids), receive_uuids },
+	[P_SIZES]	    = { 0, sizeof(struct p_sizes), receive_sizes },
+	[P_STATE]	    = { 0, sizeof(struct p_state), receive_state },
+	[P_STATE_CHG_REQ]   = { 0, sizeof(struct p_req_state), receive_req_state },
+	[P_SYNC_UUID]       = { 0, sizeof(struct p_uuid), receive_sync_uuid },
+	[P_OV_REQUEST]      = { 0, sizeof(struct p_block_req), receive_data_request },
+	[P_OV_REPLY]        = { 1, sizeof(struct p_block_req), receive_ov_reply },
+	[P_CSUM_RS_REQUEST] = { 1, sizeof(struct p_block_req), receive_data_request },
+	[P_RS_THIN_REQ]     = { 0, sizeof(struct p_block_req), receive_data_request },
+	[P_DELAY_PROBE]     = { 0, sizeof(struct p_delay_probe93), receive_skip },
+	[P_OUT_OF_SYNC]     = { 0, sizeof(struct p_block_desc), receive_out_of_sync },
+	[P_CONN_ST_CHG_REQ] = { 0, sizeof(struct p_req_state), receive_req_state },
+	[P_PROTOCOL_UPDATE] = { 1, sizeof(struct p_protocol), receive_protocol },
+	[P_TWOPC_PREPARE] = { 0, sizeof(struct p_twopc_request), receive_twopc },
+	[P_TWOPC_PREP_RSZ]  = { 0, sizeof(struct p_twopc_request), receive_twopc },
+	[P_TWOPC_ABORT] = { 0, sizeof(struct p_twopc_request), receive_twopc },
+	[P_DAGTAG]	    = { 0, sizeof(struct p_dagtag), receive_dagtag },
+	[P_UUIDS110]	    = { 1, sizeof(struct p_uuids110), receive_uuids110 },
+	[P_PEER_DAGTAG]     = { 0, sizeof(struct p_peer_dagtag), receive_peer_dagtag },
+	[P_CURRENT_UUID]    = { 0, sizeof(struct p_current_uuid), receive_current_uuid },
+	[P_TWOPC_COMMIT]    = { 0, sizeof(struct p_twopc_request), receive_twopc },
+	[P_TRIM]	    = { 0, sizeof(struct p_trim), receive_Data },
+	[P_ZEROES]	    = { 0, sizeof(struct p_trim), receive_Data },
+	[P_RS_DEALLOCATED]  = { 0, sizeof(struct p_block_desc), receive_rs_deallocated },
+	[P_RS_DEALLOCATED_ID] = { 0, sizeof(struct p_block_ack), receive_rs_deallocated },
+	[P_DISCONNECT]      = { 0, 0, receive_disconnect },
+	[P_RS_DAGTAG_REQ]   = { 0, sizeof(struct p_rs_req), receive_dagtag_data_request },
+	[P_RS_CSUM_DAGTAG_REQ]   = { 1, sizeof(struct p_rs_req), receive_dagtag_data_request },
+	[P_RS_THIN_DAGTAG_REQ]   = { 0, sizeof(struct p_rs_req), receive_dagtag_data_request },
+	[P_OV_DAGTAG_REQ]   = { 0, sizeof(struct p_rs_req), receive_dagtag_data_request },
+	[P_OV_DAGTAG_REPLY]   = { 1, sizeof(struct p_rs_req), receive_dagtag_ov_reply },
+	[P_FLUSH_REQUESTS]  = { 0, sizeof(struct p_flush_requests), receive_flush_requests },
+	[P_FLUSH_REQUESTS_ACK] = { 0, sizeof(struct p_flush_ack), receive_flush_requests_ack },
+	[P_ENABLE_REPLICATION_NEXT] = { 0, sizeof(struct p_enable_replication),
+		receive_enable_replication_next },
+	[P_ENABLE_REPLICATION] = { 0, sizeof(struct p_enable_replication),
+		receive_enable_replication },
+};
+
+static void drbdd(struct drbd_connection *connection)
+{
+	struct packet_info pi;
+	size_t shs; /* sub header size */
+	int err;
+
+	while (get_t_state(&connection->receiver) == RUNNING) {
+		struct data_cmd const *cmd;
+
+		drbd_thread_current_set_cpu(&connection->receiver);
+		update_receiver_timing_details(connection, drbd_recv_header_maybe_unplug);
+		if (drbd_recv_header_maybe_unplug(connection, &pi))
+			goto err_out;
+
+		cmd = &drbd_cmd_handler[pi.cmd];
+		if (unlikely(pi.cmd >= ARRAY_SIZE(drbd_cmd_handler) || !cmd->fn)) {
+			drbd_err(connection, "Unexpected data packet %s (0x%04x)",
+				 drbd_packet_name(pi.cmd), pi.cmd);
+			goto err_out;
+		}
+
+		shs = cmd->pkt_size;
+		if (pi.cmd == P_SIZES && connection->agreed_features & DRBD_FF_WSAME)
+			shs += sizeof(struct o_qlim);
+		if (pi.size > shs && !cmd->expect_payload) {
+			drbd_err(connection, "No payload expected %s l:%d\n",
+				 drbd_packet_name(pi.cmd), pi.size);
+			goto err_out;
+		}
+		if (pi.size < shs) {
+			drbd_err(connection, "%s: unexpected packet size, expected:%d received:%d\n",
+				 drbd_packet_name(pi.cmd), (int)shs, pi.size);
+			goto err_out;
+		}
+
+		if (shs) {
+			update_receiver_timing_details(connection, drbd_recv_all_warn);
+			err = drbd_recv_all_warn(connection, &pi.data, shs);
+			if (err)
+				goto err_out;
+			pi.size -= shs;
+		}
+
+		update_receiver_timing_details(connection, cmd->fn);
+		err = cmd->fn(connection, &pi);
+		if (err) {
+			drbd_err(connection, "error receiving %s, e: %d l: %d!\n",
+				 drbd_packet_name(pi.cmd), err, pi.size);
+			goto err_out;
+		}
+	}
+	return;
+
+    err_out:
+	change_cstate(connection, C_PROTOCOL_ERROR, CS_HARD);
+}
+
+static void drbd_cancel_conflicting_resync_requests(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+	struct conflict_worker *submit_conflict = &device->submit_conflict;
+	struct rb_node *node;
+	bool any_queued = false;
+
+	spin_lock_irq(&device->interval_lock);
+	for (node = rb_first(&device->requests); node; node = rb_next(node)) {
+		struct drbd_interval *i = rb_entry(node, struct drbd_interval, rb);
+		struct drbd_peer_request *peer_req;
+
+		if (!drbd_interval_is_resync(i))
+			continue;
+
+		peer_req = container_of(i, struct drbd_peer_request, i);
+
+		if (peer_req->peer_device != peer_device)
+			continue;
+
+		/* Only cancel requests which are waiting for conflicts to resolve. */
+		if (test_bit(INTERVAL_SUBMITTED, &i->flags) ||
+				(test_bit(INTERVAL_SENT, &i->flags) &&
+				 !test_bit(INTERVAL_RECEIVED, &i->flags)) ||
+				test_bit(INTERVAL_CANCELED, &i->flags))
+			continue;
+
+		set_bit(INTERVAL_CANCELED, &i->flags);
+
+		dynamic_drbd_dbg(device,
+				"Cancel %s %s request at %llus+%u (sent=%d)\n",
+				test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags) ?
+					"already queued" : "unqueued",
+				drbd_interval_type_str(i),
+				(unsigned long long) i->sector, i->size,
+				test_bit(INTERVAL_SENT, &i->flags));
+
+		if (test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags))
+			continue;
+
+		set_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags);
+
+		spin_lock(&submit_conflict->lock);
+		switch (i->type) {
+		case INTERVAL_RESYNC_WRITE:
+			list_add_tail(&peer_req->w.list, &submit_conflict->resync_writes);
+			break;
+		case INTERVAL_RESYNC_READ:
+			list_add_tail(&peer_req->w.list, &submit_conflict->resync_reads);
+			break;
+		default:
+			drbd_err(peer_device, "Unexpected interval type in %s\n", __func__);
+		}
+		spin_unlock(&submit_conflict->lock);
+
+		any_queued = true;
+	}
+	spin_unlock_irq(&device->interval_lock);
+
+	if (any_queued)
+		queue_work(submit_conflict->wq, &submit_conflict->worker);
+}
+
+static void cancel_dagtag_dependent_requests(struct drbd_resource *resource, unsigned int node_id)
+{
+	struct drbd_connection *connection;
+	LIST_HEAD(work_list);
+	struct drbd_peer_request *peer_req, *t;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		spin_lock_irq(&connection->peer_reqs_lock);
+		list_for_each_entry(peer_req, &connection->dagtag_wait_ee, w.list) {
+			if (peer_req->depend_dagtag_node_id != node_id)
+				continue;
+
+			dynamic_drbd_dbg(peer_req->peer_device, "%s at %llus+%u: Wait for dagtag %llus from peer %u cancelled\n",
+					drbd_interval_type_str(&peer_req->i),
+					(unsigned long long) peer_req->i.sector, peer_req->i.size,
+					(unsigned long long) peer_req->depend_dagtag,
+					node_id);
+
+			list_move_tail(&peer_req->w.list, &work_list);
+			break;
+		}
+		spin_unlock_irq(&connection->peer_reqs_lock);
+	}
+	rcu_read_unlock();
+
+	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {
+		drbd_peer_resync_read_cancel(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+}
+
+static void cleanup_resync_leftovers(struct drbd_peer_device *peer_device)
+{
+	peer_device->rs_total = 0;
+	peer_device->rs_failed = 0;
+	D_ASSERT(peer_device, atomic_read(&peer_device->rs_pending_cnt) == 0);
+
+	timer_delete_sync(&peer_device->resync_timer);
+	resync_timer_fn(&peer_device->resync_timer);
+	timer_delete_sync(&peer_device->start_resync_timer);
+}
+
+static void free_waiting_resync_requests(struct drbd_connection *connection)
+{
+	LIST_HEAD(free_list);
+	struct drbd_peer_device *peer_device;
+	struct drbd_peer_request *peer_req, *t;
+	int vnr;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		list_for_each_entry_safe(peer_req, t, &peer_device->resync_requests, recv_order) {
+			drbd_list_del_resync_request(peer_req);
+			list_add_tail(&peer_req->w.list, &free_list);
+		}
+	}
+	rcu_read_unlock();
+
+	list_for_each_entry_safe(peer_req, t, &connection->peer_reads, recv_order) {
+		if (peer_req->i.type == INTERVAL_PEER_READ)
+			continue;
+
+		peer_req->flags &= ~EE_ON_RECV_ORDER;
+		list_del(&peer_req->recv_order);
+
+		if (peer_req->i.type == INTERVAL_RESYNC_READ)
+			list_move_tail(&peer_req->w.list, &free_list); /* from resync_ack_ee */
+		else
+			list_add_tail(&peer_req->w.list, &free_list);
+	}
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	list_for_each_entry_safe(peer_req, t, &free_list, w.list) {
+		/*
+		 * Resync write requests waiting for peers-in-sync to be sent
+		 * just need to be freed.
+		 */
+		if (test_bit(INTERVAL_COMPLETED, &peer_req->i.flags)) {
+			drbd_free_peer_req(peer_req);
+			continue;
+		}
+
+		D_ASSERT(connection, test_bit(INTERVAL_SENT, &peer_req->i.flags));
+		D_ASSERT(connection, !test_bit(INTERVAL_RECEIVED, &peer_req->i.flags));
+		D_ASSERT(connection, !(peer_req->flags & EE_TRIM));
+
+		if (peer_req->i.type == INTERVAL_RESYNC_READ)
+			atomic_sub(peer_req->i.size >> 9, &connection->rs_in_flight);
+
+		dec_rs_pending(peer_req->peer_device);
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+}
+
+static void free_dagtag_wait_requests(struct drbd_connection *connection)
+{
+	LIST_HEAD(dagtag_wait_work_list);
+	struct drbd_peer_request *peer_req, *t;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_splice_init(&connection->dagtag_wait_ee, &dagtag_wait_work_list);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	list_for_each_entry_safe(peer_req, t, &dagtag_wait_work_list, w.list) {
+		struct drbd_peer_device *peer_device = peer_req->peer_device;
+
+		/* Verify requests are placed in the interval tree when
+		 * the request is made, so they need to be removed if
+		 * the reply was waiting for a dagtag to be reached. */
+		if (peer_req->i.type == INTERVAL_OV_READ_SOURCE)
+			drbd_remove_peer_req_interval(peer_req);
+
+		drbd_free_peer_req(peer_req);
+		dec_unacked(peer_device);
+		put_ldev(peer_device->device);
+	}
+}
+
+static void drain_resync_activity(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	/*
+	 * In order to understand this function, refer to the flow diagrams in
+	 * the comments for make_resync_request(), make_ov_request() and
+	 * receive_dagtag_data_request().
+	 */
+
+	/*
+	 * We could receive data from a peer at any point. This might release a
+	 * request that is waiting for a dagtag. That would cause it to
+	 * progress to waiting for conflicts or the backing disk. So we need to
+	 * remove these requests before flushing the other stages.
+	 */
+	free_dagtag_wait_requests(connection);
+
+	/* Wait for w_resync_timer/w_e_send_csum to finish, if running. */
+	drbd_flush_workqueue(&connection->sender_work);
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+
+		kref_get(&device->kref);
+		rcu_read_unlock();
+
+		/* Cause remaining discards to be submitted. */
+		drbd_last_resync_request(peer_device, true);
+		/* Cause requests waiting due to conflicts to be canceled. */
+		drbd_cancel_conflicting_resync_requests(peer_device);
+
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+
+	/* Drain conflicting and backing requests. */
+	wait_event(connection->ee_wait, atomic_read(&connection->backing_ee_cnt) == 0);
+
+	/* Wait for work queued when backing requests finished. */
+	drbd_flush_workqueue(&connection->sender_work);
+
+	/* Clear up and remove requests that have progressed to done_ee. */
+	drbd_finish_peer_reqs(connection);
+
+	/*
+	 * Requests that are waiting for a resync reply must be removed from
+	 * the interval tree and then freed.
+	 */
+	free_waiting_resync_requests(connection);
+
+	/* Requests that are waiting for a dagtag on this connection must be
+	 * cancelled, because the dependency will never be fulfilled. */
+	cancel_dagtag_dependent_requests(connection->resource, connection->peer_node_id);
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+
+		kref_get(&device->kref);
+		rcu_read_unlock();
+
+		cleanup_resync_leftovers(peer_device);
+
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+}
+
+static void peer_device_disconnected(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+
+	if (test_and_clear_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags))
+		up_read_non_owner(&device->uuid_sem);
+
+	peer_device_init_connect_state(peer_device);
+
+	/* No need to start additional resyncs after reconnection. */
+	peer_device->resync_again = 0;
+
+	if (!drbd_suspended(device)) {
+		struct drbd_resource *resource = device->resource;
+
+		/* We need to create the new UUID immediately when we finish
+		   requests that did not reach the lost peer.
+		   But when we lost quorum we are going to finish those
+		   requests with error, therefore do not create the new UUID
+		   immediately! */
+		if (!list_empty(&resource->transfer_log) &&
+		    drbd_data_accessible(device, NOW) &&
+		    !test_bit(PRIMARY_LOST_QUORUM, &device->flags) &&
+		    test_and_clear_bit(NEW_CUR_UUID, &device->flags))
+			drbd_check_peers_new_current_uuid(device);
+	}
+
+	drbd_md_sync(device);
 
-		if (err) {
-			spin_lock_irq(&device->resource->req_lock);
-			list_del(&peer_req->w.list);
-			spin_unlock_irq(&device->resource->req_lock);
+	if (get_ldev(device)) {
+		drbd_bitmap_io(device, &drbd_bm_write_copy_pages, "write from disconnected",
+				BM_LOCK_BULK | BM_LOCK_SINGLE_SLOT, peer_device);
+		put_ldev(device);
+	}
+}
 
-			drbd_free_peer_req(device, peer_req);
-			put_ldev(device);
-			err = 0;
-			goto fail;
-		}
+static bool initiator_can_commit_or_abort(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	bool remote = resource->twopc_reply.initiator_node_id != resource->res_opts.node_id;
 
-		inc_unacked(device);
+	if (remote) {
+		u64 parents = resource->twopc_parent_nodes & ~NODE_MASK(connection->peer_node_id);
 
-		/* No put_ldev() here. Gets called in drbd_endio_write_sec_final(),
-		   as well as drbd_rs_complete_io() */
-	} else {
-	fail:
-		drbd_rs_complete_io(device, sector);
-		drbd_send_ack_ex(peer_device, P_NEG_ACK, sector, size, ID_SYNCER);
+		if (!parents)
+			return false;
+		resource->twopc_parent_nodes = parents;
 	}
 
-	atomic_add(size >> 9, &device->rs_sect_in);
+	if (test_bit(TWOPC_PREPARED, &connection->flags) &&
+	    !(test_bit(TWOPC_YES, &connection->flags) ||
+	      test_bit(TWOPC_NO, &connection->flags) ||
+	      test_bit(TWOPC_RETRY, &connection->flags)))
+		return false;
 
-	return err;
+	return true;
 }
 
-struct data_cmd {
-	int expect_payload;
-	unsigned int pkt_size;
-	int (*fn)(struct drbd_connection *, struct packet_info *);
-};
+static void cleanup_remote_state_change(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct twopc_reply *reply = &resource->twopc_reply;
+	struct twopc_request request;
+	bool remote = false;
 
-static struct data_cmd drbd_cmd_handler[] = {
-	[P_DATA]	    = { 1, sizeof(struct p_data), receive_Data },
-	[P_DATA_REPLY]	    = { 1, sizeof(struct p_data), receive_DataReply },
-	[P_RS_DATA_REPLY]   = { 1, sizeof(struct p_data), receive_RSDataReply } ,
-	[P_BARRIER]	    = { 0, sizeof(struct p_barrier), receive_Barrier } ,
-	[P_BITMAP]	    = { 1, 0, receive_bitmap } ,
-	[P_COMPRESSED_BITMAP] = { 1, 0, receive_bitmap } ,
-	[P_UNPLUG_REMOTE]   = { 0, 0, receive_UnplugRemote },
-	[P_DATA_REQUEST]    = { 0, sizeof(struct p_block_req), receive_DataRequest },
-	[P_RS_DATA_REQUEST] = { 0, sizeof(struct p_block_req), receive_DataRequest },
-	[P_SYNC_PARAM]	    = { 1, 0, receive_SyncParam },
-	[P_SYNC_PARAM89]    = { 1, 0, receive_SyncParam },
-	[P_PROTOCOL]        = { 1, sizeof(struct p_protocol), receive_protocol },
-	[P_UUIDS]	    = { 0, sizeof(struct p_uuids), receive_uuids },
-	[P_SIZES]	    = { 0, sizeof(struct p_sizes), receive_sizes },
-	[P_STATE]	    = { 0, sizeof(struct p_state), receive_state },
-	[P_STATE_CHG_REQ]   = { 0, sizeof(struct p_req_state), receive_req_state },
-	[P_SYNC_UUID]       = { 0, sizeof(struct p_rs_uuid), receive_sync_uuid },
-	[P_OV_REQUEST]      = { 0, sizeof(struct p_block_req), receive_DataRequest },
-	[P_OV_REPLY]        = { 1, sizeof(struct p_block_req), receive_DataRequest },
-	[P_CSUM_RS_REQUEST] = { 1, sizeof(struct p_block_req), receive_DataRequest },
-	[P_RS_THIN_REQ]     = { 0, sizeof(struct p_block_req), receive_DataRequest },
-	[P_DELAY_PROBE]     = { 0, sizeof(struct p_delay_probe93), receive_skip },
-	[P_OUT_OF_SYNC]     = { 0, sizeof(struct p_block_desc), receive_out_of_sync },
-	[P_CONN_ST_CHG_REQ] = { 0, sizeof(struct p_req_state), receive_req_conn_state },
-	[P_PROTOCOL_UPDATE] = { 1, sizeof(struct p_protocol), receive_protocol },
-	[P_TRIM]	    = { 0, sizeof(struct p_trim), receive_Data },
-	[P_ZEROES]	    = { 0, sizeof(struct p_trim), receive_Data },
-	[P_RS_DEALLOCATED]  = { 0, sizeof(struct p_block_desc), receive_rs_deallocated },
-};
+	write_lock_irq(&resource->state_rwlock);
+	if (resource->remote_state_change && !initiator_can_commit_or_abort(connection)) {
+		remote = reply->initiator_node_id != resource->res_opts.node_id;
 
-static void drbdd(struct drbd_connection *connection)
+		if (remote)
+			request = (struct twopc_request) {
+				.nodes_to_reach = ~0,
+				.cmd = P_TWOPC_ABORT,
+				.tid = reply->tid,
+				.initiator_node_id = reply->initiator_node_id,
+				.target_node_id = reply->target_node_id,
+				.vnr = reply->vnr,
+			};
+
+		drbd_info(connection, "Aborting %s state change %u commit not possible\n",
+			  remote ? "remote" : "local", reply->tid);
+		if (remote) {
+			timer_delete(&resource->twopc_timer);
+			__clear_remote_state_change(resource);
+		} else {
+			enum alt_rv alt_rv = abort_local_transaction(connection, 0);
+			if (alt_rv != ALT_LOCKED)
+				return;
+		}
+	}
+	write_unlock_irq(&resource->state_rwlock);
+
+	/* for a local transaction, change_cluster_wide_state() sends the P_TWOPC_ABORTs */
+	if (remote)
+		nested_twopc_abort(resource, &request);
+}
+
+static void drbd_notify_peers_lost_primary(struct drbd_connection *lost_peer)
 {
-	struct packet_info pi;
-	size_t shs; /* sub header size */
-	int err;
+	struct drbd_resource *resource = lost_peer->resource;
+	struct drbd_connection *connection;
+	u64 im;
 
-	while (get_t_state(&connection->receiver) == RUNNING) {
-		struct data_cmd const *cmd;
+	for_each_connection_ref(connection, im, resource) {
+		struct drbd_peer_device *peer_device;
+		bool send_dagtag = false;
+		int vnr;
 
-		drbd_thread_current_set_cpu(&connection->receiver);
-		update_receiver_timing_details(connection, drbd_recv_header_maybe_unplug);
-		if (drbd_recv_header_maybe_unplug(connection, &pi))
-			goto err_out;
+		if (connection == lost_peer)
+			continue;
+		if (connection->cstate[NOW] != C_CONNECTED)
+			continue;
 
-		cmd = &drbd_cmd_handler[pi.cmd];
-		if (unlikely(pi.cmd >= ARRAY_SIZE(drbd_cmd_handler) || !cmd->fn)) {
-			drbd_err(connection, "Unexpected data packet %s (0x%04x)",
-				 cmdname(pi.cmd), pi.cmd);
-			goto err_out;
-		}
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			struct drbd_device *device = peer_device->device;
+			u64 current_uuid = drbd_current_uuid(device);
+			u64 weak_nodes = drbd_weak_nodes_device(device);
 
-		shs = cmd->pkt_size;
-		if (pi.cmd == P_SIZES && connection->agreed_features & DRBD_FF_WSAME)
-			shs += sizeof(struct o_qlim);
-		if (pi.size > shs && !cmd->expect_payload) {
-			drbd_err(connection, "No payload expected %s l:%d\n",
-				 cmdname(pi.cmd), pi.size);
-			goto err_out;
-		}
-		if (pi.size < shs) {
-			drbd_err(connection, "%s: unexpected packet size, expected:%d received:%d\n",
-				 cmdname(pi.cmd), (int)shs, pi.size);
-			goto err_out;
-		}
+			if (device->disk_state[NOW] < D_INCONSISTENT ||
+			    peer_device->disk_state[NOW] < D_INCONSISTENT)
+				continue; /* Ignore if one side is diskless */
 
-		if (shs) {
-			update_receiver_timing_details(connection, drbd_recv_all_warn);
-			err = drbd_recv_all_warn(connection, pi.data, shs);
-			if (err)
-				goto err_out;
-			pi.size -= shs;
+			drbd_send_current_uuid(peer_device, current_uuid, weak_nodes);
+			send_dagtag = true;
 		}
 
-		update_receiver_timing_details(connection, cmd->fn);
-		err = cmd->fn(connection, &pi);
-		if (err) {
-			drbd_err(connection, "error receiving %s, e: %d l: %d!\n",
-				 cmdname(pi.cmd), err, pi.size);
-			goto err_out;
-		}
+		if (send_dagtag)
+			drbd_send_peer_dagtag(connection, lost_peer);
 	}
-	return;
-
-    err_out:
-	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
 }
 
 static void conn_disconnect(struct drbd_connection *connection)
 {
+	struct drbd_resource *resource = connection->resource;
 	struct drbd_peer_device *peer_device;
-	enum drbd_conns oc;
-	int vnr;
+	enum drbd_conn_state oc;
+	unsigned long irq_flags;
+	int vnr, i;
+
+	clear_bit(CONN_DRY_RUN, &connection->flags);
+	clear_bit(CONN_CONGESTED, &connection->flags);
 
-	if (connection->cstate == C_STANDALONE)
+	if (connection->cstate[NOW] == C_STANDALONE)
 		return;
 
 	/* We are about to start the cleanup after connection loss.
-	 * Make sure drbd_make_request knows about that.
+	 * Make sure drbd_submit_bio knows about that.
 	 * Usually we should be in some network failure state already,
 	 * but just in case we are not, we fix it up here.
 	 */
-	conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
+	change_cstate_tag(connection, C_NETWORK_FAILURE, CS_HARD, "disconnected", NULL);
+
+	del_connect_timer(connection);
 
 	/* ack_receiver does not clean up anything. it must not interfere, either */
-	drbd_thread_stop(&connection->ack_receiver);
 	if (connection->ack_sender) {
 		destroy_workqueue(connection->ack_sender);
 		connection->ack_sender = NULL;
 	}
-	drbd_free_sock(connection);
+
+	/* restart sender thread,
+	 * potentially get it out of blocking network operations */
+	drbd_thread_stop(&connection->sender);
+	drbd_thread_start(&connection->sender);
+
+	mutex_lock(&resource->conf_update);
+	drbd_transport_shutdown(connection, CLOSE_CONNECTION);
+	mutex_unlock(&resource->conf_update);
+
+	cleanup_remote_state_change(connection);
+
+	drain_resync_activity(connection);
+
+	connection->after_reconciliation.lost_node_id = -1;
+
+	/* Wait for current activity to cease.  This includes waiting for
+	 * peer_request queued to the submitter workqueue. */
+	wait_event(connection->ee_wait,
+		atomic_read(&connection->active_ee_cnt) == 0);
+
+	/* wait for all w_e_end_data_req, w_e_end_rsdata_req, w_send_barrier,
+	 * etc. which may still be on the worker queue to be "canceled" */
+	drbd_flush_workqueue(&connection->sender_work);
+
+	drbd_finish_peer_reqs(connection);
+
+	/* This second workqueue flush is necessary, since drbd_finish_peer_reqs()
+	   might have issued a work again. The one before drbd_finish_peer_reqs() is
+	   necessary to reclaim net_ee in drbd_finish_peer_reqs(). */
+	drbd_flush_workqueue(&connection->sender_work);
 
 	rcu_read_lock();
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
 		struct drbd_device *device = peer_device->device;
+
 		kref_get(&device->kref);
 		rcu_read_unlock();
-		drbd_disconnected(peer_device);
+
+		peer_device_disconnected(peer_device);
+		if (get_ldev(device)) {
+			drbd_reconsider_queue_parameters(device, device->ldev);
+			put_ldev(device);
+		} else {
+			drbd_reconsider_queue_parameters(device, NULL);
+		}
+
 		kref_put(&device->kref, drbd_destroy_device);
 		rcu_read_lock();
 	}
 	rcu_read_unlock();
 
+	/* Apply these changes after peer_device_disconnected() because that
+	 * may cause the loss of other connections to be detected, which can
+	 * change the suspended state. */
+	tl_walk(connection, &connection->req_not_net_done,
+			resource->cached_susp ? CONNECTION_LOST_WHILE_SUSPENDED : CONNECTION_LOST);
+
+	i = drbd_free_peer_reqs(connection, &connection->done_ee);
+	if (i)
+		drbd_info(connection, "done_ee not empty, killed %u entries\n", i);
+	i = drbd_free_peer_reqs(connection, &connection->dagtag_wait_ee);
+	if (i)
+		drbd_info(connection, "dagtag_wait_ee not empty, killed %u entries\n", i);
+	i = drbd_free_peer_reqs(connection, &connection->resync_ack_ee);
+	if (i)
+		drbd_info(connection, "resync_ack_ee not empty, killed %u entries\n", i);
+
+	cleanup_unacked_peer_requests(connection);
+	cleanup_peer_ack_list(connection);
+
+	i = atomic_read(&connection->pp_in_use);
+	if (i)
+		drbd_info(connection, "pp_in_use = %d, expected 0\n", i);
+	i = atomic_read(&connection->pp_in_use_by_net);
+	if (i)
+		drbd_info(connection, "pp_in_use_by_net = %d, expected 0\n", i);
+
 	if (!list_empty(&connection->current_epoch->list))
 		drbd_err(connection, "ASSERTION FAILED: connection->current_epoch->list not empty\n");
 	/* ok, no more ee's on the fly, it is safe to reset the epoch_size */
 	atomic_set(&connection->current_epoch->epoch_size, 0);
 	connection->send.seen_any_write_yet = false;
+	connection->send.current_dagtag_sector =
+		resource->dagtag_sector - ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT) - 1;
+	connection->current_epoch->oldest_unconfirmed_peer_req = NULL;
+
+	/* Indicate that last_dagtag_sector may no longer be up-to-date. We
+	 * need to keep last_dagtag_sector because we may still need it to
+	 * resolve a reconciliation resync. However, we need to avoid issuing a
+	 * resync request dependent on that dagtag because the resync source
+	 * may not be aware of the dagtag, even though it has newer data. This
+	 * can occur if the peer has been re-started since the request with the
+	 * dagtag.
+	 * */
+	clear_bit(RECEIVED_DAGTAG, &connection->flags);
+
+	/* Release any threads waiting for a barrier to be acked. */
+	clear_bit(BARRIER_ACK_PENDING, &connection->flags);
+	wake_up(&resource->barrier_wait);
 
 	drbd_info(connection, "Connection closed\n");
 
-	if (conn_highest_role(connection) == R_PRIMARY && conn_highest_pdsk(connection) >= D_UNKNOWN)
+	if (resource->role[NOW] == R_PRIMARY &&
+	    connection->fencing_policy != FP_DONT_CARE &&
+	    conn_highest_pdsk(connection) >= D_UNKNOWN)
 		conn_try_outdate_peer_async(connection);
 
-	spin_lock_irq(&connection->resource->req_lock);
-	oc = connection->cstate;
-	if (oc >= C_UNCONNECTED)
-		_conn_request_state(connection, NS(conn, C_UNCONNECTED), CS_VERBOSE);
-
-	spin_unlock_irq(&connection->resource->req_lock);
-
-	if (oc == C_DISCONNECTING)
-		conn_request_state(connection, NS(conn, C_STANDALONE), CS_VERBOSE | CS_HARD);
-}
-
-static int drbd_disconnected(struct drbd_peer_device *peer_device)
-{
-	struct drbd_device *device = peer_device->device;
-	unsigned int i;
-
-	/* wait for current activity to cease. */
-	spin_lock_irq(&device->resource->req_lock);
-	_drbd_wait_ee_list_empty(device, &device->active_ee);
-	_drbd_wait_ee_list_empty(device, &device->sync_ee);
-	_drbd_wait_ee_list_empty(device, &device->read_ee);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	/* We do not have data structures that would allow us to
-	 * get the rs_pending_cnt down to 0 again.
-	 *  * On C_SYNC_TARGET we do not have any data structures describing
-	 *    the pending RSDataRequest's we have sent.
-	 *  * On C_SYNC_SOURCE there is no data structure that tracks
-	 *    the P_RS_DATA_REPLY blocks that we sent to the SyncTarget.
-	 *  And no, it is not the sum of the reference counts in the
-	 *  resync_LRU. The resync_LRU tracks the whole operation including
-	 *  the disk-IO, while the rs_pending_cnt only tracks the blocks
-	 *  on the fly. */
-	drbd_rs_cancel_all(device);
-	device->rs_total = 0;
-	device->rs_failed = 0;
-	atomic_set(&device->rs_pending_cnt, 0);
-	wake_up(&device->misc_wait);
-
-	timer_delete_sync(&device->resync_timer);
-	resync_timer_fn(&device->resync_timer);
-
-	/* wait for all w_e_end_data_req, w_e_end_rsdata_req, w_send_barrier,
-	 * w_make_resync_request etc. which may still be on the worker queue
-	 * to be "canceled" */
-	drbd_flush_workqueue(&peer_device->connection->sender_work);
-
-	drbd_finish_peer_reqs(device);
-
-	/* This second workqueue flush is necessary, since drbd_finish_peer_reqs()
-	   might have issued a work again. The one before drbd_finish_peer_reqs() is
-	   necessary to reclain net_ee in drbd_finish_peer_reqs(). */
-	drbd_flush_workqueue(&peer_device->connection->sender_work);
+	drbd_maybe_khelper(NULL, connection, "disconnected");
 
-	/* need to do it again, drbd_finish_peer_reqs() may have populated it
-	 * again via drbd_try_clear_on_disk_bm(). */
-	drbd_rs_cancel_all(device);
-
-	kfree(device->p_uuid);
-	device->p_uuid = NULL;
-
-	if (!drbd_suspended(device))
-		tl_clear(peer_device->connection);
-
-	drbd_md_sync(device);
-
-	if (get_ldev(device)) {
-		drbd_bitmap_io(device, &drbd_bm_write_copy_pages,
-				"write from disconnected", BM_LOCKED_CHANGE_ALLOWED, NULL);
-		put_ldev(device);
+	begin_state_change(resource, &irq_flags, CS_VERBOSE | CS_LOCAL_ONLY);
+	oc = connection->cstate[NOW];
+	if (oc >= C_UNCONNECTED) {
+		__change_cstate(connection, C_UNCONNECTED);
+		/* drbd_receiver() has to be restarted after it returns */
+		drbd_thread_restart_nowait(&connection->receiver);
 	}
+	end_state_change(resource, &irq_flags, "disconnected");
 
-	i = atomic_read(&device->pp_in_use_by_net);
-	if (i)
-		drbd_info(device, "pp_in_use_by_net = %d, expected 0\n", i);
-	i = atomic_read(&device->pp_in_use);
-	if (i)
-		drbd_info(device, "pp_in_use = %d, expected 0\n", i);
-
-	D_ASSERT(device, list_empty(&device->read_ee));
-	D_ASSERT(device, list_empty(&device->active_ee));
-	D_ASSERT(device, list_empty(&device->sync_ee));
-	D_ASSERT(device, list_empty(&device->done_ee));
+	if (test_bit(NOTIFY_PEERS_LOST_PRIMARY, &connection->flags)) {
+		drbd_notify_peers_lost_primary(connection);
+		clear_bit(NOTIFY_PEERS_LOST_PRIMARY, &connection->flags);
+	}
 
-	return 0;
+	if (oc == C_DISCONNECTING)
+		change_cstate_tag(connection, C_STANDALONE, CS_VERBOSE | CS_HARD | CS_LOCAL_ONLY,
+				"disconnected", NULL);
 }
 
 /*
- * We support PRO_VERSION_MIN to PRO_VERSION_MAX. The protocol version
- * we can agree on is stored in agreed_pro_version.
+ * We support PRO_VERSION_MIN to PRO_VERSION_MAX.
+ * But see also drbd_protocol_version_acceptable() and module parameter
+ * drbd_protocol_version_min.
+ * The protocol version we can agree on is stored in agreed_pro_version.
  *
  * feature flags and the reserved array should be enough room for future
  * enhancements of the handshake protocol, and possible plugins...
+ * See also PRO_FEATURES.
  *
- * for now, they are expected to be zero, but ignored.
  */
 static int drbd_send_features(struct drbd_connection *connection)
 {
-	struct drbd_socket *sock;
 	struct p_connection_features *p;
 
-	sock = &connection->data;
-	p = conn_prepare_command(connection, sock);
+	p = __conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
 	if (!p)
 		return -EIO;
 	memset(p, 0, sizeof(*p));
-	p->protocol_min = cpu_to_be32(PRO_VERSION_MIN);
+	p->protocol_min = cpu_to_be32(drbd_protocol_version_min);
 	p->protocol_max = cpu_to_be32(PRO_VERSION_MAX);
+	p->sender_node_id = cpu_to_be32(connection->resource->res_opts.node_id);
+	p->receiver_node_id = cpu_to_be32(connection->peer_node_id);
 	p->feature_flags = cpu_to_be32(PRO_FEATURES);
-	return conn_send_command(connection, sock, P_CONNECTION_FEATURES, sizeof(*p), NULL, 0);
+	return __send_command(connection, -1, P_CONNECTION_FEATURES, DATA_STREAM);
 }
 
 /*
@@ -5083,9 +9952,10 @@ static int drbd_send_features(struct drbd_connection *connection)
  *  -1 peer talks different language,
  *     no point in trying again, please go standalone.
  */
-static int drbd_do_features(struct drbd_connection *connection)
+int drbd_do_features(struct drbd_connection *connection)
 {
 	/* ASSERT current == connection->receiver ... */
+	struct drbd_resource *resource = connection->resource;
 	struct p_connection_features *p;
 	const int expect = sizeof(struct p_connection_features);
 	struct packet_info pi;
@@ -5096,12 +9966,15 @@ static int drbd_do_features(struct drbd_connection *connection)
 		return 0;
 
 	err = drbd_recv_header(connection, &pi);
-	if (err)
+	if (err) {
+		if (err == -EAGAIN)
+			drbd_err(connection, "timeout while waiting for feature packet\n");
 		return 0;
+	}
 
 	if (pi.cmd != P_CONNECTION_FEATURES) {
 		drbd_err(connection, "expected ConnectionFeatures packet, received: %s (0x%04x)\n",
-			 cmdname(pi.cmd), pi.cmd);
+			 drbd_packet_name(pi.cmd), pi.cmd);
 		return -1;
 	}
 
@@ -5111,8 +9984,7 @@ static int drbd_do_features(struct drbd_connection *connection)
 		return -1;
 	}
 
-	p = pi.data;
-	err = drbd_recv_all_warn(connection, p, expect);
+	err = drbd_recv_all_warn(connection, (void **)&p, expect);
 	if (err)
 		return 0;
 
@@ -5122,42 +9994,87 @@ static int drbd_do_features(struct drbd_connection *connection)
 		p->protocol_max = p->protocol_min;
 
 	if (PRO_VERSION_MAX < p->protocol_min ||
-	    PRO_VERSION_MIN > p->protocol_max)
-		goto incompat;
+	    drbd_protocol_version_min > p->protocol_max) {
+		drbd_err(connection, "incompatible DRBD dialects: "
+		    "I support %d-%d, peer supports %d-%d\n",
+		    drbd_protocol_version_min, PRO_VERSION_MAX,
+		    p->protocol_min, p->protocol_max);
+		return -1;
+	}
+	/* Older DRBD will always expect us to agree to their max,
+	 * if it falls within our [min, max] range.
+	 * But we have a gap in there that we do not support.
+	 */
+	if (p->protocol_max > PRO_VERSION_8_MAX &&
+	    p->protocol_max < PRO_VERSION_MIN) {
+		drbd_err(connection, "incompatible DRBD 9 dialects: I support %u-%u, peer supports %u-%u\n",
+		    PRO_VERSION_MIN, PRO_VERSION_MAX,
+		    p->protocol_min, p->protocol_max);
+		return -1;
+	}
 
 	connection->agreed_pro_version = min_t(int, PRO_VERSION_MAX, p->protocol_max);
 	connection->agreed_features = PRO_FEATURES & be32_to_cpu(p->feature_flags);
 
-	drbd_info(connection, "Handshake successful: "
-	     "Agreed network protocol version %d\n", connection->agreed_pro_version);
+	if (connection->agreed_pro_version < 110) {
+		struct drbd_connection *connection2;
+		bool multiple = false;
+
+		rcu_read_lock();
+		for_each_connection_rcu(connection2, resource) {
+			if (connection == connection2)
+				continue;
+			multiple = true;
+		}
+		rcu_read_unlock();
+
+		if (multiple) {
+			drbd_err(connection, "Peer supports protocols %d-%d, but "
+				 "multiple connections are only supported in protocol "
+				 "110 and above\n", p->protocol_min, p->protocol_max);
+			return -1;
+		}
+	}
+
+	if (connection->agreed_pro_version >= 110) {
+		if (be32_to_cpu(p->sender_node_id) != connection->peer_node_id) {
+			drbd_err(connection, "Peer presented a node_id of %d instead of %d\n",
+				 be32_to_cpu(p->sender_node_id), connection->peer_node_id);
+			return 0;
+		}
+		if (be32_to_cpu(p->receiver_node_id) != resource->res_opts.node_id) {
+			drbd_err(connection, "Peer expects me to have a node_id of %d instead of %d\n",
+				 be32_to_cpu(p->receiver_node_id), resource->res_opts.node_id);
+			return 0;
+		}
+	}
+
+	drbd_info(connection, "Handshake to peer %d successful: "
+			"Agreed network protocol version %d\n",
+			connection->peer_node_id,
+			connection->agreed_pro_version);
 
-	drbd_info(connection, "Feature flags enabled on protocol level: 0x%x%s%s%s%s.\n",
+	drbd_info(connection, "Feature flags enabled on protocol level: 0x%x%s%s%s%s%s\n",
 		  connection->agreed_features,
 		  connection->agreed_features & DRBD_FF_TRIM ? " TRIM" : "",
 		  connection->agreed_features & DRBD_FF_THIN_RESYNC ? " THIN_RESYNC" : "",
 		  connection->agreed_features & DRBD_FF_WSAME ? " WRITE_SAME" : "",
-		  connection->agreed_features & DRBD_FF_WZEROES ? " WRITE_ZEROES" :
+		  connection->agreed_features & DRBD_FF_WZEROES ? " WRITE_ZEROES" : "",
+		  connection->agreed_features & DRBD_FF_RESYNC_DAGTAG ? " RESYNC_DAGTAG" :
 		  connection->agreed_features ? "" : " none");
 
 	return 1;
-
- incompat:
-	drbd_err(connection, "incompatible DRBD dialects: "
-	    "I support %d-%d, peer supports %d-%d\n",
-	    PRO_VERSION_MIN, PRO_VERSION_MAX,
-	    p->protocol_min, p->protocol_max);
-	return -1;
 }
 
 #if !defined(CONFIG_CRYPTO_HMAC) && !defined(CONFIG_CRYPTO_HMAC_MODULE)
-static int drbd_do_auth(struct drbd_connection *connection)
+int drbd_do_auth(struct drbd_connection *connection)
 {
 	drbd_err(connection, "This kernel was build without CONFIG_CRYPTO_HMAC.\n");
 	drbd_err(connection, "You need to disable 'cram-hmac-alg' in drbd.conf.\n");
 	return -1;
 }
 #else
-#define CHALLENGE_LEN 64
+#define CHALLENGE_LEN 64 /* must be multiple of 4 */
 
 /* Return value:
 	1 - auth succeeded,
@@ -5165,25 +10082,28 @@ static int drbd_do_auth(struct drbd_connection *connection)
 	-1 - auth failed, don't try again.
 */
 
-static int drbd_do_auth(struct drbd_connection *connection)
+struct auth_challenge {
+	char d[CHALLENGE_LEN];
+	u32 i;
+} __attribute__((packed));
+
+int drbd_do_auth(struct drbd_connection *connection)
 {
-	struct drbd_socket *sock;
-	char my_challenge[CHALLENGE_LEN];  /* 64 Bytes... */
-	char *response = NULL;
+	struct auth_challenge my_challenge, *peers_ch = NULL;
+	void *response;
 	char *right_response = NULL;
-	char *peers_ch = NULL;
 	unsigned int key_len;
 	char secret[SHARED_SECRET_MAX]; /* 64 byte */
 	unsigned int resp_size;
 	struct shash_desc *desc;
 	struct packet_info pi;
 	struct net_conf *nc;
-	int err, rv;
-
-	/* FIXME: Put the challenge/response into the preallocated socket buffer.  */
+	int err, rv, dig_size;
+	bool peer_is_drbd_9 = connection->agreed_pro_version >= 110;
+	void *packet_body;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
+	nc = rcu_dereference(connection->transport.net_conf);
 	key_len = strlen(nc->shared_secret);
 	memcpy(secret, nc->shared_secret, key_len);
 	rcu_read_unlock();
@@ -5204,15 +10124,16 @@ static int drbd_do_auth(struct drbd_connection *connection)
 		goto fail;
 	}
 
-	get_random_bytes(my_challenge, CHALLENGE_LEN);
+	get_random_bytes(my_challenge.d, sizeof(my_challenge.d));
 
-	sock = &connection->data;
-	if (!conn_prepare_command(connection, sock)) {
+	packet_body = __conn_prepare_command(connection, sizeof(my_challenge.d), DATA_STREAM);
+	if (!packet_body) {
 		rv = 0;
 		goto fail;
 	}
-	rv = !conn_send_command(connection, sock, P_AUTH_CHALLENGE, 0,
-				my_challenge, CHALLENGE_LEN);
+	memcpy(packet_body, my_challenge.d, sizeof(my_challenge.d));
+
+	rv = !__send_command(connection, -1, P_AUTH_CHALLENGE, DATA_STREAM);
 	if (!rv)
 		goto fail;
 
@@ -5224,61 +10145,56 @@ static int drbd_do_auth(struct drbd_connection *connection)
 
 	if (pi.cmd != P_AUTH_CHALLENGE) {
 		drbd_err(connection, "expected AuthChallenge packet, received: %s (0x%04x)\n",
-			 cmdname(pi.cmd), pi.cmd);
-		rv = -1;
-		goto fail;
-	}
-
-	if (pi.size > CHALLENGE_LEN * 2) {
-		drbd_err(connection, "expected AuthChallenge payload too big.\n");
+			 drbd_packet_name(pi.cmd), pi.cmd);
 		rv = -1;
 		goto fail;
 	}
 
-	if (pi.size < CHALLENGE_LEN) {
-		drbd_err(connection, "AuthChallenge payload too small.\n");
+	if (pi.size != sizeof(peers_ch->d)) {
+		drbd_err(connection, "unexpected AuthChallenge payload.\n");
 		rv = -1;
 		goto fail;
 	}
 
-	peers_ch = kmalloc(pi.size, GFP_NOIO);
+	peers_ch = kmalloc_obj(*peers_ch, GFP_NOIO);
 	if (!peers_ch) {
 		rv = -1;
 		goto fail;
 	}
 
-	err = drbd_recv_all_warn(connection, peers_ch, pi.size);
+	err = drbd_recv_into(connection, peers_ch->d, sizeof(peers_ch->d));
 	if (err) {
 		rv = 0;
 		goto fail;
 	}
 
-	if (!memcmp(my_challenge, peers_ch, CHALLENGE_LEN)) {
+	if (!memcmp(my_challenge.d, peers_ch->d, sizeof(my_challenge.d))) {
 		drbd_err(connection, "Peer presented the same challenge!\n");
 		rv = -1;
 		goto fail;
 	}
 
 	resp_size = crypto_shash_digestsize(connection->cram_hmac_tfm);
-	response = kmalloc(resp_size, GFP_NOIO);
+	response = __conn_prepare_command(connection, resp_size, DATA_STREAM);
 	if (!response) {
-		rv = -1;
+		rv = 0;
 		goto fail;
 	}
 
-	rv = crypto_shash_digest(desc, peers_ch, pi.size, response);
+	dig_size = pi.size;
+	if (peer_is_drbd_9) {
+		peers_ch->i = cpu_to_be32(connection->resource->res_opts.node_id);
+		dig_size += sizeof(peers_ch->i);
+	}
+
+	rv = crypto_shash_digest(desc, peers_ch->d, dig_size, response);
 	if (rv) {
-		drbd_err(connection, "crypto_hash_digest() failed with %d\n", rv);
+		drbd_err(connection, "crypto_shash_digest() failed with %d\n", rv);
 		rv = -1;
 		goto fail;
 	}
 
-	if (!conn_prepare_command(connection, sock)) {
-		rv = 0;
-		goto fail;
-	}
-	rv = !conn_send_command(connection, sock, P_AUTH_RESPONSE, 0,
-				response, resp_size);
+	rv = !__send_command(connection, -1, P_AUTH_RESPONSE, DATA_STREAM);
 	if (!rv)
 		goto fail;
 
@@ -5290,18 +10206,19 @@ static int drbd_do_auth(struct drbd_connection *connection)
 
 	if (pi.cmd != P_AUTH_RESPONSE) {
 		drbd_err(connection, "expected AuthResponse packet, received: %s (0x%04x)\n",
-			 cmdname(pi.cmd), pi.cmd);
+			 drbd_packet_name(pi.cmd), pi.cmd);
 		rv = 0;
 		goto fail;
 	}
 
 	if (pi.size != resp_size) {
-		drbd_err(connection, "expected AuthResponse payload of wrong size\n");
+		drbd_err(connection, "expected AuthResponse payload of %u bytes, received %u\n",
+				resp_size, pi.size);
 		rv = 0;
 		goto fail;
 	}
 
-	err = drbd_recv_all_warn(connection, response , resp_size);
+	err = drbd_recv_all(connection, &response, resp_size);
 	if (err) {
 		rv = 0;
 		goto fail;
@@ -5313,10 +10230,15 @@ static int drbd_do_auth(struct drbd_connection *connection)
 		goto fail;
 	}
 
-	rv = crypto_shash_digest(desc, my_challenge, CHALLENGE_LEN,
-				 right_response);
+	dig_size = sizeof(my_challenge.d);
+	if (peer_is_drbd_9) {
+		my_challenge.i = cpu_to_be32(connection->peer_node_id);
+		dig_size += sizeof(my_challenge.i);
+	}
+
+	rv = crypto_shash_digest(desc, my_challenge.d, dig_size, right_response);
 	if (rv) {
-		drbd_err(connection, "crypto_hash_digest() failed with %d\n", rv);
+		drbd_err(connection, "crypto_shash_digest() failed with %d\n", rv);
 		rv = -1;
 		goto fail;
 	}
@@ -5331,108 +10253,273 @@ static int drbd_do_auth(struct drbd_connection *connection)
 
  fail:
 	kfree(peers_ch);
-	kfree(response);
 	kfree(right_response);
 	if (desc) {
 		shash_desc_zero(desc);
 		kfree(desc);
 	}
 
-	return rv;
-}
-#endif
+	return rv;
+}
+#endif
+
+int drbd_receiver(struct drbd_thread *thi)
+{
+	struct drbd_connection *connection = thi->connection;
+
+	if (conn_connect(connection)) {
+		blk_start_plug(&connection->receiver_plug);
+		drbdd(connection);
+		blk_finish_plug(&connection->receiver_plug);
+	}
+
+	conn_disconnect(connection);
+	return 0;
+}
+
+/* ********* acknowledge sender ******** */
+
+static void drbd_check_flush_dagtag_reached(struct drbd_connection *peer_ack_connection)
+{
+	struct drbd_resource *resource = peer_ack_connection->resource;
+	struct drbd_connection *flush_requests_connection;
+	u64 peer_ack_node_mask = NODE_MASK(peer_ack_connection->peer_node_id);
+	u64 last_peer_ack_dagtag_seen = peer_ack_connection->last_peer_ack_dagtag_seen;
+	u64 im;
+
+	for_each_connection_ref(flush_requests_connection, im, resource) {
+		u64 flush_sequence;
+		u64 *sent_mask;
+		u64 flush_requests_dagtag;
+
+		spin_lock_irq(&flush_requests_connection->primary_flush_lock);
+		flush_requests_dagtag = flush_requests_connection->flush_requests_dagtag;
+		flush_sequence = flush_requests_connection->flush_sequence;
+		sent_mask = &flush_requests_connection->flush_forward_sent_mask;
+
+		if (!flush_sequence || /* Active flushes use non-zero sequence numbers */
+				*sent_mask & peer_ack_node_mask ||
+				last_peer_ack_dagtag_seen < flush_requests_dagtag) {
+			spin_unlock_irq(&flush_requests_connection->primary_flush_lock);
+			continue;
+		}
+
+		*sent_mask |= peer_ack_node_mask;
+		spin_unlock_irq(&flush_requests_connection->primary_flush_lock);
+
+		if (peer_ack_connection == flush_requests_connection)
+			drbd_send_flush_requests_ack(peer_ack_connection,
+					flush_sequence,
+					resource->res_opts.node_id);
+		else
+			drbd_send_flush_forward(peer_ack_connection,
+					flush_sequence,
+					flush_requests_connection->peer_node_id);
+	}
+}
+
+static int process_peer_ack_list(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_ack *peer_ack, *tmp;
+	u64 node_id_mask;
+	int err = 0;
+
+	node_id_mask = NODE_MASK(connection->peer_node_id);
+
+	spin_lock_irq(&resource->peer_ack_lock);
+	peer_ack = list_first_entry(&resource->peer_ack_list, struct drbd_peer_ack, list);
+	while (&peer_ack->list != &resource->peer_ack_list) {
+		u64 pending_mask = peer_ack->pending_mask;
+		u64 mask = peer_ack->mask;
+		u64 dagtag_sector = peer_ack->dagtag_sector;
+
+		tmp = list_next_entry(peer_ack, list);
+
+		if (!(peer_ack->queued_mask & node_id_mask)) {
+			peer_ack = tmp;
+			continue;
+		}
+
+		/*
+		 * After disconnecting, queue_peer_ack_send() sets
+		 * last_peer_ack_dagtag_seen directly. Do not jump back if we
+		 * process a peer ack with a lower dagtag here shortly after.
+		 */
+		connection->last_peer_ack_dagtag_seen =
+			max(connection->last_peer_ack_dagtag_seen, dagtag_sector);
+
+		peer_ack->queued_mask &= ~node_id_mask;
+		drbd_destroy_peer_ack_if_done(peer_ack);
+		peer_ack = tmp;
+
+		if (!(pending_mask & node_id_mask))
+			continue;
+		spin_unlock_irq(&resource->peer_ack_lock);
+
+		err = drbd_send_peer_ack(connection, mask, dagtag_sector);
+
+		spin_lock_irq(&resource->peer_ack_lock);
+		if (err)
+			break;
+	}
+	spin_unlock_irq(&resource->peer_ack_lock);
+
+	if (!err && connection->agreed_pro_version >= 123)
+		drbd_check_flush_dagtag_reached(connection);
+
+	return err;
+}
+
+static int got_peers_in_sync(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct p_peer_block_desc *p = pi->data;
+	sector_t sector;
+	u64 in_sync_b;
+	int size;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	device = peer_device->device;
 
-int drbd_receiver(struct drbd_thread *thi)
-{
-	struct drbd_connection *connection = thi->connection;
-	int h;
+	if (get_ldev(device)) {
+		unsigned long modified;
 
-	drbd_info(connection, "receiver (re)started\n");
+		sector = be64_to_cpu(p->sector);
+		size = be32_to_cpu(p->size);
+		in_sync_b = node_ids_to_bitmap(device, be64_to_cpu(p->mask));
 
-	do {
-		h = conn_connect(connection);
-		if (h == 0) {
-			conn_disconnect(connection);
-			schedule_timeout_interruptible(HZ);
-		}
-		if (h == -1) {
-			drbd_warn(connection, "Discarding network configuration.\n");
-			conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
-		}
-	} while (h == 0);
+		modified = drbd_set_sync(device, sector, size, 0, in_sync_b);
 
-	if (h > 0) {
-		blk_start_plug(&connection->receiver_plug);
-		drbdd(connection);
-		blk_finish_plug(&connection->receiver_plug);
-	}
+		/* If we are SyncSource then we rely on P_PEERS_IN_SYNC from
+		 * the peer to inform us of sync progress. Otherwise only send
+		 * peers-in-sync when we have actually cleared some bits.
+		 * This prevents an infinite loop with the peer. */
+		if (modified || peer_device->repl_state[NOW] == L_SYNC_SOURCE)
+			drbd_queue_update_peers(peer_device, sector, sector + (size >> SECTOR_SHIFT));
 
-	conn_disconnect(connection);
+		put_ldev(device);
+	}
 
-	drbd_info(connection, "receiver terminated\n");
 	return 0;
 }
 
-/* ********* acknowledge sender ******** */
-
-static int got_conn_RqSReply(struct drbd_connection *connection, struct packet_info *pi)
+static int got_RqSReply(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct p_req_state_reply *p = pi->data;
 	int retcode = be32_to_cpu(p->retcode);
 
-	if (retcode >= SS_SUCCESS) {
-		set_bit(CONN_WD_ST_CHG_OKAY, &connection->flags);
-	} else {
-		set_bit(CONN_WD_ST_CHG_FAIL, &connection->flags);
-		drbd_err(connection, "Requested state change failed by peer: %s (%d)\n",
-			 drbd_set_st_err_str(retcode), retcode);
+	if (retcode >= SS_SUCCESS)
+		set_bit(TWOPC_YES, &connection->flags);
+	else {
+		set_bit(TWOPC_NO, &connection->flags);
+		dynamic_drbd_dbg(connection, "Requested state change failed by peer: %s (%d)\n",
+			   drbd_set_st_err_str(retcode), retcode);
 	}
-	wake_up(&connection->ping_wait);
+
+	wake_up_all(&connection->resource->state_wait);
 
 	return 0;
 }
 
-static int got_RqSReply(struct drbd_connection *connection, struct packet_info *pi)
-{
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_req_state_reply *p = pi->data;
-	int retcode = be32_to_cpu(p->retcode);
-
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
-
-	if (test_bit(CONN_WD_ST_CHG_REQ, &connection->flags)) {
-		D_ASSERT(device, connection->agreed_pro_version < 100);
-		return got_conn_RqSReply(connection, pi);
-	}
+static int got_twopc_reply(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct p_twopc_reply *p = pi->data;
+
+	write_lock_irq(&resource->state_rwlock);
+	if (resource->twopc_reply.initiator_node_id == be32_to_cpu(p->initiator_node_id) &&
+	    resource->twopc_reply.tid == be32_to_cpu(p->tid)) {
+		dynamic_drbd_dbg(connection, "Got a %s reply for state change %u\n",
+			   drbd_packet_name(pi->cmd),
+			   resource->twopc_reply.tid);
+
+		if (pi->cmd == P_TWOPC_YES) {
+			struct drbd_peer_device *peer_device;
+			u64 reachable_nodes;
+			u64 max_size;
+
+			reachable_nodes = be64_to_cpu(p->reachable_nodes);
+
+			switch (resource->twopc.type) {
+			case TWOPC_STATE_CHANGE:
+				if (resource->res_opts.node_id ==
+				    resource->twopc_reply.initiator_node_id &&
+				    connection->peer_node_id ==
+				    resource->twopc_reply.target_node_id) {
+					resource->twopc_reply.target_reachable_nodes |=
+						reachable_nodes;
+				} else {
+					resource->twopc_reply.reachable_nodes |=
+						reachable_nodes;
+				}
+				resource->twopc_reply.primary_nodes |=
+					be64_to_cpu(p->primary_nodes);
+				resource->twopc_reply.weak_nodes |=
+					be64_to_cpu(p->weak_nodes);
+				break;
+			case TWOPC_RESIZE:
+				resource->twopc_reply.reachable_nodes |= reachable_nodes;
+				resource->twopc_reply.diskful_primary_nodes |=
+					be64_to_cpu(p->diskful_primary_nodes);
+				max_size = be64_to_cpu(p->max_possible_size);
+				resource->twopc_reply.max_possible_size =
+					min_t(sector_t, resource->twopc_reply.max_possible_size,
+					      max_size);
+				peer_device = conn_peer_device(connection, resource->twopc_reply.vnr);
+				if (peer_device)
+					peer_device->max_size = max_size;
+				break;
+			}
+		}
 
-	if (retcode >= SS_SUCCESS) {
-		set_bit(CL_ST_CHG_SUCCESS, &device->flags);
+		if (pi->cmd == P_TWOPC_YES)
+			set_bit(TWOPC_YES, &connection->flags);
+		else if (pi->cmd == P_TWOPC_NO)
+			set_bit(TWOPC_NO, &connection->flags);
+		else if (pi->cmd == P_TWOPC_RETRY)
+			set_bit(TWOPC_RETRY, &connection->flags);
+		drbd_maybe_cluster_wide_reply(resource);
 	} else {
-		set_bit(CL_ST_CHG_FAIL, &device->flags);
-		drbd_err(device, "Requested state change failed by peer: %s (%d)\n",
-			drbd_set_st_err_str(retcode), retcode);
+		dynamic_drbd_dbg(connection, "Ignoring %s reply for state change %u\n",
+			   drbd_packet_name(pi->cmd),
+			   be32_to_cpu(p->tid));
 	}
-	wake_up(&device->state_wait);
+	write_unlock_irq(&resource->state_rwlock);
 
 	return 0;
 }
 
-static int got_Ping(struct drbd_connection *connection, struct packet_info *pi)
+void twopc_connection_down(struct drbd_connection *connection)
 {
-	return drbd_send_ping_ack(connection);
+	struct drbd_resource *resource = connection->resource;
+
+	if (resource->twopc_reply.initiator_node_id != -1 &&
+	    test_bit(TWOPC_PREPARED, &connection->flags)) {
+		set_bit(TWOPC_RETRY, &connection->flags);
+		drbd_maybe_cluster_wide_reply(resource);
+	}
+}
 
+static int got_Ping(struct drbd_connection *connection, struct packet_info *pi)
+{
+	queue_work(ping_ack_sender, &connection->send_ping_ack_work);
+	return 0;
 }
 
 static int got_PingAck(struct drbd_connection *connection, struct packet_info *pi)
 {
-	/* restore idle timeout */
-	connection->meta.socket->sk->sk_rcvtimeo = connection->net_conf->ping_int*HZ;
-	if (!test_and_set_bit(GOT_PING_ACK, &connection->flags))
-		wake_up(&connection->ping_wait);
+	clear_bit(PING_TIMEOUT_ACTIVE, &connection->flags);
+	set_rcvtimeo(connection, REGULAR_TIMEOUT);
+
+	if (test_bit(PING_PENDING, &connection->flags)) {
+		clear_bit(PING_PENDING, &connection->flags);
+		wake_up_all(&connection->resource->state_wait);
+	}
 
 	return 0;
 }
@@ -5441,6 +10528,7 @@ static int got_IsInSync(struct drbd_connection *connection, struct packet_info *
 {
 	struct drbd_peer_device *peer_device;
 	struct drbd_device *device;
+	struct drbd_peer_request *peer_req;
 	struct p_block_ack *p = pi->data;
 	sector_t sector = be64_to_cpu(p->sector);
 	int blksize = be32_to_cpu(p->blksize);
@@ -5450,69 +10538,74 @@ static int got_IsInSync(struct drbd_connection *connection, struct packet_info *
 		return -EIO;
 	device = peer_device->device;
 
-	D_ASSERT(device, peer_device->connection->agreed_pro_version >= 89);
+	D_ASSERT(device, connection->agreed_pro_version >= 89);
 
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
+	/* Do not rely on the block_id from older peers. */
+	if (connection->agreed_pro_version < 122)
+		p->block_id = ID_SYNCER;
+
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_RESYNC_WRITE),
+			sector, blksize, p->block_id);
+	if (!peer_req)
+		return -EIO;
+
+	dec_rs_pending(peer_device);
+
+	set_bit(INTERVAL_RECEIVED, &peer_req->i.flags);
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_del(&peer_req->w.list);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
 	if (get_ldev(device)) {
-		drbd_rs_complete_io(device, sector);
 		drbd_set_in_sync(peer_device, sector, blksize);
 		/* rs_same_csums is supposed to count in units of BM_BLOCK_SIZE */
-		device->rs_same_csum += (blksize >> BM_BLOCK_SHIFT);
+		peer_device->rs_same_csum += (blksize >> device->ldev->md.bm_block_shift);
 		put_ldev(device);
 	}
-	dec_rs_pending(peer_device);
-	atomic_add(blksize >> 9, &device->rs_sect_in);
+	rs_sectors_came_in(peer_device, blksize);
 
+	drbd_remove_peer_req_interval(peer_req);
+	drbd_resync_request_complete(peer_req);
 	return 0;
 }
 
 static int
 validate_req_change_req_state(struct drbd_peer_device *peer_device, u64 id, sector_t sector,
-			      struct rb_root *root, const char *func,
+			      enum drbd_interval_type type, const char *func,
 			      enum drbd_req_event what, bool missing_ok)
 {
 	struct drbd_device *device = peer_device->device;
 	struct drbd_request *req;
-	struct bio_and_error m;
 
-	spin_lock_irq(&device->resource->req_lock);
-	req = find_request(device, root, id, sector, missing_ok, func);
-	if (unlikely(!req)) {
-		spin_unlock_irq(&device->resource->req_lock);
+	spin_lock_irq(&device->interval_lock);
+	req = find_request(device, type, id, sector, missing_ok, func);
+	spin_unlock_irq(&device->interval_lock);
+	if (unlikely(!req))
 		return -EIO;
-	}
-	__req_mod(req, what, peer_device, &m);
-	spin_unlock_irq(&device->resource->req_lock);
+	req_mod(req, what, peer_device);
 
-	if (m.bio)
-		complete_master_bio(device, &m);
 	return 0;
 }
 
 static int got_BlockAck(struct drbd_connection *connection, struct packet_info *pi)
 {
 	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
 	struct p_block_ack *p = pi->data;
 	sector_t sector = be64_to_cpu(p->sector);
-	int blksize = be32_to_cpu(p->blksize);
 	enum drbd_req_event what;
 
 	peer_device = conn_peer_device(connection, pi->vnr);
 	if (!peer_device)
 		return -EIO;
-	device = peer_device->device;
 
 	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
 
-	if (p->block_id == ID_SYNCER) {
-		drbd_set_in_sync(peer_device, sector, blksize);
-		dec_rs_pending(peer_device);
-		return 0;
-	}
 	switch (pi->cmd) {
-	case P_RS_WRITE_ACK:
+	case P_RS_WRITE_ACK: /* agreed_pro_version < 122 */
+	case P_WRITE_ACK_IN_SYNC:
 		what = WRITE_ACKED_BY_PEER_AND_SIS;
 		break;
 	case P_WRITE_ACK:
@@ -5521,209 +10614,597 @@ static int got_BlockAck(struct drbd_connection *connection, struct packet_info *
 	case P_RECV_ACK:
 		what = RECV_ACKED_BY_PEER;
 		break;
-	case P_SUPERSEDED:
-		what = CONFLICT_RESOLVED;
-		break;
-	case P_RETRY_WRITE:
-		what = POSTPONE_WRITE;
-		break;
 	default:
 		BUG();
 	}
 
-	return validate_req_change_req_state(peer_device, p->block_id, sector,
-					     &device->write_requests, __func__,
-					     what, false);
+	return validate_req_change_req_state(peer_device, p->block_id, sector,
+					     INTERVAL_LOCAL_WRITE, __func__,
+					     what, false);
+}
+
+/* Process acks for resync writes. */
+static int got_RSWriteAck(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct p_block_ack *p = pi->data;
+	bool is_neg_ack = pi->cmd == P_NEG_ACK || pi->cmd == P_RS_NEG_ACK;
+	sector_t sector = be64_to_cpu(p->sector);
+	int size = be32_to_cpu(p->blksize);
+	struct drbd_peer_request *peer_req;
+	struct drbd_peer_request *peer_req_tmp;
+	/* With agreed_pro_version < 122, one ack may correspond to multiple peer requests. */
+	LIST_HEAD(peer_reqs);
+
+	/* P_RS_WRITE_ACK used to be used instead of P_WRITE_ACK_IN_SYNC. */
+	if (connection->agreed_pro_version < 122 && p->block_id != ID_SYNCER)
+		return got_BlockAck(connection, pi);
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+
+	if (is_neg_ack && peer_device->disk_state[NOW] == D_UP_TO_DATE)
+		set_bit(GOT_NEG_ACK, &peer_device->flags);
+
+	find_resync_requests(peer_device, &peer_reqs, sector, size, p->block_id);
+
+	if (list_empty(&peer_reqs)) {
+		drbd_err(peer_device, "Unexpected resync write ack at %llus+%u\n",
+				(unsigned long long) sector, size);
+		return -EIO;
+	}
+
+	if (p->block_id != ID_SYNCER && peer_reqs.next->next != &peer_reqs) {
+		drbd_err(peer_device, "Resync write ack at %llus+%u matches multiple requests\n",
+				(unsigned long long) sector, size);
+		return -EIO;
+	}
+
+	if (is_neg_ack)
+		drbd_rs_failed_io(peer_device, sector, size);
+	else
+		drbd_set_in_sync(peer_device, sector, size);
+
+	atomic_sub(size >> 9, &connection->rs_in_flight);
+
+	list_for_each_entry_safe(peer_req, peer_req_tmp, &peer_reqs, w.list) {
+		dec_rs_pending(peer_device);
+		drbd_resync_read_req_mod(peer_req, INTERVAL_RECEIVED);
+	}
+	return 0;
+}
+
+static int got_NegAck(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct p_block_ack *p = pi->data;
+	sector_t sector = be64_to_cpu(p->sector);
+	int size = be32_to_cpu(p->blksize);
+	int err;
+
+	/* P_NEG_ACK used to be used instead of P_RS_NEG_ACK. */
+	if (p->block_id == ID_SYNCER)
+		return got_RSWriteAck(connection, pi);
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+
+	if (peer_device->disk_state[NOW] == D_UP_TO_DATE)
+		set_bit(GOT_NEG_ACK, &peer_device->flags);
+
+	err = validate_req_change_req_state(peer_device, p->block_id, sector,
+			INTERVAL_LOCAL_WRITE, __func__, NEG_ACKED, true);
+	if (err) {
+		/* Protocol A has no P_WRITE_ACKs, but has P_NEG_ACKs.
+		   The master bio might already be completed, therefore the
+		   request is no longer in the collision hash. */
+		/* In Protocol B we might already have got a P_RECV_ACK
+		   but then get a P_NEG_ACK afterwards. */
+		drbd_set_out_of_sync(peer_device, sector, size);
+	}
+	return 0;
+}
+
+static int got_NegDReply(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct p_block_ack *p = pi->data;
+	sector_t sector = be64_to_cpu(p->sector);
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+
+	drbd_warn_ratelimit(peer_device, "Got NegDReply; Sector %llus, len %u.\n",
+			(unsigned long long)sector, be32_to_cpu(p->blksize));
+
+	return validate_req_change_req_state(peer_device, p->block_id, sector,
+					     INTERVAL_LOCAL_READ, __func__,
+					     NEG_ACKED, false);
+}
+
+void drbd_unsuccessful_resync_request(struct drbd_peer_request *peer_req, bool failed)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+
+	if (get_ldev_if_state(device, D_DETACHING)) {
+		if (failed) {
+			drbd_rs_failed_io(peer_device, peer_req->i.sector, peer_req->i.size);
+		} else {
+			if (drbd_interval_is_verify(&peer_req->i)) {
+				drbd_verify_skipped_block(peer_device, peer_req->i.sector, peer_req->i.size);
+				verify_progress(peer_device, peer_req->i.sector, peer_req->i.size);
+			} else {
+				set_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags);
+			}
+		}
+
+		rs_sectors_came_in(peer_device, peer_req->i.size);
+		mod_timer(&peer_device->resync_timer, jiffies + RS_MAKE_REQS_INTV);
+		put_ldev(device);
+	}
+
+	drbd_remove_peer_req_interval(peer_req);
+	drbd_free_peer_req(peer_req);
+}
+
+static int got_NegRSDReply(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_peer_request *peer_req;
+	sector_t sector;
+	int size;
+	u64 block_id;
+	struct p_block_ack *p = pi->data;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+
+	sector = be64_to_cpu(p->sector);
+	size = be32_to_cpu(p->blksize);
+
+	/* Prior to protocol version 122, block_id may be meaningless. */
+	block_id = peer_device->connection->agreed_pro_version >= 122 ? p->block_id : ID_SYNCER;
+
+	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_RESYNC_WRITE) |
+			INTERVAL_TYPE_MASK(INTERVAL_OV_READ_SOURCE),
+			sector, size, block_id);
+	if (!peer_req)
+		return -EIO;
+
+	dec_rs_pending(peer_device);
+
+	if (pi->cmd == P_RS_CANCEL_AHEAD)
+		set_bit(SYNC_TARGET_TO_BEHIND, &peer_device->flags);
+
+	drbd_unsuccessful_resync_request(peer_req, pi->cmd == P_NEG_RS_DREPLY);
+	return 0;
+}
+
+static int got_BarrierAck(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_barrier_ack *p = pi->data;
+
+	return tl_release(connection, 0, 0, p->barrier, be32_to_cpu(p->set_size));
+}
+
+static int got_confirm_stable(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct p_confirm_stable *p = pi->data;
+
+	return tl_release(connection, p->oldest_block_id, p->youngest_block_id, 0,
+			  be32_to_cpu(p->set_size));
+}
+
+static int got_OVResult(struct drbd_connection *connection, struct packet_info *pi)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
+	struct drbd_peer_request *peer_req;
+	sector_t sector;
+	int size;
+	u64 block_id;
+	u32 seq_num;
+	enum ov_result result;
+
+	peer_device = conn_peer_device(connection, pi->vnr);
+	if (!peer_device)
+		return -EIO;
+	device = peer_device->device;
+
+	if (pi->cmd == P_OV_RESULT) {
+		struct p_block_ack *p = pi->data;
+
+		sector = be64_to_cpu(p->sector);
+		size = be32_to_cpu(p->blksize);
+		block_id = ID_SYNCER;
+		seq_num = be32_to_cpu(p->seq_num);
+		result = drbd_block_id_to_ov_result(be64_to_cpu(p->block_id));
+	} else { /* P_OV_RESULT_ID */
+		struct p_ov_result *p = pi->data;
+
+		sector = be64_to_cpu(p->sector);
+		size = be32_to_cpu(p->blksize);
+		block_id = p->block_id;
+		seq_num = be32_to_cpu(p->seq_num);
+		result = be32_to_cpu(p->result);
+	}
+
+	update_peer_seq(peer_device, seq_num);
+
+	peer_req = find_resync_request(peer_device, INTERVAL_TYPE_MASK(INTERVAL_OV_READ_TARGET),
+			sector, size, block_id);
+	if (!peer_req)
+		return -EIO;
+
+	drbd_remove_peer_req_interval(peer_req);
+
+	/* This may be a request that we could not cancel because the peer does
+	 * not understand P_RS_CANCEL. Treat it as a skipped block. */
+	if (connection->agreed_pro_version < 110 && test_bit(INTERVAL_CONFLICT, &peer_req->i.flags))
+		result = OV_RESULT_SKIP;
+
+	drbd_free_peer_req(peer_req);
+	peer_req = NULL;
+
+	if (result == OV_RESULT_SKIP)
+		drbd_verify_skipped_block(peer_device, sector, size);
+	if (result == OV_RESULT_OUT_OF_SYNC)
+		drbd_ov_out_of_sync_found(peer_device, sector, size);
+	else
+		ov_out_of_sync_print(peer_device);
+
+	if (!get_ldev(device))
+		return 0;
+
+	dec_rs_pending(peer_device);
+
+	verify_progress(peer_device, sector, size);
+
+	put_ldev(device);
+	return 0;
+}
+
+static int got_skip(struct drbd_connection *connection, struct packet_info *pi)
+{
+	return 0;
+}
+
+static u64 node_id_to_mask(struct drbd_peer_md *peer_md, int node_id) __must_hold(local)
+{
+	int bitmap_bit = peer_md[node_id].bitmap_index;
+	return (bitmap_bit >= 0) ? NODE_MASK(bitmap_bit) : 0;
+}
+
+static u64 node_ids_to_bitmap(struct drbd_device *device, u64 node_ids) __must_hold(local)
+{
+	struct drbd_peer_md *peer_md = device->ldev->md.peers;
+	u64 bitmap_bits = 0;
+	int node_id;
+
+	for_each_set_bit(node_id, (unsigned long *)&node_ids, DRBD_NODE_ID_MAX)
+		bitmap_bits |= node_id_to_mask(peer_md, node_id);
+	return bitmap_bits;
+}
+
+static struct drbd_peer_request *drbd_send_oos_next_req(struct drbd_connection *peer_ack_connection,
+		int oos_node_id, struct drbd_peer_request *peer_req)
+{
+	lockdep_assert_held(&peer_ack_connection->send_oos_lock);
+
+	if (peer_req == NULL)
+		peer_req = list_entry(&peer_ack_connection->send_oos,
+				struct drbd_peer_request, recv_order);
+
+	list_for_each_entry_continue(peer_req, &peer_ack_connection->send_oos, recv_order) {
+		if (NODE_MASK(oos_node_id) & peer_req->send_oos_pending)
+			return peer_req;
+	}
+
+	return NULL;
 }
 
-static int got_NegAck(struct drbd_connection *connection, struct packet_info *pi)
+static void drbd_send_oos_from(struct drbd_connection *oos_connection, int peer_ack_node_id)
 {
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_block_ack *p = pi->data;
-	sector_t sector = be64_to_cpu(p->sector);
-	int size = be32_to_cpu(p->blksize);
-	int err;
+	int oos_node_id = oos_connection->peer_node_id;
+	struct drbd_resource *resource = oos_connection->resource;
+	struct drbd_connection *peer_ack_connection;
+	struct drbd_peer_request *peer_req;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	rcu_read_lock();
+	peer_ack_connection = drbd_connection_by_node_id(resource, peer_ack_node_id);
+	/* Valid to use peer_ack_connection after unlock because we have kref */
+	rcu_read_unlock();
 
-	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+	spin_lock_irq(&peer_ack_connection->send_oos_lock);
+	peer_req = drbd_send_oos_next_req(peer_ack_connection, oos_node_id, NULL);
+	spin_unlock_irq(&peer_ack_connection->send_oos_lock);
 
-	if (p->block_id == ID_SYNCER) {
-		dec_rs_pending(peer_device);
-		drbd_rs_failed_io(peer_device, sector, size);
-		return 0;
+	while (peer_req) {
+		struct drbd_peer_device *peer_device =
+			conn_peer_device(oos_connection, peer_req->peer_device->device->vnr);
+		struct drbd_peer_request *free_peer_req = NULL;
+
+		/* Ignore errors and keep iterating to clear up list */
+		drbd_send_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
+
+		spin_lock_irq(&peer_ack_connection->send_oos_lock);
+		peer_req->send_oos_pending &= ~NODE_MASK(oos_node_id);
+		if (!peer_req->send_oos_pending)
+			free_peer_req = peer_req;
+
+		peer_req = drbd_send_oos_next_req(peer_ack_connection, oos_node_id, peer_req);
+		spin_unlock_irq(&peer_ack_connection->send_oos_lock);
+
+		if (free_peer_req)
+			drbd_free_peer_req(free_peer_req);
 	}
 
-	err = validate_req_change_req_state(peer_device, p->block_id, sector,
-					    &device->write_requests, __func__,
-					    NEG_ACKED, true);
-	if (err) {
-		/* Protocol A has no P_WRITE_ACKs, but has P_NEG_ACKs.
-		   The master bio might already be completed, therefore the
-		   request is no longer in the collision hash. */
-		/* In Protocol B we might already have got a P_RECV_ACK
-		   but then get a P_NEG_ACK afterwards. */
-		drbd_set_out_of_sync(peer_device, sector, size);
+	kref_put(&peer_ack_connection->kref, drbd_destroy_connection);
+}
+
+int drbd_send_out_of_sync_wf(struct drbd_work *w, int cancel)
+{
+	struct drbd_connection *oos_connection = container_of(w, struct drbd_connection,
+			send_oos_work);
+	unsigned long send_oos_from_mask = READ_ONCE(oos_connection->send_oos_from_mask);
+	int peer_ack_node_id;
+
+	for_each_set_bit(peer_ack_node_id, &send_oos_from_mask, sizeof(unsigned long)) {
+		clear_bit(peer_ack_node_id, &oos_connection->send_oos_from_mask);
+		drbd_send_oos_from(oos_connection, peer_ack_node_id);
 	}
+
 	return 0;
 }
 
-static int got_NegDReply(struct drbd_connection *connection, struct packet_info *pi)
+static bool is_sync_source(struct drbd_peer_device *peer_device)
+{
+	return is_sync_source_state(peer_device, NOW) ||
+		peer_device->repl_state[NOW] == L_WF_BITMAP_S;
+}
+
+static u64 drbd_calculate_send_oos_pending(struct drbd_device *device, u64 in_sync)
 {
 	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_block_ack *p = pi->data;
-	sector_t sector = be64_to_cpu(p->sector);
+	u64 send_oos_pending = 0;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (!(NODE_MASK(peer_device->node_id) & in_sync) &&
+				is_sync_source(peer_device))
+			send_oos_pending |= NODE_MASK(peer_device->node_id);
+	}
+	rcu_read_unlock();
 
-	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+	return send_oos_pending;
+}
 
-	drbd_err(device, "Got NegDReply; Sector %llus, len %u.\n",
-	    (unsigned long long)sector, be32_to_cpu(p->blksize));
+static void drbd_queue_send_out_of_sync(struct drbd_connection *peer_ack_connection,
+		struct list_head *send_oos_peer_req_list, u64 any_send_oos_pending)
+{
+	struct drbd_resource *resource = peer_ack_connection->resource;
+	int peer_ack_node_id = peer_ack_connection->peer_node_id;
+	struct drbd_connection *oos_connection;
 
-	return validate_req_change_req_state(peer_device, p->block_id, sector,
-					     &device->read_requests, __func__,
-					     NEG_ACKED, false);
+	if (!any_send_oos_pending)
+		return;
+
+	spin_lock_irq(&peer_ack_connection->send_oos_lock);
+	list_splice_tail(send_oos_peer_req_list, &peer_ack_connection->send_oos);
+	spin_unlock_irq(&peer_ack_connection->send_oos_lock);
+
+	/* Take state_rwlock to ensure work is queued on sender that is still running */
+	read_lock_irq(&resource->state_rwlock);
+	for_each_connection(oos_connection, resource) {
+		if (!(NODE_MASK(oos_connection->peer_node_id) & any_send_oos_pending) ||
+				oos_connection->cstate[NOW] < C_CONNECTED)
+			continue;
+
+		if (test_and_set_bit(peer_ack_node_id, &oos_connection->send_oos_from_mask))
+			continue; /* Only get kref if we set the bit here */
+
+		kref_get(&peer_ack_connection->kref);
+		drbd_queue_work_if_unqueued(&oos_connection->sender_work,
+				&oos_connection->send_oos_work);
+	}
+	read_unlock_irq(&resource->state_rwlock);
 }
 
-static int got_NegRSDReply(struct drbd_connection *connection, struct packet_info *pi)
+static int got_peer_ack(struct drbd_connection *connection, struct packet_info *pi)
 {
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	sector_t sector;
-	int size;
-	struct p_block_ack *p = pi->data;
+	struct p_peer_ack *p = pi->data;
+	u64 dagtag, in_sync;
+	struct drbd_peer_request *peer_req, *tmp;
+	struct list_head work_list;
+	u64 any_send_oos_pending = 0;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	dagtag = be64_to_cpu(p->dagtag);
+	in_sync = be64_to_cpu(p->mask);
 
-	sector = be64_to_cpu(p->sector);
-	size = be32_to_cpu(p->blksize);
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_for_each_entry(peer_req, &connection->peer_requests, recv_order) {
+		if (dagtag == peer_req->dagtag_sector)
+			goto found;
+	}
+	spin_unlock_irq(&connection->peer_reqs_lock);
 
-	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+	drbd_err(connection, "peer request with dagtag %llu not found\n", dagtag);
+	return -EIO;
 
-	dec_rs_pending(peer_device);
+found:
+	list_cut_position(&work_list, &connection->peer_requests, &peer_req->recv_order);
+	spin_unlock_irq(&connection->peer_reqs_lock);
 
-	if (get_ldev_if_state(device, D_FAILED)) {
-		drbd_rs_complete_io(device, sector);
-		switch (pi->cmd) {
-		case P_NEG_RS_DREPLY:
-			drbd_rs_failed_io(peer_device, sector, size);
-			break;
-		case P_RS_CANCEL:
-			break;
-		default:
-			BUG();
+	list_for_each_entry_safe(peer_req, tmp, &work_list, recv_order) {
+		struct drbd_peer_device *peer_device = peer_req->peer_device;
+		struct drbd_device *device = peer_device->device;
+		u64 in_sync_b, mask;
+
+		D_ASSERT(peer_device, peer_req->flags & EE_IN_ACTLOG);
+
+		if (get_ldev(device)) {
+			if ((peer_req->flags & EE_WAS_ERROR) == 0)
+				in_sync_b = node_ids_to_bitmap(device, in_sync);
+			else
+				in_sync_b = 0;
+			mask = ~node_id_to_mask(device->ldev->md.peers,
+						connection->peer_node_id);
+
+			drbd_set_sync(device, peer_req->i.sector,
+				      peer_req->i.size, ~in_sync_b, mask);
+			drbd_al_complete_io(device, &peer_req->i);
+			put_ldev(device);
 		}
-		put_ldev(device);
+
+		peer_req->send_oos_pending = drbd_calculate_send_oos_pending(device, in_sync);
+		any_send_oos_pending |= peer_req->send_oos_pending;
+		if (!peer_req->send_oos_pending)
+			drbd_free_peer_req(peer_req);
 	}
 
+	drbd_queue_send_out_of_sync(connection, &work_list, any_send_oos_pending);
 	return 0;
 }
 
-static int got_BarrierAck(struct drbd_connection *connection, struct packet_info *pi)
+void apply_unacked_peer_requests(struct drbd_connection *connection)
 {
-	struct p_barrier_ack *p = pi->data;
-	struct drbd_peer_device *peer_device;
-	int vnr;
-
-	tl_release(connection, p->barrier, be32_to_cpu(p->set_size));
+	struct drbd_peer_request *peer_req;
+	unsigned long flags;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+	spin_lock_irqsave(&connection->peer_reqs_lock, flags);
+	list_for_each_entry(peer_req, &connection->peer_requests, recv_order) {
+		struct drbd_peer_device *peer_device = peer_req->peer_device;
 		struct drbd_device *device = peer_device->device;
+		int bitmap_index = peer_device->bitmap_index;
+		u64 mask = ~(bitmap_index != -1 ? 1UL << bitmap_index : 0UL);
 
-		if (device->state.conn == C_AHEAD &&
-		    atomic_read(&device->ap_in_flight) == 0 &&
-		    !test_and_set_bit(AHEAD_TO_SYNC_SOURCE, &device->flags)) {
-			device->start_resync_timer.expires = jiffies + HZ;
-			add_timer(&device->start_resync_timer);
-		}
+		drbd_set_sync(device, peer_req->i.sector, peer_req->i.size,
+			      mask, mask);
 	}
-	rcu_read_unlock();
-
-	return 0;
+	spin_unlock_irqrestore(&connection->peer_reqs_lock, flags);
 }
 
-static int got_OVResult(struct drbd_connection *connection, struct packet_info *pi)
+static void cleanup_unacked_peer_requests(struct drbd_connection *connection)
 {
-	struct drbd_peer_device *peer_device;
-	struct drbd_device *device;
-	struct p_block_ack *p = pi->data;
-	struct drbd_device_work *dw;
-	sector_t sector;
-	int size;
+	struct drbd_peer_request *peer_req, *tmp;
+	LIST_HEAD(work_list);
+	u64 any_send_oos_pending = 0;
 
-	peer_device = conn_peer_device(connection, pi->vnr);
-	if (!peer_device)
-		return -EIO;
-	device = peer_device->device;
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_splice_init(&connection->peer_requests, &work_list);
+	spin_unlock_irq(&connection->peer_reqs_lock);
 
-	sector = be64_to_cpu(p->sector);
-	size = be32_to_cpu(p->blksize);
+	list_for_each_entry_safe(peer_req, tmp, &work_list, recv_order) {
+		struct drbd_peer_device *peer_device = peer_req->peer_device;
+		struct drbd_device *device = peer_device->device;
+		int bitmap_index = peer_device->bitmap_index;
+		u64 mask = ~(bitmap_index != -1 ? 1UL << bitmap_index : 0UL);
 
-	update_peer_seq(peer_device, be32_to_cpu(p->seq_num));
+		if (get_ldev(device)) {
+			drbd_set_sync(device, peer_req->i.sector, peer_req->i.size,
+				      mask, mask);
+			drbd_al_complete_io(device, &peer_req->i);
+			put_ldev(device);
+		}
 
-	if (be64_to_cpu(p->block_id) == ID_OUT_OF_SYNC)
-		drbd_ov_out_of_sync_found(peer_device, sector, size);
-	else
-		ov_out_of_sync_print(peer_device);
+		peer_req->send_oos_pending = drbd_calculate_send_oos_pending(device, 0);
+		any_send_oos_pending |= peer_req->send_oos_pending;
+		if (!peer_req->send_oos_pending)
+			drbd_free_peer_req(peer_req);
+	}
 
-	if (!get_ldev(device))
-		return 0;
+	drbd_queue_send_out_of_sync(connection, &work_list, any_send_oos_pending);
+}
 
-	drbd_rs_complete_io(device, sector);
-	dec_rs_pending(peer_device);
+static void cleanup_peer_ack_list(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_peer_ack *peer_ack, *tmp;
+	struct drbd_request *req;
+	int idx = connection->peer_node_id;
+	u64 node_id_mask = NODE_MASK(idx);
+
+	spin_lock_irq(&resource->peer_ack_lock);
+	list_for_each_entry_safe(peer_ack, tmp, &resource->peer_ack_list, list) {
+		if (!(peer_ack->queued_mask & node_id_mask))
+			continue;
+		peer_ack->queued_mask &= ~node_id_mask;
+		drbd_destroy_peer_ack_if_done(peer_ack);
+	}
+	req = resource->peer_ack_req;
+	if (req)
+		req->net_rq_state[idx] &= ~RQ_NET_SENT;
+	spin_unlock_irq(&resource->peer_ack_lock);
+}
 
-	--device->ov_left;
+int drbd_flush_ack_wf(struct drbd_work *w, int unused)
+{
+	struct drbd_connection *connection =
+		container_of(w, struct drbd_connection, flush_ack_work);
+	int primary_node_id;
 
-	/* let's advance progress step marks only for every other megabyte */
-	if ((device->ov_left & 0x200) == 0x200)
-		drbd_advance_rs_marks(peer_device, device->ov_left);
+	for (primary_node_id = 0; primary_node_id < DRBD_PEERS_MAX; primary_node_id++) {
+		u64 flush_sequence;
 
-	if (device->ov_left == 0) {
-		dw = kmalloc_obj(*dw, GFP_NOIO);
-		if (dw) {
-			dw->w.cb = w_ov_finished;
-			dw->device = device;
-			drbd_queue_work(&peer_device->connection->sender_work, &dw->w);
-		} else {
-			drbd_err(device, "kmalloc(dw) failed.");
-			ov_out_of_sync_print(peer_device);
-			drbd_resync_finished(peer_device);
-		}
+		spin_lock_irq(&connection->flush_ack_lock);
+		flush_sequence = connection->flush_ack_sequence[primary_node_id];
+		connection->flush_ack_sequence[primary_node_id] = 0;
+		spin_unlock_irq(&connection->flush_ack_lock);
+
+		if (flush_sequence) /* Active flushes use non-zero sequence numbers */
+			drbd_send_flush_requests_ack(connection, flush_sequence, primary_node_id);
 	}
-	put_ldev(device);
+
 	return 0;
 }
 
-static int got_skip(struct drbd_connection *connection, struct packet_info *pi)
+static int got_flush_forward(struct drbd_connection *connection, struct packet_info *pi)
 {
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_connection *initiator_connection;
+	struct p_flush_forward *p = pi->data;
+	u64 flush_sequence = be64_to_cpu(p->flush_sequence);
+	int initiator_node_id = be32_to_cpu(p->initiator_node_id);
+
+	rcu_read_lock();
+	initiator_connection = drbd_connection_by_node_id(resource, initiator_node_id);
+	if (!initiator_connection) {
+		rcu_read_unlock();
+		return 0;
+	}
+
+	spin_lock_irq(&initiator_connection->flush_ack_lock);
+	initiator_connection->flush_ack_sequence[connection->peer_node_id] = flush_sequence;
+	drbd_queue_work_if_unqueued(&initiator_connection->sender_work,
+			&initiator_connection->flush_ack_work);
+	spin_unlock_irq(&initiator_connection->flush_ack_lock);
+	rcu_read_unlock();
 	return 0;
 }
 
-struct meta_sock_cmd {
-	size_t pkt_size;
-	int (*fn)(struct drbd_connection *connection, struct packet_info *);
-};
-
-static void set_rcvtimeo(struct drbd_connection *connection, bool ping_timeout)
+static void set_rcvtimeo(struct drbd_connection *connection, enum rcv_timeou_kind kind)
 {
-	long t;
+	struct drbd_transport *transport = &connection->transport;
+	struct drbd_transport_ops *tr_ops = &transport->class->ops;
+	bool ping_timeout = kind == PING_TIMEOUT;
 	struct net_conf *nc;
+	long t;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
+	nc = rcu_dereference(transport->net_conf);
 	t = ping_timeout ? nc->ping_timeo : nc->ping_int;
 	rcu_read_unlock();
 
@@ -5731,202 +11212,265 @@ static void set_rcvtimeo(struct drbd_connection *connection, bool ping_timeout)
 	if (ping_timeout)
 		t /= 10;
 
-	connection->meta.socket->sk->sk_rcvtimeo = t;
+	tr_ops->set_rcvtimeo(transport, CONTROL_STREAM, t);
 }
 
-static void set_ping_timeout(struct drbd_connection *connection)
+void drbd_send_ping_wf(struct work_struct *ws)
 {
-	set_rcvtimeo(connection, 1);
-}
+	struct drbd_connection *connection =
+		container_of(ws, struct drbd_connection, send_ping_work);
+	int err;
 
-static void set_idle_timeout(struct drbd_connection *connection)
-{
-	set_rcvtimeo(connection, 0);
+	set_rcvtimeo(connection, PING_TIMEOUT);
+	set_bit(PING_TIMEOUT_ACTIVE, &connection->flags);
+	err = drbd_send_ping(connection);
+	if (err)
+		change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
 }
 
+struct meta_sock_cmd {
+	size_t pkt_size;
+	int (*fn)(struct drbd_connection *connection, struct packet_info *);
+};
+
 static struct meta_sock_cmd ack_receiver_tbl[] = {
-	[P_PING]	    = { 0, got_Ping },
-	[P_PING_ACK]	    = { 0, got_PingAck },
-	[P_RECV_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_WRITE_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_RS_WRITE_ACK]    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_SUPERSEDED]   = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_NEG_ACK]	    = { sizeof(struct p_block_ack), got_NegAck },
-	[P_NEG_DREPLY]	    = { sizeof(struct p_block_ack), got_NegDReply },
-	[P_NEG_RS_DREPLY]   = { sizeof(struct p_block_ack), got_NegRSDReply },
-	[P_OV_RESULT]	    = { sizeof(struct p_block_ack), got_OVResult },
-	[P_BARRIER_ACK]	    = { sizeof(struct p_barrier_ack), got_BarrierAck },
-	[P_STATE_CHG_REPLY] = { sizeof(struct p_req_state_reply), got_RqSReply },
-	[P_RS_IS_IN_SYNC]   = { sizeof(struct p_block_ack), got_IsInSync },
-	[P_DELAY_PROBE]     = { sizeof(struct p_delay_probe93), got_skip },
-	[P_RS_CANCEL]       = { sizeof(struct p_block_ack), got_NegRSDReply },
-	[P_CONN_ST_CHG_REPLY]={ sizeof(struct p_req_state_reply), got_conn_RqSReply },
-	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_PING]	      = { 0, got_Ping },
+	[P_PING_ACK]	      = { 0, got_PingAck },
+	[P_RECV_ACK]	      = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_WRITE_ACK]	      = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_WRITE_ACK_IN_SYNC] = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_NEG_ACK]	      = { sizeof(struct p_block_ack), got_NegAck },
+	[P_NEG_DREPLY]	      = { sizeof(struct p_block_ack), got_NegDReply },
+	[P_NEG_RS_DREPLY]     = { sizeof(struct p_block_ack), got_NegRSDReply },
+	[P_RS_WRITE_ACK]      = { sizeof(struct p_block_ack), got_RSWriteAck },
+	[P_RS_NEG_ACK]	      = { sizeof(struct p_block_ack), got_RSWriteAck },
+	[P_OV_RESULT]	      = { sizeof(struct p_block_ack), got_OVResult },
+	[P_OV_RESULT_ID]      = { sizeof(struct p_ov_result), got_OVResult },
+	[P_BARRIER_ACK]	      = { sizeof(struct p_barrier_ack), got_BarrierAck },
+	[P_CONFIRM_STABLE]    = { sizeof(struct p_confirm_stable), got_confirm_stable },
+	[P_STATE_CHG_REPLY]   = { sizeof(struct p_req_state_reply), got_RqSReply },
+	[P_RS_IS_IN_SYNC]     = { sizeof(struct p_block_ack), got_IsInSync },
+	[P_DELAY_PROBE]	      = { sizeof(struct p_delay_probe93), got_skip },
+	[P_RS_CANCEL]	      = { sizeof(struct p_block_ack), got_NegRSDReply },
+	[P_RS_CANCEL_AHEAD]   = { sizeof(struct p_block_ack), got_NegRSDReply },
+	[P_CONN_ST_CHG_REPLY] = { sizeof(struct p_req_state_reply), got_RqSReply },
+	[P_PEER_ACK]	      = { sizeof(struct p_peer_ack), got_peer_ack },
+	[P_PEERS_IN_SYNC]     = { sizeof(struct p_peer_block_desc), got_peers_in_sync },
+	[P_TWOPC_YES]	      = { sizeof(struct p_twopc_reply), got_twopc_reply },
+	[P_TWOPC_NO]	      = { sizeof(struct p_twopc_reply), got_twopc_reply },
+	[P_TWOPC_RETRY]	      = { sizeof(struct p_twopc_reply), got_twopc_reply },
+	[P_FLUSH_FORWARD]     = { sizeof(struct p_flush_forward), got_flush_forward },
 };
 
-int drbd_ack_receiver(struct drbd_thread *thi)
+static void fillup_buffer_from(struct drbd_mutable_buffer *to_fill, unsigned int need, struct drbd_const_buffer *pool)
 {
-	struct drbd_connection *connection = thi->connection;
-	struct meta_sock_cmd *cmd = NULL;
+	if (to_fill->avail < need) {
+		unsigned int missing = min(need - to_fill->avail, pool->avail);
+
+		memcpy(to_fill->buffer + to_fill->avail, pool->buffer, missing);
+		pool->buffer += missing;
+		pool->avail -= missing;
+		to_fill->avail += missing;
+	}
+}
+
+static int decode_meta_cmd(struct drbd_connection *connection, const u8 *pos, struct packet_info *pi)
+{
+	int header_version, payload_size;
+	struct meta_sock_cmd *cmd;
+
+	/*
+	 * A ping packet (via the control stream) can overtake the
+	 * feature packet. We might get it with a different header version
+	 * than expected since we will agree on the protocol version
+	 * by receiving the feature packet.
+	 */
+	header_version = __decode_header(pos, pi);
+	if (header_version < 0) {
+		drbd_err(connection, "Wrong magic value 0x%08x in protocol version %d [control]\n",
+			 be32_to_cpu(*(__be32 *)pos), header_version);
+		return -EINVAL;
+	}
+
+	if (pi->cmd >= ARRAY_SIZE(ack_receiver_tbl)) {
+		drbd_err(connection, "Unexpected meta packet %s (0x%04x)\n",
+			 drbd_packet_name(pi->cmd), pi->cmd);
+		return -ENOENT;
+	}
+
+	cmd = &ack_receiver_tbl[pi->cmd];
+	payload_size = cmd->pkt_size;
+	if (pi->size != payload_size) {
+		drbd_err(connection, "Wrong packet size on meta (c: %d, l: %d)\n",
+			 pi->cmd, pi->size);
+		return -EINVAL;
+	}
+
+	return payload_size;
+}
+
+static int process_previous_part(struct drbd_connection *connection, struct drbd_const_buffer *pool)
+{
+	struct drbd_mutable_buffer *buffer = &connection->reassemble_buffer;
+	int payload_size, packet_size;
+	unsigned int header_size;
 	struct packet_info pi;
-	unsigned long pre_recv_jif;
-	int rv;
-	void *buf    = connection->meta.rbuf;
-	int received = 0;
-	unsigned int header_size = drbd_header_size(connection);
-	int expect   = header_size;
-	bool ping_timeout_active = false;
+	int err;
+
+	fillup_buffer_from(buffer, sizeof(u32), pool);
+	if (buffer->avail < sizeof(u32))
+		return 0;
 
-	sched_set_fifo_low(current);
+	header_size = decode_header_size(buffer->buffer);
+	fillup_buffer_from(buffer, header_size, pool);
+	if (buffer->avail < header_size)
+		return 0;
 
-	while (get_t_state(thi) == RUNNING) {
-		drbd_thread_current_set_cpu(thi);
+	payload_size = decode_meta_cmd(connection, buffer->buffer, &pi);
+	if (payload_size < 0)
+		return payload_size;
 
-		if (test_and_clear_bit(SEND_PING, &connection->flags)) {
-			if (drbd_send_ping(connection)) {
-				drbd_err(connection, "drbd_send_ping has failed\n");
-				goto reconnect;
-			}
-			set_ping_timeout(connection);
-			ping_timeout_active = true;
-		}
-
-		pre_recv_jif = jiffies;
-		rv = drbd_recv_short(connection->meta.socket, buf, expect-received, 0);
-
-		/* Note:
-		 * -EINTR	 (on meta) we got a signal
-		 * -EAGAIN	 (on meta) rcvtimeo expired
-		 * -ECONNRESET	 other side closed the connection
-		 * -ERESTARTSYS  (on data) we got a signal
-		 * rv <  0	 other than above: unexpected error!
-		 * rv == expected: full header or command
-		 * rv <  expected: "woken" by signal during receive
-		 * rv == 0	 : "connection shut down by peer"
-		 */
-		if (likely(rv > 0)) {
-			received += rv;
-			buf	 += rv;
-		} else if (rv == 0) {
-			if (test_bit(DISCONNECT_SENT, &connection->flags)) {
-				long t;
-				rcu_read_lock();
-				t = rcu_dereference(connection->net_conf)->ping_timeo * HZ/10;
-				rcu_read_unlock();
-
-				t = wait_event_timeout(connection->ping_wait,
-						       connection->cstate < C_WF_REPORT_PARAMS,
-						       t);
-				if (t)
-					break;
-			}
-			drbd_err(connection, "meta connection shut down by peer.\n");
+	packet_size = header_size + payload_size;
+	fillup_buffer_from(buffer, packet_size, pool);
+	if (buffer->avail < packet_size)
+		return 0;
+
+	err = ack_receiver_tbl[pi.cmd].fn(connection, &pi);
+	connection->reassemble_buffer.avail = 0;
+	return err;
+}
+
+void drbd_control_data_ready(struct drbd_transport *transport, struct drbd_const_buffer *pool)
+{
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+	unsigned int header_size;
+	int err;
+
+	if (connection->cstate[NOW] < C_TEAR_DOWN)
+		return;
+
+	if (connection->reassemble_buffer.avail) {
+		err = process_previous_part(connection, pool);
+		if (err < 0)
 			goto reconnect;
-		} else if (rv == -EAGAIN) {
-			/* If the data socket received something meanwhile,
-			 * that is good enough: peer is still alive. */
-			if (time_after(connection->last_received, pre_recv_jif))
-				continue;
-			if (ping_timeout_active) {
-				drbd_err(connection, "PingAck did not arrive in time.\n");
-				goto reconnect;
-			}
-			set_bit(SEND_PING, &connection->flags);
-			continue;
-		} else if (rv == -EINTR) {
-			/* maybe drbd_thread_stop(): the while condition will notice.
-			 * maybe woken for send_ping: we'll send a ping above,
-			 * and change the rcvtimeo */
-			flush_signals(current);
-			continue;
-		} else {
-			drbd_err(connection, "sock_recvmsg returned %d\n", rv);
+	}
+
+	while (pool->avail >= sizeof(u32)) {
+		int payload_size, packet_size;
+		struct packet_info pi;
+
+		header_size = decode_header_size(pool->buffer);
+		if (header_size > pool->avail)
+			goto keep_part;
+
+		payload_size = decode_meta_cmd(connection, pool->buffer, &pi);
+		if (payload_size < 0) {
+			err = payload_size;
 			goto reconnect;
 		}
 
-		if (received == expect && cmd == NULL) {
-			if (decode_header(connection, connection->meta.rbuf, &pi))
-				goto reconnect;
-			cmd = &ack_receiver_tbl[pi.cmd];
-			if (pi.cmd >= ARRAY_SIZE(ack_receiver_tbl) || !cmd->fn) {
-				drbd_err(connection, "Unexpected meta packet %s (0x%04x)\n",
-					 cmdname(pi.cmd), pi.cmd);
-				goto disconnect;
-			}
-			expect = header_size + cmd->pkt_size;
-			if (pi.size != expect - header_size) {
-				drbd_err(connection, "Wrong packet size on meta (c: %d, l: %d)\n",
-					pi.cmd, pi.size);
-				goto reconnect;
-			}
-		}
-		if (received == expect) {
-			bool err;
+		packet_size = header_size + payload_size;
+		if (packet_size > pool->avail)
+			goto keep_part;
 
-			err = cmd->fn(connection, &pi);
-			if (err) {
-				drbd_err(connection, "%ps failed\n", cmd->fn);
-				goto reconnect;
-			}
+		err = ack_receiver_tbl[pi.cmd].fn(connection, &pi);
+		if (err)
+			goto reconnect;
 
-			connection->last_received = jiffies;
+		pool->buffer += packet_size;
+		pool->avail -= packet_size;
+	}
+	if (pool->avail > 0) {
+keep_part:
+		memcpy(connection->reassemble_buffer.buffer, pool->buffer, pool->avail);
+		connection->reassemble_buffer.avail = pool->avail;
+		pool->avail = 0;
+	}
+	return;
 
-			if (cmd == &ack_receiver_tbl[P_PING_ACK]) {
-				set_idle_timeout(connection);
-				ping_timeout_active = false;
-			}
+reconnect:
+	change_cstate(connection, err == -EPROTO ? C_PROTOCOL_ERROR : C_NETWORK_FAILURE, CS_HARD);
+}
+
+void drbd_control_event(struct drbd_transport *transport, enum drbd_tr_event event)
+{
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
 
-			buf	 = connection->meta.rbuf;
-			received = 0;
-			expect	 = header_size;
-			cmd	 = NULL;
+	if (event == TIMEOUT) {
+		if (!test_bit(PING_TIMEOUT_ACTIVE, &connection->flags)) {
+			schedule_work(&connection->send_ping_work);
+			return;
+		} else {
+			if (connection->cstate[NOW] == C_CONNECTED)
+				drbd_warn(connection, "PingAck did not arrive in time.\n");
 		}
+	} else /* event == CLOSED_BY_PEER */ {
+		if (connection->cstate[NOW] == C_CONNECTED && disconnect_expected(connection))
+			return;
+		drbd_warn(connection, "meta connection shut down by peer.\n");
 	}
 
-	if (0) {
-reconnect:
-		conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
-		conn_md_sync(connection);
-	}
-	if (0) {
-disconnect:
-		conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
-	}
+	change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+}
 
-	drbd_info(connection, "ack_receiver terminated\n");
+static bool disconnect_expected(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	bool expect_disconnect;
 
-	return 0;
+	/* We are reacting to a not-committed state change! The disconnect might
+	   get aborted. This is not a problem worth much more complex code.
+	   In the unlikely case, it happens that a two-phase-commit of a graceful
+	   disconnect gets aborted and the control connection breaks in exactly
+	   this time window, we will notice it as soon as sending something on the
+	   control stream. */
+	read_lock_irq(&resource->state_rwlock);
+	expect_disconnect = resource->remote_state_change &&
+		drbd_twopc_between_peer_and_me(connection) &&
+		resource->twopc_reply.is_disconnect;
+	read_unlock_irq(&resource->state_rwlock);
+	return expect_disconnect;
 }
 
 void drbd_send_acks_wf(struct work_struct *ws)
 {
-	struct drbd_peer_device *peer_device =
-		container_of(ws, struct drbd_peer_device, send_acks_work);
-	struct drbd_connection *connection = peer_device->connection;
-	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection =
+		container_of(ws, struct drbd_connection, send_acks_work);
+	struct drbd_transport *transport = &connection->transport;
 	struct net_conf *nc;
 	int tcp_cork, err;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
+	nc = rcu_dereference(transport->net_conf);
 	tcp_cork = nc->tcp_cork;
 	rcu_read_unlock();
 
+	/* TODO: conditionally cork; it may hurt latency if we cork without
+	   much to send */
 	if (tcp_cork)
-		tcp_sock_set_cork(connection->meta.socket->sk, true);
+		drbd_cork(connection, CONTROL_STREAM);
+	err = drbd_finish_peer_reqs(connection);
 
-	err = drbd_finish_peer_reqs(device);
-	kref_put(&device->kref, drbd_destroy_device);
-	/* get is in drbd_endio_write_sec_final(). That is necessary to keep the
-	   struct work_struct send_acks_work alive, which is in the peer_device object */
+	/* but unconditionally uncork unless disabled */
+	if (tcp_cork)
+		drbd_uncork(connection, CONTROL_STREAM);
 
-	if (err) {
-		conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
-		return;
-	}
+	if (err)
+		change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+}
 
-	if (tcp_cork)
-		tcp_sock_set_cork(connection->meta.socket->sk, false);
+void drbd_send_peer_ack_wf(struct work_struct *ws)
+{
+	struct drbd_connection *connection =
+		container_of(ws, struct drbd_connection, peer_ack_work);
 
-	return;
+	if (process_peer_ack_list(connection))
+		change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
 }
+
+EXPORT_SYMBOL(drbd_alloc_pages); /* for transports */
+EXPORT_SYMBOL(drbd_free_pages);
+EXPORT_SYMBOL(drbd_control_data_ready);
+EXPORT_SYMBOL(drbd_control_event);
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 70f75ef079457..e88b5da15c1e1 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -18,7 +18,6 @@
 #include "drbd_int.h"
 #include "drbd_req.h"
 
-
 static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size);
 
 static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src)
@@ -28,160 +27,553 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio
 	req = mempool_alloc(&drbd_request_mempool, GFP_NOIO);
 	if (!req)
 		return NULL;
+
 	memset(req, 0, sizeof(*req));
 
-	req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
-		      | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0)
-		      | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0);
+	kref_get(&device->kref);
+
 	req->device = device;
 	req->master_bio = bio_src;
 	req->epoch = 0;
 
 	drbd_clear_interval(&req->i);
-	req->i.sector     = bio_src->bi_iter.bi_sector;
-	req->i.size      = bio_src->bi_iter.bi_size;
-	req->i.local = true;
-	req->i.waiting = false;
+	req->i.sector = bio_src->bi_iter.bi_sector;
+	req->i.size = bio_src->bi_iter.bi_size;
+	req->i.type = bio_data_dir(bio_src) == WRITE ? INTERVAL_LOCAL_WRITE : INTERVAL_LOCAL_READ;
 
 	INIT_LIST_HEAD(&req->tl_requests);
-	INIT_LIST_HEAD(&req->w.list);
+	INIT_LIST_HEAD(&req->list);
 	INIT_LIST_HEAD(&req->req_pending_master_completion);
 	INIT_LIST_HEAD(&req->req_pending_local);
 
 	/* one reference to be put by __drbd_make_request */
 	atomic_set(&req->completion_ref, 1);
-	/* one kref as long as completion_ref > 0 */
+	/* one reference as long as completion_ref > 0 */
+	refcount_set(&req->done_ref, 1);
+	/* one reference as long as done_ref > 0 */
+	refcount_set(&req->oos_send_ref, 1);
+	/* one kref as long as oos_send_ref > 0 */
 	kref_init(&req->kref);
+	spin_lock_init(&req->rq_lock);
+
+	req->local_rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
+			      | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0)
+			      | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0);
+
 	return req;
 }
 
+void drbd_reclaim_req(struct rcu_head *rp)
+{
+	struct drbd_request *req = container_of(rp, struct drbd_request, rcu);
+
+	kref_put(&req->device->kref, drbd_destroy_device);
+
+	mempool_free(req, &drbd_request_mempool);
+}
+
+static u64 peer_ack_mask(struct drbd_request *req)
+{
+	struct drbd_resource *resource = req->device->resource;
+	struct drbd_connection *connection;
+	u64 mask = 0;
+
+	spin_lock_irq(&req->rq_lock);
+	if (req->local_rq_state & RQ_LOCAL_OK)
+		mask |= NODE_MASK(resource->res_opts.node_id);
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		int node_id = connection->peer_node_id;
+
+		if (req->net_rq_state[node_id] & RQ_NET_OK)
+			mask |= NODE_MASK(node_id);
+	}
+	rcu_read_unlock();
+	spin_unlock_irq(&req->rq_lock);
+
+	return mask;
+}
+
+static void queue_peer_ack_send(struct drbd_resource *resource,
+		struct drbd_request *req, struct drbd_peer_ack *peer_ack)
+{
+	struct drbd_connection *connection;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		unsigned int node_id = connection->peer_node_id;
+		if (connection->agreed_pro_version < 110 ||
+				connection->cstate[NOW] != C_CONNECTED) {
+			connection->last_peer_ack_dagtag_seen = peer_ack->dagtag_sector;
+			continue;
+		}
+
+		if (req->net_rq_state[node_id] & RQ_NET_SENT)
+			peer_ack->pending_mask |= NODE_MASK(node_id);
+
+		peer_ack->queued_mask |= NODE_MASK(node_id);
+		queue_work(connection->ack_sender, &connection->peer_ack_work);
+	}
+	rcu_read_unlock();
+}
+
+void drbd_destroy_peer_ack_if_done(struct drbd_peer_ack *peer_ack)
+{
+	struct drbd_resource *resource = peer_ack->resource;
+
+	lockdep_assert_held(&resource->peer_ack_lock);
+
+	if (peer_ack->queued_mask)
+		return;
+
+	list_del(&peer_ack->list);
+	kfree(peer_ack);
+}
+
+int w_queue_peer_ack(struct drbd_work *w, int cancel)
+{
+	struct drbd_resource *resource =
+		container_of(w, struct drbd_resource, peer_ack_work);
+	LIST_HEAD(work_list);
+	struct drbd_request *req, *tmp;
+
+	spin_lock_irq(&resource->peer_ack_lock);
+	list_splice_init(&resource->peer_ack_req_list, &work_list);
+	spin_unlock_irq(&resource->peer_ack_lock);
+
+	list_for_each_entry_safe(req, tmp, &work_list, list) {
+		struct drbd_peer_ack *peer_ack =
+			kzalloc_obj(struct drbd_peer_ack);
+
+		peer_ack->resource = resource;
+		INIT_LIST_HEAD(&peer_ack->list);
+		peer_ack->mask = peer_ack_mask(req);
+		peer_ack->dagtag_sector = req->dagtag_sector;
+
+		spin_lock_irq(&resource->peer_ack_lock);
+		list_add_tail(&peer_ack->list, &resource->peer_ack_list);
+		queue_peer_ack_send(resource, req, peer_ack);
+		drbd_destroy_peer_ack_if_done(peer_ack);
+		spin_unlock_irq(&resource->peer_ack_lock);
+
+		kref_put(&req->kref, drbd_req_destroy);
+	}
+	return 0;
+}
+
+void drbd_queue_peer_ack(struct drbd_resource *resource, struct drbd_request *req)
+{
+	lockdep_assert_held(&resource->peer_ack_lock);
+
+	list_add_tail(&req->list, &resource->peer_ack_req_list);
+	drbd_queue_work_if_unqueued(&resource->work, &resource->peer_ack_work);
+}
+
+static bool peer_ack_differs(struct drbd_request *req1, struct drbd_request *req2)
+{
+	unsigned int max_node_id = req1->device->resource->max_node_id;
+	unsigned int node_id;
+
+	for (node_id = 0; node_id <= max_node_id; node_id++)
+		if ((req1->net_rq_state[node_id] & RQ_NET_OK) !=
+		    (req2->net_rq_state[node_id] & RQ_NET_OK))
+			return true;
+	return false;
+}
+
+static bool peer_ack_window_full(struct drbd_request *req)
+{
+	struct drbd_resource *resource = req->device->resource;
+	u32 peer_ack_window = resource->res_opts.peer_ack_window;
+	u64 last_dagtag = resource->last_peer_acked_dagtag + peer_ack_window;
+
+	return dagtag_newer_eq(req->dagtag_sector, last_dagtag);
+}
+
 static void drbd_remove_request_interval(struct rb_root *root,
 					 struct drbd_request *req)
 {
 	struct drbd_device *device = req->device;
-	struct drbd_interval *i = &req->i;
-
-	drbd_remove_interval(root, i);
+	unsigned long flags;
 
-	/* Wake up any processes waiting for this request to complete.  */
-	if (i->waiting)
-		wake_up(&device->misc_wait);
+	spin_lock_irqsave(&device->interval_lock, flags);
+	drbd_remove_interval(root, &req->i);
+	spin_unlock_irqrestore(&device->interval_lock, flags);
 }
 
 void drbd_req_destroy(struct kref *kref)
 {
 	struct drbd_request *req = container_of(kref, struct drbd_request, kref);
+
+	call_rcu(&req->rcu, drbd_reclaim_req);
+}
+
+static void drbd_req_done(struct drbd_request *req)
+{
 	struct drbd_device *device = req->device;
-	const unsigned s = req->rq_state;
+	struct drbd_resource *resource = device->resource;
+	struct drbd_peer_device *peer_device;
+	unsigned int s = req->local_rq_state;
+	unsigned long modified_mask = 0;
+
+	lockdep_assert_held(&resource->state_rwlock);
+	lockdep_assert_irqs_disabled();
+
+#ifdef CONFIG_DRBD_TIMING_STATS
+	if (s & RQ_WRITE && req->i.size != 0) {
+		spin_lock(&device->timing_lock); /* local irq already disabled */
+		device->reqs++;
+		ktime_aggregate(device, req, in_actlog_kt);
+		ktime_aggregate(device, req, pre_submit_kt);
+		for_each_peer_device(peer_device, device) {
+			int node_id = peer_device->node_id;
+			unsigned ns = req->net_rq_state[node_id];
+			if (!(ns & RQ_NET_MASK))
+				continue;
+			ktime_aggregate_pd(peer_device, node_id, req, pre_send_kt);
+			ktime_aggregate_pd(peer_device, node_id, req, acked_kt);
+			ktime_aggregate_pd(peer_device, node_id, req, net_done_kt);
+		}
+		spin_unlock(&device->timing_lock);
+	}
+#endif
+
+	/* paranoia */
+	for_each_peer_device(peer_device, device) {
+		unsigned ns = req->net_rq_state[peer_device->node_id];
+		if (!(ns & RQ_NET_MASK))
+			continue;
+		if (ns & RQ_NET_DONE)
+			continue;
+
+		drbd_err(device,
+			"%s: Logic BUG rq_state: (0:%x, %d:%x), completion_ref = %d\n",
+			__func__, s, peer_device->node_id, ns, atomic_read(&req->completion_ref));
+		return;
+	}
 
+	/* more paranoia */
 	if ((req->master_bio && !(s & RQ_POSTPONED)) ||
-		atomic_read(&req->completion_ref) ||
-		(s & RQ_LOCAL_PENDING) ||
-		((s & RQ_NET_MASK) && !(s & RQ_NET_DONE))) {
-		drbd_err(device, "drbd_req_destroy: Logic BUG rq_state = 0x%x, completion_ref = %d\n",
-				s, atomic_read(&req->completion_ref));
+		atomic_read(&req->completion_ref) || (s & RQ_LOCAL_PENDING)) {
+		drbd_err(device, "%s: Logic BUG master_bio:%d rq_state: %x, completion_ref = %d\n",
+				__func__, !!req->master_bio, s, atomic_read(&req->completion_ref));
 		return;
 	}
 
-	/* If called from mod_rq_state (expected normal case) or
-	 * drbd_send_and_submit (the less likely normal path), this holds the
-	 * req_lock, and req->tl_requests will typicaly be on ->transfer_log,
-	 * though it may be still empty (never added to the transfer log).
-	 *
-	 * If called from do_retry(), we do NOT hold the req_lock, but we are
-	 * still allowed to unconditionally list_del(&req->tl_requests),
-	 * because it will be on a local on-stack list only. */
-	list_del_init(&req->tl_requests);
-
 	/* finally remove the request from the conflict detection
 	 * respective block_id verification interval tree. */
-	if (!drbd_interval_empty(&req->i)) {
-		struct rb_root *root;
+	if (s & RQ_WRITE && !drbd_interval_empty(&req->i))
+		drbd_remove_request_interval(&device->requests, req);
+
+	/* There is a special case:
+	 * we may notice late that IO was suspended,
+	 * and postpone, or schedule for retry, a write,
+	 * before it even was submitted or sent.
+	 * In that case we do not want to touch the bitmap at all.
+	 */
+	if ((s & RQ_WRITE) && (s & (RQ_POSTPONED|RQ_LOCAL_MASK)) != RQ_POSTPONED &&
+			req->i.size && get_ldev_if_state(device, D_DETACHING)) {
+		struct drbd_peer_md *peer_md = device->ldev->md.peers;
+		unsigned long bits = -1, mask = -1;
+		int node_id, max_node_id = device->resource->max_node_id;
+
+		for (node_id = 0; node_id <= max_node_id; node_id++) {
+			unsigned int net_rq_state;
+
+			net_rq_state = req->net_rq_state[node_id];
+			if (net_rq_state & RQ_NET_OK) {
+				int bitmap_index = peer_md[node_id].bitmap_index;
+
+				if (bitmap_index == -1)
+					continue;
+
+				if (net_rq_state & RQ_NET_SIS)
+					clear_bit(bitmap_index, &bits);
+				else
+					clear_bit(bitmap_index, &mask);
+			}
+		}
+		if (device->bitmap)
+			modified_mask =
+				drbd_set_sync(device, req->i.sector, req->i.size, bits, mask);
+		put_ldev(device);
+	}
 
-		if (s & RQ_WRITE)
-			root = &device->write_requests;
-		else
-			root = &device->read_requests;
-		drbd_remove_request_interval(root, req);
-	} else if (s & (RQ_NET_MASK & ~RQ_NET_DONE) && req->i.size != 0)
-		drbd_err(device, "drbd_req_destroy: Logic BUG: interval empty, but: rq_state=0x%x, sect=%llu, size=%u\n",
-			s, (unsigned long long)req->i.sector, req->i.size);
-
-	/* if it was a write, we may have to set the corresponding
-	 * bit(s) out-of-sync first. If it had a local part, we need to
-	 * release the reference to the activity log. */
 	if (s & RQ_WRITE) {
-		/* Set out-of-sync unless both OK flags are set
-		 * (local only or remote failed).
-		 * Other places where we set out-of-sync:
-		 * READ with local io-error */
-
-		/* There is a special case:
-		 * we may notice late that IO was suspended,
-		 * and postpone, or schedule for retry, a write,
-		 * before it even was submitted or sent.
-		 * In that case we do not want to touch the bitmap at all.
-		 */
-		struct drbd_peer_device *peer_device = first_peer_device(device);
-		if ((s & (RQ_POSTPONED|RQ_LOCAL_MASK|RQ_NET_MASK)) != RQ_POSTPONED) {
-			if (!(s & RQ_NET_OK) || !(s & RQ_LOCAL_OK))
-				drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size);
+		for_each_peer_device(peer_device, device) {
+			if (!(req->net_rq_state[peer_device->node_id] & RQ_NET_PENDING_OOS))
+				continue;
+
+			if (s & RQ_POSTPONED) {
+				drbd_err(device, "%s: Logic BUG RQ_NET_PENDING_OOS|RQ_POSTPONED\n",
+						__func__);
+				continue;
+			}
 
-			if ((s & RQ_NET_OK) && (s & RQ_LOCAL_OK) && (s & RQ_NET_SIS))
-				drbd_set_in_sync(peer_device, req->i.sector, req->i.size);
+			/*
+			 * As an optimization, we only send out-of-sync if we
+			 * set some bit for this peer. If we are not
+			 * replicating to this peer and the same block(s) are
+			 * overwritten several times, the peer only needs to be
+			 * informed of the first change.
+			 */
+			if (peer_device->bitmap_index != -1 &&
+					test_bit(peer_device->bitmap_index, &modified_mask))
+				_req_mod(req, READY_FOR_NET, peer_device);
+			else
+				_req_mod(req, SKIP_OOS, peer_device);
+
+			wake_up(&peer_device->connection->sender_work.q_wait);
 		}
+	}
+
+	/* one might be tempted to move the drbd_al_complete_io
+	 * to the local io completion callback drbd_request_endio.
+	 * but, if this was a mirror write, we may only
+	 * drbd_al_complete_io after this is RQ_NET_DONE,
+	 * otherwise the extent could be dropped from the al
+	 * before it has actually been written on the peer.
+	 * if we crash before our peer knows about the request,
+	 * but after the extent has been dropped from the al,
+	 * we would forget to resync the corresponding extent.
+	 */
+	if (s & RQ_IN_ACT_LOG) {
+		if (get_ldev_if_state(device, D_DETACHING)) {
+			if (drbd_al_complete_io(device, &req->i))
+				set_bit(INTERVAL_AL_EXTENT_LAST, &req->i.flags);
+			put_ldev(device);
+		} else if (drbd_device_ratelimit(device, BACKEND)) {
+			drbd_warn(device, "Should have called drbd_al_complete_io(, %llu, %u), but my Disk seems to have failed :(\n",
+					(unsigned long long) req->i.sector, req->i.size);
 
-		/* one might be tempted to move the drbd_al_complete_io
-		 * to the local io completion callback drbd_request_endio.
-		 * but, if this was a mirror write, we may only
-		 * drbd_al_complete_io after this is RQ_NET_DONE,
-		 * otherwise the extent could be dropped from the al
-		 * before it has actually been written on the peer.
-		 * if we crash before our peer knows about the request,
-		 * but after the extent has been dropped from the al,
-		 * we would forget to resync the corresponding extent.
-		 */
-		if (s & RQ_IN_ACT_LOG) {
-			if (get_ldev_if_state(device, D_FAILED)) {
-				drbd_al_complete_io(device, &req->i);
-				put_ldev(device);
-			} else if (drbd_ratelimit()) {
-				drbd_warn(device, "Should have called drbd_al_complete_io(, %llu, %u), "
-					 "but my Disk seems to have failed :(\n",
-					 (unsigned long long) req->i.sector, req->i.size);
-			}
 		}
 	}
+}
 
-	mempool_free(req, &drbd_request_mempool);
+static void drbd_req_oos_sent(struct drbd_request *req)
+{
+	struct drbd_device *device = req->device;
+	struct drbd_resource *resource = device->resource;
+	unsigned int s = req->local_rq_state;
+
+	lockdep_assert_held(&resource->state_rwlock);
+	lockdep_assert_irqs_disabled();
+
+	if (s & RQ_WRITE && req->i.size) {
+		struct drbd_resource *resource = device->resource;
+		struct drbd_request *peer_ack_req;
+
+		spin_lock(&resource->peer_ack_lock); /* local irq already disabled */
+		peer_ack_req = resource->peer_ack_req;
+		if (peer_ack_req) {
+			bool al_extent_last = test_bit(INTERVAL_AL_EXTENT_LAST, &req->i.flags);
+
+			if (peer_ack_differs(req, peer_ack_req) ||
+			    (al_extent_last && atomic_read(&device->ap_actlog_cnt)) ||
+			    peer_ack_window_full(req)) {
+				drbd_queue_peer_ack(resource, peer_ack_req);
+				peer_ack_req = NULL;
+			} else
+				kref_put(&peer_ack_req->kref, drbd_req_destroy);
+		}
+		resource->peer_ack_req = req;
+
+		if (!peer_ack_req)
+			resource->last_peer_acked_dagtag = req->dagtag_sector;
+		spin_unlock(&resource->peer_ack_lock);
+
+		mod_timer(&resource->peer_ack_timer,
+			  jiffies + resource->res_opts.peer_ack_delay * HZ / 1000);
+	} else
+		kref_put(&req->kref, drbd_req_destroy);
 }
 
-static void wake_all_senders(struct drbd_connection *connection)
+static void wake_all_senders(struct drbd_resource *resource)
 {
-	wake_up(&connection->sender_work.q_wait);
+	struct drbd_connection *connection;
+	/* We need make sure any update is visible before we wake up the
+	 * threads that may check the values in their wait_event() condition.
+	 * Do we need smp_mb here? Or rather switch to atomic_t? */
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource)
+		wake_up(&connection->sender_work.q_wait);
+	rcu_read_unlock();
 }
 
-/* must hold resource->req_lock */
-void start_new_tl_epoch(struct drbd_connection *connection)
+bool start_new_tl_epoch(struct drbd_resource *resource)
 {
+	unsigned long flags;
+	bool new_epoch_started;
+
+	spin_lock_irqsave(&resource->current_tle_lock, flags);
 	/* no point closing an epoch, if it is empty, anyways. */
-	if (connection->current_tle_writes == 0)
-		return;
+	if (resource->current_tle_writes == 0) {
+		new_epoch_started = false;
+	} else {
+		resource->current_tle_writes = 0;
+		atomic_inc(&resource->current_tle_nr);
+		wake_all_senders(resource);
+		new_epoch_started = true;
+	}
+	spin_unlock_irqrestore(&resource->current_tle_lock, flags);
 
-	connection->current_tle_writes = 0;
-	atomic_inc(&connection->current_tle_nr);
-	wake_all_senders(connection);
+	return new_epoch_started;
 }
 
 void complete_master_bio(struct drbd_device *device,
 		struct bio_and_error *m)
 {
+	int rw = bio_data_dir(m->bio);
 	if (unlikely(m->error))
 		m->bio->bi_status = errno_to_blk_status(m->error);
 	bio_endio(m->bio);
-	dec_ap_bio(device);
+	dec_ap_bio(device, rw);
+}
+
+static void queue_conflicting_resync_write(
+		struct conflict_worker *submit_conflict, struct drbd_interval *i)
+{
+	struct drbd_peer_request *peer_req = container_of(i, struct drbd_peer_request, i);
+
+	list_add_tail(&peer_req->w.list, &submit_conflict->resync_writes);
+}
+
+static void queue_conflicting_resync_read(
+		struct conflict_worker *submit_conflict, struct drbd_interval *i)
+{
+	struct drbd_peer_request *peer_req = container_of(i, struct drbd_peer_request, i);
+
+	list_add_tail(&peer_req->w.list, &submit_conflict->resync_reads);
+}
+
+static void queue_conflicting_write(
+		struct conflict_worker *submit_conflict, struct drbd_interval *i)
+{
+	struct drbd_request *req = container_of(i, struct drbd_request, i);
+
+	list_add_tail(&req->list, &submit_conflict->writes);
+}
+
+static void queue_conflicting_peer_write(
+		struct conflict_worker *submit_conflict, struct drbd_interval *i)
+{
+	struct drbd_peer_request *peer_req = container_of(i, struct drbd_peer_request, i);
+
+	list_add_tail(&peer_req->w.list, &submit_conflict->peer_writes);
+}
+
+/* Queue any conflicting requests in this interval to be submitted. */
+void drbd_release_conflicts(struct drbd_device *device, struct drbd_interval *release_interval)
+{
+	struct conflict_worker *submit_conflict = &device->submit_conflict;
+	struct drbd_interval *i;
+	bool any_queued = false;
+
+	lockdep_assert_held(&device->interval_lock);
+
+	drbd_for_each_overlap(i, &device->requests, release_interval->sector, release_interval->size) {
+		if (test_bit(INTERVAL_SUBMITTED, &i->flags))
+			continue;
+
+		/* If we are waiting for a reply from the peer, then there is
+		 * no need to process the conflict. */
+		if (test_bit(INTERVAL_SENT, &i->flags) && !test_bit(INTERVAL_RECEIVED, &i->flags))
+			continue;
+
+		dynamic_drbd_dbg(device,
+				"%s %s request at %llus+%u after conflict with %llus+%u\n",
+				test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags) ? "Already queued" : "Queue",
+				drbd_interval_type_str(i),
+				(unsigned long long) i->sector, i->size,
+				(unsigned long long) release_interval->sector, release_interval->size);
+
+		if (test_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags))
+			continue;
+
+		/* Verify requests never wait for conflicting intervals. If
+		 * there are no conflicts, they are marked direcly as
+		 * submitted. Hence we should not see any here. */
+		if (unlikely(drbd_interval_is_verify(i))) {
+			if (drbd_ratelimit())
+				drbd_err(device, "Found verify request that was not yet submitted\n");
+			continue;
+		}
+
+		set_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &i->flags);
+
+		spin_lock(&submit_conflict->lock);
+		/* Queue the request regardless of whether other conflicts
+		 * remain. The conflict submitter will only actually submit the
+		 * request if there are no conflicts. */
+		switch (i->type) {
+		case INTERVAL_LOCAL_WRITE:
+			queue_conflicting_write(submit_conflict, i);
+			break;
+		case INTERVAL_PEER_WRITE:
+			queue_conflicting_peer_write(submit_conflict, i);
+			break;
+		case INTERVAL_RESYNC_WRITE:
+			queue_conflicting_resync_write(submit_conflict, i);
+			break;
+		case INTERVAL_RESYNC_READ:
+			queue_conflicting_resync_read(submit_conflict, i);
+			break;
+		default:
+			BUG();
+		}
+		spin_unlock(&submit_conflict->lock);
+
+		any_queued = true;
+	}
+
+	if (any_queued)
+		queue_work(submit_conflict->wq, &submit_conflict->worker);
 }
 
+void drbd_put_ref_tl_walk(struct drbd_request *req, int done_put, int oos_send_put)
+{
+	struct drbd_resource *resource = req->device->resource;
+
+	lockdep_assert_held(&resource->state_rwlock);
+
+	while (req) {
+		struct drbd_request *next_write;
+		bool done = false;
+		bool oos_sent = false;
+
+		if (done_put && refcount_sub_and_test(done_put, &req->done_ref)) {
+			done = true;
+			drbd_req_done(req);
+			oos_send_put++;
+		}
+
+		if (oos_send_put && refcount_sub_and_test(oos_send_put, &req->oos_send_ref))
+			oos_sent = true;
+
+		if (!done && !oos_sent)
+			break;
+
+		spin_lock(&resource->tl_update_lock); /* local irq already disabled */
+		next_write = req->next_write;
+		if (oos_sent) {
+			list_del_rcu(&req->tl_requests);
+			if (resource->tl_previous_write == req)
+				resource->tl_previous_write = NULL;
+		} else if (done) {
+			set_bit(INTERVAL_DONE, &req->i.flags);
+		}
+		spin_unlock(&resource->tl_update_lock);
+
+		if (oos_sent)
+			/* potentially destroy */
+			drbd_req_oos_sent(req);
+
+		req = next_write;
+		done_put = done ? 1 : 0;
+		oos_send_put = oos_sent ? 1 : 0;
+	}
+}
 
 /* Helper for __req_mod().
  * Set m->bio to the master bio, if it is fit to be completed,
@@ -192,30 +584,11 @@ void complete_master_bio(struct drbd_device *device,
 static
 void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m)
 {
-	const unsigned s = req->rq_state;
+	const unsigned s = req->local_rq_state;
 	struct drbd_device *device = req->device;
-	int error, ok;
-
-	/* we must not complete the master bio, while it is
-	 *	still being processed by _drbd_send_zc_bio (drbd_send_dblock)
-	 *	not yet acknowledged by the peer
-	 *	not yet completed by the local io subsystem
-	 * these flags may get cleared in any order by
-	 *	the worker,
-	 *	the receiver,
-	 *	the bio_endio completion callbacks.
-	 */
-	if ((s & RQ_LOCAL_PENDING && !(s & RQ_LOCAL_ABORTED)) ||
-	    (s & RQ_NET_QUEUED) || (s & RQ_NET_PENDING) ||
-	    (s & RQ_COMPLETION_SUSP)) {
-		drbd_err(device, "drbd_req_complete: Logic BUG rq_state = 0x%x\n", s);
-		return;
-	}
-
-	if (!req->master_bio) {
-		drbd_err(device, "drbd_req_complete: Logic BUG, master_bio == NULL!\n");
-		return;
-	}
+	struct drbd_peer_device *peer_device;
+	unsigned long flags;
+	int error, ok = 0;
 
 	/*
 	 * figure out whether to report success or failure.
@@ -230,69 +603,136 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m)
 	 * local completion error, if any, has been stored as ERR_PTR
 	 * in private_bio within drbd_request_endio.
 	 */
-	ok = (s & RQ_LOCAL_OK) || (s & RQ_NET_OK);
+	if (s & RQ_LOCAL_OK)
+		++ok;
 	error = PTR_ERR(req->private_bio);
 
+	for_each_peer_device(peer_device, device) {
+		unsigned ns = req->net_rq_state[peer_device->node_id];
+		/* any net ok ok local ok is good enough to complete this bio as OK */
+		if (ns & RQ_NET_OK)
+			++ok;
+		/* paranoia */
+		/* we must not complete the master bio, while it is
+		 *	still being processed by _drbd_send_zc_bio (drbd_send_dblock),
+		 *	respectively still needed for the second drbd_csum_bio() there.
+		 *	not yet acknowledged by the peer
+		 *	not yet completed by the local io subsystem
+		 * these flags may get cleared in any order by
+		 *	the worker,
+		 *	the sender,
+		 *	the receiver,
+		 *	the bio_endio completion callbacks.
+		 */
+		if (!(ns & RQ_NET_MASK))
+			continue;
+		if (ns & RQ_NET_PENDING_OOS)
+			continue;
+		if (!(ns & (RQ_NET_PENDING|RQ_NET_QUEUED)))
+			continue;
+
+		drbd_err(device,
+			"drbd_req_complete: Logic BUG rq_state: (0:%x, %d:%x), completion_ref = %d\n",
+			 s, peer_device->node_id, ns, atomic_read(&req->completion_ref));
+		return;
+	}
+
+	/* more paranoia */
+	if (atomic_read(&req->completion_ref) ||
+	    ((s & RQ_LOCAL_PENDING) && !(s & RQ_LOCAL_ABORTED))) {
+		drbd_err(device, "drbd_req_complete: Logic BUG rq_state: %x, completion_ref = %d\n",
+				s, atomic_read(&req->completion_ref));
+		return;
+	}
+
+	if (!req->master_bio) {
+		drbd_err(device, "drbd_req_complete: Logic BUG, master_bio == NULL!\n");
+		return;
+	}
+
 	/* Before we can signal completion to the upper layers,
 	 * we may need to close the current transfer log epoch.
-	 * We are within the request lock, so we can simply compare
-	 * the request epoch number with the current transfer log
-	 * epoch number.  If they match, increase the current_tle_nr,
-	 * and reset the transfer log epoch write_cnt.
+	 * We simply compare the request epoch number with the current
+	 * transfer log epoch number.
+	 * With very specific timing, this may cause unnecessary barriers
+	 * to be sent, but that is harmless.
+	 *
+	 * There is no need to close the transfer log epoch for empty flushes.
+	 * The completion of the previous requests had the required effect on
+	 * the peers already.
 	 */
-	if (op_is_write(bio_op(req->master_bio)) &&
-	    req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr))
-		start_new_tl_epoch(first_peer_device(device)->connection);
+	if (bio_data_dir(req->master_bio) == WRITE &&
+	    likely(req->i.size != 0) &&
+	    req->epoch == atomic_read(&device->resource->current_tle_nr))
+		start_new_tl_epoch(device->resource);
 
 	/* Update disk stats */
 	bio_end_io_acct(req->master_bio, req->start_jif);
 
-	/* If READ failed,
-	 * have it be pushed back to the retry work queue,
-	 * so it will re-enter __drbd_make_request(),
-	 * and be re-assigned to a suitable local or remote path,
-	 * or failed if we do not have access to good data anymore.
-	 *
-	 * Unless it was failed early by __drbd_make_request(),
-	 * because no path was available, in which case
-	 * it was not even added to the transfer_log.
-	 *
-	 * read-ahead may fail, and will not be retried.
-	 *
-	 * WRITE should have used all available paths already.
-	 */
-	if (!ok &&
-	    bio_op(req->master_bio) == REQ_OP_READ &&
-	    !(req->master_bio->bi_opf & REQ_RAHEAD) &&
-	    !list_empty(&req->tl_requests))
-		req->rq_state |= RQ_POSTPONED;
-
-	if (!(req->rq_state & RQ_POSTPONED)) {
-		m->error = ok ? 0 : (error ?: -EIO);
+	if (device->cached_err_io) {
+		ok = 0;
+		req->local_rq_state &= ~RQ_POSTPONED;
+	} else if (!ok &&
+		   bio_op(req->master_bio) == REQ_OP_READ &&
+		   !(req->master_bio->bi_opf & REQ_RAHEAD) &&
+		   !list_empty(&req->tl_requests)) {
+		/* If READ failed,
+		 * have it be pushed back to the retry work queue,
+		 * so it will re-enter __drbd_make_request(),
+		 * and be re-assigned to a suitable local or remote path,
+		 * or failed if we do not have access to good data anymore.
+		 *
+		 * Unless it was failed early by __drbd_make_request(),
+		 * because no path was available, in which case
+		 * it was not even added to the transfer_log.
+		 *
+		 * read-ahead may fail, and will not be retried.
+		 *
+		 * WRITE should have used all available paths already.
+		 */
+		req->local_rq_state |= RQ_POSTPONED;
+	}
+
+	if (!(req->local_rq_state & RQ_POSTPONED)) {
+		struct drbd_resource *resource = device->resource;
+		bool quorum =
+			resource->res_opts.on_no_quorum == ONQ_IO_ERROR ?
+			resource->cached_all_devices_have_quorum : true;
+
+		m->error = ok && quorum ? 0 : (error ?: -EIO);
 		m->bio = req->master_bio;
 		req->master_bio = NULL;
-		/* We leave it in the tree, to be able to verify later
-		 * write-acks in protocol != C during resync.
-		 * But we mark it as "complete", so it won't be counted as
-		 * conflict in a multi-primary setup. */
-		req->i.completed = true;
+
+		if (req->local_rq_state & RQ_WRITE) {
+			spin_lock_irqsave(&device->interval_lock, flags);
+			/* We leave it in the tree, to be able to verify later
+			 * write-acks in protocol != C during resync.
+			 * But we mark it as "complete", so it won't be counted as
+			 * conflict in a multi-primary setup.
+			 */
+			set_bit(INTERVAL_COMPLETED, &req->i.flags);
+			drbd_release_conflicts(device, &req->i);
+			spin_unlock_irqrestore(&device->interval_lock, flags);
+		}
 	}
 
-	if (req->i.waiting)
-		wake_up(&device->misc_wait);
+	if (!(req->local_rq_state & RQ_WRITE))
+		drbd_remove_request_interval(&device->read_requests, req);
 
 	/* Either we are about to complete to upper layers,
 	 * or we will restart this request.
 	 * In either case, the request object will be destroyed soon,
 	 * so better remove it from all lists. */
+	spin_lock_irqsave(&device->pending_completion_lock, flags);
 	list_del_init(&req->req_pending_master_completion);
+	spin_unlock_irqrestore(&device->pending_completion_lock, flags);
 }
 
-/* still holds resource->req_lock */
 static void drbd_req_put_completion_ref(struct drbd_request *req, struct bio_and_error *m, int put)
 {
-	struct drbd_device *device = req->device;
-	D_ASSERT(device, m || (req->rq_state & RQ_POSTPONED));
+	D_ASSERT(req->device, m || (req->local_rq_state & RQ_POSTPONED));
+
+	lockdep_assert_held(&req->device->resource->state_rwlock);
 
 	if (!put)
 		return;
@@ -304,229 +744,368 @@ static void drbd_req_put_completion_ref(struct drbd_request *req, struct bio_and
 
 	/* local completion may still come in later,
 	 * we need to keep the req object around. */
-	if (req->rq_state & RQ_LOCAL_ABORTED)
+	if (req->local_rq_state & RQ_LOCAL_ABORTED)
 		return;
 
-	if (req->rq_state & RQ_POSTPONED) {
+	if (req->local_rq_state & RQ_POSTPONED) {
 		/* don't destroy the req object just yet,
 		 * but queue it for retry */
 		drbd_restart_request(req);
 		return;
 	}
 
-	kref_put(&req->kref, drbd_req_destroy);
+	drbd_put_ref_tl_walk(req, 1, 0);
 }
 
-static void set_if_null_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
+void drbd_set_pending_out_of_sync(struct drbd_peer_device *peer_device)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	if (!connection)
-		return;
-	if (connection->req_next == NULL)
-		connection->req_next = req;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_resource *resource = device->resource;
+	const int node_id = peer_device->node_id;
+	struct drbd_request *req;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(req, &resource->transfer_log, tl_requests) {
+		unsigned int local_rq_state, net_rq_state;
+
+		/*
+		 * This is similar to the bitmap modification performed in
+		 * drbd_req_done(), but simplified for this special case.
+		 */
+
+		spin_lock_irq(&req->rq_lock);
+		local_rq_state = req->local_rq_state;
+		net_rq_state = req->net_rq_state[node_id];
+		spin_unlock_irq(&req->rq_lock);
+
+		if (!(local_rq_state & RQ_WRITE))
+			continue;
+
+		if ((local_rq_state & (RQ_POSTPONED|RQ_LOCAL_MASK)) == RQ_POSTPONED)
+			continue;
+
+		if (!req->i.size)
+			continue;
+
+		if (net_rq_state & RQ_NET_OK)
+			continue;
+
+		drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size);
+	}
+	rcu_read_unlock();
 }
 
-static void advance_conn_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
+static void advance_conn_req_next(struct drbd_connection *connection, struct drbd_request *req)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	struct drbd_request *iter = req;
-	if (!connection)
+	struct drbd_request *found_req = NULL;
+	/* Only the sender thread comes here. No other caller context of req_mod() ever arrives here */
+	if (connection->todo.req_next != req)
 		return;
-	if (connection->req_next != req)
-		return;
-
-	req = NULL;
-	list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) {
-		const unsigned int s = iter->rq_state;
+	rcu_read_lock();
+	list_for_each_entry_continue_rcu(req, &connection->resource->transfer_log, tl_requests) {
+		const unsigned s = req->net_rq_state[connection->peer_node_id];
 
-		if (s & RQ_NET_QUEUED) {
-			req = iter;
+		if (likely(s & RQ_NET_QUEUED)) {
+			found_req = req;
 			break;
 		}
 	}
-	connection->req_next = req;
+	rcu_read_unlock();
+	connection->todo.req_next = found_req;
 }
 
-static void set_if_null_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
+/**
+ * set_cache_ptr_if_null() - Set caching pointer to given request if not currently set.
+ * @connection: DRBD connection to operate on.
+ * @cache_ptr: Pointer to set.
+ * @req: Request to potentially set the pointer to.
+ *
+ * The caching pointer system is designed to track the oldest request in the
+ * transfer log fulfilling some condition. In particular, a combination of
+ * flags towards a given peer. This condition must guarantee that the request
+ * will not be destroyed.
+ *
+ * This system is implemented by set_cache_ptr_if_null() and
+ * advance_cache_ptr(). A request must be in the transfer log and fulfil the
+ * condition before set_cache_ptr_if_null() is called. If
+ * set_cache_ptr_if_null() is called before this request is in the transfer log
+ * or before it fulfils the condition, the pointer may be advanced past this
+ * request, or unset, which also has the effect of skipping the request.
+ *
+ * Once the condition is no longer fulfilled for a request, advance_cache_ptr()
+ * must be called. If the caching pointer currently points to this request,
+ * this will advance it to the next request fulfilling the condition.
+ *
+ * set_cache_ptr_if_null() may be called concurrently with advance_cache_ptr().
+ */
+static void set_cache_ptr_if_null(struct drbd_connection *connection,
+		struct drbd_request **cache_ptr, struct drbd_request *req)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	if (!connection)
-		return;
-	if (connection->req_ack_pending == NULL)
-		connection->req_ack_pending = req;
+	spin_lock(&connection->advance_cache_ptr_lock); /* local IRQ already disabled */
+	if (*cache_ptr == NULL) {
+		smp_wmb(); /* make list_add_tail_rcu(req, transfer_log) visible before cache_ptr */
+		WRITE_ONCE(*cache_ptr, req);
+	}
+	spin_unlock(&connection->advance_cache_ptr_lock);
 }
 
-static void advance_conn_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
+/* See set_cache_ptr_if_null(). */
+static void advance_cache_ptr(struct drbd_connection *connection,
+			      struct drbd_request __rcu **cache_ptr, struct drbd_request *req,
+			      unsigned int is_set, unsigned int is_clear)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	struct drbd_request *iter = req;
-	if (!connection)
-		return;
-	if (connection->req_ack_pending != req)
-		return;
+	struct drbd_request *old_req;
+	struct drbd_request *found_req = NULL;
 
-	req = NULL;
-	list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) {
-		const unsigned int s = iter->rq_state;
+	/*
+	 * Prevent concurrent updates of the same caching pointer. Otherwise if
+	 * this function is called concurrently for a given caching pointer,
+	 * the call for the older request may advance the pointer to the newer
+	 * request, although the newer request has concurrently been modified
+	 * such that it no longer fulfils the condition.
+	 */
+	spin_lock(&connection->advance_cache_ptr_lock); /* local IRQ already disabled */
 
-		if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING)) {
-			req = iter;
+	rcu_read_lock();
+	old_req = rcu_dereference(*cache_ptr);
+	if (old_req != req) {
+		rcu_read_unlock();
+		spin_unlock(&connection->advance_cache_ptr_lock);
+		return;
+	}
+	list_for_each_entry_continue_rcu(req, &connection->resource->transfer_log, tl_requests) {
+		const unsigned s = READ_ONCE(req->net_rq_state[connection->peer_node_id]);
+		if (!(s & RQ_NET_MASK))
+			continue;
+		if (((s & is_set) == is_set) && !(s & is_clear)) {
+			found_req = req;
 			break;
 		}
 	}
-	connection->req_ack_pending = req;
+
+	WRITE_ONCE(*cache_ptr, found_req);
+	rcu_read_unlock();
+
+	spin_unlock(&connection->advance_cache_ptr_lock);
 }
 
-static void set_if_null_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
+/* for wsame, discard, and zero-out requests, the payload (amount of data we
+ * need to send) is much smaller than the number of storage sectors affected */
+static unsigned int req_payload_sectors(struct drbd_request *req)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	if (!connection)
-		return;
-	if (connection->req_not_net_done == NULL)
-		connection->req_not_net_done = req;
+	/* actually: physical_block_size,
+	 * but lets just hardcode 4k in sectors: */
+	if (unlikely(req->local_rq_state & RQ_WSAME))
+		return 8;
+	/* really only a few bytes, but let's pretend one sector */
+	if (unlikely(req->local_rq_state & (RQ_UNMAP|RQ_ZEROES)))
+		return 1;
+	/* other have all the data as payload on the wire */
+	return req->i.size >> 9;
 }
 
-static void advance_conn_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
+static bool drbd_sender_needs_master_bio(unsigned int net_rq_state)
 {
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	struct drbd_request *iter = req;
-	if (!connection)
-		return;
-	if (connection->req_not_net_done != req)
-		return;
-
-	req = NULL;
-	list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) {
-		const unsigned int s = iter->rq_state;
-
-		if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE)) {
-			req = iter;
-			break;
-		}
-	}
-	connection->req_not_net_done = req;
+	return (net_rq_state & RQ_NET_QUEUED) && !(net_rq_state & RQ_NET_DONE);
 }
 
 /* I'd like this to be the only place that manipulates
  * req->completion_ref and req->kref. */
 static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
+		struct drbd_peer_device *peer_device,
 		int clear, int set)
 {
-	struct drbd_device *device = req->device;
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	unsigned s = req->rq_state;
+	unsigned int old_local, old_net = 0, new_net = 0;
+	unsigned int set_local = set & RQ_STATE_0_MASK;
+	unsigned int clear_local = clear & RQ_STATE_0_MASK;
 	int c_put = 0;
-
-	if (drbd_suspended(device) && !((s | clear) & RQ_COMPLETION_SUSP))
-		set |= RQ_COMPLETION_SUSP;
+	int d_put = 0;
+	int o_put = 0;
+	const int idx = peer_device ? peer_device->node_id : -1;
+	struct drbd_connection *connection = NULL;
+	bool unchanged;
+
+	set &= ~RQ_STATE_0_MASK;
+	clear &= ~RQ_STATE_0_MASK;
+
+	if (idx == -1) {
+		/* do not try to manipulate net state bits
+		 * without an associated state slot! */
+		BUG_ON(set);
+		BUG_ON(clear);
+	}
 
 	/* apply */
+	spin_lock(&req->rq_lock); /* local IRQ already disabled */
 
-	req->rq_state &= ~clear;
-	req->rq_state |= set;
+	old_local = req->local_rq_state;
+	req->local_rq_state &= ~clear_local;
+	req->local_rq_state |= set_local;
+
+	if (idx != -1) {
+		old_net = req->net_rq_state[idx];
+		new_net = (req->net_rq_state[idx] & ~clear) | set;
+		WRITE_ONCE(req->net_rq_state[idx], new_net);
+		connection = peer_device->connection;
+	}
 
 	/* no change? */
-	if (req->rq_state == s)
+	unchanged = req->local_rq_state == old_local &&
+	  (idx == -1 || req->net_rq_state[idx] == old_net);
+
+	if (unchanged) {
+		spin_unlock(&req->rq_lock);
 		return;
+	}
 
 	/* intent: get references */
 
-	kref_get(&req->kref);
-
-	if (!(s & RQ_LOCAL_PENDING) && (set & RQ_LOCAL_PENDING))
+	if (!(old_local & RQ_LOCAL_PENDING) && (set_local & RQ_LOCAL_PENDING))
 		atomic_inc(&req->completion_ref);
 
-	if (!(s & RQ_NET_PENDING) && (set & RQ_NET_PENDING)) {
-		inc_ap_pending(device);
+	if (!(old_net & RQ_NET_PENDING) && (set & RQ_NET_PENDING)) {
+		inc_ap_pending(peer_device);
 		atomic_inc(&req->completion_ref);
 	}
 
-	if (!(s & RQ_NET_QUEUED) && (set & RQ_NET_QUEUED)) {
+	if (!(old_net & RQ_NET_QUEUED) && (set & RQ_NET_QUEUED)) {
+		/* Keep request on transfer log while queued for sender */
+		refcount_inc(&req->oos_send_ref);
+	}
+
+	if (!drbd_sender_needs_master_bio(old_net) && drbd_sender_needs_master_bio(new_net)) {
+		/*
+		 * This completion ref is necessary to avoid premature
+		 * completion in case a WRITE_ACKED_BY_PEER comes in before the
+		 * sender can do HANDED_OVER_TO_NETWORK.
+		 */
 		atomic_inc(&req->completion_ref);
-		set_if_null_req_next(peer_device, req);
 	}
 
-	if (!(s & RQ_EXP_BARR_ACK) && (set & RQ_EXP_BARR_ACK))
-		kref_get(&req->kref); /* wait for the DONE */
+	if (!(old_net & RQ_NET_READY) && (set & RQ_NET_READY) &&
+			!(req->net_rq_state[idx] & RQ_NET_DONE))
+		set_cache_ptr_if_null(connection, &connection->req_not_net_done, req);
+
+	if (!(old_net & RQ_EXP_BARR_ACK) && (set & RQ_EXP_BARR_ACK))
+		refcount_inc(&req->done_ref); /* wait for the DONE */
 
-	if (!(s & RQ_NET_SENT) && (set & RQ_NET_SENT)) {
+	if (!(old_net & RQ_NET_SENT) && (set & RQ_NET_SENT)) {
 		/* potentially already completed in the ack_receiver thread */
-		if (!(s & RQ_NET_DONE)) {
-			atomic_add(req->i.size >> 9, &device->ap_in_flight);
-			set_if_null_req_not_net_done(peer_device, req);
-		}
-		if (req->rq_state & RQ_NET_PENDING)
-			set_if_null_req_ack_pending(peer_device, req);
+		if (!(old_net & RQ_NET_DONE))
+			atomic_add(req_payload_sectors(req), &peer_device->connection->ap_in_flight);
+		if (req->net_rq_state[idx] & RQ_NET_PENDING)
+			set_cache_ptr_if_null(connection, &connection->req_ack_pending, req);
 	}
 
-	if (!(s & RQ_COMPLETION_SUSP) && (set & RQ_COMPLETION_SUSP))
+	if (!(old_local & RQ_COMPLETION_SUSP) && (set_local & RQ_COMPLETION_SUSP))
 		atomic_inc(&req->completion_ref);
 
+	spin_unlock(&req->rq_lock);
+
 	/* progress: put references */
 
-	if ((s & RQ_COMPLETION_SUSP) && (clear & RQ_COMPLETION_SUSP))
+	if ((old_local & RQ_COMPLETION_SUSP) && (clear_local & RQ_COMPLETION_SUSP))
 		++c_put;
 
-	if (!(s & RQ_LOCAL_ABORTED) && (set & RQ_LOCAL_ABORTED)) {
-		D_ASSERT(device, req->rq_state & RQ_LOCAL_PENDING);
+	if (!(old_local & RQ_LOCAL_ABORTED) && (set_local & RQ_LOCAL_ABORTED)) {
+		D_ASSERT(req->device, req->local_rq_state & RQ_LOCAL_PENDING);
 		++c_put;
 	}
 
-	if ((s & RQ_LOCAL_PENDING) && (clear & RQ_LOCAL_PENDING)) {
-		if (req->rq_state & RQ_LOCAL_ABORTED)
-			kref_put(&req->kref, drbd_req_destroy);
+	if ((old_local & RQ_LOCAL_PENDING) && (clear_local & RQ_LOCAL_PENDING)) {
+		struct drbd_device *device = req->device;
+
+		if (req->local_rq_state & RQ_LOCAL_ABORTED)
+			++d_put;
 		else
 			++c_put;
+		spin_lock(&device->pending_completion_lock); /* local irq already disabled */
 		list_del_init(&req->req_pending_local);
+		spin_unlock(&device->pending_completion_lock);
 	}
 
-	if ((s & RQ_NET_PENDING) && (clear & RQ_NET_PENDING)) {
-		dec_ap_pending(device);
+	if ((old_net & RQ_NET_PENDING) && (clear & RQ_NET_PENDING)) {
+		dec_ap_pending(peer_device);
 		++c_put;
-		req->acked_jif = jiffies;
-		advance_conn_req_ack_pending(peer_device, req);
+		ktime_get_accounting(req->acked_kt[peer_device->node_id]);
+		advance_cache_ptr(connection, &connection->req_ack_pending,
+				  req, RQ_NET_SENT | RQ_NET_PENDING, 0);
 	}
 
-	if ((s & RQ_NET_QUEUED) && (clear & RQ_NET_QUEUED)) {
-		++c_put;
-		advance_conn_req_next(peer_device, req);
+	if ((old_net & RQ_NET_QUEUED) && (clear & RQ_NET_QUEUED)) {
+		++o_put;
+		advance_conn_req_next(connection, req);
 	}
 
-	if (!(s & RQ_NET_DONE) && (set & RQ_NET_DONE)) {
-		if (s & RQ_NET_SENT)
-			atomic_sub(req->i.size >> 9, &device->ap_in_flight);
-		if (s & RQ_EXP_BARR_ACK)
-			kref_put(&req->kref, drbd_req_destroy);
-		req->net_done_jif = jiffies;
+	if (drbd_sender_needs_master_bio(old_net) && !drbd_sender_needs_master_bio(new_net))
+		++c_put;
 
-		/* in ahead/behind mode, or just in case,
-		 * before we finally destroy this request,
-		 * the caching pointers must not reference it anymore */
-		advance_conn_req_next(peer_device, req);
-		advance_conn_req_ack_pending(peer_device, req);
-		advance_conn_req_not_net_done(peer_device, req);
-	}
+	if (!(old_net & RQ_NET_DONE) && (set & RQ_NET_DONE)) {
+		if (old_net & RQ_NET_SENT)
+			atomic_sub(req_payload_sectors(req),
+					&peer_device->connection->ap_in_flight);
+		if (old_net & RQ_EXP_BARR_ACK)
+			++d_put;
+		ktime_get_accounting(req->net_done_kt[peer_device->node_id]);
 
-	/* potentially complete and destroy */
+		advance_cache_ptr(connection, &connection->req_not_net_done,
+				  req, 0, RQ_NET_DONE);
+	}
 
-	/* If we made progress, retry conflicting peer requests, if any. */
-	if (req->i.waiting)
-		wake_up(&device->misc_wait);
+	if ((old_net & RQ_NET_PENDING_OOS) && (clear & RQ_NET_PENDING_OOS)) {
+		if (peer_device->repl_state[NOW] == L_AHEAD &&
+		    atomic_read(&peer_device->connection->ap_in_flight) == 0) {
+			struct drbd_peer_device *pd;
+			int vnr;
+			/* The first peer device to notice that it is time to
+			 * go Ahead -> SyncSource tries to trigger that
+			 * transition for *all* peer devices currently in
+			 * L_AHEAD for this connection. */
+			idr_for_each_entry(&peer_device->connection->peer_devices, pd, vnr) {
+				if (pd->repl_state[NOW] != L_AHEAD)
+					continue;
+				if (test_and_set_bit(AHEAD_TO_SYNC_SOURCE, &pd->flags))
+					continue; /* already done */
+				pd->start_resync_side = L_SYNC_SOURCE;
+				mod_timer(&pd->start_resync_timer, jiffies + HZ);
+			}
+		}
+	}
 
+	/* potentially complete and destroy */
 	drbd_req_put_completion_ref(req, m, c_put);
-	kref_put(&req->kref, drbd_req_destroy);
+
+	/* req cannot have been destroyed if there are still references */
+	if (d_put || o_put)
+		/* potentially destroy */
+		drbd_put_ref_tl_walk(req, d_put, o_put);
 }
 
 static void drbd_report_io_error(struct drbd_device *device, struct drbd_request *req)
 {
-	if (!drbd_ratelimit())
+	if (!drbd_device_ratelimit(device, BACKEND))
 		return;
 
 	drbd_warn(device, "local %s IO error sector %llu+%u on %pg\n",
-			(req->rq_state & RQ_WRITE) ? "WRITE" : "READ",
-			(unsigned long long)req->i.sector,
-			req->i.size >> 9,
-			device->ldev->backing_bdev);
+		  (req->local_rq_state & RQ_WRITE) ? "WRITE" : "READ",
+		  (unsigned long long)req->i.sector,
+		  req->i.size >> 9,
+		  device->ldev->backing_bdev);
+}
+
+static int drbd_protocol_state_bits(struct drbd_connection *connection)
+{
+	struct net_conf *nc;
+	int p;
+
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	p = nc->wire_protocol;
+	rcu_read_unlock();
+
+	return p == DRBD_PROT_C ? RQ_EXP_WRITE_ACK :
+		p == DRBD_PROT_B ? RQ_EXP_RECEIVE_ACK : 0;
+
 }
 
 /* Helper for HANDED_OVER_TO_NETWORK.
@@ -535,11 +1114,12 @@ static void drbd_report_io_error(struct drbd_device *device, struct drbd_request
  * --> If so, clear PENDING and set NET_OK below.
  * If it is a protocol A write, but not RQ_PENDING anymore, neg-ack was faster
  * (and we must not set RQ_NET_OK) */
-static inline bool is_pending_write_protocol_A(struct drbd_request *req)
+static inline bool is_pending_write_protocol_A(struct drbd_request *req, int idx)
 {
-	return (req->rq_state &
-		   (RQ_WRITE|RQ_NET_PENDING|RQ_EXP_WRITE_ACK|RQ_EXP_RECEIVE_ACK))
-		== (RQ_WRITE|RQ_NET_PENDING);
+	return (req->local_rq_state & RQ_WRITE) == 0 ? 0 :
+		(req->net_rq_state[idx] &
+		   (RQ_NET_PENDING|RQ_EXP_WRITE_ACK|RQ_EXP_RECEIVE_ACK))
+		==  RQ_NET_PENDING;
 }
 
 /* obviously this could be coded as many single functions
@@ -550,95 +1130,76 @@ static inline bool is_pending_write_protocol_A(struct drbd_request *req)
  * but having it this way
  *  enforces that it is all in this one place, where it is easier to audit,
  *  it makes it obvious that whatever "event" "happens" to a request should
- *  happen "atomically" within the req_lock,
+ *  happen with the state_rwlock read lock held,
  *  and it enforces that we have to think in a very structured manner
  *  about the "events" that may happen to a request during its life time ...
  *
  *
  * peer_device == NULL means local disk
  */
-int __req_mod(struct drbd_request *req, enum drbd_req_event what,
+void __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		struct drbd_peer_device *peer_device,
 		struct bio_and_error *m)
 {
-	struct drbd_device *const device = req->device;
-	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
+	struct drbd_device *device = req->device;
 	struct net_conf *nc;
-	int p, rv = 0;
+	unsigned long flags;
+	int p;
+	int idx;
+
+	lockdep_assert_held(&device->resource->state_rwlock);
 
 	if (m)
 		m->bio = NULL;
 
+	idx = peer_device ? peer_device->node_id : -1;
+
 	switch (what) {
 	default:
 		drbd_err(device, "LOGIC BUG in %s:%u\n", __FILE__ , __LINE__);
 		break;
 
-	/* does not happen...
-	 * initialization done in drbd_req_new
-	case CREATED:
-		break;
-		*/
-
-	case TO_BE_SENT: /* via network */
-		/* reached via __drbd_make_request
-		 * and from w_read_retry_remote */
-		D_ASSERT(device, !(req->rq_state & RQ_NET_MASK));
-		rcu_read_lock();
-		nc = rcu_dereference(connection->net_conf);
-		p = nc->wire_protocol;
-		rcu_read_unlock();
-		req->rq_state |=
-			p == DRBD_PROT_C ? RQ_EXP_WRITE_ACK :
-			p == DRBD_PROT_B ? RQ_EXP_RECEIVE_ACK : 0;
-		mod_rq_state(req, m, 0, RQ_NET_PENDING);
-		break;
-
 	case TO_BE_SUBMITTED: /* locally */
 		/* reached via __drbd_make_request */
-		D_ASSERT(device, !(req->rq_state & RQ_LOCAL_MASK));
-		mod_rq_state(req, m, 0, RQ_LOCAL_PENDING);
+		D_ASSERT(device, !(req->local_rq_state & RQ_LOCAL_MASK));
+		mod_rq_state(req, m, peer_device, 0, RQ_LOCAL_PENDING);
 		break;
 
 	case COMPLETED_OK:
-		if (req->rq_state & RQ_WRITE)
+		if (req->local_rq_state & RQ_WRITE)
 			device->writ_cnt += req->i.size >> 9;
 		else
 			device->read_cnt += req->i.size >> 9;
 
-		mod_rq_state(req, m, RQ_LOCAL_PENDING,
+		mod_rq_state(req, m, peer_device, RQ_LOCAL_PENDING,
 				RQ_LOCAL_COMPLETED|RQ_LOCAL_OK);
 		break;
 
 	case ABORT_DISK_IO:
-		mod_rq_state(req, m, 0, RQ_LOCAL_ABORTED);
+		mod_rq_state(req, m, peer_device, 0, RQ_LOCAL_ABORTED);
 		break;
 
 	case WRITE_COMPLETED_WITH_ERROR:
 		drbd_report_io_error(device, req);
-		__drbd_chk_io_error(device, DRBD_WRITE_ERROR);
-		mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
+		mod_rq_state(req, m, peer_device, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
 		break;
 
 	case READ_COMPLETED_WITH_ERROR:
-		drbd_set_out_of_sync(first_peer_device(device),
-				req->i.sector, req->i.size);
+		drbd_set_all_out_of_sync(device, req->i.sector, req->i.size);
 		drbd_report_io_error(device, req);
-		__drbd_chk_io_error(device, DRBD_READ_ERROR);
 		fallthrough;
 	case READ_AHEAD_COMPLETED_WITH_ERROR:
-		/* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */
-		mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
+		mod_rq_state(req, m, peer_device, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
 		break;
 
 	case DISCARD_COMPLETED_NOTSUPP:
 	case DISCARD_COMPLETED_WITH_ERROR:
 		/* I'd rather not detach from local disk just because it
 		 * failed a REQ_OP_DISCARD. */
-		mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
+		mod_rq_state(req, m, peer_device, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
 		break;
 
-	case QUEUE_FOR_NET_READ:
+	case NEW_NET_READ:
 		/* READ, and
 		 * no local disk,
 		 * or target area marked as invalid,
@@ -650,27 +1211,19 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 * Corresponding drbd_remove_request_interval is in
 		 * drbd_req_complete() */
 		D_ASSERT(device, drbd_interval_empty(&req->i));
+		spin_lock_irqsave(&device->interval_lock, flags);
 		drbd_insert_interval(&device->read_requests, &req->i);
+		spin_unlock_irqrestore(&device->interval_lock, flags);
 
-		set_bit(UNPLUG_REMOTE, &device->flags);
-
-		D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
-		D_ASSERT(device, (req->rq_state & RQ_LOCAL_MASK) == 0);
-		mod_rq_state(req, m, 0, RQ_NET_QUEUED);
-		req->w.cb = w_send_read_req;
-		drbd_queue_work(&connection->sender_work,
-				&req->w);
+		D_ASSERT(device, !(req->net_rq_state[idx] & RQ_NET_MASK));
+		D_ASSERT(device, !(req->local_rq_state & RQ_LOCAL_MASK));
+		mod_rq_state(req, m, peer_device, 0, RQ_NET_PENDING|RQ_NET_QUEUED);
 		break;
 
-	case QUEUE_FOR_NET_WRITE:
+	case NEW_NET_WRITE:
 		/* assert something? */
 		/* from __drbd_make_request only */
 
-		/* Corresponding drbd_remove_request_interval is in
-		 * drbd_req_complete() */
-		D_ASSERT(device, drbd_interval_empty(&req->i));
-		drbd_insert_interval(&device->write_requests, &req->i);
-
 		/* NOTE
 		 * In case the req ended up on the transfer log before being
 		 * queued on the worker, it could lead to this request being
@@ -685,85 +1238,109 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 *
 		 * Add req to the (now) current epoch (barrier). */
 
-		/* otherwise we may lose an unplug, which may cause some remote
-		 * io-scheduler timeout to expire, increasing maximum latency,
-		 * hurting performance. */
-		set_bit(UNPLUG_REMOTE, &device->flags);
+		D_ASSERT(device, !(req->net_rq_state[idx] & RQ_NET_MASK));
 
 		/* queue work item to send data */
-		D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
-		mod_rq_state(req, m, 0, RQ_NET_QUEUED|RQ_EXP_BARR_ACK);
-		req->w.cb =  w_send_dblock;
-		drbd_queue_work(&connection->sender_work,
-				&req->w);
+		mod_rq_state(req, m, peer_device, 0, RQ_NET_PENDING|RQ_NET_QUEUED|RQ_EXP_BARR_ACK|
+				drbd_protocol_state_bits(peer_device->connection));
+
+		/* Close the epoch, in case it outgrew the limit.
+		 * Or if this is a "batch bio", and some of our peers is "old",
+		 * because a batch bio "storm" (like, large scale discarding
+		 * during mkfs time) would be likely to starve out the peers
+		 * activity log, if it is smaller than ours (or we don't have
+		 * any).  And a fix for the resulting potential distributed
+		 * deadlock was only implemented with P_CONFIRM_STABLE with
+		 * protocol version 114.
+		 */
+		if (device->resource->cached_min_aggreed_protocol_version < 114 &&
+		    (req->local_rq_state & (RQ_UNMAP|RQ_WSAME|RQ_ZEROES)))
+			p = 1;
+		else {
+			rcu_read_lock();
+			nc = rcu_dereference(peer_device->connection->transport.net_conf);
+			p = nc->max_epoch_size;
+			rcu_read_unlock();
+		}
+		if (device->resource->current_tle_writes >= p)
+			start_new_tl_epoch(device->resource);
+		break;
 
-		/* close the epoch, in case it outgrew the limit */
-		rcu_read_lock();
-		nc = rcu_dereference(connection->net_conf);
-		p = nc->max_epoch_size;
-		rcu_read_unlock();
-		if (connection->current_tle_writes >= p)
-			start_new_tl_epoch(connection);
+	case NEW_NET_OOS:
+		/* We will just send P_OUT_OF_SYNC to this peer. The request is
+		 * "done" from the start in the sense that everything necessary
+		 * in the data stage has been done.
+		 */
+		mod_rq_state(req, m, peer_device, 0, RQ_NET_PENDING_OOS|RQ_NET_QUEUED|RQ_NET_DONE);
+		break;
 
+	case READY_FOR_NET:
+		mod_rq_state(req, m, peer_device, 0, RQ_NET_READY);
 		break;
 
-	case QUEUE_FOR_SEND_OOS:
-		mod_rq_state(req, m, 0, RQ_NET_QUEUED);
-		req->w.cb =  w_send_out_of_sync;
-		drbd_queue_work(&connection->sender_work,
-				&req->w);
+	case SKIP_OOS:
+		mod_rq_state(req, m, peer_device, RQ_NET_PENDING_OOS, RQ_NET_READY);
 		break;
 
-	case READ_RETRY_REMOTE_CANCELED:
+	case OOS_HANDED_TO_NETWORK:
 	case SEND_CANCELED:
 	case SEND_FAILED:
-		/* real cleanup will be done from tl_clear.  just update flags
-		 * so it is no longer marked as on the worker queue */
-		mod_rq_state(req, m, RQ_NET_QUEUED, 0);
+		/* Sending P_OUT_OF_SYNC is irrelevant if the connection was
+		 * lost. Hence, when the intention was to send P_OUT_OF_SYNC,
+		 * the effect of successfully sending the packet and connection
+		 * loss are the same.
+		 *
+		 * Otherwise just update flags so it is no longer marked as on
+		 * the sender queue; real cleanup will be done from
+		 * tl_walk(,CONNECTION_LOST*).
+		 */
+		mod_rq_state(req, m, peer_device, RQ_NET_PENDING_OOS|RQ_NET_QUEUED, 0);
 		break;
 
 	case HANDED_OVER_TO_NETWORK:
 		/* assert something? */
-		if (is_pending_write_protocol_A(req))
+		if (is_pending_write_protocol_A(req, idx))
 			/* this is what is dangerous about protocol A:
 			 * pretend it was successfully written on the peer. */
-			mod_rq_state(req, m, RQ_NET_QUEUED|RQ_NET_PENDING,
-						RQ_NET_SENT|RQ_NET_OK);
+			mod_rq_state(req, m, peer_device, RQ_NET_QUEUED|RQ_NET_PENDING,
+				     RQ_NET_SENT|RQ_NET_OK);
 		else
-			mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_SENT);
+			mod_rq_state(req, m, peer_device, RQ_NET_QUEUED, RQ_NET_SENT);
 		/* It is still not yet RQ_NET_DONE until the
 		 * corresponding epoch barrier got acked as well,
 		 * so we know what to dirty on connection loss. */
 		break;
 
-	case OOS_HANDED_TO_NETWORK:
-		/* Was not set PENDING, no longer QUEUED, so is now DONE
-		 * as far as this connection is concerned. */
-		mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_DONE);
-		break;
-
-	case CONNECTION_LOST_WHILE_PENDING:
-		/* transfer log cleanup after connection loss */
-		mod_rq_state(req, m,
-				RQ_NET_OK|RQ_NET_PENDING|RQ_COMPLETION_SUSP,
-				RQ_NET_DONE);
-		break;
+	case CONNECTION_LOST:
+	case CONNECTION_LOST_WHILE_SUSPENDED:
+		/* Only apply to requests that were for this peer but not done. */
+		if (!(req->net_rq_state[idx] & RQ_NET_MASK) || req->net_rq_state[idx] & RQ_NET_DONE)
+			break;
 
-	case CONFLICT_RESOLVED:
-		/* for superseded conflicting writes of multiple primaries,
-		 * there is no need to keep anything in the tl, potential
-		 * node crashes are covered by the activity log.
+		/* For protocol A, or when not suspended, we consider the
+		 * request to be lost towards this peer.
+		 *
+		 * Protocol B&C requests are kept while suspended because
+		 * resending is allowed. If such a request is pending to this
+		 * peer, we suspend its completion until IO is resumed. This is
+		 * a conservative simplification. We could complete it while
+		 * suspended once we know it has been received by "enough"
+		 * peers. However, we do not track that.
 		 *
-		 * If this request had been marked as RQ_POSTPONED before,
-		 * it will actually not be completed, but "restarted",
-		 * resubmitted from the retry worker context. */
-		D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
-		D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
-		mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_DONE|RQ_NET_OK);
+		 * If the request is no longer pending to this peer, then we
+		 * have already received the corresponding ack. The request may
+		 * complete as far as this peer is concerned. */
+		if (what == CONNECTION_LOST ||
+				!(req->net_rq_state[idx] & (RQ_EXP_RECEIVE_ACK|RQ_EXP_WRITE_ACK)))
+			mod_rq_state(req, m, peer_device, RQ_NET_PENDING|RQ_NET_OK, RQ_NET_DONE);
+		else if (req->net_rq_state[idx] & RQ_NET_PENDING)
+			mod_rq_state(req, m, peer_device, 0, RQ_COMPLETION_SUSP);
 		break;
 
 	case WRITE_ACKED_BY_PEER_AND_SIS:
-		req->rq_state |= RQ_NET_SIS;
+		spin_lock_irqsave(&req->rq_lock, flags);
+		req->net_rq_state[idx] |= RQ_NET_SIS;
+		spin_unlock_irqrestore(&req->rq_lock, flags);
 		fallthrough;
 	case WRITE_ACKED_BY_PEER:
 		/* Normal operation protocol C: successfully written on peer.
@@ -775,155 +1352,162 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 * for volatile write-back caches on lower level devices. */
 		goto ack_common;
 	case RECV_ACKED_BY_PEER:
-		D_ASSERT(device, req->rq_state & RQ_EXP_RECEIVE_ACK);
+		D_ASSERT(device, req->net_rq_state[idx] & RQ_EXP_RECEIVE_ACK);
 		/* protocol B; pretends to be successfully written on peer.
 		 * see also notes above in HANDED_OVER_TO_NETWORK about
 		 * protocol != C */
 	ack_common:
-		mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK);
-		break;
-
-	case POSTPONE_WRITE:
-		D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
-		/* If this node has already detected the write conflict, the
-		 * worker will be waiting on misc_wait.  Wake it up once this
-		 * request has completed locally.
-		 */
-		D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
-		req->rq_state |= RQ_POSTPONED;
-		if (req->i.waiting)
-			wake_up(&device->misc_wait);
-		/* Do not clear RQ_NET_PENDING. This request will make further
-		 * progress via restart_conflicting_writes() or
-		 * fail_postponed_requests(). Hopefully. */
+		mod_rq_state(req, m, peer_device, RQ_NET_PENDING, RQ_NET_OK);
 		break;
 
 	case NEG_ACKED:
-		mod_rq_state(req, m, RQ_NET_OK|RQ_NET_PENDING, 0);
+		mod_rq_state(req, m, peer_device, RQ_NET_OK|RQ_NET_PENDING,
+			     (req->local_rq_state & RQ_WRITE) ? 0 : RQ_NET_DONE);
 		break;
 
-	case FAIL_FROZEN_DISK_IO:
-		if (!(req->rq_state & RQ_LOCAL_COMPLETED))
-			break;
-		mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
+	case COMPLETION_RESUMED:
+		mod_rq_state(req, m, peer_device, RQ_COMPLETION_SUSP, 0);
 		break;
 
-	case RESTART_FROZEN_DISK_IO:
-		if (!(req->rq_state & RQ_LOCAL_COMPLETED))
+	case CANCEL_SUSPENDED_IO:
+		/* Only apply to requests that were for this peer but not done. */
+		if (!(req->net_rq_state[idx] & RQ_NET_MASK) || req->net_rq_state[idx] & RQ_NET_DONE)
 			break;
 
-		mod_rq_state(req, m,
-				RQ_COMPLETION_SUSP|RQ_LOCAL_COMPLETED,
-				RQ_LOCAL_PENDING);
-
-		rv = MR_READ;
-		if (bio_data_dir(req->master_bio) == WRITE)
-			rv = MR_WRITE;
-
-		get_ldev(device); /* always succeeds in this call path */
-		req->w.cb = w_restart_disk_io;
-		drbd_queue_work(&connection->sender_work,
-				&req->w);
+		/* CONNECTION_LOST_WHILE_SUSPENDED followed by
+		 * CANCEL_SUSPENDED_IO should be essentially the same as
+		 * CONNECTION_LOST. Make the corresponding changes. The
+		 * RQ_COMPLETION_SUSP flag is handled by COMPLETION_RESUMED. */
+		mod_rq_state(req, m, peer_device, RQ_NET_PENDING|RQ_NET_OK, RQ_NET_DONE);
 		break;
 
 	case RESEND:
-		/* Simply complete (local only) READs. */
-		if (!(req->rq_state & RQ_WRITE) && !req->w.cb) {
-			mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
-			break;
-		}
-
 		/* If RQ_NET_OK is already set, we got a P_WRITE_ACK or P_RECV_ACK
 		   before the connection loss (B&C only); only P_BARRIER_ACK
 		   (or the local completion?) was missing when we suspended.
 		   Throwing them out of the TL here by pretending we got a BARRIER_ACK.
-		   During connection handshake, we ensure that the peer was not rebooted. */
-		if (!(req->rq_state & RQ_NET_OK)) {
-			/* FIXME could this possibly be a req->dw.cb == w_send_out_of_sync?
-			 * in that case we must not set RQ_NET_PENDING. */
-
-			mod_rq_state(req, m, RQ_COMPLETION_SUSP, RQ_NET_QUEUED|RQ_NET_PENDING);
-			if (req->w.cb) {
-				/* w.cb expected to be w_send_dblock, or w_send_read_req */
-				drbd_queue_work(&connection->sender_work,
-						&req->w);
-				rv = req->rq_state & RQ_WRITE ? MR_WRITE : MR_READ;
-			} /* else: FIXME can this happen? */
+		   During connection handshake, we ensure that the peer was not rebooted.
+
+		   Protocol A requests always have RQ_NET_OK removed when the
+		   connection is lost, so this will never apply to them.
+
+		   Resending is only allowed on synchronous connections,
+		   where all requests not yet completed to upper layers would
+		   be in the same "reorder-domain", there can not possibly be
+		   any dependency between incomplete requests, and we are
+		   allowed to complete this one "out-of-sequence".
+		 */
+		if (req->net_rq_state[idx] & RQ_NET_OK)
+			goto barrier_acked;
+
+		/* Only apply to requests that are pending a response from
+		 * this peer. */
+		if (!(req->net_rq_state[idx] & RQ_NET_PENDING))
 			break;
-		}
-		fallthrough;	/* to BARRIER_ACKED */
+
+		D_ASSERT(device, !(req->net_rq_state[idx] & RQ_NET_QUEUED));
+		mod_rq_state(req, m, peer_device, RQ_NET_SENT, RQ_NET_QUEUED);
+		break;
 
 	case BARRIER_ACKED:
+barrier_acked:
 		/* barrier ack for READ requests does not make sense */
-		if (!(req->rq_state & RQ_WRITE))
+		if (!(req->local_rq_state & RQ_WRITE))
 			break;
 
-		if (req->rq_state & RQ_NET_PENDING) {
+		if (req->net_rq_state[idx] & RQ_NET_PENDING) {
 			/* barrier came in before all requests were acked.
 			 * this is bad, because if the connection is lost now,
 			 * we won't be able to clean them up... */
 			drbd_err(device, "FIXME (BARRIER_ACKED but pending)\n");
+			mod_rq_state(req, m, peer_device, RQ_NET_PENDING, RQ_NET_OK);
 		}
-		/* Allowed to complete requests, even while suspended.
-		 * As this is called for all requests within a matching epoch,
+		/* As this is called for all requests within a matching epoch,
 		 * we need to filter, and only set RQ_NET_DONE for those that
 		 * have actually been on the wire. */
-		mod_rq_state(req, m, RQ_COMPLETION_SUSP,
-				(req->rq_state & RQ_NET_MASK) ? RQ_NET_DONE : 0);
+		if (req->net_rq_state[idx] & RQ_NET_MASK)
+			mod_rq_state(req, m, peer_device, 0, RQ_NET_DONE);
 		break;
 
 	case DATA_RECEIVED:
-		D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
-		mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK|RQ_NET_DONE);
+		D_ASSERT(device, req->net_rq_state[idx] & RQ_NET_PENDING);
+		mod_rq_state(req, m, peer_device, RQ_NET_PENDING, RQ_NET_OK|RQ_NET_DONE);
 		break;
 
-	case QUEUE_AS_DRBD_BARRIER:
-		start_new_tl_epoch(connection);
-		mod_rq_state(req, m, 0, RQ_NET_OK|RQ_NET_DONE);
+	case BARRIER_SENT:
+		mod_rq_state(req, m, peer_device, 0, RQ_NET_OK|RQ_NET_DONE);
 		break;
-	}
-
-	return rv;
+	};
 }
 
 /* we may do a local read if:
  * - we are consistent (of course),
  * - or we are generally inconsistent,
- *   BUT we are still/already IN SYNC for this area.
+ *   BUT we are still/already IN SYNC with all peers for this area.
  *   since size may be bigger than BM_BLOCK_SIZE,
  *   we may need to check several bits.
  */
 static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size)
 {
+	struct drbd_md *md = &device->ldev->md;
+	struct drbd_bitmap *bm;
+	unsigned int node_id;
+	unsigned int n_checked = 0;
+
 	unsigned long sbnr, ebnr;
 	sector_t esector, nr_sectors;
 
-	if (device->state.disk == D_UP_TO_DATE)
+	if (device->disk_state[NOW] == D_UP_TO_DATE)
 		return true;
-	if (device->state.disk != D_INCONSISTENT)
+	if (device->disk_state[NOW] != D_INCONSISTENT)
 		return false;
 	esector = sector + (size >> 9) - 1;
 	nr_sectors = get_capacity(device->vdisk);
 	D_ASSERT(device, sector  < nr_sectors);
 	D_ASSERT(device, esector < nr_sectors);
 
-	sbnr = BM_SECT_TO_BIT(sector);
-	ebnr = BM_SECT_TO_BIT(esector);
+	bm = device->bitmap;
+	if (!bm)
+		return true;
+
+	sbnr = bm_sect_to_bit(bm, sector);
+	ebnr = bm_sect_to_bit(bm, esector);
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &md->peers[node_id];
 
-	return drbd_bm_count_bits(device, sbnr, ebnr) == 0;
+		/* Skip bitmap indexes which are not assigned to a peer. */
+		if (!(peer_md->flags & MDF_HAVE_BITMAP))
+			continue;
+
+		if (drbd_bm_count_bits(device, peer_md->bitmap_index, sbnr, ebnr))
+			return false;
+		++n_checked;
+	}
+	if (n_checked == 0) {
+		drbd_err_ratelimit(device, "No valid bitmap slots found to check!\n");
+		return false;
+	}
+	return true;
 }
 
-static bool remote_due_to_read_balancing(struct drbd_device *device, sector_t sector,
+/* TODO improve for more than one peer.
+ * also take into account the drbd protocol. */
+static bool remote_due_to_read_balancing(struct drbd_device *device,
+		struct drbd_peer_device *peer_device, sector_t sector,
 		enum drbd_read_balancing rbm)
 {
 	int stripe_shift;
 
 	switch (rbm) {
 	case RB_CONGESTED_REMOTE:
+		/* originally, this used the bdi congestion framework,
+		 * but that was removed in linux 5.18.
+		 * so just never report the lower device as congested. */
 		return false;
 	case RB_LEAST_PENDING:
 		return atomic_read(&device->local_cnt) >
-			atomic_read(&device->ap_pending_cnt) + atomic_read(&device->rs_pending_cnt);
+			atomic_read(&peer_device->ap_pending_cnt) + atomic_read(&peer_device->rs_pending_cnt);
 	case RB_32K_STRIPING:  /* stripe_shift = 15 */
 	case RB_64K_STRIPING:
 	case RB_128K_STRIPING:
@@ -942,63 +1526,32 @@ static bool remote_due_to_read_balancing(struct drbd_device *device, sector_t se
 	}
 }
 
-/*
- * complete_conflicting_writes  -  wait for any conflicting write requests
- *
- * The write_requests tree contains all active write requests which we
- * currently know about.  Wait for any requests to complete which conflict with
- * the new one.
- *
- * Only way out: remove the conflicting intervals from the tree.
- */
-static void complete_conflicting_writes(struct drbd_request *req)
-{
-	DEFINE_WAIT(wait);
-	struct drbd_device *device = req->device;
-	struct drbd_interval *i;
-	sector_t sector = req->i.sector;
-	int size = req->i.size;
-
-	for (;;) {
-		drbd_for_each_overlap(i, &device->write_requests, sector, size) {
-			/* Ignore, if already completed to upper layers. */
-			if (i->completed)
-				continue;
-			/* Handle the first found overlap.  After the schedule
-			 * we have to restart the tree walk. */
-			break;
-		}
-		if (!i)	/* if any */
-			break;
-
-		/* Indicate to wake up device->misc_wait on progress.  */
-		prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE);
-		i->waiting = true;
-		spin_unlock_irq(&device->resource->req_lock);
-		schedule();
-		spin_lock_irq(&device->resource->req_lock);
-	}
-	finish_wait(&device->misc_wait, &wait);
-}
-
-/* called within req_lock */
-static void maybe_pull_ahead(struct drbd_device *device)
+static void __maybe_pull_ahead(struct drbd_device *device, struct drbd_connection *connection)
 {
-	struct drbd_connection *connection = first_peer_device(device)->connection;
 	struct net_conf *nc;
 	bool congested = false;
 	enum drbd_on_congestion on_congestion;
+	u32 cong_fill = 0, cong_extents = 0;
+	struct drbd_peer_device *peer_device = conn_peer_device(connection, device->vnr);
 
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	on_congestion = nc ? nc->on_congestion : OC_BLOCK;
-	rcu_read_unlock();
-	if (on_congestion == OC_BLOCK ||
-	    connection->agreed_pro_version < 96)
+	lockdep_assert_held(&device->resource->state_rwlock);
+
+	if (connection->agreed_pro_version < 96)
+		return;
+
+	nc = rcu_dereference(connection->transport.net_conf);
+	if (nc) {
+		on_congestion = nc->on_congestion;
+		cong_fill = nc->cong_fill;
+		cong_extents = nc->cong_extents;
+	} else {
+		on_congestion = OC_BLOCK;
+	}
+	if (on_congestion == OC_BLOCK)
 		return;
 
-	if (on_congestion == OC_PULL_AHEAD && device->state.conn == C_AHEAD)
-		return; /* nothing to do ... */
+	if (!drbd_should_do_remote(peer_device, NOW))
+		return; /* Ignore congestion if we are not replicating writes */
 
 	/* If I don't even have good local storage, we can not reasonably try
 	 * to pull ahead of the peer. We also need the local reference to make
@@ -1007,42 +1560,122 @@ static void maybe_pull_ahead(struct drbd_device *device)
 	if (!get_ldev_if_state(device, D_UP_TO_DATE))
 		return;
 
-	if (nc->cong_fill &&
-	    atomic_read(&device->ap_in_flight) >= nc->cong_fill) {
-		drbd_info(device, "Congestion-fill threshold reached\n");
-		congested = true;
+	if (test_and_set_bit(HANDLING_CONGESTION, &peer_device->flags))
+		goto out;
+
+	/* if an other volume already found that we are congested, short circuit. */
+	congested = test_bit(CONN_CONGESTED, &connection->flags);
+
+	if (!congested && cong_fill) {
+		int n = atomic_read(&connection->ap_in_flight) +
+			atomic_read(&connection->rs_in_flight);
+		if (n >= cong_fill) {
+			drbd_info(device, "Congestion-fill threshold reached (%d >= %d)\n", n, cong_fill);
+			congested = true;
+		}
 	}
 
-	if (device->act_log->used >= nc->cong_extents) {
-		drbd_info(device, "Congestion-extents threshold reached\n");
+	if (!congested && device->act_log->used >= cong_extents) {
+		drbd_info(device, "Congestion-extents threshold reached (%d >= %d)\n",
+			device->act_log->used, cong_extents);
 		congested = true;
 	}
 
 	if (congested) {
-		/* start a new epoch for non-mirrored writes */
-		start_new_tl_epoch(first_peer_device(device)->connection);
-
-		if (on_congestion == OC_PULL_AHEAD)
-			_drbd_set_state(_NS(device, conn, C_AHEAD), 0, NULL);
-		else  /*nc->on_congestion == OC_DISCONNECT */
-			_drbd_set_state(_NS(device, conn, C_DISCONNECTING), 0, NULL);
+		set_bit(CONN_CONGESTED, &connection->flags);
+		drbd_peer_device_post_work(peer_device, HANDLE_CONGESTION);
+	} else {
+		clear_bit(HANDLING_CONGESTION, &peer_device->flags);
 	}
+out:
 	put_ldev(device);
 }
 
-/* If this returns false, and req->private_bio is still set,
- * this should be submitted locally.
+static void maybe_pull_ahead(struct drbd_device *device)
+{
+	struct drbd_connection *connection;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, device->resource)
+		if (connection->cstate[NOW] == C_CONNECTED)
+			__maybe_pull_ahead(device, connection);
+	rcu_read_unlock();
+}
+
+bool drbd_should_do_remote(struct drbd_peer_device *peer_device, enum which_state which)
+{
+	enum drbd_disk_state peer_disk_state = peer_device->disk_state[which];
+	enum drbd_repl_state repl_state = peer_device->repl_state[which];
+	bool replication = peer_device->replication[which];
+
+	return peer_disk_state == D_UP_TO_DATE ||
+		(peer_disk_state == D_INCONSISTENT && replication &&
+		 (repl_state == L_ESTABLISHED ||
+		  (repl_state >= L_WF_BITMAP_T && repl_state < L_AHEAD)));
+	/* Before proto 96 that was >= CONNECTED instead of >= L_WF_BITMAP_T.
+	   That is equivalent since before 96 IO was frozen in the L_WF_BITMAP*
+	   states. */
+}
+
+static bool drbd_should_send_out_of_sync(struct drbd_peer_device *peer_device)
+{
+	enum drbd_disk_state peer_disk_state = peer_device->disk_state[NOW];
+	enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+	bool replication = peer_device->replication[NOW];
+
+	return repl_state == L_AHEAD ||
+		repl_state == L_WF_BITMAP_S ||
+		(repl_state >= L_ESTABLISHED &&
+		 (peer_disk_state == D_OUTDATED ||
+		  (peer_disk_state == D_INCONSISTENT && !replication)));
+
+	/* proto 96 check omitted, there was no L_AHEAD back then,
+	 * peer disk was never Outdated while connection was established,
+	 * and IO was frozen during bitmap exchange */
+}
+
+/* Prefer to read from protcol C peers, then B, last A */
+static u64 calc_nodes_to_read_from(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	u64 candidates[DRBD_PROT_C] = {};
+	int wp;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		struct net_conf *nc;
+
+		if (peer_device->disk_state[NOW] != D_UP_TO_DATE)
+			continue;
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		if (!nc || !nc->allow_remote_read)
+			continue;
+		wp = nc->wire_protocol;
+		candidates[wp - 1] |= NODE_MASK(peer_device->node_id);
+	}
+	rcu_read_unlock();
+
+	for (wp = DRBD_PROT_C; wp >= DRBD_PROT_A; wp--) {
+		if (candidates[wp - 1])
+			return candidates[wp - 1];
+	}
+	return 0;
+}
+
+/* If this returns NULL, and req->private_bio is still set,
+ * the request should be submitted locally.
  *
- * If it returns false, but req->private_bio is not set,
+ * If it returns NULL, but req->private_bio is not set,
  * we do not have access to good data :(
  *
  * Otherwise, this destroys req->private_bio, if any,
- * and returns true.
+ * and returns the peer device which should be asked for data.
  */
-static bool do_remote_read(struct drbd_request *req)
+static struct drbd_peer_device *find_peer_device_for_read(struct drbd_request *req)
 {
+	struct drbd_peer_device *peer_device;
 	struct drbd_device *device = req->device;
-	enum drbd_read_balancing rbm;
+	enum drbd_read_balancing rbm = RB_PREFER_REMOTE;
 
 	if (req->private_bio) {
 		if (!drbd_may_do_local_read(device,
@@ -1053,90 +1686,122 @@ static bool do_remote_read(struct drbd_request *req)
 		}
 	}
 
-	if (device->state.pdsk != D_UP_TO_DATE)
-		return false;
-
-	if (req->private_bio == NULL)
-		return true;
-
-	/* TODO: improve read balancing decisions, take into account drbd
-	 * protocol, pending requests etc. */
-
-	rcu_read_lock();
-	rbm = rcu_dereference(device->ldev->disk_conf)->read_balancing;
-	rcu_read_unlock();
-
-	if (rbm == RB_PREFER_LOCAL && req->private_bio)
-		return false; /* submit locally */
+	if (device->disk_state[NOW] > D_DISKLESS) {
+		rcu_read_lock();
+		rbm = rcu_dereference(device->ldev->disk_conf)->read_balancing;
+		rcu_read_unlock();
+		if (rbm == RB_PREFER_LOCAL && req->private_bio) {
+			return NULL; /* submit locally */
+		}
+	}
 
-	if (remote_due_to_read_balancing(device, req->i.sector, rbm)) {
-		if (req->private_bio) {
-			bio_put(req->private_bio);
-			req->private_bio = NULL;
-			put_ldev(device);
+	/* TODO: improve read balancing decisions, allow user to configure node weights */
+	while (true) {
+		if (!device->read_nodes)
+			device->read_nodes = calc_nodes_to_read_from(device);
+		if (device->read_nodes) {
+			int peer_node_id = __ffs64(device->read_nodes);
+			device->read_nodes &= ~NODE_MASK(peer_node_id);
+			peer_device = peer_device_by_node_id(device, peer_node_id);
+			if (!peer_device)
+				continue;
+			if (peer_device->disk_state[NOW] != D_UP_TO_DATE)
+				continue;
+			if (req->private_bio &&
+			    !remote_due_to_read_balancing(device, peer_device, req->i.sector, rbm))
+				peer_device = NULL;
+		} else {
+			peer_device = NULL;
 		}
-		return true;
+		break;
 	}
 
-	return false;
+	if (peer_device && req->private_bio) {
+		bio_put(req->private_bio);
+		req->private_bio = NULL;
+		put_ldev(device);
+	}
+	return peer_device;
 }
 
-bool drbd_should_do_remote(union drbd_dev_state s)
+static int drbd_process_empty_flush(struct drbd_request *req)
 {
-	return s.pdsk == D_UP_TO_DATE ||
-		(s.pdsk >= D_INCONSISTENT &&
-		 s.conn >= C_WF_BITMAP_T &&
-		 s.conn < C_AHEAD);
-	/* Before proto 96 that was >= CONNECTED instead of >= C_WF_BITMAP_T.
-	   That is equivalent since before 96 IO was frozen in the C_WF_BITMAP*
-	   states. */
-}
+	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device;
+	int count = 0;
+
+	for_each_peer_device(peer_device, device) {
+		/* When a flush is submitted, the expectation is that the data
+		 * is written somewhere in a usable form. Hence only
+		 * D_UP_TO_DATE peers are included and not all peers that
+		 * receive the data. */
+		if (peer_device->disk_state[NOW] == D_UP_TO_DATE) {
+			++count;
+
+			/* An empty flush indicates that all previously
+			 * completed requests should be written out to stable
+			 * storage. Request completion already triggers a
+			 * barrier to be sent and the current epoch closed. The
+			 * barrier causes the data to be written out unless
+			 * that is configured not to be necessary.
+			 *
+			 * Hence there is nothing more to be done to cause the
+			 * writing out to persistent storage which was
+			 * requested. We just mark the request so that we know
+			 * that a flush has effectively occurred on this peer
+			 * so that we can complete it successfully.
+			 *
+			 * We _should_ wait for any outstanding barriers to
+			 * protocol C peers to be acked before completing this
+			 * request, so that we are sure that the previously
+			 * completed requests have really been written out
+			 * there too. However, DRBD has never yet implemented
+			 * this. */
+			_req_mod(req, BARRIER_SENT, peer_device);
+		}
+	}
 
-static bool drbd_should_send_out_of_sync(union drbd_dev_state s)
-{
-	return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
-	/* pdsk = D_INCONSISTENT as a consequence. Protocol 96 check not necessary
-	   since we enter state C_AHEAD only if proto >= 96 */
+	return count;
 }
 
-/* returns number of connections (== 1, for drbd 8.4)
- * expected to actually write this data,
+/* returns the number of connections expected to actually write this data,
  * which does NOT include those that we are L_AHEAD for. */
 static int drbd_process_write_request(struct drbd_request *req)
 {
 	struct drbd_device *device = req->device;
-	struct drbd_peer_device *peer_device = first_peer_device(device);
+	struct drbd_peer_device *peer_device;
 	int remote, send_oos;
+	int count = 0;
 
-	remote = drbd_should_do_remote(device->state);
-	send_oos = drbd_should_send_out_of_sync(device->state);
+	for_each_peer_device(peer_device, device) {
+		remote = drbd_should_do_remote(peer_device, NOW);
+		send_oos = drbd_should_send_out_of_sync(peer_device);
 
-	/* Need to replicate writes.  Unless it is an empty flush,
-	 * which is better mapped to a DRBD P_BARRIER packet,
-	 * also for drbd wire protocol compatibility reasons.
-	 * If this was a flush, just start a new epoch.
-	 * Unless the current epoch was empty anyways, or we are not currently
-	 * replicating, in which case there is no point. */
-	if (unlikely(req->i.size == 0)) {
-		/* The only size==0 bios we expect are empty flushes. */
-		D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH);
-		if (remote)
-			_req_mod(req, QUEUE_AS_DRBD_BARRIER, peer_device);
-		return remote;
-	}
+		if (!remote && !send_oos)
+			continue;
 
-	if (!remote && !send_oos)
-		return 0;
+		D_ASSERT(device, !(remote && send_oos));
 
-	D_ASSERT(device, !(remote && send_oos));
+		if (remote) {
+			++count;
+			_req_mod(req, NEW_NET_WRITE, peer_device);
+		} else
+			_req_mod(req, NEW_NET_OOS, peer_device);
+	}
+
+	return count;
+}
 
-	if (remote) {
-		_req_mod(req, TO_BE_SENT, peer_device);
-		_req_mod(req, QUEUE_FOR_NET_WRITE, peer_device);
-	} else if (drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size))
-		_req_mod(req, QUEUE_FOR_SEND_OOS, peer_device);
+static void drbd_request_ready_for_net(struct drbd_request *req)
+{
+	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device;
 
-	return remote;
+	for_each_peer_device(peer_device, device) {
+		/* Do not mark RQ_NET_PENDING_OOS requests ready yet */
+		if (req->net_rq_state[peer_device->node_id] & RQ_NET_PENDING)
+			_req_mod(req, READY_FOR_NET, peer_device);
+	}
 }
 
 static void drbd_process_discard_or_zeroes_req(struct drbd_request *req, int flags)
@@ -1162,45 +1827,66 @@ drbd_submit_req_private_bio(struct drbd_request *req)
 	else
 		type = DRBD_FAULT_DT_RD;
 
+	bio_set_dev(bio, device->ldev->backing_bdev);
+
 	/* State may have changed since we grabbed our reference on the
-	 * ->ldev member. Double check, and short-circuit to endio.
+	 * device->ldev member. Double check, and short-circuit to endio.
 	 * In case the last activity log transaction failed to get on
 	 * stable storage, and this is a WRITE, we may not even submit
 	 * this bio. */
 	if (get_ldev(device)) {
-		if (drbd_insert_fault(device, type))
-			bio_io_error(bio);
-		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
+		if (drbd_insert_fault(device, type)) {
+			bio->bi_status = BLK_STS_IOERR;
+			bio_endio(bio);
+		} else if (bio_op(bio) == REQ_OP_WRITE_ZEROES) {
 			drbd_process_discard_or_zeroes_req(req, EE_ZEROOUT |
 			    ((bio->bi_opf & REQ_NOUNMAP) ? 0 : EE_TRIM));
-		else if (bio_op(bio) == REQ_OP_DISCARD)
+		} else if (bio_op(bio) == REQ_OP_DISCARD) {
 			drbd_process_discard_or_zeroes_req(req, EE_TRIM);
-		else
+		} else {
 			submit_bio_noacct(bio);
+		}
 		put_ldev(device);
-	} else
-		bio_io_error(bio);
-}
+	} else {
+		bio->bi_status = BLK_STS_IOERR;
+		bio_endio(bio);
+	}
+ }
 
 static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req)
 {
-	spin_lock_irq(&device->resource->req_lock);
-	list_add_tail(&req->tl_requests, &device->submit.writes);
+	if (req->private_bio)
+		atomic_inc(&device->ap_actlog_cnt);
+	spin_lock_irq(&device->pending_completion_lock);
 	list_add_tail(&req->req_pending_master_completion,
 			&device->pending_master_completion[1 /* WRITE */]);
-	spin_unlock_irq(&device->resource->req_lock);
+	spin_unlock_irq(&device->pending_completion_lock);
+	spin_lock(&device->submit.lock);
+	list_add_tail(&req->list, &device->submit.writes);
+	spin_unlock(&device->submit.lock);
 	queue_work(device->submit.wq, &device->submit.worker);
 	/* do_submit() may sleep internally on al_wait, too */
 	wake_up(&device->al_wait);
 }
 
-/* returns the new drbd_request pointer, if the caller is expected to
- * drbd_send_and_submit() it (to save latency), or NULL if we queued the
- * request on the submitter thread.
+static void drbd_req_in_actlog(struct drbd_request *req)
+{
+	req->local_rq_state |= RQ_IN_ACT_LOG;
+	ktime_get_accounting(req->in_actlog_kt);
+	atomic_sub(interval_to_al_extents(&req->i), &req->device->wait_for_actlog_ecnt);
+}
+
+/* returns the new drbd_request pointer, if the caller is expected to submit it
+ * (to save latency), or NULL if we queued the request on the submitter thread.
  * Returns ERR_PTR(-ENOMEM) if we cannot allocate a drbd_request.
  */
+#ifndef CONFIG_DRBD_TIMING_STATS
+#define drbd_request_prepare(d, b, k, j) drbd_request_prepare(d, b, j)
+#endif
 static struct drbd_request *
-drbd_request_prepare(struct drbd_device *device, struct bio *bio)
+drbd_request_prepare(struct drbd_device *device, struct bio *bio,
+		ktime_t start_kt,
+		unsigned long start_jif)
 {
 	const int rw = bio_data_dir(bio);
 	struct drbd_request *req;
@@ -1208,44 +1894,65 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio)
 	/* allocate outside of all locks; */
 	req = drbd_req_new(device, bio);
 	if (!req) {
-		dec_ap_bio(device);
-		/* only pass the error to the upper layers.
-		 * if user cannot handle io errors, that's not our business. */
 		drbd_err(device, "could not kmalloc() req\n");
-		bio->bi_status = BLK_STS_RESOURCE;
-		bio_endio(bio);
-		return ERR_PTR(-ENOMEM);
+		goto no_mem;
 	}
 
 	/* Update disk stats */
 	req->start_jif = bio_start_io_acct(req->master_bio);
 
 	if (get_ldev(device)) {
-		req->private_bio = bio_alloc_clone(device->ldev->backing_bdev,
-						   bio, GFP_NOIO,
-						   &drbd_io_bio_set);
+		req->private_bio = bio_alloc_clone(device->ldev->backing_bdev, bio, GFP_NOIO, &drbd_io_bio_set);
+		if (!req->private_bio) {
+			drbd_err(device, "could not bio_alloc_clone() req->private_bio\n");
+			kfree(req);
+			put_ldev(device);
+			goto no_mem;
+		}
 		req->private_bio->bi_private = req;
 		req->private_bio->bi_end_io = drbd_request_endio;
 	}
 
+	ktime_get_accounting_assign(req->start_kt, start_kt);
+
+	if (rw != WRITE || req->i.size == 0)
+		return req;
+
+	/* Let the activity log know we are about to use it...
+	 * FIXME
+	 * Needs to slow down to not congest on the activity log, in case we
+	 * have multiple primaries and the peer sends huge scattered epochs.
+	 * See also how peer_requests are handled
+	 * in receive_Data() { ... drbd_wait_for_activity_log_extents(); ... }
+	 */
+	if (req->private_bio)
+		atomic_add(interval_to_al_extents(&req->i), &device->wait_for_actlog_ecnt);
+
 	/* process discards always from our submitter thread */
-	if (bio_op(bio) == REQ_OP_WRITE_ZEROES ||
-	    bio_op(bio) == REQ_OP_DISCARD)
+	if ((bio_op(bio) == REQ_OP_WRITE_ZEROES) ||
+	    (bio_op(bio) == REQ_OP_DISCARD))
 		goto queue_for_submitter_thread;
 
-	if (rw == WRITE && req->private_bio && req->i.size
-	&& !test_bit(AL_SUSPENDED, &device->flags)) {
+	if (req->private_bio && !test_bit(AL_SUSPENDED, &device->flags)) {
 		if (!drbd_al_begin_io_fastpath(device, &req->i))
 			goto queue_for_submitter_thread;
-		req->rq_state |= RQ_IN_ACT_LOG;
-		req->in_actlog_jif = jiffies;
+		drbd_req_in_actlog(req);
 	}
 	return req;
 
  queue_for_submitter_thread:
-	atomic_inc(&device->ap_actlog_cnt);
+	ktime_aggregate_delta(device, req->start_kt, before_queue_kt);
 	drbd_queue_write(device, req);
 	return NULL;
+
+ no_mem:
+	dec_ap_bio(device, rw);
+	/* only pass the error to the upper layers.
+	 * if user cannot handle io errors, that's not our business.
+	 */
+	bio->bi_status = BLK_STS_RESOURCE;
+	bio_endio(bio);
+	return ERR_PTR(-ENOMEM);
 }
 
 /* Require at least one path to current data.
@@ -1260,8 +1967,17 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio)
  */
 static bool may_do_writes(struct drbd_device *device)
 {
-	const union drbd_dev_state s = device->state;
-	return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE;
+	struct drbd_peer_device *peer_device;
+
+	if (device->disk_state[NOW] == D_UP_TO_DATE)
+		return true;
+
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->disk_state[NOW] == D_UP_TO_DATE)
+		    return true;
+	}
+
+	return false;
 }
 
 struct drbd_plug_cb {
@@ -1273,21 +1989,23 @@ struct drbd_plug_cb {
 static void drbd_unplug(struct blk_plug_cb *cb, bool from_schedule)
 {
 	struct drbd_plug_cb *plug = container_of(cb, struct drbd_plug_cb, cb);
-	struct drbd_resource *resource = plug->cb.data;
 	struct drbd_request *req = plug->most_recent_req;
+	struct drbd_resource *resource = req->device->resource;
 
 	kfree(cb);
 	if (!req)
 		return;
 
-	spin_lock_irq(&resource->req_lock);
+	read_lock_irq(&resource->state_rwlock);
 	/* In case the sender did not process it yet, raise the flag to
 	 * have it followed with P_UNPLUG_REMOTE just after. */
-	req->rq_state |= RQ_UNPLUG;
+	spin_lock(&req->rq_lock);
+	req->local_rq_state |= RQ_UNPLUG;
+	spin_unlock(&req->rq_lock);
 	/* but also queue a generic unplug */
 	drbd_queue_unplug(req->device);
 	kref_put(&req->kref, drbd_req_destroy);
-	spin_unlock_irq(&resource->req_lock);
+	read_unlock_irq(&resource->state_rwlock);
 }
 
 static struct drbd_plug_cb* drbd_check_plugged(struct drbd_resource *resource)
@@ -1307,40 +2025,34 @@ static struct drbd_plug_cb* drbd_check_plugged(struct drbd_resource *resource)
 static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req)
 {
 	struct drbd_request *tmp = plug->most_recent_req;
-	/* Will be sent to some peer.
-	 * Remember to tag it with UNPLUG_REMOTE on unplug */
+	/* Will be sent to some peer. */
 	kref_get(&req->kref);
 	plug->most_recent_req = req;
 	if (tmp)
 		kref_put(&tmp->kref, drbd_req_destroy);
 }
 
-static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req)
+static void drbd_send_and_submit(struct drbd_request *req)
 {
+	struct drbd_device *device = req->device;
 	struct drbd_resource *resource = device->resource;
-	struct drbd_peer_device *peer_device = first_peer_device(device);
+	struct drbd_peer_device *peer_device = NULL; /* for read */
 	const int rw = bio_data_dir(req->master_bio);
 	struct bio_and_error m = { NULL, };
 	bool no_remote = false;
 	bool submit_private_bio = false;
 
-	spin_lock_irq(&resource->req_lock);
-	if (rw == WRITE) {
-		/* This may temporarily give up the req_lock,
-		 * but will re-aquire it before it returns here.
-		 * Needs to be before the check on drbd_suspended() */
-		complete_conflicting_writes(req);
-		/* no more giving up req_lock from now on! */
+	read_lock_irq(&resource->state_rwlock);
 
+	if (rw == WRITE) {
 		/* check for congestion, and potentially stop sending
 		 * full data updates, but start sending "dirty bits" only. */
 		maybe_pull_ahead(device);
 	}
 
-
 	if (drbd_suspended(device)) {
 		/* push back and retry: */
-		req->rq_state |= RQ_POSTPONED;
+		req->local_rq_state |= RQ_POSTPONED;
 		if (req->private_bio) {
 			bio_put(req->private_bio);
 			req->private_bio = NULL;
@@ -1349,44 +2061,87 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 		goto out;
 	}
 
-	/* We fail READ early, if we can not serve it.
-	 * We must do this before req is registered on any lists.
-	 * Otherwise, drbd_req_complete() will queue failed READ for retry. */
-	if (rw != WRITE) {
-		if (!do_remote_read(req) && !req->private_bio)
+	if (rw == WRITE) {
+		if (!may_do_writes(device)) {
+			if (req->private_bio) {
+				bio_put(req->private_bio);
+				req->private_bio = NULL;
+				put_ldev(device);
+			}
+			goto nodata;
+		}
+	} else {
+		/* We fail READ early, if we can not serve it.
+		 * We must do this before req is registered on any lists.
+		 * Otherwise, drbd_req_complete() will queue failed READ for retry. */
+		peer_device = find_peer_device_for_read(req);
+		if (!peer_device && !req->private_bio)
 			goto nodata;
 	}
 
-	/* which transfer log epoch does this belong to? */
-	req->epoch = atomic_read(&first_peer_device(device)->connection->current_tle_nr);
+	spin_lock(&resource->tl_update_lock); /* local irq already disabled */
+	if (rw == WRITE) {
+		/* Update dagtag_sector before determining current_tle_nr so
+		 * that senders can detect if there are requests currently
+		 * being submitted. Updates are protected by tl_update_lock,
+		 * but reads are not, so WRITE_ONCE(). */
+		WRITE_ONCE(resource->dagtag_sector, resource->dagtag_sector + (req->i.size >> 9));
+		/* Ensure that the written value is visible to the senders. */
+		smp_wmb();
+	}
+	req->dagtag_sector = resource->dagtag_sector;
 
-	/* no point in adding empty flushes to the transfer log,
-	 * they are mapped to drbd barriers already. */
-	if (likely(req->i.size!=0)) {
-		if (rw == WRITE)
-			first_peer_device(device)->connection->current_tle_writes++;
+	spin_lock(&resource->current_tle_lock);
+	/* which transfer log epoch does this belong to? */
+	req->epoch = atomic_read(&resource->current_tle_nr);
+	if (rw == WRITE && likely(req->i.size != 0))
+		resource->current_tle_writes++;
+	spin_unlock(&resource->current_tle_lock);
 
-		list_add_tail(&req->tl_requests, &first_peer_device(device)->connection->transfer_log);
-	}
+	/* A size==0 bio can only be an empty flush, which is mapped to a DRBD
+	 * P_BARRIER packet. */
+	if (unlikely(req->i.size == 0)) {
+		/* The only size==0 bios we expect are empty flushes. */
+		D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH);
 
-	if (rw == WRITE) {
-		if (req->private_bio && !may_do_writes(device)) {
-			bio_put(req->private_bio);
-			req->private_bio = NULL;
-			put_ldev(device);
-			goto nodata;
-		}
-		if (!drbd_process_write_request(req))
+		if (!drbd_process_empty_flush(req))
 			no_remote = true;
 	} else {
-		/* We either have a private_bio, or we can read from remote.
-		 * Otherwise we had done the goto nodata above. */
-		if (req->private_bio == NULL) {
-			_req_mod(req, TO_BE_SENT, peer_device);
-			_req_mod(req, QUEUE_FOR_NET_READ, peer_device);
-		} else
-			no_remote = true;
+		if (rw == WRITE) {
+			struct drbd_request *prev_write = resource->tl_previous_write;
+			resource->tl_previous_write = req;
+
+			if (prev_write) {
+				if (!test_bit(INTERVAL_DONE, &prev_write->i.flags))
+					refcount_inc(&req->done_ref);
+				refcount_inc(&req->oos_send_ref);
+				prev_write->next_write = req;
+			}
+
+			if (!drbd_process_write_request(req))
+				no_remote = true;
+		} else {
+			if (peer_device)
+				_req_mod(req, NEW_NET_READ, peer_device);
+			else
+				no_remote = true;
+		}
+
+		/* req may now be accessed by other threads - do not modify
+		 * "immutable" fields after this point */
+		list_add_tail_rcu(&req->tl_requests, &resource->transfer_log);
+
+		/* Do this after adding to the transfer log so that the
+		 * caching pointer req_not_net_done is set if
+		 * necessary. */
+		drbd_request_ready_for_net(req);
 	}
+	spin_unlock(&resource->tl_update_lock);
+
+	if (rw == WRITE)
+		wake_all_senders(resource);
+	else if (peer_device)
+		wake_up(&peer_device->connection->sender_work.q_wait);
 
 	if (no_remote == false) {
 		struct drbd_plug_cb *plug = drbd_check_plugged(resource);
@@ -1396,29 +2151,38 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 
 	/* If it took the fast path in drbd_request_prepare, add it here.
 	 * The slow path has added it already. */
+	spin_lock(&device->pending_completion_lock); /* local irq already disabled */
 	if (list_empty(&req->req_pending_master_completion))
 		list_add_tail(&req->req_pending_master_completion,
 			&device->pending_master_completion[rw == WRITE]);
 	if (req->private_bio) {
-		/* needs to be marked within the same spinlock */
+		/* pre_submit_jif is used in request_timer_fn() */
 		req->pre_submit_jif = jiffies;
+		ktime_get_accounting(req->pre_submit_kt);
 		list_add_tail(&req->req_pending_local,
 			&device->pending_completion[rw == WRITE]);
 		_req_mod(req, TO_BE_SUBMITTED, NULL);
-		/* but we need to give up the spinlock to submit */
+		/* needs to be marked within the same spinlock
+		 * but we need to give up the spinlock to submit */
 		submit_private_bio = true;
-	} else if (no_remote) {
+		spin_unlock(&device->pending_completion_lock);
+	} else {
+		spin_unlock(&device->pending_completion_lock);
+		if (no_remote) {
 nodata:
-		if (drbd_ratelimit())
-			drbd_err(device, "IO ERROR: neither local nor remote data, sector %llu+%u\n",
-					(unsigned long long)req->i.sector, req->i.size >> 9);
-		/* A write may have been queued for send_oos, however.
-		 * So we can not simply free it, we must go through drbd_req_put_completion_ref() */
+			drbd_err_ratelimit(req->device,
+				"IO ERROR: neither local nor remote data, sector %llu+%u\n",
+				 (unsigned long long)req->i.sector, req->i.size >> 9);
+			/* A write may have been queued for send_oos, however.
+			 * So we can not simply free it, we must go through
+			 * drbd_req_put_completion_ref()
+			 */
+		}
 	}
 
 out:
 	drbd_req_put_completion_ref(req, &m, 1);
-	spin_unlock_irq(&resource->req_lock);
+	read_unlock_irq(&resource->state_rwlock);
 
 	/* Even though above is a kref_put(), this is safe.
 	 * As long as we still need to submit our private bio,
@@ -1428,114 +2192,394 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 	 * That's why we cannot check on req->private_bio. */
 	if (submit_private_bio)
 		drbd_submit_req_private_bio(req);
+
 	if (m.bio)
 		complete_master_bio(device, &m);
 }
 
-void __drbd_make_request(struct drbd_device *device, struct bio *bio)
+/* Insert the request into the tree of writes. Pass it through to be submitted
+ * if possible. Otherwise it will be submitted asynchronously via
+ * drbd_release_conflicts once the conflict has been resolved. */
+static void drbd_conflict_submit_write(struct drbd_request *req)
+{
+	struct drbd_device *device = req->device;
+	struct drbd_interval *conflict;
+
+	spin_lock_irq(&device->interval_lock);
+	clear_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &req->i.flags);
+	conflict = drbd_find_conflict(device, &req->i, 0);
+	if (drbd_interval_empty(&req->i))
+		drbd_insert_interval(&device->requests, &req->i);
+	if (!conflict) {
+		set_bit(INTERVAL_SUBMITTED, &req->i.flags);
+	} else if (drbd_interval_is_local(conflict)) {
+		struct drbd_request *conflicting_req =
+			container_of(conflict, struct drbd_request, i);
+
+		if (conflicting_req->local_rq_state & RQ_POSTPONED) {
+			req->local_rq_state |= RQ_POSTPONED;
+
+			/*
+			 * Remove interval from tree to prevent req from being
+			 * queued when conflicts are released.
+			 */
+			drbd_remove_interval(&device->requests, &req->i);
+		}
+	}
+	spin_unlock_irq(&device->interval_lock);
+
+	/*
+	 * If there is a conflict, the request will be submitted once the
+	 * conflict has cleared.
+	 */
+	if (!conflict) {
+		drbd_send_and_submit(req);
+	} else if (req->local_rq_state & RQ_POSTPONED) {
+		if (req->private_bio) {
+			bio_put(req->private_bio);
+			req->private_bio = NULL;
+			put_ldev(device);
+		}
+		drbd_req_put_completion_ref(req, NULL, 1);
+	}
+}
+
+static bool inc_ap_bio_cond(struct drbd_device *device, int rw)
+{
+	int ap_bio_cnt;
+	bool rv;
+
+	read_lock_irq(&device->resource->state_rwlock);
+	rv = may_inc_ap_bio(device);
+	read_unlock_irq(&device->resource->state_rwlock);
+	if (!rv)
+		return false;
+
+	/* check need for new current uuid _AFTER_ ensuring IO is not suspended via may_inc_ap_bio */
+	if (test_bit(NEW_CUR_UUID, &device->flags)) {
+		if (!test_and_set_bit(WRITING_NEW_CUR_UUID, &device->flags))
+			drbd_device_post_work(device, MAKE_NEW_CUR_UUID);
+
+		return false;
+	}
+
+	do {
+		unsigned int nr_requests = device->resource->res_opts.nr_requests;
+
+		ap_bio_cnt = atomic_read(&device->ap_bio_cnt[rw]);
+		if (ap_bio_cnt >= nr_requests)
+			return false;
+	} while (atomic_cmpxchg(&device->ap_bio_cnt[rw], ap_bio_cnt, ap_bio_cnt + 1) != ap_bio_cnt);
+
+	return true;
+}
+
+static void inc_ap_bio(struct drbd_device *device, int rw)
+{
+	/* we wait here
+	 *    as long as the device is suspended
+	 *    until the bitmap is no longer on the fly during connection
+	 *    handshake as long as we would exceed the max_buffer limit.
+	 *
+	 * to avoid races with the reconnect code,
+	 * we need to atomic_inc within the spinlock. */
+
+	wait_event(device->misc_wait, inc_ap_bio_cond(device, rw));
+}
+
+void __drbd_make_request(struct drbd_device *device, struct bio *bio,
+		ktime_t start_kt,
+		unsigned long start_jif)
 {
-	struct drbd_request *req = drbd_request_prepare(device, bio);
+	const int rw = bio_data_dir(bio);
+	struct drbd_request *req;
+
+	inc_ap_bio(device, bio_data_dir(bio));
+	req = drbd_request_prepare(device, bio, start_kt, start_jif);
 	if (IS_ERR_OR_NULL(req))
 		return;
-	drbd_send_and_submit(device, req);
+
+	if (rw == WRITE)
+		drbd_conflict_submit_write(req);
+	else
+		drbd_send_and_submit(req);
+}
+
+/* Work function to submit requests once they are released after conflicts. The
+ * queued requests are processed and, if no other conflict is found, submitted. */
+void drbd_do_submit_conflict(struct work_struct *ws)
+{
+	struct drbd_device *device = container_of(ws, struct drbd_device, submit_conflict.worker);
+	struct drbd_peer_request *peer_req, *peer_req_tmp;
+	struct drbd_request *req, *tmp;
+	LIST_HEAD(resync_writes);
+	LIST_HEAD(resync_reads);
+	LIST_HEAD(writes);
+	LIST_HEAD(peer_writes);
+
+	spin_lock_irq(&device->submit_conflict.lock);
+	list_splice_init(&device->submit_conflict.resync_writes, &resync_writes);
+	list_splice_init(&device->submit_conflict.resync_reads, &resync_reads);
+	list_splice_init(&device->submit_conflict.writes, &writes);
+	list_splice_init(&device->submit_conflict.peer_writes, &peer_writes);
+	spin_unlock_irq(&device->submit_conflict.lock);
+
+	/* Delete the list entries when iterating them so that they can be re-used
+	 * for adding them to the conflict lists again once the
+	 * submit_conflict_queued flag has been cleared. */
+
+	list_for_each_entry_safe(peer_req, peer_req_tmp, &resync_writes, w.list) {
+		list_del_init(&peer_req->w.list);
+		if (!test_bit(INTERVAL_SENT, &peer_req->i.flags))
+			drbd_conflict_send_resync_request(peer_req);
+		else
+			drbd_conflict_submit_resync_request(peer_req);
+	}
+
+	list_for_each_entry_safe(peer_req, peer_req_tmp, &resync_reads, w.list) {
+		list_del_init(&peer_req->w.list);
+		drbd_conflict_submit_peer_read(peer_req);
+	}
+
+	list_for_each_entry_safe(req, tmp, &writes, list) {
+		list_del_init(&req->list);
+		drbd_conflict_submit_write(req);
+	}
+
+	list_for_each_entry_safe(peer_req, peer_req_tmp, &peer_writes, w.list) {
+		list_del_init(&peer_req->w.list);
+		drbd_conflict_submit_peer_write(peer_req);
+	}
+}
+
+/* helpers for do_submit */
+
+struct incoming_pending {
+	/* from drbd_submit_bio() or receive_Data() */
+	struct list_head incoming;
+	/* for non-blocking fill-up # of updates in the transaction */
+	struct list_head more_incoming;
+	/* to be submitted after next AL-transaction commit */
+	struct list_head pending;
+	/* need cleanup */
+	struct list_head cleanup;
+};
+
+struct waiting_for_act_log {
+	struct incoming_pending requests;
+	struct incoming_pending peer_requests;
+};
+
+static void ipb_init(struct incoming_pending *ipb)
+{
+	INIT_LIST_HEAD(&ipb->incoming);
+	INIT_LIST_HEAD(&ipb->more_incoming);
+	INIT_LIST_HEAD(&ipb->pending);
+	INIT_LIST_HEAD(&ipb->cleanup);
+}
+
+static void wfa_init(struct waiting_for_act_log *wfa)
+{
+	ipb_init(&wfa->requests);
+	ipb_init(&wfa->peer_requests);
+}
+
+#define wfa_lists_empty(_wfa, name)	\
+	(list_empty(&(_wfa)->requests.name) && list_empty(&(_wfa)->peer_requests.name))
+#define wfa_splice_tail_init(_wfa, from, to) do { \
+	list_splice_tail_init(&(_wfa)->requests.from, &(_wfa)->requests.to); \
+	list_splice_tail_init(&(_wfa)->peer_requests.from, &(_wfa)->peer_requests.to); \
+	} while (0)
+
+static void __drbd_submit_peer_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	int err;
+
+	peer_req->flags |= EE_IN_ACTLOG;
+	atomic_sub(interval_to_al_extents(&peer_req->i), &device->wait_for_actlog_ecnt);
+	atomic_dec(&device->wait_for_actlog);
+	list_del_init(&peer_req->w.list);
+
+	err = drbd_submit_peer_request(peer_req);
+
+	if (err)
+		drbd_cleanup_after_failed_submit_peer_write(peer_req);
 }
 
-static void submit_fast_path(struct drbd_device *device, struct list_head *incoming)
+static void submit_fast_path(struct drbd_device *device, struct waiting_for_act_log *wfa)
 {
 	struct blk_plug plug;
 	struct drbd_request *req, *tmp;
+	struct drbd_peer_request *pr, *pr_tmp;
 
 	blk_start_plug(&plug);
-	list_for_each_entry_safe(req, tmp, incoming, tl_requests) {
+	list_for_each_entry_safe(pr, pr_tmp, &wfa->peer_requests.incoming, w.list) {
+		if (!drbd_al_begin_io_fastpath(pr->peer_device->device, &pr->i))
+			continue;
+
+		__drbd_submit_peer_request(pr);
+	}
+	list_for_each_entry_safe(req, tmp, &wfa->requests.incoming, list) {
 		const int rw = bio_data_dir(req->master_bio);
 
-		if (rw == WRITE /* rw != WRITE should not even end up here! */
-		&& req->private_bio && req->i.size
-		&& !test_bit(AL_SUSPENDED, &device->flags)) {
+		if (rw == WRITE && req->private_bio && req->i.size
+				&& !test_bit(AL_SUSPENDED, &device->flags)) {
 			if (!drbd_al_begin_io_fastpath(device, &req->i))
 				continue;
 
-			req->rq_state |= RQ_IN_ACT_LOG;
-			req->in_actlog_jif = jiffies;
+			drbd_req_in_actlog(req);
 			atomic_dec(&device->ap_actlog_cnt);
 		}
 
-		list_del_init(&req->tl_requests);
-		drbd_send_and_submit(device, req);
+		list_del_init(&req->list);
+		drbd_conflict_submit_write(req);
 	}
 	blk_finish_plug(&plug);
 }
 
+static struct drbd_request *wfa_next_request(struct waiting_for_act_log *wfa)
+{
+	struct list_head *lh = !list_empty(&wfa->requests.more_incoming) ?
+			&wfa->requests.more_incoming : &wfa->requests.incoming;
+	return list_first_entry_or_null(lh, struct drbd_request, list);
+}
+
+static struct drbd_peer_request *wfa_next_peer_request(struct waiting_for_act_log *wfa)
+{
+	struct list_head *lh = !list_empty(&wfa->peer_requests.more_incoming) ?
+			&wfa->peer_requests.more_incoming : &wfa->peer_requests.incoming;
+	return list_first_entry_or_null(lh, struct drbd_peer_request, w.list);
+}
+
 static bool prepare_al_transaction_nonblock(struct drbd_device *device,
-					    struct list_head *incoming,
-					    struct list_head *pending,
-					    struct list_head *later)
+					    struct waiting_for_act_log *wfa)
 {
+	struct drbd_peer_request *peer_req;
 	struct drbd_request *req;
-	int wake = 0;
+	bool made_progress = false;
 	int err;
 
 	spin_lock_irq(&device->al_lock);
-	while ((req = list_first_entry_or_null(incoming, struct drbd_request, tl_requests))) {
+
+	/* Don't even try, if someone has it locked right now. */
+	if (test_bit(__LC_LOCKED, &device->act_log->flags))
+		goto out;
+
+	while ((peer_req = wfa_next_peer_request(wfa))) {
+		if (peer_req->peer_device->connection->cstate[NOW] < C_CONNECTED) {
+			list_move_tail(&peer_req->w.list, &wfa->peer_requests.cleanup);
+			made_progress = true;
+			continue;
+		}
+		err = drbd_al_begin_io_nonblock(device, &peer_req->i);
+		if (err) {
+			if (err != -ENOBUFS && drbd_ratelimit())
+				drbd_err(device, "Unexpected error %d from drbd_al_begin_io_nonblock\n", err);
+			break;
+		}
+		list_move_tail(&peer_req->w.list, &wfa->peer_requests.pending);
+		made_progress = true;
+	}
+	while ((req = wfa_next_request(wfa))) {
+		ktime_aggregate_delta(device, req->start_kt, before_al_begin_io_kt);
 		err = drbd_al_begin_io_nonblock(device, &req->i);
-		if (err == -ENOBUFS)
+		if (err) {
+			if (err != -ENOBUFS && drbd_ratelimit())
+				drbd_err(device, "Unexpected error %d from drbd_al_begin_io_nonblock\n", err);
 			break;
-		if (err == -EBUSY)
-			wake = 1;
-		if (err)
-			list_move_tail(&req->tl_requests, later);
-		else
-			list_move_tail(&req->tl_requests, pending);
+		}
+		list_move_tail(&req->list, &wfa->requests.pending);
+		made_progress = true;
 	}
+ out:
 	spin_unlock_irq(&device->al_lock);
-	if (wake)
-		wake_up(&device->al_wait);
-	return !list_empty(pending);
+	return made_progress;
 }
 
-static void send_and_submit_pending(struct drbd_device *device, struct list_head *pending)
+static void send_and_submit_pending(struct drbd_device *device, struct waiting_for_act_log *wfa)
 {
 	struct blk_plug plug;
-	struct drbd_request *req;
+	struct drbd_request *req, *tmp;
+	struct drbd_peer_request *pr, *pr_tmp;
 
 	blk_start_plug(&plug);
-	while ((req = list_first_entry_or_null(pending, struct drbd_request, tl_requests))) {
-		req->rq_state |= RQ_IN_ACT_LOG;
-		req->in_actlog_jif = jiffies;
+	list_for_each_entry_safe(pr, pr_tmp, &wfa->peer_requests.pending, w.list) {
+		__drbd_submit_peer_request(pr);
+	}
+	list_for_each_entry_safe(req, tmp, &wfa->requests.pending, list) {
+		drbd_req_in_actlog(req);
 		atomic_dec(&device->ap_actlog_cnt);
-		list_del_init(&req->tl_requests);
-		drbd_send_and_submit(device, req);
+		list_del_init(&req->list);
+		drbd_conflict_submit_write(req);
 	}
 	blk_finish_plug(&plug);
 }
 
+/* more: for non-blocking fill-up # of updates in the transaction */
+static bool grab_new_incoming_requests(struct drbd_device *device, struct waiting_for_act_log *wfa, bool more)
+{
+	/* grab new incoming requests */
+	struct list_head *reqs = more ? &wfa->requests.more_incoming : &wfa->requests.incoming;
+	struct list_head *peer_reqs = more ? &wfa->peer_requests.more_incoming : &wfa->peer_requests.incoming;
+	bool found_new = false;
+
+	spin_lock(&device->submit.lock);
+	found_new = !list_empty(&device->submit.writes);
+	list_splice_tail_init(&device->submit.writes, reqs);
+	found_new |= !list_empty(&device->submit.peer_writes);
+	list_splice_tail_init(&device->submit.peer_writes, peer_reqs);
+	spin_unlock(&device->submit.lock);
+
+	return found_new;
+}
+
 void do_submit(struct work_struct *ws)
 {
 	struct drbd_device *device = container_of(ws, struct drbd_device, submit.worker);
-	LIST_HEAD(incoming);	/* from drbd_make_request() */
-	LIST_HEAD(pending);	/* to be submitted after next AL-transaction commit */
-	LIST_HEAD(busy);	/* blocked by resync requests */
+	struct waiting_for_act_log wfa;
+	bool made_progress;
 
-	/* grab new incoming requests */
-	spin_lock_irq(&device->resource->req_lock);
-	list_splice_tail_init(&device->submit.writes, &incoming);
-	spin_unlock_irq(&device->resource->req_lock);
+	wfa_init(&wfa);
+
+	grab_new_incoming_requests(device, &wfa, false);
 
 	for (;;) {
 		DEFINE_WAIT(wait);
 
-		/* move used-to-be-busy back to front of incoming */
-		list_splice_init(&busy, &incoming);
-		submit_fast_path(device, &incoming);
-		if (list_empty(&incoming))
+		submit_fast_path(device, &wfa);
+		if (wfa_lists_empty(&wfa, incoming))
 			break;
 
 		for (;;) {
+			/*
+			 * We put ourselves on device->al_wait, then check if
+			 * we can need to actually sleep and wait for someone
+			 * else to make progress.
+			 *
+			 * We need to sleep if we cannot activate enough
+			 * activity log extents for even one single request.
+			 * That would mean that all (peer-)requests in our
+			 * incoming lists target "cold" activity log extents,
+			 * all activity log extent slots are have on-going
+			 * in-flight IO (are "hot"), and no idle or free slot
+			 * is available.
+			 *
+			 * prepare_to_wait() can internally cause a wake_up()
+			 * as well, though, so this may appear to busy-loop
+			 * a couple times, but should settle down quickly.
+			 *
+			 * When application requests make sufficient progress,
+			 * some refcount on some extent will eventually drop to
+			 * zero, we will be woken up, and can try to move that
+			 * now idle extent to "cold", and recycle its slot for
+			 * one of the extents we'd like to become hot.
+			 */
 			prepare_to_wait(&device->al_wait, &wait, TASK_UNINTERRUPTIBLE);
 
-			list_splice_init(&busy, &incoming);
-			prepare_al_transaction_nonblock(device, &incoming, &pending, &busy);
-			if (!list_empty(&pending))
+			made_progress = prepare_al_transaction_nonblock(device, &wfa);
+			if (made_progress)
 				break;
 
 			schedule();
@@ -1551,15 +2595,12 @@ void do_submit(struct work_struct *ws)
 			 * effectively blocking all new requests until we made
 			 * at least _some_ progress with what we currently have.
 			 */
-			if (!list_empty(&incoming))
+			if (!wfa_lists_empty(&wfa, incoming))
 				continue;
 
-			/* Nothing moved to pending, but nothing left
-			 * on incoming: all moved to busy!
-			 * Grab new and iterate. */
-			spin_lock_irq(&device->resource->req_lock);
-			list_splice_tail_init(&device->submit.writes, &incoming);
-			spin_unlock_irq(&device->resource->req_lock);
+			/* Nothing moved to pending, but nothing left on
+			 * incoming. Grab new and iterate. */
+			grab_new_incoming_requests(device, &wfa, false);
 		}
 		finish_wait(&device->al_wait, &wait);
 
@@ -1567,81 +2608,215 @@ void do_submit(struct work_struct *ws)
 		 * had been processed, skip ahead to commit, and iterate
 		 * without splicing in more incoming requests from upper layers.
 		 *
-		 * Else, if all incoming have been processed,
-		 * they have become either "pending" (to be submitted after
-		 * next transaction commit) or "busy" (blocked by resync).
+		 * Else, if all incoming have been processed, they have become
+		 * "pending" (to be submitted after next transaction commit).
 		 *
 		 * Maybe more was queued, while we prepared the transaction?
 		 * Try to stuff those into this transaction as well.
 		 * Be strictly non-blocking here,
 		 * we already have something to commit.
 		 *
-		 * Commit if we don't make any more progres.
+		 * Commit as soon as we don't make any more progress.
 		 */
 
-		while (list_empty(&incoming)) {
-			LIST_HEAD(more_pending);
-			LIST_HEAD(more_incoming);
-			bool made_progress;
-
+		while (wfa_lists_empty(&wfa, incoming)) {
 			/* It is ok to look outside the lock,
 			 * it's only an optimization anyways */
-			if (list_empty(&device->submit.writes))
+			if (list_empty(&device->submit.writes) &&
+			    list_empty(&device->submit.peer_writes))
 				break;
 
-			spin_lock_irq(&device->resource->req_lock);
-			list_splice_tail_init(&device->submit.writes, &more_incoming);
-			spin_unlock_irq(&device->resource->req_lock);
-
-			if (list_empty(&more_incoming))
+			if (!grab_new_incoming_requests(device, &wfa, true))
 				break;
 
-			made_progress = prepare_al_transaction_nonblock(device, &more_incoming, &more_pending, &busy);
+			made_progress = prepare_al_transaction_nonblock(device, &wfa);
 
-			list_splice_tail_init(&more_pending, &pending);
-			list_splice_tail_init(&more_incoming, &incoming);
+			wfa_splice_tail_init(&wfa, more_incoming, incoming);
 			if (!made_progress)
 				break;
 		}
+		if (!list_empty(&wfa.peer_requests.cleanup))
+			drbd_cleanup_peer_requests_wfa(device, &wfa.peer_requests.cleanup);
 
 		drbd_al_begin_io_commit(device);
-		send_and_submit_pending(device, &pending);
+
+		send_and_submit_pending(device, &wfa);
+	}
+}
+
+static bool drbd_reject_write_early(struct drbd_device *device, struct bio *bio)
+{
+	struct drbd_resource *resource = device->resource;
+
+	/* If you "mount -o ro", then later "mount -o remount,rw", you can end
+	 * up with a DRBD "Secondary" receiving WRITE requests from the VFS.
+	 * We cannot have that. */
+
+	if (bio_data_dir(bio) == READ)
+		return false;
+
+	if (resource->role[NOW] != R_PRIMARY) {
+		/* You can fsync() on an O_RDONLY fd. Only be noisy
+		 * if there is data.  Ratelimit on per device "unspec"
+		 * ratelimit state before kmalloc / adding the specific
+		 * openers hint.
+		 */
+		if (bio_has_data(bio) && drbd_device_ratelimit(device, GENERIC)) {
+			char *buf = kmalloc(128, __GFP_NORETRY);
+
+			if (buf)
+				youngest_and_oldest_opener_to_str(device, buf, 128);
+			drbd_err(device,
+				"Rejected WRITE request, not in Primary role.%s\n", buf ?: "");
+			kfree(buf);
+		}
+		return true;
+	} else if (device->open_cnt == 0) {
+		drbd_err_ratelimit(device, "WRITE request, but open_cnt == 0!\n");
+	} else if (!device->writable && bio_has_data(bio)) {
+		/*
+		 * If the resource was (temporarily, auto) promoted,
+		 * a remount,rw may have succeeded without marking the device
+		 * open_cnt as "writable".  Once we let writes through, we need
+		 * _all_ openers to release(), before we attempt to auto-demote
+		 * again, so we mark it writable here.  Grab the open_release
+		 * mutex to protect against races with new openers.
+		 */
+		mutex_lock(&resource->open_release);
+		drbd_info(device, "open_cnt:%d, implicitly promoted to writable\n",
+			device->open_cnt);
+		device->writable = true;
+		mutex_unlock(&resource->open_release);
+	}
+	return false;
+}
+
+/* Check if bio is "bad", likely to be rejected by lower layers or peers:
+ * Must not be too large, must not be unaligned.
+ */
+static bool bio_bad(struct drbd_device *device, struct bio *bio)
+{
+	unsigned int bss_mask = queue_logical_block_size(device->rq_queue) / SECTOR_SIZE - 1;
+	unsigned int bs_mask = queue_logical_block_size(device->rq_queue) - 1;
+	unsigned long long sector = bio->bi_iter.bi_sector;
+	unsigned int size = bio->bi_iter.bi_size;
+
+	if (size > DRBD_MAX_BATCH_BIO_SIZE || (size & bs_mask) || (sector & bss_mask)) {
+		char comm[TASK_COMM_LEN];
+
+		get_task_comm(comm, current);
+		drbd_warn(device, "bad bio: %llu +%u 0x%x submitted by %s[%u]\n",
+			sector, size, bio->bi_opf, comm, task_pid_nr(current));
+		return true;
 	}
+
+	return false;
 }
 
+/* drbd_submit_bio() - entry point for data into DRBD
+ *
+ * Request handling flow:
+ *
+ *                                    drbd_submit_bio
+ *                                           |
+ *                                           v          wait for AL
+ * do_retry -----------------------> __drbd_make_request --------> drbd_queue_write
+ *     ^                                     |                          |
+ *     |                                     |                         ...
+ *     |                                     |                          |
+ *     |                                     |                          v    AL extent active
+ *     |     drbd_do_submit_conflict --------+                     do_submit ----------------+
+ *     |                ^                    |                          |                    |
+ *    ...               |                    |                          v                    v
+ *     |               ...                   |               send_and_submit_pending   submit_fast_path
+ *     |                |                    v                          |                    |
+ *     |                +----------- drbd_conflict_submit_write <-------+--------------------+
+ *     |                  conflict           |
+ *     |                                     v
+ * drbd_restart_request <----------- drbd_send_and_submit
+ *                      RQ_POSTPONED         |
+ *                                           v
+ *                                   Request state machine
+ */
 void drbd_submit_bio(struct bio *bio)
 {
 	struct drbd_device *device = bio->bi_bdev->bd_disk->private_data;
+#ifdef CONFIG_DRBD_TIMING_STATS
+	ktime_t start_kt;
+#endif
+	unsigned long start_jif;
+
+	if (drbd_reject_write_early(device, bio)) {
+		bio->bi_status = BLK_STS_IOERR;
+		bio_endio(bio);
+		return;
+	}
 
 	bio = bio_split_to_limits(bio);
 	if (!bio)
 		return;
 
-	/*
-	 * what we "blindly" assume:
+	if (device->cached_err_io || bio_bad(device, bio)) {
+		bio->bi_status = BLK_STS_IOERR;
+		bio_endio(bio);
+		return;
+	}
+
+	/* This is both an optimization: READ of size 0, nothing to do
+	 * and a workaround: (older) ZFS explodes on size zero reads, see
+	 * https://github.com/zfsonlinux/zfs/issues/8379
+	 * Actually don't do anything for size zero bios.
+	 * Add a "WARN_ONCE", so we can tell the caller to stop doing this.
 	 */
-	D_ASSERT(device, IS_ALIGNED(bio->bi_iter.bi_size, 512));
+	if (bio_op(bio) == REQ_OP_READ && bio->bi_iter.bi_size == 0) {
+		WARN_ONCE(1, "size zero read from upper layers");
+		bio_endio(bio);
+		return;
+	}
+
+	ktime_get_accounting(start_kt);
+	start_jif = jiffies;
+
+	__drbd_make_request(device, bio, start_kt, start_jif);
+}
+
+static unsigned long time_min_in_future(unsigned long now,
+		unsigned long t1, unsigned long t2)
+{
+	bool t1_in_future = time_after(t1, now);
+	bool t2_in_future = time_after(t2, now);
+
+	/* Ensure that we never return a time in the past. */
+	t1 = t1_in_future ? t1 : now;
+	t2 = t2_in_future ? t2 : now;
 
-	inc_ap_bio(device);
-	__drbd_make_request(device, bio);
+	if (!t1_in_future)
+		return t2;
+
+	if (!t2_in_future)
+		return t1;
+
+	return time_after(t1, t2) ? t2 : t1;
 }
 
 static bool net_timeout_reached(struct drbd_request *net_req,
-		struct drbd_connection *connection,
+		struct drbd_peer_device *peer_device,
 		unsigned long now, unsigned long ent,
 		unsigned int ko_count, unsigned int timeout)
 {
-	struct drbd_device *device = net_req->device;
+	struct drbd_connection *connection = peer_device->connection;
+	int peer_node_id = peer_device->node_id;
+	unsigned long pre_send_jif = net_req->pre_send_jif[peer_node_id];
 
-	if (!time_after(now, net_req->pre_send_jif + ent))
+	if (!time_after(now, pre_send_jif + ent))
 		return false;
 
 	if (time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent))
 		return false;
 
-	if (net_req->rq_state & RQ_NET_PENDING) {
-		drbd_warn(device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
-			jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
+	if (net_req->net_rq_state[peer_node_id] & RQ_NET_PENDING) {
+		drbd_warn(peer_device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
+			jiffies_to_msecs(now - pre_send_jif), ko_count, timeout);
 		return true;
 	}
 
@@ -1650,9 +2825,12 @@ static bool net_timeout_reached(struct drbd_request *net_req,
 	 * Check if we sent the barrier already.  We should not blame the peer
 	 * for being unresponsive, if we did not even ask it yet. */
 	if (net_req->epoch == connection->send.current_epoch_nr) {
-		drbd_warn(device,
-			"We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread blocked?\n",
-			jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
+		/* It is OK for the barrier to be delayed for a long time for a
+		 * suspended request. */
+		if (!(net_req->local_rq_state & RQ_COMPLETION_SUSP))
+			drbd_warn(peer_device,
+					"We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread blocked?\n",
+					jiffies_to_msecs(now - pre_send_jif), ko_count, timeout);
 		return false;
 	}
 
@@ -1673,7 +2851,7 @@ static bool net_timeout_reached(struct drbd_request *net_req,
 	 * barrier packet is relevant enough.
 	 */
 	if (time_after(now, connection->send.last_sent_barrier_jif + ent)) {
-		drbd_warn(device, "Remote failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
+		drbd_warn(peer_device, "Remote failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
 			connection->send.last_sent_barrier_jif, now,
 			jiffies_to_msecs(now - connection->send.last_sent_barrier_jif), ko_count, timeout);
 		return true;
@@ -1690,7 +2868,7 @@ static bool net_timeout_reached(struct drbd_request *net_req,
  * - the connection was established (resp. disk was attached)
  *   for longer than the timeout already.
  * Note that for 32bit jiffies and very stable connections/disks,
- * we may have a wrap around, which is catched by
+ * we may have a wrap around, which is caught by
  *   !time_in_range(now, last_..._jif, last_..._jif + timeout).
  *
  * Side effect: once per 32bit wrap-around interval, which means every
@@ -1700,92 +2878,199 @@ static bool net_timeout_reached(struct drbd_request *net_req,
 
 void request_timer_fn(struct timer_list *t)
 {
-	struct drbd_device *device = timer_container_of(device, t,
-							request_timer);
-	struct drbd_connection *connection = first_peer_device(device)->connection;
-	struct drbd_request *req_read, *req_write, *req_peer; /* oldest request */
-	struct net_conf *nc;
-	unsigned long oldest_submit_jif;
-	unsigned long ent = 0, dt = 0, et, nt; /* effective timeout = ko_count * timeout */
-	unsigned long now;
-	unsigned int ko_count = 0, timeout = 0;
+	struct drbd_device *device = timer_container_of(device, t, request_timer);
+	struct drbd_resource *resource = device->resource;
+	struct drbd_connection *connection;
+	struct drbd_request *req_read, *req_write;
+	unsigned long oldest_submit_jif, irq_flags;
+	unsigned long disk_timeout = 0, effective_timeout = 0, now = jiffies, next_trigger_time = now;
+	bool restart_timer = false, io_error = false;
+	unsigned long timeout_peers = 0;
+	int node_id;
 
 	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	if (nc && device->state.conn >= C_WF_REPORT_PARAMS) {
-		ko_count = nc->ko_count;
-		timeout = nc->timeout;
-	}
-
 	if (get_ldev(device)) { /* implicit state.disk >= D_INCONSISTENT */
-		dt = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10;
+		disk_timeout = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10;
 		put_ldev(device);
 	}
 	rcu_read_unlock();
 
+	/* FIXME right now, this basically does a full transfer log walk *every time* */
+	read_lock_irq(&resource->state_rwlock);
+	if (disk_timeout) {
+		unsigned long write_pre_submit_jif = 0, read_pre_submit_jif = 0;
+
+		spin_lock(&device->pending_completion_lock); /* local irq already disabled */
+		req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pending_local);
+		req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pending_local);
+		spin_unlock(&device->pending_completion_lock);
+
+		if (req_write)
+			write_pre_submit_jif = req_write->pre_submit_jif;
+		if (req_read)
+			read_pre_submit_jif = req_read->pre_submit_jif;
+		oldest_submit_jif =
+			(req_write && req_read)
+			? (time_before(write_pre_submit_jif, read_pre_submit_jif)
+			  ? write_pre_submit_jif : read_pre_submit_jif)
+			: req_write ? write_pre_submit_jif
+			: req_read ? read_pre_submit_jif : now;
+
+		if (device->disk_state[NOW] > D_FAILED) {
+			effective_timeout = min_not_zero(effective_timeout, disk_timeout);
+			next_trigger_time = time_min_in_future(now,
+					next_trigger_time, oldest_submit_jif + disk_timeout);
+			restart_timer = true;
+		}
 
-	ent = timeout * HZ/10 * ko_count;
-	et = min_not_zero(dt, ent);
+		if (time_after(now, oldest_submit_jif + disk_timeout) &&
+		    !time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + disk_timeout))
+			io_error = true;
+	}
+	for_each_connection(connection, resource) {
+		struct drbd_peer_device *peer_device = conn_peer_device(connection, device->vnr);
+		struct net_conf *nc;
+		struct drbd_request *req;
+		unsigned long effective_net_timeout = 0;
+		unsigned long pre_send_jif = now;
+		unsigned int ko_count = 0, timeout = 0;
 
-	if (!et)
-		return; /* Recurring timer stopped */
+		rcu_read_lock();
+		nc = rcu_dereference(connection->transport.net_conf);
+		if (nc) {
+			/* effective timeout = ko_count * timeout */
+			if (connection->cstate[NOW] == C_CONNECTED) {
+				ko_count = nc->ko_count;
+				timeout = nc->timeout;
+				effective_net_timeout = timeout * HZ/10 * ko_count;
+			}
+		}
+		rcu_read_unlock();
 
-	now = jiffies;
-	nt = now + et;
+		/* This connection is not established,
+		 * or has the effective timeout disabled.
+		 * no timer restart needed (for this connection). */
+		if (!effective_net_timeout)
+			continue;
 
-	spin_lock_irq(&device->resource->req_lock);
-	req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pending_local);
-	req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pending_local);
+		/* maybe the oldest request waiting for the peer is in fact still
+		 * blocking in tcp sendmsg.  That's ok, though, that's handled via the
+		 * socket send timeout, requesting a ping, and bumping ko-count in
+		 * drbd_stream_send_timed_out().
+		 */
 
-	/* maybe the oldest request waiting for the peer is in fact still
-	 * blocking in tcp sendmsg.  That's ok, though, that's handled via the
-	 * socket send timeout, requesting a ping, and bumping ko-count in
-	 * we_should_drop_the_connection().
-	 */
+		/* check the oldest request we did successfully sent,
+		 * but which is still waiting for an ACK. */
+		req = connection->req_ack_pending;
+
+		/* If we don't have such request (e.g. protocol A)
+		 * check the oldest request which is still waiting on its epoch
+		 * closing barrier ack. */
+		if (!req) {
+			req = connection->req_not_net_done;
+
+			/* If we did not send the request yet then pre_send_jif
+			 * is not set. Treat this the same as when there are no
+			 * requests pending. */
+			if (req && !(req->net_rq_state[connection->peer_node_id] & RQ_NET_SENT))
+				req = NULL;
+		}
 
-	/* check the oldest request we did successfully sent,
-	 * but which is still waiting for an ACK. */
-	req_peer = connection->req_ack_pending;
+		if (req)
+			pre_send_jif = req->pre_send_jif[connection->peer_node_id];
+
+		effective_timeout = min_not_zero(effective_timeout, effective_net_timeout);
+		next_trigger_time = time_min_in_future(now,
+				next_trigger_time, pre_send_jif + effective_net_timeout);
+		/* Restart the timer, even if there are no pending requests at all.
+		 * We currently do not re-arm from the submit path. */
+		restart_timer = true;
+
+		/* We have one timer per "device",
+		 * but the "oldest" request is per "connection".
+		 * Evaluate the oldest peer request only in one timer! */
+		if (req == NULL || req->device != device)
+			continue;
+
+		if (net_timeout_reached(req, peer_device, now, effective_net_timeout, ko_count, timeout)) {
+			dynamic_drbd_dbg(peer_device, "Request at %llus+%u timed out\n",
+					(unsigned long long) req->i.sector,
+					req->i.size);
+			timeout_peers |= NODE_MASK(connection->peer_node_id);
+		}
+	}
+	read_unlock_irq(&resource->state_rwlock);
 
-	/* if we don't have such request (e.g. protocoll A)
-	 * check the oldest requests which is still waiting on its epoch
-	 * closing barrier ack. */
-	if (!req_peer)
-		req_peer = connection->req_not_net_done;
+	if (io_error) {
+		drbd_warn(device, "Local backing device failed to meet the disk-timeout\n");
+		drbd_handle_io_error(device, DRBD_FORCE_DETACH);
+	}
 
-	/* evaluate the oldest peer request only in one timer! */
-	if (req_peer && req_peer->device != device)
-		req_peer = NULL;
+	BUILD_BUG_ON(sizeof(timeout_peers) * 8 < DRBD_NODE_ID_MAX);
+	for_each_set_bit(node_id, &timeout_peers, DRBD_NODE_ID_MAX) {
+		connection = drbd_get_connection_by_node_id(resource, node_id);
+		if (!connection)
+			continue;
+		begin_state_change(resource, &irq_flags, CS_VERBOSE | CS_HARD);
+		__change_cstate(connection, C_TIMEOUT);
+		end_state_change(resource, &irq_flags, "timeout");
+		kref_put(&connection->kref, drbd_destroy_connection);
+	}
 
-	/* do we have something to evaluate? */
-	if (req_peer == NULL && req_write == NULL && req_read == NULL)
-		goto out;
+	if (restart_timer) {
+		next_trigger_time = time_min_in_future(now, next_trigger_time, now + effective_timeout);
+		mod_timer(&device->request_timer, next_trigger_time);
+	}
+}
 
-	oldest_submit_jif =
-		(req_write && req_read)
-		? ( time_before(req_write->pre_submit_jif, req_read->pre_submit_jif)
-		  ? req_write->pre_submit_jif : req_read->pre_submit_jif )
-		: req_write ? req_write->pre_submit_jif
-		: req_read ? req_read->pre_submit_jif : now;
+/**
+ * drbd_handle_io_error_: Handle the on_io_error setting, should be called from all io completion handlers
+ * @device: DRBD device.
+ * @df:     Detach flags indicating the kind of IO that failed.
+ * @where:  Calling function name.
+ */
+void drbd_handle_io_error_(struct drbd_device *device,
+	enum drbd_force_detach_flags df, const char *where)
+{
+	unsigned long flags;
+	enum drbd_io_error_p ep;
 
-	if (ent && req_peer && net_timeout_reached(req_peer, connection, now, ent, ko_count, timeout))
-		_conn_request_state(connection, NS(conn, C_TIMEOUT), CS_VERBOSE | CS_HARD);
+	write_lock_irqsave(&device->resource->state_rwlock, flags);
 
-	if (dt && oldest_submit_jif != now &&
-		 time_after(now, oldest_submit_jif + dt) &&
-		!time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + dt)) {
-		drbd_warn(device, "Local backing device failed to meet the disk-timeout\n");
-		__drbd_chk_io_error(device, DRBD_FORCE_DETACH);
+	rcu_read_lock();
+	ep = rcu_dereference(device->ldev->disk_conf)->on_io_error;
+	rcu_read_unlock();
+	switch (ep) {
+	case EP_PASS_ON: /* FIXME would this be better named "Ignore"? */
+		if (df == DRBD_READ_ERROR ||  df == DRBD_WRITE_ERROR) {
+			if (drbd_device_ratelimit(device, BACKEND))
+				drbd_err(device, "Local IO failed in %s.\n", where);
+			if (device->disk_state[NOW] > D_INCONSISTENT) {
+				begin_state_change_locked(device->resource, CS_HARD);
+				__change_disk_state(device, D_INCONSISTENT);
+				end_state_change_locked(device->resource, "local-io-error");
+			}
+			break;
+		}
+		fallthrough;	/* for DRBD_META_IO_ERROR or DRBD_FORCE_DETACH */
+	case EP_DETACH:
+	case EP_CALL_HELPER:
+		/* Force-detach is not really an IO error, but rather a
+		 * desperate measure to try to deal with a completely
+		 * unresponsive lower level IO stack.
+		 * Still it should be treated as a WRITE error.
+		 */
+		if (df == DRBD_FORCE_DETACH)
+			set_bit(FORCE_DETACH, &device->flags);
+		if (device->disk_state[NOW] > D_FAILED) {
+			begin_state_change_locked(device->resource, CS_HARD);
+			__change_disk_state(device, D_FAILED);
+			end_state_change_locked(device->resource, "local-io-error");
+			drbd_err(device,
+				"Local IO failed in %s. Detaching...\n", where);
+		}
+		break;
 	}
 
-	/* Reschedule timer for the nearest not already expired timeout.
-	 * Fallback to now + min(effective network timeout, disk timeout). */
-	ent = (ent && req_peer && time_before(now, req_peer->pre_send_jif + ent))
-		? req_peer->pre_send_jif + ent : now + et;
-	dt = (dt && oldest_submit_jif != now && time_before(now, oldest_submit_jif + dt))
-		? oldest_submit_jif + dt : now + et;
-	nt = time_before(ent, dt) ? ent : dt;
-out:
-	spin_unlock_irq(&device->resource->req_lock);
-	mod_timer(&device->request_timer, nt);
+	write_unlock_irqrestore(&device->resource->state_rwlock, flags);
 }
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index 9ae860e7591b3..e5770401cb7a8 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -64,36 +64,31 @@
  */
 
 enum drbd_req_event {
-	CREATED,
-	TO_BE_SENT,
 	TO_BE_SUBMITTED,
 
-	/* XXX yes, now I am inconsistent...
-	 * these are not "events" but "actions"
-	 * oh, well... */
-	QUEUE_FOR_NET_WRITE,
-	QUEUE_FOR_NET_READ,
-	QUEUE_FOR_SEND_OOS,
+	NEW_NET_READ,
+	NEW_NET_WRITE,
+	NEW_NET_OOS,
+	READY_FOR_NET,
+	SKIP_OOS,
 
-	/* An empty flush is queued as P_BARRIER,
-	 * which will cause it to complete "successfully",
-	 * even if the local disk flush failed.
+	/* For an empty flush, mark that a corresponding barrier has been sent
+	 * to this peer. This causes it to complete "successfully", even if the
+	 * local disk flush failed.
 	 *
 	 * Just like "real" requests, empty flushes (blkdev_issue_flush()) will
 	 * only see an error if neither local nor remote data is reachable. */
-	QUEUE_AS_DRBD_BARRIER,
+	BARRIER_SENT,
 
 	SEND_CANCELED,
 	SEND_FAILED,
 	HANDED_OVER_TO_NETWORK,
 	OOS_HANDED_TO_NETWORK,
-	CONNECTION_LOST_WHILE_PENDING,
-	READ_RETRY_REMOTE_CANCELED,
+	CONNECTION_LOST,
+	CONNECTION_LOST_WHILE_SUSPENDED,
 	RECV_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER_AND_SIS, /* and set_in_sync */
-	CONFLICT_RESOLVED,
-	POSTPONE_WRITE,
 	NEG_ACKED,
 	BARRIER_ACKED, /* in protocol A and B */
 	DATA_RECEIVED, /* (remote read) */
@@ -107,82 +102,93 @@ enum drbd_req_event {
 
 	ABORT_DISK_IO,
 	RESEND,
-	FAIL_FROZEN_DISK_IO,
-	RESTART_FROZEN_DISK_IO,
+	CANCEL_SUSPENDED_IO,
+	COMPLETION_RESUMED,
 	NOTHING,
 };
 
-/* encoding of request states for now.  we don't actually need that many bits.
- * we don't need to do atomic bit operations either, since most of the time we
- * need to look at the connection state and/or manipulate some lists at the
- * same time, so we should hold the request lock anyways.
+/*
+ * Encoding of request states. Modifications are protected by rq_lock. We don't
+ * do atomic bit operations.
  */
 enum drbd_req_state_bits {
-	/* 3210
-	 * 0000: no local possible
-	 * 0001: to be submitted
-	 *    UNUSED, we could map: 011: submitted, completion still pending
-	 * 0110: completed ok
-	 * 0010: completed with error
-	 * 1001: Aborted (before completion)
-	 * 1x10: Aborted and completed -> free
-	 */
-	__RQ_LOCAL_PENDING,
-	__RQ_LOCAL_COMPLETED,
-	__RQ_LOCAL_OK,
-	__RQ_LOCAL_ABORTED,
-
-	/* 87654
-	 * 00000: no network possible
-	 * 00001: to be send
-	 * 00011: to be send, on worker queue
-	 * 00101: sent, expecting recv_ack (B) or write_ack (C)
-	 * 11101: sent,
-	 *        recv_ack (B) or implicit "ack" (A),
-	 *        still waiting for the barrier ack.
-	 *        master_bio may already be completed and invalidated.
-	 * 11100: write acked (C),
-	 *        data received (for remote read, any protocol)
-	 *        or finally the barrier ack has arrived (B,A)...
-	 *        request can be freed
-	 * 01100: neg-acked (write, protocol C)
-	 *        or neg-d-acked (read, any protocol)
-	 *        or killed from the transfer log
-	 *        during cleanup after connection loss
-	 *        request can be freed
-	 * 01000: canceled or send failed...
-	 *        request can be freed
+	/*
+	 * Here are the possible combinations of the core net flags pending, pending-oos,
+	 * queued, ready, sent, done, ok.
+	 *
+	 * <none>:
+	 *   No network required, or not yet processed.
+	 * pending,queued:
+	 *   To be sent, must not be processed yet.
+	 * pending,queued,ready:
+	 *   To be sent, processing allowed.
+	 * pending,ready,sent:
+	 *   Sent, expecting P_RECV_ACK (B) or P_WRITE_ACK (C).
+	 * queued,ready,ok:
+	 *   P_RECV_ACK (B) or P_WRITE_ACK (C) received before request marked
+	 *   as having been sent.
+	 * ready,sent,ok:
+	 *   Sent, implicit "ack" (A), P_RECV_ACK (B) or P_WRITE_ACK (C) received.
+	 *   Still waiting for the barrier ack.
+	 *   master_bio may already be completed and invalidated.
+	 * pending:
+	 *   Intended for this peer, but connection lost before processing
+	 *   allowed.
+	 * pending,ready:
+	 *   Intended for this peer, but connection lost. If
+	 *   IO is suspended, it will stay in this state until the connection
+	 *   is restored or IO is resumed.
+	 * ready,sent,done,ok:
+	 *   Data received (for remote read, any protocol),
+	 *   or finally the barrier ack has arrived.
+	 * ready,sent,done:
+	 *   Received P_NEG_ACK for write (protocol C, or we are SyncSource),
+	 *   or P_NEG_DREPLY for read (any protocol).
+	 *   Or cleaned up after connection loss after send.
+	 * pending-oos,queued,done:
+	 *   P_OUT_OF_SYNC to be sent, must not be processed yet.
+	 * pending-oos,queued,ready,done:
+	 *   P_OUT_OF_SYNC to be sent, processing allowed.
+	 * queued,ready,done:
+	 *   P_OUT_OF_SYNC was intended, but skipped.
+	 * done:
+	 *   P_OUT_OF_SYNC was intended, but connection lost before processing
+	 *   allowed.
+	 * ready,done:
+	 *   P_OUT_OF_SYNC sent.
+	 *   Or cleaned up after connection loss, either before send or when
+	 *   only P_OUT_OF_SYNC was intended.
 	 */
 
-	/* if "SENT" is not set, yet, this can still fail or be canceled.
-	 * if "SENT" is set already, we still wait for an Ack packet.
-	 * when cleared, the master_bio may be completed.
-	 * in (B,A) the request object may still linger on the transaction log
-	 * until the corresponding barrier ack comes in */
+	/* Pending some network interaction towards the peer apart from
+	 * barriers or P_OUT_OF_SYNC.
+	 * If "sent" is not yet set, this can still fail or be canceled.
+	 * While set, the master_bio may not be completed. */
 	__RQ_NET_PENDING,
 
-	/* If it is QUEUED, and it is a WRITE, it is also registered in the
-	 * transfer log. Currently we need this flag to avoid conflicts between
-	 * worker canceling the request and tl_clear_barrier killing it from
-	 * transfer log.  We should restructure the code so this conflict does
-	 * no longer occur. */
+	/* Pending send of P_OUT_OF_SYNC */
+	__RQ_NET_PENDING_OOS,
+
+	/* The sender might store pointers to it */
 	__RQ_NET_QUEUED,
 
-	/* well, actually only "handed over to the network stack".
-	 *
-	 * TODO can potentially be dropped because of the similar meaning
-	 * of RQ_NET_SENT and ~RQ_NET_QUEUED.
-	 * however it is not exactly the same. before we drop it
-	 * we must ensure that we can tell a request with network part
-	 * from a request without, regardless of what happens to it. */
+	/* Ready for processing by the sender */
+	__RQ_NET_READY,
+
+	/* Well, actually only "handed over to the network stack". */
 	__RQ_NET_SENT,
 
-	/* when set, the request may be freed (if RQ_NET_QUEUED is clear).
-	 * basically this means the corresponding P_BARRIER_ACK was received */
+	/* When set, the data stage is done, as far as interaction with this
+	 * peer is concerned. Basically this means the corresponding
+	 * P_BARRIER_ACK was received. */
 	__RQ_NET_DONE,
 
-	/* whether or not we know (C) or pretend (B,A) that the write
-	 * was successfully written on the peer.
+	/* Set when the request was successful. That is, the corresponding
+	 * condition is fulfilled:
+	 * - The write was sent (A)
+	 * - Receipt of the write was acknowledged (B)
+	 * - The write was successfully written on the peer (C)
+	 * - Read data was received
 	 */
 	__RQ_NET_OK,
 
@@ -192,6 +198,29 @@ enum drbd_req_state_bits {
 	/* keep this last, its for the RQ_NET_MASK */
 	__RQ_NET_MAX,
 
+	/* We expect a receive ACK (wire proto B) */
+	__RQ_EXP_RECEIVE_ACK,
+
+	/* We expect a write ACK (wite proto C) */
+	__RQ_EXP_WRITE_ACK,
+
+	/* waiting for a barrier ack, did an extra kref_get */
+	__RQ_EXP_BARR_ACK,
+
+	/* 4321
+	 * 0000: no local possible
+	 * 0001: to be submitted
+	 *    UNUSED, we could map: 011: submitted, completion still pending
+	 * 0110: completed ok
+	 * 0010: completed with error
+	 * 1001: Aborted (before completion)
+	 * 1x10: Aborted and completed -> free
+	 */
+	__RQ_LOCAL_PENDING,
+	__RQ_LOCAL_COMPLETED,
+	__RQ_LOCAL_OK,
+	__RQ_LOCAL_ABORTED,
+
 	/* Set when this is a write, clear for a read */
 	__RQ_WRITE,
 	__RQ_WSAME,
@@ -212,26 +241,11 @@ enum drbd_req_state_bits {
 	/* would have been completed,
 	 * but was not, because of drbd_suspended() */
 	__RQ_COMPLETION_SUSP,
-
-	/* We expect a receive ACK (wire proto B) */
-	__RQ_EXP_RECEIVE_ACK,
-
-	/* We expect a write ACK (wite proto C) */
-	__RQ_EXP_WRITE_ACK,
-
-	/* waiting for a barrier ack, did an extra kref_get */
-	__RQ_EXP_BARR_ACK,
 };
-
-#define RQ_LOCAL_PENDING   (1UL << __RQ_LOCAL_PENDING)
-#define RQ_LOCAL_COMPLETED (1UL << __RQ_LOCAL_COMPLETED)
-#define RQ_LOCAL_OK        (1UL << __RQ_LOCAL_OK)
-#define RQ_LOCAL_ABORTED   (1UL << __RQ_LOCAL_ABORTED)
-
-#define RQ_LOCAL_MASK      ((RQ_LOCAL_ABORTED << 1)-1)
-
 #define RQ_NET_PENDING     (1UL << __RQ_NET_PENDING)
+#define RQ_NET_PENDING_OOS (1UL << __RQ_NET_PENDING_OOS)
 #define RQ_NET_QUEUED      (1UL << __RQ_NET_QUEUED)
+#define RQ_NET_READY       (1UL << __RQ_NET_READY)
 #define RQ_NET_SENT        (1UL << __RQ_NET_SENT)
 #define RQ_NET_DONE        (1UL << __RQ_NET_DONE)
 #define RQ_NET_OK          (1UL << __RQ_NET_OK)
@@ -239,6 +253,18 @@ enum drbd_req_state_bits {
 
 #define RQ_NET_MASK        (((1UL << __RQ_NET_MAX)-1) & ~RQ_LOCAL_MASK)
 
+#define RQ_EXP_RECEIVE_ACK (1UL << __RQ_EXP_RECEIVE_ACK)
+#define RQ_EXP_WRITE_ACK   (1UL << __RQ_EXP_WRITE_ACK)
+#define RQ_EXP_BARR_ACK    (1UL << __RQ_EXP_BARR_ACK)
+
+#define RQ_LOCAL_PENDING   (1UL << __RQ_LOCAL_PENDING)
+#define RQ_LOCAL_COMPLETED (1UL << __RQ_LOCAL_COMPLETED)
+#define RQ_LOCAL_OK        (1UL << __RQ_LOCAL_OK)
+#define RQ_LOCAL_ABORTED   (1UL << __RQ_LOCAL_ABORTED)
+
+#define RQ_LOCAL_MASK      \
+	(RQ_LOCAL_ABORTED | RQ_LOCAL_OK | RQ_LOCAL_COMPLETED | RQ_LOCAL_PENDING)
+
 #define RQ_WRITE           (1UL << __RQ_WRITE)
 #define RQ_WSAME           (1UL << __RQ_WSAME)
 #define RQ_UNMAP           (1UL << __RQ_UNMAP)
@@ -247,14 +273,25 @@ enum drbd_req_state_bits {
 #define RQ_UNPLUG          (1UL << __RQ_UNPLUG)
 #define RQ_POSTPONED	   (1UL << __RQ_POSTPONED)
 #define RQ_COMPLETION_SUSP (1UL << __RQ_COMPLETION_SUSP)
-#define RQ_EXP_RECEIVE_ACK (1UL << __RQ_EXP_RECEIVE_ACK)
-#define RQ_EXP_WRITE_ACK   (1UL << __RQ_EXP_WRITE_ACK)
-#define RQ_EXP_BARR_ACK    (1UL << __RQ_EXP_BARR_ACK)
 
-/* For waking up the frozen transfer log mod_req() has to return if the request
-   should be counted in the epoch object*/
-#define MR_WRITE       1
-#define MR_READ        2
+
+/* these flags go into local_rq_state,
+ * orhter flags go into their respective net_rq_state[idx] */
+#define RQ_STATE_0_MASK	\
+	(RQ_LOCAL_MASK  |\
+	 RQ_WRITE       |\
+	 RQ_WSAME       |\
+	 RQ_UNMAP       |\
+	 RQ_ZEROES      |\
+	 RQ_IN_ACT_LOG  |\
+	 RQ_UNPLUG      |\
+	 RQ_POSTPONED   |\
+	 RQ_COMPLETION_SUSP)
+
+static inline bool drbd_req_is_write(struct drbd_request *req)
+{
+	return req->local_rq_state & RQ_WRITE;
+}
 
 /* Short lived temporary struct on the stack.
  * We could squirrel the error to be returned into
@@ -264,61 +301,63 @@ struct bio_and_error {
 	int error;
 };
 
-extern void start_new_tl_epoch(struct drbd_connection *connection);
-extern void drbd_req_destroy(struct kref *kref);
-extern int __req_mod(struct drbd_request *req, enum drbd_req_event what,
-		struct drbd_peer_device *peer_device,
-		struct bio_and_error *m);
-extern void complete_master_bio(struct drbd_device *device,
-		struct bio_and_error *m);
-extern void request_timer_fn(struct timer_list *t);
-extern void tl_restart(struct drbd_connection *connection, enum drbd_req_event what);
-extern void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what);
-extern void tl_abort_disk_io(struct drbd_device *device);
+bool start_new_tl_epoch(struct drbd_resource *resource);
+void drbd_req_destroy(struct kref *kref);
+void __req_mod(struct drbd_request *req, enum drbd_req_event what,
+	       struct drbd_peer_device *peer_device, struct bio_and_error *m);
+void complete_master_bio(struct drbd_device *device, struct bio_and_error *m);
+void drbd_release_conflicts(struct drbd_device *device,
+			    struct drbd_interval *release_interval);
+void drbd_put_ref_tl_walk(struct drbd_request *req, int done_put, int oos_send_put);
+void drbd_set_pending_out_of_sync(struct drbd_peer_device *peer_device);
+void request_timer_fn(struct timer_list *t);
+void tl_walk(struct drbd_connection *connection,
+	     struct drbd_request **from_req, enum drbd_req_event what);
+void __tl_walk(struct drbd_resource * const resource,
+	       struct drbd_connection * const connection,
+	       struct drbd_request **from_req, const enum drbd_req_event what);
+void drbd_destroy_peer_ack_if_done(struct drbd_peer_ack *peer_ack);
+int w_queue_peer_ack(struct drbd_work *w, int cancel);
+void drbd_queue_peer_ack(struct drbd_resource *resource,
+			 struct drbd_request *req);
+bool drbd_should_do_remote(struct drbd_peer_device *peer_device,
+			   enum which_state which);
+void drbd_reclaim_req(struct rcu_head *rp);
 
 /* this is in drbd_main.c */
-extern void drbd_restart_request(struct drbd_request *req);
+void drbd_restart_request(struct drbd_request *req);
+void drbd_restart_suspended_reqs(struct drbd_resource *resource);
 
 /* use this if you don't want to deal with calling complete_master_bio()
  * outside the spinlock, e.g. when walking some list on cleanup. */
-static inline int _req_mod(struct drbd_request *req, enum drbd_req_event what,
+static inline void _req_mod(struct drbd_request *req, enum drbd_req_event what,
 		struct drbd_peer_device *peer_device)
 {
 	struct drbd_device *device = req->device;
 	struct bio_and_error m;
-	int rv;
 
 	/* __req_mod possibly frees req, do not touch req after that! */
-	rv = __req_mod(req, what, peer_device, &m);
+	__req_mod(req, what, peer_device, &m);
 	if (m.bio)
 		complete_master_bio(device, &m);
-
-	return rv;
 }
 
-/* completion of master bio is outside of our spinlock.
- * We still may or may not be inside some irqs disabled section
- * of the lower level driver completion callback, so we need to
- * spin_lock_irqsave here. */
-static inline int req_mod(struct drbd_request *req,
+/* completion of master bio is outside of spinlock.
+ * If you need it irqsave, do it your self!
+ * Which means: don't use from bio endio callback. */
+static inline void req_mod(struct drbd_request *req,
 		enum drbd_req_event what,
 		struct drbd_peer_device *peer_device)
 {
-	unsigned long flags;
 	struct drbd_device *device = req->device;
 	struct bio_and_error m;
-	int rv;
 
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	rv = __req_mod(req, what, peer_device, &m);
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
+	read_lock_irq(&device->resource->state_rwlock);
+	__req_mod(req, what, peer_device, &m);
+	read_unlock_irq(&device->resource->state_rwlock);
 
 	if (m.bio)
 		complete_master_bio(device, &m);
-
-	return rv;
 }
 
-extern bool drbd_should_do_remote(union drbd_dev_state);
-
 #endif
diff --git a/drivers/block/drbd/drbd_sender.c b/drivers/block/drbd/drbd_sender.c
new file mode 100644
index 0000000000000..d128b48e21408
--- /dev/null
+++ b/drivers/block/drbd/drbd_sender.c
@@ -0,0 +1,3826 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+   drbd_sender.c
+
+   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
+
+   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
+   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
+   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
+
+
+ */
+
+#include <linux/drbd.h>
+#include <linux/sched.h>
+#include <linux/sched/signal.h>
+#include <linux/wait.h>
+#include <linux/mm.h>
+#include <linux/memcontrol.h> /* needed on kernels <4.3 */
+#include <linux/mm_inline.h>
+#include <linux/slab.h>
+#include <linux/random.h>
+#include <linux/overflow.h>
+#include <linux/part_stat.h>
+
+#include "drbd_int.h"
+#include <linux/drbd_protocol.h>
+#include "drbd_req.h"
+#include "drbd_meta_data.h"
+
+void drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device);
+
+static int make_ov_request(struct drbd_peer_device *, int);
+static int make_resync_request(struct drbd_peer_device *, int);
+static bool should_send_barrier(struct drbd_connection *, unsigned int epoch);
+static void maybe_send_barrier(struct drbd_connection *, unsigned int);
+static unsigned long get_work_bits(const unsigned long mask, unsigned long *flags);
+
+/* endio handlers:
+ *   drbd_md_endio (defined here)
+ *   drbd_request_endio (defined here)
+ *   drbd_peer_request_endio (defined here)
+ *   drbd_bm_endio (defined in drbd_bitmap.c)
+ *
+ * For all these callbacks, note the following:
+ * The callbacks will be called in irq context by the IDE drivers,
+ * and in Softirqs/Tasklets/BH context by the SCSI drivers.
+ * Try to get the locking right :)
+ *
+ */
+
+/* used for synchronous meta data and bitmap IO
+ * submitted by drbd_md_sync_page_io()
+ */
+void drbd_md_endio(struct bio *bio)
+{
+	struct drbd_device *device;
+
+	blk_status_t status = bio->bi_status;
+
+	device = bio->bi_private;
+	device->md_io.error = blk_status_to_errno(status);
+
+	/* special case: drbd_md_read() during drbd_adm_attach() */
+	if (device->ldev)
+		put_ldev(device);
+	bio_put(bio);
+
+	/* We grabbed an extra reference in _drbd_md_sync_page_io() to be able
+	 * to timeout on the lower level device, and eventually detach from it.
+	 * If this io completion runs after that timeout expired, this
+	 * drbd_md_put_buffer() may allow us to finally try and re-attach.
+	 * During normal operation, this only puts that extra reference
+	 * down to 1 again.
+	 * Make sure we first drop the reference, and only then signal
+	 * completion, or we may (in drbd_al_read_log()) cycle so fast into the
+	 * next drbd_md_sync_page_io(), that we trigger the
+	 * ASSERT(atomic_read(&mdev->md_io_in_use) == 1) there.
+	 */
+	drbd_md_put_buffer(device);
+	device->md_io.done = 1;
+	wake_up(&device->misc_wait);
+}
+
+/* reads on behalf of the partner,
+ * "submitted" by the receiver
+ */
+static void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	bool io_error;
+
+	device->read_cnt += peer_req->i.size >> 9;
+	io_error = test_bit(__EE_WAS_ERROR, &peer_req->flags);
+
+	drbd_queue_work(&connection->sender_work, &peer_req->w);
+	peer_req = NULL; /* peer_req may be freed. */
+
+	/*
+	 * Decrement counter after queuing work to avoid a moment where
+	 * backing_ee_cnt is zero and the sender work list is empty.
+	 */
+	if (atomic_dec_and_test(&connection->backing_ee_cnt))
+		wake_up(&connection->ee_wait);
+
+	if (io_error)
+		drbd_handle_io_error(device, DRBD_READ_ERROR);
+
+	put_ldev(device);
+}
+
+static int is_failed_barrier(int ee_flags)
+{
+	return (ee_flags & (EE_IS_BARRIER|EE_WAS_ERROR|EE_RESUBMITTED|EE_TRIM|EE_ZEROOUT))
+		== (EE_IS_BARRIER|EE_WAS_ERROR);
+}
+
+static bool drbd_peer_request_is_merged(struct drbd_peer_request *peer_req,
+		sector_t main_sector, sector_t main_sector_end)
+{
+	/*
+	 * We do not send overlapping resync requests. So any request which is
+	 * in the corresponding range and for which we have received a reply
+	 * must be a merged request. EE_TRIM implies that we have received a
+	 * reply.
+	 */
+	return peer_req->i.sector >= main_sector &&
+		peer_req->i.sector + (peer_req->i.size >> SECTOR_SHIFT) <= main_sector_end &&
+			peer_req->i.type == INTERVAL_RESYNC_WRITE &&
+			(peer_req->flags & EE_TRIM);
+}
+
+int drbd_unmerge_discard(struct drbd_peer_request *peer_req_main, struct list_head *list)
+{
+	struct drbd_peer_device *peer_device = peer_req_main->peer_device;
+	struct drbd_peer_request *peer_req = peer_req_main;
+	sector_t main_sector = peer_req_main->i.sector;
+	sector_t main_sector_end = main_sector + (peer_req_main->i.size >> SECTOR_SHIFT);
+	int merged_count = 0;
+
+	list_for_each_entry_continue(peer_req, &peer_device->resync_requests, recv_order) {
+		if (!drbd_peer_request_is_merged(peer_req, main_sector, main_sector_end))
+			break;
+
+		merged_count++;
+		list_add_tail(&peer_req->w.list, list);
+	}
+
+	return merged_count;
+}
+
+/* writes on behalf of the partner, or resync writes,
+ * "submitted" by the receiver, final stage.  */
+void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local)
+{
+	unsigned long flags = 0;
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	enum drbd_interval_type type;
+	bool do_wake;
+
+	/* if this is a failed barrier request, disable use of barriers,
+	 * and schedule for resubmission */
+	if (is_failed_barrier(peer_req->flags)) {
+		drbd_bump_write_ordering(device->resource, device->ldev, WO_BDEV_FLUSH);
+		spin_lock_irqsave(&connection->peer_reqs_lock, flags);
+		peer_req->flags = (peer_req->flags & ~EE_WAS_ERROR) | EE_RESUBMITTED;
+		peer_req->w.cb = w_e_reissue;
+		/* put_ldev actually happens below, once we come here again. */
+		__release(local);
+		spin_unlock_irqrestore(&connection->peer_reqs_lock, flags);
+		drbd_queue_work(&connection->sender_work, &peer_req->w);
+		if (atomic_dec_and_test(&connection->active_ee_cnt))
+			wake_up(&connection->ee_wait);
+		return;
+	}
+
+	/* after we moved peer_req to done_ee,
+	 * we may no longer access it,
+	 * it may be freed/reused already!
+	 * (as soon as we release the peer_reqs_lock) */
+	type = peer_req->i.type;
+
+	if (peer_req->flags & EE_WAS_ERROR) {
+		/* In protocol != C, we usually do not send write acks.
+		 * In case of a write error, send the neg ack anyways.
+		 * This only applies to to application writes, not to resync. */
+		if (peer_req->i.type == INTERVAL_PEER_WRITE) {
+			if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags))
+				inc_unacked(peer_device);
+		}
+		drbd_set_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
+		drbd_handle_io_error(device, DRBD_WRITE_ERROR);
+	}
+
+	spin_lock_irqsave(&connection->peer_reqs_lock, flags);
+	device->writ_cnt += peer_req->i.size >> 9;
+	atomic_inc(&connection->done_ee_cnt);
+	list_add_tail(&peer_req->w.list, &connection->done_ee);
+	if (peer_req->i.type == INTERVAL_RESYNC_WRITE && peer_req->flags & EE_TRIM) {
+		LIST_HEAD(merged);
+		int merged_count;
+
+		merged_count = drbd_unmerge_discard(peer_req, &merged);
+		list_splice_tail(&merged, &connection->done_ee);
+		atomic_add(merged_count, &connection->done_ee_cnt);
+	}
+	peer_req = NULL; /* may be freed after unlock */
+	spin_unlock_irqrestore(&connection->peer_reqs_lock, flags);
+
+	/*
+	 * Do not remove from the requests tree here: we did not send the
+	 * Ack yet.
+	 * Removed from the tree from "drbd_finish_peer_reqs" within the
+	 * appropriate callback (e_end_block/e_end_resync_block) or from
+	 * cleanup functions if the connection is lost.
+	 */
+
+	if (connection->cstate[NOW] == C_CONNECTED)
+		queue_work(connection->ack_sender, &connection->send_acks_work);
+
+	if (type == INTERVAL_RESYNC_WRITE)
+		do_wake = atomic_dec_and_test(&connection->backing_ee_cnt);
+	else
+		do_wake = atomic_dec_and_test(&connection->active_ee_cnt);
+
+	if (do_wake)
+		wake_up(&connection->ee_wait);
+
+	put_ldev(device);
+}
+
+/* writes on behalf of the partner, or resync writes,
+ * "submitted" by the receiver.
+ */
+void drbd_peer_request_endio(struct bio *bio)
+{
+	struct drbd_peer_request *peer_req = bio->bi_private;
+	struct drbd_device *device = peer_req->peer_device->device;
+	bool is_write = bio_data_dir(bio) == WRITE;
+	bool is_discard = bio_op(bio) == REQ_OP_WRITE_ZEROES ||
+			  bio_op(bio) == REQ_OP_DISCARD;
+
+	blk_status_t status = bio->bi_status;
+
+	if (status && drbd_device_ratelimit(device, BACKEND))
+		drbd_warn(device, "%s: error=%d s=%llus\n",
+				is_write ? (is_discard ? "discard" : "write")
+					: "read", status,
+				(unsigned long long)peer_req->i.sector);
+
+	if (status)
+		set_bit(__EE_WAS_ERROR, &peer_req->flags);
+
+	bio_put(bio); /* no need for the bio anymore */
+	if (atomic_dec_and_test(&peer_req->pending_bios)) {
+		if (is_write)
+			drbd_endio_write_sec_final(peer_req);
+		else
+			drbd_endio_read_sec_final(peer_req);
+	}
+}
+
+/* Not static to increase the likelyhood that it will show up in a stack trace */
+void drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
+{
+	panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n",
+		device->minor, device->resource->name, device->vnr);
+}
+
+
+/* read, readA or write requests on R_PRIMARY coming from drbd_submit_bio
+ */
+void drbd_request_endio(struct bio *bio)
+{
+	unsigned long flags;
+	struct drbd_request *req = bio->bi_private;
+	struct drbd_device *device = req->device;
+	struct bio_and_error m;
+	enum drbd_req_event what;
+
+	blk_status_t status = bio->bi_status;
+
+	/* If this request was aborted locally before,
+	 * but now was completed "successfully",
+	 * chances are that this caused arbitrary data corruption.
+	 *
+	 * "aborting" requests, or force-detaching the disk, is intended for
+	 * completely blocked/hung local backing devices which do no longer
+	 * complete requests at all, not even do error completions.  In this
+	 * situation, usually a hard-reset and failover is the only way out.
+	 *
+	 * By "aborting", basically faking a local error-completion,
+	 * we allow for a more graceful switchover by cleanly migrating services.
+	 * Still the affected node has to be rebooted "soon".
+	 *
+	 * By completing these requests, we allow the upper layers to re-use
+	 * the associated data pages.
+	 *
+	 * If later the local backing device "recovers", and now DMAs some data
+	 * from disk into the original request pages, in the best case it will
+	 * just put random data into unused pages; but typically it will corrupt
+	 * meanwhile completely unrelated data, causing all sorts of damage.
+	 *
+	 * Which means delayed successful completion,
+	 * especially for READ requests,
+	 * is a reason to panic().
+	 *
+	 * We assume that a delayed *error* completion is OK,
+	 * though we still will complain noisily about it.
+	 */
+	if (unlikely(req->local_rq_state & RQ_LOCAL_ABORTED)) {
+		if (drbd_device_ratelimit(device, BACKEND))
+			drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n");
+
+		if (!status)
+			drbd_panic_after_delayed_completion_of_aborted_request(device);
+	}
+
+	/* to avoid recursion in __req_mod */
+	if (unlikely(status)) {
+		enum req_op op = bio_op(bio);
+		if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
+			if (status == BLK_STS_NOTSUPP)
+				what = DISCARD_COMPLETED_NOTSUPP;
+			else
+				what = DISCARD_COMPLETED_WITH_ERROR;
+		} else if (op == REQ_OP_READ) {
+			if (bio->bi_opf & REQ_RAHEAD)
+				what = READ_AHEAD_COMPLETED_WITH_ERROR;
+			else
+				what = READ_COMPLETED_WITH_ERROR;
+		} else {
+			what = WRITE_COMPLETED_WITH_ERROR;
+		}
+	} else {
+		what = COMPLETED_OK;
+	}
+
+	bio_put(req->private_bio);
+	req->private_bio = ERR_PTR(blk_status_to_errno(status));
+
+	/* it is legal to fail read-ahead, no drbd_handle_io_error for READ_AHEAD_COMPLETED_WITH_ERROR */
+	if (what == WRITE_COMPLETED_WITH_ERROR)
+		drbd_handle_io_error(device, DRBD_WRITE_ERROR);
+	else if (what == READ_COMPLETED_WITH_ERROR)
+		drbd_handle_io_error(device, DRBD_READ_ERROR);
+
+	spin_lock_irqsave(&device->interval_lock, flags);
+	set_bit(INTERVAL_BACKING_COMPLETED, &req->i.flags);
+	if (req->local_rq_state & RQ_WRITE)
+		drbd_release_conflicts(device, &req->i);
+	spin_unlock_irqrestore(&device->interval_lock, flags);
+
+	/* not req_mod(), we need irqsave here! */
+	read_lock_irqsave(&device->resource->state_rwlock, flags);
+	__req_mod(req, what, NULL, &m);
+	read_unlock_irqrestore(&device->resource->state_rwlock, flags);
+	put_ldev(device);
+
+	if (m.bio)
+		complete_master_bio(device, &m);
+}
+
+struct dagtag_find_result {
+	int err;
+	unsigned int node_id;
+	u64 dagtag;
+};
+
+static struct dagtag_find_result find_current_dagtag(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	struct dagtag_find_result ret = { 0 };
+
+	read_lock_irq(&resource->state_rwlock);
+
+	if (resource->role[NOW] == R_PRIMARY) {
+		/* Sending data and sending resync requests are not
+		 * synchronized with each other, so our peer may need to wait
+		 * until it has received more data before it can reply to this
+		 * request. */
+		ret.node_id = resource->res_opts.node_id;
+		ret.dagtag = resource->dagtag_sector;
+	} else {
+		for_each_connection(connection, resource) {
+			if (connection->peer_role[NOW] != R_PRIMARY)
+				continue;
+
+			/* Do not depend on a stale dagtag. */
+			if (!test_bit(RECEIVED_DAGTAG, &connection->flags))
+				continue;
+
+			if (ret.dagtag) {
+				if (drbd_ratelimit())
+					drbd_err(resource, "Refusing to resync due to multiple remote primaries\n");
+				ret.err = 1;
+				break;
+			} else {
+				ret.node_id = connection->peer_node_id;
+				ret.dagtag = atomic64_read(&connection->last_dagtag_sector);
+			}
+		}
+	}
+
+	read_unlock_irq(&resource->state_rwlock);
+
+	return ret;
+}
+
+static void send_resync_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct dagtag_find_result dagtag_result;
+
+	if (!(connection->agreed_features & DRBD_FF_RESYNC_DAGTAG) &&
+			drbd_al_active(peer_device->device, peer_req->i.sector, peer_req->i.size)) {
+		dynamic_drbd_dbg(peer_device,
+				"Abort resync request at %llus+%u due to activity",
+				(unsigned long long) peer_req->i.sector, peer_req->i.size);
+
+		drbd_unsuccessful_resync_request(peer_req, false);
+		return;
+	}
+
+	inc_rs_pending(peer_device);
+
+	dagtag_result = find_current_dagtag(peer_device->device->resource);
+	if (dagtag_result.err) {
+		change_cstate(peer_device->connection, C_DISCONNECTING, CS_HARD);
+		return;
+	}
+
+	if (peer_req->flags & EE_HAS_DIGEST) {
+		enum drbd_packet cmd = connection->agreed_features & DRBD_FF_RESYNC_DAGTAG ?
+			P_RS_CSUM_DAGTAG_REQ : P_CSUM_RS_REQUEST;
+
+		void *digest = drbd_prepare_drequest_csum(peer_req, cmd,
+				peer_req->digest->digest_size,
+				dagtag_result.node_id, dagtag_result.dagtag);
+		if (!digest)
+			return;
+
+		memcpy(digest, peer_req->digest->digest, peer_req->digest->digest_size);
+
+		/* We are now finished with the digest, so we can free it.
+		 * If we don't, the reference will be lost when the block_id
+		 * field of the union is used for the reply. */
+		peer_req->flags &= ~EE_HAS_DIGEST;
+		kfree(peer_req->digest);
+		peer_req->digest = NULL;
+
+		drbd_send_command(peer_device, cmd, DATA_STREAM);
+	} else {
+		enum drbd_packet cmd;
+		if (connection->agreed_features & DRBD_FF_RESYNC_DAGTAG)
+			cmd = peer_req->flags & EE_RS_THIN_REQ ? P_RS_THIN_DAGTAG_REQ : P_RS_DAGTAG_REQ;
+		else
+			cmd = peer_req->flags & EE_RS_THIN_REQ ? P_RS_THIN_REQ : P_RS_DATA_REQUEST;
+
+		drbd_send_rs_request(peer_device, cmd,
+				peer_req->i.sector, peer_req->i.size, peer_req->block_id,
+				dagtag_result.node_id, dagtag_result.dagtag);
+	}
+}
+
+void drbd_conflict_send_resync_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_req->peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	bool conflict;
+	bool canceled;
+
+	spin_lock_irq(&device->interval_lock);
+	clear_bit(INTERVAL_SUBMIT_CONFLICT_QUEUED, &peer_req->i.flags);
+	canceled = test_bit(INTERVAL_CANCELED, &peer_req->i.flags);
+	conflict = drbd_find_conflict(device, &peer_req->i, CONFLICT_FLAG_IGNORE_SAME_PEER);
+	if (drbd_interval_empty(&peer_req->i))
+		drbd_insert_interval(&device->requests, &peer_req->i);
+	if (!conflict)
+		set_bit(INTERVAL_SENT, &peer_req->i.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	if (!conflict) {
+		send_resync_request(peer_req);
+	} else if (canceled) {
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+
+	if ((!conflict || canceled) && atomic_dec_and_test(&connection->backing_ee_cnt))
+		wake_up(&connection->ee_wait);
+}
+
+void drbd_csum_pages(struct crypto_shash *tfm, struct page *page, void *digest)
+{
+	SHASH_DESC_ON_STACK(desc, tfm);
+
+	desc->tfm = tfm;
+
+	crypto_shash_init(desc);
+
+	page_chain_for_each(page) {
+		unsigned off = page_chain_offset(page);
+		unsigned len = page_chain_size(page);
+		u8 *src;
+		src = kmap_atomic(page);
+		crypto_shash_update(desc, src + off, len);
+		kunmap_atomic(src);
+	}
+	crypto_shash_final(desc, digest);
+	shash_desc_zero(desc);
+}
+
+void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest)
+{
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+	SHASH_DESC_ON_STACK(desc, tfm);
+
+	desc->tfm = tfm;
+
+	crypto_shash_init(desc);
+
+	bio_for_each_segment(bvec, bio, iter) {
+		u8 *src;
+		src = bvec_kmap_local(&bvec);
+		crypto_shash_update(desc, src, bvec.bv_len);
+		kunmap_local(src);
+	}
+	crypto_shash_final(desc, digest);
+	shash_desc_zero(desc);
+}
+
+/* MAYBE merge common code with w_e_end_ov_req */
+static int w_e_send_csum(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	int digest_size;
+	int err = 0;
+	struct digest_info *di;
+
+	if (unlikely(cancel))
+		goto out;
+
+	/* Do not add to interval tree if already disconnected or resync aborted */
+	if (!repl_is_sync_target(peer_device->repl_state[NOW]))
+		goto out;
+
+	if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0))
+		goto out;
+
+	digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm);
+
+	di = kmalloc(sizeof(*di) + digest_size, GFP_NOIO);
+	if (!di) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	di->digest_size = digest_size;
+	di->digest = (((char *)di)+sizeof(struct digest_info));
+
+	drbd_csum_pages(peer_device->connection->csums_tfm, peer_req->page_chain.head, di->digest);
+	/* Free pages before continuing.
+	 * In case we block on congestion, we could otherwise run into
+	 * some distributed deadlock, if the other side blocks on
+	 * congestion as well, because our receiver blocks in
+	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
+	drbd_free_page_chain(&peer_device->connection->transport, &peer_req->page_chain);
+
+	/* Use the same drbd_peer_request for tracking resync request and for
+	 * writing, if that is necessary. */
+	peer_req->digest = di;
+	peer_req->flags |= EE_HAS_DIGEST;
+
+	atomic_inc(&connection->backing_ee_cnt);
+	drbd_conflict_send_resync_request(peer_req);
+	return 0;
+
+out:
+	atomic_sub(peer_req->i.size >> SECTOR_SHIFT, &peer_device->device->rs_sect_ev);
+	drbd_free_peer_req(peer_req);
+
+	if (unlikely(err))
+		drbd_err(peer_device, "drbd_send_drequest(..., csum) failed\n");
+	return err;
+}
+
+static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, int size)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_request *peer_req;
+
+	if (!get_ldev(device))
+		return -EIO;
+
+	/* Do not wait if no memory is immediately available.  */
+	peer_req = drbd_alloc_peer_req(peer_device, GFP_TRY & ~__GFP_RECLAIM);
+	if (!peer_req)
+		goto defer;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_add_tail(&peer_req->recv_order, &peer_device->resync_requests);
+	peer_req->flags |= EE_ON_RECV_ORDER;
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	if (size) {
+		drbd_alloc_page_chain(&connection->transport,
+			&peer_req->page_chain, DIV_ROUND_UP(size, PAGE_SIZE), GFP_TRY);
+		if (!peer_req->page_chain.head)
+			goto defer2;
+	}
+	peer_req->i.size = size;
+	peer_req->i.sector = sector;
+	/* This will be a resync write once we receive the data back from the
+	 * peer, assuming the checksums differ. */
+	peer_req->i.type = INTERVAL_RESYNC_WRITE;
+	peer_req->requested_size = size;
+
+	peer_req->w.cb = w_e_send_csum;
+	peer_req->opf = REQ_OP_READ;
+
+	atomic_inc(&connection->backing_ee_cnt);
+	atomic_add(size >> 9, &device->rs_sect_ev);
+	if (drbd_submit_peer_request(peer_req) == 0)
+		return 0;
+
+	/* If it failed because of ENOMEM, retry should help.  If it failed
+	 * because bio_add_page failed (probably broken lower level driver),
+	 * retry may or may not help.
+	 * If it does not, you may need to force disconnect. */
+
+defer2:
+	drbd_free_peer_req(peer_req);
+defer:
+	put_ldev(device);
+	return -EAGAIN;
+}
+
+static int make_one_resync_request(struct drbd_peer_device *peer_device, int discard_granularity, sector_t sector, int size)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_peer_request *peer_req;
+
+	/* Do not wait if no memory is immediately available.  */
+	peer_req = drbd_alloc_peer_req(peer_device, GFP_TRY & ~__GFP_RECLAIM);
+	if (!peer_req) {
+		drbd_err(device, "Could not allocate resync request\n");
+		put_ldev(device);
+		return -EAGAIN;
+	}
+
+	peer_req->i.size = size;
+	peer_req->i.sector = sector;
+	peer_req->i.type = INTERVAL_RESYNC_WRITE;
+	peer_req->requested_size = size;
+
+	if (size == discard_granularity)
+		peer_req->flags |= EE_RS_THIN_REQ;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	list_add_tail(&peer_req->recv_order, &peer_device->resync_requests);
+	peer_req->flags |= EE_ON_RECV_ORDER;
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	atomic_inc(&connection->backing_ee_cnt);
+	drbd_conflict_send_resync_request(peer_req);
+	return 0;
+}
+
+int w_resync_timer(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_device *peer_device =
+		container_of(w, struct drbd_peer_device, resync_work);
+
+	switch (peer_device->repl_state[NOW]) {
+	case L_VERIFY_S:
+		make_ov_request(peer_device, cancel);
+		break;
+	case L_SYNC_TARGET:
+		make_resync_request(peer_device, cancel);
+		break;
+	default:
+		if (atomic_read(&peer_device->rs_sect_in) >= peer_device->rs_in_flight) {
+			struct drbd_resource *resource = peer_device->device->resource;
+			unsigned long irq_flags;
+			begin_state_change(resource, &irq_flags, 0);
+			peer_device->resync_active[NEW] = false;
+			end_state_change(resource, &irq_flags, "resync-inactive");
+		}
+		break;
+	}
+
+	return 0;
+}
+
+int w_send_dagtag(struct drbd_work *w, int cancel)
+{
+	struct drbd_connection *connection =
+		container_of(w, struct drbd_connection, send_dagtag_work);
+	struct drbd_resource *resource = connection->resource;
+	int err;
+	u64 dagtag_sector;
+
+	if (cancel)
+		return 0;
+
+	read_lock_irq(&resource->state_rwlock);
+	dagtag_sector = connection->send_dagtag;
+	/* It is OK to use the value outside the lock, because the work will be
+	 * queued again if it is changed. */
+	read_unlock_irq(&resource->state_rwlock);
+
+	/* Only send if no request with a newer dagtag has been sent. This can
+	 * occur if a write arrives after the state change and is processed
+	 * before this work item. */
+	if (dagtag_newer_eq(connection->send.current_dagtag_sector, dagtag_sector))
+		return 0;
+
+	err = drbd_send_dagtag(connection, dagtag_sector);
+	if (err)
+		return err;
+
+	connection->send.current_dagtag_sector = dagtag_sector;
+	return 0;
+}
+
+int w_send_uuids(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_device *peer_device =
+		container_of(w, struct drbd_peer_device, propagate_uuids_work);
+
+	if (peer_device->repl_state[NOW] < L_ESTABLISHED ||
+	    !test_bit(INITIAL_STATE_SENT, &peer_device->flags))
+		return 0;
+
+	drbd_send_uuids(peer_device, 0, 0);
+
+	return 0;
+}
+
+bool drbd_any_flush_pending(struct drbd_resource *resource)
+{
+	unsigned long flags;
+	struct drbd_connection *primary_connection;
+	bool any_flush_pending = false;
+
+	spin_lock_irqsave(&resource->initiator_flush_lock, flags);
+	rcu_read_lock();
+	for_each_connection_rcu(primary_connection, resource) {
+		if (primary_connection->pending_flush_mask) {
+			any_flush_pending = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+	spin_unlock_irqrestore(&resource->initiator_flush_lock, flags);
+
+	return any_flush_pending;
+}
+
+void resync_timer_fn(struct timer_list *t)
+{
+	struct drbd_peer_device *peer_device = timer_container_of(peer_device, t, resync_timer);
+
+	drbd_queue_work_if_unqueued(
+		&peer_device->connection->sender_work,
+		&peer_device->resync_work);
+}
+
+static void fifo_set(struct fifo_buffer *fb, int value)
+{
+	int i;
+
+	for (i = 0; i < fb->size; i++)
+		fb->values[i] = value;
+}
+
+static int fifo_push(struct fifo_buffer *fb, int value)
+{
+	int ov;
+
+	ov = fb->values[fb->head_index];
+	fb->values[fb->head_index++] = value;
+
+	if (fb->head_index >= fb->size)
+		fb->head_index = 0;
+
+	return ov;
+}
+
+static void fifo_add_val(struct fifo_buffer *fb, int value)
+{
+	int i;
+
+	for (i = 0; i < fb->size; i++)
+		fb->values[i] += value;
+}
+
+struct fifo_buffer *fifo_alloc(unsigned int fifo_size)
+{
+	struct fifo_buffer *fb;
+
+	fb = kzalloc_flex(*fb, values, fifo_size, GFP_NOIO);
+	if (!fb)
+		return NULL;
+
+	fb->head_index = 0;
+	fb->size = fifo_size;
+	fb->total = 0;
+
+	return fb;
+}
+
+/* FIXME by choosing to calculate in nano seconds, we now have several do_div()
+ * in here, which I find very ugly.
+ */
+static int drbd_rs_controller(struct drbd_peer_device *peer_device, u64 sect_in, u64 duration_ns)
+{
+	const u64 max_duration_ns = RS_MAKE_REQS_INTV_NS * 10;
+	struct peer_device_conf *pdc;
+	unsigned int want;     /* The number of sectors we want in-flight */
+	int req_sect; /* Number of sectors to request in this turn */
+	int correction; /* Number of sectors more we need in-flight */
+	int cps; /* correction per invocation of drbd_rs_controller() */
+	int steps; /* Number of time steps to plan ahead */
+	int curr_corr;
+	u64 max_sect;
+	struct fifo_buffer *plan;
+	u64 duration_ms;
+
+	if (duration_ns == 0)
+		duration_ns = 1;
+	else if (duration_ns > max_duration_ns)
+		duration_ns = max_duration_ns;
+
+	if (duration_ns < RS_MAKE_REQS_INTV_NS) {
+		/* Scale sect_in so that it represents the number of sectors which
+		 * would have arrived if the cycle had lasted the normal time
+		 * (RS_MAKE_REQS_INTV). */
+		sect_in = sect_in * RS_MAKE_REQS_INTV_NS;
+		do_div(sect_in, duration_ns);
+	}
+
+	pdc = rcu_dereference(peer_device->conf);
+	plan = rcu_dereference(peer_device->rs_plan_s);
+
+	steps = plan->size; /* (pdc->c_plan_ahead * 10 * RS_MAKE_REQS_INTV) / HZ; */
+
+	if (peer_device->rs_in_flight + sect_in == 0) { /* At start of resync */
+		want = ((pdc->resync_rate * 2 * RS_MAKE_REQS_INTV) / HZ) * steps;
+	} else { /* normal path */
+		if (pdc->c_fill_target) {
+			want = pdc->c_fill_target;
+		} else {
+			u64 tmp = sect_in * pdc->c_delay_target * NSEC_PER_SEC;
+			do_div(tmp, (duration_ns * 10));
+			want = tmp;
+		}
+	}
+
+	correction = want - peer_device->rs_in_flight - plan->total;
+
+	/* Plan ahead */
+	cps = correction / steps;
+	fifo_add_val(plan, cps);
+	plan->total += cps * steps;
+
+	/* What we do in this step */
+	curr_corr = fifo_push(plan, 0);
+	plan->total -= curr_corr;
+
+	req_sect = sect_in + curr_corr;
+	if (req_sect < 0)
+		req_sect = 0;
+
+	if (pdc->c_max_rate == 0) {
+		/* No rate limiting. */
+		max_sect = ~0ULL;
+	} else {
+		max_sect = (u64)pdc->c_max_rate * 2 * RS_MAKE_REQS_INTV_NS;
+		do_div(max_sect, NSEC_PER_SEC);
+	}
+
+	duration_ms = duration_ns;
+	do_div(duration_ms, NSEC_PER_MSEC);
+	dynamic_drbd_dbg(peer_device, "dur=%lluns (%llums) sect_in=%llu in_flight=%d wa=%u co=%d st=%d cps=%d cc=%d rs=%d mx=%llu\n",
+		 duration_ns, duration_ms, sect_in, peer_device->rs_in_flight, want, correction,
+		 steps, cps, curr_corr, req_sect, max_sect);
+
+	if (req_sect > max_sect)
+		req_sect = max_sect;
+
+	return req_sect;
+}
+
+/* Calculate how many 4k sized blocks we want to resync this time.
+ * Because peer nodes may have different bitmap granularity, and won't
+ * be able to clear "partial bits", make sure we try to request
+ * multiples of the bitmap block size (of myself or the peer, whichever
+ * is larger) in one go. Return value is scaled to our bm_block_size.
+ * If both peers operate at 4k granularity, or at identical granularity,
+ * this should not change behavior.
+ * If we operate at different granularities, we may need to improve our
+ * drbd_rs_controller() as well to get intended resync rates,
+ * especially if you configure for rather small c-max-rate.
+ */
+static int drbd_rs_number_requests(struct drbd_peer_device *peer_device)
+{
+	struct net_conf *nc;
+	ktime_t duration, now;
+	unsigned int sect_in;  /* Number of sectors that came in since the last turn */
+	int number, mxb;
+	int effective_resync_request_size;
+	struct drbd_bitmap *bm = peer_device->device->bitmap;
+
+	sect_in = atomic_xchg(&peer_device->rs_sect_in, 0);
+	peer_device->rs_in_flight -= sect_in;
+
+	now = ktime_get();
+	duration = ktime_sub(now, peer_device->rs_last_mk_req_kt);
+	peer_device->rs_last_mk_req_kt = now;
+
+	rcu_read_lock();
+	nc = rcu_dereference(peer_device->connection->transport.net_conf);
+	mxb = nc ? nc->max_buffers : 0;
+	if (rcu_dereference(peer_device->rs_plan_s)->size) {
+		number = drbd_rs_controller(peer_device, sect_in, ktime_to_ns(duration));
+		number = sect_to_bit(number, BM_BLOCK_SHIFT_4k);
+	} else {
+		number = RS_MAKE_REQS_INTV * rcu_dereference(peer_device->conf)->resync_rate
+			/ ((BM_BLOCK_SIZE_4k/1024) * HZ);
+	}
+	rcu_read_unlock();
+
+	/* Don't have more than "max-buffers"/2 in-flight.
+	 * Otherwise we may cause the remote site to stall on drbd_alloc_pages(),
+	 * potentially causing a distributed deadlock on congestion during
+	 * online-verify or (checksum-based) resync, if max-buffers,
+	 * socket buffer sizes and resync rate settings are mis-configured.
+	 * Note that "number" is in units of "bm_bytes_per_bit",
+	 * mxb (as used here, and in drbd_alloc_pages on the peer) is
+	 * "number of pages" (typically 4k), and "rs_in_flight" is in "sectors"
+	 * (512 Byte). Convert everything to sectors and back.
+	 */
+	{
+		int mxb_sect = mxb << (PAGE_SHIFT - 9);
+		int num_sect = bit_to_sect(number, BM_BLOCK_SHIFT_4k);
+
+		if (mxb_sect - peer_device->rs_in_flight < num_sect) {
+			num_sect = mxb_sect - peer_device->rs_in_flight;
+			number = sect_to_bit(num_sect, BM_BLOCK_SHIFT_4k);
+		}
+	}
+
+	/* BM_BLOCK_SIZE_MAX/BM_BLOCK_SIZE_4k? Maybe. But do not round up unless we have to. */
+	effective_resync_request_size =
+		1 << (max(bm->bm_block_shift, peer_device->bm_block_shift) - BM_BLOCK_SHIFT_4k);
+	number = ALIGN(number, effective_resync_request_size);
+	peer_device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE_4k/1024) / RS_MAKE_REQS_INTV;
+	return number >> (bm->bm_block_shift - BM_BLOCK_SHIFT_4k);
+}
+
+static int resync_delay(bool request_ok, int number, int done)
+{
+	if (request_ok && number > 0 && done > 0) {
+		/* Requests in-flight. Adjusting the standard delay to
+		 * mitigate rounding and other errors, that cause 'done'
+		 * to be different from the optimal 'number'.  (usually
+		 * in the range of 66ms to 133ms) */
+		return RS_MAKE_REQS_INTV * done / number;
+	}
+
+	return RS_MAKE_REQS_INTV;
+}
+
+void drbd_rs_all_in_flight_came_back(struct drbd_peer_device *peer_device, int rs_sect_in)
+{
+	unsigned int max_bio_size_kb = DRBD_MAX_BIO_SIZE / 1024;
+	struct drbd_device *device = peer_device->device;
+	unsigned int c_max_rate, interval, latency, m, amount_kb;
+	unsigned int rs_kib_in = rs_sect_in / 2;
+	ktime_t latency_kt;
+	bool kickstart;
+
+	if (get_ldev(device)) {
+		max_bio_size_kb = queue_max_hw_sectors(device->rq_queue) / 2;
+		put_ldev(device);
+	}
+
+	rcu_read_lock();
+	c_max_rate = rcu_dereference(peer_device->conf)->c_max_rate;
+	rcu_read_unlock();
+
+	latency_kt = ktime_sub(ktime_get(), peer_device->rs_last_mk_req_kt);
+	latency = nsecs_to_jiffies(ktime_to_ns(latency_kt));
+
+	m = max_bio_size_kb > rs_kib_in ? max_bio_size_kb / rs_kib_in : 1;
+	if (c_max_rate != 0)
+		interval = rs_kib_in * m * HZ / c_max_rate;
+	else
+		interval = 0;
+	/* interval holds the ideal pace in which we should request max_bio_size */
+
+	if (peer_device->repl_state[NOW] == L_SYNC_TARGET) {
+		/* Only run resync_work early if we are definitely making
+		 * progress. Otherwise we might continually lock a resync
+		 * extent even when all the requests are canceled. This can
+		 * cause application IO to be blocked for an indefinitely long
+		 * time. */
+		if (test_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags))
+			return;
+	}
+
+	amount_kb = c_max_rate / (HZ / RS_MAKE_REQS_INTV);
+	kickstart = rs_kib_in < amount_kb / 2 && latency < RS_MAKE_REQS_INTV / 2;
+	/* In case the latency of the link and remote IO subsystem is small and
+	   the controller was clearly issuing a too small number of requests,
+	   kickstart it by scheduling it immediately */
+
+	if (kickstart || interval <= latency) {
+		drbd_queue_work_if_unqueued(
+			&peer_device->connection->sender_work,
+			&peer_device->resync_work);
+		return;
+	}
+
+	if (interval < RS_MAKE_REQS_INTV)
+		mod_timer(&peer_device->resync_timer, jiffies + (interval - latency));
+}
+
+static void drbd_enable_peer_replication(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	unsigned long irq_flags;
+	struct drbd_peer_device *peer_device;
+
+	begin_state_change(resource, &irq_flags, CS_VERBOSE);
+	for_each_peer_device(peer_device, device)
+		peer_device->peer_replication[NEW] = true;
+	end_state_change(resource, &irq_flags, "enable-peer-replication");
+}
+
+/* Returns whether whole resync is finished. */
+static bool drbd_resync_check_finished(struct drbd_peer_device *peer_device)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_resource *resource = connection->resource;
+	bool resync_requests_complete;
+	unsigned long bitmap_weight;
+	unsigned long last_resync_pass_bits;
+	bool peer_replication;
+
+	/* Test whether resync pass finished */
+	if (drbd_bm_find_next(peer_device, peer_device->resync_next_bit) < DRBD_END_OF_BITMAP)
+		return false;
+
+	if (drbd_any_flush_pending(resource))
+		return false;
+
+	spin_lock_irq(&connection->peer_reqs_lock);
+	resync_requests_complete = list_empty(&peer_device->resync_requests);
+	spin_unlock_irq(&connection->peer_reqs_lock);
+
+	if (!resync_requests_complete)
+		return false;
+
+	last_resync_pass_bits = peer_device->last_resync_pass_bits;
+	bitmap_weight = drbd_bm_total_weight(peer_device);
+	peer_device->last_resync_pass_bits = bitmap_weight;
+
+	peer_replication = drbd_all_peer_replication(device, NOW);
+	dynamic_drbd_dbg(peer_device, "Resync pass complete last:%lu out-of-sync:%lu failed:%lu replication:%s\n",
+			last_resync_pass_bits, bitmap_weight, peer_device->rs_failed,
+			peer_replication ? "enabled" : "disabled");
+
+	if (!peer_replication) {
+		if (peer_device->rs_failed == 0 && bitmap_weight > 0 &&
+				bitmap_weight < last_resync_pass_bits / 2) {
+			/* Start next pass with replication still disabled */
+			peer_device->resync_next_bit = 0;
+			return false;
+		}
+
+		drbd_enable_peer_replication(device);
+		return false;
+	}
+
+	if (peer_device->rs_failed == 0 && bitmap_weight > 0) {
+		/* Start next pass. Replication is enabled. */
+		peer_device->resync_next_bit = 0;
+		return false;
+	}
+
+	drbd_resync_finished(peer_device, D_MASK);
+	return true;
+}
+
+static bool send_buffer_half_full(struct drbd_peer_device *peer_device)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_transport *transport = &connection->transport;
+	bool half_full = false;
+
+	mutex_lock(&connection->mutex[DATA_STREAM]);
+	if (transport->class->ops.stream_ok(transport, DATA_STREAM)) {
+		struct drbd_transport_stats transport_stats;
+		int queued, sndbuf;
+
+		transport->class->ops.stats(transport, &transport_stats);
+		queued = transport_stats.send_buffer_used;
+		sndbuf = transport_stats.send_buffer_size;
+		if (queued > sndbuf / 2) {
+			half_full = true;
+			transport->class->ops.hint(transport, DATA_STREAM, NOSPACE);
+		}
+	} else {
+		half_full = true;
+	}
+	mutex_unlock(&connection->mutex[DATA_STREAM]);
+
+	return half_full;
+}
+
+static int optimal_bits_for_alignment(unsigned long bit, int bm_block_shift)
+{
+	int max_bio_bits = DRBD_MAX_BIO_SIZE >> bm_block_shift;
+
+	/* under the assumption that we find a big block of out-of-sync blocks
+	   in the bitmap, calculate the optimal request size so that the
+	   request sizes get bigger, and each request is "perfectly" aligned.
+	   (In case the backing device is a RAID5)
+	   for an odd number, it returns 1.
+	   for anything dividable by 2, it returns 2.
+	   for 3 it returns 1 so that the next request size can be 4.
+	   and so on...
+	*/
+
+	/* Only consider the lower order bits up to the size of max_bio_bits.
+	 * This prevents overflows when converting to int. */
+	bit = bit & (max_bio_bits - 1);
+
+	if (bit == 0)
+		return max_bio_bits;
+
+	return 1 << __ffs(bit);
+}
+
+static int round_to_powerof_2(int value)
+{
+	int l2 = fls(value) - 1;
+	int smaller = 1 << l2;
+	int bigger = smaller << 1;
+
+	if (value == 0)
+		return 0;
+
+	return value - smaller < bigger - value ? smaller : bigger;
+}
+
+static bool adjacent(sector_t sector1, int size, sector_t sector2)
+{
+	return sector1 + (size >> SECTOR_SHIFT) == sector2;
+}
+
+/* make_resync_request() - initiate resync requests as required
+ *
+ * Request handling flow:
+ *
+ *                     checksum resync
+ * make_resync_request --------+
+ *       |                     v
+ *       |               read_for_csum
+ *       |                     |
+ *       |                     v
+ *       |          drbd_submit_peer_request
+ *       |                     |
+ *       |                    ... backing device
+ *       |                     |
+ *       |                     v
+ *       |           drbd_peer_request_endio
+ *       |                     |
+ *       |                     v
+ *       |          drbd_endio_read_sec_final
+ *       |                     |
+ *       V                    ... sender_work
+ * make_one_resync_request     |
+ *       |                     v
+ *       +---------------- w_e_send_csum
+ *       |
+ *       v                             conflict
+ * drbd_conflict_send_resync_request -------+
+ *       |                ^                 |
+ *       |                |                ...
+ *       |                |                 |
+ *       |                |                 v
+ *       v                +---- drbd_do_submit_conflict
+ * send_resync_request
+ *       |
+ *      ... via peer
+ *       |
+ *       +----------------------------+
+ *       |                            |
+ *       v                            v
+ * receive_RSDataReply      receive_rs_deallocated
+ *       |                            |
+ *       |                           ... using list resync_requests
+ *       |                            |
+ *       v                            v
+ * recv_resync_read        drbd_process_rs_discards
+ *       |                            |
+ *       |                            v
+ *       +----------------- drbd_submit_rs_discard
+ *       |
+ *       v                             conflict
+ * drbd_conflict_submit_resync_request -----+
+ *       |                ^                 |
+ *       |                |                ...
+ *       |                |                 |
+ *       |                |                 v
+ *       v                +---- drbd_do_submit_conflict
+ * drbd_submit_peer_request
+ *       |
+ *      ... backing device
+ *       |
+ *       v
+ * drbd_peer_request_endio
+ *       |
+ *       v
+ * drbd_endio_write_sec_final
+ *       |
+ *      ... done_ee
+ *       |
+ *       v
+ * drbd_finish_peer_reqs
+ *       |
+ *       v
+ * e_end_resync_block
+ *       |
+ *       v
+ * drbd_resync_request_complete
+ */
+static int make_resync_request(struct drbd_peer_device *peer_device, int cancel)
+{
+	int optimal_bits_alignment, optimal_bits_rate, discard_granularity = 0;
+	int number = 0, rollback_i, size = 0, i = 0, optimal_bits;
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap; /* deref bm members only after get_ldev() */
+	const sector_t capacity = get_capacity(device->vdisk);
+	bool request_ok = true;
+	unsigned long bit;
+	sector_t sector, prev_sector = 0;
+	unsigned int peer_bm_block_shift = peer_device->bm_block_shift;
+	unsigned int bm_block_shift, bits_per_peer_bit;
+
+	if (unlikely(cancel))
+		return 0;
+
+	if (test_bit(SYNC_TARGET_TO_BEHIND, &peer_device->flags)) {
+		/* If a P_RS_CANCEL_AHEAD on control socket overtook the
+		 * already queued data and state change to Ahead/Behind,
+		 * don't add more resync requests, just wait it out. */
+		drbd_info_ratelimit(peer_device, "peer pulled ahead during resync\n");
+		return 0;
+	}
+
+	if (drbd_resync_check_finished(peer_device))
+		return 0;
+
+	if (!get_ldev(device)) {
+		/* Since we only need to access device->rsync a
+		   get_ldev_if_state(device,D_FAILED) would be sufficient, but
+		   to continue resync with a broken disk makes no sense at
+		   all */
+		drbd_err(device, "Disk broke down during resync!\n");
+		return 0;
+	}
+
+	if (send_buffer_half_full(peer_device)) {
+		/* We still want to reschedule ourselves, so do not return. */
+		goto skip_request;
+	}
+
+	if (connection->agreed_features & DRBD_FF_THIN_RESYNC) {
+		rcu_read_lock();
+		discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity;
+		rcu_read_unlock();
+	}
+
+	bm_block_shift = bm->bm_block_shift;
+	number = drbd_rs_number_requests(peer_device);
+	if (number < discard_granularity >> bm_block_shift)
+		number = discard_granularity >> bm_block_shift;
+
+	/*
+	 * Drain resync requests when we jump back to avoid conflicts that are
+	 * resolved in an arbitrary order, leading to an unexpected ordering of
+	 * requests being completed.
+	 */
+	if (test_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags) &&
+			peer_device->rs_in_flight > 0) {
+		/*
+		 * The rs_in_flight counter does not include discards waiting
+		 * to be merged. Hence we may jump back while there are
+		 * discards waiting to be merged. In this situation, we may
+		 * make a resync request that conflicts with a discard. Allow
+		 * the discard to be merged here so that the conflict is
+		 * resolved.
+		 */
+		drbd_process_rs_discards(peer_device, false);
+		goto skip_request;
+	}
+
+	/* don't let rs_sectors_came_in() re-schedule us "early"
+	 * just because the first reply came "fast", ... */
+	peer_device->rs_in_flight += bm_bit_to_sect(device->bitmap, number);
+
+	bits_per_peer_bit = peer_bm_block_shift > bm_block_shift ?
+		1 << (peer_bm_block_shift - bm_block_shift) : 1;
+
+	clear_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags);
+	for (; i < number; i += bits_per_peer_bit) {
+		int err;
+
+		/* If we are aborting the requests or the peer is canceling
+		 * them, there is no need to flood the connection with
+		 * requests. Back off now. */
+		if (i > 0 && test_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags)) {
+			request_ok = false;
+			goto request_done;
+		}
+
+		if ((number - i) < discard_granularity >> bm_block_shift)
+			goto request_done;
+
+		bit  = drbd_bm_find_next(peer_device, peer_device->resync_next_bit);
+		if (bit == DRBD_END_OF_BITMAP) {
+			peer_device->resync_next_bit = drbd_bm_bits(device);
+			goto request_done;
+		}
+
+		bit = ALIGN_DOWN(bit, bits_per_peer_bit);
+		sector = bm_bit_to_sect(bm, bit);
+
+		if (drbd_rs_c_min_rate_throttle(peer_device)) {
+			peer_device->resync_next_bit = bit;
+			goto request_done;
+		}
+
+		if (adjacent(prev_sector, size, sector) && (number - i) < size >> bm_block_shift) {
+			/* When making requests in an out-of-sync area, ensure that the size
+			   of successive requests does not decrease. This allows the next
+			   make_resync_request call to start with optimal alignment. */
+			goto request_done;
+		}
+
+		prev_sector = sector;
+		size = bm_block_size(bm) * bits_per_peer_bit;
+		optimal_bits_alignment = optimal_bits_for_alignment(bit,
+						max(peer_bm_block_shift, bm_block_shift));
+		optimal_bits_rate = round_to_powerof_2(number - i);
+		optimal_bits = min(optimal_bits_alignment, optimal_bits_rate) - 1;
+
+		/* try to find some adjacent bits. */
+		rollback_i = i;
+		while (optimal_bits-- > 0) {
+			if (discard_granularity && size == discard_granularity)
+				break;
+
+			if (drbd_bm_count_bits(device, peer_device->bitmap_index,
+					       bit + bits_per_peer_bit,
+					       bit + bits_per_peer_bit * 2 - 1) == 0)
+				break;
+			size += bm_block_size(bm) * bits_per_peer_bit;
+			bit += bits_per_peer_bit;
+			i += bits_per_peer_bit;
+		}
+
+		/* set the offset to start the next drbd_bm_find_next from */
+		peer_device->resync_next_bit = bit + bits_per_peer_bit;
+
+		/* adjust very last sectors, in case we are oddly sized */
+		if (sector + (size>>9) > capacity)
+			size = (capacity-sector)<<9;
+
+		if (peer_device->use_csums)
+			err = read_for_csum(peer_device, sector, size);
+		else
+			err = make_one_resync_request(peer_device, discard_granularity, sector, size);
+
+		switch (err) {
+		case -EIO: /* Disk failure */
+			put_ldev(device);
+			return -EIO;
+		case -EAGAIN: /* allocation failed, or ldev busy */
+			set_bit(RS_REQUEST_UNSUCCESSFUL, &peer_device->flags);
+			peer_device->resync_next_bit = bm_sect_to_bit(bm, sector);
+			i = rollback_i;
+			goto request_done;
+		case 0:
+			/* everything ok */
+			break;
+		default:
+			BUG();
+		}
+	}
+
+request_done:
+	/* ... but do a correction, in case we had to break/goto request_done; */
+	peer_device->rs_in_flight -= (number - i) * bm_sect_per_bit(bm);
+
+	if (peer_device->resync_next_bit >= drbd_bm_bits(device)) {
+		/*
+		 * Last resync request sent in this pass. There will be no
+		 * replies for subsequent sectors so discard merging should
+		 * stop here.
+		 */
+		drbd_last_resync_request(peer_device, false);
+	}
+
+skip_request:
+	/* Always reschedule ourselves as a form of polling to detect the end of a resync pass. */
+	mod_timer(&peer_device->resync_timer, jiffies + resync_delay(request_ok, number, i));
+
+	if (peer_device->rs_in_flight > 0 && request_ok) {
+		int rs_sect_in = atomic_read(&peer_device->rs_sect_in);
+
+		if (rs_sect_in >= peer_device->rs_in_flight) {
+			/*
+			 * In case replies were received before correction to
+			 * rs_in_flight, consider whether to schedule ourselves
+			 * early.
+			 */
+			drbd_rs_all_in_flight_came_back(peer_device, rs_sect_in);
+		}
+	}
+	put_ldev(device);
+	return 0;
+}
+
+static void send_ov_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct dagtag_find_result dagtag_result;
+	enum drbd_packet cmd = peer_device->connection->agreed_features & DRBD_FF_RESYNC_DAGTAG ?
+		P_OV_DAGTAG_REQ : P_OV_REQUEST;
+
+	inc_rs_pending(peer_device);
+
+	dagtag_result = find_current_dagtag(peer_device->device->resource);
+	if (dagtag_result.err) {
+		change_cstate(peer_device->connection, C_DISCONNECTING, CS_HARD);
+		return;
+	}
+
+	drbd_send_rs_request(peer_device, cmd,
+			peer_req->i.sector, peer_req->i.size, peer_req->block_id,
+			dagtag_result.node_id, dagtag_result.dagtag);
+}
+
+static void drbd_conflict_send_ov_request(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+
+	spin_lock_irq(&device->interval_lock);
+	if (drbd_find_conflict(device, &peer_req->i, 0))
+		set_bit(INTERVAL_CONFLICT, &peer_req->i.flags);
+	drbd_insert_interval(&device->requests, &peer_req->i);
+	set_bit(INTERVAL_SENT, &peer_req->i.flags);
+	/* Mark as submitted now, since OV requests do not have a second
+	 * conflict resolution stage when the reply is received. */
+	set_bit(INTERVAL_SUBMITTED, &peer_req->i.flags);
+	spin_unlock_irq(&device->interval_lock);
+
+	/* If there were conflicts we will skip the block. However, we send a
+	 * request anyway because the protocol doesn't include any way to mark
+	 * a block as skipped without having sent any request. */
+	send_ov_request(peer_req);
+}
+
+/* make_ov_request() - initiate online verify requests as required
+ *
+ * Request handling flow:
+ *
+ * make_ov_request
+ *        |
+ *        v
+ * drbd_conflict_send_ov_request
+ *        |
+ *        v
+ * send_ov_request
+ *        |
+ *       ... via peer
+ *        |
+ *        v
+ * receive_dagtag_ov_reply
+ *        |
+ *        v
+ * receive_common_ov_reply
+ *        |
+ *        v              dagtag waiting
+ * drbd_peer_resync_read --------------+
+ *        |                            |
+ *        |                           ... dagtag_wait_ee
+ *        |                            |
+ *        |                            v
+ *        +--------------- release_dagtag_wait
+ *        |
+ *        v
+ * drbd_conflict_submit_peer_read
+ *        |
+ *        v
+ * drbd_submit_peer_request
+ *        |
+ *       ... backing device
+ *        |
+ *        v
+ * drbd_peer_request_endio
+ *        |
+ *        v
+ * drbd_endio_read_sec_final
+ *        |
+ *       ... sender_work
+ *        |
+ *        v
+ * w_e_end_ov_reply
+ */
+static int make_ov_request(struct drbd_peer_device *peer_device, int cancel)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap;
+	struct drbd_connection *connection = peer_device->connection;
+	int number, i, size;
+	sector_t sector;
+	const sector_t capacity = get_capacity(device->vdisk);
+	bool stop_sector_reached = false;
+
+	if (unlikely(cancel))
+		return 1;
+
+	number = drbd_rs_number_requests(peer_device);
+	sector = peer_device->ov_position;
+
+	/* don't let rs_sectors_came_in() re-schedule us "early"
+	 * just because the first reply came "fast", ... */
+	peer_device->rs_in_flight += bm_bit_to_sect(bm, number);
+	for (i = 0; i < number; i++) {
+		struct drbd_peer_request *peer_req;
+
+		if (sector >= capacity)
+			break;
+
+		/* We check for "finished" only in the reply path:
+		 * w_e_end_ov_reply().
+		 * We need to send at least one request out. */
+		stop_sector_reached = sector > peer_device->ov_start_sector
+			&& verify_can_do_stop_sector(peer_device)
+			&& sector >= peer_device->ov_stop_sector;
+		if (stop_sector_reached)
+			break;
+
+		if (drbd_rs_c_min_rate_throttle(peer_device))
+			break;
+
+		size = bm_block_size(bm);
+		if (sector + (size>>9) > capacity)
+			size = (capacity-sector)<<9;
+
+		/* Do not wait if no memory is immediately available.  */
+		peer_req = drbd_alloc_peer_req(peer_device, GFP_TRY & ~__GFP_RECLAIM);
+		if (!peer_req) {
+			drbd_err(device, "Could not allocate online verify request\n");
+			put_ldev(device);
+			return 0;
+		}
+
+		peer_req->i.size = size;
+		peer_req->i.sector = sector;
+		peer_req->i.type = INTERVAL_OV_READ_SOURCE;
+
+		spin_lock_irq(&connection->peer_reqs_lock);
+		list_add_tail(&peer_req->recv_order, &connection->peer_reads);
+		peer_req->flags |= EE_ON_RECV_ORDER;
+		spin_unlock_irq(&connection->peer_reqs_lock);
+
+		drbd_conflict_send_ov_request(peer_req);
+
+		sector += bm_sect_per_bit(bm);
+	}
+	/* ... but do a correction, in case we had to break; ... */
+	peer_device->rs_in_flight -= bm_bit_to_sect(bm, number-i);
+	peer_device->ov_position = sector;
+	if (stop_sector_reached)
+		return 1;
+	/* ... and in case that raced with the receiver,
+	 * reschedule ourselves right now */
+	if (i > 0 && atomic_read(&peer_device->rs_sect_in) >= peer_device->rs_in_flight)
+		drbd_queue_work_if_unqueued(
+			&peer_device->connection->sender_work,
+			&peer_device->resync_work);
+	else
+		mod_timer(&peer_device->resync_timer, jiffies + resync_delay(true, number, i));
+	return 1;
+}
+
+struct resync_finished_work {
+	struct drbd_peer_device_work pdw;
+	enum drbd_disk_state new_peer_disk_state;
+};
+
+static int w_resync_finished(struct drbd_work *w, int cancel)
+{
+	struct resync_finished_work *rfw = container_of(
+		container_of(w, struct drbd_peer_device_work, w),
+		struct resync_finished_work, pdw);
+
+	if (!cancel)
+		drbd_resync_finished(rfw->pdw.peer_device, rfw->new_peer_disk_state);
+	kfree(rfw);
+
+	return 0;
+}
+
+static long ping_timeout(struct drbd_connection *connection)
+{
+	struct net_conf *nc;
+	long timeout;
+
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	timeout = nc->ping_timeo * HZ / 10;
+	rcu_read_unlock();
+
+	return timeout;
+}
+
+static int send_ping_peer(struct drbd_connection *connection)
+{
+	bool was_pending = test_and_set_bit(PING_PENDING, &connection->flags);
+	int err = 0;
+
+	if (!was_pending) {
+		err = drbd_send_ping(connection);
+		if (err)
+			change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+	}
+
+	return err;
+}
+
+void drbd_ping_peer(struct drbd_connection *connection)
+{
+	long r, timeout = ping_timeout(connection);
+	int err;
+
+	err = send_ping_peer(connection);
+	if (err)
+		return;
+
+	r = wait_event_timeout(connection->resource->state_wait,
+			       !test_bit(PING_PENDING, &connection->flags) ||
+			       connection->cstate[NOW] < C_CONNECTED,
+			       timeout);
+	if (r > 0)
+		return;
+
+	drbd_warn(connection, "PingAck did not arrive in time\n");
+	change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+}
+
+/* caller needs to hold rcu_read_lock, state_rwlock, adm_mutex or conf_update */
+struct drbd_peer_device *peer_device_by_node_id(struct drbd_device *device, int node_id)
+{
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->node_id == node_id)
+			return peer_device;
+	}
+
+	return NULL;
+}
+
+static void __outdate_peer_disk_by_mask(struct drbd_device *device, u64 nodes)
+{
+	struct drbd_peer_device *peer_device;
+	int node_id;
+
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		if (!(nodes & NODE_MASK(node_id)))
+			continue;
+		peer_device = peer_device_by_node_id(device, node_id);
+		if (peer_device && peer_device->disk_state[NEW] >= D_CONSISTENT)
+			__change_peer_disk_state(peer_device, D_OUTDATED);
+	}
+}
+
+/* An annoying corner case is if we are resync target towards a bunch
+   of nodes. One of the resyncs finished as STABLE_RESYNC, the others
+   as UNSTABLE_RESYNC. */
+static bool was_resync_stable(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *device = peer_device->device;
+
+	if (test_bit(UNSTABLE_RESYNC, &peer_device->flags) &&
+	    !test_bit(STABLE_RESYNC, &device->flags))
+		return false;
+
+	set_bit(STABLE_RESYNC, &device->flags);
+	/* that STABLE_RESYNC bit gets reset if in any other ongoing resync
+	   we receive something from a resync source that is marked with
+	   UNSTABLE RESYNC. */
+
+	return true;
+}
+
+static u64 __cancel_other_resyncs(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	u64 target_m = 0;
+
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->repl_state[NEW] == L_PAUSED_SYNC_T) {
+			target_m |= NODE_MASK(peer_device->node_id);
+			__change_repl_state(peer_device, L_ESTABLISHED);
+		}
+	}
+
+	return target_m;
+}
+
+static void resync_again(struct drbd_device *device, u64 source_m, u64 target_m)
+{
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->resync_again) {
+			u64 m = NODE_MASK(peer_device->node_id);
+			enum drbd_repl_state new_repl_state =
+				source_m & m ? L_WF_BITMAP_S :
+				target_m & m ? L_WF_BITMAP_T :
+				L_ESTABLISHED;
+
+			if (new_repl_state != L_ESTABLISHED) {
+				peer_device->resync_again--;
+				begin_state_change_locked(device->resource, CS_VERBOSE);
+				__change_repl_state(peer_device, new_repl_state);
+				end_state_change_locked(device->resource, "resync-again");
+			}
+		}
+	}
+}
+
+static void init_resync_stable_bits(struct drbd_peer_device *first_target_pd)
+{
+	struct drbd_device *device = first_target_pd->device;
+	struct drbd_peer_device *peer_device;
+
+	clear_bit(UNSTABLE_RESYNC, &first_target_pd->flags);
+
+	/* Clear the device wide STABLE_RESYNC flag when becoming
+	   resync target on the first peer_device. */
+	for_each_peer_device(peer_device, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NOW];
+		if (peer_device == first_target_pd)
+			continue;
+		if (repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T)
+			return;
+	}
+	clear_bit(STABLE_RESYNC, &device->flags);
+}
+
+static void after_reconciliation_resync(struct drbd_connection *connection)
+{
+	struct drbd_connection *lost_peer =
+		drbd_get_connection_by_node_id(connection->resource,
+					       connection->after_reconciliation.lost_node_id);
+
+	if (lost_peer) {
+		if (lost_peer->cstate[NOW] < C_CONNECTED)
+			atomic64_set(&lost_peer->last_dagtag_sector,
+				connection->after_reconciliation.dagtag_sector);
+
+		kref_put(&lost_peer->kref, drbd_destroy_connection);
+	}
+
+	connection->after_reconciliation.lost_node_id = -1;
+}
+
+static void try_to_get_resynced_from_primary(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	struct drbd_peer_device *peer_device;
+	struct drbd_connection *connection;
+
+	read_lock_irq(&resource->state_rwlock);
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->connection->peer_role[NEW] == R_PRIMARY &&
+		    peer_device->disk_state[NEW] == D_UP_TO_DATE)
+			goto found;
+	}
+	peer_device = NULL;
+found:
+	read_unlock_irq(&resource->state_rwlock);
+
+	if (!peer_device)
+		return;
+
+	connection = peer_device->connection;
+	if (connection->agreed_pro_version < 118) {
+		drbd_warn(connection,
+			  "peer is lower than protocol vers 118, reconnecting to get resynced\n");
+		change_cstate(connection, C_PROTOCOL_ERROR, CS_HARD);
+		return;
+	}
+
+	drbd_send_uuids(peer_device, 0, 0);
+	drbd_start_resync(peer_device, L_SYNC_TARGET, "resync-from-primary");
+}
+
+static void queue_resync_finished(struct drbd_peer_device *peer_device, enum drbd_disk_state new_peer_disk_state)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct resync_finished_work *rfw;
+
+	rfw = kmalloc_obj(struct resync_finished_work, GFP_ATOMIC);
+	if (!rfw) {
+		drbd_err(peer_device, "Warn failed to kmalloc(dw).\n");
+		return;
+	}
+
+	rfw->pdw.w.cb = w_resync_finished;
+	rfw->pdw.peer_device = peer_device;
+	rfw->new_peer_disk_state = new_peer_disk_state;
+	drbd_queue_work(&connection->sender_work, &rfw->pdw.w);
+}
+
+void drbd_resync_finished(struct drbd_peer_device *peer_device,
+			 enum drbd_disk_state new_peer_disk_state)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	enum drbd_repl_state *repl_state = peer_device->repl_state;
+	enum drbd_repl_state old_repl_state = L_ESTABLISHED;
+	bool try_to_get_resynced_from_primary_flag = false;
+	u64 source_m = 0, target_m = 0;
+	unsigned long db, dt, dbdt;
+	unsigned long n_oos;
+	char *khelper_cmd = NULL;
+	int verify_done = 0;
+	bool aborted = false;
+	int bm_block_shift = device->last_bm_block_shift;
+	sector_t final_peers_in_sync_end;
+
+	if (repl_state[NOW] == L_SYNC_SOURCE || repl_state[NOW] == L_PAUSED_SYNC_S) {
+		/* Make sure all queued w_update_peers() executed. */
+		if (current == device->resource->worker.task) {
+			queue_resync_finished(peer_device, new_peer_disk_state);
+			return;
+		} else {
+			drbd_flush_workqueue(&device->resource->work);
+		}
+	}
+
+	if (!down_write_trylock(&device->uuid_sem)) {
+		if (current == device->resource->worker.task) {
+			queue_resync_finished(peer_device, new_peer_disk_state);
+			return;
+		} else {
+			down_write(&device->uuid_sem);
+		}
+	}
+
+	dt = (jiffies - peer_device->rs_start - peer_device->rs_paused) / HZ;
+	if (dt <= 0)
+		dt = 1;
+	db = peer_device->rs_total;
+	/* adjust for verify start and stop sectors, respective reached position */
+	if (repl_state[NOW] == L_VERIFY_S || repl_state[NOW] == L_VERIFY_T)
+		db -= atomic64_read(&peer_device->ov_left);
+
+	dbdt = bit_to_kb(db/dt, bm_block_shift);
+	peer_device->rs_paused /= HZ;
+
+	if (!get_ldev(device)) {
+		up_write(&device->uuid_sem);
+		goto out;
+	}
+
+	drbd_ping_peer(connection);
+
+	write_lock_irq(&device->resource->state_rwlock);
+	begin_state_change_locked(device->resource, CS_VERBOSE);
+	old_repl_state = repl_state[NOW];
+
+	verify_done = (repl_state[NOW] == L_VERIFY_S || repl_state[NOW] == L_VERIFY_T);
+
+	/* This protects us against multiple calls (that can happen in the presence
+	   of application IO), and against connectivity loss just before we arrive here. */
+	if (peer_device->repl_state[NOW] <= L_ESTABLISHED)
+		goto out_unlock;
+
+	/*
+	 * This protects us against a race with the peer when finishing a
+	 * resync at the same time as entering Ahead-Behind mode.
+	 */
+	if (peer_device->repl_state[NOW] == L_BEHIND)
+		goto out_unlock;
+
+	peer_device->resync_active[NEW] = false;
+	__change_repl_state(peer_device, L_ESTABLISHED);
+
+	aborted = device->disk_state[NOW] == D_OUTDATED && new_peer_disk_state == D_INCONSISTENT;
+	{
+	char tmp[sizeof(" but 01234567890123456789 4k blocks skipped")] = "";
+	if (verify_done && peer_device->ov_skipped)
+		snprintf(tmp, sizeof(tmp), " but %lu %lluk blocks skipped",
+			peer_device->ov_skipped, bit_to_kb(1, bm_block_shift));
+	drbd_info(peer_device, "%s %s%s (total %lu sec; paused %lu sec; %lu K/sec)\n",
+		  verify_done ? "Online verify" : "Resync",
+		  aborted ? "aborted" : "done", tmp,
+		  dt + peer_device->rs_paused, peer_device->rs_paused, dbdt);
+	}
+
+	n_oos = drbd_bm_total_weight(peer_device);
+
+	if (repl_state[NOW] == L_VERIFY_S || repl_state[NOW] == L_VERIFY_T) {
+		if (n_oos) {
+			drbd_alert(peer_device, "Online verify found %lu %lluk blocks out of sync!\n",
+			      n_oos, bit_to_kb(1, bm_block_shift));
+			khelper_cmd = "out-of-sync";
+		}
+	} else {
+		if (!aborted && peer_device->rs_failed == 0 && n_oos != 0)
+			drbd_warn(peer_device, "expected n_oos:%lu to be 0\n", n_oos);
+
+		if (repl_state[NOW] == L_SYNC_TARGET || repl_state[NOW] == L_PAUSED_SYNC_T)
+			khelper_cmd = "after-resync-target";
+
+		if (peer_device->use_csums && peer_device->rs_total) {
+			const unsigned long s = peer_device->rs_same_csum;
+			const unsigned long t = peer_device->rs_total;
+			const int ratio =
+				(t == 0)     ? 0 :
+			(t < 100000) ? ((s*100)/t) : (s/(t/100));
+			drbd_info(peer_device, "%u %% had equal checksums, eliminated: %lluK; "
+			     "transferred %lluK total %lluK\n",
+			     ratio,
+			     bit_to_kb(peer_device->rs_same_csum, bm_block_shift),
+			     bit_to_kb(peer_device->rs_total - peer_device->rs_same_csum,
+					bm_block_shift),
+			     bit_to_kb(peer_device->rs_total, bm_block_shift));
+		}
+	}
+
+	if (peer_device->rs_failed) {
+		drbd_info(peer_device, "            %lu failed blocks\n", peer_device->rs_failed);
+
+		if (repl_state[NOW] == L_SYNC_TARGET || repl_state[NOW] == L_PAUSED_SYNC_T) {
+			__change_disk_state(device, D_INCONSISTENT);
+			__change_peer_disk_state(peer_device, D_UP_TO_DATE);
+		} else {
+			__change_disk_state(device, D_UP_TO_DATE);
+			__change_peer_disk_state(peer_device, D_INCONSISTENT);
+		}
+	} else {
+		if (repl_state[NOW] == L_SYNC_TARGET || repl_state[NOW] == L_PAUSED_SYNC_T) {
+			bool stable_resync = was_resync_stable(peer_device);
+			if (stable_resync) {
+				enum drbd_disk_state new_disk_state = peer_device->disk_state[NOW];
+				if (new_disk_state < D_UP_TO_DATE &&
+				    test_bit(SYNC_SRC_CRASHED_PRI, &peer_device->flags)) {
+					try_to_get_resynced_from_primary_flag = true;
+					set_bit(CRASHED_PRIMARY, &device->flags);
+				}
+				__change_disk_state(device, new_disk_state);
+			}
+
+			if (device->disk_state[NEW] == D_UP_TO_DATE)
+				target_m = __cancel_other_resyncs(device);
+
+			if (stable_resync && test_bit(UUIDS_RECEIVED, &peer_device->flags)) {
+				const int node_id = device->resource->res_opts.node_id;
+				int i;
+
+				u64 newer = drbd_uuid_resync_finished(peer_device);
+				__outdate_peer_disk_by_mask(device, newer);
+				drbd_print_uuids(peer_device, "updated UUIDs");
+
+				/* Now the two UUID sets are equal, update what we
+				 * know of the peer. */
+				peer_device->current_uuid = drbd_current_uuid(device);
+				peer_device->bitmap_uuids[node_id] = drbd_bitmap_uuid(peer_device);
+				for (i = 0; i < ARRAY_SIZE(peer_device->history_uuids); i++)
+					peer_device->history_uuids[i] =
+						drbd_history_uuid(device, i);
+			} else {
+				if (!test_bit(UUIDS_RECEIVED, &peer_device->flags))
+					drbd_err(peer_device, "BUG: uuids were not received!\n");
+
+				if (test_bit(UNSTABLE_RESYNC, &peer_device->flags))
+					drbd_info(peer_device, "Peer was unstable during resync\n");
+			}
+		} else if (repl_state[NOW] == L_SYNC_SOURCE || repl_state[NOW] == L_PAUSED_SYNC_S) {
+			if (new_peer_disk_state != D_MASK)
+				__change_peer_disk_state(peer_device, new_peer_disk_state);
+			if (peer_device->connection->agreed_pro_version < 110) {
+				drbd_uuid_set_bitmap(peer_device, 0UL);
+				drbd_print_uuids(peer_device, "updated UUIDs");
+			}
+		}
+	}
+
+out_unlock:
+	end_state_change_locked(device->resource, "resync-finished");
+
+	put_ldev(device);
+
+	peer_device->rs_total  = 0;
+	peer_device->rs_failed = 0;
+	peer_device->rs_paused = 0;
+
+	if (old_repl_state == L_SYNC_TARGET || old_repl_state == L_PAUSED_SYNC_T)
+		target_m |= NODE_MASK(peer_device->node_id);
+	else if (old_repl_state == L_SYNC_SOURCE || old_repl_state == L_PAUSED_SYNC_S)
+		source_m |= NODE_MASK(peer_device->node_id);
+
+	resync_again(device, source_m, target_m);
+	write_unlock_irq(&device->resource->state_rwlock);
+	up_write(&device->uuid_sem);
+	if (connection->after_reconciliation.lost_node_id != -1)
+		after_reconciliation_resync(connection);
+
+	/* Potentially send final P_PEERS_IN_SYNC. */
+	final_peers_in_sync_end = min(get_capacity(device->vdisk),
+			(peer_device->last_peers_in_sync_end | PEERS_IN_SYNC_STEP_SECT_MASK) + 1);
+	drbd_queue_update_peers(peer_device,
+			peer_device->last_peers_in_sync_end, final_peers_in_sync_end);
+
+out:
+	/* reset start sector, if we reached end of device */
+	if (verify_done && atomic64_read(&peer_device->ov_left) == 0)
+		peer_device->ov_start_sector = 0;
+
+	drbd_md_sync_if_dirty(device);
+
+	if (khelper_cmd)
+		drbd_maybe_khelper(device, connection, khelper_cmd);
+
+	if (try_to_get_resynced_from_primary_flag)
+		try_to_get_resynced_from_primary(device);
+}
+
+
+/**
+ * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST
+ * @w:		work object.
+ * @cancel:	The connection will be closed anyways
+ */
+int w_e_end_data_req(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	int err;
+
+	if (unlikely(cancel)) {
+		err = 0;
+		goto out;
+	}
+
+	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
+		err = drbd_send_block(peer_device, P_DATA_REPLY, peer_req);
+	} else {
+		drbd_err_ratelimit(peer_device, "Sending NegDReply. sector=%llus.\n",
+			    (unsigned long long)peer_req->i.sector);
+
+		err = drbd_send_ack(peer_device, P_NEG_DREPLY, peer_req);
+	}
+	if (unlikely(err))
+		drbd_err(peer_device, "drbd_send_block() failed\n");
+
+out:
+	dec_unacked(peer_device);
+	drbd_free_peer_req(peer_req);
+
+	return err;
+}
+
+void
+drbd_resync_read_req_mod(struct drbd_peer_request *peer_req, enum drbd_interval_flags bit_to_set)
+{
+	const unsigned long done_mask = 1UL << INTERVAL_SENT | 1UL << INTERVAL_RECEIVED;
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	unsigned long nflags, oflags, new_flag;
+
+	new_flag = 1UL << bit_to_set;
+	if (!(new_flag & done_mask))
+		drbd_err(peer_device, "BUG: %s: Unexpected flag 0x%lx\n", __func__, new_flag);
+
+	do {
+		oflags = READ_ONCE(peer_req->i.flags);
+		nflags = oflags | new_flag;
+	} while (cmpxchg(&peer_req->i.flags, oflags, nflags) != oflags);
+
+	if (new_flag & oflags)
+		drbd_err(peer_device, "BUG: %s: Flag 0x%lx already set\n", __func__, new_flag);
+
+	if ((nflags & done_mask) == done_mask) {
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+}
+
+static bool all_zero(struct drbd_peer_request *peer_req)
+{
+	struct page *page = peer_req->page_chain.head;
+	unsigned int len = peer_req->i.size;
+
+	page_chain_for_each(page) {
+		unsigned int l = min_t(unsigned int, len, PAGE_SIZE);
+		unsigned int i, words = l / sizeof(long);
+		unsigned long *d;
+
+		d = kmap_atomic(page);
+		for (i = 0; i < words; i++) {
+			if (d[i]) {
+				kunmap_atomic(d);
+				return false;
+			}
+		}
+		kunmap_atomic(d);
+		len -= l;
+	}
+
+	return true;
+}
+
+static bool al_resync_extent_active(struct drbd_device *device, sector_t sector, unsigned int size)
+{
+	sector_t resync_extent_sector = sector & ~LEGACY_BM_EXT_SECT_MASK;
+	sector_t end_sector = sector + (size >> SECTOR_SHIFT);
+	sector_t resync_extent_end_sector =
+		(end_sector + LEGACY_BM_EXT_SECT_MASK) & ~LEGACY_BM_EXT_SECT_MASK;
+	return drbd_al_active(device,
+			resync_extent_sector,
+			(resync_extent_end_sector - resync_extent_sector) << SECTOR_SHIFT);
+}
+
+static int drbd_rs_reply(struct drbd_peer_device *peer_device, struct drbd_peer_request *peer_req, bool *expect_ack)
+{
+	struct drbd_connection *connection = peer_device->connection;
+	struct drbd_device *device = peer_device->device;
+	int err;
+	bool eq = false;
+
+	if (peer_req->flags & EE_HAS_DIGEST) {
+		struct digest_info *di = peer_req->digest;
+		int digest_size;
+		void *digest = NULL;
+
+		/* quick hack to try to avoid a race against reconfiguration.
+		 * a real fix would be much more involved,
+		 * introducing more locking mechanisms */
+		if (connection->csums_tfm) {
+			digest_size = crypto_shash_digestsize(connection->csums_tfm);
+			D_ASSERT(device, digest_size == di->digest_size);
+			digest = kmalloc(digest_size, GFP_NOIO);
+			if (digest) {
+				drbd_csum_pages(connection->csums_tfm, peer_req->page_chain.head, digest);
+				eq = !memcmp(digest, di->digest, digest_size);
+				kfree(digest);
+			}
+		}
+
+		peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */
+		kfree(di);
+	}
+
+	if (eq) {
+		drbd_set_in_sync(peer_device, peer_req->i.sector, peer_req->i.size);
+		/* rs_same_csums unit is BM_BLOCK_SIZE */
+		peer_device->rs_same_csum += peer_req->i.size >> device->ldev->md.bm_block_shift;
+		err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req);
+	} else {
+		inc_rs_pending(peer_device);
+		/*
+		 * If we send back as P_RS_DEALLOCATED,
+		 * this is overestimating "in-flight" accounting.
+		 * But needed to be properly balanced with
+		 * the atomic_sub() in got_RSWriteAck.
+		 */
+		atomic_add(peer_req->i.size >> 9, &connection->rs_in_flight);
+
+		spin_lock_irq(&connection->peer_reqs_lock);
+		list_add_tail(&peer_req->w.list, &connection->resync_ack_ee);
+		spin_unlock_irq(&connection->peer_reqs_lock);
+
+		if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req)) {
+			err = drbd_send_rs_deallocated(peer_device, peer_req);
+		} else {
+			err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req);
+		}
+		*expect_ack = true;
+
+		drbd_free_page_chain(&connection->transport, &peer_req->page_chain);
+
+		drbd_resync_read_req_mod(peer_req, INTERVAL_SENT);
+		peer_req = NULL;
+	}
+
+	return err;
+}
+
+/**
+ * w_e_end_rsdata_req() - Reply to a resync request.
+ * @w:		work object.
+ * @cancel:	The connection is being closed
+ *
+ * Worker callback to send P_RS_DATA_REPLY or a related packet after completing
+ * a resync read.
+ *
+ * Return: Error code or 0 on success.
+ */
+int w_e_end_rsdata_req(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	int err;
+	bool expect_ack = false;
+
+	if (unlikely(cancel) || connection->cstate[NOW] < C_CONNECTED) {
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+		dec_unacked(peer_device);
+		return 0;
+	}
+
+	if (peer_device->repl_state[NOW] == L_AHEAD) {
+		err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
+	} else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
+		if (unlikely(peer_device->disk_state[NOW] < D_INCONSISTENT)) {
+			if (connection->agreed_features & DRBD_FF_RESYNC_DAGTAG) {
+				drbd_err_ratelimit(peer_device,
+						"Sending P_RS_CANCEL, partner DISKLESS!\n");
+				err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
+			} else {
+				/*
+				 * A peer that does not support DRBD_FF_RESYNC_DAGTAG does not
+				 * expect to receive P_RS_CANCEL after losing its disk.
+				 */
+				drbd_err_ratelimit(peer_device,
+						"Not sending resync reply, partner DISKLESS!\n");
+				err = 0;
+			}
+		} else if (connection->agreed_pro_version >= 110 &&
+				!(connection->agreed_features & DRBD_FF_RESYNC_DAGTAG) &&
+				al_resync_extent_active(peer_device->device,
+					peer_req->i.sector, peer_req->i.size)) {
+			/* DRBD versions without DRBD_FF_RESYNC_DAGTAG lock
+			 * 128MiB "resync extents" in the activity log whenever
+			 * they make resync requests. Some of these versions
+			 * also lock activity lock extents when receiving
+			 * P_DATA. In particular, DRBD 9.0 and 9.1. This can
+			 * cause a deadlock if we send resync replies in these
+			 * extents as follows:
+			 * * Node is SyncTarget towards us
+			 * * Node locks a resync extent and sends P_RS_DATA_REQUEST
+			 * * Node receives P_DATA write in this extent; write
+			 *   waits for resync extent to be unlocked
+			 * * Node receives P_BARRIER (protocol A); receiver
+			 *   thread blocks waiting for write to complete
+			 * * We reply to P_RS_DATA_REQUEST, but it is never
+			 *   processed because receiver thread is blocked
+			 *
+			 * Break the deadlock by canceling instead. This is
+			 * sent on the control socket so it will be processed. */
+			dynamic_drbd_dbg(peer_device,
+					"Cancel resync request at %llus+%u due to activity",
+					(unsigned long long) peer_req->i.sector, peer_req->i.size);
+
+			err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
+		} else {
+			err = drbd_rs_reply(peer_device, peer_req, &expect_ack);
+
+			/* If expect_ack is true, peer_req may already have been freed. */
+			if (expect_ack)
+				peer_req = NULL;
+		}
+	} else {
+		drbd_err_ratelimit(peer_device, "Sending NegRSDReply. sector %llus.\n",
+		    (unsigned long long)peer_req->i.sector);
+
+		err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
+
+		/* update resync data with failure */
+		drbd_rs_failed_io(peer_device, peer_req->i.sector, peer_req->i.size);
+	}
+
+	dec_unacked(peer_device);
+
+	if (!expect_ack) {
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+	}
+
+	if (unlikely(err))
+		drbd_err(peer_device, "Sending resync reply failed\n");
+	return err;
+}
+
+int w_e_end_ov_req(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	int digest_size;
+	void *digest;
+	sector_t sector = peer_req->i.sector;
+	unsigned int size = peer_req->i.size;
+	struct dagtag_find_result dagtag_result;
+	int err = 0;
+	enum drbd_packet cmd = connection->agreed_features & DRBD_FF_RESYNC_DAGTAG ?
+		P_OV_DAGTAG_REPLY : P_OV_REPLY;
+
+	if (unlikely(cancel) || connection->cstate[NOW] < C_CONNECTED)
+		goto out;
+
+	if (!(connection->agreed_features & DRBD_FF_RESYNC_DAGTAG) &&
+		al_resync_extent_active(peer_device->device, peer_req->i.sector, peer_req->i.size)) {
+		/* A peer that does not support DRBD_FF_RESYNC_DAGTAG expects
+		 * online verify to be exclusive with 128MiB "resync extents"
+		 * in the activity log. If such a verify source sends a request
+		 * but we receive an overlapping write before the request then
+		 * we will read newer data for the verify transaction than the
+		 * source did. So we may detect spurious out-of-sync blocks.
+		 *
+		 * In addition, we may trigger a deadlock in such a peer by
+		 * sending a reply if it is waiting for writes to drain due to
+		 * a P_BARRIER packet. See w_e_end_rsdata_req for details.
+		 *
+		 * Prevent these issues by canceling instead.
+		 */
+		dynamic_drbd_dbg(peer_device,
+				"Cancel online verify request at %llus+%u due to activity",
+				(unsigned long long) peer_req->i.sector, peer_req->i.size);
+
+		spin_lock_irq(&device->interval_lock);
+		set_bit(INTERVAL_CONFLICT, &peer_req->i.flags);
+		spin_unlock_irq(&device->interval_lock);
+	}
+
+	if (test_bit(INTERVAL_CONFLICT, &peer_req->i.flags)) {
+		if (connection->agreed_pro_version < 110) {
+			if (drbd_ratelimit())
+				drbd_warn(peer_device, "Verify request conflicts but cannot cancel, "
+						"peer may report spurious out-of-sync\n");
+		} else {
+			drbd_verify_skipped_block(peer_device, sector, size);
+			verify_progress(peer_device, sector, size);
+			drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
+			goto out;
+		}
+	}
+
+	dagtag_result = find_current_dagtag(peer_device->device->resource);
+	if (dagtag_result.err)
+		goto out;
+
+	set_bit(INTERVAL_SENT, &peer_req->i.flags);
+
+	digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
+	/* FIXME if this allocation fails, online verify will not terminate! */
+	digest = drbd_prepare_drequest_csum(peer_req, cmd, digest_size,
+			dagtag_result.node_id, dagtag_result.dagtag);
+	if (!digest) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	if (!(peer_req->flags & EE_WAS_ERROR))
+		drbd_csum_pages(peer_device->connection->verify_tfm,
+				peer_req->page_chain.head, digest);
+	else
+		memset(digest, 0, digest_size);
+
+	/* Free pages before send.
+	 * In case we block on congestion, we could otherwise run into
+	 * some distributed deadlock, if the other side blocks on
+	 * congestion as well, because our receiver blocks in
+	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
+	drbd_free_page_chain(&peer_device->connection->transport, &peer_req->page_chain);
+
+	inc_rs_pending(peer_device);
+
+	err = drbd_send_command(peer_device, cmd, DATA_STREAM);
+	if (err)
+		goto out_rs_pending;
+
+	dec_unacked(peer_device);
+	return 0;
+
+out_rs_pending:
+	dec_rs_pending(peer_device);
+out:
+	drbd_remove_peer_req_interval(peer_req);
+	drbd_free_peer_req(peer_req);
+	dec_unacked(peer_device);
+	return err;
+}
+
+void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device, sector_t sector, int size)
+{
+	if (peer_device->ov_last_oos_start + peer_device->ov_last_oos_size == sector) {
+		peer_device->ov_last_oos_size += size>>9;
+	} else {
+		ov_out_of_sync_print(peer_device);
+		peer_device->ov_last_oos_start = sector;
+		peer_device->ov_last_oos_size = size>>9;
+	}
+	drbd_set_out_of_sync(peer_device, sector, size);
+}
+
+void verify_progress(struct drbd_peer_device *peer_device,
+		const sector_t sector, const unsigned int size)
+{
+	bool stop_sector_reached =
+		(peer_device->repl_state[NOW] == L_VERIFY_S) &&
+		verify_can_do_stop_sector(peer_device) &&
+		(sector + (size>>9)) >= peer_device->ov_stop_sector;
+
+	unsigned long ov_left = atomic64_dec_return(&peer_device->ov_left);
+
+	/* let's advance progress step marks only for every other megabyte */
+	if ((ov_left & 0x1ff) == 0)
+		drbd_advance_rs_marks(peer_device, ov_left);
+
+	if (ov_left == 0 || stop_sector_reached)
+		drbd_peer_device_post_work(peer_device, RS_DONE);
+}
+
+static bool digest_equal(struct drbd_peer_request *peer_req)
+{
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_device *device = peer_device->device;
+	struct digest_info *di;
+	void *digest;
+	int digest_size;
+	bool eq = false;
+
+	di = peer_req->digest;
+
+	digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
+	digest = kmalloc(digest_size, GFP_NOIO);
+	if (digest) {
+		drbd_csum_pages(peer_device->connection->verify_tfm, peer_req->page_chain.head, digest);
+
+		D_ASSERT(device, digest_size == di->digest_size);
+		eq = !memcmp(digest, di->digest, digest_size);
+		kfree(digest);
+	}
+
+	return eq;
+}
+
+int w_e_end_ov_reply(struct drbd_work *w, int cancel)
+{
+	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
+	struct drbd_peer_device *peer_device = peer_req->peer_device;
+	struct drbd_connection *connection = peer_device->connection;
+	sector_t sector = peer_req->i.sector;
+	unsigned int size = peer_req->i.size;
+	u64 block_id = peer_req->block_id;
+	enum ov_result result;
+	bool al_conflict = false;
+	int err;
+
+	if (unlikely(cancel)) {
+		drbd_remove_peer_req_interval(peer_req);
+		drbd_free_peer_req(peer_req);
+		dec_unacked(peer_device);
+		return 0;
+	}
+
+	if (!(connection->agreed_features & DRBD_FF_RESYNC_DAGTAG) &&
+		al_resync_extent_active(peer_device->device, peer_req->i.sector, peer_req->i.size)) {
+		/* A peer that does not support DRBD_FF_RESYNC_DAGTAG expects
+		 * online verify to be exclusive with 128MiB "resync extents"
+		 * in the activity log. We may have received an overlapping
+		 * write before issuing this read, which the peer did not have
+		 * at the time of its read. So we may detect spurious
+		 * out-of-sync blocks.
+		 *
+		 * Prevent this by skipping instead.
+		 */
+		dynamic_drbd_dbg(peer_device,
+				"Skip online verify block at %llus+%u due to activity",
+				(unsigned long long) peer_req->i.sector, peer_req->i.size);
+
+		al_conflict = true;
+	}
+
+	if (test_bit(INTERVAL_CONFLICT, &peer_req->i.flags) || al_conflict) {
+		/* DRBD versions without DRBD_FF_RESYNC_DAGTAG do not know about
+		 * OV_RESULT_SKIP, but they treat it the same as OV_RESULT_IN_SYNC which is
+		 * the best we can do here anyway. */
+		result = OV_RESULT_SKIP;
+		drbd_verify_skipped_block(peer_device, sector, size);
+	} else if (likely((peer_req->flags & EE_WAS_ERROR) == 0) && digest_equal(peer_req)) {
+		result = OV_RESULT_IN_SYNC;
+		ov_out_of_sync_print(peer_device);
+	} else {
+		result = OV_RESULT_OUT_OF_SYNC;
+		drbd_ov_out_of_sync_found(peer_device, sector, size);
+	}
+
+	/* Free peer_req and pages before send.
+	 * In case we block on congestion, we could otherwise run into
+	 * some distributed deadlock, if the other side blocks on
+	 * congestion as well, because our receiver blocks in
+	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
+	drbd_remove_peer_req_interval(peer_req);
+	drbd_free_peer_req(peer_req);
+	peer_req = NULL;
+
+	err = drbd_send_ov_result(peer_device, sector, size, block_id, result);
+
+	dec_unacked(peer_device);
+
+	verify_progress(peer_device, sector, size);
+
+	return err;
+}
+
+/* FIXME
+ * We need to track the number of pending barrier acks,
+ * and to be able to wait for them.
+ * See also comment in drbd_adm_attach before drbd_suspend_io.
+ */
+static int drbd_send_barrier(struct drbd_connection *connection)
+{
+	struct p_barrier *p;
+	int err;
+
+	p = conn_prepare_command(connection, sizeof(*p), DATA_STREAM);
+	if (!p)
+		return -EIO;
+
+	p->barrier = connection->send.current_epoch_nr;
+	p->pad = 0;
+	connection->send.last_sent_epoch_nr = connection->send.current_epoch_nr;
+	connection->send.current_epoch_writes = 0;
+	connection->send.last_sent_barrier_jif = jiffies;
+
+	set_bit(BARRIER_ACK_PENDING, &connection->flags);
+	err = send_command(connection, -1, P_BARRIER, DATA_STREAM);
+	if (err) {
+		clear_bit(BARRIER_ACK_PENDING, &connection->flags);
+		wake_up(&connection->resource->barrier_wait);
+	}
+	return err;
+}
+
+static bool need_unplug(struct drbd_connection *connection)
+{
+	unsigned i = connection->todo.unplug_slot;
+	return dagtag_newer_eq(connection->send.current_dagtag_sector,
+			connection->todo.unplug_dagtag_sector[i]);
+}
+
+static void maybe_send_unplug_remote(struct drbd_connection *connection, bool send_anyways)
+{
+	if (need_unplug(connection)) {
+		/* Yes, this is non-atomic wrt. its use in drbd_unplug_fn.
+		 * We save a spin_lock_irq, and worst case
+		 * we occasionally miss an unplug event. */
+
+		/* Paranoia: to avoid a continuous stream of unplug-hints,
+		 * in case we never get any unplug events */
+		connection->todo.unplug_dagtag_sector[connection->todo.unplug_slot] =
+			connection->send.current_dagtag_sector + (1ULL << 63);
+		/* advance the current unplug slot */
+		connection->todo.unplug_slot ^= 1;
+	} else if (!send_anyways)
+		return;
+
+	if (connection->cstate[NOW] < C_CONNECTED)
+		return;
+
+	if (!conn_prepare_command(connection, 0, DATA_STREAM))
+		return;
+
+	send_command(connection, -1, P_UNPLUG_REMOTE, DATA_STREAM);
+}
+
+static bool __drbd_may_sync_now(struct drbd_peer_device *peer_device)
+{
+	struct drbd_device *other_device = peer_device->device;
+	int ret = true;
+
+	rcu_read_lock();
+	while (1) {
+		struct drbd_peer_device *other_peer_device;
+		int resync_after;
+
+		if (!other_device->ldev || other_device->disk_state[NOW] == D_DISKLESS)
+			break;
+		resync_after = rcu_dereference(other_device->ldev->disk_conf)->resync_after;
+		if (resync_after == -1)
+			break;
+		other_device = minor_to_device(resync_after);
+		if (!other_device)
+			break;
+		for_each_peer_device_rcu(other_peer_device, other_device) {
+			if ((other_peer_device->repl_state[NOW] >= L_SYNC_SOURCE &&
+			     other_peer_device->repl_state[NOW] <= L_PAUSED_SYNC_T) ||
+			    other_peer_device->resync_susp_dependency[NOW] ||
+			    other_peer_device->resync_susp_peer[NOW] ||
+			    other_peer_device->resync_susp_user[NOW]) {
+				ret = false;
+				goto break_unlock;
+			}
+		}
+	}
+break_unlock:
+	rcu_read_unlock();
+
+	return ret;
+}
+
+/**
+ * drbd_pause_after() - Pause resync on all devices that may not resync now
+ * @device:	DRBD device.
+ *
+ * Called from process context only (admin command and after_state_ch).
+ */
+static bool drbd_pause_after(struct drbd_device *device)
+{
+	struct drbd_device *other_device;
+	bool changed = false;
+	int vnr;
+
+	/* FIXME seriously inefficient with many devices,
+	 * while also ignoring the input "device" argument :-( */
+	rcu_read_lock();
+	idr_for_each_entry(&drbd_devices, other_device, vnr) {
+		struct drbd_peer_device *other_peer_device;
+
+		begin_state_change_locked(other_device->resource, CS_HARD);
+		if (other_device->disk_state[NOW] == D_DISKLESS) {
+			abort_state_change_locked(other_device->resource);
+			continue;
+		}
+		for_each_peer_device_rcu(other_peer_device, other_device) {
+			if (other_peer_device->repl_state[NOW] == L_OFF)
+				continue;
+			if (!__drbd_may_sync_now(other_peer_device))
+				__change_resync_susp_dependency(other_peer_device, true);
+		}
+		if (end_state_change_locked(other_device->resource, "resync-after") !=
+				SS_NOTHING_TO_DO)
+			changed = true;
+	}
+	rcu_read_unlock();
+	return changed;
+}
+
+/**
+ * drbd_resume_next() - Resume resync on all devices that may resync now
+ * @device:	DRBD device.
+ *
+ * Called from process context only (admin command and sender).
+ */
+static bool drbd_resume_next(struct drbd_device *device)
+{
+	struct drbd_device *other_device;
+	bool changed = false;
+	int vnr;
+
+	/* FIXME seriously inefficient with many devices,
+	 * while also ignoring the input "device" argument :-( */
+	rcu_read_lock();
+	idr_for_each_entry(&drbd_devices, other_device, vnr) {
+		struct drbd_peer_device *other_peer_device;
+
+		begin_state_change_locked(other_device->resource, CS_HARD);
+		if (other_device->disk_state[NOW] == D_DISKLESS) {
+			abort_state_change_locked(other_device->resource);
+			continue;
+		}
+		for_each_peer_device_rcu(other_peer_device, other_device) {
+			if (other_peer_device->repl_state[NOW] == L_OFF)
+				continue;
+			if (other_peer_device->resync_susp_dependency[NOW] &&
+			    __drbd_may_sync_now(other_peer_device))
+				__change_resync_susp_dependency(other_peer_device, false);
+		}
+		if (end_state_change_locked(other_device->resource, "resync-after") !=
+				SS_NOTHING_TO_DO)
+			changed = true;
+	}
+	rcu_read_unlock();
+	return changed;
+}
+
+void resume_next_sg(struct drbd_device *device)
+{
+	lock_all_resources();
+	while (drbd_resume_next(device))
+		; /* Iterate if some state changed. */
+	unlock_all_resources();
+}
+
+void suspend_other_sg(struct drbd_device *device)
+{
+	lock_all_resources();
+	while (drbd_pause_after(device))
+		; /* Iterate if some state changed. */
+	unlock_all_resources();
+}
+
+/* caller must hold resources_mutex */
+enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int resync_after)
+{
+	struct drbd_device *other_device;
+	int rv = NO_ERROR;
+
+	if (resync_after == -1)
+		return NO_ERROR;
+	if (resync_after < -1)
+		return ERR_RESYNC_AFTER;
+	other_device = minor_to_device(resync_after);
+
+	/* You are free to depend on diskless, non-existing,
+	 * or not yet/no longer existing minors.
+	 * We only reject dependency loops.
+	 * We cannot follow the dependency chain beyond a detached or
+	 * missing minor.
+	 */
+	if (!other_device)
+		return NO_ERROR;
+
+	/* check for loops */
+	rcu_read_lock();
+	while (1) {
+		if (other_device == device) {
+			rv = ERR_RESYNC_AFTER_CYCLE;
+			break;
+		}
+
+		if (!other_device)
+			break;
+
+		if (!get_ldev_if_state(other_device, D_NEGOTIATING))
+			break;
+		resync_after = rcu_dereference(other_device->ldev->disk_conf)->resync_after;
+		put_ldev(other_device);
+
+		/* dependency chain ends here, no cycles. */
+		if (resync_after == -1)
+			break;
+
+		/* follow the dependency chain */
+		other_device = minor_to_device(resync_after);
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+/* caller must hold resources_mutex */
+void drbd_resync_after_changed(struct drbd_device *device)
+{
+	while (drbd_pause_after(device) || drbd_resume_next(device))
+		/* do nothing */ ;
+}
+
+void drbd_rs_controller_reset(struct drbd_peer_device *peer_device)
+{
+	struct gendisk *disk = peer_device->device->ldev->backing_bdev->bd_disk;
+	struct fifo_buffer *plan;
+
+	atomic_set(&peer_device->rs_sect_in, 0);
+	atomic_set(&peer_device->device->rs_sect_ev, 0);  /* FIXME: ??? */
+	peer_device->rs_last_mk_req_kt = ktime_get();
+	peer_device->rs_in_flight = 0;
+	peer_device->rs_last_events = (int)part_stat_read_accum(disk->part0, sectors);
+
+	/* Updating the RCU protected object in place is necessary since
+	   this function gets called from atomic context.
+	   It is valid since all other updates also lead to an completely
+	   empty fifo */
+	rcu_read_lock();
+	plan = rcu_dereference(peer_device->rs_plan_s);
+	plan->total = 0;
+	fifo_set(plan, 0);
+	rcu_read_unlock();
+}
+
+void start_resync_timer_fn(struct timer_list *t)
+{
+	struct drbd_peer_device *peer_device = timer_container_of(peer_device, t,
+			start_resync_timer);
+	drbd_peer_device_post_work(peer_device, RS_START);
+}
+
+bool drbd_stable_sync_source_present(struct drbd_peer_device *except_peer_device, enum which_state which)
+{
+	struct drbd_device *device = except_peer_device->device;
+	struct drbd_peer_device *peer_device;
+	u64 authoritative_nodes = 0;
+	bool rv = false;
+
+	if (!(except_peer_device->uuid_flags & UUID_FLAG_STABLE))
+		authoritative_nodes = except_peer_device->uuid_node_mask;
+
+	/* If a peer considers himself as unstable and sees me as an authoritative
+	   node, then we have a stable resync source! */
+	if (authoritative_nodes & NODE_MASK(device->resource->res_opts.node_id))
+		return true;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_repl_state repl_state;
+		struct net_conf *nc;
+
+		if (peer_device == except_peer_device)
+			continue;
+
+		repl_state = peer_device->repl_state[which];
+
+		if (repl_state == L_ESTABLISHED ||
+				repl_state == L_WF_BITMAP_S ||
+				(repl_state >= L_SYNC_SOURCE && repl_state < L_AHEAD)) {
+			if (authoritative_nodes & NODE_MASK(peer_device->node_id)) {
+				rv = true;
+				break;
+			}
+
+			nc = rcu_dereference(peer_device->connection->transport.net_conf);
+			/* Restricting the clause the two_primaries not allowed, otherwise
+			   we need to ensure here that we are neighbor of all primaries,
+			   and that is a lot more challenging. */
+
+			if ((!nc->two_primaries &&
+			     peer_device->connection->peer_role[which] == R_PRIMARY) ||
+			    ((repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) &&
+			     peer_device->uuid_flags & UUID_FLAG_STABLE)) {
+				rv = true;
+				break;
+			}
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+static void do_start_resync(struct drbd_peer_device *peer_device)
+{
+	/*
+	 * Postpone resync in any of these situations:
+	 * - There is still activity from a previous resync according to rs_pending_cnt.
+	 * - The resync as SyncTarget is still active.
+	 * - The transition from Ahead to SyncSource if there is any activity on this peer device.
+	 */
+	if (atomic_read(&peer_device->rs_pending_cnt) ||
+			peer_device->resync_active[NOW] ||
+			(peer_device->repl_state[NOW] == L_AHEAD &&
+			 atomic_read(&peer_device->unacked_cnt))) {
+		drbd_warn(peer_device, "postponing start_resync ...\n");
+		mod_timer(&peer_device->start_resync_timer, jiffies + HZ/10);
+		return;
+	}
+
+	drbd_start_resync(peer_device, peer_device->start_resync_side, "postponed-resync");
+	clear_bit(AHEAD_TO_SYNC_SOURCE, &peer_device->flags);
+}
+
+static void handle_congestion(struct drbd_peer_device *peer_device)
+{
+	struct drbd_resource *resource = peer_device->device->resource;
+	unsigned long irq_flags;
+	struct net_conf *nc;
+	enum drbd_on_congestion on_congestion;
+
+	rcu_read_lock();
+	nc = rcu_dereference(peer_device->connection->transport.net_conf);
+	if (nc) {
+		on_congestion = nc->on_congestion;
+
+		begin_state_change(resource, &irq_flags, CS_VERBOSE | CS_HARD);
+		/* congestion may have cleared since it was detected */
+		if (atomic_read(&peer_device->connection->ap_in_flight) > 0) {
+			if (on_congestion == OC_PULL_AHEAD)
+				__change_repl_state(peer_device, L_AHEAD);
+			else if (on_congestion == OC_DISCONNECT)
+				__change_cstate(peer_device->connection, C_DISCONNECTING);
+		}
+		end_state_change(resource, &irq_flags, "congestion");
+	}
+	rcu_read_unlock();
+
+	clear_bit(HANDLING_CONGESTION, &peer_device->flags);
+}
+
+/**
+ * drbd_start_resync() - Start the resync process
+ * @peer_device: The DRBD peer device to start the resync on.
+ * @side: Direction of the resync; which side am I? Either L_SYNC_SOURCE or
+ * 	  L_SYNC_TARGET.
+ * @tag: State change tag to print in status messages.
+ *
+ * This function might bring you directly into one of the
+ * C_PAUSED_SYNC_* states.
+ */
+void drbd_start_resync(struct drbd_peer_device *peer_device, enum drbd_repl_state side,
+		const char *tag)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_connection *connection = peer_device->connection;
+	enum drbd_disk_state finished_resync_pdsk = D_UNKNOWN;
+	enum drbd_repl_state repl_state;
+	int r;
+
+	read_lock_irq(&device->resource->state_rwlock);
+	repl_state = peer_device->repl_state[NOW];
+	read_unlock_irq(&device->resource->state_rwlock);
+	if (repl_state < L_ESTABLISHED) {
+		/* Connection closed meanwhile. */
+		return;
+	}
+	if (repl_state >= L_SYNC_SOURCE && repl_state < L_AHEAD) {
+		drbd_err(peer_device, "Resync already running!\n");
+		return;
+	}
+
+	if (!test_bit(B_RS_H_DONE, &peer_device->flags)) {
+		if (side == L_SYNC_TARGET) {
+			r = drbd_maybe_khelper(device, connection, "before-resync-target");
+			if (r == DRBD_UMH_DISABLED)
+				goto skip_helper;
+
+			r = (r >> 8) & 0xff;
+			if (r > 0) {
+				drbd_info(device, "before-resync-target handler returned %d, "
+					 "dropping connection.\n", r);
+				change_cstate(connection, C_DISCONNECTING, CS_HARD);
+				return;
+			}
+		} else /* L_SYNC_SOURCE */ {
+			r = drbd_maybe_khelper(device, connection, "before-resync-source");
+			if (r == DRBD_UMH_DISABLED)
+				goto skip_helper;
+
+			r = (r >> 8) & 0xff;
+			if (r > 0) {
+				if (r == 3) {
+					drbd_info(device, "before-resync-source handler returned %d, "
+						 "ignoring. Old userland tools?", r);
+				} else {
+					drbd_info(device, "before-resync-source handler returned %d, "
+						 "dropping connection.\n", r);
+					change_cstate(connection, C_DISCONNECTING, CS_HARD);
+					return;
+				}
+			}
+		}
+	}
+
+skip_helper:
+
+	if (side == L_SYNC_TARGET && drbd_current_uuid(device) == UUID_JUST_CREATED) {
+		/* prepare to continue an interrupted initial resync later */
+		if (get_ldev(device)) {
+			const int my_node_id = device->resource->res_opts.node_id;
+			u64 peer_bitmap_uuid = peer_device->bitmap_uuids[my_node_id];
+
+			if (peer_bitmap_uuid) {
+				down_write(&device->uuid_sem);
+				_drbd_uuid_set_current(device, peer_bitmap_uuid);
+				up_write(&device->uuid_sem);
+				drbd_print_uuids(peer_device, "setting UUIDs to");
+			}
+			put_ldev(device);
+		}
+	}
+
+	if (down_trylock(&device->resource->state_sem)) {
+		/* Retry later and let the worker make progress in the
+		 * meantime; two-phase commits depend on that.  */
+		set_bit(B_RS_H_DONE, &peer_device->flags);
+		peer_device->start_resync_side = side;
+		mod_timer(&peer_device->start_resync_timer, jiffies + HZ/5);
+		return;
+	}
+
+	lock_all_resources();
+	clear_bit(B_RS_H_DONE, &peer_device->flags);
+	if (connection->cstate[NOW] < C_CONNECTED ||
+	    !get_ldev_if_state(device, D_NEGOTIATING)) {
+		unlock_all_resources();
+		goto out;
+	}
+
+	begin_state_change_locked(device->resource, CS_VERBOSE);
+	__change_resync_susp_dependency(peer_device, !__drbd_may_sync_now(peer_device));
+	__change_repl_state(peer_device, side);
+	if (side == L_SYNC_TARGET)
+		init_resync_stable_bits(peer_device);
+	finished_resync_pdsk = peer_device->resync_finished_pdsk;
+	peer_device->resync_finished_pdsk = D_UNKNOWN;
+	r = end_state_change_locked(device->resource, tag);
+	repl_state = peer_device->repl_state[NOW];
+
+	if (repl_state < L_ESTABLISHED)
+		r = SS_UNKNOWN_ERROR;
+
+	if (r == SS_SUCCESS)
+		drbd_pause_after(device);
+
+	unlock_all_resources();
+	put_ldev(device);
+    out:
+	up(&device->resource->state_sem);
+	if (finished_resync_pdsk != D_UNKNOWN)
+		drbd_resync_finished(peer_device, finished_resync_pdsk);
+}
+
+static void update_on_disk_bitmap(struct drbd_peer_device *peer_device, bool resync_done)
+{
+	struct drbd_device *device = peer_device->device;
+	peer_device->rs_last_writeout = jiffies;
+
+	if (!get_ldev(device))
+		return;
+
+	drbd_bm_write_lazy(device, 0);
+
+	if (resync_done) {
+		if (is_verify_state(peer_device, NOW)) {
+			ov_out_of_sync_print(peer_device);
+			ov_skipped_print(peer_device);
+			drbd_resync_finished(peer_device, D_MASK);
+		} else if (is_sync_state(peer_device, NOW)) {
+			drbd_resync_finished(peer_device, D_MASK);
+		}
+	}
+
+	/* update timestamp, in case it took a while to write out stuff */
+	peer_device->rs_last_writeout = jiffies;
+	put_ldev(device);
+}
+
+static void go_diskless(struct drbd_device *device)
+{
+	D_ASSERT(device, device->disk_state[NOW] == D_FAILED ||
+			 device->disk_state[NOW] == D_DETACHING);
+	/* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
+	 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
+	 * the protected members anymore, though, so once put_ldev reaches zero
+	 * again, it will be safe to free them. */
+
+	/* Try to write changed bitmap pages, read errors may have just
+	 * set some bits outside the area covered by the activity log.
+	 *
+	 * If we have an IO error during the bitmap writeout,
+	 * we will want a full sync next time, just in case.
+	 * (Do we want a specific meta data flag for this?)
+	 *
+	 * If that does not make it to stable storage either,
+	 * we cannot do anything about that anymore.
+	 *
+	 * We still need to check if both bitmap and ldev are present, we may
+	 * end up here after a failed attach, before ldev was even assigned.
+	 */
+	if (device->bitmap && device->ldev) {
+		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
+					       "detach",
+					       BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK,
+					       NULL)) {
+			if (test_bit(CRASHED_PRIMARY, &device->flags)) {
+				struct drbd_peer_device *peer_device;
+
+				rcu_read_lock();
+				for_each_peer_device_rcu(peer_device, device)
+					drbd_md_set_peer_flag(peer_device, MDF_PEER_FULL_SYNC);
+				rcu_read_unlock();
+			}
+		}
+	}
+
+	drbd_md_sync_if_dirty(device);
+	drbd_bm_free(device);
+	change_disk_state(device, D_DISKLESS, CS_HARD, "go-diskless", NULL);
+}
+
+static int do_md_sync(struct drbd_device *device)
+{
+	drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
+	drbd_md_sync(device);
+	return 0;
+}
+
+/* only called from drbd_worker thread, no locking */
+void __update_timing_details(
+		struct drbd_thread_timing_details *tdp,
+		unsigned int *cb_nr,
+		void *cb,
+		const char *fn, const unsigned int line)
+{
+	unsigned int i = *cb_nr % DRBD_THREAD_DETAILS_HIST;
+	struct drbd_thread_timing_details *td = tdp + i;
+
+	td->start_jif = jiffies;
+	td->cb_addr = cb;
+	td->caller_fn = fn;
+	td->line = line;
+	td->cb_nr = *cb_nr;
+
+	i = (i+1) % DRBD_THREAD_DETAILS_HIST;
+	td = tdp + i;
+	memset(td, 0, sizeof(*td));
+
+	++(*cb_nr);
+}
+
+static bool all_responded(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	bool all_responded = true;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (!test_bit(CHECKING_PEER, &connection->flags))
+			continue;
+		if (connection->cstate[NOW] < C_CONNECTED) {
+			clear_bit(CHECKING_PEER, &connection->flags);
+			continue;
+		}
+		if (test_bit(PING_PENDING, &connection->flags)) {
+			all_responded = false;
+			continue;
+		} else {
+			clear_bit(CHECKING_PEER, &connection->flags);
+		}
+	}
+	rcu_read_unlock();
+
+	return all_responded;
+}
+
+void drbd_check_peers(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	long t, timeo = LONG_MAX;
+	unsigned long start;
+	bool check_ongoing;
+	u64 im;
+
+	check_ongoing = test_and_set_bit(CHECKING_PEERS, &resource->flags);
+	if (check_ongoing) {
+		wait_event(resource->state_wait,
+			   !test_bit(CHECKING_PEERS, &resource->flags));
+		return;
+	}
+
+	start = jiffies;
+	for_each_connection_ref(connection, im, resource) {
+		if (connection->cstate[NOW] < C_CONNECTED)
+			continue;
+		set_bit(CHECKING_PEER, &connection->flags);
+		send_ping_peer(connection);
+		t = ping_timeout(connection);
+		if (t < timeo)
+			timeo = t;
+	}
+
+	while (!wait_event_timeout(resource->state_wait, all_responded(resource), timeo)) {
+		unsigned long waited = jiffies - start;
+
+		timeo = LONG_MAX;
+		rcu_read_lock();
+		for_each_connection_rcu(connection, resource) {
+			if (!test_bit(CHECKING_PEER, &connection->flags))
+				continue;
+			t = ping_timeout(connection);
+			if (waited >= t) {
+				drbd_warn(connection, "peer failed to send PingAck in time\n");
+				change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+				clear_bit(CHECKING_PEER, &connection->flags);
+				continue;
+			}
+			if (t - waited < timeo)
+				timeo = t - waited;
+		}
+		rcu_read_unlock();
+	}
+
+	clear_bit(CHECKING_PEERS, &resource->flags);
+	wake_up_all(&resource->state_wait);
+}
+
+void drbd_check_peers_new_current_uuid(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+
+	drbd_check_peers(resource);
+
+	if (device->have_quorum[NOW] && drbd_data_accessible(device, NOW))
+		drbd_uuid_new_current(device, false);
+}
+
+static void make_new_current_uuid(struct drbd_device *device)
+{
+	drbd_check_peers_new_current_uuid(device);
+
+	get_work_bits(1UL << NEW_CUR_UUID | 1UL << WRITING_NEW_CUR_UUID, &device->flags);
+	wake_up(&device->misc_wait);
+}
+
+static void do_device_work(struct drbd_device *device, const unsigned long todo)
+{
+	if (test_bit(MD_SYNC, &todo))
+		do_md_sync(device);
+	if (test_bit(GO_DISKLESS, &todo))
+		go_diskless(device);
+	if (test_bit(MAKE_NEW_CUR_UUID, &todo))
+		make_new_current_uuid(device);
+}
+
+static void do_peer_device_work(struct drbd_peer_device *peer_device, const unsigned long todo)
+{
+	if (test_bit(RS_PROGRESS, &todo))
+		drbd_broadcast_peer_device_state(peer_device);
+	if (test_bit(RS_DONE, &todo) ||
+	    test_bit(RS_LAZY_BM_WRITE, &todo))
+		update_on_disk_bitmap(peer_device, test_bit(RS_DONE, &todo));
+	if (test_bit(RS_START, &todo))
+		do_start_resync(peer_device);
+	if (test_bit(HANDLE_CONGESTION, &todo))
+		handle_congestion(peer_device);
+}
+
+#define DRBD_DEVICE_WORK_MASK	\
+	((1UL << GO_DISKLESS)	\
+	|(1UL << MD_SYNC)	\
+	|(1UL << MAKE_NEW_CUR_UUID)\
+	)
+
+#define DRBD_PEER_DEVICE_WORK_MASK	\
+	((1UL << RS_START)		\
+	|(1UL << RS_LAZY_BM_WRITE)	\
+	|(1UL << RS_PROGRESS)		\
+	|(1UL << RS_DONE)		\
+	|(1UL << HANDLE_CONGESTION)     \
+	)
+
+static unsigned long get_work_bits(const unsigned long mask, unsigned long *flags)
+{
+	unsigned long old, new;
+	do {
+		old = *flags;
+		new = old & ~mask;
+	} while (cmpxchg(flags, old, new) != old);
+	return old & mask;
+}
+
+static void __do_unqueued_peer_device_work(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		unsigned long todo = get_work_bits(DRBD_PEER_DEVICE_WORK_MASK, &peer_device->flags);
+		if (!todo)
+			continue;
+
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		do_peer_device_work(peer_device, todo);
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+}
+
+static void do_unqueued_peer_device_work(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	u64 im;
+
+	for_each_connection_ref(connection, im, resource)
+		__do_unqueued_peer_device_work(connection);
+}
+
+static void do_unqueued_device_work(struct drbd_resource *resource)
+{
+	struct drbd_device *device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		unsigned long todo = get_work_bits(DRBD_DEVICE_WORK_MASK, &device->flags);
+		if (!todo)
+			continue;
+
+		kref_get(&device->kref);
+		rcu_read_unlock();
+		do_device_work(device, todo);
+		kref_put(&device->kref, drbd_destroy_device);
+		rcu_read_lock();
+	}
+	rcu_read_unlock();
+}
+
+static bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list)
+{
+	spin_lock_irq(&queue->q_lock);
+	list_splice_tail_init(&queue->q, work_list);
+	spin_unlock_irq(&queue->q_lock);
+	return !list_empty(work_list);
+}
+
+static struct drbd_request *__next_request_for_connection(
+		struct drbd_connection *connection)
+{
+	struct drbd_request *req;
+
+	list_for_each_entry_rcu(req, &connection->resource->transfer_log, tl_requests) {
+		unsigned s = req->net_rq_state[connection->peer_node_id];
+
+		if (likely(s & RQ_NET_QUEUED))
+			return req;
+	}
+	return NULL;
+}
+
+static struct drbd_request *tl_next_request_for_connection(
+		struct drbd_connection *connection, bool wait_ready)
+{
+	if (connection->todo.req_next == NULL)
+		connection->todo.req_next = __next_request_for_connection(connection);
+
+	if (connection->todo.req_next == NULL) {
+		connection->todo.req = NULL;
+	} else {
+		unsigned int s = connection->todo.req_next->net_rq_state[connection->peer_node_id];
+
+		if (likely((s & RQ_NET_READY) || !wait_ready)) {
+			connection->todo.req = connection->todo.req_next;
+			connection->send.seen_dagtag_sector = connection->todo.req->dagtag_sector;
+		} else {
+			/* Leave the request in "req_next" until it is ready */
+			connection->todo.req = NULL;
+		}
+	}
+
+	/*
+	 * Advancement of todo.req_next happens in advance_conn_req_next(),
+	 * called from mod_rq_state()
+	 */
+
+	return connection->todo.req;
+}
+
+static void maybe_send_state_after_ahead(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (test_and_clear_bit(SEND_STATE_AFTER_AHEAD, &peer_device->flags)) {
+			peer_device->todo.was_sending_out_of_sync = false;
+			rcu_read_unlock();
+			drbd_send_current_state(peer_device);
+			rcu_read_lock();
+		}
+	}
+	rcu_read_unlock();
+}
+
+/* This finds the next not yet processed request from
+ * connection->resource->transfer_log.
+ * It also moves all currently queued connection->sender_work
+ * to connection->todo.work_list.
+ */
+static bool check_sender_todo(struct drbd_connection *connection)
+{
+	rcu_read_lock();
+	tl_next_request_for_connection(connection, true);
+
+	/* FIXME can we get rid of this additional lock? */
+	spin_lock_irq(&connection->sender_work.q_lock);
+	list_splice_tail_init(&connection->sender_work.q, &connection->todo.work_list);
+	spin_unlock_irq(&connection->sender_work.q_lock);
+	rcu_read_unlock();
+
+	return connection->todo.req
+		|| need_unplug(connection)
+		|| !list_empty(&connection->todo.work_list);
+}
+
+static bool drbd_send_barrier_next_oos(struct drbd_connection *connection)
+{
+	if (!connection->todo.req_next)
+		return false;
+
+	return connection->todo.req_next->net_rq_state[connection->peer_node_id]
+		& RQ_NET_PENDING_OOS;
+}
+
+static void wait_for_sender_todo(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+	DEFINE_WAIT(wait);
+	struct net_conf *nc;
+	int uncork, cork;
+	bool got_something = 0;
+
+	got_something = check_sender_todo(connection);
+	if (got_something)
+		return;
+
+	/* Still nothing to do?
+	 * Maybe we still need to close the current epoch,
+	 * even if no new requests are queued yet.
+	 *
+	 * Also, poke TCP, just in case.
+	 * Then wait for new work (or signal). */
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	uncork = nc ? nc->tcp_cork : 0;
+	rcu_read_unlock();
+	if (uncork)
+		drbd_uncork(connection, DATA_STREAM);
+
+	for (;;) {
+		int send_barrier;
+		prepare_to_wait(&connection->sender_work.q_wait, &wait,
+				TASK_INTERRUPTIBLE);
+		if (check_sender_todo(connection) || signal_pending(current)) {
+			break;
+		}
+
+		/* We found nothing new to do, no to-be-communicated request,
+		 * no other work item.  We may still need to close the last
+		 * epoch.  Next incoming request epoch will be connection ->
+		 * current transfer log epoch number.  If that is different
+		 * from the epoch of the last request we communicated, we want
+		 * to send the epoch separating barrier now.
+		 */
+		send_barrier = should_send_barrier(connection,
+					atomic_read(&resource->current_tle_nr));
+
+		if (send_barrier) {
+			/* Ensure that we read the most recent
+			 * resource->dagtag_sector value. */
+			smp_rmb();
+			/* If a request is currently being submitted it may not
+			 * have been picked up by this sender, even though it
+			 * belongs to the old epoch. Ensure that we are
+			 * up-to-date with the most recently submitted dagtag
+			 * to ensure that we do not send a barrier early in
+			 * this case. If there is such a request then this
+			 * sender will be woken, so it is OK to schedule().
+			 *
+			 * If we have found a request that is
+			 * RQ_NET_PENDING_OOS, but not yet RQ_NET_READY, then
+			 * we also need to send a barrier.
+			 */
+			if (dagtag_newer_eq(connection->send.seen_dagtag_sector,
+						READ_ONCE(resource->dagtag_sector))
+					|| drbd_send_barrier_next_oos(connection)) {
+				finish_wait(&connection->sender_work.q_wait, &wait);
+				maybe_send_barrier(connection,
+						connection->send.current_epoch_nr + 1);
+				continue;
+			}
+		}
+
+		if (test_and_clear_bit(SEND_STATE_AFTER_AHEAD_C, &connection->flags)) {
+			finish_wait(&connection->sender_work.q_wait, &wait);
+			maybe_send_state_after_ahead(connection);
+			continue;
+		}
+
+		/* drbd_send() may have called flush_signals() */
+		if (get_t_state(&connection->sender) != RUNNING)
+			break;
+
+		schedule();
+		/* may be woken up for other things but new work, too,
+		 * e.g. if the current epoch got closed.
+		 * In which case we send the barrier above. */
+	}
+	finish_wait(&connection->sender_work.q_wait, &wait);
+
+	/* someone may have changed the config while we have been waiting above. */
+	rcu_read_lock();
+	nc = rcu_dereference(connection->transport.net_conf);
+	cork = nc ? nc->tcp_cork : 0;
+	rcu_read_unlock();
+
+	if (cork)
+		drbd_cork(connection, DATA_STREAM);
+	else if (!uncork)
+		drbd_uncork(connection, DATA_STREAM);
+}
+
+static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch)
+{
+	if (!connection->send.seen_any_write_yet) {
+		connection->send.seen_any_write_yet = true;
+		connection->send.current_epoch_nr = epoch;
+		connection->send.current_epoch_writes = 0;
+		connection->send.last_sent_barrier_jif = jiffies;
+	}
+}
+
+static bool should_send_barrier(struct drbd_connection *connection, unsigned int epoch)
+{
+	if (!connection->send.seen_any_write_yet)
+		return false;
+	return connection->send.current_epoch_nr != epoch;
+}
+static void maybe_send_barrier(struct drbd_connection *connection, unsigned int epoch)
+{
+	/* re-init if first write on this connection */
+	if (should_send_barrier(connection, epoch)) {
+		if (connection->send.current_epoch_writes)
+			drbd_send_barrier(connection);
+		connection->send.current_epoch_nr = epoch;
+	}
+}
+
+static int process_one_request(struct drbd_connection *connection)
+{
+	struct bio_and_error m;
+	struct drbd_request *req = connection->todo.req;
+	struct drbd_device *device = req->device;
+	struct drbd_peer_device *peer_device =
+			conn_peer_device(connection, device->vnr);
+	unsigned s = req->net_rq_state[peer_device->node_id];
+	bool do_send_unplug = req->local_rq_state & RQ_UNPLUG;
+	int err = 0;
+	enum drbd_req_event what;
+
+	/* pre_send_jif[] is used in net_timeout_reached() */
+	req->pre_send_jif[peer_device->node_id] = jiffies;
+	ktime_get_accounting(req->pre_send_kt[peer_device->node_id]);
+	if (drbd_req_is_write(req)) {
+		/* If a WRITE does not expect a barrier ack,
+		 * we are supposed to only send an "out of sync" info packet */
+		if (s & RQ_EXP_BARR_ACK) {
+			u64 current_dagtag_sector =
+				req->dagtag_sector - (req->i.size >> 9);
+
+			re_init_if_first_write(connection, req->epoch);
+			maybe_send_barrier(connection, req->epoch);
+			if (current_dagtag_sector != connection->send.current_dagtag_sector)
+				drbd_send_dagtag(connection, current_dagtag_sector);
+
+			connection->send.current_epoch_writes++;
+			connection->send.current_dagtag_sector = req->dagtag_sector;
+
+			if (peer_device->todo.was_sending_out_of_sync) {
+				clear_bit(SEND_STATE_AFTER_AHEAD, &peer_device->flags);
+				peer_device->todo.was_sending_out_of_sync = false;
+				drbd_send_current_state(peer_device);
+			}
+
+			err = drbd_send_dblock(peer_device, req);
+			what = err ? SEND_FAILED : HANDED_OVER_TO_NETWORK;
+		} else {
+			/* this time, no connection->send.current_epoch_writes++;
+			 * If it was sent, it was the closing barrier for the last
+			 * replicated epoch, before we went into AHEAD mode.
+			 * No more barriers will be sent, until we leave AHEAD mode again. */
+			maybe_send_barrier(connection, req->epoch);
+
+			/* make sure the state change to L_AHEAD/L_BEHIND
+			 * arrives before the first set-out-of-sync information */
+			if (!peer_device->todo.was_sending_out_of_sync) {
+				peer_device->todo.was_sending_out_of_sync = true;
+				drbd_send_current_state(peer_device);
+			}
+
+			/* When this flag is not set, sending OOS may be skipped */
+			if (s & RQ_NET_PENDING_OOS)
+				err = drbd_send_out_of_sync(peer_device,
+						req->i.sector, req->i.size);
+			/* This event has the appropriate effect even if OOS skipped or failed */
+			what = OOS_HANDED_TO_NETWORK;
+		}
+	} else {
+		maybe_send_barrier(connection, req->epoch);
+		err = drbd_send_drequest(peer_device,
+				req->i.sector, req->i.size, (unsigned long)req);
+		what = err ? SEND_FAILED : HANDED_OVER_TO_NETWORK;
+	}
+
+	read_lock_irq(&connection->resource->state_rwlock);
+	__req_mod(req, what, peer_device, &m);
+	read_unlock_irq(&connection->resource->state_rwlock);
+
+	check_sender_todo(connection);
+
+	if (m.bio)
+		complete_master_bio(device, &m);
+
+	do_send_unplug = do_send_unplug && what == HANDED_OVER_TO_NETWORK;
+	maybe_send_unplug_remote(connection, do_send_unplug);
+
+	return err;
+}
+
+static int process_sender_todo(struct drbd_connection *connection)
+{
+	struct drbd_work *w = NULL;
+
+	/* Process all currently pending work items,
+	 * or requests from the transfer log.
+	 *
+	 * Right now, work items do not require any strict ordering wrt. the
+	 * request stream, so lets just do simple interleaved processing.
+	 *
+	 * Stop processing as soon as an error is encountered.
+	 */
+	if (!connection->todo.req) {
+		update_sender_timing_details(connection, maybe_send_unplug_remote);
+		maybe_send_unplug_remote(connection, false);
+	} else if (list_empty(&connection->todo.work_list)) {
+		update_sender_timing_details(connection, process_one_request);
+		return process_one_request(connection);
+	}
+
+	while (!list_empty(&connection->todo.work_list)) {
+		int err;
+
+		w = list_first_entry(&connection->todo.work_list, struct drbd_work, list);
+		list_del_init(&w->list);
+		update_sender_timing_details(connection, w->cb);
+		err = w->cb(w, connection->cstate[NOW] < C_CONNECTED);
+		if (err)
+			return err;
+
+		if (connection->todo.req) {
+			update_sender_timing_details(connection, process_one_request);
+			err = process_one_request(connection);
+		}
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+int drbd_sender(struct drbd_thread *thi)
+{
+	struct drbd_connection *connection = thi->connection;
+	struct drbd_work *w;
+	struct drbd_peer_device *peer_device;
+	int vnr;
+	int err;
+
+	/* Should we drop this? Or reset even more stuff? */
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		peer_device->send_cnt = 0;
+		peer_device->recv_cnt = 0;
+	}
+	rcu_read_unlock();
+
+	while (get_t_state(thi) == RUNNING) {
+		drbd_thread_current_set_cpu(thi);
+
+		if (list_empty(&connection->todo.work_list) &&
+		    connection->todo.req == NULL) {
+			update_sender_timing_details(connection, wait_for_sender_todo);
+			wait_for_sender_todo(connection);
+		}
+
+		if (signal_pending(current)) {
+			flush_signals(current);
+			if (get_t_state(thi) == RUNNING) {
+				drbd_warn(connection, "Sender got an unexpected signal\n");
+				continue;
+			}
+			break;
+		}
+
+		if (get_t_state(thi) != RUNNING)
+			break;
+
+		err = process_sender_todo(connection);
+		if (err)
+			change_cstate(connection, C_NETWORK_FAILURE, CS_HARD);
+	}
+
+	/* cleanup all currently unprocessed requests */
+	if (!connection->todo.req) {
+		rcu_read_lock();
+		tl_next_request_for_connection(connection, false);
+		rcu_read_unlock();
+	}
+	while (connection->todo.req) {
+		struct bio_and_error m;
+		struct drbd_request *req = connection->todo.req;
+		struct drbd_device *device = req->device;
+		peer_device = conn_peer_device(connection, device->vnr);
+
+		read_lock_irq(&connection->resource->state_rwlock);
+		__req_mod(req, SEND_CANCELED, peer_device, &m);
+		read_unlock_irq(&connection->resource->state_rwlock);
+		if (m.bio)
+			complete_master_bio(device, &m);
+
+		rcu_read_lock();
+		tl_next_request_for_connection(connection, false);
+		rcu_read_unlock();
+	}
+
+	/* cancel all still pending works */
+	do {
+		while (!list_empty(&connection->todo.work_list)) {
+			w = list_first_entry(&connection->todo.work_list, struct drbd_work, list);
+			list_del_init(&w->list);
+			w->cb(w, 1);
+		}
+		dequeue_work_batch(&connection->sender_work, &connection->todo.work_list);
+	} while (!list_empty(&connection->todo.work_list));
+
+	return 0;
+}
+
+int drbd_worker(struct drbd_thread *thi)
+{
+	LIST_HEAD(work_list);
+	struct drbd_resource *resource = thi->resource;
+	struct drbd_work *w;
+
+	while (get_t_state(thi) == RUNNING) {
+		drbd_thread_current_set_cpu(thi);
+
+		if (list_empty(&work_list)) {
+			bool w, d, p;
+
+			update_worker_timing_details(resource, dequeue_work_batch);
+			wait_event_interruptible(resource->work.q_wait,
+				(w = dequeue_work_batch(&resource->work, &work_list),
+				 d = test_and_clear_bit(DEVICE_WORK_PENDING, &resource->flags),
+				 p = test_and_clear_bit(PEER_DEVICE_WORK_PENDING, &resource->flags),
+				 w || d || p));
+
+			if (p) {
+				update_worker_timing_details(resource, do_unqueued_peer_device_work);
+				do_unqueued_peer_device_work(resource);
+			}
+
+			if (d) {
+				update_worker_timing_details(resource, do_unqueued_device_work);
+				do_unqueued_device_work(resource);
+			}
+		}
+
+		if (signal_pending(current)) {
+			flush_signals(current);
+			if (get_t_state(thi) == RUNNING) {
+				drbd_warn(resource, "Worker got an unexpected signal\n");
+				continue;
+			}
+			break;
+		}
+
+		if (get_t_state(thi) != RUNNING)
+			break;
+
+
+		while (!list_empty(&work_list)) {
+			w = list_first_entry(&work_list, struct drbd_work, list);
+			list_del_init(&w->list);
+			update_worker_timing_details(resource, w->cb);
+			w->cb(w, 0);
+		}
+	}
+
+	do {
+		if (test_and_clear_bit(DEVICE_WORK_PENDING, &resource->flags)) {
+			update_worker_timing_details(resource, do_unqueued_device_work);
+			do_unqueued_device_work(resource);
+		}
+		if (test_and_clear_bit(PEER_DEVICE_WORK_PENDING, &resource->flags)) {
+			update_worker_timing_details(resource, do_unqueued_peer_device_work);
+			do_unqueued_peer_device_work(resource);
+		}
+		while (!list_empty(&work_list)) {
+			w = list_first_entry(&work_list, struct drbd_work, list);
+			list_del_init(&w->list);
+			update_worker_timing_details(resource, w->cb);
+			w->cb(w, 1);
+		}
+		dequeue_work_batch(&resource->work, &work_list);
+	} while (!list_empty(&work_list) ||
+		 test_bit(DEVICE_WORK_PENDING, &resource->flags) ||
+		 test_bit(PEER_DEVICE_WORK_PENDING, &resource->flags));
+
+	return 0;
+}
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index adcba7f1d8ea4..24e8f100890c1 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -13,199 +13,414 @@
 
  */
 
-#include <linux/drbd_limits.h>
+#include <linux/random.h>
+#include <linux/jiffies.h>
 #include "drbd_int.h"
-#include "drbd_protocol.h"
+#include <linux/drbd_protocol.h>
 #include "drbd_req.h"
 #include "drbd_state_change.h"
 
-struct after_state_chg_work {
+
+struct after_state_change_work {
 	struct drbd_work w;
-	struct drbd_device *device;
-	union drbd_state os;
-	union drbd_state ns;
-	enum chg_state_flags flags;
-	struct completion *done;
 	struct drbd_state_change *state_change;
+	struct completion *done;
+};
+
+struct quorum_info {
+	int up_to_date;
+	int present;
+	int voters;
+	int quorum_at;
+	int min_redundancy_at;
+};
+
+struct quorum_detail {
+	int up_to_date;
+	int present;
+	int outdated;
+	int diskless;
+	int missing_diskless;
+	int quorumless;
+	int unknown;
+	int quorate_peers;
+};
+
+struct change_context {
+	struct drbd_resource *resource;
+	int vnr;
+	union drbd_state mask;
+	union drbd_state val;
+	int target_node_id;
+	enum chg_state_flags flags;
+	bool change_local_state_last;
+	const char **err_str;
+};
+
+enum change_phase {
+	PH_LOCAL_COMMIT,
+	PH_PREPARE,
+	PH_84_COMMIT,
+	PH_COMMIT,
 };
 
-enum sanitize_state_warnings {
-	NO_WARNING,
-	ABORTED_ONLINE_VERIFY,
-	ABORTED_RESYNC,
-	CONNECTION_LOST_NEGOTIATING,
-	IMPLICITLY_UPGRADED_DISK,
-	IMPLICITLY_UPGRADED_PDSK,
+struct change_disk_state_context {
+	struct change_context context;
+	struct drbd_device *device;
 };
 
+static bool lost_contact_to_peer_data(enum drbd_disk_state *peer_disk_state);
+static bool peer_returns_diskless(struct drbd_peer_device *peer_device,
+				  enum drbd_disk_state os, enum drbd_disk_state ns);
+static void print_state_change(struct drbd_resource *resource, const char *prefix, const char *tag);
+static void finish_state_change(struct drbd_resource *, const char *tag);
+static int w_after_state_change(struct drbd_work *w, int unused);
+static enum drbd_state_rv is_valid_soft_transition(struct drbd_resource *);
+static enum drbd_state_rv is_valid_transition(struct drbd_resource *resource);
+static void sanitize_state(struct drbd_resource *resource);
+static void ensure_exposed_data_uuid(struct drbd_device *device);
+static enum drbd_state_rv change_peer_state(struct drbd_connection *, int, union drbd_state,
+					    union drbd_state, unsigned long *);
+static void check_wrongly_set_mdf_exists(struct drbd_device *);
+static void update_members(struct drbd_resource *resource);
+static bool calc_data_accessible(struct drbd_state_change *state_change, int n_device,
+				 enum which_state which);
+
+/* We need to stay consistent if we are neighbor of a diskless primary with
+   different UUID. This function should be used if the device was D_UP_TO_DATE
+   before.
+ */
+static bool may_return_to_up_to_date(struct drbd_device *device, enum which_state which)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = true;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->disk_state[which] == D_DISKLESS &&
+		    peer_device->connection->peer_role[which] == R_PRIMARY &&
+		    peer_device->current_uuid != drbd_current_uuid(device)) {
+			rv = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+/**
+ * may_be_up_to_date()  -  check if transition from D_CONSISTENT to D_UP_TO_DATE is allowed
+ * @device: DRBD device.
+ * @which: OLD or NEW
+ *
+ * When fencing is enabled, it may only transition from D_CONSISTENT to D_UP_TO_DATE
+ * when ether all peers are connected, or outdated.
+ */
+static bool may_be_up_to_date(struct drbd_device *device, enum which_state which) __must_hold(local)
+{
+	bool all_peers_outdated = true;
+	int node_id;
+
+	if (!may_return_to_up_to_date(device, which))
+		return false;
+
+	rcu_read_lock();
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
+		struct drbd_peer_device *peer_device;
+		enum drbd_disk_state peer_disk_state;
+		bool want_bitmap = true;
+
+		if (node_id == device->ldev->md.node_id)
+			continue;
+
+		if (!(peer_md->flags & MDF_HAVE_BITMAP) && !(peer_md->flags & MDF_NODE_EXISTS))
+			continue;
+
+		if (!(peer_md->flags & MDF_PEER_FENCING))
+			continue;
+		peer_device = peer_device_by_node_id(device, node_id);
+		if (peer_device) {
+			struct peer_device_conf *pdc = rcu_dereference(peer_device->conf);
+			want_bitmap = pdc->bitmap;
+			peer_disk_state = peer_device->disk_state[NEW];
+		} else {
+			peer_disk_state = D_UNKNOWN;
+		}
+
+		switch (peer_disk_state) {
+		case D_DISKLESS:
+			if (!(peer_md->flags & MDF_PEER_DEVICE_SEEN))
+				continue;
+			fallthrough;
+		case D_ATTACHING:
+		case D_DETACHING:
+		case D_FAILED:
+		case D_NEGOTIATING:
+		case D_UNKNOWN:
+			if (!want_bitmap)
+				continue;
+			if ((peer_md->flags & MDF_PEER_OUTDATED))
+				continue;
+			break;
+		case D_INCONSISTENT:
+		case D_OUTDATED:
+			continue;
+		case D_CONSISTENT:
+		case D_UP_TO_DATE:
+			/* These states imply that there is a connection. If there is
+			   a connection we do not need to insist that the peer was
+			   outdated. */
+			continue;
+		case D_MASK:
+			break;
+		}
+
+		all_peers_outdated = false;
+	}
+	rcu_read_unlock();
+	return all_peers_outdated;
+}
+
+static bool stable_up_to_date_neighbor(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->disk_state[NEW] == D_UP_TO_DATE &&
+		    peer_device->uuid_flags & UUID_FLAG_STABLE && /* primary is also stable */
+		    peer_device->current_uuid == drbd_current_uuid(device)) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+/**
+ * disk_state_from_md()  -  determine initial disk state
+ * @device: DRBD device.
+ *
+ * When a disk is attached to a device, we set the disk state to D_NEGOTIATING.
+ * We then wait for all connected peers to send the peer disk state.  Once that
+ * has happened, we can determine the actual disk state based on the peer disk
+ * states and the state of the disk itself.
+ *
+ * The initial disk state becomes D_UP_TO_DATE without fencing or when we know
+ * that all peers have been outdated, and D_CONSISTENT otherwise.
+ *
+ * The caller either needs to have a get_ldev() reference, or need to call
+ * this function only if disk_state[NOW] >= D_NEGOTIATING and holding the
+ * state_rwlock.
+ */
+enum drbd_disk_state disk_state_from_md(struct drbd_device *device) __must_hold(local)
+{
+	enum drbd_disk_state disk_state;
+
+	if (!drbd_md_test_flag(device->ldev, MDF_CONSISTENT))
+		disk_state = D_INCONSISTENT;
+	else if (!drbd_md_test_flag(device->ldev, MDF_WAS_UP_TO_DATE))
+		disk_state = D_OUTDATED;
+	else
+		disk_state = may_be_up_to_date(device, NOW) ? D_UP_TO_DATE : D_CONSISTENT;
+
+	return disk_state;
+}
+
+bool is_suspended_fen(struct drbd_resource *resource, enum which_state which)
+{
+	struct drbd_connection *connection;
+	bool rv = false;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->susp_fen[which]) {
+			rv = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+bool resource_is_suspended(struct drbd_resource *resource, enum which_state which)
+{
+	bool rv = resource->susp_user[which] || resource->susp_nod[which] ||
+		resource->susp_quorum[which] ||	resource->susp_uuid[which];
+
+	if (rv)
+		return rv;
+
+	return is_suspended_fen(resource, which);
+}
+
 static void count_objects(struct drbd_resource *resource,
-			  unsigned int *n_devices,
-			  unsigned int *n_connections)
+			  struct drbd_state_change_object_count *ocnt)
 {
+	struct drbd_path *path;
 	struct drbd_device *device;
 	struct drbd_connection *connection;
 	int vnr;
 
-	*n_devices = 0;
-	*n_connections = 0;
+	lockdep_assert_held(&resource->state_rwlock);
+
+	ocnt->n_devices = 0;
+	ocnt->n_connections = 0;
+	ocnt->n_paths = 0;
 
 	idr_for_each_entry(&resource->devices, device, vnr)
-		(*n_devices)++;
-	for_each_connection(connection, resource)
-		(*n_connections)++;
+		ocnt->n_devices++;
+	for_each_connection(connection, resource) {
+		ocnt->n_connections++;
+		list_for_each_entry(path, &connection->transport.paths, list) {
+			ocnt->n_paths++;
+		}
+	}
 }
 
-static struct drbd_state_change *alloc_state_change(unsigned int n_devices, unsigned int n_connections, gfp_t gfp)
+static struct drbd_state_change *alloc_state_change(struct drbd_state_change_object_count *ocnt, gfp_t flags)
 {
 	struct drbd_state_change *state_change;
-	unsigned int size, n;
+	unsigned int size;
 
 	size = sizeof(struct drbd_state_change) +
-	       n_devices * sizeof(struct drbd_device_state_change) +
-	       n_connections * sizeof(struct drbd_connection_state_change) +
-	       n_devices * n_connections * sizeof(struct drbd_peer_device_state_change);
-	state_change = kmalloc(size, gfp);
+	       ocnt->n_devices * sizeof(struct drbd_device_state_change) +
+	       ocnt->n_connections * sizeof(struct drbd_connection_state_change) +
+	       ocnt->n_devices * ocnt->n_connections * sizeof(struct drbd_peer_device_state_change) +
+	       ocnt->n_paths * sizeof(struct drbd_path_state);
+	state_change = kzalloc(size, flags);
 	if (!state_change)
 		return NULL;
-	state_change->n_devices = n_devices;
-	state_change->n_connections = n_connections;
+	state_change->n_connections = ocnt->n_connections;
+	state_change->n_devices = ocnt->n_devices;
+	state_change->n_paths = ocnt->n_paths;
 	state_change->devices = (void *)(state_change + 1);
-	state_change->connections = (void *)&state_change->devices[n_devices];
-	state_change->peer_devices = (void *)&state_change->connections[n_connections];
-	state_change->resource->resource = NULL;
-	for (n = 0; n < n_devices; n++)
-		state_change->devices[n].device = NULL;
-	for (n = 0; n < n_connections; n++)
-		state_change->connections[n].connection = NULL;
+	state_change->connections = (void *)&state_change->devices[ocnt->n_devices];
+	state_change->peer_devices = (void *)&state_change->connections[ocnt->n_connections];
+	state_change->paths = (void *)&state_change->peer_devices[ocnt->n_devices*ocnt->n_connections];
 	return state_change;
 }
 
-struct drbd_state_change *remember_old_state(struct drbd_resource *resource, gfp_t gfp)
+struct drbd_state_change *remember_state_change(struct drbd_resource *resource, gfp_t gfp)
 {
 	struct drbd_state_change *state_change;
 	struct drbd_device *device;
-	unsigned int n_devices;
 	struct drbd_connection *connection;
-	unsigned int n_connections;
+	struct drbd_state_change_object_count ocnt;
 	int vnr;
 
 	struct drbd_device_state_change *device_state_change;
 	struct drbd_peer_device_state_change *peer_device_state_change;
 	struct drbd_connection_state_change *connection_state_change;
+	struct drbd_path_state *path_state; /* yes, not a _change :-( */
+
+	lockdep_assert_held(&resource->state_rwlock);
 
-	/* Caller holds req_lock spinlock.
-	 * No state, no device IDR, no connections lists can change. */
-	count_objects(resource, &n_devices, &n_connections);
-	state_change = alloc_state_change(n_devices, n_connections, gfp);
+	count_objects(resource, &ocnt);
+	state_change = alloc_state_change(&ocnt, gfp);
 	if (!state_change)
 		return NULL;
 
 	kref_get(&resource->kref);
 	state_change->resource->resource = resource;
-	state_change->resource->role[OLD] =
-		conn_highest_role(first_connection(resource));
-	state_change->resource->susp[OLD] = resource->susp;
-	state_change->resource->susp_nod[OLD] = resource->susp_nod;
-	state_change->resource->susp_fen[OLD] = resource->susp_fen;
-
-	connection_state_change = state_change->connections;
-	for_each_connection(connection, resource) {
-		kref_get(&connection->kref);
-		connection_state_change->connection = connection;
-		connection_state_change->cstate[OLD] =
-			connection->cstate;
-		connection_state_change->peer_role[OLD] =
-			conn_highest_peer(connection);
-		connection_state_change++;
-	}
+	memcpy(state_change->resource->role,
+	       resource->role, sizeof(resource->role));
+	memcpy(state_change->resource->susp,
+	       resource->susp_user, sizeof(resource->susp_user));
+	memcpy(state_change->resource->susp_nod,
+	       resource->susp_nod, sizeof(resource->susp_nod));
+	memcpy(state_change->resource->susp_uuid,
+	       resource->susp_uuid, sizeof(resource->susp_uuid));
+	memcpy(state_change->resource->fail_io,
+	       resource->fail_io, sizeof(resource->fail_io));
 
 	device_state_change = state_change->devices;
 	peer_device_state_change = state_change->peer_devices;
 	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+
 		kref_get(&device->kref);
 		device_state_change->device = device;
-		device_state_change->disk_state[OLD] = device->state.disk;
+		memcpy(device_state_change->disk_state,
+		       device->disk_state, sizeof(device->disk_state));
+		memcpy(device_state_change->have_quorum,
+		       device->have_quorum, sizeof(device->have_quorum));
 
 		/* The peer_devices for each device have to be enumerated in
 		   the order of the connections. We may not use for_each_peer_device() here. */
 		for_each_connection(connection, resource) {
-			struct drbd_peer_device *peer_device;
-
 			peer_device = conn_peer_device(connection, device->vnr);
+
 			peer_device_state_change->peer_device = peer_device;
-			peer_device_state_change->disk_state[OLD] =
-				device->state.pdsk;
-			peer_device_state_change->repl_state[OLD] =
-				max_t(enum drbd_conns,
-				      C_WF_REPORT_PARAMS, device->state.conn);
-			peer_device_state_change->resync_susp_user[OLD] =
-				device->state.user_isp;
-			peer_device_state_change->resync_susp_peer[OLD] =
-				device->state.peer_isp;
-			peer_device_state_change->resync_susp_dependency[OLD] =
-				device->state.aftr_isp;
+			memcpy(peer_device_state_change->disk_state,
+			       peer_device->disk_state, sizeof(peer_device->disk_state));
+			memcpy(peer_device_state_change->repl_state,
+			       peer_device->repl_state, sizeof(peer_device->repl_state));
+			memcpy(peer_device_state_change->resync_susp_user,
+			       peer_device->resync_susp_user,
+			       sizeof(peer_device->resync_susp_user));
+			memcpy(peer_device_state_change->resync_susp_peer,
+			       peer_device->resync_susp_peer,
+			       sizeof(peer_device->resync_susp_peer));
+			memcpy(peer_device_state_change->resync_susp_dependency,
+			       peer_device->resync_susp_dependency,
+			       sizeof(peer_device->resync_susp_dependency));
+			memcpy(peer_device_state_change->resync_susp_other_c,
+			       peer_device->resync_susp_other_c,
+			       sizeof(peer_device->resync_susp_other_c));
+			memcpy(peer_device_state_change->resync_active,
+			       peer_device->resync_active,
+			       sizeof(peer_device->resync_active));
+			memcpy(peer_device_state_change->replication,
+			       peer_device->replication,
+			       sizeof(peer_device->replication));
+			memcpy(peer_device_state_change->peer_replication,
+			       peer_device->peer_replication,
+			       sizeof(peer_device->peer_replication));
 			peer_device_state_change++;
 		}
 		device_state_change++;
 	}
 
-	return state_change;
-}
-
-static void remember_new_state(struct drbd_state_change *state_change)
-{
-	struct drbd_resource_state_change *resource_state_change;
-	struct drbd_resource *resource;
-	unsigned int n;
-
-	if (!state_change)
-		return;
-
-	resource_state_change = &state_change->resource[0];
-	resource = resource_state_change->resource;
-
-	resource_state_change->role[NEW] =
-		conn_highest_role(first_connection(resource));
-	resource_state_change->susp[NEW] = resource->susp;
-	resource_state_change->susp_nod[NEW] = resource->susp_nod;
-	resource_state_change->susp_fen[NEW] = resource->susp_fen;
-
-	for (n = 0; n < state_change->n_devices; n++) {
-		struct drbd_device_state_change *device_state_change =
-			&state_change->devices[n];
-		struct drbd_device *device = device_state_change->device;
-
-		device_state_change->disk_state[NEW] = device->state.disk;
-	}
+	connection_state_change = state_change->connections;
+	path_state = state_change->paths;
+	for_each_connection(connection, resource) {
+		struct drbd_path *path;
 
-	for (n = 0; n < state_change->n_connections; n++) {
-		struct drbd_connection_state_change *connection_state_change =
-			&state_change->connections[n];
-		struct drbd_connection *connection =
-			connection_state_change->connection;
+		kref_get(&connection->kref);
+		connection_state_change->connection = connection;
+		memcpy(connection_state_change->cstate,
+		       connection->cstate, sizeof(connection->cstate));
+		memcpy(connection_state_change->peer_role,
+		       connection->peer_role, sizeof(connection->peer_role));
+		memcpy(connection_state_change->susp_fen,
+		       connection->susp_fen, sizeof(connection->susp_fen));
+
+		list_for_each_entry(path, &connection->transport.paths, list) {
+			/* Share the connection kref with above.
+			 * Could also share the pointer, but would then need to
+			 * remember an additional n_paths per connection
+			 * count/offset (connection_state_change->n_paths++)
+			 * to be able to associate the paths with its connection.
+			 * So why not directly store the pointer here again. */
+			path_state->connection = connection;
+			kref_get(&path->kref);
+			path_state->path = path;
+			path_state->path_established = test_bit(TR_ESTABLISHED, &path->flags);
+
+			path_state++;
+		}
 
-		connection_state_change->cstate[NEW] = connection->cstate;
-		connection_state_change->peer_role[NEW] =
-			conn_highest_peer(connection);
+		connection_state_change++;
 	}
 
-	for (n = 0; n < state_change->n_devices * state_change->n_connections; n++) {
-		struct drbd_peer_device_state_change *peer_device_state_change =
-			&state_change->peer_devices[n];
-		struct drbd_device *device =
-			peer_device_state_change->peer_device->device;
-		union drbd_dev_state state = device->state;
-
-		peer_device_state_change->disk_state[NEW] = state.pdsk;
-		peer_device_state_change->repl_state[NEW] =
-			max_t(enum drbd_conns, C_WF_REPORT_PARAMS, state.conn);
-		peer_device_state_change->resync_susp_user[NEW] =
-			state.user_isp;
-		peer_device_state_change->resync_susp_peer[NEW] =
-			state.peer_isp;
-		peer_device_state_change->resync_susp_dependency[NEW] =
-			state.aftr_isp;
-	}
+	return state_change;
 }
 
 void copy_old_to_new_state_change(struct drbd_state_change *state_change)
@@ -219,7 +434,8 @@ void copy_old_to_new_state_change(struct drbd_state_change *state_change)
 	OLD_TO_NEW(resource_state_change->role);
 	OLD_TO_NEW(resource_state_change->susp);
 	OLD_TO_NEW(resource_state_change->susp_nod);
-	OLD_TO_NEW(resource_state_change->susp_fen);
+	OLD_TO_NEW(resource_state_change->susp_uuid);
+	OLD_TO_NEW(resource_state_change->fail_io);
 
 	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
 		struct drbd_connection_state_change *connection_state_change =
@@ -227,6 +443,7 @@ void copy_old_to_new_state_change(struct drbd_state_change *state_change)
 
 		OLD_TO_NEW(connection_state_change->peer_role);
 		OLD_TO_NEW(connection_state_change->cstate);
+		OLD_TO_NEW(connection_state_change->susp_fen);
 	}
 
 	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
@@ -234,6 +451,7 @@ void copy_old_to_new_state_change(struct drbd_state_change *state_change)
 			&state_change->devices[n_device];
 
 		OLD_TO_NEW(device_state_change->disk_state);
+		OLD_TO_NEW(device_state_change->have_quorum);
 	}
 
 	n_peer_devices = state_change->n_devices * state_change->n_connections;
@@ -246,6 +464,10 @@ void copy_old_to_new_state_change(struct drbd_state_change *state_change)
 		OLD_TO_NEW(p->resync_susp_user);
 		OLD_TO_NEW(p->resync_susp_peer);
 		OLD_TO_NEW(p->resync_susp_dependency);
+		OLD_TO_NEW(p->resync_susp_other_c);
+		OLD_TO_NEW(p->resync_active);
+		OLD_TO_NEW(p->replication);
+		OLD_TO_NEW(p->peer_replication);
 	}
 
 #undef OLD_TO_NEW
@@ -258,737 +480,1397 @@ void forget_state_change(struct drbd_state_change *state_change)
 	if (!state_change)
 		return;
 
-	if (state_change->resource->resource)
+	if (state_change->resource->resource) {
 		kref_put(&state_change->resource->resource->kref, drbd_destroy_resource);
+	}
 	for (n = 0; n < state_change->n_devices; n++) {
 		struct drbd_device *device = state_change->devices[n].device;
 
-		if (device)
+		if (device) {
 			kref_put(&device->kref, drbd_destroy_device);
+		}
 	}
 	for (n = 0; n < state_change->n_connections; n++) {
 		struct drbd_connection *connection =
 			state_change->connections[n].connection;
 
-		if (connection)
+		if (connection) {
 			kref_put(&connection->kref, drbd_destroy_connection);
+		}
+	}
+	for (n = 0; n < state_change->n_paths; n++) {
+		struct drbd_path *path = state_change->paths[n].path;
+		if (path) {
+			kref_put(&path->kref, drbd_destroy_path);
+		}
 	}
 	kfree(state_change);
 }
 
-static int w_after_state_ch(struct drbd_work *w, int unused);
-static void after_state_ch(struct drbd_device *device, union drbd_state os,
-			   union drbd_state ns, enum chg_state_flags flags,
-			   struct drbd_state_change *);
-static enum drbd_state_rv is_valid_state(struct drbd_device *, union drbd_state);
-static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_connection *);
-static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
-static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os,
-				       union drbd_state ns, enum sanitize_state_warnings *warn);
-
-static inline bool is_susp(union drbd_state s)
+static bool state_has_changed(struct drbd_resource *resource)
 {
-        return s.susp || s.susp_nod || s.susp_fen;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	int vnr;
+
+	if (resource->state_change_flags & CS_FORCE_RECALC)
+		return true;
+
+	if (resource->role[OLD] != resource->role[NEW] ||
+	    resource->susp_user[OLD] != resource->susp_user[NEW] ||
+	    resource->susp_nod[OLD] != resource->susp_nod[NEW] ||
+	    resource->susp_quorum[OLD] != resource->susp_quorum[NEW] ||
+	    resource->susp_uuid[OLD] != resource->susp_uuid[NEW] ||
+	    resource->fail_io[OLD] != resource->fail_io[NEW])
+		return true;
+
+	for_each_connection(connection, resource) {
+		if (connection->cstate[OLD] != connection->cstate[NEW] ||
+		    connection->peer_role[OLD] != connection->peer_role[NEW] ||
+		    connection->susp_fen[OLD] != connection->susp_fen[NEW])
+			return true;
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+
+		if (device->disk_state[OLD] != device->disk_state[NEW] ||
+		    device->have_quorum[OLD] != device->have_quorum[NEW])
+			return true;
+
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->disk_state[OLD] != peer_device->disk_state[NEW] ||
+			    peer_device->repl_state[OLD] != peer_device->repl_state[NEW] ||
+			    peer_device->resync_susp_user[OLD] !=
+				peer_device->resync_susp_user[NEW] ||
+			    peer_device->resync_susp_peer[OLD] !=
+				peer_device->resync_susp_peer[NEW] ||
+			    peer_device->resync_susp_dependency[OLD] !=
+				peer_device->resync_susp_dependency[NEW] ||
+			    peer_device->resync_susp_other_c[OLD] !=
+				peer_device->resync_susp_other_c[NEW] ||
+			    peer_device->resync_active[OLD] !=
+				peer_device->resync_active[NEW] ||
+			    peer_device->replication[OLD] !=
+				peer_device->replication[NEW] ||
+			    peer_device->peer_replication[OLD] !=
+				peer_device->peer_replication[NEW] ||
+			    peer_device->uuid_flags & UUID_FLAG_GOT_STABLE)
+				return true;
+		}
+	}
+	return false;
 }
 
-bool conn_all_vols_unconf(struct drbd_connection *connection)
+static void ___begin_state_change(struct drbd_resource *resource)
 {
-	struct drbd_peer_device *peer_device;
-	bool rv = true;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
 	int vnr;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		if (device->state.disk != D_DISKLESS ||
-		    device->state.conn != C_STANDALONE ||
-		    device->state.role != R_SECONDARY) {
-			rv = false;
-			break;
-		}
+	resource->role[NEW] = resource->role[NOW];
+	resource->susp_user[NEW] = resource->susp_user[NOW];
+	resource->susp_nod[NEW] = resource->susp_nod[NOW];
+	resource->susp_quorum[NEW] = resource->susp_quorum[NOW];
+	resource->susp_uuid[NEW] = resource->susp_uuid[NOW];
+	resource->fail_io[NEW] = resource->fail_io[NOW];
+
+	for_each_connection_rcu(connection, resource) {
+		connection->cstate[NEW] = connection->cstate[NOW];
+		connection->peer_role[NEW] = connection->peer_role[NOW];
+		connection->susp_fen[NEW] = connection->susp_fen[NOW];
 	}
-	rcu_read_unlock();
 
-	return rv;
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+
+		device->disk_state[NEW] = device->disk_state[NOW];
+		device->have_quorum[NEW] = device->have_quorum[NOW];
+
+		for_each_peer_device_rcu(peer_device, device) {
+			peer_device->disk_state[NEW] = peer_device->disk_state[NOW];
+			peer_device->repl_state[NEW] = peer_device->repl_state[NOW];
+			peer_device->resync_susp_user[NEW] =
+				peer_device->resync_susp_user[NOW];
+			peer_device->resync_susp_peer[NEW] =
+				peer_device->resync_susp_peer[NOW];
+			peer_device->resync_susp_dependency[NEW] =
+				peer_device->resync_susp_dependency[NOW];
+			peer_device->resync_susp_other_c[NEW] =
+				peer_device->resync_susp_other_c[NOW];
+			peer_device->resync_active[NEW] =
+				peer_device->resync_active[NOW];
+			peer_device->replication[NEW] =
+				peer_device->replication[NOW];
+			peer_device->peer_replication[NEW] =
+				peer_device->peer_replication[NOW];
+		}
+	}
 }
 
-/* Unfortunately the states where not correctly ordered, when
-   they where defined. therefore can not use max_t() here. */
-static enum drbd_role max_role(enum drbd_role role1, enum drbd_role role2)
+static void __begin_state_change(struct drbd_resource *resource)
 {
-	if (role1 == R_PRIMARY || role2 == R_PRIMARY)
-		return R_PRIMARY;
-	if (role1 == R_SECONDARY || role2 == R_SECONDARY)
-		return R_SECONDARY;
-	return R_UNKNOWN;
+	rcu_read_lock();
+	___begin_state_change(resource);
 }
 
-static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2)
+static enum drbd_state_rv try_state_change(struct drbd_resource *resource)
 {
-	if (role1 == R_UNKNOWN || role2 == R_UNKNOWN)
-		return R_UNKNOWN;
-	if (role1 == R_SECONDARY || role2 == R_SECONDARY)
-		return R_SECONDARY;
-	return R_PRIMARY;
+	enum drbd_state_rv rv;
+
+	if (!state_has_changed(resource))
+		return SS_NOTHING_TO_DO;
+	sanitize_state(resource);
+	rv = is_valid_transition(resource);
+	if (rv >= SS_SUCCESS && !(resource->state_change_flags & CS_HARD))
+		rv = is_valid_soft_transition(resource);
+	return rv;
 }
 
-enum drbd_role conn_highest_role(struct drbd_connection *connection)
+static void apply_update_to_exposed_data_uuid(struct drbd_resource *resource)
 {
-	enum drbd_role role = R_SECONDARY;
-	struct drbd_peer_device *peer_device;
+	struct drbd_device *device;
 	int vnr;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		role = max_role(role, device->state.role);
-	}
-	rcu_read_unlock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		u64 nedu = device->next_exposed_data_uuid;
+		int changed = 0;
 
-	return role;
+		if (!nedu)
+			continue;
+		if (device->disk_state[NOW] < D_INCONSISTENT)
+			changed = drbd_uuid_set_exposed(device, nedu, false);
+
+		device->next_exposed_data_uuid = 0;
+		if (changed)
+			drbd_info(device, "Executing delayed exposed data uuid update: %016llX\n",
+				  (unsigned long long)device->exposed_data_uuid);
+		else
+			drbd_info(device, "Canceling delayed exposed data uuid update\n");
+	}
 }
 
-enum drbd_role conn_highest_peer(struct drbd_connection *connection)
+void __clear_remote_state_change(struct drbd_resource *resource)
 {
-	enum drbd_role peer = R_UNKNOWN;
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	bool is_connect = resource->twopc_reply.is_connect;
+	int initiator_node_id = resource->twopc_reply.initiator_node_id;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		peer = max_role(peer, device->state.peer);
+	resource->remote_state_change = false;
+	resource->twopc_reply.initiator_node_id = -1;
+	resource->twopc_reply.tid = 0;
+
+	if (is_connect && resource->twopc_prepare_reply_cmd == 0) {
+		struct drbd_connection *connection;
+
+		rcu_read_lock();
+		connection = drbd_connection_by_node_id(resource, initiator_node_id);
+		if (connection)
+			abort_connect(connection);
+		rcu_read_unlock();
 	}
-	rcu_read_unlock();
 
-	return peer;
+	wake_up_all(&resource->twopc_wait);
+
+	/* Do things that where postponed to after two-phase commits finished */
+	apply_update_to_exposed_data_uuid(resource);
 }
 
-enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection)
+static bool state_is_stable(struct drbd_device *device)
 {
-	enum drbd_disk_state disk_state = D_DISKLESS;
 	struct drbd_peer_device *peer_device;
-	int vnr;
+	bool stable = true;
+
+	/* DO NOT add a default clause, we want the compiler to warn us
+	 * for any newly introduced state we may have forgotten to add here */
 
 	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		disk_state = max_t(enum drbd_disk_state, disk_state, device->state.disk);
+	for_each_peer_device_rcu(peer_device, device) {
+		switch (peer_device->repl_state[NOW]) {
+		/* New io is only accepted when the peer device is unknown or there is
+		 * a well-established connection. */
+		case L_OFF:
+		case L_ESTABLISHED:
+		case L_SYNC_SOURCE:
+		case L_SYNC_TARGET:
+		case L_VERIFY_S:
+		case L_VERIFY_T:
+		case L_PAUSED_SYNC_S:
+		case L_PAUSED_SYNC_T:
+		case L_AHEAD:
+		case L_BEHIND:
+		case L_STARTING_SYNC_S:
+		case L_STARTING_SYNC_T:
+			break;
+
+			/* Allow IO in BM exchange states with new protocols */
+		case L_WF_BITMAP_S:
+			if (peer_device->connection->agreed_pro_version < 96)
+				stable = false;
+			break;
+
+			/* no new io accepted in these states */
+		case L_WF_BITMAP_T:
+		case L_WF_SYNC_UUID:
+			stable = false;
+			break;
+		}
+		if (!stable)
+			break;
 	}
 	rcu_read_unlock();
 
-	return disk_state;
+	switch (device->disk_state[NOW]) {
+	case D_DISKLESS:
+	case D_INCONSISTENT:
+	case D_OUTDATED:
+	case D_CONSISTENT:
+	case D_UP_TO_DATE:
+	case D_FAILED:
+	case D_DETACHING:
+		/* disk state is stable as well. */
+		break;
+
+	/* no new io accepted during transitional states */
+	case D_ATTACHING:
+	case D_NEGOTIATING:
+	case D_UNKNOWN:
+	case D_MASK:
+		stable = false;
+	}
+
+	return stable;
 }
 
-enum drbd_disk_state conn_lowest_disk(struct drbd_connection *connection)
+static bool drbd_state_change_is_connect(struct drbd_resource *resource)
 {
-	enum drbd_disk_state disk_state = D_MASK;
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	struct drbd_connection *connection;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk);
+	for_each_connection(connection, resource) {
+		if (connection->cstate[NOW] == C_CONNECTING &&
+				connection->cstate[NEW] == C_CONNECTED)
+			return true;
 	}
-	rcu_read_unlock();
 
-	return disk_state;
+	return false;
 }
 
-enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection)
+static struct after_state_change_work *alloc_after_state_change_work(struct drbd_resource *resource)
 {
-	enum drbd_disk_state disk_state = D_DISKLESS;
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	struct after_state_change_work *work;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		disk_state = max_t(enum drbd_disk_state, disk_state, device->state.pdsk);
+	lockdep_assert_held(&resource->state_rwlock);
+
+	/* If the resource is already "unregistered", the worker thread
+	 * is gone, there is no-one to consume the work item and release
+	 * the associated refcounts. Just don't even create it.
+	 */
+	if (test_bit(R_UNREGISTERED, &resource->flags))
+		return NULL;
+
+	work = kmalloc_obj(*work, GFP_ATOMIC);
+	if (work) {
+		work->state_change = remember_state_change(resource, GFP_ATOMIC);
+		if (!work->state_change) {
+			kfree(work);
+			work = NULL;
+		}
 	}
-	rcu_read_unlock();
+	if (!work)
+		drbd_err(resource, "Could not allocate after state change work\n");
 
-	return disk_state;
+	return work;
 }
 
-enum drbd_conns conn_lowest_conn(struct drbd_connection *connection)
+static void queue_after_state_change_work(struct drbd_resource *resource,
+					  struct completion *done,
+					  struct after_state_change_work *work)
 {
-	enum drbd_conns conn = C_MASK;
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	if (work) {
+		work->w.cb = w_after_state_change;
+		work->done = done;
+		drbd_queue_work(&resource->work, &work->w);
+	} else if (done) {
+		complete(done);
+	}
+}
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		conn = min_t(enum drbd_conns, conn, device->state.conn);
+static enum drbd_state_rv ___end_state_change(struct drbd_resource *resource, struct completion *done,
+					      enum drbd_state_rv rv, const char *tag)
+{
+	enum chg_state_flags flags = resource->state_change_flags;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	bool is_connect;
+	unsigned int pro_ver;
+	int vnr;
+	bool all_devs_have_quorum = true;
+	struct after_state_change_work *work;
+
+	if (flags & CS_ABORT)
+		goto out;
+	if (rv >= SS_SUCCESS)
+		rv = try_state_change(resource);
+	if (rv < SS_SUCCESS) {
+		if (flags & CS_VERBOSE) {
+			drbd_err(resource, "State change failed: %s (%d)\n",
+					drbd_set_st_err_str(rv), rv);
+			print_state_change(resource, "Failed: ", tag);
+		}
+		goto out;
 	}
-	rcu_read_unlock();
+	if (flags & CS_PREPARE)
+		goto out;
 
-	return conn;
-}
+	update_members(resource);
+	finish_state_change(resource, tag);
 
-static bool no_peer_wf_report_params(struct drbd_connection *connection)
-{
-	struct drbd_peer_device *peer_device;
-	int vnr;
-	bool rv = true;
+	/* Check whether we are establishing a connection before applying the change. */
+	is_connect = drbd_state_change_is_connect(resource);
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-		if (peer_device->device->state.conn == C_WF_REPORT_PARAMS) {
-			rv = false;
-			break;
+	/* This remembers the state change, so call before applying the change. */
+	work = alloc_after_state_change_work(resource);
+
+	/* changes to local_cnt and device flags should be visible before
+	 * changes to state, which again should be visible before anything else
+	 * depending on that change happens. */
+	smp_wmb();
+	resource->role[NOW] = resource->role[NEW];
+	resource->susp_user[NOW] = resource->susp_user[NEW];
+	resource->susp_nod[NOW] = resource->susp_nod[NEW];
+	resource->susp_quorum[NOW] = resource->susp_quorum[NEW];
+	resource->susp_uuid[NOW] = resource->susp_uuid[NEW];
+	resource->fail_io[NOW] = resource->fail_io[NEW];
+	resource->cached_susp = resource_is_suspended(resource, NEW);
+
+	pro_ver = PRO_VERSION_MAX;
+	for_each_connection(connection, resource) {
+		connection->cstate[NOW] = connection->cstate[NEW];
+		connection->peer_role[NOW] = connection->peer_role[NEW];
+		connection->susp_fen[NOW] = connection->susp_fen[NEW];
+
+		pro_ver = min_t(unsigned int, pro_ver,
+			connection->agreed_pro_version);
+
+		wake_up(&connection->ee_wait);
+	}
+	resource->cached_min_aggreed_protocol_version = pro_ver;
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct res_opts *o = &resource->res_opts;
+		struct drbd_peer_device *peer_device;
+
+		device->disk_state[NOW] = device->disk_state[NEW];
+		device->have_quorum[NOW] = device->have_quorum[NEW];
+
+		if (!device->have_quorum[NOW])
+			all_devs_have_quorum = false;
+
+		for_each_peer_device(peer_device, device) {
+			peer_device->disk_state[NOW] = peer_device->disk_state[NEW];
+			peer_device->repl_state[NOW] = peer_device->repl_state[NEW];
+			peer_device->resync_susp_user[NOW] =
+				peer_device->resync_susp_user[NEW];
+			peer_device->resync_susp_peer[NOW] =
+				peer_device->resync_susp_peer[NEW];
+			peer_device->resync_susp_dependency[NOW] =
+				peer_device->resync_susp_dependency[NEW];
+			peer_device->resync_susp_other_c[NOW] =
+				peer_device->resync_susp_other_c[NEW];
+			peer_device->resync_active[NOW] =
+				peer_device->resync_active[NEW];
+			peer_device->replication[NOW] =
+				peer_device->replication[NEW];
+			peer_device->peer_replication[NOW] =
+				peer_device->peer_replication[NEW];
 		}
-	rcu_read_unlock();
+		device->cached_state_unstable = !state_is_stable(device);
+		device->cached_err_io =
+			(o->on_no_quorum == ONQ_IO_ERROR && !device->have_quorum[NOW]) ||
+			(o->on_no_data == OND_IO_ERROR && !drbd_data_accessible(device, NOW)) ||
+			resource->fail_io[NEW];
+	}
+	resource->cached_all_devices_have_quorum = all_devs_have_quorum;
+	smp_wmb(); /* Make the NEW_CUR_UUID bit visible after the state change! */
 
-	return rv;
-}
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+		if (test_bit(__NEW_CUR_UUID, &device->flags)) {
+			clear_bit(__NEW_CUR_UUID, &device->flags);
+			set_bit(NEW_CUR_UUID, &device->flags);
+		}
+		ensure_exposed_data_uuid(device);
+
+		wake_up(&device->al_wait);
+		wake_up(&device->misc_wait);
+
+		/* Due to the exclusivity of two-phase commits, there can only
+		 * be one connection being established at once. Hence it is OK
+		 * to release uuid_sem for all connections if the state change
+		 * is establishing any connection. */
+		if (is_connect) {
+			for_each_peer_device(peer_device, device) {
+				if (test_and_clear_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags))
+					up_read_non_owner(&device->uuid_sem);
+			}
+		}
+	}
 
-static void wake_up_all_devices(struct drbd_connection *connection)
-{
-	struct drbd_peer_device *peer_device;
-	int vnr;
+	wake_up_all(&resource->state_wait);
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-		wake_up(&peer_device->device->state_wait);
+	/* Call this after applying the state change from NEW to NOW. */
+	queue_after_state_change_work(resource, done, work);
+out:
 	rcu_read_unlock();
 
-}
+	if ((flags & CS_TWOPC) && !(flags & CS_PREPARE))
+		__clear_remote_state_change(resource);
 
+	resource->state_change_err_str = NULL;
+	return rv;
+}
 
-/**
- * cl_wide_st_chg() - true if the state change is a cluster wide one
- * @device:	DRBD device.
- * @os:		old (current) state.
- * @ns:		new (wanted) state.
- */
-static int cl_wide_st_chg(struct drbd_device *device,
-			  union drbd_state os, union drbd_state ns)
+void state_change_lock(struct drbd_resource *resource, unsigned long *irq_flags, enum chg_state_flags flags)
 {
-	return (os.conn >= C_CONNECTED && ns.conn >= C_CONNECTED &&
-		 ((os.role != R_PRIMARY && ns.role == R_PRIMARY) ||
-		  (os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
-		  (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S) ||
-		  (os.disk != D_FAILED && ns.disk == D_FAILED))) ||
-		(os.conn >= C_CONNECTED && ns.conn == C_DISCONNECTING) ||
-		(os.conn == C_CONNECTED && ns.conn == C_VERIFY_S) ||
-		(os.conn == C_CONNECTED && ns.conn == C_WF_REPORT_PARAMS);
+	if ((flags & CS_SERIALIZE) && !(flags & (CS_ALREADY_SERIALIZED | CS_PREPARED))) {
+		WARN_ONCE(current == resource->worker.task,
+			"worker should not initiate state changes with CS_SERIALIZE\n");
+		down(&resource->state_sem);
+	}
+	write_lock_irqsave(&resource->state_rwlock, *irq_flags);
+	resource->state_change_flags = flags;
 }
 
-static union drbd_state
-apply_mask_val(union drbd_state os, union drbd_state mask, union drbd_state val)
+static void __state_change_unlock(struct drbd_resource *resource, unsigned long *irq_flags, struct completion *done)
 {
-	union drbd_state ns;
-	ns.i = (os.i & ~mask.i) | val.i;
-	return ns;
+	enum chg_state_flags flags = resource->state_change_flags;
+
+	resource->state_change_flags = 0;
+	write_unlock_irqrestore(&resource->state_rwlock, *irq_flags);
+	if (done && expect(resource, current != resource->worker.task))
+		wait_for_completion(done);
+	if ((flags & CS_SERIALIZE) && !(flags & (CS_ALREADY_SERIALIZED | CS_PREPARE)))
+		up(&resource->state_sem);
 }
 
-enum drbd_state_rv
-drbd_change_state(struct drbd_device *device, enum chg_state_flags f,
-		  union drbd_state mask, union drbd_state val)
+void state_change_unlock(struct drbd_resource *resource, unsigned long *irq_flags)
 {
-	unsigned long flags;
-	union drbd_state ns;
-	enum drbd_state_rv rv;
-
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	ns = apply_mask_val(drbd_read_state(device), mask, val);
-	rv = _drbd_set_state(device, ns, f, NULL);
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-
-	return rv;
+	__state_change_unlock(resource, irq_flags, NULL);
 }
 
-/**
- * drbd_force_state() - Impose a change which happens outside our control on our state
- * @device:	DRBD device.
- * @mask:	mask of state bits to change.
- * @val:	value of new state bits.
- */
-void drbd_force_state(struct drbd_device *device,
-	union drbd_state mask, union drbd_state val)
+void begin_state_change_locked(struct drbd_resource *resource, enum chg_state_flags flags)
 {
-	drbd_change_state(device, CS_HARD, mask, val);
+	BUG_ON(flags & (CS_SERIALIZE | CS_WAIT_COMPLETE | CS_PREPARE | CS_ABORT));
+	resource->state_change_flags = flags;
+	__begin_state_change(resource);
 }
 
-static enum drbd_state_rv
-_req_st_cond(struct drbd_device *device, union drbd_state mask,
-	     union drbd_state val)
+enum drbd_state_rv end_state_change_locked(struct drbd_resource *resource, const char *tag)
 {
-	union drbd_state os, ns;
-	unsigned long flags;
-	enum drbd_state_rv rv;
-
-	if (test_and_clear_bit(CL_ST_CHG_SUCCESS, &device->flags))
-		return SS_CW_SUCCESS;
+	return ___end_state_change(resource, NULL, SS_SUCCESS, tag);
+}
 
-	if (test_and_clear_bit(CL_ST_CHG_FAIL, &device->flags))
-		return SS_CW_FAILED_BY_PEER;
+void begin_state_change(struct drbd_resource *resource, unsigned long *irq_flags, enum chg_state_flags flags)
+{
+	state_change_lock(resource, irq_flags, flags);
+	__begin_state_change(resource);
+}
 
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	os = drbd_read_state(device);
-	ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL);
-	rv = is_valid_transition(os, ns);
-	if (rv >= SS_SUCCESS)
-		rv = SS_UNKNOWN_ERROR;  /* cont waiting, otherwise fail. */
+static enum drbd_state_rv __end_state_change(struct drbd_resource *resource,
+					     unsigned long *irq_flags,
+					     enum drbd_state_rv rv,
+					     const char *tag)
+{
+	enum chg_state_flags flags = resource->state_change_flags;
+	struct completion __done, *done = NULL;
 
-	if (!cl_wide_st_chg(device, os, ns))
-		rv = SS_CW_NO_NEED;
-	if (rv == SS_UNKNOWN_ERROR) {
-		rv = is_valid_state(device, ns);
-		if (rv >= SS_SUCCESS) {
-			rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection);
-			if (rv >= SS_SUCCESS)
-				rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */
-		}
+	if ((flags & CS_WAIT_COMPLETE) && !(flags & (CS_PREPARE | CS_ABORT))) {
+		done = &__done;
+		init_completion(done);
 	}
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-
+	rv = ___end_state_change(resource, done, rv, tag);
+	__state_change_unlock(resource, irq_flags, rv >= SS_SUCCESS ? done : NULL);
 	return rv;
 }
 
-/**
- * drbd_req_state() - Perform an eventually cluster wide state change
- * @device:	DRBD device.
- * @mask:	mask of state bits to change.
- * @val:	value of new state bits.
- * @f:		flags
- *
- * Should not be called directly, use drbd_request_state() or
- * _drbd_request_state().
- */
-static enum drbd_state_rv
-drbd_req_state(struct drbd_device *device, union drbd_state mask,
-	       union drbd_state val, enum chg_state_flags f)
+enum drbd_state_rv end_state_change(struct drbd_resource *resource, unsigned long *irq_flags,
+		const char *tag)
 {
-	struct completion done;
-	unsigned long flags;
-	union drbd_state os, ns;
-	enum drbd_state_rv rv;
-	void *buffer = NULL;
-
-	init_completion(&done);
-
-	if (f & CS_SERIALIZE)
-		mutex_lock(device->state_mutex);
-	if (f & CS_INHIBIT_MD_IO)
-		buffer = drbd_md_get_buffer(device, __func__);
-
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	os = drbd_read_state(device);
-	ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL);
-	rv = is_valid_transition(os, ns);
-	if (rv < SS_SUCCESS) {
-		spin_unlock_irqrestore(&device->resource->req_lock, flags);
-		goto abort;
-	}
+	return __end_state_change(resource, irq_flags, SS_SUCCESS, tag);
+}
 
-	if (cl_wide_st_chg(device, os, ns)) {
-		rv = is_valid_state(device, ns);
-		if (rv == SS_SUCCESS)
-			rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection);
-		spin_unlock_irqrestore(&device->resource->req_lock, flags);
+void abort_state_change(struct drbd_resource *resource, unsigned long *irq_flags)
+{
+	resource->state_change_flags &= ~CS_VERBOSE;
+	__end_state_change(resource, irq_flags, SS_UNKNOWN_ERROR, NULL);
+}
 
-		if (rv < SS_SUCCESS) {
-			if (f & CS_VERBOSE)
-				print_st_err(device, os, ns, rv);
-			goto abort;
-		}
+void abort_state_change_locked(struct drbd_resource *resource)
+{
+	resource->state_change_flags &= ~CS_VERBOSE;
+	___end_state_change(resource, NULL, SS_UNKNOWN_ERROR, NULL);
+}
 
-		if (drbd_send_state_req(first_peer_device(device), mask, val)) {
-			rv = SS_CW_FAILED_BY_PEER;
-			if (f & CS_VERBOSE)
-				print_st_err(device, os, ns, rv);
-			goto abort;
-		}
+static void begin_remote_state_change(struct drbd_resource *resource, unsigned long *irq_flags)
+{
+	rcu_read_unlock();
+	write_unlock_irqrestore(&resource->state_rwlock, *irq_flags);
+}
 
-		wait_event(device->state_wait,
-			(rv = _req_st_cond(device, mask, val)));
+static void __end_remote_state_change(struct drbd_resource *resource, enum chg_state_flags flags)
+{
+	rcu_read_lock();
+	resource->state_change_flags = flags;
+	___begin_state_change(resource);
+}
 
-		if (rv < SS_SUCCESS) {
-			if (f & CS_VERBOSE)
-				print_st_err(device, os, ns, rv);
-			goto abort;
-		}
-		spin_lock_irqsave(&device->resource->req_lock, flags);
-		ns = apply_mask_val(drbd_read_state(device), mask, val);
-		rv = _drbd_set_state(device, ns, f, &done);
-	} else {
-		rv = _drbd_set_state(device, ns, f, &done);
-	}
+static void end_remote_state_change(struct drbd_resource *resource, unsigned long *irq_flags, enum chg_state_flags flags)
+{
+	write_lock_irqsave(&resource->state_rwlock, *irq_flags);
+	__end_remote_state_change(resource, flags);
+}
 
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
+void clear_remote_state_change(struct drbd_resource *resource)
+{
+	unsigned long irq_flags;
 
-	if (f & CS_WAIT_COMPLETE && rv == SS_SUCCESS) {
-		D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
-		wait_for_completion(&done);
-	}
+	write_lock_irqsave(&resource->state_rwlock, irq_flags);
+	__clear_remote_state_change(resource);
+	write_unlock_irqrestore(&resource->state_rwlock, irq_flags);
+}
 
-abort:
-	if (buffer)
-		drbd_md_put_buffer(device);
-	if (f & CS_SERIALIZE)
-		mutex_unlock(device->state_mutex);
+static union drbd_state drbd_get_resource_state(struct drbd_resource *resource, enum which_state which)
+{
+	union drbd_state rv = { {
+		.conn = C_STANDALONE,  /* really: undefined */
+		/* (user_isp, peer_isp, and aftr_isp are undefined as well.) */
+		.disk = D_UNKNOWN,  /* really: undefined */
+		.role = resource->role[which],
+		.peer = R_UNKNOWN,  /* really: undefined */
+		.susp = resource->susp_user[which] || resource->susp_quorum[which] || resource->susp_uuid[which],
+		.susp_nod = resource->susp_nod[which],
+		.susp_fen = is_suspended_fen(resource, which),
+		.pdsk = D_UNKNOWN,  /* really: undefined */
+	} };
 
 	return rv;
 }
 
-/**
- * _drbd_request_state() - Request a state change (with flags)
- * @device:	DRBD device.
- * @mask:	mask of state bits to change.
- * @val:	value of new state bits.
- * @f:		flags
- *
- * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE
- * flag, or when logging of failed state change requests is not desired.
- */
-enum drbd_state_rv
-_drbd_request_state(struct drbd_device *device, union drbd_state mask,
-		    union drbd_state val, enum chg_state_flags f)
+union drbd_state drbd_get_device_state(struct drbd_device *device, enum which_state which)
 {
-	enum drbd_state_rv rv;
+	union drbd_state rv = drbd_get_resource_state(device->resource, which);
 
-	wait_event(device->state_wait,
-		   (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE);
+	rv.disk = device->disk_state[which];
+	rv.quorum = device->have_quorum[which];
 
 	return rv;
 }
 
-/*
- * We grab drbd_md_get_buffer(), because we don't want to "fail" the disk while
- * there is IO in-flight: the transition into D_FAILED for detach purposes
- * may get misinterpreted as actual IO error in a confused endio function.
- *
- * We wrap it all into wait_event(), to retry in case the drbd_req_state()
- * returns SS_IN_TRANSIENT_STATE.
- *
- * To avoid potential deadlock with e.g. the receiver thread trying to grab
- * drbd_md_get_buffer() while trying to get out of the "transient state", we
- * need to grab and release the meta data buffer inside of that wait_event loop.
- */
-static enum drbd_state_rv
-request_detach(struct drbd_device *device)
+union drbd_state drbd_get_peer_device_state(struct drbd_peer_device *peer_device, enum which_state which)
 {
-	return drbd_req_state(device, NS(disk, D_FAILED),
-			CS_VERBOSE | CS_ORDERED | CS_INHIBIT_MD_IO);
-}
+	struct drbd_connection *connection = peer_device->connection;
+	union drbd_state rv;
 
-int drbd_request_detach_interruptible(struct drbd_device *device)
-{
-	int ret, rv;
+	rv = drbd_get_device_state(peer_device->device, which);
+	rv.user_isp = peer_device->resync_susp_user[which];
+	rv.peer_isp = peer_device->resync_susp_peer[which];
+	rv.aftr_isp = resync_susp_comb_dep(peer_device, which);
+	rv.conn = combined_conn_state(peer_device, which);
+	rv.peer = connection->peer_role[which];
+	rv.pdsk = peer_device->disk_state[which];
 
-	drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */
-	wait_event_interruptible(device->state_wait,
-		(rv = request_detach(device)) != SS_IN_TRANSIENT_STATE);
-	drbd_resume_io(device);
+	return rv;
+}
 
-	ret = wait_event_interruptible(device->misc_wait,
-			device->state.disk != D_FAILED);
+enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection)
+{
+	enum drbd_disk_state disk_state = D_DISKLESS;
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	if (rv == SS_IS_DISKLESS)
-		rv = SS_NOTHING_TO_DO;
-	if (ret)
-		rv = ERR_INTR;
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		disk_state = max_t(enum drbd_disk_state, disk_state, device->disk_state[NOW]);
+	}
+	rcu_read_unlock();
 
-	return rv;
+	return disk_state;
 }
 
-enum drbd_state_rv
-_drbd_request_state_holding_state_mutex(struct drbd_device *device, union drbd_state mask,
-		    union drbd_state val, enum chg_state_flags f)
+enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection)
 {
-	enum drbd_state_rv rv;
-
-	BUG_ON(f & CS_SERIALIZE);
+	enum drbd_disk_state disk_state = D_DISKLESS;
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	wait_event_cmd(device->state_wait,
-		       (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE,
-		       mutex_unlock(device->state_mutex),
-		       mutex_lock(device->state_mutex));
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+		disk_state = max_t(enum drbd_disk_state, disk_state, peer_device->disk_state[NOW]);
+	rcu_read_unlock();
 
-	return rv;
+	return disk_state;
 }
 
-static void print_st(struct drbd_device *device, const char *name, union drbd_state ns)
+static bool suspend_reason_changed(struct drbd_resource *resource)
 {
-	drbd_err(device, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n",
-	    name,
-	    drbd_conn_str(ns.conn),
-	    drbd_role_str(ns.role),
-	    drbd_role_str(ns.peer),
-	    drbd_disk_str(ns.disk),
-	    drbd_disk_str(ns.pdsk),
-	    is_susp(ns) ? 's' : 'r',
-	    ns.aftr_isp ? 'a' : '-',
-	    ns.peer_isp ? 'p' : '-',
-	    ns.user_isp ? 'u' : '-',
-	    ns.susp_fen ? 'F' : '-',
-	    ns.susp_nod ? 'N' : '-'
-	    );
+	return resource->susp_user[OLD] != resource->susp_user[NEW] ||
+		resource->susp_nod[OLD] != resource->susp_nod[NEW] ||
+		resource->susp_quorum[OLD] != resource->susp_quorum[NEW] ||
+		resource->susp_uuid[OLD] != resource->susp_uuid[NEW] ||
+		is_suspended_fen(resource, OLD) != is_suspended_fen(resource, NEW);
 }
 
-void print_st_err(struct drbd_device *device, union drbd_state os,
-	          union drbd_state ns, enum drbd_state_rv err)
+static bool resync_suspended(struct drbd_peer_device *peer_device, enum which_state which)
 {
-	if (err == SS_IN_TRANSIENT_STATE)
-		return;
-	drbd_err(device, "State change failed: %s\n", drbd_set_st_err_str(err));
-	print_st(device, " state", os);
-	print_st(device, "wanted", ns);
+	return peer_device->resync_susp_user[which] ||
+	       peer_device->resync_susp_peer[which] ||
+	       resync_susp_comb_dep(peer_device, which);
 }
 
-static long print_state_change(char *pb, union drbd_state os, union drbd_state ns,
-			       enum chg_state_flags flags)
+static int scnprintf_resync_suspend_flags(char *buffer, size_t size,
+					  struct drbd_peer_device *peer_device,
+					  enum which_state which)
 {
-	char *pbp;
-	pbp = pb;
-	*pbp = 0;
-
-	if (ns.role != os.role && flags & CS_DC_ROLE)
-		pbp += sprintf(pbp, "role( %s -> %s ) ",
-			       drbd_role_str(os.role),
-			       drbd_role_str(ns.role));
-	if (ns.peer != os.peer && flags & CS_DC_PEER)
-		pbp += sprintf(pbp, "peer( %s -> %s ) ",
-			       drbd_role_str(os.peer),
-			       drbd_role_str(ns.peer));
-	if (ns.conn != os.conn && flags & CS_DC_CONN)
-		pbp += sprintf(pbp, "conn( %s -> %s ) ",
-			       drbd_conn_str(os.conn),
-			       drbd_conn_str(ns.conn));
-	if (ns.disk != os.disk && flags & CS_DC_DISK)
-		pbp += sprintf(pbp, "disk( %s -> %s ) ",
-			       drbd_disk_str(os.disk),
-			       drbd_disk_str(ns.disk));
-	if (ns.pdsk != os.pdsk && flags & CS_DC_PDSK)
-		pbp += sprintf(pbp, "pdsk( %s -> %s ) ",
-			       drbd_disk_str(os.pdsk),
-			       drbd_disk_str(ns.pdsk));
+	struct drbd_device *device = peer_device->device;
+	char *b = buffer, *end = buffer + size;
 
-	return pbp - pb;
-}
+	if (!resync_suspended(peer_device, which))
+		return scnprintf(buffer, size, "no");
 
-static void drbd_pr_state_change(struct drbd_device *device, union drbd_state os, union drbd_state ns,
-				 enum chg_state_flags flags)
-{
-	char pb[300];
-	char *pbp = pb;
+	if (peer_device->resync_susp_user[which])
+		b += scnprintf(b, end - b, "user,");
+	if (peer_device->resync_susp_peer[which])
+		b += scnprintf(b, end - b, "peer,");
+	if (peer_device->resync_susp_dependency[which])
+		b += scnprintf(b, end - b, "after dependency,");
+	if (peer_device->resync_susp_other_c[which])
+		b += scnprintf(b, end - b, "connection dependency,");
+	if (is_sync_source_state(peer_device, which) && device->disk_state[which] <= D_INCONSISTENT)
+		b += scnprintf(b, end - b, "disk inconsistent,");
 
-	pbp += print_state_change(pbp, os, ns, flags ^ CS_DC_MASK);
+	*(--b) = 0;
 
-	if (ns.aftr_isp != os.aftr_isp)
-		pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ",
-			       os.aftr_isp,
-			       ns.aftr_isp);
-	if (ns.peer_isp != os.peer_isp)
-		pbp += sprintf(pbp, "peer_isp( %d -> %d ) ",
-			       os.peer_isp,
-			       ns.peer_isp);
-	if (ns.user_isp != os.user_isp)
-		pbp += sprintf(pbp, "user_isp( %d -> %d ) ",
-			       os.user_isp,
-			       ns.user_isp);
+	return b - buffer;
+}
 
-	if (pbp != pb)
-		drbd_info(device, "%s\n", pb);
+static int scnprintf_io_suspend_flags(char *buffer, size_t size,
+				      struct drbd_resource *resource,
+				      enum which_state which)
+{
+	char *b = buffer, *end = buffer + size;
+
+	if (!resource_is_suspended(resource, which))
+		return scnprintf(buffer, size, "no");
+
+	if (resource->susp_user[which])
+		b += scnprintf(b, end - b, "user,");
+	if (resource->susp_nod[which])
+		b += scnprintf(b, end - b, "no-disk,");
+	if (is_suspended_fen(resource, which))
+		b += scnprintf(b, end - b, "fencing,");
+	if (resource->susp_quorum[which])
+		b += scnprintf(b, end - b, "quorum,");
+	if (resource->susp_uuid[which])
+		b += scnprintf(b, end - b, "uuid,");
+	*(--b) = 0;
+
+	return b - buffer;
 }
 
-static void conn_pr_state_change(struct drbd_connection *connection, union drbd_state os, union drbd_state ns,
-				 enum chg_state_flags flags)
+static void print_state_change(struct drbd_resource *resource, const char *prefix, const char *tag)
 {
-	char pb[300];
-	char *pbp = pb;
+	char buffer[150], *b, *end = buffer + sizeof(buffer);
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	enum drbd_role *role = resource->role;
+	bool *fail_io = resource->fail_io;
+	int vnr;
 
-	pbp += print_state_change(pbp, os, ns, flags);
+	b = buffer;
+	if (role[OLD] != role[NEW])
+		b += scnprintf(b, end - b, "role( %s -> %s ) ",
+			       drbd_role_str(role[OLD]),
+			       drbd_role_str(role[NEW]));
+	if (suspend_reason_changed(resource)) {
+		b += scnprintf(b, end - b, "susp-io( ");
+		b += scnprintf_io_suspend_flags(b, end - b, resource, OLD);
+		b += scnprintf(b, end - b, " -> ");
+		b += scnprintf_io_suspend_flags(b, end - b, resource, NEW);
+		b += scnprintf(b, end - b, " ) ");
+	}
+	if (fail_io[OLD] != fail_io[NEW])
+		b += scnprintf(b, end - b, "force-io-failures( %s -> %s ) ",
+			       fail_io[OLD] ? "yes" : "no",
+			       fail_io[NEW] ? "yes" : "no");
+	if (b != buffer) {
+		*(b-1) = 0;
+		drbd_info(resource, "%s%s%s%s%s\n", prefix, buffer,
+			tag ? " [" : "", tag ?: "", tag ? "]" : "");
+	}
 
-	if (is_susp(ns) != is_susp(os) && flags & CS_DC_SUSP)
-		pbp += sprintf(pbp, "susp( %d -> %d ) ",
-			       is_susp(os),
-			       is_susp(ns));
+	for_each_connection(connection, resource) {
+		enum drbd_conn_state *cstate = connection->cstate;
+		enum drbd_role *peer_role = connection->peer_role;
+
+		b = buffer;
+		if (cstate[OLD] != cstate[NEW])
+			b += scnprintf(b, end - b, "conn( %s -> %s ) ",
+				       drbd_conn_str(cstate[OLD]),
+				       drbd_conn_str(cstate[NEW]));
+		if (peer_role[OLD] != peer_role[NEW])
+			b += scnprintf(b, end - b, "peer( %s -> %s ) ",
+				       drbd_role_str(peer_role[OLD]),
+				       drbd_role_str(peer_role[NEW]));
+
+		if (b != buffer) {
+			*(b-1) = 0;
+			drbd_info(connection, "%s%s%s%s%s\n", prefix, buffer,
+				tag ? " [" : "", tag ?: "", tag ? "]" : "");
+		}
+	}
 
-	if (pbp != pb)
-		drbd_info(connection, "%s\n", pb);
-}
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+		enum drbd_disk_state *disk_state = device->disk_state;
+		bool *have_quorum = device->have_quorum;
+
+		b = buffer;
+		if (disk_state[OLD] != disk_state[NEW])
+			b += scnprintf(b, end - b, "disk( %s -> %s ) ",
+				       drbd_disk_str(disk_state[OLD]),
+				       drbd_disk_str(disk_state[NEW]));
+		if (have_quorum[OLD] != have_quorum[NEW])
+			b += scnprintf(b, end - b, "quorum( %s -> %s ) ",
+				       have_quorum[OLD] ? "yes" : "no",
+				       have_quorum[NEW] ? "yes" : "no");
+		if (b != buffer) {
+			*(b-1) = 0;
+			drbd_info(device, "%s%s%s%s%s\n", prefix, buffer,
+				tag ? " [" : "", tag ?: "", tag ? "]" : "");
+		}
 
+		for_each_peer_device(peer_device, device) {
+			enum drbd_disk_state *peer_disk_state = peer_device->disk_state;
+			enum drbd_repl_state *repl_state = peer_device->repl_state;
+			bool *replication = peer_device->replication;
+			bool *peer_replication = peer_device->peer_replication;
+
+			b = buffer;
+			if (peer_disk_state[OLD] != peer_disk_state[NEW])
+				b += scnprintf(b, end - b, "pdsk( %s -> %s ) ",
+					       drbd_disk_str(peer_disk_state[OLD]),
+					       drbd_disk_str(peer_disk_state[NEW]));
+			if (repl_state[OLD] != repl_state[NEW])
+				b += scnprintf(b, end - b, "repl( %s -> %s ) ",
+					       drbd_repl_str(repl_state[OLD]),
+					       drbd_repl_str(repl_state[NEW]));
+
+			if (resync_suspended(peer_device, OLD) !=
+			    resync_suspended(peer_device, NEW)) {
+				b += scnprintf(b, end - b, "resync-susp( ");
+				b += scnprintf_resync_suspend_flags(b, end - b, peer_device, OLD);
+				b += scnprintf(b, end - b, " -> ");
+				b += scnprintf_resync_suspend_flags(b, end - b, peer_device, NEW);
+				b += scnprintf(b, end - b, " ) ");
+			}
 
-/**
- * is_valid_state() - Returns an SS_ error code if ns is not valid
- * @device:	DRBD device.
- * @ns:		State to consider.
- */
-static enum drbd_state_rv
-is_valid_state(struct drbd_device *device, union drbd_state ns)
-{
-	/* See drbd_state_sw_errors in drbd_strings.c */
+			if (replication[OLD] != replication[NEW])
+				b += scnprintf(b, end - b, "replication( %s -> %s ) ",
+					       replication[OLD] ? "yes" : "no",
+					       replication[NEW] ? "yes" : "no");
 
-	enum drbd_fencing_p fp;
-	enum drbd_state_rv rv = SS_SUCCESS;
-	struct net_conf *nc;
+			if (peer_replication[OLD] != peer_replication[NEW])
+				b += scnprintf(b, end - b, "peer_replication( %s -> %s ) ",
+					       peer_replication[OLD] ? "yes" : "no",
+					       peer_replication[NEW] ? "yes" : "no");
 
-	rcu_read_lock();
-	fp = FP_DONT_CARE;
-	if (get_ldev(device)) {
-		fp = rcu_dereference(device->ldev->disk_conf)->fencing;
-		put_ldev(device);
+			if (b != buffer) {
+				*(b-1) = 0;
+				drbd_info(peer_device, "%s%s%s%s%s\n", prefix, buffer,
+					tag ? " [" : "", tag ?: "", tag ? "]" : "");
+			}
+		}
 	}
+}
+
+static bool local_disk_may_be_outdated(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
 
-	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-	if (nc) {
-		if (!nc->two_primaries && ns.role == R_PRIMARY) {
-			if (ns.peer == R_PRIMARY)
-				rv = SS_TWO_PRIMARIES;
-			else if (conn_highest_peer(first_peer_device(device)->connection) == R_PRIMARY)
-				rv = SS_O_VOL_PEER_PRI;
+	if (device->resource->role[NEW] == R_PRIMARY) {
+		for_each_peer_device(peer_device, device) {
+			if (peer_device->disk_state[NEW] == D_UP_TO_DATE &&
+			    peer_device->repl_state[NEW] == L_WF_BITMAP_T)
+				return true;
 		}
+		return false;
+	}
+
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->connection->peer_role[NEW] == R_PRIMARY &&
+		    peer_device->repl_state[NEW] > L_OFF)
+			goto have_primary_neighbor;
 	}
 
-	if (rv <= 0)
-		goto out; /* already found a reason to abort */
-	else if (ns.role == R_SECONDARY && device->open_cnt)
-		rv = SS_DEVICE_IN_USE;
+	return true;	/* No neighbor primary, I might be outdated*/
 
-	else if (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.disk < D_UP_TO_DATE)
-		rv = SS_NO_UP_TO_DATE_DISK;
+have_primary_neighbor:
+	/* Allow self outdating while connecting to a diskless primary. */
+	if (peer_device->disk_state[NEW] == D_DISKLESS &&
+	    peer_device->repl_state[OLD] == L_OFF && peer_device->repl_state[NEW] == L_ESTABLISHED)
+		return true;
 
-	else if (fp >= FP_RESOURCE &&
-		 ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk >= D_UNKNOWN)
-		rv = SS_PRIMARY_NOP;
+	for_each_peer_device(peer_device, device) {
+		enum drbd_repl_state repl_state = peer_device->repl_state[NEW];
+		switch (repl_state) {
+		case L_WF_BITMAP_S:
+		case L_STARTING_SYNC_S:
+		case L_SYNC_SOURCE:
+		case L_PAUSED_SYNC_S:
+		case L_AHEAD:
+		case L_ESTABLISHED:
+		case L_VERIFY_S:
+		case L_VERIFY_T:
+		case L_OFF:
+			continue;
+		case L_WF_SYNC_UUID:
+		case L_WF_BITMAP_T:
+		case L_STARTING_SYNC_T:
+		case L_SYNC_TARGET:
+		case L_PAUSED_SYNC_T:
+		case L_BEHIND:
+			return true;
+		}
+	}
 
-	else if (ns.role == R_PRIMARY && ns.disk <= D_INCONSISTENT && ns.pdsk <= D_INCONSISTENT)
-		rv = SS_NO_UP_TO_DATE_DISK;
+	return false;
+}
 
-	else if (ns.conn > C_CONNECTED && ns.disk < D_INCONSISTENT)
-		rv = SS_NO_LOCAL_DISK;
+static int calc_quorum_at(s32 setting, int voters)
+{
+	int quorum_at;
 
-	else if (ns.conn > C_CONNECTED && ns.pdsk < D_INCONSISTENT)
-		rv = SS_NO_REMOTE_DISK;
+	switch (setting) {
+	case QOU_MAJORITY:
+		quorum_at = voters / 2 + 1;
+		break;
+	case QOU_ALL:
+		quorum_at = voters;
+		break;
+	default:
+		quorum_at = setting;
+	}
 
-	else if (ns.conn > C_CONNECTED && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)
-		rv = SS_NO_UP_TO_DATE_DISK;
+	return quorum_at;
+}
 
-	else if ((ns.conn == C_CONNECTED ||
-		  ns.conn == C_WF_BITMAP_S ||
-		  ns.conn == C_SYNC_SOURCE ||
-		  ns.conn == C_PAUSED_SYNC_S) &&
-		  ns.disk == D_OUTDATED)
-		rv = SS_CONNECTED_OUTDATES;
+static void __calc_quorum_with_disk(struct drbd_device *device, struct quorum_detail *qd)
+{
+	struct drbd_resource *resource = device->resource;
+	const u64 quorumless_nodes = device->have_quorum[NOW] ? ~resource->members : 0;
+	const int my_node_id = resource->res_opts.node_id;
+	int node_id;
 
-	else if (nc && (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
-		 (nc->verify_alg[0] == 0))
-		rv = SS_NO_VERIFY_ALG;
+	check_wrongly_set_mdf_exists(device);
 
-	else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
-		  first_peer_device(device)->connection->agreed_pro_version < 88)
-		rv = SS_NOT_SUPPORTED;
+	rcu_read_lock();
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
+		struct drbd_peer_device *peer_device;
+		enum drbd_disk_state disk_state;
+		enum drbd_repl_state repl_state;
+		bool is_intentional_diskless;
+		struct net_conf *nc;
+
+		if (node_id == my_node_id) {
+			disk_state = device->disk_state[NEW];
+			if (disk_state > D_DISKLESS) {
+				if (disk_state == D_UP_TO_DATE)
+					qd->up_to_date++;
+				else
+					qd->present++;
+			}
+			continue;
+		}
 
-	else if (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)
-		rv = SS_NO_UP_TO_DATE_DISK;
+		/* Ignore non existing nodes.
+		   Note: a fresh (before connected once), intentional diskless peer
+		   gets ignored as well by this.
+		   A fresh diskful peer counts! (since it has MDF_HAVE_BITMAP) */
+		if (!(peer_md->flags & (MDF_HAVE_BITMAP | MDF_NODE_EXISTS | MDF_PEER_DEVICE_SEEN)))
+			continue;
 
-	else if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) &&
-                 ns.pdsk == D_UNKNOWN)
-		rv = SS_NEED_CONNECTION;
+		peer_device = peer_device_by_node_id(device, node_id);
 
-	else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN)
-		rv = SS_CONNECTED_OUTDATES;
+		if (peer_device) {
+			is_intentional_diskless = !want_bitmap(peer_device);
+			nc = rcu_dereference(peer_device->connection->transport.net_conf);
+			if (nc && !nc->allow_remote_read) {
+				dynamic_drbd_dbg(peer_device,
+						 "Excluding from quorum calculation because allow-remote-read = no\n");
+				continue;
+			}
+		} else {
+			is_intentional_diskless = !(peer_md->flags & MDF_PEER_DEVICE_SEEN);
+		}
 
-out:
+		repl_state = peer_device ? peer_device->repl_state[NEW] : L_OFF;
+		disk_state = peer_device ? peer_device->disk_state[NEW] : D_UNKNOWN;
+
+		if (repl_state == L_OFF) {
+			if (is_intentional_diskless)
+				/* device should be diskless but is absent */
+				qd->missing_diskless++;
+			else if (disk_state <= D_OUTDATED || peer_md->flags & MDF_PEER_OUTDATED)
+				qd->outdated++;
+			else if (NODE_MASK(node_id) & quorumless_nodes)
+				qd->quorumless++;
+			else
+				qd->unknown++;
+		} else {
+			if (disk_state == D_DISKLESS && is_intentional_diskless)
+				qd->diskless++;
+			else if (disk_state == D_UP_TO_DATE)
+				qd->up_to_date++;
+			else
+				qd->present++;
+		}
+	}
 	rcu_read_unlock();
-
-	return rv;
 }
 
-/**
- * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible
- * This function limits state transitions that may be declined by DRBD. I.e.
- * user requests (aka soft transitions).
- * @os:		old state.
- * @ns:		new state.
- * @connection:  DRBD connection.
- */
-static enum drbd_state_rv
-is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_connection *connection)
+static void __calc_quorum_no_disk(struct drbd_device *device, struct quorum_detail *qd)
 {
-	enum drbd_state_rv rv = SS_SUCCESS;
-
-	if ((ns.conn == C_STARTING_SYNC_T || ns.conn == C_STARTING_SYNC_S) &&
-	    os.conn > C_CONNECTED)
-		rv = SS_RESYNC_RUNNING;
+	struct drbd_resource *resource = device->resource;
+	const u64 quorumless_nodes = device->have_quorum[NOW] ? ~resource->members : 0;
+	struct drbd_peer_device *peer_device;
+	bool is_intentional_diskless;
+
+	if (device->disk_state[NEW] == D_DISKLESS) {
+		/* We only want to consider ourselves as a diskless node when
+		 * we actually intended to be diskless in the config. Otherwise,
+		 * we shouldn't get a vote in the quorum process, so count
+		 * ourselves as unknown. */
+		if (device->device_conf.intentional_diskless)
+			qd->diskless++;
+		else
+			qd->unknown++;
+	}
 
-	if (ns.conn == C_DISCONNECTING && os.conn == C_STANDALONE)
-		rv = SS_ALREADY_STANDALONE;
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_disk_state disk_state;
+		enum drbd_repl_state repl_state;
+		struct net_conf *nc;
+
+		repl_state = peer_device->repl_state[NEW];
+		disk_state = peer_device->disk_state[NEW];
+
+		is_intentional_diskless = !want_bitmap(peer_device);
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		if (nc && !nc->allow_remote_read) {
+			dynamic_drbd_dbg(peer_device,
+					 "Excluding from quorum calculation because allow-remote-read = no\n");
+			continue;
+		}
 
-	if (ns.disk > D_ATTACHING && os.disk == D_DISKLESS)
-		rv = SS_IS_DISKLESS;
+		if (repl_state == L_OFF) {
+			if (is_intentional_diskless)
+				/* device should be diskless but is absent */
+				qd->missing_diskless++;
+			else if (disk_state <= D_OUTDATED)
+				qd->outdated++;
+			else if (NODE_MASK(peer_device->node_id) & quorumless_nodes)
+				qd->quorumless++;
+			else
+				qd->unknown++;
+		} else {
+			if (disk_state == D_DISKLESS && is_intentional_diskless)
+				qd->diskless++;
+			else if (disk_state == D_UP_TO_DATE)
+				qd->up_to_date++;
+			else
+				qd->present++;
+		}
 
-	if (ns.conn == C_WF_CONNECTION && os.conn < C_UNCONNECTED)
-		rv = SS_NO_NET_CONFIG;
+		if (disk_state == D_UP_TO_DATE && test_bit(PEER_QUORATE, &peer_device->flags))
+			qd->quorate_peers++;
 
-	if (ns.disk == D_OUTDATED && os.disk < D_OUTDATED && os.disk != D_ATTACHING)
-		rv = SS_LOWER_THAN_OUTDATED;
 
-	if (ns.conn == C_DISCONNECTING && os.conn == C_UNCONNECTED)
-		rv = SS_IN_TRANSIENT_STATE;
+	}
+	rcu_read_unlock();
+}
 
-	/* While establishing a connection only allow cstate to change.
-	   Delay/refuse role changes, detach attach etc... (they do not touch cstate) */
-	if (test_bit(STATE_SENT, &connection->flags) &&
-	    !((ns.conn == C_WF_REPORT_PARAMS && os.conn == C_WF_CONNECTION) ||
-	      (ns.conn >= C_CONNECTED && os.conn == C_WF_REPORT_PARAMS)))
-		rv = SS_IN_TRANSIENT_STATE;
+static bool calc_quorum(struct drbd_device *device, struct quorum_info *qi)
+{
+	struct drbd_resource *resource = device->resource;
+	int voters, quorum_at, diskless_majority_at, min_redundancy_at;
+	struct quorum_detail qd = {};
+	bool have_quorum;
 
-	/* Do not promote during resync handshake triggered by "force primary".
-	 * This is a hack. It should really be rejected by the peer during the
-	 * cluster wide state change request. */
-	if (os.role != R_PRIMARY && ns.role == R_PRIMARY
-		&& ns.pdsk == D_UP_TO_DATE
-		&& ns.disk != D_UP_TO_DATE && ns.disk != D_DISKLESS
-		&& (ns.conn <= C_WF_SYNC_UUID || ns.conn != os.conn))
-			rv = SS_IN_TRANSIENT_STATE;
+	if (device->disk_state[NEW] > D_ATTACHING && get_ldev_if_state(device, D_ATTACHING)) {
+		__calc_quorum_with_disk(device, &qd);
+		put_ldev(device);
+	} else {
+		__calc_quorum_no_disk(device, &qd);
+	}
 
-	if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED)
-		rv = SS_NEED_CONNECTION;
+	/* Check if a partition containing all missing nodes might have quorum */
+	voters = qd.outdated + qd.quorumless + qd.unknown + qd.up_to_date + qd.present;
+	quorum_at = calc_quorum_at(resource->res_opts.quorum, voters);
+	if (qd.outdated + qd.quorumless + qd.unknown >= quorum_at) {
+		/* when the missing nodes have the quorum, give up the quorumless */
+		qd.unknown += qd.quorumless;
+		qd.quorumless = 0;
+	}
 
-	if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
-	    ns.conn != os.conn && os.conn > C_CONNECTED)
-		rv = SS_RESYNC_RUNNING;
+	/* When all the absent nodes are D_OUTDATED (no one D_UNKNOWN), we can be
+	   sure that the other partition is not able to promote. ->
+	   We remove them from the voters. -> We have quorum */
+	if (qd.unknown)
+		voters = qd.outdated + qd.quorumless + qd.unknown + qd.up_to_date + qd.present;
+	else
+		voters = qd.up_to_date + qd.present;
+
+	quorum_at = calc_quorum_at(resource->res_opts.quorum, voters);
+	diskless_majority_at = calc_quorum_at(QOU_MAJORITY, qd.diskless + qd.missing_diskless);
+	min_redundancy_at = calc_quorum_at(resource->res_opts.quorum_min_redundancy, voters);
+
+	if (qi) {
+		qi->voters = voters;
+		qi->up_to_date = qd.up_to_date;
+		qi->present = qd.present;
+		qi->quorum_at = quorum_at;
+		qi->min_redundancy_at = min_redundancy_at;
+	}
 
-	if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) &&
-	    os.conn < C_CONNECTED)
-		rv = SS_NEED_CONNECTION;
+	have_quorum = qd.quorate_peers ||
+		((qd.up_to_date + qd.present) >= quorum_at && qd.up_to_date >= min_redundancy_at);
+
+	if (!have_quorum && voters != 0 && voters % 2 == 0 && qd.up_to_date + qd.present == quorum_at - 1 &&
+		/* It is an even number of nodes (think 2) and we failed by one vote.
+		   Check if we have majority of the diskless nodes connected.
+		   Using the diskless nodes a tie-breaker! */
+	    qd.diskless >= diskless_majority_at && device->have_quorum[NOW]) {
+		have_quorum = true;
+		if (!test_bit(TIEBREAKER_QUORUM, &device->flags)) {
+			set_bit(TIEBREAKER_QUORUM, &device->flags);
+			drbd_info(device, "Would lose quorum, but using tiebreaker logic to keep\n");
+		}
+	} else {
+		clear_bit(TIEBREAKER_QUORUM, &device->flags);
+	}
 
-	if ((ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE)
-	    && os.conn < C_WF_REPORT_PARAMS)
-		rv = SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */
+	return have_quorum;
+}
 
-	if (ns.conn == C_DISCONNECTING && ns.pdsk == D_OUTDATED &&
-	    os.conn < C_CONNECTED && os.pdsk > D_OUTDATED)
-		rv = SS_OUTDATE_WO_CONN;
+static __printf(2, 3) void _drbd_state_err(struct change_context *context, const char *fmt, ...)
+{
+	struct drbd_resource *resource = context->resource;
+	const char *err_str;
+	va_list args;
+
+	va_start(args, fmt);
+	err_str = kvasprintf(GFP_ATOMIC, fmt, args);
+	va_end(args);
+	if (!err_str)
+		return;
+	if (context->flags & CS_VERBOSE)
+		drbd_err(resource, "%s\n", err_str);
 
-	return rv;
+	if (context->err_str)
+		*context->err_str = err_str;
+	else
+		kfree(err_str);
 }
 
-static enum drbd_state_rv
-is_valid_conn_transition(enum drbd_conns oc, enum drbd_conns nc)
+static __printf(2, 3) void drbd_state_err(struct drbd_resource *resource, const char *fmt, ...)
 {
-	/* no change -> nothing to do, at least for the connection part */
-	if (oc == nc)
-		return SS_NOTHING_TO_DO;
+	const char *err_str;
+	va_list args;
 
-	/* disconnect of an unconfigured connection does not make sense */
-	if (oc == C_STANDALONE && nc == C_DISCONNECTING)
-		return SS_ALREADY_STANDALONE;
+	va_start(args, fmt);
+	err_str = kvasprintf(GFP_ATOMIC, fmt, args);
+	va_end(args);
+	if (!err_str)
+		return;
+	if (resource->state_change_flags & CS_VERBOSE)
+		drbd_err(resource, "%s\n", err_str);
 
-	/* from C_STANDALONE, we start with C_UNCONNECTED */
-	if (oc == C_STANDALONE && nc != C_UNCONNECTED)
-		return SS_NEED_CONNECTION;
+	if (resource->state_change_err_str)
+		*resource->state_change_err_str = err_str;
+	else
+		kfree(err_str);
+}
 
-	/* When establishing a connection we need to go through WF_REPORT_PARAMS!
-	   Necessary to do the right thing upon invalidate-remote on a disconnected resource */
-	if (oc < C_WF_REPORT_PARAMS && nc >= C_CONNECTED)
-		return SS_NEED_CONNECTION;
+static enum drbd_state_rv __is_valid_soft_transition(struct drbd_resource *resource)
+{
+	enum drbd_role *role = resource->role;
+	bool *fail_io = resource->fail_io;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	bool in_handshake = false;
+	int vnr;
+
+	/* See drbd_state_sw_errors in drbd_strings.c */
+
+	if (role[OLD] != R_PRIMARY && role[NEW] == R_PRIMARY) {
+		for_each_connection_rcu(connection, resource) {
+			struct net_conf *nc;
+
+			nc = rcu_dereference(connection->transport.net_conf);
+			if (!nc || nc->two_primaries)
+				continue;
+			if (connection->peer_role[NEW] == R_PRIMARY)
+				return SS_TWO_PRIMARIES;
+		}
+	}
+
+	for_each_connection_rcu(connection, resource) {
+		struct drbd_peer_device *peer_device;
+
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			if (test_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags) &&
+			    peer_device->repl_state[NOW] == L_OFF) {
+				in_handshake = true;
+				goto handshake_found;
+			}
+		}
+	}
+handshake_found:
+
+	if (in_handshake && role[OLD] != role[NEW])
+		return SS_IN_TRANSIENT_STATE;
+
+	if (role[OLD] == R_SECONDARY && role[NEW] == R_PRIMARY && fail_io[NEW])
+		return SS_DEVICE_IN_USE;
+
+	for_each_connection_rcu(connection, resource) {
+		enum drbd_conn_state *cstate = connection->cstate;
+		enum drbd_role *peer_role = connection->peer_role;
+		struct net_conf *nc;
+		bool two_primaries;
+
+		if (cstate[NEW] == C_DISCONNECTING && cstate[OLD] == C_STANDALONE)
+			return SS_ALREADY_STANDALONE;
+
+		if (cstate[NEW] == C_CONNECTING && cstate[OLD] < C_UNCONNECTED)
+			return SS_NO_NET_CONFIG;
+
+		if (cstate[NEW] == C_DISCONNECTING && cstate[OLD] == C_UNCONNECTED)
+			return SS_IN_TRANSIENT_STATE;
+
+		nc = rcu_dereference(connection->transport.net_conf);
+		two_primaries = nc ? nc->two_primaries : false;
+		if (peer_role[NEW] == R_PRIMARY && peer_role[OLD] != R_PRIMARY && !two_primaries) {
+			if (role[NOW] == R_PRIMARY)
+				return SS_TWO_PRIMARIES;
+			if (!fail_io[NEW]) {
+				idr_for_each_entry(&resource->devices, device, vnr) {
+					if (!device->writable && device->open_cnt)
+						return SS_PRIMARY_READER;
+					/*
+					 * One might be tempted to add "|| open_rw_cont" here.
+					 * That is wrong. The promotion of a rw opener will be
+					 * handled in its own two-phase commit.
+					 * Returning SS_PRIMARY_READER for a rw_opener might
+					 * causes confusion for the caller, if that then waits
+					 * for the read-only openers to go away.
+					 */
+				}
+			}
+		}
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		enum drbd_disk_state *disk_state = device->disk_state;
+		struct drbd_peer_device *peer_device;
+		bool any_disk_up_to_date[2];
+		enum which_state which;
+		int nr_negotiating = 0;
+
+		if (in_handshake &&
+		    ((disk_state[OLD] < D_ATTACHING && disk_state[NEW] == D_ATTACHING) ||
+		     (disk_state[OLD] > D_DETACHING && disk_state[NEW] == D_DETACHING)))
+			return SS_IN_TRANSIENT_STATE;
+
+		if (role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY && device->writable &&
+		    !(resource->state_change_flags & CS_FS_IGN_OPENERS))
+			return SS_DEVICE_IN_USE;
+
+		if (disk_state[NEW] > D_ATTACHING && disk_state[OLD] == D_DISKLESS)
+			return SS_IS_DISKLESS;
+
+		if (disk_state[NEW] == D_OUTDATED && disk_state[OLD] < D_OUTDATED &&
+		    disk_state[OLD] != D_ATTACHING && disk_state[OLD] != D_NEGOTIATING) {
+			/* Do not allow outdate of inconsistent or diskless.
+			   But we have to allow Inconsistent -> Outdated if a resync
+			   finishes over one connection, and is paused on other connections */
+
+			for_each_peer_device_rcu(peer_device, device) {
+				enum drbd_repl_state *repl_state = peer_device->repl_state;
+				if (repl_state[OLD] == L_SYNC_TARGET && repl_state[NEW] == L_ESTABLISHED)
+					goto allow;
+			}
+			return SS_LOWER_THAN_OUTDATED;
+		}
+		allow:
+
+		for (which = OLD; which <= NEW; which++)
+			any_disk_up_to_date[which] = drbd_data_accessible(device, which);
+
+		/* Prevent becoming primary while there is not data accessible
+		   and prevent detach or disconnect while primary */
+		if (!(role[OLD] == R_PRIMARY && !any_disk_up_to_date[OLD]) &&
+		     (role[NEW] == R_PRIMARY && !any_disk_up_to_date[NEW]))
+			return SS_NO_UP_TO_DATE_DISK;
+
+		/* Prevent detach or disconnect while held open read only */
+		if (!device->writable && device->open_cnt &&
+		    any_disk_up_to_date[OLD] && !any_disk_up_to_date[NEW])
+			return SS_NO_UP_TO_DATE_DISK;
+
+		if (disk_state[NEW] == D_NEGOTIATING)
+			nr_negotiating++;
+
+		/* Prevent promote when there is no quorum and
+		 * prevent graceful disconnect/detach that would kill quorum
+		 */
+		if ((role[OLD] == R_SECONDARY || device->have_quorum[OLD]) &&
+		    role[NEW] == R_PRIMARY && !device->have_quorum[NEW]) {
+			struct quorum_info qi;
+
+			calc_quorum(device, &qi);
+
+			if (disk_state[NEW] <= D_ATTACHING)
+				drbd_state_err(resource, "no UpToDate peer with quorum");
+			else if (qi.up_to_date + qi.present < qi.quorum_at)
+				drbd_state_err(resource, "%d of %d nodes visible, need %d for quorum",
+					       qi.up_to_date + qi.present, qi.voters, qi.quorum_at);
+			else if (qi.up_to_date < qi.min_redundancy_at)
+				drbd_state_err(resource, "%d of %d nodes up_to_date, need %d for "
+					       "quorum-minimum-redundancy",
+					       qi.up_to_date, qi.voters, qi.min_redundancy_at);
+			return SS_NO_QUORUM;
+		}
+
+		for_each_peer_device_rcu(peer_device, device) {
+			enum drbd_disk_state *peer_disk_state = peer_device->disk_state;
+			enum drbd_repl_state *repl_state = peer_device->repl_state;
+
+			if (peer_disk_state[NEW] == D_NEGOTIATING)
+				nr_negotiating++;
+
+			if (nr_negotiating > 1)
+				return SS_IN_TRANSIENT_STATE;
+
+			if (peer_device->connection->fencing_policy >= FP_RESOURCE &&
+			    !(role[OLD] == R_PRIMARY && repl_state[OLD] < L_ESTABLISHED && !(peer_disk_state[OLD] <= D_OUTDATED)) &&
+			     (role[NEW] == R_PRIMARY && repl_state[NEW] < L_ESTABLISHED && !(peer_disk_state[NEW] <= D_OUTDATED)))
+				return SS_PRIMARY_NOP;
+
+			if (!(repl_state[OLD] > L_ESTABLISHED && disk_state[OLD] < D_INCONSISTENT) &&
+			     (repl_state[NEW] > L_ESTABLISHED && disk_state[NEW] < D_INCONSISTENT))
+				return SS_NO_LOCAL_DISK;
+
+			if (!(repl_state[OLD] > L_ESTABLISHED && peer_disk_state[OLD] < D_INCONSISTENT) &&
+			     (repl_state[NEW] > L_ESTABLISHED && peer_disk_state[NEW] < D_INCONSISTENT))
+				return SS_NO_REMOTE_DISK;
+
+			if (disk_state[OLD] > D_OUTDATED && disk_state[NEW] == D_OUTDATED &&
+			    !local_disk_may_be_outdated(device))
+				return SS_CONNECTED_OUTDATES;
+
+			if (!(repl_state[OLD] == L_VERIFY_S || repl_state[OLD] == L_VERIFY_T) &&
+			     (repl_state[NEW] == L_VERIFY_S || repl_state[NEW] == L_VERIFY_T)) {
+				struct net_conf *nc = rcu_dereference(peer_device->connection->transport.net_conf);
+
+				if (!nc || nc->verify_alg[0] == 0)
+					return SS_NO_VERIFY_ALG;
+			}
+
+			if (!(repl_state[OLD] == L_VERIFY_S || repl_state[OLD] == L_VERIFY_T) &&
+			     (repl_state[NEW] == L_VERIFY_S || repl_state[NEW] == L_VERIFY_T) &&
+				  peer_device->connection->agreed_pro_version < 88)
+				return SS_NOT_SUPPORTED;
+
+			if (repl_is_sync_source(repl_state[OLD]) &&
+					repl_state[NEW] == L_WF_BITMAP_S)
+				return SS_RESYNC_RUNNING;
+
+			if (repl_is_sync_target(repl_state[OLD]) &&
+					repl_state[NEW] == L_WF_BITMAP_T)
+				return SS_RESYNC_RUNNING;
+
+			if (repl_state[NEW] != repl_state[OLD] &&
+			    (repl_state[NEW] == L_STARTING_SYNC_T || repl_state[NEW] == L_STARTING_SYNC_S) &&
+			    repl_state[OLD] > L_ESTABLISHED)
+				return SS_RESYNC_RUNNING;
+
+			if ((repl_state[NEW] == L_VERIFY_S || repl_state[NEW] == L_VERIFY_T) && repl_state[OLD] < L_ESTABLISHED)
+				return SS_NEED_CONNECTION;
+
+			if ((repl_state[NEW] == L_VERIFY_S || repl_state[NEW] == L_VERIFY_T) &&
+			    repl_state[NEW] != repl_state[OLD] && repl_state[OLD] > L_ESTABLISHED)
+				return SS_RESYNC_RUNNING;
+
+			if ((repl_state[NEW] == L_STARTING_SYNC_S || repl_state[NEW] == L_STARTING_SYNC_T) &&
+			    repl_state[OLD] < L_ESTABLISHED)
+				return SS_NEED_CONNECTION;
+
+			if ((repl_state[NEW] == L_SYNC_TARGET || repl_state[NEW] == L_SYNC_SOURCE)
+			    && repl_state[OLD] < L_OFF)
+				return SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */
+
+			if ((peer_disk_state[NEW] > D_DISKLESS && peer_disk_state[NEW] != D_UNKNOWN) &&
+			    peer_disk_state[OLD] == D_DISKLESS && !want_bitmap(peer_device))
+				return SS_ATTACH_NO_BITMAP;  /* peer with --bitmap=no wannts to attach ??? */
+		}
+	}
+
+	return SS_SUCCESS;
+}
+
+/**
+ * is_valid_soft_transition() - Returns an SS_ error code if state[NEW] is not valid
+ *
+ * "Soft" transitions are voluntary state changes which drbd may decline, such
+ * as a user request to promote a resource to primary.  Opposed to that are
+ * involuntary or "hard" transitions like a network connection loss.
+ *
+ * When deciding if a "soft" transition should be allowed, "hard" transitions
+ * may already have forced the resource into a critical state.  It may take
+ * several "soft" transitions to get the resource back to normal.  To allow
+ * those, rather than checking if the desired new state is valid, we can only
+ * check if the desired new state is "at least as good" as the current state.
+ *
+ * @resource:	DRBD resource
+ */
+static enum drbd_state_rv is_valid_soft_transition(struct drbd_resource *resource)
+{
+	enum drbd_state_rv rv;
+
+	rcu_read_lock();
+	rv = __is_valid_soft_transition(resource);
+	rcu_read_unlock();
+
+	return rv;
+}
+
+static enum drbd_state_rv
+is_valid_conn_transition(enum drbd_conn_state oc, enum drbd_conn_state nc)
+{
+	/* no change -> nothing to do, at least for the connection part */
+	if (oc == nc)
+		return SS_NOTHING_TO_DO;
+
+	/* disconnect of an unconfigured connection does not make sense */
+	if (oc == C_STANDALONE && nc == C_DISCONNECTING)
+		return SS_ALREADY_STANDALONE;
+
+	/* from C_STANDALONE, we start with C_UNCONNECTED */
+	if (oc == C_STANDALONE && nc != C_UNCONNECTED)
+		return SS_NEED_CONNECTION;
 
 	/* After a network error only C_UNCONNECTED or C_DISCONNECTING may follow. */
 	if (oc >= C_TIMEOUT && oc <= C_TEAR_DOWN && nc != C_UNCONNECTED && nc != C_DISCONNECTING)
@@ -1007,1391 +1889,4542 @@ is_valid_conn_transition(enum drbd_conns oc, enum drbd_conns nc)
  * This limits hard state transitions. Hard state transitions are facts there are
  * imposed on DRBD by the environment. E.g. disk broke or network broke down.
  * But those hard state transitions are still not allowed to do everything.
- * @ns:		new state.
- * @os:		old state.
+ * @resource: DRBD resource.
  */
-static enum drbd_state_rv
-is_valid_transition(union drbd_state os, union drbd_state ns)
+static enum drbd_state_rv is_valid_transition(struct drbd_resource *resource)
 {
 	enum drbd_state_rv rv;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	int vnr;
 
-	rv = is_valid_conn_transition(os.conn, ns.conn);
+	for_each_connection(connection, resource) {
+		rv = is_valid_conn_transition(connection->cstate[OLD], connection->cstate[NEW]);
+		if (rv < SS_SUCCESS)
+			return rv;
+	}
 
-	/* we cannot fail (again) if we already detached */
-	if (ns.disk == D_FAILED && os.disk == D_DISKLESS)
-		rv = SS_IS_DISKLESS;
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		/* we cannot fail (again) if we already detached */
+		if ((device->disk_state[NEW] == D_FAILED || device->disk_state[NEW] == D_DETACHING) &&
+		    device->disk_state[OLD] == D_DISKLESS) {
+			return SS_IS_DISKLESS;
+		}
+	}
 
-	return rv;
+	return SS_SUCCESS;
 }
 
-static void print_sanitize_warnings(struct drbd_device *device, enum sanitize_state_warnings warn)
+static bool is_sync_target_other_c(struct drbd_peer_device *ign_peer_device)
 {
-	static const char *msg_table[] = {
-		[NO_WARNING] = "",
-		[ABORTED_ONLINE_VERIFY] = "Online-verify aborted.",
-		[ABORTED_RESYNC] = "Resync aborted.",
-		[CONNECTION_LOST_NEGOTIATING] = "Connection lost while negotiating, no data!",
-		[IMPLICITLY_UPGRADED_DISK] = "Implicitly upgraded disk",
-		[IMPLICITLY_UPGRADED_PDSK] = "Implicitly upgraded pdsk",
-	};
+	struct drbd_device *device = ign_peer_device->device;
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device) {
+		enum drbd_repl_state r;
+
+		if (peer_device == ign_peer_device)
+			continue;
 
-	if (warn != NO_WARNING)
-		drbd_warn(device, "%s\n", msg_table[warn]);
+		r = peer_device->repl_state[NEW];
+		if (r == L_SYNC_TARGET || r == L_PAUSED_SYNC_T)
+			return true;
+	}
+
+	return false;
 }
 
-/**
- * sanitize_state() - Resolves implicitly necessary additional changes to a state transition
- * @device:	DRBD device.
- * @os:		old state.
- * @ns:		new state.
- * @warn:	placeholder for returned state warning.
- *
- * When we loose connection, we have to set the state of the peers disk (pdsk)
- * to D_UNKNOWN. This rule and many more along those lines are in this function.
- */
-static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os,
-				       union drbd_state ns, enum sanitize_state_warnings *warn)
+static void drbd_start_other_targets_paused(struct drbd_peer_device *peer_device)
 {
-	enum drbd_fencing_p fp;
-	enum drbd_disk_state disk_min, disk_max, pdsk_min, pdsk_max;
+	struct drbd_device *device = peer_device->device;
+	struct drbd_peer_device *p;
 
-	if (warn)
-		*warn = NO_WARNING;
+	for_each_peer_device(p, device) {
+		if (p == peer_device)
+			continue;
 
-	fp = FP_DONT_CARE;
-	if (get_ldev(device)) {
-		rcu_read_lock();
-		fp = rcu_dereference(device->ldev->disk_conf)->fencing;
-		rcu_read_unlock();
-		put_ldev(device);
+		if (p->disk_state[NEW] >= D_INCONSISTENT && p->repl_state[NEW] == L_ESTABLISHED)
+			p->repl_state[NEW] = L_PAUSED_SYNC_T;
 	}
+}
 
-	/* Implications from connection to peer and peer_isp */
-	if (ns.conn < C_CONNECTED) {
-		ns.peer_isp = 0;
-		ns.peer = R_UNKNOWN;
-		if (ns.pdsk > D_UNKNOWN || ns.pdsk < D_INCONSISTENT)
-			ns.pdsk = D_UNKNOWN;
-	}
+static bool drbd_is_sync_target_candidate(struct drbd_peer_device *peer_device)
+{
+	if (!repl_is_sync_target(peer_device->repl_state[NEW]))
+		return false;
 
-	/* Clear the aftr_isp when becoming unconfigured */
-	if (ns.conn == C_STANDALONE && ns.disk == D_DISKLESS && ns.role == R_SECONDARY)
-		ns.aftr_isp = 0;
+	if (peer_device->resync_susp_dependency[NEW] ||
+			peer_device->resync_susp_peer[NEW] ||
+			peer_device->resync_susp_user[NEW])
+		return false;
 
-	/* An implication of the disk states onto the connection state */
-	/* Abort resync if a disk fails/detaches */
-	if (ns.conn > C_CONNECTED && (ns.disk <= D_FAILED || ns.pdsk <= D_FAILED)) {
-		if (warn)
-			*warn = ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T ?
-				ABORTED_ONLINE_VERIFY : ABORTED_RESYNC;
-		ns.conn = C_CONNECTED;
-	}
+	if (peer_device->disk_state[NEW] < D_OUTDATED)
+		return false;
 
-	/* Connection breaks down before we finished "Negotiating" */
-	if (ns.conn < C_CONNECTED && ns.disk == D_NEGOTIATING &&
-	    get_ldev_if_state(device, D_NEGOTIATING)) {
-		if (device->ed_uuid == device->ldev->md.uuid[UI_CURRENT]) {
-			ns.disk = device->new_state_tmp.disk;
-			ns.pdsk = device->new_state_tmp.pdsk;
-		} else {
-			if (warn)
-				*warn = CONNECTION_LOST_NEGOTIATING;
-			ns.disk = D_DISKLESS;
-			ns.pdsk = D_UNKNOWN;
-		}
-		put_ldev(device);
-	}
+	return true;
 
-	/* D_CONSISTENT and D_OUTDATED vanish when we get connected */
-	if (ns.conn >= C_CONNECTED && ns.conn < C_AHEAD) {
-		if (ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED)
-			ns.disk = D_UP_TO_DATE;
-		if (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED)
-			ns.pdsk = D_UP_TO_DATE;
-	}
-
-	/* Implications of the connection state on the disk states */
-	disk_min = D_DISKLESS;
-	disk_max = D_UP_TO_DATE;
-	pdsk_min = D_INCONSISTENT;
-	pdsk_max = D_UNKNOWN;
-	switch ((enum drbd_conns)ns.conn) {
-	case C_WF_BITMAP_T:
-	case C_PAUSED_SYNC_T:
-	case C_STARTING_SYNC_T:
-	case C_WF_SYNC_UUID:
-	case C_BEHIND:
-		disk_min = D_INCONSISTENT;
-		disk_max = D_OUTDATED;
-		pdsk_min = D_UP_TO_DATE;
-		pdsk_max = D_UP_TO_DATE;
-		break;
-	case C_VERIFY_S:
-	case C_VERIFY_T:
-		disk_min = D_UP_TO_DATE;
-		disk_max = D_UP_TO_DATE;
-		pdsk_min = D_UP_TO_DATE;
-		pdsk_max = D_UP_TO_DATE;
-		break;
-	case C_CONNECTED:
-		disk_min = D_DISKLESS;
-		disk_max = D_UP_TO_DATE;
-		pdsk_min = D_DISKLESS;
-		pdsk_max = D_UP_TO_DATE;
-		break;
-	case C_WF_BITMAP_S:
-	case C_PAUSED_SYNC_S:
-	case C_STARTING_SYNC_S:
-	case C_AHEAD:
-		disk_min = D_UP_TO_DATE;
-		disk_max = D_UP_TO_DATE;
-		pdsk_min = D_INCONSISTENT;
-		pdsk_max = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/
-		break;
-	case C_SYNC_TARGET:
-		disk_min = D_INCONSISTENT;
-		disk_max = D_INCONSISTENT;
-		pdsk_min = D_UP_TO_DATE;
-		pdsk_max = D_UP_TO_DATE;
-		break;
-	case C_SYNC_SOURCE:
-		disk_min = D_UP_TO_DATE;
-		disk_max = D_UP_TO_DATE;
-		pdsk_min = D_INCONSISTENT;
-		pdsk_max = D_INCONSISTENT;
-		break;
-	case C_STANDALONE:
-	case C_DISCONNECTING:
-	case C_UNCONNECTED:
-	case C_TIMEOUT:
-	case C_BROKEN_PIPE:
-	case C_NETWORK_FAILURE:
-	case C_PROTOCOL_ERROR:
-	case C_TEAR_DOWN:
-	case C_WF_CONNECTION:
-	case C_WF_REPORT_PARAMS:
-	case C_MASK:
-		break;
-	}
-	if (ns.disk > disk_max)
-		ns.disk = disk_max;
+}
 
-	if (ns.disk < disk_min) {
-		if (warn)
-			*warn = IMPLICITLY_UPGRADED_DISK;
-		ns.disk = disk_min;
-	}
-	if (ns.pdsk > pdsk_max)
-		ns.pdsk = pdsk_max;
+static void drbd_select_sync_target(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	struct drbd_peer_device *target_current = NULL;
+	struct drbd_peer_device *target_active = NULL;
+	struct drbd_peer_device *target_desired = NULL;
 
-	if (ns.pdsk < pdsk_min) {
-		if (warn)
-			*warn = IMPLICITLY_UPGRADED_PDSK;
-		ns.pdsk = pdsk_min;
+	/* Find current and active resync peers. */
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->repl_state[OLD] == L_SYNC_TARGET && drbd_is_sync_target_candidate(peer_device))
+			target_current = peer_device;
+
+		if (peer_device->resync_active[NEW])
+			target_active = peer_device;
 	}
 
-	if (fp == FP_STONITH &&
-	    (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk > D_OUTDATED) &&
-	    !(os.role == R_PRIMARY && os.conn < C_CONNECTED && os.pdsk > D_OUTDATED))
-		ns.susp_fen = 1; /* Suspend IO while fence-peer handler runs (peer lost) */
+	/* Choose desired resync peer. */
+	for_each_peer_device_rcu(peer_device, device) {
+		if (!drbd_is_sync_target_candidate(peer_device))
+			continue;
 
-	if (device->resource->res_opts.on_no_data == OND_SUSPEND_IO &&
-	    (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) &&
-	    !(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE))
-		ns.susp_nod = 1; /* Suspend IO while no data available (no accessible data available) */
+		if (target_desired && drbd_bm_total_weight(peer_device) > drbd_bm_total_weight(target_desired))
+			continue;
 
-	if (ns.aftr_isp || ns.peer_isp || ns.user_isp) {
-		if (ns.conn == C_SYNC_SOURCE)
-			ns.conn = C_PAUSED_SYNC_S;
-		if (ns.conn == C_SYNC_TARGET)
-			ns.conn = C_PAUSED_SYNC_T;
-	} else {
-		if (ns.conn == C_PAUSED_SYNC_S)
-			ns.conn = C_SYNC_SOURCE;
-		if (ns.conn == C_PAUSED_SYNC_T)
-			ns.conn = C_SYNC_TARGET;
+		target_desired = peer_device;
 	}
 
-	return ns;
-}
+	/* Keep current resync target if the alternative has less than 1MiB
+	 * storage (256 bits) less to resync. */
+	if (target_current && target_desired &&
+			drbd_bm_total_weight(target_current) < drbd_bm_total_weight(target_desired) + 256UL)
+		target_desired = target_current;
 
-void drbd_resume_al(struct drbd_device *device)
-{
-	if (test_and_clear_bit(AL_SUSPENDED, &device->flags))
-		drbd_info(device, "Resumed AL updates\n");
-}
+	/* Do not activate/unpause a resync if some other is still active. */
+	if (target_desired && target_active && target_desired != target_active)
+		target_desired = NULL;
 
-/* helper for _drbd_set_state */
-static void set_ov_position(struct drbd_peer_device *peer_device, enum drbd_conns cs)
-{
-	struct drbd_device *device = peer_device->device;
+	/* Activate resync (if not already active). */
+	if (target_desired)
+		target_desired->resync_active[NEW] = true;
 
-	if (peer_device->connection->agreed_pro_version < 90)
-		device->ov_start_sector = 0;
-	device->rs_total = drbd_bm_bits(device);
-	device->ov_position = 0;
-	if (cs == C_VERIFY_T) {
-		/* starting online verify from an arbitrary position
-		 * does not fit well into the existing protocol.
-		 * on C_VERIFY_T, we initialize ov_left and friends
-		 * implicitly in receive_DataRequest once the
-		 * first P_OV_REQUEST is received */
-		device->ov_start_sector = ~(sector_t)0;
-	} else {
-		unsigned long bit = BM_SECT_TO_BIT(device->ov_start_sector);
-		if (bit >= device->rs_total) {
-			device->ov_start_sector =
-				BM_BIT_TO_SECT(device->rs_total - 1);
-			device->rs_total = 1;
-		} else
-			device->rs_total -= bit;
-		device->ov_position = device->ov_start_sector;
+	/* Make sure that the targets are correctly paused/unpaused. */
+	for_each_peer_device_rcu(peer_device, device) {
+		enum drbd_repl_state *repl_state = peer_device->repl_state;
+
+		peer_device->resync_susp_other_c[NEW] = target_desired && peer_device != target_desired;
+
+		if (!repl_is_sync_target(repl_state[NEW]))
+			continue;
+
+		peer_device->repl_state[NEW] = peer_device == target_desired ? L_SYNC_TARGET : L_PAUSED_SYNC_T;
 	}
-	device->ov_left = device->rs_total;
 }
 
-/**
- * _drbd_set_state() - Set a new DRBD state
- * @device:	DRBD device.
- * @ns:		new state.
- * @flags:	Flags
- * @done:	Optional completion, that will get completed after the after_state_ch() finished
- *
- * Caller needs to hold req_lock. Do not call directly.
- */
-enum drbd_state_rv
-_drbd_set_state(struct drbd_device *device, union drbd_state ns,
-	        enum chg_state_flags flags, struct completion *done)
+static bool drbd_change_to_inconsistent(enum drbd_disk_state *disk_state,
+		enum drbd_conn_state *cstate)
 {
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	union drbd_state os;
-	enum drbd_state_rv rv = SS_SUCCESS;
-	enum sanitize_state_warnings ssw;
-	struct after_state_chg_work *ascw;
-	struct drbd_state_change *state_change;
-
-	os = drbd_read_state(device);
+	return !(disk_state[OLD] == D_INCONSISTENT && cstate[OLD] == C_CONNECTED) &&
+		(disk_state[NEW] == D_INCONSISTENT && cstate[NEW] == C_CONNECTED);
+}
 
-	ns = sanitize_state(device, os, ns, &ssw);
-	if (ns.i == os.i)
-		return SS_NOTHING_TO_DO;
+static void sanitize_state(struct drbd_resource *resource)
+{
+	enum drbd_role *role = resource->role;
+	struct drbd_connection *connection;
+	struct drbd_device *device;
+	bool maybe_crashed_primary = false;
+	bool volume_lost_data_access = false;
+	bool volumes_have_data_access = true;
+	bool resource_has_quorum = true;
+	int connected_primaries = 0;
+	int vnr;
 
-	rv = is_valid_transition(os, ns);
-	if (rv < SS_SUCCESS)
-		return rv;
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		enum drbd_conn_state *cstate = connection->cstate;
 
-	if (!(flags & CS_HARD)) {
-		/*  pre-state-change checks ; only look at ns  */
-		/* See drbd_state_sw_errors in drbd_strings.c */
+		if (cstate[NEW] < C_CONNECTED)
+			connection->peer_role[NEW] = R_UNKNOWN;
 
-		rv = is_valid_state(device, ns);
-		if (rv < SS_SUCCESS) {
-			/* If the old state was illegal as well, then let
-			   this happen...*/
+		if (connection->peer_role[OLD] == R_PRIMARY && cstate[OLD] == C_CONNECTED &&
+		    ((cstate[NEW] >= C_TIMEOUT && cstate[NEW] <= C_PROTOCOL_ERROR) ||
+		     (cstate[NEW] == C_DISCONNECTING && resource->state_change_flags & CS_HARD)))
+			/* implies also C_BROKEN_PIPE and C_NETWORK_FAILURE */
+			maybe_crashed_primary = true;
 
-			if (is_valid_state(device, os) == rv)
-				rv = is_valid_soft_transition(os, ns, connection);
-		} else
-			rv = is_valid_soft_transition(os, ns, connection);
+		if (connection->peer_role[NEW] == R_PRIMARY)
+			connected_primaries++;
 	}
 
-	if (rv < SS_SUCCESS) {
-		if (flags & CS_VERBOSE)
-			print_st_err(device, os, ns, rv);
-		return rv;
-	}
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+		enum drbd_disk_state *disk_state = device->disk_state;
+		bool lost_connection = false;
+		bool have_good_peer = false;
+
+		if (disk_state[OLD] == D_DISKLESS && disk_state[NEW] == D_DETACHING)
+			disk_state[NEW] = D_DISKLESS;
+
+		if ((resource->state_change_flags & CS_IGN_OUTD_FAIL) &&
+		    disk_state[OLD] < D_OUTDATED && disk_state[NEW] == D_OUTDATED)
+			disk_state[NEW] = disk_state[OLD];
+
+		/* Is disk state negotiation finished? */
+		if (disk_state[OLD] == D_NEGOTIATING && disk_state[NEW] == D_NEGOTIATING) {
+			int all = 0, target = 0, no_result = 0;
+			bool up_to_date_neighbor = false;
+
+			for_each_peer_device_rcu(peer_device, device) {
+				enum drbd_repl_state repl_state = peer_device->repl_state[NEW];
+				enum drbd_repl_state nr = peer_device->negotiation_result;
+				enum drbd_disk_state pdsk = peer_device->disk_state[NEW];
+
+				if (pdsk < D_NEGOTIATING || repl_state == L_OFF)
+					continue;
+
+				if (pdsk == D_UP_TO_DATE)
+					up_to_date_neighbor = true;
+
+				all++;
+				if (nr == L_NEG_NO_RESULT)
+					no_result++;
+				else if (nr == L_NEGOTIATING)
+					goto stay_negotiating;
+				else if (nr == L_WF_BITMAP_T)
+					target++;
+				else if (nr != L_ESTABLISHED && nr != L_WF_BITMAP_S)
+					drbd_err(peer_device, "Unexpected nr = %s\n", drbd_repl_str(nr));
+			}
 
-	print_sanitize_warnings(device, ssw);
+			/* negotiation finished */
+			if (no_result > 0 && no_result == all)
+				disk_state[NEW] = D_DETACHING;
+			else if (target)
+				disk_state[NEW] = D_INCONSISTENT;
+			else
+				disk_state[NEW] = up_to_date_neighbor ? D_UP_TO_DATE :
+					disk_state_from_md(device);
+
+			for_each_peer_device_rcu(peer_device, device) {
+				enum drbd_repl_state nr = peer_device->negotiation_result;
+
+				if (peer_device->connection->cstate[NEW] < C_CONNECTED ||
+				    nr == L_NEGOTIATING)
+					continue;
+
+				if (nr == L_NEG_NO_RESULT)
+					nr = L_ESTABLISHED;
+
+				if (nr == L_WF_BITMAP_S && disk_state[NEW] == D_INCONSISTENT) {
+					/* Should be sync source for one peer and sync
+					   target for an other peer. Delay the sync source
+					   role */
+					nr = L_PAUSED_SYNC_S;
+					peer_device->resync_susp_other_c[NEW] = true;
+					drbd_warn(peer_device, "Finish me\n");
+				}
+				peer_device->repl_state[NEW] = nr;
+			}
+		}
+	stay_negotiating:
+
+		for_each_peer_device_rcu(peer_device, device) {
+			enum drbd_repl_state *repl_state = peer_device->repl_state;
+			enum drbd_disk_state *peer_disk_state = peer_device->disk_state;
+			struct drbd_connection *connection = peer_device->connection;
+			enum drbd_conn_state *cstate = connection->cstate;
+
+			if (peer_disk_state[NEW] == D_UP_TO_DATE &&
+			    (device->exposed_data_uuid & ~UUID_PRIMARY) ==
+			    (peer_device->current_uuid & ~UUID_PRIMARY))
+				have_good_peer = true;
+
+			if (repl_state[NEW] < L_ESTABLISHED) {
+				peer_device->resync_susp_peer[NEW] = false;
+				if (peer_disk_state[NEW] > D_UNKNOWN ||
+				    peer_disk_state[NEW] < D_INCONSISTENT)
+					peer_disk_state[NEW] = D_UNKNOWN;
+			}
+			if (repl_state[OLD] >= L_ESTABLISHED && repl_state[NEW] < L_ESTABLISHED) {
+				lost_connection = true;
+				peer_device->resync_active[NEW] = false;
+			}
 
-	drbd_pr_state_change(device, os, ns, flags);
+			/* Clear the aftr_isp when becoming unconfigured */
+			if (cstate[NEW] == C_STANDALONE &&
+			    disk_state[NEW] == D_DISKLESS &&
+			    role[NEW] == R_SECONDARY)
+				peer_device->resync_susp_dependency[NEW] = false;
+
+			/* Abort resync if a disk fails/detaches */
+			if (repl_state[NEW] > L_ESTABLISHED &&
+			    (disk_state[NEW] <= D_FAILED ||
+			     peer_disk_state[NEW] <= D_FAILED)) {
+				repl_state[NEW] = L_ESTABLISHED;
+				clear_bit(RECONCILIATION_RESYNC, &peer_device->flags);
+				peer_device->resync_active[NEW] = false;
+			}
 
-	/* Display changes to the susp* flags that where caused by the call to
-	   sanitize_state(). Only display it here if we where not called from
-	   _conn_request_state() */
-	if (!(flags & CS_DC_SUSP))
-		conn_pr_state_change(connection, os, ns,
-				     (flags & ~CS_DC_MASK) | CS_DC_SUSP);
+			/* Suspend IO while fence-peer handler runs (peer lost) */
+			if (connection->fencing_policy == FP_STONITH &&
+			    (role[NEW] == R_PRIMARY &&
+			     repl_state[NEW] < L_ESTABLISHED &&
+			     peer_disk_state[NEW] == D_UNKNOWN) &&
+			    (role[OLD] != R_PRIMARY ||
+			     peer_disk_state[OLD] != D_UNKNOWN))
+				connection->susp_fen[NEW] = true;
+		}
 
-	/* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
-	 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
-	 * drbd_ldev_destroy() won't happen before our corresponding
-	 * after_state_ch works run, where we put_ldev again. */
-	if ((os.disk != D_FAILED && ns.disk == D_FAILED) ||
-	    (os.disk != D_DISKLESS && ns.disk == D_DISKLESS))
-		atomic_inc(&device->local_cnt);
+		drbd_select_sync_target(device);
+
+		for_each_peer_device_rcu(peer_device, device) {
+			enum drbd_repl_state *repl_state = peer_device->repl_state;
+			enum drbd_disk_state *peer_disk_state = peer_device->disk_state;
+			struct drbd_connection *connection = peer_device->connection;
+			enum drbd_conn_state *cstate = connection->cstate;
+			enum drbd_disk_state min_disk_state, max_disk_state;
+			enum drbd_disk_state min_peer_disk_state, max_peer_disk_state;
+			enum drbd_role *peer_role = connection->peer_role;
+			bool uuids_match, cond;
+
+			/* Pause a SyncSource until it finishes resync as target on other connections */
+			if (repl_state[OLD] != L_SYNC_SOURCE && repl_state[NEW] == L_SYNC_SOURCE &&
+			    is_sync_target_other_c(peer_device))
+				peer_device->resync_susp_other_c[NEW] = true;
+
+			if (resync_suspended(peer_device, NEW)) {
+				if (repl_state[NEW] == L_SYNC_SOURCE)
+					repl_state[NEW] = L_PAUSED_SYNC_S;
+			} else {
+				if (repl_state[NEW] == L_PAUSED_SYNC_S)
+					repl_state[NEW] = L_SYNC_SOURCE;
+			}
 
-	if (!is_sync_state(os.conn) && is_sync_state(ns.conn))
-		clear_bit(RS_DONE, &device->flags);
+			/* Implication of the repl state on other peer's repl state */
+			if (repl_state[OLD] != L_STARTING_SYNC_T && repl_state[NEW] == L_STARTING_SYNC_T)
+				drbd_start_other_targets_paused(peer_device);
+
+			/* D_CONSISTENT vanish when we get connected (pre 9.0) */
+			if (connection->agreed_pro_version < 110 &&
+			    repl_state[NEW] >= L_ESTABLISHED && repl_state[NEW] < L_AHEAD) {
+				if (disk_state[NEW] == D_CONSISTENT)
+					disk_state[NEW] = D_UP_TO_DATE;
+				if (peer_disk_state[NEW] == D_CONSISTENT)
+					peer_disk_state[NEW] = D_UP_TO_DATE;
+			}
 
-	/* FIXME: Have any flags been set earlier in this function already? */
-	state_change = remember_old_state(device->resource, GFP_ATOMIC);
+			/* Implications of the repl state on the disk states */
+			min_disk_state = D_DISKLESS;
+			max_disk_state = D_UP_TO_DATE;
+			min_peer_disk_state = D_INCONSISTENT;
+			max_peer_disk_state = D_UNKNOWN;
+			switch (repl_state[NEW]) {
+			case L_OFF:
+				/* values from above */
+				break;
+			case L_WF_BITMAP_T:
+			case L_STARTING_SYNC_T:
+			case L_WF_SYNC_UUID:
+			case L_BEHIND:
+				min_disk_state = D_INCONSISTENT;
+				max_disk_state = D_OUTDATED;
+				min_peer_disk_state = D_INCONSISTENT;
+				max_peer_disk_state = D_UP_TO_DATE;
+				break;
+			case L_VERIFY_S:
+			case L_VERIFY_T:
+				min_disk_state = D_INCONSISTENT;
+				max_disk_state = D_UP_TO_DATE;
+				min_peer_disk_state = D_INCONSISTENT;
+				max_peer_disk_state = D_UP_TO_DATE;
+				break;
+			case L_ESTABLISHED:
+				min_disk_state = D_DISKLESS;
+				max_disk_state = D_UP_TO_DATE;
+				min_peer_disk_state = D_DISKLESS;
+				max_peer_disk_state = D_UP_TO_DATE;
+				break;
+			case L_WF_BITMAP_S:
+			case L_PAUSED_SYNC_S:
+			case L_STARTING_SYNC_S:
+			case L_AHEAD:
+				min_disk_state = D_INCONSISTENT;
+				max_disk_state = D_UP_TO_DATE;
+				min_peer_disk_state = D_INCONSISTENT;
+				max_peer_disk_state = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/
+				break;
+			case L_PAUSED_SYNC_T:
+			case L_SYNC_TARGET:
+				min_disk_state = D_INCONSISTENT;
+				max_disk_state = D_INCONSISTENT;
+				min_peer_disk_state = D_INCONSISTENT;
+				max_peer_disk_state = D_UP_TO_DATE;
+				break;
+			case L_SYNC_SOURCE:
+				min_disk_state = D_INCONSISTENT;
+				max_disk_state = D_UP_TO_DATE;
+				min_peer_disk_state = D_INCONSISTENT;
+				max_peer_disk_state = D_INCONSISTENT;
+				break;
+			}
 
-	/* changes to local_cnt and device flags should be visible before
-	 * changes to state, which again should be visible before anything else
-	 * depending on that change happens. */
-	smp_wmb();
-	device->state.i = ns.i;
-	device->resource->susp = ns.susp;
-	device->resource->susp_nod = ns.susp_nod;
-	device->resource->susp_fen = ns.susp_fen;
-	smp_wmb();
+			/* Implications of the repl state on the disk states */
+			if (disk_state[NEW] > max_disk_state)
+				disk_state[NEW] = max_disk_state;
+
+			if (disk_state[NEW] < min_disk_state)
+				disk_state[NEW] = min_disk_state;
+
+			if (peer_disk_state[NEW] > max_peer_disk_state)
+				peer_disk_state[NEW] = max_peer_disk_state;
+
+			if (peer_disk_state[NEW] < min_peer_disk_state)
+				peer_disk_state[NEW] = min_peer_disk_state;
+
+			/* A detach is a cluster wide transaction. The peer_disk_state updates
+			   are coming in while we have it prepared. When the cluster wide
+			   state change gets committed prevent D_DISKLESS -> D_FAILED */
+			if (peer_disk_state[OLD] == D_DISKLESS &&
+			    (peer_disk_state[NEW] == D_FAILED || peer_disk_state[NEW] == D_DETACHING))
+				peer_disk_state[NEW] = D_DISKLESS;
+
+			/* Upgrade myself from D_OUTDATED if..
+			   1) We connect to stable D_UP_TO_DATE(or D_CONSISTENT) peer without resync
+			   2) The peer just became stable
+			   3) the peer was stable and just became D_UP_TO_DATE */
+			if (repl_state[NEW] == L_ESTABLISHED && disk_state[NEW] == D_OUTDATED &&
+			    peer_disk_state[NEW] >= D_CONSISTENT && test_bit(UUIDS_RECEIVED, &peer_device->flags) &&
+			    peer_device->uuid_flags & UUID_FLAG_STABLE &&
+			    (repl_state[OLD] < L_ESTABLISHED ||
+			     peer_device->uuid_flags & UUID_FLAG_GOT_STABLE ||
+			     peer_disk_state[OLD] == D_OUTDATED))
+				disk_state[NEW] = peer_disk_state[NEW];
+
+			/* The attempted resync made us D_OUTDATED, roll that back in case */
+			if (repl_state[OLD] == L_WF_BITMAP_T && repl_state[NEW] == L_OFF &&
+			    disk_state[NEW] == D_OUTDATED &&
+			    stable_up_to_date_neighbor(device) && may_be_up_to_date(device, NEW))
+				disk_state[NEW] = D_UP_TO_DATE;
+
+			/* clause intentional here, the D_CONSISTENT form above might trigger this */
+			if (repl_state[OLD] < L_ESTABLISHED && repl_state[NEW] >= L_ESTABLISHED &&
+			    disk_state[NEW] == D_CONSISTENT && may_be_up_to_date(device, NEW))
+				disk_state[NEW] = D_UP_TO_DATE;
+
+			/* Follow a neighbor that goes from D_CONSISTENT TO D_UP_TO_DATE */
+			if (disk_state[NEW] == D_CONSISTENT &&
+			    peer_disk_state[OLD] == D_CONSISTENT &&
+			    peer_disk_state[NEW] == D_UP_TO_DATE &&
+			    peer_device->uuid_flags & UUID_FLAG_STABLE)
+				disk_state[NEW] = D_UP_TO_DATE;
+
+			peer_device->uuid_flags &= ~UUID_FLAG_GOT_STABLE;
+
+			uuids_match =
+				(peer_device->current_uuid & ~UUID_PRIMARY) ==
+				(drbd_current_uuid(device) & ~UUID_PRIMARY);
+
+			if (peer_role[OLD] == R_UNKNOWN && peer_role[NEW] == R_PRIMARY &&
+			    peer_disk_state[NEW] == D_DISKLESS && disk_state[NEW] >= D_NEGOTIATING) {
+				/* Got connected to a diskless primary */
+				if (uuids_match && !is_sync_target_other_c(peer_device)) {
+					if (device->disk_state[NOW] < D_UP_TO_DATE) {
+						drbd_info(peer_device, "Upgrading local disk to D_UP_TO_DATE since current UUID matches.\n");
+						disk_state[NEW] = D_UP_TO_DATE;
+					}
+				} else {
+					set_bit(TRY_TO_GET_RESYNC, &device->flags);
+					if (disk_state[NEW] == D_UP_TO_DATE) {
+						drbd_info(peer_device, "Downgrading local disk to D_CONSISTENT since current UUID differs.\n");
+						disk_state[NEW] = D_CONSISTENT;
+						/* This is a "safety net"; it can only happen if fencing and quorum
+						   are both disabled. This alone would be racy, look for
+						   "Do not trust this guy!" (see also may_return_to_up_to_date()) */
+					}
+				}
+			}
 
-	remember_new_state(state_change);
+			if (connection->agreed_features & DRBD_FF_RS_SKIP_UUID)
+				cond = have_good_peer &&
+					(device->exposed_data_uuid & ~UUID_PRIMARY) !=
+					(peer_device->current_uuid & ~UUID_PRIMARY);
+			else
+				cond = peer_disk_state[OLD] == D_UNKNOWN &&
+					role[NEW] == R_PRIMARY && !uuids_match;
+
+			if (disk_state[NEW] == D_DISKLESS && peer_disk_state[NEW] == D_UP_TO_DATE &&
+			    cond) {
+				/* Do not trust this guy!
+				   He wants to be D_UP_TO_DATE, but has a different current
+				   UUID. Do not accept him as D_UP_TO_DATE but downgrade that to
+				   D_CONSISTENT here.
+				*/
+				peer_disk_state[NEW] = D_CONSISTENT;
+			}
 
-	/* put replicated vs not-replicated requests in seperate epochs */
-	if (drbd_should_do_remote((union drbd_dev_state)os.i) !=
-	    drbd_should_do_remote((union drbd_dev_state)ns.i))
-		start_new_tl_epoch(connection);
+			/*
+			 * Determine whether peer will disable replication due to this transition.
+			 *
+			 * This matches the condition on the peer below.
+			 */
+			if (drbd_change_to_inconsistent(disk_state, cstate) ||
+					(!repl_is_sync_target(repl_state[OLD]) &&
+					 repl_is_sync_target(repl_state[NEW])))
+				peer_device->peer_replication[NEW] =
+					test_bit(PEER_REPLICATION_NEXT, &peer_device->flags);
+
+			/*
+			 * Decide whether to disable replication when the peer
+			 * transitions to Inconsistent. Only consider the disk
+			 * state when we are Connected because we want to wait
+			 * until we know whether replication should be enabled
+			 * on the next transition to Inconsistent. This is
+			 * communicated with the P_ENABLE_REPLICATION_NEXT
+			 * packet.
+			 *
+			 * Also re-evaluate whether to disable replication when
+			 * we become SyncSource, even when the peer's disk was
+			 * already Inconsistent. This is relevant when
+			 * switching between Ahead-Behind+Inconsistent and
+			 * SyncSource-SyncTarget.
+			 *
+			 * This matches the condition on the peer above.
+			 */
+			if (drbd_change_to_inconsistent(peer_disk_state, cstate) ||
+					(!repl_is_sync_source(repl_state[OLD]) &&
+					 repl_is_sync_source(repl_state[NEW])))
+				peer_device->replication[NEW] =
+					test_bit(REPLICATION_NEXT, &peer_device->flags);
+
+			/*
+			 * Not strictly necessary, since "replication" is only
+			 * considered when the peer disk is Inconsistent, but
+			 * it makes the logs clearer.
+			 */
+			if (peer_disk_state[OLD] == D_INCONSISTENT &&
+					peer_disk_state[NEW] != D_INCONSISTENT)
+				peer_device->replication[NEW] = true;
+		}
 
-	if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING)
-		drbd_print_uuids(device, "attached to UUIDs");
+		if (resource->res_opts.quorum != QOU_OFF)
+			device->have_quorum[NEW] = calc_quorum(device, NULL);
+		else
+			device->have_quorum[NEW] = true;
 
-	/* Wake up role changes, that were delayed because of connection establishing */
-	if (os.conn == C_WF_REPORT_PARAMS && ns.conn != C_WF_REPORT_PARAMS &&
-	    no_peer_wf_report_params(connection)) {
-		clear_bit(STATE_SENT, &connection->flags);
-		wake_up_all_devices(connection);
-	}
+		if (!device->have_quorum[NEW] && disk_state[NEW] == D_UP_TO_DATE &&
+		    test_bit(RESTORE_QUORUM, &device->flags)) {
+			device->have_quorum[NEW] = true;
+			set_bit(RESTORING_QUORUM, &device->flags);
+		}
 
-	wake_up(&device->misc_wait);
-	wake_up(&device->state_wait);
-	wake_up(&connection->ping_wait);
+		if (!device->have_quorum[NEW])
+			resource_has_quorum = false;
 
-	/* Aborted verify run, or we reached the stop sector.
-	 * Log the last position, unless end-of-device. */
-	if ((os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) &&
-	    ns.conn <= C_CONNECTED) {
-		device->ov_start_sector =
-			BM_BIT_TO_SECT(drbd_bm_bits(device) - device->ov_left);
-		if (device->ov_left)
-			drbd_info(device, "Online Verify reached sector %llu\n",
-				(unsigned long long)device->ov_start_sector);
-	}
+		/* Suspend IO if we have no accessible data available.
+		 * Policy may be extended later to be able to suspend
+		 * if redundancy falls below a certain level. */
+		if (role[NEW] == R_PRIMARY && !drbd_data_accessible(device, NEW)) {
+			volumes_have_data_access = false;
+			if (role[OLD] != R_PRIMARY || drbd_data_accessible(device, OLD))
+				volume_lost_data_access = true;
+		}
 
-	if ((os.conn == C_PAUSED_SYNC_T || os.conn == C_PAUSED_SYNC_S) &&
-	    (ns.conn == C_SYNC_TARGET  || ns.conn == C_SYNC_SOURCE)) {
-		drbd_info(device, "Syncer continues.\n");
-		device->rs_paused += (long)jiffies
-				  -(long)device->rs_mark_time[device->rs_last_mark];
-		if (ns.conn == C_SYNC_TARGET)
-			mod_timer(&device->resync_timer, jiffies);
-	}
+		if (lost_connection && disk_state[NEW] == D_NEGOTIATING)
+			disk_state[NEW] = disk_state_from_md(device);
 
-	if ((os.conn == C_SYNC_TARGET  || os.conn == C_SYNC_SOURCE) &&
-	    (ns.conn == C_PAUSED_SYNC_T || ns.conn == C_PAUSED_SYNC_S)) {
-		drbd_info(device, "Resync suspended\n");
-		device->rs_mark_time[device->rs_last_mark] = jiffies;
+		if (maybe_crashed_primary && !connected_primaries &&
+		    disk_state[NEW] == D_UP_TO_DATE && role[NOW] == R_SECONDARY)
+			disk_state[NEW] = D_CONSISTENT;
 	}
+	rcu_read_unlock();
 
-	if (os.conn == C_CONNECTED &&
-	    (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T)) {
-		unsigned long now = jiffies;
-		int i;
-
-		set_ov_position(peer_device, ns.conn);
-		device->rs_start = now;
-		device->rs_last_sect_ev = 0;
-		device->ov_last_oos_size = 0;
-		device->ov_last_oos_start = 0;
-
-		for (i = 0; i < DRBD_SYNC_MARKS; i++) {
-			device->rs_mark_left[i] = device->ov_left;
-			device->rs_mark_time[i] = now;
-		}
+	if (volumes_have_data_access)
+		resource->susp_nod[NEW] = false;
+	if (volume_lost_data_access && resource->res_opts.on_no_data == OND_SUSPEND_IO)
+		resource->susp_nod[NEW] = true;
 
-		drbd_rs_controller_reset(peer_device);
+	resource->susp_quorum[NEW] =
+		resource->res_opts.on_no_quorum == ONQ_SUSPEND_IO ? !resource_has_quorum : false;
 
-		if (ns.conn == C_VERIFY_S) {
-			drbd_info(device, "Starting Online Verify from sector %llu\n",
-					(unsigned long long)device->ov_position);
-			mod_timer(&device->resync_timer, jiffies);
+	if (!resource->susp_uuid[OLD] &&
+	    resource_is_suspended(resource, OLD) && !resource_is_suspended(resource, NEW)) {
+		idr_for_each_entry(&resource->devices, device, vnr) {
+			if (test_bit(NEW_CUR_UUID, &device->flags)) {
+				resource->susp_uuid[NEW] = true;
+				break;
+			}
 		}
 	}
 
-	if (get_ldev(device)) {
-		u32 mdf = device->ldev->md.flags & ~(MDF_CONSISTENT|MDF_PRIMARY_IND|
-						 MDF_CONNECTED_IND|MDF_WAS_UP_TO_DATE|
-						 MDF_PEER_OUT_DATED|MDF_CRASHED_PRIMARY);
-
-		mdf &= ~MDF_AL_CLEAN;
-		if (test_bit(CRASHED_PRIMARY, &device->flags))
-			mdf |= MDF_CRASHED_PRIMARY;
-		if (device->state.role == R_PRIMARY ||
-		    (device->state.pdsk < D_INCONSISTENT && device->state.peer == R_PRIMARY))
-			mdf |= MDF_PRIMARY_IND;
-		if (device->state.conn > C_WF_REPORT_PARAMS)
-			mdf |= MDF_CONNECTED_IND;
-		if (device->state.disk > D_INCONSISTENT)
-			mdf |= MDF_CONSISTENT;
-		if (device->state.disk > D_OUTDATED)
-			mdf |= MDF_WAS_UP_TO_DATE;
-		if (device->state.pdsk <= D_OUTDATED && device->state.pdsk >= D_INCONSISTENT)
-			mdf |= MDF_PEER_OUT_DATED;
-		if (mdf != device->ldev->md.flags) {
-			device->ldev->md.flags = mdf;
-			drbd_md_mark_dirty(device);
-		}
-		if (os.disk < D_CONSISTENT && ns.disk >= D_CONSISTENT)
-			drbd_set_ed_uuid(device, device->ldev->md.uuid[UI_CURRENT]);
-		put_ldev(device);
-	}
-
-	/* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */
-	if (os.disk == D_INCONSISTENT && os.pdsk == D_INCONSISTENT &&
-	    os.peer == R_SECONDARY && ns.peer == R_PRIMARY)
-		set_bit(CONSIDER_RESYNC, &device->flags);
-
-	/* Receiver should clean up itself */
-	if (os.conn != C_DISCONNECTING && ns.conn == C_DISCONNECTING)
-		drbd_thread_stop_nowait(&connection->receiver);
-
-	/* Now the receiver finished cleaning up itself, it should die */
-	if (os.conn != C_STANDALONE && ns.conn == C_STANDALONE)
-		drbd_thread_stop_nowait(&connection->receiver);
-
-	/* Upon network failure, we need to restart the receiver. */
-	if (os.conn > C_WF_CONNECTION &&
-	    ns.conn <= C_TEAR_DOWN && ns.conn >= C_TIMEOUT)
-		drbd_thread_restart_nowait(&connection->receiver);
-
-	/* Resume AL writing if we get a connection */
-	if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED) {
-		drbd_resume_al(device);
-		connection->connect_cnt++;
-	}
-
-	/* remember last attach time so request_timer_fn() won't
-	 * kill newly established sessions while we are still trying to thaw
-	 * previously frozen IO */
-	if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) &&
-	    ns.disk > D_NEGOTIATING)
-		device->last_reattach_jif = jiffies;
-
-	ascw = kmalloc_obj(*ascw, GFP_ATOMIC);
-	if (ascw) {
-		ascw->os = os;
-		ascw->ns = ns;
-		ascw->flags = flags;
-		ascw->w.cb = w_after_state_ch;
-		ascw->device = device;
-		ascw->done = done;
-		ascw->state_change = state_change;
-		drbd_queue_work(&connection->sender_work,
-				&ascw->w);
-	} else {
-		drbd_err(device, "Could not kmalloc an ascw\n");
+	if (role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY &&
+	    (resource->state_change_flags & CS_FS_IGN_OPENERS)) {
+		int rw_count, ro_count;
+		drbd_open_counts(resource, &rw_count, &ro_count);
+		if (rw_count)
+			resource->fail_io[NEW] = true;
 	}
+}
 
-	return rv;
+void drbd_resume_al(struct drbd_device *device)
+{
+	if (test_and_clear_bit(AL_SUSPENDED, &device->flags))
+		drbd_info(device, "Resumed AL updates\n");
 }
 
-static int w_after_state_ch(struct drbd_work *w, int unused)
+static bool drbd_need_twopc_after_lost_peer(struct drbd_connection *connection)
 {
-	struct after_state_chg_work *ascw =
-		container_of(w, struct after_state_chg_work, w);
-	struct drbd_device *device = ascw->device;
+	enum drbd_conn_state *cstate = connection->cstate;
 
-	after_state_ch(device, ascw->os, ascw->ns, ascw->flags, ascw->state_change);
-	forget_state_change(ascw->state_change);
-	if (ascw->flags & CS_WAIT_COMPLETE)
-		complete(ascw->done);
-	kfree(ascw);
+	/* Is the state change a disconnect? */
+	if (!(cstate[OLD] == C_CONNECTED && cstate[NEW] < C_CONNECTED))
+		return false;
 
-	return 0;
+	/*
+	 * The peer did not provide reachable_nodes when disconnecting, so
+	 * trigger a twopc ourselves.
+	 */
+	if (!(connection->agreed_features & DRBD_FF_2PC_V2))
+		return true;
+
+	/* Trigger a twopc if it was a non-graceful disconnect. */
+	return cstate[NEW] != C_TEAR_DOWN;
 }
 
-static void abw_start_sync(struct drbd_device *device, int rv)
+static void drbd_schedule_empty_twopc(struct drbd_resource *resource)
 {
-	if (rv) {
-		drbd_err(device, "Writing the bitmap failed not starting resync.\n");
-		_drbd_request_state(device, NS(conn, C_CONNECTED), CS_VERBOSE);
+	kref_get(&resource->kref);
+	if (!schedule_work(&resource->empty_twopc)) {
+		kref_put(&resource->kref, drbd_destroy_resource);
+	}
+}
+
+/*
+ * We cache a node mask of the online members of the cluster. It might
+ * be off because a node is still marked as online immediately after
+ * it crashes. That means it might have an online mark for an already
+ * offline node. On the other hand, we guarantee that it never has
+ * a zero for an online node.
+ */
+static void update_members(struct drbd_resource *resource)
+{
+	enum chg_state_flags flags = resource->state_change_flags;
+	struct twopc_reply *reply = &resource->twopc_reply;
+	const int my_node_id = resource->res_opts.node_id;
+	struct drbd_connection *connection;
+
+	/* in case we initiated 2PC we know the reachable nodes */
+	if (flags & CS_TWOPC && reply->initiator_node_id == my_node_id) {
+		resource->members = reply->reachable_nodes;
 		return;
 	}
 
-	switch (device->state.conn) {
-	case C_STARTING_SYNC_T:
-		_drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE);
-		break;
-	case C_STARTING_SYNC_S:
-		drbd_start_resync(device, C_SYNC_SOURCE);
-		break;
+	/* In case I am 2PC target of a connect or non-graceful disconnect */
+	for_each_connection(connection, resource) {
+		enum drbd_conn_state *cstate = connection->cstate;
+		const int peer_node_mask = NODE_MASK(connection->peer_node_id);
+
+		/* add a fresh connection to the members */
+		if (cstate[OLD] < C_CONNECTED && cstate[NEW] == C_CONNECTED)
+			resource->members |= peer_node_mask;
+
+		/* Connection to peer lost. Check if we should remove it from the members */
+		if (drbd_need_twopc_after_lost_peer(connection) &&
+				resource->members & peer_node_mask)
+			drbd_schedule_empty_twopc(resource);
 	}
 }
 
-int drbd_bitmap_io_from_worker(struct drbd_device *device,
-		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
-		char *why, enum bm_flag flags,
-		struct drbd_peer_device *peer_device)
+static bool drbd_any_peer_device_up_to_date(struct drbd_connection *connection)
 {
-	int rv;
+	int vnr;
+	struct drbd_peer_device *peer_device;
 
-	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (peer_device->disk_state[NEW] == D_UP_TO_DATE)
+			return true;
+	}
 
-	/* open coded non-blocking drbd_suspend_io(device); */
-	atomic_inc(&device->suspend_cnt);
+	return false;
+}
 
-	drbd_bm_lock(device, why, flags);
-	rv = io_fn(device, peer_device);
-	drbd_bm_unlock(device);
+/* Whether replication is enabled on all peers for this device */
+bool drbd_all_peer_replication(struct drbd_device *device, enum which_state which)
+{
+	struct drbd_peer_device *peer_device;
+	bool all_peer_replication = true;
 
-	drbd_resume_io(device);
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		if (!peer_device->peer_replication[which])
+			all_peer_replication = false;
+	}
+	rcu_read_unlock();
 
-	return rv;
+	return all_peer_replication;
 }
 
-int notify_resource_state_change(struct sk_buff *skb,
-				  unsigned int seq,
-				  void *state_change,
+/* As drbd_all_peer_replication() but takes a state change object */
+static bool drbd_all_peer_replication_change(struct drbd_state_change *state_change, int n_device,
+		enum which_state which)
+{
+	int n_connection;
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[
+				n_device * state_change->n_connections + n_connection];
+
+		if (!peer_device_state_change->peer_replication[which])
+			return false;
+	}
+
+	return true;
+}
+
+static void drbd_determine_flush_pending(struct drbd_resource *resource)
+{
+	struct drbd_device *device;
+	struct drbd_connection *primary_connection;
+	struct drbd_connection *up_to_date_connection;
+	int vnr;
+	bool send_flush_requests = false;
+
+	/* Clear any bits if we no longer expect or require a flush ack */
+	spin_lock(&resource->initiator_flush_lock);
+	for_each_connection(primary_connection, resource) {
+		u64 *pending_flush_mask = &primary_connection->pending_flush_mask;
+
+		/*
+		 * Clear bits if we no longer expect or require a flush ack due
+		 * to loss of connection to the Primary peer.
+		 */
+		if (primary_connection->cstate[NEW] != C_CONNECTED) {
+			if (*pending_flush_mask)
+				*pending_flush_mask = 0;
+			continue;
+		}
+
+		/*
+		 * Clear bits if we no longer expect or require a flush ack
+		 * because the peer that was UpToDate is no longer UpToDate.
+		 * For instance, if we lose the connection to that peer.
+		 */
+		for_each_connection(up_to_date_connection, resource) {
+			u64 up_to_date_mask = NODE_MASK(up_to_date_connection->peer_node_id);
+
+			if (drbd_any_peer_device_up_to_date(up_to_date_connection))
+				continue;
+
+			if (*pending_flush_mask & up_to_date_mask)
+				*pending_flush_mask &= ~up_to_date_mask;
+		}
+	}
+	spin_unlock(&resource->initiator_flush_lock);
+
+	/* Check if we need a new flush */
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+
+		for_each_peer_device(peer_device, device) {
+			if (!(is_sync_target_state(peer_device, NOW) &&
+					drbd_all_peer_replication(device, NOW)) &&
+					is_sync_target_state(peer_device, NEW) &&
+					drbd_all_peer_replication(device, NEW))
+				send_flush_requests = true;
+		}
+	}
+
+	if (!send_flush_requests)
+		return;
+
+	/* We need a new flush. Mark which acks we are waiting for. */
+	spin_lock(&resource->initiator_flush_lock);
+	resource->current_flush_sequence++;
+
+	for_each_connection(primary_connection, resource) {
+		primary_connection->pending_flush_mask = 0;
+
+		if (primary_connection->peer_role[NEW] != R_PRIMARY)
+			continue;
+
+		if (primary_connection->agreed_pro_version < 123)
+			continue;
+
+		for_each_connection(up_to_date_connection, resource) {
+			u64 up_to_date_mask = NODE_MASK(up_to_date_connection->peer_node_id);
+
+			if (!drbd_any_peer_device_up_to_date(up_to_date_connection))
+				continue;
+
+			if (up_to_date_connection->agreed_pro_version < 123)
+				continue;
+
+			primary_connection->pending_flush_mask |= up_to_date_mask;
+		}
+	}
+	spin_unlock(&resource->initiator_flush_lock);
+}
+
+static void set_ov_position(struct drbd_peer_device *peer_device,
+			    enum drbd_repl_state repl_state)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap;
+
+	if (peer_device->connection->agreed_pro_version < 90)
+		peer_device->ov_start_sector = 0;
+	peer_device->rs_total = drbd_bm_bits(device);
+	peer_device->ov_position = 0;
+	if (repl_state == L_VERIFY_T) {
+		/* starting online verify from an arbitrary position
+		 * does not fit well into the existing protocol.
+		 * on L_VERIFY_T, we initialize ov_left and friends
+		 * implicitly in receive_common_data_request once the
+		 * first P_OV_REQUEST is received */
+		peer_device->ov_start_sector = ~(sector_t)0;
+	} else {
+		unsigned long bit = bm_sect_to_bit(bm, peer_device->ov_start_sector);
+		if (bit >= peer_device->rs_total) {
+			peer_device->ov_start_sector =
+				bm_bit_to_sect(bm, peer_device->rs_total - 1);
+			peer_device->rs_total = 1;
+		} else
+			peer_device->rs_total -= bit;
+		peer_device->ov_position = peer_device->ov_start_sector;
+	}
+	atomic64_set(&peer_device->ov_left, peer_device->rs_total);
+	peer_device->ov_skipped = 0;
+}
+
+static void initialize_resync_progress_marks(struct drbd_peer_device *peer_device)
+{
+	unsigned long tw = drbd_bm_total_weight(peer_device);
+	unsigned long now = jiffies;
+	int i;
+
+	peer_device->rs_last_progress_report_ts = now;
+	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
+		peer_device->rs_mark_left[i] = tw;
+		peer_device->rs_mark_time[i] = now;
+	}
+}
+
+static void initialize_resync(struct drbd_peer_device *peer_device)
+{
+	unsigned long tw = drbd_bm_total_weight(peer_device);
+	unsigned long now = jiffies;
+
+	peer_device->last_peers_in_sync_end = 0;
+	peer_device->resync_next_bit = 0;
+	peer_device->last_resync_pass_bits = tw;
+	peer_device->rs_failed = 0;
+	peer_device->rs_paused = 0;
+	peer_device->rs_same_csum = 0;
+	peer_device->rs_total = tw;
+	peer_device->rs_start = now;
+	peer_device->rs_last_writeout = now;
+	initialize_resync_progress_marks(peer_device);
+	drbd_rs_controller_reset(peer_device);
+}
+
+/* Is there a primary with access to up to date data known */
+static bool primary_and_data_present(struct drbd_device *device)
+{
+	bool up_to_date_data = device->disk_state[NEW] == D_UP_TO_DATE;
+	struct drbd_resource *resource = device->resource;
+	bool primary = resource->role[NEW] == R_PRIMARY;
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device) {
+		struct drbd_connection *connection = peer_device->connection;
+
+		/* Do not consider the peer if we are disconnecting. */
+		if (resource->remote_state_change &&
+				drbd_twopc_between_peer_and_me(connection) &&
+				resource->twopc_reply.is_disconnect)
+			continue;
+
+		if (connection->peer_role[NEW] == R_PRIMARY)
+			primary = true;
+
+		if (peer_device->disk_state[NEW] == D_UP_TO_DATE)
+			up_to_date_data = true;
+	}
+
+	return primary && up_to_date_data;
+}
+
+static bool extra_ldev_ref_for_after_state_chg(enum drbd_disk_state *disk_state)
+{
+	return (disk_state[OLD] != D_FAILED && disk_state[NEW] == D_FAILED) ||
+	       (disk_state[OLD] != D_DETACHING && disk_state[NEW] == D_DETACHING) ||
+	       (disk_state[OLD] != D_DISKLESS && disk_state[NEW] == D_DISKLESS);
+}
+
+static bool has_starting_resyncs(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (peer_device->repl_state[NEW] > L_ESTABLISHED)
+			return true;
+	}
+	return false;
+}
+
+static bool should_try_become_up_to_date(struct drbd_device *device, enum drbd_disk_state *disk_state,
+		enum which_state which)
+{
+	return disk_state[OLD] == D_UP_TO_DATE && disk_state[NEW] == D_CONSISTENT &&
+			may_return_to_up_to_date(device, which);
+}
+
+/**
+ * finish_state_change  -  carry out actions triggered by a state change
+ * @resource: DBRD resource.
+ * @tag: State change tag to print in status messages.
+ */
+static void finish_state_change(struct drbd_resource *resource, const char *tag)
+{
+	enum drbd_role *role = resource->role;
+	bool *susp_uuid = resource->susp_uuid;
+	struct drbd_device *device;
+	struct drbd_connection *connection;
+	bool starting_resync = false;
+	bool start_new_epoch = false;
+	bool lost_a_primary_peer = false;
+	bool some_peer_is_primary = false;
+	bool some_peer_request_in_flight = false;
+	bool resource_suspended[2];
+	bool unfreeze_io = false;
+	int vnr;
+
+	print_state_change(resource, "", tag);
+
+	resource_suspended[OLD] = resource_is_suspended(resource, OLD);
+	resource_suspended[NEW] = resource_is_suspended(resource, NEW);
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		bool *have_quorum = device->have_quorum;
+		struct drbd_peer_device *peer_device;
+
+		for_each_peer_device(peer_device, device) {
+			struct drbd_connection *connection = peer_device->connection;
+			bool did, should;
+
+			did = drbd_should_do_remote(peer_device, NOW);
+			should = drbd_should_do_remote(peer_device, NEW);
+
+			if (!did && should) {
+				/* Since "did" is false, the request with this
+				 * dagtag and prior requests were not be marked
+				 * to be sent to this peer. Hence this will not
+				 * send a dagtag packet before the
+				 * corresponding data packet.
+				 *
+				 * It is possible that this peer does not
+				 * actually have the data corresponding to this
+				 * dagtag. However in that case, the disk state
+				 * of that peer will not be D_UP_TO_DATE, so it
+				 * not be relevant what dagtag we have sent it. */
+				connection->send_dagtag = resource->dagtag_sector;
+				drbd_queue_work_if_unqueued(
+						&connection->sender_work,
+						&connection->send_dagtag_work);
+			}
+
+			if (did != should)
+				start_new_epoch = true;
+
+			if (peer_device->repl_state[OLD] != L_WF_BITMAP_S &&
+					peer_device->repl_state[NEW] == L_WF_BITMAP_S)
+				clear_bit(B_RS_H_DONE, &peer_device->flags);
+
+			if (peer_device->repl_state[OLD] != L_WF_BITMAP_T &&
+					peer_device->repl_state[NEW] == L_WF_BITMAP_T)
+				clear_bit(B_RS_H_DONE, &peer_device->flags);
+
+			if (!is_sync_state(peer_device, NOW) &&
+			    is_sync_state(peer_device, NEW)) {
+				clear_bit(RS_DONE, &peer_device->flags);
+				clear_bit(B_RS_H_DONE, &peer_device->flags);
+				clear_bit(SYNC_TARGET_TO_BEHIND, &peer_device->flags);
+			}
+		}
+
+		if (role[NEW] == R_PRIMARY && !have_quorum[NEW])
+			set_bit(PRIMARY_LOST_QUORUM, &device->flags);
+	}
+	if (start_new_epoch)
+		start_new_tl_epoch(resource);
+
+	spin_lock(&resource->peer_ack_lock);
+	if (role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY && resource->peer_ack_req) {
+		resource->last_peer_acked_dagtag = resource->peer_ack_req->dagtag_sector;
+		drbd_queue_peer_ack(resource, resource->peer_ack_req);
+		resource->peer_ack_req = NULL;
+	}
+	spin_unlock(&resource->peer_ack_lock);
+
+	drbd_determine_flush_pending(resource);
+
+	if (!resource->fail_io[OLD] && resource->fail_io[NEW])
+		drbd_warn(resource, "Failing IOs\n");
+
+	for_each_connection(connection, resource) {
+		enum drbd_role *peer_role = connection->peer_role;
+		enum drbd_conn_state *cstate = connection->cstate;
+
+		if (peer_role[NEW] == R_PRIMARY)
+			some_peer_is_primary = true;
+
+		switch (cstate[NEW]) {
+		case C_CONNECTED:
+			if (atomic_read(&connection->active_ee_cnt)
+					|| atomic_read(&connection->done_ee_cnt))
+				some_peer_request_in_flight = true;
+			break;
+		case C_STANDALONE:
+		case C_UNCONNECTED:
+		case C_CONNECTING:
+			/* maybe others are safe as well? which ones? */
+			break;
+		default:
+			/* if we just disconnected, there may still be some request in flight. */
+			some_peer_request_in_flight = true;
+		}
+
+		if (some_peer_is_primary && some_peer_request_in_flight)
+			break;
+	}
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		struct drbd_peer_device *peer_device;
+		struct drbd_bitmap *bm = device->bitmap;
+		enum drbd_disk_state *disk_state = device->disk_state;
+		bool create_new_uuid = false;
+
+		if (test_bit(RESTORING_QUORUM, &device->flags) &&
+		    !device->have_quorum[OLD] && device->have_quorum[NEW]) {
+			clear_bit(RESTORING_QUORUM, &device->flags);
+			drbd_info(resource, "Restored quorum from before reboot\n");
+		}
+
+		if (test_bit(RESTORE_QUORUM, &device->flags) &&
+		    (device->have_quorum[NEW] || disk_state[NEW] < D_UP_TO_DATE))
+			clear_bit(RESTORE_QUORUM, &device->flags);
+
+		if (disk_state[OLD] != D_NEGOTIATING && disk_state[NEW] == D_NEGOTIATING) {
+			for_each_peer_device(peer_device, device)
+				peer_device->negotiation_result = L_NEGOTIATING;
+		}
+
+		/* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
+		 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
+		 * drbd_ldev_destroy() won't happen before our corresponding
+		 * w_after_state_change works run, where we put_ldev again. */
+		if (extra_ldev_ref_for_after_state_chg(disk_state))
+			atomic_inc(&device->local_cnt);
+
+		if (disk_state[OLD] != D_DISKLESS && disk_state[NEW] == D_DISKLESS) {
+			/* who knows if we are ever going to be attached again,
+			 * and whether that will be the same device, or a newly
+			 * initialized one. */
+			for_each_peer_device(peer_device, device)
+				peer_device->bitmap_index = -1;
+		}
+
+		if (disk_state[OLD] == D_ATTACHING && disk_state[NEW] >= D_NEGOTIATING)
+			drbd_info(device, "attached to current UUID: %016llX\n", device->ldev->md.current_uuid);
+
+		for_each_peer_device(peer_device, device) {
+			enum drbd_repl_state *repl_state = peer_device->repl_state;
+			enum drbd_disk_state *peer_disk_state = peer_device->disk_state;
+			struct drbd_connection *connection = peer_device->connection;
+			enum drbd_role *peer_role = connection->peer_role;
+
+			if (repl_state[OLD] <= L_ESTABLISHED && repl_state[NEW] == L_WF_BITMAP_S)
+				starting_resync = true;
+
+			if ((disk_state[OLD] != D_UP_TO_DATE || peer_disk_state[OLD] != D_UP_TO_DATE) &&
+			    (disk_state[NEW] == D_UP_TO_DATE && peer_disk_state[NEW] == D_UP_TO_DATE)) {
+				clear_bit(CRASHED_PRIMARY, &device->flags);
+				if (test_bit(UUIDS_RECEIVED, &peer_device->flags))
+					peer_device->uuid_flags &= ~((u64)UUID_FLAG_CRASHED_PRIMARY);
+			}
+
+			/* Aborted verify run, or we reached the stop sector.
+			 * Log the last position, unless end-of-device. */
+			if ((repl_state[OLD] == L_VERIFY_S || repl_state[OLD] == L_VERIFY_T) &&
+			    repl_state[NEW] <= L_ESTABLISHED) {
+				unsigned long ov_left = atomic64_read(&peer_device->ov_left);
+
+				peer_device->ov_start_sector =
+					bm_bit_to_sect(bm, drbd_bm_bits(device) - ov_left);
+				if (ov_left)
+					drbd_info(peer_device, "Online Verify reached sector %llu\n",
+						  (unsigned long long)peer_device->ov_start_sector);
+			}
+
+			if ((repl_state[OLD] == L_PAUSED_SYNC_T || repl_state[OLD] == L_PAUSED_SYNC_S) &&
+			    (repl_state[NEW] == L_SYNC_TARGET  || repl_state[NEW] == L_SYNC_SOURCE)) {
+				drbd_info(peer_device, "Syncer continues.\n");
+				peer_device->rs_paused += (long)jiffies
+						  -(long)peer_device->rs_mark_time[peer_device->rs_last_mark];
+				initialize_resync_progress_marks(peer_device);
+				peer_device->resync_next_bit = 0;
+			}
+
+			if ((repl_state[OLD] == L_SYNC_TARGET  || repl_state[OLD] == L_SYNC_SOURCE) &&
+			    (repl_state[NEW] == L_PAUSED_SYNC_T || repl_state[NEW] == L_PAUSED_SYNC_S)) {
+				drbd_info(peer_device, "Resync suspended\n");
+				peer_device->rs_mark_time[peer_device->rs_last_mark] = jiffies;
+			}
+
+
+			if (repl_state[OLD] > L_ESTABLISHED && repl_state[NEW] <= L_ESTABLISHED)
+				clear_bit(RECONCILIATION_RESYNC, &peer_device->flags);
+
+			if (repl_state[OLD] >= L_ESTABLISHED && repl_state[NEW] < L_ESTABLISHED)
+				clear_bit(AHEAD_TO_SYNC_SOURCE, &peer_device->flags);
+
+			if (repl_state[OLD] == L_ESTABLISHED &&
+			    (repl_state[NEW] == L_VERIFY_S || repl_state[NEW] == L_VERIFY_T)) {
+				unsigned long now = jiffies;
+				int i;
+
+				set_ov_position(peer_device, repl_state[NEW]);
+				peer_device->rs_start = now;
+				peer_device->ov_last_oos_size = 0;
+				peer_device->ov_last_oos_start = 0;
+				peer_device->ov_last_skipped_size = 0;
+				peer_device->ov_last_skipped_start = 0;
+				peer_device->rs_last_writeout = now;
+				peer_device->rs_last_progress_report_ts = now;
+				for (i = 0; i < DRBD_SYNC_MARKS; i++) {
+					peer_device->rs_mark_left[i] = peer_device->rs_total;
+					peer_device->rs_mark_time[i] = now;
+				}
+
+				drbd_rs_controller_reset(peer_device);
+			} else if (!(repl_state[OLD] >= L_SYNC_SOURCE && repl_state[OLD] <= L_PAUSED_SYNC_T) &&
+				   (repl_state[NEW] >= L_SYNC_SOURCE && repl_state[NEW] <= L_PAUSED_SYNC_T)) {
+				initialize_resync(peer_device);
+			}
+
+			if (disk_state[NEW] != D_NEGOTIATING && get_ldev(device)) {
+				if (peer_device->bitmap_index != -1) {
+					enum drbd_disk_state pdsk = peer_device->disk_state[NEW];
+					u32 mdf = device->ldev->md.peers[peer_device->node_id].flags;
+					/* Do NOT clear MDF_PEER_DEVICE_SEEN here.
+					 * We want to be able to refuse a resize beyond "last agreed" size,
+					 * even if the peer is currently detached.
+					 */
+					mdf &= ~(MDF_PEER_CONNECTED | MDF_PEER_OUTDATED | MDF_PEER_FENCING);
+					if (repl_state[NEW] > L_OFF)
+						mdf |= MDF_PEER_CONNECTED;
+					if (pdsk >= D_INCONSISTENT) {
+						if (pdsk <= D_OUTDATED)
+							mdf |= MDF_PEER_OUTDATED;
+						if (pdsk != D_UNKNOWN)
+							mdf |= MDF_PEER_DEVICE_SEEN;
+					}
+					if (pdsk == D_DISKLESS && !want_bitmap(peer_device))
+						mdf &= ~MDF_PEER_DEVICE_SEEN;
+					if (peer_device->connection->fencing_policy != FP_DONT_CARE)
+						mdf |= MDF_PEER_FENCING;
+					if (mdf != device->ldev->md.peers[peer_device->node_id].flags) {
+						device->ldev->md.peers[peer_device->node_id].flags = mdf;
+						drbd_md_mark_dirty(device);
+					}
+				}
+
+				/* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */
+				if (disk_state[OLD] == D_INCONSISTENT &&
+				    peer_disk_state[OLD] == D_INCONSISTENT && peer_disk_state[NEW] == D_UP_TO_DATE &&
+				    peer_role[OLD] == R_SECONDARY && peer_role[NEW] == R_PRIMARY)
+					set_bit(CONSIDER_RESYNC, &peer_device->flags);
+
+				/* Resume AL writing if we get a connection */
+				if (repl_state[OLD] < L_ESTABLISHED && repl_state[NEW] >= L_ESTABLISHED)
+					drbd_resume_al(device);
+				put_ldev(device);
+			}
+
+			if (repl_state[OLD] == L_AHEAD && repl_state[NEW] == L_SYNC_SOURCE) {
+				set_bit(SEND_STATE_AFTER_AHEAD, &peer_device->flags);
+				set_bit(SEND_STATE_AFTER_AHEAD_C, &connection->flags);
+
+				clear_bit(CONN_CONGESTED, &connection->flags);
+				wake_up(&connection->sender_work.q_wait);
+			}
+
+			/* We start writing locally without replicating the changes,
+			 * better start a new data generation */
+			if (repl_state[OLD] != L_AHEAD && repl_state[NEW] == L_AHEAD)
+				create_new_uuid = true;
+
+			if (lost_contact_to_peer_data(peer_disk_state)) {
+				if (role[NEW] == R_PRIMARY && !test_bit(UNREGISTERED, &device->flags) &&
+				    drbd_data_accessible(device, NEW))
+					create_new_uuid = true;
+
+				if (connection->agreed_pro_version < 110 &&
+				    peer_role[NEW] == R_PRIMARY &&
+				    disk_state[NEW] >= D_UP_TO_DATE)
+					create_new_uuid = true;
+			}
+			if (peer_returns_diskless(peer_device, peer_disk_state[OLD], peer_disk_state[NEW])) {
+				if (role[NEW] == R_PRIMARY && !test_bit(UNREGISTERED, &device->flags) &&
+				    disk_state[NEW] == D_UP_TO_DATE)
+					create_new_uuid = true;
+			}
+
+			if (disk_state[OLD] > D_FAILED && disk_state[NEW] == D_FAILED &&
+			    role[NEW] == R_PRIMARY && drbd_data_accessible(device, NEW))
+				create_new_uuid = true;
+
+			if (peer_disk_state[NEW] < D_UP_TO_DATE && test_bit(GOT_NEG_ACK, &peer_device->flags))
+				clear_bit(GOT_NEG_ACK, &peer_device->flags);
+
+			if (repl_state[OLD] > L_ESTABLISHED && repl_state[NEW] <= L_ESTABLISHED)
+				clear_bit(SYNC_SRC_CRASHED_PRI, &peer_device->flags);
+
+			if (peer_role[OLD] != peer_role[NEW] || role[OLD] != role[NEW] ||
+			    peer_disk_state[OLD] != peer_disk_state[NEW])
+				drbd_update_mdf_al_disabled(device, NEW);
+		}
+
+		if (disk_state[OLD] >= D_INCONSISTENT && disk_state[NEW] < D_INCONSISTENT &&
+		    role[NEW] == R_PRIMARY && drbd_data_accessible(device, NEW))
+			create_new_uuid = true;
+
+		if (role[OLD] == R_SECONDARY && role[NEW] == R_PRIMARY)
+			create_new_uuid = true;
+
+		/* Only a single new current uuid when susp_uuid becomes true */
+		if (create_new_uuid && !susp_uuid[OLD])
+			set_bit(__NEW_CUR_UUID, &device->flags);
+
+		if (disk_state[NEW] != D_NEGOTIATING && get_ldev_if_state(device, D_DETACHING)) {
+			u32 mdf = device->ldev->md.flags;
+			bool graceful_detach = disk_state[NEW] == D_DETACHING && !test_bit(FORCE_DETACH, &device->flags);
+
+			/* For now, always require a drbdmeta apply-al run,
+			 * even if that ends up only re-initializing the AL */
+			mdf &= ~MDF_AL_CLEAN;
+			/* reset some flags to what we know now */
+			mdf &= ~MDF_CRASHED_PRIMARY;
+			if (test_bit(CRASHED_PRIMARY, &device->flags) ||
+			    (role[NEW] == R_PRIMARY && !graceful_detach))
+				mdf |= MDF_CRASHED_PRIMARY;
+			mdf &= ~MDF_PRIMARY_LOST_QUORUM;
+			if (test_bit(PRIMARY_LOST_QUORUM, &device->flags))
+				mdf |= MDF_PRIMARY_LOST_QUORUM;
+			/* Do not touch MDF_CONSISTENT if we are D_FAILED */
+			if (disk_state[NEW] >= D_INCONSISTENT) {
+				mdf &= ~(MDF_CONSISTENT | MDF_WAS_UP_TO_DATE);
+
+				if (disk_state[NEW] > D_INCONSISTENT)
+					mdf |= MDF_CONSISTENT;
+				if (disk_state[NEW] > D_OUTDATED)
+					mdf |= MDF_WAS_UP_TO_DATE;
+			} else if ((disk_state[NEW] == D_FAILED || disk_state[NEW] == D_DETACHING) &&
+				   mdf & MDF_WAS_UP_TO_DATE &&
+				   primary_and_data_present(device)) {
+				/* There are cases when we still can update meta-data even if disk
+				   state is failed.... Clear MDF_WAS_UP_TO_DATE if appropriate */
+				mdf &= ~MDF_WAS_UP_TO_DATE;
+			}
+
+/*
+ * MDF_PRIMARY_IND  IS set: apply activity log after crash
+ * MDF_PRIMARY_IND NOT set: do not apply, forget and re-initialize activity log after crash.
+ * We want the MDF_PRIMARY_IND set *always* before our backend could possibly
+ * be target of write requests, whether we are Secondary or Primary ourselves.
+ *
+ * We want to avoid to clear that flag just because we lost the connection to a
+ * detached Primary, but before all in-flight IO was drained, because we may
+ * have some dirty bits not yet persisted.
+ *
+ * We want it cleared only once we are *certain* that we no longer see any Primary,
+ * are not Primary ourselves, AND all previously received WRITE (peer-) requests
+ * have been processed, NOTHING is in flight against our backend anymore,
+ * AND we have successfully written out any dirty bitmap pages.
+ *
+ *
+ * MDF_PEER_DEVICE_SEEN ... The peer had a backing device at some point
+ * MDF_NODE_EXISTS ... We have seen evidence that this node exists in the cluster.
+ *   Note: This bit does **not** get set when a new peer/connection is created with
+ *   `drbdsetup new-peer ...`.  The bit gets set when we establish a connection
+ *   successfully for the first time or we learn via other nodes about the
+ *   existence.
+ */
+
+			/* set, if someone is/becomes primary */
+			if (role[NEW] == R_PRIMARY || some_peer_is_primary)
+				mdf |= MDF_PRIMARY_IND;
+			/* clear, if */
+			else if (/* NO peer requests in flight, AND */
+				 !some_peer_request_in_flight &&
+				 (graceful_detach ||
+				  /* or everyone secondary ... */
+				  (role[NEW] == R_SECONDARY && !some_peer_is_primary &&
+				   /* ... and not detaching because of IO error. */
+				   disk_state[NEW] >= D_INCONSISTENT)))
+				mdf &= ~MDF_PRIMARY_IND;
+
+			if (device->have_quorum[NEW])
+				mdf |= MDF_HAVE_QUORUM;
+			else
+				mdf &= ~MDF_HAVE_QUORUM;
+			/* apply changed flags to md.flags,
+			 * and "schedule" for write-out */
+			if (mdf != device->ldev->md.flags ||
+			    device->ldev->md.members != resource->members) {
+				device->ldev->md.flags = mdf;
+				device->ldev->md.members = resource->members;
+				drbd_md_mark_dirty(device);
+			}
+			if (disk_state[OLD] < D_CONSISTENT && disk_state[NEW] >= D_CONSISTENT)
+				drbd_uuid_set_exposed(device, device->ldev->md.current_uuid, true);
+			put_ldev(device);
+		}
+
+		/* remember last attach time so request_timer_fn() won't
+		 * kill newly established sessions while we are still trying to thaw
+		 * previously frozen IO */
+		if ((disk_state[OLD] == D_ATTACHING || disk_state[OLD] == D_NEGOTIATING) &&
+		    disk_state[NEW] > D_NEGOTIATING)
+			device->last_reattach_jif = jiffies;
+
+		if (!device->have_quorum[OLD] && device->have_quorum[NEW])
+			clear_bit(PRIMARY_LOST_QUORUM, &device->flags);
+
+		if (resource_suspended[NEW] &&
+		    !(role[OLD] == R_PRIMARY && !drbd_data_accessible(device, OLD)) &&
+		     (role[NEW] == R_PRIMARY && !drbd_data_accessible(device, NEW)) &&
+		    resource->res_opts.on_no_data == OND_IO_ERROR)
+			unfreeze_io = true;
+
+		if (!resource->fail_io[OLD] && resource->fail_io[NEW])
+			unfreeze_io = true;
+
+		if (role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY)
+			clear_bit(NEW_CUR_UUID, &device->flags);
+
+		if (should_try_become_up_to_date(device, disk_state, NEW))
+			set_bit(TRY_BECOME_UP_TO_DATE_PENDING, &resource->flags);
+	}
+
+	for_each_connection(connection, resource) {
+		enum drbd_conn_state *cstate = connection->cstate;
+		enum drbd_role *peer_role = connection->peer_role;
+
+		/*
+		 * If we lose connection to a Primary node then we need to
+		 * inform our peers so that we can potentially do a
+		 * reconciliation resync. The function conn_disconnect()
+		 * informs the peers. So we must set the flag before stopping
+		 * the receiver.
+		 */
+		if (cstate[OLD] == C_CONNECTED && cstate[NEW] < C_CONNECTED &&
+				peer_role[OLD] == R_PRIMARY)
+			set_bit(NOTIFY_PEERS_LOST_PRIMARY, &connection->flags);
+
+		/* Receiver should clean up itself */
+		if (cstate[OLD] != C_DISCONNECTING && cstate[NEW] == C_DISCONNECTING)
+			drbd_thread_stop_nowait(&connection->receiver);
+
+		/* Now the receiver finished cleaning up itself, it should die */
+		if (cstate[OLD] != C_STANDALONE && cstate[NEW] == C_STANDALONE)
+			drbd_thread_stop_nowait(&connection->receiver);
+
+		/* Upon network failure, we need to restart the receiver. */
+		if (cstate[OLD] >= C_CONNECTING &&
+		    cstate[NEW] <= C_TEAR_DOWN && cstate[NEW] >= C_TIMEOUT)
+			drbd_thread_restart_nowait(&connection->receiver);
+
+		if (cstate[OLD] == C_CONNECTED && cstate[NEW] < C_CONNECTED)
+			twopc_connection_down(connection);
+
+		/* remember last connect time so request_timer_fn() won't
+		 * kill newly established sessions while we are still trying to thaw
+		 * previously frozen IO */
+		if (cstate[OLD] < C_CONNECTED && cstate[NEW] == C_CONNECTED)
+			connection->last_reconnect_jif = jiffies;
+
+		if (resource_suspended[OLD]) {
+			enum drbd_req_event walk_event = -1;
+
+			/* If we resume IO without this connection, then we
+			 * need to cancel suspended requests. */
+			if ((!resource_suspended[NEW] || unfreeze_io) && cstate[NEW] < C_CONNECTED)
+				walk_event = CANCEL_SUSPENDED_IO;
+			/* On reconnection when we have been suspended we need
+			 * to process suspended requests. If there are resyncs,
+			 * that means that it was not a simple disconnect and
+			 * reconnect, so we cannot resend. We must cancel
+			 * instead. */
+			else if (cstate[OLD] < C_CONNECTED && cstate[NEW] == C_CONNECTED)
+				walk_event = has_starting_resyncs(connection) ? CANCEL_SUSPENDED_IO : RESEND;
+
+			if (walk_event != -1)
+				__tl_walk(resource, connection, &connection->req_not_net_done, walk_event);
+
+			/* Since we are in finish_state_change(), and the state
+			 * was previously not C_CONNECTED, the sender cannot
+			 * have received any requests yet. So it will find any
+			 * requests to resend when it rescans the transfer log. */
+			if (walk_event == RESEND)
+				wake_up(&connection->sender_work.q_wait);
+		}
+
+		if (cstate[OLD] == C_CONNECTED && cstate[NEW] < C_CONNECTED)
+			set_bit(RECONNECT, &connection->flags);
+
+		if (starting_resync && peer_role[NEW] == R_PRIMARY)
+			apply_unacked_peer_requests(connection);
+
+		if (peer_role[OLD] == R_PRIMARY && peer_role[NEW] == R_UNKNOWN)
+			lost_a_primary_peer = true;
+	}
+
+	if (lost_a_primary_peer) {
+		idr_for_each_entry(&resource->devices, device, vnr) {
+			struct drbd_peer_device *peer_device;
+
+			for_each_peer_device(peer_device, device) {
+				enum drbd_repl_state repl_state = peer_device->repl_state[NEW];
+
+				if (!test_bit(UNSTABLE_RESYNC, &peer_device->flags) &&
+				    (repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T) &&
+				    !(peer_device->uuid_flags & UUID_FLAG_STABLE) &&
+				    !drbd_stable_sync_source_present(peer_device, NEW))
+					set_bit(UNSTABLE_RESYNC, &peer_device->flags);
+			}
+		}
+	}
+
+	if (resource_suspended[OLD] && !resource_suspended[NEW])
+		drbd_restart_suspended_reqs(resource);
+
+	if ((resource_suspended[OLD] && !resource_suspended[NEW]) || unfreeze_io)
+		__tl_walk(resource, NULL, NULL, COMPLETION_RESUMED);
+}
+
+static void abw_start_sync(struct drbd_device *device,
+			   struct drbd_peer_device *peer_device, int rv)
+{
+	struct drbd_peer_device *pd;
+
+	if (rv) {
+		drbd_err(device, "Writing the bitmap failed not starting resync.\n");
+		stable_change_repl_state(peer_device, L_ESTABLISHED, CS_VERBOSE, "start-sync");
+		return;
+	}
+
+	switch (peer_device->repl_state[NOW]) {
+	case L_STARTING_SYNC_T:
+		/* Since the number of set bits changed and the other peer_devices are
+		   lready in L_PAUSED_SYNC_T state, we need to set rs_total here */
+		rcu_read_lock();
+		for_each_peer_device_rcu(pd, device)
+			initialize_resync(pd);
+		rcu_read_unlock();
+
+		if (peer_device->connection->agreed_pro_version < 110)
+			stable_change_repl_state(peer_device, L_WF_SYNC_UUID, CS_VERBOSE,
+					"start-sync");
+		else
+			drbd_start_resync(peer_device, L_SYNC_TARGET, "start-sync");
+		break;
+	case L_STARTING_SYNC_S:
+		drbd_start_resync(peer_device, L_SYNC_SOURCE, "start-sync");
+		break;
+	default:
+		break;
+	}
+}
+
+int drbd_bitmap_io_from_worker(struct drbd_device *device,
+		int (*io_fn)(struct drbd_device *, struct drbd_peer_device *),
+		char *why, enum bm_flag flags,
+		struct drbd_peer_device *peer_device)
+{
+	int rv;
+
+	D_ASSERT(device, current == device->resource->worker.task);
+
+	if (!device->bitmap)
+		return 0;
+
+	/* open coded non-blocking drbd_suspend_io(device); */
+	atomic_inc(&device->suspend_cnt);
+
+	if (flags & BM_LOCK_SINGLE_SLOT)
+		drbd_bm_slot_lock(peer_device, why, flags);
+	else
+		drbd_bm_lock(device, why, flags);
+	rv = io_fn(device, peer_device);
+	if (flags & BM_LOCK_SINGLE_SLOT)
+		drbd_bm_slot_unlock(peer_device);
+	else
+		drbd_bm_unlock(device);
+
+	drbd_resume_io(device);
+
+	return rv;
+}
+
+static bool state_change_is_susp_fen(struct drbd_state_change *state_change,
+					    enum which_state which)
+{
+	int n_connection;
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+				&state_change->connections[n_connection];
+
+		if (connection_state_change->susp_fen[which])
+			return true;
+	}
+
+	return false;
+}
+
+static bool state_change_is_susp_quorum(struct drbd_state_change *state_change,
+					       enum which_state which)
+{
+	struct drbd_resource *resource = state_change->resource[0].resource;
+	int n_device;
+
+	if (resource->res_opts.on_no_quorum != ONQ_SUSPEND_IO)
+		return false;
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change =
+				&state_change->devices[n_device];
+
+		if (!device_state_change->have_quorum[which])
+			return true;
+	}
+
+	return false;
+}
+
+static bool resync_susp_comb_dep_sc(struct drbd_state_change *state_change,
+				    unsigned int n_device, int n_connection,
+				    enum which_state which)
+{
+	struct drbd_peer_device_state_change *peer_device_state_change =
+		&state_change->peer_devices[n_device * state_change->n_connections + n_connection];
+	struct drbd_device_state_change *device_state_change = &state_change->devices[n_device];
+	bool resync_susp_dependency = peer_device_state_change->resync_susp_dependency[which];
+	bool resync_susp_other_c = peer_device_state_change->resync_susp_other_c[which];
+	enum drbd_repl_state repl_state = peer_device_state_change->repl_state[which];
+	enum drbd_disk_state disk_state = device_state_change->disk_state[which];
+
+	return resync_susp_dependency || resync_susp_other_c ||
+		((repl_state == L_SYNC_SOURCE || repl_state == L_PAUSED_SYNC_S)
+		 && disk_state <= D_INCONSISTENT);
+}
+
+static union drbd_state state_change_word(struct drbd_state_change *state_change,
+					  unsigned int n_device, int n_connection,
+					  enum which_state which)
+{
+	struct drbd_resource_state_change *resource_state_change =
+		&state_change->resource[0];
+	struct drbd_device_state_change *device_state_change =
+		&state_change->devices[n_device];
+	union drbd_state state = { {
+		.role = R_UNKNOWN,
+		.peer = R_UNKNOWN,
+		.conn = C_STANDALONE,
+		.disk = D_UNKNOWN,
+		.pdsk = D_UNKNOWN,
+	} };
+
+	state.role = resource_state_change->role[which];
+	state.susp = resource_state_change->susp[which] || state_change_is_susp_quorum(state_change, which) ||
+		resource_state_change->susp_uuid[which];
+	state.susp_nod = resource_state_change->susp_nod[which];
+	state.susp_fen = state_change_is_susp_fen(state_change, which);
+	state.quorum = device_state_change->have_quorum[which];
+	state.disk = device_state_change->disk_state[which];
+	if (n_connection != -1) {
+		struct drbd_connection_state_change *connection_state_change =
+			&state_change->connections[n_connection];
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[n_device * state_change->n_connections + n_connection];
+
+		state.peer = connection_state_change->peer_role[which];
+		state.conn = peer_device_state_change->repl_state[which];
+		if (state.conn <= L_OFF)
+			state.conn = connection_state_change->cstate[which];
+		state.pdsk = peer_device_state_change->disk_state[which];
+		state.aftr_isp = resync_susp_comb_dep_sc(state_change, n_device, n_connection, which);
+		state.peer_isp = peer_device_state_change->resync_susp_peer[which];
+		state.user_isp = peer_device_state_change->resync_susp_user[which];
+	}
+	return state;
+}
+
+int notify_resource_state_change(struct sk_buff *skb,
+				  unsigned int seq,
+				  void *state_change,
 				  enum drbd_notification_type type)
 {
-	struct drbd_resource_state_change *resource_state_change = state_change;
-	struct drbd_resource *resource = resource_state_change->resource;
-	struct resource_info resource_info = {
-		.res_role = resource_state_change->role[NEW],
-		.res_susp = resource_state_change->susp[NEW],
-		.res_susp_nod = resource_state_change->susp_nod[NEW],
-		.res_susp_fen = resource_state_change->susp_fen[NEW],
-	};
+	struct drbd_resource_state_change *resource_state_change =
+		((struct drbd_state_change *)state_change)->resource;
+	struct drbd_resource *resource = resource_state_change->resource;
+	struct resource_info resource_info = {
+		.res_role = resource_state_change->role[NEW],
+		.res_susp = resource_state_change->susp[NEW],
+		.res_susp_nod = resource_state_change->susp_nod[NEW],
+		.res_susp_fen = state_change_is_susp_fen(state_change, NEW),
+		.res_susp_quorum = state_change_is_susp_quorum(state_change, NEW) ||
+			resource_state_change->susp_uuid[NEW],
+		.res_fail_io = resource_state_change->fail_io[NEW],
+	};
+
+	return notify_resource_state(skb, seq, resource, &resource_info, NULL, type);
+}
+
+int notify_connection_state_change(struct sk_buff *skb,
+				    unsigned int seq,
+				    void *state_change,
+				    enum drbd_notification_type type)
+{
+	struct drbd_connection_state_change *connection_state_change = state_change;
+	struct drbd_connection *connection = connection_state_change->connection;
+	struct connection_info connection_info = {
+		.conn_connection_state = connection_state_change->cstate[NEW],
+		.conn_role = connection_state_change->peer_role[NEW],
+	};
+
+	return notify_connection_state(skb, seq, connection, &connection_info, type);
+}
+
+int notify_device_state_change(struct sk_buff *skb,
+				unsigned int seq,
+				void *state_change,
+				enum drbd_notification_type type)
+{
+	struct drbd_device_state_change *device_state_change = state_change;
+	struct drbd_device *device = device_state_change->device;
+	struct device_info device_info;
+	device_state_change_to_info(&device_info, device_state_change);
+
+	return notify_device_state(skb, seq, device, &device_info, type);
+}
+
+int notify_peer_device_state_change(struct sk_buff *skb,
+				     unsigned int seq,
+				     void *state_change,
+				     enum drbd_notification_type type)
+{
+	struct drbd_peer_device_state_change *peer_device_state_change = state_change;
+	struct drbd_peer_device *peer_device = peer_device_state_change->peer_device;
+	struct peer_device_info peer_device_info;
+	peer_device_state_change_to_info(&peer_device_info, state_change);
+
+	return notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type);
+}
+
+static void notify_state_change(struct drbd_state_change *state_change)
+{
+	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+	bool resource_state_has_changed;
+	unsigned int n_device, n_connection, n_peer_device, n_peer_devices;
+	int (*last_func)(struct sk_buff *, unsigned int, void *,
+			  enum drbd_notification_type) = NULL;
+	void *last_arg = NULL;
+
+#define HAS_CHANGED(state) ((state)[OLD] != (state)[NEW])
+#define FINAL_STATE_CHANGE(type) \
+	({ if (last_func) \
+		last_func(NULL, 0, last_arg, type); \
+	})
+#define REMEMBER_STATE_CHANGE(func, arg, type) \
+	({ FINAL_STATE_CHANGE(type | NOTIFY_CONTINUES); \
+	   last_func = (typeof(last_func))func; \
+	   last_arg = arg; \
+	 })
+
+	mutex_lock(&notification_mutex);
+
+	resource_state_has_changed =
+		HAS_CHANGED(resource_state_change->role) ||
+		HAS_CHANGED(resource_state_change->susp) ||
+		HAS_CHANGED(resource_state_change->susp_nod) ||
+		HAS_CHANGED(resource_state_change->susp_uuid) ||
+		state_change_is_susp_fen(state_change, OLD) !=
+		state_change_is_susp_fen(state_change, NEW) ||
+		state_change_is_susp_quorum(state_change, OLD) !=
+		state_change_is_susp_quorum(state_change, NEW) ||
+		HAS_CHANGED(resource_state_change->fail_io);
+
+	if (resource_state_has_changed)
+		REMEMBER_STATE_CHANGE(notify_resource_state_change,
+				      state_change, NOTIFY_CHANGE);
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+				&state_change->connections[n_connection];
+
+		if (HAS_CHANGED(connection_state_change->peer_role) ||
+		    HAS_CHANGED(connection_state_change->cstate))
+			REMEMBER_STATE_CHANGE(notify_connection_state_change,
+					      connection_state_change, NOTIFY_CHANGE);
+	}
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change =
+			&state_change->devices[n_device];
+
+		if (HAS_CHANGED(device_state_change->disk_state) ||
+		    HAS_CHANGED(device_state_change->have_quorum))
+			REMEMBER_STATE_CHANGE(notify_device_state_change,
+					      device_state_change, NOTIFY_CHANGE);
+	}
+
+	n_peer_devices = state_change->n_devices * state_change->n_connections;
+	for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) {
+		struct drbd_peer_device_state_change *p =
+			&state_change->peer_devices[n_peer_device];
+
+		if (HAS_CHANGED(p->disk_state) ||
+		    HAS_CHANGED(p->repl_state) ||
+		    HAS_CHANGED(p->resync_susp_user) ||
+		    HAS_CHANGED(p->resync_susp_peer) ||
+		    HAS_CHANGED(p->resync_susp_dependency) ||
+		    HAS_CHANGED(p->resync_susp_other_c))
+			REMEMBER_STATE_CHANGE(notify_peer_device_state_change,
+					      p, NOTIFY_CHANGE);
+	}
+
+	FINAL_STATE_CHANGE(NOTIFY_CHANGE);
+	mutex_unlock(&notification_mutex);
+
+#undef HAS_CHANGED
+#undef FINAL_STATE_CHANGE
+#undef REMEMBER_STATE_CHANGE
+}
+
+static void send_role_to_all_peers(struct drbd_state_change *state_change)
+{
+	unsigned int n_connection;
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+			&state_change->connections[n_connection];
+		struct drbd_connection *connection = connection_state_change->connection;
+		enum drbd_conn_state new_cstate = connection_state_change->cstate[NEW];
+
+		if (new_cstate < C_CONNECTED)
+			continue;
+
+		if (connection->agreed_pro_version < 110) {
+			unsigned int n_device;
+
+			/* Before DRBD 9, the role is a device attribute
+			 * instead of a resource attribute. */
+			for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+				struct drbd_peer_device *peer_device =
+					state_change->peer_devices[n_connection].peer_device;
+				union drbd_state state =
+					state_change_word(state_change, n_device, n_connection, NEW);
+
+				drbd_send_state(peer_device, state);
+			}
+		} else {
+			union drbd_state state = { {
+				.role = state_change->resource[0].role[NEW],
+			} };
+
+			conn_send_state(connection, state);
+		}
+	}
+}
+
+static void send_new_state_to_all_peer_devices(struct drbd_state_change *state_change, int n_device)
+{
+	unsigned int n_connection;
+
+	BUG_ON(state_change->n_devices <= n_device);
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[n_device * state_change->n_connections + n_connection];
+		struct drbd_peer_device *peer_device = peer_device_state_change->peer_device;
+		union drbd_state new_state = state_change_word(state_change, n_device, n_connection, NEW);
+
+		if (new_state.conn >= C_CONNECTED)
+			drbd_send_state(peer_device, new_state);
+	}
+}
+
+/* This function is supposed to have the same semantics as drbd_device_stable() in drbd_main.c
+   A primary is stable since it is authoritative.
+   Unstable are neighbors of a primary and resync target nodes.
+   Nodes further away from a primary are stable! Do no confuse with "weak".*/
+static bool calc_device_stable(struct drbd_state_change *state_change, int n_device, enum which_state which)
+{
+	int n_connection;
+
+	if (state_change->resource->role[which] == R_PRIMARY)
+		return true;
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change =
+			&state_change->connections[n_connection];
+		enum drbd_role *peer_role = connection_state_change->peer_role;
+
+		if (peer_role[which] == R_PRIMARY)
+			return false;
+	}
+
+	return true;
+}
+
+static bool calc_resync_target(struct drbd_state_change *state_change, int n_device, enum which_state which)
+{
+	int n_connection;
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[n_device * state_change->n_connections + n_connection];
+		enum drbd_repl_state *repl_state = peer_device_state_change->repl_state;
+
+		switch (repl_state[which]) {
+		case L_WF_BITMAP_T:
+		case L_SYNC_TARGET:
+		case L_PAUSED_SYNC_T:
+			return true;
+		default:
+			continue;
+		}
+	}
+
+	return false;
+}
+
+/* takes old and new peer disk state */
+static bool lost_contact_to_peer_data(enum drbd_disk_state *peer_disk_state)
+{
+	enum drbd_disk_state os = peer_disk_state[OLD];
+	enum drbd_disk_state ns = peer_disk_state[NEW];
+
+	return (os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED)
+		&& (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED);
+}
+
+static bool peer_returns_diskless(struct drbd_peer_device *peer_device,
+				  enum drbd_disk_state os, enum drbd_disk_state ns)
+{
+	struct drbd_device *device = peer_device->device;
+	bool rv = false;
+
+	/* Scenario, starting with normal operation
+	 * Connected Primary/Secondary UpToDate/UpToDate
+	 * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen)
+	 * ...
+	 * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!)
+	 */
+
+	if (get_ldev(device)) {
+		if (os == D_UNKNOWN && (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED) &&
+		    drbd_bitmap_uuid(peer_device) == 0)
+			rv = true;
+		put_ldev(device);
+	}
+	return rv;
+}
+
+static void check_may_resume_io_after_fencing(struct drbd_state_change *state_change, int n_connection)
+{
+	struct drbd_connection_state_change *connection_state_change = &state_change->connections[n_connection];
+	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+	struct drbd_connection *connection = connection_state_change->connection;
+	struct drbd_resource *resource = resource_state_change->resource;
+	bool all_peer_disks_outdated = true;
+	bool all_peer_disks_connected = true;
+	struct drbd_peer_device *peer_device;
+	unsigned long irq_flags;
+	int vnr, n_device;
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[n_device * state_change->n_connections + n_connection];
+		enum drbd_repl_state *repl_state = peer_device_state_change->repl_state;
+		enum drbd_disk_state *peer_disk_state = peer_device_state_change->disk_state;
+
+		if (peer_disk_state[NEW] > D_OUTDATED)
+			all_peer_disks_outdated = false;
+		if (repl_state[NEW] < L_ESTABLISHED)
+			all_peer_disks_connected = false;
+	}
+
+	/* case1: The outdate peer handler is successful: */
+	if (all_peer_disks_outdated) {
+		rcu_read_lock();
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			struct drbd_device *device = peer_device->device;
+			if (test_and_clear_bit(NEW_CUR_UUID, &device->flags)) {
+				kref_get(&device->kref);
+				rcu_read_unlock();
+				drbd_uuid_new_current(device, false);
+				kref_put(&device->kref, drbd_destroy_device);
+				rcu_read_lock();
+			}
+		}
+		rcu_read_unlock();
+		begin_state_change(resource, &irq_flags, CS_VERBOSE);
+		__change_io_susp_fencing(connection, false);
+		end_state_change(resource, &irq_flags, "after-fencing");
+	}
+	/* case2: The connection was established again: */
+	if (all_peer_disks_connected) {
+		rcu_read_lock();
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+			struct drbd_device *device = peer_device->device;
+			clear_bit(NEW_CUR_UUID, &device->flags);
+		}
+		rcu_read_unlock();
+		begin_state_change(resource, &irq_flags, CS_VERBOSE);
+		__change_io_susp_fencing(connection, false);
+		end_state_change(resource, &irq_flags, "after-fencing");
+	}
+}
+
+static bool drbd_should_unfence(struct drbd_state_change *state_change, int n_connection)
+{
+	bool some_peer_was_not_up_to_date = false;
+	int n_device;
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change =
+			&state_change->devices[n_device];
+		enum drbd_disk_state *disk_state = device_state_change->disk_state;
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[
+				n_device * state_change->n_connections + n_connection];
+		enum drbd_disk_state *peer_disk_state = peer_device_state_change->disk_state;
+
+		/* Do not unfence if some volume is not yet up-to-date. */
+		if (disk_state[NEW] != D_UP_TO_DATE || peer_disk_state[NEW] != D_UP_TO_DATE)
+			return false;
+
+		/* Only unfence when the final volume becomes up-to-date. */
+		if (peer_disk_state[OLD] != D_UP_TO_DATE)
+			some_peer_was_not_up_to_date = true;
+	}
+
+	return some_peer_was_not_up_to_date;
+}
+
+static bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device)
+{
+	bool csums_after_crash_only;
+	rcu_read_lock();
+	csums_after_crash_only = rcu_dereference(connection->transport.net_conf)->csums_after_crash_only;
+	rcu_read_unlock();
+	return connection->agreed_pro_version >= 89 &&		/* supported? */
+		connection->csums_tfm &&			/* configured? */
+		(csums_after_crash_only == false		/* use for each resync? */
+		 || test_bit(CRASHED_PRIMARY, &device->flags));	/* or only after Primary crash? */
+}
+
+static void drbd_run_resync(struct drbd_peer_device *peer_device, enum drbd_repl_state repl_state)
+{
+	struct drbd_device *device = peer_device->device;
+	struct drbd_bitmap *bm = device->bitmap;
+	struct drbd_connection *connection = peer_device->connection;
+	enum drbd_repl_state side = repl_is_sync_target(repl_state) ? L_SYNC_TARGET : L_SYNC_SOURCE;
+
+	drbd_info(peer_device, "Began resync as %s (will sync %llu KB [%lu bits set]).\n",
+			drbd_repl_str(repl_state),
+			bm_bit_to_kb(bm, peer_device->rs_total),
+			(unsigned long) peer_device->rs_total);
+
+	if (side == L_SYNC_TARGET)
+		drbd_uuid_set_exposed(device, peer_device->current_uuid, false);
+
+	peer_device->use_csums = side == L_SYNC_TARGET ?
+		use_checksum_based_resync(connection, device) : false;
+
+	if (side == L_SYNC_TARGET &&
+			!(peer_device->uuid_flags & UUID_FLAG_STABLE) &&
+			!drbd_stable_sync_source_present(peer_device, NOW))
+		set_bit(UNSTABLE_RESYNC, &peer_device->flags);
+
+	/* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid
+	 * with w_send_oos, or the sync target will get confused as to
+	 * how much bits to resync.  We cannot do that always, because for an
+	 * empty resync and protocol < 95, we need to do it here, as we call
+	 * drbd_resync_finished from here in that case.
+	 * We drbd_gen_and_send_sync_uuid here for protocol < 96,
+	 * and from after_state_ch otherwise. */
+	if (side == L_SYNC_SOURCE && connection->agreed_pro_version < 96)
+		drbd_gen_and_send_sync_uuid(peer_device);
+
+	if (connection->agreed_pro_version < 95 && peer_device->rs_total == 0) {
+		/* This still has a race (about when exactly the peers
+		 * detect connection loss) that can lead to a full sync
+		 * on next handshake. In 8.3.9 we fixed this with explicit
+		 * resync-finished notifications, but the fix
+		 * introduces a protocol change.  Sleeping for some
+		 * time longer than the ping interval + timeout on the
+		 * SyncSource, to give the SyncTarget the chance to
+		 * detect connection loss, then waiting for a ping
+		 * response (implicit in drbd_resync_finished) reduces
+		 * the race considerably, but does not solve it. */
+		if (side == L_SYNC_SOURCE) {
+			struct net_conf *nc;
+			int timeo;
+
+			rcu_read_lock();
+			nc = rcu_dereference(connection->transport.net_conf);
+			timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9;
+			rcu_read_unlock();
+			schedule_timeout_interruptible(timeo);
+		}
+		drbd_resync_finished(peer_device, D_MASK);
+	}
+
+	/* ns.conn may already be != peer_device->repl_state[NOW],
+	 * we may have been paused in between, or become paused until
+	 * the timer triggers.
+	 * No matter, that is handled in resync_timer_fn() */
+	if (repl_state == L_SYNC_TARGET || repl_state == L_PAUSED_SYNC_T)
+		drbd_uuid_resync_starting(peer_device);
+
+	drbd_md_sync_if_dirty(device);
+}
+
+
+/*
+ * Perform after state change actions that may sleep.
+ */
+static int w_after_state_change(struct drbd_work *w, int unused)
+{
+	struct after_state_change_work *work =
+		container_of(w, struct after_state_change_work, w);
+	struct drbd_state_change *state_change = work->state_change;
+	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
+	struct drbd_resource *resource = resource_state_change->resource;
+	enum drbd_role *role = resource_state_change->role;
+	bool *susp_uuid = resource_state_change->susp_uuid;
+	struct drbd_peer_device *send_state_others = NULL;
+	int n_device, n_connection;
+	bool still_connected = false;
+	bool try_become_up_to_date = false;
+	bool healed_primary = false;
+	bool send_flush_requests = false;
+
+	notify_state_change(state_change);
+
+	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
+		struct drbd_device_state_change *device_state_change = &state_change->devices[n_device];
+		struct drbd_device *device = device_state_change->device;
+		enum drbd_disk_state *disk_state = device_state_change->disk_state;
+		bool have_ldev = extra_ldev_ref_for_after_state_chg(disk_state);
+		bool *have_quorum = device_state_change->have_quorum;
+		bool effective_disk_size_determined = false;
+		bool device_stable[2], resync_target[2];
+		bool data_accessible[2];
+		bool all_peer_replication[2];
+		bool resync_finished = false;
+		bool some_peer_demoted = false;
+		bool new_current_uuid = false;
+		enum which_state which;
+
+		for (which = OLD; which <= NEW; which++) {
+			device_stable[which] = calc_device_stable(state_change, n_device, which);
+			resync_target[which] = calc_resync_target(state_change, n_device, which);
+			data_accessible[which] =
+				calc_data_accessible(state_change, n_device, which);
+			all_peer_replication[which] =
+				drbd_all_peer_replication_change(state_change, n_device, which);
+
+		}
+
+		if (disk_state[NEW] == D_UP_TO_DATE)
+			effective_disk_size_determined = true;
+
+		for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+			struct drbd_peer_device_state_change *peer_device_state_change =
+				&state_change->peer_devices[
+					n_device * state_change->n_connections + n_connection];
+			struct drbd_peer_device *peer_device = peer_device_state_change->peer_device;
+			enum drbd_disk_state *peer_disk_state = peer_device_state_change->disk_state;
+			enum drbd_repl_state *repl_state = peer_device_state_change->repl_state;
+
+			if ((repl_state[OLD] == L_SYNC_TARGET || repl_state[OLD] == L_PAUSED_SYNC_T) &&
+			    repl_state[NEW] == L_ESTABLISHED)
+				resync_finished = true;
+
+			if (disk_state[OLD] == D_INCONSISTENT && disk_state[NEW] == D_UP_TO_DATE &&
+			    peer_disk_state[OLD] == D_INCONSISTENT && peer_disk_state[NEW] == D_UP_TO_DATE)
+				send_state_others = peer_device;
+
+			/* connect without resync or remote attach without resync */
+			if (disk_state[NOW] >= D_OUTDATED && repl_state[NEW] == L_ESTABLISHED &&
+			    ((repl_state[OLD] == L_OFF && peer_disk_state[NEW] >= D_OUTDATED) ||
+			     (peer_disk_state[OLD] == D_DISKLESS && peer_disk_state[NEW] >= D_OUTDATED))) {
+				u64 peer_current_uuid = peer_device->current_uuid & ~UUID_PRIMARY;
+				u64 my_current_uuid = drbd_current_uuid(device) & ~UUID_PRIMARY;
+
+				if (peer_current_uuid == my_current_uuid && get_ldev(device)) {
+					down_write(&device->uuid_sem);
+					drbd_uuid_set_bitmap(peer_device, 0);
+					up_write(&device->uuid_sem);
+					drbd_print_uuids(peer_device, "cleared bm UUID and bitmap");
+					drbd_bitmap_io_from_worker(device, &drbd_bmio_clear_one_peer,
+								   "clearing bm one peer", BM_LOCK_CLEAR | BM_LOCK_BULK,
+								   peer_device);
+					put_ldev(device);
+				}
+			}
+		}
+
+		if (role[NEW] == R_PRIMARY && !data_accessible[OLD] && data_accessible[NEW])
+			healed_primary = true;
+
+		for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+			struct drbd_connection_state_change *connection_state_change = &state_change->connections[n_connection];
+			struct drbd_connection *connection = connection_state_change->connection;
+			enum drbd_conn_state *cstate = connection_state_change->cstate;
+			enum drbd_role *peer_role = connection_state_change->peer_role;
+			struct drbd_peer_device_state_change *peer_device_state_change =
+				&state_change->peer_devices[
+					n_device * state_change->n_connections + n_connection];
+			struct drbd_peer_device *peer_device = peer_device_state_change->peer_device;
+			enum drbd_repl_state *repl_state = peer_device_state_change->repl_state;
+			enum drbd_disk_state *peer_disk_state = peer_device_state_change->disk_state;
+			bool *resync_susp_user = peer_device_state_change->resync_susp_user;
+			bool *resync_susp_peer = peer_device_state_change->resync_susp_peer;
+			bool *resync_susp_dependency = peer_device_state_change->resync_susp_dependency;
+			union drbd_state new_state =
+				state_change_word(state_change, n_device, n_connection, NEW);
+			bool send_uuids, send_state = false;
+
+			/* In case we finished a resync as resync-target update all neighbors
+			 * about having a bitmap_uuid of 0 towards the previous sync-source.
+			 * That needs to go out before sending the new disk state
+			 * to avoid a race where the other node might downgrade our disk
+			 * state due to old UUID values.
+			 *
+			 * Also check the replication state to ensure that we
+			 * do not send these extra UUIDs before the initial
+			 * handshake. */
+			send_uuids = resync_finished &&
+				peer_disk_state[NEW] != D_UNKNOWN &&
+				repl_state[NEW] > L_OFF;
+
+			/* Send UUIDs again if they changed while establishing the connection */
+			if (repl_state[OLD] == L_OFF && repl_state[NEW] > L_OFF &&
+			    peer_device->comm_current_uuid != drbd_resolved_uuid(peer_device, NULL))
+				send_uuids = true;
+
+			if (repl_state[NEW] > L_OFF && device_stable[OLD] != device_stable[NEW])
+				send_uuids = true;
+
+			if (send_uuids)
+				drbd_send_uuids(peer_device, 0, 0);
+
+			if (peer_disk_state[NEW] == D_UP_TO_DATE)
+				effective_disk_size_determined = true;
+
+			if (!(role[OLD] == R_PRIMARY && !data_accessible[OLD]) &&
+			     (role[NEW] == R_PRIMARY && !data_accessible[NEW]) &&
+			    !test_bit(UNREGISTERED, &device->flags))
+				drbd_maybe_khelper(device, connection, "pri-on-incon-degr");
+
+			/* Became sync source.  With protocol >= 96, we still need to send out
+			 * the sync uuid now. Need to do that before any drbd_send_state, or
+			 * the other side may go "paused sync" before receiving the sync uuids,
+			 * which is unexpected. */
+			if (!(repl_state[OLD] == L_SYNC_SOURCE || repl_state[OLD] == L_PAUSED_SYNC_S) &&
+			     (repl_state[NEW] == L_SYNC_SOURCE || repl_state[NEW] == L_PAUSED_SYNC_S) &&
+			    connection->agreed_pro_version >= 96 && connection->agreed_pro_version < 110 &&
+			    get_ldev(device)) {
+				drbd_gen_and_send_sync_uuid(peer_device);
+				put_ldev(device);
+			}
+
+			/* Do not change the order of the if above and the two below... */
+			if (peer_disk_state[OLD] < D_NEGOTIATING &&
+			    peer_disk_state[NEW] == D_NEGOTIATING) { /* attach on the peer */
+				/* we probably will start a resync soon.
+				 * make sure those things are properly reset. */
+				peer_device->rs_total = 0;
+				peer_device->rs_failed = 0;
+
+				drbd_send_uuids(peer_device, 0, 0);
+				drbd_send_state(peer_device, new_state);
+			}
+			/* No point in queuing send_bitmap if we don't have a connection
+			 * anymore, so check also the _current_ state, not only the new state
+			 * at the time this work was queued. */
+			if (repl_state[OLD] != L_WF_BITMAP_S && repl_state[NEW] == L_WF_BITMAP_S &&
+			    peer_device->repl_state[NOW] == L_WF_BITMAP_S) {
+				/* Now that the connection is L_WF_BITMAP_S,
+				 * new requests will be sent to the peer as
+				 * P_OUT_OF_SYNC packets. However, active
+				 * requests may not have been communicated to
+				 * the peer and may not yet be marked in the
+				 * local bitmap. Mark these requests in the
+				 * bitmap before reading and sending that
+				 * bitmap. This may set bits unnecessarily, but
+				 * it does no harm to resync a small amount of
+				 * additional data. */
+				drbd_set_pending_out_of_sync(peer_device);
+				drbd_queue_bitmap_io(device, &drbd_send_bitmap, NULL,
+						"send_bitmap (WFBitMapS)",
+						BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK | BM_LOCK_SINGLE_SLOT,
+						peer_device);
+			}
+
+			if (peer_role[OLD] == R_PRIMARY && peer_role[NEW] == R_SECONDARY)
+				some_peer_demoted = true;
+
+			/* Last part of the attaching process ... */
+			if (cstate[NEW] == C_CONNECTED && /* repl_state[NEW] might still be L_OFF */
+			    disk_state[OLD] == D_ATTACHING && disk_state[NEW] >= D_NEGOTIATING) {
+				drbd_send_sizes(peer_device, 0, 0);  /* to start sync... */
+				drbd_send_uuids(peer_device, 0, 0);
+				drbd_send_state(peer_device, new_state);
+			}
+
+			/* Started resync, tell peer if drbd9 */
+			if (repl_state[NEW] >= L_SYNC_SOURCE && repl_state[NEW] <= L_PAUSED_SYNC_T &&
+			    (repl_state[OLD] < L_SYNC_SOURCE || repl_state[OLD] > L_PAUSED_SYNC_T))
+				send_state = true;
+
+			/* We want to pause/continue resync, tell peer. */
+			if (repl_state[NEW] >= L_ESTABLISHED &&
+			    ((resync_susp_comb_dep_sc(state_change, n_device, n_connection, OLD) !=
+			      resync_susp_comb_dep_sc(state_change, n_device, n_connection, NEW)) ||
+			     (resync_susp_user[OLD] != resync_susp_user[NEW])))
+				send_state = true;
+
+			/* finished resync, tell sync source */
+			if ((repl_state[OLD] == L_SYNC_TARGET || repl_state[OLD] == L_PAUSED_SYNC_T) &&
+			    repl_state[NEW] == L_ESTABLISHED)
+				send_state = true;
+
+			/* In case one of the isp bits got set, suspend other devices. */
+			if (!(resync_susp_dependency[OLD] || resync_susp_peer[OLD] || resync_susp_user[OLD]) &&
+			     (resync_susp_dependency[NEW] || resync_susp_peer[NEW] || resync_susp_user[NEW]))
+				suspend_other_sg(device);
+
+			/* Make sure the peer gets informed about eventual state
+			   changes (ISP bits) while we were in L_OFF. */
+			if (repl_state[OLD] == L_OFF && repl_state[NEW] >= L_ESTABLISHED)
+				send_state = true;
+
+			if (repl_state[OLD] != L_AHEAD && repl_state[NEW] == L_AHEAD)
+				send_state = true;
+
+			/* We are in the progress to start a full sync. SyncTarget sets all slots. */
+			if (repl_state[OLD] != L_STARTING_SYNC_T && repl_state[NEW] == L_STARTING_SYNC_T)
+				drbd_queue_bitmap_io(device,
+					&drbd_bmio_set_all_n_write, &abw_start_sync,
+					"set_n_write from StartingSync",
+					BM_LOCK_CLEAR | BM_LOCK_BULK,
+					peer_device);
+
+			/* We are in the progress to start a full sync. SyncSource one slot. */
+			if (repl_state[OLD] != L_STARTING_SYNC_S && repl_state[NEW] == L_STARTING_SYNC_S)
+				drbd_queue_bitmap_io(device,
+					&drbd_bmio_set_n_write, &abw_start_sync,
+					"set_n_write from StartingSync",
+					BM_LOCK_CLEAR | BM_LOCK_BULK,
+					peer_device);
+
+			/* Disks got bigger while they were detached */
+			if (disk_state[NEW] > D_NEGOTIATING && peer_disk_state[NEW] > D_NEGOTIATING &&
+			    test_and_clear_bit(RESYNC_AFTER_NEG, &peer_device->flags)) {
+				if (repl_state[NEW] == L_ESTABLISHED)
+					resync_after_online_grow(peer_device);
+			}
+
+			/* A resync finished or aborted, wake paused devices... */
+			if ((repl_state[OLD] > L_ESTABLISHED && repl_state[NEW] <= L_ESTABLISHED) ||
+			    (resync_susp_peer[OLD] && !resync_susp_peer[NEW]) ||
+			    (resync_susp_user[OLD] && !resync_susp_user[NEW]))
+				resume_next_sg(device);
+
+			/* sync target done with resync. Explicitly notify all peers. Our sync
+			   source should even know by himself, but the others need that info. */
+			if (disk_state[OLD] < D_UP_TO_DATE && repl_state[OLD] >= L_SYNC_SOURCE && repl_state[NEW] == L_ESTABLISHED)
+				send_new_state_to_all_peer_devices(state_change, n_device);
+
+			/* Outdated myself, or became D_UP_TO_DATE tell peers
+			 * Do not do it, when the local node was forced from R_SECONDARY to R_PRIMARY,
+			 * because that is part of the 2-phase-commit and that is necessary to trigger
+			 * the initial resync. */
+			if ((disk_state[NEW] >= D_INCONSISTENT && disk_state[NEW] != disk_state[OLD] &&
+			     repl_state[OLD] >= L_ESTABLISHED && repl_state[NEW] >= L_ESTABLISHED) &&
+			    !(role[OLD] == R_SECONDARY && role[NEW] == R_PRIMARY))
+				send_state = true;
+
+			/* diskless peers need to be informed about quorum changes, since they consider
+			   the quorum state of the diskfull nodes. */
+			if (have_quorum[OLD] != have_quorum[NEW] && disk_state[NEW] >= D_INCONSISTENT)
+				send_state = true;
+
+			/* Skipped resync with peer_device, tell others... */
+			if (send_state_others && send_state_others != peer_device)
+				send_state = true;
+
+			/* This triggers bitmap writeout of potentially still unwritten pages
+			 * if the resync finished cleanly, or aborted because of peer disk
+			 * failure, or on transition from resync back to AHEAD/BEHIND.
+			 *
+			 * Connection loss is handled in conn_disconnect() by the receiver.
+			 *
+			 * For resync aborted because of local disk failure, we cannot do
+			 * any bitmap writeout anymore.
+			 *
+			 * No harm done if some bits change during this phase.
+			 */
+			if ((repl_state[OLD] > L_ESTABLISHED && repl_state[OLD] < L_AHEAD) &&
+			    (repl_state[NEW] == L_ESTABLISHED || repl_state[NEW] >= L_AHEAD) &&
+			    get_ldev(device)) {
+				drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL,
+					"write from resync_finished", BM_LOCK_BULK,
+					NULL);
+				put_ldev(device);
+			}
+
+			/* Verify finished, or reached stop sector.  Peer did not know about
+			 * the stop sector, and we may even have changed the stop sector during
+			 * verify to interrupt/stop early.  Send the new state. */
+			if (repl_state[OLD] == L_VERIFY_S && repl_state[NEW] == L_ESTABLISHED
+			    && verify_can_do_stop_sector(peer_device))
+				send_new_state_to_all_peer_devices(state_change, n_device);
+
+			if (disk_state[NEW] == D_DISKLESS &&
+			    cstate[NEW] == C_STANDALONE &&
+			    role[NEW] == R_SECONDARY) {
+				if (resync_susp_dependency[OLD] != resync_susp_dependency[NEW])
+					resume_next_sg(device);
+			}
+
+			if (device_stable[OLD] && !device_stable[NEW] &&
+			    repl_state[NEW] >= L_ESTABLISHED && get_ldev(device)) {
+				/* Inform peers about being unstable...
+				   Maybe it would be a better idea to have the stable bit as
+				   part of the state (and being sent with the state) */
+				drbd_send_uuids(peer_device, 0, 0);
+				put_ldev(device);
+			}
+
+			if (send_state && cstate[NEW] == C_CONNECTED)
+				drbd_send_state(peer_device, new_state);
+
+			if (((!device_stable[OLD] && device_stable[NEW]) ||
+			     (resync_target[OLD] && !resync_target[NEW] && device_stable[NEW])) &&
+			    !(repl_state[OLD] == L_SYNC_TARGET || repl_state[OLD] == L_PAUSED_SYNC_T) &&
+			    !(peer_role[OLD] == R_PRIMARY) && disk_state[NEW] >= D_OUTDATED &&
+			    repl_state[NEW] >= L_ESTABLISHED &&
+			    get_ldev(device)) {
+				/* Offer all peers a resync, with the exception of ...
+				   ... the node that made me up-to-date (with a resync)
+				   ... I was primary
+				   ... the peer that transitioned from primary to secondary
+				*/
+				drbd_send_uuids(peer_device, UUID_FLAG_GOT_STABLE, 0);
+				put_ldev(device);
+			}
+
+			if (peer_disk_state[OLD] == D_UP_TO_DATE &&
+			    (peer_disk_state[NEW] == D_FAILED || peer_disk_state[NEW] == D_INCONSISTENT) &&
+			    test_and_clear_bit(NEW_CUR_UUID, &device->flags))
+				/* When a peer disk goes from D_UP_TO_DATE to D_FAILED or D_INCONSISTENT
+				   we know that a write failed on that node. Therefore we need to create
+				   the new UUID right now (not wait for the next write to come in) */
+				new_current_uuid = true;
+
+			if (disk_state[OLD] > D_FAILED && disk_state[NEW] == D_FAILED &&
+			    role[NEW] == R_PRIMARY && test_and_clear_bit(NEW_CUR_UUID, &device->flags))
+				new_current_uuid = true;
+
+			if (repl_state[OLD] != L_VERIFY_S && repl_state[NEW] == L_VERIFY_S) {
+				drbd_info(peer_device, "Starting Online Verify from sector %llu\n",
+						(unsigned long long)peer_device->ov_position);
+				drbd_queue_work_if_unqueued(
+						&peer_device->connection->sender_work,
+						&peer_device->resync_work);
+			}
+
+			if (!repl_is_sync(repl_state[OLD]) && repl_is_sync(repl_state[NEW]))
+				drbd_run_resync(peer_device, repl_state[NEW]);
+
+			if (repl_is_sync(repl_state[OLD]) && !repl_is_sync(repl_state[NEW]))
+				drbd_last_resync_request(peer_device, false);
+
+			if (peer_device_state_change->repl_state[OLD] != L_SYNC_TARGET &&
+					peer_device_state_change->repl_state[NEW] == L_SYNC_TARGET)
+				drbd_queue_work_if_unqueued(
+						&peer_device->connection->sender_work,
+						&peer_device->resync_work);
+
+			if (!(repl_is_sync_target(repl_state[OLD]) &&
+					all_peer_replication[OLD]) &&
+					repl_is_sync_target(repl_state[NEW]) &&
+					all_peer_replication[NEW])
+				send_flush_requests = true;
+
+			if (!peer_device_state_change->peer_replication[OLD] &&
+					peer_device_state_change->peer_replication[NEW])
+				drbd_send_enable_replication(peer_device, true);
+		}
+
+		if (((role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY) || some_peer_demoted) &&
+		    get_ldev(device)) {
+			/* The some_peer_demoted case is superseded by
+			 * handle_neighbor_demotion(). We keep this call for
+			 * compatibility until support for protocol version 121
+			 * is removed.
+			 *
+			 * No changes to the bitmap expected after this point, so write out any
+			 * changes up to now to ensure that the metadata disk has the full
+			 * bitmap content. Even if the bitmap changes (e.g. it was dual primary)
+			 * no harm was done if it did change. */
+			drbd_bitmap_io_from_worker(device, &drbd_bm_write,
+						   "demote", BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK,
+						   NULL);
+			put_ldev(device);
+		}
+
+		/* Make sure the effective disk size is stored in the metadata
+		 * if a local disk is attached and either the local disk state
+		 * or a peer disk state is D_UP_TO_DATE.  */
+		if (effective_disk_size_determined && get_ldev(device)) {
+			sector_t size = get_capacity(device->vdisk);
+			if (device->ldev->md.effective_size != size) {
+				char ppb[10];
+
+				drbd_info(device, "persisting effective size = %s (%llu KB)\n",
+				     ppsize(ppb, size >> 1),
+				     (unsigned long long)size >> 1);
+				device->ldev->md.effective_size = size;
+				drbd_md_mark_dirty(device);
+			}
+			put_ldev(device);
+		}
+
+		/* first half of local IO error, failure to attach,
+		 * or administrative detach */
+		if ((disk_state[OLD] != D_FAILED && disk_state[NEW] == D_FAILED) ||
+		    (disk_state[OLD] != D_DETACHING && disk_state[NEW] == D_DETACHING)) {
+			enum drbd_io_error_p eh = EP_PASS_ON;
+			int was_io_error = 0;
+
+			/* Our cleanup here with the transition to D_DISKLESS.
+			 * It is still not safe to dereference ldev here, since
+			 * we might come from an failed Attach before ldev was set. */
+			if (have_ldev && device->ldev) {
+				rcu_read_lock();
+				eh = rcu_dereference(device->ldev->disk_conf)->on_io_error;
+				rcu_read_unlock();
+
+				was_io_error = disk_state[NEW] == D_FAILED;
+
+				/* Intentionally call this handler first, before drbd_send_state().
+				 * See: 2932204 drbd: call local-io-error handler early
+				 * People may chose to hard-reset the box from this handler.
+				 * It is useful if this looks like a "regular node crash". */
+				if (was_io_error && eh == EP_CALL_HELPER)
+					drbd_maybe_khelper(device, NULL, "local-io-error");
+
+				/* Immediately allow completion of all application IO,
+				 * that waits for completion from the local disk,
+				 * if this was a force-detach due to disk_timeout
+				 * or administrator request (drbdsetup detach --force).
+				 * Do NOT abort otherwise.
+				 * Aborting local requests may cause serious problems,
+				 * if requests are completed to upper layers already,
+				 * and then later the already submitted local bio completes.
+				 * This can cause DMA into former bio pages that meanwhile
+				 * have been re-used for other things.
+				 * So aborting local requests may cause crashes,
+				 * or even worse, silent data corruption.
+				 */
+				if (test_and_clear_bit(FORCE_DETACH, &device->flags))
+					tl_abort_disk_io(device);
+
+				send_new_state_to_all_peer_devices(state_change, n_device);
+
+				/* In case we want to get something to stable storage still,
+				 * this may be the last chance.
+				 * Following put_ldev may transition to D_DISKLESS. */
+				drbd_bitmap_io_from_worker(device, &drbd_bm_write,
+						"detach", BM_LOCK_SET | BM_LOCK_CLEAR | BM_LOCK_BULK,
+						NULL);
+				drbd_md_sync_if_dirty(device);
+			}
+		}
+
+		/* second half of local IO error, failure to attach,
+		 * or administrative detach,
+		 * after local_cnt references have reached zero again */
+		if (disk_state[OLD] != D_DISKLESS && disk_state[NEW] == D_DISKLESS) {
+			/* We must still be diskless,
+			 * re-attach has to be serialized with this! */
+			if (device->disk_state[NOW] != D_DISKLESS)
+				drbd_err(device,
+					"ASSERT FAILED: disk is %s while going diskless\n",
+					drbd_disk_str(device->disk_state[NOW]));
+
+			/* we may need to cancel the md_sync timer */
+			timer_delete_sync(&device->md_sync_timer);
+
+			if (have_ldev)
+				send_new_state_to_all_peer_devices(state_change, n_device);
+		}
+
+		if (have_ldev)
+			put_ldev(device);
+
+		/* Notify peers that I had a local IO error and did not detach. */
+		if (disk_state[OLD] == D_UP_TO_DATE && disk_state[NEW] == D_INCONSISTENT)
+			send_new_state_to_all_peer_devices(state_change, n_device);
+
+		/* Testing EMPTY_TWOPC_PENDING would cause more queuing than necessary */
+		if (should_try_become_up_to_date(device, disk_state, NOW))
+			try_become_up_to_date = true;
+
+		if (test_bit(TRY_TO_GET_RESYNC, &device->flags)) {
+			/* Got connected to a diskless primary */
+			clear_bit(TRY_TO_GET_RESYNC, &device->flags);
+			drbd_try_to_get_resynced(device);
+		}
+
+		drbd_md_sync_if_dirty(device);
+
+		if (role[NEW] == R_PRIMARY && have_quorum[OLD] && !have_quorum[NEW])
+			drbd_maybe_khelper(device, NULL, "quorum-lost");
+
+		if (!susp_uuid[OLD] && susp_uuid[NEW] &&
+		    test_and_clear_bit(NEW_CUR_UUID, &device->flags))
+			new_current_uuid = true;
+
+		if (new_current_uuid)
+			drbd_uuid_new_current(device, false);
+
+		if (disk_state[OLD] > D_DISKLESS && disk_state[NEW] == D_DISKLESS)
+			drbd_reconsider_queue_parameters(device, NULL);
+	}
+
+	if (role[OLD] == R_PRIMARY && role[NEW] == R_SECONDARY)
+		send_role_to_all_peers(state_change);
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change = &state_change->connections[n_connection];
+		struct drbd_connection *connection = connection_state_change->connection;
+		enum drbd_conn_state *cstate = connection_state_change->cstate;
+		bool *susp_fen = connection_state_change->susp_fen;
+		enum drbd_fencing_policy fencing_policy;
+
+		if (connection_state_change->peer_role[NEW] == R_PRIMARY && send_flush_requests &&
+				connection->agreed_pro_version >= 123) {
+			u64 current_flush_sequence;
+
+			spin_lock_irq(&resource->initiator_flush_lock);
+			/* Requirement: At least the value from the corresponding state change */
+			current_flush_sequence = resource->current_flush_sequence;
+			spin_unlock_irq(&resource->initiator_flush_lock);
+
+			drbd_send_flush_requests(connection, current_flush_sequence);
+		}
+
+		/* Upon network configuration, we need to start the receiver */
+		if (cstate[OLD] == C_STANDALONE && cstate[NEW] == C_UNCONNECTED)
+			drbd_thread_start(&connection->receiver);
+
+		if (susp_fen[NEW])
+			check_may_resume_io_after_fencing(state_change, n_connection);
+
+		rcu_read_lock();
+		fencing_policy = connection->fencing_policy;
+		rcu_read_unlock();
+		if (fencing_policy != FP_DONT_CARE &&
+				drbd_should_unfence(state_change, n_connection))
+			drbd_maybe_khelper(NULL, connection, "unfence-peer");
+	}
+
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_connection_state_change *connection_state_change = &state_change->connections[n_connection];
+		enum drbd_conn_state *cstate = connection_state_change->cstate;
+
+		if (cstate[NEW] == C_CONNECTED || cstate[NEW] == C_CONNECTING)
+			still_connected = true;
+	}
+
+	if (susp_uuid[NEW]) {
+		unsigned long irq_flags;
+
+		begin_state_change(resource, &irq_flags, CS_VERBOSE);
+		resource->susp_uuid[NEW] = false;
+		end_state_change(resource, &irq_flags, "susp-uuid");
+	}
+
+	if (try_become_up_to_date || healed_primary)
+		drbd_schedule_empty_twopc(resource);
+
+	if (!still_connected)
+		mod_timer_pending(&resource->twopc_timer, jiffies);
+
+	if (work->done)
+		complete(work->done);
+	forget_state_change(state_change);
+	kfree(work);
+
+	return 0;
+}
+
+static bool local_state_change(enum chg_state_flags flags)
+{
+	return flags & (CS_HARD | CS_LOCAL_ONLY);
+}
+
+static enum drbd_state_rv
+__peer_request(struct drbd_connection *connection, int vnr,
+	       union drbd_state mask, union drbd_state val)
+{
+	enum drbd_state_rv rv = SS_SUCCESS;
+
+	if (connection->cstate[NOW] == C_CONNECTED) {
+		enum drbd_packet cmd = (vnr == -1) ? P_CONN_ST_CHG_REQ : P_STATE_CHG_REQ;
+		if (!conn_send_state_req(connection, vnr, cmd, mask, val)) {
+			set_bit(TWOPC_PREPARED, &connection->flags);
+			rv = SS_CW_SUCCESS;
+		}
+	}
+	return rv;
+}
+
+static enum drbd_state_rv __peer_reply(struct drbd_connection *connection)
+{
+	if (test_and_clear_bit(TWOPC_NO, &connection->flags))
+		return SS_CW_FAILED_BY_PEER;
+	if (test_and_clear_bit(TWOPC_YES, &connection->flags) ||
+	    !test_bit(TWOPC_PREPARED, &connection->flags))
+		return SS_CW_SUCCESS;
+
+	/* This is DRBD 9.x <-> 8.4 compat code.
+	 * Consistent with __peer_request() above:
+	 * No more connection: fake success. */
+	if (connection->cstate[NOW] != C_CONNECTED)
+		return SS_SUCCESS;
+	return SS_UNKNOWN_ERROR;
+}
+
+static bool when_done_lock(struct drbd_resource *resource,
+			   unsigned long *irq_flags)
+{
+	write_lock_irqsave(&resource->state_rwlock, *irq_flags);
+	if (!resource->remote_state_change && !test_bit(TWOPC_WORK_PENDING, &resource->flags))
+		return true;
+	write_unlock_irqrestore(&resource->state_rwlock, *irq_flags);
+	return false;
+}
+
+/**
+ * complete_remote_state_change  -  Wait for other remote state changes to complete
+ * @resource: DRBD resource.
+ * @irq_flags: IRQ flags from begin_state_change.
+ */
+static void complete_remote_state_change(struct drbd_resource *resource,
+					 unsigned long *irq_flags)
+{
+	if (resource->remote_state_change) {
+		enum chg_state_flags flags = resource->state_change_flags;
+
+		begin_remote_state_change(resource, irq_flags);
+		for (;;) {
+			long t = twopc_timeout(resource);
+
+			t = wait_event_timeout(resource->twopc_wait,
+				   when_done_lock(resource, irq_flags), t);
+			if (t)
+				break;
+			if (when_done_lock(resource, irq_flags)) {
+				drbd_info(resource, "Two-phase commit: "
+					  "not woken up in time\n");
+				break;
+			}
+		}
+		__end_remote_state_change(resource, flags);
+	}
+}
+
+static enum drbd_state_rv
+change_peer_state(struct drbd_connection *connection, int vnr,
+		  union drbd_state mask, union drbd_state val, unsigned long *irq_flags)
+{
+	struct drbd_resource *resource = connection->resource;
+	enum chg_state_flags flags = resource->state_change_flags | CS_TWOPC;
+	enum drbd_state_rv rv;
+
+	if (!expect(resource, flags & CS_SERIALIZE))
+		return SS_CW_FAILED_BY_PEER;
+
+	complete_remote_state_change(resource, irq_flags);
+
+	resource->remote_state_change = true;
+	resource->twopc_reply.initiator_node_id = resource->res_opts.node_id;
+	resource->twopc_reply.tid = 0;
+	begin_remote_state_change(resource, irq_flags);
+	rv = __peer_request(connection, vnr, mask, val);
+	if (rv == SS_CW_SUCCESS) {
+		wait_event(resource->state_wait,
+			((rv = __peer_reply(connection)) != SS_UNKNOWN_ERROR));
+		clear_bit(TWOPC_PREPARED, &connection->flags);
+	}
+	end_remote_state_change(resource, irq_flags, flags);
+	return rv;
+}
+
+static enum drbd_state_rv
+__cluster_wide_request(struct drbd_resource *resource, struct twopc_request *request,
+		       u64 reach_immediately)
+{
+	enum drbd_packet cmd = request->cmd;
+	struct drbd_connection *connection;
+	enum drbd_state_rv rv = SS_SUCCESS;
+	u64 im;
+
+	for_each_connection_ref(connection, im, resource) {
+		u64 mask;
+		int err;
+
+		clear_bit(TWOPC_PREPARED, &connection->flags);
+
+		if (connection->agreed_pro_version < 110)
+			continue;
+		mask = NODE_MASK(connection->peer_node_id);
+		if (reach_immediately & mask)
+			set_bit(TWOPC_PREPARED, &connection->flags);
+		else
+			continue;
+
+		clear_bit(TWOPC_YES, &connection->flags);
+		clear_bit(TWOPC_NO, &connection->flags);
+		clear_bit(TWOPC_RETRY, &connection->flags);
+
+		err = conn_send_twopc_request(connection, request);
+		if (err) {
+			clear_bit(TWOPC_PREPARED, &connection->flags);
+			wake_up(&resource->work.q_wait);
+			continue;
+		}
+		if (cmd == P_TWOPC_PREPARE || cmd == P_TWOPC_PREP_RSZ)
+			schedule_work(&connection->send_ping_work);
+		rv = SS_CW_SUCCESS;
+	}
+	return rv;
+}
+
+bool drbd_twopc_between_peer_and_me(struct drbd_connection *connection)
+{
+	const int my_node_id = connection->resource->res_opts.node_id;
+	struct twopc_reply *o = &connection->resource->twopc_reply;
+
+	return ((o->target_node_id == my_node_id || o->target_node_id == -1) &&
+		o->initiator_node_id == connection->peer_node_id) ||
+		((o->target_node_id == connection->peer_node_id || o->target_node_id == -1) &&
+		 o->initiator_node_id == my_node_id);
+}
+
+bool cluster_wide_reply_ready(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	bool connect_ready = true;
+	bool have_no = resource->twopc_reply.state_change_failed;
+	bool have_retry = false;
+	bool all_yes = true;
+
+	if (test_bit(TWOPC_ABORT_LOCAL, &resource->flags))
+		return true;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->agreed_pro_version >= 118 &&
+				!idr_is_empty(&resource->devices) &&
+				resource->twopc_reply.is_connect &&
+				drbd_twopc_between_peer_and_me(connection) &&
+				!test_bit(CONN_HANDSHAKE_READY, &connection->flags))
+			connect_ready = false;
+
+		if (!test_bit(TWOPC_PREPARED, &connection->flags))
+			continue;
+		if (test_bit(TWOPC_NO, &connection->flags))
+			have_no = true;
+		if (test_bit(TWOPC_RETRY, &connection->flags))
+			have_retry = true;
+		if (!test_bit(TWOPC_YES, &connection->flags))
+			all_yes = false;
+	}
+	rcu_read_unlock();
+
+	return have_retry || (connect_ready && (have_no || all_yes));
+}
+
+static enum drbd_state_rv get_cluster_wide_reply(struct drbd_resource *resource,
+						 struct change_context *context)
+{
+	struct drbd_connection *connection, *failed_by = NULL;
+	bool handshake_disconnect = false;
+	bool handshake_retry = false;
+	bool have_no = resource->twopc_reply.state_change_failed;
+	bool have_retry = false;
+	enum drbd_state_rv rv = SS_CW_SUCCESS;
+
+	if (test_bit(TWOPC_ABORT_LOCAL, &resource->flags))
+		return SS_CONCURRENT_ST_CHG;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (resource->twopc_reply.is_connect &&
+				drbd_twopc_between_peer_and_me(connection)) {
+			if (test_bit(CONN_HANDSHAKE_DISCONNECT, &connection->flags))
+				handshake_disconnect = true;
+			if (test_bit(CONN_HANDSHAKE_RETRY, &connection->flags))
+				handshake_retry = true;
+		}
+
+		if (!test_bit(TWOPC_PREPARED, &connection->flags))
+			continue;
+		if (test_bit(TWOPC_NO, &connection->flags)) {
+			failed_by = connection;
+			have_no = true;
+		}
+		if (test_bit(TWOPC_RETRY, &connection->flags))
+			have_retry = true;
+	}
+
+	if (have_retry)
+		rv = SS_CONCURRENT_ST_CHG;
+	else if (handshake_retry)
+		rv = SS_HANDSHAKE_RETRY;
+	else if (handshake_disconnect)
+		rv = SS_HANDSHAKE_DISCONNECT;
+	else if (have_no) {
+		if (context && failed_by)
+			_drbd_state_err(context, "Declined by peer %s (id: %d), see the kernel log there",
+					rcu_dereference(failed_by->transport.net_conf)->name,
+					failed_by->peer_node_id);
+		rv = SS_CW_FAILED_BY_PEER;
+	}
+	rcu_read_unlock();
+
+	if (rv == SS_CW_SUCCESS && test_bit(TWOPC_RECV_SIZES_ERR, &resource->flags))
+		rv = SS_HANDSHAKE_DISCONNECT;
+
+	return rv;
+}
+
+static bool supports_two_phase_commit(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	bool supported = true;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[NOW] != C_CONNECTED)
+			continue;
+		if (connection->agreed_pro_version < 110) {
+			supported = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return supported;
+}
+
+static struct drbd_connection *get_first_connection(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection = NULL;
+
+	rcu_read_lock();
+	if (!list_empty(&resource->connections)) {
+		connection = first_connection(resource);
+		kref_get(&connection->kref);
+	}
+	rcu_read_unlock();
+	return connection;
+}
+
+/* That two_primaries is a connection option is one of those things of
+   the past, that should be cleaned up!! it should be a resource config!
+   Here is a inaccurate heuristic */
+static bool multiple_primaries_allowed(struct drbd_resource *resource)
+{
+	struct drbd_connection *connection;
+	bool allowed = false;
+	struct net_conf *nc;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		nc = rcu_dereference(connection->transport.net_conf);
+		if (nc && nc->two_primaries) {
+			allowed = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return allowed;
+}
+
+static enum drbd_state_rv
+check_primaries_distances(struct drbd_resource *resource)
+{
+	struct twopc_reply *reply = &resource->twopc_reply;
+	int nr_primaries = hweight64(reply->primary_nodes);
+	u64 common_server;
+
+	if (nr_primaries <= 1)
+		return SS_SUCCESS;
+	if (nr_primaries > 1 && !multiple_primaries_allowed(resource))
+		return SS_TWO_PRIMARIES;
+	/* All primaries directly connected. Good */
+	if (!(reply->primary_nodes & reply->weak_nodes))
+		return SS_SUCCESS;
+
+	/* For virtualization setups with diskless hypervisors (R_PRIMARY) and one
+	   or multiple storage servers (R_SECONDARY) allow live-migration between the
+	   hypervisors. */
+	common_server = ~reply->weak_nodes;
+	if (common_server) {
+		int node_id;
+		/* Only allow if the new primary is diskless. See also far_away_change()
+		   in drbd_receiver.c for the diskless check on the other primary */
+		if ((reply->primary_nodes & NODE_MASK(resource->res_opts.node_id)) &&
+		    drbd_have_local_disk(resource))
+			return SS_WEAKLY_CONNECTED;
+
+		for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+			struct drbd_connection *connection;
+			struct net_conf *nc;
+			bool two_primaries;
+
+			if (!(common_server & NODE_MASK(node_id)))
+				continue;
+			connection = drbd_connection_by_node_id(resource, node_id);
+			if (!connection)
+				continue;
+
+			rcu_read_lock();
+			nc = rcu_dereference(connection->transport.net_conf);
+			two_primaries = nc ? nc->two_primaries : false;
+			rcu_read_unlock();
+
+			if (!two_primaries)
+				return SS_TWO_PRIMARIES;
+		}
+
+		return SS_SUCCESS;
+	}
+	return SS_WEAKLY_CONNECTED;
+}
+
+static enum drbd_state_rv
+check_ro_cnt_and_primary(struct drbd_resource *resource)
+{
+	struct twopc_reply *reply = &resource->twopc_reply;
+	struct drbd_connection *connection;
+	enum drbd_state_rv rv = SS_SUCCESS;
+	struct net_conf *nc;
+
+	if (drbd_open_ro_count(resource) == 0)
+		return rv;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		nc = rcu_dereference(connection->transport.net_conf);
+		if (!nc->two_primaries &&
+		    NODE_MASK(connection->peer_node_id) & reply->primary_nodes) {
+			rv = SS_PRIMARY_READER;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return rv;
+}
+
+long twopc_retry_timeout(struct drbd_resource *resource, int retries)
+{
+	struct drbd_connection *connection;
+	int connections = 0;
+	long timeout = 0;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (connection->cstate[NOW] < C_CONNECTING)
+			continue;
+		connections++;
+	}
+	rcu_read_unlock();
+
+	if (connections > 0) {
+		if (retries > 5)
+			retries = 5;
+		timeout = resource->res_opts.twopc_retry_timeout *
+			  HZ / 10 * connections * (1 << retries);
+		timeout = get_random_u32_below(timeout);
+	}
+	return timeout;
+}
+
+void abort_connect(struct drbd_connection *connection)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (test_and_clear_bit(HOLDING_UUID_READ_LOCK, &peer_device->flags))
+			up_read_non_owner(&peer_device->device->uuid_sem);
+		clear_bit(INITIAL_STATE_SENT, &peer_device->flags);
+		clear_bit(INITIAL_STATE_RECEIVED, &peer_device->flags);
+		clear_bit(UUIDS_RECEIVED, &peer_device->flags);
+		clear_bit(CURRENT_UUID_RECEIVED, &peer_device->flags);
+	}
+	rcu_read_unlock();
+}
+
+static void twopc_phase2(struct drbd_resource *resource,
+			 struct twopc_request *request,
+			 u64 reach_immediately)
+{
+	struct drbd_connection *connection;
+	u64 im;
+
+	for_each_connection_ref(connection, im, resource) {
+		u64 mask = NODE_MASK(connection->peer_node_id);
+		if (!(reach_immediately & mask))
+			continue;
+
+		conn_send_twopc_request(connection, request);
+	}
+}
+
+void drbd_print_cluster_wide_state_change(struct drbd_resource *resource, const char *message,
+		unsigned int tid, unsigned int initiator_node_id, int target_node_id,
+		union drbd_state mask, union drbd_state val)
+{
+	char buffer[150], *b, *end = buffer + sizeof(buffer);
+
+	b = buffer;
+	b += scnprintf(b, end - b, "%u->", initiator_node_id);
+	if (target_node_id == -1)
+		b += scnprintf(b, end - b, "all");
+	else
+		b += scnprintf(b, end - b, "%d", target_node_id);
+
+	if (mask.role)
+		b += scnprintf(b, end - b, " role( %s )", drbd_role_str(val.role));
+
+	if (mask.peer)
+		b += scnprintf(b, end - b, " peer( %s )", drbd_role_str(val.peer));
+
+	if (mask.conn) {
+		if (val.conn > C_CONNECTED)
+			b += scnprintf(b, end - b, " repl( %s )", drbd_repl_str(val.conn));
+		else
+			b += scnprintf(b, end - b, " conn( %s )", drbd_conn_str(val.conn));
+	}
+
+	if (mask.disk)
+		b += scnprintf(b, end - b, " disk( %s )", drbd_disk_str(val.disk));
+
+	if (mask.pdsk)
+		b += scnprintf(b, end - b, " pdsk( %s )", drbd_disk_str(val.pdsk));
+
+	// Any of "susp-io( user )", "susp-io( quorum )" or "susp-io( uuid )"
+	if (mask.susp)
+		b += scnprintf(b, end - b, " %ssusp-io", val.susp ? "+" : "-");
+
+	if (mask.susp_nod)
+		b += scnprintf(b, end - b, " susp-io( %sno-disk )", val.susp_nod ? "+" : "-");
+
+	if (mask.susp_fen)
+		b += scnprintf(b, end - b, " susp-io( %sfencing )", val.susp_fen ? "+" : "-");
+
+	if (mask.user_isp)
+		b += scnprintf(b, end - b, " resync-susp( %suser )", val.user_isp ? "+" : "-");
+
+	if (mask.peer_isp)
+		b += scnprintf(b, end - b, " resync-susp( %speer )", val.peer_isp ? "+" : "-");
+
+	if (mask.aftr_isp)
+		b += scnprintf(b, end - b, " resync-susp( %safter dependency )",
+				val.aftr_isp ? "+" : "-");
+
+	if (!mask.i)
+		b += scnprintf(b, end - b, " empty");
+
+	drbd_info(resource, "%s %u: %s\n", message, tid, buffer);
+}
+
+/**
+ * change_cluster_wide_state  -  Cluster-wide two-phase commit
+ * @change: The callback function that does the actual state change.
+ * @context: State change context.
+ * @tag: State change tag to print in status messages.
+ *
+ * Perform a two-phase commit transaction among all (reachable) nodes in the
+ * cluster.  In our transaction model, the initiator of a transaction is also
+ * the coordinator.
+ *
+ * In phase one of the transaction, the coordinator sends all nodes in the
+ * cluster a P_TWOPC_PREPARE packet.  Each node replies with either P_TWOPC_YES
+ * if it consents or with P_TWOPC_NO if it denies the transaction.  Once all
+ * replies have been received, the coordinator sends all nodes in the cluster a
+ * P_TWOPC_COMMIT or P_TWOPC_ABORT packet to finish the transaction.
+ *
+ * When a node in the cluster is busy with another transaction, it replies with
+ * P_TWOPC_NO.  The coordinator is then responsible for retrying the
+ * transaction.
+ *
+ * Since a cluster is not guaranteed to always be fully connected, some nodes
+ * will not be directly reachable from other nodes.  In order to still reach
+ * all nodes in the cluster, participants will forward requests to nodes which
+ * haven't received the request yet:
+ *
+ * The nodes_to_reach field in requests indicates which nodes have received the
+ * request already.  Before forwarding a request to a peer, a node removes
+ * itself from nodes_to_reach; it then sends the request to all directly
+ * connected nodes in nodes_to_reach.
+ *
+ * If there are redundant paths in the cluster, requests will reach some nodes
+ * more than once.  Nodes remember when they are taking part in a transaction;
+ * they detect duplicate requests and reply to them with P_TWOPC_YES packets.
+ * (Transactions are identified by the node id of the initiator and a random,
+ * unique-enough transaction identifier.)
+ *
+ * A configurable timeout determines how long a coordinator or participant will
+ * wait for a transaction to finish.  A transaction that times out is assumed
+ * to have aborted.
+ */
+static enum drbd_state_rv
+change_cluster_wide_state(bool (*change)(struct change_context *, enum change_phase),
+			  struct change_context *context, const char *tag)
+{
+	struct drbd_resource *resource = context->resource;
+	unsigned long irq_flags;
+	struct twopc_request request;
+	struct twopc_reply *reply = &resource->twopc_reply;
+	struct drbd_connection *connection, *target_connection = NULL;
+	enum drbd_state_rv rv;
+	u64 reach_immediately;
+	int retries = 1;
+	unsigned long start_time;
+	bool have_peers;
+
+	begin_state_change(resource, &irq_flags, context->flags | CS_LOCAL_ONLY);
+	resource->state_change_err_str = context->err_str;
+
+	if (local_state_change(context->flags)) {
+		/* Not a cluster-wide state change. */
+		change(context, PH_LOCAL_COMMIT);
+		return end_state_change(resource, &irq_flags, tag);
+	} else {
+		if (!change(context, PH_PREPARE)) {
+			/* Not a cluster-wide state change. */
+			return end_state_change(resource, &irq_flags, tag);
+		}
+		rv = try_state_change(resource);
+		if (rv != SS_SUCCESS) {
+			/* Failure or nothing to do. */
+			/* abort_state_change(resource, &irq_flags); */
+			if (rv == SS_NOTHING_TO_DO)
+				resource->state_change_flags &= ~CS_VERBOSE;
+			return __end_state_change(resource, &irq_flags, rv, tag);
+		}
+		/* Really a cluster-wide state change. */
+	}
+
+	if (!supports_two_phase_commit(resource)) {
+		connection = get_first_connection(resource);
+		rv = SS_SUCCESS;
+		if (connection) {
+			rv = change_peer_state(connection, context->vnr, context->mask, context->val, &irq_flags);
+			kref_put(&connection->kref, drbd_destroy_connection);
+		}
+		if (rv >= SS_SUCCESS)
+			change(context, PH_84_COMMIT);
+		return __end_state_change(resource, &irq_flags, rv, tag);
+	}
+
+	if (!expect(resource, context->flags & CS_SERIALIZE || context->mask.i == 0)) {
+		rv = SS_CW_FAILED_BY_PEER;
+		return __end_state_change(resource, &irq_flags, rv, tag);
+	}
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (!expect(connection, current != connection->receiver.task)) {
+			rcu_read_unlock();
+			BUG();
+		}
+	}
+	rcu_read_unlock();
+
+    retry:
+	if (current == resource->worker.task && resource->remote_state_change)
+		return __end_state_change(resource, &irq_flags, SS_CONCURRENT_ST_CHG, tag);
+
+	complete_remote_state_change(resource, &irq_flags);
+	start_time = jiffies;
+	resource->state_change_err_str = context->err_str;
+
+	*reply = (struct twopc_reply) { 0 };
+
+	reach_immediately = directly_connected_nodes(resource, NOW);
+	if (context->target_node_id != -1) {
+		struct drbd_connection *connection;
+
+		/* Fail if the target node is no longer directly reachable. */
+		connection = drbd_get_connection_by_node_id(resource, context->target_node_id);
+		if (!connection) {
+			rv = SS_NEED_CONNECTION;
+			return __end_state_change(resource, &irq_flags, rv, tag);
+		}
+
+		if (!(connection->cstate[NOW] == C_CONNECTED ||
+		      (connection->cstate[NOW] == C_CONNECTING &&
+		       context->mask.conn == conn_MASK &&
+		       context->val.conn == C_CONNECTED))) {
+			rv = SS_NEED_CONNECTION;
+
+			kref_put(&connection->kref, drbd_destroy_connection);
+			return __end_state_change(resource, &irq_flags, rv, tag);
+		}
+		target_connection = connection;
+
+		/* For connect transactions, add the target node id. */
+		reach_immediately |= NODE_MASK(context->target_node_id);
+	}
+
+	do
+		reply->tid = get_random_u32();
+	while (!reply->tid);
+
+	clear_bit(TWOPC_RECV_SIZES_ERR, &resource->flags);
+	request.tid = reply->tid;
+	request.initiator_node_id = resource->res_opts.node_id;
+	request.target_node_id = context->target_node_id;
+	request.nodes_to_reach = ~(reach_immediately | NODE_MASK(resource->res_opts.node_id));
+	request.vnr = context->vnr;
+	request.cmd = P_TWOPC_PREPARE;
+	request.flags = TWOPC_HAS_REACHABLE;
+
+	resource->twopc.type = TWOPC_STATE_CHANGE;
+	resource->twopc.state_change.mask = context->mask;
+	resource->twopc.state_change.val = context->val;
+	resource->twopc.state_change.primary_nodes = 0;
+	resource->twopc.state_change.reachable_nodes = 0;
+	resource->twopc_parent_nodes = 0;
+	resource->remote_state_change = true;
+
+	drbd_print_cluster_wide_state_change(resource, "Preparing cluster-wide state change",
+			request.tid, resource->res_opts.node_id, context->target_node_id,
+			context->mask, context->val);
+
+	reply->initiator_node_id = resource->res_opts.node_id;
+	reply->target_node_id = context->target_node_id;
+
+	reply->reachable_nodes = directly_connected_nodes(resource, NOW) |
+				       NODE_MASK(resource->res_opts.node_id);
+	if (context->mask.conn == conn_MASK && context->val.conn == C_CONNECTED) {
+		reply->reachable_nodes |= NODE_MASK(context->target_node_id);
+		reply->target_reachable_nodes = reply->reachable_nodes;
+		reply->is_connect = 1;
+		drbd_init_connect_state(target_connection);
+	} else if (context->mask.conn == conn_MASK && context->val.conn == C_DISCONNECTING) {
+		reply->target_reachable_nodes = NODE_MASK(context->target_node_id);
+		reply->reachable_nodes &= ~reply->target_reachable_nodes;
+		reply->is_disconnect = 1;
+	} else {
+		reply->target_reachable_nodes = reply->reachable_nodes;
+	}
+
+	D_ASSERT(resource, !test_bit(TWOPC_WORK_PENDING, &resource->flags));
+	begin_remote_state_change(resource, &irq_flags);
+	rv = __cluster_wide_request(resource, &request, reach_immediately);
+
+	/* If we are changing state attached to a particular connection then we
+	 * expect that connection to remain connected. A failure to send
+	 * P_TWOPC_PREPARE on that connection is a failure for the whole
+	 * cluster-wide state change. */
+	if (target_connection && !test_bit(TWOPC_PREPARED, &target_connection->flags))
+		rv = SS_NEED_CONNECTION;
+
+	have_peers = rv == SS_CW_SUCCESS;
+	if (have_peers) {
+		long t;
+
+		if (context->mask.conn == conn_MASK && context->val.conn == C_CONNECTED &&
+		    target_connection->agreed_pro_version >= 118)
+			conn_connect2(target_connection);
+
+		t = wait_event_interruptible_timeout(resource->state_wait,
+						     cluster_wide_reply_ready(resource),
+						     twopc_timeout(resource));
+		if (t > 0)
+			rv = get_cluster_wide_reply(resource, context);
+		else
+			rv = t == 0 ? SS_TIMEOUT : SS_INTERRUPTED;
+
+		/* while waiting for the replies, reach_immediately might have changed. */
+		reach_immediately = directly_connected_nodes(resource, NOW);
+		if (target_connection && target_connection->cstate[NOW] == C_CONNECTING)
+			reach_immediately |= NODE_MASK(context->target_node_id);
+
+		request.nodes_to_reach =
+			~(reach_immediately | NODE_MASK(resource->res_opts.node_id));
+
+		if (rv == SS_CW_SUCCESS) {
+			u64 directly_reachable = reach_immediately |
+				NODE_MASK(resource->res_opts.node_id);
+
+			if (context->mask.conn == conn_MASK && context->val.conn == C_DISCONNECTING)
+				directly_reachable &= ~NODE_MASK(context->target_node_id);
+
+			if ((context->mask.role == role_MASK && context->val.role == R_PRIMARY) ||
+			    (context->mask.role != role_MASK && resource->role[NOW] == R_PRIMARY)) {
+				reply->primary_nodes |= NODE_MASK(resource->res_opts.node_id);
+				if (drbd_res_data_accessible(resource))
+					reply->weak_nodes |= ~directly_reachable;
+			}
+
+			/*
+			 * When a node is Primary and has access to UpToDate data, it sets
+			 * weak_nodes to the mask of those it is not connected to. This includes the
+			 * bits for nodes which are not configured, so will always have some set
+			 * bits. Thus if there is a Primary node and no bits are set in weak_nodes,
+			 * the Primary cannot have access to UpToDate data.
+			 */
+			if (reply->primary_nodes && !reply->weak_nodes)
+				request.flags |= TWOPC_PRI_INCAPABLE;
+
+			drbd_info(resource, "State change %u: primary_nodes=%lX, weak_nodes=%lX\n",
+				  reply->tid, (unsigned long)reply->primary_nodes,
+				  (unsigned long)reply->weak_nodes);
+
+			if ((context->mask.role == role_MASK && context->val.role == R_PRIMARY) ||
+			    (context->mask.conn == conn_MASK && context->val.conn == C_CONNECTED))
+				rv = check_primaries_distances(resource);
+
+			if (rv >= SS_SUCCESS &&
+			    context->mask.conn == conn_MASK && context->val.conn == C_CONNECTED)
+				rv = check_ro_cnt_and_primary(resource);
+
+			if (!(context->mask.conn == conn_MASK && context->val.conn == C_DISCONNECTING) ||
+			    (reply->reachable_nodes & reply->target_reachable_nodes)) {
+				/* The cluster is still connected after this
+				 * transaction: either this transaction does
+				 * not disconnect a connection, or there are
+				 * redundant connections.  */
+
+				u64 m;
+
+				m = reply->reachable_nodes | reply->target_reachable_nodes;
+				reply->reachable_nodes = m;
+				reply->target_reachable_nodes = m;
+			} else {
+				rcu_read_lock();
+				for_each_connection_rcu(connection, resource) {
+					int node_id = connection->peer_node_id;
+
+					if (node_id == context->target_node_id) {
+						drbd_info(connection, "Cluster is now split\n");
+						break;
+					}
+				}
+				rcu_read_unlock();
+			}
+
+			resource->twopc.state_change.primary_nodes = reply->primary_nodes;
+			resource->twopc.state_change.reachable_nodes =
+				reply->target_reachable_nodes;
+		}
+
+		if (context->mask.conn == conn_MASK && context->val.conn == C_CONNECTED &&
+		    target_connection->agreed_pro_version >= 118) {
+			wait_initial_states_received(target_connection);
+
+			if (rv >= SS_SUCCESS && test_bit(TWOPC_RECV_SIZES_ERR, &resource->flags))
+				rv = SS_HANDSHAKE_DISCONNECT;
+		}
+	}
+
+	request.cmd = rv >= SS_SUCCESS ? P_TWOPC_COMMIT : P_TWOPC_ABORT;
+	if (rv < SS_SUCCESS && target_connection)
+		abort_connect(target_connection);
+
+	if ((rv == SS_TIMEOUT || rv == SS_CONCURRENT_ST_CHG) &&
+	    !(context->flags & CS_DONT_RETRY)) {
+		long timeout = twopc_retry_timeout(resource, retries++);
+		drbd_info(resource, "Retrying cluster-wide state change after %ums\n",
+			  jiffies_to_msecs(timeout));
+		if (have_peers)
+			twopc_phase2(resource, &request, reach_immediately);
+		if (target_connection) {
+			kref_put(&target_connection->kref, drbd_destroy_connection);
+			target_connection = NULL;
+		}
+		clear_remote_state_change(resource);
+		schedule_timeout_interruptible(timeout);
+		end_remote_state_change(resource, &irq_flags, context->flags | CS_TWOPC);
+		goto retry;
+	}
+
+	if (rv >= SS_SUCCESS)
+		drbd_info(resource, "Committing cluster-wide state change %u (%ums)\n",
+			  request.tid,
+			  jiffies_to_msecs(jiffies - start_time));
+	else
+		drbd_info(resource, "Aborting cluster-wide state change %u (%ums) rv = %d\n",
+			  request.tid,
+			  jiffies_to_msecs(jiffies - start_time),
+			  rv);
+
+	if (have_peers && context->change_local_state_last) {
+		set_bit(TWOPC_STATE_CHANGE_PENDING, &resource->flags);
+		twopc_phase2(resource, &request, reach_immediately);
+	}
+
+	end_remote_state_change(resource, &irq_flags, context->flags | CS_TWOPC);
+	clear_bit(TWOPC_STATE_CHANGE_PENDING, &resource->flags);
+	if (rv >= SS_SUCCESS) {
+		change(context, PH_COMMIT);
+		rv = end_state_change(resource, &irq_flags, tag);
+		if (rv < SS_SUCCESS)
+			drbd_err(resource, "FATAL: Local commit of already committed %u failed! \n",
+				 request.tid);
+	} else {
+		abort_state_change(resource, &irq_flags);
+	}
+
+	if (have_peers && !context->change_local_state_last)
+		twopc_phase2(resource, &request, reach_immediately);
+
+	if (target_connection) {
+		kref_put(&target_connection->kref, drbd_destroy_connection);
+	}
+	return rv;
+}
+
+enum determine_dev_size
+change_cluster_wide_device_size(struct drbd_device *device,
+				sector_t local_max_size,
+				uint64_t new_user_size,
+				enum dds_flags dds_flags,
+				struct resize_parms *rs)
+{
+	struct drbd_resource *resource = device->resource;
+	struct twopc_reply *reply = &resource->twopc_reply;
+	struct twopc_request request;
+	unsigned long start_time;
+	unsigned long irq_flags;
+	enum drbd_state_rv rv;
+	enum determine_dev_size dd;
+	u64 reach_immediately;
+	bool have_peers, commit_it;
+	sector_t new_size = 0;
+	int retries = 1;
+
+retry:
+	rv = drbd_support_2pc_resize(resource);
+	if (rv < SS_SUCCESS)
+		return DS_2PC_NOT_SUPPORTED;
+
+	state_change_lock(resource, &irq_flags, CS_VERBOSE | CS_LOCAL_ONLY);
+	rcu_read_lock();
+	complete_remote_state_change(resource, &irq_flags);
+	start_time = jiffies;
+	reach_immediately = directly_connected_nodes(resource, NOW);
+
+	*reply = (struct twopc_reply) { 0 };
+
+	do
+		reply->tid = get_random_u32();
+	while (!reply->tid);
+
+	request.tid = reply->tid;
+	request.initiator_node_id = resource->res_opts.node_id;
+	request.target_node_id = -1;
+	request.nodes_to_reach = ~(reach_immediately | NODE_MASK(resource->res_opts.node_id));
+	request.vnr = device->vnr;
+	request.cmd = P_TWOPC_PREP_RSZ;
+	request.flags = 0;
+	resource->twopc.type = TWOPC_RESIZE;
+	resource->twopc.resize.dds_flags = dds_flags;
+	resource->twopc.resize.user_size = new_user_size;
+	resource->twopc.resize.diskful_primary_nodes = 0;
+	resource->twopc.resize.new_size = 0;
+	resource->twopc_parent_nodes = 0;
+	resource->remote_state_change = true;
+
+	reply->initiator_node_id = resource->res_opts.node_id;
+	reply->target_node_id = -1;
+	reply->max_possible_size = local_max_size;
+	reply->reachable_nodes = reach_immediately | NODE_MASK(resource->res_opts.node_id);
+	reply->target_reachable_nodes = reply->reachable_nodes;
+	if (resource->role[NOW] == R_PRIMARY)
+		reply->diskful_primary_nodes = NODE_MASK(resource->res_opts.node_id);
+	rcu_read_unlock();
+	state_change_unlock(resource, &irq_flags);
+
+	drbd_info(device, "Preparing cluster-wide size change %u "
+		  "(local_max_size = %llu KB, user_cap = %llu KB)\n",
+		  request.tid,
+		  (unsigned long long)local_max_size >> 1,
+		  (unsigned long long)new_user_size >> 1);
+
+	rv = __cluster_wide_request(resource, &request, reach_immediately);
+
+	have_peers = rv == SS_CW_SUCCESS;
+	if (have_peers) {
+		if (wait_event_timeout(resource->state_wait,
+				       cluster_wide_reply_ready(resource),
+				       twopc_timeout(resource)))
+			rv = get_cluster_wide_reply(resource, NULL);
+		else
+			rv = SS_TIMEOUT;
+
+		if (rv == SS_TIMEOUT || rv == SS_CONCURRENT_ST_CHG) {
+			long timeout = twopc_retry_timeout(resource, retries++);
+
+			drbd_info(device, "Retrying cluster-wide size change after %ums\n",
+				  jiffies_to_msecs(timeout));
+
+			request.cmd = P_TWOPC_ABORT;
+			twopc_phase2(resource, &request, reach_immediately);
+
+			clear_remote_state_change(resource);
+			schedule_timeout_interruptible(timeout);
+			goto retry;
+		}
+	}
+
+	if (rv >= SS_SUCCESS) {
+		new_size = drbd_new_dev_size(device, reply->max_possible_size,
+						new_user_size, dds_flags | DDSF_2PC);
+		commit_it = new_size != get_capacity(device->vdisk);
+
+		if (commit_it) {
+			resource->twopc.resize.new_size = new_size;
+			resource->twopc.resize.diskful_primary_nodes = reply->diskful_primary_nodes;
+			drbd_info(device, "Committing cluster-wide size change %u (%ums)\n",
+				  request.tid,
+				  jiffies_to_msecs(jiffies - start_time));
+		} else {
+			drbd_info(device, "Aborting cluster-wide size change %u (%ums) size unchanged\n",
+				  request.tid,
+				  jiffies_to_msecs(jiffies - start_time));
+		}
+	} else {
+		commit_it = false;
+		drbd_info(device, "Aborting cluster-wide size change %u (%ums) rv = %d\n",
+			  request.tid,
+			  jiffies_to_msecs(jiffies - start_time),
+			  rv);
+	}
+
+	request.cmd = commit_it ? P_TWOPC_COMMIT : P_TWOPC_ABORT;
+	if (have_peers)
+		twopc_phase2(resource, &request, reach_immediately);
+
+	if (commit_it) {
+		struct twopc_resize *tr = &resource->twopc.resize;
+
+		tr->diskful_primary_nodes = reply->diskful_primary_nodes;
+		tr->new_size = new_size;
+		tr->dds_flags = dds_flags;
+		tr->user_size = new_user_size;
+
+		dd = drbd_commit_size_change(device, rs, reach_immediately);
+	} else {
+		if (rv == SS_CW_FAILED_BY_PEER)
+			dd = DS_2PC_NOT_SUPPORTED;
+		else if (rv >= SS_SUCCESS)
+			dd = DS_UNCHANGED;
+		else
+			dd = DS_2PC_ERR;
+	}
 
-	return notify_resource_state(skb, seq, resource, &resource_info, type);
+	clear_remote_state_change(resource);
+	return dd;
 }
 
-int notify_connection_state_change(struct sk_buff *skb,
-				    unsigned int seq,
-				    void *state_change,
-				    enum drbd_notification_type type)
+static void twopc_end_nested(struct drbd_resource *resource, enum drbd_packet cmd)
 {
-	struct drbd_connection_state_change *p = state_change;
-	struct drbd_connection *connection = p->connection;
-	struct connection_info connection_info = {
-		.conn_connection_state = p->cstate[NEW],
-		.conn_role = p->peer_role[NEW],
-	};
+	struct drbd_connection *twopc_parent;
+	u64 im;
+	struct twopc_reply twopc_reply;
+	u64 twopc_parent_nodes = 0;
+
+	write_lock_irq(&resource->state_rwlock);
+	twopc_reply = resource->twopc_reply;
+	/* Only send replies if we are in a twopc and have not yet sent replies. */
+	if (twopc_reply.tid && resource->twopc_prepare_reply_cmd == 0) {
+		resource->twopc_prepare_reply_cmd = cmd;
+		twopc_parent_nodes = resource->twopc_parent_nodes;
+	}
+	clear_bit(TWOPC_WORK_PENDING, &resource->flags);
+	write_unlock_irq(&resource->state_rwlock);
 
-	return notify_connection_state(skb, seq, connection, &connection_info, type);
+	if (!twopc_reply.tid)
+		return;
+
+	for_each_connection_ref(twopc_parent, im, resource) {
+		if (!(twopc_parent_nodes & NODE_MASK(twopc_parent->peer_node_id)))
+			continue;
+
+		if (twopc_reply.is_disconnect)
+			set_bit(DISCONNECT_EXPECTED, &twopc_parent->flags);
+
+		dynamic_drbd_dbg(twopc_parent, "Nested state change %u result: %s\n",
+			   twopc_reply.tid, drbd_packet_name(cmd));
+
+		drbd_send_twopc_reply(twopc_parent, cmd, &twopc_reply);
+	}
+	wake_up_all(&resource->twopc_wait);
 }
 
-int notify_device_state_change(struct sk_buff *skb,
-				unsigned int seq,
-				void *state_change,
-				enum drbd_notification_type type)
+static void __nested_twopc_work(struct drbd_resource *resource)
 {
-	struct drbd_device_state_change *device_state_change = state_change;
-	struct drbd_device *device = device_state_change->device;
-	struct device_info device_info = {
-		.dev_disk_state = device_state_change->disk_state[NEW],
-	};
+	enum drbd_state_rv rv;
+	enum drbd_packet cmd;
 
-	return notify_device_state(skb, seq, device, &device_info, type);
+	rv = get_cluster_wide_reply(resource, NULL);
+	if (rv >= SS_SUCCESS)
+		cmd = P_TWOPC_YES;
+	else if (rv == SS_CONCURRENT_ST_CHG || rv == SS_HANDSHAKE_RETRY)
+		cmd = P_TWOPC_RETRY;
+	else
+		cmd = P_TWOPC_NO;
+	twopc_end_nested(resource, cmd);
 }
 
-int notify_peer_device_state_change(struct sk_buff *skb,
-				     unsigned int seq,
-				     void *state_change,
-				     enum drbd_notification_type type)
+void nested_twopc_work(struct work_struct *work)
+{
+	struct drbd_resource *resource =
+		container_of(work, struct drbd_resource, twopc_work);
+
+	__nested_twopc_work(resource);
+
+	kref_put(&resource->kref, drbd_destroy_resource);
+}
+
+void drbd_maybe_cluster_wide_reply(struct drbd_resource *resource)
+{
+	lockdep_assert_held(&resource->state_rwlock);
+
+	if (!resource->remote_state_change || !cluster_wide_reply_ready(resource))
+		return;
+
+	if (resource->twopc_reply.initiator_node_id == resource->res_opts.node_id) {
+		wake_up_all(&resource->state_wait);
+		return;
+	}
+
+	if (test_and_set_bit(TWOPC_WORK_PENDING, &resource->flags))
+		return;
+
+	kref_get(&resource->kref);
+	schedule_work(&resource->twopc_work);
+}
+
+enum drbd_state_rv
+nested_twopc_request(struct drbd_resource *resource, struct twopc_request *request)
+{
+	u64 nodes_to_reach, reach_immediately;
+	enum drbd_packet cmd = request->cmd;
+	enum drbd_state_rv rv;
+	bool have_peers;
+
+	write_lock_irq(&resource->state_rwlock);
+	nodes_to_reach = request->nodes_to_reach;
+	reach_immediately = directly_connected_nodes(resource, NOW) & nodes_to_reach;
+	nodes_to_reach &= ~(reach_immediately | NODE_MASK(resource->res_opts.node_id));
+	request->nodes_to_reach = nodes_to_reach;
+	write_unlock_irq(&resource->state_rwlock);
+
+	rv = __cluster_wide_request(resource, request, reach_immediately);
+	have_peers = rv == SS_CW_SUCCESS;
+	if (cmd == P_TWOPC_PREPARE || cmd == P_TWOPC_PREP_RSZ) {
+		if (rv < SS_SUCCESS)
+			twopc_end_nested(resource, P_TWOPC_NO);
+		else if (!have_peers && cluster_wide_reply_ready(resource)) /* no nested nodes */
+			__nested_twopc_work(resource);
+	}
+	return rv;
+}
+
+static bool has_up_to_date_peer_disks(struct drbd_device *device)
 {
-	struct drbd_peer_device_state_change *p = state_change;
-	struct drbd_peer_device *peer_device = p->peer_device;
-	struct peer_device_info peer_device_info = {
-		.peer_repl_state = p->repl_state[NEW],
-		.peer_disk_state = p->disk_state[NEW],
-		.peer_resync_susp_user = p->resync_susp_user[NEW],
-		.peer_resync_susp_peer = p->resync_susp_peer[NEW],
-		.peer_resync_susp_dependency = p->resync_susp_dependency[NEW],
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device)
+		if (peer_device->disk_state[NEW] == D_UP_TO_DATE)
+			return true;
+	return false;
+}
+
+static void disconnect_where_resync_target(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device)
+		if (is_sync_target_state(peer_device, NEW))
+			__change_cstate(peer_device->connection, C_TEAR_DOWN);
+}
+
+static bool do_change_role(struct change_context *context, enum change_phase phase)
+{
+	struct drbd_resource *resource = context->resource;
+	enum drbd_role role = context->val.role;
+	int flags = context->flags;
+	struct drbd_device *device;
+	int vnr;
+
+	resource->role[NEW] = role;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (role == R_PRIMARY && (flags & CS_FP_LOCAL_UP_TO_DATE)) {
+			if (device->disk_state[NEW] < D_UP_TO_DATE &&
+			    device->disk_state[NEW] >= D_INCONSISTENT &&
+			    !has_up_to_date_peer_disks(device)) {
+				device->disk_state[NEW] = D_UP_TO_DATE;
+				/* adding it to the context so that it gets sent to the peers */
+				context->mask.disk |= disk_MASK;
+				context->val.disk |= D_UP_TO_DATE;
+				disconnect_where_resync_target(device);
+			}
+		}
+
+		if (role == R_PRIMARY && (flags & CS_FP_OUTDATE_PEERS)) {
+			struct drbd_peer_device *peer_device;
+			for_each_peer_device_rcu(peer_device, device) {
+				if (peer_device->disk_state[NEW] == D_UNKNOWN)
+					__change_peer_disk_state(peer_device, D_OUTDATED);
+			}
+		}
+
+		if (role == R_PRIMARY && phase == PH_COMMIT) {
+			u64 reachable_nodes = resource->twopc_reply.reachable_nodes;
+			struct drbd_peer_device *peer_device;
+
+			for_each_peer_device_rcu(peer_device, device) {
+				if (NODE_MASK(peer_device->node_id) & reachable_nodes &&
+				    peer_device->disk_state[NEW] == D_UNKNOWN &&
+				    want_bitmap(peer_device))
+					__change_peer_disk_state(peer_device, D_OUTDATED);
+			}
+		}
+	}
+	rcu_read_unlock();
+
+	return phase != PH_PREPARE ||
+		context->resource->role[NOW] != context->val.role;
+}
+
+enum drbd_state_rv change_role(struct drbd_resource *resource,
+			       enum drbd_role role,
+			       enum chg_state_flags flags,
+			       const char *tag,
+			       const char **err_str)
+{
+	struct change_context role_context = {
+		.resource = resource,
+		.vnr = -1,
+		.mask = { { .role = role_MASK } },
+		.val = { { .role = role } },
+		.target_node_id = -1,
+		.flags = flags | CS_SERIALIZE,
+		.err_str = err_str,
 	};
+	enum drbd_state_rv rv;
+	bool got_state_sem = false;
 
-	return notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type);
+	if (role == R_SECONDARY) {
+		if (!(flags & CS_ALREADY_SERIALIZED)) {
+			down(&resource->state_sem);
+			got_state_sem = true;
+			role_context.flags |= CS_ALREADY_SERIALIZED;
+		}
+		role_context.change_local_state_last = true;
+	}
+	rv = change_cluster_wide_state(do_change_role, &role_context, tag);
+	if (got_state_sem)
+		up(&resource->state_sem);
+	return rv;
 }
 
-static void broadcast_state_change(struct drbd_state_change *state_change)
+void __change_io_susp_user(struct drbd_resource *resource, bool value)
 {
-	struct drbd_resource_state_change *resource_state_change = &state_change->resource[0];
-	bool resource_state_has_changed;
-	unsigned int n_device, n_connection, n_peer_device, n_peer_devices;
-	int (*last_func)(struct sk_buff *, unsigned int,
-		void *, enum drbd_notification_type) = NULL;
-	void *last_arg = NULL;
+	resource->susp_user[NEW] = value;
+}
 
-#define HAS_CHANGED(state) ((state)[OLD] != (state)[NEW])
-#define FINAL_STATE_CHANGE(type) \
-	({ if (last_func) \
-		last_func(NULL, 0, last_arg, type); \
-	})
-#define REMEMBER_STATE_CHANGE(func, arg, type) \
-	({ FINAL_STATE_CHANGE(type | NOTIFY_CONTINUES); \
-	   last_func = func; \
-	   last_arg = arg; \
-	 })
+enum drbd_state_rv change_io_susp_user(struct drbd_resource *resource,
+				       bool value,
+				       enum chg_state_flags flags)
+{
+	unsigned long irq_flags;
 
-	mutex_lock(&notification_mutex);
+	begin_state_change(resource, &irq_flags, flags);
+	__change_io_susp_user(resource, value);
+	return end_state_change(resource, &irq_flags, value ? "suspend-io" : "resume-io");
+}
 
-	resource_state_has_changed =
-	    HAS_CHANGED(resource_state_change->role) ||
-	    HAS_CHANGED(resource_state_change->susp) ||
-	    HAS_CHANGED(resource_state_change->susp_nod) ||
-	    HAS_CHANGED(resource_state_change->susp_fen);
+void __change_io_susp_no_data(struct drbd_resource *resource, bool value)
+{
+	resource->susp_nod[NEW] = value;
+}
 
-	if (resource_state_has_changed)
-		REMEMBER_STATE_CHANGE(notify_resource_state_change,
-				      resource_state_change, NOTIFY_CHANGE);
+void __change_io_susp_fencing(struct drbd_connection *connection, bool value)
+{
+	connection->susp_fen[NEW] = value;
+}
 
-	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
-		struct drbd_connection_state_change *connection_state_change =
-				&state_change->connections[n_connection];
+void __change_io_susp_quorum(struct drbd_resource *resource, bool value)
+{
+	resource->susp_quorum[NEW] = value;
+}
 
-		if (HAS_CHANGED(connection_state_change->peer_role) ||
-		    HAS_CHANGED(connection_state_change->cstate))
-			REMEMBER_STATE_CHANGE(notify_connection_state_change,
-					      connection_state_change, NOTIFY_CHANGE);
+void __change_disk_state(struct drbd_device *device, enum drbd_disk_state disk_state)
+{
+	device->disk_state[NEW] = disk_state;
+}
+
+void __downgrade_disk_states(struct drbd_resource *resource, enum drbd_disk_state disk_state)
+{
+	struct drbd_device *device;
+	int vnr;
+
+	rcu_read_lock();
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (device->disk_state[NEW] > disk_state)
+			__change_disk_state(device, disk_state);
+	}
+	rcu_read_unlock();
+}
+
+void __outdate_myself(struct drbd_resource *resource)
+{
+	struct drbd_device *device;
+	int vnr;
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (device->disk_state[NOW] > D_OUTDATED)
+			__change_disk_state(device, D_OUTDATED);
+	}
+}
+
+static bool device_has_connected_peer_devices(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+
+	for_each_peer_device(peer_device, device)
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED)
+			return true;
+	return false;
+}
+
+static bool device_has_peer_devices_with_disk(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+	bool rv = false;
+
+	for_each_peer_device(peer_device, device) {
+		if (peer_device->connection->cstate[NOW] == C_CONNECTED) {
+			/* We expect to receive up-to-date UUIDs soon.
+			   To avoid a race in receive_state, "clear" uuids while
+			   holding state_rwlock. I.e. atomic with the state change */
+			clear_bit(UUIDS_RECEIVED, &peer_device->flags);
+			if (peer_device->disk_state[NOW] > D_DISKLESS &&
+			    peer_device->disk_state[NOW] != D_UNKNOWN)
+				rv = true;
+		}
+	}
+
+	return rv;
+}
+
+static void restore_outdated_in_pdsk(struct drbd_device *device)
+{
+	struct drbd_peer_device *peer_device;
+
+	if (!get_ldev_if_state(device, D_ATTACHING))
+		return;
+
+	for_each_peer_device(peer_device, device) {
+		int node_id = peer_device->connection->peer_node_id;
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
+
+		if ((peer_md->flags & MDF_PEER_OUTDATED) &&
+		    peer_device->disk_state[NEW] == D_UNKNOWN)
+			__change_peer_disk_state(peer_device, D_OUTDATED);
+	}
+
+	put_ldev(device);
+}
+
+static bool do_twopc_after_lost_peer(struct change_context *context, enum change_phase phase)
+{
+	struct drbd_resource *resource = context->resource;
+	struct twopc_reply *reply = &resource->twopc_reply;
+	u64 directly_reachable = directly_connected_nodes(resource, NEW) |
+		NODE_MASK(resource->res_opts.node_id);
+	bool pri_incapable = reply->primary_nodes && !reply->weak_nodes; /* TWOPC_PRI_INCAPABLE */
+
+	if (phase == PH_COMMIT && (reply->primary_nodes & ~directly_reachable && !pri_incapable)) {
+		__outdate_myself(resource);
+	} else {
+		struct drbd_device *device;
+		int vnr;
+
+		idr_for_each_entry(&resource->devices, device, vnr) {
+			if (device->disk_state[NOW] == D_CONSISTENT &&
+			    may_return_to_up_to_date(device, NOW))
+				__change_disk_state(device, D_UP_TO_DATE);
+		}
 	}
 
-	for (n_device = 0; n_device < state_change->n_devices; n_device++) {
-		struct drbd_device_state_change *device_state_change =
-			&state_change->devices[n_device];
+	return phase != PH_PREPARE || reply->reachable_nodes != NODE_MASK(resource->res_opts.node_id);
+}
+
+static enum drbd_state_rv twopc_after_lost_peer(struct drbd_resource *resource,
+						enum chg_state_flags flags)
+{
+	struct change_context context = {
+		.resource = resource,
+		.vnr = -1,
+		.mask = { },
+		.val = { },
+		.target_node_id = -1,
+		.flags = flags | (resource->res_opts.quorum != QOU_OFF ? CS_FORCE_RECALC : 0),
+		.change_local_state_last = false,
+	};
+
+	/* The other nodes get the request for an empty state change. I.e. they
+	   will agree to this change request. At commit time we know where to
+	   go from the D_CONSISTENT, since we got the primary mask. */
+	return change_cluster_wide_state(do_twopc_after_lost_peer, &context, "lost-peer");
+}
+
+void drbd_empty_twopc_work_fn(struct work_struct *work)
+{
+	struct drbd_resource *resource = container_of(work, struct drbd_resource, empty_twopc);
 
-		if (HAS_CHANGED(device_state_change->disk_state))
-			REMEMBER_STATE_CHANGE(notify_device_state_change,
-					      device_state_change, NOTIFY_CHANGE);
-	}
+	twopc_after_lost_peer(resource, CS_VERBOSE);
 
-	n_peer_devices = state_change->n_devices * state_change->n_connections;
-	for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) {
-		struct drbd_peer_device_state_change *p =
-			&state_change->peer_devices[n_peer_device];
+	clear_bit(TRY_BECOME_UP_TO_DATE_PENDING, &resource->flags);
+	wake_up_all(&resource->state_wait);
 
-		if (HAS_CHANGED(p->disk_state) ||
-		    HAS_CHANGED(p->repl_state) ||
-		    HAS_CHANGED(p->resync_susp_user) ||
-		    HAS_CHANGED(p->resync_susp_peer) ||
-		    HAS_CHANGED(p->resync_susp_dependency))
-			REMEMBER_STATE_CHANGE(notify_peer_device_state_change,
-					      p, NOTIFY_CHANGE);
+	kref_put(&resource->kref, drbd_destroy_resource);
+}
+
+static bool do_change_disk_state(struct change_context *context, enum change_phase phase)
+{
+	struct drbd_device *device =
+		container_of(context, struct change_disk_state_context, context)->device;
+	bool cluster_wide_state_change = false;
+
+	if (device->disk_state[NOW] == D_ATTACHING &&
+	    context->val.disk == D_NEGOTIATING) {
+		if (device_has_peer_devices_with_disk(device)) {
+			cluster_wide_state_change =
+				supports_two_phase_commit(device->resource);
+		} else {
+			/* very last part of attach */
+			context->val.disk = disk_state_from_md(device);
+			restore_outdated_in_pdsk(device);
+		}
+	} else if (device->disk_state[NOW] != D_DETACHING &&
+		   context->val.disk == D_DETACHING &&
+		   device_has_connected_peer_devices(device)) {
+		cluster_wide_state_change = true;
 	}
+	__change_disk_state(device, context->val.disk);
+	return phase != PH_PREPARE || cluster_wide_state_change;
+}
 
-	FINAL_STATE_CHANGE(NOTIFY_CHANGE);
-	mutex_unlock(&notification_mutex);
+enum drbd_state_rv change_disk_state(struct drbd_device *device,
+				     enum drbd_disk_state disk_state,
+				     enum chg_state_flags flags,
+				     const char *tag,
+				     const char **err_str)
+{
+	struct change_disk_state_context disk_state_context = {
+		.context = {
+			.resource = device->resource,
+			.vnr = device->vnr,
+			.mask = { { .disk = disk_MASK } },
+			.val = { { .disk = disk_state } },
+			.target_node_id = -1,
+			.flags = flags,
+			.change_local_state_last = true,
+			.err_str = err_str,
+		},
+		.device = device,
+	};
 
-#undef HAS_CHANGED
-#undef FINAL_STATE_CHANGE
-#undef REMEMBER_STATE_CHANGE
+	return change_cluster_wide_state(do_change_disk_state,
+					 &disk_state_context.context, tag);
 }
 
-/* takes old and new peer disk state */
-static bool lost_contact_to_peer_data(enum drbd_disk_state os, enum drbd_disk_state ns)
+void __change_cstate(struct drbd_connection *connection, enum drbd_conn_state cstate)
 {
-	if ((os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED)
-	&&  (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED))
-		return true;
+	if (cstate == C_DISCONNECTING)
+		set_bit(DISCONNECT_EXPECTED, &connection->flags);
 
-	/* Scenario, starting with normal operation
-	 * Connected Primary/Secondary UpToDate/UpToDate
-	 * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen)
-	 * ...
-	 * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!)
-	 */
-	if (os == D_UNKNOWN
-	&&  (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED))
-		return true;
+	connection->cstate[NEW] = cstate;
+	if (cstate < C_CONNECTED) {
+		struct drbd_peer_device *peer_device;
+		int vnr;
 
-	return false;
+		rcu_read_lock();
+		idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
+			__change_repl_state(peer_device, L_OFF);
+		rcu_read_unlock();
+	}
 }
 
-/**
- * after_state_ch() - Perform after state change actions that may sleep
- * @device:	DRBD device.
- * @os:		old state.
- * @ns:		new state.
- * @flags:	Flags
- * @state_change: state change to broadcast
- */
-static void after_state_ch(struct drbd_device *device, union drbd_state os,
-			   union drbd_state ns, enum chg_state_flags flags,
-			   struct drbd_state_change *state_change)
+static bool connection_has_connected_peer_devices(struct drbd_connection *connection)
 {
-	struct drbd_resource *resource = device->resource;
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	struct sib_info sib;
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	broadcast_state_change(state_change);
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (peer_device->repl_state[NOW] >= L_ESTABLISHED)
+			return true;
+	}
+	return false;
+}
 
-	sib.sib_reason = SIB_STATE_CHANGE;
-	sib.os = os;
-	sib.ns = ns;
+enum outdate_what { OUTDATE_NOTHING, OUTDATE_DISKS, OUTDATE_PEER_DISKS };
 
-	if ((os.disk != D_UP_TO_DATE || os.pdsk != D_UP_TO_DATE)
-	&&  (ns.disk == D_UP_TO_DATE && ns.pdsk == D_UP_TO_DATE)) {
-		clear_bit(CRASHED_PRIMARY, &device->flags);
-		if (device->p_uuid)
-			device->p_uuid[UI_FLAGS] &= ~((u64)2);
+static enum outdate_what outdate_on_disconnect(struct drbd_connection *connection)
+{
+	struct drbd_resource *resource = connection->resource;
+
+	if (connection->cstate[NOW] == C_CONNECTED &&
+	    (connection->fencing_policy >= FP_RESOURCE ||
+	     connection->resource->res_opts.quorum != QOU_OFF) &&
+	    resource->role[NOW] != connection->peer_role[NOW]) {
+		/* primary politely disconnects from secondary,
+		 * tells peer to please outdate itself */
+		if (resource->role[NOW] == R_PRIMARY)
+			return OUTDATE_PEER_DISKS;
+
+		/* secondary politely disconnect from primary,
+		 * proposes to outdate itself. */
+		if (connection->peer_role[NOW] == R_PRIMARY)
+			return OUTDATE_DISKS;
 	}
+	return OUTDATE_NOTHING;
+}
 
-	/* Inform userspace about the change... */
-	drbd_bcast_event(device, &sib);
+static void __change_cstate_and_outdate(struct drbd_connection *connection,
+					enum drbd_conn_state cstate,
+					enum outdate_what outdate_what)
+{
+	__change_cstate(connection, cstate);
+	switch (outdate_what) {
+	case OUTDATE_DISKS:
+		__downgrade_disk_states(connection->resource, D_OUTDATED);
+		break;
+	case OUTDATE_PEER_DISKS:
+		__downgrade_peer_disk_states(connection, D_OUTDATED);
+		break;
+	case OUTDATE_NOTHING:
+		break;
+	}
+}
 
-	if (!(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE) &&
-	    (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE))
-		drbd_khelper(device, "pri-on-incon-degr");
+void apply_connect(struct drbd_connection *connection, bool commit)
+{
+	struct drbd_peer_device *peer_device;
+	int vnr;
 
-	/* Here we have the actions that are performed after a
-	   state change. This function might sleep */
+	if (!commit || connection->cstate[NEW] != C_CONNECTED)
+		return;
 
-	if (ns.susp_nod) {
-		enum drbd_req_event what = NOTHING;
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		struct drbd_device *device = peer_device->device;
+		union drbd_state s = peer_device->connect_state;
 
-		spin_lock_irq(&device->resource->req_lock);
-		if (os.conn < C_CONNECTED && conn_lowest_conn(connection) >= C_CONNECTED)
-			what = RESEND;
+		if (s.disk != D_MASK)
+			__change_disk_state(device, s.disk);
+		if (device->disk_state[NOW] != D_NEGOTIATING)
+			__change_repl_state(peer_device, s.conn);
+		__change_peer_disk_state(peer_device, s.pdsk);
+		__change_resync_susp_peer(peer_device, s.peer_isp);
 
-		if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) &&
-		    conn_lowest_disk(connection) == D_UP_TO_DATE)
-			what = RESTART_FROZEN_DISK_IO;
+		if (s.conn == L_OFF)
+			__change_cstate(connection, C_DISCONNECTING);
 
-		if (resource->susp_nod && what != NOTHING) {
-			_tl_restart(connection, what);
-			_conn_request_state(connection,
-					    (union drbd_state) { { .susp_nod = 1 } },
-					    (union drbd_state) { { .susp_nod = 0 } },
-					    CS_VERBOSE);
-		}
-		spin_unlock_irq(&device->resource->req_lock);
+		if (commit)
+			clear_bit(DISCARD_MY_DATA, &peer_device->flags);
 	}
+}
 
-	if (ns.susp_fen) {
-		spin_lock_irq(&device->resource->req_lock);
-		if (resource->susp_fen && conn_lowest_conn(connection) >= C_CONNECTED) {
-			/* case2: The connection was established again: */
-			struct drbd_peer_device *peer_device;
-			int vnr;
-
-			rcu_read_lock();
-			idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-				clear_bit(NEW_CUR_UUID, &peer_device->device->flags);
-			rcu_read_unlock();
-
-			/* We should actively create a new uuid, _before_
-			 * we resume/resent, if the peer is diskless
-			 * (recovery from a multiple error scenario).
-			 * Currently, this happens with a slight delay
-			 * below when checking lost_contact_to_peer_data() ...
-			 */
-			_tl_restart(connection, RESEND);
-			_conn_request_state(connection,
-					    (union drbd_state) { { .susp_fen = 1 } },
-					    (union drbd_state) { { .susp_fen = 0 } },
-					    CS_VERBOSE);
-		}
-		spin_unlock_irq(&device->resource->req_lock);
-	}
-
-	/* Became sync source.  With protocol >= 96, we still need to send out
-	 * the sync uuid now. Need to do that before any drbd_send_state, or
-	 * the other side may go "paused sync" before receiving the sync uuids,
-	 * which is unexpected. */
-	if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) &&
-	    (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) &&
-	    connection->agreed_pro_version >= 96 && get_ldev(device)) {
-		drbd_gen_and_send_sync_uuid(peer_device);
-		put_ldev(device);
-	}
+struct change_cstate_context {
+	struct change_context context;
+	struct drbd_connection *connection;
+	enum outdate_what outdate_what;
+};
 
-	/* Do not change the order of the if above and the two below... */
-	if (os.pdsk == D_DISKLESS &&
-	    ns.pdsk > D_DISKLESS && ns.pdsk != D_UNKNOWN) {      /* attach on the peer */
-		/* we probably will start a resync soon.
-		 * make sure those things are properly reset. */
-		device->rs_total = 0;
-		device->rs_failed = 0;
-		atomic_set(&device->rs_pending_cnt, 0);
-		drbd_rs_cancel_all(device);
-
-		drbd_send_uuids(peer_device);
-		drbd_send_state(peer_device, ns);
-	}
-	/* No point in queuing send_bitmap if we don't have a connection
-	 * anymore, so check also the _current_ state, not only the new state
-	 * at the time this work was queued. */
-	if (os.conn != C_WF_BITMAP_S && ns.conn == C_WF_BITMAP_S &&
-	    device->state.conn == C_WF_BITMAP_S)
-		drbd_queue_bitmap_io(device, &drbd_send_bitmap, NULL,
-				"send_bitmap (WFBitMapS)",
-				BM_LOCKED_TEST_ALLOWED, peer_device);
-
-	/* Lost contact to peer's copy of the data */
-	if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) {
-		if (get_ldev(device)) {
-			if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) &&
-			    device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
-				if (drbd_suspended(device)) {
-					set_bit(NEW_CUR_UUID, &device->flags);
-				} else {
-					drbd_uuid_new_current(device);
-					drbd_send_uuids(peer_device);
-				}
+static bool do_change_cstate(struct change_context *context, enum change_phase phase)
+{
+	struct change_cstate_context *cstate_context =
+		container_of(context, struct change_cstate_context, context);
+	struct drbd_connection *connection = cstate_context->connection;
+	struct drbd_resource *resource = context->resource;
+	struct twopc_reply *reply = &resource->twopc_reply;
+
+	if (phase == PH_PREPARE) {
+		cstate_context->outdate_what = OUTDATE_NOTHING;
+		if (context->val.conn == C_DISCONNECTING && !(context->flags & CS_HARD)) {
+			cstate_context->outdate_what =
+				outdate_on_disconnect(connection);
+			switch (cstate_context->outdate_what) {
+			case OUTDATE_DISKS:
+				context->mask.disk = disk_MASK;
+				context->val.disk = D_OUTDATED;
+				break;
+			case OUTDATE_PEER_DISKS:
+				context->mask.pdsk = pdsk_MASK;
+				context->val.pdsk = D_OUTDATED;
+				break;
+			case OUTDATE_NOTHING:
+				break;
 			}
-			put_ldev(device);
 		}
 	}
+	if ((context->val.conn == C_CONNECTED && connection->cstate[NEW] == C_CONNECTING) ||
+	    context->val.conn != C_CONNECTED)
+		__change_cstate_and_outdate(connection,
+					    context->val.conn,
+					    cstate_context->outdate_what);
+
+	if (context->val.conn == C_CONNECTED &&
+	    connection->agreed_pro_version >= 117)
+		apply_connect(connection, phase == PH_COMMIT);
+
+	if (phase == PH_COMMIT) {
+		u64 directly_reachable = directly_connected_nodes(resource, NEW) |
+			NODE_MASK(resource->res_opts.node_id);
+
+		if (reply->primary_nodes & ~directly_reachable)
+			__outdate_myself(resource);
+	}
 
-	if (ns.pdsk < D_INCONSISTENT && get_ldev(device)) {
-		if (os.peer != R_PRIMARY && ns.peer == R_PRIMARY &&
-		    device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
-			drbd_uuid_new_current(device);
-			drbd_send_uuids(peer_device);
-		}
-		/* D_DISKLESS Peer becomes secondary */
-		if (os.peer == R_PRIMARY && ns.peer == R_SECONDARY)
-			/* We may still be Primary ourselves.
-			 * No harm done if the bitmap still changes,
-			 * redirtied pages will follow later. */
-			drbd_bitmap_io_from_worker(device, &drbd_bm_write,
-				"demote diskless peer", BM_LOCKED_SET_ALLOWED, peer_device);
-		put_ldev(device);
+	if (context->val.conn == C_CONNECTED && connection->peer_role[NOW] == R_UNKNOWN) {
+		enum drbd_role target_role =
+			(reply->primary_nodes & NODE_MASK(context->target_node_id)) ?
+			R_PRIMARY : R_SECONDARY;
+
+		__change_peer_role(connection, target_role);
 	}
 
-	/* Write out all changed bits on demote.
-	 * Though, no need to da that just yet
-	 * if there is a resync going on still */
-	if (os.role == R_PRIMARY && ns.role == R_SECONDARY &&
-		device->state.conn <= C_CONNECTED && get_ldev(device)) {
-		/* No changes to the bitmap expected this time, so assert that,
-		 * even though no harm was done if it did change. */
-		drbd_bitmap_io_from_worker(device, &drbd_bm_write,
-				"demote", BM_LOCKED_TEST_ALLOWED, peer_device);
-		put_ldev(device);
+	return phase != PH_PREPARE ||
+	       context->val.conn == C_CONNECTED ||
+	       (context->val.conn == C_DISCONNECTING &&
+		connection_has_connected_peer_devices(connection));
+}
+
+/**
+ * change_cstate_tag()  -  change the connection state of a connection
+ * @connection: DRBD connection.
+ * @cstate: The connection state to change to.
+ * @flags: State change flags.
+ * @tag: State change tag to print in status messages.
+ * @err_str: Pointer to save the error string to.
+ *
+ * When disconnecting from a peer, we may also need to outdate the local or
+ * peer disks depending on the fencing policy.  This cannot easily be split
+ * into two state changes.
+ */
+enum drbd_state_rv change_cstate_tag(struct drbd_connection *connection,
+				    enum drbd_conn_state cstate,
+				    enum chg_state_flags flags,
+				    const char *tag,
+				    const char **err_str)
+{
+	struct change_cstate_context cstate_context = {
+		.context = {
+			.resource = connection->resource,
+			.vnr = -1,
+			.mask = { { .conn = conn_MASK } },
+			.val = { { .conn = cstate } },
+			.target_node_id = connection->peer_node_id,
+			.flags = flags,
+			.change_local_state_last = true,
+			.err_str = err_str,
+		},
+		.connection = connection,
+	};
+
+	if (cstate == C_CONNECTED) {
+		cstate_context.context.mask.role = role_MASK;
+		cstate_context.context.val.role = connection->resource->role[NOW];
 	}
 
-	/* Last part of the attaching process ... */
-	if (ns.conn >= C_CONNECTED &&
-	    os.disk == D_ATTACHING && ns.disk == D_NEGOTIATING) {
-		drbd_send_sizes(peer_device, 0, 0);  /* to start sync... */
-		drbd_send_uuids(peer_device);
-		drbd_send_state(peer_device, ns);
-	}
-
-	/* We want to pause/continue resync, tell peer. */
-	if (ns.conn >= C_CONNECTED &&
-	     ((os.aftr_isp != ns.aftr_isp) ||
-	      (os.user_isp != ns.user_isp)))
-		drbd_send_state(peer_device, ns);
-
-	/* In case one of the isp bits got set, suspend other devices. */
-	if ((!os.aftr_isp && !os.peer_isp && !os.user_isp) &&
-	    (ns.aftr_isp || ns.peer_isp || ns.user_isp))
-		suspend_other_sg(device);
-
-	/* Make sure the peer gets informed about eventual state
-	   changes (ISP bits) while we were in WFReportParams. */
-	if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED)
-		drbd_send_state(peer_device, ns);
-
-	if (os.conn != C_AHEAD && ns.conn == C_AHEAD)
-		drbd_send_state(peer_device, ns);
-
-	/* We are in the progress to start a full sync... */
-	if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
-	    (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S))
-		/* no other bitmap changes expected during this phase */
-		drbd_queue_bitmap_io(device,
-			&drbd_bmio_set_n_write, &abw_start_sync,
-			"set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED,
-			peer_device);
-
-	/* first half of local IO error, failure to attach,
-	 * or administrative detach */
-	if (os.disk != D_FAILED && ns.disk == D_FAILED) {
-		enum drbd_io_error_p eh = EP_PASS_ON;
-		int was_io_error = 0;
-		/* corresponding get_ldev was in _drbd_set_state, to serialize
-		 * our cleanup here with the transition to D_DISKLESS.
-		 * But is is still not save to dreference ldev here, since
-		 * we might come from an failed Attach before ldev was set. */
-		if (device->ldev) {
-			rcu_read_lock();
-			eh = rcu_dereference(device->ldev->disk_conf)->on_io_error;
-			rcu_read_unlock();
+	/*
+	 * Hard connection state changes like a protocol error or forced
+	 * disconnect may occur while we are holding resource->state_sem.  In
+	 * that case, omit CS_SERIALIZE so that we don't deadlock trying to
+	 * grab that mutex again.
+	 */
+	if (!(flags & CS_HARD))
+		cstate_context.context.flags |= CS_SERIALIZE;
 
-			was_io_error = test_and_clear_bit(WAS_IO_ERROR, &device->flags);
-
-			/* Intentionally call this handler first, before drbd_send_state().
-			 * See: 2932204 drbd: call local-io-error handler early
-			 * People may chose to hard-reset the box from this handler.
-			 * It is useful if this looks like a "regular node crash". */
-			if (was_io_error && eh == EP_CALL_HELPER)
-				drbd_khelper(device, "local-io-error");
-
-			/* Immediately allow completion of all application IO,
-			 * that waits for completion from the local disk,
-			 * if this was a force-detach due to disk_timeout
-			 * or administrator request (drbdsetup detach --force).
-			 * Do NOT abort otherwise.
-			 * Aborting local requests may cause serious problems,
-			 * if requests are completed to upper layers already,
-			 * and then later the already submitted local bio completes.
-			 * This can cause DMA into former bio pages that meanwhile
-			 * have been re-used for other things.
-			 * So aborting local requests may cause crashes,
-			 * or even worse, silent data corruption.
-			 */
-			if (test_and_clear_bit(FORCE_DETACH, &device->flags))
-				tl_abort_disk_io(device);
+	return change_cluster_wide_state(do_change_cstate, &cstate_context.context, tag);
+}
 
-			/* current state still has to be D_FAILED,
-			 * there is only one way out: to D_DISKLESS,
-			 * and that may only happen after our put_ldev below. */
-			if (device->state.disk != D_FAILED)
-				drbd_err(device,
-					"ASSERT FAILED: disk is %s during detach\n",
-					drbd_disk_str(device->state.disk));
+void __change_peer_role(struct drbd_connection *connection, enum drbd_role peer_role)
+{
+	connection->peer_role[NEW] = peer_role;
+}
 
-			if (ns.conn >= C_CONNECTED)
-				drbd_send_state(peer_device, ns);
+void __change_repl_state(struct drbd_peer_device *peer_device, enum drbd_repl_state repl_state)
+{
+	peer_device->repl_state[NEW] = repl_state;
+	if (repl_state > L_OFF)
+		peer_device->connection->cstate[NEW] = C_CONNECTED;
+}
 
-			drbd_rs_cancel_all(device);
+struct change_repl_context {
+	struct change_context context;
+	struct drbd_peer_device *peer_device;
+};
 
-			/* In case we want to get something to stable storage still,
-			 * this may be the last chance.
-			 * Following put_ldev may transition to D_DISKLESS. */
-			drbd_md_sync(device);
-		}
-		put_ldev(device);
-	}
+static bool do_change_repl_state(struct change_context *context, enum change_phase phase)
+{
+	struct change_repl_context *repl_context =
+		container_of(context, struct change_repl_context, context);
+	struct drbd_peer_device *peer_device = repl_context->peer_device;
+	enum drbd_repl_state *repl_state = peer_device->repl_state;
+	enum drbd_repl_state new_repl_state = context->val.conn;
+	bool cluster_wide = context->flags & CS_CLUSTER_WIDE;
+
+	__change_repl_state(peer_device, new_repl_state);
+
+	return phase != PH_PREPARE ||
+		((repl_state[NOW] >= L_ESTABLISHED &&
+		  (new_repl_state == L_STARTING_SYNC_S || new_repl_state == L_STARTING_SYNC_T)) ||
+		 (repl_state[NOW] == L_ESTABLISHED &&
+		  (new_repl_state == L_VERIFY_S || new_repl_state == L_OFF)) ||
+		 (repl_state[NOW] == L_ESTABLISHED && cluster_wide &&
+		  (new_repl_state == L_WF_BITMAP_S || new_repl_state == L_WF_BITMAP_T)));
+}
 
-	/* second half of local IO error, failure to attach,
-	 * or administrative detach,
-	 * after local_cnt references have reached zero again */
-	if (os.disk != D_DISKLESS && ns.disk == D_DISKLESS) {
-		/* We must still be diskless,
-		 * re-attach has to be serialized with this! */
-		if (device->state.disk != D_DISKLESS)
-			drbd_err(device,
-				 "ASSERT FAILED: disk is %s while going diskless\n",
-				 drbd_disk_str(device->state.disk));
-
-		if (ns.conn >= C_CONNECTED)
-			drbd_send_state(peer_device, ns);
-		/* corresponding get_ldev in __drbd_set_state
-		 * this may finally trigger drbd_ldev_destroy. */
-		put_ldev(device);
-	}
+enum drbd_state_rv change_repl_state(struct drbd_peer_device *peer_device,
+				     enum drbd_repl_state new_repl_state,
+				     enum chg_state_flags flags,
+				     const char *tag)
+{
+	struct change_repl_context repl_context = {
+		.context = {
+			.resource = peer_device->device->resource,
+			.vnr = peer_device->device->vnr,
+			.mask = { { .conn = conn_MASK } },
+			.val = { { .conn = new_repl_state } },
+			.target_node_id = peer_device->node_id,
+			.flags = flags
+		},
+		.peer_device = peer_device
+	};
 
-	/* Notify peer that I had a local IO error, and did not detached.. */
-	if (os.disk == D_UP_TO_DATE && ns.disk == D_INCONSISTENT && ns.conn >= C_CONNECTED)
-		drbd_send_state(peer_device, ns);
-
-	/* Disks got bigger while they were detached */
-	if (ns.disk > D_NEGOTIATING && ns.pdsk > D_NEGOTIATING &&
-	    test_and_clear_bit(RESYNC_AFTER_NEG, &device->flags)) {
-		if (ns.conn == C_CONNECTED)
-			resync_after_online_grow(device);
-	}
-
-	/* A resync finished or aborted, wake paused devices... */
-	if ((os.conn > C_CONNECTED && ns.conn <= C_CONNECTED) ||
-	    (os.peer_isp && !ns.peer_isp) ||
-	    (os.user_isp && !ns.user_isp))
-		resume_next_sg(device);
-
-	/* sync target done with resync.  Explicitly notify peer, even though
-	 * it should (at least for non-empty resyncs) already know itself. */
-	if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED)
-		drbd_send_state(peer_device, ns);
-
-	/* Verify finished, or reached stop sector.  Peer did not know about
-	 * the stop sector, and we may even have changed the stop sector during
-	 * verify to interrupt/stop early.  Send the new state. */
-	if (os.conn == C_VERIFY_S && ns.conn == C_CONNECTED
-	&& verify_can_do_stop_sector(device))
-		drbd_send_state(peer_device, ns);
-
-	/* This triggers bitmap writeout of potentially still unwritten pages
-	 * if the resync finished cleanly, or aborted because of peer disk
-	 * failure, or on transition from resync back to AHEAD/BEHIND.
-	 *
-	 * Connection loss is handled in drbd_disconnected() by the receiver.
-	 *
-	 * For resync aborted because of local disk failure, we cannot do
-	 * any bitmap writeout anymore.
-	 *
-	 * No harm done if some bits change during this phase.
-	 */
-	if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) &&
-	    (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) {
-		drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL,
-			"write from resync_finished", BM_LOCKED_CHANGE_ALLOWED,
-			peer_device);
-		put_ldev(device);
-	}
+	if (new_repl_state == L_WF_BITMAP_S || new_repl_state == L_VERIFY_S)
+		repl_context.context.change_local_state_last = true;
 
-	if (ns.disk == D_DISKLESS &&
-	    ns.conn == C_STANDALONE &&
-	    ns.role == R_SECONDARY) {
-		if (os.aftr_isp != ns.aftr_isp)
-			resume_next_sg(device);
-	}
+	return change_cluster_wide_state(do_change_repl_state, &repl_context.context, tag);
+}
 
-	drbd_md_sync(device);
+enum drbd_state_rv stable_change_repl_state(struct drbd_peer_device *peer_device,
+					    enum drbd_repl_state repl_state,
+					    enum chg_state_flags flags,
+					    const char *tag)
+{
+	return stable_state_change(peer_device->device->resource,
+		change_repl_state(peer_device, repl_state, flags, tag));
 }
 
-struct after_conn_state_chg_work {
-	struct drbd_work w;
-	enum drbd_conns oc;
-	union drbd_state ns_min;
-	union drbd_state ns_max; /* new, max state, over all devices */
-	enum chg_state_flags flags;
-	struct drbd_connection *connection;
-	struct drbd_state_change *state_change;
-};
+void __change_peer_disk_state(struct drbd_peer_device *peer_device, enum drbd_disk_state disk_state)
+{
+	peer_device->disk_state[NEW] = disk_state;
+}
 
-static int w_after_conn_state_ch(struct drbd_work *w, int unused)
+void __downgrade_peer_disk_states(struct drbd_connection *connection, enum drbd_disk_state disk_state)
 {
-	struct after_conn_state_chg_work *acscw =
-		container_of(w, struct after_conn_state_chg_work, w);
-	struct drbd_connection *connection = acscw->connection;
-	enum drbd_conns oc = acscw->oc;
-	union drbd_state ns_max = acscw->ns_max;
 	struct drbd_peer_device *peer_device;
 	int vnr;
 
-	broadcast_state_change(acscw->state_change);
-	forget_state_change(acscw->state_change);
-	kfree(acscw);
+	rcu_read_lock();
+	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
+		if (peer_device->disk_state[NEW] > disk_state)
+			__change_peer_disk_state(peer_device, disk_state);
+	}
+	rcu_read_unlock();
+}
 
-	/* Upon network configuration, we need to start the receiver */
-	if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED)
-		drbd_thread_start(&connection->receiver);
+enum drbd_state_rv change_peer_disk_state(struct drbd_peer_device *peer_device,
+					  enum drbd_disk_state disk_state,
+					  enum chg_state_flags flags,
+					  const char *tag)
+{
+	struct drbd_resource *resource = peer_device->device->resource;
+	unsigned long irq_flags;
 
-	if (oc == C_DISCONNECTING && ns_max.conn == C_STANDALONE) {
-		struct net_conf *old_conf;
+	begin_state_change(resource, &irq_flags, flags);
+	__change_peer_disk_state(peer_device, disk_state);
+	return end_state_change(resource, &irq_flags, tag);
+}
 
-		mutex_lock(&notification_mutex);
-		idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
-			notify_peer_device_state(NULL, 0, peer_device, NULL,
-						 NOTIFY_DESTROY | NOTIFY_CONTINUES);
-		notify_connection_state(NULL, 0, connection, NULL, NOTIFY_DESTROY);
-		mutex_unlock(&notification_mutex);
+void __change_resync_susp_user(struct drbd_peer_device *peer_device,
+				       bool value)
+{
+	peer_device->resync_susp_user[NEW] = value;
+}
 
-		mutex_lock(&connection->resource->conf_update);
-		old_conf = connection->net_conf;
-		connection->my_addr_len = 0;
-		connection->peer_addr_len = 0;
-		RCU_INIT_POINTER(connection->net_conf, NULL);
-		conn_free_crypto(connection);
-		mutex_unlock(&connection->resource->conf_update);
+enum drbd_state_rv change_resync_susp_user(struct drbd_peer_device *peer_device,
+						   bool value,
+						   enum chg_state_flags flags)
+{
+	struct drbd_resource *resource = peer_device->device->resource;
+	unsigned long irq_flags;
 
-		kvfree_rcu_mightsleep(old_conf);
-	}
+	begin_state_change(resource, &irq_flags, flags);
+	__change_resync_susp_user(peer_device, value);
+	return end_state_change(resource, &irq_flags, value ? "pause-sync" : "resume-sync");
+}
 
-	if (ns_max.susp_fen) {
-		/* case1: The outdate peer handler is successful: */
-		if (ns_max.pdsk <= D_OUTDATED) {
-			rcu_read_lock();
-			idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-				struct drbd_device *device = peer_device->device;
-				if (test_bit(NEW_CUR_UUID, &device->flags)) {
-					drbd_uuid_new_current(device);
-					clear_bit(NEW_CUR_UUID, &device->flags);
-				}
-			}
-			rcu_read_unlock();
-			spin_lock_irq(&connection->resource->req_lock);
-			_tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
-			_conn_request_state(connection,
-					    (union drbd_state) { { .susp_fen = 1 } },
-					    (union drbd_state) { { .susp_fen = 0 } },
-					    CS_VERBOSE);
-			spin_unlock_irq(&connection->resource->req_lock);
-		}
-	}
-	conn_md_sync(connection);
-	kref_put(&connection->kref, drbd_destroy_connection);
+void __change_resync_susp_peer(struct drbd_peer_device *peer_device,
+				       bool value)
+{
+	peer_device->resync_susp_peer[NEW] = value;
+}
 
-	return 0;
+void __change_resync_susp_dependency(struct drbd_peer_device *peer_device,
+					     bool value)
+{
+	peer_device->resync_susp_dependency[NEW] = value;
 }
 
-static void conn_old_common_state(struct drbd_connection *connection, union drbd_state *pcs, enum chg_state_flags *pf)
+static void log_current_uuids(struct drbd_device *device)
 {
-	enum chg_state_flags flags = ~0;
 	struct drbd_peer_device *peer_device;
-	int vnr, first_vol = 1;
-	union drbd_dev_state os, cs = {
-		{ .role = R_SECONDARY,
-		  .peer = R_UNKNOWN,
-		  .conn = connection->cstate,
-		  .disk = D_DISKLESS,
-		  .pdsk = D_UNKNOWN,
-		} };
+	struct drbd_connection *connection;
+	char msg[120];
+	int ret, pos = 0;
 
 	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		os = device->state;
-
-		if (first_vol) {
-			cs = os;
-			first_vol = 0;
+	for_each_peer_device_rcu(peer_device, device) {
+		if (peer_device->disk_state[NOW] != D_UP_TO_DATE)
 			continue;
+		connection = peer_device->connection;
+		ret = snprintf(msg + pos, 120 - pos, "%s: %016llX ",
+			       rcu_dereference(connection->transport.net_conf)->name,
+			       peer_device->current_uuid);
+		if (ret > 0)
+			pos += ret;
+		if (pos >= 120)
+			break;
+	}
+	rcu_read_unlock();
+	drbd_warn(device, "%s", msg);
+}
+
+bool drbd_res_data_accessible(struct drbd_resource *resource)
+{
+	bool data_accessible = false;
+	struct drbd_device *device;
+	int vnr;
+
+	idr_for_each_entry(&resource->devices, device, vnr) {
+		if (drbd_data_accessible(device, NOW)) {
+			data_accessible = true;
+			break;
 		}
+	}
 
-		if (cs.role != os.role)
-			flags &= ~CS_DC_ROLE;
+	return data_accessible;
+}
 
-		if (cs.peer != os.peer)
-			flags &= ~CS_DC_PEER;
+/**
+ * calc_data_accessible() - returns if up-to-data data is reachable
+ *
+ * @state_change: where to get the state information from
+ * @n_device:     index into the devices array
+ * @which:        OLD or NEW
+ *
+ * calc_data_accessible() returns true if either the local disk is up-to-date
+ * or of the peers. The related drbd_data_accessible() computes the same
+ * result from different inputs.
+ */
+static bool calc_data_accessible(struct drbd_state_change *state_change, int n_device,
+				 enum which_state which)
+{
+	struct drbd_device_state_change *device_state_change = &state_change->devices[n_device];
+	enum drbd_disk_state *disk_state = device_state_change->disk_state;
+	int n_connection;
 
-		if (cs.conn != os.conn)
-			flags &= ~CS_DC_CONN;
+	if (disk_state[which] == D_UP_TO_DATE)
+		return true;
 
-		if (cs.disk != os.disk)
-			flags &= ~CS_DC_DISK;
+	for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) {
+		struct drbd_peer_device_state_change *peer_device_state_change =
+			&state_change->peer_devices[
+				n_device * state_change->n_connections + n_connection];
+		struct drbd_peer_device *peer_device = peer_device_state_change->peer_device;
+		enum drbd_disk_state *peer_disk_state = peer_device_state_change->disk_state;
+		struct net_conf *nc;
+		bool allow_remote_read;
 
-		if (cs.pdsk != os.pdsk)
-			flags &= ~CS_DC_PDSK;
+		rcu_read_lock();
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		allow_remote_read = nc->allow_remote_read;
+		rcu_read_unlock();
+		if (nc && !allow_remote_read)
+			continue;
+		if (peer_disk_state[which] == D_UP_TO_DATE)
+			return true;
 	}
-	rcu_read_unlock();
 
-	*pf |= CS_DC_MASK;
-	*pf &= flags;
-	(*pcs).i = cs.i;
+	return false;
 }
 
-static enum drbd_state_rv
-conn_is_valid_transition(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-			 enum chg_state_flags flags)
+/**
+ * drbd_data_accessible() - returns if up-to-data data is reachable
+ *
+ * @device: the device, the question is about
+ * @which:  OLD, NEW, or NOW (Only use OLD within a state change!)
+ *
+ * drbd_data_accessible() returns true if either the local disk is up-to-date
+ * or of the peers. The related calc_data_accessible() computes the same
+ * result from different inputs.
+ */
+bool drbd_data_accessible(struct drbd_device *device, enum which_state which)
 {
-	enum drbd_state_rv rv = SS_SUCCESS;
-	union drbd_state ns, os;
 	struct drbd_peer_device *peer_device;
-	int vnr;
+	bool data_accessible = false;
 
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		os = drbd_read_state(device);
-		ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL);
-
-		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
-			ns.disk = os.disk;
+	if (device->disk_state[which] == D_UP_TO_DATE)
+		return true;
 
-		if (ns.i == os.i)
+	rcu_read_lock();
+	for_each_peer_device_rcu(peer_device, device) {
+		struct net_conf *nc;
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		if (nc && !nc->allow_remote_read)
 			continue;
-
-		rv = is_valid_transition(os, ns);
-
-		if (rv >= SS_SUCCESS && !(flags & CS_HARD)) {
-			rv = is_valid_state(device, ns);
-			if (rv < SS_SUCCESS) {
-				if (is_valid_state(device, os) == rv)
-					rv = is_valid_soft_transition(os, ns, connection);
-			} else
-				rv = is_valid_soft_transition(os, ns, connection);
-		}
-
-		if (rv < SS_SUCCESS) {
-			if (flags & CS_VERBOSE)
-				print_st_err(device, os, ns, rv);
+		if (peer_device->disk_state[which] == D_UP_TO_DATE) {
+			data_accessible = true;
 			break;
 		}
 	}
 	rcu_read_unlock();
 
-	return rv;
+	return data_accessible;
 }
-
-static void
-conn_set_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-	       union drbd_state *pns_min, union drbd_state *pns_max, enum chg_state_flags flags)
+/* drbd_data_accessible() and exposable_data_uuid() have the same structure. By intention. */
+static u64 exposable_data_uuid(struct drbd_device *device)
 {
-	union drbd_state ns, os, ns_max = { };
-	union drbd_state ns_min = {
-		{ .role = R_MASK,
-		  .peer = R_MASK,
-		  .conn = val.conn,
-		  .disk = D_MASK,
-		  .pdsk = D_MASK
-		} };
 	struct drbd_peer_device *peer_device;
-	enum drbd_state_rv rv;
-	int vnr, number_of_volumes = 0;
-
-	if (mask.conn == C_MASK) {
-		/* remember last connect time so request_timer_fn() won't
-		 * kill newly established sessions while we are still trying to thaw
-		 * previously frozen IO */
-		if (connection->cstate != C_WF_REPORT_PARAMS && val.conn == C_WF_REPORT_PARAMS)
-			connection->last_reconnect_jif = jiffies;
+	u64 uuid = 0;
 
-		connection->cstate = val.conn;
+	if (get_ldev_if_state(device, D_UP_TO_DATE)) {
+		uuid = device->ldev->md.current_uuid;
+		put_ldev(device);
+		return uuid;
 	}
 
 	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		number_of_volumes++;
-		os = drbd_read_state(device);
-		ns = apply_mask_val(os, mask, val);
-		ns = sanitize_state(device, os, ns, NULL);
-
-		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
-			ns.disk = os.disk;
-
-		rv = _drbd_set_state(device, ns, flags, NULL);
-		BUG_ON(rv < SS_SUCCESS);
-		ns.i = device->state.i;
-		ns_max.role = max_role(ns.role, ns_max.role);
-		ns_max.peer = max_role(ns.peer, ns_max.peer);
-		ns_max.conn = max_t(enum drbd_conns, ns.conn, ns_max.conn);
-		ns_max.disk = max_t(enum drbd_disk_state, ns.disk, ns_max.disk);
-		ns_max.pdsk = max_t(enum drbd_disk_state, ns.pdsk, ns_max.pdsk);
-
-		ns_min.role = min_role(ns.role, ns_min.role);
-		ns_min.peer = min_role(ns.peer, ns_min.peer);
-		ns_min.conn = min_t(enum drbd_conns, ns.conn, ns_min.conn);
-		ns_min.disk = min_t(enum drbd_disk_state, ns.disk, ns_min.disk);
-		ns_min.pdsk = min_t(enum drbd_disk_state, ns.pdsk, ns_min.pdsk);
+	for_each_peer_device_rcu(peer_device, device) {
+		struct net_conf *nc;
+		nc = rcu_dereference(peer_device->connection->transport.net_conf);
+		if (nc && !nc->allow_remote_read)
+			continue;
+		if (peer_device->disk_state[NOW] == D_UP_TO_DATE &&
+		    (uuid & ~UUID_PRIMARY) != (peer_device->current_uuid & ~UUID_PRIMARY)) {
+			if (!uuid) {
+				uuid = peer_device->current_uuid;
+				continue;
+			}
+			drbd_err(device, "Multiple UpToDate peers have different current UUIDs\n");
+			log_current_uuids(device);
+		}
 	}
 	rcu_read_unlock();
 
-	if (number_of_volumes == 0) {
-		ns_min = ns_max = (union drbd_state) { {
-				.role = R_SECONDARY,
-				.peer = R_UNKNOWN,
-				.conn = val.conn,
-				.disk = D_DISKLESS,
-				.pdsk = D_UNKNOWN
-			} };
-	}
-
-	ns_min.susp = ns_max.susp = connection->resource->susp;
-	ns_min.susp_nod = ns_max.susp_nod = connection->resource->susp_nod;
-	ns_min.susp_fen = ns_max.susp_fen = connection->resource->susp_fen;
-
-	*pns_min = ns_min;
-	*pns_max = ns_max;
+	return uuid;
 }
 
-static enum drbd_state_rv
-_conn_rq_cond(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
+static void ensure_exposed_data_uuid(struct drbd_device *device)
 {
-	enum drbd_state_rv err, rv = SS_UNKNOWN_ERROR; /* continue waiting */;
-
-	if (test_and_clear_bit(CONN_WD_ST_CHG_OKAY, &connection->flags))
-		rv = SS_CW_SUCCESS;
-
-	if (test_and_clear_bit(CONN_WD_ST_CHG_FAIL, &connection->flags))
-		rv = SS_CW_FAILED_BY_PEER;
+	u64 uuid = exposable_data_uuid(device);
 
-	err = conn_is_valid_transition(connection, mask, val, 0);
-	if (err == SS_SUCCESS && connection->cstate == C_WF_REPORT_PARAMS)
-		return rv;
+	if (uuid)
+		drbd_uuid_set_exposed(device, uuid, true);
 
-	return err;
 }
 
-enum drbd_state_rv
-_conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-		    enum chg_state_flags flags)
+/* Between 9.1.7 and 9.1.12 drbd was setting MDF_NODE_EXISTS for all peers.
+ * With that the flag got useless. It is a meta-data flag that persists.
+ * Clear it for all not configured nodes if we find it in every peer slot.
+ */
+static void check_wrongly_set_mdf_exists(struct drbd_device *device)
 {
-	enum drbd_state_rv rv = SS_SUCCESS;
-	struct after_conn_state_chg_work *acscw;
-	enum drbd_conns oc = connection->cstate;
-	union drbd_state ns_max, ns_min, os;
-	bool have_mutex = false;
-	struct drbd_state_change *state_change;
+	struct drbd_resource *resource = device->resource;
+	const int my_node_id = resource->res_opts.node_id;
+	bool wrong = true;
+	int node_id;
 
-	if (mask.conn) {
-		rv = is_valid_conn_transition(oc, val.conn);
-		if (rv < SS_SUCCESS)
-			goto abort;
-	}
+	if (!get_ldev(device))
+		return;
 
-	rv = conn_is_valid_transition(connection, mask, val, flags);
-	if (rv < SS_SUCCESS)
-		goto abort;
-
-	if (oc == C_WF_REPORT_PARAMS && val.conn == C_DISCONNECTING &&
-	    !(flags & (CS_LOCAL_ONLY | CS_HARD))) {
-
-		/* This will be a cluster-wide state change.
-		 * Need to give up the spinlock, grab the mutex,
-		 * then send the state change request, ... */
-		spin_unlock_irq(&connection->resource->req_lock);
-		mutex_lock(&connection->cstate_mutex);
-		have_mutex = true;
-
-		set_bit(CONN_WD_ST_CHG_REQ, &connection->flags);
-		if (conn_send_state_req(connection, mask, val)) {
-			/* sending failed. */
-			clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags);
-			rv = SS_CW_FAILED_BY_PEER;
-			/* need to re-aquire the spin lock, though */
-			goto abort_unlocked;
-		}
-
-		if (val.conn == C_DISCONNECTING)
-			set_bit(DISCONNECT_SENT, &connection->flags);
-
-		/* ... and re-aquire the spinlock.
-		 * If _conn_rq_cond() returned >= SS_SUCCESS, we must call
-		 * conn_set_state() within the same spinlock. */
-		spin_lock_irq(&connection->resource->req_lock);
-		wait_event_lock_irq(connection->ping_wait,
-				(rv = _conn_rq_cond(connection, mask, val)),
-				connection->resource->req_lock);
-		clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags);
-		if (rv < SS_SUCCESS)
-			goto abort;
-	}
-
-	state_change = remember_old_state(connection->resource, GFP_ATOMIC);
-	conn_old_common_state(connection, &os, &flags);
-	flags |= CS_DC_SUSP;
-	conn_set_state(connection, mask, val, &ns_min, &ns_max, flags);
-	conn_pr_state_change(connection, os, ns_max, flags);
-	remember_new_state(state_change);
-
-	acscw = kmalloc_obj(*acscw, GFP_ATOMIC);
-	if (acscw) {
-		acscw->oc = os.conn;
-		acscw->ns_min = ns_min;
-		acscw->ns_max = ns_max;
-		acscw->flags = flags;
-		acscw->w.cb = w_after_conn_state_ch;
-		kref_get(&connection->kref);
-		acscw->connection = connection;
-		acscw->state_change = state_change;
-		drbd_queue_work(&connection->sender_work, &acscw->w);
-	} else {
-		drbd_err(connection, "Could not kmalloc an acscw\n");
-	}
+	rcu_read_lock();
 
- abort:
-	if (have_mutex) {
-		/* mutex_unlock() "... must not be used in interrupt context.",
-		 * so give up the spinlock, then re-aquire it */
-		spin_unlock_irq(&connection->resource->req_lock);
- abort_unlocked:
-		mutex_unlock(&connection->cstate_mutex);
-		spin_lock_irq(&connection->resource->req_lock);
-	}
-	if (rv < SS_SUCCESS && flags & CS_VERBOSE) {
-		drbd_err(connection, "State change failed: %s\n", drbd_set_st_err_str(rv));
-		drbd_err(connection, " mask = 0x%x val = 0x%x\n", mask.i, val.i);
-		drbd_err(connection, " old_conn:%s wanted_conn:%s\n", drbd_conn_str(oc), drbd_conn_str(val.conn));
-	}
-	return rv;
-}
+	for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+		struct drbd_peer_device *peer_device = peer_device_by_node_id(device, node_id);
+		struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
 
-enum drbd_state_rv
-conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-		   enum chg_state_flags flags)
-{
-	enum drbd_state_rv rv;
+		if (!(peer_md->flags & MDF_NODE_EXISTS || peer_device || node_id == my_node_id)) {
+			wrong = false;
+			break;
+		}
+	}
 
-	spin_lock_irq(&connection->resource->req_lock);
-	rv = _conn_request_state(connection, mask, val, flags);
-	spin_unlock_irq(&connection->resource->req_lock);
+	if (wrong) {
+		for (node_id = 0; node_id < DRBD_NODE_ID_MAX; node_id++) {
+			struct drbd_peer_device *peer_device = peer_device_by_node_id(device, node_id);
+			struct drbd_peer_md *peer_md = &device->ldev->md.peers[node_id];
 
-	return rv;
+			if (!peer_device)
+				peer_md->flags &= ~MDF_NODE_EXISTS;
+		}
+		if (!test_bit(WRONG_MDF_EXISTS, &resource->flags)) {
+			set_bit(WRONG_MDF_EXISTS, &resource->flags);
+			drbd_warn(resource, "Clearing excess MDF_NODE_EXISTS flags\n");
+		}
+	}
+	rcu_read_unlock();
+	put_ldev(device);
 }
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index cbaeb8018dbf5..7535fd7c6037f 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -2,26 +2,19 @@
 #ifndef DRBD_STATE_H
 #define DRBD_STATE_H
 
+#include <linux/drbd_protocol.h>
+
+struct drbd_resource;
 struct drbd_device;
 struct drbd_connection;
+struct drbd_peer_device;
+struct drbd_work;
+struct twopc_request;
 
 /**
  * DOC: DRBD State macros
  *
  * These macros are used to express state changes in easily readable form.
- *
- * The NS macros expand to a mask and a value, that can be bit ored onto the
- * current state as soon as the spinlock (req_lock) was taken.
- *
- * The _NS macros are used for state functions that get called with the
- * spinlock. These macros expand directly to the new state value.
- *
- * Besides the basic forms NS() and _NS() additional _?NS[23] are defined
- * to express state changes that affect more than one aspect of the state.
- *
- * E.g. NS2(conn, C_CONNECTED, peer, R_SECONDARY)
- * Means that the network connection was established and that the peer
- * is in secondary role.
  */
 #define role_MASK R_MASK
 #define peer_MASK R_MASK
@@ -34,141 +27,168 @@ struct drbd_connection;
 #define susp_nod_MASK 1
 #define susp_fen_MASK 1
 
-#define NS(T, S) \
-	({ union drbd_state mask; mask.i = 0; mask.T = T##_MASK; mask; }), \
-	({ union drbd_state val; val.i = 0; val.T = (S); val; })
-#define NS2(T1, S1, T2, S2) \
-	({ union drbd_state mask; mask.i = 0; mask.T1 = T1##_MASK; \
-	  mask.T2 = T2##_MASK; mask; }), \
-	({ union drbd_state val; val.i = 0; val.T1 = (S1); \
-	  val.T2 = (S2); val; })
-#define NS3(T1, S1, T2, S2, T3, S3) \
-	({ union drbd_state mask; mask.i = 0; mask.T1 = T1##_MASK; \
-	  mask.T2 = T2##_MASK; mask.T3 = T3##_MASK; mask; }), \
-	({ union drbd_state val;  val.i = 0; val.T1 = (S1); \
-	  val.T2 = (S2); val.T3 = (S3); val; })
-
-#define _NS(D, T, S) \
-	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T = (S); __ns; })
-#define _NS2(D, T1, S1, T2, S2) \
-	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T1 = (S1); \
-	__ns.T2 = (S2); __ns; })
-#define _NS3(D, T1, S1, T2, S2, T3, S3) \
-	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T1 = (S1); \
-	__ns.T2 = (S2); __ns.T3 = (S3); __ns; })
-
 enum chg_state_flags {
-	CS_HARD	         = 1 << 0,
+	CS_HARD          = 1 << 0, /* Forced state change, such as a connection loss */
 	CS_VERBOSE       = 1 << 1,
 	CS_WAIT_COMPLETE = 1 << 2,
 	CS_SERIALIZE     = 1 << 3,
-	CS_ORDERED       = CS_WAIT_COMPLETE + CS_SERIALIZE,
-	CS_LOCAL_ONLY    = 1 << 4, /* Do not consider a device pair wide state change */
-	CS_DC_ROLE       = 1 << 5, /* DC = display as connection state change */
-	CS_DC_PEER       = 1 << 6,
-	CS_DC_CONN       = 1 << 7,
-	CS_DC_DISK       = 1 << 8,
-	CS_DC_PDSK       = 1 << 9,
-	CS_DC_SUSP       = 1 << 10,
-	CS_DC_MASK       = CS_DC_ROLE + CS_DC_PEER + CS_DC_CONN + CS_DC_DISK + CS_DC_PDSK,
-	CS_IGN_OUTD_FAIL = 1 << 11,
-
-	/* Make sure no meta data IO is in flight, by calling
-	 * drbd_md_get_buffer().  Used for graceful detach. */
-	CS_INHIBIT_MD_IO = 1 << 12,
+	CS_ALREADY_SERIALIZED = 1 << 4, /* resource->state_sem already taken */
+	CS_LOCAL_ONLY    = 1 << 5, /* Do not consider a cluster-wide state change */
+	CS_PREPARE	 = 1 << 6,
+	CS_PREPARED	 = 1 << 7,
+	CS_ABORT	 = 1 << 8,
+	CS_TWOPC	 = 1 << 9,
+	CS_IGN_OUTD_FAIL = 1 << 10,
+	CS_DONT_RETRY    = 1 << 11, /* Disable internal retry. Caller has a retry loop */
+	CS_FORCE_RECALC  = 1 << 13, /* Force re-evaluation of state logic */
+	CS_CLUSTER_WIDE  = 1 << 14, /* Make this a cluster wide state change! */
+	CS_FP_LOCAL_UP_TO_DATE = 1 << 15, /* force promotion by making local disk state up_to_date */
+	CS_FP_OUTDATE_PEERS = 1 << 16, /* force promotion by marking unknown peers as outdated */
+	CS_FS_IGN_OPENERS = 1 << 17, /* force demote, ignore openers */
 };
 
-/* drbd_dev_state and drbd_state are different types. This is to stress the
-   small difference. There is no suspended flag (.susp), and no suspended
-   while fence handler runs flas (susp_fen). */
-union drbd_dev_state {
-	struct {
-#if defined(__LITTLE_ENDIAN_BITFIELD)
-		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned conn:5 ;   /* 17/32	 cstates */
-		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned _unused:1 ;
-		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
-		unsigned peer_isp:1 ;
-		unsigned user_isp:1 ;
-		unsigned _pad:11;   /* 0	 unused */
-#elif defined(__BIG_ENDIAN_BITFIELD)
-		unsigned _pad:11;
-		unsigned user_isp:1 ;
-		unsigned peer_isp:1 ;
-		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
-		unsigned _unused:1 ;
-		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned conn:5 ;   /* 17/32	 cstates */
-		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
-#else
-# error "this endianess is not supported"
-#endif
-	};
-	unsigned int i;
-};
+void drbd_resume_al(struct drbd_device *device);
 
-extern enum drbd_state_rv drbd_change_state(struct drbd_device *device,
-					    enum chg_state_flags f,
-					    union drbd_state mask,
-					    union drbd_state val);
-extern void drbd_force_state(struct drbd_device *, union drbd_state,
-			union drbd_state);
-extern enum drbd_state_rv _drbd_request_state(struct drbd_device *,
-					      union drbd_state,
-					      union drbd_state,
-					      enum chg_state_flags);
-
-extern enum drbd_state_rv
-_drbd_request_state_holding_state_mutex(struct drbd_device *, union drbd_state,
-					union drbd_state, enum chg_state_flags);
-
-extern enum drbd_state_rv _drbd_set_state(struct drbd_device *, union drbd_state,
-					  enum chg_state_flags,
-					  struct completion *done);
-extern void print_st_err(struct drbd_device *, union drbd_state,
-			union drbd_state, enum drbd_state_rv);
-
-enum drbd_state_rv
-_conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-		    enum chg_state_flags flags);
-
-enum drbd_state_rv
-conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val,
-		   enum chg_state_flags flags);
-
-extern void drbd_resume_al(struct drbd_device *device);
-extern bool conn_all_vols_unconf(struct drbd_connection *connection);
+enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection);
+enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection);
 
-/**
- * drbd_request_state() - Request a state change
- * @device:	DRBD device.
- * @mask:	mask of state bits to change.
- * @val:	value of new state bits.
- *
- * This is the most graceful way of requesting a state change. It is verbose
- * quite verbose in case the state change is not possible, and all those
- * state changes are globally serialized.
- */
-static inline int drbd_request_state(struct drbd_device *device,
-				     union drbd_state mask,
-				     union drbd_state val)
+void state_change_lock(struct drbd_resource *resource,
+		       unsigned long *irq_flags, enum chg_state_flags flags);
+void state_change_unlock(struct drbd_resource *resource,
+			 unsigned long *irq_flags);
+
+void begin_state_change(struct drbd_resource *resource,
+			unsigned long *irq_flags, enum chg_state_flags flags);
+enum drbd_state_rv end_state_change(struct drbd_resource *resource,
+				    unsigned long *irq_flags, const char *tag);
+void abort_state_change(struct drbd_resource *resource,
+			unsigned long *irq_flags);
+void abort_state_change_locked(struct drbd_resource *resource);
+
+void begin_state_change_locked(struct drbd_resource *resource,
+			       enum chg_state_flags flags);
+enum drbd_state_rv end_state_change_locked(struct drbd_resource *resource,
+					   const char *tag);
+
+void clear_remote_state_change(struct drbd_resource *resource);
+void __clear_remote_state_change(struct drbd_resource *resource);
+
+
+enum which_state;
+bool drbd_all_peer_replication(struct drbd_device *device, enum which_state which);
+union drbd_state drbd_get_device_state(struct drbd_device *device,
+				       enum which_state which);
+union drbd_state drbd_get_peer_device_state(struct drbd_peer_device *peer_device,
+					    enum which_state which);
+
+#define stable_state_change(resource, change_state) ({				\
+		enum drbd_state_rv rv;						\
+		int err;							\
+		err = wait_event_interruptible((resource)->state_wait,		\
+			(rv = (change_state)) != SS_IN_TRANSIENT_STATE);	\
+		if (err)							\
+			err = -SS_UNKNOWN_ERROR;				\
+		else								\
+			err = rv;						\
+		err;								\
+	})
+
+void nested_twopc_work(struct work_struct *work);
+void drbd_maybe_cluster_wide_reply(struct drbd_resource *resource);
+enum drbd_state_rv nested_twopc_request(struct drbd_resource *resource,
+					struct twopc_request *request);
+bool drbd_twopc_between_peer_and_me(struct drbd_connection *connection);
+bool cluster_wide_reply_ready(struct drbd_resource *resource);
+
+enum drbd_state_rv change_role(struct drbd_resource *resource,
+			       enum drbd_role role,
+			       enum chg_state_flags flags, const char *tag,
+			       const char **err_str);
+
+void __change_io_susp_user(struct drbd_resource *resource, bool value);
+enum drbd_state_rv change_io_susp_user(struct drbd_resource *resource,
+				       bool value, enum chg_state_flags flags);
+void __change_io_susp_no_data(struct drbd_resource *resource, bool value);
+void __change_io_susp_fencing(struct drbd_connection *connection, bool value);
+void __change_io_susp_quorum(struct drbd_resource *resource, bool value);
+
+void __change_disk_state(struct drbd_device *device,
+			 enum drbd_disk_state disk_state);
+void __downgrade_disk_states(struct drbd_resource *resource,
+			     enum drbd_disk_state disk_state);
+enum drbd_state_rv change_disk_state(struct drbd_device *device,
+				     enum drbd_disk_state disk_state,
+				     enum chg_state_flags flags,
+				     const char *tag, const char **err_str);
+
+void __change_cstate(struct drbd_connection *connection,
+		     enum drbd_conn_state cstate);
+enum drbd_state_rv change_cstate_tag(struct drbd_connection *connection,
+				     enum drbd_conn_state cstate,
+				     enum chg_state_flags flags,
+				     const char *tag, const char **err_str);
+static inline enum drbd_state_rv change_cstate(struct drbd_connection *connection,
+					       enum drbd_conn_state cstate,
+					       enum chg_state_flags flags)
 {
-	return _drbd_request_state(device, mask, val, CS_VERBOSE + CS_ORDERED);
+	return change_cstate_tag(connection, cstate, flags, NULL, NULL);
 }
 
-/* for use in adm_detach() (drbd_adm_detach(), drbd_adm_down()) */
-int drbd_request_detach_interruptible(struct drbd_device *device);
-
-enum drbd_role conn_highest_role(struct drbd_connection *connection);
-enum drbd_role conn_highest_peer(struct drbd_connection *connection);
-enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection);
-enum drbd_disk_state conn_lowest_disk(struct drbd_connection *connection);
-enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection);
-enum drbd_conns conn_lowest_conn(struct drbd_connection *connection);
-
+void __change_peer_role(struct drbd_connection *connection,
+			enum drbd_role peer_role);
+
+void __change_repl_state(struct drbd_peer_device *peer_device,
+			 enum drbd_repl_state repl_state);
+enum drbd_state_rv change_repl_state(struct drbd_peer_device *peer_device,
+				     enum drbd_repl_state new_repl_state,
+				     enum chg_state_flags flags,
+				     const char *tag);
+enum drbd_state_rv stable_change_repl_state(struct drbd_peer_device *peer_device,
+					    enum drbd_repl_state repl_state,
+					    enum chg_state_flags flags,
+					    const char *tag);
+
+void __change_peer_disk_state(struct drbd_peer_device *peer_device,
+			      enum drbd_disk_state disk_state);
+void __downgrade_peer_disk_states(struct drbd_connection *connection,
+				  enum drbd_disk_state disk_state);
+void __outdate_myself(struct drbd_resource *resource);
+enum drbd_state_rv change_peer_disk_state(struct drbd_peer_device *peer_device,
+					  enum drbd_disk_state disk_state,
+					  enum chg_state_flags flags,
+					  const char *tag);
+
+void __change_resync_susp_user(struct drbd_peer_device *peer_device,
+			       bool value);
+enum drbd_state_rv change_resync_susp_user(struct drbd_peer_device *peer_device,
+					   bool value,
+					   enum chg_state_flags flags);
+void __change_resync_susp_peer(struct drbd_peer_device *peer_device,
+			       bool value);
+void __change_resync_susp_dependency(struct drbd_peer_device *peer_device,
+				     bool value);
+void apply_connect(struct drbd_connection *connection, bool commit);
+
+struct drbd_work;
+
+bool resource_is_suspended(struct drbd_resource *resource,
+			   enum which_state which);
+bool is_suspended_fen(struct drbd_resource *resource, enum which_state which);
+
+enum dds_flags;
+enum determine_dev_size;
+struct resize_parms;
+
+enum determine_dev_size
+change_cluster_wide_device_size(struct drbd_device *device,
+				sector_t local_max_size,
+				uint64_t new_user_size,
+				enum dds_flags dds_flags,
+				struct resize_parms *rs);
+
+bool drbd_data_accessible(struct drbd_device *device, enum which_state which);
+bool drbd_res_data_accessible(struct drbd_resource *resource);
+
+
+void drbd_empty_twopc_work_fn(struct work_struct *work);
 #endif
diff --git a/drivers/block/drbd/drbd_state_change.h b/drivers/block/drbd/drbd_state_change.h
index a56a57d676862..bb68684a5fd3c 100644
--- a/drivers/block/drbd/drbd_state_change.h
+++ b/drivers/block/drbd/drbd_state_change.h
@@ -7,58 +7,80 @@ struct drbd_resource_state_change {
 	enum drbd_role role[2];
 	bool susp[2];
 	bool susp_nod[2];
-	bool susp_fen[2];
+	bool susp_uuid[2];
+	bool fail_io[2];
 };
 
 struct drbd_device_state_change {
 	struct drbd_device *device;
 	enum drbd_disk_state disk_state[2];
+	bool have_quorum[2];
 };
 
 struct drbd_connection_state_change {
 	struct drbd_connection *connection;
-	enum drbd_conns cstate[2];  /* drbd9: enum drbd_conn_state */
+	enum drbd_conn_state cstate[2];
 	enum drbd_role peer_role[2];
+	bool susp_fen[2];
+};
+
+/* exception: stores state, not change.
+ * for get_initial_state. */
+struct drbd_path_state {
+	struct drbd_connection *connection;
+	struct drbd_path *path;
+	/* not an array,
+	 * because it's not an array in struct drbd_path either */
+	bool path_established;
 };
 
 struct drbd_peer_device_state_change {
 	struct drbd_peer_device *peer_device;
 	enum drbd_disk_state disk_state[2];
-	enum drbd_conns repl_state[2];  /* drbd9: enum drbd_repl_state */
+	enum drbd_repl_state repl_state[2];
 	bool resync_susp_user[2];
 	bool resync_susp_peer[2];
 	bool resync_susp_dependency[2];
+	bool resync_susp_other_c[2];
+	bool resync_active[2];
+	bool replication[2];
+	bool peer_replication[2];
+};
+
+struct drbd_state_change_object_count {
+	unsigned int n_devices;
+	unsigned int n_connections;
+	unsigned int n_paths;
 };
 
 struct drbd_state_change {
 	struct list_head list;
 	unsigned int n_devices;
 	unsigned int n_connections;
+	unsigned int n_paths;
 	struct drbd_resource_state_change resource[1];
 	struct drbd_device_state_change *devices;
 	struct drbd_connection_state_change *connections;
 	struct drbd_peer_device_state_change *peer_devices;
+	struct drbd_path_state *paths;
 };
 
-extern struct drbd_state_change *remember_old_state(struct drbd_resource *, gfp_t);
-extern void copy_old_to_new_state_change(struct drbd_state_change *);
-extern void forget_state_change(struct drbd_state_change *);
+struct drbd_state_change *remember_state_change(struct drbd_resource *resource,
+						gfp_t gfp);
+void copy_old_to_new_state_change(struct drbd_state_change *state_change);
+void forget_state_change(struct drbd_state_change *state_change);
 
-extern int notify_resource_state_change(struct sk_buff *,
-					 unsigned int,
-					 void *,
-					 enum drbd_notification_type type);
-extern int notify_connection_state_change(struct sk_buff *,
-					   unsigned int,
-					   void *,
-					   enum drbd_notification_type type);
-extern int notify_device_state_change(struct sk_buff *,
-				       unsigned int,
-				       void *,
-				       enum drbd_notification_type type);
-extern int notify_peer_device_state_change(struct sk_buff *,
-					    unsigned int,
-					    void *,
-					    enum drbd_notification_type type);
+int notify_resource_state_change(struct sk_buff *skb, unsigned int seq,
+				 void *state_change,
+				 enum drbd_notification_type type);
+int notify_connection_state_change(struct sk_buff *skb, unsigned int seq,
+				   void *state_change,
+				   enum drbd_notification_type type);
+int notify_device_state_change(struct sk_buff *skb, unsigned int seq,
+			       void *state_change,
+			       enum drbd_notification_type type);
+int notify_peer_device_state_change(struct sk_buff *skb, unsigned int seq,
+				    void *state_change,
+				    enum drbd_notification_type type);
 
 #endif  /* DRBD_STATE_CHANGE_H */
diff --git a/drivers/block/drbd/drbd_strings.c b/drivers/block/drbd/drbd_strings.c
index 0a06f744b0961..8affd2904ce64 100644
--- a/drivers/block/drbd/drbd_strings.c
+++ b/drivers/block/drbd/drbd_strings.c
@@ -8,13 +8,14 @@
   Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
 
-
 */
 
 #include <linux/drbd.h>
+#include <linux/array_size.h>
 #include "drbd_strings.h"
+#include <linux/drbd_protocol.h>
 
-static const char * const drbd_conn_s_names[] = {
+static const char * const __conn_state_names[] = {
 	[C_STANDALONE]       = "StandAlone",
 	[C_DISCONNECTING]    = "Disconnecting",
 	[C_UNCONNECTED]      = "Unconnected",
@@ -22,34 +23,54 @@ static const char * const drbd_conn_s_names[] = {
 	[C_BROKEN_PIPE]      = "BrokenPipe",
 	[C_NETWORK_FAILURE]  = "NetworkFailure",
 	[C_PROTOCOL_ERROR]   = "ProtocolError",
-	[C_WF_CONNECTION]    = "WFConnection",
-	[C_WF_REPORT_PARAMS] = "WFReportParams",
 	[C_TEAR_DOWN]        = "TearDown",
-	[C_CONNECTED]        = "Connected",
-	[C_STARTING_SYNC_S]  = "StartingSyncS",
-	[C_STARTING_SYNC_T]  = "StartingSyncT",
-	[C_WF_BITMAP_S]      = "WFBitMapS",
-	[C_WF_BITMAP_T]      = "WFBitMapT",
-	[C_WF_SYNC_UUID]     = "WFSyncUUID",
-	[C_SYNC_SOURCE]      = "SyncSource",
-	[C_SYNC_TARGET]      = "SyncTarget",
-	[C_PAUSED_SYNC_S]    = "PausedSyncS",
-	[C_PAUSED_SYNC_T]    = "PausedSyncT",
-	[C_VERIFY_S]         = "VerifyS",
-	[C_VERIFY_T]         = "VerifyT",
-	[C_AHEAD]            = "Ahead",
-	[C_BEHIND]           = "Behind",
+	[C_CONNECTING]       = "Connecting",
+	[C_CONNECTED]	     = "Connected",
+};
+
+struct state_names drbd_conn_state_names = {
+	.names = __conn_state_names,
+	.size = sizeof(__conn_state_names) / sizeof(__conn_state_names[0]),
+};
+
+static const char * const __repl_state_names[] = {
+	[L_OFF]              = "Off",
+	[L_ESTABLISHED]      = "Established",
+	[L_STARTING_SYNC_S]  = "StartingSyncS",
+	[L_STARTING_SYNC_T]  = "StartingSyncT",
+	[L_WF_BITMAP_S]      = "WFBitMapS",
+	[L_WF_BITMAP_T]      = "WFBitMapT",
+	[L_WF_SYNC_UUID]     = "WFSyncUUID",
+	[L_SYNC_SOURCE]      = "SyncSource",
+	[L_SYNC_TARGET]      = "SyncTarget",
+	[L_VERIFY_S]         = "VerifyS",
+	[L_VERIFY_T]         = "VerifyT",
+	[L_PAUSED_SYNC_S]    = "PausedSyncS",
+	[L_PAUSED_SYNC_T]    = "PausedSyncT",
+	[L_AHEAD]            = "Ahead",
+	[L_BEHIND]           = "Behind",
+};
+
+struct state_names drbd_repl_state_names = {
+	.names = __repl_state_names,
+	.size = sizeof(__repl_state_names) / sizeof(__repl_state_names[0]),
 };
 
-static const char * const drbd_role_s_names[] = {
+static const char * const __role_state_names[] = {
+	[R_UNKNOWN]   = "Unknown",
 	[R_PRIMARY]   = "Primary",
 	[R_SECONDARY] = "Secondary",
-	[R_UNKNOWN]   = "Unknown"
 };
 
-static const char * const drbd_disk_s_names[] = {
+struct state_names drbd_role_state_names = {
+	.names = __role_state_names,
+	.size = sizeof(__role_state_names) / sizeof(__role_state_names[0]),
+};
+
+static const char * const __disk_state_names[] = {
 	[D_DISKLESS]     = "Diskless",
 	[D_ATTACHING]    = "Attaching",
+	[D_DETACHING]    = "Detaching",
 	[D_FAILED]       = "Failed",
 	[D_NEGOTIATING]  = "Negotiating",
 	[D_INCONSISTENT] = "Inconsistent",
@@ -59,7 +80,12 @@ static const char * const drbd_disk_s_names[] = {
 	[D_UP_TO_DATE]   = "UpToDate",
 };
 
-static const char * const drbd_state_sw_errors[] = {
+struct state_names drbd_disk_state_names = {
+	.names = __disk_state_names,
+	.size = sizeof(__disk_state_names) / sizeof(__disk_state_names[0]),
+};
+
+static const char * const __error_messages[] = {
 	[-SS_TWO_PRIMARIES] = "Multiple primaries not allowed by config",
 	[-SS_NO_UP_TO_DATE_DISK] = "Need access to UpToDate data",
 	[-SS_NO_LOCAL_DISK] = "Can not resync without local disk",
@@ -73,34 +99,163 @@ static const char * const drbd_state_sw_errors[] = {
 	[-SS_DEVICE_IN_USE] = "Device is held open by someone",
 	[-SS_NO_NET_CONFIG] = "Have no net/connection configuration",
 	[-SS_NO_VERIFY_ALG] = "Need a verify algorithm to start online verify",
-	[-SS_NEED_CONNECTION] = "Need a connection to start verify or resync",
+	[-SS_NEED_CONNECTION] = "State change requires a connection",
 	[-SS_NOT_SUPPORTED] = "Peer does not support protocol",
 	[-SS_LOWER_THAN_OUTDATED] = "Disk state is lower than outdated",
 	[-SS_IN_TRANSIENT_STATE] = "In transient state, retry after next state change",
 	[-SS_CONCURRENT_ST_CHG] = "Concurrent state changes detected and aborted",
-	[-SS_OUTDATE_WO_CONN] = "Need a connection for a graceful disconnect/outdate peer",
 	[-SS_O_VOL_PEER_PRI] = "Other vol primary on peer not allowed by config",
+	[-SS_PRIMARY_READER] = "Peer may not become primary while device is opened read-only",
+	[-SS_INTERRUPTED] = "Interrupted state change",
+	[-SS_TIMEOUT] = "Timeout in operation",
+	[-SS_WEAKLY_CONNECTED] = "Primary nodes must be strongly connected among each other",
+	[-SS_NO_QUORUM] = "No quorum",
+	[-SS_ATTACH_NO_BITMAP] = "Intentional diskless peer may not attach a disk",
+	[-SS_HANDSHAKE_DISCONNECT] = "Disconnect chosen in handshake",
+	[-SS_HANDSHAKE_RETRY] = "Retry chosen in handshake",
 };
 
-const char *drbd_conn_str(enum drbd_conns s)
+struct state_names drbd_error_messages = {
+	.names = __error_messages,
+	.size = sizeof(__error_messages) / sizeof(__error_messages[0]),
+};
+
+static const char * const __packet_names[] = {
+	[P_DATA]	        = "P_DATA",
+	[P_WSAME]               = "P_WSAME",
+	[P_TRIM]                = "P_TRIM",
+	[P_DATA_REPLY]	        = "P_DATA_REPLY",
+	[P_RS_DATA_REPLY]	= "P_RS_DATA_REPLY",
+	[P_BARRIER]	        = "P_BARRIER",
+	[P_BITMAP]	        = "P_BITMAP",
+	[P_BECOME_SYNC_TARGET]  = "P_BECOME_SYNC_TARGET",
+	[P_BECOME_SYNC_SOURCE]  = "P_BECOME_SYNC_SOURCE",
+	[P_UNPLUG_REMOTE]	= "P_UNPLUG_REMOTE",
+	[P_DATA_REQUEST]	= "P_DATA_REQUEST",
+	[P_RS_DATA_REQUEST]     = "P_RS_DATA_REQUEST",
+	[P_SYNC_PARAM]	        = "P_SYNC_PARAM",
+	[P_SYNC_PARAM89]	= "P_SYNC_PARAM89",
+	[P_PROTOCOL]            = "P_PROTOCOL",
+	[P_UUIDS]	        = "P_UUIDS",
+	[P_SIZES]	        = "P_SIZES",
+	[P_STATE]	        = "P_STATE",
+	[P_SYNC_UUID]           = "P_SYNC_UUID",
+	[P_AUTH_CHALLENGE]      = "P_AUTH_CHALLENGE",
+	[P_AUTH_RESPONSE]	= "P_AUTH_RESPONSE",
+	[P_PING]		= "P_PING",
+	[P_PING_ACK]	        = "P_PING_ACK",
+	[P_RECV_ACK]	        = "P_RECV_ACK",
+	[P_WRITE_ACK]	        = "P_WRITE_ACK",
+	[P_RS_WRITE_ACK]	= "P_RS_WRITE_ACK",
+	[P_SUPERSEDED]		= "P_SUPERSEDED",
+	[P_NEG_ACK]	        = "P_NEG_ACK",
+	[P_NEG_DREPLY]	        = "P_NEG_DREPLY",
+	[P_NEG_RS_DREPLY]	= "P_NEG_RS_DREPLY",
+	[P_BARRIER_ACK]	        = "P_BARRIER_ACK",
+	[P_STATE_CHG_REQ]       = "P_STATE_CHG_REQ",
+	[P_STATE_CHG_REPLY]     = "P_STATE_CHG_REPLY",
+	[P_OV_REQUEST]          = "P_OV_REQUEST",
+	[P_OV_REPLY]            = "P_OV_REPLY",
+	[P_OV_RESULT]           = "P_OV_RESULT",
+	[P_CSUM_RS_REQUEST]     = "P_CSUM_RS_REQUEST",
+	[P_RS_IS_IN_SYNC]	= "P_RS_IS_IN_SYNC",
+	[P_COMPRESSED_BITMAP]   = "P_COMPRESSED_BITMAP",
+	[P_DELAY_PROBE]         = "P_DELAY_PROBE",
+	[P_OUT_OF_SYNC]		= "P_OUT_OF_SYNC",
+	[P_RETRY_WRITE]		= "P_RETRY_WRITE",
+	[P_RS_CANCEL]		= "P_RS_CANCEL",
+	[P_RS_CANCEL_AHEAD]	= "P_RS_CANCEL_AHEAD",
+	[P_CONN_ST_CHG_REQ]	= "P_CONN_ST_CHG_REQ",
+	[P_CONN_ST_CHG_REPLY]	= "P_CONN_ST_CHG_REPLY",
+	[P_PROTOCOL_UPDATE]	= "P_PROTOCOL_UPDATE",
+	[P_TWOPC_PREPARE]	= "P_TWOPC_PREPARE",
+	[P_TWOPC_ABORT]		= "P_TWOPC_ABORT",
+	[P_DAGTAG]		= "P_DAGTAG",
+	[P_RS_THIN_REQ]         = "P_RS_THIN_REQ",
+	[P_RS_DEALLOCATED]      = "P_RS_DEALLOCATED",
+	[P_TWOPC_PREP_RSZ]      = "P_TWOPC_PREP_RSZ",
+	[P_ZEROES]              = "P_ZEROES",
+	[P_PEER_ACK]		= "P_PEER_ACK",
+	[P_PEERS_IN_SYNC]       = "P_PEERS_IN_SYNC",
+	[P_UUIDS110]            = "P_UUIDS110",
+	[P_PEER_DAGTAG]         = "P_PEER_DAGTAG",
+	[P_CURRENT_UUID]        = "P_CURRENT_UUID",
+	[P_TWOPC_COMMIT]	= "P_TWOPC_COMMIT",
+	[P_TWOPC_YES]		= "P_TWOPC_YES",
+	[P_TWOPC_NO]		= "P_TWOPC_NO",
+	[P_TWOPC_RETRY]		= "P_TWOPC_RETRY",
+	[P_CONFIRM_STABLE]      = "P_CONFIRM_STABLE",
+	[P_DISCONNECT]		= "P_DISCONNECT",
+	[P_RS_DAGTAG_REQ]	= "P_RS_DAGTAG_REQ",
+	[P_RS_CSUM_DAGTAG_REQ]	= "P_RS_CSUM_DAGTAG_REQ",
+	[P_RS_THIN_DAGTAG_REQ]	= "P_RS_THIN_DAGTAG_REQ",
+	[P_OV_DAGTAG_REQ]	= "P_OV_DAGTAG_REQ",
+	[P_OV_DAGTAG_REPLY]	= "P_OV_DAGTAG_REPLY",
+	[P_WRITE_ACK_IN_SYNC]   = "P_WRITE_ACK_IN_SYNC",
+	[P_RS_NEG_ACK]          = "P_RS_NEG_ACK",
+	[P_OV_RESULT_ID]        = "P_OV_RESULT_ID",
+	[P_RS_DEALLOCATED_ID]   = "P_RS_DEALLOCATED_ID",
+	[P_FLUSH_REQUESTS]      = "P_FLUSH_REQUESTS",
+	[P_FLUSH_FORWARD]       = "P_FLUSH_FORWARD",
+	[P_FLUSH_REQUESTS_ACK]  = "P_FLUSH_REQUESTS_ACK",
+	[P_ENABLE_REPLICATION_NEXT] = "P_ENABLE_REPLICATION_NEXT",
+	[P_ENABLE_REPLICATION]  = "P_ENABLE_REPLICATION",
+	/* enum drbd_packet, but not commands - obsoleted flags:
+	 *	P_MAY_IGNORE
+	 *	P_MAX_OPT_CMD
+	 */
+};
+
+struct state_names drbd_packet_names = {
+	.names = __packet_names,
+	.size = sizeof(__packet_names) / sizeof(__packet_names[0]),
+};
+
+const char *drbd_repl_str(enum drbd_repl_state s)
 {
-	/* enums are unsigned... */
-	return s > C_BEHIND ? "TOO_LARGE" : drbd_conn_s_names[s];
+	return (s < 0 || s >= drbd_repl_state_names.size ||
+		!drbd_repl_state_names.names[s]) ?
+		"?" : drbd_repl_state_names.names[s];
+}
+
+const char *drbd_conn_str(enum drbd_conn_state s)
+{
+	return (s < 0 || s >= drbd_conn_state_names.size ||
+		!drbd_conn_state_names.names[s]) ?
+		"?" : drbd_conn_state_names.names[s];
 }
 
 const char *drbd_role_str(enum drbd_role s)
 {
-	return s > R_SECONDARY   ? "TOO_LARGE" : drbd_role_s_names[s];
+	return (s < 0 || s >= drbd_role_state_names.size ||
+		!drbd_role_state_names.names[s]) ?
+		"?" : drbd_role_state_names.names[s];
 }
 
 const char *drbd_disk_str(enum drbd_disk_state s)
 {
-	return s > D_UP_TO_DATE    ? "TOO_LARGE" : drbd_disk_s_names[s];
+	return (s < 0 || s >= drbd_disk_state_names.size ||
+		!drbd_disk_state_names.names[s]) ?
+		"?" : drbd_disk_state_names.names[s];
 }
 
 const char *drbd_set_st_err_str(enum drbd_state_rv err)
 {
-	return err <= SS_AFTER_LAST_ERROR ? "TOO_SMALL" :
-	       err > SS_TWO_PRIMARIES ? "TOO_LARGE"
-			: drbd_state_sw_errors[-err];
+	return (-err < 0 || -err >= drbd_error_messages.size ||
+		!drbd_error_messages.names[-err]) ?
+		"?" : drbd_error_messages.names[-err];
+}
+
+const char *drbd_packet_name(enum drbd_packet cmd)
+{
+	/* too big for the array: 0xfffX */
+	if (cmd == P_INITIAL_META)
+		return "InitialMeta";
+	if (cmd == P_INITIAL_DATA)
+		return "InitialData";
+	if (cmd == P_CONNECTION_FEATURES)
+		return "ConnectionFeatures";
+	return (cmd < 0 || cmd >= ARRAY_SIZE(__packet_names) ||
+		!__packet_names[cmd]) ?
+	       "?" : __packet_names[cmd];
 }
diff --git a/drivers/block/drbd/drbd_strings.h b/drivers/block/drbd/drbd_strings.h
index 0201f6590f6a9..f376ce28a8150 100644
--- a/drivers/block/drbd/drbd_strings.h
+++ b/drivers/block/drbd/drbd_strings.h
@@ -2,9 +2,26 @@
 #ifndef __DRBD_STRINGS_H
 #define __DRBD_STRINGS_H
 
-extern const char *drbd_conn_str(enum drbd_conns);
-extern const char *drbd_role_str(enum drbd_role);
-extern const char *drbd_disk_str(enum drbd_disk_state);
-extern const char *drbd_set_st_err_str(enum drbd_state_rv);
+struct state_names {
+	const char * const *names;
+	unsigned int size;
+};
+
+extern struct state_names drbd_conn_state_names;
+extern struct state_names drbd_repl_state_names;
+extern struct state_names drbd_role_state_names;
+extern struct state_names drbd_disk_state_names;
+extern struct state_names drbd_error_messages;
+extern struct state_names drbd_packet_names;
+
+enum drbd_packet;
+
+const char *drbd_repl_str(enum drbd_repl_state s);
+const char *drbd_conn_str(enum drbd_conn_state s);
+const char *drbd_role_str(enum drbd_role s);
+const char *drbd_disk_str(enum drbd_disk_state s);
+const char *drbd_set_st_err_str(enum drbd_state_rv err);
+const char *drbd_packet_name(enum drbd_packet cmd);
+
 
 #endif  /* __DRBD_STRINGS_H */
diff --git a/drivers/block/drbd/drbd_transport.c b/drivers/block/drbd/drbd_transport.c
new file mode 100644
index 0000000000000..7c6128cbb8bce
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport.c
@@ -0,0 +1,379 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
+
+#include <linux/spinlock.h>
+#include <linux/module.h>
+#include <net/ipv6.h>
+#include "drbd_transport.h"
+#include "drbd_int.h"
+
+static LIST_HEAD(transport_classes);
+static DECLARE_RWSEM(transport_classes_lock);
+
+static struct drbd_transport_class *__find_transport_class(const char *transport_name)
+{
+	struct drbd_transport_class *transport_class;
+
+	list_for_each_entry(transport_class, &transport_classes, list)
+		if (!strcmp(transport_class->name, transport_name))
+			return transport_class;
+
+	return NULL;
+}
+
+int drbd_register_transport_class(struct drbd_transport_class *transport_class, int version,
+				  int drbd_transport_size)
+{
+	int rv = 0;
+	if (version != DRBD_TRANSPORT_API_VERSION) {
+		pr_err("DRBD_TRANSPORT_API_VERSION not compatible\n");
+		return -EINVAL;
+	}
+
+	if (drbd_transport_size != sizeof(struct drbd_transport)) {
+		pr_err("sizeof(drbd_transport) not compatible\n");
+		return -EINVAL;
+	}
+
+	down_write(&transport_classes_lock);
+	if (__find_transport_class(transport_class->name)) {
+		pr_err("transport class '%s' already registered\n", transport_class->name);
+		rv = -EEXIST;
+	} else {
+		list_add_tail(&transport_class->list, &transport_classes);
+		pr_info("registered transport class '%s' (version:%s)\n",
+			transport_class->name,
+			transport_class->module->version ?: "N/A");
+	}
+	up_write(&transport_classes_lock);
+	return rv;
+}
+
+void drbd_unregister_transport_class(struct drbd_transport_class *transport_class)
+{
+	down_write(&transport_classes_lock);
+	if (!__find_transport_class(transport_class->name)) {
+		pr_crit("unregistering unknown transport class '%s'\n",
+			transport_class->name);
+		BUG();
+	}
+	list_del_init(&transport_class->list);
+	pr_info("unregistered transport class '%s'\n", transport_class->name);
+	up_write(&transport_classes_lock);
+}
+
+static struct drbd_transport_class *get_transport_class(const char *name)
+{
+	struct drbd_transport_class *tc;
+
+	down_read(&transport_classes_lock);
+	tc = __find_transport_class(name);
+	if (tc && !try_module_get(tc->module))
+		tc = NULL;
+	up_read(&transport_classes_lock);
+	return tc;
+}
+
+struct drbd_transport_class *drbd_get_transport_class(const char *name)
+{
+	struct drbd_transport_class *tc = get_transport_class(name);
+
+	if (!tc) {
+		request_module("drbd_transport_%s", name);
+		tc = get_transport_class(name);
+	}
+
+	return tc;
+}
+
+void drbd_put_transport_class(struct drbd_transport_class *tc)
+{
+	/* convenient in the error cleanup path */
+	if (!tc)
+		return;
+	down_read(&transport_classes_lock);
+	module_put(tc->module);
+	up_read(&transport_classes_lock);
+}
+
+void drbd_print_transports_loaded(struct seq_file *seq)
+{
+	struct drbd_transport_class *tc;
+
+	down_read(&transport_classes_lock);
+
+	seq_puts(seq, "Transports (api:" __stringify(DRBD_TRANSPORT_API_VERSION) "):");
+	list_for_each_entry(tc, &transport_classes, list) {
+		seq_printf(seq, " %s (%s)", tc->name,
+				tc->module->version ? tc->module->version : "NONE");
+	}
+	seq_putc(seq, '\n');
+
+	up_read(&transport_classes_lock);
+}
+
+static bool addr_equal(const struct sockaddr_storage *addr1, const struct sockaddr_storage *addr2)
+{
+	if (addr1->ss_family != addr2->ss_family)
+		return false;
+
+	if (addr1->ss_family == AF_INET6) {
+		const struct sockaddr_in6 *v6a1 = (const struct sockaddr_in6 *)addr1;
+		const struct sockaddr_in6 *v6a2 = (const struct sockaddr_in6 *)addr2;
+
+		if (!ipv6_addr_equal(&v6a1->sin6_addr, &v6a2->sin6_addr))
+			return false;
+		else if (ipv6_addr_type(&v6a1->sin6_addr) & IPV6_ADDR_LINKLOCAL)
+			return v6a1->sin6_scope_id == v6a2->sin6_scope_id;
+		return true;
+	} else /* AF_INET, AF_SSOCKS, AF_SDP */ {
+		const struct sockaddr_in *v4a1 = (const struct sockaddr_in *)addr1;
+		const struct sockaddr_in *v4a2 = (const struct sockaddr_in *)addr2;
+
+		return v4a1->sin_addr.s_addr == v4a2->sin_addr.s_addr;
+	}
+}
+
+static bool addr_and_port_equal(const struct sockaddr_storage *addr1, const struct sockaddr_storage *addr2)
+{
+	if (!addr_equal(addr1, addr2))
+		return false;
+
+	if (addr1->ss_family == AF_INET6) {
+		const struct sockaddr_in6 *v6a1 = (const struct sockaddr_in6 *)addr1;
+		const struct sockaddr_in6 *v6a2 = (const struct sockaddr_in6 *)addr2;
+
+		return v6a1->sin6_port == v6a2->sin6_port;
+	} else /* AF_INET, AF_SSOCKS, AF_SDP */ {
+		const struct sockaddr_in *v4a1 = (const struct sockaddr_in *)addr1;
+		const struct sockaddr_in *v4a2 = (const struct sockaddr_in *)addr2;
+
+		return v4a1->sin_port == v4a2->sin_port;
+	}
+
+	return false;
+}
+
+static struct drbd_listener *find_listener(struct drbd_connection *connection,
+					   const struct sockaddr_storage *addr)
+{
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_listener *listener;
+
+	list_for_each_entry(listener, &resource->listeners, list) {
+		if (addr_and_port_equal(&listener->listen_addr, addr)) {
+			if (kref_get_unless_zero(&listener->kref))
+				return listener;
+		}
+	}
+	return NULL;
+}
+
+int drbd_get_listener(struct drbd_path *path)
+{
+	struct drbd_transport *transport = path->transport;
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+	struct sockaddr *addr = (struct sockaddr *)&path->my_addr;
+	struct drbd_resource *resource = connection->resource;
+	struct drbd_transport_class *tc = transport->class;
+	struct drbd_listener *listener;
+	bool needs_init = false;
+	int err;
+
+	spin_lock_bh(&resource->listeners_lock);
+	listener = find_listener(connection, (struct sockaddr_storage *)addr);
+	if (!listener) {
+		listener = kzalloc(tc->listener_instance_size, GFP_ATOMIC);
+		if (!listener) {
+			spin_unlock_bh(&resource->listeners_lock);
+			return -ENOMEM;
+		}
+		kref_init(&listener->kref);
+		INIT_LIST_HEAD(&listener->waiters);
+		listener->resource = resource;
+		listener->pending_accepts = 0;
+		spin_lock_init(&listener->waiters_lock);
+		init_completion(&listener->ready);
+		listener->listen_addr = *(struct sockaddr_storage *)addr;
+		listener->transport_class = NULL;
+
+		list_add(&listener->list, &resource->listeners);
+		needs_init = true;
+	}
+	spin_unlock_bh(&resource->listeners_lock);
+
+	if (needs_init) {
+		if (try_module_get(tc->module)) {
+			listener->transport_class = tc;
+			err = tc->ops.init_listener(transport, addr, path->net, listener);
+		} else {
+			err = -ENODEV;
+		}
+		listener->err = err;
+		complete_all(&listener->ready);
+	} else {
+		wait_for_completion(&listener->ready);
+		err = listener->err;
+	}
+
+	if (err) {
+		kref_put(&listener->kref, drbd_listener_destroy);
+		return err;
+	}
+
+	spin_lock_bh(&listener->waiters_lock);
+	kref_get(&path->kref);
+	list_add(&path->listener_link, &listener->waiters);
+	path->listener = listener;
+	spin_unlock_bh(&listener->waiters_lock);
+	/* After exposing the listener on a path, drbd_put_listenr() can destroy it. */
+
+	return 0;
+}
+
+void drbd_listener_destroy(struct kref *kref)
+{
+	struct drbd_listener *listener = container_of(kref, struct drbd_listener, kref);
+	struct drbd_transport_class *tc = listener->transport_class;
+	struct drbd_resource *resource = listener->resource;
+
+	spin_lock_bh(&resource->listeners_lock);
+	list_del(&listener->list);
+	spin_unlock_bh(&resource->listeners_lock);
+
+	if (tc) {
+		tc->ops.release_listener(listener);
+		module_put(tc->module);
+	}
+	kfree(listener);
+}
+
+void drbd_put_listener(struct drbd_path *path)
+{
+	struct drbd_listener *listener;
+
+	listener = xchg(&path->listener, NULL);
+	if (!listener)
+		return;
+
+	spin_lock_bh(&listener->waiters_lock);
+	list_del(&path->listener_link);
+	kref_put(&path->kref, drbd_destroy_path);
+	spin_unlock_bh(&listener->waiters_lock);
+	kref_put(&listener->kref, drbd_listener_destroy);
+}
+
+struct drbd_path *drbd_find_path_by_addr(struct drbd_listener *listener, struct sockaddr_storage *addr)
+{
+	struct drbd_path *path;
+
+	list_for_each_entry(path, &listener->waiters, listener_link) {
+		if (addr_equal(&path->peer_addr, addr))
+			return path;
+	}
+
+	return NULL;
+}
+
+/**
+ * drbd_stream_send_timed_out() - Tells transport if the connection should stay alive
+ * @transport:	DRBD transport to operate on.
+ * @stream:     DATA_STREAM or CONTROL_STREAM
+ *
+ * When it returns true, the transport should return -EAGAIN to its caller of the
+ * send function. When it returns false the transport should keep on trying to
+ * get the packet through.
+ */
+bool drbd_stream_send_timed_out(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+	bool drop_it;
+
+	drop_it = stream == CONTROL_STREAM || connection->cstate[NOW] < C_CONNECTED;
+
+	if (drop_it)
+		return true;
+
+	drop_it = !--connection->transport.ko_count;
+	if (!drop_it) {
+		drbd_err(connection, "[%s/%d] sending time expired, ko = %u\n",
+			 current->comm, current->pid, connection->transport.ko_count);
+		schedule_work(&connection->send_ping_work);
+	}
+
+	return drop_it;
+}
+
+bool drbd_should_abort_listening(struct drbd_transport *transport)
+{
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+	bool abort = false;
+
+	if (connection->cstate[NOW] <= C_DISCONNECTING)
+		abort = true;
+	if (signal_pending(current)) {
+		flush_signals(current);
+		smp_rmb();
+		if (get_t_state(&connection->receiver) == EXITING)
+			abort = true;
+	}
+
+	return abort;
+}
+
+/* Called by a transport if a path was established / disconnected */
+void drbd_path_event(struct drbd_transport *transport, struct drbd_path *path)
+{
+	struct drbd_connection *connection =
+		container_of(transport, struct drbd_connection, transport);
+
+	notify_path(connection, path, NOTIFY_CHANGE);
+}
+
+struct drbd_path *__drbd_next_path_ref(struct drbd_path *drbd_path,
+					      struct drbd_transport *transport)
+{
+	rcu_read_lock();
+	if (!drbd_path) {
+		drbd_path = list_first_or_null_rcu(&transport->paths, struct drbd_path, list);
+	} else {
+		struct list_head *pos;
+		bool in_list;
+
+		pos = list_next_rcu(&drbd_path->list);
+		/* Ensure list head is read before flag. */
+		smp_rmb();
+		in_list = !test_bit(TR_UNREGISTERED, &drbd_path->flags);
+		kref_put(&drbd_path->kref, drbd_destroy_path);
+
+		if (pos == &transport->paths) {
+			drbd_path = NULL;
+		} else if (in_list) {
+			drbd_path = list_entry_rcu(pos, struct drbd_path, list);
+		} else {
+			/* No longer on the list, element might be freed, restart from the start */
+			drbd_path = list_first_or_null_rcu(&transport->paths,
+					struct drbd_path, list);
+		}
+	}
+	if (drbd_path)
+		kref_get(&drbd_path->kref);
+	rcu_read_unlock();
+
+	return drbd_path;
+}
+
+/* Network transport abstractions */
+EXPORT_SYMBOL_GPL(drbd_register_transport_class);
+EXPORT_SYMBOL_GPL(drbd_unregister_transport_class);
+EXPORT_SYMBOL_GPL(drbd_get_listener);
+EXPORT_SYMBOL_GPL(drbd_put_listener);
+EXPORT_SYMBOL_GPL(drbd_find_path_by_addr);
+EXPORT_SYMBOL_GPL(drbd_stream_send_timed_out);
+EXPORT_SYMBOL_GPL(drbd_should_abort_listening);
+EXPORT_SYMBOL_GPL(drbd_path_event);
+EXPORT_SYMBOL_GPL(drbd_listener_destroy);
+EXPORT_SYMBOL_GPL(__drbd_next_path_ref);
diff --git a/drivers/block/drbd/drbd_transport.h b/drivers/block/drbd/drbd_transport.h
new file mode 100644
index 0000000000000..ff393e8d12dc0
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport.h
@@ -0,0 +1,443 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef DRBD_TRANSPORT_H
+#define DRBD_TRANSPORT_H
+
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/wait.h>
+#include <linux/socket.h>
+
+/* Whenever touch this file in a non-trivial way, increase the
+   DRBD_TRANSPORT_API_VERSION
+   So that transport compiled against an older version of this
+   header will no longer load in a module that assumes a newer
+   version. */
+#define DRBD_TRANSPORT_API_VERSION 21
+
+/* MSG_MSG_DONTROUTE and MSG_PROBE are not used by DRBD. I.e.
+   we can reuse these flags for our purposes */
+#define CALLER_BUFFER  MSG_DONTROUTE
+#define GROW_BUFFER    MSG_PROBE
+
+/*
+ * gfp_mask for allocating memory with no write-out.
+ *
+ * When drbd allocates memory on behalf of the peer, we prevent it from causing
+ * write-out because in a criss-cross setup, the write-out could lead to memory
+ * pressure on the peer, eventually leading to deadlock.
+ */
+#define GFP_TRY	(__GFP_HIGHMEM | __GFP_NOWARN | __GFP_RECLAIM)
+
+#define tr_printk(level, transport, fmt, args...)  ({		\
+	rcu_read_lock();					\
+	printk(level "drbd %s %s:%s: " fmt,			\
+	       (transport)->log_prefix,				\
+	       (transport)->class->name,			\
+	       rcu_dereference((transport)->net_conf)->name,	\
+	       ## args);					\
+	rcu_read_unlock();					\
+	})
+
+#define tr_err(transport, fmt, args...) \
+	tr_printk(KERN_ERR, transport, fmt, ## args)
+#define tr_warn(transport, fmt, args...) \
+	tr_printk(KERN_WARNING, transport, fmt, ## args)
+#define tr_notice(transport, fmt, args...) \
+	tr_printk(KERN_NOTICE, transport, fmt, ## args)
+#define tr_info(transport, fmt, args...) \
+	tr_printk(KERN_INFO, transport, fmt, ## args)
+
+#define TR_ASSERT(x, exp)							\
+	do {									\
+		if (!(exp))							\
+			tr_err(x, "ASSERTION %s FAILED in %s\n",	\
+				 #exp, __func__);				\
+	} while (0)
+
+struct drbd_resource;
+struct drbd_listener;
+struct drbd_transport;
+
+enum drbd_stream {
+	DATA_STREAM,
+	CONTROL_STREAM
+};
+
+enum drbd_tr_hints {
+	CORK,
+	UNCORK,
+	NODELAY,
+	NOSPACE,
+	QUICKACK
+};
+
+enum { /* bits in the flags word */
+	NET_CONGESTED,		/* The data socket is congested */
+	RESOLVE_CONFLICTS,	/* Set on one node, cleared on the peer! */
+};
+
+enum drbd_tr_free_op {
+	CLOSE_CONNECTION,
+	DESTROY_TRANSPORT
+};
+
+enum drbd_tr_event {
+	CLOSED_BY_PEER,
+	TIMEOUT,
+};
+
+enum drbd_tr_path_flag {
+	TR_ESTABLISHED, /* updated by the transport */
+	TR_UNREGISTERED,
+	TR_TRANSPORT_PRIVATE = 32, /* flags starting here are used exclusively by the transport */
+};
+
+/* A transport might wrap its own data structure around this. Having
+   this base class as its first member. */
+struct drbd_path {
+	struct sockaddr_storage my_addr;
+	struct sockaddr_storage peer_addr;
+
+	struct kref kref;
+
+	struct net *net;
+	int my_addr_len;
+	int peer_addr_len;
+	unsigned long flags;
+
+	struct drbd_transport *transport;
+	struct list_head list; /* paths of a connection */
+	struct list_head listener_link; /* paths waiting for an incoming connection,
+					   head is in a drbd_listener */
+	struct drbd_listener *listener;
+
+	struct rcu_head rcu;
+};
+
+/* Each transport implementation should embed a struct drbd_transport
+   into it's instance data structure. */
+struct drbd_transport {
+	struct drbd_transport_class *class;
+
+	struct list_head paths;
+
+	const char *log_prefix;		/* resource name */
+	struct net_conf __rcu *net_conf;	/* content protected by rcu */
+
+	/* These members are intended to be updated by the transport: */
+	unsigned int ko_count;
+	unsigned long flags;
+};
+
+struct drbd_transport_stats {
+	int unread_received;
+	int unacked_send;
+	int send_buffer_size;
+	int send_buffer_used;
+};
+
+/* argument to ->recv_pages() */
+struct drbd_page_chain_head {
+	struct page *head;
+	unsigned int nr_pages;
+};
+
+struct drbd_const_buffer {
+	const u8 *buffer;
+	unsigned int avail;
+};
+
+/**
+ * struct drbd_transport_ops - Operations implemented by the transport.
+ *
+ * The user of this API guarantees that all of the following will be exclusive
+ * with respect to each other for a given transport instance:
+ * * init()
+ * * free()
+ * * prepare_connect()
+ * * finish_connect()
+ * * add_path() and the subsequent list_add_tail_rcu() for the paths list
+ * * may_remove_path() and the subsequent list_del_rcu() for the paths list
+ *
+ * The connection sequence is as follows:
+ * 1. prepare_connect(), with the above exclusivity guarantee
+ * 2. connect(), this may take a long time
+ * 3. finish_connect(), with the above exclusivity guarantee
+ */
+struct drbd_transport_ops {
+	int (*init)(struct drbd_transport *);
+	void (*free)(struct drbd_transport *, enum drbd_tr_free_op free_op);
+	int (*init_listener)(struct drbd_transport *, const struct sockaddr *, struct net *net,
+			     struct drbd_listener *);
+	void (*release_listener)(struct drbd_listener *);
+	int (*prepare_connect)(struct drbd_transport *);
+	int (*connect)(struct drbd_transport *);
+	void (*finish_connect)(struct drbd_transport *);
+
+/**
+ * recv() - Receive data via the transport
+ * @transport:	The transport to use
+ * @stream:	The stream within the transport to use. Ether DATA_STREAM or CONTROL_STREAM
+ * @buf:	The function will place here the pointer to the data area
+ * @size:	Number of byte to receive
+ * @msg_flags:	Bitmask of CALLER_BUFFER, GROW_BUFFER and MSG_DONTWAIT
+ *
+ * recv() returns the requests data in a buffer (owned by the transport).
+ * You may pass MSG_DONTWAIT as flags.  Usually with the next call to recv()
+ * or recv_pages() on the same stream, the buffer may no longer be accessed
+ * by the caller. I.e. it is reclaimed by the transport.
+ *
+ * If the transport was not capable of fulfilling the complete "wish" of the
+ * caller (that means it returned a smaller size that size), the caller may
+ * call recv() again with the flag GROW_BUFFER, and *buf as returned by the
+ * previous call.
+ * Note1: This can happen if MSG_DONTWAIT was used, or if a receive timeout
+ *	was we with set_rcvtimeo().
+ * Note2: recv() is free to re-locate the buffer in such a call. I.e. to
+ *	modify *buf. Then it copies the content received so far to the new
+ *	memory location.
+ *
+ * Last not least the caller may also pass an arbitrary pointer in *buf with
+ * the CALLER_BUFFER flag. This is expected to be used for small amounts
+ * of data only
+ *
+ * Upon success the function returns the bytes read. Upon error the return
+ * code is negative. A 0 indicates that the socket was closed by the remote
+ * side.
+ */
+	int (*recv)(struct drbd_transport *, enum drbd_stream, void **buf, size_t size, int flags);
+
+/**
+ * recv_pages() - Receive bulk data via the transport's DATA_STREAM
+ * @peer_device: Identify the transport and the device
+ * @page_chain:	Here recv_pages() will place the page chain head and length
+ * @size:	Number of bytes to receive
+ *
+ * recv_pages() will return the requested amount of data from DATA_STREAM,
+ * and place it into pages allocated with drbd_alloc_pages().
+ *
+ * Upon success the function returns 0. Upon error the function returns a
+ * negative value
+ */
+	int (*recv_pages)(struct drbd_transport *, struct drbd_page_chain_head *, size_t size);
+
+	void (*stats)(struct drbd_transport *, struct drbd_transport_stats *stats);
+/**
+ * net_conf_change() - Notify about changed network configuration on the transport.
+ * @new_net_conf: The new network configuration that should be applied.
+ *
+ * net_conf_change() is called in the context of either the initial creation of the connection,
+ * or when the net_conf is changed via netlink. Note that assignment of the net_conf to the
+ * transport object happens after this function is called.
+ *
+ * On a negative (error) return value, it is expected that any changes are reverted and
+ * the old net_conf (if any) is still in effect.
+ *
+ * Upon success the function return 0. Upon error the function returns a negative value.
+ */
+	int (*net_conf_change)(struct drbd_transport *, struct net_conf *new_net_conf);
+	void (*set_rcvtimeo)(struct drbd_transport *, enum drbd_stream, long timeout);
+	long (*get_rcvtimeo)(struct drbd_transport *, enum drbd_stream);
+	int (*send_page)(struct drbd_transport *, enum drbd_stream, struct page *,
+			 int offset, size_t size, unsigned msg_flags);
+	int (*send_zc_bio)(struct drbd_transport *, struct bio *bio);
+	bool (*stream_ok)(struct drbd_transport *, enum drbd_stream);
+	bool (*hint)(struct drbd_transport *, enum drbd_stream, enum drbd_tr_hints hint);
+	void (*debugfs_show)(struct drbd_transport *, struct seq_file *m);
+
+/**
+ * add_path() - Prepare path to be added
+ * @path: The path that is being added
+ *
+ * Called before the path is added to the paths list.
+ *
+ * Return: 0 if path may be added, error code otherwise.
+ */
+	int (*add_path)(struct drbd_path *path);
+
+/**
+ * may_remove_path() - Query whether path may currently be removed
+ * @path: The path to be removed
+ *
+ * Return: true is path may be removed, false otherwise.
+ */
+	bool (*may_remove_path)(struct drbd_path *path);
+
+/**
+ * remove_path() - Clear up after path removal
+ * @path: The path that is being removed
+ *
+ * Clear up a path that is being removed. Called after the path has been
+ * removed from the list and all kref references have been put.
+ */
+	void (*remove_path)(struct drbd_path *path);
+};
+
+struct drbd_transport_class {
+	const char *name;
+	const int instance_size;
+	const int path_instance_size;
+	const int listener_instance_size;
+	struct drbd_transport_ops ops;
+
+	struct module *module;
+
+	struct list_head list;
+};
+
+
+/* An "abstract base class" for transport implementations. I.e. it
+   should be embedded into a transport specific representation of a
+   listening "socket" */
+struct drbd_listener {
+	struct kref kref;
+	struct drbd_resource *resource;
+	struct drbd_transport_class *transport_class;
+	struct list_head list; /* link for resource->listeners */
+	struct list_head waiters; /* list head for paths */
+	spinlock_t waiters_lock;
+	int pending_accepts;
+	struct sockaddr_storage listen_addr;
+	struct completion ready;
+	int err;
+};
+
+/* drbd_main.c */
+void drbd_destroy_path(struct kref *kref);
+
+/* drbd_transport.c */
+int drbd_register_transport_class(struct drbd_transport_class *transport_class,
+				  int version, int drbd_transport_size);
+void drbd_unregister_transport_class(struct drbd_transport_class *transport_class);
+struct drbd_transport_class *drbd_get_transport_class(const char *name);
+void drbd_put_transport_class(struct drbd_transport_class *tc);
+void drbd_print_transports_loaded(struct seq_file *seq);
+
+int drbd_get_listener(struct drbd_path *path);
+void drbd_put_listener(struct drbd_path *path);
+struct drbd_path *drbd_find_path_by_addr(struct drbd_listener *listener,
+					 struct sockaddr_storage *addr);
+bool drbd_stream_send_timed_out(struct drbd_transport *transport,
+				enum drbd_stream stream);
+bool drbd_should_abort_listening(struct drbd_transport *transport);
+void drbd_path_event(struct drbd_transport *transport, struct drbd_path *path);
+void drbd_listener_destroy(struct kref *kref);
+struct drbd_path *__drbd_next_path_ref(struct drbd_path *drbd_path,
+				       struct drbd_transport *transport);
+
+/* Might restart iteration, if current element is removed from list!! */
+#define for_each_path_ref(path, transport)			\
+	for (path = __drbd_next_path_ref(NULL, transport);	\
+	     path;						\
+	     path = __drbd_next_path_ref(path, transport))
+
+/* drbd_receiver.c*/
+struct page *drbd_alloc_pages(struct drbd_transport *transport,
+			      unsigned int number, gfp_t gfp_mask);
+void drbd_free_pages(struct drbd_transport *transport, struct page *page);
+void drbd_control_data_ready(struct drbd_transport *transport,
+			     struct drbd_const_buffer *pool);
+void drbd_control_event(struct drbd_transport *transport,
+			enum drbd_tr_event event);
+
+static inline void drbd_alloc_page_chain(struct drbd_transport *t,
+	struct drbd_page_chain_head *chain, unsigned int nr, gfp_t gfp_flags)
+{
+	chain->head = drbd_alloc_pages(t, nr, gfp_flags);
+	chain->nr_pages = chain->head ? nr : 0;
+}
+
+static inline void drbd_free_page_chain(struct drbd_transport *transport,
+					struct drbd_page_chain_head *chain)
+{
+	drbd_free_pages(transport, chain->head);
+	chain->head = NULL;
+	chain->nr_pages = 0;
+}
+
+/*
+ * Some helper functions to deal with our page chains.
+ */
+/* Our transports may sometimes need to only partially use a page.
+ * We need to express that somehow.  Use this struct, and "graft" it into
+ * struct page at page->lru.
+ *
+ * According to include/linux/mm.h:
+ *  | A page may be used by anyone else who does a __get_free_page().
+ *  | In this case, page_count still tracks the references, and should only
+ *  | be used through the normal accessor functions. The top bits of page->flags
+ *  | and page->virtual store page management information, but all other fields
+ *  | are unused and could be used privately, carefully. The management of this
+ *  | page is the responsibility of the one who allocated it, and those who have
+ *  | subsequently been given references to it.
+ * (we do alloc_page(), that is equivalent).
+ *
+ * Red Hat struct page is different from upstream (layout and members) :(
+ * So I am not too sure about the "all other fields", and it is not as easy to
+ * find a place where sizeof(struct drbd_page_chain) would fit on all archs and
+ * distribution-changed layouts.
+ *
+ * But (upstream) struct page also says:
+ *  | struct list_head lru;   * ...
+ *  |       * Can be used as a generic list
+ *  |       * by the page owner.
+ *
+ * On 32bit, use unsigned short for offset and size,
+ * to still fit in sizeof(page->lru).
+ */
+
+/* grafted over struct page.lru */
+struct drbd_page_chain {
+	struct page *next;	/* next page in chain, if any */
+#ifdef CONFIG_64BIT
+	unsigned int offset;	/* start offset of data within this page */
+	unsigned int size;	/* number of data bytes within this page */
+#else
+#if PAGE_SIZE > (1U<<16)
+#error "won't work."
+#endif
+	unsigned short offset;	/* start offset of data within this page */
+	unsigned short size;	/* number of data bytes within this page */
+#endif
+};
+
+static inline void dummy_for_buildbug(void)
+{
+	struct page *dummy;
+	BUILD_BUG_ON(sizeof(struct drbd_page_chain) > sizeof(dummy->lru));
+}
+
+#define page_chain_next(page) \
+	(((struct drbd_page_chain *)&(page)->lru)->next)
+#define page_chain_size(page) \
+	(((struct drbd_page_chain *)&(page)->lru)->size)
+#define page_chain_offset(page) \
+	(((struct drbd_page_chain *)&(page)->lru)->offset)
+#define set_page_chain_next(page, v) \
+	(((struct drbd_page_chain *)&(page)->lru)->next = (v))
+#define set_page_chain_size(page, v) \
+	(((struct drbd_page_chain *)&(page)->lru)->size = (v))
+#define set_page_chain_offset(page, v) \
+	(((struct drbd_page_chain *)&(page)->lru)->offset = (v))
+#define set_page_chain_next_offset_size(page, n, o, s)		\
+	(*((struct drbd_page_chain *)&(page)->lru) =		\
+	((struct drbd_page_chain) {				\
+		.next = (n),					\
+		.offset = (o),					\
+		.size = (s),					\
+	 }))
+
+#define page_chain_for_each(page) \
+	for (; page && ({ prefetch(page_chain_next(page)); 1; }); \
+			page = page_chain_next(page))
+#define page_chain_for_each_safe(page, n) \
+	for (; page && ({ n = page_chain_next(page); 1; }); page = n)
+
+#ifndef SK_CAN_REUSE
+/* This constant was introduced by Pavel Emelyanov <xemul@parallels.com> on
+   Thu Apr 19 03:39:36 2012 +0000. Before the release of linux-3.5
+   commit 4a17fd52 sock: Introduce named constants for sk_reuse */
+#define SK_CAN_REUSE   1
+#endif
+
+#endif
diff --git a/drivers/block/drbd/drbd_transport_lb-tcp.c b/drivers/block/drbd/drbd_transport_lb-tcp.c
new file mode 100644
index 0000000000000..beea1bd3e2c5a
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport_lb-tcp.c
@@ -0,0 +1,1905 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * drbd_transport_lb-tcp.c
+ *
+ * This file is part of DRBD.
+ *
+ *  Copyright (C) 2014-2023, LINBIT HA-Solutions GmbH.
+ */
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/socket.h>
+#include <linux/pkt_sched.h>
+#include <linux/sched/signal.h>
+#include <linux/net.h>
+#include <linux/tcp.h>
+#include <linux/highmem.h>
+#include <linux/bio.h>
+#include <linux/drbd_genl_api.h>
+#include <linux/drbd_config.h>
+#include <net/tcp.h>
+#include <linux/drbd_protocol.h>
+#include "drbd_transport.h"
+
+
+MODULE_AUTHOR("Philipp Reisner <philipp.reisner@linbit.com>");
+MODULE_AUTHOR("Lars Ellenberg <lars.ellenberg@linbit.com>");
+MODULE_AUTHOR("Roland Kammerer <roland.kammerer@linbit.com>");
+MODULE_DESCRIPTION("Load balancing TCP transport layer for DRBD");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(REL_VERSION);
+
+/* TCP keepalive has proven to be vital in many deployment scenarios.
+ * Without keepalive, after a device has seen a sufficiently long period of
+ * idle time, packets on our "bulk data" socket may be dropped because an
+ * overly "smart" network infrastructure decided that TCP session was stale.
+ * Note that we don't try to use this to detect "broken" tcp sessions here,
+ * these will still be handled by the DRBD effective network timeout via
+ * timeout / ko-count settings.
+ * We use this to try to keep "idle" TCP sessions "alive".
+ * Default to send a probe every 23 seconds.
+ */
+#define DRBD_KEEP_IDLE	23
+#define DRBD_KEEP_INTVL 23
+#define DRBD_KEEP_CNT	9
+static unsigned int drbd_keepcnt = DRBD_KEEP_CNT;
+module_param_named(keepcnt, drbd_keepcnt, uint, 0664);
+MODULE_PARM_DESC(keepcnt, "see tcp(7) tcp_keepalive_probes; set TCP_KEEPCNT for data sockets; default: 9");
+static unsigned int drbd_keepidle = DRBD_KEEP_IDLE;
+module_param_named(keepidle, drbd_keepidle, uint, 0664);
+MODULE_PARM_DESC(keepidle, "see tcp(7) tcp_keepalive_time; set TCP_KEEPIDLE for data sockets; default: 23s");
+static unsigned int drbd_keepintvl = DRBD_KEEP_INTVL;
+module_param_named(keepintvl, drbd_keepintvl, uint, 0664);
+MODULE_PARM_DESC(keepintvtl, "see tcp(7) tcp_keepalive_intvl; set TCP_KEEPINTVL for data sockets; default: 23s");
+
+#define DTL_CONNECTING 1
+#define DTL_LOAD_BALANCE 2
+
+struct dtl_flow;
+
+struct dtl_header {
+	u32 sequence;
+	u32 bytes;
+} __packed;
+
+struct buffer {
+	void *base;
+	void *pos;
+};
+
+struct dtl_stream {
+	unsigned int send_sequence;
+	struct dtl_flow *recv_flow;
+	unsigned int recv_sequence;
+	long rcvtimeo;
+};
+
+struct dtl_transport {
+	struct drbd_transport transport; /* Must be first! */
+	spinlock_t control_recv_lock;
+	unsigned long flags;
+	struct timer_list control_timer;
+	struct delayed_work connect_work;
+	wait_queue_head_t data_ready;
+	wait_queue_head_t write_space;
+	struct dtl_stream streams[2];
+	struct buffer rbuf;
+	int connected_paths;
+	wait_queue_head_t connected_paths_change;
+	int err;
+};
+
+struct dtl_listener {
+	struct drbd_listener listener;
+
+	struct work_struct accept_work;
+	void (*original_sk_state_change)(struct sock *sk);
+	struct socket *s_listen;
+};
+
+struct dtl_flow {
+	struct socket *sock;
+	unsigned int recv_sequence;
+	int recv_bytes; /* The number of bytes to receive before the next dtl_header */
+	struct {
+		union {
+			struct dtl_header header;
+			u8 bytes[8];
+		};
+		int avail;
+	} control_reassemble;
+
+	void (*original_sk_state_change)(struct sock *sk);
+	void (*original_sk_data_ready)(struct sock *sk);
+	void (*original_sk_write_space)(struct sock *sk);
+
+	enum drbd_stream stream_nr;
+};
+
+struct dtl_path {
+	struct drbd_path path;
+	struct dtl_flow flow[2];
+};
+
+
+static int dtl_init(struct drbd_transport *transport);
+static void dtl_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op);
+static void dtl_socket_free(struct drbd_transport *transport, struct socket **sock);
+static int dtl_prepare_connect(struct drbd_transport *transport);
+static int dtl_connect(struct drbd_transport *transport);
+static void dtl_finish_connect(struct drbd_transport *transport);
+static int dtl_recv(struct drbd_transport *transport, enum drbd_stream stream, void **buf,
+		    size_t size, int flags);
+static int dtl_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain,
+			  size_t size);
+static void dtl_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats);
+static int dtl_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf);
+static void dtl_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream,
+			     long timeout);
+static long dtl_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream);
+static int dtl_send_page(struct drbd_transport *transport, enum drbd_stream, struct page *page,
+		int offset, size_t size, unsigned int msg_flags);
+static int dtl_send_zc_bio(struct drbd_transport *, struct bio *bio);
+static bool dtl_stream_ok(struct drbd_transport *transport, enum drbd_stream stream);
+static bool dtl_hint(struct drbd_transport *transport, enum drbd_stream stream,
+		     enum drbd_tr_hints hint);
+static void dtl_debugfs_show(struct drbd_transport *transport, struct seq_file *m);
+static int dtl_add_path(struct drbd_path *path);
+static bool dtl_may_remove_path(struct drbd_path *);
+static void dtl_remove_path(struct drbd_path *);
+static void dtl_control_timer_fn(struct timer_list *t);
+static void dtl_write_space(struct sock *sk);
+static void dtl_connect_work_fn(struct work_struct *work);
+static void dtl_accept_work_fn(struct work_struct *work);
+static int dtl_set_active(struct drbd_transport *transport, bool active);
+static int dtl_path_adjust_listener(struct dtl_path *path, bool active);
+static int dtl_init_listener(struct drbd_transport *transport, const struct sockaddr *addr,
+			     struct net *net, struct drbd_listener *drbd_listener);
+static void dtl_destroy_listener(struct drbd_listener *generic_listener);
+static void dtl_set_socket_callbacks(struct dtl_transport *dtl_transport, struct dtl_flow *flow);
+
+
+static struct drbd_transport_class dtl_transport_class = {
+	.name = "lb-tcp",
+	.instance_size = sizeof(struct dtl_transport),
+	.path_instance_size = sizeof(struct dtl_path),
+	.listener_instance_size = sizeof(struct dtl_listener),
+	.ops = (struct drbd_transport_ops) {
+		.init = dtl_init,
+		.free = dtl_free,
+		.init_listener = dtl_init_listener,
+		.release_listener = dtl_destroy_listener,
+		.prepare_connect = dtl_prepare_connect,
+		.connect = dtl_connect,
+		.finish_connect = dtl_finish_connect,
+		.recv = dtl_recv,
+		.recv_pages = dtl_recv_pages,
+		.stats = dtl_stats,
+		.net_conf_change = dtl_net_conf_change,
+		.set_rcvtimeo = dtl_set_rcvtimeo,
+		.get_rcvtimeo = dtl_get_rcvtimeo,
+		.send_page = dtl_send_page,
+		.send_zc_bio = dtl_send_zc_bio,
+		.stream_ok = dtl_stream_ok,
+		.hint = dtl_hint,
+		.debugfs_show = dtl_debugfs_show,
+		.add_path = dtl_add_path,
+		.may_remove_path = dtl_may_remove_path,
+		.remove_path = dtl_remove_path,
+	},
+	.module = THIS_MODULE,
+	.list = LIST_HEAD_INIT(dtl_transport_class.list),
+};
+
+static int dtl_init(struct drbd_transport *transport)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+
+	spin_lock_init(&dtl_transport->control_recv_lock);
+
+	dtl_transport->transport.class = &dtl_transport_class;
+	timer_setup(&dtl_transport->control_timer, dtl_control_timer_fn, 0);
+
+	init_waitqueue_head(&dtl_transport->data_ready);
+	init_waitqueue_head(&dtl_transport->write_space);
+	INIT_DELAYED_WORK(&dtl_transport->connect_work, dtl_connect_work_fn);
+	dtl_transport->connected_paths = 0;
+	dtl_transport->flags = 0;
+	init_waitqueue_head(&dtl_transport->connected_paths_change);
+
+	dtl_transport->rbuf.base = (void *)__get_free_page(GFP_KERNEL);
+	dtl_transport->rbuf.pos = dtl_transport->rbuf.base;
+	if (!dtl_transport->rbuf.base)
+		return -ENOMEM;
+
+	return 0;
+}
+
+
+static void dtl_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct drbd_path *drbd_path;
+	/* free the socket specific stuff, mutexes are handled by caller */
+
+	dtl_set_active(transport, false);
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		bool was_established = test_and_clear_bit(TR_ESTABLISHED, &drbd_path->flags);
+
+		if (free_op == CLOSE_CONNECTION && was_established)
+			drbd_path_event(transport, drbd_path);
+	}
+
+	timer_delete_sync(&dtl_transport->control_timer);
+	cancel_delayed_work_sync(&dtl_transport->connect_work);
+
+	if (free_op == DESTROY_TRANSPORT) {
+		free_page((unsigned long)dtl_transport->rbuf.base);
+		dtl_transport->rbuf.base = NULL;
+	}
+}
+
+static int _dtl_send(struct dtl_transport *dtl_transport, struct dtl_flow *flow,
+		      void *buf, size_t size, unsigned int msg_flags)
+{
+	struct socket *sock = flow->sock;
+	struct kvec iov;
+	struct msghdr msg;
+	int rv, sent = 0;
+
+	/* THINK  if (signal_pending) return ... ? */
+
+	iov.iov_base = buf;
+	iov.iov_len  = size;
+
+	msg.msg_name       = NULL;
+	msg.msg_namelen    = 0;
+	msg.msg_control    = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;
+
+	do {
+		rv = kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len);
+		if (rv == -EAGAIN) {
+			struct drbd_transport *transport = &dtl_transport->transport;
+
+			if (drbd_stream_send_timed_out(transport, flow->stream_nr))
+				break;
+			continue;
+		}
+		if (rv == -EINTR) {
+			flush_signals(current);
+			rv = 0;
+		}
+		if (rv < 0)
+			break;
+		sent += rv;
+		iov.iov_base += rv;
+		iov.iov_len  -= rv;
+	} while (sent < size);
+
+	if (rv <= 0)
+		return rv;
+
+	return sent;
+}
+
+static int dtl_recv_short(struct socket *sock, void *buf, size_t size, int flags)
+{
+	struct kvec iov = {
+		.iov_base = buf,
+		.iov_len = size,
+	};
+	struct msghdr msg = {
+		.msg_flags = (flags ? flags : MSG_WAITALL | MSG_NOSIGNAL)
+	};
+
+	return kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags);
+}
+
+static void dtl_data_ready(struct sock *sk)
+{
+	struct dtl_flow *flow = sk->sk_user_data;
+	struct dtl_path *path = container_of(flow, struct dtl_path, flow[flow->stream_nr]);
+	struct dtl_transport *dtl_transport =
+		container_of(path->path.transport, struct dtl_transport, transport);
+
+	wake_up(&dtl_transport->data_ready);
+
+	flow->original_sk_data_ready(sk);
+}
+
+static int dtl_wait_data_cond(struct dtl_transport *dtl_transport,
+			      enum drbd_stream st, struct dtl_flow **rh_fl)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct dtl_stream *stream = &dtl_transport->streams[st];
+	struct drbd_path *drbd_path;
+	struct dtl_flow *flow;
+	struct tcp_sock *tp;
+	struct sock *sk;
+	int err = -ENOTCONN;
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+
+		if (!test_bit(TR_ESTABLISHED, &drbd_path->flags))
+			continue;
+		flow = &path->flow[st];
+		if (!flow->sock)
+			continue;
+		sk = flow->sock->sk;
+		tp = tcp_sk(sk);
+		if (sk->sk_state != TCP_ESTABLISHED)
+			continue;
+		if (flow->recv_sequence == stream->recv_sequence + 1)
+			goto found;
+		err = -EAGAIN;
+		if (READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq) < sizeof(struct dtl_header))
+			continue;
+		if (flow->recv_bytes)
+			continue;
+
+		*rh_fl = flow;
+		err = -EBFONT; /* Abusing strange errno to activate outer loop */
+		kref_put(&drbd_path->kref, drbd_destroy_path); /* aborting for_each_path_ref */
+		goto out;
+	}
+	if (err > 0)
+		err = -EAGAIN;
+
+	goto out;
+found:
+	kref_put(&drbd_path->kref, drbd_destroy_path); /* aborted for_each_path_ref */
+	stream->recv_sequence++;
+	stream->recv_flow = flow;
+	err = 0;
+out:
+	return err;
+}
+
+static int dtl_select_recv_flow(struct dtl_transport *dtl_transport, enum drbd_stream st,
+				struct dtl_flow **flow)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct dtl_stream *stream = &dtl_transport->streams[st];
+	long rem, timeout = stream->rcvtimeo;
+	int err;
+
+	if (stream->recv_flow) {
+		if (!stream->recv_flow->sock)
+			return -ENOTCONN;
+
+		*flow = stream->recv_flow;
+		return 0;
+	}
+
+	while (true) {
+		struct dtl_header header;
+		struct dtl_flow *rh_fl;
+
+		rem = wait_event_interruptible_timeout(dtl_transport->data_ready,
+			(err = dtl_wait_data_cond(dtl_transport, st, &rh_fl)) != -EAGAIN,
+			timeout);
+		if (rem < 0)
+			return rem;
+		if (!err)
+			break;
+		if (err != -EBFONT)
+			return err;
+
+		err = dtl_recv_short(rh_fl->sock, &header, sizeof(header), 0);
+		if (err < 0)
+			return err;
+		if (err < sizeof(header)) {
+			tr_warn(transport, "got too little %d\n", err);
+			return -EIO;
+		}
+		rh_fl->recv_sequence = be32_to_cpu(header.sequence);
+		rh_fl->recv_bytes = be32_to_cpu(header.bytes);
+		if (rh_fl->recv_sequence == stream->recv_sequence + 1) {
+			stream->recv_sequence++;
+			stream->recv_flow = rh_fl;
+			break;
+		}
+	}
+
+	*flow = stream->recv_flow;
+	return 0;
+}
+
+static void dtl_received(struct dtl_transport *dtl_transport, struct dtl_flow *flow, int size)
+{
+	if (test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags)) {
+		flow->recv_bytes -= size;
+		if (flow->recv_bytes == 0)
+			dtl_transport->streams[flow->stream_nr].recv_flow = NULL;
+	}
+}
+
+static int
+dtl_recv(struct drbd_transport *transport, enum drbd_stream st, void **buf, size_t size, int flags)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_flow *flow;
+	void *buffer;
+	int err;
+
+	err = dtl_select_recv_flow(dtl_transport, st, &flow);
+	if (err)
+		return err;
+
+	if (flags & CALLER_BUFFER) {
+		buffer = *buf;
+		err = dtl_recv_short(flow->sock, buffer, size, flags & ~CALLER_BUFFER);
+	} else if (flags & GROW_BUFFER) {
+		TR_ASSERT(transport, *buf == dtl_transport->rbuf.base);
+		buffer = dtl_transport->rbuf.pos;
+		TR_ASSERT(transport, (buffer - *buf) + size <= PAGE_SIZE);
+
+		err = dtl_recv_short(flow->sock, buffer, size, flags & ~GROW_BUFFER);
+	} else {
+		buffer = dtl_transport->rbuf.base;
+
+		err = dtl_recv_short(flow->sock, buffer, size, flags);
+		if (err > 0)
+			*buf = buffer;
+	}
+
+	if (err > 0) {
+		dtl_received(dtl_transport, flow, err);
+		dtl_transport->rbuf.pos = buffer + err;
+	}
+
+	return err;
+}
+
+static int
+_dtl_recv_page(struct dtl_transport *dtl_transport, struct page *page, int size)
+{
+	void *data = kmap_local_page(page);
+	void *pos = data;
+	struct dtl_flow *flow;
+	int err;
+
+	while (size) {
+		err = dtl_select_recv_flow(dtl_transport, DATA_STREAM, &flow);
+		if (err)
+			goto out;
+
+		err = dtl_recv_short(flow->sock, data, min(size, flow->recv_bytes), 0);
+		if (err < 0)
+			goto out;
+		size -= err;
+		pos += err;
+		dtl_received(dtl_transport, flow, err);
+	}
+	err = pos - data;
+out:
+	kunmap_local(data);
+	return err;
+}
+
+static int
+dtl_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain, size_t size)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct page *page;
+	int err;
+
+	drbd_alloc_page_chain(transport, chain, DIV_ROUND_UP(size, PAGE_SIZE), GFP_TRY);
+	page = chain->head;
+	if (!page)
+		return -ENOMEM;
+
+	page_chain_for_each(page) {
+		size_t len = min_t(int, size, PAGE_SIZE);
+
+		err = _dtl_recv_page(dtl_transport, page, len);
+		if (err < 0)
+			goto fail;
+		set_page_chain_offset(page, 0);
+		set_page_chain_size(page, len);
+		size -= err;
+	}
+	if (unlikely(size)) {
+		tr_warn(transport, "Not enough data received; missing %zu bytes\n", size);
+		err = -ENODATA;
+		goto fail;
+	}
+	return 0;
+fail:
+	drbd_free_page_chain(transport, chain);
+	return err;
+}
+
+static void dtl_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats)
+{
+	struct drbd_transport_stats s = {};
+	struct drbd_path *drbd_path;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct dtl_flow *flow = &path->flow[DATA_STREAM];
+
+		if (flow->sock) {
+			struct sock *sk = flow->sock->sk;
+			struct tcp_sock *tp = tcp_sk(sk);
+
+			s.unread_received += tp->rcv_nxt - tp->copied_seq;
+			s.unacked_send += tp->write_seq - tp->snd_una;
+			s.send_buffer_size += sk->sk_sndbuf;
+			s.send_buffer_used += sk->sk_wmem_queued;
+		}
+	}
+	rcu_read_unlock();
+
+	*stats = s;
+}
+
+static void dtl_setbufsize(struct socket *sock, unsigned int snd, unsigned int rcv)
+{
+	struct sock *sk = sock->sk;
+
+	/* open coded SO_SNDBUF, SO_RCVBUF */
+	if (snd) {
+		sk->sk_sndbuf = snd;
+		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
+		/* Wake up sending tasks if we upped the value. */
+		sk->sk_write_space(sk);
+	} else {
+		sk->sk_userlocks &= ~SOCK_SNDBUF_LOCK;
+	}
+
+	if (rcv) {
+		sk->sk_rcvbuf = rcv;
+		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
+	} else {
+		sk->sk_userlocks &= ~SOCK_RCVBUF_LOCK;
+	}
+}
+
+static bool dtl_path_cmp_addr(struct dtl_path *path)
+{
+	struct drbd_path *drbd_path = &path->path;
+	int addr_size;
+
+	addr_size = min(drbd_path->my_addr_len, drbd_path->peer_addr_len);
+	return memcmp(&drbd_path->my_addr, &drbd_path->peer_addr, addr_size) > 0;
+}
+
+static int
+dtl_try_connect(struct drbd_transport *transport, struct dtl_path *path, struct socket **ret_sock)
+{
+	const char *what;
+	struct socket *sock;
+	struct sockaddr_storage my_addr, peer_addr;
+	struct net_conf *nc;
+	int err;
+	int sndbuf_size, rcvbuf_size, connect_int;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EIO;
+	}
+	sndbuf_size = nc->sndbuf_size;
+	rcvbuf_size = nc->rcvbuf_size;
+	connect_int = nc->connect_int;
+	rcu_read_unlock();
+
+	my_addr = path->path.my_addr;
+	if (my_addr.ss_family == AF_INET6)
+		((struct sockaddr_in6 *)&my_addr)->sin6_port = 0;
+	else
+		((struct sockaddr_in *)&my_addr)->sin_port = 0; /* AF_INET & AF_SCI */
+
+	/* The network stack might change peer_addr.ss_family, so use a copy here. */
+	peer_addr = path->path.peer_addr;
+
+	what = "sock_create_kern";
+	err = sock_create_kern(path->path.net, my_addr.ss_family, SOCK_STREAM, IPPROTO_TCP,
+			       &sock);
+	if (err < 0) {
+		sock = NULL;
+		goto out;
+	}
+
+	sock->sk->sk_rcvtimeo =
+	sock->sk->sk_sndtimeo = connect_int * HZ;
+	dtl_setbufsize(sock, sndbuf_size, rcvbuf_size);
+
+	/* explicitly bind to the configured IP as source IP
+	 * for the outgoing connections.
+	 * This is needed for multihomed hosts and to be
+	 * able to use lo: interfaces for drbd.
+	 * Make sure to use 0 as port number, so linux selects
+	 * a free one dynamically.
+	 */
+	what = "bind before connect";
+	err = sock->ops->bind(sock, (struct sockaddr_unsized *) &my_addr, path->path.my_addr_len);
+	if (err < 0)
+		goto out;
+
+	/* connect may fail, peer not yet available. stay C_CONNECTING */
+	what = "connect";
+	err = sock->ops->connect(sock, (struct sockaddr_unsized *) &peer_addr,
+				   path->path.peer_addr_len, 0);
+	if (err < 0) {
+		switch (err) {
+		case -ETIMEDOUT:
+		case -EINPROGRESS:
+		case -EINTR:
+		case -ERESTARTSYS:
+		case -ECONNREFUSED:
+		case -ECONNRESET:
+		case -ENETUNREACH:
+		case -EHOSTDOWN:
+		case -EHOSTUNREACH:
+			err = -EAGAIN;
+			break;
+		case -EINVAL:
+			err = -EADDRNOTAVAIL;
+			break;
+		}
+	}
+
+out:
+	if (err < 0) {
+		if (sock)
+			sock_release(sock);
+		if (err != -EAGAIN && err != -EADDRNOTAVAIL)
+			tr_err(transport, "%s failed, err = %d\n", what, err);
+	} else {
+		*ret_sock = sock;
+	}
+
+	return err;
+}
+
+static int dtl_send_first_packet(struct dtl_transport *dtl_transport,
+				 struct dtl_flow *flow, enum drbd_packet cmd)
+{
+	struct p_header80 h;
+	int msg_flags = 0;
+	int err;
+
+	if (!flow->sock)
+		return -EIO;
+
+	if (test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags)) {
+		struct dtl_header hdr = { .sequence = 0, .bytes = cpu_to_be32(sizeof(h)) };
+
+		err = _dtl_send(dtl_transport, flow, &hdr, sizeof(hdr), msg_flags | MSG_MORE);
+		if (err < 0)
+			return err;
+	}
+
+	h.magic = cpu_to_be32(DRBD_MAGIC);
+	h.command = cpu_to_be16(cmd);
+	h.length = 0;
+
+	err = _dtl_send(dtl_transport, flow, &h, sizeof(h), msg_flags);
+
+	return err;
+}
+
+/**
+ * dtl_socket_free() - Free the socket
+ * @transport:	DRBD transport.
+ * @sock:	pointer to the pointer to the socket.
+ */
+static void dtl_socket_free(struct drbd_transport *transport, struct socket **sock)
+{
+	struct socket *s = xchg(sock, NULL);
+
+	if (!s)
+		return;
+
+	synchronize_rcu();
+	kernel_sock_shutdown(s, SHUT_RDWR);
+	sock_release(s);
+}
+
+/**
+ * dtl_socket_ok_or_free() - Free the socket if its connection is not okay
+ * @transport:	DRBD transport.
+ * @sock:	pointer to the pointer to the socket.
+ */
+static bool dtl_socket_ok_or_free(struct drbd_transport *transport, struct socket **sock)
+{
+	struct socket *s;
+	bool rv;
+
+	rcu_read_lock();
+	s = rcu_dereference(*sock);
+	rv = s && s->sk->sk_state == TCP_ESTABLISHED;
+	rcu_read_unlock();
+
+	if (s && !rv)
+		dtl_socket_free(transport, sock);
+
+	return rv;
+}
+
+static bool _dtl_path_established(struct drbd_transport *transport, struct dtl_path *path)
+{
+	return	dtl_socket_ok_or_free(transport, &path->flow[DATA_STREAM].sock) &&
+		dtl_socket_ok_or_free(transport, &path->flow[CONTROL_STREAM].sock);
+}
+
+static bool dtl_deactivate_other_paths(struct dtl_path *path)
+{
+	struct drbd_transport *transport = path->path.transport;
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	bool active = test_and_clear_bit(DTL_CONNECTING, &dtl_transport->flags);
+	struct drbd_path *drbd_path;
+
+	if (active) {
+		for_each_path_ref(drbd_path, transport)
+			dtl_path_adjust_listener(path, false);
+	}
+
+	return active;
+}
+
+static bool dtl_path_established(struct drbd_transport *transport, struct dtl_path *path)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	bool lb = test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags);
+	struct drbd_path *drbd_path = &path->path;
+	struct net_conf *nc;
+	enum drbd_stream i;
+	bool established;
+	int timeout;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	timeout = (nc->sock_check_timeo ?: nc->ping_timeo) * HZ / 10;
+	rcu_read_unlock();
+	schedule_timeout_interruptible(timeout);
+
+	established = _dtl_path_established(transport, path);
+
+	if (established && !lb) {
+		established = dtl_deactivate_other_paths(path);
+
+		if (!established) {
+			dtl_socket_free(transport, &path->flow[DATA_STREAM].sock);
+			dtl_socket_free(transport, &path->flow[CONTROL_STREAM].sock);
+		}
+	}
+
+	if (!established) {
+		if (test_and_clear_bit(TR_ESTABLISHED, &drbd_path->flags)) {
+			dtl_transport->connected_paths--;
+			drbd_path_event(transport, drbd_path);
+		}
+	} else if (!test_and_set_bit(TR_ESTABLISHED, &drbd_path->flags)) {
+		dtl_transport->connected_paths++;
+
+		for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+			if (lb) {
+				path->flow[i].recv_sequence = 0;
+				path->flow[i].recv_bytes = 0;
+			} else {
+				path->flow[i].recv_sequence = 1;
+				path->flow[i].recv_bytes = INT_MAX;
+				dtl_transport->streams[i].recv_flow = &path->flow[i];
+			}
+		}
+		wake_up(&dtl_transport->data_ready);
+		drbd_put_listener(drbd_path);
+		dtl_set_socket_callbacks(dtl_transport, &path->flow[DATA_STREAM]);
+		dtl_set_socket_callbacks(dtl_transport, &path->flow[CONTROL_STREAM]);
+		drbd_path_event(transport, drbd_path);
+	}
+
+	return established;
+}
+
+static void unregister_state_change(struct sock *sk, struct dtl_listener *listener)
+{
+	write_lock_bh(&sk->sk_callback_lock);
+	sk->sk_state_change = listener->original_sk_state_change;
+	sk->sk_user_data = NULL;
+	write_unlock_bh(&sk->sk_callback_lock);
+}
+
+
+static int dtl_receive_first_packet(struct dtl_transport *dtl_transport, struct dtl_path *path,
+				    struct socket *sock)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct p_header80 header;
+	struct net_conf *nc;
+	int err;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EIO;
+	}
+	sock->sk->sk_rcvtimeo = nc->ping_timeo * 4 * HZ / 10;
+	rcu_read_unlock();
+
+	if (test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags)) {
+		struct dtl_header hdr;
+
+		err = dtl_recv_short(sock, &hdr, sizeof(hdr), 0);
+		if (err != sizeof(hdr)) {
+			if (err >= 0)
+				err = -EIO;
+			return err;
+		}
+	}
+	err = dtl_recv_short(sock, &header, sizeof(header), 0);
+	if (err != sizeof(header)) {
+		if (err >= 0)
+			err = -EIO;
+		return err;
+	}
+	if (header.magic != cpu_to_be32(DRBD_MAGIC)) {
+		tr_err(transport, "Wrong magic value 0x%08x in receive_first_packet\n",
+			 be32_to_cpu(header.magic));
+		return -EINVAL;
+	}
+	return be16_to_cpu(header.command);
+}
+
+static struct dtl_flow *dtl_control_next_flow_in_seq(struct dtl_transport *dtl_transport)
+{
+	struct dtl_stream *stream = &dtl_transport->streams[CONTROL_STREAM];
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct drbd_path *drbd_path;
+	struct dtl_flow *flow;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+
+		flow = &path->flow[CONTROL_STREAM];
+		if (flow->sock &&
+		    flow->recv_sequence == stream->recv_sequence + 1 && flow->recv_bytes > 0) {
+			struct sock *sk = flow->sock->sk;
+			struct tcp_sock *tp = tcp_sk(sk);
+
+			if (READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq))
+				goto found;
+		}
+	}
+	flow = NULL;
+found:
+	rcu_read_unlock();
+	return flow;
+}
+
+static int dtl_control_tcp_input(read_descriptor_t *rd_desc, struct sk_buff *skb,
+				 unsigned int offset, size_t len)
+{
+	struct dtl_flow *flow = rd_desc->arg.data;
+	struct dtl_path *path = container_of(flow, struct dtl_path, flow[flow->stream_nr]);
+	struct dtl_transport *dtl_transport =
+		container_of(path->path.transport, struct dtl_transport, transport);
+	struct dtl_stream *stream = &dtl_transport->streams[CONTROL_STREAM];
+	struct drbd_transport *transport = &dtl_transport->transport;
+	int overall_avail, avail, consumed = 0;
+	struct drbd_const_buffer buffer;
+	struct skb_seq_state seq;
+
+	if (flow->recv_bytes &&
+	    flow->recv_sequence != stream->recv_sequence + 1)
+		return 0;
+
+	skb_prepare_seq_read(skb, offset, skb->len, &seq);
+	do {
+		/*
+		 * skb_seq_read() returns the length of the block assigned to buffer. This might
+		 * be more than is actually ready, so we ensure we only mark as available what
+		 * is ready.
+		 */
+		overall_avail = skb_seq_read(consumed, &buffer.buffer, &seq);
+		if (!overall_avail)
+			break;
+		avail = min_t(int, overall_avail, len - consumed);
+		while (avail) {
+			if (flow->recv_bytes == 0) {
+				const struct dtl_header *hdr = (struct dtl_header *)buffer.buffer;
+				int size = sizeof(struct dtl_header);
+				bool hdr_frag = flow->control_reassemble.avail || avail < size;
+
+				if (hdr_frag) {
+					int cra = flow->control_reassemble.avail;
+
+					size = min(size - cra, avail);
+					memcpy(flow->control_reassemble.bytes + cra, hdr, size);
+					flow->control_reassemble.avail += size;
+				}
+				consumed += size;
+				avail -= size;
+				buffer.buffer += size;
+				if (hdr_frag) {
+					if (flow->control_reassemble.avail < sizeof(*hdr))
+						continue;
+					hdr = &flow->control_reassemble.header;
+					flow->control_reassemble.avail = 0;
+				}
+
+				flow->recv_sequence = be32_to_cpu(hdr->sequence);
+				flow->recv_bytes = be32_to_cpu(hdr->bytes);
+				if (flow->recv_sequence != stream->recv_sequence + 1)
+					goto out;
+			}
+			buffer.avail = min(flow->recv_bytes, avail);
+			if (!buffer.avail)
+				continue;
+			consumed += buffer.avail;
+			avail -= buffer.avail;
+			if (test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags))
+				flow->recv_bytes -= buffer.avail;
+			drbd_control_data_ready(transport, &buffer);
+			if (flow->recv_bytes == 0)
+				stream->recv_sequence++;
+		}
+	} while (consumed < len);
+out:
+	skb_abort_seq_read(&seq);
+	return consumed;
+}
+
+static void dtl_control_data_ready(struct sock *sk)
+{
+	struct dtl_flow *flow = sk->sk_user_data;
+	struct dtl_path *path = container_of(flow, struct dtl_path, flow[flow->stream_nr]);
+	struct dtl_transport *dtl_transport =
+		container_of(path->path.transport, struct dtl_transport, transport);
+
+	read_descriptor_t rd_desc = {
+		.count = 1,
+		.arg = { .data = flow },
+	};
+	mod_timer(&dtl_transport->control_timer, jiffies + sk->sk_rcvtimeo);
+
+	spin_lock_bh(&dtl_transport->control_recv_lock);
+	tcp_read_sock(sk, &rd_desc, dtl_control_tcp_input);
+
+	/* in case another flow became the next in sequence */
+	while ((flow = dtl_control_next_flow_in_seq(dtl_transport))) {
+		sk = flow->sock->sk;
+		rd_desc.arg.data = flow;
+		tcp_read_sock(sk, &rd_desc, dtl_control_tcp_input);
+	}
+	spin_unlock_bh(&dtl_transport->control_recv_lock);
+}
+
+static void dtl_control_state_change(struct sock *sk)
+{
+	struct dtl_flow *flow = sk->sk_user_data;
+	struct dtl_path *path = container_of(flow, struct dtl_path, flow[flow->stream_nr]);
+	struct dtl_transport *dtl_transport =
+		container_of(path->path.transport, struct dtl_transport, transport);
+	struct drbd_transport *transport = &dtl_transport->transport;
+
+	switch (sk->sk_state) {
+	case TCP_FIN_WAIT1:
+	case TCP_CLOSE_WAIT:
+	case TCP_CLOSE:
+	case TCP_LAST_ACK:
+	case TCP_CLOSING:
+		drbd_control_event(transport, CLOSED_BY_PEER);
+		break;
+	default:
+		tr_warn(transport, "unhandled state %d\n", sk->sk_state);
+	}
+
+	flow->original_sk_state_change(sk);
+}
+
+static void dtl_incoming_connection(struct sock *sk)
+{
+	struct dtl_listener *listener = sk->sk_user_data;
+	void (*state_change)(struct sock *sk);
+
+	state_change = listener->original_sk_state_change;
+	state_change(sk);
+
+	spin_lock(&listener->listener.waiters_lock);
+	listener->listener.pending_accepts++;
+	spin_unlock(&listener->listener.waiters_lock);
+	kref_get(&listener->listener.kref);
+	if (!schedule_work(&listener->accept_work))
+		kref_put(&listener->listener.kref, drbd_listener_destroy);
+}
+
+static void dtl_control_timer_fn(struct timer_list *t)
+{
+	struct dtl_transport *dtl_transport = timer_container_of(dtl_transport, t, control_timer);
+	struct drbd_transport *transport = &dtl_transport->transport;
+
+	drbd_control_event(transport, TIMEOUT);
+}
+
+static void dtl_destroy_listener(struct drbd_listener *generic_listener)
+{
+	struct dtl_listener *listener =
+		container_of(generic_listener, struct dtl_listener, listener);
+
+	if (!listener->s_listen)
+		return;
+	unregister_state_change(listener->s_listen->sk, listener);
+	sock_release(listener->s_listen);
+}
+
+static int dtl_init_listener(struct drbd_transport *transport,
+			     const struct sockaddr *addr,
+			     struct net *net,
+			     struct drbd_listener *drbd_listener)
+{
+	int err, sndbuf_size, rcvbuf_size, addr_len;
+	struct sockaddr_storage my_addr;
+	struct dtl_listener *listener = container_of(drbd_listener, struct dtl_listener, listener);
+	struct socket *s_listen;
+	struct net_conf *nc;
+	const char *what = "";
+
+
+	INIT_WORK(&listener->accept_work, dtl_accept_work_fn);
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EINVAL;
+	}
+	sndbuf_size = nc->sndbuf_size;
+	rcvbuf_size = nc->rcvbuf_size;
+	rcu_read_unlock();
+
+	my_addr = *(struct sockaddr_storage *)addr;
+
+	err = sock_create_kern(net, my_addr.ss_family, SOCK_STREAM, IPPROTO_TCP, &s_listen);
+	if (err < 0) {
+		s_listen = NULL;
+		what = "sock_create_kern";
+		goto out;
+	}
+
+	s_listen->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
+	dtl_setbufsize(s_listen, sndbuf_size, rcvbuf_size);
+
+	addr_len = addr->sa_family == AF_INET6 ? sizeof(struct sockaddr_in6)
+		: sizeof(struct sockaddr_in);
+
+	err = s_listen->ops->bind(s_listen, (struct sockaddr_unsized *)&my_addr, addr_len);
+	if (err < 0) {
+		what = "bind before listen";
+		goto out;
+	}
+
+	listener->s_listen = s_listen;
+	write_lock_bh(&s_listen->sk->sk_callback_lock);
+	listener->original_sk_state_change = s_listen->sk->sk_state_change;
+	s_listen->sk->sk_state_change = dtl_incoming_connection;
+	s_listen->sk->sk_user_data = listener;
+	write_unlock_bh(&s_listen->sk->sk_callback_lock);
+
+	err = s_listen->ops->listen(s_listen, DRBD_PEERS_MAX * 2);
+	if (err < 0) {
+		what = "listen";
+		goto out;
+	}
+
+	listener->listener.listen_addr = my_addr;
+
+	return 0;
+out:
+	if (s_listen)
+		sock_release(s_listen);
+
+	if (err < 0 &&
+	    err != -EAGAIN && err != -EINTR && err != -ERESTARTSYS && err != -EADDRINUSE &&
+	    err != -EADDRNOTAVAIL)
+		tr_err(transport, "%s failed, err = %d\n", what, err);
+
+	return err;
+}
+
+static void dtl_setup_socket(struct dtl_transport *dtl_transport, struct socket *sock,
+			     struct dtl_flow *flow)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	bool use_for_data = flow->stream_nr == DATA_STREAM;
+	struct net_conf *nc;
+	long timeout = HZ;
+
+	sock->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
+	/* We are a block device, we are in the write-out path,
+	 * we may need memory to facilitate memory reclaim
+	 */
+	sock->sk->sk_use_task_frag = false;
+	sock->sk->sk_allocation = GFP_ATOMIC;
+	sk_set_memalloc(sock->sk);
+
+	sock->sk->sk_priority = use_for_data ? TC_PRIO_INTERACTIVE_BULK : TC_PRIO_INTERACTIVE;
+	tcp_sock_set_nodelay(sock->sk);
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (nc)
+		timeout = nc->timeout * HZ / 10;
+	rcu_read_unlock();
+
+	sock->sk->sk_sndtimeo = timeout;
+	sock_set_keepalive(sock->sk);
+
+	if (use_for_data) {
+		if (drbd_keepidle)
+			tcp_sock_set_keepidle(sock->sk, drbd_keepidle);
+		if (drbd_keepcnt)
+			tcp_sock_set_keepcnt(sock->sk, drbd_keepcnt);
+		if (drbd_keepintvl)
+			tcp_sock_set_keepintvl(sock->sk, drbd_keepintvl);
+	}
+	flow->sock = sock;
+}
+
+static void dtl_set_socket_callbacks(struct dtl_transport *dtl_transport, struct dtl_flow *flow)
+{
+	bool use_for_data = flow->stream_nr == DATA_STREAM;
+	struct socket *sock = flow->sock;
+
+	write_lock_bh(&sock->sk->sk_callback_lock);
+	if (sock->sk->sk_data_ready != dtl_data_ready &&
+	    sock->sk->sk_data_ready != dtl_control_data_ready) {
+		sock->sk->sk_user_data = flow;
+		flow->original_sk_data_ready = sock->sk->sk_data_ready;
+		if (use_for_data) {
+			flow->original_sk_write_space = sock->sk->sk_write_space;
+			sock->sk->sk_data_ready = dtl_data_ready;
+			sock->sk->sk_write_space = dtl_write_space;
+		} else {
+			flow->original_sk_state_change = sock->sk->sk_state_change;
+			sock->sk->sk_data_ready = dtl_control_data_ready;
+			sock->sk->sk_state_change = dtl_control_state_change;
+		}
+	}
+	write_unlock_bh(&sock->sk->sk_callback_lock);
+}
+
+static void dtl_do_first_packet(struct dtl_transport *dtl_transport, struct dtl_path *path,
+				struct socket *s)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	int fp;
+
+	fp = dtl_receive_first_packet(dtl_transport, path, s);
+
+	dtl_socket_ok_or_free(transport, &path->flow[DATA_STREAM].sock);
+	dtl_socket_ok_or_free(transport, &path->flow[CONTROL_STREAM].sock);
+
+	switch (fp) {
+	case P_INITIAL_DATA:
+		if (path->flow[DATA_STREAM].sock)
+			tr_warn(transport, "initial packet S crossed\n");
+		dtl_socket_free(transport, &path->flow[DATA_STREAM].sock);
+		dtl_setup_socket(dtl_transport, s, &path->flow[DATA_STREAM]);
+		break;
+	case P_INITIAL_META:
+		if (path->flow[CONTROL_STREAM].sock)
+			tr_warn(transport, "initial packet M crossed\n");
+		dtl_socket_free(transport, &path->flow[CONTROL_STREAM].sock);
+		dtl_setup_socket(dtl_transport, s, &path->flow[CONTROL_STREAM]);
+		break;
+	default:
+		tr_warn(transport, "Error receiving initial packet. err = %d\n", fp);
+		kernel_sock_shutdown(s, SHUT_RDWR);
+		sock_release(s);
+		return;
+	}
+
+	if (dtl_path_established(transport, path)) {
+		if (dtl_transport->connected_paths == 1 && fp == P_INITIAL_META)
+			set_bit(RESOLVE_CONFLICTS, &transport->flags);
+	} else {
+		/* successful accept, not yet both -> speed up next connect attempt */
+		if (test_bit(DTL_CONNECTING, &dtl_transport->flags))
+			mod_delayed_work(system_wq, &dtl_transport->connect_work, 1);
+	}
+
+	if (!dtl_transport->err && fp < 0)
+		dtl_transport->err = fp;
+
+	wake_up_all(&dtl_transport->connected_paths_change);
+}
+
+static void dtl_accept_work_fn(struct work_struct *work)
+{
+	struct dtl_listener *listener = container_of(work, struct dtl_listener, accept_work);
+	struct dtl_transport *dtl_transport;
+	struct drbd_path *drbd_path;
+	struct dtl_path *path;
+	struct socket *s;
+	int err, tries = 5;
+
+	while (listener->listener.pending_accepts && tries > 0) {
+		struct sockaddr_storage peer_addr;
+
+		s = NULL;
+		err = kernel_accept(listener->s_listen, &s, O_NONBLOCK);
+
+		tries--;
+		if (err || !s)
+			continue;
+
+		unregister_state_change(s->sk, listener);
+		s->ops->getname(s, (struct sockaddr *)&peer_addr, 2);
+
+		spin_lock_bh(&listener->listener.waiters_lock);
+		listener->listener.pending_accepts--;
+		drbd_path = drbd_find_path_by_addr(&listener->listener, &peer_addr);
+		if (drbd_path)
+			kref_get(&drbd_path->kref);
+		spin_unlock_bh(&listener->listener.waiters_lock);
+
+		if (!drbd_path) {
+			switch (peer_addr.ss_family) {
+				struct sockaddr_in6 *from_sin6;
+				struct sockaddr_in *from_sin;
+
+			case AF_INET6:
+				from_sin6 = (struct sockaddr_in6 *)&peer_addr;
+				pr_notice("drbd: Closing unexpected connection from %pI6\n",
+					  &from_sin6->sin6_addr);
+				break;
+			default:
+				from_sin = (struct sockaddr_in *)&peer_addr;
+				pr_notice("drbd: Closing unexpected connection from %pI4\n",
+					  &from_sin->sin_addr);
+				break;
+			}
+			kernel_sock_shutdown(s, SHUT_RDWR);
+			sock_release(s);
+			continue;
+		}
+
+		path = container_of(drbd_path, struct dtl_path, path);
+		dtl_transport = container_of(path->path.transport, struct dtl_transport, transport);
+
+		/* Do not add sockets to a path after DTL_CONNECTING was cleared! */
+		if (test_bit(DTL_CONNECTING, &dtl_transport->flags)) {
+			dtl_do_first_packet(dtl_transport, path, s);
+		} else {
+			kernel_sock_shutdown(s, SHUT_RDWR);
+			sock_release(s);
+		}
+		kref_put(&drbd_path->kref, drbd_destroy_path);
+	}
+	kref_put(&listener->listener.kref, drbd_listener_destroy);
+}
+
+static void dtl_connect_work_fn(struct work_struct *work)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(work, struct dtl_transport, connect_work.work);
+	struct drbd_transport *transport = &dtl_transport->transport;
+	int connected_paths = dtl_transport->connected_paths;
+	int err, nr_paths = 0, to_connect = 0, err_ret = 0;
+	struct drbd_path *drbd_path;
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct socket *s = NULL;
+		bool use_for_data;
+
+		nr_paths++;
+		if (_dtl_path_established(transport, path))
+			continue;
+
+		to_connect++;
+		err = dtl_try_connect(transport, path, &s);
+		if (err < 0) {
+			if (err != -EAGAIN && err != -EADDRNOTAVAIL && !err_ret)
+				err_ret = err;
+			continue;
+		}
+
+		dtl_socket_ok_or_free(transport, &path->flow[DATA_STREAM].sock);
+		dtl_socket_ok_or_free(transport, &path->flow[CONTROL_STREAM].sock);
+
+		if (!path->flow[DATA_STREAM].sock && !path->flow[CONTROL_STREAM].sock) {
+			use_for_data = dtl_path_cmp_addr(path);
+		} else if (!path->flow[DATA_STREAM].sock) {
+			use_for_data = true;
+		} else {
+			if (path->flow[CONTROL_STREAM].sock) {
+				tr_err(transport, "Logic error in conn_connect()\n");
+				dtl_socket_free(transport, &s);
+				continue;
+			}
+			use_for_data = false;
+		}
+
+		if (use_for_data) {
+			struct dtl_flow tmp_flow = path->flow[DATA_STREAM];
+
+			tmp_flow.sock = s;
+			err = dtl_send_first_packet(dtl_transport, &tmp_flow, P_INITIAL_DATA);
+			dtl_setup_socket(dtl_transport, s, &path->flow[DATA_STREAM]);
+
+		} else {
+			struct dtl_flow tmp_flow = path->flow[CONTROL_STREAM];
+
+			tmp_flow.sock = s;
+			err = dtl_send_first_packet(dtl_transport, &tmp_flow, P_INITIAL_META);
+			dtl_setup_socket(dtl_transport, s, &path->flow[CONTROL_STREAM]);
+		}
+
+		if (dtl_path_established(transport, path)) {
+			if (dtl_transport->connected_paths == 1 && !use_for_data)
+				clear_bit(RESOLVE_CONFLICTS, &transport->flags);
+		}
+	}
+
+	if (to_connect && test_bit(DTL_CONNECTING, &dtl_transport->flags)) {
+		struct net_conf *nc;
+		int connect_int = HZ;
+
+		rcu_read_lock();
+		nc = rcu_dereference(transport->net_conf);
+		if (nc)
+			connect_int = nc->connect_int;
+		rcu_read_unlock();
+
+		schedule_delayed_work(&dtl_transport->connect_work, connect_int * HZ);
+	}
+
+	if (nr_paths == to_connect && err_ret && !dtl_transport->err)
+		dtl_transport->err = err_ret;
+
+	if (connected_paths != dtl_transport->connected_paths || err_ret)
+		wake_up_all(&dtl_transport->connected_paths_change);
+}
+
+static int dtl_path_adjust_listener(struct dtl_path *path, bool active)
+{
+	struct drbd_path *drbd_path = &path->path;
+	struct drbd_listener *listener = READ_ONCE(drbd_path->listener);
+	int err = 0;
+
+	if (!active && listener)
+		drbd_put_listener(drbd_path);
+	else if (active && !listener)
+		err = drbd_get_listener(drbd_path);
+
+	return err;
+}
+
+static int dtl_set_active(struct drbd_transport *transport, bool active)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct drbd_path *drbd_path;
+
+	if (active)
+		set_bit(DTL_CONNECTING, &dtl_transport->flags);
+	else
+		clear_bit(DTL_CONNECTING, &dtl_transport->flags);
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		enum drbd_stream i;
+		int err;
+
+		for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+			if (path->flow[i].sock && path->flow[i].original_sk_state_change) {
+				write_lock_bh(&path->flow[i].sock->sk->sk_callback_lock);
+				path->flow[i].sock->sk->sk_state_change =
+					path->flow[i].original_sk_state_change;
+				write_unlock_bh(&path->flow[i].sock->sk->sk_callback_lock);
+			}
+
+			dtl_socket_free(transport, &path->flow[i].sock);
+		}
+
+		err = dtl_path_adjust_listener(path, active);
+
+		if (err) {
+			kref_put(&drbd_path->kref, drbd_destroy_path);
+			return err;
+		}
+	}
+	return 0;
+}
+
+static int dtl_prepare_connect(struct drbd_transport *transport)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+
+	dtl_transport->connected_paths = 0;
+	dtl_transport->err = 0;
+	flush_signals(current);
+	timer_delete_sync(&dtl_transport->control_timer);
+	dtl_transport->err = dtl_set_active(transport, true);
+
+	return dtl_transport->err;
+}
+
+static int dtl_connect(struct drbd_transport *transport)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	int err;
+
+	schedule_work(&dtl_transport->connect_work.work);
+	err = wait_event_interruptible(dtl_transport->connected_paths_change,
+				       dtl_transport->connected_paths > 0);
+
+	if (err < 0)
+		dtl_transport->err = err;
+
+	return dtl_transport->err;
+}
+
+static void dtl_finish_connect(struct drbd_transport *transport)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	bool lb = test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags);
+	enum drbd_stream i;
+
+	if (dtl_transport->err) {
+		dtl_set_active(transport, false);
+		cancel_delayed_work_sync(&dtl_transport->connect_work);
+	}
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+		dtl_transport->streams[i].send_sequence = 1;
+		dtl_transport->streams[i].recv_sequence = 0;
+		if (lb)
+			dtl_transport->streams[i].recv_flow = NULL;
+	}
+}
+
+static int dtl_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct drbd_path *drbd_path;
+
+	if (new_net_conf->load_balance_paths)
+		__set_bit(DTL_LOAD_BALANCE, &dtl_transport->flags);
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct socket *data_sock = path->flow[DATA_STREAM].sock;
+		struct socket *control_sock = path->flow[CONTROL_STREAM].sock;
+
+		if (data_sock)
+			dtl_setbufsize(data_sock, new_net_conf->sndbuf_size,
+				       new_net_conf->rcvbuf_size);
+
+		if (control_sock)
+			dtl_setbufsize(control_sock, new_net_conf->sndbuf_size,
+				       new_net_conf->rcvbuf_size);
+	}
+
+	return 0;
+}
+
+static void dtl_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream st, long timeout)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_stream *stream = &dtl_transport->streams[st];
+	struct drbd_path *drbd_path;
+
+	stream->rcvtimeo = timeout;
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct socket *sock = path->flow[st].sock;
+
+		if (!sock)
+			continue;
+
+		sock->sk->sk_rcvtimeo = timeout;
+
+		if (st == CONTROL_STREAM)
+			mod_timer(&dtl_transport->control_timer, jiffies + timeout);
+	}
+}
+
+static long dtl_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream st)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_stream *stream = &dtl_transport->streams[st];
+
+	return stream->rcvtimeo;
+}
+
+static bool dtl_stream_ok(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	struct drbd_path *drbd_path;
+	bool established = false;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct socket *sock = path->flow[stream].sock;
+
+		established = sock && sock->sk && sock->sk->sk_state == TCP_ESTABLISHED;
+		if (established)
+			break;
+	}
+	rcu_read_unlock();
+
+	return established;
+}
+
+static void dtl_write_space(struct sock *sk)
+{
+	struct dtl_flow *flow = sk->sk_user_data;
+	struct dtl_path *path = container_of(flow, struct dtl_path, flow[flow->stream_nr]);
+	struct dtl_transport *dtl_transport =
+		container_of(path->path.transport, struct dtl_transport, transport);
+
+	flow->original_sk_write_space(sk);
+	wake_up(&dtl_transport->write_space);
+}
+
+static int dtl_select_send_flow_cond(struct dtl_transport *dtl_transport,
+			      enum drbd_stream st, struct dtl_flow **result)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	int best_wmem = INT_MAX;
+	struct drbd_path *drbd_path;
+	struct dtl_flow *best = NULL;
+	bool empty;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct dtl_flow *flow = &path->flow[st];
+
+		if (!test_bit(TR_ESTABLISHED, &drbd_path->flags))
+			continue;
+
+		if (flow->sock) {
+			struct sock *sk = flow->sock->sk;
+			int wmem = sk_stream_min_wspace(sk);
+			/* int wmem_queued = READ_ONCE(sk->sk_wmem_queued); */
+
+			if (st == DATA_STREAM) {
+				if (wmem < best_wmem && wmem < sk->sk_sndbuf) {
+					best = flow;
+					best_wmem = wmem;
+				}
+			} else {
+				if (wmem < sk->sk_sndbuf)
+					best = flow;
+				/* Only use first established control flow. */
+				break;
+			}
+		}
+	}
+	empty = list_empty(&transport->paths);
+	rcu_read_unlock();
+
+	if (!best) {
+		if (empty)
+			return -ENOTCONN;
+
+		set_bit(NET_CONGESTED, &dtl_transport->transport.flags);
+		return -EAGAIN;
+	}
+	clear_bit(NET_CONGESTED, &dtl_transport->transport.flags);
+
+	*result = best;
+	return 0;
+}
+
+static int dtl_select_send_flow(struct dtl_transport *dtl_transport,
+				enum drbd_stream st, struct dtl_flow **result)
+{
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct net_conf *nc;
+	long rem, timeout = HZ;
+	int err;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (nc)
+		timeout = nc->timeout * HZ / 10;
+	rcu_read_unlock();
+
+	rem = wait_event_interruptible_timeout(dtl_transport->write_space,
+		(err = dtl_select_send_flow_cond(dtl_transport, st, result)) != -EAGAIN,
+		timeout);
+
+	return rem < 0 ? rem : err;
+}
+
+static int _dtl_send_page(struct dtl_transport *dtl_transport, struct dtl_flow *flow,
+			  struct page *page, int offset, size_t size, unsigned int msg_flags)
+{
+	struct msghdr msg = { .msg_flags = msg_flags | MSG_NOSIGNAL | MSG_SPLICE_PAGES };
+	struct drbd_transport *transport = &dtl_transport->transport;
+	struct socket *sock = flow->sock;
+	struct bio_vec bvec;
+	int len = size;
+	int err = -EIO;
+
+	do {
+		int sent;
+
+		bvec_set_page(&bvec, page, len, offset);
+		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
+
+		sent = sock_sendmsg(sock, &msg);
+		if (sent <= 0) {
+			if (sent == -EAGAIN) {
+				if (drbd_stream_send_timed_out(transport, flow->stream_nr))
+					break;
+				continue;
+			}
+			tr_warn(transport, "%s: size=%d len=%d sent=%d\n",
+			     __func__, (int)size, len, sent);
+			if (sent < 0)
+				err = sent;
+			break;
+		}
+		len    -= sent;
+		offset += sent;
+		/* NOTE: it may take up to twice the socket timeout to have it
+		 * return -EAGAIN, the first timeout will likely happen with a
+		 * partial send, masking the timeout.  Maybe we want to export
+		 * drbd_stream_should_continue_after_partial_send(transport, stream)
+		 * and add that to the while() condition below.
+		 */
+	} while (len > 0 /* THINK && peer_device->repl_state[NOW] >= L_ESTABLISHED */);
+
+	if (len == 0)
+		err = 0;
+
+	return err;
+}
+
+static int dtl_send_page(struct drbd_transport *transport, enum drbd_stream stream,
+			 struct page *page, int offset, size_t size, unsigned int msg_flags)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_header header;
+	struct dtl_flow *flow;
+	int err;
+
+	err = dtl_select_send_flow(dtl_transport, stream, &flow);
+	if (err)
+		return err;
+
+	if (test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags)) {
+		header.sequence = cpu_to_be32(dtl_transport->streams[stream].send_sequence++);
+		header.bytes = cpu_to_be32(size);
+
+		err = _dtl_send(dtl_transport, flow, &header, sizeof(header), msg_flags | MSG_MORE);
+		if (err < 0)
+			goto out;
+	}
+	err = _dtl_send_page(dtl_transport, flow, page, offset, size, msg_flags);
+
+out:
+	return err;
+}
+
+static int dtl_bio_chunk_size_available(struct bio *bio, int wmem_available,
+		struct bvec_iter *iter_scan)
+{
+	struct bio_vec bvec;
+	int chunk = 0;
+
+	while (chunk < wmem_available && iter_scan->bi_size) {
+		bvec = bio_iter_iovec(bio, *iter_scan);
+		chunk += bvec.bv_len;
+		bio_advance_iter_single(bio, iter_scan, bvec.bv_len);
+	}
+
+	return chunk;
+}
+
+static int dtl_send_bio_pages(struct dtl_transport *dtl_transport, struct dtl_flow *flow,
+		struct bio *bio, struct bvec_iter *iter, int chunk)
+{
+	struct bio_vec bvec;
+
+	while (chunk > 0 && iter->bi_size) {
+		int err;
+
+		bvec = bio_iter_iovec(bio, *iter);
+		err = _dtl_send_page(dtl_transport, flow, bvec.bv_page,
+				bvec.bv_offset, bvec.bv_len,
+				bio_iter_last(bvec, *iter) ? 0 : MSG_MORE);
+		if (err)
+			return err;
+		chunk -= bvec.bv_len;
+		bio_advance_iter_single(bio, iter, bvec.bv_len);
+	}
+
+	return 0;
+}
+
+static int dtl_send_zc_bio(struct drbd_transport *transport, struct bio *bio)
+{
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_stream *stream = &dtl_transport->streams[DATA_STREAM];
+	bool lb = test_bit(DTL_LOAD_BALANCE, &dtl_transport->flags);
+	struct bvec_iter iter_scan = bio->bi_iter;
+	struct bvec_iter iter = bio->bi_iter;
+	int err;
+
+	if (!bio_has_data(bio)) /* e.g. REQ_OP_DISCARD */
+		return 0;
+
+	do {
+		struct dtl_flow *flow;
+		struct sock *sk;
+		int chunk, wmem_available;
+
+		err = dtl_select_send_flow(dtl_transport, DATA_STREAM, &flow);
+		if (err)
+			goto out;
+
+		sk = flow->sock->sk;
+		wmem_available = READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
+
+		if (lb && iter.bi_size > wmem_available) {
+			chunk = dtl_bio_chunk_size_available(bio, wmem_available, &iter_scan);
+		} else {
+			chunk = iter.bi_size;
+		}
+
+		if (lb) {
+			struct dtl_header header;
+
+			header.sequence = cpu_to_be32(stream->send_sequence++);
+			header.bytes = cpu_to_be32(chunk);
+			err = _dtl_send(dtl_transport, flow, &header, sizeof(header), MSG_MORE);
+			if (err < 0)
+				goto out;
+		}
+
+		err = dtl_send_bio_pages(dtl_transport, flow, bio, &iter, chunk);
+		if (err)
+			goto out;
+	} while (iter.bi_size);
+out:
+	return err;
+}
+
+static bool dtl_hint(struct drbd_transport *transport, enum drbd_stream stream,
+		enum drbd_tr_hints hint)
+{
+	struct drbd_path *drbd_path;
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+		struct socket *sock = path->flow[stream].sock;
+
+		if (!sock)
+			continue;
+
+		switch (hint) {
+		case CORK:
+			tcp_sock_set_cork(sock->sk, true);
+			break;
+		case UNCORK:
+			tcp_sock_set_cork(sock->sk, false);
+			break;
+		case NODELAY:
+			tcp_sock_set_nodelay(sock->sk);
+			break;
+		case NOSPACE:
+			if (sock->sk->sk_socket)
+				set_bit(SOCK_NOSPACE, &sock->sk->sk_socket->flags);
+			break;
+		case QUICKACK:
+			tcp_sock_set_quickack(sock->sk, 2);
+			break;
+		}
+	}
+
+	return true;
+}
+
+static void dtl_debugfs_show_stream(struct seq_file *m, struct socket *sock)
+{
+	struct sock *sk = sock->sk;
+	struct tcp_sock *tp = tcp_sk(sk);
+
+	seq_printf(m, "unread receive buffer: %u Byte\n",
+		   tp->rcv_nxt - tp->copied_seq);
+	seq_printf(m, "unacked send buffer: %u Byte\n",
+		   tp->write_seq - tp->snd_una);
+	seq_printf(m, "send buffer size: %u Byte\n", sk->sk_sndbuf);
+	seq_printf(m, "send buffer used: %u Byte\n", sk->sk_wmem_queued);
+}
+
+static void dtl_debugfs_show(struct drbd_transport *transport, struct seq_file *m)
+{
+	struct drbd_path *drbd_path;
+
+	/* BUMP me if you change the file format/content/presentation */
+	seq_printf(m, "v: %u\n\n", 0);
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		enum drbd_stream i;
+
+		seq_printf(m, "%pI4 - %pI4:\n",
+			   &((struct sockaddr_in *)&drbd_path->my_addr)->sin_addr,
+			   &((struct sockaddr_in *)&drbd_path->peer_addr)->sin_addr);
+
+		for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+			struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+			struct socket *sock = path->flow[i].sock;
+
+			if (!sock)
+				continue;
+			seq_printf(m, "%s stream\n", i == DATA_STREAM ? "data" : "control");
+			dtl_debugfs_show_stream(m, sock);
+		}
+		seq_puts(m, "\n");
+	}
+	rcu_read_unlock();
+}
+
+static int dtl_add_path(struct drbd_path *drbd_path)
+{
+	struct drbd_transport *transport = drbd_path->transport;
+	struct dtl_transport *dtl_transport =
+		container_of(transport, struct dtl_transport, transport);
+	struct dtl_path *path = container_of(drbd_path, struct dtl_path, path);
+	bool active = test_bit(DTL_CONNECTING, &dtl_transport->flags);
+	enum drbd_stream i;
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		path->flow[i].stream_nr = i;
+
+	clear_bit(TR_ESTABLISHED, &drbd_path->flags);
+
+	return dtl_path_adjust_listener(path, active);
+}
+
+static bool dtl_may_remove_path(struct drbd_path *drbd_path)
+{
+	return !test_bit(TR_ESTABLISHED, &drbd_path->flags);
+}
+
+static void dtl_remove_path(struct drbd_path *drbd_path)
+{
+	drbd_put_listener(drbd_path);
+}
+
+static int __init dtl_initialize(void)
+{
+	return drbd_register_transport_class(&dtl_transport_class,
+					     DRBD_TRANSPORT_API_VERSION,
+					     sizeof(struct drbd_transport));
+}
+
+static void __exit dtl_cleanup(void)
+{
+	drbd_unregister_transport_class(&dtl_transport_class);
+}
+
+module_init(dtl_initialize)
+module_exit(dtl_cleanup)
diff --git a/drivers/block/drbd/drbd_transport_rdma.c b/drivers/block/drbd/drbd_transport_rdma.c
new file mode 100644
index 0000000000000..fa6dcde43d571
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport_rdma.c
@@ -0,0 +1,3531 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+   drbd_transport_rdma.c
+
+   This file is part of DRBD.
+
+   Copyright (C) 2014-2021, LINBIT HA-Solutions GmbH.
+*/
+
+#undef pr_fmt
+#define pr_fmt(fmt)	"drbd_rdma: " fmt
+
+#ifndef SENDER_COMPACTS_BVECS
+/* My benchmarking shows a limit of 30 MB/s
+ * with the current implementation of this idea.
+ * cpu bound, perf top shows mainly get_page/put_page.
+ * Without this, using the plain send_page,
+ * I achieve > 400 MB/s on the same system.
+ * => disable for now, improve later.
+ */
+#define SENDER_COMPACTS_BVECS 0
+#endif
+
+#include <linux/module.h>
+#include <linux/sched/signal.h>
+#include <linux/bio.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_cm.h>
+#include <rdma/ib_cm.h>
+#include <linux/interrupt.h>
+#include <linux/drbd_genl_api.h>
+#include <linux/drbd_protocol.h>
+#include "drbd_transport.h"
+#include "linux/drbd_config.h" /* for REL_VERSION */
+
+/* Nearly all data transfer uses the send/receive semantics. No need to
+   actually use RDMA WRITE / READ.
+
+   Only for DRBD's remote read (P_DATA_REQUEST and P_DATA_REPLY) a
+   RDMA WRITE would make a lot of sense:
+     Right now the recv_dless_read() function in DRBD is one of the few
+     remaining callers of recv(,,CALLER_BUFFER). This in turn needs a
+     memcpy().
+
+   The block_id field (64 bit) could be re-labelled to be the RKEY for
+   an RDMA WRITE. The P_DATA_REPLY packet will then only deliver the
+   news that the RDMA WRITE was executed...
+
+
+   Flow Control
+   ============
+
+   If the receiving machine can not keep up with the data rate it needs to
+   slow down the sending machine. In order to do so we keep track of the
+   number of rx_descs the peer has posted (peer_rx_descs).
+
+   If one player posts new rx_descs it tells the peer about it with a
+   dtr_flow_control packet. Those packet get never delivered to the
+   DRBD above us.
+*/
+
+MODULE_AUTHOR("Roland Kammerer <roland.kammerer@linbit.com>");
+MODULE_AUTHOR("Philipp Reisner <philipp.reisner@linbit.com>");
+MODULE_AUTHOR("Lars Ellenberg <lars.ellenberg@linbit.com>");
+MODULE_DESCRIPTION("RDMA transport layer for DRBD");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(REL_VERSION);
+
+int allocation_size;
+/* module_param(allocation_size, int, 0664);
+   MODULE_PARM_DESC(allocation_size, "Allocation size for receive buffers (page size of peer)");
+
+   That needs to be implemented in dtr_create_rx_desc() and in dtr_recv() and dtr_recv_pages() */
+
+/* If no recvbuf_size or sendbuf_size is configured use 1M plus two pages for the DATA_STREAM */
+/* Actually it is not a buffer, but the number of tx_descs or rx_descs we allow,
+   very comparable to the socket sendbuf and recvbuf sizes */
+#define RDMA_DEF_BUFFER_SIZE (DRBD_MAX_BIO_SIZE + 2 * PAGE_SIZE)
+
+/* If we can send less than 8 packets, we consider the transport as congested. */
+#define DESCS_LOW_LEVEL 8
+
+/* Assuming that a singe 4k write should be at the highest scatterd over 8
+   pages. I.e. has no parts smaller than 512 bytes.
+   Arbitrary assumption. It seems that Mellanox hardware can do up to 29
+   ppc64 page size might be 64k */
+#if (PAGE_SIZE / 512) > 28
+# define DTR_MAX_TX_SGES 28
+#else
+# define DTR_MAX_TX_SGES (PAGE_SIZE / 512)
+#endif
+
+#define DTR_MAGIC ((u32)0x5257494E)
+
+struct dtr_flow_control {
+	uint32_t magic;
+	uint32_t new_rx_descs[2];
+	uint32_t rx_desc_stolen_from_stream;
+} __packed;
+
+/* These numbers are sent within the immediate data value to identify
+   if the packet is a data, and control or a (transport private) flow_control
+   message */
+enum dtr_stream_nr {
+	ST_DATA = DATA_STREAM,
+	ST_CONTROL = CONTROL_STREAM,
+	ST_FLOW_CTRL
+};
+
+/* IB_WR_SEND_WITH_IMM and IB_WR_RDMA_WRITE_WITH_IMM
+
+   both transfer user data and a 32bit value with is delivered at the receiving
+   to the event handler of the completion queue. I.e. that can be used to queue
+   the incoming messages to different streams.
+
+   dtr_imm:
+   In order to support folding the data and the control stream into one RDMA
+   connection we use the stream field of dtr_imm: DATA_STREAM, CONTROL_STREAM
+   and FLOW_CONTROL.
+   To be able to order the messages on the receiving side before delivering them
+   to the upper layers we use a sequence number.
+
+   */
+#define SEQUENCE_BITS 30
+union dtr_immediate {
+	struct {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+		unsigned int sequence:SEQUENCE_BITS;
+		unsigned int stream:2;
+#elif defined(__BIG_ENDIAN_BITFIELD)
+		unsigned int stream:2;
+		unsigned int sequence:SEQUENCE_BITS;
+#else
+# error "this endianness is not supported"
+#endif
+	};
+	unsigned int i;
+};
+
+
+enum dtr_state_bits {
+	DSB_CONNECT_REQ,
+	DSB_CONNECTING,
+	DSB_CONNECTED,
+	DSB_ERROR,
+};
+
+#define DSM_CONNECT_REQ   (1 << DSB_CONNECT_REQ)
+#define DSM_CONNECTING    (1 << DSB_CONNECTING)
+#define DSM_CONNECTED     (1 << DSB_CONNECTED)
+#define DSM_ERROR         (1 << DSB_ERROR)
+
+enum dtr_alloc_rdma_res_causes {
+	IB_ALLOC_PD,
+	IB_CREATE_CQ_RX,
+	IB_CREATE_CQ_TX,
+	IB_REQ_NOTIFY_CQ_RX,
+	IB_REQ_NOTIFY_CQ_TX,
+	RDMA_CREATE_QP,
+	IB_GET_DMA_MR
+};
+
+struct dtr_rx_desc {
+	struct page *page;
+	struct list_head list;
+	int size;
+	unsigned int sequence;
+	struct dtr_cm *cm;
+	struct ib_sge sge;
+};
+
+struct dtr_tx_desc {
+	union {
+		struct page *page;
+		void *data;
+		struct bio *bio;
+	};
+	enum {
+		SEND_PAGE,
+		SEND_MSG,
+		SEND_BIO,
+	} type;
+	int nr_sges;
+	union dtr_immediate imm;
+	struct ib_sge sge[]; /* must be last! */
+};
+
+struct dtr_flow {
+	struct dtr_path *path;
+
+	atomic_t tx_descs_posted;
+	int tx_descs_max; /* derived from net_conf->sndbuf_size. Do not change after alloc. */
+	atomic_t peer_rx_descs; /* peer's receive window in number of rx descs */
+
+	atomic_t rx_descs_posted;
+	int rx_descs_max;  /* derived from net_conf->rcvbuf_size. Do not change after alloc. */
+
+	atomic_t rx_descs_allocated;
+	int rx_descs_want_posted;
+	atomic_t rx_descs_known_to_peer;
+};
+
+enum connect_state_enum {
+	PCS_INACTIVE,
+	PCS_REQUEST_ABORT,
+	PCS_FINISHING = PCS_REQUEST_ABORT,
+	PCS_CONNECTING,
+};
+
+struct dtr_connect_state {
+	struct delayed_work retry_connect_work;
+	atomic_t active_state; /* trying to establish a connection*/
+	atomic_t passive_state; /* listening for a connection */
+	wait_queue_head_t wq;
+	bool active; /* active = established by connect ; !active = established by accept */
+};
+
+struct dtr_path {
+	struct drbd_path path;
+
+	struct dtr_connect_state cs;
+
+	struct dtr_cm *cm; /* RCU'd and kref in cm */
+
+	struct dtr_flow flow[2];
+	spinlock_t send_flow_control_lock;
+	struct tasklet_struct flow_control_tasklet;
+	struct work_struct refill_rx_descs_work;
+};
+
+struct dtr_stream {
+	wait_queue_head_t send_wq;
+	wait_queue_head_t recv_wq;
+
+	/* for recv() to keep track of the current rx_desc:
+	 * - whenever the bytes_left of the current rx_desc == 0, we know that all data
+	 *   is consumed, and get a new rx_desc from the completion queue, and set
+	 *   current rx_desc accodingly.
+	 */
+	struct {
+		struct dtr_rx_desc *desc;
+		void *pos;
+		int bytes_left;
+	} current_rx;
+
+	unsigned long unread; /* unread received; unit: bytes */
+	struct list_head rx_descs;
+	spinlock_t rx_descs_lock;
+
+	long send_timeout;
+	long recv_timeout;
+
+	unsigned int tx_sequence;
+	unsigned int rx_sequence;
+	struct dtr_transport *rdma_transport;
+};
+
+struct dtr_transport {
+	struct drbd_transport transport;
+	struct dtr_stream stream[2];
+	int rx_allocation_size;
+	int sges_max;
+	bool active; /* connect() returned no error. I.e. C_CONNECTING or C_CONNECTED */
+
+	/* per transport rate limit state for diagnostic messages.
+	 * maybe: one for debug, one for warning, one for error?
+	 * maybe: move into generic drbd_transport an tr_{warn,err,debug}().
+	 */
+	struct ratelimit_state rate_limit;
+
+	struct timer_list control_timer;
+	atomic_t first_path_connect_err;
+	struct completion connected;
+
+	struct tasklet_struct control_tasklet;
+};
+
+struct dtr_cm {
+	struct kref kref;
+	struct rdma_cm_id *id;
+	struct dtr_path *path;
+
+	struct ib_cq *recv_cq;
+	struct ib_cq *send_cq;
+	struct ib_pd *pd;
+
+	unsigned long state; /* DSB bits / DSM masks */
+	wait_queue_head_t state_wq;
+	unsigned long last_sent_jif;
+	atomic_t tx_descs_posted;
+	struct timer_list tx_timeout;
+
+	struct work_struct tx_timeout_work;
+	struct work_struct connect_work;
+	struct work_struct establish_work;
+	struct work_struct disconnect_work;
+
+	struct list_head error_rx_descs;
+	spinlock_t error_rx_descs_lock;
+	struct work_struct end_rx_work;
+	struct work_struct end_tx_work;
+
+	struct dtr_transport *rdma_transport;
+	struct rcu_head rcu;
+};
+
+struct dtr_listener {
+	struct drbd_listener listener;
+
+	struct dtr_cm cm;
+};
+
+static int dtr_init(struct drbd_transport *transport);
+static void dtr_free(struct drbd_transport *transport, enum drbd_tr_free_op);
+static int dtr_prepare_connect(struct drbd_transport *transport);
+static int dtr_connect(struct drbd_transport *transport);
+static void dtr_finish_connect(struct drbd_transport *transport);
+static int dtr_recv(struct drbd_transport *transport, enum drbd_stream stream, void **buf, size_t size, int flags);
+static void dtr_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats);
+static int dtr_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf);
+static void dtr_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout);
+static long dtr_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream);
+static int dtr_send_page(struct drbd_transport *transport, enum drbd_stream stream, struct page *page,
+		int offset, size_t size, unsigned msg_flags);
+static int dtr_send_zc_bio(struct drbd_transport *, struct bio *bio);
+static int dtr_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain, size_t size);
+static bool dtr_stream_ok(struct drbd_transport *transport, enum drbd_stream stream);
+static bool dtr_hint(struct drbd_transport *transport, enum drbd_stream stream, enum drbd_tr_hints hint);
+static void dtr_debugfs_show(struct drbd_transport *, struct seq_file *m);
+static int dtr_add_path(struct drbd_path *path);
+static bool dtr_may_remove_path(struct drbd_path *path);
+static void dtr_remove_path(struct drbd_path *path);
+
+static int dtr_create_cm_id(struct dtr_cm *cm_context, struct net *net);
+static bool dtr_path_ok(struct dtr_path *path);
+static bool dtr_transport_ok(struct drbd_transport *transport);
+static int __dtr_post_tx_desc(struct dtr_cm *, struct dtr_tx_desc *);
+static int dtr_post_tx_desc(struct dtr_transport *, struct dtr_tx_desc *);
+static int dtr_repost_tx_desc(struct dtr_cm *old_cm, struct dtr_tx_desc *tx_desc);
+static int dtr_repost_rx_desc(struct dtr_cm *cm, struct dtr_rx_desc *rx_desc);
+static bool dtr_receive_rx_desc(struct dtr_transport *, enum drbd_stream,
+				struct dtr_rx_desc **);
+static void dtr_recycle_rx_desc(struct drbd_transport *transport,
+				enum drbd_stream stream,
+				struct dtr_rx_desc **pp_rx_desc,
+				gfp_t gfp_mask);
+static void dtr_refill_rx_desc(struct dtr_transport *rdma_transport,
+			       enum drbd_stream stream);
+static void dtr_free_tx_desc(struct dtr_cm *cm, struct dtr_tx_desc *tx_desc);
+static void dtr_free_rx_desc(struct dtr_rx_desc *rx_desc);
+static void dtr_cma_disconnect_work_fn(struct work_struct *work);
+static void dtr_disconnect_path(struct dtr_path *path);
+static void __dtr_disconnect_path(struct dtr_path *path);
+static int dtr_init_flow(struct dtr_path *path, enum drbd_stream stream);
+static int dtr_cm_alloc_rdma_res(struct dtr_cm *cm);
+static void __dtr_refill_rx_desc(struct dtr_path *path, enum drbd_stream stream);
+static int dtr_send_flow_control_msg(struct dtr_path *path, gfp_t gfp_mask);
+static struct dtr_cm *dtr_path_get_cm_connected(struct dtr_path *path);
+static void dtr_destroy_cm(struct kref *kref);
+static void dtr_destroy_cm_keep_id(struct kref *kref);
+static int dtr_activate_path(struct dtr_path *path);
+static void dtr_end_tx_work_fn(struct work_struct *work);
+static void dtr_end_rx_work_fn(struct work_struct *work);
+static void dtr_cma_retry_connect(struct dtr_path *path, struct dtr_cm *failed_cm);
+static void dtr_tx_timeout_fn(struct timer_list *t);
+static void dtr_control_timer_fn(struct timer_list *t);
+static void dtr_tx_timeout_work_fn(struct work_struct *work);
+static void dtr_cma_connect_work_fn(struct work_struct *work);
+static struct dtr_rx_desc *dtr_next_rx_desc(struct dtr_stream *rdma_stream);
+static void dtr_control_tasklet_fn(struct tasklet_struct *t);
+static int dtr_init_listener(struct drbd_transport *transport, const struct sockaddr *addr,
+			     struct net *net, struct drbd_listener *drbd_listener);
+static void dtr_destroy_listener(struct drbd_listener *generic_listener);
+
+
+static struct drbd_transport_class rdma_transport_class = {
+	.name = "rdma",
+	.instance_size = sizeof(struct dtr_transport),
+	.path_instance_size = sizeof(struct dtr_path),
+	.listener_instance_size = sizeof(struct dtr_listener),
+	.ops = (struct drbd_transport_ops) {
+		.init = dtr_init,
+		.free = dtr_free,
+		.init_listener = dtr_init_listener,
+		.release_listener = dtr_destroy_listener,
+		.prepare_connect = dtr_prepare_connect,
+		.connect = dtr_connect,
+		.finish_connect = dtr_finish_connect,
+		.recv = dtr_recv,
+		.stats = dtr_stats,
+		.net_conf_change = dtr_net_conf_change,
+		.set_rcvtimeo = dtr_set_rcvtimeo,
+		.get_rcvtimeo = dtr_get_rcvtimeo,
+		.send_page = dtr_send_page,
+		.send_zc_bio = dtr_send_zc_bio,
+		.recv_pages = dtr_recv_pages,
+		.stream_ok = dtr_stream_ok,
+		.hint = dtr_hint,
+		.debugfs_show = dtr_debugfs_show,
+		.add_path = dtr_add_path,
+		.may_remove_path = dtr_may_remove_path,
+		.remove_path = dtr_remove_path,
+	},
+	.module = THIS_MODULE,
+	.list = LIST_HEAD_INIT(rdma_transport_class.list),
+};
+
+static struct rdma_conn_param dtr_conn_param = {
+	.responder_resources = 1,
+	.initiator_depth = 1,
+	.retry_count = 10,
+	.rnr_retry_count  = 7,
+};
+
+static u32 dtr_cm_to_lkey(struct dtr_cm *cm)
+{
+	return cm->pd->local_dma_lkey;
+}
+
+static void dtr_re_init_stream(struct dtr_stream *rdma_stream)
+{
+	struct drbd_transport *transport = &rdma_stream->rdma_transport->transport;
+
+	rdma_stream->current_rx.pos = NULL;
+	rdma_stream->current_rx.bytes_left = 0;
+
+	rdma_stream->tx_sequence = 1;
+	rdma_stream->rx_sequence = 1;
+	rdma_stream->unread = 0;
+
+	TR_ASSERT(transport, list_empty(&rdma_stream->rx_descs));
+	TR_ASSERT(transport, rdma_stream->current_rx.desc == NULL);
+}
+
+static void dtr_init_stream(struct dtr_stream *rdma_stream,
+			    struct drbd_transport *transport)
+{
+	rdma_stream->current_rx.desc = NULL;
+
+	rdma_stream->recv_timeout = MAX_SCHEDULE_TIMEOUT;
+	rdma_stream->send_timeout = MAX_SCHEDULE_TIMEOUT;
+
+	init_waitqueue_head(&rdma_stream->recv_wq);
+	init_waitqueue_head(&rdma_stream->send_wq);
+	rdma_stream->rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+
+	INIT_LIST_HEAD(&rdma_stream->rx_descs);
+	spin_lock_init(&rdma_stream->rx_descs_lock);
+
+	dtr_re_init_stream(rdma_stream);
+}
+
+static int dtr_init(struct drbd_transport *transport)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	int i;
+
+	transport->class = &rdma_transport_class;
+
+	rdma_transport->rx_allocation_size = allocation_size;
+	rdma_transport->active = false;
+	rdma_transport->sges_max = DTR_MAX_TX_SGES;
+
+	ratelimit_state_init(&rdma_transport->rate_limit, 5*HZ, 4);
+	timer_setup(&rdma_transport->control_timer, dtr_control_timer_fn, 0);
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		dtr_init_stream(&rdma_transport->stream[i], transport);
+
+	tasklet_setup(&rdma_transport->control_tasklet, dtr_control_tasklet_fn);
+
+	return 0;
+}
+
+static void dtr_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct drbd_path *drbd_path;
+	int i;
+
+	rdma_transport->active = false;
+
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		struct dtr_path *path = container_of(drbd_path, struct dtr_path, path);
+
+		__dtr_disconnect_path(path);
+	}
+
+	/* Free the rx_descs that where received and not consumed. */
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		struct dtr_stream *rdma_stream = &rdma_transport->stream[i];
+		struct dtr_rx_desc *rx_desc, *tmp;
+		LIST_HEAD(rx_descs);
+
+		dtr_free_rx_desc(rdma_stream->current_rx.desc);
+		rdma_stream->current_rx.desc = NULL;
+
+		spin_lock_irq(&rdma_stream->rx_descs_lock);
+		list_splice_init(&rdma_stream->rx_descs, &rx_descs);
+		spin_unlock_irq(&rdma_stream->rx_descs_lock);
+
+		list_for_each_entry_safe(rx_desc, tmp, &rx_descs, list)
+			dtr_free_rx_desc(rx_desc);
+	}
+
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		struct dtr_path *path = container_of(drbd_path, struct dtr_path, path);
+		struct dtr_cm *cm;
+
+		cm = xchg(&path->cm, NULL); // RCU xchg
+		if (cm)
+			kref_put(&cm->kref, dtr_destroy_cm);
+	}
+
+	timer_delete_sync(&rdma_transport->control_timer);
+
+	if (free_op == DESTROY_TRANSPORT) {
+		list_for_each_entry(drbd_path, &transport->paths, list) {
+			struct dtr_path *path = container_of(drbd_path, struct dtr_path, path);
+
+			flush_delayed_work(&path->cs.retry_connect_work);
+		}
+
+		/* The transport object itself is embedded into a conneciton.
+		   Do not free it here! The function should better be called
+		   uninit. */
+	}
+}
+
+static void dtr_control_timer_fn(struct timer_list *t)
+{
+	struct dtr_transport *rdma_transport = timer_container_of(rdma_transport, t, control_timer);
+	struct drbd_transport *transport = &rdma_transport->transport;
+
+	drbd_control_event(transport, TIMEOUT);
+}
+
+static int dtr_send(struct dtr_path *path, void *buf, size_t size, gfp_t gfp_mask)
+{
+	struct ib_device *device;
+	struct dtr_tx_desc *tx_desc;
+	struct dtr_cm *cm;
+	void *send_buffer;
+	int err = -ECONNRESET;
+
+	// pr_info("%s: dtr_send() size = %d data[0]:%lx\n", rdma_stream->name, (int)size, *(unsigned long*)buf);
+
+	cm = dtr_path_get_cm_connected(path);
+	if (!cm)
+		goto out;
+
+	err = -ENOMEM;
+	tx_desc = kzalloc(sizeof(*tx_desc) + sizeof(struct ib_sge), gfp_mask);
+	if (!tx_desc)
+		goto out_put;
+
+	send_buffer = kmalloc(size, gfp_mask);
+	if (!send_buffer) {
+		kfree(tx_desc);
+		goto out_put;
+	}
+	memcpy(send_buffer, buf, size);
+
+	device = cm->id->device;
+	tx_desc->type = SEND_MSG;
+	tx_desc->data = send_buffer;
+	tx_desc->nr_sges = 1;
+	tx_desc->sge[0].addr = ib_dma_map_single(device, send_buffer, size, DMA_TO_DEVICE);
+	err = ib_dma_mapping_error(device, tx_desc->sge[0].addr);
+	if (err) {
+		kfree(tx_desc);
+		kfree(send_buffer);
+		goto out_put;
+	}
+
+	tx_desc->sge[0].lkey = dtr_cm_to_lkey(cm);
+	tx_desc->sge[0].length = size;
+	tx_desc->imm = (union dtr_immediate)
+		{ .stream = ST_FLOW_CTRL, .sequence = 0 };
+
+	err = __dtr_post_tx_desc(cm, tx_desc);
+	if (err)
+		dtr_free_tx_desc(cm, tx_desc);
+
+out_put:
+	kref_put(&cm->kref, dtr_destroy_cm);
+out:
+	return err;
+}
+
+
+static int dtr_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain, size_t size)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[DATA_STREAM];
+	struct page *page, *head = NULL, *tail = NULL;
+	int i = 0;
+
+	if (!dtr_transport_ok(transport))
+		return -ECONNRESET;
+
+	// pr_info("%s: in recv_pages, size: %zu\n", rdma_stream->name, size);
+	TR_ASSERT(transport, rdma_stream->current_rx.bytes_left == 0);
+	dtr_recycle_rx_desc(transport, DATA_STREAM, &rdma_stream->current_rx.desc, GFP_NOIO);
+	dtr_refill_rx_desc(rdma_transport, DATA_STREAM);
+
+	while (size) {
+		struct dtr_rx_desc *rx_desc = NULL;
+		long t;
+
+		t = wait_event_interruptible_timeout(rdma_stream->recv_wq,
+					dtr_receive_rx_desc(rdma_transport, DATA_STREAM, &rx_desc),
+					rdma_stream->recv_timeout);
+
+		if (t <= 0) {
+			/*
+			 * Cannot give back pages that may still be in use!
+			 * (More reason why we only have one rx_desc per page,
+			 * and don't get_page() in dtr_create_rx_desc).
+			 */
+			drbd_free_pages(transport, head);
+			return t == 0 ? -EAGAIN : -EINTR;
+		}
+
+		page = rx_desc->page;
+		/* put_page() if we would get_page() in
+		 * dtr_create_rx_desc().  but we don't. We return the page
+		 * chain to the user, which is supposed to give it back to
+		 * drbd_free_pages() eventually. */
+		rx_desc->page = NULL;
+		size -= rx_desc->size;
+
+		/* If the sender did dtr_send_page every bvec of a bio with
+		 * unaligned bvecs (as xfs often creates), rx_desc->size and
+		 * offset may well be not the PAGE_SIZE and 0 we hope for.
+		 */
+		if (tail) {
+			/* See also dtr_create_rx_desc().
+			 * For PAGE_SIZE > 4k, we may create several RR per page.
+			 * We cannot link a page to itself, though.
+			 *
+			 * Adding to size would be easy enough.
+			 * But what do we do about possible holes?
+			 * FIXME
+			 */
+			BUG_ON(page == tail);
+
+			set_page_chain_next(tail, page);
+			tail = page;
+		} else
+			head = tail = page;
+
+		set_page_chain_offset(page, 0);
+		set_page_chain_size(page, rx_desc->size);
+
+		atomic_dec(&rx_desc->cm->path->flow[DATA_STREAM].rx_descs_allocated);
+		dtr_free_rx_desc(rx_desc);
+
+		i++;
+		dtr_refill_rx_desc(rdma_transport, DATA_STREAM);
+	}
+
+	// pr_info("%s: rcvd %d pages\n", rdma_stream->name, i);
+	chain->head = head;
+	chain->nr_pages = i;
+	return 0;
+}
+
+static int _dtr_recv(struct drbd_transport *transport, enum drbd_stream stream,
+		     void **buf, size_t size, int flags)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[stream];
+	struct dtr_rx_desc *rx_desc = NULL;
+	void *buffer;
+
+	if (flags & GROW_BUFFER) {
+		/* Since transport_rdma always returns the full, requested amount
+		   of data, DRBD should never call with GROW_BUFFER! */
+		tr_err(transport, "Called with GROW_BUFFER\n");
+		return -EINVAL;
+	} else if (rdma_stream->current_rx.bytes_left == 0) {
+		long t;
+
+		dtr_recycle_rx_desc(transport, stream, &rdma_stream->current_rx.desc, GFP_NOIO);
+		if (flags & MSG_DONTWAIT) {
+			t = dtr_receive_rx_desc(rdma_transport, stream, &rx_desc);
+		} else {
+			t = wait_event_interruptible_timeout(rdma_stream->recv_wq,
+						dtr_receive_rx_desc(rdma_transport, stream, &rx_desc),
+						rdma_stream->recv_timeout);
+		}
+
+		if (t <= 0)
+			return t == 0 ? -EAGAIN : -EINTR;
+
+		// pr_info("%s: got a new page with size: %d\n", rdma_stream->name, rx_desc->size);
+		buffer = page_address(rx_desc->page);
+		rdma_stream->current_rx.desc = rx_desc;
+		rdma_stream->current_rx.pos = buffer + size;
+		rdma_stream->current_rx.bytes_left = rx_desc->size - size;
+		if (rdma_stream->current_rx.bytes_left < 0)
+			tr_warn(transport,
+				"new, requesting more (%zu) than available (%d)\n", size, rx_desc->size);
+
+		if (flags & CALLER_BUFFER)
+			memcpy(*buf, buffer, size);
+		else
+			*buf = buffer;
+
+		// pr_info("%s: recv completely new fine, returning size on\n", rdma_stream->name);
+		// pr_info("%s: rx_count: %d\n", rdma_stream->name, rdma_stream->rx_descs_posted);
+
+		return size;
+	} else { /* return next part */
+		// pr_info("recv next part on %s\n", rdma_stream->name);
+		buffer = rdma_stream->current_rx.pos;
+		rdma_stream->current_rx.pos += size;
+
+		if (rdma_stream->current_rx.bytes_left < size) {
+			tr_err(transport,
+			       "requested more than left! bytes_left = %d, size = %zu\n",
+					rdma_stream->current_rx.bytes_left, size);
+			rdma_stream->current_rx.bytes_left = 0; /* 0 left == get new entry */
+		} else {
+			rdma_stream->current_rx.bytes_left -= size;
+			// pr_info("%s: old_rx left: %d\n", rdma_stream->name, rdma_stream->current_rx.bytes_left);
+		}
+
+		if (flags & CALLER_BUFFER)
+			memcpy(*buf, buffer, size);
+		else
+			*buf = buffer;
+
+		// pr_info("%s: recv next part fine, returning size\n", rdma_stream->name);
+		return size;
+	}
+
+	return 0;
+}
+
+static int dtr_recv(struct drbd_transport *transport, enum drbd_stream stream, void **buf, size_t size, int flags)
+{
+	struct dtr_transport *rdma_transport;
+	int err;
+
+	if (!transport)
+		return -ECONNRESET;
+
+	rdma_transport = container_of(transport, struct dtr_transport, transport);
+
+	if (!dtr_transport_ok(transport))
+		return -ECONNRESET;
+
+	err = _dtr_recv(transport, stream, buf, size, flags);
+
+	dtr_refill_rx_desc(rdma_transport, stream);
+	return err;
+}
+
+static void dtr_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_path *path;
+	int sb_size = 0, sb_used = 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &transport->paths, path.list) {
+		struct dtr_flow *flow = &path->flow[DATA_STREAM];
+
+		sb_size += flow->tx_descs_max;
+		sb_used += atomic_read(&flow->tx_descs_posted);
+	}
+	rcu_read_unlock();
+
+	/* these are used by the sender, guess we should them get right */
+	stats->send_buffer_size = sb_size * DRBD_SOCKET_BUFFER_SIZE;
+	stats->send_buffer_used = sb_used * DRBD_SOCKET_BUFFER_SIZE;
+
+	/* these two for debugfs */
+	stats->unread_received = rdma_transport->stream[DATA_STREAM].unread;
+	stats->unacked_send = stats->send_buffer_used;
+
+}
+
+/* The following functions (at least)
+   dtr_path_established_work_fn(),
+   dtr_cma_accept_work_fn(), dtr_cma_accept(),
+   dtr_cma_retry_connect_work_fn(),
+   dtr_cma_retry_connect(),
+   dtr_cma_connect_fail_work_fn(), dtr_cma_connect(),
+   dtr_cma_disconnect_work_fn(), dtr_cma_disconnect(),
+   dtr_cma_event_handler()
+
+   are called from worker context or are callbacks from rdma_cm's context.
+
+   We need to make sure the path does not go away in the meantime.
+ */
+
+static int dtr_path_prepare(struct dtr_path *path, struct dtr_cm *cm, bool active)
+{
+	struct dtr_cm *cm2;
+	int i, err;
+
+	cm2 = cmpxchg(&path->cm, NULL, cm); // RCU xchg
+	if (cm2) {
+		/*
+		 * The caller needs to hold a ref on cm. dtr_path_prepare()
+		 * gifts that reference to the path. If setting the pointer in
+		 * the path fails, we have to put one ref of cm.
+		 */
+		kref_put(&cm->kref, dtr_destroy_cm);
+		return -ENOENT;
+	}
+
+	path->cs.active = active;
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		dtr_init_flow(path, i);
+
+	err = dtr_cm_alloc_rdma_res(cm);
+
+	return err;
+}
+
+static struct dtr_cm *__dtr_path_get_cm(struct dtr_path *path)
+{
+	struct dtr_cm *cm;
+
+	cm = rcu_dereference(path->cm);
+	if (cm && !kref_get_unless_zero(&cm->kref))
+		cm = NULL;
+	return cm;
+}
+
+static struct dtr_cm *dtr_path_get_cm(struct dtr_path *path)
+{
+	struct dtr_cm *cm;
+
+	rcu_read_lock();
+	cm = __dtr_path_get_cm(path);
+	rcu_read_unlock();
+	return cm;
+}
+
+static struct dtr_cm *dtr_path_get_cm_connected(struct dtr_path *path)
+{
+	struct dtr_cm *cm;
+
+	cm = dtr_path_get_cm(path);
+	if (cm && cm->state != DSM_CONNECTED) {
+		kref_put(&cm->kref, dtr_destroy_cm);
+		cm = NULL;
+	}
+	return cm;
+}
+
+static void dtr_path_established_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, establish_work);
+	struct dtr_path *path = cm->path;
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_connect_state *cs = &path->cs;
+	int i, p, err;
+
+
+	err = cm != path->cm;
+	kref_put(&cm->kref, dtr_destroy_cm);
+	if (err)
+		return;
+
+	p = atomic_cmpxchg(&cs->passive_state, PCS_CONNECTING, PCS_FINISHING);
+	if (p < PCS_CONNECTING)
+		goto out;
+
+	path->cm->state = DSM_CONNECTED;
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		__dtr_refill_rx_desc(path, i);
+	err = dtr_send_flow_control_msg(path, GFP_NOIO);
+	if (err > 0)
+		err = 0;
+	if (err)
+		tr_err(transport, "sending first flow_control_msg() failed\n");
+
+	schedule_timeout(HZ / 4);
+	if (!dtr_path_ok(path)) {
+		if (path->cs.active)
+			dtr_cma_retry_connect(path, path->cm);
+		return;
+	}
+
+	p = atomic_cmpxchg(&rdma_transport->first_path_connect_err, 1, err);
+	if (p == 1) {
+		if (cs->active)
+			set_bit(RESOLVE_CONFLICTS, &transport->flags);
+		else
+			clear_bit(RESOLVE_CONFLICTS, &transport->flags);
+		complete(&rdma_transport->connected);
+	}
+
+	set_bit(TR_ESTABLISHED, &path->path.flags);
+	drbd_path_event(transport, &path->path);
+
+out:
+	atomic_set(&cs->active_state, PCS_INACTIVE);
+	p = atomic_xchg(&cs->passive_state, PCS_INACTIVE);
+	if (p > PCS_INACTIVE)
+		drbd_put_listener(&path->path);
+
+	wake_up(&cs->wq);
+}
+
+static struct dtr_cm *dtr_alloc_cm(struct dtr_path *path)
+{
+	struct dtr_cm *cm;
+
+	cm = kzalloc_obj(*cm);
+	if (!cm)
+		return NULL;
+
+	kref_init(&cm->kref);
+	INIT_WORK(&cm->connect_work, dtr_cma_connect_work_fn);
+	INIT_WORK(&cm->establish_work, dtr_path_established_work_fn);
+	INIT_WORK(&cm->disconnect_work, dtr_cma_disconnect_work_fn);
+	INIT_WORK(&cm->end_rx_work, dtr_end_rx_work_fn);
+	INIT_WORK(&cm->end_tx_work, dtr_end_tx_work_fn);
+	INIT_WORK(&cm->tx_timeout_work, dtr_tx_timeout_work_fn);
+	INIT_LIST_HEAD(&cm->error_rx_descs);
+	spin_lock_init(&cm->error_rx_descs_lock);
+	timer_setup(&cm->tx_timeout, dtr_tx_timeout_fn, 0);
+
+	kref_get(&path->path.kref);
+	cm->path = path;
+	cm->rdma_transport = container_of(path->path.transport, struct dtr_transport, transport);
+
+	/*
+	 * We need this module in core as long as a dtr_tx_desc, a dtr_rx_desc
+	 * or a dtr_cm object exists because they might have a callback
+	 * registered in the RDMA code that will call back into this module. The
+	 * rx and tx descs have a reference to the dtr_cm object, so taking an
+	 * extra reference to the module for each dtr_cm object is sufficient.
+	 */
+	__module_get(THIS_MODULE);
+
+	return cm;
+}
+
+static int dtr_cma_accept(struct dtr_listener *listener, struct rdma_cm_id *new_cm_id, struct dtr_cm **ret_cm)
+{
+	struct sockaddr_storage *peer_addr;
+	struct dtr_connect_state *cs;
+	struct dtr_path *path;
+	struct drbd_path *drbd_path;
+	struct dtr_cm *cm;
+	int err;
+
+	*ret_cm = NULL;
+	peer_addr = &new_cm_id->route.addr.dst_addr;
+
+	spin_lock(&listener->listener.waiters_lock);
+	drbd_path = drbd_find_path_by_addr(&listener->listener, peer_addr);
+	if (drbd_path)
+		kref_get(&drbd_path->kref);
+	spin_unlock(&listener->listener.waiters_lock);
+
+	if (!drbd_path) {
+		struct sockaddr_in6 *from_sin6;
+		struct sockaddr_in *from_sin;
+
+		switch (peer_addr->ss_family) {
+		case AF_INET6:
+			from_sin6 = (struct sockaddr_in6 *)peer_addr;
+			pr_warn("Closing unexpected connection from "
+			       "%pI6\n", &from_sin6->sin6_addr);
+			break;
+		case AF_INET:
+			from_sin = (struct sockaddr_in *)peer_addr;
+			pr_warn("Closing unexpected connection from "
+				"%pI4\n", &from_sin->sin_addr);
+			break;
+		default:
+			pr_warn("Closing unexpected connection family = %d\n",
+				peer_addr->ss_family);
+		}
+
+		rdma_reject(new_cm_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
+		return -EAGAIN;
+	}
+
+	path = container_of(drbd_path, struct dtr_path, path);
+	cs = &path->cs;
+	if (atomic_read(&cs->passive_state) < PCS_CONNECTING)
+		goto reject;
+
+	cm = dtr_alloc_cm(path);
+	if (!cm) {
+		pr_err("rejecting connecting since -ENOMEM for cm\n");
+		goto reject;
+	}
+
+	cm->state = DSM_CONNECT_REQ;
+	init_waitqueue_head(&cm->state_wq);
+	new_cm_id->context = cm;
+	cm->id = new_cm_id;
+	*ret_cm = cm;
+
+	/* Expecting RDMA_CM_EVENT_ESTABLISHED, after rdma_accept(). Get
+	   the ref before dtr_path_prepare(), since that exposes the cm
+	   to the path, and the path might get destroyed, and with that
+	   going to put the cm */
+	kref_get(&cm->kref);
+
+	/* Gifting the initial kref to the path->cm pointer */
+	err = dtr_path_prepare(path, cm, false);
+	if (err) {
+		/* Returning the cm via ret_cm and an error causes the caller to put one ref */
+		goto reject;
+	}
+	kref_put(&drbd_path->kref, drbd_destroy_path);
+
+	err = rdma_accept(new_cm_id, &dtr_conn_param);
+	if (err)
+		kref_put(&cm->kref, dtr_destroy_cm);
+
+	return err;
+
+reject:
+	rdma_reject(new_cm_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
+	kref_put(&drbd_path->kref, drbd_destroy_path);
+	return -EAGAIN;
+}
+
+static int dtr_start_try_connect(struct dtr_connect_state *cs)
+{
+	struct dtr_path *path = container_of(cs, struct dtr_path, cs);
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_cm *cm;
+	int err = -ENOMEM;
+
+	cm = dtr_alloc_cm(path);
+	if (!cm)
+		goto out;
+
+	err = dtr_create_cm_id(cm, path->path.net);
+	if (err) {
+		tr_err(transport, "rdma_create_id() failed %d\n", err);
+		goto out;
+	}
+
+	/* Holding the initial reference on cm, expecting RDMA_CM_EVENT_ADDR_RESOLVED */
+	err = rdma_resolve_addr(cm->id, NULL,
+				(struct sockaddr *)&path->path.peer_addr,
+				2000);
+	if (err) {
+		tr_err(transport, "rdma_resolve_addr error %d\n", err);
+		goto out;
+	}
+
+	return 0;
+out:
+	if (cm)
+		kref_put(&cm->kref, dtr_destroy_cm);
+	return err;
+}
+
+static void dtr_cma_retry_connect_work_fn(struct work_struct *work)
+{
+	struct dtr_connect_state *cs = container_of(work, struct dtr_connect_state, retry_connect_work.work);
+	enum connect_state_enum p;
+	int err;
+
+	p = atomic_cmpxchg(&cs->active_state, PCS_REQUEST_ABORT, PCS_INACTIVE);
+	if (p != PCS_CONNECTING) {
+		wake_up(&cs->wq);
+		return;
+	}
+
+	err = dtr_start_try_connect(cs);
+	if (err) {
+		struct dtr_path *path = container_of(cs, struct dtr_path, cs);
+		struct drbd_transport *transport = path->path.transport;
+
+		tr_err(transport, "dtr_start_try_connect failed  %d\n", err);
+		schedule_delayed_work(&cs->retry_connect_work, HZ);
+	}
+}
+
+static void dtr_remove_cm_from_path(struct dtr_path *path, struct dtr_cm *failed_cm)
+{
+	struct dtr_cm *cm;
+
+	cm = cmpxchg(&path->cm, failed_cm, NULL); // RCU &path->cm
+	if (cm == failed_cm && cm->id && cm->id->qp) {
+		struct drbd_transport *transport = path->path.transport;
+		struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+		int err;
+
+		err = ib_modify_qp(cm->id->qp, &attr, IB_QP_STATE);
+		if (err)
+			tr_err(transport, "ib_modify_qp failed %d\n", err);
+
+		kref_put(&cm->kref, dtr_destroy_cm);
+	}
+}
+
+static void dtr_cma_retry_connect(struct dtr_path *path, struct dtr_cm *failed_cm)
+{
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_connect_state *cs = &path->cs;
+	long connect_int = 10 * HZ;
+	struct net_conf *nc;
+	int a;
+
+	dtr_remove_cm_from_path(path, failed_cm);
+
+	a = atomic_read(&cs->active_state);
+	if (a == PCS_INACTIVE) {
+		return;
+	} else if (a == PCS_CONNECTING) {
+		rcu_read_lock();
+		nc = rcu_dereference(transport->net_conf);
+		if (nc)
+			connect_int = nc->connect_int * HZ;
+		rcu_read_unlock();
+	} else {
+		connect_int = 1;
+	}
+	schedule_delayed_work(&cs->retry_connect_work, connect_int);
+}
+
+static void dtr_cma_connect_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, connect_work);
+	struct dtr_path *path = cm->path;
+	struct drbd_transport *transport = path->path.transport;
+	enum connect_state_enum p;
+	int err;
+
+	p = atomic_cmpxchg(&path->cs.active_state, PCS_REQUEST_ABORT, PCS_INACTIVE);
+	if (p != PCS_CONNECTING) {
+		wake_up(&path->cs.wq);
+		kref_put(&cm->kref, dtr_destroy_cm); /* for work */
+		return;
+	}
+
+	kref_get(&cm->kref); /* for the path->cm pointer */
+	err = dtr_path_prepare(path, cm, true);
+	if (err) {
+		tr_err(transport, "dtr_path_prepare() = %d\n", err);
+		goto out;
+	}
+
+	kref_get(&cm->kref); /* Expecting RDMA_CM_EVENT_ESTABLISHED */
+	set_bit(DSB_CONNECTING, &cm->state);
+	err = rdma_connect(cm->id, &dtr_conn_param);
+	if (err) {
+		if (test_and_clear_bit(DSB_CONNECTING, &cm->state))
+			kref_put(&cm->kref, dtr_destroy_cm); /* no _EVENT_ESTABLISHED */
+		tr_err(transport, "rdma_connect error %d\n", err);
+		goto out;
+	}
+
+	kref_put(&cm->kref, dtr_destroy_cm); /* for work */
+	return;
+out:
+	kref_put(&cm->kref, dtr_destroy_cm); /* for work */
+	dtr_cma_retry_connect(path, cm);
+}
+
+static void dtr_cma_disconnect_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, disconnect_work);
+	struct dtr_path *path = cm->path;
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct drbd_path *drbd_path = &path->path;
+	bool destroyed;
+	int err;
+
+	err = cm != path->cm;
+	kref_put(&cm->kref, dtr_destroy_cm);
+	if (err)
+		return;
+
+	destroyed = test_bit(TR_UNREGISTERED, &drbd_path->flags) || rdma_transport->active == false;
+	if (test_and_clear_bit(TR_ESTABLISHED, &drbd_path->flags) && !destroyed)
+		drbd_path_event(transport, drbd_path);
+
+	if (!dtr_transport_ok(transport))
+		drbd_control_event(transport, CLOSED_BY_PEER);
+
+	if (destroyed)
+		return;
+
+	/* in dtr_disconnect_path() -> __dtr_uninit_path() we free the previous
+	   cm. That causes the reference on the path to be dropped.
+	   In dtr_activate_path() -> dtr_start_try_connect() we allocate a new
+	   cm, that holds a reference on the path again.
+
+	   Bridge the gap with a reference here!
+	*/
+
+	kref_get(&path->path.kref);
+	dtr_disconnect_path(path);
+
+	/* dtr_disconnect_path() may take time, recheck here... */
+	if (test_bit(TR_UNREGISTERED, &drbd_path->flags) || rdma_transport->active == false)
+		goto abort;
+
+	if (!dtr_transport_ok(transport)) {
+		/* If there is no other connected path mark the connection as
+		   no longer active. Do not try to re-establish this path!! */
+		rdma_transport->active = false;
+		goto abort;
+	}
+
+	err = dtr_activate_path(path);
+	if (err)
+		tr_err(transport, "dtr_activate_path() = %d\n", err);
+abort:
+	kref_put(&path->path.kref, drbd_destroy_path);
+}
+
+static void dtr_cma_disconnect(struct dtr_cm *cm)
+{
+	kref_get(&cm->kref);
+	schedule_work(&cm->disconnect_work);
+}
+
+static int dtr_cma_event_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event)
+{
+	int err;
+	/* context comes from rdma_create_id() */
+	struct dtr_cm *cm = cm_id->context;
+	struct dtr_listener *listener;
+	bool connecting;
+
+	if (!cm) {
+		pr_err("id %p event %d, but no context!\n", cm_id, event->event);
+		return 0;
+	}
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		// pr_info("%s: RDMA_CM_EVENT_ADDR_RESOLVED\n", cm->name);
+		kref_get(&cm->kref); /* Expecting RDMA_CM_EVENT_ROUTE_RESOLVED */
+		err = rdma_resolve_route(cm_id, 2000);
+		if (err) {
+			kref_put(&cm->kref, dtr_destroy_cm);
+			pr_err("rdma_resolve_route error %d\n", err);
+		}
+		break;
+
+	case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		// pr_info("%s: RDMA_CM_EVENT_ROUTE_RESOLVED\n", cm->name);
+
+		kref_get(&cm->kref);
+		schedule_work(&cm->connect_work);
+		break;
+
+	case RDMA_CM_EVENT_CONNECT_REQUEST:
+		// pr_info("%s: RDMA_CM_EVENT_CONNECT_REQUEST\n", cm->name);
+		/* for listener */
+
+		listener = container_of(cm, struct dtr_listener, cm);
+		err = dtr_cma_accept(listener, cm_id, &cm);
+
+		/* I found this a bit confusing. When a new connection comes in, the callback
+		   gets called with a new rdma_cm_id. The new rdma_cm_id inherits its context
+		   pointer from the listening rdma_cm_id. The new context gets created in
+		   dtr_cma_accept() and is put into &cm here.
+		   cm now contains the accepted connection (no longer the listener); */
+		if (err) {
+			if (!cm)
+				return 1; /* caller destroy the cm_id */
+			break; /* drop the last ref of cm at function exit */
+		}
+		return 0; /* do not touch kref of the new connection */
+
+	case RDMA_CM_EVENT_CONNECT_RESPONSE:
+		// pr_info("%s: RDMA_CM_EVENT_CONNECT_RESPONSE\n", cm->name);
+		/*cm->path->cm = cm;
+		  dtr_path_established(cm->path); */
+		break;
+
+	case RDMA_CM_EVENT_ESTABLISHED:
+		// pr_info("%s: RDMA_CM_EVENT_ESTABLISHED\n", cm->name);
+		/* cm->state = DSM_CONNECTED; is set later in the work item */
+		/* This is called for active and passive connections */
+
+		connecting = test_and_clear_bit(DSB_CONNECTING, &cm->state) ||
+			test_and_clear_bit(DSB_CONNECT_REQ, &cm->state);
+		kref_get(&cm->kref); /* connected -> expect a disconnect in the future */
+		kref_get(&cm->kref); /* for the work */
+		schedule_work(&cm->establish_work);
+
+		if (!connecting)
+			return 0; /* keep ref; __dtr_disconnect_path() won */
+		break;
+
+	case RDMA_CM_EVENT_ADDR_ERROR:
+		// pr_info("%s: RDMA_CM_EVENT_ADDR_ERROR\n", cm->name);
+	case RDMA_CM_EVENT_ROUTE_ERROR:
+		// pr_info("%s: RDMA_CM_EVENT_ROUTE_ERROR\n", cm->name);
+		set_bit(DSB_ERROR, &cm->state);
+
+		dtr_cma_retry_connect(cm->path, cm);
+		break;
+
+	case RDMA_CM_EVENT_CONNECT_ERROR:
+		// pr_info("%s: RDMA_CM_EVENT_CONNECT_ERROR\n", cm->name);
+	case RDMA_CM_EVENT_UNREACHABLE:
+		// pr_info("%s: RDMA_CM_EVENT_UNREACHABLE\n", cm->name);
+	case RDMA_CM_EVENT_REJECTED:
+		// pr_info("%s: RDMA_CM_EVENT_REJECTED\n", cm->name);
+		// pr_info("event = %d, status = %d\n", event->event, event->status);
+		set_bit(DSB_ERROR, &cm->state);
+
+		dtr_cma_retry_connect(cm->path, cm);
+		connecting = test_and_clear_bit(DSB_CONNECTING, &cm->state) ||
+			test_and_clear_bit(DSB_CONNECT_REQ, &cm->state);
+		if (!connecting)
+			return 0; /* keep ref; __dtr_disconnect_path() won */
+		break;
+
+	case RDMA_CM_EVENT_DISCONNECTED:
+		// pr_info("%s: RDMA_CM_EVENT_DISCONNECTED\n", cm->name);
+		if (!test_and_clear_bit(DSB_CONNECTED, &cm->state))
+			return 0; /* keep ref on cm; probably a tx_timeout */
+
+		dtr_cma_disconnect(cm);
+		break;
+
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		// pr_info("%s: RDMA_CM_EVENT_DEVICE_REMOVAL\n", cm->name);
+		return 0;
+
+	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+		return 0;
+
+	default:
+		pr_warn("id %p context %p unexpected event %d!\n",
+				cm_id, cm, event->event);
+		return 0;
+	}
+	wake_up(&cm->state_wq);
+
+	/* by returning 1 we instruct the caller to destroy the cm_id. We
+	   are not allowed to free it within the callback, since that deadlocks! */
+	return kref_put(&cm->kref, dtr_destroy_cm_keep_id);
+}
+
+static int dtr_create_cm_id(struct dtr_cm *cm, struct net *net)
+{
+	struct rdma_cm_id *id;
+
+	cm->state = 0;
+	init_waitqueue_head(&cm->state_wq);
+
+	id = rdma_create_id(net, dtr_cma_event_handler, cm, RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(id)) {
+		cm->id = NULL;
+		set_bit(DSB_ERROR, &cm->state);
+		return PTR_ERR(id);
+	}
+
+	cm->id = id;
+	return 0;
+}
+
+/* Number of rx_descs the peer does not know */
+static int dtr_new_rx_descs(struct dtr_flow *flow)
+{
+	int posted, known;
+
+	posted = atomic_read(&flow->rx_descs_posted);
+	smp_rmb(); /* smp_wmb() is in dtr_handle_rx_cq_event() */
+	known = atomic_read(&flow->rx_descs_known_to_peer);
+
+	/* If the two decrements in dtr_handle_rx_cq_event() execute in
+	 * parallel our result might be one too low, that does not matter.
+	 * Only make sure to never return a -1 because that would matter! */
+	return max(posted - known, 0);
+}
+
+static struct dtr_rx_desc *dtr_next_rx_desc(struct dtr_stream *rdma_stream)
+{
+	struct dtr_rx_desc *rx_desc;
+
+	spin_lock_irq(&rdma_stream->rx_descs_lock);
+	rx_desc = list_first_entry_or_null(&rdma_stream->rx_descs, struct dtr_rx_desc, list);
+	if (rx_desc) {
+		if (rx_desc->sequence == rdma_stream->rx_sequence) {
+			list_del(&rx_desc->list);
+			rdma_stream->rx_sequence =
+				(rdma_stream->rx_sequence + 1) & ((1UL << SEQUENCE_BITS) - 1);
+			rdma_stream->unread -= rx_desc->size;
+		} else {
+			rx_desc = NULL;
+		}
+	}
+	spin_unlock_irq(&rdma_stream->rx_descs_lock);
+
+	return rx_desc;
+}
+
+static bool dtr_receive_rx_desc(struct dtr_transport *rdma_transport,
+				enum drbd_stream stream,
+				struct dtr_rx_desc **ptr_rx_desc)
+{
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[stream];
+	struct dtr_rx_desc *rx_desc;
+
+	rx_desc = dtr_next_rx_desc(rdma_stream);
+
+	if (rx_desc) {
+		struct dtr_cm *cm = rx_desc->cm;
+		struct dtr_transport *rdma_transport =
+			container_of(cm->path->path.transport, struct dtr_transport, transport);
+
+		INIT_LIST_HEAD(&rx_desc->list);
+		ib_dma_sync_single_for_cpu(cm->id->device, rx_desc->sge.addr,
+					   rdma_transport->rx_allocation_size, DMA_FROM_DEVICE);
+		*ptr_rx_desc = rx_desc;
+		return true;
+	} else {
+		/* The waiting thread gets woken up if a packet arrived, or if there is no
+		   new packet but we need to tell the peer about space in our receive window */
+		struct dtr_path *path;
+
+		rcu_read_lock();
+		list_for_each_entry_rcu(path, &rdma_transport->transport.paths, path.list) {
+			struct dtr_flow *flow = &path->flow[stream];
+
+			if (atomic_read(&flow->rx_descs_known_to_peer) <
+			    atomic_read(&flow->rx_descs_posted) / 8)
+				dtr_send_flow_control_msg(path, GFP_ATOMIC);
+		}
+		rcu_read_unlock();
+	}
+
+	return false;
+}
+
+static int dtr_send_flow_control_msg(struct dtr_path *path, gfp_t gfp_mask)
+{
+	struct dtr_flow_control msg;
+	enum drbd_stream i;
+	int err, n[2], rx_desc_stolen_from = -1, rx_descs = 0;
+
+	msg.magic = cpu_to_be32(DTR_MAGIC);
+
+	spin_lock_bh(&path->send_flow_control_lock);
+	/* dtr_send_flow_control_msg() is called from the receiver thread and
+	   areceiver, asender (multiple threads).
+	   determining the number of new tx_descs and subtracting this number
+	   from rx_descs_known_to_peer has to be atomic!
+	 */
+	for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+		struct dtr_flow *flow = &path->flow[i];
+
+		n[i] = dtr_new_rx_descs(flow);
+		atomic_add(n[i], &flow->rx_descs_known_to_peer);
+		rx_descs += n[i];
+
+		msg.new_rx_descs[i] = cpu_to_be32(n[i]);
+		if (rx_desc_stolen_from == -1 && atomic_dec_if_positive(&flow->peer_rx_descs) >= 0)
+			rx_desc_stolen_from = i;
+	}
+	spin_unlock_bh(&path->send_flow_control_lock);
+
+	if (rx_desc_stolen_from == -1) {
+		struct drbd_transport *transport = path->path.transport;
+		struct dtr_transport *rdma_transport =
+			container_of(transport, struct dtr_transport, transport);
+
+		if (__ratelimit(&rdma_transport->rate_limit))
+			tr_err(transport, "Not sending flow_control msg, no receive window!\n");
+		err = -ENOBUFS;
+		goto out_undo;
+	}
+
+	if (rx_descs == 0) {
+		atomic_inc(&path->flow[rx_desc_stolen_from].peer_rx_descs);
+		return 0;
+	}
+
+	msg.rx_desc_stolen_from_stream = cpu_to_be32(rx_desc_stolen_from);
+	err = dtr_send(path, &msg, sizeof(msg), gfp_mask);
+	if (err) {
+		atomic_inc(&path->flow[rx_desc_stolen_from].peer_rx_descs);
+	out_undo:
+		for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+			struct dtr_flow *flow = &path->flow[i];
+			atomic_sub(n[i], &flow->rx_descs_known_to_peer);
+		}
+	}
+	return err;
+}
+
+static void dtr_flow_control(struct dtr_flow *flow, gfp_t gfp_mask)
+{
+	int n, known_to_peer = atomic_read(&flow->rx_descs_known_to_peer);
+	int tx_descs_max = flow->tx_descs_max;
+
+	n = dtr_new_rx_descs(flow);
+	if (n > tx_descs_max / 8 || known_to_peer < tx_descs_max / 8)
+	  dtr_send_flow_control_msg(flow->path, gfp_mask);
+}
+
+static int dtr_got_flow_control_msg(struct dtr_path *path,
+				     struct dtr_flow_control *msg)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(path->path.transport, struct dtr_transport, transport);
+	struct dtr_flow *flow;
+	int i, n;
+
+	for (i = CONTROL_STREAM; i >= DATA_STREAM; i--) {
+		uint32_t new_rx_descs = be32_to_cpu(msg->new_rx_descs[i]);
+		flow = &path->flow[i];
+
+		n = atomic_add_return(new_rx_descs, &flow->peer_rx_descs);
+		wake_up_interruptible(&rdma_transport->stream[i].send_wq);
+	}
+
+	/* rdma_stream is the data_stream here... */
+	if (n >= DESCS_LOW_LEVEL) {
+		int tx_descs_posted = atomic_read(&flow->tx_descs_posted);
+		if (flow->tx_descs_max - tx_descs_posted >= DESCS_LOW_LEVEL)
+			clear_bit(NET_CONGESTED, &rdma_transport->transport.flags);
+	}
+
+	return be32_to_cpu(msg->rx_desc_stolen_from_stream);
+}
+
+static void dtr_flow_control_tasklet_fn(struct tasklet_struct *t)
+{
+	struct dtr_path *path = from_tasklet(path, t, flow_control_tasklet);
+
+	dtr_send_flow_control_msg(path, GFP_ATOMIC);
+}
+
+static void dtr_maybe_trigger_flow_control_msg(struct dtr_path *path, int rx_desc_stolen_from)
+{
+	struct dtr_flow *flow;
+	int n;
+
+	flow = &path->flow[rx_desc_stolen_from];
+	n = atomic_dec_return(&flow->rx_descs_known_to_peer);
+	/* If we get a lot of flow control messages in, but no data on this
+	 * path, we need to tell the peer that we recycled all these buffers
+	 */
+	if (n < atomic_read(&flow->rx_descs_posted) / 8)
+		tasklet_schedule(&path->flow_control_tasklet);
+}
+
+static void dtr_tx_timeout_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, tx_timeout_work);
+	struct drbd_transport *transport;
+	struct dtr_path *path = cm->path;
+
+	if (!test_and_clear_bit(DSB_CONNECTED, &cm->state) || !path)
+		goto out;
+
+	transport = path->path.transport;
+	tr_warn(transport, "%pI4 - %pI4: tx timeout\n",
+		&((struct sockaddr_in *)&path->path.my_addr)->sin_addr,
+		&((struct sockaddr_in *)&path->path.peer_addr)->sin_addr);
+
+	dtr_remove_cm_from_path(path, cm);
+
+	/* It is not sure that a RDMA_CM_EVENT_DISCONNECTED will be delivered.
+	 * Dropping ref for that here. In case it is delivered we will not drop
+	 * the ref in dtr_cma_event_handler() due to clearing DSB_CONNECTED
+	 * from cm->state */
+	kref_put(&cm->kref, dtr_destroy_cm);
+
+	clear_bit(TR_ESTABLISHED, &path->path.flags);
+	drbd_path_event(transport, &path->path);
+
+	if (!dtr_transport_ok(transport)) {
+		struct dtr_transport *rdma_transport =
+			container_of(transport, struct dtr_transport, transport);
+
+		drbd_control_event(transport, CLOSED_BY_PEER);
+		rdma_transport->active = false;
+	} else {
+		dtr_activate_path(path);
+	}
+
+out:
+	kref_put(&cm->kref, dtr_destroy_cm); /* for work (armed timer) */
+}
+
+static void dtr_tx_timeout_fn(struct timer_list *t)
+{
+	struct dtr_cm *cm = timer_container_of(cm, t, tx_timeout);
+
+	/* cm->kref for armed timer becomes a ref for the work */
+	schedule_work(&cm->tx_timeout_work);
+}
+
+static bool higher_in_sequence(unsigned int higher, unsigned int base)
+{
+	/*
+	  SEQUENCE Arithmetic: By looking at the most signifficant bit of
+	  the reduced word size we find out if the difference is positive.
+	  The difference is necessary to deal with the overflow in the
+	  sequence number space.
+	 */
+	unsigned int diff = higher - base;
+
+	return !(diff & (1 << (SEQUENCE_BITS - 1)));
+}
+
+static void __dtr_order_rx_descs(struct dtr_stream *rdma_stream,
+				 struct dtr_rx_desc *rx_desc)
+{
+	struct dtr_rx_desc *pos;
+	unsigned int seq = rx_desc->sequence;
+
+	list_for_each_entry_reverse(pos, &rdma_stream->rx_descs, list) {
+		if (higher_in_sequence(seq, pos->sequence)) { /* think: seq > pos->sequence */
+			list_add(&rx_desc->list, &pos->list);
+			return;
+		}
+	}
+	list_add(&rx_desc->list, &rdma_stream->rx_descs);
+}
+
+static void dtr_order_rx_descs(struct dtr_stream *rdma_stream,
+			       struct dtr_rx_desc *rx_desc)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&rdma_stream->rx_descs_lock, flags);
+	__dtr_order_rx_descs(rdma_stream, rx_desc);
+	rdma_stream->unread += rx_desc->size;
+	spin_unlock_irqrestore(&rdma_stream->rx_descs_lock, flags);
+}
+
+static void dtr_dec_rx_descs(struct dtr_cm *cm)
+{
+	struct dtr_flow *flow = cm->path->flow;
+	struct dtr_transport *rdma_transport = cm->rdma_transport;
+
+	/* When we get the posted rx_descs back, we do not know if they
+	 * where accoutend for the data stream or the control stream...
+	 */
+	if (atomic_dec_if_positive(&flow[DATA_STREAM].rx_descs_posted) >= 0)
+		return;
+
+	if (atomic_dec_if_positive(&flow[CONTROL_STREAM].rx_descs_posted) >= 0)
+		return;
+
+	if (__ratelimit(&rdma_transport->rate_limit)) {
+		struct drbd_transport *transport = &rdma_transport->transport;
+
+		tr_warn(transport, "rx_descs_posted underflow avoided\n");
+	}
+}
+
+static void dtr_control_data_ready(struct dtr_stream *rdma_stream, struct dtr_rx_desc *rx_desc)
+{
+	struct dtr_transport *rdma_transport = rdma_stream->rdma_transport;
+	struct drbd_transport *transport = &rdma_transport->transport;
+	struct drbd_const_buffer buffer;
+	struct dtr_cm *cm = rx_desc->cm;
+	struct dtr_path *path = cm->path;
+	struct dtr_flow *flow = &path->flow[CONTROL_STREAM];
+
+	if (atomic_read(&flow->rx_descs_known_to_peer) < atomic_read(&flow->rx_descs_posted) / 8)
+		dtr_send_flow_control_msg(path, GFP_ATOMIC);
+
+	ib_dma_sync_single_for_cpu(cm->id->device, rx_desc->sge.addr,
+				   rdma_transport->rx_allocation_size, DMA_FROM_DEVICE);
+
+	buffer.buffer = page_address(rx_desc->page);
+	buffer.avail = rx_desc->size;
+	drbd_control_data_ready(transport, &buffer);
+
+	dtr_recycle_rx_desc(transport, CONTROL_STREAM, &rx_desc, GFP_ATOMIC);
+}
+
+static void __dtr_order_rx_descs_front(struct dtr_stream *rdma_stream,
+				       struct dtr_rx_desc *rx_desc)
+{
+	struct dtr_rx_desc *pos;
+	unsigned int seq = rx_desc->sequence;
+
+	list_for_each_entry(pos, &rdma_stream->rx_descs, list) {
+		if (higher_in_sequence(seq, pos->sequence)) { /* think: seq > pos->sequence */
+			list_add(&rx_desc->list, &pos->list);
+			return;
+		}
+	}
+	list_add(&rx_desc->list, &rdma_stream->rx_descs);
+}
+
+static void dtr_control_tasklet_fn(struct tasklet_struct *t)
+{
+	struct dtr_transport *rdma_transport =
+		from_tasklet(rdma_transport, t, control_tasklet);
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[CONTROL_STREAM];
+	struct dtr_rx_desc *rx_desc, *tmp;
+	LIST_HEAD(rx_descs);
+
+	spin_lock_irq(&rdma_stream->rx_descs_lock);
+	list_splice_init(&rdma_stream->rx_descs, &rx_descs);
+	spin_unlock_irq(&rdma_stream->rx_descs_lock);
+
+	list_for_each_entry_safe(rx_desc, tmp, &rx_descs, list) {
+		if (rx_desc->sequence != rdma_stream->rx_sequence)
+			goto abort;
+		list_del(&rx_desc->list);
+		rdma_stream->rx_sequence =
+			(rdma_stream->rx_sequence + 1) & ((1UL << SEQUENCE_BITS) - 1);
+		rdma_stream->unread -= rx_desc->size;
+		dtr_control_data_ready(rdma_stream, rx_desc);
+	}
+	return;
+
+abort:
+	spin_lock_irq(&rdma_stream->rx_descs_lock);
+	list_for_each_entry_safe(rx_desc, tmp, &rx_descs, list) {
+		list_del(&rx_desc->list);
+		__dtr_order_rx_descs_front(rdma_stream, rx_desc);
+	}
+	spin_unlock_irq(&rdma_stream->rx_descs_lock);
+
+	tasklet_schedule(&rdma_transport->control_tasklet);
+}
+
+static int dtr_handle_rx_cq_event(struct ib_cq *cq, struct dtr_cm *cm)
+{
+	struct dtr_path *path = cm->path;
+	struct dtr_transport *rdma_transport =
+		container_of(path->path.transport, struct dtr_transport, transport);
+	struct dtr_rx_desc *rx_desc;
+	union dtr_immediate immediate;
+	struct ib_wc wc;
+	int ret, err;
+
+	ret = ib_poll_cq(cq, 1, &wc);
+	if (!ret)
+		return -EAGAIN;
+
+	rx_desc = (struct dtr_rx_desc *) (unsigned long) wc.wr_id;
+
+	if (wc.status != IB_WC_SUCCESS || !(wc.opcode & IB_WC_RECV)) {
+		struct drbd_transport *transport = &rdma_transport->transport;
+		unsigned long irq_flags;
+
+		switch (wc.status) {
+		case IB_WC_WR_FLUSH_ERR:
+			/* "Work Request Flushed Error: A Work Request was in
+			 * process or outstanding when the QP transitioned into
+			 * the Error State."
+			 *
+			 * Which is not entirely unexpected...
+			 */
+			break;
+
+		default:
+			if (__ratelimit(&rdma_transport->rate_limit)) {
+				tr_warn(transport,
+					"wc.status = %d (%s), wc.opcode = %d (%s)\n",
+					wc.status, wc.status == IB_WC_SUCCESS ? "ok" : "bad",
+					wc.opcode, wc.opcode & IB_WC_RECV ? "ok" : "bad");
+
+				tr_warn(transport,
+					"wc.vendor_err = %d, wc.byte_len = %d wc.imm_data = %d\n",
+					wc.vendor_err, wc.byte_len, wc.ex.imm_data);
+			}
+		}
+
+		/* dtr_free_rx_desc(NULL, rx_desc);
+		   dtr_free_rx_desc() will call drbd_free_page(), and that function
+		   should not be called from IRQ context. This callback executes
+		   in the context of the timer interrupt.
+		 */
+		spin_lock_irqsave(&cm->error_rx_descs_lock, irq_flags);
+		list_add_tail(&rx_desc->list, &cm->error_rx_descs);
+		spin_unlock_irqrestore(&cm->error_rx_descs_lock, irq_flags);
+		dtr_dec_rx_descs(cm);
+		set_bit(DSB_ERROR, &cm->state);
+
+		return 0;
+	}
+
+	rx_desc->size = wc.byte_len;
+	immediate.i = be32_to_cpu(wc.ex.imm_data);
+	if (immediate.stream == ST_FLOW_CTRL) {
+		int rx_desc_stolen_from;
+
+		ib_dma_sync_single_for_cpu(cm->id->device, rx_desc->sge.addr,
+					   rdma_transport->rx_allocation_size, DMA_FROM_DEVICE);
+		rx_desc_stolen_from = dtr_got_flow_control_msg(path, page_address(rx_desc->page));
+		err = dtr_repost_rx_desc(cm, rx_desc);
+		if (err)
+			tr_err(&rdma_transport->transport, "dtr_repost_rx_desc() failed %d", err);
+		dtr_maybe_trigger_flow_control_msg(path, rx_desc_stolen_from);
+	} else {
+		struct dtr_flow *flow = &path->flow[immediate.stream];
+		struct dtr_stream *rdma_stream = &rdma_transport->stream[immediate.stream];
+
+		atomic_dec(&flow->rx_descs_posted);
+		smp_wmb(); /* smp_rmb() is in dtr_new_rx_descs() */
+		atomic_dec(&flow->rx_descs_known_to_peer);
+
+		if (immediate.stream == ST_CONTROL)
+			mod_timer(&rdma_transport->control_timer, jiffies + rdma_stream->recv_timeout);
+
+		rx_desc->sequence = immediate.sequence;
+		dtr_order_rx_descs(rdma_stream, rx_desc);
+
+		if (immediate.stream == ST_CONTROL)
+			tasklet_schedule(&rdma_transport->control_tasklet);
+		else
+			wake_up_interruptible(&rdma_stream->recv_wq);
+
+	}
+
+	return 0;
+}
+
+static void dtr_rx_cq_event_handler(struct ib_cq *cq, void *ctx)
+{
+	struct dtr_cm *cm = ctx;
+	struct dtr_path *path = cm->path;
+	int err, rc;
+
+	do {
+		unsigned long irq_flags;
+		do {
+			err = dtr_handle_rx_cq_event(cq, cm);
+		} while (!err);
+
+		spin_lock_irqsave(&cm->error_rx_descs_lock, irq_flags);
+		if (!list_empty(&cm->error_rx_descs)) {
+			kref_get(&cm->kref);
+			if (!schedule_work(&cm->end_rx_work))
+				kref_put(&cm->kref, dtr_destroy_cm);
+		}
+		spin_unlock_irqrestore(&cm->error_rx_descs_lock, irq_flags);
+
+		rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
+		if (unlikely(rc < 0)) {
+			struct drbd_transport *transport = path->path.transport;
+			tr_err(transport, "ib_req_notify_cq failed %d\n", rc);
+			break;
+		}
+	} while (rc);
+
+	if (dtr_path_ok(path)) {
+		struct dtr_flow *flow = &path->flow[DATA_STREAM];
+
+		if (atomic_read(&flow->rx_descs_posted) < flow->rx_descs_want_posted / 2)
+			schedule_work(&path->refill_rx_descs_work);
+	}
+}
+
+static void dtr_free_tx_desc(struct dtr_cm *cm, struct dtr_tx_desc *tx_desc)
+{
+	struct ib_device *device = cm->id->device;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+	int i, nr_sges;
+
+	switch (tx_desc->type) {
+	case SEND_PAGE:
+		ib_dma_unmap_page(device, tx_desc->sge[0].addr, tx_desc->sge[0].length, DMA_TO_DEVICE);
+		put_page(tx_desc->page);
+		break;
+	case SEND_MSG:
+		ib_dma_unmap_single(device, tx_desc->sge[0].addr, tx_desc->sge[0].length, DMA_TO_DEVICE);
+		kfree(tx_desc->data);
+		break;
+	case SEND_BIO:
+		nr_sges = tx_desc->nr_sges;
+		for (i = 0; i < nr_sges; i++)
+			ib_dma_unmap_page(device, tx_desc->sge[i].addr, tx_desc->sge[i].length,
+					  DMA_TO_DEVICE);
+		bio_for_each_segment(bvec, tx_desc->bio, iter) {
+			put_page(bvec.bv_page);
+		}
+		break;
+	}
+	kfree(tx_desc);
+}
+
+static int dtr_handle_tx_cq_event(struct ib_cq *cq, struct dtr_cm *cm)
+{
+	struct dtr_path *path = cm->path;
+	struct dtr_transport *rdma_transport =
+		container_of(path->path.transport, struct dtr_transport, transport);
+	struct dtr_tx_desc *tx_desc;
+	struct ib_wc wc;
+	enum dtr_stream_nr stream_nr;
+	int ret, err;
+
+	ret = ib_poll_cq(cq, 1, &wc);
+	if (!ret)
+		return -EAGAIN;
+
+	tx_desc = (struct dtr_tx_desc *) (unsigned long) wc.wr_id;
+	stream_nr = tx_desc->imm.stream;
+
+	if (wc.status != IB_WC_SUCCESS || wc.opcode != IB_WC_SEND) {
+		struct drbd_transport *transport = &rdma_transport->transport;
+
+		if (wc.status == IB_WC_RNR_RETRY_EXC_ERR) {
+			struct dtr_flow *flow = &path->flow[stream_nr];
+			tr_err(transport, "tx_event: wc.status = IB_WC_RNR_RETRY_EXC_ERR\n");
+			tr_info(transport, "peer_rx_descs = %d", atomic_read(&flow->peer_rx_descs));
+		} else if (wc.status != IB_WC_WR_FLUSH_ERR) {
+			tr_err(transport, "tx_event: wc.status != IB_WC_SUCCESS %d\n", wc.status);
+			tr_err(transport, "wc.vendor_err = %d, wc.byte_len = %d wc.imm_data = %d\n",
+			       wc.vendor_err, wc.byte_len, wc.ex.imm_data);
+		}
+
+		set_bit(DSB_ERROR, &cm->state);
+
+		if (stream_nr != ST_FLOW_CTRL) {
+			err = dtr_repost_tx_desc(cm, tx_desc);
+			if (!err)
+				tx_desc = NULL; /* it is in the air again! Fly! */
+			else if (__ratelimit(&rdma_transport->rate_limit))
+				tr_warn(transport, "repost of tx_desc failed! %d\n", err);
+		}
+	}
+
+	if (stream_nr != ST_FLOW_CTRL) {
+		struct dtr_flow *flow = &path->flow[stream_nr];
+		struct dtr_stream *rdma_stream = &rdma_transport->stream[stream_nr];
+
+		atomic_dec(&flow->tx_descs_posted);
+		wake_up_interruptible(&rdma_stream->send_wq);
+	}
+
+	if (tx_desc)
+		dtr_free_tx_desc(cm, tx_desc);
+	if (atomic_dec_and_test(&cm->tx_descs_posted)) {
+		bool was_active = timer_delete(&cm->tx_timeout);
+
+		if (was_active)
+			kref_put(&cm->kref, dtr_destroy_cm);
+
+		if (cm->state == DSM_CONNECTED)
+			kref_put(&cm->kref, dtr_destroy_cm); /* this is _not_ the last ref */
+		else
+			schedule_work(&cm->end_tx_work); /* the last ref might be put in this work */
+	}
+
+	return 0;
+}
+
+static void dtr_tx_cq_event_handler(struct ib_cq *cq, void *ctx)
+{
+	struct dtr_cm *cm = ctx;
+	int err, rc;
+
+	do {
+		do {
+			err = dtr_handle_tx_cq_event(cq, cm);
+		} while (!err);
+
+		if (cm->state != DSM_CONNECTED)
+			break;
+
+		rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
+		if (unlikely(rc < 0)) {
+			struct drbd_transport *transport = cm->path->path.transport;
+			tr_err(transport, "ib_req_notify_cq failed %d\n", rc);
+			break;
+		}
+	} while (rc);
+}
+
+static int dtr_create_qp(struct dtr_cm *cm, int rx_descs_max, int tx_descs_max)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(cm->path->path.transport, struct dtr_transport, transport);
+	int err;
+
+	struct ib_qp_init_attr init_attr = {
+		.cap.max_send_wr = tx_descs_max,
+		.cap.max_recv_wr = rx_descs_max,
+		.cap.max_recv_sge = 1, /* We only receive into single pages */
+		.cap.max_send_sge = rdma_transport->sges_max,
+		.qp_type = IB_QPT_RC,
+		.send_cq = cm->send_cq,
+		.recv_cq = cm->recv_cq,
+		.sq_sig_type = IB_SIGNAL_REQ_WR
+	};
+
+	err = rdma_create_qp(cm->id, cm->pd, &init_attr);
+
+	return err;
+}
+
+static int dtr_post_rx_desc(struct dtr_cm *cm, struct dtr_rx_desc *rx_desc)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(cm->path->path.transport, struct dtr_transport, transport);
+	struct ib_recv_wr recv_wr;
+	const struct ib_recv_wr *recv_wr_failed;
+	int err = -EIO;
+
+	recv_wr.next = NULL;
+	recv_wr.wr_id = (unsigned long)rx_desc;
+	recv_wr.sg_list = &rx_desc->sge;
+	recv_wr.num_sge = 1;
+
+	ib_dma_sync_single_for_device(cm->id->device,
+				      rx_desc->sge.addr, rdma_transport->rx_allocation_size, DMA_FROM_DEVICE);
+
+	err = ib_post_recv(cm->id->qp, &recv_wr, &recv_wr_failed);
+	if (err)
+		tr_err(&rdma_transport->transport, "ib_post_recv error %d\n", err);
+
+	return err;
+}
+
+static void dtr_free_rx_desc(struct dtr_rx_desc *rx_desc)
+{
+	struct dtr_transport *rdma_transport;
+	struct dtr_path *path;
+	struct ib_device *device;
+	struct dtr_cm *cm;
+	int alloc_size;
+
+	if (!rx_desc)
+		return; /* Allow call with NULL */
+
+	cm = rx_desc->cm;
+	device = cm->id->device;
+	path = cm->path;
+	rdma_transport = container_of(path->path.transport, struct dtr_transport, transport);
+	alloc_size = rdma_transport->rx_allocation_size;
+	ib_dma_unmap_single(device, rx_desc->sge.addr, alloc_size, DMA_FROM_DEVICE);
+	kref_put(&cm->kref, dtr_destroy_cm);
+
+	if (rx_desc->page) {
+		struct drbd_transport *transport = &rdma_transport->transport;
+
+		/* put_page(), if we had more than one rx_desc per page,
+		 * but see comments in dtr_create_rx_desc */
+		drbd_free_pages(transport, rx_desc->page);
+	}
+	kfree(rx_desc);
+}
+
+static int dtr_create_rx_desc(struct dtr_flow *flow, gfp_t gfp_mask, bool connected_only)
+{
+	struct dtr_path *path = flow->path;
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_rx_desc *rx_desc;
+	struct page *page;
+	int err, alloc_size = rdma_transport->rx_allocation_size;
+	int nr_pages = alloc_size / PAGE_SIZE;
+	struct dtr_cm *cm;
+
+	rx_desc = kzalloc_obj(*rx_desc, gfp_mask);
+	if (!rx_desc)
+		return -ENOMEM;
+
+	/* As of now, this MUST NEVER return a highmem page!
+	 * Which means no other user may ever have requested and then given
+	 * back a highmem page!
+	 */
+	page = drbd_alloc_pages(transport, nr_pages, gfp_mask);
+	if (!page) {
+		kfree(rx_desc);
+		return -ENOMEM;
+	}
+	BUG_ON(PageHighMem(page));
+
+	err = -ECONNRESET;
+	cm = dtr_path_get_cm(path);
+	if (!cm)
+		goto out;
+	if (connected_only && cm->state != DSM_CONNECTED)
+		goto out_put;
+
+	rx_desc->cm = cm;
+	rx_desc->page = page;
+	rx_desc->size = 0;
+	rx_desc->sge.lkey = dtr_cm_to_lkey(cm);
+	rx_desc->sge.addr = ib_dma_map_single(cm->id->device, page_address(page), alloc_size,
+					      DMA_FROM_DEVICE);
+	err = ib_dma_mapping_error(cm->id->device, rx_desc->sge.addr);
+	if (err) {
+		tr_err(transport, "ib_dma_map_single() failed %d\n", err);
+		goto out_put;
+	}
+	rx_desc->sge.length = alloc_size;
+
+	atomic_inc(&flow->rx_descs_allocated);
+	atomic_inc(&flow->rx_descs_posted);
+	err = dtr_post_rx_desc(cm, rx_desc);
+	if (err) {
+		tr_err(transport, "dtr_post_rx_desc() returned %d\n", err);
+		atomic_dec(&flow->rx_descs_posted);
+		atomic_dec(&flow->rx_descs_allocated);
+		dtr_free_rx_desc(rx_desc);
+	}
+	return err;
+
+out_put:
+	kref_put(&cm->kref, dtr_destroy_cm);
+out:
+	kfree(rx_desc);
+	drbd_free_pages(transport, page);
+	return err;
+}
+
+static void dtr_refill_rx_descs_work_fn(struct work_struct *work)
+{
+	struct dtr_path *path = container_of(work, struct dtr_path, refill_rx_descs_work);
+
+	if (dtr_path_ok(path))
+		__dtr_refill_rx_desc(path, DATA_STREAM);
+}
+
+static void __dtr_refill_rx_desc(struct dtr_path *path, enum drbd_stream stream)
+{
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_flow *flow = &path->flow[stream];
+	int descs_want_posted, descs_max;
+
+	descs_max = flow->rx_descs_max;
+	descs_want_posted = flow->rx_descs_want_posted;
+
+	while (atomic_read(&flow->rx_descs_posted) < descs_want_posted &&
+	       atomic_read(&flow->rx_descs_allocated) < descs_max) {
+		int err;
+
+		err = dtr_create_rx_desc(flow, (GFP_NOIO & ~__GFP_RECLAIM) | __GFP_NOWARN, true);
+		/*
+		 * drbd_alloc_pages() goes over the configured max_buffers, but throttles the
+		 * caller with sleeping 100ms for each of those excess pages.  By calling
+		 * without __GFP_RECLAIM we request to get a -ENOMEM instead of sleeping.
+		 * We simply stop refilling then.
+		 */
+		if (err == -ENOMEM) {
+			break;
+		} else if (err) {
+			tr_err(transport, "dtr_create_rx_desc() = %d\n", err);
+			break;
+		}
+	}
+}
+
+static void dtr_refill_rx_desc(struct dtr_transport *rdma_transport,
+			       enum drbd_stream stream)
+{
+	struct drbd_transport *transport = &rdma_transport->transport;
+	struct drbd_path *drbd_path;
+
+	for_each_path_ref(drbd_path, transport) {
+		struct dtr_path *path = container_of(drbd_path, struct dtr_path, path);
+
+		if (!dtr_path_ok(path))
+			continue;
+
+		__dtr_refill_rx_desc(path, stream);
+		dtr_flow_control(&path->flow[stream], GFP_NOIO);
+	}
+}
+
+static int dtr_repost_rx_desc(struct dtr_cm *cm, struct dtr_rx_desc *rx_desc)
+{
+	int err;
+
+	rx_desc->size = 0;
+	rx_desc->sge.lkey = dtr_cm_to_lkey(cm);
+	/* rx_desc->sge.addr = rx_desc->dma_addr;
+	   rx_desc->sge.length = rx_desc->alloc_size; */
+
+	err = dtr_post_rx_desc(cm, rx_desc);
+	return err;
+}
+
+static void dtr_recycle_rx_desc(struct drbd_transport *transport,
+				enum drbd_stream stream,
+				struct dtr_rx_desc **pp_rx_desc,
+				gfp_t gfp_mask)
+{
+	struct dtr_rx_desc *rx_desc = *pp_rx_desc;
+	struct dtr_cm *cm;
+	struct dtr_path *path;
+	struct dtr_flow *flow;
+	int err;
+
+	if (!rx_desc)
+		return;
+
+	cm = rx_desc->cm;
+	path = cm->path;
+	flow = &path->flow[stream];
+
+	err = dtr_repost_rx_desc(cm, rx_desc);
+
+	if (err) {
+		dtr_free_rx_desc(rx_desc);
+	} else {
+		atomic_inc(&flow->rx_descs_posted);
+		dtr_flow_control(flow, gfp_mask);
+	}
+
+	*pp_rx_desc = NULL;
+}
+
+static int __dtr_post_tx_desc(struct dtr_cm *cm, struct dtr_tx_desc *tx_desc)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(cm->path->path.transport, struct dtr_transport, transport);
+	struct drbd_transport *transport = &rdma_transport->transport;
+	struct ib_send_wr send_wr;
+	const struct ib_send_wr *send_wr_failed;
+	struct ib_device *device = cm->id->device;
+	unsigned long timeout;
+	struct net_conf *nc;
+	int i, err = -EIO;
+	bool was_active;
+
+	send_wr.next = NULL;
+	send_wr.wr_id = (unsigned long)tx_desc;
+	send_wr.sg_list = tx_desc->sge;
+	send_wr.num_sge = tx_desc->nr_sges;
+	send_wr.ex.imm_data = cpu_to_be32(tx_desc->imm.i);
+	send_wr.opcode = IB_WR_SEND_WITH_IMM;
+	send_wr.send_flags = IB_SEND_SIGNALED;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	timeout = nc->ping_timeo;
+	rcu_read_unlock();
+
+	for (i = 0; i < tx_desc->nr_sges; i++)
+		ib_dma_sync_single_for_device(device, tx_desc->sge[i].addr,
+					      tx_desc->sge[i].length, DMA_TO_DEVICE);
+
+	if (atomic_inc_return(&cm->tx_descs_posted) == 1)
+		kref_get(&cm->kref); /* keep one extra ref as long as one tx is posted */
+
+	kref_get(&cm->kref);
+	was_active = mod_timer(&cm->tx_timeout, jiffies + timeout * HZ / 20);
+	if (was_active)
+		kref_put(&cm->kref, dtr_destroy_cm);
+
+	err = ib_post_send(cm->id->qp, &send_wr, &send_wr_failed);
+	if (err) {
+		tr_err(&rdma_transport->transport, "ib_post_send() failed %d\n", err);
+		was_active = timer_delete(&cm->tx_timeout);
+		if (!was_active)
+			was_active = cancel_work_sync(&cm->tx_timeout_work);
+		if (was_active)
+			kref_put(&cm->kref, dtr_destroy_cm);
+		if (atomic_dec_and_test(&cm->tx_descs_posted))
+			kref_put(&cm->kref, dtr_destroy_cm);
+	}
+
+	return err;
+}
+
+static struct dtr_cm *dtr_select_and_get_cm_for_tx(struct dtr_transport *rdma_transport,
+						     enum drbd_stream stream)
+{
+	struct drbd_transport *transport = &rdma_transport->transport;
+	struct dtr_path *path, *candidate = NULL;
+	unsigned long last_sent_jif = -1UL;
+	struct dtr_cm *cm;
+
+	/* Within in 16 jiffy use one path, in case we switch to an other one,
+	   use that that was used longest ago */
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &transport->paths, path.list) {
+		struct dtr_flow *flow = &path->flow[stream];
+		unsigned long ls;
+
+		cm = rcu_dereference(path->cm);
+		if (!cm || cm->state != DSM_CONNECTED)
+			continue;
+
+		/* Normal packets are not allowed to consume all of the peer's rx_descs,
+		   the last one is reserved for flow-control messages. */
+		if (atomic_read(&flow->tx_descs_posted) >= flow->tx_descs_max ||
+		    atomic_read(&flow->peer_rx_descs) <= 1)
+			continue;
+
+		ls = cm->last_sent_jif;
+		if ((ls & ~0xfUL) == (jiffies & ~0xfUL) && kref_get_unless_zero(&cm->kref)) {
+			rcu_read_unlock();
+			return cm;
+		}
+		if (ls < last_sent_jif) {
+			last_sent_jif = ls;
+			candidate = path;
+		}
+	}
+
+	if (candidate) {
+		cm = __dtr_path_get_cm(candidate);
+		cm->last_sent_jif = jiffies;
+	} else {
+		cm = NULL;
+	}
+	rcu_read_unlock();
+
+	return cm;
+}
+
+static int dtr_remap_tx_desc(struct dtr_cm *old_cm, struct dtr_cm *cm,
+			      struct dtr_tx_desc *tx_desc)
+{
+	struct ib_device *device = old_cm->id->device;
+	int i, nr_sges, err;
+	dma_addr_t a = 0;
+
+	switch (tx_desc->type) {
+	case SEND_PAGE:
+		ib_dma_unmap_page(device, tx_desc->sge[0].addr, tx_desc->sge[0].length, DMA_TO_DEVICE);
+		break;
+	case SEND_MSG:
+		ib_dma_unmap_single(device, tx_desc->sge[0].addr, tx_desc->sge[0].length, DMA_TO_DEVICE);
+		break;
+	case SEND_BIO:
+		nr_sges = tx_desc->nr_sges;
+		for (i = 0; i < nr_sges; i++)
+			ib_dma_unmap_page(device, tx_desc->sge[i].addr, tx_desc->sge[i].length,
+					  DMA_TO_DEVICE);
+		break;
+	}
+
+	device = cm->id->device;
+	switch (tx_desc->type) {
+	case SEND_PAGE:
+		a = ib_dma_map_page(device, tx_desc->page, tx_desc->sge[0].addr & ~PAGE_MASK,
+				    tx_desc->sge[0].length, DMA_TO_DEVICE);
+		break;
+	case SEND_MSG:
+		a = ib_dma_map_single(device, tx_desc->data, tx_desc->sge[0].length, DMA_TO_DEVICE);
+		break;
+	case SEND_BIO:
+#if SENDER_COMPACTS_BVECS
+		#error implement me
+#endif
+		break;
+	}
+	err = ib_dma_mapping_error(device, a);
+
+	tx_desc->sge[0].addr = a;
+	tx_desc->sge[0].lkey = dtr_cm_to_lkey(cm);
+
+	return err;
+}
+
+
+static int dtr_repost_tx_desc(struct dtr_cm *old_cm, struct dtr_tx_desc *tx_desc)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(old_cm->path->path.transport, struct dtr_transport, transport);
+	enum drbd_stream stream = tx_desc->imm.stream;
+	struct dtr_cm *cm;
+	int err;
+
+	do {
+		cm = dtr_select_and_get_cm_for_tx(rdma_transport, stream);
+		if (!cm)
+			return -ECONNRESET;
+
+		err = dtr_remap_tx_desc(old_cm, cm, tx_desc);
+		if (err)
+			continue;
+
+		err = __dtr_post_tx_desc(cm, tx_desc);
+		if (!err) {
+			struct dtr_flow *flow = &cm->path->flow[stream];
+			atomic_inc(&flow->tx_descs_posted);
+		}
+		kref_put(&cm->kref, dtr_destroy_cm);
+	} while (err);
+
+	return err;
+}
+
+static int dtr_post_tx_desc(struct dtr_transport *rdma_transport,
+			    struct dtr_tx_desc *tx_desc)
+{
+	enum drbd_stream stream = tx_desc->imm.stream;
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[stream];
+	struct ib_device *device;
+	struct dtr_flow *flow;
+	struct dtr_cm *cm;
+	int offset, err;
+	long t;
+
+retry:
+	t = wait_event_interruptible_timeout(rdma_stream->send_wq,
+			(cm = dtr_select_and_get_cm_for_tx(rdma_transport, stream)),
+			rdma_stream->send_timeout);
+
+	if (t == 0) {
+		struct dtr_transport *rdma_transport = rdma_stream->rdma_transport;
+
+		if (drbd_stream_send_timed_out(&rdma_transport->transport, stream))
+			return -EAGAIN;
+		goto retry;
+	} else if (t < 0)
+		return -EINTR;
+
+	flow = &cm->path->flow[stream];
+	if (atomic_dec_if_positive(&flow->peer_rx_descs) < 0) {
+		kref_put(&cm->kref, dtr_destroy_cm);
+		goto retry;
+	}
+	device = cm->id->device;
+	switch (tx_desc->type) {
+	case SEND_PAGE:
+		offset = tx_desc->sge[0].lkey;
+		tx_desc->sge[0].addr = ib_dma_map_page(device, tx_desc->page, offset,
+						      tx_desc->sge[0].length, DMA_TO_DEVICE);
+		err = ib_dma_mapping_error(device, tx_desc->sge[0].addr);
+		if (err)
+			goto out;
+
+		tx_desc->sge[0].lkey = dtr_cm_to_lkey(cm);
+		break;
+	case SEND_MSG:
+	case SEND_BIO:
+		BUG();
+	}
+
+	err = __dtr_post_tx_desc(cm, tx_desc);
+	if (!err)
+		atomic_inc(&flow->tx_descs_posted);
+	else
+		ib_dma_unmap_page(device, tx_desc->sge[0].addr, tx_desc->sge[0].length, DMA_TO_DEVICE);
+
+
+out:
+	// pr_info("%s: Created send_wr (%p, %p): nr_sges=%u, first seg: lkey=%x, addr=%llx, length=%d\n", rdma_stream->name, tx_desc->page, tx_desc, tx_desc->nr_sges, tx_desc->sge[0].lkey, tx_desc->sge[0].addr, tx_desc->sge[0].length);
+	kref_put(&cm->kref, dtr_destroy_cm);
+	return err;
+}
+
+static int dtr_init_flow(struct dtr_path *path, enum drbd_stream stream)
+{
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	unsigned int alloc_size = rdma_transport->rx_allocation_size;
+	unsigned int rcvbuf_size = RDMA_DEF_BUFFER_SIZE;
+	unsigned int sndbuf_size = RDMA_DEF_BUFFER_SIZE;
+	struct dtr_flow *flow = &path->flow[stream];
+	struct net_conf *nc;
+	int err = 0;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		tr_err(transport, "need net_conf\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (nc->rcvbuf_size)
+		rcvbuf_size = nc->rcvbuf_size;
+	if (nc->sndbuf_size)
+		sndbuf_size = nc->sndbuf_size;
+
+	if (stream == CONTROL_STREAM) {
+		rcvbuf_size = nc->rdma_ctrl_rcvbuf_size ?: max(rcvbuf_size / 64, alloc_size * 8);
+		sndbuf_size = nc->rdma_ctrl_sndbuf_size ?: max(sndbuf_size / 64, alloc_size * 8);
+	}
+
+	if (rcvbuf_size / DRBD_SOCKET_BUFFER_SIZE > nc->max_buffers) {
+		tr_err(transport, "Set max-buffers at least to %d, (right now it is %d).\n",
+		       rcvbuf_size / DRBD_SOCKET_BUFFER_SIZE, nc->max_buffers);
+		tr_err(transport, "This is due to rcvbuf-size = %d.\n", rcvbuf_size);
+		rcu_read_unlock();
+		err = -EINVAL;
+		goto out;
+	}
+
+	rcu_read_unlock();
+
+	flow->path = path;
+	flow->tx_descs_max = sndbuf_size / DRBD_SOCKET_BUFFER_SIZE;
+	flow->rx_descs_max = rcvbuf_size / DRBD_SOCKET_BUFFER_SIZE;
+
+	atomic_set(&flow->tx_descs_posted, 0);
+	atomic_set(&flow->peer_rx_descs, stream == CONTROL_STREAM ? 1 : 0);
+	atomic_set(&flow->rx_descs_known_to_peer, stream == CONTROL_STREAM ? 1 : 0);
+
+	atomic_set(&flow->rx_descs_posted, 0);
+	atomic_set(&flow->rx_descs_allocated, 0);
+
+	flow->rx_descs_want_posted = flow->rx_descs_max / 2;
+
+ out:
+	return err;
+}
+
+static int _dtr_cm_alloc_rdma_res(struct dtr_cm *cm,
+				    enum dtr_alloc_rdma_res_causes *cause)
+{
+	int err, i, rx_descs_max = 0, tx_descs_max = 0;
+	struct ib_cq_init_attr cq_attr = {};
+	struct dtr_path *path = cm->path;
+
+	/* Each path might be the sole path, therefore it must be able to
+	   support both streams */
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		rx_descs_max += path->flow[i].rx_descs_max;
+		tx_descs_max += path->flow[i].tx_descs_max;
+	}
+
+	/* alloc protection domain (PD) */
+	/* in 4.9 ib_alloc_pd got the ability to specify flags as second param */
+	/* so far we don't use flags, but if we start using them, we have to be
+	 * aware that the compat layer removes this parameter for old kernels */
+	cm->pd = ib_alloc_pd(cm->id->device, 0);
+	if (IS_ERR(cm->pd)) {
+		*cause = IB_ALLOC_PD;
+		err = PTR_ERR(cm->pd);
+		goto pd_failed;
+	}
+
+	/* create recv completion queue (CQ) */
+	cq_attr.cqe = rx_descs_max;
+	cm->recv_cq = ib_create_cq(cm->id->device,
+			dtr_rx_cq_event_handler, NULL, cm,
+			&cq_attr);
+	if (IS_ERR(cm->recv_cq)) {
+		*cause = IB_CREATE_CQ_RX;
+		err = PTR_ERR(cm->recv_cq);
+		goto recv_cq_failed;
+	}
+
+	/* create send completion queue (CQ) */
+	cq_attr.cqe = tx_descs_max;
+	cm->send_cq = ib_create_cq(cm->id->device,
+			dtr_tx_cq_event_handler, NULL, cm,
+			&cq_attr);
+	if (IS_ERR(cm->send_cq)) {
+		*cause = IB_CREATE_CQ_TX;
+		err = PTR_ERR(cm->send_cq);
+		goto send_cq_failed;
+	}
+
+	/* arm CQs */
+	err = ib_req_notify_cq(cm->recv_cq, IB_CQ_NEXT_COMP);
+	if (err) {
+		*cause = IB_REQ_NOTIFY_CQ_RX;
+		goto notify_failed;
+	}
+
+	err = ib_req_notify_cq(cm->send_cq, IB_CQ_NEXT_COMP);
+	if (err) {
+		*cause = IB_REQ_NOTIFY_CQ_TX;
+		goto notify_failed;
+	}
+
+	/* create a queue pair (QP) */
+	err = dtr_create_qp(cm, rx_descs_max, tx_descs_max);
+	if (err) {
+		*cause = RDMA_CREATE_QP;
+		goto createqp_failed;
+	}
+
+	/* some RDMA transports need at least one rx desc for establishing a connection */
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		dtr_create_rx_desc(&path->flow[i], GFP_NOIO, false);
+
+	return 0;
+
+createqp_failed:
+notify_failed:
+	ib_destroy_cq(cm->send_cq);
+	cm->send_cq = NULL;
+send_cq_failed:
+	ib_destroy_cq(cm->recv_cq);
+	cm->recv_cq = NULL;
+recv_cq_failed:
+	ib_dealloc_pd(cm->pd);
+	cm->pd = NULL;
+pd_failed:
+	return err;
+}
+
+
+static int dtr_cm_alloc_rdma_res(struct dtr_cm *cm)
+{
+	struct dtr_path *path = cm->path;
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	enum dtr_alloc_rdma_res_causes cause;
+	struct ib_device_attr dev_attr;
+	struct ib_udata uhw = {.outlen = 0, .inlen = 0};
+	struct ib_device *device = cm->id->device;
+	int rx_descs_max = 0, tx_descs_max = 0;
+	bool reduced = false;
+	int i, hca_max, err, dev_sge;
+
+	static const char * const err_txt[] = {
+		[IB_ALLOC_PD] = "ib_alloc_pd()",
+		[IB_CREATE_CQ_RX] = "ib_create_cq() rx",
+		[IB_CREATE_CQ_TX] = "ib_create_cq() tx",
+		[IB_REQ_NOTIFY_CQ_RX] = "ib_req_notify_cq() rx",
+		[IB_REQ_NOTIFY_CQ_TX] = "ib_req_notify_cq() tx",
+		[RDMA_CREATE_QP] = "rdma_create_qp()",
+		[IB_GET_DMA_MR] = "ib_get_dma_mr()",
+	};
+
+	err = device->ops.query_device(device, &dev_attr, &uhw);
+	if (err) {
+		tr_err(transport, "ib_query_device: %d\n", err);
+		return err;
+	}
+
+	dev_sge = min(dev_attr.max_send_sge, dev_attr.max_recv_sge);
+	if (rdma_transport->sges_max > dev_sge)
+		rdma_transport->sges_max = dev_sge;
+
+	hca_max = min(dev_attr.max_qp_wr, dev_attr.max_cqe);
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		rx_descs_max += path->flow[i].rx_descs_max;
+		tx_descs_max += path->flow[i].tx_descs_max;
+	}
+
+	if (tx_descs_max > hca_max || rx_descs_max > hca_max) {
+		int rx_correction = 0, tx_correction = 0;
+		reduced = true;
+
+		if (tx_descs_max > hca_max)
+			tx_correction = hca_max - tx_descs_max;
+
+		if (rx_descs_max > hca_max)
+			rx_correction = hca_max - rx_descs_max;
+
+		path->flow[DATA_STREAM].rx_descs_max -= rx_correction;
+		path->flow[DATA_STREAM].tx_descs_max -= tx_correction;
+
+		rx_descs_max -= rx_correction;
+		tx_descs_max -= tx_correction;
+	}
+
+	for (;;) {
+		err = _dtr_cm_alloc_rdma_res(cm, &cause);
+
+		if (err == 0 || cause != RDMA_CREATE_QP || err != -ENOMEM)
+			break;
+
+		reduced = true;
+		if (path->flow[DATA_STREAM].rx_descs_max <= 64)
+			break;
+		path->flow[DATA_STREAM].rx_descs_max -= 64;
+		if (path->flow[DATA_STREAM].tx_descs_max <= 64)
+			break;
+		path->flow[DATA_STREAM].tx_descs_max -= 64;
+		if (path->flow[CONTROL_STREAM].rx_descs_max > 8)
+			path->flow[CONTROL_STREAM].rx_descs_max -= 1;
+		if (path->flow[CONTROL_STREAM].tx_descs_max > 8)
+			path->flow[CONTROL_STREAM].tx_descs_max -= 1;
+	}
+
+	if (err) {
+		tr_err(transport, "%s failed with err = %d\n", err_txt[cause], err);
+	} else if (reduced) {
+		/* ib_create_qp() may return -ENOMEM if max_send_wr or max_recv_wr are
+		   too big. Unfortunately there is no way to find the working maxima.
+		   http://www.rdmamojo.com/2012/12/21/ibv_create_qp/
+		   Suggests "Trial end error" to find the maximal number. */
+
+		tr_warn(transport, "Needed to adjust buffer sizes for HCA\n");
+		tr_warn(transport, "rcvbuf = %d sndbuf = %d \n",
+			path->flow[DATA_STREAM].rx_descs_max * DRBD_SOCKET_BUFFER_SIZE,
+			path->flow[DATA_STREAM].tx_descs_max * DRBD_SOCKET_BUFFER_SIZE);
+		tr_warn(transport, "It is recommended to apply this change to the configuration\n");
+	}
+
+	return err;
+}
+
+static void dtr_end_rx_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, end_rx_work);
+	struct dtr_rx_desc *rx_desc, *tmp;
+	unsigned long irq_flags;
+	LIST_HEAD(rx_descs);
+
+	spin_lock_irqsave(&cm->error_rx_descs_lock, irq_flags);
+	list_splice_init(&cm->error_rx_descs, &rx_descs);
+	spin_unlock_irqrestore(&cm->error_rx_descs_lock, irq_flags);
+	list_for_each_entry_safe(rx_desc, tmp, &rx_descs, list)
+		dtr_free_rx_desc(rx_desc);
+	kref_put(&cm->kref, dtr_destroy_cm);
+}
+
+static void dtr_end_tx_work_fn(struct work_struct *work)
+{
+	struct dtr_cm *cm = container_of(work, struct dtr_cm, end_tx_work);
+
+	kref_put(&cm->kref, dtr_destroy_cm);
+}
+
+static void __dtr_disconnect_path(struct dtr_path *path)
+{
+	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
+	struct drbd_transport *transport;
+	enum connect_state_enum a, p;
+	bool was_scheduled;
+	struct dtr_cm *cm;
+	long t;
+	int err;
+
+	if (!path)
+		return;
+
+	transport = path->path.transport;
+
+	a = atomic_cmpxchg(&path->cs.active_state, PCS_CONNECTING, PCS_REQUEST_ABORT);
+	p = atomic_cmpxchg(&path->cs.passive_state, PCS_CONNECTING, PCS_INACTIVE);
+
+	switch (p) {
+	case PCS_CONNECTING:
+		drbd_put_listener(&path->path);
+		break;
+	case PCS_FINISHING:
+		t = wait_event_timeout(path->cs.wq,
+				       atomic_read(&path->cs.passive_state) == PCS_INACTIVE,
+				       HZ * 60);
+		if (t == 0)
+			tr_warn(transport, "passive_state still %d\n", atomic_read(&path->cs.passive_state));
+		fallthrough;
+	case PCS_INACTIVE:
+		break;
+	}
+
+	switch (a) {
+	case PCS_CONNECTING:
+		was_scheduled = flush_delayed_work(&path->cs.retry_connect_work);
+		if (!was_scheduled) {
+			atomic_set(&path->cs.active_state, PCS_INACTIVE);
+			break;
+		}
+		fallthrough;
+	case PCS_REQUEST_ABORT:
+		t = wait_event_timeout(path->cs.wq,
+				       atomic_read(&path->cs.active_state) == PCS_INACTIVE,
+				       HZ * 60);
+		if (t == 0)
+			tr_warn(transport, "active_state still %d\n", atomic_read(&path->cs.active_state));
+		fallthrough;
+	case PCS_INACTIVE:
+		break;
+	}
+
+	cm = dtr_path_get_cm(path);
+	if (!cm)
+		return;
+
+	err = rdma_disconnect(cm->id);
+	if (err) {
+		tr_warn(transport, "failed to disconnect, id %p context %p err %d\n",
+			cm->id, cm->id->context, err);
+		/* We are ignoring errors here on purpose */
+		goto out;
+	}
+
+	/* There might be a signal pending here. Not incorruptible! */
+	wait_event_timeout(cm->state_wq,
+			   !test_bit(DSB_CONNECTED, &cm->state),
+			   HZ);
+
+	if (test_bit(DSB_CONNECTED, &cm->state))
+		tr_warn(transport, "WARN: not properly disconnected, state = %lu\n",
+			cm->state);
+
+ out:
+	/* between dtr_alloc_cm() and dtr_cm_alloc_rdma_res() cm->id->qp is NULL */
+	if (cm->id->qp) {
+		/* With putting the QP into error state, it has to hand back
+		   all posted rx_descs */
+		err = ib_modify_qp(cm->id->qp, &attr, IB_QP_STATE);
+		if (err)
+			tr_err(transport, "ib_modify_qp failed %d\n", err);
+	}
+
+	/*
+	 * We are expecting one of RDMA_CM_EVENT_ESTABLISHED, _UNREACHABLE,
+	 * _CONNECT_ERROR, or _REJECTED on this cm. Some RDMA drivers report
+	 * these error events after unexpectedly long timeouts, while others do
+	 * not report it at all. We are no longer interested in these
+	 * events. Destroy the cm and cm_id to avoid leaking it.
+	 * This is racing with the event delivery, which drops a reference.
+	 */
+	if (test_and_clear_bit(DSB_CONNECTING, &cm->state) ||
+	    test_and_clear_bit(DSB_CONNECT_REQ, &cm->state))
+		kref_put(&cm->kref, dtr_destroy_cm);
+
+	kref_put(&cm->kref, dtr_destroy_cm);
+}
+
+static void dtr_reclaim_cm(struct rcu_head *rcu_head)
+{
+	struct dtr_cm *cm = container_of(rcu_head, struct dtr_cm, rcu);
+
+	kfree(cm);
+	module_put(THIS_MODULE);
+}
+
+/* dtr_destroy_cm() might run after the transport was destroyed */
+static void __dtr_destroy_cm(struct kref *kref, bool destroy_id)
+{
+	struct dtr_cm *cm = container_of(kref, struct dtr_cm, kref);
+
+	if (cm->id) {
+		if (cm->id->qp)
+			rdma_destroy_qp(cm->id);
+		cm->id->qp = NULL;
+	}
+
+	if (cm->send_cq) {
+		ib_destroy_cq(cm->send_cq);
+		cm->send_cq = NULL;
+	}
+
+	if (cm->recv_cq) {
+		ib_destroy_cq(cm->recv_cq);
+		cm->recv_cq = NULL;
+	}
+
+	if (cm->pd) {
+		ib_dealloc_pd(cm->pd);
+		cm->pd = NULL;
+	}
+
+	if (cm->id) {
+		/* Just in case some callback is still triggered
+		 * after we kfree'd path. */
+		cm->id->context = NULL;
+		if (destroy_id)
+			rdma_destroy_id(cm->id);
+		cm->id = NULL;
+	}
+	if (cm->path) {
+		kref_put(&cm->path->path.kref, drbd_destroy_path);
+		cm->path = NULL;
+	}
+
+	call_rcu(&cm->rcu, dtr_reclaim_cm);
+}
+
+static void dtr_destroy_cm(struct kref *kref)
+{
+	__dtr_destroy_cm(kref, true);
+}
+
+static void dtr_destroy_cm_keep_id(struct kref *kref)
+{
+	__dtr_destroy_cm(kref, false);
+}
+
+static void dtr_disconnect_path(struct dtr_path *path)
+{
+	struct dtr_cm *cm;
+
+	if (!path)
+		return;
+
+	__dtr_disconnect_path(path);
+
+	cm = xchg(&path->cm, NULL); // RCU xchg
+	if (cm)
+		kref_put(&cm->kref, dtr_destroy_cm);
+}
+
+static void dtr_destroy_listener(struct drbd_listener *generic_listener)
+{
+	struct dtr_listener *listener =
+		container_of(generic_listener, struct dtr_listener, listener);
+
+	if (listener->cm.id)
+		rdma_destroy_id(listener->cm.id);
+}
+
+static int dtr_init_listener(struct drbd_transport *transport, const struct sockaddr *addr, struct net *net, struct drbd_listener *drbd_listener)
+{
+	struct dtr_listener *listener = container_of(drbd_listener, struct dtr_listener, listener);
+	struct sockaddr_storage my_addr;
+	int err = -ENOMEM;
+
+	my_addr = *(struct sockaddr_storage *)addr;
+
+	err = dtr_create_cm_id(&listener->cm, net);
+	if (err) {
+		tr_err(transport, "rdma_create_id() failed\n");
+		goto out;
+	}
+	listener->cm.state = 0; /* listening */
+
+	err = rdma_bind_addr(listener->cm.id, (struct sockaddr *)&my_addr);
+	if (err) {
+		tr_err(transport, "rdma_bind_addr error %d\n", err);
+		goto out;
+	}
+
+	err = rdma_listen(listener->cm.id, 1);
+	if (err) {
+		tr_err(transport, "rdma_listen error %d\n", err);
+		goto out;
+	}
+
+	listener->listener.listen_addr = *(struct sockaddr_storage *)addr;
+
+	return 0;
+out:
+	if (listener->cm.id) {
+		rdma_destroy_id(listener->cm.id);
+		listener->cm.id = NULL;
+	}
+
+	return err;
+}
+
+static int dtr_activate_path(struct dtr_path *path)
+{
+	struct drbd_transport *transport = path->path.transport;
+	struct dtr_connect_state *cs;
+	int err = -ENOMEM;
+
+	cs = &path->cs;
+
+	init_waitqueue_head(&cs->wq);
+
+	atomic_set(&cs->passive_state, PCS_CONNECTING);
+	atomic_set(&cs->active_state, PCS_CONNECTING);
+
+	if (path->path.listener) {
+		tr_warn(transport, "ASSERTION FAILED: in dtr_activate_path() found listener, dropping it\n");
+		drbd_put_listener(&path->path);
+	}
+	err = drbd_get_listener(&path->path);
+	if (err)
+		goto out_no_put;
+
+	/*
+	 * Check passive_state after drbd_get_listener() completed.
+	 * __dtr_disconnect_path() sets passive_state before calling
+	 * drbd_put_listener(). That drbd_put_listner() might return
+	 * before the drbd_get_listner() here started.
+	 */
+	if (atomic_read(&cs->passive_state) != PCS_CONNECTING ||
+	    atomic_read(&cs->active_state) != PCS_CONNECTING)
+		goto out;
+
+	err = dtr_start_try_connect(cs);
+	if (err)
+		goto out;
+
+	return 0;
+
+out:
+	drbd_put_listener(&path->path);
+out_no_put:
+	atomic_set(&cs->passive_state, PCS_INACTIVE);
+	atomic_set(&cs->active_state, PCS_INACTIVE);
+	wake_up(&cs->wq);
+
+	return err;
+}
+
+static int dtr_prepare_connect(struct drbd_transport *transport)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+
+	struct dtr_stream *data_stream = NULL, *control_stream = NULL;
+	struct dtr_path *path;
+	struct net_conf *nc;
+	int timeout, err = -ENOMEM;
+
+	flush_signals(current);
+
+	if (!list_first_or_null_rcu(&transport->paths, struct drbd_path, list))
+		return -EDESTADDRREQ;
+
+	data_stream = &rdma_transport->stream[DATA_STREAM];
+	dtr_re_init_stream(data_stream);
+
+	control_stream = &rdma_transport->stream[CONTROL_STREAM];
+	dtr_re_init_stream(control_stream);
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+
+	timeout = nc->timeout * HZ / 10;
+	rcu_read_unlock();
+
+	data_stream->send_timeout = timeout;
+	control_stream->send_timeout = timeout;
+
+	atomic_set(&rdma_transport->first_path_connect_err, 1);
+	init_completion(&rdma_transport->connected);
+
+	rdma_transport->active = true;
+
+	list_for_each_entry(path, &transport->paths, path.list) {
+		err = dtr_activate_path(path);
+		if (err)
+			goto abort;
+	}
+
+	return 0;
+
+abort:
+	rdma_transport->active = false;
+	return err;
+}
+
+static int dtr_connect(struct drbd_transport *transport)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	int i, err = -ENOMEM;
+
+	err = wait_for_completion_interruptible(&rdma_transport->connected);
+	if (err) {
+		flush_signals(current);
+		goto abort;
+	}
+
+	err = atomic_read(&rdma_transport->first_path_connect_err);
+	if (err == 1)
+		err = -EAGAIN;
+	if (err)
+		goto abort;
+
+
+	/* Make sure at least one path has rx_descs... */
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+		dtr_refill_rx_desc(rdma_transport, i);
+
+	/* make sure the other side had time to create rx_descs */
+	schedule_timeout(HZ / 4);
+
+	return 0;
+
+abort:
+	rdma_transport->active = false;
+
+	return err;
+}
+
+static void dtr_finish_connect(struct drbd_transport *transport)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+
+	if (!rdma_transport->active) {
+		struct dtr_path *path;
+
+		list_for_each_entry(path, &transport->paths, path.list)
+			dtr_disconnect_path(path);
+	}
+}
+
+static int dtr_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf)
+{
+	struct net_conf *old_net_conf;
+	struct dtr_transport *dtr_transport = container_of(transport,
+		struct dtr_transport, transport);
+	int ret = 0;
+
+	rcu_read_lock();
+	old_net_conf = rcu_dereference(transport->net_conf);
+	if (old_net_conf && dtr_transport->active) {
+		if (old_net_conf->sndbuf_size != new_net_conf->sndbuf_size) {
+			tr_warn(transport, "online change of sndbuf_size not supported\n");
+			ret = -EINVAL;
+		}
+		if (old_net_conf->rcvbuf_size != new_net_conf->rcvbuf_size) {
+			tr_warn(transport, "online change of rcvbuf_size not supported\n");
+			ret = -EINVAL;
+		}
+	}
+	rcu_read_unlock();
+
+	return ret;
+}
+
+static void dtr_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+
+	rdma_transport->stream[stream].recv_timeout = timeout;
+
+	if (stream == CONTROL_STREAM)
+		mod_timer(&rdma_transport->control_timer, jiffies + timeout);
+}
+
+static long dtr_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+
+	return rdma_transport->stream[stream].recv_timeout;
+}
+
+static bool dtr_path_ok(struct dtr_path *path)
+{
+	bool r = false;
+	struct dtr_cm *cm = path->cm;
+
+	rcu_read_lock();
+	cm = rcu_dereference(path->cm);
+	if (cm) {
+		r = cm->id && cm->state == DSM_CONNECTED;
+	}
+	rcu_read_unlock();
+
+	return r;
+}
+
+static bool dtr_transport_ok(struct drbd_transport *transport)
+{
+	struct dtr_path *path;
+	bool r = false;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &transport->paths, path.list) {
+		r = dtr_path_ok(path);
+		if (r)
+			break;
+	}
+	rcu_read_unlock();
+
+	return r;
+}
+
+static bool dtr_stream_ok(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	return dtr_transport_ok(transport);
+}
+
+static void dtr_update_congested(struct drbd_transport *transport)
+{
+	struct dtr_path *path;
+	bool congested = true;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &transport->paths, path.list) {
+		struct dtr_flow *flow = &path->flow[DATA_STREAM];
+		bool path_congested = false;
+		int tx_descs_posted;
+
+		if (!dtr_path_ok(path))
+			continue;
+
+		tx_descs_posted = atomic_read(&flow->tx_descs_posted);
+		path_congested |= flow->tx_descs_max - tx_descs_posted < DESCS_LOW_LEVEL;
+		path_congested |= atomic_read(&flow->peer_rx_descs) < DESCS_LOW_LEVEL;
+
+		if (!path_congested) {
+			congested = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	if (congested)
+		set_bit(NET_CONGESTED, &transport->flags);
+}
+
+static int dtr_send_page(struct drbd_transport *transport, enum drbd_stream stream,
+			 struct page *page, int offset, size_t size, unsigned msg_flags)
+{
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_tx_desc *tx_desc;
+	int err;
+
+	// pr_info("%s: in send_page, size: %zu\n", rdma_stream->name, size);
+
+	if (!dtr_transport_ok(transport))
+		return -ECONNRESET;
+
+	tx_desc = kmalloc(sizeof(*tx_desc) + sizeof(struct ib_sge), GFP_NOIO);
+	if (!tx_desc)
+		return -ENOMEM;
+
+	get_page(page); /* The put_page() is in dtr_tx_cq_event_handler() */
+	tx_desc->type = SEND_PAGE;
+	tx_desc->page = page;
+	tx_desc->nr_sges = 1;
+	tx_desc->imm = (union dtr_immediate)
+		{ .stream = stream,
+		  .sequence = rdma_transport->stream[stream].tx_sequence++
+		};
+	tx_desc->sge[0].length = size;
+	tx_desc->sge[0].lkey = offset; /* abusing lkey fild. See dtr_post_tx_desc() */
+
+	err = dtr_post_tx_desc(rdma_transport, tx_desc);
+	if (err) {
+		put_page(page);
+		kfree(tx_desc);
+	}
+
+	if (stream == DATA_STREAM)
+		dtr_update_congested(transport);
+
+	return err;
+}
+
+#if SENDER_COMPACTS_BVECS
+static int dtr_send_bio_part(struct dtr_transport *rdma_transport,
+			     struct bio *bio, int start, int size_tx_desc, int sges)
+{
+	struct dtr_stream *rdma_stream = &rdma_transport->stream[DATA_STREAM];
+	struct dtr_tx_desc *tx_desc;
+	struct ib_device *device;
+	struct dtr_path *path = NULL;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+	int i = 0, pos = 0, done = 0, err;
+
+	if (!size_tx_desc)
+		return 0;
+
+	//tr_info(&rdma_transport->transport,
+	//	"  dtr_send_bio_part(start = %d, size = %d, sges = %d)\n",
+	//	start, size_tx_desc, sges);
+
+	tx_desc = kmalloc(sizeof(*tx_desc) + sizeof(struct ib_sge) * sges, GFP_NOIO);
+	if (!tx_desc)
+		return -ENOMEM;
+
+	tx_desc->type = SEND_BIO;
+	tx_desc->bio = bio;
+	tx_desc->nr_sges = sges;
+	device = rdma_stream->cm.id->device;
+
+	bio_for_each_segment(bvec, tx_desc->bio, iter) {
+		struct page *page = bvec.bv_page;
+		int offset = bvec.bv_offset;
+		int size = bvec.bv_len;
+		int shift = 0;
+		get_page(page);
+
+		if (pos < start || done == size_tx_desc) {
+			if (done != size_tx_desc && pos + size > start) {
+				shift = (start - pos);
+			} else {
+				pos += size;
+				continue;
+			}
+		}
+
+		pos += size;
+		offset += shift;
+		size = min(size - shift, size_tx_desc - done);
+
+		//tr_info(&rdma_transport->transport,
+		//	"   sge (i = %d, offset = %d, size = %d)\n",
+		//	i, offset, size);
+
+		tx_desc->sge[i].addr = ib_dma_map_page(device, page, offset, size, DMA_TO_DEVICE);
+		err = ib_dma_mapping_error(device, tx_desc->sge[i].addr);
+		if (err)
+			return err; // FIX THIS
+		tx_desc->sge[i].lkey = dtr_path_to_lkey(path);
+		tx_desc->sge[i].length = size;
+		done += size;
+		i++;
+	}
+
+	TR_ASSERT(&rdma_transport->transport, done == size_tx_desc);
+	tx_desc->imm = (union dtr_immediate)
+		{ .stream = ST_DATA,
+		  .sequence = rdma_transport->stream[ST_DATA].tx_sequence++
+		};
+
+	err = dtr_post_tx_desc(rdma_stream, tx_desc, &path);
+	if (err) {
+		if (path) {
+			dtr_free_tx_desc(path, tx_desc);
+		} else {
+			bio_for_each_segment(bvec, tx_desc->bio, iter) {
+				put_page(bvec.bv_page);
+			}
+			kfree(tx_desc);
+		}
+	}
+
+	return err;
+}
+#endif
+
+static int dtr_send_zc_bio(struct drbd_transport *transport, struct bio *bio)
+{
+#if SENDER_COMPACTS_BVECS
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	int start = 0, sges = 0, size_tx_desc = 0, remaining = 0, err;
+	int sges_max = rdma_transport->sges_max;
+#endif
+	int err = -EINVAL;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+
+	//tr_info(transport, "in send_zc_bio, size: %d\n", bio->bi_size);
+
+	if (!dtr_transport_ok(transport))
+		return -ECONNRESET;
+
+#if SENDER_COMPACTS_BVECS
+	bio_for_each_segment(bvec, bio, iter) {
+		size_tx_desc += bvec.bv_len;
+		//tr_info(transport, " bvec len = %d\n", bvec.bv_len);
+		if (size_tx_desc > DRBD_SOCKET_BUFFER_SIZE) {
+			remaining = size_tx_desc - DRBD_SOCKET_BUFFER_SIZE;
+			size_tx_desc = DRBD_SOCKET_BUFFER_SIZE;
+		}
+		sges++;
+		if (size_tx_desc == DRBD_SOCKET_BUFFER_SIZE || sges >= sges_max) {
+			err = dtr_send_bio_part(rdma_transport, bio, start, size_tx_desc, sges);
+			if (err)
+				goto out;
+			start += size_tx_desc;
+			sges = 0;
+			size_tx_desc = remaining;
+			if (remaining) {
+				sges++;
+				remaining = 0;
+			}
+		}
+	}
+	err = dtr_send_bio_part(rdma_transport, bio, start, size_tx_desc, sges);
+	start += size_tx_desc;
+
+	TR_ASSERT(transport, start == bio->bi_iter.bi_size);
+out:
+#else
+	bio_for_each_segment(bvec, bio, iter) {
+		err = dtr_send_page(transport, DATA_STREAM,
+			bvec.bv_page, bvec.bv_offset, bvec.bv_len,
+			0 /* flags currently unused by dtr_send_page */);
+		if (err)
+			break;
+	}
+#endif
+	if (1 /* stream == DATA_STREAM */)
+		dtr_update_congested(transport);
+
+	return err;
+}
+
+static bool dtr_hint(struct drbd_transport *transport, enum drbd_stream stream,
+		enum drbd_tr_hints hint)
+{
+	switch (hint) {
+	default: /* not implemented, but should not trigger error handling */
+		return true;
+	}
+	return true;
+}
+
+static void dtr_debugfs_show_flow(struct dtr_flow *flow, const char *name, struct seq_file *m)
+{
+	seq_printf(m,    " %-7s field:  posted\t alloc\tdesired\t  max\n", name);
+	seq_printf(m, "      tx_descs: %5d\t\t\t%5d\n", atomic_read(&flow->tx_descs_posted), flow->tx_descs_max);
+	seq_printf(m, " peer_rx_descs: %5d (receive window at peer)\n", atomic_read(&flow->peer_rx_descs));
+	seq_printf(m, "      rx_descs: %5d\t%5d\t%5d\t%5d\n", atomic_read(&flow->rx_descs_posted),
+		   atomic_read(&flow->rx_descs_allocated),
+		   flow->rx_descs_want_posted, flow->rx_descs_max);
+	seq_printf(m, " rx_peer_knows: %5d (what the peer knows about my receive window)\n\n",
+		   atomic_read(&flow->rx_descs_known_to_peer));
+}
+
+static void dtr_debugfs_show_path(struct dtr_path *path, struct seq_file *m)
+{
+	static const char *stream_names[] = {
+		[ST_DATA] = "data",
+		[ST_CONTROL] = "control",
+	};
+	static const char *state_names[] = {
+		[0] = "not connected",
+		[DSM_CONNECT_REQ] = "CONNECT_REQ",
+		[DSM_CONNECTING] = "CONNECTING",
+		[DSM_CONNECTING|DSM_CONNECT_REQ] = "CONNECTING|DSM_CONNECT_REQ",
+		[DSM_CONNECTED] = "CONNECTED",
+		[DSM_CONNECTED|DSM_CONNECT_REQ] = "CONNECTED|CONNECT_REQ",
+		[DSM_CONNECTED|DSM_CONNECTING] = "CONNECTED|CONNECTING",
+		[DSM_CONNECTED|DSM_CONNECTING|DSM_CONNECT_REQ] =
+			"CONNECTED|CONNECTING|DSM_CONNECT_REQ",
+		[DSM_ERROR] = "ERROR",
+		[DSM_ERROR|DSM_CONNECT_REQ] = "ERROR|CONNECT_REQ",
+		[DSM_ERROR|DSM_CONNECTING] = "ERROR|CONNECTING",
+		[DSM_ERROR|DSM_CONNECTING|DSM_CONNECT_REQ] = "ERROR|CONNECTING|CONNECT_REQ",
+		[DSM_ERROR|DSM_CONNECTED] = "ERROR|CONNECTED",
+		[DSM_ERROR|DSM_CONNECTED|DSM_CONNECT_REQ] = "ERROR|CONNECTED|CONNECT_REQ",
+		[DSM_ERROR|DSM_CONNECTED|DSM_CONNECTING] = "ERROR|CONNECTED|CONNECTING|",
+		[DSM_ERROR|DSM_CONNECTED|DSM_CONNECTING|DSM_CONNECT_REQ] =
+			"ERROR|CONNECTED|CONNECTING|CONNECT_REQ",
+	};
+
+	enum drbd_stream i;
+	unsigned long s = 0;
+	struct dtr_cm *cm;
+
+	rcu_read_lock();
+	cm = rcu_dereference(path->cm);
+	if (cm)
+		s = cm->state;
+	rcu_read_unlock();
+
+	seq_printf(m, "%pI4 - %pI4: %s\n",
+		   &((struct sockaddr_in *)&path->path.my_addr)->sin_addr,
+		   &((struct sockaddr_in *)&path->path.peer_addr)->sin_addr,
+		   state_names[s]);
+
+	if (dtr_path_ok(path)) {
+		for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++)
+			dtr_debugfs_show_flow(&path->flow[i], stream_names[i], m);
+	}
+}
+
+static void dtr_debugfs_show(struct drbd_transport *transport, struct seq_file *m)
+{
+	struct dtr_path *path;
+
+	/* BUMP me if you change the file format/content/presentation */
+	seq_printf(m, "v: %u\n\n", 1);
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(path, &transport->paths, path.list)
+		dtr_debugfs_show_path(path, m);
+	rcu_read_unlock();
+}
+
+static int dtr_add_path(struct drbd_path *add_path)
+{
+	struct drbd_transport *transport = add_path->transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct dtr_path *path;
+
+	path = container_of(add_path, struct dtr_path, path);
+
+	/* initialize private parts of path */
+	atomic_set(&path->cs.passive_state, PCS_INACTIVE);
+	atomic_set(&path->cs.active_state, PCS_INACTIVE);
+	spin_lock_init(&path->send_flow_control_lock);
+	tasklet_setup(&path->flow_control_tasklet, dtr_flow_control_tasklet_fn);
+	INIT_WORK(&path->refill_rx_descs_work, dtr_refill_rx_descs_work_fn);
+	INIT_DELAYED_WORK(&path->cs.retry_connect_work, dtr_cma_retry_connect_work_fn);
+
+	if (!rdma_transport->active)
+		return 0;
+
+	return dtr_activate_path(path);
+}
+
+static bool dtr_may_remove_path(struct drbd_path *del_path)
+{
+	struct drbd_transport *transport = del_path->transport;
+	struct dtr_transport *rdma_transport =
+		container_of(transport, struct dtr_transport, transport);
+	struct drbd_path *drbd_path, *connected_path = NULL;
+	int connected = 0;
+
+	if (!rdma_transport->active)
+		return true;
+
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		struct dtr_path *path = container_of(drbd_path, struct dtr_path, path);
+
+		if (dtr_path_ok(path)) {
+			connected++;
+			connected_path = drbd_path;
+		}
+	}
+
+	return connected > 1 || connected_path != del_path;
+}
+
+static void dtr_remove_path(struct drbd_path *del_path)
+{
+	struct dtr_path *path = container_of(del_path, struct dtr_path, path);
+
+	dtr_disconnect_path(path);
+}
+
+static int __init dtr_initialize(void)
+{
+	allocation_size = PAGE_SIZE;
+
+	return drbd_register_transport_class(&rdma_transport_class,
+					     DRBD_TRANSPORT_API_VERSION,
+					     sizeof(struct drbd_transport));
+}
+
+static void __exit dtr_cleanup(void)
+{
+	drbd_unregister_transport_class(&rdma_transport_class);
+}
+
+module_init(dtr_initialize)
+module_exit(dtr_cleanup)
diff --git a/drivers/block/drbd/drbd_transport_tcp.c b/drivers/block/drbd/drbd_transport_tcp.c
new file mode 100644
index 0000000000000..8ece2c56dee6c
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport_tcp.c
@@ -0,0 +1,1668 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+   drbd_transport_tcp.c
+
+   This file is part of DRBD.
+
+   Copyright (C) 2014-2017, LINBIT HA-Solutions GmbH.
+
+
+*/
+
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/socket.h>
+#include <linux/pkt_sched.h>
+#include <linux/sched/signal.h>
+#include <linux/net.h>
+#include <linux/file.h>
+#include <linux/tcp.h>
+#include <linux/highmem.h>
+#include <linux/bio.h>
+#include <linux/drbd_genl_api.h>
+#include <linux/drbd_config.h>
+#include <linux/tls.h>
+#include <net/tcp.h>
+#include <net/handshake.h>
+#include <net/tls.h>
+#include <net/tls_prot.h>
+#include <linux/drbd_protocol.h>
+#include "drbd_transport.h"
+
+
+MODULE_AUTHOR("Philipp Reisner <philipp.reisner@linbit.com>");
+MODULE_AUTHOR("Lars Ellenberg <lars.ellenberg@linbit.com>");
+MODULE_AUTHOR("Roland Kammerer <roland.kammerer@linbit.com>");
+MODULE_DESCRIPTION("TCP (SDP, SSOCKS) transport layer for DRBD");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(REL_VERSION);
+
+/* TCP keepalive has proven to be vital in many deployment scenarios.
+ * Without keepalive, after a device has seen a sufficiently long period of
+ * idle time, packets on our "bulk data" socket may be dropped because an
+ * overly "smart" network infrastructure decided that TCP session was stale.
+ * Note that we don't try to use this to detect "broken" tcp sessions here,
+ * these will still be handled by the DRBD effective network timeout via
+ * timeout / ko-count settings.
+ * We use this to try to keep "idle" TCP sessions "alive".
+ * Default to send a probe every 23 seconds.
+ */
+#define DRBD_KEEP_IDLE	23
+#define DRBD_KEEP_INTVL 23
+#define DRBD_KEEP_CNT	9
+static unsigned int drbd_keepcnt = DRBD_KEEP_CNT;
+module_param_named(keepcnt, drbd_keepcnt, uint, 0664);
+MODULE_PARM_DESC(keepcnt, "see tcp(7) tcp_keepalive_probes; set TCP_KEEPCNT for data sockets; default: 9");
+static unsigned int drbd_keepidle = DRBD_KEEP_IDLE;
+module_param_named(keepidle, drbd_keepidle, uint, 0664);
+MODULE_PARM_DESC(keepidle, "see tcp(7) tcp_keepalive_time; set TCP_KEEPIDLE for data sockets; default: 23s");
+static unsigned int drbd_keepintvl = DRBD_KEEP_INTVL;
+module_param_named(keepintvl, drbd_keepintvl, uint, 0664);
+MODULE_PARM_DESC(keepintvtl, "see tcp(7) tcp_keepalive_intvl; set TCP_KEEPINTVL for data sockets; default: 23s");
+
+static struct workqueue_struct *dtt_csocket_recv;
+
+struct buffer {
+	void *base;
+	void *pos;
+};
+
+#define DTT_CONNECTING 1
+#define DTT_DATA_READY_ARMED 2
+
+struct drbd_tcp_transport {
+	struct drbd_transport transport; /* Must be first! */
+	spinlock_t control_recv_lock;
+	unsigned long flags;
+	struct socket *stream[2];
+	struct buffer rbuf[2];
+	struct timer_list control_timer;
+	struct work_struct control_data_ready_work;
+	void (*original_control_sk_state_change)(struct sock *sk);
+	void (*original_control_sk_data_ready)(struct sock *sk);
+};
+
+struct dtt_listener {
+	struct drbd_listener listener;
+	void (*original_sk_state_change)(struct sock *sk);
+	struct socket *s_listen;
+
+	wait_queue_head_t wait; /* woken if a connection came in */
+};
+
+/* Since each path might have a different local IP address, each
+   path might need its own listener. Therefore the drbd_waiter object
+   is embedded into the dtt_path and _not_ the dtt_waiter. */
+
+struct dtt_socket_container {
+	struct list_head list;
+	struct socket *socket;
+};
+
+struct dtt_path {
+	struct drbd_path path;
+
+	struct list_head sockets; /* sockets passed to me by other receiver threads */
+};
+
+static int dtt_init(struct drbd_transport *transport);
+static void dtt_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op);
+static void dtt_socket_free(struct socket **sock);
+static int dtt_init_listener(struct drbd_transport *transport, const struct sockaddr *addr,
+			     struct net *net, struct drbd_listener *drbd_listener);
+static void dtt_destroy_listener(struct drbd_listener *generic_listener);
+static int dtt_prepare_connect(struct drbd_transport *transport);
+static int dtt_connect(struct drbd_transport *transport);
+static void dtt_finish_connect(struct drbd_transport *transport);
+static int dtt_recv(struct drbd_transport *transport, enum drbd_stream stream, void **buf, size_t size, int flags);
+static int dtt_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain, size_t size);
+static void dtt_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats);
+static int dtt_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf);
+static void dtt_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout);
+static long dtt_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream);
+static int dtt_send_page(struct drbd_transport *transport, enum drbd_stream, struct page *page,
+		int offset, size_t size, unsigned msg_flags);
+static int dtt_send_zc_bio(struct drbd_transport *, struct bio *bio);
+static bool dtt_stream_ok(struct drbd_transport *transport, enum drbd_stream stream);
+static bool dtt_hint(struct drbd_transport *transport, enum drbd_stream stream, enum drbd_tr_hints hint);
+static void dtt_debugfs_show(struct drbd_transport *transport, struct seq_file *m);
+static void dtt_update_congested(struct drbd_tcp_transport *tcp_transport);
+static int dtt_add_path(struct drbd_path *path);
+static bool dtt_may_remove_path(struct drbd_path *);
+static void dtt_remove_path(struct drbd_path *);
+static void dtt_control_timer_fn(struct timer_list *t);
+
+static struct drbd_transport_class tcp_transport_class = {
+	.name = "tcp",
+	.instance_size = sizeof(struct drbd_tcp_transport),
+	.path_instance_size = sizeof(struct dtt_path),
+	.listener_instance_size = sizeof(struct dtt_listener),
+	.ops = (struct drbd_transport_ops) {
+		.init = dtt_init,
+		.free = dtt_free,
+		.init_listener = dtt_init_listener,
+		.release_listener = dtt_destroy_listener,
+		.prepare_connect = dtt_prepare_connect,
+		.connect = dtt_connect,
+		.finish_connect = dtt_finish_connect,
+		.recv = dtt_recv,
+		.recv_pages = dtt_recv_pages,
+		.stats = dtt_stats,
+		.net_conf_change = dtt_net_conf_change,
+		.set_rcvtimeo = dtt_set_rcvtimeo,
+		.get_rcvtimeo = dtt_get_rcvtimeo,
+		.send_page = dtt_send_page,
+		.send_zc_bio = dtt_send_zc_bio,
+		.stream_ok = dtt_stream_ok,
+		.hint = dtt_hint,
+		.debugfs_show = dtt_debugfs_show,
+		.add_path = dtt_add_path,
+		.may_remove_path = dtt_may_remove_path,
+		.remove_path = dtt_remove_path,
+	},
+	.module = THIS_MODULE,
+	.list = LIST_HEAD_INIT(tcp_transport_class.list),
+};
+
+static int dtt_init(struct drbd_transport *transport)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	enum drbd_stream i;
+
+	spin_lock_init(&tcp_transport->control_recv_lock);
+	tcp_transport->transport.class = &tcp_transport_class;
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		void *buffer = (void *)__get_free_page(GFP_KERNEL);
+		if (!buffer)
+			goto fail;
+		tcp_transport->rbuf[i].base = buffer;
+		tcp_transport->rbuf[i].pos = buffer;
+	}
+	timer_setup(&tcp_transport->control_timer, dtt_control_timer_fn, 0);
+
+	return 0;
+fail:
+	free_page((unsigned long)tcp_transport->rbuf[0].base);
+	return -ENOMEM;
+}
+
+static void dtt_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	enum drbd_stream i;
+	struct drbd_path *drbd_path;
+	/* free the socket specific stuff,
+	 * mutexes are handled by caller */
+
+	clear_bit(DTT_DATA_READY_ARMED, &tcp_transport->flags);
+
+	if (tcp_transport->control_data_ready_work.func) {
+		cancel_work_sync(&tcp_transport->control_data_ready_work);
+		tcp_transport->control_data_ready_work.func = NULL;
+	}
+
+	if (tcp_transport->stream[CONTROL_STREAM] &&
+	    tcp_transport->original_control_sk_state_change) {
+		write_lock_bh(&tcp_transport->stream[CONTROL_STREAM]->sk->sk_callback_lock);
+		tcp_transport->stream[CONTROL_STREAM]->sk->sk_state_change =
+			tcp_transport->original_control_sk_state_change;
+		write_unlock_bh(&tcp_transport->stream[CONTROL_STREAM]->sk->sk_callback_lock);
+	}
+
+	synchronize_rcu();
+	for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+		dtt_socket_free(&tcp_transport->stream[i]);
+	}
+
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		bool was_established = test_and_clear_bit(TR_ESTABLISHED, &drbd_path->flags);
+
+		if (free_op == CLOSE_CONNECTION && was_established)
+			drbd_path_event(transport, drbd_path);
+	}
+
+	timer_delete_sync(&tcp_transport->control_timer);
+
+	if (free_op == DESTROY_TRANSPORT) {
+		for (i = DATA_STREAM; i <= CONTROL_STREAM; i++) {
+			free_page((unsigned long)tcp_transport->rbuf[i].base);
+			tcp_transport->rbuf[i].base = NULL;
+		}
+	}
+}
+
+static int _dtt_send(struct drbd_tcp_transport *tcp_transport, struct socket *socket,
+		      void *buf, size_t size, unsigned msg_flags)
+{
+	struct kvec iov;
+	struct msghdr msg;
+	int rv, sent = 0;
+
+	/* THINK  if (signal_pending) return ... ? */
+
+	iov.iov_base = buf;
+	iov.iov_len  = size;
+
+	msg.msg_name       = NULL;
+	msg.msg_namelen    = 0;
+	msg.msg_control    = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;
+
+	do {
+		rv = kernel_sendmsg(socket, &msg, &iov, 1, iov.iov_len);
+		if (rv == -EAGAIN) {
+			struct drbd_transport *transport = &tcp_transport->transport;
+			enum drbd_stream stream =
+				tcp_transport->stream[DATA_STREAM] == socket ?
+					DATA_STREAM : CONTROL_STREAM;
+
+			if (drbd_stream_send_timed_out(transport, stream))
+				break;
+			else
+				continue;
+		}
+		if (rv == -EINTR) {
+			flush_signals(current);
+			rv = 0;
+		}
+		if (rv < 0)
+			break;
+		sent += rv;
+		iov.iov_base += rv;
+		iov.iov_len  -= rv;
+	} while (sent < size);
+
+	if (rv <= 0)
+		return rv;
+
+	return sent;
+}
+
+static int dtt_recv_short(struct socket *socket, void *buf, size_t size, int flags)
+{
+	struct kvec iov = {
+		.iov_base = buf,
+		.iov_len = size,
+	};
+	union {
+		struct cmsghdr cmsg;
+		u8 buf[CMSG_SPACE(sizeof(u8))];
+	} u;
+	struct msghdr msg = {
+		.msg_control = &u,
+		.msg_controllen = sizeof(u),
+	};
+	int ret;
+
+	flags = flags ? flags : MSG_WAITALL | MSG_NOSIGNAL;
+
+	ret = kernel_recvmsg(socket, &msg, &iov, 1, size, flags);
+
+	if (msg.msg_controllen != sizeof(u)) {
+		u8 level, description;
+
+		switch (tls_get_record_type(socket->sk, &u.cmsg)) {
+		case 0:
+			fallthrough;
+		case TLS_RECORD_TYPE_DATA:
+			break;
+		case TLS_RECORD_TYPE_ALERT:
+			tls_alert_recv(socket->sk, &msg, &level, &description);
+			ret = (level == TLS_ALERT_LEVEL_FATAL) ? -EACCES : -EAGAIN;
+			break;
+		default:
+			/* discard this record type */
+			ret = -EAGAIN;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int dtt_recv(struct drbd_transport *transport, enum drbd_stream stream, void **buf, size_t size, int flags)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[stream];
+	void *buffer;
+	int rv;
+
+	if (!socket)
+		return -ENOTCONN;
+
+	if (flags & CALLER_BUFFER) {
+		buffer = *buf;
+		rv = dtt_recv_short(socket, buffer, size, flags & ~CALLER_BUFFER);
+	} else if (flags & GROW_BUFFER) {
+		TR_ASSERT(transport, *buf == tcp_transport->rbuf[stream].base);
+		buffer = tcp_transport->rbuf[stream].pos;
+		TR_ASSERT(transport, (buffer - *buf) + size <= PAGE_SIZE);
+
+		rv = dtt_recv_short(socket, buffer, size, flags & ~GROW_BUFFER);
+	} else {
+		buffer = tcp_transport->rbuf[stream].base;
+
+		rv = dtt_recv_short(socket, buffer, size, flags);
+		if (rv > 0)
+			*buf = buffer;
+	}
+
+	if (rv > 0)
+		tcp_transport->rbuf[stream].pos = buffer + rv;
+
+	return rv;
+}
+
+static int dtt_recv_pages(struct drbd_transport *transport, struct drbd_page_chain_head *chain, size_t size)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[DATA_STREAM];
+	struct page *page;
+	int err;
+
+	if (!socket)
+		return -ENOTCONN;
+
+	drbd_alloc_page_chain(transport, chain, DIV_ROUND_UP(size, PAGE_SIZE), GFP_TRY);
+	page = chain->head;
+	if (!page)
+		return -ENOMEM;
+
+	page_chain_for_each(page) {
+		size_t len = min_t(int, size, PAGE_SIZE);
+		void *data = kmap(page);
+		err = dtt_recv_short(socket, data, len, 0);
+		kunmap(page);
+		set_page_chain_offset(page, 0);
+		set_page_chain_size(page, len);
+		if (err < 0)
+			goto fail;
+		size -= err;
+	}
+	if (unlikely(size)) {
+		tr_warn(transport, "Not enough data received; missing %zu bytes\n", size);
+		err = -ENODATA;
+		goto fail;
+	}
+	return 0;
+fail:
+	drbd_free_page_chain(transport, chain);
+	return err;
+}
+
+static void dtt_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+
+	struct socket *socket = tcp_transport->stream[DATA_STREAM];
+
+	if (socket) {
+		struct sock *sk = socket->sk;
+		struct tcp_sock *tp = tcp_sk(sk);
+
+		stats->unread_received = tp->rcv_nxt - tp->copied_seq;
+		stats->unacked_send = tp->write_seq - tp->snd_una;
+		stats->send_buffer_size = sk->sk_sndbuf;
+		stats->send_buffer_used = sk->sk_wmem_queued;
+	}
+}
+
+static void dtt_setbufsize(struct socket *socket, unsigned int snd,
+			   unsigned int rcv)
+{
+	struct sock *sk = socket->sk;
+
+	/* open coded SO_SNDBUF, SO_RCVBUF */
+	if (snd) {
+		sk->sk_sndbuf = snd;
+		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
+		/* Wake up sending tasks if we upped the value. */
+		sk->sk_write_space(sk);
+	} else {
+		sk->sk_userlocks &= ~SOCK_SNDBUF_LOCK;
+	}
+
+	if (rcv) {
+		sk->sk_rcvbuf = rcv;
+		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
+	} else {
+		sk->sk_userlocks &= ~SOCK_RCVBUF_LOCK;
+	}
+}
+
+static bool dtt_path_cmp_addr(struct dtt_path *path)
+{
+	struct drbd_path *drbd_path = &path->path;
+	int addr_size;
+
+	addr_size = min(drbd_path->my_addr_len, drbd_path->peer_addr_len);
+	return memcmp(&drbd_path->my_addr, &drbd_path->peer_addr, addr_size) > 0;
+}
+
+static int dtt_try_connect(struct dtt_path *path, struct socket **ret_socket)
+{
+	struct drbd_transport *transport = path->path.transport;
+	const char *what;
+	struct socket *socket;
+	struct sockaddr_storage my_addr, peer_addr;
+	struct net_conf *nc;
+	int err;
+	int sndbuf_size, rcvbuf_size, connect_int;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EIO;
+	}
+	sndbuf_size = nc->sndbuf_size;
+	rcvbuf_size = nc->rcvbuf_size;
+	connect_int = nc->connect_int;
+	rcu_read_unlock();
+
+	my_addr = path->path.my_addr;
+	if (my_addr.ss_family == AF_INET6)
+		((struct sockaddr_in6 *)&my_addr)->sin6_port = 0;
+	else
+		((struct sockaddr_in *)&my_addr)->sin_port = 0; /* AF_INET & AF_SCI */
+
+	/* In some cases, the network stack can end up overwriting
+	   peer_addr.ss_family, so use a copy here. */
+	peer_addr = path->path.peer_addr;
+
+	what = "sock_create_kern";
+	err = sock_create_kern(path->path.net, my_addr.ss_family, SOCK_STREAM, IPPROTO_TCP, &socket);
+	if (err < 0) {
+		socket = NULL;
+		goto out;
+	}
+
+	socket->sk->sk_rcvtimeo =
+	socket->sk->sk_sndtimeo = connect_int * HZ;
+	dtt_setbufsize(socket, sndbuf_size, rcvbuf_size);
+
+	/* explicitly bind to the configured IP as source IP
+	*  for the outgoing connections.
+	*  This is needed for multihomed hosts and to be
+	*  able to use lo: interfaces for drbd.
+	* Make sure to use 0 as port number, so linux selects
+	*  a free one dynamically.
+	*/
+	what = "bind before connect";
+	err = socket->ops->bind(socket, (struct sockaddr_unsized *) &my_addr, path->path.my_addr_len);
+	if (err < 0)
+		goto out;
+
+	/* connect may fail, peer not yet available.
+	 * stay C_CONNECTING, don't go Disconnecting! */
+	what = "connect";
+	err = socket->ops->connect(socket, (struct sockaddr_unsized *) &peer_addr,
+				   path->path.peer_addr_len, 0);
+	if (err < 0) {
+		switch (err) {
+		case -ETIMEDOUT:
+		case -EINPROGRESS:
+		case -EINTR:
+		case -ERESTARTSYS:
+		case -ECONNREFUSED:
+		case -ECONNRESET:
+		case -ENETUNREACH:
+		case -EHOSTDOWN:
+		case -EHOSTUNREACH:
+			err = -EAGAIN;
+			break;
+		case -EINVAL:
+			err = -EADDRNOTAVAIL;
+			break;
+		}
+	}
+
+out:
+	if (err < 0) {
+		if (socket)
+			sock_release(socket);
+		if (err != -EAGAIN && err != -EADDRNOTAVAIL)
+			tr_err(transport, "%s failed, err = %d\n", what, err);
+	} else {
+		*ret_socket = socket;
+	}
+
+	return err;
+}
+
+typedef int (*tls_hello_func)(const struct tls_handshake_args *, gfp_t);
+
+struct tls_handshake_wait {
+	struct completion done;
+	int status;
+};
+
+static void tls_handshake_done(void *data, int status, key_serial_t peerid)
+{
+	struct tls_handshake_wait *wait = data;
+
+	// Normalize the error to be negative: while the error _should_ be negative
+	// it is not guaranteed: the netlink interface allows any u32 value, which is
+	// then negated and cast to int, so who knows what will be returned.
+	if (status > 0)
+		status = -status;
+
+	wait->status = status;
+	complete(&wait->done);
+}
+
+static int tls_init_hello(struct socket *sock, const char *peername,
+			  key_serial_t keyring, key_serial_t privkey,
+			  key_serial_t certificate, tls_hello_func hello,
+			  struct tls_handshake_wait *tls_wait)
+{
+	int err;
+	struct tls_handshake_args tls_args = {
+			.ta_sock = sock,
+			.ta_done = tls_handshake_done,
+			.ta_data = tls_wait,
+			.ta_peername = peername,
+			.ta_keyring = keyring,
+			.ta_my_privkey = privkey,
+			.ta_my_cert = certificate,
+	};
+
+	if (IS_ERR(sock_alloc_file(sock, O_NONBLOCK, NULL)))
+		return -EIO;
+
+	do {
+		err = hello(&tls_args, GFP_KERNEL);
+	} while (err == -EAGAIN);
+
+	return err;
+}
+
+static int tls_wait_hello(struct tls_handshake_wait *csocket_tls_wait,
+			  struct tls_handshake_wait *dsocket_tls_wait,
+			  unsigned long timeout)
+{
+	unsigned long remaining = wait_for_completion_timeout(
+		&csocket_tls_wait->done, timeout);
+	if (!remaining)
+		return -ETIMEDOUT;
+
+	if (!wait_for_completion_timeout(&dsocket_tls_wait->done, remaining))
+		return -ETIMEDOUT;
+
+	if (csocket_tls_wait->status)
+		return csocket_tls_wait->status;
+
+	return dsocket_tls_wait->status;
+}
+
+
+static int dtt_send_first_packet(struct drbd_tcp_transport *tcp_transport, struct socket *socket,
+				 enum drbd_packet cmd)
+{
+	struct p_header80 h;
+
+	h.magic = cpu_to_be32(DRBD_MAGIC);
+	h.command = cpu_to_be16(cmd);
+	h.length = 0;
+
+	return _dtt_send(tcp_transport, socket, &h, sizeof(h), 0);
+}
+
+/**
+ * dtt_socket_free() - Free the socket
+ * @socket:	pointer to the pointer to the socket.
+ */
+static void dtt_socket_free(struct socket **socket)
+{
+	if (!*socket)
+		return;
+
+	tls_handshake_cancel((*socket)->sk);
+	kernel_sock_shutdown(*socket, SHUT_RDWR);
+
+	if ((*socket)->file)
+		sockfd_put((*socket));
+	else
+		sock_release(*socket);
+
+	*socket = NULL;
+}
+
+/**
+ * dtt_socket_ok_or_free() - Free the socket if its connection is not okay
+ * @socket:	pointer to the pointer to the socket.
+ */
+static bool dtt_socket_ok_or_free(struct socket **socket)
+{
+	if (!*socket)
+		return false;
+
+	if ((*socket)->sk->sk_state == TCP_ESTABLISHED)
+		return true;
+
+	dtt_socket_free(socket);
+	return false;
+}
+
+static bool dtt_connection_established(struct drbd_transport *transport,
+				       struct socket **socket1,
+				       struct socket **socket2,
+				       struct dtt_path **first_path)
+{
+	struct net_conf *nc;
+	int timeout, good = 0;
+
+	if (!*socket1 || !*socket2)
+		return false;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	timeout = (nc->sock_check_timeo ?: nc->ping_timeo) * HZ / 10;
+	rcu_read_unlock();
+	schedule_timeout_interruptible(timeout);
+
+	good += dtt_socket_ok_or_free(socket1);
+	good += dtt_socket_ok_or_free(socket2);
+
+	if (good == 0) {
+		kref_put(&(*first_path)->path.kref, drbd_destroy_path);
+		*first_path = NULL;
+	}
+
+	return good == 2;
+}
+
+static struct dtt_path *dtt_wait_connect_cond(struct drbd_transport *transport)
+{
+	struct drbd_listener *listener;
+	struct drbd_path *drbd_path;
+	struct dtt_path *path = NULL;
+	bool rv = false;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drbd_path, &transport->paths, list) {
+		path = container_of(drbd_path, struct dtt_path, path);
+		listener = drbd_path->listener;
+
+		spin_lock_bh(&listener->waiters_lock);
+		rv = listener->pending_accepts > 0 || !list_empty(&path->sockets);
+		spin_unlock_bh(&listener->waiters_lock);
+
+		if (rv)
+			break;
+	}
+	if (rv)
+		kref_get(&path->path.kref);
+	rcu_read_unlock();
+
+	return rv ? path : NULL;
+}
+
+static void unregister_state_change(struct sock *sock, struct dtt_listener *listener)
+{
+	write_lock_bh(&sock->sk_callback_lock);
+	sock->sk_state_change = listener->original_sk_state_change;
+	sock->sk_user_data = NULL;
+	write_unlock_bh(&sock->sk_callback_lock);
+}
+
+static int dtt_wait_for_connect(struct drbd_transport *transport,
+				struct drbd_listener *drbd_listener, struct socket **socket,
+				struct dtt_path **ret_path)
+{
+	struct dtt_socket_container *socket_c;
+	struct sockaddr_storage peer_addr;
+	int connect_int, err = 0;
+	long timeo;
+	struct socket *s_estab = NULL;
+	struct net_conf *nc;
+	struct drbd_path *drbd_path2;
+	struct dtt_listener *listener = container_of(drbd_listener, struct dtt_listener, listener);
+	struct dtt_path *path = NULL;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EINVAL;
+	}
+	connect_int = nc->connect_int;
+	rcu_read_unlock();
+
+	timeo = connect_int * HZ;
+	timeo += get_random_u32_below(2) ? timeo / 7 : -timeo / 7; /* 28.5% random jitter */
+
+retry:
+	if (path)
+		kref_put(&path->path.kref, drbd_destroy_path);
+	timeo = wait_event_interruptible_timeout(listener->wait,
+			(path = dtt_wait_connect_cond(transport)),
+			timeo);
+	if (timeo <= 0)
+		return -EAGAIN;
+
+	spin_lock_bh(&listener->listener.waiters_lock);
+	socket_c = list_first_entry_or_null(&path->sockets, struct dtt_socket_container, list);
+	if (socket_c) {
+		s_estab = socket_c->socket;
+		list_del(&socket_c->list);
+		kfree(socket_c);
+	} else if (listener->listener.pending_accepts > 0) {
+		listener->listener.pending_accepts--;
+		spin_unlock_bh(&listener->listener.waiters_lock);
+
+		s_estab = NULL;
+		err = kernel_accept(listener->s_listen, &s_estab, O_NONBLOCK);
+		if (err < 0) {
+			kref_put(&path->path.kref, drbd_destroy_path);
+			return err;
+		}
+
+		/* The established socket inherits the sk_state_change callback
+		   from the listening socket. */
+		unregister_state_change(s_estab->sk, listener);
+
+		s_estab->ops->getname(s_estab, (struct sockaddr *)&peer_addr, 2);
+
+		spin_lock_bh(&listener->listener.waiters_lock);
+		drbd_path2 = drbd_find_path_by_addr(&listener->listener, &peer_addr);
+		if (!drbd_path2) {
+			struct sockaddr_in6 *from_sin6;
+			struct sockaddr_in *from_sin;
+
+			switch (peer_addr.ss_family) {
+			case AF_INET6:
+				from_sin6 = (struct sockaddr_in6 *)&peer_addr;
+				tr_notice(transport, "Closing unexpected connection from "
+				       "%pI6\n", &from_sin6->sin6_addr);
+				break;
+			default:
+				from_sin = (struct sockaddr_in *)&peer_addr;
+				tr_notice(transport, "Closing unexpected connection from "
+					 "%pI4\n", &from_sin->sin_addr);
+				break;
+			}
+
+			goto retry_locked;
+		}
+		if (drbd_path2 != &path->path) {
+			struct dtt_path *path2 =
+				container_of(drbd_path2, struct dtt_path, path);
+
+			socket_c = kmalloc_obj(*socket_c, GFP_ATOMIC);
+			if (!socket_c) {
+				tr_info(transport, /* path2->transport, */
+					"No mem, dropped an incoming connection\n");
+				goto retry_locked;
+			}
+
+			socket_c->socket = s_estab;
+			s_estab = NULL;
+			list_add_tail(&socket_c->list, &path2->sockets);
+			wake_up(&listener->wait);
+			goto retry_locked;
+		}
+		if (s_estab->sk->sk_state != TCP_ESTABLISHED)
+			goto retry_locked;
+	}
+	spin_unlock_bh(&listener->listener.waiters_lock);
+	*socket = s_estab;
+	if (*ret_path)
+		kref_put(&(*ret_path)->path.kref, drbd_destroy_path);
+	*ret_path = path;
+	return 0;
+
+retry_locked:
+	spin_unlock_bh(&listener->listener.waiters_lock);
+	dtt_socket_free(&s_estab);
+	goto retry;
+}
+
+static int dtt_receive_first_packet(struct drbd_tcp_transport *tcp_transport, struct socket *socket)
+{
+	struct drbd_transport *transport = &tcp_transport->transport;
+	struct p_header80 *h = tcp_transport->rbuf[DATA_STREAM].base;
+	const unsigned int header_size = sizeof(*h);
+	struct net_conf *nc;
+	int err;
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EIO;
+	}
+	socket->sk->sk_rcvtimeo = nc->ping_timeo * 4 * HZ / 10;
+	rcu_read_unlock();
+
+	err = dtt_recv_short(socket, h, header_size, 0);
+	if (err != header_size) {
+		if (err >= 0)
+			err = -EIO;
+		return err;
+	}
+	if (h->magic != cpu_to_be32(DRBD_MAGIC)) {
+		tr_err(transport, "Wrong magic value 0x%08x in receive_first_packet\n",
+			 be32_to_cpu(h->magic));
+		return -EINVAL;
+	}
+	return be16_to_cpu(h->command);
+}
+
+
+static int dtt_control_tcp_input(read_descriptor_t *rd_desc, struct sk_buff *skb,
+				 unsigned int offset, size_t len)
+{
+	struct drbd_transport *transport = rd_desc->arg.data;
+	unsigned int avail, consumed = 0;
+	struct skb_seq_state seq;
+
+	skb_prepare_seq_read(skb, offset, offset + len, &seq);
+	do {
+		struct drbd_const_buffer buffer;
+
+		/*
+		 * skb_seq_read() returns the length of the block assigned to buffer. This might
+		 * be more than is actually ready, so we ensure we only mark as available what
+		 * is ready.
+		 */
+		avail = skb_seq_read(consumed, &buffer.buffer, &seq);
+		if (!avail)
+			break;
+		buffer.avail = min_t(unsigned int, avail, len - consumed);
+		consumed += buffer.avail;
+		drbd_control_data_ready(transport, &buffer);
+	} while (consumed < len);
+	skb_abort_seq_read(&seq);
+
+	return consumed;
+}
+
+static void dtt_control_data_ready_work(struct work_struct *item)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(item, struct drbd_tcp_transport, control_data_ready_work);
+	struct socket *csocket = tcp_transport->stream[CONTROL_STREAM];
+	struct drbd_const_buffer drbd_buffer;
+	int n;
+
+	while (true) {
+		n = dtt_recv_short(csocket, tcp_transport->rbuf[CONTROL_STREAM].base, PAGE_SIZE,
+				   MSG_DONTWAIT | MSG_NOSIGNAL);
+		if (n <= 0)
+			break;
+
+		drbd_buffer.buffer = tcp_transport->rbuf[CONTROL_STREAM].base;
+		drbd_buffer.avail = n;
+		drbd_control_data_ready(&tcp_transport->transport, &drbd_buffer);
+	}
+}
+
+static void dtt_control_data_ready(struct sock *sock)
+{
+	struct drbd_transport *transport = sock->sk_user_data;
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+
+	read_descriptor_t rd_desc = {
+		.count = 1,
+		.arg = { .data = transport },
+	};
+
+	if (!test_bit(DTT_DATA_READY_ARMED, &tcp_transport->flags)
+	    && tcp_transport->original_control_sk_data_ready)
+		return tcp_transport->original_control_sk_data_ready(sock);
+
+	/* We have two different paths depending on if TLS is enabled or not.
+	 * If TLS is enabled, we can't use read_sock, firstly because it's not implemented for the
+	 * TLS protocol on most kernels, secondly the implementation that does exist is not safe
+	 * to call from SOFTIRQ context. Instead, we schedule a work and increment the counter of
+	 * "pending" ready events.
+	 *
+	 * In normal TCP mode, we can simply use tcp_read_sock, as that is safe to call from SOFTIRQ
+	 * contexts.
+	 */
+	mod_timer(&tcp_transport->control_timer, jiffies + sock->sk_rcvtimeo);
+	if (tcp_transport->control_data_ready_work.func) {
+		queue_work(dtt_csocket_recv, &tcp_transport->control_data_ready_work);
+	} else {
+		spin_lock_bh(&tcp_transport->control_recv_lock);
+		tcp_read_sock(sock, &rd_desc, dtt_control_tcp_input);
+		spin_unlock_bh(&tcp_transport->control_recv_lock);
+	}
+}
+
+static void dtt_control_state_change(struct sock *sock)
+{
+	struct drbd_transport *transport = sock->sk_user_data;
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+
+	switch (sock->sk_state) {
+	case TCP_FIN_WAIT1:
+	case TCP_CLOSE_WAIT:
+	case TCP_CLOSE:
+	case TCP_LAST_ACK:
+	case TCP_CLOSING:
+		drbd_control_event(transport, CLOSED_BY_PEER);
+		break;
+	default:
+		tr_warn(transport, "unhandled state %d\n", sock->sk_state);
+	}
+
+	tcp_transport->original_control_sk_state_change(sock);
+}
+
+static void dtt_incoming_connection(struct sock *sock)
+{
+	struct dtt_listener *listener = sock->sk_user_data;
+	void (*state_change)(struct sock *sock);
+
+	state_change = listener->original_sk_state_change;
+	state_change(sock);
+
+	spin_lock(&listener->listener.waiters_lock);
+	listener->listener.pending_accepts++;
+	spin_unlock(&listener->listener.waiters_lock);
+	wake_up(&listener->wait);
+}
+
+static void dtt_control_timer_fn(struct timer_list *t)
+{
+	struct drbd_tcp_transport *tcp_transport = timer_container_of(tcp_transport, t,
+			control_timer);
+	struct drbd_transport *transport = &tcp_transport->transport;
+
+	drbd_control_event(transport, TIMEOUT);
+}
+
+static void dtt_destroy_listener(struct drbd_listener *generic_listener)
+{
+	struct dtt_listener *listener =
+		container_of(generic_listener, struct dtt_listener, listener);
+
+	if (!listener->s_listen)
+		return;
+	unregister_state_change(listener->s_listen->sk, listener);
+	sock_release(listener->s_listen);
+}
+
+static int dtt_init_listener(struct drbd_transport *transport,
+			     const struct sockaddr *addr,
+			     struct net *net,
+			     struct drbd_listener *drbd_listener)
+{
+	int err, sndbuf_size, rcvbuf_size, addr_len;
+	struct sockaddr_storage my_addr;
+	struct dtt_listener *listener = container_of(drbd_listener, struct dtt_listener, listener);
+	struct socket *s_listen;
+	struct net_conf *nc;
+	const char *what = "";
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	if (!nc) {
+		rcu_read_unlock();
+		return -EINVAL;
+	}
+	sndbuf_size = nc->sndbuf_size;
+	rcvbuf_size = nc->rcvbuf_size;
+	rcu_read_unlock();
+
+	my_addr = *(struct sockaddr_storage *)addr;
+
+	err = sock_create_kern(net, my_addr.ss_family, SOCK_STREAM, IPPROTO_TCP, &s_listen);
+	if (err < 0) {
+		s_listen = NULL;
+		what = "sock_create_kern";
+		goto out;
+	}
+
+	s_listen->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
+	dtt_setbufsize(s_listen, sndbuf_size, rcvbuf_size);
+
+	addr_len = addr->sa_family == AF_INET6 ? sizeof(struct sockaddr_in6)
+		: sizeof(struct sockaddr_in);
+
+	err = s_listen->ops->bind(s_listen, (struct sockaddr_unsized *)&my_addr, addr_len);
+	if (err < 0) {
+		what = "bind before listen";
+		goto out;
+	}
+
+	listener->s_listen = s_listen;
+	write_lock_bh(&s_listen->sk->sk_callback_lock);
+	listener->original_sk_state_change = s_listen->sk->sk_state_change;
+	s_listen->sk->sk_state_change = dtt_incoming_connection;
+	s_listen->sk->sk_user_data = listener;
+	write_unlock_bh(&s_listen->sk->sk_callback_lock);
+
+	err = s_listen->ops->listen(s_listen, DRBD_PEERS_MAX * 2);
+	if (err < 0) {
+		what = "listen";
+		goto out;
+	}
+
+	listener->listener.listen_addr = my_addr;
+	init_waitqueue_head(&listener->wait);
+
+	return 0;
+out:
+	if (s_listen)
+		sock_release(s_listen);
+
+	if (err < 0 &&
+	    err != -EAGAIN && err != -EINTR && err != -ERESTARTSYS && err != -EADDRINUSE &&
+	    err != -EADDRNOTAVAIL)
+		tr_err(transport, "%s failed, err = %d\n", what, err);
+
+	return err;
+}
+
+static void dtt_cleanup_accepted_sockets(struct dtt_path *path)
+{
+	while (!list_empty(&path->sockets)) {
+		struct dtt_socket_container *socket_c =
+			list_first_entry(&path->sockets, struct dtt_socket_container, list);
+
+		list_del(&socket_c->list);
+		dtt_socket_free(&socket_c->socket);
+		kfree(socket_c);
+	}
+}
+
+static void dtt_finish_connect(struct drbd_transport *transport)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct dtt_path *path;
+
+	clear_bit(DTT_CONNECTING, &tcp_transport->flags);
+
+	list_for_each_entry(path, &transport->paths, path.list) {
+		drbd_put_listener(&path->path);
+		dtt_cleanup_accepted_sockets(path);
+	}
+}
+
+static struct dtt_path *dtt_next_path(struct dtt_path *path, struct drbd_transport *transport)
+{
+	struct drbd_path *drbd_path;
+
+	drbd_path = __drbd_next_path_ref(path ? &path->path : NULL, transport);
+
+	/* Loop when we reach the end. */
+	if (!drbd_path)
+		drbd_path = __drbd_next_path_ref(NULL, transport);
+
+	return drbd_path ? container_of(drbd_path, struct dtt_path, path) : NULL;
+}
+
+static int dtt_prepare_connect(struct drbd_transport *transport)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct dtt_path *path;
+	struct drbd_path *drbd_path;
+
+	list_for_each_entry(path, &transport->paths, path.list)
+		dtt_cleanup_accepted_sockets(path);
+
+	set_bit(DTT_CONNECTING, &tcp_transport->flags);
+
+	list_for_each_entry(drbd_path, &transport->paths, list) {
+		if (!drbd_path->listener) {
+			int err = drbd_get_listener(drbd_path);
+
+			if (err)
+				return err;
+		}
+	}
+
+	return 0;
+}
+
+static int dtt_connect(struct drbd_transport *transport)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct dtt_path *connect_to_path, *first_path = NULL;
+	struct socket *dsocket, *csocket;
+	struct net_conf *nc;
+	bool tls, dsocket_is_server = false, csocket_is_server = false;
+	char peername[64];
+	key_serial_t tls_keyring, tls_privkey, tls_certificate;
+	int timeout, err;
+	bool ok;
+	struct tls_handshake_wait csocket_tls_wait = { .status = 0 };
+	struct tls_handshake_wait dsocket_tls_wait = { .status = 0 };
+
+	dsocket = NULL;
+	csocket = NULL;
+
+	connect_to_path = dtt_next_path(NULL, transport);
+	if (!connect_to_path) {
+		err = -EDESTADDRREQ;
+		goto out;
+	}
+
+	do {
+		struct socket *s = NULL;
+
+		err = dtt_try_connect(connect_to_path, &s);
+		if (err < 0 && err != -EAGAIN)
+			goto out_release_sockets;
+
+		if (s) {
+			bool use_for_data;
+
+			if (first_path) {
+				if (first_path != connect_to_path) {
+					tr_info(transport, "initial paths crossed A - fail over\n");
+					dtt_socket_free(&dsocket);
+					dtt_socket_free(&csocket);
+				}
+
+				kref_put(&first_path->path.kref, drbd_destroy_path);
+				first_path = NULL;
+			}
+
+			kref_get(&connect_to_path->path.kref);
+			first_path = connect_to_path;
+
+			if (!dsocket && !csocket) {
+				use_for_data = dtt_path_cmp_addr(first_path);
+			} else if (!dsocket) {
+				use_for_data = true;
+			} else {
+				if (csocket) {
+					tr_err(transport, "Logic error in conn_connect()\n");
+					goto out_eagain;
+				}
+				use_for_data = false;
+			}
+
+			if (!use_for_data)
+				clear_bit(RESOLVE_CONFLICTS, &transport->flags);
+
+			err = dtt_send_first_packet(tcp_transport,
+						    s,
+						    use_for_data ? P_INITIAL_DATA : P_INITIAL_META);
+
+			if (err < 0) {
+				tr_warn(transport, "Error sending initial packet: %d\n", err);
+				dtt_socket_free(&s);
+			} else if (use_for_data) {
+				dsocket = s;
+				dsocket_is_server = false;
+			} else {
+				csocket = s;
+				csocket_is_server = false;
+			}
+		} else if (!first_path) {
+			connect_to_path = dtt_next_path(connect_to_path, transport);
+
+			/*
+			 * The final path should not be removed while
+			 * connecting, but handle the case for robustness.
+			 */
+			err = -EDESTADDRREQ;
+			if (!connect_to_path)
+				goto out_release_sockets;
+		}
+
+		if (dtt_connection_established(transport, &dsocket, &csocket, &first_path))
+			break;
+
+retry:
+		s = NULL;
+		err = dtt_wait_for_connect(transport, connect_to_path->path.listener, &s, &connect_to_path);
+		if (err < 0 && err != -EAGAIN)
+			goto out_release_sockets;
+
+		if (s) {
+			int fp = dtt_receive_first_packet(tcp_transport, s);
+
+			if (first_path) {
+				if (first_path != connect_to_path) {
+					tr_info(transport, "initial paths crossed P - fail over\n");
+					dtt_socket_free(&dsocket);
+					dtt_socket_free(&csocket);
+				}
+
+				kref_put(&first_path->path.kref, drbd_destroy_path);
+				first_path = NULL;
+			}
+
+			kref_get(&connect_to_path->path.kref);
+			first_path = connect_to_path;
+
+			dtt_socket_ok_or_free(&dsocket);
+			dtt_socket_ok_or_free(&csocket);
+			switch (fp) {
+			case P_INITIAL_DATA:
+				if (dsocket) {
+					tr_warn(transport, "initial packet S crossed\n");
+					dtt_socket_free(&dsocket);
+					dsocket = s;
+					dsocket_is_server = true;
+					goto randomize;
+				}
+				dsocket = s;
+				dsocket_is_server = true;
+				break;
+			case P_INITIAL_META:
+				set_bit(RESOLVE_CONFLICTS, &transport->flags);
+				if (csocket) {
+					tr_warn(transport, "initial packet M crossed\n");
+					dtt_socket_free(&csocket);
+					csocket = s;
+					csocket_is_server = true;
+					goto randomize;
+				}
+				csocket = s;
+				csocket_is_server = true;
+				break;
+			default:
+				tr_warn(transport, "Error receiving initial packet: %d\n", fp);
+				dtt_socket_free(&s);
+randomize:
+				if (get_random_u32_below(2))
+					goto retry;
+			}
+		}
+
+		if (drbd_should_abort_listening(transport))
+			goto out_eagain;
+
+		ok = dtt_connection_established(transport, &dsocket, &csocket, &first_path);
+	} while (!ok);
+
+	rcu_read_lock();
+	nc = rcu_dereference(transport->net_conf);
+	timeout = nc->timeout * HZ / 10;
+	tls = nc->tls;
+	memcpy(peername, nc->name, 64);
+	tls_keyring = nc->tls_keyring;
+	tls_privkey = nc->tls_privkey;
+	tls_certificate = nc->tls_certificate;
+	rcu_read_unlock();
+
+	write_lock_bh(&csocket->sk->sk_callback_lock);
+	clear_bit(DTT_DATA_READY_ARMED, &tcp_transport->flags);
+	tcp_transport->original_control_sk_data_ready = csocket->sk->sk_data_ready;
+	csocket->sk->sk_user_data = transport;
+	csocket->sk->sk_data_ready = dtt_control_data_ready;
+	write_unlock_bh(&csocket->sk->sk_callback_lock);
+
+	if (tls) {
+		csocket_tls_wait.done = COMPLETION_INITIALIZER_ONSTACK(csocket_tls_wait.done);
+		dsocket_tls_wait.done = COMPLETION_INITIALIZER_ONSTACK(dsocket_tls_wait.done);
+
+		err = tls_init_hello(
+			csocket, peername, tls_keyring, tls_privkey, tls_certificate,
+			csocket_is_server ? tls_server_hello_x509 : tls_client_hello_x509,
+			&csocket_tls_wait);
+		if (err < 0) {
+			tr_warn(transport, "Error from control socket tls handshake: %d\n", err);
+			goto out_release_sockets;
+		}
+
+		err = tls_init_hello(
+			dsocket, peername, tls_keyring, tls_privkey, tls_certificate,
+			dsocket_is_server ? tls_server_hello_x509 : tls_client_hello_x509,
+			&dsocket_tls_wait);
+		if (err < 0) {
+			tr_warn(transport, "Error from data socket tls handshake: %d\n", err);
+			goto out_release_sockets;
+		}
+
+		err = tls_wait_hello(&csocket_tls_wait, &dsocket_tls_wait, timeout);
+		if (err < 0) {
+			tr_warn(transport, "Error from tls handshake: %d\n", err);
+			goto out_release_sockets;
+		}
+
+		INIT_WORK(&tcp_transport->control_data_ready_work, dtt_control_data_ready_work);
+	}
+
+	TR_ASSERT(transport, first_path == connect_to_path);
+	set_bit(TR_ESTABLISHED, &connect_to_path->path.flags);
+	drbd_path_event(transport, &connect_to_path->path);
+
+	dsocket->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
+	csocket->sk->sk_reuse = SK_CAN_REUSE; /* SO_REUSEADDR */
+
+	/* We are a block device, we are in the write-out path,
+	 * we may need memory to facilitate memory reclaim
+	 */
+	dsocket->sk->sk_allocation = GFP_ATOMIC;
+	csocket->sk->sk_allocation = GFP_ATOMIC;
+
+	dsocket->sk->sk_use_task_frag = false;
+	csocket->sk->sk_use_task_frag = false;
+
+	sk_set_memalloc(dsocket->sk);
+	sk_set_memalloc(csocket->sk);
+
+	dsocket->sk->sk_priority = TC_PRIO_INTERACTIVE_BULK;
+	csocket->sk->sk_priority = TC_PRIO_INTERACTIVE;
+
+	/* NOT YET ...
+	 * sock.socket->sk->sk_sndtimeo = transport->net_conf->timeout*HZ/10;
+	 * sock.socket->sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
+	 * first set it to the P_CONNECTION_FEATURES timeout,
+	 * which we set to 4x the configured ping_timeout. */
+
+	/* we don't want delays.
+	 * we use tcp_sock_set_cork where appropriate, though */
+	tcp_sock_set_nodelay(dsocket->sk);
+	tcp_sock_set_nodelay(csocket->sk);
+
+	tcp_transport->stream[DATA_STREAM] = dsocket;
+	tcp_transport->stream[CONTROL_STREAM] = csocket;
+
+	dsocket->sk->sk_sndtimeo = timeout;
+	csocket->sk->sk_sndtimeo = timeout;
+
+	sock_set_keepalive(dsocket->sk);
+
+	if (drbd_keepidle)
+		tcp_sock_set_keepidle(dsocket->sk, drbd_keepidle);
+	if (drbd_keepcnt)
+		tcp_sock_set_keepcnt(dsocket->sk, drbd_keepcnt);
+	if (drbd_keepintvl)
+		tcp_sock_set_keepintvl(dsocket->sk, drbd_keepintvl);
+
+	write_lock_bh(&csocket->sk->sk_callback_lock);
+	tcp_transport->original_control_sk_state_change = csocket->sk->sk_state_change;
+	csocket->sk->sk_state_change = dtt_control_state_change;
+	set_bit(DTT_DATA_READY_ARMED, &tcp_transport->flags);
+	write_unlock_bh(&csocket->sk->sk_callback_lock);
+
+	err = 0;
+	goto out;
+
+out_eagain:
+	err = -EAGAIN;
+
+out_release_sockets:
+	dtt_socket_free(&dsocket);
+	dtt_socket_free(&csocket);
+
+out:
+	if (first_path)
+		kref_put(&first_path->path.kref, drbd_destroy_path);
+	if (connect_to_path)
+		kref_put(&connect_to_path->path.kref, drbd_destroy_path);
+
+	return err;
+}
+
+static int dtt_net_conf_change(struct drbd_transport *transport, struct net_conf *new_net_conf)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct net_conf *old_net_conf;
+	struct socket *data_socket = tcp_transport->stream[DATA_STREAM];
+	struct socket *control_socket = tcp_transport->stream[CONTROL_STREAM];
+
+	rcu_read_lock();
+	old_net_conf = rcu_dereference(transport->net_conf);
+	rcu_read_unlock();
+
+	if (old_net_conf && old_net_conf->tls != new_net_conf->tls &&
+	    (data_socket || control_socket)) {
+		tr_warn(transport, "cannot switch tls (%s -> %s) while connected\n",
+			old_net_conf->tls ? "yes" : "no", new_net_conf->tls ? "yes" : "no");
+		return -EINVAL;
+	}
+
+	if (data_socket) {
+		dtt_setbufsize(data_socket, new_net_conf->sndbuf_size, new_net_conf->rcvbuf_size);
+	}
+
+	if (control_socket) {
+		dtt_setbufsize(control_socket, new_net_conf->sndbuf_size, new_net_conf->rcvbuf_size);
+	}
+
+	return 0;
+}
+
+static void dtt_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[stream];
+
+	if (!socket)
+		return;
+
+	socket->sk->sk_rcvtimeo = timeout;
+
+	if (stream == CONTROL_STREAM)
+		mod_timer(&tcp_transport->control_timer, jiffies + timeout);
+
+}
+
+static long dtt_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[stream];
+
+	if (!socket)
+		return -ENOTCONN;
+
+	return socket->sk->sk_rcvtimeo;
+}
+
+static bool dtt_stream_ok(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[stream];
+
+	return socket && socket->sk;
+}
+
+static void dtt_update_congested(struct drbd_tcp_transport *tcp_transport)
+{
+	struct socket *socket = tcp_transport->stream[DATA_STREAM];
+	struct sock *sock;
+
+	if (!socket)
+		return;
+
+	sock = socket->sk;
+	if (sock->sk_wmem_queued > sock->sk_sndbuf * 4 / 5)
+		set_bit(NET_CONGESTED, &tcp_transport->transport.flags);
+}
+
+static int dtt_send_page(struct drbd_transport *transport, enum drbd_stream stream,
+			 struct page *page, int offset, size_t size, unsigned msg_flags)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct socket *socket = tcp_transport->stream[stream];
+	struct msghdr msg = { .msg_flags = msg_flags | MSG_NOSIGNAL | MSG_SPLICE_PAGES };
+	struct bio_vec bvec;
+	int len = size;
+	int err = -EIO;
+
+	if (!socket)
+		return -ENOTCONN;
+
+	dtt_update_congested(tcp_transport);
+	do {
+		int sent;
+
+		bvec_set_page(&bvec, page, len, offset);
+		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
+
+		sent = sock_sendmsg(socket, &msg);
+		if (sent <= 0) {
+			if (sent == -EAGAIN) {
+				if (drbd_stream_send_timed_out(transport, stream))
+					break;
+				continue;
+			}
+			tr_warn(transport, "%s: size=%d len=%d sent=%d\n",
+			     __func__, (int)size, len, sent);
+			if (sent < 0)
+				err = sent;
+			break;
+		}
+		len    -= sent;
+		offset += sent;
+		/* NOTE: it may take up to twice the socket timeout to have it
+		 * return -EAGAIN, the first timeout will likely happen with a
+		 * partial send, masking the timeout.  Maybe we want to export
+		 * drbd_stream_should_continue_after_partial_send(transport, stream)
+		 * and add that to the while() condition below.
+		 */
+	} while (len > 0 /* THINK && peer_device->repl_state[NOW] >= L_ESTABLISHED */);
+	clear_bit(NET_CONGESTED, &tcp_transport->transport.flags);
+
+	if (len == 0)
+		err = 0;
+
+	return err;
+}
+
+static int dtt_send_zc_bio(struct drbd_transport *transport, struct bio *bio)
+{
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+
+	bio_for_each_segment(bvec, bio, iter) {
+		int err;
+
+		err = dtt_send_page(transport, DATA_STREAM, bvec.bv_page,
+				      bvec.bv_offset, bvec.bv_len,
+				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+static bool dtt_hint(struct drbd_transport *transport, enum drbd_stream stream,
+		enum drbd_tr_hints hint)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	bool rv = true;
+	struct socket *socket = tcp_transport->stream[stream];
+
+	if (!socket)
+		return false;
+
+	switch (hint) {
+	case CORK:
+		tcp_sock_set_cork(socket->sk, true);
+		break;
+	case UNCORK:
+		tcp_sock_set_cork(socket->sk, false);
+		break;
+	case NODELAY:
+		tcp_sock_set_nodelay(socket->sk);
+		break;
+	case NOSPACE:
+		if (socket->sk->sk_socket)
+			set_bit(SOCK_NOSPACE, &socket->sk->sk_socket->flags);
+		break;
+	case QUICKACK:
+		tcp_sock_set_quickack(socket->sk, 2);
+		break;
+	default: /* not implemented, but should not trigger error handling */
+		return true;
+	}
+
+	return rv;
+}
+
+static void dtt_debugfs_show_stream(struct seq_file *m, struct socket *socket)
+{
+	struct sock *sk = socket->sk;
+	struct tcp_sock *tp = tcp_sk(sk);
+
+	seq_printf(m, "unread receive buffer: %u Byte\n",
+		   tp->rcv_nxt - tp->copied_seq);
+	seq_printf(m, "unacked send buffer: %u Byte\n",
+		   tp->write_seq - tp->snd_una);
+	seq_printf(m, "send buffer size: %u Byte\n", sk->sk_sndbuf);
+	seq_printf(m, "send buffer used: %u Byte\n", sk->sk_wmem_queued);
+}
+
+static void dtt_debugfs_show(struct drbd_transport *transport, struct seq_file *m)
+{
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	enum drbd_stream i;
+
+	/* BUMP me if you change the file format/content/presentation */
+	seq_printf(m, "v: %u\n\n", 0);
+
+	for (i = DATA_STREAM; i <= CONTROL_STREAM ; i++) {
+		struct socket *socket = tcp_transport->stream[i];
+
+		if (socket) {
+			seq_printf(m, "%s stream\n", i == DATA_STREAM ? "data" : "control");
+			dtt_debugfs_show_stream(m, socket);
+		}
+	}
+
+}
+
+static int dtt_add_path(struct drbd_path *drbd_path)
+{
+	struct drbd_transport *transport = drbd_path->transport;
+	struct drbd_tcp_transport *tcp_transport =
+		container_of(transport, struct drbd_tcp_transport, transport);
+	struct dtt_path *path = container_of(drbd_path, struct dtt_path, path);
+
+	clear_bit(TR_ESTABLISHED, &drbd_path->flags);
+	INIT_LIST_HEAD(&path->sockets);
+
+	if (!test_bit(DTT_CONNECTING, &tcp_transport->flags))
+		return 0;
+
+	return drbd_get_listener(drbd_path);
+}
+
+static bool dtt_may_remove_path(struct drbd_path *drbd_path)
+{
+	return !test_bit(TR_ESTABLISHED, &drbd_path->flags);
+}
+
+static void dtt_remove_path(struct drbd_path *drbd_path)
+{
+	drbd_put_listener(drbd_path);
+}
+
+static int __init dtt_initialize(void)
+{
+	dtt_csocket_recv = alloc_workqueue("dtt_csocket_recv",
+					   WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
+	if (!dtt_csocket_recv)
+		return -ENOMEM;
+	return drbd_register_transport_class(&tcp_transport_class,
+					     DRBD_TRANSPORT_API_VERSION,
+					     sizeof(struct drbd_transport));
+}
+
+static void __exit dtt_cleanup(void)
+{
+	destroy_workqueue(dtt_csocket_recv);
+	drbd_unregister_transport_class(&tcp_transport_class);
+}
+
+module_init(dtt_initialize)
+module_exit(dtt_cleanup)
diff --git a/drivers/block/drbd/drbd_transport_template.c b/drivers/block/drbd/drbd_transport_template.c
new file mode 100644
index 0000000000000..7a07dff0b5e8d
--- /dev/null
+++ b/drivers/block/drbd/drbd_transport_template.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/module.h>
+#include "drbd_transport.h"
+#include "drbd_int.h"
+
+
+MODULE_AUTHOR("xxx");
+MODULE_DESCRIPTION("xxx transport layer for DRBD");
+MODULE_LICENSE("GPL");
+
+
+struct drbd_xxx_transport {
+	struct drbd_transport transport;
+	/* xxx */
+};
+
+struct xxx_listener {
+	struct drbd_listener listener;
+	/* xxx */
+};
+
+struct xxx_waiter {
+	struct drbd_waiter waiter;
+	/* xxx */
+};
+
+static struct drbd_transport *xxx_create(struct drbd_connection *connection);
+static void xxx_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op);
+static int xxx_connect(struct drbd_transport *transport);
+static int xxx_recv(struct drbd_transport *transport, enum drbd_stream stream, void *buf, size_t size, int flags);
+static void xxx_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats);
+static void xxx_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout);
+static long xxx_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream);
+static int xxx_send_page(struct drbd_transport *transport, enum drbd_stream stream, struct page *page,
+		    int offset, size_t size, unsigned msg_flags);
+static bool xxx_stream_ok(struct drbd_transport *transport, enum drbd_stream stream);
+static bool xxx_hint(struct drbd_transport *transport, enum drbd_stream stream, enum drbd_tr_hints hint);
+
+
+static struct drbd_transport_class xxx_transport_class = {
+	.name = "xxx",
+	.create = xxx_create,
+	.list = LIST_HEAD_INIT(xxx_transport_class.list),
+};
+
+static struct drbd_transport_ops xxx_ops = {
+	.free = xxx_free,
+	.connect = xxx_connect,
+	.recv = xxx_recv,
+	.stats = xxx_stats,
+	.set_rcvtimeo = xxx_set_rcvtimeo,
+	.get_rcvtimeo = xxx_get_rcvtimeo,
+	.send_page = xxx_send_page,
+	.stream_ok = xxx_stream_ok,
+	.hint = xxx_hint,
+};
+
+
+static struct drbd_transport *xxx_create(struct drbd_connection *connection)
+{
+	struct drbd_xxx_transport *xxx_transport;
+
+	if (!try_module_get(THIS_MODULE))
+		return NULL;
+
+	xxx_transport = kzalloc_obj(struct drbd_xxx_transport);
+	if (!xxx_transport) {
+		module_put(THIS_MODULE);
+		return NULL;
+	}
+
+	xxx_transport->transport.ops = &xxx_ops;
+	xxx_transport->transport.connection = connection;
+
+	return &xxx_transport->transport;
+}
+
+static void xxx_free(struct drbd_transport *transport, enum drbd_tr_free_op free_op)
+{
+	struct drbd_xxx_transport *xxx_transport =
+		container_of(transport, struct drbd_xxx_transport, transport);
+
+	/* disconnect here */
+
+	if (free_op == DESTROY_TRANSPORT) {
+		kfree(xxx_transport);
+		module_put(THIS_MODULE);
+	}
+}
+
+static int xxx_send(struct drbd_transport *transport, enum drbd_stream stream, void *buf, size_t size, unsigned msg_flags)
+{
+	struct drbd_xxx_transport *xxx_transport =
+		container_of(transport, struct drbd_xxx_transport, transport);
+
+	return 0;
+}
+
+static int xxx_recv(struct drbd_transport *transport, enum drbd_stream stream, void *buf, size_t size, int flags)
+{
+	struct drbd_xxx_transport *xxx_transport =
+		container_of(transport, struct drbd_xxx_transport, transport);
+
+	return 0;
+}
+
+static void xxx_stats(struct drbd_transport *transport, struct drbd_transport_stats *stats)
+{
+}
+
+static int xxx_connect(struct drbd_transport *transport)
+{
+	struct drbd_xxx_transport *xxx_transport =
+		container_of(transport, struct drbd_xxx_transport, transport);
+
+	return true;
+}
+
+static void xxx_set_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream, long timeout)
+{
+}
+
+static long xxx_get_rcvtimeo(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	return 0;
+}
+
+static bool xxx_stream_ok(struct drbd_transport *transport, enum drbd_stream stream)
+{
+	return true;
+}
+
+static int xxx_send_page(struct drbd_transport *transport, enum drbd_stream stream, struct page *page,
+		    int offset, size_t size, unsigned msg_flags)
+{
+	return 0;
+}
+
+static bool xxx_hint(struct drbd_transport *transport, enum drbd_stream stream,
+		enum drbd_tr_hints hint)
+{
+	switch (hint) {
+	default: /* not implemented, but should not trigger error handling */
+		return true;
+	}
+	return true;
+}
+
+static int __init xxx_init(void)
+{
+	return drbd_register_transport_class(&xxx_transport_class);
+}
+
+static void __exit xxx_cleanup(void)
+{
+	drbd_unregister_transport_class(&xxx_transport_class);
+}
+
+module_init(xxx_init)
+module_exit(xxx_cleanup)
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
deleted file mode 100644
index 0697f99fed18d..0000000000000
--- a/drivers/block/drbd/drbd_worker.c
+++ /dev/null
@@ -1,2223 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
-   drbd_worker.c
-
-   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
-
-   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
-   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
-   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
-
-
-*/
-
-#include <linux/module.h>
-#include <linux/drbd.h>
-#include <linux/sched/signal.h>
-#include <linux/wait.h>
-#include <linux/mm.h>
-#include <linux/memcontrol.h>
-#include <linux/mm_inline.h>
-#include <linux/slab.h>
-#include <linux/random.h>
-#include <linux/string.h>
-#include <linux/scatterlist.h>
-#include <linux/part_stat.h>
-
-#include "drbd_int.h"
-#include "drbd_protocol.h"
-#include "drbd_req.h"
-
-static int make_ov_request(struct drbd_peer_device *, int);
-static int make_resync_request(struct drbd_peer_device *, int);
-
-/* endio handlers:
- *   drbd_md_endio (defined here)
- *   drbd_request_endio (defined here)
- *   drbd_peer_request_endio (defined here)
- *   drbd_bm_endio (defined in drbd_bitmap.c)
- *
- * For all these callbacks, note the following:
- * The callbacks will be called in irq context by the IDE drivers,
- * and in Softirqs/Tasklets/BH context by the SCSI drivers.
- * Try to get the locking right :)
- *
- */
-
-/* used for synchronous meta data and bitmap IO
- * submitted by drbd_md_sync_page_io()
- */
-void drbd_md_endio(struct bio *bio)
-{
-	struct drbd_device *device;
-
-	device = bio->bi_private;
-	device->md_io.error = blk_status_to_errno(bio->bi_status);
-
-	/* special case: drbd_md_read() during drbd_adm_attach() */
-	if (device->ldev)
-		put_ldev(device);
-	bio_put(bio);
-
-	/* We grabbed an extra reference in _drbd_md_sync_page_io() to be able
-	 * to timeout on the lower level device, and eventually detach from it.
-	 * If this io completion runs after that timeout expired, this
-	 * drbd_md_put_buffer() may allow us to finally try and re-attach.
-	 * During normal operation, this only puts that extra reference
-	 * down to 1 again.
-	 * Make sure we first drop the reference, and only then signal
-	 * completion, or we may (in drbd_al_read_log()) cycle so fast into the
-	 * next drbd_md_sync_page_io(), that we trigger the
-	 * ASSERT(atomic_read(&device->md_io_in_use) == 1) there.
-	 */
-	drbd_md_put_buffer(device);
-	device->md_io.done = 1;
-	wake_up(&device->misc_wait);
-}
-
-/* reads on behalf of the partner,
- * "submitted" by the receiver
- */
-static void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local)
-{
-	unsigned long flags = 0;
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	device->read_cnt += peer_req->i.size >> 9;
-	list_del(&peer_req->w.list);
-	if (list_empty(&device->read_ee))
-		wake_up(&device->ee_wait);
-	if (test_bit(__EE_WAS_ERROR, &peer_req->flags))
-		__drbd_chk_io_error(device, DRBD_READ_ERROR);
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-
-	drbd_queue_work(&peer_device->connection->sender_work, &peer_req->w);
-	put_ldev(device);
-}
-
-/* writes on behalf of the partner, or resync writes,
- * "submitted" by the receiver, final stage.  */
-void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local)
-{
-	unsigned long flags = 0;
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	struct drbd_connection *connection = peer_device->connection;
-	struct drbd_interval i;
-	int do_wake;
-	u64 block_id;
-	int do_al_complete_io;
-
-	/* after we moved peer_req to done_ee,
-	 * we may no longer access it,
-	 * it may be freed/reused already!
-	 * (as soon as we release the req_lock) */
-	i = peer_req->i;
-	do_al_complete_io = peer_req->flags & EE_CALL_AL_COMPLETE_IO;
-	block_id = peer_req->block_id;
-	peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
-
-	if (peer_req->flags & EE_WAS_ERROR) {
-		/* In protocol != C, we usually do not send write acks.
-		 * In case of a write error, send the neg ack anyways. */
-		if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags))
-			inc_unacked(device);
-		drbd_set_out_of_sync(peer_device, peer_req->i.sector, peer_req->i.size);
-	}
-
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	device->writ_cnt += peer_req->i.size >> 9;
-	list_move_tail(&peer_req->w.list, &device->done_ee);
-
-	/*
-	 * Do not remove from the write_requests tree here: we did not send the
-	 * Ack yet and did not wake possibly waiting conflicting requests.
-	 * Removed from the tree from "drbd_process_done_ee" within the
-	 * appropriate dw.cb (e_end_block/e_end_resync_block) or from
-	 * _drbd_clear_done_ee.
-	 */
-
-	do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee);
-
-	/* FIXME do we want to detach for failed REQ_OP_DISCARD?
-	 * ((peer_req->flags & (EE_WAS_ERROR|EE_TRIM)) == EE_WAS_ERROR) */
-	if (peer_req->flags & EE_WAS_ERROR)
-		__drbd_chk_io_error(device, DRBD_WRITE_ERROR);
-
-	if (connection->cstate >= C_WF_REPORT_PARAMS) {
-		kref_get(&device->kref); /* put is in drbd_send_acks_wf() */
-		if (!queue_work(connection->ack_sender, &peer_device->send_acks_work))
-			kref_put(&device->kref, drbd_destroy_device);
-	}
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-
-	if (block_id == ID_SYNCER)
-		drbd_rs_complete_io(device, i.sector);
-
-	if (do_wake)
-		wake_up(&device->ee_wait);
-
-	if (do_al_complete_io)
-		drbd_al_complete_io(device, &i);
-
-	put_ldev(device);
-}
-
-/* writes on behalf of the partner, or resync writes,
- * "submitted" by the receiver.
- */
-void drbd_peer_request_endio(struct bio *bio)
-{
-	struct drbd_peer_request *peer_req = bio->bi_private;
-	struct drbd_device *device = peer_req->peer_device->device;
-	bool is_write = bio_data_dir(bio) == WRITE;
-	bool is_discard = bio_op(bio) == REQ_OP_WRITE_ZEROES ||
-			  bio_op(bio) == REQ_OP_DISCARD;
-
-	if (bio->bi_status && drbd_ratelimit())
-		drbd_warn(device, "%s: error=%d s=%llus\n",
-				is_write ? (is_discard ? "discard" : "write")
-					: "read", bio->bi_status,
-				(unsigned long long)peer_req->i.sector);
-
-	if (bio->bi_status)
-		set_bit(__EE_WAS_ERROR, &peer_req->flags);
-
-	bio_put(bio); /* no need for the bio anymore */
-	if (atomic_dec_and_test(&peer_req->pending_bios)) {
-		if (is_write)
-			drbd_endio_write_sec_final(peer_req);
-		else
-			drbd_endio_read_sec_final(peer_req);
-	}
-}
-
-static void
-drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
-{
-	panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n",
-		device->minor, device->resource->name, device->vnr);
-}
-
-/* read, readA or write requests on R_PRIMARY coming from drbd_make_request
- */
-void drbd_request_endio(struct bio *bio)
-{
-	unsigned long flags;
-	struct drbd_request *req = bio->bi_private;
-	struct drbd_device *device = req->device;
-	struct bio_and_error m;
-	enum drbd_req_event what;
-
-	/* If this request was aborted locally before,
-	 * but now was completed "successfully",
-	 * chances are that this caused arbitrary data corruption.
-	 *
-	 * "aborting" requests, or force-detaching the disk, is intended for
-	 * completely blocked/hung local backing devices which do no longer
-	 * complete requests at all, not even do error completions.  In this
-	 * situation, usually a hard-reset and failover is the only way out.
-	 *
-	 * By "aborting", basically faking a local error-completion,
-	 * we allow for a more graceful swichover by cleanly migrating services.
-	 * Still the affected node has to be rebooted "soon".
-	 *
-	 * By completing these requests, we allow the upper layers to re-use
-	 * the associated data pages.
-	 *
-	 * If later the local backing device "recovers", and now DMAs some data
-	 * from disk into the original request pages, in the best case it will
-	 * just put random data into unused pages; but typically it will corrupt
-	 * meanwhile completely unrelated data, causing all sorts of damage.
-	 *
-	 * Which means delayed successful completion,
-	 * especially for READ requests,
-	 * is a reason to panic().
-	 *
-	 * We assume that a delayed *error* completion is OK,
-	 * though we still will complain noisily about it.
-	 */
-	if (unlikely(req->rq_state & RQ_LOCAL_ABORTED)) {
-		if (drbd_ratelimit())
-			drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n");
-
-		if (!bio->bi_status)
-			drbd_panic_after_delayed_completion_of_aborted_request(device);
-	}
-
-	/* to avoid recursion in __req_mod */
-	if (unlikely(bio->bi_status)) {
-		switch (bio_op(bio)) {
-		case REQ_OP_WRITE_ZEROES:
-		case REQ_OP_DISCARD:
-			if (bio->bi_status == BLK_STS_NOTSUPP)
-				what = DISCARD_COMPLETED_NOTSUPP;
-			else
-				what = DISCARD_COMPLETED_WITH_ERROR;
-			break;
-		case REQ_OP_READ:
-			if (bio->bi_opf & REQ_RAHEAD)
-				what = READ_AHEAD_COMPLETED_WITH_ERROR;
-			else
-				what = READ_COMPLETED_WITH_ERROR;
-			break;
-		default:
-			what = WRITE_COMPLETED_WITH_ERROR;
-			break;
-		}
-	} else {
-		what = COMPLETED_OK;
-	}
-
-	req->private_bio = ERR_PTR(blk_status_to_errno(bio->bi_status));
-	bio_put(bio);
-
-	/* not req_mod(), we need irqsave here! */
-	spin_lock_irqsave(&device->resource->req_lock, flags);
-	__req_mod(req, what, NULL, &m);
-	spin_unlock_irqrestore(&device->resource->req_lock, flags);
-	put_ldev(device);
-
-	if (m.bio)
-		complete_master_bio(device, &m);
-}
-
-void drbd_csum_ee(struct crypto_shash *tfm, struct drbd_peer_request *peer_req, void *digest)
-{
-	SHASH_DESC_ON_STACK(desc, tfm);
-	struct page *page = peer_req->pages;
-	struct page *tmp;
-	unsigned len;
-	void *src;
-
-	desc->tfm = tfm;
-
-	crypto_shash_init(desc);
-
-	src = kmap_atomic(page);
-	while ((tmp = page_chain_next(page))) {
-		/* all but the last page will be fully used */
-		crypto_shash_update(desc, src, PAGE_SIZE);
-		kunmap_atomic(src);
-		page = tmp;
-		src = kmap_atomic(page);
-	}
-	/* and now the last, possibly only partially used page */
-	len = peer_req->i.size & (PAGE_SIZE - 1);
-	crypto_shash_update(desc, src, len ?: PAGE_SIZE);
-	kunmap_atomic(src);
-
-	crypto_shash_final(desc, digest);
-	shash_desc_zero(desc);
-}
-
-void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest)
-{
-	SHASH_DESC_ON_STACK(desc, tfm);
-	struct bio_vec bvec;
-	struct bvec_iter iter;
-
-	desc->tfm = tfm;
-
-	crypto_shash_init(desc);
-
-	bio_for_each_segment(bvec, bio, iter) {
-		u8 *src;
-
-		src = bvec_kmap_local(&bvec);
-		crypto_shash_update(desc, src, bvec.bv_len);
-		kunmap_local(src);
-	}
-	crypto_shash_final(desc, digest);
-	shash_desc_zero(desc);
-}
-
-/* MAYBE merge common code with w_e_end_ov_req */
-static int w_e_send_csum(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	int digest_size;
-	void *digest;
-	int err = 0;
-
-	if (unlikely(cancel))
-		goto out;
-
-	if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0))
-		goto out;
-
-	digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm);
-	digest = kmalloc(digest_size, GFP_NOIO);
-	if (digest) {
-		sector_t sector = peer_req->i.sector;
-		unsigned int size = peer_req->i.size;
-		drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest);
-		/* Free peer_req and pages before send.
-		 * In case we block on congestion, we could otherwise run into
-		 * some distributed deadlock, if the other side blocks on
-		 * congestion as well, because our receiver blocks in
-		 * drbd_alloc_pages due to pp_in_use > max_buffers. */
-		drbd_free_peer_req(device, peer_req);
-		peer_req = NULL;
-		inc_rs_pending(peer_device);
-		err = drbd_send_drequest_csum(peer_device, sector, size,
-					      digest, digest_size,
-					      P_CSUM_RS_REQUEST);
-		kfree(digest);
-	} else {
-		drbd_err(device, "kmalloc() of digest failed.\n");
-		err = -ENOMEM;
-	}
-
-out:
-	if (peer_req)
-		drbd_free_peer_req(device, peer_req);
-
-	if (unlikely(err))
-		drbd_err(device, "drbd_send_drequest(..., csum) failed\n");
-	return err;
-}
-
-#define GFP_TRY	(__GFP_HIGHMEM | __GFP_NOWARN)
-
-static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, int size)
-{
-	struct drbd_device *device = peer_device->device;
-	struct drbd_peer_request *peer_req;
-
-	if (!get_ldev(device))
-		return -EIO;
-
-	/* GFP_TRY, because if there is no memory available right now, this may
-	 * be rescheduled for later. It is "only" background resync, after all. */
-	peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector,
-				       size, size, GFP_TRY);
-	if (!peer_req)
-		goto defer;
-
-	peer_req->w.cb = w_e_send_csum;
-	peer_req->opf = REQ_OP_READ;
-	spin_lock_irq(&device->resource->req_lock);
-	list_add_tail(&peer_req->w.list, &device->read_ee);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	atomic_add(size >> 9, &device->rs_sect_ev);
-	if (drbd_submit_peer_request(peer_req) == 0)
-		return 0;
-
-	/* If it failed because of ENOMEM, retry should help.  If it failed
-	 * because bio_add_page failed (probably broken lower level driver),
-	 * retry may or may not help.
-	 * If it does not, you may need to force disconnect. */
-	spin_lock_irq(&device->resource->req_lock);
-	list_del(&peer_req->w.list);
-	spin_unlock_irq(&device->resource->req_lock);
-
-	drbd_free_peer_req(device, peer_req);
-defer:
-	put_ldev(device);
-	return -EAGAIN;
-}
-
-int w_resync_timer(struct drbd_work *w, int cancel)
-{
-	struct drbd_device *device =
-		container_of(w, struct drbd_device, resync_work);
-
-	switch (device->state.conn) {
-	case C_VERIFY_S:
-		make_ov_request(first_peer_device(device), cancel);
-		break;
-	case C_SYNC_TARGET:
-		make_resync_request(first_peer_device(device), cancel);
-		break;
-	}
-
-	return 0;
-}
-
-void resync_timer_fn(struct timer_list *t)
-{
-	struct drbd_device *device = timer_container_of(device, t,
-							resync_timer);
-
-	drbd_queue_work_if_unqueued(
-		&first_peer_device(device)->connection->sender_work,
-		&device->resync_work);
-}
-
-static void fifo_set(struct fifo_buffer *fb, int value)
-{
-	int i;
-
-	for (i = 0; i < fb->size; i++)
-		fb->values[i] = value;
-}
-
-static int fifo_push(struct fifo_buffer *fb, int value)
-{
-	int ov;
-
-	ov = fb->values[fb->head_index];
-	fb->values[fb->head_index++] = value;
-
-	if (fb->head_index >= fb->size)
-		fb->head_index = 0;
-
-	return ov;
-}
-
-static void fifo_add_val(struct fifo_buffer *fb, int value)
-{
-	int i;
-
-	for (i = 0; i < fb->size; i++)
-		fb->values[i] += value;
-}
-
-struct fifo_buffer *fifo_alloc(unsigned int fifo_size)
-{
-	struct fifo_buffer *fb;
-
-	fb = kzalloc_flex(*fb, values, fifo_size, GFP_NOIO);
-	if (!fb)
-		return NULL;
-
-	fb->head_index = 0;
-	fb->size = fifo_size;
-	fb->total = 0;
-
-	return fb;
-}
-
-static int drbd_rs_controller(struct drbd_peer_device *peer_device, unsigned int sect_in)
-{
-	struct drbd_device *device = peer_device->device;
-	struct disk_conf *dc;
-	unsigned int want;     /* The number of sectors we want in-flight */
-	int req_sect; /* Number of sectors to request in this turn */
-	int correction; /* Number of sectors more we need in-flight */
-	int cps; /* correction per invocation of drbd_rs_controller() */
-	int steps; /* Number of time steps to plan ahead */
-	int curr_corr;
-	int max_sect;
-	struct fifo_buffer *plan;
-
-	dc = rcu_dereference(device->ldev->disk_conf);
-	plan = rcu_dereference(device->rs_plan_s);
-
-	steps = plan->size; /* (dc->c_plan_ahead * 10 * SLEEP_TIME) / HZ; */
-
-	if (device->rs_in_flight + sect_in == 0) { /* At start of resync */
-		want = ((dc->resync_rate * 2 * SLEEP_TIME) / HZ) * steps;
-	} else { /* normal path */
-		want = dc->c_fill_target ? dc->c_fill_target :
-			sect_in * dc->c_delay_target * HZ / (SLEEP_TIME * 10);
-	}
-
-	correction = want - device->rs_in_flight - plan->total;
-
-	/* Plan ahead */
-	cps = correction / steps;
-	fifo_add_val(plan, cps);
-	plan->total += cps * steps;
-
-	/* What we do in this step */
-	curr_corr = fifo_push(plan, 0);
-	plan->total -= curr_corr;
-
-	req_sect = sect_in + curr_corr;
-	if (req_sect < 0)
-		req_sect = 0;
-
-	max_sect = (dc->c_max_rate * 2 * SLEEP_TIME) / HZ;
-	if (req_sect > max_sect)
-		req_sect = max_sect;
-
-	/*
-	drbd_warn(device, "si=%u if=%d wa=%u co=%d st=%d cps=%d pl=%d cc=%d rs=%d\n",
-		 sect_in, device->rs_in_flight, want, correction,
-		 steps, cps, device->rs_planed, curr_corr, req_sect);
-	*/
-
-	return req_sect;
-}
-
-static int drbd_rs_number_requests(struct drbd_peer_device *peer_device)
-{
-	struct drbd_device *device = peer_device->device;
-	unsigned int sect_in;  /* Number of sectors that came in since the last turn */
-	int number, mxb;
-
-	sect_in = atomic_xchg(&device->rs_sect_in, 0);
-	device->rs_in_flight -= sect_in;
-
-	rcu_read_lock();
-	mxb = drbd_get_max_buffers(device) / 2;
-	if (rcu_dereference(device->rs_plan_s)->size) {
-		number = drbd_rs_controller(peer_device, sect_in) >> (BM_BLOCK_SHIFT - 9);
-		device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME;
-	} else {
-		device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate;
-		number = SLEEP_TIME * device->c_sync_rate  / ((BM_BLOCK_SIZE / 1024) * HZ);
-	}
-	rcu_read_unlock();
-
-	/* Don't have more than "max-buffers"/2 in-flight.
-	 * Otherwise we may cause the remote site to stall on drbd_alloc_pages(),
-	 * potentially causing a distributed deadlock on congestion during
-	 * online-verify or (checksum-based) resync, if max-buffers,
-	 * socket buffer sizes and resync rate settings are mis-configured. */
-
-	/* note that "number" is in units of "BM_BLOCK_SIZE" (which is 4k),
-	 * mxb (as used here, and in drbd_alloc_pages on the peer) is
-	 * "number of pages" (typically also 4k),
-	 * but "rs_in_flight" is in "sectors" (512 Byte). */
-	if (mxb - device->rs_in_flight/8 < number)
-		number = mxb - device->rs_in_flight/8;
-
-	return number;
-}
-
-static int make_resync_request(struct drbd_peer_device *const peer_device, int cancel)
-{
-	struct drbd_device *const device = peer_device->device;
-	struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
-	unsigned long bit;
-	sector_t sector;
-	const sector_t capacity = get_capacity(device->vdisk);
-	int max_bio_size;
-	int number, rollback_i, size;
-	int align, requeue = 0;
-	int i = 0;
-	int discard_granularity = 0;
-
-	if (unlikely(cancel))
-		return 0;
-
-	if (device->rs_total == 0) {
-		/* empty resync? */
-		drbd_resync_finished(peer_device);
-		return 0;
-	}
-
-	if (!get_ldev(device)) {
-		/* Since we only need to access device->rsync a
-		   get_ldev_if_state(device,D_FAILED) would be sufficient, but
-		   to continue resync with a broken disk makes no sense at
-		   all */
-		drbd_err(device, "Disk broke down during resync!\n");
-		return 0;
-	}
-
-	if (connection->agreed_features & DRBD_FF_THIN_RESYNC) {
-		rcu_read_lock();
-		discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity;
-		rcu_read_unlock();
-	}
-
-	max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9;
-	number = drbd_rs_number_requests(peer_device);
-	if (number <= 0)
-		goto requeue;
-
-	for (i = 0; i < number; i++) {
-		/* Stop generating RS requests when half of the send buffer is filled,
-		 * but notify TCP that we'd like to have more space. */
-		mutex_lock(&connection->data.mutex);
-		if (connection->data.socket) {
-			struct sock *sk = connection->data.socket->sk;
-			int queued = sk->sk_wmem_queued;
-			int sndbuf = sk->sk_sndbuf;
-			if (queued > sndbuf / 2) {
-				requeue = 1;
-				if (sk->sk_socket)
-					set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-			}
-		} else
-			requeue = 1;
-		mutex_unlock(&connection->data.mutex);
-		if (requeue)
-			goto requeue;
-
-next_sector:
-		size = BM_BLOCK_SIZE;
-		bit  = drbd_bm_find_next(device, device->bm_resync_fo);
-
-		if (bit == DRBD_END_OF_BITMAP) {
-			device->bm_resync_fo = drbd_bm_bits(device);
-			put_ldev(device);
-			return 0;
-		}
-
-		sector = BM_BIT_TO_SECT(bit);
-
-		if (drbd_try_rs_begin_io(peer_device, sector)) {
-			device->bm_resync_fo = bit;
-			goto requeue;
-		}
-		device->bm_resync_fo = bit + 1;
-
-		if (unlikely(drbd_bm_test_bit(device, bit) == 0)) {
-			drbd_rs_complete_io(device, sector);
-			goto next_sector;
-		}
-
-#if DRBD_MAX_BIO_SIZE > BM_BLOCK_SIZE
-		/* try to find some adjacent bits.
-		 * we stop if we have already the maximum req size.
-		 *
-		 * Additionally always align bigger requests, in order to
-		 * be prepared for all stripe sizes of software RAIDs.
-		 */
-		align = 1;
-		rollback_i = i;
-		while (i < number) {
-			if (size + BM_BLOCK_SIZE > max_bio_size)
-				break;
-
-			/* Be always aligned */
-			if (sector & ((1<<(align+3))-1))
-				break;
-
-			if (discard_granularity && size == discard_granularity)
-				break;
-
-			/* do not cross extent boundaries */
-			if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0)
-				break;
-			/* now, is it actually dirty, after all?
-			 * caution, drbd_bm_test_bit is tri-state for some
-			 * obscure reason; ( b == 0 ) would get the out-of-band
-			 * only accidentally right because of the "oddly sized"
-			 * adjustment below */
-			if (drbd_bm_test_bit(device, bit+1) != 1)
-				break;
-			bit++;
-			size += BM_BLOCK_SIZE;
-			if ((BM_BLOCK_SIZE << align) <= size)
-				align++;
-			i++;
-		}
-		/* if we merged some,
-		 * reset the offset to start the next drbd_bm_find_next from */
-		if (size > BM_BLOCK_SIZE)
-			device->bm_resync_fo = bit + 1;
-#endif
-
-		/* adjust very last sectors, in case we are oddly sized */
-		if (sector + (size>>9) > capacity)
-			size = (capacity-sector)<<9;
-
-		if (device->use_csums) {
-			switch (read_for_csum(peer_device, sector, size)) {
-			case -EIO: /* Disk failure */
-				put_ldev(device);
-				return -EIO;
-			case -EAGAIN: /* allocation failed, or ldev busy */
-				drbd_rs_complete_io(device, sector);
-				device->bm_resync_fo = BM_SECT_TO_BIT(sector);
-				i = rollback_i;
-				goto requeue;
-			case 0:
-				/* everything ok */
-				break;
-			default:
-				BUG();
-			}
-		} else {
-			int err;
-
-			inc_rs_pending(peer_device);
-			err = drbd_send_drequest(peer_device,
-						 size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST,
-						 sector, size, ID_SYNCER);
-			if (err) {
-				drbd_err(device, "drbd_send_drequest() failed, aborting...\n");
-				dec_rs_pending(peer_device);
-				put_ldev(device);
-				return err;
-			}
-		}
-	}
-
-	if (device->bm_resync_fo >= drbd_bm_bits(device)) {
-		/* last syncer _request_ was sent,
-		 * but the P_RS_DATA_REPLY not yet received.  sync will end (and
-		 * next sync group will resume), as soon as we receive the last
-		 * resync data block, and the last bit is cleared.
-		 * until then resync "work" is "inactive" ...
-		 */
-		put_ldev(device);
-		return 0;
-	}
-
- requeue:
-	device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));
-	mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);
-	put_ldev(device);
-	return 0;
-}
-
-static int make_ov_request(struct drbd_peer_device *peer_device, int cancel)
-{
-	struct drbd_device *device = peer_device->device;
-	int number, i, size;
-	sector_t sector;
-	const sector_t capacity = get_capacity(device->vdisk);
-	bool stop_sector_reached = false;
-
-	if (unlikely(cancel))
-		return 1;
-
-	number = drbd_rs_number_requests(peer_device);
-
-	sector = device->ov_position;
-	for (i = 0; i < number; i++) {
-		if (sector >= capacity)
-			return 1;
-
-		/* We check for "finished" only in the reply path:
-		 * w_e_end_ov_reply().
-		 * We need to send at least one request out. */
-		stop_sector_reached = i > 0
-			&& verify_can_do_stop_sector(device)
-			&& sector >= device->ov_stop_sector;
-		if (stop_sector_reached)
-			break;
-
-		size = BM_BLOCK_SIZE;
-
-		if (drbd_try_rs_begin_io(peer_device, sector)) {
-			device->ov_position = sector;
-			goto requeue;
-		}
-
-		if (sector + (size>>9) > capacity)
-			size = (capacity-sector)<<9;
-
-		inc_rs_pending(peer_device);
-		if (drbd_send_ov_request(first_peer_device(device), sector, size)) {
-			dec_rs_pending(peer_device);
-			return 0;
-		}
-		sector += BM_SECT_PER_BIT;
-	}
-	device->ov_position = sector;
-
- requeue:
-	device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9));
-	if (i == 0 || !stop_sector_reached)
-		mod_timer(&device->resync_timer, jiffies + SLEEP_TIME);
-	return 1;
-}
-
-int w_ov_finished(struct drbd_work *w, int cancel)
-{
-	struct drbd_device_work *dw =
-		container_of(w, struct drbd_device_work, w);
-	struct drbd_device *device = dw->device;
-	kfree(dw);
-	ov_out_of_sync_print(first_peer_device(device));
-	drbd_resync_finished(first_peer_device(device));
-
-	return 0;
-}
-
-static int w_resync_finished(struct drbd_work *w, int cancel)
-{
-	struct drbd_device_work *dw =
-		container_of(w, struct drbd_device_work, w);
-	struct drbd_device *device = dw->device;
-	kfree(dw);
-
-	drbd_resync_finished(first_peer_device(device));
-
-	return 0;
-}
-
-static void ping_peer(struct drbd_device *device)
-{
-	struct drbd_connection *connection = first_peer_device(device)->connection;
-
-	clear_bit(GOT_PING_ACK, &connection->flags);
-	request_ping(connection);
-	wait_event(connection->ping_wait,
-		   test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED);
-}
-
-int drbd_resync_finished(struct drbd_peer_device *peer_device)
-{
-	struct drbd_device *device = peer_device->device;
-	struct drbd_connection *connection = peer_device->connection;
-	unsigned long db, dt, dbdt;
-	unsigned long n_oos;
-	union drbd_state os, ns;
-	struct drbd_device_work *dw;
-	char *khelper_cmd = NULL;
-	int verify_done = 0;
-
-	/* Remove all elements from the resync LRU. Since future actions
-	 * might set bits in the (main) bitmap, then the entries in the
-	 * resync LRU would be wrong. */
-	if (drbd_rs_del_all(device)) {
-		/* In case this is not possible now, most probably because
-		 * there are P_RS_DATA_REPLY Packets lingering on the worker's
-		 * queue (or even the read operations for those packets
-		 * is not finished by now).   Retry in 100ms. */
-
-		schedule_timeout_interruptible(HZ / 10);
-		dw = kmalloc_obj(struct drbd_device_work, GFP_ATOMIC);
-		if (dw) {
-			dw->w.cb = w_resync_finished;
-			dw->device = device;
-			drbd_queue_work(&connection->sender_work, &dw->w);
-			return 1;
-		}
-		drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n");
-	}
-
-	dt = (jiffies - device->rs_start - device->rs_paused) / HZ;
-	if (dt <= 0)
-		dt = 1;
-
-	db = device->rs_total;
-	/* adjust for verify start and stop sectors, respective reached position */
-	if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T)
-		db -= device->ov_left;
-
-	dbdt = Bit2KB(db/dt);
-	device->rs_paused /= HZ;
-
-	if (!get_ldev(device))
-		goto out;
-
-	ping_peer(device);
-
-	spin_lock_irq(&device->resource->req_lock);
-	os = drbd_read_state(device);
-
-	verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T);
-
-	/* This protects us against multiple calls (that can happen in the presence
-	   of application IO), and against connectivity loss just before we arrive here. */
-	if (os.conn <= C_CONNECTED)
-		goto out_unlock;
-
-	ns = os;
-	ns.conn = C_CONNECTED;
-
-	drbd_info(device, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n",
-	     verify_done ? "Online verify" : "Resync",
-	     dt + device->rs_paused, device->rs_paused, dbdt);
-
-	n_oos = drbd_bm_total_weight(device);
-
-	if (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) {
-		if (n_oos) {
-			drbd_alert(device, "Online verify found %lu %dk block out of sync!\n",
-			      n_oos, Bit2KB(1));
-			khelper_cmd = "out-of-sync";
-		}
-	} else {
-		D_ASSERT(device, (n_oos - device->rs_failed) == 0);
-
-		if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T)
-			khelper_cmd = "after-resync-target";
-
-		if (device->use_csums && device->rs_total) {
-			const unsigned long s = device->rs_same_csum;
-			const unsigned long t = device->rs_total;
-			const int ratio =
-				(t == 0)     ? 0 :
-			(t < 100000) ? ((s*100)/t) : (s/(t/100));
-			drbd_info(device, "%u %% had equal checksums, eliminated: %luK; "
-			     "transferred %luK total %luK\n",
-			     ratio,
-			     Bit2KB(device->rs_same_csum),
-			     Bit2KB(device->rs_total - device->rs_same_csum),
-			     Bit2KB(device->rs_total));
-		}
-	}
-
-	if (device->rs_failed) {
-		drbd_info(device, "            %lu failed blocks\n", device->rs_failed);
-
-		if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) {
-			ns.disk = D_INCONSISTENT;
-			ns.pdsk = D_UP_TO_DATE;
-		} else {
-			ns.disk = D_UP_TO_DATE;
-			ns.pdsk = D_INCONSISTENT;
-		}
-	} else {
-		ns.disk = D_UP_TO_DATE;
-		ns.pdsk = D_UP_TO_DATE;
-
-		if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) {
-			if (device->p_uuid) {
-				int i;
-				for (i = UI_BITMAP ; i <= UI_HISTORY_END ; i++)
-					_drbd_uuid_set(device, i, device->p_uuid[i]);
-				drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_CURRENT]);
-				_drbd_uuid_set(device, UI_CURRENT, device->p_uuid[UI_CURRENT]);
-			} else {
-				drbd_err(device, "device->p_uuid is NULL! BUG\n");
-			}
-		}
-
-		if (!(os.conn == C_VERIFY_S || os.conn == C_VERIFY_T)) {
-			/* for verify runs, we don't update uuids here,
-			 * so there would be nothing to report. */
-			drbd_uuid_set_bm(device, 0UL);
-			drbd_print_uuids(device, "updated UUIDs");
-			if (device->p_uuid) {
-				/* Now the two UUID sets are equal, update what we
-				 * know of the peer. */
-				int i;
-				for (i = UI_CURRENT ; i <= UI_HISTORY_END ; i++)
-					device->p_uuid[i] = device->ldev->md.uuid[i];
-			}
-		}
-	}
-
-	_drbd_set_state(device, ns, CS_VERBOSE, NULL);
-out_unlock:
-	spin_unlock_irq(&device->resource->req_lock);
-
-	/* If we have been sync source, and have an effective fencing-policy,
-	 * once *all* volumes are back in sync, call "unfence". */
-	if (os.conn == C_SYNC_SOURCE) {
-		enum drbd_disk_state disk_state = D_MASK;
-		enum drbd_disk_state pdsk_state = D_MASK;
-		enum drbd_fencing_p fp = FP_DONT_CARE;
-
-		rcu_read_lock();
-		fp = rcu_dereference(device->ldev->disk_conf)->fencing;
-		if (fp != FP_DONT_CARE) {
-			struct drbd_peer_device *peer_device;
-			int vnr;
-			idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-				struct drbd_device *device = peer_device->device;
-				disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk);
-				pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk);
-			}
-		}
-		rcu_read_unlock();
-		if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE)
-			conn_khelper(connection, "unfence-peer");
-	}
-
-	put_ldev(device);
-out:
-	device->rs_total  = 0;
-	device->rs_failed = 0;
-	device->rs_paused = 0;
-
-	/* reset start sector, if we reached end of device */
-	if (verify_done && device->ov_left == 0)
-		device->ov_start_sector = 0;
-
-	drbd_md_sync(device);
-
-	if (khelper_cmd)
-		drbd_khelper(device, khelper_cmd);
-
-	return 1;
-}
-
-/**
- * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST
- * @w:		work object.
- * @cancel:	The connection will be closed anyways
- */
-int w_e_end_data_req(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	int err;
-
-	if (unlikely(cancel)) {
-		err = 0;
-		goto out;
-	}
-
-	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		err = drbd_send_block(peer_device, P_DATA_REPLY, peer_req);
-	} else {
-		if (drbd_ratelimit())
-			drbd_err(device, "Sending NegDReply. sector=%llus.\n",
-			    (unsigned long long)peer_req->i.sector);
-
-		err = drbd_send_ack(peer_device, P_NEG_DREPLY, peer_req);
-	}
-
-	if (unlikely(err))
-		drbd_err(device, "drbd_send_block() failed\n");
-out:
-	dec_unacked(device);
-	drbd_free_peer_req(device, peer_req);
-
-	return err;
-}
-
-static bool all_zero(struct drbd_peer_request *peer_req)
-{
-	struct page *page = peer_req->pages;
-	unsigned int len = peer_req->i.size;
-
-	page_chain_for_each(page) {
-		unsigned int l = min_t(unsigned int, len, PAGE_SIZE);
-		unsigned int i, words = l / sizeof(long);
-		unsigned long *d;
-
-		d = kmap_atomic(page);
-		for (i = 0; i < words; i++) {
-			if (d[i]) {
-				kunmap_atomic(d);
-				return false;
-			}
-		}
-		kunmap_atomic(d);
-		len -= l;
-	}
-
-	return true;
-}
-
-/**
- * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST
- * @w:		work object.
- * @cancel:	The connection will be closed anyways
- */
-int w_e_end_rsdata_req(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	int err;
-
-	if (unlikely(cancel)) {
-		err = 0;
-		goto out;
-	}
-
-	if (get_ldev_if_state(device, D_FAILED)) {
-		drbd_rs_complete_io(device, peer_req->i.sector);
-		put_ldev(device);
-	}
-
-	if (device->state.conn == C_AHEAD) {
-		err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req);
-	} else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		if (likely(device->state.pdsk >= D_INCONSISTENT)) {
-			inc_rs_pending(peer_device);
-			if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req))
-				err = drbd_send_rs_deallocated(peer_device, peer_req);
-			else
-				err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req);
-		} else {
-			if (drbd_ratelimit())
-				drbd_err(device, "Not sending RSDataReply, "
-				    "partner DISKLESS!\n");
-			err = 0;
-		}
-	} else {
-		if (drbd_ratelimit())
-			drbd_err(device, "Sending NegRSDReply. sector %llus.\n",
-			    (unsigned long long)peer_req->i.sector);
-
-		err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
-
-		/* update resync data with failure */
-		drbd_rs_failed_io(peer_device, peer_req->i.sector, peer_req->i.size);
-	}
-	if (unlikely(err))
-		drbd_err(device, "drbd_send_block() failed\n");
-out:
-	dec_unacked(device);
-	drbd_free_peer_req(device, peer_req);
-
-	return err;
-}
-
-int w_e_end_csum_rs_req(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	struct digest_info *di;
-	int digest_size;
-	void *digest = NULL;
-	int err, eq = 0;
-
-	if (unlikely(cancel)) {
-		err = 0;
-		goto out;
-	}
-
-	if (get_ldev(device)) {
-		drbd_rs_complete_io(device, peer_req->i.sector);
-		put_ldev(device);
-	}
-
-	di = peer_req->digest;
-
-	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		/* quick hack to try to avoid a race against reconfiguration.
-		 * a real fix would be much more involved,
-		 * introducing more locking mechanisms */
-		if (peer_device->connection->csums_tfm) {
-			digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm);
-			D_ASSERT(device, digest_size == di->digest_size);
-			digest = kmalloc(digest_size, GFP_NOIO);
-		}
-		if (digest) {
-			drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest);
-			eq = !memcmp(digest, di->digest, digest_size);
-			kfree(digest);
-		}
-
-		if (eq) {
-			drbd_set_in_sync(peer_device, peer_req->i.sector, peer_req->i.size);
-			/* rs_same_csums unit is BM_BLOCK_SIZE */
-			device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT;
-			err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req);
-		} else {
-			inc_rs_pending(peer_device);
-			peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */
-			peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */
-			kfree(di);
-			err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req);
-		}
-	} else {
-		err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req);
-		if (drbd_ratelimit())
-			drbd_err(device, "Sending NegDReply. I guess it gets messy.\n");
-	}
-	if (unlikely(err))
-		drbd_err(device, "drbd_send_block/ack() failed\n");
-out:
-	dec_unacked(device);
-	drbd_free_peer_req(device, peer_req);
-
-	return err;
-}
-
-int w_e_end_ov_req(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	sector_t sector = peer_req->i.sector;
-	unsigned int size = peer_req->i.size;
-	int digest_size;
-	void *digest;
-	int err = 0;
-
-	if (unlikely(cancel))
-		goto out;
-
-	digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
-	digest = kmalloc(digest_size, GFP_NOIO);
-	if (!digest) {
-		err = 1;	/* terminate the connection in case the allocation failed */
-		goto out;
-	}
-
-	if (likely(!(peer_req->flags & EE_WAS_ERROR)))
-		drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest);
-	else
-		memset(digest, 0, digest_size);
-
-	/* Free e and pages before send.
-	 * In case we block on congestion, we could otherwise run into
-	 * some distributed deadlock, if the other side blocks on
-	 * congestion as well, because our receiver blocks in
-	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
-	drbd_free_peer_req(device, peer_req);
-	peer_req = NULL;
-	inc_rs_pending(peer_device);
-	err = drbd_send_drequest_csum(peer_device, sector, size, digest, digest_size, P_OV_REPLY);
-	if (err)
-		dec_rs_pending(peer_device);
-	kfree(digest);
-
-out:
-	if (peer_req)
-		drbd_free_peer_req(device, peer_req);
-	dec_unacked(device);
-	return err;
-}
-
-void drbd_ov_out_of_sync_found(struct drbd_peer_device *peer_device, sector_t sector, int size)
-{
-	struct drbd_device *device = peer_device->device;
-	if (device->ov_last_oos_start + device->ov_last_oos_size == sector) {
-		device->ov_last_oos_size += size>>9;
-	} else {
-		device->ov_last_oos_start = sector;
-		device->ov_last_oos_size = size>>9;
-	}
-	drbd_set_out_of_sync(peer_device, sector, size);
-}
-
-int w_e_end_ov_reply(struct drbd_work *w, int cancel)
-{
-	struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w);
-	struct drbd_peer_device *peer_device = peer_req->peer_device;
-	struct drbd_device *device = peer_device->device;
-	struct digest_info *di;
-	void *digest;
-	sector_t sector = peer_req->i.sector;
-	unsigned int size = peer_req->i.size;
-	int digest_size;
-	int err, eq = 0;
-	bool stop_sector_reached = false;
-
-	if (unlikely(cancel)) {
-		drbd_free_peer_req(device, peer_req);
-		dec_unacked(device);
-		return 0;
-	}
-
-	/* after "cancel", because after drbd_disconnect/drbd_rs_cancel_all
-	 * the resync lru has been cleaned up already */
-	if (get_ldev(device)) {
-		drbd_rs_complete_io(device, peer_req->i.sector);
-		put_ldev(device);
-	}
-
-	di = peer_req->digest;
-
-	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {
-		digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm);
-		digest = kmalloc(digest_size, GFP_NOIO);
-		if (digest) {
-			drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest);
-
-			D_ASSERT(device, digest_size == di->digest_size);
-			eq = !memcmp(digest, di->digest, digest_size);
-			kfree(digest);
-		}
-	}
-
-	/* Free peer_req and pages before send.
-	 * In case we block on congestion, we could otherwise run into
-	 * some distributed deadlock, if the other side blocks on
-	 * congestion as well, because our receiver blocks in
-	 * drbd_alloc_pages due to pp_in_use > max_buffers. */
-	drbd_free_peer_req(device, peer_req);
-	if (!eq)
-		drbd_ov_out_of_sync_found(peer_device, sector, size);
-	else
-		ov_out_of_sync_print(peer_device);
-
-	err = drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size,
-			       eq ? ID_IN_SYNC : ID_OUT_OF_SYNC);
-
-	dec_unacked(device);
-
-	--device->ov_left;
-
-	/* let's advance progress step marks only for every other megabyte */
-	if ((device->ov_left & 0x200) == 0x200)
-		drbd_advance_rs_marks(peer_device, device->ov_left);
-
-	stop_sector_reached = verify_can_do_stop_sector(device) &&
-		(sector + (size>>9)) >= device->ov_stop_sector;
-
-	if (device->ov_left == 0 || stop_sector_reached) {
-		ov_out_of_sync_print(peer_device);
-		drbd_resync_finished(peer_device);
-	}
-
-	return err;
-}
-
-/* FIXME
- * We need to track the number of pending barrier acks,
- * and to be able to wait for them.
- * See also comment in drbd_adm_attach before drbd_suspend_io.
- */
-static int drbd_send_barrier(struct drbd_connection *connection)
-{
-	struct p_barrier *p;
-	struct drbd_socket *sock;
-
-	sock = &connection->data;
-	p = conn_prepare_command(connection, sock);
-	if (!p)
-		return -EIO;
-	p->barrier = connection->send.current_epoch_nr;
-	p->pad = 0;
-	connection->send.current_epoch_writes = 0;
-	connection->send.last_sent_barrier_jif = jiffies;
-
-	return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0);
-}
-
-static int pd_send_unplug_remote(struct drbd_peer_device *pd)
-{
-	struct drbd_socket *sock = &pd->connection->data;
-	if (!drbd_prepare_command(pd, sock))
-		return -EIO;
-	return drbd_send_command(pd, sock, P_UNPLUG_REMOTE, 0, NULL, 0);
-}
-
-int w_send_write_hint(struct drbd_work *w, int cancel)
-{
-	struct drbd_device *device =
-		container_of(w, struct drbd_device, unplug_work);
-
-	if (cancel)
-		return 0;
-	return pd_send_unplug_remote(first_peer_device(device));
-}
-
-static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch)
-{
-	if (!connection->send.seen_any_write_yet) {
-		connection->send.seen_any_write_yet = true;
-		connection->send.current_epoch_nr = epoch;
-		connection->send.current_epoch_writes = 0;
-		connection->send.last_sent_barrier_jif = jiffies;
-	}
-}
-
-static void maybe_send_barrier(struct drbd_connection *connection, unsigned int epoch)
-{
-	/* re-init if first write on this connection */
-	if (!connection->send.seen_any_write_yet)
-		return;
-	if (connection->send.current_epoch_nr != epoch) {
-		if (connection->send.current_epoch_writes)
-			drbd_send_barrier(connection);
-		connection->send.current_epoch_nr = epoch;
-	}
-}
-
-int w_send_out_of_sync(struct drbd_work *w, int cancel)
-{
-	struct drbd_request *req = container_of(w, struct drbd_request, w);
-	struct drbd_device *device = req->device;
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
-	struct drbd_connection *const connection = peer_device->connection;
-	int err;
-
-	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED, peer_device);
-		return 0;
-	}
-	req->pre_send_jif = jiffies;
-
-	/* this time, no connection->send.current_epoch_writes++;
-	 * If it was sent, it was the closing barrier for the last
-	 * replicated epoch, before we went into AHEAD mode.
-	 * No more barriers will be sent, until we leave AHEAD mode again. */
-	maybe_send_barrier(connection, req->epoch);
-
-	err = drbd_send_out_of_sync(peer_device, req);
-	req_mod(req, OOS_HANDED_TO_NETWORK, peer_device);
-
-	return err;
-}
-
-/**
- * w_send_dblock() - Worker callback to send a P_DATA packet in order to mirror a write request
- * @w:		work object.
- * @cancel:	The connection will be closed anyways
- */
-int w_send_dblock(struct drbd_work *w, int cancel)
-{
-	struct drbd_request *req = container_of(w, struct drbd_request, w);
-	struct drbd_device *device = req->device;
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
-	struct drbd_connection *connection = peer_device->connection;
-	bool do_send_unplug = req->rq_state & RQ_UNPLUG;
-	int err;
-
-	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED, peer_device);
-		return 0;
-	}
-	req->pre_send_jif = jiffies;
-
-	re_init_if_first_write(connection, req->epoch);
-	maybe_send_barrier(connection, req->epoch);
-	connection->send.current_epoch_writes++;
-
-	err = drbd_send_dblock(peer_device, req);
-	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK, peer_device);
-
-	if (do_send_unplug && !err)
-		pd_send_unplug_remote(peer_device);
-
-	return err;
-}
-
-/**
- * w_send_read_req() - Worker callback to send a read request (P_DATA_REQUEST) packet
- * @w:		work object.
- * @cancel:	The connection will be closed anyways
- */
-int w_send_read_req(struct drbd_work *w, int cancel)
-{
-	struct drbd_request *req = container_of(w, struct drbd_request, w);
-	struct drbd_device *device = req->device;
-	struct drbd_peer_device *const peer_device = first_peer_device(device);
-	struct drbd_connection *connection = peer_device->connection;
-	bool do_send_unplug = req->rq_state & RQ_UNPLUG;
-	int err;
-
-	if (unlikely(cancel)) {
-		req_mod(req, SEND_CANCELED, peer_device);
-		return 0;
-	}
-	req->pre_send_jif = jiffies;
-
-	/* Even read requests may close a write epoch,
-	 * if there was any yet. */
-	maybe_send_barrier(connection, req->epoch);
-
-	err = drbd_send_drequest(peer_device, P_DATA_REQUEST, req->i.sector, req->i.size,
-				 (unsigned long)req);
-
-	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK, peer_device);
-
-	if (do_send_unplug && !err)
-		pd_send_unplug_remote(peer_device);
-
-	return err;
-}
-
-int w_restart_disk_io(struct drbd_work *w, int cancel)
-{
-	struct drbd_request *req = container_of(w, struct drbd_request, w);
-	struct drbd_device *device = req->device;
-
-	if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG)
-		drbd_al_begin_io(device, &req->i);
-
-	req->private_bio = bio_alloc_clone(device->ldev->backing_bdev,
-					   req->master_bio, GFP_NOIO,
-					  &drbd_io_bio_set);
-	req->private_bio->bi_private = req;
-	req->private_bio->bi_end_io = drbd_request_endio;
-	submit_bio_noacct(req->private_bio);
-
-	return 0;
-}
-
-static int _drbd_may_sync_now(struct drbd_device *device)
-{
-	struct drbd_device *odev = device;
-	int resync_after;
-
-	while (1) {
-		if (!odev->ldev || odev->state.disk == D_DISKLESS)
-			return 1;
-		rcu_read_lock();
-		resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after;
-		rcu_read_unlock();
-		if (resync_after == -1)
-			return 1;
-		odev = minor_to_device(resync_after);
-		if (!odev)
-			return 1;
-		if ((odev->state.conn >= C_SYNC_SOURCE &&
-		     odev->state.conn <= C_PAUSED_SYNC_T) ||
-		    odev->state.aftr_isp || odev->state.peer_isp ||
-		    odev->state.user_isp)
-			return 0;
-	}
-}
-
-/**
- * drbd_pause_after() - Pause resync on all devices that may not resync now
- * @device:	DRBD device.
- *
- * Called from process context only (admin command and after_state_ch).
- */
-static bool drbd_pause_after(struct drbd_device *device)
-{
-	bool changed = false;
-	struct drbd_device *odev;
-	int i;
-
-	rcu_read_lock();
-	idr_for_each_entry(&drbd_devices, odev, i) {
-		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
-			continue;
-		if (!_drbd_may_sync_now(odev) &&
-		    _drbd_set_state(_NS(odev, aftr_isp, 1),
-				    CS_HARD, NULL) != SS_NOTHING_TO_DO)
-			changed = true;
-	}
-	rcu_read_unlock();
-
-	return changed;
-}
-
-/**
- * drbd_resume_next() - Resume resync on all devices that may resync now
- * @device:	DRBD device.
- *
- * Called from process context only (admin command and worker).
- */
-static bool drbd_resume_next(struct drbd_device *device)
-{
-	bool changed = false;
-	struct drbd_device *odev;
-	int i;
-
-	rcu_read_lock();
-	idr_for_each_entry(&drbd_devices, odev, i) {
-		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
-			continue;
-		if (odev->state.aftr_isp) {
-			if (_drbd_may_sync_now(odev) &&
-			    _drbd_set_state(_NS(odev, aftr_isp, 0),
-					    CS_HARD, NULL) != SS_NOTHING_TO_DO)
-				changed = true;
-		}
-	}
-	rcu_read_unlock();
-	return changed;
-}
-
-void resume_next_sg(struct drbd_device *device)
-{
-	lock_all_resources();
-	drbd_resume_next(device);
-	unlock_all_resources();
-}
-
-void suspend_other_sg(struct drbd_device *device)
-{
-	lock_all_resources();
-	drbd_pause_after(device);
-	unlock_all_resources();
-}
-
-/* caller must lock_all_resources() */
-enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor)
-{
-	struct drbd_device *odev;
-	int resync_after;
-
-	if (o_minor == -1)
-		return NO_ERROR;
-	if (o_minor < -1 || o_minor > MINORMASK)
-		return ERR_RESYNC_AFTER;
-
-	/* check for loops */
-	odev = minor_to_device(o_minor);
-	while (1) {
-		if (odev == device)
-			return ERR_RESYNC_AFTER_CYCLE;
-
-		/* You are free to depend on diskless, non-existing,
-		 * or not yet/no longer existing minors.
-		 * We only reject dependency loops.
-		 * We cannot follow the dependency chain beyond a detached or
-		 * missing minor.
-		 */
-		if (!odev || !odev->ldev || odev->state.disk == D_DISKLESS)
-			return NO_ERROR;
-
-		rcu_read_lock();
-		resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after;
-		rcu_read_unlock();
-		/* dependency chain ends here, no cycles. */
-		if (resync_after == -1)
-			return NO_ERROR;
-
-		/* follow the dependency chain */
-		odev = minor_to_device(resync_after);
-	}
-}
-
-/* caller must lock_all_resources() */
-void drbd_resync_after_changed(struct drbd_device *device)
-{
-	int changed;
-
-	do {
-		changed  = drbd_pause_after(device);
-		changed |= drbd_resume_next(device);
-	} while (changed);
-}
-
-void drbd_rs_controller_reset(struct drbd_peer_device *peer_device)
-{
-	struct drbd_device *device = peer_device->device;
-	struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
-	struct fifo_buffer *plan;
-
-	atomic_set(&device->rs_sect_in, 0);
-	atomic_set(&device->rs_sect_ev, 0);
-	device->rs_in_flight = 0;
-	device->rs_last_events =
-		(int)part_stat_read_accum(disk->part0, sectors);
-
-	/* Updating the RCU protected object in place is necessary since
-	   this function gets called from atomic context.
-	   It is valid since all other updates also lead to an completely
-	   empty fifo */
-	rcu_read_lock();
-	plan = rcu_dereference(device->rs_plan_s);
-	plan->total = 0;
-	fifo_set(plan, 0);
-	rcu_read_unlock();
-}
-
-void start_resync_timer_fn(struct timer_list *t)
-{
-	struct drbd_device *device = timer_container_of(device, t,
-							start_resync_timer);
-	drbd_device_post_work(device, RS_START);
-}
-
-static void do_start_resync(struct drbd_device *device)
-{
-	if (atomic_read(&device->unacked_cnt) || atomic_read(&device->rs_pending_cnt)) {
-		drbd_warn(device, "postponing start_resync ...\n");
-		device->start_resync_timer.expires = jiffies + HZ/10;
-		add_timer(&device->start_resync_timer);
-		return;
-	}
-
-	drbd_start_resync(device, C_SYNC_SOURCE);
-	clear_bit(AHEAD_TO_SYNC_SOURCE, &device->flags);
-}
-
-static bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device)
-{
-	bool csums_after_crash_only;
-	rcu_read_lock();
-	csums_after_crash_only = rcu_dereference(connection->net_conf)->csums_after_crash_only;
-	rcu_read_unlock();
-	return connection->agreed_pro_version >= 89 &&		/* supported? */
-		connection->csums_tfm &&			/* configured? */
-		(csums_after_crash_only == false		/* use for each resync? */
-		 || test_bit(CRASHED_PRIMARY, &device->flags));	/* or only after Primary crash? */
-}
-
-/**
- * drbd_start_resync() - Start the resync process
- * @device:	DRBD device.
- * @side:	Either C_SYNC_SOURCE or C_SYNC_TARGET
- *
- * This function might bring you directly into one of the
- * C_PAUSED_SYNC_* states.
- */
-void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
-{
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
-	union drbd_state ns;
-	int r;
-
-	if (device->state.conn >= C_SYNC_SOURCE && device->state.conn < C_AHEAD) {
-		drbd_err(device, "Resync already running!\n");
-		return;
-	}
-
-	if (!connection) {
-		drbd_err(device, "No connection to peer, aborting!\n");
-		return;
-	}
-
-	if (!test_bit(B_RS_H_DONE, &device->flags)) {
-		if (side == C_SYNC_TARGET) {
-			/* Since application IO was locked out during C_WF_BITMAP_T and
-			   C_WF_SYNC_UUID we are still unmodified. Before going to C_SYNC_TARGET
-			   we check that we might make the data inconsistent. */
-			r = drbd_khelper(device, "before-resync-target");
-			r = (r >> 8) & 0xff;
-			if (r > 0) {
-				drbd_info(device, "before-resync-target handler returned %d, "
-					 "dropping connection.\n", r);
-				conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD);
-				return;
-			}
-		} else /* C_SYNC_SOURCE */ {
-			r = drbd_khelper(device, "before-resync-source");
-			r = (r >> 8) & 0xff;
-			if (r > 0) {
-				if (r == 3) {
-					drbd_info(device, "before-resync-source handler returned %d, "
-						 "ignoring. Old userland tools?", r);
-				} else {
-					drbd_info(device, "before-resync-source handler returned %d, "
-						 "dropping connection.\n", r);
-					conn_request_state(connection,
-							   NS(conn, C_DISCONNECTING), CS_HARD);
-					return;
-				}
-			}
-		}
-	}
-
-	if (current == connection->worker.task) {
-		/* The worker should not sleep waiting for state_mutex,
-		   that can take long */
-		if (!mutex_trylock(device->state_mutex)) {
-			set_bit(B_RS_H_DONE, &device->flags);
-			device->start_resync_timer.expires = jiffies + HZ/5;
-			add_timer(&device->start_resync_timer);
-			return;
-		}
-	} else {
-		mutex_lock(device->state_mutex);
-	}
-
-	lock_all_resources();
-	clear_bit(B_RS_H_DONE, &device->flags);
-	/* Did some connection breakage or IO error race with us? */
-	if (device->state.conn < C_CONNECTED
-	|| !get_ldev_if_state(device, D_NEGOTIATING)) {
-		unlock_all_resources();
-		goto out;
-	}
-
-	ns = drbd_read_state(device);
-
-	ns.aftr_isp = !_drbd_may_sync_now(device);
-
-	ns.conn = side;
-
-	if (side == C_SYNC_TARGET)
-		ns.disk = D_INCONSISTENT;
-	else /* side == C_SYNC_SOURCE */
-		ns.pdsk = D_INCONSISTENT;
-
-	r = _drbd_set_state(device, ns, CS_VERBOSE, NULL);
-	ns = drbd_read_state(device);
-
-	if (ns.conn < C_CONNECTED)
-		r = SS_UNKNOWN_ERROR;
-
-	if (r == SS_SUCCESS) {
-		unsigned long tw = drbd_bm_total_weight(device);
-		unsigned long now = jiffies;
-		int i;
-
-		device->rs_failed    = 0;
-		device->rs_paused    = 0;
-		device->rs_same_csum = 0;
-		device->rs_last_sect_ev = 0;
-		device->rs_total     = tw;
-		device->rs_start     = now;
-		for (i = 0; i < DRBD_SYNC_MARKS; i++) {
-			device->rs_mark_left[i] = tw;
-			device->rs_mark_time[i] = now;
-		}
-		drbd_pause_after(device);
-		/* Forget potentially stale cached per resync extent bit-counts.
-		 * Open coded drbd_rs_cancel_all(device), we already have IRQs
-		 * disabled, and know the disk state is ok. */
-		spin_lock(&device->al_lock);
-		lc_reset(device->resync);
-		device->resync_locked = 0;
-		device->resync_wenr = LC_FREE;
-		spin_unlock(&device->al_lock);
-	}
-	unlock_all_resources();
-
-	if (r == SS_SUCCESS) {
-		wake_up(&device->al_wait); /* for lc_reset() above */
-		/* reset rs_last_bcast when a resync or verify is started,
-		 * to deal with potential jiffies wrap. */
-		device->rs_last_bcast = jiffies - HZ;
-
-		drbd_info(device, "Began resync as %s (will sync %lu KB [%lu bits set]).\n",
-		     drbd_conn_str(ns.conn),
-		     (unsigned long) device->rs_total << (BM_BLOCK_SHIFT-10),
-		     (unsigned long) device->rs_total);
-		if (side == C_SYNC_TARGET) {
-			device->bm_resync_fo = 0;
-			device->use_csums = use_checksum_based_resync(connection, device);
-		} else {
-			device->use_csums = false;
-		}
-
-		/* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid
-		 * with w_send_oos, or the sync target will get confused as to
-		 * how much bits to resync.  We cannot do that always, because for an
-		 * empty resync and protocol < 95, we need to do it here, as we call
-		 * drbd_resync_finished from here in that case.
-		 * We drbd_gen_and_send_sync_uuid here for protocol < 96,
-		 * and from after_state_ch otherwise. */
-		if (side == C_SYNC_SOURCE && connection->agreed_pro_version < 96)
-			drbd_gen_and_send_sync_uuid(peer_device);
-
-		if (connection->agreed_pro_version < 95 && device->rs_total == 0) {
-			/* This still has a race (about when exactly the peers
-			 * detect connection loss) that can lead to a full sync
-			 * on next handshake. In 8.3.9 we fixed this with explicit
-			 * resync-finished notifications, but the fix
-			 * introduces a protocol change.  Sleeping for some
-			 * time longer than the ping interval + timeout on the
-			 * SyncSource, to give the SyncTarget the chance to
-			 * detect connection loss, then waiting for a ping
-			 * response (implicit in drbd_resync_finished) reduces
-			 * the race considerably, but does not solve it. */
-			if (side == C_SYNC_SOURCE) {
-				struct net_conf *nc;
-				int timeo;
-
-				rcu_read_lock();
-				nc = rcu_dereference(connection->net_conf);
-				timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9;
-				rcu_read_unlock();
-				schedule_timeout_interruptible(timeo);
-			}
-			drbd_resync_finished(peer_device);
-		}
-
-		drbd_rs_controller_reset(peer_device);
-		/* ns.conn may already be != device->state.conn,
-		 * we may have been paused in between, or become paused until
-		 * the timer triggers.
-		 * No matter, that is handled in resync_timer_fn() */
-		if (ns.conn == C_SYNC_TARGET)
-			mod_timer(&device->resync_timer, jiffies);
-
-		drbd_md_sync(device);
-	}
-	put_ldev(device);
-out:
-	mutex_unlock(device->state_mutex);
-}
-
-static void update_on_disk_bitmap(struct drbd_peer_device *peer_device, bool resync_done)
-{
-	struct drbd_device *device = peer_device->device;
-	struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, };
-	device->rs_last_bcast = jiffies;
-
-	if (!get_ldev(device))
-		return;
-
-	drbd_bm_write_lazy(device, 0);
-	if (resync_done && is_sync_state(device->state.conn))
-		drbd_resync_finished(peer_device);
-
-	drbd_bcast_event(device, &sib);
-	/* update timestamp, in case it took a while to write out stuff */
-	device->rs_last_bcast = jiffies;
-	put_ldev(device);
-}
-
-static void drbd_ldev_destroy(struct drbd_device *device)
-{
-	lc_destroy(device->resync);
-	device->resync = NULL;
-	lc_destroy(device->act_log);
-	device->act_log = NULL;
-
-	__acquire(local);
-	drbd_backing_dev_free(device, device->ldev);
-	device->ldev = NULL;
-	__release(local);
-
-	clear_bit(GOING_DISKLESS, &device->flags);
-	wake_up(&device->misc_wait);
-}
-
-static void go_diskless(struct drbd_device *device)
-{
-	struct drbd_peer_device *peer_device = first_peer_device(device);
-	D_ASSERT(device, device->state.disk == D_FAILED);
-	/* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
-	 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
-	 * the protected members anymore, though, so once put_ldev reaches zero
-	 * again, it will be safe to free them. */
-
-	/* Try to write changed bitmap pages, read errors may have just
-	 * set some bits outside the area covered by the activity log.
-	 *
-	 * If we have an IO error during the bitmap writeout,
-	 * we will want a full sync next time, just in case.
-	 * (Do we want a specific meta data flag for this?)
-	 *
-	 * If that does not make it to stable storage either,
-	 * we cannot do anything about that anymore.
-	 *
-	 * We still need to check if both bitmap and ldev are present, we may
-	 * end up here after a failed attach, before ldev was even assigned.
-	 */
-	if (device->bitmap && device->ldev) {
-		/* An interrupted resync or similar is allowed to recounts bits
-		 * while we detach.
-		 * Any modifications would not be expected anymore, though.
-		 */
-		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
-					"detach", BM_LOCKED_TEST_ALLOWED, peer_device)) {
-			if (test_bit(WAS_READ_ERROR, &device->flags)) {
-				drbd_md_set_flag(device, MDF_FULL_SYNC);
-				drbd_md_sync(device);
-			}
-		}
-	}
-
-	drbd_force_state(device, NS(disk, D_DISKLESS));
-}
-
-static int do_md_sync(struct drbd_device *device)
-{
-	drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
-	drbd_md_sync(device);
-	return 0;
-}
-
-/* only called from drbd_worker thread, no locking */
-void __update_timing_details(
-		struct drbd_thread_timing_details *tdp,
-		unsigned int *cb_nr,
-		void *cb,
-		const char *fn, const unsigned int line)
-{
-	unsigned int i = *cb_nr % DRBD_THREAD_DETAILS_HIST;
-	struct drbd_thread_timing_details *td = tdp + i;
-
-	td->start_jif = jiffies;
-	td->cb_addr = cb;
-	td->caller_fn = fn;
-	td->line = line;
-	td->cb_nr = *cb_nr;
-
-	i = (i+1) % DRBD_THREAD_DETAILS_HIST;
-	td = tdp + i;
-	memset(td, 0, sizeof(*td));
-
-	++(*cb_nr);
-}
-
-static void do_device_work(struct drbd_device *device, const unsigned long todo)
-{
-	if (test_bit(MD_SYNC, &todo))
-		do_md_sync(device);
-	if (test_bit(RS_DONE, &todo) ||
-	    test_bit(RS_PROGRESS, &todo))
-		update_on_disk_bitmap(first_peer_device(device), test_bit(RS_DONE, &todo));
-	if (test_bit(GO_DISKLESS, &todo))
-		go_diskless(device);
-	if (test_bit(DESTROY_DISK, &todo))
-		drbd_ldev_destroy(device);
-	if (test_bit(RS_START, &todo))
-		do_start_resync(device);
-}
-
-#define DRBD_DEVICE_WORK_MASK	\
-	((1UL << GO_DISKLESS)	\
-	|(1UL << DESTROY_DISK)	\
-	|(1UL << MD_SYNC)	\
-	|(1UL << RS_START)	\
-	|(1UL << RS_PROGRESS)	\
-	|(1UL << RS_DONE)	\
-	)
-
-static unsigned long get_work_bits(unsigned long *flags)
-{
-	unsigned long old, new;
-	do {
-		old = *flags;
-		new = old & ~DRBD_DEVICE_WORK_MASK;
-	} while (cmpxchg(flags, old, new) != old);
-	return old & DRBD_DEVICE_WORK_MASK;
-}
-
-static void do_unqueued_work(struct drbd_connection *connection)
-{
-	struct drbd_peer_device *peer_device;
-	int vnr;
-
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		unsigned long todo = get_work_bits(&device->flags);
-		if (!todo)
-			continue;
-
-		kref_get(&device->kref);
-		rcu_read_unlock();
-		do_device_work(device, todo);
-		kref_put(&device->kref, drbd_destroy_device);
-		rcu_read_lock();
-	}
-	rcu_read_unlock();
-}
-
-static bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list)
-{
-	spin_lock_irq(&queue->q_lock);
-	list_splice_tail_init(&queue->q, work_list);
-	spin_unlock_irq(&queue->q_lock);
-	return !list_empty(work_list);
-}
-
-static void wait_for_work(struct drbd_connection *connection, struct list_head *work_list)
-{
-	DEFINE_WAIT(wait);
-	struct net_conf *nc;
-	int uncork, cork;
-
-	dequeue_work_batch(&connection->sender_work, work_list);
-	if (!list_empty(work_list))
-		return;
-
-	/* Still nothing to do?
-	 * Maybe we still need to close the current epoch,
-	 * even if no new requests are queued yet.
-	 *
-	 * Also, poke TCP, just in case.
-	 * Then wait for new work (or signal). */
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	uncork = nc ? nc->tcp_cork : 0;
-	rcu_read_unlock();
-	if (uncork) {
-		mutex_lock(&connection->data.mutex);
-		if (connection->data.socket)
-			tcp_sock_set_cork(connection->data.socket->sk, false);
-		mutex_unlock(&connection->data.mutex);
-	}
-
-	for (;;) {
-		int send_barrier;
-		prepare_to_wait(&connection->sender_work.q_wait, &wait, TASK_INTERRUPTIBLE);
-		spin_lock_irq(&connection->resource->req_lock);
-		spin_lock(&connection->sender_work.q_lock);	/* FIXME get rid of this one? */
-		if (!list_empty(&connection->sender_work.q))
-			list_splice_tail_init(&connection->sender_work.q, work_list);
-		spin_unlock(&connection->sender_work.q_lock);	/* FIXME get rid of this one? */
-		if (!list_empty(work_list) || signal_pending(current)) {
-			spin_unlock_irq(&connection->resource->req_lock);
-			break;
-		}
-
-		/* We found nothing new to do, no to-be-communicated request,
-		 * no other work item.  We may still need to close the last
-		 * epoch.  Next incoming request epoch will be connection ->
-		 * current transfer log epoch number.  If that is different
-		 * from the epoch of the last request we communicated, it is
-		 * safe to send the epoch separating barrier now.
-		 */
-		send_barrier =
-			atomic_read(&connection->current_tle_nr) !=
-			connection->send.current_epoch_nr;
-		spin_unlock_irq(&connection->resource->req_lock);
-
-		if (send_barrier)
-			maybe_send_barrier(connection,
-					connection->send.current_epoch_nr + 1);
-
-		if (test_bit(DEVICE_WORK_PENDING, &connection->flags))
-			break;
-
-		/* drbd_send() may have called flush_signals() */
-		if (get_t_state(&connection->worker) != RUNNING)
-			break;
-
-		schedule();
-		/* may be woken up for other things but new work, too,
-		 * e.g. if the current epoch got closed.
-		 * In which case we send the barrier above. */
-	}
-	finish_wait(&connection->sender_work.q_wait, &wait);
-
-	/* someone may have changed the config while we have been waiting above. */
-	rcu_read_lock();
-	nc = rcu_dereference(connection->net_conf);
-	cork = nc ? nc->tcp_cork : 0;
-	rcu_read_unlock();
-	mutex_lock(&connection->data.mutex);
-	if (connection->data.socket) {
-		if (cork)
-			tcp_sock_set_cork(connection->data.socket->sk, true);
-		else if (!uncork)
-			tcp_sock_set_cork(connection->data.socket->sk, false);
-	}
-	mutex_unlock(&connection->data.mutex);
-}
-
-int drbd_worker(struct drbd_thread *thi)
-{
-	struct drbd_connection *connection = thi->connection;
-	struct drbd_work *w = NULL;
-	struct drbd_peer_device *peer_device;
-	LIST_HEAD(work_list);
-	int vnr;
-
-	while (get_t_state(thi) == RUNNING) {
-		drbd_thread_current_set_cpu(thi);
-
-		if (list_empty(&work_list)) {
-			update_worker_timing_details(connection, wait_for_work);
-			wait_for_work(connection, &work_list);
-		}
-
-		if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) {
-			update_worker_timing_details(connection, do_unqueued_work);
-			do_unqueued_work(connection);
-		}
-
-		if (signal_pending(current)) {
-			flush_signals(current);
-			if (get_t_state(thi) == RUNNING) {
-				drbd_warn(connection, "Worker got an unexpected signal\n");
-				continue;
-			}
-			break;
-		}
-
-		if (get_t_state(thi) != RUNNING)
-			break;
-
-		if (!list_empty(&work_list)) {
-			w = list_first_entry(&work_list, struct drbd_work, list);
-			list_del_init(&w->list);
-			update_worker_timing_details(connection, w->cb);
-			if (w->cb(w, connection->cstate < C_WF_REPORT_PARAMS) == 0)
-				continue;
-			if (connection->cstate >= C_WF_REPORT_PARAMS)
-				conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD);
-		}
-	}
-
-	do {
-		if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) {
-			update_worker_timing_details(connection, do_unqueued_work);
-			do_unqueued_work(connection);
-		}
-		if (!list_empty(&work_list)) {
-			w = list_first_entry(&work_list, struct drbd_work, list);
-			list_del_init(&w->list);
-			update_worker_timing_details(connection, w->cb);
-			w->cb(w, 1);
-		} else
-			dequeue_work_batch(&connection->sender_work, &work_list);
-	} while (!list_empty(&work_list) || test_bit(DEVICE_WORK_PENDING, &connection->flags));
-
-	rcu_read_lock();
-	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
-		struct drbd_device *device = peer_device->device;
-		D_ASSERT(device, device->state.disk == D_DISKLESS && device->state.conn == C_STANDALONE);
-		kref_get(&device->kref);
-		rcu_read_unlock();
-		drbd_device_cleanup(device);
-		kref_put(&device->kref, drbd_destroy_device);
-		rcu_read_lock();
-	}
-	rcu_read_unlock();
-
-	return 0;
-}
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 63aeb7a76a8c9..9e1ebb8e80da6 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -808,7 +808,7 @@ static void ublk_dev_param_basic_apply(struct ublk_device *ub)
 
 static int ublk_integrity_flags(u32 flags)
 {
-	int ret_flags = 0;
+	int ret_flags = BLK_SPLIT_INTERVAL_CAPABLE;
 
 	if (flags & LBMD_PI_CAP_INTEGRITY) {
 		flags &= ~LBMD_PI_CAP_INTEGRITY;
diff --git a/drivers/block/zram/backend_lz4.c b/drivers/block/zram/backend_lz4.c
index 04e1866147608..c449d511ba86f 100644
--- a/drivers/block/zram/backend_lz4.c
+++ b/drivers/block/zram/backend_lz4.c
@@ -14,13 +14,38 @@ struct lz4_ctx {
 
 static void lz4_release_params(struct zcomp_params *params)
 {
+	LZ4_stream_t *dict_stream = params->drv_data;
+
+	params->drv_data = NULL;
+	if (!dict_stream)
+		return;
+
+	kfree(dict_stream);
 }
 
 static int lz4_setup_params(struct zcomp_params *params)
 {
+	LZ4_stream_t *dict_stream;
+	int ret;
+
 	if (params->level == ZCOMP_PARAM_NOT_SET)
 		params->level = LZ4_ACCELERATION_DEFAULT;
 
+	if (!params->dict || !params->dict_sz)
+		return 0;
+
+	dict_stream = kzalloc_obj(*dict_stream, GFP_KERNEL);
+	if (!dict_stream)
+		return -ENOMEM;
+
+	ret = LZ4_loadDict(dict_stream,
+			   params->dict, params->dict_sz);
+	if (ret != params->dict_sz) {
+		kfree(dict_stream);
+		return -EINVAL;
+	}
+	params->drv_data = dict_stream;
+
 	return 0;
 }
 
@@ -79,9 +104,7 @@ static int lz4_compress(struct zcomp_params *params, struct zcomp_ctx *ctx,
 					zctx->mem);
 	} else {
 		/* Cstrm needs to be reset */
-		ret = LZ4_loadDict(zctx->cstrm, params->dict, params->dict_sz);
-		if (ret != params->dict_sz)
-			return -EINVAL;
+		memcpy(zctx->cstrm, params->drv_data, sizeof(*zctx->cstrm));
 		ret = LZ4_compress_fast_continue(zctx->cstrm, req->src,
 						 req->dst, req->src_len,
 						 req->dst_len, params->level);
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
index a771a8ecc540b..974c4691887e6 100644
--- a/drivers/block/zram/zcomp.c
+++ b/drivers/block/zram/zcomp.c
@@ -84,9 +84,14 @@ static const struct zcomp_ops *lookup_backend_ops(const char *comp)
 	return backends[i];
 }
 
-bool zcomp_available_algorithm(const char *comp)
+const char *zcomp_lookup_backend_name(const char *comp)
 {
-	return lookup_backend_ops(comp) != NULL;
+	const struct zcomp_ops *backend = lookup_backend_ops(comp);
+
+	if (backend)
+		return backend->name;
+
+	return NULL;
 }
 
 /* show available compressors */
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h
index eacfd3f7d61d9..81a0f3f6ff487 100644
--- a/drivers/block/zram/zcomp.h
+++ b/drivers/block/zram/zcomp.h
@@ -80,7 +80,7 @@ struct zcomp {
 int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node);
 int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node);
 ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at);
-bool zcomp_available_algorithm(const char *comp);
+const char *zcomp_lookup_backend_name(const char *comp);
 
 struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params);
 void zcomp_destroy(struct zcomp *comp);
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index a324ede6206d3..c2afd1c34f4a6 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -917,9 +917,8 @@ static void zram_account_writeback_submit(struct zram *zram)
 
 static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
 {
-	u32 size, index = req->pps->index;
-	int err, prio;
-	bool huge;
+	u32 index = req->pps->index;
+	int err;
 
 	err = blk_status_to_errno(req->bio.bi_status);
 	if (err) {
@@ -946,28 +945,13 @@ static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
 		goto out;
 	}
 
-	if (zram->compressed_wb) {
-		/*
-		 * ZRAM_WB slots get freed, we need to preserve data required
-		 * for read decompression.
-		 */
-		size = get_slot_size(zram, index);
-		prio = get_slot_comp_priority(zram, index);
-		huge = test_slot_flag(zram, index, ZRAM_HUGE);
-	}
-
-	slot_free(zram, index);
-	set_slot_flag(zram, index, ZRAM_WB);
+	clear_slot_flag(zram, index, ZRAM_IDLE);
+	if (test_slot_flag(zram, index, ZRAM_HUGE))
+		atomic64_dec(&zram->stats.huge_pages);
+	atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size);
+	zs_free(zram->mem_pool, get_slot_handle(zram, index));
 	set_slot_handle(zram, index, req->blk_idx);
-
-	if (zram->compressed_wb) {
-		if (huge)
-			set_slot_flag(zram, index, ZRAM_HUGE);
-		set_slot_size(zram, index, size);
-		set_slot_comp_priority(zram, index, prio);
-	}
-
-	atomic64_inc(&zram->stats.pages_stored);
+	set_slot_flag(zram, index, ZRAM_WB);
 
 out:
 	slot_unlock(zram, index);
@@ -1212,9 +1196,9 @@ static int parse_mode(char *val, u32 *mode)
 	return 0;
 }
 
-static int scan_slots_for_writeback(struct zram *zram, u32 mode,
-				    unsigned long lo, unsigned long hi,
-				    struct zram_pp_ctl *ctl)
+static void scan_slots_for_writeback(struct zram *zram, u32 mode,
+				     unsigned long lo, unsigned long hi,
+				     struct zram_pp_ctl *ctl)
 {
 	u32 index = lo;
 
@@ -1246,8 +1230,6 @@ static int scan_slots_for_writeback(struct zram *zram, u32 mode,
 			break;
 		index++;
 	}
-
-	return 0;
 }
 
 static ssize_t writeback_store(struct device *dev,
@@ -1445,21 +1427,21 @@ static void zram_async_read_endio(struct bio *bio)
 	queue_work(system_highpri_wq, &req->work);
 }
 
-static void read_from_bdev_async(struct zram *zram, struct page *page,
-				 u32 index, unsigned long blk_idx,
-				 struct bio *parent)
+static int read_from_bdev_async(struct zram *zram, struct page *page,
+				u32 index, unsigned long blk_idx,
+				struct bio *parent)
 {
 	struct zram_rb_req *req;
 	struct bio *bio;
 
 	req = kmalloc_obj(*req, GFP_NOIO);
 	if (!req)
-		return;
+		return -ENOMEM;
 
 	bio = bio_alloc(zram->bdev, 1, parent->bi_opf, GFP_NOIO);
 	if (!bio) {
 		kfree(req);
-		return;
+		return -ENOMEM;
 	}
 
 	req->zram = zram;
@@ -1475,6 +1457,8 @@ static void read_from_bdev_async(struct zram *zram, struct page *page,
 	__bio_add_page(bio, page, PAGE_SIZE, 0);
 	bio_inc_remaining(parent);
 	submit_bio(bio);
+
+	return 0;
 }
 
 static void zram_sync_read(struct work_struct *w)
@@ -1523,8 +1507,7 @@ static int read_from_bdev(struct zram *zram, struct page *page, u32 index,
 			return -EIO;
 		return read_from_bdev_sync(zram, page, index, blk_idx);
 	}
-	read_from_bdev_async(zram, page, index, blk_idx, parent);
-	return 0;
+	return read_from_bdev_async(zram, page, index, blk_idx, parent);
 }
 #else
 static inline void reset_bdev(struct zram *zram) {};
@@ -1635,45 +1618,62 @@ static void zram_debugfs_register(struct zram *zram) {};
 static void zram_debugfs_unregister(struct zram *zram) {};
 #endif
 
-static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg)
+/* Only algo parameter given, lookup by algo name */
+static int lookup_algo_priority(struct zram *zram, const char *algo,
+				u32 min_prio)
 {
-	/* Do not free statically defined compression algorithms */
-	if (zram->comp_algs[prio] != default_compressor)
-		kfree(zram->comp_algs[prio]);
+	s32 prio;
+
+	for (prio = min_prio; prio < ZRAM_MAX_COMPS; prio++) {
+		if (!zram->comp_algs[prio])
+			continue;
 
+		if (!strcmp(zram->comp_algs[prio], algo))
+			return prio;
+	}
+
+	return -EINVAL;
+}
+
+/* Both algo and priority parameters given, validate them */
+static int validate_algo_priority(struct zram *zram, const char *algo, u32 prio)
+{
+	if (prio >= ZRAM_MAX_COMPS)
+		return -EINVAL;
+	/* No algo at given priority */
+	if (!zram->comp_algs[prio])
+		return -EINVAL;
+	/* A different algo at given priority */
+	if (strcmp(zram->comp_algs[prio], algo))
+		return -EINVAL;
+	return 0;
+}
+
+static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg)
+{
 	zram->comp_algs[prio] = alg;
 }
 
 static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf)
 {
-	char *compressor;
+	const char *alg;
 	size_t sz;
 
 	sz = strlen(buf);
 	if (sz >= ZRAM_MAX_ALGO_NAME_SZ)
 		return -E2BIG;
 
-	compressor = kstrdup(buf, GFP_KERNEL);
-	if (!compressor)
-		return -ENOMEM;
-
-	/* ignore trailing newline */
-	if (sz > 0 && compressor[sz - 1] == '\n')
-		compressor[sz - 1] = 0x00;
-
-	if (!zcomp_available_algorithm(compressor)) {
-		kfree(compressor);
+	alg = zcomp_lookup_backend_name(buf);
+	if (!alg)
 		return -EINVAL;
-	}
 
 	guard(rwsem_write)(&zram->dev_lock);
 	if (init_done(zram)) {
-		kfree(compressor);
 		pr_info("Can't change algorithm for initialized device\n");
 		return -EBUSY;
 	}
 
-	comp_algorithm_set(zram, prio, compressor);
+	comp_algorithm_set(zram, prio, alg);
 	return 0;
 }
 
@@ -1721,6 +1721,7 @@ static ssize_t algorithm_params_store(struct device *dev,
 	char *args, *param, *val, *algo = NULL, *dict_path = NULL;
 	struct deflate_params deflate_params;
 	struct zram *zram = dev_to_zram(dev);
+	bool prio_param = false;
 	int ret;
 
 	deflate_params.winbits = ZCOMP_PARAM_NOT_SET;
@@ -1733,6 +1734,7 @@ static ssize_t algorithm_params_store(struct device *dev,
 			return -EINVAL;
 
 		if (!strcmp(param, "priority")) {
+			prio_param = true;
 			ret = kstrtoint(val, 10, &prio);
 			if (ret)
 				return ret;
@@ -1764,24 +1766,26 @@ static ssize_t algorithm_params_store(struct device *dev,
 		}
 	}
 
-	/* Lookup priority by algorithm name */
-	if (algo) {
-		s32 p;
+	guard(rwsem_write)(&zram->dev_lock);
+	if (init_done(zram))
+		return -EBUSY;
 
-		prio = -EINVAL;
-		for (p = ZRAM_PRIMARY_COMP; p < ZRAM_MAX_COMPS; p++) {
-			if (!zram->comp_algs[p])
-				continue;
+	if (prio_param) {
+		if (prio < ZRAM_PRIMARY_COMP || prio >= ZRAM_MAX_COMPS)
+			return -EINVAL;
+	}
 
-			if (!strcmp(zram->comp_algs[p], algo)) {
-				prio = p;
-				break;
-			}
-		}
+	if (algo && prio_param) {
+		ret = validate_algo_priority(zram, algo, prio);
+		if (ret)
+			return ret;
 	}
 
-	if (prio < ZRAM_PRIMARY_COMP || prio >= ZRAM_MAX_COMPS)
-		return -EINVAL;
+	if (algo && !prio_param) {
+		prio = lookup_algo_priority(zram, algo, ZRAM_PRIMARY_COMP);
+		if (prio < 0)
+			return -EINVAL;
+	}
 
 	ret = comp_params_store(zram, prio, level, dict_path, &deflate_params);
 	return ret ? ret : len;
@@ -2010,8 +2014,13 @@ static void slot_free(struct zram *zram, u32 index)
 	set_slot_comp_priority(zram, index, 0);
 
 	if (test_slot_flag(zram, index, ZRAM_HUGE)) {
+		/*
+		 * Writeback completion decrements ->huge_pages but keeps
+		 * ZRAM_HUGE flag for deferred decompression path.
+		 */
+		if (!test_slot_flag(zram, index, ZRAM_WB))
+			atomic64_dec(&zram->stats.huge_pages);
 		clear_slot_flag(zram, index, ZRAM_HUGE);
-		atomic64_dec(&zram->stats.huge_pages);
 	}
 
 	if (test_slot_flag(zram, index, ZRAM_WB)) {
@@ -2345,8 +2354,20 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 #define RECOMPRESS_IDLE		(1 << 0)
 #define RECOMPRESS_HUGE		(1 << 1)
 
-static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,
-				     struct zram_pp_ctl *ctl)
+static bool highest_priority_algorithm(struct zram *zram, u32 prio)
+{
+	u32 p;
+
+	for (p = prio + 1; p < ZRAM_MAX_COMPS; p++) {
+		if (zram->comp_algs[p])
+			return false;
+	}
+
+	return true;
+}
+
+static void scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio,
+				      struct zram_pp_ctl *ctl)
 {
 	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
 	unsigned long index;
@@ -2371,8 +2392,8 @@ static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,
 		    test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE))
 			goto next;
 
-		/* Already compressed with same of higher priority */
-		if (get_slot_comp_priority(zram, index) + 1 >= prio_max)
+		/* Already compressed with same or higher priority */
+		if (get_slot_comp_priority(zram, index) >= prio)
 			goto next;
 
 		ok = place_pp_slot(zram, ctl, index);
@@ -2381,8 +2402,6 @@ static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,
 		if (!ok)
 			break;
 	}
-
-	return 0;
 }
 
 /*
@@ -2393,8 +2412,7 @@ static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,
  * Corresponding ZRAM slot should be locked.
  */
 static int recompress_slot(struct zram *zram, u32 index, struct page *page,
-			   u64 *num_recomp_pages, u32 threshold, u32 prio,
-			   u32 prio_max)
+			   u64 *num_recomp_pages, u32 threshold, u32 prio)
 {
 	struct zcomp_strm *zstrm = NULL;
 	unsigned long handle_old;
@@ -2428,51 +2446,10 @@ static int recompress_slot(struct zram *zram, u32 index, struct page *page,
 	 */
 	clear_slot_flag(zram, index, ZRAM_IDLE);
 
-	class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old);
-
-	prio = max(prio, get_slot_comp_priority(zram, index) + 1);
-	/*
-	 * Recompression slots scan should not select slots that are
-	 * already compressed with a higher priority algorithm, but
-	 * just in case
-	 */
-	if (prio >= prio_max)
-		return 0;
-
-	/*
-	 * Iterate the secondary comp algorithms list (in order of priority)
-	 * and try to recompress the page.
-	 */
-	for (; prio < prio_max; prio++) {
-		if (!zram->comps[prio])
-			continue;
-
-		zstrm = zcomp_stream_get(zram->comps[prio]);
-		src = kmap_local_page(page);
-		ret = zcomp_compress(zram->comps[prio], zstrm,
-				     src, &comp_len_new);
-		kunmap_local(src);
-
-		if (ret) {
-			zcomp_stream_put(zstrm);
-			zstrm = NULL;
-			break;
-		}
-
-		class_index_new = zs_lookup_class_index(zram->mem_pool,
-							comp_len_new);
-
-		/* Continue until we make progress */
-		if (class_index_new >= class_index_old ||
-		    (threshold && comp_len_new >= threshold)) {
-			zcomp_stream_put(zstrm);
-			zstrm = NULL;
-			continue;
-		}
-
-		/* Recompression was successful so break out */
-		break;
-	}
+	zstrm = zcomp_stream_get(zram->comps[prio]);
+	src = kmap_local_page(page);
+	ret = zcomp_compress(zram->comps[prio], zstrm, src, &comp_len_new);
+	kunmap_local(src);
 
 	/*
 	 * Decrement the limit (if set) on pages we can recompress, even
@@ -2483,21 +2460,27 @@ static int recompress_slot(struct zram *zram, u32 index, struct page *page,
 	if (*num_recomp_pages)
 		*num_recomp_pages -= 1;
 
-	/* Compression error */
-	if (ret)
+	if (ret) {
+		zcomp_stream_put(zstrm);
 		return ret;
+	}
+
+	class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old);
+	class_index_new = zs_lookup_class_index(zram->mem_pool, comp_len_new);
+
+	if (class_index_new >= class_index_old ||
+	    (threshold && comp_len_new >= threshold)) {
+		zcomp_stream_put(zstrm);
 
-	if (!zstrm) {
 		/*
 		 * Secondary algorithms failed to re-compress the page
 		 * in a way that would save memory.
 		 *
-		 * Mark the object incompressible if the max-priority
-		 * algorithm couldn't re-compress it.
+		 * Mark the object incompressible if the max-priority (the
+		 * last configured one) algorithm couldn't re-compress it.
 		 */
-		if (prio < zram->num_active_comps)
-			return 0;
-		set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);
+		if (highest_priority_algorithm(zram, prio))
+			set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);
 		return 0;
 	}
 
@@ -2542,15 +2525,13 @@ static ssize_t recompress_store(struct device *dev,
 	char *args, *param, *val, *algo = NULL;
 	u64 num_recomp_pages = ULLONG_MAX;
 	struct zram_pp_ctl *ctl = NULL;
-	struct zram_pp_slot *pps;
+	s32 prio = ZRAM_SECONDARY_COMP;
 	u32 mode = 0, threshold = 0;
-	u32 prio, prio_max;
+	struct zram_pp_slot *pps;
 	struct page *page = NULL;
+	bool prio_param = false;
 	ssize_t ret;
 
-	prio = ZRAM_SECONDARY_COMP;
-	prio_max = zram->num_active_comps;
-
 	args = skip_spaces(buf);
 	while (*args) {
 		args = next_arg(args, &param, &val);
@@ -2596,14 +2577,10 @@ static ssize_t recompress_store(struct device *dev,
 		}
 
 		if (!strcmp(param, "priority")) {
-			ret = kstrtouint(val, 10, &prio);
+			prio_param = true;
+			ret = kstrtoint(val, 10, &prio);
 			if (ret)
 				return ret;
-
-			if (prio == ZRAM_PRIMARY_COMP)
-				prio = ZRAM_SECONDARY_COMP;
-
-			prio_max = prio + 1;
 			continue;
 		}
 	}
@@ -2615,32 +2592,26 @@ static ssize_t recompress_store(struct device *dev,
 	if (!init_done(zram))
 		return -EINVAL;
 
-	if (algo) {
-		bool found = false;
-
-		for (; prio < ZRAM_MAX_COMPS; prio++) {
-			if (!zram->comp_algs[prio])
-				continue;
-
-			if (!strcmp(zram->comp_algs[prio], algo)) {
-				prio_max = prio + 1;
-				found = true;
-				break;
-			}
-		}
+	if (prio_param) {
+		if (prio < ZRAM_SECONDARY_COMP || prio >= ZRAM_MAX_COMPS)
+			return -EINVAL;
+	}
 
-		if (!found) {
-			ret = -EINVAL;
-			goto out;
-		}
+	if (algo && prio_param) {
+		ret = validate_algo_priority(zram, algo, prio);
+		if (ret)
+			return ret;
 	}
 
-	prio_max = min(prio_max, (u32)zram->num_active_comps);
-	if (prio >= prio_max) {
-		ret = -EINVAL;
-		goto out;
+	if (algo && !prio_param) {
+		prio = lookup_algo_priority(zram, algo, ZRAM_SECONDARY_COMP);
+		if (prio < 0)
+			return -EINVAL;
 	}
 
+	if (!zram->comps[prio])
+		return -EINVAL;
+
 	page = alloc_page(GFP_KERNEL);
 	if (!page) {
 		ret = -ENOMEM;
@@ -2653,7 +2624,7 @@ static ssize_t recompress_store(struct device *dev,
 		goto out;
 	}
 
-	scan_slots_for_recompress(zram, mode, prio_max, ctl);
+	scan_slots_for_recompress(zram, mode, prio, ctl);
 
 	ret = len;
 	while ((pps = select_pp_slot(ctl))) {
@@ -2667,8 +2638,7 @@ static ssize_t recompress_store(struct device *dev,
 			goto next;
 
 		err = recompress_slot(zram, pps->index, page,
-				      &num_recomp_pages, threshold,
-				      prio, prio_max);
+				      &num_recomp_pages, threshold, prio);
 next:
 		slot_unlock(zram, pps->index);
 		release_pp_slot(zram, pps);
@@ -2848,15 +2818,10 @@ static void zram_destroy_comps(struct zram *zram)
 		if (!comp)
 			continue;
 		zcomp_destroy(comp);
-		zram->num_active_comps--;
 	}
 
-	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
-		/* Do not free statically defined compression algorithms */
-		if (zram->comp_algs[prio] != default_compressor)
-			kfree(zram->comp_algs[prio]);
+	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++)
 		zram->comp_algs[prio] = NULL;
-	}
 
 	zram_comp_params_reset(zram);
 }
@@ -2917,7 +2882,6 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr,
 		}
 
 		zram->comps[prio] = comp;
-		zram->num_active_comps++;
 	}
 	zram->disksize = disksize;
 	set_capacity_and_notify(zram->disk, zram->disksize >> SECTOR_SHIFT);
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index f0de8f8218f5c..08d1774c15db6 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -125,7 +125,6 @@ struct zram {
 	 */
 	u64 disksize;	/* bytes */
 	const char *comp_algs[ZRAM_MAX_COMPS];
-	s8 num_active_comps;
 	/*
 	 * zram is claimed so open request will be failed
 	 */
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index d33cc70eec662..463d59890befe 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -223,7 +223,7 @@ int btbcm_patchram(struct hci_dev *hdev, const struct firmware *fw)
 		err = PTR_ERR(skb);
 		bt_dev_err(hdev, "BCM: Download Minidrv command failed (%d)",
 			   err);
-		goto done;
+		return err;
 	}
 	kfree_skb(skb);
 
@@ -242,8 +242,7 @@ int btbcm_patchram(struct hci_dev *hdev, const struct firmware *fw)
 
 		if (fw_size < cmd->plen) {
 			bt_dev_err(hdev, "BCM: Patch is corrupted");
-			err = -EINVAL;
-			goto done;
+			return -EINVAL;
 		}
 
 		cmd_param = fw_ptr;
@@ -258,7 +257,7 @@ int btbcm_patchram(struct hci_dev *hdev, const struct firmware *fw)
 			err = PTR_ERR(skb);
 			bt_dev_err(hdev, "BCM: Patch command %04x failed (%d)",
 				   opcode, err);
-			goto done;
+			return err;
 		}
 		kfree_skb(skb);
 	}
@@ -266,8 +265,7 @@ int btbcm_patchram(struct hci_dev *hdev, const struct firmware *fw)
 	/* 250 msec delay after Launch Ram completes */
 	msleep(250);
 
-done:
-	return err;
+	return 0;
 }
 EXPORT_SYMBOL(btbcm_patchram);
 
@@ -507,6 +505,7 @@ static const struct bcm_subver_table bcm_uart_subver_table[] = {
 	{ 0x6119, "BCM4345C0"	},	/* 003.001.025 */
 	{ 0x6606, "BCM4345C5"	},	/* 003.006.006 */
 	{ 0x230f, "BCM4356A2"	},	/* 001.003.015 */
+	{ 0x2310, "BCM4343A2"	},	/* 001.003.016 */
 	{ 0x220e, "BCM20702A1"  },	/* 001.002.014 */
 	{ 0x420d, "BCM4349B1"	},	/* 002.002.013 */
 	{ 0x420e, "BCM4349B1"	},	/* 002.002.014 */
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index 246b6205c5e0e..8353932ed6307 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -35,6 +35,19 @@ enum {
 	DSM_SET_RESET_METHOD = 3,
 };
 
+/* Hybrid ECDSA + LMS */
+#define BTINTEL_RSA_HEADER_VER		0x00010000
+#define BTINTEL_ECDSA_HEADER_VER	0x00020000
+#define BTINTEL_HYBRID_HEADER_VER	0x00069700
+#define BTINTEL_ECDSA_OFFSET		128
+#define BTINTEL_CSS_HEADER_SIZE		128
+#define BTINTEL_ECDSA_PUB_KEY_SIZE	96
+#define BTINTEL_ECDSA_SIG_SIZE		96
+#define BTINTEL_LMS_OFFSET		320
+#define BTINTEL_LMS_PUB_KEY_SIZE	52
+#define BTINTEL_LMS_SIG_SIZE		1744
+#define BTINTEL_CMD_BUFFER_OFFSET	2116
+
 #define BTINTEL_BT_DOMAIN		0x12
 #define BTINTEL_SAR_LEGACY		0
 #define BTINTEL_SAR_INC_PWR		1
@@ -484,6 +497,8 @@ int btintel_version_info_tlv(struct hci_dev *hdev,
 	case 0x1d:	/* BlazarU (BzrU) */
 	case 0x1e:	/* BlazarI (Bzr) */
 	case 0x1f:      /* Scorpious Peak */
+	case 0x20:	/* Scorpious Peak2 */
+	case 0x21:	/* Scorpious Peak2 F */
 	case 0x22:	/* BlazarIW (BzrIW) */
 		break;
 	default:
@@ -505,8 +520,8 @@ int btintel_version_info_tlv(struct hci_dev *hdev,
 			return -EINVAL;
 		}
 
-		/* Secure boot engine type should be either 1 (ECDSA) or 0 (RSA) */
-		if (version->sbe_type > 0x01) {
+		/* Secure boot engine type can be 0 (RSA), 1 (ECDSA), 2 (LMS), 3 (ECDSA + LMS) */
+		if (version->sbe_type > 0x03) {
 			bt_dev_err(hdev, "Unsupported Intel secure boot engine type (0x%x)",
 				   version->sbe_type);
 			return -EINVAL;
@@ -1025,6 +1040,48 @@ static int btintel_sfi_ecdsa_header_secure_send(struct hci_dev *hdev,
 	return 0;
 }
 
+static int btintel_sfi_hybrid_header_secure_send(struct hci_dev *hdev,
+						 const struct firmware *fw)
+{
+	int err;
+
+	err = btintel_secure_send(hdev, 0x00, BTINTEL_CSS_HEADER_SIZE, fw->data);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware CSS header (%d)", err);
+		return err;
+	}
+
+	err = btintel_secure_send(hdev, 0x03, BTINTEL_ECDSA_PUB_KEY_SIZE,
+				  fw->data + BTINTEL_ECDSA_OFFSET);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware ECDSA pkey (%d)", err);
+		return err;
+	}
+
+	err = btintel_secure_send(hdev, 0x02, BTINTEL_ECDSA_SIG_SIZE,
+				  fw->data + BTINTEL_ECDSA_OFFSET + BTINTEL_ECDSA_PUB_KEY_SIZE);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware ECDSA signature (%d)", err);
+		return err;
+	}
+
+	err = btintel_secure_send(hdev, 0x05, BTINTEL_LMS_PUB_KEY_SIZE,
+				  fw->data + BTINTEL_LMS_OFFSET);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware LMS pkey (%d)", err);
+		return err;
+	}
+
+	err = btintel_secure_send(hdev, 0x04, BTINTEL_LMS_SIG_SIZE,
+				  fw->data + BTINTEL_LMS_OFFSET + BTINTEL_LMS_PUB_KEY_SIZE);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware LMS signature (%d)", err);
+		return err;
+	}
+
+	return 0;
+}
+
 static int btintel_download_firmware_payload(struct hci_dev *hdev,
 					     const struct firmware *fw,
 					     size_t offset)
@@ -1198,11 +1255,12 @@ static int btintel_download_fw_tlv(struct hci_dev *hdev,
 	 * Command Buffer.
 	 *
 	 * CSS Header byte positions 0x08 to 0x0B represent the CSS Header
-	 * version: RSA(0x00010000) , ECDSA (0x00020000)
+	 * version: RSA(0x00010000) , ECDSA (0x00020000) , HYBRID (0x00069700)
 	 */
 	css_header_ver = get_unaligned_le32(fw->data + CSS_HEADER_OFFSET);
-	if (css_header_ver != 0x00010000) {
-		bt_dev_err(hdev, "Invalid CSS Header version");
+	if (css_header_ver != BTINTEL_RSA_HEADER_VER &&
+	    css_header_ver != BTINTEL_HYBRID_HEADER_VER) {
+		bt_dev_err(hdev, "Invalid CSS Header version: 0x%8.8x", css_header_ver);
 		return -EINVAL;
 	}
 
@@ -1220,15 +1278,15 @@ static int btintel_download_fw_tlv(struct hci_dev *hdev,
 		err = btintel_download_firmware_payload(hdev, fw, RSA_HEADER_LEN);
 		if (err)
 			return err;
-	} else if (hw_variant >= 0x17) {
+	} else if (hw_variant >= 0x17 && css_header_ver == BTINTEL_RSA_HEADER_VER) {
 		/* Check if CSS header for ECDSA follows the RSA header */
 		if (fw->data[ECDSA_OFFSET] != 0x06)
 			return -EINVAL;
 
 		/* Check if the CSS Header version is ECDSA(0x00020000) */
 		css_header_ver = get_unaligned_le32(fw->data + ECDSA_OFFSET + CSS_HEADER_OFFSET);
-		if (css_header_ver != 0x00020000) {
-			bt_dev_err(hdev, "Invalid CSS Header version");
+		if (css_header_ver != BTINTEL_ECDSA_HEADER_VER) {
+			bt_dev_err(hdev, "Invalid CSS Header version: 0x%8.8x", css_header_ver);
 			return -EINVAL;
 		}
 
@@ -1251,6 +1309,14 @@ static int btintel_download_fw_tlv(struct hci_dev *hdev,
 			if (err)
 				return err;
 		}
+	} else if (hw_variant >= 0x20 && css_header_ver == BTINTEL_HYBRID_HEADER_VER) {
+		err = btintel_sfi_hybrid_header_secure_send(hdev, fw);
+		if (err)
+			return err;
+
+		err = btintel_download_firmware_payload(hdev, fw, BTINTEL_CMD_BUFFER_OFFSET);
+		if (err)
+			return err;
 	}
 	return 0;
 }
@@ -2749,35 +2815,40 @@ static int btintel_set_dsbr(struct hci_dev *hdev, struct intel_version_tlv *ver)
 
 	struct btintel_dsbr_cmd cmd;
 	struct sk_buff *skb;
-	u32 dsbr, cnvi;
-	u8 status;
+	u32 dsbr;
+	u8 status, hw_variant;
 	int err;
 
-	cnvi = ver->cnvi_top & 0xfff;
+	hw_variant = INTEL_HW_VARIANT(ver->cnvi_bt);
 	/* DSBR command needs to be sent for,
 	 * 1. BlazarI or BlazarIW + B0 step product in IML image.
 	 * 2. Gale Peak2 or BlazarU in OP image.
 	 * 3. Scorpious Peak in IML image.
+	 * 4. Scorpious Peak2 onwards + PCIe transport in IML image.
 	 */
 
-	switch (cnvi) {
-	case BTINTEL_CNVI_BLAZARI:
-	case BTINTEL_CNVI_BLAZARIW:
+	switch (hw_variant) {
+	case BTINTEL_HWID_BZRI:
+	case BTINTEL_HWID_BZRIW:
 		if (ver->img_type == BTINTEL_IMG_IML &&
 		    INTEL_CNVX_TOP_STEP(ver->cnvi_top) == 0x01)
 			break;
 		return 0;
-	case BTINTEL_CNVI_GAP:
-	case BTINTEL_CNVI_BLAZARU:
+	case BTINTEL_HWID_GAP:
+	case BTINTEL_HWID_BZRU:
 		if (ver->img_type == BTINTEL_IMG_OP &&
 		    hdev->bus == HCI_USB)
 			break;
 		return 0;
-	case BTINTEL_CNVI_SCP:
+	case BTINTEL_HWID_SCP:
 		if (ver->img_type == BTINTEL_IMG_IML)
 			break;
 		return 0;
 	default:
+		/* Scorpius Peak2 onwards */
+		if (hw_variant >= BTINTEL_HWID_SCP2 && hdev->bus == HCI_PCI
+		    && ver->img_type == BTINTEL_IMG_IML)
+			break;
 		return 0;
 	}
 
@@ -3254,6 +3325,8 @@ void btintel_set_msft_opcode(struct hci_dev *hdev, u8 hw_variant)
 	case 0x1d:
 	case 0x1e:
 	case 0x1f:
+	case 0x20:
+	case 0x21:
 	case 0x22:
 		hci_set_msft_opcode(hdev, 0xFC1E);
 		break;
@@ -3595,6 +3668,8 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 	case 0x1d:
 	case 0x1e:
 	case 0x1f:
+	case 0x20:
+	case 0x21:
 	case 0x22:
 		/* Display version information of TLV type */
 		btintel_version_info_tlv(hdev, &ver_tlv);
diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
index 431998049e686..0e9ca99aaaaea 100644
--- a/drivers/bluetooth/btintel.h
+++ b/drivers/bluetooth/btintel.h
@@ -54,11 +54,13 @@ struct intel_tlv {
 
 #define BTINTEL_HCI_OP_RESET	0xfc01
 
-#define BTINTEL_CNVI_BLAZARI		0x900
-#define BTINTEL_CNVI_BLAZARIW		0x901
-#define BTINTEL_CNVI_GAP		0x910
-#define BTINTEL_CNVI_BLAZARU		0x930
-#define BTINTEL_CNVI_SCP		0xA00
+#define BTINTEL_CNVI_BLAZARI		0x900	/* BlazarI - Lunar Lake */
+#define BTINTEL_CNVI_BLAZARIW		0x901	/* BlazarIW - Wildcat Lake */
+#define BTINTEL_CNVI_GAP		0x910	/* Gale Peak2 - Meteor Lake */
+#define BTINTEL_CNVI_BLAZARU		0x930	/* BlazarU - Meteor Lake */
+#define BTINTEL_CNVI_SCP		0xA00	/* Scorpius Peak - Panther Lake */
+#define BTINTEL_CNVI_SCP2		0xA10	/* Scorpius Peak2 - Nova Lake */
+#define BTINTEL_CNVI_SCP2F		0xA20	/* Scorpius Peak2F - Nova Lake */
 
 /* CNVR */
 #define BTINTEL_CNVR_FMP2		0x910
@@ -69,6 +71,14 @@ struct intel_tlv {
 
 #define BTINTEL_FWID_MAXLEN 64
 
+/* CNVi Hardware variant */
+#define BTINTEL_HWID_GAP	0x1c	/* Gale Peak2 - Meteor Lake */
+#define BTINTEL_HWID_BZRI	0x1e	/* BlazarI - Lunar Lake */
+#define BTINTEL_HWID_BZRU	0x1d	/* BlazarU - Meteor Lake */
+#define BTINTEL_HWID_SCP	0x1f	/* Scorpius Peak - Panther Lake */
+#define BTINTEL_HWID_SCP2	0x20	/* Scorpius Peak2 - Nova Lake */
+#define BTINTEL_HWID_BZRIW	0x22	/* BlazarIW - Wildcat Lake */
+
 struct intel_version_tlv {
 	u32	cnvi_top;
 	u32	cnvr_top;
diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c
index 37b744e35bc45..05f82bc3f0d78 100644
--- a/drivers/bluetooth/btintel_pcie.c
+++ b/drivers/bluetooth/btintel_pcie.c
@@ -9,7 +9,9 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/firmware.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
+#include <linux/string.h>
 #include <linux/wait.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
@@ -45,6 +47,10 @@ static const struct pci_device_id btintel_pcie_table[] = {
 	{ BTINTEL_PCI_DEVICE(0xE376, PCI_ANY_ID) },
 	 /* Scorpious, Panther Lake-H404 */
 	{ BTINTEL_PCI_DEVICE(0xE476, PCI_ANY_ID) },
+	 /* Scorpious2, Nova Lake-PCD-H */
+	{ BTINTEL_PCI_DEVICE(0xD346, PCI_ANY_ID) },
+	 /* Scorpious2, Nova Lake-PCD-S */
+	{ BTINTEL_PCI_DEVICE(0x6E74, PCI_ANY_ID) },
 	{ 0 }
 };
 MODULE_DEVICE_TABLE(pci, btintel_pcie_table);
@@ -72,6 +78,9 @@ struct btintel_pcie_dev_recovery {
 #define BTINTEL_PCIE_SCP_HWEXP_SIZE		4096
 #define BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR		0xB030F800
 
+#define BTINTEL_PCIE_SCP2_HWEXP_SIZE		4096
+#define BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR	0xB031D000
+
 #define BTINTEL_PCIE_MAGIC_NUM	0xA5A5A5A5
 
 #define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER	0x17A2
@@ -268,7 +277,7 @@ static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev)
 	if (!skb)
 		return;
 
-	snprintf(buf, sizeof(buf), "%s", "---- Dump of debug registers ---");
+	strscpy(buf, "---- Dump of debug registers ---");
 	bt_dev_dbg(hdev, "%s", buf);
 	skb_put_data(skb, buf, strlen(buf));
 
@@ -1224,11 +1233,16 @@ static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data)
 			return;
 		len = BTINTEL_PCIE_BLZR_HWEXP_SIZE; /* exception data length */
 		addr = BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR;
-	break;
+		break;
 	case BTINTEL_CNVI_SCP:
 		len = BTINTEL_PCIE_SCP_HWEXP_SIZE;
 		addr = BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR;
-	break;
+		break;
+	case BTINTEL_CNVI_SCP2:
+	case BTINTEL_CNVI_SCP2F:
+		len = BTINTEL_PCIE_SCP2_HWEXP_SIZE;
+		addr = BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR;
+		break;
 	default:
 		bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top);
 		return;
@@ -2093,6 +2107,8 @@ static int btintel_pcie_setup_internal(struct hci_dev *hdev)
 	switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) {
 	case 0x1e:	/* BzrI */
 	case 0x1f:	/* ScP  */
+	case 0x20:	/* ScP2 */
+	case 0x21:	/* ScP2 F */
 	case 0x22:	/* BzrIW */
 		/* Display version information of TLV type */
 		btintel_version_info_tlv(hdev, &ver_tlv);
@@ -2372,7 +2388,7 @@ static int btintel_pcie_hci_drv_read_info(struct hci_dev *hdev, void *data,
 	u16 opcode, num_supported_commands =
 		ARRAY_SIZE(btintel_pcie_hci_drv_supported_commands);
 
-	rp_size = sizeof(*rp) + num_supported_commands * 2;
+	rp_size = struct_size(rp, supported_commands, num_supported_commands);
 
 	rp = kmalloc(rp_size, GFP_KERNEL);
 	if (!rp)
diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c
index fa7533578f85c..55516b4602db6 100644
--- a/drivers/bluetooth/btmtk.c
+++ b/drivers/bluetooth/btmtk.c
@@ -205,9 +205,14 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 				}
 			}
 
+			/* If retry exhausted goto err_release_fw */
+			if (retry == 0) {
+				err = -EIO;
+				goto err_release_fw;
+			}
+
 			fw_ptr += section_offset;
 			wmt_params.op = BTMTK_WMT_PATCH_DWNLD;
-			wmt_params.status = NULL;
 
 			while (dl_size > 0) {
 				dlen = min_t(int, 250, dl_size);
@@ -225,7 +230,14 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 				wmt_params.data = fw_ptr;
 
 				err = wmt_cmd_sync(hdev, &wmt_params);
-				if (err < 0) {
+				/* Status BTMTK_WMT_PATCH_PROGRESS indicates firmware is
+				 * in process of being downloaded, which is not expected to
+				 * occur here.
+				 */
+				if (status == BTMTK_WMT_PATCH_PROGRESS) {
+					err = -EIO;
+					goto err_release_fw;
+				} else if (err < 0) {
 					bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)",
 						   err);
 					goto err_release_fw;
@@ -1326,12 +1338,16 @@ int btmtk_usb_setup(struct hci_dev *hdev)
 	case 0x7922:
 	case 0x7925:
 	case 0x7961:
+	case 0x7902:
 		btmtk_fw_get_filename(fw_bin_name, sizeof(fw_bin_name), dev_id,
 				      fw_version, fw_flavor);
 
 		err = btmtk_setup_firmware_79xx(hdev, fw_bin_name,
 						btmtk_usb_hci_wmt_sync);
 		if (err < 0) {
+			/* retry once if setup firmware error */
+			if (!test_and_set_bit(BTMTK_FIRMWARE_DL_RETRY, &btmtk_data->flags))
+				btmtk_reset_sync(hdev);
 			bt_dev_err(hdev, "Failed to set up firmware (%d)", err);
 			return err;
 		}
@@ -1359,6 +1375,9 @@ int btmtk_usb_setup(struct hci_dev *hdev)
 		hci_set_msft_opcode(hdev, 0xFD30);
 		hci_set_aosp_capable(hdev);
 
+		/* Clear BTMTK_FIRMWARE_DL_RETRY if setup successfully */
+		test_and_clear_bit(BTMTK_FIRMWARE_DL_RETRY, &btmtk_data->flags);
+
 		/* Set up ISO interface after protocol enabled */
 		if (test_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags)) {
 			if (!btmtk_usb_isointf_init(hdev))
diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h
index 5df7c32966247..adaf385626ee6 100644
--- a/drivers/bluetooth/btmtk.h
+++ b/drivers/bluetooth/btmtk.h
@@ -5,6 +5,7 @@
 #define FIRMWARE_MT7663		"mediatek/mt7663pr2h.bin"
 #define FIRMWARE_MT7668		"mediatek/mt7668pr2h.bin"
 #define FIRMWARE_MT7922		"mediatek/BT_RAM_CODE_MT7922_1_1_hdr.bin"
+#define FIRMWARE_MT7902		"mediatek/BT_RAM_CODE_MT7902_1_1_hdr.bin"
 #define FIRMWARE_MT7961		"mediatek/BT_RAM_CODE_MT7961_1_2_hdr.bin"
 #define FIRMWARE_MT7925		"mediatek/mt7925/BT_RAM_CODE_MT7925_1_1_hdr.bin"
 
@@ -147,6 +148,7 @@ enum {
 	BTMTK_HW_RESET_ACTIVE,
 	BTMTK_ISOPKT_OVER_INTR,
 	BTMTK_ISOPKT_RUNNING,
+	BTMTK_FIRMWARE_DL_RETRY,
 };
 
 typedef int (*btmtk_reset_sync_func_t)(struct hci_dev *, void *);
diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c
index e986e5af51ae1..042064464d341 100644
--- a/drivers/bluetooth/btmtksdio.c
+++ b/drivers/bluetooth/btmtksdio.c
@@ -42,24 +42,35 @@ struct btmtksdio_data {
 	const char *fwname;
 	u16 chipid;
 	bool lp_mbox_supported;
+	bool pm_runtime_supported;
 };
 
 static const struct btmtksdio_data mt7663_data = {
 	.fwname = FIRMWARE_MT7663,
 	.chipid = 0x7663,
 	.lp_mbox_supported = false,
+	.pm_runtime_supported = true,
 };
 
 static const struct btmtksdio_data mt7668_data = {
 	.fwname = FIRMWARE_MT7668,
 	.chipid = 0x7668,
 	.lp_mbox_supported = false,
+	.pm_runtime_supported = true,
 };
 
 static const struct btmtksdio_data mt7921_data = {
 	.fwname = FIRMWARE_MT7961,
 	.chipid = 0x7921,
 	.lp_mbox_supported = true,
+	.pm_runtime_supported = true,
+};
+
+static const struct btmtksdio_data mt7902_data = {
+	.fwname = FIRMWARE_MT7902,
+	.chipid = 0x7902,
+	.lp_mbox_supported = false,
+	.pm_runtime_supported = false,
 };
 
 static const struct sdio_device_id btmtksdio_table[] = {
@@ -69,6 +80,8 @@ static const struct sdio_device_id btmtksdio_table[] = {
 	 .driver_data = (kernel_ulong_t)&mt7668_data },
 	{SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7961),
 	 .driver_data = (kernel_ulong_t)&mt7921_data },
+	{SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7902),
+	.driver_data = (kernel_ulong_t)&mt7902_data },
 	{ }	/* Terminating entry */
 };
 MODULE_DEVICE_TABLE(sdio, btmtksdio_table);
@@ -1090,6 +1103,7 @@ static int btmtksdio_setup(struct hci_dev *hdev)
 	set_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state);
 
 	switch (bdev->data->chipid) {
+	case 0x7902:
 	case 0x7921:
 		if (test_bit(BTMTKSDIO_HW_RESET_ACTIVE, &bdev->tx_state)) {
 			err = btmtksdio_mtk_reg_read(hdev, MT7921_DLSTATUS,
@@ -1167,22 +1181,24 @@ static int btmtksdio_setup(struct hci_dev *hdev)
 	delta = ktime_sub(rettime, calltime);
 	duration = (unsigned long long)ktime_to_ns(delta) >> 10;
 
-	pm_runtime_set_autosuspend_delay(bdev->dev,
-					 MTKBTSDIO_AUTOSUSPEND_DELAY);
-	pm_runtime_use_autosuspend(bdev->dev);
+	if (bdev->data->pm_runtime_supported) {
+		pm_runtime_set_autosuspend_delay(bdev->dev,
+						 MTKBTSDIO_AUTOSUSPEND_DELAY);
+		pm_runtime_use_autosuspend(bdev->dev);
 
-	err = pm_runtime_set_active(bdev->dev);
-	if (err < 0)
-		return err;
+		err = pm_runtime_set_active(bdev->dev);
+		if (err < 0)
+			return err;
 
-	/* Default forbid runtime auto suspend, that can be allowed by
-	 * enable_autosuspend flag or the PM runtime entry under sysfs.
-	 */
-	pm_runtime_forbid(bdev->dev);
-	pm_runtime_enable(bdev->dev);
+		/* Default forbid runtime auto suspend, that can be allowed by
+		 * enable_autosuspend flag or the PM runtime entry under sysfs.
+		 */
+		pm_runtime_forbid(bdev->dev);
+		pm_runtime_enable(bdev->dev);
 
-	if (enable_autosuspend)
-		pm_runtime_allow(bdev->dev);
+		if (enable_autosuspend)
+			pm_runtime_allow(bdev->dev);
+	}
 
 	bt_dev_info(hdev, "Device setup in %llu usecs", duration);
 
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index 74f820e89655e..dda76365726f0 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -787,6 +787,8 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 	 */
 	if (soc_type == QCA_WCN3988)
 		rom_ver = ((soc_ver & 0x00000f00) >> 0x05) | (soc_ver & 0x0000000f);
+	else if (soc_type == QCA_WCN3998)
+		rom_ver = ((soc_ver & 0x0000f000) >> 0x07) | (soc_ver & 0x0000000f);
 	else
 		rom_ver = ((soc_ver & 0x00000f00) >> 0x04) | (soc_ver & 0x0000000f);
 
@@ -799,6 +801,14 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 		snprintf(config.fwname, sizeof(config.fwname), "qca/%s", rampatch_name);
 	} else {
 		switch (soc_type) {
+		case QCA_QCA2066:
+			snprintf(config.fwname, sizeof(config.fwname),
+				 "qca/hpbtfw%02x.tlv", rom_ver);
+			break;
+		case QCA_QCA6390:
+			snprintf(config.fwname, sizeof(config.fwname),
+				 "qca/htbtfw%02x.tlv", rom_ver);
+			break;
 		case QCA_WCN3950:
 			snprintf(config.fwname, sizeof(config.fwname),
 				 "qca/cmbtfw%02x.tlv", rom_ver);
@@ -813,14 +823,6 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 			snprintf(config.fwname, sizeof(config.fwname),
 				 "qca/apbtfw%02x.tlv", rom_ver);
 			break;
-		case QCA_QCA2066:
-			snprintf(config.fwname, sizeof(config.fwname),
-				 "qca/hpbtfw%02x.tlv", rom_ver);
-			break;
-		case QCA_QCA6390:
-			snprintf(config.fwname, sizeof(config.fwname),
-				 "qca/htbtfw%02x.tlv", rom_ver);
-			break;
 		case QCA_WCN6750:
 			/* Choose mbn file by default.If mbn file is not found
 			 * then choose tlv file
@@ -890,6 +892,16 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 		}
 	} else {
 		switch (soc_type) {
+		case QCA_QCA2066:
+			qca_get_nvm_name_by_board(config.fwname,
+						  sizeof(config.fwname),
+						  "hpnv", soc_type, ver,
+						  rom_ver, boardid);
+			break;
+		case QCA_QCA6390:
+			snprintf(config.fwname, sizeof(config.fwname),
+				 "qca/htnv%02x.bin", rom_ver);
+			break;
 		case QCA_WCN3950:
 			if (le32_to_cpu(ver.soc_id) == QCA_WCN3950_SOC_ID_T)
 				variant = "t";
@@ -912,15 +924,6 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 			snprintf(config.fwname, sizeof(config.fwname),
 				 "qca/apnv%02x.bin", rom_ver);
 			break;
-		case QCA_QCA2066:
-			qca_get_nvm_name_by_board(config.fwname,
-				sizeof(config.fwname), "hpnv", soc_type, ver,
-				rom_ver, boardid);
-			break;
-		case QCA_QCA6390:
-			snprintf(config.fwname, sizeof(config.fwname),
-				 "qca/htnv%02x.bin", rom_ver);
-			break;
 		case QCA_WCN6750:
 			snprintf(config.fwname, sizeof(config.fwname),
 				 "qca/msnv%02x.bin", rom_ver);
@@ -954,9 +957,9 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 	}
 
 	switch (soc_type) {
-	case QCA_WCN3991:
 	case QCA_QCA2066:
 	case QCA_QCA6390:
+	case QCA_WCN3991:
 	case QCA_WCN6750:
 	case QCA_WCN6855:
 	case QCA_WCN7850:
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index a1c5eb993e478..f9d515ee9124e 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -671,7 +671,17 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3606), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
-
+	/* MediaTek MT7902 Bluetooth devices */
+	{ USB_DEVICE(0x0e8d, 0x1ede), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3579), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3580), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3594), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3596), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	/* MediaTek MT7922 Bluetooth devices */
 	{ USB_DEVICE(0x13d3, 0x3585), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
@@ -703,8 +713,12 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0489, 0xe170), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe174), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x04ca, 0x3804), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3807), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK |
@@ -789,7 +803,7 @@ static const struct usb_device_id quirks_table[] = {
 	{ USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
-	  					     BTUSB_WIDEBAND_SPEECH },
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0bda, 0x8771), .driver_info = BTUSB_REALTEK |
@@ -2468,6 +2482,7 @@ static int btusb_setup_csr(struct hci_dev *hdev)
 			     HCI_INIT_TIMEOUT);
 	if (IS_ERR(skb)) {
 		int err = PTR_ERR(skb);
+
 		bt_dev_err(hdev, "CSR: Local version failed (%d)", err);
 		return err;
 	}
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index 91acf24f1ef54..91c96ad123422 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -541,6 +541,8 @@ static int download_firmware(struct ll_device *lldev)
 	if (err || !fw->data || !fw->size) {
 		bt_dev_err(lldev->hu.hdev, "request_firmware failed(errno %d) for %s",
 			   err, bts_scr_name);
+		if (!err)
+			release_firmware(fw);
 		return -EINVAL;
 	}
 	ptr = (void *)fw->data;
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index bb9f002aa85e9..c17a462aef554 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -236,8 +236,7 @@ struct qca_serdev {
 
 static int qca_regulator_enable(struct qca_serdev *qcadev);
 static void qca_regulator_disable(struct qca_serdev *qcadev);
-static void qca_power_shutdown(struct hci_uart *hu);
-static int qca_power_off(struct hci_dev *hdev);
+static void qca_power_off(struct hci_uart *hu);
 static void qca_controller_memdump(struct work_struct *work);
 static void qca_dmp_hdr(struct hci_dev *hdev, struct sk_buff *skb);
 
@@ -723,6 +722,10 @@ static int qca_close(struct hci_uart *hu)
 
 	BT_DBG("hu %p qca close", hu);
 
+	/* BT core skips qca_hci_shutdown() which calls qca_power_off() on rmmod */
+	if (!test_bit(QCA_BT_OFF, &qca->flags))
+		qca_power_off(hu);
+
 	serial_clock_vote(HCI_IBS_VOTE_STATS_UPDATE, hu);
 
 	skb_queue_purge(&qca->tx_wait_q);
@@ -1850,6 +1853,7 @@ static int qca_power_on(struct hci_dev *hdev)
 		return 0;
 
 	switch (soc_type) {
+	case QCA_QCA6390:
 	case QCA_WCN3950:
 	case QCA_WCN3988:
 	case QCA_WCN3990:
@@ -1858,7 +1862,6 @@ static int qca_power_on(struct hci_dev *hdev)
 	case QCA_WCN6750:
 	case QCA_WCN6855:
 	case QCA_WCN7850:
-	case QCA_QCA6390:
 		ret = qca_regulator_init(hu);
 		break;
 
@@ -2047,7 +2050,7 @@ static int qca_setup(struct hci_uart *hu)
 
 out:
 	if (ret) {
-		qca_power_shutdown(hu);
+		qca_power_off(hu);
 
 		if (retries < MAX_INIT_RETRIES) {
 			bt_dev_warn(hdev, "Retry BT power ON:%d", retries);
@@ -2096,6 +2099,18 @@ static const struct hci_uart_proto qca_proto = {
 	.dequeue	= qca_dequeue,
 };
 
+static const struct qca_device_data qca_soc_data_qca2066 __maybe_unused = {
+	.soc_type = QCA_QCA2066,
+	.num_vregs = 0,
+	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES |
+			QCA_CAP_HFP_HW_OFFLOAD,
+};
+
+static const struct qca_device_data qca_soc_data_qca6390 __maybe_unused = {
+	.soc_type = QCA_QCA6390,
+	.num_vregs = 0,
+};
+
 static const struct qca_device_data qca_soc_data_wcn3950 __maybe_unused = {
 	.soc_type = QCA_WCN3950,
 	.vregs = (struct qca_vreg []) {
@@ -2152,18 +2167,6 @@ static const struct qca_device_data qca_soc_data_wcn3998 __maybe_unused = {
 	.num_vregs = 4,
 };
 
-static const struct qca_device_data qca_soc_data_qca2066 __maybe_unused = {
-	.soc_type = QCA_QCA2066,
-	.num_vregs = 0,
-	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES |
-			QCA_CAP_HFP_HW_OFFLOAD,
-};
-
-static const struct qca_device_data qca_soc_data_qca6390 __maybe_unused = {
-	.soc_type = QCA_QCA6390,
-	.num_vregs = 0,
-};
-
 static const struct qca_device_data qca_soc_data_wcn6750 __maybe_unused = {
 	.soc_type = QCA_WCN6750,
 	.vregs = (struct qca_vreg []) {
@@ -2211,7 +2214,7 @@ static const struct qca_device_data qca_soc_data_wcn7850 __maybe_unused = {
 			QCA_CAP_HFP_HW_OFFLOAD,
 };
 
-static void qca_power_shutdown(struct hci_uart *hu)
+static void qca_power_off(struct hci_uart *hu)
 {
 	struct qca_serdev *qcadev;
 	struct qca_data *qca = hu->priv;
@@ -2238,6 +2241,18 @@ static void qca_power_shutdown(struct hci_uart *hu)
 	qcadev = serdev_device_get_drvdata(hu->serdev);
 	power = qcadev->bt_power;
 
+	switch (soc_type) {
+	case QCA_WCN3988:
+	case QCA_WCN3990:
+	case QCA_WCN3991:
+	case QCA_WCN3998:
+		host_set_baudrate(hu, 2400);
+		qca_send_power_pulse(hu, false);
+		break;
+	default:
+		break;
+	}
+
 	if (power && power->pwrseq) {
 		pwrseq_power_off(power->pwrseq);
 		set_bit(QCA_BT_OFF, &qca->flags);
@@ -2249,8 +2264,6 @@ static void qca_power_shutdown(struct hci_uart *hu)
 	case QCA_WCN3990:
 	case QCA_WCN3991:
 	case QCA_WCN3998:
-		host_set_baudrate(hu, 2400);
-		qca_send_power_pulse(hu, false);
 		qca_regulator_disable(qcadev);
 		break;
 
@@ -2261,7 +2274,7 @@ static void qca_power_shutdown(struct hci_uart *hu)
 		qca_regulator_disable(qcadev);
 		if (qcadev->sw_ctrl) {
 			sw_ctrl_state = gpiod_get_value_cansleep(qcadev->sw_ctrl);
-			bt_dev_dbg(hu->hdev, "SW_CTRL is %d", sw_ctrl_state);
+			BT_DBG("SW_CTRL is %d", sw_ctrl_state);
 		}
 		break;
 
@@ -2272,7 +2285,7 @@ static void qca_power_shutdown(struct hci_uart *hu)
 	set_bit(QCA_BT_OFF, &qca->flags);
 }
 
-static int qca_power_off(struct hci_dev *hdev)
+static int qca_hci_shutdown(struct hci_dev *hdev)
 {
 	struct hci_uart *hu = hci_get_drvdata(hdev);
 	struct qca_data *qca = hu->priv;
@@ -2291,7 +2304,7 @@ static int qca_power_off(struct hci_dev *hdev)
 		usleep_range(8000, 10000);
 	}
 
-	qca_power_shutdown(hu);
+	qca_power_off(hu);
 	return 0;
 }
 
@@ -2402,6 +2415,7 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 		qcadev->btsoc_type = QCA_ROME;
 
 	switch (qcadev->btsoc_type) {
+	case QCA_QCA6390:
 	case QCA_WCN3950:
 	case QCA_WCN3988:
 	case QCA_WCN3990:
@@ -2410,7 +2424,6 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 	case QCA_WCN6750:
 	case QCA_WCN6855:
 	case QCA_WCN7850:
-	case QCA_QCA6390:
 		qcadev->bt_power = devm_kzalloc(&serdev->dev,
 						sizeof(struct qca_power),
 						GFP_KERNEL);
@@ -2422,9 +2435,14 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 	}
 
 	switch (qcadev->btsoc_type) {
+	case QCA_WCN3950:
+	case QCA_WCN3988:
+	case QCA_WCN3990:
+	case QCA_WCN3991:
+	case QCA_WCN3998:
+	case QCA_WCN6750:
 	case QCA_WCN6855:
 	case QCA_WCN7850:
-	case QCA_WCN6750:
 		if (!device_property_present(&serdev->dev, "enable-gpios")) {
 			/*
 			 * Backward compatibility with old DT sources. If the
@@ -2446,12 +2464,7 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 			else
 				break;
 		}
-		fallthrough;
-	case QCA_WCN3950:
-	case QCA_WCN3988:
-	case QCA_WCN3990:
-	case QCA_WCN3991:
-	case QCA_WCN3998:
+
 		qcadev->bt_power->dev = &serdev->dev;
 		err = qca_init_regulators(qcadev->bt_power, data->vregs,
 					  data->num_vregs);
@@ -2530,7 +2543,7 @@ static int qca_serdev_probe(struct serdev_device *serdev)
 
 	if (power_ctrl_enabled) {
 		hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_SETUP);
-		hdev->shutdown = qca_power_off;
+		hdev->shutdown = qca_hci_shutdown;
 	}
 
 	if (data) {
@@ -2565,7 +2578,7 @@ static void qca_serdev_remove(struct serdev_device *serdev)
 	case QCA_WCN6855:
 	case QCA_WCN7850:
 		if (power->vregs_on)
-			qca_power_shutdown(&qcadev->serdev_hu);
+			qca_power_off(&qcadev->serdev_hu);
 		break;
 	default:
 		break;
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 2a1b46f07080b..3181d8aa32a3e 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -38,36 +38,6 @@ config BRCMSTB_GISB_ARB
 	  arbiter. This driver provides timeout and target abort error handling
 	  and internal bus master decoding.
 
-config BT1_APB
-	bool "Baikal-T1 APB-bus driver"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
-	select REGMAP_MMIO
-	help
-	  Baikal-T1 AXI-APB bridge is used to access the SoC subsystem CSRs.
-	  IO requests are routed to this bus by means of the DW AMBA 3 AXI
-	  Interconnect. In case of any APB protocol collisions, slave device
-	  not responding on timeout an IRQ is raised with an erroneous address
-	  reported to the APB terminator (APB Errors Handler Block). This
-	  driver provides the interrupt handler to detect the erroneous
-	  address, prints an error message about the address fault, updates an
-	  errors counter. The counter and the APB-bus operations timeout can be
-	  accessed via corresponding sysfs nodes.
-
-config BT1_AXI
-	bool "Baikal-T1 AXI-bus driver"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
-	select MFD_SYSCON
-	help
-	  AXI3-bus is the main communication bus connecting all high-speed
-	  peripheral IP-cores with RAM controller and with MIPS P5600 cores on
-	  Baikal-T1 SoC. Traffic arbitration is done by means of DW AMBA 3 AXI
-	  Interconnect (so called AXI Main Interconnect) routing IO requests
-	  from one SoC block to another. This driver provides a way to detect
-	  any bus protocol errors and device not responding situations by
-	  means of an embedded on top of the interconnect errors handler
-	  block (EHB). AXI Interconnect QoS arbitration tuning is currently
-	  unsupported.
-
 config MOXTET
 	tristate "CZ.NIC Turris Mox module configuration bus"
 	depends on SPI_MASTER && OF
@@ -169,6 +139,16 @@ config QCOM_SSC_BLOCK_BUS
 	  i2c/spi/uart controllers, a hexagon core, and a clock controller
 	  which provides clocks for the above.
 
+config STM32_DBG_BUS
+	tristate "OP-TEE based debug access bus"
+	depends on OPTEE && STM32_FIREWALL
+	depends on ARCH_STM32 || COMPILE_TEST
+	help
+	  Select this to get the support for the OP-TEE based STM32 debug bus
+	  driver that is used to handle debug-related peripherals on STM32
+	  platforms when the debug configuration is not accessible by the
+	  normal world.
+
 config STM32_FIREWALL
 	bool "STM32 Firewall framework"
 	depends on (ARCH_STM32 || COMPILE_TEST) && OF
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index 8e693fe8a03a5..a01f97fef3e80 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -13,8 +13,6 @@ obj-$(CONFIG_MOXTET)		+= moxtet.o
 # DPAA2 fsl-mc bus
 obj-$(CONFIG_FSL_MC_BUS)	+= fsl-mc/
 
-obj-$(CONFIG_BT1_APB)		+= bt1-apb.o
-obj-$(CONFIG_BT1_AXI)		+= bt1-axi.o
 obj-$(CONFIG_IMX_AIPSTZ)	+= imx-aipstz.o
 obj-$(CONFIG_IMX_WEIM)		+= imx-weim.o
 obj-$(CONFIG_INTEL_IXP4XX_EB)	+= intel-ixp4xx-eb.o
@@ -27,6 +25,7 @@ obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 obj-$(CONFIG_OMAP_OCP2SCP)	+= omap-ocp2scp.o
 obj-$(CONFIG_QCOM_EBI2)		+= qcom-ebi2.o
 obj-$(CONFIG_QCOM_SSC_BLOCK_BUS)	+= qcom-ssc-block-bus.o
+obj-$(CONFIG_STM32_DBG_BUS)	+= stm32_dbg_bus.o
 obj-$(CONFIG_STM32_FIREWALL)	+= stm32_firewall.o stm32_rifsc.o stm32_etzpc.o
 obj-$(CONFIG_SUN50I_DE2_BUS)	+= sun50i-de2.o
 obj-$(CONFIG_SUNXI_RSB)		+= sunxi-rsb.o
diff --git a/drivers/bus/bt1-apb.c b/drivers/bus/bt1-apb.c
deleted file mode 100644
index 7463124b6dd92..0000000000000
--- a/drivers/bus/bt1-apb.c
+++ /dev/null
@@ -1,396 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 APB-bus driver
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/device.h>
-#include <linux/atomic.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/nmi.h>
-#include <linux/of.h>
-#include <linux/regmap.h>
-#include <linux/clk.h>
-#include <linux/reset.h>
-#include <linux/time64.h>
-#include <linux/sysfs.h>
-
-#define APB_EHB_ISR			0x00
-#define APB_EHB_ISR_PENDING		BIT(0)
-#define APB_EHB_ISR_MASK		BIT(1)
-#define APB_EHB_ADDR			0x04
-#define APB_EHB_TIMEOUT			0x08
-
-#define APB_EHB_TIMEOUT_MIN		0x000003FFU
-#define APB_EHB_TIMEOUT_MAX		0xFFFFFFFFU
-
-/*
- * struct bt1_apb - Baikal-T1 APB EHB private data
- * @dev: Pointer to the device structure.
- * @regs: APB EHB registers map.
- * @res: No-device error injection memory region.
- * @irq: Errors IRQ number.
- * @rate: APB-bus reference clock rate.
- * @pclk: APB-reference clock.
- * @prst: APB domain reset line.
- * @count: Number of errors detected.
- */
-struct bt1_apb {
-	struct device *dev;
-
-	struct regmap *regs;
-	void __iomem *res;
-	int irq;
-
-	unsigned long rate;
-	struct clk *pclk;
-
-	struct reset_control *prst;
-
-	atomic_t count;
-};
-
-static const struct regmap_config bt1_apb_regmap_cfg = {
-	.reg_bits = 32,
-	.val_bits = 32,
-	.reg_stride = 4,
-	.max_register = APB_EHB_TIMEOUT,
-	.fast_io = true
-};
-
-static inline unsigned long bt1_apb_n_to_timeout_us(struct bt1_apb *apb, u32 n)
-{
-	u64 timeout = (u64)n * USEC_PER_SEC;
-
-	do_div(timeout, apb->rate);
-
-	return timeout;
-
-}
-
-static inline unsigned long bt1_apb_timeout_to_n_us(struct bt1_apb *apb,
-						    unsigned long timeout)
-{
-	u64 n = (u64)timeout * apb->rate;
-
-	do_div(n, USEC_PER_SEC);
-
-	return n;
-
-}
-
-static irqreturn_t bt1_apb_isr(int irq, void *data)
-{
-	struct bt1_apb *apb = data;
-	u32 addr = 0;
-
-	regmap_read(apb->regs, APB_EHB_ADDR, &addr);
-
-	dev_crit_ratelimited(apb->dev,
-		"APB-bus fault %d: Slave access timeout at 0x%08x\n",
-		atomic_inc_return(&apb->count),
-		addr);
-
-	/*
-	 * Print backtrace on each CPU. This might be pointless if the fault
-	 * has happened on the same CPU as the IRQ handler is executed or
-	 * the other core proceeded further execution despite the error.
-	 * But if it's not, by looking at the trace we would get straight to
-	 * the cause of the problem.
-	 */
-	trigger_all_cpu_backtrace();
-
-	regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_PENDING, 0);
-
-	return IRQ_HANDLED;
-}
-
-static void bt1_apb_clear_data(void *data)
-{
-	struct bt1_apb *apb = data;
-	struct platform_device *pdev = to_platform_device(apb->dev);
-
-	platform_set_drvdata(pdev, NULL);
-}
-
-static struct bt1_apb *bt1_apb_create_data(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct bt1_apb *apb;
-	int ret;
-
-	apb = devm_kzalloc(dev, sizeof(*apb), GFP_KERNEL);
-	if (!apb)
-		return ERR_PTR(-ENOMEM);
-
-	ret = devm_add_action(dev, bt1_apb_clear_data, apb);
-	if (ret) {
-		dev_err(dev, "Can't add APB EHB data clear action\n");
-		return ERR_PTR(ret);
-	}
-
-	apb->dev = dev;
-	atomic_set(&apb->count, 0);
-	platform_set_drvdata(pdev, apb);
-
-	return apb;
-}
-
-static int bt1_apb_request_regs(struct bt1_apb *apb)
-{
-	struct platform_device *pdev = to_platform_device(apb->dev);
-	void __iomem *regs;
-
-	regs = devm_platform_ioremap_resource_byname(pdev, "ehb");
-	if (IS_ERR(regs)) {
-		dev_err(apb->dev, "Couldn't map APB EHB registers\n");
-		return PTR_ERR(regs);
-	}
-
-	apb->regs = devm_regmap_init_mmio(apb->dev, regs, &bt1_apb_regmap_cfg);
-	if (IS_ERR(apb->regs)) {
-		dev_err(apb->dev, "Couldn't create APB EHB regmap\n");
-		return PTR_ERR(apb->regs);
-	}
-
-	apb->res = devm_platform_ioremap_resource_byname(pdev, "nodev");
-	if (IS_ERR(apb->res))
-		dev_err(apb->dev, "Couldn't map reserved region\n");
-
-	return PTR_ERR_OR_ZERO(apb->res);
-}
-
-static int bt1_apb_request_rst(struct bt1_apb *apb)
-{
-	int ret;
-
-	apb->prst = devm_reset_control_get_optional_exclusive(apb->dev, "prst");
-	if (IS_ERR(apb->prst))
-		return dev_err_probe(apb->dev, PTR_ERR(apb->prst),
-				     "Couldn't get reset control line\n");
-
-	ret = reset_control_deassert(apb->prst);
-	if (ret)
-		dev_err(apb->dev, "Failed to deassert the reset line\n");
-
-	return ret;
-}
-
-static int bt1_apb_request_clk(struct bt1_apb *apb)
-{
-	apb->pclk = devm_clk_get_enabled(apb->dev, "pclk");
-	if (IS_ERR(apb->pclk))
-		return dev_err_probe(apb->dev, PTR_ERR(apb->pclk),
-				     "Couldn't get APB clock descriptor\n");
-
-	apb->rate = clk_get_rate(apb->pclk);
-	if (!apb->rate) {
-		dev_err(apb->dev, "Invalid clock rate\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static void bt1_apb_clear_irq(void *data)
-{
-	struct bt1_apb *apb = data;
-
-	regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_MASK, 0);
-}
-
-static int bt1_apb_request_irq(struct bt1_apb *apb)
-{
-	struct platform_device *pdev = to_platform_device(apb->dev);
-	int ret;
-
-	apb->irq = platform_get_irq(pdev, 0);
-	if (apb->irq < 0)
-		return apb->irq;
-
-	ret = devm_request_irq(apb->dev, apb->irq, bt1_apb_isr, IRQF_SHARED,
-			       "bt1-apb", apb);
-	if (ret) {
-		dev_err(apb->dev, "Couldn't request APB EHB IRQ\n");
-		return ret;
-	}
-
-	ret = devm_add_action(apb->dev, bt1_apb_clear_irq, apb);
-	if (ret) {
-		dev_err(apb->dev, "Can't add APB EHB IRQs clear action\n");
-		return ret;
-	}
-
-	/* Unmask IRQ and clear it' pending flag. */
-	regmap_update_bits(apb->regs, APB_EHB_ISR,
-			   APB_EHB_ISR_PENDING | APB_EHB_ISR_MASK,
-			   APB_EHB_ISR_MASK);
-
-	return 0;
-}
-
-static ssize_t count_show(struct device *dev, struct device_attribute *attr,
-			  char *buf)
-{
-	struct bt1_apb *apb = dev_get_drvdata(dev);
-
-	return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&apb->count));
-}
-static DEVICE_ATTR_RO(count);
-
-static ssize_t timeout_show(struct device *dev, struct device_attribute *attr,
-			    char *buf)
-{
-	struct bt1_apb *apb = dev_get_drvdata(dev);
-	unsigned long timeout;
-	int ret;
-	u32 n;
-
-	ret = regmap_read(apb->regs, APB_EHB_TIMEOUT, &n);
-	if (ret)
-		return ret;
-
-	timeout = bt1_apb_n_to_timeout_us(apb, n);
-
-	return scnprintf(buf, PAGE_SIZE, "%lu\n", timeout);
-}
-
-static ssize_t timeout_store(struct device *dev,
-			     struct device_attribute *attr,
-			     const char *buf, size_t count)
-{
-	struct bt1_apb *apb = dev_get_drvdata(dev);
-	unsigned long timeout;
-	int ret;
-	u32 n;
-
-	if (kstrtoul(buf, 0, &timeout) < 0)
-		return -EINVAL;
-
-	n = bt1_apb_timeout_to_n_us(apb, timeout);
-	n = clamp(n, APB_EHB_TIMEOUT_MIN, APB_EHB_TIMEOUT_MAX);
-
-	ret = regmap_write(apb->regs, APB_EHB_TIMEOUT, n);
-
-	return ret ?: count;
-}
-static DEVICE_ATTR_RW(timeout);
-
-static ssize_t inject_error_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	return scnprintf(buf, PAGE_SIZE, "Error injection: nodev irq\n");
-}
-
-static ssize_t inject_error_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *data, size_t count)
-{
-	struct bt1_apb *apb = dev_get_drvdata(dev);
-
-	/*
-	 * Either dummy read from the unmapped address in the APB IO area
-	 * or manually set the IRQ status.
-	 */
-	if (sysfs_streq(data, "nodev"))
-		readl(apb->res);
-	else if (sysfs_streq(data, "irq"))
-		regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_PENDING,
-				   APB_EHB_ISR_PENDING);
-	else
-		return -EINVAL;
-
-	return count;
-}
-static DEVICE_ATTR_RW(inject_error);
-
-static struct attribute *bt1_apb_sysfs_attrs[] = {
-	&dev_attr_count.attr,
-	&dev_attr_timeout.attr,
-	&dev_attr_inject_error.attr,
-	NULL
-};
-ATTRIBUTE_GROUPS(bt1_apb_sysfs);
-
-static void bt1_apb_remove_sysfs(void *data)
-{
-	struct bt1_apb *apb = data;
-
-	device_remove_groups(apb->dev, bt1_apb_sysfs_groups);
-}
-
-static int bt1_apb_init_sysfs(struct bt1_apb *apb)
-{
-	int ret;
-
-	ret = device_add_groups(apb->dev, bt1_apb_sysfs_groups);
-	if (ret) {
-		dev_err(apb->dev, "Failed to create EHB APB sysfs nodes\n");
-		return ret;
-	}
-
-	ret = devm_add_action_or_reset(apb->dev, bt1_apb_remove_sysfs, apb);
-	if (ret)
-		dev_err(apb->dev, "Can't add APB EHB sysfs remove action\n");
-
-	return ret;
-}
-
-static int bt1_apb_probe(struct platform_device *pdev)
-{
-	struct bt1_apb *apb;
-	int ret;
-
-	apb = bt1_apb_create_data(pdev);
-	if (IS_ERR(apb))
-		return PTR_ERR(apb);
-
-	ret = bt1_apb_request_regs(apb);
-	if (ret)
-		return ret;
-
-	ret = bt1_apb_request_rst(apb);
-	if (ret)
-		return ret;
-
-	ret = bt1_apb_request_clk(apb);
-	if (ret)
-		return ret;
-
-	ret = bt1_apb_request_irq(apb);
-	if (ret)
-		return ret;
-
-	ret = bt1_apb_init_sysfs(apb);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static const struct of_device_id bt1_apb_of_match[] = {
-	{ .compatible = "baikal,bt1-apb" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, bt1_apb_of_match);
-
-static struct platform_driver bt1_apb_driver = {
-	.probe = bt1_apb_probe,
-	.driver = {
-		.name = "bt1-apb",
-		.of_match_table = bt1_apb_of_match
-	}
-};
-module_platform_driver(bt1_apb_driver);
-
-MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
-MODULE_DESCRIPTION("Baikal-T1 APB-bus driver");
diff --git a/drivers/bus/bt1-axi.c b/drivers/bus/bt1-axi.c
deleted file mode 100644
index a5254c73bf43a..0000000000000
--- a/drivers/bus/bt1-axi.c
+++ /dev/null
@@ -1,292 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 AXI-bus driver
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/bitfield.h>
-#include <linux/device.h>
-#include <linux/atomic.h>
-#include <linux/regmap.h>
-#include <linux/platform_device.h>
-#include <linux/mfd/syscon.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/nmi.h>
-#include <linux/of.h>
-#include <linux/clk.h>
-#include <linux/reset.h>
-#include <linux/sysfs.h>
-
-#define BT1_AXI_WERRL			0x110
-#define BT1_AXI_WERRH			0x114
-#define BT1_AXI_WERRH_TYPE		BIT(23)
-#define BT1_AXI_WERRH_ADDR_FLD		24
-#define BT1_AXI_WERRH_ADDR_MASK		GENMASK(31, BT1_AXI_WERRH_ADDR_FLD)
-
-/*
- * struct bt1_axi - Baikal-T1 AXI-bus private data
- * @dev: Pointer to the device structure.
- * @qos_regs: AXI Interconnect QoS tuning registers.
- * @sys_regs: Baikal-T1 System Controller registers map.
- * @irq: Errors IRQ number.
- * @aclk: AXI reference clock.
- * @arst: AXI Interconnect reset line.
- * @count: Number of errors detected.
- */
-struct bt1_axi {
-	struct device *dev;
-
-	void __iomem *qos_regs;
-	struct regmap *sys_regs;
-	int irq;
-
-	struct clk *aclk;
-
-	struct reset_control *arst;
-
-	atomic_t count;
-};
-
-static irqreturn_t bt1_axi_isr(int irq, void *data)
-{
-	struct bt1_axi *axi = data;
-	u32 low = 0, high = 0;
-
-	regmap_read(axi->sys_regs, BT1_AXI_WERRL, &low);
-	regmap_read(axi->sys_regs, BT1_AXI_WERRH, &high);
-
-	dev_crit_ratelimited(axi->dev,
-		"AXI-bus fault %d: %s at 0x%x%08x\n",
-		atomic_inc_return(&axi->count),
-		high & BT1_AXI_WERRH_TYPE ? "no slave" : "slave protocol error",
-		high, low);
-
-	/*
-	 * Print backtrace on each CPU. This might be pointless if the fault
-	 * has happened on the same CPU as the IRQ handler is executed or
-	 * the other core proceeded further execution despite the error.
-	 * But if it's not, by looking at the trace we would get straight to
-	 * the cause of the problem.
-	 */
-	trigger_all_cpu_backtrace();
-
-	return IRQ_HANDLED;
-}
-
-static void bt1_axi_clear_data(void *data)
-{
-	struct bt1_axi *axi = data;
-	struct platform_device *pdev = to_platform_device(axi->dev);
-
-	platform_set_drvdata(pdev, NULL);
-}
-
-static struct bt1_axi *bt1_axi_create_data(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct bt1_axi *axi;
-	int ret;
-
-	axi = devm_kzalloc(dev, sizeof(*axi), GFP_KERNEL);
-	if (!axi)
-		return ERR_PTR(-ENOMEM);
-
-	ret = devm_add_action(dev, bt1_axi_clear_data, axi);
-	if (ret) {
-		dev_err(dev, "Can't add AXI EHB data clear action\n");
-		return ERR_PTR(ret);
-	}
-
-	axi->dev = dev;
-	atomic_set(&axi->count, 0);
-	platform_set_drvdata(pdev, axi);
-
-	return axi;
-}
-
-static int bt1_axi_request_regs(struct bt1_axi *axi)
-{
-	struct platform_device *pdev = to_platform_device(axi->dev);
-	struct device *dev = axi->dev;
-
-	axi->sys_regs = syscon_regmap_lookup_by_phandle(dev->of_node, "syscon");
-	if (IS_ERR(axi->sys_regs)) {
-		dev_err(dev, "Couldn't find syscon registers\n");
-		return PTR_ERR(axi->sys_regs);
-	}
-
-	axi->qos_regs = devm_platform_ioremap_resource_byname(pdev, "qos");
-	if (IS_ERR(axi->qos_regs))
-		dev_err(dev, "Couldn't map AXI-bus QoS registers\n");
-
-	return PTR_ERR_OR_ZERO(axi->qos_regs);
-}
-
-static int bt1_axi_request_rst(struct bt1_axi *axi)
-{
-	int ret;
-
-	axi->arst = devm_reset_control_get_optional_exclusive(axi->dev, "arst");
-	if (IS_ERR(axi->arst))
-		return dev_err_probe(axi->dev, PTR_ERR(axi->arst),
-				     "Couldn't get reset control line\n");
-
-	ret = reset_control_deassert(axi->arst);
-	if (ret)
-		dev_err(axi->dev, "Failed to deassert the reset line\n");
-
-	return ret;
-}
-
-static int bt1_axi_request_clk(struct bt1_axi *axi)
-{
-	axi->aclk = devm_clk_get_enabled(axi->dev, "aclk");
-	if (IS_ERR(axi->aclk))
-		return dev_err_probe(axi->dev, PTR_ERR(axi->aclk),
-				     "Couldn't get AXI Interconnect clock\n");
-
-	return 0;
-}
-
-static int bt1_axi_request_irq(struct bt1_axi *axi)
-{
-	struct platform_device *pdev = to_platform_device(axi->dev);
-	int ret;
-
-	axi->irq = platform_get_irq(pdev, 0);
-	if (axi->irq < 0)
-		return axi->irq;
-
-	ret = devm_request_irq(axi->dev, axi->irq, bt1_axi_isr, IRQF_SHARED,
-			       "bt1-axi", axi);
-	if (ret)
-		dev_err(axi->dev, "Couldn't request AXI EHB IRQ\n");
-
-	return ret;
-}
-
-static ssize_t count_show(struct device *dev,
-			  struct device_attribute *attr, char *buf)
-{
-	struct bt1_axi *axi = dev_get_drvdata(dev);
-
-	return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&axi->count));
-}
-static DEVICE_ATTR_RO(count);
-
-static ssize_t inject_error_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	return scnprintf(buf, PAGE_SIZE, "Error injection: bus unaligned\n");
-}
-
-static ssize_t inject_error_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *data, size_t count)
-{
-	struct bt1_axi *axi = dev_get_drvdata(dev);
-
-	/*
-	 * Performing unaligned read from the memory will cause the CM2 bus
-	 * error while unaligned writing - the AXI bus write error handled
-	 * by this driver.
-	 */
-	if (sysfs_streq(data, "bus"))
-		readb(axi->qos_regs);
-	else if (sysfs_streq(data, "unaligned"))
-		writeb(0, axi->qos_regs);
-	else
-		return -EINVAL;
-
-	return count;
-}
-static DEVICE_ATTR_RW(inject_error);
-
-static struct attribute *bt1_axi_sysfs_attrs[] = {
-	&dev_attr_count.attr,
-	&dev_attr_inject_error.attr,
-	NULL
-};
-ATTRIBUTE_GROUPS(bt1_axi_sysfs);
-
-static void bt1_axi_remove_sysfs(void *data)
-{
-	struct bt1_axi *axi = data;
-
-	device_remove_groups(axi->dev, bt1_axi_sysfs_groups);
-}
-
-static int bt1_axi_init_sysfs(struct bt1_axi *axi)
-{
-	int ret;
-
-	ret = device_add_groups(axi->dev, bt1_axi_sysfs_groups);
-	if (ret) {
-		dev_err(axi->dev, "Failed to add sysfs files group\n");
-		return ret;
-	}
-
-	ret = devm_add_action_or_reset(axi->dev, bt1_axi_remove_sysfs, axi);
-	if (ret)
-		dev_err(axi->dev, "Can't add AXI EHB sysfs remove action\n");
-
-	return ret;
-}
-
-static int bt1_axi_probe(struct platform_device *pdev)
-{
-	struct bt1_axi *axi;
-	int ret;
-
-	axi = bt1_axi_create_data(pdev);
-	if (IS_ERR(axi))
-		return PTR_ERR(axi);
-
-	ret = bt1_axi_request_regs(axi);
-	if (ret)
-		return ret;
-
-	ret = bt1_axi_request_rst(axi);
-	if (ret)
-		return ret;
-
-	ret = bt1_axi_request_clk(axi);
-	if (ret)
-		return ret;
-
-	ret = bt1_axi_request_irq(axi);
-	if (ret)
-		return ret;
-
-	ret = bt1_axi_init_sysfs(axi);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static const struct of_device_id bt1_axi_of_match[] = {
-	{ .compatible = "baikal,bt1-axi" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, bt1_axi_of_match);
-
-static struct platform_driver bt1_axi_driver = {
-	.probe = bt1_axi_probe,
-	.driver = {
-		.name = "bt1-axi",
-		.of_match_table = bt1_axi_of_match
-	}
-};
-module_platform_driver(bt1_axi_driver);
-
-MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
-MODULE_DESCRIPTION("Baikal-T1 AXI-bus driver");
diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c
index db67442addad2..a85706826fa06 100644
--- a/drivers/bus/fsl-mc/dprc-driver.c
+++ b/drivers/bus/fsl-mc/dprc-driver.c
@@ -609,9 +609,8 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 {
 	struct device *parent_dev = mc_dev->dev.parent;
 	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev);
-	struct irq_domain *mc_msi_domain;
+	struct irq_domain *mc_msi_domain = NULL;
 	bool mc_io_created = false;
-	bool msi_domain_set = false;
 	bool uapi_created = false;
 	u16 major_ver, minor_ver;
 	size_t region_size;
@@ -652,14 +651,12 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 		uapi_created = true;
 	}
 
-	mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev);
-	if (!mc_msi_domain) {
+	mc_msi_domain = fsl_mc_get_msi_parent(&mc_dev->dev);
+	if (!mc_msi_domain)
 		dev_warn(&mc_dev->dev,
 			 "WARNING: MC bus without interrupt support\n");
-	} else {
+	else
 		dev_set_msi_domain(&mc_dev->dev, mc_msi_domain);
-		msi_domain_set = true;
-	}
 
 	error = dprc_open(mc_dev->mc_io, 0, mc_dev->obj_desc.id,
 			  &mc_dev->mc_handle);
@@ -699,8 +696,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 	(void)dprc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
 
 error_cleanup_msi_domain:
-	if (msi_domain_set)
-		dev_set_msi_domain(&mc_dev->dev, NULL);
+	dev_set_msi_domain(&mc_dev->dev, NULL);
 
 	if (mc_io_created) {
 		fsl_destroy_mc_io(mc_dev->mc_io);
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index c117745cf2065..1a6c8e3c0b27f 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -854,14 +854,12 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
 	} else {
 		/*
 		 * A non-DPRC object has to be a child of a DPRC, use the
-		 * parent's ICID and interrupt domain.
+		 * parent's ICID.
 		 */
 		mc_dev->icid = parent_mc_dev->icid;
 		mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
 		mc_dev->dev.dma_mask = &mc_dev->dma_mask;
 		mc_dev->dev.coherent_dma_mask = mc_dev->dma_mask;
-		dev_set_msi_domain(&mc_dev->dev,
-				   dev_get_msi_domain(&parent_mc_dev->dev));
 	}
 
 	/*
diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c
index 82cd69f7884c6..be38b43803dea 100644
--- a/drivers/bus/fsl-mc/fsl-mc-msi.c
+++ b/drivers/bus/fsl-mc/fsl-mc-msi.c
@@ -15,53 +15,16 @@
 
 #include "fsl-mc-private.h"
 
-#ifdef GENERIC_MSI_DOMAIN_OPS
-/*
- * Generate a unique ID identifying the interrupt (only used within the MSI
- * irqdomain.  Combine the icid with the interrupt index.
- */
-static irq_hw_number_t fsl_mc_domain_calc_hwirq(struct fsl_mc_device *dev,
-						struct msi_desc *desc)
-{
-	/*
-	 * Make the base hwirq value for ICID*10000 so it is readable
-	 * as a decimal value in /proc/interrupts.
-	 */
-	return (irq_hw_number_t)(desc->msi_index + (dev->icid * 10000));
-}
-
-static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg,
-				struct msi_desc *desc)
-{
-	arg->desc = desc;
-	arg->hwirq = fsl_mc_domain_calc_hwirq(to_fsl_mc_device(desc->dev),
-					      desc);
-}
-#else
-#define fsl_mc_msi_set_desc NULL
-#endif
-
-static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info)
+static void fsl_mc_write_msi_msg(struct msi_desc *msi_desc, struct msi_msg *msg)
 {
-	struct msi_domain_ops *ops = info->ops;
-
-	if (!ops)
-		return;
-
-	/*
-	 * set_desc should not be set by the caller
-	 */
-	if (!ops->set_desc)
-		ops->set_desc = fsl_mc_msi_set_desc;
-}
-
-static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev,
-				   struct fsl_mc_device_irq *mc_dev_irq,
-				   struct msi_desc *msi_desc)
-{
-	int error;
+	struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev);
+	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
+	struct fsl_mc_device_irq *mc_dev_irq = &mc_bus->irq_resources[msi_desc->msi_index];
 	struct fsl_mc_device *owner_mc_dev = mc_dev_irq->mc_dev;
 	struct dprc_irq_cfg irq_cfg;
+	int error;
+
+	msi_desc->msg = *msg;
 
 	/*
 	 * msi_desc->msg.address is 0x0 when this function is invoked in
@@ -110,118 +73,27 @@ static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev,
 	}
 }
 
-/*
- * NOTE: This function is invoked with interrupts disabled
- */
-static void fsl_mc_msi_write_msg(struct irq_data *irq_data,
-				 struct msi_msg *msg)
-{
-	struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
-	struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev);
-	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
-	struct fsl_mc_device_irq *mc_dev_irq =
-		&mc_bus->irq_resources[msi_desc->msi_index];
-
-	msi_desc->msg = *msg;
-
-	/*
-	 * Program the MSI (paddr, value) pair in the device:
-	 */
-	__fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq, msi_desc);
-}
-
-static void fsl_mc_msi_update_chip_ops(struct msi_domain_info *info)
-{
-	struct irq_chip *chip = info->chip;
-
-	if (!chip)
-		return;
-
-	/*
-	 * irq_write_msi_msg should not be set by the caller
-	 */
-	if (!chip->irq_write_msi_msg)
-		chip->irq_write_msi_msg = fsl_mc_msi_write_msg;
-}
-
-/**
- * fsl_mc_msi_create_irq_domain - Create a fsl-mc MSI interrupt domain
- * @fwnode:	Optional firmware node of the interrupt controller
- * @info:	MSI domain info
- * @parent:	Parent irq domain
- *
- * Updates the domain and chip ops and creates a fsl-mc MSI
- * interrupt domain.
- *
- * Returns:
- * A domain pointer or NULL in case of failure.
- */
-struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
-						struct msi_domain_info *info,
-						struct irq_domain *parent)
-{
-	struct irq_domain *domain;
-
-	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE)))
-		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
-	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
-		fsl_mc_msi_update_dom_ops(info);
-	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
-		fsl_mc_msi_update_chip_ops(info);
-	info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS;
-
-	domain = msi_create_irq_domain(fwnode, info, parent);
-	if (domain)
-		irq_domain_update_bus_token(domain, DOMAIN_BUS_FSL_MC_MSI);
-
-	return domain;
-}
-
-struct irq_domain *fsl_mc_find_msi_domain(struct device *dev)
+struct irq_domain *fsl_mc_get_msi_parent(struct device *dev)
 {
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
 	struct device *root_dprc_dev;
 	struct device *bus_dev;
-	struct irq_domain *msi_domain;
-	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
 
 	fsl_mc_get_root_dprc(dev, &root_dprc_dev);
 	bus_dev = root_dprc_dev->parent;
 
-	if (bus_dev->of_node) {
-		msi_domain = of_msi_map_get_device_domain(dev,
-						  mc_dev->icid,
-						  DOMAIN_BUS_FSL_MC_MSI);
-
-		/*
-		 * if the msi-map property is missing assume that all the
-		 * child containers inherit the domain from the parent
-		 */
-		if (!msi_domain)
-
-			msi_domain = of_msi_get_domain(bus_dev,
-						bus_dev->of_node,
-						DOMAIN_BUS_FSL_MC_MSI);
-	} else {
-		msi_domain = iort_get_device_domain(dev, mc_dev->icid,
-						    DOMAIN_BUS_FSL_MC_MSI);
-	}
-
-	return msi_domain;
+	return (bus_dev->of_node ?
+		of_msi_get_domain(bus_dev, bus_dev->of_node, DOMAIN_BUS_NEXUS) :
+		iort_get_device_domain(bus_dev, mc_dev->icid, DOMAIN_BUS_NEXUS));
 }
 
 int fsl_mc_msi_domain_alloc_irqs(struct device *dev,  unsigned int irq_count)
 {
 	int error = msi_setup_device_data(dev);
-
 	if (error)
 		return error;
 
-	/*
-	 * NOTE: Calling this function will trigger the invocation of the
-	 * its_fsl_mc_msi_prepare() callback
-	 */
-	error = msi_domain_alloc_irqs_range(dev, MSI_DEFAULT_DOMAIN, 0, irq_count - 1);
-
+	error = platform_device_msi_init_and_alloc_irqs(dev, irq_count, fsl_mc_write_msi_msg);
 	if (error)
 		dev_err(dev, "Failed to allocate IRQs\n");
 	return error;
@@ -231,3 +103,15 @@ void fsl_mc_msi_domain_free_irqs(struct device *dev)
 {
 	msi_domain_free_irqs_all(dev, MSI_DEFAULT_DOMAIN);
 }
+
+u32 fsl_mc_get_msi_id(struct device *dev)
+{
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
+	struct device *root_dprc_dev;
+
+	fsl_mc_get_root_dprc(dev, &root_dprc_dev);
+
+	return (root_dprc_dev->parent->of_node ?
+		of_msi_xlate(dev, NULL, mc_dev->icid) :
+		iort_msi_map_id(dev, mc_dev->icid));
+}
diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h
index beed4c53533d8..197edcc8cde46 100644
--- a/drivers/bus/fsl-mc/fsl-mc-private.h
+++ b/drivers/bus/fsl-mc/fsl-mc-private.h
@@ -641,7 +641,7 @@ int fsl_mc_msi_domain_alloc_irqs(struct device *dev,
 
 void fsl_mc_msi_domain_free_irqs(struct device *dev);
 
-struct irq_domain *fsl_mc_find_msi_domain(struct device *dev);
+struct irq_domain *fsl_mc_get_msi_parent(struct device *dev);
 
 int __must_check fsl_create_mc_io(struct device *dev,
 				  phys_addr_t mc_portal_phys_addr,
diff --git a/drivers/bus/imx-aipstz.c b/drivers/bus/imx-aipstz.c
index 5fdf377f5d063..83371e5b35a2d 100644
--- a/drivers/bus/imx-aipstz.c
+++ b/drivers/bus/imx-aipstz.c
@@ -11,9 +11,19 @@
 #include <linux/regmap.h>
 
 #define IMX_AIPSTZ_MPR0 0x0
+#define IMX_AIPSTZ_OPACR0 0x40
+#define IMX_AIPSTZ_OPACR1 0x44
+#define IMX_AIPSTZ_OPACR2 0x48
+#define IMX_AIPSTZ_OPACR3 0x4c
+#define IMX_AIPSTZ_OPACR4 0x50
 
 struct imx_aipstz_config {
 	u32 mpr0;
+	u32 opacr0;
+	u32 opacr1;
+	u32 opacr2;
+	u32 opacr3;
+	u32 opacr4;
 };
 
 struct imx_aipstz_data {
@@ -24,6 +34,11 @@ struct imx_aipstz_data {
 static void imx_aipstz_apply_default(struct imx_aipstz_data *data)
 {
 	writel(data->default_cfg->mpr0, data->base + IMX_AIPSTZ_MPR0);
+	writel(data->default_cfg->opacr0, data->base + IMX_AIPSTZ_OPACR0);
+	writel(data->default_cfg->opacr1, data->base + IMX_AIPSTZ_OPACR1);
+	writel(data->default_cfg->opacr2, data->base + IMX_AIPSTZ_OPACR2);
+	writel(data->default_cfg->opacr3, data->base + IMX_AIPSTZ_OPACR3);
+	writel(data->default_cfg->opacr4, data->base + IMX_AIPSTZ_OPACR4);
 }
 
 static const struct of_device_id imx_aipstz_match_table[] = {
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index e3d0a3cbaf944..0277e1ab11988 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -367,7 +367,7 @@ static void mhi_ep_read_completion(struct mhi_ep_buf_info *buf_info)
 				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
 							     MHI_TRE_DATA_GET_LEN(el),
 							     MHI_EV_CC_EOB);
-				if (ret < 0) {
+				if (ret) {
 					dev_err(&mhi_chan->mhi_dev->dev,
 						"Error sending transfer compl. event\n");
 					goto err_free_tre_buf;
@@ -383,7 +383,7 @@ static void mhi_ep_read_completion(struct mhi_ep_buf_info *buf_info)
 				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
 							     MHI_TRE_DATA_GET_LEN(el),
 							     MHI_EV_CC_EOT);
-				if (ret < 0) {
+				if (ret) {
 					dev_err(&mhi_chan->mhi_dev->dev,
 						"Error sending transfer compl. event\n");
 					goto err_free_tre_buf;
@@ -449,7 +449,7 @@ static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl,
 
 		dev_dbg(dev, "Reading %zd bytes from channel (%u)\n", tr_len, ring->ch_id);
 		ret = mhi_cntrl->read_async(mhi_cntrl, &buf_info);
-		if (ret < 0) {
+		if (ret) {
 			dev_err(&mhi_chan->mhi_dev->dev, "Error reading from channel\n");
 			goto err_free_buf_addr;
 		}
@@ -494,7 +494,7 @@ static int mhi_ep_process_ch_ring(struct mhi_ep_ring *ring)
 	} else {
 		/* UL channel */
 		ret = mhi_ep_read_channel(mhi_cntrl, ring);
-		if (ret < 0) {
+		if (ret) {
 			dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n");
 			return ret;
 		}
@@ -591,7 +591,7 @@ int mhi_ep_queue_skb(struct mhi_ep_device *mhi_dev, struct sk_buff *skb)
 
 		dev_dbg(dev, "Writing %zd bytes to channel (%u)\n", tr_len, ring->ch_id);
 		ret = mhi_cntrl->write_async(mhi_cntrl, &buf_info);
-		if (ret < 0) {
+		if (ret) {
 			dev_err(dev, "Error writing to the channel\n");
 			goto err_exit;
 		}
diff --git a/drivers/bus/mhi/ep/ring.c b/drivers/bus/mhi/ep/ring.c
index 9375b16ff2a5d..405ce16c02a89 100644
--- a/drivers/bus/mhi/ep/ring.c
+++ b/drivers/bus/mhi/ep/ring.c
@@ -49,7 +49,7 @@ static int __mhi_ep_cache_ring(struct mhi_ep_ring *ring, size_t end)
 		buf_info.dev_addr = &ring->ring_cache[start];
 
 		ret = mhi_cntrl->read_sync(mhi_cntrl, &buf_info);
-		if (ret < 0)
+		if (ret)
 			return ret;
 	} else {
 		buf_info.size = (ring->ring_size - start) * sizeof(struct mhi_ring_element);
@@ -57,7 +57,7 @@ static int __mhi_ep_cache_ring(struct mhi_ep_ring *ring, size_t end)
 		buf_info.dev_addr = &ring->ring_cache[start];
 
 		ret = mhi_cntrl->read_sync(mhi_cntrl, &buf_info);
-		if (ret < 0)
+		if (ret)
 			return ret;
 
 		if (end) {
@@ -66,7 +66,7 @@ static int __mhi_ep_cache_ring(struct mhi_ep_ring *ring, size_t end)
 			buf_info.size = end * sizeof(struct mhi_ring_element);
 
 			ret = mhi_cntrl->read_sync(mhi_cntrl, &buf_info);
-			if (ret < 0)
+			if (ret)
 				return ret;
 		}
 	}
diff --git a/drivers/bus/mhi/host/boot.c b/drivers/bus/mhi/host/boot.c
index f16a1e67a667d..19c84913cfb95 100644
--- a/drivers/bus/mhi/host/boot.c
+++ b/drivers/bus/mhi/host/boot.c
@@ -308,7 +308,6 @@ static void mhi_free_bhi_buffer(struct mhi_controller *mhi_cntrl,
 	struct mhi_buf *mhi_buf = image_info->mhi_buf;
 
 	dma_free_coherent(mhi_cntrl->cntrl_dev, mhi_buf->len, mhi_buf->buf, mhi_buf->dma_addr);
-	kfree(image_info->mhi_buf);
 	kfree(image_info);
 }
 
@@ -322,7 +321,6 @@ void mhi_free_bhie_table(struct mhi_controller *mhi_cntrl,
 		dma_free_coherent(mhi_cntrl->cntrl_dev, mhi_buf->len,
 				  mhi_buf->buf, mhi_buf->dma_addr);
 
-	kfree(image_info->mhi_buf);
 	kfree(image_info);
 }
 
@@ -333,15 +331,10 @@ static int mhi_alloc_bhi_buffer(struct mhi_controller *mhi_cntrl,
 	struct image_info *img_info;
 	struct mhi_buf *mhi_buf;
 
-	img_info = kzalloc_obj(*img_info);
+	img_info = kzalloc_flex(*img_info, mhi_buf, 1);
 	if (!img_info)
 		return -ENOMEM;
 
-	/* Allocate memory for entry */
-	img_info->mhi_buf = kzalloc_obj(*img_info->mhi_buf);
-	if (!img_info->mhi_buf)
-		goto error_alloc_mhi_buf;
-
 	/* Allocate and populate vector table */
 	mhi_buf = img_info->mhi_buf;
 
@@ -358,8 +351,6 @@ static int mhi_alloc_bhi_buffer(struct mhi_controller *mhi_cntrl,
 	return 0;
 
 error_alloc_segment:
-	kfree(mhi_buf);
-error_alloc_mhi_buf:
 	kfree(img_info);
 
 	return -ENOMEM;
@@ -375,14 +366,11 @@ int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl,
 	struct image_info *img_info;
 	struct mhi_buf *mhi_buf;
 
-	img_info = kzalloc_obj(*img_info);
+	img_info = kzalloc_flex(*img_info, mhi_buf, segments);
 	if (!img_info)
 		return -ENOMEM;
 
-	/* Allocate memory for entries */
-	img_info->mhi_buf = kzalloc_objs(*img_info->mhi_buf, segments);
-	if (!img_info->mhi_buf)
-		goto error_alloc_mhi_buf;
+	img_info->entries = segments;
 
 	/* Allocate and populate vector table */
 	mhi_buf = img_info->mhi_buf;
@@ -402,7 +390,6 @@ int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl,
 	}
 
 	img_info->bhi_vec = img_info->mhi_buf[segments - 1].buf;
-	img_info->entries = segments;
 	*image_info = img_info;
 
 	return 0;
@@ -411,9 +398,6 @@ int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl,
 	for (--i, --mhi_buf; i >= 0; i--, mhi_buf--)
 		dma_free_coherent(mhi_cntrl->cntrl_dev, mhi_buf->len,
 				  mhi_buf->buf, mhi_buf->dma_addr);
-	kfree(img_info->mhi_buf);
-
-error_alloc_mhi_buf:
 	kfree(img_info);
 
 	return -ENOMEM;
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index 0884a384b77fc..391ab146f501c 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -253,6 +253,13 @@ static const struct mhi_channel_config mhi_qcom_qdu100_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(41, "MHI_PHC", 32, 4),
 	MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 256, 5),
 	MHI_CHANNEL_CONFIG_DL(47, "IP_SW0", 256, 5),
+	MHI_CHANNEL_CONFIG_UL(48, "IP_SW1", 256, 6),
+	MHI_CHANNEL_CONFIG_DL(49, "IP_SW1", 256, 6),
+	MHI_CHANNEL_CONFIG_UL(50, "IP_ETH0", 256, 7),
+	MHI_CHANNEL_CONFIG_DL(51, "IP_ETH0", 256, 7),
+	MHI_CHANNEL_CONFIG_UL(52, "IP_ETH1", 256, 8),
+	MHI_CHANNEL_CONFIG_DL(53, "IP_ETH1", 256, 8),
+
 };
 
 static struct mhi_event_config mhi_qcom_qdu100_events[] = {
@@ -268,6 +275,7 @@ static struct mhi_event_config mhi_qcom_qdu100_events[] = {
 	MHI_EVENT_CONFIG_SW_DATA(5, 512),
 	MHI_EVENT_CONFIG_SW_DATA(6, 512),
 	MHI_EVENT_CONFIG_SW_DATA(7, 512),
+	MHI_EVENT_CONFIG_SW_DATA(8, 512),
 };
 
 static const struct mhi_controller_config mhi_qcom_qdu100_config = {
@@ -407,6 +415,16 @@ static const struct mhi_pci_dev_info mhi_qcom_sdx55_info = {
 	.sideband_wake = false,
 };
 
+static const struct mhi_pci_dev_info mhi_qcom_sdx35_info = {
+	.name = "qcom-sdx35m",
+	.config = &modem_qcom_v2_mhiv_config,
+	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+	.dma_data_width = 32,
+	.mru_default = 32768,
+	.sideband_wake = false,
+	.edl_trigger = true,
+};
+
 static const struct mhi_pci_dev_info mhi_qcom_sdx24_info = {
 	.name = "qcom-sdx24",
 	.edl = "qcom/prog_firehose_sdx24.mbn",
@@ -788,6 +806,8 @@ static const struct mhi_channel_config mhi_telit_fn990_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
 	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
+	MHI_CHANNEL_CONFIG_UL(94, "NMEA", 32, 1),
+	MHI_CHANNEL_CONFIG_DL(95, "NMEA", 32, 1),
 	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
 	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
 };
@@ -839,6 +859,8 @@ static const struct mhi_channel_config mhi_telit_fn920c04_channels[] = {
 	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
 	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
+	MHI_CHANNEL_CONFIG_UL(94, "NMEA", 32, 1),
+	MHI_CHANNEL_CONFIG_DL(95, "NMEA", 32, 1),
 	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2),
 	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3),
 };
@@ -909,6 +931,8 @@ static const struct pci_device_id mhi_pci_id_table[] = {
 	/* Telit FN920C04 (sdx35) */
 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x2020),
 		.driver_data = (kernel_ulong_t) &mhi_telit_fn920c04_info },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x011a),
+		.driver_data = (kernel_ulong_t) &mhi_qcom_sdx35_info },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0304),
 		.driver_data = (kernel_ulong_t) &mhi_qcom_sdx24_info },
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0306, PCI_VENDOR_ID_QCOM, 0x010c),
@@ -1393,7 +1417,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_unregister;
 	}
 
-	err = mhi_sync_power_up(mhi_cntrl);
+	err = mhi_async_power_up(mhi_cntrl);
 	if (err) {
 		dev_err(&pdev->dev, "failed to power up MHI controller\n");
 		goto err_unprepare;
@@ -1428,6 +1452,7 @@ static void mhi_pci_remove(struct pci_dev *pdev)
 	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
 	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
 
+	pm_runtime_forbid(&pdev->dev);
 	pci_disable_sriov(pdev);
 
 	if (pdev->is_physfn)
diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c
index 3f00d953fb9a0..c920bd6fbaafd 100644
--- a/drivers/bus/simple-pm-bus.c
+++ b/drivers/bus/simple-pm-bus.c
@@ -36,7 +36,7 @@ static int simple_pm_bus_probe(struct platform_device *pdev)
 	 * that's not listed in simple_pm_bus_of_match. We don't want to do any
 	 * of the simple-pm-bus tasks for these devices, so return early.
 	 */
-	if (pdev->driver_override)
+	if (device_has_driver_override(&pdev->dev))
 		return 0;
 
 	match = of_match_device(dev->driver->of_match_table, dev);
@@ -78,7 +78,7 @@ static void simple_pm_bus_remove(struct platform_device *pdev)
 {
 	const void *data = of_device_get_match_data(&pdev->dev);
 
-	if (pdev->driver_override || data)
+	if (device_has_driver_override(&pdev->dev) || data)
 		return;
 
 	dev_dbg(&pdev->dev, "%s\n", __func__);
diff --git a/drivers/bus/stm32_dbg_bus.c b/drivers/bus/stm32_dbg_bus.c
new file mode 100644
index 0000000000000..e30ef3465609e
--- /dev/null
+++ b/drivers/bus/stm32_dbg_bus.c
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, STMicroelectronics - All Rights Reserved
+ */
+
+#include <linux/bus/stm32_firewall.h>
+#include <linux/bus/stm32_firewall_device.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/tee_drv.h>
+#include <linux/types.h>
+
+enum stm32_dbg_profile {
+	PERIPHERAL_DBG_PROFILE	= 0,
+	HDP_DBG_PROFILE		= 1,
+};
+
+enum stm32_dbg_pta_command {
+	/*
+	 * PTA_CMD_GRANT_DBG_ACCESS - Verify the debug configuration against the given debug profile
+	 * and grant access or not
+	 *
+	 * [in]     value[0].a  Debug profile to grant access to.
+	 */
+	PTA_CMD_GRANT_DBG_ACCESS,
+};
+
+/**
+ * struct stm32_dbg_bus - OP-TEE based STM32 debug bus private data
+ * @dev: STM32 debug bus device.
+ * @ctx: OP-TEE context handler.
+ */
+struct stm32_dbg_bus {
+	struct device *dev;
+	struct tee_context *ctx;
+};
+
+/* Expect at most 1 instance of this driver */
+static struct stm32_dbg_bus *stm32_dbg_bus_priv;
+
+static int stm32_dbg_pta_open_session(u32 *id)
+{
+	struct tee_client_device *dbg_bus_dev = to_tee_client_device(stm32_dbg_bus_priv->dev);
+	struct tee_ioctl_open_session_arg sess_arg;
+	int ret;
+
+	memset(&sess_arg, 0, sizeof(sess_arg));
+	export_uuid(sess_arg.uuid, &dbg_bus_dev->id.uuid);
+	sess_arg.clnt_login = TEE_IOCTL_LOGIN_REE_KERNEL;
+
+	ret = tee_client_open_session(stm32_dbg_bus_priv->ctx, &sess_arg, NULL);
+	if (ret < 0 || sess_arg.ret) {
+		dev_err(stm32_dbg_bus_priv->dev, "Failed opening tee session, err: %#x\n",
+			sess_arg.ret);
+		return -EOPNOTSUPP;
+	}
+
+	*id = sess_arg.session;
+
+	return 0;
+}
+
+static void stm32_dbg_pta_close_session(u32 id)
+{
+	tee_client_close_session(stm32_dbg_bus_priv->ctx, id);
+}
+
+static int stm32_dbg_bus_grant_access(struct stm32_firewall_controller *ctrl, u32 dbg_profile)
+{
+	struct tee_ioctl_invoke_arg inv_arg = {0};
+	struct tee_param param[1] = {0};
+	u32 session_id;
+	int ret;
+
+	if (dbg_profile != PERIPHERAL_DBG_PROFILE && dbg_profile != HDP_DBG_PROFILE)
+		return -EOPNOTSUPP;
+
+	ret = stm32_dbg_pta_open_session(&session_id);
+	if (ret)
+		return ret;
+
+	inv_arg.func = PTA_CMD_GRANT_DBG_ACCESS;
+	inv_arg.session = session_id;
+	inv_arg.num_params = 1;
+	param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT;
+	param[0].u.value.a = dbg_profile;
+
+	ret = tee_client_invoke_func(stm32_dbg_bus_priv->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_dbg(stm32_dbg_bus_priv->dev,
+			"When invoking function, err %x, TEE returns: %x\n", ret, inv_arg.ret);
+		if (!ret)
+			ret = -EACCES;
+	}
+
+	stm32_dbg_pta_close_session(session_id);
+
+	return ret;
+}
+
+/* Implement mandatory release_access ops even if it does nothing*/
+static void stm32_dbg_bus_release_access(struct stm32_firewall_controller *ctrl, u32 dbg_profile)
+{
+}
+
+static int stm32_dbg_bus_plat_probe(struct platform_device *pdev)
+{
+	struct stm32_firewall_controller *dbg_controller;
+	int ret;
+
+	/* Defer if OP-TEE service is not yet available */
+	if (!stm32_dbg_bus_priv)
+		return -EPROBE_DEFER;
+
+	dbg_controller = devm_kzalloc(&pdev->dev, sizeof(*dbg_controller), GFP_KERNEL);
+	if (!dbg_controller)
+		return dev_err_probe(&pdev->dev, -ENOMEM, "Couldn't allocate debug controller\n");
+
+	dbg_controller->dev = &pdev->dev;
+	dbg_controller->mmio = NULL;
+	dbg_controller->name = dev_driver_string(dbg_controller->dev);
+	dbg_controller->type = STM32_PERIPHERAL_FIREWALL;
+	dbg_controller->grant_access = stm32_dbg_bus_grant_access;
+	dbg_controller->release_access = stm32_dbg_bus_release_access;
+
+	ret = stm32_firewall_controller_register(dbg_controller);
+	if (ret) {
+		dev_err(dbg_controller->dev, "Couldn't register as a firewall controller: %d", ret);
+		return ret;
+	}
+
+	ret = stm32_firewall_populate_bus(dbg_controller);
+	if (ret) {
+		dev_err(dbg_controller->dev, "Couldn't populate debug bus: %d", ret);
+		stm32_firewall_controller_unregister(dbg_controller);
+		return ret;
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	ret = of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
+	if (ret) {
+		dev_err(dbg_controller->dev, "Couldn't populate the node: %d", ret);
+		stm32_firewall_controller_unregister(dbg_controller);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id stm32_dbg_bus_of_match[] = {
+	{ .compatible = "st,stm32mp131-dbg-bus", },
+	{ .compatible = "st,stm32mp151-dbg-bus", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, stm32_dbg_bus_of_match);
+
+static struct platform_driver stm32_dbg_bus_driver = {
+	.probe = stm32_dbg_bus_plat_probe,
+	.driver = {
+		.name = "stm32-dbg-bus",
+		.of_match_table = stm32_dbg_bus_of_match,
+	},
+};
+
+static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data)
+{
+	return (ver->impl_id == TEE_IMPL_ID_OPTEE);
+}
+
+static void stm32_dbg_bus_remove(struct tee_client_device *tee_dev)
+{
+	tee_client_close_context(stm32_dbg_bus_priv->ctx);
+	stm32_dbg_bus_priv = NULL;
+
+	of_platform_depopulate(&tee_dev->dev);
+}
+
+static int stm32_dbg_bus_probe(struct tee_client_device *tee_dev)
+{
+	struct device *dev = &tee_dev->dev;
+	struct stm32_dbg_bus *priv;
+	int ret = 0;
+
+	if (stm32_dbg_bus_priv)
+		return dev_err_probe(dev, -EBUSY,
+				     "A STM32 debug bus device is already initialized\n");
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	/* Open context with TEE driver */
+	priv->ctx = tee_client_open_context(NULL, optee_ctx_match, NULL, NULL);
+	if (IS_ERR_OR_NULL(priv->ctx))
+		return dev_err_probe(dev, PTR_ERR_OR_ZERO(priv->ctx), "Cannot open TEE context\n");
+
+	stm32_dbg_bus_priv = priv;
+	stm32_dbg_bus_priv->dev = dev;
+
+	return ret;
+}
+
+static const struct tee_client_device_id optee_dbg_bus_id_table[] = {
+	{UUID_INIT(0xdd05bc8b, 0x9f3b, 0x49f0,
+		   0xb6, 0x49, 0x01, 0xaa, 0x10, 0xc1, 0xc2, 0x10)},
+	{}
+};
+
+static struct tee_client_driver stm32_optee_dbg_bus_driver = {
+	.id_table = optee_dbg_bus_id_table,
+	.probe = stm32_dbg_bus_probe,
+	.remove = stm32_dbg_bus_remove,
+	.driver = {
+		.name = "optee_dbg_bus",
+	},
+};
+
+static void __exit stm32_optee_dbg_bus_driver_exit(void)
+{
+	platform_driver_unregister(&stm32_dbg_bus_driver);
+	tee_client_driver_unregister(&stm32_optee_dbg_bus_driver);
+}
+module_exit(stm32_optee_dbg_bus_driver_exit);
+
+static int __init stm32_optee_dbg_bus_driver_init(void)
+{
+	int err;
+
+	err = tee_client_driver_register(&stm32_optee_dbg_bus_driver);
+	if (err)
+		return err;
+
+	err = platform_driver_register(&stm32_dbg_bus_driver);
+	if (err)
+		tee_client_driver_unregister(&stm32_optee_dbg_bus_driver);
+
+	return err;
+}
+module_init(stm32_optee_dbg_bus_driver_init);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Gatien Chevallier <gatien.chevallier@foss.st.com>");
+MODULE_DESCRIPTION("OP-TEE based STM32 debug access bus driver");
diff --git a/drivers/bus/stm32_etzpc.c b/drivers/bus/stm32_etzpc.c
index 7fc0f16960be3..4918a14e507ef 100644
--- a/drivers/bus/stm32_etzpc.c
+++ b/drivers/bus/stm32_etzpc.c
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/bus/stm32_firewall.h>
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/init.h>
@@ -16,8 +17,6 @@
 #include <linux/platform_device.h>
 #include <linux/types.h>
 
-#include "stm32_firewall.h"
-
 /*
  * ETZPC registers
  */
diff --git a/drivers/bus/stm32_firewall.c b/drivers/bus/stm32_firewall.c
index 7e7afe8007dbc..e3619dba8b069 100644
--- a/drivers/bus/stm32_firewall.c
+++ b/drivers/bus/stm32_firewall.c
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/bus/stm32_firewall.h>
 #include <linux/bus/stm32_firewall_device.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -18,8 +19,6 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 
-#include "stm32_firewall.h"
-
 /* Corresponds to STM32_FIREWALL_MAX_EXTRA_ARGS + firewall ID */
 #define STM32_FIREWALL_MAX_ARGS		(STM32_FIREWALL_MAX_EXTRA_ARGS + 1)
 
@@ -185,6 +184,48 @@ void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall, u32 su
 }
 EXPORT_SYMBOL_GPL(stm32_firewall_release_access_by_id);
 
+int stm32_firewall_get_grant_all_access(struct device *dev, struct stm32_firewall **firewall,
+					int *nb_firewall)
+{
+	struct stm32_firewall *loc_firewall;
+	int err;
+	int i;
+
+	*nb_firewall = of_count_phandle_with_args(dev->of_node, "access-controllers",
+						  "#access-controller-cells");
+	if (*nb_firewall < 0)
+		return *nb_firewall;
+
+	if (!*nb_firewall) {
+		*firewall = NULL;
+		return 0;
+	}
+
+	loc_firewall = devm_kcalloc(dev, *nb_firewall, sizeof(*loc_firewall), GFP_KERNEL);
+	if (!loc_firewall)
+		return -ENOMEM;
+
+	/* Get stm32 firewall information */
+	err = stm32_firewall_get_firewall(dev->of_node, loc_firewall, *nb_firewall);
+	if (err)
+		return err;
+
+	for (i = 0; i < *nb_firewall; i++) {
+		err = stm32_firewall_grant_access(&loc_firewall[i]);
+		if (err) {
+			while (i--)
+				stm32_firewall_release_access(&loc_firewall[i]);
+
+			return err;
+		}
+	}
+
+	*firewall = loc_firewall;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(stm32_firewall_get_grant_all_access);
+
 /* Firewall controller API */
 
 int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller)
@@ -241,7 +282,6 @@ EXPORT_SYMBOL_GPL(stm32_firewall_controller_unregister);
 int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller)
 {
 	struct stm32_firewall *firewalls;
-	struct device_node *child;
 	struct device *parent;
 	unsigned int i;
 	int len;
@@ -251,30 +291,25 @@ int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_contr
 
 	dev_dbg(parent, "Populating %s system bus\n", dev_name(firewall_controller->dev));
 
-	for_each_available_child_of_node(dev_of_node(parent), child) {
+	for_each_available_child_of_node_scoped(dev_of_node(parent), child) {
 		/* The access-controllers property is mandatory for firewall bus devices */
 		len = of_count_phandle_with_args(child, "access-controllers",
 						 "#access-controller-cells");
-		if (len <= 0) {
-			of_node_put(child);
+		if (len <= 0)
 			return -EINVAL;
-		}
 
 		firewalls = kzalloc_objs(*firewalls, len);
-		if (!firewalls) {
-			of_node_put(child);
+		if (!firewalls)
 			return -ENOMEM;
-		}
 
 		err = stm32_firewall_get_firewall(child, firewalls, (unsigned int)len);
 		if (err) {
 			kfree(firewalls);
-			of_node_put(child);
 			return err;
 		}
 
 		for (i = 0; i < len; i++) {
-			if (firewall_controller->grant_access(firewall_controller,
+			if (firewall_controller->grant_access(firewalls[i].firewall_ctrl,
 							      firewalls[i].firewall_id)) {
 				/*
 				 * Peripheral access not allowed or not defined.
diff --git a/drivers/bus/stm32_firewall.h b/drivers/bus/stm32_firewall.h
deleted file mode 100644
index e5fac85fe3460..0000000000000
--- a/drivers/bus/stm32_firewall.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2023, STMicroelectronics - All Rights Reserved
- */
-
-#ifndef _STM32_FIREWALL_H
-#define _STM32_FIREWALL_H
-
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/types.h>
-
-/**
- * STM32_PERIPHERAL_FIREWALL:		This type of firewall protects peripherals
- * STM32_MEMORY_FIREWALL:		This type of firewall protects memories/subsets of memory
- *					zones
- * STM32_NOTYPE_FIREWALL:		Undefined firewall type
- */
-
-#define STM32_PERIPHERAL_FIREWALL	BIT(1)
-#define STM32_MEMORY_FIREWALL		BIT(2)
-#define STM32_NOTYPE_FIREWALL		BIT(3)
-
-/**
- * struct stm32_firewall_controller - Information on firewall controller supplying services
- *
- * @name:			Name of the firewall controller
- * @dev:			Device reference of the firewall controller
- * @mmio:			Base address of the firewall controller
- * @entry:			List entry of the firewall controller list
- * @type:			Type of firewall
- * @max_entries:		Number of entries covered by the firewall
- * @grant_access:		Callback used to grant access for a device access against a
- *				firewall controller
- * @release_access:		Callback used to release resources taken by a device when access was
- *				granted
- * @grant_memory_range_access:	Callback used to grant access for a device to a given memory region
- */
-struct stm32_firewall_controller {
-	const char *name;
-	struct device *dev;
-	void __iomem *mmio;
-	struct list_head entry;
-	unsigned int type;
-	unsigned int max_entries;
-
-	int (*grant_access)(struct stm32_firewall_controller *ctrl, u32 id);
-	void (*release_access)(struct stm32_firewall_controller *ctrl, u32 id);
-	int (*grant_memory_range_access)(struct stm32_firewall_controller *ctrl, phys_addr_t paddr,
-					 size_t size);
-};
-
-/**
- * stm32_firewall_controller_register - Register a firewall controller to the STM32 firewall
- *					framework
- * @firewall_controller:	Firewall controller to register
- *
- * Returns 0 in case of success or -ENODEV if no controller was given.
- */
-int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller);
-
-/**
- * stm32_firewall_controller_unregister - Unregister a firewall controller from the STM32
- *					  firewall framework
- * @firewall_controller:	Firewall controller to unregister
- */
-void stm32_firewall_controller_unregister(struct stm32_firewall_controller *firewall_controller);
-
-/**
- * stm32_firewall_populate_bus - Populate device tree nodes that have a correct firewall
- *				 configuration. This is used at boot-time only, as a sanity check
- *				 between device tree and firewalls hardware configurations to
- *				 prevent a kernel crash when a device driver is not granted access
- *
- * @firewall_controller:	Firewall controller which nodes will be populated or not
- *
- * Returns 0 in case of success or appropriate errno code if error occurred.
- */
-int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller);
-
-#endif /* _STM32_FIREWALL_H */
diff --git a/drivers/bus/stm32_rifsc.c b/drivers/bus/stm32_rifsc.c
index debeaf8ea1bd2..19d10379dcefb 100644
--- a/drivers/bus/stm32_rifsc.c
+++ b/drivers/bus/stm32_rifsc.c
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/bus/stm32_firewall.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -15,10 +16,9 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/string.h>
 #include <linux/types.h>
 
-#include "stm32_firewall.h"
-
 /*
  * RIFSC offset register
  */
@@ -450,7 +450,7 @@ static void stm32_rifsc_fill_rimu_dbg_entry(struct rifsc_dbg_private *rifsc,
 	const struct stm32_rifsc_resources_names *dbg_names = rifsc->res_names;
 	u32 rimc_attr = readl_relaxed(rifsc->mmio + RIFSC_RIMC_ATTR0 + 0x4 * i);
 
-	snprintf(dbg_entry->m_name, sizeof(dbg_entry->m_name), "%s", dbg_names->initiator_names[i]);
+	strscpy(dbg_entry->m_name, dbg_names->initiator_names[i]);
 	dbg_entry->m_cid = FIELD_GET(RIFSC_RIMC_MCID_MASK, rimc_attr);
 	dbg_entry->cidsel = rimc_attr & RIFSC_RIMC_CIDSEL;
 	dbg_entry->m_sec = rimc_attr & RIFSC_RIMC_MSEC;
@@ -469,8 +469,7 @@ static void stm32_rifsc_fill_dev_dbg_entry(struct rifsc_dbg_private *rifsc,
 	sec_cfgr = readl_relaxed(rifsc->mmio + RIFSC_RISC_SECCFGR0 + 0x4 * reg_id);
 	priv_cfgr = readl_relaxed(rifsc->mmio + RIFSC_RISC_PRIVCFGR0 + 0x4 * reg_id);
 
-	snprintf(dbg_entry->dev_name, sizeof(dbg_entry->dev_name), "%s",
-		 dbg_names->device_names[i]);
+	strscpy(dbg_entry->dev_name, dbg_names->device_names[i]);
 	dbg_entry->dev_id = i;
 	dbg_entry->dev_cid_filt_en = cid_cfgr & CIDCFGR_CFEN;
 	dbg_entry->dev_sem_en = cid_cfgr & CIDCFGR_SEMEN;
@@ -688,34 +687,6 @@ static int stm32_rifsc_grant_access(struct stm32_firewall_controller *ctrl, u32
 	sec_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_SECCFGR0 + 0x4 * reg_id);
 	cid_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_PER0_CIDCFGR + 0x8 * firewall_id);
 
-	/* First check conditions for semaphore mode, which doesn't take into account static CID. */
-	if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) {
-		if (cid_reg_value & BIT(RIF_CID1 + SEMWL_SHIFT)) {
-			/* Static CID is irrelevant if semaphore mode */
-			goto skip_cid_check;
-		} else {
-			dev_dbg(rifsc_controller->dev,
-				"Invalid bus semaphore configuration: index %d\n", firewall_id);
-			return -EACCES;
-		}
-	}
-
-	/*
-	 * Skip CID check if CID filtering isn't enabled or filtering is enabled on CID0, which
-	 * corresponds to whatever CID.
-	 */
-	if (!(cid_reg_value & CIDCFGR_CFEN) ||
-	    FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) == RIF_CID0)
-		goto skip_cid_check;
-
-	/* Coherency check with the CID configuration */
-	if (FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) != RIF_CID1) {
-		dev_dbg(rifsc_controller->dev, "Invalid CID configuration for peripheral: %d\n",
-			firewall_id);
-		return -EACCES;
-	}
-
-skip_cid_check:
 	/* Check security configuration */
 	if (sec_reg_value & BIT(reg_offset)) {
 		dev_dbg(rifsc_controller->dev,
@@ -723,19 +694,31 @@ static int stm32_rifsc_grant_access(struct stm32_firewall_controller *ctrl, u32
 		return -EACCES;
 	}
 
-	/*
-	 * If the peripheral is in semaphore mode, take the semaphore so that
-	 * the CID1 has the ownership.
-	 */
-	if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) {
+	/* Skip CID check if CID filtering isn't enabled */
+	if (!(cid_reg_value & CIDCFGR_CFEN))
+		goto skip_cid_check;
+
+	/* First check conditions for semaphore mode, which doesn't take into account static CID. */
+	if (cid_reg_value & CIDCFGR_SEMEN) {
+		if (!(cid_reg_value & BIT(RIF_CID1 + SEMWL_SHIFT))) {
+			dev_dbg(rifsc_controller->dev,
+				"Invalid bus semaphore configuration: index %d\n", firewall_id);
+			return -EACCES;
+		}
+
 		rc = stm32_rif_acquire_semaphore(rifsc_controller, firewall_id);
 		if (rc) {
-			dev_err(rifsc_controller->dev,
+			dev_dbg(rifsc_controller->dev,
 				"Couldn't acquire semaphore for peripheral: %d\n", firewall_id);
 			return rc;
 		}
+	} else if (FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) != RIF_CID1) {
+		dev_dbg(rifsc_controller->dev, "Invalid CID configuration for peripheral: %d\n",
+			firewall_id);
+		return -EACCES;
 	}
 
+skip_cid_check:
 	return 0;
 }
 
diff --git a/drivers/cache/ax45mp_cache.c b/drivers/cache/ax45mp_cache.c
index 1d7dd3d2c101c..934c5087ec2bd 100644
--- a/drivers/cache/ax45mp_cache.c
+++ b/drivers/cache/ax45mp_cache.c
@@ -178,11 +178,11 @@ static const struct of_device_id ax45mp_cache_ids[] = {
 
 static int __init ax45mp_cache_init(void)
 {
-	struct device_node *np;
 	struct resource res;
 	int ret;
 
-	np = of_find_matching_node(NULL, ax45mp_cache_ids);
+	struct device_node *np __free(device_node) =
+		of_find_matching_node(NULL, ax45mp_cache_ids);
 	if (!of_device_is_available(np))
 		return -ENODEV;
 
diff --git a/drivers/cache/starfive_starlink_cache.c b/drivers/cache/starfive_starlink_cache.c
index 24c7d078ca227..3a25d2d7c70ca 100644
--- a/drivers/cache/starfive_starlink_cache.c
+++ b/drivers/cache/starfive_starlink_cache.c
@@ -102,11 +102,11 @@ static const struct of_device_id starlink_cache_ids[] = {
 
 static int __init starlink_cache_init(void)
 {
-	struct device_node *np;
 	u32 block_size;
 	int ret;
 
-	np = of_find_matching_node(NULL, starlink_cache_ids);
+	struct device_node *np __free(device_node) =
+		of_find_matching_node(NULL, starlink_cache_ids);
 	if (!of_device_is_available(np))
 		return -ENODEV;
 
diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c
index 60dd09a56f50e..8f128cc40147b 100644
--- a/drivers/char/hpet.c
+++ b/drivers/char/hpet.c
@@ -354,8 +354,9 @@ static __init int hpet_mmap_enable(char *str)
 }
 __setup("hpet_mmap=", hpet_mmap_enable);
 
-static int hpet_mmap(struct file *file, struct vm_area_struct *vma)
+static int hpet_mmap_prepare(struct vm_area_desc *desc)
 {
+	struct file *file = desc->file;
 	struct hpet_dev *devp;
 	unsigned long addr;
 
@@ -368,11 +369,12 @@ static int hpet_mmap(struct file *file, struct vm_area_struct *vma)
 	if (addr & (PAGE_SIZE - 1))
 		return -ENOSYS;
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-	return vm_iomap_memory(vma, addr, PAGE_SIZE);
+	desc->page_prot = pgprot_noncached(desc->page_prot);
+	mmap_action_simple_ioremap(desc, addr, PAGE_SIZE);
+	return 0;
 }
 #else
-static int hpet_mmap(struct file *file, struct vm_area_struct *vma)
+static int hpet_mmap_prepare(struct vm_area_desc *desc)
 {
 	return -ENOSYS;
 }
@@ -710,7 +712,7 @@ static const struct file_operations hpet_fops = {
 	.open = hpet_open,
 	.release = hpet_release,
 	.fasync = hpet_fasync,
-	.mmap = hpet_mmap,
+	.mmap_prepare = hpet_mmap_prepare,
 };
 
 static int hpet_is_known(struct hpet_data *hdp)
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index c41f51c82eddc..869ac87a4b6ae 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -987,7 +987,7 @@ static int deliver_response(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 		mutex_lock(&intf->user_msgs_mutex);
 		list_add_tail(&msg->link, &intf->user_msgs);
 		mutex_unlock(&intf->user_msgs_mutex);
-		queue_work(system_wq, &intf->smi_work);
+		queue_work(system_percpu_wq, &intf->smi_work);
 	}
 
 	return rv;
@@ -4977,7 +4977,7 @@ void ipmi_smi_msg_received(struct ipmi_smi *intf,
 	if (run_to_completion)
 		smi_work(&intf->smi_work);
 	else
-		queue_work(system_wq, &intf->smi_work);
+		queue_work(system_percpu_wq, &intf->smi_work);
 }
 EXPORT_SYMBOL(ipmi_smi_msg_received);
 
@@ -4987,7 +4987,7 @@ void ipmi_smi_watchdog_pretimeout(struct ipmi_smi *intf)
 		return;
 
 	atomic_set(&intf->watchdog_pretimeouts_to_deliver, 1);
-	queue_work(system_wq, &intf->smi_work);
+	queue_work(system_percpu_wq, &intf->smi_work);
 }
 EXPORT_SYMBOL(ipmi_smi_watchdog_pretimeout);
 
@@ -5162,7 +5162,7 @@ static bool ipmi_timeout_handler(struct ipmi_smi *intf,
 				       flags);
 	}
 
-	queue_work(system_wq, &intf->smi_work);
+	queue_work(system_percpu_wq, &intf->smi_work);
 
 	return need_timer;
 }
@@ -5218,7 +5218,7 @@ static void ipmi_timeout(struct timer_list *unused)
 	if (atomic_read(&stop_operation))
 		return;
 
-	queue_work(system_wq, &ipmi_timer_work);
+	queue_work(system_percpu_wq, &ipmi_timer_work);
 }
 
 static void need_waiter(struct ipmi_smi *intf)
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index cca4529431f8a..5fd421e48c04b 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -520,7 +520,7 @@ static int mmap_zero_prepare(struct vm_area_desc *desc)
 #ifndef CONFIG_MMU
 	return -ENOSYS;
 #endif
-	if (vma_desc_test_flags(desc, VMA_SHARED_BIT))
+	if (vma_desc_test(desc, VMA_SHARED_BIT))
 		return shmem_zero_setup_desc(desc);
 
 	desc->action.success_hook = mmap_zero_private_success;
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 7ff4d29911fd8..6422546c45c35 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -56,11 +56,7 @@
 #include <linux/sched/isolation.h>
 #include <crypto/chacha.h>
 #include <crypto/blake2s.h>
-#ifdef CONFIG_VDSO_GETRANDOM
-#include <vdso/getrandom.h>
 #include <vdso/datapage.h>
-#include <vdso/vsyscall.h>
-#endif
 #include <asm/archrandom.h>
 #include <asm/processor.h>
 #include <asm/irq.h>
@@ -269,7 +265,7 @@ static void crng_reseed(struct work_struct *work)
 	if (next_gen == ULONG_MAX)
 		++next_gen;
 	WRITE_ONCE(base_crng.generation, next_gen);
-#ifdef CONFIG_VDSO_GETRANDOM
+
 	/* base_crng.generation's invalid value is ULONG_MAX, while
 	 * vdso_k_rng_data->generation's invalid value is 0, so add one to the
 	 * former to arrive at the latter. Use smp_store_release so that this
@@ -283,8 +279,9 @@ static void crng_reseed(struct work_struct *work)
 	 * because the vDSO side only checks whether the value changed, without
 	 * actually using or interpreting the value.
 	 */
-	smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
-#endif
+	if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
+		smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
+
 	if (!static_branch_likely(&crng_is_ready))
 		crng_init = CRNG_READY;
 	spin_unlock_irqrestore(&base_crng.lock, flags);
@@ -544,18 +541,16 @@ DEFINE_BATCHED_ENTROPY(u16)
 DEFINE_BATCHED_ENTROPY(u32)
 DEFINE_BATCHED_ENTROPY(u64)
 
-u32 __get_random_u32_below(u32 ceil)
+/*
+ * This is the slow path for variable ceil. It is still fast, most of the time,
+ * by doing traditional reciprocal multiplication and opportunistically
+ * comparing the lower half to ceil itself, before falling back to computing a
+ * larger bound, and then rejecting samples whose lower half would indicate a
+ * range indivisible by ceil. The use of `-ceil % ceil` is analogous to `2^32 %
+ * ceil`, but is computable in 32-bits.
+ */
+u32 __limit_random_u32_below(u32 ceil, u32 rand)
 {
-	/*
-	 * This is the slow path for variable ceil. It is still fast, most of
-	 * the time, by doing traditional reciprocal multiplication and
-	 * opportunistically comparing the lower half to ceil itself, before
-	 * falling back to computing a larger bound, and then rejecting samples
-	 * whose lower half would indicate a range indivisible by ceil. The use
-	 * of `-ceil % ceil` is analogous to `2^32 % ceil`, but is computable
-	 * in 32-bits.
-	 */
-	u32 rand = get_random_u32();
 	u64 mult;
 
 	/*
@@ -577,6 +572,12 @@ u32 __get_random_u32_below(u32 ceil)
 	}
 	return mult >> 32;
 }
+EXPORT_SYMBOL_GPL(__limit_random_u32_below);
+
+u32 __get_random_u32_below(u32 ceil)
+{
+	return __limit_random_u32_below(ceil, get_random_u32());
+}
 EXPORT_SYMBOL(__get_random_u32_below);
 
 #ifdef CONFIG_SMP
@@ -734,9 +735,8 @@ static void __cold _credit_init_bits(size_t bits)
 		if (system_dfl_wq)
 			queue_work(system_dfl_wq, &set_ready);
 		atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
-#ifdef CONFIG_VDSO_GETRANDOM
-		WRITE_ONCE(vdso_k_rng_data->is_ready, true);
-#endif
+		if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
+			WRITE_ONCE(vdso_k_rng_data->is_ready, true);
 		wake_up_interruptible(&crng_init_wait);
 		kill_fasync(&fasync, SIGIO, POLL_IN);
 		pr_notice("crng init done\n");
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 3d803b4cf5c13..8cc300b90b5fd 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -531,6 +531,7 @@ source "drivers/clk/starfive/Kconfig"
 source "drivers/clk/sunxi/Kconfig"
 source "drivers/clk/sunxi-ng/Kconfig"
 source "drivers/clk/tegra/Kconfig"
+source "drivers/clk/tenstorrent/Kconfig"
 source "drivers/clk/thead/Kconfig"
 source "drivers/clk/stm32/Kconfig"
 source "drivers/clk/ti/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index f7bce3951a30d..f52cf3ac64fcc 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -155,6 +155,7 @@ obj-y					+= starfive/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
 obj-y					+= sunxi-ng/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
+obj-y					+= tenstorrent/
 obj-$(CONFIG_ARCH_THEAD)		+= thead/
 obj-y					+= ti/
 obj-$(CONFIG_CLK_UNIPHIER)		+= uniphier/
diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c
index 6b286ea6f1218..7c562559ad8bb 100644
--- a/drivers/clk/clk-scmi.c
+++ b/drivers/clk/clk-scmi.c
@@ -12,7 +12,6 @@
 #include <linux/of.h>
 #include <linux/module.h>
 #include <linux/scmi_protocol.h>
-#include <asm/div64.h>
 
 #define NOT_ATOMIC	false
 #define ATOMIC		true
@@ -57,35 +56,17 @@ static unsigned long scmi_clk_recalc_rate(struct clk_hw *hw,
 static int scmi_clk_determine_rate(struct clk_hw *hw,
 				   struct clk_rate_request *req)
 {
-	u64 fmin, fmax, ftmp;
+	int ret;
 	struct scmi_clk *clk = to_scmi_clk(hw);
 
 	/*
-	 * We can't figure out what rate it will be, so just return the
-	 * rate back to the caller. scmi_clk_recalc_rate() will be called
-	 * after the rate is set and we'll know what rate the clock is
+	 * If we could not get a better rate scmi_clk_recalc_rate() will be
+	 * called after the rate is set and we'll know what rate the clock is
 	 * running at then.
 	 */
-	if (clk->info->rate_discrete)
-		return 0;
-
-	fmin = clk->info->range.min_rate;
-	fmax = clk->info->range.max_rate;
-	if (req->rate <= fmin) {
-		req->rate = fmin;
-
-		return 0;
-	} else if (req->rate >= fmax) {
-		req->rate = fmax;
-
-		return 0;
-	}
-
-	ftmp = req->rate - fmin;
-	ftmp += clk->info->range.step_size - 1; /* to round up */
-	do_div(ftmp, clk->info->range.step_size);
-
-	req->rate = ftmp * clk->info->range.step_size + fmin;
+	ret = scmi_proto_clk_ops->determine_rate(clk->ph, clk->id, &req->rate);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -221,7 +202,6 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk,
 			     const struct clk_ops *scmi_ops)
 {
 	int ret;
-	unsigned long min_rate, max_rate;
 
 	struct clk_init_data init = {
 		.flags = CLK_GET_RATE_NOCACHE,
@@ -236,20 +216,8 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk,
 	if (ret)
 		return ret;
 
-	if (sclk->info->rate_discrete) {
-		int num_rates = sclk->info->list.num_rates;
-
-		if (num_rates <= 0)
-			return -EINVAL;
-
-		min_rate = sclk->info->list.rates[0];
-		max_rate = sclk->info->list.rates[num_rates - 1];
-	} else {
-		min_rate = sclk->info->range.min_rate;
-		max_rate = sclk->info->range.max_rate;
-	}
-
-	clk_hw_set_rate_range(&sclk->hw, min_rate, max_rate);
+	clk_hw_set_rate_range(&sclk->hw, sclk->info->min_rate,
+			      sclk->info->max_rate);
 	return ret;
 }
 
diff --git a/drivers/clk/imx/clk-fracn-gppll.c b/drivers/clk/imx/clk-fracn-gppll.c
index 89ed7749bf47e..4048c16c0578b 100644
--- a/drivers/clk/imx/clk-fracn-gppll.c
+++ b/drivers/clk/imx/clk-fracn-gppll.c
@@ -85,9 +85,11 @@ static const struct imx_fracn_gppll_rate_table fracn_tbl[] = {
 	PLL_FRACN_GP(519750000U, 173, 25, 100, 1, 8),
 	PLL_FRACN_GP(498000000U, 166, 0, 1, 0, 8),
 	PLL_FRACN_GP(484000000U, 121, 0, 1, 0, 6),
+	PLL_FRACN_GP(477400000U, 119, 35, 100, 0, 6),
 	PLL_FRACN_GP(445333333U, 167, 0, 1, 0, 9),
 	PLL_FRACN_GP(400000000U, 200, 0, 1, 0, 12),
 	PLL_FRACN_GP(393216000U, 163, 84, 100, 0, 10),
+	PLL_FRACN_GP(333333333U, 125, 0, 1, 1, 9),
 	PLL_FRACN_GP(332600000U, 138, 584, 1000, 0, 10),
 	PLL_FRACN_GP(300000000U, 150, 0, 1, 0, 12),
 	PLL_FRACN_GP(241900000U, 201, 584, 1000, 0, 20),
diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c
index f726c00aba721..35e6b59c01dbe 100644
--- a/drivers/clk/imx/clk-imx6q.c
+++ b/drivers/clk/imx/clk-imx6q.c
@@ -188,9 +188,11 @@ static void of_assigned_ldb_sels(struct device_node *node,
 		}
 		if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) {
 			pr_err("ccm: parent clock %d not in ccm\n", index);
+			of_node_put(clkspec.np);
 			return;
 		}
 		parent = clkspec.args[0];
+		of_node_put(clkspec.np);
 
 		rc = of_parse_phandle_with_args(node, "assigned-clocks",
 				"#clock-cells", index, &clkspec);
@@ -198,9 +200,11 @@ static void of_assigned_ldb_sels(struct device_node *node,
 			return;
 		if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) {
 			pr_err("ccm: child clock %d not in ccm\n", index);
+			of_node_put(clkspec.np);
 			return;
 		}
 		child = clkspec.args[0];
+		of_node_put(clkspec.np);
 
 		if (child != IMX6QDL_CLK_LDB_DI0_SEL &&
 		    child != IMX6QDL_CLK_LDB_DI1_SEL)
@@ -238,8 +242,11 @@ static bool pll6_bypassed(struct device_node *node)
 			return false;
 
 		if (clkspec.np == node &&
-		    clkspec.args[0] == IMX6QDL_PLL6_BYPASS)
+		    clkspec.args[0] == IMX6QDL_PLL6_BYPASS) {
+			of_node_put(clkspec.np);
 			break;
+		}
+		of_node_put(clkspec.np);
 	}
 
 	/* PLL6 bypass is not part of the assigned clock list */
@@ -249,6 +256,9 @@ static bool pll6_bypassed(struct device_node *node)
 	ret = of_parse_phandle_with_args(node, "assigned-clock-parents",
 					 "#clock-cells", index, &clkspec);
 
+	if (!ret)
+		of_node_put(clkspec.np);
+
 	if (clkspec.args[0] != IMX6QDL_CLK_PLL6)
 		return true;
 
diff --git a/drivers/clk/imx/clk-imx8-acm.c b/drivers/clk/imx/clk-imx8-acm.c
index 790f7e44b11e2..07dca6f31cf8e 100644
--- a/drivers/clk/imx/clk-imx8-acm.c
+++ b/drivers/clk/imx/clk-imx8-acm.c
@@ -371,7 +371,8 @@ static int imx8_acm_clk_probe(struct platform_device *pdev)
 	for (i = 0; i < priv->soc_data->num_sels; i++) {
 		hws[sels[i].clkid] = devm_clk_hw_register_mux_parent_data_table(dev,
 										sels[i].name, sels[i].parents,
-										sels[i].num_parents, 0,
+										sels[i].num_parents,
+										CLK_SET_RATE_NO_REPARENT,
 										base + sels[i].reg,
 										sels[i].shift, sels[i].width,
 										0, NULL, NULL);
diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c
index f70ed231b92d6..cedc8a02aa1f0 100644
--- a/drivers/clk/imx/clk-imx8mq.c
+++ b/drivers/clk/imx/clk-imx8mq.c
@@ -237,7 +237,7 @@ static const char * const imx8mq_dsi_esc_sels[] = {"osc_25m", "sys2_pll_100m", "
 static const char * const imx8mq_csi1_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m",
 					      "sys2_pll_1000m", "sys3_pll_out", "audio_pll2_out", "video_pll1_out", };
 
-static const char * const imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m",
+static const char * const imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_333m", "sys2_pll_100m", "sys1_pll_800m",
 					     "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", };
 
 static const char * const imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m",
@@ -246,7 +246,7 @@ static const char * const imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m",
 static const char * const imx8mq_csi2_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m",
 					      "sys2_pll_1000m", "sys3_pll_out", "audio_pll2_out", "video_pll1_out", };
 
-static const char * const imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m",
+static const char * const imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_333m", "sys2_pll_100m", "sys1_pll_800m",
 					     "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", };
 
 static const char * const imx8mq_csi2_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m",
diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c
index 7552aaafc3395..39600ee22be30 100644
--- a/drivers/clk/imx/clk-pll14xx.c
+++ b/drivers/clk/imx/clk-pll14xx.c
@@ -151,7 +151,7 @@ static void imx_pll14xx_calc_settings(struct clk_pll14xx *pll, unsigned long rat
 	/* First try if we can get the desired rate from one of the static entries */
 	tt = imx_get_pll_settings(pll, rate);
 	if (tt) {
-		pr_debug("%s: in=%ld, want=%ld, Using PLL setting from table\n",
+		pr_debug("%s: in=%lu, want=%lu, Using PLL setting from table\n",
 			 clk_hw_get_name(&pll->hw), prate, rate);
 		t->rate = tt->rate;
 		t->mdiv = tt->mdiv;
@@ -173,7 +173,7 @@ static void imx_pll14xx_calc_settings(struct clk_pll14xx *pll, unsigned long rat
 
 	if (rate >= rate_min && rate <= rate_max) {
 		kdiv = pll1443x_calc_kdiv(mdiv, pdiv, sdiv, rate, prate);
-		pr_debug("%s: in=%ld, want=%ld Only adjust kdiv %ld -> %d\n",
+		pr_debug("%s: in=%lu, want=%lu Only adjust kdiv %ld -> %d\n",
 			 clk_hw_get_name(&pll->hw), prate, rate,
 			 FIELD_GET(KDIV_MASK, pll_div_ctl1), kdiv);
 		fout = pll14xx_calc_rate(pll, mdiv, pdiv, sdiv, kdiv, prate);
@@ -211,7 +211,7 @@ static void imx_pll14xx_calc_settings(struct clk_pll14xx *pll, unsigned long rat
 		}
 	}
 found:
-	pr_debug("%s: in=%ld, want=%ld got=%d (pdiv=%d sdiv=%d mdiv=%d kdiv=%d)\n",
+	pr_debug("%s: in=%lu, want=%lu got=%u (pdiv=%d sdiv=%d mdiv=%d kdiv=%d)\n",
 		 clk_hw_get_name(&pll->hw), prate, rate, t->rate, t->pdiv, t->sdiv,
 		 t->mdiv, t->kdiv);
 }
diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c
index a85ec48a798b5..9b33df9967ece 100644
--- a/drivers/clk/imx/clk-scu.c
+++ b/drivers/clk/imx/clk-scu.c
@@ -706,8 +706,7 @@ struct clk_hw *imx_clk_scu_alloc_dev(const char *name,
 	if (ret)
 		goto put_device;
 
-	ret = driver_set_override(&pdev->dev, &pdev->driver_override,
-				  "imx-scu-clk", strlen("imx-scu-clk"));
+	ret = device_set_driver_override(&pdev->dev, "imx-scu-clk");
 	if (ret)
 		goto put_device;
 
diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c
index 41eb38552a9c7..1fbd8011fde21 100644
--- a/drivers/clk/imx/clk-vf610.c
+++ b/drivers/clk/imx/clk-vf610.c
@@ -11,6 +11,13 @@
 
 #include "clk.h"
 
+/*
+ * The VF610_CLK_END corresponds to ones defined in
+ * include/dt-bindings/clock/vf610-clock.h
+ * It shall be the value of the last defined clock +1
+ */
+#define VF610_CLK_END 196
+
 #define CCM_CCR			(ccm_base + 0x00)
 #define CCM_CSR			(ccm_base + 0x04)
 #define CCM_CCSR		(ccm_base + 0x08)
@@ -313,6 +320,11 @@ static void __init vf610_clocks_init(struct device_node *ccm_node)
 	clk[VF610_CLK_ENET_TS] = imx_clk_gate("enet_ts", "enet_ts_sel", CCM_CSCDR1, 23);
 	clk[VF610_CLK_ENET0] = imx_clk_gate2("enet0", "ipg_bus", CCM_CCGR9, CCM_CCGRx_CGn(0));
 	clk[VF610_CLK_ENET1] = imx_clk_gate2("enet1", "ipg_bus", CCM_CCGR9, CCM_CCGRx_CGn(1));
+	clk[VF610_CLK_ESW] = imx_clk_gate2("esw", "ipg_bus", CCM_CCGR10, CCM_CCGRx_CGn(8));
+	clk[VF610_CLK_ESW_MAC_TAB0] = imx_clk_gate2("esw_tab0", "ipg_bus", CCM_CCGR10, CCM_CCGRx_CGn(12));
+	clk[VF610_CLK_ESW_MAC_TAB1] = imx_clk_gate2("esw_tab1", "ipg_bus", CCM_CCGR10, CCM_CCGRx_CGn(13));
+	clk[VF610_CLK_ESW_MAC_TAB2] = imx_clk_gate2("esw_tab2", "ipg_bus", CCM_CCGR10, CCM_CCGRx_CGn(14));
+	clk[VF610_CLK_ESW_MAC_TAB3] = imx_clk_gate2("esw_tab3", "ipg_bus", CCM_CCGR10, CCM_CCGRx_CGn(15));
 
 	clk[VF610_CLK_PIT] = imx_clk_gate2("pit", "ipg_bus", CCM_CCGR1, CCM_CCGRx_CGn(7));
 
diff --git a/drivers/clk/microchip/clk-mpfs-ccc.c b/drivers/clk/microchip/clk-mpfs-ccc.c
index 3a3ea2d142f8a..0a76a1aaa50f7 100644
--- a/drivers/clk/microchip/clk-mpfs-ccc.c
+++ b/drivers/clk/microchip/clk-mpfs-ccc.c
@@ -178,7 +178,7 @@ static int mpfs_ccc_register_outputs(struct device *dev, struct mpfs_ccc_out_hw_
 			return dev_err_probe(dev, ret, "failed to register clock id: %d\n",
 					     out_hw->id);
 
-		data->hw_data.hws[out_hw->id] = &out_hw->divider.hw;
+		data->hw_data.hws[out_hw->id - 2] = &out_hw->divider.hw;
 	}
 
 	return 0;
@@ -234,6 +234,10 @@ static int mpfs_ccc_probe(struct platform_device *pdev)
 	unsigned int num_clks;
 	int ret;
 
+	/*
+	 * If DLLs get added here, mpfs_ccc_register_outputs() currently packs
+	 * sparse clock IDs in the hws array
+	 */
 	num_clks = ARRAY_SIZE(mpfs_ccc_pll_clks) + ARRAY_SIZE(mpfs_ccc_pll0out_clks) +
 		   ARRAY_SIZE(mpfs_ccc_pll1out_clks);
 
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index a8a86ea6bb744..e45e353eb4f00 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -19,33 +19,68 @@ menuconfig COMMON_CLK_QCOM
 
 if COMMON_CLK_QCOM
 
+config CLK_ELIZA_GCC
+	tristate "Eliza Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	help
+	  Support for the global clock controller on Eliza devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  I2C, USB, UFS, SDCC, etc.
+
+config CLK_ELIZA_TCSRCC
+	tristate "Eliza TCSR Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	help
+	  Support for the TCSR clock controller on Eliza devices.
+	  Say Y if you want to use peripheral devices such as USB/PCIe/UFS.
+
 config CLK_GLYMUR_DISPCC
-	tristate "GLYMUR Display Clock Controller"
+	tristate "Glymur Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_GLYMUR_GCC
 	help
 	  Support for the display clock controllers on Qualcomm
-	  Technologies, Inc. GLYMUR devices.
+	  Technologies, Inc. Glymur devices.
 	  Say Y if you want to support display devices and functionality such as
 	  splash screen.
 
 config CLK_GLYMUR_GCC
-	tristate "GLYMUR Global Clock Controller"
+	tristate "Glymur Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
 	help
-	  Support for the global clock controller on GLYMUR devices.
+	  Support for the global clock controller on Glymur devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  I2C, USB, UFS, SDCC, etc.
 
+config CLK_GLYMUR_GPUCC
+	tristate "GLYMUR Graphics Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_GLYMUR_GCC
+	help
+	  Support for the graphics clock controller on GLYMUR devices.
+	  Say Y if you want to support graphics controller devices and
+	  functionality such as 3D graphics.
+
 config CLK_GLYMUR_TCSRCC
-	tristate "GLYMUR TCSR Clock Controller"
+	tristate "Glymur TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
 	help
-	  Support for the TCSR clock controller on GLYMUR devices.
+	  Support for the TCSR clock controller on Glymur devices.
 	  Say Y if you want to use peripheral devices such as USB/PCIe/EDP.
 
+config CLK_GLYMUR_VIDEOCC
+	tristate "Glymur Video Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_GLYMUR_GCC
+	help
+	  Support for the video clock controller on Glymur devices.
+	  Say Y if you want to support video devices and functionality such as
+	  video encode and decode.
+
 config CLK_KAANAPALI_CAMCC
 	tristate "Kaanapali Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -314,6 +349,14 @@ config IPQ_GCC_5018
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  i2c, USB, SD/eMMC, etc.
 
+config IPQ_GCC_5210
+	tristate "IPQ5210 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	help
+	  Support for the global clock controller on ipq5210 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, SD/eMMC, etc.
+
 config IPQ_GCC_5332
 	tristate "IPQ5332 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -392,7 +435,7 @@ config IPQ_NSSCC_9574
 
 config IPQ_NSSCC_QCA8K
 	tristate "QCA8K(QCA8386 or QCA8084) NSS Clock Controller"
-	depends on MDIO_BUS
+	depends on PHYLIB
 	help
 	  Support for NSS(Network SubSystem) clock controller on
 	  qca8386/qca8084 chip.
@@ -1579,10 +1622,10 @@ config SM_VIDEOCC_8250
 config SM_VIDEOCC_8350
 	tristate "SM8350 Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
-	select SM_GCC_8350
+	depends on SM_GCC_8350 || SC_GCC_8280XP
 	select QCOM_GDSC
 	help
-	  Support for the video clock controller on SM8350 devices.
+	  Support for the video clock controller on SM8350 or SC8280XP devices.
 	  Say Y if you want to support video devices and functionality such as
 	  video encode and decode.
 
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 6b0ad8832b55f..807a7c9071a7d 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -20,10 +20,14 @@ clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o
 # Keep alphabetically sorted by config
 obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o
 obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o
+obj-$(CONFIG_CLK_ELIZA_GCC) += gcc-eliza.o
+obj-$(CONFIG_CLK_ELIZA_TCSRCC) += tcsrcc-eliza.o
 obj-$(CONFIG_CLK_GFM_LPASS_SM8250) += lpass-gfm-sm8250.o
 obj-$(CONFIG_CLK_GLYMUR_DISPCC) += dispcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o
+obj-$(CONFIG_CLK_GLYMUR_GPUCC) += gpucc-glymur.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o
+obj-$(CONFIG_CLK_GLYMUR_VIDEOCC) += videocc-glymur.o
 obj-$(CONFIG_CLK_KAANAPALI_CAMCC) += cambistmclkcc-kaanapali.o camcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_DISPCC) += dispcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o
@@ -43,6 +47,7 @@ obj-$(CONFIG_IPQ_APSS_6018) += apss-ipq6018.o
 obj-$(CONFIG_IPQ_CMN_PLL) += ipq-cmn-pll.o
 obj-$(CONFIG_IPQ_GCC_4019) += gcc-ipq4019.o
 obj-$(CONFIG_IPQ_GCC_5018) += gcc-ipq5018.o
+obj-$(CONFIG_IPQ_GCC_5210) += gcc-ipq5210.o
 obj-$(CONFIG_IPQ_GCC_5332) += gcc-ipq5332.o
 obj-$(CONFIG_IPQ_GCC_5424) += gcc-ipq5424.o
 obj-$(CONFIG_IPQ_GCC_6018) += gcc-ipq6018.o
diff --git a/drivers/clk/qcom/cambistmclkcc-kaanapali.c b/drivers/clk/qcom/cambistmclkcc-kaanapali.c
index 066c1087b0b65..6ad912403b8bc 100644
--- a/drivers/clk/qcom/cambistmclkcc-kaanapali.c
+++ b/drivers/clk/qcom/cambistmclkcc-kaanapali.c
@@ -6,9 +6,7 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,kaanapali-cambistmclkcc.h>
diff --git a/drivers/clk/qcom/camcc-kaanapali.c b/drivers/clk/qcom/camcc-kaanapali.c
index 82967993fcffe..c848ca99e9dff 100644
--- a/drivers/clk/qcom/camcc-kaanapali.c
+++ b/drivers/clk/qcom/camcc-kaanapali.c
@@ -6,9 +6,7 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,kaanapali-camcc.h>
diff --git a/drivers/clk/qcom/camcc-sc8180x.c b/drivers/clk/qcom/camcc-sc8180x.c
index 388fedf1dc815..0291e2f3ea802 100644
--- a/drivers/clk/qcom/camcc-sc8180x.c
+++ b/drivers/clk/qcom/camcc-sc8180x.c
@@ -7,7 +7,6 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,sc8180x-camcc.h>
@@ -63,6 +62,7 @@ static const struct alpha_pll_config cam_cc_pll0_config = {
 
 static struct clk_alpha_pll cam_cc_pll0 = {
 	.offset = 0x0,
+	.config = &cam_cc_pll0_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -138,6 +138,7 @@ static const struct alpha_pll_config cam_cc_pll1_config = {
 
 static struct clk_alpha_pll cam_cc_pll1 = {
 	.offset = 0x1000,
+	.config = &cam_cc_pll1_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -167,6 +168,7 @@ static const struct alpha_pll_config cam_cc_pll2_config = {
 
 static struct clk_alpha_pll cam_cc_pll2 = {
 	.offset = 0x2000,
+	.config = &cam_cc_pll2_config,
 	.vco_table = regera_vco,
 	.num_vco = ARRAY_SIZE(regera_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_REGERA],
@@ -219,6 +221,7 @@ static const struct alpha_pll_config cam_cc_pll3_config = {
 
 static struct clk_alpha_pll cam_cc_pll3 = {
 	.offset = 0x3000,
+	.config = &cam_cc_pll3_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -248,6 +251,7 @@ static const struct alpha_pll_config cam_cc_pll4_config = {
 
 static struct clk_alpha_pll cam_cc_pll4 = {
 	.offset = 0x4000,
+	.config = &cam_cc_pll4_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -277,6 +281,7 @@ static const struct alpha_pll_config cam_cc_pll5_config = {
 
 static struct clk_alpha_pll cam_cc_pll5 = {
 	.offset = 0x4078,
+	.config = &cam_cc_pll5_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -306,6 +311,7 @@ static const struct alpha_pll_config cam_cc_pll6_config = {
 
 static struct clk_alpha_pll cam_cc_pll6 = {
 	.offset = 0x40f0,
+	.config = &cam_cc_pll6_config,
 	.vco_table = trion_vco,
 	.num_vco = ARRAY_SIZE(trion_vco),
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION],
@@ -2813,6 +2819,21 @@ static const struct qcom_reset_map cam_cc_sc8180x_resets[] = {
 	[CAM_CC_MCLK7_BCR] = { 0x50e0 },
 };
 
+static struct clk_alpha_pll *cam_cc_sc8180x_plls[] = {
+	&cam_cc_pll0,
+	&cam_cc_pll1,
+	&cam_cc_pll2,
+	&cam_cc_pll3,
+	&cam_cc_pll4,
+	&cam_cc_pll5,
+	&cam_cc_pll6,
+};
+
+static u32 cam_cc_sc8180x_critical_cbcrs[] = {
+	0xc1e4, /* CAM_CC_GDSC_CLK */
+	0xc200, /* CAM_CC_SLEEP_CLK */
+};
+
 static const struct regmap_config cam_cc_sc8180x_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -2821,6 +2842,13 @@ static const struct regmap_config cam_cc_sc8180x_regmap_config = {
 	.fast_io = true,
 };
 
+static struct qcom_cc_driver_data cam_cc_sc8180x_driver_data = {
+	.alpha_plls = cam_cc_sc8180x_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_cc_sc8180x_plls),
+	.clk_cbcrs = cam_cc_sc8180x_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_sc8180x_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc cam_cc_sc8180x_desc = {
 	.config = &cam_cc_sc8180x_regmap_config,
 	.clks = cam_cc_sc8180x_clocks,
@@ -2829,6 +2857,8 @@ static const struct qcom_cc_desc cam_cc_sc8180x_desc = {
 	.num_resets = ARRAY_SIZE(cam_cc_sc8180x_resets),
 	.gdscs = cam_cc_sc8180x_gdscs,
 	.num_gdscs = ARRAY_SIZE(cam_cc_sc8180x_gdscs),
+	.use_rpm = true,
+	.driver_data = &cam_cc_sc8180x_driver_data,
 };
 
 static const struct of_device_id cam_cc_sc8180x_match_table[] = {
@@ -2839,40 +2869,7 @@ MODULE_DEVICE_TABLE(of, cam_cc_sc8180x_match_table);
 
 static int cam_cc_sc8180x_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	ret = devm_pm_runtime_enable(&pdev->dev);
-	if (ret)
-		return ret;
-
-	ret = pm_runtime_resume_and_get(&pdev->dev);
-	if (ret)
-		return ret;
-
-	regmap = qcom_cc_map(pdev, &cam_cc_sc8180x_desc);
-	if (IS_ERR(regmap)) {
-		pm_runtime_put(&pdev->dev);
-		return PTR_ERR(regmap);
-	}
-
-	clk_trion_pll_configure(&cam_cc_pll0, regmap, &cam_cc_pll0_config);
-	clk_trion_pll_configure(&cam_cc_pll1, regmap, &cam_cc_pll1_config);
-	clk_regera_pll_configure(&cam_cc_pll2, regmap, &cam_cc_pll2_config);
-	clk_trion_pll_configure(&cam_cc_pll3, regmap, &cam_cc_pll3_config);
-	clk_trion_pll_configure(&cam_cc_pll4, regmap, &cam_cc_pll4_config);
-	clk_trion_pll_configure(&cam_cc_pll5, regmap, &cam_cc_pll5_config);
-	clk_trion_pll_configure(&cam_cc_pll6, regmap, &cam_cc_pll6_config);
-
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xc1e4); /* CAM_CC_GDSC_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc200); /* CAM_CC_SLEEP_CLK */
-
-	ret = qcom_cc_really_probe(&pdev->dev, &cam_cc_sc8180x_desc, regmap);
-
-	pm_runtime_put(&pdev->dev);
-
-	return ret;
+	return qcom_cc_probe(pdev, &cam_cc_sc8180x_desc);
 }
 
 static struct platform_driver cam_cc_sc8180x_driver = {
diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c
index 547729b1a8ee0..6a54481cc6ae1 100644
--- a/drivers/clk/qcom/clk-rpmh.c
+++ b/drivers/clk/qcom/clk-rpmh.c
@@ -372,6 +372,8 @@ DEFINE_CLK_RPMH_VRM(rf_clk3, _d, "rfclkd3", 1);
 DEFINE_CLK_RPMH_VRM(rf_clk4, _d, "rfclkd4", 1);
 
 DEFINE_CLK_RPMH_VRM(rf_clk3, _a2, "rfclka3", 2);
+DEFINE_CLK_RPMH_VRM(rf_clk4, _a2, "rfclka4", 2);
+DEFINE_CLK_RPMH_VRM(rf_clk5, _a2, "rfclka5", 2);
 
 DEFINE_CLK_RPMH_VRM(clk1, _a1, "clka1", 1);
 DEFINE_CLK_RPMH_VRM(clk2, _a1, "clka2", 1);
@@ -940,6 +942,29 @@ static const struct clk_rpmh_desc clk_rpmh_kaanapali = {
 	.num_clks = ARRAY_SIZE(kaanapali_rpmh_clocks),
 };
 
+static struct clk_hw *eliza_rpmh_clocks[] = {
+	[RPMH_CXO_CLK]          = &clk_rpmh_bi_tcxo_div2.hw,
+	[RPMH_CXO_CLK_A]        = &clk_rpmh_bi_tcxo_div2_ao.hw,
+	[RPMH_LN_BB_CLK1]	= &clk_rpmh_clk6_a2.hw,
+	[RPMH_LN_BB_CLK1_A]	= &clk_rpmh_clk6_a2_ao.hw,
+	[RPMH_LN_BB_CLK3]	= &clk_rpmh_clk8_a2.hw,
+	[RPMH_LN_BB_CLK3_A]	= &clk_rpmh_clk8_a2_ao.hw,
+	[RPMH_RF_CLK1]		= &clk_rpmh_rf_clk1_a.hw,
+	[RPMH_RF_CLK1_A]	= &clk_rpmh_rf_clk1_a_ao.hw,
+	[RPMH_RF_CLK2]		= &clk_rpmh_rf_clk2_a.hw,
+	[RPMH_RF_CLK2_A]	= &clk_rpmh_rf_clk2_a_ao.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_rf_clk4_a2.hw,
+	[RPMH_RF_CLK4_A]	= &clk_rpmh_rf_clk4_a2_ao.hw,
+	[RPMH_RF_CLK5]		= &clk_rpmh_rf_clk5_a2.hw,
+	[RPMH_RF_CLK5_A]	= &clk_rpmh_rf_clk5_a2_ao.hw,
+	[RPMH_IPA_CLK]		= &clk_rpmh_ipa.hw,
+};
+
+static const struct clk_rpmh_desc clk_rpmh_eliza = {
+	.clks = eliza_rpmh_clocks,
+	.num_clks = ARRAY_SIZE(eliza_rpmh_clocks),
+};
+
 static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec,
 					 void *data)
 {
@@ -1029,6 +1054,7 @@ static int clk_rpmh_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id clk_rpmh_match_table[] = {
+	{ .compatible = "qcom,eliza-rpmh-clk", .data = &clk_rpmh_eliza},
 	{ .compatible = "qcom,glymur-rpmh-clk", .data = &clk_rpmh_glymur},
 	{ .compatible = "qcom,kaanapali-rpmh-clk", .data = &clk_rpmh_kaanapali},
 	{ .compatible = "qcom,milos-rpmh-clk", .data = &clk_rpmh_milos},
diff --git a/drivers/clk/qcom/dispcc-glymur.c b/drivers/clk/qcom/dispcc-glymur.c
index 5203fa6383f6a..fd085cb906672 100644
--- a/drivers/clk/qcom/dispcc-glymur.c
+++ b/drivers/clk/qcom/dispcc-glymur.c
@@ -6,9 +6,7 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,glymur-dispcc.h>
@@ -417,7 +415,7 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_aux_clk_src = {
 		.parent_data = disp_cc_parent_data_1,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_1),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_dp_ops,
+		.ops = &clk_rcg2_shared_ops,
 	},
 };
 
@@ -747,7 +745,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = {
 			&disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -762,7 +759,6 @@ static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = {
 			&disp_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -1978,5 +1974,5 @@ static struct platform_driver disp_cc_glymur_driver = {
 
 module_platform_driver(disp_cc_glymur_driver);
 
-MODULE_DESCRIPTION("QTI DISPCC GLYMUR Driver");
+MODULE_DESCRIPTION("QTI DISPCC Glymur Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/dispcc-kaanapali.c b/drivers/clk/qcom/dispcc-kaanapali.c
index baae2ec1f72aa..5ec4d2ab6b674 100644
--- a/drivers/clk/qcom/dispcc-kaanapali.c
+++ b/drivers/clk/qcom/dispcc-kaanapali.c
@@ -6,9 +6,7 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,kaanapali-dispcc.h>
@@ -800,7 +798,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = {
 			&disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -815,7 +812,6 @@ static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = {
 			&disp_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/dispcc-milos.c b/drivers/clk/qcom/dispcc-milos.c
index 95b6dd89d9ae3..0a483fb6683af 100644
--- a/drivers/clk/qcom/dispcc-milos.c
+++ b/drivers/clk/qcom/dispcc-milos.c
@@ -4,12 +4,10 @@
  * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
  */
 
-#include <linux/clk.h>
 #include <linux/clk-provider.h>
-#include <linux/err.h>
 #include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
@@ -394,7 +392,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = {
 			&disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/dispcc-sc8280xp.c b/drivers/clk/qcom/dispcc-sc8280xp.c
index 5903a759d4af4..e91dfed0f37e9 100644
--- a/drivers/clk/qcom/dispcc-sc8280xp.c
+++ b/drivers/clk/qcom/dispcc-sc8280xp.c
@@ -1160,7 +1160,6 @@ static struct clk_regmap_div disp0_cc_mdss_byte0_div_clk_src = {
 			&disp0_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -1175,7 +1174,6 @@ static struct clk_regmap_div disp1_cc_mdss_byte0_div_clk_src = {
 			&disp1_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -1190,7 +1188,6 @@ static struct clk_regmap_div disp0_cc_mdss_byte1_div_clk_src = {
 			&disp0_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -1205,7 +1202,6 @@ static struct clk_regmap_div disp1_cc_mdss_byte1_div_clk_src = {
 			&disp1_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/dispcc-sm4450.c b/drivers/clk/qcom/dispcc-sm4450.c
index e8752d01c8e62..2fdacc26df698 100644
--- a/drivers/clk/qcom/dispcc-sm4450.c
+++ b/drivers/clk/qcom/dispcc-sm4450.c
@@ -335,7 +335,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = {
 			&disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/dispcc-sm6115.c b/drivers/clk/qcom/dispcc-sm6115.c
index 8ae25d51db946..75bd57213079f 100644
--- a/drivers/clk/qcom/dispcc-sm6115.c
+++ b/drivers/clk/qcom/dispcc-sm6115.c
@@ -22,6 +22,7 @@
 #include "clk-regmap-divider.h"
 #include "common.h"
 #include "gdsc.h"
+#include "reset.h"
 
 enum {
 	DT_BI_TCXO,
@@ -511,6 +512,10 @@ static struct clk_branch disp_cc_sleep_clk = {
 	},
 };
 
+static const struct qcom_reset_map disp_cc_sm6115_resets[] = {
+	[DISP_CC_MDSS_CORE_BCR] = { 0x2000 },
+};
+
 static struct gdsc mdss_gdsc = {
 	.gdscr = 0x3000,
 	.pd = {
@@ -561,6 +566,8 @@ static const struct qcom_cc_desc disp_cc_sm6115_desc = {
 	.config = &disp_cc_sm6115_regmap_config,
 	.clks = disp_cc_sm6115_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm6115_clocks),
+	.resets = disp_cc_sm6115_resets,
+	.num_resets = ARRAY_SIZE(disp_cc_sm6115_resets),
 	.gdscs = disp_cc_sm6115_gdscs,
 	.num_gdscs = ARRAY_SIZE(disp_cc_sm6115_gdscs),
 };
diff --git a/drivers/clk/qcom/dispcc-sm6125.c b/drivers/clk/qcom/dispcc-sm6125.c
index 851d38a487d3c..2c67abcfef122 100644
--- a/drivers/clk/qcom/dispcc-sm6125.c
+++ b/drivers/clk/qcom/dispcc-sm6125.c
@@ -17,6 +17,7 @@
 #include "clk-regmap.h"
 #include "common.h"
 #include "gdsc.h"
+#include "reset.h"
 
 enum {
 	P_BI_TCXO,
@@ -607,6 +608,10 @@ static struct clk_branch disp_cc_xo_clk = {
 	},
 };
 
+static const struct qcom_reset_map disp_cc_sm6125_resets[] = {
+	[DISP_CC_MDSS_CORE_BCR] = { 0x2000 },
+};
+
 static struct gdsc mdss_gdsc = {
 	.gdscr = 0x3000,
 	.pd = {
@@ -663,6 +668,8 @@ static const struct qcom_cc_desc disp_cc_sm6125_desc = {
 	.config = &disp_cc_sm6125_regmap_config,
 	.clks = disp_cc_sm6125_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm6125_clocks),
+	.resets = disp_cc_sm6125_resets,
+	.num_resets = ARRAY_SIZE(disp_cc_sm6125_resets),
 	.gdscs = disp_cc_sm6125_gdscs,
 	.num_gdscs = ARRAY_SIZE(disp_cc_sm6125_gdscs),
 };
diff --git a/drivers/clk/qcom/dispcc-sm8250.c b/drivers/clk/qcom/dispcc-sm8250.c
index 8f433e1e70283..e59cdadd56479 100644
--- a/drivers/clk/qcom/dispcc-sm8250.c
+++ b/drivers/clk/qcom/dispcc-sm8250.c
@@ -578,7 +578,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = {
 		.name = "disp_cc_mdss_pclk0_clk_src",
 		.parent_data = disp_cc_parent_data_6,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_6),
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
 		.ops = &clk_pixel_ops,
 	},
 };
@@ -592,7 +592,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = {
 		.name = "disp_cc_mdss_pclk1_clk_src",
 		.parent_data = disp_cc_parent_data_6,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_6),
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
 		.ops = &clk_pixel_ops,
 	},
 };
@@ -632,7 +632,7 @@ static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = {
 		.parent_data = disp_cc_parent_data_1,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_1),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_shared_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c
index 9ce9fd28e55b2..2e91332dd92ab 100644
--- a/drivers/clk/qcom/dispcc-sm8450.c
+++ b/drivers/clk/qcom/dispcc-sm8450.c
@@ -409,7 +409,7 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_aux_clk_src = {
 		.parent_data = disp_cc_parent_data_1,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_1),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_dp_ops,
+		.ops = &clk_rcg2_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/dispcc0-sa8775p.c b/drivers/clk/qcom/dispcc0-sa8775p.c
index aeda9cf4bfee8..b248fa9705873 100644
--- a/drivers/clk/qcom/dispcc0-sa8775p.c
+++ b/drivers/clk/qcom/dispcc0-sa8775p.c
@@ -591,7 +591,6 @@ static struct clk_regmap_div mdss_0_disp_cc_mdss_byte0_div_clk_src = {
 			&mdss_0_disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -606,7 +605,6 @@ static struct clk_regmap_div mdss_0_disp_cc_mdss_byte1_div_clk_src = {
 			&mdss_0_disp_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/dispcc1-sa8775p.c b/drivers/clk/qcom/dispcc1-sa8775p.c
index cd55d1c119024..9882edbb79f9e 100644
--- a/drivers/clk/qcom/dispcc1-sa8775p.c
+++ b/drivers/clk/qcom/dispcc1-sa8775p.c
@@ -591,7 +591,6 @@ static struct clk_regmap_div mdss_1_disp_cc_mdss_byte0_div_clk_src = {
 			&mdss_1_disp_cc_mdss_byte0_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
@@ -606,7 +605,6 @@ static struct clk_regmap_div mdss_1_disp_cc_mdss_byte1_div_clk_src = {
 			&mdss_1_disp_cc_mdss_byte1_clk_src.clkr.hw,
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
 		.ops = &clk_regmap_div_ops,
 	},
 };
diff --git a/drivers/clk/qcom/gcc-eliza.c b/drivers/clk/qcom/gcc-eliza.c
new file mode 100644
index 0000000000000..06ee1469badd3
--- /dev/null
+++ b/drivers/clk/qcom/gcc-eliza.c
@@ -0,0 +1,3104 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_SLEEP_CLK,
+	DT_PCIE_0_PIPE_CLK,
+	DT_PCIE_1_PIPE_CLK,
+	DT_UFS_PHY_RX_SYMBOL_0_CLK,
+	DT_UFS_PHY_RX_SYMBOL_1_CLK,
+	DT_UFS_PHY_TX_SYMBOL_0_CLK,
+	DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GCC_GPLL0_OUT_EVEN,
+	P_GCC_GPLL0_OUT_MAIN,
+	P_GCC_GPLL4_OUT_MAIN,
+	P_GCC_GPLL7_OUT_MAIN,
+	P_GCC_GPLL8_OUT_MAIN,
+	P_GCC_GPLL9_OUT_MAIN,
+	P_PCIE_0_PIPE_CLK,
+	P_PCIE_1_PIPE_CLK,
+	P_SLEEP_CLK,
+	P_UFS_PHY_RX_SYMBOL_0_CLK,
+	P_UFS_PHY_RX_SYMBOL_1_CLK,
+	P_UFS_PHY_TX_SYMBOL_0_CLK,
+	P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+static struct clk_alpha_pll gcc_gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ole_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gcc_gpll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gpll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll4 = {
+	.offset = 0x4000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ole_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll7 = {
+	.offset = 0x7000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll7",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ole_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll8 = {
+	.offset = 0x8000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll8",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ole_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll9 = {
+	.offset = 0x9000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll9",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ole_ops,
+		},
+	},
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL7_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll7.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL8_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll8.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data gcc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map gcc_parent_map_7[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL9_OUT_MAIN, 2 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_7[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll9.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 },
+	{ P_BI_TCXO, 2 },
+	{ P_GCC_GPLL0_OUT_EVEN, 3 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK },
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_0_pipe_clk_src = {
+	.reg = 0x6b080,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_PCIE_0_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_1_pipe_clk_src = {
+	.reg = 0xac07c,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_PCIE_1_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_0_clk_src = {
+	.reg = 0x77068,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_1_clk_src = {
+	.reg = 0x770ec,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_1_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_tx_symbol_0_clk_src = {
+	.reg = 0x77058,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_TX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = {
+	.reg = 0x39070,
+	.shift = 0,
+	.width = 2,
+	.parent_map = gcc_parent_map_8,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk_src",
+			.parent_data = gcc_parent_data_8,
+			.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+			.ops = &clk_regmap_mux_closest_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp1_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp2_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp3_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_aux_clk_src = {
+	.cmd_rcgr = 0x6b084,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_aux_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = {
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = {
+	.cmd_rcgr = 0x6b068,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_phy_rchng_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_aux_clk_src = {
+	.cmd_rcgr = 0xac080,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_aux_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = {
+	.cmd_rcgr = 0xac064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_phy_rchng_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pdm2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0),
+	F(250000000, P_GCC_GPLL7_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_qspi_ref_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_qspi_ref_clk_src",
+	.parent_data = gcc_parent_data_4,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_qspi_ref_clk_src = {
+	.cmd_rcgr = 0x188c0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_qspi_ref_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(61440000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 625),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = {
+	.cmd_rcgr = 0x18014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = {
+	.cmd_rcgr = 0x18150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s3_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(61440000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 625),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = {
+	.cmd_rcgr = 0x182a0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = {
+	.cmd_rcgr = 0x183dc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s5_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(61440000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 625),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(128000000, P_GCC_GPLL0_OUT_MAIN, 1, 16, 75),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = {
+	.cmd_rcgr = 0x18518,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s5_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s6_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = {
+	.cmd_rcgr = 0x18654,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s7_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = {
+	.cmd_rcgr = 0x18790,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = {
+	.cmd_rcgr = 0x1e014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = {
+	.cmd_rcgr = 0x1e150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap2_s2_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(61440000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 625),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(125000000, P_GCC_GPLL7_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s2_clk_src",
+	.parent_data = gcc_parent_data_4,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = {
+	.cmd_rcgr = 0x1e28c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_qupv3_wrap2_s2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = {
+	.cmd_rcgr = 0x1e3c8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = {
+	.cmd_rcgr = 0x1e504,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = {
+	.cmd_rcgr = 0x1e640,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s5_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s6_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s6_clk_src",
+	.parent_data = gcc_parent_data_4,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s6_clk_src = {
+	.cmd_rcgr = 0x1e77c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s6_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s7_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s7_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s7_clk_src = {
+	.cmd_rcgr = 0x1e8b8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s3_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s7_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = {
+	F(144000, P_BI_TCXO, 16, 3, 25),
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(20000000, P_GCC_GPLL0_OUT_EVEN, 5, 1, 3),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(192000000, P_GCC_GPLL8_OUT_MAIN, 2, 0, 0),
+	F(384000000, P_GCC_GPLL8_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0xa9018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_sdcc1_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_apps_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = {
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0),
+	F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0xa9040,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_ice_core_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0),
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x1401c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc2_apps_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = {
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0),
+	F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = {
+	.cmd_rcgr = 0x77034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_axi_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = {
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0),
+	F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = {
+	.cmd_rcgr = 0x7708c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_ice_core_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = {
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = {
+	.cmd_rcgr = 0x770c0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = {
+	.cmd_rcgr = 0x770a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_unipro_core_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = {
+	F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0),
+	F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_master_clk_src = {
+	.cmd_rcgr = 0x39030,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x39048,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
+	.cmd_rcgr = 0x39074,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb3_prim_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_pcie_0_pipe_div2_clk_src = {
+	.reg = 0x6b0a4,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_pipe_div2_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_pcie_0_pipe_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_pcie_1_pipe_div2_clk_src = {
+	.reg = 0xac0a0,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_pipe_div2_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_pcie_1_pipe_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_qupv3_wrap1_s2_clk_src = {
+	.reg = 0x1828c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap1_s2_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = {
+	.reg = 0x39060,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb30_prim_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_aggre_noc_pcie_axi_clk = {
+	.halt_reg = 0x10068,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10068,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(30),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_noc_pcie_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_ufs_phy_axi_clk = {
+	.halt_reg = 0x770f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770f4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_usb3_prim_axi_clk = {
+	.halt_reg = 0x39094,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x39094,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x38004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_hf_axi_clk = {
+	.halt_reg = 0x26014,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x26014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_sf_axi_clk = {
+	.halt_reg = 0x26024,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x26024,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_pcie_anoc_ahb_clk = {
+	.halt_reg = 0x10050,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_pcie_anoc_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x39090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x39090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_pcie_sf_axi_clk = {
+	.halt_reg = 0x10058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x10058,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_pcie_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_gpu_axi_clk = {
+	.halt_reg = 0x71158,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x71158,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71158,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ddrss_gpu_axi_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_pcie_sf_qtb_clk = {
+	.halt_reg = 0x1007c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x1007c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ddrss_pcie_sf_qtb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_hf_axi_clk = {
+	.halt_reg = 0x27008,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x27008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gemnoc_gfx_clk = {
+	.halt_reg = 0x71010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gemnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_cph_clk_src = {
+	.halt_reg = 0x71150,
+	.halt_check = BRANCH_HALT_ENABLE_VOTED,
+	.clkr = {
+		.enable_reg = 0x71150,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_cph_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_cph_clk_src = {
+	.halt_reg = 0x71154,
+	.halt_check = BRANCH_HALT_ENABLE_VOTED,
+	.clkr = {
+		.enable_reg = 0x71154,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_div_cph_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0_out_even.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_smmu_vote_clk = {
+	.halt_reg = 0x7d000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_smmu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmu_tcu_vote_clk = {
+	.halt_reg = 0x7d02c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mmu_tcu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b040,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b030,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6b030,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_phy_rchng_clk = {
+	.halt_reg = 0x6b064,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_phy_rchng_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_phy_rchng_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x6b054,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_div2_clk = {
+	.halt_reg = 0x6b0a8,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_div2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_pipe_div2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = {
+	.halt_reg = 0x6b01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_aux_clk = {
+	.halt_reg = 0xac040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_cfg_ahb_clk = {
+	.halt_reg = 0xac03c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xac03c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_mstr_axi_clk = {
+	.halt_reg = 0xac02c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0xac02c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_phy_rchng_clk = {
+	.halt_reg = 0xac060,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_phy_rchng_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_phy_rchng_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_clk = {
+	.halt_reg = 0xac050,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_div2_clk = {
+	.halt_reg = 0xac0a4,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_div2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_pipe_div2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_axi_clk = {
+	.halt_reg = 0xac01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xac01c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = {
+	.halt_reg = 0xac018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x33004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x33008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_cmd_ahb_clk = {
+	.halt_reg = 0x26010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x26010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_cmd_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = {
+	.halt_reg = 0x26008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x26008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_nrt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_rt_ahb_clk = {
+	.halt_reg = 0x2600c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2600c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x2600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_rt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_gpu_ahb_clk = {
+	.halt_reg = 0x71008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_gpu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_pcie_ahb_clk = {
+	.halt_reg = 0x6b018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_pcie_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_v_cpu_ahb_clk = {
+	.halt_reg = 0x32010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x32010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_v_cpu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = {
+	.halt_reg = 0x3200c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x3200c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_vcodec_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = {
+	.halt_reg = 0x2301c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_clk = {
+	.halt_reg = 0x23008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_qspi_ref_clk = {
+	.halt_reg = 0x188bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_qspi_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s0_clk = {
+	.halt_reg = 0x18004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s1_clk = {
+	.halt_reg = 0x18140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s2_clk = {
+	.halt_reg = 0x1827c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s3_clk = {
+	.halt_reg = 0x18290,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s4_clk = {
+	.halt_reg = 0x183cc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s5_clk = {
+	.halt_reg = 0x18508,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s6_clk = {
+	.halt_reg = 0x18644,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s7_clk = {
+	.halt_reg = 0x18780,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = {
+	.halt_reg = 0x23174,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_clk = {
+	.halt_reg = 0x23160,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s0_clk = {
+	.halt_reg = 0x1e004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s1_clk = {
+	.halt_reg = 0x1e140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s2_clk = {
+	.halt_reg = 0x1e27c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s3_clk = {
+	.halt_reg = 0x1e3b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s4_clk = {
+	.halt_reg = 0x1e4f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s5_clk = {
+	.halt_reg = 0x1e630,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s6_clk = {
+	.halt_reg = 0x1e76c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s7_clk = {
+	.halt_reg = 0x1e8a8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = {
+	.halt_reg = 0x23000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = {
+	.halt_reg = 0x23004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = {
+	.halt_reg = 0x23158,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23158,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = {
+	.halt_reg = 0x2315c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2315c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0xa9004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xa9004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0xa9008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xa9008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0xa9030,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xa9030,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xa9030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ahb_clk = {
+	.halt_reg = 0x77028,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_axi_clk = {
+	.halt_reg = 0x77018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ice_core_clk = {
+	.halt_reg = 0x7707c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7707c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7707c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_phy_aux_clk = {
+	.halt_reg = 0x770bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770bc,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = {
+	.halt_reg = 0x77030,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x77030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = {
+	.halt_reg = 0x770d8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x770d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = {
+	.halt_reg = 0x7702c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x7702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_unipro_core_clk = {
+	.halt_reg = 0x7706c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7706c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7706c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_unipro_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_unipro_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_atb_clk = {
+	.halt_reg = 0x3908c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x3908c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_atb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_master_clk = {
+	.halt_reg = 0x39018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_mock_utmi_clk = {
+	.halt_reg = 0x3902c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3902c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_sleep_clk = {
+	.halt_reg = 0x39028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_aux_clk = {
+	.halt_reg = 0x39064,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39064,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = {
+	.halt_reg = 0x39068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_com_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_pipe_clk = {
+	.halt_reg = 0x3906c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x3906c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3906c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi0_clk = {
+	.halt_reg = 0x32018,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x32018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi1_clk = {
+	.halt_reg = 0x32028,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x32028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gcc_pcie_0_gdsc = {
+	.gdscr = 0x6b004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(0),
+	.pd = {
+		.name = "gcc_pcie_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_0_phy_gdsc = {
+	.gdscr = 0x6c000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(2),
+	.pd = {
+		.name = "gcc_pcie_0_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_1_gdsc = {
+	.gdscr = 0xac004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(3),
+	.pd = {
+		.name = "gcc_pcie_1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_1_phy_gdsc = {
+	.gdscr = 0xad000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(4),
+	.pd = {
+		.name = "gcc_pcie_1_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_ufs_mem_phy_gdsc = {
+	.gdscr = 0x9e000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_ufs_mem_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_ufs_phy_gdsc = {
+	.gdscr = 0x77004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_ufs_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb30_prim_gdsc = {
+	.gdscr = 0x39004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_usb30_prim_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb3_phy_gdsc = {
+	.gdscr = 0x50018,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_usb3_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gcc_eliza_clocks[] = {
+	[GCC_AGGRE_NOC_PCIE_AXI_CLK] = &gcc_aggre_noc_pcie_axi_clk.clkr,
+	[GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr,
+	[GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr,
+	[GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr,
+	[GCC_CFG_NOC_PCIE_ANOC_AHB_CLK] = &gcc_cfg_noc_pcie_anoc_ahb_clk.clkr,
+	[GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr,
+	[GCC_CNOC_PCIE_SF_AXI_CLK] = &gcc_cnoc_pcie_sf_axi_clk.clkr,
+	[GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr,
+	[GCC_DDRSS_PCIE_SF_QTB_CLK] = &gcc_ddrss_pcie_sf_qtb_clk.clkr,
+	[GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_GPLL0] = &gcc_gpll0.clkr,
+	[GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr,
+	[GCC_GPLL4] = &gcc_gpll4.clkr,
+	[GCC_GPLL7] = &gcc_gpll7.clkr,
+	[GCC_GPLL8] = &gcc_gpll8.clkr,
+	[GCC_GPLL9] = &gcc_gpll9.clkr,
+	[GCC_GPU_GEMNOC_GFX_CLK] = &gcc_gpu_gemnoc_gfx_clk.clkr,
+	[GCC_GPU_GPLL0_CPH_CLK_SRC] = &gcc_gpu_gpll0_cph_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CPH_CLK_SRC] = &gcc_gpu_gpll0_div_cph_clk_src.clkr,
+	[GCC_GPU_SMMU_VOTE_CLK] = &gcc_gpu_smmu_vote_clk.clkr,
+	[GCC_MMU_TCU_VOTE_CLK] = &gcc_mmu_tcu_vote_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK] = &gcc_pcie_0_phy_rchng_clk.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr,
+	[GCC_PCIE_0_PIPE_DIV2_CLK] = &gcc_pcie_0_pipe_div2_clk.clkr,
+	[GCC_PCIE_0_PIPE_DIV2_CLK_SRC] = &gcc_pcie_0_pipe_div2_clk_src.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr,
+	[GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr,
+	[GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr,
+	[GCC_PCIE_1_PHY_RCHNG_CLK] = &gcc_pcie_1_phy_rchng_clk.clkr,
+	[GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr,
+	[GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr,
+	[GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr,
+	[GCC_PCIE_1_PIPE_DIV2_CLK] = &gcc_pcie_1_pipe_div2_clk.clkr,
+	[GCC_PCIE_1_PIPE_DIV2_CLK_SRC] = &gcc_pcie_1_pipe_div2_clk_src.clkr,
+	[GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr,
+	[GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_QMIP_CAMERA_CMD_AHB_CLK] = &gcc_qmip_camera_cmd_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr,
+	[GCC_QMIP_GPU_AHB_CLK] = &gcc_qmip_gpu_ahb_clk.clkr,
+	[GCC_QMIP_PCIE_AHB_CLK] = &gcc_qmip_pcie_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_V_CPU_AHB_CLK] = &gcc_qmip_video_v_cpu_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK] = &gcc_qupv3_wrap1_qspi_ref_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC] = &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr,
+	[GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S6_CLK] = &gcc_qupv3_wrap2_s6_clk.clkr,
+	[GCC_QUPV3_WRAP2_S6_CLK_SRC] = &gcc_qupv3_wrap2_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S7_CLK] = &gcc_qupv3_wrap2_s7_clk.clkr,
+	[GCC_QUPV3_WRAP2_S7_CLK_SRC] = &gcc_qupv3_wrap2_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr,
+	[GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr,
+	[GCC_USB30_PRIM_ATB_CLK] = &gcc_usb30_prim_atb_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr,
+	[GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr,
+	[GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr,
+	[GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr,
+};
+
+static struct gdsc *gcc_eliza_gdscs[] = {
+	[GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc,
+	[GCC_PCIE_0_PHY_GDSC] = &gcc_pcie_0_phy_gdsc,
+	[GCC_PCIE_1_GDSC] = &gcc_pcie_1_gdsc,
+	[GCC_PCIE_1_PHY_GDSC] = &gcc_pcie_1_phy_gdsc,
+	[GCC_UFS_MEM_PHY_GDSC] = &gcc_ufs_mem_phy_gdsc,
+	[GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc,
+	[GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc,
+	[GCC_USB3_PHY_GDSC] = &gcc_usb3_phy_gdsc,
+};
+
+static const struct qcom_reset_map gcc_eliza_resets[] = {
+	[GCC_CAMERA_BCR] = { 0x26000 },
+	[GCC_DISPLAY_BCR] = { 0x27000 },
+	[GCC_GPU_BCR] = { 0x71000 },
+	[GCC_PCIE_0_BCR] = { 0x6b000 },
+	[GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 },
+	[GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 },
+	[GCC_PCIE_0_PHY_BCR] = { 0x6c01c },
+	[GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 },
+	[GCC_PCIE_1_BCR] = { 0xac000 },
+	[GCC_PCIE_1_LINK_DOWN_BCR] = { 0x8e014 },
+	[GCC_PCIE_1_NOCSR_COM_PHY_BCR] = { 0x8e020 },
+	[GCC_PCIE_1_PHY_BCR] = { 0x8e01c },
+	[GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0x8e024 },
+	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c },
+	[GCC_PCIE_PHY_COM_BCR] = { 0x6f010 },
+	[GCC_PCIE_RSCC_BCR] = { 0x11000 },
+	[GCC_PDM_BCR] = { 0x33000 },
+	[GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 },
+	[GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x12004 },
+	[GCC_SDCC1_BCR] = { 0xa9000 },
+	[GCC_SDCC2_BCR] = { 0x14000 },
+	[GCC_UFS_PHY_BCR] = { 0x77000 },
+	[GCC_USB30_PRIM_BCR] = { 0x39000 },
+	[GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 },
+	[GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 },
+	[GCC_USB3_PHY_PRIM_BCR] = { 0x50000 },
+	[GCC_USB3_PHY_SEC_BCR] = { 0x5000c },
+	[GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 },
+	[GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 },
+	[GCC_VIDEO_AXI0_CLK_ARES] = { 0x32018, 2 },
+	[GCC_VIDEO_AXI1_CLK_ARES] = { 0x32028, 2 },
+	[GCC_VIDEO_BCR] = { 0x32000 },
+};
+
+static u32 gcc_eliza_critical_cbcrs[] = {
+	0xa0004, /* GCC_CAM_BIST_MCLK_AHB_CLK */
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26034, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32038, /* GCC_VIDEO_XO_CLK */
+};
+
+static const struct clk_rcg_dfs_data gcc_eliza_dfs_clocks[] = {
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_qspi_ref_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s6_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s7_clk_src),
+};
+
+static const struct regmap_config gcc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1f41f0,
+	.fast_io = true,
+};
+
+static void clk_eliza_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static struct qcom_cc_driver_data gcc_eliza_driver_data = {
+	.clk_cbcrs = gcc_eliza_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_eliza_critical_cbcrs),
+	.dfs_rcgs = gcc_eliza_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_eliza_dfs_clocks),
+	.clk_regs_configure = clk_eliza_regs_configure,
+};
+
+static const struct qcom_cc_desc gcc_eliza_desc = {
+	.config = &gcc_eliza_regmap_config,
+	.clks = gcc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(gcc_eliza_clocks),
+	.resets = gcc_eliza_resets,
+	.num_resets = ARRAY_SIZE(gcc_eliza_resets),
+	.gdscs = gcc_eliza_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_eliza_gdscs),
+	.driver_data = &gcc_eliza_driver_data,
+};
+
+static const struct of_device_id gcc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_eliza_match_table);
+
+static int gcc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_eliza_desc);
+}
+
+static struct platform_driver gcc_eliza_driver = {
+	.probe = gcc_eliza_probe,
+	.driver = {
+		.name = "gcc-eliza",
+		.of_match_table = gcc_eliza_match_table,
+	},
+};
+
+static int __init gcc_eliza_init(void)
+{
+	return platform_driver_register(&gcc_eliza_driver);
+}
+subsys_initcall(gcc_eliza_init);
+
+static void __exit gcc_eliza_exit(void)
+{
+	platform_driver_unregister(&gcc_eliza_driver);
+}
+module_exit(gcc_eliza_exit);
+
+MODULE_DESCRIPTION("QTI GCC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c
index 238e205735ed5..1a5d3d1827054 100644
--- a/drivers/clk/qcom/gcc-glymur.c
+++ b/drivers/clk/qcom/gcc-glymur.c
@@ -6,7 +6,6 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
@@ -8507,6 +8506,7 @@ static const struct qcom_reset_map gcc_glymur_resets[] = {
 	[GCC_VIDEO_AXI0_CLK_ARES] = { 0x3201c, 2 },
 	[GCC_VIDEO_AXI1_CLK_ARES] = { 0x32044, 2 },
 	[GCC_VIDEO_BCR] = { 0x32000 },
+	[GCC_VIDEO_AXI0C_CLK_ARES] = { 0x32030, 2 },
 };
 
 static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = {
@@ -8611,5 +8611,5 @@ static void __exit gcc_glymur_exit(void)
 }
 module_exit(gcc_glymur_exit);
 
-MODULE_DESCRIPTION("QTI GCC GLYMUR Driver");
+MODULE_DESCRIPTION("QTI GCC Glymur Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-ipq5210.c b/drivers/clk/qcom/gcc-ipq5210.c
new file mode 100644
index 0000000000000..3a786a21bdff5
--- /dev/null
+++ b/drivers/clk/qcom/gcc-ipq5210.c
@@ -0,0 +1,2661 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,ipq5210-gcc.h>
+#include <dt-bindings/reset/qcom,ipq5210-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "reset.h"
+
+enum {
+	DT_XO,
+	DT_SLEEP_CLK,
+	DT_PCIE30_PHY0_PIPE_CLK,
+	DT_PCIE30_PHY1_PIPE_CLK,
+	DT_USB3_PHY0_CC_PIPE_CLK,
+	DT_NSS_CMN_CLK,
+};
+
+enum {
+	P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC,
+	P_GPLL0_OUT_AUX,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL2_OUT_AUX,
+	P_GPLL2_OUT_MAIN,
+	P_GPLL4_OUT_AUX,
+	P_GPLL4_OUT_MAIN,
+	P_NSS_CMN_CLK,
+	P_SLEEP_CLK,
+	P_USB3PHY_0_PIPE,
+	P_XO,
+};
+
+static const struct clk_parent_data gcc_parent_data_xo = { .index = DT_XO };
+
+static struct clk_alpha_pll gpll0_main = {
+	.offset = 0x20000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll0_main",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor gpll0_div2 = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0_div2",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gpll0_main.clkr.hw
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0 = {
+	.offset = 0x20000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gpll0_main.clkr.hw
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll2_main = {
+	.offset = 0x21000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll2_main",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll2[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll2 = {
+	.offset = 0x21000,
+	.post_div_table = post_div_table_gpll2,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll2),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll2",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll2_main.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4_main = {
+	.offset = 0x22000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll4_main",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+			/*
+			 * There are no consumers for this GPLL in kernel yet,
+			 * (will be added soon), so the clock framework
+			 * disables this source. But some of the clocks
+			 * initialized by boot loaders uses this source. So we
+			 * need to keep this clock ON. Add the
+			 * CLK_IGNORE_UNUSED flag so the clock will not be
+			 * disabled. Once the consumer in kernel is added, we
+			 * can get rid of this flag.
+			 */
+			.flags = CLK_IS_CRITICAL,
+		},
+	},
+};
+static const struct parent_map gcc_parent_map_xo[] = {
+	{ P_XO, 0 },
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_XO, 0 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_XO },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_XO, 0 },
+	{ P_NSS_CMN_CLK, 1 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_GPLL2_OUT_AUX, 3 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_XO },
+	{ .index = DT_NSS_CMN_CLK },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL2_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_6[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_7[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_7[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4_main.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_9[] = {
+	{ P_XO, 0 },
+	{ P_GPLL4_OUT_AUX, 1 },
+	{ P_GPLL0_OUT_MAIN, 3 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_9[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll4_main.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_10[] = {
+	{ P_XO, 0 },
+	{ P_GPLL4_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_10[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll4_main.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_11[] = {
+	{ P_XO, 0 },
+	{ P_GPLL4_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_11[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll4_main.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_12[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL2_OUT_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_12[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_13[] = {
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_13[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct freq_tbl ftbl_gcc_adss_pwm_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_adss_pwm_clk_src = {
+	.cmd_rcgr = 0x1c004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_adss_pwm_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_nss_ts_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_nss_ts_clk_src = {
+	.cmd_rcgr = 0x17088,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nss_ts_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_system_noc_bfdcd_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	F(266666667, P_GPLL4_OUT_MAIN, 4.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_system_noc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x2e004,
+	.freq_tbl = ftbl_gcc_system_noc_bfdcd_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_system_noc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_nssnoc_memnoc_bfdcd_clk_src[] = {
+	F(429000000, P_NSS_CMN_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_nssnoc_memnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x17004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_nssnoc_memnoc_bfdcd_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nssnoc_memnoc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie0_axi_m_clk_src[] = {
+	F(200000000, P_GPLL4_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie0_axi_m_clk_src = {
+	.cmd_rcgr = 0x28018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_axi_m_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie0_axi_s_clk_src = {
+	.cmd_rcgr = 0x28020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_axi_s_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie0_rchng_clk_src = {
+	.cmd_rcgr = 0x28028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_rchng_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie1_axi_m_clk_src[] = {
+	F(266666667, P_GPLL4_OUT_MAIN, 4.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie1_axi_m_clk_src = {
+	.cmd_rcgr = 0x29018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie1_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_axi_m_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie1_axi_s_clk_src = {
+	.cmd_rcgr = 0x29020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_axi_s_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie1_rchng_clk_src = {
+	.cmd_rcgr = 0x29028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_rchng_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_aux_clk_src[] = {
+	F(20000000, P_GPLL0_OUT_MAIN, 10, 1, 4),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_aux_clk_src = {
+	.cmd_rcgr = 0x28004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_pcie_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_aux_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap_se0_clk_src[] = {
+	F(960000, P_XO, 10, 2, 5),
+	F(3686636, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 2, 217),
+	F(4800000, P_XO, 5, 0, 0),
+	F(7373272, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 4, 217),
+	F(9600000, P_XO, 2.5, 0, 0),
+	F(14746544, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 8, 217),
+	F(16000000, P_GPLL0_OUT_MAIN, 10, 1, 5),
+	F(24000000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2),
+	F(32000000, P_GPLL0_OUT_MAIN, 1, 1, 25),
+	F(40000000, P_GPLL0_OUT_MAIN, 1, 1, 20),
+	F(46400000, P_GPLL0_OUT_MAIN, 2, 29, 250),
+	F(48000000, P_GPLL0_OUT_MAIN, 1, 3, 50),
+	F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(51200000, P_GPLL0_OUT_MAIN, 1, 8, 125),
+	F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 100),
+	F(58986175, P_GPLL0_OUT_MAIN, 1, 16, 217),
+	F(60000000, P_GPLL0_OUT_MAIN, 1, 3, 40),
+	F(64000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se0_clk_src = {
+	.cmd_rcgr = 0x4004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se0_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se1_clk_src = {
+	.cmd_rcgr = 0x5004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se1_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se2_clk_src = {
+	.cmd_rcgr = 0x2018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se3_clk_src = {
+	.cmd_rcgr = 0x2034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se3_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se4_clk_src = {
+	.cmd_rcgr = 0x3018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se4_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se5_clk_src = {
+	.cmd_rcgr = 0x3034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se5_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = {
+	F(144000, P_XO, 16, 12, 125),
+	F(400000, P_XO, 12, 1, 5),
+	F(24000000, P_GPLL2_OUT_MAIN, 12, 1, 2),
+	F(48000000, P_GPLL2_OUT_MAIN, 12, 0, 0),
+	F(96000000, P_GPLL2_OUT_MAIN, 6, 0, 0),
+	F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(192000000, P_GPLL2_OUT_MAIN, 3, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x33004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_sdcc1_apps_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_apps_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = {
+	F(300000000, P_GPLL4_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x33018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_ice_core_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_uniphy_sys_clk_src = {
+	.cmd_rcgr = 0x17090,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_uniphy_sys_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb0_aux_clk_src = {
+	.cmd_rcgr = 0x2c018,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_aux_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb0_master_clk_src[] = {
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb0_master_clk_src = {
+	.cmd_rcgr = 0x2c004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb0_master_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb0_mock_utmi_clk_src[] = {
+	F(60000000, P_GPLL4_OUT_AUX, 10, 1, 2),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb0_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x2c02c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_9,
+	.freq_tbl = ftbl_gcc_usb0_mock_utmi_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_9,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qdss_at_clk_src[] = {
+	F(240000000, P_GPLL4_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qdss_at_clk_src = {
+	.cmd_rcgr = 0x2d004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_qdss_at_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qdss_at_clk_src",
+		.parent_data = gcc_parent_data_10,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qdss_tsctr_clk_src[] = {
+	F(600000000, P_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qdss_tsctr_clk_src = {
+	.cmd_rcgr = 0x2d01c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_qdss_tsctr_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qdss_tsctr_clk_src",
+		.parent_data = gcc_parent_data_10,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcnoc_bfdcd_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x31004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcnoc_bfdcd_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcnoc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+		/*
+		 * There are no consumers for this source in kernel yet,
+		 * (will be added soon), so the clock framework
+		 * disables this source. But some of the clocks
+		 * initialized by boot loaders uses this source. So we
+		 * need to keep this clock ON. Add the
+		 * CLK_IGNORE_UNUSED flag so the clock will not be
+		 * disabled. Once the consumer in kernel is added, we
+		 * can get rid of this flag.
+		 */
+		.flags = CLK_IS_CRITICAL,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qpic_io_macro_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	F(400000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qpic_io_macro_clk_src = {
+	.cmd_rcgr = 0x32004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_12,
+	.freq_tbl = ftbl_gcc_qpic_io_macro_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qpic_io_macro_clk_src",
+		.parent_data = gcc_parent_data_12,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qpic_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qpic_clk_src = {
+	.cmd_rcgr = 0x32020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_12,
+	.freq_tbl = ftbl_gcc_qpic_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qpic_clk_src",
+		.parent_data = gcc_parent_data_12,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pon_tm2x_clk_src[] = {
+	F(342860000, P_GPLL4_OUT_MAIN, 3.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pon_tm2x_clk_src = {
+	.cmd_rcgr = 0x3c004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_11,
+	.freq_tbl = ftbl_gcc_pon_tm2x_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pon_tm2x_clk_src",
+		.parent_data = gcc_parent_data_11,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_11),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sleep_clk_src = {
+	.cmd_rcgr = 0x3400c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_13,
+	.freq_tbl = ftbl_gcc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sleep_clk_src",
+		.parent_data = gcc_parent_data_13,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_13),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_lpass_sway_clk_src[] = {
+	F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_lpass_sway_clk_src = {
+	.cmd_rcgr = 0x27004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_lpass_sway_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_lpass_sway_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_lpass_axim_clk_src = {
+	.cmd_rcgr = 0x2700c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_lpass_sway_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_lpass_axim_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_nssnoc_memnoc_div_clk_src = {
+	.reg = 0x1700c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nssnoc_memnoc_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_nssnoc_memnoc_bfdcd_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb0_mock_utmi_div_clk_src = {
+	.reg = 0x2c040,
+	.shift = 0,
+	.width = 2,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_mock_utmi_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb0_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_fixed_factor gcc_pon_tm_div_clk_src = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pon_tm_div_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gcc_pon_tm2x_clk_src.clkr.hw
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_branch gcc_adss_pwm_clk = {
+	.halt_reg = 0x1c00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1c00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_adss_pwm_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_adss_pwm_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_pcie0_1lane_s_clk = {
+	.halt_reg = 0x31088,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x31088,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_pcie0_1lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_pcie1_2lane_s_clk = {
+	.halt_reg = 0x3108c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3108c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_pcie1_2lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_usb_clk = {
+	.halt_reg = 0x310a8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x310a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_usb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdio_ahb_clk = {
+	.halt_reg = 0x17040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mdio_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdio_gephy_ahb_clk = {
+	.halt_reg = 0x17098,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17098,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mdio_gephy_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ts_clk = {
+	.halt_reg = 0x17018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nss_ts_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nss_ts_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nsscc_clk = {
+	.halt_reg = 0x17034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nsscc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nsscfg_clk = {
+	.halt_reg = 0x1702c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nsscfg_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_atb_clk = {
+	.halt_reg = 0x17014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_atb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_at_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_memnoc_1_clk = {
+	.halt_reg = 0x17084,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17084,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_memnoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nssnoc_memnoc_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_memnoc_clk = {
+	.halt_reg = 0x17024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_memnoc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nssnoc_memnoc_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_nsscc_clk = {
+	.halt_reg = 0x17030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_nsscc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 gcc_xo_clk_src = {
+	.cmd_rcgr = 0x34004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_xo,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_xo_clk_src",
+		.parent_data = &gcc_parent_data_xo,
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor gcc_xo_div4_clk_src = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_xo_div4_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gcc_xo_clk_src.clkr.hw
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_branch gcc_gephy_sys_clk = {
+	.halt_reg = 0x2a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gephy_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_pcnoc_1_clk = {
+	.halt_reg = 0x17080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_pcnoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_qosgen_ref_clk = {
+	.halt_reg = 0x1701c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_qosgen_ref_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&gcc_xo_div4_clk_src.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_snoc_1_clk = {
+	.halt_reg = 0x1707c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1707c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_snoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_system_noc_bfdcd_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_snoc_clk = {
+	.halt_reg = 0x17028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_snoc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_timeout_ref_clk = {
+	.halt_reg = 0x17020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_timeout_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_xo_div4_clk_src.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_xo_dcd_clk = {
+	.halt_reg = 0x17074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_xo_dcd_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_ahb_clk = {
+	.halt_reg = 0x28030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_aux_clk = {
+	.halt_reg = 0x28070,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_m_clk = {
+	.halt_reg = 0x28038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_s_bridge_clk = {
+	.halt_reg = 0x28048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_s_clk = {
+	.halt_reg = 0x28040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie0_pipe_clk_src = {
+	.reg = 0x28064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY0_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_pipe_clk = {
+	.halt_reg = 0x28068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x28068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie0_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_ahb_clk = {
+	.halt_reg = 0x29030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_aux_clk = {
+	.halt_reg = 0x29074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_m_clk = {
+	.halt_reg = 0x29038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_s_bridge_clk = {
+	.halt_reg = 0x29048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_s_clk = {
+	.halt_reg = 0x29040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie1_pipe_clk_src = {
+	.reg = 0x29064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie1_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY1_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_pipe_clk = {
+	.halt_reg = 0x29068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x29068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie1_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qrng_ahb_clk = {
+	.halt_reg = 0x13024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qrng_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_ahb_mst_clk = {
+	.halt_reg = 0x1014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_ahb_mst_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_ahb_slv_clk = {
+	.halt_reg = 0x102c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_ahb_slv_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se0_clk = {
+	.halt_reg = 0x4020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se1_clk = {
+	.halt_reg = 0x5020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se2_clk = {
+	.halt_reg = 0x202c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x202c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se3_clk = {
+	.halt_reg = 0x2048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se4_clk = {
+	.halt_reg = 0x302c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se5_clk = {
+	.halt_reg = 0x3048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x3303c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3303c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x3302c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x33034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_snoc_pcie0_axi_m_clk = {
+	.halt_reg = 0x2e04c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_snoc_pcie0_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_snoc_pcie1_axi_m_clk = {
+	.halt_reg = 0x2e050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_snoc_pcie1_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_ahb_clk = {
+	.halt_reg = 0x1704c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1704c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_sys_clk = {
+	.halt_reg = 0x17048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy0_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_ahb_clk = {
+	.halt_reg = 0x1705c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1705c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_sys_clk = {
+	.halt_reg = 0x17058,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy1_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_ahb_clk = {
+	.halt_reg = 0x1706c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1706c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy2_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_sys_clk = {
+	.halt_reg = 0x17068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy2_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_aux_clk = {
+	.halt_reg = 0x2c04c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_master_clk = {
+	.halt_reg = 0x2c044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_mock_utmi_clk = {
+	.halt_reg = 0x2c050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_mock_utmi_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_phy_cfg_ahb_clk = {
+	.halt_reg = 0x2c05c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c05c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_phy_cfg_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_usb0_pipe_clk_src = {
+	.reg = 0x2c074,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_USB3_PHY0_CC_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_pipe_clk = {
+	.halt_reg = 0x2c054,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2c054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_usb0_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_sleep_clk = {
+	.halt_reg = 0x2c058,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2c058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_sleep_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sleep_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_rchng_clk = {
+	.halt_reg = 0x28028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie0_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_rchng_clk = {
+	.halt_reg = 0x29028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie1_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_ahb_clk = {
+	.halt_reg = 0x32010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x32010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_clk = {
+	.halt_reg = 0x32028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x32028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qpic_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_io_macro_clk = {
+	.halt_reg = 0x3200c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_io_macro_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qpic_io_macro_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_ahb_clk = {
+	.halt_reg = 0x3a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cmn_12gpll_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_sys_clk = {
+	.halt_reg = 0x3a008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cmn_12gpll_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qdss_at_clk = {
+	.halt_reg = 0x2d034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2d034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qdss_at_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_at_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qdss_dap_clk = {
+	.halt_reg = 0x2d058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qdss_dap_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_tsctr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pon_apb_clk = {
+	.halt_reg = 0x3c01c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pon_apb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pon_tm_clk = {
+	.halt_reg = 0x3c014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pon_tm_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pon_tm_div_clk_src.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pon_tm2x_clk = {
+	.halt_reg = 0x3c00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pon_tm2x_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pon_tm2x_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_snoc_lpass_clk = {
+	.halt_reg = 0x2e028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_snoc_lpass_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_lpass_axim_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_lpass_sway_clk = {
+	.halt_reg = 0x27014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x27014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_lpass_sway_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_lpass_sway_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_lpass_cfg_clk = {
+	.halt_reg = 0x31020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x31020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_lpass_cfg_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_lpass_sway_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_lpass_core_axim_clk = {
+	.halt_reg = 0x27018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x27018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_lpass_core_axim_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_lpass_axim_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static __maybe_unused struct clk_regmap *gcc_ipq5210_clocks[] = {
+	[GCC_ADSS_PWM_CLK] = &gcc_adss_pwm_clk.clkr,
+	[GCC_ADSS_PWM_CLK_SRC] = &gcc_adss_pwm_clk_src.clkr,
+	[GCC_CMN_12GPLL_AHB_CLK] = &gcc_cmn_12gpll_ahb_clk.clkr,
+	[GCC_CMN_12GPLL_SYS_CLK] = &gcc_cmn_12gpll_sys_clk.clkr,
+	[GCC_CNOC_LPASS_CFG_CLK] = &gcc_cnoc_lpass_cfg_clk.clkr,
+	[GCC_CNOC_PCIE0_1LANE_S_CLK] = &gcc_cnoc_pcie0_1lane_s_clk.clkr,
+	[GCC_CNOC_PCIE1_2LANE_S_CLK] = &gcc_cnoc_pcie1_2lane_s_clk.clkr,
+	[GCC_CNOC_USB_CLK] = &gcc_cnoc_usb_clk.clkr,
+	[GCC_GEPHY_SYS_CLK] = &gcc_gephy_sys_clk.clkr,
+	[GCC_LPASS_AXIM_CLK_SRC] = &gcc_lpass_axim_clk_src.clkr,
+	[GCC_LPASS_CORE_AXIM_CLK] = &gcc_lpass_core_axim_clk.clkr,
+	[GCC_LPASS_SWAY_CLK] = &gcc_lpass_sway_clk.clkr,
+	[GCC_LPASS_SWAY_CLK_SRC] = &gcc_lpass_sway_clk_src.clkr,
+	[GCC_MDIO_AHB_CLK] = &gcc_mdio_ahb_clk.clkr,
+	[GCC_MDIO_GEPHY_AHB_CLK] = &gcc_mdio_gephy_ahb_clk.clkr,
+	[GCC_NSS_TS_CLK] = &gcc_nss_ts_clk.clkr,
+	[GCC_NSS_TS_CLK_SRC] = &gcc_nss_ts_clk_src.clkr,
+	[GCC_NSSCC_CLK] = &gcc_nsscc_clk.clkr,
+	[GCC_NSSCFG_CLK] = &gcc_nsscfg_clk.clkr,
+	[GCC_NSSNOC_ATB_CLK] = &gcc_nssnoc_atb_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_1_CLK] = &gcc_nssnoc_memnoc_1_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_BFDCD_CLK_SRC] = &gcc_nssnoc_memnoc_bfdcd_clk_src.clkr,
+	[GCC_NSSNOC_MEMNOC_CLK] = &gcc_nssnoc_memnoc_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_DIV_CLK_SRC] = &gcc_nssnoc_memnoc_div_clk_src.clkr,
+	[GCC_NSSNOC_NSSCC_CLK] = &gcc_nssnoc_nsscc_clk.clkr,
+	[GCC_NSSNOC_PCNOC_1_CLK] = &gcc_nssnoc_pcnoc_1_clk.clkr,
+	[GCC_NSSNOC_QOSGEN_REF_CLK] = &gcc_nssnoc_qosgen_ref_clk.clkr,
+	[GCC_NSSNOC_SNOC_1_CLK] = &gcc_nssnoc_snoc_1_clk.clkr,
+	[GCC_NSSNOC_SNOC_CLK] = &gcc_nssnoc_snoc_clk.clkr,
+	[GCC_NSSNOC_TIMEOUT_REF_CLK] = &gcc_nssnoc_timeout_ref_clk.clkr,
+	[GCC_NSSNOC_XO_DCD_CLK] = &gcc_nssnoc_xo_dcd_clk.clkr,
+	[GCC_PCIE0_AHB_CLK] = &gcc_pcie0_ahb_clk.clkr,
+	[GCC_PCIE0_AUX_CLK] = &gcc_pcie0_aux_clk.clkr,
+	[GCC_PCIE0_AXI_M_CLK] = &gcc_pcie0_axi_m_clk.clkr,
+	[GCC_PCIE0_AXI_M_CLK_SRC] = &gcc_pcie0_axi_m_clk_src.clkr,
+	[GCC_PCIE0_AXI_S_BRIDGE_CLK] = &gcc_pcie0_axi_s_bridge_clk.clkr,
+	[GCC_PCIE0_AXI_S_CLK] = &gcc_pcie0_axi_s_clk.clkr,
+	[GCC_PCIE0_AXI_S_CLK_SRC] = &gcc_pcie0_axi_s_clk_src.clkr,
+	[GCC_PCIE0_PIPE_CLK] = &gcc_pcie0_pipe_clk.clkr,
+	[GCC_PCIE0_PIPE_CLK_SRC] = &gcc_pcie0_pipe_clk_src.clkr,
+	[GCC_PCIE0_RCHNG_CLK] = &gcc_pcie0_rchng_clk.clkr,
+	[GCC_PCIE0_RCHNG_CLK_SRC] = &gcc_pcie0_rchng_clk_src.clkr,
+	[GCC_PCIE1_AHB_CLK] = &gcc_pcie1_ahb_clk.clkr,
+	[GCC_PCIE1_AUX_CLK] = &gcc_pcie1_aux_clk.clkr,
+	[GCC_PCIE1_AXI_M_CLK] = &gcc_pcie1_axi_m_clk.clkr,
+	[GCC_PCIE1_AXI_M_CLK_SRC] = &gcc_pcie1_axi_m_clk_src.clkr,
+	[GCC_PCIE1_AXI_S_BRIDGE_CLK] = &gcc_pcie1_axi_s_bridge_clk.clkr,
+	[GCC_PCIE1_AXI_S_CLK] = &gcc_pcie1_axi_s_clk.clkr,
+	[GCC_PCIE1_AXI_S_CLK_SRC] = &gcc_pcie1_axi_s_clk_src.clkr,
+	[GCC_PCIE1_PIPE_CLK] = &gcc_pcie1_pipe_clk.clkr,
+	[GCC_PCIE1_PIPE_CLK_SRC] = &gcc_pcie1_pipe_clk_src.clkr,
+	[GCC_PCIE1_RCHNG_CLK] = &gcc_pcie1_rchng_clk.clkr,
+	[GCC_PCIE1_RCHNG_CLK_SRC] = &gcc_pcie1_rchng_clk_src.clkr,
+	[GCC_PCIE_AUX_CLK_SRC] = &gcc_pcie_aux_clk_src.clkr,
+	[GCC_PCNOC_BFDCD_CLK_SRC] = &gcc_pcnoc_bfdcd_clk_src.clkr,
+	[GCC_PON_APB_CLK] = &gcc_pon_apb_clk.clkr,
+	[GCC_PON_TM_CLK] = &gcc_pon_tm_clk.clkr,
+	[GCC_PON_TM2X_CLK] = &gcc_pon_tm2x_clk.clkr,
+	[GCC_PON_TM2X_CLK_SRC] = &gcc_pon_tm2x_clk_src.clkr,
+	[GCC_QDSS_AT_CLK] = &gcc_qdss_at_clk.clkr,
+	[GCC_QDSS_AT_CLK_SRC] = &gcc_qdss_at_clk_src.clkr,
+	[GCC_QDSS_DAP_CLK] = &gcc_qdss_dap_clk.clkr,
+	[GCC_QDSS_TSCTR_CLK_SRC] = &gcc_qdss_tsctr_clk_src.clkr,
+	[GCC_QPIC_AHB_CLK] = &gcc_qpic_ahb_clk.clkr,
+	[GCC_QPIC_CLK] = &gcc_qpic_clk.clkr,
+	[GCC_QPIC_CLK_SRC] = &gcc_qpic_clk_src.clkr,
+	[GCC_QPIC_IO_MACRO_CLK] = &gcc_qpic_io_macro_clk.clkr,
+	[GCC_QPIC_IO_MACRO_CLK_SRC] = &gcc_qpic_io_macro_clk_src.clkr,
+	[GCC_QRNG_AHB_CLK] = &gcc_qrng_ahb_clk.clkr,
+	[GCC_QUPV3_AHB_MST_CLK] = &gcc_qupv3_ahb_mst_clk.clkr,
+	[GCC_QUPV3_AHB_SLV_CLK] = &gcc_qupv3_ahb_slv_clk.clkr,
+	[GCC_QUPV3_WRAP_SE0_CLK] = &gcc_qupv3_wrap_se0_clk.clkr,
+	[GCC_QUPV3_WRAP_SE0_CLK_SRC] = &gcc_qupv3_wrap_se0_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE1_CLK] = &gcc_qupv3_wrap_se1_clk.clkr,
+	[GCC_QUPV3_WRAP_SE1_CLK_SRC] = &gcc_qupv3_wrap_se1_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE2_CLK] = &gcc_qupv3_wrap_se2_clk.clkr,
+	[GCC_QUPV3_WRAP_SE2_CLK_SRC] = &gcc_qupv3_wrap_se2_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE3_CLK] = &gcc_qupv3_wrap_se3_clk.clkr,
+	[GCC_QUPV3_WRAP_SE3_CLK_SRC] = &gcc_qupv3_wrap_se3_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE4_CLK] = &gcc_qupv3_wrap_se4_clk.clkr,
+	[GCC_QUPV3_WRAP_SE4_CLK_SRC] = &gcc_qupv3_wrap_se4_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE5_CLK] = &gcc_qupv3_wrap_se5_clk.clkr,
+	[GCC_QUPV3_WRAP_SE5_CLK_SRC] = &gcc_qupv3_wrap_se5_clk_src.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr,
+	[GCC_SLEEP_CLK_SRC] = &gcc_sleep_clk_src.clkr,
+	[GCC_SNOC_LPASS_CLK] = &gcc_snoc_lpass_clk.clkr,
+	[GCC_SNOC_PCIE0_AXI_M_CLK] = &gcc_snoc_pcie0_axi_m_clk.clkr,
+	[GCC_SNOC_PCIE1_AXI_M_CLK] = &gcc_snoc_pcie1_axi_m_clk.clkr,
+	[GCC_SYSTEM_NOC_BFDCD_CLK_SRC] = &gcc_system_noc_bfdcd_clk_src.clkr,
+	[GCC_UNIPHY0_AHB_CLK] = &gcc_uniphy0_ahb_clk.clkr,
+	[GCC_UNIPHY0_SYS_CLK] = &gcc_uniphy0_sys_clk.clkr,
+	[GCC_UNIPHY1_AHB_CLK] = &gcc_uniphy1_ahb_clk.clkr,
+	[GCC_UNIPHY1_SYS_CLK] = &gcc_uniphy1_sys_clk.clkr,
+	[GCC_UNIPHY2_AHB_CLK] = &gcc_uniphy2_ahb_clk.clkr,
+	[GCC_UNIPHY2_SYS_CLK] = &gcc_uniphy2_sys_clk.clkr,
+	[GCC_UNIPHY_SYS_CLK_SRC] = &gcc_uniphy_sys_clk_src.clkr,
+	[GCC_USB0_AUX_CLK] = &gcc_usb0_aux_clk.clkr,
+	[GCC_USB0_AUX_CLK_SRC] = &gcc_usb0_aux_clk_src.clkr,
+	[GCC_USB0_MASTER_CLK] = &gcc_usb0_master_clk.clkr,
+	[GCC_USB0_MASTER_CLK_SRC] = &gcc_usb0_master_clk_src.clkr,
+	[GCC_USB0_MOCK_UTMI_CLK] = &gcc_usb0_mock_utmi_clk.clkr,
+	[GCC_USB0_MOCK_UTMI_CLK_SRC] = &gcc_usb0_mock_utmi_clk_src.clkr,
+	[GCC_USB0_MOCK_UTMI_DIV_CLK_SRC] = &gcc_usb0_mock_utmi_div_clk_src.clkr,
+	[GCC_USB0_PHY_CFG_AHB_CLK] = &gcc_usb0_phy_cfg_ahb_clk.clkr,
+	[GCC_USB0_PIPE_CLK] = &gcc_usb0_pipe_clk.clkr,
+	[GCC_USB0_PIPE_CLK_SRC] = &gcc_usb0_pipe_clk_src.clkr,
+	[GCC_USB0_SLEEP_CLK] = &gcc_usb0_sleep_clk.clkr,
+	[GCC_XO_CLK_SRC] = &gcc_xo_clk_src.clkr,
+	[GPLL0_MAIN] = &gpll0_main.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL2_MAIN] = &gpll2_main.clkr,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL4_MAIN] = &gpll4_main.clkr,
+};
+
+static const struct qcom_reset_map gcc_ipq5210_resets[] = {
+	[GCC_ADSS_BCR] = { 0x1c000 },
+	[GCC_ADSS_PWM_ARES] = { 0x1c00c, 2 },
+	[GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR] = { 0x38000 },
+	[GCC_APC0_VOLTAGE_DROOP_DETECTOR_GPLL0_ARES] = { 0x3800c, 2 },
+	[GCC_APSS_AHB_ARES] = { 0x24014, 2 },
+	[GCC_APSS_ATB_ARES] = { 0x24034, 2 },
+	[GCC_APSS_AXI_ARES] = { 0x24018, 2 },
+	[GCC_APSS_TS_ARES] = { 0x24030, 2 },
+	[GCC_BOOT_ROM_AHB_ARES] = { 0x1302c, 2 },
+	[GCC_BOOT_ROM_BCR] = { 0x13028 },
+	[GCC_GEPHY_BCR] = { 0x2a000 },
+	[GCC_GEPHY_SYS_ARES] = { 0x2a004, 2 },
+	[GCC_GP1_ARES] = { 0x8018, 2 },
+	[GCC_GP2_ARES] = { 0x9018, 2 },
+	[GCC_GP3_ARES] = { 0xa018, 2 },
+	[GCC_MDIO_AHB_ARES] = { 0x17040, 2 },
+	[GCC_MDIO_BCR] = { 0x1703c },
+	[GCC_MDIO_GEPHY_AHB_ARES] = { 0x17098, 2 },
+	[GCC_NSS_BCR] = { 0x17000 },
+	[GCC_NSS_TS_ARES] = { 0x17018, 2 },
+	[GCC_NSSCC_ARES] = { 0x17034, 2 },
+	[GCC_NSSCFG_ARES] = { 0x1702c, 2 },
+	[GCC_NSSNOC_ATB_ARES] = { 0x17014, 2 },
+	[GCC_NSSNOC_MEMNOC_1_ARES] = { 0x17084, 2 },
+	[GCC_NSSNOC_MEMNOC_ARES] = { 0x17024, 2 },
+	[GCC_NSSNOC_NSSCC_ARES] = { 0x17030, 2 },
+	[GCC_NSSNOC_PCNOC_1_ARES] = { 0x17080, 2 },
+	[GCC_NSSNOC_QOSGEN_REF_ARES] = { 0x1701c, 2 },
+	[GCC_NSSNOC_SNOC_1_ARES] = { 0x1707c, 2 },
+	[GCC_NSSNOC_SNOC_ARES] = { 0x17028, 2 },
+	[GCC_NSSNOC_TIMEOUT_REF_ARES] = { 0x17020, 2 },
+	[GCC_NSSNOC_XO_DCD_ARES] = { 0x17074, 2 },
+	[GCC_PCIE0_AHB_ARES] = { 0x28030, 2 },
+	[GCC_PCIE0_AUX_ARES] = { 0x28070, 2 },
+	[GCC_PCIE0_AXI_M_ARES] = { 0x28038, 2 },
+	[GCC_PCIE0_AXI_S_BRIDGE_ARES] = { 0x28048, 2 },
+	[GCC_PCIE0_AXI_S_ARES] = { 0x28040, 2 },
+	[GCC_PCIE0_BCR] = { 0x28000 },
+	[GCC_PCIE0_LINK_DOWN_BCR] = { 0x28054 },
+	[GCC_PCIE0_PIPE_RESET] = { 0x28058, 0 },
+	[GCC_PCIE0_CORE_STICKY_RESET] = { 0x28058, 1 },
+	[GCC_PCIE0_AXI_S_STICKY_RESET] = { 0x28058, 2 },
+	[GCC_PCIE0_AXI_S_RESET] = { 0x28058, 3 },
+	[GCC_PCIE0_AXI_M_STICKY_RESET] = { 0x28058, 4 },
+	[GCC_PCIE0_AXI_M_RESET] = { 0x28058, 5 },
+	[GCC_PCIE0_AUX_RESET] = { 0x28058, 6 },
+	[GCC_PCIE0_AHB_RESET] = { 0x28058, 7 },
+	[GCC_PCIE0_PHY_BCR] = { 0x28060 },
+	[GCC_PCIE0_PIPE_ARES] = { 0x28068, 2 },
+	[GCC_PCIE0PHY_PHY_BCR] = { 0x2805c },
+	[GCC_PCIE1_AHB_ARES] = { 0x29030, 2 },
+	[GCC_PCIE1_AUX_ARES] = { 0x29074, 2 },
+	[GCC_PCIE1_AXI_M_ARES] = { 0x29038, 2 },
+	[GCC_PCIE1_AXI_S_BRIDGE_ARES] = { 0x29048, 2 },
+	[GCC_PCIE1_AXI_S_ARES] = { 0x29040, 2 },
+	[GCC_PCIE1_BCR] = { 0x29000 },
+	[GCC_PCIE1_LINK_DOWN_BCR] = { 0x29054 },
+	[GCC_PCIE1_PIPE_RESET] = { 0x29058, 0 },
+	[GCC_PCIE1_CORE_STICKY_RESET] = { 0x29058, 1 },
+	[GCC_PCIE1_AXI_S_STICKY_RESET] = { 0x29058, 2 },
+	[GCC_PCIE1_AXI_S_RESET] = { 0x29058, 3 },
+	[GCC_PCIE1_AXI_M_STICKY_RESET] = { 0x29058, 4 },
+	[GCC_PCIE1_AXI_M_RESET] = { 0x29058, 5 },
+	[GCC_PCIE1_AUX_RESET] = { 0x29058, 6 },
+	[GCC_PCIE1_AHB_RESET] = { 0x29058, 7 },
+	[GCC_PCIE1_PHY_BCR] = { 0x29060 },
+	[GCC_PCIE1_PIPE_ARES] = { 0x29068, 2 },
+	[GCC_PCIE1PHY_PHY_BCR] = { 0x2905c },
+	[GCC_QRNG_AHB_ARES] = { 0x13024, 2 },
+	[GCC_QRNG_BCR] = { 0x13020 },
+	[GCC_QUPV3_2X_CORE_ARES] = { 0x1020, 2 },
+	[GCC_QUPV3_AHB_MST_ARES] = { 0x1014, 2 },
+	[GCC_QUPV3_AHB_SLV_ARES] = { 0x102c, 2 },
+	[GCC_QUPV3_BCR] = { 0x1000 },
+	[GCC_QUPV3_CORE_ARES] = { 0x1018, 2 },
+	[GCC_QUPV3_WRAP_SE0_ARES] = { 0x4020, 2 },
+	[GCC_QUPV3_WRAP_SE0_BCR] = { 0x4000 },
+	[GCC_QUPV3_WRAP_SE1_ARES] = { 0x5020, 2 },
+	[GCC_QUPV3_WRAP_SE1_BCR] = { 0x5000 },
+	[GCC_QUPV3_WRAP_SE2_ARES] = { 0x202c, 2 },
+	[GCC_QUPV3_WRAP_SE2_BCR] = { 0x2000 },
+	[GCC_QUPV3_WRAP_SE3_ARES] = { 0x2048, 2 },
+	[GCC_QUPV3_WRAP_SE3_BCR] = { 0x2030 },
+	[GCC_QUPV3_WRAP_SE4_ARES] = { 0x302c, 2 },
+	[GCC_QUPV3_WRAP_SE4_BCR] = { 0x3000 },
+	[GCC_QUPV3_WRAP_SE5_ARES] = { 0x3048, 2 },
+	[GCC_QUPV3_WRAP_SE5_BCR] = { 0x3030 },
+	[GCC_QUSB2_0_PHY_BCR] = { 0x2c068 },
+	[GCC_SDCC1_AHB_ARES] = { 0x3303c, 2 },
+	[GCC_SDCC1_APPS_ARES] = { 0x3302c, 2 },
+	[GCC_SDCC1_ICE_CORE_ARES] = { 0x33034, 2 },
+	[GCC_SDCC_BCR] = { 0x33000 },
+	[GCC_TLMM_AHB_ARES] = { 0x3e004, 2 },
+	[GCC_TLMM_ARES] = { 0x3e008, 2 },
+	[GCC_TLMM_BCR] = { 0x3e000 },
+	[GCC_UNIPHY0_AHB_ARES] = { 0x1704c, 2 },
+	[GCC_UNIPHY0_BCR] = { 0x17044 },
+	[GCC_UNIPHY0_SYS_ARES] = { 0x17048, 2 },
+	[GCC_UNIPHY1_AHB_ARES] = { 0x1705c, 2 },
+	[GCC_UNIPHY1_BCR] = { 0x17054 },
+	[GCC_UNIPHY1_SYS_ARES] = { 0x17058, 2 },
+	[GCC_UNIPHY2_AHB_ARES] = { 0x1706c, 2 },
+	[GCC_UNIPHY2_BCR] = { 0x17064 },
+	[GCC_UNIPHY2_SYS_ARES] = { 0x17068, 2 },
+	[GCC_USB0_AUX_ARES] = { 0x2c04c, 2 },
+	[GCC_USB0_MASTER_ARES] = { 0x2c044, 2 },
+	[GCC_USB0_MOCK_UTMI_ARES] = { 0x2c050, 2 },
+	[GCC_USB0_PHY_BCR] = { 0x2c06c },
+	[GCC_USB0_PHY_CFG_AHB_ARES] = { 0x2c05c, 2 },
+	[GCC_USB0_PIPE_ARES] = { 0x2c054, 2 },
+	[GCC_USB0_SLEEP_ARES] = { 0x2c058, 2 },
+	[GCC_USB3PHY_0_PHY_BCR] = { 0x2c070 },
+	[GCC_USB_BCR] = { 0x2c000 },
+	[GCC_QDSS_BCR] = { 0x2d000 },
+};
+
+static const struct of_device_id gcc_ipq5210_match_table[] = {
+	{ .compatible = "qcom,ipq5210-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_ipq5210_match_table);
+
+static const struct regmap_config gcc_ipq5210_regmap_config = {
+	.reg_bits       = 32,
+	.reg_stride     = 4,
+	.val_bits       = 32,
+	.max_register   = 0x3f024,
+	.fast_io        = true,
+};
+
+static struct clk_hw *gcc_ipq5210_hws[] = {
+	&gpll0_div2.hw,
+	&gcc_xo_div4_clk_src.hw,
+	&gcc_pon_tm_div_clk_src.hw,
+};
+
+static const struct qcom_cc_desc gcc_ipq5210_desc = {
+	.config = &gcc_ipq5210_regmap_config,
+	.clks = gcc_ipq5210_clocks,
+	.num_clks = ARRAY_SIZE(gcc_ipq5210_clocks),
+	.resets = gcc_ipq5210_resets,
+	.num_resets = ARRAY_SIZE(gcc_ipq5210_resets),
+	.clk_hws = gcc_ipq5210_hws,
+	.num_clk_hws = ARRAY_SIZE(gcc_ipq5210_hws),
+};
+
+static int gcc_ipq5210_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_ipq5210_desc);
+}
+
+static struct platform_driver gcc_ipq5210_driver = {
+	.probe = gcc_ipq5210_probe,
+	.driver = {
+		.name   = "qcom,gcc-ipq5210",
+		.of_match_table = gcc_ipq5210_match_table,
+	},
+};
+
+static int __init gcc_ipq5210_init(void)
+{
+	return platform_driver_register(&gcc_ipq5210_driver);
+}
+core_initcall(gcc_ipq5210_init);
+
+static void __exit gcc_ipq5210_exit(void)
+{
+	platform_driver_unregister(&gcc_ipq5210_driver);
+}
+module_exit(gcc_ipq5210_exit);
+
+MODULE_DESCRIPTION("QTI GCC IPQ5210 Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-ipq6018.c b/drivers/clk/qcom/gcc-ipq6018.c
index d4fc491a18b22..6943dc511534f 100644
--- a/drivers/clk/qcom/gcc-ipq6018.c
+++ b/drivers/clk/qcom/gcc-ipq6018.c
@@ -400,7 +400,7 @@ static struct clk_branch gcc_xo_clk_src = {
 				.fw_name = "xo",
 			},
 			.num_parents = 1,
-			.flags = CLK_SET_RATE_PARENT,
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL,
 			.ops = &clk_branch2_ops,
 		},
 	},
diff --git a/drivers/clk/qcom/gcc-kaanapali.c b/drivers/clk/qcom/gcc-kaanapali.c
index b9743284927d7..210ec7afbb67d 100644
--- a/drivers/clk/qcom/gcc-kaanapali.c
+++ b/drivers/clk/qcom/gcc-kaanapali.c
@@ -6,7 +6,6 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
diff --git a/drivers/clk/qcom/gcc-sc8180x.c b/drivers/clk/qcom/gcc-sc8180x.c
index 31e788e22ab4a..88b95d5326d90 100644
--- a/drivers/clk/qcom/gcc-sc8180x.c
+++ b/drivers/clk/qcom/gcc-sc8180x.c
@@ -4172,7 +4172,7 @@ static struct gdsc usb30_sec_gdsc = {
 	.pd = {
 		.name = "usb30_sec_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4190,7 +4190,7 @@ static struct gdsc usb30_prim_gdsc = {
 	.pd = {
 		.name = "usb30_prim_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4199,7 +4199,7 @@ static struct gdsc pcie_0_gdsc = {
 	.pd = {
 		.name = "pcie_0_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4226,7 +4226,7 @@ static struct gdsc pcie_1_gdsc = {
 	.pd = {
 		.name = "pcie_1_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4235,7 +4235,7 @@ static struct gdsc pcie_2_gdsc = {
 	.pd = {
 		.name = "pcie_2_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4253,7 +4253,7 @@ static struct gdsc pcie_3_gdsc = {
 	.pd = {
 		.name = "pcie_3_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
@@ -4262,10 +4262,55 @@ static struct gdsc usb30_mp_gdsc = {
 	.pd = {
 		.name = "usb30_mp_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = POLL_CFG_GDSCR,
 };
 
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = {
+	.gdscr = 0x7d050,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = {
+	.gdscr = 0x7d058,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf_gdsc = {
+	.gdscr = 0x7d054,
+	.pd = {
+		.name = "hlos1_vote_mmnoc_mmu_tbu_sf_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc hlos1_vote_turing_mmu_tbu0_gdsc = {
+	.gdscr = 0x7d05c,
+	.pd = {
+		.name = "hlos1_vote_turing_mmu_tbu0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc hlos1_vote_turing_mmu_tbu1_gdsc = {
+	.gdscr = 0x7d060,
+	.pd = {
+		.name = "hlos1_vote_turing_mmu_tbu1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
 static struct clk_regmap *gcc_sc8180x_clocks[] = {
 	[GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr,
 	[GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr,
@@ -4560,7 +4605,7 @@ static const struct qcom_reset_map gcc_sc8180x_resets[] = {
 	[GCC_VIDEO_AXI1_CLK_BCR] = { .reg = 0xb028, .bit = 2, .udelay = 150 },
 };
 
-static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = {
+static const struct clk_rcg_dfs_data gcc_sc8180x_dfs_clocks[] = {
 	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src),
 	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src),
 	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src),
@@ -4595,6 +4640,24 @@ static struct gdsc *gcc_sc8180x_gdscs[] = {
 	[USB30_MP_GDSC] = &usb30_mp_gdsc,
 	[USB30_PRIM_GDSC] = &usb30_prim_gdsc,
 	[USB30_SEC_GDSC] = &usb30_sec_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc,
+	[HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf_gdsc,
+	[HLOS1_VOTE_TURING_MMU_TBU0_GDSC] = &hlos1_vote_turing_mmu_tbu0_gdsc,
+	[HLOS1_VOTE_TURING_MMU_TBU1_GDSC] = &hlos1_vote_turing_mmu_tbu1_gdsc,
+};
+
+static u32 gcc_sc8180x_critical_cbcrs[] = {
+	0xb004,  /* GCC_VIDEO_AHB_CLK */
+	0xb008,  /* GCC_CAMERA_AHB_CLK */
+	0xb00c,  /* GCC_DISP_AHB_CLK */
+	0xb040,  /* GCC_VIDEO_XO_CLK */
+	0xb044,  /* GCC_CAMERA_XO_CLK */
+	0xb048,  /* GCC_DISP_XO_CLK */
+	0x48004, /* GCC_CPUSS_GNOC_CLK */
+	0x48190, /* GCC_CPUSS_DVM_BUS_CLK */
+	0x4d004, /* GCC_NPU_CFG_AHB_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
 };
 
 static const struct regmap_config gcc_sc8180x_regmap_config = {
@@ -4605,6 +4668,21 @@ static const struct regmap_config gcc_sc8180x_regmap_config = {
 	.fast_io	= true,
 };
 
+static void clk_sc8180x_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Disable the GPLL0 active input to NPU and GPU via MISC registers */
+	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
+}
+
+static struct qcom_cc_driver_data gcc_sc8180x_driver_data = {
+	.clk_cbcrs = gcc_sc8180x_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sc8180x_critical_cbcrs),
+	.dfs_rcgs = gcc_sc8180x_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_sc8180x_dfs_clocks),
+	.clk_regs_configure = clk_sc8180x_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sc8180x_desc = {
 	.config = &gcc_sc8180x_regmap_config,
 	.clks = gcc_sc8180x_clocks,
@@ -4613,6 +4691,8 @@ static const struct qcom_cc_desc gcc_sc8180x_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sc8180x_resets),
 	.gdscs = gcc_sc8180x_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sc8180x_gdscs),
+	.use_rpm = true,
+	.driver_data = &gcc_sc8180x_driver_data,
 };
 
 static const struct of_device_id gcc_sc8180x_match_table[] = {
@@ -4623,35 +4703,7 @@ MODULE_DEVICE_TABLE(of, gcc_sc8180x_match_table);
 
 static int gcc_sc8180x_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sc8180x_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xb004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb008); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb00c); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb040); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb044); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb048); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x48004); /* GCC_CPUSS_GNOC_CLK */
-	qcom_branch_set_clk_en(regmap, 0x48190); /* GCC_CPUSS_DVM_BUS_CLK */
-	qcom_branch_set_clk_en(regmap, 0x4d004); /* GCC_NPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-
-	/* Disable the GPLL0 active input to NPU and GPU via MISC registers */
-	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-					ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sc8180x_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sc8180x_desc);
 }
 
 static struct platform_driver gcc_sc8180x_driver = {
diff --git a/drivers/clk/qcom/gpucc-glymur.c b/drivers/clk/qcom/gpucc-glymur.c
new file mode 100644
index 0000000000000..1a1d946347d00
--- /dev/null
+++ b/drivers/clk/qcom/gpucc-glymur.c
@@ -0,0 +1,618 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,glymur-gpucc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_GPLL0_OUT_MAIN,
+	DT_GPLL0_OUT_MAIN_DIV,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL0_OUT_MAIN_DIV,
+	P_GPU_CC_PLL0_OUT_EVEN,
+	P_GPU_CC_PLL0_OUT_MAIN,
+	P_GPU_CC_PLL0_OUT_ODD,
+};
+
+static const struct pll_vco taycan_eko_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 1150.0 MHz Configuration */
+static const struct alpha_pll_config gpu_cc_pll0_config = {
+	.l = 0x3b,
+	.alpha = 0xe555,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll gpu_cc_pll0 = {
+	.offset = 0x0,
+	.config = &gpu_cc_pll0_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpu_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpu_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gpu_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpu_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpu_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+static const struct parent_map gpu_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct parent_map gpu_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPU_CC_PLL0_OUT_MAIN, 1 },
+	{ P_GPU_CC_PLL0_OUT_EVEN, 2 },
+	{ P_GPU_CC_PLL0_OUT_ODD, 3 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .hw = &gpu_cc_pll0_out_even.clkr.hw },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct freq_tbl ftbl_gpu_cc_ff_clk_src[] = {
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_ff_clk_src = {
+	.cmd_rcgr = 0x9474,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_0,
+	.freq_tbl = ftbl_gpu_cc_ff_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_ff_clk_src",
+		.parent_data = gpu_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gpu_cc_gmu_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(575000000, P_GPU_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	F(700000000, P_GPU_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	F(725000000, P_GPU_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	F(750000000, P_GPU_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_gmu_clk_src = {
+	.cmd_rcgr = 0x9318,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_1,
+	.freq_tbl = ftbl_gpu_cc_gmu_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_gmu_clk_src",
+		.parent_data = gpu_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gpu_cc_hub_clk_src[] = {
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	F(400000000, P_GPLL0_OUT_MAIN, 1.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_hub_clk_src = {
+	.cmd_rcgr = 0x93f0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_1,
+	.freq_tbl = ftbl_gpu_cc_hub_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_hub_clk_src",
+		.parent_data = gpu_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div gpu_cc_hub_div_clk_src = {
+	.reg = 0x9430,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_hub_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpu_cc_hub_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gpu_cc_ahb_clk = {
+	.halt_reg = 0x90bc,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x90bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_accu_shift_clk = {
+	.halt_reg = 0x9108,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9108,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_accu_shift_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_ff_clk = {
+	.halt_reg = 0x90ec,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x90ec,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_ff_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_ff_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_gmu_clk = {
+	.halt_reg = 0x90d4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x90d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_gmu_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_gmu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cxo_clk = {
+	.halt_reg = 0x90e4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x90e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cxo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_demet_clk = {
+	.halt_reg = 0x9010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_demet_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_dpm_clk = {
+	.halt_reg = 0x910c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x910c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_dpm_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_freq_measure_clk = {
+	.halt_reg = 0x900c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x900c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_freq_measure_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gpu_smmu_vote_clk = {
+	.halt_reg = 0x7000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gpu_smmu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_accu_shift_clk = {
+	.halt_reg = 0x9070,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_accu_shift_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_acd_ahb_ff_clk = {
+	.halt_reg = 0x9068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_acd_ahb_ff_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_ff_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_ahb_ff_clk = {
+	.halt_reg = 0x9064,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9064,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_ahb_ff_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_ff_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_gmu_clk = {
+	.halt_reg = 0x9060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_gmu_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_gmu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_rcg_ahb_ff_clk = {
+	.halt_reg = 0x906c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x906c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_rcg_ahb_ff_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_ff_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_aon_clk = {
+	.halt_reg = 0x93ec,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x93ec,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_aon_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_cx_int_clk = {
+	.halt_reg = 0x90e8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x90e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_cx_int_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_memnoc_gfx_clk = {
+	.halt_reg = 0x90f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x90f0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_memnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_rscc_hub_aon_clk = {
+	.halt_reg = 0x93e8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x93e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_rscc_hub_aon_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_sleep_clk = {
+	.halt_reg = 0x90cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x90cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gpu_cc_cx_gdsc = {
+	.gdscr = 0x9080,
+	.gds_hw_ctrl = 0x9094,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gpu_cc_cx_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gpu_cc_glymur_clocks[] = {
+	[GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr,
+	[GPU_CC_CX_ACCU_SHIFT_CLK] = &gpu_cc_cx_accu_shift_clk.clkr,
+	[GPU_CC_CX_FF_CLK] = &gpu_cc_cx_ff_clk.clkr,
+	[GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr,
+	[GPU_CC_CXO_CLK] = &gpu_cc_cxo_clk.clkr,
+	[GPU_CC_DEMET_CLK] = &gpu_cc_demet_clk.clkr,
+	[GPU_CC_DPM_CLK] = &gpu_cc_dpm_clk.clkr,
+	[GPU_CC_FF_CLK_SRC] = &gpu_cc_ff_clk_src.clkr,
+	[GPU_CC_FREQ_MEASURE_CLK] = &gpu_cc_freq_measure_clk.clkr,
+	[GPU_CC_GMU_CLK_SRC] = &gpu_cc_gmu_clk_src.clkr,
+	[GPU_CC_GPU_SMMU_VOTE_CLK] = &gpu_cc_gpu_smmu_vote_clk.clkr,
+	[GPU_CC_GX_ACCU_SHIFT_CLK] = &gpu_cc_gx_accu_shift_clk.clkr,
+	[GPU_CC_GX_ACD_AHB_FF_CLK] = &gpu_cc_gx_acd_ahb_ff_clk.clkr,
+	[GPU_CC_GX_AHB_FF_CLK] = &gpu_cc_gx_ahb_ff_clk.clkr,
+	[GPU_CC_GX_GMU_CLK] = &gpu_cc_gx_gmu_clk.clkr,
+	[GPU_CC_GX_RCG_AHB_FF_CLK] = &gpu_cc_gx_rcg_ahb_ff_clk.clkr,
+	[GPU_CC_HUB_AON_CLK] = &gpu_cc_hub_aon_clk.clkr,
+	[GPU_CC_HUB_CLK_SRC] = &gpu_cc_hub_clk_src.clkr,
+	[GPU_CC_HUB_CX_INT_CLK] = &gpu_cc_hub_cx_int_clk.clkr,
+	[GPU_CC_HUB_DIV_CLK_SRC] = &gpu_cc_hub_div_clk_src.clkr,
+	[GPU_CC_MEMNOC_GFX_CLK] = &gpu_cc_memnoc_gfx_clk.clkr,
+	[GPU_CC_PLL0] = &gpu_cc_pll0.clkr,
+	[GPU_CC_PLL0_OUT_EVEN] = &gpu_cc_pll0_out_even.clkr,
+	[GPU_CC_RSCC_HUB_AON_CLK] = &gpu_cc_rscc_hub_aon_clk.clkr,
+	[GPU_CC_SLEEP_CLK] = &gpu_cc_sleep_clk.clkr,
+};
+
+static struct gdsc *gpu_cc_glymur_gdscs[] = {
+	[GPU_CC_CX_GDSC] = &gpu_cc_cx_gdsc,
+};
+
+static const struct qcom_reset_map gpu_cc_glymur_resets[] = {
+	[GPU_CC_CB_BCR] = { 0x93a0 },
+	[GPU_CC_CX_BCR] = { 0x907c },
+	[GPU_CC_FAST_HUB_BCR] = { 0x93e4 },
+	[GPU_CC_FF_BCR] = { 0x9470 },
+	[GPU_CC_GMU_BCR] = { 0x9314 },
+	[GPU_CC_GX_BCR] = { 0x905c },
+	[GPU_CC_XO_BCR] = { 0x9000 },
+};
+
+static struct clk_alpha_pll *gpu_cc_glymur_plls[] = {
+	&gpu_cc_pll0,
+};
+
+static u32 gpu_cc_glymur_critical_cbcrs[] = {
+	0x93a4, /* GPU_CC_CB_CLK */
+	0x9008, /* GPU_CC_CXO_AON_CLK */
+	0x9004, /* GPU_CC_RSCC_XO_AON_CLK */
+};
+
+static const struct regmap_config gpu_cc_glymur_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x95e8,
+	.fast_io = true,
+};
+
+static struct qcom_cc_driver_data gpu_cc_glymur_driver_data = {
+	.alpha_plls = gpu_cc_glymur_plls,
+	.num_alpha_plls = ARRAY_SIZE(gpu_cc_glymur_plls),
+	.clk_cbcrs = gpu_cc_glymur_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_glymur_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc gpu_cc_glymur_desc = {
+	.config = &gpu_cc_glymur_regmap_config,
+	.clks = gpu_cc_glymur_clocks,
+	.num_clks = ARRAY_SIZE(gpu_cc_glymur_clocks),
+	.resets = gpu_cc_glymur_resets,
+	.num_resets = ARRAY_SIZE(gpu_cc_glymur_resets),
+	.gdscs = gpu_cc_glymur_gdscs,
+	.num_gdscs = ARRAY_SIZE(gpu_cc_glymur_gdscs),
+	.use_rpm = true,
+	.driver_data = &gpu_cc_glymur_driver_data,
+};
+
+static const struct of_device_id gpu_cc_glymur_match_table[] = {
+	{ .compatible = "qcom,glymur-gpucc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gpu_cc_glymur_match_table);
+
+static int gpu_cc_glymur_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gpu_cc_glymur_desc);
+}
+
+static struct platform_driver gpu_cc_glymur_driver = {
+	.probe = gpu_cc_glymur_probe,
+	.driver = {
+		.name = "gpucc-glymur",
+		.of_match_table = gpu_cc_glymur_match_table,
+	},
+};
+
+module_platform_driver(gpu_cc_glymur_driver);
+
+MODULE_DESCRIPTION("QTI GPUCC Glymur Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gpucc-kaanapali.c b/drivers/clk/qcom/gpucc-kaanapali.c
index 52be48c15c672..d93d06067fbfe 100644
--- a/drivers/clk/qcom/gpucc-kaanapali.c
+++ b/drivers/clk/qcom/gpucc-kaanapali.c
@@ -6,7 +6,6 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
diff --git a/drivers/clk/qcom/gxclkctl-kaanapali.c b/drivers/clk/qcom/gxclkctl-kaanapali.c
index c209ce5fe4f00..795ce40e028b2 100644
--- a/drivers/clk/qcom/gxclkctl-kaanapali.c
+++ b/drivers/clk/qcom/gxclkctl-kaanapali.c
@@ -6,7 +6,6 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
@@ -52,6 +51,7 @@ static const struct qcom_cc_desc gx_clkctl_kaanapali_desc = {
 };
 
 static const struct of_device_id gx_clkctl_kaanapali_match_table[] = {
+	{ .compatible = "qcom,glymur-gxclkctl" },
 	{ .compatible = "qcom,kaanapali-gxclkctl" },
 	{ }
 };
diff --git a/drivers/clk/qcom/tcsrcc-eliza.c b/drivers/clk/qcom/tcsrcc-eliza.c
new file mode 100644
index 0000000000000..5a47a4c77cb5b
--- /dev/null
+++ b/drivers/clk/qcom/tcsrcc-eliza.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-tcsr.h>
+
+#include "clk-branch.h"
+#include "clk-regmap.h"
+#include "common.h"
+
+enum {
+	DT_BI_TCXO_PAD,
+};
+
+static struct clk_branch tcsr_hdmi_clkref_en = {
+	.halt_reg = 0x14,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x14,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_hdmi_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_pcie_0_clkref_en = {
+	.halt_reg = 0x0,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_pcie_0_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_pcie_1_clkref_en = {
+	.halt_reg = 0x1c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x1c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_pcie_1_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_ufs_clkref_en = {
+	.halt_reg = 0x8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_ufs_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb2_clkref_en = {
+	.halt_reg = 0x4,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb2_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb3_clkref_en = {
+	.halt_reg = 0x10,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x10,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb3_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *tcsr_cc_eliza_clocks[] = {
+	[TCSR_HDMI_CLKREF_EN] = &tcsr_hdmi_clkref_en.clkr,
+	[TCSR_PCIE_0_CLKREF_EN] = &tcsr_pcie_0_clkref_en.clkr,
+	[TCSR_PCIE_1_CLKREF_EN] = &tcsr_pcie_1_clkref_en.clkr,
+	[TCSR_UFS_CLKREF_EN] = &tcsr_ufs_clkref_en.clkr,
+	[TCSR_USB2_CLKREF_EN] = &tcsr_usb2_clkref_en.clkr,
+	[TCSR_USB3_CLKREF_EN] = &tcsr_usb3_clkref_en.clkr,
+};
+
+static const struct regmap_config tcsr_cc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_desc tcsr_cc_eliza_desc = {
+	.config = &tcsr_cc_eliza_regmap_config,
+	.clks = tcsr_cc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(tcsr_cc_eliza_clocks),
+};
+
+static const struct of_device_id tcsr_cc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-tcsr" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tcsr_cc_eliza_match_table);
+
+static int tcsr_cc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &tcsr_cc_eliza_desc);
+}
+
+static struct platform_driver tcsr_cc_eliza_driver = {
+	.probe = tcsr_cc_eliza_probe,
+	.driver = {
+		.name = "tcsr_cc-eliza",
+		.of_match_table = tcsr_cc_eliza_match_table,
+	},
+};
+
+static int __init tcsr_cc_eliza_init(void)
+{
+	return platform_driver_register(&tcsr_cc_eliza_driver);
+}
+subsys_initcall(tcsr_cc_eliza_init);
+
+static void __exit tcsr_cc_eliza_exit(void)
+{
+	platform_driver_unregister(&tcsr_cc_eliza_driver);
+}
+module_exit(tcsr_cc_eliza_exit);
+
+MODULE_DESCRIPTION("QTI TCSR_CC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-glymur.c b/drivers/clk/qcom/tcsrcc-glymur.c
index 215bc2ac548da..9c0edebcdbb12 100644
--- a/drivers/clk/qcom/tcsrcc-glymur.c
+++ b/drivers/clk/qcom/tcsrcc-glymur.c
@@ -6,7 +6,6 @@
 #include <linux/clk-provider.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
@@ -309,5 +308,5 @@ static void __exit tcsr_cc_glymur_exit(void)
 }
 module_exit(tcsr_cc_glymur_exit);
 
-MODULE_DESCRIPTION("QTI TCSRCC GLYMUR Driver");
+MODULE_DESCRIPTION("QTI TCSRCC Glymur Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-kaanapali.c b/drivers/clk/qcom/tcsrcc-kaanapali.c
index 4da77367c9e03..db46d639edb80 100644
--- a/drivers/clk/qcom/tcsrcc-kaanapali.c
+++ b/drivers/clk/qcom/tcsrcc-kaanapali.c
@@ -5,7 +5,6 @@
 
 #include <linux/clk-provider.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
diff --git a/drivers/clk/qcom/tcsrcc-sm8750.c b/drivers/clk/qcom/tcsrcc-sm8750.c
index 242e320986ef6..46af987601971 100644
--- a/drivers/clk/qcom/tcsrcc-sm8750.c
+++ b/drivers/clk/qcom/tcsrcc-sm8750.c
@@ -4,8 +4,8 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
diff --git a/drivers/clk/qcom/videocc-glymur.c b/drivers/clk/qcom/videocc-glymur.c
new file mode 100644
index 0000000000000..bb3aae6b83960
--- /dev/null
+++ b/drivers/clk/qcom/videocc-glymur.c
@@ -0,0 +1,532 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,glymur-videocc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_SLEEP_CLK,
+	P_VIDEO_CC_PLL0_OUT_MAIN,
+};
+
+static const struct pll_vco taycan_eko_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll0_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000008,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll video_cc_pll0 = {
+	.offset = 0x0,
+	.config = &video_cc_pll0_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct parent_map video_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map video_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll0.clkr.hw },
+};
+
+static const struct parent_map video_cc_parent_map_2[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_2[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct freq_tbl ftbl_video_cc_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_ahb_clk_src = {
+	.cmd_rcgr = 0x8018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_ahb_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = {
+	F(720000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1014000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1098000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1332000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1600000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1965000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0_clk_src = {
+	.cmd_rcgr = 0x8000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_1,
+	.freq_tbl = ftbl_video_cc_mvs0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0_clk_src",
+		.parent_data = video_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x8120,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_2,
+	.freq_tbl = ftbl_video_cc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_sleep_clk_src",
+		.parent_data = video_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 video_cc_xo_clk_src = {
+	.cmd_rcgr = 0x80f8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_xo_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div video_cc_mvs0_div_clk_src = {
+	.reg = 0x809c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div video_cc_mvs0c_div2_div_clk_src = {
+	.reg = 0x8060,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0c_div2_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div video_cc_mvs1_div_clk_src = {
+	.reg = 0x80d8,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs1_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch video_cc_mvs0_clk = {
+	.halt_reg = 0x807c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x807c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x807c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_freerun_clk = {
+	.halt_reg = 0x808c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x808c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_shift_clk = {
+	.halt_reg = 0x8114,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8114,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8114,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_clk = {
+	.halt_reg = 0x804c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x804c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_div2_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_freerun_clk = {
+	.halt_reg = 0x805c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x805c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_div2_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_shift_clk = {
+	.halt_reg = 0x811c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x811c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x811c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs1_clk = {
+	.halt_reg = 0x80b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80b8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs1_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs1_freerun_clk = {
+	.halt_reg = 0x80c8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x80c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs1_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs1_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs1_shift_clk = {
+	.halt_reg = 0x8118,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8118,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8118,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs1_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc video_cc_mvs0c_gdsc = {
+	.gdscr = 0x8034,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0c_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0_gdsc = {
+	.gdscr = 0x8068,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &video_cc_mvs0c_gdsc.pd,
+};
+
+static struct gdsc video_cc_mvs1_gdsc = {
+	.gdscr = 0x80a4,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *video_cc_glymur_clocks[] = {
+	[VIDEO_CC_AHB_CLK_SRC] = &video_cc_ahb_clk_src.clkr,
+	[VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr,
+	[VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr,
+	[VIDEO_CC_MVS0_DIV_CLK_SRC] = &video_cc_mvs0_div_clk_src.clkr,
+	[VIDEO_CC_MVS0_FREERUN_CLK] = &video_cc_mvs0_freerun_clk.clkr,
+	[VIDEO_CC_MVS0_SHIFT_CLK] = &video_cc_mvs0_shift_clk.clkr,
+	[VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr,
+	[VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC] = &video_cc_mvs0c_div2_div_clk_src.clkr,
+	[VIDEO_CC_MVS0C_FREERUN_CLK] = &video_cc_mvs0c_freerun_clk.clkr,
+	[VIDEO_CC_MVS0C_SHIFT_CLK] = &video_cc_mvs0c_shift_clk.clkr,
+	[VIDEO_CC_MVS1_CLK] = &video_cc_mvs1_clk.clkr,
+	[VIDEO_CC_MVS1_DIV_CLK_SRC] = &video_cc_mvs1_div_clk_src.clkr,
+	[VIDEO_CC_MVS1_FREERUN_CLK] = &video_cc_mvs1_freerun_clk.clkr,
+	[VIDEO_CC_MVS1_SHIFT_CLK] = &video_cc_mvs1_shift_clk.clkr,
+	[VIDEO_CC_PLL0] = &video_cc_pll0.clkr,
+	[VIDEO_CC_SLEEP_CLK_SRC] = &video_cc_sleep_clk_src.clkr,
+	[VIDEO_CC_XO_CLK_SRC] = &video_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *video_cc_glymur_gdscs[] = {
+	[VIDEO_CC_MVS0_GDSC] = &video_cc_mvs0_gdsc,
+	[VIDEO_CC_MVS0C_GDSC] = &video_cc_mvs0c_gdsc,
+	[VIDEO_CC_MVS1_GDSC] = &video_cc_mvs1_gdsc,
+};
+
+static const struct qcom_reset_map video_cc_glymur_resets[] = {
+	[VIDEO_CC_INTERFACE_BCR] = { 0x80dc },
+	[VIDEO_CC_MVS0_BCR] = { 0x8064 },
+	[VIDEO_CC_MVS0C_FREERUN_CLK_ARES] = { 0x805c, 2 },
+	[VIDEO_CC_MVS0C_BCR] = { 0x8030 },
+	[VIDEO_CC_MVS0_FREERUN_CLK_ARES] = { 0x808c, 2 },
+	[VIDEO_CC_MVS1_FREERUN_CLK_ARES] = { 0x80c8, 2 },
+	[VIDEO_CC_MVS1_BCR] = { 0x80a0 },
+};
+
+static struct clk_alpha_pll *video_cc_glymur_plls[] = {
+	&video_cc_pll0,
+};
+
+static u32 video_cc_glymur_critical_cbcrs[] = {
+	0x80e0, /* VIDEO_CC_AHB_CLK */
+	0x8138, /* VIDEO_CC_SLEEP_CLK */
+	0x8110, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct regmap_config video_cc_glymur_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9f54,
+	.fast_io = true,
+};
+
+static void clk_glymur_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Update CTRL_IN register */
+	regmap_update_bits(regmap, 0x9f24, BIT(0), BIT(0));
+}
+
+static struct qcom_cc_driver_data video_cc_glymur_driver_data = {
+	.alpha_plls = video_cc_glymur_plls,
+	.num_alpha_plls = ARRAY_SIZE(video_cc_glymur_plls),
+	.clk_cbcrs = video_cc_glymur_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_glymur_critical_cbcrs),
+	.clk_regs_configure = clk_glymur_regs_configure,
+};
+
+static struct qcom_cc_desc video_cc_glymur_desc = {
+	.config = &video_cc_glymur_regmap_config,
+	.clks = video_cc_glymur_clocks,
+	.num_clks = ARRAY_SIZE(video_cc_glymur_clocks),
+	.resets = video_cc_glymur_resets,
+	.num_resets = ARRAY_SIZE(video_cc_glymur_resets),
+	.gdscs = video_cc_glymur_gdscs,
+	.num_gdscs = ARRAY_SIZE(video_cc_glymur_gdscs),
+	.use_rpm = true,
+	.driver_data = &video_cc_glymur_driver_data,
+};
+
+static const struct of_device_id video_cc_glymur_match_table[] = {
+	{ .compatible = "qcom,glymur-videocc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, video_cc_glymur_match_table);
+
+static int video_cc_glymur_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &video_cc_glymur_desc);
+}
+
+static struct platform_driver video_cc_glymur_driver = {
+	.probe = video_cc_glymur_probe,
+	.driver = {
+		.name = "videocc-glymur",
+		.of_match_table = video_cc_glymur_match_table,
+	},
+};
+
+module_platform_driver(video_cc_glymur_driver);
+
+MODULE_DESCRIPTION("QTI VIDEOCC Glymur Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/videocc-sm8750.c b/drivers/clk/qcom/videocc-sm8750.c
index 823aca2bdd345..5c1034dd5f575 100644
--- a/drivers/clk/qcom/videocc-sm8750.c
+++ b/drivers/clk/qcom/videocc-sm8750.c
@@ -7,7 +7,6 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,sm8750-videocc.h>
diff --git a/drivers/clk/renesas/r9a09g056-cpg.c b/drivers/clk/renesas/r9a09g056-cpg.c
index fead173cae8b2..549c882f9a18d 100644
--- a/drivers/clk/renesas/r9a09g056-cpg.c
+++ b/drivers/clk/renesas/r9a09g056-cpg.c
@@ -289,6 +289,26 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = {
 						BUS_MSTOP(5, BIT(13))),
 	DEF_MOD("wdt_3_clk_loco",		CLK_QEXTAL, 5, 2, 2, 18,
 						BUS_MSTOP(5, BIT(13))),
+	DEF_MOD("rtc_0_clk_rtc",		CLK_PLLCM33_DIV16, 5, 3, 2, 19,
+						BUS_MSTOP(3, BIT(11) | BIT(12))),
+	DEF_MOD("rspi_0_pclk",			CLK_PLLCLN_DIV8, 5, 4, 2, 20,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 5, 2, 21,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_tclk",			CLK_PLLCLN_DIV8, 5, 6, 2, 22,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_1_pclk",			CLK_PLLCLN_DIV8, 5, 7, 2, 23,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 8, 2, 24,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_tclk",			CLK_PLLCLN_DIV8, 5, 9, 2, 25,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_2_pclk",			CLK_PLLCLN_DIV8, 5, 10, 2, 26,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 11, 2, 27,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_tclk",			CLK_PLLCLN_DIV8, 5, 12, 2, 28,
+						BUS_MSTOP(11, BIT(2))),
 	DEF_MOD("rsci0_pclk",			CLK_PLLCLN_DIV16, 5, 13, 2, 29,
 						BUS_MSTOP(11, BIT(3))),
 	DEF_MOD("rsci0_tclk",			CLK_PLLCLN_DIV16, 5, 14, 2, 30,
@@ -389,24 +409,6 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = {
 						BUS_MSTOP(11, BIT(12))),
 	DEF_MOD("rsci9_ps_ps1_n",		CLK_PLLCLN_DIV64, 8, 14, 4, 14,
 						BUS_MSTOP(11, BIT(12))),
-	DEF_MOD("rspi_0_pclk",			CLK_PLLCLN_DIV8, 5, 4, 2, 20,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_0_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 5, 2, 21,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_0_tclk",			CLK_PLLCLN_DIV8, 5, 6, 2, 22,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_1_pclk",			CLK_PLLCLN_DIV8, 5, 7, 2, 23,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_1_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 8, 2, 24,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_1_tclk",			CLK_PLLCLN_DIV8, 5, 9, 2, 25,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_2_pclk",			CLK_PLLCLN_DIV8, 5, 10, 2, 26,
-						BUS_MSTOP(11, BIT(2))),
-	DEF_MOD("rspi_2_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 11, 2, 27,
-						BUS_MSTOP(11, BIT(2))),
-	DEF_MOD("rspi_2_tclk",			CLK_PLLCLN_DIV8, 5, 12, 2, 28,
-						BUS_MSTOP(11, BIT(2))),
 	DEF_MOD("scif_0_clk_pck",		CLK_PLLCM33_DIV16, 8, 15, 4, 15,
 						BUS_MSTOP(3, BIT(14))),
 	DEF_MOD("i3c_0_pclkrw",			CLK_PLLCLN_DIV16, 9, 0, 4, 16,
@@ -593,6 +595,8 @@ static const struct rzv2h_reset r9a09g056_resets[] __initconst = {
 	DEF_RST(9, 2, 4, 3),		/* RSCI8_TRESETN */
 	DEF_RST(9, 3, 4, 4),		/* RSCI9_PRESETN */
 	DEF_RST(9, 4, 4, 5),		/* RSCI9_TRESETN */
+	DEF_RST(7, 9, 3, 10),		/* RTC_0_RST_RTC */
+	DEF_RST(7, 10, 3, 11),		/* RTC_0_RST_RTC_V */
 	DEF_RST(7, 11, 3, 12),		/* RSPI_0_PRESETN */
 	DEF_RST(7, 12, 3, 13),		/* RSPI_0_TRESETN */
 	DEF_RST(7, 13, 3, 14),		/* RSPI_1_PRESETN */
diff --git a/drivers/clk/renesas/r9a09g057-cpg.c b/drivers/clk/renesas/r9a09g057-cpg.c
index 6943cad318b5d..c3174f40fdb40 100644
--- a/drivers/clk/renesas/r9a09g057-cpg.c
+++ b/drivers/clk/renesas/r9a09g057-cpg.c
@@ -280,22 +280,30 @@ static const struct rzv2h_mod_clk r9a09g057_mod_clks[] __initconst = {
 						BUS_MSTOP(11, BIT(15))),
 	DEF_MOD("gtm_7_pclk",			CLK_PLLCLN_DIV16, 4, 10, 2, 10,
 						BUS_MSTOP(12, BIT(0))),
-	DEF_MOD("wdt_0_clkp",			CLK_PLLCM33_DIV16, 4, 11, 2, 11,
-						BUS_MSTOP(3, BIT(10))),
-	DEF_MOD("wdt_0_clk_loco",		CLK_QEXTAL, 4, 12, 2, 12,
-						BUS_MSTOP(3, BIT(10))),
 	DEF_MOD("wdt_1_clkp",			CLK_PLLCLN_DIV16, 4, 13, 2, 13,
 						BUS_MSTOP(1, BIT(0))),
 	DEF_MOD("wdt_1_clk_loco",		CLK_QEXTAL, 4, 14, 2, 14,
 						BUS_MSTOP(1, BIT(0))),
-	DEF_MOD("wdt_2_clkp",			CLK_PLLCLN_DIV16, 4, 15, 2, 15,
-						BUS_MSTOP(5, BIT(12))),
-	DEF_MOD("wdt_2_clk_loco",		CLK_QEXTAL, 5, 0, 2, 16,
-						BUS_MSTOP(5, BIT(12))),
-	DEF_MOD("wdt_3_clkp",			CLK_PLLCLN_DIV16, 5, 1, 2, 17,
-						BUS_MSTOP(5, BIT(13))),
-	DEF_MOD("wdt_3_clk_loco",		CLK_QEXTAL, 5, 2, 2, 18,
-						BUS_MSTOP(5, BIT(13))),
+	DEF_MOD("rtc_0_clk_rtc",		CLK_PLLCM33_DIV16, 5, 3, 2, 19,
+						BUS_MSTOP(3, BIT(11) | BIT(12))),
+	DEF_MOD("rspi_0_pclk",			CLK_PLLCLN_DIV8, 5, 4, 2, 20,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 5, 2, 21,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_tclk",			CLK_PLLCLN_DIV8, 5, 6, 2, 22,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_1_pclk",			CLK_PLLCLN_DIV8, 5, 7, 2, 23,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 8, 2, 24,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_tclk",			CLK_PLLCLN_DIV8, 5, 9, 2, 25,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_2_pclk",			CLK_PLLCLN_DIV8, 5, 10, 2, 26,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 11, 2, 27,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_tclk",			CLK_PLLCLN_DIV8, 5, 12, 2, 28,
+						BUS_MSTOP(11, BIT(2))),
 	DEF_MOD("rsci0_pclk",			CLK_PLLCLN_DIV16, 5, 13, 2, 29,
 						BUS_MSTOP(11, BIT(3))),
 	DEF_MOD("rsci0_tclk",			CLK_PLLCLN_DIV16, 5, 14, 2, 30,
@@ -396,26 +404,6 @@ static const struct rzv2h_mod_clk r9a09g057_mod_clks[] __initconst = {
 						BUS_MSTOP(11, BIT(12))),
 	DEF_MOD("rsci9_ps_ps1_n",		CLK_PLLCLN_DIV64, 8, 14, 4, 14,
 						BUS_MSTOP(11, BIT(12))),
-	DEF_MOD("rtc_0_clk_rtc",		CLK_PLLCM33_DIV16, 5, 3, 2, 19,
-						BUS_MSTOP(3, BIT(11) | BIT(12))),
-	DEF_MOD("rspi_0_pclk",			CLK_PLLCLN_DIV8, 5, 4, 2, 20,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_0_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 5, 2, 21,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_0_tclk",			CLK_PLLCLN_DIV8, 5, 6, 2, 22,
-						BUS_MSTOP(11, BIT(0))),
-	DEF_MOD("rspi_1_pclk",			CLK_PLLCLN_DIV8, 5, 7, 2, 23,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_1_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 8, 2, 24,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_1_tclk",			CLK_PLLCLN_DIV8, 5, 9, 2, 25,
-						BUS_MSTOP(11, BIT(1))),
-	DEF_MOD("rspi_2_pclk",			CLK_PLLCLN_DIV8, 5, 10, 2, 26,
-						BUS_MSTOP(11, BIT(2))),
-	DEF_MOD("rspi_2_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 11, 2, 27,
-						BUS_MSTOP(11, BIT(2))),
-	DEF_MOD("rspi_2_tclk",			CLK_PLLCLN_DIV8, 5, 12, 2, 28,
-						BUS_MSTOP(11, BIT(2))),
 	DEF_MOD("scif_0_clk_pck",		CLK_PLLCM33_DIV16, 8, 15, 4, 15,
 						BUS_MSTOP(3, BIT(14))),
 	DEF_MOD("i3c_0_pclkrw",			CLK_PLLCLN_DIV16, 9, 0, 4, 16,
@@ -598,10 +586,7 @@ static const struct rzv2h_reset r9a09g057_resets[] __initconst = {
 	DEF_RST(7, 2, 3, 3),		/* GTM_5_PRESETZ */
 	DEF_RST(7, 3, 3, 4),		/* GTM_6_PRESETZ */
 	DEF_RST(7, 4, 3, 5),		/* GTM_7_PRESETZ */
-	DEF_RST(7, 5, 3, 6),		/* WDT_0_RESET */
 	DEF_RST(7, 6, 3, 7),		/* WDT_1_RESET */
-	DEF_RST(7, 7, 3, 8),		/* WDT_2_RESET */
-	DEF_RST(7, 8, 3, 9),		/* WDT_3_RESET */
 	DEF_RST(8, 1, 3, 18),		/* RSCI0_PRESETN */
 	DEF_RST(8, 2, 3, 19),		/* RSCI0_TRESETN */
 	DEF_RST(8, 3, 3, 20),		/* RSCI1_PRESETN */
diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig
index 5cf1e0fd6fb32..7e14335020618 100644
--- a/drivers/clk/rockchip/Kconfig
+++ b/drivers/clk/rockchip/Kconfig
@@ -16,6 +16,13 @@ config CLK_PX30
 	help
 	  Build the driver for PX30 Clock Driver.
 
+config CLK_RV1103B
+	bool "Rockchip RV1103B clock controller support"
+	depends on ARM || COMPILE_TEST
+	default y
+	help
+	  Build the driver for RV1103B Clock Driver.
+
 config CLK_RV110X
 	bool "Rockchip RV110x clock controller support"
 	depends on ARM || COMPILE_TEST
diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile
index 4d8cbb2044c7c..7c984ee006c6d 100644
--- a/drivers/clk/rockchip/Makefile
+++ b/drivers/clk/rockchip/Makefile
@@ -18,6 +18,7 @@ clk-rockchip-y += gate-link.o
 clk-rockchip-$(CONFIG_RESET_CONTROLLER) += softrst.o
 
 obj-$(CONFIG_CLK_PX30)          += clk-px30.o
+obj-$(CONFIG_CLK_RV1103B)	+= clk-rv1103b.o
 obj-$(CONFIG_CLK_RV110X)        += clk-rv1108.o
 obj-$(CONFIG_CLK_RV1126)        += clk-rv1126.o
 obj-$(CONFIG_CLK_RV1126B)	+= clk-rv1126b.o rst-rv1126b.o
diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c
index 74eabf9b2ae2e..d571c4b0c35f7 100644
--- a/drivers/clk/rockchip/clk-rk3568.c
+++ b/drivers/clk/rockchip/clk-rk3568.c
@@ -827,6 +827,8 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = {
 			RK3568_CLKGATE_CON(12), 3, GFLAGS),
 	GATE(CLK_PCIE20_AUX_NDFT, "clk_pcie20_aux_ndft", "xin24m", 0,
 			RK3568_CLKGATE_CON(12), 4, GFLAGS),
+	GATE(CLK_PCIE20_PIPE_DFT, "clk_pcie20_pipe_dft", "aclk_pipe", CLK_IGNORE_UNUSED,
+			RK3568_CLKGATE_CON(12), 5, GFLAGS),
 	GATE(ACLK_PCIE30X1_MST, "aclk_pcie30x1_mst", "aclk_pipe", 0,
 			RK3568_CLKGATE_CON(12), 8, GFLAGS),
 	GATE(ACLK_PCIE30X1_SLV, "aclk_pcie30x1_slv", "aclk_pipe", 0,
@@ -837,6 +839,8 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = {
 			RK3568_CLKGATE_CON(12), 11, GFLAGS),
 	GATE(CLK_PCIE30X1_AUX_NDFT, "clk_pcie30x1_aux_ndft", "xin24m", 0,
 			RK3568_CLKGATE_CON(12), 12, GFLAGS),
+	GATE(CLK_PCIE30X1_PIPE_DFT, "clk_pcie30x1_pipe_dft", "aclk_pipe", CLK_IGNORE_UNUSED,
+			RK3568_CLKGATE_CON(12), 13, GFLAGS),
 	GATE(ACLK_PCIE30X2_MST, "aclk_pcie30x2_mst", "aclk_pipe", 0,
 			RK3568_CLKGATE_CON(13), 0, GFLAGS),
 	GATE(ACLK_PCIE30X2_SLV, "aclk_pcie30x2_slv", "aclk_pipe", 0,
@@ -847,6 +851,8 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = {
 			RK3568_CLKGATE_CON(13), 3, GFLAGS),
 	GATE(CLK_PCIE30X2_AUX_NDFT, "clk_pcie30x2_aux_ndft", "xin24m", 0,
 			RK3568_CLKGATE_CON(13), 4, GFLAGS),
+	GATE(CLK_PCIE30X2_PIPE_DFT, "clk_pcie30x2_pipe_dft", "aclk_pipe", CLK_IGNORE_UNUSED,
+			RK3568_CLKGATE_CON(13), 5, GFLAGS),
 	GATE(ACLK_SATA0, "aclk_sata0", "aclk_pipe", 0,
 			RK3568_CLKGATE_CON(11), 0, GFLAGS),
 	GATE(CLK_SATA0_PMALIVE, "clk_sata0_pmalive", "gpll_20m", 0,
diff --git a/drivers/clk/rockchip/clk-rv1103b.c b/drivers/clk/rockchip/clk-rv1103b.c
new file mode 100644
index 0000000000000..7da1fda5e1b95
--- /dev/null
+++ b/drivers/clk/rockchip/clk-rv1103b.c
@@ -0,0 +1,658 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2024 Rockchip Electronics Co. Ltd.
+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <dt-bindings/clock/rockchip,rv1103b-cru.h>
+#include "clk.h"
+
+#define RV1103B_GRF_SOC_STATUS0		0x10
+#define RV1103B_FRAC_MAX_PRATE		1200000000
+#define PVTPLL_SRC_SEL_PVTPLL		(BIT(0) | BIT(16))
+
+enum rv1103b_plls {
+	dpll,
+	gpll,
+};
+
+static struct rockchip_pll_rate_table rv1103b_pll_rates[] = {
+	/* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */
+	RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0),
+	{ /* sentinel */ },
+};
+
+#define RV1103B_DIV_ACLK_CORE_MASK	0x1f
+#define RV1103B_DIV_ACLK_CORE_SHIFT	0
+#define RV1103B_DIV_PCLK_DBG_MASK	0x1f
+#define RV1103B_DIV_PCLK_DBG_SHIFT	8
+
+#define RV1103B_CLKSEL0(_aclk_core)						\
+{										\
+	.reg = RV1103B_CORECLKSEL_CON(2),					\
+	.val = HIWORD_UPDATE(_aclk_core - 1, RV1103B_DIV_ACLK_CORE_MASK,	\
+			     RV1103B_DIV_ACLK_CORE_SHIFT),			\
+}
+
+#define RV1103B_CLKSEL1(_pclk_dbg)						\
+{										\
+	.reg = RV1103B_CORECLKSEL_CON(2),					\
+	.val = HIWORD_UPDATE(_pclk_dbg - 1, RV1103B_DIV_PCLK_DBG_MASK,		\
+			     RV1103B_DIV_PCLK_DBG_SHIFT),			\
+}
+
+#define RV1103B_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg)			\
+{										\
+	.prate = _prate,							\
+	.divs = {								\
+		RV1103B_CLKSEL0(_aclk_core),					\
+		RV1103B_CLKSEL1(_pclk_dbg),					\
+	},									\
+}
+
+static struct rockchip_cpuclk_rate_table rv1103b_cpuclk_rates[] __initdata = {
+	RV1103B_CPUCLK_RATE(1608000000, 4, 10),
+	RV1103B_CPUCLK_RATE(1512000000, 4, 10),
+	RV1103B_CPUCLK_RATE(1416000000, 4, 10),
+	RV1103B_CPUCLK_RATE(1296000000, 3, 10),
+	RV1103B_CPUCLK_RATE(1200000000, 3, 10),
+	RV1103B_CPUCLK_RATE(1188000000, 3, 8),
+	RV1103B_CPUCLK_RATE(1104000000, 2, 8),
+	RV1103B_CPUCLK_RATE(1008000000, 2, 8),
+	RV1103B_CPUCLK_RATE(816000000, 2, 6),
+	RV1103B_CPUCLK_RATE(600000000, 2, 4),
+	RV1103B_CPUCLK_RATE(594000000, 2, 4),
+	RV1103B_CPUCLK_RATE(408000000, 1, 3),
+	RV1103B_CPUCLK_RATE(396000000, 1, 3),
+};
+
+PNAME(mux_pll_p)		= { "xin24m" };
+PNAME(mux_200m_100m_p)		= { "clk_gpll_div6", "clk_gpll_div12" };
+PNAME(mux_gpll_24m_p)		= { "gpll", "xin24m" };
+PNAME(mux_480m_400m_300m_200m_p) = { "clk_gpll_div2p5", "clk_gpll_div3", "clk_gpll_div4", "clk_gpll_div6" };
+PNAME(mux_480m_400m_300m_p)	= { "clk_gpll_div2p5", "clk_gpll_div3", "clk_gpll_div4" };
+PNAME(mux_300m_200m_p)		= { "clk_gpll_div4", "clk_gpll_div6" };
+PNAME(mux_600m_480m_400m_p)	= { "clk_gpll_div2", "clk_gpll_div2p5", "clk_gpll_div3" };
+PNAME(mux_400m_300m_p)		= { "clk_gpll_div3", "clk_gpll_div4" };
+PNAME(mux_100m_24m_p)		= { "clk_gpll_div12", "xin24m" };
+PNAME(mux_200m_24m_p)		= { "clk_gpll_div6", "xin24m" };
+PNAME(mux_200m_100m_50m_24m_p)	= { "clk_gpll_div6", "clk_gpll_div12", "clk_gpll_div24", "xin24m" };
+PNAME(mux_300m_200m_100m_p)	= { "clk_gpll_div4", "clk_gpll_div6", "clk_gpll_div12" };
+PNAME(sclk_uart0_src_p)		= { "clk_uart0_src", "clk_uart0_frac", "xin24m" };
+PNAME(sclk_uart1_src_p)		= { "clk_uart1_src", "clk_uart1_frac", "xin24m" };
+PNAME(sclk_uart2_src_p)		= { "clk_uart2_src", "clk_uart2_frac", "xin24m" };
+PNAME(mclk_sai_src_p)		= { "clk_sai_src", "clk_sai_frac", "mclk_sai_from_io", "xin_osc0_half" };
+PNAME(clk_freq_pwm0_src_p)	= { "sclk_sai_from_io", "mclk_sai_from_io", "clk_testout_out" };
+PNAME(clk_counter_pwm0_src_p)	= { "sclk_sai_from_io", "mclk_sai_from_io", "clk_testout_out" };
+PNAME(clk_mipi0_out2io_p)	= { "clk_ref_mipi0", "xin24m" };
+PNAME(clk_mipi1_out2io_p)	= { "clk_ref_mipi1", "xin24m" };
+PNAME(mclk_sai_out2io_p)	= { "mclk_sai_src", "xin_osc0_half" };
+PNAME(aclk_npu_root_p)		= { "clk_npu_src", "clk_npu_pvtpll" };
+PNAME(clk_core_vepu_p)		= { "clk_vepu_src", "clk_vepu_pvtpll" };
+PNAME(lsclk_vi_root_p)		= { "clk_gpll_div6", "lsclk_vi_100m" };
+PNAME(clk_core_isp_p)		= { "clk_isp_src", "clk_isp_pvtpll_src" };
+PNAME(lsclk_pmu_root_p)		= { "xin24m", "clk_rc_osc_io" };
+PNAME(xin_rc_div_p)		= { "xin24m", "clk_rc_osc_io" };
+PNAME(clk_32k_p)		= { "xin_rc_div", "clk_32k_rtc", "clk_32k_io" };
+PNAME(dbclk_pmu_gpio0_p)	= { "xin24m", "clk_32k" };
+PNAME(sclk_sfc_2x_pmu1_p)	= { "clk_gpll_div12", "clk_rc_osc_io" };
+PNAME(mux_armclk_p)		= { "armclk_gpll", "clk_core_pvtpll" };
+
+static struct rockchip_pll_clock rv1103b_pll_clks[] __initdata = {
+	[dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p,
+		     CLK_IS_CRITICAL, RV1103B_PLL_CON(16),
+		     RV1103B_MODE_CON, 0, 10, 0, rv1103b_pll_rates),
+	[gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p,
+		     CLK_IS_CRITICAL, RV1103B_PLL_CON(24),
+		     RV1103B_MODE_CON, 0, 10, 0, rv1103b_pll_rates),
+};
+
+#define MFLAGS CLK_MUX_HIWORD_MASK
+#define DFLAGS CLK_DIVIDER_HIWORD_MASK
+#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE)
+
+static struct rockchip_clk_branch rv1103b_clk_uart0_fracmux __initdata =
+	MUX(SCLK_UART0_SRC, "sclk_uart0_src", sclk_uart0_src_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(32), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rv1103b_clk_uart1_fracmux __initdata =
+	MUX(SCLK_UART1_SRC, "sclk_uart1_src", sclk_uart1_src_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(32), 10, 2, MFLAGS);
+
+static struct rockchip_clk_branch rv1103b_clk_uart2_fracmux __initdata =
+	MUX(SCLK_UART2_SRC, "sclk_uart2_src", sclk_uart2_src_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(32), 12, 2, MFLAGS);
+
+static struct rockchip_clk_branch rv1103b_rcdiv_pmu_fracmux __initdata =
+	MUX(CLK_32K, "clk_32k", clk_32k_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT,
+			RK3568_PMU_CLKSEL_CON(0), 0, 2, MFLAGS);
+
+static struct rockchip_clk_branch rv1103b_clk_branches[] __initdata = {
+
+	/*       Clock Definition       */
+	FACTOR(XIN_OSC0_HALF, "xin_osc0_half", "xin24m", 0, 1, 2),
+
+	COMPOSITE_NOGATE(0, "armclk_gpll", mux_gpll_24m_p, CLK_IS_CRITICAL,
+			RV1103B_CLKSEL_CON(37), 12, 1, MFLAGS, 13, 3, DFLAGS),
+
+	/* pd_top */
+	COMPOSITE_NOMUX(CLK_GPLL_DIV24, "clk_gpll_div24", "gpll", 0,
+			RV1103B_CLKSEL_CON(0), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 0, GFLAGS),
+	COMPOSITE_NOMUX(CLK_GPLL_DIV12, "clk_gpll_div12", "gpll", 0,
+			RV1103B_CLKSEL_CON(0), 5, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 1, GFLAGS),
+	COMPOSITE_NOMUX(CLK_GPLL_DIV6, "clk_gpll_div6", "gpll", 0,
+			RV1103B_CLKSEL_CON(1), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 3, GFLAGS),
+	COMPOSITE_NOMUX(CLK_GPLL_DIV4, "clk_gpll_div4", "gpll", 0,
+			RV1103B_CLKSEL_CON(1), 10, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 5, GFLAGS),
+	COMPOSITE_NOMUX(CLK_GPLL_DIV3, "clk_gpll_div3", "gpll", 0,
+			RV1103B_CLKSEL_CON(2), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 7, GFLAGS),
+	COMPOSITE_NOMUX_HALFDIV(CLK_GPLL_DIV2P5, "clk_gpll_div2p5", "gpll", 0,
+			RV1103B_CLKSEL_CON(2), 5, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 8, GFLAGS),
+	COMPOSITE_NOMUX(CLK_GPLL_DIV2, "clk_gpll_div2", "gpll", 0,
+			RV1103B_CLKSEL_CON(2), 10, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(0), 9, GFLAGS),
+	COMPOSITE_NOMUX(CLK_UART0_SRC, "clk_uart0_src", "gpll", 0,
+			RV1103B_CLKSEL_CON(5), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(1), 0, GFLAGS),
+	COMPOSITE_NOMUX(CLK_UART1_SRC, "clk_uart1_src", "gpll", 0,
+			RV1103B_CLKSEL_CON(5), 5, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(1), 1, GFLAGS),
+	COMPOSITE_NOMUX(CLK_UART2_SRC, "clk_uart2_src", "gpll", 0,
+			RV1103B_CLKSEL_CON(5), 10, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(1), 2, GFLAGS),
+	COMPOSITE_FRACMUX(CLK_UART0_FRAC, "clk_uart0_frac", "clk_uart0_src", 0,
+			RV1103B_CLKSEL_CON(10), 0,
+			RV1103B_CLKGATE_CON(1), 6, GFLAGS,
+			&rv1103b_clk_uart0_fracmux),
+	COMPOSITE_FRACMUX(CLK_UART1_FRAC, "clk_uart1_frac", "clk_uart1_src", 0,
+			RV1103B_CLKSEL_CON(11), 0,
+			RV1103B_CLKGATE_CON(1), 7, GFLAGS,
+			&rv1103b_clk_uart1_fracmux),
+	COMPOSITE_FRACMUX(CLK_UART2_FRAC, "clk_uart2_frac", "clk_uart2_src", 0,
+			RV1103B_CLKSEL_CON(12), 0,
+			RV1103B_CLKGATE_CON(1), 8, GFLAGS,
+			&rv1103b_clk_uart2_fracmux),
+	GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_src", 0,
+			RV1103B_CLKGATE_CON(3), 3, GFLAGS),
+	GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_src", 0,
+			RV1103B_CLKGATE_CON(3), 4, GFLAGS),
+	GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_src", 0,
+			RV1103B_CLKGATE_CON(3), 8, GFLAGS),
+
+	COMPOSITE_NOMUX(CLK_SAI_SRC, "clk_sai_src", "gpll", 0,
+			RV1103B_CLKSEL_CON(20), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(1), 12, GFLAGS),
+	MUX(MCLK_SAI_SRC, "mclk_sai_src", mclk_sai_src_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(35), 10, 2, MFLAGS),
+	GATE(MCLK_SAI, "mclk_sai", "mclk_sai_src", 0,
+			RV1103B_CLKGATE_CON(5), 5, GFLAGS),
+
+	COMPOSITE_NODIV(LSCLK_NPU_SRC, "lsclk_npu_src", mux_200m_100m_p, CLK_IS_CRITICAL,
+			RV1103B_CLKSEL_CON(30), 0, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(2), 0, GFLAGS),
+	COMPOSITE(CLK_NPU_SRC, "clk_npu_src", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(37), 0, 1, MFLAGS, 1, 2, DFLAGS,
+			RV1103B_CLKGATE_CON(5), 12, GFLAGS),
+	COMPOSITE_NODIV(ACLK_VEPU_SRC, "aclk_vepu_src", mux_480m_400m_300m_200m_p, 0,
+			RV1103B_CLKSEL_CON(30), 8, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(2), 4, GFLAGS),
+	COMPOSITE(CLK_VEPU_SRC, "clk_vepu_src", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(37), 4, 1, MFLAGS, 5, 2, DFLAGS,
+			RV1103B_CLKGATE_CON(5), 13, GFLAGS),
+	COMPOSITE_NODIV(ACLK_VI_SRC, "aclk_vi_src", mux_480m_400m_300m_p, CLK_IS_CRITICAL,
+			RV1103B_CLKSEL_CON(30), 12, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(2), 8, GFLAGS),
+	COMPOSITE(CLK_ISP_SRC, "clk_isp_src", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(37), 8, 1, MFLAGS, 9, 2, DFLAGS,
+			RV1103B_CLKGATE_CON(5), 14, GFLAGS),
+	COMPOSITE_NODIV(DCLK_VICAP, "dclk_vicap", mux_300m_200m_p, 0,
+			RV1103B_CLKSEL_CON(30), 14, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(2), 9, GFLAGS),
+	COMPOSITE(CCLK_EMMC, "cclk_emmc", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(31), 15, 1, MFLAGS, 0, 8, DFLAGS,
+			RV1103B_CLKGATE_CON(2), 10, GFLAGS),
+	COMPOSITE(CCLK_SDMMC0, "cclk_sdmmc0", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(32), 15, 1, MFLAGS, 0, 8, DFLAGS,
+			RV1103B_CLKGATE_CON(2), 11, GFLAGS),
+	COMPOSITE(SCLK_SFC_2X, "sclk_sfc_2x", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(33), 15, 1, MFLAGS, 0, 8, DFLAGS,
+			RV1103B_CLKGATE_CON(2), 12, GFLAGS),
+	COMPOSITE_NODIV(LSCLK_PERI_SRC, "lsclk_peri_src", mux_300m_200m_p, CLK_IS_CRITICAL,
+			RV1103B_CLKSEL_CON(31), 9, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(3), 0, GFLAGS),
+	COMPOSITE_NODIV(ACLK_PERI_SRC, "aclk_peri_src", mux_600m_480m_400m_p, CLK_IS_CRITICAL,
+			RV1103B_CLKSEL_CON(31), 10, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(3), 1, GFLAGS),
+	COMPOSITE_NODIV(HCLK_HPMCU, "hclk_hpmcu", mux_400m_300m_p, 0,
+			RV1103B_CLKSEL_CON(31), 12, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(3), 2, GFLAGS),
+		COMPOSITE_NODIV(CLK_I2C_PMU, "clk_i2c_pmu", mux_100m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 0, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 0, GFLAGS),
+	COMPOSITE_NODIV(CLK_I2C_PERI, "clk_i2c_peri", mux_200m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 1, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 4, GFLAGS),
+	COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_100m_50m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 2, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 5, GFLAGS),
+	COMPOSITE_NODIV(CLK_PWM0_SRC, "clk_pwm0_src", mux_100m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 12, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 10, GFLAGS),
+	COMPOSITE_NODIV(CLK_PWM1, "clk_pwm1", mux_100m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 13, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 11, GFLAGS),
+	COMPOSITE_NODIV(CLK_PWM2, "clk_pwm2", mux_100m_24m_p, 0,
+			RV1103B_CLKSEL_CON(34), 14, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(4), 12, GFLAGS),
+	COMPOSITE_NODIV(DCLK_DECOM_SRC, "dclk_decom_src", mux_480m_400m_300m_p, 0,
+			RV1103B_CLKSEL_CON(35), 0, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 0, GFLAGS),
+	COMPOSITE(CCLK_SDMMC1, "cclk_sdmmc1", mux_gpll_24m_p, 0,
+			RV1103B_CLKSEL_CON(36), 15, 1, MFLAGS, 0, 8, DFLAGS,
+			RV1103B_CLKGATE_CON(5), 1, GFLAGS),
+	COMPOSITE_NODIV(CLK_CORE_CRYPTO, "clk_core_crypto", mux_300m_200m_100m_p, 0,
+			RV1103B_CLKSEL_CON(35), 2, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 2, GFLAGS),
+	COMPOSITE_NODIV(CLK_PKA_CRYPTO, "clk_pka_crypto", mux_300m_200m_100m_p, 0,
+			RV1103B_CLKSEL_CON(35), 4, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 3, GFLAGS),
+	COMPOSITE_NODIV(CLK_CORE_RGA, "clk_core_rga", mux_400m_300m_p, 0,
+			RV1103B_CLKSEL_CON(35), 8, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 4, GFLAGS),
+
+	GATE(PCLK_TOP_ROOT, "pclk_top_root", "clk_gpll_div12", CLK_IS_CRITICAL,
+			RV1103B_CLKGATE_CON(6), 0, GFLAGS),
+	COMPOSITE_NOMUX(CLK_REF_MIPI0, "clk_ref_mipi0", "clk_gpll_div2", 0,
+			RV1103B_CLKSEL_CON(40), 0, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(6), 3, GFLAGS),
+	COMPOSITE_NODIV(CLK_MIPI0_OUT2IO, "clk_mipi0_out2io", clk_mipi0_out2io_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(40), 6, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(6), 4, GFLAGS),
+	COMPOSITE_NOMUX(CLK_REF_MIPI1, "clk_ref_mipi1", "clk_gpll_div2", 0,
+			RV1103B_CLKSEL_CON(40), 8, 5, DFLAGS,
+			RV1103B_CLKGATE_CON(6), 5, GFLAGS),
+	COMPOSITE_NODIV(CLK_MIPI1_OUT2IO, "clk_mipi1_out2io", clk_mipi1_out2io_p, CLK_SET_RATE_PARENT,
+			RV1103B_CLKSEL_CON(40), 14, 1, MFLAGS,
+			RV1103B_CLKGATE_CON(6), 6, GFLAGS),
+	COMPOSITE(MCLK_SAI_OUT2IO, "mclk_sai_out2io", mclk_sai_out2io_p, 0,
+			RV1103B_CLKSEL_CON(41), 7, 1, MFLAGS, 13, 3, DFLAGS,
+			RV1103B_CLKGATE_CON(6), 9, GFLAGS),
+
+	/* pd_vpu */
+	COMPOSITE_NODIV(ACLK_NPU_ROOT, "aclk_npu_root", aclk_npu_root_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
+			RV1103B_NPUCLKSEL_CON(0), 1, 1, MFLAGS,
+			RV1103B_NPUCLKGATE_CON(0), 1, GFLAGS),
+	GATE(HCLK_RKNN, "hclk_rknn", "lsclk_npu_src", 0,
+			RV1103B_NPUCLKGATE_CON(0), 4, GFLAGS),
+	GATE(ACLK_RKNN, "aclk_rknn", "aclk_npu_root", 0,
+			RV1103B_NPUCLKGATE_CON(0), 5, GFLAGS),
+
+	/* pd_vepu */
+	COMPOSITE_NOMUX(LSCLK_VEPU_ROOT, "lsclk_vepu_root", "aclk_vepu_src", CLK_IS_CRITICAL,
+			RV1103B_VEPUCLKSEL_CON(0), 2, 2, DFLAGS,
+			RV1103B_VEPUCLKGATE_CON(0), 0, GFLAGS),
+	GATE(HCLK_VEPU, "hclk_vepu", "lsclk_vepu_root", 0,
+			RV1103B_VEPUCLKGATE_CON(0), 4, GFLAGS),
+	GATE(ACLK_VEPU, "aclk_vepu", "aclk_vepu_src", 0,
+			RV1103B_VEPUCLKGATE_CON(0), 5, GFLAGS),
+	COMPOSITE_NODIV(CLK_CORE_VEPU, "clk_core_vepu", clk_core_vepu_p, 0,
+			RV1103B_VEPUCLKSEL_CON(0), 1, 1, MFLAGS,
+			RV1103B_VEPUCLKGATE_CON(0), 6, GFLAGS),
+	GATE(PCLK_ACODEC, "pclk_acodec", "lsclk_vepu_root", 0,
+			RV1103B_VEPUCLKGATE_CON(0), 13, GFLAGS),
+	GATE(PCLK_USBPHY, "pclk_usbphy", "lsclk_vepu_root", 0,
+			RV1103B_VEPUCLKGATE_CON(0), 14, GFLAGS),
+
+	/* pd_vi */
+	FACTOR(LSCLK_VI_100M, "lsclk_vi_100m", "clk_gpll_div6", 0, 1, 2),
+	COMPOSITE_NODIV(LSCLK_VI_ROOT, "lsclk_vi_root", lsclk_vi_root_p, CLK_IS_CRITICAL,
+			RV1103B_VICLKSEL_CON(0), 3, 1, MFLAGS,
+			RV1103B_VICLKGATE_CON(0), 0, GFLAGS),
+	GATE(HCLK_ISP, "hclk_isp", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(0), 4, GFLAGS),
+	GATE(ACLK_ISP, "aclk_isp", "aclk_vi_src", 0,
+			RV1103B_VICLKGATE_CON(0), 5, GFLAGS),
+	COMPOSITE_NODIV(CLK_CORE_ISP, "clk_core_isp", clk_core_isp_p, 0,
+			RV1103B_VICLKSEL_CON(0), 1, 1, MFLAGS,
+			RV1103B_VICLKGATE_CON(0), 6, GFLAGS),
+	GATE(ACLK_VICAP, "aclk_vicap", "aclk_vi_src", 0,
+			RV1103B_VICLKGATE_CON(1), 2, GFLAGS),
+	GATE(HCLK_VICAP, "hclk_vicap", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 3, GFLAGS),
+	GATE(ISP0CLK_VICAP, "isp0clk_vicap", "clk_core_isp", 0,
+			RV1103B_VICLKGATE_CON(1), 8, GFLAGS),
+	GATE(PCLK_CSI2HOST0, "pclk_csi2host0", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 9, GFLAGS),
+	GATE(PCLK_CSI2HOST1, "pclk_csi2host1", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 11, GFLAGS),
+	GATE(HCLK_EMMC, "hclk_emmc", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 13, GFLAGS),
+	GATE(HCLK_SFC, "hclk_sfc", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 14, GFLAGS),
+	GATE(HCLK_SFC_XIP, "hclk_sfc_xip", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(1), 15, GFLAGS),
+	GATE(HCLK_SDMMC0, "hclk_sdmmc0", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(2), 0, GFLAGS),
+	GATE(PCLK_CSIPHY, "pclk_csiphy", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(2), 2, GFLAGS),
+	GATE(PCLK_GPIO1, "pclk_gpio1", "lsclk_vi_root", 0,
+			RV1103B_VICLKGATE_CON(2), 3, GFLAGS),
+	GATE(DBCLK_GPIO1, "dbclk_gpio1", "xin24m", 0,
+			RV1103B_VICLKGATE_CON(2), 4, GFLAGS),
+
+	/* pd_ddr */
+	GATE(LSCLK_DDR_ROOT, "lsclk_ddr_root", "clk_gpll_div12", CLK_IS_CRITICAL,
+			RV1103B_DDRCLKGATE_CON(0), 0, GFLAGS),
+	GATE(CLK_TIMER_DDRMON, "clk_timer_ddrmon", "xin24m", 0,
+			RV1103B_DDRCLKGATE_CON(0), 4, GFLAGS),
+	FACTOR(0, "sclk_ddr", "dpll", 0, 1, 2),
+
+	/* pd_pmu */
+	COMPOSITE(LSCLK_PMU_ROOT, "lsclk_pmu_root", lsclk_pmu_root_p, CLK_IS_CRITICAL,
+			RV1103B_PMUCLKSEL_CON(2), 4, 1, MFLAGS, 0, 2, DFLAGS,
+			RV1103B_PMUCLKGATE_CON(0), 0, GFLAGS),
+	GATE(PCLK_PMU, "pclk_pmu", "lsclk_pmu_root", CLK_IS_CRITICAL,
+			RV1103B_PMUCLKGATE_CON(0), 2, GFLAGS),
+	MUX(XIN_RC_SRC, "xin_rc_src", xin_rc_div_p, 0,
+			RV1103B_PMUCLKSEL_CON(0), 2, 1, MFLAGS),
+	COMPOSITE_FRACMUX(XIN_RC_DIV, "xin_rc_div", "xin_rc_src", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL,
+			RV1103B_PMUCLKSEL_CON(1), 0,
+			RV1103B_PMUCLKGATE_CON(0), 3, GFLAGS,
+			&rv1103b_rcdiv_pmu_fracmux),
+	GATE(PCLK_PMU_GPIO0, "pclk_pmu_gpio0", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(0), 4, GFLAGS),
+	COMPOSITE_NODIV(DBCLK_PMU_GPIO0, "dbclk_pmu_gpio0", dbclk_pmu_gpio0_p, 0,
+			RK3568_PMU_CLKSEL_CON(0), 3, 1, MFLAGS,
+			RV1103B_PMUCLKGATE_CON(0), 5, GFLAGS),
+	GATE(PCLK_PWM0, "pclk_pwm0", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(2), 0, GFLAGS),
+	GATE(CLK_PWM0, "clk_pwm0", "clk_pwm0_src", 0,
+			RV1103B_PMUCLKGATE_CON(2), 1, GFLAGS),
+	GATE(CLK_OSC_PWM0, "clk_osc_pwm0", "xin24m", 0,
+			RV1103B_PMUCLKGATE_CON(2), 2, GFLAGS),
+	GATE(CLK_RC_PWM0, "clk_rc_pwm0", "clk_32k", 0,
+			RV1103B_PMUCLKGATE_CON(2), 3, GFLAGS),
+	GATE(PCLK_I2C0, "pclk_i2c0", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(0), 12, GFLAGS),
+	GATE(CLK_I2C0, "clk_i2c0", "clk_i2c_pmu", 0,
+			RV1103B_PMUCLKGATE_CON(0), 13, GFLAGS),
+	GATE(PCLK_UART0, "pclk_uart0", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(0), 14, GFLAGS),
+	GATE(CLK_REFOUT, "clk_refout", "xin24m", 0,
+			RV1103B_PMUCLKGATE_CON(1), 4, GFLAGS),
+	GATE(CLK_PREROLL, "clk_preroll", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(1), 6, GFLAGS),
+	GATE(CLK_PREROLL_32K, "clk_preroll_32k", "clk_32k", 0,
+			RV1103B_PMUCLKGATE_CON(1), 7, GFLAGS),
+	GATE(CLK_LPMCU_PMU, "clk_lpmcu_pmu", "lsclk_pmu_root", 0,
+			RV1103B_PMUCLKGATE_CON(2), 12, GFLAGS),
+
+	/* pd_pmu1 */
+	GATE(PCLK_SPI2AHB, "pclk_spi2ahb", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 0, GFLAGS),
+	GATE(HCLK_SPI2AHB, "hclk_spi2ahb", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 1, GFLAGS),
+	GATE(PCLK_WDT_LPMCU, "pclk_wdt_lpmcu", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 9, GFLAGS),
+	GATE(TCLK_WDT_LPMCU, "tclk_wdt_lpmcu", "xin24m", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 10, GFLAGS),
+	GATE(HCLK_SFC_PMU1, "hclk_sfc_pmu1", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 12, GFLAGS),
+	GATE(HCLK_SFC_XIP_PMU1, "hclk_sfc_xip_pmu1", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 13, GFLAGS),
+	COMPOSITE_NODIV(SCLK_SFC_2X_PMU1, "sclk_sfc_2x_pmu1", sclk_sfc_2x_pmu1_p, 0,
+			RV1103B_PMU1CLKSEL_CON(0), 8, 1, MFLAGS,
+			RV1103B_PMU1CLKGATE_CON(0), 14, GFLAGS),
+	GATE(CLK_LPMCU, "clk_lpmcu", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(1), 0, GFLAGS),
+	GATE(CLK_LPMCU_RTC, "clk_lpmcu_rtc", "xin24m", 0,
+			RV1103B_PMU1CLKGATE_CON(1), 4, GFLAGS),
+	GATE(PCLK_LPMCU_MAILBOX, "pclk_lpmcu_mailbox", "lsclk_pmu_root", 0,
+			RV1103B_PMU1CLKGATE_CON(1), 8, GFLAGS),
+
+	/* pd_peri */
+	COMPOSITE_NOMUX(PCLK_PERI_ROOT, "pclk_peri_root", "lsclk_peri_src", CLK_IS_CRITICAL,
+			RV1103B_PERICLKSEL_CON(0), 0, 2, DFLAGS,
+			RV1103B_PERICLKGATE_CON(0), 0, GFLAGS),
+	COMPOSITE_NOMUX(PCLK_RTC_ROOT, "pclk_rtc_root", "lsclk_peri_src", CLK_IS_CRITICAL,
+			RV1103B_PERICLKSEL_CON(2), 12, 4, DFLAGS,
+			RV1103B_PERICLKGATE_CON(0), 8, GFLAGS),
+	GATE(CLK_TIMER_ROOT, "clk_timer_root", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(0), 1, GFLAGS),
+	GATE(PCLK_TIMER, "pclk_timer", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 0, GFLAGS),
+	GATE(CLK_TIMER0, "clk_timer0", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 1, GFLAGS),
+	GATE(CLK_TIMER1, "clk_timer1", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 2, GFLAGS),
+	GATE(CLK_TIMER2, "clk_timer2", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 3, GFLAGS),
+	GATE(CLK_TIMER3, "clk_timer3", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 4, GFLAGS),
+	GATE(CLK_TIMER4, "clk_timer4", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 5, GFLAGS),
+	GATE(CLK_TIMER5, "clk_timer5", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 6, GFLAGS),
+	GATE(PCLK_STIMER, "pclk_stimer", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 7, GFLAGS),
+	GATE(CLK_STIMER0, "clk_stimer0", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 8, GFLAGS),
+	GATE(CLK_STIMER1, "clk_stimer1", "clk_timer_root", 0,
+			RV1103B_PERICLKGATE_CON(1), 9, GFLAGS),
+	GATE(PCLK_WDT_NS, "pclk_wdt_ns", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 0, GFLAGS),
+	GATE(TCLK_WDT_NS, "tclk_wdt_ns", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(2), 1, GFLAGS),
+	GATE(PCLK_WDT_S, "pclk_wdt_s", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 2, GFLAGS),
+	GATE(TCLK_WDT_S, "tclk_wdt_s", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(2), 3, GFLAGS),
+	GATE(PCLK_WDT_HPMCU, "pclk_wdt_hpmcu", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 4, GFLAGS),
+	GATE(TCLK_WDT_HPMCU, "tclk_wdt_hpmcu", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(2), 5, GFLAGS),
+	GATE(PCLK_I2C1, "pclk_i2c1", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 6, GFLAGS),
+	GATE(CLK_I2C1, "clk_i2c1", "clk_i2c_peri", 0,
+			RV1103B_PERICLKGATE_CON(2), 7, GFLAGS),
+	GATE(PCLK_I2C2, "pclk_i2c2", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 8, GFLAGS),
+	GATE(CLK_I2C2, "clk_i2c2", "clk_i2c_peri", 0,
+			RV1103B_PERICLKGATE_CON(2), 9, GFLAGS),
+	GATE(PCLK_I2C3, "pclk_i2c3", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 10, GFLAGS),
+	GATE(CLK_I2C3, "clk_i2c3", "clk_i2c_peri", 0,
+			RV1103B_PERICLKGATE_CON(2), 11, GFLAGS),
+	GATE(PCLK_I2C4, "pclk_i2c4", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(2), 12, GFLAGS),
+	GATE(CLK_I2C4, "clk_i2c4", "clk_i2c_peri", 0,
+			RV1103B_PERICLKGATE_CON(2), 13, GFLAGS),
+	GATE(PCLK_SPI0, "pclk_spi0", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(3), 10, GFLAGS),
+	GATE(PCLK_PWM1, "pclk_pwm1", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(4), 6, GFLAGS),
+	GATE(CLK_OSC_PWM1, "clk_osc_pwm1", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(4), 8, GFLAGS),
+	GATE(PCLK_PWM2, "pclk_pwm2", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(4), 12, GFLAGS),
+	GATE(CLK_OSC_PWM2, "clk_osc_pwm2", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(4), 13, GFLAGS),
+	GATE(PCLK_UART2, "pclk_uart2", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(3), 0, GFLAGS),
+	GATE(PCLK_UART1, "pclk_uart1", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(3), 2, GFLAGS),
+	GATE(ACLK_RKDMA, "aclk_rkdma", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(5), 8, GFLAGS),
+	GATE(PCLK_TSADC, "pclk_tsadc", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(5), 9, GFLAGS),
+	COMPOSITE_NOMUX(CLK_TSADC, "clk_tsadc", "xin24m", 0,
+			RV1103B_PERICLKSEL_CON(0), 4, 5, DFLAGS,
+			RV1103B_PERICLKGATE_CON(5), 10, GFLAGS),
+	COMPOSITE_NOMUX(CLK_TSADC_TSEN, "clk_tsadc_tsen", "xin24m", 0,
+			RV1103B_PERICLKSEL_CON(0), 10, 5, DFLAGS,
+			RV1103B_PERICLKGATE_CON(5), 11, GFLAGS),
+	GATE(PCLK_SARADC, "pclk_saradc", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(5), 12, GFLAGS),
+	COMPOSITE_NOMUX(CLK_SARADC, "clk_saradc", "xin24m", 0,
+			RV1103B_PERICLKSEL_CON(1), 0, 3, DFLAGS,
+			RV1103B_PERICLKGATE_CON(5), 13, GFLAGS),
+	GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(6), 3, GFLAGS),
+	GATE(DBCLK_GPIO2, "dbclk_gpio2", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(6), 4, GFLAGS),
+	GATE(ACLK_USBOTG, "aclk_usbotg", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(6), 9, GFLAGS),
+	GATE(CLK_REF_USBOTG, "clk_ref_usbotg", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(6), 10, GFLAGS),
+	GATE(HCLK_SDMMC1, "hclk_sdmmc1", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(7), 0, GFLAGS),
+	GATE(HCLK_SAI, "hclk_sai", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(7), 1, GFLAGS),
+	GATE(ACLK_CRYPTO, "aclk_crypto", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(8), 2, GFLAGS),
+	GATE(HCLK_CRYPTO, "hclk_crypto", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(8), 3, GFLAGS),
+	GATE(HCLK_RK_RNG_S, "hclk_rk_rng_s", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(8), 5, GFLAGS),
+	GATE(HCLK_RK_RNG_NS, "hclk_rk_rng_ns", "hclk_rk_rng_s", 0,
+			RV1103B_PERICLKGATE_CON(8), 4, GFLAGS),
+	GATE(PCLK_OTPC_NS, "pclk_otpc_ns", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(8), 6, GFLAGS),
+	GATE(CLK_OTPC_ROOT_NS, "clk_otpc_root_ns", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(8), 7, GFLAGS),
+	GATE(CLK_SBPI_OTPC_NS, "clk_sbpi_otpc_ns", "clk_otpc_root_ns", 0,
+			RV1103B_PERICLKGATE_CON(8), 8, GFLAGS),
+	COMPOSITE_NOMUX(CLK_USER_OTPC_NS, "clk_user_otpc_ns", "clk_otpc_root_ns", 0,
+			RV1103B_PERICLKSEL_CON(1), 4, 3, DFLAGS,
+			RV1103B_PERICLKGATE_CON(8), 9, GFLAGS),
+	GATE(PCLK_OTPC_S, "pclk_otpc_s", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(8), 10, GFLAGS),
+	GATE(CLK_OTPC_ROOT_S, "clk_otpc_root_s", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(8), 11, GFLAGS),
+	GATE(CLK_SBPI_OTPC_S, "clk_sbpi_otpc_s", "clk_otpc_root_s", 0,
+			RV1103B_PERICLKGATE_CON(8), 12, GFLAGS),
+	COMPOSITE_NOMUX(CLK_USER_OTPC_S, "clk_user_otpc_s", "clk_otpc_root_s", 0,
+			RV1103B_PERICLKSEL_CON(1), 8, 3, DFLAGS,
+			RV1103B_PERICLKGATE_CON(8), 13, GFLAGS),
+	GATE(PCLK_OTP_MASK, "pclk_otp_mask", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(8), 15, GFLAGS),
+	GATE(HCLK_RGA, "hclk_rga", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(9), 0, GFLAGS),
+	GATE(ACLK_RGA, "aclk_rga", "aclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(9), 1, GFLAGS),
+	GATE(ACLK_MAC, "aclk_mac", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(9), 3, GFLAGS),
+	GATE(PCLK_MAC, "pclk_mac", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(9), 4, GFLAGS),
+	GATE(CLK_MACPHY, "clk_macphy", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(9), 11, GFLAGS),
+	GATE(ACLK_SPINLOCK, "aclk_spinlock", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(10), 0, GFLAGS),
+	GATE(HCLK_CACHE, "hclk_cache", "hclk_hpmcu", 0,
+			RV1103B_PERICLKGATE_CON(10), 1, GFLAGS),
+	GATE(PCLK_HPMCU_MAILBOX, "pclk_hpmcu_mailbox", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(10), 2, GFLAGS),
+	GATE(PCLK_HPMCU_INTMUX, "pclk_hpmcu_intmux", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(10), 3, GFLAGS),
+	GATE(CLK_HPMCU, "clk_hpmcu", "hclk_hpmcu", 0,
+			RV1103B_PERICLKGATE_CON(10), 4, GFLAGS),
+	GATE(CLK_HPMCU_RTC, "clk_hpmcu_rtc", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(10), 8, GFLAGS),
+	GATE(DCLK_DECOM, "dclk_decom", "dclk_decom_src", 0,
+			RV1103B_PERICLKGATE_CON(11), 0, GFLAGS),
+	GATE(ACLK_DECOM, "aclk_decom", "aclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(11), 1, GFLAGS),
+	GATE(PCLK_DECOM, "pclk_decom", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(11), 2, GFLAGS),
+	GATE(ACLK_SYS_SRAM, "aclk_sys_sram", "lsclk_peri_src", CLK_IS_CRITICAL,
+			RV1103B_PERICLKGATE_CON(11), 3, GFLAGS),
+	GATE(PCLK_DMA2DDR, "pclk_dma2ddr", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(11), 4, GFLAGS),
+	GATE(ACLK_DMA2DDR, "aclk_dma2ddr", "aclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(11), 5, GFLAGS),
+	GATE(PCLK_DCF, "pclk_dcf", "pclk_peri_root", 0,
+			RV1103B_PERICLKGATE_CON(11), 6, GFLAGS),
+	GATE(ACLK_DCF, "aclk_dcf", "lsclk_peri_src", 0,
+			RV1103B_PERICLKGATE_CON(11), 7, GFLAGS),
+	COMPOSITE_NOMUX(MCLK_ACODEC_TX, "mclk_acodec_tx", "mclk_sai_src", 0,
+			RV1103B_PERICLKSEL_CON(2), 0, 3, DFLAGS,
+			RV1103B_PERICLKGATE_CON(11), 9, GFLAGS),
+	GATE(CLK_REF_USBPHY, "clk_ref_usbphy", "xin24m", 0,
+			RV1103B_PERICLKGATE_CON(11), 12, GFLAGS),
+
+	/* io */
+	COMPOSITE_NODIV(CLK_FREQ_PWM0_SRC, "clk_freq_pwm0_src", clk_freq_pwm0_src_p, 0,
+			RV1103B_CLKSEL_CON(35), 12, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 6, GFLAGS),
+	GATE(CLK_FREQ_PWM0, "clk_freq_pwm0", "clk_freq_pwm0_src", 0,
+			RV1103B_PMUCLKGATE_CON(2), 4, GFLAGS),
+	COMPOSITE_NODIV(CLK_COUNTER_PWM0_SRC, "clk_counter_pwm0_src", clk_counter_pwm0_src_p, 0,
+			RV1103B_CLKSEL_CON(35), 14, 2, MFLAGS,
+			RV1103B_CLKGATE_CON(5), 7, GFLAGS),
+	GATE(CLK_COUNTER_PWM0, "clk_counter_pwm0", "clk_counter_pwm0_src", 0,
+			RV1103B_PMUCLKGATE_CON(2), 5, GFLAGS),
+	GATE(SCLK_SPI2AHB, "sclk_spi2ahb", "sclk_spi2ahb_io", 0,
+			RV1103B_PMU1CLKGATE_CON(0), 2, GFLAGS),
+	GATE(CLK_UTMI_USBOTG, "clk_utmi_usbotg", "clk_utmi_usbotg_io", 0,
+			RV1103B_PERICRU_IP_CON, 14, GFLAGS),
+};
+
+static struct rockchip_clk_branch rv1103b_armclk __initdata =
+	MUX(ARMCLK, "armclk", mux_armclk_p, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT,
+			RV1103B_CORECLKSEL_CON(0), 1, 1, MFLAGS);
+
+static void __init rv1103b_clk_init(struct device_node *np)
+{
+	struct rockchip_clk_provider *ctx;
+	unsigned long clk_nr;
+	void __iomem *reg_base;
+
+	clk_nr = rockchip_clk_find_max_clk_id(rv1103b_clk_branches,
+					      ARRAY_SIZE(rv1103b_clk_branches)) + 1;
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: could not map cru region\n", __func__);
+		return;
+	}
+
+	ctx = rockchip_clk_init(np, reg_base, clk_nr);
+	if (IS_ERR(ctx)) {
+		pr_err("%s: rockchip clk init failed\n", __func__);
+		iounmap(reg_base);
+		return;
+	}
+
+	rockchip_clk_register_plls(ctx, rv1103b_pll_clks,
+				   ARRAY_SIZE(rv1103b_pll_clks),
+				   RV1103B_GRF_SOC_STATUS0);
+
+	rockchip_clk_register_branches(ctx, rv1103b_clk_branches,
+				       ARRAY_SIZE(rv1103b_clk_branches));
+
+	rockchip_clk_register_armclk_multi_pll(ctx, &rv1103b_armclk,
+					       rv1103b_cpuclk_rates,
+					       ARRAY_SIZE(rv1103b_cpuclk_rates));
+
+	rockchip_register_restart_notifier(ctx, RV1103B_GLB_SRST_FST, NULL);
+
+	rockchip_clk_of_add_provider(np, ctx);
+
+	/* pvtpll src init */
+	writel_relaxed(PVTPLL_SRC_SEL_PVTPLL, reg_base + RV1103B_CORECLKSEL_CON(0));
+	writel_relaxed(PVTPLL_SRC_SEL_PVTPLL, reg_base + RV1103B_NPUCLKSEL_CON(0));
+	writel_relaxed(PVTPLL_SRC_SEL_PVTPLL, reg_base + RV1103B_VICLKSEL_CON(0));
+	writel_relaxed(PVTPLL_SRC_SEL_PVTPLL, reg_base + RV1103B_VEPUCLKSEL_CON(0));
+}
+
+CLK_OF_DECLARE(rv1103b_cru, "rockchip,rv1103b-cru", rv1103b_clk_init);
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index b2fff1d13a4ab..cf0f5f11c34b5 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -66,6 +66,55 @@ struct clk;
 #define PX30_PMU_CLKGATE_CON(x)		((x) * 0x4 + 0x80)
 #define PX30_PMU_MODE			0x0020
 
+#define RV1103B_TOPCRU_BASE		0x60000
+#define RV1103B_PERICRU_BASE		0x0
+#define RV1103B_VICRU_BASE		0x30000
+#define RV1103B_NPUCRU_BASE		0x20000
+#define RV1103B_CORECRU_BASE		0x40000
+#define RV1103B_VEPUCRU_BASE		0x10000
+#define RV1103B_DDRCRU_BASE		0x50000
+#define RV1103B_SUBDDRCRU_BASE		0x58000
+#define RV1103B_PMUCRU_BASE		0x70000
+#define RV1103B_PMU1CRU_BASE		0x80000
+
+#define RV1103B_PMUCLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_PMUCRU_BASE)
+#define RV1103B_PMUCLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_PMUCRU_BASE)
+#define RV1103B_PMUSOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_PMUCRU_BASE)
+#define RV1103B_PMU1CLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_PMU1CRU_BASE)
+#define RV1103B_PMU1CLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_PMU1CRU_BASE)
+#define RV1103B_PMU1SOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_PMU1CRU_BASE)
+#define RV1103B_PLL_CON(x)		((x) * 0x4 + RV1103B_TOPCRU_BASE)
+#define RV1103B_MODE_CON		(0x280 + RV1103B_TOPCRU_BASE)
+#define RV1103B_CLKSEL_CON(x)		((x) * 0x4 + 0x300 + RV1103B_TOPCRU_BASE)
+#define RV1103B_CLKGATE_CON(x)		((x) * 0x4 + 0x800 + RV1103B_TOPCRU_BASE)
+#define RV1103B_SOFTRST_CON(x)		((x) * 0x4 + 0xa00 + RV1103B_TOPCRU_BASE)
+#define RV1103B_GLB_SRST_FST		(0xc08 + RV1103B_TOPCRU_BASE)
+#define RV1103B_GLB_SRST_SND		(0xc0c + RV1103B_TOPCRU_BASE)
+#define RV1103B_CLK_SAI_FRAC_DIV_HIGH	(0xcc0 + RV1103B_TOPCRU_BASE)
+#define RV1103B_PERICLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_PERICRU_BASE)
+#define RV1103B_PERICLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_PERICRU_BASE)
+#define RV1103B_PERISOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_PERICRU_BASE)
+#define RV1103B_PERICRU_IP_CON		(0xc08 + RV1103B_PERICRU_BASE)
+#define RV1103B_VICLKSEL_CON(x)		((x) * 0x4 + 0x300 + RV1103B_VICRU_BASE)
+#define RV1103B_VICLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_VICRU_BASE)
+#define RV1103B_VISOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_VICRU_BASE)
+#define RV1103B_NPUCLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_NPUCRU_BASE)
+#define RV1103B_NPUCLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_NPUCRU_BASE)
+#define RV1103B_NPUSOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_NPUCRU_BASE)
+#define RV1103B_CORECLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_CORECRU_BASE)
+#define RV1103B_CORECLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_CORECRU_BASE)
+#define RV1103B_CORESOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_CORECRU_BASE)
+#define RV1103B_VEPUCLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_VEPUCRU_BASE)
+#define RV1103B_VEPUCLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_VEPUCRU_BASE)
+#define RV1103B_VEPUSOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_VEPUCRU_BASE)
+#define RV1103B_DDRCLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_DDRCRU_BASE)
+#define RV1103B_DDRCLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_DDRCRU_BASE)
+#define RV1103B_DDRSOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_DDRCRU_BASE)
+#define RV1103B_SUBDDRCLKSEL_CON(x)	((x) * 0x4 + 0x300 + RV1103B_SUBDDRCRU_BASE)
+#define RV1103B_SUBDDRCLKGATE_CON(x)	((x) * 0x4 + 0x800 + RV1103B_SUBDDRCRU_BASE)
+#define RV1103B_SUBDDRSOFTRST_CON(x)	((x) * 0x4 + 0xa00 + RV1103B_SUBDDRCRU_BASE)
+#define RV1103B_SUBDDRMODE_CON		(0x280 + RV1103B_SUBDDRCRU_BASE)
+
 #define RV1108_PLL_CON(x)		((x) * 0x4)
 #define RV1108_CLKSEL_CON(x)		((x) * 0x4 + 0x60)
 #define RV1108_CLKGATE_CON(x)		((x) * 0x4 + 0x120)
diff --git a/drivers/clk/samsung/Makefile b/drivers/clk/samsung/Makefile
index f3657f2e1b98c..b3c4ef4e0dbff 100644
--- a/drivers/clk/samsung/Makefile
+++ b/drivers/clk/samsung/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_EXYNOS_5410_COMMON_CLK)	+= clk-exynos5410.o
 obj-$(CONFIG_EXYNOS_5420_COMMON_CLK)	+= clk-exynos5420.o
 obj-$(CONFIG_EXYNOS_5420_COMMON_CLK)	+= clk-exynos5-subcmu.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-artpec8.o
+obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-artpec9.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos5433.o
 obj-$(CONFIG_EXYNOS_AUDSS_CLK_CON) += clk-exynos-audss.o
 obj-$(CONFIG_EXYNOS_CLKOUT)	+= clk-exynos-clkout.o
diff --git a/drivers/clk/samsung/clk-acpm.c b/drivers/clk/samsung/clk-acpm.c
index b90809ce3f882..d8944160793a0 100644
--- a/drivers/clk/samsung/clk-acpm.c
+++ b/drivers/clk/samsung/clk-acpm.c
@@ -20,7 +20,7 @@ struct acpm_clk {
 	u32 id;
 	struct clk_hw hw;
 	unsigned int mbox_chan_id;
-	const struct acpm_handle *handle;
+	struct acpm_handle *handle;
 };
 
 struct acpm_clk_variant {
@@ -113,7 +113,7 @@ static int acpm_clk_register(struct device *dev, struct acpm_clk *aclk,
 
 static int acpm_clk_probe(struct platform_device *pdev)
 {
-	const struct acpm_handle *acpm_handle;
+	struct acpm_handle *acpm_handle;
 	struct clk_hw_onecell_data *clk_data;
 	struct clk_hw **hws;
 	struct device *dev = &pdev->dev;
diff --git a/drivers/clk/samsung/clk-artpec9.c b/drivers/clk/samsung/clk-artpec9.c
new file mode 100644
index 0000000000000..2eaf8117638cd
--- /dev/null
+++ b/drivers/clk/samsung/clk-artpec9.c
@@ -0,0 +1,1224 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd.
+ *             https://www.samsung.com
+ * Copyright (c) 2025  Axis Communications AB.
+ *             https://www.axis.com
+ *
+ * Common Clock Framework support for ARTPEC-9 SoC.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/clock/axis,artpec9-clk.h>
+
+#include "clk.h"
+#include "clk-exynos-arm64.h"
+
+/* NOTE: Must be equal to the last clock ID increased by one */
+#define CMU_CMU_NR_CLK				(CLK_DOUT_CMU_VIO_AUDIO + 1)
+#define CMU_BUS_NR_CLK				(CLK_MOUT_BUS_ACLK_USER + 1)
+#define CMU_CORE_NR_CLK				(CLK_MOUT_CORE_ACLK_USER + 1)
+#define CMU_CPUCL_NR_CLK			(CLK_GOUT_CPUCL_CSSYS_IPCLKPORT_PCLKDBG + 1)
+#define CMU_FSYS0_NR_CLK			(CLK_GOUT_FSYS0_PWM_IPCLKPORT_I_PCLK_S0 + 1)
+#define CMU_FSYS1_NR_CLK			(CLK_GOUT_FSYS1_XHB_USB_IPCLKPORT_CLK + 1)
+#define CMU_IMEM_NR_CLK				(CLK_GOUT_IMEM_PCLK_TMU0_APBIF + 1)
+#define CMU_PERI_NR_CLK				(CLK_GOUT_PERI_UART2_SCLK_UART + 1)
+
+/* Register Offset definitions for CMU_CMU (0x12c00000) */
+#define PLL_LOCKTIME_PLL_AUDIO					0x0000
+#define PLL_LOCKTIME_PLL_SHARED0				0x0004
+#define PLL_LOCKTIME_PLL_SHARED1				0x0008
+#define PLL_CON0_PLL_AUDIO					0x0100
+#define PLL_CON0_PLL_SHARED0					0x0120
+#define PLL_CON0_PLL_SHARED1					0x0140
+#define CLK_CON_MUX_CLKCMU_BUS					0x1000
+#define CLK_CON_MUX_CLKCMU_CDC_CORE				0x1004
+#define CLK_CON_MUX_CLKCMU_CORE_MAIN				0x1008
+#define CLK_CON_MUX_CLKCMU_CPUCL_SWITCH				0x100c
+#define CLK_CON_MUX_CLKCMU_VIO_AUDIO				0x1010
+#define CLK_CON_MUX_CLKCMU_DLP_CORE				0x1014
+#define CLK_CON_MUX_CLKCMU_FSYS0_BUS				0x1018
+#define CLK_CON_MUX_CLKCMU_FSYS0_IP				0x101c
+#define CLK_CON_MUX_CLKCMU_FSYS1_BUS				0x1020
+#define CLK_CON_MUX_CLKCMU_FSYS1_SCAN0				0x1024
+#define CLK_CON_MUX_CLKCMU_FSYS1_SCAN1				0x1028
+#define CLK_CON_MUX_CLKCMU_GPU_2D				0x102c
+#define CLK_CON_MUX_CLKCMU_GPU_3D				0x1030
+#define CLK_CON_MUX_CLKCMU_IMEM_ACLK				0x1034
+#define CLK_CON_MUX_CLKCMU_IMEM_CA5				0x1038
+#define CLK_CON_MUX_CLKCMU_IMEM_JPEG				0x103c
+#define CLK_CON_MUX_CLKCMU_IMEM_SSS				0x1040
+#define CLK_CON_MUX_CLKCMU_IPA_CORE				0x1044
+#define CLK_CON_MUX_CLKCMU_MIF_BUSP				0x1048
+#define CLK_CON_MUX_CLKCMU_MIF_SWITCH				0x104c
+#define CLK_CON_MUX_CLKCMU_PERI_DISP				0x1050
+#define CLK_CON_MUX_CLKCMU_PERI_IP				0x1054
+#define CLK_CON_MUX_CLKCMU_RSP_CORE				0x1058
+#define CLK_CON_MUX_CLKCMU_TRFM					0x105c
+#define CLK_CON_MUX_CLKCMU_VIO_CORE				0x1060
+#define CLK_CON_MUX_CLKCMU_VIO_CORE_L				0x1064
+#define CLK_CON_MUX_CLKCMU_VIP0					0x1068
+#define CLK_CON_MUX_CLKCMU_VIP1					0x106c
+#define CLK_CON_MUX_CLKCMU_VPP_CORE				0x1070
+#define CLK_CON_DIV_CLKCMU_ADD					0x1800
+#define CLK_CON_DIV_CLKCMU_BUS					0x1804
+#define CLK_CON_DIV_CLKCMU_CDC_CORE				0x1808
+#define CLK_CON_DIV_CLKCMU_CORE_MAIN				0x180c
+#define CLK_CON_DIV_CLKCMU_CPUCL_SWITCH				0x1810
+#define CLK_CON_DIV_CLKCMU_DLP_CORE				0x1814
+#define CLK_CON_DIV_CLKCMU_FSYS0_BUS				0x1818
+#define CLK_CON_DIV_CLKCMU_FSYS0_IP				0x181c
+#define CLK_CON_DIV_CLKCMU_FSYS1_BUS				0x1820
+#define CLK_CON_DIV_CLKCMU_FSYS1_SCAN0				0x1824
+#define CLK_CON_DIV_CLKCMU_FSYS1_SCAN1				0x1828
+#define CLK_CON_DIV_CLKCMU_GPU_2D				0x182c
+#define CLK_CON_DIV_CLKCMU_GPU_3D				0x1830
+#define CLK_CON_DIV_CLKCMU_IMEM_ACLK				0x1834
+#define CLK_CON_DIV_CLKCMU_IMEM_CA5				0x1838
+#define CLK_CON_DIV_CLKCMU_IMEM_JPEG				0x183c
+#define CLK_CON_DIV_CLKCMU_IMEM_SSS				0x1840
+#define CLK_CON_DIV_CLKCMU_IPA_CORE				0x1844
+#define CLK_CON_DIV_CLKCMU_LCPU					0x1848
+#define CLK_CON_DIV_CLKCMU_MIF_BUSP				0x184c
+#define CLK_CON_DIV_CLKCMU_MIF_SWITCH				0x1850
+#define CLK_CON_DIV_CLKCMU_PERI_DISP				0x1854
+#define CLK_CON_DIV_CLKCMU_PERI_IP				0x1858
+#define CLK_CON_DIV_CLKCMU_RSP_CORE				0x185c
+#define CLK_CON_DIV_CLKCMU_TRFM					0x1860
+#define CLK_CON_DIV_CLKCMU_VIO_AUDIO				0x1864
+#define CLK_CON_DIV_CLKCMU_VIO_CORE				0x1868
+#define CLK_CON_DIV_CLKCMU_VIO_CORE_L				0x186c
+#define CLK_CON_DIV_CLKCMU_VIP0					0x1870
+#define CLK_CON_DIV_CLKCMU_VIP1					0x1874
+#define CLK_CON_DIV_CLKCMU_VPP_CORE				0x1878
+#define CLK_CON_DIV_PLL_SHARED0_DIV2				0x187c
+#define CLK_CON_DIV_PLL_SHARED0_DIV3				0x1880
+#define CLK_CON_DIV_PLL_SHARED0_DIV4				0x1884
+#define CLK_CON_DIV_PLL_SHARED1_DIV2				0x1888
+#define CLK_CON_DIV_PLL_SHARED1_DIV3				0x188c
+#define CLK_CON_DIV_PLL_SHARED1_DIV4				0x1890
+
+static const unsigned long cmu_cmu_clk_regs[] __initconst = {
+	PLL_LOCKTIME_PLL_AUDIO,
+	PLL_LOCKTIME_PLL_SHARED0,
+	PLL_LOCKTIME_PLL_SHARED1,
+	PLL_CON0_PLL_AUDIO,
+	PLL_CON0_PLL_SHARED0,
+	PLL_CON0_PLL_SHARED1,
+	CLK_CON_MUX_CLKCMU_BUS,
+	CLK_CON_MUX_CLKCMU_CDC_CORE,
+	CLK_CON_MUX_CLKCMU_CORE_MAIN,
+	CLK_CON_MUX_CLKCMU_CPUCL_SWITCH,
+	CLK_CON_MUX_CLKCMU_DLP_CORE,
+	CLK_CON_MUX_CLKCMU_FSYS0_BUS,
+	CLK_CON_MUX_CLKCMU_FSYS0_IP,
+	CLK_CON_MUX_CLKCMU_FSYS1_BUS,
+	CLK_CON_MUX_CLKCMU_FSYS1_SCAN0,
+	CLK_CON_MUX_CLKCMU_FSYS1_SCAN1,
+	CLK_CON_MUX_CLKCMU_GPU_2D,
+	CLK_CON_MUX_CLKCMU_GPU_3D,
+	CLK_CON_MUX_CLKCMU_IMEM_ACLK,
+	CLK_CON_MUX_CLKCMU_IMEM_CA5,
+	CLK_CON_MUX_CLKCMU_IMEM_JPEG,
+	CLK_CON_MUX_CLKCMU_IMEM_SSS,
+	CLK_CON_MUX_CLKCMU_IPA_CORE,
+	CLK_CON_MUX_CLKCMU_MIF_BUSP,
+	CLK_CON_MUX_CLKCMU_MIF_SWITCH,
+	CLK_CON_MUX_CLKCMU_PERI_DISP,
+	CLK_CON_MUX_CLKCMU_PERI_IP,
+	CLK_CON_MUX_CLKCMU_RSP_CORE,
+	CLK_CON_MUX_CLKCMU_TRFM,
+	CLK_CON_MUX_CLKCMU_VIO_CORE,
+	CLK_CON_MUX_CLKCMU_VIO_CORE_L,
+	CLK_CON_MUX_CLKCMU_VIP0,
+	CLK_CON_MUX_CLKCMU_VIP1,
+	CLK_CON_MUX_CLKCMU_VPP_CORE,
+	CLK_CON_DIV_CLKCMU_ADD,
+	CLK_CON_DIV_CLKCMU_BUS,
+	CLK_CON_DIV_CLKCMU_CDC_CORE,
+	CLK_CON_DIV_CLKCMU_CORE_MAIN,
+	CLK_CON_DIV_CLKCMU_CPUCL_SWITCH,
+	CLK_CON_DIV_CLKCMU_VIO_AUDIO,
+	CLK_CON_DIV_CLKCMU_DLP_CORE,
+	CLK_CON_DIV_CLKCMU_FSYS0_BUS,
+	CLK_CON_DIV_CLKCMU_FSYS0_IP,
+	CLK_CON_DIV_CLKCMU_FSYS1_BUS,
+	CLK_CON_DIV_CLKCMU_FSYS1_SCAN0,
+	CLK_CON_DIV_CLKCMU_FSYS1_SCAN1,
+	CLK_CON_DIV_CLKCMU_GPU_2D,
+	CLK_CON_DIV_CLKCMU_GPU_3D,
+	CLK_CON_DIV_CLKCMU_IMEM_ACLK,
+	CLK_CON_DIV_CLKCMU_IMEM_CA5,
+	CLK_CON_DIV_CLKCMU_IMEM_JPEG,
+	CLK_CON_DIV_CLKCMU_IMEM_SSS,
+	CLK_CON_DIV_CLKCMU_IPA_CORE,
+	CLK_CON_DIV_CLKCMU_LCPU,
+	CLK_CON_DIV_CLKCMU_MIF_BUSP,
+	CLK_CON_DIV_CLKCMU_MIF_SWITCH,
+	CLK_CON_DIV_CLKCMU_PERI_DISP,
+	CLK_CON_DIV_CLKCMU_PERI_IP,
+	CLK_CON_DIV_CLKCMU_RSP_CORE,
+	CLK_CON_DIV_CLKCMU_TRFM,
+	CLK_CON_DIV_CLKCMU_VIO_AUDIO,
+	CLK_CON_DIV_CLKCMU_VIO_CORE,
+	CLK_CON_DIV_CLKCMU_VIO_CORE_L,
+	CLK_CON_DIV_CLKCMU_VIP0,
+	CLK_CON_DIV_CLKCMU_VIP1,
+	CLK_CON_DIV_CLKCMU_VPP_CORE,
+	CLK_CON_DIV_PLL_SHARED0_DIV2,
+	CLK_CON_DIV_PLL_SHARED0_DIV3,
+	CLK_CON_DIV_PLL_SHARED0_DIV4,
+	CLK_CON_DIV_PLL_SHARED1_DIV2,
+	CLK_CON_DIV_PLL_SHARED1_DIV3,
+	CLK_CON_DIV_PLL_SHARED1_DIV4,
+};
+
+static const struct samsung_pll_rate_table artpec9_pll_audio_rates[] __initconst = {
+	PLL_A9FRACO_RATE(25 * MHZ, 589824000U, 94, 1, 3, 6238440),
+};
+
+static const struct samsung_pll_clock cmu_cmu_pll_clks[] __initconst = {
+	PLL(pll_a9fracm, CLK_FOUT_SHARED0_PLL, "fout_pll_shared0", "fin_pll",
+	    PLL_LOCKTIME_PLL_SHARED0, PLL_CON0_PLL_SHARED0, NULL),
+	PLL(pll_a9fracm, CLK_FOUT_SHARED1_PLL, "fout_pll_shared1", "fin_pll",
+	    PLL_LOCKTIME_PLL_SHARED1, PLL_CON0_PLL_SHARED1, NULL),
+	PLL(pll_a9fraco, CLK_FOUT_AUDIO_PLL, "fout_pll_audio", "fin_pll",
+	    PLL_LOCKTIME_PLL_AUDIO, PLL_CON0_PLL_AUDIO, artpec9_pll_audio_rates),
+};
+
+PNAME(mout_clkcmu_bus_bus_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div2",
+				 "dout_pll_shared1_div3", "dout_pll_shared1_div4" };
+PNAME(mout_clkcmu_cdc_core_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div3",
+				  "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_core_main_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div2",
+				   "dout_pll_shared1_div3", "dout_pll_shared1_div4" };
+PNAME(mout_clkcmu_cpucl_switch_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div2",
+				      "dout_pll_shared0_div3", "dout_pll_shared1_div3" };
+PNAME(mout_clkcmu_dlp_core_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				  "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_fsys0_bus_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				   "dout_pll_shared1_div4", "dout_pll_shared1_div2" };
+PNAME(mout_clkcmu_fsys0_ip_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div3",
+				  "dout_pll_shared1_div2", "dout_pll_shared0_div3" };
+PNAME(mout_clkcmu_fsys1_bus_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				   "dout_pll_shared1_div2", "dout_pll_shared0_div4" };
+PNAME(mout_clkcmu_fsys1_scan0_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div4" };
+PNAME(mout_clkcmu_fsys1_scan1_p) = { "dout_pll_shared1_div3", "dout_pll_shared1_div4" };
+PNAME(mout_clkcmu_gpu_3d_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				"dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_gpu_2d_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				"dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_imem_aclk_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				   "dout_pll_shared1_div4", "dout_pll_shared1_div2" };
+PNAME(mout_clkcmu_imem_ca5_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div2",
+				  "dout_pll_shared1_div3", "mout_clk_pll_shared1" };
+PNAME(mout_clkcmu_imem_jpeg_p) = { "dout_pll_shared0_div2", "dout_pll_shared0_div3",
+				   "dout_pll_shared1_div2", "dout_pll_shared1_div3" };
+PNAME(mout_clkcmu_imem_sss_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div2" };
+PNAME(mout_clkcmu_ipa_core_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				  "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_mif_switch_p) = { "fout_pll_shared1", "mout_clkcmu_pll_shared0",
+				    "dout_pll_shared0_div2", "dout_pll_shared0_div3" };
+PNAME(mout_clkcmu_mif_busp_p) = { "dout_pll_shared1_div3", "dout_pll_shared1_div4",
+				  "dout_pll_shared0_div4", "dout_pll_shared0_div2" };
+PNAME(mout_clkcmu_peri_disp_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				   "dout_pll_shared1_div4", "dout_pll_shared1_div2" };
+PNAME(mout_clkcmu_peri_ip_p) = { "fout_pll_fsys1", "dout_pll_shared1_2",
+				 "dout_pll_shared1_div4", "dout_pll_shared0_div2" };
+PNAME(mout_clkcmu_rsp_core_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				  "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_trfm_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+			      "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_vio_core_l_p) = { "dout_pll_shared0_div2", "dout_pll_shared1_div3",
+				    "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_vio_core_p) = { "fout_pll_fsys1", "dout_pll_shared0_div2",
+				  "dout_pll_shared1_div3", "dout_pll_shared1_div2" };
+PNAME(mout_clkcmu_vio_audio_p) = { "fout_pll_audio", "mout_clkcmu_pll_audio" };
+PNAME(mout_clkcmu_vip0_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+			      "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_vip1_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+			      "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_vpp_core_p) = { "dout_pll_shared1_div3", "dout_pll_shared0_div2",
+				  "dout_pll_shared1_div2", "mout_clk_pll_fsys1" };
+PNAME(mout_clkcmu_pll_shared0_p) = { "fin_pll", "fout_pll_shared0" };
+PNAME(mout_clkcmu_pll_shared1_p) = { "fin_pll", "fout_pll_shared1" };
+PNAME(mout_clkcmu_pll_audio_p) = { "fin_pll", "fout_pll_audio" };
+
+static const struct samsung_mux_clock cmu_cmu_mux_clks[] __initconst = {
+	MUX(0, "mout_clkcmu_pll_shared0", mout_clkcmu_pll_shared0_p, PLL_CON0_PLL_SHARED0, 4, 1),
+	MUX(0, "mout_clkcmu_pll_shared1", mout_clkcmu_pll_shared1_p, PLL_CON0_PLL_SHARED1, 4, 1),
+	MUX(0, "mout_clkcmu_pll_audio", mout_clkcmu_pll_audio_p, PLL_CON0_PLL_AUDIO, 4, 1),
+	MUX(0, "mout_clkcmu_bus_bus", mout_clkcmu_bus_bus_p, CLK_CON_MUX_CLKCMU_BUS, 0, 2),
+	nMUX(0, "mout_clkcmu_cdc_core", mout_clkcmu_cdc_core_p, CLK_CON_MUX_CLKCMU_CDC_CORE, 0, 2),
+	MUX(0, "mout_clkcmu_core_main", mout_clkcmu_core_main_p,
+	    CLK_CON_MUX_CLKCMU_CORE_MAIN, 0, 2),
+	MUX(0, "mout_clkcmu_cpucl_switch", mout_clkcmu_cpucl_switch_p,
+	    CLK_CON_MUX_CLKCMU_CPUCL_SWITCH, 0, 2),
+	nMUX(0, "mout_clkcmu_dlp_core", mout_clkcmu_dlp_core_p, CLK_CON_MUX_CLKCMU_DLP_CORE, 0, 2),
+	MUX(0, "mout_clkcmu_fsys0_bus", mout_clkcmu_fsys0_bus_p,
+	    CLK_CON_MUX_CLKCMU_FSYS0_BUS, 0, 2),
+	MUX(0, "mout_clkcmu_fsys0_ip", mout_clkcmu_fsys0_ip_p, CLK_CON_MUX_CLKCMU_FSYS0_IP, 0, 2),
+	MUX(0, "mout_clkcmu_fsys1_bus", mout_clkcmu_fsys1_bus_p,
+	    CLK_CON_MUX_CLKCMU_FSYS1_BUS, 0, 2),
+	MUX(0, "mout_clkcmu_fsys1_scan0", mout_clkcmu_fsys1_scan0_p,
+	    CLK_CON_MUX_CLKCMU_FSYS1_SCAN0, 0, 1),
+	MUX(0, "mout_clkcmu_fsys1_scan1", mout_clkcmu_fsys1_scan1_p,
+	    CLK_CON_MUX_CLKCMU_FSYS1_SCAN1, 0, 1),
+	MUX(0, "mout_clkcmu_gpu_2d", mout_clkcmu_gpu_2d_p, CLK_CON_MUX_CLKCMU_GPU_2D, 0, 2),
+	MUX(0, "mout_clkcmu_gpu_3d", mout_clkcmu_gpu_3d_p, CLK_CON_MUX_CLKCMU_GPU_3D, 0, 2),
+	MUX(0, "mout_clkcmu_imem_aclk", mout_clkcmu_imem_aclk_p,
+	    CLK_CON_MUX_CLKCMU_IMEM_ACLK, 0, 2),
+	MUX(0, "mout_clkcmu_imem_ca5", mout_clkcmu_imem_ca5_p, CLK_CON_MUX_CLKCMU_IMEM_CA5, 0, 2),
+	MUX(0, "mout_clkcmu_imem_jpeg", mout_clkcmu_imem_jpeg_p,
+	    CLK_CON_MUX_CLKCMU_IMEM_JPEG, 0, 2),
+	MUX(0, "mout_clkcmu_imem_sss", mout_clkcmu_imem_sss_p, CLK_CON_MUX_CLKCMU_IMEM_SSS, 0, 1),
+	MUX(0, "mout_clkcmu_ipa_core", mout_clkcmu_ipa_core_p, CLK_CON_MUX_CLKCMU_IPA_CORE, 0, 2),
+	MUX(0, "mout_clkcmu_mif_busp", mout_clkcmu_mif_busp_p, CLK_CON_MUX_CLKCMU_MIF_BUSP, 0, 2),
+	MUX(0, "mout_clkcmu_mif_switch", mout_clkcmu_mif_switch_p,
+	    CLK_CON_MUX_CLKCMU_MIF_SWITCH, 0, 2),
+	MUX(0, "mout_clkcmu_peri_disp", mout_clkcmu_peri_disp_p,
+	    CLK_CON_MUX_CLKCMU_PERI_DISP, 0, 2),
+	MUX(0, "mout_clkcmu_peri_ip", mout_clkcmu_peri_ip_p, CLK_CON_MUX_CLKCMU_PERI_IP, 0, 2),
+	MUX(0, "mout_clkcmu_rsp_core", mout_clkcmu_rsp_core_p, CLK_CON_MUX_CLKCMU_RSP_CORE, 0, 2),
+	MUX(0, "mout_clkcmu_trfm", mout_clkcmu_trfm_p, CLK_CON_MUX_CLKCMU_TRFM, 0, 2),
+	MUX(0, "mout_clkcmu_vio_core", mout_clkcmu_vio_core_p, CLK_CON_MUX_CLKCMU_VIO_CORE, 0, 2),
+	MUX(0, "mout_clkcmu_vio_core_l", mout_clkcmu_vio_core_l_p,
+	    CLK_CON_MUX_CLKCMU_VIO_CORE_L, 0, 2),
+	MUX(0, "mout_clkcmu_vio_audio", mout_clkcmu_vio_audio_p,
+	    CLK_CON_MUX_CLKCMU_VIO_AUDIO, 0, 1),
+	MUX(0, "mout_clkcmu_vip0", mout_clkcmu_vip0_p, CLK_CON_MUX_CLKCMU_VIP0, 0, 2),
+	MUX(0, "mout_clkcmu_vip1", mout_clkcmu_vip1_p, CLK_CON_MUX_CLKCMU_VIP1, 0, 2),
+	MUX(0, "mout_clkcmu_vpp_core", mout_clkcmu_vpp_core_p, CLK_CON_MUX_CLKCMU_VPP_CORE, 0, 2),
+};
+
+static const struct samsung_div_clock cmu_cmu_div_clks[] __initconst = {
+	DIV(CLK_DOUT_CMU_ADD, "dout_clkcmu_add", "gate_clkcmu_add", CLK_CON_DIV_CLKCMU_ADD, 0, 8),
+	DIV(CLK_DOUT_CMU_BUS, "dout_clkcmu_bus",
+	    "gate_clkcmu_bus_bus", CLK_CON_DIV_CLKCMU_BUS, 0, 4),
+	DIV_F(CLK_DOUT_CMU_CDC_CORE, "dout_clkcmu_cdc_core",
+	      "gate_clkcmu_cdc_core", CLK_CON_DIV_CLKCMU_CDC_CORE, 0, 4, CLK_SET_RATE_PARENT, 0),
+	DIV(CLK_DOUT_CMU_CORE_MAIN, "dout_clkcmu_core_main",
+	    "gate_clkcmu_core_main", CLK_CON_DIV_CLKCMU_CORE_MAIN, 0, 4),
+	DIV(CLK_DOUT_CMU_CPUCL_SWITCH, "dout_clkcmu_cpucl_switch",
+	    "gate_clkcmu_cpucl_switch", CLK_CON_DIV_CLKCMU_CPUCL_SWITCH, 0, 3),
+	DIV_F(CLK_DOUT_CMU_DLP_CORE, "dout_clkcmu_dlp_core",
+	      "gate_clkcmu_dlp_core", CLK_CON_DIV_CLKCMU_DLP_CORE, 0, 4, CLK_SET_RATE_PARENT, 0),
+	DIV(CLK_DOUT_CMU_FSYS0_BUS, "dout_clkcmu_fsys0_bus",
+	    "gate_clkcmu_fsys0_bus", CLK_CON_DIV_CLKCMU_FSYS0_BUS, 0, 4),
+	DIV(CLK_DOUT_CMU_FSYS0_IP, "dout_clkcmu_fsys0_ip",
+	    "gate_clkcmu_fsys0_ip", CLK_CON_DIV_CLKCMU_FSYS0_IP, 0, 9),
+	DIV(CLK_DOUT_CMU_FSYS1_BUS, "dout_clkcmu_fsys1_bus",
+	    "gate_clkcmu_fsys1_bus", CLK_CON_DIV_CLKCMU_FSYS1_BUS, 0, 4),
+	DIV(CLK_DOUT_CMU_FSYS1_SCAN0, "dout_clkcmu_fsys1_scan0",
+	    "gate_clkcmu_fsys1_scan0", CLK_CON_DIV_CLKCMU_FSYS1_SCAN0, 0, 4),
+	DIV(CLK_DOUT_CMU_FSYS1_SCAN1, "dout_clkcmu_fsys1_scan1",
+	    "gate_clkcmu_fsys1_scan1", CLK_CON_DIV_CLKCMU_FSYS1_SCAN1, 0, 4),
+	DIV(CLK_DOUT_CMU_GPU_2D, "dout_clkcmu_gpu_2d",
+	    "gate_clkcmu_gpu_2d", CLK_CON_DIV_CLKCMU_GPU_2D, 0, 4),
+	DIV(CLK_DOUT_CMU_GPU_3D, "dout_clkcmu_gpu_3d",
+	    "gate_clkcmu_gpu_3d", CLK_CON_DIV_CLKCMU_GPU_3D, 0, 4),
+	DIV(CLK_DOUT_CMU_IMEM_ACLK, "dout_clkcmu_imem_aclk",
+	    "gate_clkcmu_imem_aclk", CLK_CON_DIV_CLKCMU_IMEM_ACLK, 0, 4),
+	DIV(CLK_DOUT_CMU_IMEM_CA5, "dout_clkcmu_imem_ca5",
+	    "gate_clkcmu_imem_ca5", CLK_CON_DIV_CLKCMU_IMEM_CA5, 0, 4),
+	DIV(CLK_DOUT_CMU_IMEM_JPEG, "dout_clkcmu_imem_jpeg",
+	    "gate_clkcmu_imem_jpeg", CLK_CON_DIV_CLKCMU_IMEM_JPEG, 0, 4),
+	DIV(CLK_DOUT_CMU_IMEM_SSS, "dout_clkcmu_imem_sss",
+	    "gate_clkcmu_imem_sss", CLK_CON_DIV_CLKCMU_IMEM_SSS, 0, 4),
+	DIV(CLK_DOUT_CMU_IPA_CORE, "dout_clkcmu_ipa_core",
+	    "gate_clkcmu_ipa_core", CLK_CON_DIV_CLKCMU_IPA_CORE, 0, 4),
+	DIV(CLK_DOUT_CMU_LCPU, "dout_clkcmu_lcpu",
+	    "gate_clkcmu_lcpu", CLK_CON_DIV_CLKCMU_LCPU, 0, 4),
+	DIV(CLK_DOUT_CMU_MIF_BUSP, "dout_clkcmu_mif_busp",
+	    "gate_clkcmu_mif_busp", CLK_CON_DIV_CLKCMU_MIF_BUSP, 0, 3),
+	DIV(CLK_DOUT_CMU_MIF_SWITCH, "dout_clkcmu_mif_switch",
+	    "gate_clkcmu_mif_switch", CLK_CON_DIV_CLKCMU_MIF_SWITCH, 0, 4),
+	DIV(CLK_DOUT_CMU_PERI_DISP, "dout_clkcmu_peri_disp",
+	    "gate_clkcmu_peri_disp", CLK_CON_DIV_CLKCMU_PERI_DISP, 0, 4),
+	DIV(CLK_DOUT_CMU_PERI_IP, "dout_clkcmu_peri_ip",
+	    "gate_clkcmu_peri_ip", CLK_CON_DIV_CLKCMU_PERI_IP, 0, 4),
+	DIV(CLK_DOUT_CMU_RSP_CORE, "dout_clkcmu_rsp_core",
+	    "gate_clkcmu_rsp_core", CLK_CON_DIV_CLKCMU_RSP_CORE, 0, 4),
+	DIV(CLK_DOUT_CMU_TRFM, "dout_clkcmu_trfm",
+	    "gate_clkcmu_trfm", CLK_CON_DIV_CLKCMU_TRFM, 0, 4),
+	DIV(CLK_DOUT_CMU_VIO_CORE, "dout_clkcmu_vio_core",
+	    "gate_clkcmu_vio_core", CLK_CON_DIV_CLKCMU_VIO_CORE, 0, 4),
+	DIV(CLK_DOUT_CMU_VIO_CORE_L, "dout_clkcmu_vio_core_l",
+	    "gate_clkcmu_vio_core_l", CLK_CON_DIV_CLKCMU_VIO_CORE_L, 0, 4),
+	DIV(CLK_DOUT_CMU_VIO_AUDIO, "dout_clkcmu_vio_audio",
+	    "gate_clkcmu_vio_audio", CLK_CON_DIV_CLKCMU_VIO_AUDIO, 0, 4),
+	DIV(CLK_DOUT_CMU_VIP0, "dout_clkcmu_vip0",
+	    "gate_clkcmu_vip0", CLK_CON_DIV_CLKCMU_VIP0, 0, 4),
+	DIV(CLK_DOUT_CMU_VIP1, "dout_clkcmu_vip1",
+	    "gate_clkcmu_vip1", CLK_CON_DIV_CLKCMU_VIP1, 0, 4),
+	DIV(CLK_DOUT_CMU_VPP_CORE, "dout_clkcmu_vpp_core",
+	    "gate_clkcmu_vpp_core", CLK_CON_DIV_CLKCMU_VPP_CORE, 0, 4),
+	DIV(CLK_DOUT_SHARED0_DIV2, "dout_pll_shared0_div2",
+	    "mout_clkcmu_pll_shared0", CLK_CON_DIV_PLL_SHARED0_DIV2, 0, 1),
+	DIV(CLK_DOUT_SHARED0_DIV3, "dout_pll_shared0_div3",
+	    "mout_clkcmu_pll_shared0", CLK_CON_DIV_PLL_SHARED0_DIV3, 0, 2),
+	DIV(CLK_DOUT_SHARED0_DIV4, "dout_pll_shared0_div4",
+	    "dout_pll_shared0_div2", CLK_CON_DIV_PLL_SHARED0_DIV4, 0, 1),
+	DIV(CLK_DOUT_SHARED1_DIV2, "dout_pll_shared1_div2",
+	    "mout_clkcmu_pll_shared1", CLK_CON_DIV_PLL_SHARED1_DIV2, 0, 1),
+	DIV(CLK_DOUT_SHARED1_DIV3, "dout_pll_shared1_div3",
+	    "mout_clkcmu_pll_shared1", CLK_CON_DIV_PLL_SHARED1_DIV3, 0, 2),
+	DIV(CLK_DOUT_SHARED1_DIV4, "dout_pll_shared1_div4",
+	    "dout_pll_shared1_div2", CLK_CON_DIV_PLL_SHARED1_DIV4, 0, 1),
+};
+
+static const struct samsung_cmu_info cmu_cmu_info __initconst = {
+	.pll_clks		= cmu_cmu_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(cmu_cmu_pll_clks),
+	.mux_clks		= cmu_cmu_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_cmu_mux_clks),
+	.div_clks		= cmu_cmu_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(cmu_cmu_div_clks),
+	.nr_clk_ids		= CMU_CMU_NR_CLK,
+	.clk_regs		= cmu_cmu_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_cmu_clk_regs),
+};
+
+/* Register Offset definitions for CMU_BUS (0x13410000) */
+#define PLL_CON0_MUX_CLK_BUS_ACLK_USER				0x0100
+
+static const unsigned long cmu_bus_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLK_BUS_ACLK_USER,
+};
+
+PNAME(mout_clk_bus_aclk_user_p) = {"fin_pll", "dout_clkcmu_bus_bus",};
+
+static const struct samsung_mux_clock cmu_bus_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_BUS_ACLK_USER, "mout_clk_bus_aclk_user", mout_clk_bus_aclk_user_p,
+	    PLL_CON0_MUX_CLK_BUS_ACLK_USER, 4, 1),
+};
+
+static const struct samsung_cmu_info cmu_bus_info __initconst = {
+	.mux_clks		= cmu_bus_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_bus_mux_clks),
+	.nr_clk_ids		= CMU_BUS_NR_CLK,
+	.clk_regs		= cmu_bus_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_bus_clk_regs),
+};
+
+/* Register Offset definitions for CMU_CORE (0x12c10000) */
+#define PLL_CON0_MUX_CLK_CORE_ACLK_USER				0x0100
+
+static const unsigned long cmu_core_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLK_CORE_ACLK_USER,
+};
+
+PNAME(mout_clk_core_aclk_user_p) = {"fin_pll", "dout_clkcmu_core_main",};
+
+static const struct samsung_mux_clock cmu_core_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_CORE_ACLK_USER, "mout_clk_core_aclk_user", mout_clk_core_aclk_user_p,
+	    PLL_CON0_MUX_CLK_CORE_ACLK_USER, 4, 1),
+};
+
+static const struct samsung_cmu_info cmu_core_info __initconst = {
+	.mux_clks		= cmu_core_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_core_mux_clks),
+	.nr_clk_ids		= CMU_CORE_NR_CLK,
+	.clk_regs		= cmu_core_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_core_clk_regs),
+};
+
+/* Register Offset definitions for CMU_CPUCL (0x12810000) */
+#define PLL_LOCKTIME_PLL0_CPUCL					0x0000
+#define PLL_LOCKTIME_PLL1_CPUCL					0x0008
+#define PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_SCU_USER		0x0100
+#define PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_USER			0x0120
+#define PLL_CON0_PLL0_CPUCL					0x0140
+#define PLL_CON0_PLL1_CPUCL					0x0160
+#define CLK_CON_MUX_CLK_CPUCL_PLL				0x1000
+#define CLK_CON_MUX_CLK_CPUCL_PLL_SCU				0x1004
+#define CLK_CON_DIV_CLK_CPUCL_CLUSTER_PERIPHCLK			0x1800
+#define CLK_CON_DIV_CLK_CPUCL_CLUSTER_GICCLK			0x1804
+#define CLK_CON_DIV_CLK_CPUCL_CLUSTER_PCLK			0x1808
+#define CLK_CON_DIV_CLK_CPUCL_CMUREF				0x180c
+#define CLK_CON_DIV_CLK_CPUCL_CPU				0x1810
+#define CLK_CON_DIV_CLK_CPUCL_CLUSTER_ATCLK			0x1818
+#define CLK_CON_DIV_CLK_CPUCL_CLUSTER_SCU			0x181c
+#define CLK_CON_DIV_CLK_CPUCL_DBG				0x1820
+#define CLK_CON_GAT_CLK_CLUSTER_CPU				0x2008
+#define CLK_CON_GAT_CLK_CPUCL_SHORTSTOP				0x200c
+#define CSSYS_IPCLKPORT_ATCLK					0x2070
+#define CSSYS_IPCLKPORT_PCLKDBG					0x2074
+#define DMYQCH_CON_CSSYS_QCH					0x3000
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK0				0x3104
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK1				0x3108
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK2				0x310c
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK3				0x3110
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK4				0x3114
+#define DMYQCH_CON_CLUSTER_QCH_CORECLK5				0x3118
+#define DMYQCH_CON_CLUSTER_QCH_PERIPHCLK			0x311c
+
+static const unsigned long cmu_cpucl_clk_regs[] __initconst = {
+	PLL_LOCKTIME_PLL0_CPUCL,
+	PLL_LOCKTIME_PLL1_CPUCL,
+	PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_SCU_USER,
+	PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_USER,
+	PLL_CON0_PLL0_CPUCL,
+	PLL_CON0_PLL1_CPUCL,
+	CLK_CON_MUX_CLK_CPUCL_PLL,
+	CLK_CON_MUX_CLK_CPUCL_PLL_SCU,
+	CLK_CON_DIV_CLK_CPUCL_CLUSTER_PERIPHCLK,
+	CLK_CON_DIV_CLK_CPUCL_CLUSTER_GICCLK,
+	CLK_CON_DIV_CLK_CPUCL_CLUSTER_PCLK,
+	CLK_CON_DIV_CLK_CPUCL_CMUREF,
+	CLK_CON_DIV_CLK_CPUCL_CPU,
+	CLK_CON_DIV_CLK_CPUCL_CLUSTER_ATCLK,
+	CLK_CON_DIV_CLK_CPUCL_CLUSTER_SCU,
+	CLK_CON_DIV_CLK_CPUCL_DBG,
+	CLK_CON_GAT_CLK_CLUSTER_CPU,
+	CLK_CON_GAT_CLK_CPUCL_SHORTSTOP,
+	CSSYS_IPCLKPORT_ATCLK,
+	CSSYS_IPCLKPORT_PCLKDBG,
+	DMYQCH_CON_CSSYS_QCH,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK0,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK1,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK2,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK3,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK4,
+	DMYQCH_CON_CLUSTER_QCH_CORECLK5,
+	DMYQCH_CON_CLUSTER_QCH_PERIPHCLK,
+};
+
+/* rate_table must be in descending order  */
+static const struct samsung_pll_rate_table artpec9_pll_cpucl_rates[] __initconst = {
+	PLL_35XX_RATE(25 * MHZ, 1400000000U, 56, 1, 0),
+	PLL_35XX_RATE(25 * MHZ, 1100000000U, 44, 1, 0),
+	PLL_35XX_RATE(25 * MHZ,  850000000U, 34, 1, 0),
+};
+
+static const struct samsung_pll_clock cmu_cpucl_pll_clks[] __initconst = {
+	PLL(pll_a9fracm, CLK_FOUT_CPUCL_PLL0, "fout_pll0_cpucl", "fin_pll",
+	    PLL_LOCKTIME_PLL0_CPUCL, PLL_CON0_PLL0_CPUCL, artpec9_pll_cpucl_rates),
+	PLL(pll_a9fracm, CLK_FOUT_CPUCL_PLL1, "fout_pll1_cpucl", "fin_pll",
+	    PLL_LOCKTIME_PLL1_CPUCL, PLL_CON0_PLL1_CPUCL, artpec9_pll_cpucl_rates),
+};
+
+PNAME(mout_clkcmu_cpucl_switch_scu_user_p) = { "fin_pll", "dout_clkcmu_cpucl_switch" };
+PNAME(mout_clkcmu_cpucl_switch_user_p) = { "fin_pll", "dout_clkcmu_cpucl_switch" };
+PNAME(mout_pll0_cpucl_p) = { "fin_pll", "fout_pll0_cpucl" };
+PNAME(mout_clk_cpucl_pll0_p) = { "mout_pll0_cpucl", "mout_clkcmu_cpucl_switch_user" };
+PNAME(mout_pll1_cpucl_p) = { "fin_pll", "fout_pll1_cpucl" };
+PNAME(mout_clk_cpucl_pll_scu_p) = { "mout_pll1_cpucl", "mout_clkcmu_cpucl_switch_scu_user" };
+
+static const struct samsung_mux_clock cmu_cpucl_mux_clks[] __initconst = {
+	MUX_F(0, "mout_pll0_cpucl", mout_pll0_cpucl_p,
+	      PLL_CON0_PLL0_CPUCL, 4, 1, CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0),
+	MUX_F(0, "mout_pll1_cpucl", mout_pll1_cpucl_p,
+	      PLL_CON0_PLL1_CPUCL, 4, 1, CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0),
+	MUX(CLK_MOUT_CPUCL_SWITCH_SCU_USER, "mout_clkcmu_cpucl_switch_scu_user",
+	    mout_clkcmu_cpucl_switch_scu_user_p, PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_SCU_USER, 4, 1),
+	MUX(CLK_MOUT_CPUCL_SWITCH_USER, "mout_clkcmu_cpucl_switch_user",
+	    mout_clkcmu_cpucl_switch_user_p, PLL_CON0_MUX_CLKCMU_CPUCL_SWITCH_USER, 4, 1),
+	MUX_F(CLK_MOUT_CPUCL_PLL0, "mout_clk_cpucl_pll0",
+	      mout_clk_cpucl_pll0_p, CLK_CON_MUX_CLK_CPUCL_PLL, 0, 1, CLK_SET_RATE_PARENT, 0),
+	MUX_F(CLK_MOUT_CPUCL_PLL_SCU, "mout_clk_cpucl_pll_scu", mout_clk_cpucl_pll_scu_p,
+	      CLK_CON_MUX_CLK_CPUCL_PLL_SCU, 0, 1, CLK_SET_RATE_PARENT, 0),
+};
+
+static const struct samsung_fixed_factor_clock cpucl_ffactor_clks[] __initconst = {
+	FFACTOR(CLK_DOUT_CPUCL_CPU, "dout_clk_cpucl_cpu",
+		"mout_clk_cpucl_pll0", 1, 1, CLK_SET_RATE_PARENT),
+};
+
+static const struct samsung_div_clock cmu_cpucl_div_clks[] __initconst = {
+	DIV(CLK_DOUT_CPUCL_CLUSTER_PERIPHCLK, "dout_clk_cluster_periphclk",
+	    "clk_con_gat_clk_cluster_cpu", CLK_CON_DIV_CLK_CPUCL_CLUSTER_PERIPHCLK, 0, 4),
+	DIV(CLK_DOUT_CPUCL_CLUSTER_GICCLK, "dout_clk_cluster_gicclk",
+	    "clk_con_gat_clk_cluster_cpu", CLK_CON_DIV_CLK_CPUCL_CLUSTER_GICCLK, 0, 4),
+	DIV(CLK_DOUT_CPUCL_CLUSTER_PCLK, "dout_clk_cluster_pclk",
+	    "clk_con_gat_clk_cluster_cpu", CLK_CON_DIV_CLK_CPUCL_CLUSTER_PCLK, 0, 4),
+	DIV(CLK_DOUT_CPUCL_CMUREF, "dout_clk_cpucl_cmuref",
+	    "dout_clk_cpucl_cpu", CLK_CON_DIV_CLK_CPUCL_CMUREF, 0, 3),
+	DIV(CLK_DOUT_CPUCL_CLUSTER_ATCLK, "dout_clk_cluster_atclk",
+	    "clk_con_gat_clk_cluster_cpu", CLK_CON_DIV_CLK_CPUCL_CLUSTER_ATCLK, 0, 4),
+	DIV_F(CLK_DOUT_CPUCL_CLUSTER_SCU, "dout_clk_cluster_scu", "mout_clk_cpucl_pll_scu",
+	      CLK_CON_DIV_CLK_CPUCL_CLUSTER_SCU, 0, 4, CLK_SET_RATE_PARENT, 0),
+	DIV(CLK_DOUT_CPUCL_DBG, "dout_clk_cpucl_dbg",
+	    "dout_clk_cpucl_cpu", CLK_CON_DIV_CLK_CPUCL_DBG, 0, 4),
+};
+
+static const struct samsung_gate_clock cmu_cpucl_gate_clks[] __initconst = {
+	GATE(CLK_GOUT_CPUCL_CLUSTER_CPU, "clk_con_gat_clk_cluster_cpu",
+	     "clk_con_gat_clk_cpucl_shortstop", CLK_CON_GAT_CLK_CLUSTER_CPU, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_CPUCL_SHORTSTOP, "clk_con_gat_clk_cpucl_shortstop", "dout_clk_cpucl_cpu",
+	     CLK_CON_GAT_CLK_CPUCL_SHORTSTOP, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_CPUCL_CSSYS_IPCLKPORT_ATCLK, "cssys_ipclkport_atclk", "dout_clk_cpucl_dbg",
+	     CSSYS_IPCLKPORT_ATCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_CPUCL_CSSYS_IPCLKPORT_PCLKDBG, "cssys_ipclkport_pclkdbg",
+	     "dout_clk_cpucl_dbg", CSSYS_IPCLKPORT_PCLKDBG, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+};
+
+static const struct samsung_cmu_info cmu_cpucl_info __initconst = {
+	.pll_clks		= cmu_cpucl_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(cmu_cpucl_pll_clks),
+	.fixed_factor_clks	= cpucl_ffactor_clks,
+	.nr_fixed_factor_clks	= ARRAY_SIZE(cpucl_ffactor_clks),
+	.mux_clks		= cmu_cpucl_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_cpucl_mux_clks),
+	.div_clks		= cmu_cpucl_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(cmu_cpucl_div_clks),
+	.gate_clks              = cmu_cpucl_gate_clks,
+	.nr_gate_clks           = ARRAY_SIZE(cmu_cpucl_gate_clks),
+	.nr_clk_ids		= CMU_CPUCL_NR_CLK,
+	.clk_regs		= cmu_cpucl_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_cpucl_clk_regs),
+};
+
+/* Register Offset definitions for CMU_FSYS0 (0x14410000) */
+#define PLL_CON0_MUX_CLK_FSYS0_BUS_USER				0x0100
+#define PLL_CON0_MUX_CLK_FSYS0_IP_USER				0x0120
+#define PLL_CON0_MUX_CLK_FSYS0_MAIN_USER			0x0140
+#define CLK_CON_DIV_CLK_FSYS0_125				0x1800
+#define CLK_CON_DIV_CLK_FSYS0_ADC				0x1804
+#define CLK_CON_DIV_CLK_FSYS0_BUS_300				0x1808
+#define CLK_CON_DIV_CLK_FSYS0_EQOS0				0x1814
+#define CLK_CON_DIV_CLK_FSYS0_EQOS1				0x1818
+#define CLK_CON_DIV_CLK_FSYS0_EQOS_250				0x181C
+#define CLK_CON_DIV_CLK_FSYS0_MMC_CARD0				0x1820
+#define CLK_CON_DIV_CLK_FSYS0_MMC_CARD1				0x1824
+#define CLK_CON_DIV_CLK_FSYS0_MMC_CARD2				0x1828
+#define CLK_CON_DIV_CLK_FSYS0_QSPI				0x182c
+#define CLK_CON_DIV_CLK_FSYS0_SFMC_NAND				0x1830
+#define CLK_CON_FSYS0_I2C0_IPCLKPORT_I_PCLK			0x2040
+#define CLK_CON_FSYS0_I2C1_IPCLKPORT_I_PCLK			0x2044
+#define CLK_CON_MMC0_IPCLKPORT_I_ACLK				0x2078
+#define CLK_CON_MMC1_IPCLKPORT_I_ACLK				0x2080
+#define CLK_CON_MMC2_IPCLKPORT_I_ACLK				0x2088
+#define CLK_CON_PWM_IPCLKPORT_I_PCLK_S0				0x2090
+#define CLK_CON_DMYQCH_CON_ADC_WRAP_QCH				0x3000
+#define CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH			0x3004
+#define CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH			0x3008
+#define CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH			0x3010
+#define CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH			0x3014
+#define CLK_CON_DMYQCH_CON_MMC0_QCH				0x3018
+#define CLK_CON_DMYQCH_CON_MMC1_QCH				0x301c
+#define CLK_CON_DMYQCH_CON_MMC2_QCH				0x3020
+#define CLK_CON_DMYQCH_CON_QSPI_QCH				0x3024
+#define CLK_CON_DMYQCH_CON_SFMC_QCH				0x3028
+
+static const unsigned long cmu_fsys0_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLK_FSYS0_BUS_USER,
+	PLL_CON0_MUX_CLK_FSYS0_IP_USER,
+	PLL_CON0_MUX_CLK_FSYS0_MAIN_USER,
+	CLK_CON_DIV_CLK_FSYS0_125,
+	CLK_CON_DIV_CLK_FSYS0_ADC,
+	CLK_CON_DIV_CLK_FSYS0_BUS_300,
+	CLK_CON_DIV_CLK_FSYS0_EQOS0,
+	CLK_CON_DIV_CLK_FSYS0_EQOS1,
+	CLK_CON_DIV_CLK_FSYS0_EQOS_250,
+	CLK_CON_DIV_CLK_FSYS0_MMC_CARD0,
+	CLK_CON_DIV_CLK_FSYS0_MMC_CARD1,
+	CLK_CON_DIV_CLK_FSYS0_MMC_CARD2,
+	CLK_CON_DIV_CLK_FSYS0_QSPI,
+	CLK_CON_DIV_CLK_FSYS0_SFMC_NAND,
+	CLK_CON_FSYS0_I2C0_IPCLKPORT_I_PCLK,
+	CLK_CON_FSYS0_I2C1_IPCLKPORT_I_PCLK,
+	CLK_CON_MMC0_IPCLKPORT_I_ACLK,
+	CLK_CON_MMC1_IPCLKPORT_I_ACLK,
+	CLK_CON_MMC2_IPCLKPORT_I_ACLK,
+	CLK_CON_PWM_IPCLKPORT_I_PCLK_S0,
+	CLK_CON_DMYQCH_CON_ADC_WRAP_QCH,
+	CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH,
+	CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH,
+	CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH,
+	CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH,
+	CLK_CON_DMYQCH_CON_MMC0_QCH,
+	CLK_CON_DMYQCH_CON_MMC1_QCH,
+	CLK_CON_DMYQCH_CON_MMC2_QCH,
+	CLK_CON_DMYQCH_CON_QSPI_QCH,
+	CLK_CON_DMYQCH_CON_SFMC_QCH,
+};
+
+PNAME(mout_fsys0_bus_user_p) = { "fin_pll", "dout_clkcmu_fsys0_bus" };
+PNAME(mout_fsys0_ip_user_p) = { "fin_pll", "dout_clkcmu_fsys0_ip" };
+PNAME(mout_fsys0_main_user_p) = { "fin_pll", "fout_pll_fsys1" };
+
+static const struct samsung_mux_clock cmu_fsys0_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_FSYS0_BUS_USER, "mout_fsys0_bus_user",
+	    mout_fsys0_bus_user_p, PLL_CON0_MUX_CLK_FSYS0_BUS_USER, 4, 1),
+	MUX(CLK_MOUT_FSYS0_IP_USER, "mout_fsys0_ip_user",
+	    mout_fsys0_ip_user_p, PLL_CON0_MUX_CLK_FSYS0_IP_USER, 4, 1),
+	MUX(CLK_MOUT_FSYS0_MAIN_USER, "mout_fsys0_main_user",
+	    mout_fsys0_main_user_p, PLL_CON0_MUX_CLK_FSYS0_MAIN_USER, 4, 1),
+};
+
+static const struct samsung_div_clock cmu_fsys0_div_clks[] __initconst = {
+	DIV(CLK_DOUT_FSYS0_125, "dout_fsys0_125", "mout_fsys0_main_user",
+	    CLK_CON_DIV_CLK_FSYS0_125, 0, 5),
+	DIV(CLK_DOUT_FSYS0_ADC, "dout_fsys0_adc", "mout_fsys0_main_user",
+	    CLK_CON_DIV_CLK_FSYS0_ADC, 0, 7),
+	DIV(CLK_DOUT_FSYS0_BUS_300, "dout_fsys0_bus_300", "mout_fsys0_bus_user",
+	    CLK_CON_DIV_CLK_FSYS0_BUS_300, 0, 4),
+	DIV(CLK_DOUT_FSYS0_EQOS0, "dout_fsys0_eqos0", "dout_fsys0_eqos_250",
+	    CLK_CON_DIV_CLK_FSYS0_EQOS0, 0, 7),
+	DIV(CLK_DOUT_FSYS0_EQOS1, "dout_fsys0_eqos1", "dout_fsys0_eqos_250",
+	    CLK_CON_DIV_CLK_FSYS0_EQOS1, 0, 7),
+	DIV(0, "dout_fsys0_eqos_250", "mout_fsys0_main_user",
+	    CLK_CON_DIV_CLK_FSYS0_EQOS_250, 0, 4),
+	DIV(CLK_DOUT_FSYS0_MMC_CARD0, "dout_fsys0_mmc_card0", "mout_fsys0_ip_user",
+	    CLK_CON_DIV_CLK_FSYS0_MMC_CARD0, 0, 10),
+	DIV(CLK_DOUT_FSYS0_MMC_CARD1, "dout_fsys0_mmc_card1", "mout_fsys0_ip_user",
+	    CLK_CON_DIV_CLK_FSYS0_MMC_CARD1, 0, 10),
+	DIV(CLK_DOUT_FSYS0_MMC_CARD2, "dout_fsys0_mmc_card2", "mout_fsys0_ip_user",
+	    CLK_CON_DIV_CLK_FSYS0_MMC_CARD2, 0, 10),
+	DIV(CLK_DOUT_FSYS0_QSPI, "dout_fsys0_qspi", "mout_fsys0_ip_user",
+	    CLK_CON_DIV_CLK_FSYS0_QSPI, 0, 4),
+	DIV(CLK_DOUT_FSYS0_SFMC_NAND, "dout_fsys0_sfmc_nand", "mout_fsys0_ip_user",
+	    CLK_CON_DIV_CLK_FSYS0_SFMC_NAND, 0, 4),
+};
+
+static const struct samsung_gate_clock cmu_fsys0_gate_clks[] __initconst = {
+	GATE(0, "adc_wrap_ipclkport_clk", "dout_fsys0_adc",
+	     CLK_CON_DMYQCH_CON_ADC_WRAP_QCH, 1, CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_ACLK_I, "eqos_top0_ipclkport_aclk_i",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_CLK_CSR_I, "eqos_top0_ipclkport_clk_csr_i",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_I_RGMII_PHASE_CLK_250,
+	     "eqos_top0_ipclkport_i_rgmii_phase_clk_250",
+	     "dout_fsys0_eqos_250", CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_I_RGMII_TXCLK, "eqos_top0_ipclkport_i_rgmii_txclk",
+	     "dout_fsys0_eqos0", CLK_CON_DMYQCH_CON_EQOS_TOP0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_I_RGMII_PHASE_CLK_250,
+	     "eqos_top1_ipclkport_i_rgmii_phase_clk_250",
+	     "dout_fsys0_eqos_250", CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_I_RGMII_TXCLK, "eqos_top1_ipclkport_i_rgmii_txclk",
+	     "dout_fsys0_eqos1", CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_ACLK_I, "eqos_top1_ipclkport_aclk_i",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_CLK_CSR_I, "eqos_top1_ipclkport_clk_csr_i",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_EQOS_TOP1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_APB_S_PCLK, "i3c0_ipclkport_i_apb_s_pclk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_CORE_CLK, "i3c0_ipclkport_i_core_clk",
+	     "dout_fsys0_125", CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_DMA_CLK, "i3c0_ipclkport_i_dma_clk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH,
+	     1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_HDR_TX_CLK, "i3c0_ipclkport_i_hdr_tx_clk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C0_QCH,
+	     1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_APB_S_PCLK, "i3c1_ipclkport_i_apb_s_pclk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH,
+	     1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_CORE_CLK, "i3c1_ipclkport_i_core_clk",
+	     "dout_fsys0_125", CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_DMA_CLK, "i3c1_ipclkport_i_dma_clk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH,
+	     1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_HDR_TX_CLK, "i3c1_ipclkport_i_hdr_tx_clk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_FSYS0_I3C1_QCH,
+	     1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_MMC0_IPCLKPORT_SDCLKIN, "mmc0_ipclkport_sdclkin",
+	     "dout_fsys0_mmc_card0", CLK_CON_DMYQCH_CON_MMC0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_MMC1_IPCLKPORT_SDCLKIN, "mmc1_ipclkport_sdclkin",
+	     "dout_fsys0_mmc_card1", CLK_CON_DMYQCH_CON_MMC1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_MMC2_IPCLKPORT_SDCLKIN, "mmc2_ipclkport_sdclkin",
+	     "dout_fsys0_mmc_card2", CLK_CON_DMYQCH_CON_MMC2_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_QSPI_IPCLKPORT_HCLK, "qspi_ipclkport_hclk",
+	     "dout_fsys0_bus_300", CLK_CON_DMYQCH_CON_QSPI_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_QSPI_IPCLKPORT_SSI_CLK, "qspi_ipclkport_ssi_clk", "dout_fsys0_qspi",
+	     CLK_CON_DMYQCH_CON_QSPI_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_SFMC_IPCLKPORT_I_ACLK_NAND, "sfmc_ipclkport_i_aclk_nand",
+	     "dout_fsys0_sfmc_nand", CLK_CON_DMYQCH_CON_SFMC_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_FSYS0_I2C0_IPCLKPORT_I_PCLK, "i2c0_ipclkport_i_pclk", "dout_fsys0_bus_300",
+	     CLK_CON_FSYS0_I2C0_IPCLKPORT_I_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_I2C1_IPCLKPORT_I_PCLK, "i2c1_ipclkport_i_pclk", "dout_fsys0_bus_300",
+	     CLK_CON_FSYS0_I2C1_IPCLKPORT_I_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_MMC0_IPCLKPORT_I_ACLK, "mmc0_ipclkport_i_aclk", "dout_fsys0_bus_300",
+	     CLK_CON_MMC0_IPCLKPORT_I_ACLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_MMC1_IPCLKPORT_I_ACLK, "mmc1_ipclkport_i_aclk", "dout_fsys0_bus_300",
+	     CLK_CON_MMC1_IPCLKPORT_I_ACLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_MMC2_IPCLKPORT_I_ACLK, "mmc2_ipclkport_i_aclk", "dout_fsys0_bus_300",
+	     CLK_CON_MMC2_IPCLKPORT_I_ACLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS0_PWM_IPCLKPORT_I_PCLK_S0, "pwm_ipclkport_i_pclk", "dout_fsys0_bus_300",
+	     CLK_CON_PWM_IPCLKPORT_I_PCLK_S0, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+};
+
+static const struct samsung_cmu_info cmu_fsys0_info __initconst = {
+	.mux_clks		= cmu_fsys0_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_fsys0_mux_clks),
+	.div_clks		= cmu_fsys0_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(cmu_fsys0_div_clks),
+	.gate_clks              = cmu_fsys0_gate_clks,
+	.nr_gate_clks           = ARRAY_SIZE(cmu_fsys0_gate_clks),
+	.nr_clk_ids		= CMU_FSYS0_NR_CLK,
+	.clk_regs		= cmu_fsys0_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_fsys0_clk_regs),
+};
+
+/* Register Offset definitions for CMU_FSYS1 (0x14c10000) */
+#define PLL_LOCKTIME_PLL_FSYS1						0x0000
+#define PLL_CON0_MUX_CLK_FSYS1_BUS_USER					0x0100
+#define PLL_CON0_MUX_CLK_FSYS1_SCAN0_USER				0x0120
+#define PLL_CON0_MUX_CLK_FSYS1_SCAN1_USER				0x0140
+#define PLL_CON0_PLL_FSYS1						0x0160
+#define CLK_CON_DIV_CLK_FSYS1_200					0x1808
+#define CLK_CON_DIV_CLK_FSYS1_BUS_300					0x1810
+#define CLK_CON_DIV_CLK_FSYS1_OTP_MEM					0x1814
+#define CLK_CON_DIV_CLK_FSYS1_PCIE_PHY_REFCLK_SYSPLL			0x1818
+#define CLK_CON_FSYS1_UART0_IPCLKPORT_I_PCLK				0x202c
+#define CLK_CON_FSYS1_UART0_IPCLKPORT_I_SCLK_UART			0x2030
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_300		0x205c
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_DBI_ACLK_SOC		0x2068
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_MSTR_ACLK_SOC	0x206c
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_SLV_ACLK_SOC		0x2070
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_DBI_ACLK_SOC		0x2078
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_MSTR_ACLK_SOC	0x2080
+#define CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_SLV_ACLK_SOC		0x2084
+#define CLK_CON_USB20DRD_IPCLKPORT_ACLK_PHYCTRL_20			0x209c
+#define CLK_CON_USB20DRD_IPCLKPORT_BUS_CLK_EARLY			0x20a0
+#define CLK_CON_XHB_AHBBR_FSYS1_IPCLKPORT_CLK				0x20a8
+#define CLK_CON_XHB_USB_IPCLKPORT_CLK					0x20ac
+#define CLK_CON_DMYQCH_CON_TZ400_QCH					0x3004
+#define CLK_CON_DMYQCH_CON_PCIE_TOP_QCH_PHY_100				0x309c
+#define CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_0_0				0x3050
+#define CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_1_0				0x3058
+
+static const unsigned long cmu_fsys1_clk_regs[] __initconst = {
+	PLL_LOCKTIME_PLL_FSYS1,
+	PLL_CON0_MUX_CLK_FSYS1_BUS_USER,
+	PLL_CON0_MUX_CLK_FSYS1_SCAN0_USER,
+	PLL_CON0_MUX_CLK_FSYS1_SCAN1_USER,
+	PLL_CON0_PLL_FSYS1,
+	CLK_CON_DIV_CLK_FSYS1_200,
+	CLK_CON_DIV_CLK_FSYS1_BUS_300,
+	CLK_CON_DIV_CLK_FSYS1_OTP_MEM,
+	CLK_CON_DIV_CLK_FSYS1_PCIE_PHY_REFCLK_SYSPLL,
+	CLK_CON_FSYS1_UART0_IPCLKPORT_I_PCLK,
+	CLK_CON_FSYS1_UART0_IPCLKPORT_I_SCLK_UART,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_300,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_DBI_ACLK_SOC,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_MSTR_ACLK_SOC,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_SLV_ACLK_SOC,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_DBI_ACLK_SOC,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_MSTR_ACLK_SOC,
+	CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_SLV_ACLK_SOC,
+	CLK_CON_USB20DRD_IPCLKPORT_ACLK_PHYCTRL_20,
+	CLK_CON_USB20DRD_IPCLKPORT_BUS_CLK_EARLY,
+	CLK_CON_XHB_AHBBR_FSYS1_IPCLKPORT_CLK,
+	CLK_CON_XHB_USB_IPCLKPORT_CLK,
+	CLK_CON_DMYQCH_CON_TZ400_QCH,
+	CLK_CON_DMYQCH_CON_PCIE_TOP_QCH_PHY_100,
+	CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_0_0,
+	CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_1_0
+};
+
+static const struct samsung_pll_rate_table artpec9_pll_fsys1_rates[] __initconst = {
+	PLL_35XX_RATE(25 * MHZ, 2000000000U, 80, 1, 0),
+};
+
+static const struct samsung_pll_clock cmu_fsys1_pll_clks[] __initconst = {
+	PLL(pll_a9fracm, CLK_FOUT_FSYS1_PLL, "fout_pll_fsys1", "fin_pll",
+	    PLL_LOCKTIME_PLL_FSYS1, PLL_CON0_PLL_FSYS1, artpec9_pll_fsys1_rates),
+};
+
+PNAME(mout_fsys1_scan0_user_p) = { "fin_pll", "dout_clkcmu_fsys1_scan0" };
+PNAME(mout_fsys1_scan1_user_p) = { "fin_pll", "dout_clkcmu_fsys1_scan1" };
+PNAME(mout_fsys1_bus_user_p) = { "fin_pll", "dout_clkcmu_fsys1_bus" };
+PNAME(mout_fsys_pll_fsys_p) = { "fin_pll", "fout_pll_fsys1" };
+
+static const struct samsung_mux_clock cmu_fsys1_mux_clks[] __initconst = {
+	MUX(0, "mout_clk_pll_fsys1", mout_fsys_pll_fsys_p, PLL_CON0_PLL_FSYS1, 4, 1),
+	MUX(CLK_MOUT_FSYS1_SCAN0_USER, "mout_fsys1_scan0_user",
+	    mout_fsys1_scan0_user_p, PLL_CON0_MUX_CLK_FSYS1_SCAN0_USER, 4, 1),
+	MUX(CLK_MOUT_FSYS1_SCAN1_USER, "mout_fsys1_scan1_user",
+	    mout_fsys1_scan1_user_p, PLL_CON0_MUX_CLK_FSYS1_SCAN1_USER, 4, 1),
+	MUX(CLK_MOUT_FSYS1_BUS_USER, "mout_fsys1_bus_user",
+	    mout_fsys1_bus_user_p, PLL_CON0_MUX_CLK_FSYS1_BUS_USER, 4, 1),
+};
+
+static const struct samsung_div_clock cmu_fsys1_div_clks[] __initconst = {
+	DIV(CLK_DOUT_FSYS1_200, "dout_fsys1_200", "mout_clk_pll_fsys1",
+	    CLK_CON_DIV_CLK_FSYS1_200, 0, 4),
+	DIV(CLK_DOUT_FSYS1_BUS_300, "dout_fsys1_bus_300", "mout_fsys1_bus_user",
+	    CLK_CON_DIV_CLK_FSYS1_BUS_300, 0, 4),
+	DIV(CLK_DOUT_FSYS1_OTP_MEM, "dout_fsys1_otp_mem", "fin_pll",
+	    CLK_CON_DIV_CLK_FSYS1_OTP_MEM, 0, 4),
+	DIV(CLK_DOUT_FSYS1_PCIE_PHY_REFCLK_SYSPLL, "dout_fsys1_pcie_phy_refclk_syspll",
+	    "mout_clk_pll_fsys1", CLK_CON_DIV_CLK_FSYS1_PCIE_PHY_REFCLK_SYSPLL, 0, 5),
+};
+
+static const struct samsung_gate_clock cmu_fsys1_gate_clks[] __initconst = {
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_100,
+	     "pcie_top_ipclkport_pcie_phy_apb2cr_pclk_100", "dout_fsys1_pcie_phy_refclk_syspll",
+	     CLK_CON_DMYQCH_CON_PCIE_TOP_QCH_PHY_100, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(0, "tzc400_ipclkport_aclk0", "mout_fsys1_bus_user",
+	     CLK_CON_DMYQCH_CON_TZ400_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "tzc400_ipclkport_aclk1", "mout_fsys1_bus_user",
+	     CLK_CON_DMYQCH_CON_TZ400_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "tzc400_ipclkport_pclk", "dout_fsys1_bus_300",
+	     CLK_CON_DMYQCH_CON_TZ400_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_UART0_PCLK, "uart", "dout_fsys1_bus_300",
+	     CLK_CON_FSYS1_UART0_IPCLKPORT_I_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_UART0_SCLK_UART, "clk_uart_baud0", "dout_fsys1_200",
+	     CLK_CON_FSYS1_UART0_IPCLKPORT_I_SCLK_UART, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_300,
+	     "pcie_top_ipclkport_pcie_phy_apb2cr_pclk_300", "dout_fsys1_bus_300",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_300, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_DBI_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x1_dbi_aclk_soc", "dout_fsys1_bus_300",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_DBI_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_MSTR_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x1_mstr_aclk_soc", "mout_fsys1_bus_user",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_MSTR_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_SLV_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x1_slv_aclk_soc", "mout_fsys1_bus_user",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X1_SLV_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_DBI_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x2_dbi_aclk_soc", "dout_fsys1_bus_300",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_DBI_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_MSTR_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x2_mstr_aclk_soc", "mout_fsys1_bus_user",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_MSTR_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_SLV_ACLK_SOC,
+	     "pcie_top_ipclkport_pcie_sub_con_x2_slv_aclk_soc", "mout_fsys1_bus_user",
+	     CLK_CON_PCIE_TOP_IPCLKPORT_PCIE_SUB_CON_X2_SLV_ACLK_SOC,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_USB20DRD_IPCLKPORT_ACLK_PHYCTRL_20,
+	     "usb20drd_ipclkport_aclk_phyctrl_20", "dout_fsys1_bus_300",
+	     CLK_CON_USB20DRD_IPCLKPORT_ACLK_PHYCTRL_20,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_USB20DRD_IPCLKPORT_BUS_CLK_EARLY, "usb20drd_ipclkport_bus_clk_early",
+	     "dout_fsys1_bus_300", CLK_CON_USB20DRD_IPCLKPORT_BUS_CLK_EARLY,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_XHB_AHBBR_FSYS1_IPCLKPORT_CLK, "xhb_ahbbr_fsys1_ipclkport_clk",
+	     "dout_fsys1_bus_300", CLK_CON_XHB_AHBBR_FSYS1_IPCLKPORT_CLK,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_FSYS1_XHB_USB_IPCLKPORT_CLK, "xhb_usb_ipclkport_clk", "dout_fsys1_bus_300",
+	     CLK_CON_XHB_USB_IPCLKPORT_CLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_mmu_fsys1_qch_u_tbu_0_0", "mout_fsys1_bus_user",
+	     CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_0_0, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_mmu_fsys1_qch_u_tbu_1_0", "mout_fsys1_bus_user",
+	     CLK_CON_QCH_CON_MMU_FSYS1_QCH_U_TBU_1_0, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+};
+
+static const struct samsung_cmu_info cmu_fsys1_info __initconst = {
+	.pll_clks		= cmu_fsys1_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(cmu_fsys1_pll_clks),
+	.mux_clks		= cmu_fsys1_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_fsys1_mux_clks),
+	.div_clks		= cmu_fsys1_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(cmu_fsys1_div_clks),
+	.gate_clks              = cmu_fsys1_gate_clks,
+	.nr_gate_clks           = ARRAY_SIZE(cmu_fsys1_gate_clks),
+	.nr_clk_ids		= CMU_FSYS1_NR_CLK,
+	.clk_regs		= cmu_fsys1_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_fsys1_clk_regs),
+};
+
+/* Register Offset definitions for CMU_IMEM (0x10010000) */
+#define PLL_CON0_MUX_CLK_IMEM_ACLK_USER				0x0100
+#define PLL_CON0_MUX_CLK_IMEM_CA5_USER				0x0120
+#define PLL_CON0_MUX_CLK_IMEM_JPEG_USER				0x0140
+#define PLL_CON0_MUX_CLK_IMEM_SSS_USER				0x0160
+#define CLK_CON_MCT0_IPCLKPORT_PCLK					0x20b4
+#define CLK_CON_MCT1_IPCLKPORT_PCLK					0x20b8
+#define CLK_CON_MCT2_IPCLKPORT_PCLK					0x20bc
+#define CLK_CON_MCT3_IPCLKPORT_PCLK					0x20c0
+#define CLK_CON_TMU_APB_IPCLKPORT_PCLK					0x20d4
+#define CLK_CON_DMYQCH_CON_CA5_0_QCH					0x3008
+#define CLK_CON_DMYQCH_CON_CA5_1_QCH					0x3018
+#define CLK_CON_DMYQCH_CON_INTMEM_QCH					0x3020
+#define CLK_CON_QCH_CON_GIC_CA55_QCHANNEL_SLAVE_0			0x306c
+#define CLK_CON_QCH_CON_GIC_CA5_0_QCH					0x3078
+#define CLK_CON_QCH_CON_GIC_CA5_1_QCH					0x307c
+#define CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_0_0				0x30ac
+#define CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_1_0				0x30b4
+
+static const unsigned long cmu_imem_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLK_IMEM_ACLK_USER,
+	PLL_CON0_MUX_CLK_IMEM_CA5_USER,
+	PLL_CON0_MUX_CLK_IMEM_JPEG_USER,
+	PLL_CON0_MUX_CLK_IMEM_SSS_USER,
+	CLK_CON_MCT0_IPCLKPORT_PCLK,
+	CLK_CON_MCT1_IPCLKPORT_PCLK,
+	CLK_CON_MCT2_IPCLKPORT_PCLK,
+	CLK_CON_MCT3_IPCLKPORT_PCLK,
+	CLK_CON_TMU_APB_IPCLKPORT_PCLK,
+	CLK_CON_DMYQCH_CON_CA5_0_QCH,
+	CLK_CON_DMYQCH_CON_CA5_1_QCH,
+	CLK_CON_DMYQCH_CON_INTMEM_QCH,
+	CLK_CON_QCH_CON_GIC_CA55_QCHANNEL_SLAVE_0,
+	CLK_CON_QCH_CON_GIC_CA5_0_QCH,
+	CLK_CON_QCH_CON_GIC_CA5_1_QCH,
+	CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_0_0,
+	CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_1_0
+};
+
+PNAME(mout_imem_aclk_user_p) = { "fin_pll", "dout_clkcmu_imem_aclk" };
+PNAME(mout_imem_ca5_user_p) = { "fin_pll", "dout_clkcmu_imem_ca5" };
+PNAME(mout_imem_jpeg_user_p) = { "fin_pll", "dout_clkcmu_imem_jpeg" };
+PNAME(mout_imem_sss_user_p) = { "fin_pll", "dout_clkcmu_imem_sss" };
+
+static const struct samsung_mux_clock cmu_imem_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_IMEM_ACLK_USER, "mout_clk_imem_aclk_user",
+	    mout_imem_aclk_user_p, PLL_CON0_MUX_CLK_IMEM_ACLK_USER, 4, 1),
+	MUX(CLK_MOUT_IMEM_CA5_USER, "mout_clk_imem_ca5_user",
+	    mout_imem_ca5_user_p, PLL_CON0_MUX_CLK_IMEM_CA5_USER, 4, 1),
+	MUX(CLK_MOUT_IMEM_SSS_USER, "mout_clk_imem_sss_user",
+	    mout_imem_sss_user_p, PLL_CON0_MUX_CLK_IMEM_SSS_USER, 4, 1),
+	MUX(CLK_MOUT_IMEM_JPEG_USER, "mout_clk_imem_jpeg_user",
+	    mout_imem_jpeg_user_p, PLL_CON0_MUX_CLK_IMEM_JPEG_USER, 4, 1),
+};
+
+static const struct samsung_fixed_factor_clock imem_ffactor_clks[] __initconst = {
+	FFACTOR(CLK_DOUT_IMEM_PCLK, "dout_clk_imem_pclk", "mout_clk_imem_aclk_user", 1, 2, 0),
+};
+
+static const struct samsung_gate_clock cmu_imem_gate_clks[] __initconst = {
+	GATE(CLK_GOUT_IMEM_CA5_0_IPCLKPORT_ATCLK, "ca5_0_ipclkport_atclk",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_IMEM_CA5_0_IPCLKPORT_CLKIN, "ca5_0_ipclkport_clkin",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_IMEM_CA5_0_IPCLKPORT_PCLK_DBG, "ca5_0_ipclkport_pclk_dbg",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_0_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_IMEM_CA5_1_IPCLKPORT_ATCLK, "ca5_1_ipclkport_atclk",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_IMEM_CA5_1_IPCLKPORT_CLKIN, "ca5_1_ipclkport_clkin",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_IMEM_CA5_1_IPCLKPORT_PCLK_DBG, "ca5_1_ipclkport_pclk_dbg",
+	     "mout_clk_imem_ca5_user", CLK_CON_DMYQCH_CON_CA5_1_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(0, "intmem_ipclkport_aclk", "mout_clk_imem_aclk_user",
+	     CLK_CON_DMYQCH_CON_INTMEM_QCH, 1, CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+	GATE(CLK_GOUT_IMEM_MCT0_PCLK, "mct0", "dout_clk_imem_pclk",
+	     CLK_CON_MCT0_IPCLKPORT_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_IMEM_MCT1_PCLK, "mct1", "dout_clk_imem_pclk",
+	     CLK_CON_MCT1_IPCLKPORT_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_IMEM_MCT2_PCLK, "mct2", "dout_clk_imem_pclk",
+	     CLK_CON_MCT2_IPCLKPORT_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_IMEM_MCT3_PCLK, "mct3", "dout_clk_imem_pclk",
+	     CLK_CON_MCT3_IPCLKPORT_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_IMEM_PCLK_TMU0_APBIF, "tmu_apb_ipclkport_pclk", "dout_clk_imem_pclk",
+	     CLK_CON_TMU_APB_IPCLKPORT_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_gic_ca55_qchannel_slave_0", "dout_clk_imem_pclk",
+	     CLK_CON_QCH_CON_GIC_CA55_QCHANNEL_SLAVE_0, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_gic_ca5_0_qch", "dout_clk_imem_pclk",
+	     CLK_CON_QCH_CON_GIC_CA5_0_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_gic_ca5_1_qch", "dout_clk_imem_pclk",
+	     CLK_CON_QCH_CON_GIC_CA5_1_QCH, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_mmu_imem_qch_u_tbu_0_0", "mout_clk_imem_ca5_user",
+	     CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_0_0, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(0, "qch_con_mmu_imem_qch_u_tbu_1_0", "mout_clk_imem_ca5_user",
+	     CLK_CON_QCH_CON_MMU_IMEM_QCH_U_TBU_1_0, 1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+};
+
+static const struct samsung_cmu_info cmu_imem_info __initconst = {
+	.fixed_factor_clks	= imem_ffactor_clks,
+	.nr_fixed_factor_clks	= ARRAY_SIZE(imem_ffactor_clks),
+	.mux_clks		= cmu_imem_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_imem_mux_clks),
+	.gate_clks              = cmu_imem_gate_clks,
+	.nr_gate_clks           = ARRAY_SIZE(cmu_imem_gate_clks),
+	.nr_clk_ids		= CMU_IMEM_NR_CLK,
+	.clk_regs		= cmu_imem_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_imem_clk_regs),
+};
+
+static void __init artpec9_cmu_imem_init(struct device_node *np)
+{
+	exynos_arm64_register_cmu(NULL, np, &cmu_imem_info);
+}
+
+CLK_OF_DECLARE(artpec9_cmu_imem, "axis,artpec9-cmu-imem", artpec9_cmu_imem_init);
+
+/* Register Offset definitions for CMU_PERI (0x14010000) */
+#define PLL_CON0_MUX_CLK_PERI_DISP_USER				0x0100
+#define PLL_CON0_MUX_CLK_PERI_IP_USER				0x0120
+#define CLK_CON_DIV_CLK_PERI_125				0x1800
+#define CLK_CON_DIV_CLK_PERI_PCLK				0x180c
+#define CLK_CON_DIV_CLK_PERI_SPI				0x1810
+#define CLK_CON_DIV_CLK_PERI_UART1				0x1814
+#define CLK_CON_DIV_CLK_PERI_UART2				0x1818
+#define CLK_CON_APB_ASYNC_DSIM_IPCLKPORT_PCLKS				0x2000
+#define CLK_CON_PERI_I2C2_IPCLKPORT_I_PCLK				0x202c
+#define CLK_CON_PERI_I2C3_IPCLKPORT_I_PCLK				0x2030
+#define CLK_CON_PERI_SPI0_IPCLKPORT_I_PCLK				0x2054
+#define CLK_CON_PERI_SPI0_IPCLKPORT_I_SCLK_SPI				0x2058
+#define CLK_CON_PERI_UART1_IPCLKPORT_I_PCLK				0x205c
+#define CLK_CON_PERI_UART1_IPCLKPORT_I_SCLK_UART			0x2060
+#define CLK_CON_PERI_UART2_IPCLKPORT_I_PCLK				0x2064
+#define CLK_CON_PERI_UART2_IPCLKPORT_I_SCLK_UART			0x2068
+#define CLK_CON_DMYQCH_CON_DMA4DSIM_QCH					0x3000
+#define CLK_CON_DMYQCH_CON_PERI_I3C2_QCH				0x3004
+#define CLK_CON_DMYQCH_CON_PERI_I3C3_QCH				0x3008
+
+static const unsigned long cmu_peri_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLK_PERI_DISP_USER,
+	PLL_CON0_MUX_CLK_PERI_IP_USER,
+	CLK_CON_DIV_CLK_PERI_125,
+	CLK_CON_DIV_CLK_PERI_PCLK,
+	CLK_CON_DIV_CLK_PERI_SPI,
+	CLK_CON_DIV_CLK_PERI_UART1,
+	CLK_CON_DIV_CLK_PERI_UART2,
+	CLK_CON_APB_ASYNC_DSIM_IPCLKPORT_PCLKS,
+	CLK_CON_PERI_I2C2_IPCLKPORT_I_PCLK,
+	CLK_CON_PERI_I2C3_IPCLKPORT_I_PCLK,
+	CLK_CON_PERI_SPI0_IPCLKPORT_I_PCLK,
+	CLK_CON_PERI_SPI0_IPCLKPORT_I_SCLK_SPI,
+	CLK_CON_PERI_UART1_IPCLKPORT_I_PCLK,
+	CLK_CON_PERI_UART1_IPCLKPORT_I_SCLK_UART,
+	CLK_CON_PERI_UART2_IPCLKPORT_I_PCLK,
+	CLK_CON_PERI_UART2_IPCLKPORT_I_SCLK_UART,
+	CLK_CON_DMYQCH_CON_DMA4DSIM_QCH,
+	CLK_CON_DMYQCH_CON_PERI_I3C2_QCH,
+	CLK_CON_DMYQCH_CON_PERI_I3C3_QCH,
+};
+
+PNAME(mout_peri_ip_user_p) = { "fin_pll", "dout_clkcmu_peri_ip" };
+PNAME(mout_peri_disp_user_p) = { "fin_pll", "dout_clkcmu_peri_disp" };
+
+static const struct samsung_mux_clock cmu_peri_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_PERI_IP_USER, "mout_peri_ip_user", mout_peri_ip_user_p,
+	    PLL_CON0_MUX_CLK_PERI_IP_USER, 4, 1),
+	MUX(CLK_MOUT_PERI_DISP_USER, "mout_peri_disp_user", mout_peri_disp_user_p,
+	    PLL_CON0_MUX_CLK_PERI_DISP_USER, 4, 1),
+};
+
+static const struct samsung_div_clock cmu_peri_div_clks[] __initconst = {
+	DIV(CLK_DOUT_PERI_125, "dout_peri_125", "mout_peri_ip_user",
+	    CLK_CON_DIV_CLK_PERI_125, 0, 4),
+	DIV(CLK_DOUT_PERI_PCLK, "dout_peri_pclk", "mout_peri_ip_user",
+	    CLK_CON_DIV_CLK_PERI_PCLK, 0, 4),
+	DIV(CLK_DOUT_PERI_SPI, "dout_peri_spi", "mout_peri_ip_user",
+	    CLK_CON_DIV_CLK_PERI_SPI, 0, 13),
+	DIV(CLK_DOUT_PERI_UART1, "dout_peri_uart1", "mout_peri_ip_user",
+	    CLK_CON_DIV_CLK_PERI_UART1, 0, 10),
+	DIV(CLK_DOUT_PERI_UART2, "dout_peri_uart2", "mout_peri_ip_user",
+	    CLK_CON_DIV_CLK_PERI_UART2, 0, 10),
+};
+
+static const struct samsung_gate_clock cmu_peri_gate_clks[] __initconst = {
+	GATE(CLK_GOUT_PERI_DMA4DSIM_IPCLKPORT_CLK_APB_CLK, "dma4dsim_ipclkport_clk_apb_clk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_DMA4DSIM_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_PERI_DMA4DSIM_IPCLKPORT_CLK_AXI_CLK, "dma4dsim_ipclkport_clk_axi_clk",
+	     "mout_peri_disp_user", CLK_CON_DMYQCH_CON_DMA4DSIM_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_PERI_I3C2_IPCLKPORT_I_APB_S_PCLK, "peri_i3c2_ipclkport_i_apb_s_pclk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C2_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I3C2_IPCLKPORT_I_CORE_CLK, "peri_i3c2_ipclkport_i_core_clk",
+	     "dout_peri_125", CLK_CON_DMYQCH_CON_PERI_I3C2_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_PERI_I3C2_IPCLKPORT_I_DMA_CLK, "peri_i3c2_ipclkport_i_dma_clk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C2_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I3C2_IPCLKPORT_I_HDR_TX_CLK, "peri_i3c2_ipclkport_i_hdr_tx_clk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C2_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I3C3_IPCLKPORT_I_APB_S_PCLK, "peri_i3c3_ipclkport_i_apb_s_pclk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C3_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I3C3_IPCLKPORT_I_CORE_CLK, "peri_i3c3_ipclkport_i_core_clk",
+	     "dout_peri_125", CLK_CON_DMYQCH_CON_PERI_I3C3_QCH, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(CLK_GOUT_PERI_I3C3_IPCLKPORT_I_DMA_CLK, "peri_i3c3_ipclkport_i_dma_clk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C3_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I3C3_IPCLKPORT_I_HDR_TX_CLK, "peri_i3c3_ipclkport_i_hdr_tx_clk",
+	     "dout_peri_pclk", CLK_CON_DMYQCH_CON_PERI_I3C3_QCH, 1,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_APB_ASYNC_DSIM_IPCLKPORT_PCLKS, "apb_async_dsim_ipclkport_pclks",
+	     "dout_peri_pclk", CLK_CON_APB_ASYNC_DSIM_IPCLKPORT_PCLKS, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I2C2_IPCLKPORT_I_PCLK, "peri_i2c2_ipclkport_i_pclk",
+	     "dout_peri_pclk", CLK_CON_PERI_I2C2_IPCLKPORT_I_PCLK, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_I2C3_IPCLKPORT_I_PCLK, "peri_i2c3_ipclkport_i_pclk",
+	     "dout_peri_pclk", CLK_CON_PERI_I2C3_IPCLKPORT_I_PCLK, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_SPI0_PCLK, "peri_spi0_ipclkport_i_pclk",
+	     "dout_peri_pclk", CLK_CON_PERI_SPI0_IPCLKPORT_I_PCLK, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_SPI0_SCLK_SPI, "peri_spi0_ipclkport_i_sclk_spi",
+	     "dout_peri_spi", CLK_CON_PERI_SPI0_IPCLKPORT_I_SCLK_SPI, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_UART1_PCLK, "uart1", "dout_peri_pclk",
+	     CLK_CON_PERI_UART1_IPCLKPORT_I_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_UART1_SCLK_UART, "clk_uart_baud1", "dout_peri_uart1",
+	     CLK_CON_PERI_UART1_IPCLKPORT_I_SCLK_UART, 21,
+	     CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_UART2_PCLK, "uart2", "dout_peri_pclk",
+	     CLK_CON_PERI_UART2_IPCLKPORT_I_PCLK, 21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+	GATE(CLK_GOUT_PERI_UART2_SCLK_UART, "clk_uart_baud2", "dout_peri_uart2",
+	     CLK_CON_PERI_UART2_IPCLKPORT_I_SCLK_UART,
+	     21, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
+};
+
+static const struct samsung_cmu_info cmu_peri_info __initconst = {
+	.mux_clks		= cmu_peri_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(cmu_peri_mux_clks),
+	.div_clks		= cmu_peri_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(cmu_peri_div_clks),
+	.gate_clks              = cmu_peri_gate_clks,
+	.nr_gate_clks           = ARRAY_SIZE(cmu_peri_gate_clks),
+	.nr_clk_ids		= CMU_PERI_NR_CLK,
+	.clk_regs		= cmu_peri_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(cmu_peri_clk_regs),
+};
+
+static int __init artpec9_cmu_probe(struct platform_device *pdev)
+{
+	const struct samsung_cmu_info *info;
+	struct device *dev = &pdev->dev;
+
+	info = of_device_get_match_data(dev);
+	exynos_arm64_register_cmu(dev, dev->of_node, info);
+
+	return 0;
+}
+
+static const struct of_device_id artpec9_cmu_of_match[] = {
+	{
+		.compatible = "axis,artpec9-cmu-cmu",
+		.data = &cmu_cmu_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-bus",
+		.data = &cmu_bus_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-core",
+		.data = &cmu_core_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-cpucl",
+		.data = &cmu_cpucl_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-fsys0",
+		.data = &cmu_fsys0_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-fsys1",
+		.data = &cmu_fsys1_info,
+	}, {
+		.compatible = "axis,artpec9-cmu-peri",
+		.data = &cmu_peri_info,
+	}, {
+	},
+};
+
+static struct platform_driver artpec9_cmu_driver __refdata = {
+	.driver = {
+		.name = "artpec9-cmu",
+		.of_match_table = artpec9_cmu_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe = artpec9_cmu_probe,
+};
+
+static int __init artpec9_cmu_init(void)
+{
+	return platform_driver_register(&artpec9_cmu_driver);
+}
+core_initcall(artpec9_cmu_init);
diff --git a/drivers/clk/samsung/clk-exynosautov920.c b/drivers/clk/samsung/clk-exynosautov920.c
index d0617c7fff3a6..04cd40c71d134 100644
--- a/drivers/clk/samsung/clk-exynosautov920.c
+++ b/drivers/clk/samsung/clk-exynosautov920.c
@@ -30,6 +30,7 @@
 #define CLKS_NR_M2M                     (CLK_DOUT_M2M_NOCP + 1)
 #define CLKS_NR_MFC                     (CLK_DOUT_MFC_NOCP + 1)
 #define CLKS_NR_MFD                     (CLK_DOUT_MFD_NOCP + 1)
+#define CLKS_NR_G3D			(CLK_MOUT_G3D_NOCP_USER + 1)
 
 /* ---- CMU_TOP ------------------------------------------------------------ */
 
@@ -1942,6 +1943,54 @@ static const struct samsung_cmu_info mfd_cmu_info __initconst = {
 	.clk_name               = "noc",
 };
 
+/* ---- CMU_G3D --------------------------------------------------------- */
+
+/* Register Offset definitions for CMU_G3D (0x1a000000) */
+#define PLL_LOCKTIME_PLL_G3D                    0x0
+#define PLL_CON3_PLL_G3D                        0x10c
+#define CLK_CON_MUX_MUX_CLK_G3D_NOC             0x1000
+#define PLL_CON0_MUX_CLKCMU_G3D_NOCP_USER       0x600
+#define PLL_CON0_MUX_CLKCMU_G3D_SWITCH_USER     0x610
+
+static const unsigned long g3d_clk_regs[] __initconst = {
+	PLL_LOCKTIME_PLL_G3D,
+	PLL_CON3_PLL_G3D,
+	CLK_CON_MUX_MUX_CLK_G3D_NOC,
+	PLL_CON0_MUX_CLKCMU_G3D_NOCP_USER,
+	PLL_CON0_MUX_CLKCMU_G3D_SWITCH_USER,
+};
+
+static const struct samsung_pll_clock g3d_pll_clks[] __initconst = {
+	/* CMU_G3D_PLL */
+	PLL(pll_531x, FOUT_PLL_G3D, "fout_pll_g3d", "oscclk",
+	    PLL_LOCKTIME_PLL_G3D, PLL_CON3_PLL_G3D, NULL),
+};
+
+/* List of parent clocks for Muxes in CMU_G3D */
+PNAME(mout_clk_g3d_noc_p) = { "oscclk", "fout_pll_g3d", "mout_clkcmu_g3d_switch_user"};
+PNAME(mout_clkcmu_g3d_switch_user_p) = { "oscclk", "dout_clkcmu_g3d_switch" };
+PNAME(mout_clkcmu_g3d_nocp_user_p) = { "oscclk", "dout_clkcmu_g3d_nocp" };
+
+static const struct samsung_mux_clock g3d_mux_clks[] __initconst = {
+	MUX(CLK_MOUT_G3D_NOC, "mout_clk_g3d_noc",
+	    mout_clk_g3d_noc_p, CLK_CON_MUX_MUX_CLK_G3D_NOC, 0, 2),
+	MUX(CLK_MOUT_G3D_SWITCH_USER, "mout_clkcmu_g3d_switch_user",
+	    mout_clkcmu_g3d_switch_user_p, PLL_CON0_MUX_CLKCMU_G3D_SWITCH_USER, 4, 1),
+	MUX(CLK_MOUT_G3D_NOCP_USER, "mout_clkcmu_g3d_nocp_user",
+	    mout_clkcmu_g3d_nocp_user_p, PLL_CON0_MUX_CLKCMU_G3D_NOCP_USER, 4, 1),
+};
+
+static const struct samsung_cmu_info g3d_cmu_info __initconst = {
+	.pll_clks               = g3d_pll_clks,
+	.nr_pll_clks            = ARRAY_SIZE(g3d_pll_clks),
+	.mux_clks               = g3d_mux_clks,
+	.nr_mux_clks            = ARRAY_SIZE(g3d_mux_clks),
+	.nr_clk_ids             = CLKS_NR_G3D,
+	.clk_regs               = g3d_clk_regs,
+	.nr_clk_regs            = ARRAY_SIZE(g3d_clk_regs),
+	.clk_name               = "noc",
+};
+
 static int __init exynosautov920_cmu_probe(struct platform_device *pdev)
 {
 	const struct samsung_cmu_info *info;
@@ -1981,6 +2030,9 @@ static const struct of_device_id exynosautov920_cmu_of_match[] = {
 	}, {
 		.compatible = "samsung,exynosautov920-cmu-mfd",
 		.data = &mfd_cmu_info,
+	}, {
+		.compatible = "samsung,exynosautov920-cmu-g3d",
+		.data = &g3d_cmu_info,
 	},
 	{ }
 };
diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c
index 44a8ecd332fdd..d2bcd3a9daf89 100644
--- a/drivers/clk/samsung/clk-gs101.c
+++ b/drivers/clk/samsung/clk-gs101.c
@@ -339,7 +339,7 @@
 #define GENERALIO_ACD_CHANNEL_3				0x3f0c
 #define GENERALIO_ACD_MASK				0x3f14
 
-static const unsigned long cmu_top_clk_regs[] __initconst = {
+static const unsigned long top_clk_regs[] __initconst = {
 	PLL_LOCKTIME_PLL_SHARED0,
 	PLL_LOCKTIME_PLL_SHARED1,
 	PLL_LOCKTIME_PLL_SHARED2,
@@ -638,7 +638,7 @@ static const unsigned long cmu_top_clk_regs[] __initconst = {
 	GENERALIO_ACD_MASK,
 };
 
-static const struct samsung_pll_clock cmu_top_pll_clks[] __initconst = {
+static const struct samsung_pll_clock top_pll_clks[] __initconst = {
 	/* CMU_TOP_PURECLKCOMP */
 	PLL(pll_0517x, CLK_FOUT_SHARED0_PLL, "fout_shared0_pll", "oscclk",
 	    PLL_LOCKTIME_PLL_SHARED0, PLL_CON3_PLL_SHARED0,
@@ -952,7 +952,7 @@ PNAME(mout_cmu_cmuref_p)	= { "mout_cmu_top_boost_option1",
  * For gates remove _UID _BLK _IPCLKPORT and _RSTNSYNC
  */
 
-static const struct samsung_mux_clock cmu_top_mux_clks[] __initconst = {
+static const struct samsung_mux_clock top_mux_clks[] __initconst = {
 	MUX(CLK_MOUT_PLL_SHARED0, "mout_pll_shared0", mout_pll_shared0_p,
 	    PLL_CON0_PLL_SHARED0, 4, 1),
 	MUX(CLK_MOUT_PLL_SHARED1, "mout_pll_shared1", mout_pll_shared1_p,
@@ -1108,7 +1108,7 @@ static const struct samsung_mux_clock cmu_top_mux_clks[] __initconst = {
 	    CLK_CON_MUX_MUX_CMU_CMUREF, 0, 1),
 };
 
-static const struct samsung_div_clock cmu_top_div_clks[] __initconst = {
+static const struct samsung_div_clock top_div_clks[] __initconst = {
 	DIV(CLK_DOUT_CMU_BO_BUS, "dout_cmu_bo_bus", "gout_cmu_bo_bus",
 	    CLK_CON_DIV_CLKCMU_BO_BUS, 0, 4),
 	DIV(CLK_DOUT_CMU_BUS0_BUS, "dout_cmu_bus0_bus", "gout_cmu_bus0_bus",
@@ -1253,13 +1253,13 @@ static const struct samsung_div_clock cmu_top_div_clks[] __initconst = {
 	    "mout_pll_shared3", CLK_CON_DIV_PLL_SHARED3_DIV2, 0, 1),
 };
 
-static const struct samsung_fixed_factor_clock cmu_top_ffactor[] __initconst = {
+static const struct samsung_fixed_factor_clock top_ffactor_clks[] __initconst = {
 	FFACTOR(CLK_DOUT_CMU_HSI0_USBDPDBG, "dout_cmu_hsi0_usbdpdbg",
 		"gout_cmu_hsi0_usbdpdbg", 1, 4, 0),
 	FFACTOR(CLK_DOUT_CMU_OTP, "dout_cmu_otp", "oscclk", 1, 8, 0),
 };
 
-static const struct samsung_gate_clock cmu_top_gate_clks[] __initconst = {
+static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 	GATE(CLK_GOUT_CMU_BUS0_BOOST, "gout_cmu_bus0_boost",
 	     "mout_cmu_boost_option1", CLK_CON_GAT_CLKCMU_BUS0_BOOST, 21, 0, 0),
 	GATE(CLK_GOUT_CMU_BUS1_BOOST, "gout_cmu_bus1_boost",
@@ -1425,19 +1425,19 @@ static const struct samsung_gate_clock cmu_top_gate_clks[] __initconst = {
 };
 
 static const struct samsung_cmu_info top_cmu_info __initconst = {
-	.pll_clks		= cmu_top_pll_clks,
-	.nr_pll_clks		= ARRAY_SIZE(cmu_top_pll_clks),
-	.mux_clks		= cmu_top_mux_clks,
-	.nr_mux_clks		= ARRAY_SIZE(cmu_top_mux_clks),
-	.div_clks		= cmu_top_div_clks,
-	.nr_div_clks		= ARRAY_SIZE(cmu_top_div_clks),
-	.fixed_factor_clks	= cmu_top_ffactor,
-	.nr_fixed_factor_clks	= ARRAY_SIZE(cmu_top_ffactor),
-	.gate_clks		= cmu_top_gate_clks,
-	.nr_gate_clks		= ARRAY_SIZE(cmu_top_gate_clks),
+	.pll_clks		= top_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(top_pll_clks),
+	.mux_clks		= top_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(top_mux_clks),
+	.div_clks		= top_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(top_div_clks),
+	.fixed_factor_clks	= top_ffactor_clks,
+	.nr_fixed_factor_clks	= ARRAY_SIZE(top_ffactor_clks),
+	.gate_clks		= top_gate_clks,
+	.nr_gate_clks		= ARRAY_SIZE(top_gate_clks),
 	.nr_clk_ids		= CLKS_NR_TOP,
-	.clk_regs		= cmu_top_clk_regs,
-	.nr_clk_regs		= ARRAY_SIZE(cmu_top_clk_regs),
+	.clk_regs		= top_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(top_clk_regs),
 	.auto_clock_gate	= true,
 	.gate_dbg_offset	= GS101_GATE_DBG_OFFSET,
 	.option_offset		= CMU_CMU_TOP_CONTROLLER_OPTION,
@@ -2434,15 +2434,15 @@ PNAME(mout_hsi0_usb31drd_p)		= { "fout_usb_pll",
 					    "dout_hsi0_usb31drd",
 					    "fout_usb_pll" };
 
-static const struct samsung_pll_rate_table cmu_hsi0_usb_pll_rates[] __initconst = {
+static const struct samsung_pll_rate_table hsi0_usb_pll_rates[] __initconst = {
 	PLL_35XX_RATE(24576000, 19200000, 150, 6, 5),
 	{ /* sentinel */ }
 };
 
-static const struct samsung_pll_clock cmu_hsi0_pll_clks[] __initconst = {
+static const struct samsung_pll_clock hsi0_pll_clks[] __initconst = {
 	PLL(pll_0518x, CLK_FOUT_USB_PLL, "fout_usb_pll", "oscclk",
 	    PLL_LOCKTIME_PLL_USB, PLL_CON3_PLL_USB,
-	    cmu_hsi0_usb_pll_rates),
+	    hsi0_usb_pll_rates),
 };
 
 static const struct samsung_mux_clock hsi0_mux_clks[] __initconst = {
@@ -2660,8 +2660,8 @@ static const struct samsung_fixed_rate_clock hsi0_fixed_clks[] __initconst = {
 };
 
 static const struct samsung_cmu_info hsi0_cmu_info __initconst = {
-	.pll_clks		= cmu_hsi0_pll_clks,
-	.nr_pll_clks		= ARRAY_SIZE(cmu_hsi0_pll_clks),
+	.pll_clks		= hsi0_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(hsi0_pll_clks),
 	.mux_clks		= hsi0_mux_clks,
 	.nr_mux_clks		= ARRAY_SIZE(hsi0_mux_clks),
 	.div_clks		= hsi0_div_clks,
@@ -2791,7 +2791,7 @@ static const struct samsung_cmu_info hsi0_cmu_info __initconst = {
 #define QCH_CON_UFS_EMBD_QCH_FMP													0x3094
 #define QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2												0x3c00
 
-static const unsigned long cmu_hsi2_clk_regs[] __initconst = {
+static const unsigned long hsi2_clk_regs[] __initconst = {
 	PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER,
 	PLL_CON1_MUX_CLKCMU_HSI2_BUS_USER,
 	PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER,
@@ -3166,8 +3166,8 @@ static const struct samsung_cmu_info hsi2_cmu_info __initconst = {
 	.gate_clks		= hsi2_gate_clks,
 	.nr_gate_clks		= ARRAY_SIZE(hsi2_gate_clks),
 	.nr_clk_ids		= CLKS_NR_HSI2,
-	.clk_regs		= cmu_hsi2_clk_regs,
-	.nr_clk_regs		= ARRAY_SIZE(cmu_hsi2_clk_regs),
+	.clk_regs		= hsi2_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(hsi2_clk_regs),
 	.sysreg_clk_regs	= dcrg_memclk_sysreg,
 	.nr_sysreg_clk_regs	= ARRAY_SIZE(dcrg_memclk_sysreg),
 	.clk_name		= "bus",
diff --git a/drivers/clk/samsung/clk-pll.c b/drivers/clk/samsung/clk-pll.c
index 026ac556fa2e3..fdb84bcec9127 100644
--- a/drivers/clk/samsung/clk-pll.c
+++ b/drivers/clk/samsung/clk-pll.c
@@ -201,6 +201,9 @@ static const struct clk_ops samsung_pll3000_clk_ops = {
 #define PLL35XX_LOCK_STAT_SHIFT	(29)
 #define PLL35XX_ENABLE_SHIFT	(31)
 
+/* A9FRACM is similar to PLL35xx, except that MDIV is bit different */
+#define PLLA9FRACM_MDIV_SHIFT	(14)
+
 static unsigned long samsung_pll35xx_recalc_rate(struct clk_hw *hw,
 				unsigned long parent_rate)
 {
@@ -209,7 +212,12 @@ static unsigned long samsung_pll35xx_recalc_rate(struct clk_hw *hw,
 	u64 fvco = parent_rate;
 
 	pll_con = readl_relaxed(pll->con_reg);
-	mdiv = (pll_con >> PLL35XX_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
+
+	if (pll->type == pll_a9fracm)
+		mdiv = (pll_con >> PLLA9FRACM_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
+	else
+		mdiv = (pll_con >> PLL35XX_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
+
 	pdiv = (pll_con >> PLL35XX_PDIV_SHIFT) & PLL35XX_PDIV_MASK;
 	sdiv = (pll_con >> PLL35XX_SDIV_SHIFT) & PLL35XX_SDIV_MASK;
 
@@ -219,12 +227,15 @@ static unsigned long samsung_pll35xx_recalc_rate(struct clk_hw *hw,
 	return (unsigned long)fvco;
 }
 
-static inline bool samsung_pll35xx_mp_change(
-		const struct samsung_pll_rate_table *rate, u32 pll_con)
+static inline bool samsung_pll35xx_mp_change(u32 pll_type,
+					     const struct samsung_pll_rate_table *rate, u32 pll_con)
 {
 	u32 old_mdiv, old_pdiv;
 
-	old_mdiv = (pll_con >> PLL35XX_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
+	if (pll_type == pll_a9fracm)
+		old_mdiv = (pll_con >> PLLA9FRACM_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
+	else
+		old_mdiv = (pll_con >> PLL35XX_MDIV_SHIFT) & PLL35XX_MDIV_MASK;
 	old_pdiv = (pll_con >> PLL35XX_PDIV_SHIFT) & PLL35XX_PDIV_MASK;
 
 	return (rate->mdiv != old_mdiv || rate->pdiv != old_pdiv);
@@ -236,6 +247,12 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate,
 	struct samsung_clk_pll *pll = to_clk_pll(hw);
 	const struct samsung_pll_rate_table *rate;
 	u32 tmp;
+	u32 mdiv_shift;
+
+	if (pll->type == pll_a9fracm)
+		mdiv_shift = PLLA9FRACM_MDIV_SHIFT;
+	else
+		mdiv_shift = PLL35XX_MDIV_SHIFT;
 
 	/* Get required rate settings from table */
 	rate = samsung_get_pll_settings(pll, drate);
@@ -247,7 +264,7 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate,
 
 	tmp = readl_relaxed(pll->con_reg);
 
-	if (!(samsung_pll35xx_mp_change(rate, tmp))) {
+	if (!(samsung_pll35xx_mp_change(pll->type, rate, tmp))) {
 		/* If only s change, change just s value only*/
 		tmp &= ~(PLL35XX_SDIV_MASK << PLL35XX_SDIV_SHIFT);
 		tmp |= rate->sdiv << PLL35XX_SDIV_SHIFT;
@@ -257,7 +274,7 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate,
 	}
 
 	/* Set PLL lock time. */
-	if (pll->type == pll_142xx || pll->type == pll_1017x)
+	if (pll->type == pll_142xx || pll->type == pll_1017x || pll->type == pll_a9fracm)
 		writel_relaxed(rate->pdiv * PLL142XX_LOCK_FACTOR,
 			pll->lock_reg);
 	else
@@ -265,10 +282,10 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate,
 			pll->lock_reg);
 
 	/* Change PLL PMS values */
-	tmp &= ~((PLL35XX_MDIV_MASK << PLL35XX_MDIV_SHIFT) |
+	tmp &= ~((PLL35XX_MDIV_MASK << mdiv_shift) |
 			(PLL35XX_PDIV_MASK << PLL35XX_PDIV_SHIFT) |
 			(PLL35XX_SDIV_MASK << PLL35XX_SDIV_SHIFT));
-	tmp |= (rate->mdiv << PLL35XX_MDIV_SHIFT) |
+	tmp |= (rate->mdiv << mdiv_shift) |
 			(rate->pdiv << PLL35XX_PDIV_SHIFT) |
 			(rate->sdiv << PLL35XX_SDIV_SHIFT);
 	writel_relaxed(tmp, pll->con_reg);
@@ -1428,6 +1445,149 @@ static const struct clk_ops samsung_pll1031x_clk_min_ops = {
 	.recalc_rate = samsung_pll1031x_recalc_rate,
 };
 
+/*
+ * PLLA9FRACO Clock Type
+ */
+#define PLLA9FRACO_LOCK_FACTOR		(500)
+
+#define PLLA9FRACO_MDIV_MASK		(0x3ff)
+#define PLLA9FRACO_PDIV_MASK		(0x3f)
+#define PLLA9FRACO_SDIV_MASK		(0x7)
+#define PLLA9FRACO_MDIV_SHIFT		(14)
+#define PLLA9FRACO_PDIV_SHIFT		(8)
+#define PLLA9FRACO_SDIV_SHIFT		(0)
+
+#define PLLA9FRACO_PLL_CON5_DIV_FRAC	(0x14)
+#define PLLA9FRACO_KDIV_MASK		(0xffffff)
+#define PLLA9FRACO_KDIV_SHIFT		(0)
+#define PLLA9FRACO_DAC_MODE		BIT(30)
+#define PLLA9FRACO_DSM_EN		BIT(31)
+#define PLLA9FRACO_FOUTPOSTDIVEN	BIT(3)
+#define PLLA9FRACO_MUX_SEL		BIT(4)
+#define PLLA9FRACO_ENABLE_SHIFT		(31)
+#define PLLA9FRACO_LOCK_STAT_SHIFT	(29)
+
+static unsigned long samsung_a9fraco_recalc_rate(struct clk_hw *hw,
+						 unsigned long parent_rate)
+{
+	struct samsung_clk_pll *pll = to_clk_pll(hw);
+	u32 pll_con0, pll_con5;
+	u64 mdiv, pdiv, sdiv, kdiv;
+	u64 fvco = parent_rate;
+
+	pll_con0 = readl_relaxed(pll->con_reg);
+	pll_con5 = readl_relaxed(pll->con_reg + PLLA9FRACO_PLL_CON5_DIV_FRAC);
+	mdiv = (pll_con0 >> PLLA9FRACO_MDIV_SHIFT) & PLLA9FRACO_MDIV_MASK;
+	pdiv = (pll_con0 >> PLLA9FRACO_PDIV_SHIFT) & PLLA9FRACO_PDIV_MASK;
+	sdiv = (pll_con0 >> PLLA9FRACO_SDIV_SHIFT) & PLLA9FRACO_SDIV_MASK;
+	kdiv = (pll_con5 & PLLA9FRACO_KDIV_MASK);
+
+	/* fvco = fref * (M + K/2^24) / p * (S+1) */
+	fvco *= mdiv;
+	fvco = (fvco << 24) + kdiv;
+	fvco = div64_u64(fvco, ((pdiv * (sdiv + 1)) << 24));
+
+	return (unsigned long)fvco;
+}
+
+static bool samsung_a9fraco_mpk_change(u32 pll_con0, u32 pll_con5,
+				       const struct samsung_pll_rate_table *rate)
+{
+	u32 old_mdiv, old_pdiv, old_kdiv;
+
+	old_mdiv = (pll_con0 >> PLLA9FRACO_MDIV_SHIFT) & PLLA9FRACO_MDIV_MASK;
+	old_pdiv = (pll_con0 >> PLLA9FRACO_PDIV_SHIFT) & PLLA9FRACO_PDIV_MASK;
+	old_kdiv = (pll_con5 >> PLLA9FRACO_KDIV_SHIFT) & PLLA9FRACO_KDIV_MASK;
+
+	return (old_mdiv != rate->mdiv || old_pdiv != rate->pdiv || old_kdiv != rate->kdiv);
+}
+
+static int samsung_a9fraco_set_rate(struct clk_hw *hw, unsigned long drate, unsigned long prate)
+{
+	struct samsung_clk_pll *pll = to_clk_pll(hw);
+	const struct samsung_pll_rate_table *rate;
+	u32 con0, con5;
+	int ret;
+
+	/* Get required rate settings from table */
+	rate = samsung_get_pll_settings(pll, drate);
+	if (!rate) {
+		pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__,
+		       drate, clk_hw_get_name(hw));
+		return -EINVAL;
+	}
+
+	con0 = readl_relaxed(pll->con_reg);
+	con5 = readl_relaxed(pll->con_reg + PLLA9FRACO_PLL_CON5_DIV_FRAC);
+
+	if (!(samsung_a9fraco_mpk_change(con0, con5, rate))) {
+		/* If only s change, change just s value only */
+		con0 &= ~(PLLA9FRACO_SDIV_MASK << PLLA9FRACO_SDIV_SHIFT);
+		con0 |= rate->sdiv << PLLA9FRACO_SDIV_SHIFT;
+		writel_relaxed(con0, pll->con_reg);
+
+		return 0;
+	}
+
+	/* Select OSCCLK (0) */
+	con0 = readl_relaxed(pll->con_reg);
+	con0 &= ~(PLLA9FRACO_MUX_SEL);
+	writel_relaxed(con0, pll->con_reg);
+
+	/* Disable PLL */
+	con0 &= ~BIT(PLLA9FRACO_ENABLE_SHIFT);
+	writel_relaxed(con0, pll->con_reg);
+
+	/* Set PLL lock time. */
+	writel_relaxed(rate->pdiv * PLLA9FRACO_LOCK_FACTOR, pll->lock_reg);
+
+	/* Set PLL M, P, and S values. */
+	con0 &= ~((PLLA9FRACO_MDIV_MASK << PLLA9FRACO_MDIV_SHIFT) |
+		  (PLLA9FRACO_PDIV_MASK << PLLA9FRACO_PDIV_SHIFT) |
+		  (PLLA9FRACO_SDIV_MASK << PLLA9FRACO_SDIV_SHIFT));
+
+	/* The field FOUTPOSTDIVEN should always be 1, else FOUT might be 0 Hz. */
+	con0 |= (rate->mdiv << PLLA9FRACO_MDIV_SHIFT) |
+		(rate->pdiv << PLLA9FRACO_PDIV_SHIFT) |
+		(rate->sdiv << PLLA9FRACO_SDIV_SHIFT) | (PLLA9FRACO_FOUTPOSTDIVEN);
+
+	/* Set PLL K, DSM_EN and DAC_MODE values. */
+	con5 = readl_relaxed(pll->con_reg + PLLA9FRACO_PLL_CON5_DIV_FRAC);
+	con5 &= ~((PLLA9FRACO_KDIV_MASK << PLLA9FRACO_KDIV_SHIFT) |
+		  PLLA9FRACO_DSM_EN | PLLA9FRACO_DAC_MODE);
+	con5 |= (rate->kdiv << PLLA9FRACO_KDIV_SHIFT) | PLLA9FRACO_DSM_EN | PLLA9FRACO_DAC_MODE;
+
+	/* Write configuration to PLL */
+	writel_relaxed(con0, pll->con_reg);
+	writel_relaxed(con5, pll->con_reg + PLLA9FRACO_PLL_CON5_DIV_FRAC);
+
+	/* Enable PLL */
+	con0 = readl_relaxed(pll->con_reg);
+	con0 |= BIT(PLLA9FRACO_ENABLE_SHIFT);
+	writel_relaxed(con0, pll->con_reg);
+
+	/* Wait for PLL lock if the PLL is enabled */
+	ret = samsung_pll_lock_wait(pll, BIT(pll->lock_offs));
+	if (ret < 0)
+		return ret;
+
+	/* Select FOUT (1) */
+	con0 |= (PLLA9FRACO_MUX_SEL);
+	writel_relaxed(con0, pll->con_reg);
+
+	return 0;
+}
+
+static const struct clk_ops samsung_a9fraco_clk_ops = {
+	.recalc_rate = samsung_a9fraco_recalc_rate,
+	.determine_rate = samsung_pll_determine_rate,
+	.set_rate = samsung_a9fraco_set_rate,
+};
+
+static const struct clk_ops samsung_a9fraco_clk_min_ops = {
+	.recalc_rate = samsung_a9fraco_recalc_rate,
+};
+
 static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 				const struct samsung_pll_clock *pll_clk)
 {
@@ -1477,6 +1637,7 @@ static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 	case pll_1452x:
 	case pll_142xx:
 	case pll_1017x:
+	case pll_a9fracm:
 		pll->enable_offs = PLL35XX_ENABLE_SHIFT;
 		pll->lock_offs = PLL35XX_LOCK_STAT_SHIFT;
 		if (!pll->rate_table)
@@ -1578,6 +1739,14 @@ static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 		else
 			init.ops = &samsung_pll1031x_clk_ops;
 		break;
+	case pll_a9fraco:
+		pll->enable_offs = PLLA9FRACO_ENABLE_SHIFT;
+		pll->lock_offs = PLLA9FRACO_LOCK_STAT_SHIFT;
+		if (!pll->rate_table)
+			init.ops = &samsung_a9fraco_clk_min_ops;
+		else
+			init.ops = &samsung_a9fraco_clk_ops;
+		break;
 	default:
 		pr_warn("%s: Unknown pll type for pll clk %s\n",
 			__func__, pll_clk->name);
diff --git a/drivers/clk/samsung/clk-pll.h b/drivers/clk/samsung/clk-pll.h
index 6c8bb7f26da54..d6eb3246611b8 100644
--- a/drivers/clk/samsung/clk-pll.h
+++ b/drivers/clk/samsung/clk-pll.h
@@ -51,6 +51,8 @@ enum samsung_pll_type {
 	pll_4311,
 	pll_1017x,
 	pll_1031x,
+	pll_a9fracm,
+	pll_a9fraco,
 };
 
 #define PLL_RATE(_fin, _m, _p, _s, _k, _ks) \
@@ -58,6 +60,11 @@ enum samsung_pll_type {
 #define PLL_VALID_RATE(_fin, _fout, _m, _p, _s, _k, _ks) ((_fout) + \
 	BUILD_BUG_ON_ZERO(PLL_RATE(_fin, _m, _p, _s, _k, _ks) != (_fout)))
 
+#define PLL_FRACO_RATE(_fin, _m, _p, _s, _k, _ks) \
+	((u64)(_fin) * (BIT(_ks) * (_m) + (_k)) / BIT(_ks) / ((_p) * ((_s) + 1)))
+#define PLL_FRACO_VALID_RATE(_fin, _fout, _m, _p, _s, _k, _ks) ((_fout) + \
+	BUILD_BUG_ON_ZERO(PLL_FRACO_RATE(_fin, _m, _p, _s, _k, _ks) != (_fout)))
+
 #define PLL_35XX_RATE(_fin, _rate, _m, _p, _s)			\
 	{							\
 		.rate	=	PLL_VALID_RATE(_fin, _rate,	\
@@ -111,6 +118,16 @@ enum samsung_pll_type {
 		.vsel	=	(_vsel),			\
 	}
 
+#define PLL_A9FRACO_RATE(_fin, _rate, _m, _p, _s, _k)		\
+	{							\
+		.rate	=	PLL_FRACO_VALID_RATE(_fin, _rate, \
+				_m, _p, _s, _k, 24),		\
+		.mdiv	=	(_m),				\
+		.pdiv	=	(_p),				\
+		.sdiv	=	(_s),				\
+		.kdiv	=	(_k),				\
+	}
+
 /* NOTE: Rate table should be kept sorted in descending order. */
 
 struct samsung_pll_rate_table {
diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c
index 94b2bccc7d02f..91e5cdbc79d76 100644
--- a/drivers/clk/samsung/clk.c
+++ b/drivers/clk/samsung/clk.c
@@ -359,8 +359,8 @@ void __init samsung_clk_register_gate(struct samsung_clk_provider *ctx,
 				ctx->reg_base + list->offset, list->bit_idx,
 				list->gate_flags, &ctx->lock);
 		if (IS_ERR(clk_hw)) {
-			pr_err("%s: failed to register clock %s: %ld\n", __func__,
-				list->name, PTR_ERR(clk_hw));
+			pr_err("%s: failed to register clock %s: %pe\n", __func__,
+			       list->name, clk_hw);
 			continue;
 		}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
index 0339c4af0fe5b..db0e36d8838e7 100644
--- a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
+++ b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
@@ -83,9 +83,22 @@ static SUNXI_CCU_MUX_DATA_WITH_GATE(r_pwmctrl_clk, "r-pwmctrl",
 static SUNXI_CCU_GATE_HW(bus_r_pwmctrl_clk, "bus-r-pwmctrl",
 			 &r_apb0_clk.common.hw, 0x13c, BIT(0), 0);
 
-/* SPI clock is /M/N (same as new MMC?) */
+static const struct clk_parent_data r_spi_parents[] = {
+	{ .fw_name = "hosc" },
+	{ .fw_name = "pll-periph" },
+	{ .name = "pll-periph0-300M" },
+	{ .name = "pll-periph1-300M" },
+	{ .name = "pll-audio" },
+};
+static SUNXI_CCU_DUALDIV_MUX_GATE(r_spi_clk, "r-spi", r_spi_parents, 0x150,
+				  0, 5,		/* M */
+				  8, 5,		/* P */
+				  24, 3,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
 static SUNXI_CCU_GATE_HW(bus_r_spi_clk, "bus-r-spi",
 			 &r_ahb_clk.common.hw, 0x15c, BIT(0), 0);
+
 static SUNXI_CCU_GATE_HW(bus_r_spinlock_clk, "bus-r-spinlock",
 			 &r_ahb_clk.common.hw, 0x16c, BIT(0), 0);
 static SUNXI_CCU_GATE_HW(bus_r_msgbox_clk, "bus-r-msgbox",
@@ -138,6 +151,7 @@ static struct ccu_common *sun55i_a523_r_ccu_clks[] = {
 	&bus_r_twd_clk.common,
 	&r_pwmctrl_clk.common,
 	&bus_r_pwmctrl_clk.common,
+	&r_spi_clk.common,
 	&bus_r_spi_clk.common,
 	&bus_r_spinlock_clk.common,
 	&bus_r_msgbox_clk.common,
@@ -169,6 +183,7 @@ static struct clk_hw_onecell_data sun55i_a523_r_hw_clks = {
 		[CLK_BUS_R_TWD]		= &bus_r_twd_clk.common.hw,
 		[CLK_R_PWMCTRL]		= &r_pwmctrl_clk.common.hw,
 		[CLK_BUS_R_PWMCTRL]	= &bus_r_pwmctrl_clk.common.hw,
+		[CLK_R_SPI]		= &r_spi_clk.common.hw,
 		[CLK_BUS_R_SPI]		= &bus_r_spi_clk.common.hw,
 		[CLK_BUS_R_SPINLOCK]	= &bus_r_spinlock_clk.common.hw,
 		[CLK_BUS_R_MSGBOX]	= &bus_r_msgbox_clk.common.hw,
diff --git a/drivers/clk/tenstorrent/Kconfig b/drivers/clk/tenstorrent/Kconfig
new file mode 100644
index 0000000000000..9d4391eeeae0f
--- /dev/null
+++ b/drivers/clk/tenstorrent/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config TENSTORRENT_ATLANTIS_PRCM
+	tristate "Support for Tenstorrent Atlantis PRCM Clock Controller"
+	depends on ARCH_TENSTORRENT || COMPILE_TEST
+	default ARCH_TENSTORRENT
+	select REGMAP_MMIO
+	select AUXILIARY_BUS
+	select MFD_SYSCON
+	help
+	  Say yes here to support the different clock
+	  controllers found in the Tenstorrent Atlantis SoC.
+	  This includes the clocks from the RCPU, HSIO, MMIO
+	  and PCIE domain.
diff --git a/drivers/clk/tenstorrent/Makefile b/drivers/clk/tenstorrent/Makefile
new file mode 100644
index 0000000000000..95d87bac7bf58
--- /dev/null
+++ b/drivers/clk/tenstorrent/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_TENSTORRENT_ATLANTIS_PRCM)	+= atlantis-prcm.o
diff --git a/drivers/clk/tenstorrent/atlantis-prcm.c b/drivers/clk/tenstorrent/atlantis-prcm.c
new file mode 100644
index 0000000000000..6d4386eeb7dae
--- /dev/null
+++ b/drivers/clk/tenstorrent/atlantis-prcm.c
@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Tenstorrent Atlantis PRCM Clock Driver
+ *
+ * Copyright (c) 2026 Tenstorrent
+ */
+
+#include <dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+/* RCPU Clock Register Offsets */
+#define PLL_RCPU_CFG_REG	0x0000
+#define PLL_NOCC_CFG_REG	0x0004
+#define NOCC_CLK_CFG_REG	0x0008
+#define RCPU_DIV_CFG_REG	0x000C
+#define RCPU_BLK_CG_REG		0x0014
+#define LSIO_BLK_CG_REG		0x0018
+#define PLL_RCPU_EN_REG		0x011C
+#define PLL_NOCC_EN_REG		0x0120
+#define BUS_CG_REG		0x01FC
+
+/* PLL Bit Definitions */
+#define PLL_CFG_EN_BIT		BIT(0)
+#define PLL_CFG_BYPASS_BIT	BIT(1)
+#define PLL_CFG_REFDIV_MASK	GENMASK(7, 2)
+#define PLL_CFG_REFDIV_SHIFT	2
+#define PLL_CFG_POSTDIV1_MASK	GENMASK(10, 8)
+#define PLL_CFG_POSTDIV1_SHIFT	8
+#define PLL_CFG_POSTDIV2_MASK	GENMASK(13, 11)
+#define PLL_CFG_POSTDIV2_SHIFT	11
+#define PLL_CFG_FBDIV_MASK	GENMASK(25, 14)
+#define PLL_CFG_FBDIV_SHIFT	14
+#define PLL_CFG_LKDT_BIT	BIT(30)
+#define PLL_CFG_LOCK_BIT	BIT(31)
+#define PLL_LOCK_TIMEOUT_US	1000
+#define PLL_BYPASS_WAIT_US	500
+
+struct atlantis_clk_common {
+	int clkid;
+	struct regmap *regmap;
+	struct clk_hw hw;
+};
+
+static inline struct atlantis_clk_common *
+hw_to_atlantis_clk_common(struct clk_hw *hw)
+{
+	return container_of(hw, struct atlantis_clk_common, hw);
+}
+
+struct atlantis_clk_mux_config {
+	u8 shift;
+	u8 width;
+	u32 reg_offset;
+};
+
+struct atlantis_clk_mux {
+	struct atlantis_clk_common common;
+	struct atlantis_clk_mux_config config;
+};
+
+struct atlantis_clk_gate_config {
+	u32 reg_offset;
+	u32 enable;
+};
+
+struct atlantis_clk_gate {
+	struct atlantis_clk_common common;
+	struct atlantis_clk_gate_config config;
+};
+
+struct atlantis_clk_divider_config {
+	u8 shift;
+	u8 width;
+	u32 flags;
+	u32 reg_offset;
+};
+
+struct atlantis_clk_divider {
+	struct atlantis_clk_common common;
+	struct atlantis_clk_divider_config config;
+};
+
+struct atlantis_clk_pll_config {
+	u32 tbl_num;
+	u32 reg_offset;
+	u32 en_reg_offset;
+	u32 cg_reg_offset;
+	u32 cg_reg_enable;
+};
+
+/* Models a PLL with Bypass Functionality and Enable Bit + an optional Gate Clock at it's output */
+struct atlantis_clk_pll {
+	struct atlantis_clk_common common;
+	struct atlantis_clk_pll_config config;
+};
+
+struct atlantis_clk_gate_shared_config {
+	u32 reg_offset;
+	u32 enable;
+	unsigned int *share_count;
+	spinlock_t *refcount_lock;
+};
+
+struct atlantis_clk_gate_shared {
+	struct atlantis_clk_common common;
+	struct atlantis_clk_gate_shared_config config;
+};
+
+struct atlantis_clk_fixed_factor_config {
+	unsigned int mult;
+	unsigned int div;
+};
+
+struct atlantis_clk_fixed_factor {
+	struct atlantis_clk_fixed_factor_config config;
+	struct atlantis_clk_common common;
+};
+
+static inline struct atlantis_clk_mux *hw_to_atlantis_clk_mux(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_mux, common);
+}
+
+static inline struct atlantis_clk_gate *
+hw_to_atlantis_clk_gate(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_gate, common);
+}
+
+static inline struct atlantis_clk_divider *
+hw_to_atlantis_clk_divider(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_divider, common);
+}
+
+static inline struct atlantis_clk_pll *hw_to_atlantis_pll(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_pll, common);
+}
+
+static inline struct atlantis_clk_gate_shared *
+hw_to_atlantis_clk_gate_shared(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_gate_shared, common);
+}
+
+static inline struct atlantis_clk_fixed_factor *
+hw_to_atlantis_clk_fixed_factor(struct clk_hw *hw)
+{
+	struct atlantis_clk_common *common = hw_to_atlantis_clk_common(hw);
+
+	return container_of(common, struct atlantis_clk_fixed_factor, common);
+}
+
+static u8 atlantis_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct atlantis_clk_mux *mux = hw_to_atlantis_clk_mux(hw);
+	u32 val;
+
+	regmap_read(mux->common.regmap, mux->config.reg_offset, &val);
+	val >>= mux->config.shift;
+	val &= (BIT(mux->config.width) - 1);
+
+	return val;
+}
+
+static int atlantis_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct atlantis_clk_mux *mux = hw_to_atlantis_clk_mux(hw);
+	u32 val = index;
+
+	return regmap_update_bits(mux->common.regmap, mux->config.reg_offset,
+				  (BIT(mux->config.width) - 1) << mux->config.shift,
+				  val << mux->config.shift);
+}
+
+static int atlantis_clk_mux_determine_rate(struct clk_hw *hw,
+					   struct clk_rate_request *req)
+{
+	return clk_mux_determine_rate_flags(hw, req, hw->init->flags);
+}
+
+static const struct clk_ops atlantis_clk_mux_ops = {
+	.get_parent = atlantis_clk_mux_get_parent,
+	.set_parent = atlantis_clk_mux_set_parent,
+	.determine_rate = atlantis_clk_mux_determine_rate,
+};
+
+static int atlantis_clk_gate_endisable(struct clk_hw *hw, int enable)
+{
+	struct atlantis_clk_gate *gate = hw_to_atlantis_clk_gate(hw);
+
+	if (enable)
+		return regmap_set_bits(gate->common.regmap,
+				       gate->config.reg_offset,
+				       gate->config.enable);
+	else
+		return regmap_clear_bits(gate->common.regmap,
+					 gate->config.reg_offset,
+					 gate->config.enable);
+}
+
+static int atlantis_clk_gate_enable(struct clk_hw *hw)
+{
+	return atlantis_clk_gate_endisable(hw, 1);
+}
+
+static void atlantis_clk_gate_disable(struct clk_hw *hw)
+{
+	atlantis_clk_gate_endisable(hw, 0);
+}
+
+static int atlantis_clk_gate_is_enabled(struct clk_hw *hw)
+{
+	struct atlantis_clk_gate *gate = hw_to_atlantis_clk_gate(hw);
+
+	return regmap_test_bits(gate->common.regmap, gate->config.reg_offset, gate->config.enable);
+}
+
+static const struct clk_ops atlantis_clk_gate_ops = {
+	.enable = atlantis_clk_gate_enable,
+	.disable = atlantis_clk_gate_disable,
+	.is_enabled = atlantis_clk_gate_is_enabled,
+};
+
+static unsigned long atlantis_clk_divider_recalc_rate(struct clk_hw *hw,
+						      unsigned long parent_rate)
+{
+	struct atlantis_clk_divider *divider = hw_to_atlantis_clk_divider(hw);
+	u32 val;
+
+	regmap_read(divider->common.regmap, divider->config.reg_offset, &val);
+
+	val >>= divider->config.shift;
+	val &= ((1 << (divider->config.width)) - 1);
+
+	return DIV_ROUND_UP_ULL((u64)parent_rate, val + 1);
+}
+
+static const struct clk_ops atlantis_clk_divider_ops = {
+	.recalc_rate = atlantis_clk_divider_recalc_rate,
+};
+
+static unsigned long
+atlantis_clk_fixed_factor_recalc_rate(struct clk_hw *hw,
+				      unsigned long parent_rate)
+{
+	struct atlantis_clk_fixed_factor *factor =
+		hw_to_atlantis_clk_fixed_factor(hw);
+	unsigned long long rate;
+
+	rate = (unsigned long long)parent_rate * factor->config.mult;
+	do_div(rate, factor->config.div);
+
+	return (unsigned long)rate;
+}
+
+static const struct clk_ops atlantis_clk_fixed_factor_ops = {
+	.recalc_rate = atlantis_clk_fixed_factor_recalc_rate,
+};
+
+static int atlantis_clk_pll_is_enabled(struct clk_hw *hw)
+{
+	struct atlantis_clk_pll *pll = hw_to_atlantis_pll(hw);
+	u32 val, en_val, cg_val;
+
+	regmap_read(pll->common.regmap, pll->config.reg_offset, &val);
+	regmap_read(pll->common.regmap, pll->config.en_reg_offset, &en_val);
+	regmap_read(pll->common.regmap, pll->config.cg_reg_offset, &cg_val);
+
+	/* Check if PLL is powered on, locked, not bypassed and Gate clk is enabled */
+	return !!(en_val & PLL_CFG_EN_BIT) && !!(val & PLL_CFG_LOCK_BIT) &&
+	       (!pll->config.cg_reg_enable || (cg_val & pll->config.cg_reg_enable)) &&
+	       !(val & PLL_CFG_BYPASS_BIT);
+}
+
+static int atlantis_clk_pll_enable(struct clk_hw *hw)
+{
+	struct atlantis_clk_pll *pll = hw_to_atlantis_pll(hw);
+	u32 val, en_val, cg_val;
+	int ret;
+
+	regmap_read(pll->common.regmap, pll->config.reg_offset, &val);
+	regmap_read(pll->common.regmap, pll->config.en_reg_offset, &en_val);
+	regmap_read(pll->common.regmap, pll->config.cg_reg_offset, &cg_val);
+
+	/* Check if PLL is already enabled, locked, not bypassed and Gate clk is enabled */
+	if ((en_val & PLL_CFG_EN_BIT) && (val & PLL_CFG_LOCK_BIT) &&
+	    (!pll->config.cg_reg_enable || (cg_val & pll->config.cg_reg_enable)) &&
+	    !(val & PLL_CFG_BYPASS_BIT)) {
+		return 0;
+	}
+
+	/* Step 1: Set bypass mode first */
+	regmap_update_bits(pll->common.regmap, pll->config.reg_offset,
+			   PLL_CFG_BYPASS_BIT, PLL_CFG_BYPASS_BIT);
+
+	/* Step 2: Enable PLL (clear then set power bit) */
+	regmap_update_bits(pll->common.regmap, pll->config.en_reg_offset,
+			   PLL_CFG_EN_BIT, 0);
+
+	regmap_update_bits(pll->common.regmap, pll->config.en_reg_offset,
+			   PLL_CFG_EN_BIT, PLL_CFG_EN_BIT);
+
+	/* Step 3: Wait for PLL lock */
+	ret = regmap_read_poll_timeout(pll->common.regmap,
+				       pll->config.reg_offset, val,
+				       val & PLL_CFG_LOCK_BIT,
+				       PLL_BYPASS_WAIT_US, PLL_LOCK_TIMEOUT_US);
+	if (ret) {
+		pr_err("PLL failed to lock within timeout\n");
+		return ret;
+	}
+
+	/* Step 4: Switch from bypass to PLL output */
+	regmap_update_bits(pll->common.regmap, pll->config.reg_offset,
+			   PLL_CFG_BYPASS_BIT, 0);
+
+	/* Enable Gate clk at PLL Output */
+	return regmap_update_bits(pll->common.regmap, pll->config.cg_reg_offset,
+				  pll->config.cg_reg_enable,
+				  pll->config.cg_reg_enable);
+}
+
+static void atlantis_clk_pll_disable(struct clk_hw *hw)
+{
+	struct atlantis_clk_pll *pll = hw_to_atlantis_pll(hw);
+
+	/* Step 1: Switch to bypass mode before disabling */
+	regmap_update_bits(pll->common.regmap, pll->config.reg_offset,
+			   PLL_CFG_BYPASS_BIT, PLL_CFG_BYPASS_BIT);
+	/* Step 2: Power down PLL */
+	regmap_update_bits(pll->common.regmap, pll->config.en_reg_offset,
+			   PLL_CFG_EN_BIT, 0);
+}
+
+static unsigned long atlantis_clk_pll_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct atlantis_clk_pll *pll = hw_to_atlantis_pll(hw);
+
+	u32 val, refdiv, fbdiv, postdiv1, postdiv2;
+	u64 fout;
+
+	regmap_read(pll->common.regmap, pll->config.reg_offset, &val);
+
+	if (val & PLL_CFG_BYPASS_BIT)
+		return parent_rate;
+
+	refdiv = FIELD_GET(PLL_CFG_REFDIV_MASK, val);
+	fbdiv = FIELD_GET(PLL_CFG_FBDIV_MASK, val);
+	postdiv1 = FIELD_GET(PLL_CFG_POSTDIV1_MASK, val);
+	postdiv2 = FIELD_GET(PLL_CFG_POSTDIV2_MASK, val);
+
+	if (!refdiv)
+		refdiv = 1;
+	if (!postdiv1)
+		postdiv1 = 1;
+	if (!postdiv2)
+		postdiv2 = 1;
+	if (!fbdiv)
+		return 0;
+
+	fout = div64_u64((u64)parent_rate * fbdiv,
+			 refdiv * postdiv1 * postdiv2);
+
+	return fout;
+}
+
+static const struct clk_ops atlantis_clk_pll_ops = {
+	.enable = atlantis_clk_pll_enable,
+	.disable = atlantis_clk_pll_disable,
+	.recalc_rate = atlantis_clk_pll_recalc_rate,
+	.is_enabled = atlantis_clk_pll_is_enabled,
+};
+
+static int atlantis_clk_gate_shared_enable(struct clk_hw *hw)
+{
+	struct atlantis_clk_gate_shared *gate =
+		hw_to_atlantis_clk_gate_shared(hw);
+	bool need_enable;
+
+	scoped_guard(spinlock_irqsave, gate->config.refcount_lock)
+	{
+		need_enable = (*gate->config.share_count)++ == 0;
+		if (need_enable) {
+			regmap_set_bits(gate->common.regmap,
+					gate->config.reg_offset,
+					gate->config.enable);
+		}
+	}
+
+	if (need_enable) {
+		if (!regmap_test_bits(gate->common.regmap,
+				      gate->config.reg_offset,
+				      gate->config.enable)) {
+			pr_warn("%s: gate enable %d failed to enable\n",
+				clk_hw_get_name(hw), gate->config.enable);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+static void atlantis_clk_gate_shared_disable(struct clk_hw *hw)
+{
+	struct atlantis_clk_gate_shared *gate =
+		hw_to_atlantis_clk_gate_shared(hw);
+
+	scoped_guard(spinlock_irqsave, gate->config.refcount_lock)
+	{
+		if (WARN_ON(*gate->config.share_count == 0))
+			return;
+		if (--(*gate->config.share_count) > 0)
+			return;
+
+		regmap_clear_bits(gate->common.regmap,
+				  gate->config.reg_offset,
+				  gate->config.enable);
+	}
+}
+
+static int atlantis_clk_gate_shared_is_enabled(struct clk_hw *hw)
+{
+	struct atlantis_clk_gate_shared *gate =
+		hw_to_atlantis_clk_gate_shared(hw);
+
+	return regmap_test_bits(gate->common.regmap, gate->config.reg_offset, gate->config.enable);
+}
+
+static void atlantis_clk_gate_shared_disable_unused(struct clk_hw *hw)
+{
+	struct atlantis_clk_gate_shared *gate =
+		hw_to_atlantis_clk_gate_shared(hw);
+
+	scoped_guard(spinlock_irqsave, gate->config.refcount_lock)
+	{
+		if (*gate->config.share_count == 0)
+			regmap_clear_bits(gate->common.regmap,
+					  gate->config.reg_offset,
+					  gate->config.enable);
+	}
+}
+
+static const struct clk_ops atlantis_clk_gate_shared_ops = {
+	.enable = atlantis_clk_gate_shared_enable,
+	.disable = atlantis_clk_gate_shared_disable,
+	.disable_unused = atlantis_clk_gate_shared_disable_unused,
+	.is_enabled = atlantis_clk_gate_shared_is_enabled,
+};
+
+#define ATLANTIS_PLL_CONFIG(_reg_offset, _en_reg_offset, _cg_reg_offset, \
+			    _cg_reg_enable)                              \
+	{                                                                \
+		.reg_offset = (_reg_offset),                             \
+		.en_reg_offset = (_en_reg_offset),                       \
+		.cg_reg_offset = (_cg_reg_offset),                       \
+		.cg_reg_enable = (_cg_reg_enable),                       \
+	}
+
+#define ATLANTIS_PLL_DEFINE(_clkid, _name, _parent, _reg_offset,               \
+			    _en_reg_offset, _cg_reg_offset, _cg_reg_enable,    \
+			    _flags)                                            \
+	static struct atlantis_clk_pll _name = {                               \
+		.config = ATLANTIS_PLL_CONFIG(_reg_offset, _en_reg_offset,     \
+					      _cg_reg_offset, _cg_reg_enable), \
+		.common = { .clkid = _clkid,                                   \
+			    .hw.init = CLK_HW_INIT_PARENTS_DATA(               \
+				    #_name, _parent, &atlantis_clk_pll_ops,    \
+				    _flags) },                                 \
+	}
+#define ATLANTIS_MUX_CONFIG(_shift, _width, _reg_offset)                    \
+	{                                                                   \
+		.shift = _shift, .width = _width, .reg_offset = _reg_offset \
+	}
+
+#define ATLANTIS_MUX_DEFINE(_clkid, _name, _parents, _reg_offset, _shift,    \
+			    _width, _flags)                                  \
+	static struct atlantis_clk_mux _name = {                             \
+		.config = ATLANTIS_MUX_CONFIG(_shift, _width, _reg_offset),  \
+		.common = { .clkid = _clkid,                                 \
+			    .hw.init = CLK_HW_INIT_PARENTS_DATA(             \
+				    #_name, _parents, &atlantis_clk_mux_ops, \
+				    _flags) }                                \
+	}
+
+#define ATLANTIS_DIVIDER_CONFIG(_shift, _width, _flags, _reg_offset) \
+	{                                                            \
+		.shift = _shift, .width = _width, .flags = _flags,   \
+		.reg_offset = _reg_offset                            \
+	}
+
+#define ATLANTIS_DIVIDER_DEFINE(_clkid, _name, _parent, _reg_offset, _shift, \
+				_width, _divflags, _flags)                   \
+	static struct atlantis_clk_divider _name = {                         \
+		.config = ATLANTIS_DIVIDER_CONFIG(_shift, _width, _divflags, \
+						  _reg_offset),              \
+		.common = { .clkid = _clkid,                                 \
+			    .hw.init = CLK_HW_INIT_HW(                       \
+				    #_name, &_parent.common.hw,              \
+				    &atlantis_clk_divider_ops, _flags) }     \
+	}
+#define ATLANTIS_GATE_CONFIG(_enable, _reg_offset)           \
+	{                                                    \
+		.enable = _enable, .reg_offset = _reg_offset \
+	}
+
+#define ATLANTIS_GATE_DEFINE(_clkid, _name, _parent, _reg_offset, _enable, \
+			     _flags)                                       \
+	static struct atlantis_clk_gate _name = {                          \
+		.config = ATLANTIS_GATE_CONFIG(_enable, _reg_offset),      \
+		.common = { .clkid = _clkid,                               \
+			    .hw.init = CLK_HW_INIT_HW(                     \
+				    #_name, &_parent.common.hw,            \
+				    &atlantis_clk_gate_ops, _flags) }      \
+	}
+#define ATLANTIS_GATE_SHARED_CONFIG(_reg_offset, _enable, _share_count)      \
+	{                                                                    \
+		.reg_offset = _reg_offset, .enable = _enable,                \
+		.share_count = _share_count, .refcount_lock = &refcount_lock \
+	}
+#define ATLANTIS_GATE_SHARED_DEFINE(_clkid, _name, _parent, _reg_offset,     \
+				    _enable, _share_count, _flags)           \
+	static struct atlantis_clk_gate_shared _name = {                     \
+		.config = ATLANTIS_GATE_SHARED_CONFIG(_reg_offset, _enable,  \
+						      _share_count),         \
+		.common = { .clkid = _clkid,                                 \
+			    .hw.init = CLK_HW_INIT_HW(                       \
+				    #_name, &_parent.common.hw,              \
+				    &atlantis_clk_gate_shared_ops, _flags) } \
+	}
+#define ATLANTIS_FIXED_FACTOR_DEFINE(_clkid, _name, _parent, _mult, _div,     \
+				     _flags)                                  \
+	static struct atlantis_clk_fixed_factor _name = {                     \
+		.config = { .mult = _mult, .div = _div },                     \
+		.common = { .clkid = _clkid,                                  \
+			    .hw.init = CLK_HW_INIT_HW(                        \
+				    #_name, &_parent.common.hw,               \
+				    &atlantis_clk_fixed_factor_ops, _flags) } \
+	}
+
+static DEFINE_SPINLOCK(refcount_lock); /* Lock for refcount value accesses */
+
+static const struct regmap_config atlantis_prcm_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xFFFC,
+	.cache_type = REGCACHE_NONE,
+};
+
+struct atlantis_prcm_data {
+	struct clk_hw **hws;
+	size_t num;
+	const char *reset_name;
+};
+
+static const struct clk_parent_data osc_24m_clk[] = {
+	{ .index = 0 },
+};
+
+ATLANTIS_PLL_DEFINE(CLK_RCPU_PLL, rcpu_pll_clk, osc_24m_clk, PLL_RCPU_CFG_REG,
+		    PLL_RCPU_EN_REG, BUS_CG_REG, 0, /* No Gate Clk at Output */
+		    CLK_GET_RATE_NOCACHE | CLK_IS_CRITICAL);
+
+static const struct clk_parent_data rcpu_root_parents[] = {
+	{ .index = 0 },
+	{ .hw = &rcpu_pll_clk.common.hw },
+};
+
+ATLANTIS_MUX_DEFINE(CLK_RCPU_ROOT, rcpu_root_clk, rcpu_root_parents,
+		    RCPU_DIV_CFG_REG, 0, 1, CLK_SET_RATE_NO_REPARENT);
+
+ATLANTIS_DIVIDER_DEFINE(CLK_RCPU_DIV2, rcpu_div2_clk, rcpu_root_clk,
+			RCPU_DIV_CFG_REG, 2, 4, 0, 0);
+ATLANTIS_DIVIDER_DEFINE(CLK_RCPU_DIV4, rcpu_div4_clk, rcpu_root_clk,
+			RCPU_DIV_CFG_REG, 7, 4, 0, 0);
+ATLANTIS_DIVIDER_DEFINE(CLK_RCPU_RTC, rcpu_rtc_clk, rcpu_div4_clk,
+			RCPU_DIV_CFG_REG, 12, 6, 0, 0);
+
+ATLANTIS_GATE_DEFINE(CLK_SMNDMA0_ACLK, rcpu_dma0_clk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(0), 0);
+ATLANTIS_GATE_DEFINE(CLK_SMNDMA1_ACLK, rcpu_dma1_clk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(1), 0);
+ATLANTIS_GATE_DEFINE(CLK_WDT0_PCLK, sl_wdt0_pclk, rcpu_div4_clk,
+		     RCPU_BLK_CG_REG, BIT(2), 0);
+ATLANTIS_GATE_DEFINE(CLK_WDT1_PCLK, sl_wdt1_pclk, rcpu_div4_clk,
+		     RCPU_BLK_CG_REG, BIT(3), 0);
+ATLANTIS_GATE_DEFINE(CLK_TIMER_PCLK, sl_timer_pclk, rcpu_div4_clk,
+		     RCPU_BLK_CG_REG, BIT(4), 0);
+ATLANTIS_GATE_DEFINE(CLK_PVTC_PCLK, sl_pvtc_pclk, rcpu_div4_clk,
+		     RCPU_BLK_CG_REG, BIT(12), 0);
+ATLANTIS_GATE_DEFINE(CLK_PMU_PCLK, sl_pmu_pclk, rcpu_div4_clk, RCPU_BLK_CG_REG,
+		     BIT(13), 0);
+ATLANTIS_GATE_DEFINE(CLK_MAILBOX_HCLK, rcpu_ipc_clk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(14), 0);
+ATLANTIS_GATE_DEFINE(CLK_SEC_SPACC_HCLK, sec_spacc_hclk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(26), 0);
+ATLANTIS_GATE_DEFINE(CLK_SEC_OTP_HCLK, sec_otp_hclk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(28), 0);
+ATLANTIS_GATE_DEFINE(CLK_TRNG_PCLK, sec_trng_pclk, rcpu_div4_clk,
+		     RCPU_BLK_CG_REG, BIT(29), 0);
+ATLANTIS_GATE_DEFINE(CLK_SEC_CRC_HCLK, sec_crc_hclk, rcpu_div2_clk,
+		     RCPU_BLK_CG_REG, BIT(30), 0);
+
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_SMN_HCLK, rcpu_smn_hclk, rcpu_div2_clk, 1, 1,
+			     0);
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_AHB0_HCLK, rcpu_ahb0_hclk, rcpu_div2_clk, 1, 1,
+			     0);
+
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_SMN_PCLK, rcpu_smn_pclk, rcpu_div4_clk, 1, 1,
+			     0);
+
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_SMN_CLK, rcpu_smn_clk, rcpu_root_clk, 1, 1, 0);
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_SCRATCHPAD_CLK, rcpu_scratchpad_aclk,
+			     rcpu_root_clk, 1, 1, 0);
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_RCPU_CORE_CLK, rcpu_core_clk, rcpu_root_clk, 1,
+			     1, 0);
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_RCPU_ROM_CLK, rcpu_rom_aclk, rcpu_root_clk, 1,
+			     1, 0);
+
+static struct atlantis_clk_fixed_factor
+	otp_load_clk = { .config = { .mult = 1, .div = 1 },
+			 .common = {
+				 .clkid = CLK_OTP_LOAD_CLK,
+				 .hw.init = CLK_HW_INIT_PARENTS_DATA(
+					 "otp_load_clk", osc_24m_clk,
+					 &atlantis_clk_fixed_factor_ops,
+					 CLK_SET_RATE_NO_REPARENT),
+			 } };
+
+ATLANTIS_PLL_DEFINE(CLK_NOC_PLL, nocc_pll_clk, osc_24m_clk, PLL_NOCC_CFG_REG,
+		    PLL_NOCC_EN_REG, BUS_CG_REG, BIT(0),
+		    CLK_GET_RATE_NOCACHE | CLK_IS_CRITICAL);
+
+static const struct clk_parent_data nocc_mux_parents[] = {
+	{ .index = 0 },
+	{ .hw = &nocc_pll_clk.common.hw },
+};
+
+ATLANTIS_MUX_DEFINE(CLK_NOCC_CLK, nocc_clk, nocc_mux_parents, NOCC_CLK_CFG_REG,
+		    0, 1, CLK_SET_RATE_NO_REPARENT);
+
+ATLANTIS_DIVIDER_DEFINE(CLK_NOCC_DIV2, nocc_div2_clk, nocc_clk,
+			NOCC_CLK_CFG_REG, 1, 4, 0, 0);
+ATLANTIS_DIVIDER_DEFINE(CLK_NOCC_DIV4, nocc_div4_clk, nocc_clk,
+			NOCC_CLK_CFG_REG, 5, 4, 0, 0);
+ATLANTIS_DIVIDER_DEFINE(CLK_NOCC_RTC, nocc_rtc_clk, nocc_div4_clk,
+			NOCC_CLK_CFG_REG, 9, 6, 0, 0);
+ATLANTIS_DIVIDER_DEFINE(CLK_NOCC_CAN, nocc_can_clk, nocc_clk, NOCC_CLK_CFG_REG,
+			15, 4, 0, 0);
+
+static unsigned int refcnt_qspi;
+ATLANTIS_GATE_SHARED_DEFINE(CLK_QSPI_SCLK, lsio_qspi_sclk, nocc_clk,
+			    LSIO_BLK_CG_REG, BIT(0), &refcnt_qspi, 0);
+ATLANTIS_GATE_SHARED_DEFINE(CLK_QSPI_HCLK, lsio_qspi_hclk, nocc_div2_clk,
+			    LSIO_BLK_CG_REG, BIT(0), &refcnt_qspi, 0);
+ATLANTIS_GATE_DEFINE(CLK_I2C0_PCLK, lsio_i2c0_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(1), 0);
+ATLANTIS_GATE_DEFINE(CLK_I2C1_PCLK, lsio_i2c1_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(2), 0);
+ATLANTIS_GATE_DEFINE(CLK_I2C2_PCLK, lsio_i2c2_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(3), 0);
+ATLANTIS_GATE_DEFINE(CLK_I2C3_PCLK, lsio_i2c3_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(4), 0);
+ATLANTIS_GATE_DEFINE(CLK_I2C4_PCLK, lsio_i2c4_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(5), 0);
+
+ATLANTIS_GATE_DEFINE(CLK_UART0_PCLK, lsio_uart0_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(6), 0);
+ATLANTIS_GATE_DEFINE(CLK_UART1_PCLK, lsio_uart1_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(7), 0);
+ATLANTIS_GATE_DEFINE(CLK_UART2_PCLK, lsio_uart2_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(8), 0);
+ATLANTIS_GATE_DEFINE(CLK_UART3_PCLK, lsio_uart3_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(9), 0);
+ATLANTIS_GATE_DEFINE(CLK_UART4_PCLK, lsio_uart4_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(10), 0);
+ATLANTIS_GATE_DEFINE(CLK_SPI0_PCLK, lsio_spi0_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(11), 0);
+ATLANTIS_GATE_DEFINE(CLK_SPI1_PCLK, lsio_spi1_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(12), 0);
+ATLANTIS_GATE_DEFINE(CLK_SPI2_PCLK, lsio_spi2_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(13), 0);
+ATLANTIS_GATE_DEFINE(CLK_SPI3_PCLK, lsio_spi3_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(14), 0);
+ATLANTIS_GATE_DEFINE(CLK_GPIO_PCLK, lsio_gpio_pclk, nocc_div4_clk,
+		     LSIO_BLK_CG_REG, BIT(15), 0);
+
+static unsigned int refcnt_can0;
+ATLANTIS_GATE_SHARED_DEFINE(CLK_CAN0_HCLK, lsio_can0_hclk, nocc_div2_clk,
+			    LSIO_BLK_CG_REG, BIT(17), &refcnt_can0, 0);
+ATLANTIS_GATE_SHARED_DEFINE(CLK_CAN0_CLK, lsio_can0_clk, nocc_can_clk,
+			    LSIO_BLK_CG_REG, BIT(17), &refcnt_can0, 0);
+
+static unsigned int refcnt_can1;
+ATLANTIS_GATE_SHARED_DEFINE(CLK_CAN1_HCLK, lsio_can1_hclk, nocc_div2_clk,
+			    LSIO_BLK_CG_REG, BIT(18), &refcnt_can1, 0);
+ATLANTIS_GATE_SHARED_DEFINE(CLK_CAN1_CLK, lsio_can1_clk, nocc_can_clk,
+			    LSIO_BLK_CG_REG, BIT(18), &refcnt_can1, 0);
+
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_CAN0_TIMER_CLK, lsio_can0_timer_clk,
+				 nocc_rtc_clk, 1, 1, 0);
+ATLANTIS_FIXED_FACTOR_DEFINE(CLK_CAN1_TIMER_CLK, lsio_can1_timer_clk,
+				 nocc_rtc_clk, 1, 1, 0);
+
+static struct clk_hw *atlantis_rcpu_clks[] = {
+	[CLK_RCPU_PLL]		= &rcpu_pll_clk.common.hw,
+	[CLK_RCPU_ROOT]		= &rcpu_root_clk.common.hw,
+	[CLK_RCPU_DIV2]		= &rcpu_div2_clk.common.hw,
+	[CLK_RCPU_DIV4]		= &rcpu_div4_clk.common.hw,
+	[CLK_RCPU_RTC]		= &rcpu_rtc_clk.common.hw,
+	[CLK_SMNDMA0_ACLK]	= &rcpu_dma0_clk.common.hw,
+	[CLK_SMNDMA1_ACLK]	= &rcpu_dma1_clk.common.hw,
+	[CLK_WDT0_PCLK]		= &sl_wdt0_pclk.common.hw,
+	[CLK_WDT1_PCLK]		= &sl_wdt1_pclk.common.hw,
+	[CLK_TIMER_PCLK]	= &sl_timer_pclk.common.hw,
+	[CLK_PVTC_PCLK]		= &sl_pvtc_pclk.common.hw,
+	[CLK_PMU_PCLK]		= &sl_pmu_pclk.common.hw,
+	[CLK_MAILBOX_HCLK]	= &rcpu_ipc_clk.common.hw,
+	[CLK_SEC_SPACC_HCLK]	= &sec_spacc_hclk.common.hw,
+	[CLK_SEC_OTP_HCLK]	= &sec_otp_hclk.common.hw,
+	[CLK_TRNG_PCLK]		= &sec_trng_pclk.common.hw,
+	[CLK_SEC_CRC_HCLK]	= &sec_crc_hclk.common.hw,
+	[CLK_SMN_HCLK]		= &rcpu_smn_hclk.common.hw,
+	[CLK_AHB0_HCLK]		= &rcpu_ahb0_hclk.common.hw,
+	[CLK_SMN_PCLK]		= &rcpu_smn_pclk.common.hw,
+	[CLK_SMN_CLK]		= &rcpu_smn_clk.common.hw,
+	[CLK_SCRATCHPAD_CLK]	= &rcpu_scratchpad_aclk.common.hw,
+	[CLK_RCPU_CORE_CLK]	= &rcpu_core_clk.common.hw,
+	[CLK_RCPU_ROM_CLK]	= &rcpu_rom_aclk.common.hw,
+	[CLK_OTP_LOAD_CLK]	= &otp_load_clk.common.hw,
+	[CLK_NOC_PLL]		= &nocc_pll_clk.common.hw,
+	[CLK_NOCC_CLK]		= &nocc_clk.common.hw,
+	[CLK_NOCC_DIV2]		= &nocc_div2_clk.common.hw,
+	[CLK_NOCC_DIV4]		= &nocc_div4_clk.common.hw,
+	[CLK_NOCC_RTC]		= &nocc_rtc_clk.common.hw,
+	[CLK_NOCC_CAN]		= &nocc_can_clk.common.hw,
+	[CLK_QSPI_SCLK]		= &lsio_qspi_sclk.common.hw,
+	[CLK_QSPI_HCLK]		= &lsio_qspi_hclk.common.hw,
+	[CLK_I2C0_PCLK]		= &lsio_i2c0_pclk.common.hw,
+	[CLK_I2C1_PCLK]		= &lsio_i2c1_pclk.common.hw,
+	[CLK_I2C2_PCLK]		= &lsio_i2c2_pclk.common.hw,
+	[CLK_I2C3_PCLK]		= &lsio_i2c3_pclk.common.hw,
+	[CLK_I2C4_PCLK]		= &lsio_i2c4_pclk.common.hw,
+	[CLK_UART0_PCLK]	= &lsio_uart0_pclk.common.hw,
+	[CLK_UART1_PCLK]	= &lsio_uart1_pclk.common.hw,
+	[CLK_UART2_PCLK]	= &lsio_uart2_pclk.common.hw,
+	[CLK_UART3_PCLK]	= &lsio_uart3_pclk.common.hw,
+	[CLK_UART4_PCLK]	= &lsio_uart4_pclk.common.hw,
+	[CLK_SPI0_PCLK]		= &lsio_spi0_pclk.common.hw,
+	[CLK_SPI1_PCLK]		= &lsio_spi1_pclk.common.hw,
+	[CLK_SPI2_PCLK]		= &lsio_spi2_pclk.common.hw,
+	[CLK_SPI3_PCLK]		= &lsio_spi3_pclk.common.hw,
+	[CLK_GPIO_PCLK]		= &lsio_gpio_pclk.common.hw,
+	[CLK_CAN0_HCLK]		= &lsio_can0_hclk.common.hw,
+	[CLK_CAN0_CLK]		= &lsio_can0_clk.common.hw,
+	[CLK_CAN1_HCLK]		= &lsio_can1_hclk.common.hw,
+	[CLK_CAN1_CLK]		= &lsio_can1_clk.common.hw,
+	[CLK_CAN0_TIMER_CLK]	= &lsio_can0_timer_clk.common.hw,
+	[CLK_CAN1_TIMER_CLK]	= &lsio_can1_timer_clk.common.hw,
+};
+
+static const struct atlantis_prcm_data atlantis_prcm_rcpu_data = {
+	.hws = atlantis_rcpu_clks,
+	.num = ARRAY_SIZE(atlantis_rcpu_clks),
+	.reset_name = "rcpu-reset"
+};
+
+static int atlantis_prcm_clocks_register(struct device *dev,
+					 struct regmap *regmap,
+					 const struct atlantis_prcm_data *data)
+{
+	struct clk_hw_onecell_data *clk_data;
+	int i, ret;
+	size_t num_clks = data->num;
+
+	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, data->num),
+				GFP_KERNEL);
+	if (!clk_data)
+		return -ENOMEM;
+
+	for (i = 0; i < data->num; i++) {
+		struct clk_hw *hw = data->hws[i];
+		struct atlantis_clk_common *common =
+			hw_to_atlantis_clk_common(hw);
+		common->regmap = regmap;
+
+		ret = devm_clk_hw_register(dev, hw);
+		if (ret)
+			return ret;
+
+		clk_data->hws[common->clkid] = hw;
+	}
+
+	clk_data->num = num_clks;
+
+	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
+}
+
+static int atlantis_prcm_probe(struct platform_device *pdev)
+{
+	const struct atlantis_prcm_data *data;
+	struct auxiliary_device *reset_adev;
+	struct regmap *regmap;
+	void __iomem *base;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return dev_err_probe(dev, PTR_ERR(base),
+				     "Failed to map registers\n");
+
+	regmap = devm_regmap_init_mmio(dev, base, &atlantis_prcm_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "Failed to init regmap\n");
+
+	data = of_device_get_match_data(dev);
+
+	ret = atlantis_prcm_clocks_register(dev, regmap, data);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register clocks\n");
+
+	reset_adev = devm_auxiliary_device_create(dev, data->reset_name, NULL);
+	if (!reset_adev)
+		return dev_err_probe(dev, -ENODEV, "failed to register resets\n");
+
+	return 0;
+}
+
+static const struct of_device_id atlantis_prcm_of_match[] = {
+	{
+		.compatible = "tenstorrent,atlantis-prcm-rcpu",
+		.data = &atlantis_prcm_rcpu_data,
+	},
+	{}
+
+};
+MODULE_DEVICE_TABLE(of, atlantis_prcm_of_match);
+
+static struct platform_driver atlantis_prcm_driver = {
+	.probe = atlantis_prcm_probe,
+	.driver = {
+		.name = "atlantis-prcm",
+		.of_match_table = atlantis_prcm_of_match,
+	},
+};
+module_platform_driver(atlantis_prcm_driver);
+
+MODULE_DESCRIPTION("Tenstorrent Atlantis PRCM Clock Controller Driver");
+MODULE_AUTHOR("Anirudh Srinivasan <asrinivasan@oss.tenstorrent.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index fd91127065454..d1a33a231a44c 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -596,7 +596,6 @@ config CLKSRC_VERSATILE
 config CLKSRC_MIPS_GIC
 	bool
 	depends on MIPS_GIC
-	select CLOCKSOURCE_WATCHDOG
 	select TIMER_OF
 
 config CLKSRC_PXA
diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c
index 48a8a607a84c2..b91e0b5ac394d 100644
--- a/drivers/comedi/comedi_fops.c
+++ b/drivers/comedi/comedi_fops.c
@@ -2588,7 +2588,7 @@ static int comedi_mmap(struct file *file, struct vm_area_struct *vma)
 	 * remap_pfn_range() because we call remap_pfn_range() in a loop.
 	 */
 	if (retval)
-		zap_vma_ptes(vma, vma->vm_start, size);
+		zap_special_vma_range(vma, vma->vm_start, size);
 #endif
 
 	if (retval == 0) {
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 4014bc9dd73a2..47c9b031f1b3b 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -248,7 +248,7 @@ config ARM_TEGRA186_CPUFREQ
 
 config ARM_TEGRA194_CPUFREQ
 	tristate "Tegra194 CPUFreq support"
-	depends on ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC || (64BIT && COMPILE_TEST)
+	depends on ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC
 	depends on TEGRA_BPMP
 	default ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC
 	help
diff --git a/drivers/cpufreq/airoha-cpufreq.c b/drivers/cpufreq/airoha-cpufreq.c
index b6b1cdc4d11d6..3e7770860d13c 100644
--- a/drivers/cpufreq/airoha-cpufreq.c
+++ b/drivers/cpufreq/airoha-cpufreq.c
@@ -115,15 +115,10 @@ MODULE_DEVICE_TABLE(of, airoha_cpufreq_match_list);
 
 static int __init airoha_cpufreq_init(void)
 {
-	struct device_node *np = of_find_node_by_path("/");
 	const struct of_device_id *match;
 	int ret;
 
-	if (!np)
-		return -ENODEV;
-
-	match = of_match_node(airoha_cpufreq_match_list, np);
-	of_node_put(np);
+	match = of_machine_get_match(airoha_cpufreq_match_list);
 	if (!match)
 		return -ENODEV;
 
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 011f35cb47b94..5dfb109cf1f4e 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -50,7 +50,8 @@ struct cppc_freq_invariance {
 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
 static struct kthread_worker *kworker_fie;
 
-static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+static int cppc_perf_from_fbctrs(u64 reference_perf,
+				 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
 				 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
 
 /**
@@ -70,7 +71,7 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi)
 	struct cppc_perf_fb_ctrs fb_ctrs = {0};
 	struct cppc_cpudata *cpu_data;
 	unsigned long local_freq_scale;
-	u64 perf;
+	u64 perf, ref_perf;
 
 	cpu_data = cppc_fi->cpu_data;
 
@@ -79,7 +80,9 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi)
 		return;
 	}
 
-	perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs);
+	ref_perf = cpu_data->perf_caps.reference_perf;
+	perf = cppc_perf_from_fbctrs(ref_perf,
+				     &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs);
 	if (!perf)
 		return;
 
@@ -287,6 +290,21 @@ static inline void cppc_freq_invariance_exit(void)
 }
 #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
 
+static void cppc_cpufreq_update_perf_limits(struct cppc_cpudata *cpu_data,
+					    struct cpufreq_policy *policy)
+{
+	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
+	u32 min_perf, max_perf;
+
+	min_perf = cppc_khz_to_perf(caps, policy->min);
+	max_perf = cppc_khz_to_perf(caps, policy->max);
+
+	cpu_data->perf_ctrls.min_perf =
+		clamp_t(u32, min_perf, caps->lowest_perf, caps->highest_perf);
+	cpu_data->perf_ctrls.max_perf =
+		clamp_t(u32, max_perf, caps->lowest_perf, caps->highest_perf);
+}
+
 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
 				   unsigned int target_freq,
 				   unsigned int relation)
@@ -298,6 +316,8 @@ static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
 
 	cpu_data->perf_ctrls.desired_perf =
 			cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
+	cppc_cpufreq_update_perf_limits(cpu_data, policy);
+
 	freqs.old = policy->cur;
 	freqs.new = target_freq;
 
@@ -322,8 +342,9 @@ static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
 
 	desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
 	cpu_data->perf_ctrls.desired_perf = desired_perf;
-	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
+	cppc_cpufreq_update_perf_limits(cpu_data, policy);
 
+	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
 	if (ret) {
 		pr_debug("Failed to set target on CPU:%d. ret:%d\n",
 			 cpu, ret);
@@ -594,6 +615,12 @@ static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
 		goto free_mask;
 	}
 
+	ret = cppc_get_perf(cpu, &cpu_data->perf_ctrls);
+	if (ret) {
+		pr_debug("Err reading CPU%d perf ctrls: ret:%d\n", cpu, ret);
+		goto free_mask;
+	}
+
 	return cpu_data;
 
 free_mask:
@@ -723,13 +750,11 @@ static inline u64 get_delta(u64 t1, u64 t0)
 	return (u32)t1 - (u32)t0;
 }
 
-static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0,
+static int cppc_perf_from_fbctrs(u64 reference_perf,
+				 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
 				 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
 {
 	u64 delta_reference, delta_delivered;
-	u64 reference_perf;
-
-	reference_perf = fb_ctrs_t0->reference_perf;
 
 	delta_reference = get_delta(fb_ctrs_t1->reference,
 				    fb_ctrs_t0->reference);
@@ -766,7 +791,7 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
 	struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
 	struct cppc_cpudata *cpu_data;
-	u64 delivered_perf;
+	u64 delivered_perf, reference_perf;
 	int ret;
 
 	if (!policy)
@@ -783,7 +808,9 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
 			return 0;
 	}
 
-	delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1);
+	reference_perf = cpu_data->perf_caps.reference_perf;
+	delivered_perf = cppc_perf_from_fbctrs(reference_perf,
+					       &fb_ctrs_t0, &fb_ctrs_t1);
 	if (!delivered_perf)
 		goto out_invalid_counters;
 
@@ -849,6 +876,7 @@ static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf)
 static ssize_t store_auto_select(struct cpufreq_policy *policy,
 				 const char *buf, size_t count)
 {
+	struct cppc_cpudata *cpu_data = policy->driver_data;
 	bool val;
 	int ret;
 
@@ -860,6 +888,29 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy,
 	if (ret)
 		return ret;
 
+	cpu_data->perf_ctrls.auto_sel = val;
+
+	if (val) {
+		u32 old_min_perf = cpu_data->perf_ctrls.min_perf;
+		u32 old_max_perf = cpu_data->perf_ctrls.max_perf;
+
+		/*
+		 * When enabling autonomous selection, program MIN_PERF and
+		 * MAX_PERF from current policy limits so that the platform
+		 * uses the correct performance bounds immediately.
+		 */
+		cppc_cpufreq_update_perf_limits(cpu_data, policy);
+
+		ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls);
+		if (ret) {
+			cpu_data->perf_ctrls.min_perf = old_min_perf;
+			cpu_data->perf_ctrls.max_perf = old_max_perf;
+			cppc_set_auto_sel(policy->cpu, false);
+			cpu_data->perf_ctrls.auto_sel = false;
+			return ret;
+		}
+	}
+
 	return count;
 }
 
@@ -910,19 +961,48 @@ static ssize_t store_##_name(struct cpufreq_policy *policy,		\
 CPPC_CPUFREQ_ATTR_RW_U64(auto_act_window, cppc_get_auto_act_window,
 			 cppc_set_auto_act_window)
 
-CPPC_CPUFREQ_ATTR_RW_U64(energy_performance_preference_val,
-			 cppc_get_epp_perf, cppc_set_epp)
+static ssize_t
+show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf)
+{
+	return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf);
+}
+
+static ssize_t
+store_energy_performance_preference_val(struct cpufreq_policy *policy,
+					const char *buf, size_t count)
+{
+	struct cppc_cpudata *cpu_data = policy->driver_data;
+	u64 val;
+	int ret;
+
+	ret = kstrtou64(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	ret = cppc_set_epp(policy->cpu, val);
+	if (ret)
+		return ret;
+
+	cpu_data->perf_ctrls.energy_perf = val;
+
+	return count;
+}
+
+CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited,
+			 cppc_set_perf_limited)
 
 cpufreq_freq_attr_ro(freqdomain_cpus);
 cpufreq_freq_attr_rw(auto_select);
 cpufreq_freq_attr_rw(auto_act_window);
 cpufreq_freq_attr_rw(energy_performance_preference_val);
+cpufreq_freq_attr_rw(perf_limited);
 
 static struct freq_attr *cppc_cpufreq_attr[] = {
 	&freqdomain_cpus,
 	&auto_select,
 	&auto_act_window,
 	&energy_performance_preference_val,
+	&perf_limited,
 	NULL,
 };
 
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 25fd3b191b7e7..ff1204c666b19 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -159,6 +159,7 @@ static const struct of_device_id blocklist[] __initconst = {
 	{ .compatible = "qcom,qcm2290", },
 	{ .compatible = "qcom,qcm6490", },
 	{ .compatible = "qcom,qcs404", },
+	{ .compatible = "qcom,qcs8300", },
 	{ .compatible = "qcom,qdu1000", },
 	{ .compatible = "qcom,sa8155p" },
 	{ .compatible = "qcom,sa8540p" },
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 277884d91913c..2082a9e4384f4 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -2367,8 +2367,8 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy,
 	target_freq = __resolve_freq(policy, target_freq, policy->min,
 				     policy->max, relation);
 
-	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
-		 policy->cpu, target_freq, relation, old_target_freq);
+	pr_debug("CPU %u: cur %u kHz -> target %u kHz (req %u kHz, rel %u)\n",
+		policy->cpu, policy->cur, target_freq, old_target_freq, relation);
 
 	/*
 	 * This might look like a redundant call as we are checking it again
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 168c23fd7fcac..732fd4d1a8037 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -21,6 +21,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/sysfs.h>
 
 /* Ondemand Sampling types */
 enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};
@@ -57,7 +58,7 @@ static ssize_t file_name##_show						\
 {									\
 	struct dbs_data *dbs_data = to_dbs_data(attr_set);		\
 	struct _gov##_dbs_tuners *tuners = dbs_data->tuners;		\
-	return sprintf(buf, "%u\n", tuners->file_name);			\
+	return sysfs_emit(buf, "%u\n", tuners->file_name);		\
 }
 
 #define gov_show_one_common(file_name)					\
@@ -65,7 +66,7 @@ static ssize_t file_name##_show						\
 (struct gov_attr_set *attr_set, char *buf)				\
 {									\
 	struct dbs_data *dbs_data = to_dbs_data(attr_set);		\
-	return sprintf(buf, "%u\n", dbs_data->file_name);		\
+	return sysfs_emit(buf, "%u\n", dbs_data->file_name);		\
 }
 
 #define gov_attr_ro(_name)						\
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 11c58af419006..51938c5a47ca9 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -3472,7 +3472,7 @@ static int intel_pstate_update_status(const char *buf, size_t size)
 {
 	if (size == 3 && !strncmp(buf, "off", size)) {
 		if (!intel_pstate_driver)
-			return -EINVAL;
+			return 0;
 
 		if (hwp_active)
 			return -EBUSY;
diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c
index b8081acba928f..e6d28d162442a 100644
--- a/drivers/cpufreq/qcom-cpufreq-nvmem.c
+++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c
@@ -291,17 +291,9 @@ static int qcom_cpufreq_ipq8064_name_version(struct device *cpu_dev,
 	ret = qcom_smem_get_soc_id(&msm_id);
 	if (ret == -ENODEV) {
 		const struct of_device_id *match;
-		struct device_node *root;
-
-		root = of_find_node_by_path("/");
-		if (!root) {
-			ret = -ENODEV;
-			goto exit;
-		}
 
 		/* Fallback to compatible match with no SMEM initialized */
-		match = of_match_node(qcom_cpufreq_ipq806x_match_list, root);
-		of_node_put(root);
+		match = of_machine_get_match(qcom_cpufreq_ipq806x_match_list);
 		if (!match) {
 			ret = -ENODEV;
 			goto exit;
@@ -647,14 +639,10 @@ MODULE_DEVICE_TABLE(of, qcom_cpufreq_match_list);
  */
 static int __init qcom_cpufreq_init(void)
 {
-	struct device_node *np __free(device_node) = of_find_node_by_path("/");
 	const struct of_device_id *match;
 	int ret;
 
-	if (!np)
-		return -ENODEV;
-
-	match = of_match_node(qcom_cpufreq_match_list, np);
+	match = of_machine_get_match(qcom_cpufreq_match_list);
 	if (!match)
 		return -ENODEV;
 
diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c
index 7a41cfc71a464..c6375e14d445d 100644
--- a/drivers/cpufreq/tegra194-cpufreq.c
+++ b/drivers/cpufreq/tegra194-cpufreq.c
@@ -196,7 +196,7 @@ static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = {
 	.refclk_delta_min = 16000,
 };
 
-static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = {
+static const struct tegra_cpufreq_soc tegra238_cpufreq_soc = {
 	.ops = &tegra234_cpufreq_ops,
 	.actmon_cntr_base = 0x4000,
 	.maxcpus_per_cluster = 8,
@@ -807,7 +807,7 @@ static void tegra194_cpufreq_remove(struct platform_device *pdev)
 static const struct of_device_id tegra194_cpufreq_of_match[] = {
 	{ .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc },
 	{ .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc },
-	{ .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc },
+	{ .compatible = "nvidia,tegra238-ccplex-cluster", .data = &tegra238_cpufreq_soc },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, tegra194_cpufreq_of_match);
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
index 3d1129aeed02b..a01abc1622eb7 100644
--- a/drivers/cpufreq/ti-cpufreq.c
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -502,16 +502,6 @@ static const struct of_device_id ti_cpufreq_of_match[]  __maybe_unused = {
 	{},
 };
 
-static const struct of_device_id *ti_cpufreq_match_node(void)
-{
-	struct device_node *np __free(device_node) = of_find_node_by_path("/");
-	const struct of_device_id *match;
-
-	match = of_match_node(ti_cpufreq_of_match, np);
-
-	return match;
-}
-
 static int ti_cpufreq_probe(struct platform_device *pdev)
 {
 	u32 version[VERSION_COUNT];
@@ -596,7 +586,7 @@ static int __init ti_cpufreq_init(void)
 	const struct of_device_id *match;
 
 	/* Check to ensure we are on a compatible platform */
-	match = ti_cpufreq_match_node();
+	match = of_machine_get_match(ti_cpufreq_of_match);
 	if (match)
 		platform_device_register_data(NULL, "ti-cpufreq", -1, match,
 					      sizeof(*match));
diff --git a/drivers/cpuidle/governors/gov.h b/drivers/cpuidle/governors/gov.h
index 99e067d9668c1..cd06a2e7b5067 100644
--- a/drivers/cpuidle/governors/gov.h
+++ b/drivers/cpuidle/governors/gov.h
@@ -10,5 +10,10 @@
  * check the time till the closest expected timer event.
  */
 #define RESIDENCY_THRESHOLD_NS	(15 * NSEC_PER_USEC)
+/*
+ * If the closest timer is in this range, the governor idle state selection need
+ * not be adjusted after the scheduler tick has been stopped.
+ */
+#define SAFE_TIMER_RANGE_NS	(2 * TICK_NSEC)
 
 #endif /* __CPUIDLE_GOVERNOR_H */
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 899ff16ff1fe2..544a5d5930077 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -261,13 +261,16 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns);
 		/*
 		 * If the tick is already stopped, the cost of possible short
-		 * idle duration misprediction is much higher, because the CPU
-		 * may be stuck in a shallow idle state for a long time as a
-		 * result of it.  In that case, say we might mispredict and use
-		 * the known time till the closest timer event for the idle
-		 * state selection.
+		 * idle duration misprediction is higher because the CPU may get
+		 * stuck in a shallow idle state then.  To avoid that, if
+		 * predicted_ns is small enough, say it might be mispredicted
+		 * and use the known time till the closest timer for idle state
+		 * selection unless that timer is going to trigger within
+		 * SAFE_TIMER_RANGE_NS in which case it can be regarded as a
+		 * sufficient safety net.
 		 */
-		if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC)
+		if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC &&
+		    data->next_timer_ns > SAFE_TIMER_RANGE_NS)
 			predicted_ns = data->next_timer_ns;
 	} else {
 		/*
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
index bec0142377b8f..ac43b9b013b3f 100644
--- a/drivers/cpuidle/governors/teo.c
+++ b/drivers/cpuidle/governors/teo.c
@@ -407,50 +407,13 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 	 * better choice.
 	 */
 	if (2 * idx_intercept_sum > cpu_data->total - idx_hit_sum) {
-		int min_idx = idx0;
-
-		if (tick_nohz_tick_stopped()) {
-			/*
-			 * Look for the shallowest idle state below the current
-			 * candidate one whose target residency is at least
-			 * equal to the tick period length.
-			 */
-			while (min_idx < idx &&
-			       drv->states[min_idx].target_residency_ns < TICK_NSEC)
-				min_idx++;
-
-			/*
-			 * Avoid selecting a state with a lower index, but with
-			 * the same target residency as the current candidate
-			 * one.
-			 */
-			if (drv->states[min_idx].target_residency_ns ==
-					drv->states[idx].target_residency_ns)
-				goto constraint;
-		}
-
-		/*
-		 * If the minimum state index is greater than or equal to the
-		 * index of the state with the maximum intercepts metric and
-		 * the corresponding state is enabled, there is no need to look
-		 * at the deeper states.
-		 */
-		if (min_idx >= intercept_max_idx &&
-		    !dev->states_usage[min_idx].disable) {
-			idx = min_idx;
-			goto constraint;
-		}
-
 		/*
 		 * Look for the deepest enabled idle state, at most as deep as
 		 * the one with the maximum intercepts metric, whose target
 		 * residency had not been greater than the idle duration in over
 		 * a half of the relevant cases in the past.
-		 *
-		 * Take the possible duration limitation present if the tick
-		 * has been stopped already into account.
 		 */
-		for (i = idx - 1, intercept_sum = 0; i >= min_idx; i--) {
+		for (i = idx - 1, intercept_sum = 0; i >= idx0; i--) {
 			intercept_sum += cpu_data->state_bins[i].intercepts;
 
 			if (dev->states_usage[i].disable)
@@ -463,7 +426,6 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		}
 	}
 
-constraint:
 	/*
 	 * If there is a latency constraint, it may be necessary to select an
 	 * idle state shallower than the current candidate one.
@@ -472,13 +434,13 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		idx = constraint_idx;
 
 	/*
-	 * If either the candidate state is state 0 or its target residency is
-	 * low enough, there is basically nothing more to do, but if the sleep
-	 * length is not updated, the subsequent wakeup will be counted as an
-	 * "intercept" which may be problematic in the cases when timer wakeups
-	 * are dominant.  Namely, it may effectively prevent deeper idle states
-	 * from being selected at one point even if no imminent timers are
-	 * scheduled.
+	 * If the tick has not been stopped and either the candidate state is
+	 * state 0 or its target residency is low enough, there is basically
+	 * nothing more to do, but if the sleep length is not updated, the
+	 * subsequent wakeup will be counted as an "intercept".  That may be
+	 * problematic in the cases when timer wakeups are dominant because it
+	 * may effectively prevent deeper idle states from being selected at one
+	 * point even if no imminent timers are scheduled.
 	 *
 	 * However, frequent timers in the RESIDENCY_THRESHOLD_NS range on one
 	 * CPU are unlikely (user space has a default 50 us slack value for
@@ -494,7 +456,8 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 	 * shallow idle states regardless of the wakeup type, so the sleep
 	 * length need not be known in that case.
 	 */
-	if ((!idx || drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS) &&
+	if (!tick_nohz_tick_stopped() && (!idx ||
+	     drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS) &&
 	    (2 * cpu_data->short_idles >= cpu_data->total ||
 	     latency_req < LATENCY_THRESHOLD_NS))
 		goto out_tick;
@@ -502,6 +465,30 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 	duration_ns = tick_nohz_get_sleep_length(&delta_tick);
 	cpu_data->sleep_length_ns = duration_ns;
 
+	/*
+	 * If the tick has been stopped and the closest timer is too far away,
+	 * update the selection to prevent the CPU from getting stuck in a
+	 * shallow idle state for too long.
+	 */
+	if (tick_nohz_tick_stopped() && duration_ns > SAFE_TIMER_RANGE_NS &&
+	    drv->states[idx].target_residency_ns < TICK_NSEC) {
+		/*
+		 * Look for the deepest enabled idle state with exit latency
+		 * within the PM QoS limit and with target residency within
+		 * duration_ns.
+		 */
+		for (i = constraint_idx; i > idx; i--) {
+			if (dev->states_usage[i].disable)
+				continue;
+
+			if (drv->states[i].target_residency_ns <= duration_ns) {
+				idx = i;
+				break;
+			}
+		}
+		return idx;
+	}
+
 	if (!idx)
 		goto out_tick;
 
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 322ae8854e3ea..283bbc650b5b2 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -38,7 +38,6 @@ obj-y += stm32/
 obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
 obj-$(CONFIG_CRYPTO_DEV_TEGRA) += tegra/
 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
-#obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
 obj-y += inside-secure/
 obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index b6a77c8d439cc..9c380351d2f9f 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -261,6 +261,7 @@ static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm)
 	if (IS_ERR(fallback)) {
 		dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n",
 			alg, PTR_ERR(fallback));
+		atmel_ecc_i2c_client_free(ctx->client);
 		return PTR_ERR(fallback);
 	}
 
diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c
index da3cd986b1eb7..0e275dbdc8c55 100644
--- a/drivers/crypto/atmel-i2c.c
+++ b/drivers/crypto/atmel-i2c.c
@@ -72,8 +72,8 @@ EXPORT_SYMBOL(atmel_i2c_init_read_config_cmd);
 
 int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr)
 {
-	if (addr < 0 || addr > OTP_ZONE_SIZE)
-		return -1;
+	if (addr >= OTP_ZONE_SIZE / 4)
+		return -EINVAL;
 
 	cmd->word_addr = COMMAND;
 	cmd->opcode = OPCODE_READ;
@@ -370,7 +370,7 @@ int atmel_i2c_probe(struct i2c_client *client)
 		}
 	}
 
-	if (bus_clk_rate > 1000000L) {
+	if (bus_clk_rate > I2C_MAX_FAST_MODE_PLUS_FREQ) {
 		dev_err(dev, "%u exceeds maximum supported clock frequency (1MHz)\n",
 			bus_clk_rate);
 		return -EINVAL;
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 1f1341a16c42a..002b629025534 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -404,20 +404,13 @@ static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
 
 static struct atmel_sha_dev *atmel_sha_find_dev(struct atmel_sha_ctx *tctx)
 {
-	struct atmel_sha_dev *dd = NULL;
-	struct atmel_sha_dev *tmp;
+	struct atmel_sha_dev *dd;
 
 	spin_lock_bh(&atmel_sha.lock);
-	if (!tctx->dd) {
-		list_for_each_entry(tmp, &atmel_sha.dev_list, list) {
-			dd = tmp;
-			break;
-		}
-		tctx->dd = dd;
-	} else {
-		dd = tctx->dd;
-	}
-
+	if (!tctx->dd)
+		tctx->dd = list_first_entry_or_null(&atmel_sha.dev_list,
+						    struct atmel_sha_dev, list);
+	dd = tctx->dd;
 	spin_unlock_bh(&atmel_sha.lock);
 
 	return dd;
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
index 98d1023007e31..691531647fd65 100644
--- a/drivers/crypto/atmel-sha204a.c
+++ b/drivers/crypto/atmel-sha204a.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/sysfs.h>
 #include <linux/workqueue.h>
 #include "atmel-i2c.h"
 
@@ -95,19 +96,24 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
 static int atmel_sha204a_otp_read(struct i2c_client *client, u16 addr, u8 *otp)
 {
 	struct atmel_i2c_cmd cmd;
-	int ret = -1;
+	int ret;
 
-	if (atmel_i2c_init_read_otp_cmd(&cmd, addr) < 0) {
+	ret = atmel_i2c_init_read_otp_cmd(&cmd, addr);
+	if (ret < 0) {
 		dev_err(&client->dev, "failed, invalid otp address %04X\n",
 			addr);
 		return ret;
 	}
 
 	ret = atmel_i2c_send_receive(client, &cmd);
+	if (ret < 0) {
+		dev_err(&client->dev, "failed to read otp at %04X\n", addr);
+		return ret;
+	}
 
 	if (cmd.data[0] == 0xff) {
 		dev_err(&client->dev, "failed, device not ready\n");
-		return -EINVAL;
+		return -EIO;
 	}
 
 	memcpy(otp, cmd.data+1, 4);
@@ -120,21 +126,22 @@ static ssize_t otp_show(struct device *dev,
 {
 	u16 addr;
 	u8 otp[OTP_ZONE_SIZE];
-	char *str = buf;
 	struct i2c_client *client = to_i2c_client(dev);
-	int i;
+	ssize_t len = 0;
+	int i, ret;
 
-	for (addr = 0; addr < OTP_ZONE_SIZE/4; addr++) {
-		if (atmel_sha204a_otp_read(client, addr, otp + addr * 4) < 0) {
+	for (addr = 0; addr < OTP_ZONE_SIZE / 4; addr++) {
+		ret = atmel_sha204a_otp_read(client, addr, otp + addr * 4);
+		if (ret < 0) {
 			dev_err(dev, "failed to read otp zone\n");
-			break;
+			return ret;
 		}
 	}
 
-	for (i = 0; i < addr*2; i++)
-		str += sprintf(str, "%02X", otp[i]);
-	str += sprintf(str, "\n");
-	return str - buf;
+	for (i = 0; i < OTP_ZONE_SIZE; i++)
+		len += sysfs_emit_at(buf, len, "%02X", otp[i]);
+	len += sysfs_emit_at(buf, len, "\n");
+	return len;
 }
 static DEVICE_ATTR_RO(otp);
 
@@ -174,10 +181,6 @@ static int atmel_sha204a_probe(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "failed to register RNG (%d)\n", ret);
 
-	/* otp read out */
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
 	ret = sysfs_create_group(&client->dev.kobj, &atmel_sha204a_groups);
 	if (ret) {
 		dev_err(&client->dev, "failed to register sysfs entry\n");
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index 278c0df3c92f2..643e507f9c020 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -294,8 +294,8 @@ static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 	} else {
-		dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
-					   dd->dma_size, DMA_FROM_DEVICE);
+		dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
+					dd->dma_size, DMA_FROM_DEVICE);
 
 		/* copy data */
 		count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
@@ -619,8 +619,8 @@ static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
 			dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 			dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 		} else {
-			dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
-				dd->dma_size, DMA_FROM_DEVICE);
+			dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
+						dd->dma_size, DMA_FROM_DEVICE);
 
 			/* copy data */
 			count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 167372936ca70..553994228a17b 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -19,6 +19,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/fsl/mc.h>
 #include <linux/kernel.h>
+#include <linux/string.h>
 #include <linux/string_choices.h>
 #include <soc/fsl/dpaa2-io.h>
 #include <soc/fsl/dpaa2-fd.h>
@@ -4644,16 +4645,12 @@ static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
 	alg = &halg->halg.base;
 
 	if (keyed) {
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->hmac_name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->hmac_driver_name);
+		strscpy(alg->cra_name, template->hmac_name);
+		strscpy(alg->cra_driver_name, template->hmac_driver_name);
 		t_alg->is_hmac = true;
 	} else {
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->driver_name);
+		strscpy(alg->cra_name, template->name);
+		strscpy(alg->cra_driver_name, template->driver_name);
 		t_alg->ahash_alg.setkey = NULL;
 		t_alg->is_hmac = false;
 	}
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 628c43a7efc41..e0a23c55c10e0 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -1913,16 +1913,12 @@ caam_hash_alloc(struct caam_hash_template *template,
 	alg = &halg->halg.base;
 
 	if (keyed) {
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->hmac_name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->hmac_driver_name);
+		strscpy(alg->cra_name, template->hmac_name);
+		strscpy(alg->cra_driver_name, template->hmac_driver_name);
 		t_alg->is_hmac = true;
 	} else {
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->driver_name);
+		strscpy(alg->cra_name, template->name);
+		strscpy(alg->cra_driver_name, template->driver_name);
 		halg->setkey = NULL;
 		t_alg->is_hmac = false;
 	}
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index 8b2dfc11289bd..4edeea6da071f 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -1965,11 +1965,11 @@ static int sev_get_firmware(struct device *dev,
 /* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
 static int sev_update_firmware(struct device *dev)
 {
-	struct sev_data_download_firmware *data;
+	struct sev_data_download_firmware data;
 	const struct firmware *firmware;
 	int ret, error, order;
 	struct page *p;
-	u64 data_size;
+	void *fw_blob;
 
 	if (!sev_version_greater_or_equal(0, 15)) {
 		dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
@@ -1981,16 +1981,7 @@ static int sev_update_firmware(struct device *dev)
 		return -1;
 	}
 
-	/*
-	 * SEV FW expects the physical address given to it to be 32
-	 * byte aligned. Memory allocated has structure placed at the
-	 * beginning followed by the firmware being passed to the SEV
-	 * FW. Allocate enough memory for data structure + alignment
-	 * padding + SEV FW.
-	 */
-	data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32);
-
-	order = get_order(firmware->size + data_size);
+	order = get_order(firmware->size);
 	p = alloc_pages(GFP_KERNEL, order);
 	if (!p) {
 		ret = -1;
@@ -2001,20 +1992,20 @@ static int sev_update_firmware(struct device *dev)
 	 * Copy firmware data to a kernel allocated contiguous
 	 * memory region.
 	 */
-	data = page_address(p);
-	memcpy(page_address(p) + data_size, firmware->data, firmware->size);
+	fw_blob = page_address(p);
+	memcpy(fw_blob, firmware->data, firmware->size);
 
-	data->address = __psp_pa(page_address(p) + data_size);
-	data->len = firmware->size;
+	data.address = __psp_pa(fw_blob);
+	data.len = firmware->size;
 
-	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
+	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
 
 	/*
 	 * A quirk for fixing the committed TCB version, when upgrading from
 	 * earlier firmware version than 1.50.
 	 */
 	if (!ret && !sev_version_greater_or_equal(1, 50))
-		ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
+		ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
 
 	if (ret)
 		dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
@@ -2408,10 +2399,8 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
 		 * in Firmware state on failure. Use snp_reclaim_pages() to
 		 * transition either case back to Hypervisor-owned state.
 		 */
-		if (snp_reclaim_pages(__pa(data), 1, true)) {
-			snp_leak_pages(__page_to_pfn(status_page), 1);
+		if (snp_reclaim_pages(__pa(data), 1, true))
 			return -EFAULT;
-		}
 	}
 
 	if (ret)
diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c
index b7fd9795062d4..76858bb4fcc22 100644
--- a/drivers/crypto/inside-secure/eip93/eip93-main.c
+++ b/drivers/crypto/inside-secure/eip93/eip93-main.c
@@ -36,6 +36,14 @@ static struct eip93_alg_template *eip93_algs[] = {
 	&eip93_alg_cbc_aes,
 	&eip93_alg_ctr_aes,
 	&eip93_alg_rfc3686_aes,
+	&eip93_alg_md5,
+	&eip93_alg_sha1,
+	&eip93_alg_sha224,
+	&eip93_alg_sha256,
+	&eip93_alg_hmac_md5,
+	&eip93_alg_hmac_sha1,
+	&eip93_alg_hmac_sha224,
+	&eip93_alg_hmac_sha256,
 	&eip93_alg_authenc_hmac_md5_cbc_des,
 	&eip93_alg_authenc_hmac_sha1_cbc_des,
 	&eip93_alg_authenc_hmac_sha224_cbc_des,
@@ -52,14 +60,6 @@ static struct eip93_alg_template *eip93_algs[] = {
 	&eip93_alg_authenc_hmac_sha1_rfc3686_aes,
 	&eip93_alg_authenc_hmac_sha224_rfc3686_aes,
 	&eip93_alg_authenc_hmac_sha256_rfc3686_aes,
-	&eip93_alg_md5,
-	&eip93_alg_sha1,
-	&eip93_alg_sha224,
-	&eip93_alg_sha256,
-	&eip93_alg_hmac_md5,
-	&eip93_alg_hmac_sha1,
-	&eip93_alg_hmac_sha224,
-	&eip93_alg_hmac_sha256,
 };
 
 inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask)
diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h
index 0490b8d151311..116b3fbb6ad79 100644
--- a/drivers/crypto/inside-secure/eip93/eip93-regs.h
+++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h
@@ -109,7 +109,7 @@
 #define EIP93_REG_PE_BUF_THRESH			0x10c
 #define   EIP93_PE_OUTBUF_THRESH		GENMASK(23, 16)
 #define   EIP93_PE_INBUF_THRESH			GENMASK(7, 0)
-#define EIP93_REG_PE_INBUF_COUNT		0x100
+#define EIP93_REG_PE_INBUF_COUNT		0x110
 #define EIP93_REG_PE_OUTBUF_COUNT		0x114
 #define EIP93_REG_PE_BUF_RW_PNTR		0x118 /* BUF_PNTR */
 
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 660f45ab86472..fb4936e7afa2d 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -1204,12 +1204,13 @@ static struct safexcel_alg_template *safexcel_algs[] = {
 	&safexcel_alg_hmac_sha256,
 	&safexcel_alg_hmac_sha384,
 	&safexcel_alg_hmac_sha512,
+	&safexcel_alg_authenc_hmac_md5_cbc_aes,
 	&safexcel_alg_authenc_hmac_sha1_cbc_aes,
 	&safexcel_alg_authenc_hmac_sha224_cbc_aes,
 	&safexcel_alg_authenc_hmac_sha256_cbc_aes,
 	&safexcel_alg_authenc_hmac_sha384_cbc_aes,
 	&safexcel_alg_authenc_hmac_sha512_cbc_aes,
-	&safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
+	&safexcel_alg_authenc_hmac_md5_ctr_aes,
 	&safexcel_alg_authenc_hmac_sha1_ctr_aes,
 	&safexcel_alg_authenc_hmac_sha224_ctr_aes,
 	&safexcel_alg_authenc_hmac_sha256_ctr_aes,
@@ -1241,11 +1242,14 @@ static struct safexcel_alg_template *safexcel_algs[] = {
 	&safexcel_alg_hmac_sha3_256,
 	&safexcel_alg_hmac_sha3_384,
 	&safexcel_alg_hmac_sha3_512,
-	&safexcel_alg_authenc_hmac_sha1_cbc_des,
+	&safexcel_alg_authenc_hmac_md5_cbc_des3_ede,
+	&safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
 	&safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
 	&safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
 	&safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
 	&safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
+	&safexcel_alg_authenc_hmac_md5_cbc_des,
+	&safexcel_alg_authenc_hmac_sha1_cbc_des,
 	&safexcel_alg_authenc_hmac_sha256_cbc_des,
 	&safexcel_alg_authenc_hmac_sha224_cbc_des,
 	&safexcel_alg_authenc_hmac_sha512_cbc_des,
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index 0f27367a85fa2..52fd460c0e9bb 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -945,12 +945,13 @@ extern struct safexcel_alg_template safexcel_alg_hmac_sha224;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha256;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha384;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha512;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes;
-extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_ctr_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes;
@@ -982,11 +983,14 @@ extern struct safexcel_alg_template safexcel_alg_hmac_sha3_224;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha3_256;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha3_384;
 extern struct safexcel_alg_template safexcel_alg_hmac_sha3_512;
-extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des3_ede;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des;
 extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des;
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index 27b1800574173..a8349b684693e 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -17,6 +17,7 @@
 #include <crypto/internal/des.h>
 #include <crypto/gcm.h>
 #include <crypto/ghash.h>
+#include <crypto/md5.h>
 #include <crypto/poly1305.h>
 #include <crypto/sha1.h>
 #include <crypto/sha2.h>
@@ -462,6 +463,9 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 
 	/* Auth key */
 	switch (ctx->hash_alg) {
+	case CONTEXT_CONTROL_CRYPTO_ALG_MD5:
+		alg = "safexcel-md5";
+		break;
 	case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 		alg = "safexcel-sha1";
 		break;
@@ -1662,6 +1666,42 @@ static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
 	return 0;
 }
 
+static int safexcel_aead_md5_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_cra_init(tfm);
+	ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
+	ctx->state_sz = MD5_DIGEST_SIZE;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_aes = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5,
+	.alg.aead = {
+		.setkey = safexcel_aead_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = AES_BLOCK_SIZE,
+		.maxauthsize = MD5_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(md5),cbc(aes))",
+			.cra_driver_name = "safexcel-authenc-hmac-md5-cbc-aes",
+			.cra_priority = SAFEXCEL_CRA_PRIORITY,
+			.cra_flags = CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_ALLOCATES_MEMORY |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_md5_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -1842,6 +1882,43 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
 	},
 };
 
+static int safexcel_aead_md5_des3_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_md5_cra_init(tfm);
+	ctx->alg = SAFEXCEL_3DES; /* override default */
+	ctx->blocksz = DES3_EDE_BLOCK_SIZE;
+	ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des3_ede = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5,
+	.alg.aead = {
+		.setkey = safexcel_aead_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = DES3_EDE_BLOCK_SIZE,
+		.maxauthsize = MD5_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+			.cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des3_ede",
+			.cra_priority = SAFEXCEL_CRA_PRIORITY,
+			.cra_flags = CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_ALLOCATES_MEMORY |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_md5_des3_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -2027,6 +2104,43 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = {
 	},
 };
 
+static int safexcel_aead_md5_des_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_md5_cra_init(tfm);
+	ctx->alg = SAFEXCEL_DES; /* override default */
+	ctx->blocksz = DES_BLOCK_SIZE;
+	ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_cbc_des = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_MD5,
+	.alg.aead = {
+		.setkey = safexcel_aead_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = DES_BLOCK_SIZE,
+		.maxauthsize = MD5_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(md5),cbc(des))",
+			.cra_driver_name = "safexcel-authenc-hmac-md5-cbc-des",
+			.cra_priority = SAFEXCEL_CRA_PRIORITY,
+			.cra_flags = CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_ALLOCATES_MEMORY |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = DES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_md5_des_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -2212,6 +2326,41 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = {
 	},
 };
 
+static int safexcel_aead_md5_ctr_cra_init(struct crypto_tfm *tfm)
+{
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	safexcel_aead_md5_cra_init(tfm);
+	ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+	return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_md5_ctr_aes = {
+	.type = SAFEXCEL_ALG_TYPE_AEAD,
+	.algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_MD5,
+	.alg.aead = {
+		.setkey = safexcel_aead_setkey,
+		.encrypt = safexcel_aead_encrypt,
+		.decrypt = safexcel_aead_decrypt,
+		.ivsize = CTR_RFC3686_IV_SIZE,
+		.maxauthsize = MD5_DIGEST_SIZE,
+		.base = {
+			.cra_name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
+			.cra_driver_name = "safexcel-authenc-hmac-md5-ctr-aes",
+			.cra_priority = SAFEXCEL_CRA_PRIORITY,
+			.cra_flags = CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_ALLOCATES_MEMORY |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+			.cra_alignmask = 0,
+			.cra_init = safexcel_aead_md5_ctr_cra_init,
+			.cra_exit = safexcel_aead_cra_exit,
+			.cra_module = THIS_MODULE,
+		},
+	},
+};
+
 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
 {
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c
index f9f1018a28236..09d4f547e082c 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
 /* Copyright(c) 2014 - 2020 Intel Corporation */
+#include <linux/delay.h>
 #include <linux/firmware.h>
 #include <linux/pci.h>
 #include "adf_cfg.h"
@@ -162,8 +163,14 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev)
 static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae)
 {
 	struct adf_fw_loader_data *loader_data = accel_dev->fw_loader;
+	unsigned long reset_delay;
 
 	qat_hal_reset(loader_data->fw_loader);
+
+	reset_delay = loader_data->fw_loader->chip_info->reset_delay_us;
+	if (reset_delay)
+		fsleep(reset_delay);
+
 	if (qat_hal_clr_reset(loader_data->fw_loader))
 		return -EFAULT;
 
diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c
index f31556beed8b6..89bfd8761d756 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c
@@ -321,7 +321,7 @@ static ssize_t cap_rem_show(struct device *dev, struct device_attribute *attr,
 {
 	struct adf_rl_interface_data *data;
 	struct adf_accel_dev *accel_dev;
-	int ret, rem_cap;
+	int rem_cap;
 
 	accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev));
 	if (!accel_dev)
@@ -336,23 +336,19 @@ static ssize_t cap_rem_show(struct device *dev, struct device_attribute *attr,
 	if (rem_cap < 0)
 		return rem_cap;
 
-	ret = sysfs_emit(buf, "%u\n", rem_cap);
-
-	return ret;
+	return sysfs_emit(buf, "%u\n", rem_cap);
 }
 
 static ssize_t cap_rem_store(struct device *dev, struct device_attribute *attr,
 			     const char *buf, size_t count)
 {
-	unsigned int val;
 	int ret;
 
 	ret = sysfs_match_string(rl_services, buf);
 	if (ret < 0)
 		return ret;
 
-	val = ret;
-	ret = set_param_u(dev, CAP_REM_SRV, val);
+	ret = set_param_u(dev, CAP_REM_SRV, ret);
 	if (ret)
 		return ret;
 
diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h
index 6887930c7995e..e74cafa95f1cc 100644
--- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h
+++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h
@@ -27,6 +27,7 @@ struct icp_qat_fw_loader_chip_info {
 	int mmp_sram_size;
 	bool nn;
 	bool lm2lm3;
+	u16 reset_delay_us;
 	u32 lm_size;
 	u32 icp_rst_csr;
 	u32 icp_rst_mask;
diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c
index 7a6ba6f22e3e8..1c3d1311f1c7a 100644
--- a/drivers/crypto/intel/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c
@@ -9,17 +9,18 @@
 #include "icp_qat_hal.h"
 #include "icp_qat_uclo.h"
 
-#define BAD_REGADDR	       0xffff
-#define MAX_RETRY_TIMES	   10000
-#define INIT_CTX_ARB_VALUE	0x0
-#define INIT_CTX_ENABLE_VALUE     0x0
-#define INIT_PC_VALUE	     0x0
-#define INIT_WAKEUP_EVENTS_VALUE  0x1
-#define INIT_SIG_EVENTS_VALUE     0x1
-#define INIT_CCENABLE_VALUE       0x2000
-#define RST_CSR_QAT_LSB	   20
-#define RST_CSR_AE_LSB		  0
-#define MC_TIMESTAMP_ENABLE       (0x1 << 7)
+#define BAD_REGADDR			0xffff
+#define MAX_RETRY_TIMES			10000
+#define INIT_CTX_ARB_VALUE		0x0
+#define INIT_CTX_ENABLE_VALUE		0x0
+#define INIT_PC_VALUE			0x0
+#define INIT_WAKEUP_EVENTS_VALUE	0x1
+#define INIT_SIG_EVENTS_VALUE		0x1
+#define INIT_CCENABLE_VALUE		0x2000
+#define RST_CSR_QAT_LSB			20
+#define RST_CSR_AE_LSB			0
+#define MC_TIMESTAMP_ENABLE		(0x1 << 7)
+#define MIN_RESET_DELAY_US		3
 
 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
 	(~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
@@ -713,8 +714,10 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 		handle->chip_info->wakeup_event_val = 0x80000000;
 		handle->chip_info->fw_auth = true;
 		handle->chip_info->css_3k = true;
-		if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX)
+		if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) {
 			handle->chip_info->dual_sign = true;
+			handle->chip_info->reset_delay_us = MIN_RESET_DELAY_US;
+		}
 		handle->chip_info->tgroup_share_ustore = true;
 		handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
 		handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
index 09e6a8474d1ab..e0f38d32bc935 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
@@ -10,6 +10,7 @@
 
 #include <linux/ctype.h>
 #include <linux/firmware.h>
+#include <linux/string.h>
 #include <linux/string_choices.h>
 #include "otx_cpt_common.h"
 #include "otx_cptpf_ucode.h"
@@ -509,13 +510,12 @@ EXPORT_SYMBOL_GPL(otx_cpt_uc_supports_eng_type);
 static void print_ucode_info(struct otx_cpt_eng_grp_info *eng_grp,
 			     char *buf, int size)
 {
-	if (eng_grp->mirror.is_ena) {
+	if (eng_grp->mirror.is_ena)
 		scnprintf(buf, size, "%s (shared with engine_group%d)",
 			  eng_grp->g->grp[eng_grp->mirror.idx].ucode[0].ver_str,
 			  eng_grp->mirror.idx);
-	} else {
-		scnprintf(buf, size, "%s", eng_grp->ucode[0].ver_str);
-	}
+	else
+		strscpy(buf, eng_grp->ucode[0].ver_str, size);
 }
 
 static void print_engs_info(struct otx_cpt_eng_grp_info *eng_grp,
diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c
index 329f60ad422e6..9214bbfc868f5 100644
--- a/drivers/crypto/padlock-sha.c
+++ b/drivers/crypto/padlock-sha.c
@@ -332,6 +332,13 @@ static int __init padlock_init(void)
 	if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN))
 		return -ENODEV;
 
+	/*
+	 * Skip family 0x07 and newer used by Zhaoxin processors,
+	 * as the driver's self-tests fail on these CPUs.
+	 */
+	if (c->x86 >= 0x07)
+		return -ENODEV;
+
 	/* Register the newly added algorithm module if on *
 	* VIA Nano processor, or else just do as before */
 	if (c->x86_model < 0x0f) {
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index 846e1d42775d1..79e92bc3f7d33 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -5,6 +5,7 @@
  */
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
+#include <linux/string.h>
 #include <crypto/gcm.h>
 #include <crypto/authenc.h>
 #include <crypto/internal/aead.h>
@@ -35,7 +36,6 @@ static void qce_aead_done(void *data)
 	u32 status;
 	unsigned int totallen;
 	unsigned char tag[SHA256_DIGEST_SIZE] = {0};
-	int ret = 0;
 
 	diff_dst = (req->src != req->dst) ? true : false;
 	dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
@@ -79,8 +79,7 @@ static void qce_aead_done(void *data)
 	} else if (!IS_CCM(rctx->flags)) {
 		totallen = req->cryptlen + req->assoclen - ctx->authsize;
 		scatterwalk_map_and_copy(tag, req->src, totallen, ctx->authsize, 0);
-		ret = memcmp(result_buf->auth_iv, tag, ctx->authsize);
-		if (ret) {
+		if (memcmp(result_buf->auth_iv, tag, ctx->authsize)) {
 			pr_err("Bad message error\n");
 			error = -EBADMSG;
 		}
@@ -144,16 +143,12 @@ qce_aead_prepare_dst_buf(struct aead_request *req)
 
 		sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg,
 				     rctx->assoclen);
-		if (IS_ERR(sg)) {
-			ret = PTR_ERR(sg);
+		if (IS_ERR(sg))
 			goto dst_tbl_free;
-		}
 		/* dst buffer */
 		sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen);
-		if (IS_ERR(sg)) {
-			ret = PTR_ERR(sg);
+		if (IS_ERR(sg))
 			goto dst_tbl_free;
-		}
 		totallen = rctx->cryptlen + rctx->assoclen;
 	} else {
 		if (totallen) {
@@ -768,9 +763,8 @@ static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_devi
 
 	alg = &tmpl->alg.aead;
 
-	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
-	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		 def->drv_name);
+	strscpy(alg->base.cra_name, def->name);
+	strscpy(alg->base.cra_driver_name, def->drv_name);
 
 	alg->base.cra_blocksize		= def->blocksize;
 	alg->chunksize			= def->chunksize;
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index 402e4e64347dc..1b37121cbcdcb 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -6,6 +6,7 @@
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
+#include <linux/string.h>
 #include <crypto/internal/hash.h>
 
 #include "common.h"
@@ -489,9 +490,8 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def,
 	base->cra_module = THIS_MODULE;
 	base->cra_init = qce_ahash_cra_init;
 
-	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
-	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		 def->drv_name);
+	strscpy(base->cra_name, def->name);
+	strscpy(base->cra_driver_name, def->drv_name);
 
 	INIT_LIST_HEAD(&tmpl->entry);
 	tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AHASH;
diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
index 4ad3a17020101..db0b648a56eb1 100644
--- a/drivers/crypto/qce/skcipher.c
+++ b/drivers/crypto/qce/skcipher.c
@@ -7,6 +7,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/moduleparam.h>
+#include <linux/string.h>
 #include <linux/types.h>
 #include <linux/errno.h>
 #include <crypto/aes.h>
@@ -446,9 +447,8 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def,
 
 	alg = &tmpl->alg.skcipher;
 
-	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
-	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		 def->drv_name);
+	strscpy(alg->base.cra_name, def->name);
+	strscpy(alg->base.cra_driver_name, def->drv_name);
 
 	alg->base.cra_blocksize		= def->blocksize;
 	alg->chunksize			= def->chunksize;
diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c
index c1dc1e43e117e..2e2d97d17e6c3 100644
--- a/drivers/crypto/starfive/jh7110-aes.c
+++ b/drivers/crypto/starfive/jh7110-aes.c
@@ -1003,7 +1003,7 @@ static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
 
 static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm)
 {
-	return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac(aes-lib))");
+	return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac-aes-lib)");
 }
 
 static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm)
diff --git a/drivers/crypto/ti/Kconfig b/drivers/crypto/ti/Kconfig
index a3692ceec49bc..6027e12de279d 100644
--- a/drivers/crypto/ti/Kconfig
+++ b/drivers/crypto/ti/Kconfig
@@ -6,6 +6,7 @@ config CRYPTO_DEV_TI_DTHEV2
 	select CRYPTO_SKCIPHER
 	select CRYPTO_ECB
 	select CRYPTO_CBC
+	select CRYPTO_CTR
 	select CRYPTO_XTS
 	help
 	  This enables support for the TI DTHE V2 hw cryptography engine
diff --git a/drivers/crypto/ti/dthev2-aes.c b/drivers/crypto/ti/dthev2-aes.c
index 156729ccc50ec..bf7d4dcb4cd7d 100644
--- a/drivers/crypto/ti/dthev2-aes.c
+++ b/drivers/crypto/ti/dthev2-aes.c
@@ -63,6 +63,7 @@
 enum aes_ctrl_mode_masks {
 	AES_CTRL_ECB_MASK = 0x00,
 	AES_CTRL_CBC_MASK = BIT(5),
+	AES_CTRL_CTR_MASK = BIT(6),
 	AES_CTRL_XTS_MASK = BIT(12) | BIT(11),
 };
 
@@ -74,6 +75,8 @@ enum aes_ctrl_mode_masks {
 #define DTHE_AES_CTRL_KEYSIZE_24B		BIT(4)
 #define DTHE_AES_CTRL_KEYSIZE_32B		(BIT(3) | BIT(4))
 
+#define DTHE_AES_CTRL_CTR_WIDTH_128B		(BIT(7) | BIT(8))
+
 #define DTHE_AES_CTRL_SAVE_CTX_SET		BIT(29)
 
 #define DTHE_AES_CTRL_OUTPUT_READY		BIT_MASK(0)
@@ -100,25 +103,27 @@ static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm)
 	return 0;
 }
 
-static int dthe_cipher_xts_init_tfm(struct crypto_skcipher *tfm)
+static int dthe_cipher_init_tfm_fallback(struct crypto_skcipher *tfm)
 {
 	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 	struct dthe_data *dev_data = dthe_get_dev(ctx);
+	const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm));
 
 	ctx->dev_data = dev_data;
 	ctx->keylen = 0;
 
-	ctx->skcipher_fb = crypto_alloc_sync_skcipher("xts(aes)", 0,
+	ctx->skcipher_fb = crypto_alloc_sync_skcipher(alg_name, 0,
 						      CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(ctx->skcipher_fb)) {
-		dev_err(dev_data->dev, "fallback driver xts(aes) couldn't be loaded\n");
+		dev_err(dev_data->dev, "fallback driver %s couldn't be loaded\n",
+			alg_name);
 		return PTR_ERR(ctx->skcipher_fb);
 	}
 
 	return 0;
 }
 
-static void dthe_cipher_xts_exit_tfm(struct crypto_skcipher *tfm)
+static void dthe_cipher_exit_tfm(struct crypto_skcipher *tfm)
 {
 	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 
@@ -156,6 +161,24 @@ static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsig
 	return dthe_aes_setkey(tfm, key, keylen);
 }
 
+static int dthe_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen)
+{
+	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
+	int ret = dthe_aes_setkey(tfm, key, keylen);
+
+	if (ret)
+		return ret;
+
+	ctx->aes_mode = DTHE_AES_CTR;
+
+	crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK);
+	crypto_sync_skcipher_set_flags(ctx->skcipher_fb,
+				       crypto_skcipher_get_flags(tfm) &
+				       CRYPTO_TFM_REQ_MASK);
+
+	return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen);
+}
+
 static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen)
 {
 	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
@@ -171,8 +194,8 @@ static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsig
 
 	crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK);
 	crypto_sync_skcipher_set_flags(ctx->skcipher_fb,
-				  crypto_skcipher_get_flags(tfm) &
-				  CRYPTO_TFM_REQ_MASK);
+				       crypto_skcipher_get_flags(tfm) &
+				       CRYPTO_TFM_REQ_MASK);
 
 	return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen);
 }
@@ -236,6 +259,10 @@ static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx,
 	case DTHE_AES_CBC:
 		ctrl_val |= AES_CTRL_CBC_MASK;
 		break;
+	case DTHE_AES_CTR:
+		ctrl_val |= AES_CTRL_CTR_MASK;
+		ctrl_val |= DTHE_AES_CTRL_CTR_WIDTH_128B;
+		break;
 	case DTHE_AES_XTS:
 		ctrl_val |= AES_CTRL_XTS_MASK;
 		break;
@@ -251,6 +278,22 @@ static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx,
 	writel_relaxed(ctrl_val, aes_base_reg + DTHE_P_AES_CTRL);
 }
 
+static int dthe_aes_do_fallback(struct skcipher_request *req)
+{
+	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+	struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req);
+
+	SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb);
+
+	skcipher_request_set_callback(subreq, skcipher_request_flags(req),
+				      req->base.complete, req->base.data);
+	skcipher_request_set_crypt(subreq, req->src, req->dst,
+				   req->cryptlen, req->iv);
+
+	return rctx->enc ? crypto_skcipher_encrypt(subreq) :
+		crypto_skcipher_decrypt(subreq);
+}
+
 static void dthe_aes_dma_in_callback(void *data)
 {
 	struct skcipher_request *req = (struct skcipher_request *)data;
@@ -271,7 +314,7 @@ static int dthe_aes_run(struct crypto_engine *engine, void *areq)
 	struct scatterlist *dst = req->dst;
 
 	int src_nents = sg_nents_for_len(src, len);
-	int dst_nents;
+	int dst_nents = sg_nents_for_len(dst, len);
 
 	int src_mapped_nents;
 	int dst_mapped_nents;
@@ -305,25 +348,62 @@ static int dthe_aes_run(struct crypto_engine *engine, void *areq)
 		dst_dir  = DMA_FROM_DEVICE;
 	}
 
+	/*
+	 * CTR mode can operate on any input length, but the hardware
+	 * requires input length to be a multiple of the block size.
+	 * We need to handle the padding in the driver.
+	 */
+	if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) {
+		unsigned int pad_size = AES_BLOCK_SIZE - (req->cryptlen % AES_BLOCK_SIZE);
+		u8 *pad_buf = rctx->padding;
+		struct scatterlist *sg;
+
+		len += pad_size;
+		src_nents++;
+		dst_nents++;
+
+		src = kmalloc_array(src_nents, sizeof(*src), GFP_ATOMIC);
+		if (!src) {
+			ret = -ENOMEM;
+			goto aes_ctr_src_alloc_err;
+		}
+
+		sg_init_table(src, src_nents);
+		sg = dthe_copy_sg(src, req->src, req->cryptlen);
+		memzero_explicit(pad_buf, AES_BLOCK_SIZE);
+		sg_set_buf(sg, pad_buf, pad_size);
+
+		if (diff_dst) {
+			dst = kmalloc_array(dst_nents, sizeof(*dst), GFP_ATOMIC);
+			if (!dst) {
+				ret = -ENOMEM;
+				goto aes_ctr_dst_alloc_err;
+			}
+
+			sg_init_table(dst, dst_nents);
+			sg = dthe_copy_sg(dst, req->dst, req->cryptlen);
+			sg_set_buf(sg, pad_buf, pad_size);
+		} else {
+			dst = src;
+		}
+	}
+
 	tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx);
 	rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx);
 
 	src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir);
 	if (src_mapped_nents == 0) {
 		ret = -EINVAL;
-		goto aes_err;
+		goto aes_map_src_err;
 	}
 
 	if (!diff_dst) {
-		dst_nents = src_nents;
 		dst_mapped_nents = src_mapped_nents;
 	} else {
-		dst_nents = sg_nents_for_len(dst, len);
 		dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir);
 		if (dst_mapped_nents == 0) {
-			dma_unmap_sg(tx_dev, src, src_nents, src_dir);
 			ret = -EINVAL;
-			goto aes_err;
+			goto aes_map_dst_err;
 		}
 	}
 
@@ -353,8 +433,8 @@ static int dthe_aes_run(struct crypto_engine *engine, void *areq)
 	else
 		dthe_aes_set_ctrl_key(ctx, rctx, (u32 *)req->iv);
 
-	writel_relaxed(lower_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_0);
-	writel_relaxed(upper_32_bits(req->cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_1);
+	writel_relaxed(lower_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_0);
+	writel_relaxed(upper_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_1);
 
 	dmaengine_submit(desc_in);
 	dmaengine_submit(desc_out);
@@ -386,11 +466,26 @@ static int dthe_aes_run(struct crypto_engine *engine, void *areq)
 	}
 
 aes_prep_err:
-	dma_unmap_sg(tx_dev, src, src_nents, src_dir);
 	if (dst_dir != DMA_BIDIRECTIONAL)
 		dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir);
+aes_map_dst_err:
+	dma_unmap_sg(tx_dev, src, src_nents, src_dir);
+
+aes_map_src_err:
+	if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) {
+		memzero_explicit(rctx->padding, AES_BLOCK_SIZE);
+		if (diff_dst)
+			kfree(dst);
+aes_ctr_dst_alloc_err:
+		kfree(src);
+aes_ctr_src_alloc_err:
+		/*
+		 * Fallback to software if ENOMEM
+		 */
+		if (ret == -ENOMEM)
+			ret = dthe_aes_do_fallback(req);
+	}
 
-aes_err:
 	local_bh_disable();
 	crypto_finalize_skcipher_request(dev_data->engine, req, ret);
 	local_bh_enable();
@@ -400,7 +495,6 @@ static int dthe_aes_run(struct crypto_engine *engine, void *areq)
 static int dthe_aes_crypt(struct skcipher_request *req)
 {
 	struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
-	struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req);
 	struct dthe_data *dev_data = dthe_get_dev(ctx);
 	struct crypto_engine *engine;
 
@@ -408,20 +502,14 @@ static int dthe_aes_crypt(struct skcipher_request *req)
 	 * If data is not a multiple of AES_BLOCK_SIZE:
 	 * - need to return -EINVAL for ECB, CBC as they are block ciphers
 	 * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS
+	 * - do nothing for CTR
 	 */
 	if (req->cryptlen % AES_BLOCK_SIZE) {
-		if (ctx->aes_mode == DTHE_AES_XTS) {
-			SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb);
-
-			skcipher_request_set_callback(subreq, skcipher_request_flags(req),
-						      req->base.complete, req->base.data);
-			skcipher_request_set_crypt(subreq, req->src, req->dst,
-						   req->cryptlen, req->iv);
+		if (ctx->aes_mode == DTHE_AES_XTS)
+			return dthe_aes_do_fallback(req);
 
-			return rctx->enc ? crypto_skcipher_encrypt(subreq) :
-				crypto_skcipher_decrypt(subreq);
-		}
-		return -EINVAL;
+		if (ctx->aes_mode != DTHE_AES_CTR)
+			return -EINVAL;
 	}
 
 	/*
@@ -501,8 +589,33 @@ static struct skcipher_engine_alg cipher_algs[] = {
 		.op.do_one_request = dthe_aes_run,
 	}, /* CBC AES */
 	{
-		.base.init			= dthe_cipher_xts_init_tfm,
-		.base.exit			= dthe_cipher_xts_exit_tfm,
+		.base.init			= dthe_cipher_init_tfm_fallback,
+		.base.exit			= dthe_cipher_exit_tfm,
+		.base.setkey			= dthe_aes_ctr_setkey,
+		.base.encrypt			= dthe_aes_encrypt,
+		.base.decrypt			= dthe_aes_decrypt,
+		.base.min_keysize		= AES_MIN_KEY_SIZE,
+		.base.max_keysize		= AES_MAX_KEY_SIZE,
+		.base.ivsize			= AES_IV_SIZE,
+		.base.chunksize			= AES_BLOCK_SIZE,
+		.base.base = {
+			.cra_name		= "ctr(aes)",
+			.cra_driver_name	= "ctr-aes-dthev2",
+			.cra_priority		= 299,
+			.cra_flags		= CRYPTO_ALG_TYPE_SKCIPHER |
+						  CRYPTO_ALG_ASYNC |
+						  CRYPTO_ALG_KERN_DRIVER_ONLY |
+						  CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize		= 1,
+			.cra_ctxsize		= sizeof(struct dthe_tfm_ctx),
+			.cra_reqsize		= sizeof(struct dthe_aes_req_ctx),
+			.cra_module		= THIS_MODULE,
+		},
+		.op.do_one_request = dthe_aes_run,
+	}, /* CTR AES */
+	{
+		.base.init			= dthe_cipher_init_tfm_fallback,
+		.base.exit			= dthe_cipher_exit_tfm,
 		.base.setkey			= dthe_aes_xts_setkey,
 		.base.encrypt			= dthe_aes_encrypt,
 		.base.decrypt			= dthe_aes_decrypt,
diff --git a/drivers/crypto/ti/dthev2-common.c b/drivers/crypto/ti/dthev2-common.c
index c39d37933b9ee..a2ad79bec105a 100644
--- a/drivers/crypto/ti/dthev2-common.c
+++ b/drivers/crypto/ti/dthev2-common.c
@@ -48,6 +48,25 @@ struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx)
 	return dev_data;
 }
 
+struct scatterlist *dthe_copy_sg(struct scatterlist *dst,
+				 struct scatterlist *src,
+				 int buflen)
+{
+	struct scatterlist *from_sg, *to_sg;
+	int sglen;
+
+	for (to_sg = dst, from_sg = src; buflen && from_sg; buflen -= sglen) {
+		sglen = from_sg->length;
+		if (sglen > buflen)
+			sglen = buflen;
+		sg_set_buf(to_sg, sg_virt(from_sg), sglen);
+		from_sg = sg_next(from_sg);
+		to_sg = sg_next(to_sg);
+	}
+
+	return to_sg;
+}
+
 static int dthe_dma_init(struct dthe_data *dev_data)
 {
 	int ret;
diff --git a/drivers/crypto/ti/dthev2-common.h b/drivers/crypto/ti/dthev2-common.h
index c7a06a4c353ff..5239ee93c9442 100644
--- a/drivers/crypto/ti/dthev2-common.h
+++ b/drivers/crypto/ti/dthev2-common.h
@@ -36,6 +36,7 @@
 enum dthe_aes_mode {
 	DTHE_AES_ECB = 0,
 	DTHE_AES_CBC,
+	DTHE_AES_CTR,
 	DTHE_AES_XTS,
 };
 
@@ -92,10 +93,12 @@ struct dthe_tfm_ctx {
 /**
  * struct dthe_aes_req_ctx - AES engine req ctx struct
  * @enc: flag indicating encryption or decryption operation
+ * @padding: padding buffer for handling unaligned data
  * @aes_compl: Completion variable for use in manual completion in case of DMA callback failure
  */
 struct dthe_aes_req_ctx {
 	int enc;
+	u8 padding[AES_BLOCK_SIZE];
 	struct completion aes_compl;
 };
 
@@ -103,6 +106,20 @@ struct dthe_aes_req_ctx {
 
 struct dthe_data *dthe_get_dev(struct dthe_tfm_ctx *ctx);
 
+/**
+ * dthe_copy_sg - Copy sg entries from src to dst
+ * @dst: Destination sg to be filled
+ * @src: Source sg to be copied from
+ * @buflen: Number of bytes to be copied
+ *
+ * Description:
+ *   Copy buflen bytes of data from src to dst.
+ *
+ **/
+struct scatterlist *dthe_copy_sg(struct scatterlist *dst,
+				 struct scatterlist *src,
+				 int buflen);
+
 int dthe_register_aes_algs(void);
 void dthe_unregister_aes_algs(void);
 
diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h
index e559bdadf4f91..0c2efdc832578 100644
--- a/drivers/crypto/virtio/virtio_crypto_common.h
+++ b/drivers/crypto/virtio/virtio_crypto_common.h
@@ -11,6 +11,7 @@
 #include <linux/crypto.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
+#include <linux/workqueue.h>
 #include <crypto/aead.h>
 #include <crypto/aes.h>
 #include <crypto/engine.h>
@@ -29,7 +30,7 @@ struct data_queue {
 	char name[32];
 
 	struct crypto_engine *engine;
-	struct tasklet_struct done_task;
+	struct work_struct done_work;
 };
 
 struct virtio_crypto {
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
index 955bff8820dab..ee83bf6568f04 100644
--- a/drivers/crypto/virtio/virtio_crypto_core.c
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -70,9 +70,9 @@ int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterl
 	return 0;
 }
 
-static void virtcrypto_done_task(unsigned long data)
+static void virtcrypto_done_work(struct work_struct *work)
 {
-	struct data_queue *data_vq = (struct data_queue *)data;
+	struct data_queue *data_vq = from_work(data_vq, work, done_work);
 	struct virtqueue *vq = data_vq->vq;
 	struct virtio_crypto_request *vc_req;
 	unsigned long flags;
@@ -96,7 +96,7 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq)
 	struct virtio_crypto *vcrypto = vq->vdev->priv;
 	struct data_queue *dq = &vcrypto->data_vq[vq->index];
 
-	tasklet_schedule(&dq->done_task);
+	queue_work(system_bh_wq, &dq->done_work);
 }
 
 static int virtcrypto_find_vqs(struct virtio_crypto *vi)
@@ -150,8 +150,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
 			ret = -ENOMEM;
 			goto err_engine;
 		}
-		tasklet_init(&vi->data_vq[i].done_task, virtcrypto_done_task,
-				(unsigned long)&vi->data_vq[i]);
+		INIT_WORK(&vi->data_vq[i].done_work, virtcrypto_done_work);
 	}
 
 	kfree(vqs_info);
@@ -501,7 +500,7 @@ static void virtcrypto_remove(struct virtio_device *vdev)
 	if (virtcrypto_dev_started(vcrypto))
 		virtcrypto_dev_stop(vcrypto);
 	for (i = 0; i < vcrypto->max_data_queues; i++)
-		tasklet_kill(&vcrypto->data_vq[i].done_task);
+		cancel_work_sync(&vcrypto->data_vq[i].done_work);
 	virtio_reset_device(vdev);
 	virtcrypto_free_unused_reqs(vcrypto);
 	virtcrypto_clear_crypto_engines(vcrypto);
diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig
index 4589bf11d3fe0..80aeb0d556bd7 100644
--- a/drivers/cxl/Kconfig
+++ b/drivers/cxl/Kconfig
@@ -59,6 +59,7 @@ config CXL_ACPI
 	tristate "CXL ACPI: Platform Support"
 	depends on ACPI
 	depends on ACPI_NUMA
+	depends on CXL_PMEM || !CXL_PMEM
 	default CXL_BUS
 	select ACPI_TABLE_LIB
 	select ACPI_HMAT
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 5b0570df0fd9c..5539e941782f6 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -224,4 +224,6 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 		    u16 *return_code);
 #endif
 
+resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
+					   struct cxl_dport *dport);
 #endif /* __CXL_CORE_H__ */
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index c222e98ae7364..cb5d5a047a9da 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -94,7 +94,6 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
 	struct cxl_hdm *cxlhdm;
 	void __iomem *hdm;
 	u32 ctrl;
-	int i;
 
 	if (!info)
 		return false;
@@ -113,22 +112,16 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
 		return false;
 
 	/*
-	 * If any decoders are committed already, there should not be any
-	 * emulated DVSEC decoders.
+	 * If HDM decoders are globally enabled, do not fall back to DVSEC
+	 * range emulation. Zeroed decoder registers after region teardown
+	 * do not imply absence of HDM capability.
+	 *
+	 * Falling back to DVSEC here would treat the decoder as AUTO and
+	 * may incorrectly latch default interleave settings.
 	 */
-	for (i = 0; i < cxlhdm->decoder_count; i++) {
-		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
-		dev_dbg(&info->port->dev,
-			"decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
-			info->port->id, i,
-			FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
-			readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
-			readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
-			readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
-			readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
-		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
-			return false;
-	}
+	ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
+	if (ctrl & CXL_HDM_DECODER_ENABLE)
+		return false;
 
 	return true;
 }
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index e7a6452bf5445..7c6c5b7450a54 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -893,7 +893,7 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, "CXL");
 
-void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
+void cxl_event_trace_record(struct cxl_memdev *cxlmd,
 			    enum cxl_event_log_type type,
 			    enum cxl_event_type event_type,
 			    const uuid_t *uuid, union cxl_event *evt)
@@ -920,6 +920,7 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 		 * translations. Take topology mutation locks and lookup
 		 * { HPA, REGION } from { DPA, MEMDEV } in the event record.
 		 */
+		guard(device)(&cxlmd->dev);
 		guard(rwsem_read)(&cxl_rwsem.region);
 		guard(rwsem_read)(&cxl_rwsem.dpa);
 
@@ -968,7 +969,7 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 }
 EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, "CXL");
 
-static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd,
+static void __cxl_event_trace_record(struct cxl_memdev *cxlmd,
 				     enum cxl_event_log_type type,
 				     struct cxl_event_record_raw *record)
 {
@@ -1301,7 +1302,7 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
 	 * Require an endpoint to be safe otherwise the driver can not
 	 * be sure that the device is unmapped.
 	 */
-	if (endpoint && cxl_num_decoders_committed(endpoint) == 0)
+	if (cxlmd->dev.driver && cxl_num_decoders_committed(endpoint) == 0)
 		return __cxl_mem_sanitize(mds, cmd);
 
 	return -EBUSY;
@@ -1521,23 +1522,21 @@ int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, "CXL");
 
-struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
+struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial,
+						 u16 dvsec)
 {
 	struct cxl_memdev_state *mds;
 	int rc;
 
-	mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
+	mds = devm_cxl_dev_state_create(dev, CXL_DEVTYPE_CLASSMEM, serial,
+					dvsec, struct cxl_memdev_state, cxlds,
+					true);
 	if (!mds) {
 		dev_err(dev, "No memory available\n");
 		return ERR_PTR(-ENOMEM);
 	}
 
 	mutex_init(&mds->event.log_lock);
-	mds->cxlds.dev = dev;
-	mds->cxlds.reg_map.host = dev;
-	mds->cxlds.cxl_mbox.host = dev;
-	mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;
-	mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
 
 	rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier);
 	if (rc == -EOPNOTSUPP)
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 273c22118d3d8..99e422594885a 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -656,6 +656,30 @@ static void detach_memdev(struct work_struct *work)
 
 static struct lock_class_key cxl_memdev_key;
 
+struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
+						 enum cxl_devtype type,
+						 u64 serial, u16 dvsec,
+						 size_t size, bool has_mbox)
+{
+	struct cxl_dev_state *cxlds = devm_kzalloc(dev, size, GFP_KERNEL);
+
+	if (!cxlds)
+		return NULL;
+
+	cxlds->dev = dev;
+	cxlds->type = type;
+	cxlds->serial = serial;
+	cxlds->cxl_dvsec = dvsec;
+	cxlds->reg_map.host = dev;
+	cxlds->reg_map.resource = CXL_RESOURCE_NONE;
+
+	if (has_mbox)
+		cxlds->cxl_mbox.host = dev;
+
+	return cxlds;
+}
+EXPORT_SYMBOL_NS_GPL(_devm_cxl_dev_state_create, "CXL");
+
 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 					   const struct file_operations *fops,
 					   const struct cxl_memdev_attach *attach)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index f96ce884a2130..d1f487b3d809a 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -696,6 +696,63 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, "CXL");
 
+static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
+				  struct cxl_register_map *map,
+				  struct cxl_dport *dport)
+{
+	resource_size_t component_reg_phys;
+
+	*map = (struct cxl_register_map) {
+		.host = &pdev->dev,
+		.resource = CXL_RESOURCE_NONE,
+	};
+
+	component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
+	if (component_reg_phys == CXL_RESOURCE_NONE)
+		return -ENXIO;
+
+	map->resource = component_reg_phys;
+	map->reg_type = CXL_REGLOC_RBI_COMPONENT;
+	map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
+
+	return 0;
+}
+
+int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+		       struct cxl_register_map *map)
+{
+	int rc;
+
+	rc = cxl_find_regblock(pdev, type, map);
+
+	/*
+	 * If the Register Locator DVSEC does not exist, check if it
+	 * is an RCH and try to extract the Component Registers from
+	 * an RCRB.
+	 */
+	if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) {
+		struct cxl_dport *dport;
+		struct cxl_port *port __free(put_cxl_port) =
+			cxl_pci_find_port(pdev, &dport);
+		if (!port)
+			return -EPROBE_DEFER;
+
+		rc = cxl_rcrb_get_comp_regs(pdev, map, dport);
+		if (rc)
+			return rc;
+
+		rc = cxl_dport_map_rcd_linkcap(pdev, dport);
+		if (rc)
+			return rc;
+
+	} else if (rc) {
+		return rc;
+	}
+
+	return cxl_setup_regs(map);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, "CXL");
+
 int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c)
 {
 	int speed, bw;
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 0c5957d1d3294..c5aacd7054f1d 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -552,10 +552,13 @@ static void cxl_port_release(struct device *dev)
 	xa_destroy(&port->dports);
 	xa_destroy(&port->regions);
 	ida_free(&cxl_port_ida, port->id);
-	if (is_cxl_root(port))
+
+	if (is_cxl_root(port)) {
 		kfree(to_cxl_root(port));
-	else
+	} else {
+		put_device(dev->parent);
 		kfree(port);
+	}
 }
 
 static ssize_t decoders_committed_show(struct device *dev,
@@ -707,6 +710,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
 		struct cxl_port *iter;
 
 		dev->parent = &parent_port->dev;
+		get_device(dev->parent);
 		port->depth = parent_port->depth + 1;
 		port->parent_dport = parent_dport;
 
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 42874948b589b..6a71d29c36b63 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -485,22 +485,14 @@ static ssize_t interleave_ways_show(struct device *dev,
 
 static const struct attribute_group *get_cxl_region_target_group(void);
 
-static ssize_t interleave_ways_store(struct device *dev,
-				     struct device_attribute *attr,
-				     const char *buf, size_t len)
+static int set_interleave_ways(struct cxl_region *cxlr, int val)
 {
-	struct cxl_region *cxlr = to_cxl_region(dev);
 	struct cxl_root_decoder *cxlrd = cxlr->cxlrd;
 	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 	struct cxl_region_params *p = &cxlr->params;
-	unsigned int val, save;
-	int rc;
+	int save, rc;
 	u8 iw;
 
-	rc = kstrtouint(buf, 0, &val);
-	if (rc)
-		return rc;
-
 	rc = ways_to_eiw(val, &iw);
 	if (rc)
 		return rc;
@@ -515,9 +507,7 @@ static ssize_t interleave_ways_store(struct device *dev,
 		return -EINVAL;
 	}
 
-	ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
-	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
-		return rc;
+	lockdep_assert_held_write(&cxl_rwsem.region);
 
 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 		return -EBUSY;
@@ -525,10 +515,31 @@ static ssize_t interleave_ways_store(struct device *dev,
 	save = p->interleave_ways;
 	p->interleave_ways = val;
 	rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
-	if (rc) {
+	if (rc)
 		p->interleave_ways = save;
+
+	return rc;
+}
+
+static ssize_t interleave_ways_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t len)
+{
+	struct cxl_region *cxlr = to_cxl_region(dev);
+	int val;
+	int rc;
+
+	rc = kstrtoint(buf, 0, &val);
+	if (rc)
+		return rc;
+
+	ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
+	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
+		return rc;
+
+	rc = set_interleave_ways(cxlr, val);
+	if (rc)
 		return rc;
-	}
 
 	return len;
 }
@@ -548,21 +559,14 @@ static ssize_t interleave_granularity_show(struct device *dev,
 	return sysfs_emit(buf, "%d\n", p->interleave_granularity);
 }
 
-static ssize_t interleave_granularity_store(struct device *dev,
-					    struct device_attribute *attr,
-					    const char *buf, size_t len)
+static int set_interleave_granularity(struct cxl_region *cxlr, int val)
 {
-	struct cxl_region *cxlr = to_cxl_region(dev);
 	struct cxl_root_decoder *cxlrd = cxlr->cxlrd;
 	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 	struct cxl_region_params *p = &cxlr->params;
-	int rc, val;
+	int rc;
 	u16 ig;
 
-	rc = kstrtoint(buf, 0, &val);
-	if (rc)
-		return rc;
-
 	rc = granularity_to_eig(val, &ig);
 	if (rc)
 		return rc;
@@ -578,14 +582,33 @@ static ssize_t interleave_granularity_store(struct device *dev,
 	if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 		return -EINVAL;
 
-	ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
-	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
-		return rc;
+	lockdep_assert_held_write(&cxl_rwsem.region);
 
 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 		return -EBUSY;
 
 	p->interleave_granularity = val;
+	return 0;
+}
+
+static ssize_t interleave_granularity_store(struct device *dev,
+					    struct device_attribute *attr,
+					    const char *buf, size_t len)
+{
+	struct cxl_region *cxlr = to_cxl_region(dev);
+	int rc, val;
+
+	rc = kstrtoint(buf, 0, &val);
+	if (rc)
+		return rc;
+
+	ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
+	if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
+		return rc;
+
+	rc = set_interleave_granularity(cxlr, val);
+	if (rc)
+		return rc;
 
 	return len;
 }
@@ -2650,7 +2673,8 @@ static ssize_t create_ram_region_show(struct device *dev,
 }
 
 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
-					  enum cxl_partition_mode mode, int id)
+					  enum cxl_partition_mode mode, int id,
+					  enum cxl_decoder_type target_type)
 {
 	int rc;
 
@@ -2672,7 +2696,7 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
 		return ERR_PTR(-EBUSY);
 	}
 
-	return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
+	return devm_cxl_add_region(cxlrd, id, mode, target_type);
 }
 
 static ssize_t create_region_store(struct device *dev, const char *buf,
@@ -2686,7 +2710,7 @@ static ssize_t create_region_store(struct device *dev, const char *buf,
 	if (rc != 1)
 		return -EINVAL;
 
-	cxlr = __create_region(cxlrd, mode, id);
+	cxlr = __create_region(cxlrd, mode, id, CXL_DECODER_HOSTONLYMEM);
 	if (IS_ERR(cxlr))
 		return PTR_ERR(cxlr);
 
@@ -2950,13 +2974,15 @@ static int __cxl_dpa_to_region(struct device *dev, void *arg)
 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
 {
 	struct cxl_dpa_to_region_context ctx;
-	struct cxl_port *port;
+	struct cxl_port *port = cxlmd->endpoint;
+
+	if (!cxlmd->dev.driver)
+		return NULL;
 
 	ctx = (struct cxl_dpa_to_region_context) {
 		.dpa = dpa,
 	};
-	port = cxlmd->endpoint;
-	if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
+	if (cxl_num_decoders_committed(port))
 		device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
 
 	return ctx.cxlr;
@@ -3854,8 +3880,10 @@ static int __construct_region(struct cxl_region *cxlr,
 	}
 
 	rc = sysfs_update_group(&cxlr->dev.kobj, &cxl_region_group);
-	if (rc)
+	if (rc) {
+		kfree(res);
 		return rc;
+	}
 
 	rc = insert_resource(cxlrd->res, res);
 	if (rc) {
@@ -3902,7 +3930,8 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
 
 	do {
 		cxlr = __create_region(cxlrd, cxlds->part[part].mode,
-				       atomic_read(&cxlrd->region_id));
+				       atomic_read(&cxlrd->region_id),
+				       cxled->cxld.target_type);
 	} while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
 
 	if (IS_ERR(cxlr)) {
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index a010b32143422..93710cf4f0a69 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -641,4 +641,3 @@ resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
 		return CXL_RESOURCE_NONE;
 	return __rcrb_to_component(dev, &dport->rcrb, CXL_RCRB_UPSTREAM);
 }
-EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, "CXL");
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 9b947286eb9b0..8194447f75d30 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -12,6 +12,7 @@
 #include <linux/node.h>
 #include <linux/io.h>
 #include <linux/range.h>
+#include <cxl/cxl.h>
 
 extern const struct nvdimm_security_ops *cxl_security_ops;
 
@@ -201,97 +202,6 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 #define   CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)
 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20
 
-/*
- * Using struct_group() allows for per register-block-type helper routines,
- * without requiring block-type agnostic code to include the prefix.
- */
-struct cxl_regs {
-	/*
-	 * Common set of CXL Component register block base pointers
-	 * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure
-	 * @ras: CXL 2.0 8.2.5.9 CXL RAS Capability Structure
-	 */
-	struct_group_tagged(cxl_component_regs, component,
-		void __iomem *hdm_decoder;
-		void __iomem *ras;
-	);
-	/*
-	 * Common set of CXL Device register block base pointers
-	 * @status: CXL 2.0 8.2.8.3 Device Status Registers
-	 * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers
-	 * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers
-	 */
-	struct_group_tagged(cxl_device_regs, device_regs,
-		void __iomem *status, *mbox, *memdev;
-	);
-
-	struct_group_tagged(cxl_pmu_regs, pmu_regs,
-		void __iomem *pmu;
-	);
-
-	/*
-	 * RCH downstream port specific RAS register
-	 * @aer: CXL 3.0 8.2.1.1 RCH Downstream Port RCRB
-	 */
-	struct_group_tagged(cxl_rch_regs, rch_regs,
-		void __iomem *dport_aer;
-	);
-
-	/*
-	 * RCD upstream port specific PCIe cap register
-	 * @pcie_cap: CXL 3.0 8.2.1.2 RCD Upstream Port RCRB
-	 */
-	struct_group_tagged(cxl_rcd_regs, rcd_regs,
-		void __iomem *rcd_pcie_cap;
-	);
-};
-
-struct cxl_reg_map {
-	bool valid;
-	int id;
-	unsigned long offset;
-	unsigned long size;
-};
-
-struct cxl_component_reg_map {
-	struct cxl_reg_map hdm_decoder;
-	struct cxl_reg_map ras;
-};
-
-struct cxl_device_reg_map {
-	struct cxl_reg_map status;
-	struct cxl_reg_map mbox;
-	struct cxl_reg_map memdev;
-};
-
-struct cxl_pmu_reg_map {
-	struct cxl_reg_map pmu;
-};
-
-/**
- * struct cxl_register_map - DVSEC harvested register block mapping parameters
- * @host: device for devm operations and logging
- * @base: virtual base of the register-block-BAR + @block_offset
- * @resource: physical resource base of the register block
- * @max_size: maximum mapping size to perform register search
- * @reg_type: see enum cxl_regloc_type
- * @component_map: cxl_reg_map for component registers
- * @device_map: cxl_reg_maps for device registers
- * @pmu_map: cxl_reg_maps for CXL Performance Monitoring Units
- */
-struct cxl_register_map {
-	struct device *host;
-	void __iomem *base;
-	resource_size_t resource;
-	resource_size_t max_size;
-	u8 reg_type;
-	union {
-		struct cxl_component_reg_map component_map;
-		struct cxl_device_reg_map device_map;
-		struct cxl_pmu_reg_map pmu_map;
-	};
-};
-
 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
 			      struct cxl_component_reg_map *map);
 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
@@ -312,8 +222,6 @@ int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
 		      struct cxl_register_map *map);
 int cxl_setup_regs(struct cxl_register_map *map);
 struct cxl_dport;
-resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
-					   struct cxl_dport *dport);
 int cxl_dport_map_rcd_linkcap(struct pci_dev *pdev, struct cxl_dport *dport);
 
 #define CXL_RESOURCE_NONE ((resource_size_t) -1)
@@ -497,11 +405,6 @@ struct cxl_region_params {
 	resource_size_t cache_size;
 };
 
-enum cxl_partition_mode {
-	CXL_PARTMODE_RAM,
-	CXL_PARTMODE_PMEM,
-};
-
 /*
  * Indicate whether this region has been assembled by autodetection or
  * userspace assembly. Prevent endpoint decoders outside of automatic
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index e21d744d639bd..02054f233fc5d 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -113,8 +113,6 @@ int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 			 resource_size_t base, resource_size_t len,
 			 resource_size_t skipped);
 
-#define CXL_NR_PARTITIONS_MAX 2
-
 struct cxl_dpa_info {
 	u64 size;
 	struct cxl_dpa_part_info {
@@ -373,87 +371,6 @@ struct cxl_security_state {
 	struct kernfs_node *sanitize_node;
 };
 
-/*
- * enum cxl_devtype - delineate type-2 from a generic type-3 device
- * @CXL_DEVTYPE_DEVMEM - Vendor specific CXL Type-2 device implementing HDM-D or
- *			 HDM-DB, no requirement that this device implements a
- *			 mailbox, or other memory-device-standard manageability
- *			 flows.
- * @CXL_DEVTYPE_CLASSMEM - Common class definition of a CXL Type-3 device with
- *			   HDM-H and class-mandatory memory device registers
- */
-enum cxl_devtype {
-	CXL_DEVTYPE_DEVMEM,
-	CXL_DEVTYPE_CLASSMEM,
-};
-
-/**
- * struct cxl_dpa_perf - DPA performance property entry
- * @dpa_range: range for DPA address
- * @coord: QoS performance data (i.e. latency, bandwidth)
- * @cdat_coord: raw QoS performance data from CDAT
- * @qos_class: QoS Class cookies
- */
-struct cxl_dpa_perf {
-	struct range dpa_range;
-	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
-	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
-	int qos_class;
-};
-
-/**
- * struct cxl_dpa_partition - DPA partition descriptor
- * @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res)
- * @perf: performance attributes of the partition from CDAT
- * @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic...
- */
-struct cxl_dpa_partition {
-	struct resource res;
-	struct cxl_dpa_perf perf;
-	enum cxl_partition_mode mode;
-};
-
-/**
- * struct cxl_dev_state - The driver device state
- *
- * cxl_dev_state represents the CXL driver/device state.  It provides an
- * interface to mailbox commands as well as some cached data about the device.
- * Currently only memory devices are represented.
- *
- * @dev: The device associated with this CXL state
- * @cxlmd: The device representing the CXL.mem capabilities of @dev
- * @reg_map: component and ras register mapping parameters
- * @regs: Class device "Device" registers
- * @cxl_dvsec: Offset to the PCIe device DVSEC
- * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH)
- * @media_ready: Indicate whether the device media is usable
- * @dpa_res: Overall DPA resource tree for the device
- * @part: DPA partition array
- * @nr_partitions: Number of DPA partitions
- * @serial: PCIe Device Serial Number
- * @type: Generic Memory Class device or Vendor Specific Memory device
- * @cxl_mbox: CXL mailbox context
- * @cxlfs: CXL features context
- */
-struct cxl_dev_state {
-	struct device *dev;
-	struct cxl_memdev *cxlmd;
-	struct cxl_register_map reg_map;
-	struct cxl_device_regs regs;
-	int cxl_dvsec;
-	bool rcd;
-	bool media_ready;
-	struct resource dpa_res;
-	struct cxl_dpa_partition part[CXL_NR_PARTITIONS_MAX];
-	unsigned int nr_partitions;
-	u64 serial;
-	enum cxl_devtype type;
-	struct cxl_mailbox cxl_mbox;
-#ifdef CONFIG_CXL_FEATURES
-	struct cxl_features_state *cxlfs;
-#endif
-};
-
 static inline resource_size_t cxl_pmem_size(struct cxl_dev_state *cxlds)
 {
 	/*
@@ -858,13 +775,14 @@ int cxl_dev_state_identify(struct cxl_memdev_state *mds);
 int cxl_await_media_ready(struct cxl_dev_state *cxlds);
 int cxl_enumerate_cmds(struct cxl_memdev_state *mds);
 int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info);
-struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev);
+struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial,
+						 u16 dvsec);
 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 				unsigned long *cmds);
 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 				  unsigned long *cmds);
 void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status);
-void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
+void cxl_event_trace_record(struct cxl_memdev *cxlmd,
 			    enum cxl_event_log_type type,
 			    enum cxl_event_type event_type,
 			    const uuid_t *uuid, union cxl_event *evt);
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 0cf64218aa16e..b826eb53cf7ba 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -74,6 +74,17 @@ static inline bool cxl_pci_flit_256(struct pci_dev *pdev)
 	return lnksta2 & PCI_EXP_LNKSTA2_FLIT;
 }
 
+/*
+ * Assume that the caller has already validated that @pdev has CXL
+ * capabilities, any RCiEP with CXL capabilities is treated as a
+ * Restricted CXL Device (RCD) and finds upstream port and endpoint
+ * registers in a Root Complex Register Block (RCRB).
+ */
+static inline bool is_cxl_restricted(struct pci_dev *pdev)
+{
+	return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
+}
+
 struct cxl_dev_state;
 void read_cdat_data(struct cxl_port *port);
 
@@ -101,4 +112,6 @@ static inline void devm_cxl_port_ras_setup(struct cxl_port *port)
 }
 #endif
 
+int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+		       struct cxl_register_map *map);
 #endif /* __CXL_PCI_H__ */
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index fbb300a018302..a878d9fb3e58e 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -465,76 +465,6 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
 	return 0;
 }
 
-/*
- * Assume that any RCIEP that emits the CXL memory expander class code
- * is an RCD
- */
-static bool is_cxl_restricted(struct pci_dev *pdev)
-{
-	return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
-}
-
-static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
-				  struct cxl_register_map *map,
-				  struct cxl_dport *dport)
-{
-	resource_size_t component_reg_phys;
-
-	*map = (struct cxl_register_map) {
-		.host = &pdev->dev,
-		.resource = CXL_RESOURCE_NONE,
-	};
-
-	struct cxl_port *port __free(put_cxl_port) =
-		cxl_pci_find_port(pdev, &dport);
-	if (!port)
-		return -EPROBE_DEFER;
-
-	component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
-	if (component_reg_phys == CXL_RESOURCE_NONE)
-		return -ENXIO;
-
-	map->resource = component_reg_phys;
-	map->reg_type = CXL_REGLOC_RBI_COMPONENT;
-	map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
-
-	return 0;
-}
-
-static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
-			      struct cxl_register_map *map)
-{
-	int rc;
-
-	rc = cxl_find_regblock(pdev, type, map);
-
-	/*
-	 * If the Register Locator DVSEC does not exist, check if it
-	 * is an RCH and try to extract the Component Registers from
-	 * an RCRB.
-	 */
-	if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) {
-		struct cxl_dport *dport;
-		struct cxl_port *port __free(put_cxl_port) =
-			cxl_pci_find_port(pdev, &dport);
-		if (!port)
-			return -EPROBE_DEFER;
-
-		rc = cxl_rcrb_get_comp_regs(pdev, map, dport);
-		if (rc)
-			return rc;
-
-		rc = cxl_dport_map_rcd_linkcap(pdev, dport);
-		if (rc)
-			return rc;
-
-	} else if (rc) {
-		return rc;
-	}
-
-	return cxl_setup_regs(map);
-}
-
 static void free_event_buf(void *buf)
 {
 	kvfree(buf);
@@ -865,25 +795,25 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	int rc, pmu_count;
 	unsigned int i;
 	bool irq_avail;
+	u16 dvsec;
 
 	rc = pcim_enable_device(pdev);
 	if (rc)
 		return rc;
 	pci_set_master(pdev);
 
-	mds = cxl_memdev_state_create(&pdev->dev);
+	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_DEVICE);
+	if (!dvsec)
+		pci_warn(pdev, "Device DVSEC not present, skip CXL.mem init\n");
+
+	mds = cxl_memdev_state_create(&pdev->dev, pci_get_dsn(pdev), dvsec);
 	if (IS_ERR(mds))
 		return PTR_ERR(mds);
 	cxlds = &mds->cxlds;
 	pci_set_drvdata(pdev, cxlds);
 
 	cxlds->rcd = is_cxl_restricted(pdev);
-	cxlds->serial = pci_get_dsn(pdev);
-	cxlds->cxl_dvsec = pci_find_dvsec_capability(
-		pdev, PCI_VENDOR_ID_CXL, PCI_DVSEC_CXL_DEVICE);
-	if (!cxlds->cxl_dvsec)
-		dev_warn(&pdev->dev,
-			 "Device DVSEC not present, skip CXL.mem init\n");
 
 	rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
 	if (rc)
@@ -1043,6 +973,9 @@ static void cxl_reset_done(struct pci_dev *pdev)
 	 * that no longer exists.
 	 */
 	guard(device)(&cxlmd->dev);
+	if (!cxlmd->dev.driver)
+		return;
+
 	if (cxlmd->endpoint &&
 	    cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) {
 		dev_crit(dev, "SBR happened without memory regions removal.\n");
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 082ec0f1c3a04..261dff7ced9f1 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -554,7 +554,7 @@ static __exit void cxl_pmem_exit(void)
 
 MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support");
 MODULE_LICENSE("GPL v2");
-module_init(cxl_pmem_init);
+subsys_initcall(cxl_pmem_init);
 module_exit(cxl_pmem_exit);
 MODULE_IMPORT_NS("CXL");
 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index 528e81240c4d4..381021c2e0318 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -24,7 +24,7 @@ static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags,
 		return -ENXIO;
 
 	/* prevent private mappings from being established */
-	if (!vma_flags_test(&flags, VMA_MAYSHARE_BIT)) {
+	if (!vma_flags_test_any(&flags, VMA_MAYSHARE_BIT)) {
 		dev_info_ratelimited(dev,
 				"%s: %s: fail, attempted private mapping\n",
 				current->comm, func);
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index c0a74091b9041..82dd9a43dc621 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -38,6 +38,7 @@
 
 static struct class *devfreq_class;
 static struct dentry *devfreq_debugfs;
+static const struct attribute_group gov_attr_group;
 
 /*
  * devfreq core provides delayed work based load monitoring helper
@@ -146,10 +147,9 @@ void devfreq_get_freq_range(struct devfreq *devfreq,
 					     DEV_PM_QOS_MIN_FREQUENCY);
 	qos_max_freq = dev_pm_qos_read_value(devfreq->dev.parent,
 					     DEV_PM_QOS_MAX_FREQUENCY);
-	*min_freq = max(*min_freq, (unsigned long)HZ_PER_KHZ * qos_min_freq);
+	*min_freq = max(*min_freq, HZ_PER_KHZ * qos_min_freq);
 	if (qos_max_freq != PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE)
-		*max_freq = min(*max_freq,
-				(unsigned long)HZ_PER_KHZ * qos_max_freq);
+		*max_freq = min(*max_freq, HZ_PER_KHZ * qos_max_freq);
 
 	/* Apply constraints from OPP interface */
 	*max_freq = clamp(*max_freq, devfreq->scaling_min_freq, devfreq->scaling_max_freq);
@@ -785,11 +785,6 @@ static void devfreq_dev_release(struct device *dev)
 	kfree(devfreq);
 }
 
-static void create_sysfs_files(struct devfreq *devfreq,
-				const struct devfreq_governor *gov);
-static void remove_sysfs_files(struct devfreq *devfreq,
-				const struct devfreq_governor *gov);
-
 /**
  * devfreq_add_device() - Add devfreq feature to the device
  * @dev:	the device to add devfreq feature.
@@ -956,7 +951,10 @@ struct devfreq *devfreq_add_device(struct device *dev,
 			 __func__);
 		goto err_init;
 	}
-	create_sysfs_files(devfreq, devfreq->governor);
+
+	err = sysfs_update_group(&devfreq->dev.kobj, &gov_attr_group);
+	if (err)
+		goto err_init;
 
 	list_add(&devfreq->node, &devfreq_list);
 
@@ -995,12 +993,9 @@ int devfreq_remove_device(struct devfreq *devfreq)
 
 	devfreq_cooling_unregister(devfreq->cdev);
 
-	if (devfreq->governor) {
+	if (devfreq->governor)
 		devfreq->governor->event_handler(devfreq,
 						 DEVFREQ_GOV_STOP, NULL);
-		remove_sysfs_files(devfreq, devfreq->governor);
-	}
-
 	device_unregister(&devfreq->dev);
 
 	return 0;
@@ -1460,7 +1455,6 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 			 __func__, df->governor->name, ret);
 		goto out;
 	}
-	remove_sysfs_files(df, df->governor);
 
 	/*
 	 * Start the new governor and create the specific sysfs files
@@ -1489,7 +1483,7 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 	 * Create the sysfs files for the new governor. But if failed to start
 	 * the new governor, restore the sysfs files of previous governor.
 	 */
-	create_sysfs_files(df, df->governor);
+	ret = sysfs_update_group(&df->dev.kobj, &gov_attr_group);
 
 out:
 	mutex_unlock(&devfreq_list_lock);
@@ -1807,14 +1801,17 @@ static struct attribute *devfreq_attrs[] = {
 	&dev_attr_trans_stat.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(devfreq);
 
 static ssize_t polling_interval_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
 {
 	struct devfreq *df = to_devfreq(dev);
 
-	if (!df->profile)
+	/* Protect against race between sysfs attrs update and read/write */
+	guard(mutex)(&devfreq_list_lock);
+
+	if (!df->profile || !df->governor ||
+	    !IS_SUPPORTED_ATTR(df->governor->attrs, POLLING_INTERVAL))
 		return -EINVAL;
 
 	return sprintf(buf, "%d\n", df->profile->polling_ms);
@@ -1828,7 +1825,10 @@ static ssize_t polling_interval_store(struct device *dev,
 	unsigned int value;
 	int ret;
 
-	if (!df->governor)
+	guard(mutex)(&devfreq_list_lock);
+
+	if (!df->governor ||
+	    !IS_SUPPORTED_ATTR(df->governor->attrs, POLLING_INTERVAL))
 		return -EINVAL;
 
 	ret = sscanf(buf, "%u", &value);
@@ -1847,7 +1847,10 @@ static ssize_t timer_show(struct device *dev,
 {
 	struct devfreq *df = to_devfreq(dev);
 
-	if (!df->profile)
+	guard(mutex)(&devfreq_list_lock);
+
+	if (!df->profile || !df->governor ||
+	    !IS_SUPPORTED_ATTR(df->governor->attrs, TIMER))
 		return -EINVAL;
 
 	return sprintf(buf, "%s\n", timer_name[df->profile->timer]);
@@ -1861,7 +1864,10 @@ static ssize_t timer_store(struct device *dev, struct device_attribute *attr,
 	int timer = -1;
 	int ret = 0, i;
 
-	if (!df->governor || !df->profile)
+	guard(mutex)(&devfreq_list_lock);
+
+	if (!df->governor || !df->profile ||
+	    !IS_SUPPORTED_ATTR(df->governor->attrs, TIMER))
 		return -EINVAL;
 
 	ret = sscanf(buf, "%16s", str_timer);
@@ -1905,37 +1911,47 @@ static ssize_t timer_store(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(timer);
 
-#define CREATE_SYSFS_FILE(df, name)					\
-{									\
-	int ret;							\
-	ret = sysfs_create_file(&df->dev.kobj, &dev_attr_##name.attr);	\
-	if (ret < 0) {							\
-		dev_warn(&df->dev,					\
-			"Unable to create attr(%s)\n", "##name");	\
-	}								\
-}									\
+static struct attribute *governor_attrs[] = {
+	&dev_attr_polling_interval.attr,
+	&dev_attr_timer.attr,
+	NULL
+};
 
-/* Create the specific sysfs files which depend on each governor. */
-static void create_sysfs_files(struct devfreq *devfreq,
-				const struct devfreq_governor *gov)
+static umode_t gov_attr_visible(struct kobject *kobj,
+				struct attribute *attr, int n)
 {
-	if (IS_SUPPORTED_ATTR(gov->attrs, POLLING_INTERVAL))
-		CREATE_SYSFS_FILE(devfreq, polling_interval);
-	if (IS_SUPPORTED_ATTR(gov->attrs, TIMER))
-		CREATE_SYSFS_FILE(devfreq, timer);
-}
+	struct device *dev = kobj_to_dev(kobj);
+	struct devfreq *df = to_devfreq(dev);
 
-/* Remove the specific sysfs files which depend on each governor. */
-static void remove_sysfs_files(struct devfreq *devfreq,
-				const struct devfreq_governor *gov)
-{
-	if (IS_SUPPORTED_ATTR(gov->attrs, POLLING_INTERVAL))
-		sysfs_remove_file(&devfreq->dev.kobj,
-				&dev_attr_polling_interval.attr);
-	if (IS_SUPPORTED_ATTR(gov->attrs, TIMER))
-		sysfs_remove_file(&devfreq->dev.kobj, &dev_attr_timer.attr);
+	if (!df->governor || !df->governor->attrs)
+		return 0;
+
+	if (attr == &dev_attr_polling_interval.attr &&
+	    IS_SUPPORTED_ATTR(df->governor->attrs, POLLING_INTERVAL))
+		return attr->mode;
+
+	if (attr == &dev_attr_timer.attr &&
+	    IS_SUPPORTED_ATTR(df->governor->attrs, TIMER))
+		return attr->mode;
+
+	return 0;
 }
 
+static const struct attribute_group devfreq_group = {
+	.attrs = devfreq_attrs,
+};
+
+static const struct attribute_group gov_attr_group = {
+	.attrs = governor_attrs,
+	.is_visible = gov_attr_visible,
+};
+
+static const struct attribute_group *devfreq_groups[] = {
+	&devfreq_group,
+	&gov_attr_group,
+	NULL
+};
+
 /**
  * devfreq_summary_show() - Show the summary of the devfreq devices
  * @s:		seq_file instance to show the summary of devfreq devices
diff --git a/drivers/dibs/dibs_main.c b/drivers/dibs/dibs_main.c
index f1816361b74d3..14c3e2d84902e 100644
--- a/drivers/dibs/dibs_main.c
+++ b/drivers/dibs/dibs_main.c
@@ -19,7 +19,9 @@
 MODULE_DESCRIPTION("Direct Internal Buffer Sharing class");
 MODULE_LICENSE("GPL");
 
-static struct class *dibs_class;
+static const struct class dibs_class = {
+	.name		= "dibs",
+};
 
 /* use an array rather a list for fast mapping: */
 static struct dibs_client *clients[MAX_DIBS_CLIENTS];
@@ -137,7 +139,7 @@ struct dibs_dev *dibs_dev_alloc(void)
 	if (!dibs)
 		return dibs;
 	dibs->dev.release = dibs_dev_release;
-	dibs->dev.class = dibs_class;
+	dibs->dev.class = &dibs_class;
 	device_initialize(&dibs->dev);
 
 	return dibs;
@@ -253,9 +255,9 @@ static int __init dibs_init(void)
 {
 	int rc;
 
-	dibs_class = class_create("dibs");
-	if (IS_ERR(dibs_class))
-		return PTR_ERR(dibs_class);
+	rc = class_register(&dibs_class);
+	if (rc)
+		return rc;
 
 	rc = dibs_loopback_init();
 	if (rc)
@@ -267,7 +269,7 @@ static int __init dibs_init(void)
 static void __exit dibs_exit(void)
 {
 	dibs_loopback_exit();
-	class_destroy(dibs_class);
+	class_unregister(&dibs_class);
 }
 
 subsys_initcall(dibs_init);
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 012d22e941d6f..8d4f2f89f24e3 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -40,18 +40,6 @@ config UDMABUF
 	  A driver to let userspace turn memfd regions into dma-bufs.
 	  Qemu can use this to create host dmabufs for guest framebuffers.
 
-config DMABUF_MOVE_NOTIFY
-	bool "Move notify between drivers (EXPERIMENTAL)"
-	default n
-	depends on DMA_SHARED_BUFFER
-	help
-	  Don't pin buffers if the dynamic DMA-buf interface is available on
-	  both the exporter as well as the importer. This fixes a security
-	  problem where userspace is able to pin unrestricted amounts of memory
-	  through DMA-buf.
-	  This is marked experimental because we don't yet have a consistent
-	  execution context and memory management between drivers.
-
 config DMABUF_DEBUG
 	bool "DMA-BUF debug checks"
 	depends on DMA_SHARED_BUFFER
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 11711874a325b..b0bc204bae14b 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -916,8 +916,7 @@ static bool
 dma_buf_pin_on_map(struct dma_buf_attachment *attach)
 {
 	return attach->dmabuf->ops->pin &&
-		(!dma_buf_attachment_is_dynamic(attach) ||
-		 !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY));
+	       !dma_buf_attachment_is_dynamic(attach);
 }
 
 /**
@@ -981,7 +980,7 @@ dma_buf_pin_on_map(struct dma_buf_attachment *attach)
  * 3. Exporters must hold the dma-buf reservation lock when calling these
  *    functions:
  *
- *     - dma_buf_move_notify()
+ *     - dma_buf_invalidate_mappings()
  */
 
 /**
@@ -1017,9 +1016,6 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 	if (WARN_ON(!dmabuf || !dev))
 		return ERR_PTR(-EINVAL);
 
-	if (WARN_ON(importer_ops && !importer_ops->move_notify))
-		return ERR_PTR(-EINVAL);
-
 	attach = kzalloc_obj(*attach);
 	if (!attach)
 		return ERR_PTR(-ENOMEM);
@@ -1130,7 +1126,7 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_pin, "DMA_BUF");
  *
  * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move
  * any mapping of @attach again and inform the importer through
- * &dma_buf_attach_ops.move_notify.
+ * &dma_buf_attach_ops.invalidate_mappings.
  */
 void dma_buf_unpin(struct dma_buf_attachment *attach)
 {
@@ -1323,24 +1319,71 @@ void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach,
 EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment_unlocked, "DMA_BUF");
 
 /**
- * dma_buf_move_notify - notify attachments that DMA-buf is moving
+ * dma_buf_attach_revocable - check if a DMA-buf importer implements
+ * revoke semantics.
+ * @attach: the DMA-buf attachment to check
+ *
+ * Returns true if the DMA-buf importer can support the revoke sequence
+ * explained in dma_buf_invalidate_mappings() within bounded time. Meaning the
+ * importer implements invalidate_mappings() and ensures that unmap is called as
+ * a result.
+ */
+bool dma_buf_attach_revocable(struct dma_buf_attachment *attach)
+{
+	return attach->importer_ops &&
+	       attach->importer_ops->invalidate_mappings;
+}
+EXPORT_SYMBOL_NS_GPL(dma_buf_attach_revocable, "DMA_BUF");
+
+/**
+ * dma_buf_invalidate_mappings - notify attachments that DMA-buf is moving
  *
  * @dmabuf:	[in]	buffer which is moving
  *
  * Informs all attachments that they need to destroy and recreate all their
- * mappings.
+ * mappings. If the attachment is dynamic then the dynamic importer is expected
+ * to invalidate any caches it has of the mapping result and perform a new
+ * mapping request before allowing HW to do any further DMA.
+ *
+ * If the attachment is pinned then this informs the pinned importer that the
+ * underlying mapping is no longer available. Pinned importers may take this is
+ * as a permanent revocation and never establish new mappings so exporters
+ * should not trigger it lightly.
+ *
+ * Upon return importers may continue to access the DMA-buf memory. The caller
+ * must do two additional waits to ensure that the memory is no longer being
+ * accessed:
+ *  1) Until dma_resv_wait_timeout() retires fences the importer is allowed to
+ *     fully access the memory.
+ *  2) Until the importer calls unmap it is allowed to speculatively
+ *     read-and-discard the memory. It must not write to the memory.
+ *
+ * A caller wishing to use dma_buf_invalidate_mappings() to fully stop access to
+ * the DMA-buf must wait for both. Dynamic callers can often use just the first.
+ *
+ * All importers providing a invalidate_mappings() op must ensure that unmap is
+ * called within bounded time after the op.
+ *
+ * Pinned importers that do not support a invalidate_mappings() op will
+ * eventually perform unmap when they are done with the buffer, which may be an
+ * ubounded time from calling this function. dma_buf_attach_revocable() can be
+ * used to prevent such importers from attaching.
+ *
+ * Importers are free to request a new mapping in parallel as this function
+ * returns.
  */
-void dma_buf_move_notify(struct dma_buf *dmabuf)
+void dma_buf_invalidate_mappings(struct dma_buf *dmabuf)
 {
 	struct dma_buf_attachment *attach;
 
 	dma_resv_assert_held(dmabuf->resv);
 
 	list_for_each_entry(attach, &dmabuf->attachments, node)
-		if (attach->importer_ops)
-			attach->importer_ops->move_notify(attach);
+		if (attach->importer_ops &&
+		    attach->importer_ops->invalidate_mappings)
+			attach->importer_ops->invalidate_mappings(attach);
 }
-EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, "DMA_BUF");
+EXPORT_SYMBOL_NS_GPL(dma_buf_invalidate_mappings, "DMA_BUF");
 
 /**
  * DOC: cpu access
@@ -1708,7 +1751,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused)
 
 
 		spin_lock(&buf_obj->name_lock);
-		seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n",
+		seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08llu\t%s\n",
 				buf_obj->size,
 				buf_obj->file->f_flags, buf_obj->file->f_mode,
 				file_count(buf_obj->file),
diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c
index 37e2c6179d779..089f694695248 100644
--- a/drivers/dma-buf/dma-fence-array.c
+++ b/drivers/dma-buf/dma-fence-array.c
@@ -200,15 +200,28 @@ void dma_fence_array_init(struct dma_fence_array *array,
 			  u64 context, unsigned seqno,
 			  bool signal_on_any)
 {
+	static struct lock_class_key dma_fence_array_lock_key;
+
 	WARN_ON(!num_fences || !fences);
 
 	array->num_fences = num_fences;
 
-	spin_lock_init(&array->lock);
-	dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
-		       context, seqno);
+	dma_fence_init(&array->base, &dma_fence_array_ops, NULL, context,
+		       seqno);
 	init_irq_work(&array->work, irq_dma_fence_array_work);
 
+	/*
+	 * dma_fence_array_enable_signaling() is invoked while holding
+	 * array->base.inline_lock and may call dma_fence_add_callback()
+	 * on the underlying fences, which takes their inline_lock.
+	 *
+	 * Since both locks share the same lockdep class, this legitimate
+	 * nesting confuses lockdep and triggers a recursive locking
+	 * warning. Assign a separate lockdep class to the array lock
+	 * to model this hierarchy correctly.
+	 */
+	lockdep_set_class(&array->base.inline_lock, &dma_fence_array_lock_key);
+
 	atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
 	array->fences = fences;
 
diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c
index a8a90acf4f34d..a588f55ea4d3d 100644
--- a/drivers/dma-buf/dma-fence-chain.c
+++ b/drivers/dma-buf/dma-fence-chain.c
@@ -242,10 +242,10 @@ void dma_fence_chain_init(struct dma_fence_chain *chain,
 			  struct dma_fence *fence,
 			  uint64_t seqno)
 {
+	static struct lock_class_key dma_fence_chain_lock_key;
 	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
 	uint64_t context;
 
-	spin_lock_init(&chain->lock);
 	rcu_assign_pointer(chain->prev, prev);
 	chain->fence = fence;
 	chain->prev_seqno = 0;
@@ -261,9 +261,21 @@ void dma_fence_chain_init(struct dma_fence_chain *chain,
 			seqno = max(prev->seqno, seqno);
 	}
 
-	dma_fence_init64(&chain->base, &dma_fence_chain_ops, &chain->lock,
+	dma_fence_init64(&chain->base, &dma_fence_chain_ops, NULL,
 			 context, seqno);
 
+	/*
+	 * dma_fence_chain_enable_signaling() is invoked while holding
+	 * chain->base.inline_lock and may call dma_fence_add_callback()
+	 * on the underlying fences, which takes their inline_lock.
+	 *
+	 * Since both locks share the same lockdep class, this legitimate
+	 * nesting confuses lockdep and triggers a recursive locking
+	 * warning. Assign a separate lockdep class to the chain lock
+	 * to model this hierarchy correctly.
+	 */
+	lockdep_set_class(&chain->base.inline_lock, &dma_fence_chain_lock_key);
+
 	/*
 	 * Chaining dma_fence_chain container together is only allowed through
 	 * the prev fence and not through the contained fence.
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 35afcfcac5910..1826ba73094c8 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -24,7 +24,6 @@ EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
 EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
 EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
 
-static DEFINE_SPINLOCK(dma_fence_stub_lock);
 static struct dma_fence dma_fence_stub;
 
 /*
@@ -123,12 +122,9 @@ static const struct dma_fence_ops dma_fence_stub_ops = {
 
 static int __init dma_fence_init_stub(void)
 {
-	dma_fence_init(&dma_fence_stub, &dma_fence_stub_ops,
-		       &dma_fence_stub_lock, 0, 0);
-
+	dma_fence_init(&dma_fence_stub, &dma_fence_stub_ops, NULL, 0, 0);
 	set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 		&dma_fence_stub.flags);
-
 	dma_fence_signal(&dma_fence_stub);
 	return 0;
 }
@@ -160,11 +156,7 @@ struct dma_fence *dma_fence_allocate_private_stub(ktime_t timestamp)
 	if (fence == NULL)
 		return NULL;
 
-	dma_fence_init(fence,
-		       &dma_fence_stub_ops,
-		       &dma_fence_stub_lock,
-		       0, 0);
-
+	dma_fence_init(fence, &dma_fence_stub_ops, NULL, 0, 0);
 	set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 		&fence->flags);
 
@@ -343,7 +335,6 @@ void __dma_fence_might_wait(void)
 }
 #endif
 
-
 /**
  * dma_fence_signal_timestamp_locked - signal completion of a fence
  * @fence: the fence to signal
@@ -362,15 +353,25 @@ void __dma_fence_might_wait(void)
 void dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 				      ktime_t timestamp)
 {
+	const struct dma_fence_ops *ops;
 	struct dma_fence_cb *cur, *tmp;
 	struct list_head cb_list;
 
-	lockdep_assert_held(fence->lock);
+	dma_fence_assert_held(fence);
 
 	if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 				      &fence->flags)))
 		return;
 
+	/*
+	 * When neither a release nor a wait operation is specified set the ops
+	 * pointer to NULL to allow the fence structure to become independent
+	 * from who originally issued it.
+	 */
+	ops = rcu_dereference_protected(fence->ops, true);
+	if (!ops->release && !ops->wait)
+		RCU_INIT_POINTER(fence->ops, NULL);
+
 	/* Stash the cb_list before replacing it with the timestamp */
 	list_replace(&fence->cb_list, &cb_list);
 
@@ -404,9 +405,9 @@ void dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
 	if (WARN_ON(!fence))
 		return;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	dma_fence_signal_timestamp_locked(fence, timestamp);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 }
 EXPORT_SYMBOL(dma_fence_signal_timestamp);
 
@@ -465,9 +466,9 @@ bool dma_fence_check_and_signal(struct dma_fence *fence)
 	unsigned long flags;
 	bool ret;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	ret = dma_fence_check_and_signal_locked(fence);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	return ret;
 }
@@ -493,9 +494,9 @@ void dma_fence_signal(struct dma_fence *fence)
 
 	tmp = dma_fence_begin_signalling();
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	dma_fence_signal_timestamp_locked(fence, ktime_get());
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	dma_fence_end_signalling(tmp);
 }
@@ -522,6 +523,7 @@ EXPORT_SYMBOL(dma_fence_signal);
 signed long
 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 {
+	const struct dma_fence_ops *ops;
 	signed long ret;
 
 	if (WARN_ON(timeout < 0))
@@ -533,15 +535,22 @@ dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 
 	dma_fence_enable_sw_signaling(fence);
 
-	if (trace_dma_fence_wait_start_enabled()) {
-		rcu_read_lock();
-		trace_dma_fence_wait_start(fence);
+	rcu_read_lock();
+	ops = rcu_dereference(fence->ops);
+	trace_dma_fence_wait_start(fence);
+	if (ops && ops->wait) {
+		/*
+		 * Implementing the wait ops is deprecated and not supported for
+		 * issuers of fences who need their lifetime to be independent
+		 * of their module after they signal, so it is ok to use the
+		 * ops outside the RCU protected section.
+		 */
+		rcu_read_unlock();
+		ret = ops->wait(fence, intr, timeout);
+	} else {
 		rcu_read_unlock();
-	}
-	if (fence->ops->wait)
-		ret = fence->ops->wait(fence, intr, timeout);
-	else
 		ret = dma_fence_default_wait(fence, intr, timeout);
+	}
 	if (trace_dma_fence_wait_end_enabled()) {
 		rcu_read_lock();
 		trace_dma_fence_wait_end(fence);
@@ -562,6 +571,7 @@ void dma_fence_release(struct kref *kref)
 {
 	struct dma_fence *fence =
 		container_of(kref, struct dma_fence, refcount);
+	const struct dma_fence_ops *ops;
 
 	rcu_read_lock();
 	trace_dma_fence_destroy(fence);
@@ -587,18 +597,18 @@ void dma_fence_release(struct kref *kref)
 		 * don't leave chains dangling. We set the error flag first
 		 * so that the callbacks know this signal is due to an error.
 		 */
-		spin_lock_irqsave(fence->lock, flags);
+		dma_fence_lock_irqsave(fence, flags);
 		fence->error = -EDEADLK;
 		dma_fence_signal_locked(fence);
-		spin_unlock_irqrestore(fence->lock, flags);
+		dma_fence_unlock_irqrestore(fence, flags);
 	}
 
-	rcu_read_unlock();
-
-	if (fence->ops->release)
-		fence->ops->release(fence);
+	ops = rcu_dereference(fence->ops);
+	if (ops && ops->release)
+		ops->release(fence);
 	else
 		dma_fence_free(fence);
+	rcu_read_unlock();
 }
 EXPORT_SYMBOL(dma_fence_release);
 
@@ -617,9 +627,10 @@ EXPORT_SYMBOL(dma_fence_free);
 
 static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 {
+	const struct dma_fence_ops *ops;
 	bool was_set;
 
-	lockdep_assert_held(fence->lock);
+	dma_fence_assert_held(fence);
 
 	was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 				   &fence->flags);
@@ -627,14 +638,18 @@ static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 	if (dma_fence_test_signaled_flag(fence))
 		return false;
 
-	if (!was_set && fence->ops->enable_signaling) {
+	rcu_read_lock();
+	ops = rcu_dereference(fence->ops);
+	if (!was_set && ops && ops->enable_signaling) {
 		trace_dma_fence_enable_signal(fence);
 
-		if (!fence->ops->enable_signaling(fence)) {
+		if (!ops->enable_signaling(fence)) {
+			rcu_read_unlock();
 			dma_fence_signal_locked(fence);
 			return false;
 		}
 	}
+	rcu_read_unlock();
 
 	return true;
 }
@@ -651,9 +666,9 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	__dma_fence_enable_signaling(fence);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 }
 EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
 
@@ -693,8 +708,7 @@ int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
 		return -ENOENT;
 	}
 
-	spin_lock_irqsave(fence->lock, flags);
-
+	dma_fence_lock_irqsave(fence, flags);
 	if (__dma_fence_enable_signaling(fence)) {
 		cb->func = func;
 		list_add_tail(&cb->node, &fence->cb_list);
@@ -702,8 +716,7 @@ int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
 		INIT_LIST_HEAD(&cb->node);
 		ret = -ENOENT;
 	}
-
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	return ret;
 }
@@ -726,9 +739,9 @@ int dma_fence_get_status(struct dma_fence *fence)
 	unsigned long flags;
 	int status;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	status = dma_fence_get_status_locked(fence);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	return status;
 }
@@ -758,13 +771,11 @@ dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
 	unsigned long flags;
 	bool ret;
 
-	spin_lock_irqsave(fence->lock, flags);
-
+	dma_fence_lock_irqsave(fence, flags);
 	ret = !list_empty(&cb->node);
 	if (ret)
 		list_del_init(&cb->node);
-
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	return ret;
 }
@@ -803,7 +814,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 	unsigned long flags;
 	signed long ret = timeout ? timeout : 1;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 
 	if (dma_fence_test_signaled_flag(fence))
 		goto out;
@@ -827,11 +838,11 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 			__set_current_state(TASK_INTERRUPTIBLE);
 		else
 			__set_current_state(TASK_UNINTERRUPTIBLE);
-		spin_unlock_irqrestore(fence->lock, flags);
+		dma_fence_unlock_irqrestore(fence, flags);
 
 		ret = schedule_timeout(ret);
 
-		spin_lock_irqsave(fence->lock, flags);
+		dma_fence_lock_irqsave(fence, flags);
 		if (ret > 0 && intr && signal_pending(current))
 			ret = -ERESTARTSYS;
 	}
@@ -841,7 +852,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 	__set_current_state(TASK_RUNNING);
 
 out:
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 	return ret;
 }
 EXPORT_SYMBOL(dma_fence_default_wait);
@@ -1007,8 +1018,13 @@ EXPORT_SYMBOL(dma_fence_wait_any_timeout);
  */
 void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
 {
-	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))
-		fence->ops->set_deadline(fence, deadline);
+	const struct dma_fence_ops *ops;
+
+	rcu_read_lock();
+	ops = rcu_dereference(fence->ops);
+	if (ops && ops->set_deadline && !dma_fence_is_signaled(fence))
+		ops->set_deadline(fence, deadline);
+	rcu_read_unlock();
 }
 EXPORT_SYMBOL(dma_fence_set_deadline);
 
@@ -1045,16 +1061,26 @@ static void
 __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
 	         spinlock_t *lock, u64 context, u64 seqno, unsigned long flags)
 {
-	BUG_ON(!lock);
 	BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
 
 	kref_init(&fence->refcount);
-	fence->ops = ops;
+	/*
+	 * While it is counter intuitive to protect a constant function pointer
+	 * table by RCU it allows modules to wait for an RCU grace period
+	 * before they unload, to make sure that nobody is executing their
+	 * functions any more.
+	 */
+	RCU_INIT_POINTER(fence->ops, ops);
 	INIT_LIST_HEAD(&fence->cb_list);
-	fence->lock = lock;
 	fence->context = context;
 	fence->seqno = seqno;
-	fence->flags = flags;
+	fence->flags = flags | BIT(DMA_FENCE_FLAG_INITIALIZED_BIT);
+	if (lock) {
+		fence->extern_lock = lock;
+	} else {
+		spin_lock_init(&fence->inline_lock);
+		fence->flags |= BIT(DMA_FENCE_FLAG_INLINE_LOCK_BIT);
+	}
 	fence->error = 0;
 
 	trace_dma_fence_init(fence);
@@ -1064,7 +1090,7 @@ __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
  * dma_fence_init - Initialize a custom fence.
  * @fence: the fence to initialize
  * @ops: the dma_fence_ops for operations on this fence
- * @lock: the irqsafe spinlock to use for locking this fence
+ * @lock: optional irqsafe spinlock to use for locking this fence
  * @context: the execution context this fence is run on
  * @seqno: a linear increasing sequence number for this context
  *
@@ -1074,6 +1100,10 @@ __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
  *
  * context and seqno are used for easy comparison between fences, allowing
  * to check which fence is later by simply using dma_fence_later().
+ *
+ * It is strongly discouraged to provide an external lock because this couples
+ * lock and fence life time. This is only allowed for legacy use cases when
+ * multiple fences need to be prevented from signaling out of order.
  */
 void
 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
@@ -1087,7 +1117,7 @@ EXPORT_SYMBOL(dma_fence_init);
  * dma_fence_init64 - Initialize a custom fence with 64-bit seqno support.
  * @fence: the fence to initialize
  * @ops: the dma_fence_ops for operations on this fence
- * @lock: the irqsafe spinlock to use for locking this fence
+ * @lock: optional irqsafe spinlock to use for locking this fence
  * @context: the execution context this fence is run on
  * @seqno: a linear increasing sequence number for this context
  *
@@ -1097,6 +1127,10 @@ EXPORT_SYMBOL(dma_fence_init);
  *
  * Context and seqno are used for easy comparison between fences, allowing
  * to check which fence is later by simply using dma_fence_later().
+ *
+ * It is strongly discouraged to provide an external lock because this couples
+ * lock and fence life time. This is only allowed for legacy use cases when
+ * multiple fences need to be prevented from signaling out of order.
  */
 void
 dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,
@@ -1129,13 +1163,14 @@ EXPORT_SYMBOL(dma_fence_init64);
  */
 const char __rcu *dma_fence_driver_name(struct dma_fence *fence)
 {
-	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
-			 "RCU protection is required for safe access to returned string");
+	const struct dma_fence_ops *ops;
 
+	/* RCU protection is required for safe access to returned string */
+	ops = rcu_dereference(fence->ops);
 	if (!dma_fence_test_signaled_flag(fence))
-		return fence->ops->get_driver_name(fence);
+		return (const char __rcu *)ops->get_driver_name(fence);
 	else
-		return "detached-driver";
+		return (const char __rcu *)"detached-driver";
 }
 EXPORT_SYMBOL(dma_fence_driver_name);
 
@@ -1161,12 +1196,13 @@ EXPORT_SYMBOL(dma_fence_driver_name);
  */
 const char __rcu *dma_fence_timeline_name(struct dma_fence *fence)
 {
-	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
-			 "RCU protection is required for safe access to returned string");
+	const struct dma_fence_ops *ops;
 
+	/* RCU protection is required for safe access to returned string */
+	ops = rcu_dereference(fence->ops);
 	if (!dma_fence_test_signaled_flag(fence))
-		return fence->ops->get_timeline_name(fence);
+		return (const char __rcu *)ops->get_driver_name(fence);
 	else
-		return "signaled-timeline";
+		return (const char __rcu *)"signaled-timeline";
 }
 EXPORT_SYMBOL(dma_fence_timeline_name);
diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c
index bea3e9858aca5..ce9e6c04897ff 100644
--- a/drivers/dma-buf/dma-resv.c
+++ b/drivers/dma-buf/dma-resv.c
@@ -790,8 +790,11 @@ static int __init dma_resv_lockdep(void)
 	mmap_read_lock(mm);
 	ww_acquire_init(&ctx, &reservation_ww_class);
 	ret = dma_resv_lock(&obj, &ctx);
-	if (ret == -EDEADLK)
+	if (ret) {
+		/* Only EDEADLK from the error injection is possible here */
+		WARN_ON(ret != -EDEADLK);
 		dma_resv_lock_slow(&obj, &ctx);
+	}
 	fs_reclaim_acquire(GFP_KERNEL);
 	/* for unmap_mapping_range on trylocked buffer objects in shrinkers */
 	i_mmap_lock_write(&mapping);
diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c
index bd3370b9a3f6d..92865786cfc99 100644
--- a/drivers/dma-buf/heaps/cma_heap.c
+++ b/drivers/dma-buf/heaps/cma_heap.c
@@ -329,10 +329,7 @@ static struct dma_buf *cma_heap_allocate(struct dma_heap *heap,
 		struct page *page = cma_pages;
 
 		while (nr_clear_pages > 0) {
-			void *vaddr = kmap_local_page(page);
-
-			clear_page(vaddr);
-			kunmap_local(vaddr);
+			clear_highpage(page);
 			/*
 			 * Avoid wasting time zeroing memory if the process
 			 * has been killed by SIGKILL.
@@ -343,7 +340,7 @@ static struct dma_buf *cma_heap_allocate(struct dma_heap *heap,
 			nr_clear_pages--;
 		}
 	} else {
-		memset(page_address(cma_pages), 0, size);
+		clear_pages(page_address(cma_pages), pagecount);
 	}
 
 	buffer->pages = kmalloc_objs(*buffer->pages, pagecount);
diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c
index 73ed6fd48a13f..0d9d524d79b6d 100644
--- a/drivers/dma-buf/st-dma-fence.c
+++ b/drivers/dma-buf/st-dma-fence.c
@@ -14,43 +14,26 @@
 
 #include "selftest.h"
 
-static struct kmem_cache *slab_fences;
-
-static struct mock_fence {
-	struct dma_fence base;
-	struct spinlock lock;
-} *to_mock_fence(struct dma_fence *f) {
-	return container_of(f, struct mock_fence, base);
-}
-
 static const char *mock_name(struct dma_fence *f)
 {
 	return "mock";
 }
 
-static void mock_fence_release(struct dma_fence *f)
-{
-	kmem_cache_free(slab_fences, to_mock_fence(f));
-}
-
 static const struct dma_fence_ops mock_ops = {
 	.get_driver_name = mock_name,
 	.get_timeline_name = mock_name,
-	.release = mock_fence_release,
 };
 
 static struct dma_fence *mock_fence(void)
 {
-	struct mock_fence *f;
+	struct dma_fence *f;
 
-	f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
+	f = kmalloc(sizeof(*f), GFP_KERNEL);
 	if (!f)
 		return NULL;
 
-	spin_lock_init(&f->lock);
-	dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
-
-	return &f->base;
+	dma_fence_init(f, &mock_ops, NULL, 0, 0);
+	return f;
 }
 
 static int sanitycheck(void *arg)
@@ -100,6 +83,11 @@ static int test_signaling(void *arg)
 		goto err_free;
 	}
 
+	if (rcu_dereference_protected(f->ops, true)) {
+		pr_err("Fence ops not cleared on signal\n");
+		goto err_free;
+	}
+
 	err = 0;
 err_free:
 	dma_fence_put(f);
@@ -410,8 +398,10 @@ struct race_thread {
 
 static void __wait_for_callbacks(struct dma_fence *f)
 {
-	spin_lock_irq(f->lock);
-	spin_unlock_irq(f->lock);
+	unsigned long flags;
+
+	dma_fence_lock_irqsave(f, flags);
+	dma_fence_unlock_irqrestore(f, flags);
 }
 
 static int thread_signal_callback(void *arg)
@@ -538,19 +528,7 @@ int dma_fence(void)
 		SUBTEST(test_stub),
 		SUBTEST(race_signal_callback),
 	};
-	int ret;
 
 	pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
-
-	slab_fences = KMEM_CACHE(mock_fence,
-				 SLAB_TYPESAFE_BY_RCU |
-				 SLAB_HWCACHE_ALIGN);
-	if (!slab_fences)
-		return -ENOMEM;
-
-	ret = subtests(tests, NULL);
-
-	kmem_cache_destroy(slab_fences);
-
-	return ret;
+	return subtests(tests, NULL);
 }
diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 963a72324d16e..8df20b0218a9e 100644
--- a/drivers/dma-buf/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -156,12 +156,12 @@ static void timeline_fence_release(struct dma_fence *fence)
 	struct sync_timeline *parent = dma_fence_parent(fence);
 	unsigned long flags;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	if (!list_empty(&pt->link)) {
 		list_del(&pt->link);
 		rb_erase(&pt->node, &parent->pt_tree);
 	}
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	sync_timeline_put(parent);
 	dma_fence_free(fence);
@@ -179,7 +179,7 @@ static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadlin
 	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
 	unsigned long flags;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
 		if (ktime_before(deadline, pt->deadline))
 			pt->deadline = deadline;
@@ -187,7 +187,7 @@ static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadlin
 		pt->deadline = deadline;
 		__set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags);
 	}
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 }
 
 static const struct dma_fence_ops timeline_fence_ops = {
@@ -431,13 +431,13 @@ static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long a
 		goto put_fence;
 	}
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	if (!test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
 		ret = -ENOENT;
 		goto unlock;
 	}
 	data.deadline_ns = ktime_to_ns(pt->deadline);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	dma_fence_put(fence);
 
@@ -450,7 +450,7 @@ static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long a
 	return 0;
 
 unlock:
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 put_fence:
 	dma_fence_put(fence);
 
diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h
index 02af347293d02..c49324505b20c 100644
--- a/drivers/dma-buf/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -47,7 +47,7 @@ struct sync_timeline {
 
 static inline struct sync_timeline *dma_fence_parent(struct dma_fence *fence)
 {
-	return container_of(fence->lock, struct sync_timeline, lock);
+	return container_of(fence->extern_lock, struct sync_timeline, lock);
 }
 
 /**
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 66cda7cc9f7ab..ae6a682c9f769 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -376,29 +376,6 @@ config K3_DMA
 	  Support the DMA engine for Hisilicon K3 platform
 	  devices.
 
-config LOONGSON1_APB_DMA
-	tristate "Loongson1 APB DMA support"
-	depends on MACH_LOONGSON32 || COMPILE_TEST
-	select DMA_ENGINE
-	select DMA_VIRTUAL_CHANNELS
-	help
-	  This selects support for the APB DMA controller in Loongson1 SoCs,
-	  which is required by Loongson1 NAND and audio support.
-
-config LOONGSON2_APB_DMA
-	tristate "Loongson2 APB DMA support"
-	depends on LOONGARCH || COMPILE_TEST
-	select DMA_ENGINE
-	select DMA_VIRTUAL_CHANNELS
-	help
-	  Support for the Loongson2 APB DMA controller driver. The
-	  DMA controller is having single DMA channel which can be
-	  configured for different peripherals like audio, nand, sdio
-	  etc which is in APB bus.
-
-	  This DMA controller transfers data from memory to peripheral fifo.
-	  It does not support memory to memory data transfer.
-
 config LPC18XX_DMAMUX
 	bool "NXP LPC18xx/43xx DMA MUX for PL080"
 	depends on ARCH_LPC18XX || COMPILE_TEST
@@ -505,7 +482,7 @@ config MV_XOR_V2
 	  platforms.
 
 config MXS_DMA
-	bool "MXS DMA support"
+	tristate "MXS DMA support"
 	depends on ARCH_MXS || ARCH_MXC || COMPILE_TEST
 	select STMP_DEVICE
 	select DMA_ENGINE
@@ -610,6 +587,15 @@ config SPRD_DMA
 	help
 	  Enable support for the on-chip DMA controller on Spreadtrum platform.
 
+config SWITCHTEC_DMA
+	tristate "Switchtec PSX/PFX Switch DMA Engine Support"
+	depends on PCI
+	select DMA_ENGINE
+	help
+	  Some Switchtec PSX/PFX PCIe Switches support additional DMA engines.
+	  These are exposed via an extra function on the switch's upstream
+	  port.
+
 config TXX9_DMAC
 	tristate "Toshiba TXx9 SoC DMA support"
 	depends on MACH_TX49XX
@@ -774,6 +760,8 @@ source "drivers/dma/fsl-dpaa2-qdma/Kconfig"
 
 source "drivers/dma/lgm/Kconfig"
 
+source "drivers/dma/loongson/Kconfig"
+
 source "drivers/dma/stm32/Kconfig"
 
 # clients
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index a54d7688392b1..14aa086629d5c 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -49,8 +49,6 @@ obj-$(CONFIG_INTEL_IDMA64) += idma64.o
 obj-$(CONFIG_INTEL_IOATDMA) += ioat/
 obj-y += idxd/
 obj-$(CONFIG_K3_DMA) += k3dma.o
-obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o
-obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o
 obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o
 obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o
 obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o
@@ -74,6 +72,7 @@ obj-$(CONFIG_SF_PDMA) += sf-pdma/
 obj-$(CONFIG_SOPHGO_CV1800B_DMAMUX) += cv1800b-dmamux.o
 obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o
 obj-$(CONFIG_SPRD_DMA) += sprd-dma.o
+obj-$(CONFIG_SWITCHTEC_DMA) += switchtec_dma.o
 obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
 obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o
 obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
@@ -87,6 +86,7 @@ obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/
 obj-$(CONFIG_INTEL_LDMA) += lgm/
 
 obj-y += amd/
+obj-y += loongson/
 obj-y += mediatek/
 obj-y += qcom/
 obj-y += stm32/
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
index eb65872c5d5cd..45c2c8e4bc451 100644
--- a/drivers/dma/dma-axi-dmac.c
+++ b/drivers/dma/dma-axi-dmac.c
@@ -134,6 +134,7 @@ struct axi_dmac_desc {
 	struct axi_dmac_chan *chan;
 
 	bool cyclic;
+	bool cyclic_eot;
 	bool have_partial_xfer;
 
 	unsigned int num_submitted;
@@ -162,6 +163,7 @@ struct axi_dmac_chan {
 	bool hw_cyclic;
 	bool hw_2d;
 	bool hw_sg;
+	bool hw_cyclic_hotfix;
 };
 
 struct axi_dmac {
@@ -227,29 +229,94 @@ static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
 	return true;
 }
 
+static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
+{
+	return list_first_entry_or_null(&chan->active_descs,
+					struct axi_dmac_desc, vdesc.node);
+}
+
+static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac,
+						    struct axi_dmac_chan *chan)
+{
+	struct axi_dmac_desc *active = axi_dmac_active_desc(chan);
+	struct virt_dma_desc *vdesc;
+	struct axi_dmac_desc *desc;
+	unsigned int val;
+
+	/*
+	 * Just play safe and ignore any SOF if we have an active cyclic transfer
+	 * flagged to end. We'll start it as soon as the current cyclic one ends.
+	 */
+	if (active && active->cyclic_eot)
+		return NULL;
+
+	/*
+	 * It means a SW cyclic transfer is in place so we should just return
+	 * the same descriptor. SW cyclic transfer termination is handled
+	 * in axi_dmac_transfer_done().
+	 */
+	if (chan->next_desc)
+		return chan->next_desc;
+
+	vdesc = vchan_next_desc(&chan->vchan);
+	if (!vdesc)
+		return NULL;
+
+	if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) {
+		struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev;
+
+		dev_warn(dev, "Discarding non EOT transfer after cyclic\n");
+		list_del(&vdesc->node);
+		return NULL;
+	}
+
+	list_move_tail(&vdesc->node, &chan->active_descs);
+	desc = to_axi_dmac_desc(vdesc);
+	chan->next_desc = desc;
+
+	if (!active || !active->cyclic)
+		return desc;
+
+	active->cyclic_eot = true;
+
+	if (chan->hw_sg) {
+		unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST;
+		/*
+		 * Let's then stop the current cyclic transfer by making sure we
+		 * get an EOT interrupt and to open the cyclic loop by marking
+		 * the last segment.
+		 */
+		active->sg[active->num_sgs - 1].hw->flags = flags;
+		return NULL;
+	}
+
+	/*
+	 * Clear the cyclic bit if there's no Scatter-Gather HW so that we get
+	 * at the end of the transfer.
+	 */
+	val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS);
+	val &= ~AXI_DMAC_FLAG_CYCLIC;
+	axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val);
+
+	return NULL;
+}
+
 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 {
 	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
-	struct virt_dma_desc *vdesc;
 	struct axi_dmac_desc *desc;
 	struct axi_dmac_sg *sg;
 	unsigned int flags = 0;
 	unsigned int val;
 
+	desc = axi_dmac_get_next_desc(dmac, chan);
+	if (!desc)
+		return;
+
 	val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
 	if (val) /* Queue is full, wait for the next SOT IRQ */
 		return;
 
-	desc = chan->next_desc;
-
-	if (!desc) {
-		vdesc = vchan_next_desc(&chan->vchan);
-		if (!vdesc)
-			return;
-		list_move_tail(&vdesc->node, &chan->active_descs);
-		desc = to_axi_dmac_desc(vdesc);
-		chan->next_desc = desc;
-	}
 	sg = &desc->sg[desc->num_submitted];
 
 	/* Already queued in cyclic mode. Wait for it to finish */
@@ -291,10 +358,12 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 	 * call, enable hw cyclic mode to avoid unnecessary interrupts.
 	 */
 	if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) {
-		if (chan->hw_sg)
+		if (chan->hw_sg) {
 			desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ;
-		else if (desc->num_sgs == 1)
+		} else if (desc->num_sgs == 1) {
+			chan->next_desc = NULL;
 			flags |= AXI_DMAC_FLAG_CYCLIC;
+		}
 	}
 
 	if (chan->hw_partial_xfer)
@@ -312,12 +381,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 	axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
 }
 
-static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
-{
-	return list_first_entry_or_null(&chan->active_descs,
-		struct axi_dmac_desc, vdesc.node);
-}
-
 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
 	struct axi_dmac_sg *sg)
 {
@@ -398,6 +461,61 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
 	}
 }
 
+static bool axi_dmac_handle_cyclic_eot(struct axi_dmac_chan *chan,
+				       struct axi_dmac_desc *active)
+{
+	struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev;
+	struct virt_dma_desc *vdesc;
+
+	/* wrap around */
+	active->num_completed = 0;
+
+	if (active->cyclic_eot) {
+		/*
+		 * It means an HW cyclic transfer was marked to stop. And we
+		 * know we have something to schedule, so start the next
+		 * transfer now the cyclic one is done.
+		 */
+		list_del(&active->vdesc.node);
+		vchan_cookie_complete(&active->vdesc);
+
+		if (chan->hw_cyclic_hotfix) {
+			struct axi_dmac *dmac = chan_to_axi_dmac(chan);
+			/*
+			 * In older IP cores, ending a cyclic transfer by clearing
+			 * the CYCLIC flag does not guarantee a graceful end.
+			 * It can happen that some data (of the next frame) is
+			 * already prefetched and will be wrongly visible in the
+			 * next transfer. To workaround this, we need to reenable
+			 * the core so everything is flushed. Newer cores handles
+			 * this correctly and do not require this "hotfix". The
+			 * SG IP also does not require this.
+			 */
+			dev_dbg(dev, "HW cyclic hotfix\n");
+			axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
+			axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE);
+		}
+
+		return true;
+	}
+
+	vdesc = vchan_next_desc(&chan->vchan);
+	if (!vdesc)
+		return false;
+	if (!(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) {
+		dev_warn(dev, "Discarding non EOT transfer after cyclic\n");
+		list_del(&vdesc->node);
+		return false;
+	}
+
+	/* then let's end the cyclic transfer */
+	chan->next_desc = NULL;
+	list_del(&active->vdesc.node);
+	vchan_cookie_complete(&active->vdesc);
+
+	return true;
+}
+
 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 	unsigned int completed_transfers)
 {
@@ -416,6 +534,7 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 	if (chan->hw_sg) {
 		if (active->cyclic) {
 			vchan_cyclic_callback(&active->vdesc);
+			start_next = axi_dmac_handle_cyclic_eot(chan, active);
 		} else {
 			list_del(&active->vdesc.node);
 			vchan_cookie_complete(&active->vdesc);
@@ -445,7 +564,8 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 			if (active->num_completed == active->num_sgs ||
 			    sg->partial_len) {
 				if (active->cyclic) {
-					active->num_completed = 0; /* wrap around */
+					/* keep start_next as is, if already true... */
+					start_next |= axi_dmac_handle_cyclic_eot(chan, active);
 				} else {
 					list_del(&active->vdesc.node);
 					vchan_cookie_complete(&active->vdesc);
@@ -657,7 +777,12 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs,
 					      vecs[i].len, dsg);
 	}
 
-	desc->cyclic = false;
+	desc->cyclic = flags & DMA_PREP_REPEAT;
+	if (desc->cyclic) {
+		/* Chain the last descriptor to the first, and remove its "last" flag */
+		desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST;
+		desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys;
+	}
 
 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 }
@@ -1053,6 +1178,9 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
 		chan->length_align_mask = chan->address_align_mask;
 	}
 
+	if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg)
+		chan->hw_cyclic_hotfix = true;
+
 	return 0;
 }
 
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 27a8980b03dd2..405bd2fbb4a3b 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -31,29 +31,29 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/platform_device.h>
-#include <linux/dma-mapping.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
+#include <linux/acpi.h>
+#include <linux/acpi_dma.h>
 #include <linux/device.h>
+#include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/hardirq.h>
-#include <linux/spinlock.h>
+#include <linux/idr.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/mempool.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/numa.h>
 #include <linux/of.h>
-#include <linux/property.h>
+#include <linux/of_dma.h>
 #include <linux/percpu.h>
-#include <linux/rcupdate.h>
-#include <linux/mutex.h>
-#include <linux/jiffies.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/rculist.h>
-#include <linux/idr.h>
+#include <linux/rcupdate.h>
 #include <linux/slab.h>
-#include <linux/acpi.h>
-#include <linux/acpi_dma.h>
-#include <linux/of_dma.h>
-#include <linux/mempool.h>
-#include <linux/numa.h>
+#include <linux/spinlock.h>
 
 #include "dmaengine.h"
 
@@ -765,7 +765,7 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
 	mutex_lock(&dma_list_mutex);
 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
 		/* Finds a DMA controller with matching device node */
-		if (np && device->dev->of_node && np != device->dev->of_node)
+		if (np && !device_match_of_node(device->dev, np))
 			continue;
 
 		chan = find_candidate(device, mask, fn, fn_param);
@@ -943,12 +943,14 @@ static void dmaenginem_release_channel(void *chan)
 
 struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name)
 {
-	struct dma_chan *chan = dma_request_chan(dev, name);
-	int ret = 0;
+	struct dma_chan *chan;
+	int ret;
 
-	if (!IS_ERR(chan))
-		ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan);
+	chan = dma_request_chan(dev, name);
+	if (IS_ERR(chan))
+		return chan;
 
+	ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index 5d74bc29cf896..4d53f077e9d25 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -50,6 +50,7 @@
 #define AXI_DMA_FLAG_HAS_APB_REGS	BIT(0)
 #define AXI_DMA_FLAG_HAS_RESETS		BIT(1)
 #define AXI_DMA_FLAG_USE_CFG2		BIT(2)
+#define AXI_DMA_FLAG_ARG0_AS_CHAN	BIT(3)
 
 static inline void
 axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val)
@@ -342,8 +343,8 @@ static void axi_desc_put(struct axi_dma_desc *desc)
 	kfree(desc);
 	atomic_sub(descs_put, &chan->descs_allocated);
 	dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n",
-		axi_chan_name(chan), descs_put,
-		atomic_read(&chan->descs_allocated));
+		 axi_chan_name(chan), descs_put,
+		 atomic_read(&chan->descs_allocated));
 }
 
 static void vchan_desc_put(struct virt_dma_desc *vdesc)
@@ -353,7 +354,7 @@ static void vchan_desc_put(struct virt_dma_desc *vdesc)
 
 static enum dma_status
 dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
-		  struct dma_tx_state *txstate)
+		   struct dma_tx_state *txstate)
 {
 	struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);
 	struct virt_dma_desc *vdesc;
@@ -419,6 +420,7 @@ static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set)
 
 	iowrite32(val, chan->chip->apb_regs + offset);
 }
+
 /* Called in chan locked context */
 static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
 				      struct axi_dma_desc *first)
@@ -491,7 +493,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan)
 
 	desc = vd_to_axi_desc(vd);
 	dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan),
-		vd->tx.cookie);
+		 vd->tx.cookie);
 	axi_chan_block_xfer_start(chan, desc);
 }
 
@@ -592,8 +594,6 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set)
 			(chan->id * DMA_APB_HS_SEL_BIT_SIZE));
 	reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE));
 	lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0);
-
-	return;
 }
 
 /*
@@ -1162,7 +1162,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id)
 		axi_chan_irq_clear(chan, status);
 
 		dev_vdbg(chip->dev, "%s %u IRQ status: 0x%08x\n",
-			axi_chan_name(chan), i, status);
+			 axi_chan_name(chan), i, status);
 
 		if (status & DWAXIDMAC_IRQ_ALL_ERR)
 			axi_chan_handle_err(chan, status);
@@ -1358,16 +1358,27 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev)
 static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,
 					    struct of_dma *ofdma)
 {
+	unsigned int handshake = dma_spec->args[0];
 	struct dw_axi_dma *dw = ofdma->of_dma_data;
-	struct axi_dma_chan *chan;
+	struct axi_dma_chan *chan = NULL;
 	struct dma_chan *dchan;
 
-	dchan = dma_get_any_slave_channel(&dw->dma);
+	if (dw->hdata->use_handshake_as_channel_number) {
+		if (handshake >= dw->hdata->nr_channels)
+			return NULL;
+
+		chan = &dw->chan[handshake];
+		dchan = dma_get_slave_channel(&chan->vc.chan);
+	} else {
+		dchan = dma_get_any_slave_channel(&dw->dma);
+	}
+
 	if (!dchan)
 		return NULL;
 
-	chan = dchan_to_axi_dma_chan(dchan);
-	chan->hw_handshake_num = dma_spec->args[0];
+	if (!chan)
+		chan = dchan_to_axi_dma_chan(dchan);
+	chan->hw_handshake_num = handshake;
 	return dchan;
 }
 
@@ -1451,7 +1462,7 @@ static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip)
 		if (chip->irq[i] < 0)
 			return chip->irq[i];
 		ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt,
-				IRQF_SHARED, KBUILD_MODNAME, chip);
+				       IRQF_SHARED, KBUILD_MODNAME, chip);
 		if (ret < 0)
 			return ret;
 	}
@@ -1506,6 +1517,8 @@ static int dw_probe(struct platform_device *pdev)
 			return ret;
 	}
 
+	chip->dw->hdata->use_handshake_as_channel_number = !!(flags & AXI_DMA_FLAG_ARG0_AS_CHAN);
+
 	chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2);
 
 	chip->core_clk = devm_clk_get(chip->dev, "core-clk");
@@ -1645,7 +1658,7 @@ static void dw_remove(struct platform_device *pdev)
 	of_dma_controller_free(chip->dev->of_node);
 
 	list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
-			vc.chan.device_node) {
+				 vc.chan.device_node) {
 		list_del(&chan->vc.chan.device_node);
 		tasklet_kill(&chan->vc.task);
 	}
@@ -1661,6 +1674,9 @@ static const struct of_device_id dw_dma_of_id_table[] = {
 	}, {
 		.compatible = "intel,kmb-axi-dma",
 		.data = (void *)AXI_DMA_FLAG_HAS_APB_REGS,
+	}, {
+		.compatible = "sophgo,cv1800b-axi-dma",
+		.data = (void *)AXI_DMA_FLAG_ARG0_AS_CHAN,
 	}, {
 		.compatible = "starfive,jh7110-axi-dma",
 		.data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2),
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
index b842e6a8d90d3..67cc199e24d1f 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
@@ -34,6 +34,7 @@ struct dw_axi_dma_hcfg {
 	bool	reg_map_8_channels;
 	bool	restrict_axi_burst_len;
 	bool	use_cfg2;
+	bool	use_handshake_as_channel_number;
 };
 
 struct axi_dma_chan {
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index e7d698b352d37..c2feb3adc79fa 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -223,6 +223,43 @@ static int dw_edma_device_config(struct dma_chan *dchan,
 				 struct dma_slave_config *config)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+	bool cfg_non_ll;
+	int non_ll = 0;
+
+	chan->non_ll = false;
+	if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) {
+		if (config->peripheral_config &&
+		    config->peripheral_size != sizeof(int)) {
+			dev_err(dchan->device->dev,
+				"config param peripheral size mismatch\n");
+			return -EINVAL;
+		}
+
+		/*
+		 * When there is no valid LLP base address available then the
+		 * default DMA ops will use the non-LL mode.
+		 *
+		 * Cases where LL mode is enabled and client wants to use the
+		 * non-LL mode then also client can do so via providing the
+		 * peripheral_config param.
+		 */
+		cfg_non_ll = chan->dw->chip->cfg_non_ll;
+		if (config->peripheral_config) {
+			non_ll = *(int *)config->peripheral_config;
+
+			if (cfg_non_ll && !non_ll) {
+				dev_err(dchan->device->dev, "invalid configuration\n");
+				return -EINVAL;
+			}
+		}
+
+		if (cfg_non_ll || non_ll)
+			chan->non_ll = true;
+	} else if (config->peripheral_config) {
+		dev_err(dchan->device->dev,
+			"peripheral config param applicable only for HDMA\n");
+		return -EINVAL;
+	}
 
 	memcpy(&chan->config, config, sizeof(*config));
 	chan->configured = true;
@@ -358,6 +395,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 	struct dw_edma_desc *desc;
 	u64 src_addr, dst_addr;
 	size_t fsz = 0;
+	u32 bursts_max;
 	u32 cnt = 0;
 	int i;
 
@@ -415,6 +453,13 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		return NULL;
 	}
 
+	/*
+	 * For non-LL mode, only a single burst can be handled
+	 * in a single chunk unlike LL mode where multiple bursts
+	 * can be configured in a single chunk.
+	 */
+	bursts_max = chan->non_ll ? 1 : chan->ll_max;
+
 	desc = dw_edma_alloc_desc(chan);
 	if (unlikely(!desc))
 		goto err_alloc;
@@ -450,7 +495,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
 			break;
 
-		if (chunk->bursts_alloc == chan->ll_max) {
+		if (chunk->bursts_alloc == bursts_max) {
 			chunk = dw_edma_alloc_chunk(desc);
 			if (unlikely(!chunk))
 				goto err_alloc;
@@ -663,7 +708,96 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 	chan->status = EDMA_ST_IDLE;
 }
 
-static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
+static void dw_edma_emul_irq_ack(struct irq_data *d)
+{
+	struct dw_edma *dw = irq_data_get_irq_chip_data(d);
+
+	dw_edma_core_ack_emulated_irq(dw);
+}
+
+/*
+ * irq_chip implementation for interrupt-emulation doorbells.
+ *
+ * The emulated source has no mask/unmask mechanism. With handle_level_irq(),
+ * the flow is therefore:
+ *   1) .irq_ack() deasserts the source
+ *   2) registered handlers (if any) are dispatched
+ * Since deassertion is already done in .irq_ack(), handlers do not need to take
+ * care of it, hence IRQCHIP_ONESHOT_SAFE.
+ */
+static struct irq_chip dw_edma_emul_irqchip = {
+	.name		= "dw-edma-emul",
+	.irq_ack	= dw_edma_emul_irq_ack,
+	.flags		= IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE,
+};
+
+static int dw_edma_emul_irq_alloc(struct dw_edma *dw)
+{
+	struct dw_edma_chip *chip = dw->chip;
+	int virq;
+
+	chip->db_irq = 0;
+	chip->db_offset = ~0;
+
+	/*
+	 * Only meaningful when the core provides the deassert sequence
+	 * for interrupt emulation.
+	 */
+	if (!dw->core->ack_emulated_irq)
+		return 0;
+
+	/*
+	 * Allocate a single, requestable Linux virtual IRQ number.
+	 * Use >= 1 so that 0 can remain a "not available" sentinel.
+	 */
+	virq = irq_alloc_desc(NUMA_NO_NODE);
+	if (virq < 0)
+		return virq;
+
+	irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq);
+	irq_set_chip_data(virq, dw);
+	irq_set_noprobe(virq);
+
+	chip->db_irq = virq;
+	chip->db_offset = dw_edma_core_db_offset(dw);
+
+	return 0;
+}
+
+static void dw_edma_emul_irq_free(struct dw_edma *dw)
+{
+	struct dw_edma_chip *chip = dw->chip;
+
+	if (!chip)
+		return;
+	if (chip->db_irq <= 0)
+		return;
+
+	irq_free_descs(chip->db_irq, 1);
+	chip->db_irq = 0;
+	chip->db_offset = ~0;
+}
+
+static inline irqreturn_t dw_edma_interrupt_emulated(void *data)
+{
+	struct dw_edma_irq *dw_irq = data;
+	struct dw_edma *dw = dw_irq->dw;
+	int db_irq = dw->chip->db_irq;
+
+	if (db_irq > 0) {
+		/*
+		 * Interrupt emulation may assert the IRQ line without updating the
+		 * normal DONE/ABORT status bits. With a shared IRQ handler we
+		 * cannot reliably detect such events by status registers alone, so
+		 * always perform the core-specific deassert sequence.
+		 */
+		generic_handle_irq(db_irq);
+		return IRQ_HANDLED;
+	}
+	return IRQ_NONE;
+}
+
+static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data)
 {
 	struct dw_edma_irq *dw_irq = data;
 
@@ -672,7 +806,7 @@ static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
 				       dw_edma_abort_interrupt);
 }
 
-static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
+static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
 {
 	struct dw_edma_irq *dw_irq = data;
 
@@ -681,12 +815,33 @@ static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
 				       dw_edma_abort_interrupt);
 }
 
-static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
+static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
+{
+	irqreturn_t ret = IRQ_NONE;
+
+	ret |= dw_edma_interrupt_write_inner(irq, data);
+	ret |= dw_edma_interrupt_emulated(data);
+
+	return ret;
+}
+
+static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
 {
 	irqreturn_t ret = IRQ_NONE;
 
-	ret |= dw_edma_interrupt_write(irq, data);
-	ret |= dw_edma_interrupt_read(irq, data);
+	ret |= dw_edma_interrupt_read_inner(irq, data);
+	ret |= dw_edma_interrupt_emulated(data);
+
+	return ret;
+}
+
+static inline irqreturn_t dw_edma_interrupt_common(int irq, void *data)
+{
+	irqreturn_t ret = IRQ_NONE;
+
+	ret |= dw_edma_interrupt_write_inner(irq, data);
+	ret |= dw_edma_interrupt_read_inner(irq, data);
+	ret |= dw_edma_interrupt_emulated(data);
 
 	return ret;
 }
@@ -844,6 +999,7 @@ static int dw_edma_irq_request(struct dw_edma *dw,
 {
 	struct dw_edma_chip *chip = dw->chip;
 	struct device *dev = dw->chip->dev;
+	struct msi_desc *msi_desc;
 	u32 wr_mask = 1;
 	u32 rd_mask = 1;
 	int i, err = 0;
@@ -895,9 +1051,12 @@ static int dw_edma_irq_request(struct dw_edma *dw,
 					  &dw->irq[i]);
 			if (err)
 				goto err_irq_free;
-
-			if (irq_get_msi_desc(irq))
+			msi_desc = irq_get_msi_desc(irq);
+			if (msi_desc) {
 				get_cached_msi_msg(irq, &dw->irq[i].msi);
+				if (!msi_desc->pci.msi_attrib.is_msix)
+					dw->irq[i].msi.data = dw->irq[0].msi.data + i;
+			}
 		}
 
 		dw->nr_irqs = i;
@@ -973,6 +1132,11 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 	if (err)
 		return err;
 
+	/* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */
+	err = dw_edma_emul_irq_alloc(dw);
+	if (err)
+		dev_warn(dev, "Failed to allocate emulation IRQ: %d\n", err);
+
 	/* Setup write/read channels */
 	err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
 	if (err)
@@ -988,6 +1152,7 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 err_irq_free:
 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
 		free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
+	dw_edma_emul_irq_free(dw);
 
 	return err;
 }
@@ -1010,6 +1175,7 @@ int dw_edma_remove(struct dw_edma_chip *chip)
 	/* Free irqs */
 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
 		free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
+	dw_edma_emul_irq_free(dw);
 
 	/* Deregister eDMA device */
 	dma_async_device_unregister(&dw->dma);
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 71894b9e0b153..902574b1ba867 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -86,6 +86,7 @@ struct dw_edma_chan {
 	u8				configured;
 
 	struct dma_slave_config		config;
+	bool				non_ll;
 };
 
 struct dw_edma_irq {
@@ -126,6 +127,8 @@ struct dw_edma_core_ops {
 	void (*start)(struct dw_edma_chunk *chunk, bool first);
 	void (*ch_config)(struct dw_edma_chan *chan);
 	void (*debugfs_on)(struct dw_edma *dw);
+	void (*ack_emulated_irq)(struct dw_edma *dw);
+	resource_size_t (*db_offset)(struct dw_edma *dw);
 };
 
 struct dw_edma_sg {
@@ -206,4 +209,19 @@ void dw_edma_core_debugfs_on(struct dw_edma *dw)
 	dw->core->debugfs_on(dw);
 }
 
+static inline int dw_edma_core_ack_emulated_irq(struct dw_edma *dw)
+{
+	if (!dw->core->ack_emulated_irq)
+		return -EOPNOTSUPP;
+
+	dw->core->ack_emulated_irq(dw);
+	return 0;
+}
+
+static inline resource_size_t
+dw_edma_core_db_offset(struct dw_edma *dw)
+{
+	return dw->core->db_offset(dw);
+}
+
 #endif /* _DW_EDMA_CORE_H */
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 83230acaa5970..0b30ce1385036 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -14,14 +14,35 @@
 #include <linux/pci-epf.h>
 #include <linux/msi.h>
 #include <linux/bitfield.h>
+#include <linux/sizes.h>
 
 #include "dw-edma-core.h"
 
-#define DW_PCIE_VSEC_DMA_ID			0x6
-#define DW_PCIE_VSEC_DMA_BAR			GENMASK(10, 8)
-#define DW_PCIE_VSEC_DMA_MAP			GENMASK(2, 0)
-#define DW_PCIE_VSEC_DMA_WR_CH			GENMASK(9, 0)
-#define DW_PCIE_VSEC_DMA_RD_CH			GENMASK(25, 16)
+/* Synopsys */
+#define DW_PCIE_SYNOPSYS_VSEC_DMA_ID		0x6
+#define DW_PCIE_SYNOPSYS_VSEC_DMA_BAR		GENMASK(10, 8)
+#define DW_PCIE_SYNOPSYS_VSEC_DMA_MAP		GENMASK(2, 0)
+#define DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH		GENMASK(9, 0)
+#define DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH		GENMASK(25, 16)
+
+/* AMD MDB (Xilinx) specific defines */
+#define PCI_DEVICE_ID_XILINX_B054		0xb054
+
+#define DW_PCIE_XILINX_MDB_VSEC_DMA_ID		0x6
+#define DW_PCIE_XILINX_MDB_VSEC_ID		0x20
+#define DW_PCIE_XILINX_MDB_VSEC_DMA_BAR		GENMASK(10, 8)
+#define DW_PCIE_XILINX_MDB_VSEC_DMA_MAP		GENMASK(2, 0)
+#define DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH	GENMASK(9, 0)
+#define DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH	GENMASK(25, 16)
+
+#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH	0xc
+#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW	0x8
+#define DW_PCIE_XILINX_MDB_INVALID_ADDR		(~0ULL)
+
+#define DW_PCIE_XILINX_MDB_LL_OFF_GAP		0x200000
+#define DW_PCIE_XILINX_MDB_LL_SIZE		0x800
+#define DW_PCIE_XILINX_MDB_DT_OFF_GAP		0x100000
+#define DW_PCIE_XILINX_MDB_DT_SIZE		0x800
 
 #define DW_BLOCK(a, b, c) \
 	{ \
@@ -50,6 +71,7 @@ struct dw_edma_pcie_data {
 	u8				irqs;
 	u16				wr_ch_cnt;
 	u16				rd_ch_cnt;
+	u64				devmem_phys_off;
 };
 
 static const struct dw_edma_pcie_data snps_edda_data = {
@@ -90,6 +112,64 @@ static const struct dw_edma_pcie_data snps_edda_data = {
 	.rd_ch_cnt			= 2,
 };
 
+static const struct dw_edma_pcie_data xilinx_mdb_data = {
+	/* MDB registers location */
+	.rg.bar				= BAR_0,
+	.rg.off				= SZ_4K,	/*  4 Kbytes */
+	.rg.sz				= SZ_8K,	/*  8 Kbytes */
+
+	/* Other */
+	.mf				= EDMA_MF_HDMA_NATIVE,
+	.irqs				= 1,
+	.wr_ch_cnt			= 8,
+	.rd_ch_cnt			= 8,
+};
+
+static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata,
+					   enum pci_barno bar, off_t start_off,
+					   off_t ll_off_gap, size_t ll_size,
+					   off_t dt_off_gap, size_t dt_size)
+{
+	u16 wr_ch = pdata->wr_ch_cnt;
+	u16 rd_ch = pdata->rd_ch_cnt;
+	off_t off;
+	u16 i;
+
+	off = start_off;
+
+	/* Write channel LL region */
+	for (i = 0; i < wr_ch; i++) {
+		pdata->ll_wr[i].bar = bar;
+		pdata->ll_wr[i].off = off;
+		pdata->ll_wr[i].sz = ll_size;
+		off += ll_off_gap;
+	}
+
+	/* Read channel LL region */
+	for (i = 0; i < rd_ch; i++) {
+		pdata->ll_rd[i].bar = bar;
+		pdata->ll_rd[i].off = off;
+		pdata->ll_rd[i].sz = ll_size;
+		off += ll_off_gap;
+	}
+
+	/* Write channel data region */
+	for (i = 0; i < wr_ch; i++) {
+		pdata->dt_wr[i].bar = bar;
+		pdata->dt_wr[i].off = off;
+		pdata->dt_wr[i].sz = dt_size;
+		off += dt_off_gap;
+	}
+
+	/* Read channel data region */
+	for (i = 0; i < rd_ch; i++) {
+		pdata->dt_rd[i].bar = bar;
+		pdata->dt_rd[i].off = off;
+		pdata->dt_rd[i].sz = dt_size;
+		off += dt_off_gap;
+	}
+}
+
 static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr)
 {
 	return pci_irq_vector(to_pci_dev(dev), nr);
@@ -114,15 +194,15 @@ static const struct dw_edma_plat_ops dw_edma_pcie_plat_ops = {
 	.pci_address = dw_edma_pcie_address,
 };
 
-static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
-					   struct dw_edma_pcie_data *pdata)
+static void dw_edma_pcie_get_synopsys_dma_data(struct pci_dev *pdev,
+					       struct dw_edma_pcie_data *pdata)
 {
 	u32 val, map;
 	u16 vsec;
 	u64 off;
 
 	vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_SYNOPSYS,
-					DW_PCIE_VSEC_DMA_ID);
+					DW_PCIE_SYNOPSYS_VSEC_DMA_ID);
 	if (!vsec)
 		return;
 
@@ -131,9 +211,9 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
 	    PCI_VNDR_HEADER_LEN(val) != 0x18)
 		return;
 
-	pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability DMA\n");
+	pci_dbg(pdev, "Detected Synopsys PCIe Vendor-Specific Extended Capability DMA\n");
 	pci_read_config_dword(pdev, vsec + 0x8, &val);
-	map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val);
+	map = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_MAP, val);
 	if (map != EDMA_MF_EDMA_LEGACY &&
 	    map != EDMA_MF_EDMA_UNROLL &&
 	    map != EDMA_MF_HDMA_COMPAT &&
@@ -141,13 +221,55 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
 		return;
 
 	pdata->mf = map;
-	pdata->rg.bar = FIELD_GET(DW_PCIE_VSEC_DMA_BAR, val);
+	pdata->rg.bar = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_BAR, val);
 
 	pci_read_config_dword(pdev, vsec + 0xc, &val);
 	pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt,
-				 FIELD_GET(DW_PCIE_VSEC_DMA_WR_CH, val));
+				 FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH, val));
 	pdata->rd_ch_cnt = min_t(u16, pdata->rd_ch_cnt,
-				 FIELD_GET(DW_PCIE_VSEC_DMA_RD_CH, val));
+				 FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH, val));
+
+	pci_read_config_dword(pdev, vsec + 0x14, &val);
+	off = val;
+	pci_read_config_dword(pdev, vsec + 0x10, &val);
+	off <<= 32;
+	off |= val;
+	pdata->rg.off = off;
+}
+
+static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev,
+					     struct dw_edma_pcie_data *pdata)
+{
+	u32 val, map;
+	u16 vsec;
+	u64 off;
+
+	pdata->devmem_phys_off = DW_PCIE_XILINX_MDB_INVALID_ADDR;
+
+	vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX,
+					DW_PCIE_XILINX_MDB_VSEC_DMA_ID);
+	if (!vsec)
+		return;
+
+	pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
+	if (PCI_VNDR_HEADER_REV(val) != 0x00 ||
+	    PCI_VNDR_HEADER_LEN(val) != 0x18)
+		return;
+
+	pci_dbg(pdev, "Detected Xilinx PCIe Vendor-Specific Extended Capability DMA\n");
+	pci_read_config_dword(pdev, vsec + 0x8, &val);
+	map = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_MAP, val);
+	if (map != EDMA_MF_HDMA_NATIVE)
+		return;
+
+	pdata->mf = map;
+	pdata->rg.bar = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_BAR, val);
+
+	pci_read_config_dword(pdev, vsec + 0xc, &val);
+	pdata->wr_ch_cnt = min(pdata->wr_ch_cnt,
+			       FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH, val));
+	pdata->rd_ch_cnt = min(pdata->rd_ch_cnt,
+			       FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH, val));
 
 	pci_read_config_dword(pdev, vsec + 0x14, &val);
 	off = val;
@@ -155,6 +277,31 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
 	off <<= 32;
 	off |= val;
 	pdata->rg.off = off;
+
+	vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX,
+					DW_PCIE_XILINX_MDB_VSEC_ID);
+	if (!vsec)
+		return;
+
+	pci_read_config_dword(pdev,
+			      vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH,
+			      &val);
+	off = val;
+	pci_read_config_dword(pdev,
+			      vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW,
+			      &val);
+	off <<= 32;
+	off |= val;
+	pdata->devmem_phys_off = off;
+}
+
+static u64 dw_edma_get_phys_addr(struct pci_dev *pdev,
+				 struct dw_edma_pcie_data *pdata,
+				 enum pci_barno bar)
+{
+	if (pdev->vendor == PCI_VENDOR_ID_XILINX)
+		return pdata->devmem_phys_off;
+	return pci_bus_address(pdev, bar);
 }
 
 static int dw_edma_pcie_probe(struct pci_dev *pdev,
@@ -165,6 +312,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	struct dw_edma_chip *chip;
 	int err, nr_irqs;
 	int i, mask;
+	bool non_ll = false;
 
 	struct dw_edma_pcie_data *vsec_data __free(kfree) =
 		kmalloc_obj(*vsec_data);
@@ -184,7 +332,32 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	 * Tries to find if exists a PCIe Vendor-Specific Extended Capability
 	 * for the DMA, if one exists, then reconfigures it.
 	 */
-	dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data);
+	dw_edma_pcie_get_synopsys_dma_data(pdev, vsec_data);
+
+	if (pdev->vendor == PCI_VENDOR_ID_XILINX) {
+		dw_edma_pcie_get_xilinx_dma_data(pdev, vsec_data);
+
+		/*
+		 * There is no valid address found for the LL memory
+		 * space on the device side. In the absence of LL base
+		 * address use the non-LL mode or simple mode supported by
+		 * the HDMA IP.
+		 */
+		if (vsec_data->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR)
+			non_ll = true;
+
+		/*
+		 * Configure the channel LL and data blocks if number of
+		 * channels enabled in VSEC capability are more than the
+		 * channels configured in xilinx_mdb_data.
+		 */
+		if (!non_ll)
+			dw_edma_set_chan_region_offset(vsec_data, BAR_2, 0,
+						       DW_PCIE_XILINX_MDB_LL_OFF_GAP,
+						       DW_PCIE_XILINX_MDB_LL_SIZE,
+						       DW_PCIE_XILINX_MDB_DT_OFF_GAP,
+						       DW_PCIE_XILINX_MDB_DT_SIZE);
+	}
 
 	/* Mapping PCI BAR regions */
 	mask = BIT(vsec_data->rg.bar);
@@ -231,6 +404,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	chip->mf = vsec_data->mf;
 	chip->nr_irqs = nr_irqs;
 	chip->ops = &dw_edma_pcie_plat_ops;
+	chip->cfg_non_ll = non_ll;
 
 	chip->ll_wr_cnt = vsec_data->wr_ch_cnt;
 	chip->ll_rd_cnt = vsec_data->rd_ch_cnt;
@@ -239,7 +413,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	if (!chip->reg_base)
 		return -ENOMEM;
 
-	for (i = 0; i < chip->ll_wr_cnt; i++) {
+	for (i = 0; i < chip->ll_wr_cnt && !non_ll; i++) {
 		struct dw_edma_region *ll_region = &chip->ll_region_wr[i];
 		struct dw_edma_region *dt_region = &chip->dt_region_wr[i];
 		struct dw_edma_block *ll_block = &vsec_data->ll_wr[i];
@@ -250,7 +424,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		ll_region->vaddr.io += ll_block->off;
-		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
+		ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
+							 ll_block->bar);
 		ll_region->paddr += ll_block->off;
 		ll_region->sz = ll_block->sz;
 
@@ -259,12 +434,13 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		dt_region->vaddr.io += dt_block->off;
-		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
+		dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
+							 dt_block->bar);
 		dt_region->paddr += dt_block->off;
 		dt_region->sz = dt_block->sz;
 	}
 
-	for (i = 0; i < chip->ll_rd_cnt; i++) {
+	for (i = 0; i < chip->ll_rd_cnt && !non_ll; i++) {
 		struct dw_edma_region *ll_region = &chip->ll_region_rd[i];
 		struct dw_edma_region *dt_region = &chip->dt_region_rd[i];
 		struct dw_edma_block *ll_block = &vsec_data->ll_rd[i];
@@ -275,7 +451,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		ll_region->vaddr.io += ll_block->off;
-		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
+		ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
+							 ll_block->bar);
 		ll_region->paddr += ll_block->off;
 		ll_region->sz = ll_block->sz;
 
@@ -284,7 +461,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		dt_region->vaddr.io += dt_block->off;
-		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
+		dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
+							 dt_block->bar);
 		dt_region->paddr += dt_block->off;
 		dt_region->sz = dt_block->sz;
 	}
@@ -367,6 +545,8 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev)
 
 static const struct pci_device_id dw_edma_pcie_id_table[] = {
 	{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) },
+	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054),
+	  (kernel_ulong_t)&xilinx_mdb_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table);
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index b75fdaffad9a4..69e8279adec81 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -509,6 +509,25 @@ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 	dw_edma_v0_debugfs_on(dw);
 }
 
+static void dw_edma_v0_core_ack_emulated_irq(struct dw_edma *dw)
+{
+	/*
+	 * Interrupt emulation may assert the IRQ without setting
+	 * DONE/ABORT status bits. A zero write to INT_CLEAR deasserts the
+	 * emulated IRQ, while being a no-op for real interrupts.
+	 */
+	SET_BOTH_32(dw, int_clear, 0);
+}
+
+static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw)
+{
+	/*
+	 * rd_int_status is chosen arbitrarily, but wr_int_status would be
+	 * equally suitable.
+	 */
+	return offsetof(struct dw_edma_v0_regs, rd_int_status);
+}
+
 static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.off = dw_edma_v0_core_off,
 	.ch_count = dw_edma_v0_core_ch_count,
@@ -517,6 +536,8 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.start = dw_edma_v0_core_start,
 	.ch_config = dw_edma_v0_core_ch_config,
 	.debugfs_on = dw_edma_v0_core_debugfs_on,
+	.ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq,
+	.db_offset = dw_edma_v0_core_db_offset,
 };
 
 void dw_edma_v0_core_register(struct dw_edma *dw)
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index e3f8db4fe909a..632abb8b481cf 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -225,7 +225,7 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 		readl(chunk->ll_region.vaddr.io);
 }
 
-static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
+static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first)
 {
 	struct dw_edma_chan *chan = chunk->chan;
 	struct dw_edma *dw = chan->dw;
@@ -252,10 +252,10 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
 			  lower_32_bits(chunk->ll_region.paddr));
 		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
 			  upper_32_bits(chunk->ll_region.paddr));
+		/* Set consumer cycle */
+		SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
+			HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
 	}
-	/* Set consumer cycle */
-	SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
-		  HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
 
 	dw_hdma_v0_sync_ll_data(chunk);
 
@@ -263,6 +263,68 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
 	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
 }
 
+static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
+{
+	struct dw_edma_chan *chan = chunk->chan;
+	struct dw_edma *dw = chan->dw;
+	struct dw_edma_burst *child;
+	u32 val;
+
+	child = list_first_entry_or_null(&chunk->burst->list,
+					 struct dw_edma_burst, list);
+	if (!child)
+		return;
+
+	SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN);
+
+	/* Source address */
+	SET_CH_32(dw, chan->dir, chan->id, sar.lsb,
+		  lower_32_bits(child->sar));
+	SET_CH_32(dw, chan->dir, chan->id, sar.msb,
+		  upper_32_bits(child->sar));
+
+	/* Destination address */
+	SET_CH_32(dw, chan->dir, chan->id, dar.lsb,
+		  lower_32_bits(child->dar));
+	SET_CH_32(dw, chan->dir, chan->id, dar.msb,
+		  upper_32_bits(child->dar));
+
+	/* Transfer size */
+	SET_CH_32(dw, chan->dir, chan->id, transfer_size, child->sz);
+
+	/* Interrupt setup */
+	val = GET_CH_32(dw, chan->dir, chan->id, int_setup) |
+			HDMA_V0_STOP_INT_MASK |
+			HDMA_V0_ABORT_INT_MASK |
+			HDMA_V0_LOCAL_STOP_INT_EN |
+			HDMA_V0_LOCAL_ABORT_INT_EN;
+
+	if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) {
+		val |= HDMA_V0_REMOTE_STOP_INT_EN |
+		       HDMA_V0_REMOTE_ABORT_INT_EN;
+	}
+
+	SET_CH_32(dw, chan->dir, chan->id, int_setup, val);
+
+	/* Channel control setup */
+	val = GET_CH_32(dw, chan->dir, chan->id, control1);
+	val &= ~HDMA_V0_LINKLIST_EN;
+	SET_CH_32(dw, chan->dir, chan->id, control1, val);
+
+	SET_CH_32(dw, chan->dir, chan->id, doorbell,
+		  HDMA_V0_DOORBELL_START);
+}
+
+static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
+{
+	struct dw_edma_chan *chan = chunk->chan;
+
+	if (chan->non_ll)
+		dw_hdma_v0_core_non_ll_start(chunk);
+	else
+		dw_hdma_v0_core_ll_start(chunk, first);
+}
+
 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
@@ -283,6 +345,12 @@ static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 	dw_hdma_v0_debugfs_on(dw);
 }
 
+static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw)
+{
+	/* Implement once the correct offset is known. */
+	return ~0;
+}
+
 static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.off = dw_hdma_v0_core_off,
 	.ch_count = dw_hdma_v0_core_ch_count,
@@ -291,6 +359,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.start = dw_hdma_v0_core_start,
 	.ch_config = dw_hdma_v0_core_ch_config,
 	.debugfs_on = dw_hdma_v0_core_debugfs_on,
+	.db_offset = dw_hdma_v0_core_db_offset,
 };
 
 void dw_hdma_v0_core_register(struct dw_edma *dw)
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index eab5fd7177e54..7759ba9b4850f 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -12,6 +12,7 @@
 #include <linux/dmaengine.h>
 
 #define HDMA_V0_MAX_NR_CH			8
+#define HDMA_V0_CH_EN				BIT(0)
 #define HDMA_V0_LOCAL_ABORT_INT_EN		BIT(6)
 #define HDMA_V0_REMOTE_ABORT_INT_EN		BIT(5)
 #define HDMA_V0_LOCAL_STOP_INT_EN		BIT(4)
diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c
index 6a38738e56e20..bb7531c456dfa 100644
--- a/drivers/dma/fsl-edma-common.c
+++ b/drivers/dma/fsl-edma-common.c
@@ -905,8 +905,7 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan)
 	fsl_chan->is_sw = false;
 	fsl_chan->srcid = 0;
 	fsl_chan->is_remote = false;
-	if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK)
-		clk_disable_unprepare(fsl_chan->clk);
+	clk_disable_unprepare(fsl_chan->clk);
 }
 
 void fsl_edma_cleanup_vchan(struct dma_device *dmadev)
diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c
index dbcdd1e683190..36155ab1602a9 100644
--- a/drivers/dma/fsl-edma-main.c
+++ b/drivers/dma/fsl-edma-main.c
@@ -317,10 +317,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec,
 			return NULL;
 		i = fsl_chan - fsl_edma->chans;
 
-		fsl_chan->priority = dma_spec->args[1];
-		fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX;
-		fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE;
-		fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO;
+		if (!b_chmux && i != dma_spec->args[0])
+			continue;
 
 		if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1))
 			continue;
@@ -328,17 +326,15 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec,
 		if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1))
 			continue;
 
-		if (!b_chmux && i == dma_spec->args[0]) {
-			chan = dma_get_slave_channel(chan);
-			chan->device->privatecnt++;
-			return chan;
-		} else if (b_chmux && !fsl_chan->srcid) {
-			/* if controller support channel mux, choose a free channel */
-			chan = dma_get_slave_channel(chan);
-			chan->device->privatecnt++;
-			fsl_chan->srcid = dma_spec->args[0];
-			return chan;
-		}
+		fsl_chan->srcid = dma_spec->args[0];
+		fsl_chan->priority = dma_spec->args[1];
+		fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX;
+		fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE;
+		fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO;
+
+		chan = dma_get_slave_channel(chan);
+		chan->device->privatecnt++;
+		return chan;
 	}
 	return NULL;
 }
@@ -709,16 +705,14 @@ static int fsl_edma_probe(struct platform_device *pdev)
 	int ret, i;
 
 	drvdata = device_get_match_data(&pdev->dev);
-	if (!drvdata) {
-		dev_err(&pdev->dev, "unable to find driver data\n");
-		return -EINVAL;
-	}
+	if (!drvdata)
+		return dev_err_probe(&pdev->dev, -EINVAL,
+				     "unable to find driver data\n");
 
 	ret = of_property_read_u32(np, "dma-channels", &chans);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't get dma-channels.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't get dma-channels.\n");
 
 	fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans),
 				GFP_KERNEL);
@@ -742,10 +736,10 @@ static int fsl_edma_probe(struct platform_device *pdev)
 
 	if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) {
 		fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma");
-		if (IS_ERR(fsl_edma->dmaclk)) {
-			dev_err(&pdev->dev, "Missing DMA block clock.\n");
-			return PTR_ERR(fsl_edma->dmaclk);
-		}
+		if (IS_ERR(fsl_edma->dmaclk))
+			return dev_err_probe(&pdev->dev,
+					     PTR_ERR(fsl_edma->dmaclk),
+					     "Missing DMA block clock.\n");
 	}
 
 	ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2);
@@ -769,11 +763,10 @@ static int fsl_edma_probe(struct platform_device *pdev)
 
 		sprintf(clkname, "dmamux%d", i);
 		fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname);
-		if (IS_ERR(fsl_edma->muxclk[i])) {
-			dev_err(&pdev->dev, "Missing DMAMUX block clock.\n");
-			/* on error: disable all previously enabled clks */
-			return PTR_ERR(fsl_edma->muxclk[i]);
-		}
+		if (IS_ERR(fsl_edma->muxclk[i]))
+			return dev_err_probe(&pdev->dev,
+					     PTR_ERR(fsl_edma->muxclk[i]),
+					     "Missing DMAMUX block clock.\n");
 	}
 
 	fsl_edma->big_endian = of_property_read_bool(np, "big-endian");
@@ -882,22 +875,17 @@ static int fsl_edma_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, fsl_edma);
 
-	ret = dma_async_device_register(&fsl_edma->dma_dev);
-	if (ret) {
-		dev_err(&pdev->dev,
-			"Can't register Freescale eDMA engine. (%d)\n", ret);
-		return ret;
-	}
+	ret = dmaenginem_async_device_register(&fsl_edma->dma_dev);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't register Freescale eDMA engine.\n");
 
-	ret = of_dma_controller_register(np,
+	ret = devm_of_dma_controller_register(&pdev->dev, np,
 			drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate,
 			fsl_edma);
-	if (ret) {
-		dev_err(&pdev->dev,
-			"Can't register Freescale eDMA of_dma. (%d)\n", ret);
-		dma_async_device_unregister(&fsl_edma->dma_dev);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't register Freescale eDMA of_dma.\n");
 
 	/* enable round robin arbitration */
 	if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG))
@@ -908,12 +896,9 @@ static int fsl_edma_probe(struct platform_device *pdev)
 
 static void fsl_edma_remove(struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
 	struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev);
 
 	fsl_edma_irq_exit(pdev, fsl_edma);
-	of_dma_controller_free(np);
-	dma_async_device_unregister(&fsl_edma->dma_dev);
 	fsl_edma_cleanup_vchan(&fsl_edma->dma_dev);
 }
 
diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c
index 0bbff9df362f8..df843fad0ece5 100644
--- a/drivers/dma/fsl-qdma.c
+++ b/drivers/dma/fsl-qdma.c
@@ -1127,22 +1127,19 @@ static int fsl_qdma_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 
 	ret = of_property_read_u32(np, "dma-channels", &chans);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't get dma-channels.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't get dma-channels.\n");
 
 	ret = of_property_read_u32(np, "block-offset", &blk_off);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't get block-offset.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't get block-offset.\n");
 
 	ret = of_property_read_u32(np, "block-number", &blk_num);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't get block-number.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't get block-number.\n");
 
 	blk_num = min_t(int, blk_num, num_online_cpus());
 
@@ -1167,10 +1164,8 @@ static int fsl_qdma_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ret = of_property_read_u32(np, "fsl,dma-queues", &queues);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't get queues.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "Can't get queues.\n");
 
 	fsl_qdma->desc_allocated = 0;
 	fsl_qdma->n_chans = chans;
@@ -1231,28 +1226,24 @@ static int fsl_qdma_probe(struct platform_device *pdev)
 	fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all;
 
 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
-	if (ret) {
-		dev_err(&pdev->dev, "dma_set_mask failure.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "dma_set_mask failure.\n");
 
 	platform_set_drvdata(pdev, fsl_qdma);
 
 	ret = fsl_qdma_reg_init(fsl_qdma);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't Initialize the qDMA engine.\n");
 
 	ret = fsl_qdma_irq_init(pdev, fsl_qdma);
 	if (ret)
 		return ret;
 
 	ret = dma_async_device_register(&fsl_qdma->dma_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't register NXP Layerscape qDMA engine.\n");
 
 	return 0;
 }
diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c
index c37d233535f9e..0366c7cf35020 100644
--- a/drivers/dma/idxd/cdev.c
+++ b/drivers/dma/idxd/cdev.c
@@ -158,11 +158,7 @@ static const struct device_type idxd_cdev_file_type = {
 static void idxd_cdev_dev_release(struct device *dev)
 {
 	struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
-	struct idxd_cdev_context *cdev_ctx;
-	struct idxd_wq *wq = idxd_cdev->wq;
 
-	cdev_ctx = &ictx[wq->idxd->data->type];
-	ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
 	kfree(idxd_cdev);
 }
 
@@ -582,11 +578,15 @@ int idxd_wq_add_cdev(struct idxd_wq *wq)
 
 void idxd_wq_del_cdev(struct idxd_wq *wq)
 {
+	struct idxd_cdev_context *cdev_ctx;
 	struct idxd_cdev *idxd_cdev;
 
 	idxd_cdev = wq->idxd_cdev;
 	wq->idxd_cdev = NULL;
 	cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
+
+	cdev_ctx = &ictx[wq->idxd->data->type];
+	ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
 	put_device(cdev_dev(idxd_cdev));
 }
 
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index c26128529ff49..131138483b877 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -175,6 +175,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq)
 	free_descs(wq);
 	dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
 	sbitmap_queue_free(&wq->sbq);
+	wq->type = IDXD_WQT_NONE;
 }
 EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD");
 
@@ -382,7 +383,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
 	lockdep_assert_held(&wq->wq_lock);
 	wq->state = IDXD_WQ_DISABLED;
 	memset(wq->wqcfg, 0, idxd->wqcfg_size);
-	wq->type = IDXD_WQT_NONE;
 	wq->threshold = 0;
 	wq->priority = 0;
 	wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
@@ -831,8 +831,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd)
 	struct device *dev = &idxd->pdev->dev;
 	struct idxd_evl *evl = idxd->evl;
 
-	gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
-	if (!gencfg.evl_en)
+	if (!evl)
 		return;
 
 	mutex_lock(&evl->lock);
@@ -1125,7 +1124,11 @@ int idxd_device_config(struct idxd_device *idxd)
 {
 	int rc;
 
-	lockdep_assert_held(&idxd->dev_lock);
+	guard(spinlock)(&idxd->dev_lock);
+
+	if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
+		return 0;
+
 	rc = idxd_wqs_setup(idxd);
 	if (rc < 0)
 		return rc;
@@ -1332,6 +1335,11 @@ void idxd_wq_free_irq(struct idxd_wq *wq)
 
 	free_irq(ie->vector, ie);
 	idxd_flush_pending_descs(ie);
+
+	/* The interrupt might have been already released by FLR */
+	if (ie->int_handle == INVALID_INT_HANDLE)
+		return;
+
 	if (idxd->request_int_handles)
 		idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX);
 	idxd_device_clear_perm_entry(idxd, ie);
@@ -1340,6 +1348,23 @@ void idxd_wq_free_irq(struct idxd_wq *wq)
 	ie->pasid = IOMMU_PASID_INVALID;
 }
 
+void idxd_wq_flush_descs(struct idxd_wq *wq)
+{
+	struct idxd_irq_entry *ie = &wq->ie;
+	struct idxd_device *idxd = wq->idxd;
+
+	guard(mutex)(&wq->wq_lock);
+
+	if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL)
+		return;
+
+	idxd_flush_pending_descs(ie);
+	if (idxd->request_int_handles)
+		idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX);
+	idxd_device_clear_perm_entry(idxd, ie);
+	ie->int_handle = INVALID_INT_HANDLE;
+}
+
 int idxd_wq_request_irq(struct idxd_wq *wq)
 {
 	struct idxd_device *idxd = wq->idxd;
@@ -1454,11 +1479,7 @@ int idxd_drv_enable_wq(struct idxd_wq *wq)
 		}
 	}
 
-	rc = 0;
-	spin_lock(&idxd->dev_lock);
-	if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
-		rc = idxd_device_config(idxd);
-	spin_unlock(&idxd->dev_lock);
+	rc = idxd_device_config(idxd);
 	if (rc < 0) {
 		dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
 		goto err;
@@ -1533,7 +1554,6 @@ void idxd_drv_disable_wq(struct idxd_wq *wq)
 	idxd_wq_reset(wq);
 	idxd_wq_free_resources(wq);
 	percpu_ref_exit(&wq->wq_active);
-	wq->type = IDXD_WQT_NONE;
 	wq->client_count = 0;
 }
 EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD");
@@ -1554,10 +1574,7 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
 	}
 
 	/* Device configuration */
-	spin_lock(&idxd->dev_lock);
-	if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
-		rc = idxd_device_config(idxd);
-	spin_unlock(&idxd->dev_lock);
+	rc = idxd_device_config(idxd);
 	if (rc < 0)
 		return -ENXIO;
 
diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
index dbecd699237e3..9937b671f6376 100644
--- a/drivers/dma/idxd/dma.c
+++ b/drivers/dma/idxd/dma.c
@@ -194,6 +194,22 @@ static void idxd_dma_release(struct dma_device *device)
 	kfree(idxd_dma);
 }
 
+static int idxd_dma_terminate_all(struct dma_chan *c)
+{
+	struct idxd_wq *wq = to_idxd_wq(c);
+
+	idxd_wq_flush_descs(wq);
+
+	return 0;
+}
+
+static void idxd_dma_synchronize(struct dma_chan *c)
+{
+	struct idxd_wq *wq = to_idxd_wq(c);
+
+	idxd_wq_drain(wq);
+}
+
 int idxd_register_dma_device(struct idxd_device *idxd)
 {
 	struct idxd_dma_dev *idxd_dma;
@@ -224,6 +240,8 @@ int idxd_register_dma_device(struct idxd_device *idxd)
 	dma->device_issue_pending = idxd_dma_issue_pending;
 	dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
 	dma->device_free_chan_resources = idxd_dma_free_chan_resources;
+	dma->device_terminate_all = idxd_dma_terminate_all;
+	dma->device_synchronize = idxd_dma_synchronize;
 
 	rc = dma_async_device_register(dma);
 	if (rc < 0) {
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index ea8c4daed38d4..ce78b9a7c641c 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -803,6 +803,7 @@ void idxd_wq_quiesce(struct idxd_wq *wq);
 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
 void idxd_wq_free_irq(struct idxd_wq *wq);
 int idxd_wq_request_irq(struct idxd_wq *wq);
+void idxd_wq_flush_descs(struct idxd_wq *wq);
 
 /* submission */
 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index fb80803d5b573..f1cfc7790d950 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -973,7 +973,8 @@ static void idxd_device_config_restore(struct idxd_device *idxd,
 
 	idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit;
 
-	idxd->evl->size = saved_evl->size;
+	if (idxd->evl)
+		idxd->evl->size = saved_evl->size;
 
 	for (i = 0; i < idxd->max_groups; i++) {
 		struct idxd_group *saved_group, *group;
@@ -1104,12 +1105,10 @@ static void idxd_reset_done(struct pci_dev *pdev)
 	idxd_device_config_restore(idxd, idxd->idxd_saved);
 
 	/* Re-configure IDXD device if allowed. */
-	if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
-		rc = idxd_device_config(idxd);
-		if (rc < 0) {
-			dev_err(dev, "HALT: %s config fails\n", idxd_name);
-			goto out;
-		}
+	rc = idxd_device_config(idxd);
+	if (rc < 0) {
+		dev_err(dev, "HALT: %s config fails\n", idxd_name);
+		goto out;
 	}
 
 	/* Bind IDXD device to driver. */
@@ -1147,6 +1146,7 @@ static void idxd_reset_done(struct pci_dev *pdev)
 	}
 out:
 	kfree(idxd->idxd_saved);
+	idxd->idxd_saved = NULL;
 }
 
 static const struct pci_error_handlers idxd_error_handler = {
diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c
index 7782f8c51c32a..6a25e1fd0e62c 100644
--- a/drivers/dma/idxd/irq.c
+++ b/drivers/dma/idxd/irq.c
@@ -397,6 +397,17 @@ static void idxd_device_flr(struct work_struct *work)
 		dev_err(&idxd->pdev->dev, "FLR failed\n");
 }
 
+static void idxd_wqs_flush_descs(struct idxd_device *idxd)
+{
+	int i;
+
+	for (i = 0; i < idxd->max_wqs; i++) {
+		struct idxd_wq *wq = idxd->wqs[i];
+
+		idxd_wq_flush_descs(wq);
+	}
+}
+
 static irqreturn_t idxd_halt(struct idxd_device *idxd)
 {
 	union gensts_reg gensts;
@@ -415,6 +426,11 @@ static irqreturn_t idxd_halt(struct idxd_device *idxd)
 		} else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) {
 			idxd->state = IDXD_DEV_HALTED;
 			idxd_mask_error_interrupts(idxd);
+			/* Flush all pending descriptors, and disable
+			 * interrupts, they will be re-enabled when FLR
+			 * concludes.
+			 */
+			idxd_wqs_flush_descs(idxd);
 			dev_dbg(&idxd->pdev->dev,
 				"idxd halted, doing FLR. After FLR, configs are restored\n");
 			INIT_WORK(&idxd->work, idxd_device_flr);
diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c
index 6db1c5fcedc58..03217041b8b3e 100644
--- a/drivers/dma/idxd/submit.c
+++ b/drivers/dma/idxd/submit.c
@@ -138,7 +138,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
 	 */
 	list_for_each_entry_safe(d, t, &flist, list) {
 		list_del_init(&d->list);
-		idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true,
+		idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true,
 				      NULL, NULL);
 	}
 }
diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
index cc2c83d7f7104..6d251095c350f 100644
--- a/drivers/dma/idxd/sysfs.c
+++ b/drivers/dma/idxd/sysfs.c
@@ -1836,6 +1836,7 @@ static void idxd_conf_device_release(struct device *dev)
 {
 	struct idxd_device *idxd = confdev_to_idxd(dev);
 
+	destroy_workqueue(idxd->wq);
 	kfree(idxd->groups);
 	bitmap_free(idxd->wq_enable_map);
 	kfree(idxd->wqs);
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 4c8196d780012..3d527883776b4 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -2265,34 +2265,24 @@ static int sdma_probe(struct platform_device *pdev)
 	if (IS_ERR(sdma->regs))
 		return PTR_ERR(sdma->regs);
 
-	sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
+	sdma->clk_ipg = devm_clk_get_prepared(&pdev->dev, "ipg");
 	if (IS_ERR(sdma->clk_ipg))
 		return PTR_ERR(sdma->clk_ipg);
 
-	sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
+	sdma->clk_ahb = devm_clk_get_prepared(&pdev->dev, "ahb");
 	if (IS_ERR(sdma->clk_ahb))
 		return PTR_ERR(sdma->clk_ahb);
 
-	ret = clk_prepare(sdma->clk_ipg);
-	if (ret)
-		return ret;
-
-	ret = clk_prepare(sdma->clk_ahb);
-	if (ret)
-		goto err_clk;
-
 	ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0,
 				dev_name(&pdev->dev), sdma);
 	if (ret)
-		goto err_irq;
+		return ret;
 
 	sdma->irq = irq;
 
 	sdma->script_addrs = kzalloc_obj(*sdma->script_addrs);
-	if (!sdma->script_addrs) {
-		ret = -ENOMEM;
-		goto err_irq;
-	}
+	if (!sdma->script_addrs)
+		return -ENOMEM;
 
 	/* initially no scripts available */
 	saddr_arr = (s32 *)sdma->script_addrs;
@@ -2333,11 +2323,11 @@ static int sdma_probe(struct platform_device *pdev)
 
 	ret = sdma_init(sdma);
 	if (ret)
-		goto err_init;
+		return ret;
 
 	ret = sdma_event_remap(sdma);
 	if (ret)
-		goto err_init;
+		return ret;
 
 	if (sdma->drvdata->script_addrs)
 		sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
@@ -2363,18 +2353,16 @@ static int sdma_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sdma);
 
-	ret = dma_async_device_register(&sdma->dma_device);
-	if (ret) {
-		dev_err(&pdev->dev, "unable to register\n");
-		goto err_init;
-	}
+	ret = dmaenginem_async_device_register(&sdma->dma_device);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "unable to register\n");
 
 	if (np) {
-		ret = of_dma_controller_register(np, sdma_xlate, sdma);
-		if (ret) {
-			dev_err(&pdev->dev, "failed to register controller\n");
-			goto err_register;
-		}
+		ret = devm_of_dma_controller_register(&pdev->dev, np,
+						      sdma_xlate, sdma);
+		if (ret)
+			return dev_err_probe(&pdev->dev, ret,
+					     "failed to register controller\n");
 
 		spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
 		ret = of_address_to_resource(spba_bus, 0, &spba_res);
@@ -2401,16 +2389,6 @@ static int sdma_probe(struct platform_device *pdev)
 	}
 
 	return 0;
-
-err_register:
-	dma_async_device_unregister(&sdma->dma_device);
-err_init:
-	kfree(sdma->script_addrs);
-err_irq:
-	clk_unprepare(sdma->clk_ahb);
-err_clk:
-	clk_unprepare(sdma->clk_ipg);
-	return ret;
 }
 
 static void sdma_remove(struct platform_device *pdev)
@@ -2419,10 +2397,6 @@ static void sdma_remove(struct platform_device *pdev)
 	int i;
 
 	devm_free_irq(&pdev->dev, sdma->irq, sdma);
-	dma_async_device_unregister(&sdma->dma_device);
-	kfree(sdma->script_addrs);
-	clk_unprepare(sdma->clk_ahb);
-	clk_unprepare(sdma->clk_ipg);
 	/* Kill the tasklet */
 	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
 		struct sdma_channel *sdmac = &sdma->channel[i];
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 12a4a4860a74e..e8a880f338c65 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -140,12 +140,6 @@ struct ioatdma_chan {
 	int prev_intr_coalesce;
 };
 
-struct ioat_sysfs_entry {
-	struct attribute attr;
-	ssize_t (*show)(struct dma_chan *, char *);
-	ssize_t (*store)(struct dma_chan *, const char *, size_t);
-};
-
 /**
  * struct ioat_sed_ent - wrapper around super extended hardware descriptor
  * @hw: hardware SED
@@ -195,11 +189,8 @@ struct ioat_ring_ent {
 	struct ioat_sed_ent *sed;
 };
 
-extern const struct sysfs_ops ioat_sysfs_ops;
-extern struct ioat_sysfs_entry ioat_version_attr;
-extern struct ioat_sysfs_entry ioat_cap_attr;
 extern int ioat_pending_level;
-extern struct kobj_type ioat_ktype;
+extern const struct kobj_type ioat_ktype;
 extern struct kmem_cache *ioat_cache;
 extern struct kmem_cache *ioat_sed_cache;
 
@@ -402,7 +393,7 @@ void ioat_issue_pending(struct dma_chan *chan);
 /* IOAT Init functions */
 bool is_bwd_ioat(struct pci_dev *pdev);
 struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
+void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type);
 void ioat_kobject_del(struct ioatdma_device *ioat_dma);
 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
 void ioat_stop(struct ioatdma_chan *ioat_chan);
diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c
index 168adf28c5b13..e796ddb5383f8 100644
--- a/drivers/dma/ioat/sysfs.c
+++ b/drivers/dma/ioat/sysfs.c
@@ -14,6 +14,12 @@
 
 #include "../dmaengine.h"
 
+struct ioat_sysfs_entry {
+	struct attribute attr;
+	ssize_t (*show)(struct dma_chan *, char *);
+	ssize_t (*store)(struct dma_chan *, const char *, size_t);
+};
+
 static ssize_t cap_show(struct dma_chan *c, char *page)
 {
 	struct dma_device *dma = c->device;
@@ -26,7 +32,7 @@ static ssize_t cap_show(struct dma_chan *c, char *page)
 		       dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
 
 }
-struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
+static const struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
 
 static ssize_t version_show(struct dma_chan *c, char *page)
 {
@@ -36,15 +42,15 @@ static ssize_t version_show(struct dma_chan *c, char *page)
 	return sprintf(page, "%d.%d\n",
 		       ioat_dma->version >> 4, ioat_dma->version & 0xf);
 }
-struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
+static const struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
 
 static ssize_t
 ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 {
-	struct ioat_sysfs_entry *entry;
+	const struct ioat_sysfs_entry *entry;
 	struct ioatdma_chan *ioat_chan;
 
-	entry = container_of(attr, struct ioat_sysfs_entry, attr);
+	entry = container_of_const(attr, struct ioat_sysfs_entry, attr);
 	ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
 
 	if (!entry->show)
@@ -56,10 +62,10 @@ static ssize_t
 ioat_attr_store(struct kobject *kobj, struct attribute *attr,
 const char *page, size_t count)
 {
-	struct ioat_sysfs_entry *entry;
+	const struct ioat_sysfs_entry *entry;
 	struct ioatdma_chan *ioat_chan;
 
-	entry = container_of(attr, struct ioat_sysfs_entry, attr);
+	entry = container_of_const(attr, struct ioat_sysfs_entry, attr);
 	ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
 
 	if (!entry->store)
@@ -67,12 +73,12 @@ const char *page, size_t count)
 	return entry->store(&ioat_chan->dma_chan, page, count);
 }
 
-const struct sysfs_ops ioat_sysfs_ops = {
+static const struct sysfs_ops ioat_sysfs_ops = {
 	.show	= ioat_attr_show,
 	.store  = ioat_attr_store,
 };
 
-void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type)
+void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type)
 {
 	struct dma_device *dma = &ioat_dma->dma_dev;
 	struct dma_chan *c;
@@ -114,7 +120,7 @@ static ssize_t ring_size_show(struct dma_chan *c, char *page)
 
 	return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);
 }
-static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+static const struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
 
 static ssize_t ring_active_show(struct dma_chan *c, char *page)
 {
@@ -123,7 +129,7 @@ static ssize_t ring_active_show(struct dma_chan *c, char *page)
 	/* ...taken outside the lock, no need to be precise */
 	return sprintf(page, "%d\n", ioat_ring_active(ioat_chan));
 }
-static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+static const struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
 
 static ssize_t intr_coalesce_show(struct dma_chan *c, char *page)
 {
@@ -148,9 +154,9 @@ size_t count)
 	return count;
 }
 
-static struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce);
+static const struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce);
 
-static struct attribute *ioat_attrs[] = {
+static const struct attribute *const ioat_attrs[] = {
 	&ring_size_attr.attr,
 	&ring_active_attr.attr,
 	&ioat_cap_attr.attr,
@@ -160,7 +166,7 @@ static struct attribute *ioat_attrs[] = {
 };
 ATTRIBUTE_GROUPS(ioat);
 
-struct kobj_type ioat_ktype = {
+const struct kobj_type ioat_ktype = {
 	.sysfs_ops = &ioat_sysfs_ops,
 	.default_groups = ioat_groups,
 };
diff --git a/drivers/dma/loongson/Kconfig b/drivers/dma/loongson/Kconfig
new file mode 100644
index 0000000000000..c4e62dce5d4f0
--- /dev/null
+++ b/drivers/dma/loongson/Kconfig
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Loongson DMA controllers drivers
+#
+if MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST
+
+config LOONGSON1_APB_DMA
+	tristate "Loongson1 APB DMA support"
+	depends on MACH_LOONGSON32 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  This selects support for the APB DMA controller in Loongson1 SoCs,
+	  which is required by Loongson1 NAND and audio support.
+
+config LOONGSON2_APB_DMA
+	tristate "Loongson2 APB DMA support"
+	depends on MACH_LOONGSON64 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  Support for the Loongson2 APB DMA controller driver. The
+	  DMA controller is having single DMA channel which can be
+	  configured for different peripherals like audio, nand, sdio
+	  etc which is in APB bus.
+
+	  This DMA controller transfers data from memory to peripheral fifo.
+	  It does not support memory to memory data transfer.
+
+config LOONGSON2_APB_CMC_DMA
+	tristate "Loongson2 Chain Multi-Channel DMA support"
+	depends on MACH_LOONGSON64 || COMPILE_TEST
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  Support for the Loongson Chain Multi-Channel DMA controller driver.
+	  It is discovered on the Loongson-2K chip (Loongson-2K0300/Loongson-2K3000),
+	  which has 4/8 channels internally, enabling bidirectional data transfer
+	  between devices and memory.
+
+endif
diff --git a/drivers/dma/loongson/Makefile b/drivers/dma/loongson/Makefile
new file mode 100644
index 0000000000000..48c19781e7298
--- /dev/null
+++ b/drivers/dma/loongson/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o
+obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o
+obj-$(CONFIG_LOONGSON2_APB_CMC_DMA) += loongson2-apb-cmc-dma.o
diff --git a/drivers/dma/loongson/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c
new file mode 100644
index 0000000000000..89786cbd20ab5
--- /dev/null
+++ b/drivers/dma/loongson/loongson1-apb-dma.c
@@ -0,0 +1,660 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Loongson-1 APB DMA Controller
+ *
+ * Copyright (C) 2015-2024 Keguang Zhang <keguang.zhang@gmail.com>
+ */
+
+#include <linux/dmapool.h>
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+/* Loongson-1 DMA Control Register */
+#define LS1X_DMA_CTRL		0x0
+
+/* DMA Control Register Bits */
+#define LS1X_DMA_STOP		BIT(4)
+#define LS1X_DMA_START		BIT(3)
+#define LS1X_DMA_ASK_VALID	BIT(2)
+
+/* DMA Next Field Bits */
+#define LS1X_DMA_NEXT_VALID	BIT(0)
+
+/* DMA Command Field Bits */
+#define LS1X_DMA_RAM2DEV	BIT(12)
+#define LS1X_DMA_INT		BIT(1)
+#define LS1X_DMA_INT_MASK	BIT(0)
+
+#define LS1X_DMA_LLI_ALIGNMENT	64
+#define LS1X_DMA_LLI_ADDR_MASK	GENMASK(31, __ffs(LS1X_DMA_LLI_ALIGNMENT))
+#define LS1X_DMA_MAX_CHANNELS	3
+
+enum ls1x_dmadesc_offsets {
+	LS1X_DMADESC_NEXT = 0,
+	LS1X_DMADESC_SADDR,
+	LS1X_DMADESC_DADDR,
+	LS1X_DMADESC_LENGTH,
+	LS1X_DMADESC_STRIDE,
+	LS1X_DMADESC_CYCLES,
+	LS1X_DMADESC_CMD,
+	LS1X_DMADESC_SIZE
+};
+
+struct ls1x_dma_lli {
+	unsigned int hw[LS1X_DMADESC_SIZE];
+	dma_addr_t phys;
+	struct list_head node;
+} __aligned(LS1X_DMA_LLI_ALIGNMENT);
+
+struct ls1x_dma_desc {
+	struct virt_dma_desc vd;
+	struct list_head lli_list;
+};
+
+struct ls1x_dma_chan {
+	struct virt_dma_chan vc;
+	struct dma_pool *lli_pool;
+	phys_addr_t src_addr;
+	phys_addr_t dst_addr;
+	enum dma_slave_buswidth src_addr_width;
+	enum dma_slave_buswidth dst_addr_width;
+	unsigned int bus_width;
+	void __iomem *reg_base;
+	int irq;
+	bool is_cyclic;
+	struct ls1x_dma_lli *curr_lli;
+};
+
+struct ls1x_dma {
+	struct dma_device ddev;
+	unsigned int nr_chans;
+	struct ls1x_dma_chan chan[];
+};
+
+static irqreturn_t ls1x_dma_irq_handler(int irq, void *data);
+
+#define to_ls1x_dma_chan(dchan)		\
+	container_of(dchan, struct ls1x_dma_chan, vc.chan)
+
+#define to_ls1x_dma_desc(d)		\
+	container_of(d, struct ls1x_dma_desc, vd)
+
+static inline struct device *chan2dev(struct dma_chan *chan)
+{
+	return &chan->dev->device;
+}
+
+static inline int ls1x_dma_query(struct ls1x_dma_chan *chan,
+				 dma_addr_t *lli_phys)
+{
+	struct dma_chan *dchan = &chan->vc.chan;
+	int val, ret;
+
+	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
+	val |= LS1X_DMA_ASK_VALID;
+	val |= dchan->chan_id;
+	writel(val, chan->reg_base + LS1X_DMA_CTRL);
+	ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val,
+					!(val & LS1X_DMA_ASK_VALID), 0, 3000);
+	if (ret)
+		dev_err(chan2dev(dchan), "failed to query DMA\n");
+
+	return ret;
+}
+
+static inline int ls1x_dma_start(struct ls1x_dma_chan *chan,
+				 dma_addr_t *lli_phys)
+{
+	struct dma_chan *dchan = &chan->vc.chan;
+	struct device *dev = chan2dev(dchan);
+	int val, ret;
+
+	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
+	val |= LS1X_DMA_START;
+	val |= dchan->chan_id;
+	writel(val, chan->reg_base + LS1X_DMA_CTRL);
+	ret = readl_poll_timeout(chan->reg_base + LS1X_DMA_CTRL, val,
+				 !(val & LS1X_DMA_START), 0, 1000);
+	if (!ret)
+		dev_dbg(dev, "start DMA with lli_phys=%pad\n", lli_phys);
+	else
+		dev_err(dev, "failed to start DMA\n");
+
+	return ret;
+}
+
+static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan)
+{
+	int val = readl(chan->reg_base + LS1X_DMA_CTRL);
+
+	writel(val | LS1X_DMA_STOP, chan->reg_base + LS1X_DMA_CTRL);
+}
+
+static void ls1x_dma_free_chan_resources(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct device *dev = chan2dev(dchan);
+
+	dma_free_coherent(dev, sizeof(struct ls1x_dma_lli),
+			  chan->curr_lli, chan->curr_lli->phys);
+	dma_pool_destroy(chan->lli_pool);
+	chan->lli_pool = NULL;
+	devm_free_irq(dev, chan->irq, chan);
+	vchan_free_chan_resources(&chan->vc);
+}
+
+static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct device *dev = chan2dev(dchan);
+	dma_addr_t phys;
+	int ret;
+
+	ret = devm_request_irq(dev, chan->irq, ls1x_dma_irq_handler,
+			       IRQF_SHARED, dma_chan_name(dchan), chan);
+	if (ret) {
+		dev_err(dev, "failed to request IRQ %d\n", chan->irq);
+		return ret;
+	}
+
+	chan->lli_pool = dma_pool_create(dma_chan_name(dchan), dev,
+					 sizeof(struct ls1x_dma_lli),
+					 __alignof__(struct ls1x_dma_lli), 0);
+	if (!chan->lli_pool)
+		return -ENOMEM;
+
+	/* allocate memory for querying the current lli */
+	dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
+	chan->curr_lli = dma_alloc_coherent(dev, sizeof(struct ls1x_dma_lli),
+					    &phys, GFP_KERNEL);
+	if (!chan->curr_lli) {
+		dma_pool_destroy(chan->lli_pool);
+		return -ENOMEM;
+	}
+	chan->curr_lli->phys = phys;
+
+	return 0;
+}
+
+static void ls1x_dma_free_desc(struct virt_dma_desc *vd)
+{
+	struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(vd->tx.chan);
+	struct ls1x_dma_lli *lli, *_lli;
+
+	list_for_each_entry_safe(lli, _lli, &desc->lli_list, node) {
+		list_del(&lli->node);
+		dma_pool_free(chan->lli_pool, lli, lli->phys);
+	}
+
+	kfree(desc);
+}
+
+static struct ls1x_dma_desc *ls1x_dma_alloc_desc(void)
+{
+	struct ls1x_dma_desc *desc;
+
+	desc = kzalloc_obj(*desc, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	INIT_LIST_HEAD(&desc->lli_list);
+
+	return desc;
+}
+
+static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc,
+			     struct scatterlist *sgl, unsigned int sg_len,
+			     enum dma_transfer_direction dir, bool is_cyclic)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL;
+	struct device *dev = chan2dev(dchan);
+	struct list_head *pos = NULL;
+	struct scatterlist *sg;
+	unsigned int dev_addr, cmd, i;
+
+	switch (dir) {
+	case DMA_MEM_TO_DEV:
+		dev_addr = chan->dst_addr;
+		chan->bus_width = chan->dst_addr_width;
+		cmd = LS1X_DMA_RAM2DEV | LS1X_DMA_INT;
+		break;
+	case DMA_DEV_TO_MEM:
+		dev_addr = chan->src_addr;
+		chan->bus_width = chan->src_addr_width;
+		cmd = LS1X_DMA_INT;
+		break;
+	default:
+		dev_err(dev, "unsupported DMA direction: %s\n",
+			dmaengine_get_direction_text(dir));
+		return -EINVAL;
+	}
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		dma_addr_t buf_addr = sg_dma_address(sg);
+		size_t buf_len = sg_dma_len(sg);
+		dma_addr_t phys;
+
+		if (!is_dma_copy_aligned(dchan->device, buf_addr, 0, buf_len)) {
+			dev_err(dev, "buffer is not aligned\n");
+			return -EINVAL;
+		}
+
+		/* allocate HW descriptors */
+		lli = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, &phys);
+		if (!lli) {
+			dev_err(dev, "failed to alloc lli %u\n", i);
+			return -ENOMEM;
+		}
+
+		/* setup HW descriptors */
+		lli->phys = phys;
+		lli->hw[LS1X_DMADESC_SADDR] = buf_addr;
+		lli->hw[LS1X_DMADESC_DADDR] = dev_addr;
+		lli->hw[LS1X_DMADESC_LENGTH] = buf_len / chan->bus_width;
+		lli->hw[LS1X_DMADESC_STRIDE] = 0;
+		lli->hw[LS1X_DMADESC_CYCLES] = 1;
+		lli->hw[LS1X_DMADESC_CMD] = cmd;
+
+		if (prev)
+			prev->hw[LS1X_DMADESC_NEXT] =
+			    lli->phys | LS1X_DMA_NEXT_VALID;
+		prev = lli;
+
+		if (!first)
+			first = lli;
+
+		list_add_tail(&lli->node, &desc->lli_list);
+	}
+
+	if (is_cyclic) {
+		lli->hw[LS1X_DMADESC_NEXT] = first->phys | LS1X_DMA_NEXT_VALID;
+		chan->is_cyclic = is_cyclic;
+	}
+
+	list_for_each(pos, &desc->lli_list) {
+		lli = list_entry(pos, struct ls1x_dma_lli, node);
+		print_hex_dump_debug("LLI: ", DUMP_PREFIX_OFFSET, 16, 4,
+				     lli, sizeof(*lli), false);
+	}
+
+	return 0;
+}
+
+static struct dma_async_tx_descriptor *
+ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+		       unsigned int sg_len, enum dma_transfer_direction dir,
+		       unsigned long flags, void *context)
+{
+	struct ls1x_dma_desc *desc;
+
+	dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n",
+		sg_len, flags, dmaengine_get_direction_text(dir));
+
+	desc = ls1x_dma_alloc_desc();
+	if (!desc)
+		return NULL;
+
+	if (ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, false)) {
+		ls1x_dma_free_desc(&desc->vd);
+		return NULL;
+	}
+
+	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *
+ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
+			 size_t buf_len, size_t period_len,
+			 enum dma_transfer_direction dir, unsigned long flags)
+{
+	struct ls1x_dma_desc *desc;
+	struct scatterlist *sgl;
+	unsigned int sg_len;
+	unsigned int i;
+	int ret;
+
+	dev_dbg(chan2dev(dchan),
+		"buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n",
+		buf_len, period_len, flags, dmaengine_get_direction_text(dir));
+
+	desc = ls1x_dma_alloc_desc();
+	if (!desc)
+		return NULL;
+
+	/* allocate the scatterlist */
+	sg_len = buf_len / period_len;
+	sgl = kmalloc_objs(*sgl, sg_len, GFP_NOWAIT);
+	if (!sgl)
+		return NULL;
+
+	sg_init_table(sgl, sg_len);
+	for (i = 0; i < sg_len; ++i) {
+		sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(buf_addr)),
+			    period_len, offset_in_page(buf_addr));
+		sg_dma_address(&sgl[i]) = buf_addr;
+		sg_dma_len(&sgl[i]) = period_len;
+		buf_addr += period_len;
+	}
+
+	ret = ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, true);
+	kfree(sgl);
+	if (ret) {
+		ls1x_dma_free_desc(&desc->vd);
+		return NULL;
+	}
+
+	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
+}
+
+static int ls1x_dma_slave_config(struct dma_chan *dchan,
+				 struct dma_slave_config *config)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	chan->src_addr = config->src_addr;
+	chan->src_addr_width = config->src_addr_width;
+	chan->dst_addr = config->dst_addr;
+	chan->dst_addr_width = config->dst_addr_width;
+
+	return 0;
+}
+
+static int ls1x_dma_pause(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	int ret;
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+	/* save the current lli */
+	ret = ls1x_dma_query(chan, &chan->curr_lli->phys);
+	if (!ret)
+		ls1x_dma_stop(chan);
+
+	return ret;
+}
+
+static int ls1x_dma_resume(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+
+	return ls1x_dma_start(chan, &chan->curr_lli->phys);
+}
+
+static int ls1x_dma_terminate_all(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct virt_dma_desc *vd;
+	LIST_HEAD(head);
+
+	ls1x_dma_stop(chan);
+
+	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
+		vd = vchan_next_desc(&chan->vc);
+		if (vd)
+			vchan_terminate_vdesc(vd);
+
+		vchan_get_all_descriptors(&chan->vc, &head);
+	}
+
+	vchan_dma_desc_free_list(&chan->vc, &head);
+
+	return 0;
+}
+
+static void ls1x_dma_synchronize(struct dma_chan *dchan)
+{
+	vchan_synchronize(to_virt_chan(dchan));
+}
+
+static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan,
+					  dma_cookie_t cookie,
+					  struct dma_tx_state *state)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+	struct virt_dma_desc *vd;
+	enum dma_status status;
+	size_t bytes = 0;
+
+	status = dma_cookie_status(dchan, cookie, state);
+	if (status == DMA_COMPLETE)
+		return status;
+
+	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
+		vd = vchan_find_desc(&chan->vc, cookie);
+		if (vd) {
+			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+			struct ls1x_dma_lli *lli;
+			dma_addr_t next_phys;
+
+			/* get the current lli */
+			if (ls1x_dma_query(chan, &chan->curr_lli->phys))
+				return status;
+
+			/* locate the current lli */
+			next_phys = chan->curr_lli->hw[LS1X_DMADESC_NEXT];
+			list_for_each_entry(lli, &desc->lli_list, node)
+				if (lli->hw[LS1X_DMADESC_NEXT] == next_phys)
+					break;
+
+			dev_dbg(chan2dev(dchan), "current lli_phys=%pad",
+				&lli->phys);
+
+			/* count the residues */
+			list_for_each_entry_from(lli, &desc->lli_list, node)
+				bytes += lli->hw[LS1X_DMADESC_LENGTH] *
+					 chan->bus_width;
+		}
+	}
+
+	dma_set_residue(state, bytes);
+
+	return status;
+}
+
+static void ls1x_dma_issue_pending(struct dma_chan *dchan)
+{
+	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
+
+	guard(spinlock_irqsave)(&chan->vc.lock);
+
+	if (vchan_issue_pending(&chan->vc)) {
+		struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
+
+		if (vd) {
+			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
+			struct ls1x_dma_lli *lli;
+
+			lli = list_first_entry(&desc->lli_list,
+					       struct ls1x_dma_lli, node);
+			ls1x_dma_start(chan, &lli->phys);
+		}
+	}
+}
+
+static irqreturn_t ls1x_dma_irq_handler(int irq, void *data)
+{
+	struct ls1x_dma_chan *chan = data;
+	struct dma_chan *dchan = &chan->vc.chan;
+	struct device *dev = chan2dev(dchan);
+	struct virt_dma_desc *vd;
+
+	scoped_guard(spinlock, &chan->vc.lock) {
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd) {
+			dev_warn(dev,
+				 "IRQ %d with no active desc on channel %d\n",
+				 irq, dchan->chan_id);
+			return IRQ_NONE;
+		}
+
+		if (chan->is_cyclic) {
+			vchan_cyclic_callback(vd);
+		} else {
+			list_del(&vd->node);
+			vchan_cookie_complete(vd);
+		}
+	}
+
+	dev_dbg(dev, "DMA IRQ %d on channel %d\n", irq, dchan->chan_id);
+
+	return IRQ_HANDLED;
+}
+
+static int ls1x_dma_chan_probe(struct platform_device *pdev,
+			       struct ls1x_dma *dma)
+{
+	void __iomem *reg_base;
+	int id;
+
+	reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(reg_base))
+		return PTR_ERR(reg_base);
+
+	for (id = 0; id < dma->nr_chans; id++) {
+		struct ls1x_dma_chan *chan = &dma->chan[id];
+		char pdev_irqname[16];
+
+		snprintf(pdev_irqname, sizeof(pdev_irqname), "ch%d", id);
+		chan->irq = platform_get_irq_byname(pdev, pdev_irqname);
+		if (chan->irq < 0)
+			return dev_err_probe(&pdev->dev, chan->irq,
+					     "failed to get IRQ for ch%d\n",
+					     id);
+
+		chan->reg_base = reg_base;
+		chan->vc.desc_free = ls1x_dma_free_desc;
+		vchan_init(&chan->vc, &dma->ddev);
+	}
+
+	return 0;
+}
+
+static void ls1x_dma_chan_remove(struct ls1x_dma *dma)
+{
+	int id;
+
+	for (id = 0; id < dma->nr_chans; id++) {
+		struct ls1x_dma_chan *chan = &dma->chan[id];
+
+		if (chan->vc.chan.device == &dma->ddev) {
+			list_del(&chan->vc.chan.device_node);
+			tasklet_kill(&chan->vc.task);
+		}
+	}
+}
+
+static int ls1x_dma_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dma_device *ddev;
+	struct ls1x_dma *dma;
+	int ret;
+
+	ret = platform_irq_count(pdev);
+	if (ret <= 0 || ret > LS1X_DMA_MAX_CHANNELS)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid number of IRQ channels: %d\n",
+				     ret);
+
+	dma = devm_kzalloc(dev, struct_size(dma, chan, ret), GFP_KERNEL);
+	if (!dma)
+		return -ENOMEM;
+	dma->nr_chans = ret;
+
+	/* initialize DMA device */
+	ddev = &dma->ddev;
+	ddev->dev = dev;
+	ddev->copy_align = DMAENGINE_ALIGN_4_BYTES;
+	ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	ddev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+	ddev->device_alloc_chan_resources = ls1x_dma_alloc_chan_resources;
+	ddev->device_free_chan_resources = ls1x_dma_free_chan_resources;
+	ddev->device_prep_slave_sg = ls1x_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = ls1x_dma_prep_dma_cyclic;
+	ddev->device_config = ls1x_dma_slave_config;
+	ddev->device_pause = ls1x_dma_pause;
+	ddev->device_resume = ls1x_dma_resume;
+	ddev->device_terminate_all = ls1x_dma_terminate_all;
+	ddev->device_synchronize = ls1x_dma_synchronize;
+	ddev->device_tx_status = ls1x_dma_tx_status;
+	ddev->device_issue_pending = ls1x_dma_issue_pending;
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	INIT_LIST_HEAD(&ddev->channels);
+
+	/* initialize DMA channels */
+	ret = ls1x_dma_chan_probe(pdev, dma);
+	if (ret)
+		goto err;
+
+	ret = dmaenginem_async_device_register(ddev);
+	if (ret) {
+		dev_err(dev, "failed to register DMA device\n");
+		goto err;
+	}
+
+	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id,
+					 ddev);
+	if (ret) {
+		dev_err(dev, "failed to register DMA controller\n");
+		goto err;
+	}
+
+	platform_set_drvdata(pdev, dma);
+	dev_info(dev, "Loongson1 DMA driver registered\n");
+
+	return 0;
+
+err:
+	ls1x_dma_chan_remove(dma);
+
+	return ret;
+}
+
+static void ls1x_dma_remove(struct platform_device *pdev)
+{
+	struct ls1x_dma *dma = platform_get_drvdata(pdev);
+
+	of_dma_controller_free(pdev->dev.of_node);
+	ls1x_dma_chan_remove(dma);
+}
+
+static const struct of_device_id ls1x_dma_match[] = {
+	{ .compatible = "loongson,ls1b-apbdma" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ls1x_dma_match);
+
+static struct platform_driver ls1x_dma_driver = {
+	.probe = ls1x_dma_probe,
+	.remove = ls1x_dma_remove,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = ls1x_dma_match,
+	},
+};
+
+module_platform_driver(ls1x_dma_driver);
+
+MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>");
+MODULE_DESCRIPTION("Loongson-1 APB DMA Controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
new file mode 100644
index 0000000000000..1c9a542edc859
--- /dev/null
+++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
@@ -0,0 +1,730 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Loongson-2 Chain Multi-Channel DMA Controller driver
+ *
+ * Copyright (C) 2024-2026 Loongson Technology Corporation Limited
+ */
+
+#include <linux/acpi.h>
+#include <linux/acpi_dma.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+#define LOONGSON2_CMCDMA_ISR		0x0	/* DMA Interrupt Status Register */
+#define LOONGSON2_CMCDMA_IFCR		0x4	/* DMA Interrupt Flag Clear Register */
+#define LOONGSON2_CMCDMA_CCR		0x8	/* DMA Channel Configuration Register */
+#define LOONGSON2_CMCDMA_CNDTR		0xc	/* DMA Channel Transmit Count Register */
+#define LOONGSON2_CMCDMA_CPAR		0x10	/* DMA Channel Peripheral Address Register */
+#define LOONGSON2_CMCDMA_CMAR		0x14	/* DMA Channel Memory Address Register */
+
+/* Bitfields of DMA interrupt status register */
+#define LOONGSON2_CMCDMA_TCI		BIT(1) /* Transfer Complete Interrupt */
+#define LOONGSON2_CMCDMA_HTI		BIT(2) /* Half Transfer Interrupt */
+#define LOONGSON2_CMCDMA_TEI		BIT(3) /* Transfer Error Interrupt */
+
+#define LOONGSON2_CMCDMA_MASKI		\
+	(LOONGSON2_CMCDMA_TCI | LOONGSON2_CMCDMA_HTI | LOONGSON2_CMCDMA_TEI)
+
+/* Bitfields of DMA channel x Configuration Register */
+#define LOONGSON2_CMCDMA_CCR_EN		BIT(0) /* Stream Enable */
+#define LOONGSON2_CMCDMA_CCR_TCIE	BIT(1) /* Transfer Complete Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_HTIE	BIT(2) /* Half Transfer Complete Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_TEIE	BIT(3) /* Transfer Error Interrupt Enable */
+#define LOONGSON2_CMCDMA_CCR_DIR	BIT(4) /* Data Transfer Direction */
+#define LOONGSON2_CMCDMA_CCR_CIRC	BIT(5) /* Circular mode */
+#define LOONGSON2_CMCDMA_CCR_PINC	BIT(6) /* Peripheral increment mode */
+#define LOONGSON2_CMCDMA_CCR_MINC	BIT(7) /* Memory increment mode */
+#define LOONGSON2_CMCDMA_CCR_PSIZE_MASK	GENMASK(9, 8)
+#define LOONGSON2_CMCDMA_CCR_MSIZE_MASK	GENMASK(11, 10)
+#define LOONGSON2_CMCDMA_CCR_PL_MASK	GENMASK(13, 12)
+#define LOONGSON2_CMCDMA_CCR_M2M	BIT(14)
+
+#define LOONGSON2_CMCDMA_CCR_CFG_MASK	\
+	(LOONGSON2_CMCDMA_CCR_PINC | LOONGSON2_CMCDMA_CCR_MINC | LOONGSON2_CMCDMA_CCR_PL_MASK)
+
+#define LOONGSON2_CMCDMA_CCR_IRQ_MASK	\
+	(LOONGSON2_CMCDMA_CCR_TCIE | LOONGSON2_CMCDMA_CCR_HTIE | LOONGSON2_CMCDMA_CCR_TEIE)
+
+#define LOONGSON2_CMCDMA_STREAM_MASK	\
+	(LOONGSON2_CMCDMA_CCR_CFG_MASK | LOONGSON2_CMCDMA_CCR_IRQ_MASK)
+
+#define LOONGSON2_CMCDMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+					 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+					 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+#define LOONSON2_CMCDMA_MAX_DATA_ITEMS	SZ_64K
+
+struct loongson2_cmc_dma_chan_reg {
+	u32 ccr;
+	u32 cndtr;
+	u32 cpar;
+	u32 cmar;
+};
+
+struct loongson2_cmc_dma_sg_req {
+	u32 len;
+	struct loongson2_cmc_dma_chan_reg chan_reg;
+};
+
+struct loongson2_cmc_dma_desc {
+	struct virt_dma_desc vdesc;
+	bool cyclic;
+	u32 num_sgs;
+	struct loongson2_cmc_dma_sg_req sg_req[] __counted_by(num_sgs);
+};
+
+struct loongson2_cmc_dma_chan {
+	struct virt_dma_chan vchan;
+	struct dma_slave_config	dma_sconfig;
+	struct loongson2_cmc_dma_desc *desc;
+	u32 id;
+	u32 irq;
+	u32 next_sg;
+	struct loongson2_cmc_dma_chan_reg chan_reg;
+};
+
+struct loongson2_cmc_dma_dev {
+	struct dma_device ddev;
+	struct clk *dma_clk;
+	void __iomem *base;
+	u32 nr_channels;
+	u32 chan_reg_offset;
+	struct loongson2_cmc_dma_chan chan[] __counted_by(nr_channels);
+};
+
+struct loongson2_cmc_dma_config {
+	u32 max_channels;
+	u32 chan_reg_offset;
+};
+
+static const struct loongson2_cmc_dma_config ls2k0300_cmc_dma_config = {
+	.max_channels = 8,
+	.chan_reg_offset = 0x14,
+};
+
+static const struct loongson2_cmc_dma_config ls2k3000_cmc_dma_config = {
+	.max_channels = 4,
+	.chan_reg_offset = 0x18,
+};
+
+static struct loongson2_cmc_dma_dev *lmdma_get_dev(struct loongson2_cmc_dma_chan *lchan)
+{
+	return container_of(lchan->vchan.chan.device, struct loongson2_cmc_dma_dev, ddev);
+}
+
+static struct loongson2_cmc_dma_chan *to_lmdma_chan(struct dma_chan *chan)
+{
+	return container_of(chan, struct loongson2_cmc_dma_chan, vchan.chan);
+}
+
+static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc)
+{
+	return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc);
+}
+
+static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan)
+{
+	return &lchan->vchan.chan.dev->device;
+}
+
+static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id)
+{
+	return readl(lddev->base + (reg + lddev->chan_reg_offset * id));
+}
+
+static void loongson2_cmc_dma_write(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id, u32 val)
+{
+	writel(val, lddev->base + (reg + lddev->chan_reg_offset * id));
+}
+
+static int loongson2_cmc_dma_get_width(enum dma_slave_buswidth width)
+{
+	switch (width) {
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		return ffs(width) - 1;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int loongson2_cmc_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *config)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	memcpy(&lchan->dma_sconfig, config, sizeof(*config));
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_irq_clear(struct loongson2_cmc_dma_chan *lchan, u32 flags)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 ifcr;
+
+	ifcr = flags << (4 * lchan->id);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_IFCR, 0, ifcr);
+}
+
+static void loongson2_cmc_dma_stop(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 ccr;
+
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+	ccr &= ~(LOONGSON2_CMCDMA_CCR_IRQ_MASK | LOONGSON2_CMCDMA_CCR_EN);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, ccr);
+
+	loongson2_cmc_dma_irq_clear(lchan, LOONGSON2_CMCDMA_MASKI);
+}
+
+static int loongson2_cmc_dma_terminate_all(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	LIST_HEAD(head);
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		if (lchan->desc) {
+			vchan_terminate_vdesc(&lchan->desc->vdesc);
+			loongson2_cmc_dma_stop(lchan);
+			lchan->desc = NULL;
+		}
+		vchan_get_all_descriptors(&lchan->vchan, &head);
+	}
+
+	vchan_dma_desc_free_list(&lchan->vchan, &head);
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_synchronize(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	vchan_synchronize(&lchan->vchan);
+}
+
+static void loongson2_cmc_dma_start_transfer(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct loongson2_cmc_dma_sg_req *sg_req;
+	struct loongson2_cmc_dma_chan_reg *reg;
+	struct virt_dma_desc *vdesc;
+
+	loongson2_cmc_dma_stop(lchan);
+
+	if (!lchan->desc) {
+		vdesc = vchan_next_desc(&lchan->vchan);
+		if (!vdesc)
+			return;
+
+		list_del(&vdesc->node);
+		lchan->desc = to_lmdma_desc(vdesc);
+		lchan->next_sg = 0;
+	}
+
+	if (lchan->next_sg == lchan->desc->num_sgs)
+		lchan->next_sg = 0;
+
+	sg_req = &lchan->desc->sg_req[lchan->next_sg];
+	reg = &sg_req->chan_reg;
+
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id, reg->cndtr);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CPAR, lchan->id, reg->cpar);
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, lchan->id, reg->cmar);
+
+	lchan->next_sg++;
+
+	/* Start DMA */
+	reg->ccr |= LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
+}
+
+static void loongson2_cmc_dma_configure_next_sg(struct loongson2_cmc_dma_chan *lchan)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct loongson2_cmc_dma_sg_req *sg_req;
+	u32 ccr, id = lchan->id;
+
+	if (lchan->next_sg == lchan->desc->num_sgs)
+		lchan->next_sg = 0;
+
+	/* Stop to update mem addr */
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, id);
+	ccr &= ~LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);
+
+	sg_req = &lchan->desc->sg_req[lchan->next_sg];
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, id, sg_req->chan_reg.cmar);
+
+	/* Start transition */
+	ccr |= LOONGSON2_CMCDMA_CCR_EN;
+	loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);
+}
+
+static void loongson2_cmc_dma_handle_chan_done(struct loongson2_cmc_dma_chan *lchan)
+{
+	if (!lchan->desc)
+		return;
+
+	if (lchan->desc->cyclic) {
+		vchan_cyclic_callback(&lchan->desc->vdesc);
+		/* LOONGSON2_CMCDMA_CCR_CIRC mode don't need update register */
+		if (lchan->desc->num_sgs == 1)
+			return;
+		loongson2_cmc_dma_configure_next_sg(lchan);
+		lchan->next_sg++;
+	} else {
+		if (lchan->next_sg == lchan->desc->num_sgs) {
+			vchan_cookie_complete(&lchan->desc->vdesc);
+			lchan->desc = NULL;
+		}
+		loongson2_cmc_dma_start_transfer(lchan);
+	}
+}
+
+static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid)
+{
+	struct loongson2_cmc_dma_chan *lchan = devid;
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	struct device *dev = chan2dev(lchan);
+	u32 ists, status, ccr;
+
+	scoped_guard(spinlock, &lchan->vchan.lock) {
+		ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+		ists = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_ISR, 0);
+		status = (ists >> (4 * lchan->id)) & LOONGSON2_CMCDMA_MASKI;
+
+		loongson2_cmc_dma_irq_clear(lchan, status);
+
+		if (status & LOONGSON2_CMCDMA_TCI) {
+			loongson2_cmc_dma_handle_chan_done(lchan);
+			status &= ~LOONGSON2_CMCDMA_TCI;
+		}
+
+		if (status & LOONGSON2_CMCDMA_HTI)
+			status &= ~LOONGSON2_CMCDMA_HTI;
+
+		if (status & LOONGSON2_CMCDMA_TEI) {
+			dev_err(dev, "DMA Transform Error.\n");
+			if (!(ccr & LOONGSON2_CMCDMA_CCR_EN))
+				dev_err(dev, "Channel disabled by HW.\n");
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) {
+		dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan);
+		loongson2_cmc_dma_start_transfer(lchan);
+	}
+}
+
+static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan,
+					    enum dma_transfer_direction direction,
+					    enum dma_slave_buswidth *buswidth, u32 buf_len)
+{
+	struct dma_slave_config	sconfig = lchan->dma_sconfig;
+	struct device *dev = chan2dev(lchan);
+	int dev_width;
+	u32 ccr;
+
+	switch (direction) {
+	case DMA_MEM_TO_DEV:
+		dev_width = loongson2_cmc_dma_get_width(sconfig.dst_addr_width);
+		if (dev_width < 0) {
+			dev_err(dev, "DMA_MEM_TO_DEV bus width not supported\n");
+			return dev_width;
+		}
+		lchan->chan_reg.cpar = sconfig.dst_addr;
+		ccr = LOONGSON2_CMCDMA_CCR_DIR;
+		*buswidth = sconfig.dst_addr_width;
+		break;
+	case DMA_DEV_TO_MEM:
+		dev_width = loongson2_cmc_dma_get_width(sconfig.src_addr_width);
+		if (dev_width < 0) {
+			dev_err(dev, "DMA_DEV_TO_MEM bus width not supported\n");
+			return dev_width;
+		}
+		lchan->chan_reg.cpar = sconfig.src_addr;
+		ccr = LOONGSON2_CMCDMA_CCR_MINC;
+		*buswidth = sconfig.src_addr_width;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ccr |= FIELD_PREP(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, dev_width) |
+	       FIELD_PREP(LOONGSON2_CMCDMA_CCR_MSIZE_MASK, dev_width);
+
+	/* Set DMA control register */
+	lchan->chan_reg.ccr &= ~(LOONGSON2_CMCDMA_CCR_PSIZE_MASK | LOONGSON2_CMCDMA_CCR_MSIZE_MASK);
+	lchan->chan_reg.ccr |= ccr;
+
+	return 0;
+}
+
+static struct dma_async_tx_descriptor *
+loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 sg_len,
+				enum dma_transfer_direction direction,
+				unsigned long flags, void *context)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct loongson2_cmc_dma_desc *desc;
+	enum dma_slave_buswidth buswidth;
+	struct scatterlist *sg;
+	u32 num_items, i;
+	int ret;
+
+	desc = kzalloc_flex(*desc, sg_req, sg_len, GFP_NOWAIT);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg));
+		if (ret)
+			return ERR_PTR(ret);
+
+		num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth);
+		if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
+			dev_err(chan2dev(lchan), "Number of items not supported\n");
+			kfree(desc);
+			return ERR_PTR(-EINVAL);
+		}
+
+		desc->sg_req[i].len = sg_dma_len(sg);
+		desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
+		desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
+		desc->sg_req[i].chan_reg.cmar = sg_dma_address(sg);
+		desc->sg_req[i].chan_reg.cndtr = num_items;
+	}
+
+	desc->num_sgs = sg_len;
+	desc->cyclic = false;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+static struct dma_async_tx_descriptor *
+loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+				  size_t period_len, enum dma_transfer_direction direction,
+				  unsigned long flags)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct loongson2_cmc_dma_desc *desc;
+	enum dma_slave_buswidth buswidth;
+	u32 num_periods, num_items, i;
+	int ret;
+
+	if (unlikely(buf_len % period_len))
+		return ERR_PTR(-EINVAL);
+
+	ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, period_len);
+	if (ret)
+		return ERR_PTR(ret);
+
+	num_items = DIV_ROUND_UP(period_len, buswidth);
+	if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
+		dev_err(chan2dev(lchan), "Number of items not supported\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Enable Circular mode */
+	if (buf_len == period_len)
+		lchan->chan_reg.ccr |= LOONGSON2_CMCDMA_CCR_CIRC;
+
+	num_periods = DIV_ROUND_UP(buf_len, period_len);
+	desc = kzalloc_flex(*desc, sg_req, num_periods, GFP_NOWAIT);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < num_periods; i++) {
+		desc->sg_req[i].len = period_len;
+		desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
+		desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
+		desc->sg_req[i].chan_reg.cmar = buf_addr;
+		desc->sg_req[i].chan_reg.cndtr = num_items;
+		buf_addr += period_len;
+	}
+
+	desc->num_sgs = num_periods;
+	desc->cyclic = true;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+static size_t loongson2_cmc_dma_desc_residue(struct loongson2_cmc_dma_chan *lchan,
+					     struct loongson2_cmc_dma_desc *desc, u32 next_sg)
+{
+	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
+	u32 residue, width, ndtr, ccr, i;
+
+	ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
+	width = FIELD_GET(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, ccr);
+
+	ndtr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id);
+	residue = ndtr << width;
+
+	if (lchan->desc->cyclic && next_sg == 0)
+		return residue;
+
+	for (i = next_sg; i < desc->num_sgs; i++)
+		residue += desc->sg_req[i].len;
+
+	return residue;
+}
+
+static enum dma_status loongson2_cmc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
+						   struct dma_tx_state *state)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct virt_dma_desc *vdesc;
+	enum dma_status status;
+
+	status = dma_cookie_status(chan, cookie, state);
+	if (status == DMA_COMPLETE || !state)
+		return status;
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		vdesc = vchan_find_desc(&lchan->vchan, cookie);
+		if (lchan->desc && cookie == lchan->desc->vdesc.tx.cookie)
+			state->residue = loongson2_cmc_dma_desc_residue(lchan, lchan->desc,
+									lchan->next_sg);
+		else if (vdesc)
+			state->residue = loongson2_cmc_dma_desc_residue(lchan,
+									to_lmdma_desc(vdesc), 0);
+	}
+
+	return status;
+}
+
+static void loongson2_cmc_dma_free_chan_resources(struct dma_chan *chan)
+{
+	vchan_free_chan_resources(to_virt_chan(chan));
+}
+
+static void loongson2_cmc_dma_desc_free(struct virt_dma_desc *vdesc)
+{
+	kfree(to_lmdma_desc(vdesc));
+}
+
+static bool loongson2_cmc_dma_acpi_filter(struct dma_chan *chan, void *param)
+{
+	struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
+	struct acpi_dma_spec *dma_spec = param;
+
+	memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
+	lchan->chan_reg.ccr = dma_spec->chan_id & LOONGSON2_CMCDMA_STREAM_MASK;
+
+	return true;
+}
+
+static int loongson2_cmc_dma_acpi_controller_register(struct loongson2_cmc_dma_dev *lddev)
+{
+	struct device *dev = lddev->ddev.dev;
+	struct acpi_dma_filter_info *info;
+
+	if (!is_acpi_node(dev_fwnode(dev)))
+		return 0;
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	dma_cap_zero(info->dma_cap);
+	info->dma_cap = lddev->ddev.cap_mask;
+	info->filter_fn = loongson2_cmc_dma_acpi_filter;
+
+	return devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info);
+}
+
+static struct dma_chan *loongson2_cmc_dma_of_xlate(struct of_phandle_args *dma_spec,
+						   struct of_dma *ofdma)
+{
+	struct loongson2_cmc_dma_dev *lddev = ofdma->of_dma_data;
+	struct device *dev = lddev->ddev.dev;
+	struct loongson2_cmc_dma_chan *lchan;
+	struct dma_chan *chan;
+
+	if (dma_spec->args_count < 2)
+		return ERR_PTR(-EINVAL);
+
+	if (dma_spec->args[0] >= lddev->nr_channels) {
+		dev_err(dev, "Invalid channel id.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	lchan = &lddev->chan[dma_spec->args[0]];
+	chan = dma_get_slave_channel(&lchan->vchan.chan);
+	if (!chan) {
+		dev_err(dev, "No more channels available.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
+	lchan->chan_reg.ccr = dma_spec->args[1] & LOONGSON2_CMCDMA_STREAM_MASK;
+
+	return chan;
+}
+
+static int loongson2_cmc_dma_of_controller_register(struct loongson2_cmc_dma_dev *lddev)
+{
+	struct device *dev = lddev->ddev.dev;
+
+	if (!is_of_node(dev_fwnode(dev)))
+		return 0;
+
+	return of_dma_controller_register(dev->of_node, loongson2_cmc_dma_of_xlate, lddev);
+}
+
+static int loongson2_cmc_dma_probe(struct platform_device *pdev)
+{
+	const struct loongson2_cmc_dma_config *config;
+	struct loongson2_cmc_dma_chan *lchan;
+	struct loongson2_cmc_dma_dev *lddev;
+	struct device *dev = &pdev->dev;
+	struct dma_device *ddev;
+	u32 nr_chans, i;
+	int ret;
+
+	config = (const struct loongson2_cmc_dma_config *)device_get_match_data(dev);
+	if (!config)
+		return -EINVAL;
+
+	ret = device_property_read_u32(dev, "dma-channels", &nr_chans);
+	if (ret || nr_chans > config->max_channels) {
+		dev_err(dev, "missing or invalid dma-channels property\n");
+		nr_chans = config->max_channels;
+	}
+
+	lddev = devm_kzalloc(dev, struct_size(lddev, chan, nr_chans), GFP_KERNEL);
+	if (!lddev)
+		return -ENOMEM;
+
+	lddev->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(lddev->base))
+		return PTR_ERR(lddev->base);
+
+	platform_set_drvdata(pdev, lddev);
+	lddev->nr_channels = nr_chans;
+	lddev->chan_reg_offset = config->chan_reg_offset;
+
+	lddev->dma_clk = devm_clk_get_optional_enabled(dev, NULL);
+	if (IS_ERR(lddev->dma_clk))
+		return dev_err_probe(dev, PTR_ERR(lddev->dma_clk), "Failed to get dma clock\n");
+
+	ddev = &lddev->ddev;
+	ddev->dev = dev;
+
+	dma_cap_zero(ddev->cap_mask);
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	dma_cap_set(DMA_PRIVATE, ddev->cap_mask);
+	dma_cap_set(DMA_CYCLIC, ddev->cap_mask);
+
+	ddev->device_free_chan_resources = loongson2_cmc_dma_free_chan_resources;
+	ddev->device_config = loongson2_cmc_dma_slave_config;
+	ddev->device_prep_slave_sg = loongson2_cmc_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = loongson2_cmc_dma_prep_dma_cyclic;
+	ddev->device_issue_pending = loongson2_cmc_dma_issue_pending;
+	ddev->device_synchronize = loongson2_cmc_dma_synchronize;
+	ddev->device_tx_status = loongson2_cmc_dma_tx_status;
+	ddev->device_terminate_all = loongson2_cmc_dma_terminate_all;
+
+	ddev->max_sg_burst = LOONSON2_CMCDMA_MAX_DATA_ITEMS;
+	ddev->src_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
+	ddev->dst_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	INIT_LIST_HEAD(&ddev->channels);
+
+	for (i = 0; i < nr_chans; i++) {
+		lchan = &lddev->chan[i];
+
+		lchan->id = i;
+		lchan->vchan.desc_free = loongson2_cmc_dma_desc_free;
+		vchan_init(&lchan->vchan, ddev);
+	}
+
+	ret = dmaenginem_async_device_register(ddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n");
+
+	for (i = 0; i < nr_chans; i++) {
+		lchan = &lddev->chan[i];
+
+		lchan->irq = platform_get_irq(pdev, i);
+		if (lchan->irq < 0)
+			return lchan->irq;
+
+		ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED,
+				       dev_name(chan2dev(lchan)), lchan);
+		if (ret)
+			return ret;
+	}
+
+	ret = loongson2_cmc_dma_acpi_controller_register(lddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register dma controller with ACPI.\n");
+
+	ret = loongson2_cmc_dma_of_controller_register(lddev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register dma controller with FDT.\n");
+
+	dev_info(dev, "Loongson-2 Multi-Channel DMA Controller registered successfully.\n");
+
+	return 0;
+}
+
+static void loongson2_cmc_dma_remove(struct platform_device *pdev)
+{
+	of_dma_controller_free(pdev->dev.of_node);
+}
+
+static const struct of_device_id loongson2_cmc_dma_of_match[] = {
+	{ .compatible = "loongson,ls2k0300-dma", .data = &ls2k0300_cmc_dma_config },
+	{ .compatible = "loongson,ls2k3000-dma", .data = &ls2k3000_cmc_dma_config },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match);
+
+static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = {
+	{ "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match);
+
+static struct platform_driver loongson2_cmc_dma_driver = {
+	.driver = {
+		.name = "loongson2-apb-cmc-dma",
+		.of_match_table = loongson2_cmc_dma_of_match,
+		.acpi_match_table = loongson2_cmc_dma_acpi_match,
+	},
+	.probe = loongson2_cmc_dma_probe,
+	.remove = loongson2_cmc_dma_remove,
+};
+module_platform_driver(loongson2_cmc_dma_driver);
+
+MODULE_DESCRIPTION("Loongson-2 Chain Multi-Channel DMA Controller driver");
+MODULE_AUTHOR("Loongson Technology Corporation Limited");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c
new file mode 100644
index 0000000000000..aceb069e71fca
--- /dev/null
+++ b/drivers/dma/loongson/loongson2-apb-dma.c
@@ -0,0 +1,686 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for the Loongson-2 APB DMA Controller
+ *
+ * Copyright (C) 2017-2023 Loongson Corporation
+ */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+/* Global Configuration Register */
+#define LDMA_ORDER_ERG		0x0
+
+/* Bitfield definitions */
+
+/* Bitfields in Global Configuration Register */
+#define LDMA_64BIT_EN		BIT(0) /* 1: 64 bit support */
+#define LDMA_UNCOHERENT_EN	BIT(1) /* 0: cache, 1: uncache */
+#define LDMA_ASK_VALID		BIT(2)
+#define LDMA_START		BIT(3) /* DMA start operation */
+#define LDMA_STOP		BIT(4) /* DMA stop operation */
+#define LDMA_CONFIG_MASK	GENMASK_ULL(4, 0) /* DMA controller config bits mask */
+
+/* Bitfields in ndesc_addr field of HW descriptor */
+#define LDMA_DESC_EN		BIT(0) /*1: The next descriptor is valid */
+#define LDMA_DESC_ADDR_LOW	GENMASK(31, 1)
+
+/* Bitfields in cmd field of HW descriptor */
+#define LDMA_INT		BIT(1) /* Enable DMA interrupts */
+#define LDMA_DATA_DIRECTION	BIT(12) /* 1: write to device, 0: read from device */
+
+#define LDMA_SLAVE_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
+
+#define LDMA_MAX_TRANS_LEN	U32_MAX
+
+/*--  descriptors  -----------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_hw_desc - DMA HW descriptor
+ * @ndesc_addr: the next descriptor low address.
+ * @mem_addr: memory low address.
+ * @apb_addr: device buffer address.
+ * @len: length of a piece of carried content, in words.
+ * @step_len: length between two moved memory data blocks.
+ * @step_times: number of blocks to be carried in a single DMA operation.
+ * @cmd: descriptor command or state.
+ * @stats: DMA status.
+ * @high_ndesc_addr: the next descriptor high address.
+ * @high_mem_addr: memory high address.
+ * @reserved: reserved
+ */
+struct ls2x_dma_hw_desc {
+	u32 ndesc_addr;
+	u32 mem_addr;
+	u32 apb_addr;
+	u32 len;
+	u32 step_len;
+	u32 step_times;
+	u32 cmd;
+	u32 stats;
+	u32 high_ndesc_addr;
+	u32 high_mem_addr;
+	u32 reserved[2];
+} __packed;
+
+/*
+ * struct ls2x_dma_sg - ls2x dma scatter gather entry
+ * @hw: the pointer to DMA HW descriptor.
+ * @llp: physical address of the DMA HW descriptor.
+ * @phys: destination or source address(mem).
+ * @len: number of Bytes to read.
+ */
+struct ls2x_dma_sg {
+	struct ls2x_dma_hw_desc	*hw;
+	dma_addr_t		llp;
+	dma_addr_t		phys;
+	u32			len;
+};
+
+/*
+ * struct ls2x_dma_desc - software descriptor
+ * @vdesc: pointer to the virtual dma descriptor.
+ * @cyclic: flag to dma cyclic
+ * @burst_size: burst size of transaction, in words.
+ * @desc_num: number of sg entries.
+ * @direction: transfer direction, to or from device.
+ * @status: dma controller status.
+ * @sg: array of sgs.
+ */
+struct ls2x_dma_desc {
+	struct virt_dma_desc		vdesc;
+	bool				cyclic;
+	size_t				burst_size;
+	u32				desc_num;
+	enum dma_transfer_direction	direction;
+	enum dma_status			status;
+	struct ls2x_dma_sg		sg[] __counted_by(desc_num);
+};
+
+/*--  Channels  --------------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel
+ * @vchan: virtual dma channel entry.
+ * @desc: pointer to the ls2x sw dma descriptor.
+ * @pool: hw desc table
+ * @irq: irq line
+ * @sconfig: configuration for slave transfers, passed via .device_config
+ */
+struct ls2x_dma_chan {
+	struct virt_dma_chan	vchan;
+	struct ls2x_dma_desc	*desc;
+	void			*pool;
+	int			irq;
+	struct dma_slave_config	sconfig;
+};
+
+/*--  Controller  ------------------------------------------------------*/
+
+/*
+ * struct ls2x_dma_priv - LS2X APB DMAC specific information
+ * @ddev: dmaengine dma_device object members
+ * @dma_clk: DMAC clock source
+ * @regs: memory mapped register base
+ * @lchan: channel to store ls2x_dma_chan structures
+ */
+struct ls2x_dma_priv {
+	struct dma_device	ddev;
+	struct clk		*dma_clk;
+	void __iomem		*regs;
+	struct ls2x_dma_chan	lchan;
+};
+
+/*--  Helper functions  ------------------------------------------------*/
+
+static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc)
+{
+	return container_of(vdesc, struct ls2x_dma_desc, vdesc);
+}
+
+static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan)
+{
+	return container_of(chan, struct ls2x_dma_chan, vchan.chan);
+}
+
+static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev)
+{
+	return container_of(ddev, struct ls2x_dma_priv, ddev);
+}
+
+static struct device *chan2dev(struct dma_chan *chan)
+{
+	return &chan->dev->device;
+}
+
+static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan);
+	struct ls2x_dma_desc *desc = to_ldma_desc(vdesc);
+	int i;
+
+	for (i = 0; i < desc->desc_num; i++) {
+		if (desc->sg[i].hw)
+			dma_pool_free(lchan->pool, desc->sg[i].hw,
+				      desc->sg[i].llp);
+	}
+
+	kfree(desc);
+}
+
+static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd)
+{
+	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
+	u64 val;
+
+	val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK;
+	val |= LDMA_64BIT_EN | cmd;
+	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
+}
+
+static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan)
+{
+	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
+	struct ls2x_dma_sg *ldma_sg;
+	struct virt_dma_desc *vdesc;
+	u64 val;
+
+	/* Get the next descriptor */
+	vdesc = vchan_next_desc(&lchan->vchan);
+	if (!vdesc) {
+		lchan->desc = NULL;
+		return;
+	}
+
+	list_del(&vdesc->node);
+	lchan->desc = to_ldma_desc(vdesc);
+	ldma_sg = &lchan->desc->sg[0];
+
+	/* Start DMA */
+	lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG);
+	val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START;
+	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
+}
+
+static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan)
+{
+	u32 maxburst, buswidth;
+
+	/* Reject definitely invalid configurations */
+	if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) &&
+	    (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS))
+		return 0;
+
+	if (lchan->sconfig.direction == DMA_MEM_TO_DEV) {
+		maxburst = lchan->sconfig.dst_maxburst;
+		buswidth = lchan->sconfig.dst_addr_width;
+	} else {
+		maxburst = lchan->sconfig.src_maxburst;
+		buswidth = lchan->sconfig.src_addr_width;
+	}
+
+	/* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */
+	return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN;
+}
+
+static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index,
+			       struct ls2x_dma_desc *desc)
+{
+	struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index];
+	u32 num_segments, segment_size;
+
+	if (desc->direction == DMA_MEM_TO_DEV) {
+		ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION;
+		ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr;
+	} else {
+		ldma_sg->hw->cmd = LDMA_INT;
+		ldma_sg->hw->apb_addr = lchan->sconfig.src_addr;
+	}
+
+	ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys);
+	ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys);
+
+	/* Split into multiple equally sized segments if necessary */
+	num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size);
+	segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments);
+
+	/* Word count register takes input in words */
+	ldma_sg->hw->len = segment_size;
+	ldma_sg->hw->step_times = num_segments;
+	ldma_sg->hw->step_len = 0;
+
+	/* lets make a link list */
+	if (sg_index) {
+		desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN;
+		desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp);
+	}
+}
+
+/*--  DMA Engine API  --------------------------------------------------*/
+
+/*
+ * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel
+ * @chan: allocate descriptor resources for this channel
+ *
+ * return - the number of allocated descriptors
+ */
+static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	/* Create a pool of consistent memory blocks for hardware descriptors */
+	lchan->pool = dma_pool_create(dev_name(chan2dev(chan)),
+				      chan->device->dev, PAGE_SIZE,
+				      __alignof__(struct ls2x_dma_hw_desc), 0);
+	if (!lchan->pool) {
+		dev_err(chan2dev(chan), "No memory for descriptors\n");
+		return -ENOMEM;
+	}
+
+	return 1;
+}
+
+/*
+ * ls2x_dma_free_chan_resources - free all channel resources
+ * @chan: DMA channel
+ */
+static void ls2x_dma_free_chan_resources(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	vchan_free_chan_resources(to_virt_chan(chan));
+	dma_pool_destroy(lchan->pool);
+	lchan->pool = NULL;
+}
+
+/*
+ * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: tx descriptor status flags
+ * @context: transaction context (ignored)
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+		       u32 sg_len, enum dma_transfer_direction direction,
+		       unsigned long flags, void *context)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	struct ls2x_dma_desc *desc;
+	struct scatterlist *sg;
+	size_t burst_size;
+	int i;
+
+	if (unlikely(!sg_len || !is_slave_direction(direction)))
+		return NULL;
+
+	burst_size = ls2x_dmac_detect_burst(lchan);
+	if (!burst_size)
+		return NULL;
+
+	desc = kzalloc_flex(*desc, sg, sg_len, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	desc->desc_num = sg_len;
+	desc->direction = direction;
+	desc->burst_size = burst_size;
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
+
+		/* Allocate DMA capable memory for hardware descriptor */
+		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
+		if (!ldma_sg->hw) {
+			desc->desc_num = i;
+			ls2x_dma_desc_free(&desc->vdesc);
+			return NULL;
+		}
+
+		ldma_sg->phys = sg_dma_address(sg);
+		ldma_sg->len = sg_dma_len(sg);
+
+		ls2x_dma_fill_desc(lchan, i, desc);
+	}
+
+	/* Setting the last descriptor enable bit */
+	desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN;
+	desc->status = DMA_IN_PROGRESS;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+/*
+ * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer
+ * @chan: the DMA channel to prepare
+ * @buf_addr: physical DMA address where the buffer starts
+ * @buf_len: total number of bytes for the entire buffer
+ * @period_len: number of bytes for each period
+ * @direction: transfer direction, to or from device
+ * @flags: tx descriptor status flags
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+			 size_t period_len, enum dma_transfer_direction direction,
+			 unsigned long flags)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	struct ls2x_dma_desc *desc;
+	size_t burst_size;
+	u32 num_periods;
+	int i;
+
+	if (unlikely(!buf_len || !period_len))
+		return NULL;
+
+	if (unlikely(!is_slave_direction(direction)))
+		return NULL;
+
+	burst_size = ls2x_dmac_detect_burst(lchan);
+	if (!burst_size)
+		return NULL;
+
+	num_periods = buf_len / period_len;
+	desc = kzalloc_flex(*desc, sg, num_periods, GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	desc->desc_num = num_periods;
+	desc->direction = direction;
+	desc->burst_size = burst_size;
+
+	/* Build cyclic linked list */
+	for (i = 0; i < num_periods; i++) {
+		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
+
+		/* Allocate DMA capable memory for hardware descriptor */
+		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
+		if (!ldma_sg->hw) {
+			desc->desc_num = i;
+			ls2x_dma_desc_free(&desc->vdesc);
+			return NULL;
+		}
+
+		ldma_sg->phys = buf_addr + period_len * i;
+		ldma_sg->len = period_len;
+
+		ls2x_dma_fill_desc(lchan, i, desc);
+	}
+
+	/* Lets make a cyclic list */
+	desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN;
+	desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp);
+	desc->cyclic = true;
+	desc->status = DMA_IN_PROGRESS;
+
+	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
+}
+
+/*
+ * ls2x_slave_config - set slave configuration for channel
+ * @chan: dma channel
+ * @cfg: slave configuration
+ *
+ * Sets slave configuration for channel
+ */
+static int ls2x_dma_slave_config(struct dma_chan *chan,
+				 struct dma_slave_config *config)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	memcpy(&lchan->sconfig, config, sizeof(*config));
+	return 0;
+}
+
+/*
+ * ls2x_dma_issue_pending - push pending transactions to the hardware
+ * @chan: channel
+ *
+ * When this function is called, all pending transactions are pushed to the
+ * hardware and executed.
+ */
+static void ls2x_dma_issue_pending(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc)
+		ls2x_dma_start_transfer(lchan);
+}
+
+/*
+ * ls2x_dma_terminate_all - terminate all transactions
+ * @chan: channel
+ *
+ * Stops all DMA transactions.
+ */
+static int ls2x_dma_terminate_all(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+	LIST_HEAD(head);
+
+	scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
+		/* Setting stop cmd */
+		ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		if (lchan->desc) {
+			vchan_terminate_vdesc(&lchan->desc->vdesc);
+			lchan->desc = NULL;
+		}
+
+		vchan_get_all_descriptors(&lchan->vchan, &head);
+	}
+
+	vchan_dma_desc_free_list(&lchan->vchan, &head);
+	return 0;
+}
+
+/*
+ * ls2x_dma_synchronize - Synchronizes the termination of transfers to the
+ * current context.
+ * @chan: channel
+ */
+static void ls2x_dma_synchronize(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	vchan_synchronize(&lchan->vchan);
+}
+
+static int ls2x_dma_pause(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) {
+		ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		lchan->desc->status = DMA_PAUSED;
+	}
+
+	return 0;
+}
+
+static int ls2x_dma_resume(struct dma_chan *chan)
+{
+	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
+
+	guard(spinlock_irqsave)(&lchan->vchan.lock);
+
+	if (lchan->desc && lchan->desc->status == DMA_PAUSED) {
+		lchan->desc->status = DMA_IN_PROGRESS;
+		ls2x_dma_write_cmd(lchan, LDMA_START);
+	}
+
+	return 0;
+}
+
+/*
+ * ls2x_dma_isr - LS2X DMA Interrupt handler
+ * @irq: IRQ number
+ * @dev_id: Pointer to ls2x_dma_chan
+ *
+ * Return: IRQ_HANDLED/IRQ_NONE
+ */
+static irqreturn_t ls2x_dma_isr(int irq, void *dev_id)
+{
+	struct ls2x_dma_chan *lchan = dev_id;
+	struct ls2x_dma_desc *desc;
+
+	scoped_guard(spinlock, &lchan->vchan.lock) {
+		desc = lchan->desc;
+		if (desc) {
+			if (desc->cyclic) {
+				vchan_cyclic_callback(&desc->vdesc);
+			} else {
+				desc->status = DMA_COMPLETE;
+				vchan_cookie_complete(&desc->vdesc);
+				ls2x_dma_start_transfer(lchan);
+			}
+
+			/* ls2x_dma_start_transfer() updates lchan->desc */
+			if (!lchan->desc)
+				ls2x_dma_write_cmd(lchan, LDMA_STOP);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int ls2x_dma_chan_init(struct platform_device *pdev,
+			      struct ls2x_dma_priv *priv)
+{
+	struct ls2x_dma_chan *lchan = &priv->lchan;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	lchan->irq = platform_get_irq(pdev, 0);
+	if (lchan->irq < 0)
+		return lchan->irq;
+
+	ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING,
+			       dev_name(&pdev->dev), lchan);
+	if (ret)
+		return ret;
+
+	/* Initialize channels related values */
+	INIT_LIST_HEAD(&priv->ddev.channels);
+	lchan->vchan.desc_free = ls2x_dma_desc_free;
+	vchan_init(&lchan->vchan, &priv->ddev);
+
+	return 0;
+}
+
+/*
+ * ls2x_dma_probe - Driver probe function
+ * @pdev: Pointer to the platform_device structure
+ *
+ * Return: '0' on success and failure value on error
+ */
+static int ls2x_dma_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ls2x_dma_priv *priv;
+	struct dma_device *ddev;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->regs))
+		return dev_err_probe(dev, PTR_ERR(priv->regs),
+				     "devm_platform_ioremap_resource failed.\n");
+
+	priv->dma_clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(priv->dma_clk))
+		return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "Couldn't start the clock.\n");
+
+	ret = ls2x_dma_chan_init(pdev, priv);
+	if (ret)
+		return ret;
+
+	ddev = &priv->ddev;
+	ddev->dev = dev;
+	dma_cap_zero(ddev->cap_mask);
+	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
+	dma_cap_set(DMA_CYCLIC, ddev->cap_mask);
+
+	ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources;
+	ddev->device_free_chan_resources = ls2x_dma_free_chan_resources;
+	ddev->device_tx_status = dma_cookie_status;
+	ddev->device_issue_pending = ls2x_dma_issue_pending;
+	ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg;
+	ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic;
+	ddev->device_config = ls2x_dma_slave_config;
+	ddev->device_terminate_all = ls2x_dma_terminate_all;
+	ddev->device_synchronize = ls2x_dma_synchronize;
+	ddev->device_pause = ls2x_dma_pause;
+	ddev->device_resume = ls2x_dma_resume;
+
+	ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS;
+	ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS;
+	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+
+	ret = dmaenginem_async_device_register(&priv->ddev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n");
+
+	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to register dma controller.\n");
+
+	platform_set_drvdata(pdev, priv);
+
+	dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n");
+	return 0;
+}
+
+/*
+ * ls2x_dma_remove - Driver remove function
+ * @pdev: Pointer to the platform_device structure
+ */
+static void ls2x_dma_remove(struct platform_device *pdev)
+{
+	of_dma_controller_free(pdev->dev.of_node);
+}
+
+static const struct of_device_id ls2x_dma_of_match_table[] = {
+	{ .compatible = "loongson,ls2k1000-apbdma" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table);
+
+static struct platform_driver ls2x_dmac_driver = {
+	.probe		= ls2x_dma_probe,
+	.remove		= ls2x_dma_remove,
+	.driver = {
+		.name	= "ls2x-apbdma",
+		.of_match_table	= ls2x_dma_of_match_table,
+	},
+};
+module_platform_driver(ls2x_dmac_driver);
+
+MODULE_DESCRIPTION("Loongson-2 APB DMA Controller driver");
+MODULE_AUTHOR("Loongson Technology Corporation Limited");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson1-apb-dma.c b/drivers/dma/loongson1-apb-dma.c
deleted file mode 100644
index 2e347aba9af81..0000000000000
--- a/drivers/dma/loongson1-apb-dma.c
+++ /dev/null
@@ -1,660 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for Loongson-1 APB DMA Controller
- *
- * Copyright (C) 2015-2024 Keguang Zhang <keguang.zhang@gmail.com>
- */
-
-#include <linux/dmapool.h>
-#include <linux/dma-mapping.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/iopoll.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_dma.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include "dmaengine.h"
-#include "virt-dma.h"
-
-/* Loongson-1 DMA Control Register */
-#define LS1X_DMA_CTRL		0x0
-
-/* DMA Control Register Bits */
-#define LS1X_DMA_STOP		BIT(4)
-#define LS1X_DMA_START		BIT(3)
-#define LS1X_DMA_ASK_VALID	BIT(2)
-
-/* DMA Next Field Bits */
-#define LS1X_DMA_NEXT_VALID	BIT(0)
-
-/* DMA Command Field Bits */
-#define LS1X_DMA_RAM2DEV	BIT(12)
-#define LS1X_DMA_INT		BIT(1)
-#define LS1X_DMA_INT_MASK	BIT(0)
-
-#define LS1X_DMA_LLI_ALIGNMENT	64
-#define LS1X_DMA_LLI_ADDR_MASK	GENMASK(31, __ffs(LS1X_DMA_LLI_ALIGNMENT))
-#define LS1X_DMA_MAX_CHANNELS	3
-
-enum ls1x_dmadesc_offsets {
-	LS1X_DMADESC_NEXT = 0,
-	LS1X_DMADESC_SADDR,
-	LS1X_DMADESC_DADDR,
-	LS1X_DMADESC_LENGTH,
-	LS1X_DMADESC_STRIDE,
-	LS1X_DMADESC_CYCLES,
-	LS1X_DMADESC_CMD,
-	LS1X_DMADESC_SIZE
-};
-
-struct ls1x_dma_lli {
-	unsigned int hw[LS1X_DMADESC_SIZE];
-	dma_addr_t phys;
-	struct list_head node;
-} __aligned(LS1X_DMA_LLI_ALIGNMENT);
-
-struct ls1x_dma_desc {
-	struct virt_dma_desc vd;
-	struct list_head lli_list;
-};
-
-struct ls1x_dma_chan {
-	struct virt_dma_chan vc;
-	struct dma_pool *lli_pool;
-	phys_addr_t src_addr;
-	phys_addr_t dst_addr;
-	enum dma_slave_buswidth src_addr_width;
-	enum dma_slave_buswidth dst_addr_width;
-	unsigned int bus_width;
-	void __iomem *reg_base;
-	int irq;
-	bool is_cyclic;
-	struct ls1x_dma_lli *curr_lli;
-};
-
-struct ls1x_dma {
-	struct dma_device ddev;
-	unsigned int nr_chans;
-	struct ls1x_dma_chan chan[];
-};
-
-static irqreturn_t ls1x_dma_irq_handler(int irq, void *data);
-
-#define to_ls1x_dma_chan(dchan)		\
-	container_of(dchan, struct ls1x_dma_chan, vc.chan)
-
-#define to_ls1x_dma_desc(d)		\
-	container_of(d, struct ls1x_dma_desc, vd)
-
-static inline struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
-static inline int ls1x_dma_query(struct ls1x_dma_chan *chan,
-				 dma_addr_t *lli_phys)
-{
-	struct dma_chan *dchan = &chan->vc.chan;
-	int val, ret;
-
-	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
-	val |= LS1X_DMA_ASK_VALID;
-	val |= dchan->chan_id;
-	writel(val, chan->reg_base + LS1X_DMA_CTRL);
-	ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val,
-					!(val & LS1X_DMA_ASK_VALID), 0, 3000);
-	if (ret)
-		dev_err(chan2dev(dchan), "failed to query DMA\n");
-
-	return ret;
-}
-
-static inline int ls1x_dma_start(struct ls1x_dma_chan *chan,
-				 dma_addr_t *lli_phys)
-{
-	struct dma_chan *dchan = &chan->vc.chan;
-	struct device *dev = chan2dev(dchan);
-	int val, ret;
-
-	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
-	val |= LS1X_DMA_START;
-	val |= dchan->chan_id;
-	writel(val, chan->reg_base + LS1X_DMA_CTRL);
-	ret = readl_poll_timeout(chan->reg_base + LS1X_DMA_CTRL, val,
-				 !(val & LS1X_DMA_START), 0, 1000);
-	if (!ret)
-		dev_dbg(dev, "start DMA with lli_phys=%pad\n", lli_phys);
-	else
-		dev_err(dev, "failed to start DMA\n");
-
-	return ret;
-}
-
-static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan)
-{
-	int val = readl(chan->reg_base + LS1X_DMA_CTRL);
-
-	writel(val | LS1X_DMA_STOP, chan->reg_base + LS1X_DMA_CTRL);
-}
-
-static void ls1x_dma_free_chan_resources(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct device *dev = chan2dev(dchan);
-
-	dma_free_coherent(dev, sizeof(struct ls1x_dma_lli),
-			  chan->curr_lli, chan->curr_lli->phys);
-	dma_pool_destroy(chan->lli_pool);
-	chan->lli_pool = NULL;
-	devm_free_irq(dev, chan->irq, chan);
-	vchan_free_chan_resources(&chan->vc);
-}
-
-static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct device *dev = chan2dev(dchan);
-	dma_addr_t phys;
-	int ret;
-
-	ret = devm_request_irq(dev, chan->irq, ls1x_dma_irq_handler,
-			       IRQF_SHARED, dma_chan_name(dchan), chan);
-	if (ret) {
-		dev_err(dev, "failed to request IRQ %d\n", chan->irq);
-		return ret;
-	}
-
-	chan->lli_pool = dma_pool_create(dma_chan_name(dchan), dev,
-					 sizeof(struct ls1x_dma_lli),
-					 __alignof__(struct ls1x_dma_lli), 0);
-	if (!chan->lli_pool)
-		return -ENOMEM;
-
-	/* allocate memory for querying the current lli */
-	dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
-	chan->curr_lli = dma_alloc_coherent(dev, sizeof(struct ls1x_dma_lli),
-					    &phys, GFP_KERNEL);
-	if (!chan->curr_lli) {
-		dma_pool_destroy(chan->lli_pool);
-		return -ENOMEM;
-	}
-	chan->curr_lli->phys = phys;
-
-	return 0;
-}
-
-static void ls1x_dma_free_desc(struct virt_dma_desc *vd)
-{
-	struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(vd->tx.chan);
-	struct ls1x_dma_lli *lli, *_lli;
-
-	list_for_each_entry_safe(lli, _lli, &desc->lli_list, node) {
-		list_del(&lli->node);
-		dma_pool_free(chan->lli_pool, lli, lli->phys);
-	}
-
-	kfree(desc);
-}
-
-static struct ls1x_dma_desc *ls1x_dma_alloc_desc(void)
-{
-	struct ls1x_dma_desc *desc;
-
-	desc = kzalloc_obj(*desc, GFP_NOWAIT);
-	if (!desc)
-		return NULL;
-
-	INIT_LIST_HEAD(&desc->lli_list);
-
-	return desc;
-}
-
-static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc,
-			     struct scatterlist *sgl, unsigned int sg_len,
-			     enum dma_transfer_direction dir, bool is_cyclic)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL;
-	struct device *dev = chan2dev(dchan);
-	struct list_head *pos = NULL;
-	struct scatterlist *sg;
-	unsigned int dev_addr, cmd, i;
-
-	switch (dir) {
-	case DMA_MEM_TO_DEV:
-		dev_addr = chan->dst_addr;
-		chan->bus_width = chan->dst_addr_width;
-		cmd = LS1X_DMA_RAM2DEV | LS1X_DMA_INT;
-		break;
-	case DMA_DEV_TO_MEM:
-		dev_addr = chan->src_addr;
-		chan->bus_width = chan->src_addr_width;
-		cmd = LS1X_DMA_INT;
-		break;
-	default:
-		dev_err(dev, "unsupported DMA direction: %s\n",
-			dmaengine_get_direction_text(dir));
-		return -EINVAL;
-	}
-
-	for_each_sg(sgl, sg, sg_len, i) {
-		dma_addr_t buf_addr = sg_dma_address(sg);
-		size_t buf_len = sg_dma_len(sg);
-		dma_addr_t phys;
-
-		if (!is_dma_copy_aligned(dchan->device, buf_addr, 0, buf_len)) {
-			dev_err(dev, "buffer is not aligned\n");
-			return -EINVAL;
-		}
-
-		/* allocate HW descriptors */
-		lli = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, &phys);
-		if (!lli) {
-			dev_err(dev, "failed to alloc lli %u\n", i);
-			return -ENOMEM;
-		}
-
-		/* setup HW descriptors */
-		lli->phys = phys;
-		lli->hw[LS1X_DMADESC_SADDR] = buf_addr;
-		lli->hw[LS1X_DMADESC_DADDR] = dev_addr;
-		lli->hw[LS1X_DMADESC_LENGTH] = buf_len / chan->bus_width;
-		lli->hw[LS1X_DMADESC_STRIDE] = 0;
-		lli->hw[LS1X_DMADESC_CYCLES] = 1;
-		lli->hw[LS1X_DMADESC_CMD] = cmd;
-
-		if (prev)
-			prev->hw[LS1X_DMADESC_NEXT] =
-			    lli->phys | LS1X_DMA_NEXT_VALID;
-		prev = lli;
-
-		if (!first)
-			first = lli;
-
-		list_add_tail(&lli->node, &desc->lli_list);
-	}
-
-	if (is_cyclic) {
-		lli->hw[LS1X_DMADESC_NEXT] = first->phys | LS1X_DMA_NEXT_VALID;
-		chan->is_cyclic = is_cyclic;
-	}
-
-	list_for_each(pos, &desc->lli_list) {
-		lli = list_entry(pos, struct ls1x_dma_lli, node);
-		print_hex_dump_debug("LLI: ", DUMP_PREFIX_OFFSET, 16, 4,
-				     lli, sizeof(*lli), false);
-	}
-
-	return 0;
-}
-
-static struct dma_async_tx_descriptor *
-ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
-		       unsigned int sg_len, enum dma_transfer_direction dir,
-		       unsigned long flags, void *context)
-{
-	struct ls1x_dma_desc *desc;
-
-	dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n",
-		sg_len, flags, dmaengine_get_direction_text(dir));
-
-	desc = ls1x_dma_alloc_desc();
-	if (!desc)
-		return NULL;
-
-	if (ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, false)) {
-		ls1x_dma_free_desc(&desc->vd);
-		return NULL;
-	}
-
-	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
-}
-
-static struct dma_async_tx_descriptor *
-ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
-			 size_t buf_len, size_t period_len,
-			 enum dma_transfer_direction dir, unsigned long flags)
-{
-	struct ls1x_dma_desc *desc;
-	struct scatterlist *sgl;
-	unsigned int sg_len;
-	unsigned int i;
-	int ret;
-
-	dev_dbg(chan2dev(dchan),
-		"buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n",
-		buf_len, period_len, flags, dmaengine_get_direction_text(dir));
-
-	desc = ls1x_dma_alloc_desc();
-	if (!desc)
-		return NULL;
-
-	/* allocate the scatterlist */
-	sg_len = buf_len / period_len;
-	sgl = kmalloc_objs(*sgl, sg_len, GFP_NOWAIT);
-	if (!sgl)
-		return NULL;
-
-	sg_init_table(sgl, sg_len);
-	for (i = 0; i < sg_len; ++i) {
-		sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(buf_addr)),
-			    period_len, offset_in_page(buf_addr));
-		sg_dma_address(&sgl[i]) = buf_addr;
-		sg_dma_len(&sgl[i]) = period_len;
-		buf_addr += period_len;
-	}
-
-	ret = ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, true);
-	kfree(sgl);
-	if (ret) {
-		ls1x_dma_free_desc(&desc->vd);
-		return NULL;
-	}
-
-	return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags);
-}
-
-static int ls1x_dma_slave_config(struct dma_chan *dchan,
-				 struct dma_slave_config *config)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-
-	chan->src_addr = config->src_addr;
-	chan->src_addr_width = config->src_addr_width;
-	chan->dst_addr = config->dst_addr;
-	chan->dst_addr_width = config->dst_addr_width;
-
-	return 0;
-}
-
-static int ls1x_dma_pause(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	int ret;
-
-	guard(spinlock_irqsave)(&chan->vc.lock);
-	/* save the current lli */
-	ret = ls1x_dma_query(chan, &chan->curr_lli->phys);
-	if (!ret)
-		ls1x_dma_stop(chan);
-
-	return ret;
-}
-
-static int ls1x_dma_resume(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-
-	guard(spinlock_irqsave)(&chan->vc.lock);
-
-	return ls1x_dma_start(chan, &chan->curr_lli->phys);
-}
-
-static int ls1x_dma_terminate_all(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct virt_dma_desc *vd;
-	LIST_HEAD(head);
-
-	ls1x_dma_stop(chan);
-
-	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
-		vd = vchan_next_desc(&chan->vc);
-		if (vd)
-			vchan_terminate_vdesc(vd);
-
-		vchan_get_all_descriptors(&chan->vc, &head);
-	}
-
-	vchan_dma_desc_free_list(&chan->vc, &head);
-
-	return 0;
-}
-
-static void ls1x_dma_synchronize(struct dma_chan *dchan)
-{
-	vchan_synchronize(to_virt_chan(dchan));
-}
-
-static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan,
-					  dma_cookie_t cookie,
-					  struct dma_tx_state *state)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct virt_dma_desc *vd;
-	enum dma_status status;
-	size_t bytes = 0;
-
-	status = dma_cookie_status(dchan, cookie, state);
-	if (status == DMA_COMPLETE)
-		return status;
-
-	scoped_guard(spinlock_irqsave, &chan->vc.lock) {
-		vd = vchan_find_desc(&chan->vc, cookie);
-		if (vd) {
-			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
-			struct ls1x_dma_lli *lli;
-			dma_addr_t next_phys;
-
-			/* get the current lli */
-			if (ls1x_dma_query(chan, &chan->curr_lli->phys))
-				return status;
-
-			/* locate the current lli */
-			next_phys = chan->curr_lli->hw[LS1X_DMADESC_NEXT];
-			list_for_each_entry(lli, &desc->lli_list, node)
-				if (lli->hw[LS1X_DMADESC_NEXT] == next_phys)
-					break;
-
-			dev_dbg(chan2dev(dchan), "current lli_phys=%pad",
-				&lli->phys);
-
-			/* count the residues */
-			list_for_each_entry_from(lli, &desc->lli_list, node)
-				bytes += lli->hw[LS1X_DMADESC_LENGTH] *
-					 chan->bus_width;
-		}
-	}
-
-	dma_set_residue(state, bytes);
-
-	return status;
-}
-
-static void ls1x_dma_issue_pending(struct dma_chan *dchan)
-{
-	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-
-	guard(spinlock_irqsave)(&chan->vc.lock);
-
-	if (vchan_issue_pending(&chan->vc)) {
-		struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
-
-		if (vd) {
-			struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd);
-			struct ls1x_dma_lli *lli;
-
-			lli = list_first_entry(&desc->lli_list,
-					       struct ls1x_dma_lli, node);
-			ls1x_dma_start(chan, &lli->phys);
-		}
-	}
-}
-
-static irqreturn_t ls1x_dma_irq_handler(int irq, void *data)
-{
-	struct ls1x_dma_chan *chan = data;
-	struct dma_chan *dchan = &chan->vc.chan;
-	struct device *dev = chan2dev(dchan);
-	struct virt_dma_desc *vd;
-
-	scoped_guard(spinlock, &chan->vc.lock) {
-		vd = vchan_next_desc(&chan->vc);
-		if (!vd) {
-			dev_warn(dev,
-				 "IRQ %d with no active desc on channel %d\n",
-				 irq, dchan->chan_id);
-			return IRQ_NONE;
-		}
-
-		if (chan->is_cyclic) {
-			vchan_cyclic_callback(vd);
-		} else {
-			list_del(&vd->node);
-			vchan_cookie_complete(vd);
-		}
-	}
-
-	dev_dbg(dev, "DMA IRQ %d on channel %d\n", irq, dchan->chan_id);
-
-	return IRQ_HANDLED;
-}
-
-static int ls1x_dma_chan_probe(struct platform_device *pdev,
-			       struct ls1x_dma *dma)
-{
-	void __iomem *reg_base;
-	int id;
-
-	reg_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(reg_base))
-		return PTR_ERR(reg_base);
-
-	for (id = 0; id < dma->nr_chans; id++) {
-		struct ls1x_dma_chan *chan = &dma->chan[id];
-		char pdev_irqname[16];
-
-		snprintf(pdev_irqname, sizeof(pdev_irqname), "ch%d", id);
-		chan->irq = platform_get_irq_byname(pdev, pdev_irqname);
-		if (chan->irq < 0)
-			return dev_err_probe(&pdev->dev, chan->irq,
-					     "failed to get IRQ for ch%d\n",
-					     id);
-
-		chan->reg_base = reg_base;
-		chan->vc.desc_free = ls1x_dma_free_desc;
-		vchan_init(&chan->vc, &dma->ddev);
-	}
-
-	return 0;
-}
-
-static void ls1x_dma_chan_remove(struct ls1x_dma *dma)
-{
-	int id;
-
-	for (id = 0; id < dma->nr_chans; id++) {
-		struct ls1x_dma_chan *chan = &dma->chan[id];
-
-		if (chan->vc.chan.device == &dma->ddev) {
-			list_del(&chan->vc.chan.device_node);
-			tasklet_kill(&chan->vc.task);
-		}
-	}
-}
-
-static int ls1x_dma_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct dma_device *ddev;
-	struct ls1x_dma *dma;
-	int ret;
-
-	ret = platform_irq_count(pdev);
-	if (ret <= 0 || ret > LS1X_DMA_MAX_CHANNELS)
-		return dev_err_probe(dev, -EINVAL,
-				     "Invalid number of IRQ channels: %d\n",
-				     ret);
-
-	dma = devm_kzalloc(dev, struct_size(dma, chan, ret), GFP_KERNEL);
-	if (!dma)
-		return -ENOMEM;
-	dma->nr_chans = ret;
-
-	/* initialize DMA device */
-	ddev = &dma->ddev;
-	ddev->dev = dev;
-	ddev->copy_align = DMAENGINE_ALIGN_4_BYTES;
-	ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
-				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
-				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
-	ddev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
-				BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
-				BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
-	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
-	ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
-	ddev->device_alloc_chan_resources = ls1x_dma_alloc_chan_resources;
-	ddev->device_free_chan_resources = ls1x_dma_free_chan_resources;
-	ddev->device_prep_slave_sg = ls1x_dma_prep_slave_sg;
-	ddev->device_prep_dma_cyclic = ls1x_dma_prep_dma_cyclic;
-	ddev->device_config = ls1x_dma_slave_config;
-	ddev->device_pause = ls1x_dma_pause;
-	ddev->device_resume = ls1x_dma_resume;
-	ddev->device_terminate_all = ls1x_dma_terminate_all;
-	ddev->device_synchronize = ls1x_dma_synchronize;
-	ddev->device_tx_status = ls1x_dma_tx_status;
-	ddev->device_issue_pending = ls1x_dma_issue_pending;
-	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
-	INIT_LIST_HEAD(&ddev->channels);
-
-	/* initialize DMA channels */
-	ret = ls1x_dma_chan_probe(pdev, dma);
-	if (ret)
-		goto err;
-
-	ret = dmaenginem_async_device_register(ddev);
-	if (ret) {
-		dev_err(dev, "failed to register DMA device\n");
-		goto err;
-	}
-
-	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id,
-					 ddev);
-	if (ret) {
-		dev_err(dev, "failed to register DMA controller\n");
-		goto err;
-	}
-
-	platform_set_drvdata(pdev, dma);
-	dev_info(dev, "Loongson1 DMA driver registered\n");
-
-	return 0;
-
-err:
-	ls1x_dma_chan_remove(dma);
-
-	return ret;
-}
-
-static void ls1x_dma_remove(struct platform_device *pdev)
-{
-	struct ls1x_dma *dma = platform_get_drvdata(pdev);
-
-	of_dma_controller_free(pdev->dev.of_node);
-	ls1x_dma_chan_remove(dma);
-}
-
-static const struct of_device_id ls1x_dma_match[] = {
-	{ .compatible = "loongson,ls1b-apbdma" },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, ls1x_dma_match);
-
-static struct platform_driver ls1x_dma_driver = {
-	.probe = ls1x_dma_probe,
-	.remove = ls1x_dma_remove,
-	.driver = {
-		.name = KBUILD_MODNAME,
-		.of_match_table = ls1x_dma_match,
-	},
-};
-
-module_platform_driver(ls1x_dma_driver);
-
-MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>");
-MODULE_DESCRIPTION("Loongson-1 APB DMA Controller driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/dma/loongson2-apb-dma.c b/drivers/dma/loongson2-apb-dma.c
deleted file mode 100644
index b981475e6779b..0000000000000
--- a/drivers/dma/loongson2-apb-dma.c
+++ /dev/null
@@ -1,705 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for the Loongson-2 APB DMA Controller
- *
- * Copyright (C) 2017-2023 Loongson Corporation
- */
-
-#include <linux/clk.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmapool.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/io-64-nonatomic-lo-hi.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_dma.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include "dmaengine.h"
-#include "virt-dma.h"
-
-/* Global Configuration Register */
-#define LDMA_ORDER_ERG		0x0
-
-/* Bitfield definitions */
-
-/* Bitfields in Global Configuration Register */
-#define LDMA_64BIT_EN		BIT(0) /* 1: 64 bit support */
-#define LDMA_UNCOHERENT_EN	BIT(1) /* 0: cache, 1: uncache */
-#define LDMA_ASK_VALID		BIT(2)
-#define LDMA_START		BIT(3) /* DMA start operation */
-#define LDMA_STOP		BIT(4) /* DMA stop operation */
-#define LDMA_CONFIG_MASK	GENMASK_ULL(4, 0) /* DMA controller config bits mask */
-
-/* Bitfields in ndesc_addr field of HW descriptor */
-#define LDMA_DESC_EN		BIT(0) /*1: The next descriptor is valid */
-#define LDMA_DESC_ADDR_LOW	GENMASK(31, 1)
-
-/* Bitfields in cmd field of HW descriptor */
-#define LDMA_INT		BIT(1) /* Enable DMA interrupts */
-#define LDMA_DATA_DIRECTION	BIT(12) /* 1: write to device, 0: read from device */
-
-#define LDMA_SLAVE_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
-				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
-
-#define LDMA_MAX_TRANS_LEN	U32_MAX
-
-/*--  descriptors  -----------------------------------------------------*/
-
-/*
- * struct ls2x_dma_hw_desc - DMA HW descriptor
- * @ndesc_addr: the next descriptor low address.
- * @mem_addr: memory low address.
- * @apb_addr: device buffer address.
- * @len: length of a piece of carried content, in words.
- * @step_len: length between two moved memory data blocks.
- * @step_times: number of blocks to be carried in a single DMA operation.
- * @cmd: descriptor command or state.
- * @stats: DMA status.
- * @high_ndesc_addr: the next descriptor high address.
- * @high_mem_addr: memory high address.
- * @reserved: reserved
- */
-struct ls2x_dma_hw_desc {
-	u32 ndesc_addr;
-	u32 mem_addr;
-	u32 apb_addr;
-	u32 len;
-	u32 step_len;
-	u32 step_times;
-	u32 cmd;
-	u32 stats;
-	u32 high_ndesc_addr;
-	u32 high_mem_addr;
-	u32 reserved[2];
-} __packed;
-
-/*
- * struct ls2x_dma_sg - ls2x dma scatter gather entry
- * @hw: the pointer to DMA HW descriptor.
- * @llp: physical address of the DMA HW descriptor.
- * @phys: destination or source address(mem).
- * @len: number of Bytes to read.
- */
-struct ls2x_dma_sg {
-	struct ls2x_dma_hw_desc	*hw;
-	dma_addr_t		llp;
-	dma_addr_t		phys;
-	u32			len;
-};
-
-/*
- * struct ls2x_dma_desc - software descriptor
- * @vdesc: pointer to the virtual dma descriptor.
- * @cyclic: flag to dma cyclic
- * @burst_size: burst size of transaction, in words.
- * @desc_num: number of sg entries.
- * @direction: transfer direction, to or from device.
- * @status: dma controller status.
- * @sg: array of sgs.
- */
-struct ls2x_dma_desc {
-	struct virt_dma_desc		vdesc;
-	bool				cyclic;
-	size_t				burst_size;
-	u32				desc_num;
-	enum dma_transfer_direction	direction;
-	enum dma_status			status;
-	struct ls2x_dma_sg		sg[] __counted_by(desc_num);
-};
-
-/*--  Channels  --------------------------------------------------------*/
-
-/*
- * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel
- * @vchan: virtual dma channel entry.
- * @desc: pointer to the ls2x sw dma descriptor.
- * @pool: hw desc table
- * @irq: irq line
- * @sconfig: configuration for slave transfers, passed via .device_config
- */
-struct ls2x_dma_chan {
-	struct virt_dma_chan	vchan;
-	struct ls2x_dma_desc	*desc;
-	void			*pool;
-	int			irq;
-	struct dma_slave_config	sconfig;
-};
-
-/*--  Controller  ------------------------------------------------------*/
-
-/*
- * struct ls2x_dma_priv - LS2X APB DMAC specific information
- * @ddev: dmaengine dma_device object members
- * @dma_clk: DMAC clock source
- * @regs: memory mapped register base
- * @lchan: channel to store ls2x_dma_chan structures
- */
-struct ls2x_dma_priv {
-	struct dma_device	ddev;
-	struct clk		*dma_clk;
-	void __iomem		*regs;
-	struct ls2x_dma_chan	lchan;
-};
-
-/*--  Helper functions  ------------------------------------------------*/
-
-static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc)
-{
-	return container_of(vdesc, struct ls2x_dma_desc, vdesc);
-}
-
-static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan)
-{
-	return container_of(chan, struct ls2x_dma_chan, vchan.chan);
-}
-
-static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev)
-{
-	return container_of(ddev, struct ls2x_dma_priv, ddev);
-}
-
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
-static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan);
-	struct ls2x_dma_desc *desc = to_ldma_desc(vdesc);
-	int i;
-
-	for (i = 0; i < desc->desc_num; i++) {
-		if (desc->sg[i].hw)
-			dma_pool_free(lchan->pool, desc->sg[i].hw,
-				      desc->sg[i].llp);
-	}
-
-	kfree(desc);
-}
-
-static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd)
-{
-	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
-	u64 val;
-
-	val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK;
-	val |= LDMA_64BIT_EN | cmd;
-	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
-}
-
-static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan)
-{
-	struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device);
-	struct ls2x_dma_sg *ldma_sg;
-	struct virt_dma_desc *vdesc;
-	u64 val;
-
-	/* Get the next descriptor */
-	vdesc = vchan_next_desc(&lchan->vchan);
-	if (!vdesc) {
-		lchan->desc = NULL;
-		return;
-	}
-
-	list_del(&vdesc->node);
-	lchan->desc = to_ldma_desc(vdesc);
-	ldma_sg = &lchan->desc->sg[0];
-
-	/* Start DMA */
-	lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG);
-	val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START;
-	lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG);
-}
-
-static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan)
-{
-	u32 maxburst, buswidth;
-
-	/* Reject definitely invalid configurations */
-	if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) &&
-	    (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS))
-		return 0;
-
-	if (lchan->sconfig.direction == DMA_MEM_TO_DEV) {
-		maxburst = lchan->sconfig.dst_maxburst;
-		buswidth = lchan->sconfig.dst_addr_width;
-	} else {
-		maxburst = lchan->sconfig.src_maxburst;
-		buswidth = lchan->sconfig.src_addr_width;
-	}
-
-	/* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */
-	return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN;
-}
-
-static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index,
-			       struct ls2x_dma_desc *desc)
-{
-	struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index];
-	u32 num_segments, segment_size;
-
-	if (desc->direction == DMA_MEM_TO_DEV) {
-		ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION;
-		ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr;
-	} else {
-		ldma_sg->hw->cmd = LDMA_INT;
-		ldma_sg->hw->apb_addr = lchan->sconfig.src_addr;
-	}
-
-	ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys);
-	ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys);
-
-	/* Split into multiple equally sized segments if necessary */
-	num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size);
-	segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments);
-
-	/* Word count register takes input in words */
-	ldma_sg->hw->len = segment_size;
-	ldma_sg->hw->step_times = num_segments;
-	ldma_sg->hw->step_len = 0;
-
-	/* lets make a link list */
-	if (sg_index) {
-		desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN;
-		desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp);
-	}
-}
-
-/*--  DMA Engine API  --------------------------------------------------*/
-
-/*
- * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel
- * @chan: allocate descriptor resources for this channel
- *
- * return - the number of allocated descriptors
- */
-static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-
-	/* Create a pool of consistent memory blocks for hardware descriptors */
-	lchan->pool = dma_pool_create(dev_name(chan2dev(chan)),
-				      chan->device->dev, PAGE_SIZE,
-				      __alignof__(struct ls2x_dma_hw_desc), 0);
-	if (!lchan->pool) {
-		dev_err(chan2dev(chan), "No memory for descriptors\n");
-		return -ENOMEM;
-	}
-
-	return 1;
-}
-
-/*
- * ls2x_dma_free_chan_resources - free all channel resources
- * @chan: DMA channel
- */
-static void ls2x_dma_free_chan_resources(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-
-	vchan_free_chan_resources(to_virt_chan(chan));
-	dma_pool_destroy(lchan->pool);
-	lchan->pool = NULL;
-}
-
-/*
- * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
- * @chan: DMA channel
- * @sgl: scatterlist to transfer to/from
- * @sg_len: number of entries in @scatterlist
- * @direction: DMA direction
- * @flags: tx descriptor status flags
- * @context: transaction context (ignored)
- *
- * Return: Async transaction descriptor on success and NULL on failure
- */
-static struct dma_async_tx_descriptor *
-ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
-		       u32 sg_len, enum dma_transfer_direction direction,
-		       unsigned long flags, void *context)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	struct ls2x_dma_desc *desc;
-	struct scatterlist *sg;
-	size_t burst_size;
-	int i;
-
-	if (unlikely(!sg_len || !is_slave_direction(direction)))
-		return NULL;
-
-	burst_size = ls2x_dmac_detect_burst(lchan);
-	if (!burst_size)
-		return NULL;
-
-	desc = kzalloc_flex(*desc, sg, sg_len, GFP_NOWAIT);
-	if (!desc)
-		return NULL;
-
-	desc->desc_num = sg_len;
-	desc->direction = direction;
-	desc->burst_size = burst_size;
-
-	for_each_sg(sgl, sg, sg_len, i) {
-		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
-
-		/* Allocate DMA capable memory for hardware descriptor */
-		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
-		if (!ldma_sg->hw) {
-			desc->desc_num = i;
-			ls2x_dma_desc_free(&desc->vdesc);
-			return NULL;
-		}
-
-		ldma_sg->phys = sg_dma_address(sg);
-		ldma_sg->len = sg_dma_len(sg);
-
-		ls2x_dma_fill_desc(lchan, i, desc);
-	}
-
-	/* Setting the last descriptor enable bit */
-	desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN;
-	desc->status = DMA_IN_PROGRESS;
-
-	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
-}
-
-/*
- * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer
- * @chan: the DMA channel to prepare
- * @buf_addr: physical DMA address where the buffer starts
- * @buf_len: total number of bytes for the entire buffer
- * @period_len: number of bytes for each period
- * @direction: transfer direction, to or from device
- * @flags: tx descriptor status flags
- *
- * Return: Async transaction descriptor on success and NULL on failure
- */
-static struct dma_async_tx_descriptor *
-ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
-			 size_t period_len, enum dma_transfer_direction direction,
-			 unsigned long flags)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	struct ls2x_dma_desc *desc;
-	size_t burst_size;
-	u32 num_periods;
-	int i;
-
-	if (unlikely(!buf_len || !period_len))
-		return NULL;
-
-	if (unlikely(!is_slave_direction(direction)))
-		return NULL;
-
-	burst_size = ls2x_dmac_detect_burst(lchan);
-	if (!burst_size)
-		return NULL;
-
-	num_periods = buf_len / period_len;
-	desc = kzalloc_flex(*desc, sg, num_periods, GFP_NOWAIT);
-	if (!desc)
-		return NULL;
-
-	desc->desc_num = num_periods;
-	desc->direction = direction;
-	desc->burst_size = burst_size;
-
-	/* Build cyclic linked list */
-	for (i = 0; i < num_periods; i++) {
-		struct ls2x_dma_sg *ldma_sg = &desc->sg[i];
-
-		/* Allocate DMA capable memory for hardware descriptor */
-		ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp);
-		if (!ldma_sg->hw) {
-			desc->desc_num = i;
-			ls2x_dma_desc_free(&desc->vdesc);
-			return NULL;
-		}
-
-		ldma_sg->phys = buf_addr + period_len * i;
-		ldma_sg->len = period_len;
-
-		ls2x_dma_fill_desc(lchan, i, desc);
-	}
-
-	/* Lets make a cyclic list */
-	desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN;
-	desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp);
-	desc->cyclic = true;
-	desc->status = DMA_IN_PROGRESS;
-
-	return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
-}
-
-/*
- * ls2x_slave_config - set slave configuration for channel
- * @chan: dma channel
- * @cfg: slave configuration
- *
- * Sets slave configuration for channel
- */
-static int ls2x_dma_slave_config(struct dma_chan *chan,
-				 struct dma_slave_config *config)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-
-	memcpy(&lchan->sconfig, config, sizeof(*config));
-	return 0;
-}
-
-/*
- * ls2x_dma_issue_pending - push pending transactions to the hardware
- * @chan: channel
- *
- * When this function is called, all pending transactions are pushed to the
- * hardware and executed.
- */
-static void ls2x_dma_issue_pending(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	unsigned long flags;
-
-	spin_lock_irqsave(&lchan->vchan.lock, flags);
-	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc)
-		ls2x_dma_start_transfer(lchan);
-	spin_unlock_irqrestore(&lchan->vchan.lock, flags);
-}
-
-/*
- * ls2x_dma_terminate_all - terminate all transactions
- * @chan: channel
- *
- * Stops all DMA transactions.
- */
-static int ls2x_dma_terminate_all(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	unsigned long flags;
-	LIST_HEAD(head);
-
-	spin_lock_irqsave(&lchan->vchan.lock, flags);
-	/* Setting stop cmd */
-	ls2x_dma_write_cmd(lchan, LDMA_STOP);
-	if (lchan->desc) {
-		vchan_terminate_vdesc(&lchan->desc->vdesc);
-		lchan->desc = NULL;
-	}
-
-	vchan_get_all_descriptors(&lchan->vchan, &head);
-	spin_unlock_irqrestore(&lchan->vchan.lock, flags);
-
-	vchan_dma_desc_free_list(&lchan->vchan, &head);
-	return 0;
-}
-
-/*
- * ls2x_dma_synchronize - Synchronizes the termination of transfers to the
- * current context.
- * @chan: channel
- */
-static void ls2x_dma_synchronize(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-
-	vchan_synchronize(&lchan->vchan);
-}
-
-static int ls2x_dma_pause(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	unsigned long flags;
-
-	spin_lock_irqsave(&lchan->vchan.lock, flags);
-	if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) {
-		ls2x_dma_write_cmd(lchan, LDMA_STOP);
-		lchan->desc->status = DMA_PAUSED;
-	}
-	spin_unlock_irqrestore(&lchan->vchan.lock, flags);
-
-	return 0;
-}
-
-static int ls2x_dma_resume(struct dma_chan *chan)
-{
-	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
-	unsigned long flags;
-
-	spin_lock_irqsave(&lchan->vchan.lock, flags);
-	if (lchan->desc && lchan->desc->status == DMA_PAUSED) {
-		lchan->desc->status = DMA_IN_PROGRESS;
-		ls2x_dma_write_cmd(lchan, LDMA_START);
-	}
-	spin_unlock_irqrestore(&lchan->vchan.lock, flags);
-
-	return 0;
-}
-
-/*
- * ls2x_dma_isr - LS2X DMA Interrupt handler
- * @irq: IRQ number
- * @dev_id: Pointer to ls2x_dma_chan
- *
- * Return: IRQ_HANDLED/IRQ_NONE
- */
-static irqreturn_t ls2x_dma_isr(int irq, void *dev_id)
-{
-	struct ls2x_dma_chan *lchan = dev_id;
-	struct ls2x_dma_desc *desc;
-
-	spin_lock(&lchan->vchan.lock);
-	desc = lchan->desc;
-	if (desc) {
-		if (desc->cyclic) {
-			vchan_cyclic_callback(&desc->vdesc);
-		} else {
-			desc->status = DMA_COMPLETE;
-			vchan_cookie_complete(&desc->vdesc);
-			ls2x_dma_start_transfer(lchan);
-		}
-
-		/* ls2x_dma_start_transfer() updates lchan->desc */
-		if (!lchan->desc)
-			ls2x_dma_write_cmd(lchan, LDMA_STOP);
-	}
-	spin_unlock(&lchan->vchan.lock);
-
-	return IRQ_HANDLED;
-}
-
-static int ls2x_dma_chan_init(struct platform_device *pdev,
-			      struct ls2x_dma_priv *priv)
-{
-	struct ls2x_dma_chan *lchan = &priv->lchan;
-	struct device *dev = &pdev->dev;
-	int ret;
-
-	lchan->irq = platform_get_irq(pdev, 0);
-	if (lchan->irq < 0)
-		return lchan->irq;
-
-	ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING,
-			       dev_name(&pdev->dev), lchan);
-	if (ret)
-		return ret;
-
-	/* Initialize channels related values */
-	INIT_LIST_HEAD(&priv->ddev.channels);
-	lchan->vchan.desc_free = ls2x_dma_desc_free;
-	vchan_init(&lchan->vchan, &priv->ddev);
-
-	return 0;
-}
-
-/*
- * ls2x_dma_probe - Driver probe function
- * @pdev: Pointer to the platform_device structure
- *
- * Return: '0' on success and failure value on error
- */
-static int ls2x_dma_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ls2x_dma_priv *priv;
-	struct dma_device *ddev;
-	int ret;
-
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(priv->regs))
-		return dev_err_probe(dev, PTR_ERR(priv->regs),
-				     "devm_platform_ioremap_resource failed.\n");
-
-	priv->dma_clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(priv->dma_clk))
-		return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n");
-
-	ret = clk_prepare_enable(priv->dma_clk);
-	if (ret)
-		return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n");
-
-	ret = ls2x_dma_chan_init(pdev, priv);
-	if (ret)
-		goto disable_clk;
-
-	ddev = &priv->ddev;
-	ddev->dev = dev;
-	dma_cap_zero(ddev->cap_mask);
-	dma_cap_set(DMA_SLAVE, ddev->cap_mask);
-	dma_cap_set(DMA_CYCLIC, ddev->cap_mask);
-
-	ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources;
-	ddev->device_free_chan_resources = ls2x_dma_free_chan_resources;
-	ddev->device_tx_status = dma_cookie_status;
-	ddev->device_issue_pending = ls2x_dma_issue_pending;
-	ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg;
-	ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic;
-	ddev->device_config = ls2x_dma_slave_config;
-	ddev->device_terminate_all = ls2x_dma_terminate_all;
-	ddev->device_synchronize = ls2x_dma_synchronize;
-	ddev->device_pause = ls2x_dma_pause;
-	ddev->device_resume = ls2x_dma_resume;
-
-	ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS;
-	ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS;
-	ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
-
-	ret = dma_async_device_register(&priv->ddev);
-	if (ret < 0)
-		goto disable_clk;
-
-	ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv);
-	if (ret < 0)
-		goto unregister_dmac;
-
-	platform_set_drvdata(pdev, priv);
-
-	dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n");
-	return 0;
-
-unregister_dmac:
-	dma_async_device_unregister(&priv->ddev);
-disable_clk:
-	clk_disable_unprepare(priv->dma_clk);
-
-	return ret;
-}
-
-/*
- * ls2x_dma_remove - Driver remove function
- * @pdev: Pointer to the platform_device structure
- */
-static void ls2x_dma_remove(struct platform_device *pdev)
-{
-	struct ls2x_dma_priv *priv = platform_get_drvdata(pdev);
-
-	of_dma_controller_free(pdev->dev.of_node);
-	dma_async_device_unregister(&priv->ddev);
-	clk_disable_unprepare(priv->dma_clk);
-}
-
-static const struct of_device_id ls2x_dma_of_match_table[] = {
-	{ .compatible = "loongson,ls2k1000-apbdma" },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table);
-
-static struct platform_driver ls2x_dmac_driver = {
-	.probe		= ls2x_dma_probe,
-	.remove		= ls2x_dma_remove,
-	.driver = {
-		.name	= "ls2x-apbdma",
-		.of_match_table	= ls2x_dma_of_match_table,
-	},
-};
-module_platform_driver(ls2x_dmac_driver);
-
-MODULE_DESCRIPTION("Loongson-2 APB DMA Controller driver");
-MODULE_AUTHOR("Loongson Technology Corporation Limited");
-MODULE_LICENSE("GPL");
diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c
index cfb9962417ef6..7acb3d29dad30 100644
--- a/drivers/dma/mxs-dma.c
+++ b/drivers/dma/mxs-dma.c
@@ -744,20 +744,19 @@ static int mxs_dma_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	const struct mxs_dma_type *dma_type;
+	struct device *dev = &pdev->dev;
 	struct mxs_dma_engine *mxs_dma;
 	int ret, i;
 
-	mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL);
+	mxs_dma = devm_kzalloc(dev, sizeof(*mxs_dma), GFP_KERNEL);
 	if (!mxs_dma)
 		return -ENOMEM;
 
 	ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to read dma-channels\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to read dma-channels\n");
 
-	dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev);
+	dma_type = (struct mxs_dma_type *)of_device_get_match_data(dev);
 	mxs_dma->type = dma_type->type;
 	mxs_dma->dev_id = dma_type->id;
 
@@ -765,7 +764,7 @@ static int mxs_dma_probe(struct platform_device *pdev)
 	if (IS_ERR(mxs_dma->base))
 		return PTR_ERR(mxs_dma->base);
 
-	mxs_dma->clk = devm_clk_get(&pdev->dev, NULL);
+	mxs_dma->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(mxs_dma->clk))
 		return PTR_ERR(mxs_dma->clk);
 
@@ -795,10 +794,10 @@ static int mxs_dma_probe(struct platform_device *pdev)
 		return ret;
 
 	mxs_dma->pdev = pdev;
-	mxs_dma->dma_device.dev = &pdev->dev;
+	mxs_dma->dma_device.dev = dev;
 
 	/* mxs_dma gets 65535 bytes maximum sg size */
-	dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES);
+	dma_set_max_seg_size(dev, MAX_XFER_BYTES);
 
 	mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources;
 	mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources;
@@ -815,18 +814,15 @@ static int mxs_dma_probe(struct platform_device *pdev)
 	mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan;
 
 	ret = dmaenginem_async_device_register(&mxs_dma->dma_device);
-	if (ret) {
-		dev_err(mxs_dma->dma_device.dev, "unable to register\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "unable to register\n");
 
-	ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma);
-	if (ret) {
-		dev_err(mxs_dma->dma_device.dev,
-			"failed to register controller\n");
-	}
+	ret = devm_of_dma_controller_register(dev, np, mxs_dma_xlate, mxs_dma);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to register controller\n");
 
-	dev_info(mxs_dma->dma_device.dev, "initialized\n");
+	dev_info(dev, "initialized\n");
 
 	return 0;
 }
@@ -840,3 +836,6 @@ static struct platform_driver mxs_dma_driver = {
 };
 
 builtin_platform_driver(mxs_dma_driver);
+
+MODULE_DESCRIPTION("MXS DMA driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c
index d84ca551b2bf8..625ff29024de0 100644
--- a/drivers/dma/sh/rz-dmac.c
+++ b/drivers/dma/sh/rz-dmac.c
@@ -10,11 +10,13 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
 #include <linux/irqchip/irq-renesas-rzv2h.h>
+#include <linux/irqchip/irq-renesas-rzt2h.h>
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -94,9 +96,16 @@ struct rz_dmac_icu {
 	u8 dmac_index;
 };
 
+struct rz_dmac_info {
+	void (*icu_register_dma_req)(struct platform_device *icu_dev,
+				     u8 dmac_index, u8 dmac_channel, u16 req_no);
+	u16 default_dma_req_no;
+};
+
 struct rz_dmac {
 	struct dma_device engine;
 	struct rz_dmac_icu icu;
+	const struct rz_dmac_info *info;
 	struct device *dev;
 	struct reset_control *rstc;
 	void __iomem *base;
@@ -105,8 +114,6 @@ struct rz_dmac {
 	unsigned int n_channels;
 	struct rz_dmac_chan *channels;
 
-	bool has_icu;
-
 	DECLARE_BITMAP(modules, 1024);
 };
 
@@ -117,10 +124,12 @@ struct rz_dmac {
  * Registers
  */
 
+#define CRTB				0x0020
 #define CHSTAT				0x0024
 #define CHCTRL				0x0028
 #define CHCFG				0x002c
 #define NXLA				0x0038
+#define CRLA				0x003c
 
 #define DCTRL				0x0000
 
@@ -131,10 +140,12 @@ struct rz_dmac {
 #define CHANNEL_8_15_COMMON_BASE	0x0700
 
 #define CHSTAT_ER			BIT(4)
+#define CHSTAT_SUS			BIT(3)
 #define CHSTAT_EN			BIT(0)
 
 #define CHCTRL_CLRINTMSK		BIT(17)
 #define CHCTRL_CLRSUS			BIT(9)
+#define CHCTRL_SETSUS			BIT(8)
 #define CHCTRL_CLRTC			BIT(6)
 #define CHCTRL_CLREND			BIT(5)
 #define CHCTRL_CLRRQ			BIT(4)
@@ -265,15 +276,12 @@ static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
 {
 	struct dma_chan *chan = &channel->vc.chan;
 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
-	unsigned long flags;
 	u32 nxla;
 	u32 chctrl;
 	u32 chstat;
 
 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 
-	local_irq_save(flags);
-
 	rz_dmac_lmdesc_recycle(channel);
 
 	nxla = channel->lmdesc.base_dma +
@@ -288,21 +296,16 @@ static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
 		rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
 		rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
 	}
-
-	local_irq_restore(flags);
 }
 
 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
 {
 	struct dma_chan *chan = &channel->vc.chan;
 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
-	unsigned long flags;
 
 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 
-	local_irq_save(flags);
 	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
-	local_irq_restore(flags);
 }
 
 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
@@ -318,6 +321,16 @@ static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
 	rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
 }
 
+static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index,
+				   int req_no)
+{
+	if (dmac->info->icu_register_dma_req)
+		dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
+						 index, req_no);
+	else
+		rz_dmac_set_dmars_register(dmac, index, req_no);
+}
+
 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
 {
 	struct dma_chan *chan = &channel->vc.chan;
@@ -335,13 +348,7 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
 	lmdesc->chext = 0;
 	lmdesc->header = HEADER_LV;
 
-	if (dmac->has_icu) {
-		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
-					   channel->index,
-					   RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
-	} else {
-		rz_dmac_set_dmars_register(dmac, channel->index, 0);
-	}
+	rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
 
 	channel->chcfg = chcfg;
 	channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
@@ -392,12 +399,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
 
 	channel->lmdesc.tail = lmdesc;
 
-	if (dmac->has_icu) {
-		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
-					   channel->index, channel->mid_rid);
-	} else {
-		rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
-	}
+	rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
 
 	channel->chctrl = CHCTRL_SETEN;
 }
@@ -447,6 +449,7 @@ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
 		if (!desc)
 			break;
 
+		/* No need to lock. This is called only for the 1st client. */
 		list_add_tail(&desc->node, &channel->ld_free);
 		channel->descs_allocated++;
 	}
@@ -461,15 +464,12 @@ static void rz_dmac_free_chan_resources(struct dma_chan *chan)
 {
 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
-	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 	struct rz_dmac_desc *desc, *_desc;
 	unsigned long flags;
-	unsigned int i;
 
 	spin_lock_irqsave(&channel->vc.lock, flags);
 
-	for (i = 0; i < DMAC_NR_LMDESC; i++)
-		lmdesc[i].header = 0;
+	rz_lmdesc_setup(channel, channel->lmdesc.base);
 
 	rz_dmac_disable_hw(channel);
 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
@@ -502,18 +502,21 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
 		__func__, channel->index, &src, &dest, len);
 
-	if (list_empty(&channel->ld_free))
-		return NULL;
+	scoped_guard(spinlock_irqsave, &channel->vc.lock) {
+		if (list_empty(&channel->ld_free))
+			return NULL;
+
+		desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 
-	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
+		desc->type = RZ_DMAC_DESC_MEMCPY;
+		desc->src = src;
+		desc->dest = dest;
+		desc->len = len;
+		desc->direction = DMA_MEM_TO_MEM;
 
-	desc->type = RZ_DMAC_DESC_MEMCPY;
-	desc->src = src;
-	desc->dest = dest;
-	desc->len = len;
-	desc->direction = DMA_MEM_TO_MEM;
+		list_move_tail(channel->ld_free.next, &channel->ld_queue);
+	}
 
-	list_move_tail(channel->ld_free.next, &channel->ld_queue);
 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 }
 
@@ -529,42 +532,41 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	int dma_length = 0;
 	int i = 0;
 
-	if (list_empty(&channel->ld_free))
-		return NULL;
+	scoped_guard(spinlock_irqsave, &channel->vc.lock) {
+		if (list_empty(&channel->ld_free))
+			return NULL;
 
-	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
+		desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 
-	for_each_sg(sgl, sg, sg_len, i) {
-		dma_length += sg_dma_len(sg);
-	}
+		for_each_sg(sgl, sg, sg_len, i)
+			dma_length += sg_dma_len(sg);
 
-	desc->type = RZ_DMAC_DESC_SLAVE_SG;
-	desc->sg = sgl;
-	desc->sgcount = sg_len;
-	desc->len = dma_length;
-	desc->direction = direction;
+		desc->type = RZ_DMAC_DESC_SLAVE_SG;
+		desc->sg = sgl;
+		desc->sgcount = sg_len;
+		desc->len = dma_length;
+		desc->direction = direction;
 
-	if (direction == DMA_DEV_TO_MEM)
-		desc->src = channel->src_per_address;
-	else
-		desc->dest = channel->dst_per_address;
+		if (direction == DMA_DEV_TO_MEM)
+			desc->src = channel->src_per_address;
+		else
+			desc->dest = channel->dst_per_address;
+
+		list_move_tail(channel->ld_free.next, &channel->ld_queue);
+	}
 
-	list_move_tail(channel->ld_free.next, &channel->ld_queue);
 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 }
 
 static int rz_dmac_terminate_all(struct dma_chan *chan)
 {
 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
-	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 	unsigned long flags;
-	unsigned int i;
 	LIST_HEAD(head);
 
-	rz_dmac_disable_hw(channel);
 	spin_lock_irqsave(&channel->vc.lock, flags);
-	for (i = 0; i < DMAC_NR_LMDESC; i++)
-		lmdesc[i].header = 0;
+	rz_dmac_disable_hw(channel);
+	rz_lmdesc_setup(channel, channel->lmdesc.base);
 
 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
@@ -675,13 +677,185 @@ static void rz_dmac_device_synchronize(struct dma_chan *chan)
 	if (ret < 0)
 		dev_warn(dmac->dev, "DMA Timeout");
 
-	if (dmac->has_icu) {
-		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
-					   channel->index,
-					   RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
-	} else {
-		rz_dmac_set_dmars_register(dmac, channel->index, 0);
+	rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
+}
+
+static struct rz_lmdesc *
+rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc)
+{
+	struct rz_lmdesc *next = ++lmdesc;
+
+	if (next >= base + DMAC_NR_LMDESC)
+		next = base;
+
+	return next;
+}
+
+static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u32 crla)
+{
+	struct rz_lmdesc *lmdesc = channel->lmdesc.head;
+	struct dma_chan *chan = &channel->vc.chan;
+	struct rz_dmac *dmac = to_rz_dmac(chan->device);
+	u32 residue = 0, i = 0;
+
+	while (lmdesc->nxla != crla) {
+		lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
+		if (++i >= DMAC_NR_LMDESC)
+			return 0;
+	}
+
+	/* Calculate residue from next lmdesc to end of virtual desc */
+	while (lmdesc->chcfg & CHCFG_DEM) {
+		residue += lmdesc->tb;
+		lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
 	}
+
+	dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue);
+
+	return residue;
+}
+
+static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel,
+				    dma_cookie_t cookie)
+{
+	struct rz_dmac_desc *current_desc, *desc;
+	enum dma_status status;
+	u32 crla, crtb, i;
+
+	/* Get current processing virtual descriptor */
+	current_desc = list_first_entry(&channel->ld_active,
+					struct rz_dmac_desc, node);
+	if (!current_desc)
+		return 0;
+
+	/*
+	 * If the cookie corresponds to a descriptor that has been completed
+	 * there is no residue. The same check has already been performed by the
+	 * caller but without holding the channel lock, so the descriptor could
+	 * now be complete.
+	 */
+	status = dma_cookie_status(&channel->vc.chan, cookie, NULL);
+	if (status == DMA_COMPLETE)
+		return 0;
+
+	/*
+	 * If the cookie doesn't correspond to the currently processing virtual
+	 * descriptor then the descriptor hasn't been processed yet, and the
+	 * residue is equal to the full descriptor size. Also, a client driver
+	 * is possible to call this function before rz_dmac_irq_handler_thread()
+	 * runs. In this case, the running descriptor will be the next
+	 * descriptor, and will appear in the done list. So, if the argument
+	 * cookie matches the done list's cookie, we can assume the residue is
+	 * zero.
+	 */
+	if (cookie != current_desc->vd.tx.cookie) {
+		list_for_each_entry(desc, &channel->ld_free, node) {
+			if (cookie == desc->vd.tx.cookie)
+				return 0;
+		}
+
+		list_for_each_entry(desc, &channel->ld_queue, node) {
+			if (cookie == desc->vd.tx.cookie)
+				return desc->len;
+		}
+
+		list_for_each_entry(desc, &channel->ld_active, node) {
+			if (cookie == desc->vd.tx.cookie)
+				return desc->len;
+		}
+
+		/*
+		 * No descriptor found for the cookie, there's thus no residue.
+		 * This shouldn't happen if the calling driver passes a correct
+		 * cookie value.
+		 */
+		WARN(1, "No descriptor for cookie!");
+		return 0;
+	}
+
+	/*
+	 * We need to read two registers. Make sure the hardware does not move
+	 * to next lmdesc while reading the current lmdesc. Trying it 3 times
+	 * should be enough: initial read, retry, retry for the paranoid.
+	 */
+	for (i = 0; i < 3; i++) {
+		crla = rz_dmac_ch_readl(channel, CRLA, 1);
+		crtb = rz_dmac_ch_readl(channel, CRTB, 1);
+		/* Still the same? */
+		if (crla == rz_dmac_ch_readl(channel, CRLA, 1))
+			break;
+	}
+
+	WARN_ONCE(i >= 3, "residue might not be continuous!");
+
+	/*
+	 * Calculate number of bytes transferred in processing virtual descriptor.
+	 * One virtual descriptor can have many lmdesc.
+	 */
+	return crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, crla);
+}
+
+static enum dma_status rz_dmac_tx_status(struct dma_chan *chan,
+					 dma_cookie_t cookie,
+					 struct dma_tx_state *txstate)
+{
+	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+	enum dma_status status;
+	u32 residue;
+
+	status = dma_cookie_status(chan, cookie, txstate);
+	if (status == DMA_COMPLETE || !txstate)
+		return status;
+
+	scoped_guard(spinlock_irqsave, &channel->vc.lock) {
+		u32 val;
+
+		residue = rz_dmac_chan_get_residue(channel, cookie);
+
+		val = rz_dmac_ch_readl(channel, CHSTAT, 1);
+		if (val & CHSTAT_SUS)
+			status = DMA_PAUSED;
+	}
+
+	/* if there's no residue and no paused, the cookie is complete */
+	if (!residue && status != DMA_PAUSED)
+		return DMA_COMPLETE;
+
+	dma_set_residue(txstate, residue);
+
+	return status;
+}
+
+static int rz_dmac_device_pause(struct dma_chan *chan)
+{
+	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+	u32 val;
+
+	guard(spinlock_irqsave)(&channel->vc.lock);
+
+	val = rz_dmac_ch_readl(channel, CHSTAT, 1);
+	if (!(val & CHSTAT_EN))
+		return 0;
+
+	rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1);
+	return read_poll_timeout_atomic(rz_dmac_ch_readl, val,
+					(val & CHSTAT_SUS), 1, 1024,
+					false, channel, CHSTAT, 1);
+}
+
+static int rz_dmac_device_resume(struct dma_chan *chan)
+{
+	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+	u32 val;
+
+	guard(spinlock_irqsave)(&channel->vc.lock);
+
+	/* Do not check CHSTAT_SUS but rely on HW capabilities. */
+
+	rz_dmac_ch_writel(channel, CHCTRL_CLRSUS, CHCTRL, 1);
+	return read_poll_timeout_atomic(rz_dmac_ch_readl, val,
+					!(val & CHSTAT_SUS), 1, 1024,
+					false, channel, CHSTAT, 1);
 }
 
 /*
@@ -693,20 +867,23 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
 {
 	struct dma_chan *chan = &channel->vc.chan;
 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
-	u32 chstat, chctrl;
+	u32 chstat;
 
 	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 	if (chstat & CHSTAT_ER) {
 		dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
 			channel->index, chstat);
-		rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
-		goto done;
+
+		scoped_guard(spinlock_irqsave, &channel->vc.lock)
+			rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
+		return;
 	}
 
-	chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
-	rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
-done:
-	return;
+	/*
+	 * No need to lock. This just clears the END interrupt. Writing
+	 * zeros to CHCTRL is just ignored by HW.
+	 */
+	rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1);
 }
 
 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
@@ -870,14 +1047,13 @@ static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
 	uint32_t dmac_index;
 	int ret;
 
-	ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args);
-	if (ret == -ENOENT)
+	if (!dmac->info->icu_register_dma_req)
 		return 0;
+
+	ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args);
 	if (ret)
 		return ret;
 
-	dmac->has_icu = true;
-
 	dmac->icu.pdev = of_find_device_by_node(args.np);
 	of_node_put(args.np);
 	if (!dmac->icu.pdev) {
@@ -932,6 +1108,7 @@ static int rz_dmac_probe(struct platform_device *pdev)
 	if (!dmac)
 		return -ENOMEM;
 
+	dmac->info = device_get_match_data(&pdev->dev);
 	dmac->dev = &pdev->dev;
 	platform_set_drvdata(pdev, dmac);
 
@@ -949,23 +1126,22 @@ static int rz_dmac_probe(struct platform_device *pdev)
 	if (IS_ERR(dmac->base))
 		return PTR_ERR(dmac->base);
 
-	if (!dmac->has_icu) {
+	if (!dmac->info->icu_register_dma_req) {
 		dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
 		if (IS_ERR(dmac->ext_base))
 			return PTR_ERR(dmac->ext_base);
 	}
 
 	/* Register interrupt handler for error */
-	irq = platform_get_irq_byname(pdev, irqname);
-	if (irq < 0)
-		return irq;
-
-	ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
-			       irqname, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
-			irq, ret);
-		return ret;
+	irq = platform_get_irq_byname_optional(pdev, irqname);
+	if (irq > 0) {
+		ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
+				       irqname, NULL);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
+				irq, ret);
+			return ret;
+		}
 	}
 
 	/* Initialize the channels. */
@@ -1003,6 +1179,7 @@ static int rz_dmac_probe(struct platform_device *pdev)
 	engine = &dmac->engine;
 	dma_cap_set(DMA_SLAVE, engine->cap_mask);
 	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
+	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
 	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
 
@@ -1010,13 +1187,15 @@ static int rz_dmac_probe(struct platform_device *pdev)
 
 	engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
 	engine->device_free_chan_resources = rz_dmac_free_chan_resources;
-	engine->device_tx_status = dma_cookie_status;
+	engine->device_tx_status = rz_dmac_tx_status;
 	engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
 	engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
 	engine->device_config = rz_dmac_config;
 	engine->device_terminate_all = rz_dmac_terminate_all;
 	engine->device_issue_pending = rz_dmac_issue_pending;
 	engine->device_synchronize = rz_dmac_device_synchronize;
+	engine->device_pause = rz_dmac_device_pause;
+	engine->device_resume = rz_dmac_device_resume;
 
 	engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
 	dma_set_max_seg_size(engine->dev, U32_MAX);
@@ -1070,9 +1249,24 @@ static void rz_dmac_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 }
 
+static const struct rz_dmac_info rz_dmac_v2h_info = {
+	.icu_register_dma_req = rzv2h_icu_register_dma_req,
+	.default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT,
+};
+
+static const struct rz_dmac_info rz_dmac_t2h_info = {
+	.icu_register_dma_req = rzt2h_icu_register_dma_req,
+	.default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT,
+};
+
+static const struct rz_dmac_info rz_dmac_generic_info = {
+	.default_dma_req_no = 0,
+};
+
 static const struct of_device_id of_rz_dmac_match[] = {
-	{ .compatible = "renesas,r9a09g057-dmac", },
-	{ .compatible = "renesas,rz-dmac", },
+	{ .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info },
+	{ .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info },
+	{ .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c
new file mode 100644
index 0000000000000..3ef9286406159
--- /dev/null
+++ b/drivers/dma/switchtec_dma.c
@@ -0,0 +1,1437 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip Switchtec(tm) DMA Controller Driver
+ * Copyright (c) 2025, Kelvin Cao <kelvin.cao@microchip.com>
+ * Copyright (c) 2025, Microchip Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/circ_buf.h>
+#include <linux/dmaengine.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+
+#include "dmaengine.h"
+
+MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Kelvin Cao");
+
+#define	SWITCHTEC_DMAC_CHAN_CTRL_OFFSET		0x1000
+#define	SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET	0x160000
+
+#define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE		0x1000
+#define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE		0x80
+
+#define SWITCHTEC_REG_CAP		0x80
+#define SWITCHTEC_REG_CHAN_CNT		0x84
+#define SWITCHTEC_REG_TAG_LIMIT		0x90
+#define SWITCHTEC_REG_CHAN_STS_VEC	0x94
+#define SWITCHTEC_REG_SE_BUF_CNT	0x98
+#define SWITCHTEC_REG_SE_BUF_BASE	0x9a
+
+#define SWITCHTEC_DESC_MAX_SIZE		0x100000
+
+#define SWITCHTEC_CHAN_CTRL_PAUSE	BIT(0)
+#define SWITCHTEC_CHAN_CTRL_HALT	BIT(1)
+#define SWITCHTEC_CHAN_CTRL_RESET	BIT(2)
+#define SWITCHTEC_CHAN_CTRL_ERR_PAUSE	BIT(3)
+
+#define SWITCHTEC_CHAN_STS_PAUSED	BIT(9)
+#define SWITCHTEC_CHAN_STS_HALTED	BIT(10)
+#define SWITCHTEC_CHAN_STS_PAUSED_MASK	GENMASK(29, 13)
+
+#define SWITCHTEC_INVALID_HFID 0xffff
+
+#define SWITCHTEC_DMA_SQ_SIZE	SZ_32K
+#define SWITCHTEC_DMA_CQ_SIZE	SZ_32K
+
+#define SWITCHTEC_DMA_RING_SIZE	SZ_32K
+
+static const char * const channel_status_str[] = {
+	[13] = "received a VDM with length error status",
+	[14] = "received a VDM or Cpl with Unsupported Request error status",
+	[15] = "received a VDM or Cpl with Completion Abort error status",
+	[16] = "received a VDM with ECRC error status",
+	[17] = "received a VDM with EP error status",
+	[18] = "received a VDM with Reserved Cpl error status",
+	[19] = "received only part of split SE CplD",
+	[20] = "the ISP_DMAC detected a Completion Time Out",
+	[21] = "received a Cpl with Unsupported Request status",
+	[22] = "received a Cpl with Completion Abort status",
+	[23] = "received a Cpl with a reserved status",
+	[24] = "received a TLP with ECRC error status in its metadata",
+	[25] = "received a TLP with the EP bit set in the header",
+	[26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID",
+	[27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt",
+	[28] = "a reserved opcode was detected in an SE",
+	[29] = "received a SE Cpl with error status",
+};
+
+struct chan_hw_regs {
+	u16 cq_head;
+	u16 rsvd1;
+	u16 sq_tail;
+	u16 rsvd2;
+	u8 ctrl;
+	u8 rsvd3[3];
+	u16 status;
+	u16 rsvd4;
+};
+
+#define PERF_BURST_SCALE_MASK	GENMASK_U32(3,   2)
+#define PERF_MRRS_MASK		GENMASK_U32(6,   4)
+#define PERF_INTERVAL_MASK	GENMASK_U32(10,  8)
+#define PERF_BURST_SIZE_MASK	GENMASK_U32(14, 12)
+#define PERF_ARB_WEIGHT_MASK	GENMASK_U32(31, 24)
+
+#define SE_BUF_BASE_MASK	GENMASK_U32(10,  2)
+#define SE_BUF_LEN_MASK		GENMASK_U32(20, 12)
+#define SE_THRESH_MASK		GENMASK_U32(31, 23)
+
+#define SWITCHTEC_CHAN_ENABLE	BIT(1)
+
+struct chan_fw_regs {
+	u32 valid_en_se;
+	u32 cq_base_lo;
+	u32 cq_base_hi;
+	u16 cq_size;
+	u16 rsvd1;
+	u32 sq_base_lo;
+	u32 sq_base_hi;
+	u16 sq_size;
+	u16 rsvd2;
+	u32 int_vec;
+	u32 perf_cfg;
+	u32 rsvd3;
+	u32 perf_latency_selector;
+	u32 perf_fetched_se_cnt_lo;
+	u32 perf_fetched_se_cnt_hi;
+	u32 perf_byte_cnt_lo;
+	u32 perf_byte_cnt_hi;
+	u32 rsvd4;
+	u16 perf_se_pending;
+	u16 perf_se_buf_empty;
+	u32 perf_chan_idle;
+	u32 perf_lat_max;
+	u32 perf_lat_min;
+	u32 perf_lat_last;
+	u16 sq_current;
+	u16 sq_phase;
+	u16 cq_current;
+	u16 cq_phase;
+};
+
+struct switchtec_dma_chan {
+	struct switchtec_dma_dev *swdma_dev;
+	struct dma_chan dma_chan;
+	struct chan_hw_regs __iomem *mmio_chan_hw;
+	struct chan_fw_regs __iomem *mmio_chan_fw;
+
+	/* Serialize hardware control register access */
+	spinlock_t hw_ctrl_lock;
+
+	struct tasklet_struct desc_task;
+
+	/* Serialize descriptor preparation */
+	spinlock_t submit_lock;
+	bool ring_active;
+	int cid;
+
+	/* Serialize completion processing */
+	spinlock_t complete_lock;
+	bool comp_ring_active;
+
+	/* channel index and irq */
+	int index;
+	int irq;
+
+	/*
+	 * In driver context, head is advanced by producer while
+	 * tail is advanced by consumer.
+	 */
+
+	/* the head and tail for both desc_ring and hw_sq */
+	int head;
+	int tail;
+	int phase_tag;
+	struct switchtec_dma_hw_se_desc *hw_sq;
+	dma_addr_t dma_addr_sq;
+
+	/* the tail for hw_cq */
+	int cq_tail;
+	struct switchtec_dma_hw_ce *hw_cq;
+	dma_addr_t dma_addr_cq;
+
+	struct list_head list;
+
+	struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE];
+};
+
+struct switchtec_dma_dev {
+	struct dma_device dma_dev;
+	struct pci_dev __rcu *pdev;
+	void __iomem *bar;
+
+	struct switchtec_dma_chan **swdma_chans;
+	int chan_cnt;
+	int chan_status_irq;
+};
+
+enum chan_op {
+	ENABLE_CHAN,
+	DISABLE_CHAN,
+};
+
+enum switchtec_dma_opcode {
+	SWITCHTEC_DMA_OPC_MEMCPY = 0,
+	SWITCHTEC_DMA_OPC_RDIMM = 0x1,
+	SWITCHTEC_DMA_OPC_WRIMM = 0x2,
+	SWITCHTEC_DMA_OPC_RHI = 0x6,
+	SWITCHTEC_DMA_OPC_NOP = 0x7,
+};
+
+struct switchtec_dma_hw_se_desc {
+	u8 opc;
+	u8 ctrl;
+	__le16 tlp_setting;
+	__le16 rsvd1;
+	__le16 cid;
+	__le32 byte_cnt;
+	__le32 addr_lo; /* SADDR_LO/WIADDR_LO */
+	__le32 addr_hi; /* SADDR_HI/WIADDR_HI */
+	__le32 daddr_lo;
+	__le32 daddr_hi;
+	__le16 dfid;
+	__le16 sfid;
+};
+
+#define SWITCHTEC_SE_DFM		BIT(5)
+#define SWITCHTEC_SE_LIOF		BIT(6)
+#define SWITCHTEC_SE_BRR		BIT(7)
+#define SWITCHTEC_SE_CID_MASK		GENMASK(15, 0)
+
+#define SWITCHTEC_CE_SC_LEN_ERR		BIT(0)
+#define SWITCHTEC_CE_SC_UR		BIT(1)
+#define SWITCHTEC_CE_SC_CA		BIT(2)
+#define SWITCHTEC_CE_SC_RSVD_CPL	BIT(3)
+#define SWITCHTEC_CE_SC_ECRC_ERR	BIT(4)
+#define SWITCHTEC_CE_SC_EP_SET		BIT(5)
+#define SWITCHTEC_CE_SC_D_RD_CTO	BIT(8)
+#define SWITCHTEC_CE_SC_D_RIMM_UR	BIT(9)
+#define SWITCHTEC_CE_SC_D_RIMM_CA	BIT(10)
+#define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL	BIT(11)
+#define SWITCHTEC_CE_SC_D_ECRC		BIT(12)
+#define SWITCHTEC_CE_SC_D_EP_SET	BIT(13)
+#define SWITCHTEC_CE_SC_D_BAD_CONNID	BIT(14)
+#define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR	BIT(15)
+#define SWITCHTEC_CE_SC_D_INVD_CMD	BIT(16)
+#define SWITCHTEC_CE_SC_MASK		GENMASK(16, 0)
+
+struct switchtec_dma_hw_ce {
+	__le32 rdimm_cpl_dw0;
+	__le32 rdimm_cpl_dw1;
+	__le32 rsvd1;
+	__le32 cpl_byte_cnt;
+	__le16 sq_head;
+	__le16 rsvd2;
+	__le32 rsvd3;
+	__le32 sts_code;
+	__le16 cid;
+	__le16 phase_tag;
+};
+
+struct switchtec_dma_desc {
+	struct dma_async_tx_descriptor txd;
+	struct switchtec_dma_hw_se_desc *hw;
+	u32 orig_size;
+	bool completed;
+};
+
+static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask,
+				bool set)
+{
+	u32 status;
+
+	return readl_poll_timeout_atomic(&chan_hw->status, status,
+					 (set && (status & mask)) ||
+					 (!set && !(status & mask)),
+					 10, 100 * USEC_PER_MSEC);
+}
+
+static int halt_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+	int ret;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		ret = -ENODEV;
+		goto unlock_and_exit;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl);
+	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+
+unlock_and_exit:
+	rcu_read_unlock();
+	return ret;
+}
+
+static int unhalt_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+	u8 ctrl;
+	int ret;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		ret = -ENODEV;
+		goto unlock_and_exit;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	ctrl = readb(&chan_hw->ctrl);
+	ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT;
+	writeb(ctrl, &chan_hw->ctrl);
+	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+
+unlock_and_exit:
+	rcu_read_unlock();
+	return ret;
+}
+
+static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw)
+{
+	readl(&chan_hw->cq_head);
+}
+
+static int reset_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE,
+	       &chan_hw->ctrl);
+	flush_pci_write(chan_hw);
+
+	udelay(1000);
+
+	writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+	flush_pci_write(chan_hw);
+
+	rcu_read_unlock();
+	return 0;
+}
+
+static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+
+	flush_pci_write(chan_hw);
+
+	rcu_read_unlock();
+
+	/* wait 60ms to ensure no pending CEs */
+	mdelay(60);
+
+	return reset_channel(swdma_chan);
+}
+
+static int channel_op(struct switchtec_dma_chan *swdma_chan, int op)
+{
+	struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw;
+	struct pci_dev *pdev;
+	u32 valid_en_se;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+
+	valid_en_se = readl(&chan_fw->valid_en_se);
+	if (op == ENABLE_CHAN)
+		valid_en_se |= SWITCHTEC_CHAN_ENABLE;
+	else
+		valid_en_se &= ~SWITCHTEC_CHAN_ENABLE;
+
+	writel(valid_en_se, &chan_fw->valid_en_se);
+
+	rcu_read_unlock();
+	return 0;
+}
+
+static int enable_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	return channel_op(swdma_chan, ENABLE_CHAN);
+}
+
+static int disable_channel(struct switchtec_dma_chan *swdma_chan)
+{
+	return channel_op(swdma_chan, DISABLE_CHAN);
+}
+
+static void
+switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan)
+{
+	struct device *chan_dev = &swdma_chan->dma_chan.dev->device;
+	struct switchtec_dma_desc *desc;
+	struct switchtec_dma_hw_ce *ce;
+	struct dmaengine_result res;
+	int tail, cid, se_idx, i;
+	__le16 phase_tag;
+	u32 sts_code;
+	__le32 *p;
+
+	do {
+		spin_lock_bh(&swdma_chan->complete_lock);
+		if (!swdma_chan->comp_ring_active) {
+			spin_unlock_bh(&swdma_chan->complete_lock);
+			break;
+		}
+
+		ce = &swdma_chan->hw_cq[swdma_chan->cq_tail];
+		/*
+		 * phase_tag is updated by hardware, ensure the value is
+		 * not from the cache
+		 */
+		phase_tag = smp_load_acquire(&ce->phase_tag);
+		if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) {
+			spin_unlock_bh(&swdma_chan->complete_lock);
+			break;
+		}
+
+		cid = le16_to_cpu(ce->cid);
+		se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1);
+		desc = swdma_chan->desc_ring[se_idx];
+
+		tail = swdma_chan->tail;
+
+		res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt);
+
+		sts_code = le32_to_cpu(ce->sts_code);
+
+		if (!(sts_code & SWITCHTEC_CE_SC_MASK)) {
+			res.result = DMA_TRANS_NOERROR;
+		} else {
+			if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO)
+				res.result = DMA_TRANS_READ_FAILED;
+			else
+				res.result = DMA_TRANS_WRITE_FAILED;
+
+			dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid,
+				(u32)(sts_code & SWITCHTEC_CE_SC_MASK));
+
+			p = (__le32 *)ce;
+			for (i = 0; i < sizeof(*ce) / 4; i++) {
+				dev_err(chan_dev, "CE DW%d: 0x%08x\n", i,
+					le32_to_cpu(*p));
+				p++;
+			}
+		}
+
+		desc->completed = true;
+
+		swdma_chan->cq_tail++;
+		swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1;
+
+		rcu_read_lock();
+		if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) {
+			rcu_read_unlock();
+			spin_unlock_bh(&swdma_chan->complete_lock);
+			return;
+		}
+		writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head);
+		rcu_read_unlock();
+
+		if (swdma_chan->cq_tail == 0)
+			swdma_chan->phase_tag = !swdma_chan->phase_tag;
+
+		/*  Out of order CE */
+		if (se_idx != tail) {
+			spin_unlock_bh(&swdma_chan->complete_lock);
+			continue;
+		}
+
+		do {
+			dma_cookie_complete(&desc->txd);
+			dma_descriptor_unmap(&desc->txd);
+			dmaengine_desc_get_callback_invoke(&desc->txd, &res);
+			desc->txd.callback = NULL;
+			desc->txd.callback_result = NULL;
+			desc->completed = false;
+
+			tail++;
+			tail &= SWITCHTEC_DMA_SQ_SIZE - 1;
+
+			/*
+			 * Ensure the desc updates are visible before updating
+			 * the tail index
+			 */
+			smp_store_release(&swdma_chan->tail, tail);
+			desc = swdma_chan->desc_ring[swdma_chan->tail];
+			if (!desc->completed)
+				break;
+		} while (CIRC_CNT(READ_ONCE(swdma_chan->head), swdma_chan->tail,
+				  SWITCHTEC_DMA_SQ_SIZE));
+
+		spin_unlock_bh(&swdma_chan->complete_lock);
+	} while (1);
+}
+
+static void
+switchtec_dma_abort_desc(struct switchtec_dma_chan *swdma_chan, int force)
+{
+	struct switchtec_dma_desc *desc;
+	struct dmaengine_result res;
+
+	if (!force)
+		switchtec_dma_cleanup_completed(swdma_chan);
+
+	spin_lock_bh(&swdma_chan->complete_lock);
+
+	while (CIRC_CNT(swdma_chan->head, swdma_chan->tail,
+			SWITCHTEC_DMA_SQ_SIZE) >= 1) {
+		desc = swdma_chan->desc_ring[swdma_chan->tail];
+
+		res.residue = desc->orig_size;
+		res.result = DMA_TRANS_ABORTED;
+
+		dma_cookie_complete(&desc->txd);
+		dma_descriptor_unmap(&desc->txd);
+		if (!force)
+			dmaengine_desc_get_callback_invoke(&desc->txd, &res);
+		desc->txd.callback = NULL;
+		desc->txd.callback_result = NULL;
+
+		swdma_chan->tail++;
+		swdma_chan->tail &= SWITCHTEC_DMA_SQ_SIZE - 1;
+	}
+
+	spin_unlock_bh(&swdma_chan->complete_lock);
+}
+
+static void switchtec_dma_chan_stop(struct switchtec_dma_chan *swdma_chan)
+{
+	int rc;
+
+	rc = halt_channel(swdma_chan);
+	if (rc)
+		return;
+
+	rcu_read_lock();
+	if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) {
+		rcu_read_unlock();
+		return;
+	}
+
+	writel(0, &swdma_chan->mmio_chan_fw->sq_base_lo);
+	writel(0, &swdma_chan->mmio_chan_fw->sq_base_hi);
+	writel(0, &swdma_chan->mmio_chan_fw->cq_base_lo);
+	writel(0, &swdma_chan->mmio_chan_fw->cq_base_hi);
+
+	rcu_read_unlock();
+}
+
+static int switchtec_dma_terminate_all(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+
+	spin_lock_bh(&swdma_chan->complete_lock);
+	swdma_chan->comp_ring_active = false;
+	spin_unlock_bh(&swdma_chan->complete_lock);
+
+	return pause_reset_channel(swdma_chan);
+}
+
+static void switchtec_dma_synchronize(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+
+	int rc;
+
+	switchtec_dma_abort_desc(swdma_chan, 1);
+
+	rc = enable_channel(swdma_chan);
+	if (rc)
+		return;
+
+	rc = reset_channel(swdma_chan);
+	if (rc)
+		return;
+
+	rc = unhalt_channel(swdma_chan);
+	if (rc)
+		return;
+
+	spin_lock_bh(&swdma_chan->submit_lock);
+	swdma_chan->head = 0;
+	spin_unlock_bh(&swdma_chan->submit_lock);
+
+	spin_lock_bh(&swdma_chan->complete_lock);
+	swdma_chan->comp_ring_active = true;
+	swdma_chan->phase_tag = 0;
+	swdma_chan->tail = 0;
+	swdma_chan->cq_tail = 0;
+	swdma_chan->cid = 0;
+	dma_cookie_init(chan);
+	spin_unlock_bh(&swdma_chan->complete_lock);
+}
+
+static struct dma_async_tx_descriptor *
+switchtec_dma_prep_desc(struct dma_chan *c, u16 dst_fid, dma_addr_t dma_dst,
+			u16 src_fid, dma_addr_t dma_src, u64 data,
+			size_t len, unsigned long flags)
+	__acquires(swdma_chan->submit_lock)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(c, struct switchtec_dma_chan, dma_chan);
+	struct switchtec_dma_desc *desc;
+	int head, tail;
+
+	spin_lock_bh(&swdma_chan->submit_lock);
+
+	if (!swdma_chan->ring_active)
+		goto err_unlock;
+
+	tail = READ_ONCE(swdma_chan->tail);
+	head = swdma_chan->head;
+
+	if (!CIRC_SPACE(head, tail, SWITCHTEC_DMA_RING_SIZE))
+		goto err_unlock;
+
+	desc = swdma_chan->desc_ring[head];
+
+	if (src_fid != SWITCHTEC_INVALID_HFID &&
+	    dst_fid != SWITCHTEC_INVALID_HFID)
+		desc->hw->ctrl |= SWITCHTEC_SE_DFM;
+
+	if (flags & DMA_PREP_INTERRUPT)
+		desc->hw->ctrl |= SWITCHTEC_SE_LIOF;
+
+	if (flags & DMA_PREP_FENCE)
+		desc->hw->ctrl |= SWITCHTEC_SE_BRR;
+
+	desc->txd.flags = flags;
+
+	desc->completed = false;
+	desc->hw->opc = SWITCHTEC_DMA_OPC_MEMCPY;
+	desc->hw->addr_lo = cpu_to_le32(lower_32_bits(dma_src));
+	desc->hw->addr_hi = cpu_to_le32(upper_32_bits(dma_src));
+	desc->hw->daddr_lo = cpu_to_le32(lower_32_bits(dma_dst));
+	desc->hw->daddr_hi = cpu_to_le32(upper_32_bits(dma_dst));
+	desc->hw->byte_cnt = cpu_to_le32(len);
+	desc->hw->tlp_setting = 0;
+	desc->hw->dfid = cpu_to_le16(dst_fid);
+	desc->hw->sfid = cpu_to_le16(src_fid);
+	swdma_chan->cid &= SWITCHTEC_SE_CID_MASK;
+	desc->hw->cid = cpu_to_le16(swdma_chan->cid++);
+	desc->orig_size = len;
+
+	/* return with the lock held, it will be released in tx_submit */
+
+	return &desc->txd;
+
+err_unlock:
+	/*
+	 * Keep sparse happy by restoring an even lock count on
+	 * this lock.
+	 */
+	__acquire(swdma_chan->submit_lock);
+
+	spin_unlock_bh(&swdma_chan->submit_lock);
+	return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+switchtec_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dst,
+			  dma_addr_t dma_src, size_t len, unsigned long flags)
+	__acquires(swdma_chan->submit_lock)
+{
+	if (len > SWITCHTEC_DESC_MAX_SIZE) {
+		/*
+		 * Keep sparse happy by restoring an even lock count on
+		 * this lock.
+		 */
+		__acquire(swdma_chan->submit_lock);
+		return NULL;
+	}
+
+	return switchtec_dma_prep_desc(c, SWITCHTEC_INVALID_HFID, dma_dst,
+				       SWITCHTEC_INVALID_HFID, dma_src, 0, len,
+				       flags);
+}
+
+static dma_cookie_t
+switchtec_dma_tx_submit(struct dma_async_tx_descriptor *desc)
+	__releases(swdma_chan->submit_lock)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(desc->chan, struct switchtec_dma_chan, dma_chan);
+	dma_cookie_t cookie;
+	int head;
+
+	head = swdma_chan->head + 1;
+	head &= SWITCHTEC_DMA_RING_SIZE - 1;
+
+	/*
+	 * Ensure the desc updates are visible before updating the head index
+	 */
+	smp_store_release(&swdma_chan->head, head);
+
+	cookie = dma_cookie_assign(desc);
+
+	spin_unlock_bh(&swdma_chan->submit_lock);
+
+	return cookie;
+}
+
+static enum dma_status switchtec_dma_tx_status(struct dma_chan *chan,
+		dma_cookie_t cookie, struct dma_tx_state *txstate)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+	enum dma_status ret;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+	if (ret == DMA_COMPLETE)
+		return ret;
+
+	/*
+	 * For jobs where the interrupts are disabled, this is the only place
+	 * to process the completions returned by the hardware. Callers that
+	 * disable interrupts must call tx_status() to determine when a job
+	 * is done, so it is safe to process completions here. If a job has
+	 * interrupts enabled, then the completions will normally be processed
+	 * in the tasklet that is triggered by the interrupt and tx_status()
+	 * does not need to be called.
+	 */
+	switchtec_dma_cleanup_completed(swdma_chan);
+
+	return dma_cookie_status(chan, cookie, txstate);
+}
+
+static void switchtec_dma_issue_pending(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+	struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
+
+	/*
+	 * The sq_tail register is actually for the head of the
+	 * submisssion queue. Chip has the opposite define of head/tail
+	 * to the Linux kernel.
+	 */
+
+	rcu_read_lock();
+	if (!rcu_dereference(swdma_dev->pdev)) {
+		rcu_read_unlock();
+		return;
+	}
+
+	spin_lock_bh(&swdma_chan->submit_lock);
+	writew(swdma_chan->head, &swdma_chan->mmio_chan_hw->sq_tail);
+	spin_unlock_bh(&swdma_chan->submit_lock);
+
+	rcu_read_unlock();
+}
+
+static int switchtec_dma_pause(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+	int ret;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		ret = -ENODEV;
+		goto unlock_and_exit;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl);
+	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, true);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+
+unlock_and_exit:
+	rcu_read_unlock();
+	return ret;
+}
+
+static int switchtec_dma_resume(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
+	struct pci_dev *pdev;
+	int ret;
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
+	if (!pdev) {
+		ret = -ENODEV;
+		goto unlock_and_exit;
+	}
+
+	spin_lock(&swdma_chan->hw_ctrl_lock);
+	writeb(0, &chan_hw->ctrl);
+	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, false);
+	spin_unlock(&swdma_chan->hw_ctrl_lock);
+
+unlock_and_exit:
+	rcu_read_unlock();
+	return ret;
+}
+
+static void switchtec_dma_desc_task(unsigned long data)
+{
+	struct switchtec_dma_chan *swdma_chan = (void *)data;
+
+	switchtec_dma_cleanup_completed(swdma_chan);
+}
+
+static irqreturn_t switchtec_dma_isr(int irq, void *chan)
+{
+	struct switchtec_dma_chan *swdma_chan = chan;
+
+	if (swdma_chan->comp_ring_active)
+		tasklet_schedule(&swdma_chan->desc_task);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma)
+{
+	struct switchtec_dma_dev *swdma_dev = dma;
+	struct dma_device *dma_dev = &swdma_dev->dma_dev;
+	struct switchtec_dma_chan *swdma_chan;
+	struct chan_hw_regs __iomem *chan_hw;
+	struct device *chan_dev;
+	struct dma_chan *chan;
+	u32 chan_status;
+	int bit;
+
+	list_for_each_entry(chan, &dma_dev->channels, device_node) {
+		swdma_chan = container_of(chan, struct switchtec_dma_chan,
+					  dma_chan);
+		chan_dev = &swdma_chan->dma_chan.dev->device;
+		chan_hw = swdma_chan->mmio_chan_hw;
+
+		rcu_read_lock();
+		if (!rcu_dereference(swdma_dev->pdev)) {
+			rcu_read_unlock();
+			goto out;
+		}
+
+		chan_status = readl(&chan_hw->status);
+		chan_status &= SWITCHTEC_CHAN_STS_PAUSED_MASK;
+		rcu_read_unlock();
+
+		bit = ffs(chan_status);
+		if (!bit)
+			dev_dbg(chan_dev, "No pause bit set.\n");
+		else
+			dev_err(chan_dev, "Paused, %s\n",
+				channel_status_str[bit - 1]);
+	}
+
+out:
+	return IRQ_HANDLED;
+}
+
+static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan)
+{
+	struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
+	size_t size;
+	int i;
+
+	size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq);
+	if (swdma_chan->hw_sq)
+		dma_free_coherent(swdma_dev->dma_dev.dev, size,
+				  swdma_chan->hw_sq, swdma_chan->dma_addr_sq);
+
+	size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq);
+	if (swdma_chan->hw_cq)
+		dma_free_coherent(swdma_dev->dma_dev.dev, size,
+				  swdma_chan->hw_cq, swdma_chan->dma_addr_cq);
+
+	for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++)
+		kfree(swdma_chan->desc_ring[i]);
+}
+
+static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan)
+{
+	struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
+	struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw;
+	struct switchtec_dma_desc *desc;
+	struct pci_dev *pdev;
+	size_t size;
+	int rc, i;
+
+	swdma_chan->head = 0;
+	swdma_chan->tail = 0;
+	swdma_chan->cq_tail = 0;
+
+	size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq);
+	swdma_chan->hw_sq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size,
+					       &swdma_chan->dma_addr_sq,
+					       GFP_NOWAIT);
+	if (!swdma_chan->hw_sq) {
+		rc = -ENOMEM;
+		goto free_and_exit;
+	}
+
+	size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq);
+	swdma_chan->hw_cq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size,
+					       &swdma_chan->dma_addr_cq,
+					       GFP_NOWAIT);
+	if (!swdma_chan->hw_cq) {
+		rc = -ENOMEM;
+		goto free_and_exit;
+	}
+
+	/* reset host phase tag */
+	swdma_chan->phase_tag = 0;
+
+	for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) {
+		desc = kzalloc_obj(*desc, GFP_NOWAIT);
+		if (!desc) {
+			rc = -ENOMEM;
+			goto free_and_exit;
+		}
+
+		dma_async_tx_descriptor_init(&desc->txd, &swdma_chan->dma_chan);
+		desc->txd.tx_submit = switchtec_dma_tx_submit;
+		desc->hw = &swdma_chan->hw_sq[i];
+		desc->completed = true;
+
+		swdma_chan->desc_ring[i] = desc;
+	}
+
+	rcu_read_lock();
+	pdev = rcu_dereference(swdma_dev->pdev);
+	if (!pdev) {
+		rcu_read_unlock();
+		rc = -ENODEV;
+		goto free_and_exit;
+	}
+
+	/* set sq/cq */
+	writel(lower_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_lo);
+	writel(upper_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_hi);
+	writel(lower_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_lo);
+	writel(upper_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_hi);
+
+	writew(SWITCHTEC_DMA_SQ_SIZE, &swdma_chan->mmio_chan_fw->sq_size);
+	writew(SWITCHTEC_DMA_CQ_SIZE, &swdma_chan->mmio_chan_fw->cq_size);
+
+	rcu_read_unlock();
+	return 0;
+
+free_and_exit:
+	switchtec_dma_free_desc(swdma_chan);
+	return rc;
+}
+
+static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+	struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
+	u32 perf_cfg;
+	int rc;
+
+	rc = switchtec_dma_alloc_desc(swdma_chan);
+	if (rc)
+		return rc;
+
+	rc = enable_channel(swdma_chan);
+	if (rc)
+		return rc;
+
+	rc = reset_channel(swdma_chan);
+	if (rc)
+		return rc;
+
+	rc = unhalt_channel(swdma_chan);
+	if (rc)
+		return rc;
+
+	swdma_chan->ring_active = true;
+	swdma_chan->comp_ring_active = true;
+	swdma_chan->cid = 0;
+
+	dma_cookie_init(chan);
+
+	rcu_read_lock();
+	if (!rcu_dereference(swdma_dev->pdev)) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+
+	perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg);
+	rcu_read_unlock();
+
+	dev_dbg(&chan->dev->device, "Burst Size:  0x%x\n",
+		FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg));
+
+	dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n",
+		FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg));
+
+	dev_dbg(&chan->dev->device, "Interval:    0x%x\n",
+		FIELD_GET(PERF_INTERVAL_MASK, perf_cfg));
+
+	dev_dbg(&chan->dev->device, "Arb Weight:  0x%x\n",
+		FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg));
+
+	dev_dbg(&chan->dev->device, "MRRS:        0x%x\n",
+		FIELD_GET(PERF_MRRS_MASK, perf_cfg));
+
+	return SWITCHTEC_DMA_SQ_SIZE;
+}
+
+static void switchtec_dma_free_chan_resources(struct dma_chan *chan)
+{
+	struct switchtec_dma_chan *swdma_chan =
+		container_of(chan, struct switchtec_dma_chan, dma_chan);
+
+	spin_lock_bh(&swdma_chan->submit_lock);
+	swdma_chan->ring_active = false;
+	spin_unlock_bh(&swdma_chan->submit_lock);
+
+	spin_lock_bh(&swdma_chan->complete_lock);
+	swdma_chan->comp_ring_active = false;
+	spin_unlock_bh(&swdma_chan->complete_lock);
+
+	switchtec_dma_chan_stop(swdma_chan);
+	switchtec_dma_abort_desc(swdma_chan, 0);
+	switchtec_dma_free_desc(swdma_chan);
+
+	disable_channel(swdma_chan);
+}
+
+static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev,
+				   struct pci_dev *pdev, int i)
+{
+	struct dma_device *dma = &swdma_dev->dma_dev;
+	struct switchtec_dma_chan *swdma_chan;
+	u32 valid_en_se, thresh;
+	int se_buf_len, irq, rc;
+	struct dma_chan *chan;
+
+	swdma_chan = kzalloc_obj(*swdma_chan, GFP_KERNEL);
+	if (!swdma_chan)
+		return -ENOMEM;
+
+	swdma_chan->phase_tag = 0;
+	swdma_chan->index = i;
+	swdma_chan->swdma_dev = swdma_dev;
+
+	spin_lock_init(&swdma_chan->hw_ctrl_lock);
+	spin_lock_init(&swdma_chan->submit_lock);
+	spin_lock_init(&swdma_chan->complete_lock);
+	tasklet_init(&swdma_chan->desc_task, switchtec_dma_desc_task,
+		     (unsigned long)swdma_chan);
+
+	swdma_chan->mmio_chan_fw =
+		swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET +
+		i * SWITCHTEC_DMA_CHAN_FW_REGS_SIZE;
+	swdma_chan->mmio_chan_hw =
+		swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CTRL_OFFSET +
+		i * SWITCHTEC_DMA_CHAN_HW_REGS_SIZE;
+
+	swdma_dev->swdma_chans[i] = swdma_chan;
+
+	rc = pause_reset_channel(swdma_chan);
+	if (rc)
+		goto free_and_exit;
+
+	/* init perf tuner */
+	writel(FIELD_PREP(PERF_BURST_SCALE_MASK, 1) |
+	       FIELD_PREP(PERF_MRRS_MASK, 3) |
+	       FIELD_PREP(PERF_BURST_SIZE_MASK, 6) |
+	       FIELD_PREP(PERF_ARB_WEIGHT_MASK, 1),
+	       &swdma_chan->mmio_chan_fw->perf_cfg);
+
+	valid_en_se = readl(&swdma_chan->mmio_chan_fw->valid_en_se);
+
+	dev_dbg(&pdev->dev, "Channel %d: SE buffer base %d\n", i,
+		FIELD_GET(SE_BUF_BASE_MASK, valid_en_se));
+
+	se_buf_len = FIELD_GET(SE_BUF_LEN_MASK, valid_en_se);
+	dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len);
+
+	thresh = se_buf_len / 2;
+	valid_en_se |= FIELD_GET(SE_THRESH_MASK, thresh);
+	writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se);
+
+	/* request irqs */
+	irq = readl(&swdma_chan->mmio_chan_fw->int_vec);
+	dev_dbg(&pdev->dev, "Channel %d: CE irq vector %d\n", i, irq);
+
+	rc = pci_request_irq(pdev, irq, switchtec_dma_isr, NULL, swdma_chan,
+			     KBUILD_MODNAME);
+	if (rc)
+		goto free_and_exit;
+
+	swdma_chan->irq = irq;
+
+	chan = &swdma_chan->dma_chan;
+	chan->device = dma;
+	dma_cookie_init(chan);
+
+	list_add_tail(&chan->device_node, &dma->channels);
+
+	return 0;
+
+free_and_exit:
+	kfree(swdma_chan);
+	return rc;
+}
+
+static int switchtec_dma_chan_free(struct pci_dev *pdev,
+				   struct switchtec_dma_chan *swdma_chan)
+{
+	spin_lock_bh(&swdma_chan->submit_lock);
+	swdma_chan->ring_active = false;
+	spin_unlock_bh(&swdma_chan->submit_lock);
+
+	spin_lock_bh(&swdma_chan->complete_lock);
+	swdma_chan->comp_ring_active = false;
+	spin_unlock_bh(&swdma_chan->complete_lock);
+
+	pci_free_irq(pdev, swdma_chan->irq, swdma_chan);
+	tasklet_kill(&swdma_chan->desc_task);
+
+	switchtec_dma_chan_stop(swdma_chan);
+
+	return 0;
+}
+
+static int switchtec_dma_chans_release(struct pci_dev *pdev,
+				       struct switchtec_dma_dev *swdma_dev)
+{
+	int i;
+
+	for (i = 0; i < swdma_dev->chan_cnt; i++)
+		switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]);
+
+	return 0;
+}
+
+static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev,
+					 struct pci_dev *pdev, int chan_cnt)
+{
+	struct dma_device *dma = &swdma_dev->dma_dev;
+	int base, cnt, rc, i;
+
+	swdma_dev->swdma_chans = kcalloc(chan_cnt, sizeof(*swdma_dev->swdma_chans),
+					 GFP_KERNEL);
+
+	if (!swdma_dev->swdma_chans)
+		return -ENOMEM;
+
+	base = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_BASE);
+	cnt = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_CNT);
+
+	dev_dbg(&pdev->dev, "EP SE buffer base %d\n", base);
+	dev_dbg(&pdev->dev, "EP SE buffer count %d\n", cnt);
+
+	INIT_LIST_HEAD(&dma->channels);
+
+	for (i = 0; i < chan_cnt; i++) {
+		rc = switchtec_dma_chan_init(swdma_dev, pdev, i);
+		if (rc) {
+			dev_err(&pdev->dev, "Channel %d: init channel failed\n",
+				i);
+			chan_cnt = i;
+			goto err_exit;
+		}
+	}
+
+	return chan_cnt;
+
+err_exit:
+	for (i = 0; i < chan_cnt; i++)
+		switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]);
+
+	kfree(swdma_dev->swdma_chans);
+
+	return rc;
+}
+
+static void switchtec_dma_release(struct dma_device *dma_dev)
+{
+	struct switchtec_dma_dev *swdma_dev =
+		container_of(dma_dev, struct switchtec_dma_dev, dma_dev);
+	int i;
+
+	for (i = 0; i < swdma_dev->chan_cnt; i++)
+		kfree(swdma_dev->swdma_chans[i]);
+
+	kfree(swdma_dev->swdma_chans);
+
+	put_device(dma_dev->dev);
+	kfree(swdma_dev);
+}
+
+static int switchtec_dma_create(struct pci_dev *pdev)
+{
+	struct switchtec_dma_dev *swdma_dev;
+	int chan_cnt, nr_vecs, irq, rc;
+	struct dma_device *dma;
+	struct dma_chan *chan;
+
+	/*
+	 * Create the switchtec dma device
+	 */
+	swdma_dev = kzalloc_obj(*swdma_dev, GFP_KERNEL);
+	if (!swdma_dev)
+		return -ENOMEM;
+
+	swdma_dev->bar = ioremap(pci_resource_start(pdev, 0),
+				 pci_resource_len(pdev, 0));
+
+	RCU_INIT_POINTER(swdma_dev->pdev, pdev);
+
+	nr_vecs = pci_msix_vec_count(pdev);
+	rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
+	if (rc < 0)
+		goto err_exit;
+
+	irq = readw(swdma_dev->bar + SWITCHTEC_REG_CHAN_STS_VEC);
+	pci_dbg(pdev, "Channel pause irq vector %d\n", irq);
+
+	rc = pci_request_irq(pdev, irq, NULL, switchtec_dma_chan_status_isr,
+			     swdma_dev, KBUILD_MODNAME);
+	if (rc)
+		goto err_exit;
+
+	swdma_dev->chan_status_irq = irq;
+
+	chan_cnt = readl(swdma_dev->bar + SWITCHTEC_REG_CHAN_CNT);
+	if (!chan_cnt) {
+		pci_err(pdev, "No channel configured.\n");
+		rc = -ENXIO;
+		goto err_exit;
+	}
+
+	chan_cnt = switchtec_dma_chans_enumerate(swdma_dev, pdev, chan_cnt);
+	if (chan_cnt < 0) {
+		pci_err(pdev, "Failed to enumerate dma channels: %d\n",
+			chan_cnt);
+		rc = -ENXIO;
+		goto err_exit;
+	}
+
+	swdma_dev->chan_cnt = chan_cnt;
+
+	dma = &swdma_dev->dma_dev;
+	dma->copy_align = DMAENGINE_ALIGN_8_BYTES;
+	dma_cap_set(DMA_MEMCPY, dma->cap_mask);
+	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
+	dma->dev = get_device(&pdev->dev);
+
+	dma->device_alloc_chan_resources = switchtec_dma_alloc_chan_resources;
+	dma->device_free_chan_resources = switchtec_dma_free_chan_resources;
+	dma->device_prep_dma_memcpy = switchtec_dma_prep_memcpy;
+	dma->device_tx_status = switchtec_dma_tx_status;
+	dma->device_issue_pending = switchtec_dma_issue_pending;
+	dma->device_pause = switchtec_dma_pause;
+	dma->device_resume = switchtec_dma_resume;
+	dma->device_terminate_all = switchtec_dma_terminate_all;
+	dma->device_synchronize = switchtec_dma_synchronize;
+	dma->device_release = switchtec_dma_release;
+
+	rc = dma_async_device_register(dma);
+	if (rc) {
+		pci_err(pdev, "Failed to register dma device: %d\n", rc);
+		goto err_chans_release_exit;
+	}
+
+	pci_dbg(pdev, "Channel count: %d\n", chan_cnt);
+
+	list_for_each_entry(chan, &dma->channels, device_node)
+		pci_dbg(pdev, "%s\n", dma_chan_name(chan));
+
+	pci_set_drvdata(pdev, swdma_dev);
+
+	return 0;
+
+err_chans_release_exit:
+	switchtec_dma_chans_release(pdev, swdma_dev);
+
+err_exit:
+	if (swdma_dev->chan_status_irq)
+		free_irq(swdma_dev->chan_status_irq, swdma_dev);
+
+	iounmap(swdma_dev->bar);
+	kfree(swdma_dev);
+	return rc;
+}
+
+static int switchtec_dma_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *id)
+{
+	int rc;
+
+	rc = pci_enable_device(pdev);
+	if (rc)
+		return rc;
+
+	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+
+	rc = pci_request_mem_regions(pdev, KBUILD_MODNAME);
+	if (rc)
+		goto err_disable;
+
+	pci_set_master(pdev);
+
+	rc = switchtec_dma_create(pdev);
+	if (rc)
+		goto err_free;
+
+	return 0;
+
+err_free:
+	pci_free_irq_vectors(pdev);
+	pci_release_mem_regions(pdev);
+
+err_disable:
+	pci_disable_device(pdev);
+
+	return rc;
+}
+
+static void switchtec_dma_remove(struct pci_dev *pdev)
+{
+	struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev);
+
+	switchtec_dma_chans_release(pdev, swdma_dev);
+
+	rcu_assign_pointer(swdma_dev->pdev, NULL);
+	synchronize_rcu();
+
+	pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev);
+
+	pci_free_irq_vectors(pdev);
+
+	dma_async_device_unregister(&swdma_dev->dma_dev);
+
+	iounmap(swdma_dev->bar);
+	pci_release_mem_regions(pdev);
+	pci_disable_device(pdev);
+}
+
+/*
+ * Also use the class code to identify the devices, as some of the
+ * device IDs are also used for other devices with other classes by
+ * Microsemi.
+ */
+#define SW_ID(vendor_id, device_id) \
+	{ \
+		.vendor     = vendor_id, \
+		.device     = device_id, \
+		.subvendor  = PCI_ANY_ID, \
+		.subdevice  = PCI_ANY_ID, \
+		.class      = PCI_CLASS_SYSTEM_OTHER << 8, \
+		.class_mask = 0xffffffff, \
+	}
+
+static const struct pci_device_id switchtec_dma_pci_tbl[] = {
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4000), /* PFX 100XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4084), /* PFX 84XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4068), /* PFX 68XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4052), /* PFX 52XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4036), /* PFX 36XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4028), /* PFX 28XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4100), /* PSX 100XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4184), /* PSX 84XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4168), /* PSX 68XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4152), /* PSX 52XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4136), /* PSX 36XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4128), /* PSX 28XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4352), /* PFXA 52XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4336), /* PFXA 36XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4328), /* PFXA 28XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4452), /* PSXA 52XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4436), /* PSXA 36XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4428), /* PSXA 28XG4 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5000), /* PFX 100XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5084), /* PFX 84XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5068), /* PFX 68XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5052), /* PFX 52XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5036), /* PFX 36XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5028), /* PFX 28XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5100), /* PSX 100XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5184), /* PSX 84XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5168), /* PSX 68XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5152), /* PSX 52XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5136), /* PSX 36XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5128), /* PSX 28XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5300), /* PFXA 100XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5384), /* PFXA 84XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5368), /* PFXA 68XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5352), /* PFXA 52XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5336), /* PFXA 36XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5328), /* PFXA 28XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5400), /* PSXA 100XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5484), /* PSXA 84XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5468), /* PSXA 68XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5452), /* PSXA 52XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5436), /* PSXA 36XG5 */
+	SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5428), /* PSXA 28XG5 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1001), /* PCI1001 16XG4 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1002), /* PCI1002 16XG4 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1003), /* PCI1003 16XG4 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1004), /* PCI1004 16XG4 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1005), /* PCI1005 16XG4 */
+	SW_ID(PCI_VENDOR_ID_EFAR,      0x1006), /* PCI1006 16XG4 */
+	{0}
+};
+MODULE_DEVICE_TABLE(pci, switchtec_dma_pci_tbl);
+
+static struct pci_driver switchtec_dma_pci_driver = {
+	.name           = KBUILD_MODNAME,
+	.id_table       = switchtec_dma_pci_tbl,
+	.probe          = switchtec_dma_probe,
+	.remove		= switchtec_dma_remove,
+};
+module_pci_driver(switchtec_dma_pci_driver);
diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
index d02a4dac22919..90a22a730cc9c 100644
--- a/drivers/dma/xilinx/xdma.c
+++ b/drivers/dma/xilinx/xdma.c
@@ -61,6 +61,8 @@ struct xdma_desc_block {
  * @dir: Transferring direction of the channel
  * @cfg: Transferring config of the channel
  * @irq: IRQ assigned to the channel
+ * @last_interrupt: task for comppleting last interrupt
+ * @stop_requested: stop request flag
  */
 struct xdma_chan {
 	struct virt_dma_chan		vchan;
@@ -1234,8 +1236,8 @@ static int xdma_probe(struct platform_device *pdev)
 
 	xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
 					   &xdma_regmap_config);
-	if (!xdev->rmap) {
-		xdma_err(xdev, "config regmap failed: %d", ret);
+	if (IS_ERR(xdev->rmap)) {
+		xdma_err(xdev, "config regmap failed: %pe", xdev->rmap);
 		goto failed;
 	}
 	INIT_LIST_HEAD(&xdev->dma_dev.channels);
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index b53292e02448f..404235c173538 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -997,16 +997,16 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,
 					      struct xilinx_cdma_tx_segment,
 					      node);
 			cdma_hw = &cdma_seg->hw;
-			residue += (cdma_hw->control - cdma_hw->status) &
-				   chan->xdev->max_buffer_len;
+			residue += (cdma_hw->control & chan->xdev->max_buffer_len) -
+			           (cdma_hw->status & chan->xdev->max_buffer_len);
 		} else if (chan->xdev->dma_config->dmatype ==
 			   XDMA_TYPE_AXIDMA) {
 			axidma_seg = list_entry(entry,
 						struct xilinx_axidma_tx_segment,
 						node);
 			axidma_hw = &axidma_seg->hw;
-			residue += (axidma_hw->control - axidma_hw->status) &
-				   chan->xdev->max_buffer_len;
+			residue += (axidma_hw->control & chan->xdev->max_buffer_len) -
+			           (axidma_hw->status & chan->xdev->max_buffer_len);
 		} else {
 			aximcdma_seg =
 				list_entry(entry,
@@ -1014,8 +1014,8 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,
 					   node);
 			aximcdma_hw = &aximcdma_seg->hw;
 			residue +=
-				(aximcdma_hw->control - aximcdma_hw->status) &
-				chan->xdev->max_buffer_len;
+				(aximcdma_hw->control & chan->xdev->max_buffer_len) -
+				(aximcdma_hw->status & chan->xdev->max_buffer_len);
 		}
 	}
 
@@ -1235,14 +1235,6 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
 
 	dma_cookie_init(dchan);
 
-	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
-		/* For AXI DMA resetting once channel will reset the
-		 * other channel as well so enable the interrupts here.
-		 */
-		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
-			      XILINX_DMA_DMAXR_ALL_IRQ_MASK);
-	}
-
 	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)
 		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
 			     XILINX_CDMA_CR_SGMODE);
@@ -1564,8 +1556,29 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 	if (chan->err)
 		return;
 
-	if (list_empty(&chan->pending_list))
+	if (list_empty(&chan->pending_list)) {
+		if (chan->cyclic) {
+			struct xilinx_dma_tx_descriptor *desc;
+			struct list_head *entry;
+
+			desc = list_last_entry(&chan->done_list,
+					       struct xilinx_dma_tx_descriptor, node);
+			list_for_each(entry, &desc->segments) {
+				struct xilinx_axidma_tx_segment *axidma_seg;
+				struct xilinx_axidma_desc_hw *axidma_hw;
+				axidma_seg = list_entry(entry,
+							struct xilinx_axidma_tx_segment,
+							node);
+				axidma_hw = &axidma_seg->hw;
+				axidma_hw->status = 0;
+			}
+
+			list_splice_tail_init(&chan->done_list, &chan->active_list);
+			chan->desc_pendingcount = 0;
+			chan->idle = false;
+		}
 		return;
+	}
 
 	if (!chan->idle)
 		return;
@@ -1591,6 +1604,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 			     head_desc->async_tx.phys);
 	reg  &= ~XILINX_DMA_CR_DELAY_MAX;
 	reg  |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT;
+	reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK;
 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
 
 	xilinx_dma_start(chan);
@@ -3024,7 +3038,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
 		return -EINVAL;
 	}
 
-	xdev->common.directions |= chan->direction;
+	xdev->common.directions |= BIT(chan->direction);
 
 	/* Request the interrupt */
 	chan->irq = of_irq_get(node, chan->tdest);
@@ -3180,7 +3194,7 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 					= axivdma_clk_init;
 	struct device_node *node = pdev->dev.of_node;
 	struct xilinx_dma_device *xdev;
-	struct device_node *child, *np = pdev->dev.of_node;
+	struct device_node *np = pdev->dev.of_node;
 	u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width;
 	int i, err;
 
@@ -3320,12 +3334,10 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, xdev);
 
 	/* Initialize the channels */
-	for_each_child_of_node(node, child) {
+	for_each_child_of_node_scoped(node, child) {
 		err = xilinx_dma_child_probe(xdev, child);
-		if (err < 0) {
-			of_node_put(child);
+		if (err < 0)
 			goto error;
-		}
 	}
 
 	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
diff --git a/drivers/dpll/zl3073x/Makefile b/drivers/dpll/zl3073x/Makefile
index bd324c7fe7101..906ec3fbcc202 100644
--- a/drivers/dpll/zl3073x/Makefile
+++ b/drivers/dpll/zl3073x/Makefile
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
 obj-$(CONFIG_ZL3073X)		+= zl3073x.o
-zl3073x-objs			:= core.o devlink.o dpll.o flash.o fw.o	\
-				   out.o prop.o ref.o synth.o
+zl3073x-objs			:= chan.o core.o devlink.o dpll.o	\
+				   flash.o fw.o out.o prop.o ref.o synth.o
 
 obj-$(CONFIG_ZL3073X_I2C)	+= zl3073x_i2c.o
 zl3073x_i2c-objs		:= i2c.o
diff --git a/drivers/dpll/zl3073x/chan.c b/drivers/dpll/zl3073x/chan.c
new file mode 100644
index 0000000000000..2f48ca2391494
--- /dev/null
+++ b/drivers/dpll/zl3073x/chan.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/cleanup.h>
+#include <linux/dev_printk.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include "chan.h"
+#include "core.h"
+
+/**
+ * zl3073x_chan_state_update - update DPLL channel status from HW
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: DPLL channel index
+ *
+ * Return: 0 on success, <0 on error
+ */
+int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
+{
+	struct zl3073x_chan *chan = &zldev->chan[index];
+	int rc;
+
+	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MON_STATUS(index),
+			     &chan->mon_status);
+	if (rc)
+		return rc;
+
+	return zl3073x_read_u8(zldev, ZL_REG_DPLL_REFSEL_STATUS(index),
+			       &chan->refsel_status);
+}
+
+/**
+ * zl3073x_chan_state_fetch - fetch DPLL channel state from hardware
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: DPLL channel index to fetch state for
+ *
+ * Reads the mode_refsel register and reference priority registers for
+ * the given DPLL channel and stores the raw values for later use.
+ *
+ * Return: 0 on success, <0 on error
+ */
+int zl3073x_chan_state_fetch(struct zl3073x_dev *zldev, u8 index)
+{
+	struct zl3073x_chan *chan = &zldev->chan[index];
+	int rc, i;
+
+	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
+			     &chan->mode_refsel);
+	if (rc)
+		return rc;
+
+	dev_dbg(zldev->dev, "DPLL%u mode: %u, ref: %u\n", index,
+		zl3073x_chan_mode_get(chan), zl3073x_chan_ref_get(chan));
+
+	rc = zl3073x_chan_state_update(zldev, index);
+	if (rc)
+		return rc;
+
+	dev_dbg(zldev->dev,
+		"DPLL%u lock_state: %u, ho: %u, sel_state: %u, sel_ref: %u\n",
+		index, zl3073x_chan_lock_state_get(chan),
+		zl3073x_chan_is_ho_ready(chan) ? 1 : 0,
+		zl3073x_chan_refsel_state_get(chan),
+		zl3073x_chan_refsel_ref_get(chan));
+
+	guard(mutex)(&zldev->multiop_lock);
+
+	/* Read DPLL configuration from mailbox */
+	rc = zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_RD,
+			   ZL_REG_DPLL_MB_MASK, BIT(index));
+	if (rc)
+		return rc;
+
+	/* Read reference priority registers */
+	for (i = 0; i < ARRAY_SIZE(chan->ref_prio); i++) {
+		rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_REF_PRIO(i),
+				     &chan->ref_prio[i]);
+		if (rc)
+			return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * zl3073x_chan_state_get - get current DPLL channel state
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: DPLL channel index to get state for
+ *
+ * Return: pointer to given DPLL channel state
+ */
+const struct zl3073x_chan *zl3073x_chan_state_get(struct zl3073x_dev *zldev,
+						  u8 index)
+{
+	return &zldev->chan[index];
+}
+
+/**
+ * zl3073x_chan_state_set - commit DPLL channel state changes to hardware
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: DPLL channel index to set state for
+ * @chan: desired channel state
+ *
+ * Skips the HW write if the configuration is unchanged, and otherwise
+ * writes only the changed registers to hardware. The mode_refsel register
+ * is written directly, while the reference priority registers are written
+ * via the DPLL mailbox interface.
+ *
+ * Return: 0 on success, <0 on HW error
+ */
+int zl3073x_chan_state_set(struct zl3073x_dev *zldev, u8 index,
+			   const struct zl3073x_chan *chan)
+{
+	struct zl3073x_chan *dchan = &zldev->chan[index];
+	int rc, i;
+
+	/* Skip HW write if configuration hasn't changed */
+	if (!memcmp(&dchan->cfg, &chan->cfg, sizeof(chan->cfg)))
+		return 0;
+
+	/* Direct register write for mode_refsel */
+	if (dchan->mode_refsel != chan->mode_refsel) {
+		rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
+				      chan->mode_refsel);
+		if (rc)
+			return rc;
+		dchan->mode_refsel = chan->mode_refsel;
+	}
+
+	/* Mailbox write for ref_prio if changed */
+	if (!memcmp(dchan->ref_prio, chan->ref_prio, sizeof(chan->ref_prio))) {
+		dchan->cfg = chan->cfg;
+		return 0;
+	}
+
+	guard(mutex)(&zldev->multiop_lock);
+
+	/* Read DPLL configuration into mailbox */
+	rc = zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_RD,
+			   ZL_REG_DPLL_MB_MASK, BIT(index));
+	if (rc)
+		return rc;
+
+	/* Update changed ref_prio registers */
+	for (i = 0; i < ARRAY_SIZE(chan->ref_prio); i++) {
+		if (dchan->ref_prio[i] != chan->ref_prio[i]) {
+			rc = zl3073x_write_u8(zldev,
+					      ZL_REG_DPLL_REF_PRIO(i),
+					      chan->ref_prio[i]);
+			if (rc)
+				return rc;
+		}
+	}
+
+	/* Commit DPLL configuration */
+	rc = zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_WR,
+			   ZL_REG_DPLL_MB_MASK, BIT(index));
+	if (rc)
+		return rc;
+
+	/* After successful write store new state */
+	dchan->cfg = chan->cfg;
+
+	return 0;
+}
diff --git a/drivers/dpll/zl3073x/chan.h b/drivers/dpll/zl3073x/chan.h
new file mode 100644
index 0000000000000..e0f02d3432086
--- /dev/null
+++ b/drivers/dpll/zl3073x/chan.h
@@ -0,0 +1,179 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef _ZL3073X_CHAN_H
+#define _ZL3073X_CHAN_H
+
+#include <linux/bitfield.h>
+#include <linux/stddef.h>
+#include <linux/types.h>
+
+#include "regs.h"
+
+struct zl3073x_dev;
+
+/**
+ * struct zl3073x_chan - DPLL channel state
+ * @mode_refsel: mode and reference selection register value
+ * @ref_prio: reference priority registers (4 bits per ref, P/N packed)
+ * @mon_status: monitor status register value
+ * @refsel_status: reference selection status register value
+ */
+struct zl3073x_chan {
+	struct_group(cfg,
+		u8	mode_refsel;
+		u8	ref_prio[ZL3073X_NUM_REFS / 2];
+	);
+	struct_group(stat,
+		u8	mon_status;
+		u8	refsel_status;
+	);
+};
+
+int zl3073x_chan_state_fetch(struct zl3073x_dev *zldev, u8 index);
+const struct zl3073x_chan *zl3073x_chan_state_get(struct zl3073x_dev *zldev,
+						 u8 index);
+int zl3073x_chan_state_set(struct zl3073x_dev *zldev, u8 index,
+			   const struct zl3073x_chan *chan);
+
+int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index);
+
+/**
+ * zl3073x_chan_mode_get - get DPLL channel operating mode
+ * @chan: pointer to channel state
+ *
+ * Return: reference selection mode of the given DPLL channel
+ */
+static inline u8 zl3073x_chan_mode_get(const struct zl3073x_chan *chan)
+{
+	return FIELD_GET(ZL_DPLL_MODE_REFSEL_MODE, chan->mode_refsel);
+}
+
+/**
+ * zl3073x_chan_ref_get - get manually selected reference
+ * @chan: pointer to channel state
+ *
+ * Return: reference selected in forced reference lock mode
+ */
+static inline u8 zl3073x_chan_ref_get(const struct zl3073x_chan *chan)
+{
+	return FIELD_GET(ZL_DPLL_MODE_REFSEL_REF, chan->mode_refsel);
+}
+
+/**
+ * zl3073x_chan_mode_set - set DPLL channel operating mode
+ * @chan: pointer to channel state
+ * @mode: mode to set
+ */
+static inline void zl3073x_chan_mode_set(struct zl3073x_chan *chan, u8 mode)
+{
+	chan->mode_refsel &= ~ZL_DPLL_MODE_REFSEL_MODE;
+	chan->mode_refsel |= FIELD_PREP(ZL_DPLL_MODE_REFSEL_MODE, mode);
+}
+
+/**
+ * zl3073x_chan_ref_set - set manually selected reference
+ * @chan: pointer to channel state
+ * @ref: reference to set
+ */
+static inline void zl3073x_chan_ref_set(struct zl3073x_chan *chan, u8 ref)
+{
+	chan->mode_refsel &= ~ZL_DPLL_MODE_REFSEL_REF;
+	chan->mode_refsel |= FIELD_PREP(ZL_DPLL_MODE_REFSEL_REF, ref);
+}
+
+/**
+ * zl3073x_chan_ref_prio_get - get reference priority
+ * @chan: pointer to channel state
+ * @ref: input reference index
+ *
+ * Return: priority of the given reference <0, 15>
+ */
+static inline u8
+zl3073x_chan_ref_prio_get(const struct zl3073x_chan *chan, u8 ref)
+{
+	u8 val = chan->ref_prio[ref / 2];
+
+	if (!(ref & 1))
+		return FIELD_GET(ZL_DPLL_REF_PRIO_REF_P, val);
+	else
+		return FIELD_GET(ZL_DPLL_REF_PRIO_REF_N, val);
+}
+
+/**
+ * zl3073x_chan_ref_prio_set - set reference priority
+ * @chan: pointer to channel state
+ * @ref: input reference index
+ * @prio: priority to set
+ */
+static inline void
+zl3073x_chan_ref_prio_set(struct zl3073x_chan *chan, u8 ref, u8 prio)
+{
+	u8 *val = &chan->ref_prio[ref / 2];
+
+	if (!(ref & 1)) {
+		*val &= ~ZL_DPLL_REF_PRIO_REF_P;
+		*val |= FIELD_PREP(ZL_DPLL_REF_PRIO_REF_P, prio);
+	} else {
+		*val &= ~ZL_DPLL_REF_PRIO_REF_N;
+		*val |= FIELD_PREP(ZL_DPLL_REF_PRIO_REF_N, prio);
+	}
+}
+
+/**
+ * zl3073x_chan_ref_is_selectable - check if reference is selectable
+ * @chan: pointer to channel state
+ * @ref: input reference index
+ *
+ * Return: true if the reference priority is not NONE, false otherwise
+ */
+static inline bool
+zl3073x_chan_ref_is_selectable(const struct zl3073x_chan *chan, u8 ref)
+{
+	return zl3073x_chan_ref_prio_get(chan, ref) != ZL_DPLL_REF_PRIO_NONE;
+}
+
+/**
+ * zl3073x_chan_lock_state_get - get DPLL channel lock state
+ * @chan: pointer to channel state
+ *
+ * Return: lock state of the given DPLL channel
+ */
+static inline u8 zl3073x_chan_lock_state_get(const struct zl3073x_chan *chan)
+{
+	return FIELD_GET(ZL_DPLL_MON_STATUS_STATE, chan->mon_status);
+}
+
+/**
+ * zl3073x_chan_is_ho_ready - check if holdover is ready
+ * @chan: pointer to channel state
+ *
+ * Return: true if holdover is ready, false otherwise
+ */
+static inline bool zl3073x_chan_is_ho_ready(const struct zl3073x_chan *chan)
+{
+	return !!FIELD_GET(ZL_DPLL_MON_STATUS_HO_READY, chan->mon_status);
+}
+
+/**
+ * zl3073x_chan_refsel_state_get - get reference selection state
+ * @chan: pointer to channel state
+ *
+ * Return: reference selection state of the given DPLL channel
+ */
+static inline u8 zl3073x_chan_refsel_state_get(const struct zl3073x_chan *chan)
+{
+	return FIELD_GET(ZL_DPLL_REFSEL_STATUS_STATE, chan->refsel_status);
+}
+
+/**
+ * zl3073x_chan_refsel_ref_get - get currently selected reference in auto mode
+ * @chan: pointer to channel state
+ *
+ * Return: reference selected by the DPLL in automatic mode
+ */
+static inline u8 zl3073x_chan_refsel_ref_get(const struct zl3073x_chan *chan)
+{
+	return FIELD_GET(ZL_DPLL_REFSEL_STATUS_REFSEL, chan->refsel_status);
+}
+
+#endif /* _ZL3073X_CHAN_H */
diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c
index 37f3c33570eef..6363002d48d46 100644
--- a/drivers/dpll/zl3073x/core.c
+++ b/drivers/dpll/zl3073x/core.c
@@ -20,79 +20,30 @@
 #include "dpll.h"
 #include "regs.h"
 
-/* Chip IDs for zl30731 */
-static const u16 zl30731_ids[] = {
-	0x0E93,
-	0x1E93,
-	0x2E93,
+#define ZL_CHIP_INFO(_id, _nchannels, _flags)				\
+	{ .id = (_id), .num_channels = (_nchannels), .flags = (_flags) }
+
+static const struct zl3073x_chip_info zl3073x_chip_ids[] = {
+	ZL_CHIP_INFO(0x0E30, 2, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x0E93, 1, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x0E94, 2, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x0E95, 3, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x0E96, 4, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x0E97, 5, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x1E93, 1, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x1E94, 2, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x1E95, 3, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x1E96, 4, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x1E97, 5, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x1F60, 2, ZL3073X_FLAG_REF_PHASE_COMP_32),
+	ZL_CHIP_INFO(0x2E93, 1, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x2E94, 2, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x2E95, 3, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x2E96, 4, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x2E97, 5, ZL3073X_FLAG_DIE_TEMP),
+	ZL_CHIP_INFO(0x3FC4, 2, ZL3073X_FLAG_DIE_TEMP),
 };
 
-const struct zl3073x_chip_info zl30731_chip_info = {
-	.ids = zl30731_ids,
-	.num_ids = ARRAY_SIZE(zl30731_ids),
-	.num_channels = 1,
-};
-EXPORT_SYMBOL_NS_GPL(zl30731_chip_info, "ZL3073X");
-
-/* Chip IDs for zl30732 */
-static const u16 zl30732_ids[] = {
-	0x0E30,
-	0x0E94,
-	0x1E94,
-	0x1F60,
-	0x2E94,
-	0x3FC4,
-};
-
-const struct zl3073x_chip_info zl30732_chip_info = {
-	.ids = zl30732_ids,
-	.num_ids = ARRAY_SIZE(zl30732_ids),
-	.num_channels = 2,
-};
-EXPORT_SYMBOL_NS_GPL(zl30732_chip_info, "ZL3073X");
-
-/* Chip IDs for zl30733 */
-static const u16 zl30733_ids[] = {
-	0x0E95,
-	0x1E95,
-	0x2E95,
-};
-
-const struct zl3073x_chip_info zl30733_chip_info = {
-	.ids = zl30733_ids,
-	.num_ids = ARRAY_SIZE(zl30733_ids),
-	.num_channels = 3,
-};
-EXPORT_SYMBOL_NS_GPL(zl30733_chip_info, "ZL3073X");
-
-/* Chip IDs for zl30734 */
-static const u16 zl30734_ids[] = {
-	0x0E96,
-	0x1E96,
-	0x2E96,
-};
-
-const struct zl3073x_chip_info zl30734_chip_info = {
-	.ids = zl30734_ids,
-	.num_ids = ARRAY_SIZE(zl30734_ids),
-	.num_channels = 4,
-};
-EXPORT_SYMBOL_NS_GPL(zl30734_chip_info, "ZL3073X");
-
-/* Chip IDs for zl30735 */
-static const u16 zl30735_ids[] = {
-	0x0E97,
-	0x1E97,
-	0x2E97,
-};
-
-const struct zl3073x_chip_info zl30735_chip_info = {
-	.ids = zl30735_ids,
-	.num_ids = ARRAY_SIZE(zl30735_ids),
-	.num_channels = 5,
-};
-EXPORT_SYMBOL_NS_GPL(zl30735_chip_info, "ZL3073X");
-
 #define ZL_RANGE_OFFSET		0x80
 #define ZL_PAGE_SIZE		0x80
 #define ZL_NUM_PAGES		256
@@ -588,17 +539,26 @@ zl3073x_dev_state_fetch(struct zl3073x_dev *zldev)
 		}
 	}
 
+	for (i = 0; i < zldev->info->num_channels; i++) {
+		rc = zl3073x_chan_state_fetch(zldev, i);
+		if (rc) {
+			dev_err(zldev->dev,
+				"Failed to fetch channel state: %pe\n",
+				ERR_PTR(rc));
+			return rc;
+		}
+	}
+
 	return rc;
 }
 
 static void
-zl3073x_dev_ref_status_update(struct zl3073x_dev *zldev)
+zl3073x_dev_ref_states_update(struct zl3073x_dev *zldev)
 {
 	int i, rc;
 
 	for (i = 0; i < ZL3073X_NUM_REFS; i++) {
-		rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(i),
-				     &zldev->ref[i].mon_status);
+		rc = zl3073x_ref_state_update(zldev, i);
 		if (rc)
 			dev_warn(zldev->dev,
 				 "Failed to get REF%u status: %pe\n", i,
@@ -606,6 +566,20 @@ zl3073x_dev_ref_status_update(struct zl3073x_dev *zldev)
 	}
 }
 
+static void
+zl3073x_dev_chan_states_update(struct zl3073x_dev *zldev)
+{
+	int i, rc;
+
+	for (i = 0; i < zldev->info->num_channels; i++) {
+		rc = zl3073x_chan_state_update(zldev, i);
+		if (rc)
+			dev_warn(zldev->dev,
+				 "Failed to get DPLL%u state: %pe\n", i,
+				 ERR_PTR(rc));
+	}
+}
+
 /**
  * zl3073x_ref_phase_offsets_update - update reference phase offsets
  * @zldev: pointer to zl3073x_dev structure
@@ -728,8 +702,11 @@ zl3073x_dev_periodic_work(struct kthread_work *work)
 	struct zl3073x_dpll *zldpll;
 	int rc;
 
-	/* Update input references status */
-	zl3073x_dev_ref_status_update(zldev);
+	/* Update input references' states */
+	zl3073x_dev_ref_states_update(zldev);
+
+	/* Update DPLL channels' states */
+	zl3073x_dev_chan_states_update(zldev);
 
 	/* Update DPLL-to-connected-ref phase offsets registers */
 	rc = zl3073x_ref_phase_offsets_update(zldev, -1);
@@ -942,7 +919,7 @@ static void zl3073x_dev_dpll_fini(void *ptr)
 }
 
 static int
-zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls)
+zl3073x_devm_dpll_init(struct zl3073x_dev *zldev)
 {
 	struct kthread_worker *kworker;
 	struct zl3073x_dpll *zldpll;
@@ -952,7 +929,7 @@ zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls)
 	INIT_LIST_HEAD(&zldev->dplls);
 
 	/* Allocate all DPLLs */
-	for (i = 0; i < num_dplls; i++) {
+	for (i = 0; i < zldev->info->num_channels; i++) {
 		zldpll = zl3073x_dpll_alloc(zldev, i);
 		if (IS_ERR(zldpll)) {
 			dev_err_probe(zldev->dev, PTR_ERR(zldpll),
@@ -992,14 +969,12 @@ zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls)
 /**
  * zl3073x_dev_probe - initialize zl3073x device
  * @zldev: pointer to zl3073x device
- * @chip_info: chip info based on compatible
  *
  * Common initialization of zl3073x device structure.
  *
  * Returns: 0 on success, <0 on error
  */
-int zl3073x_dev_probe(struct zl3073x_dev *zldev,
-		      const struct zl3073x_chip_info *chip_info)
+int zl3073x_dev_probe(struct zl3073x_dev *zldev)
 {
 	u16 id, revision, fw_ver;
 	unsigned int i;
@@ -1011,18 +986,17 @@ int zl3073x_dev_probe(struct zl3073x_dev *zldev,
 	if (rc)
 		return rc;
 
-	/* Check it matches */
-	for (i = 0; i < chip_info->num_ids; i++) {
-		if (id == chip_info->ids[i])
+	/* Detect chip variant */
+	for (i = 0; i < ARRAY_SIZE(zl3073x_chip_ids); i++) {
+		if (zl3073x_chip_ids[i].id == id)
 			break;
 	}
 
-	if (i == chip_info->num_ids) {
+	if (i == ARRAY_SIZE(zl3073x_chip_ids))
 		return dev_err_probe(zldev->dev, -ENODEV,
-				     "Unknown or non-match chip ID: 0x%0x\n",
-				     id);
-	}
-	zldev->chip_id = id;
+				     "Unknown chip ID: 0x%04x\n", id);
+
+	zldev->info = &zl3073x_chip_ids[i];
 
 	/* Read revision, firmware version and custom config version */
 	rc = zl3073x_read_u16(zldev, ZL_REG_REVISION, &revision);
@@ -1061,7 +1035,7 @@ int zl3073x_dev_probe(struct zl3073x_dev *zldev,
 				     "Failed to initialize mutex\n");
 
 	/* Register DPLL channels */
-	rc = zl3073x_devm_dpll_init(zldev, chip_info->num_channels);
+	rc = zl3073x_devm_dpll_init(zldev);
 	if (rc)
 		return rc;
 
diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h
index fd2af3c62a7d5..99440620407da 100644
--- a/drivers/dpll/zl3073x/core.h
+++ b/drivers/dpll/zl3073x/core.h
@@ -9,6 +9,7 @@
 #include <linux/mutex.h>
 #include <linux/types.h>
 
+#include "chan.h"
 #include "out.h"
 #include "ref.h"
 #include "regs.h"
@@ -18,27 +19,39 @@ struct device;
 struct regmap;
 struct zl3073x_dpll;
 
-/*
- * Hardware limits for ZL3073x chip family
+
+enum zl3073x_flags {
+	ZL3073X_FLAG_REF_PHASE_COMP_32_BIT,
+	ZL3073X_FLAG_DIE_TEMP_BIT,
+	ZL3073X_FLAGS_NBITS /* must be last */
+};
+
+#define __ZL3073X_FLAG(name)	BIT(ZL3073X_FLAG_ ## name ## _BIT)
+#define ZL3073X_FLAG_REF_PHASE_COMP_32	__ZL3073X_FLAG(REF_PHASE_COMP_32)
+#define ZL3073X_FLAG_DIE_TEMP		__ZL3073X_FLAG(DIE_TEMP)
+
+/**
+ * struct zl3073x_chip_info - chip variant identification
+ * @id: chip ID
+ * @num_channels: number of DPLL channels supported by this variant
+ * @flags: chip variant flags
  */
-#define ZL3073X_MAX_CHANNELS	5
-#define ZL3073X_NUM_REFS	10
-#define ZL3073X_NUM_OUTS	10
-#define ZL3073X_NUM_SYNTHS	5
-#define ZL3073X_NUM_INPUT_PINS	ZL3073X_NUM_REFS
-#define ZL3073X_NUM_OUTPUT_PINS	(ZL3073X_NUM_OUTS * 2)
-#define ZL3073X_NUM_PINS	(ZL3073X_NUM_INPUT_PINS + \
-				 ZL3073X_NUM_OUTPUT_PINS)
+struct zl3073x_chip_info {
+	u16		id;
+	u8		num_channels;
+	unsigned long	flags;
+};
 
 /**
  * struct zl3073x_dev - zl3073x device
  * @dev: pointer to device
  * @regmap: regmap to access device registers
+ * @info: detected chip info
  * @multiop_lock: to serialize multiple register operations
- * @chip_id: chip ID read from hardware
  * @ref: array of input references' invariants
  * @out: array of outs' invariants
  * @synth: array of synths' invariants
+ * @chan: array of DPLL channels' state
  * @dplls: list of DPLLs
  * @kworker: thread for periodic work
  * @work: periodic work
@@ -46,15 +59,16 @@ struct zl3073x_dpll;
  * @phase_avg_factor: phase offset measurement averaging factor
  */
 struct zl3073x_dev {
-	struct device		*dev;
-	struct regmap		*regmap;
-	struct mutex		multiop_lock;
-	u16			chip_id;
+	struct device			*dev;
+	struct regmap			*regmap;
+	const struct zl3073x_chip_info	*info;
+	struct mutex			multiop_lock;
 
 	/* Invariants */
 	struct zl3073x_ref	ref[ZL3073X_NUM_REFS];
 	struct zl3073x_out	out[ZL3073X_NUM_OUTS];
 	struct zl3073x_synth	synth[ZL3073X_NUM_SYNTHS];
+	struct zl3073x_chan	chan[ZL3073X_MAX_CHANNELS];
 
 	/* DPLL channels */
 	struct list_head	dplls;
@@ -68,22 +82,10 @@ struct zl3073x_dev {
 	u8			phase_avg_factor;
 };
 
-struct zl3073x_chip_info {
-	const u16	*ids;
-	size_t		num_ids;
-	int		num_channels;
-};
-
-extern const struct zl3073x_chip_info zl30731_chip_info;
-extern const struct zl3073x_chip_info zl30732_chip_info;
-extern const struct zl3073x_chip_info zl30733_chip_info;
-extern const struct zl3073x_chip_info zl30734_chip_info;
-extern const struct zl3073x_chip_info zl30735_chip_info;
 extern const struct regmap_config zl3073x_regmap_config;
 
 struct zl3073x_dev *zl3073x_devm_alloc(struct device *dev);
-int zl3073x_dev_probe(struct zl3073x_dev *zldev,
-		      const struct zl3073x_chip_info *chip_info);
+int zl3073x_dev_probe(struct zl3073x_dev *zldev);
 
 int zl3073x_dev_start(struct zl3073x_dev *zldev, bool full);
 void zl3073x_dev_stop(struct zl3073x_dev *zldev);
@@ -158,18 +160,7 @@ int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel);
 static inline bool
 zl3073x_dev_is_ref_phase_comp_32bit(struct zl3073x_dev *zldev)
 {
-	switch (zldev->chip_id) {
-	case 0x0E30:
-	case 0x0E93:
-	case 0x0E94:
-	case 0x0E95:
-	case 0x0E96:
-	case 0x0E97:
-	case 0x1F60:
-		return true;
-	default:
-		return false;
-	}
+	return zldev->info->flags & ZL3073X_FLAG_REF_PHASE_COMP_32;
 }
 
 static inline bool
diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c
index aaa14ea5e670f..a29f606318f6d 100644
--- a/drivers/dpll/zl3073x/dpll.c
+++ b/drivers/dpll/zl3073x/dpll.c
@@ -34,7 +34,6 @@
  * @dir: pin direction
  * @id: pin id
  * @prio: pin priority <0, 14>
- * @selectable: pin is selectable in automatic mode
  * @esync_control: embedded sync is controllable
  * @phase_gran: phase adjustment granularity
  * @pin_state: last saved pin state
@@ -50,7 +49,6 @@ struct zl3073x_dpll_pin {
 	enum dpll_pin_direction	dir;
 	u8			id;
 	u8			prio;
-	bool			selectable;
 	bool			esync_control;
 	s32			phase_gran;
 	enum dpll_pin_state	pin_state;
@@ -245,156 +243,27 @@ zl3073x_dpll_input_pin_frequency_set(const struct dpll_pin *dpll_pin,
 	return zl3073x_ref_state_set(zldev, ref_id, &ref);
 }
 
-/**
- * zl3073x_dpll_selected_ref_get - get currently selected reference
- * @zldpll: pointer to zl3073x_dpll
- * @ref: place to store selected reference
- *
- * Check for currently selected reference the DPLL should be locked to
- * and stores its index to given @ref.
- *
- * Return: 0 on success, <0 on error
- */
-static int
-zl3073x_dpll_selected_ref_get(struct zl3073x_dpll *zldpll, u8 *ref)
-{
-	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 state, value;
-	int rc;
-
-	switch (zldpll->refsel_mode) {
-	case ZL_DPLL_MODE_REFSEL_MODE_AUTO:
-		/* For automatic mode read refsel_status register */
-		rc = zl3073x_read_u8(zldev,
-				     ZL_REG_DPLL_REFSEL_STATUS(zldpll->id),
-				     &value);
-		if (rc)
-			return rc;
-
-		/* Extract reference state */
-		state = FIELD_GET(ZL_DPLL_REFSEL_STATUS_STATE, value);
-
-		/* Return the reference only if the DPLL is locked to it */
-		if (state == ZL_DPLL_REFSEL_STATUS_STATE_LOCK)
-			*ref = FIELD_GET(ZL_DPLL_REFSEL_STATUS_REFSEL, value);
-		else
-			*ref = ZL3073X_DPLL_REF_NONE;
-		break;
-	case ZL_DPLL_MODE_REFSEL_MODE_REFLOCK:
-		/* For manual mode return stored value */
-		*ref = zldpll->forced_ref;
-		break;
-	default:
-		/* For other modes like NCO, freerun... there is no input ref */
-		*ref = ZL3073X_DPLL_REF_NONE;
-		break;
-	}
-
-	return 0;
-}
-
-/**
- * zl3073x_dpll_selected_ref_set - select reference in manual mode
- * @zldpll: pointer to zl3073x_dpll
- * @ref: input reference to be selected
- *
- * Selects the given reference for the DPLL channel it should be
- * locked to.
- *
- * Return: 0 on success, <0 on error
- */
-static int
-zl3073x_dpll_selected_ref_set(struct zl3073x_dpll *zldpll, u8 ref)
-{
-	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 mode, mode_refsel;
-	int rc;
-
-	mode = zldpll->refsel_mode;
-
-	switch (mode) {
-	case ZL_DPLL_MODE_REFSEL_MODE_REFLOCK:
-		/* Manual mode with ref selected */
-		if (ref == ZL3073X_DPLL_REF_NONE) {
-			switch (zldpll->lock_status) {
-			case DPLL_LOCK_STATUS_LOCKED_HO_ACQ:
-			case DPLL_LOCK_STATUS_HOLDOVER:
-				/* Switch to forced holdover */
-				mode = ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER;
-				break;
-			default:
-				/* Switch to freerun */
-				mode = ZL_DPLL_MODE_REFSEL_MODE_FREERUN;
-				break;
-			}
-			/* Keep selected reference */
-			ref = zldpll->forced_ref;
-		} else if (ref == zldpll->forced_ref) {
-			/* No register update - same mode and same ref */
-			return 0;
-		}
-		break;
-	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
-	case ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER:
-		/* Manual mode without no ref */
-		if (ref == ZL3073X_DPLL_REF_NONE)
-			/* No register update - keep current mode */
-			return 0;
-
-		/* Switch to reflock mode and update ref selection */
-		mode = ZL_DPLL_MODE_REFSEL_MODE_REFLOCK;
-		break;
-	default:
-		/* For other modes like automatic or NCO ref cannot be selected
-		 * manually
-		 */
-		return -EOPNOTSUPP;
-	}
-
-	/* Build mode_refsel value */
-	mode_refsel = FIELD_PREP(ZL_DPLL_MODE_REFSEL_MODE, mode) |
-		      FIELD_PREP(ZL_DPLL_MODE_REFSEL_REF, ref);
-
-	/* Update dpll_mode_refsel register */
-	rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(zldpll->id),
-			      mode_refsel);
-	if (rc)
-		return rc;
-
-	/* Store new mode and forced reference */
-	zldpll->refsel_mode = mode;
-	zldpll->forced_ref = ref;
-
-	return rc;
-}
-
 /**
  * zl3073x_dpll_connected_ref_get - get currently connected reference
  * @zldpll: pointer to zl3073x_dpll
- * @ref: place to store selected reference
  *
- * Looks for currently connected the DPLL is locked to and stores its index
- * to given @ref.
+ * Looks for currently connected reference the DPLL is locked to.
  *
- * Return: 0 on success, <0 on error
+ * Return: reference index if locked, ZL3073X_DPLL_REF_NONE otherwise
  */
-static int
-zl3073x_dpll_connected_ref_get(struct zl3073x_dpll *zldpll, u8 *ref)
+static u8
+zl3073x_dpll_connected_ref_get(struct zl3073x_dpll *zldpll)
 {
-	struct zl3073x_dev *zldev = zldpll->dev;
-	int rc;
+	const struct zl3073x_chan *chan = zl3073x_chan_state_get(zldpll->dev,
+								 zldpll->id);
+	u8 state;
 
-	/* Get currently selected input reference */
-	rc = zl3073x_dpll_selected_ref_get(zldpll, ref);
-	if (rc)
-		return rc;
+	/* A reference is connected only when the DPLL is locked to it */
+	state = zl3073x_chan_refsel_state_get(chan);
+	if (state == ZL_DPLL_REFSEL_STATUS_STATE_LOCK)
+		return zl3073x_chan_refsel_ref_get(chan);
 
-	/* If the monitor indicates an error nothing is connected */
-	if (ZL3073X_DPLL_REF_IS_VALID(*ref) &&
-	    !zl3073x_dev_ref_is_status_ok(zldev, *ref))
-		*ref = ZL3073X_DPLL_REF_NONE;
-
-	return 0;
+	return ZL3073X_DPLL_REF_NONE;
 }
 
 static int
@@ -410,12 +279,9 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin,
 	const struct zl3073x_ref *ref;
 	u8 conn_id, ref_id;
 	s64 ref_phase;
-	int rc;
 
 	/* Get currently connected reference */
-	rc = zl3073x_dpll_connected_ref_get(zldpll, &conn_id);
-	if (rc)
-		return rc;
+	conn_id = zl3073x_dpll_connected_ref_get(zldpll);
 
 	/* Report phase offset only for currently connected pin if the phase
 	 * monitor feature is disabled and only if the input pin signal is
@@ -453,7 +319,7 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin,
 
 	*phase_offset = ref_phase * DPLL_PHASE_OFFSET_DIVIDER;
 
-	return rc;
+	return 0;
 }
 
 static int
@@ -516,98 +382,6 @@ zl3073x_dpll_input_pin_phase_adjust_set(const struct dpll_pin *dpll_pin,
 	return zl3073x_ref_state_set(zldev, ref_id, &ref);
 }
 
-/**
- * zl3073x_dpll_ref_prio_get - get priority for given input pin
- * @pin: pointer to pin
- * @prio: place to store priority
- *
- * Reads current priority for the given input pin and stores the value
- * to @prio.
- *
- * Return: 0 on success, <0 on error
- */
-static int
-zl3073x_dpll_ref_prio_get(struct zl3073x_dpll_pin *pin, u8 *prio)
-{
-	struct zl3073x_dpll *zldpll = pin->dpll;
-	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 ref, ref_prio;
-	int rc;
-
-	guard(mutex)(&zldev->multiop_lock);
-
-	/* Read DPLL configuration */
-	rc = zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_RD,
-			   ZL_REG_DPLL_MB_MASK, BIT(zldpll->id));
-	if (rc)
-		return rc;
-
-	/* Read reference priority - one value for P&N pins (4 bits/pin) */
-	ref = zl3073x_input_pin_ref_get(pin->id);
-	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_REF_PRIO(ref / 2),
-			     &ref_prio);
-	if (rc)
-		return rc;
-
-	/* Select nibble according pin type */
-	if (zl3073x_dpll_is_p_pin(pin))
-		*prio = FIELD_GET(ZL_DPLL_REF_PRIO_REF_P, ref_prio);
-	else
-		*prio = FIELD_GET(ZL_DPLL_REF_PRIO_REF_N, ref_prio);
-
-	return rc;
-}
-
-/**
- * zl3073x_dpll_ref_prio_set - set priority for given input pin
- * @pin: pointer to pin
- * @prio: place to store priority
- *
- * Sets priority for the given input pin.
- *
- * Return: 0 on success, <0 on error
- */
-static int
-zl3073x_dpll_ref_prio_set(struct zl3073x_dpll_pin *pin, u8 prio)
-{
-	struct zl3073x_dpll *zldpll = pin->dpll;
-	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 ref, ref_prio;
-	int rc;
-
-	guard(mutex)(&zldev->multiop_lock);
-
-	/* Read DPLL configuration into mailbox */
-	rc = zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_RD,
-			   ZL_REG_DPLL_MB_MASK, BIT(zldpll->id));
-	if (rc)
-		return rc;
-
-	/* Read reference priority - one value shared between P&N pins */
-	ref = zl3073x_input_pin_ref_get(pin->id);
-	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_REF_PRIO(ref / 2), &ref_prio);
-	if (rc)
-		return rc;
-
-	/* Update nibble according pin type */
-	if (zl3073x_dpll_is_p_pin(pin)) {
-		ref_prio &= ~ZL_DPLL_REF_PRIO_REF_P;
-		ref_prio |= FIELD_PREP(ZL_DPLL_REF_PRIO_REF_P, prio);
-	} else {
-		ref_prio &= ~ZL_DPLL_REF_PRIO_REF_N;
-		ref_prio |= FIELD_PREP(ZL_DPLL_REF_PRIO_REF_N, prio);
-	}
-
-	/* Update reference priority */
-	rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_REF_PRIO(ref / 2), ref_prio);
-	if (rc)
-		return rc;
-
-	/* Commit configuration */
-	return zl3073x_mb_op(zldev, ZL_REG_DPLL_MB_SEM, ZL_DPLL_MB_SEM_WR,
-			     ZL_REG_DPLL_MB_MASK, BIT(zldpll->id));
-}
-
 /**
  * zl3073x_dpll_ref_state_get - get status for given input pin
  * @pin: pointer to pin
@@ -624,17 +398,14 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin,
 {
 	struct zl3073x_dpll *zldpll = pin->dpll;
 	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 ref, ref_conn;
-	int rc;
+	const struct zl3073x_chan *chan;
+	u8 ref;
 
+	chan = zl3073x_chan_state_get(zldev, zldpll->id);
 	ref = zl3073x_input_pin_ref_get(pin->id);
 
-	/* Get currently connected reference */
-	rc = zl3073x_dpll_connected_ref_get(zldpll, &ref_conn);
-	if (rc)
-		return rc;
-
-	if (ref == ref_conn) {
+	/* Check if the pin reference is connected */
+	if (ref == zl3073x_dpll_connected_ref_get(zldpll)) {
 		*state = DPLL_PIN_STATE_CONNECTED;
 		return 0;
 	}
@@ -643,8 +414,9 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin,
 	 * selectable and its monitor does not report any error then report
 	 * pin as selectable.
 	 */
-	if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
-	    zl3073x_dev_ref_is_status_ok(zldev, ref) && pin->selectable) {
+	if (zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
+	    zl3073x_dev_ref_is_status_ok(zldev, ref) &&
+	    zl3073x_chan_ref_is_selectable(chan, ref)) {
 		*state = DPLL_PIN_STATE_SELECTABLE;
 		return 0;
 	}
@@ -678,68 +450,81 @@ zl3073x_dpll_input_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
-	u8 new_ref;
+	struct zl3073x_chan chan;
+	u8 mode, ref;
 	int rc;
 
-	switch (zldpll->refsel_mode) {
+	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	ref = zl3073x_input_pin_ref_get(pin->id);
+	mode = zl3073x_chan_mode_get(&chan);
+
+	switch (mode) {
 	case ZL_DPLL_MODE_REFSEL_MODE_REFLOCK:
-	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
-	case ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER:
 		if (state == DPLL_PIN_STATE_CONNECTED) {
 			/* Choose the pin as new selected reference */
-			new_ref = zl3073x_input_pin_ref_get(pin->id);
+			zl3073x_chan_ref_set(&chan, ref);
 		} else if (state == DPLL_PIN_STATE_DISCONNECTED) {
-			/* No reference */
-			new_ref = ZL3073X_DPLL_REF_NONE;
+			/* Choose new mode based on lock status */
+			switch (zldpll->lock_status) {
+			case DPLL_LOCK_STATUS_LOCKED_HO_ACQ:
+			case DPLL_LOCK_STATUS_HOLDOVER:
+				mode = ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER;
+				break;
+			default:
+				mode = ZL_DPLL_MODE_REFSEL_MODE_FREERUN;
+				break;
+			}
+			zl3073x_chan_mode_set(&chan, mode);
 		} else {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "Invalid pin state for manual mode");
-			return -EINVAL;
+			goto invalid_state;
+		}
+		break;
+	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
+	case ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER:
+		if (state == DPLL_PIN_STATE_CONNECTED) {
+			/* Choose the pin as new selected reference */
+			zl3073x_chan_ref_set(&chan, ref);
+			/* Switch to reflock mode */
+			zl3073x_chan_mode_set(&chan,
+					      ZL_DPLL_MODE_REFSEL_MODE_REFLOCK);
+		} else if (state != DPLL_PIN_STATE_DISCONNECTED) {
+			goto invalid_state;
 		}
-
-		rc = zl3073x_dpll_selected_ref_set(zldpll, new_ref);
 		break;
-
 	case ZL_DPLL_MODE_REFSEL_MODE_AUTO:
 		if (state == DPLL_PIN_STATE_SELECTABLE) {
-			if (pin->selectable)
+			if (zl3073x_chan_ref_is_selectable(&chan, ref))
 				return 0; /* Pin is already selectable */
 
 			/* Restore pin priority in HW */
-			rc = zl3073x_dpll_ref_prio_set(pin, pin->prio);
-			if (rc)
-				return rc;
-
-			/* Mark pin as selectable */
-			pin->selectable = true;
+			zl3073x_chan_ref_prio_set(&chan, ref, pin->prio);
 		} else if (state == DPLL_PIN_STATE_DISCONNECTED) {
-			if (!pin->selectable)
+			if (!zl3073x_chan_ref_is_selectable(&chan, ref))
 				return 0; /* Pin is already disconnected */
 
 			/* Set pin priority to none in HW */
-			rc = zl3073x_dpll_ref_prio_set(pin,
-						       ZL_DPLL_REF_PRIO_NONE);
-			if (rc)
-				return rc;
-
-			/* Mark pin as non-selectable */
-			pin->selectable = false;
+			zl3073x_chan_ref_prio_set(&chan, ref,
+						  ZL_DPLL_REF_PRIO_NONE);
 		} else {
-			NL_SET_ERR_MSG(extack,
-				       "Invalid pin state for automatic mode");
-			return -EINVAL;
+			goto invalid_state;
 		}
 		break;
-
 	default:
 		/* In other modes we cannot change input reference */
 		NL_SET_ERR_MSG(extack,
 			       "Pin state cannot be changed in current mode");
-		rc = -EOPNOTSUPP;
-		break;
+		return -EOPNOTSUPP;
 	}
 
-	return rc;
+	/* Commit DPLL channel changes */
+	rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
+	if (rc)
+		return rc;
+
+	return 0;
+invalid_state:
+	NL_SET_ERR_MSG_MOD(extack, "Invalid pin state for this device mode");
+	return -EINVAL;
 }
 
 static int
@@ -759,15 +544,21 @@ zl3073x_dpll_input_pin_prio_set(const struct dpll_pin *dpll_pin, void *pin_priv,
 				const struct dpll_device *dpll, void *dpll_priv,
 				u32 prio, struct netlink_ext_ack *extack)
 {
+	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
+	struct zl3073x_chan chan;
+	u8 ref;
 	int rc;
 
 	if (prio > ZL_DPLL_REF_PRIO_MAX)
 		return -EINVAL;
 
 	/* If the pin is selectable then update HW registers */
-	if (pin->selectable) {
-		rc = zl3073x_dpll_ref_prio_set(pin, prio);
+	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	ref = zl3073x_input_pin_ref_get(pin->id);
+	if (zl3073x_chan_ref_is_selectable(&chan, ref)) {
+		zl3073x_chan_ref_prio_set(&chan, ref, prio);
+		rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
 		if (rc)
 			return rc;
 	}
@@ -1065,6 +856,25 @@ zl3073x_dpll_output_pin_state_on_dpll_get(const struct dpll_pin *dpll_pin,
 	return 0;
 }
 
+static int
+zl3073x_dpll_temp_get(const struct dpll_device *dpll, void *dpll_priv,
+		      s32 *temp, struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	struct zl3073x_dev *zldev = zldpll->dev;
+	u16 val;
+	int rc;
+
+	rc = zl3073x_read_u16(zldev, ZL_REG_DIE_TEMP_STATUS, &val);
+	if (rc)
+		return rc;
+
+	/* Register value is in units of 0.1 C, convert to millidegrees */
+	*temp = (s16)val * 100;
+
+	return 0;
+}
+
 static int
 zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
 			     enum dpll_lock_status *status,
@@ -1072,11 +882,11 @@ zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
 			     struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
-	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 mon_status, state;
-	int rc;
+	const struct zl3073x_chan *chan;
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 
-	switch (zldpll->refsel_mode) {
+	switch (zl3073x_chan_mode_get(chan)) {
 	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
 	case ZL_DPLL_MODE_REFSEL_MODE_NCO:
 		/* In FREERUN and NCO modes the DPLL is always unlocked */
@@ -1087,16 +897,9 @@ zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
 		break;
 	}
 
-	/* Read DPLL monitor status */
-	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MON_STATUS(zldpll->id),
-			     &mon_status);
-	if (rc)
-		return rc;
-	state = FIELD_GET(ZL_DPLL_MON_STATUS_STATE, mon_status);
-
-	switch (state) {
+	switch (zl3073x_chan_lock_state_get(chan)) {
 	case ZL_DPLL_MON_STATUS_STATE_LOCK:
-		if (FIELD_GET(ZL_DPLL_MON_STATUS_HO_READY, mon_status))
+		if (zl3073x_chan_is_ho_ready(chan))
 			*status = DPLL_LOCK_STATUS_LOCKED_HO_ACQ;
 		else
 			*status = DPLL_LOCK_STATUS_LOCKED;
@@ -1106,8 +909,9 @@ zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
 		*status = DPLL_LOCK_STATUS_HOLDOVER;
 		break;
 	default:
-		dev_warn(zldev->dev, "Unknown DPLL monitor status: 0x%02x\n",
-			 mon_status);
+		dev_warn(zldpll->dev->dev,
+			 "Unknown DPLL monitor status: 0x%02x\n",
+			 chan->mon_status);
 		*status = DPLL_LOCK_STATUS_UNLOCKED;
 		break;
 	}
@@ -1121,13 +925,16 @@ zl3073x_dpll_supported_modes_get(const struct dpll_device *dpll,
 				 struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
+	const struct zl3073x_chan *chan;
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 
 	/* We support switching between automatic and manual mode, except in
 	 * a case where the DPLL channel is configured to run in NCO mode.
 	 * In this case, report only the manual mode to which the NCO is mapped
 	 * as the only supported one.
 	 */
-	if (zldpll->refsel_mode != ZL_DPLL_MODE_REFSEL_MODE_NCO)
+	if (zl3073x_chan_mode_get(chan) != ZL_DPLL_MODE_REFSEL_MODE_NCO)
 		__set_bit(DPLL_MODE_AUTOMATIC, modes);
 
 	__set_bit(DPLL_MODE_MANUAL, modes);
@@ -1140,8 +947,11 @@ zl3073x_dpll_mode_get(const struct dpll_device *dpll, void *dpll_priv,
 		      enum dpll_mode *mode, struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
+	const struct zl3073x_chan *chan;
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 
-	switch (zldpll->refsel_mode) {
+	switch (zl3073x_chan_mode_get(chan)) {
 	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
 	case ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER:
 	case ZL_DPLL_MODE_REFSEL_MODE_NCO:
@@ -1220,14 +1030,12 @@ zl3073x_dpll_mode_set(const struct dpll_device *dpll, void *dpll_priv,
 		      enum dpll_mode mode, struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
-	u8 hw_mode, mode_refsel, ref;
+	struct zl3073x_chan chan;
+	u8 hw_mode, ref;
 	int rc;
 
-	rc = zl3073x_dpll_selected_ref_get(zldpll, &ref);
-	if (rc) {
-		NL_SET_ERR_MSG_MOD(extack, "failed to get selected reference");
-		return rc;
-	}
+	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	ref = zl3073x_chan_refsel_ref_get(&chan);
 
 	if (mode == DPLL_MODE_MANUAL) {
 		/* We are switching from automatic to manual mode:
@@ -1250,44 +1058,32 @@ zl3073x_dpll_mode_set(const struct dpll_device *dpll, void *dpll_priv,
 		 *   it is selectable after switch to automatic mode
 		 * - switch to automatic mode
 		 */
-		struct zl3073x_dpll_pin *pin;
-
-		pin = zl3073x_dpll_pin_get_by_ref(zldpll, ref);
-		if (pin && !pin->selectable) {
-			/* Restore pin priority in HW */
-			rc = zl3073x_dpll_ref_prio_set(pin, pin->prio);
-			if (rc) {
-				NL_SET_ERR_MSG_MOD(extack,
-						   "failed to restore pin priority");
-				return rc;
+		if (ZL3073X_DPLL_REF_IS_VALID(ref) &&
+		    !zl3073x_chan_ref_is_selectable(&chan, ref)) {
+			struct zl3073x_dpll_pin *pin;
+
+			pin = zl3073x_dpll_pin_get_by_ref(zldpll, ref);
+			if (pin) {
+				/* Restore pin priority in HW */
+				zl3073x_chan_ref_prio_set(&chan, ref,
+							  pin->prio);
 			}
-
-			pin->selectable = true;
 		}
 
 		hw_mode = ZL_DPLL_MODE_REFSEL_MODE_AUTO;
 	}
 
-	/* Build mode_refsel value */
-	mode_refsel = FIELD_PREP(ZL_DPLL_MODE_REFSEL_MODE, hw_mode);
-
+	zl3073x_chan_mode_set(&chan, hw_mode);
 	if (ZL3073X_DPLL_REF_IS_VALID(ref))
-		mode_refsel |= FIELD_PREP(ZL_DPLL_MODE_REFSEL_REF, ref);
+		zl3073x_chan_ref_set(&chan, ref);
 
-	/* Update dpll_mode_refsel register */
-	rc = zl3073x_write_u8(zldpll->dev, ZL_REG_DPLL_MODE_REFSEL(zldpll->id),
-			      mode_refsel);
+	rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
 	if (rc) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "failed to set reference selection mode");
 		return rc;
 	}
 
-	zldpll->refsel_mode = hw_mode;
-
-	if (ZL3073X_DPLL_REF_IS_VALID(ref))
-		zldpll->forced_ref = ref;
-
 	return 0;
 }
 
@@ -1428,18 +1224,16 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index)
 	pin->phase_gran = props->dpll_props.phase_gran;
 
 	if (zl3073x_dpll_is_input_pin(pin)) {
-		rc = zl3073x_dpll_ref_prio_get(pin, &pin->prio);
-		if (rc)
-			goto err_prio_get;
+		const struct zl3073x_chan *chan;
+		u8 ref;
+
+		chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+		ref = zl3073x_input_pin_ref_get(pin->id);
+		pin->prio = zl3073x_chan_ref_prio_get(chan, ref);
 
-		if (pin->prio == ZL_DPLL_REF_PRIO_NONE) {
-			/* Clamp prio to max value & mark pin non-selectable */
+		if (pin->prio == ZL_DPLL_REF_PRIO_NONE)
+			/* Clamp prio to max value */
 			pin->prio = ZL_DPLL_REF_PRIO_MAX;
-			pin->selectable = false;
-		} else {
-			/* Mark pin as selectable */
-			pin->selectable = true;
-		}
 	}
 
 	/* Create or get existing DPLL pin */
@@ -1468,7 +1262,6 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index)
 
 err_register:
 	dpll_pin_put(pin->dpll_pin, &pin->tracker);
-err_prio_get:
 	pin->dpll_pin = NULL;
 err_pin_get:
 	zl3073x_pin_props_put(props);
@@ -1540,15 +1333,18 @@ zl3073x_dpll_pin_is_registrable(struct zl3073x_dpll *zldpll,
 				enum dpll_pin_direction dir, u8 index)
 {
 	struct zl3073x_dev *zldev = zldpll->dev;
+	const struct zl3073x_chan *chan;
 	bool is_diff, is_enabled;
 	const char *name;
 
+	chan = zl3073x_chan_state_get(zldev, zldpll->id);
+
 	if (dir == DPLL_PIN_DIRECTION_INPUT) {
 		u8 ref_id = zl3073x_input_pin_ref_get(index);
 		const struct zl3073x_ref *ref;
 
 		/* Skip the pin if the DPLL is running in NCO mode */
-		if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_NCO)
+		if (zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_NCO)
 			return false;
 
 		name = "REF";
@@ -1656,20 +1452,11 @@ static int
 zl3073x_dpll_device_register(struct zl3073x_dpll *zldpll)
 {
 	struct zl3073x_dev *zldev = zldpll->dev;
-	u8 dpll_mode_refsel;
 	int rc;
 
-	/* Read DPLL mode and forcibly selected reference */
-	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(zldpll->id),
-			     &dpll_mode_refsel);
-	if (rc)
-		return rc;
-
-	/* Extract mode and selected input reference */
-	zldpll->refsel_mode = FIELD_GET(ZL_DPLL_MODE_REFSEL_MODE,
-					dpll_mode_refsel);
-	zldpll->forced_ref = FIELD_GET(ZL_DPLL_MODE_REFSEL_REF,
-				       dpll_mode_refsel);
+	zldpll->ops = zl3073x_dpll_device_ops;
+	if (zldev->info->flags & ZL3073X_FLAG_DIE_TEMP)
+		zldpll->ops.temp_get = zl3073x_dpll_temp_get;
 
 	zldpll->dpll_dev = dpll_device_get(zldev->clock_id, zldpll->id,
 					   THIS_MODULE, &zldpll->tracker);
@@ -1682,7 +1469,7 @@ zl3073x_dpll_device_register(struct zl3073x_dpll *zldpll)
 
 	rc = dpll_device_register(zldpll->dpll_dev,
 				  zl3073x_prop_dpll_type_get(zldev, zldpll->id),
-				  &zl3073x_dpll_device_ops, zldpll);
+				  &zldpll->ops, zldpll);
 	if (rc) {
 		dpll_device_put(zldpll->dpll_dev, &zldpll->tracker);
 		zldpll->dpll_dev = NULL;
@@ -1705,8 +1492,7 @@ zl3073x_dpll_device_unregister(struct zl3073x_dpll *zldpll)
 
 	cancel_work_sync(&zldpll->change_work);
 
-	dpll_device_unregister(zldpll->dpll_dev, &zl3073x_dpll_device_ops,
-			       zldpll);
+	dpll_device_unregister(zldpll->dpll_dev, &zldpll->ops, zldpll);
 	dpll_device_put(zldpll->dpll_dev, &zldpll->tracker);
 	zldpll->dpll_dev = NULL;
 }
@@ -1822,8 +1608,10 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 	struct zl3073x_dev *zldev = zldpll->dev;
 	enum dpll_lock_status lock_status;
 	struct device *dev = zldev->dev;
+	const struct zl3073x_chan *chan;
 	struct zl3073x_dpll_pin *pin;
 	int rc;
+	u8 mode;
 
 	zldpll->check_count++;
 
@@ -1845,8 +1633,10 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 	/* Input pin monitoring does make sense only in automatic
 	 * or forced reference modes.
 	 */
-	if (zldpll->refsel_mode != ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
-	    zldpll->refsel_mode != ZL_DPLL_MODE_REFSEL_MODE_REFLOCK)
+	chan = zl3073x_chan_state_get(zldev, zldpll->id);
+	mode = zl3073x_chan_mode_get(chan);
+	if (mode != ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
+	    mode != ZL_DPLL_MODE_REFSEL_MODE_REFLOCK)
 		return;
 
 	/* Update phase offset latch registers for this DPLL if the phase
diff --git a/drivers/dpll/zl3073x/dpll.h b/drivers/dpll/zl3073x/dpll.h
index c65c798c37927..115ee4f67e7ab 100644
--- a/drivers/dpll/zl3073x/dpll.h
+++ b/drivers/dpll/zl3073x/dpll.h
@@ -13,10 +13,9 @@
  * @list: this DPLL list entry
  * @dev: pointer to multi-function parent device
  * @id: DPLL index
- * @refsel_mode: reference selection mode
- * @forced_ref: selected reference in forced reference lock mode
  * @check_count: periodic check counter
  * @phase_monitor: is phase offset monitor enabled
+ * @ops: DPLL device operations for this instance
  * @dpll_dev: pointer to registered DPLL device
  * @tracker: tracking object for the acquired reference
  * @lock_status: last saved DPLL lock status
@@ -27,10 +26,9 @@ struct zl3073x_dpll {
 	struct list_head		list;
 	struct zl3073x_dev		*dev;
 	u8				id;
-	u8				refsel_mode;
-	u8				forced_ref;
 	u8				check_count;
 	bool				phase_monitor;
+	struct dpll_device_ops		ops;
 	struct dpll_device		*dpll_dev;
 	dpll_tracker			tracker;
 	enum dpll_lock_status		lock_status;
diff --git a/drivers/dpll/zl3073x/i2c.c b/drivers/dpll/zl3073x/i2c.c
index 7bbfdd4ed8671..979df85826abc 100644
--- a/drivers/dpll/zl3073x/i2c.c
+++ b/drivers/dpll/zl3073x/i2c.c
@@ -22,40 +22,25 @@ static int zl3073x_i2c_probe(struct i2c_client *client)
 		return dev_err_probe(dev, PTR_ERR(zldev->regmap),
 				     "Failed to initialize regmap\n");
 
-	return zl3073x_dev_probe(zldev, i2c_get_match_data(client));
+	return zl3073x_dev_probe(zldev);
 }
 
 static const struct i2c_device_id zl3073x_i2c_id[] = {
-	{
-		.name = "zl30731",
-		.driver_data = (kernel_ulong_t)&zl30731_chip_info,
-	},
-	{
-		.name = "zl30732",
-		.driver_data = (kernel_ulong_t)&zl30732_chip_info,
-	},
-	{
-		.name = "zl30733",
-		.driver_data = (kernel_ulong_t)&zl30733_chip_info,
-	},
-	{
-		.name = "zl30734",
-		.driver_data = (kernel_ulong_t)&zl30734_chip_info,
-	},
-	{
-		.name = "zl30735",
-		.driver_data = (kernel_ulong_t)&zl30735_chip_info,
-	},
+	{ "zl30731" },
+	{ "zl30732" },
+	{ "zl30733" },
+	{ "zl30734" },
+	{ "zl30735" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, zl3073x_i2c_id);
 
 static const struct of_device_id zl3073x_i2c_of_match[] = {
-	{ .compatible = "microchip,zl30731", .data = &zl30731_chip_info },
-	{ .compatible = "microchip,zl30732", .data = &zl30732_chip_info },
-	{ .compatible = "microchip,zl30733", .data = &zl30733_chip_info },
-	{ .compatible = "microchip,zl30734", .data = &zl30734_chip_info },
-	{ .compatible = "microchip,zl30735", .data = &zl30735_chip_info },
+	{ .compatible = "microchip,zl30731" },
+	{ .compatible = "microchip,zl30732" },
+	{ .compatible = "microchip,zl30733" },
+	{ .compatible = "microchip,zl30734" },
+	{ .compatible = "microchip,zl30735" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, zl3073x_i2c_of_match);
diff --git a/drivers/dpll/zl3073x/out.c b/drivers/dpll/zl3073x/out.c
index 86829a0c1c022..eb5628aebcee8 100644
--- a/drivers/dpll/zl3073x/out.c
+++ b/drivers/dpll/zl3073x/out.c
@@ -106,12 +106,32 @@ const struct zl3073x_out *zl3073x_out_state_get(struct zl3073x_dev *zldev,
 	return &zldev->out[index];
 }
 
+/**
+ * zl3073x_out_state_set - commit output state changes to hardware
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: output index to set state for
+ * @out: desired output state
+ *
+ * Validates that invariant fields have not been modified, skips the HW
+ * write if the mutable configuration is unchanged, and otherwise writes
+ * only the changed cfg fields to hardware via the mailbox interface.
+ *
+ * Return: 0 on success, -EINVAL if invariants changed, <0 on HW error
+ */
 int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index,
 			  const struct zl3073x_out *out)
 {
 	struct zl3073x_out *dout = &zldev->out[index];
 	int rc;
 
+	/* Reject attempts to change invariant fields (set at fetch only) */
+	if (WARN_ON(memcmp(&dout->inv, &out->inv, sizeof(out->inv))))
+		return -EINVAL;
+
+	/* Skip HW write if configuration hasn't changed */
+	if (!memcmp(&dout->cfg, &out->cfg, sizeof(out->cfg)))
+		return 0;
+
 	guard(mutex)(&zldev->multiop_lock);
 
 	/* Read output configuration into mailbox */
@@ -146,12 +166,7 @@ int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index,
 		return rc;
 
 	/* After successful commit store new state */
-	dout->div = out->div;
-	dout->width = out->width;
-	dout->esync_n_period = out->esync_n_period;
-	dout->esync_n_width = out->esync_n_width;
-	dout->mode = out->mode;
-	dout->phase_comp = out->phase_comp;
+	dout->cfg = out->cfg;
 
 	return 0;
 }
diff --git a/drivers/dpll/zl3073x/out.h b/drivers/dpll/zl3073x/out.h
index 318f9bb8da3a0..edf40432bba5f 100644
--- a/drivers/dpll/zl3073x/out.h
+++ b/drivers/dpll/zl3073x/out.h
@@ -4,6 +4,7 @@
 #define _ZL3073X_OUT_H
 
 #include <linux/bitfield.h>
+#include <linux/stddef.h>
 #include <linux/types.h>
 
 #include "regs.h"
@@ -17,17 +18,21 @@ struct zl3073x_dev;
  * @esync_n_period: embedded sync or n-pin period (for n-div formats)
  * @esync_n_width: embedded sync or n-pin pulse width
  * @phase_comp: phase compensation
- * @ctrl: output control
  * @mode: output mode
+ * @ctrl: output control
  */
 struct zl3073x_out {
-	u32	div;
-	u32	width;
-	u32	esync_n_period;
-	u32	esync_n_width;
-	s32	phase_comp;
-	u8	ctrl;
-	u8	mode;
+	struct_group(cfg, /* Config */
+		u32	div;
+		u32	width;
+		u32	esync_n_period;
+		u32	esync_n_width;
+		s32	phase_comp;
+		u8	mode;
+	);
+	struct_group(inv, /* Invariants */
+		u8	ctrl;
+	);
 };
 
 int zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index);
diff --git a/drivers/dpll/zl3073x/ref.c b/drivers/dpll/zl3073x/ref.c
index 6b65e61039999..825ac30bcd6f7 100644
--- a/drivers/dpll/zl3073x/ref.c
+++ b/drivers/dpll/zl3073x/ref.c
@@ -51,6 +51,21 @@ zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult)
 	return -EINVAL;
 }
 
+/**
+ * zl3073x_ref_state_update - update input reference status from HW
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: input reference index
+ *
+ * Return: 0 on success, <0 on error
+ */
+int zl3073x_ref_state_update(struct zl3073x_dev *zldev, u8 index)
+{
+	struct zl3073x_ref *ref = &zldev->ref[index];
+
+	return zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(index),
+			       &ref->mon_status);
+}
+
 /**
  * zl3073x_ref_state_fetch - fetch input reference state from hardware
  * @zldev: pointer to zl3073x_dev structure
@@ -73,18 +88,17 @@ int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index)
 		struct zl3073x_ref *p_ref = ref - 1; /* P-pin counterpart*/
 
 		/* Copy the shared items from the P-pin */
-		ref->config = p_ref->config;
-		ref->esync_n_div = p_ref->esync_n_div;
-		ref->freq_base = p_ref->freq_base;
-		ref->freq_mult = p_ref->freq_mult;
-		ref->freq_ratio_m = p_ref->freq_ratio_m;
-		ref->freq_ratio_n = p_ref->freq_ratio_n;
-		ref->phase_comp = p_ref->phase_comp;
-		ref->sync_ctrl = p_ref->sync_ctrl;
+		ref->cfg = p_ref->cfg;
+		ref->inv = p_ref->inv;
 
 		return 0; /* Finish - no non-shared items for now */
 	}
 
+	/* Read reference status */
+	rc = zl3073x_ref_state_update(zldev, index);
+	if (rc)
+		return rc;
+
 	guard(mutex)(&zldev->multiop_lock);
 
 	/* Read reference configuration */
@@ -154,12 +168,32 @@ zl3073x_ref_state_get(struct zl3073x_dev *zldev, u8 index)
 	return &zldev->ref[index];
 }
 
+/**
+ * zl3073x_ref_state_set - commit input reference state changes to hardware
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: input reference index to set state for
+ * @ref: desired reference state
+ *
+ * Validates that invariant fields have not been modified, skips the HW
+ * write if the mutable configuration is unchanged, and otherwise writes
+ * only the changed cfg fields to hardware via the mailbox interface.
+ *
+ * Return: 0 on success, -EINVAL if invariants changed, <0 on HW error
+ */
 int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index,
 			  const struct zl3073x_ref *ref)
 {
 	struct zl3073x_ref *dref = &zldev->ref[index];
 	int rc;
 
+	/* Reject attempts to change invariant fields (set at init only) */
+	if (WARN_ON(memcmp(&dref->inv, &ref->inv, sizeof(ref->inv))))
+		return -EINVAL;
+
+	/* Skip HW write if configuration hasn't changed */
+	if (!memcmp(&dref->cfg, &ref->cfg, sizeof(ref->cfg)))
+		return 0;
+
 	guard(mutex)(&zldev->multiop_lock);
 
 	/* Read reference configuration into mailbox */
@@ -207,13 +241,7 @@ int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index,
 		return rc;
 
 	/* After successful commit store new state */
-	dref->freq_base = ref->freq_base;
-	dref->freq_mult = ref->freq_mult;
-	dref->freq_ratio_m = ref->freq_ratio_m;
-	dref->freq_ratio_n = ref->freq_ratio_n;
-	dref->esync_n_div = ref->esync_n_div;
-	dref->sync_ctrl = ref->sync_ctrl;
-	dref->phase_comp = ref->phase_comp;
+	dref->cfg = ref->cfg;
 
 	return 0;
 }
diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h
index 0d8618f5ce8df..06d8d4d97ea26 100644
--- a/drivers/dpll/zl3073x/ref.h
+++ b/drivers/dpll/zl3073x/ref.h
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/math64.h>
+#include <linux/stddef.h>
 #include <linux/types.h>
 
 #include "regs.h"
@@ -13,28 +14,34 @@ struct zl3073x_dev;
 
 /**
  * struct zl3073x_ref - input reference state
- * @ffo: current fractional frequency offset
  * @phase_comp: phase compensation
  * @esync_n_div: divisor for embedded sync or n-divided signal formats
  * @freq_base: frequency base
  * @freq_mult: frequnecy multiplier
  * @freq_ratio_m: FEC mode multiplier
  * @freq_ratio_n: FEC mode divisor
- * @config: reference config
  * @sync_ctrl: reference sync control
+ * @config: reference config
+ * @ffo: current fractional frequency offset
  * @mon_status: reference monitor status
  */
 struct zl3073x_ref {
-	s64	ffo;
-	u64	phase_comp;
-	u32	esync_n_div;
-	u16	freq_base;
-	u16	freq_mult;
-	u16	freq_ratio_m;
-	u16	freq_ratio_n;
-	u8	config;
-	u8	sync_ctrl;
-	u8	mon_status;
+	struct_group(cfg, /* Configuration */
+		u64	phase_comp;
+		u32	esync_n_div;
+		u16	freq_base;
+		u16	freq_mult;
+		u16	freq_ratio_m;
+		u16	freq_ratio_n;
+		u8	sync_ctrl;
+	);
+	struct_group(inv, /* Invariants */
+		u8	config;
+	);
+	struct_group(stat, /* Status */
+		s64	ffo;
+		u8	mon_status;
+	);
 };
 
 int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index);
@@ -45,6 +52,8 @@ const struct zl3073x_ref *zl3073x_ref_state_get(struct zl3073x_dev *zldev,
 int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index,
 			  const struct zl3073x_ref *ref);
 
+int zl3073x_ref_state_update(struct zl3073x_dev *zldev, u8 index);
+
 int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult);
 
 /**
diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h
index 5573d7188406b..5ae50cb761a97 100644
--- a/drivers/dpll/zl3073x/regs.h
+++ b/drivers/dpll/zl3073x/regs.h
@@ -6,6 +6,18 @@
 #include <linux/bitfield.h>
 #include <linux/bits.h>
 
+/*
+ * Hardware limits for ZL3073x chip family
+ */
+#define ZL3073X_MAX_CHANNELS	5
+#define ZL3073X_NUM_REFS	10
+#define ZL3073X_NUM_OUTS	10
+#define ZL3073X_NUM_SYNTHS	5
+#define ZL3073X_NUM_INPUT_PINS	ZL3073X_NUM_REFS
+#define ZL3073X_NUM_OUTPUT_PINS	(ZL3073X_NUM_OUTS * 2)
+#define ZL3073X_NUM_PINS	(ZL3073X_NUM_INPUT_PINS + \
+				 ZL3073X_NUM_OUTPUT_PINS)
+
 /*
  * Register address structure:
  * ===========================
@@ -78,6 +90,8 @@
 #define ZL_REG_RESET_STATUS			ZL_REG(0, 0x18, 1)
 #define ZL_REG_RESET_STATUS_RESET		BIT(0)
 
+#define ZL_REG_DIE_TEMP_STATUS			ZL_REG(0, 0x44, 2)
+
 /*************************
  * Register Page 2, Status
  *************************/
diff --git a/drivers/dpll/zl3073x/spi.c b/drivers/dpll/zl3073x/spi.c
index af901b4d6dda0..f024f42b78d05 100644
--- a/drivers/dpll/zl3073x/spi.c
+++ b/drivers/dpll/zl3073x/spi.c
@@ -22,40 +22,25 @@ static int zl3073x_spi_probe(struct spi_device *spi)
 		return dev_err_probe(dev, PTR_ERR(zldev->regmap),
 				     "Failed to initialize regmap\n");
 
-	return zl3073x_dev_probe(zldev, spi_get_device_match_data(spi));
+	return zl3073x_dev_probe(zldev);
 }
 
 static const struct spi_device_id zl3073x_spi_id[] = {
-	{
-		.name = "zl30731",
-		.driver_data = (kernel_ulong_t)&zl30731_chip_info
-	},
-	{
-		.name = "zl30732",
-		.driver_data = (kernel_ulong_t)&zl30732_chip_info,
-	},
-	{
-		.name = "zl30733",
-		.driver_data = (kernel_ulong_t)&zl30733_chip_info,
-	},
-	{
-		.name = "zl30734",
-		.driver_data = (kernel_ulong_t)&zl30734_chip_info,
-	},
-	{
-		.name = "zl30735",
-		.driver_data = (kernel_ulong_t)&zl30735_chip_info,
-	},
+	{ "zl30731" },
+	{ "zl30732" },
+	{ "zl30733" },
+	{ "zl30734" },
+	{ "zl30735" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(spi, zl3073x_spi_id);
 
 static const struct of_device_id zl3073x_spi_of_match[] = {
-	{ .compatible = "microchip,zl30731", .data = &zl30731_chip_info },
-	{ .compatible = "microchip,zl30732", .data = &zl30732_chip_info },
-	{ .compatible = "microchip,zl30733", .data = &zl30733_chip_info },
-	{ .compatible = "microchip,zl30734", .data = &zl30734_chip_info },
-	{ .compatible = "microchip,zl30735", .data = &zl30735_chip_info },
+	{ .compatible = "microchip,zl30731" },
+	{ .compatible = "microchip,zl30732" },
+	{ .compatible = "microchip,zl30733" },
+	{ .compatible = "microchip,zl30734" },
+	{ .compatible = "microchip,zl30735" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, zl3073x_spi_of_match);
diff --git a/drivers/dpll/zl3073x/synth.h b/drivers/dpll/zl3073x/synth.h
index 6c55eb8a888c2..89e13ea2e6d94 100644
--- a/drivers/dpll/zl3073x/synth.h
+++ b/drivers/dpll/zl3073x/synth.h
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/math64.h>
+#include <linux/stddef.h>
 #include <linux/types.h>
 
 #include "regs.h"
@@ -20,11 +21,13 @@ struct zl3073x_dev;
  * @ctrl: synth control
  */
 struct zl3073x_synth {
-	u32	freq_mult;
-	u16	freq_base;
-	u16	freq_m;
-	u16	freq_n;
-	u8	ctrl;
+	struct_group(inv, /* Invariants */
+		u32	freq_mult;
+		u16	freq_base;
+		u16	freq_m;
+		u16	freq_n;
+		u8	ctrl;
+	);
 };
 
 int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 synth_id);
@@ -32,9 +35,6 @@ int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 synth_id);
 const struct zl3073x_synth *zl3073x_synth_state_get(struct zl3073x_dev *zldev,
 						    u8 synth_id);
 
-int zl3073x_synth_state_set(struct zl3073x_dev *zldev, u8 synth_id,
-			    const struct zl3073x_synth *synth);
-
 /**
  * zl3073x_synth_dpll_get - get DPLL ID the synth is driven by
  * @synth: pointer to synth state
diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h
index 7248d24c4908d..f3e84172caa9c 100644
--- a/drivers/edac/altera_edac.h
+++ b/drivers/edac/altera_edac.h
@@ -172,7 +172,6 @@ struct altr_sdram_prv_data {
 	int ecc_irq_clr_mask;
 	int ecc_cnt_rst_offset;
 	int ecc_cnt_rst_mask;
-	struct edac_dev_sysfs_attribute *eccmgr_sysfs_attr;
 	int ecc_enable_mask;
 	int ce_set_mask;
 	int ue_set_mask;
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 8908ab881c855..c6aa69dbd9fb1 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -3863,6 +3863,9 @@ static int per_family_init(struct amd64_pvt *pvt)
 				pvt->max_mcs		= 8;
 			}
 			break;
+		case 0x40 ... 0x4f:
+			pvt->max_mcs			= 4;
+			break;
 		case 0x60 ... 0x6f:
 			pvt->flags.zn_regs_v2		= 1;
 			break;
diff --git a/drivers/edac/edac_device.h b/drivers/edac/edac_device.h
index 034711d71ebf6..24c1921aa4903 100644
--- a/drivers/edac/edac_device.h
+++ b/drivers/edac/edac_device.h
@@ -163,7 +163,7 @@ struct edac_device_ctl_info {
 	 * If attributes are desired, then set to array of attributes
 	 * If no attributes are desired, leave NULL
 	 */
-	struct edac_dev_sysfs_attribute *sysfs_attributes;
+	const struct edac_dev_sysfs_attribute *sysfs_attributes;
 
 	/* pointer to main 'edac' subsys in sysfs */
 	const struct bus_type *edac_subsys;
diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c
index fcebc4ffea26d..b1c2717cd0237 100644
--- a/drivers/edac/edac_device_sysfs.c
+++ b/drivers/edac/edac_device_sysfs.c
@@ -21,8 +21,6 @@
 #define EDAC_DEVICE_SYMLINK	"device"
 
 #define to_edacdev(k) container_of(k, struct edac_device_ctl_info, kobj)
-#define to_edacdev_attr(a) container_of(a, struct edacdev_attribute, attr)
-
 
 /*
  * Set of edac_device_ctl_info attribute store/show functions
@@ -111,14 +109,14 @@ struct ctl_info_attribute {
 };
 
 #define to_ctl_info(k) container_of(k, struct edac_device_ctl_info, kobj)
-#define to_ctl_info_attr(a) container_of(a,struct ctl_info_attribute,attr)
+#define to_ctl_info_attr(a) container_of_const(a, struct ctl_info_attribute, attr)
 
 /* Function to 'show' fields from the edac_dev 'ctl_info' structure */
 static ssize_t edac_dev_ctl_info_show(struct kobject *kobj,
 				struct attribute *attr, char *buffer)
 {
 	struct edac_device_ctl_info *edac_dev = to_ctl_info(kobj);
-	struct ctl_info_attribute *ctl_info_attr = to_ctl_info_attr(attr);
+	const struct ctl_info_attribute *ctl_info_attr = to_ctl_info_attr(attr);
 
 	if (ctl_info_attr->show)
 		return ctl_info_attr->show(edac_dev, buffer);
@@ -131,7 +129,7 @@ static ssize_t edac_dev_ctl_info_store(struct kobject *kobj,
 				const char *buffer, size_t count)
 {
 	struct edac_device_ctl_info *edac_dev = to_ctl_info(kobj);
-	struct ctl_info_attribute *ctl_info_attr = to_ctl_info_attr(attr);
+	const struct ctl_info_attribute *ctl_info_attr = to_ctl_info_attr(attr);
 
 	if (ctl_info_attr->store)
 		return ctl_info_attr->store(edac_dev, buffer, count);
@@ -145,7 +143,7 @@ static const struct sysfs_ops device_ctl_info_ops = {
 };
 
 #define CTL_INFO_ATTR(_name,_mode,_show,_store)        \
-static struct ctl_info_attribute attr_ctl_info_##_name = {      \
+static const struct ctl_info_attribute attr_ctl_info_##_name = {      \
 	.attr = {.name = __stringify(_name), .mode = _mode },   \
 	.show   = _show,                                        \
 	.store  = _store,                                       \
@@ -163,7 +161,7 @@ CTL_INFO_ATTR(poll_msec, S_IRUGO | S_IWUSR,
 	edac_device_ctl_poll_msec_show, edac_device_ctl_poll_msec_store);
 
 /* Base Attributes of the EDAC_DEVICE ECC object */
-static struct attribute *device_ctrl_attrs[] = {
+static const struct attribute *const device_ctrl_attrs[] = {
 	&attr_ctl_info_panic_on_ue.attr,
 	&attr_ctl_info_log_ue.attr,
 	&attr_ctl_info_log_ce.attr,
@@ -575,8 +573,7 @@ static void edac_device_delete_block(struct edac_device_ctl_info *edac_dev,
 		for (i = 0; i < block->nr_attribs; i++, sysfs_attrib++) {
 
 			/* remove each block_attrib file */
-			sysfs_remove_file(&block->kobj,
-				(struct attribute *) sysfs_attrib);
+			sysfs_remove_file(&block->kobj, &sysfs_attrib->attr);
 		}
 	}
 
@@ -726,7 +723,7 @@ static void edac_device_delete_instances(struct edac_device_ctl_info *edac_dev)
 static int edac_device_add_main_sysfs_attributes(
 			struct edac_device_ctl_info *edac_dev)
 {
-	struct edac_dev_sysfs_attribute *sysfs_attrib;
+	const struct edac_dev_sysfs_attribute *sysfs_attrib;
 	int err = 0;
 
 	sysfs_attrib = edac_dev->sysfs_attributes;
@@ -736,7 +733,7 @@ static int edac_device_add_main_sysfs_attributes(
 		 */
 		while (sysfs_attrib->attr.name != NULL) {
 			err = sysfs_create_file(&edac_dev->kobj,
-				(struct attribute*) sysfs_attrib);
+				&sysfs_attrib->attr);
 			if (err)
 				goto err_out;
 
@@ -755,7 +752,7 @@ static int edac_device_add_main_sysfs_attributes(
 static void edac_device_remove_main_sysfs_attributes(
 			struct edac_device_ctl_info *edac_dev)
 {
-	struct edac_dev_sysfs_attribute *sysfs_attrib;
+	const struct edac_dev_sysfs_attribute *sysfs_attrib;
 
 	/* if there are main attributes, defined, remove them. First,
 	 * point to the start of the array and iterate over it
@@ -764,8 +761,7 @@ static void edac_device_remove_main_sysfs_attributes(
 	sysfs_attrib = edac_dev->sysfs_attributes;
 	if (sysfs_attrib) {
 		while (sysfs_attrib->attr.name != NULL) {
-			sysfs_remove_file(&edac_dev->kobj,
-					(struct attribute *) sysfs_attrib);
+			sysfs_remove_file(&edac_dev->kobj, &sysfs_attrib->attr);
 			sysfs_attrib++;
 		}
 	}
diff --git a/drivers/edac/edac_pci_sysfs.c b/drivers/edac/edac_pci_sysfs.c
index 446fef0a93993..9f437f648e4e3 100644
--- a/drivers/edac/edac_pci_sysfs.c
+++ b/drivers/edac/edac_pci_sysfs.c
@@ -68,7 +68,7 @@ static ssize_t instance_npe_count_show(struct edac_pci_ctl_info *pci,
 }
 
 #define to_instance(k) container_of(k, struct edac_pci_ctl_info, kobj)
-#define to_instance_attr(a) container_of(a, struct instance_attribute, attr)
+#define to_instance_attr(a) container_of_const(a, struct instance_attribute, attr)
 
 /* DEVICE instance kobject release() function */
 static void edac_pci_instance_release(struct kobject *kobj)
@@ -98,7 +98,7 @@ static ssize_t edac_pci_instance_show(struct kobject *kobj,
 				struct attribute *attr, char *buffer)
 {
 	struct edac_pci_ctl_info *pci = to_instance(kobj);
-	struct instance_attribute *instance_attr = to_instance_attr(attr);
+	const struct instance_attribute *instance_attr = to_instance_attr(attr);
 
 	if (instance_attr->show)
 		return instance_attr->show(pci, buffer);
@@ -111,7 +111,7 @@ static ssize_t edac_pci_instance_store(struct kobject *kobj,
 				const char *buffer, size_t count)
 {
 	struct edac_pci_ctl_info *pci = to_instance(kobj);
-	struct instance_attribute *instance_attr = to_instance_attr(attr);
+	const struct instance_attribute *instance_attr = to_instance_attr(attr);
 
 	if (instance_attr->store)
 		return instance_attr->store(pci, buffer, count);
@@ -125,7 +125,7 @@ static const struct sysfs_ops pci_instance_ops = {
 };
 
 #define INSTANCE_ATTR(_name, _mode, _show, _store)	\
-static struct instance_attribute attr_instance_##_name = {	\
+static const struct instance_attribute attr_instance_##_name = {	\
 	.attr	= {.name = __stringify(_name), .mode = _mode },	\
 	.show	= _show,					\
 	.store	= _store,					\
@@ -135,7 +135,7 @@ INSTANCE_ATTR(pe_count, S_IRUGO, instance_pe_count_show, NULL);
 INSTANCE_ATTR(npe_count, S_IRUGO, instance_npe_count_show, NULL);
 
 /* pci instance attributes */
-static struct attribute *pci_instance_attrs[] = {
+static const struct attribute *const pci_instance_attrs[] = {
 	&attr_instance_pe_count.attr,
 	&attr_instance_npe_count.attr,
 	NULL
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 89b3e8cc38b1a..6f61b76142307 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -580,6 +580,10 @@ static bool i10nm_mc_decode_available(struct mce *mce)
 		if (bank < 13 || bank > 20)
 			return false;
 		break;
+	case GNR:
+		if (bank < 13 || bank > 24)
+			return false;
+		break;
 	default:
 		return false;
 	}
@@ -637,6 +641,16 @@ static bool i10nm_mc_decode(struct decoded_addr *res)
 		res->rank         = GET_BITFIELD(m->misc, 57, 57);
 		res->dimm         = GET_BITFIELD(m->misc, 58, 58);
 		break;
+	case GNR:
+		res->imc          = m->bank - 13;
+		res->channel      = 0;
+		res->column       = GET_BITFIELD(m->misc, 9, 18) << 2;
+		res->row          = GET_BITFIELD(m->misc, 19, 36);
+		res->bank_group   = GET_BITFIELD(m->misc, 39, 41);
+		res->bank_address = GET_BITFIELD(m->misc, 37, 38);
+		res->rank         = GET_BITFIELD(m->misc, 55, 56);
+		res->dimm         = GET_BITFIELD(m->misc, 57, 57);
+		break;
 	default:
 		return false;
 	}
diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c
index 0f783173d0ecd..b87c4c6471ead 100644
--- a/drivers/edac/i7core_edac.c
+++ b/drivers/edac/i7core_edac.c
@@ -240,9 +240,9 @@ struct pci_id_table {
 struct i7core_dev {
 	struct list_head	list;
 	u8			socket;
-	struct pci_dev		**pdev;
-	int			n_devs;
 	struct mem_ctl_info	*mci;
+	int			n_devs;
+	struct pci_dev		*pdev[] __counted_by(n_devs);
 };
 
 struct i7core_pvt {
@@ -455,18 +455,12 @@ static struct i7core_dev *alloc_i7core_dev(u8 socket,
 {
 	struct i7core_dev *i7core_dev;
 
-	i7core_dev = kzalloc_obj(*i7core_dev);
+	i7core_dev = kzalloc_flex(*i7core_dev, pdev, table->n_devs);
 	if (!i7core_dev)
 		return NULL;
 
-	i7core_dev->pdev = kzalloc_objs(*i7core_dev->pdev, table->n_devs);
-	if (!i7core_dev->pdev) {
-		kfree(i7core_dev);
-		return NULL;
-	}
-
-	i7core_dev->socket = socket;
 	i7core_dev->n_devs = table->n_devs;
+	i7core_dev->socket = socket;
 	list_add_tail(&i7core_dev->list, &i7core_edac_list);
 
 	return i7core_dev;
@@ -475,7 +469,6 @@ static struct i7core_dev *alloc_i7core_dev(u8 socket,
 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 {
 	list_del(&i7core_dev->list);
-	kfree(i7core_dev->pdev);
 	kfree(i7core_dev);
 }
 
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index af3c12284a1e5..bd252cb3c38e7 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -689,36 +689,46 @@ static void decode_mc6_mce(struct mce *m)
 }
 
 static const char * const smca_long_names[] = {
-	[SMCA_LS ... SMCA_LS_V2]	= "Load Store Unit",
-	[SMCA_IF]			= "Instruction Fetch Unit",
-	[SMCA_L2_CACHE]			= "L2 Cache",
+	[SMCA_CS ... SMCA_CS_V2]	= "Coherent Station",
+	[SMCA_DACC_BE]			= "DACC Back-end Unit",
+	[SMCA_DACC_FE]			= "DACC Front-end Unit",
 	[SMCA_DE]			= "Decode Unit",
-	[SMCA_RESERVED]			= "Reserved",
+	[SMCA_EDDR5CMN]			= "eDDR5 CMN Unit",
 	[SMCA_EX]			= "Execution Unit",
 	[SMCA_FP]			= "Floating Point Unit",
+	[SMCA_GMI_PCS]			= "Global Memory Interconnect PCS Unit",
+	[SMCA_GMI_PHY]			= "Global Memory Interconnect PHY Unit",
+	[SMCA_IF]			= "Instruction Fetch Unit",
+	[SMCA_L2_CACHE]			= "L2 Cache",
 	[SMCA_L3_CACHE]			= "L3 Cache",
-	[SMCA_CS ... SMCA_CS_V2]	= "Coherent Slave",
-	[SMCA_PIE]			= "Power, Interrupts, etc.",
-
-	/* UMC v2 is separate because both of them can exist in a single system. */
-	[SMCA_UMC]			= "Unified Memory Controller",
-	[SMCA_UMC_V2]			= "Unified Memory Controller v2",
-	[SMCA_PB]			= "Parameter Block",
-	[SMCA_PSP ... SMCA_PSP_V2]	= "Platform Security Processor",
-	[SMCA_SMU ... SMCA_SMU_V2]	= "System Management Unit",
+	[SMCA_LS ... SMCA_LS_V2]	= "Load Store Unit",
 	[SMCA_MP5]			= "Microprocessor 5 Unit",
+	[SMCA_MPART]			= "MPART Unit",
+	[SMCA_MPASP ... SMCA_MPASP_V2]	= "MPASP Unit",
+	[SMCA_MPDACC]			= "MPDACC Unit",
 	[SMCA_MPDMA]			= "MPDMA Unit",
+	[SMCA_MPM]			= "MPM Unit",
+	[SMCA_MPRAS]			= "MPRAS Unit",
+	[SMCA_NBIF]			= "NBIF Unit",
 	[SMCA_NBIO]			= "Northbridge IO Unit",
+	[SMCA_PB]			= "Parameter Block",
 	[SMCA_PCIE ... SMCA_PCIE_V2]	= "PCI Express Unit",
-	[SMCA_XGMI_PCS]			= "Ext Global Memory Interconnect PCS Unit",
-	[SMCA_NBIF]			= "NBIF Unit",
-	[SMCA_SHUB]			= "System Hub Unit",
+	[SMCA_PCIE_PL]			= "PCIe Link Unit",
+	[SMCA_PIE]			= "Power, Interrupts, etc.",
+	[SMCA_PSP ... SMCA_PSP_V2]	= "Platform Security Processor",
+	[SMCA_RESERVED]			= "Reserved",
 	[SMCA_SATA]			= "SATA Unit",
+	[SMCA_SHUB]			= "System Hub Unit",
+	[SMCA_SMU ... SMCA_SMU_V2]	= "System Management Unit",
+	[SMCA_SSBDCI]			= "Die to Die Interconnect Unit",
+
+	/* UMC v2 is separate because both of them can exist in a single system. */
+	[SMCA_UMC]			= "Unified Memory Controller",
+	[SMCA_UMC_V2]			= "Unified Memory Controller v2",
 	[SMCA_USB]			= "USB Unit",
-	[SMCA_GMI_PCS]			= "Global Memory Interconnect PCS Unit",
-	[SMCA_XGMI_PHY]			= "Ext Global Memory Interconnect PHY Unit",
 	[SMCA_WAFL_PHY]			= "WAFL PHY Unit",
-	[SMCA_GMI_PHY]			= "Global Memory Interconnect PHY Unit",
+	[SMCA_XGMI_PCS]			= "Ext Global Memory Interconnect PCS Unit",
+	[SMCA_XGMI_PHY]			= "Ext Global Memory Interconnect PHY Unit",
 };
 
 static const char *smca_get_long_name(enum smca_bank_types t)
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index a45dc6b35edef..277f1c6bd5222 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -399,7 +399,7 @@ static ssize_t mpc85xx_l2_inject_ctrl_store(struct edac_device_ctl_info
 	return 0;
 }
 
-static struct edac_dev_sysfs_attribute mpc85xx_l2_sysfs_attributes[] = {
+static const struct edac_dev_sysfs_attribute mpc85xx_l2_sysfs_attributes[] = {
 	{
 	 .attr = {
 		  .name = "inject_data_hi",
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 09d4e816404b8..7b282dfd093f2 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -364,11 +364,11 @@ struct sbridge_dev {
 	int			seg;
 	u8			bus, mc;
 	u8			node_id, source_id;
-	struct pci_dev		**pdev;
 	enum domain		dom;
 	int			n_devs;
 	int			i_devs;
 	struct mem_ctl_info	*mci;
+	struct pci_dev		*pdev[] __counted_by(n_devs);
 };
 
 struct knl_pvt {
@@ -771,21 +771,14 @@ static struct sbridge_dev *alloc_sbridge_dev(int seg, u8 bus, enum domain dom,
 {
 	struct sbridge_dev *sbridge_dev;
 
-	sbridge_dev = kzalloc_obj(*sbridge_dev);
+	sbridge_dev = kzalloc_flex(*sbridge_dev, pdev, table->n_devs_per_imc);
 	if (!sbridge_dev)
 		return NULL;
 
-	sbridge_dev->pdev = kzalloc_objs(*sbridge_dev->pdev,
-					 table->n_devs_per_imc);
-	if (!sbridge_dev->pdev) {
-		kfree(sbridge_dev);
-		return NULL;
-	}
-
+	sbridge_dev->n_devs = table->n_devs_per_imc;
 	sbridge_dev->seg = seg;
 	sbridge_dev->bus = bus;
 	sbridge_dev->dom = dom;
-	sbridge_dev->n_devs = table->n_devs_per_imc;
 	list_add_tail(&sbridge_dev->list, &sbridge_edac_list);
 
 	return sbridge_dev;
@@ -794,7 +787,6 @@ static struct sbridge_dev *alloc_sbridge_dev(int seg, u8 bus, enum domain dom,
 static void free_sbridge_dev(struct sbridge_dev *sbridge_dev)
 {
 	list_del(&sbridge_dev->list);
-	kfree(sbridge_dev->pdev);
 	kfree(sbridge_dev);
 }
 
diff --git a/drivers/edac/versalnet_edac.c b/drivers/edac/versalnet_edac.c
index 2cbc13d9bd00b..0b47ed7fed638 100644
--- a/drivers/edac/versalnet_edac.c
+++ b/drivers/edac/versalnet_edac.c
@@ -70,6 +70,8 @@
 #define XDDR5_BUS_WIDTH_32		1
 #define XDDR5_BUS_WIDTH_16		2
 
+#define MC_NAME_LEN			32
+
 /**
  * struct ecc_error_info - ECC error log information.
  * @burstpos:		Burst position.
@@ -760,7 +762,17 @@ static void versal_edac_release(struct device *dev)
 	kfree(dev);
 }
 
-static int init_versalnet(struct mc_priv *priv, struct platform_device *pdev)
+static void remove_one_mc(struct mc_priv *priv, int i)
+{
+	struct mem_ctl_info *mci;
+
+	mci = priv->mci[i];
+	device_unregister(mci->pdev);
+	edac_mc_del_mc(mci->pdev);
+	edac_mc_free(mci);
+}
+
+static int init_one_mc(struct mc_priv *priv, struct platform_device *pdev, int i)
 {
 	u32 num_chans, rank, dwidth, config;
 	struct edac_mc_layer layers[2];
@@ -768,102 +780,110 @@ static int init_versalnet(struct mc_priv *priv, struct platform_device *pdev)
 	struct device *dev;
 	enum dev_type dt;
 	char *name;
-	int rc, i;
-
-	for (i = 0; i < NUM_CONTROLLERS; i++) {
-		config = priv->adec[CONF + i * ADEC_NUM];
-		num_chans = FIELD_GET(MC5_NUM_CHANS_MASK, config);
-		rank = 1 << FIELD_GET(MC5_RANK_MASK, config);
-		dwidth = FIELD_GET(MC5_BUS_WIDTH_MASK, config);
-
-		switch (dwidth) {
-		case XDDR5_BUS_WIDTH_16:
-			dt = DEV_X16;
-			break;
-		case XDDR5_BUS_WIDTH_32:
-			dt = DEV_X32;
-			break;
-		case XDDR5_BUS_WIDTH_64:
-			dt = DEV_X64;
-			break;
-		default:
-			dt = DEV_UNKNOWN;
-		}
+	int rc;
 
-		if (dt == DEV_UNKNOWN)
-			continue;
+	config = priv->adec[CONF + i * ADEC_NUM];
+	num_chans = FIELD_GET(MC5_NUM_CHANS_MASK, config);
+	rank = 1 << FIELD_GET(MC5_RANK_MASK, config);
+	dwidth = FIELD_GET(MC5_BUS_WIDTH_MASK, config);
+
+	switch (dwidth) {
+	case XDDR5_BUS_WIDTH_16:
+		dt = DEV_X16;
+		break;
+	case XDDR5_BUS_WIDTH_32:
+		dt = DEV_X32;
+		break;
+	case XDDR5_BUS_WIDTH_64:
+		dt = DEV_X64;
+		break;
+	default:
+		dt = DEV_UNKNOWN;
+	}
 
-		/* Find the first enabled device and register that one. */
-		layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
-		layers[0].size = rank;
-		layers[0].is_virt_csrow = true;
-		layers[1].type = EDAC_MC_LAYER_CHANNEL;
-		layers[1].size = num_chans;
-		layers[1].is_virt_csrow = false;
+	if (dt == DEV_UNKNOWN)
+		return 0;
 
-		rc = -ENOMEM;
-		mci = edac_mc_alloc(i, ARRAY_SIZE(layers), layers,
-				    sizeof(struct mc_priv));
-		if (!mci) {
-			edac_printk(KERN_ERR, EDAC_MC, "Failed memory allocation for MC%d\n", i);
-			goto err_alloc;
-		}
+	/* Find the first enabled device and register that one. */
+	layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
+	layers[0].size = rank;
+	layers[0].is_virt_csrow = true;
+	layers[1].type = EDAC_MC_LAYER_CHANNEL;
+	layers[1].size = num_chans;
+	layers[1].is_virt_csrow = false;
+
+	rc = -ENOMEM;
+	name = kzalloc(MC_NAME_LEN, GFP_KERNEL);
+	if (!name)
+		return rc;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		goto err_name_free;
+
+	mci = edac_mc_alloc(i, ARRAY_SIZE(layers), layers, sizeof(struct mc_priv));
+	if (!mci) {
+		edac_printk(KERN_ERR, EDAC_MC, "Failed memory allocation for MC%d\n", i);
+		goto err_dev_free;
+	}
 
-		priv->mci[i] = mci;
-		priv->dwidth = dt;
+	sprintf(name, "versal-net-ddrmc5-edac-%d", i);
 
-		dev = kzalloc_obj(*dev);
-		dev->release = versal_edac_release;
-		name = kmalloc(32, GFP_KERNEL);
-		sprintf(name, "versal-net-ddrmc5-edac-%d", i);
-		dev->init_name = name;
-		rc = device_register(dev);
-		if (rc)
-			goto err_alloc;
+	dev->init_name = name;
+	dev->release = versal_edac_release;
 
-		mci->pdev = dev;
+	rc = device_register(dev);
+	if (rc)
+		goto err_mc_free;
 
-		platform_set_drvdata(pdev, priv);
+	mci->pdev = dev;
+	mc_init(mci, dev);
 
-		mc_init(mci, dev);
-		rc = edac_mc_add_mc(mci);
-		if (rc) {
-			edac_printk(KERN_ERR, EDAC_MC, "Failed to register MC%d with EDAC core\n", i);
-			goto err_alloc;
-		}
+	rc = edac_mc_add_mc(mci);
+	if (rc) {
+		edac_printk(KERN_ERR, EDAC_MC, "Failed to register MC%d with EDAC core\n", i);
+		goto err_unreg;
 	}
-	return 0;
 
-err_alloc:
-	while (i--) {
-		mci = priv->mci[i];
-		if (!mci)
-			continue;
-
-		if (mci->pdev) {
-			device_unregister(mci->pdev);
-			edac_mc_del_mc(mci->pdev);
-		}
+	priv->mci[i] = mci;
+	priv->dwidth = dt;
 
-		edac_mc_free(mci);
-	}
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+
+err_unreg:
+	device_unregister(mci->pdev);
+err_mc_free:
+	edac_mc_free(mci);
+err_dev_free:
+	kfree(dev);
+err_name_free:
+	kfree(name);
 
 	return rc;
 }
 
-static void remove_versalnet(struct mc_priv *priv)
+static int init_versalnet(struct mc_priv *priv, struct platform_device *pdev)
 {
-	struct mem_ctl_info *mci;
-	int i;
+	int rc, i;
 
 	for (i = 0; i < NUM_CONTROLLERS; i++) {
-		device_unregister(priv->mci[i]->pdev);
-		mci = edac_mc_del_mc(priv->mci[i]->pdev);
-		if (!mci)
-			return;
+		rc = init_one_mc(priv, pdev, i);
+		if (rc) {
+			while (i--)
+				remove_one_mc(priv, i);
 
-		edac_mc_free(mci);
+			return rc;
+		}
 	}
+	return 0;
+}
+
+static void remove_versalnet(struct mc_priv *priv)
+{
+	for (int i = 0; i < NUM_CONTROLLERS; i++)
+		remove_one_mc(priv, i);
 }
 
 static int mc_probe(struct platform_device *pdev)
diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig
index aec46bf03302c..68d9df7d2dae0 100644
--- a/drivers/extcon/Kconfig
+++ b/drivers/extcon/Kconfig
@@ -158,6 +158,7 @@ config EXTCON_PTN5150
 	tristate "NXP PTN5150 CC LOGIC USB EXTCON support"
 	depends on I2C && (GPIOLIB || COMPILE_TEST)
 	depends on USB_ROLE_SWITCH || !USB_ROLE_SWITCH
+	depends on TYPEC || !TYPEC
 	select REGMAP_I2C
 	help
 	  Say Y here to enable support for USB peripheral and USB host
diff --git a/drivers/extcon/extcon-intel-int3496.c b/drivers/extcon/extcon-intel-int3496.c
index ded1a85a55495..7d16d5b7d58f7 100644
--- a/drivers/extcon/extcon-intel-int3496.c
+++ b/drivers/extcon/extcon-intel-int3496.c
@@ -106,7 +106,7 @@ static irqreturn_t int3496_thread_isr(int irq, void *priv)
 	struct int3496_data *data = priv;
 
 	/* Let the pin settle before processing it */
-	mod_delayed_work(system_wq, &data->work, DEBOUNCE_TIME);
+	mod_delayed_work(system_percpu_wq, &data->work, DEBOUNCE_TIME);
 
 	return IRQ_HANDLED;
 }
@@ -181,7 +181,7 @@ static int int3496_probe(struct platform_device *pdev)
 	}
 
 	/* process id-pin so that we start with the right status */
-	queue_delayed_work(system_wq, &data->work, 0);
+	queue_delayed_work(system_percpu_wq, &data->work, 0);
 	flush_delayed_work(&data->work);
 
 	platform_set_drvdata(pdev, data);
diff --git a/drivers/extcon/extcon-ptn5150.c b/drivers/extcon/extcon-ptn5150.c
index 78ad86c4a3bee..eca1b140aeb0f 100644
--- a/drivers/extcon/extcon-ptn5150.c
+++ b/drivers/extcon/extcon-ptn5150.c
@@ -18,6 +18,7 @@
 #include <linux/extcon-provider.h>
 #include <linux/gpio/consumer.h>
 #include <linux/usb/role.h>
+#include <linux/usb/typec_mux.h>
 
 /* PTN5150 registers */
 #define PTN5150_REG_DEVICE_ID			0x01
@@ -38,7 +39,11 @@
 #define PTN5150_REG_DEVICE_ID_VERSION		GENMASK(7, 3)
 #define PTN5150_REG_DEVICE_ID_VENDOR		GENMASK(2, 0)
 
+#define PTN5150_POLARITY_CC1			0x1
+#define PTN5150_POLARITY_CC2			0x2
+
 #define PTN5150_REG_CC_PORT_ATTACHMENT		GENMASK(4, 2)
+#define PTN5150_REG_CC_POLARITY			GENMASK(1, 0)
 #define PTN5150_REG_CC_VBUS_DETECTION		BIT(7)
 #define PTN5150_REG_INT_CABLE_ATTACH_MASK	BIT(0)
 #define PTN5150_REG_INT_CABLE_DETACH_MASK	BIT(1)
@@ -53,6 +58,7 @@ struct ptn5150_info {
 	int irq;
 	struct work_struct irq_work;
 	struct mutex mutex;
+	struct typec_switch *orient_sw;
 	struct usb_role_switch *role_sw;
 };
 
@@ -71,6 +77,7 @@ static const struct regmap_config ptn5150_regmap_config = {
 
 static void ptn5150_check_state(struct ptn5150_info *info)
 {
+	enum typec_orientation orient = TYPEC_ORIENTATION_NONE;
 	unsigned int port_status, reg_data, vbus;
 	enum usb_role usb_role = USB_ROLE_NONE;
 	int ret;
@@ -81,6 +88,23 @@ static void ptn5150_check_state(struct ptn5150_info *info)
 		return;
 	}
 
+	orient = FIELD_GET(PTN5150_REG_CC_POLARITY, reg_data);
+	switch (orient) {
+	case PTN5150_POLARITY_CC1:
+		orient = TYPEC_ORIENTATION_NORMAL;
+		break;
+	case PTN5150_POLARITY_CC2:
+		orient = TYPEC_ORIENTATION_REVERSE;
+		break;
+	default:
+		orient = TYPEC_ORIENTATION_NONE;
+		break;
+	}
+
+	ret = typec_switch_set(info->orient_sw, orient);
+	if (ret)
+		dev_err(info->dev, "failed to set orientation: %d\n", ret);
+
 	port_status = FIELD_GET(PTN5150_REG_CC_PORT_ATTACHMENT, reg_data);
 
 	switch (port_status) {
@@ -152,6 +176,12 @@ static void ptn5150_irq_work(struct work_struct *work)
 				dev_err(info->dev,
 					"failed to set none role: %d\n",
 					ret);
+
+			ret = typec_switch_set(info->orient_sw,
+					       TYPEC_ORIENTATION_NONE);
+			if (ret)
+				dev_err(info->dev,
+					"failed to set orientation: %d\n", ret);
 		}
 	}
 
@@ -219,12 +249,14 @@ static void ptn5150_work_sync_and_put(void *data)
 
 	cancel_work_sync(&info->irq_work);
 	usb_role_switch_put(info->role_sw);
+	typec_switch_put(info->orient_sw);
 }
 
 static int ptn5150_i2c_probe(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
 	struct device_node *np = i2c->dev.of_node;
+	struct fwnode_handle *connector;
 	struct ptn5150_info *info;
 	int ret;
 
@@ -311,7 +343,17 @@ static int ptn5150_i2c_probe(struct i2c_client *i2c)
 	if (ret)
 		return -EINVAL;
 
+	connector = device_get_named_child_node(dev, "connector");
+	if (connector) {
+		info->orient_sw = fwnode_typec_switch_get(connector);
+		if (IS_ERR(info->orient_sw))
+			return dev_err_probe(info->dev, PTR_ERR(info->orient_sw),
+					"failed to get orientation switch\n");
+	}
+
 	info->role_sw = usb_role_switch_get(info->dev);
+	if (!info->role_sw && connector)
+		info->role_sw = fwnode_usb_role_switch_get(connector);
 	if (IS_ERR(info->role_sw))
 		return dev_err_probe(info->dev, PTR_ERR(info->role_sw),
 				     "failed to get role switch\n");
@@ -331,6 +373,19 @@ static int ptn5150_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
+static int ptn5150_resume(struct device *dev)
+{
+	struct i2c_client *i2c = to_i2c_client(dev);
+	struct ptn5150_info *info = i2c_get_clientdata(i2c);
+
+	/* Need to check possible pending interrupt events */
+	schedule_work(&info->irq_work);
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(ptn5150_pm_ops, NULL, ptn5150_resume);
+
 static const struct of_device_id ptn5150_dt_match[] = {
 	{ .compatible = "nxp,ptn5150" },
 	{ },
@@ -346,6 +401,7 @@ MODULE_DEVICE_TABLE(i2c, ptn5150_i2c_id);
 static struct i2c_driver ptn5150_i2c_driver = {
 	.driver		= {
 		.name	= "ptn5150",
+		.pm = pm_sleep_ptr(&ptn5150_pm_ops),
 		.of_match_table = ptn5150_dt_match,
 	},
 	.probe		= ptn5150_i2c_probe,
diff --git a/drivers/extcon/extcon-usbc-tusb320.c b/drivers/extcon/extcon-usbc-tusb320.c
index 2eab341de6b76..920b034218502 100644
--- a/drivers/extcon/extcon-usbc-tusb320.c
+++ b/drivers/extcon/extcon-usbc-tusb320.c
@@ -454,20 +454,18 @@ static int tusb320_typec_probe(struct i2c_client *client,
 	priv->port_type = priv->cap.type;
 
 	/* This goes into register 0x8 field CURRENT_MODE_ADVERTISE */
-	ret = fwnode_property_read_string(connector, "typec-power-opmode", &cap_str);
-	if (ret)
-		goto err_put;
-
-	ret = typec_find_pwr_opmode(cap_str);
-	if (ret < 0)
-		goto err_put;
-
-	priv->pwr_opmode = ret;
+	if (!fwnode_property_read_string(connector, "typec-power-opmode",
+					 &cap_str)) {
+		ret = typec_find_pwr_opmode(cap_str);
+		if (ret < 0)
+			goto err_put;
+		priv->pwr_opmode = ret;
 
-	/* Initialize the hardware with the devicetree settings. */
-	ret = tusb320_set_adv_pwr_mode(priv);
-	if (ret)
-		goto err_put;
+		/* Initialize the hardware with the devicetree settings. */
+		ret = tusb320_set_adv_pwr_mode(priv);
+		if (ret)
+			goto err_put;
+	}
 
 	priv->cap.revision		= USB_TYPEC_REV_1_1;
 	priv->cap.accessory[0]		= TYPEC_ACCESSORY_AUDIO;
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index d9e9815a5f962..98d85cc114a7f 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -1366,10 +1366,10 @@ void extcon_dev_unregister(struct extcon_dev *edev)
 		return;
 	}
 
-	ida_free(&extcon_dev_ids, edev->id);
-
 	device_unregister(&edev->dev);
 
+	ida_free(&extcon_dev_ids, edev->id);
+
 	if (edev->mutually_exclusive && edev->max_supported) {
 		for (index = 0; edev->mutually_exclusive[index];
 				index++)
diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c
index f1a2bee39bf11..82b3b6d9ed2df 100644
--- a/drivers/firewire/net.c
+++ b/drivers/firewire/net.c
@@ -257,9 +257,10 @@ static void fwnet_header_cache_update(struct hh_cache *hh,
 	memcpy((u8 *)hh->hh_data + HH_DATA_OFF(FWNET_HLEN), haddr, net->addr_len);
 }
 
-static int fwnet_header_parse(const struct sk_buff *skb, unsigned char *haddr)
+static int fwnet_header_parse(const struct sk_buff *skb, const struct net_device *dev,
+			      unsigned char *haddr)
 {
-	memcpy(haddr, skb->dev->dev_addr, FWNET_ALEN);
+	memcpy(haddr, dev->dev_addr, FWNET_ALEN);
 
 	return FWNET_ALEN;
 }
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index 12a625387d6e6..f2f94d4d533e8 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -205,12 +205,12 @@ static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt)
 	return 0;
 }
 
-static int ffa_rxtx_unmap(u16 vm_id)
+static int ffa_rxtx_unmap(void)
 {
 	ffa_value_t ret;
 
 	invoke_ffa_fn((ffa_value_t){
-		      .a0 = FFA_RXTX_UNMAP, .a1 = PACK_TARGET_INFO(vm_id, 0),
+		      .a0 = FFA_RXTX_UNMAP,
 		      }, &ret);
 
 	if (ret.a0 == FFA_ERROR)
@@ -2097,7 +2097,7 @@ static int __init ffa_init(void)
 
 	pr_err("failed to setup partitions\n");
 	ffa_notifications_cleanup();
-	ffa_rxtx_unmap(drv_info->vm_id);
+	ffa_rxtx_unmap();
 free_pages:
 	if (drv_info->tx_buffer)
 		free_pages_exact(drv_info->tx_buffer, rxtx_bufsz);
@@ -2112,7 +2112,7 @@ static void __exit ffa_exit(void)
 {
 	ffa_notifications_cleanup();
 	ffa_partitions_cleanup();
-	ffa_rxtx_unmap(drv_info->vm_id);
+	ffa_rxtx_unmap();
 	free_pages_exact(drv_info->tx_buffer, drv_info->rxtx_bufsz);
 	free_pages_exact(drv_info->rx_buffer, drv_info->rxtx_bufsz);
 	kfree(drv_info);
diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
index 22267bbd0f4db..cd1331c2fc403 100644
--- a/drivers/firmware/arm_scmi/base.c
+++ b/drivers/firmware/arm_scmi/base.c
@@ -7,6 +7,7 @@
 
 #define pr_fmt(fmt) "SCMI Notifications BASE - " fmt
 
+#include <linux/math.h>
 #include <linux/module.h>
 #include <linux/scmi_protocol.h>
 
@@ -219,8 +220,7 @@ scmi_base_implementation_list_get(const struct scmi_protocol_handle *ph,
 		}
 
 		real_list_sz = t->rx.len - sizeof(u32);
-		calc_list_sz = (1 + (loop_num_ret - 1) / sizeof(u32)) *
-				sizeof(u32);
+		calc_list_sz = round_up(loop_num_ret, sizeof(u32));
 		if (calc_list_sz != real_list_sz) {
 			dev_warn(dev,
 				 "Malformed reply - real_sz:%zd  calc_sz:%u  (loop_num_ret:%d)\n",
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index ab36871650a1f..082fb0db8681a 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2018-2022 ARM Ltd.
  */
 
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/limits.h>
 #include <linux/sort.h>
@@ -156,13 +157,26 @@ struct scmi_clock_rate_notify_payld {
 	__le32 rate_high;
 };
 
+struct scmi_clock_desc {
+	u32 id;
+	unsigned int tot_rates;
+	struct scmi_clock_rates r;
+#define	RATE_MIN	0
+#define	RATE_MAX	1
+#define	RATE_STEP	2
+	struct scmi_clock_info info;
+};
+
+#define to_desc(p)	(container_of(p, struct scmi_clock_desc, info))
+
 struct clock_info {
 	int num_clocks;
 	int max_async_req;
 	bool notify_rate_changed_cmd;
 	bool notify_rate_change_requested_cmd;
 	atomic_t cur_async_req;
-	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkds;
+#define CLOCK_INFO(c, i)	(&(((c)->clkds + (i))->info))
 	int (*clock_config_set)(const struct scmi_protocol_handle *ph,
 				u32 clk_id, enum clk_state state,
 				enum scmi_clock_oem_config oem_type,
@@ -184,7 +198,7 @@ scmi_clock_domain_lookup(struct clock_info *ci, u32 clk_id)
 	if (clk_id >= ci->num_clocks)
 		return ERR_PTR(-EINVAL);
 
-	return ci->clk + clk_id;
+	return CLOCK_INFO(ci, clk_id);
 }
 
 static int
@@ -225,8 +239,7 @@ scmi_clock_protocol_attributes_get(const struct scmi_protocol_handle *ph,
 
 struct scmi_clk_ipriv {
 	struct device *dev;
-	u32 clk_id;
-	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkd;
 };
 
 static void iter_clk_possible_parents_prepare_message(void *message, unsigned int desc_index,
@@ -235,7 +248,7 @@ static void iter_clk_possible_parents_prepare_message(void *message, unsigned in
 	struct scmi_msg_clock_possible_parents *msg = message;
 	const struct scmi_clk_ipriv *p = priv;
 
-	msg->id = cpu_to_le32(p->clk_id);
+	msg->id = cpu_to_le32(p->clkd->id);
 	/* Set the number of OPPs to be skipped/already read */
 	msg->skip_parents = cpu_to_le32(desc_index);
 }
@@ -245,7 +258,6 @@ static int iter_clk_possible_parents_update_state(struct scmi_iterator_state *st
 {
 	const struct scmi_msg_resp_clock_possible_parents *r = response;
 	struct scmi_clk_ipriv *p = priv;
-	struct device *dev = ((struct scmi_clk_ipriv *)p)->dev;
 	u32 flags;
 
 	flags = le32_to_cpu(r->num_parent_flags);
@@ -257,14 +269,15 @@ static int iter_clk_possible_parents_update_state(struct scmi_iterator_state *st
 	 * assume it's returned+remaining on first call.
 	 */
 	if (!st->max_resources) {
-		p->clk->num_parents = st->num_returned + st->num_remaining;
-		p->clk->parents = devm_kcalloc(dev, p->clk->num_parents,
-					       sizeof(*p->clk->parents),
-					       GFP_KERNEL);
-		if (!p->clk->parents) {
-			p->clk->num_parents = 0;
+		int num_parents = st->num_returned + st->num_remaining;
+
+		p->clkd->info.parents = devm_kcalloc(p->dev, num_parents,
+						     sizeof(*p->clkd->info.parents),
+						     GFP_KERNEL);
+		if (!p->clkd->info.parents)
 			return -ENOMEM;
-		}
+
+		/* max_resources is used by the iterators to control bounds */
 		st->max_resources = st->num_returned + st->num_remaining;
 	}
 
@@ -279,29 +292,29 @@ static int iter_clk_possible_parents_process_response(const struct scmi_protocol
 	const struct scmi_msg_resp_clock_possible_parents *r = response;
 	struct scmi_clk_ipriv *p = priv;
 
-	u32 *parent = &p->clk->parents[st->desc_index + st->loop_idx];
+	p->clkd->info.parents[st->desc_index + st->loop_idx] =
+		le32_to_cpu(r->possible_parents[st->loop_idx]);
 
-	*parent = le32_to_cpu(r->possible_parents[st->loop_idx]);
+	/* Count only effectively discovered parents */
+	p->clkd->info.num_parents++;
 
 	return 0;
 }
 
-static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph, u32 clk_id,
-				       struct scmi_clock_info *clk)
+static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph,
+				       u32 clk_id, struct clock_info *cinfo)
 {
 	struct scmi_iterator_ops ops = {
 		.prepare_message = iter_clk_possible_parents_prepare_message,
 		.update_state = iter_clk_possible_parents_update_state,
 		.process_response = iter_clk_possible_parents_process_response,
 	};
-
+	struct scmi_clock_desc *clkd = &cinfo->clkds[clk_id];
 	struct scmi_clk_ipriv ppriv = {
-		.clk_id = clk_id,
-		.clk = clk,
+		.clkd = clkd,
 		.dev = ph->dev,
 	};
 	void *iter;
-	int ret;
 
 	iter = ph->hops->iter_response_init(ph, &ops, 0,
 					    CLOCK_POSSIBLE_PARENTS_GET,
@@ -310,9 +323,7 @@ static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph, u3
 	if (IS_ERR(iter))
 		return PTR_ERR(iter);
 
-	ret = ph->hops->iter_response_run(iter);
-
-	return ret;
+	return ph->hops->iter_response_run(iter);
 }
 
 static int
@@ -351,7 +362,7 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph,
 	u32 attributes;
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_clock_attributes *attr;
-	struct scmi_clock_info *clk = cinfo->clk + clk_id;
+	struct scmi_clock_info *clk = CLOCK_INFO(cinfo, clk_id);
 
 	ret = ph->xops->xfer_get_init(ph, CLOCK_ATTRIBUTES,
 				      sizeof(clk_id), sizeof(*attr), &t);
@@ -392,10 +403,16 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph,
 		    SUPPORTS_RATE_CHANGE_REQUESTED_NOTIF(attributes))
 			clk->rate_change_requested_notifications = true;
 		if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3) {
-			if (SUPPORTS_PARENT_CLOCK(attributes))
-				scmi_clock_possible_parents(ph, clk_id, clk);
-			if (SUPPORTS_GET_PERMISSIONS(attributes))
-				scmi_clock_get_permissions(ph, clk_id, clk);
+			if (SUPPORTS_PARENT_CLOCK(attributes)) {
+				ret = scmi_clock_possible_parents(ph, clk_id, cinfo);
+				if (ret)
+					return ret;
+			}
+			if (SUPPORTS_GET_PERMISSIONS(attributes)) {
+				ret = scmi_clock_get_permissions(ph, clk_id, clk);
+				if (ret)
+					return ret;
+			}
 			if (SUPPORTS_EXTENDED_CONFIG(attributes))
 				clk->extended_config = true;
 		}
@@ -423,7 +440,7 @@ static void iter_clk_describe_prepare_message(void *message,
 	struct scmi_msg_clock_describe_rates *msg = message;
 	const struct scmi_clk_ipriv *p = priv;
 
-	msg->id = cpu_to_le32(p->clk_id);
+	msg->id = cpu_to_le32(p->clkd->id);
 	/* Set the number of rates to be skipped/already read */
 	msg->rate_index = cpu_to_le32(desc_index);
 }
@@ -456,20 +473,33 @@ iter_clk_describe_update_state(struct scmi_iterator_state *st,
 	flags = le32_to_cpu(r->num_rates_flags);
 	st->num_remaining = NUM_REMAINING(flags);
 	st->num_returned = NUM_RETURNED(flags);
-	p->clk->rate_discrete = RATE_DISCRETE(flags);
+	p->clkd->r.rate_discrete = RATE_DISCRETE(flags);
 
 	/* Warn about out of spec replies ... */
-	if (!p->clk->rate_discrete &&
+	if (!p->clkd->r.rate_discrete &&
 	    (st->num_returned != 3 || st->num_remaining != 0)) {
 		dev_warn(p->dev,
 			 "Out-of-spec CLOCK_DESCRIBE_RATES reply for %s - returned:%d remaining:%d rx_len:%zd\n",
-			 p->clk->name, st->num_returned, st->num_remaining,
+			 p->clkd->info.name, st->num_returned, st->num_remaining,
 			 st->rx_len);
 
 		SCMI_QUIRK(clock_rates_triplet_out_of_spec,
 			   QUIRK_OUT_OF_SPEC_TRIPLET);
 	}
 
+	if (!st->max_resources) {
+		unsigned int tot_rates = st->num_returned + st->num_remaining;
+
+		p->clkd->r.rates = devm_kcalloc(p->dev, tot_rates,
+						sizeof(*p->clkd->r.rates), GFP_KERNEL);
+		if (!p->clkd->r.rates)
+			return -ENOMEM;
+
+		/* max_resources is used by the iterators to control bounds */
+		p->clkd->tot_rates = tot_rates;
+		st->max_resources = tot_rates;
+	}
+
 	return 0;
 }
 
@@ -478,38 +508,20 @@ iter_clk_describe_process_response(const struct scmi_protocol_handle *ph,
 				   const void *response,
 				   struct scmi_iterator_state *st, void *priv)
 {
-	int ret = 0;
 	struct scmi_clk_ipriv *p = priv;
 	const struct scmi_msg_resp_clock_describe_rates *r = response;
 
-	if (!p->clk->rate_discrete) {
-		switch (st->desc_index + st->loop_idx) {
-		case 0:
-			p->clk->range.min_rate = RATE_TO_U64(r->rate[0]);
-			break;
-		case 1:
-			p->clk->range.max_rate = RATE_TO_U64(r->rate[1]);
-			break;
-		case 2:
-			p->clk->range.step_size = RATE_TO_U64(r->rate[2]);
-			break;
-		default:
-			ret = -EINVAL;
-			break;
-		}
-	} else {
-		u64 *rate = &p->clk->list.rates[st->desc_index + st->loop_idx];
+	p->clkd->r.rates[p->clkd->r.num_rates] = RATE_TO_U64(r->rate[st->loop_idx]);
 
-		*rate = RATE_TO_U64(r->rate[st->loop_idx]);
-		p->clk->list.num_rates++;
-	}
+	/* Count only effectively discovered rates */
+	p->clkd->r.num_rates++;
 
-	return ret;
+	return 0;
 }
 
 static int
-scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
-			      struct scmi_clock_info *clk)
+scmi_clock_describe_rates_get_full(const struct scmi_protocol_handle *ph,
+				   struct scmi_clock_desc *clkd)
 {
 	int ret;
 	void *iter;
@@ -519,12 +531,16 @@ scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
 		.process_response = iter_clk_describe_process_response,
 	};
 	struct scmi_clk_ipriv cpriv = {
-		.clk_id = clk_id,
-		.clk = clk,
+		.clkd = clkd,
 		.dev = ph->dev,
 	};
 
-	iter = ph->hops->iter_response_init(ph, &ops, SCMI_MAX_NUM_RATES,
+	/*
+	 * Using tot_rates as max_resources parameter here so as to trigger
+	 * the dynamic allocation only when strictly needed: when trying a
+	 * full enumeration after a lazy one tot_rates will be non-zero.
+	 */
+	iter = ph->hops->iter_response_init(ph, &ops, clkd->tot_rates,
 					    CLOCK_DESCRIBE_RATES,
 					    sizeof(struct scmi_msg_clock_describe_rates),
 					    &cpriv);
@@ -535,18 +551,96 @@ scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
 	if (ret)
 		return ret;
 
-	if (!clk->rate_discrete) {
-		dev_dbg(ph->dev, "Min %llu Max %llu Step %llu Hz\n",
-			clk->range.min_rate, clk->range.max_rate,
-			clk->range.step_size);
-	} else if (clk->list.num_rates) {
-		sort(clk->list.rates, clk->list.num_rates,
-		     sizeof(clk->list.rates[0]), rate_cmp_func, NULL);
+	/* empty set ? */
+	if (!clkd->r.num_rates)
+		return 0;
+
+	if (clkd->r.rate_discrete && PROTOCOL_REV_MAJOR(ph->version) == 0x1)
+		sort(clkd->r.rates, clkd->r.num_rates,
+		     sizeof(clkd->r.rates[0]), rate_cmp_func, NULL);
+
+	return 0;
+}
+
+static int
+scmi_clock_describe_rates_get_lazy(const struct scmi_protocol_handle *ph,
+				   struct scmi_clock_desc *clkd)
+{
+	struct scmi_iterator_ops ops = {
+		.prepare_message = iter_clk_describe_prepare_message,
+		.update_state = iter_clk_describe_update_state,
+		.process_response = iter_clk_describe_process_response,
+	};
+	struct scmi_clk_ipriv cpriv = {
+		.clkd = clkd,
+		.dev = ph->dev,
+	};
+	unsigned int first, last;
+	void *iter;
+	int ret;
+
+	iter = ph->hops->iter_response_init(ph, &ops, 0, CLOCK_DESCRIBE_RATES,
+					    sizeof(struct scmi_msg_clock_describe_rates),
+					    &cpriv);
+	if (IS_ERR(iter))
+		return PTR_ERR(iter);
+
+	/* Try to grab a triplet, so that in case is NON-discrete we are done */
+	first = 0;
+	last = 2;
+	ret = ph->hops->iter_response_run_bound(iter, &first, &last);
+	if (ret)
+		goto out;
+
+	/* If discrete grab the last value, which should be the max */
+	if (clkd->r.rate_discrete && clkd->tot_rates > 3) {
+		first = clkd->tot_rates - 1;
+		last = clkd->tot_rates - 1;
+		ret = ph->hops->iter_response_run_bound(iter, &first, &last);
 	}
 
+out:
+	ph->hops->iter_response_cleanup(iter);
+
 	return ret;
 }
 
+static int
+scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph,
+			      u32 clk_id, struct clock_info *cinfo)
+{
+	struct scmi_clock_desc *clkd = &cinfo->clkds[clk_id];
+	int ret;
+
+	/*
+	 * Since only after SCMI Clock v1.0 the returned rates are guaranteed to
+	 * be discovered in ascending order, lazy enumeration cannot be use for
+	 * SCMI Clock v1.0 protocol.
+	 */
+	if (PROTOCOL_REV_MAJOR(ph->version) > 0x1)
+		ret = scmi_clock_describe_rates_get_lazy(ph, clkd);
+	else
+		ret = scmi_clock_describe_rates_get_full(ph, clkd);
+
+	if (ret)
+		return ret;
+
+	clkd->info.min_rate = clkd->r.rates[RATE_MIN];
+	if (!clkd->r.rate_discrete) {
+		clkd->info.max_rate = clkd->r.rates[RATE_MAX];
+		dev_dbg(ph->dev, "Min %llu Max %llu Step %llu Hz\n",
+			clkd->r.rates[RATE_MIN], clkd->r.rates[RATE_MAX],
+			clkd->r.rates[RATE_STEP]);
+	} else {
+		clkd->info.max_rate = clkd->r.rates[clkd->r.num_rates - 1];
+		dev_dbg(ph->dev, "Clock:%s DISCRETE:%d -> Min %llu Max %llu\n",
+			clkd->info.name, clkd->r.rate_discrete,
+			clkd->info.min_rate, clkd->info.max_rate);
+	}
+
+	return 0;
+}
+
 static int
 scmi_clock_rate_get(const struct scmi_protocol_handle *ph,
 		    u32 clk_id, u64 *value)
@@ -624,6 +718,75 @@ static int scmi_clock_rate_set(const struct scmi_protocol_handle *ph,
 	return ret;
 }
 
+static int scmi_clock_determine_rate(const struct scmi_protocol_handle *ph,
+				     u32 clk_id, unsigned long *rate)
+{
+	u64 fmin, fmax, ftmp;
+	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkd;
+	struct clock_info *ci = ph->get_priv(ph);
+
+	if (!rate)
+		return -EINVAL;
+
+	clk = scmi_clock_domain_lookup(ci, clk_id);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	clkd = to_desc(clk);
+
+	/*
+	 * If we can't figure out what rate it will be, so just return the
+	 * rate back to the caller.
+	 */
+	if (clkd->r.rate_discrete)
+		return 0;
+
+	fmin = clk->min_rate;
+	fmax = clk->max_rate;
+	if (*rate <= fmin) {
+		*rate = fmin;
+		return 0;
+	} else if (*rate >= fmax) {
+		*rate = fmax;
+		return 0;
+	}
+
+	ftmp = *rate - fmin;
+	ftmp += clkd->r.rates[RATE_STEP] - 1; /* to round up */
+	ftmp = div64_ul(ftmp, clkd->r.rates[RATE_STEP]);
+
+	*rate = ftmp * clkd->r.rates[RATE_STEP] + fmin;
+
+	return 0;
+}
+
+static const struct scmi_clock_rates *
+scmi_clock_all_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id)
+{
+	struct clock_info *ci = ph->get_priv(ph);
+	struct scmi_clock_desc *clkd;
+	struct scmi_clock_info *clk;
+
+	clk = scmi_clock_domain_lookup(ci, clk_id);
+	if (IS_ERR(clk) || !clk->name[0])
+		return NULL;
+
+	clkd = to_desc(clk);
+	/* Needs full enumeration ? */
+	if (clkd->r.rate_discrete && clkd->tot_rates != clkd->r.num_rates) {
+		int ret;
+
+		/* rates[] is already allocated BUT we need to re-enumerate */
+		clkd->r.num_rates = 0;
+		ret = scmi_clock_describe_rates_get_full(ph, clkd);
+		if (ret)
+			return NULL;
+	}
+
+	return &clkd->r;
+}
+
 static int
 scmi_clock_config_set(const struct scmi_protocol_handle *ph, u32 clk_id,
 		      enum clk_state state,
@@ -936,6 +1099,8 @@ static const struct scmi_clk_proto_ops clk_proto_ops = {
 	.info_get = scmi_clock_info_get,
 	.rate_get = scmi_clock_rate_get,
 	.rate_set = scmi_clock_rate_set,
+	.determine_rate = scmi_clock_determine_rate,
+	.all_rates_get = scmi_clock_all_rates_get,
 	.enable = scmi_clock_enable,
 	.disable = scmi_clock_disable,
 	.state_get = scmi_clock_state_get,
@@ -1080,17 +1245,20 @@ static int scmi_clock_protocol_init(const struct scmi_protocol_handle *ph)
 	if (ret)
 		return ret;
 
-	cinfo->clk = devm_kcalloc(ph->dev, cinfo->num_clocks,
-				  sizeof(*cinfo->clk), GFP_KERNEL);
-	if (!cinfo->clk)
+	cinfo->clkds = devm_kcalloc(ph->dev, cinfo->num_clocks,
+				    sizeof(*cinfo->clkds), GFP_KERNEL);
+	if (!cinfo->clkds)
 		return -ENOMEM;
 
 	for (clkid = 0; clkid < cinfo->num_clocks; clkid++) {
-		struct scmi_clock_info *clk = cinfo->clk + clkid;
-
+		cinfo->clkds[clkid].id = clkid;
 		ret = scmi_clock_attributes_get(ph, clkid, cinfo);
-		if (!ret)
-			scmi_clock_describe_rates_get(ph, clkid, clk);
+		if (ret)
+			return ret;
+
+		ret = scmi_clock_describe_rates_get(ph, clkid, cinfo);
+		if (ret)
+			return ret;
 	}
 
 	if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3) {
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index 7c35c95fddbaf..7c9617d080a02 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -235,6 +235,9 @@ struct scmi_transport_ops {
  *		      to have an execution latency lesser-equal to the threshold
  *		      should be considered for atomic mode operation: such
  *		      decision is finally left up to the SCMI drivers.
+ * @no_completion_irq: Flag to indicate that this transport has no completion
+ *		       interrupt and has to be polled. This is similar to the
+ *		       force_polling below, except this is set via DT property.
  * @force_polling: Flag to force this whole transport to use SCMI core polling
  *		   mechanism instead of completion interrupts even if available.
  * @sync_cmds_completed_on_ret: Flag to indicate that the transport assures
@@ -254,6 +257,7 @@ struct scmi_desc {
 	int max_msg;
 	int max_msg_size;
 	unsigned int atomic_threshold;
+	bool no_completion_irq;
 	const bool force_polling;
 	const bool sync_cmds_completed_on_ret;
 	const bool atomic_enabled;
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index 3e76a3204ba4f..cb4865fd8af29 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -17,6 +17,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/bitmap.h>
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/export.h>
@@ -1789,68 +1790,72 @@ static void *scmi_iterator_init(const struct scmi_protocol_handle *ph,
 				size_t tx_size, void *priv)
 {
 	int ret;
-	struct scmi_iterator *i;
 
-	i = devm_kzalloc(ph->dev, sizeof(*i), GFP_KERNEL);
+	struct scmi_iterator *i __free(kfree) = kzalloc(sizeof(*i), GFP_KERNEL);
 	if (!i)
 		return ERR_PTR(-ENOMEM);
 
+	if (!ops || !ph)
+		return ERR_PTR(-EINVAL);
+
 	i->ph = ph;
 	i->ops = ops;
 	i->priv = priv;
 
 	ret = ph->xops->xfer_get_init(ph, msg_id, tx_size, 0, &i->t);
-	if (ret) {
-		devm_kfree(ph->dev, i);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	i->state.max_resources = max_resources;
 	i->msg = i->t->tx.buf;
 	i->resp = i->t->rx.buf;
 
-	return i;
+	return no_free_ptr(i);
 }
 
-static int scmi_iterator_run(void *iter)
+static int __scmi_iterator_run(void *iter, unsigned int *start, unsigned int *end)
 {
-	int ret = -EINVAL;
+	int ret;
 	struct scmi_iterator_ops *iops;
 	const struct scmi_protocol_handle *ph;
 	struct scmi_iterator_state *st;
-	struct scmi_iterator *i = iter;
+	struct scmi_iterator *i;
 
-	if (!i || !i->ops || !i->ph)
-		return ret;
+	if (!iter)
+		return -EINVAL;
 
+	i = iter;
 	iops = i->ops;
 	ph = i->ph;
 	st = &i->state;
 
+	/* Reinitialize state for next run */
+	st->num_returned = 0;
+	st->num_remaining = 0;
+	st->desc_index = start ? *start : 0;
+
 	do {
 		iops->prepare_message(i->msg, st->desc_index, i->priv);
 		ret = ph->xops->do_xfer(ph, i->t);
 		if (ret)
-			break;
+			return ret;
 
 		st->rx_len = i->t->rx.len;
 		ret = iops->update_state(st, i->resp, i->priv);
 		if (ret)
-			break;
+			return ret;
 
 		if (st->num_returned > st->max_resources - st->desc_index) {
 			dev_err(ph->dev,
 				"No. of resources can't exceed %d\n",
 				st->max_resources);
-			ret = -EINVAL;
-			break;
+			return -EINVAL;
 		}
 
-		for (st->loop_idx = 0; st->loop_idx < st->num_returned;
-		     st->loop_idx++) {
+		for (st->loop_idx = 0; st->loop_idx < st->num_returned; st->loop_idx++) {
 			ret = iops->process_response(ph, i->resp, st, i->priv);
 			if (ret)
-				goto out;
+				return ret;
 		}
 
 		st->desc_index += st->num_returned;
@@ -1859,16 +1864,36 @@ static int scmi_iterator_run(void *iter)
 		 * check for both returned and remaining to avoid infinite
 		 * loop due to buggy firmware
 		 */
-	} while (st->num_returned && st->num_remaining);
+	} while (st->num_returned && st->num_remaining &&
+		 (!end || (st->desc_index <= min(*end, st->max_resources - 1))));
 
-out:
-	/* Finalize and destroy iterator */
-	ph->xops->xfer_put(ph, i->t);
-	devm_kfree(ph->dev, i);
+	return 0;
+}
+
+static void scmi_iterator_cleanup(void *iter)
+{
+	struct scmi_iterator *i = iter;
+
+	i->ph->xops->xfer_put(i->ph, i->t);
+	kfree(i);
+}
+
+static int scmi_iterator_run(void *iter)
+{
+	int ret;
+
+	ret = __scmi_iterator_run(iter, NULL, NULL);
+	scmi_iterator_cleanup(iter);
 
 	return ret;
 }
 
+static int scmi_iterator_run_bound(void *iter, unsigned int *start,
+				   unsigned int *end)
+{
+	return __scmi_iterator_run(iter, start, end);
+}
+
 struct scmi_msg_get_fc_info {
 	__le32 domain;
 	__le32 message_id;
@@ -2047,6 +2072,8 @@ static const struct scmi_proto_helpers_ops helpers_ops = {
 	.get_max_msg_size = scmi_common_get_max_msg_size,
 	.iter_response_init = scmi_iterator_init,
 	.iter_response_run = scmi_iterator_run,
+	.iter_response_run_bound = scmi_iterator_run_bound,
+	.iter_response_cleanup = scmi_iterator_cleanup,
 	.protocol_msg_check = scmi_protocol_msg_check,
 	.fastchannel_init = scmi_common_fastchannel_init,
 	.fastchannel_db_ring = scmi_common_fastchannel_db_ring,
@@ -2735,6 +2762,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 	cinfo->is_p2a = !tx;
 	cinfo->rx_timeout_ms = info->desc->max_rx_timeout_ms;
 	cinfo->max_msg_size = info->desc->max_msg_size;
+	cinfo->no_completion_irq = info->desc->no_completion_irq;
 
 	/* Create a unique name for this transport device */
 	snprintf(name, 32, "__scmi_transport_device_%s_%02X",
@@ -3150,6 +3178,9 @@ static const struct scmi_desc *scmi_transport_setup(struct device *dev)
 	if (ret && ret != -EINVAL)
 		dev_err(dev, "Malformed arm,max-msg DT property.\n");
 
+	trans->desc.no_completion_irq = of_property_read_bool(dev->of_node,
+							      "arm,no-completion-irq");
+
 	dev_info(dev,
 		 "SCMI max-rx-timeout: %dms / max-msg-size: %dbytes / max-msg: %d\n",
 		 trans->desc.max_rx_timeout_ms, trans->desc.max_msg_size,
diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c
index 9168794adae43..40ec184eedaec 100644
--- a/drivers/firmware/arm_scmi/notify.c
+++ b/drivers/firmware/arm_scmi/notify.c
@@ -1066,7 +1066,7 @@ static int scmi_register_event_handler(struct scmi_notify_instance *ni,
  * since at creation time we usually want to have all setup and ready before
  * events really start flowing.
  *
- * Return: A properly refcounted handler on Success, NULL on Failure
+ * Return: A properly refcounted handler on Success, ERR_PTR on Failure
  */
 static inline struct scmi_event_handler *
 __scmi_event_handler_get_ops(struct scmi_notify_instance *ni,
@@ -1113,7 +1113,7 @@ __scmi_event_handler_get_ops(struct scmi_notify_instance *ni,
 	}
 	mutex_unlock(&ni->pending_mtx);
 
-	return hndl;
+	return hndl ?: ERR_PTR(-ENODEV);
 }
 
 static struct scmi_event_handler *
diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h
index 4c75970326e65..e2ef604c16ef6 100644
--- a/drivers/firmware/arm_scmi/protocols.h
+++ b/drivers/firmware/arm_scmi/protocols.h
@@ -189,13 +189,13 @@ struct scmi_protocol_handle {
 
 /**
  * struct scmi_iterator_state  - Iterator current state descriptor
- * @desc_index: Starting index for the current mulit-part request.
+ * @desc_index: Starting index for the current multi-part request.
  * @num_returned: Number of returned items in the last multi-part reply.
  * @num_remaining: Number of remaining items in the multi-part message.
  * @max_resources: Maximum acceptable number of items, configured by the caller
  *		   depending on the underlying resources that it is querying.
  * @loop_idx: The iterator loop index in the current multi-part reply.
- * @rx_len: Size in bytes of the currenly processed message; it can be used by
+ * @rx_len: Size in bytes of the currently processed message; it can be used by
  *	    the user of the iterator to verify a reply size.
  * @priv: Optional pointer to some additional state-related private data setup
  *	  by the caller during the iterations.
@@ -259,7 +259,15 @@ struct scmi_fc_info {
  *			multi-part responses using the custom operations
  *			provided in @ops.
  * @iter_response_run: A common helper to trigger the run of a previously
- *		       initialized iterator.
+ *		       initialized iterator. Note that unbound iterators are
+ *		       automatically cleaned up.
+ * @iter_response_run_bound: A common helper to trigger the run of a previously
+ *			     initialized iterator, but only within the
+ *			     specified, optional, @start and @end resource
+ *			     indexes. Note that these bound-iterators need
+ *			     explicit cleanup via @iter_response_bound_cleanup.
+ * @iter_response_bound_cleanup: A common helper to finally release the iterator
+ *				 for bound iterators.
  * @protocol_msg_check: A common helper to check is a specific protocol message
  *			is supported.
  * @fastchannel_init: A common helper used to initialize FC descriptors by
@@ -276,6 +284,9 @@ struct scmi_proto_helpers_ops {
 				    unsigned int max_resources, u8 msg_id,
 				    size_t tx_size, void *priv);
 	int (*iter_response_run)(void *iter);
+	int (*iter_response_run_bound)(void *iter,
+				       unsigned int *start, unsigned int *end);
+	void (*iter_response_cleanup)(void *iter);
 	int (*protocol_msg_check)(const struct scmi_protocol_handle *ph,
 				  u32 message_id, u32 *attributes);
 	void (*fastchannel_init)(const struct scmi_protocol_handle *ph,
diff --git a/drivers/firmware/arm_scmi/quirks.h b/drivers/firmware/arm_scmi/quirks.h
index a71fde85a5272..d8ba60b956522 100644
--- a/drivers/firmware/arm_scmi/quirks.h
+++ b/drivers/firmware/arm_scmi/quirks.h
@@ -20,10 +20,10 @@
  * named as _qn.
  */
 #define SCMI_QUIRK(_qn, _blk)						\
-	do {								\
+	({								\
 		if (static_branch_unlikely(&(scmi_quirk_ ## _qn)))	\
 			(_blk);						\
-	} while (0)
+	})
 
 void scmi_quirks_initialize(void);
 void scmi_quirks_enable(struct device *dev, const char *vend,
@@ -34,10 +34,10 @@ void scmi_quirks_enable(struct device *dev, const char *vend,
 #define DECLARE_SCMI_QUIRK(_qn)
 /* Force quirks compilation even when SCMI Quirks are disabled */
 #define SCMI_QUIRK(_qn, _blk)						\
-	do {								\
+	({								\
 		if (0)							\
 			(_blk);						\
-	} while (0)
+	})
 
 static inline void scmi_quirks_initialize(void) { }
 static inline void scmi_quirks_enable(struct device *dev, const char *vend,
diff --git a/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c b/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
index 0ada753367efe..637973fb45e65 100644
--- a/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
+++ b/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
@@ -27,6 +27,7 @@ enum scmi_imx_misc_protocol_cmd {
 	SCMI_IMX_MISC_CTRL_GET	= 0x4,
 	SCMI_IMX_MISC_DISCOVER_BUILD_INFO = 0x6,
 	SCMI_IMX_MISC_CTRL_NOTIFY = 0x8,
+	SCMI_IMX_MISC_RESET_REASON_GET = 0xA,
 	SCMI_IMX_MISC_CFG_INFO_GET = 0xC,
 	SCMI_IMX_MISC_SYSLOG_GET = 0xD,
 	SCMI_IMX_MISC_BOARD_INFO = 0xE,
@@ -89,6 +90,37 @@ struct scmi_imx_misc_cfg_info_out {
 	u8 cfgname[MISC_MAX_CFGNAME];
 };
 
+struct scmi_imx_misc_reset_reason_in {
+#define MISC_REASON_FLAG_SYSTEM	BIT(0)
+	__le32 flags;
+};
+
+struct scmi_imx_misc_reset_reason_out {
+	/* Boot reason flags */
+#define MISC_BOOT_FLAG_VLD		BIT(31)
+#define MISC_BOOT_FLAG_ORG_VLD		BIT(28)
+#define MISC_BOOT_FLAG_ORIGIN		GENMASK(27, 24)
+#define MISC_BOOT_FLAG_O_SHIFT		24
+#define MISC_BOOT_FLAG_ERR_VLD		BIT(23)
+#define MISC_BOOT_FLAG_ERR_ID		GENMASK(22, 8)
+#define MISC_BOOT_FLAG_E_SHIFT		8
+#define MISC_BOOT_FLAG_REASON		GENMASK(7, 0)
+	__le32 b_flags;
+	/* Shutdown reason flags */
+#define MISC_SHUTDOWN_FLAG_VLD		BIT(31)
+#define MISC_SHUTDOWN_FLAG_EXT_LEN	GENMASK(30, 29)
+#define MISC_SHUTDOWN_FLAG_ORG_VLD	BIT(28)
+#define MISC_SHUTDOWN_FLAG_ORIGIN	GENMASK(27, 24)
+#define MISC_SHUTDOWN_FLAG_O_SHIFT	24
+#define MISC_SHUTDOWN_FLAG_ERR_VLD	BIT(23)
+#define MISC_SHUTDOWN_FLAG_ERR_ID	GENMASK(22, 8)
+#define MISC_SHUTDOWN_FLAG_E_SHIFT	8
+#define MISC_SHUTDOWN_FLAG_REASON	GENMASK(7, 0)
+	__le32 s_flags;
+	/* Array of extended info words */
+	__le32 extinfo[MISC_EXT_INFO_LEN_MAX];
+};
+
 struct scmi_imx_misc_syslog_in {
 	__le32 flags;
 	__le32 index;
@@ -452,11 +484,65 @@ static int scmi_imx_misc_syslog_get(const struct scmi_protocol_handle *ph, u16 *
 	return ph->hops->iter_response_run(iter);
 }
 
+static int scmi_imx_misc_reset_reason(const struct scmi_protocol_handle *ph, bool system,
+				      struct scmi_imx_misc_reset_reason *boot_r,
+				      struct scmi_imx_misc_reset_reason *shut_r,
+				      u32 *extinfo)
+{
+	struct scmi_imx_misc_reset_reason_in *in;
+	struct scmi_imx_misc_reset_reason_out *out;
+	struct scmi_xfer *t;
+	int ret;
+
+	ret = ph->xops->xfer_get_init(ph, SCMI_IMX_MISC_RESET_REASON_GET, sizeof(*in),
+				      sizeof(*out), &t);
+	if (ret)
+		return ret;
+
+	in = t->tx.buf;
+	if (system)
+		in->flags = le32_encode_bits(1, MISC_REASON_FLAG_SYSTEM);
+	else
+		in->flags = cpu_to_le32(0);
+
+	ret = ph->xops->do_xfer(ph, t);
+	if (!ret) {
+		out = t->rx.buf;
+		if (boot_r) {
+			boot_r->valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_VLD);
+			boot_r->orig_valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ORG_VLD);
+			boot_r->err_valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ERR_VLD);
+			boot_r->reason = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_REASON);
+			boot_r->origin = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ORIGIN);
+			boot_r->errid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ERR_ID);
+		}
+
+		if (shut_r) {
+			shut_r->valid = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_VLD);
+			shut_r->orig_valid = le32_get_bits(out->s_flags,
+							   MISC_SHUTDOWN_FLAG_ORG_VLD);
+			shut_r->err_valid = le32_get_bits(out->s_flags,
+							  MISC_SHUTDOWN_FLAG_ERR_VLD);
+			shut_r->reason = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_REASON);
+			shut_r->origin = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_ORIGIN);
+			shut_r->errid = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_ERR_ID);
+		}
+
+		if (extinfo)
+			memcpy_from_le32(extinfo, out->extinfo, MISC_EXT_INFO_LEN_MAX);
+	}
+
+	ph->xops->xfer_put(ph, t);
+
+	return ret;
+}
+
 static const struct scmi_imx_misc_proto_ops scmi_imx_misc_proto_ops = {
 	.misc_ctrl_set = scmi_imx_misc_ctrl_set,
 	.misc_ctrl_get = scmi_imx_misc_ctrl_get,
 	.misc_ctrl_req_notify = scmi_imx_misc_ctrl_notify,
 	.misc_syslog = scmi_imx_misc_syslog_get,
+	.misc_reset_reason = scmi_imx_misc_reset_reason,
 };
 
 static int scmi_imx_misc_protocol_init(const struct scmi_protocol_handle *ph)
diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c
index 00e74449ce09e..2acad5fa5a286 100644
--- a/drivers/firmware/arm_scpi.c
+++ b/drivers/firmware/arm_scpi.c
@@ -18,6 +18,7 @@
 
 #include <linux/bitmap.h>
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/export.h>
@@ -940,13 +941,13 @@ static int scpi_probe(struct platform_device *pdev)
 		int idx = scpi_drvinfo->num_chans;
 		struct scpi_chan *pchan = scpi_drvinfo->channels + idx;
 		struct mbox_client *cl = &pchan->cl;
-		struct device_node *shmem = of_parse_phandle(np, "shmem", idx);
+		struct device_node *shmem __free(device_node) =
+			of_parse_phandle(np, "shmem", idx);
 
 		if (!of_match_node(shmem_of_match, shmem))
 			return -ENXIO;
 
 		ret = of_address_to_resource(shmem, 0, &res);
-		of_node_put(shmem);
 		if (ret) {
 			dev_err(dev, "failed to get SCPI payload mem resource\n");
 			return ret;
diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c
index 5d8be0ac7c5e6..bb93b24c659dd 100644
--- a/drivers/firmware/cirrus/cs_dsp.c
+++ b/drivers/firmware/cirrus/cs_dsp.c
@@ -9,6 +9,7 @@
  *                         Cirrus Logic International Semiconductor Ltd.
  */
 
+#include <kunit/static_stub.h>
 #include <kunit/visibility.h>
 #include <linux/cleanup.h>
 #include <linux/ctype.h>
@@ -18,6 +19,7 @@
 #include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/ratelimit.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
@@ -30,45 +32,47 @@
 /*
  * When the KUnit test is running the error-case tests will cause a lot
  * of messages. Rate-limit to prevent overflowing the kernel log buffer
- * during KUnit test runs.
+ * during KUnit test runs and allow the test to redirect this function.
+ * In normal (not KUnit) builds this collapses to only return true.
  */
-#if IS_ENABLED(CONFIG_FW_CS_DSP_KUNIT_TEST)
-bool cs_dsp_suppress_err_messages;
-EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_err_messages);
+VISIBLE_IF_KUNIT bool cs_dsp_can_emit_message(void)
+{
+	KUNIT_STATIC_STUB_REDIRECT(cs_dsp_can_emit_message);
 
-bool cs_dsp_suppress_warn_messages;
-EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_warn_messages);
+	if (IS_ENABLED(CONFIG_FW_CS_DSP_KUNIT_TEST)) {
+		static DEFINE_RATELIMIT_STATE(_rs,
+					      DEFAULT_RATELIMIT_INTERVAL,
+					      DEFAULT_RATELIMIT_BURST);
+		return __ratelimit(&_rs);
+	}
 
-bool cs_dsp_suppress_info_messages;
-EXPORT_SYMBOL_IF_KUNIT(cs_dsp_suppress_info_messages);
+	return true;
+}
+EXPORT_SYMBOL_IF_KUNIT(cs_dsp_can_emit_message);
 
-#define cs_dsp_err(_dsp, fmt, ...) \
-	do { \
-		if (!cs_dsp_suppress_err_messages) \
-			dev_err_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
+#define cs_dsp_err(_dsp, fmt, ...)						   \
+	do {									   \
+		if (cs_dsp_can_emit_message())					   \
+			dev_err(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
 	} while (false)
-#define cs_dsp_warn(_dsp, fmt, ...) \
-	do { \
-		if (!cs_dsp_suppress_warn_messages) \
-			dev_warn_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
+
+#define cs_dsp_warn(_dsp, fmt, ...)						    \
+	do {									    \
+		if (cs_dsp_can_emit_message())					    \
+			dev_warn(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
 	} while (false)
-#define cs_dsp_info(_dsp, fmt, ...) \
-	do { \
-		if (!cs_dsp_suppress_info_messages) \
-			dev_info_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
+
+#define cs_dsp_info(_dsp, fmt, ...)						    \
+	do {									    \
+		if (cs_dsp_can_emit_message())					    \
+			dev_info(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
+	} while (false)
+
+#define cs_dsp_dbg(_dsp, fmt, ...)						   \
+	do {									   \
+		if (cs_dsp_can_emit_message())					   \
+			dev_dbg(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__); \
 	} while (false)
-#define cs_dsp_dbg(_dsp, fmt, ...) \
-	dev_dbg_ratelimited(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__)
-#else
-#define cs_dsp_err(_dsp, fmt, ...) \
-	dev_err(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__)
-#define cs_dsp_warn(_dsp, fmt, ...) \
-	dev_warn(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__)
-#define cs_dsp_info(_dsp, fmt, ...) \
-	dev_info(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__)
-#define cs_dsp_dbg(_dsp, fmt, ...) \
-	dev_dbg(_dsp->dev, "%s: " fmt, _dsp->name, ##__VA_ARGS__)
-#endif
 
 #define ADSP1_CONTROL_1                   0x00
 #define ADSP1_CONTROL_2                   0x02
@@ -515,6 +519,7 @@ void cs_dsp_init_debugfs(struct cs_dsp *dsp, struct dentry *debugfs_root)
 
 	debugfs_create_bool("booted", 0444, root, &dsp->booted);
 	debugfs_create_bool("running", 0444, root, &dsp->running);
+	debugfs_create_bool("hibernating", 0444, root, &dsp->hibernating);
 	debugfs_create_x32("fw_id", 0444, root, &dsp->fw_id);
 	debugfs_create_x32("fw_version", 0444, root, &dsp->fw_id_version);
 
@@ -703,7 +708,7 @@ int cs_dsp_coeff_write_acked_control(struct cs_dsp_coeff_ctl *ctl, unsigned int
 
 	lockdep_assert_held(&dsp->pwr_lock);
 
-	if (!dsp->running)
+	if (!dsp->running || dsp->hibernating)
 		return -EPERM;
 
 	ret = cs_dsp_coeff_base_reg(ctl, &reg, 0);
@@ -827,7 +832,7 @@ int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl,
 	}
 
 	ctl->set = 1;
-	if (ctl->enabled && ctl->dsp->running)
+	if (ctl->enabled && ctl->dsp->running && !ctl->dsp->hibernating)
 		ret = cs_dsp_coeff_write_ctrl_raw(ctl, off, buf, len);
 
 	if (ret < 0)
@@ -920,12 +925,12 @@ int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl,
 		return -EINVAL;
 
 	if (ctl->flags & WMFW_CTL_FLAG_VOLATILE) {
-		if (ctl->enabled && ctl->dsp->running)
+		if (ctl->enabled && ctl->dsp->running && !ctl->dsp->hibernating)
 			return cs_dsp_coeff_read_ctrl_raw(ctl, off, buf, len);
 		else
 			return -EPERM;
 	} else {
-		if (!ctl->flags && ctl->enabled && ctl->dsp->running)
+		if (!ctl->flags && ctl->enabled && ctl->dsp->running && !ctl->dsp->hibernating)
 			ret = cs_dsp_coeff_read_ctrl_raw(ctl, 0, ctl->cache, ctl->len);
 
 		if (buf != ctl->cache)
@@ -1108,6 +1113,44 @@ static int cs_dsp_create_control(struct cs_dsp *dsp,
 	return ret;
 }
 
+
+/**
+ * cs_dsp_hibernate() - Disable or enable all controls for a DSP
+ * @dsp: pointer to DSP structure
+ * @hibernate: whether to set controls to cache only mode
+ *
+ * When @hibernate is true, the DSP is entering hibernation mode where the
+ * regmap is inaccessible, and all controls become cache only.
+ * When @hibernate is false, the DSP has exited hibernation mode. If the DSP
+ * is running, all controls are re-synced to the DSP.
+ *
+ */
+void cs_dsp_hibernate(struct cs_dsp *dsp, bool hibernate)
+{
+	mutex_lock(&dsp->pwr_lock);
+
+	if (!dsp->running) {
+		cs_dsp_dbg(dsp, "Cannot hibernate, DSP not running\n");
+		goto out;
+	}
+
+	if (dsp->hibernating == hibernate)
+		goto out;
+
+	cs_dsp_dbg(dsp, "Set hibernating to %d\n", hibernate);
+	dsp->hibernating = hibernate;
+
+	if (!dsp->hibernating && dsp->running) {
+		int ret = cs_dsp_coeff_sync_controls(dsp);
+
+		if (ret)
+			cs_dsp_err(dsp, "Error syncing controls: %d\n", ret);
+	}
+out:
+	mutex_unlock(&dsp->pwr_lock);
+}
+EXPORT_SYMBOL_NS_GPL(cs_dsp_hibernate, "FW_CS_DSP");
+
 struct cs_dsp_coeff_parsed_alg {
 	int id;
 	const u8 *name;
@@ -2510,6 +2553,7 @@ int cs_dsp_adsp1_power_up(struct cs_dsp *dsp,
 		goto err_ena;
 
 	dsp->booted = true;
+	dsp->hibernating = false;
 
 	/* Start the core running */
 	regmap_update_bits(dsp->regmap, dsp->base + ADSP1_CONTROL_30,
@@ -2788,6 +2832,7 @@ int cs_dsp_power_up(struct cs_dsp *dsp,
 		dsp->ops->disable_core(dsp);
 
 	dsp->booted = true;
+	dsp->hibernating = false;
 
 	mutex_unlock(&dsp->pwr_lock);
 
diff --git a/drivers/firmware/cirrus/cs_dsp.h b/drivers/firmware/cirrus/cs_dsp.h
index adf543004aea3..04d768d08d033 100644
--- a/drivers/firmware/cirrus/cs_dsp.h
+++ b/drivers/firmware/cirrus/cs_dsp.h
@@ -10,9 +10,7 @@
 #define FW_CS_DSP_H
 
 #if IS_ENABLED(CONFIG_KUNIT)
-extern bool cs_dsp_suppress_err_messages;
-extern bool cs_dsp_suppress_warn_messages;
-extern bool cs_dsp_suppress_info_messages;
+bool cs_dsp_can_emit_message(void);
 #endif
 
 #endif /* ifndef FW_CS_DSP_H */
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_bin.c b/drivers/firmware/cirrus/test/cs_dsp_test_bin.c
index 66140caeebb5e..63416838f8653 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_bin.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_bin.c
@@ -7,6 +7,7 @@
 
 #include <kunit/device.h>
 #include <kunit/resource.h>
+#include <kunit/static_stub.h>
 #include <kunit/test.h>
 #include <linux/build_bug.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
@@ -2155,6 +2156,15 @@ static void bin_patch_name_and_info(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, reg_val, payload_data);
 }
 
+static bool cs_dsp_bin_test_can_emit_message_hook(void)
+{
+#if defined(DEBUG)
+	return true;
+#else
+	return false;
+#endif
+}
+
 static int cs_dsp_bin_test_common_init(struct kunit *test, struct cs_dsp *dsp,
 					int wmdr_ver)
 {
@@ -2239,16 +2249,12 @@ static int cs_dsp_bin_test_common_init(struct kunit *test, struct cs_dsp *dsp,
 	 * The large number of test cases will cause an unusually large amount
 	 * of dev_info() messages from cs_dsp, so suppress these.
 	 */
-	cs_dsp_suppress_info_messages = true;
+	kunit_activate_static_stub(test, cs_dsp_can_emit_message,
+				   cs_dsp_bin_test_can_emit_message_hook);
 
 	return 0;
 }
 
-static void cs_dsp_bin_test_exit(struct kunit *test)
-{
-	cs_dsp_suppress_info_messages = false;
-}
-
 static int cs_dsp_bin_test_halo_init_common(struct kunit *test, int wmdr_ver)
 {
 	struct cs_dsp *dsp;
@@ -2833,28 +2839,29 @@ static struct kunit_case cs_dsp_bin_test_cases_adsp2[] = {
 static struct kunit_suite cs_dsp_bin_test_halo = {
 	.name = "cs_dsp_bin_halo",
 	.init = cs_dsp_bin_test_halo_init,
-	.exit = cs_dsp_bin_test_exit,
 	.test_cases = cs_dsp_bin_test_cases_halo,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_bin_test_halo_wmdr3 = {
 	.name = "cs_dsp_bin_halo_wmdr_v3",
 	.init = cs_dsp_bin_test_halo_wmdr3_init,
 	.test_cases = cs_dsp_bin_test_cases_halo_wmdr3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_bin_test_adsp2_32bit = {
 	.name = "cs_dsp_bin_adsp2_32bit",
 	.init = cs_dsp_bin_test_adsp2_32bit_init,
-	.exit = cs_dsp_bin_test_exit,
 	.test_cases = cs_dsp_bin_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_bin_test_adsp2_16bit = {
 	.name = "cs_dsp_bin_adsp2_16bit",
 	.init = cs_dsp_bin_test_adsp2_16bit_init,
-	.exit = cs_dsp_bin_test_exit,
 	.test_cases = cs_dsp_bin_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_bin_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c b/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c
index 9b2763b369702..f879c5467c98a 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_bin_error.c
@@ -8,6 +8,7 @@
 
 #include <kunit/device.h>
 #include <kunit/resource.h>
+#include <kunit/static_stub.h>
 #include <kunit/test.h>
 #include <linux/build_bug.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
@@ -380,11 +381,13 @@ static void bin_block_payload_len_garbage(struct kunit *test)
 			0);
 }
 
-static void cs_dsp_bin_err_test_exit(struct kunit *test)
+static bool cs_dsp_bin_err_test_can_emit_message_hook(void)
 {
-	cs_dsp_suppress_err_messages = false;
-	cs_dsp_suppress_warn_messages = false;
-	cs_dsp_suppress_info_messages = false;
+#if defined(DEBUG)
+	return true;
+#else
+	return false;
+#endif
 }
 
 static int cs_dsp_bin_err_test_common_init(struct kunit *test, struct cs_dsp *dsp,
@@ -482,9 +485,8 @@ static int cs_dsp_bin_err_test_common_init(struct kunit *test, struct cs_dsp *ds
 	 * Testing error conditions can produce a lot of log output
 	 * from cs_dsp error messages, so suppress messages.
 	 */
-	cs_dsp_suppress_err_messages = true;
-	cs_dsp_suppress_warn_messages = true;
-	cs_dsp_suppress_info_messages = true;
+	kunit_activate_static_stub(test, cs_dsp_can_emit_message,
+				   cs_dsp_bin_err_test_can_emit_message_hook);
 
 	return 0;
 }
@@ -584,22 +586,22 @@ static struct kunit_case cs_dsp_bin_err_test_cases[] = {
 static struct kunit_suite cs_dsp_bin_err_test_halo = {
 	.name = "cs_dsp_bin_err_halo",
 	.init = cs_dsp_bin_err_test_halo_init,
-	.exit = cs_dsp_bin_err_test_exit,
 	.test_cases = cs_dsp_bin_err_test_cases,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_bin_err_test_adsp2_32bit = {
 	.name = "cs_dsp_bin_err_adsp2_32bit",
 	.init = cs_dsp_bin_err_test_adsp2_32bit_init,
-	.exit = cs_dsp_bin_err_test_exit,
 	.test_cases = cs_dsp_bin_err_test_cases,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_bin_err_test_adsp2_16bit = {
 	.name = "cs_dsp_bin_err_adsp2_16bit",
 	.init = cs_dsp_bin_err_test_adsp2_16bit_init,
-	.exit = cs_dsp_bin_err_test_exit,
 	.test_cases = cs_dsp_bin_err_test_cases,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_bin_err_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_control_cache.c b/drivers/firmware/cirrus/test/cs_dsp_test_control_cache.c
index ebca3a4ab0f1a..b03cd7c37f6e4 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_control_cache.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_control_cache.c
@@ -3248,30 +3248,35 @@ static struct kunit_suite cs_dsp_ctl_cache_test_halo = {
 	.name = "cs_dsp_ctl_cache_wmfwV3_halo",
 	.init = cs_dsp_ctl_cache_test_halo_init,
 	.test_cases = cs_dsp_ctl_cache_test_cases_v3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_cache_test_adsp2_32bit_wmfw1 = {
 	.name = "cs_dsp_ctl_cache_wmfwV1_adsp2_32bit",
 	.init = cs_dsp_ctl_cache_test_adsp2_32bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_cache_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_cache_test_adsp2_32bit_wmfw2 = {
 	.name = "cs_dsp_ctl_cache_wmfwV2_adsp2_32bit",
 	.init = cs_dsp_ctl_cache_test_adsp2_32bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_cache_test_cases_v2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_cache_test_adsp2_16bit_wmfw1 = {
 	.name = "cs_dsp_ctl_cache_wmfwV1_adsp2_16bit",
 	.init = cs_dsp_ctl_cache_test_adsp2_16bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_cache_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_cache_test_adsp2_16bit_wmfw2 = {
 	.name = "cs_dsp_ctl_cache_wmfwV2_adsp2_16bit",
 	.init = cs_dsp_ctl_cache_test_adsp2_16bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_cache_test_cases_v2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_ctl_cache_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_control_parse.c b/drivers/firmware/cirrus/test/cs_dsp_test_control_parse.c
index 942ba1af5e7c1..368703767644a 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_control_parse.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_control_parse.c
@@ -1805,30 +1805,35 @@ static struct kunit_suite cs_dsp_ctl_parse_test_halo = {
 	.name = "cs_dsp_ctl_parse_wmfwV3_halo",
 	.init = cs_dsp_ctl_parse_test_halo_init,
 	.test_cases = cs_dsp_ctl_parse_test_cases_v2_v3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_parse_test_adsp2_32bit_wmfw1 = {
 	.name = "cs_dsp_ctl_parse_wmfwV1_adsp2_32bit",
 	.init = cs_dsp_ctl_parse_test_adsp2_32bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_parse_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_parse_test_adsp2_32bit_wmfw2 = {
 	.name = "cs_dsp_ctl_parse_wmfwV2_adsp2_32bit",
 	.init = cs_dsp_ctl_parse_test_adsp2_32bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_parse_test_cases_v2_v3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_parse_test_adsp2_16bit_wmfw1 = {
 	.name = "cs_dsp_ctl_parse_wmfwV1_adsp2_16bit",
 	.init = cs_dsp_ctl_parse_test_adsp2_16bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_parse_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_parse_test_adsp2_16bit_wmfw2 = {
 	.name = "cs_dsp_ctl_parse_wmfwV2_adsp2_16bit",
 	.init = cs_dsp_ctl_parse_test_adsp2_16bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_parse_test_cases_v2_v3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_ctl_parse_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_control_rw.c b/drivers/firmware/cirrus/test/cs_dsp_test_control_rw.c
index bda00a95d4f9f..d800f163b099e 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_control_rw.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_control_rw.c
@@ -2636,30 +2636,35 @@ static struct kunit_suite cs_dsp_ctl_rw_test_halo = {
 	.name = "cs_dsp_ctl_rw_wmfwV3_halo",
 	.init = cs_dsp_ctl_rw_test_halo_init,
 	.test_cases = cs_dsp_ctl_rw_test_cases_halo,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_rw_test_adsp2_32bit_wmfw1 = {
 	.name = "cs_dsp_ctl_rw_wmfwV1_adsp2_32bit",
 	.init = cs_dsp_ctl_rw_test_adsp2_32bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_rw_test_cases_adsp,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_rw_test_adsp2_32bit_wmfw2 = {
 	.name = "cs_dsp_ctl_rw_wmfwV2_adsp2_32bit",
 	.init = cs_dsp_ctl_rw_test_adsp2_32bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_rw_test_cases_adsp,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_rw_test_adsp2_16bit_wmfw1 = {
 	.name = "cs_dsp_ctl_rw_wmfwV1_adsp2_16bit",
 	.init = cs_dsp_ctl_rw_test_adsp2_16bit_wmfw1_init,
 	.test_cases = cs_dsp_ctl_rw_test_cases_adsp,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_ctl_rw_test_adsp2_16bit_wmfw2 = {
 	.name = "cs_dsp_ctl_rw_wmfwV2_adsp2_16bit",
 	.init = cs_dsp_ctl_rw_test_adsp2_16bit_wmfw2_init,
 	.test_cases = cs_dsp_ctl_rw_test_cases_adsp,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_ctl_rw_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c
index f02cb6cf76386..1de70e81a8687 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw.c
@@ -8,6 +8,7 @@
 
 #include <kunit/device.h>
 #include <kunit/resource.h>
+#include <kunit/static_stub.h>
 #include <kunit/test.h>
 #include <linux/build_bug.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
@@ -1775,6 +1776,15 @@ static void wmfw_load_with_info(struct kunit *test)
 	KUNIT_EXPECT_MEMEQ(test, readback, payload_data, payload_size_bytes);
 }
 
+static bool cs_dsp_wmfw_test_can_emit_message_hook(void)
+{
+#if defined(DEBUG)
+	return true;
+#else
+	return false;
+#endif
+}
+
 static int cs_dsp_wmfw_test_common_init(struct kunit *test, struct cs_dsp *dsp,
 					int wmfw_version)
 {
@@ -1863,16 +1873,12 @@ static int cs_dsp_wmfw_test_common_init(struct kunit *test, struct cs_dsp *dsp,
 	 * The large number of test cases will cause an unusually large amount
 	 * of dev_info() messages from cs_dsp, so suppress these.
 	 */
-	cs_dsp_suppress_info_messages = true;
+	kunit_activate_static_stub(test, cs_dsp_can_emit_message,
+				   cs_dsp_wmfw_test_can_emit_message_hook);
 
 	return 0;
 }
 
-static void cs_dsp_wmfw_test_exit(struct kunit *test)
-{
-	cs_dsp_suppress_info_messages = false;
-}
-
 static int cs_dsp_wmfw_test_halo_init(struct kunit *test)
 {
 	struct cs_dsp *dsp;
@@ -2180,50 +2186,50 @@ static struct kunit_case cs_dsp_wmfw_test_cases_adsp2[] = {
 static struct kunit_suite cs_dsp_wmfw_test_halo = {
 	.name = "cs_dsp_wmfwV3_halo",
 	.init = cs_dsp_wmfw_test_halo_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_halo,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw0 = {
 	.name = "cs_dsp_wmfwV0_adsp2_32bit",
 	.init = cs_dsp_wmfw_test_adsp2_32bit_wmfw0_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw1 = {
 	.name = "cs_dsp_wmfwV1_adsp2_32bit",
 	.init = cs_dsp_wmfw_test_adsp2_32bit_wmfw1_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_32bit_wmfw2 = {
 	.name = "cs_dsp_wmfwV2_adsp2_32bit",
 	.init = cs_dsp_wmfw_test_adsp2_32bit_wmfw2_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw0 = {
 	.name = "cs_dsp_wmfwV0_adsp2_16bit",
 	.init = cs_dsp_wmfw_test_adsp2_16bit_wmfw0_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw1 = {
 	.name = "cs_dsp_wmfwV1_adsp2_16bit",
 	.init = cs_dsp_wmfw_test_adsp2_16bit_wmfw1_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_test_adsp2_16bit_wmfw2 = {
 	.name = "cs_dsp_wmfwV2_adsp2_16bit",
 	.init = cs_dsp_wmfw_test_adsp2_16bit_wmfw2_init,
-	.exit = cs_dsp_wmfw_test_exit,
 	.test_cases = cs_dsp_wmfw_test_cases_adsp2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_wmfw_test_halo,
diff --git a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c
index 37162d12e2fa7..e7bf5dc474f59 100644
--- a/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c
+++ b/drivers/firmware/cirrus/test/cs_dsp_test_wmfw_error.c
@@ -8,6 +8,7 @@
 
 #include <kunit/device.h>
 #include <kunit/resource.h>
+#include <kunit/static_stub.h>
 #include <kunit/test.h>
 #include <linux/build_bug.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
@@ -989,11 +990,13 @@ static void wmfw_v2_coeff_description_exceeds_block(struct kunit *test)
 			-EOVERFLOW);
 }
 
-static void cs_dsp_wmfw_err_test_exit(struct kunit *test)
+static bool cs_dsp_wmfw_err_test_can_emit_message_hook(void)
 {
-	cs_dsp_suppress_err_messages = false;
-	cs_dsp_suppress_warn_messages = false;
-	cs_dsp_suppress_info_messages = false;
+#if defined(DEBUG)
+	return true;
+#else
+	return false;
+#endif
 }
 
 static int cs_dsp_wmfw_err_test_common_init(struct kunit *test, struct cs_dsp *dsp,
@@ -1080,9 +1083,8 @@ static int cs_dsp_wmfw_err_test_common_init(struct kunit *test, struct cs_dsp *d
 	 * Testing error conditions can produce a lot of log output
 	 * from cs_dsp error messages, so suppress messages.
 	 */
-	cs_dsp_suppress_err_messages = true;
-	cs_dsp_suppress_warn_messages = true;
-	cs_dsp_suppress_info_messages = true;
+	kunit_activate_static_stub(test, cs_dsp_can_emit_message,
+				   cs_dsp_wmfw_err_test_can_emit_message_hook);
 
 	return 0;
 }
@@ -1304,50 +1306,50 @@ static struct kunit_case cs_dsp_wmfw_err_test_cases_v3[] = {
 static struct kunit_suite cs_dsp_wmfw_err_test_halo = {
 	.name = "cs_dsp_wmfwV3_err_halo",
 	.init = cs_dsp_wmfw_err_test_halo_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v3,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_32bit_wmfw0 = {
 	.name = "cs_dsp_wmfwV0_err_adsp2_32bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_32bit_wmfw0_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v0,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_32bit_wmfw1 = {
 	.name = "cs_dsp_wmfwV1_err_adsp2_32bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_32bit_wmfw1_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_32bit_wmfw2 = {
 	.name = "cs_dsp_wmfwV2_err_adsp2_32bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_32bit_wmfw2_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_16bit_wmfw0 = {
 	.name = "cs_dsp_wmfwV0_err_adsp2_16bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_16bit_wmfw0_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v0,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_16bit_wmfw1 = {
 	.name = "cs_dsp_wmfwV1_err_adsp2_16bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_16bit_wmfw1_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v1,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 static struct kunit_suite cs_dsp_wmfw_err_test_adsp2_16bit_wmfw2 = {
 	.name = "cs_dsp_wmfwV2_err_adsp2_16bit",
 	.init = cs_dsp_wmfw_err_test_adsp2_16bit_wmfw2_init,
-	.exit = cs_dsp_wmfw_err_test_exit,
 	.test_cases = cs_dsp_wmfw_err_test_cases_v2,
+	.attr.speed = KUNIT_SPEED_SLOW,
 };
 
 kunit_test_suites(&cs_dsp_wmfw_err_test_halo,
diff --git a/drivers/firmware/efi/efi-bgrt.c b/drivers/firmware/efi/efi-bgrt.c
index 6aafdb67dbca4..1da451582812a 100644
--- a/drivers/firmware/efi/efi-bgrt.c
+++ b/drivers/firmware/efi/efi-bgrt.c
@@ -29,11 +29,12 @@ void __init efi_bgrt_init(struct acpi_table_header *table)
 	void *image;
 	struct bmp_header bmp_header;
 	struct acpi_table_bgrt *bgrt = &bgrt_tab;
+	int mem_type;
 
 	if (acpi_disabled)
 		return;
 
-	if (!efi_enabled(EFI_MEMMAP))
+	if (!efi_enabled(EFI_MEMMAP) && !efi_enabled(EFI_PARAVIRT))
 		return;
 
 	if (table->length < sizeof(bgrt_tab)) {
@@ -62,7 +63,9 @@ void __init efi_bgrt_init(struct acpi_table_header *table)
 		goto out;
 	}
 
-	if (efi_mem_type(bgrt->image_address) != EFI_BOOT_SERVICES_DATA) {
+	mem_type = efi_mem_type(bgrt->image_address);
+	if (mem_type != EFI_BOOT_SERVICES_DATA &&
+	    mem_type != EFI_ACPI_RECLAIM_MEMORY) {
 		pr_notice("Ignoring BGRT: invalid image address\n");
 		goto out;
 	}
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index b2fb92a4bbd11..49b32a1aa125c 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -423,7 +423,7 @@ static int __init efisubsys_init(void)
 		 * ordered workqueue (which creates only one execution context)
 		 * should suffice for all our needs.
 		 */
-		efi_rts_wq = alloc_ordered_workqueue("efi_rts_wq", 0);
+		efi_rts_wq = alloc_ordered_workqueue("efi_runtime", WQ_SYSFS);
 		if (!efi_rts_wq) {
 			pr_err("Creating efi_rts_wq failed, EFI runtime services disabled.\n");
 			clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
@@ -983,18 +983,12 @@ char * __init efi_md_typeattr_format(char *buf, size_t size,
  */
 u64 efi_mem_attributes(unsigned long phys_addr)
 {
-	efi_memory_desc_t *md;
+	efi_memory_desc_t md;
 
-	if (!efi_enabled(EFI_MEMMAP))
+	if (efi_mem_desc_lookup(phys_addr, &md))
 		return 0;
 
-	for_each_efi_memory_desc(md) {
-		if ((md->phys_addr <= phys_addr) &&
-		    (phys_addr < (md->phys_addr +
-		    (md->num_pages << EFI_PAGE_SHIFT))))
-			return md->attribute;
-	}
-	return 0;
+	return md.attribute;
 }
 
 /*
@@ -1007,18 +1001,15 @@ u64 efi_mem_attributes(unsigned long phys_addr)
  */
 int efi_mem_type(unsigned long phys_addr)
 {
-	const efi_memory_desc_t *md;
+	efi_memory_desc_t md;
 
-	if (!efi_enabled(EFI_MEMMAP))
+	if (!efi_enabled(EFI_MEMMAP) && !efi_enabled(EFI_PARAVIRT))
 		return -ENOTSUPP;
 
-	for_each_efi_memory_desc(md) {
-		if ((md->phys_addr <= phys_addr) &&
-		    (phys_addr < (md->phys_addr +
-				  (md->num_pages << EFI_PAGE_SHIFT))))
-			return md->type;
-	}
-	return -EINVAL;
+	if (efi_mem_desc_lookup(phys_addr, &md))
+		return -EINVAL;
+
+	return md.type;
 }
 
 int efi_status_to_err(efi_status_t status)
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index e386ffd009b7e..fbb8d4e33256a 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -11,12 +11,11 @@ cflags-y			:= $(KBUILD_CFLAGS)
 
 cflags-$(CONFIG_X86_32)		:= -march=i386
 cflags-$(CONFIG_X86_64)		:= -mcmodel=small
-cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ -std=gnu11 -fms-extensions \
+cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ $(CC_FLAGS_DIALECT) \
 				   -fPIC -fno-strict-aliasing -mno-red-zone \
 				   -mno-mmx -mno-sse -fshort-wchar \
 				   -Wno-pointer-sign \
 				   $(call cc-disable-warning, address-of-packed-member) \
-				   $(if $(CONFIG_CC_IS_CLANG),-Wno-gnu -Wno-microsoft-anon-tag) \
 				   -fno-asynchronous-unwind-tables \
 				   $(CLANG_FLAGS)
 
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index 6a337f1f8787b..23b3543d3041b 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -32,8 +32,8 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size,
 {
 	int node, num_rsv;
 	int status;
-	u32 fdt_val32;
-	u64 fdt_val64;
+	fdt32_t fdt_val32;
+	fdt64_t fdt_val64;
 
 	/* Do some checks on provided FDT, if it exists: */
 	if (orig_fdt) {
@@ -100,13 +100,13 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size,
 	if (status)
 		goto fdt_set_fail;
 
-	fdt_val64 = U64_MAX; /* placeholder */
+	fdt_val64 = cpu_to_fdt64(U64_MAX); /* placeholder */
 
 	status = fdt_setprop_var(fdt, node, "linux,uefi-mmap-start", fdt_val64);
 	if (status)
 		goto fdt_set_fail;
 
-	fdt_val32 = U32_MAX; /* placeholder */
+	fdt_val32 = cpu_to_fdt32(U32_MAX); /* placeholder */
 
 	status = fdt_setprop_var(fdt, node, "linux,uefi-mmap-size", fdt_val32);
 	if (status)
@@ -147,8 +147,8 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size,
 static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map)
 {
 	int node = fdt_path_offset(fdt, "/chosen");
-	u64 fdt_val64;
-	u32 fdt_val32;
+	fdt64_t fdt_val64;
+	fdt32_t fdt_val32;
 	int err;
 
 	if (node < 0)
diff --git a/drivers/firmware/google/Kconfig b/drivers/firmware/google/Kconfig
index 41b78f5cb7351..b78c644fa2539 100644
--- a/drivers/firmware/google/Kconfig
+++ b/drivers/firmware/google/Kconfig
@@ -59,11 +59,12 @@ config GOOGLE_MEMCONSOLE_X86_LEGACY
 
 config GOOGLE_FRAMEBUFFER_COREBOOT
 	tristate "Coreboot Framebuffer"
-	depends on FB_SIMPLE || DRM_SIMPLEDRM
 	depends on GOOGLE_COREBOOT_TABLE
 	help
 	  This option enables the kernel to search for a framebuffer in
-	  the coreboot table.  If found, it is registered with simplefb.
+	  the coreboot table.  If found, it is registered with a platform
+	  device of type coreboot-framebuffer. Using the old device of
+	  type simple-framebuffer is deprecated.
 
 config GOOGLE_MEMCONSOLE_COREBOOT
 	tristate "Firmware Memory Console"
diff --git a/drivers/firmware/google/cbmem.c b/drivers/firmware/google/cbmem.c
index 54c3b8b05e5d6..3397bacdfdbec 100644
--- a/drivers/firmware/google/cbmem.c
+++ b/drivers/firmware/google/cbmem.c
@@ -12,6 +12,7 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/kobject.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c
index 882db32e51be9..c769631ea15d7 100644
--- a/drivers/firmware/google/coreboot_table.c
+++ b/drivers/firmware/google/coreboot_table.c
@@ -14,6 +14,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -21,6 +22,16 @@
 
 #include "coreboot_table.h"
 
+/* Coreboot table header structure */
+struct coreboot_table_header {
+	char signature[4];
+	u32 header_bytes;
+	u32 header_checksum;
+	u32 table_bytes;
+	u32 table_checksum;
+	u32 table_entries;
+};
+
 #define CB_DEV(d) container_of(d, struct coreboot_device, dev)
 #define CB_DRV(d) container_of_const(d, struct coreboot_driver, drv)
 
@@ -251,7 +262,7 @@ static void __exit coreboot_table_driver_exit(void)
 	bus_unregister(&coreboot_bus_type);
 }
 
-module_init(coreboot_table_driver_init);
+subsys_initcall(coreboot_table_driver_init);
 module_exit(coreboot_table_driver_exit);
 
 MODULE_AUTHOR("Google, Inc.");
diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h
index bb6f0f7299b46..616ca3903e5c6 100644
--- a/drivers/firmware/google/coreboot_table.h
+++ b/drivers/firmware/google/coreboot_table.h
@@ -12,65 +12,8 @@
 #ifndef __COREBOOT_TABLE_H
 #define __COREBOOT_TABLE_H
 
+#include <linux/coreboot.h>
 #include <linux/device.h>
-#include <linux/mod_devicetable.h>
-
-/* Coreboot table header structure */
-struct coreboot_table_header {
-	char signature[4];
-	u32 header_bytes;
-	u32 header_checksum;
-	u32 table_bytes;
-	u32 table_checksum;
-	u32 table_entries;
-};
-
-/* List of coreboot entry structures that is used */
-/* Generic */
-struct coreboot_table_entry {
-	u32 tag;
-	u32 size;
-};
-
-/* Points to a CBMEM entry */
-struct lb_cbmem_ref {
-	u32 tag;
-	u32 size;
-
-	u64 cbmem_addr;
-};
-
-#define LB_TAG_CBMEM_ENTRY 0x31
-
-/* Corresponds to LB_TAG_CBMEM_ENTRY */
-struct lb_cbmem_entry {
-	u32 tag;
-	u32 size;
-
-	u64 address;
-	u32 entry_size;
-	u32 id;
-};
-
-/* Describes framebuffer setup by coreboot */
-struct lb_framebuffer {
-	u32 tag;
-	u32 size;
-
-	u64 physical_address;
-	u32 x_resolution;
-	u32 y_resolution;
-	u32 bytes_per_line;
-	u8  bits_per_pixel;
-	u8  red_mask_pos;
-	u8  red_mask_size;
-	u8  green_mask_pos;
-	u8  green_mask_size;
-	u8  blue_mask_pos;
-	u8  blue_mask_size;
-	u8  reserved_mask_pos;
-	u8  reserved_mask_size;
-};
 
 /* A device, additionally with information from coreboot. */
 struct coreboot_device {
diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c
index c68c9f56370f2..2c63a9bd0dcb4 100644
--- a/drivers/firmware/google/framebuffer-coreboot.c
+++ b/drivers/firmware/google/framebuffer-coreboot.c
@@ -12,30 +12,87 @@
 #include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/pci.h>
 #include <linux/platform_data/simplefb.h>
 #include <linux/platform_device.h>
 #include <linux/sysfb.h>
 
 #include "coreboot_table.h"
 
-#define CB_TAG_FRAMEBUFFER 0x12
+#if defined(CONFIG_PCI)
+static bool framebuffer_pci_dev_is_enabled(struct pci_dev *pdev)
+{
+	/*
+	 * TODO: Try to integrate this code into the PCI subsystem
+	 */
+	int ret;
+	u16 command;
+
+	ret = pci_read_config_word(pdev, PCI_COMMAND, &command);
+	if (ret != PCIBIOS_SUCCESSFUL)
+		return false;
+	if (!(command & PCI_COMMAND_MEMORY))
+		return false;
+	return true;
+}
+
+static struct pci_dev *framebuffer_parent_pci_dev(struct resource *res)
+{
+	struct pci_dev *pdev = NULL;
+	const struct resource *r = NULL;
+
+	while (!r && (pdev = pci_get_base_class(PCI_BASE_CLASS_DISPLAY, pdev)))
+		r = pci_find_resource(pdev, res);
+
+	if (!r || !pdev)
+		return NULL; /* not found; not an error */
+
+	if (!framebuffer_pci_dev_is_enabled(pdev)) {
+		pci_dev_put(pdev);
+		return ERR_PTR(-ENODEV);
+	}
 
-static const struct simplefb_format formats[] = SIMPLEFB_FORMATS;
+	return pdev;
+}
+#else
+static struct pci_dev *framebuffer_parent_pci_dev(struct resource *res)
+{
+	return NULL;
+}
+#endif
+
+static struct device *framebuffer_parent_dev(struct resource *res)
+{
+	struct pci_dev *pdev;
+
+	pdev = framebuffer_parent_pci_dev(res);
+	if (IS_ERR(pdev))
+		return ERR_CAST(pdev);
+	else if (pdev)
+		return &pdev->dev;
+
+	return NULL;
+}
 
 static int framebuffer_probe(struct coreboot_device *dev)
 {
-	int i;
-	u32 length;
 	struct lb_framebuffer *fb = &dev->framebuffer;
+	struct device *parent;
 	struct platform_device *pdev;
 	struct resource res;
+	int ret;
+#if !IS_ENABLED(CONFIG_DRM_COREBOOTDRM)
 	struct simplefb_platform_data pdata = {
 		.width = fb->x_resolution,
 		.height = fb->y_resolution,
 		.stride = fb->bytes_per_line,
 		.format = NULL,
 	};
+	int i;
+	static const struct simplefb_format formats[] = SIMPLEFB_FORMATS;
+#endif
 
 	/*
 	 * On coreboot systems, the advertised LB_TAG_FRAMEBUFFER entry
@@ -53,6 +110,29 @@ static int framebuffer_probe(struct coreboot_device *dev)
 	if (!fb->physical_address)
 		return -ENODEV;
 
+	res = DEFINE_RES_MEM(fb->physical_address,
+			     PAGE_ALIGN(fb->y_resolution * fb->bytes_per_line));
+	if (res.end <= res.start)
+		return -EINVAL;
+
+	parent = framebuffer_parent_dev(&res);
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
+
+#if IS_ENABLED(CONFIG_DRM_COREBOOTDRM)
+	pdev = platform_device_register_resndata(parent, "coreboot-framebuffer", 0,
+						 &res, 1, fb, fb->size);
+	if (IS_ERR(pdev)) {
+		pr_warn("coreboot: could not register framebuffer\n");
+		ret = PTR_ERR(pdev);
+		goto out_put_device_parent;
+	}
+#else
+	/*
+	 * FIXME: Coreboot systems should use a driver that binds to
+	 *        coreboot-framebuffer devices. Remove support for
+	 *        simple-framebuffer at some point.
+	 */
 	for (i = 0; i < ARRAY_SIZE(formats); ++i) {
 		if (fb->bits_per_pixel     == formats[i].bits_per_pixel &&
 		    fb->red_mask_pos       == formats[i].red.offset &&
@@ -63,35 +143,28 @@ static int framebuffer_probe(struct coreboot_device *dev)
 		    fb->blue_mask_size     == formats[i].blue.length)
 			pdata.format = formats[i].name;
 	}
-	if (!pdata.format)
-		return -ENODEV;
-
-	memset(&res, 0, sizeof(res));
-	res.flags = IORESOURCE_MEM | IORESOURCE_BUSY;
-	res.name = "Coreboot Framebuffer";
-	res.start = fb->physical_address;
-	length = PAGE_ALIGN(fb->y_resolution * fb->bytes_per_line);
-	res.end = res.start + length - 1;
-	if (res.end <= res.start)
-		return -EINVAL;
+	if (!pdata.format) {
+		ret = -ENODEV;
+		goto out_put_device_parent;
+	}
 
-	pdev = platform_device_register_resndata(&dev->dev,
+	pdev = platform_device_register_resndata(parent,
 						 "simple-framebuffer", 0,
 						 &res, 1, &pdata,
 						 sizeof(pdata));
-	if (IS_ERR(pdev))
+	if (IS_ERR(pdev)) {
+		ret = PTR_ERR(pdev);
 		pr_warn("coreboot: could not register framebuffer\n");
-	else
-		dev_set_drvdata(&dev->dev, pdev);
-
-	return PTR_ERR_OR_ZERO(pdev);
-}
+		goto out_put_device_parent;
+	}
+#endif
 
-static void framebuffer_remove(struct coreboot_device *dev)
-{
-	struct platform_device *pdev = dev_get_drvdata(&dev->dev);
+	ret = 0;
 
-	platform_device_unregister(pdev);
+out_put_device_parent:
+	if (parent)
+		put_device(parent);
+	return ret;
 }
 
 static const struct coreboot_device_id framebuffer_ids[] = {
@@ -102,7 +175,6 @@ MODULE_DEVICE_TABLE(coreboot, framebuffer_ids);
 
 static struct coreboot_driver framebuffer_driver = {
 	.probe = framebuffer_probe,
-	.remove = framebuffer_remove,
 	.drv = {
 		.name = "framebuffer",
 	},
diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c
index c5f08617aa8dd..4aa9b1cad3c35 100644
--- a/drivers/firmware/google/memconsole-coreboot.c
+++ b/drivers/firmware/google/memconsole-coreboot.c
@@ -10,6 +10,7 @@
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 
 #include "memconsole.h"
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c
index 54349e579b0df..dd058291250b6 100644
--- a/drivers/firmware/google/vpd.c
+++ b/drivers/firmware/google/vpd.c
@@ -13,6 +13,7 @@
 #include <linux/kernel.h>
 #include <linux/kobject.h>
 #include <linux/list.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
diff --git a/drivers/firmware/imx/sm-misc.c b/drivers/firmware/imx/sm-misc.c
index 0a8ada329c9de..16b5ff833d212 100644
--- a/drivers/firmware/imx/sm-misc.c
+++ b/drivers/firmware/imx/sm-misc.c
@@ -18,6 +18,29 @@ static const struct scmi_imx_misc_proto_ops *imx_misc_ctrl_ops;
 static struct scmi_protocol_handle *ph;
 struct notifier_block scmi_imx_misc_ctrl_nb;
 
+static const char * const rst_imx95[] = {
+	"cm33_lockup", "cm33_swreq", "cm7_lockup", "cm7_swreq", "fccu",
+	"jtag_sw", "ele", "tempsense", "wdog1", "wdog2", "wdog3", "wdog4",
+	"wdog5", "jtag", "cm33_exc", "bbm", "sw", "sm_err", "fusa_sreco",
+	"pmic", "unused", "unused", "unused", "unused", "unused", "unused",
+	"unused", "unused", "unused", "unused", "unused", "por",
+};
+
+static const char * const rst_imx94[] = {
+	"cm33_lockup", "cm33_swreq", "cm70_lockup", "cm70_swreq", "fccu",
+	"jtag_sw", "ele", "tempsense", "wdog1", "wdog2", "wdog3", "wdog4",
+	"wdog5", "jtag", "wdog6", "wdog7", "wdog8", "wo_netc", "cm33s_lockup",
+	"cm33s_swreq", "cm71_lockup", "cm71_swreq", "cm33_exc", "bbm", "sw",
+	"sm_err", "fusa_sreco", "pmic", "unused", "unused", "unused", "por",
+};
+
+static const struct of_device_id allowlist[] = {
+	{ .compatible = "fsl,imx952", .data = rst_imx95 },
+	{ .compatible = "fsl,imx95", .data = rst_imx95 },
+	{ .compatible = "fsl,imx94", .data = rst_imx94 },
+	{ /* Sentinel */ }
+};
+
 int scmi_imx_misc_ctrl_set(u32 id, u32 val)
 {
 	if (!ph)
@@ -75,6 +98,54 @@ static void scmi_imx_misc_put(void *p)
 	debugfs_remove((struct dentry *)p);
 }
 
+static int scmi_imx_misc_get_reason(struct scmi_device *sdev)
+{
+	struct scmi_imx_misc_reset_reason boot, shutdown;
+	const char **rst;
+	bool system = true;
+	int ret;
+
+	if (!of_machine_device_match(allowlist))
+		return 0;
+
+	rst = (const char **)of_machine_get_match_data(allowlist);
+
+	ret = imx_misc_ctrl_ops->misc_reset_reason(ph, system, &boot, &shutdown, NULL);
+	if (!ret) {
+		if (boot.valid)
+			dev_info(&sdev->dev, "%s Boot reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[boot.reason],
+				 boot.orig_valid ? boot.origin : -1,
+				 boot.err_valid ? boot.errid : -1);
+		if (shutdown.valid)
+			dev_info(&sdev->dev, "%s shutdown reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[shutdown.reason],
+				 shutdown.orig_valid ? shutdown.origin : -1,
+				 shutdown.err_valid ? shutdown.errid : -1);
+	} else {
+		dev_err(&sdev->dev, "Failed to get system reset reason: %d\n", ret);
+	}
+
+	system = false;
+	ret = imx_misc_ctrl_ops->misc_reset_reason(ph, system, &boot, &shutdown, NULL);
+	if (!ret) {
+		if (boot.valid)
+			dev_info(&sdev->dev, "%s Boot reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[boot.reason],
+				 boot.orig_valid ? boot.origin : -1,
+				 boot.err_valid ? boot.errid : -1);
+		if (shutdown.valid)
+			dev_info(&sdev->dev, "%s shutdown reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[shutdown.reason],
+				 shutdown.orig_valid ? shutdown.origin : -1,
+				 shutdown.err_valid ? shutdown.errid : -1);
+	} else {
+		dev_err(&sdev->dev, "Failed to get lm reset reason: %d\n", ret);
+	}
+
+	return 0;
+}
+
 static int scmi_imx_misc_ctrl_probe(struct scmi_device *sdev)
 {
 	const struct scmi_handle *handle = sdev->handle;
@@ -133,6 +204,8 @@ static int scmi_imx_misc_ctrl_probe(struct scmi_device *sdev)
 	scmi_imx_dentry = debugfs_create_dir("scmi_imx", NULL);
 	debugfs_create_file("syslog", 0444, scmi_imx_dentry, &sdev->dev, &syslog_fops);
 
+	scmi_imx_misc_get_reason(sdev);
+
 	return devm_add_action_or_reset(&sdev->dev, scmi_imx_misc_put, scmi_imx_dentry);
 }
 
diff --git a/drivers/firmware/microchip/mpfs-auto-update.c b/drivers/firmware/microchip/mpfs-auto-update.c
index 46b19d803446e..1211fd8d04633 100644
--- a/drivers/firmware/microchip/mpfs-auto-update.c
+++ b/drivers/firmware/microchip/mpfs-auto-update.c
@@ -113,10 +113,6 @@ static enum fw_upload_err mpfs_auto_update_prepare(struct fw_upload *fw_uploader
 	 * be added here.
 	 */
 
-	priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller);
-	if (!priv->flash)
-		return FW_UPLOAD_ERR_HW_ERROR;
-
 	erase_size = round_up(erase_size, (u64)priv->flash->erasesize);
 
 	/*
@@ -427,6 +423,12 @@ static int mpfs_auto_update_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(priv->sys_controller),
 				     "Could not register as a sub device of the system controller\n");
 
+	priv->flash = mpfs_sys_controller_get_flash(priv->sys_controller);
+	if (IS_ERR_OR_NULL(priv->flash)) {
+		dev_dbg(dev, "No flash connected to the system controller, auto-update not supported\n");
+		return -ENODEV;
+	}
+
 	priv->dev = dev;
 	platform_set_drvdata(pdev, priv);
 
diff --git a/drivers/firmware/qcom/qcom_qseecom_uefisecapp.c b/drivers/firmware/qcom/qcom_qseecom_uefisecapp.c
index 98a463e9774bf..befa68d1dcaf4 100644
--- a/drivers/firmware/qcom/qcom_qseecom_uefisecapp.c
+++ b/drivers/firmware/qcom/qcom_qseecom_uefisecapp.c
@@ -699,20 +699,18 @@ static DEFINE_MUTEX(__qcuefi_lock);
 
 static int qcuefi_set_reference(struct qcuefi_client *qcuefi)
 {
-	mutex_lock(&__qcuefi_lock);
+	guard(mutex)(&__qcuefi_lock);
 
-	if (qcuefi && __qcuefi) {
-		mutex_unlock(&__qcuefi_lock);
+	if (qcuefi && __qcuefi)
 		return -EEXIST;
-	}
 
 	__qcuefi = qcuefi;
 
-	mutex_unlock(&__qcuefi_lock);
 	return 0;
 }
 
 static struct qcuefi_client *qcuefi_acquire(void)
+	__acquires(__qcuefi_lock)
 {
 	mutex_lock(&__qcuefi_lock);
 	if (!__qcuefi) {
@@ -723,6 +721,7 @@ static struct qcuefi_client *qcuefi_acquire(void)
 }
 
 static void qcuefi_release(void)
+	__releases(__qcuefi_lock)
 {
 	mutex_unlock(&__qcuefi_lock);
 }
diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c
index 8fbc96693a55f..33333384ac81e 100644
--- a/drivers/firmware/qcom/qcom_scm.c
+++ b/drivers/firmware/qcom/qcom_scm.c
@@ -199,19 +199,18 @@ static int qcom_scm_bw_enable(void)
 	if (!__scm->path)
 		return 0;
 
-	mutex_lock(&__scm->scm_bw_lock);
+	guard(mutex)(&__scm->scm_bw_lock);
+
 	if (!__scm->scm_vote_count) {
 		ret = icc_set_bw(__scm->path, 0, UINT_MAX);
 		if (ret < 0) {
 			dev_err(__scm->dev, "failed to set bandwidth request\n");
-			goto err_bw;
+			return ret;
 		}
 	}
 	__scm->scm_vote_count++;
-err_bw:
-	mutex_unlock(&__scm->scm_bw_lock);
 
-	return ret;
+	return 0;
 }
 
 static void qcom_scm_bw_disable(void)
@@ -2290,11 +2289,13 @@ EXPORT_SYMBOL_GPL(qcom_scm_qseecom_app_send);
  */
 static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "asus,vivobook-s15" },
+	{ .compatible = "asus,vivobook-s15-x1p4" },
 	{ .compatible = "asus,zenbook-a14-ux3407qa" },
 	{ .compatible = "asus,zenbook-a14-ux3407ra" },
 	{ .compatible = "dell,inspiron-14-plus-7441" },
 	{ .compatible = "dell,latitude-7455" },
 	{ .compatible = "dell,xps13-9345" },
+	{ .compatible = "ecs,liva-qc710" },
 	{ .compatible = "hp,elitebook-ultra-g1q" },
 	{ .compatible = "hp,omnibook-x14" },
 	{ .compatible = "huawei,gaokun3" },
@@ -2309,7 +2310,9 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "microsoft,denali", },
 	{ .compatible = "microsoft,romulus13", },
 	{ .compatible = "microsoft,romulus15", },
+	{ .compatible = "qcom,glymur-crd" },
 	{ .compatible = "qcom,hamoa-iot-evk" },
+	{ .compatible = "qcom,purwa-iot-evk" },
 	{ .compatible = "qcom,sc8180x-primus" },
 	{ .compatible = "qcom,x1e001de-devkit" },
 	{ .compatible = "qcom,x1e80100-crd" },
diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.c b/drivers/firmware/samsung/exynos-acpm-dvfs.c
index 1c5b2b143bcc3..06bdf62dea1f3 100644
--- a/drivers/firmware/samsung/exynos-acpm-dvfs.c
+++ b/drivers/firmware/samsung/exynos-acpm-dvfs.c
@@ -5,6 +5,7 @@
  * Copyright 2025 Linaro Ltd.
  */
 
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
 #include <linux/firmware/samsung/exynos-acpm-protocol.h>
 #include <linux/ktime.h>
@@ -24,12 +25,12 @@ static void acpm_dvfs_set_xfer(struct acpm_xfer *xfer, u32 *cmd, size_t cmdlen,
 			       unsigned int acpm_chan_id, bool response)
 {
 	xfer->acpm_chan_id = acpm_chan_id;
+	xfer->txcnt = cmdlen;
 	xfer->txd = cmd;
-	xfer->txlen = cmdlen;
 
 	if (response) {
+		xfer->rxcnt = cmdlen;
 		xfer->rxd = cmd;
-		xfer->rxlen = cmdlen;
 	}
 }
 
@@ -42,7 +43,7 @@ static void acpm_dvfs_init_set_rate_cmd(u32 cmd[4], unsigned int clk_id,
 	cmd[3] = ktime_to_ms(ktime_get());
 }
 
-int acpm_dvfs_set_rate(const struct acpm_handle *handle,
+int acpm_dvfs_set_rate(struct acpm_handle *handle,
 		       unsigned int acpm_chan_id, unsigned int clk_id,
 		       unsigned long rate)
 {
@@ -50,7 +51,7 @@ int acpm_dvfs_set_rate(const struct acpm_handle *handle,
 	u32 cmd[4];
 
 	acpm_dvfs_init_set_rate_cmd(cmd, clk_id, rate);
-	acpm_dvfs_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id, false);
+	acpm_dvfs_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id, false);
 
 	return acpm_do_xfer(handle, &xfer);
 }
@@ -62,7 +63,7 @@ static void acpm_dvfs_init_get_rate_cmd(u32 cmd[4], unsigned int clk_id)
 	cmd[3] = ktime_to_ms(ktime_get());
 }
 
-unsigned long acpm_dvfs_get_rate(const struct acpm_handle *handle,
+unsigned long acpm_dvfs_get_rate(struct acpm_handle *handle,
 				 unsigned int acpm_chan_id, unsigned int clk_id)
 {
 	struct acpm_xfer xfer;
@@ -70,7 +71,7 @@ unsigned long acpm_dvfs_get_rate(const struct acpm_handle *handle,
 	int ret;
 
 	acpm_dvfs_init_get_rate_cmd(cmd, clk_id);
-	acpm_dvfs_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id, true);
+	acpm_dvfs_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id, true);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.h b/drivers/firmware/samsung/exynos-acpm-dvfs.h
index 9f2778e649c9d..b37b154261029 100644
--- a/drivers/firmware/samsung/exynos-acpm-dvfs.h
+++ b/drivers/firmware/samsung/exynos-acpm-dvfs.h
@@ -11,10 +11,10 @@
 
 struct acpm_handle;
 
-int acpm_dvfs_set_rate(const struct acpm_handle *handle,
+int acpm_dvfs_set_rate(struct acpm_handle *handle,
 		       unsigned int acpm_chan_id, unsigned int id,
 		       unsigned long rate);
-unsigned long acpm_dvfs_get_rate(const struct acpm_handle *handle,
+unsigned long acpm_dvfs_get_rate(struct acpm_handle *handle,
 				 unsigned int acpm_chan_id,
 				 unsigned int clk_id);
 
diff --git a/drivers/firmware/samsung/exynos-acpm-pmic.c b/drivers/firmware/samsung/exynos-acpm-pmic.c
index 961d7599e4224..0c50993cc9a83 100644
--- a/drivers/firmware/samsung/exynos-acpm-pmic.c
+++ b/drivers/firmware/samsung/exynos-acpm-pmic.c
@@ -41,7 +41,7 @@ static const int acpm_pmic_linux_errmap[] = {
 	[2] = -EACCES, /* Write register can't be accessed or issues to access it. */
 };
 
-static int acpm_pmic_to_linux_err(int err)
+static int acpm_pmic_to_linux_err(unsigned int err)
 {
 	if (err >= 0 && err < ARRAY_SIZE(acpm_pmic_linux_errmap))
 		return acpm_pmic_linux_errmap[err];
@@ -63,8 +63,8 @@ static void acpm_pmic_set_xfer(struct acpm_xfer *xfer, u32 *cmd, size_t cmdlen,
 {
 	xfer->txd = cmd;
 	xfer->rxd = cmd;
-	xfer->txlen = cmdlen;
-	xfer->rxlen = cmdlen;
+	xfer->txcnt = cmdlen;
+	xfer->rxcnt = cmdlen;
 	xfer->acpm_chan_id = acpm_chan_id;
 }
 
@@ -77,7 +77,7 @@ static void acpm_pmic_init_read_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan)
 	cmd[3] = ktime_to_ms(ktime_get());
 }
 
-int acpm_pmic_read_reg(const struct acpm_handle *handle,
+int acpm_pmic_read_reg(struct acpm_handle *handle,
 		       unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 		       u8 *buf)
 {
@@ -86,7 +86,7 @@ int acpm_pmic_read_reg(const struct acpm_handle *handle,
 	int ret;
 
 	acpm_pmic_init_read_cmd(cmd, type, reg, chan);
-	acpm_pmic_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id);
+	acpm_pmic_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
@@ -107,7 +107,7 @@ static void acpm_pmic_init_bulk_read_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan,
 		 FIELD_PREP(ACPM_PMIC_VALUE, count);
 }
 
-int acpm_pmic_bulk_read(const struct acpm_handle *handle,
+int acpm_pmic_bulk_read(struct acpm_handle *handle,
 			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			u8 count, u8 *buf)
 {
@@ -119,7 +119,7 @@ int acpm_pmic_bulk_read(const struct acpm_handle *handle,
 		return -EINVAL;
 
 	acpm_pmic_init_bulk_read_cmd(cmd, type, reg, chan, count);
-	acpm_pmic_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id);
+	acpm_pmic_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
@@ -150,7 +150,7 @@ static void acpm_pmic_init_write_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan,
 	cmd[3] = ktime_to_ms(ktime_get());
 }
 
-int acpm_pmic_write_reg(const struct acpm_handle *handle,
+int acpm_pmic_write_reg(struct acpm_handle *handle,
 			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			u8 value)
 {
@@ -159,7 +159,7 @@ int acpm_pmic_write_reg(const struct acpm_handle *handle,
 	int ret;
 
 	acpm_pmic_init_write_cmd(cmd, type, reg, chan, value);
-	acpm_pmic_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id);
+	acpm_pmic_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
@@ -187,7 +187,7 @@ static void acpm_pmic_init_bulk_write_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan,
 	}
 }
 
-int acpm_pmic_bulk_write(const struct acpm_handle *handle,
+int acpm_pmic_bulk_write(struct acpm_handle *handle,
 			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			 u8 count, const u8 *buf)
 {
@@ -199,7 +199,7 @@ int acpm_pmic_bulk_write(const struct acpm_handle *handle,
 		return -EINVAL;
 
 	acpm_pmic_init_bulk_write_cmd(cmd, type, reg, chan, count, buf);
-	acpm_pmic_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id);
+	acpm_pmic_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
@@ -220,7 +220,7 @@ static void acpm_pmic_init_update_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan,
 	cmd[3] = ktime_to_ms(ktime_get());
 }
 
-int acpm_pmic_update_reg(const struct acpm_handle *handle,
+int acpm_pmic_update_reg(struct acpm_handle *handle,
 			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			 u8 value, u8 mask)
 {
@@ -229,7 +229,7 @@ int acpm_pmic_update_reg(const struct acpm_handle *handle,
 	int ret;
 
 	acpm_pmic_init_update_cmd(cmd, type, reg, chan, value, mask);
-	acpm_pmic_set_xfer(&xfer, cmd, sizeof(cmd), acpm_chan_id);
+	acpm_pmic_set_xfer(&xfer, cmd, ARRAY_SIZE(cmd), acpm_chan_id);
 
 	ret = acpm_do_xfer(handle, &xfer);
 	if (ret)
diff --git a/drivers/firmware/samsung/exynos-acpm-pmic.h b/drivers/firmware/samsung/exynos-acpm-pmic.h
index 078421888a140..88ae9aada2aea 100644
--- a/drivers/firmware/samsung/exynos-acpm-pmic.h
+++ b/drivers/firmware/samsung/exynos-acpm-pmic.h
@@ -11,19 +11,19 @@
 
 struct acpm_handle;
 
-int acpm_pmic_read_reg(const struct acpm_handle *handle,
+int acpm_pmic_read_reg(struct acpm_handle *handle,
 		       unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 		       u8 *buf);
-int acpm_pmic_bulk_read(const struct acpm_handle *handle,
+int acpm_pmic_bulk_read(struct acpm_handle *handle,
 			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			u8 count, u8 *buf);
-int acpm_pmic_write_reg(const struct acpm_handle *handle,
+int acpm_pmic_write_reg(struct acpm_handle *handle,
 			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			u8 value);
-int acpm_pmic_bulk_write(const struct acpm_handle *handle,
+int acpm_pmic_bulk_write(struct acpm_handle *handle,
 			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			 u8 count, const u8 *buf);
-int acpm_pmic_update_reg(const struct acpm_handle *handle,
+int acpm_pmic_update_reg(struct acpm_handle *handle,
 			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
 			 u8 value, u8 mask);
 #endif /* __EXYNOS_ACPM_PMIC_H__ */
diff --git a/drivers/firmware/samsung/exynos-acpm.c b/drivers/firmware/samsung/exynos-acpm.c
index 0cb269c704601..16c46ed608371 100644
--- a/drivers/firmware/samsung/exynos-acpm.c
+++ b/drivers/firmware/samsung/exynos-acpm.c
@@ -205,7 +205,7 @@ static void acpm_get_saved_rx(struct acpm_chan *achan,
 	rx_seqnum = FIELD_GET(ACPM_PROTOCOL_SEQNUM, rx_data->cmd[0]);
 
 	if (rx_seqnum == tx_seqnum) {
-		memcpy(xfer->rxd, rx_data->cmd, xfer->rxlen);
+		memcpy(xfer->rxd, rx_data->cmd, xfer->rxcnt * sizeof(*xfer->rxd));
 		clear_bit(rx_seqnum - 1, achan->bitmap_seqnum);
 	}
 }
@@ -258,8 +258,7 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
 
 		if (rx_data->response) {
 			if (rx_seqnum == tx_seqnum) {
-				__ioread32_copy(xfer->rxd, addr,
-						xfer->rxlen / 4);
+				__ioread32_copy(xfer->rxd, addr, xfer->rxcnt);
 				rx_set = true;
 				clear_bit(seqnum, achan->bitmap_seqnum);
 			} else {
@@ -269,8 +268,7 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
 				 * clear yet the bitmap. It will be cleared
 				 * after the response is copied to the request.
 				 */
-				__ioread32_copy(rx_data->cmd, addr,
-						xfer->rxlen / 4);
+				__ioread32_copy(rx_data->cmd, addr, xfer->rxcnt);
 			}
 		} else {
 			clear_bit(seqnum, achan->bitmap_seqnum);
@@ -412,7 +410,7 @@ static int acpm_wait_for_message_response(struct acpm_chan *achan,
  *
  * Return: 0 on success, -errno otherwise.
  */
-int acpm_do_xfer(const struct acpm_handle *handle, const struct acpm_xfer *xfer)
+int acpm_do_xfer(struct acpm_handle *handle, const struct acpm_xfer *xfer)
 {
 	struct acpm_info *acpm = handle_to_acpm_info(handle);
 	struct exynos_mbox_msg msg;
@@ -425,7 +423,9 @@ int acpm_do_xfer(const struct acpm_handle *handle, const struct acpm_xfer *xfer)
 
 	achan = &acpm->chans[xfer->acpm_chan_id];
 
-	if (!xfer->txd || xfer->txlen > achan->mlen || xfer->rxlen > achan->mlen)
+	if (!xfer->txd ||
+	    (xfer->txcnt * sizeof(*xfer->txd) > achan->mlen) ||
+	    (xfer->rxcnt * sizeof(*xfer->rxd) > achan->mlen))
 		return -EINVAL;
 
 	if (!achan->poll_completion) {
@@ -448,7 +448,7 @@ int acpm_do_xfer(const struct acpm_handle *handle, const struct acpm_xfer *xfer)
 
 		/* Write TX command. */
 		__iowrite32_copy(achan->tx.base + achan->mlen * tx_front,
-				 xfer->txd, xfer->txlen / 4);
+				 xfer->txd, xfer->txcnt);
 
 		/* Advance TX front. */
 		writel(idx, achan->tx.front);
@@ -674,7 +674,7 @@ static int acpm_probe(struct platform_device *pdev)
  * acpm_handle_put() - release the handle acquired by acpm_get_by_phandle.
  * @handle:	Handle acquired by acpm_get_by_phandle.
  */
-static void acpm_handle_put(const struct acpm_handle *handle)
+static void acpm_handle_put(struct acpm_handle *handle)
 {
 	struct acpm_info *acpm = handle_to_acpm_info(handle);
 	struct device *dev = acpm->dev;
@@ -700,9 +700,11 @@ static void devm_acpm_release(struct device *dev, void *res)
  * @np:		ACPM device tree node.
  *
  * Return: pointer to handle on success, ERR_PTR(-errno) otherwise.
+ *
+ * Note: handle CANNOT be pointer to const
  */
-static const struct acpm_handle *acpm_get_by_node(struct device *dev,
-						  struct device_node *np)
+static struct acpm_handle *acpm_get_by_node(struct device *dev,
+					    struct device_node *np)
 {
 	struct platform_device *pdev;
 	struct device_link *link;
@@ -743,10 +745,10 @@ static const struct acpm_handle *acpm_get_by_node(struct device *dev,
  *
  * Return: pointer to handle on success, ERR_PTR(-errno) otherwise.
  */
-const struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
-						struct device_node *np)
+struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
+					  struct device_node *np)
 {
-	const struct acpm_handle **ptr, *handle;
+	struct acpm_handle **ptr, *handle;
 
 	ptr = devres_alloc(devm_acpm_release, sizeof(*ptr), GFP_KERNEL);
 	if (!ptr)
diff --git a/drivers/firmware/samsung/exynos-acpm.h b/drivers/firmware/samsung/exynos-acpm.h
index 2d14cb58f98c9..5df8354dc96ce 100644
--- a/drivers/firmware/samsung/exynos-acpm.h
+++ b/drivers/firmware/samsung/exynos-acpm.h
@@ -8,16 +8,16 @@
 #define __EXYNOS_ACPM_H__
 
 struct acpm_xfer {
-	const u32 *txd;
-	u32 *rxd;
-	size_t txlen;
-	size_t rxlen;
+	const u32 *txd __counted_by_ptr(txcnt);
+	u32 *rxd __counted_by_ptr(rxcnt);
+	size_t txcnt;
+	size_t rxcnt;
 	unsigned int acpm_chan_id;
 };
 
 struct acpm_handle;
 
-int acpm_do_xfer(const struct acpm_handle *handle,
+int acpm_do_xfer(struct acpm_handle *handle,
 		 const struct acpm_xfer *xfer);
 
 #endif /* __EXYNOS_ACPM_H__ */
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
index ca68c38aa4a14..8c275bd48a0d9 100644
--- a/drivers/fpga/fpga-bridge.c
+++ b/drivers/fpga/fpga-bridge.c
@@ -290,7 +290,7 @@ static ssize_t name_show(struct device *dev,
 {
 	struct fpga_bridge *bridge = to_fpga_bridge(dev);
 
-	return sprintf(buf, "%s\n", bridge->name);
+	return sysfs_emit(buf, "%s\n", bridge->name);
 }
 
 static ssize_t state_show(struct device *dev,
diff --git a/drivers/gpib/agilent_82357a/agilent_82357a.c b/drivers/gpib/agilent_82357a/agilent_82357a.c
index e1349afbf9333..770ba6eb40d1d 100644
--- a/drivers/gpib/agilent_82357a/agilent_82357a.c
+++ b/drivers/gpib/agilent_82357a/agilent_82357a.c
@@ -1479,7 +1479,7 @@ static int agilent_82357a_driver_probe(struct usb_interface *interface,
 
 	if (mutex_lock_interruptible(&agilent_82357a_hotplug_lock))
 		return -ERESTARTSYS;
-	usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	usb_dev = interface_to_usbdev(interface);
 	for (i = 0; i < MAX_NUM_82357A_INTERFACES; ++i) {
 		if (!agilent_82357a_driver_interfaces[i]) {
 			agilent_82357a_driver_interfaces[i] = interface;
@@ -1490,14 +1490,12 @@ static int agilent_82357a_driver_probe(struct usb_interface *interface,
 		}
 	}
 	if (i == MAX_NUM_82357A_INTERFACES) {
-		usb_put_dev(usb_dev);
 		mutex_unlock(&agilent_82357a_hotplug_lock);
 		dev_err(&usb_dev->dev, "out of space in agilent_82357a_driver_interfaces[]\n");
 		return -1;
 	}
 	path = kmalloc(path_length, GFP_KERNEL);
 	if (!path) {
-		usb_put_dev(usb_dev);
 		mutex_unlock(&agilent_82357a_hotplug_lock);
 		return -ENOMEM;
 	}
@@ -1539,7 +1537,6 @@ static void agilent_82357a_driver_disconnect(struct usb_interface *interface)
 	}
 	if (i == MAX_NUM_82357A_INTERFACES)
 		dev_err(&usb_dev->dev, "unable to find interface - bug?\n");
-	usb_put_dev(usb_dev);
 
 	mutex_unlock(&agilent_82357a_hotplug_lock);
 }
diff --git a/drivers/gpib/common/gpib_os.c b/drivers/gpib/common/gpib_os.c
index be757db993a56..0a8b981fc5791 100644
--- a/drivers/gpib/common/gpib_os.c
+++ b/drivers/gpib/common/gpib_os.c
@@ -2169,10 +2169,13 @@ void init_gpib_status_queue(struct gpib_status_queue *device)
 	device->dropped_byte = 0;
 }
 
-static struct class *gpib_class;
+static const struct class gpib_class = {
+	.name	= "gpib_common",
+};
 
 static int __init gpib_common_init_module(void)
 {
+	int err;
 	int i;
 
 	pr_info("GPIB core driver\n");
@@ -2181,14 +2184,14 @@ static int __init gpib_common_init_module(void)
 		pr_err("gpib: can't get major %d\n", GPIB_CODE);
 		return -EIO;
 	}
-	gpib_class = class_create("gpib_common");
-	if (IS_ERR(gpib_class)) {
+	err = class_register(&gpib_class);
+	if (err) {
 		pr_err("gpib: failed to create gpib class\n");
 		unregister_chrdev(GPIB_CODE, "gpib");
-		return PTR_ERR(gpib_class);
+		return err;
 	}
 	for (i = 0; i < GPIB_MAX_NUM_BOARDS; ++i)
-		board_array[i].gpib_dev = device_create(gpib_class, NULL,
+		board_array[i].gpib_dev = device_create(&gpib_class, NULL,
 							MKDEV(GPIB_CODE, i), NULL, "gpib%i", i);
 
 	return 0;
@@ -2199,9 +2202,9 @@ static void __exit gpib_common_exit_module(void)
 	int i;
 
 	for (i = 0; i < GPIB_MAX_NUM_BOARDS; ++i)
-		device_destroy(gpib_class, MKDEV(GPIB_CODE, i));
+		device_destroy(&gpib_class, MKDEV(GPIB_CODE, i));
 
-	class_destroy(gpib_class);
+	class_unregister(&gpib_class);
 	unregister_chrdev(GPIB_CODE, "gpib");
 }
 
diff --git a/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c b/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c
index ee781d2f0b8e1..389dde5a8481e 100644
--- a/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c
+++ b/drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c
@@ -51,10 +51,10 @@ MODULE_DESCRIPTION("GPIB driver for LPVO usb devices");
  *
  */
 
-static const struct usb_device_id skel_table[] = {
+static const struct usb_device_id lpvo_table[] = {
 	{ }					   /* Terminating entry */
 };
-MODULE_DEVICE_TABLE(usb, skel_table);
+MODULE_DEVICE_TABLE(usb, lpvo_table);
 
 /*
  *   ***  Diagnostics and Debug  ***
@@ -182,15 +182,11 @@ static int usb_minors[MAX_DEV];			   /* usb minors */
 static int assigned_usb_minors;		   /* mask of filled slots */
 static struct mutex minors_lock;     /* operations on usb_minors are to be protected */
 
-/*
- * usb-skeleton prototypes
- */
-
-struct usb_skel;
-static ssize_t skel_do_write(struct usb_skel *, const char *, size_t);
-static ssize_t skel_do_read(struct usb_skel *, char *, size_t);
-static int skel_do_open(struct gpib_board *, int);
-static int skel_do_release(struct gpib_board *);
+struct lpvo;
+static ssize_t lpvo_do_write(struct lpvo *, const char *, size_t);
+static ssize_t lpvo_do_read(struct lpvo *, char *, size_t);
+static int lpvo_do_open(struct gpib_board *, int);
+static int lpvo_do_release(struct gpib_board *);
 
 /*
  *  usec_diff : take difference in MICROsec between two 'timespec'
@@ -218,7 +214,7 @@ static inline int usec_diff(struct timespec64 *a, struct timespec64 *b)
 
 static int write_loop(void *dev, char *msg, int leng)
 {
-	return skel_do_write(dev, msg, leng);
+	return lpvo_do_write(dev, msg, leng);
 }
 
 /**
@@ -246,7 +242,7 @@ static int send_command(struct gpib_board *board, char *msg, int leng)
 	if (retval < 0)
 		return retval;
 
-	nchar = skel_do_read(GPIB_DEV, buffer, 64);
+	nchar = lpvo_do_read(GPIB_DEV, buffer, 64);
 
 	if (nchar < 0) {
 		dev_err(board->gpib_dev, " return from read: %d\n", nchar);
@@ -310,7 +306,7 @@ static int one_char(struct gpib_board *board, struct char_buf *b)
 		return b->inbuf[b->last - b->nchar--];
 	}
 	ktime_get_real_ts64 (&before);
-	b->nchar = skel_do_read(GPIB_DEV, b->inbuf, INBUF_SIZE);
+	b->nchar = lpvo_do_read(GPIB_DEV, b->inbuf, INBUF_SIZE);
 	b->last = b->nchar;
 	ktime_get_real_ts64 (&after);
 
@@ -445,12 +441,12 @@ static int usb_gpib_attach(struct gpib_board *board, const struct gpib_board_con
 	if (!board->private_data)
 		return -ENOMEM;
 
-	retval = skel_do_open(board, usb_minors[j]);
+	retval = lpvo_do_open(board, usb_minors[j]);
 
-	DIA_LOG(1, "Skel open: %d\n", retval);
+	DIA_LOG(1, "lpvo open: %d\n", retval);
 
 	if (retval) {
-		dev_err(board->gpib_dev, "skel open failed.\n");
+		dev_err(board->gpib_dev, "lpvo open failed.\n");
 		kfree(board->private_data);
 		board->private_data = NULL;
 		return -ENODEV;
@@ -517,8 +513,8 @@ static void usb_gpib_detach(struct gpib_board *board)
 			write_loop(GPIB_DEV, USB_GPIB_OFF, strlen(USB_GPIB_OFF));
 			msleep(100);
 			DIA_LOG(1, "%s", "GPIB off\n");
-			retval = skel_do_release(board);
-			DIA_LOG(1, "skel release -> %d\n", retval);
+			retval = lpvo_do_release(board);
+			DIA_LOG(1, "lpvo release -> %d\n", retval);
 		}
 		kfree(board->private_data);
 		board->private_data = NULL;
@@ -772,8 +768,8 @@ static int usb_gpib_read(struct gpib_board *board,
 		if (retval < 0)
 			return retval;
 
-		retval = skel_do_read(GPIB_DEV, inbuf, 1);
-		retval += skel_do_read(GPIB_DEV, inbuf + 1, 1);
+		retval = lpvo_do_read(GPIB_DEV, inbuf, 1);
+		retval += lpvo_do_read(GPIB_DEV, inbuf + 1, 1);
 
 		ktime_get_real_ts64 (&after);
 
@@ -1197,12 +1193,12 @@ static int write_latency_timer(struct usb_device *udev)
  *  written by Greg Kroah-Hartman and available in the kernel tree.	     *
  *									     *
  *  Functions skel_open() and skel_release() have been rewritten and named   *
- *  skel_do_open() and skel_do_release() to process the attach and detach    *
+ *  lpvo_do_open() and lpvo_do_release() to process the attach and detach    *
  *  requests coming from gpib_config.					     *
  *									     *
- *  Functions skel_read() and skel_write() have been split into a	     *
- *  skel_do_read() and skel_do_write(), that cover the kernel stuff of read  *
- *  and write operations, and the original skel_read() and skel_write(),     *
+ *  Functions lpvo_read() and lpvo_write() have been split into a	     *
+ *  lpvo_do_read() and lpvo_do_write(), that cover the kernel stuff of read  *
+ *  and write operations, and the original lpvo_read() and lpvo_write(),     *
  *  that handle communication with user space and call their _do_ companion. *
  *									     *
  *  Only the _do_ versions are used by the lpvo_usb_gpib driver; other ones  *
@@ -1230,7 +1226,7 @@ static int write_latency_timer(struct usb_device *udev)
 #include <linux/mutex.h>
 
 /* Get a minor range for your devices from the usb maintainer */
-#define USB_SKEL_MINOR_BASE	   192
+#define USB_LPVO_MINOR_BASE	   192
 
 /*   private defines   */
 
@@ -1245,7 +1241,7 @@ static int write_latency_timer(struct usb_device *udev)
 #define USER_DEVICE 1		      /* compile for device(s) in user space */
 
 /* Structure to hold all of our device specific stuff */
-struct usb_skel {
+struct lpvo {
 	struct usb_device     *udev;		     /* the usb device for this device */
 	struct usb_interface  *interface;	     /* the interface for this device */
 	struct semaphore      limit_sem;	     /* limiting the number of writes in progress */
@@ -1265,14 +1261,14 @@ struct usb_skel {
 	wait_queue_head_t     bulk_in_wait;	     /* to wait for an ongoing read */
 };
 
-#define to_skel_dev(d) container_of(d, struct usb_skel, kref)
+#define to_lpvo_dev(d) container_of(d, struct lpvo, kref)
 
-static struct usb_driver skel_driver;
-static void skel_draw_down(struct usb_skel *dev);
+static struct usb_driver lpvo_driver;
+static void lpvo_draw_down(struct lpvo *dev);
 
-static void skel_delete(struct kref *kref)
+static void lpvo_delete(struct kref *kref)
 {
-	struct usb_skel *dev = to_skel_dev(kref);
+	struct lpvo *dev = to_lpvo_dev(kref);
 
 	usb_free_urb(dev->bulk_in_urb);
 	usb_put_dev(dev->udev);
@@ -1281,16 +1277,16 @@ static void skel_delete(struct kref *kref)
 }
 
 /*
- * skel_do_open() - to be called by usb_gpib_attach
+ * lpvo_do_open() - to be called by usb_gpib_attach
  */
 
-static int skel_do_open(struct gpib_board *board, int subminor)
+static int lpvo_do_open(struct gpib_board *board, int subminor)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	struct usb_interface *interface;
 	int retval = 0;
 
-	interface = usb_find_interface(&skel_driver, subminor);
+	interface = usb_find_interface(&lpvo_driver, subminor);
 	if (!interface) {
 		dev_err(board->gpib_dev, "can't find device for minor %d\n", subminor);
 		retval = -ENODEV;
@@ -1318,12 +1314,12 @@ static int skel_do_open(struct gpib_board *board, int subminor)
 }
 
 /*
- * skel_do_release() - to be called by usb_gpib_detach
+ * lpvo_do_release() - to be called by usb_gpib_detach
  */
 
-static int skel_do_release(struct gpib_board *board)
+static int lpvo_do_release(struct gpib_board *board)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 
 	dev = GPIB_DEV;
 	if (!dev)
@@ -1336,7 +1332,7 @@ static int skel_do_release(struct gpib_board *board)
 	mutex_unlock(&dev->io_mutex);
 
 	/* decrement the count on our device */
-	kref_put(&dev->kref, skel_delete);
+	kref_put(&dev->kref, lpvo_delete);
 	return 0;
 }
 
@@ -1344,9 +1340,9 @@ static int skel_do_release(struct gpib_board *board)
  * read functions
  */
 
-static void skel_read_bulk_callback(struct urb *urb)
+static void lpvo_read_bulk_callback(struct urb *urb)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	unsigned long flags;
 
 	dev = urb->context;
@@ -1370,7 +1366,7 @@ static void skel_read_bulk_callback(struct urb *urb)
 	wake_up_interruptible(&dev->bulk_in_wait);
 }
 
-static int skel_do_read_io(struct usb_skel *dev, size_t count)
+static int lpvo_do_read_io(struct lpvo *dev, size_t count)
 {
 	int rv;
 
@@ -1381,7 +1377,7 @@ static int skel_do_read_io(struct usb_skel *dev, size_t count)
 					  dev->bulk_in_endpoint_addr),
 			  dev->bulk_in_buffer,
 			  min(dev->bulk_in_size, count),
-			  skel_read_bulk_callback,
+			  lpvo_read_bulk_callback,
 			  dev);
 	/* tell everybody to leave the URB alone */
 	spin_lock_irq(&dev->err_lock);
@@ -1406,10 +1402,10 @@ static int skel_do_read_io(struct usb_skel *dev, size_t count)
 }
 
 /*
- * skel_do_read() - read operations from lpvo_usb_gpib
+ * lpvo_do_read() - read operations from lpvo_usb_gpib
  */
 
-static ssize_t skel_do_read(struct usb_skel *dev, char *buffer, size_t count)
+static ssize_t lpvo_do_read(struct lpvo *dev, char *buffer, size_t count)
 {
 	int rv;
 	bool ongoing_io;
@@ -1487,7 +1483,7 @@ static ssize_t skel_do_read(struct usb_skel *dev, char *buffer, size_t count)
 			 * it seems that requests for less than dev->bulk_in_size
 			 *  are not accepted
 			 */
-			rv = skel_do_read_io(dev, dev->bulk_in_size);
+			rv = lpvo_do_read_io(dev, dev->bulk_in_size);
 			if (rv < 0)
 				goto exit;
 			else
@@ -1534,10 +1530,10 @@ static ssize_t skel_do_read(struct usb_skel *dev, char *buffer, size_t count)
 		 * asked for by the lpvo_usb_gpib layer.
 		 */
 //		  if (available < count)
-//			  skel_do_read_io(dev, dev->bulk_in_size);
+//			  lpvo_do_read_io(dev, dev->bulk_in_size);
 	} else {
 		/* no data in the buffer */
-		rv = skel_do_read_io(dev, dev->bulk_in_size);
+		rv = lpvo_do_read_io(dev, dev->bulk_in_size);
 		if (rv < 0)
 			goto exit;
 		else
@@ -1557,9 +1553,9 @@ static ssize_t skel_do_read(struct usb_skel *dev, char *buffer, size_t count)
  * write functions
  */
 
-static void skel_write_bulk_callback(struct urb *urb)
+static void lpvo_write_bulk_callback(struct urb *urb)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	unsigned long flags;
 
 	dev = urb->context;
@@ -1584,10 +1580,10 @@ static void skel_write_bulk_callback(struct urb *urb)
 }
 
 /*
- * skel_do_write() - write operations from lpvo_usb_gpib
+ * lpvo_do_write() - write operations from lpvo_usb_gpib
  */
 
-static ssize_t skel_do_write(struct usb_skel *dev, const char *buffer, size_t count)
+static ssize_t lpvo_do_write(struct lpvo *dev, const char *buffer, size_t count)
 {
 	int retval = 0;
 	struct urb *urb = NULL;
@@ -1655,7 +1651,7 @@ static ssize_t skel_do_write(struct usb_skel *dev, const char *buffer, size_t co
 	/* initialize the urb properly */
 	usb_fill_bulk_urb(urb, dev->udev,
 			  usb_sndbulkpipe(dev->udev, dev->bulk_out_endpoint_addr),
-			  buf, writesize, skel_write_bulk_callback, dev);
+			  buf, writesize, lpvo_write_bulk_callback, dev);
 	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 	usb_anchor_urb(urb, &dev->submitted);
 
@@ -1694,9 +1690,9 @@ static ssize_t skel_do_write(struct usb_skel *dev, const char *buffer, size_t co
 
 #if USER_DEVICE	 /* conditional compilation of user space device */
 
-static int skel_flush(struct file *file, fl_owner_t id)
+static int lpvo_flush(struct file *file, fl_owner_t id)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	int res;
 
 	dev = file->private_data;
@@ -1705,7 +1701,7 @@ static int skel_flush(struct file *file, fl_owner_t id)
 
 	/* wait for io to stop */
 	mutex_lock(&dev->io_mutex);
-	skel_draw_down(dev);
+	lpvo_draw_down(dev);
 
 	/* read out errors, leave subsequent opens a clean slate */
 	spin_lock_irq(&dev->err_lock);
@@ -1718,16 +1714,16 @@ static int skel_flush(struct file *file, fl_owner_t id)
 	return res;
 }
 
-static int skel_open(struct inode *inode, struct file *file)
+static int lpvo_open(struct inode *inode, struct file *file)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	struct usb_interface *interface;
 	int subminor;
 	int retval = 0;
 
 	subminor = iminor(inode);
 
-	interface = usb_find_interface(&skel_driver, subminor);
+	interface = usb_find_interface(&lpvo_driver, subminor);
 	if (!interface) {
 		pr_err("can't find device for minor %d\n", subminor);
 		retval = -ENODEV;
@@ -1754,9 +1750,9 @@ static int skel_open(struct inode *inode, struct file *file)
 	return retval;
 }
 
-static int skel_release(struct inode *inode, struct file *file)
+static int lpvo_release(struct inode *inode, struct file *file)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 
 	dev = file->private_data;
 	if (!dev)
@@ -1769,7 +1765,7 @@ static int skel_release(struct inode *inode, struct file *file)
 	mutex_unlock(&dev->io_mutex);
 
 	/* decrement the count on our device */
-	kref_put(&dev->kref, skel_delete);
+	kref_put(&dev->kref, lpvo_delete);
 	return 0;
 }
 
@@ -1777,10 +1773,10 @@ static int skel_release(struct inode *inode, struct file *file)
  * user space access to read function
  */
 
-static ssize_t skel_read(struct file *file, char __user *buffer, size_t count,
+static ssize_t lpvo_read(struct file *file, char __user *buffer, size_t count,
 			 loff_t *ppos)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	char *buf;
 	ssize_t rv;
 
@@ -1790,7 +1786,7 @@ static ssize_t skel_read(struct file *file, char __user *buffer, size_t count,
 	if (!buf)
 		return -ENOMEM;
 
-	rv = skel_do_read(dev, buf, count);
+	rv = lpvo_do_read(dev, buf, count);
 
 	if (rv > 0) {
 		if (copy_to_user(buffer, buf, rv)) {
@@ -1806,10 +1802,10 @@ static ssize_t skel_read(struct file *file, char __user *buffer, size_t count,
  * user space access to write function
  */
 
-static ssize_t skel_write(struct file *file, const char __user *user_buffer,
+static ssize_t lpvo_write(struct file *file, const char __user *user_buffer,
 			  size_t count, loff_t *ppos)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	char *buf;
 	ssize_t rv;
 
@@ -1824,20 +1820,20 @@ static ssize_t skel_write(struct file *file, const char __user *user_buffer,
 		return -EFAULT;
 	}
 
-	rv = skel_do_write(dev, buf, count);
+	rv = lpvo_do_write(dev, buf, count);
 	kfree(buf);
 	return rv;
 }
 #endif
 
-static const struct file_operations skel_fops = {
+static const struct file_operations lpvo_fops = {
 	.owner =	THIS_MODULE,
 #if USER_DEVICE
-	.read =	   skel_read,
-	.write =   skel_write,
-	.open =	   skel_open,
-	.release = skel_release,
-	.flush =   skel_flush,
+	.read =	   lpvo_read,
+	.write =   lpvo_write,
+	.open =	   lpvo_open,
+	.release = lpvo_release,
+	.flush =   lpvo_flush,
 	.llseek =  noop_llseek,
 #endif
 };
@@ -1847,17 +1843,17 @@ static const struct file_operations skel_fops = {
  * and to have the device registered with the driver core
  */
 #if USER_DEVICE
-static struct usb_class_driver skel_class = {
+static struct usb_class_driver lpvo_class = {
 	.name =		       "lpvo_raw%d",
-	.fops =		       &skel_fops,
-	.minor_base =	     USB_SKEL_MINOR_BASE,
+	.fops =		       &lpvo_fops,
+	.minor_base =	     USB_LPVO_MINOR_BASE,
 };
 #endif
 
-static int skel_probe(struct usb_interface *interface,
+static int lpvo_probe(struct usb_interface *interface,
 		      const struct usb_device_id *id)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	struct usb_endpoint_descriptor *bulk_in, *bulk_out;
 	int retval;
 	char *device_path;
@@ -1916,7 +1912,7 @@ static int skel_probe(struct usb_interface *interface,
 
 #if USER_DEVICE
 	/* we can register the device now, as it is ready */
-	retval = usb_register_dev(interface, &skel_class);
+	retval = usb_register_dev(interface, &lpvo_class);
 	if (retval) {
 		/* something prevented us from registering this driver */
 		dev_err(&interface->dev,
@@ -1934,14 +1930,14 @@ static int skel_probe(struct usb_interface *interface,
 
 error:
 	/* this frees allocated memory */
-	kref_put(&dev->kref, skel_delete);
+	kref_put(&dev->kref, lpvo_delete);
 
 	return retval;
 }
 
-static void skel_disconnect(struct usb_interface *interface)
+static void lpvo_disconnect(struct usb_interface *interface)
 {
-	struct usb_skel *dev;
+	struct lpvo *dev;
 	int minor = interface->minor;
 
 	usb_gpib_exit_module(minor);	  /* first, disactivate the lpvo */
@@ -1951,7 +1947,7 @@ static void skel_disconnect(struct usb_interface *interface)
 
 #if USER_DEVICE
 	/* give back our minor */
-	usb_deregister_dev(interface, &skel_class);
+	usb_deregister_dev(interface, &lpvo_class);
 #endif
 
 	/* prevent more I/O from starting */
@@ -1962,10 +1958,10 @@ static void skel_disconnect(struct usb_interface *interface)
 	usb_kill_anchored_urbs(&dev->submitted);
 
 	/* decrement our usage count */
-	kref_put(&dev->kref, skel_delete);
+	kref_put(&dev->kref, lpvo_delete);
 }
 
-static void skel_draw_down(struct usb_skel *dev)
+static void lpvo_draw_down(struct lpvo *dev)
 {
 	int time;
 
@@ -1975,34 +1971,34 @@ static void skel_draw_down(struct usb_skel *dev)
 	usb_kill_urb(dev->bulk_in_urb);
 }
 
-static int skel_suspend(struct usb_interface *intf, pm_message_t message)
+static int lpvo_suspend(struct usb_interface *intf, pm_message_t message)
 {
-	struct usb_skel *dev = usb_get_intfdata(intf);
+	struct lpvo *dev = usb_get_intfdata(intf);
 
 	if (!dev)
 		return 0;
-	skel_draw_down(dev);
+	lpvo_draw_down(dev);
 	return 0;
 }
 
-static int skel_resume(struct usb_interface *intf)
+static int lpvo_resume(struct usb_interface *intf)
 {
 	return 0;
 }
 
-static int skel_pre_reset(struct usb_interface *intf)
+static int lpvo_pre_reset(struct usb_interface *intf)
 {
-	struct usb_skel *dev = usb_get_intfdata(intf);
+	struct lpvo *dev = usb_get_intfdata(intf);
 
 	mutex_lock(&dev->io_mutex);
-	skel_draw_down(dev);
+	lpvo_draw_down(dev);
 
 	return 0;
 }
 
-static int skel_post_reset(struct usb_interface *intf)
+static int lpvo_post_reset(struct usb_interface *intf)
 {
-	struct usb_skel *dev = usb_get_intfdata(intf);
+	struct lpvo *dev = usb_get_intfdata(intf);
 
 	/* we are sure no URBs are active - no locking needed */
 	dev->errors = -EPIPE;
@@ -2011,16 +2007,16 @@ static int skel_post_reset(struct usb_interface *intf)
 	return 0;
 }
 
-static struct usb_driver skel_driver = {
+static struct usb_driver lpvo_driver = {
 	.name =			NAME,
-	.probe =		skel_probe,
-	.disconnect =		skel_disconnect,
-	.suspend =		skel_suspend,
-	.resume =		skel_resume,
-	.pre_reset =		skel_pre_reset,
-	.post_reset =		skel_post_reset,
-	.id_table =		skel_table,
+	.probe =		lpvo_probe,
+	.disconnect =		lpvo_disconnect,
+	.suspend =		lpvo_suspend,
+	.resume =		lpvo_resume,
+	.pre_reset =		lpvo_pre_reset,
+	.post_reset =		lpvo_post_reset,
+	.id_table =		lpvo_table,
 	.supports_autosuspend = 1,
 };
 
-module_usb_driver(skel_driver);
+module_usb_driver(lpvo_driver);
diff --git a/drivers/gpib/nec7210/nec7210.c b/drivers/gpib/nec7210/nec7210.c
index bbf39367f5e4f..f15d38dfa4ccb 100644
--- a/drivers/gpib/nec7210/nec7210.c
+++ b/drivers/gpib/nec7210/nec7210.c
@@ -1107,15 +1107,3 @@ void nec7210_locking_iomem_write_byte(struct nec7210_priv *priv, u8 data,
 	spin_unlock_irqrestore(&priv->register_page_lock, flags);
 }
 EXPORT_SYMBOL(nec7210_locking_iomem_write_byte);
-
-static int __init nec7210_init_module(void)
-{
-	return 0;
-}
-
-static void __exit nec7210_exit_module(void)
-{
-}
-
-module_init(nec7210_init_module);
-module_exit(nec7210_exit_module);
diff --git a/drivers/gpib/ni_usb/ni_usb_gpib.c b/drivers/gpib/ni_usb/ni_usb_gpib.c
index a24cd65213620..0bbc13ecebf91 100644
--- a/drivers/gpib/ni_usb/ni_usb_gpib.c
+++ b/drivers/gpib/ni_usb/ni_usb_gpib.c
@@ -2431,7 +2431,6 @@ static int ni_usb_driver_probe(struct usb_interface *interface,	const struct usb
 	static const int path_length = 1024;
 
 	mutex_lock(&ni_usb_hotplug_lock);
-	usb_get_dev(usb_dev);
 	for (i = 0; i < MAX_NUM_NI_USB_INTERFACES; i++) {
 		if (!ni_usb_driver_interfaces[i]) {
 			ni_usb_driver_interfaces[i] = interface;
@@ -2440,14 +2439,12 @@ static int ni_usb_driver_probe(struct usb_interface *interface,	const struct usb
 		}
 	}
 	if (i == MAX_NUM_NI_USB_INTERFACES) {
-		usb_put_dev(usb_dev);
 		mutex_unlock(&ni_usb_hotplug_lock);
 		dev_err(&usb_dev->dev, "ni_usb_driver_interfaces[] full\n");
 		return -1;
 	}
 	path = kmalloc(path_length, GFP_KERNEL);
 	if (!path) {
-		usb_put_dev(usb_dev);
 		mutex_unlock(&ni_usb_hotplug_lock);
 		return -ENOMEM;
 	}
@@ -2488,7 +2485,6 @@ static void ni_usb_driver_disconnect(struct usb_interface *interface)
 	}
 	if (i == MAX_NUM_NI_USB_INTERFACES)
 		dev_err(&usb_dev->dev, "unable to find interface  bug?\n");
-	usb_put_dev(usb_dev);
 	mutex_unlock(&ni_usb_hotplug_lock);
 }
 
diff --git a/drivers/gpib/tms9914/tms9914.c b/drivers/gpib/tms9914/tms9914.c
index 72a11596a35eb..1411297e62178 100644
--- a/drivers/gpib/tms9914/tms9914.c
+++ b/drivers/gpib/tms9914/tms9914.c
@@ -899,16 +899,3 @@ void tms9914_iomem_write_byte(struct tms9914_priv *priv, u8 data, unsigned int r
 		udelay(1);
 }
 EXPORT_SYMBOL_GPL(tms9914_iomem_write_byte);
-
-static int __init tms9914_init_module(void)
-{
-	return 0;
-}
-
-static void __exit tms9914_exit_module(void)
-{
-}
-
-module_init(tms9914_init_module);
-module_exit(tms9914_exit_module);
-
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index b45fb799e36c1..6d39c87125137 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -95,7 +95,6 @@ config GPIO_CDEV
 
 config GPIO_CDEV_V1
 	bool "Support GPIO ABI Version 1"
-	default y
 	depends on GPIO_CDEV
 	help
 	  Say Y here to support version 1 of the GPIO CDEV ABI.
@@ -103,8 +102,6 @@ config GPIO_CDEV_V1
 	  This ABI version is deprecated.
 	  Please use the latest ABI for new developments.
 
-	  If unsure, say Y.
-
 config GPIO_GENERIC
 	depends on HAS_IOMEM # Only for IOMEM drivers
 	tristate
@@ -1440,6 +1437,7 @@ config GPIO_JANZ_TTL
 config GPIO_KEMPLD
 	tristate "Kontron ETX / COMexpress GPIO"
 	depends on MFD_KEMPLD
+	select GPIOLIB_IRQCHIP
 	help
 	  This enables support for the PLD GPIO interface on some Kontron ETX
 	  and COMexpress (ETXexpress) modules.
diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO
index 5acaeab029ec6..7ce80fde1f17e 100644
--- a/drivers/gpio/TODO
+++ b/drivers/gpio/TODO
@@ -58,34 +58,6 @@ Work items:
 
 -------------------------------------------------------------------------------
 
-Get rid of <linux/of_gpio.h>
-
-This header and helpers appeared at one point when there was no proper
-driver infrastructure for doing simpler MMIO GPIO devices and there was
-no core support for parsing device tree GPIOs from the core library with
-the [devm_]gpiod_get() calls we have today that will implicitly go into
-the device tree back-end. It is legacy and should not be used in new code.
-
-Work items:
-
-- Change all consumer drivers that #include <linux/of_gpio.h> to
-  #include <linux/gpio/consumer.h> and stop doing custom parsing of the
-  GPIO lines from the device tree. This can be tricky and often involves
-  changing board files, etc.
-
-- Pull semantics for legacy device tree (OF) GPIO lookups into
-  gpiolib-of.c: in some cases subsystems are doing custom flags and
-  lookups for polarity inversion, open drain and what not. As we now
-  handle this with generic OF bindings, pull all legacy handling into
-  gpiolib so the library API becomes narrow and deep and handle all
-  legacy bindings internally. (See e.g. commits 6953c57ab172,
-  6a537d48461d etc)
-
-- Delete <linux/of_gpio.h> when all the above is complete and everything
-  uses <linux/gpio/consumer.h> or <linux/gpio/driver.h> instead.
-
--------------------------------------------------------------------------------
-
 Collect drivers
 
 Collect GPIO drivers from arch/* and other places that should be placed
diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c
index 208b71c59d585..b1d32d590cf83 100644
--- a/drivers/gpio/gpio-bcm-kona.c
+++ b/drivers/gpio/gpio-bcm-kona.c
@@ -58,15 +58,6 @@
 #define LOCK_CODE				0xffffffff
 #define UNLOCK_CODE				0x00000000
 
-struct bcm_kona_gpio {
-	void __iomem *reg_base;
-	int num_bank;
-	raw_spinlock_t lock;
-	struct gpio_chip gpio_chip;
-	struct irq_domain *irq_domain;
-	struct bcm_kona_gpio_bank *banks;
-};
-
 struct bcm_kona_gpio_bank {
 	int id;
 	int irq;
@@ -90,6 +81,15 @@ struct bcm_kona_gpio_bank {
 	struct bcm_kona_gpio *kona_gpio;
 };
 
+struct bcm_kona_gpio {
+	void __iomem *reg_base;
+	int num_bank;
+	raw_spinlock_t lock;
+	struct gpio_chip gpio_chip;
+	struct irq_domain *irq_domain;
+	struct bcm_kona_gpio_bank banks[] __counted_by(num_bank);
+};
+
 static inline void bcm_kona_gpio_write_lock_regs(void __iomem *reg_base,
 						int bank_id, u32 lockcode)
 {
@@ -584,12 +584,6 @@ static int bcm_kona_gpio_probe(struct platform_device *pdev)
 	int ret;
 	int i;
 
-	kona_gpio = devm_kzalloc(dev, sizeof(*kona_gpio), GFP_KERNEL);
-	if (!kona_gpio)
-		return -ENOMEM;
-
-	kona_gpio->gpio_chip = template_chip;
-	chip = &kona_gpio->gpio_chip;
 	ret = platform_irq_count(pdev);
 	if (!ret) {
 		dev_err(dev, "Couldn't determine # GPIO banks\n");
@@ -597,6 +591,11 @@ static int bcm_kona_gpio_probe(struct platform_device *pdev)
 	} else if (ret < 0) {
 		return dev_err_probe(dev, ret, "Couldn't determine GPIO banks\n");
 	}
+
+	kona_gpio = devm_kzalloc(dev, struct_size(kona_gpio, banks, ret), GFP_KERNEL);
+	if (!kona_gpio)
+		return -ENOMEM;
+
 	kona_gpio->num_bank = ret;
 
 	if (kona_gpio->num_bank > GPIO_MAX_BANK_NUM) {
@@ -604,13 +603,9 @@ static int bcm_kona_gpio_probe(struct platform_device *pdev)
 			GPIO_MAX_BANK_NUM);
 		return -ENXIO;
 	}
-	kona_gpio->banks = devm_kcalloc(dev,
-					kona_gpio->num_bank,
-					sizeof(*kona_gpio->banks),
-					GFP_KERNEL);
-	if (!kona_gpio->banks)
-		return -ENOMEM;
 
+	kona_gpio->gpio_chip = template_chip;
+	chip = &kona_gpio->gpio_chip;
 	chip->parent = dev;
 	chip->ngpio = kona_gpio->num_bank * GPIO_PER_BANK;
 
diff --git a/drivers/gpio/gpio-bd9571mwv.c b/drivers/gpio/gpio-bd9571mwv.c
index 7c95bb36511e1..cc5b1746f2fe8 100644
--- a/drivers/gpio/gpio-bd9571mwv.c
+++ b/drivers/gpio/gpio-bd9571mwv.c
@@ -69,7 +69,7 @@ static int bd9571mwv_gpio_get(struct gpio_chip *chip, unsigned int offset)
 	if (ret < 0)
 		return ret;
 
-	return val & BIT(offset);
+	return !!(val & BIT(offset));
 }
 
 static int bd9571mwv_gpio_set(struct gpio_chip *chip, unsigned int offset,
diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c
index 2352d099709c2..44ca798cf832a 100644
--- a/drivers/gpio/gpio-brcmstb.c
+++ b/drivers/gpio/gpio-brcmstb.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
-// Copyright (C) 2015-2017 Broadcom
+// Copyright (C) 2015-2017, 2026 Broadcom
 
 #include <linux/bitops.h>
 #include <linux/gpio/driver.h>
@@ -54,6 +54,7 @@ struct brcmstb_gpio_priv {
 	int parent_irq;
 	int num_gpios;
 	int parent_wake_irq;
+	bool suspended;
 };
 
 #define MAX_GPIO_PER_BANK       32
@@ -95,15 +96,13 @@ static int brcmstb_gpio_hwirq_to_offset(irq_hw_number_t hwirq,
 	return hwirq - bank->chip.gc.offset;
 }
 
-static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
-		unsigned int hwirq, bool enable)
+static void __brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
+				     irq_hw_number_t hwirq, bool enable)
 {
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
 	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(hwirq, bank));
 	u32 imask;
 
-	guard(gpio_generic_lock_irqsave)(&bank->chip);
-
 	imask = gpio_generic_read_reg(&bank->chip,
 				      priv->reg_base + GIO_MASK(bank->id));
 	if (enable)
@@ -114,6 +113,13 @@ static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
 			       priv->reg_base + GIO_MASK(bank->id), imask);
 }
 
+static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank,
+				   irq_hw_number_t hwirq, bool enable)
+{
+	guard(gpio_generic_lock_irqsave)(&bank->chip);
+	__brcmstb_gpio_set_imask(bank, hwirq, enable);
+}
+
 static int brcmstb_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
 	struct brcmstb_gpio_priv *priv = brcmstb_gpio_gc_to_priv(gc);
@@ -132,7 +138,21 @@ static void brcmstb_gpio_irq_mask(struct irq_data *d)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 
-	brcmstb_gpio_set_imask(bank, d->hwirq, false);
+	brcmstb_gpio_set_imask(bank, irqd_to_hwirq(d), false);
+}
+
+static void brcmstb_gpio_irq_mask_ack(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
+	struct brcmstb_gpio_priv *priv = bank->parent_priv;
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(hwirq, bank));
+
+	guard(gpio_generic_lock_irqsave)(&bank->chip);
+	__brcmstb_gpio_set_imask(bank, hwirq, false);
+	gpio_generic_write_reg(&bank->chip,
+			       priv->reg_base + GIO_STAT(bank->id), mask);
 }
 
 static void brcmstb_gpio_irq_unmask(struct irq_data *d)
@@ -140,7 +160,7 @@ static void brcmstb_gpio_irq_unmask(struct irq_data *d)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 
-	brcmstb_gpio_set_imask(bank, d->hwirq, true);
+	brcmstb_gpio_set_imask(bank, irqd_to_hwirq(d), true);
 }
 
 static void brcmstb_gpio_irq_ack(struct irq_data *d)
@@ -148,7 +168,7 @@ static void brcmstb_gpio_irq_ack(struct irq_data *d)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
-	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(d->hwirq, bank));
+	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(irqd_to_hwirq(d), bank));
 
 	gpio_generic_write_reg(&bank->chip,
 			       priv->reg_base + GIO_STAT(bank->id), mask);
@@ -159,7 +179,7 @@ static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
-	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(d->hwirq, bank));
+	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(irqd_to_hwirq(d), bank));
 	u32 edge_insensitive, iedge_insensitive;
 	u32 edge_config, iedge_config;
 	u32 level, ilevel;
@@ -221,6 +241,9 @@ static int brcmstb_gpio_priv_set_wake(struct brcmstb_gpio_priv *priv,
 {
 	int ret = 0;
 
+	if (priv->parent_wake_irq == priv->parent_irq)
+		return ret;
+
 	if (enable)
 		ret = enable_irq_wake(priv->parent_wake_irq);
 	else
@@ -236,7 +259,7 @@ static int brcmstb_gpio_irq_set_wake(struct irq_data *d, unsigned int enable)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct brcmstb_gpio_bank *bank = gpiochip_get_data(gc);
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
-	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(d->hwirq, bank));
+	u32 mask = BIT(brcmstb_gpio_hwirq_to_offset(irqd_to_hwirq(d), bank));
 
 	/*
 	 * Do not do anything specific for now, suspend/resume callbacks will
@@ -271,6 +294,11 @@ static void brcmstb_gpio_irq_bank_handler(struct brcmstb_gpio_bank *bank)
 	while ((status = brcmstb_gpio_get_active_irqs(bank))) {
 		unsigned int offset;
 
+		if (priv->suspended && bank->wake_active & status) {
+			priv->suspended = false;
+			pm_wakeup_event(&priv->pdev->dev, 0);
+		}
+
 		for_each_set_bit(offset, &status, 32) {
 			if (offset >= bank->width)
 				dev_warn(&priv->pdev->dev,
@@ -444,18 +472,18 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 	}
 
 	if (of_property_read_bool(np, "wakeup-source")) {
+		/*
+		 * Set wakeup capability so we can process boot-time
+		 * "wakeups" (e.g., from S5 cold boot).
+		 */
+		device_set_wakeup_capable(dev, true);
+		device_wakeup_enable(dev);
 		priv->parent_wake_irq = platform_get_irq(pdev, 1);
 		if (priv->parent_wake_irq < 0) {
-			priv->parent_wake_irq = 0;
+			priv->parent_wake_irq = priv->parent_irq;
 			dev_warn(dev,
 				"Couldn't get wake IRQ - GPIOs will not be able to wake from sleep");
 		} else {
-			/*
-			 * Set wakeup capability so we can process boot-time
-			 * "wakeups" (e.g., from S5 cold boot)
-			 */
-			device_set_wakeup_capable(dev, true);
-			device_wakeup_enable(dev);
 			err = devm_request_irq(dev, priv->parent_wake_irq,
 					       brcmstb_gpio_wake_irq_handler,
 					       IRQF_SHARED,
@@ -466,18 +494,17 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 				goto out_free_domain;
 			}
 		}
+		priv->irq_chip.irq_set_wake = brcmstb_gpio_irq_set_wake;
 	}
 
 	priv->irq_chip.name = dev_name(dev);
 	priv->irq_chip.irq_disable = brcmstb_gpio_irq_mask;
 	priv->irq_chip.irq_mask = brcmstb_gpio_irq_mask;
+	priv->irq_chip.irq_mask_ack = brcmstb_gpio_irq_mask_ack;
 	priv->irq_chip.irq_unmask = brcmstb_gpio_irq_unmask;
 	priv->irq_chip.irq_ack = brcmstb_gpio_irq_ack;
 	priv->irq_chip.irq_set_type = brcmstb_gpio_irq_set_type;
 
-	if (priv->parent_wake_irq)
-		priv->irq_chip.irq_set_wake = brcmstb_gpio_irq_set_wake;
-
 	irq_set_chained_handler_and_data(priv->parent_irq,
 					 brcmstb_gpio_irq_handler, priv);
 	irq_set_status_flags(priv->parent_irq, IRQ_DISABLE_UNLAZY);
@@ -500,16 +527,11 @@ static void brcmstb_gpio_bank_save(struct brcmstb_gpio_priv *priv,
 					priv->reg_base + GIO_BANK_OFF(bank->id, i));
 }
 
-static void brcmstb_gpio_quiesce(struct device *dev, bool save)
+static void brcmstb_gpio_quiesce(struct brcmstb_gpio_priv *priv, bool save)
 {
-	struct brcmstb_gpio_priv *priv = dev_get_drvdata(dev);
 	struct brcmstb_gpio_bank *bank;
 	u32 imask;
 
-	/* disable non-wake interrupt */
-	if (priv->parent_irq >= 0)
-		disable_irq(priv->parent_irq);
-
 	list_for_each_entry(bank, &priv->bank_list, node) {
 		if (save)
 			brcmstb_gpio_bank_save(priv, bank);
@@ -527,8 +549,13 @@ static void brcmstb_gpio_quiesce(struct device *dev, bool save)
 
 static void brcmstb_gpio_shutdown(struct platform_device *pdev)
 {
+	struct brcmstb_gpio_priv *priv = dev_get_drvdata(&pdev->dev);
+
+	if (priv->parent_irq > 0)
+		disable_irq(priv->parent_irq);
+
 	/* Enable GPIO for S5 cold boot */
-	brcmstb_gpio_quiesce(&pdev->dev, false);
+	brcmstb_gpio_quiesce(priv, false);
 }
 
 static void brcmstb_gpio_bank_restore(struct brcmstb_gpio_priv *priv,
@@ -544,7 +571,30 @@ static void brcmstb_gpio_bank_restore(struct brcmstb_gpio_priv *priv,
 
 static int brcmstb_gpio_suspend(struct device *dev)
 {
-	brcmstb_gpio_quiesce(dev, true);
+	struct brcmstb_gpio_priv *priv = dev_get_drvdata(dev);
+
+	if (priv->parent_irq > 0)
+		priv->suspended = true;
+
+	return 0;
+}
+
+static int brcmstb_gpio_suspend_noirq(struct device *dev)
+{
+	struct brcmstb_gpio_priv *priv = dev_get_drvdata(dev);
+
+	/* Catch any wakeup sources occurring between suspend and noirq */
+	if (!priv->suspended)
+		return -EBUSY;
+
+	if (priv->parent_irq > 0)
+		disable_irq(priv->parent_irq);
+
+	brcmstb_gpio_quiesce(priv, true);
+
+	if (priv->parent_wake_irq)
+		enable_irq(priv->parent_irq);
+
 	return 0;
 }
 
@@ -552,25 +602,24 @@ static int brcmstb_gpio_resume(struct device *dev)
 {
 	struct brcmstb_gpio_priv *priv = dev_get_drvdata(dev);
 	struct brcmstb_gpio_bank *bank;
-	bool need_wakeup_event = false;
 
-	list_for_each_entry(bank, &priv->bank_list, node) {
-		need_wakeup_event |= !!__brcmstb_gpio_get_active_irqs(bank);
-		brcmstb_gpio_bank_restore(priv, bank);
-	}
+	if (priv->parent_wake_irq)
+		disable_irq(priv->parent_irq);
 
-	if (priv->parent_wake_irq && need_wakeup_event)
-		pm_wakeup_event(dev, 0);
+	priv->suspended = false;
 
-	/* enable non-wake interrupt */
-	if (priv->parent_irq >= 0)
+	list_for_each_entry(bank, &priv->bank_list, node)
+		brcmstb_gpio_bank_restore(priv, bank);
+
+	if (priv->parent_irq > 0)
 		enable_irq(priv->parent_irq);
 
 	return 0;
 }
 
 static const struct dev_pm_ops brcmstb_gpio_pm_ops = {
-	.suspend_noirq = pm_sleep_ptr(brcmstb_gpio_suspend),
+	.suspend = pm_sleep_ptr(brcmstb_gpio_suspend),
+	.suspend_noirq = pm_sleep_ptr(brcmstb_gpio_suspend_noirq),
 	.resume_noirq = pm_sleep_ptr(brcmstb_gpio_resume),
 };
 
diff --git a/drivers/gpio/gpio-cgbc.c b/drivers/gpio/gpio-cgbc.c
index 0efa1b61001ad..84b5ed3c6e22b 100644
--- a/drivers/gpio/gpio-cgbc.c
+++ b/drivers/gpio/gpio-cgbc.c
@@ -47,8 +47,8 @@ static int cgbc_gpio_get(struct gpio_chip *chip, unsigned int offset)
 
 	if (ret)
 		return ret;
-	else
-		return (int)(val & (u8)BIT(offset));
+
+	return !!(val & BIT(offset));
 }
 
 static int __cgbc_gpio_set(struct gpio_chip *chip, unsigned int offset,
diff --git a/drivers/gpio/gpio-cs5535.c b/drivers/gpio/gpio-cs5535.c
index 8affe4e9f90e9..5f5373d86397c 100644
--- a/drivers/gpio/gpio-cs5535.c
+++ b/drivers/gpio/gpio-cs5535.c
@@ -39,10 +39,6 @@ static ulong mask = GPIO_DEFAULT_MASK;
 module_param_named(mask, mask, ulong, 0444);
 MODULE_PARM_DESC(mask, "GPIO channel mask.");
 
-/*
- * FIXME: convert this singleton driver to use the state container
- * design pattern, see Documentation/driver-api/driver-model/design-patterns.rst
- */
 static struct cs5535_gpio_chip {
 	struct gpio_chip chip;
 	resource_size_t base;
@@ -285,30 +281,29 @@ static const char * const cs5535_gpio_names[] = {
 	"GPIO28", NULL, NULL, NULL,
 };
 
-static struct cs5535_gpio_chip cs5535_gpio_chip = {
-	.chip = {
-		.owner = THIS_MODULE,
-		.label = DRV_NAME,
-
-		.base = 0,
-		.ngpio = 32,
-		.names = cs5535_gpio_names,
-		.request = chip_gpio_request,
-
-		.get = chip_gpio_get,
-		.set = chip_gpio_set,
-
-		.direction_input = chip_direction_input,
-		.direction_output = chip_direction_output,
-	},
-};
-
 static int cs5535_gpio_probe(struct platform_device *pdev)
 {
+	struct cs5535_gpio_chip *priv;
+	struct gpio_chip *gc;
 	struct resource *res;
 	int err = -EIO;
 	ulong mask_orig = mask;
 
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	gc = &priv->chip;
+	gc->owner = THIS_MODULE;
+	gc->label = DRV_NAME;
+	gc->ngpio = 32;
+	gc->names = cs5535_gpio_names;
+	gc->request = chip_gpio_request;
+	gc->get = chip_gpio_get;
+	gc->set = chip_gpio_set;
+	gc->direction_input = chip_direction_input;
+	gc->direction_output = chip_direction_output;
+
 	/* There are two ways to get the GPIO base address; one is by
 	 * fetching it from MSR_LBAR_GPIO, the other is by reading the
 	 * PCI BAR info.  The latter method is easier (especially across
@@ -329,9 +324,9 @@ static int cs5535_gpio_probe(struct platform_device *pdev)
 	}
 
 	/* set up the driver-specific struct */
-	cs5535_gpio_chip.base = res->start;
-	cs5535_gpio_chip.pdev = pdev;
-	spin_lock_init(&cs5535_gpio_chip.lock);
+	priv->base = res->start;
+	priv->pdev = pdev;
+	spin_lock_init(&priv->lock);
 
 	dev_info(&pdev->dev, "reserved resource region %pR\n", res);
 
@@ -347,8 +342,7 @@ static int cs5535_gpio_probe(struct platform_device *pdev)
 				mask_orig, mask);
 
 	/* finally, register with the generic GPIO API */
-	return devm_gpiochip_add_data(&pdev->dev, &cs5535_gpio_chip.chip,
-				      &cs5535_gpio_chip);
+	return devm_gpiochip_add_data(&pdev->dev, gc, priv);
 }
 
 static struct platform_driver cs5535_gpio_driver = {
diff --git a/drivers/gpio/gpio-da9055.c b/drivers/gpio/gpio-da9055.c
index a09bd6eb93cfb..1949a6ea8ec61 100644
--- a/drivers/gpio/gpio-da9055.c
+++ b/drivers/gpio/gpio-da9055.c
@@ -55,7 +55,7 @@ static int da9055_gpio_get(struct gpio_chip *gc, unsigned offset)
 			return ret;
 	}
 
-	return ret & (1 << offset);
+	return !!(ret & (1 << offset));
 
 }
 
diff --git a/drivers/gpio/gpio-htc-egpio.c b/drivers/gpio/gpio-htc-egpio.c
index 72935d6dbebf0..d15423c718d03 100644
--- a/drivers/gpio/gpio-htc-egpio.c
+++ b/drivers/gpio/gpio-htc-egpio.c
@@ -46,8 +46,8 @@ struct egpio_info {
 	uint              chained_irq;
 
 	/* egpio info */
-	struct egpio_chip *chip;
 	int               nchips;
+	struct egpio_chip chip[] __counted_by(nchips);
 };
 
 static inline void egpio_writew(u16 value, struct egpio_info *ei, int reg)
@@ -270,10 +270,12 @@ static int __init egpio_probe(struct platform_device *pdev)
 	int               i;
 
 	/* Initialize ei data structure. */
-	ei = devm_kzalloc(&pdev->dev, sizeof(*ei), GFP_KERNEL);
+	ei = devm_kzalloc(&pdev->dev, struct_size(ei, chip, pdata->num_chips), GFP_KERNEL);
 	if (!ei)
 		return -ENOMEM;
 
+	ei->nchips = pdata->num_chips;
+
 	spin_lock_init(&ei->lock);
 
 	/* Find chained irq */
@@ -302,13 +304,6 @@ static int __init egpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, ei);
 
-	ei->nchips = pdata->num_chips;
-	ei->chip = devm_kcalloc(&pdev->dev,
-				ei->nchips, sizeof(struct egpio_chip),
-				GFP_KERNEL);
-	if (!ei->chip)
-		return -ENOMEM;
-
 	for (i = 0; i < ei->nchips; i++) {
 		ei->chip[i].reg_start = pdata->chip[i].reg_start;
 		ei->chip[i].cached_values = pdata->chip[i].initial_values;
diff --git a/drivers/gpio/gpio-kempld.c b/drivers/gpio/gpio-kempld.c
index 923aad3ab4d45..5a63df3ea5fa7 100644
--- a/drivers/gpio/gpio-kempld.c
+++ b/drivers/gpio/gpio-kempld.c
@@ -11,20 +11,35 @@
 #include <linux/module.h>
 #include <linux/bitops.h>
 #include <linux/errno.h>
+#include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/gpio/driver.h>
 #include <linux/mfd/kempld.h>
 
 #define KEMPLD_GPIO_MAX_NUM		16
 #define KEMPLD_GPIO_MASK(x)		(BIT((x) % 8))
-#define KEMPLD_GPIO_DIR_NUM(x)		(0x40 + (x) / 8)
-#define KEMPLD_GPIO_LVL_NUM(x)		(0x42 + (x) / 8)
+#define KEMPLD_GPIO_DIR			0x40
+#define KEMPLD_GPIO_LVL			0x42
+#define KEMPLD_GPIO_STS			0x44
 #define KEMPLD_GPIO_EVT_LVL_EDGE	0x46
+#define KEMPLD_GPIO_EVT_LOW_HIGH	0x48
 #define KEMPLD_GPIO_IEN			0x4A
+#define KEMPLD_GPIO_OUT_LVL		0x4E
+
+/* The IRQ to use if none was configured in the BIOS */
+static unsigned int gpio_irq;
+module_param_hw(gpio_irq, uint, irq, 0444);
+MODULE_PARM_DESC(gpio_irq, "Set legacy GPIO IRQ (1-15)");
 
 struct kempld_gpio_data {
 	struct gpio_chip		chip;
 	struct kempld_device_data	*pld;
+	u8				out_lvl_reg;
+
+	struct mutex			irq_lock;
+	u16				ien;
+	u16				evt_low_high;
+	u16				evt_lvl_edge;
 };
 
 /*
@@ -32,24 +47,25 @@ struct kempld_gpio_data {
  * kempld_get_mutex must be called prior to calling this function.
  */
 static void kempld_gpio_bitop(struct kempld_device_data *pld,
-			      u8 reg, u8 bit, u8 val)
+			      u8 reg, unsigned int bit, bool val)
 {
 	u8 status;
 
-	status = kempld_read8(pld, reg);
+	status = kempld_read8(pld, reg + (bit / 8));
 	if (val)
 		status |= KEMPLD_GPIO_MASK(bit);
 	else
 		status &= ~KEMPLD_GPIO_MASK(bit);
-	kempld_write8(pld, reg, status);
+	kempld_write8(pld, reg + (bit / 8), status);
 }
 
-static int kempld_gpio_get_bit(struct kempld_device_data *pld, u8 reg, u8 bit)
+static int kempld_gpio_get_bit(struct kempld_device_data *pld,
+			       u8 reg, unsigned int bit)
 {
 	u8 status;
 
 	kempld_get_mutex(pld);
-	status = kempld_read8(pld, reg);
+	status = kempld_read8(pld, reg + (bit / 8));
 	kempld_release_mutex(pld);
 
 	return !!(status & KEMPLD_GPIO_MASK(bit));
@@ -60,7 +76,34 @@ static int kempld_gpio_get(struct gpio_chip *chip, unsigned offset)
 	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
-	return !!kempld_gpio_get_bit(pld, KEMPLD_GPIO_LVL_NUM(offset), offset);
+	return !!kempld_gpio_get_bit(pld, KEMPLD_GPIO_LVL, offset);
+}
+
+static int kempld_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
+				    unsigned long *bits)
+{
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+	struct kempld_device_data *pld = gpio->pld;
+	u8 reg = KEMPLD_GPIO_LVL;
+	unsigned int shift;
+
+	bits[0] &= ~mask[0];
+
+	kempld_get_mutex(pld);
+
+	/* Try to reduce to a single 8 bits access if possible */
+	for (shift = 0; shift < gpio->chip.ngpio; shift += 8, reg++) {
+		unsigned long msk = (mask[0] >> shift) & 0xff;
+
+		if (!msk)
+			continue;
+
+		bits[0] |= (kempld_read8(pld, reg) & msk) << shift;
+	}
+
+	kempld_release_mutex(pld);
+
+	return 0;
 }
 
 static int kempld_gpio_set(struct gpio_chip *chip, unsigned int offset,
@@ -70,7 +113,38 @@ static int kempld_gpio_set(struct gpio_chip *chip, unsigned int offset,
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
-	kempld_gpio_bitop(pld, KEMPLD_GPIO_LVL_NUM(offset), offset, value);
+	kempld_gpio_bitop(pld, gpio->out_lvl_reg, offset, value);
+	kempld_release_mutex(pld);
+
+	return 0;
+}
+
+static int kempld_gpio_set_multiple(struct gpio_chip *chip,
+				    unsigned long *mask, unsigned long *bits)
+{
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+	struct kempld_device_data *pld = gpio->pld;
+	u8 reg = gpio->out_lvl_reg;
+	unsigned int shift;
+
+	kempld_get_mutex(pld);
+
+	/* Try to reduce to a single 8 bits access if possible */
+	for (shift = 0; shift < gpio->chip.ngpio; shift += 8, reg++) {
+		u8 val, msk = mask[0] >> shift;
+
+		if (!msk)
+			continue;
+
+		if (msk != 0xFF)
+			val = kempld_read8(pld, reg) & ~msk;
+		else
+			val = 0;
+
+		val |= (bits[0] >> shift) & msk;
+		kempld_write8(pld, reg, val);
+	}
+
 	kempld_release_mutex(pld);
 
 	return 0;
@@ -82,7 +156,7 @@ static int kempld_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
-	kempld_gpio_bitop(pld, KEMPLD_GPIO_DIR_NUM(offset), offset, 0);
+	kempld_gpio_bitop(pld, KEMPLD_GPIO_DIR, offset, 0);
 	kempld_release_mutex(pld);
 
 	return 0;
@@ -95,8 +169,8 @@ static int kempld_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 	struct kempld_device_data *pld = gpio->pld;
 
 	kempld_get_mutex(pld);
-	kempld_gpio_bitop(pld, KEMPLD_GPIO_LVL_NUM(offset), offset, value);
-	kempld_gpio_bitop(pld, KEMPLD_GPIO_DIR_NUM(offset), offset, 1);
+	kempld_gpio_bitop(pld, gpio->out_lvl_reg, offset, value);
+	kempld_gpio_bitop(pld, KEMPLD_GPIO_DIR, offset, 1);
 	kempld_release_mutex(pld);
 
 	return 0;
@@ -107,7 +181,7 @@ static int kempld_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
 	struct kempld_device_data *pld = gpio->pld;
 
-	if (kempld_gpio_get_bit(pld, KEMPLD_GPIO_DIR_NUM(offset), offset))
+	if (kempld_gpio_get_bit(pld, KEMPLD_GPIO_DIR, offset))
 		return GPIO_LINE_DIRECTION_OUT;
 
 	return GPIO_LINE_DIRECTION_IN;
@@ -133,6 +207,180 @@ static int kempld_gpio_pincount(struct kempld_device_data *pld)
 	return evt ? __ffs(evt) : 16;
 }
 
+static void kempld_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+
+	gpio->ien &= ~BIT(irqd_to_hwirq(data));
+	gpiochip_disable_irq(chip, irqd_to_hwirq(data));
+}
+
+static void kempld_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+
+	gpiochip_enable_irq(chip, irqd_to_hwirq(data));
+	gpio->ien |= BIT(irqd_to_hwirq(data));
+}
+
+static int kempld_irq_set_type(struct irq_data *data, unsigned int type)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+		gpio->evt_low_high |= BIT(data->hwirq);
+		gpio->evt_lvl_edge |= BIT(data->hwirq);
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		gpio->evt_low_high &= ~BIT(data->hwirq);
+		gpio->evt_lvl_edge |= BIT(data->hwirq);
+		break;
+	case IRQ_TYPE_LEVEL_HIGH:
+		gpio->evt_low_high |= BIT(data->hwirq);
+		gpio->evt_lvl_edge &= ~BIT(data->hwirq);
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		gpio->evt_low_high &= ~BIT(data->hwirq);
+		gpio->evt_lvl_edge &= ~BIT(data->hwirq);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void kempld_irq_bus_lock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+
+	mutex_lock(&gpio->irq_lock);
+}
+
+static void kempld_irq_bus_sync_unlock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct kempld_gpio_data *gpio = gpiochip_get_data(chip);
+	struct kempld_device_data *pld = gpio->pld;
+
+	kempld_get_mutex(pld);
+	kempld_write16(pld, KEMPLD_GPIO_EVT_LVL_EDGE, gpio->evt_lvl_edge);
+	kempld_write16(pld, KEMPLD_GPIO_EVT_LOW_HIGH, gpio->evt_low_high);
+	kempld_write16(pld, KEMPLD_GPIO_IEN, gpio->ien);
+	kempld_release_mutex(pld);
+
+	mutex_unlock(&gpio->irq_lock);
+}
+
+static const struct irq_chip kempld_irqchip = {
+	.name			= "kempld-gpio",
+	.irq_mask		= kempld_irq_mask,
+	.irq_unmask		= kempld_irq_unmask,
+	.irq_set_type		= kempld_irq_set_type,
+	.irq_bus_lock		= kempld_irq_bus_lock,
+	.irq_bus_sync_unlock	= kempld_irq_bus_sync_unlock,
+	.flags			= IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static irqreturn_t kempld_gpio_irq_handler(int irq, void *data)
+{
+	struct kempld_gpio_data *gpio = data;
+	struct gpio_chip *chip = &gpio->chip;
+	unsigned int pin, child_irq;
+	unsigned long status;
+
+	kempld_get_mutex(gpio->pld);
+
+	status = kempld_read16(gpio->pld, KEMPLD_GPIO_STS);
+	if (status)
+		kempld_write16(gpio->pld, KEMPLD_GPIO_STS, status);
+
+	kempld_release_mutex(gpio->pld);
+
+	status &= gpio->ien;
+	if (!status)
+		return IRQ_NONE;
+
+	for_each_set_bit(pin, &status, chip->ngpio) {
+		child_irq = irq_find_mapping(chip->irq.domain, pin);
+		handle_nested_irq(child_irq);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int kempld_gpio_irq_init(struct device *dev,
+				struct kempld_gpio_data *gpio)
+{
+	struct kempld_device_data *pld = gpio->pld;
+	struct gpio_chip *chip = &gpio->chip;
+	struct gpio_irq_chip *girq;
+	unsigned int irq;
+	int ret;
+
+	/* Get the IRQ configured by the BIOS in the PLD */
+	kempld_get_mutex(pld);
+	irq = kempld_read8(pld, KEMPLD_IRQ_GPIO);
+	kempld_release_mutex(pld);
+
+	if (irq == 0xff) {
+		dev_info(dev, "GPIO controller has no IRQ support\n");
+		return 0;
+	}
+
+	/* Allow overriding the IRQ with the module parameter */
+	if (gpio_irq > 0) {
+		dev_warn(dev, "Forcing IRQ to %d\n", gpio_irq);
+		irq &= ~KEMPLD_IRQ_GPIO_MASK;
+		irq |= gpio_irq & KEMPLD_IRQ_GPIO_MASK;
+	}
+
+	if (!(irq & KEMPLD_IRQ_GPIO_MASK)) {
+		dev_warn(dev, "No IRQ configured\n");
+		return 0;
+	}
+
+	/* Get the current config, disable all child interrupts, clear them
+	 * and set the parent IRQ
+	 */
+	kempld_get_mutex(pld);
+	gpio->evt_low_high = kempld_read16(pld, KEMPLD_GPIO_EVT_LOW_HIGH);
+	gpio->evt_lvl_edge = kempld_read16(pld, KEMPLD_GPIO_EVT_LVL_EDGE);
+	kempld_write16(pld, KEMPLD_GPIO_IEN, 0);
+	kempld_write16(pld, KEMPLD_GPIO_STS, 0xFFFF);
+	kempld_write16(pld, KEMPLD_IRQ_GPIO, irq);
+	kempld_release_mutex(pld);
+
+	girq = &chip->irq;
+	gpio_irq_chip_set_chip(girq, &kempld_irqchip);
+
+	girq->parent_handler = NULL;
+	girq->num_parents = 0;
+	girq->parents = NULL;
+	girq->default_type = IRQ_TYPE_NONE;
+	girq->handler = handle_simple_irq;
+	girq->threaded = true;
+
+	mutex_init(&gpio->irq_lock);
+
+	ret = devm_request_threaded_irq(dev, irq & KEMPLD_IRQ_GPIO_MASK,
+					NULL, kempld_gpio_irq_handler,
+					IRQF_ONESHOT, chip->label,
+					gpio);
+	if (ret) {
+		dev_err(dev, "failed to request irq %d\n", irq);
+		return ret;
+	}
+
+	return 0;
+}
+
 static int kempld_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -152,6 +400,15 @@ static int kempld_gpio_probe(struct platform_device *pdev)
 	if (!gpio)
 		return -ENOMEM;
 
+	/* Starting with version 2.8 there is a dedicated register for the
+	 * output state, earlier versions share the register used to read
+	 * the line level.
+	 */
+	if (pld->info.spec_major > 2 || pld->info.spec_minor >= 8)
+		gpio->out_lvl_reg = KEMPLD_GPIO_OUT_LVL;
+	else
+		gpio->out_lvl_reg = KEMPLD_GPIO_LVL;
+
 	gpio->pld = pld;
 
 	platform_set_drvdata(pdev, gpio);
@@ -169,13 +426,19 @@ static int kempld_gpio_probe(struct platform_device *pdev)
 	chip->direction_output = kempld_gpio_direction_output;
 	chip->get_direction = kempld_gpio_get_direction;
 	chip->get = kempld_gpio_get;
+	chip->get_multiple = kempld_gpio_get_multiple;
 	chip->set = kempld_gpio_set;
+	chip->set_multiple = kempld_gpio_set_multiple;
 	chip->ngpio = kempld_gpio_pincount(pld);
 	if (chip->ngpio == 0) {
 		dev_err(dev, "No GPIO pins detected\n");
 		return -ENODEV;
 	}
 
+	ret = kempld_gpio_irq_init(dev, gpio);
+	if (ret)
+		return ret;
+
 	ret = devm_gpiochip_add_data(dev, chip, gpio);
 	if (ret) {
 		dev_err(dev, "Could not register GPIO chip\n");
diff --git a/drivers/gpio/gpio-lp873x.c b/drivers/gpio/gpio-lp873x.c
index 5376708a81bfe..f4413fa5a8110 100644
--- a/drivers/gpio/gpio-lp873x.c
+++ b/drivers/gpio/gpio-lp873x.c
@@ -55,7 +55,7 @@ static int lp873x_gpio_get(struct gpio_chip *chip, unsigned int offset)
 	if (ret < 0)
 		return ret;
 
-	return val & BIT(offset * BITS_PER_GPO);
+	return !!(val & BIT(offset * BITS_PER_GPO));
 }
 
 static int lp873x_gpio_set(struct gpio_chip *chip, unsigned int offset,
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index a61d670ceeda6..281ba1740a6a0 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -10,14 +10,18 @@
  *  Derived from drivers/gpio/pca953x.c
  */
 
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/string.h>
+#include <linux/cleanup.h>
+#include <linux/err.h>
+#include <linux/device.h>
 #include <linux/gpio/driver.h>
-#include <linux/interrupt.h>
 #include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/platform_data/max732x.h>
+#include <linux/slab.h>
+#include <linux/string.h>
 
 /*
  * Each port of MAX732x (including MAX7319) falls into one of the
@@ -207,22 +211,20 @@ static void max732x_gpio_set_mask(struct gpio_chip *gc, unsigned off, int mask,
 	uint8_t reg_out;
 	int ret;
 
-	mutex_lock(&chip->lock);
+	guard(mutex)(&chip->lock);
 
 	reg_out = (off > 7) ? chip->reg_out[1] : chip->reg_out[0];
 	reg_out = (reg_out & ~mask) | (val & mask);
 
 	ret = max732x_writeb(chip, is_group_a(chip, off), reg_out);
 	if (ret < 0)
-		goto out;
+		return;
 
 	/* update the shadow register then */
 	if (off > 7)
 		chip->reg_out[1] = reg_out;
 	else
 		chip->reg_out[0] = reg_out;
-out:
-	mutex_unlock(&chip->lock);
 }
 
 static int max732x_gpio_set_value(struct gpio_chip *gc, unsigned int off,
@@ -329,7 +331,7 @@ static void max732x_irq_update_mask(struct max732x_chip *chip)
 	if (chip->irq_features == INT_NO_MASK)
 		return;
 
-	mutex_lock(&chip->lock);
+	guard(mutex)(&chip->lock);
 
 	switch (chip->irq_features) {
 	case INT_INDEP_MASK:
@@ -342,8 +344,6 @@ static void max732x_irq_update_mask(struct max732x_chip *chip)
 		max732x_writeb(chip, 1, (uint8_t)msg);
 		break;
 	}
-
-	mutex_unlock(&chip->lock);
 }
 
 static void max732x_irq_mask(struct irq_data *d)
diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c
index edbcaad57d009..0941d034a49c5 100644
--- a/drivers/gpio/gpio-mmio.c
+++ b/drivers/gpio/gpio-mmio.c
@@ -42,18 +42,16 @@ o        `                     ~~~~\___/~~~~    ` controller in FPGA is ,.`
 
 #include <linux/bitops.h>
 #include <linux/cleanup.h>
-#include <linux/compiler.h>
 #include <linux/err.h>
-#include <linux/init.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
+#include <linux/limits.h>
 #include <linux/log2.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
-#include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
diff --git a/drivers/gpio/gpio-mpsse.c b/drivers/gpio/gpio-mpsse.c
index 12191aeb65668..a859deab2bca8 100644
--- a/drivers/gpio/gpio-mpsse.c
+++ b/drivers/gpio/gpio-mpsse.c
@@ -548,13 +548,6 @@ static void gpio_mpsse_ida_remove(void *data)
 	ida_free(&gpio_mpsse_ida, priv->id);
 }
 
-static void gpio_mpsse_usb_put_dev(void *data)
-{
-	struct mpsse_priv *priv = data;
-
-	usb_put_dev(priv->udev);
-}
-
 static int mpsse_init_valid_mask(struct gpio_chip *chip,
 				 unsigned long *valid_mask,
 				 unsigned int ngpios)
@@ -598,11 +591,7 @@ static int gpio_mpsse_probe(struct usb_interface *interface,
 
 	INIT_LIST_HEAD(&priv->workers);
 
-	priv->udev = usb_get_dev(interface_to_usbdev(interface));
-	err = devm_add_action_or_reset(dev, gpio_mpsse_usb_put_dev, priv);
-	if (err)
-		return err;
-
+	priv->udev = interface_to_usbdev(interface);
 	priv->intf = interface;
 	priv->intf_id = interface->cur_altsetting->desc.bInterfaceNumber;
 
diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c
index 4cf91528f547e..5e3152c2e51a8 100644
--- a/drivers/gpio/gpio-realtek-otto.c
+++ b/drivers/gpio/gpio-realtek-otto.c
@@ -351,6 +351,10 @@ static const struct of_device_id realtek_gpio_of_match[] = {
 	{
 		.compatible = "realtek,rtl9310-gpio",
 	},
+	{
+		.compatible = "realtek,rtl9607-gpio",
+		.data = (void *)GPIO_PORTS_REVERSED,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, realtek_gpio_of_match);
diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c
index 0fff4a699f12d..ac1b939283eb7 100644
--- a/drivers/gpio/gpio-rockchip.c
+++ b/drivers/gpio/gpio-rockchip.c
@@ -296,7 +296,7 @@ static int rockchip_gpio_set_config(struct gpio_chip *gc, unsigned int offset,
 		 */
 		return -ENOTSUPP;
 	default:
-		return -ENOTSUPP;
+		return gpiochip_generic_config(gc, offset, config);
 	}
 }
 
diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
index 13b87c8e6d0ca..f32674230237e 100644
--- a/drivers/gpio/gpio-sim.c
+++ b/drivers/gpio/gpio-sim.c
@@ -40,6 +40,7 @@
 
 #define GPIO_SIM_NGPIO_MAX	1024
 #define GPIO_SIM_PROP_MAX	5 /* Max 4 properties + sentinel. */
+#define GPIO_SIM_HOG_PROP_MAX	5
 #define GPIO_SIM_NUM_ATTRS	3 /* value, pull and sentinel */
 
 static DEFINE_IDA(gpio_sim_ida);
@@ -561,8 +562,6 @@ struct gpio_sim_device {
 	 */
 	struct mutex lock;
 
-	struct gpiod_hog *hogs;
-
 	struct list_head bank_list;
 };
 
@@ -655,6 +654,7 @@ struct gpio_sim_hog {
 
 	char *name;
 	int dir;
+	bool active_low;
 };
 
 static struct gpio_sim_hog *to_gpio_sim_hog(struct config_item *item)
@@ -774,102 +774,6 @@ static void gpio_sim_set_reserved_ranges(struct gpio_sim_bank *bank,
 	}
 }
 
-static void gpio_sim_remove_hogs(struct gpio_sim_device *dev)
-{
-	struct gpiod_hog *hog;
-
-	if (!dev->hogs)
-		return;
-
-	gpiod_remove_hogs(dev->hogs);
-
-	for (hog = dev->hogs; hog->chip_label; hog++) {
-		kfree(hog->chip_label);
-		kfree(hog->line_name);
-	}
-
-	kfree(dev->hogs);
-	dev->hogs = NULL;
-}
-
-static int gpio_sim_add_hogs(struct gpio_sim_device *dev)
-{
-	unsigned int num_hogs = 0, idx = 0;
-	struct gpio_sim_bank *bank;
-	struct gpio_sim_line *line;
-	struct gpiod_hog *hog;
-
-	list_for_each_entry(bank, &dev->bank_list, siblings) {
-		list_for_each_entry(line, &bank->line_list, siblings) {
-			if (line->offset >= bank->num_lines)
-				continue;
-
-			if (line->hog)
-				num_hogs++;
-		}
-	}
-
-	if (!num_hogs)
-		return 0;
-
-	/* Allocate one more for the sentinel. */
-	dev->hogs = kzalloc_objs(*dev->hogs, num_hogs + 1);
-	if (!dev->hogs)
-		return -ENOMEM;
-
-	list_for_each_entry(bank, &dev->bank_list, siblings) {
-		list_for_each_entry(line, &bank->line_list, siblings) {
-			if (line->offset >= bank->num_lines)
-				continue;
-
-			if (!line->hog)
-				continue;
-
-			hog = &dev->hogs[idx++];
-
-			/*
-			 * We need to make this string manually because at this
-			 * point the device doesn't exist yet and so dev_name()
-			 * is not available.
-			 */
-			if (gpio_sim_bank_has_label(bank))
-				hog->chip_label = kstrdup(bank->label,
-							  GFP_KERNEL);
-			else
-				hog->chip_label = kasprintf(GFP_KERNEL,
-							"gpio-sim.%u:%pfwP",
-							dev->id,
-							bank->swnode);
-			if (!hog->chip_label) {
-				gpio_sim_remove_hogs(dev);
-				return -ENOMEM;
-			}
-
-			/*
-			 * We need to duplicate this because the hog config
-			 * item can be removed at any time (and we can't block
-			 * it) and gpiolib doesn't make a deep copy of the hog
-			 * data.
-			 */
-			if (line->hog->name) {
-				hog->line_name = kstrdup(line->hog->name,
-							 GFP_KERNEL);
-				if (!hog->line_name) {
-					gpio_sim_remove_hogs(dev);
-					return -ENOMEM;
-				}
-			}
-
-			hog->chip_hwnum = line->offset;
-			hog->dflags = line->hog->dir;
-		}
-	}
-
-	gpiod_add_hogs(dev->hogs);
-
-	return 0;
-}
-
 static struct fwnode_handle *
 gpio_sim_make_bank_swnode(struct gpio_sim_bank *bank,
 			  struct fwnode_handle *parent)
@@ -917,12 +821,61 @@ gpio_sim_make_bank_swnode(struct gpio_sim_bank *bank,
 	return fwnode_create_software_node(properties, parent);
 }
 
+static int gpio_sim_bank_add_hogs(struct gpio_sim_bank *bank)
+{
+	struct property_entry properties[GPIO_SIM_HOG_PROP_MAX];
+	struct fwnode_handle *swnode;
+	struct gpio_sim_line *line;
+	struct gpio_sim_hog *hog;
+	unsigned int idx;
+	u32 gpios[2];
+
+	list_for_each_entry(line, &bank->line_list, siblings) {
+		if (!line->hog)
+			continue;
+
+		hog = line->hog;
+
+		gpios[0] = line->offset;
+		gpios[1] = hog->active_low ? 1 : 0;
+
+		memset(properties, 0, sizeof(properties));
+
+		idx = 0;
+		properties[idx++] = PROPERTY_ENTRY_BOOL("gpio-hog");
+		properties[idx++] = PROPERTY_ENTRY_U32_ARRAY("gpios", gpios);
+		properties[idx++] = PROPERTY_ENTRY_STRING("line-name", hog->name);
+
+		switch (hog->dir) {
+		case GPIOD_IN:
+			properties[idx++] = PROPERTY_ENTRY_BOOL("input");
+			break;
+		case GPIOD_OUT_HIGH:
+			properties[idx++] = PROPERTY_ENTRY_BOOL("output-high");
+			break;
+		case GPIOD_OUT_LOW:
+			properties[idx++] = PROPERTY_ENTRY_BOOL("output-low");
+			break;
+		default:
+			/* Would have been validated at configfs store. */
+			WARN(1, "Unexpected hog direction value: %d", hog->dir);
+			return -EINVAL;
+		}
+
+		swnode = fwnode_create_software_node(properties, bank->swnode);
+		if (IS_ERR(swnode))
+			return PTR_ERR(swnode);
+	}
+
+	return 0;
+}
+
 static void gpio_sim_remove_swnode_recursive(struct fwnode_handle *swnode)
 {
 	struct fwnode_handle *child;
 
 	fwnode_for_each_child_node(swnode, child)
-		fwnode_remove_software_node(child);
+		gpio_sim_remove_swnode_recursive(child);
 
 	fwnode_remove_software_node(swnode);
 }
@@ -977,12 +930,12 @@ static int gpio_sim_device_activate(struct gpio_sim_device *dev)
 			gpio_sim_remove_swnode_recursive(swnode);
 			return ret;
 		}
-	}
 
-	ret = gpio_sim_add_hogs(dev);
-	if (ret) {
-		gpio_sim_remove_swnode_recursive(swnode);
-		return ret;
+		ret = gpio_sim_bank_add_hogs(bank);
+		if (ret) {
+			gpio_sim_remove_swnode_recursive(swnode);
+			return ret;
+		}
 	}
 
 	pdevinfo.name = "gpio-sim";
@@ -991,7 +944,6 @@ static int gpio_sim_device_activate(struct gpio_sim_device *dev)
 
 	ret = dev_sync_probe_register(&dev->probe_data, &pdevinfo);
 	if (ret) {
-		gpio_sim_remove_hogs(dev);
 		gpio_sim_remove_swnode_recursive(swnode);
 		return ret;
 	}
@@ -1007,7 +959,6 @@ static void gpio_sim_device_deactivate(struct gpio_sim_device *dev)
 
 	swnode = dev_fwnode(&dev->probe_data.pdev->dev);
 	dev_sync_probe_unregister(&dev->probe_data);
-	gpio_sim_remove_hogs(dev);
 	gpio_sim_remove_swnode_recursive(swnode);
 }
 
@@ -1365,9 +1316,46 @@ gpio_sim_hog_config_direction_store(struct config_item *item,
 
 CONFIGFS_ATTR(gpio_sim_hog_config_, direction);
 
+static ssize_t gpio_sim_hog_config_active_low_show(struct config_item *item,
+						   char *page)
+{
+	struct gpio_sim_hog *hog = to_gpio_sim_hog(item);
+	struct gpio_sim_device *dev = gpio_sim_hog_get_device(hog);
+
+	guard(mutex)(&dev->lock);
+
+	return sprintf(page, "%c\n", hog->active_low ? '1' : '0');
+}
+
+static ssize_t
+gpio_sim_hog_config_active_low_store(struct config_item *item,
+				     const char *page, size_t count)
+{
+	struct gpio_sim_hog *hog = to_gpio_sim_hog(item);
+	struct gpio_sim_device *dev = gpio_sim_hog_get_device(hog);
+	bool active_low;
+	int ret;
+
+	guard(mutex)(&dev->lock);
+
+	if (gpio_sim_device_is_live(dev))
+		return -EBUSY;
+
+	ret = kstrtobool(page, &active_low);
+	if (ret)
+		return ret;
+
+	hog->active_low = active_low;
+
+	return count;
+}
+
+CONFIGFS_ATTR(gpio_sim_hog_config_, active_low);
+
 static struct configfs_attribute *gpio_sim_hog_config_attrs[] = {
 	&gpio_sim_hog_config_attr_name,
 	&gpio_sim_hog_config_attr_direction,
+	&gpio_sim_hog_config_attr_active_low,
 	NULL
 };
 
diff --git a/drivers/gpio/gpio-spacemit-k1.c b/drivers/gpio/gpio-spacemit-k1.c
index dbd2e81094b93..5fe813b7f9bb5 100644
--- a/drivers/gpio/gpio-spacemit-k1.c
+++ b/drivers/gpio/gpio-spacemit-k1.c
@@ -228,6 +228,7 @@ static int spacemit_gpio_add_bank(struct spacemit_gpio *sg,
 	gc->label		= dev_name(dev);
 	gc->request		= gpiochip_generic_request;
 	gc->free		= gpiochip_generic_free;
+	gc->set_config		= gpiochip_generic_config;
 	gc->ngpio		= SPACEMIT_NR_GPIOS_PER_BANK;
 	gc->base		= -1;
 	gc->of_gpio_n_cells	= 3;
diff --git a/drivers/gpio/gpio-stp-xway.c b/drivers/gpio/gpio-stp-xway.c
index 493c027afdd62..78d6c78d4aab5 100644
--- a/drivers/gpio/gpio-stp-xway.c
+++ b/drivers/gpio/gpio-stp-xway.c
@@ -102,7 +102,7 @@ static int xway_stp_get(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct xway_stp *chip = gpiochip_get_data(gc);
 
-	return (xway_stp_r32(chip->virt, XWAY_STP_CPU0) & BIT(gpio));
+	return !!(xway_stp_r32(chip->virt, XWAY_STP_CPU0) & BIT(gpio));
 }
 
 /**
diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c
index 9c874f07be755..aa7c3e44234f5 100644
--- a/drivers/gpio/gpio-tegra186.c
+++ b/drivers/gpio/gpio-tegra186.c
@@ -125,7 +125,6 @@ struct tegra_gpio_soc {
 struct tegra_gpio {
 	struct gpio_chip gpio;
 	unsigned int num_irq;
-	unsigned int *irq;
 
 	const struct tegra_gpio_soc *soc;
 	unsigned int num_irqs_per_bank;
@@ -133,6 +132,8 @@ struct tegra_gpio {
 
 	void __iomem *secure;
 	void __iomem *base;
+
+	unsigned int irq[] __counted_by(num_irq);
 };
 
 static const struct tegra_gpio_port *
@@ -857,12 +858,18 @@ static int tegra186_gpio_probe(struct platform_device *pdev)
 	struct device_node *np;
 	struct resource *res;
 	char **names;
-	int err;
+	int node, err;
+
+	err = platform_irq_count(pdev);
+	if (err < 0)
+		return err;
 
-	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+	gpio = devm_kzalloc(&pdev->dev, struct_size(gpio, irq, err), GFP_KERNEL);
 	if (!gpio)
 		return -ENOMEM;
 
+	gpio->num_irq = err;
+
 	gpio->soc = device_get_match_data(&pdev->dev);
 	gpio->gpio.label = gpio->soc->name;
 	gpio->gpio.parent = &pdev->dev;
@@ -889,21 +896,10 @@ static int tegra186_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gpio->base))
 		return PTR_ERR(gpio->base);
 
-	err = platform_irq_count(pdev);
-	if (err < 0)
-		return err;
-
-	gpio->num_irq = err;
-
 	err = tegra186_gpio_irqs_per_bank(gpio);
 	if (err < 0)
 		return err;
 
-	gpio->irq = devm_kcalloc(&pdev->dev, gpio->num_irq, sizeof(*gpio->irq),
-				 GFP_KERNEL);
-	if (!gpio->irq)
-		return -ENOMEM;
-
 	for (i = 0; i < gpio->num_irq; i++) {
 		err = platform_get_irq(pdev, i);
 		if (err < 0)
@@ -937,16 +933,22 @@ static int tegra186_gpio_probe(struct platform_device *pdev)
 	if (!names)
 		return -ENOMEM;
 
+	node = dev_to_node(&pdev->dev);
+
 	for (i = 0, offset = 0; i < gpio->soc->num_ports; i++) {
 		const struct tegra_gpio_port *port = &gpio->soc->ports[i];
 		char *name;
 
 		for (j = 0; j < port->pins; j++) {
-			if (gpio->soc->prefix)
-				name = devm_kasprintf(gpio->gpio.parent, GFP_KERNEL, "%s-P%s.%02x",
-						      gpio->soc->prefix, port->name, j);
+			if (node >= 0)
+				name = devm_kasprintf(gpio->gpio.parent, GFP_KERNEL,
+						      "%d-%sP%s.%02x", node,
+						      gpio->soc->prefix ?: "",
+						      port->name, j);
 			else
-				name = devm_kasprintf(gpio->gpio.parent, GFP_KERNEL, "P%s.%02x",
+				name = devm_kasprintf(gpio->gpio.parent, GFP_KERNEL,
+						      "%sP%s.%02x",
+						      gpio->soc->prefix ?: "",
 						      port->name, j);
 			if (!name)
 				return -ENOMEM;
@@ -1373,6 +1375,9 @@ static const struct tegra_gpio_soc tegra256_main_soc = {
 	.has_vm_support = true,
 };
 
+/* Macro to define GPIO name prefix with separator */
+#define TEGRA_GPIO_PREFIX(_x)	_x "-"
+
 #define TEGRA410_COMPUTE_GPIO_PORT(_name, _bank, _port, _pins)	\
 	TEGRA_GPIO_PORT(TEGRA410_COMPUTE, _name, _bank, _port, _pins)
 
@@ -1388,7 +1393,7 @@ static const struct tegra_gpio_soc tegra410_compute_soc = {
 	.num_ports = ARRAY_SIZE(tegra410_compute_ports),
 	.ports = tegra410_compute_ports,
 	.name = "tegra410-gpio-compute",
-	.prefix = "COMPUTE",
+	.prefix = TEGRA_GPIO_PREFIX("COMPUTE"),
 	.num_irqs_per_bank = 8,
 	.instance = 0,
 };
@@ -1418,7 +1423,7 @@ static const struct tegra_gpio_soc tegra410_system_soc = {
 	.num_ports = ARRAY_SIZE(tegra410_system_ports),
 	.ports = tegra410_system_ports,
 	.name = "tegra410-gpio-system",
-	.prefix = "SYSTEM",
+	.prefix = TEGRA_GPIO_PREFIX("SYSTEM"),
 	.num_irqs_per_bank = 8,
 	.instance = 0,
 };
diff --git a/drivers/gpio/gpio-tps65086.c b/drivers/gpio/gpio-tps65086.c
index 84b17b83476f5..df770ecf28bc4 100644
--- a/drivers/gpio/gpio-tps65086.c
+++ b/drivers/gpio/gpio-tps65086.c
@@ -50,7 +50,7 @@ static int tps65086_gpio_get(struct gpio_chip *chip, unsigned offset)
 	if (ret < 0)
 		return ret;
 
-	return val & BIT(4 + offset);
+	return !!(val & BIT(4 + offset));
 }
 
 static int tps65086_gpio_set(struct gpio_chip *chip, unsigned int offset,
diff --git a/drivers/gpio/gpio-ts4800.c b/drivers/gpio/gpio-ts4800.c
index 992ee231db9ff..0207c2b813f41 100644
--- a/drivers/gpio/gpio-ts4800.c
+++ b/drivers/gpio/gpio-ts4800.c
@@ -11,7 +11,6 @@
 #include <linux/platform_device.h>
 #include <linux/property.h>
 
-#define DEFAULT_PIN_NUMBER      16
 #define INPUT_REG_OFFSET        0x00
 #define OUTPUT_REG_OFFSET       0x02
 #define DIRECTION_REG_OFFSET    0x04
@@ -23,7 +22,6 @@ static int ts4800_gpio_probe(struct platform_device *pdev)
 	struct gpio_generic_chip *chip;
 	void __iomem *base_addr;
 	int retval;
-	u32 ngpios;
 
 	chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
 	if (!chip)
@@ -33,12 +31,6 @@ static int ts4800_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(base_addr))
 		return PTR_ERR(base_addr);
 
-	retval = device_property_read_u32(dev, "ngpios", &ngpios);
-	if (retval == -EINVAL)
-		ngpios = DEFAULT_PIN_NUMBER;
-	else if (retval)
-		return retval;
-
 	config = (struct gpio_generic_chip_config) {
 		.dev = dev,
 		.sz = 2,
@@ -52,8 +44,6 @@ static int ts4800_gpio_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, retval,
 				     "failed to initialize the generic GPIO chip\n");
 
-	chip->gc.ngpio = ngpios;
-
 	return devm_gpiochip_add_data(dev, &chip->gc, NULL);
 }
 
diff --git a/drivers/gpio/gpio-viperboard.c b/drivers/gpio/gpio-viperboard.c
index 15e495c109d2a..89087fd48a819 100644
--- a/drivers/gpio/gpio-viperboard.c
+++ b/drivers/gpio/gpio-viperboard.c
@@ -288,7 +288,7 @@ static int vprbrd_gpiob_get(struct gpio_chip *chip,
 
 	/* if io is set to output, just return the saved value */
 	if (gpio->gpiob_out & (1 << offset))
-		return gpio->gpiob_val & (1 << offset);
+		return !!(gpio->gpiob_val & (1 << offset));
 
 	mutex_lock(&vb->lock);
 
diff --git a/drivers/gpio/gpiolib-acpi-core.c b/drivers/gpio/gpiolib-acpi-core.c
index ced6375d1badf..09f860200a059 100644
--- a/drivers/gpio/gpiolib-acpi-core.c
+++ b/drivers/gpio/gpiolib-acpi-core.c
@@ -1220,75 +1220,6 @@ static void acpi_gpiochip_free_regions(struct acpi_gpio_chip *achip)
 	}
 }
 
-static struct gpio_desc *
-acpi_gpiochip_parse_own_gpio(struct acpi_gpio_chip *achip,
-			     struct fwnode_handle *fwnode,
-			     const char **name,
-			     unsigned long *lflags,
-			     enum gpiod_flags *dflags)
-{
-	struct gpio_chip *chip = achip->chip;
-	struct gpio_desc *desc;
-	u32 gpios[2];
-	int ret;
-
-	*lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
-	*dflags = GPIOD_ASIS;
-	*name = NULL;
-
-	ret = fwnode_property_read_u32_array(fwnode, "gpios", gpios,
-					     ARRAY_SIZE(gpios));
-	if (ret < 0)
-		return ERR_PTR(ret);
-
-	desc = gpiochip_get_desc(chip, gpios[0]);
-	if (IS_ERR(desc))
-		return desc;
-
-	if (gpios[1])
-		*lflags |= GPIO_ACTIVE_LOW;
-
-	if (fwnode_property_present(fwnode, "input"))
-		*dflags |= GPIOD_IN;
-	else if (fwnode_property_present(fwnode, "output-low"))
-		*dflags |= GPIOD_OUT_LOW;
-	else if (fwnode_property_present(fwnode, "output-high"))
-		*dflags |= GPIOD_OUT_HIGH;
-	else
-		return ERR_PTR(-EINVAL);
-
-	fwnode_property_read_string(fwnode, "line-name", name);
-
-	return desc;
-}
-
-static void acpi_gpiochip_scan_gpios(struct acpi_gpio_chip *achip)
-{
-	struct gpio_chip *chip = achip->chip;
-
-	device_for_each_child_node_scoped(chip->parent, fwnode) {
-		unsigned long lflags;
-		enum gpiod_flags dflags;
-		struct gpio_desc *desc;
-		const char *name;
-		int ret;
-
-		if (!fwnode_property_present(fwnode, "gpio-hog"))
-			continue;
-
-		desc = acpi_gpiochip_parse_own_gpio(achip, fwnode, &name,
-						    &lflags, &dflags);
-		if (IS_ERR(desc))
-			continue;
-
-		ret = gpiod_hog(desc, name, lflags, dflags);
-		if (ret) {
-			dev_err(chip->parent, "Failed to hog GPIO\n");
-			return;
-		}
-	}
-}
-
 void acpi_gpiochip_add(struct gpio_chip *chip)
 {
 	struct acpi_gpio_chip *acpi_gpio;
@@ -1321,7 +1252,6 @@ void acpi_gpiochip_add(struct gpio_chip *chip)
 	}
 
 	acpi_gpiochip_request_regions(acpi_gpio);
-	acpi_gpiochip_scan_gpios(acpi_gpio);
 	acpi_dev_clear_dependencies(adev);
 }
 
diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c
index 73ae77f0f2133..f36b7c06996d7 100644
--- a/drivers/gpio/gpiolib-cdev.c
+++ b/drivers/gpio/gpiolib-cdev.c
@@ -1586,15 +1586,16 @@ static const struct file_operations line_fileops = {
 #endif
 };
 
+DEFINE_FREE(linereq_free, struct linereq *, if (!IS_ERR_OR_NULL(_T)) linereq_free(_T))
+
 static int linereq_create(struct gpio_device *gdev, void __user *ip)
 {
 	struct gpio_v2_line_request ulr;
 	struct gpio_v2_line_config *lc;
-	struct linereq *lr;
-	struct file *file;
+	struct linereq *lr __free(linereq_free) = NULL;
 	u64 flags, edflags;
 	unsigned int i;
-	int fd, ret;
+	int ret;
 
 	if (copy_from_user(&ulr, ip, sizeof(ulr)))
 		return -EFAULT;
@@ -1627,10 +1628,8 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 		/* label is only initialized if consumer is set */
 		lr->label = kstrndup(ulr.consumer, sizeof(ulr.consumer) - 1,
 				     GFP_KERNEL);
-		if (!lr->label) {
-			ret = -ENOMEM;
-			goto out_free_linereq;
-		}
+		if (!lr->label)
+			return -ENOMEM;
 	}
 
 	mutex_init(&lr->config_mutex);
@@ -1649,14 +1648,12 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 		u32 offset = ulr.offsets[i];
 		struct gpio_desc *desc = gpio_device_get_desc(gdev, offset);
 
-		if (IS_ERR(desc)) {
-			ret = PTR_ERR(desc);
-			goto out_free_linereq;
-		}
+		if (IS_ERR(desc))
+			return PTR_ERR(desc);
 
 		ret = gpiod_request_user(desc, lr->label);
 		if (ret)
-			goto out_free_linereq;
+			return ret;
 
 		lr->lines[i].desc = desc;
 		flags = gpio_v2_line_config_flags(lc, i);
@@ -1664,7 +1661,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 
 		ret = gpiod_set_transitory(desc, false);
 		if (ret < 0)
-			goto out_free_linereq;
+			return ret;
 
 		edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS;
 		/*
@@ -1676,16 +1673,16 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 
 			ret = gpiod_direction_output_nonotify(desc, val);
 			if (ret)
-				goto out_free_linereq;
+				return ret;
 		} else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
 			ret = gpiod_direction_input_nonotify(desc);
 			if (ret)
-				goto out_free_linereq;
+				return ret;
 
 			ret = edge_detector_setup(&lr->lines[i], lc, i,
 						  edflags);
 			if (ret)
-				goto out_free_linereq;
+				return ret;
 		}
 
 		lr->lines[i].edflags = edflags;
@@ -1700,44 +1697,25 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 	ret = blocking_notifier_chain_register(&gdev->device_notifier,
 					       &lr->device_unregistered_nb);
 	if (ret)
-		goto out_free_linereq;
-
-	fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
-	if (fd < 0) {
-		ret = fd;
-		goto out_free_linereq;
-	}
+		return ret;
 
-	file = anon_inode_getfile("gpio-line", &line_fileops, lr,
-				  O_RDONLY | O_CLOEXEC);
-	if (IS_ERR(file)) {
-		ret = PTR_ERR(file);
-		goto out_put_unused_fd;
-	}
+	FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
+		   anon_inode_getfile("gpio-line", &line_fileops, lr,
+				      O_RDONLY | O_CLOEXEC));
+	if (fdf.err)
+		return fdf.err;
+	retain_and_null_ptr(lr);
 
-	ulr.fd = fd;
-	if (copy_to_user(ip, &ulr, sizeof(ulr))) {
-		/*
-		 * fput() will trigger the release() callback, so do not go onto
-		 * the regular error cleanup path here.
-		 */
-		fput(file);
-		put_unused_fd(fd);
+	ulr.fd = fd_prepare_fd(fdf);
+	if (copy_to_user(ip, &ulr, sizeof(ulr)))
 		return -EFAULT;
-	}
 
-	fd_install(fd, file);
+	fd_publish(fdf);
 
 	dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
-		lr->num_lines);
+		ulr.num_lines);
 
 	return 0;
-
-out_put_unused_fd:
-	put_unused_fd(fd);
-out_free_linereq:
-	linereq_free(lr);
-	return ret;
 }
 
 #ifdef CONFIG_GPIO_CDEV_V1
@@ -2010,16 +1988,16 @@ static irqreturn_t lineevent_irq_handler(int irq, void *p)
 	return IRQ_WAKE_THREAD;
 }
 
+DEFINE_FREE(lineevent_free, struct lineevent_state *, if (!IS_ERR_OR_NULL(_T)) lineevent_free(_T))
+
 static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 {
 	struct gpioevent_request eventreq;
-	struct lineevent_state *le;
+	struct lineevent_state *le __free(lineevent_free) = NULL;
 	struct gpio_desc *desc;
-	struct file *file;
 	u32 offset;
 	u32 lflags;
 	u32 eflags;
-	int fd;
 	int ret;
 	int irq, irqflags = 0;
 	char *label;
@@ -2064,15 +2042,13 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 		le->label = kstrndup(eventreq.consumer_label,
 				     sizeof(eventreq.consumer_label) - 1,
 				     GFP_KERNEL);
-		if (!le->label) {
-			ret = -ENOMEM;
-			goto out_free_le;
-		}
+		if (!le->label)
+			return -ENOMEM;
 	}
 
 	ret = gpiod_request_user(desc, le->label);
 	if (ret)
-		goto out_free_le;
+		return ret;
 	le->desc = desc;
 	le->eflags = eflags;
 
@@ -2080,15 +2056,13 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 
 	ret = gpiod_direction_input(desc);
 	if (ret)
-		goto out_free_le;
+		return ret;
 
 	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED);
 
 	irq = gpiod_to_irq(desc);
-	if (irq <= 0) {
-		ret = -ENODEV;
-		goto out_free_le;
-	}
+	if (irq <= 0)
+		return -ENODEV;
 
 	if (eflags & GPIOEVENT_REQUEST_RISING_EDGE)
 		irqflags |= test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags) ?
@@ -2105,13 +2079,11 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 	ret = blocking_notifier_chain_register(&gdev->device_notifier,
 					       &le->device_unregistered_nb);
 	if (ret)
-		goto out_free_le;
+		return ret;
 
 	label = make_irq_label(le->label);
-	if (IS_ERR(label)) {
-		ret = PTR_ERR(label);
-		goto out_free_le;
-	}
+	if (IS_ERR(label))
+		return PTR_ERR(label);
 
 	/* Request a thread to read the events */
 	ret = request_threaded_irq(irq,
@@ -2122,46 +2094,25 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 				   le);
 	if (ret) {
 		free_irq_label(label);
-		goto out_free_le;
+		return ret;
 	}
 
 	le->irq = irq;
 
-	fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
-	if (fd < 0) {
-		ret = fd;
-		goto out_free_le;
-	}
-
-	file = anon_inode_getfile("gpio-event",
-				  &lineevent_fileops,
-				  le,
-				  O_RDONLY | O_CLOEXEC);
-	if (IS_ERR(file)) {
-		ret = PTR_ERR(file);
-		goto out_put_unused_fd;
-	}
+	FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
+		   anon_inode_getfile("gpio-event", &lineevent_fileops, le,
+				      O_RDONLY | O_CLOEXEC));
+	if (fdf.err)
+		return fdf.err;
+	retain_and_null_ptr(le);
 
-	eventreq.fd = fd;
-	if (copy_to_user(ip, &eventreq, sizeof(eventreq))) {
-		/*
-		 * fput() will trigger the release() callback, so do not go onto
-		 * the regular error cleanup path here.
-		 */
-		fput(file);
-		put_unused_fd(fd);
+	eventreq.fd = fd_prepare_fd(fdf);
+	if (copy_to_user(ip, &eventreq, sizeof(eventreq)))
 		return -EFAULT;
-	}
 
-	fd_install(fd, file);
+	fd_publish(fdf);
 
 	return 0;
-
-out_put_unused_fd:
-	put_unused_fd(fd);
-out_free_le:
-	lineevent_free(le);
-	return ret;
 }
 
 static void gpio_v2_line_info_to_v1(struct gpio_v2_line_info *info_v2,
@@ -2689,12 +2640,6 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file)
 	struct gpio_chardev_data *cdev;
 	int ret = -ENOMEM;
 
-	guard(srcu)(&gdev->srcu);
-
-	/* Fail on open if the backing gpiochip is gone */
-	if (!rcu_access_pointer(gdev->chip))
-		return -ENODEV;
-
 	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
 	if (!cdev)
 		return -ENOMEM;
@@ -2782,9 +2727,9 @@ static const struct file_operations gpio_fileops = {
 #endif
 };
 
-int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt)
+int gpiolib_cdev_register(struct gpio_chip *gc, dev_t devt)
 {
-	struct gpio_chip *gc;
+	struct gpio_device *gdev = gc->gpiodev;
 	int ret;
 
 	cdev_init(&gdev->chrdev, &gpio_fileops);
@@ -2802,14 +2747,6 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt)
 		return ret;
 	}
 
-	guard(srcu)(&gdev->srcu);
-	gc = srcu_dereference(gdev->chip, &gdev->srcu);
-	if (!gc) {
-		cdev_device_del(&gdev->chrdev, &gdev->dev);
-		destroy_workqueue(gdev->line_state_wq);
-		return -ENODEV;
-	}
-
 	gpiochip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id);
 
 	return 0;
diff --git a/drivers/gpio/gpiolib-cdev.h b/drivers/gpio/gpiolib-cdev.h
index b42644cbffb80..4a9cb3335d990 100644
--- a/drivers/gpio/gpiolib-cdev.h
+++ b/drivers/gpio/gpiolib-cdev.h
@@ -7,7 +7,7 @@
 
 struct gpio_device;
 
-int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt);
+int gpiolib_cdev_register(struct gpio_chip *gc, dev_t devt);
 void gpiolib_cdev_unregister(struct gpio_device *gdev);
 
 #endif /* GPIOLIB_CDEV_H */
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index ef1ac68b94b78..2c923d17541f2 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -10,11 +10,11 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/errno.h>
+#include <linux/fwnode.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/of_gpio.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/slab.h>
 #include <linux/string.h>
@@ -446,32 +446,6 @@ static struct gpio_desc *of_get_named_gpiod_flags(const struct device_node *np,
 	return desc;
 }
 
-/**
- * of_get_named_gpio() - Get a GPIO number to use with GPIO API
- * @np:		device node to get GPIO from
- * @propname:	Name of property containing gpio specifier(s)
- * @index:	index of the GPIO
- *
- * **DEPRECATED** This function is deprecated and must not be used in new code.
- *
- * Returns:
- * GPIO number to use with Linux generic GPIO API, or one of the errno
- * value on the error condition.
- */
-int of_get_named_gpio(const struct device_node *np, const char *propname,
-		      int index)
-{
-	struct gpio_desc *desc;
-
-	desc = of_get_named_gpiod_flags(np, propname, index, NULL);
-
-	if (IS_ERR(desc))
-		return PTR_ERR(desc);
-	else
-		return desc_to_gpio(desc);
-}
-EXPORT_SYMBOL_GPL(of_get_named_gpio);
-
 /* Converts gpio_lookup_flags into bitmask of GPIO_* values */
 static unsigned long of_convert_gpio_flags(enum of_gpio_flags flags)
 {
@@ -542,6 +516,10 @@ static struct gpio_desc *of_find_gpio_rename(struct device_node *np,
 		{ "reset",	"reset-n-io",	"marvell,nfc-uart" },
 		{ "reset",	"reset-n-io",	"mrvl,nfc-uart" },
 #endif
+#if IS_ENABLED(CONFIG_NFC_S3FWRN5_I2C)
+		{ "en",		"s3fwrn5,en-gpios",	"samsung,s3fwrn5-i2c" },
+		{ "wake",	"s3fwrn5,fw-gpios",	"samsung,s3fwrn5-i2c" },
+#endif
 #if IS_ENABLED(CONFIG_PCI_LANTIQ)
 		/* MIPS Lantiq PCI */
 		{ "reset",	"gpio-reset",	"lantiq,pci-xway" },
@@ -735,139 +713,26 @@ struct gpio_desc *of_find_gpio(struct device_node *np, const char *con_id,
 	return desc;
 }
 
-/**
- * of_parse_own_gpio() - Get a GPIO hog descriptor, names and flags for GPIO API
- * @np:		device node to get GPIO from
- * @chip:	GPIO chip whose hog is parsed
- * @idx:	Index of the GPIO to parse
- * @name:	GPIO line name
- * @lflags:	bitmask of gpio_lookup_flags GPIO_* values - returned from
- *		of_find_gpio() or of_parse_own_gpio()
- * @dflags:	gpiod_flags - optional GPIO initialization flags
- *
- * Returns:
- * GPIO descriptor to use with Linux GPIO API, or one of the errno
- * value on the error condition.
- */
-static struct gpio_desc *of_parse_own_gpio(struct device_node *np,
-					   struct gpio_chip *chip,
-					   unsigned int idx, const char **name,
-					   unsigned long *lflags,
-					   enum gpiod_flags *dflags)
+int of_gpiochip_get_lflags(struct gpio_chip *chip,
+			   struct fwnode_reference_args *gpiospec,
+			   unsigned long *lflags)
 {
-	struct device_node *chip_np;
 	enum of_gpio_flags xlate_flags;
-	struct of_phandle_args gpiospec;
+	struct of_phandle_args args;
 	struct gpio_desc *desc;
-	unsigned int i;
-	u32 tmp;
-	int ret;
 
-	chip_np = dev_of_node(&chip->gpiodev->dev);
-	if (!chip_np)
-		return ERR_PTR(-EINVAL);
+	args.np = to_of_node(gpiospec->fwnode);
+	args.args_count = gpiospec->nargs;
 
-	xlate_flags = 0;
-	*lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
-	*dflags = GPIOD_ASIS;
+	for (int i = 0; i < args.args_count; i++)
+		args.args[i] = gpiospec->args[i];
 
-	ret = of_property_read_u32(chip_np, "#gpio-cells", &tmp);
-	if (ret)
-		return ERR_PTR(ret);
-
-	gpiospec.np = chip_np;
-	gpiospec.args_count = tmp;
-
-	for (i = 0; i < tmp; i++) {
-		ret = of_property_read_u32_index(np, "gpios", idx * tmp + i,
-						 &gpiospec.args[i]);
-		if (ret)
-			return ERR_PTR(ret);
-	}
-
-	desc = of_xlate_and_get_gpiod_flags(chip, &gpiospec, &xlate_flags);
+	desc = of_xlate_and_get_gpiod_flags(chip, &args, &xlate_flags);
 	if (IS_ERR(desc))
-		return desc;
+		return PTR_ERR(desc);
 
 	*lflags = of_convert_gpio_flags(xlate_flags);
 
-	if (of_property_read_bool(np, "input"))
-		*dflags |= GPIOD_IN;
-	else if (of_property_read_bool(np, "output-low"))
-		*dflags |= GPIOD_OUT_LOW;
-	else if (of_property_read_bool(np, "output-high"))
-		*dflags |= GPIOD_OUT_HIGH;
-	else {
-		pr_warn("GPIO line %d (%pOFn): no hogging state specified, bailing out\n",
-			desc_to_gpio(desc), np);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if (name && of_property_read_string(np, "line-name", name))
-		*name = np->name;
-
-	return desc;
-}
-
-/**
- * of_gpiochip_add_hog - Add all hogs in a hog device node
- * @chip:	gpio chip to act on
- * @hog:	device node describing the hogs
- *
- * Returns:
- * 0 on success, or negative errno on failure.
- */
-static int of_gpiochip_add_hog(struct gpio_chip *chip, struct device_node *hog)
-{
-	enum gpiod_flags dflags;
-	struct gpio_desc *desc;
-	unsigned long lflags;
-	const char *name;
-	unsigned int i;
-	int ret;
-
-	for (i = 0;; i++) {
-		desc = of_parse_own_gpio(hog, chip, i, &name, &lflags, &dflags);
-		if (IS_ERR(desc))
-			break;
-
-		ret = gpiod_hog(desc, name, lflags, dflags);
-		if (ret < 0)
-			return ret;
-
-#ifdef CONFIG_OF_DYNAMIC
-		WRITE_ONCE(desc->hog, hog);
-#endif
-	}
-
-	return 0;
-}
-
-/**
- * of_gpiochip_scan_gpios - Scan gpio-controller for gpio definitions
- * @chip:	gpio chip to act on
- *
- * This is only used by of_gpiochip_add to request/set GPIO initial
- * configuration.
- *
- * Returns:
- * 0 on success, or negative errno on failure.
- */
-static int of_gpiochip_scan_gpios(struct gpio_chip *chip)
-{
-	int ret;
-
-	for_each_available_child_of_node_scoped(dev_of_node(&chip->gpiodev->dev), np) {
-		if (!of_property_read_bool(np, "gpio-hog"))
-			continue;
-
-		ret = of_gpiochip_add_hog(chip, np);
-		if (ret < 0)
-			return ret;
-
-		of_node_set_flag(np, OF_POPULATED);
-	}
-
 	return 0;
 }
 
@@ -922,7 +787,7 @@ static int of_gpio_notify(struct notifier_block *nb, unsigned long action,
 		if (!gdev)
 			return NOTIFY_DONE;	/* not for us */
 
-		ret = of_gpiochip_add_hog(gpio_device_get_chip(gdev), rd->dn);
+		ret = gpiochip_add_hog(gpio_device_get_chip(gdev), of_fwnode_handle(rd->dn));
 		if (ret < 0) {
 			pr_err("%s: failed to add hogs for %pOF\n", __func__,
 			       rd->dn);
@@ -1201,16 +1066,24 @@ int of_gpiochip_add(struct gpio_chip *chip)
 
 	of_node_get(np);
 
-	ret = of_gpiochip_scan_gpios(chip);
-	if (ret)
-		of_node_put(np);
+	for_each_available_child_of_node_scoped(np, child) {
+		if (of_property_read_bool(child, "gpio-hog"))
+			of_node_set_flag(child, OF_POPULATED);
+	}
 
 	return ret;
 }
 
 void of_gpiochip_remove(struct gpio_chip *chip)
 {
-	of_node_put(dev_of_node(&chip->gpiodev->dev));
+	struct device_node *np = dev_of_node(&chip->gpiodev->dev);
+
+	for_each_child_of_node_scoped(np, child) {
+		if (of_property_present(child, "gpio-hog"))
+			of_node_clear_flag(child, OF_POPULATED);
+	}
+
+	of_node_put(np);
 }
 
 bool of_gpiochip_instance_match(struct gpio_chip *gc, unsigned int index)
diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h
index 2257f7a498a10..218cfe5bc4ac3 100644
--- a/drivers/gpio/gpiolib-of.h
+++ b/drivers/gpio/gpiolib-of.h
@@ -10,6 +10,7 @@
 
 struct device_node;
 struct fwnode_handle;
+struct fwnode_reference_args;
 
 struct gpio_chip;
 struct gpio_desc;
@@ -24,6 +25,9 @@ int of_gpiochip_add(struct gpio_chip *gc);
 void of_gpiochip_remove(struct gpio_chip *gc);
 bool of_gpiochip_instance_match(struct gpio_chip *gc, unsigned int index);
 int of_gpio_count(const struct fwnode_handle *fwnode, const char *con_id);
+int of_gpiochip_get_lflags(struct gpio_chip *chip,
+			   struct fwnode_reference_args *gpiospec,
+			   unsigned long *lflags);
 #else
 static inline struct gpio_desc *of_find_gpio(struct device_node *np,
 					     const char *con_id,
@@ -44,6 +48,12 @@ static inline int of_gpio_count(const struct fwnode_handle *fwnode,
 {
 	return 0;
 }
+static inline int of_gpiochip_get_lflags(struct gpio_chip *chip,
+					 struct fwnode_reference_args *gpiospec,
+					 unsigned long *lflags)
+{
+	return -ENOENT;
+}
 #endif /* CONFIG_OF_GPIO */
 
 extern struct notifier_block gpio_of_notifier;
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index 270e8060e7616..fc06b0c2881b9 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -983,10 +983,10 @@ void gpiod_unexport(struct gpio_desc *desc)
 }
 EXPORT_SYMBOL_GPL(gpiod_unexport);
 
-int gpiochip_sysfs_register(struct gpio_device *gdev)
+int gpiochip_sysfs_register(struct gpio_chip *gc)
 {
+	struct gpio_device *gdev = gc->gpiodev;
 	struct gpiodev_data *data;
-	struct gpio_chip *chip;
 	struct device *parent;
 	int err;
 
@@ -999,18 +999,12 @@ int gpiochip_sysfs_register(struct gpio_device *gdev)
 	if (!class_is_registered(&gpio_class))
 		return 0;
 
-	guard(srcu)(&gdev->srcu);
-
-	chip = srcu_dereference(gdev->chip, &gdev->srcu);
-	if (!chip)
-		return -ENODEV;
-
 	/*
 	 * For sysfs backward compatibility we need to preserve this
 	 * preferred parenting to the gpio_chip parent field, if set.
 	 */
-	if (chip->parent)
-		parent = chip->parent;
+	if (gc->parent)
+		parent = gc->parent;
 	else
 		parent = &gdev->dev;
 
@@ -1029,7 +1023,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev)
 						    MKDEV(0, 0), data,
 						    gpiochip_groups,
 						    GPIOCHIP_NAME "%d",
-						    chip->base);
+						    gc->base);
 	if (IS_ERR(data->cdev_base)) {
 		err = PTR_ERR(data->cdev_base);
 		kfree(data);
@@ -1053,11 +1047,11 @@ int gpiochip_sysfs_register(struct gpio_device *gdev)
 	return 0;
 }
 
-void gpiochip_sysfs_unregister(struct gpio_device *gdev)
+void gpiochip_sysfs_unregister(struct gpio_chip *gc)
 {
+	struct gpio_device *gdev = gc->gpiodev;
 	struct gpiodev_data *data;
 	struct gpio_desc *desc;
-	struct gpio_chip *chip;
 
 	guard(mutex)(&sysfs_lock);
 
@@ -1065,13 +1059,8 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev)
 	if (!data)
 		return;
 
-	guard(srcu)(&gdev->srcu);
-	chip = srcu_dereference(gdev->chip, &gdev->srcu);
-	if (!chip)
-		return;
-
 	/* unregister gpiod class devices owned by sysfs */
-	for_each_gpio_desc_with_flag(chip, desc, GPIOD_FLAG_SYSFS) {
+	for_each_gpio_desc_with_flag(gc, desc, GPIOD_FLAG_SYSFS) {
 		gpiod_unexport_unlocked(desc);
 		gpiod_free(desc);
 	}
@@ -1090,10 +1079,9 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev)
  */
 static int gpiofind_sysfs_register(struct gpio_chip *gc, const void *data)
 {
-	struct gpio_device *gdev = gc->gpiodev;
 	int ret;
 
-	ret = gpiochip_sysfs_register(gdev);
+	ret = gpiochip_sysfs_register(gc);
 	if (ret)
 		gpiochip_err(gc, "failed to register the sysfs entry: %d\n", ret);
 
diff --git a/drivers/gpio/gpiolib-sysfs.h b/drivers/gpio/gpiolib-sysfs.h
index b794b396d6a52..d0998de043a22 100644
--- a/drivers/gpio/gpiolib-sysfs.h
+++ b/drivers/gpio/gpiolib-sysfs.h
@@ -7,17 +7,17 @@ struct gpio_device;
 
 #ifdef CONFIG_GPIO_SYSFS
 
-int gpiochip_sysfs_register(struct gpio_device *gdev);
-void gpiochip_sysfs_unregister(struct gpio_device *gdev);
+int gpiochip_sysfs_register(struct gpio_chip *gc);
+void gpiochip_sysfs_unregister(struct gpio_chip *gc);
 
 #else
 
-static inline int gpiochip_sysfs_register(struct gpio_device *gdev)
+static inline int gpiochip_sysfs_register(struct gpio_chip *gc)
 {
 	return 0;
 }
 
-static inline void gpiochip_sysfs_unregister(struct gpio_device *gdev)
+static inline void gpiochip_sysfs_unregister(struct gpio_chip *gc)
 {
 }
 
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index ada572aaebd69..cb3c6fe39abf3 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -11,6 +11,7 @@
 #include <linux/errno.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/fwnode.h>
 #include <linux/idr.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
@@ -102,9 +103,6 @@ static DEFINE_MUTEX(gpio_devices_lock);
 /* Ensures coherence during read-only accesses to the list of GPIO devices. */
 DEFINE_STATIC_SRCU(gpio_devices_srcu);
 
-static DEFINE_MUTEX(gpio_machine_hogs_mutex);
-static LIST_HEAD(gpio_machine_hogs);
-
 const char *const gpio_suffixes[] = { "gpios", "gpio", NULL };
 
 static void gpiochip_free_hogs(struct gpio_chip *gc);
@@ -340,7 +338,15 @@ struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev)
 }
 EXPORT_SYMBOL_GPL(gpio_device_get_chip);
 
-/* dynamic allocation of GPIOs, e.g. on a hotplugged device */
+/**
+ * gpiochip_find_base_unlocked() - Find a global GPIO number base
+ * @ngpio: Number of consecutive GPIOs to number
+ *
+ * Finds and allocates a consecutive range of unsigned integers representing
+ * the GPIOs on the system. Using this numberspace outside of gpiolibs
+ * internals is STRONGLY DISCOURAGED, drivers and consumers should NOT concern
+ * themselves with this numberspace.
+ */
 static int gpiochip_find_base_unlocked(u16 ngpio)
 {
 	unsigned int base = GPIO_DYNAMIC_BASE;
@@ -881,24 +887,27 @@ static const struct device_type gpio_dev_type = {
 };
 
 #ifdef CONFIG_GPIO_CDEV
-#define gcdev_register(gdev, devt)	gpiolib_cdev_register((gdev), (devt))
+#define gcdev_register(gc, devt)	gpiolib_cdev_register((gc), (devt))
 #define gcdev_unregister(gdev)		gpiolib_cdev_unregister((gdev))
 #else
 /*
  * gpiolib_cdev_register() indirectly calls device_add(), which is still
  * required even when cdev is not selected.
  */
-#define gcdev_register(gdev, devt)	device_add(&(gdev)->dev)
+#define gcdev_register(gc, devt)	device_add(&(gc)->gpiodev->dev)
 #define gcdev_unregister(gdev)		device_del(&(gdev)->dev)
 #endif
 
-static int gpiochip_setup_dev(struct gpio_device *gdev)
+/*
+ * An initial reference count has been held in gpiochip_add_data_with_key().
+ * The caller should drop the reference via gpio_device_put() on errors.
+ */
+static int gpiochip_setup_dev(struct gpio_chip *gc)
 {
+	struct gpio_device *gdev = gc->gpiodev;
 	struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev);
 	int ret;
 
-	device_initialize(&gdev->dev);
-
 	/*
 	 * If fwnode doesn't belong to another device, it's safe to clear its
 	 * initialized flag.
@@ -906,11 +915,11 @@ static int gpiochip_setup_dev(struct gpio_device *gdev)
 	if (fwnode && !fwnode->dev)
 		fwnode_dev_initialized(fwnode, false);
 
-	ret = gcdev_register(gdev, gpio_devt);
+	ret = gcdev_register(gc, gpio_devt);
 	if (ret)
 		return ret;
 
-	ret = gpiochip_sysfs_register(gdev);
+	ret = gpiochip_sysfs_register(gc);
 	if (ret)
 		goto err_remove_device;
 
@@ -924,49 +933,120 @@ static int gpiochip_setup_dev(struct gpio_device *gdev)
 	return ret;
 }
 
-static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog)
+int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode)
 {
+	struct fwnode_handle *gc_node = dev_fwnode(&gc->gpiodev->dev);
+	struct fwnode_reference_args gpiospec;
+	enum gpiod_flags dflags;
 	struct gpio_desc *desc;
-	int rv;
+	unsigned long lflags;
+	const char *name;
+	int ret, argc;
+	u32 gpios[3]; /* We support up to three-cell bindings. */
+	u32 cells;
+
+	lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
+	dflags = GPIOD_ASIS;
+	name = NULL;
+
+	argc = fwnode_property_count_u32(fwnode, "gpios");
+	if (argc < 0)
+		return argc;
+	if (argc > 3)
+		return -EINVAL;
 
-	desc = gpiochip_get_desc(gc, hog->chip_hwnum);
-	if (IS_ERR(desc)) {
-		gpiochip_err(gc, "%s: unable to get GPIO desc: %ld\n",
-			     __func__, PTR_ERR(desc));
-		return;
+	ret = fwnode_property_read_u32_array(fwnode, "gpios", gpios, argc);
+	if (ret < 0)
+		return ret;
+
+	if (is_of_node(fwnode)) {
+		/*
+		 * OF-nodes need some additional special handling for
+		 * translating of devicetree flags.
+		 */
+		ret = fwnode_property_read_u32(gc_node, "#gpio-cells", &cells);
+		if (ret)
+			return ret;
+		if (!ret && argc != cells)
+			return -EINVAL;
+
+		memset(&gpiospec, 0, sizeof(gpiospec));
+		gpiospec.fwnode = fwnode;
+		gpiospec.nargs = argc;
+
+		for (int i = 0; i < argc; i++)
+			gpiospec.args[i] = gpios[i];
+
+		ret = of_gpiochip_get_lflags(gc, &gpiospec, &lflags);
+		if (ret)
+			return ret;
+	} else {
+		/*
+		 * GPIO_ACTIVE_LOW is currently the only lookup flag
+		 * supported for non-OF firmware nodes.
+		 */
+		if (gpios[1])
+			lflags |= GPIO_ACTIVE_LOW;
 	}
 
-	rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags);
-	if (rv)
-		gpiod_err(desc, "%s: unable to hog GPIO line (%s:%u): %d\n",
-			  __func__, gc->label, hog->chip_hwnum, rv);
+	if (fwnode_property_present(fwnode, "input"))
+		dflags |= GPIOD_IN;
+	else if (fwnode_property_present(fwnode, "output-low"))
+		dflags |= GPIOD_OUT_LOW;
+	else if (fwnode_property_present(fwnode, "output-high"))
+		dflags |= GPIOD_OUT_HIGH;
+	else
+		return -EINVAL;
+
+	fwnode_property_read_string(fwnode, "line-name", &name);
+
+	desc = gpiochip_get_desc(gc, gpios[0]);
+	if (IS_ERR(desc))
+		return PTR_ERR(desc);
+
+	return gpiod_hog(desc, name, lflags, dflags);
 }
 
-static void machine_gpiochip_add(struct gpio_chip *gc)
+static int gpiochip_hog_lines(struct gpio_chip *gc)
 {
-	struct gpiod_hog *hog;
+	int ret;
 
-	guard(mutex)(&gpio_machine_hogs_mutex);
+	device_for_each_child_node_scoped(&gc->gpiodev->dev, fwnode) {
+		if (!fwnode_property_present(fwnode, "gpio-hog"))
+			continue;
 
-	list_for_each_entry(hog, &gpio_machine_hogs, list) {
-		if (!strcmp(gc->label, hog->chip_label))
-			gpiochip_machine_hog(gc, hog);
+		ret = gpiochip_add_hog(gc, fwnode);
+		if (ret)
+			return ret;
 	}
+
+	return 0;
 }
 
 static void gpiochip_setup_devs(void)
 {
 	struct gpio_device *gdev;
+	struct gpio_chip *gc;
 	int ret;
 
 	guard(srcu)(&gpio_devices_srcu);
 
 	list_for_each_entry_srcu(gdev, &gpio_devices, list,
 				 srcu_read_lock_held(&gpio_devices_srcu)) {
-		ret = gpiochip_setup_dev(gdev);
-		if (ret)
+		guard(srcu)(&gdev->srcu);
+
+		gc = srcu_dereference(gdev->chip, &gdev->srcu);
+		if (!gc) {
+			dev_err(&gdev->dev, "Underlying GPIO chip is gone\n");
+			continue;
+		}
+
+		ret = gpiochip_setup_dev(gc);
+		if (ret) {
+			gpio_device_put(gdev);
 			dev_err(&gdev->dev,
 				"Failed to initialize gpio device (%d)\n", ret);
+		}
 	}
 }
 
@@ -1047,71 +1127,72 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	int base = 0;
 	int ret;
 
-	/*
-	 * First: allocate and populate the internal stat container, and
-	 * set up the struct device.
-	 */
 	gdev = kzalloc(sizeof(*gdev), GFP_KERNEL);
 	if (!gdev)
 		return -ENOMEM;
-
-	gdev->dev.type = &gpio_dev_type;
-	gdev->dev.bus = &gpio_bus_type;
-	gdev->dev.parent = gc->parent;
-	rcu_assign_pointer(gdev->chip, gc);
-
 	gc->gpiodev = gdev;
 	gpiochip_set_data(gc, data);
 
-	device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc));
-
 	ret = ida_alloc(&gpio_ida, GFP_KERNEL);
 	if (ret < 0)
 		goto err_free_gdev;
 	gdev->id = ret;
 
-	ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id);
+	ret = init_srcu_struct(&gdev->srcu);
 	if (ret)
 		goto err_free_ida;
+	rcu_assign_pointer(gdev->chip, gc);
 
-	if (gc->parent && gc->parent->driver)
-		gdev->owner = gc->parent->driver->owner;
-	else if (gc->owner)
-		/* TODO: remove chip->owner */
-		gdev->owner = gc->owner;
-	else
-		gdev->owner = THIS_MODULE;
+	ret = init_srcu_struct(&gdev->desc_srcu);
+	if (ret)
+		goto err_cleanup_gdev_srcu;
+
+	ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id);
+	if (ret)
+		goto err_cleanup_desc_srcu;
+
+	device_initialize(&gdev->dev);
+	/*
+	 * After this point any allocated resources to `gdev` will be
+	 * free():ed by gpiodev_release().  If you add new resources
+	 * then make sure they get free():ed there.
+	 */
+	gdev->dev.type = &gpio_dev_type;
+	gdev->dev.bus = &gpio_bus_type;
+	gdev->dev.parent = gc->parent;
+	device_set_node(&gdev->dev, gpiochip_choose_fwnode(gc));
 
 	ret = gpiochip_get_ngpios(gc, &gdev->dev);
 	if (ret)
-		goto err_free_dev_name;
+		goto err_put_device;
+	gdev->ngpio = gc->ngpio;
 
 	gdev->descs = kcalloc(gc->ngpio, sizeof(*gdev->descs), GFP_KERNEL);
 	if (!gdev->descs) {
 		ret = -ENOMEM;
-		goto err_free_dev_name;
+		goto err_put_device;
 	}
 
 	gdev->label = kstrdup_const(gc->label ?: "unknown", GFP_KERNEL);
 	if (!gdev->label) {
 		ret = -ENOMEM;
-		goto err_free_descs;
+		goto err_put_device;
 	}
 
-	gdev->ngpio = gc->ngpio;
 	gdev->can_sleep = gc->can_sleep;
-
 	rwlock_init(&gdev->line_state_lock);
 	RAW_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier);
 	BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier);
-
-	ret = init_srcu_struct(&gdev->srcu);
-	if (ret)
-		goto err_free_label;
-
-	ret = init_srcu_struct(&gdev->desc_srcu);
-	if (ret)
-		goto err_cleanup_gdev_srcu;
+#ifdef CONFIG_PINCTRL
+	INIT_LIST_HEAD(&gdev->pin_ranges);
+#endif
+	if (gc->parent && gc->parent->driver)
+		gdev->owner = gc->parent->driver->owner;
+	else if (gc->owner)
+		/* TODO: remove chip->owner */
+		gdev->owner = gc->owner;
+	else
+		gdev->owner = THIS_MODULE;
 
 	scoped_guard(mutex, &gpio_devices_lock) {
 		/*
@@ -1127,7 +1208,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 			if (base < 0) {
 				ret = base;
 				base = 0;
-				goto err_cleanup_desc_srcu;
+				goto err_put_device;
 			}
 
 			/*
@@ -1147,14 +1228,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 		ret = gpiodev_add_to_list_unlocked(gdev);
 		if (ret) {
 			gpiochip_err(gc, "GPIO integer space overlap, cannot add chip\n");
-			goto err_cleanup_desc_srcu;
+			goto err_put_device;
 		}
 	}
 
-#ifdef CONFIG_PINCTRL
-	INIT_LIST_HEAD(&gdev->pin_ranges);
-#endif
-
 	if (gc->names)
 		gpiochip_set_desc_names(gc);
 
@@ -1196,7 +1273,9 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 
 	acpi_gpiochip_add(gc);
 
-	machine_gpiochip_add(gc);
+	ret = gpiochip_hog_lines(gc);
+	if (ret)
+		goto err_remove_of_chip;
 
 	ret = gpiochip_irqchip_init_valid_mask(gc);
 	if (ret)
@@ -1223,7 +1302,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	 * Otherwise, defer until later.
 	 */
 	if (gpiolib_initialized) {
-		ret = gpiochip_setup_dev(gdev);
+		ret = gpiochip_setup_dev(gc);
 		if (ret)
 			goto err_teardown_shared;
 	}
@@ -1248,25 +1327,19 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
 	scoped_guard(mutex, &gpio_devices_lock)
 		list_del_rcu(&gdev->list);
 	synchronize_srcu(&gpio_devices_srcu);
-	if (gdev->dev.release) {
-		/* release() has been registered by gpiochip_setup_dev() */
-		gpio_device_put(gdev);
-		goto err_print_message;
-	}
+err_put_device:
+	gpio_device_put(gdev);
+	goto err_print_message;
+
 err_cleanup_desc_srcu:
 	cleanup_srcu_struct(&gdev->desc_srcu);
 err_cleanup_gdev_srcu:
 	cleanup_srcu_struct(&gdev->srcu);
-err_free_label:
-	kfree_const(gdev->label);
-err_free_descs:
-	kfree(gdev->descs);
-err_free_dev_name:
-	kfree(dev_name(&gdev->dev));
 err_free_ida:
 	ida_free(&gpio_ida, gdev->id);
 err_free_gdev:
 	kfree(gdev);
+
 err_print_message:
 	/* failures here can mean systems won't boot... */
 	if (ret != -EPROBE_DEFER) {
@@ -1289,7 +1362,7 @@ void gpiochip_remove(struct gpio_chip *gc)
 	struct gpio_device *gdev = gc->gpiodev;
 
 	/* FIXME: should the legacy sysfs handling be moved to gpio_device? */
-	gpiochip_sysfs_unregister(gdev);
+	gpiochip_sysfs_unregister(gc);
 	gpiochip_free_hogs(gc);
 	gpiochip_free_remaining_irqs(gc);
 
@@ -1394,7 +1467,16 @@ EXPORT_SYMBOL_GPL(gpio_device_find_by_label);
 
 static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, const void *fwnode)
 {
-	return device_match_fwnode(&gc->gpiodev->dev, fwnode);
+	struct device *dev = &gc->gpiodev->dev;
+	struct fwnode_handle *node = dev_fwnode(dev);
+
+	if (IS_ERR(fwnode))
+		return 0;
+
+	if (device_match_fwnode(dev, fwnode))
+		return 1;
+
+	return node && node->secondary == fwnode;
 }
 
 /**
@@ -2465,8 +2547,10 @@ int gpiod_request_commit(struct gpio_desc *desc, const char *label)
 		return -EBUSY;
 
 	offset = gpiod_hwgpio(desc);
-	if (!gpiochip_line_is_valid(guard.gc, offset))
-		return -EINVAL;
+	if (!gpiochip_line_is_valid(guard.gc, offset)) {
+		ret = -EINVAL;
+		goto out_clear_bit;
+	}
 
 	/* NOTE:  gpio_request() can be called in early boot,
 	 * before IRQs are enabled, for non-sleeping (SOC) GPIOs.
@@ -4472,42 +4556,6 @@ void gpiod_remove_lookup_table(struct gpiod_lookup_table *table)
 }
 EXPORT_SYMBOL_GPL(gpiod_remove_lookup_table);
 
-/**
- * gpiod_add_hogs() - register a set of GPIO hogs from machine code
- * @hogs: table of gpio hog entries with a zeroed sentinel at the end
- */
-void gpiod_add_hogs(struct gpiod_hog *hogs)
-{
-	struct gpiod_hog *hog;
-
-	guard(mutex)(&gpio_machine_hogs_mutex);
-
-	for (hog = &hogs[0]; hog->chip_label; hog++) {
-		list_add_tail(&hog->list, &gpio_machine_hogs);
-
-		/*
-		 * The chip may have been registered earlier, so check if it
-		 * exists and, if so, try to hog the line now.
-		 */
-		struct gpio_device *gdev __free(gpio_device_put) =
-				gpio_device_find_by_label(hog->chip_label);
-		if (gdev)
-			gpiochip_machine_hog(gpio_device_get_chip(gdev), hog);
-	}
-}
-EXPORT_SYMBOL_GPL(gpiod_add_hogs);
-
-void gpiod_remove_hogs(struct gpiod_hog *hogs)
-{
-	struct gpiod_hog *hog;
-
-	guard(mutex)(&gpio_machine_hogs_mutex);
-
-	for (hog = &hogs[0]; hog->chip_label; hog++)
-		list_del(&hog->list);
-}
-EXPORT_SYMBOL_GPL(gpiod_remove_hogs);
-
 static bool gpiod_match_lookup_table(struct device *dev,
 				     const struct gpiod_lookup_table *table)
 {
@@ -5320,23 +5368,14 @@ core_initcall(gpiolib_dev_init);
 
 #ifdef CONFIG_DEBUG_FS
 
-static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev)
+static void gpiolib_dbg_show(struct seq_file *s, struct gpio_chip *gc)
 {
 	bool active_low, is_irq, is_out;
 	struct gpio_desc *desc;
 	unsigned int gpio = 0;
-	struct gpio_chip *gc;
 	unsigned long flags;
 	int value;
 
-	guard(srcu)(&gdev->srcu);
-
-	gc = srcu_dereference(gdev->chip, &gdev->srcu);
-	if (!gc) {
-		seq_puts(s, "Underlying GPIO chip is gone\n");
-		return;
-	}
-
 	for_each_gpio_desc(gc, desc) {
 		guard(srcu)(&desc->gdev->desc_srcu);
 		flags = READ_ONCE(desc->flags);
@@ -5449,7 +5488,7 @@ static int gpiolib_seq_show(struct seq_file *s, void *v)
 	if (gc->dbg_show)
 		gc->dbg_show(s, gc);
 	else
-		gpiolib_dbg_show(s, gdev);
+		gpiolib_dbg_show(s, gc);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 3abb90385829e..dc4cb61a93187 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -17,11 +17,14 @@
 #include <linux/module.h>
 #include <linux/notifier.h>
 #include <linux/spinlock.h>
+#include <linux/string.h>
 #include <linux/srcu.h>
 #include <linux/workqueue.h>
 
 #define GPIOCHIP_NAME	"gpiochip"
 
+struct fwnode_handle;
+
 /**
  * struct gpio_device - internal state container for GPIO devices
  * @dev: the GPIO device struct
@@ -107,7 +110,7 @@ extern const char *const gpio_suffixes[];
 		if (con_id)								\
 			snprintf(propname, sizeof(propname), "%s-%s", con_id, __gs);	\
 		else									\
-			snprintf(propname, sizeof(propname), "%s", __gs);		\
+			strscpy(propname, __gs);					\
 		1;									\
 	     });									\
 	     __suffixes++)
@@ -273,6 +276,7 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
 int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce);
 int gpiod_hog(struct gpio_desc *desc, const char *name,
 		unsigned long lflags, enum gpiod_flags dflags);
+int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode);
 int gpiochip_get_ngpios(struct gpio_chip *gc, struct device *dev);
 struct gpio_desc *gpiochip_get_desc(struct gpio_chip *gc, unsigned int hwnum);
 const char *gpiod_get_label(struct gpio_desc *desc);
diff --git a/drivers/gpu/Kconfig b/drivers/gpu/Kconfig
new file mode 100644
index 0000000000000..ebb2ad4b7ea0b
--- /dev/null
+++ b/drivers/gpu/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config GPU_BUDDY
+       bool
+       help
+         A page based buddy allocator for GPU memory.
+
+config GPU_BUDDY_KUNIT_TEST
+       tristate "KUnit tests for GPU buddy allocator" if !KUNIT_ALL_TESTS
+       depends on GPU_BUDDY && KUNIT
+       default KUNIT_ALL_TESTS
+       help
+         KUnit tests for the GPU buddy allocator.
diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
index 36a54d4566303..b4e5e338efa23 100644
--- a/drivers/gpu/Makefile
+++ b/drivers/gpu/Makefile
@@ -2,7 +2,9 @@
 # drm/tegra depends on host1x, so if both drivers are built-in care must be
 # taken to initialize them in the correct order. Link order is the only way
 # to ensure this currently.
-obj-y			+= host1x/ drm/ vga/
+# Similarly, buddy must come first since it is used by other drivers.
+obj-$(CONFIG_GPU_BUDDY)	+= buddy.o
+obj-y			+= host1x/ drm/ vga/ tests/
 obj-$(CONFIG_IMX_IPUV3_CORE)	+= ipu-v3/
 obj-$(CONFIG_TRACE_GPU_MEM)		+= trace/
 obj-$(CONFIG_NOVA_CORE)		+= nova-core/
diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c
new file mode 100644
index 0000000000000..52686672e99f5
--- /dev/null
+++ b/drivers/gpu/buddy.c
@@ -0,0 +1,1525 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+
+#include <linux/bug.h>
+#include <linux/export.h>
+#include <linux/kmemleak.h>
+#include <linux/module.h>
+#include <linux/sizes.h>
+
+#include <linux/gpu_buddy.h>
+
+/**
+ * gpu_buddy_assert - assert a condition in the buddy allocator
+ * @condition: condition expected to be true
+ *
+ * When CONFIG_KUNIT is enabled, evaluates @condition and, if false, triggers
+ * a WARN_ON() and also calls kunit_fail_current_test() so that any running
+ * kunit test is properly marked as failed. The stringified condition is
+ * included in the failure message for easy identification.
+ *
+ * When CONFIG_KUNIT is not enabled, this reduces to WARN_ON() so production
+ * builds retain the same warning semantics as before.
+ */
+#if IS_ENABLED(CONFIG_KUNIT)
+#include <kunit/test-bug.h>
+#define gpu_buddy_assert(condition) do {						\
+	if (WARN_ON(!(condition)))						\
+		kunit_fail_current_test("gpu_buddy_assert(" #condition ")");	\
+} while (0)
+#else
+#define gpu_buddy_assert(condition) WARN_ON(!(condition))
+#endif
+
+static struct kmem_cache *slab_blocks;
+
+static unsigned int
+gpu_buddy_block_state(struct gpu_buddy_block *block)
+{
+	return block->header & GPU_BUDDY_HEADER_STATE;
+}
+
+static bool
+gpu_buddy_block_is_allocated(struct gpu_buddy_block *block)
+{
+	return gpu_buddy_block_state(block) == GPU_BUDDY_ALLOCATED;
+}
+
+static bool
+gpu_buddy_block_is_split(struct gpu_buddy_block *block)
+{
+	return gpu_buddy_block_state(block) == GPU_BUDDY_SPLIT;
+}
+
+static unsigned int gpu_buddy_block_offset_alignment(struct gpu_buddy_block *block)
+{
+	u64 offset = gpu_buddy_block_offset(block);
+
+	if (!offset)
+		/*
+		 * __ffs64(0) is undefined; offset 0 is maximally aligned, so return
+		 * a value greater than any possible alignment.
+		 */
+		return 64 + 1;
+
+	return __ffs64(offset);
+}
+
+RB_DECLARE_CALLBACKS_MAX(static, gpu_buddy_augment_cb,
+			 struct gpu_buddy_block, rb,
+			 unsigned int, subtree_max_alignment,
+			 gpu_buddy_block_offset_alignment);
+
+static struct gpu_buddy_block *gpu_block_alloc(struct gpu_buddy *mm,
+					       struct gpu_buddy_block *parent,
+					       unsigned int order,
+					       u64 offset)
+{
+	struct gpu_buddy_block *block;
+
+	BUG_ON(order > GPU_BUDDY_MAX_ORDER);
+
+	block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL);
+	if (!block)
+		return NULL;
+
+	block->header = offset;
+	block->header |= order;
+	block->parent = parent;
+
+	RB_CLEAR_NODE(&block->rb);
+
+	BUG_ON(block->header & GPU_BUDDY_HEADER_UNUSED);
+	return block;
+}
+
+static void gpu_block_free(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block)
+{
+	kmem_cache_free(slab_blocks, block);
+}
+
+static enum gpu_buddy_free_tree
+get_block_tree(struct gpu_buddy_block *block)
+{
+	return gpu_buddy_block_is_clear(block) ?
+	       GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
+}
+
+static struct gpu_buddy_block *
+rbtree_get_free_block(const struct rb_node *node)
+{
+	return node ? rb_entry(node, struct gpu_buddy_block, rb) : NULL;
+}
+
+static struct gpu_buddy_block *
+rbtree_last_free_block(struct rb_root *root)
+{
+	return rbtree_get_free_block(rb_last(root));
+}
+
+static bool rbtree_is_empty(struct rb_root *root)
+{
+	return RB_EMPTY_ROOT(root);
+}
+
+static void rbtree_insert(struct gpu_buddy *mm,
+			  struct gpu_buddy_block *block,
+			  enum gpu_buddy_free_tree tree)
+{
+	struct rb_node **link, *parent = NULL;
+	unsigned int block_alignment, order;
+	struct gpu_buddy_block *node;
+	struct rb_root *root;
+
+	order = gpu_buddy_block_order(block);
+	block_alignment = gpu_buddy_block_offset_alignment(block);
+
+	root = &mm->free_trees[tree][order];
+	link = &root->rb_node;
+
+	while (*link) {
+		parent = *link;
+		node = rbtree_get_free_block(parent);
+		/*
+		 * Manual augmentation update during insertion traversal. Required
+		 * because rb_insert_augmented() only calls rotate callback during
+		 * rotations. This ensures all ancestors on the insertion path have
+		 * correct subtree_max_alignment values.
+		 */
+		if (node->subtree_max_alignment < block_alignment)
+			node->subtree_max_alignment = block_alignment;
+
+		if (gpu_buddy_block_offset(block) < gpu_buddy_block_offset(node))
+			link = &parent->rb_left;
+		else
+			link = &parent->rb_right;
+	}
+
+	block->subtree_max_alignment = block_alignment;
+	rb_link_node(&block->rb, parent, link);
+	rb_insert_augmented(&block->rb, root, &gpu_buddy_augment_cb);
+}
+
+static void rbtree_remove(struct gpu_buddy *mm,
+			  struct gpu_buddy_block *block)
+{
+	unsigned int order = gpu_buddy_block_order(block);
+	enum gpu_buddy_free_tree tree;
+	struct rb_root *root;
+
+	tree = get_block_tree(block);
+	root = &mm->free_trees[tree][order];
+
+	rb_erase_augmented(&block->rb, root, &gpu_buddy_augment_cb);
+	RB_CLEAR_NODE(&block->rb);
+}
+
+static void clear_reset(struct gpu_buddy_block *block)
+{
+	block->header &= ~GPU_BUDDY_HEADER_CLEAR;
+}
+
+static void mark_cleared(struct gpu_buddy_block *block)
+{
+	block->header |= GPU_BUDDY_HEADER_CLEAR;
+}
+
+static void mark_allocated(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block)
+{
+	block->header &= ~GPU_BUDDY_HEADER_STATE;
+	block->header |= GPU_BUDDY_ALLOCATED;
+
+	rbtree_remove(mm, block);
+}
+
+static void mark_free(struct gpu_buddy *mm,
+		      struct gpu_buddy_block *block)
+{
+	enum gpu_buddy_free_tree tree;
+
+	block->header &= ~GPU_BUDDY_HEADER_STATE;
+	block->header |= GPU_BUDDY_FREE;
+
+	tree = get_block_tree(block);
+	rbtree_insert(mm, block, tree);
+}
+
+static void mark_split(struct gpu_buddy *mm,
+		       struct gpu_buddy_block *block)
+{
+	block->header &= ~GPU_BUDDY_HEADER_STATE;
+	block->header |= GPU_BUDDY_SPLIT;
+
+	rbtree_remove(mm, block);
+}
+
+static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
+{
+	return s1 <= e2 && e1 >= s2;
+}
+
+static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
+{
+	return s1 <= s2 && e1 >= e2;
+}
+
+static struct gpu_buddy_block *
+__get_buddy(struct gpu_buddy_block *block)
+{
+	struct gpu_buddy_block *parent;
+
+	parent = block->parent;
+	if (!parent)
+		return NULL;
+
+	if (parent->left == block)
+		return parent->right;
+
+	return parent->left;
+}
+
+static unsigned int __gpu_buddy_free(struct gpu_buddy *mm,
+				     struct gpu_buddy_block *block,
+				     bool force_merge)
+{
+	struct gpu_buddy_block *parent;
+	unsigned int order;
+
+	while ((parent = block->parent)) {
+		struct gpu_buddy_block *buddy;
+
+		buddy = __get_buddy(block);
+
+		if (!gpu_buddy_block_is_free(buddy))
+			break;
+
+		if (!force_merge) {
+			/*
+			 * Check the block and its buddy clear state and exit
+			 * the loop if they both have the dissimilar state.
+			 */
+			if (gpu_buddy_block_is_clear(block) !=
+			    gpu_buddy_block_is_clear(buddy))
+				break;
+
+			if (gpu_buddy_block_is_clear(block))
+				mark_cleared(parent);
+		}
+
+		rbtree_remove(mm, buddy);
+		if (force_merge && gpu_buddy_block_is_clear(buddy))
+			mm->clear_avail -= gpu_buddy_block_size(mm, buddy);
+
+		gpu_block_free(mm, block);
+		gpu_block_free(mm, buddy);
+
+		block = parent;
+	}
+
+	order = gpu_buddy_block_order(block);
+	mark_free(mm, block);
+
+	return order;
+}
+
+static int __force_merge(struct gpu_buddy *mm,
+			 u64 start,
+			 u64 end,
+			 unsigned int min_order)
+{
+	unsigned int tree, order;
+	int i;
+
+	if (!min_order)
+		return -ENOMEM;
+
+	if (min_order > mm->max_order)
+		return -EINVAL;
+
+	for_each_free_tree(tree) {
+		for (i = min_order - 1; i >= 0; i--) {
+			struct rb_node *iter = rb_last(&mm->free_trees[tree][i]);
+
+			while (iter) {
+				struct gpu_buddy_block *block, *buddy;
+				u64 block_start, block_end;
+
+				block = rbtree_get_free_block(iter);
+				iter = rb_prev(iter);
+
+				if (!block || !block->parent)
+					continue;
+
+				block_start = gpu_buddy_block_offset(block);
+				block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
+
+				if (!contains(start, end, block_start, block_end))
+					continue;
+
+				buddy = __get_buddy(block);
+				if (!gpu_buddy_block_is_free(buddy))
+					continue;
+
+				gpu_buddy_assert(gpu_buddy_block_is_clear(block) !=
+						 gpu_buddy_block_is_clear(buddy));
+
+				/*
+				 * Advance to the next node when the current node is the buddy,
+				 * as freeing the block will also remove its buddy from the tree.
+				 */
+				if (iter == &buddy->rb)
+					iter = rb_prev(iter);
+
+				rbtree_remove(mm, block);
+				if (gpu_buddy_block_is_clear(block))
+					mm->clear_avail -= gpu_buddy_block_size(mm, block);
+
+				order = __gpu_buddy_free(mm, block, true);
+				if (order >= min_order)
+					return 0;
+			}
+		}
+	}
+
+	return -ENOMEM;
+}
+
+/**
+ * gpu_buddy_init - init memory manager
+ *
+ * @mm: GPU buddy manager to initialize
+ * @size: size in bytes to manage
+ * @chunk_size: minimum page size in bytes for our allocations
+ *
+ * Initializes the memory manager and its resources.
+ *
+ * Returns:
+ * 0 on success, error code on failure.
+ */
+int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
+{
+	unsigned int i, j, root_count = 0;
+	u64 offset = 0;
+
+	if (size < chunk_size)
+		return -EINVAL;
+
+	if (chunk_size < SZ_4K)
+		return -EINVAL;
+
+	if (!is_power_of_2(chunk_size))
+		return -EINVAL;
+
+	size = round_down(size, chunk_size);
+
+	mm->size = size;
+	mm->avail = size;
+	mm->clear_avail = 0;
+	mm->chunk_size = chunk_size;
+	mm->max_order = ilog2(size) - ilog2(chunk_size);
+
+	BUG_ON(mm->max_order > GPU_BUDDY_MAX_ORDER);
+
+	mm->free_trees = kmalloc_array(GPU_BUDDY_MAX_FREE_TREES,
+				       sizeof(*mm->free_trees),
+				       GFP_KERNEL);
+	if (!mm->free_trees)
+		return -ENOMEM;
+
+	for_each_free_tree(i) {
+		mm->free_trees[i] = kmalloc_array(mm->max_order + 1,
+						  sizeof(struct rb_root),
+						  GFP_KERNEL);
+		if (!mm->free_trees[i])
+			goto out_free_tree;
+
+		for (j = 0; j <= mm->max_order; ++j)
+			mm->free_trees[i][j] = RB_ROOT;
+	}
+
+	mm->n_roots = hweight64(size);
+
+	mm->roots = kmalloc_array(mm->n_roots,
+				  sizeof(struct gpu_buddy_block *),
+				  GFP_KERNEL);
+	if (!mm->roots)
+		goto out_free_tree;
+
+	/*
+	 * Split into power-of-two blocks, in case we are given a size that is
+	 * not itself a power-of-two.
+	 */
+	do {
+		struct gpu_buddy_block *root;
+		unsigned int order;
+		u64 root_size;
+
+		order = ilog2(size) - ilog2(chunk_size);
+		root_size = chunk_size << order;
+
+		root = gpu_block_alloc(mm, NULL, order, offset);
+		if (!root)
+			goto out_free_roots;
+
+		mark_free(mm, root);
+
+		BUG_ON(root_count > mm->max_order);
+		BUG_ON(gpu_buddy_block_size(mm, root) < chunk_size);
+
+		mm->roots[root_count] = root;
+
+		offset += root_size;
+		size -= root_size;
+		root_count++;
+	} while (size);
+
+	return 0;
+
+out_free_roots:
+	while (root_count--)
+		gpu_block_free(mm, mm->roots[root_count]);
+	kfree(mm->roots);
+out_free_tree:
+	while (i--)
+		kfree(mm->free_trees[i]);
+	kfree(mm->free_trees);
+	return -ENOMEM;
+}
+EXPORT_SYMBOL(gpu_buddy_init);
+
+/**
+ * gpu_buddy_fini - tear down the memory manager
+ *
+ * @mm: GPU buddy manager to free
+ *
+ * Cleanup memory manager resources and the freetree
+ */
+void gpu_buddy_fini(struct gpu_buddy *mm)
+{
+	u64 root_size, size, start;
+	unsigned int order;
+	int i;
+
+	size = mm->size;
+
+	for (i = 0; i < mm->n_roots; ++i) {
+		order = ilog2(size) - ilog2(mm->chunk_size);
+		start = gpu_buddy_block_offset(mm->roots[i]);
+		__force_merge(mm, start, start + size, order);
+
+		gpu_buddy_assert(gpu_buddy_block_is_free(mm->roots[i]));
+
+		gpu_block_free(mm, mm->roots[i]);
+
+		root_size = mm->chunk_size << order;
+		size -= root_size;
+	}
+
+	gpu_buddy_assert(mm->avail == mm->size);
+
+	for_each_free_tree(i)
+		kfree(mm->free_trees[i]);
+	kfree(mm->free_trees);
+	kfree(mm->roots);
+}
+EXPORT_SYMBOL(gpu_buddy_fini);
+
+static int split_block(struct gpu_buddy *mm,
+		       struct gpu_buddy_block *block)
+{
+	unsigned int block_order = gpu_buddy_block_order(block) - 1;
+	u64 offset = gpu_buddy_block_offset(block);
+
+	BUG_ON(!gpu_buddy_block_is_free(block));
+	BUG_ON(!gpu_buddy_block_order(block));
+
+	block->left = gpu_block_alloc(mm, block, block_order, offset);
+	if (!block->left)
+		return -ENOMEM;
+
+	block->right = gpu_block_alloc(mm, block, block_order,
+				       offset + (mm->chunk_size << block_order));
+	if (!block->right) {
+		gpu_block_free(mm, block->left);
+		return -ENOMEM;
+	}
+
+	mark_split(mm, block);
+
+	if (gpu_buddy_block_is_clear(block)) {
+		mark_cleared(block->left);
+		mark_cleared(block->right);
+		clear_reset(block);
+	}
+
+	mark_free(mm, block->left);
+	mark_free(mm, block->right);
+
+	return 0;
+}
+
+/**
+ * gpu_buddy_reset_clear - reset blocks clear state
+ *
+ * @mm: GPU buddy manager
+ * @is_clear: blocks clear state
+ *
+ * Reset the clear state based on @is_clear value for each block
+ * in the freetree.
+ */
+void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear)
+{
+	enum gpu_buddy_free_tree src_tree, dst_tree;
+	u64 root_size, size, start;
+	unsigned int order;
+	int i;
+
+	size = mm->size;
+	for (i = 0; i < mm->n_roots; ++i) {
+		order = ilog2(size) - ilog2(mm->chunk_size);
+		start = gpu_buddy_block_offset(mm->roots[i]);
+		__force_merge(mm, start, start + size, order);
+
+		root_size = mm->chunk_size << order;
+		size -= root_size;
+	}
+
+	src_tree = is_clear ? GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
+	dst_tree = is_clear ? GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
+
+	for (i = 0; i <= mm->max_order; ++i) {
+		struct rb_root *root = &mm->free_trees[src_tree][i];
+		struct gpu_buddy_block *block, *tmp;
+
+		rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
+			rbtree_remove(mm, block);
+			if (is_clear) {
+				mark_cleared(block);
+				mm->clear_avail += gpu_buddy_block_size(mm, block);
+			} else {
+				clear_reset(block);
+				mm->clear_avail -= gpu_buddy_block_size(mm, block);
+			}
+
+			rbtree_insert(mm, block, dst_tree);
+		}
+	}
+}
+EXPORT_SYMBOL(gpu_buddy_reset_clear);
+
+/**
+ * gpu_buddy_free_block - free a block
+ *
+ * @mm: GPU buddy manager
+ * @block: block to be freed
+ */
+void gpu_buddy_free_block(struct gpu_buddy *mm,
+			  struct gpu_buddy_block *block)
+{
+	BUG_ON(!gpu_buddy_block_is_allocated(block));
+	mm->avail += gpu_buddy_block_size(mm, block);
+	if (gpu_buddy_block_is_clear(block))
+		mm->clear_avail += gpu_buddy_block_size(mm, block);
+
+	__gpu_buddy_free(mm, block, false);
+}
+EXPORT_SYMBOL(gpu_buddy_free_block);
+
+static void __gpu_buddy_free_list(struct gpu_buddy *mm,
+				  struct list_head *objects,
+				  bool mark_clear,
+				  bool mark_dirty)
+{
+	struct gpu_buddy_block *block, *on;
+
+	gpu_buddy_assert(!(mark_dirty && mark_clear));
+
+	list_for_each_entry_safe(block, on, objects, link) {
+		if (mark_clear)
+			mark_cleared(block);
+		else if (mark_dirty)
+			clear_reset(block);
+		gpu_buddy_free_block(mm, block);
+		cond_resched();
+	}
+	INIT_LIST_HEAD(objects);
+}
+
+static void gpu_buddy_free_list_internal(struct gpu_buddy *mm,
+					 struct list_head *objects)
+{
+	/*
+	 * Don't touch the clear/dirty bit, since allocation is still internal
+	 * at this point. For example we might have just failed part of the
+	 * allocation.
+	 */
+	__gpu_buddy_free_list(mm, objects, false, false);
+}
+
+/**
+ * gpu_buddy_free_list - free blocks
+ *
+ * @mm: GPU buddy manager
+ * @objects: input list head to free blocks
+ * @flags: optional flags like GPU_BUDDY_CLEARED
+ */
+void gpu_buddy_free_list(struct gpu_buddy *mm,
+			 struct list_head *objects,
+			 unsigned int flags)
+{
+	bool mark_clear = flags & GPU_BUDDY_CLEARED;
+
+	__gpu_buddy_free_list(mm, objects, mark_clear, !mark_clear);
+}
+EXPORT_SYMBOL(gpu_buddy_free_list);
+
+static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags)
+{
+	bool needs_clear = flags & GPU_BUDDY_CLEAR_ALLOCATION;
+
+	return needs_clear != gpu_buddy_block_is_clear(block);
+}
+
+static struct gpu_buddy_block *
+__alloc_range_bias(struct gpu_buddy *mm,
+		   u64 start, u64 end,
+		   unsigned int order,
+		   unsigned long flags,
+		   bool fallback)
+{
+	u64 req_size = mm->chunk_size << order;
+	struct gpu_buddy_block *block;
+	struct gpu_buddy_block *buddy;
+	LIST_HEAD(dfs);
+	int err;
+	int i;
+
+	end = end - 1;
+
+	for (i = 0; i < mm->n_roots; ++i)
+		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
+
+	do {
+		u64 block_start;
+		u64 block_end;
+
+		block = list_first_entry_or_null(&dfs,
+						 struct gpu_buddy_block,
+						 tmp_link);
+		if (!block)
+			break;
+
+		list_del(&block->tmp_link);
+
+		if (gpu_buddy_block_order(block) < order)
+			continue;
+
+		block_start = gpu_buddy_block_offset(block);
+		block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
+
+		if (!overlaps(start, end, block_start, block_end))
+			continue;
+
+		if (gpu_buddy_block_is_allocated(block))
+			continue;
+
+		if (block_start < start || block_end > end) {
+			u64 adjusted_start = max(block_start, start);
+			u64 adjusted_end = min(block_end, end);
+
+			if (round_down(adjusted_end + 1, req_size) <=
+			    round_up(adjusted_start, req_size))
+				continue;
+		}
+
+		if (!fallback && block_incompatible(block, flags))
+			continue;
+
+		if (contains(start, end, block_start, block_end) &&
+		    order == gpu_buddy_block_order(block)) {
+			/*
+			 * Find the free block within the range.
+			 */
+			if (gpu_buddy_block_is_free(block))
+				return block;
+
+			continue;
+		}
+
+		if (!gpu_buddy_block_is_split(block)) {
+			err = split_block(mm, block);
+			if (unlikely(err))
+				goto err_undo;
+		}
+
+		list_add(&block->right->tmp_link, &dfs);
+		list_add(&block->left->tmp_link, &dfs);
+	} while (1);
+
+	return ERR_PTR(-ENOSPC);
+
+err_undo:
+	/*
+	 * We really don't want to leave around a bunch of split blocks, since
+	 * bigger is better, so make sure we merge everything back before we
+	 * free the allocated blocks.
+	 */
+	buddy = __get_buddy(block);
+	if (buddy &&
+	    (gpu_buddy_block_is_free(block) &&
+	     gpu_buddy_block_is_free(buddy)))
+		__gpu_buddy_free(mm, block, false);
+	return ERR_PTR(err);
+}
+
+static struct gpu_buddy_block *
+__gpu_buddy_alloc_range_bias(struct gpu_buddy *mm,
+			     u64 start, u64 end,
+			     unsigned int order,
+			     unsigned long flags)
+{
+	struct gpu_buddy_block *block;
+	bool fallback = false;
+
+	block = __alloc_range_bias(mm, start, end, order,
+				   flags, fallback);
+	if (IS_ERR(block))
+		return __alloc_range_bias(mm, start, end, order,
+					  flags, !fallback);
+
+	return block;
+}
+
+static struct gpu_buddy_block *
+get_maxblock(struct gpu_buddy *mm,
+	     unsigned int order,
+	     enum gpu_buddy_free_tree tree)
+{
+	struct gpu_buddy_block *max_block = NULL, *block = NULL;
+	struct rb_root *root;
+	unsigned int i;
+
+	for (i = order; i <= mm->max_order; ++i) {
+		root = &mm->free_trees[tree][i];
+		block = rbtree_last_free_block(root);
+		if (!block)
+			continue;
+
+		if (!max_block) {
+			max_block = block;
+			continue;
+		}
+
+		if (gpu_buddy_block_offset(block) >
+		    gpu_buddy_block_offset(max_block)) {
+			max_block = block;
+		}
+	}
+
+	return max_block;
+}
+
+static struct gpu_buddy_block *
+alloc_from_freetree(struct gpu_buddy *mm,
+		    unsigned int order,
+		    unsigned long flags)
+{
+	struct gpu_buddy_block *block = NULL;
+	struct rb_root *root;
+	enum gpu_buddy_free_tree tree;
+	unsigned int tmp;
+	int err;
+
+	tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ?
+		GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
+
+	if (flags & GPU_BUDDY_TOPDOWN_ALLOCATION) {
+		block = get_maxblock(mm, order, tree);
+		if (block)
+			/* Store the obtained block order */
+			tmp = gpu_buddy_block_order(block);
+	} else {
+		for (tmp = order; tmp <= mm->max_order; ++tmp) {
+			/* Get RB tree root for this order and tree */
+			root = &mm->free_trees[tree][tmp];
+			block = rbtree_last_free_block(root);
+			if (block)
+				break;
+		}
+	}
+
+	if (!block) {
+		/* Try allocating from the other tree */
+		tree = (tree == GPU_BUDDY_CLEAR_TREE) ?
+			GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
+
+		for (tmp = order; tmp <= mm->max_order; ++tmp) {
+			root = &mm->free_trees[tree][tmp];
+			block = rbtree_last_free_block(root);
+			if (block)
+				break;
+		}
+
+		if (!block)
+			return ERR_PTR(-ENOSPC);
+	}
+
+	BUG_ON(!gpu_buddy_block_is_free(block));
+
+	while (tmp != order) {
+		err = split_block(mm, block);
+		if (unlikely(err))
+			goto err_undo;
+
+		block = block->right;
+		tmp--;
+	}
+	return block;
+
+err_undo:
+	if (tmp != order)
+		__gpu_buddy_free(mm, block, false);
+	return ERR_PTR(err);
+}
+
+static bool
+gpu_buddy_can_offset_align(u64 size, u64 min_block_size)
+{
+	return size < min_block_size && is_power_of_2(size);
+}
+
+static bool gpu_buddy_subtree_can_satisfy(struct rb_node *node,
+					  unsigned int alignment)
+{
+	struct gpu_buddy_block *block;
+
+	block = rbtree_get_free_block(node);
+	return block->subtree_max_alignment >= alignment;
+}
+
+static struct gpu_buddy_block *
+gpu_buddy_find_block_aligned(struct gpu_buddy *mm,
+			     enum gpu_buddy_free_tree tree,
+			     unsigned int order,
+			     unsigned int alignment,
+			     unsigned long flags)
+{
+	struct rb_root *root = &mm->free_trees[tree][order];
+	struct rb_node *rb = root->rb_node;
+
+	while (rb) {
+		struct gpu_buddy_block *block = rbtree_get_free_block(rb);
+		struct rb_node *left_node = rb->rb_left, *right_node = rb->rb_right;
+
+		if (right_node) {
+			if (gpu_buddy_subtree_can_satisfy(right_node, alignment)) {
+				rb = right_node;
+				continue;
+			}
+		}
+
+		if (gpu_buddy_block_offset_alignment(block) >= alignment)
+			return block;
+
+		if (left_node) {
+			if (gpu_buddy_subtree_can_satisfy(left_node, alignment)) {
+				rb = left_node;
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	return NULL;
+}
+
+static struct gpu_buddy_block *
+gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
+				    u64 size,
+				    u64 min_block_size,
+				    unsigned long flags)
+{
+	struct gpu_buddy_block *block = NULL;
+	unsigned int order, tmp, alignment;
+	struct gpu_buddy_block *buddy;
+	enum gpu_buddy_free_tree tree;
+	unsigned long pages;
+	int err;
+
+	alignment = ilog2(min_block_size);
+	pages = size >> ilog2(mm->chunk_size);
+	order = fls(pages) - 1;
+
+	tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ?
+		GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
+
+	for (tmp = order; tmp <= mm->max_order; ++tmp) {
+		block = gpu_buddy_find_block_aligned(mm, tree, tmp,
+						     alignment, flags);
+		if (!block) {
+			tree = (tree == GPU_BUDDY_CLEAR_TREE) ?
+				GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
+			block = gpu_buddy_find_block_aligned(mm, tree, tmp,
+							     alignment, flags);
+		}
+
+		if (block)
+			break;
+	}
+
+	if (!block)
+		return ERR_PTR(-ENOSPC);
+
+	while (gpu_buddy_block_order(block) > order) {
+		struct gpu_buddy_block *left, *right;
+
+		err = split_block(mm, block);
+		if (unlikely(err))
+			goto err_undo;
+
+		left  = block->left;
+		right = block->right;
+
+		if (gpu_buddy_block_offset_alignment(right) >= alignment)
+			block = right;
+		else
+			block = left;
+	}
+
+	return block;
+
+err_undo:
+	/*
+	 * We really don't want to leave around a bunch of split blocks, since
+	 * bigger is better, so make sure we merge everything back before we
+	 * free the allocated blocks.
+	 */
+	buddy = __get_buddy(block);
+	if (buddy &&
+	    (gpu_buddy_block_is_free(block) &&
+	     gpu_buddy_block_is_free(buddy)))
+		__gpu_buddy_free(mm, block, false);
+	return ERR_PTR(err);
+}
+
+static int __alloc_range(struct gpu_buddy *mm,
+			 struct list_head *dfs,
+			 u64 start, u64 size,
+			 struct list_head *blocks,
+			 u64 *total_allocated_on_err)
+{
+	struct gpu_buddy_block *block;
+	struct gpu_buddy_block *buddy;
+	u64 total_allocated = 0;
+	LIST_HEAD(allocated);
+	u64 end;
+	int err;
+
+	end = start + size - 1;
+
+	do {
+		u64 block_start;
+		u64 block_end;
+
+		block = list_first_entry_or_null(dfs,
+						 struct gpu_buddy_block,
+						 tmp_link);
+		if (!block)
+			break;
+
+		list_del(&block->tmp_link);
+
+		block_start = gpu_buddy_block_offset(block);
+		block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
+
+		if (!overlaps(start, end, block_start, block_end))
+			continue;
+
+		if (gpu_buddy_block_is_allocated(block)) {
+			err = -ENOSPC;
+			goto err_free;
+		}
+
+		if (contains(start, end, block_start, block_end)) {
+			if (gpu_buddy_block_is_free(block)) {
+				mark_allocated(mm, block);
+				total_allocated += gpu_buddy_block_size(mm, block);
+				mm->avail -= gpu_buddy_block_size(mm, block);
+				if (gpu_buddy_block_is_clear(block))
+					mm->clear_avail -= gpu_buddy_block_size(mm, block);
+				list_add_tail(&block->link, &allocated);
+				continue;
+			} else if (!mm->clear_avail) {
+				err = -ENOSPC;
+				goto err_free;
+			}
+		}
+
+		if (!gpu_buddy_block_is_split(block)) {
+			err = split_block(mm, block);
+			if (unlikely(err))
+				goto err_undo;
+		}
+
+		list_add(&block->right->tmp_link, dfs);
+		list_add(&block->left->tmp_link, dfs);
+	} while (1);
+
+	if (total_allocated < size) {
+		err = -ENOSPC;
+		goto err_free;
+	}
+
+	list_splice_tail(&allocated, blocks);
+
+	return 0;
+
+err_undo:
+	/*
+	 * We really don't want to leave around a bunch of split blocks, since
+	 * bigger is better, so make sure we merge everything back before we
+	 * free the allocated blocks.
+	 */
+	buddy = __get_buddy(block);
+	if (buddy &&
+	    (gpu_buddy_block_is_free(block) &&
+	     gpu_buddy_block_is_free(buddy)))
+		__gpu_buddy_free(mm, block, false);
+
+err_free:
+	if (err == -ENOSPC && total_allocated_on_err) {
+		list_splice_tail(&allocated, blocks);
+		*total_allocated_on_err = total_allocated;
+	} else {
+		gpu_buddy_free_list_internal(mm, &allocated);
+	}
+
+	return err;
+}
+
+static int __gpu_buddy_alloc_range(struct gpu_buddy *mm,
+				   u64 start,
+				   u64 size,
+				   u64 *total_allocated_on_err,
+				   struct list_head *blocks)
+{
+	LIST_HEAD(dfs);
+	int i;
+
+	for (i = 0; i < mm->n_roots; ++i)
+		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
+
+	return __alloc_range(mm, &dfs, start, size,
+			     blocks, total_allocated_on_err);
+}
+
+static int __alloc_contig_try_harder(struct gpu_buddy *mm,
+				     u64 size,
+				     u64 min_block_size,
+				     struct list_head *blocks)
+{
+	u64 rhs_offset, lhs_offset, lhs_size, filled;
+	struct gpu_buddy_block *block;
+	unsigned int tree, order;
+	LIST_HEAD(blocks_lhs);
+	unsigned long pages;
+	u64 modify_size;
+	int err;
+
+	modify_size = rounddown_pow_of_two(size);
+	pages = modify_size >> ilog2(mm->chunk_size);
+	order = fls(pages) - 1;
+	if (order == 0)
+		return -ENOSPC;
+
+	for_each_free_tree(tree) {
+		struct rb_root *root;
+		struct rb_node *iter;
+
+		root = &mm->free_trees[tree][order];
+		if (rbtree_is_empty(root))
+			continue;
+
+		iter = rb_last(root);
+		while (iter) {
+			block = rbtree_get_free_block(iter);
+
+			/* Allocate blocks traversing RHS */
+			rhs_offset = gpu_buddy_block_offset(block);
+			err =  __gpu_buddy_alloc_range(mm, rhs_offset, size,
+						       &filled, blocks);
+			if (!err || err != -ENOSPC)
+				return err;
+
+			lhs_size = max((size - filled), min_block_size);
+			if (!IS_ALIGNED(lhs_size, min_block_size))
+				lhs_size = round_up(lhs_size, min_block_size);
+
+			/* Allocate blocks traversing LHS */
+			lhs_offset = gpu_buddy_block_offset(block) - lhs_size;
+			err =  __gpu_buddy_alloc_range(mm, lhs_offset, lhs_size,
+						       NULL, &blocks_lhs);
+			if (!err) {
+				list_splice(&blocks_lhs, blocks);
+				return 0;
+			} else if (err != -ENOSPC) {
+				gpu_buddy_free_list_internal(mm, blocks);
+				return err;
+			}
+			/* Free blocks for the next iteration */
+			gpu_buddy_free_list_internal(mm, blocks);
+
+			iter = rb_prev(iter);
+		}
+	}
+
+	return -ENOSPC;
+}
+
+/**
+ * gpu_buddy_block_trim - free unused pages
+ *
+ * @mm: GPU buddy manager
+ * @start: start address to begin the trimming.
+ * @new_size: original size requested
+ * @blocks: Input and output list of allocated blocks.
+ * MUST contain single block as input to be trimmed.
+ * On success will contain the newly allocated blocks
+ * making up the @new_size. Blocks always appear in
+ * ascending order
+ *
+ * For contiguous allocation, we round up the size to the nearest
+ * power of two value, drivers consume *actual* size, so remaining
+ * portions are unused and can be optionally freed with this function
+ *
+ * Returns:
+ * 0 on success, error code on failure.
+ */
+int gpu_buddy_block_trim(struct gpu_buddy *mm,
+			 u64 *start,
+			 u64 new_size,
+			 struct list_head *blocks)
+{
+	struct gpu_buddy_block *parent;
+	struct gpu_buddy_block *block;
+	u64 block_start, block_end;
+	LIST_HEAD(dfs);
+	u64 new_start;
+	int err;
+
+	if (!list_is_singular(blocks))
+		return -EINVAL;
+
+	block = list_first_entry(blocks,
+				 struct gpu_buddy_block,
+				 link);
+
+	block_start = gpu_buddy_block_offset(block);
+	block_end = block_start + gpu_buddy_block_size(mm, block);
+
+	if (WARN_ON(!gpu_buddy_block_is_allocated(block)))
+		return -EINVAL;
+
+	if (new_size > gpu_buddy_block_size(mm, block))
+		return -EINVAL;
+
+	if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size))
+		return -EINVAL;
+
+	if (new_size == gpu_buddy_block_size(mm, block))
+		return 0;
+
+	new_start = block_start;
+	if (start) {
+		new_start = *start;
+
+		if (new_start < block_start)
+			return -EINVAL;
+
+		if (!IS_ALIGNED(new_start, mm->chunk_size))
+			return -EINVAL;
+
+		if (range_overflows(new_start, new_size, block_end))
+			return -EINVAL;
+	}
+
+	list_del(&block->link);
+	mark_free(mm, block);
+	mm->avail += gpu_buddy_block_size(mm, block);
+	if (gpu_buddy_block_is_clear(block))
+		mm->clear_avail += gpu_buddy_block_size(mm, block);
+
+	/* Prevent recursively freeing this node */
+	parent = block->parent;
+	block->parent = NULL;
+
+	list_add(&block->tmp_link, &dfs);
+	err =  __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL);
+	if (err) {
+		mark_allocated(mm, block);
+		mm->avail -= gpu_buddy_block_size(mm, block);
+		if (gpu_buddy_block_is_clear(block))
+			mm->clear_avail -= gpu_buddy_block_size(mm, block);
+		list_add(&block->link, blocks);
+	}
+
+	block->parent = parent;
+	return err;
+}
+EXPORT_SYMBOL(gpu_buddy_block_trim);
+
+static struct gpu_buddy_block *
+__gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
+			 u64 start, u64 end,
+			 u64 size, u64 min_block_size,
+			 unsigned int order,
+			 unsigned long flags)
+{
+	if (flags & GPU_BUDDY_RANGE_ALLOCATION)
+		/* Allocate traversing within the range */
+		return  __gpu_buddy_alloc_range_bias(mm, start, end,
+						     order, flags);
+	else if (size < min_block_size)
+		/* Allocate from an offset-aligned region without size rounding */
+		return gpu_buddy_offset_aligned_allocation(mm, size,
+							   min_block_size,
+							   flags);
+	else
+		/* Allocate from freetree */
+		return alloc_from_freetree(mm, order, flags);
+}
+
+/**
+ * gpu_buddy_alloc_blocks - allocate power-of-two blocks
+ *
+ * @mm: GPU buddy manager to allocate from
+ * @start: start of the allowed range for this block
+ * @end: end of the allowed range for this block
+ * @size: size of the allocation in bytes
+ * @min_block_size: alignment of the allocation
+ * @blocks: output list head to add allocated blocks
+ * @flags: GPU_BUDDY_*_ALLOCATION flags
+ *
+ * alloc_range_bias() called on range limitations, which traverses
+ * the tree and returns the desired block.
+ *
+ * alloc_from_freetree() called when *no* range restrictions
+ * are enforced, which picks the block from the freetree.
+ *
+ * Returns:
+ * 0 on success, error code on failure.
+ */
+int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
+			   u64 start, u64 end, u64 size,
+			   u64 min_block_size,
+			   struct list_head *blocks,
+			   unsigned long flags)
+{
+	struct gpu_buddy_block *block = NULL;
+	u64 original_size, original_min_size;
+	unsigned int min_order, order;
+	LIST_HEAD(allocated);
+	unsigned long pages;
+	int err;
+
+	if (size < mm->chunk_size)
+		return -EINVAL;
+
+	if (min_block_size < mm->chunk_size)
+		return -EINVAL;
+
+	if (!is_power_of_2(min_block_size))
+		return -EINVAL;
+
+	if (!IS_ALIGNED(start | end | size, mm->chunk_size))
+		return -EINVAL;
+
+	if (end > mm->size)
+		return -EINVAL;
+
+	if (range_overflows(start, size, mm->size))
+		return -EINVAL;
+
+	/* Actual range allocation */
+	if (start + size == end) {
+		if (!IS_ALIGNED(start | end, min_block_size))
+			return -EINVAL;
+
+		return __gpu_buddy_alloc_range(mm, start, size, NULL, blocks);
+	}
+
+	original_size = size;
+	original_min_size = min_block_size;
+
+	/* Roundup the size to power of 2 */
+	if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) {
+		size = roundup_pow_of_two(size);
+		min_block_size = size;
+		/*
+		 * Normalize the requested size to min_block_size for regular allocations.
+		 * Offset-aligned allocations intentionally skip size rounding.
+		 */
+	} else if (!gpu_buddy_can_offset_align(size, min_block_size)) {
+		size = round_up(size, min_block_size);
+	}
+
+	pages = size >> ilog2(mm->chunk_size);
+	order = fls(pages) - 1;
+	min_order = ilog2(min_block_size) - ilog2(mm->chunk_size);
+
+	if (order > mm->max_order || size > mm->size) {
+		if ((flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) &&
+		    !(flags & GPU_BUDDY_RANGE_ALLOCATION))
+			return __alloc_contig_try_harder(mm, original_size,
+							 original_min_size, blocks);
+
+		return -EINVAL;
+	}
+
+	do {
+		order = min(order, (unsigned int)fls(pages) - 1);
+		BUG_ON(order > mm->max_order);
+		/*
+		 * Regular allocations must not allocate blocks smaller than min_block_size.
+		 * Offset-aligned allocations deliberately bypass this constraint.
+		 */
+		BUG_ON(size >= min_block_size && order < min_order);
+
+		do {
+			unsigned int fallback_order;
+
+			block = __gpu_buddy_alloc_blocks(mm, start,
+							 end,
+							 size,
+							 min_block_size,
+							 order,
+							 flags);
+			if (!IS_ERR(block))
+				break;
+
+			if (size < min_block_size) {
+				fallback_order = order;
+			} else if (order == min_order) {
+				fallback_order = min_order;
+			} else {
+				order--;
+				continue;
+			}
+
+			/* Try allocation through force merge method */
+			if (mm->clear_avail &&
+			    !__force_merge(mm, start, end, fallback_order)) {
+				block = __gpu_buddy_alloc_blocks(mm, start,
+								 end,
+								 size,
+								 min_block_size,
+								 fallback_order,
+								 flags);
+				if (!IS_ERR(block)) {
+					order = fallback_order;
+					break;
+				}
+			}
+
+			/*
+			 * Try contiguous block allocation through
+			 * try harder method.
+			 */
+			if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION &&
+			    !(flags & GPU_BUDDY_RANGE_ALLOCATION))
+				return __alloc_contig_try_harder(mm,
+								 original_size,
+								 original_min_size,
+								 blocks);
+			err = -ENOSPC;
+			goto err_free;
+		} while (1);
+
+		mark_allocated(mm, block);
+		mm->avail -= gpu_buddy_block_size(mm, block);
+		if (gpu_buddy_block_is_clear(block))
+			mm->clear_avail -= gpu_buddy_block_size(mm, block);
+		kmemleak_update_trace(block);
+		list_add_tail(&block->link, &allocated);
+
+		pages -= BIT(order);
+
+		if (!pages)
+			break;
+	} while (1);
+
+	/* Trim the allocated block to the required size */
+	if (!(flags & GPU_BUDDY_TRIM_DISABLE) &&
+	    original_size != size) {
+		struct list_head *trim_list;
+		LIST_HEAD(temp);
+		u64 trim_size;
+
+		trim_list = &allocated;
+		trim_size = original_size;
+
+		if (!list_is_singular(&allocated)) {
+			block = list_last_entry(&allocated, typeof(*block), link);
+			list_move(&block->link, &temp);
+			trim_list = &temp;
+			trim_size = gpu_buddy_block_size(mm, block) -
+				(size - original_size);
+		}
+
+		gpu_buddy_block_trim(mm,
+				     NULL,
+				     trim_size,
+				     trim_list);
+
+		if (!list_empty(&temp))
+			list_splice_tail(trim_list, &allocated);
+	}
+
+	list_splice_tail(&allocated, blocks);
+	return 0;
+
+err_free:
+	gpu_buddy_free_list_internal(mm, &allocated);
+	return err;
+}
+EXPORT_SYMBOL(gpu_buddy_alloc_blocks);
+
+/**
+ * gpu_buddy_block_print - print block information
+ *
+ * @mm: GPU buddy manager
+ * @block: GPU buddy block
+ */
+void gpu_buddy_block_print(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block)
+{
+	u64 start = gpu_buddy_block_offset(block);
+	u64 size = gpu_buddy_block_size(mm, block);
+
+	pr_info("%#018llx-%#018llx: %llu\n", start, start + size, size);
+}
+EXPORT_SYMBOL(gpu_buddy_block_print);
+
+/**
+ * gpu_buddy_print - print allocator state
+ *
+ * @mm: GPU buddy manager
+ * @p: GPU printer to use
+ */
+void gpu_buddy_print(struct gpu_buddy *mm)
+{
+	int order;
+
+	pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
+		mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
+
+	for (order = mm->max_order; order >= 0; order--) {
+		struct gpu_buddy_block *block, *tmp;
+		struct rb_root *root;
+		u64 count = 0, free;
+		unsigned int tree;
+
+		for_each_free_tree(tree) {
+			root = &mm->free_trees[tree][order];
+
+			rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
+				BUG_ON(!gpu_buddy_block_is_free(block));
+				count++;
+			}
+		}
+
+		free = count * (mm->chunk_size << order);
+		if (free < SZ_1M)
+			pr_info("order-%2d free: %8llu KiB, blocks: %llu\n",
+				order, free >> 10, count);
+		else
+			pr_info("order-%2d free: %8llu MiB, blocks: %llu\n",
+				order, free >> 20, count);
+	}
+}
+EXPORT_SYMBOL(gpu_buddy_print);
+
+static void gpu_buddy_module_exit(void)
+{
+	kmem_cache_destroy(slab_blocks);
+}
+
+static int __init gpu_buddy_module_init(void)
+{
+	slab_blocks = KMEM_CACHE(gpu_buddy_block, 0);
+	if (!slab_blocks)
+		return -ENOMEM;
+
+	return 0;
+}
+
+module_init(gpu_buddy_module_init);
+module_exit(gpu_buddy_module_exit);
+
+MODULE_DESCRIPTION("GPU Buddy Allocator");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index d3d52310c9cc1..5386248e75b6d 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -55,7 +55,7 @@ config DRM_DRAW
 
 config DRM_PANIC
 	bool "Display a user-friendly message when a kernel panic occurs"
-	depends on DRM
+	depends on DRM && PRINTK
 	select FONT_SUPPORT
 	select DRM_DRAW
 	help
@@ -130,6 +130,16 @@ config DRM_PANIC_SCREEN_QR_VERSION
 	  Smaller QR code are easier to read, but will contain less debugging
 	  data. Default is 40.
 
+config DRM_RAS
+	bool "DRM RAS support"
+	depends on DRM
+	depends on NET
+	help
+	  Enables the DRM RAS (Reliability, Availability and Serviceability)
+	  support for DRM drivers. This provides a Generic Netlink interface
+	  for error reporting and queries.
+	  If in doubt, say "N".
+
 config DRM_DEBUG_DP_MST_TOPOLOGY_REFS
         bool "Enable refcount backtrace history in the DP MST helpers"
 	depends on STACKTRACE_SUPPORT
@@ -220,6 +230,7 @@ config DRM_GPUSVM
 config DRM_BUDDY
 	tristate
 	depends on DRM
+	select GPU_BUDDY
 	help
 	  A page based buddy allocator
 
@@ -269,10 +280,6 @@ config DRM_SCHED
 config DRM_PANEL_BACKLIGHT_QUIRKS
 	tristate
 
-config DRM_LIB_RANDOM
-	bool
-	default n
-
 config DRM_PRIVACY_SCREEN
 	bool
 	default n
@@ -335,6 +342,7 @@ source "drivers/gpu/drm/udl/Kconfig"
 source "drivers/gpu/drm/v3d/Kconfig"
 source "drivers/gpu/drm/vboxvideo/Kconfig"
 source "drivers/gpu/drm/vc4/Kconfig"
+source "drivers/gpu/drm/verisilicon/Kconfig"
 source "drivers/gpu/drm/vgem/Kconfig"
 source "drivers/gpu/drm/virtio/Kconfig"
 source "drivers/gpu/drm/vkms/Kconfig"
diff --git a/drivers/gpu/drm/Kconfig.debug b/drivers/gpu/drm/Kconfig.debug
index 05dc43c0b8c5a..3b7886865335a 100644
--- a/drivers/gpu/drm/Kconfig.debug
+++ b/drivers/gpu/drm/Kconfig.debug
@@ -69,7 +69,6 @@ config DRM_KUNIT_TEST
 	select DRM_EXPORT_FOR_TESTS if m
 	select DRM_GEM_SHMEM_HELPER
 	select DRM_KUNIT_TEST_HELPERS
-	select DRM_LIB_RANDOM
 	select DRM_SYSFB_HELPER
 	select PRIME_NUMBERS
 	default KUNIT_ALL_TESTS
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index ec2c5ff823826..e97faabcd7830 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -78,7 +78,6 @@ drm-$(CONFIG_DRM_CLIENT) += \
 	drm_client_event.o \
 	drm_client_modeset.o \
 	drm_client_sysrq.o
-drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_PANEL) += drm_panel.o
 drm-$(CONFIG_OF) += drm_of.o
@@ -94,6 +93,7 @@ drm-$(CONFIG_DRM_ACCEL) += ../../accel/drm_accel.o
 drm-$(CONFIG_DRM_PANIC) += drm_panic.o
 drm-$(CONFIG_DRM_DRAW) += drm_draw.o
 drm-$(CONFIG_DRM_PANIC_SCREEN_QR_CODE) += drm_panic_qr.o
+drm-$(CONFIG_DRM_RAS) += drm_ras.o drm_ras_nl.o drm_ras_genl_family.o
 obj-$(CONFIG_DRM)	+= drm.o
 
 obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
@@ -237,11 +237,12 @@ obj-y			+= solomon/
 obj-$(CONFIG_DRM_SPRD) += sprd/
 obj-$(CONFIG_DRM_LOONGSON) += loongson/
 obj-$(CONFIG_DRM_POWERVR) += imagination/
+obj-$(CONFIG_DRM_VERISILICON_DC) += verisilicon/
 
 # Ensure drm headers are self-contained and pass kernel-doc
 hdrtest-files := \
 	$(shell cd $(src) && find . -maxdepth 1 -name 'drm_*.h') \
-	$(shell cd $(src) && find display lib -name '*.h')
+	$(shell cd $(src) && find display -name '*.h')
 
 always-$(CONFIG_DRM_HEADER_TEST) += \
 	$(patsubst %.h,%.hdrtest, $(hdrtest-files))
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 8e22882b66aa4..6a7e9bfec59eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -47,7 +47,7 @@ subdir-ccflags-$(CONFIG_DRM_AMDGPU_WERROR) += -Werror
 amdgpu-y := amdgpu_drv.o
 
 # add KMS driver
-amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
+amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
 	amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \
 	atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \
 	amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \
@@ -86,7 +86,7 @@ amdgpu-y += \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \
 	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
 	nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
-	cyan_skillfish_reg_init.o soc_v1_0.o
+	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o
 
 # add DF block
 amdgpu-y += \
@@ -213,6 +213,7 @@ amdgpu-y += \
 	vcn_v4_0_5.o \
 	vcn_v5_0_0.o \
 	vcn_v5_0_1.o \
+	vcn_v5_0_2.o \
 	amdgpu_jpeg.o \
 	jpeg_v1_0.o \
 	jpeg_v2_0.o \
@@ -223,6 +224,7 @@ amdgpu-y += \
 	jpeg_v4_0_5.o \
 	jpeg_v5_0_0.o \
 	jpeg_v5_0_1.o \
+	jpeg_v5_0_2.o \
 	jpeg_v5_3_0.o
 
 # add VPE block
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 447e734c362be..892c90b8d063b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -81,6 +81,7 @@
 #include "amdgpu_sdma.h"
 #include "amdgpu_lsdma.h"
 #include "amdgpu_nbio.h"
+#include "amdgpu_reg_access.h"
 #include "amdgpu_hdp.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_virt.h"
@@ -217,9 +218,7 @@ extern struct amdgpu_watchdog_timer amdgpu_watchdog_timer;
 extern int amdgpu_async_gfx_ring;
 extern int amdgpu_mcbp;
 extern int amdgpu_discovery;
-extern int amdgpu_mes;
 extern int amdgpu_mes_log_enable;
-extern int amdgpu_mes_kiq;
 extern int amdgpu_uni_mes;
 extern int amdgpu_noretry;
 extern int amdgpu_force_asic_type;
@@ -328,6 +327,7 @@ struct kfd_vm_fault_info;
 struct amdgpu_hive_info;
 struct amdgpu_reset_context;
 struct amdgpu_reset_control;
+struct amdgpu_coredump_info;
 
 enum amdgpu_cp_irq {
 	AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -680,21 +680,6 @@ void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
 /*
  * Core structure, functions and helpers.
  */
-typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t);
-typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
-
-typedef uint32_t (*amdgpu_rreg_ext_t)(struct amdgpu_device*, uint64_t);
-typedef void (*amdgpu_wreg_ext_t)(struct amdgpu_device*, uint64_t, uint32_t);
-
-typedef uint64_t (*amdgpu_rreg64_t)(struct amdgpu_device*, uint32_t);
-typedef void (*amdgpu_wreg64_t)(struct amdgpu_device*, uint32_t, uint64_t);
-
-typedef uint64_t (*amdgpu_rreg64_ext_t)(struct amdgpu_device*, uint64_t);
-typedef void (*amdgpu_wreg64_ext_t)(struct amdgpu_device*, uint64_t, uint64_t);
-
-typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
-typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
-
 struct amdgpu_mmio_remap {
 	u32 reg_offset;
 	resource_size_t bus_addr;
@@ -791,6 +776,12 @@ struct amd_powerplay {
 					  (rid == 0x01) || \
 					  (rid == 0x10))))
 
+enum amdgpu_mqd_update_flag {
+       AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE = 1,
+       AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE = 2,
+       AMDGPU_UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
+};
+
 struct amdgpu_mqd_prop {
 	uint64_t mqd_gpu_addr;
 	uint64_t hqd_base_gpu_addr;
@@ -811,6 +802,10 @@ struct amdgpu_mqd_prop {
 	uint64_t fence_address;
 	bool tmz_queue;
 	bool kernel_queue;
+	uint32_t *cu_mask;
+	uint32_t cu_mask_count;
+	uint32_t cu_flags;
+	bool is_user_cu_masked;
 };
 
 struct amdgpu_mqd {
@@ -906,42 +901,8 @@ struct amdgpu_device {
 	/* protects concurrent MM_INDEX/DATA based register access */
 	spinlock_t mmio_idx_lock;
 	struct amdgpu_mmio_remap        rmmio_remap;
-	/* protects concurrent SMC based register access */
-	spinlock_t smc_idx_lock;
-	amdgpu_rreg_t			smc_rreg;
-	amdgpu_wreg_t			smc_wreg;
-	/* protects concurrent PCIE register access */
-	spinlock_t pcie_idx_lock;
-	amdgpu_rreg_t			pcie_rreg;
-	amdgpu_wreg_t			pcie_wreg;
-	amdgpu_rreg_t			pciep_rreg;
-	amdgpu_wreg_t			pciep_wreg;
-	amdgpu_rreg_ext_t		pcie_rreg_ext;
-	amdgpu_wreg_ext_t		pcie_wreg_ext;
-	amdgpu_rreg64_t			pcie_rreg64;
-	amdgpu_wreg64_t			pcie_wreg64;
-	amdgpu_rreg64_ext_t			pcie_rreg64_ext;
-	amdgpu_wreg64_ext_t			pcie_wreg64_ext;
-	/* protects concurrent UVD register access */
-	spinlock_t uvd_ctx_idx_lock;
-	amdgpu_rreg_t			uvd_ctx_rreg;
-	amdgpu_wreg_t			uvd_ctx_wreg;
-	/* protects concurrent DIDT register access */
-	spinlock_t didt_idx_lock;
-	amdgpu_rreg_t			didt_rreg;
-	amdgpu_wreg_t			didt_wreg;
-	/* protects concurrent gc_cac register access */
-	spinlock_t gc_cac_idx_lock;
-	amdgpu_rreg_t			gc_cac_rreg;
-	amdgpu_wreg_t			gc_cac_wreg;
-	/* protects concurrent se_cac register access */
-	spinlock_t se_cac_idx_lock;
-	amdgpu_rreg_t			se_cac_rreg;
-	amdgpu_wreg_t			se_cac_wreg;
-	/* protects concurrent ENDPOINT (audio) register access */
-	spinlock_t audio_endpt_idx_lock;
-	amdgpu_block_rreg_t		audio_endpt_rreg;
-	amdgpu_block_wreg_t		audio_endpt_wreg;
+	/* Indirect register access blocks */
+	struct amdgpu_reg_access reg;
 	struct amdgpu_doorbell		doorbell;
 
 	/* clock/pll info */
@@ -1187,6 +1148,11 @@ struct amdgpu_device {
 
 	struct amdgpu_reset_domain	*reset_domain;
 
+#ifdef CONFIG_DEV_COREDUMP
+	struct amdgpu_coredump_info	*coredump;
+	struct work_struct		coredump_work;
+#endif
+
 	struct mutex			benchmark_mutex;
 
 	bool                            scpm_enabled;
@@ -1297,42 +1263,6 @@ size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 
 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 			       void *buf, size_t size, bool write);
-uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
-			    uint32_t inst, uint32_t reg_addr, char reg_name[],
-			    uint32_t expected_value, uint32_t mask);
-uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
-			    uint32_t reg, uint32_t acc_flags);
-u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
-				    u64 reg_addr);
-uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
-				uint32_t reg, uint32_t acc_flags,
-				uint32_t xcc_id);
-void amdgpu_device_wreg(struct amdgpu_device *adev,
-			uint32_t reg, uint32_t v,
-			uint32_t acc_flags);
-void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
-				     u64 reg_addr, u32 reg_data);
-void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
-			    uint32_t reg, uint32_t v,
-			    uint32_t acc_flags,
-			    uint32_t xcc_id);
-void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
-			     uint32_t reg, uint32_t v, uint32_t xcc_id);
-void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value);
-uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
-
-u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
-				u32 reg_addr);
-u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
-				  u32 reg_addr);
-u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
-				  u64 reg_addr);
-void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
-				 u32 reg_addr, u32 reg_data);
-void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
-				   u32 reg_addr, u64 reg_data);
-void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
-				   u64 reg_addr, u64 reg_data);
 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev);
 bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev,
 				       enum amd_asic_type asic_type);
@@ -1372,28 +1302,30 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 #define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
 #define RREG32_XCC(reg, inst) amdgpu_device_xcc_rreg(adev, (reg), 0, inst)
 #define WREG32_XCC(reg, v, inst) amdgpu_device_xcc_wreg(adev, (reg), (v), 0, inst)
-#define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg))
-#define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v))
-#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg))
-#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v))
-#define RREG32_PCIE_EXT(reg) adev->pcie_rreg_ext(adev, (reg))
-#define WREG32_PCIE_EXT(reg, v) adev->pcie_wreg_ext(adev, (reg), (v))
-#define RREG64_PCIE(reg) adev->pcie_rreg64(adev, (reg))
-#define WREG64_PCIE(reg, v) adev->pcie_wreg64(adev, (reg), (v))
-#define RREG64_PCIE_EXT(reg) adev->pcie_rreg64_ext(adev, (reg))
-#define WREG64_PCIE_EXT(reg, v) adev->pcie_wreg64_ext(adev, (reg), (v))
-#define RREG32_SMC(reg) adev->smc_rreg(adev, (reg))
-#define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v))
-#define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg))
-#define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v))
-#define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg))
-#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
-#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
-#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
-#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg))
-#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v))
-#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
-#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
+#define RREG32_PCIE(reg) amdgpu_reg_pcie_rd32(adev, (reg))
+#define WREG32_PCIE(reg, v) amdgpu_reg_pcie_wr32(adev, (reg), (v))
+#define RREG32_PCIE_PORT(reg) amdgpu_reg_pciep_rd32(adev, (reg))
+#define WREG32_PCIE_PORT(reg, v) amdgpu_reg_pciep_wr32(adev, (reg), (v))
+#define RREG32_PCIE_EXT(reg) amdgpu_reg_pcie_ext_rd32(adev, (reg))
+#define WREG32_PCIE_EXT(reg, v) amdgpu_reg_pcie_ext_wr32(adev, (reg), (v))
+#define RREG64_PCIE(reg) amdgpu_reg_pcie_rd64(adev, (reg))
+#define WREG64_PCIE(reg, v) amdgpu_reg_pcie_wr64(adev, (reg), (v))
+#define RREG64_PCIE_EXT(reg) amdgpu_reg_pcie_ext_rd64(adev, (reg))
+#define WREG64_PCIE_EXT(reg, v) amdgpu_reg_pcie_ext_wr64(adev, (reg), (v))
+#define RREG32_SMC(reg) amdgpu_reg_smc_rd32(adev, (reg))
+#define WREG32_SMC(reg, v) amdgpu_reg_smc_wr32(adev, (reg), (v))
+#define RREG32_UVD_CTX(reg) amdgpu_reg_uvd_ctx_rd32(adev, (reg))
+#define WREG32_UVD_CTX(reg, v) amdgpu_reg_uvd_ctx_wr32(adev, (reg), (v))
+#define RREG32_DIDT(reg) amdgpu_reg_didt_rd32(adev, (reg))
+#define WREG32_DIDT(reg, v) amdgpu_reg_didt_wr32(adev, (reg), (v))
+#define RREG32_GC_CAC(reg) amdgpu_reg_gc_cac_rd32(adev, (reg))
+#define WREG32_GC_CAC(reg, v) amdgpu_reg_gc_cac_wr32(adev, (reg), (v))
+#define RREG32_SE_CAC(reg) amdgpu_reg_se_cac_rd32(adev, (reg))
+#define WREG32_SE_CAC(reg, v) amdgpu_reg_se_cac_wr32(adev, (reg), (v))
+#define RREG32_AUDIO_ENDPT(block, reg) \
+	amdgpu_reg_audio_endpt_rd32(adev, (block), (reg))
+#define WREG32_AUDIO_ENDPT(block, reg, v) \
+	amdgpu_reg_audio_endpt_wr32(adev, (block), (reg), (v))
 #define WREG32_P(reg, val, mask)				\
 	do {							\
 		uint32_t tmp_ = RREG32(reg);			\
@@ -1523,10 +1455,6 @@ void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
 		struct amdgpu_ring *ring);
 
 void amdgpu_device_halt(struct amdgpu_device *adev);
-u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
-				u32 reg);
-void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
-				u32 reg, u32 v);
 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev);
 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
 					    struct dma_fence *gang);
@@ -1536,6 +1464,8 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,
 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev);
 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring);
 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset);
+void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
+				   const struct amdgpu_vm_pte_funcs *vm_pte_funcs);
 
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
index 965c7e688535a..bcb180f9d3ff8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
@@ -330,7 +330,7 @@ static uint32_t kgd_gfx_v12_1_set_address_watch(struct amdgpu_device *adev,
 
 	watch_address_cntl = 0;
 	watch_address_low = lower_32_bits(watch_address);
-	watch_address_high = upper_32_bits(watch_address) & 0xffff;
+	watch_address_high = upper_32_bits(watch_address) & 0x1ffffff;
 
 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
 			TCP_WATCH0_CNTL,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 1cbba9803d314..6f3c68cde75eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -35,6 +35,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 	struct dma_fence *fence;
 	int i, r;
 
+	mutex_lock(&adev->mman.default_entity.lock);
 	stime = ktime_get();
 	for (i = 0; i < n; i++) {
 		r = amdgpu_copy_buffer(adev, &adev->mman.default_entity,
@@ -49,6 +50,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 	}
 
 exit_do_move:
+	mutex_unlock(&adev->mman.default_entity.lock);
 	etime = ktime_get();
 	*time_ms = ktime_ms_delta(etime, stime);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 4662bfbe70b2d..43864df8af046 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -36,6 +36,7 @@
 
 #define AMDGPU_BO_LIST_MAX_PRIORITY	32u
 #define AMDGPU_BO_LIST_NUM_BUCKETS	(AMDGPU_BO_LIST_MAX_PRIORITY + 1)
+#define AMDGPU_BO_LIST_MAX_ENTRIES	(128 * 1024)
 
 static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu)
 {
@@ -188,6 +189,9 @@ int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
 	const uint32_t bo_number = in->bo_number;
 	struct drm_amdgpu_bo_list_entry *info;
 
+	if (bo_number > AMDGPU_BO_LIST_MAX_ENTRIES)
+		return -EINVAL;
+
 	/* copy the handle array from userspace to a kernel buffer */
 	if (likely(info_size == bo_info_size)) {
 		info = vmemdup_array_user(uptr, bo_number, info_size);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index d1bf2e150c1ad..b04fa9fd90b79 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -246,10 +246,10 @@ amdgpu_connector_find_encoder(struct drm_connector *connector,
 	return NULL;
 }
 
-static struct edid *
+static const struct drm_edid *
 amdgpu_connector_get_hardcoded_edid(struct amdgpu_device *adev)
 {
-	return drm_edid_duplicate(drm_edid_raw(adev->mode_info.bios_hardcoded_edid));
+	return drm_edid_dup(adev->mode_info.bios_hardcoded_edid);
 }
 
 static void amdgpu_connector_get_edid(struct drm_connector *connector)
@@ -268,8 +268,8 @@ static void amdgpu_connector_get_edid(struct drm_connector *connector)
 	if ((amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) !=
 	     ENCODER_OBJECT_ID_NONE) &&
 	    amdgpu_connector->ddc_bus->has_aux) {
-		amdgpu_connector->edid = drm_get_edid(connector,
-						      &amdgpu_connector->ddc_bus->aux.ddc);
+		amdgpu_connector->edid = drm_edid_read_ddc(connector,
+							  &amdgpu_connector->ddc_bus->aux.ddc);
 	} else if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
 		   (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
 		struct amdgpu_connector_atom_dig *dig = amdgpu_connector->con_priv;
@@ -277,14 +277,14 @@ static void amdgpu_connector_get_edid(struct drm_connector *connector)
 		if ((dig->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT ||
 		     dig->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) &&
 		    amdgpu_connector->ddc_bus->has_aux)
-			amdgpu_connector->edid = drm_get_edid(connector,
-							      &amdgpu_connector->ddc_bus->aux.ddc);
+			amdgpu_connector->edid = drm_edid_read_ddc(connector,
+								  &amdgpu_connector->ddc_bus->aux.ddc);
 		else if (amdgpu_connector->ddc_bus)
-			amdgpu_connector->edid = drm_get_edid(connector,
-							      &amdgpu_connector->ddc_bus->adapter);
+			amdgpu_connector->edid = drm_edid_read_ddc(connector,
+								  &amdgpu_connector->ddc_bus->adapter);
 	} else if (amdgpu_connector->ddc_bus) {
-		amdgpu_connector->edid = drm_get_edid(connector,
-						      &amdgpu_connector->ddc_bus->adapter);
+		amdgpu_connector->edid = drm_edid_read_ddc(connector,
+							  &amdgpu_connector->ddc_bus->adapter);
 	}
 
 	if (!amdgpu_connector->edid) {
@@ -292,30 +292,22 @@ static void amdgpu_connector_get_edid(struct drm_connector *connector)
 		if (((connector->connector_type == DRM_MODE_CONNECTOR_LVDS) ||
 		     (connector->connector_type == DRM_MODE_CONNECTOR_eDP))) {
 			amdgpu_connector->edid = amdgpu_connector_get_hardcoded_edid(adev);
-			drm_connector_update_edid_property(connector, amdgpu_connector->edid);
+			drm_edid_connector_update(connector, amdgpu_connector->edid);
 		}
 	}
 }
 
-static void amdgpu_connector_free_edid(struct drm_connector *connector)
-{
-	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
-
-	kfree(amdgpu_connector->edid);
-	amdgpu_connector->edid = NULL;
-}
-
 static int amdgpu_connector_ddc_get_modes(struct drm_connector *connector)
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 	int ret;
 
 	if (amdgpu_connector->edid) {
-		drm_connector_update_edid_property(connector, amdgpu_connector->edid);
-		ret = drm_add_edid_modes(connector, amdgpu_connector->edid);
+		drm_edid_connector_update(connector, amdgpu_connector->edid);
+		ret = drm_edid_connector_add_modes(connector);
 		return ret;
 	}
-	drm_connector_update_edid_property(connector, NULL);
+	drm_edid_connector_update(connector, NULL);
 	return 0;
 }
 
@@ -754,7 +746,7 @@ static void amdgpu_connector_destroy(struct drm_connector *connector)
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-	amdgpu_connector_free_edid(connector);
+	drm_edid_free(amdgpu_connector->edid);
 	kfree(amdgpu_connector->con_priv);
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
@@ -873,7 +865,7 @@ amdgpu_connector_vga_detect(struct drm_connector *connector, bool force)
 		dret = amdgpu_display_ddc_probe(amdgpu_connector, false);
 	if (dret) {
 		amdgpu_connector->detected_by_load = false;
-		amdgpu_connector_free_edid(connector);
+		drm_edid_free(amdgpu_connector->edid);
 		amdgpu_connector_get_edid(connector);
 
 		if (!amdgpu_connector->edid) {
@@ -883,13 +875,13 @@ amdgpu_connector_vga_detect(struct drm_connector *connector, bool force)
 			ret = connector_status_connected;
 		} else {
 			amdgpu_connector->use_digital =
-				!!(amdgpu_connector->edid->input & DRM_EDID_INPUT_DIGITAL);
+				drm_edid_is_digital(amdgpu_connector->edid);
 
 			/* some oems have boards with separate digital and analog connectors
 			 * with a shared ddc line (often vga + hdmi)
 			 */
 			if (amdgpu_connector->use_digital && amdgpu_connector->shared_ddc) {
-				amdgpu_connector_free_edid(connector);
+				drm_edid_free(amdgpu_connector->edid);
 				ret = connector_status_disconnected;
 			} else {
 				ret = connector_status_connected;
@@ -984,7 +976,7 @@ static void amdgpu_connector_shared_ddc(enum drm_connector_status *status,
 					/* hpd is our only option in this case */
 					if (!amdgpu_display_hpd_sense(adev,
 								      amdgpu_connector->hpd.hpd)) {
-						amdgpu_connector_free_edid(connector);
+						drm_edid_free(amdgpu_connector->edid);
 						*status = connector_status_disconnected;
 					}
 				}
@@ -1053,7 +1045,7 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
 	}
 	if (dret) {
 		amdgpu_connector->detected_by_load = false;
-		amdgpu_connector_free_edid(connector);
+		drm_edid_free(amdgpu_connector->edid);
 		amdgpu_connector_get_edid(connector);
 
 		if (!amdgpu_connector->edid) {
@@ -1063,13 +1055,13 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
 			broken_edid = true; /* defer use_digital to later */
 		} else {
 			amdgpu_connector->use_digital =
-				!!(amdgpu_connector->edid->input & DRM_EDID_INPUT_DIGITAL);
+				drm_edid_is_digital(amdgpu_connector->edid);
 
 			/* some oems have boards with separate digital and analog connectors
 			 * with a shared ddc line (often vga + hdmi)
 			 */
 			if ((!amdgpu_connector->use_digital) && amdgpu_connector->shared_ddc) {
-				amdgpu_connector_free_edid(connector);
+				drm_edid_free(amdgpu_connector->edid);
 				ret = connector_status_disconnected;
 			} else {
 				ret = connector_status_connected;
@@ -1239,6 +1231,8 @@ static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector
 		case CONNECTOR_OBJECT_ID_HDMI_TYPE_B:
 			max_digital_pixel_clock_khz = max_dvi_single_link_pixel_clock * 2;
 			break;
+		default:
+			return MODE_BAD;
 		}
 
 		/* When the display EDID claims that it's an HDMI display,
@@ -1417,7 +1411,7 @@ amdgpu_connector_dp_detect(struct drm_connector *connector, bool force)
 		goto out;
 	}
 
-	amdgpu_connector_free_edid(connector);
+	drm_edid_free(amdgpu_connector->edid);
 
 	if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
 	    (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 24e4b4fc91564..c048217615c1f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -84,13 +84,6 @@ static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
 	if (r)
 		return r;
 
-	/*
-	 * Abort if there is no run queue associated with this entity.
-	 * Possibly because of disabled HW IP.
-	 */
-	if (entity->rq == NULL)
-		return -EINVAL;
-
 	/* Check if we can add this IB to some existing job */
 	for (i = 0; i < p->gang_size; ++i)
 		if (p->entities[i] == entity)
@@ -1747,6 +1740,13 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
 	struct drm_amdgpu_fence *fences;
 	int r;
 
+	/*
+	 * fence_count must be non-zero; dma_fence_wait_any_timeout()
+	 * does not accept an empty fence array.
+	 */
+	if (!wait->in.fence_count)
+		return -EINVAL;
+
 	/* Get the fences from userspace */
 	fences = memdup_array_user(u64_to_user_ptr(wait->in.fences),
 				   wait->in.fence_count,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index f2c038c91c708..7af86a32c0c5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -231,13 +231,19 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
 	} else {
 		struct amdgpu_fpriv *fpriv;
 
-		fpriv = container_of(ctx->ctx_mgr, struct amdgpu_fpriv, ctx_mgr);
+		/* TODO: Stop using fpriv here, we only need the xcp_id. */
+		fpriv = container_of(ctx->mgr, struct amdgpu_fpriv, ctx_mgr);
 		r = amdgpu_xcp_select_scheds(adev, hw_ip, hw_prio, fpriv,
 						&num_scheds, &scheds);
 		if (r)
 			goto error_free_entity;
 	}
 
+	if (num_scheds == 0) {
+		r = -EINVAL;
+		goto error_free_entity;
+	}
+
 	/* disable load balance if the hw engine retains context among dependent jobs */
 	if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
 	    hw_ip == AMDGPU_HW_IP_VCN_DEC ||
@@ -348,7 +354,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
 	else
 		ctx->stable_pstate = current_stable_pstate;
 
-	ctx->ctx_mgr = &(fpriv->ctx_mgr);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index 090dfe86f75ba..cf8d700a22fe8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -44,20 +44,19 @@ struct amdgpu_ctx_entity {
 
 struct amdgpu_ctx {
 	struct kref			refcount;
-	struct amdgpu_ctx_mgr		*mgr;
+	spinlock_t			ring_lock;
 	unsigned			reset_counter;
 	unsigned			reset_counter_query;
-	uint64_t			generation;
-	spinlock_t			ring_lock;
-	struct amdgpu_ctx_entity	*entities[AMDGPU_HW_IP_NUM][AMDGPU_MAX_ENTITY_NUM];
-	bool				preamble_presented;
 	int32_t				init_priority;
 	int32_t				override_priority;
+	uint32_t			stable_pstate;
 	atomic_t			guilty;
+	bool				preamble_presented;
+	uint64_t			generation;
 	unsigned long			ras_counter_ce;
 	unsigned long			ras_counter_ue;
-	uint32_t			stable_pstate;
-	struct amdgpu_ctx_mgr		*ctx_mgr;
+	struct amdgpu_ctx_mgr		*mgr;
+	struct amdgpu_ctx_entity	*entities[AMDGPU_HW_IP_NUM][AMDGPU_MAX_ENTITY_NUM];
 };
 
 struct amdgpu_ctx_mgr {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index f7467af2e102e..aa9239b310a35 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -39,6 +39,7 @@
 
 #include "amdgpu_reset.h"
 #include "amdgpu_psp_ta.h"
+#include "amdgpu_userq.h"
 
 #if defined(CONFIG_DEBUG_FS)
 
@@ -638,7 +639,7 @@ static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 	if (size & 0x3 || *pos & 0x3)
 		return -EINVAL;
 
-	if (!adev->didt_rreg)
+	if (!adev->reg.didt.rreg)
 		return -EOPNOTSUPP;
 
 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
@@ -696,7 +697,7 @@ static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user
 	if (size & 0x3 || *pos & 0x3)
 		return -EINVAL;
 
-	if (!adev->didt_wreg)
+	if (!adev->reg.didt.wreg)
 		return -EOPNOTSUPP;
 
 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
@@ -752,7 +753,7 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 	ssize_t result = 0;
 	int r;
 
-	if (!adev->smc_rreg)
+	if (!adev->reg.smc.rreg)
 		return -EOPNOTSUPP;
 
 	if (size & 0x3 || *pos & 0x3)
@@ -810,7 +811,7 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *
 	ssize_t result = 0;
 	int r;
 
-	if (!adev->smc_wreg)
+	if (!adev->reg.smc.wreg)
 		return -EOPNOTSUPP;
 
 	if (size & 0x3 || *pos & 0x3)
@@ -2156,6 +2157,53 @@ static const struct file_operations amdgpu_pt_info_fops = {
 	.release = single_release,
 };
 
+static int amdgpu_mqd_info_read(struct seq_file *m, void *unused)
+{
+	struct amdgpu_usermode_queue *queue = m->private;
+	struct amdgpu_bo *bo;
+	int r;
+
+	if (!queue || !queue->mqd.obj)
+		return -EINVAL;
+
+	bo = amdgpu_bo_ref(queue->mqd.obj);
+	r = amdgpu_bo_reserve(bo, true);
+	if (r) {
+		amdgpu_bo_unref(&bo);
+		return -EINVAL;
+	}
+
+	seq_printf(m, "queue_type: %d\n", queue->queue_type);
+	seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));
+
+	amdgpu_bo_unreserve(bo);
+	amdgpu_bo_unref(&bo);
+
+	return 0;
+}
+
+static int amdgpu_mqd_info_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, amdgpu_mqd_info_read, inode->i_private);
+}
+
+static const struct file_operations amdgpu_mqd_info_fops = {
+	.owner = THIS_MODULE,
+	.open = amdgpu_mqd_info_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+void amdgpu_debugfs_userq_init(struct drm_file *file, struct amdgpu_usermode_queue *queue, int qid)
+{
+	char queue_name[32];
+
+	scnprintf(queue_name, sizeof(queue_name), "queue_%d", qid);
+	queue->debugfs_queue = debugfs_create_dir(queue_name, file->debugfs_client);
+	debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
+}
+
 void amdgpu_debugfs_vm_init(struct drm_file *file)
 {
 	debugfs_create_file("vm_pagetable_info", 0444, file->debugfs_client, file,
@@ -2174,4 +2222,9 @@ int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
 void amdgpu_debugfs_vm_init(struct drm_file *file)
 {
 }
+void amdgpu_debugfs_userq_init(struct drm_file *file,
+			       struct amdgpu_usermode_queue *queue,
+			       int qid)
+{
+}
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
index e7b3c38e51864..e88b4a1e564c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
@@ -25,6 +25,7 @@
 /*
  * Debugfs
  */
+struct amdgpu_usermode_queue;
 
 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
 int amdgpu_debugfs_init(struct amdgpu_device *adev);
@@ -34,4 +35,7 @@ void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_vm_init(struct drm_file *file);
+void amdgpu_debugfs_userq_init(struct drm_file *file,
+			       struct amdgpu_usermode_queue *queue,
+			       int qid);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
index c38e7371bafce..2b54a67437c26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
@@ -32,8 +32,13 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
 		     bool vram_lost, struct amdgpu_job *job)
 {
 }
+void amdgpu_coredump_init(struct amdgpu_device *adev)
+{
+}
 #else
 
+#define AMDGPU_CORE_DUMP_SIZE_MAX (256 * 1024 * 1024)
+
 const char *hw_ip_names[MAX_HWIP] = {
 	[GC_HWIP]		= "GC",
 	[HDP_HWIP]		= "HDP",
@@ -196,11 +201,9 @@ static void amdgpu_devcoredump_fw_info(struct amdgpu_device *adev,
 }
 
 static ssize_t
-amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
-			void *data, size_t datalen)
+amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_info *coredump)
 {
 	struct drm_printer p;
-	struct amdgpu_coredump_info *coredump = data;
 	struct drm_print_iterator iter;
 	struct amdgpu_vm_fault_info *fault_info;
 	struct amdgpu_ip_block *ip_block;
@@ -208,7 +211,6 @@ amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
 
 	iter.data = buffer;
 	iter.offset = 0;
-	iter.start = offset;
 	iter.remain = count;
 
 	p = drm_coredump_printer(&iter);
@@ -261,6 +263,8 @@ amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
 		}
 	}
 
+	amdgpu_discovery_dump(coredump->adev, &p);
+
 	/* IP firmware information */
 	drm_printf(&p, "\nIP Firmwares\n");
 	amdgpu_devcoredump_fw_info(coredump->adev, &p);
@@ -320,9 +324,63 @@ amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
 	return count - iter.remain;
 }
 
+static ssize_t
+amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
+			void *data, size_t datalen)
+{
+	struct amdgpu_coredump_info *coredump = data;
+	ssize_t byte_copied;
+
+	if (!coredump)
+		return -ENODEV;
+
+	if (!coredump->formatted)
+		return -ENODEV;
+
+	if (offset >= coredump->formatted_size)
+		return 0;
+
+	byte_copied = count < coredump->formatted_size - offset ? count :
+		coredump->formatted_size - offset;
+	memcpy(buffer, coredump->formatted + offset, byte_copied);
+
+	return byte_copied;
+}
+
 static void amdgpu_devcoredump_free(void *data)
 {
-	kfree(data);
+	struct amdgpu_coredump_info *coredump = data;
+
+	kvfree(coredump->formatted);
+	kvfree(data);
+}
+
+static void amdgpu_devcoredump_deferred_work(struct work_struct *work)
+{
+	struct amdgpu_device *adev = container_of(work, typeof(*adev), coredump_work);
+	struct amdgpu_coredump_info *coredump = adev->coredump;
+
+	/* Do a one-time preparation of the coredump output because
+	 * repeatingly calling drm_coredump_printer is very slow.
+	 */
+	coredump->formatted_size = amdgpu_devcoredump_format(
+		NULL, AMDGPU_CORE_DUMP_SIZE_MAX, coredump);
+	coredump->formatted = kvzalloc(coredump->formatted_size, GFP_KERNEL);
+	if (!coredump->formatted) {
+		amdgpu_devcoredump_free(coredump);
+		goto end;
+	}
+
+	amdgpu_devcoredump_format(coredump->formatted, coredump->formatted_size, coredump);
+
+	/* If there's an existing coredump for this device, the free function will be
+	 * called immediately so coredump might be invalid after the call to dev_coredumpm.
+	 */
+	dev_coredumpm(coredump->adev->dev, THIS_MODULE, coredump, 0, GFP_NOWAIT,
+		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
+
+end:
+	adev->coredump = NULL;
 }
 
 void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
@@ -332,6 +390,10 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
 	struct amdgpu_coredump_info *coredump;
 	struct drm_sched_job *s_job;
 
+	/* No need to generate a new coredump if there's one in progress already. */
+	if (work_pending(&adev->coredump_work))
+		return;
+
 	coredump = kzalloc_obj(*coredump, GFP_NOWAIT);
 	if (!coredump)
 		return;
@@ -358,11 +420,20 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
 
 	ktime_get_ts64(&coredump->reset_time);
 
-	dev_coredumpm(dev->dev, THIS_MODULE, coredump, 0, GFP_NOWAIT,
-		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
+	/* Update the current coredump pointer (no lock needed, this function can only be called
+	 * from a single thread)
+	 */
+	adev->coredump = coredump;
+	/* Kick off coredump formatting to a worker thread. */
+	queue_work(system_unbound_wq, &adev->coredump_work);
 
 	drm_info(dev, "AMDGPU device coredump file has been created\n");
 	drm_info(dev, "Check your /sys/class/drm/card%d/device/devcoredump/data\n",
 		 dev->primary->index);
 }
+
+void amdgpu_coredump_init(struct amdgpu_device *adev)
+{
+	INIT_WORK(&adev->coredump_work, amdgpu_devcoredump_deferred_work);
+}
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
index ef9772c6bcc9e..b3582d0b4ca43 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
@@ -35,12 +35,19 @@ struct amdgpu_coredump_info {
 	struct amdgpu_device            *adev;
 	struct amdgpu_task_info         reset_task_info;
 	struct timespec64               reset_time;
+
 	bool                            skip_vram_check;
 	bool                            reset_vram_lost;
 	struct amdgpu_ring              *ring;
+	/* Readable form of coredevdump, generate once to speed up
+	 * reading it (see drm_coredump_printer's documentation).
+	 */
+	ssize_t				formatted_size;
+	char				*formatted;
 };
 #endif
 
 void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
 		     bool vram_lost, struct amdgpu_job *job);
+void amdgpu_coredump_init(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index d8296dfc5e8a5..ac5769d9e75ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -846,558 +846,6 @@ bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 	return false;
 }
 
-/**
- * amdgpu_device_rreg - read a memory mapped IO or indirect register
- *
- * @adev: amdgpu_device pointer
- * @reg: dword aligned register offset
- * @acc_flags: access flags which require special behavior
- *
- * Returns the 32 bit value from the offset specified.
- */
-uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
-			    uint32_t reg, uint32_t acc_flags)
-{
-	uint32_t ret;
-
-	if (amdgpu_device_skip_hw_access(adev))
-		return 0;
-
-	if ((reg * 4) < adev->rmmio_size) {
-		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, 0);
-			up_read(&adev->reset_domain->sem);
-		} else {
-			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
-		}
-	} else {
-		ret = adev->pcie_rreg(adev, reg * 4);
-	}
-
-	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
-
-	return ret;
-}
-
-/*
- * MMIO register read with bytes helper functions
- * @offset:bytes offset from MMIO start
- */
-
-/**
- * amdgpu_mm_rreg8 - read a memory mapped IO register
- *
- * @adev: amdgpu_device pointer
- * @offset: byte aligned register offset
- *
- * Returns the 8 bit value from the offset specified.
- */
-uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
-{
-	if (amdgpu_device_skip_hw_access(adev))
-		return 0;
-
-	if (offset < adev->rmmio_size)
-		return (readb(adev->rmmio + offset));
-	BUG();
-}
-
-
-/**
- * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
- *
- * @adev: amdgpu_device pointer
- * @reg: dword aligned register offset
- * @acc_flags: access flags which require special behavior
- * @xcc_id: xcc accelerated compute core id
- *
- * Returns the 32 bit value from the offset specified.
- */
-uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
-				uint32_t reg, uint32_t acc_flags,
-				uint32_t xcc_id)
-{
-	uint32_t ret, rlcg_flag;
-
-	if (amdgpu_device_skip_hw_access(adev))
-		return 0;
-
-	if ((reg * 4) < adev->rmmio_size) {
-		if (amdgpu_sriov_vf(adev) &&
-		    !amdgpu_sriov_runtime(adev) &&
-		    adev->gfx.rlc.rlcg_reg_access_supported &&
-		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
-							 GC_HWIP, false,
-							 &rlcg_flag)) {
-			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
-		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-			up_read(&adev->reset_domain->sem);
-		} else {
-			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
-		}
-	} else {
-		ret = adev->pcie_rreg(adev, reg * 4);
-	}
-
-	return ret;
-}
-
-/*
- * MMIO register write with bytes helper functions
- * @offset:bytes offset from MMIO start
- * @value: the value want to be written to the register
- */
-
-/**
- * amdgpu_mm_wreg8 - read a memory mapped IO register
- *
- * @adev: amdgpu_device pointer
- * @offset: byte aligned register offset
- * @value: 8 bit value to write
- *
- * Writes the value specified to the offset specified.
- */
-void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
-{
-	if (amdgpu_device_skip_hw_access(adev))
-		return;
-
-	if (offset < adev->rmmio_size)
-		writeb(value, adev->rmmio + offset);
-	else
-		BUG();
-}
-
-/**
- * amdgpu_device_wreg - write to a memory mapped IO or indirect register
- *
- * @adev: amdgpu_device pointer
- * @reg: dword aligned register offset
- * @v: 32 bit value to write to the register
- * @acc_flags: access flags which require special behavior
- *
- * Writes the value specified to the offset specified.
- */
-void amdgpu_device_wreg(struct amdgpu_device *adev,
-			uint32_t reg, uint32_t v,
-			uint32_t acc_flags)
-{
-	if (amdgpu_device_skip_hw_access(adev))
-		return;
-
-	if ((reg * 4) < adev->rmmio_size) {
-		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, 0);
-			up_read(&adev->reset_domain->sem);
-		} else {
-			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
-		}
-	} else {
-		adev->pcie_wreg(adev, reg * 4, v);
-	}
-
-	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
-}
-
-/**
- * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
- *
- * @adev: amdgpu_device pointer
- * @reg: mmio/rlc register
- * @v: value to write
- * @xcc_id: xcc accelerated compute core id
- *
- * this function is invoked only for the debugfs register access
- */
-void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
-			     uint32_t reg, uint32_t v,
-			     uint32_t xcc_id)
-{
-	if (amdgpu_device_skip_hw_access(adev))
-		return;
-
-	if (amdgpu_sriov_fullaccess(adev) &&
-	    adev->gfx.rlc.funcs &&
-	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
-		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
-			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
-	} else if ((reg * 4) >= adev->rmmio_size) {
-		adev->pcie_wreg(adev, reg * 4, v);
-	} else {
-		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
-	}
-}
-
-/**
- * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
- *
- * @adev: amdgpu_device pointer
- * @reg: dword aligned register offset
- * @v: 32 bit value to write to the register
- * @acc_flags: access flags which require special behavior
- * @xcc_id: xcc accelerated compute core id
- *
- * Writes the value specified to the offset specified.
- */
-void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
-			uint32_t reg, uint32_t v,
-			uint32_t acc_flags, uint32_t xcc_id)
-{
-	uint32_t rlcg_flag;
-
-	if (amdgpu_device_skip_hw_access(adev))
-		return;
-
-	if ((reg * 4) < adev->rmmio_size) {
-		if (amdgpu_sriov_vf(adev) &&
-		    !amdgpu_sriov_runtime(adev) &&
-		    adev->gfx.rlc.rlcg_reg_access_supported &&
-		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
-							 GC_HWIP, true,
-							 &rlcg_flag)) {
-			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
-		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-			up_read(&adev->reset_domain->sem);
-		} else {
-			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
-		}
-	} else {
-		adev->pcie_wreg(adev, reg * 4, v);
-	}
-}
-
-/**
- * amdgpu_device_indirect_rreg - read an indirect register
- *
- * @adev: amdgpu_device pointer
- * @reg_addr: indirect register address to read from
- *
- * Returns the value of indirect register @reg_addr
- */
-u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
-				u32 reg_addr)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_data_offset;
-	u32 r;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	r = readl(pcie_data_offset);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-
-	return r;
-}
-
-u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
-				    u64 reg_addr)
-{
-	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
-	u32 r;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_index_hi_offset;
-	void __iomem *pcie_data_offset;
-
-	if (unlikely(!adev->nbio.funcs)) {
-		pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
-		pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
-	} else {
-		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-	}
-
-	if (reg_addr >> 32) {
-		if (unlikely(!adev->nbio.funcs))
-			pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
-		else
-			pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
-	} else {
-		pcie_index_hi = 0;
-	}
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-	if (pcie_index_hi != 0)
-		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
-				pcie_index_hi * 4;
-
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	r = readl(pcie_data_offset);
-
-	/* clear the high bits */
-	if (pcie_index_hi != 0) {
-		writel(0, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-
-	return r;
-}
-
-/**
- * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
- *
- * @adev: amdgpu_device pointer
- * @reg_addr: indirect register address to read from
- *
- * Returns the value of indirect register @reg_addr
- */
-u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
-				  u32 reg_addr)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_data_offset;
-	u64 r;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-
-	/* read low 32 bits */
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	r = readl(pcie_data_offset);
-	/* read high 32 bits */
-	writel(reg_addr + 4, pcie_index_offset);
-	readl(pcie_index_offset);
-	r |= ((u64)readl(pcie_data_offset) << 32);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-
-	return r;
-}
-
-u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
-				  u64 reg_addr)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	unsigned long pcie_index_hi = 0;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_index_hi_offset;
-	void __iomem *pcie_data_offset;
-	u64 r;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
-		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-	if (pcie_index_hi != 0)
-		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
-			pcie_index_hi * 4;
-
-	/* read low 32 bits */
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	r = readl(pcie_data_offset);
-	/* read high 32 bits */
-	writel(reg_addr + 4, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	r |= ((u64)readl(pcie_data_offset) << 32);
-
-	/* clear the high bits */
-	if (pcie_index_hi != 0) {
-		writel(0, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-
-	return r;
-}
-
-/**
- * amdgpu_device_indirect_wreg - write an indirect register address
- *
- * @adev: amdgpu_device pointer
- * @reg_addr: indirect register offset
- * @reg_data: indirect register data
- *
- */
-void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
-				 u32 reg_addr, u32 reg_data)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_data_offset;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	writel(reg_data, pcie_data_offset);
-	readl(pcie_data_offset);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-}
-
-void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
-				     u64 reg_addr, u32 reg_data)
-{
-	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_index_hi_offset;
-	void __iomem *pcie_data_offset;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
-		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
-	else
-		pcie_index_hi = 0;
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-	if (pcie_index_hi != 0)
-		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
-				pcie_index_hi * 4;
-
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	writel(reg_data, pcie_data_offset);
-	readl(pcie_data_offset);
-
-	/* clear the high bits */
-	if (pcie_index_hi != 0) {
-		writel(0, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-}
-
-/**
- * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
- *
- * @adev: amdgpu_device pointer
- * @reg_addr: indirect register offset
- * @reg_data: indirect register data
- *
- */
-void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
-				   u32 reg_addr, u64 reg_data)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_data_offset;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-
-	/* write low 32 bits */
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
-	readl(pcie_data_offset);
-	/* write high 32 bits */
-	writel(reg_addr + 4, pcie_index_offset);
-	readl(pcie_index_offset);
-	writel((u32)(reg_data >> 32), pcie_data_offset);
-	readl(pcie_data_offset);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-}
-
-void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
-				   u64 reg_addr, u64 reg_data)
-{
-	unsigned long flags, pcie_index, pcie_data;
-	unsigned long pcie_index_hi = 0;
-	void __iomem *pcie_index_offset;
-	void __iomem *pcie_index_hi_offset;
-	void __iomem *pcie_data_offset;
-
-	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
-	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
-	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
-		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
-	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
-	if (pcie_index_hi != 0)
-		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
-				pcie_index_hi * 4;
-
-	/* write low 32 bits */
-	writel(reg_addr, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
-	readl(pcie_data_offset);
-	/* write high 32 bits */
-	writel(reg_addr + 4, pcie_index_offset);
-	readl(pcie_index_offset);
-	if (pcie_index_hi != 0) {
-		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-	writel((u32)(reg_data >> 32), pcie_data_offset);
-	readl(pcie_data_offset);
-
-	/* clear the high bits */
-	if (pcie_index_hi != 0) {
-		writel(0, pcie_index_hi_offset);
-		readl(pcie_index_hi_offset);
-	}
-
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-}
-
 /**
  * amdgpu_device_get_rev_id - query device rev_id
  *
@@ -1410,149 +858,6 @@ u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
 	return adev->nbio.funcs->get_rev_id(adev);
 }
 
-/**
- * amdgpu_invalid_rreg - dummy reg read function
- *
- * @adev: amdgpu_device pointer
- * @reg: offset of register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- * Returns the value in the register.
- */
-static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
-{
-	dev_err(adev->dev, "Invalid callback to read register 0x%04X\n", reg);
-	BUG();
-	return 0;
-}
-
-static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
-{
-	dev_err(adev->dev, "Invalid callback to read register 0x%llX\n", reg);
-	BUG();
-	return 0;
-}
-
-/**
- * amdgpu_invalid_wreg - dummy reg write function
- *
- * @adev: amdgpu_device pointer
- * @reg: offset of register
- * @v: value to write to the register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- */
-static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
-{
-	dev_err(adev->dev,
-		"Invalid callback to write register 0x%04X with 0x%08X\n", reg,
-		v);
-	BUG();
-}
-
-static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
-{
-	dev_err(adev->dev,
-		"Invalid callback to write register 0x%llX with 0x%08X\n", reg,
-		v);
-	BUG();
-}
-
-/**
- * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
- *
- * @adev: amdgpu_device pointer
- * @reg: offset of register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- * Returns the value in the register.
- */
-static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
-{
-	dev_err(adev->dev, "Invalid callback to read 64 bit register 0x%04X\n",
-		reg);
-	BUG();
-	return 0;
-}
-
-static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
-{
-	dev_err(adev->dev, "Invalid callback to read register 0x%llX\n", reg);
-	BUG();
-	return 0;
-}
-
-/**
- * amdgpu_invalid_wreg64 - dummy reg write function
- *
- * @adev: amdgpu_device pointer
- * @reg: offset of register
- * @v: value to write to the register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- */
-static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
-{
-	dev_err(adev->dev,
-		"Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
-		reg, v);
-	BUG();
-}
-
-static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
-{
-	dev_err(adev->dev,
-		"Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
-		reg, v);
-	BUG();
-}
-
-/**
- * amdgpu_block_invalid_rreg - dummy reg read function
- *
- * @adev: amdgpu_device pointer
- * @block: offset of instance
- * @reg: offset of register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- * Returns the value in the register.
- */
-static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
-					  uint32_t block, uint32_t reg)
-{
-	dev_err(adev->dev,
-		"Invalid callback to read register 0x%04X in block 0x%04X\n",
-		reg, block);
-	BUG();
-	return 0;
-}
-
-/**
- * amdgpu_block_invalid_wreg - dummy reg write function
- *
- * @adev: amdgpu_device pointer
- * @block: offset of instance
- * @reg: offset of register
- * @v: value to write to the register
- *
- * Dummy register read function.  Used for register blocks
- * that certain asics don't have (all asics).
- */
-static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
-				      uint32_t block,
-				      uint32_t reg, uint32_t v)
-{
-	dev_err(adev->dev,
-		"Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
-		reg, block, v);
-	BUG();
-}
-
 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)
 {
 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
@@ -3158,9 +2463,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 	if (r)
 		goto init_failed;
 
-	if (adev->mman.buffer_funcs_ring &&
-	    adev->mman.buffer_funcs_ring->sched.ready)
-		amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_set_buffer_funcs_status(adev, true);
 
 	/* Don't init kfd if whole hive need to be reset during init */
 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
@@ -4059,8 +3362,7 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
 
 	r = amdgpu_device_ip_resume_phase2(adev);
 
-	if (adev->mman.buffer_funcs_ring->sched.ready)
-		amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_set_buffer_funcs_status(adev, true);
 
 	if (r)
 		return r;
@@ -4120,17 +3422,6 @@ bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev,
 	case CHIP_VERDE:
 	case CHIP_OLAND:
 		return amdgpu_dc != 0 && IS_ENABLED(CONFIG_DRM_AMD_DC_SI);
-	case CHIP_KAVERI:
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-		/*
-		 * We have systems in the wild with these ASICs that require
-		 * TRAVIS and NUTMEG support which is not supported with DC.
-		 *
-		 * Fallback to the non-DC driver here by default so as not to
-		 * cause regressions.
-		 */
-		return amdgpu_dc > 0;
 	default:
 		return amdgpu_dc != 0;
 #else
@@ -4408,26 +3699,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
 
-	adev->smc_rreg = &amdgpu_invalid_rreg;
-	adev->smc_wreg = &amdgpu_invalid_wreg;
-	adev->pcie_rreg = &amdgpu_invalid_rreg;
-	adev->pcie_wreg = &amdgpu_invalid_wreg;
-	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
-	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
-	adev->pciep_rreg = &amdgpu_invalid_rreg;
-	adev->pciep_wreg = &amdgpu_invalid_wreg;
-	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
-	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
-	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
-	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
-	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
-	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
-	adev->didt_rreg = &amdgpu_invalid_rreg;
-	adev->didt_wreg = &amdgpu_invalid_wreg;
-	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
-	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
-	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
-	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
+	amdgpu_reg_access_init(adev);
 
 	dev_info(
 		adev->dev,
@@ -4472,13 +3744,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		return r;
 
 	spin_lock_init(&adev->mmio_idx_lock);
-	spin_lock_init(&adev->smc_idx_lock);
-	spin_lock_init(&adev->pcie_idx_lock);
-	spin_lock_init(&adev->uvd_ctx_idx_lock);
-	spin_lock_init(&adev->didt_idx_lock);
-	spin_lock_init(&adev->gc_cac_idx_lock);
-	spin_lock_init(&adev->se_cac_idx_lock);
-	spin_lock_init(&adev->audio_endpt_idx_lock);
 	spin_lock_init(&adev->mm_stats.lock);
 	spin_lock_init(&adev->virt.rlcg_reg_lock);
 	spin_lock_init(&adev->wb.lock);
@@ -4516,6 +3781,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
 	INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
 
+	amdgpu_coredump_init(adev);
+
 	adev->gfx.gfx_off_req_count = 1;
 	adev->gfx.gfx_off_residency = 0;
 	adev->gfx.gfx_off_entrycount = 0;
@@ -4613,9 +3880,21 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	amdgpu_gmc_noretry_set(adev);
 	/* Need to get xgmi info early to decide the reset behavior*/
 	if (adev->gmc.xgmi.supported) {
-		r = adev->gfxhub.funcs->get_xgmi_info(adev);
-		if (r)
-			return r;
+		if (adev->gfxhub.funcs &&
+		    adev->gfxhub.funcs->get_xgmi_info) {
+			r = adev->gfxhub.funcs->get_xgmi_info(adev);
+			if (r)
+				return r;
+		}
+	}
+
+	if (adev->gmc.xgmi.connected_to_cpu) {
+		if (adev->mmhub.funcs &&
+		    adev->mmhub.funcs->get_xgmi_info) {
+			r = adev->mmhub.funcs->get_xgmi_info(adev);
+			if (r)
+				return r;
+		}
 	}
 
 	/* enable PCIE atomic ops */
@@ -5207,8 +4486,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
 	return 0;
 
 unwind_evict:
-	if (adev->mman.buffer_funcs_ring->sched.ready)
-		amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_set_buffer_funcs_status(adev, true);
 	amdgpu_fence_driver_hw_init(adev);
 
 unwind_userq:
@@ -5942,8 +5220,7 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
 				if (r)
 					goto out;
 
-				if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
-					amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
+				amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
 
 				r = amdgpu_device_ip_resume_phase3(tmp_adev);
 				if (r)
@@ -6321,7 +5598,7 @@ static void amdgpu_device_halt_activities(struct amdgpu_device *adev,
 			if (!amdgpu_ring_sched_ready(ring))
 				continue;
 
-			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
+			drm_sched_wqueue_stop(&ring->sched);
 
 			if (need_emergency_restart)
 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
@@ -6405,7 +5682,7 @@ static int amdgpu_device_sched_resume(struct list_head *device_list,
 			if (!amdgpu_ring_sched_ready(ring))
 				continue;
 
-			drm_sched_start(&ring->sched, 0);
+			drm_sched_wqueue_start(&ring->sched);
 		}
 
 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
@@ -7388,39 +6665,6 @@ void amdgpu_device_halt(struct amdgpu_device *adev)
 	pci_wait_for_pending_transaction(pdev);
 }
 
-u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
-				u32 reg)
-{
-	unsigned long flags, address, data;
-	u32 r;
-
-	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	WREG32(address, reg * 4);
-	(void)RREG32(address);
-	r = RREG32(data);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-	return r;
-}
-
-void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
-				u32 reg, u32 v)
-{
-	unsigned long flags, address, data;
-
-	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
-	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
-
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
-	WREG32(address, reg * 4);
-	(void)RREG32(address);
-	WREG32(data, v);
-	(void)RREG32(data);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
-}
-
 /**
  * amdgpu_device_get_gang - return a reference to the current gang
  * @adev: amdgpu_device pointer
@@ -7603,36 +6847,6 @@ bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
 	}
 }
 
-uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
-		uint32_t inst, uint32_t reg_addr, char reg_name[],
-		uint32_t expected_value, uint32_t mask)
-{
-	uint32_t ret = 0;
-	uint32_t old_ = 0;
-	uint32_t tmp_ = RREG32(reg_addr);
-	uint32_t loop = adev->usec_timeout;
-
-	while ((tmp_ & (mask)) != (expected_value)) {
-		if (old_ != tmp_) {
-			loop = adev->usec_timeout;
-			old_ = tmp_;
-		} else
-			udelay(1);
-		tmp_ = RREG32(reg_addr);
-		loop--;
-		if (!loop) {
-			dev_warn(
-				adev->dev,
-				"Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
-				inst, reg_name, (uint32_t)expected_value,
-				(uint32_t)(tmp_ & (mask)));
-			ret = -ETIMEDOUT;
-			break;
-		}
-	}
-	return ret;
-}
-
 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
 {
 	ssize_t size = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index af3d2fd61cf3f..0eb0c62d2f4f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -87,6 +87,7 @@
 #include "sdma_v7_1.h"
 #include "lsdma_v6_0.h"
 #include "lsdma_v7_0.h"
+#include "lsdma_v7_1.h"
 #include "vcn_v2_0.h"
 #include "jpeg_v2_0.h"
 #include "vcn_v3_0.h"
@@ -111,8 +112,10 @@
 #include "smuio_v15_0_8.h"
 #include "vcn_v5_0_0.h"
 #include "vcn_v5_0_1.h"
+#include "vcn_v5_0_2.h"
 #include "jpeg_v5_0_0.h"
 #include "jpeg_v5_0_1.h"
+#include "jpeg_v5_0_2.h"
 #include "jpeg_v5_3_0.h"
 
 #include "amdgpu_ras_mgr.h"
@@ -132,6 +135,7 @@ MODULE_FIRMWARE("amdgpu/picasso_ip_discovery.bin");
 MODULE_FIRMWARE("amdgpu/arcturus_ip_discovery.bin");
 MODULE_FIRMWARE("amdgpu/aldebaran_ip_discovery.bin");
 
+/* Note: These registers are consistent across all the SOCs */
 #define mmIP_DISCOVERY_VERSION  0x16A00
 #define mmRCC_CONFIG_MEMSIZE	0xde3
 #define mmMP0_SMN_C2PMSG_33	0x16061
@@ -139,6 +143,10 @@ MODULE_FIRMWARE("amdgpu/aldebaran_ip_discovery.bin");
 #define mmMM_INDEX_HI		0x6
 #define mmMM_DATA		0x1
 
+#define mmDRIVER_SCRATCH_0	0x94
+#define mmDRIVER_SCRATCH_1	0x95
+#define mmDRIVER_SCRATCH_2	0x96
+
 static const char *hw_id_names[HW_ID_MAX] = {
 	[MP1_HWID]		= "MP1",
 	[MP2_HWID]		= "MP2",
@@ -253,39 +261,12 @@ static int hw_id_map[MAX_HWIP] = {
 	[ATU_HWIP]	= ATU_HWID,
 };
 
-static int amdgpu_discovery_read_binary_from_sysmem(struct amdgpu_device *adev, uint8_t *binary)
-{
-	u64 tmr_offset, tmr_size, pos;
-	void *discv_regn;
-	int ret;
-
-	ret = amdgpu_acpi_get_tmr_info(adev, &tmr_offset, &tmr_size);
-	if (ret)
-		return ret;
-
-	pos = tmr_offset + tmr_size - DISCOVERY_TMR_OFFSET;
-
-	/* This region is read-only and reserved from system use */
-	discv_regn = memremap(pos, adev->discovery.size, MEMREMAP_WC);
-	if (discv_regn) {
-		memcpy(binary, discv_regn, adev->discovery.size);
-		memunmap(discv_regn);
-		return 0;
-	}
-
-	return -ENOENT;
-}
-
-#define IP_DISCOVERY_V2		2
-#define IP_DISCOVERY_V4		4
-
-static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev,
-						 uint8_t *binary)
+static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev,
+					 bool *is_tmr_in_sysmem)
 {
-	bool sz_valid = true;
-	uint64_t vram_size;
-	int i, ret = 0;
-	u32 msg;
+	u64 vram_size, tmr_offset, tmr_size;
+	u32 msg, tmr_offset_lo, tmr_offset_hi;
+	int i, ret;
 
 	if (!amdgpu_sriov_vf(adev)) {
 		/* It can take up to two second for IFWI init to complete on some dGPUs,
@@ -305,51 +286,102 @@ static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev,
 	}
 
 	vram_size = RREG32(mmRCC_CONFIG_MEMSIZE);
-	if (!vram_size || vram_size == U32_MAX)
-		sz_valid = false;
+	if (vram_size == U32_MAX)
+		return -ENXIO;
+	else if (!vram_size)
+		*is_tmr_in_sysmem = true;
 	else
-		vram_size <<= 20;
+		*is_tmr_in_sysmem = false;
 
-	/*
-	 * If in VRAM, discovery TMR is marked for reservation. If it is in system mem,
-	 * then it is not required to be reserved.
-	 */
-	if (sz_valid) {
-		if (amdgpu_sriov_vf(adev) && adev->virt.is_dynamic_crit_regn_enabled) {
-			/* For SRIOV VFs with dynamic critical region enabled,
-			 * we will get the IPD binary via below call.
-			 * If dynamic critical is disabled, fall through to normal seq.
-			 */
-			if (amdgpu_virt_get_dynamic_data_info(adev,
-						AMD_SRIOV_MSG_IPD_TABLE_ID, binary,
-						&adev->discovery.size)) {
-				dev_err(adev->dev,
-						"failed to read discovery info from dynamic critical region.");
-				ret = -EINVAL;
-				goto exit;
-			}
+	/* init the default tmr size and offset */
+	adev->discovery.size = DISCOVERY_TMR_SIZE;
+	if (vram_size)
+		adev->discovery.offset = (vram_size << 20) - DISCOVERY_TMR_OFFSET;
+
+	if (amdgpu_sriov_vf(adev)) {
+		if (adev->virt.is_dynamic_crit_regn_enabled) {
+			adev->discovery.offset =
+				adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset;
+			adev->discovery.size =
+				adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb << 10;
+			if (!adev->discovery.offset || !adev->discovery.size)
+				return -EINVAL;
 		} else {
-			uint64_t pos = vram_size - DISCOVERY_TMR_OFFSET;
+			goto out;
+		}
+	} else {
+		tmr_size = RREG32(mmDRIVER_SCRATCH_2);
+		if (tmr_size) {
+			/* It's preferred to transition to PSP mailbox reg interface
+			 * for both bare-metal and passthrough if available */
+			adev->discovery.size = (u32)tmr_size;
+			tmr_offset_lo = RREG32(mmDRIVER_SCRATCH_0);
+			tmr_offset_hi = RREG32(mmDRIVER_SCRATCH_1);
+			adev->discovery.offset = ((u64)le32_to_cpu(tmr_offset_hi) << 32 |
+						  le32_to_cpu(tmr_offset_lo));
+		} else if (!vram_size) {
+			/* fall back to apci approach to query tmr offset if vram_size is 0 */
+			ret = amdgpu_acpi_get_tmr_info(adev, &tmr_offset, &tmr_size);
+			if (ret)
+				return ret;
+			adev->discovery.size = (u32)tmr_size;
+			adev->discovery.offset = tmr_offset + tmr_size - DISCOVERY_TMR_OFFSET;
+		}
+	}
+out:
+	adev->discovery.bin = kzalloc(adev->discovery.size, GFP_KERNEL);
+	if (!adev->discovery.bin)
+		return -ENOMEM;
+	adev->discovery.debugfs_blob.data = adev->discovery.bin;
+	adev->discovery.debugfs_blob.size = adev->discovery.size;
+
+	return 0;
+}
+
+static int amdgpu_discovery_read_binary_from_sysmem(struct amdgpu_device *adev, uint8_t *binary)
+{
+	void *discv_regn;
+
+	/* This region is read-only and reserved from system use */
+	discv_regn = memremap(adev->discovery.offset, adev->discovery.size, MEMREMAP_WC);
+	if (discv_regn) {
+		memcpy(binary, discv_regn, adev->discovery.size);
+		memunmap(discv_regn);
+		return 0;
+	}
+
+	return -ENOENT;
+}
 
-			amdgpu_device_vram_access(adev, pos, (uint32_t *)binary,
-					adev->discovery.size, false);
+#define IP_DISCOVERY_V2		2
+#define IP_DISCOVERY_V4		4
+
+static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev,
+						 uint8_t *binary,
+						 bool is_tmr_in_sysmem)
+{
+	int ret = 0;
+
+	if (!is_tmr_in_sysmem) {
+		if (amdgpu_sriov_vf(adev) &&
+		    amdgpu_sriov_xgmi_connected_to_cpu(adev)) {
+			ret = amdgpu_discovery_read_binary_from_sysmem(adev, binary);
+		} else {
+			amdgpu_device_vram_access(adev, adev->discovery.offset,
+						  (uint32_t *)binary,
+						  adev->discovery.size, false);
 			adev->discovery.reserve_tmr = true;
 		}
 	} else {
 		ret = amdgpu_discovery_read_binary_from_sysmem(adev, binary);
 	}
 
-	if (ret)
-		dev_err(adev->dev,
-			"failed to read discovery info from memory, vram size read: %llx",
-			vram_size);
-exit:
 	return ret;
 }
 
 static int amdgpu_discovery_read_binary_from_file(struct amdgpu_device *adev,
-							uint8_t *binary,
-							const char *fw_name)
+						  uint8_t *binary,
+						  const char *fw_name)
 {
 	const struct firmware *fw;
 	int r;
@@ -431,14 +463,12 @@ static void amdgpu_discovery_harvest_config_quirk(struct amdgpu_device *adev)
 }
 
 static int amdgpu_discovery_verify_npsinfo(struct amdgpu_device *adev,
-					   struct binary_header *bhdr)
+					   struct table_info *info)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct table_info *info;
 	uint16_t checksum;
 	uint16_t offset;
 
-	info = &bhdr->table_list[NPS_INFO];
 	offset = le16_to_cpu(info->offset);
 	checksum = le16_to_cpu(info->checksum);
 
@@ -491,23 +521,130 @@ static const char *amdgpu_discovery_get_fw_name(struct amdgpu_device *adev)
 	}
 }
 
-static int amdgpu_discovery_init(struct amdgpu_device *adev)
+static int amdgpu_discovery_get_table_info(struct amdgpu_device *adev,
+					   struct table_info **info,
+					   uint16_t table_id)
+{
+	struct binary_header *bhdr =
+		(struct binary_header *)adev->discovery.bin;
+	struct binary_header_v2 *bhdrv2;
+
+	switch (bhdr->version_major) {
+	case 2:
+		bhdrv2 = (struct binary_header_v2 *)adev->discovery.bin;
+		*info = &bhdrv2->table_list[table_id];
+		break;
+	case 1:
+		*info = &bhdr->table_list[table_id];
+		break;
+	default:
+		dev_err(adev->dev, "Invalid ip discovery table version\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int amdgpu_discovery_table_check(struct amdgpu_device *adev,
+					uint8_t *discovery_bin,
+					uint16_t table_id)
 {
+	int r, act_val, exp_val, table_size;
+	uint16_t offset, checksum;
 	struct table_info *info;
+	bool check_table = true;
+	char *table_name;
+
+	r = amdgpu_discovery_get_table_info(adev, &info, table_id);
+	if (r)
+		return r;
+	offset = le16_to_cpu(info->offset);
+	checksum = le16_to_cpu(info->checksum);
+
+	switch (table_id) {
+	case IP_DISCOVERY: {
+		struct ip_discovery_header *ihdr =
+			(struct ip_discovery_header *)(discovery_bin + offset);
+		act_val = le32_to_cpu(ihdr->signature);
+		exp_val = DISCOVERY_TABLE_SIGNATURE;
+		table_size = le16_to_cpu(ihdr->size);
+		table_name = "data table";
+		break;
+	}
+	case GC: {
+		struct gpu_info_header *ghdr =
+			(struct gpu_info_header *)(discovery_bin + offset);
+		act_val = le32_to_cpu(ghdr->table_id);
+		exp_val = GC_TABLE_ID;
+		table_size = le16_to_cpu(ghdr->size);
+		table_name = "gc table";
+		break;
+	}
+	case HARVEST_INFO: {
+		struct harvest_info_header *hhdr =
+			(struct harvest_info_header *)(discovery_bin + offset);
+		act_val = le32_to_cpu(hhdr->signature);
+		exp_val = HARVEST_TABLE_SIGNATURE;
+		table_size = sizeof(struct harvest_table);
+		table_name = "harvest table";
+		break;
+	}
+	case VCN_INFO: {
+		struct vcn_info_header *vhdr =
+			(struct vcn_info_header *)(discovery_bin + offset);
+		act_val = le32_to_cpu(vhdr->table_id);
+		exp_val = VCN_INFO_TABLE_ID;
+		table_size = le32_to_cpu(vhdr->size_bytes);
+		table_name = "vcn table";
+		break;
+	}
+	case MALL_INFO: {
+		struct mall_info_header *mhdr =
+			(struct mall_info_header *)(discovery_bin + offset);
+		act_val = le32_to_cpu(mhdr->table_id);
+		exp_val = MALL_INFO_TABLE_ID;
+		table_size = le32_to_cpu(mhdr->size_bytes);
+		table_name = "mall table";
+		check_table = false;
+		break;
+	}
+	default:
+		dev_err(adev->dev, "invalid ip discovery table id %d specified\n", table_id);
+		check_table = false;
+		break;
+	}
+
+	if (check_table && offset) {
+		if (act_val != exp_val) {
+			dev_err(adev->dev, "invalid ip discovery %s signature\n", table_name);
+			return -EINVAL;
+		}
+
+		if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
+						      table_size, checksum)) {
+			dev_err(adev->dev, "invalid ip discovery %s checksum\n", table_name);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int amdgpu_discovery_init(struct amdgpu_device *adev)
+{
 	struct binary_header *bhdr;
 	uint8_t *discovery_bin;
 	const char *fw_name;
 	uint16_t offset;
 	uint16_t size;
 	uint16_t checksum;
+	uint16_t table_id;
+	bool is_tmr_in_sysmem;
 	int r;
 
-	adev->discovery.bin = kzalloc(DISCOVERY_TMR_SIZE, GFP_KERNEL);
-	if (!adev->discovery.bin)
-		return -ENOMEM;
-	adev->discovery.size = DISCOVERY_TMR_SIZE;
-	adev->discovery.debugfs_blob.data = adev->discovery.bin;
-	adev->discovery.debugfs_blob.size = adev->discovery.size;
+	r = amdgpu_discovery_get_tmr_info(adev, &is_tmr_in_sysmem);
+	if (r)
+		return r;
 
 	discovery_bin = adev->discovery.bin;
 	/* Read from file if it is the preferred option */
@@ -520,7 +657,8 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev)
 			goto out;
 	} else {
 		drm_dbg(&adev->ddev, "use ip discovery information from memory");
-		r = amdgpu_discovery_read_binary_from_mem(adev, discovery_bin);
+		r = amdgpu_discovery_read_binary_from_mem(adev, discovery_bin,
+							  is_tmr_in_sysmem);
 		if (r)
 			goto out;
 	}
@@ -547,118 +685,10 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev)
 		goto out;
 	}
 
-	info = &bhdr->table_list[IP_DISCOVERY];
-	offset = le16_to_cpu(info->offset);
-	checksum = le16_to_cpu(info->checksum);
-
-	if (offset) {
-		struct ip_discovery_header *ihdr =
-			(struct ip_discovery_header *)(discovery_bin + offset);
-		if (le32_to_cpu(ihdr->signature) != DISCOVERY_TABLE_SIGNATURE) {
-			dev_err(adev->dev, "invalid ip discovery data table signature\n");
-			r = -EINVAL;
-			goto out;
-		}
-
-		if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
-						      le16_to_cpu(ihdr->size),
-						      checksum)) {
-			dev_err(adev->dev, "invalid ip discovery data table checksum\n");
-			r = -EINVAL;
-			goto out;
-		}
-	}
-
-	info = &bhdr->table_list[GC];
-	offset = le16_to_cpu(info->offset);
-	checksum = le16_to_cpu(info->checksum);
-
-	if (offset) {
-		struct gpu_info_header *ghdr =
-			(struct gpu_info_header *)(discovery_bin + offset);
-
-		if (le32_to_cpu(ghdr->table_id) != GC_TABLE_ID) {
-			dev_err(adev->dev, "invalid ip discovery gc table id\n");
-			r = -EINVAL;
-			goto out;
-		}
-
-		if (!amdgpu_discovery_verify_checksum(adev, discovery_bin + offset,
-						      le32_to_cpu(ghdr->size),
-						      checksum)) {
-			dev_err(adev->dev, "invalid gc data table checksum\n");
-			r = -EINVAL;
-			goto out;
-		}
-	}
-
-	info = &bhdr->table_list[HARVEST_INFO];
-	offset = le16_to_cpu(info->offset);
-	checksum = le16_to_cpu(info->checksum);
-
-	if (offset) {
-		struct harvest_info_header *hhdr =
-			(struct harvest_info_header *)(discovery_bin + offset);
-
-		if (le32_to_cpu(hhdr->signature) != HARVEST_TABLE_SIGNATURE) {
-			dev_err(adev->dev, "invalid ip discovery harvest table signature\n");
-			r = -EINVAL;
-			goto out;
-		}
-
-		if (!amdgpu_discovery_verify_checksum(adev,
-			    discovery_bin + offset,
-			    sizeof(struct harvest_table), checksum)) {
-			dev_err(adev->dev, "invalid harvest data table checksum\n");
-			r = -EINVAL;
-			goto out;
-		}
-	}
-
-	info = &bhdr->table_list[VCN_INFO];
-	offset = le16_to_cpu(info->offset);
-	checksum = le16_to_cpu(info->checksum);
-
-	if (offset) {
-		struct vcn_info_header *vhdr =
-			(struct vcn_info_header *)(discovery_bin + offset);
-
-		if (le32_to_cpu(vhdr->table_id) != VCN_INFO_TABLE_ID) {
-			dev_err(adev->dev, "invalid ip discovery vcn table id\n");
-			r = -EINVAL;
-			goto out;
-		}
-
-		if (!amdgpu_discovery_verify_checksum(adev,
-			    discovery_bin + offset,
-			    le32_to_cpu(vhdr->size_bytes), checksum)) {
-			dev_err(adev->dev, "invalid vcn data table checksum\n");
-			r = -EINVAL;
-			goto out;
-		}
-	}
-
-	info = &bhdr->table_list[MALL_INFO];
-	offset = le16_to_cpu(info->offset);
-	checksum = le16_to_cpu(info->checksum);
-
-	if (0 && offset) {
-		struct mall_info_header *mhdr =
-			(struct mall_info_header *)(discovery_bin + offset);
-
-		if (le32_to_cpu(mhdr->table_id) != MALL_INFO_TABLE_ID) {
-			dev_err(adev->dev, "invalid ip discovery mall table id\n");
-			r = -EINVAL;
-			goto out;
-		}
-
-		if (!amdgpu_discovery_verify_checksum(adev,
-			    discovery_bin + offset,
-			    le32_to_cpu(mhdr->size_bytes), checksum)) {
-			dev_err(adev->dev, "invalid mall data table checksum\n");
-			r = -EINVAL;
+	for (table_id = 0; table_id <= MALL_INFO; table_id++) {
+		r = amdgpu_discovery_table_check(adev, discovery_bin, table_id);
+		if (r)
 			goto out;
-		}
 	}
 
 	return 0;
@@ -770,14 +800,15 @@ static void amdgpu_discovery_read_from_harvest_table(struct amdgpu_device *adev,
 						     uint32_t *umc_harvest_count)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	struct harvest_table *harvest_info;
 	u16 offset;
 	int i;
-	uint32_t umc_harvest_config = 0;
+	u64 umc_harvest_config = 0;
 
-	bhdr = (struct binary_header *)discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[HARVEST_INFO].offset);
+	if (amdgpu_discovery_get_table_info(adev, &info, HARVEST_INFO))
+		return;
+	offset = le16_to_cpu(info->offset);
 
 	if (!offset) {
 		dev_err(adev->dev, "invalid harvest table offset\n");
@@ -830,7 +861,7 @@ static void amdgpu_discovery_read_from_harvest_table(struct amdgpu_device *adev,
 		}
 	}
 
-	adev->umc.active_mask = ((1 << adev->umc.node_inst_num) - 1) &
+	adev->umc.active_mask = ((1ULL << adev->umc.node_inst_num) - 1ULL) &
 				~umc_harvest_config;
 }
 
@@ -1195,13 +1226,8 @@ static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev,
 					ip_hw_instance->num_instance);
 			ip_hw_instance->num_base_addresses = ip->num_base_address;
 
-			for (kk = 0; kk < ip_hw_instance->num_base_addresses; kk++) {
-				if (reg_base_64)
-					ip_hw_instance->base_addr[kk] =
-						lower_32_bits(le64_to_cpu(ip->base_address_64[kk])) & 0x3FFFFFFF;
-				else
-					ip_hw_instance->base_addr[kk] = ip->base_address[kk];
-			}
+			for (kk = 0; kk < ip_hw_instance->num_base_addresses; kk++)
+				ip_hw_instance->base_addr[kk] = ip->base_address[kk];
 
 			kobject_init(&ip_hw_instance->kobj, &ip_hw_instance_ktype);
 			ip_hw_instance->kobj.kset = &ip_hw_id->hw_id_kset;
@@ -1224,7 +1250,7 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev)
 {
 	struct ip_discovery_top *ip_top = adev->discovery.ip_top;
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	struct ip_discovery_header *ihdr;
 	struct die_header *dhdr;
 	struct kset *die_kset = &ip_top->die_kset;
@@ -1232,10 +1258,12 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev)
 	size_t ip_offset;
 	int ii, res;
 
-	bhdr = (struct binary_header *)discovery_bin;
+	res = amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY);
+	if (res)
+		return res;
 	ihdr = (struct ip_discovery_header
 			*)(discovery_bin +
-			   le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset));
+			   le16_to_cpu(info->offset));
 	num_dies = le16_to_cpu(ihdr->num_dies);
 
 	DRM_DEBUG("number of dies: %d\n", num_dies);
@@ -1379,12 +1407,54 @@ static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev)
 	kobject_put(&ip_top->kobj);
 }
 
+/* devcoredump support */
+void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p)
+{
+	struct ip_discovery_top *ip_top = adev->discovery.ip_top;
+	struct ip_die_entry *ip_die_entry;
+	struct list_head *el_die, *el_hw_id, *el_hw_inst;
+	struct ip_hw_id *hw_id;
+	struct kset *die_kset;
+	struct ip_hw_instance *ip_inst;
+	int i = 0, j;
+
+	die_kset = &ip_top->die_kset;
+
+	drm_printf(p, "\nHW IP Discovery\n");
+	spin_lock(&die_kset->list_lock);
+	list_for_each(el_die, &die_kset->list) {
+		drm_printf(p, "die %d\n", i++);
+		ip_die_entry = to_ip_die_entry(list_to_kobj(el_die));
+
+		list_for_each(el_hw_id, &ip_die_entry->ip_kset.list) {
+			hw_id = to_ip_hw_id(list_to_kobj(el_hw_id));
+			drm_printf(p, "hw_id %d %s\n", hw_id->hw_id, hw_id_names[hw_id->hw_id]);
+
+			list_for_each(el_hw_inst, &hw_id->hw_id_kset.list) {
+				ip_inst = to_ip_hw_instance(list_to_kobj(el_hw_inst));
+				drm_printf(p, "\tinstance %d\n", ip_inst->num_instance);
+				drm_printf(p, "\tmajor %d\n", ip_inst->major);
+				drm_printf(p, "\tminor %d\n", ip_inst->minor);
+				drm_printf(p, "\trevision %d\n", ip_inst->revision);
+				drm_printf(p, "\tharvest 0x%01X\n", ip_inst->harvest);
+				drm_printf(p, "\tnum_base_addresses %d\n",
+					   ip_inst->num_base_addresses);
+				for (j = 0; j < ip_inst->num_base_addresses; j++)
+					drm_printf(p, "\tbase_addr[%d] 0x%08X\n",
+						   j, ip_inst->base_addr[j]);
+			}
+		}
+	}
+	spin_unlock(&die_kset->list_lock);
+}
+
+
 /* ================================================== */
 
 static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
 {
 	uint8_t num_base_address, subrev, variant;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	struct ip_discovery_header *ihdr;
 	struct die_header *dhdr;
 	uint8_t *discovery_bin;
@@ -1409,10 +1479,12 @@ static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
 	adev->sdma.sdma_mask = 0;
 	adev->vcn.inst_mask = 0;
 	adev->jpeg.inst_mask = 0;
-	bhdr = (struct binary_header *)discovery_bin;
+	r = amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY);
+	if (r)
+		return r;
 	ihdr = (struct ip_discovery_header
 			*)(discovery_bin +
-			   le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset));
+			   le16_to_cpu(info->offset));
 	num_dies = le16_to_cpu(ihdr->num_dies);
 
 	DRM_DEBUG("number of dies: %d\n", num_dies);
@@ -1585,14 +1657,15 @@ static void amdgpu_discovery_harvest_ip(struct amdgpu_device *adev)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
 	struct ip_discovery_header *ihdr;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	int vcn_harvest_count = 0;
 	int umc_harvest_count = 0;
-	uint16_t offset, ihdr_ver;
+	uint16_t ihdr_ver;
 
-	bhdr = (struct binary_header *)discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset);
-	ihdr = (struct ip_discovery_header *)(discovery_bin + offset);
+	if (amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY))
+		return;
+	ihdr = (struct ip_discovery_header *)(discovery_bin +
+					      le16_to_cpu(info->offset));
 	ihdr_ver = le16_to_cpu(ihdr->version);
 	/*
 	 * Harvest table does not fit Navi1x and legacy GPUs,
@@ -1640,7 +1713,7 @@ union gc_info {
 static int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	union gc_info *gc_info;
 	u16 offset;
 
@@ -1649,8 +1722,9 @@ static int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
-	bhdr = (struct binary_header *)discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[GC].offset);
+	if (amdgpu_discovery_get_table_info(adev, &info, GC))
+		return -EINVAL;
+	offset = le16_to_cpu(info->offset);
 
 	if (!offset)
 		return 0;
@@ -1749,7 +1823,7 @@ union mall_info {
 static int amdgpu_discovery_get_mall_info(struct amdgpu_device *adev)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	union mall_info *mall_info;
 	u32 u, mall_size_per_umc, m_s_present, half_use;
 	u64 mall_size;
@@ -1760,8 +1834,9 @@ static int amdgpu_discovery_get_mall_info(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
-	bhdr = (struct binary_header *)discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[MALL_INFO].offset);
+	if (amdgpu_discovery_get_table_info(adev, &info, MALL_INFO))
+		return -EINVAL;
+	offset = le16_to_cpu(info->offset);
 
 	if (!offset)
 		return 0;
@@ -1806,7 +1881,7 @@ union vcn_info {
 static int amdgpu_discovery_get_vcn_info(struct amdgpu_device *adev)
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	union vcn_info *vcn_info;
 	u16 offset;
 	int v;
@@ -1826,8 +1901,9 @@ static int amdgpu_discovery_get_vcn_info(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
-	bhdr = (struct binary_header *)discovery_bin;
-	offset = le16_to_cpu(bhdr->table_list[VCN_INFO].offset);
+	if (amdgpu_discovery_get_table_info(adev, &info, VCN_INFO))
+		return -EINVAL;
+	offset = le16_to_cpu(info->offset);
 
 	if (!offset)
 		return 0;
@@ -1864,14 +1940,26 @@ static int amdgpu_discovery_refresh_nps_info(struct amdgpu_device *adev,
 	uint64_t vram_size, pos, offset;
 	struct nps_info_header *nhdr;
 	struct binary_header bhdr;
+	struct binary_header_v2 bhdrv2;
 	uint16_t checksum;
 
 	vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20;
 	pos = vram_size - DISCOVERY_TMR_OFFSET;
 	amdgpu_device_vram_access(adev, pos, &bhdr, sizeof(bhdr), false);
 
-	offset = le16_to_cpu(bhdr.table_list[NPS_INFO].offset);
-	checksum = le16_to_cpu(bhdr.table_list[NPS_INFO].checksum);
+	switch (bhdr.version_major) {
+	case 2:
+		amdgpu_device_vram_access(adev, pos, &bhdrv2, sizeof(bhdrv2), false);
+		offset = le16_to_cpu(bhdrv2.table_list[NPS_INFO].offset);
+		checksum = le16_to_cpu(bhdrv2.table_list[NPS_INFO].checksum);
+		break;
+	case 1:
+		offset = le16_to_cpu(bhdr.table_list[NPS_INFO].offset);
+		checksum = le16_to_cpu(bhdr.table_list[NPS_INFO].checksum);
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	amdgpu_device_vram_access(adev, (pos + offset), nps_data,
 				  sizeof(*nps_data), false);
@@ -1894,7 +1982,7 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 {
 	uint8_t *discovery_bin = adev->discovery.bin;
 	struct amdgpu_gmc_memrange *mem_ranges;
-	struct binary_header *bhdr;
+	struct table_info *info;
 	union nps_info *nps_info;
 	union nps_info nps_data;
 	u16 offset;
@@ -1915,14 +2003,15 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 			return -EINVAL;
 		}
 
-		bhdr = (struct binary_header *)discovery_bin;
-		offset = le16_to_cpu(bhdr->table_list[NPS_INFO].offset);
+		if (amdgpu_discovery_get_table_info(adev, &info, NPS_INFO))
+			return -EINVAL;
+		offset = le16_to_cpu(info->offset);
 
 		if (!offset)
 			return -ENOENT;
 
 		/* If verification fails, return as if NPS table doesn't exist */
-		if (amdgpu_discovery_verify_npsinfo(adev, bhdr))
+		if (amdgpu_discovery_verify_npsinfo(adev, info))
 			return -ENOENT;
 
 		nps_info = (union nps_info *)(discovery_bin + offset);
@@ -2298,6 +2387,7 @@ static int amdgpu_discovery_set_display_ip_blocks(struct amdgpu_device *adev)
 		case IP_VERSION(3, 5, 1):
 		case IP_VERSION(3, 6, 0):
 		case IP_VERSION(4, 1, 0):
+		case IP_VERSION(4, 2, 0):
 			/* TODO: Fix IP version. DC code expects version 4.0.1 */
 			if (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(4, 1, 0))
 				adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 0, 1);
@@ -2561,6 +2651,10 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev)
 			amdgpu_device_ip_block_add(adev, &vcn_v5_0_1_ip_block);
 			amdgpu_device_ip_block_add(adev, &jpeg_v5_0_1_ip_block);
 			break;
+		case IP_VERSION(5, 0, 2):
+			amdgpu_device_ip_block_add(adev, &vcn_v5_0_2_ip_block);
+			amdgpu_device_ip_block_add(adev, &jpeg_v5_0_2_ip_block);
+			break;
 		default:
 			dev_err(adev->dev,
 				"Failed to add vcn/jpeg ip block(UVD_HWIP:0x%x)\n",
@@ -3225,6 +3319,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(7, 0, 1):
 		adev->lsdma.funcs = &lsdma_v7_0_funcs;
 		break;
+	case IP_VERSION(7, 1, 0):
+		adev->lsdma.funcs = &lsdma_v7_1_funcs;
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
index 4ce04486cc319..a7aeb47887a34 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -30,10 +30,12 @@
 #define DISCOVERY_TMR_OFFSET    (64 << 10)
 
 struct ip_discovery_top;
+struct drm_printer;
 
 struct amdgpu_discovery_info {
 	struct debugfs_blob_wrapper debugfs_blob;
 	struct ip_discovery_top *ip_top;
+	uint64_t offset;
 	uint32_t size;
 	uint8_t *bin;
 	bool reserve_tmr;
@@ -47,4 +49,6 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 				  struct amdgpu_gmc_memrange **ranges,
 				  int *range_cnt, bool refresh);
 
+void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p);
+
 #endif /* __AMDGPU_DISCOVERY__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index bef9dce2e7eac..f5cd685424427 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -1738,21 +1738,6 @@ bool amdgpu_crtc_get_scanout_position(struct drm_crtc *crtc,
 						  stime, etime, mode);
 }
 
-static bool
-amdgpu_display_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj)
-{
-	struct drm_device *dev = adev_to_drm(adev);
-	struct drm_fb_helper *fb_helper = dev->fb_helper;
-
-	if (!fb_helper || !fb_helper->buffer)
-		return false;
-
-	if (gem_to_amdgpu_bo(fb_helper->buffer->gem) != robj)
-		return false;
-
-	return true;
-}
-
 int amdgpu_display_suspend_helper(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev_to_drm(adev);
@@ -1775,7 +1760,6 @@ int amdgpu_display_suspend_helper(struct amdgpu_device *adev)
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 		struct drm_framebuffer *fb = crtc->primary->fb;
-		struct amdgpu_bo *robj;
 
 		if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
 			struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
@@ -1790,8 +1774,9 @@ int amdgpu_display_suspend_helper(struct amdgpu_device *adev)
 		if (!fb || !fb->obj[0])
 			continue;
 
-		robj = gem_to_amdgpu_bo(fb->obj[0]);
-		if (!amdgpu_display_robj_is_fb(adev, robj)) {
+		if (!drm_fb_helper_gem_is_fb(dev->fb_helper, fb->obj[0])) {
+			struct amdgpu_bo *robj = gem_to_amdgpu_bo(fb->obj[0]);
+
 			r = amdgpu_bo_reserve(robj, true);
 			if (r == 0) {
 				amdgpu_bo_unpin(robj);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 656c267dbe587..b33c300e26e2e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -134,13 +134,9 @@ static int amdgpu_dma_buf_pin(struct dma_buf_attachment *attach)
 	 * notifiers are disabled, only allow pinning in VRAM when move
 	 * notiers are enabled.
 	 */
-	if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) {
-		domains &= ~AMDGPU_GEM_DOMAIN_VRAM;
-	} else {
-		list_for_each_entry(attach, &dmabuf->attachments, node)
-			if (!attach->peer2peer)
-				domains &= ~AMDGPU_GEM_DOMAIN_VRAM;
-	}
+	list_for_each_entry(attach, &dmabuf->attachments, node)
+		if (!attach->peer2peer)
+			domains &= ~AMDGPU_GEM_DOMAIN_VRAM;
 
 	if (domains & AMDGPU_GEM_DOMAIN_VRAM)
 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
@@ -456,7 +452,7 @@ amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
 }
 
 /**
- * amdgpu_dma_buf_move_notify - &attach.move_notify implementation
+ * amdgpu_dma_buf_move_notify - &attach.invalidate_mappings implementation
  *
  * @attach: the DMA-buf attachment
  *
@@ -534,7 +530,7 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
 
 static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = {
 	.allow_peer2peer = true,
-	.move_notify = amdgpu_dma_buf_move_notify
+	.invalidate_mappings = amdgpu_dma_buf_move_notify
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 95d26f086d545..a44baa9ee78dc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -223,9 +223,7 @@ uint amdgpu_dc_visual_confirm;
 int amdgpu_async_gfx_ring = 1;
 int amdgpu_mcbp = -1;
 int amdgpu_discovery = -1;
-int amdgpu_mes;
 int amdgpu_mes_log_enable = 0;
-int amdgpu_mes_kiq;
 int amdgpu_uni_mes = 1;
 int amdgpu_noretry = -1;
 int amdgpu_force_asic_type = -1;
@@ -690,15 +688,6 @@ MODULE_PARM_DESC(discovery,
 	"Allow driver to discover hardware IPs from IP Discovery table at the top of VRAM");
 module_param_named(discovery, amdgpu_discovery, int, 0444);
 
-/**
- * DOC: mes (int)
- * Enable Micro Engine Scheduler. This is a new hw scheduling engine for gfx, sdma, and compute.
- * (0 = disabled (default), 1 = enabled)
- */
-MODULE_PARM_DESC(mes,
-	"Enable Micro Engine Scheduler (0 = disabled (default), 1 = enabled)");
-module_param_named(mes, amdgpu_mes, int, 0444);
-
 /**
  * DOC: mes_log_enable (int)
  * Enable Micro Engine Scheduler log. This is used to enable/disable MES internal log.
@@ -708,15 +697,6 @@ MODULE_PARM_DESC(mes_log_enable,
 	"Enable Micro Engine Scheduler log (0 = disabled (default), 1 = enabled)");
 module_param_named(mes_log_enable, amdgpu_mes_log_enable, int, 0444);
 
-/**
- * DOC: mes_kiq (int)
- * Enable Micro Engine Scheduler KIQ. This is a new engine pipe for kiq.
- * (0 = disabled (default), 1 = enabled)
- */
-MODULE_PARM_DESC(mes_kiq,
-	"Enable Micro Engine Scheduler KIQ (0 = disabled (default), 1 = enabled)");
-module_param_named(mes_kiq, amdgpu_mes_kiq, int, 0444);
-
 /**
  * DOC: uni_mes (int)
  * Enable Unified Micro Engine Scheduler. This is a new engine pipe for unified scheduler.
@@ -859,8 +839,8 @@ module_param_named_unsafe(no_queue_eviction_on_vm_fault, amdgpu_no_queue_evictio
 /**
  * DOC: mtype_local (int)
  */
-int amdgpu_mtype_local;
-MODULE_PARM_DESC(mtype_local, "MTYPE for local memory (0 = MTYPE_RW (default), 1 = MTYPE_NC, 2 = MTYPE_CC)");
+int amdgpu_mtype_local = -1;
+MODULE_PARM_DESC(mtype_local, "MTYPE for local memory (default: ASIC dependent, 0 = MTYPE_RW, 1 = MTYPE_NC, 2 = MTYPE_CC)");
 module_param_named_unsafe(mtype_local, amdgpu_mtype_local, int, 0444);
 
 /**
@@ -2972,9 +2952,11 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
 	int idx;
 
 	if (fpriv && drm_dev_enter(dev, &idx)) {
-		fpriv->evf_mgr.fd_closing = true;
-		amdgpu_eviction_fence_destroy(&fpriv->evf_mgr);
+		amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
+		amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
+		amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
 		amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
+		amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
 		drm_dev_exit(idx);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index 3b588c7740ec8..55e8b9cc2f9f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -25,9 +25,6 @@
 #include <drm/drm_exec.h>
 #include "amdgpu.h"
 
-#define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name)
-#define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr)
-
 static const char *
 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence)
 {
@@ -43,127 +40,79 @@ amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f)
 	return ef->timeline_name;
 }
 
-int
-amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				    struct drm_exec *exec)
+static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
 {
-	struct amdgpu_eviction_fence *old_ef, *new_ef;
-	struct drm_gem_object *obj;
-	unsigned long index;
-	int ret;
-
-	if (evf_mgr->ev_fence &&
-	    !dma_fence_is_signaled(&evf_mgr->ev_fence->base))
-		return 0;
-	/*
-	 * Steps to replace eviction fence:
-	 * * lock all objects in exec (caller)
-	 * * create a new eviction fence
-	 * * update new eviction fence in evf_mgr
-	 * * attach the new eviction fence to BOs
-	 * * release the old fence
-	 * * unlock the objects (caller)
-	 */
-	new_ef = amdgpu_eviction_fence_create(evf_mgr);
-	if (!new_ef) {
-		DRM_ERROR("Failed to create new eviction fence\n");
-		return -ENOMEM;
-	}
-
-	/* Update the eviction fence now */
-	spin_lock(&evf_mgr->ev_fence_lock);
-	old_ef = evf_mgr->ev_fence;
-	evf_mgr->ev_fence = new_ef;
-	spin_unlock(&evf_mgr->ev_fence_lock);
+	struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f);
 
-	/* Attach the new fence */
-	drm_exec_for_each_locked_object(exec, index, obj) {
-		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
-
-		if (!bo)
-			continue;
-		ret = amdgpu_eviction_fence_attach(evf_mgr, bo);
-		if (ret) {
-			DRM_ERROR("Failed to attch new eviction fence\n");
-			goto free_err;
-		}
-	}
-
-	/* Free old fence */
-	if (old_ef)
-		dma_fence_put(&old_ef->base);
-	return 0;
-
-free_err:
-	kfree(new_ef);
-	return ret;
+	schedule_work(&ev_fence->evf_mgr->suspend_work);
+	return true;
 }
 
+static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
+	.get_driver_name = amdgpu_eviction_fence_get_driver_name,
+	.get_timeline_name = amdgpu_eviction_fence_get_timeline_name,
+	.enable_signaling = amdgpu_eviction_fence_enable_signaling,
+};
+
 static void
 amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 {
-	struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work);
-	struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr);
+	struct amdgpu_eviction_fence_mgr *evf_mgr =
+		container_of(work, struct amdgpu_eviction_fence_mgr,
+			     suspend_work);
+	struct amdgpu_fpriv *fpriv =
+		container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);
 	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
-	struct amdgpu_eviction_fence *ev_fence;
+	struct dma_fence *ev_fence;
 
 	mutex_lock(&uq_mgr->userq_mutex);
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	if (ev_fence)
-		dma_fence_get(&ev_fence->base);
-	else
-		goto unlock;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
-	amdgpu_userq_evict(uq_mgr, ev_fence);
+	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	amdgpu_userq_evict(uq_mgr, !evf_mgr->shutdown);
 
-	mutex_unlock(&uq_mgr->userq_mutex);
-	dma_fence_put(&ev_fence->base);
-	return;
-
-unlock:
-	spin_unlock(&evf_mgr->ev_fence_lock);
+	/*
+	 * Signaling the eviction fence must be done while holding the
+	 * userq_mutex. Otherwise we won't resume the queues before issuing the
+	 * next fence.
+	 */
+	dma_fence_signal(ev_fence);
+	dma_fence_put(ev_fence);
 	mutex_unlock(&uq_mgr->userq_mutex);
 }
 
-static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
+int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				struct amdgpu_bo *bo)
 {
-	struct amdgpu_eviction_fence_mgr *evf_mgr;
-	struct amdgpu_eviction_fence *ev_fence;
-
-	if (!f)
-		return true;
-
-	ev_fence = to_ev_fence(f);
-	evf_mgr = ev_fence->evf_mgr;
+	struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	struct ttm_operation_ctx ctx = { false, false };
+	struct dma_resv *resv = bo->tbo.base.resv;
+	int ret;
 
-	schedule_delayed_work(&evf_mgr->suspend_work, 0);
-	return true;
-}
+	if (!dma_fence_is_signaled(ev_fence)) {
 
-static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
-	.get_driver_name = amdgpu_eviction_fence_get_driver_name,
-	.get_timeline_name = amdgpu_eviction_fence_get_timeline_name,
-	.enable_signaling = amdgpu_eviction_fence_enable_signaling,
-};
+		amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+		if (!ret)
+			dma_resv_add_fence(resv, ev_fence,
+					   DMA_RESV_USAGE_BOOKKEEP);
+	} else {
+		ret = 0;
+	}
 
-void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				  struct amdgpu_eviction_fence *ev_fence)
-{
-	spin_lock(&evf_mgr->ev_fence_lock);
-	dma_fence_signal(&ev_fence->base);
-	spin_unlock(&evf_mgr->ev_fence_lock);
+	dma_fence_put(ev_fence);
+	return ret;
 }
 
-struct amdgpu_eviction_fence *
-amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
+int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
+			 struct drm_exec *exec)
 {
 	struct amdgpu_eviction_fence *ev_fence;
+	struct drm_gem_object *obj;
+	unsigned long index;
 
+	/* Create and initialize a new eviction fence */
 	ev_fence = kzalloc_obj(*ev_fence);
 	if (!ev_fence)
-		return NULL;
+		return -ENOMEM;
 
 	ev_fence->evf_mgr = evf_mgr;
 	get_task_comm(ev_fence->timeline_name, current);
@@ -171,56 +120,22 @@ amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
 	dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,
 			 &ev_fence->lock, evf_mgr->ev_fence_ctx,
 			 atomic_inc_return(&evf_mgr->ev_fence_seq));
-	return ev_fence;
-}
 
-void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr)
-{
-	struct amdgpu_eviction_fence *ev_fence;
-
-	/* Wait for any pending work to execute */
-	flush_delayed_work(&evf_mgr->suspend_work);
-
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
-	if (!ev_fence)
-		return;
-
-	dma_fence_wait(&ev_fence->base, false);
+	/* Remember it for newly added BOs */
+	dma_fence_put(evf_mgr->ev_fence);
+	evf_mgr->ev_fence = &ev_fence->base;
 
-	/* Last unref of ev_fence */
-	dma_fence_put(&ev_fence->base);
-}
-
-int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				 struct amdgpu_bo *bo)
-{
-	struct amdgpu_eviction_fence *ev_fence;
-	struct dma_resv *resv = bo->tbo.base.resv;
-	int ret;
-
-	if (!resv)
-		return 0;
+	/* And add it to all existing BOs */
+	drm_exec_for_each_locked_object(exec, index, obj) {
+		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
 
-	ret = dma_resv_reserve_fences(resv, 1);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Failed to resv fence space\n");
-		return ret;
+		amdgpu_evf_mgr_attach_fence(evf_mgr, bo);
 	}
-
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	if (ev_fence)
-		dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP);
-	spin_unlock(&evf_mgr->ev_fence_lock);
-
 	return 0;
 }
 
-void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				  struct amdgpu_bo *bo)
+void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo)
 {
 	struct dma_fence *stub = dma_fence_get_stub();
 
@@ -229,13 +144,31 @@ void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
 	dma_fence_put(stub);
 }
 
-int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
+void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
-	/* This needs to be done one time per open */
 	atomic_set(&evf_mgr->ev_fence_seq, 0);
 	evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);
-	spin_lock_init(&evf_mgr->ev_fence_lock);
+	evf_mgr->ev_fence = dma_fence_get_stub();
 
-	INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
-	return 0;
+	INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
+}
+
+void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	evf_mgr->shutdown = true;
+	/* Make sure that the shutdown is visible to the suspend work */
+	flush_work(&evf_mgr->suspend_work);
+}
+
+void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
+		       false);
+	/* Make sure that we are done with the last suspend work */
+	flush_work(&evf_mgr->suspend_work);
+}
+
+void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	dma_fence_put(evf_mgr->ev_fence);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
index fcd867b7147dd..2a750add4e7b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h
@@ -25,6 +25,8 @@
 #ifndef AMDGPU_EV_FENCE_H_
 #define AMDGPU_EV_FENCE_H_
 
+#include <linux/dma-fence.h>
+
 struct amdgpu_eviction_fence {
 	struct dma_fence base;
 	spinlock_t	 lock;
@@ -35,35 +37,36 @@ struct amdgpu_eviction_fence {
 struct amdgpu_eviction_fence_mgr {
 	u64			ev_fence_ctx;
 	atomic_t		ev_fence_seq;
-	spinlock_t		ev_fence_lock;
-	struct amdgpu_eviction_fence *ev_fence;
-	struct delayed_work	suspend_work;
-	uint8_t fd_closing;
-};
-
-/* Eviction fence helper functions */
-struct amdgpu_eviction_fence *
-amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr);
 
-void
-amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr);
-
-int
-amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_bo *bo);
+	/*
+	 * Only updated while holding the VM resv lock.
+	 * Only signaled while holding the userq mutex.
+	 */
+	struct dma_fence __rcu	*ev_fence;
+	struct work_struct	suspend_work;
+	bool			shutdown;
+};
 
-void
-amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_bo *bo);
+static inline struct dma_fence *
+amdgpu_evf_mgr_get_fence(struct amdgpu_eviction_fence_mgr *evf_mgr)
+{
+	struct dma_fence *ev_fence;
 
-int
-amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
+	rcu_read_lock();
+	ev_fence = dma_fence_get_rcu_safe(&evf_mgr->ev_fence);
+	rcu_read_unlock();
+	return ev_fence;
+}
 
-void
-amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
-			     struct amdgpu_eviction_fence *ev_fence);
+int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				struct amdgpu_bo *bo);
+int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
+			 struct drm_exec *exec);
+void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
+				 struct amdgpu_bo *bo);
+void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr);
+void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr);
 
-int
-amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
-				    struct drm_exec *exec);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 1054d66c54fac..d209591e3710a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -89,16 +89,6 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
 	return seq;
 }
 
-static void amdgpu_fence_save_fence_wptr_start(struct amdgpu_fence *af)
-{
-	af->fence_wptr_start = af->ring->wptr;
-}
-
-static void amdgpu_fence_save_fence_wptr_end(struct amdgpu_fence *af)
-{
-	af->fence_wptr_end = af->ring->wptr;
-}
-
 /**
  * amdgpu_fence_emit - emit a fence on the requested ring
  *
@@ -107,16 +97,14 @@ static void amdgpu_fence_save_fence_wptr_end(struct amdgpu_fence *af)
  * @flags: flags to pass into the subordinate .emit_fence() call
  *
  * Emits a fence command on the requested ring (all asics).
- * Returns 0 on success, -ENOMEM on failure.
  */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
-		      unsigned int flags)
+void amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
+		       unsigned int flags)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct dma_fence *fence;
 	struct dma_fence __rcu **ptr;
 	uint32_t seq;
-	int r;
 
 	fence = &af->base;
 	af->ring = ring;
@@ -126,11 +114,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
 		       &ring->fence_drv.lock,
 		       adev->fence_context + ring->idx, seq);
 
-	amdgpu_fence_save_fence_wptr_start(af);
 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
-	amdgpu_fence_save_fence_wptr_end(af);
-	amdgpu_fence_save_wptr(af);
+
 	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
 	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
@@ -141,10 +127,13 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
 		rcu_read_unlock();
 
 		if (old) {
-			r = dma_fence_wait(old, false);
+			/*
+			 * dma_fence_wait(old, false) is not interruptible.
+			 * It will not return an error in this case.
+			 * So we can safely ignore the return value.
+			 */
+			dma_fence_wait(old, false);
 			dma_fence_put(old);
-			if (r)
-				return r;
 		}
 	}
 
@@ -154,8 +143,6 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
 	 * emitting the fence would mess up the hardware ring buffer.
 	 */
 	rcu_assign_pointer(*ptr, dma_fence_get(fence));
-
-	return 0;
 }
 
 /**
@@ -241,7 +228,6 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring)
 
 	do {
 		struct dma_fence *fence, **ptr;
-		struct amdgpu_fence *am_fence;
 
 		++last_seq;
 		last_seq &= drv->num_fences_mask;
@@ -254,12 +240,6 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring)
 		if (!fence)
 			continue;
 
-		/* Save the wptr in the fence driver so we know what the last processed
-		 * wptr was.  This is required for re-emitting the ring state for
-		 * queues that are reset but are not guilty and thus have no guilty fence.
-		 */
-		am_fence = container_of(fence, struct amdgpu_fence, base);
-		drv->signalled_wptr = am_fence->wptr;
 		dma_fence_signal(fence);
 		dma_fence_put(fence);
 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
@@ -708,25 +688,29 @@ void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
  */
 
 /**
- * amdgpu_fence_driver_update_timedout_fence_state - Update fence state and set errors
+ * amdgpu_ring_set_fence_errors_and_reemit - Set dma_fence errors and reemit
  *
- * @af: fence of the ring to update
+ * @ring: the ring to operate on
+ * @guilty_fence: fence of the ring to update
  *
  */
-void amdgpu_fence_driver_update_timedout_fence_state(struct amdgpu_fence *af)
+void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
+					     struct amdgpu_fence *guilty_fence)
 {
 	struct dma_fence *unprocessed;
 	struct dma_fence __rcu **ptr;
 	struct amdgpu_fence *fence;
-	struct amdgpu_ring *ring = af->ring;
 	unsigned long flags;
 	u32 seq, last_seq;
-	bool reemitted = false;
+	unsigned int i;
+	bool is_guilty_fence;
+	bool is_guilty_context;
 
 	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
 
-	/* mark all fences from the guilty context with an error */
+	ring->reemit = true;
+	amdgpu_ring_alloc(ring, ring->ring_backup_entries_to_copy);
 	spin_lock_irqsave(&ring->fence_drv.lock, flags);
 	do {
 		last_seq++;
@@ -738,39 +722,45 @@ void amdgpu_fence_driver_update_timedout_fence_state(struct amdgpu_fence *af)
 
 		if (unprocessed && !dma_fence_is_signaled_locked(unprocessed)) {
 			fence = container_of(unprocessed, struct amdgpu_fence, base);
+			is_guilty_fence = fence == guilty_fence;
+			is_guilty_context = fence->context == guilty_fence->context;
 
-			if (fence->reemitted > 1)
-				reemitted = true;
-			else if (fence == af)
+			/* mark all fences from the guilty context with an error */
+			if (is_guilty_fence)
 				dma_fence_set_error(&fence->base, -ETIME);
-			else if (fence->context == af->context)
+			else if (is_guilty_context)
 				dma_fence_set_error(&fence->base, -ECANCELED);
+
+			/* reemit the packet stream and update wptrs */
+			fence->ib_wptr = ring->wptr;
+			for (i = 0; i < fence->ib_dw_size; i++) {
+				/* Skip the IB(s) for the guilty context. */
+				if (is_guilty_context &&
+				    i >= fence->skip_ib_dw_start_offset &&
+				    i < fence->skip_ib_dw_end_offset)
+					amdgpu_ring_write(ring, ring->funcs->nop);
+				else
+					amdgpu_ring_write(ring,
+							  ring->ring_backup[fence->backup_idx + i]);
+			}
 		}
 		rcu_read_unlock();
 	} while (last_seq != seq);
 	spin_unlock_irqrestore(&ring->fence_drv.lock, flags);
-
-	if (reemitted) {
-		/* if we've already reemitted once then just cancel everything */
-		amdgpu_fence_driver_force_completion(af->ring);
-		af->ring->ring_backup_entries_to_copy = 0;
-	}
-}
-
-void amdgpu_fence_save_wptr(struct amdgpu_fence *af)
-{
-	af->wptr = af->ring->wptr;
+	amdgpu_ring_commit(ring);
+	ring->reemit = false;
 }
 
 static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
-						   u64 start_wptr, u64 end_wptr)
+						   struct amdgpu_fence *af)
 {
-	unsigned int first_idx = start_wptr & ring->buf_mask;
-	unsigned int last_idx = end_wptr & ring->buf_mask;
+	unsigned int first_idx = af->ib_wptr & ring->buf_mask;
+	unsigned int dw_size = af->ib_dw_size;
 	unsigned int i;
 
+	af->backup_idx = ring->ring_backup_entries_to_copy;
 	/* Backup the contents of the ring buffer. */
-	for (i = first_idx; i != last_idx; ++i, i &= ring->buf_mask)
+	for (i = first_idx; dw_size > 0; ++i, i &= ring->buf_mask, --dw_size)
 		ring->ring_backup[ring->ring_backup_entries_to_copy++] = ring->ring[i];
 }
 
@@ -780,12 +770,10 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
 	struct dma_fence *unprocessed;
 	struct dma_fence __rcu **ptr;
 	struct amdgpu_fence *fence;
-	u64 wptr;
 	u32 seq, last_seq;
 
 	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
-	wptr = ring->fence_drv.signalled_wptr;
 	ring->ring_backup_entries_to_copy = 0;
 
 	do {
@@ -799,21 +787,7 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
 		if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
 			fence = container_of(unprocessed, struct amdgpu_fence, base);
 
-			/* save everything if the ring is not guilty, otherwise
-			 * just save the content from other contexts.
-			 */
-			if (!fence->reemitted &&
-			    (!guilty_fence || (fence->context != guilty_fence->context))) {
-				amdgpu_ring_backup_unprocessed_command(ring, wptr,
-								       fence->wptr);
-			} else if (!fence->reemitted) {
-				/* always save the fence */
-				amdgpu_ring_backup_unprocessed_command(ring,
-								       fence->fence_wptr_start,
-								       fence->fence_wptr_end);
-			}
-			wptr = fence->wptr;
-			fence->reemitted++;
+			amdgpu_ring_backup_unprocessed_command(ring, fence);
 		}
 		rcu_read_unlock();
 	} while (last_seq != seq);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c
index 514bd302365f2..841e1b3a017e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c
@@ -106,6 +106,7 @@ static bool is_fru_eeprom_supported(struct amdgpu_device *adev, u32 *fru_addr)
 				*fru_addr = FRU_EEPROM_MADDR_8;
 			return true;
 	case IP_VERSION(13, 0, 12):
+	case IP_VERSION(15, 0, 8):
 			if (fru_addr)
 				*fru_addr = FRU_EEPROM_MADDR_INV;
 			return true;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index a6107109a2b86..01dc73309d73c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -258,18 +258,15 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 
 	amdgpu_vm_bo_update_shared(abo);
 	bo_va = amdgpu_vm_bo_find(vm, abo);
-	if (!bo_va)
+	if (!bo_va) {
 		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
-	else
+		r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
+		if (r)
+			goto out_unlock;
+	} else {
 		++bo_va->ref_count;
-
-	/* attach gfx eviction fence */
-	r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
-	if (r) {
-		DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
-		amdgpu_bo_unreserve(abo);
-		return r;
 	}
+
 	drm_exec_fini(&exec);
 
 	/* Validate and add eviction fence to DMABuf imports with dynamic
@@ -337,7 +334,7 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj,
 	}
 
 	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
-		amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
+		amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo);
 
 	bo_va = amdgpu_vm_bo_find(vm, bo);
 	if (!bo_va || --bo_va->ref_count)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 77578ecc67828..cab3196a87fb1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -503,6 +503,55 @@ void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
 			      &ring->mqd_ptr);
 }
 
+void amdgpu_gfx_mqd_symmetrically_map_cu_mask(struct amdgpu_device *adev, const uint32_t *cu_mask,
+					      uint32_t cu_mask_count, uint32_t *se_mask)
+{
+	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
+	struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
+	uint32_t cu_per_sh[8][4] = {0};
+	int i, se, sh, cu, cu_bitmap_sh_mul;
+	int xcc_inst = ffs(adev->gfx.xcc_mask) - 1;
+	bool wgp_mode_req = amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0);
+	int cu_inc = wgp_mode_req ? 2 : 1;
+	uint32_t en_mask = wgp_mode_req ? 0x3 : 0x1;
+	int num_xcc, inc, inst = 0;
+
+	if (xcc_inst < 0)
+		xcc_inst = 0;
+
+	num_xcc = hweight16(adev->gfx.xcc_mask);
+	if (!num_xcc)
+		num_xcc = 1;
+
+	inc = cu_inc * num_xcc;
+
+	cu_bitmap_sh_mul = 2;
+
+	for (se = 0; se < gfx_info->max_shader_engines; se++)
+		for (sh = 0; sh < gfx_info->max_sh_per_se; sh++)
+			cu_per_sh[se][sh] = hweight32(
+				cu_info->bitmap[xcc_inst][se % 4][sh + (se / 4) *
+				cu_bitmap_sh_mul]);
+
+	for (i = 0; i < gfx_info->max_shader_engines; i++)
+		se_mask[i] = 0;
+
+	i = inst;
+	for (cu = 0; cu < 16; cu += cu_inc) {
+		for (sh = 0; sh < gfx_info->max_sh_per_se; sh++) {
+			for (se = 0; se < gfx_info->max_shader_engines; se++) {
+				if (cu_per_sh[se][sh] > cu) {
+					if ((i / 32) < cu_mask_count && (cu_mask[i / 32] & (1 << (i % 32))))
+						se_mask[se] |= en_mask << (cu + sh * 16);
+					i += inc;
+					if (i >= cu_mask_count * 32)
+						return;
+				}
+			}
+		}
+	}
+}
+
 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 {
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 585cc8e81bb28..2785eda6fea52 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -463,6 +463,7 @@ struct amdgpu_gfx {
 	struct amdgpu_irq_src		cp_ecc_error_irq;
 	struct amdgpu_irq_src		sq_irq;
 	struct amdgpu_irq_src		rlc_gc_fed_irq;
+	struct amdgpu_irq_src		rlc_poison_irq;
 	struct sq_work			sq_work;
 
 	/* gfx status */
@@ -583,6 +584,8 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 			   unsigned mqd_size, int xcc_id);
 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id);
+void amdgpu_gfx_mqd_symmetrically_map_cu_mask(struct amdgpu_device *adev, const uint32_t *cu_mask,
+					      uint32_t cu_mask_count, uint32_t *se_mask);
 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id);
 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id);
 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 5179fa008626e..a0940db1cd36c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -742,7 +742,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	 * translation. Avoid this by doing the invalidation from the SDMA
 	 * itself at least for GART.
 	 */
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&adev->mman.default_entity.lock);
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
@@ -755,7 +755,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	fence = amdgpu_job_submit(job);
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&adev->mman.default_entity.lock);
 
 	dma_fence_wait(fence, false);
 	dma_fence_put(fence);
@@ -763,7 +763,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	return;
 
 error_alloc:
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&adev->mman.default_entity.lock);
 	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 0e8a52d965738..b9fdc3276e816 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -94,6 +94,10 @@ enum amdgpu_memory_partition {
 #define AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE 0x20
 #define AMDGPU_GMC9_FAULT_SOURCE_DATA_EXE   0x10
 
+#define AMDGPU_GMC121_FAULT_SOURCE_DATA_READ  0x400000
+#define AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE 0x200000
+#define AMDGPU_GMC121_FAULT_SOURCE_DATA_EXE   0x100000
+
 /*
  * GMC page fault information
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index ac276bb53c7c2..620fddde4c4dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -324,17 +324,13 @@ int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size)
 {
 	struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
 	struct ttm_resource_manager *man = &mgr->manager;
-	uint64_t start, size;
 
 	man->use_tt = true;
 	man->func = &amdgpu_gtt_mgr_func;
 
 	ttm_resource_manager_init(man, &adev->mman.bdev, gtt_size);
 
-	start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
-	start += amdgpu_vce_required_gart_pages(adev);
-	size = (adev->gmc.gart_size >> PAGE_SHIFT) - start;
-	drm_mm_init(&mgr->mm, start, size);
+	drm_mm_init(&mgr->mm, 0, adev->gmc.gart_size >> PAGE_SHIFT);
 	spin_lock_init(&mgr->lock);
 
 	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 3a7bab87b5d80..63f62c670df56 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -129,6 +129,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	struct amdgpu_ib *ib = &ibs[0];
 	struct dma_fence *tmp = NULL;
 	struct amdgpu_fence *af;
+	struct amdgpu_fence *vm_af;
 	bool need_ctx_switch;
 	struct amdgpu_vm *vm;
 	uint64_t fence_ctx;
@@ -215,25 +216,28 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		dma_fence_put(tmp);
 	}
 
-	if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync)
-		ring->funcs->emit_mem_sync(ring);
+	if (job) {
+		vm_af = job->hw_vm_fence;
+		/* VM sequence */
+		vm_af->ib_wptr = ring->wptr;
+		amdgpu_vm_flush(ring, job, need_pipe_sync);
+		vm_af->ib_dw_size =
+			amdgpu_ring_get_dw_distance(ring, vm_af->ib_wptr, ring->wptr);
+	}
 
-	if (ring->funcs->emit_wave_limit &&
-	    ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH)
-		ring->funcs->emit_wave_limit(ring, true);
+	/* IB sequence */
+	af->ib_wptr = ring->wptr;
+	amdgpu_ring_ib_begin(ring);
 
 	if (ring->funcs->insert_start)
 		ring->funcs->insert_start(ring);
 
-	if (job) {
-		r = amdgpu_vm_flush(ring, job, need_pipe_sync);
-		if (r) {
-			amdgpu_ring_undo(ring);
-			goto free_fence;
-		}
-	}
+	if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync)
+		ring->funcs->emit_mem_sync(ring);
 
-	amdgpu_ring_ib_begin(ring);
+	if (ring->funcs->emit_wave_limit &&
+	    ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH)
+		ring->funcs->emit_wave_limit(ring, true);
 
 	if (ring->funcs->emit_gfx_shadow && adev->gfx.cp_gfx_shadow)
 		amdgpu_ring_emit_gfx_shadow(ring, shadow_va, csa_va, gds_va,
@@ -243,6 +247,9 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		cond_exec = amdgpu_ring_init_cond_exec(ring,
 						       ring->cond_exe_gpu_addr);
 
+	/* Skip the IB for guilty contexts */
+	af->skip_ib_dw_start_offset =
+		amdgpu_ring_get_dw_distance(ring, af->ib_wptr, ring->wptr);
 	amdgpu_device_flush_hdp(adev, ring);
 
 	if (need_ctx_switch)
@@ -281,6 +288,9 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		amdgpu_ring_emit_frame_cntl(ring, false, secure);
 
 	amdgpu_device_invalidate_hdp(adev, ring);
+	/* Skip the IB for guilty contexts */
+	af->skip_ib_dw_end_offset =
+		amdgpu_ring_get_dw_distance(ring, af->ib_wptr, ring->wptr);
 
 	if (ib->flags & AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE)
 		fence_flags |= AMDGPU_FENCE_FLAG_TC_WB_ONLY;
@@ -297,14 +307,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		amdgpu_ring_init_cond_exec(ring, ring->cond_exe_gpu_addr);
 	}
 
-	r = amdgpu_fence_emit(ring, af, fence_flags);
-	if (r) {
-		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
-		if (job && job->vmid)
-			amdgpu_vmid_reset(adev, ring->vm_hub, job->vmid);
-		amdgpu_ring_undo(ring);
-		goto free_fence;
-	}
+	amdgpu_fence_emit(ring, af, fence_flags);
 	*f = &af->base;
 	/* get a ref for the job */
 	if (job)
@@ -323,15 +326,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	    ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH)
 		ring->funcs->emit_wave_limit(ring, false);
 
-	/* Save the wptr associated with this fence.
-	 * This must be last for resets to work properly
-	 * as we need to save the wptr associated with this
-	 * fence so we know what rings contents to backup
-	 * after we reset the queue.
-	 */
-	amdgpu_fence_save_wptr(af);
-
 	amdgpu_ring_ib_end(ring);
+
+	af->ib_dw_size = amdgpu_ring_get_dw_distance(ring, af->ib_wptr, ring->wptr);
+
 	amdgpu_ring_commit(ring);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index d7c1ffbf7626c..07771721af9d2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -92,7 +92,6 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	struct drm_wedge_task_info *info = NULL;
 	struct amdgpu_task_info *ti = NULL;
 	struct amdgpu_device *adev = ring->adev;
-	enum drm_gpu_sched_stat status = DRM_GPU_SCHED_STAT_RESET;
 	int idx, r;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
@@ -147,8 +146,6 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 				ring->sched.name);
 			drm_dev_wedged_event(adev_to_drm(adev),
 					     DRM_WEDGE_RECOVERY_NONE, info);
-			/* This is needed to add the job back to the pending list */
-			status = DRM_GPU_SCHED_STAT_NO_HANG;
 			goto exit;
 		}
 		dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name);
@@ -184,7 +181,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 exit:
 	amdgpu_vm_put_task_info(ti);
 	drm_dev_exit(idx);
-	return status;
+	/* This is needed to add the job back to the pending list */
+	return DRM_GPU_SCHED_STAT_NO_HANG;
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -289,9 +287,10 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 	unsigned i;
 
 	/* Check if any fences were initialized */
-	if (job->base.s_fence && job->base.s_fence->finished.ops)
+	if (job->base.s_fence &&
+	    dma_fence_was_initialized(&job->base.s_fence->finished))
 		f = &job->base.s_fence->finished;
-	else if (job->hw_fence && job->hw_fence->base.ops)
+	else if (dma_fence_was_initialized(&job->hw_fence->base))
 		f = &job->hw_fence->base;
 	else
 		f = NULL;
@@ -308,11 +307,11 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 
 	amdgpu_sync_free(&job->explicit_sync);
 
-	if (job->hw_fence->base.ops)
+	if (dma_fence_was_initialized(&job->hw_fence->base))
 		dma_fence_put(&job->hw_fence->base);
 	else
 		kfree(job->hw_fence);
-	if (job->hw_vm_fence->base.ops)
+	if (dma_fence_was_initialized(&job->hw_vm_fence->base))
 		dma_fence_put(&job->hw_vm_fence->base);
 	else
 		kfree(job->hw_vm_fence);
@@ -346,11 +345,11 @@ void amdgpu_job_free(struct amdgpu_job *job)
 	if (job->gang_submit != &job->base.s_fence->scheduled)
 		dma_fence_put(job->gang_submit);
 
-	if (job->hw_fence->base.ops)
+	if (dma_fence_was_initialized(&job->hw_fence->base))
 		dma_fence_put(&job->hw_fence->base);
 	else
 		kfree(job->hw_fence);
-	if (job->hw_vm_fence->base.ops)
+	if (dma_fence_was_initialized(&job->hw_vm_fence->base))
 		dma_fence_put(&job->hw_vm_fence->base);
 	else
 		kfree(job->hw_vm_fence);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 7f19554b9ad11..06efce38f3238 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1522,10 +1522,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 			 "Failed to init usermode queue manager (%d), use legacy workload submission only\n",
 			 r);
 
-	r = amdgpu_eviction_fence_init(&fpriv->evf_mgr);
-	if (r)
-		goto error_vm;
-
+	amdgpu_evf_mgr_init(&fpriv->evf_mgr);
 	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
 
 	file_priv->driver_priv = fpriv;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index bcf2a067dc410..f80e3aca9c78e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -159,9 +159,9 @@ struct amdgpu_mes {
 
 	int				hung_queue_db_array_size;
 	int				hung_queue_hqd_info_offset;
-	struct amdgpu_bo		*hung_queue_db_array_gpu_obj[AMDGPU_MAX_MES_PIPES];
-	uint64_t			hung_queue_db_array_gpu_addr[AMDGPU_MAX_MES_PIPES];
-	void				*hung_queue_db_array_cpu_addr[AMDGPU_MAX_MES_PIPES];
+	struct amdgpu_bo		*hung_queue_db_array_gpu_obj[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t			hung_queue_db_array_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+	void				*hung_queue_db_array_cpu_addr[AMDGPU_MAX_MES_INST_PIPES];
 
 	/* cooperative dispatch */
 	bool                enable_coop_mode;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
index 1ca9d4ed8063a..6b8214650e5d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
@@ -63,14 +63,40 @@ struct amdgpu_mmhub_funcs {
 				uint64_t page_table_base);
 	void (*update_power_gating)(struct amdgpu_device *adev,
                                 bool enable);
+	int (*get_xgmi_info)(struct amdgpu_device *adev);
+};
+
+struct amdgpu_mmhub_client_ids {
+	const char * const (*names)[2];
+	unsigned int size;
 };
 
 struct amdgpu_mmhub {
 	struct ras_common_if *ras_if;
 	const struct amdgpu_mmhub_funcs *funcs;
 	struct amdgpu_mmhub_ras  *ras;
+	struct amdgpu_mmhub_client_ids client_ids;
 };
 
+static inline void
+amdgpu_mmhub_init_client_info(struct amdgpu_mmhub *mmhub,
+			      const char * const (*names)[2],
+			      unsigned int size)
+{
+	mmhub->client_ids.names = names;
+	mmhub->client_ids.size = size;
+}
+
+static inline const char *
+amdgpu_mmhub_client_name(struct amdgpu_mmhub *mmhub,
+			  u32 cid, bool is_write)
+{
+	if (cid < mmhub->client_ids.size)
+		return mmhub->client_ids.names[cid][is_write];
+
+	return NULL;
+}
+
 int amdgpu_mmhub_ras_sw_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index e244c12ceb238..90352284c5ee2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -624,7 +624,7 @@ struct amdgpu_connector {
 	bool use_digital;
 	/* we need to mind the EDID between detect
 	   and get modes due to analog/digital/tvencoder */
-	struct edid *edid;
+	const struct drm_edid *edid;
 	void *con_priv;
 	bool dac_load_detect;
 	bool detected_by_load; /* if the connection status was determined by load */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 1fb9564006966..3220ab8547756 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -1274,7 +1274,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 
 	if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) &&
 	    old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
-		dma_buf_move_notify(abo->tbo.base.dma_buf);
+		dma_buf_invalidate_mappings(abo->tbo.base.dma_buf);
 
 	/* move_notify is called before move happens */
 	trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
@@ -1676,9 +1676,9 @@ u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
 	attachment = READ_ONCE(bo->tbo.base.import_attach);
 
 	if (attachment)
-		seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino);
+		seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino);
 	else if (dma_buf)
-		seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino);
+		seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino);
 
 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 27b67da9fdac2..d39b695cd9259 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -3096,6 +3096,13 @@ static int psp_load_non_psp_fw(struct psp_context *psp)
 			 */
 			continue;
 
+		/* IMU ucode is part of IFWI and MP0 15.0.8 would load it */
+		if (amdgpu_ip_version(adev, MP0_HWIP, 0) ==
+		    IP_VERSION(15, 0, 8) &&
+		    (ucode->ucode_id == AMDGPU_UCODE_ID_IMU_I ||
+		    ucode->ucode_id == AMDGPU_UCODE_ID_IMU_D))
+			continue;
+
 		psp_print_fw_hdr(psp, ucode);
 
 		ret = psp_execute_ip_fw_load(psp, ucode);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index c0336ca9bf6a8..6edcb77132993 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -21,6 +21,7 @@
  *
  *
  */
+#include "amdgpu_reg_access.h"
 #include <linux/debugfs.h>
 #include <linux/list.h>
 #include <linux/module.h>
@@ -42,6 +43,7 @@
 #include "amdgpu_reset.h"
 #include "amdgpu_psp.h"
 #include "amdgpu_ras_mgr.h"
+#include "amdgpu_virt_ras_cmd.h"
 
 #ifdef CONFIG_X86_MCE_AMD
 #include <asm/mce.h>
@@ -228,19 +230,30 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev,
 		return 0;
 
 	if (amdgpu_sriov_vf(adev)) {
-		if (amdgpu_virt_check_vf_critical_region(adev, address, &hit))
-			return -EPERM;
-		return hit ? -EACCES : 0;
+		if (amdgpu_uniras_enabled(adev)) {
+			if (amdgpu_virt_ras_check_address_validity(adev, address, &hit))
+				return -EPERM;
+			if (hit)
+				return -EACCES;
+		} else {
+			if (amdgpu_virt_check_vf_critical_region(adev, address, &hit))
+				return -EPERM;
+			return hit ? -EACCES : 0;
+		}
 	}
 
 	if ((address >= adev->gmc.mc_vram_size) ||
 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT))
 		return -EFAULT;
 
-	if (amdgpu_uniras_enabled(adev))
-		count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address,
-			page_pfns, ARRAY_SIZE(page_pfns));
-	else
+	if (amdgpu_uniras_enabled(adev)) {
+		if (amdgpu_sriov_vf(adev))
+			count = amdgpu_virt_ras_convert_retired_address(adev, address,
+				page_pfns, ARRAY_SIZE(page_pfns));
+		else
+			count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address,
+				page_pfns, ARRAY_SIZE(page_pfns));
+	} else
 		count = amdgpu_umc_lookup_bad_pages_in_a_row(adev,
 				address, page_pfns, ARRAY_SIZE(page_pfns));
 
@@ -2237,6 +2250,7 @@ static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 12):
 	case IP_VERSION(13, 0, 14):
+	case IP_VERSION(13, 0, 15):
 		ret = true;
 		break;
 	default:
@@ -3118,9 +3132,11 @@ static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
 				enum amdgpu_memory_partition nps)
 {
 	int i = 0;
+	uint64_t chan_idx_v2;
 	enum amdgpu_memory_partition save_nps;
 
 	save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
+	chan_idx_v2 = bps[0].retired_page & UMC_CHANNEL_IDX_V2;
 
 	/*old asics just have pa in eeprom*/
 	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
@@ -3132,7 +3148,7 @@ static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
 	for (i = 0; i < adev->umc.retire_unit; i++)
 		bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
 
-	if (save_nps) {
+	if (save_nps || chan_idx_v2) {
 		if (save_nps == nps) {
 			if (amdgpu_umc_pages_in_a_row(adev, err_data,
 					bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))
@@ -3176,10 +3192,12 @@ static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
 				enum amdgpu_memory_partition nps)
 {
 	int i = 0;
+	uint64_t chan_idx_v2;
 	enum amdgpu_memory_partition save_nps;
 
 	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
 		save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
+		chan_idx_v2 = bps->retired_page & UMC_CHANNEL_IDX_V2;
 		bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
 	} else {
 		/* if pmfw manages eeprom, save_nps is not stored on eeprom,
@@ -3201,16 +3219,19 @@ static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
 			err_data->err_addr[i].mcumc_id = bps->mcumc_id;
 		}
 	} else {
-		if (bps->address) {
+		if (save_nps || chan_idx_v2) {
 			if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))
 				return -EINVAL;
 		} else {
 			/* for specific old eeprom data, mca address is not stored,
 			 * calc it from pa
 			 */
-			if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
-				&(bps->address), AMDGPU_NPS1_PARTITION_MODE))
-				return -EINVAL;
+			if (bps->address == 0)
+				if (amdgpu_umc_pa2mca(adev,
+					bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
+					&(bps->address),
+					AMDGPU_NPS1_PARTITION_MODE))
+					return -EINVAL;
 
 			if (amdgpu_ras_mca2pa(adev, bps, err_data))
 				return -EOPNOTSUPP;
@@ -3980,6 +4001,7 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
 		case IP_VERSION(13, 0, 6):
 		case IP_VERSION(13, 0, 12):
 		case IP_VERSION(13, 0, 14):
+		case IP_VERSION(13, 0, 15):
 			return true;
 		default:
 			return false;
@@ -3993,6 +4015,7 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
 		case IP_VERSION(13, 0, 10):
 		case IP_VERSION(13, 0, 12):
 		case IP_VERSION(13, 0, 14):
+		case IP_VERSION(13, 0, 15):
 		case IP_VERSION(14, 0, 3):
 			return true;
 		default:
@@ -4162,7 +4185,8 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
 		adev->aca.is_enabled =
 			(amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
 			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
-			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14));
+			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
+			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 15));
 	}
 
 	/* bad page feature is not applicable to specific app platform */
@@ -4250,6 +4274,7 @@ static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 2):
 	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 12):
+	case IP_VERSION(13, 0, 15):
 		con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT;
 		break;
 	case IP_VERSION(13, 0, 14):
@@ -5516,11 +5541,11 @@ static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
 	 * is changed. In such case, replace the aqua_vanjaram implementation
 	 * with more common helper */
 	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
-		   aqua_vanjaram_encode_ext_smn_addressing(instance);
+		   amdgpu_reg_get_smn_base64(adev, MP0_HWIP, instance);
 	fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
 
 	reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
-		   aqua_vanjaram_encode_ext_smn_addressing(instance);
+		   amdgpu_reg_get_smn_base64(adev, MP0_HWIP, instance);
 	boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
 
 	socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
@@ -5586,7 +5611,7 @@ static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
 	int retry_loop;
 
 	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
-		   aqua_vanjaram_encode_ext_smn_addressing(instance);
+		   amdgpu_reg_get_smn_base64(adev, MP0_HWIP, instance);
 
 	for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
 		reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
@@ -5665,7 +5690,7 @@ int amdgpu_ras_add_critical_region(struct amdgpu_device *adev,
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct amdgpu_vram_mgr_resource *vres;
 	struct ras_critical_region *region;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	int ret = 0;
 
 	if (!bo || !bo->tbo.resource)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index 6fba9d5b29ea6..44fba4b6aa92a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -508,6 +508,9 @@ int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control)
 	control->bad_channel_bitmap = 0;
 	amdgpu_dpm_send_hbm_bad_channel_flag(adev, control->bad_channel_bitmap);
 	con->update_channel_flag = false;
+	/* there is no record on eeprom now, clear the counter */
+	if (con->eh_data)
+		con->eh_data->count_saved = 0;
 
 	amdgpu_ras_debugfs_set_ret_size(control);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
new file mode 100644
index 0000000000000..540040c76058b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
@@ -0,0 +1,958 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/delay.h>
+
+#include "amdgpu.h"
+#include "amdgpu_reset.h"
+#include "amdgpu_trace.h"
+#include "amdgpu_virt.h"
+#include "amdgpu_reg_access.h"
+
+#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
+#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
+#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
+
+void amdgpu_reg_access_init(struct amdgpu_device *adev)
+{
+	spin_lock_init(&adev->reg.smc.lock);
+	adev->reg.smc.rreg = NULL;
+	adev->reg.smc.wreg = NULL;
+
+	spin_lock_init(&adev->reg.uvd_ctx.lock);
+	adev->reg.uvd_ctx.rreg = NULL;
+	adev->reg.uvd_ctx.wreg = NULL;
+
+	spin_lock_init(&adev->reg.didt.lock);
+	adev->reg.didt.rreg = NULL;
+	adev->reg.didt.wreg = NULL;
+
+	spin_lock_init(&adev->reg.gc_cac.lock);
+	adev->reg.gc_cac.rreg = NULL;
+	adev->reg.gc_cac.wreg = NULL;
+
+	spin_lock_init(&adev->reg.se_cac.lock);
+	adev->reg.se_cac.rreg = NULL;
+	adev->reg.se_cac.wreg = NULL;
+
+	spin_lock_init(&adev->reg.audio_endpt.lock);
+	adev->reg.audio_endpt.rreg = NULL;
+	adev->reg.audio_endpt.wreg = NULL;
+
+	spin_lock_init(&adev->reg.pcie.lock);
+	adev->reg.pcie.rreg = NULL;
+	adev->reg.pcie.wreg = NULL;
+	adev->reg.pcie.rreg_ext = NULL;
+	adev->reg.pcie.wreg_ext = NULL;
+	adev->reg.pcie.rreg64 = NULL;
+	adev->reg.pcie.wreg64 = NULL;
+	adev->reg.pcie.rreg64_ext = NULL;
+	adev->reg.pcie.wreg64_ext = NULL;
+	adev->reg.pcie.port_rreg = NULL;
+	adev->reg.pcie.port_wreg = NULL;
+}
+
+uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.smc.rreg) {
+		dev_err_once(adev->dev, "SMC register read not supported\n");
+		return 0;
+	}
+	return adev->reg.smc.rreg(adev, reg);
+}
+
+void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	if (!adev->reg.smc.wreg) {
+		dev_err_once(adev->dev, "SMC register write not supported\n");
+		return;
+	}
+	adev->reg.smc.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_uvd_ctx_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.uvd_ctx.rreg) {
+		dev_err_once(adev->dev,
+			     "UVD_CTX register read not supported\n");
+		return 0;
+	}
+	return adev->reg.uvd_ctx.rreg(adev, reg);
+}
+
+void amdgpu_reg_uvd_ctx_wr32(struct amdgpu_device *adev, uint32_t reg,
+			     uint32_t v)
+{
+	if (!adev->reg.uvd_ctx.wreg) {
+		dev_err_once(adev->dev,
+			     "UVD_CTX register write not supported\n");
+		return;
+	}
+	adev->reg.uvd_ctx.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_didt_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.didt.rreg) {
+		dev_err_once(adev->dev, "DIDT register read not supported\n");
+		return 0;
+	}
+	return adev->reg.didt.rreg(adev, reg);
+}
+
+void amdgpu_reg_didt_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	if (!adev->reg.didt.wreg) {
+		dev_err_once(adev->dev, "DIDT register write not supported\n");
+		return;
+	}
+	adev->reg.didt.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_gc_cac_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.gc_cac.rreg) {
+		dev_err_once(adev->dev, "GC_CAC register read not supported\n");
+		return 0;
+	}
+	return adev->reg.gc_cac.rreg(adev, reg);
+}
+
+void amdgpu_reg_gc_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v)
+{
+	if (!adev->reg.gc_cac.wreg) {
+		dev_err_once(adev->dev,
+			     "GC_CAC register write not supported\n");
+		return;
+	}
+	adev->reg.gc_cac.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_se_cac_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.se_cac.rreg) {
+		dev_err_once(adev->dev, "SE_CAC register read not supported\n");
+		return 0;
+	}
+	return adev->reg.se_cac.rreg(adev, reg);
+}
+
+void amdgpu_reg_se_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v)
+{
+	if (!adev->reg.se_cac.wreg) {
+		dev_err_once(adev->dev,
+			     "SE_CAC register write not supported\n");
+		return;
+	}
+	adev->reg.se_cac.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_audio_endpt_rd32(struct amdgpu_device *adev, uint32_t block,
+				     uint32_t reg)
+{
+	if (!adev->reg.audio_endpt.rreg) {
+		dev_err_once(adev->dev,
+			     "AUDIO_ENDPT register read not supported\n");
+		return 0;
+	}
+	return adev->reg.audio_endpt.rreg(adev, block, reg);
+}
+
+void amdgpu_reg_audio_endpt_wr32(struct amdgpu_device *adev, uint32_t block,
+				 uint32_t reg, uint32_t v)
+{
+	if (!adev->reg.audio_endpt.wreg) {
+		dev_err_once(adev->dev,
+			     "AUDIO_ENDPT register write not supported\n");
+		return;
+	}
+	adev->reg.audio_endpt.wreg(adev, block, reg, v);
+}
+
+uint32_t amdgpu_reg_pcie_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.pcie.rreg) {
+		dev_err_once(adev->dev, "PCIE register read not supported\n");
+		return 0;
+	}
+	return adev->reg.pcie.rreg(adev, reg);
+}
+
+void amdgpu_reg_pcie_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	if (!adev->reg.pcie.wreg) {
+		dev_err_once(adev->dev, "PCIE register write not supported\n");
+		return;
+	}
+	adev->reg.pcie.wreg(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_pcie_ext_rd32(struct amdgpu_device *adev, uint64_t reg)
+{
+	if (!adev->reg.pcie.rreg_ext) {
+		dev_err_once(adev->dev, "PCIE EXT register read not supported\n");
+		return 0;
+	}
+	return adev->reg.pcie.rreg_ext(adev, reg);
+}
+
+void amdgpu_reg_pcie_ext_wr32(struct amdgpu_device *adev, uint64_t reg,
+			      uint32_t v)
+{
+	if (!adev->reg.pcie.wreg_ext) {
+		dev_err_once(adev->dev, "PCIE EXT register write not supported\n");
+		return;
+	}
+	adev->reg.pcie.wreg_ext(adev, reg, v);
+}
+
+uint64_t amdgpu_reg_pcie_rd64(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.pcie.rreg64) {
+		dev_err_once(adev->dev, "PCIE 64-bit register read not supported\n");
+		return 0;
+	}
+	return adev->reg.pcie.rreg64(adev, reg);
+}
+
+void amdgpu_reg_pcie_wr64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
+{
+	if (!adev->reg.pcie.wreg64) {
+		dev_err_once(adev->dev, "PCIE 64-bit register write not supported\n");
+		return;
+	}
+	adev->reg.pcie.wreg64(adev, reg, v);
+}
+
+uint64_t amdgpu_reg_pcie_ext_rd64(struct amdgpu_device *adev, uint64_t reg)
+{
+	if (!adev->reg.pcie.rreg64_ext) {
+		dev_err_once(adev->dev, "PCIE EXT 64-bit register read not supported\n");
+		return 0;
+	}
+	return adev->reg.pcie.rreg64_ext(adev, reg);
+}
+
+void amdgpu_reg_pcie_ext_wr64(struct amdgpu_device *adev, uint64_t reg,
+			      uint64_t v)
+{
+	if (!adev->reg.pcie.wreg64_ext) {
+		dev_err_once(adev->dev, "PCIE EXT 64-bit register write not supported\n");
+		return;
+	}
+	adev->reg.pcie.wreg64_ext(adev, reg, v);
+}
+
+uint32_t amdgpu_reg_pciep_rd32(struct amdgpu_device *adev, uint32_t reg)
+{
+	if (!adev->reg.pcie.port_rreg) {
+		dev_err_once(adev->dev, "PCIEP register read not supported\n");
+		return 0;
+	}
+	return adev->reg.pcie.port_rreg(adev, reg);
+}
+
+void amdgpu_reg_pciep_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	if (!adev->reg.pcie.port_wreg) {
+		dev_err_once(adev->dev, "PCIEP register write not supported\n");
+		return;
+	}
+	adev->reg.pcie.port_wreg(adev, reg, v);
+}
+
+static int amdgpu_reg_get_smn_base_version(struct amdgpu_device *adev)
+{
+	struct pci_dev *pdev = adev->pdev;
+	int id;
+
+	if (amdgpu_sriov_vf(adev))
+		return -EOPNOTSUPP;
+
+	id = (pdev->device >> 4) & 0xFFFF;
+	if (id == 0x74A || id == 0x74B || id == 0x75A || id == 0x75B)
+		return 1;
+
+	return -EOPNOTSUPP;
+}
+
+uint64_t amdgpu_reg_get_smn_base64(struct amdgpu_device *adev,
+				   enum amd_hw_ip_block_type block,
+				   int die_inst)
+{
+	if (!adev->reg.smn.get_smn_base) {
+		int version = amdgpu_reg_get_smn_base_version(adev);
+		switch (version) {
+		case 1:
+			return amdgpu_reg_smn_v1_0_get_base(adev, block,
+							    die_inst);
+		default:
+			dev_err_once(
+				adev->dev,
+				"SMN base address query not supported for this device\n");
+			return 0;
+		}
+	}
+	return adev->reg.smn.get_smn_base(adev, block, die_inst);
+}
+
+uint64_t amdgpu_reg_smn_v1_0_get_base(struct amdgpu_device *adev,
+				      enum amd_hw_ip_block_type block,
+				      int die_inst)
+{
+	uint64_t smn_base;
+
+	if (die_inst == 0)
+		return 0;
+
+	switch (block) {
+	case XGMI_HWIP:
+	case NBIO_HWIP:
+	case MP0_HWIP:
+	case UMC_HWIP:
+	case DF_HWIP:
+		smn_base = ((uint64_t)(die_inst & 0x3) << 32) | (1ULL << 34);
+		break;
+	default:
+		dev_warn_once(
+			adev->dev,
+			"SMN base address query not supported for this block %d\n",
+			block);
+		smn_base = 0;
+		break;
+	}
+
+	return smn_base;
+}
+
+/*
+ * register access helper functions.
+ */
+
+/**
+ * amdgpu_device_rreg - read a memory mapped IO or indirect register
+ *
+ * @adev: amdgpu_device pointer
+ * @reg: dword aligned register offset
+ * @acc_flags: access flags which require special behavior
+ *
+ * Returns the 32 bit value from the offset specified.
+ */
+uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t acc_flags)
+{
+	uint32_t ret;
+
+	if (amdgpu_device_skip_hw_access(adev))
+		return 0;
+
+	if ((reg * 4) < adev->rmmio_size) {
+		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
+		    amdgpu_sriov_runtime(adev) &&
+		    down_read_trylock(&adev->reset_domain->sem)) {
+			ret = amdgpu_kiq_rreg(adev, reg, 0);
+			up_read(&adev->reset_domain->sem);
+		} else {
+			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
+		}
+	} else {
+		ret = amdgpu_reg_pcie_rd32(adev, reg * 4);
+	}
+
+	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
+
+	return ret;
+}
+
+/*
+ * MMIO register read with bytes helper functions
+ * @offset:bytes offset from MMIO start
+ */
+
+/**
+ * amdgpu_mm_rreg8 - read a memory mapped IO register
+ *
+ * @adev: amdgpu_device pointer
+ * @offset: byte aligned register offset
+ *
+ * Returns the 8 bit value from the offset specified.
+ */
+uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
+{
+	if (amdgpu_device_skip_hw_access(adev))
+		return 0;
+
+	if (offset < adev->rmmio_size)
+		return (readb(adev->rmmio + offset));
+	BUG();
+}
+
+/**
+ * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
+ *
+ * @adev: amdgpu_device pointer
+ * @reg: dword aligned register offset
+ * @acc_flags: access flags which require special behavior
+ * @xcc_id: xcc accelerated compute core id
+ *
+ * Returns the 32 bit value from the offset specified.
+ */
+uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, uint32_t reg,
+				uint32_t acc_flags, uint32_t xcc_id)
+{
+	uint32_t ret, rlcg_flag;
+
+	if (amdgpu_device_skip_hw_access(adev))
+		return 0;
+
+	if ((reg * 4) < adev->rmmio_size) {
+		if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) &&
+		    adev->gfx.rlc.rlcg_reg_access_supported &&
+		    amdgpu_virt_get_rlcg_reg_access_flag(
+			    adev, acc_flags, GC_HWIP, false, &rlcg_flag)) {
+			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
+						      GET_INST(GC, xcc_id));
+		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
+			   amdgpu_sriov_runtime(adev) &&
+			   down_read_trylock(&adev->reset_domain->sem)) {
+			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+			up_read(&adev->reset_domain->sem);
+		} else {
+			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
+		}
+	} else {
+		ret = amdgpu_reg_pcie_rd32(adev, reg * 4);
+	}
+
+	return ret;
+}
+
+/*
+ * MMIO register write with bytes helper functions
+ * @offset:bytes offset from MMIO start
+ * @value: the value want to be written to the register
+ */
+
+/**
+ * amdgpu_mm_wreg8 - read a memory mapped IO register
+ *
+ * @adev: amdgpu_device pointer
+ * @offset: byte aligned register offset
+ * @value: 8 bit value to write
+ *
+ * Writes the value specified to the offset specified.
+ */
+void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
+{
+	if (amdgpu_device_skip_hw_access(adev))
+		return;
+
+	if (offset < adev->rmmio_size)
+		writeb(value, adev->rmmio + offset);
+	else
+		BUG();
+}
+
+/**
+ * amdgpu_device_wreg - write to a memory mapped IO or indirect register
+ *
+ * @adev: amdgpu_device pointer
+ * @reg: dword aligned register offset
+ * @v: 32 bit value to write to the register
+ * @acc_flags: access flags which require special behavior
+ *
+ * Writes the value specified to the offset specified.
+ */
+void amdgpu_device_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
+			uint32_t acc_flags)
+{
+	if (amdgpu_device_skip_hw_access(adev))
+		return;
+
+	if ((reg * 4) < adev->rmmio_size) {
+		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
+		    amdgpu_sriov_runtime(adev) &&
+		    down_read_trylock(&adev->reset_domain->sem)) {
+			amdgpu_kiq_wreg(adev, reg, v, 0);
+			up_read(&adev->reset_domain->sem);
+		} else {
+			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
+		}
+	} else {
+		amdgpu_reg_pcie_wr32(adev, reg * 4, v);
+	}
+
+	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
+}
+
+/**
+ * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
+ *
+ * @adev: amdgpu_device pointer
+ * @reg: mmio/rlc register
+ * @v: value to write
+ * @xcc_id: xcc accelerated compute core id
+ *
+ * this function is invoked only for the debugfs register access
+ */
+void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg,
+			     uint32_t v, uint32_t xcc_id)
+{
+	if (amdgpu_device_skip_hw_access(adev))
+		return;
+
+	if (amdgpu_sriov_fullaccess(adev) && adev->gfx.rlc.funcs &&
+	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
+		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
+			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
+	} else if ((reg * 4) >= adev->rmmio_size) {
+		amdgpu_reg_pcie_wr32(adev, reg * 4, v);
+	} else {
+		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
+	}
+}
+
+/**
+ * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
+ *
+ * @adev: amdgpu_device pointer
+ * @reg: dword aligned register offset
+ * @v: 32 bit value to write to the register
+ * @acc_flags: access flags which require special behavior
+ * @xcc_id: xcc accelerated compute core id
+ *
+ * Writes the value specified to the offset specified.
+ */
+void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v, uint32_t acc_flags, uint32_t xcc_id)
+{
+	uint32_t rlcg_flag;
+
+	if (amdgpu_device_skip_hw_access(adev))
+		return;
+
+	if ((reg * 4) < adev->rmmio_size) {
+		if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) &&
+		    adev->gfx.rlc.rlcg_reg_access_supported &&
+		    amdgpu_virt_get_rlcg_reg_access_flag(
+			    adev, acc_flags, GC_HWIP, true, &rlcg_flag)) {
+			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
+						GET_INST(GC, xcc_id));
+		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
+			   amdgpu_sriov_runtime(adev) &&
+			   down_read_trylock(&adev->reset_domain->sem)) {
+			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+			up_read(&adev->reset_domain->sem);
+		} else {
+			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
+		}
+	} else {
+		amdgpu_reg_pcie_wr32(adev, reg * 4, v);
+	}
+}
+
+/**
+ * amdgpu_device_indirect_rreg - read an indirect register
+ *
+ * @adev: amdgpu_device pointer
+ * @reg_addr: indirect register address to read from
+ *
+ * Returns the value of indirect register @reg_addr
+ */
+u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, u32 reg_addr)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_data_offset;
+	u32 r;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	r = readl(pcie_data_offset);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+
+	return r;
+}
+
+u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, u64 reg_addr)
+{
+	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
+	u32 r;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_index_hi_offset;
+	void __iomem *pcie_data_offset;
+
+	if (unlikely(!adev->nbio.funcs)) {
+		pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
+		pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
+	} else {
+		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+	}
+
+	if (reg_addr >> 32) {
+		if (unlikely(!adev->nbio.funcs))
+			pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
+		else
+			pcie_index_hi =
+				adev->nbio.funcs->get_pcie_index_hi_offset(
+					adev);
+	} else {
+		pcie_index_hi = 0;
+	}
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+	if (pcie_index_hi != 0)
+		pcie_index_hi_offset =
+			(void __iomem *)adev->rmmio + pcie_index_hi * 4;
+
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	r = readl(pcie_data_offset);
+
+	/* clear the high bits */
+	if (pcie_index_hi != 0) {
+		writel(0, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+
+	return r;
+}
+
+/**
+ * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
+ *
+ * @adev: amdgpu_device pointer
+ * @reg_addr: indirect register address to read from
+ *
+ * Returns the value of indirect register @reg_addr
+ */
+u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, u32 reg_addr)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_data_offset;
+	u64 r;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+
+	/* read low 32 bits */
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	r = readl(pcie_data_offset);
+	/* read high 32 bits */
+	writel(reg_addr + 4, pcie_index_offset);
+	readl(pcie_index_offset);
+	r |= ((u64)readl(pcie_data_offset) << 32);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+
+	return r;
+}
+
+u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, u64 reg_addr)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	unsigned long pcie_index_hi = 0;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_index_hi_offset;
+	void __iomem *pcie_data_offset;
+	u64 r;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
+		pcie_index_hi =
+			adev->nbio.funcs->get_pcie_index_hi_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+	if (pcie_index_hi != 0)
+		pcie_index_hi_offset =
+			(void __iomem *)adev->rmmio + pcie_index_hi * 4;
+
+	/* read low 32 bits */
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	r = readl(pcie_data_offset);
+	/* read high 32 bits */
+	writel(reg_addr + 4, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	r |= ((u64)readl(pcie_data_offset) << 32);
+
+	/* clear the high bits */
+	if (pcie_index_hi != 0) {
+		writel(0, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+
+	return r;
+}
+
+/**
+ * amdgpu_device_indirect_wreg - write an indirect register address
+ *
+ * @adev: amdgpu_device pointer
+ * @reg_addr: indirect register offset
+ * @reg_data: indirect register data
+ *
+ */
+void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, u32 reg_addr,
+				 u32 reg_data)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_data_offset;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	writel(reg_data, pcie_data_offset);
+	readl(pcie_data_offset);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+}
+
+void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, u64 reg_addr,
+				     u32 reg_data)
+{
+	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_index_hi_offset;
+	void __iomem *pcie_data_offset;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
+		pcie_index_hi =
+			adev->nbio.funcs->get_pcie_index_hi_offset(adev);
+	else
+		pcie_index_hi = 0;
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+	if (pcie_index_hi != 0)
+		pcie_index_hi_offset =
+			(void __iomem *)adev->rmmio + pcie_index_hi * 4;
+
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	writel(reg_data, pcie_data_offset);
+	readl(pcie_data_offset);
+
+	/* clear the high bits */
+	if (pcie_index_hi != 0) {
+		writel(0, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+}
+
+/**
+ * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
+ *
+ * @adev: amdgpu_device pointer
+ * @reg_addr: indirect register offset
+ * @reg_data: indirect register data
+ *
+ */
+void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, u32 reg_addr,
+				   u64 reg_data)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_data_offset;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+
+	/* write low 32 bits */
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
+	readl(pcie_data_offset);
+	/* write high 32 bits */
+	writel(reg_addr + 4, pcie_index_offset);
+	readl(pcie_index_offset);
+	writel((u32)(reg_data >> 32), pcie_data_offset);
+	readl(pcie_data_offset);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+}
+
+void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, u64 reg_addr,
+				       u64 reg_data)
+{
+	unsigned long flags, pcie_index, pcie_data;
+	unsigned long pcie_index_hi = 0;
+	void __iomem *pcie_index_offset;
+	void __iomem *pcie_index_hi_offset;
+	void __iomem *pcie_data_offset;
+
+	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
+	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
+	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
+		pcie_index_hi =
+			adev->nbio.funcs->get_pcie_index_hi_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
+	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
+	if (pcie_index_hi != 0)
+		pcie_index_hi_offset =
+			(void __iomem *)adev->rmmio + pcie_index_hi * 4;
+
+	/* write low 32 bits */
+	writel(reg_addr, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
+	readl(pcie_data_offset);
+	/* write high 32 bits */
+	writel(reg_addr + 4, pcie_index_offset);
+	readl(pcie_index_offset);
+	if (pcie_index_hi != 0) {
+		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+	writel((u32)(reg_data >> 32), pcie_data_offset);
+	readl(pcie_data_offset);
+
+	/* clear the high bits */
+	if (pcie_index_hi != 0) {
+		writel(0, pcie_index_hi_offset);
+		readl(pcie_index_hi_offset);
+	}
+
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+}
+
+u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags, address, data;
+	u32 r;
+
+	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
+	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	WREG32(address, reg * 4);
+	(void)RREG32(address);
+	r = RREG32(data);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+	return r;
+}
+
+void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags, address, data;
+
+	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
+	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
+
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
+	WREG32(address, reg * 4);
+	(void)RREG32(address);
+	WREG32(data, v);
+	(void)RREG32(data);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
+}
+
+uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
+				    uint32_t reg_addr, char reg_name[],
+				    uint32_t expected_value, uint32_t mask)
+{
+	uint32_t ret = 0;
+	uint32_t old_ = 0;
+	uint32_t tmp_ = RREG32(reg_addr);
+	uint32_t loop = adev->usec_timeout;
+
+	while ((tmp_ & (mask)) != (expected_value)) {
+		if (old_ != tmp_) {
+			loop = adev->usec_timeout;
+			old_ = tmp_;
+		} else
+			udelay(1);
+		tmp_ = RREG32(reg_addr);
+		loop--;
+		if (!loop) {
+			dev_warn(
+				adev->dev,
+				"Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
+				inst, reg_name, (uint32_t)expected_value,
+				(uint32_t)(tmp_ & (mask)));
+			ret = -ETIMEDOUT;
+			break;
+		}
+	}
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
new file mode 100644
index 0000000000000..4d88e5cd19fc9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_REG_ACCESS_H__
+#define __AMDGPU_REG_ACCESS_H__
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+
+#include "amdgpu_ip.h"
+
+struct amdgpu_device;
+
+typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device *, uint32_t);
+typedef void (*amdgpu_wreg_t)(struct amdgpu_device *, uint32_t, uint32_t);
+typedef uint32_t (*amdgpu_rreg_ext_t)(struct amdgpu_device *, uint64_t);
+typedef void (*amdgpu_wreg_ext_t)(struct amdgpu_device *, uint64_t, uint32_t);
+typedef uint64_t (*amdgpu_rreg64_t)(struct amdgpu_device *, uint32_t);
+typedef void (*amdgpu_wreg64_t)(struct amdgpu_device *, uint32_t, uint64_t);
+typedef uint64_t (*amdgpu_rreg64_ext_t)(struct amdgpu_device *, uint64_t);
+typedef void (*amdgpu_wreg64_ext_t)(struct amdgpu_device *, uint64_t, uint64_t);
+
+typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device *, uint32_t,
+					uint32_t);
+typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device *, uint32_t, uint32_t,
+				    uint32_t);
+typedef uint64_t (*amdgpu_reg_get_smn_base64_t)(struct amdgpu_device *adev,
+					 enum amd_hw_ip_block_type block,
+					 int die_inst);
+
+struct amdgpu_reg_ind {
+	spinlock_t lock;
+	amdgpu_rreg_t rreg;
+	amdgpu_wreg_t wreg;
+};
+
+struct amdgpu_reg_ind_blk {
+	spinlock_t lock;
+	amdgpu_block_rreg_t rreg;
+	amdgpu_block_wreg_t wreg;
+};
+
+struct amdgpu_reg_pcie_ind {
+	spinlock_t lock;
+	amdgpu_rreg_t rreg;
+	amdgpu_wreg_t wreg;
+	amdgpu_rreg_ext_t rreg_ext;
+	amdgpu_wreg_ext_t wreg_ext;
+	amdgpu_rreg64_t rreg64;
+	amdgpu_wreg64_t wreg64;
+	amdgpu_rreg64_ext_t rreg64_ext;
+	amdgpu_wreg64_ext_t wreg64_ext;
+	amdgpu_rreg_t port_rreg;
+	amdgpu_wreg_t port_wreg;
+};
+
+struct amdgpu_reg_smn_ext {
+	amdgpu_reg_get_smn_base64_t get_smn_base;
+};
+
+struct amdgpu_reg_access {
+	struct amdgpu_reg_ind smc;
+	struct amdgpu_reg_ind uvd_ctx;
+	struct amdgpu_reg_ind didt;
+	struct amdgpu_reg_ind gc_cac;
+	struct amdgpu_reg_ind se_cac;
+	struct amdgpu_reg_ind_blk audio_endpt;
+	struct amdgpu_reg_pcie_ind pcie;
+	struct amdgpu_reg_smn_ext smn;
+};
+
+void amdgpu_reg_access_init(struct amdgpu_device *adev);
+uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+uint32_t amdgpu_reg_uvd_ctx_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_uvd_ctx_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+uint32_t amdgpu_reg_didt_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_didt_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+uint32_t amdgpu_reg_gc_cac_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_gc_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v);
+uint32_t amdgpu_reg_se_cac_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_se_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v);
+uint32_t amdgpu_reg_audio_endpt_rd32(struct amdgpu_device *adev, uint32_t block,
+				     uint32_t reg);
+void amdgpu_reg_audio_endpt_wr32(struct amdgpu_device *adev, uint32_t block,
+				 uint32_t reg, uint32_t v);
+uint32_t amdgpu_reg_pcie_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_pcie_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+uint32_t amdgpu_reg_pcie_ext_rd32(struct amdgpu_device *adev, uint64_t reg);
+void amdgpu_reg_pcie_ext_wr32(struct amdgpu_device *adev, uint64_t reg,
+			      uint32_t v);
+uint64_t amdgpu_reg_pcie_rd64(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_pcie_wr64(struct amdgpu_device *adev, uint32_t reg, uint64_t v);
+uint64_t amdgpu_reg_pcie_ext_rd64(struct amdgpu_device *adev, uint64_t reg);
+void amdgpu_reg_pcie_ext_wr64(struct amdgpu_device *adev, uint64_t reg,
+			      uint64_t v);
+uint32_t amdgpu_reg_pciep_rd32(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_reg_pciep_wr32(struct amdgpu_device *adev, uint32_t reg,
+			   uint32_t v);
+uint64_t amdgpu_reg_get_smn_base64(struct amdgpu_device *adev,
+				   enum amd_hw_ip_block_type block,
+				   int die_inst);
+uint64_t amdgpu_reg_smn_v1_0_get_base(struct amdgpu_device *adev,
+				      enum amd_hw_ip_block_type block,
+				      int die_inst);
+
+uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t acc_flags);
+uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, uint32_t reg,
+				uint32_t acc_flags, uint32_t xcc_id);
+void amdgpu_device_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
+			uint32_t acc_flags);
+void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, uint32_t reg,
+			    uint32_t v, uint32_t acc_flags, uint32_t xcc_id);
+void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg,
+			     uint32_t v, uint32_t xcc_id);
+void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset,
+		     uint8_t value);
+uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
+
+u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, u32 reg_addr);
+u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, u64 reg_addr);
+u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, u32 reg_addr);
+u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, u64 reg_addr);
+void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, u32 reg_addr,
+				 u32 reg_data);
+void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, u64 reg_addr,
+				     u32 reg_data);
+void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, u32 reg_addr,
+				   u64 reg_data);
+void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, u64 reg_addr,
+				       u64 reg_data);
+
+u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg);
+void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v);
+
+uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
+				    uint32_t reg_addr, char reg_name[],
+				    uint32_t expected_value, uint32_t mask);
+
+#endif /* __AMDGPU_REG_ACCESS_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
index be2e56ce1355e..8908d9e08a304 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
@@ -55,7 +55,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
 				    uint64_t start, uint64_t size,
 				    struct amdgpu_res_cursor *cur)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	struct list_head *head, *next;
 	struct drm_mm_node *node;
 
@@ -71,7 +71,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
 		head = &to_amdgpu_vram_mgr_resource(res)->blocks;
 
 		block = list_first_entry_or_null(head,
-						 struct drm_buddy_block,
+						 struct gpu_buddy_block,
 						 link);
 		if (!block)
 			goto fallback;
@@ -81,7 +81,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
 
 			next = block->link.next;
 			if (next != head)
-				block = list_entry(next, struct drm_buddy_block, link);
+				block = list_entry(next, struct gpu_buddy_block, link);
 		}
 
 		cur->start = amdgpu_vram_mgr_block_start(block) + start;
@@ -125,7 +125,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
  */
 static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	struct drm_mm_node *node;
 	struct list_head *next;
 
@@ -146,7 +146,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
 		block = cur->node;
 
 		next = block->link.next;
-		block = list_entry(next, struct drm_buddy_block, link);
+		block = list_entry(next, struct gpu_buddy_block, link);
 
 		cur->node = block;
 		cur->start = amdgpu_vram_mgr_block_start(block);
@@ -175,7 +175,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
  */
 static inline bool amdgpu_res_cleared(struct amdgpu_res_cursor *cur)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 
 	switch (cur->mem_type) {
 	case TTM_PL_VRAM:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 4638a686a84e7..66e8a2f7afcf6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -90,10 +90,13 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)
 	ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
 
 	/* Make sure we aren't trying to allocate more space
-	 * than the maximum for one submission
+	 * than the maximum for one submission.  Skip for reemit
+	 * since we may be reemitting several submissions.
 	 */
-	if (WARN_ON_ONCE(ndw > ring->max_dw))
-		return -ENOMEM;
+	if (!ring->reemit) {
+		if (WARN_ON_ONCE(ndw > ring->max_dw))
+			return -ENOMEM;
+	}
 
 	ring->count_dw = ndw;
 	ring->wptr_old = ring->wptr;
@@ -104,29 +107,6 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)
 	return 0;
 }
 
-/**
- * amdgpu_ring_alloc_reemit - allocate space on the ring buffer for reemit
- *
- * @ring: amdgpu_ring structure holding ring information
- * @ndw: number of dwords to allocate in the ring buffer
- *
- * Allocate @ndw dwords in the ring buffer (all asics).
- * doesn't check the max_dw limit as we may be reemitting
- * several submissions.
- */
-static void amdgpu_ring_alloc_reemit(struct amdgpu_ring *ring, unsigned int ndw)
-{
-	/* Align requested size with padding so unlock_commit can
-	 * pad safely */
-	ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
-
-	ring->count_dw = ndw;
-	ring->wptr_old = ring->wptr;
-
-	if (ring->funcs->begin_use)
-		ring->funcs->begin_use(ring);
-}
-
 /**
  * amdgpu_ring_insert_nop - insert NOP packets
  *
@@ -479,10 +459,10 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
 	if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
 		return false;
 
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	if (!dma_fence_is_signaled_locked(fence))
 		dma_fence_set_error(fence, -ENODATA);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	while (!dma_fence_is_signaled(fence) &&
 	       ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0)
@@ -875,7 +855,6 @@ void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring,
 int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
 				 struct amdgpu_fence *guilty_fence)
 {
-	unsigned int i;
 	int r;
 
 	/* verify that the ring is functional */
@@ -883,16 +862,9 @@ int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
 	if (r)
 		return r;
 
-	/* set an error on all fences from the context */
-	if (guilty_fence)
-		amdgpu_fence_driver_update_timedout_fence_state(guilty_fence);
-	/* Re-emit the non-guilty commands */
-	if (ring->ring_backup_entries_to_copy) {
-		amdgpu_ring_alloc_reemit(ring, ring->ring_backup_entries_to_copy);
-		for (i = 0; i < ring->ring_backup_entries_to_copy; i++)
-			amdgpu_ring_write(ring, ring->ring_backup[i]);
-		amdgpu_ring_commit(ring);
-	}
+	/* set an error on all fences from the context and reemit */
+	amdgpu_ring_set_fence_errors_and_reemit(ring, guilty_fence);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index cb0fb1a989d2f..ce5af137ee400 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -121,7 +121,6 @@ struct amdgpu_fence_driver {
 	/* sync_seq is protected by ring emission lock */
 	uint32_t			sync_seq;
 	atomic_t			last_seq;
-	u64				signalled_wptr;
 	bool				initialized;
 	struct amdgpu_irq_src		*irq_src;
 	unsigned			irq_type;
@@ -146,23 +145,23 @@ struct amdgpu_fence {
 	struct amdgpu_ring		*ring;
 	ktime_t				start_timestamp;
 
-	/* wptr for the total submission for resets */
-	u64				wptr;
+	/* location and size of the IB */
+	u64				ib_wptr;
+	unsigned int			ib_dw_size;
+	unsigned int			skip_ib_dw_start_offset;
+	unsigned int			skip_ib_dw_end_offset;
 	/* fence context for resets */
 	u64				context;
-	/* has this fence been reemitted */
-	unsigned int			reemitted;
-	/* wptr for the fence for the submission */
-	u64				fence_wptr_start;
-	u64				fence_wptr_end;
+	/* idx for ring backups */
+	unsigned int			backup_idx;
 };
 
 extern const struct drm_sched_backend_ops amdgpu_sched_ops;
 
 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error);
 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
-void amdgpu_fence_driver_update_timedout_fence_state(struct amdgpu_fence *af);
-void amdgpu_fence_save_wptr(struct amdgpu_fence *af);
+void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
+					     struct amdgpu_fence *guilty_fence);
 
 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);
 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
@@ -172,8 +171,8 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev);
 int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
-		      unsigned int flags);
+void amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
+		       unsigned int flags);
 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
 			      uint32_t timeout);
 bool amdgpu_fence_process(struct amdgpu_ring *ring);
@@ -313,6 +312,7 @@ struct amdgpu_ring {
 	/* backups for resets */
 	uint32_t		*ring_backup;
 	unsigned int		ring_backup_entries_to_copy;
+	bool			reemit;
 	unsigned		rptr_offs;
 	u64			rptr_gpu_addr;
 	u32			*rptr_cpu_addr;
@@ -522,6 +522,17 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring,
 	ring->count_dw -= count_dw;
 }
 
+static inline unsigned int amdgpu_ring_get_dw_distance(struct amdgpu_ring *ring,
+						       u64 start_wptr, u64 end_wptr)
+{
+	unsigned int start = start_wptr & ring->buf_mask;
+	unsigned int end = end_wptr & ring->buf_mask;
+
+	if (end < start)
+		end += ring->ring_size >> 2;
+	return end - start;
+}
+
 /**
  * amdgpu_ring_patch_cond_exec - patch dw count of conditional execute
  * @ring: amdgpu_ring structure
@@ -532,18 +543,13 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring,
 static inline void amdgpu_ring_patch_cond_exec(struct amdgpu_ring *ring,
 					       unsigned int offset)
 {
-	unsigned cur;
-
 	if (!ring->funcs->init_cond_exec)
 		return;
 
 	WARN_ON(offset > ring->buf_mask);
 	WARN_ON(ring->ring[offset] != 0);
 
-	cur = (ring->wptr - 1) & ring->buf_mask;
-	if (cur < offset)
-		cur += ring->ring_size >> 2;
-	ring->ring[offset] = cur - offset;
+	ring->ring[offset] = amdgpu_ring_get_dw_distance(ring, offset, ring->wptr - 1);
 }
 
 int amdgpu_ring_test_helper(struct amdgpu_ring *ring);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index eeaa56c8d129f..b4ab309bf08ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -163,7 +163,8 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 }
 
 static struct dma_fence *
-amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_job *job, u32 num_dw)
+amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity,
+		      struct amdgpu_job *job, u32 num_dw)
 {
 	struct amdgpu_ring *ring;
 
@@ -171,6 +172,8 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_job *job, u32 nu
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
 
+	lockdep_assert_held(&entity->lock);
+
 	return amdgpu_job_submit(job);
 }
 
@@ -228,9 +231,7 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 
 	*size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset);
 
-	*addr = adev->gmc.gart_start;
-	*addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
-		AMDGPU_GPU_PAGE_SIZE;
+	*addr = amdgpu_compute_gart_address(&adev->gmc, entity, window);
 	*addr += offset;
 
 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
@@ -248,7 +249,7 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 	src_addr += job->ibs[0].gpu_addr;
 
 	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
-	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
+	dst_addr += (entity->gart_window_offs[window] >> AMDGPU_GPU_PAGE_SHIFT) * 8;
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
 				dst_addr, num_bytes, 0);
 
@@ -269,7 +270,7 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 		amdgpu_gart_map_vram_range(adev, pa, 0, num_pages, flags, cpu_addr);
 	}
 
-	dma_fence_put(amdgpu_ttm_job_submit(adev, job, num_dw));
+	dma_fence_put(amdgpu_ttm_job_submit(adev, entity, job, num_dw));
 	return 0;
 }
 
@@ -313,7 +314,7 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 	amdgpu_res_first(src->mem, src->offset, size, &src_mm);
 	amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm);
 
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&entity->lock);
 	while (src_mm.remaining) {
 		uint64_t from, to, cur_size, tiling_flags;
 		uint32_t num_type, data_format, max_com, write_compress_disable;
@@ -368,7 +369,7 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 		amdgpu_res_next(&dst_mm, cur_size);
 	}
 error:
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&entity->lock);
 	*f = fence;
 	return r;
 }
@@ -1580,7 +1581,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	if (r)
 		goto out;
 
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&adev->mman.default_entity.lock);
 	amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm);
 	src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) +
 		src_mm.start;
@@ -1591,8 +1592,8 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
 				PAGE_SIZE, 0);
 
-	fence = amdgpu_ttm_job_submit(adev, job, num_dw);
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	fence = amdgpu_ttm_job_submit(adev, &adev->mman.default_entity, job, num_dw);
+	mutex_unlock(&adev->mman.default_entity.lock);
 
 	if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout))
 		r = -ETIMEDOUT;
@@ -1908,7 +1909,7 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton MMIO_REMAP BO
+ * amdgpu_ttm_alloc_mmio_remap_bo - Allocate the singleton MMIO_REMAP BO
  * @adev: amdgpu device
  *
  * Allocates a global BO with backing AMDGPU_PL_MMIO_REMAP when the
@@ -2013,6 +2014,50 @@ static void amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device *adev)
 	adev->rmmio_remap.bo = NULL;
 }
 
+static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr,
+					 struct amdgpu_ttm_buffer_entity *entity,
+					 enum drm_sched_priority prio,
+					 struct drm_gpu_scheduler **scheds,
+					 int num_schedulers,
+					 u32 num_gart_windows)
+{
+	int i, r, num_pages;
+
+	r = drm_sched_entity_init(&entity->base, prio, scheds, num_schedulers, NULL);
+	if (r)
+		return r;
+
+	mutex_init(&entity->lock);
+
+	if (ARRAY_SIZE(entity->gart_window_offs) < num_gart_windows)
+		return -EINVAL;
+	if (num_gart_windows == 0)
+		return 0;
+
+	num_pages = num_gart_windows * AMDGPU_GTT_MAX_TRANSFER_SIZE;
+	r = amdgpu_gtt_mgr_alloc_entries(mgr, &entity->gart_node, num_pages,
+					 DRM_MM_INSERT_BEST);
+	if (r) {
+		drm_sched_entity_destroy(&entity->base);
+		return r;
+	}
+
+	for (i = 0; i < num_gart_windows; i++) {
+		entity->gart_window_offs[i] =
+			amdgpu_gtt_node_to_byte_offset(&entity->gart_node) +
+				i * AMDGPU_GTT_MAX_TRANSFER_SIZE * PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+static void amdgpu_ttm_buffer_entity_fini(struct amdgpu_gtt_mgr *mgr,
+					  struct amdgpu_ttm_buffer_entity *entity)
+{
+	amdgpu_gtt_mgr_free_entries(mgr, &entity->gart_node);
+	drm_sched_entity_destroy(&entity->base);
+}
+
 /*
  * amdgpu_ttm_init - Init the memory management (ttm) as well as various
  * gtt/vram related fields.
@@ -2027,8 +2072,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	uint64_t gtt_size;
 	int r;
 
-	mutex_init(&adev->mman.gtt_window_lock);
-
 	dma_set_max_seg_size(adev->dev, UINT_MAX);
 	/* No others user of address space so set it to 0 */
 	r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,
@@ -2313,42 +2356,49 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 		struct amdgpu_ring *ring;
 		struct drm_gpu_scheduler *sched;
 
+		if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) {
+			dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
+			return;
+		}
+
 		ring = adev->mman.buffer_funcs_ring;
 		sched = &ring->sched;
-		r = drm_sched_entity_init(&adev->mman.default_entity.base,
-					  DRM_SCHED_PRIORITY_KERNEL, &sched,
-					  1, NULL);
-		if (r) {
+		r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
+						  &adev->mman.default_entity,
+						  DRM_SCHED_PRIORITY_KERNEL,
+						  &sched, 1, 0);
+		if (r < 0) {
 			dev_err(adev->dev,
-				"Failed setting up TTM BO move entity (%d)\n",
-				r);
+				"Failed setting up TTM entity (%d)\n", r);
 			return;
 		}
 
-		r = drm_sched_entity_init(&adev->mman.clear_entity.base,
-					  DRM_SCHED_PRIORITY_NORMAL, &sched,
-					  1, NULL);
-		if (r) {
+		r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
+						  &adev->mman.clear_entity,
+						  DRM_SCHED_PRIORITY_NORMAL,
+						  &sched, 1, 1);
+		if (r < 0) {
 			dev_err(adev->dev,
-				"Failed setting up TTM BO clear entity (%d)\n",
-				r);
-			goto error_free_entity;
+				"Failed setting up TTM BO clear entity (%d)\n", r);
+			goto error_free_default_entity;
 		}
 
-		r = drm_sched_entity_init(&adev->mman.move_entity.base,
-					  DRM_SCHED_PRIORITY_NORMAL, &sched,
-					  1, NULL);
-		if (r) {
+		r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
+						  &adev->mman.move_entity,
+						  DRM_SCHED_PRIORITY_NORMAL,
+						  &sched, 1, 2);
+		if (r < 0) {
 			dev_err(adev->dev,
-				"Failed setting up TTM BO move entity (%d)\n",
-				r);
-			drm_sched_entity_destroy(&adev->mman.clear_entity.base);
-			goto error_free_entity;
+				"Failed setting up TTM BO move entity (%d)\n", r);
+			goto error_free_clear_entity;
 		}
 	} else {
-		drm_sched_entity_destroy(&adev->mman.default_entity.base);
-		drm_sched_entity_destroy(&adev->mman.clear_entity.base);
-		drm_sched_entity_destroy(&adev->mman.move_entity.base);
+		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+					      &adev->mman.default_entity);
+		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+					      &adev->mman.clear_entity);
+		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+					      &adev->mman.move_entity);
 		/* Drop all the old fences since re-creating the scheduler entities
 		 * will allocate new contexts.
 		 */
@@ -2365,8 +2415,12 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 
 	return;
 
-error_free_entity:
-	drm_sched_entity_destroy(&adev->mman.default_entity.base);
+error_free_clear_entity:
+	amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+				      &adev->mman.clear_entity);
+error_free_default_entity:
+	amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+				      &adev->mman.default_entity);
 }
 
 static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
@@ -2440,7 +2494,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
 		byte_count -= cur_size_in_bytes;
 	}
 
-	*fence = amdgpu_ttm_job_submit(adev, job, num_dw);
+	*fence = amdgpu_ttm_job_submit(adev, entity, job, num_dw);
 
 	return 0;
 
@@ -2483,7 +2537,7 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
 		byte_count -= cur_size;
 	}
 
-	*fence = amdgpu_ttm_job_submit(adev, job, num_dw);
+	*fence = amdgpu_ttm_job_submit(adev, entity, job, num_dw);
 	return 0;
 }
 
@@ -2503,6 +2557,7 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 			    struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct amdgpu_ttm_buffer_entity *entity;
 	struct amdgpu_res_cursor cursor;
 	u64 addr;
 	int r = 0;
@@ -2513,11 +2568,12 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 	if (!fence)
 		return -EINVAL;
 
+	entity = &adev->mman.clear_entity;
 	*fence = dma_fence_get_stub();
 
 	amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
 
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&entity->lock);
 	while (cursor.remaining) {
 		struct dma_fence *next = NULL;
 		u64 size;
@@ -2530,13 +2586,12 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 		/* Never clear more than 256MiB at once to avoid timeouts */
 		size = min(cursor.size, 256ULL << 20);
 
-		r = amdgpu_ttm_map_buffer(&adev->mman.clear_entity,
-					  &bo->tbo, bo->tbo.resource, &cursor,
-					  1, false, &size, &addr);
+		r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &cursor,
+					  0, false, &size, &addr);
 		if (r)
 			goto err;
 
-		r = amdgpu_ttm_fill_mem(adev, &adev->mman.clear_entity, 0, addr, size, resv,
+		r = amdgpu_ttm_fill_mem(adev, entity, 0, addr, size, resv,
 					&next, true,
 					AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
 		if (r)
@@ -2548,7 +2603,7 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 		amdgpu_res_next(&cursor, size);
 	}
 err:
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&entity->lock);
 
 	return r;
 }
@@ -2573,7 +2628,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 
 	amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst);
 
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&entity->lock);
 	while (dst.remaining) {
 		struct dma_fence *next;
 		uint64_t cur_size, to;
@@ -2582,7 +2637,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 		cur_size = min(dst.size, 256ULL << 20);
 
 		r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst,
-					  1, false, &cur_size, &to);
+					  0, false, &cur_size, &to);
 		if (r)
 			goto error;
 
@@ -2598,7 +2653,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 		amdgpu_res_next(&dst, cur_size);
 	}
 error:
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&entity->lock);
 	if (f)
 		*f = dma_fence_get(fence);
 	dma_fence_put(fence);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 143201ecea3fe..bf101215757e7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -29,6 +29,7 @@
 #include <drm/ttm/ttm_placement.h>
 #include "amdgpu_vram_mgr.h"
 #include "amdgpu_hmm.h"
+#include "amdgpu_gmc.h"
 
 #define AMDGPU_PL_GDS		(TTM_PL_PRIV + 0)
 #define AMDGPU_PL_GWS		(TTM_PL_PRIV + 1)
@@ -38,8 +39,7 @@
 #define AMDGPU_PL_MMIO_REMAP	(TTM_PL_PRIV + 5)
 #define __AMDGPU_PL_NUM	(TTM_PL_PRIV + 6)
 
-#define AMDGPU_GTT_MAX_TRANSFER_SIZE	512
-#define AMDGPU_GTT_NUM_TRANSFER_WINDOWS	2
+#define AMDGPU_GTT_MAX_TRANSFER_SIZE	1024
 
 extern const struct attribute_group amdgpu_vram_mgr_attr_group;
 extern const struct attribute_group amdgpu_gtt_mgr_attr_group;
@@ -54,6 +54,9 @@ struct amdgpu_gtt_mgr {
 
 struct amdgpu_ttm_buffer_entity {
 	struct drm_sched_entity base;
+	struct mutex		lock;
+	struct drm_mm_node	gart_node;
+	u64			gart_window_offs[2];
 };
 
 struct amdgpu_mman {
@@ -67,8 +70,7 @@ struct amdgpu_mman {
 	struct amdgpu_ring			*buffer_funcs_ring;
 	bool					buffer_funcs_enabled;
 
-	struct mutex				gtt_window_lock;
-
+	/* @default_entity: for workarounds, has no gart windows */
 	struct amdgpu_ttm_buffer_entity default_entity;
 	struct amdgpu_ttm_buffer_entity clear_entity;
 	struct amdgpu_ttm_buffer_entity move_entity;
@@ -205,6 +207,27 @@ static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
 }
 #endif
 
+/**
+ * amdgpu_compute_gart_address() - Returns GART address of an entity's window
+ * @gmc: The &struct amdgpu_gmc instance to use
+ * @entity: The &struct amdgpu_ttm_buffer_entity owning the GART window
+ * @index: The window to use (must be 0 or 1)
+ */
+static inline u64 amdgpu_compute_gart_address(struct amdgpu_gmc *gmc,
+					      struct amdgpu_ttm_buffer_entity *entity,
+					      int index)
+{
+	return gmc->gart_start + entity->gart_window_offs[index];
+}
+
+/**
+ * amdgpu_gtt_node_to_byte_offset() - Returns a byte offset of a gtt node
+ */
+static inline u64 amdgpu_gtt_node_to_byte_offset(const struct drm_mm_node *gtt_node)
+{
+	return gtt_node->start * (u64)PAGE_SIZE;
+}
+
 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct amdgpu_hmm_range *range);
 int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
 			      uint64_t *user_addr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 1ab61e7b35db6..6d9e96fabd581 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -291,22 +291,22 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 			break;
 		case 5:
 			/* rlc_hdr v2_5 */
-			DRM_INFO("rlc_iram_ucode_size_bytes: %u\n",
+			DRM_DEBUG("rlc_iram_ucode_size_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_iram_ucode_size_bytes));
-			DRM_INFO("rlc_iram_ucode_offset_bytes: %u\n",
+			DRM_DEBUG("rlc_iram_ucode_offset_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_iram_ucode_offset_bytes));
-			DRM_INFO("rlc_dram_ucode_size_bytes: %u\n",
+			DRM_DEBUG("rlc_dram_ucode_size_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_dram_ucode_size_bytes));
-			DRM_INFO("rlc_dram_ucode_offset_bytes: %u\n",
+			DRM_DEBUG("rlc_dram_ucode_offset_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_dram_ucode_offset_bytes));
 			/* rlc_hdr v2_5 */
-			DRM_INFO("rlc_1_iram_ucode_size_bytes: %u\n",
+			DRM_DEBUG("rlc_1_iram_ucode_size_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_iram_ucode_size_bytes));
-			DRM_INFO("rlc_1_iram_ucode_offset_bytes: %u\n",
+			DRM_DEBUG("rlc_1_iram_ucode_offset_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_iram_ucode_offset_bytes));
-			DRM_INFO("rlc_1_dram_ucode_size_bytes: %u\n",
+			DRM_DEBUG("rlc_1_dram_ucode_size_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_dram_ucode_size_bytes));
-			DRM_INFO("rlc_1_dram_ucode_offset_bytes: %u\n",
+			DRM_DEBUG("rlc_1_dram_ucode_offset_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_dram_ucode_offset_bytes));
 			break;
 		default:
@@ -1150,7 +1150,7 @@ int amdgpu_ucode_create_bo(struct amdgpu_device *adev)
 	if ((adev->firmware.load_type != AMDGPU_FW_LOAD_DIRECT) &&
 	    (adev->firmware.load_type != AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)) {
 		amdgpu_bo_create_kernel(adev, adev->firmware.fw_size, PAGE_SIZE,
-			(amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf) ?
+			(amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf || adev->gmc.xgmi.connected_to_cpu) ?
 			AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 			&adev->firmware.fw_buf,
 			&adev->firmware.fw_buf_mc,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 4df0f9d5ad11b..0238c2798de4e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -398,6 +398,17 @@ int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+int amdgpu_umc_uniras_process_ecc_irq(struct amdgpu_device *adev,
+			struct amdgpu_irq_src *source,
+			struct amdgpu_iv_entry *entry)
+{
+	struct ras_ih_info ih_info = {0};
+
+	ih_info.block = RAS_BLOCK_ID__UMC;
+	amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
+	return 0;
+}
+
 int amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 		uint64_t err_addr,
 		uint64_t retired_page,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 28dff750c47e3..8494a55ebf76c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -161,6 +161,9 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
 int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
 		struct amdgpu_irq_src *source,
 		struct amdgpu_iv_entry *entry);
+int amdgpu_umc_uniras_process_ecc_irq(struct amdgpu_device *adev,
+		struct amdgpu_irq_src *source,
+		struct amdgpu_iv_entry *entry);
 int amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 		uint64_t err_addr,
 		uint64_t retired_page,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 7c450350847d6..d94f4966fea9b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -156,7 +156,7 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 	struct dma_fence *fence;
 	struct amdgpu_userq_mgr *uq_mgr;
 
-	if (!queue || !queue->userq_mgr)
+	if (!queue->userq_mgr)
 		return;
 
 	uq_mgr = queue->userq_mgr;
@@ -472,17 +472,16 @@ void
 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
 			     struct amdgpu_eviction_fence_mgr *evf_mgr)
 {
-	struct amdgpu_eviction_fence *ev_fence;
+	struct dma_fence *ev_fence;
 
 retry:
 	/* Flush any pending resume work to create ev_fence */
 	flush_delayed_work(&uq_mgr->resume_work);
 
 	mutex_lock(&uq_mgr->userq_mutex);
-	spin_lock(&evf_mgr->ev_fence_lock);
-	ev_fence = evf_mgr->ev_fence;
-	spin_unlock(&evf_mgr->ev_fence_lock);
-	if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) {
+	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
+	if (dma_fence_is_signaled(ev_fence)) {
+		dma_fence_put(ev_fence);
 		mutex_unlock(&uq_mgr->userq_mutex);
 		/*
 		 * Looks like there was no pending resume work,
@@ -491,6 +490,7 @@ amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
 		schedule_delayed_work(&uq_mgr->resume_work, 0);
 		goto retry;
 	}
+	dma_fence_put(ev_fence);
 }
 
 int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
@@ -623,13 +623,14 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	int r = 0;
 
 	cancel_delayed_work_sync(&uq_mgr->resume_work);
+
+	/* Cancel any pending hang detection work and cleanup */
+	cancel_delayed_work_sync(&queue->hang_detect_work);
+
 	mutex_lock(&uq_mgr->userq_mutex);
+	queue->hang_detect_fence = NULL;
 	amdgpu_userq_wait_for_last_fence(queue);
-	/* Cancel any pending hang detection work and cleanup */
-	if (queue->hang_detect_fence) {
-		cancel_delayed_work_sync(&queue->hang_detect_work);
-		queue->hang_detect_fence = NULL;
-	}
+
 	r = amdgpu_bo_reserve(queue->db_obj.obj, true);
 	if (!r) {
 		amdgpu_bo_unpin(queue->db_obj.obj);
@@ -709,46 +710,6 @@ static int amdgpu_userq_priority_permit(struct drm_file *filp,
 	return -EACCES;
 }
 
-#if defined(CONFIG_DEBUG_FS)
-static int amdgpu_mqd_info_read(struct seq_file *m, void *unused)
-{
-	struct amdgpu_usermode_queue *queue = m->private;
-	struct amdgpu_bo *bo;
-	int r;
-
-	if (!queue || !queue->mqd.obj)
-		return -EINVAL;
-
-	bo = amdgpu_bo_ref(queue->mqd.obj);
-	r = amdgpu_bo_reserve(bo, true);
-	if (r) {
-		amdgpu_bo_unref(&bo);
-		return -EINVAL;
-	}
-
-	seq_printf(m, "queue_type: %d\n", queue->queue_type);
-	seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));
-
-	amdgpu_bo_unreserve(bo);
-	amdgpu_bo_unref(&bo);
-
-	return 0;
-}
-
-static int amdgpu_mqd_info_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, amdgpu_mqd_info_read, inode->i_private);
-}
-
-static const struct file_operations amdgpu_mqd_info_fops = {
-	.owner = THIS_MODULE,
-	.open = amdgpu_mqd_info_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-#endif
-
 static int
 amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 {
@@ -758,7 +719,6 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	const struct amdgpu_userq_funcs *uq_funcs;
 	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_db_info db_info;
-	char *queue_name;
 	bool skip_map_queue;
 	u32 qid;
 	uint64_t index;
@@ -819,17 +779,15 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	    amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
 	    amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
 		r = -EINVAL;
-		kfree(queue);
-		goto unlock;
+		goto free_queue;
 	}
 
 	/* Convert relative doorbell offset into absolute doorbell index */
 	index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
 	if (index == (uint64_t)-EINVAL) {
 		drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
-		kfree(queue);
 		r = -EINVAL;
-		goto unlock;
+		goto free_queue;
 	}
 
 	queue->doorbell_index = index;
@@ -837,41 +795,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	r = amdgpu_userq_fence_driver_alloc(adev, queue);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
-		goto unlock;
+		goto free_queue;
 	}
 
 	r = uq_funcs->mqd_create(queue, &args->in);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
-		amdgpu_userq_fence_driver_free(queue);
-		kfree(queue);
-		goto unlock;
-	}
-
-	/* drop this refcount during queue destroy */
-	kref_init(&queue->refcount);
-
-	/* Wait for mode-1 reset to complete */
-	down_read(&adev->reset_domain->sem);
-	r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
-	if (r) {
-		kfree(queue);
-		up_read(&adev->reset_domain->sem);
-		goto unlock;
-	}
-
-	r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
-		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
-	if (r) {
-		drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n");
-		amdgpu_userq_fence_driver_free(queue);
-		uq_funcs->mqd_destroy(queue);
-		kfree(queue);
-		r = -ENOMEM;
-		up_read(&adev->reset_domain->sem);
-		goto unlock;
+		goto clean_fence_driver;
 	}
-	up_read(&adev->reset_domain->sem);
 
 	/* don't map the queue if scheduling is halted */
 	if (adev->userq_halt_for_enforce_isolation &&
@@ -884,31 +815,52 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		r = amdgpu_userq_map_helper(queue);
 		if (r) {
 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
-			xa_erase(&uq_mgr->userq_xa, qid);
-			amdgpu_userq_fence_driver_free(queue);
-			uq_funcs->mqd_destroy(queue);
-			kfree(queue);
-			goto unlock;
+			down_read(&adev->reset_domain->sem);
+			goto clean_mqd;
 		}
 	}
 
-	queue_name = kasprintf(GFP_KERNEL, "queue-%d", qid);
-	if (!queue_name) {
+	/* drop this refcount during queue destroy */
+	kref_init(&queue->refcount);
+
+	/* Wait for mode-1 reset to complete */
+	down_read(&adev->reset_domain->sem);
+
+	r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
+		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
+	if (r) {
+		if (!skip_map_queue)
+			amdgpu_userq_unmap_helper(queue);
+
 		r = -ENOMEM;
-		goto unlock;
+		goto clean_mqd;
 	}
 
-#if defined(CONFIG_DEBUG_FS)
-	/* Queue dentry per client to hold MQD information   */
-	queue->debugfs_queue = debugfs_create_dir(queue_name, filp->debugfs_client);
-	debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
-#endif
+	r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
+	if (r) {
+		xa_erase(&uq_mgr->userq_xa, qid);
+		if (!skip_map_queue)
+			amdgpu_userq_unmap_helper(queue);
+
+		goto clean_mqd;
+	}
+	up_read(&adev->reset_domain->sem);
+
+	amdgpu_debugfs_userq_init(filp, queue, qid);
 	amdgpu_userq_init_hang_detect_work(queue);
-	kfree(queue_name);
 
 	args->out.queue_id = qid;
 	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
+	mutex_unlock(&uq_mgr->userq_mutex);
+	return 0;
 
+clean_mqd:
+	uq_funcs->mqd_destroy(queue);
+	up_read(&adev->reset_domain->sem);
+clean_fence_driver:
+	amdgpu_userq_fence_driver_free(queue);
+free_queue:
+	kfree(queue);
 unlock:
 	mutex_unlock(&uq_mgr->userq_mutex);
 
@@ -1245,7 +1197,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 		dma_fence_wait(bo_va->last_pt_update, false);
 	dma_fence_wait(vm->last_update, false);
 
-	ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec);
+	ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
 	if (ret)
 		drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
 
@@ -1265,11 +1217,13 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
 {
 	struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
 	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
+	struct dma_fence *ev_fence;
 	int ret;
 
-	flush_delayed_work(&fpriv->evf_mgr.suspend_work);
-
 	mutex_lock(&uq_mgr->userq_mutex);
+	ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr);
+	if (!dma_fence_is_signaled(ev_fence))
+		goto unlock;
 
 	ret = amdgpu_userq_vm_validate(uq_mgr);
 	if (ret) {
@@ -1285,6 +1239,7 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
 
 unlock:
 	mutex_unlock(&uq_mgr->userq_mutex);
+	dma_fence_put(ev_fence);
 }
 
 static int
@@ -1360,11 +1315,8 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 }
 
 void
-amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
-		   struct amdgpu_eviction_fence *ev_fence)
+amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, bool schedule_resume)
 {
-	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
-	struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	int ret;
 
@@ -1377,16 +1329,8 @@ amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
 	if (ret)
 		dev_err(adev->dev, "Failed to evict userqueue\n");
 
-	/* Signal current eviction fence */
-	amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
-
-	if (evf_mgr->fd_closing) {
-		cancel_delayed_work_sync(&uq_mgr->resume_work);
-		return;
-	}
-
-	/* Schedule a resume work */
-	schedule_delayed_work(&uq_mgr->resume_work, 0);
+	if (schedule_resume)
+		schedule_delayed_work(&uq_mgr->resume_work, 0);
 }
 
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
@@ -1401,6 +1345,11 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 	return 0;
 }
 
+void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
+{
+	cancel_delayed_work_sync(&userq_mgr->resume_work);
+}
+
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 {
 	struct amdgpu_usermode_queue *queue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 736c1d38297c8..f0abc16d02cce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -82,6 +82,8 @@ struct amdgpu_usermode_queue {
 struct amdgpu_userq_funcs {
 	int (*mqd_create)(struct amdgpu_usermode_queue *queue,
 			  struct drm_amdgpu_userq_in *args);
+	int (*mqd_update)(struct amdgpu_usermode_queue *queue,
+			  struct drm_amdgpu_userq_in *args);
 	void (*mqd_destroy)(struct amdgpu_usermode_queue *uq);
 	int (*unmap)(struct amdgpu_usermode_queue *queue);
 	int (*map)(struct amdgpu_usermode_queue *queue);
@@ -121,6 +123,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
 			  struct amdgpu_device *adev);
 
+void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
 
 int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
@@ -131,7 +134,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
 				 struct amdgpu_userq_obj *userq_obj);
 
 void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
-			struct amdgpu_eviction_fence *ev_fence);
+			bool schedule_resume);
 
 void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
 				  struct amdgpu_eviction_fence_mgr *evf_mgr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 5239b06b9ab03..781896c9fd267 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -461,33 +461,31 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *filp)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct drm_amdgpu_userq_signal *args = data;
+	const unsigned int num_write_bo_handles = args->num_bo_write_handles;
+	const unsigned int num_read_bo_handles = args->num_bo_read_handles;
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
-	struct drm_amdgpu_userq_signal *args = data;
-	struct drm_gem_object **gobj_write = NULL;
-	struct drm_gem_object **gobj_read = NULL;
-	struct amdgpu_usermode_queue *queue = NULL;
+	struct drm_gem_object **gobj_write, **gobj_read;
+	u32 *syncobj_handles, num_syncobj_handles;
 	struct amdgpu_userq_fence *userq_fence;
+	struct amdgpu_usermode_queue *queue = NULL;
 	struct drm_syncobj **syncobj = NULL;
-	u32 *bo_handles_write, num_write_bo_handles;
-	u32 *syncobj_handles, num_syncobj_handles;
-	u32 *bo_handles_read, num_read_bo_handles;
-	int r, i, entry, rentry, wentry;
 	struct dma_fence *fence;
 	struct drm_exec exec;
+	int r, i, entry;
 	u64 wptr;
 
 	if (!amdgpu_userq_enabled(dev))
 		return -ENOTSUPP;
 
-	if (args->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES ||
-	    args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
+	if (args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
 	    args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
 		return -EINVAL;
 
 	num_syncobj_handles = args->num_syncobj_handles;
-	syncobj_handles = memdup_user(u64_to_user_ptr(args->syncobj_handles),
-				      size_mul(sizeof(u32), num_syncobj_handles));
+	syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
+					    num_syncobj_handles, sizeof(u32));
 	if (IS_ERR(syncobj_handles))
 		return PTR_ERR(syncobj_handles);
 
@@ -506,51 +504,19 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	num_read_bo_handles = args->num_bo_read_handles;
-	bo_handles_read = memdup_user(u64_to_user_ptr(args->bo_read_handles),
-				      sizeof(u32) * num_read_bo_handles);
-	if (IS_ERR(bo_handles_read)) {
-		r = PTR_ERR(bo_handles_read);
+	r = drm_gem_objects_lookup(filp,
+				   u64_to_user_ptr(args->bo_read_handles),
+				   num_read_bo_handles,
+				   &gobj_read);
+	if (r)
 		goto free_syncobj;
-	}
-
-	/* Array of pointers to the GEM read objects */
-	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
-	if (!gobj_read) {
-		r = -ENOMEM;
-		goto free_bo_handles_read;
-	}
-
-	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
-		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
-		if (!gobj_read[rentry]) {
-			r = -ENOENT;
-			goto put_gobj_read;
-		}
-	}
 
-	num_write_bo_handles = args->num_bo_write_handles;
-	bo_handles_write = memdup_user(u64_to_user_ptr(args->bo_write_handles),
-				       sizeof(u32) * num_write_bo_handles);
-	if (IS_ERR(bo_handles_write)) {
-		r = PTR_ERR(bo_handles_write);
+	r = drm_gem_objects_lookup(filp,
+				   u64_to_user_ptr(args->bo_write_handles),
+				   num_write_bo_handles,
+				   &gobj_write);
+	if (r)
 		goto put_gobj_read;
-	}
-
-	/* Array of pointers to the GEM write objects */
-	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
-	if (!gobj_write) {
-		r = -ENOMEM;
-		goto free_bo_handles_write;
-	}
-
-	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
-		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
-		if (!gobj_write[wentry]) {
-			r = -ENOENT;
-			goto put_gobj_write;
-		}
-	}
 
 	/* Retrieve the user queue */
 	queue = amdgpu_userq_get(userq_mgr, args->queue_id);
@@ -629,17 +595,13 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 exec_fini:
 	drm_exec_fini(&exec);
 put_gobj_write:
-	while (wentry-- > 0)
-		drm_gem_object_put(gobj_write[wentry]);
-	kfree(gobj_write);
-free_bo_handles_write:
-	kfree(bo_handles_write);
+	for (i = 0; i < num_write_bo_handles; i++)
+		drm_gem_object_put(gobj_write[i]);
+	kvfree(gobj_write);
 put_gobj_read:
-	while (rentry-- > 0)
-		drm_gem_object_put(gobj_read[rentry]);
-	kfree(gobj_read);
-free_bo_handles_read:
-	kfree(bo_handles_read);
+	for (i = 0; i < num_read_bo_handles; i++)
+		drm_gem_object_put(gobj_read[i]);
+	kvfree(gobj_read);
 free_syncobj:
 	while (entry-- > 0)
 		if (syncobj[entry])
@@ -654,371 +616,399 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	return r;
 }
 
-int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
-			    struct drm_file *filp)
+/* Count the number of expected fences so userspace can alloc a buffer */
+static int
+amdgpu_userq_wait_count_fences(struct drm_file *filp,
+			       struct drm_amdgpu_userq_wait *wait_info,
+			       u32 *syncobj_handles, u32 *timeline_points,
+			       u32 *timeline_handles,
+			       struct drm_gem_object **gobj_write,
+			       struct drm_gem_object **gobj_read)
 {
-	u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write;
-	u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
-	struct drm_amdgpu_userq_fence_info *fence_info = NULL;
-	struct drm_amdgpu_userq_wait *wait_info = data;
-	struct amdgpu_fpriv *fpriv = filp->driver_priv;
-	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
-	struct amdgpu_usermode_queue *waitq = NULL;
-	struct drm_gem_object **gobj_write;
-	struct drm_gem_object **gobj_read;
-	struct dma_fence **fences = NULL;
-	u16 num_points, num_fences = 0;
-	int r, i, rentry, wentry, cnt;
+	int num_read_bo_handles, num_write_bo_handles;
+	struct dma_fence_unwrap iter;
+	struct dma_fence *fence, *f;
+	unsigned int num_fences = 0;
 	struct drm_exec exec;
+	int i, r;
+
+	/*
+	 * This needs to be outside of the lock provided by drm_exec for
+	 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
+	 */
+
+	/* Count timeline fences */
+	for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
+		r = drm_syncobj_find_fence(filp, timeline_handles[i],
+					   timeline_points[i],
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			return r;
+
+		dma_fence_unwrap_for_each(f, &iter, fence)
+			num_fences++;
 
-	if (!amdgpu_userq_enabled(dev))
-		return -ENOTSUPP;
+		dma_fence_put(fence);
+	}
 
-	if (wait_info->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES ||
-	    wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
-	    wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
-		return -EINVAL;
+	/* Count boolean fences */
+	for (i = 0; i < wait_info->num_syncobj_handles; i++) {
+		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			return r;
 
-	num_read_bo_handles = wait_info->num_bo_read_handles;
-	bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles),
-				      size_mul(sizeof(u32), num_read_bo_handles));
-	if (IS_ERR(bo_handles_read))
-		return PTR_ERR(bo_handles_read);
+		num_fences++;
+		dma_fence_put(fence);
+	}
 
+	/* Lock all the GEM objects */
+	/* TODO: It is actually not necessary to lock them */
+	num_read_bo_handles = wait_info->num_bo_read_handles;
 	num_write_bo_handles = wait_info->num_bo_write_handles;
-	bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles),
-				       size_mul(sizeof(u32), num_write_bo_handles));
-	if (IS_ERR(bo_handles_write)) {
-		r = PTR_ERR(bo_handles_write);
-		goto free_bo_handles_read;
-	}
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
+		      num_read_bo_handles + num_write_bo_handles);
 
-	num_syncobj = wait_info->num_syncobj_handles;
-	syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles),
-				      size_mul(sizeof(u32), num_syncobj));
-	if (IS_ERR(syncobj_handles)) {
-		r = PTR_ERR(syncobj_handles);
-		goto free_bo_handles_write;
+	drm_exec_until_all_locked(&exec) {
+		r = drm_exec_prepare_array(&exec, gobj_read,
+					   num_read_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
+
+		r = drm_exec_prepare_array(&exec, gobj_write,
+					   num_write_bo_handles, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (r)
+			goto error_unlock;
 	}
 
-	num_points = wait_info->num_syncobj_timeline_handles;
-	timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles),
-				       sizeof(u32) * num_points);
-	if (IS_ERR(timeline_handles)) {
-		r = PTR_ERR(timeline_handles);
-		goto free_syncobj_handles;
+	/* Count read fences */
+	for (i = 0; i < num_read_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
+
+		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
+					DMA_RESV_USAGE_READ, fence)
+			num_fences++;
 	}
 
-	timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points),
-				      sizeof(u32) * num_points);
-	if (IS_ERR(timeline_points)) {
-		r = PTR_ERR(timeline_points);
-		goto free_timeline_handles;
+	/* Count write fences */
+	for (i = 0; i < num_write_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
+
+		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
+					DMA_RESV_USAGE_WRITE, fence)
+			num_fences++;
 	}
 
-	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
-	if (!gobj_read) {
+	wait_info->num_fences = num_fences;
+	r = 0;
+
+error_unlock:
+	/* Unlock all the GEM objects */
+	drm_exec_fini(&exec);
+	return r;
+}
+
+static int
+amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
+				    struct drm_amdgpu_userq_wait *wait_info,
+				    u32 *syncobj_handles, u32 *timeline_points,
+				    u32 *timeline_handles,
+				    struct drm_gem_object **gobj_write,
+				    struct drm_gem_object **gobj_read)
+{
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
+	struct drm_amdgpu_userq_fence_info *fence_info;
+	int num_read_bo_handles, num_write_bo_handles;
+	struct amdgpu_usermode_queue *waitq;
+	struct dma_fence **fences, *fence, *f;
+	struct dma_fence_unwrap iter;
+	int num_points, num_syncobj;
+	unsigned int num_fences = 0;
+	struct drm_exec exec;
+	int i, cnt, r;
+
+	fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
+				   GFP_KERNEL);
+	if (!fence_info)
+		return -ENOMEM;
+
+	fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
+			       GFP_KERNEL);
+	if (!fences) {
 		r = -ENOMEM;
-		goto free_timeline_points;
+		goto free_fence_info;
 	}
 
-	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
-		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
-		if (!gobj_read[rentry]) {
-			r = -ENOENT;
-			goto put_gobj_read;
+	/* Retrieve timeline fences */
+	num_points = wait_info->num_syncobj_timeline_handles;
+	for (i = 0; i < num_points; i++) {
+		r = drm_syncobj_find_fence(filp, timeline_handles[i],
+					   timeline_points[i],
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			goto free_fences;
+
+		dma_fence_unwrap_for_each(f, &iter, fence) {
+			if (num_fences >= wait_info->num_fences) {
+				r = -EINVAL;
+				dma_fence_put(fence);
+				goto free_fences;
+			}
+
+			fences[num_fences++] = dma_fence_get(f);
 		}
-	}
 
-	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
-	if (!gobj_write) {
-		r = -ENOMEM;
-		goto put_gobj_read;
+		dma_fence_put(fence);
 	}
 
-	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
-		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
-		if (!gobj_write[wentry]) {
-			r = -ENOENT;
-			goto put_gobj_write;
+	/* Retrieve boolean fences */
+	num_syncobj = wait_info->num_syncobj_handles;
+	for (i = 0; i < num_syncobj; i++) {
+		struct dma_fence *fence;
+
+		r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
+					   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+					   &fence);
+		if (r)
+			goto free_fences;
+
+		if (num_fences >= wait_info->num_fences) {
+			dma_fence_put(fence);
+			r = -EINVAL;
+			goto free_fences;
 		}
+
+		/* Give the reference to the fence array */
+		fences[num_fences++] = fence;
 	}
 
+	/* Lock all the GEM objects */
+	num_read_bo_handles = wait_info->num_bo_read_handles;
+	num_write_bo_handles = wait_info->num_bo_write_handles;
 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
-		      (num_read_bo_handles + num_write_bo_handles));
+		      num_read_bo_handles + num_write_bo_handles);
 
-	/* Lock all BOs with retry handling */
 	drm_exec_until_all_locked(&exec) {
-		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
+		r = drm_exec_prepare_array(&exec, gobj_read,
+					   num_read_bo_handles, 1);
 		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			drm_exec_fini(&exec);
-			goto put_gobj_write;
-		}
+		if (r)
+			goto error_unlock;
 
-		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
+		r = drm_exec_prepare_array(&exec, gobj_write,
+					   num_write_bo_handles, 1);
 		drm_exec_retry_on_contention(&exec);
-		if (r) {
-			drm_exec_fini(&exec);
-			goto put_gobj_write;
-		}
+		if (r)
+			goto error_unlock;
 	}
 
-	if (!wait_info->num_fences) {
-		if (num_points) {
-			struct dma_fence_unwrap iter;
-			struct dma_fence *fence;
-			struct dma_fence *f;
-
-			for (i = 0; i < num_points; i++) {
-				r = drm_syncobj_find_fence(filp, timeline_handles[i],
-							   timeline_points[i],
-							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-							   &fence);
-				if (r)
-					goto exec_fini;
-
-				dma_fence_unwrap_for_each(f, &iter, fence)
-					num_fences++;
+	/* Retrieve GEM read objects fence */
+	for (i = 0; i < num_read_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-				dma_fence_put(fence);
+		dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
+					DMA_RESV_USAGE_READ, fence) {
+			if (num_fences >= wait_info->num_fences) {
+				r = -EINVAL;
+				goto error_unlock;
 			}
+
+			fences[num_fences++] = dma_fence_get(fence);
 		}
+	}
 
-		/* Count syncobj's fence */
-		for (i = 0; i < num_syncobj; i++) {
-			struct dma_fence *fence;
+	/* Retrieve GEM write objects fence */
+	for (i = 0; i < num_write_bo_handles; i++) {
+		struct dma_resv_iter resv_cursor;
+		struct dma_fence *fence;
 
-			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
-						   0,
-						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-						   &fence);
-			if (r)
-				goto exec_fini;
+		dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
+					DMA_RESV_USAGE_WRITE, fence) {
+			if (num_fences >= wait_info->num_fences) {
+				r = -EINVAL;
+				goto error_unlock;
+			}
 
-			num_fences++;
-			dma_fence_put(fence);
+			fences[num_fences++] = dma_fence_get(fence);
 		}
+	}
 
-		/* Count GEM objects fence */
-		for (i = 0; i < num_read_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+	drm_exec_fini(&exec);
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
-						DMA_RESV_USAGE_READ, fence)
-				num_fences++;
-		}
+	/*
+	 * Keep only the latest fences to reduce the number of values
+	 * given back to userspace.
+	 */
+	num_fences = dma_fence_dedup_array(fences, num_fences);
 
-		for (i = 0; i < num_write_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+	waitq = amdgpu_userq_get(userq_mgr, wait_info->waitq_id);
+	if (!waitq) {
+		r = -EINVAL;
+		goto free_fences;
+	}
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
-						DMA_RESV_USAGE_WRITE, fence)
-				num_fences++;
-		}
+	for (i = 0, cnt = 0; i < num_fences; i++) {
+		struct amdgpu_userq_fence_driver *fence_drv;
+		struct amdgpu_userq_fence *userq_fence;
+		u32 index;
 
-		/*
-		 * Passing num_fences = 0 means that userspace doesn't want to
-		 * retrieve userq_fence_info. If num_fences = 0 we skip filling
-		 * userq_fence_info and return the actual number of fences on
-		 * args->num_fences.
-		 */
-		wait_info->num_fences = num_fences;
-	} else {
-		/* Array of fence info */
-		fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL);
-		if (!fence_info) {
-			r = -ENOMEM;
-			goto exec_fini;
-		}
+		userq_fence = to_amdgpu_userq_fence(fences[i]);
+		if (!userq_fence) {
+			/*
+			 * Just waiting on other driver fences should
+			 * be good for now
+			 */
+			r = dma_fence_wait(fences[i], true);
+			if (r)
+				goto put_waitq;
 
-		/* Array of fences */
-		fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL);
-		if (!fences) {
-			r = -ENOMEM;
-			goto free_fence_info;
+			continue;
 		}
 
-		/* Retrieve GEM read objects fence */
-		for (i = 0; i < num_read_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
-
-			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
-						DMA_RESV_USAGE_READ, fence) {
-				if (num_fences >= wait_info->num_fences) {
-					r = -EINVAL;
-					goto free_fences;
-				}
-
-				fences[num_fences++] = fence;
-				dma_fence_get(fence);
-			}
-		}
+		fence_drv = userq_fence->fence_drv;
+		/*
+		 * We need to make sure the user queue release their reference
+		 * to the fence drivers at some point before queue destruction.
+		 * Otherwise, we would gather those references until we don't
+		 * have any more space left and crash.
+		 */
+		r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
+			     xa_limit_32b, GFP_KERNEL);
+		if (r)
+			goto put_waitq;
 
-		/* Retrieve GEM write objects fence */
-		for (i = 0; i < num_write_bo_handles; i++) {
-			struct dma_resv_iter resv_cursor;
-			struct dma_fence *fence;
+		amdgpu_userq_fence_driver_get(fence_drv);
 
-			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
-						DMA_RESV_USAGE_WRITE, fence) {
-				if (num_fences >= wait_info->num_fences) {
-					r = -EINVAL;
-					goto free_fences;
-				}
+		/* Store drm syncobj's gpu va address and value */
+		fence_info[cnt].va = fence_drv->va;
+		fence_info[cnt].value = fences[i]->seqno;
 
-				fences[num_fences++] = fence;
-				dma_fence_get(fence);
-			}
-		}
+		/* Increment the actual userq fence count */
+		cnt++;
+	}
+	wait_info->num_fences = cnt;
 
-		if (num_points) {
-			struct dma_fence_unwrap iter;
-			struct dma_fence *fence;
-			struct dma_fence *f;
-
-			for (i = 0; i < num_points; i++) {
-				r = drm_syncobj_find_fence(filp, timeline_handles[i],
-							   timeline_points[i],
-							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-							   &fence);
-				if (r)
-					goto free_fences;
-
-				dma_fence_unwrap_for_each(f, &iter, fence) {
-					if (num_fences >= wait_info->num_fences) {
-						r = -EINVAL;
-						dma_fence_put(fence);
-						goto free_fences;
-					}
-
-					dma_fence_get(f);
-					fences[num_fences++] = f;
-				}
+	/* Copy userq fence info to user space */
+	if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
+			 fence_info, cnt * sizeof(*fence_info)))
+		r = -EFAULT;
+	else
+		r = 0;
 
-				dma_fence_put(fence);
-			}
-		}
+put_waitq:
+	amdgpu_userq_put(waitq);
 
-		/* Retrieve syncobj's fence */
-		for (i = 0; i < num_syncobj; i++) {
-			struct dma_fence *fence;
+free_fences:
+	while (num_fences--)
+		dma_fence_put(fences[num_fences]);
+	kfree(fences);
 
-			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
-						   0,
-						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
-						   &fence);
-			if (r)
-				goto free_fences;
+free_fence_info:
+	kfree(fence_info);
+	return r;
 
-			if (num_fences >= wait_info->num_fences) {
-				r = -EINVAL;
-				dma_fence_put(fence);
-				goto free_fences;
-			}
+error_unlock:
+	drm_exec_fini(&exec);
+	goto free_fences;
+}
 
-			fences[num_fences++] = fence;
-		}
+int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *filp)
+{
+	int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
+	u32 *syncobj_handles, *timeline_points, *timeline_handles;
+	struct drm_amdgpu_userq_wait *wait_info = data;
+	struct drm_gem_object **gobj_write;
+	struct drm_gem_object **gobj_read;
+	void __user *ptr;
+	int r;
 
-		/*
-		 * Keep only the latest fences to reduce the number of values
-		 * given back to userspace.
-		 */
-		num_fences = dma_fence_dedup_array(fences, num_fences);
+	if (!amdgpu_userq_enabled(dev))
+		return -ENOTSUPP;
 
-		waitq = amdgpu_userq_get(userq_mgr, wait_info->waitq_id);
-		if (!waitq) {
-			r = -EINVAL;
-			goto free_fences;
-		}
+	if (wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
+	    wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
+		return -EINVAL;
 
-		for (i = 0, cnt = 0; i < num_fences; i++) {
-			struct amdgpu_userq_fence_driver *fence_drv;
-			struct amdgpu_userq_fence *userq_fence;
-			u32 index;
-
-			userq_fence = to_amdgpu_userq_fence(fences[i]);
-			if (!userq_fence) {
-				/*
-				 * Just waiting on other driver fences should
-				 * be good for now
-				 */
-				r = dma_fence_wait(fences[i], true);
-				if (r) {
-					dma_fence_put(fences[i]);
-					goto free_fences;
-				}
-
-				dma_fence_put(fences[i]);
-				continue;
-			}
+	num_syncobj = wait_info->num_syncobj_handles;
+	ptr = u64_to_user_ptr(wait_info->syncobj_handles);
+	syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
+	if (IS_ERR(syncobj_handles))
+		return PTR_ERR(syncobj_handles);
 
-			fence_drv = userq_fence->fence_drv;
-			/*
-			 * We need to make sure the user queue release their reference
-			 * to the fence drivers at some point before queue destruction.
-			 * Otherwise, we would gather those references until we don't
-			 * have any more space left and crash.
-			 */
-			r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
-				     xa_limit_32b, GFP_KERNEL);
-			if (r)
-				goto free_fences;
+	num_points = wait_info->num_syncobj_timeline_handles;
+	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
+	timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
+	if (IS_ERR(timeline_handles)) {
+		r = PTR_ERR(timeline_handles);
+		goto free_syncobj_handles;
+	}
 
-			amdgpu_userq_fence_driver_get(fence_drv);
+	ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
+	timeline_points = memdup_array_user(ptr, num_points, sizeof(u32));
+	if (IS_ERR(timeline_points)) {
+		r = PTR_ERR(timeline_points);
+		goto free_timeline_handles;
+	}
 
-			/* Store drm syncobj's gpu va address and value */
-			fence_info[cnt].va = fence_drv->va;
-			fence_info[cnt].value = fences[i]->seqno;
+	num_read_bo_handles = wait_info->num_bo_read_handles;
+	ptr = u64_to_user_ptr(wait_info->bo_read_handles),
+	r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
+	if (r)
+		goto free_timeline_points;
 
-			dma_fence_put(fences[i]);
-			/* Increment the actual userq fence count */
-			cnt++;
-		}
+	num_write_bo_handles = wait_info->num_bo_write_handles;
+	ptr = u64_to_user_ptr(wait_info->bo_write_handles),
+	r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
+				   &gobj_write);
+	if (r)
+		goto put_gobj_read;
 
-		wait_info->num_fences = cnt;
-		/* Copy userq fence info to user space */
-		if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
-				 fence_info, wait_info->num_fences * sizeof(*fence_info))) {
-			r = -EFAULT;
-			goto free_fences;
-		}
+	/*
+	 * Passing num_fences = 0 means that userspace doesn't want to
+	 * retrieve userq_fence_info. If num_fences = 0 we skip filling
+	 * userq_fence_info and return the actual number of fences on
+	 * args->num_fences.
+	 */
+	if (!wait_info->num_fences) {
+		r = amdgpu_userq_wait_count_fences(filp, wait_info,
+						   syncobj_handles,
+						   timeline_points,
+						   timeline_handles,
+						   gobj_write,
+						   gobj_read);
+	} else {
+		r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
+							syncobj_handles,
+							timeline_points,
+							timeline_handles,
+							gobj_write,
+							gobj_read);
 	}
 
-free_fences:
-	if (fences) {
-		while (num_fences-- > 0)
-			dma_fence_put(fences[num_fences]);
-		kfree(fences);
-	}
-free_fence_info:
-	kfree(fence_info);
-exec_fini:
-	drm_exec_fini(&exec);
-put_gobj_write:
-	while (wentry-- > 0)
-		drm_gem_object_put(gobj_write[wentry]);
-	kfree(gobj_write);
+	while (num_write_bo_handles--)
+		drm_gem_object_put(gobj_write[num_write_bo_handles]);
+	kvfree(gobj_write);
+
 put_gobj_read:
-	while (rentry-- > 0)
-		drm_gem_object_put(gobj_read[rentry]);
-	kfree(gobj_read);
+	while (num_read_bo_handles--)
+		drm_gem_object_put(gobj_read[num_read_bo_handles]);
+	kvfree(gobj_read);
+
 free_timeline_points:
 	kfree(timeline_points);
 free_timeline_handles:
 	kfree(timeline_handles);
 free_syncobj_handles:
 	kfree(syncobj_handles);
-free_bo_handles_write:
-	kfree(bo_handles_write);
-free_bo_handles_read:
-	kfree(bo_handles_read);
-
-	if (waitq)
-		amdgpu_userq_put(waitq);
-
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index a7d8f1ce6ac2d..eb4a15db2ef20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -450,24 +450,6 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 	}
 }
 
-/**
- * amdgpu_vce_required_gart_pages() - gets number of GART pages required by VCE
- *
- * @adev: amdgpu_device pointer
- *
- * Returns how many GART pages we need before GTT for the VCE IP block.
- * For VCE1, see vce_v1_0_ensure_vcpu_bo_32bit_addr for details.
- * For VCE2+, this is not needed so return zero.
- */
-u32 amdgpu_vce_required_gart_pages(struct amdgpu_device *adev)
-{
-	/* VCE IP block not added yet, so can't use amdgpu_ip_version */
-	if (adev->family == AMDGPU_FAMILY_SI)
-		return 512;
-
-	return 0;
-}
-
 /**
  * amdgpu_vce_get_create_msg - generate a VCE create msg
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 1c3464ce50378..778c714c8385d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -52,6 +52,7 @@ struct amdgpu_vce {
 	uint32_t                srbm_soft_reset;
 	unsigned		num_rings;
 	uint32_t		keyselect;
+	struct drm_mm_node	gart_node;
 };
 
 int amdgpu_vce_early_init(struct amdgpu_device *adev);
@@ -61,7 +62,6 @@ int amdgpu_vce_entity_init(struct amdgpu_device *adev, struct amdgpu_ring *ring)
 int amdgpu_vce_suspend(struct amdgpu_device *adev);
 int amdgpu_vce_resume(struct amdgpu_device *adev);
 void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp);
-u32 amdgpu_vce_required_gart_pages(struct amdgpu_device *adev);
 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 			     struct amdgpu_ib *ib);
 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 75ae9b429420e..03d95dca93d75 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -63,6 +63,7 @@
 #define FIRMWARE_VCN4_0_6_1		"amdgpu/vcn_4_0_6_1.bin"
 #define FIRMWARE_VCN5_0_0		"amdgpu/vcn_5_0_0.bin"
 #define FIRMWARE_VCN5_0_1		"amdgpu/vcn_5_0_1.bin"
+#define FIRMWARE_VCN5_0_2		"amdgpu/vcn_5_0_2.bin"
 #define FIRMWARE_VCN5_3_0		"amdgpu/vcn_5_3_0.bin"
 
 MODULE_FIRMWARE(FIRMWARE_RAVEN);
@@ -91,6 +92,7 @@ MODULE_FIRMWARE(FIRMWARE_VCN4_0_6);
 MODULE_FIRMWARE(FIRMWARE_VCN4_0_6_1);
 MODULE_FIRMWARE(FIRMWARE_VCN5_0_0);
 MODULE_FIRMWARE(FIRMWARE_VCN5_0_1);
+MODULE_FIRMWARE(FIRMWARE_VCN5_0_2);
 MODULE_FIRMWARE(FIRMWARE_VCN5_3_0);
 
 static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
@@ -1095,7 +1097,8 @@ int amdgpu_vcn_unified_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	long r;
 
 	if ((amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(4, 0, 3)) &&
-	    (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(5, 0, 1))) {
+	    (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(5, 0, 1)) &&
+	    (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(5, 0, 2))) {
 		r = amdgpu_vcn_enc_ring_test_ib(ring, timeout);
 		if (r)
 			goto error;
@@ -1132,7 +1135,8 @@ void amdgpu_vcn_setup_ucode(struct amdgpu_device *adev, int i)
 			return;
 
 		if ((amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(4, 0, 3) ||
-		     amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(5, 0, 1))
+		     amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(5, 0, 1) ||
+		     amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(5, 0, 2))
 		    && (i > 0))
 			return;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 275745aa58292..dba7ea16a10d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -605,10 +605,10 @@ static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev)
 
 #ifdef MODULE
 	if (THIS_MODULE->version != NULL)
-		strcpy(vf2pf_info->driver_version, THIS_MODULE->version);
+		strscpy(vf2pf_info->driver_version, THIS_MODULE->version);
 	else
 #endif
-		strcpy(vf2pf_info->driver_version, "N/A");
+		strscpy(vf2pf_info->driver_version, "N/A");
 
 	vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all
 	vf2pf_info->driver_cert = 0;
@@ -950,11 +950,6 @@ int amdgpu_virt_init_critical_region(struct amdgpu_device *adev)
 	if (adev->virt.req_init_data_ver != GPU_CRIT_REGION_V2)
 		return 0;
 
-	if (init_hdr_offset < 0) {
-		dev_err(adev->dev, "Invalid init header offset\n");
-		return -EINVAL;
-	}
-
 	vram_size = RREG32(mmRCC_CONFIG_MEMSIZE);
 	if (!vram_size || vram_size == U32_MAX)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 886fbce0bfd1d..9da0c6e9b8695 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -162,6 +162,7 @@ enum AMDGIM_FEATURE_FLAG {
 	AMDGIM_FEATURE_RAS_TELEMETRY = (1 << 10),
 	AMDGIM_FEATURE_RAS_CPER = (1 << 11),
 	AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK = (1 << 12),
+	AMDGIM_FEATURE_XGMI_CONNECTED_TO_CPU = (1 << 13),
 };
 
 enum AMDGIM_REG_ACCESS_FLAG {
@@ -412,6 +413,9 @@ struct amdgpu_video_codec_info;
 #define amdgpu_sriov_xgmi_ta_ext_peer_link_en(adev) \
 ((adev)->virt.gim_feature & AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK)
 
+#define amdgpu_sriov_xgmi_connected_to_cpu(adev) \
+((adev)->virt.gim_feature & AMDGIM_FEATURE_XGMI_CONNECTED_TO_CPU)
+
 static inline bool is_virtual_machine(void)
 {
 #if defined(CONFIG_X86)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index f2beb980e3c3a..3f5712fc72163 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -138,6 +138,20 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 }
 
+/**
+ * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
+ *
+ * @vm: VM to test against.
+ * @bo: BO to be tested.
+ *
+ * Returns true if the BO shares the dma_resv object with the root PD and is
+ * always guaranteed to be valid inside the VM.
+ */
+bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
+{
+	return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;
+}
+
 /**
  * amdgpu_vm_bo_evicted - vm_bo is evicted
  *
@@ -764,12 +778,9 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  * @need_pipe_sync: is pipe sync needed
  *
  * Emit a VM flush when it is necessary.
- *
- * Returns:
- * 0 on success, errno otherwise.
  */
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
-		    bool need_pipe_sync)
+void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
+		     bool need_pipe_sync)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];
@@ -783,8 +794,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	bool cleaner_shader_needed = false;
 	bool pasid_mapping_needed = false;
 	struct dma_fence *fence = NULL;
-	unsigned int patch;
-	int r;
+	unsigned int patch = 0;
 
 	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 		gds_switch_needed = true;
@@ -812,9 +822,20 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 
 	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync &&
 	    !cleaner_shader_needed)
-		return 0;
+		return;
 
 	amdgpu_ring_ib_begin(ring);
+
+	/* There is no matching insert_end for this on purpose for the vm flush.
+	 * The IB portion of the submission has both.  Having multiple
+	 * insert_start sequences is ok, but you can only have one insert_end
+	 * per submission based on the way VCN FW works.  For JPEG
+	 * you can as many insert_start and insert_end sequences as you like as
+	 * long as the rest of the packets come between start and end sequences.
+	 */
+	if (ring->funcs->insert_start)
+		ring->funcs->insert_start(ring);
+
 	if (ring->funcs->init_cond_exec)
 		patch = amdgpu_ring_init_cond_exec(ring,
 						   ring->cond_exe_gpu_addr);
@@ -845,9 +866,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	}
 
 	if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) {
-		r = amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
-		if (r)
-			return r;
+		amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
 		fence = &job->hw_vm_fence->base;
 		/* get a ref for the job */
 		dma_fence_get(fence);
@@ -892,7 +911,6 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	}
 
 	amdgpu_ring_ib_end(ring);
-	return 0;
 }
 
 /**
@@ -1069,7 +1087,10 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
 	}
 
 	/* Prepare a TLB flush fence to be attached to PTs */
-	if (!params->unlocked) {
+	/* The check for need_tlb_fence should be dropped once we
+	 * sort out the issues with KIQ/MES TLB invalidation timeouts.
+	 */
+	if (!params->unlocked && vm->need_tlb_fence) {
 		amdgpu_vm_tlb_fence_create(params->adev, vm, fence);
 
 		/* Makes sure no PD/PT is freed before the flush */
@@ -2602,6 +2623,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	ttm_lru_bulk_move_init(&vm->lru_bulk_move);
 
 	vm->is_compute_context = false;
+	vm->need_tlb_fence = amdgpu_userq_enabled(&adev->ddev);
 
 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
 				    AMDGPU_VM_USE_CPU_FOR_GFX);
@@ -2739,6 +2761,7 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	dma_fence_put(vm->last_update);
 	vm->last_update = dma_fence_get_stub();
 	vm->is_compute_context = true;
+	vm->need_tlb_fence = true;
 
 unreserve_bo:
 	amdgpu_bo_unreserve(vm->root.bo);
@@ -2785,8 +2808,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	dma_fence_put(vm->last_unlocked);
 	dma_fence_wait(vm->last_tlb_flush, false);
 	/* Make sure that all fence callbacks have completed */
-	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
-	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
+	dma_fence_lock_irqsave(vm->last_tlb_flush, flags);
+	dma_fence_unlock_irqrestore(vm->last_tlb_flush, flags);
 	dma_fence_put(vm->last_tlb_flush);
 
 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
@@ -3188,20 +3211,6 @@ void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
 }
 
-/**
- * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
- *
- * @vm: VM to test against.
- * @bo: BO to be tested.
- *
- * Returns true if the BO shares the dma_resv object with the root PD and is
- * always guaranteed to be valid inside the VM.
- */
-bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
-{
-	return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;
-}
-
 void amdgpu_vm_print_task_info(struct amdgpu_device *adev,
 			       struct amdgpu_task_info *task_info)
 {
@@ -3210,3 +3219,20 @@ void amdgpu_vm_print_task_info(struct amdgpu_device *adev,
 		task_info->process_name, task_info->tgid,
 		task_info->task.comm, task_info->task.pid);
 }
+
+void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
+				   const struct amdgpu_vm_pte_funcs *vm_pte_funcs)
+{
+	struct drm_gpu_scheduler *sched;
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->sdma.has_page_queue)
+			sched = &adev->sdma.instance[i].page.sched;
+		else
+			sched = &adev->sdma.instance[i].ring.sched;
+		adev->vm_manager.vm_pte_scheds[i] = sched;
+	}
+	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
+	adev->vm_manager.vm_pte_funcs = vm_pte_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 806d62ed61eff..641fe91b4f035 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -441,6 +441,8 @@ struct amdgpu_vm {
 	struct ttm_lru_bulk_move lru_bulk_move;
 	/* Flag to indicate if VM is used for compute */
 	bool			is_compute_context;
+	/* Flag to indicate if VM needs a TLB fence (KFD or KGD) */
+	bool			need_tlb_fence;
 
 	/* Memory partition number, -1 means any partition */
 	int8_t			mem_id;
@@ -513,7 +515,7 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       struct ww_acquire_ctx *ticket,
 		       int (*callback)(void *p, struct amdgpu_bo *bo),
 		       void *param);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
+void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm, bool immediate);
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
@@ -639,7 +641,7 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
 	 * sure that the dma_fence structure isn't freed up.
 	 */
 	rcu_read_lock();
-	lock = vm->last_tlb_flush->lock;
+	lock = dma_fence_spinlock(vm->last_tlb_flush);
 	rcu_read_unlock();
 
 	spin_lock_irqsave(lock, flags);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 6c9b3e21e15c3..2a241a5b12c42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -25,6 +25,7 @@
 #include <linux/dma-mapping.h>
 #include <drm/ttm/ttm_range_manager.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_buddy.h>
 
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
@@ -52,15 +53,15 @@ to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
 	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
 }
 
-static inline struct drm_buddy_block *
+static inline struct gpu_buddy_block *
 amdgpu_vram_mgr_first_block(struct list_head *list)
 {
-	return list_first_entry_or_null(list, struct drm_buddy_block, link);
+	return list_first_entry_or_null(list, struct gpu_buddy_block, link);
 }
 
 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	u64 start, size;
 
 	block = amdgpu_vram_mgr_first_block(head);
@@ -71,7 +72,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
 		start = amdgpu_vram_mgr_block_start(block);
 		size = amdgpu_vram_mgr_block_size(block);
 
-		block = list_entry(block->link.next, struct drm_buddy_block, link);
+		block = list_entry(block->link.next, struct gpu_buddy_block, link);
 		if (start + size != amdgpu_vram_mgr_block_start(block))
 			return false;
 	}
@@ -81,7 +82,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
 
 static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	u64 size = 0;
 
 	list_for_each_entry(block, head, link)
@@ -254,7 +255,7 @@ const struct attribute_group amdgpu_vram_mgr_attr_group = {
  * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
  */
 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
-				    struct drm_buddy_block *block)
+				    struct gpu_buddy_block *block)
 {
 	u64 start = amdgpu_vram_mgr_block_start(block);
 	u64 end = start + amdgpu_vram_mgr_block_size(block);
@@ -279,7 +280,7 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct ttm_resource *res = bo->tbo.resource;
 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	u64 usage = 0;
 
 	if (amdgpu_gmc_vram_full_visible(&adev->gmc))
@@ -299,15 +300,15 @@ static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
 {
 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 	struct amdgpu_vram_reservation *rsv, *temp;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	uint64_t vis_usage;
 
 	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
-		if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
+		if (gpu_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
 					   rsv->size, mm->chunk_size, &rsv->allocated,
-					   DRM_BUDDY_RANGE_ALLOCATION))
+					   GPU_BUDDY_RANGE_ALLOCATION))
 			continue;
 
 		block = amdgpu_vram_mgr_first_block(&rsv->allocated);
@@ -403,7 +404,7 @@ int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr,
 			uint64_t address, struct amdgpu_vram_block_info *info)
 {
 	struct amdgpu_vram_mgr_resource *vres;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	u64 start, size;
 	int ret = -ENOENT;
 
@@ -450,8 +451,8 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	struct amdgpu_vram_mgr_resource *vres;
 	u64 size, remaining_size, lpfn, fpfn;
 	unsigned int adjust_dcc_size = 0;
-	struct drm_buddy *mm = &mgr->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &mgr->mm;
+	struct gpu_buddy_block *block;
 	unsigned long pages_per_block;
 	int r;
 
@@ -493,17 +494,17 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	INIT_LIST_HEAD(&vres->blocks);
 
 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
-		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
+		vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
 
 	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
-		vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
+		vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
 
 	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED)
-		vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION;
+		vres->flags |= GPU_BUDDY_CLEAR_ALLOCATION;
 
 	if (fpfn || lpfn != mgr->mm.size)
 		/* Allocate blocks in desired range */
-		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
+		vres->flags |= GPU_BUDDY_RANGE_ALLOCATION;
 
 	if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC &&
 	    adev->gmc.gmc_funcs->get_dcc_alignment)
@@ -516,7 +517,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 		dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size);
 		remaining_size = (u64)dcc_size;
 
-		vres->flags |= DRM_BUDDY_TRIM_DISABLE;
+		vres->flags |= GPU_BUDDY_TRIM_DISABLE;
 	}
 
 	mutex_lock(&mgr->lock);
@@ -536,7 +537,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 
 		BUG_ON(min_block_size < mm->chunk_size);
 
-		r = drm_buddy_alloc_blocks(mm, fpfn,
+		r = gpu_buddy_alloc_blocks(mm, fpfn,
 					   lpfn,
 					   size,
 					   min_block_size,
@@ -545,7 +546,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 
 		if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul &&
 		    !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) {
-			vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION;
+			vres->flags &= ~GPU_BUDDY_CONTIGUOUS_ALLOCATION;
 			pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT),
 						tbo->page_alignment);
 
@@ -566,7 +567,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	list_add_tail(&vres->vres_node, &mgr->allocated_vres_list);
 
 	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {
-		struct drm_buddy_block *dcc_block;
+		struct gpu_buddy_block *dcc_block;
 		unsigned long dcc_start;
 		u64 trim_start;
 
@@ -576,7 +577,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 			roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block),
 				adjust_dcc_size);
 		trim_start = (u64)dcc_start;
-		drm_buddy_block_trim(mm, &trim_start,
+		gpu_buddy_block_trim(mm, &trim_start,
 				     (u64)vres->base.size,
 				     &vres->blocks);
 	}
@@ -614,7 +615,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	return 0;
 
 error_free_blocks:
-	drm_buddy_free_list(mm, &vres->blocks, 0);
+	gpu_buddy_free_list(mm, &vres->blocks, 0);
 	mutex_unlock(&mgr->lock);
 error_fini:
 	ttm_resource_fini(man, &vres->base);
@@ -637,8 +638,8 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
-	struct drm_buddy *mm = &mgr->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &mgr->mm;
+	struct gpu_buddy_block *block;
 	uint64_t vis_usage = 0;
 
 	mutex_lock(&mgr->lock);
@@ -649,7 +650,7 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
 	list_for_each_entry(block, &vres->blocks, link)
 		vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
 
-	drm_buddy_free_list(mm, &vres->blocks, vres->flags);
+	gpu_buddy_free_list(mm, &vres->blocks, vres->flags);
 	amdgpu_vram_mgr_do_reserve(man);
 	mutex_unlock(&mgr->lock);
 
@@ -688,7 +689,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
 	if (!*sgt)
 		return -ENOMEM;
 
-	/* Determine the number of DRM_BUDDY blocks to export */
+	/* Determine the number of GPU_BUDDY blocks to export */
 	amdgpu_res_first(res, offset, length, &cursor);
 	while (cursor.remaining) {
 		num_entries++;
@@ -704,10 +705,10 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
 		sg->length = 0;
 
 	/*
-	 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
-	 * @note: Use iterator api to get first the DRM_BUDDY block
+	 * Walk down GPU_BUDDY blocks to populate scatterlist nodes
+	 * @note: Use iterator api to get first the GPU_BUDDY block
 	 * and the number of bytes from it. Access the following
-	 * DRM_BUDDY block(s) if more buffer needs to exported
+	 * GPU_BUDDY block(s) if more buffer needs to exported
 	 */
 	amdgpu_res_first(res, offset, length, &cursor);
 	for_each_sgtable_sg((*sgt), sg, i) {
@@ -792,10 +793,10 @@ uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
 void amdgpu_vram_mgr_clear_reset_blocks(struct amdgpu_device *adev)
 {
 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 
 	mutex_lock(&mgr->lock);
-	drm_buddy_reset_clear(mm, false);
+	gpu_buddy_reset_clear(mm, false);
 	mutex_unlock(&mgr->lock);
 }
 
@@ -815,7 +816,7 @@ static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
 				       size_t size)
 {
 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 
 	/* Check each drm buddy block individually */
 	list_for_each_entry(block, &mgr->blocks, link) {
@@ -848,7 +849,7 @@ static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
 				       size_t size)
 {
 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 
 	/* Check each drm buddy block individually */
 	list_for_each_entry(block, &mgr->blocks, link) {
@@ -877,7 +878,7 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
 				  struct drm_printer *printer)
 {
 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 	struct amdgpu_vram_reservation *rsv;
 
 	drm_printf(printer, "  vis usage:%llu\n",
@@ -930,7 +931,7 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
 	mgr->default_page_size = PAGE_SIZE;
 
 	man->func = &amdgpu_vram_mgr_func;
-	err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
+	err = gpu_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
 	if (err)
 		return err;
 
@@ -965,11 +966,11 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
 		kfree(rsv);
 
 	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
-		drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
+		gpu_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
 		kfree(rsv);
 	}
 	if (!adev->gmc.is_app_apu)
-		drm_buddy_fini(&mgr->mm);
+		gpu_buddy_fini(&mgr->mm);
 	mutex_unlock(&mgr->lock);
 
 	ttm_resource_manager_cleanup(man);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
index 5f5fd9a911c26..429a21a2e9b23 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
@@ -24,11 +24,11 @@
 #ifndef __AMDGPU_VRAM_MGR_H__
 #define __AMDGPU_VRAM_MGR_H__
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 
 struct amdgpu_vram_mgr {
 	struct ttm_resource_manager manager;
-	struct drm_buddy mm;
+	struct gpu_buddy mm;
 	/* protects access to buffer objects */
 	struct mutex lock;
 	struct list_head reservations_pending;
@@ -57,19 +57,19 @@ struct amdgpu_vram_mgr_resource {
 	struct amdgpu_vres_task task;
 };
 
-static inline u64 amdgpu_vram_mgr_block_start(struct drm_buddy_block *block)
+static inline u64 amdgpu_vram_mgr_block_start(struct gpu_buddy_block *block)
 {
-	return drm_buddy_block_offset(block);
+	return gpu_buddy_block_offset(block);
 }
 
-static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block)
+static inline u64 amdgpu_vram_mgr_block_size(struct gpu_buddy_block *block)
 {
-	return (u64)PAGE_SIZE << drm_buddy_block_order(block);
+	return (u64)PAGE_SIZE << gpu_buddy_block_order(block);
 }
 
-static inline bool amdgpu_vram_mgr_is_cleared(struct drm_buddy_block *block)
+static inline bool amdgpu_vram_mgr_is_cleared(struct gpu_buddy_block *block)
 {
-	return drm_buddy_block_is_clear(block);
+	return gpu_buddy_block_is_clear(block);
 }
 
 static inline struct amdgpu_vram_mgr_resource *
@@ -82,8 +82,8 @@ static inline void amdgpu_vram_mgr_set_cleared(struct ttm_resource *res)
 {
 	struct amdgpu_vram_mgr_resource *ares = to_amdgpu_vram_mgr_resource(res);
 
-	WARN_ON(ares->flags & DRM_BUDDY_CLEARED);
-	ares->flags |= DRM_BUDDY_CLEARED;
+	WARN_ON(ares->flags & GPU_BUDDY_CLEARED);
+	ares->flags |= GPU_BUDDY_CLEARED;
 }
 
 int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 11e56df1d91b7..e63d05c477a08 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -338,7 +338,7 @@ static u32 xgmi_v6_4_get_link_status(struct amdgpu_device *adev, int global_link
 	if (!(adev->aid_mask & BIT(i)))
 		return U32_MAX;
 
-	addr += adev->asic_funcs->encode_ext_smn_addressing(i);
+	addr += amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, i);
 
 	return RREG32_PCIE_EXT(addr);
 }
@@ -347,6 +347,9 @@ int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
 {
 	u32 xgmi_state_reg_val;
 
+	if (amdgpu_sriov_vf(adev))
+		return AMDGPU_XGMI_LINK_NA;
+
 	if (adev->gmc.xgmi.num_physical_nodes <= 1)
 		return -EINVAL;
 
@@ -469,12 +472,12 @@ static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(ddev);
 	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
-	int i;
+	int i, offset = 0;
 
 	for (i = 0; i < top->num_nodes; i++)
-		sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);
+		offset += sysfs_emit_at(buf, offset, "%02x ", top->nodes[i].num_hops);
 
-	return sysfs_emit(buf, "%s\n", buf);
+	return offset + sysfs_emit_at(buf, offset, "\n");
 }
 
 static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
@@ -484,12 +487,12 @@ static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(ddev);
 	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
-	int i;
+	int i, offset = 0;
 
 	for (i = 0; i < top->num_nodes; i++)
-		sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);
+		offset += sysfs_emit_at(buf, offset, "%02x ", top->nodes[i].num_links);
 
-	return sysfs_emit(buf, "%s\n", buf);
+	return offset + sysfs_emit_at(buf, offset, "\n");
 }
 
 static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
@@ -1290,7 +1293,10 @@ static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
 
 static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
 {
-	WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
+	uint64_t smn_base =
+		amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst);
+
+	WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL);
 }
 
 static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
@@ -1500,6 +1506,7 @@ static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct a
 					    u64 mca_base, struct ras_err_data *err_data)
 {
 	int xgmi_inst = mcm_info->die_id;
+	uint64_t smn_base;
 	u64 status = 0;
 
 	status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
@@ -1516,8 +1523,8 @@ static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct a
 	default:
 		break;
 	}
-
-	WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
+	smn_base = amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst);
+	WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL);
 }
 
 static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
index cffb2f805de24..847cfd1fd0046 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
@@ -161,7 +161,8 @@ union amd_sriov_msg_feature_flags {
 		uint32_t ras_telemetry		: 1;
 		uint32_t ras_cper		: 1;
 		uint32_t xgmi_ta_ext_peer_link	: 1;
-		uint32_t reserved		: 19;
+		uint32_t xgmi_connected_to_cpu  : 1;
+		uint32_t reserved		: 18;
 	} flags;
 	uint32_t all;
 };
@@ -470,14 +471,23 @@ struct amd_sriov_ras_chk_criti {
 	uint32_t hit;
 };
 
+union amd_sriov_ras_host_push {
+	struct amd_sriov_ras_telemetry_error_count error_count;
+	struct amd_sriov_ras_cper_dump cper_dump;
+	struct amd_sriov_ras_chk_criti chk_criti;
+};
+
+#define AMD_SRIOV_UNIRAS_BLOCKS_BUF_SIZE 4096
+#define AMD_SRIOV_UNIRAS_CMD_MAX_SIZE (4096 * 13)
+struct amd_sriov_uniras_shared_mem {
+	uint8_t blocks_ecc_buf[AMD_SRIOV_UNIRAS_BLOCKS_BUF_SIZE];
+	uint8_t cmd_buf[AMD_SRIOV_UNIRAS_CMD_MAX_SIZE];
+};
+
 struct amdsriov_ras_telemetry {
 	struct amd_sriov_ras_telemetry_header header;
-
-	union {
-		struct amd_sriov_ras_telemetry_error_count error_count;
-		struct amd_sriov_ras_cper_dump cper_dump;
-		struct amd_sriov_ras_chk_criti chk_criti;
-	} body;
+	union amd_sriov_ras_host_push body;
+	struct amd_sriov_uniras_shared_mem uniras_shared_mem;
 };
 
 /* version data stored in MAILBOX_MSGBUF_RCV_DW1 for future expansion */
@@ -510,6 +520,10 @@ _Static_assert(AMD_SRIOV_MSG_RESERVE_UCODE % 4 == 0,
 _Static_assert(AMD_SRIOV_MSG_RESERVE_UCODE > AMD_SRIOV_UCODE_ID__MAX,
 	       "AMD_SRIOV_MSG_RESERVE_UCODE must be bigger than AMD_SRIOV_UCODE_ID__MAX");
 
+_Static_assert(
+	sizeof(struct amdsriov_ras_telemetry) <= AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10,
+"amdsriov_ras_telemetry must be " stringification(AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1) " KB");
+
 #undef _stringification
 #undef stringification
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
index d9842aa25283a..72ea37dbfea8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
@@ -58,25 +58,6 @@ void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
 	adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT << 1;
 }
 
-/* Fixed pattern for smn addressing on different AIDs:
- *   bit[34]: indicate cross AID access
- *   bit[33:32]: indicate target AID id
- * AID id range is 0 ~ 3 as maximum AID number is 4.
- */
-u64 aqua_vanjaram_encode_ext_smn_addressing(int ext_id)
-{
-	u64 ext_offset;
-
-	/* local routing and bit[34:32] will be zeros */
-	if (ext_id == 0)
-		return 0;
-
-	/* Initiated from host, accessing to all non-zero aids are cross traffic */
-	ext_offset = ((u64)(ext_id & 0x3) << 32) | (1ULL << 34);
-
-	return ext_offset;
-}
-
 static enum amdgpu_gfx_partition
 __aqua_vanjaram_calc_xcp_mode(struct amdgpu_xcp_mgr *xcp_mgr)
 {
@@ -590,7 +571,7 @@ static void aqua_read_smn_ext(struct amdgpu_device *adev,
 			      uint64_t smn_addr, int i)
 {
 	regdata->addr =
-		smn_addr + adev->asic_funcs->encode_ext_smn_addressing(i);
+		smn_addr + amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, i);
 	regdata->value = RREG32_PCIE_EXT(regdata->addr);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index e4ce3029d3fb7..6e37961f6be53 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1462,8 +1462,6 @@ static void atom_get_vbios_pn(struct atom_context *ctx)
 
 		ctx->vbios_pn[count] = 0;
 	}
-
-	drm_info(ctx->card->dev, "ATOM BIOS: %s\n", ctx->vbios_pn);
 }
 
 static void atom_get_vbios_version(struct atom_context *ctx)
@@ -1520,6 +1518,30 @@ static void atom_get_vbios_build(struct atom_context *ctx)
 		strscpy(ctx->build_num, str, len);
 }
 
+static inline void atom_print_vbios_info(struct atom_context *ctx)
+{
+	char vbios_info[256];
+	int off = 0;
+
+	if (ctx->vbios_pn[0])
+		off += scnprintf(vbios_info + off, sizeof(vbios_info) - off,
+				 "%s", ctx->vbios_pn);
+	if (ctx->build_num[0])
+		off += scnprintf(vbios_info + off, sizeof(vbios_info) - off,
+				 "%sbuild: %s", off ? ", " : "",
+				 ctx->build_num);
+	if (ctx->vbios_ver_str[0])
+		off += scnprintf(vbios_info + off, sizeof(vbios_info) - off,
+				 "%sver: %s", off ? ", " : "",
+				 ctx->vbios_ver_str);
+	if (ctx->date[0])
+		off += scnprintf(vbios_info + off, sizeof(vbios_info) - off,
+				 "%s%.10s", off ? ", " : "",
+				 ctx->date);
+	if (off)
+		drm_info(ctx->card->dev, "ATOM BIOS: %s\n", vbios_info);
+}
+
 struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
 {
 	int base;
@@ -1582,6 +1604,8 @@ struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
 	atom_get_vbios_version(ctx);
 	atom_get_vbios_build(ctx);
 
+	atom_print_vbios_info(ctx);
+
 	return ctx;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index c081784a19c4f..29954c7d61b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -154,11 +154,11 @@ static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(mmPCIE_INDEX, reg);
 	(void)RREG32(mmPCIE_INDEX);
 	r = RREG32(mmPCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -166,12 +166,12 @@ static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(mmPCIE_INDEX, reg);
 	(void)RREG32(mmPCIE_INDEX);
 	WREG32(mmPCIE_DATA, v);
 	(void)RREG32(mmPCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
@@ -179,10 +179,10 @@ static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmSMC_IND_INDEX_0, (reg));
 	r = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 	return r;
 }
 
@@ -190,10 +190,10 @@ static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmSMC_IND_INDEX_0, (reg));
 	WREG32(mmSMC_IND_DATA_0, (v));
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 }
 
 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
@@ -201,10 +201,10 @@ static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	r = RREG32(mmUVD_CTX_DATA);
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 	return r;
 }
 
@@ -212,10 +212,10 @@ static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	WREG32(mmUVD_CTX_DATA, (v));
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 }
 
 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
@@ -223,10 +223,10 @@ static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(mmDIDT_IND_INDEX, (reg));
 	r = RREG32(mmDIDT_IND_DATA);
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 	return r;
 }
 
@@ -234,10 +234,10 @@ static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(mmDIDT_IND_INDEX, (reg));
 	WREG32(mmDIDT_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 }
 
 static const u32 bonaire_golden_spm_registers[] =
@@ -1027,7 +1027,7 @@ static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 	dw_ptr = (u32 *)bios;
 	length_dw = ALIGN(length_bytes, 4) / 4;
 	/* take the smc lock since we are using the smc index */
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	/* set rom index to 0 */
 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 	WREG32(mmSMC_IND_DATA_0, 0);
@@ -1035,7 +1035,7 @@ static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 	for (i = 0; i < length_dw; i++)
 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return true;
 }
@@ -1984,14 +1984,14 @@ static int cik_common_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->smc_rreg = &cik_smc_rreg;
-	adev->smc_wreg = &cik_smc_wreg;
-	adev->pcie_rreg = &cik_pcie_rreg;
-	adev->pcie_wreg = &cik_pcie_wreg;
-	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
-	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
-	adev->didt_rreg = &cik_didt_rreg;
-	adev->didt_wreg = &cik_didt_wreg;
+	adev->reg.smc.rreg = cik_smc_rreg;
+	adev->reg.smc.wreg = cik_smc_wreg;
+	adev->reg.pcie.rreg = &cik_pcie_rreg;
+	adev->reg.pcie.wreg = &cik_pcie_wreg;
+	adev->reg.uvd_ctx.rreg = &cik_uvd_ctx_rreg;
+	adev->reg.uvd_ctx.wreg = &cik_uvd_ctx_wreg;
+	adev->reg.didt.rreg = &cik_didt_rreg;
+	adev->reg.didt.wreg = &cik_didt_wreg;
 
 	adev->asic_funcs = &cik_asic_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 9e8715b4739da..22780c09177d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -53,7 +53,6 @@ static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
 static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
-static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);
 static int cik_sdma_soft_reset(struct amdgpu_ip_block *ip_block);
 
 u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);
@@ -919,6 +918,14 @@ static void cik_enable_sdma_mgls(struct amdgpu_device *adev,
 	}
 }
 
+static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = cik_sdma_vm_copy_pte,
+
+	.write_pte = cik_sdma_vm_write_pte,
+	.set_pte_pde = cik_sdma_vm_set_pte_pde,
+};
+
 static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -933,7 +940,7 @@ static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block)
 	cik_sdma_set_ring_funcs(adev);
 	cik_sdma_set_irq_funcs(adev);
 	cik_sdma_set_buffer_funcs(adev);
-	cik_sdma_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &cik_sdma_vm_pte_funcs);
 
 	return 0;
 }
@@ -1337,26 +1344,6 @@ static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = cik_sdma_vm_copy_pte,
-
-	.write_pte = cik_sdma_vm_write_pte,
-	.set_pte_pde = cik_sdma_vm_set_pte_pde,
-};
-
-static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version cik_sdma_ip_block =
 {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index a7ffe10eea1b4..c8f465158e711 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -175,10 +175,10 @@ static u32 dce_v10_0_audio_endpt_rreg(struct amdgpu_device *adev,
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset, reg);
 	r = RREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 
 	return r;
 }
@@ -188,10 +188,10 @@ static void dce_v10_0_audio_endpt_wreg(struct amdgpu_device *adev,
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset, reg);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset, v);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 }
 
 static u32 dce_v10_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
@@ -1298,7 +1298,7 @@ static void dce_v10_0_audio_write_speaker_allocation(struct drm_encoder *encoder
 		return;
 	}
 
-	sad_count = drm_edid_to_speaker_allocation(amdgpu_connector->edid, &sadb);
+	sad_count = drm_edid_to_speaker_allocation(drm_edid_raw(amdgpu_connector->edid), &sadb);
 	if (sad_count < 0) {
 		DRM_ERROR("Couldn't read Speaker Allocation Data Block: %d\n", sad_count);
 		sad_count = 0;
@@ -1368,7 +1368,7 @@ static void dce_v10_0_audio_write_sad_regs(struct drm_encoder *encoder)
 		return;
 	}
 
-	sad_count = drm_edid_to_sad(amdgpu_connector->edid, &sads);
+	sad_count = drm_edid_to_sad(drm_edid_raw(amdgpu_connector->edid), &sads);
 	if (sad_count < 0)
 		DRM_ERROR("Couldn't read SADs: %d\n", sad_count);
 	if (sad_count <= 0)
@@ -2750,8 +2750,8 @@ static int dce_v10_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->audio_endpt_rreg = &dce_v10_0_audio_endpt_rreg;
-	adev->audio_endpt_wreg = &dce_v10_0_audio_endpt_wreg;
+	adev->reg.audio_endpt.rreg = &dce_v10_0_audio_endpt_rreg;
+	adev->reg.audio_endpt.wreg = &dce_v10_0_audio_endpt_wreg;
 
 	dce_v10_0_set_display_funcs(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index a72e20db5363b..58d0da5c2a74f 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -138,10 +138,10 @@ static u32 dce_v6_0_audio_endpt_rreg(struct amdgpu_device *adev,
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset, reg);
 	r = RREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 
 	return r;
 }
@@ -151,11 +151,11 @@ static void dce_v6_0_audio_endpt_wreg(struct amdgpu_device *adev,
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset,
 		reg | AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_WRITE_EN_MASK);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset, v);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 }
 
 static u32 dce_v6_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
@@ -1265,7 +1265,7 @@ static void dce_v6_0_audio_write_speaker_allocation(struct drm_encoder *encoder)
 		return;
 	}
 
-	sad_count = drm_edid_to_speaker_allocation(amdgpu_connector->edid, &sadb);
+	sad_count = drm_edid_to_speaker_allocation(drm_edid_raw(amdgpu_connector->edid), &sadb);
 	if (sad_count < 0) {
 		DRM_ERROR("Couldn't read Speaker Allocation Data Block: %d\n", sad_count);
 		sad_count = 0;
@@ -1346,7 +1346,7 @@ static void dce_v6_0_audio_write_sad_regs(struct drm_encoder *encoder)
 		return;
 	}
 
-	sad_count = drm_edid_to_sad(amdgpu_connector->edid, &sads);
+	sad_count = drm_edid_to_sad(drm_edid_raw(amdgpu_connector->edid), &sads);
 	if (sad_count < 0)
 		DRM_ERROR("Couldn't read SADs: %d\n", sad_count);
 	if (sad_count <= 0)
@@ -2697,8 +2697,8 @@ static int dce_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->audio_endpt_rreg = &dce_v6_0_audio_endpt_rreg;
-	adev->audio_endpt_wreg = &dce_v6_0_audio_endpt_wreg;
+	adev->reg.audio_endpt.rreg = &dce_v6_0_audio_endpt_rreg;
+	adev->reg.audio_endpt.wreg = &dce_v6_0_audio_endpt_wreg;
 
 	dce_v6_0_set_display_funcs(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 4221c7b7c5060..6d19f6d94d252 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -126,10 +126,10 @@ static u32 dce_v8_0_audio_endpt_rreg(struct amdgpu_device *adev,
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset, reg);
 	r = RREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 
 	return r;
 }
@@ -139,10 +139,10 @@ static void dce_v8_0_audio_endpt_wreg(struct amdgpu_device *adev,
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->audio_endpt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.audio_endpt.lock, flags);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_INDEX + block_offset, reg);
 	WREG32(mmAZALIA_F0_CODEC_ENDPOINT_DATA + block_offset, v);
-	spin_unlock_irqrestore(&adev->audio_endpt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.audio_endpt.lock, flags);
 }
 
 static u32 dce_v8_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
@@ -1271,7 +1271,7 @@ static void dce_v8_0_audio_write_speaker_allocation(struct drm_encoder *encoder)
 		return;
 	}
 
-	sad_count = drm_edid_to_speaker_allocation(amdgpu_connector->edid, &sadb);
+	sad_count = drm_edid_to_speaker_allocation(drm_edid_raw(amdgpu_connector->edid), &sadb);
 	if (sad_count < 0) {
 		DRM_ERROR("Couldn't read Speaker Allocation Data Block: %d\n", sad_count);
 		sad_count = 0;
@@ -1339,7 +1339,7 @@ static void dce_v8_0_audio_write_sad_regs(struct drm_encoder *encoder)
 		return;
 	}
 
-	sad_count = drm_edid_to_sad(amdgpu_connector->edid, &sads);
+	sad_count = drm_edid_to_sad(drm_edid_raw(amdgpu_connector->edid), &sads);
 	if (sad_count < 0)
 		DRM_ERROR("Couldn't read SADs: %d\n", sad_count);
 	if (sad_count <= 0)
@@ -2655,8 +2655,8 @@ static int dce_v8_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->audio_endpt_rreg = &dce_v8_0_audio_endpt_rreg;
-	adev->audio_endpt_wreg = &dce_v8_0_audio_endpt_wreg;
+	adev->reg.audio_endpt.rreg = &dce_v8_0_audio_endpt_rreg;
+	adev->reg.audio_endpt.wreg = &dce_v8_0_audio_endpt_wreg;
 
 	dce_v8_0_set_display_funcs(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index 621aeca538803..7e7e6c389895a 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -51,7 +51,7 @@ static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev,
 	address = adev->nbio.funcs->get_pcie_index_offset(adev);
 	data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 	WREG32(data, ficaa_val);
 
@@ -61,7 +61,7 @@ static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev,
 	WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 	ficadh_val = RREG32(data);
 
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 
 	return (((ficadh_val & 0xFFFFFFFFFFFFFFFF) << 32) | ficadl_val);
 }
@@ -74,7 +74,7 @@ static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val,
 	address = adev->nbio.funcs->get_pcie_index_offset(adev);
 	data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 	WREG32(data, ficaa_val);
 
@@ -84,7 +84,7 @@ static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val,
 	WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 	WREG32(data, ficadh_val);
 
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 /*
@@ -102,12 +102,12 @@ static void df_v3_6_perfmon_rreg(struct amdgpu_device *adev,
 	address = adev->nbio.funcs->get_pcie_index_offset(adev);
 	data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(address, lo_addr);
 	*lo_val = RREG32(data);
 	WREG32(address, hi_addr);
 	*hi_val = RREG32(data);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 /*
@@ -124,12 +124,12 @@ static void df_v3_6_perfmon_wreg(struct amdgpu_device *adev, uint32_t lo_addr,
 	address = adev->nbio.funcs->get_pcie_index_offset(adev);
 	data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(address, lo_addr);
 	WREG32(data, lo_val);
 	WREG32(address, hi_addr);
 	WREG32(data, hi_val);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 /* same as perfmon_wreg but return status on write value check */
@@ -143,7 +143,7 @@ static int df_v3_6_perfmon_arm_with_status(struct amdgpu_device *adev,
 	address = adev->nbio.funcs->get_pcie_index_offset(adev);
 	data = adev->nbio.funcs->get_pcie_data_offset(adev);
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(address, lo_addr);
 	WREG32(data, lo_val);
 	WREG32(address, hi_addr);
@@ -153,7 +153,7 @@ static int df_v3_6_perfmon_arm_with_status(struct amdgpu_device *adev,
 	lo_val_rb = RREG32(data);
 	WREG32(address, hi_addr);
 	hi_val_rb = RREG32(data);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 
 	if (!(lo_val == lo_val_rb && hi_val == hi_val_rb))
 		return -EBUSY;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 427975b5a1d97..b1a1b8a10a08b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -4238,6 +4238,37 @@ static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
 	return gfx_v11_0_cp_gfx_start(adev);
 }
 
+static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev,
+					      struct v11_compute_mqd *mqd,
+					      struct amdgpu_mqd_prop *prop)
+{
+	uint32_t se_mask[8] = {0};
+	uint32_t wa_mask;
+	bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE |
+					  AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE);
+
+	if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count))
+		return;
+
+	if (has_wa_flag) {
+		wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ?
+			  0xffff : 0xffffffff;
+		mqd->compute_static_thread_mgmt_se0 = wa_mask;
+		mqd->compute_static_thread_mgmt_se1 = wa_mask;
+		mqd->compute_static_thread_mgmt_se2 = wa_mask;
+		mqd->compute_static_thread_mgmt_se3 = wa_mask;
+		return;
+	}
+
+	amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask,
+						prop->cu_mask_count, se_mask);
+
+	mqd->compute_static_thread_mgmt_se0 = se_mask[0];
+	mqd->compute_static_thread_mgmt_se1 = se_mask[1];
+	mqd->compute_static_thread_mgmt_se2 = se_mask[2];
+	mqd->compute_static_thread_mgmt_se3 = se_mask[3];
+}
+
 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 				      struct amdgpu_mqd_prop *prop)
 {
@@ -4372,6 +4403,8 @@ static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	/* set UQ fenceaddress */
 	mqd->fence_address_lo = lower_32_bits(prop->fence_address);
 	mqd->fence_address_hi = upper_32_bits(prop->fence_address);
+	/* set CU mask */
+	gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 79ea1af363a53..a418ae609c363 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -3109,6 +3109,37 @@ static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
 	return gfx_v12_0_cp_gfx_start(adev);
 }
 
+static void gfx_v12_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev,
+					      struct v12_compute_mqd *mqd,
+					      struct amdgpu_mqd_prop *prop)
+{
+	uint32_t se_mask[8] = {0};
+	uint32_t wa_mask;
+	bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE |
+					  AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE);
+
+	if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count))
+		return;
+
+	if (has_wa_flag) {
+		wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ?
+			  0xffff : 0xffffffff;
+		mqd->compute_static_thread_mgmt_se0 = wa_mask;
+		mqd->compute_static_thread_mgmt_se1 = wa_mask;
+		mqd->compute_static_thread_mgmt_se2 = wa_mask;
+		mqd->compute_static_thread_mgmt_se3 = wa_mask;
+		return;
+	}
+
+	amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask,
+						prop->cu_mask_count, se_mask);
+
+	mqd->compute_static_thread_mgmt_se0 = se_mask[0];
+	mqd->compute_static_thread_mgmt_se1 = se_mask[1];
+	mqd->compute_static_thread_mgmt_se2 = se_mask[2];
+	mqd->compute_static_thread_mgmt_se3 = se_mask[3];
+}
+
 static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 				      struct amdgpu_mqd_prop *prop)
 {
@@ -3242,6 +3273,8 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	/* set UQ fenceaddress */
 	mqd->fence_address_lo = lower_32_bits(prop->fence_address);
 	mqd->fence_address_hi = upper_32_bits(prop->fence_address);
+	/* set CU mask */
+	gfx_v12_0_compute_mqd_set_cu_mask(adev, mqd, prop);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index eb9725ae1607a..db49582a211f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -45,6 +45,7 @@
 #include "v12_structs.h"
 #include "gfx_v12_1.h"
 #include "mes_v12_1.h"
+#include "amdgpu_ras_mgr.h"
 
 #define GFX12_MEC_HPD_SIZE	2048
 #define NUM_SIMD_PER_CU_GFX12_1	4
@@ -136,7 +137,6 @@ static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring,
 			  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
 			  PACKET3_MAP_QUEUES_ME((me)) |
 			  PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
-			  PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
 			  PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
 			  PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
 	amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
@@ -245,8 +245,7 @@ static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 			  /* memory (1) or register (0) */
 			  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
 			   WAIT_REG_MEM_OPERATION(opt) | /* wait */
-			   WAIT_REG_MEM_FUNCTION(3) |  /* equal */
-			   WAIT_REG_MEM_ENGINE(eng_sel)));
+			   WAIT_REG_MEM_FUNCTION(3)));  /* equal */
 
 	if (mem_space)
 		BUG_ON(addr0 & 0x3); /* Dword align */
@@ -1155,11 +1154,13 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 		break;
 	}
 
-	/* recalculate compute rings to use based on hardware configuration */
-	num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
-			     adev->gfx.mec.num_queue_per_pipe) / 2;
-	adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
-					  num_compute_rings);
+	if (adev->gfx.num_compute_rings) {
+		/* recalculate compute rings to use based on hardware configuration */
+		num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
+				     adev->gfx.mec.num_queue_per_pipe) / 2;
+		adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
+						  num_compute_rings);
+	}
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 
@@ -1184,6 +1185,13 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	/* RLC POISON Error */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_RLC,
+				GFX_12_1_0__SRCID__RLC_POISON_INTERRUPT,
+				&adev->gfx.rlc_poison_irq);
+	if (r)
+		return r;
+
 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
 
 	r = gfx_v12_1_rlc_init(adev);
@@ -1405,7 +1413,7 @@ static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
 	/*
 	 * Configure apertures:
 	 * LDS:         0x20000000'00000000 - 0x20000001'00000000 (4GB)
-	 * Scratch:     0x10000000'00000000 - 0x10000001'00000000 (4GB)
+	 * Scratch:     0x10000000'00000000 - 0x11ffffff'ffffffff (128PB 57-bit)
 	 */
 	sh_mem_bases = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
 				     (adev->gmc.private_aperture_start >> 58));
@@ -2629,24 +2637,6 @@ static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev,
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG, data);
 }
 
-static void gfx_v12_1_xcc_setup_tcp_thrashing_ctrl(struct amdgpu_device *adev,
-					 int xcc_id)
-{
-	uint32_t val;
-
-	/* Set the TCP UTCL0 register to enable atomics */
-	val = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
-					regTCP_UTCL0_THRASHING_CTRL);
-	val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL, THRASHING_EN, 0x2);
-	val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL,
-					RETRY_FRAGMENT_THRESHOLD_UP_EN, 0x1);
-	val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL,
-					RETRY_FRAGMENT_THRESHOLD_DOWN_EN, 0x1);
-
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
-					regTCP_UTCL0_THRASHING_CTRL, val);
-}
-
 static void gfx_v12_1_xcc_enable_atomics(struct amdgpu_device *adev,
 					 int xcc_id)
 {
@@ -2695,7 +2685,6 @@ static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev)
 	for (i = 0; i < NUM_XCC(adev->gfx.xcc_mask); i++) {
 		gfx_v12_1_xcc_disable_burst(adev, i);
 		gfx_v12_1_xcc_enable_atomics(adev, i);
-		gfx_v12_1_xcc_setup_tcp_thrashing_ctrl(adev, i);
 		gfx_v12_1_xcc_disable_early_write_ack(adev, i);
 		gfx_v12_1_xcc_disable_tcp_spill_cache(adev, i);
 	}
@@ -2794,6 +2783,33 @@ static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev,
 	gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
 }
 
+static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev,
+					      bool enable)
+{
+	unsigned int irq_type;
+	int m, p, r;
+
+	if (adev->gfx.disable_kq) {
+		for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
+			for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
+				irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+					+ (m * adev->gfx.mec.num_pipe_per_mec)
+					+ p;
+				if (enable)
+					r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
+							   irq_type);
+				else
+					r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
+							   irq_type);
+				if (r)
+					return r;
+			}
+		}
+	}
+
+	return 0;
+}
+
 static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -2801,6 +2817,7 @@ static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
 
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+	gfx_v12_1_set_userq_eop_interrupts(adev, false);
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 	for (i = 0; i < num_xcc; i++) {
@@ -2868,10 +2885,26 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+
+	switch (amdgpu_user_queue) {
+	case -1:
+	default:
+		adev->gfx.disable_kq = true;
+		adev->gfx.disable_uq = true;
+		break;
+	case 0:
+		adev->gfx.disable_kq = false;
+		adev->gfx.disable_uq = true;
+		break;
+	}
+
 	adev->gfx.funcs = &gfx_v12_1_gfx_funcs;
 
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	if (adev->gfx.disable_kq)
+		adev->gfx.num_compute_rings = 0;
+	else
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
 
 	gfx_v12_1_set_kiq_pm4_funcs(adev);
 	gfx_v12_1_set_ring_funcs(adev);
@@ -2898,6 +2931,10 @@ static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	r = gfx_v12_1_set_userq_eop_interrupts(adev, true);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -3381,11 +3418,10 @@ static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 
 static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 {
-	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 	uint32_t seq = ring->fence_drv.sync_seq;
 	uint64_t addr = ring->fence_drv.gpu_addr;
 
-	gfx_v12_1_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
+	gfx_v12_1_wait_reg_mem(ring, 0, 1, 0, lower_32_bits(addr),
 			       upper_32_bits(addr), seq, 0xffffffff, 4);
 }
 
@@ -3424,8 +3460,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
 
 	/* write fence seq to the "addr" */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
-				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
+	amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
 	amdgpu_ring_write(ring, lower_32_bits(addr));
 	amdgpu_ring_write(ring, upper_32_bits(addr));
 	amdgpu_ring_write(ring, lower_32_bits(seq));
@@ -3433,8 +3468,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
 	if (flags & AMDGPU_FENCE_FLAG_INT) {
 		/* set register to trigger INT */
 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
-					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
+		amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
 		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
 		amdgpu_ring_write(ring, 0);
 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
@@ -3493,9 +3527,7 @@ static void gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
 						   uint32_t reg0, uint32_t reg1,
 						   uint32_t ref, uint32_t mask)
 {
-	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
-
-	gfx_v12_1_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
+	gfx_v12_1_wait_reg_mem(ring, 0, 0, 1, reg0, reg1,
 			       ref, mask, 0x20);
 }
 
@@ -3630,12 +3662,6 @@ static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
 			return -EINVAL;
 
 		switch (me_id) {
-		case 0:
-			if (pipe_id == 0)
-				amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
-			else
-				amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
-			break;
 		case 1:
 		case 2:
 			for (i = 0; i < adev->gfx.num_compute_rings; i++) {
@@ -3652,6 +3678,9 @@ static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
 					amdgpu_fence_process(ring);
 			}
 			break;
+		default:
+			dev_dbg(adev->dev, "Unexpected me %d in eop_irq\n", me_id);
+			break;
 		}
 	}
 
@@ -3719,29 +3748,23 @@ static void gfx_v12_1_handle_priv_fault(struct amdgpu_device *adev,
 	if (xcc_id == -EINVAL)
 		return;
 
-	switch (me_id) {
-	case 0:
-		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-			ring = &adev->gfx.gfx_ring[i];
-			/* we only enabled 1 gfx queue per pipe for now */
-			if (ring->me == me_id && ring->pipe == pipe_id)
-				drm_sched_fault(&ring->sched);
-		}
-		break;
-	case 1:
-	case 2:
-		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-			ring = &adev->gfx.compute_ring
+	if (!adev->gfx.disable_kq) {
+		switch (me_id) {
+		case 1:
+		case 2:
+			for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+				ring = &adev->gfx.compute_ring
 					[i +
 					 xcc_id * adev->gfx.num_compute_rings];
-			if (ring->me == me_id && ring->pipe == pipe_id &&
-			    ring->queue == queue_id)
-				drm_sched_fault(&ring->sched);
+				if (ring->me == me_id && ring->pipe == pipe_id &&
+				    ring->queue == queue_id)
+					drm_sched_fault(&ring->sched);
+			}
+			break;
+		default:
+			dev_dbg(adev->dev, "Unexpected me %d in priv_fault\n", me_id);
+			break;
 		}
-		break;
-	default:
-		BUG();
-		break;
 	}
 }
 
@@ -3763,6 +3786,35 @@ static int gfx_v12_1_priv_inst_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	uint32_t rlc_fed_status = 0;
+	uint32_t ras_blk = RAS_BLOCK_ID__GFX;
+	struct ras_ih_info ih_info = {0};
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++)
+		rlc_fed_status |= RREG32(SOC15_REG_OFFSET(GC,
+					GET_INST(GC, i), regRLC_RLCS_FED_STATUS));
+
+	if (!rlc_fed_status)
+		return 0;
+
+	if (REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA0_FED_ERR) ||
+	    REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA1_FED_ERR))
+		ras_blk = RAS_BLOCK_ID__SDMA;
+
+	dev_warn(adev->dev, "RLC %d FED IRQ\n", ras_blk);
+
+	ih_info.block = ras_blk;
+	ih_info.reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+	amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
+	return 0;
+}
+
 static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
 {
 	const unsigned int gcr_cntl =
@@ -3887,6 +3939,10 @@ static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_inst_irq_funcs = {
 	.process = gfx_v12_1_priv_inst_irq,
 };
 
+static const struct amdgpu_irq_src_funcs gfx_v12_1_rlc_poison_irq_funcs = {
+	.process = gfx_v12_1_rlc_poison_irq,
+};
+
 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
@@ -3897,6 +3953,9 @@ static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
 
 	adev->gfx.priv_inst_irq.num_types = 1;
 	adev->gfx.priv_inst_irq.funcs = &gfx_v12_1_priv_inst_irq_funcs;
+
+	adev->gfx.rlc_poison_irq.num_types = 1;
+	adev->gfx.rlc_poison_irq.funcs = &gfx_v12_1_rlc_poison_irq_funcs;
 }
 
 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
index 9a5c82c8db53b..21a07530c64d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
@@ -53,16 +53,10 @@
 
 /* Packet 3 types */
 #define	PACKET3_NOP					0x10
-#define	PACKET3_SET_BASE				0x11
-#define		PACKET3_BASE_INDEX(x)                  ((x) << 0)
-#define			CE_PARTITION_BASE		3
 #define	PACKET3_CLEAR_STATE				0x12
 #define	PACKET3_INDEX_BUFFER_SIZE			0x13
 #define	PACKET3_DISPATCH_DIRECT				0x15
 #define	PACKET3_DISPATCH_INDIRECT			0x16
-#define	PACKET3_INDIRECT_BUFFER_END			0x17
-#define	PACKET3_INDIRECT_BUFFER_CNST_END		0x19
-#define	PACKET3_ATOMIC_GDS				0x1D
 #define	PACKET3_ATOMIC_MEM				0x1E
 #define	PACKET3_OCCLUSION_QUERY				0x1F
 #define	PACKET3_SET_PREDICATION				0x20
@@ -74,47 +68,42 @@
 #define	PACKET3_INDEX_BASE				0x26
 #define	PACKET3_DRAW_INDEX_2				0x27
 #define	PACKET3_CONTEXT_CONTROL				0x28
-#define	PACKET3_INDEX_TYPE				0x2A
 #define	PACKET3_DRAW_INDIRECT_MULTI			0x2C
 #define	PACKET3_DRAW_INDEX_AUTO				0x2D
 #define	PACKET3_NUM_INSTANCES				0x2F
 #define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
-#define	PACKET3_INDIRECT_BUFFER_PRIV			0x32
-#define	PACKET3_INDIRECT_BUFFER_CNST			0x33
-#define	PACKET3_COND_INDIRECT_BUFFER_CNST		0x33
-#define	PACKET3_STRMOUT_BUFFER_UPDATE			0x34
 #define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
-#define	PACKET3_DRAW_PREAMBLE				0x36
 #define	PACKET3_WRITE_DATA				0x37
-#define		WRITE_DATA_DST_SEL(x)                   ((x) << 8)
+#define		WRITE_DATA_DST_SEL(x)                   (((x) & 0xf) << 8)
 		/* 0 - register
-		 * 1 - memory (sync - via GRBM)
-		 * 2 - gl2
-		 * 3 - gds
+		 * 1 - reserved
+		 * 2 - tc_l2
+		 * 3 - reserved
 		 * 4 - reserved
-		 * 5 - memory (async - direct)
+		 * 5 - memory (same as tc_l2)
+         * 6 - memory_mapped_adc_persistent_state
 		 */
-#define		WR_ONE_ADDR                             (1 << 16)
+#define		WRITE_DATA_SCOPE(x)                     (((x) & 0x3) << 12)
+#define		WRITE_DATA_MODE(x)                      (((x) & 0x3) << 14)
+        /* 0 - local xcd
+         * 1 - remote/local aid
+         * 2 - remote xcd
+         * 3 - remote mid
+         */
+#define		WRITE_DATA_ADDR_INCR                    (1 << 16)
+#define		WRITE_DATA_MID_DIE_ID(x)                (((x) & 0x3) << 18)
 #define		WR_CONFIRM                              (1 << 20)
-#define		WRITE_DATA_CACHE_POLICY(x)              ((x) << 25)
-		/* 0 - LRU
-		 * 1 - Stream
-		 */
-#define		WRITE_DATA_ENGINE_SEL(x)                ((x) << 30)
-		/* 0 - me
-		 * 1 - pfp
-		 * 2 - ce
+#define		WRITE_DATA_XCD_DIE_ID(x)                (((x) & 0xf) << 21)
+#define		WRITE_DATA_TEMPORAL(x)                  (((x) & 0x3) << 25)
+		/* 0 - rt
+		 * 1 - nt
+         * 2 - ht
+         * 3 - lu
 		 */
+#define		WRITE_DATA_COOP_DISABLE                 (1 << 27)
 #define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
-#define	PACKET3_MEM_SEMAPHORE				0x39
-#              define PACKET3_SEM_USE_MAILBOX       (0x1 << 16)
-#              define PACKET3_SEM_SEL_SIGNAL_TYPE   (0x1 << 20) /* 0 = increment, 1 = write 1 */
-#              define PACKET3_SEM_SEL_SIGNAL	    (0x6 << 29)
-#              define PACKET3_SEM_SEL_WAIT	    (0x7 << 29)
-#define	PACKET3_DRAW_INDEX_MULTI_INST			0x3A
-#define	PACKET3_COPY_DW					0x3B
 #define	PACKET3_WAIT_REG_MEM				0x3C
-#define		WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+#define		WAIT_REG_MEM_FUNCTION(x)                (((x) & 0x7) << 0)
 		/* 0 - always
 		 * 1 - <
 		 * 2 - <=
@@ -123,33 +112,66 @@
 		 * 5 - >=
 		 * 6 - >
 		 */
-#define		WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+#define		WAIT_REG_MEM_MEM_SPACE(x)               (((x) & 0x3) << 4)
 		/* 0 - reg
 		 * 1 - mem
 		 */
-#define		WAIT_REG_MEM_OPERATION(x)               ((x) << 6)
+#define		WAIT_REG_MEM_OPERATION(x)               (((x) & 0x3) << 6)
 		/* 0 - wait_reg_mem
 		 * 1 - wr_wait_wr_reg
 		 */
-#define		WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
-		/* 0 - me
-		 * 1 - pfp
+#define		WAIT_REG_MEM_MODE(x)                    (((x) & 0x3) << 10)
+        /* 0 - local xcd
+         * 1 - remote/local aid
+         * 2 - remote xcd
+         * 3 - remote mid
+         */
+#define		WAIT_REG_MEM_MID_DIE_ID(x)              (((x) & 0x3) << 12)
+#define		WAIT_REG_MEM_XCD_DIE_ID(x)              (((x) & 0xf) << 14)
+#define		WAIT_REG_MEM_MES_INTR_PIPE(x)           (((x) & 0x3) << 22)
+#define		WAIT_REG_MEM_MES_ACTION(x)              (((x) & 0x1) << 24)
+#define		WAIT_REG_MEM_TEMPORAL(x)                (((x) & 0x3) << 25)
+		/* 0 - rt
+		 * 1 - nt
+         * 2 - ht
+         * 3 - lu
 		 */
 #define	PACKET3_INDIRECT_BUFFER				0x3F
 #define		INDIRECT_BUFFER_VALID                   (1 << 23)
-#define		INDIRECT_BUFFER_CACHE_POLICY(x)         ((x) << 28)
-		/* 0 - LRU
-		 * 1 - Stream
-		 * 2 - Bypass
+#define		INDIRECT_BUFFER_TEMPORAL(x)             (x) << 28)
+		/* 0 - rt
+		 * 1 - nt
+		 * 2 - ht
+         * 3 - lu
 		 */
-#define		INDIRECT_BUFFER_PRE_ENB(x)		((x) << 21)
-#define		INDIRECT_BUFFER_PRE_RESUME(x)           ((x) << 30)
 #define	PACKET3_COND_INDIRECT_BUFFER			0x3F
 #define	PACKET3_COPY_DATA				0x40
-#define	PACKET3_CP_DMA					0x41
+#define		COPY_DATA_SRC_SEL(x)                    (((x) & 0xf) << 0)
+#define		COPY_DATA_DST_SEL(x)                    (((x) & 0xf) << 8)
+#define		COPY_DATA_SRC_SCOPE(x)                  (((x) & 0x3) << 4)
+#define		COPY_DATA_DST_SCOPE(x)                  (((x) & 0x3) << 27)
+#define		COPY_DATA_MODE(x)                       (((x) & 0x3) << 6)
+        /* 0 - local xcd
+         * 1 - remote/local aid
+         * 2 - remote xcd
+         * 3 - remote mid
+         */
+#define		COPY_DATA_SRC_TEMPORAL(x)               (((x) & 0x3) << 13)
+#define		COPY_DATA_DST_TEMPORAL(x)               (((x) & 0x3) << 25)
+		/* 0 - rt
+		 * 1 - nt
+         * 2 - ht
+         * 3 - lu
+		 */
+#define		COPY_DATA_COUNT_SEL                     (1 << 16)
+#define		COPY_DATA_SRC_DST_REMOTE_MODE(x)        (((x)) & 0x1 << 16)
+        /* 0 - src remote
+         * 1 - dst remote
+         */
+#define		COPY_DATA_MID_DIE_ID(x)                 (((x) & 0x3) << 18)
+#define		COPY_DATA_XCD_DIE_ID(x)                 (((x) & 0xf) << 21)
+#define		COPY_DATA_PQ_EXE_STATUS                 (1 << 27)
 #define	PACKET3_PFP_SYNC_ME				0x42
-#define	PACKET3_SURFACE_SYNC				0x43
-#define	PACKET3_ME_INITIALIZE				0x44
 #define	PACKET3_COND_WRITE				0x45
 #define	PACKET3_EVENT_WRITE				0x46
 #define		EVENT_TYPE(x)                           ((x) << 0)
@@ -160,8 +182,6 @@
 		 * 3 - SAMPLE_STREAMOUTSTAT*
 		 * 4 - *S_PARTIAL_FLUSH
 		 */
-#define	PACKET3_EVENT_WRITE_EOP				0x47
-#define	PACKET3_EVENT_WRITE_EOS				0x48
 #define	PACKET3_RELEASE_MEM				0x49
 #define		PACKET3_RELEASE_MEM_EVENT_TYPE(x)	    ((x) << 0)
 #define		PACKET3_RELEASE_MEM_EVENT_INDEX(x)	    ((x) << 8)
@@ -180,27 +200,30 @@
 		 * 2 - temporal__release_mem__ht
 		 * 3 - temporal__release_mem__lu
 		 */
-#define		PACKET3_RELEASE_MEM_EXECUTE		(1 << 28)
+#define		PACKET3_RELEASE_MEM_PQ_EXE_STATUS		(1 << 28)
+#define		PACKET3_RELEASE_MEM_GCR_GLK_INV		    (1 << 30)
 
-#define		PACKET3_RELEASE_MEM_DATA_SEL(x)		((x) << 29)
-		/* 0 - discard
-		 * 1 - send low 32bit data
-		 * 2 - send 64bit data
-		 * 3 - send 64bit GPU counter value
-		 * 4 - send 64bit sys counter value
+#define		PACKET3_RELEASE_MEM_DST_SEL(x)		((x) << 16)
+		/* 0 - memory controller
+		 * 1 - TC/L2
+         * 2 - register
 		 */
+#define		PACKET3_RELEASE_MEM_MES_INTR_PIPE(x)    ((x) << 20)
+#define		PACKET3_RELEASE_MEM_MES_ACTION_ID(x)    ((x) << 22)
 #define		PACKET3_RELEASE_MEM_INT_SEL(x)		((x) << 24)
 		/* 0 - none
 		 * 1 - interrupt only (DATA_SEL = 0)
 		 * 2 - interrupt when data write is confirmed
 		 */
-#define		PACKET3_RELEASE_MEM_DST_SEL(x)		((x) << 16)
-		/* 0 - MC
-		 * 1 - TC/L2
+#define		PACKET3_RELEASE_MEM_ADD_DOOREBLL_OFFSET(x)		(1 << 28)
+#define		PACKET3_RELEASE_MEM_DATA_SEL(x)		((x) << 29)
+		/* 0 - discard
+		 * 1 - send low 32bit data
+		 * 2 - send 64bit data
+		 * 3 - send 64bit GPU counter value
+		 * 4 - send 64bit sys counter value
 		 */
 
-
-
 #define	PACKET3_PREAMBLE_CNTL				0x4A
 #              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
 #              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
@@ -218,26 +241,29 @@
 		/* 0 - ME
 		 * 1 - PFP
 		 */
-#              define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13)
-		/* 0 - LRU
-		 * 1 - Stream
+#              define PACKET3_DMA_DATA_SRC_TEMPORAL(x)  ((x) << 13)
+		/* 0 - rt
+		 * 1 - nt
+         * 2 - ht
+         * 3 - lu
 		 */
-#              define PACKET3_DMA_DATA_DST_SEL(x)  ((x) << 20)
+#              define PACKET3_DMA_DATA_SRC_SCOPE(x)     ((x) << 15)
+#              define PACKET3_DMA_DATA_DST_SEL(x)       ((x) << 20)
 		/* 0 - DST_ADDR using DAS
 		 * 1 - GDS
 		 * 3 - DST_ADDR using L2
 		 */
-#              define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25)
+#              define PACKET3_DMA_DATA_DST_TEMPORAL(x)  ((x) << 25)
 		/* 0 - LRU
 		 * 1 - Stream
 		 */
-#              define PACKET3_DMA_DATA_SRC_SEL(x)  ((x) << 29)
+#              define PACKET3_DMA_DATA_DST_SCOPE(x)     ((x) << 27)
+#              define PACKET3_DMA_DATA_SRC_SEL(x)       ((x) << 29)
 		/* 0 - SRC_ADDR using SAS
 		 * 1 - GDS
 		 * 2 - DATA
 		 * 3 - SRC_ADDR using L2
 		 */
-#              define PACKET3_DMA_DATA_CP_SYNC     (1 << 31)
 /* COMMAND */
 #              define PACKET3_DMA_DATA_CMD_SAS     (1 << 26)
 		/* 0 - memory
@@ -247,13 +273,11 @@
 		/* 0 - memory
 		 * 1 - register
 		 */
-#              define PACKET3_DMA_DATA_CMD_SAIC    (1 << 28)
-#              define PACKET3_DMA_DATA_CMD_DAIC    (1 << 29)
-#              define PACKET3_DMA_DATA_CMD_RAW_WAIT  (1 << 30)
+#              define PACKET3_DMA_DATA_CMD_SAIC     (1 << 28)
+#              define PACKET3_DMA_DATA_CMD_DAIC     (1 << 29)
+#              define PACKET3_DMA_DATA_CMD_RAW_WAIT (1 << 30)
+#              define PACKET3_DMA_DATA_CMD_DIS_WC   (1 << 30)
 #define	PACKET3_CONTEXT_REG_RMW				0x51
-#define	PACKET3_GFX_CNTX_UPDATE				0x52
-#define	PACKET3_BLK_CNTX_UPDATE				0x53
-#define	PACKET3_INCR_UPDT_STATE				0x55
 #define	PACKET3_ACQUIRE_MEM				0x58
 /* 1.  HEADER
  * 2.  COHER_CNTL [30:0]
@@ -307,10 +331,7 @@
 		 * 2: REVERSE
 		 */
 #define 	PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA  (1 << 18)
-#define	PACKET3_REWIND					0x59
-#define	PACKET3_INTERRUPT				0x5A
 #define	PACKET3_GEN_PDEPTE				0x5B
-#define	PACKET3_INDIRECT_BUFFER_PASID			0x5C
 #define	PACKET3_PRIME_UTCL2				0x5D
 #define	PACKET3_LOAD_UCONFIG_REG			0x5E
 #define	PACKET3_LOAD_SH_REG				0x5F
@@ -324,12 +345,6 @@
 #define	PACKET3_SET_CONTEXT_REG				0x69
 #define		PACKET3_SET_CONTEXT_REG_START			0x0000a000
 #define		PACKET3_SET_CONTEXT_REG_END			0x0000a400
-#define	PACKET3_SET_CONTEXT_REG_INDEX			0x6A
-#define	PACKET3_SET_VGPR_REG_DI_MULTI			0x71
-#define	PACKET3_SET_SH_REG_DI				0x72
-#define	PACKET3_SET_CONTEXT_REG_INDIRECT		0x73
-#define	PACKET3_SET_SH_REG_DI_MULTI			0x74
-#define	PACKET3_GFX_PIPE_LOCK				0x75
 #define	PACKET3_SET_SH_REG				0x76
 #define		PACKET3_SET_SH_REG_START			0x00002c00
 #define		PACKET3_SET_SH_REG_END				0x00003000
@@ -339,47 +354,19 @@
 #define		PACKET3_SET_UCONFIG_REG_START			0x0000c000
 #define		PACKET3_SET_UCONFIG_REG_END			0x0000c400
 #define	PACKET3_SET_UCONFIG_REG_INDEX			0x7A
-#define	PACKET3_FORWARD_HEADER				0x7C
-#define	PACKET3_SCRATCH_RAM_WRITE			0x7D
-#define	PACKET3_SCRATCH_RAM_READ			0x7E
-#define	PACKET3_LOAD_CONST_RAM				0x80
-#define	PACKET3_WRITE_CONST_RAM				0x81
-#define	PACKET3_DUMP_CONST_RAM				0x83
-#define	PACKET3_INCREMENT_CE_COUNTER			0x84
-#define	PACKET3_INCREMENT_DE_COUNTER			0x85
-#define	PACKET3_WAIT_ON_CE_COUNTER			0x86
-#define	PACKET3_WAIT_ON_DE_COUNTER_DIFF			0x88
-#define	PACKET3_SWITCH_BUFFER				0x8B
 #define	PACKET3_DISPATCH_DRAW_PREAMBLE			0x8C
-#define	PACKET3_DISPATCH_DRAW_PREAMBLE_ACE		0x8C
 #define	PACKET3_DISPATCH_DRAW				0x8D
-#define	PACKET3_DISPATCH_DRAW_ACE			0x8D
-#define	PACKET3_GET_LOD_STATS				0x8E
-#define	PACKET3_DRAW_MULTI_PREAMBLE			0x8F
-#define	PACKET3_FRAME_CONTROL				0x90
-#			define FRAME_TMZ	(1 << 0)
-#			define FRAME_CMD(x) ((x) << 28)
-			/*
-			 * x=0: tmz_begin
-			 * x=1: tmz_end
-			 */
 #define	PACKET3_INDEX_ATTRIBUTES_INDIRECT		0x91
 #define	PACKET3_WAIT_REG_MEM64				0x93
-#define	PACKET3_COND_PREEMPT				0x94
 #define	PACKET3_HDP_FLUSH				0x95
-#define	PACKET3_COPY_DATA_RB				0x96
 #define	PACKET3_INVALIDATE_TLBS				0x98
 #define PACKET3_INVALIDATE_TLBS_DST_SEL(x)     ((x) << 0)
 #define PACKET3_INVALIDATE_TLBS_ALL_HUB(x)     ((x) << 4)
 #define PACKET3_INVALIDATE_TLBS_PASID(x)       ((x) << 5)
 #define PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(x)  ((x) << 29)
 
-#define	PACKET3_AQL_PACKET				0x99
 #define	PACKET3_DMA_DATA_FILL_MULTI			0x9A
 #define	PACKET3_SET_SH_REG_INDEX			0x9B
-#define	PACKET3_DRAW_INDIRECT_COUNT_MULTI		0x9C
-#define	PACKET3_DRAW_INDEX_INDIRECT_COUNT_MULTI		0x9D
-#define	PACKET3_DUMP_CONST_RAM_OFFSET			0x9E
 #define	PACKET3_LOAD_CONTEXT_REG_INDEX			0x9F
 #define	PACKET3_SET_RESOURCES				0xA0
 /* 1. header
@@ -394,7 +381,6 @@
 #              define PACKET3_SET_RESOURCES_VMID_MASK(x)     ((x) << 0)
 #              define PACKET3_SET_RESOURCES_UNMAP_LATENTY(x) ((x) << 16)
 #              define PACKET3_SET_RESOURCES_QUEUE_TYPE(x)    ((x) << 29)
-#define PACKET3_MAP_PROCESS				0xA1
 #define PACKET3_MAP_QUEUES				0xA2
 /* 1. header
  * 2. CONTROL
@@ -411,11 +397,10 @@
 #              define PACKET3_MAP_QUEUES_PIPE(x)            ((x) << 16)
 #              define PACKET3_MAP_QUEUES_ME(x)              ((x) << 18)
 #              define PACKET3_MAP_QUEUES_QUEUE_TYPE(x)      ((x) << 21)
-#              define PACKET3_MAP_QUEUES_ALLOC_FORMAT(x)    ((x) << 24)
+#              define PACKET3_MAP_QUEUES_QUEUE_GROUP(x)     ((x) << 24)
 #              define PACKET3_MAP_QUEUES_ENGINE_SEL(x)      ((x) << 26)
 #              define PACKET3_MAP_QUEUES_NUM_QUEUES(x)      ((x) << 29)
 /* CONTROL2 */
-#              define PACKET3_MAP_QUEUES_CHECK_DISABLE(x)   ((x) << 1)
 #              define PACKET3_MAP_QUEUES_DOORBELL_OFFSET(x) ((x) << 2)
 #define	PACKET3_UNMAP_QUEUES				0xA3
 /* 1. header
@@ -464,12 +449,6 @@
 #              define PACKET3_QUERY_STATUS_PASID(x)            ((x) << 0)
 /* CONTROL2b */
 #              define PACKET3_QUERY_STATUS_DOORBELL_OFFSET(x)  ((x) << 2)
-#              define PACKET3_QUERY_STATUS_ENG_SEL(x)          ((x) << 25)
-#define	PACKET3_RUN_LIST				0xA5
-#define	PACKET3_MAP_PROCESS_VM				0xA6
-/* GFX11 */
-#define	PACKET3_SET_Q_PREEMPTION_MODE			0xF0
-#              define PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(x)  ((x) << 0)
-#              define PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM    (1 << 0)
+#              define PACKET3_QUERY_STATUS_ENG_SEL(x)          ((x) << 28)
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 7e9d753f4a808..95be105671ece 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -2355,7 +2355,7 @@ static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
 			ring = &adev->gfx.sw_gfx_ring[i];
 			ring->ring_obj = NULL;
-			sprintf(ring->name, amdgpu_sw_ring_name(i));
+			strscpy(ring->name, amdgpu_sw_ring_name(i), sizeof(ring->name));
 			ring->use_doorbell = true;
 			ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
 			ring->is_sw_ring = true;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
index 4aa004ee2c4d9..3544eb42dca60 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
@@ -146,71 +146,15 @@ static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *ade
 	uint32_t tmp;
 	int i;
 
-	for_each_inst(i, xcc_mask) {
-		/* Program the AGP BAR */
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_BASE_LO32, 0);
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_BASE_HI32, 0);
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_BOT_LO32,
-				 lower_32_bits(adev->gmc.agp_start >> 24));
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_BOT_HI32,
-				 upper_32_bits(adev->gmc.agp_start >> 24));
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_TOP_LO32,
-				 lower_32_bits(adev->gmc.agp_end >> 24));
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCMC_VM_AGP_TOP_HI32,
-				 upper_32_bits(adev->gmc.agp_end >> 24));
+	/*TODO: revisit whether the SRIOV guest access to theseregisters
+	 * is blocked by security policy or not */
+	if (amdgpu_sriov_vf(adev))
+		return;
 
-		if (!amdgpu_sriov_vf(adev)) {
-			/* Program the system aperture low logical page number. */
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
-				     lower_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18));
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
-				     upper_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18));
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
-				     lower_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18));
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
-				     upper_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18));
-
-			/* Set default page address. */
-			value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
-				     (u32)(value >> 12));
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
-				     (u32)(value >> 44));
-
-			/* Program "protection fault". */
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
-				     (u32)(adev->dummy_page_addr >> 12));
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
-				     (u32)((u64)adev->dummy_page_addr >> 44));
-
-			tmp = RREG32_SOC15(GC, GET_INST(GC, i),
-					   regGCVM_L2_PROTECTION_FAULT_CNTL2);
-			tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
-					    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
-			tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
-					    ENABLE_RETRY_FAULT_INTERRUPT, 0x1);
-			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
-		}
-
-		/* In the case squeezing vram into GART aperture, we don't use
-		 * FB aperture and AGP aperture. Disable them.
-		 */
+	for_each_inst(i, xcc_mask) {
 		if (adev->gmc.pdb0_bo) {
+			/* Disable agp and system aperture
+			 * when vmid0 page table is enabled */
 			WREG32_SOC15(GC, GET_INST(GC, i),
 				     regGCMC_VM_FB_LOCATION_TOP_LO32, 0);
 			WREG32_SOC15(GC, GET_INST(GC, i),
@@ -225,7 +169,8 @@ static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *ade
 			WREG32_SOC15(GC, GET_INST(GC, i),
 				     regGCMC_VM_AGP_TOP_HI32, 0);
 			WREG32_SOC15(GC, GET_INST(GC, i),
-				     regGCMC_VM_AGP_BOT_LO32, 0xFFFFFFFF);
+				     regGCMC_VM_AGP_BOT_LO32,
+				     0xFFFFFFFF);
 			WREG32_SOC15(GC, GET_INST(GC, i),
 				     regGCMC_VM_AGP_BOT_HI32, 1);
 			WREG32_SOC15(GC, GET_INST(GC, i),
@@ -238,7 +183,69 @@ static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *ade
 				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
 			WREG32_SOC15(GC, GET_INST(GC, i),
 				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
+		} else {
+			/* Program the AGP BAR */
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_BASE_LO32, 0);
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_BASE_HI32, 0);
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_BOT_LO32,
+					 lower_32_bits(adev->gmc.agp_start >> 24));
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_BOT_HI32,
+					 upper_32_bits(adev->gmc.agp_start >> 24));
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_TOP_LO32,
+					 lower_32_bits(adev->gmc.agp_end >> 24));
+			WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+					 regGCMC_VM_AGP_TOP_HI32,
+					 upper_32_bits(adev->gmc.agp_end >> 24));
+
+			/* Program the system aperture low logical page number. */
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+				     lower_32_bits(min(adev->gmc.fb_start,
+						   adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+				     upper_32_bits(min(adev->gmc.fb_start,
+						   adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
+				     lower_32_bits(max(adev->gmc.fb_end,
+						   adev->gmc.agp_end) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
+				     upper_32_bits(max(adev->gmc.fb_end,
+						   adev->gmc.agp_end) >> 18));
 		}
+
+		/* Set default page address. */
+		value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+			     (u32)(value >> 12));
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+			     (u32)(value >> 44));
+
+		/* Program "protection fault". */
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+			     (u32)(adev->dummy_page_addr >> 12));
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+			     (u32)((u64)adev->dummy_page_addr >> 44));
+
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+				   regGCVM_L2_PROTECTION_FAULT_CNTL2);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				    ENABLE_RETRY_FAULT_INTERRUPT, 0x1);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 	}
 }
 
@@ -325,10 +332,18 @@ static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev,
 		WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL3, tmp);
 
 		tmp = regGCVM_L2_CNTL4_DEFAULT;
-		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
-				    VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
-		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
-				    VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+		if (adev->gmc.xgmi.connected_to_cpu) {
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+					    VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+					    VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+		} else {
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+					    VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+					    VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+		}
+
 		WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL4, tmp);
 
 		tmp = regGCVM_L2_CNTL5_DEFAULT;
@@ -807,47 +822,6 @@ static void gfxhub_v12_1_init(struct amdgpu_device *adev)
 	gfxhub_v12_1_xcc_init(adev, xcc_mask);
 }
 
-static int gfxhub_v12_1_get_xgmi_info(struct amdgpu_device *adev)
-{
-	u32 max_num_physical_nodes;
-	u32 max_physical_node_id;
-	u32 xgmi_lfb_cntl;
-	u32 max_region;
-	u64 seg_size;
-
-	xgmi_lfb_cntl = RREG32_SOC15(GC, GET_INST(GC, 0),
-				     regGCMC_VM_XGMI_LFB_CNTL);
-	seg_size = REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, 0),
-				 regGCMC_VM_XGMI_LFB_SIZE),
-				 GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
-	max_region = REG_GET_FIELD(xgmi_lfb_cntl,
-				   GCMC_VM_XGMI_LFB_CNTL,
-				   PF_MAX_REGION);
-
-	max_num_physical_nodes   = 8;
-	max_physical_node_id     = 7;
-
-	/* PF_MAX_REGION=0 means xgmi is disabled */
-	if (max_region || adev->gmc.xgmi.connected_to_cpu) {
-		adev->gmc.xgmi.num_physical_nodes = max_region + 1;
-
-		if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
-			return -EINVAL;
-
-		adev->gmc.xgmi.physical_node_id =
-			REG_GET_FIELD(xgmi_lfb_cntl,
-				      GCMC_VM_XGMI_LFB_CNTL,
-				      PF_LFB_REGION);
-
-		if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
-			return -EINVAL;
-
-		adev->gmc.xgmi.node_segment_size = seg_size;
-	}
-
-	return 0;
-}
-
 const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs = {
 	.get_fb_location = gfxhub_v12_1_get_fb_location,
 	.get_mc_fb_offset = gfxhub_v12_1_get_mc_fb_offset,
@@ -856,7 +830,6 @@ const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs = {
 	.gart_disable = gfxhub_v12_1_gart_disable,
 	.set_fault_enable_default = gfxhub_v12_1_set_fault_enable_default,
 	.init = gfxhub_v12_1_init,
-	.get_xgmi_info = gfxhub_v12_1_get_xgmi_info,
 };
 
 static int gfxhub_v12_1_xcp_resume(void *handle, uint32_t inst_mask)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index b9671fc39e2a8..f1079bd8cf001 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -636,6 +636,11 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (adev->smuio.funcs &&
+	    adev->smuio.funcs->is_host_gpu_xgmi_supported)
+		adev->gmc.xgmi.connected_to_cpu =
+			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
+
 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 	case IP_VERSION(12, 1, 0):
 		gmc_v12_1_set_gmc_funcs(adev);
@@ -654,9 +659,15 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
 	adev->gmc.shared_aperture_end =
 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
+
 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
-	adev->gmc.private_aperture_end =
-		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))
+		adev->gmc.private_aperture_end =
+			adev->gmc.private_aperture_start + (1ULL << 57) - 1;
+	else
+		adev->gmc.private_aperture_end =
+			adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+
 	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
 	return 0;
@@ -685,17 +696,23 @@ static void gmc_v12_0_vram_gtt_location(struct amdgpu_device *adev,
 
 	base = adev->mmhub.funcs->get_fb_location(adev);
 
-	amdgpu_gmc_set_agp_default(adev, mc);
-	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
-	amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_LOW);
-	if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
-		amdgpu_gmc_agp_location(adev, mc);
-
+	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
+		amdgpu_gmc_sysvm_location(adev, mc);
+	} else {
+		amdgpu_gmc_set_agp_default(adev, mc);
+		amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+		amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_LOW);
+		if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
+			amdgpu_gmc_agp_location(adev, mc);
+	}
 	/* base offset of vram pages */
 	if (amdgpu_sriov_vf(adev))
 		adev->vm_manager.vram_base_offset = 0;
 	else
 		adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
+
+	adev->vm_manager.vram_base_offset +=
+		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
 }
 
 /**
@@ -711,12 +728,17 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
 {
 	int r;
 
-	/* size in MB on si */
-	adev->gmc.mc_vram_size =
-		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
+	if (adev->gmc.xgmi.connected_to_cpu)
+		adev->gmc.mc_vram_size =
+			adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
+	else
+		adev->gmc.mc_vram_size =
+			adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
+
 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
 
-	if (!(adev->flags & AMD_IS_APU)) {
+	if (!(adev->flags & AMD_IS_APU) &&
+	    !adev->gmc.xgmi.connected_to_cpu) {
 		r = amdgpu_device_resize_fb_bar(adev);
 		if (r)
 			return r;
@@ -726,8 +748,12 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
 
 #ifdef CONFIG_X86_64
-	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
-		adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev);
+	if (((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
+	    (adev->gmc.xgmi.connected_to_cpu)) {
+		adev->gmc.aper_base =
+			adev->mmhub.funcs->get_mc_fb_offset(adev) +
+			adev->gmc.xgmi.physical_node_id *
+			adev->gmc.xgmi.node_segment_size;
 		adev->gmc.aper_size = adev->gmc.real_vram_size;
 	}
 #endif
@@ -756,6 +782,14 @@ static int gmc_v12_0_gart_init(struct amdgpu_device *adev)
 		return 0;
 	}
 
+	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
+		adev->gmc.vmid0_page_table_depth = 1;
+		adev->gmc.vmid0_page_table_block_size = 12;
+	} else {
+		adev->gmc.vmid0_page_table_depth = 0;
+		adev->gmc.vmid0_page_table_block_size = 0;
+	}
+
 	/* Initialize common gart structure */
 	r = amdgpu_gart_init(adev);
 	if (r)
@@ -766,7 +800,14 @@ static int gmc_v12_0_gart_init(struct amdgpu_device *adev)
 				    AMDGPU_PTE_EXECUTABLE |
 				    AMDGPU_PTE_IS_PTE;
 
-	return amdgpu_gart_table_vram_alloc(adev);
+	r = amdgpu_gart_table_vram_alloc(adev);
+	if (r)
+		return r;
+
+	if (amdgpu_gmc_is_pdb0_enabled(adev))
+		r = amdgpu_gmc_pdb0_alloc(adev);
+
+	return r;
 }
 
 static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
@@ -852,11 +893,15 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(12, 1, 0)) &&
-	    !amdgpu_sriov_vf(adev)) {
+	if (!amdgpu_sriov_vf(adev)) {
 		/* interrupt sent to DF. */
-		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 0, 0))
+			r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
+				      &adev->gmc.ecc_irq);
+		else
+			r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_DF, 0,
 				      &adev->gmc.ecc_irq);
+
 		if (r)
 			return r;
 	}
@@ -938,6 +983,7 @@ static int gmc_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	amdgpu_vm_manager_fini(adev);
 	gmc_v12_0_gart_fini(adev);
 	amdgpu_gem_force_release(adev);
+	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
 	amdgpu_bo_fini(adev);
 
 	return 0;
@@ -957,6 +1003,9 @@ static int gmc_v12_0_gart_enable(struct amdgpu_device *adev)
 	int r;
 	bool value;
 
+	if (adev->gmc.xgmi.connected_to_cpu)
+		amdgpu_gmc_init_pdb0(adev);
+
 	if (adev->gart.bo == NULL) {
 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
 		return -EINVAL;
@@ -978,6 +1027,7 @@ static int gmc_v12_0_gart_enable(struct amdgpu_device *adev)
 
 	drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled (table at 0x%016llX).\n",
 		 (unsigned)(adev->gmc.gart_size >> 20),
+		 (adev->gmc.pdb0_bo) ? (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo) :
 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index ef6e550ce7c3d..7ea7b9c30bca9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -121,7 +121,7 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 
 	if (entry->src_id == UTCL2_1_0__SRCID__RETRY) {
 		retry_fault = true;
-		write_fault = !!(entry->src_data[1] & 0x200000);
+		write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE);
 	}
 
 	if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) {
@@ -345,9 +345,7 @@ static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 		return;
 	}
 
-	mutex_lock(&adev->mman.gtt_window_lock);
 	gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0);
-	mutex_unlock(&adev->mman.gtt_window_lock);
 	return;
 }
 
@@ -526,20 +524,27 @@ static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
 	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
 	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
-	unsigned int mtype, mtype_local;
+	unsigned int mtype, mtype_local, mtype_remote;
 	bool snoop = false;
 	bool is_local = false;
+	bool is_aid_a1;
 
 	switch (gc_ip_version) {
 	case IP_VERSION(12, 1, 0):
-		mtype_local = MTYPE_RW;
-		if (amdgpu_mtype_local == 1) {
+		is_aid_a1 = (adev->rev_id & 0x10);
+
+		mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
+		mtype_remote = is_aid_a1 ? MTYPE_NC : MTYPE_UC;
+		if (amdgpu_mtype_local == 0) {
+			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
+			mtype_local = MTYPE_RW;
+		} else if (amdgpu_mtype_local == 1) {
 			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
 			mtype_local = MTYPE_NC;
 		} else if (amdgpu_mtype_local == 2) {
-			DRM_INFO_ONCE("MTYPE_CC not supported, using MTYPE_RW instead for local memory\n");
+			DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
 		} else {
-			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
+			DRM_INFO_ONCE("Using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
 		}
 
 		is_local = (is_vram && adev == bo_adev);
@@ -549,10 +554,7 @@ static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 		} else if (ext_coherent) {
 			mtype = is_local ? mtype_local : MTYPE_UC;
 		} else {
-			if (is_local)
-				mtype = mtype_local;
-			else
-				mtype = MTYPE_NC;
+			mtype = is_local ? mtype_local : mtype_remote;
 		}
 		break;
 	default:
@@ -623,10 +625,17 @@ static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = {
 	.process = gmc_v12_1_process_interrupt,
 };
 
+static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = {
+	.process = amdgpu_umc_uniras_process_ecc_irq,
+};
+
 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->gmc.vm_fault.num_types = 1;
 	adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs;
+
+	adev->gmc.ecc_irq.num_types = 1;
+	adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs;
 }
 
 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index e35ed0cc2ec62..1ca0202cfdea8 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -660,35 +660,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 			gfxhub_client_ids[cid],
 			cid);
 	} else {
-		switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-		case IP_VERSION(9, 0, 0):
-			mmhub_cid = mmhub_client_ids_vega10[cid][rw];
-			break;
-		case IP_VERSION(9, 3, 0):
-			mmhub_cid = mmhub_client_ids_vega12[cid][rw];
-			break;
-		case IP_VERSION(9, 4, 0):
-			mmhub_cid = mmhub_client_ids_vega20[cid][rw];
-			break;
-		case IP_VERSION(9, 4, 1):
-			mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
-			break;
-		case IP_VERSION(9, 1, 0):
-		case IP_VERSION(9, 2, 0):
-			mmhub_cid = mmhub_client_ids_raven[cid][rw];
-			break;
-		case IP_VERSION(1, 5, 0):
-		case IP_VERSION(2, 4, 0):
-			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
-			break;
-		case IP_VERSION(1, 8, 0):
-		case IP_VERSION(9, 4, 2):
-			mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
-			break;
-		default:
-			mmhub_cid = NULL;
-			break;
-		}
+		mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 			mmhub_cid ? mmhub_cid : "unknown", cid);
 	}
@@ -1428,6 +1400,52 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
 	}
 }
 
+static void gmc_v9_0_init_mmhub_client_info(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
+	case IP_VERSION(9, 0, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_vega10,
+					     ARRAY_SIZE(mmhub_client_ids_vega10));
+		break;
+	case IP_VERSION(9, 3, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_vega12,
+					     ARRAY_SIZE(mmhub_client_ids_vega12));
+		break;
+	case IP_VERSION(9, 4, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_vega20,
+					     ARRAY_SIZE(mmhub_client_ids_vega20));
+		break;
+	case IP_VERSION(9, 4, 1):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_arcturus,
+					     ARRAY_SIZE(mmhub_client_ids_arcturus));
+		break;
+	case IP_VERSION(9, 1, 0):
+	case IP_VERSION(9, 2, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_raven,
+					     ARRAY_SIZE(mmhub_client_ids_raven));
+		break;
+	case IP_VERSION(1, 5, 0):
+	case IP_VERSION(2, 4, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_renoir,
+					     ARRAY_SIZE(mmhub_client_ids_renoir));
+		break;
+	case IP_VERSION(1, 8, 0):
+	case IP_VERSION(9, 4, 2):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_aldebaran,
+					     ARRAY_SIZE(mmhub_client_ids_aldebaran));
+		break;
+	default:
+		break;
+	}
+}
+
 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
 {
 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
@@ -1445,6 +1463,8 @@ static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
 		break;
 	}
+
+	gmc_v9_0_init_mmhub_client_info(adev);
 }
 
 static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
index 451828bf583e4..1fbe904f4223b 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
@@ -289,6 +289,13 @@ static uint32_t ih_v7_0_setup_retry_doorbell(u32 doorbell_index)
 	return val;
 }
 
+#define regIH_RING1_CLIENT_CFG_INDEX_V7_1             0x122
+#define regIH_RING1_CLIENT_CFG_INDEX_V7_1_BASE_IDX    0
+#define regIH_RING1_CLIENT_CFG_DATA_V7_1              0x123
+#define regIH_RING1_CLIENT_CFG_DATA_V7_1_BASE_IDX     0
+#define regIH_CHICKEN_V7_1                            0x129
+#define regIH_CHICKEN_V7_1_BASE_IDX                   0
+
 /**
  * ih_v7_0_irq_init - init and enable the interrupt ring
  *
@@ -307,6 +314,7 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 	u32 tmp;
 	int ret;
 	int i;
+	u32 reg_addr;
 
 	/* disable irqs */
 	ret = ih_v7_0_toggle_interrupts(adev, false);
@@ -318,10 +326,15 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 	if (unlikely((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
 		     (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO))) {
 		if (ih[0]->use_bus_addr) {
-			ih_chicken = RREG32_SOC15(OSSSYS, 0, regIH_CHICKEN);
+			if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0))
+				reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_CHICKEN_V7_1);
+			else
+				reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_CHICKEN);
+			ih_chicken = RREG32(reg_addr);
+			/* The reg fields definitions are identical in ih v7_0 and ih v7_1 */
 			ih_chicken = REG_SET_FIELD(ih_chicken,
 					IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1);
-			WREG32_SOC15(OSSSYS, 0, regIH_CHICKEN, ih_chicken);
+			WREG32(reg_addr, ih_chicken);
 		}
 	}
 
@@ -358,17 +371,26 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 
 	/* Redirect the interrupts to IH RB1 for dGPU */
 	if (adev->irq.ih1.ring_size) {
-		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX);
+		if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0))
+			reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX_V7_1);
+		else
+			reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX);
+		tmp = RREG32(reg_addr);
+		/* The reg fields definitions are identical in ih v7_0 and ih v7_1 */
 		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_INDEX, INDEX, 0);
-		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX, tmp);
+		WREG32(reg_addr, tmp);
 
-		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA);
+		if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0))
+			reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA_V7_1);
+		else
+			reg_addr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA);
+		tmp = RREG32(reg_addr);
+		/* The reg fields definitions are identical in ih v7_0 and ih v7_1 */
 		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, CLIENT_ID, 0xa);
 		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, SOURCE_ID, 0x0);
 		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA,
 				    SOURCE_ID_MATCH_ENABLE, 0x1);
-
-		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA, tmp);
+		WREG32(reg_addr, tmp);
 	}
 
 	pci_set_master(adev->pdev);
diff --git a/drivers/gpu/drm/amd/amdgpu/isp_v4_1_1.c b/drivers/gpu/drm/amd/amdgpu/isp_v4_1_1.c
index b3590b33cab9e..485ecdec96184 100644
--- a/drivers/gpu/drm/amd/amdgpu/isp_v4_1_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/isp_v4_1_1.c
@@ -129,7 +129,7 @@ static int isp_genpd_add_device(struct device *dev, void *data)
 	if (!pdev)
 		return -EINVAL;
 
-	if (!dev->type->name) {
+	if (!dev->type || !dev->type->name) {
 		drm_dbg(&adev->ddev, "Invalid device type to add\n");
 		goto exit;
 	}
@@ -165,7 +165,7 @@ static int isp_genpd_remove_device(struct device *dev, void *data)
 	if (!pdev)
 		return -EINVAL;
 
-	if (!dev->type->name) {
+	if (!dev->type || !dev->type->name) {
 		drm_dbg(&adev->ddev, "Invalid device type to remove\n");
 		goto exit;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
new file mode 100644
index 0000000000000..285c459379c4a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
@@ -0,0 +1,840 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_jpeg.h"
+#include "amdgpu_pm.h"
+#include "soc15.h"
+#include "soc15d.h"
+#include "jpeg_v4_0_3.h"
+#include "jpeg_v5_0_2.h"
+#include "mmsch_v5_0.h"
+
+#include "vcn/vcn_5_0_0_offset.h"
+#include "vcn/vcn_5_0_0_sh_mask.h"
+#include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
+
+static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev);
+static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev);
+static int jpeg_v5_0_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
+					     enum amd_powergating_state state);
+static void jpeg_v5_0_2_dec_ring_set_wptr(struct amdgpu_ring *ring);
+
+static int amdgpu_ih_srcid_jpeg[] = {
+	VCN_5_0__SRCID__JPEG_DECODE,
+	VCN_5_0__SRCID__JPEG1_DECODE,
+	VCN_5_0__SRCID__JPEG2_DECODE,
+	VCN_5_0__SRCID__JPEG3_DECODE,
+	VCN_5_0__SRCID__JPEG4_DECODE,
+	VCN_5_0__SRCID__JPEG5_DECODE,
+	VCN_5_0__SRCID__JPEG6_DECODE,
+	VCN_5_0__SRCID__JPEG7_DECODE,
+	VCN_5_0__SRCID__JPEG8_DECODE,
+	VCN_5_0__SRCID__JPEG9_DECODE,
+};
+
+static const struct amdgpu_hwip_reg_entry jpeg_reg_list_5_0_2[] = {
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_POWER_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_INT_STAT),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_ADDR_MODE),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_Y_GFX10_TILING_SURFACE),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_UV_GFX10_TILING_SURFACE),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_PITCH),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_UV_PITCH),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC8_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC8_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC8_UVD_JRBC_STATUS),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC9_UVD_JRBC_RB_RPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC9_UVD_JRBC_RB_WPTR),
+	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC9_UVD_JRBC_STATUS),
+};
+
+static int jpeg_v5_0_2_core_reg_offset(u32 pipe)
+{
+	if (pipe <= AMDGPU_MAX_JPEG_RINGS_4_0_3)
+		return ((0x40 * pipe) - 0xc80);
+	else
+		return ((0x40 * pipe) - 0x440);
+}
+
+/**
+ * jpeg_v5_0_2_early_init - set function pointers
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Set ring and irq function pointers
+ */
+static int jpeg_v5_0_2_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (!adev->jpeg.num_jpeg_inst || adev->jpeg.num_jpeg_inst > AMDGPU_MAX_JPEG_INSTANCES)
+		return -ENOENT;
+
+	adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
+	jpeg_v5_0_2_set_dec_ring_funcs(adev);
+	jpeg_v5_0_2_set_irq_funcs(adev);
+
+	return 0;
+}
+
+/**
+ * jpeg_v5_0_2_sw_init - sw init for JPEG block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Load firmware and sw initialization
+ */
+static int jpeg_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ring *ring;
+	int i, j, r, jpeg_inst;
+
+	for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+		/* JPEG TRAP */
+		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+				      amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
+		if (r)
+			return r;
+	}
+
+	r = amdgpu_jpeg_sw_init(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_jpeg_resume(adev);
+	if (r)
+		return r;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		jpeg_inst = GET_INST(JPEG, i);
+
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			ring = &adev->jpeg.inst[i].ring_dec[j];
+			ring->use_doorbell = false;
+			ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
+			ring->doorbell_index =
+				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+				1 + j + 11 * jpeg_inst;
+			sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
+			r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
+					     AMDGPU_RING_PRIO_DEFAULT, NULL);
+			if (r)
+				return r;
+
+			adev->jpeg.internal.jpeg_pitch[j] =
+				regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET;
+			adev->jpeg.inst[i].external.jpeg_pitch[j] =
+				SOC15_REG_OFFSET1(JPEG, jpeg_inst, regUVD_JRBC_SCRATCH0,
+						  (j ? jpeg_v5_0_2_core_reg_offset(j) : 0));
+		}
+	}
+
+	r = amdgpu_jpeg_reg_dump_init(adev, jpeg_reg_list_5_0_2, ARRAY_SIZE(jpeg_reg_list_5_0_2));
+	if (r)
+		return r;
+
+	adev->jpeg.supported_reset =
+		amdgpu_get_soft_full_reset_mask(&adev->jpeg.inst[0].ring_dec[0]);
+	adev->jpeg.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
+	r = amdgpu_jpeg_sysfs_reset_mask_init(adev);
+
+	return r;
+}
+
+/**
+ * jpeg_v5_0_2_sw_fini - sw fini for JPEG block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * JPEG suspend and free up sw allocation
+ */
+static int jpeg_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = amdgpu_jpeg_suspend(adev);
+	if (r)
+		return r;
+
+	amdgpu_jpeg_sysfs_reset_mask_fini(adev);
+
+	r = amdgpu_jpeg_sw_fini(adev);
+
+	return r;
+}
+
+/**
+ * jpeg_v5_0_2_hw_init - start and test JPEG block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ */
+static int jpeg_v5_0_2_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ring *ring;
+	int i, j, r, jpeg_inst, tmp;
+
+	if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100)
+		adev->jpeg.caps |= AMDGPU_JPEG_CAPS(RRMT_ENABLED);
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		jpeg_inst = GET_INST(JPEG, i);
+		ring = adev->jpeg.inst[i].ring_dec;
+
+		/* Remove JPEG Tile antihang mechanism */
+		tmp = RREG32_SOC15(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS);
+		tmp &= (~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
+		WREG32_SOC15(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS, tmp);
+
+		if (ring->use_doorbell)
+			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+				 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 11 * jpeg_inst,
+				 adev->jpeg.inst[i].aid_id);
+
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			ring = &adev->jpeg.inst[i].ring_dec[j];
+			if (ring->use_doorbell)
+				WREG32_SOC15_OFFSET(VCN, GET_INST(VCN, i), regVCN_JPEG_DB_CTRL,
+						    ring->pipe,
+						    ring->doorbell_index <<
+						    VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
+						    VCN_JPEG_DB_CTRL__EN_MASK);
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				return r;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * jpeg_v5_0_2_hw_fini - stop the hardware block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Stop the JPEG block, mark ring as not ready any more
+ */
+static int jpeg_v5_0_2_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int ret = 0;
+
+	cancel_delayed_work_sync(&adev->jpeg.idle_work);
+
+		if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
+			ret = jpeg_v5_0_2_set_powergating_state(ip_block, AMD_PG_STATE_GATE);
+
+	return ret;
+}
+
+/**
+ * jpeg_v5_0_2_suspend - suspend JPEG block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * HW fini and suspend JPEG block
+ */
+static int jpeg_v5_0_2_suspend(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = jpeg_v5_0_2_hw_fini(ip_block);
+	if (r)
+		return r;
+
+	r = amdgpu_jpeg_suspend(adev);
+
+	return r;
+}
+
+/**
+ * jpeg_v5_0_2_resume - resume JPEG block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Resume firmware and hw init JPEG block
+ */
+static int jpeg_v5_0_2_resume(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = amdgpu_jpeg_resume(adev);
+	if (r)
+		return r;
+
+	r = jpeg_v5_0_2_hw_init(ip_block);
+
+	return r;
+}
+
+static void jpeg_v5_0_2_init_inst(struct amdgpu_device *adev, int i)
+{
+	int jpeg_inst = GET_INST(JPEG, i);
+
+	/* disable anti hang mechanism */
+	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS), 0,
+		 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
+
+	/* keep the JPEG in static PG mode */
+	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS), 0,
+		 ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK);
+
+	/* MJPEG global tiling registers */
+	WREG32_SOC15(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG,
+		     adev->gfx.config.gb_addr_config);
+
+	/* enable JMI channel */
+	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL), 0,
+		 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
+}
+
+static void jpeg_v5_0_2_deinit_inst(struct amdgpu_device *adev, int i)
+{
+	int jpeg_inst = GET_INST(JPEG, i);
+	/* reset JMI */
+	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL),
+		 UVD_JMI_CNTL__SOFT_RESET_MASK,
+		 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
+
+	/* enable anti hang mechanism */
+	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS),
+		 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
+		 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
+}
+
+static void jpeg_v5_0_2_init_jrbc(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u32 reg, data, mask;
+	int jpeg_inst = GET_INST(JPEG, ring->me);
+	int reg_offset = ring->pipe ? jpeg_v5_0_2_core_reg_offset(ring->pipe) : 0;
+
+	/* enable System Interrupt for JRBC */
+	reg = SOC15_REG_OFFSET(JPEG, jpeg_inst, regJPEG_SYS_INT_EN);
+	if (ring->pipe < AMDGPU_MAX_JPEG_RINGS_4_0_3) {
+		data = JPEG_SYS_INT_EN__DJRBC0_MASK << ring->pipe;
+		mask = ~(JPEG_SYS_INT_EN__DJRBC0_MASK << ring->pipe);
+		WREG32_P(reg, data, mask);
+	} else {
+		data = JPEG_SYS_INT_EN__DJRBC0_MASK << (ring->pipe+12);
+		mask = ~(JPEG_SYS_INT_EN__DJRBC0_MASK << (ring->pipe+12));
+		WREG32_P(reg, data, mask);
+	}
+
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_LMI_JRBC_RB_VMID,
+			    reg_offset, 0);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JRBC_RB_CNTL,
+			    reg_offset,
+			    (0x00000001L | 0x00000002L));
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
+			    reg_offset, lower_32_bits(ring->gpu_addr));
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
+			    reg_offset, upper_32_bits(ring->gpu_addr));
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JRBC_RB_RPTR,
+			    reg_offset, 0);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JRBC_RB_WPTR,
+			    reg_offset, 0);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JRBC_RB_CNTL,
+			    reg_offset, 0x00000002L);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JRBC_RB_SIZE,
+			    reg_offset, ring->ring_size / 4);
+	ring->wptr = RREG32_SOC15_OFFSET(JPEG, jpeg_inst, regUVD_JRBC_RB_WPTR,
+					 reg_offset);
+}
+
+/**
+ * jpeg_v5_0_2_start - start JPEG block
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Setup and start the JPEG block
+ */
+static int jpeg_v5_0_2_start(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	int i, j;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		jpeg_v5_0_2_init_inst(adev, i);
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			ring = &adev->jpeg.inst[i].ring_dec[j];
+			jpeg_v5_0_2_init_jrbc(ring);
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * jpeg_v5_0_2_stop - stop JPEG block
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * stop the JPEG block
+ */
+static int jpeg_v5_0_2_stop(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i)
+		jpeg_v5_0_2_deinit_inst(adev, i);
+
+	return 0;
+}
+
+/**
+ * jpeg_v5_0_2_dec_ring_get_rptr - get read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware read pointer
+ */
+static uint64_t jpeg_v5_0_2_dec_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	return RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC_RB_RPTR,
+				   ring->pipe ? jpeg_v5_0_2_core_reg_offset(ring->pipe) : 0);
+}
+
+/**
+ * jpeg_v5_0_2_dec_ring_get_wptr - get write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware write pointer
+ */
+static uint64_t jpeg_v5_0_2_dec_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring->use_doorbell)
+		return adev->wb.wb[ring->wptr_offs];
+
+	return RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC_RB_WPTR,
+				   ring->pipe ? jpeg_v5_0_2_core_reg_offset(ring->pipe) : 0);
+}
+
+/**
+ * jpeg_v5_0_2_dec_ring_set_wptr - set write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the write pointer to the hardware
+ */
+static void jpeg_v5_0_2_dec_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring->use_doorbell) {
+		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
+		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+	} else {
+		WREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me),
+				    regUVD_JRBC_RB_WPTR,
+				    (ring->pipe ? jpeg_v5_0_2_core_reg_offset(ring->pipe) : 0),
+				    lower_32_bits(ring->wptr));
+	}
+}
+
+static bool jpeg_v5_0_2_is_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	bool ret = false;
+	int i, j;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			int reg_offset = (j ? jpeg_v5_0_2_core_reg_offset(j) : 0);
+
+			ret &= ((RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, i),
+				regUVD_JRBC_STATUS, reg_offset) &
+				UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
+				UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
+		}
+	}
+
+	return ret;
+}
+
+static int jpeg_v5_0_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int ret = 0;
+	int i, j;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			int reg_offset = (j ? jpeg_v5_0_2_core_reg_offset(j) : 0);
+
+			ret &= SOC15_WAIT_ON_RREG_OFFSET(JPEG, GET_INST(JPEG, i),
+							 regUVD_JRBC_STATUS, reg_offset,
+							 UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
+							 UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
+		}
+	}
+	return ret;
+}
+
+static int jpeg_v5_0_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+					     enum amd_clockgating_state state)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	bool enable = state == AMD_CG_STATE_GATE;
+
+	int i;
+
+	if (!enable)
+		return 0;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		if (!jpeg_v5_0_2_is_idle(ip_block))
+			return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int jpeg_v5_0_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
+					     enum amd_powergating_state state)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int ret;
+
+	if (state == adev->jpeg.cur_state)
+		return 0;
+
+	if (state == AMD_PG_STATE_GATE)
+		ret = jpeg_v5_0_2_stop(adev);
+	else
+		ret = jpeg_v5_0_2_start(adev);
+
+	if (!ret)
+		adev->jpeg.cur_state = state;
+
+	return ret;
+}
+
+static int jpeg_v5_0_2_set_interrupt_state(struct amdgpu_device *adev,
+					   struct amdgpu_irq_src *source,
+					   unsigned int type,
+					   enum amdgpu_interrupt_state state)
+{
+	return 0;
+}
+
+static int jpeg_v5_0_2_process_interrupt(struct amdgpu_device *adev,
+					 struct amdgpu_irq_src *source,
+					 struct amdgpu_iv_entry *entry)
+{
+	u32 i, inst;
+
+	i = node_id_to_phys_map[entry->node_id];
+	DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
+
+	for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
+		if (adev->jpeg.inst[inst].aid_id == i)
+			break;
+
+	if (inst >= adev->jpeg.num_jpeg_inst) {
+		dev_WARN_ONCE(adev->dev, 1,
+			      "Interrupt received for unknown JPEG instance %d",
+			      entry->node_id);
+		return 0;
+	}
+
+	switch (entry->src_id) {
+	case VCN_5_0__SRCID__JPEG_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
+		break;
+	case VCN_5_0__SRCID__JPEG1_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
+		break;
+	case VCN_5_0__SRCID__JPEG2_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
+		break;
+	case VCN_5_0__SRCID__JPEG3_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
+		break;
+	case VCN_5_0__SRCID__JPEG4_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
+		break;
+	case VCN_5_0__SRCID__JPEG5_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
+		break;
+	case VCN_5_0__SRCID__JPEG6_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
+		break;
+	case VCN_5_0__SRCID__JPEG7_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
+		break;
+	case VCN_5_0__SRCID__JPEG8_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[8]);
+		break;
+	case VCN_5_0__SRCID__JPEG9_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[9]);
+		break;
+	default:
+		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
+			      entry->src_id, entry->src_data[0]);
+		break;
+	}
+
+	return 0;
+}
+
+static void jpeg_v5_0_2_core_stall_reset(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	int jpeg_inst = GET_INST(JPEG, ring->me);
+	int reg_offset = ring->pipe ? jpeg_v5_0_2_core_reg_offset(ring->pipe) : 0;
+
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JMI0_UVD_JMI_CLIENT_STALL,
+			    reg_offset, 0x1F);
+	SOC15_WAIT_ON_RREG_OFFSET(JPEG, jpeg_inst,
+				  regUVD_JMI0_UVD_JMI_CLIENT_CLEAN_STATUS,
+				  reg_offset, 0x1F, 0x1F);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JMI0_JPEG_LMI_DROP,
+			    reg_offset, 0x1F);
+	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CORE_RST_CTRL, 1 << ring->pipe);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JMI0_UVD_JMI_CLIENT_STALL,
+			    reg_offset, 0x00);
+	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
+			    regUVD_JMI0_JPEG_LMI_DROP,
+			    reg_offset, 0x00);
+	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CORE_RST_CTRL, 0x00);
+}
+
+static int jpeg_v5_0_2_ring_reset(struct amdgpu_ring *ring,
+				  unsigned int vmid,
+				  struct amdgpu_fence *timedout_fence)
+{
+	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
+	jpeg_v5_0_2_core_stall_reset(ring);
+	jpeg_v5_0_2_init_jrbc(ring);
+	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+}
+
+static const struct amd_ip_funcs jpeg_v5_0_2_ip_funcs = {
+	.name = "jpeg_v5_0_2",
+	.early_init = jpeg_v5_0_2_early_init,
+	.late_init = NULL,
+	.sw_init = jpeg_v5_0_2_sw_init,
+	.sw_fini = jpeg_v5_0_2_sw_fini,
+	.hw_init = jpeg_v5_0_2_hw_init,
+	.hw_fini = jpeg_v5_0_2_hw_fini,
+	.suspend = jpeg_v5_0_2_suspend,
+	.resume = jpeg_v5_0_2_resume,
+	.is_idle = jpeg_v5_0_2_is_idle,
+	.wait_for_idle = jpeg_v5_0_2_wait_for_idle,
+	.check_soft_reset = NULL,
+	.pre_soft_reset = NULL,
+	.soft_reset = NULL,
+	.post_soft_reset = NULL,
+	.set_clockgating_state = jpeg_v5_0_2_set_clockgating_state,
+	.set_powergating_state = jpeg_v5_0_2_set_powergating_state,
+	.dump_ip_state = amdgpu_jpeg_dump_ip_state,
+	.print_ip_state = amdgpu_jpeg_print_ip_state,
+};
+
+static const struct amdgpu_ring_funcs jpeg_v5_0_2_dec_ring_vm_funcs = {
+	.type = AMDGPU_RING_TYPE_VCN_JPEG,
+	.align_mask = 0xf,
+	.get_rptr = jpeg_v5_0_2_dec_ring_get_rptr,
+	.get_wptr = jpeg_v5_0_2_dec_ring_get_wptr,
+	.set_wptr = jpeg_v5_0_2_dec_ring_set_wptr,
+	.emit_frame_size =
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
+		8 + /* jpeg_v5_0_2_dec_ring_emit_vm_flush */
+		22 + 22 + /* jpeg_v5_0_2_dec_ring_emit_fence x2 vm fence */
+		8 + 16,
+	.emit_ib_size = 22, /* jpeg_v5_0_2_dec_ring_emit_ib */
+	.emit_ib = jpeg_v4_0_3_dec_ring_emit_ib,
+	.emit_fence = jpeg_v4_0_3_dec_ring_emit_fence,
+	.emit_vm_flush = jpeg_v4_0_3_dec_ring_emit_vm_flush,
+	.emit_hdp_flush = jpeg_v4_0_3_ring_emit_hdp_flush,
+	.test_ring = amdgpu_jpeg_dec_ring_test_ring,
+	.test_ib = amdgpu_jpeg_dec_ring_test_ib,
+	.insert_nop = jpeg_v4_0_3_dec_ring_nop,
+	.insert_start = jpeg_v4_0_3_dec_ring_insert_start,
+	.insert_end = jpeg_v4_0_3_dec_ring_insert_end,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.begin_use = amdgpu_jpeg_ring_begin_use,
+	.end_use = amdgpu_jpeg_ring_end_use,
+	.emit_wreg = jpeg_v4_0_3_dec_ring_emit_wreg,
+	.emit_reg_wait = jpeg_v4_0_3_dec_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+	.reset = jpeg_v5_0_2_ring_reset,
+};
+
+static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev)
+{
+	int i, j, jpeg_inst;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+			adev->jpeg.inst[i].ring_dec[j].funcs = &jpeg_v5_0_2_dec_ring_vm_funcs;
+			adev->jpeg.inst[i].ring_dec[j].me = i;
+			adev->jpeg.inst[i].ring_dec[j].pipe = j;
+		}
+		jpeg_inst = GET_INST(JPEG, i);
+		adev->jpeg.inst[i].aid_id =
+			jpeg_inst / adev->jpeg.num_inst_per_aid;
+	}
+}
+
+static const struct amdgpu_irq_src_funcs jpeg_v5_0_2_irq_funcs = {
+	.set = jpeg_v5_0_2_set_interrupt_state,
+	.process = jpeg_v5_0_2_process_interrupt,
+};
+
+static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i)
+		adev->jpeg.inst->irq.num_types += adev->jpeg.num_jpeg_rings;
+
+	adev->jpeg.inst->irq.funcs = &jpeg_v5_0_2_irq_funcs;
+}
+
+const struct amdgpu_ip_block_version jpeg_v5_0_2_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_JPEG,
+	.major = 5,
+	.minor = 0,
+	.rev = 2,
+	.funcs = &jpeg_v5_0_2_ip_funcs,
+};
+
+#if 0
+static int jpeg_v5_0_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
+					enum aca_smu_type type, void *data)
+{
+	struct aca_bank_info info;
+	u64 misc0;
+	int ret;
+
+	ret = aca_bank_info_decode(bank, &info);
+	if (ret)
+		return ret;
+
+	misc0 = bank->regs[ACA_REG_IDX_MISC0];
+	switch (type) {
+	case ACA_SMU_TYPE_UE:
+		bank->aca_err_type = ACA_ERROR_TYPE_UE;
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
+						1ULL);
+		break;
+	case ACA_SMU_TYPE_CE:
+		bank->aca_err_type = ACA_ERROR_TYPE_CE;
+		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
+						ACA_REG__MISC0__ERRCNT(misc0));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+/* reference to smu driver if header file */
+static int jpeg_v5_0_2_err_codes[] = {
+	16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-9][S|D] */
+	24, 25, 26, 27, 28, 29, 30, 31,
+	48, 49, 50, 51,
+};
+
+static bool jpeg_v5_0_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
+					enum aca_smu_type type, void *data)
+{
+	u32 instlo;
+
+	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
+	instlo &= GENMASK(31, 1);
+
+	if (instlo != mmSMNAID_AID0_MCA_SMU)
+		return false;
+
+	if (aca_bank_check_error_codes(handle->adev, bank,
+					jpeg_v5_0_2_err_codes,
+					ARRAY_SIZE(jpeg_v5_0_2_err_codes)))
+		return false;
+
+	return true;
+}
+
+static const struct aca_bank_ops jpeg_v5_0_2_aca_bank_ops = {
+	.aca_bank_parser = jpeg_v5_0_2_aca_bank_parser,
+	.aca_bank_is_valid = jpeg_v5_0_2_aca_bank_is_valid,
+};
+
+static const struct aca_info jpeg_v5_0_2_aca_info = {
+	.hwip = ACA_HWIP_TYPE_SMU,
+	.mask = ACA_ERROR_UE_MASK,
+	.bank_ops = &jpeg_v5_0_2_aca_bank_ops,
+};
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.h b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.h
new file mode 100644
index 0000000000000..c201a2c42a0dc
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2025-2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __JPEG_V5_0_2_H__
+#define __JPEG_V5_0_2_H__
+
+extern const struct amdgpu_ip_block_version jpeg_v5_0_2_ip_block;
+
+#define regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET			0x4094
+#define regUVD_JRBC_EXTERNAL_MCM_ADDR_INTERNAL_OFFSET			0x1bffe
+
+#define regUVD_JRBC0_UVD_JRBC_RB_WPTR                                                         0x0640
+#define regUVD_JRBC0_UVD_JRBC_RB_WPTR_BASE_IDX                                                1
+#define regUVD_JRBC0_UVD_JRBC_STATUS                                                          0x0649
+#define regUVD_JRBC0_UVD_JRBC_STATUS_BASE_IDX                                                 1
+#define regUVD_JRBC0_UVD_JRBC_RB_RPTR                                                         0x064a
+#define regUVD_JRBC0_UVD_JRBC_RB_RPTR_BASE_IDX                                                1
+#define regUVD_JRBC1_UVD_JRBC_RB_WPTR                                                         0x0000
+#define regUVD_JRBC1_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC1_UVD_JRBC_STATUS                                                          0x0009
+#define regUVD_JRBC1_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC1_UVD_JRBC_RB_RPTR                                                         0x000a
+#define regUVD_JRBC1_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC2_UVD_JRBC_RB_WPTR                                                         0x0040
+#define regUVD_JRBC2_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC2_UVD_JRBC_STATUS                                                          0x0049
+#define regUVD_JRBC2_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC2_UVD_JRBC_RB_RPTR                                                         0x004a
+#define regUVD_JRBC2_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC3_UVD_JRBC_RB_WPTR                                                         0x0080
+#define regUVD_JRBC3_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC3_UVD_JRBC_STATUS                                                          0x0089
+#define regUVD_JRBC3_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC3_UVD_JRBC_RB_RPTR                                                         0x008a
+#define regUVD_JRBC3_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC4_UVD_JRBC_RB_WPTR                                                         0x00c0
+#define regUVD_JRBC4_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC4_UVD_JRBC_STATUS                                                          0x00c9
+#define regUVD_JRBC4_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC4_UVD_JRBC_RB_RPTR                                                         0x00ca
+#define regUVD_JRBC4_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC5_UVD_JRBC_RB_WPTR                                                         0x0100
+#define regUVD_JRBC5_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC5_UVD_JRBC_STATUS                                                          0x0109
+#define regUVD_JRBC5_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC5_UVD_JRBC_RB_RPTR                                                         0x010a
+#define regUVD_JRBC5_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC6_UVD_JRBC_RB_WPTR                                                         0x0140
+#define regUVD_JRBC6_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC6_UVD_JRBC_STATUS                                                          0x0149
+#define regUVD_JRBC6_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC6_UVD_JRBC_RB_RPTR                                                         0x014a
+#define regUVD_JRBC6_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC7_UVD_JRBC_RB_WPTR                                                         0x0180
+#define regUVD_JRBC7_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC7_UVD_JRBC_STATUS                                                          0x0189
+#define regUVD_JRBC7_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC7_UVD_JRBC_RB_RPTR                                                         0x018a
+#define regUVD_JRBC7_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC8_UVD_JRBC_RB_WPTR                                                         0x01c0
+#define regUVD_JRBC8_UVD_JRBC_RB_WPTR_BASE_IDX                                                0
+#define regUVD_JRBC8_UVD_JRBC_STATUS                                                          0x01c9
+#define regUVD_JRBC8_UVD_JRBC_STATUS_BASE_IDX                                                 0
+#define regUVD_JRBC8_UVD_JRBC_RB_RPTR                                                         0x01ca
+#define regUVD_JRBC8_UVD_JRBC_RB_RPTR_BASE_IDX                                                0
+#define regUVD_JRBC9_UVD_JRBC_RB_WPTR                                                         0x0440
+#define regUVD_JRBC9_UVD_JRBC_RB_WPTR_BASE_IDX                                                1
+#define regUVD_JRBC9_UVD_JRBC_STATUS                                                          0x0449
+#define regUVD_JRBC9_UVD_JRBC_STATUS_BASE_IDX                                                 1
+#define regUVD_JRBC9_UVD_JRBC_RB_RPTR                                                         0x044a
+#define regUVD_JRBC9_UVD_JRBC_RB_RPTR_BASE_IDX                                                1
+#define regUVD_JMI0_JPEG_LMI_DROP                                                             0x0663
+#define regUVD_JMI0_JPEG_LMI_DROP_BASE_IDX                                                    1
+#define regUVD_JMI0_UVD_JMI_CLIENT_STALL                                                      0x067a
+#define regUVD_JMI0_UVD_JMI_CLIENT_STALL_BASE_IDX                                             1
+#define regUVD_JMI0_UVD_JMI_CLIENT_CLEAN_STATUS                                               0x067b
+#define regUVD_JMI0_UVD_JMI_CLIENT_CLEAN_STATUS_BASE_IDX                                      1
+#define regJPEG_CORE_RST_CTRL                                                                 0x072e
+#define regJPEG_CORE_RST_CTRL_BASE_IDX                                                        1
+
+#define regVCN_RRMT_CNTL                          0x0940
+#define regVCN_RRMT_CNTL_BASE_IDX                 1
+
+enum amdgpu_jpeg_v5_0_2_sub_block {
+	AMDGPU_JPEG_V5_0_2_JPEG0 = 0,
+	AMDGPU_JPEG_V5_0_2_JPEG1,
+
+	AMDGPU_JPEG_V5_0_2_MAX_SUB_BLOCK,
+};
+
+#endif /* __JPEG_V5_0_2_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
new file mode 100644
index 0000000000000..d93a0e65ce7de
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/delay.h>
+#include "amdgpu.h"
+#include "lsdma_v7_1.h"
+#include "amdgpu_lsdma.h"
+
+#include "lsdma/lsdma_7_1_0_offset.h"
+#include "lsdma/lsdma_7_1_0_sh_mask.h"
+
+static int lsdma_v7_1_wait_pio_status(struct amdgpu_device *adev)
+{
+	return amdgpu_lsdma_wait_for(adev, SOC15_REG_OFFSET(LSDMA, 0, regLSDMA_PIO_STATUS),
+			LSDMA_PIO_STATUS__PIO_IDLE_MASK | LSDMA_PIO_STATUS__PIO_FIFO_EMPTY_MASK,
+			LSDMA_PIO_STATUS__PIO_IDLE_MASK | LSDMA_PIO_STATUS__PIO_FIFO_EMPTY_MASK);
+}
+
+static int lsdma_v7_1_copy_mem(struct amdgpu_device *adev,
+			       uint64_t src_addr,
+			       uint64_t dst_addr,
+			       uint64_t size)
+{
+	int ret;
+	uint32_t tmp;
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_SRC_ADDR_LO, lower_32_bits(src_addr));
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_SRC_ADDR_HI, upper_32_bits(src_addr));
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_DST_ADDR_LO, lower_32_bits(dst_addr));
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_DST_ADDR_HI, upper_32_bits(dst_addr));
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_CONTROL, 0x0);
+
+	tmp = RREG32_SOC15(LSDMA, 0, regLSDMA_PIO_COMMAND);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, COUNT, size);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, RAW_WAIT, 0);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, CONSTANT_FILL, 0);
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_COMMAND, tmp);
+
+	ret = lsdma_v7_1_wait_pio_status(adev);
+	if (ret)
+		dev_err(adev->dev, "LSDMA PIO failed to copy memory!\n");
+
+	return ret;
+}
+
+static int lsdma_v7_1_fill_mem(struct amdgpu_device *adev,
+			       uint64_t dst_addr,
+			       uint32_t data,
+			       uint64_t size)
+{
+	int ret;
+	uint32_t tmp;
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_CONSTFILL_DATA, data);
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_DST_ADDR_LO, lower_32_bits(dst_addr));
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_DST_ADDR_HI, upper_32_bits(dst_addr));
+
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_CONTROL, 0x0);
+
+	tmp = RREG32_SOC15(LSDMA, 0, regLSDMA_PIO_COMMAND);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, COUNT, size);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, RAW_WAIT, 0);
+	tmp = REG_SET_FIELD(tmp, LSDMA_PIO_COMMAND, CONSTANT_FILL, 1);
+	WREG32_SOC15(LSDMA, 0, regLSDMA_PIO_COMMAND, tmp);
+
+	ret = lsdma_v7_1_wait_pio_status(adev);
+	if (ret)
+		dev_err(adev->dev, "LSDMA PIO failed to fill memory!\n");
+
+	return ret;
+}
+
+const struct amdgpu_lsdma_funcs lsdma_v7_1_funcs = {
+	.copy_mem = lsdma_v7_1_copy_mem,
+	.fill_mem = lsdma_v7_1_fill_mem,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.h b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.h
new file mode 100644
index 0000000000000..3d1ab605849a0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __LSDMA_V7_1_H__
+#define __LSDMA_V7_1_H__
+
+#include "soc15_common.h"
+
+extern const struct amdgpu_lsdma_funcs lsdma_v7_1_funcs;
+
+#endif /* __LSDMA_V7_1_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index 7b8c670d0a9ed..5dcc2c32644ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -31,6 +31,8 @@
 #include "gc/gc_11_0_0_default.h"
 #include "v12_structs.h"
 #include "mes_v12_api_def.h"
+#include "gfx_v12_1_pkt.h"
+#include "sdma_v7_1_0_pkt_open.h"
 
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes.bin");
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes1.bin");
@@ -41,6 +43,7 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id);
 static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
 static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
 static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
+static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id);
 
 #define MES_EOP_SIZE   2048
 
@@ -491,16 +494,12 @@ static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes,
 }
 static uint32_t mes_v12_1_get_xcc_from_reg(uint32_t reg_offset)
 {
-	/* Check xcc reg offset range */
-	uint32_t xcc = (reg_offset & XCC_MID_MASK) ? 4 : 0;
-	/* Each XCC has two register ranges.
-	 * These are represented in reg_offset[17:16]
-	 */
-	return ((reg_offset >> 16) & 0x3) + xcc;
+	return ((reg_offset >> 16) & 0x7);
 }
 
 static void mes_v12_1_get_rrmt(uint32_t reg, uint32_t xcc_id,
-				 struct RRMT_OPTION *rrmt_opt)
+			       struct RRMT_OPTION *rrmt_opt,
+			       uint32_t *out_reg)
 {
 	uint32_t normalized_reg = soc_v1_0_normalize_xcc_reg_offset(reg);
 
@@ -509,8 +508,12 @@ static void mes_v12_1_get_rrmt(uint32_t reg, uint32_t xcc_id,
 		rrmt_opt->mode = (xcc_id == rrmt_opt->xcd_die_id) ?
 			 MES_RRMT_MODE_LOCAL_XCD : MES_RRMT_MODE_REMOTE_XCD;
 	} else {
-		rrmt_opt->mode = MES_RRMT_MODE_LOCAL_REMOTE_AID;
+		rrmt_opt->mode = MES_RRMT_MODE_REMOTE_MID;
+		if (soc_v1_0_mid1_reg_range(reg))
+			rrmt_opt->mid_die_id = 1;
 	}
+
+	*out_reg = soc_v1_0_normalize_reg_offset(reg);
 }
 
 static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
@@ -534,65 +537,44 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
 	switch (input->op) {
 	case MES_MISC_OP_READ_REG:
 		misc_pkt.opcode = MESAPI_MISC__READ_REG;
-		misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
 		mes_v12_1_get_rrmt(input->read_reg.reg_offset,
 				   GET_INST(GC, input->xcc_id),
-				   &misc_pkt.read_reg.rrmt_opt);
-		if (misc_pkt.read_reg.rrmt_opt.mode != MES_RRMT_MODE_REMOTE_MID) {
-			misc_pkt.read_reg.reg_offset =
-				soc_v1_0_normalize_xcc_reg_offset(misc_pkt.read_reg.reg_offset);
-		}
+				   &misc_pkt.read_reg.rrmt_opt,
+				   &misc_pkt.read_reg.reg_offset);
 		break;
 	case MES_MISC_OP_WRITE_REG:
 		misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
-		misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
 		misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
 		mes_v12_1_get_rrmt(input->write_reg.reg_offset,
 				   GET_INST(GC, input->xcc_id),
-				   &misc_pkt.write_reg.rrmt_opt);
-		if (misc_pkt.write_reg.rrmt_opt.mode != MES_RRMT_MODE_REMOTE_MID) {
-			misc_pkt.write_reg.reg_offset =
-				soc_v1_0_normalize_xcc_reg_offset(misc_pkt.write_reg.reg_offset);
-		}
+				   &misc_pkt.write_reg.rrmt_opt,
+				   &misc_pkt.write_reg.reg_offset);
 		break;
 	case MES_MISC_OP_WRM_REG_WAIT:
 		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
 		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
-		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 		misc_pkt.wait_reg_mem.reg_offset2 = 0;
 		mes_v12_1_get_rrmt(input->wrm_reg.reg0,
 				   GET_INST(GC, input->xcc_id),
-				   &misc_pkt.wait_reg_mem.rrmt_opt1);
-		if (misc_pkt.wait_reg_mem.rrmt_opt1.mode != MES_RRMT_MODE_REMOTE_MID) {
-			misc_pkt.wait_reg_mem.reg_offset1 =
-				soc_v1_0_normalize_xcc_reg_offset(misc_pkt.wait_reg_mem.reg_offset1);
-		}
+				   &misc_pkt.wait_reg_mem.rrmt_opt1,
+				   &misc_pkt.wait_reg_mem.reg_offset1);
 		break;
 	case MES_MISC_OP_WRM_REG_WR_WAIT:
 		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
 		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
-		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
-		misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
 		mes_v12_1_get_rrmt(input->wrm_reg.reg0,
 				   GET_INST(GC, input->xcc_id),
-				   &misc_pkt.wait_reg_mem.rrmt_opt1);
+				   &misc_pkt.wait_reg_mem.rrmt_opt1,
+				   &misc_pkt.wait_reg_mem.reg_offset1);
 		mes_v12_1_get_rrmt(input->wrm_reg.reg1,
 				   GET_INST(GC, input->xcc_id),
-				   &misc_pkt.wait_reg_mem.rrmt_opt2);
-
-		if (misc_pkt.wait_reg_mem.rrmt_opt1.mode != MES_RRMT_MODE_REMOTE_MID) {
-			misc_pkt.wait_reg_mem.reg_offset1 =
-				soc_v1_0_normalize_xcc_reg_offset(misc_pkt.wait_reg_mem.reg_offset1);
-		}
-		if (misc_pkt.wait_reg_mem.rrmt_opt2.mode != MES_RRMT_MODE_REMOTE_MID) {
-			misc_pkt.wait_reg_mem.reg_offset2 =
-				soc_v1_0_normalize_xcc_reg_offset(misc_pkt.wait_reg_mem.reg_offset2);
-		}
+				   &misc_pkt.wait_reg_mem.rrmt_opt2,
+				   &misc_pkt.wait_reg_mem.reg_offset2);
 		break;
 	case MES_MISC_OP_SET_SHADER_DEBUGGER:
 		pipe = AMDGPU_MES_SCHED_PIPE;
@@ -1611,7 +1593,6 @@ static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 			amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[inst],
 					      &adev->mes.eop_gpu_addr[inst],
 					      NULL);
-			amdgpu_ucode_release(&adev->mes.fw[inst]);
 
 			if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
 				amdgpu_bo_free_kernel(&adev->mes.ring[inst].mqd_obj,
@@ -1622,6 +1603,9 @@ static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 		}
 	}
 
+	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++)
+		amdgpu_ucode_release(&adev->mes.fw[pipe]);
+
 	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
 		if (!adev->enable_uni_mes) {
 			amdgpu_bo_free_kernel(&adev->gfx.kiq[xcc_id].ring.mqd_obj,
@@ -1947,10 +1931,31 @@ static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
+static int mes_v12_1_late_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	/* TODO: remove it if issue fixed. */
+	if (adev->mes.enable_coop_mode)
+		return 0;
+
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		/* for COOP mode, only test master xcc. */
+		if (adev->mes.enable_coop_mode &&
+		    adev->mes.master_xcc_ids[xcc_id] != xcc_id)
+			continue;
+
+		mes_v12_1_self_test(adev, xcc_id);
+	}
+
+	return 0;
+}
+
 static const struct amd_ip_funcs mes_v12_1_ip_funcs = {
 	.name = "mes_v12_1",
 	.early_init = mes_v12_1_early_init,
-	.late_init = NULL,
+	.late_init = mes_v12_1_late_init,
 	.sw_init = mes_v12_1_sw_init,
 	.sw_fini = mes_v12_1_sw_fini,
 	.hw_init = mes_v12_1_hw_init,
@@ -1966,3 +1971,312 @@ const struct amdgpu_ip_block_version mes_v12_1_ip_block = {
 	.rev = 0,
 	.funcs = &mes_v12_1_ip_funcs,
 };
+
+static int mes_v12_1_alloc_test_buf(struct amdgpu_device *adev,
+				    struct amdgpu_bo **bo, uint64_t *addr,
+				    void **ptr, int size)
+{
+	amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+				bo, addr, ptr);
+	if (!*bo) {
+		dev_err(adev->dev, "failed to allocate test buffer bo\n");
+		return -ENOMEM;
+	}
+	memset(*ptr, 0, size);
+	return 0;
+}
+
+static int mes_v12_1_map_test_bo(struct amdgpu_device *adev,
+				 struct amdgpu_bo *bo, struct amdgpu_vm *vm,
+				 struct amdgpu_bo_va **bo_va, u64 va, int size)
+{
+	struct amdgpu_sync sync;
+	int r;
+
+	r = amdgpu_map_static_csa(adev, vm, bo, bo_va, va, size);
+	if (r)
+		return r;
+
+	amdgpu_sync_create(&sync);
+
+	r = amdgpu_vm_bo_update(adev, *bo_va, false);
+	if (r) {
+		dev_err(adev->dev, "failed to do vm_bo_update on meta data\n");
+		goto error;
+	}
+	amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update, GFP_KERNEL);
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev, "failed to update pdes on meta data\n");
+		goto error;
+	}
+	amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);
+	amdgpu_sync_wait(&sync, false);
+
+error:
+	amdgpu_sync_free(&sync);
+	return 0;
+}
+
+static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id,
+			       u32 *queue_ptr, u64 fence_gpu_addr,
+			       void *fence_cpu_ptr, void *wptr_cpu_addr,
+			       u64 doorbell_idx, int queue_type)
+{
+	volatile uint32_t *cpu_ptr = fence_cpu_ptr;
+	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	int sdma_ring_align = 0x10, compute_ring_align = 0x100;
+	uint32_t tmp, xcc_offset;
+	int r = 0, i, wptr = 0;
+
+	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
+		if (!adev->mes.enable_coop_mode) {
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regSCRATCH_REG0, 0xCAFEDEAD);
+		} else {
+			for (i = 0; i < num_xcc; i++) {
+				if (adev->mes.master_xcc_ids[i] == xcc_id)
+					WREG32_SOC15(GC, GET_INST(GC, i),
+					       regSCRATCH_REG0, 0xCAFEDEAD);
+			}
+		}
+
+		xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
+		queue_ptr[wptr++] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
+		queue_ptr[wptr++] = xcc_offset - PACKET3_SET_UCONFIG_REG_START;
+		queue_ptr[wptr++] = 0xDEADBEEF;
+
+		for (i = wptr; i < compute_ring_align; i++)
+			queue_ptr[wptr++] = PACKET3(PACKET3_NOP, 0x3FFF);
+
+	}  else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
+		*cpu_ptr = 0xCAFEDEAD;
+
+		queue_ptr[wptr++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+			SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+		queue_ptr[wptr++] = lower_32_bits(fence_gpu_addr);
+		queue_ptr[wptr++] = upper_32_bits(fence_gpu_addr);
+		queue_ptr[wptr++] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+		queue_ptr[wptr++] = 0xDEADBEEF;
+
+		for (i = wptr; i < sdma_ring_align; i++)
+			queue_ptr[wptr++] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+
+		wptr <<= 2;
+	}
+
+	atomic64_set((atomic64_t *)wptr_cpu_addr, wptr);
+	WDOORBELL64(doorbell_idx, wptr);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (queue_type == AMDGPU_RING_TYPE_SDMA) {
+			tmp = le32_to_cpu(*cpu_ptr);
+		} else {
+			if (!adev->mes.enable_coop_mode) {
+				tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+						   regSCRATCH_REG0);
+			} else {
+				for (i = 0; i < num_xcc; i++) {
+					if (xcc_id != adev->mes.master_xcc_ids[i])
+						continue;
+
+					tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+							   regSCRATCH_REG0);
+					if (tmp != 0xDEADBEEF)
+						break;
+				}
+			}
+		}
+
+		if (tmp == 0xDEADBEEF)
+			break;
+
+		if (amdgpu_emu_mode == 1)
+			msleep(1);
+		else
+			udelay(1);
+	}
+
+	if (i >= adev->usec_timeout) {
+		dev_err(adev->dev, "xcc%d: mes self test (%s) failed\n", xcc_id,
+		      queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
+
+		while (halt_if_hws_hang)
+			schedule();
+
+		r = -ETIMEDOUT;
+	} else {
+		dev_info(adev->dev, "xcc%d: mes self test (%s) pass\n", xcc_id,
+		      queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
+	}
+
+	return r;
+}
+
+#define USER_CTX_SIZE (PAGE_SIZE * 2)
+#define USER_CTX_VA AMDGPU_VA_RESERVED_BOTTOM
+#define RING_OFFSET(addr) ((addr))
+#define EOP_OFFSET(addr)  ((addr) + PAGE_SIZE)
+#define WPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64))
+#define RPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 2)
+#define FENCE_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 3)
+
+static int mes_v12_1_test_queue(struct amdgpu_device *adev, int xcc_id,
+		     int pasid, struct amdgpu_vm *vm, u64 meta_gpu_addr,
+		     u64 queue_gpu_addr, void *ctx_ptr, int queue_type)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+	struct amdgpu_mqd *mqd_mgr = &adev->mqds[queue_type];
+	struct amdgpu_mqd_prop mqd_prop = {0};
+	struct mes_add_queue_input add_queue = {0};
+	struct mes_remove_queue_input remove_queue = {0};
+	struct amdgpu_bo *mqd_bo = NULL;
+	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	int i, r, off, mqd_size, mqd_count = 1;
+	void *mqd_ptr = NULL;
+	u64 mqd_gpu_addr, doorbell_idx;
+
+	/* extra one page size padding for mes fw */
+	mqd_size = mqd_mgr->mqd_size + PAGE_SIZE;
+
+	if (queue_type == AMDGPU_RING_TYPE_SDMA) {
+		doorbell_idx = adev->mes.db_start_dw_offset +	\
+			adev->doorbell_index.sdma_engine[0];
+	} else {
+		doorbell_idx = adev->mes.db_start_dw_offset + \
+			adev->doorbell_index.userqueue_start;
+	}
+
+	if (adev->mes.enable_coop_mode &&
+	    queue_type == AMDGPU_RING_TYPE_COMPUTE) {
+		for (i = 0, mqd_count = 0; i < num_xcc; i++) {
+			if (adev->mes.master_xcc_ids[i] == xcc_id)
+				mqd_count++;
+		}
+		mqd_size *= mqd_count;
+	}
+
+	r = mes_v12_1_alloc_test_buf(adev, &mqd_bo, &mqd_gpu_addr,
+				     &mqd_ptr, mqd_size * mqd_count);
+	if (r < 0)
+		return r;
+
+	mqd_prop.mqd_gpu_addr = mqd_gpu_addr;
+	mqd_prop.hqd_base_gpu_addr = RING_OFFSET(USER_CTX_VA);
+	mqd_prop.eop_gpu_addr = EOP_OFFSET(USER_CTX_VA);
+	mqd_prop.wptr_gpu_addr = WPTR_OFFSET(USER_CTX_VA);
+	mqd_prop.rptr_gpu_addr = RPTR_OFFSET(USER_CTX_VA);
+	mqd_prop.doorbell_index = doorbell_idx;
+	mqd_prop.queue_size = PAGE_SIZE;
+	mqd_prop.mqd_stride_size = mqd_size;
+	mqd_prop.use_doorbell = true;
+	mqd_prop.hqd_active = false;
+
+	mqd_mgr->init_mqd(adev, mqd_ptr, &mqd_prop);
+	if (mqd_count > 1) {
+		for (i = 1; i < mqd_count; i++) {
+			off = mqd_size * i;
+			mqd_prop.mqd_gpu_addr = mqd_gpu_addr + off;
+			mqd_mgr->init_mqd(adev, (char *)mqd_ptr + off,
+					  &mqd_prop);
+		}
+	}
+
+	add_queue.xcc_id = xcc_id;
+	add_queue.process_id = pasid;
+	add_queue.page_table_base_addr = adev->vm_manager.vram_base_offset +
+		amdgpu_bo_gpu_offset(vm->root.bo) - adev->gmc.vram_start;
+	add_queue.process_va_start = 0;
+	add_queue.process_va_end = adev->vm_manager.max_pfn - 1;
+	add_queue.process_context_addr = meta_gpu_addr;
+	add_queue.gang_context_addr = meta_gpu_addr + AMDGPU_MES_PROC_CTX_SIZE;
+	add_queue.doorbell_offset = doorbell_idx;
+	add_queue.mqd_addr = mqd_gpu_addr;
+	add_queue.wptr_addr = mqd_prop.wptr_gpu_addr;
+	add_queue.wptr_mc_addr = WPTR_OFFSET(queue_gpu_addr);
+	add_queue.queue_type = queue_type;
+	add_queue.vm_cntx_cntl = hub->vm_cntx_cntl;
+
+	r = mes_v12_1_add_hw_queue(&adev->mes, &add_queue);
+	if (r)
+		goto error;
+
+	mes_v12_1_test_ring(adev, xcc_id, (u32 *)RING_OFFSET((char *)ctx_ptr),
+			    FENCE_OFFSET(USER_CTX_VA),
+			    FENCE_OFFSET((char *)ctx_ptr),
+			    WPTR_OFFSET((char *)ctx_ptr),
+			    doorbell_idx, queue_type);
+
+	remove_queue.xcc_id = xcc_id;
+	remove_queue.doorbell_offset = doorbell_idx;
+	remove_queue.gang_context_addr = add_queue.gang_context_addr;
+	r = mes_v12_1_remove_hw_queue(&adev->mes, &remove_queue);
+
+error:
+	amdgpu_bo_free_kernel(&mqd_bo, &mqd_gpu_addr, &mqd_ptr);
+	return r;
+}
+
+static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id)
+{
+	int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE,
+		              /* AMDGPU_RING_TYPE_SDMA */ };
+	struct amdgpu_bo_va *bo_va = NULL;
+	struct amdgpu_vm *vm = NULL;
+	struct amdgpu_bo *meta_bo = NULL, *ctx_bo = NULL;
+	void *meta_ptr = NULL, *ctx_ptr = NULL;
+	u64 meta_gpu_addr, ctx_gpu_addr;
+	int size, i, r, pasid;;
+
+	pasid = amdgpu_pasid_alloc(16);
+	if (pasid < 0)
+		pasid = 0;
+
+	size = AMDGPU_MES_PROC_CTX_SIZE + AMDGPU_MES_GANG_CTX_SIZE;
+	r = mes_v12_1_alloc_test_buf(adev, &meta_bo, &meta_gpu_addr,
+				     &meta_ptr, size);
+	if (r < 0)
+		goto err2;
+
+	r = mes_v12_1_alloc_test_buf(adev, &ctx_bo, &ctx_gpu_addr,
+				     &ctx_ptr, USER_CTX_SIZE);
+	if (r < 0)
+		goto err2;
+
+	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	if (!vm) {
+		r = -ENOMEM;
+		goto err2;
+	}
+
+	r = amdgpu_vm_init(adev, vm, -1, pasid);
+	if (r)
+		goto err1;
+
+	r = mes_v12_1_map_test_bo(adev, ctx_bo, vm, &bo_va,
+				  USER_CTX_VA, USER_CTX_SIZE);
+	if (r)
+		goto err0;
+
+	for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
+		memset(ctx_ptr, 0, USER_CTX_SIZE);
+
+		r = mes_v12_1_test_queue(adev, xcc_id, pasid, vm, meta_gpu_addr,
+					 ctx_gpu_addr, ctx_ptr, queue_types[i]);
+		if (r)
+			break;
+	}
+
+	amdgpu_unmap_static_csa(adev, vm, ctx_bo, bo_va, USER_CTX_VA);
+err0:
+	amdgpu_vm_fini(adev, vm);
+err1:
+	kfree(vm);
+err2:
+	amdgpu_bo_free_kernel(&meta_bo, &meta_gpu_addr, &meta_ptr);
+	amdgpu_bo_free_kernel(&ctx_bo, &ctx_gpu_addr, &ctx_ptr);
+	amdgpu_pasid_free(pasid);
+	return r;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
index a0cc8e218ca1e..42a09a277ec3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
@@ -141,7 +141,7 @@ mmhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					     uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -151,22 +151,7 @@ mmhub_v2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(2, 0, 0):
-	case IP_VERSION(2, 0, 2):
-		mmhub_cid = mmhub_client_ids_navi1x[cid][rw];
-		break;
-	case IP_VERSION(2, 1, 0):
-	case IP_VERSION(2, 1, 1):
-		mmhub_cid = mmhub_client_ids_sienna_cichlid[cid][rw];
-		break;
-	case IP_VERSION(2, 1, 2):
-		mmhub_cid = mmhub_client_ids_beige_goby[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -518,6 +503,31 @@ static const struct amdgpu_vmhub_funcs mmhub_v2_0_vmhub_funcs = {
 	.get_invalidate_req = mmhub_v2_0_get_invalidate_req,
 };
 
+static void mmhub_v2_0_init_client_info(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
+	case IP_VERSION(2, 0, 0):
+	case IP_VERSION(2, 0, 2):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_navi1x,
+					     ARRAY_SIZE(mmhub_client_ids_navi1x));
+		break;
+	case IP_VERSION(2, 1, 0):
+	case IP_VERSION(2, 1, 1):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_sienna_cichlid,
+					     ARRAY_SIZE(mmhub_client_ids_sienna_cichlid));
+		break;
+	case IP_VERSION(2, 1, 2):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_beige_goby,
+					     ARRAY_SIZE(mmhub_client_ids_beige_goby));
+		break;
+	default:
+		break;
+	}
+}
+
 static void mmhub_v2_0_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
@@ -558,6 +568,8 @@ static void mmhub_v2_0_init(struct amdgpu_device *adev)
 		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 
 	hub->vmhub_funcs = &mmhub_v2_0_vmhub_funcs;
+
+	mmhub_v2_0_init_client_info(adev);
 }
 
 static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
index 5eb8122e27469..31c479d76c421 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c
@@ -80,7 +80,7 @@ mmhub_v2_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					     uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -90,16 +90,7 @@ mmhub_v2_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(2, 3, 0):
-	case IP_VERSION(2, 4, 0):
-	case IP_VERSION(2, 4, 1):
-		mmhub_cid = mmhub_client_ids_vangogh[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -486,6 +477,10 @@ static void mmhub_v2_3_init(struct amdgpu_device *adev)
 		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 
 	hub->vmhub_funcs = &mmhub_v2_3_vmhub_funcs;
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_vangogh,
+				     ARRAY_SIZE(mmhub_client_ids_vangogh));
 }
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
index 7d5242df58a51..3d82cfa0f1b51 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
@@ -97,7 +97,7 @@ mmhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					     uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -107,15 +107,7 @@ mmhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(3, 0, 0):
-	case IP_VERSION(3, 0, 1):
-		mmhub_cid = mmhub_client_ids_v3_0_0[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -520,6 +512,10 @@ static void mmhub_v3_0_init(struct amdgpu_device *adev)
 		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXTS_DISABLE);
 
 	hub->vmhub_funcs = &mmhub_v3_0_vmhub_funcs;
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_v3_0_0,
+				     ARRAY_SIZE(mmhub_client_ids_v3_0_0));
 }
 
 static u64 mmhub_v3_0_get_fb_location(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c
index 910337dc28d10..a1b0b7b39a42a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c
@@ -104,7 +104,7 @@ mmhub_v3_0_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					      uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -114,16 +114,7 @@ mmhub_v3_0_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(3, 0, 1):
-		mmhub_cid = mmhub_client_ids_v3_0_1[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
-
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -503,6 +494,10 @@ static void mmhub_v3_0_1_init(struct amdgpu_device *adev)
 		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 
 	hub->vmhub_funcs = &mmhub_v3_0_1_vmhub_funcs;
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_v3_0_1,
+				     ARRAY_SIZE(mmhub_client_ids_v3_0_1));
 }
 
 static u64 mmhub_v3_0_1_get_fb_location(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c
index f0f182f033b98..34e8dbd47c0f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c
@@ -97,7 +97,7 @@ mmhub_v3_0_2_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					     uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -107,8 +107,7 @@ mmhub_v3_0_2_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-
-	mmhub_cid = mmhub_client_ids_v3_0_2[cid][rw];
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -509,6 +508,10 @@ static void mmhub_v3_0_2_init(struct amdgpu_device *adev)
 		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_BANK_SELECT_RESERVED_CID2);
 
 	hub->vmhub_funcs = &mmhub_v3_0_2_vmhub_funcs;
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_v3_0_2,
+				     ARRAY_SIZE(mmhub_client_ids_v3_0_2));
 }
 
 static u64 mmhub_v3_0_2_get_fb_location(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c
index daf1f8ad4ccaa..cfce7e1297d4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c
@@ -217,7 +217,7 @@ mmhub_v3_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					      uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
@@ -227,29 +227,10 @@ mmhub_v3_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
 		status);
-
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(3, 3, 0):
-	case IP_VERSION(3, 3, 2):
-		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3) ?
-			    mmhub_client_ids_v3_3[cid][rw] :
-			    cid == 0x140 ? "UMSCH" : NULL;
-		break;
-	case IP_VERSION(3, 3, 1):
-		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3_1) ?
-			    mmhub_client_ids_v3_3_1[cid][rw] :
-			    cid == 0x140 ? "UMSCH" : NULL;
-		break;
-	case IP_VERSION(3, 4, 0):
-		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_4) ?
-			mmhub_client_ids_v3_4[cid][rw] :
-		cid == 0x140 ? "UMSCH" : NULL;
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
-
+	if (cid == 0x140)
+		mmhub_cid = "UMSCH";
+	else
+		mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -640,6 +621,30 @@ static const struct amdgpu_vmhub_funcs mmhub_v3_3_vmhub_funcs = {
 	.get_invalidate_req = mmhub_v3_3_get_invalidate_req,
 };
 
+static void mmhub_v3_3_init_client_info(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
+	case IP_VERSION(3, 3, 0):
+	case IP_VERSION(3, 3, 2):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_v3_3,
+					     ARRAY_SIZE(mmhub_client_ids_v3_3));
+		break;
+	case IP_VERSION(3, 3, 1):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_v3_3_1,
+					     ARRAY_SIZE(mmhub_client_ids_v3_3_1));
+		break;
+	case IP_VERSION(3, 4, 0):
+		amdgpu_mmhub_init_client_info(&adev->mmhub,
+					     mmhub_client_ids_v3_4,
+					     ARRAY_SIZE(mmhub_client_ids_v3_4));
+		break;
+	default:
+		break;
+	}
+}
+
 static void mmhub_v3_3_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
@@ -680,6 +685,8 @@ static void mmhub_v3_3_init(struct amdgpu_device *adev)
 		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
 
 	hub->vmhub_funcs = &mmhub_v3_3_vmhub_funcs;
+
+	mmhub_v3_3_init_client_info(adev);
 }
 
 static u64 mmhub_v3_3_get_fb_location(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c
index 951998454b257..bef75c4c48d3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c
@@ -90,7 +90,7 @@ mmhub_v4_1_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					      uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS_LO32, CID);
@@ -100,14 +100,7 @@ mmhub_v4_1_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n",
 		status);
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(4, 1, 0):
-		mmhub_cid = mmhub_client_ids_v4_1_0[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -514,6 +507,10 @@ static void mmhub_v4_1_0_init(struct amdgpu_device *adev)
 		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXTS_DISABLE);
 
 	hub->vmhub_funcs = &mmhub_v4_1_0_vmhub_funcs;
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_v4_1_0,
+				     ARRAY_SIZE(mmhub_client_ids_v4_1_0));
 }
 
 static u64 mmhub_v4_1_0_get_fb_location(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
index a72770e3d0e99..29f7ed4668587 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
@@ -72,6 +72,45 @@ static const char *mmhub_client_ids_v4_2_0[][2] = {
 	[23][1] = "VCN1",
 };
 
+static int mmhub_v4_2_0_get_xgmi_info(struct amdgpu_device *adev)
+{
+	u32 max_num_physical_nodes;
+	u32 max_physical_node_id;
+	u32 xgmi_lfb_cntl;
+	u32 max_region;
+	u64 seg_size;
+
+	/* limit this callback to A + A configuration only */
+	if (!adev->gmc.xgmi.connected_to_cpu)
+		return 0;
+
+	xgmi_lfb_cntl = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0),
+				     regMMMC_VM_XGMI_LFB_CNTL);
+	seg_size = REG_GET_FIELD(
+		RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMMMC_VM_XGMI_LFB_SIZE),
+		MMMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
+	max_region =
+		REG_GET_FIELD(xgmi_lfb_cntl, MMMC_VM_XGMI_LFB_CNTL, PF_MAX_REGION);
+
+	max_num_physical_nodes   = 4;
+	max_physical_node_id     = 3;
+
+	adev->gmc.xgmi.num_physical_nodes = max_region + 1;
+
+	if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
+		return -EINVAL;
+
+	adev->gmc.xgmi.physical_node_id =
+		REG_GET_FIELD(xgmi_lfb_cntl, MMMC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
+
+	if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
+		return -EINVAL;
+
+	adev->gmc.xgmi.node_segment_size = seg_size;
+
+	return 0;
+}
+
 static u64 mmhub_v4_2_0_get_fb_location(struct amdgpu_device *adev)
 {
 	u64 base;
@@ -131,7 +170,7 @@ static void mmhub_v4_2_0_setup_vm_pt_regs(struct amdgpu_device *adev,
 static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
 						     uint32_t mid_mask)
 {
-	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+	uint64_t pt_base;
 	int i;
 
 	if (adev->gmc.pdb0_bo)
@@ -152,10 +191,10 @@ static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
 
 			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
-				     (u32)(adev->gmc.fb_end >> 12));
+				     (u32)(adev->gmc.gart_end >> 12));
 			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
-				     (u32)(adev->gmc.fb_end >> 44));
+				     (u32)(adev->gmc.gart_end >> 44));
 		} else {
 			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
 				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
@@ -190,41 +229,74 @@ static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *ade
 		return;
 
 	for_each_inst(i, mid_mask) {
-		/* Program the AGP BAR */
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BASE_LO32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BASE_HI32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BOT_LO32,
-			     lower_32_bits(adev->gmc.agp_start >> 24));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BOT_HI32,
-			     upper_32_bits(adev->gmc.agp_start >> 24));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_TOP_LO32,
-			     lower_32_bits(adev->gmc.agp_end >> 24));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_TOP_HI32,
-			     upper_32_bits(adev->gmc.agp_end >> 24));
+		if (adev->gmc.pdb0_bo) {
+			/* Disable agp and system aperture
+			 * when vmid0 page table is enabled */
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_FB_LOCATION_TOP_LO32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_FB_LOCATION_TOP_HI32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_FB_LOCATION_BASE_LO32,
+				     0xFFFFFFFF);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_FB_LOCATION_BASE_HI32, 1);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_TOP_LO32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_TOP_HI32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BOT_LO32,
+				     0xFFFFFFFF);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BOT_HI32, 1);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+				     0xFFFFFFFF);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+				     0x7F);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
+		} else {
+			/* Program the AGP BAR */
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BASE_LO32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BASE_HI32, 0);
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BOT_LO32,
+				     lower_32_bits(adev->gmc.agp_start >> 24));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_BOT_HI32,
+				     upper_32_bits(adev->gmc.agp_start >> 24));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_TOP_LO32,
+				     lower_32_bits(adev->gmc.agp_end >> 24));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_AGP_TOP_HI32,
+				     upper_32_bits(adev->gmc.agp_end >> 24));
 
-		/* Program the system aperture low logical page number. */
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
-			     lower_32_bits(min(adev->gmc.fb_start,
-					       adev->gmc.agp_start) >> 18));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
-			     upper_32_bits(min(adev->gmc.fb_start,
-					       adev->gmc.agp_start) >> 18));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
-			     lower_32_bits(max(adev->gmc.fb_end,
-					       adev->gmc.agp_end) >> 18));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
-			     upper_32_bits(max(adev->gmc.fb_end,
-					       adev->gmc.agp_end) >> 18));
+			/* Program the system aperture low logical page number. */
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+				     lower_32_bits(min(adev->gmc.fb_start,
+						   adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+				     upper_32_bits(min(adev->gmc.fb_start,
+						   adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
+				     lower_32_bits(max(adev->gmc.fb_end,
+						   adev->gmc.agp_end) >> 18));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
+				     upper_32_bits(max(adev->gmc.fb_end,
+						   adev->gmc.agp_end) >> 18));
+		}
 
 		/* Set default page address. */
 		value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
@@ -252,38 +324,6 @@ static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *ade
 		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
 			     regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 	}
-
-	/* In the case squeezing vram into GART aperture, we don't use
-	 * FB aperture and AGP aperture. Disable them.
-	 */
-	if (adev->gmc.pdb0_bo) {
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_FB_LOCATION_TOP_LO32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_FB_LOCATION_TOP_HI32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_FB_LOCATION_BASE_LO32, 0xFFFFFFFF);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_FB_LOCATION_BASE_HI32, 1);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_TOP_LO32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_TOP_HI32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BOT_LO32, 0xFFFFFFFF);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_AGP_BOT_HI32, 1);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
-			     0xFFFFFFFF);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
-			     0x7F);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
-	}
 }
 
 static void mmhub_v4_2_0_mid_init_tlb_regs(struct amdgpu_device *adev,
@@ -676,7 +716,7 @@ mmhub_v4_2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					      uint32_t status)
 {
 	uint32_t cid, rw;
-	const char *mmhub_cid = NULL;
+	const char *mmhub_cid;
 
 	cid = REG_GET_FIELD(status,
 			    MMVM_L2_PROTECTION_FAULT_STATUS_LO32, CID);
@@ -686,14 +726,7 @@ mmhub_v4_2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 	dev_err(adev->dev,
 		"MMVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n",
 		status);
-	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
-	case IP_VERSION(4, 2, 0):
-		mmhub_cid = mmhub_client_ids_v4_2_0[cid][rw];
-		break;
-	default:
-		mmhub_cid = NULL;
-		break;
-	}
+	mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
 	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
 		mmhub_cid ? mmhub_cid : "unknown", cid);
 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
@@ -784,6 +817,10 @@ static void mmhub_v4_2_0_init(struct amdgpu_device *adev)
 
 	mid_mask = adev->aid_mask;
 	mmhub_v4_2_0_mid_init(adev, mid_mask);
+
+	amdgpu_mmhub_init_client_info(&adev->mmhub,
+				     mmhub_client_ids_v4_2_0,
+				     ARRAY_SIZE(mmhub_client_ids_v4_2_0));
 }
 
 static void
@@ -883,6 +920,7 @@ const struct amdgpu_mmhub_funcs mmhub_v4_2_0_funcs = {
 	.set_fault_enable_default = mmhub_v4_2_0_set_fault_enable_default,
 	.set_clockgating = mmhub_v4_2_0_set_clockgating,
 	.get_clockgating = mmhub_v4_2_0_get_clockgating,
+	.get_xgmi_info = mmhub_v4_2_0_get_xgmi_info,
 };
 
 static int mmhub_v4_2_0_xcp_resume(void *handle, uint32_t inst_mask)
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index f17c3839aea19..7ce1a1b956065 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -283,10 +283,10 @@ static u32 nv_didt_rreg(struct amdgpu_device *adev, u32 reg)
 	address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	r = RREG32(data);
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 	return r;
 }
 
@@ -297,10 +297,10 @@ static void nv_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	WREG32(data, (v));
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 }
 
 static u32 nv_get_config_memsize(struct amdgpu_device *adev)
@@ -635,21 +635,15 @@ static int nv_common_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	adev->nbio.funcs->set_reg_remap(adev);
-	adev->smc_rreg = NULL;
-	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
-	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
-	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
-	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
-	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
-	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
-
-	/* TODO: will add them during VCN v2 implementation */
-	adev->uvd_ctx_rreg = NULL;
-	adev->uvd_ctx_wreg = NULL;
-
-	adev->didt_rreg = &nv_didt_rreg;
-	adev->didt_wreg = &nv_didt_wreg;
+	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
+	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
+	adev->reg.pcie.rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->reg.pcie.wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->reg.pcie.port_rreg = &amdgpu_device_pcie_port_rreg;
+	adev->reg.pcie.port_wreg = &amdgpu_device_pcie_port_wreg;
+
+	adev->reg.didt.rreg = &nv_didt_rreg;
+	adev->reg.didt.wreg = &nv_didt_wreg;
 
 	adev->asic_funcs = &nv_asic_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
index d1e1a43695210..a0c84f81c0c9d 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
@@ -166,7 +166,7 @@ static void psp_v13_0_bootloader_print_status(struct psp_context *psp,
 			bl_status_reg =
 				(SOC15_REG_OFFSET(MP0, 0, regMP0_SMN_C2PMSG_92)
 				 << 2) +
-				adev->asic_funcs->encode_ext_smn_addressing(i);
+				amdgpu_reg_get_smn_base64(adev, MP0_HWIP, i);
 			at += snprintf(bl_status_msg + at,
 				       PSP13_BL_STATUS_SIZE - at,
 				       " status(%02i): 0x%08x", i,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 92ce580647cdc..0090ace49024f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -51,7 +51,6 @@
 
 static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev);
 
 MODULE_FIRMWARE("amdgpu/topaz_sdma.bin");
@@ -809,6 +808,14 @@ static void sdma_v2_4_ring_emit_wreg(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, val);
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v2_4_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v2_4_vm_copy_pte,
+
+	.write_pte = sdma_v2_4_vm_write_pte,
+	.set_pte_pde = sdma_v2_4_vm_set_pte_pde,
+};
+
 static int sdma_v2_4_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -822,7 +829,7 @@ static int sdma_v2_4_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v2_4_set_ring_funcs(adev);
 	sdma_v2_4_set_buffer_funcs(adev);
-	sdma_v2_4_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v2_4_vm_pte_funcs);
 	sdma_v2_4_set_irq_funcs(adev);
 
 	return 0;
@@ -1232,26 +1239,6 @@ static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v2_4_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v2_4_vm_copy_pte,
-
-	.write_pte = sdma_v2_4_vm_write_pte,
-	.set_pte_pde = sdma_v2_4_vm_set_pte_pde,
-};
-
-static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v2_4_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version sdma_v2_4_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 2,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 1c076bd1cf73e..2526d393162ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -51,7 +51,6 @@
 
 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev);
 
 MODULE_FIRMWARE("amdgpu/tonga_sdma.bin");
@@ -1082,6 +1081,14 @@ static void sdma_v3_0_ring_emit_wreg(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, val);
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v3_0_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v3_0_vm_copy_pte,
+
+	.write_pte = sdma_v3_0_vm_write_pte,
+	.set_pte_pde = sdma_v3_0_vm_set_pte_pde,
+};
+
 static int sdma_v3_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1102,7 +1109,7 @@ static int sdma_v3_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v3_0_set_ring_funcs(adev);
 	sdma_v3_0_set_buffer_funcs(adev);
-	sdma_v3_0_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v3_0_vm_pte_funcs);
 	sdma_v3_0_set_irq_funcs(adev);
 
 	return 0;
@@ -1674,26 +1681,6 @@ static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v3_0_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v3_0_vm_copy_pte,
-
-	.write_pte = sdma_v3_0_vm_write_pte,
-	.set_pte_pde = sdma_v3_0_vm_set_pte_pde,
-};
-
-static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v3_0_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			 &adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version sdma_v3_0_ip_block =
 {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index f38004e6064e5..44f0f23e11484 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -129,7 +129,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_0[] = {
 
 static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev);
 
@@ -1751,6 +1750,14 @@ static bool sdma_v4_0_fw_support_paging_queue(struct amdgpu_device *adev)
 	}
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v4_0_vm_copy_pte,
+
+	.write_pte = sdma_v4_0_vm_write_pte,
+	.set_pte_pde = sdma_v4_0_vm_set_pte_pde,
+};
+
 static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1769,7 +1776,7 @@ static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v4_0_set_ring_funcs(adev);
 	sdma_v4_0_set_buffer_funcs(adev);
-	sdma_v4_0_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_0_vm_pte_funcs);
 	sdma_v4_0_set_irq_funcs(adev);
 	sdma_v4_0_set_ras_funcs(adev);
 
@@ -2597,48 +2604,37 @@ static void sdma_v4_0_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 22,
 	.copy_num_dw = 7,
 	.emit_copy_buffer = sdma_v4_0_emit_copy_buffer,
 
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 22,
+	.fill_num_dw = 5,
+	.emit_fill_buffer = sdma_v4_0_emit_fill_buffer,
+};
+
+static const struct amdgpu_buffer_funcs sdma_v4_4_buffer_funcs = {
+	.copy_max_bytes = 1 << 30,
+	.copy_num_dw = 7,
+	.emit_copy_buffer = sdma_v4_0_emit_copy_buffer,
+
+	.fill_max_bytes = 1 << 30,
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v4_0_emit_fill_buffer,
 };
 
 static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
+	if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= IP_VERSION(4, 4, 0))
+		adev->mman.buffer_funcs = &sdma_v4_4_buffer_funcs;
+	else
+		adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
 	if (adev->sdma.has_page_queue)
 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
 	else
 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v4_0_vm_copy_pte,
-
-	.write_pte = sdma_v4_0_vm_write_pte,
-	.set_pte_pde = sdma_v4_0_vm_set_pte_pde,
-};
-
-static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	struct drm_gpu_scheduler *sched;
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v4_0_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		if (adev->sdma.has_page_queue)
-			sched = &adev->sdma.instance[i].page.sched;
-		else
-			sched = &adev->sdma.instance[i].ring.sched;
-		adev->vm_manager.vm_pte_scheds[i] = sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 static void sdma_v4_0_get_ras_error_count(uint32_t value,
 					uint32_t instance,
 					uint32_t *sec_count)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index a1443990d5c60..78bdfed0a7fd3 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -104,7 +104,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = {
 
 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
 static void sdma_v4_4_2_update_reset_mask(struct amdgpu_device *adev);
@@ -1347,6 +1346,14 @@ static const struct amdgpu_sdma_funcs sdma_v4_4_2_sdma_funcs = {
 	.soft_reset_kernel_queue = &sdma_v4_4_2_soft_reset_engine,
 };
 
+static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v4_4_2_vm_copy_pte,
+
+	.write_pte = sdma_v4_4_2_vm_write_pte,
+	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
+};
+
 static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1362,7 +1369,7 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v4_4_2_set_ring_funcs(adev);
 	sdma_v4_4_2_set_buffer_funcs(adev);
-	sdma_v4_4_2_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_4_2_vm_pte_funcs);
 	sdma_v4_4_2_set_irq_funcs(adev);
 	sdma_v4_4_2_set_ras_funcs(adev);
 	return 0;
@@ -2298,11 +2305,11 @@ static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 30,
 	.copy_num_dw = 7,
 	.emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
 
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 30,
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
 };
@@ -2316,30 +2323,6 @@ static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v4_4_2_vm_copy_pte,
-
-	.write_pte = sdma_v4_4_2_vm_write_pte,
-	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
-};
-
-static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	struct drm_gpu_scheduler *sched;
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		if (adev->sdma.has_page_queue)
-			sched = &adev->sdma.instance[i].page.sched;
-		else
-			sched = &adev->sdma.instance[i].ring.sched;
-		adev->vm_manager.vm_pte_scheds[i] = sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 /**
  * sdma_v4_4_2_update_reset_mask - update  reset mask for SDMA
  * @adev: Pointer to the AMDGPU device structure
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index e3a035c9fecee..52f4e9e099cbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -110,7 +110,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_5_0[] = {
 
 static void sdma_v5_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v5_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v5_0_set_irq_funcs(struct amdgpu_device *adev);
 static int sdma_v5_0_stop_queue(struct amdgpu_ring *ring);
 static int sdma_v5_0_restore_queue(struct amdgpu_ring *ring);
@@ -1357,6 +1356,13 @@ static const struct amdgpu_sdma_funcs sdma_v5_0_sdma_funcs = {
 	.soft_reset_kernel_queue = &sdma_v5_0_soft_reset_engine,
 };
 
+static const struct amdgpu_vm_pte_funcs sdma_v5_0_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v5_0_vm_copy_pte,
+	.write_pte = sdma_v5_0_vm_write_pte,
+	.set_pte_pde = sdma_v5_0_vm_set_pte_pde,
+};
+
 static int sdma_v5_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1368,7 +1374,7 @@ static int sdma_v5_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v5_0_set_ring_funcs(adev);
 	sdma_v5_0_set_buffer_funcs(adev);
-	sdma_v5_0_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_0_vm_pte_funcs);
 	sdma_v5_0_set_irq_funcs(adev);
 	sdma_v5_0_set_mqd_funcs(adev);
 
@@ -2052,27 +2058,6 @@ static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev)
 	}
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v5_0_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v5_0_vm_copy_pte,
-	.write_pte = sdma_v5_0_vm_write_pte,
-	.set_pte_pde = sdma_v5_0_vm_set_pte_pde,
-};
-
-static void sdma_v5_0_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	if (adev->vm_manager.vm_pte_funcs == NULL) {
-		adev->vm_manager.vm_pte_funcs = &sdma_v5_0_vm_pte_funcs;
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			adev->vm_manager.vm_pte_scheds[i] =
-				&adev->sdma.instance[i].ring.sched;
-		}
-		adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-	}
-}
-
 const struct amdgpu_ip_block_version sdma_v5_0_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 5,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index feebaa8cd9b14..b4fb90cc8f7d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -111,7 +111,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_5_2[] = {
 
 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev);
 static int sdma_v5_2_stop_queue(struct amdgpu_ring *ring);
 static int sdma_v5_2_restore_queue(struct amdgpu_ring *ring);
@@ -1248,6 +1247,13 @@ static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v5_2_vm_copy_pte,
+	.write_pte = sdma_v5_2_vm_write_pte,
+	.set_pte_pde = sdma_v5_2_vm_set_pte_pde,
+};
+
 static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1259,7 +1265,7 @@ static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v5_2_set_ring_funcs(adev);
 	sdma_v5_2_set_buffer_funcs(adev);
-	sdma_v5_2_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_2_vm_pte_funcs);
 	sdma_v5_2_set_irq_funcs(adev);
 	sdma_v5_2_set_mqd_funcs(adev);
 
@@ -2039,11 +2045,11 @@ static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 30,
 	.copy_num_dw = 7,
 	.emit_copy_buffer = sdma_v5_2_emit_copy_buffer,
 
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 30, /* HW supports 1 << 30, but PAL uses 1 << 22 */
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v5_2_emit_fill_buffer,
 };
@@ -2056,27 +2062,6 @@ static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
 	}
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v5_2_vm_copy_pte,
-	.write_pte = sdma_v5_2_vm_write_pte,
-	.set_pte_pde = sdma_v5_2_vm_set_pte_pde,
-};
-
-static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	if (adev->vm_manager.vm_pte_funcs == NULL) {
-		adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs;
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			adev->vm_manager.vm_pte_scheds[i] =
-				&adev->sdma.instance[i].ring.sched;
-		}
-		adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-	}
-}
-
 const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 5,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index b40126f5d3ef5..b005672f2f96b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -120,7 +120,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_6_0[] = {
 
 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev);
 static int sdma_v6_0_start(struct amdgpu_device *adev);
 
@@ -1280,6 +1279,13 @@ static void sdma_v6_0_get_csa_info(struct amdgpu_device *adev,
 	csa_info->alignment = SDMA6_CSA_ALIGNMENT;
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v6_0_vm_pte_funcs = {
+	.copy_pte_num_dw = 7,
+	.copy_pte = sdma_v6_0_vm_copy_pte,
+	.write_pte = sdma_v6_0_vm_write_pte,
+	.set_pte_pde = sdma_v6_0_vm_set_pte_pde,
+};
+
 static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1308,7 +1314,7 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v6_0_set_ring_funcs(adev);
 	sdma_v6_0_set_buffer_funcs(adev);
-	sdma_v6_0_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v6_0_vm_pte_funcs);
 	sdma_v6_0_set_irq_funcs(adev);
 	sdma_v6_0_set_mqd_funcs(adev);
 	sdma_v6_0_set_ras_funcs(adev);
@@ -1878,11 +1884,11 @@ static void sdma_v6_0_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 30,
 	.copy_num_dw = 7,
 	.emit_copy_buffer = sdma_v6_0_emit_copy_buffer,
 
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 30,
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v6_0_emit_fill_buffer,
 };
@@ -1893,25 +1899,6 @@ static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v6_0_vm_pte_funcs = {
-	.copy_pte_num_dw = 7,
-	.copy_pte = sdma_v6_0_vm_copy_pte,
-	.write_pte = sdma_v6_0_vm_write_pte,
-	.set_pte_pde = sdma_v6_0_vm_set_pte_pde,
-};
-
-static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v6_0_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 6,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 8d16ef257bcb9..5679a94d0815e 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -119,7 +119,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_0[] = {
 
 static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v7_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v7_0_set_irq_funcs(struct amdgpu_device *adev);
 static int sdma_v7_0_start(struct amdgpu_device *adev);
 
@@ -1264,6 +1263,13 @@ static void sdma_v7_0_get_csa_info(struct amdgpu_device *adev,
 	csa_info->alignment = SDMA7_CSA_ALIGNMENT;
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v7_0_vm_pte_funcs = {
+	.copy_pte_num_dw = 8,
+	.copy_pte = sdma_v7_0_vm_copy_pte,
+	.write_pte = sdma_v7_0_vm_write_pte,
+	.set_pte_pde = sdma_v7_0_vm_set_pte_pde,
+};
+
 static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1294,7 +1300,7 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v7_0_set_ring_funcs(adev);
 	sdma_v7_0_set_buffer_funcs(adev);
-	sdma_v7_0_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_0_vm_pte_funcs);
 	sdma_v7_0_set_irq_funcs(adev);
 	sdma_v7_0_set_mqd_funcs(adev);
 	adev->sdma.get_csa_info = &sdma_v7_0_get_csa_info;
@@ -1829,10 +1835,10 @@ static void sdma_v7_0_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 30,
 	.copy_num_dw = 8,
 	.emit_copy_buffer = sdma_v7_0_emit_copy_buffer,
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 30,
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v7_0_emit_fill_buffer,
 };
@@ -1843,25 +1849,6 @@ static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v7_0_vm_pte_funcs = {
-	.copy_pte_num_dw = 8,
-	.copy_pte = sdma_v7_0_vm_copy_pte,
-	.write_pte = sdma_v7_0_vm_write_pte,
-	.set_pte_pde = sdma_v7_0_vm_set_pte_pde,
-};
-
-static void sdma_v7_0_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v7_0_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 7,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index 0824cba48f2e7..f20e0fc3fc743 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -110,7 +110,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_1[] = {
 
 static void sdma_v7_1_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev);
-static void sdma_v7_1_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v7_1_set_irq_funcs(struct amdgpu_device *adev);
 static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
 				uint32_t inst_mask);
@@ -1130,12 +1129,21 @@ static void sdma_v7_1_vm_set_pte_pde(struct amdgpu_ib *ib,
 	/* for physically contiguous pages (vram) */
 	u32 header = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
 
-	if (amdgpu_mtype_local)
-		header |= SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x3);
-	else
-		header |= (SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x2) |
-			   SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1) |
-			   SDMA_PKT_PTEPDE_COPY_HEADER_SCOPE(0x3));
+	/* TODO:
+	 * When VM_L2_CNTL5.WALKER_FETCH_PDE_MTYPE_ENABLE is enabled, change below MTYPE
+	 * to RW for AID A1 and UC for AID A0. NC needs additional GCR flush and need not
+	 * be supported. Also, honour amdgpu_mtype_local override. RW would additionally
+	 * require setting SCOPE bits in the header.
+	 *
+	 * header |= (SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x2:RW) |
+	 *           SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1) |
+	 *           SDMA_PKT_PTEPDE_COPY_HEADER_SCOPE(0x3:SYS_SCOPE));
+	 */
+
+	/* VM_L2_CNTL5.WALKER_FETCH_PDE_MTYPE_ENABLE is 0 which defaults to UC. So,
+	 * use MTYPE_UC (0x3). For ref. MTYPE_RW=0x2 MTYPE_NC=0x0
+	 */
+	header |= SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x3) | SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1);
 
 	ib->ptr[ib->length_dw++] = header;
 	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
@@ -1248,6 +1256,13 @@ static void sdma_v7_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
 }
 
+static const struct amdgpu_vm_pte_funcs sdma_v7_1_vm_pte_funcs = {
+	.copy_pte_num_dw = 8,
+	.copy_pte = sdma_v7_1_vm_copy_pte,
+	.write_pte = sdma_v7_1_vm_write_pte,
+	.set_pte_pde = sdma_v7_1_vm_set_pte_pde,
+};
+
 static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1261,7 +1276,7 @@ static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
 
 	sdma_v7_1_set_ring_funcs(adev);
 	sdma_v7_1_set_buffer_funcs(adev);
-	sdma_v7_1_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_1_vm_pte_funcs);
 	sdma_v7_1_set_irq_funcs(adev);
 	sdma_v7_1_set_mqd_funcs(adev);
 
@@ -1739,10 +1754,10 @@ static void sdma_v7_1_emit_fill_buffer(struct amdgpu_ib *ib,
 }
 
 static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
-	.copy_max_bytes = 0x400000,
+	.copy_max_bytes = 1 << 30,
 	.copy_num_dw = 8,
 	.emit_copy_buffer = sdma_v7_1_emit_copy_buffer,
-	.fill_max_bytes = 0x400000,
+	.fill_max_bytes = 1 << 30,
 	.fill_num_dw = 5,
 	.emit_fill_buffer = sdma_v7_1_emit_fill_buffer,
 };
@@ -1753,25 +1768,6 @@ static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs sdma_v7_1_vm_pte_funcs = {
-	.copy_pte_num_dw = 8,
-	.copy_pte = sdma_v7_1_vm_copy_pte,
-	.write_pte = sdma_v7_1_vm_write_pte,
-	.set_pte_pde = sdma_v7_1_vm_set_pte_pde,
-};
-
-static void sdma_v7_1_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &sdma_v7_1_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
 	.major = 7,
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 509d43b238f3f..c26cb3e8bff6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1027,11 +1027,11 @@ static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(AMDGPU_PCIE_INDEX, reg);
 	(void)RREG32(AMDGPU_PCIE_INDEX);
 	r = RREG32(AMDGPU_PCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -1039,12 +1039,12 @@ static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(AMDGPU_PCIE_INDEX, reg);
 	(void)RREG32(AMDGPU_PCIE_INDEX);
 	WREG32(AMDGPU_PCIE_DATA, v);
 	(void)RREG32(AMDGPU_PCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
@@ -1052,11 +1052,11 @@ static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 	(void)RREG32(PCIE_PORT_INDEX);
 	r = RREG32(PCIE_PORT_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -1064,12 +1064,12 @@ static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 	(void)RREG32(PCIE_PORT_INDEX);
 	WREG32(PCIE_PORT_DATA, (v));
 	(void)RREG32(PCIE_PORT_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
@@ -1077,10 +1077,10 @@ static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmSMC_IND_INDEX_0, (reg));
 	r = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 	return r;
 }
 
@@ -1088,10 +1088,10 @@ static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmSMC_IND_INDEX_0, (reg));
 	WREG32(mmSMC_IND_DATA_0, (v));
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 }
 
 static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
@@ -1099,10 +1099,10 @@ static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	r = RREG32(mmUVD_CTX_DATA);
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 	return r;
 }
 
@@ -1110,10 +1110,10 @@ static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	WREG32(mmUVD_CTX_DATA, (v));
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 }
 
 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
@@ -2037,16 +2037,14 @@ static int si_common_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->smc_rreg = &si_smc_rreg;
-	adev->smc_wreg = &si_smc_wreg;
-	adev->pcie_rreg = &si_pcie_rreg;
-	adev->pcie_wreg = &si_pcie_wreg;
-	adev->pciep_rreg = &si_pciep_rreg;
-	adev->pciep_wreg = &si_pciep_wreg;
-	adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
-	adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
-	adev->didt_rreg = NULL;
-	adev->didt_wreg = NULL;
+	adev->reg.smc.rreg = si_smc_rreg;
+	adev->reg.smc.wreg = si_smc_wreg;
+	adev->reg.pcie.rreg = &si_pcie_rreg;
+	adev->reg.pcie.wreg = &si_pcie_wreg;
+	adev->reg.pcie.port_rreg = &si_pciep_rreg;
+	adev->reg.pcie.port_wreg = &si_pciep_wreg;
+	adev->reg.uvd_ctx.rreg = &si_uvd_ctx_rreg;
+	adev->reg.uvd_ctx.wreg = &si_uvd_ctx_wreg;
 
 	adev->asic_funcs = &si_asic_funcs;
 
@@ -2382,10 +2380,10 @@ static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
 	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -2393,10 +2391,10 @@ static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
 	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
@@ -2404,10 +2402,10 @@ static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -2415,10 +2413,10 @@ static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 static void si_program_aspm(struct amdgpu_device *adev)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 74fcaa340d9b1..3e58feb2d5e4f 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -37,7 +37,6 @@ const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
 
 static void si_dma_set_ring_funcs(struct amdgpu_device *adev);
 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev);
-static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void si_dma_set_irq_funcs(struct amdgpu_device *adev);
 
 /**
@@ -473,6 +472,14 @@ static void si_dma_ring_emit_wreg(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, val);
 }
 
+static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
+	.copy_pte_num_dw = 5,
+	.copy_pte = si_dma_vm_copy_pte,
+
+	.write_pte = si_dma_vm_write_pte,
+	.set_pte_pde = si_dma_vm_set_pte_pde,
+};
+
 static int si_dma_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -481,7 +488,7 @@ static int si_dma_early_init(struct amdgpu_ip_block *ip_block)
 
 	si_dma_set_ring_funcs(adev);
 	si_dma_set_buffer_funcs(adev);
-	si_dma_set_vm_pte_funcs(adev);
+	amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs);
 	si_dma_set_irq_funcs(adev);
 
 	return 0;
@@ -830,26 +837,6 @@ static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
 
-static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
-	.copy_pte_num_dw = 5,
-	.copy_pte = si_dma_vm_copy_pte,
-
-	.write_pte = si_dma_vm_write_pte,
-	.set_pte_pde = si_dma_vm_set_pte_pde,
-};
-
-static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	adev->vm_manager.vm_pte_funcs = &si_dma_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		adev->vm_manager.vm_pte_scheds[i] =
-			&adev->sdma.instance[i].ring.sched;
-	}
-	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
-}
-
 const struct amdgpu_ip_block_version si_dma_ip_block =
 {
 	.type = AMD_IP_BLOCK_TYPE_SDMA,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 4e037a6978f09..b456e4541d9ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -245,10 +245,10 @@ static u32 soc15_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 	address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX);
 	data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA);
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(address, ((reg) & 0x1ff));
 	r = RREG32(data);
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 	return r;
 }
 
@@ -259,10 +259,10 @@ static void soc15_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX);
 	data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA);
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(address, ((reg) & 0x1ff));
 	WREG32(data, (v));
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 }
 
 static u32 soc15_didt_rreg(struct amdgpu_device *adev, u32 reg)
@@ -273,10 +273,10 @@ static u32 soc15_didt_rreg(struct amdgpu_device *adev, u32 reg)
 	address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	r = RREG32(data);
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 	return r;
 }
 
@@ -287,10 +287,10 @@ static void soc15_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	WREG32(data, (v));
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 }
 
 static u32 soc15_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
@@ -298,10 +298,10 @@ static u32 soc15_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.gc_cac.lock, flags);
 	WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
 	r = RREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA);
-	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.gc_cac.lock, flags);
 	return r;
 }
 
@@ -309,10 +309,10 @@ static void soc15_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.gc_cac.lock, flags);
 	WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
 	WREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.gc_cac.lock, flags);
 }
 
 static u32 soc15_se_cac_rreg(struct amdgpu_device *adev, u32 reg)
@@ -320,10 +320,10 @@ static u32 soc15_se_cac_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.se_cac.lock, flags);
 	WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
 	r = RREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA);
-	spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.se_cac.lock, flags);
 	return r;
 }
 
@@ -331,10 +331,10 @@ static void soc15_se_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.se_cac.lock, flags);
 	WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
 	WREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.se_cac.lock, flags);
 }
 
 static u32 soc15_get_config_memsize(struct amdgpu_device *adev)
@@ -952,7 +952,6 @@ static const struct amdgpu_asic_funcs aqua_vanjaram_asic_funcs =
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &soc15_supports_baco,
 	.query_video_codecs = &soc15_query_video_codecs,
-	.encode_ext_smn_addressing = &aqua_vanjaram_encode_ext_smn_addressing,
 	.get_reg_state = &aqua_vanjaram_get_reg_state,
 };
 
@@ -961,24 +960,22 @@ static int soc15_common_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	adev->nbio.funcs->set_reg_remap(adev);
-	adev->smc_rreg = NULL;
-	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
-	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
-	adev->pcie_rreg_ext = &amdgpu_device_indirect_rreg_ext;
-	adev->pcie_wreg_ext = &amdgpu_device_indirect_wreg_ext;
-	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
-	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
-	adev->pcie_rreg64_ext = &amdgpu_device_indirect_rreg64_ext;
-	adev->pcie_wreg64_ext = &amdgpu_device_indirect_wreg64_ext;
-	adev->uvd_ctx_rreg = &soc15_uvd_ctx_rreg;
-	adev->uvd_ctx_wreg = &soc15_uvd_ctx_wreg;
-	adev->didt_rreg = &soc15_didt_rreg;
-	adev->didt_wreg = &soc15_didt_wreg;
-	adev->gc_cac_rreg = &soc15_gc_cac_rreg;
-	adev->gc_cac_wreg = &soc15_gc_cac_wreg;
-	adev->se_cac_rreg = &soc15_se_cac_rreg;
-	adev->se_cac_wreg = &soc15_se_cac_wreg;
+	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
+	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
+	adev->reg.pcie.rreg_ext = &amdgpu_device_indirect_rreg_ext;
+	adev->reg.pcie.wreg_ext = &amdgpu_device_indirect_wreg_ext;
+	adev->reg.pcie.rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->reg.pcie.wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->reg.pcie.rreg64_ext = &amdgpu_device_indirect_rreg64_ext;
+	adev->reg.pcie.wreg64_ext = &amdgpu_device_indirect_wreg64_ext;
+	adev->reg.uvd_ctx.rreg = &soc15_uvd_ctx_rreg;
+	adev->reg.uvd_ctx.wreg = &soc15_uvd_ctx_wreg;
+	adev->reg.didt.rreg = &soc15_didt_rreg;
+	adev->reg.didt.wreg = &soc15_didt_wreg;
+	adev->reg.gc_cac.rreg = &soc15_gc_cac_rreg;
+	adev->reg.gc_cac.wreg = &soc15_gc_cac_wreg;
+	adev->reg.se_cac.rreg = &soc15_se_cac_rreg;
+	adev->reg.se_cac.wreg = &soc15_se_cac_wreg;
 
 	adev->rev_id = amdgpu_device_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
@@ -1200,6 +1197,7 @@ static int soc15_common_early_init(struct amdgpu_ip_block *ip_block)
 	case IP_VERSION(9, 4, 4):
 	case IP_VERSION(9, 5, 0):
 		adev->asic_funcs = &aqua_vanjaram_asic_funcs;
+		adev->reg.smn.get_smn_base = &amdgpu_reg_smn_v1_0_get_base;
 		adev->cg_flags =
 			AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_CGCG |
 			AMD_CG_SUPPORT_GFX_CGLS | AMD_CG_SUPPORT_SDMA_MGCG |
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.h b/drivers/gpu/drm/amd/amdgpu/soc15.h
index c8ac11a9cdef6..46a6477b677ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.h
@@ -118,7 +118,6 @@ int vega10_reg_base_init(struct amdgpu_device *adev);
 int vega20_reg_base_init(struct amdgpu_device *adev);
 int arct_reg_base_init(struct amdgpu_device *adev);
 int aldebaran_reg_base_init(struct amdgpu_device *adev);
-u64 aqua_vanjaram_encode_ext_smn_addressing(int ext_id);
 int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev);
 ssize_t aqua_vanjaram_get_reg_state(struct amdgpu_device *adev,
 				    enum amdgpu_reg_state reg_state, void *buf,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 242b24f73c174..a7b5a95ebebbe 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -195,19 +195,22 @@
 	__RREG32_SOC15_RLC__((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset, AMDGPU_REGS_RLC, ip##_HWIP, inst)
 
 /* inst equals to ext for some IPs */
-#define RREG32_SOC15_EXT(ip, inst, reg, ext) \
-	RREG32_PCIE_EXT((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) * 4 \
-			+ adev->asic_funcs->encode_ext_smn_addressing(ext)) \
-
-#define WREG32_SOC15_EXT(ip, inst, reg, ext, value) \
-	WREG32_PCIE_EXT((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) * 4 \
-			+ adev->asic_funcs->encode_ext_smn_addressing(ext), \
-			value) \
-
-#define RREG64_MCA(ext, mca_base, idx) \
-	RREG64_PCIE_EXT(adev->asic_funcs->encode_ext_smn_addressing(ext) + mca_base + (idx * 8))
-
-#define WREG64_MCA(ext, mca_base, idx, val) \
-	WREG64_PCIE_EXT(adev->asic_funcs->encode_ext_smn_addressing(ext) + mca_base + (idx * 8), val)
+#define RREG32_SOC15_EXT(ip, inst, reg, ext)                                 \
+	RREG32_PCIE_EXT((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + \
+			 reg) * 4 +                                          \
+			amdgpu_reg_get_smn_base64(adev, ip##_HWIP, inst))
+
+#define WREG32_SOC15_EXT(ip, inst, reg, ext, value)                         \
+	WREG32_PCIE_EXT(                                                    \
+		(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) * \
+				4 +                                         \
+			amdgpu_reg_get_smn_base64(adev, ip##_HWIP, inst),   \
+		value)
+
+#define RREG64_MCA(smn_base, mca_base, idx) \
+	RREG64_PCIE_EXT(smn_base + mca_base + (idx * 8))
+
+#define WREG64_MCA(smn_base, mca_base, idx, val) \
+	WREG64_PCIE_EXT(smn_base + mca_base + (idx * 8), val)
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index a0ad1f8a76f06..fbd1d97f33add 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -229,10 +229,10 @@ static u32 soc21_didt_rreg(struct amdgpu_device *adev, u32 reg)
 	address = SOC15_REG_OFFSET(GC, 0, regDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, regDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	r = RREG32(data);
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 	return r;
 }
 
@@ -243,10 +243,10 @@ static void soc21_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	address = SOC15_REG_OFFSET(GC, 0, regDIDT_IND_INDEX);
 	data = SOC15_REG_OFFSET(GC, 0, regDIDT_IND_DATA);
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(address, (reg));
 	WREG32(data, (v));
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 }
 
 static u32 soc21_get_config_memsize(struct amdgpu_device *adev)
@@ -589,21 +589,15 @@ static int soc21_common_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	adev->nbio.funcs->set_reg_remap(adev);
-	adev->smc_rreg = NULL;
-	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
-	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
-	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
-	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
-	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
-	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
-
-	/* TODO: will add them during VCN v2 implementation */
-	adev->uvd_ctx_rreg = NULL;
-	adev->uvd_ctx_wreg = NULL;
-
-	adev->didt_rreg = &soc21_didt_rreg;
-	adev->didt_wreg = &soc21_didt_wreg;
+	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
+	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
+	adev->reg.pcie.rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->reg.pcie.wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->reg.pcie.port_rreg = &amdgpu_device_pcie_port_rreg;
+	adev->reg.pcie.port_wreg = &amdgpu_device_pcie_port_wreg;
+
+	adev->reg.didt.rreg = &soc21_didt_rreg;
+	adev->reg.didt.wreg = &soc21_didt_wreg;
 
 	adev->asic_funcs = &soc21_asic_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c
index ecb6c3fcfbd15..d1adf19a51c45 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc24.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc24.c
@@ -362,18 +362,12 @@ static int soc24_common_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	adev->nbio.funcs->set_reg_remap(adev);
-	adev->smc_rreg = NULL;
-	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
-	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
-	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
-	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
-	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
-	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
-	adev->uvd_ctx_rreg = NULL;
-	adev->uvd_ctx_wreg = NULL;
-	adev->didt_rreg = NULL;
-	adev->didt_wreg = NULL;
+	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
+	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
+	adev->reg.pcie.rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->reg.pcie.wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->reg.pcie.port_rreg = &amdgpu_device_pcie_port_rreg;
+	adev->reg.pcie.port_wreg = &amdgpu_device_pcie_port_wreg;
 
 	adev->asic_funcs = &soc24_asic_funcs;
 
@@ -484,7 +478,7 @@ static int soc24_common_hw_init(struct amdgpu_ip_block *ip_block)
 	if (adev->nbio.funcs->remap_hdp_registers)
 		adev->nbio.funcs->remap_hdp_registers(adev);
 
-	if (adev->df.funcs->hw_init)
+	if (adev->df.funcs && adev->df.funcs->hw_init)
 		adev->df.funcs->hw_init(adev);
 
 	/* enable the doorbell aperture */
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
index 59ab952d5cce4..709b1669b07bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
@@ -41,6 +41,44 @@
 #define NORMALIZE_XCC_REG_OFFSET(offset) \
 	(offset & 0xFFFF)
 
+#define MID1_REG_RANGE_0_LOW  0x40000
+#define MID1_REG_RANGE_0_HIGH 0x80000
+#define NORMALIZE_MID_REG_OFFSET(offset) \
+		(offset & 0x3FFFF)
+
+static const struct amdgpu_video_codecs vcn_5_0_2_video_codecs_encode_vcn0 = {
+	.codec_count = 0,
+	.codec_array = NULL,
+};
+
+static const struct amdgpu_video_codec_info vcn_5_0_2_video_codecs_decode_array_vcn0[] = {
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)},
+};
+
+static const struct amdgpu_video_codecs vcn_5_0_2_video_codecs_decode_vcn0 = {
+	.codec_count = ARRAY_SIZE(vcn_5_0_2_video_codecs_decode_array_vcn0),
+	.codec_array = vcn_5_0_2_video_codecs_decode_array_vcn0,
+};
+
+static int soc_v1_0_query_video_codecs(struct amdgpu_device *adev, bool encode,
+					const struct amdgpu_video_codecs **codecs)
+{
+	switch (amdgpu_ip_version(adev, UVD_HWIP, 0)) {
+	case IP_VERSION(5, 0, 2):
+		if (encode)
+			*codecs = &vcn_5_0_2_video_codecs_encode_vcn0;
+		else
+			*codecs = &vcn_5_0_2_video_codecs_decode_vcn0;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
 /* Initialized doorbells for amdgpu including multimedia
  * KFD can use all the rest in 2M doorbell bar */
 static void soc_v1_0_doorbell_index_init(struct amdgpu_device *adev)
@@ -57,7 +95,7 @@ static void soc_v1_0_doorbell_index_init(struct amdgpu_device *adev)
 	adev->doorbell_index.userqueue_end = AMDGPU_SOC_V1_0_DOORBELL_USERQUEUE_END;
 	adev->doorbell_index.xcc_doorbell_range = AMDGPU_SOC_V1_0_DOORBELL_XCC_RANGE;
 
-	adev->doorbell_index.sdma_doorbell_range = 20;
+	adev->doorbell_index.sdma_doorbell_range = 14;
 	for (i = 0; i < adev->sdma.num_instances; i++)
 		adev->doorbell_index.sdma_engine[i] =
 			AMDGPU_SOC_V1_0_DOORBELL_sDMA_ENGINE_START +
@@ -214,23 +252,35 @@ static bool soc_v1_0_need_full_reset(struct amdgpu_device *adev)
 
 static bool soc_v1_0_need_reset_on_init(struct amdgpu_device *adev)
 {
-	u32 sol_reg;
 
-	if (adev->flags & AMD_IS_APU)
-		return false;
+	return false;
+}
 
-	/* Check sOS sign of life register to confirm sys driver and sOS
-	 * are already been loaded.
-	 */
-	sol_reg = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81);
-	if (sol_reg)
-		return true;
+static enum amd_reset_method
+soc_v1_0_asic_reset_method(struct amdgpu_device *adev)
+{
+	if ((adev->gmc.xgmi.supported && adev->gmc.xgmi.connected_to_cpu) ||
+	    (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(15, 0, 8))) {
+		if (amdgpu_reset_method != -1)
+			dev_warn_once(adev->dev, "Reset override isn't supported, using Mode2 instead.\n");
 
-	return false;
+		return AMD_RESET_METHOD_MODE2;
+	}
+
+	return amdgpu_reset_method;
 }
 
 static int soc_v1_0_asic_reset(struct amdgpu_device *adev)
 {
+	switch (soc_v1_0_asic_reset_method(adev)) {
+	case AMD_RESET_METHOD_MODE2:
+		dev_info(adev->dev, "MODE2 reset\n");
+		return amdgpu_dpm_mode2_reset(adev);
+	default:
+		dev_info(adev->dev, "Invalid reset method Not supported\n");
+		return -EOPNOTSUPP;
+	}
+
 	return 0;
 }
 
@@ -244,28 +294,24 @@ static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = {
 	.need_reset_on_init = &soc_v1_0_need_reset_on_init,
 	.encode_ext_smn_addressing = &soc_v1_0_encode_ext_smn_addressing,
 	.reset = soc_v1_0_asic_reset,
+	.reset_method = &soc_v1_0_asic_reset_method,
+	.query_video_codecs = &soc_v1_0_query_video_codecs,
 };
 
 static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->smc_rreg = NULL;
-	adev->smc_wreg = NULL;
-	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
-	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
-	adev->pcie_rreg_ext = &amdgpu_device_indirect_rreg_ext;
-	adev->pcie_wreg_ext = &amdgpu_device_indirect_wreg_ext;
-	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
-	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
-	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
-	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
-	adev->pcie_rreg64_ext = &amdgpu_device_indirect_rreg64_ext;
-	adev->pcie_wreg64_ext = &amdgpu_device_indirect_wreg64_ext;
-	adev->uvd_ctx_rreg = NULL;
-	adev->uvd_ctx_wreg = NULL;
-	adev->didt_rreg = NULL;
-	adev->didt_wreg = NULL;
+	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
+	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
+	adev->reg.pcie.rreg_ext = &amdgpu_device_indirect_rreg_ext;
+	adev->reg.pcie.wreg_ext = &amdgpu_device_indirect_wreg_ext;
+	adev->reg.pcie.rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->reg.pcie.wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->reg.pcie.port_rreg = &amdgpu_device_pcie_port_rreg;
+	adev->reg.pcie.port_wreg = &amdgpu_device_pcie_port_wreg;
+	adev->reg.pcie.rreg64_ext = &amdgpu_device_indirect_rreg64_ext;
+	adev->reg.pcie.wreg64_ext = &amdgpu_device_indirect_wreg64_ext;
 
 	adev->asic_funcs = &soc_v1_0_asic_funcs;
 
@@ -274,8 +320,9 @@ static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
 
 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 	case IP_VERSION(12, 1, 0):
-		adev->cg_flags = 0;
-		adev->pg_flags = 0;
+		adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS;
+		adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG;
 		adev->external_rev_id = adev->rev_id + 0x50;
 		break;
 	default:
@@ -815,7 +862,7 @@ int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
 {
 	int ret, i;
 	int xcc_inst_per_aid = 4;
-	uint16_t xcc_mask;
+	uint16_t xcc_mask, sdma_mask = 0;
 
 	xcc_mask = adev->gfx.xcc_mask;
 	adev->aid_mask = 0;
@@ -825,10 +872,12 @@ int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
 	}
 
 	adev->sdma.num_inst_per_xcc = 2;
-	adev->sdma.num_instances =
-		NUM_XCC(adev->gfx.xcc_mask) * adev->sdma.num_inst_per_xcc;
-	adev->sdma.sdma_mask =
-		GENMASK(adev->sdma.num_instances - 1, 0);
+	for_each_inst(i, adev->gfx.xcc_mask)
+		sdma_mask |=
+			GENMASK(adev->sdma.num_inst_per_xcc - 1, 0) <<
+			(i * adev->sdma.num_inst_per_xcc);
+	adev->sdma.sdma_mask = sdma_mask;
+	adev->sdma.num_instances = NUM_XCC(adev->sdma.sdma_mask);
 
 	ret = soc_v1_0_xcp_mgr_init(adev);
 	if (ret)
@@ -860,3 +909,31 @@ uint32_t soc_v1_0_normalize_xcc_reg_offset(uint32_t reg)
 	else
 		return reg;
 }
+
+bool soc_v1_0_mid1_reg_range(uint32_t reg)
+{
+	uint32_t normalized_reg = soc_v1_0_normalize_xcc_reg_offset(reg);
+
+	if (soc_v1_0_normalize_xcc_reg_range(normalized_reg))
+		return false;
+
+	if ((reg >= MID1_REG_RANGE_0_LOW) && (reg < MID1_REG_RANGE_0_HIGH))
+		return true;
+	else
+		return false;
+}
+
+uint32_t soc_v1_0_normalize_reg_offset(uint32_t reg)
+{
+	uint32_t normalized_reg = soc_v1_0_normalize_xcc_reg_offset(reg);
+
+	if (soc_v1_0_normalize_xcc_reg_range(normalized_reg))
+		return soc_v1_0_normalize_xcc_reg_offset(reg);
+
+	/* check if the reg offset is inside MID1. */
+	if (soc_v1_0_mid1_reg_range(reg))
+		return NORMALIZE_MID_REG_OFFSET(reg);
+
+	return reg;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
index 146996101aa07..16c220fcc4e92 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
@@ -31,7 +31,9 @@ void soc_v1_0_grbm_select(struct amdgpu_device *adev,
 			  int xcc_id);
 int soc_v1_0_init_soc_config(struct amdgpu_device *adev);
 bool soc_v1_0_normalize_xcc_reg_range(uint32_t reg);
+bool soc_v1_0_mid1_reg_range(uint32_t reg);
 uint32_t soc_v1_0_normalize_xcc_reg_offset(uint32_t reg);
+uint32_t soc_v1_0_normalize_reg_offset(uint32_t reg);
 u64 soc_v1_0_encode_ext_smn_addressing(int ext_id);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
index 1f80045775f5d..db505ab32fa0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
@@ -183,50 +183,97 @@ static void umc_v12_0_get_retire_flip_bits(struct amdgpu_device *adev)
 	if (adev->gmc.gmc_funcs->query_mem_partition_mode)
 		nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
 
-	/* default setting */
-	flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_C2_BIT;
-	flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C3_BIT;
-	flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_C4_BIT;
-	flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R13_BIT;
-	flip_bits->flip_row_bit = 13;
-	flip_bits->bit_num = 4;
-	flip_bits->r13_in_pa = UMC_V12_0_PA_R13_BIT;
-
-	if (nps == AMDGPU_NPS2_PARTITION_MODE) {
+	if (adev->gmc.num_umc == 16) {
+		/* default setting */
+		flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_C2_BIT;
+		flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C3_BIT;
+		flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_C4_BIT;
+		flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R13_BIT;
+		flip_bits->flip_row_bit = 13;
+		flip_bits->bit_num = 4;
+		flip_bits->r13_in_pa = UMC_V12_0_PA_R13_BIT;
+
+		if (nps == AMDGPU_NPS2_PARTITION_MODE) {
+			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH5_BIT;
+			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C2_BIT;
+			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B1_BIT;
+			flip_bits->r13_in_pa = UMC_V12_0_PA_R12_BIT;
+		} else if (nps == AMDGPU_NPS4_PARTITION_MODE) {
+			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH4_BIT;
+			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_CH5_BIT;
+			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B0_BIT;
+			flip_bits->r13_in_pa = UMC_V12_0_PA_R11_BIT;
+		}
+
+		switch (vram_type) {
+		case AMDGPU_VRAM_TYPE_HBM:
+			/* other nps modes are taken as nps1 */
+			if (nps == AMDGPU_NPS2_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
+			else if (nps == AMDGPU_NPS4_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
+
+			break;
+		case AMDGPU_VRAM_TYPE_HBM3E:
+			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
+			flip_bits->flip_row_bit = 12;
+
+			if (nps == AMDGPU_NPS2_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
+			else if (nps == AMDGPU_NPS4_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R10_BIT;
+
+			break;
+		default:
+			dev_warn(adev->dev,
+				"Unknown HBM type, set RAS retire flip bits to the value in NPS1 mode.\n");
+			break;
+		}
+	} else if (adev->gmc.num_umc == 8) {
+		/* default setting */
 		flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH5_BIT;
 		flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C2_BIT;
 		flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B1_BIT;
+		flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
+		flip_bits->flip_row_bit = 12;
+		flip_bits->bit_num = 4;
 		flip_bits->r13_in_pa = UMC_V12_0_PA_R12_BIT;
-	} else if (nps == AMDGPU_NPS4_PARTITION_MODE) {
-		flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH4_BIT;
-		flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_CH5_BIT;
-		flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B0_BIT;
-		flip_bits->r13_in_pa = UMC_V12_0_PA_R11_BIT;
-	}
 
-	switch (vram_type) {
-	case AMDGPU_VRAM_TYPE_HBM:
-		/* other nps modes are taken as nps1 */
-		if (nps == AMDGPU_NPS2_PARTITION_MODE)
+		if (nps == AMDGPU_NPS2_PARTITION_MODE) {
+			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH4_BIT;
+			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_CH5_BIT;
+			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B0_BIT;
+			flip_bits->r13_in_pa = UMC_V12_0_PA_R11_BIT;
+		}
+
+		switch (vram_type) {
+		case AMDGPU_VRAM_TYPE_HBM:
 			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
-		else if (nps == AMDGPU_NPS4_PARTITION_MODE)
-			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
 
-		break;
-	case AMDGPU_VRAM_TYPE_HBM3E:
-		flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
-		flip_bits->flip_row_bit = 12;
+			/* other nps modes are taken as nps1 */
+			if (nps == AMDGPU_NPS2_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
 
-		if (nps == AMDGPU_NPS2_PARTITION_MODE)
+			break;
+		case AMDGPU_VRAM_TYPE_HBM3E:
 			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-		else if (nps == AMDGPU_NPS4_PARTITION_MODE)
-			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R10_BIT;
+			flip_bits->flip_row_bit = 12;
 
-		break;
-	default:
+			if (nps == AMDGPU_NPS2_PARTITION_MODE)
+				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R10_BIT;
+
+			break;
+		default:
+			dev_warn(adev->dev,
+				"Unknown HBM type, set RAS retire flip bits to the value in NPS1 mode.\n");
+			break;
+		}
+	} else {
 		dev_warn(adev->dev,
-			"Unknown HBM type, set RAS retire flip bits to the value in NPS1 mode.\n");
-		break;
+			"Unsupported UMC number(%d), failed to set RAS flip bits.\n",
+			adev->gmc.num_umc);
+
+		return;
 	}
 
 	adev->umc.retire_unit = 0x1 << flip_bits->bit_num;
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
index 9ae4246185560..5b7b46d242c6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
@@ -47,11 +47,6 @@
 #define VCE_V1_0_DATA_SIZE	(7808 * (AMDGPU_MAX_VCE_HANDLES + 1))
 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
-#define VCE_V1_0_GART_PAGE_START \
-	(AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS)
-#define VCE_V1_0_GART_ADDR_START \
-	(VCE_V1_0_GART_PAGE_START * AMDGPU_GPU_PAGE_SIZE)
-
 static void vce_v1_0_set_ring_funcs(struct amdgpu_device *adev);
 static void vce_v1_0_set_irq_funcs(struct amdgpu_device *adev);
 
@@ -535,27 +530,29 @@ static int vce_v1_0_early_init(struct amdgpu_ip_block *ip_block)
  */
 static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev)
 {
-	u64 gpu_addr = amdgpu_bo_gpu_offset(adev->vce.vcpu_bo);
 	u64 bo_size = amdgpu_bo_size(adev->vce.vcpu_bo);
 	u64 max_vcpu_bo_addr = 0xffffffff - bo_size;
 	u64 num_pages = ALIGN(bo_size, AMDGPU_GPU_PAGE_SIZE) / AMDGPU_GPU_PAGE_SIZE;
 	u64 pa = amdgpu_gmc_vram_pa(adev, adev->vce.vcpu_bo);
 	u64 flags = AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | AMDGPU_PTE_VALID;
+	u64 vce_gart_start_offs;
+	int r;
 
-	/*
-	 * Check if the VCPU BO already has a 32-bit address.
-	 * Eg. if MC is configured to put VRAM in the low address range.
-	 */
-	if (gpu_addr <= max_vcpu_bo_addr)
-		return 0;
+	r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr,
+					 &adev->vce.gart_node, num_pages,
+					 DRM_MM_INSERT_LOW);
+	if (r)
+		return r;
+
+	vce_gart_start_offs = amdgpu_gtt_node_to_byte_offset(&adev->vce.gart_node);
 
 	/* Check if we can map the VCPU BO in GART to a 32-bit address. */
-	if (adev->gmc.gart_start + VCE_V1_0_GART_ADDR_START > max_vcpu_bo_addr)
+	if (adev->gmc.gart_start + vce_gart_start_offs > max_vcpu_bo_addr)
 		return -EINVAL;
 
-	amdgpu_gart_map_vram_range(adev, pa, VCE_V1_0_GART_PAGE_START,
+	amdgpu_gart_map_vram_range(adev, pa, adev->vce.gart_node.start,
 				   num_pages, flags, adev->gart.ptr);
-	adev->vce.gpu_addr = adev->gmc.gart_start + VCE_V1_0_GART_ADDR_START;
+	adev->vce.gpu_addr = adev->gmc.gart_start + vce_gart_start_offs;
 	if (adev->vce.gpu_addr > max_vcpu_bo_addr)
 		return -EINVAL;
 
@@ -610,7 +607,11 @@ static int vce_v1_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	return amdgpu_vce_sw_fini(adev);
+	r = amdgpu_vce_sw_fini(adev);
+
+	amdgpu_gtt_mgr_free_entries(&adev->mman.gtt_mgr, &adev->vce.gart_node);
+
+	return r;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
new file mode 100644
index 0000000000000..c3d3cc0230580
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
@@ -0,0 +1,1219 @@
+/*
+ * Copyright 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_vcn.h"
+#include "amdgpu_pm.h"
+#include "soc15.h"
+#include "soc15d.h"
+#include "soc15_hw_ip.h"
+#include "vcn_v2_0.h"
+#include "vcn_v4_0_3.h"
+
+#include "vcn/vcn_5_0_0_offset.h"
+#include "vcn/vcn_5_0_0_sh_mask.h"
+#include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
+#include "vcn_v5_0_0.h"
+#include "vcn_v5_0_1.h"
+#include "vcn_v5_0_2.h"
+
+#include <drm/drm_drv.h>
+
+static void vcn_v5_0_2_set_unified_ring_funcs(struct amdgpu_device *adev);
+static void vcn_v5_0_2_set_irq_funcs(struct amdgpu_device *adev);
+static int vcn_v5_0_2_set_pg_state(struct amdgpu_vcn_inst *vinst,
+				   enum amd_powergating_state state);
+static void vcn_v5_0_2_unified_ring_set_wptr(struct amdgpu_ring *ring);
+
+/**
+ * vcn_v5_0_2_early_init - set function pointers and load microcode
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Set ring and irq function pointers
+ * Load microcode from filesystem
+ */
+static int vcn_v5_0_2_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, r;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
+		/* re-use enc ring as unified ring */
+		adev->vcn.inst[i].num_enc_rings = 1;
+
+	vcn_v5_0_2_set_unified_ring_funcs(adev);
+	vcn_v5_0_2_set_irq_funcs(adev);
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		adev->vcn.inst[i].set_pg_state = vcn_v5_0_2_set_pg_state;
+
+		r = amdgpu_vcn_early_init(adev, i);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static void vcn_v5_0_2_fw_shared_init(struct amdgpu_device *adev, int inst_idx)
+{
+	struct amdgpu_vcn5_fw_shared *fw_shared;
+
+	fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
+
+	if (fw_shared->sq.is_enabled)
+		return;
+	fw_shared->present_flag_0 =
+		cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
+	fw_shared->sq.is_enabled = 1;
+
+	if (amdgpu_vcnfw_log)
+		amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]);
+}
+
+/**
+ * vcn_v5_0_2_sw_init - sw init for VCN block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Load firmware and sw initialization
+ */
+static int vcn_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ring *ring;
+	int i, r, vcn_inst;
+
+	/* VCN UNIFIED TRAP */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_VCN,
+		VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq);
+	if (r)
+		return r;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		vcn_inst = GET_INST(VCN, i);
+
+		r = amdgpu_vcn_sw_init(adev, i);
+		if (r)
+			return r;
+
+		amdgpu_vcn_setup_ucode(adev, i);
+
+		r = amdgpu_vcn_resume(adev, i);
+		if (r)
+			return r;
+
+		ring = &adev->vcn.inst[i].ring_enc[0];
+		ring->use_doorbell = true;
+
+		ring->doorbell_index =
+			(adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 32 * vcn_inst;
+
+		ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id);
+		sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id);
+
+		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
+					AMDGPU_RING_PRIO_DEFAULT, &adev->vcn.inst[i].sched_score);
+		if (r)
+			return r;
+
+		vcn_v5_0_2_fw_shared_init(adev, i);
+	}
+
+	/* TODO: Add queue reset mask when FW fully supports it */
+	adev->vcn.supported_reset =
+		amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
+
+	return amdgpu_vcn_sysfs_reset_mask_init(adev);
+}
+
+/**
+ * vcn_v5_0_2_sw_fini - sw fini for VCN block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * VCN suspend and free up sw allocation
+ */
+static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, r, idx;
+
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+			struct amdgpu_vcn5_fw_shared *fw_shared;
+
+			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
+			fw_shared->present_flag_0 = 0;
+			fw_shared->sq.is_enabled = 0;
+		}
+
+		drm_dev_exit(idx);
+	}
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		r = amdgpu_vcn_suspend(adev, i);
+		if (r)
+			return r;
+	}
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++)
+		amdgpu_vcn_sw_fini(adev, i);
+
+	amdgpu_vcn_sysfs_reset_mask_fini(adev);
+
+	kfree(adev->vcn.ip_dump); //TODO check
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_hw_init - start and test VCN block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Initialize the hardware, boot up the VCPU and do some testing
+ */
+static int vcn_v5_0_2_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ring *ring;
+	int i, r, vcn_inst;
+	uint32_t tmp;
+
+	if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x200)
+		adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED);
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		vcn_inst = GET_INST(VCN, i);
+		ring = &adev->vcn.inst[i].ring_enc[0];
+
+		/* Remove Video Tiles antihang mechanism */
+		tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS);
+		tmp &= (~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+		WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp);
+
+		if (ring->use_doorbell)
+			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+				((adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+				 11 * vcn_inst),
+				adev->vcn.inst[i].aid_id);
+
+		/* Re-init fw_shared, if required */
+		vcn_v5_0_2_fw_shared_init(adev, i);
+
+		r = amdgpu_ring_test_helper(ring);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_hw_fini - stop the hardware block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Stop the VCN block, mark ring as not ready any more
+ */
+static int vcn_v5_0_2_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
+
+		cancel_delayed_work_sync(&adev->vcn.inst[i].idle_work);
+		if (vinst->cur_state != AMD_PG_STATE_GATE)
+			vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
+	}
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_suspend - suspend VCN block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * HW fini and suspend VCN block
+ */
+static int vcn_v5_0_2_suspend(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r, i;
+
+	r = vcn_v5_0_2_hw_fini(ip_block);
+	if (r)
+		return r;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		r = amdgpu_vcn_suspend(ip_block->adev, i);
+		if (r)
+			return r;
+	}
+
+	return r;
+}
+
+/**
+ * vcn_v5_0_2_resume - resume VCN block
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Resume firmware and hw init VCN block
+ */
+static int vcn_v5_0_2_resume(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r, i;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
+
+		if (amdgpu_in_reset(adev))
+			vinst->cur_state = AMD_PG_STATE_GATE;
+
+		r = amdgpu_vcn_resume(ip_block->adev, i);
+		if (r)
+			return r;
+	}
+
+	r = vcn_v5_0_2_hw_init(ip_block);
+
+	return r;
+}
+
+/**
+ * vcn_v5_0_2_mc_resume - memory controller programming
+ *
+ * @vinst: VCN instance
+ *
+ * Let the VCN memory controller know it's offsets
+ */
+static void vcn_v5_0_2_mc_resume(struct amdgpu_vcn_inst *vinst)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int inst = vinst->inst;
+	uint32_t offset, size, vcn_inst;
+	const struct common_firmware_header *hdr;
+
+	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst].fw->data;
+	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
+
+	vcn_inst = GET_INST(VCN, inst);
+	/* cache window 0: fw */
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
+		WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
+		WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0);
+		offset = 0;
+	} else {
+		WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
+		WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
+		offset = size;
+		WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0,
+				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+	}
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size);
+
+	/* cache window 1: stack */
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
+		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
+		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
+
+	/* cache window 2: context */
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
+		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
+		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
+
+	/* non-cache window */
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
+		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
+		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0,
+		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)));
+}
+
+/**
+ * vcn_v5_0_2_mc_resume_dpg_mode - memory controller programming for dpg mode
+ *
+ * @vinst: VCN instance
+ * @indirect: indirectly write sram
+ *
+ * Let the VCN memory controller know it's offsets with dpg mode
+ */
+static void vcn_v5_0_2_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
+					  bool indirect)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int inst_idx = vinst->inst;
+	uint32_t offset, size;
+	const struct common_firmware_header *hdr;
+
+	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data;
+	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
+
+	/* cache window 0: fw */
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (!indirect) {
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN +
+				 inst_idx].tmr_mc_addr_lo), 0, indirect);
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN +
+				 inst_idx].tmr_mc_addr_hi), 0, indirect);
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
+		} else {
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
+			WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+				VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
+		}
+		offset = 0;
+	} else {
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
+		offset = size;
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_OFFSET0),
+			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
+	}
+
+	if (!indirect)
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
+	else
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
+
+	/* cache window 1: stack */
+	if (!indirect) {
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
+			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
+			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
+	} else {
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
+		WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
+	}
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+			VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
+
+	/* cache window 2: context */
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
+		lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset +
+			AMDGPU_VCN_STACK_SIZE), 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
+		upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset +
+			AMDGPU_VCN_STACK_SIZE), 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
+
+	/* non-cache window */
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
+		lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
+		upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_NONCACHE_SIZE0),
+		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)), 0, indirect);
+
+	/* VCN global tiling registers */
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
+}
+
+/**
+ * vcn_v5_0_2_disable_clock_gating - disable VCN clock gating
+ *
+ * @vinst: VCN instance
+ *
+ * Disable clock gating for VCN block
+ */
+static void vcn_v5_0_2_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
+{
+}
+
+/**
+ * vcn_v5_0_2_enable_clock_gating - enable VCN clock gating
+ *
+ * @vinst: VCN instance
+ *
+ * Enable clock gating for VCN block
+ */
+static void vcn_v5_0_2_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
+{
+}
+
+/**
+ * vcn_v5_0_2_pause_dpg_mode - VCN pause with dpg mode
+ *
+ * @vinst: VCN instance
+ * @new_state: pause state
+ *
+ * Pause dpg mode for VCN block
+ */
+static int vcn_v5_0_2_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
+				struct dpg_pause_state *new_state)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	uint32_t reg_data = 0;
+	int vcn_inst;
+
+	vcn_inst = GET_INST(VCN, vinst->inst);
+
+	/* pause/unpause if state is changed */
+	if (vinst->pause_state.fw_based != new_state->fw_based) {
+		DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d %s\n",
+			vinst->pause_state.fw_based, new_state->fw_based,
+			new_state->fw_based ? "VCN_DPG_STATE__PAUSE" : "VCN_DPG_STATE__UNPAUSE");
+		reg_data = RREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE) &
+				(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+
+		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
+			/* pause DPG */
+			reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+			WREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE, reg_data);
+
+			/* wait for ACK */
+			SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_DPG_PAUSE,
+					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
+					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+		} else {
+			/* unpause DPG, no need to wait */
+			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+			WREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE, reg_data);
+		}
+		vinst->pause_state.fw_based = new_state->fw_based;
+	}
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_start_dpg_mode - VCN start with dpg mode
+ *
+ * @vinst: VCN instance
+ * @indirect: indirectly write sram
+ *
+ * Start VCN block with dpg mode
+ */
+static int vcn_v5_0_2_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
+				     bool indirect)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int inst_idx = vinst->inst;
+	struct amdgpu_vcn5_fw_shared *fw_shared =
+		adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
+	struct amdgpu_ring *ring;
+	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__PAUSE};
+	int vcn_inst, ret;
+	uint32_t tmp;
+
+	vcn_inst = GET_INST(VCN, inst_idx);
+
+	/* disable register anti-hang mechanism */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1,
+		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+
+	/* enable dynamic power gating mode */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS);
+	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp);
+
+	if (indirect) {
+		adev->vcn.inst[inst_idx].dpg_sram_curr_addr =
+			(uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
+		/* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */
+		WREG32_SOC24_DPG_MODE(inst_idx, 0xDEADBEEF,
+				adev->vcn.inst[inst_idx].aid_id, 0, true);
+	}
+
+	/* enable VCPU clock */
+	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
+	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect);
+
+	/* disable master interrupt */
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect);
+
+	/* setup regUVD_LMI_CTRL */
+	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+		UVD_LMI_CTRL__REQ_MODE_MASK |
+		UVD_LMI_CTRL__CRC_RESET_MASK |
+		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
+		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
+		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
+		0x00100000L);
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect);
+
+	vcn_v5_0_2_mc_resume_dpg_mode(vinst, indirect);
+
+	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
+	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect);
+
+	/* enable LMI MC and UMC channels */
+	tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect);
+
+	/* enable master interrupt */
+	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
+		VCN, 0, regUVD_MASTINT_EN),
+		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
+
+	if (indirect) {
+		ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM);
+		if (ret) {
+			dev_err(adev->dev, "vcn sram load failed %d\n", ret);
+			return ret;
+		}
+	}
+
+	/* resetting ring, fw should not check RB ring */
+	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
+
+	/* Pause dpg */
+	vcn_v5_0_2_pause_dpg_mode(vinst, &state);
+
+	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
+
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, lower_32_bits(ring->gpu_addr));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, ring->ring_size / sizeof(uint32_t));
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
+	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0);
+
+	ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
+	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
+	/* resetting done, fw can check RB ring */
+	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
+
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL,
+		ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
+		VCN_RB1_DB_CTRL__EN_MASK);
+	/* Read DB_CTRL to flush the write DB_CTRL command. */
+	RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL);
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_start - VCN start
+ *
+ * @vinst: VCN instance
+ *
+ * Start VCN block
+ */
+static int vcn_v5_0_2_start(struct amdgpu_vcn_inst *vinst)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int i = vinst->inst;
+	struct amdgpu_vcn5_fw_shared *fw_shared;
+	struct amdgpu_ring *ring;
+	uint32_t tmp;
+	int j, k, r, vcn_inst;
+
+	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
+
+	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
+		return vcn_v5_0_2_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
+
+	vcn_inst = GET_INST(VCN, i);
+
+	/* set VCN status busy */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp);
+
+	/* enable VCPU clock */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
+		 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
+
+	/* disable master interrupt */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0,
+		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
+
+	/* enable LMI MC and UMC channels */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0,
+		 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
+	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
+	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
+
+	/* setup regUVD_LMI_CTRL */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, tmp |
+		     UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+		     UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
+		     UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+		     UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
+
+	vcn_v5_0_2_mc_resume(vinst);
+
+	/* VCN global tiling registers */
+	WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG,
+		     adev->gfx.config.gb_addr_config);
+
+	/* unblock VCPU register access */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0,
+		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
+
+	/* release VCPU reset to boot */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
+		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
+
+	for (j = 0; j < 10; ++j) {
+		uint32_t status;
+
+		for (k = 0; k < 100; ++k) {
+			status = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS);
+			if (status & 2)
+				break;
+			mdelay(1000);
+			if (amdgpu_emu_mode == 1)
+				msleep(520);
+		}
+
+		if (amdgpu_emu_mode == 1) {
+			r = -1;
+			if (status & 2) {
+				r = 0;
+				break;
+			}
+		} else {
+			r = 0;
+			if (status & 2)
+				break;
+
+			dev_err(adev->dev,
+				"VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
+			WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
+				 UVD_VCPU_CNTL__BLK_RST_MASK,
+				 ~UVD_VCPU_CNTL__BLK_RST_MASK);
+			mdelay(10);
+			WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
+				 ~UVD_VCPU_CNTL__BLK_RST_MASK);
+
+			mdelay(10);
+			r = -1;
+		}
+	}
+
+	if (r) {
+		dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
+		return r;
+	}
+
+	/* enable master interrupt */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN),
+		 UVD_MASTINT_EN__VCPU_EN_MASK,
+		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
+
+	/* clear the busy bit of VCN_STATUS */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0,
+		 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
+
+	ring = &adev->vcn.inst[i].ring_enc[0];
+
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL,
+		     ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
+		     VCN_RB1_DB_CTRL__EN_MASK);
+
+	/* Read DB_CTRL to flush the write DB_CTRL command. */
+	RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL);
+
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, ring->gpu_addr);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, ring->ring_size / 4);
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
+	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
+	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0);
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR);
+	WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, tmp);
+	ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
+
+	tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
+	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
+	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_stop_dpg_mode - VCN stop with dpg mode
+ *
+ * @vinst: VCN instance
+ *
+ * Stop VCN block with dpg mode
+ */
+static void vcn_v5_0_2_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int inst_idx = vinst->inst;
+	uint32_t tmp;
+	int vcn_inst;
+	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
+
+	vcn_inst = GET_INST(VCN, inst_idx);
+
+	/* Unpause dpg */
+	vcn_v5_0_2_pause_dpg_mode(vinst, &state);
+
+	/* Wait for power status to be 1 */
+	SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1,
+		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+
+	/* wait for read ptr to be equal to write ptr */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
+	SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
+
+	/* disable dynamic power gating mode */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0,
+		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
+}
+
+/**
+ * vcn_v5_0_2_stop - VCN stop
+ *
+ * @vinst: VCN instance
+ *
+ * Stop VCN block
+ */
+static int vcn_v5_0_2_stop(struct amdgpu_vcn_inst *vinst)
+{
+	struct amdgpu_device *adev = vinst->adev;
+	int i = vinst->inst;
+	struct amdgpu_vcn5_fw_shared *fw_shared;
+	uint32_t tmp;
+	int r = 0, vcn_inst;
+
+	vcn_inst = GET_INST(VCN, i);
+
+	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
+	fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
+
+	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
+		vcn_v5_0_2_stop_dpg_mode(vinst);
+		return 0;
+	}
+
+	/* wait for vcn idle */
+	r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
+	if (r)
+		return r;
+
+	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
+		UVD_LMI_STATUS__READ_CLEAN_MASK |
+		UVD_LMI_STATUS__WRITE_CLEAN_MASK |
+		UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
+	r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, tmp);
+	if (r)
+		return r;
+
+	/* disable LMI UMC channel */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2);
+	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp);
+	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
+		UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
+	r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, tmp);
+	if (r)
+		return r;
+
+	/* block VCPU register access */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL),
+		 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
+		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
+
+	/* reset VCPU */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
+		 UVD_VCPU_CNTL__BLK_RST_MASK,
+		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
+
+	/* disable VCPU clock */
+	WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
+		 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
+
+	/* apply soft reset */
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
+	tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
+	tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
+	tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
+	WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
+
+	/* clear status */
+	WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0);
+
+	return 0;
+}
+
+/**
+ * vcn_v5_0_2_unified_ring_get_rptr - get unified read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware unified read pointer
+ */
+static uint64_t vcn_v5_0_2_unified_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
+		DRM_ERROR("wrong ring id is identified in %s", __func__);
+
+	return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR);
+}
+
+/**
+ * vcn_v5_0_2_unified_ring_get_wptr - get unified write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware unified write pointer
+ */
+static uint64_t vcn_v5_0_2_unified_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
+		DRM_ERROR("wrong ring id is identified in %s", __func__);
+
+	if (ring->use_doorbell)
+		return *ring->wptr_cpu_addr;
+	else
+		return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR);
+}
+
+/**
+ * vcn_v5_0_2_unified_ring_set_wptr - set enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the enc write pointer to the hardware
+ */
+static void vcn_v5_0_2_unified_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
+		DRM_ERROR("wrong ring id is identified in %s", __func__);
+
+	if (ring->use_doorbell) {
+		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
+		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+	} else {
+		WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR,
+				lower_32_bits(ring->wptr));
+	}
+}
+
+static const struct amdgpu_ring_funcs vcn_v5_0_2_unified_ring_vm_funcs = {
+	.type = AMDGPU_RING_TYPE_VCN_ENC,
+	.align_mask = 0x3f,
+	.nop = VCN_ENC_CMD_NO_OP,
+	.get_rptr = vcn_v5_0_2_unified_ring_get_rptr,
+	.get_wptr = vcn_v5_0_2_unified_ring_get_wptr,
+	.set_wptr = vcn_v5_0_2_unified_ring_set_wptr,
+	.emit_frame_size = SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+			   SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
+			   4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
+			   5 +
+			   5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
+			   1, /* vcn_v2_0_enc_ring_insert_end */
+	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
+	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
+	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
+	.emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush,
+	.emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush,
+	.test_ring = amdgpu_vcn_enc_ring_test_ring,
+	.test_ib = amdgpu_vcn_unified_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_end = vcn_v2_0_enc_ring_insert_end,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.begin_use = amdgpu_vcn_ring_begin_use,
+	.end_use = amdgpu_vcn_ring_end_use,
+	.emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg,
+	.emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+/**
+ * vcn_v5_0_2_set_unified_ring_funcs - set unified ring functions
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set unified ring functions
+ */
+static void vcn_v5_0_2_set_unified_ring_funcs(struct amdgpu_device *adev)
+{
+	int i, vcn_inst;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v5_0_2_unified_ring_vm_funcs;
+		adev->vcn.inst[i].ring_enc[0].me = i;
+		vcn_inst = GET_INST(VCN, i);
+		adev->vcn.inst[i].aid_id = vcn_inst / adev->vcn.num_inst_per_aid;
+	}
+}
+
+/**
+ * vcn_v5_0_2_is_idle - check VCN block is idle
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block structure
+ *
+ * Check whether VCN block is idle
+ */
+static bool vcn_v5_0_2_is_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, ret = 1;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
+		ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == UVD_STATUS__IDLE);
+
+	return ret;
+}
+
+/**
+ * vcn_v5_0_2_wait_for_idle - wait for VCN block idle
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Wait for VCN block idle
+ */
+static int vcn_v5_0_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, ret = 0;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, UVD_STATUS__IDLE,
+			UVD_STATUS__IDLE);
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+/**
+ * vcn_v5_0_2_set_clockgating_state - set VCN block clockgating state
+ *
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ * @state: clock gating state
+ *
+ * Set VCN block clockgating state
+ */
+static int vcn_v5_0_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+					    enum amd_clockgating_state state)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	bool enable = state == AMD_CG_STATE_GATE;
+	int i;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
+
+		if (enable) {
+			if (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) != UVD_STATUS__IDLE)
+				return -EBUSY;
+			vcn_v5_0_2_enable_clock_gating(vinst);
+		} else {
+			vcn_v5_0_2_disable_clock_gating(vinst);
+		}
+	}
+
+	return 0;
+}
+
+static int vcn_v5_0_2_set_pg_state(struct amdgpu_vcn_inst *vinst,
+				   enum amd_powergating_state state)
+{
+	int ret = 0;
+
+	if (state == vinst->cur_state)
+		return 0;
+
+	if (state == AMD_PG_STATE_GATE)
+		ret = vcn_v5_0_2_stop(vinst);
+	else
+		ret = vcn_v5_0_2_start(vinst);
+
+	if (!ret)
+		vinst->cur_state = state;
+
+	return ret;
+}
+
+/**
+ * vcn_v5_0_2_process_interrupt - process VCN block interrupt
+ *
+ * @adev: amdgpu_device pointer
+ * @source: interrupt sources
+ * @entry: interrupt entry from clients and sources
+ *
+ * Process VCN block interrupt
+ */
+static int vcn_v5_0_2_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
+	struct amdgpu_iv_entry *entry)
+{
+	uint32_t i, inst;
+
+	i = node_id_to_phys_map[entry->node_id];
+
+	DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n");
+
+	for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst)
+		if (adev->vcn.inst[inst].aid_id == i)
+			break;
+
+	if (inst >= adev->vcn.num_vcn_inst) {
+		dev_WARN_ONCE(adev->dev, 1,
+				"Interrupt received for unknown VCN instance %d",
+				entry->node_id);
+		return 0;
+	}
+
+	switch (entry->src_id) {
+	case VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
+		amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]);
+		break;
+	default:
+		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
+			  entry->src_id, entry->src_data[0]);
+		break;
+	}
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs vcn_v5_0_2_irq_funcs = {
+	.process = vcn_v5_0_2_process_interrupt,
+};
+
+/**
+ * vcn_v5_0_2_set_irq_funcs - set VCN block interrupt irq functions
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set VCN block interrupt irq functions
+ */
+static void vcn_v5_0_2_set_irq_funcs(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
+		adev->vcn.inst->irq.num_types++;
+	adev->vcn.inst->irq.funcs = &vcn_v5_0_2_irq_funcs;
+}
+
+static const struct amd_ip_funcs vcn_v5_0_2_ip_funcs = {
+	.name = "vcn_v5_0_2",
+	.early_init = vcn_v5_0_2_early_init,
+	.late_init = NULL,
+	.sw_init = vcn_v5_0_2_sw_init,
+	.sw_fini = vcn_v5_0_2_sw_fini,
+	.hw_init = vcn_v5_0_2_hw_init,
+	.hw_fini = vcn_v5_0_2_hw_fini,
+	.suspend = vcn_v5_0_2_suspend,
+	.resume = vcn_v5_0_2_resume,
+	.is_idle = vcn_v5_0_2_is_idle,
+	.wait_for_idle = vcn_v5_0_2_wait_for_idle,
+	.check_soft_reset = NULL,
+	.pre_soft_reset = NULL,
+	.soft_reset = NULL,
+	.post_soft_reset = NULL,
+	.set_clockgating_state = vcn_v5_0_2_set_clockgating_state,
+	.set_powergating_state = vcn_set_powergating_state,
+};
+
+const struct amdgpu_ip_block_version vcn_v5_0_2_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_VCN,
+	.major = 5,
+	.minor = 0,
+	.rev = 2,
+	.funcs = &vcn_v5_0_2_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.h b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.h
new file mode 100644
index 0000000000000..461bdda91eb60
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __VCN_v5_0_2_H__
+#define __VCN_v5_0_2_H__
+
+extern const struct amdgpu_ip_block_version vcn_v5_0_2_ip_block;
+
+#endif /* __VCN_v5_0_2_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 6a574b6c8e631..a256320b92f33 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -299,11 +299,11 @@ static u32 vi_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32_NO_KIQ(mmPCIE_INDEX, reg);
 	(void)RREG32_NO_KIQ(mmPCIE_INDEX);
 	r = RREG32_NO_KIQ(mmPCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 	return r;
 }
 
@@ -311,12 +311,12 @@ static void vi_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
 	WREG32_NO_KIQ(mmPCIE_INDEX, reg);
 	(void)RREG32_NO_KIQ(mmPCIE_INDEX);
 	WREG32_NO_KIQ(mmPCIE_DATA, v);
 	(void)RREG32_NO_KIQ(mmPCIE_DATA);
-	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
 }
 
 static u32 vi_smc_rreg(struct amdgpu_device *adev, u32 reg)
@@ -324,10 +324,10 @@ static u32 vi_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32_NO_KIQ(mmSMC_IND_INDEX_11, (reg));
 	r = RREG32_NO_KIQ(mmSMC_IND_DATA_11);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 	return r;
 }
 
@@ -335,10 +335,10 @@ static void vi_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32_NO_KIQ(mmSMC_IND_INDEX_11, (reg));
 	WREG32_NO_KIQ(mmSMC_IND_DATA_11, (v));
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 }
 
 /* smu_8_0_d.h */
@@ -350,10 +350,10 @@ static u32 cz_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmMP0PUB_IND_INDEX, (reg));
 	r = RREG32(mmMP0PUB_IND_DATA);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 	return r;
 }
 
@@ -361,10 +361,10 @@ static void cz_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmMP0PUB_IND_INDEX, (reg));
 	WREG32(mmMP0PUB_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 }
 
 static u32 vi_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
@@ -372,10 +372,10 @@ static u32 vi_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	r = RREG32(mmUVD_CTX_DATA);
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 	return r;
 }
 
@@ -383,10 +383,10 @@ static void vi_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 	WREG32(mmUVD_CTX_DATA, (v));
-	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
 }
 
 static u32 vi_didt_rreg(struct amdgpu_device *adev, u32 reg)
@@ -394,10 +394,10 @@ static u32 vi_didt_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(mmDIDT_IND_INDEX, (reg));
 	r = RREG32(mmDIDT_IND_DATA);
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 	return r;
 }
 
@@ -405,10 +405,10 @@ static void vi_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->didt_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.didt.lock, flags);
 	WREG32(mmDIDT_IND_INDEX, (reg));
 	WREG32(mmDIDT_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
 }
 
 static u32 vi_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
@@ -416,10 +416,10 @@ static u32 vi_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
 	unsigned long flags;
 	u32 r;
 
-	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.gc_cac.lock, flags);
 	WREG32(mmGC_CAC_IND_INDEX, (reg));
 	r = RREG32(mmGC_CAC_IND_DATA);
-	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.gc_cac.lock, flags);
 	return r;
 }
 
@@ -427,10 +427,10 @@ static void vi_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.gc_cac.lock, flags);
 	WREG32(mmGC_CAC_IND_INDEX, (reg));
 	WREG32(mmGC_CAC_IND_DATA, (v));
-	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.gc_cac.lock, flags);
 }
 
 
@@ -649,7 +649,7 @@ static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 	dw_ptr = (u32 *)bios;
 	length_dw = ALIGN(length_bytes, 4) / 4;
 	/* take the smc lock since we are using the smc index */
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	/* set rom index to 0 */
 	WREG32(mmSMC_IND_INDEX_11, ixROM_INDEX);
 	WREG32(mmSMC_IND_DATA_11, 0);
@@ -657,7 +657,7 @@ static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 	WREG32(mmSMC_IND_INDEX_11, ixROM_DATA);
 	for (i = 0; i < length_dw; i++)
 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_11);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return true;
 }
@@ -1454,20 +1454,20 @@ static int vi_common_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	if (adev->flags & AMD_IS_APU) {
-		adev->smc_rreg = &cz_smc_rreg;
-		adev->smc_wreg = &cz_smc_wreg;
+		adev->reg.smc.rreg = cz_smc_rreg;
+		adev->reg.smc.wreg = cz_smc_wreg;
 	} else {
-		adev->smc_rreg = &vi_smc_rreg;
-		adev->smc_wreg = &vi_smc_wreg;
+		adev->reg.smc.rreg = vi_smc_rreg;
+		adev->reg.smc.wreg = vi_smc_wreg;
 	}
-	adev->pcie_rreg = &vi_pcie_rreg;
-	adev->pcie_wreg = &vi_pcie_wreg;
-	adev->uvd_ctx_rreg = &vi_uvd_ctx_rreg;
-	adev->uvd_ctx_wreg = &vi_uvd_ctx_wreg;
-	adev->didt_rreg = &vi_didt_rreg;
-	adev->didt_wreg = &vi_didt_wreg;
-	adev->gc_cac_rreg = &vi_gc_cac_rreg;
-	adev->gc_cac_wreg = &vi_gc_cac_wreg;
+	adev->reg.pcie.rreg = &vi_pcie_rreg;
+	adev->reg.pcie.wreg = &vi_pcie_wreg;
+	adev->reg.uvd_ctx.rreg = &vi_uvd_ctx_rreg;
+	adev->reg.uvd_ctx.wreg = &vi_uvd_ctx_wreg;
+	adev->reg.didt.rreg = &vi_didt_rreg;
+	adev->reg.didt.wreg = &vi_didt_wreg;
+	adev->reg.gc_cac.rreg = &vi_gc_cac_rreg;
+	adev->reg.gc_cac.wreg = &vi_gc_cac_wreg;
 
 	adev->asic_funcs = &vi_asic_funcs;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index 16e12c9913f94..a5d7467c2f349 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -27,7 +27,7 @@ config HSA_AMD_SVM
 
 config HSA_AMD_P2P
 	bool "HSA kernel driver support for peer-to-peer for AMD GPU devices"
-	depends on HSA_AMD && PCI_P2PDMA && DMABUF_MOVE_NOTIFY
+	depends on HSA_AMD && PCI_P2PDMA
 	help
 	  Enable peer-to-peer (P2P) communication between AMD GPUs over
 	  the PCIe bus. This can improve performance of multi-GPU compute
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
index 73acbe0b7c212..b799c70f5742a 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
@@ -97,11 +97,11 @@ static void cik_event_interrupt_wq(struct kfd_node *dev,
 		return;
 
 	if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE)
-		kfd_signal_event_interrupt(pasid, context_id, 28);
+		kfd_signal_event_interrupt(pasid, context_id, 28, true);
 	else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP)
-		kfd_signal_event_interrupt(pasid, context_id, 28);
+		kfd_signal_event_interrupt(pasid, context_id, 28, true);
 	else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG)
-		kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);
+		kfd_signal_event_interrupt(pasid, context_id & 0xff, 8, true);
 	else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
 		kfd_signal_hw_exception_event(pasid);
 	else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
index 456db81998993..d38ff404277b4 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
@@ -134,9 +134,6 @@ var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT	= 12
 var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SIZE	= 6
 #endif
 
-var TTMP11_SCHED_MODE_SHIFT			= 26
-var TTMP11_SCHED_MODE_SIZE			= 2
-var TTMP11_SCHED_MODE_MASK			= 0xC000000
 var TTMP11_DEBUG_TRAP_ENABLED_SHIFT		= 23
 var TTMP11_DEBUG_TRAP_ENABLED_MASK		= 0x800000
 var TTMP11_FIRST_REPLAY_SHIFT			= 22
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 3ddf06c755b52..ab3b2e7be9bd0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -2720,7 +2720,7 @@ static int get_wave_state(struct device_queue_manager *dqm,
 			ctl_stack, ctl_stack_used_size, save_area_used_size);
 }
 
-static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
+static int get_queue_checkpoint_info(struct device_queue_manager *dqm,
 			const struct queue *q,
 			u32 *mqd_size,
 			u32 *ctl_stack_size)
@@ -2728,6 +2728,7 @@ static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
 	struct mqd_manager *mqd_mgr;
 	enum KFD_MQD_TYPE mqd_type =
 			get_mqd_type_from_queue_type(q->properties.type);
+	int ret = 0;
 
 	dqm_lock(dqm);
 	mqd_mgr = dqm->mqd_mgrs[mqd_type];
@@ -2735,9 +2736,11 @@ static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
 	*ctl_stack_size = 0;
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
-		mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
+		ret = mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
 
 	dqm_unlock(dqm);
+
+	return ret;
 }
 
 static int checkpoint_mqd(struct device_queue_manager *dqm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index ef07e44916f80..3272328da11f9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -192,7 +192,7 @@ struct device_queue_manager_ops {
 
 	int (*reset_queues)(struct device_queue_manager *dqm,
 					uint16_t pasid);
-	void	(*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
+	int	(*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
 				  const struct queue *q, u32 *mqd_size,
 				  u32 *ctl_stack_size);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index bafc7b699dcc3..44150a71ffd51 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -142,6 +142,7 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
  * @p:     Pointer to struct kfd_process
  * @id:    ID to look up
  * @bits:  Number of valid bits in @id
+ * @signal_mailbox_updated: flag indicates if FW updates signal mailbox entry
  *
  * Finds the first signaled event with a matching partial ID. If no
  * matching signaled event is found, returns NULL. In that case the
@@ -155,7 +156,8 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
  * driver.
  */
 static struct kfd_event *lookup_signaled_event_by_partial_id(
-	struct kfd_process *p, uint32_t id, uint32_t bits)
+	struct kfd_process *p, uint32_t id, uint32_t bits,
+	bool signal_mailbox_updated)
 {
 	struct kfd_event *ev;
 
@@ -166,7 +168,8 @@ static struct kfd_event *lookup_signaled_event_by_partial_id(
 	 * and we only need a single lookup.
 	 */
 	if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
-		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
+		if (signal_mailbox_updated &&
+		    page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
 			return NULL;
 
 		return idr_find(&p->event_idr, id);
@@ -724,7 +727,7 @@ static void set_event_from_interrupt(struct kfd_process *p,
 }
 
 void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
-				uint32_t valid_id_bits)
+				uint32_t valid_id_bits, bool signal_mailbox_updated)
 {
 	struct kfd_event *ev = NULL;
 
@@ -742,7 +745,8 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
 
 	if (valid_id_bits)
 		ev = lookup_signaled_event_by_partial_id(p, partial_id,
-							 valid_id_bits);
+							 valid_id_bits,
+							 signal_mailbox_updated);
 	if (ev) {
 		set_event_from_interrupt(p, ev);
 	} else if (p->signal_page) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index 52ccfd397c2b3..1dc21c13833b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -85,6 +85,7 @@ struct kfd_event {
 #define KFD_EVENT_TYPE_MEMORY 8
 
 extern void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
-				       uint32_t valid_id_bits);
+				       uint32_t valid_id_bits,
+				       bool signal_mailbox_updated);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index e8da0b4527dc5..04c5e26f01ed9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -342,20 +342,14 @@ static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id)
 
 static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)
 {
-	if (KFD_GC_VERSION(pdd->dev) >= IP_VERSION(12, 1, 0))
-		pdd->lds_base = pdd->dev->adev->gmc.shared_aperture_start;
-	else
-		pdd->lds_base = MAKE_LDS_APP_BASE_V9();
+	pdd->lds_base = MAKE_LDS_APP_BASE_V9();
 	pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
 
 	pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM;
 	pdd->gpuvm_limit =
 		pdd->dev->kfd->shared_resources.gpuvm_size - 1;
 
-	if (KFD_GC_VERSION(pdd->dev) >= IP_VERSION(12, 1, 0))
-		pdd->scratch_base = pdd->dev->adev->gmc.private_aperture_start;
-	else
-		pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
+	pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
 	pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
 
 	/*
@@ -365,6 +359,25 @@ static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)
 	pdd->qpd.cwsr_base = AMDGPU_VA_RESERVED_TRAP_START(pdd->dev->adev);
 }
 
+static void kfd_init_apertures_v12(struct kfd_process_device *pdd, uint8_t id)
+{
+	pdd->lds_base = pdd->dev->adev->gmc.shared_aperture_start;
+	pdd->lds_limit = pdd->dev->adev->gmc.shared_aperture_end;
+
+	pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM;
+	pdd->gpuvm_limit =
+		pdd->dev->kfd->shared_resources.gpuvm_size - 1;
+
+	pdd->scratch_base = pdd->dev->adev->gmc.private_aperture_start;
+	pdd->scratch_limit = pdd->dev->adev->gmc.private_aperture_end;
+
+	/*
+	 * Place TBA/TMA on opposite side of VM hole to prevent
+	 * stray faults from triggering SVM on these pages.
+	 */
+	pdd->qpd.cwsr_base = AMDGPU_VA_RESERVED_TRAP_START(pdd->dev->adev);
+}
+
 int kfd_init_apertures(struct kfd_process *process)
 {
 	uint8_t id  = 0;
@@ -412,9 +425,11 @@ int kfd_init_apertures(struct kfd_process *process)
 				kfd_init_apertures_vi(pdd, id);
 				break;
 			default:
-				if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
+				if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0)) {
+					kfd_init_apertures_v12(pdd, id);
+				} else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) {
 					kfd_init_apertures_v9(pdd, id);
-				else {
+				} else {
 					WARN(1, "Unexpected ASIC family %u",
 					     dev->adev->asic_type);
 					return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
index 3e1ad89747979..19406ab92c5bb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
@@ -211,7 +211,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 	    client_id == SOC15_IH_CLIENTID_SE2SH ||
 	    client_id == SOC15_IH_CLIENTID_SE3SH) {
 		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
-			kfd_signal_event_interrupt(pasid, context_id0, 32);
+			kfd_signal_event_interrupt(pasid, context_id0, 32, true);
 		else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
 			encoding = REG_GET_FIELD(context_id1,
 						SQ_INTERRUPT_WORD_WAVE_CTXID1, ENCODING);
@@ -324,7 +324,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 			default:
 				break;
 			}
-			kfd_signal_event_interrupt(pasid, context_id0 & 0x7fffff, 23);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0x7fffff, 23, true);
 		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
 			   KFD_DBG_EC_TYPE_IS_PACKET(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0))) {
 			kfd_set_dbg_ev_from_interrupt(dev, pasid,
@@ -344,7 +344,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 		   client_id == SOC15_IH_CLIENTID_SDMA6 ||
 		   client_id == SOC15_IH_CLIENTID_SDMA7) {
 		if (source_id == SOC15_INTSRC_SDMA_TRAP) {
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true);
 		}
 	} else if (client_id == SOC15_IH_CLIENTID_VMC ||
 		   client_id == SOC15_IH_CLIENTID_VMC1 ||
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
index 2788a52714d10..12d81abed7481 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
@@ -353,7 +353,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
 
 		/* CP */
 		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
-			kfd_signal_event_interrupt(pasid, context_id0, 32);
+			kfd_signal_event_interrupt(pasid, context_id0, 32, true);
 		else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
 			 KFD_DBG_EC_TYPE_IS_PACKET(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0))) {
 			u32 doorbell_id = KFD_CTXID0_DOORBELL_ID(context_id0);
@@ -366,7 +366,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
 
 		/* SDMA */
 		else if (source_id == SOC21_INTSRC_SDMA_TRAP)
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true);
 		else if (source_id == SOC21_INTSRC_SDMA_ECC) {
 			event_interrupt_poison_consumption_v11(dev, pasid, source_id);
 			return;
@@ -404,7 +404,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
 			default:
 				break;
 			}
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24, true);
 		}
 
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
index 230e57dbcec58..47947b94926ba 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
@@ -28,6 +28,7 @@
 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
 #include "kfd_smi_events.h"
 #include "kfd_debug.h"
+#include "amdgpu_ras_mgr.h"
 
 /*
  * GFX12.1 SQ Interrupts
@@ -185,6 +186,7 @@ static void event_interrupt_poison_consumption_v12_1(struct kfd_node *node,
 	enum amdgpu_ras_block block = 0;
 	int ret = -EINVAL;
 	uint32_t reset = 0;
+	u64 event_id = RAS_EVENT_INVALID_ID;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, NULL);
 
 	if (!p)
@@ -220,7 +222,15 @@ static void event_interrupt_poison_consumption_v12_1(struct kfd_node *node,
 	 * resetting queue passes, do page retirement without gpu reset
 	 * resetting queue fails, fallback to gpu reset solution
 	 */
-	amdgpu_amdkfd_ras_poison_consumption_handler(node->adev, block, reset);
+	if (amdgpu_uniras_enabled(node->adev))
+		event_id = amdgpu_ras_mgr_gen_ras_event_seqno(node->adev,
+					RAS_SEQNO_TYPE_POISON_CONSUMPTION);
+
+	RAS_EVENT_LOG(node->adev, event_id,
+		      "poison is consumed by source %d, kick off gpu reset flow\n", source_id);
+
+	amdgpu_amdkfd_ras_pasid_poison_consumption_handler(node->adev,
+				block, pasid, NULL, NULL, reset);
 }
 
 static bool event_interrupt_isr_v12_1(struct kfd_node *node,
@@ -326,7 +336,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node,
 
 		/* CP */
 		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
-			kfd_signal_event_interrupt(pasid, context_id0, 32);
+			kfd_signal_event_interrupt(pasid, context_id0, 32, false);
 		else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
 			 KFD_DBG_EC_TYPE_IS_PACKET(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0))) {
 			u32 doorbell_id = KFD_CTXID0_DOORBELL_ID(context_id0);
@@ -339,7 +349,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node,
 
 		/* SDMA */
 		else if (source_id == SOC21_INTSRC_SDMA_TRAP)
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true);
 		else if (source_id == SOC21_INTSRC_SDMA_ECC) {
 			event_interrupt_poison_consumption_v12_1(node, pasid, source_id);
 			return;
@@ -377,7 +387,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node,
 			default:
 				break;
 			}
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24, true);
 		}
 
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index d76fb61869c75..1688d8e595f2f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -379,7 +379,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 	    client_id == SOC15_IH_CLIENTID_SE2SH ||
 	    client_id == SOC15_IH_CLIENTID_SE3SH) {
 		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
-			kfd_signal_event_interrupt(pasid, context_id0, 32);
+			kfd_signal_event_interrupt(pasid, context_id0, 32, true);
 		else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
 			sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1);
 			encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING);
@@ -513,7 +513,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 			default:
 				break;
 			}
-			kfd_signal_event_interrupt(pasid, sq_int_data, 24);
+			kfd_signal_event_interrupt(pasid, sq_int_data, 24, true);
 		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
 			   KFD_DBG_EC_TYPE_IS_PACKET(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0))) {
 			kfd_set_dbg_ev_from_interrupt(dev, pasid,
@@ -530,7 +530,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 		   client_id == SOC15_IH_CLIENTID_SDMA6 ||
 		   client_id == SOC15_IH_CLIENTID_SDMA7) {
 		if (source_id == SOC15_INTSRC_SDMA_TRAP) {
-			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28, true);
 		} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
 			event_interrupt_poison_consumption_v9(dev, pasid, client_id);
 			return;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index b021f1e561147..10bc81ce37cbf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -59,8 +59,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
 	void *cpu_addr;
 	int r;
 
-	/* use gart window 0 */
-	*gart_addr = adev->gmc.gart_start;
+	*gart_addr = amdgpu_compute_gart_address(&adev->gmc, entity, 0);
 
 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
 	num_bytes = npages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
@@ -78,6 +77,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
 	src_addr += job->ibs[0].gpu_addr;
 
 	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
+	dst_addr += (entity->gart_window_offs[0] >> AMDGPU_GPU_PAGE_SHIFT) * 8;
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
 				dst_addr, num_bytes, 0);
 
@@ -116,7 +116,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
  * multiple GTT_MAX_PAGES transfer, all sdma operations are serialized, wait for
  * the last sdma finish fence which is returned to check copy memory is done.
  *
- * Context: Process context, takes and releases gtt_window_lock
+ * Context: Process context
  *
  * Return:
  * 0 - OK, otherwise error code
@@ -136,9 +136,9 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 	u64 size;
 	int r;
 
-	entity = &adev->mman.default_entity;
+	entity = &adev->mman.move_entity;
 
-	mutex_lock(&adev->mman.gtt_window_lock);
+	mutex_lock(&entity->lock);
 
 	while (npages) {
 		size = min(GTT_MAX_PAGES, npages);
@@ -175,7 +175,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 	}
 
 out_unlock:
-	mutex_unlock(&adev->mman.gtt_window_lock);
+	mutex_unlock(&entity->lock);
 
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 2429d278ef0eb..06ca6235ff1b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -102,7 +102,8 @@ struct mqd_manager {
 				  u32 *ctl_stack_used_size,
 				  u32 *save_area_used_size);
 
-	void	(*get_checkpoint_info)(struct mqd_manager *mm, void *mqd, uint32_t *ctl_stack_size);
+	int	(*get_checkpoint_info)(struct mqd_manager *mm, void *mqd,
+				       uint32_t *ctl_stack_size);
 
 	void	(*checkpoint_mqd)(struct mqd_manager *mm,
 				  void *mqd,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 562d475cf4c99..bb70e57ae4d52 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -70,7 +70,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct cik_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index d6067316d7f49..77fb41e2486a4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -70,7 +70,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct v10_compute_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index e3a7acb0ccbc8..a1e3cf2384dd3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -96,7 +96,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct v11_compute_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index 0b97376fc6f9f..b3e122d7876e0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -77,7 +77,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct v12_compute_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index eef6bdce4be39..c90c0d99b1e3f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -131,7 +131,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct v12_1_compute_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index d5c234f30e8d6..979ae94ac9668 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -106,7 +106,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct v9_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static bool mqd_on_vram(struct amdgpu_device *adev)
@@ -386,11 +385,14 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
-static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
 {
 	struct v9_mqd *m = get_mqd(mqd);
 
-	*ctl_stack_size = m->cp_hqd_cntl_stack_size * NUM_XCC(mm->dev->xcc_mask);
+	if (check_mul_overflow(m->cp_hqd_cntl_stack_size, NUM_XCC(mm->dev->xcc_mask), ctl_stack_size))
+		return -EINVAL;
+
+	return 0;
 }
 
 static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 69c1b8a690b86..431a20323146b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -73,7 +73,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static void set_priority(struct vi_mqd *m, struct queue_properties *q)
 {
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
-	/* m->cp_hqd_queue_priority = q->priority; */
 }
 
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
@@ -275,10 +274,11 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
-static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+static int get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
 {
 	/* Control stack is stored in user mode */
 	*ctl_stack_size = 0;
+	return 0;
 }
 
 static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index e5b56412931bd..85f2bc3fbf85e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1525,7 +1525,7 @@ int kfd_wait_on_events(struct kfd_process *p,
 		       bool all, uint32_t *user_timeout_ms,
 		       uint32_t *wait_result);
 void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
-				uint32_t valid_id_bits);
+				uint32_t valid_id_bits, bool signal_mailbox_updated);
 void kfd_signal_hw_exception_event(u32 pasid);
 int kfd_set_event(struct kfd_process *p, uint32_t event_id);
 int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 12e24fbf8c463..a031166f270c8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -585,7 +585,7 @@ static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 		ret = kobject_init_and_add(pdd->kobj_stats,
 					   &procfs_stats_type,
 					   p->kobj,
-					   stats_dir_filename);
+					   "%s", stats_dir_filename);
 
 		if (ret) {
 			pr_warn("Creating KFD proc/stats_%s folder failed",
@@ -632,7 +632,7 @@ static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
 			return;
 
 		ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
-					   p->kobj, counters_dir_filename);
+					   p->kobj, "%s", counters_dir_filename);
 		if (ret) {
 			pr_warn("Creating KFD proc/%s folder failed",
 				counters_dir_filename);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index f5d2847e1cbb4..44e39ce222b78 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -590,7 +590,8 @@ int pqm_update_queue_properties(struct process_queue_manager *pqm,
 			return err;
 
 		if (kfd_queue_buffer_get(vm, (void *)p->queue_address, &p->ring_bo,
-					 p->queue_size)) {
+					 p->queue_size +
+					 pqn->q->properties.metadata_queue_size)) {
 			pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n",
 				 p->queue_address, p->queue_size);
 			amdgpu_bo_unreserve(vm->root.bo);
@@ -1070,6 +1071,7 @@ int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
 				  uint32_t *ctl_stack_size)
 {
 	struct process_queue_node *pqn;
+	int ret;
 
 	pqn = get_queue_by_qid(pqm, qid);
 	if (!pqn) {
@@ -1082,9 +1084,14 @@ int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
 		return -EOPNOTSUPP;
 	}
 
-	pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
+	ret = pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
 						       pqn->q, mqd_size,
 						       ctl_stack_size);
+	if (ret) {
+		pr_debug("amdkfd: Overflow while computing stack size for queue %d\n", qid);
+		return ret;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 080242f9981b0..37fdcaf7192f1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1219,7 +1219,8 @@ svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
 	bool snoop = (domain != SVM_RANGE_VRAM_DOMAIN);
 	bool coherent = flags & (KFD_IOCTL_SVM_FLAG_COHERENT | KFD_IOCTL_SVM_FLAG_EXT_COHERENT);
 	bool ext_coherent = flags & KFD_IOCTL_SVM_FLAG_EXT_COHERENT;
-	unsigned int mtype_local;
+	unsigned int mtype_local, mtype_remote;
+	bool is_aid_a1, is_local;
 
 	if (domain == SVM_RANGE_VRAM_DOMAIN)
 		bo_node = prange->svm_bo->node;
@@ -1307,20 +1308,23 @@ svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
 		mapping_flags |= AMDGPU_VM_MTYPE_NC;
 		break;
 	case IP_VERSION(12, 1, 0):
+		is_aid_a1 = (node->adev->rev_id & 0x10);
+		is_local = (domain == SVM_RANGE_VRAM_DOMAIN) &&
+				(bo_node->adev == node->adev);
+
+		mtype_local = amdgpu_mtype_local == 0 ? AMDGPU_VM_MTYPE_RW :
+				amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
+				is_aid_a1 ? AMDGPU_VM_MTYPE_RW : AMDGPU_VM_MTYPE_NC;
+		mtype_remote = is_aid_a1 ? AMDGPU_VM_MTYPE_NC : AMDGPU_VM_MTYPE_UC;
 		snoop = true;
-		if (domain == SVM_RANGE_VRAM_DOMAIN) {
-			mtype_local = amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
-								AMDGPU_VM_MTYPE_RW;
-			/* local HBM  */
-			if (bo_node->adev == node->adev)
-				mapping_flags |= mtype_local;
-			/* Remote GPU memory */
-			else
-				mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC :
-								AMDGPU_VM_MTYPE_NC;
-		/* system memory accessed by the dGPU */
+
+		if (is_local) /* local HBM  */ {
+			mapping_flags |= mtype_local;
+		} else if (ext_coherent) {
+			mapping_flags |= AMDGPU_VM_MTYPE_UC;
 		} else {
-			mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+			/* system memory or remote VRAM */
+			mapping_flags |= mtype_remote;
 		}
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index b3d6f2cd8ab6f..b8611ff8e1e6b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2015-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -91,6 +91,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_eld.h>
+#include <drm/drm_mode.h>
 #include <drm/drm_utils.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_audio_component.h>
@@ -152,6 +153,9 @@ MODULE_FIRMWARE(FIRMWARE_DCN_36_DMUB);
 #define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin"
 MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB);
 
+#define FIRMWARE_DCN_42_DMUB "amdgpu/dcn_4_2_dmcub.bin"
+MODULE_FIRMWARE(FIRMWARE_DCN_42_DMUB);
+
 /**
  * DOC: overview
  *
@@ -1368,6 +1372,7 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev)
 	case IP_VERSION(3, 5, 0):
 	case IP_VERSION(3, 5, 1):
 	case IP_VERSION(3, 6, 0):
+	case IP_VERSION(4, 2, 0):
 		hw_params.ips_sequential_ono = adev->external_rev_id > 0x10;
 		hw_params.lower_hbr3_phy_ssc = true;
 		break;
@@ -1815,6 +1820,9 @@ static void *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev)
 	case IP_VERSION(4, 0, 1):
 		bb_size = sizeof(struct dml2_soc_bb);
 		break;
+	case IP_VERSION(4, 2, 0):
+		bb_size = sizeof(struct dml2_soc_bb);
+		break;
 	default:
 		return NULL;
 	}
@@ -1859,6 +1867,9 @@ static enum dmub_ips_disable_type dm_get_default_ips_mode(
 	case IP_VERSION(3, 5, 1):
 		ret =  DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
 		break;
+	case IP_VERSION(4, 2, 0):
+		ret =  DMUB_IPS_DISABLE_ALL;
+		break;
 	default:
 		/* ASICs older than DCN35 do not have IPSs */
 		if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 5, 0))
@@ -2002,6 +2013,13 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0))
 		init_data.num_virtual_links = 1;
 
+	/* DCN42 and above dpia switch to unified link training path */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 2, 0)) {
+		init_data.flags.consolidated_dpia_dp_lt = true;
+		init_data.flags.enable_dpia_pre_training = true;
+		init_data.flags.unify_link_enc_assignment = true;
+		init_data.flags.usb4_bw_alloc_support = true;
+	}
 	retrieve_dmi_info(&adev->dm);
 	if (adev->dm.edp0_on_dp1_quirk)
 		init_data.flags.support_edp0_on_dp1 = true;
@@ -2272,6 +2290,11 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 				      &adev->dm.dmub_bo_gpu_addr,
 				      &adev->dm.dmub_bo_cpu_addr);
 
+	if (adev->dm.boot_time_crc_info.bo_ptr)
+		amdgpu_bo_free_kernel(&adev->dm.boot_time_crc_info.bo_ptr,
+					&adev->dm.boot_time_crc_info.gpu_addr,
+					&adev->dm.boot_time_crc_info.cpu_addr);
+
 	if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) {
 		for (i = 0; i < adev->dm.dc->caps.max_links; i++) {
 			if (adev->dm.hpd_rx_offload_wq[i].wq) {
@@ -2369,6 +2392,7 @@ static int load_dmcu_fw(struct amdgpu_device *adev)
 		case IP_VERSION(3, 5, 1):
 		case IP_VERSION(3, 6, 0):
 		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
 			return 0;
 		default:
 			break;
@@ -2454,6 +2478,8 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev)
 		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_7_SCRATCH_MEM
 		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_IB_MEM
 		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_SHARED_STATE
+		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_LSDMA_BUFFER
+		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_CURSOR_OFFLOAD
 	};
 	int r;
 
@@ -2502,7 +2528,9 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev)
 	case IP_VERSION(4, 0, 1):
 		dmub_asic = DMUB_ASIC_DCN401;
 		break;
-
+	case IP_VERSION(4, 2, 0):
+		dmub_asic = DMUB_ASIC_DCN42;
+		break;
 	default:
 		/* ASIC doesn't support DMUB. */
 		return 0;
@@ -2554,7 +2582,7 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev)
 	fw_meta_info_params.fw_inst_const = adev->dm.dmub_fw->data +
 					    le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
 					    PSP_HEADER_BYTES_256;
-	fw_meta_info_params.fw_bss_data = region_params.bss_data_size ? adev->dm.dmub_fw->data +
+	fw_meta_info_params.fw_bss_data = fw_meta_info_params.bss_data_size ? adev->dm.dmub_fw->data +
 					  le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
 					  le32_to_cpu(hdr->inst_const_bytes) : NULL;
 	fw_meta_info_params.custom_psp_footer_size = 0;
@@ -2715,6 +2743,54 @@ static int detect_mst_link_for_all_connectors(struct drm_device *dev)
 	return ret;
 }
 
+static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev)
+{
+	struct dm_boot_time_crc_info *bootcrc_info = NULL;
+	struct dmub_srv *dmub = NULL;
+	union dmub_fw_boot_options option = {0};
+	int ret = 0;
+	const uint32_t fb_size = 3 * 1024 * 1024;	/* 3MB for DCC pattern */
+
+	if (!adev || !adev->dm.dc || !adev->dm.dc->ctx ||
+		!adev->dm.dc->ctx->dmub_srv) {
+		return;
+	}
+
+	dmub = adev->dm.dc->ctx->dmub_srv->dmub;
+	bootcrc_info = &adev->dm.boot_time_crc_info;
+
+	if (!dmub || !dmub->hw_funcs.get_fw_boot_option) {
+		drm_dbg(adev_to_drm(adev), "failed to init boot time crc buffer\n");
+		return;
+	}
+
+	option = dmub->hw_funcs.get_fw_boot_option(dmub);
+
+	/* Return if boot time CRC is not enabled */
+	if (option.bits.bootcrc_en_at_S0i3 == 0)
+		return;
+
+	/* Create a buffer for boot time CRC */
+	ret = amdgpu_bo_create_kernel(adev, fb_size, PAGE_SIZE,
+		AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
+		&bootcrc_info->bo_ptr,
+		&bootcrc_info->gpu_addr,
+		&bootcrc_info->cpu_addr);
+
+	if (ret) {
+		drm_dbg(adev_to_drm(adev), "failed to create boot time crc buffer\n");
+	} else {
+		bootcrc_info->size = fb_size;
+
+		drm_dbg(adev_to_drm(adev), "boot time crc buffer created addr 0x%llx, size %u\n",
+			bootcrc_info->gpu_addr, bootcrc_info->size);
+
+		/* Send the buffer info to DMUB */
+		dc_dmub_srv_boot_time_crc_init(adev->dm.dc,
+			bootcrc_info->gpu_addr, bootcrc_info->size);
+	}
+}
+
 static int dm_late_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -2726,6 +2802,11 @@ static int dm_late_init(struct amdgpu_ip_block *ip_block)
 
 	dmcu = adev->dm.dc->res_pool->dmcu;
 
+	/* Init the boot time CRC (skip in resume) */
+	if ((adev->in_suspend == 0) &&
+		(amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(3, 6, 0)))
+		amdgpu_dm_boot_time_crc_init(adev);
+
 	for (i = 0; i < 16; i++)
 		linear_lut[i] = 0xFFFF * i / 15;
 
@@ -3765,6 +3846,10 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 	caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps;
 	caps->aux_support = false;
 
+	drm_object_property_set_value(&conn_base->base,
+				      adev_to_drm(adev)->mode_config.panel_type_property,
+				      caps->ext_caps->bits.oled ? DRM_MODE_PANEL_TYPE_OLED : DRM_MODE_PANEL_TYPE_UNKNOWN);
+
 	if (caps->ext_caps->bits.oled == 1
 	    /*
 	     * ||
@@ -4385,105 +4470,6 @@ static int register_hpd_handlers(struct amdgpu_device *adev)
 	return 0;
 }
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-/* Register IRQ sources and initialize IRQ callbacks */
-static int dce60_register_irq_handlers(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	struct common_irq_params *c_irq_params;
-	struct dc_interrupt_params int_params = {0};
-	int r;
-	int i;
-	unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
-
-	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
-	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
-
-	/*
-	 * Actions of amdgpu_irq_add_id():
-	 * 1. Register a set() function with base driver.
-	 *    Base driver will call set() function to enable/disable an
-	 *    interrupt in DC hardware.
-	 * 2. Register amdgpu_dm_irq_handler().
-	 *    Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
-	 *    coming from DC hardware.
-	 *    amdgpu_dm_irq_handler() will re-direct the interrupt to DC
-	 *    for acknowledging and handling.
-	 */
-
-	/* Use VBLANK interrupt */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		r = amdgpu_irq_add_id(adev, client_id, i + 1, &adev->crtc_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i + 1, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_VBLANK1 ||
-			int_params.irq_source  > DC_IRQ_SOURCE_VBLANK6) {
-			drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_crtc_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* Use GRPH_PFLIP interrupt */
-	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
-			i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
-		r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_PFLIP_FIRST ||
-			int_params.irq_source  > DC_IRQ_SOURCE_PFLIP_LAST) {
-			drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_pflip_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* HPD */
-	r = amdgpu_irq_add_id(adev, client_id,
-			VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
-	if (r) {
-		drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
-		return r;
-	}
-
-	r = register_hpd_handlers(adev);
-
-	return r;
-}
-#endif
-
 /* Register IRQ sources and initialize IRQ callbacks */
 static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 {
@@ -4492,7 +4478,12 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 	struct dc_interrupt_params int_params = {0};
 	int r;
 	int i;
+	unsigned int src_id;
 	unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
+	/* Use different interrupts for VBLANK on DCE 6 vs. newer. */
+	const unsigned int vblank_d1 =
+		adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0
+		? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1;
 
 	if (adev->family >= AMDGPU_FAMILY_AI)
 		client_id = SOC15_IH_CLIENTID_DCE;
@@ -4513,8 +4504,9 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 	 */
 
 	/* Use VBLANK interrupt */
-	for (i = VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0; i <= VISLANDS30_IV_SRCID_D6_VERTICAL_INTERRUPT0; i++) {
-		r = amdgpu_irq_add_id(adev, client_id, i, &adev->crtc_irq);
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		src_id = vblank_d1 + i;
+		r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq);
 		if (r) {
 			drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
 			return r;
@@ -4522,7 +4514,7 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 
 		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
 		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
+			dc_interrupt_to_irq_source(dc, src_id, 0);
 
 		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
 			int_params.irq_source  < DC_IRQ_SOURCE_VBLANK1 ||
@@ -4541,33 +4533,36 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 			return -ENOMEM;
 	}
 
-	/* Use VUPDATE interrupt */
-	for (i = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT; i <= VISLANDS30_IV_SRCID_D6_V_UPDATE_INT; i += 2) {
-		r = amdgpu_irq_add_id(adev, client_id, i, &adev->vupdate_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
-			return r;
-		}
+	if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
+		/* Use VUPDATE interrupt */
+		for (i = 0; i < adev->mode_info.num_crtc; i++) {
+			src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2;
+			r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq);
+			if (r) {
+				drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
+				return r;
+			}
 
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
+			int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+			int_params.irq_source =
+				dc_interrupt_to_irq_source(dc, src_id, 0);
 
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
-			int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
-			drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+				int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
+				int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
+				drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
+				return -EINVAL;
+			}
 
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
+			c_irq_params = &adev->dm.vupdate_params[
+				int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+			c_irq_params->adev = adev;
+			c_irq_params->irq_src = int_params.irq_source;
 
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_vupdate_high_irq, c_irq_params))
-			return -ENOMEM;
+			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				dm_vupdate_high_irq, c_irq_params))
+				return -ENOMEM;
+		}
 	}
 
 	/* Use GRPH_PFLIP interrupt */
@@ -4909,14 +4904,37 @@ static void dm_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(dm_state);
 }
 
+static struct drm_private_state *
+dm_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct amdgpu_device *adev = drm_to_adev(obj->dev);
+	struct dm_atomic_state *dm_state;
+	struct dc_state *context;
+
+	dm_state = kzalloc_obj(*dm_state);
+	if (!dm_state)
+		return ERR_PTR(-ENOMEM);
+
+	context = dc_state_create_current_copy(adev->dm.dc);
+	if (!context) {
+		kfree(dm_state);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	__drm_atomic_helper_private_obj_create_state(obj, &dm_state->base);
+	dm_state->context = context;
+
+	return &dm_state->base;
+}
+
 static struct drm_private_state_funcs dm_atomic_state_funcs = {
+	.atomic_create_state = dm_atomic_create_state,
 	.atomic_duplicate_state = dm_atomic_duplicate_state,
 	.atomic_destroy_state = dm_atomic_destroy_state,
 };
 
 static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 {
-	struct dm_atomic_state *state;
 	int r;
 
 	adev->mode_info.mode_config_initialized = true;
@@ -4936,42 +4954,22 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 	/* indicates support for immediate flip */
 	adev_to_drm(adev)->mode_config.async_page_flip = true;
 
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
-
-	state->context = dc_state_create_current_copy(adev->dm.dc);
-	if (!state->context) {
-		kfree(state);
-		return -ENOMEM;
-	}
-
 	drm_atomic_private_obj_init(adev_to_drm(adev),
 				    &adev->dm.atomic_obj,
-				    &state->base,
 				    &dm_atomic_state_funcs);
 
 	r = amdgpu_display_modeset_create_props(adev);
-	if (r) {
-		dc_state_release(state->context);
-		kfree(state);
+	if (r)
 		return r;
-	}
 
 #ifdef AMD_PRIVATE_COLOR
-	if (amdgpu_dm_create_color_properties(adev)) {
-		dc_state_release(state->context);
-		kfree(state);
+	if (amdgpu_dm_create_color_properties(adev))
 		return -ENOMEM;
-	}
 #endif
 
 	r = amdgpu_dm_audio_init(adev);
-	if (r) {
-		dc_state_release(state->context);
-		kfree(state);
+	if (r)
 		return r;
-	}
 
 	return 0;
 }
@@ -5539,6 +5537,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 	case IP_VERSION(3, 5, 1):
 	case IP_VERSION(3, 6, 0):
 	case IP_VERSION(4, 0, 1):
+	case IP_VERSION(4, 2, 0):
 		if (register_outbox_irq_handlers(dm->adev)) {
 			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
 			goto fail;
@@ -5563,6 +5562,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		case IP_VERSION(3, 5, 1):
 		case IP_VERSION(3, 6, 0):
 		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
 			psr_feature_enabled = true;
 			break;
 		default:
@@ -5695,11 +5695,6 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 	case CHIP_PITCAIRN:
 	case CHIP_VERDE:
 	case CHIP_OLAND:
-		if (dce60_register_irq_handlers(dm->adev)) {
-			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
-			goto fail;
-		}
-		break;
 #endif
 	case CHIP_BONAIRE:
 	case CHIP_HAWAII:
@@ -5745,6 +5740,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		case IP_VERSION(3, 5, 1):
 		case IP_VERSION(3, 6, 0):
 		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
 			if (dcn10_register_irq_handlers(dm->adev)) {
 				drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
 				goto fail;
@@ -5893,6 +5889,9 @@ static int dm_init_microcode(struct amdgpu_device *adev)
 	case IP_VERSION(4, 0, 1):
 		fw_name_dmub = FIRMWARE_DCN_401_DMUB;
 		break;
+	case IP_VERSION(4, 2, 0):
+		fw_name_dmub = FIRMWARE_DCN_42_DMUB;
+		break;
 	default:
 		/* ASIC doesn't support DMUB. */
 		return 0;
@@ -6020,6 +6019,7 @@ static int dm_early_init(struct amdgpu_ip_block *ip_block)
 		case IP_VERSION(3, 5, 1):
 		case IP_VERSION(3, 6, 0):
 		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
 			adev->mode_info.num_crtc = 4;
 			adev->mode_info.num_hpd = 4;
 			adev->mode_info.num_dig = 4;
@@ -6036,8 +6036,8 @@ static int dm_early_init(struct amdgpu_ip_block *ip_block)
 		adev->mode_info.funcs = &dm_display_funcs;
 
 	/*
-	 * Note: Do NOT change adev->audio_endpt_rreg and
-	 * adev->audio_endpt_wreg because they are initialised in
+	 * Note: Do NOT change adev->reg.audio_endpt.rreg and
+	 * adev->reg.audio_endpt.wreg because they are initialised in
 	 * amdgpu_device_init()
 	 */
 #if defined(CONFIG_DEBUG_KERNEL_DC)
@@ -8578,6 +8578,12 @@ static int to_drm_connector_type(enum signal_type st, uint32_t connector_id)
 		return DRM_MODE_CONNECTOR_VGA;
 	case SIGNAL_TYPE_DISPLAY_PORT:
 	case SIGNAL_TYPE_DISPLAY_PORT_MST:
+		/* External DP bridges have a different connector type. */
+		if (connector_id == CONNECTOR_ID_VGA)
+			return DRM_MODE_CONNECTOR_VGA;
+		else if (connector_id == CONNECTOR_ID_LVDS)
+			return DRM_MODE_CONNECTOR_LVDS;
+
 		return DRM_MODE_CONNECTOR_DisplayPort;
 	case SIGNAL_TYPE_DVI_DUAL_LINK:
 	case SIGNAL_TYPE_DVI_SINGLE_LINK:
@@ -9073,6 +9079,8 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 	if (connector_type == DRM_MODE_CONNECTOR_eDP) {
 		struct drm_privacy_screen *privacy_screen;
 
+		drm_connector_attach_panel_type_property(&aconnector->base);
+
 		privacy_screen = drm_privacy_screen_get(adev_to_drm(adev)->dev, NULL);
 		if (!IS_ERR(privacy_screen)) {
 			drm_connector_attach_privacy_screen_provider(&aconnector->base,
@@ -9250,8 +9258,7 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
 	    connector_type == DRM_MODE_CONNECTOR_HDMIB)
 		amdgpu_dm_initialize_hdmi_connector(aconnector);
 
-	if (connector_type == DRM_MODE_CONNECTOR_DisplayPort
-		|| connector_type == DRM_MODE_CONNECTOR_eDP)
+	if (dc_is_dp_signal(link->connector_signal))
 		amdgpu_dm_initialize_dp_connector(dm, aconnector, link->link_index);
 
 out_free:
@@ -12313,10 +12320,11 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 	int i;
 
 	/* Overlay cursor not supported on HW before DCN
-	 * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions
-	 * as previous DCN generations, so enable native mode on DCN401
+	 * DCN401/420 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions
+	 * as previous DCN generations, so enable native mode on DCN401/420
 	 */
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) {
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1) ||
+	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0)) {
 		*cursor_mode = DM_CURSOR_NATIVE_MODE;
 		return 0;
 	}
@@ -12736,7 +12744,8 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 		/* Check if rotation or scaling is enabled on DCN401 */
 		if ((drm_plane_mask(crtc->cursor) & new_crtc_state->plane_mask) &&
-		    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) {
+		    (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0) ||
+		    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1))) {
 			new_cursor_state = drm_atomic_get_new_plane_state(state, crtc->cursor);
 
 			is_rotated = new_cursor_state &&
@@ -13119,7 +13128,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector,
 	u16 min_vfreq;
 	u16 max_vfreq;
 
-	if (edid == NULL || edid->extensions == 0)
+	if (!edid || !edid->extensions)
 		return;
 
 	/* Find DisplayID extension */
@@ -13129,7 +13138,7 @@ static void parse_edid_displayid_vrr(struct drm_connector *connector,
 			break;
 	}
 
-	if (edid_ext == NULL)
+	if (i == edid->extensions)
 		return;
 
 	while (j < EDID_LENGTH) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 8008136717482..83fefd9023559 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -123,6 +123,20 @@ struct dm_compressor_info {
 	uint64_t gpu_addr;
 };
 
+/**
+ * struct dm_boot_time_crc_info - Buffer info used by boot time CRC
+ * @cpu_addr: MMIO cpu addr
+ * @bo_ptr: Pointer to the buffer object
+ * @gpu_addr: MMIO gpu addr
+ * @size: Size of the buffer
+ */
+struct dm_boot_time_crc_info {
+	void *cpu_addr;
+	struct amdgpu_bo *bo_ptr;
+	uint64_t gpu_addr;
+	uint32_t size;
+};
+
 typedef void (*dmub_notify_interrupt_callback_t)(struct amdgpu_device *adev, struct dmub_notification *notify);
 
 /**
@@ -698,6 +712,13 @@ struct amdgpu_display_manager {
 		struct completion replied;
 		char reply_data[0x40];  // Cannot include dmub_cmd here
 	} fused_io[8];
+
+	/**
+	 * @dm_boot_time_crc_info:
+	 *
+	 * Buffer info for the boot time crc.
+	 */
+	struct dm_boot_time_crc_info boot_time_crc_info;
 };
 
 enum dsc_clock_force_state {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
index d59ba82d3d7ca..7ee051cb3c056 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
@@ -37,24 +37,28 @@ const u64 amdgpu_dm_supported_degam_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_PQ_125_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_BT2020_INV_OETF) |
-	BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV);
+	BIT(DRM_COLOROP_1D_CURVE_GAMMA22);
 
 const u64 amdgpu_dm_supported_shaper_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_INV_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_PQ_125_INV_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_BT2020_OETF) |
-	BIT(DRM_COLOROP_1D_CURVE_GAMMA22);
+	BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV);
 
 const u64 amdgpu_dm_supported_blnd_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_PQ_125_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_BT2020_INV_OETF) |
-	BIT(DRM_COLOROP_1D_CURVE_GAMMA22_INV);
+	BIT(DRM_COLOROP_1D_CURVE_GAMMA22);
 
 #define MAX_COLOR_PIPELINE_OPS 10
 
 #define LUT3D_SIZE		17
 
+static const struct drm_colorop_funcs dm_colorop_funcs = {
+	.destroy = drm_colorop_destroy,
+};
+
 int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_prop_enum_list *list)
 {
 	struct drm_colorop *ops[MAX_COLOR_PIPELINE_OPS];
@@ -73,7 +77,7 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane,
+	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
 					      amdgpu_dm_supported_degam_tfs,
 					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
@@ -90,7 +94,8 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_mult_init(dev, ops[i], plane, DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = drm_plane_colorop_mult_init(dev, ops[i], plane, &dm_colorop_funcs,
+					  DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -105,7 +110,9 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane, DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane,
+					     &dm_colorop_funcs,
+					     DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -121,7 +128,7 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane,
+		ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
 						amdgpu_dm_supported_shaper_tfs,
 						DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
@@ -138,7 +145,9 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane, MAX_COLOR_LUT_ENTRIES,
+		ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane,
+							&dm_colorop_funcs,
+							MAX_COLOR_LUT_ENTRIES,
 							DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
 							DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
@@ -155,7 +164,8 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_3dlut_init(dev, ops[i], plane, LUT3D_SIZE,
+		ret = drm_plane_colorop_3dlut_init(dev, ops[i], plane,
+					&dm_colorop_funcs, LUT3D_SIZE,
 					DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL,
 					DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
@@ -173,7 +183,7 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane,
+	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
 					      amdgpu_dm_supported_blnd_tfs,
 					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
@@ -190,7 +200,8 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane, MAX_COLOR_LUT_ENTRIES,
+	ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane, &dm_colorop_funcs,
+						  MAX_COLOR_LUT_ENTRIES,
 						  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
 						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 24bc2a86904bc..2409ac72b1667 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -3305,6 +3305,52 @@ static int disallow_edp_enter_psr_set(void *data, u64 val)
 	return 0;
 }
 
+/* check if kernel disallow eDP enter replay state
+ * cat /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay
+ * 0: allow edp enter replay; 1: disallow
+ */
+static int disallow_edp_enter_replay_get(void *data, u64 *val)
+{
+	struct amdgpu_dm_connector *aconnector = data;
+
+	*val = (u64) aconnector->disallow_edp_enter_replay;
+	return 0;
+}
+
+/* set kernel disallow eDP enter replay state
+ * echo 0x0 /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay
+ * 0: allow edp enter replay; 1: disallow
+ *
+ * usage: test app read crc from PSR eDP rx.
+ *
+ * during kernel boot up, kernel write dpcd 0x37b to
+ * notify eDP rx replay enable.
+ * rx fw will start checking crc for rx internal logic.
+ * crc read count within dpcd 0x246 is not updated and
+ * value is 0. when eDP tx driver wants to read rx crc
+ * from dpcd 0x246, 0x270, read count 0 lead tx driver
+ * timeout.
+ *
+ * to avoid this, we add this debugfs to let test app to disbable
+ * rx replay. then test app can read non-zero crc read count.
+ *
+ * expected app sequence is as below:
+ * 1. disable eDP PHY and notify eDP rx with dpcd 0x600 = 2.
+ * 2. echo 0x1 /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay
+ * 3. enable eDP PHY and notify eDP rx with dpcd 0x600 = 1 but
+ *    without programming dpcd 0x37b.
+ * 4. read crc from rx dpcd 0x270, 0x246, etc.
+ * 5. echo 0x0 /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay.
+ *    this will let eDP back to normal with replay setup dpcd 0x37b.
+ */
+static int disallow_edp_enter_replay_set(void *data, u64 val)
+{
+	struct amdgpu_dm_connector *aconnector = data;
+
+	aconnector->disallow_edp_enter_replay = val ? true : false;
+	return 0;
+}
+
 static int dmub_trace_mask_set(void *data, u64 val)
 {
 	struct amdgpu_device *adev = data;
@@ -3433,6 +3479,10 @@ DEFINE_DEBUGFS_ATTRIBUTE(disallow_edp_enter_psr_fops,
 			disallow_edp_enter_psr_get,
 			disallow_edp_enter_psr_set, "%llu\n");
 
+DEFINE_DEBUGFS_ATTRIBUTE(disallow_edp_enter_replay_fops,
+			disallow_edp_enter_replay_get,
+			disallow_edp_enter_replay_set, "%llu\n");
+
 DEFINE_DEBUGFS_ATTRIBUTE(ips_residency_cntl_fops, ips_residency_cntl_get,
 			   ips_residency_cntl_set, "%llu\n");
 DEFINE_SHOW_ATTRIBUTE(current_backlight);
@@ -3635,6 +3685,8 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 					&allow_edp_hotplug_detection_fops);
 		debugfs_create_file("disallow_edp_enter_psr", 0644, dir, connector,
 					&disallow_edp_enter_psr_fops);
+		debugfs_create_file("disallow_edp_enter_replay", 0644, dir, connector,
+					&disallow_edp_enter_replay_fops);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(connector_debugfs_entries); i++) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index a09761f9882d1..f3fa8eb4bcce6 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -1016,6 +1016,9 @@ enum dc_edid_status dm_helpers_read_local_edid(
 	else
 		ddc = &aconnector->i2c->base;
 
+	if (link->dc->hwss.prepare_ddc)
+		link->dc->hwss.prepare_ddc(link);
+
 	/* some dongles read edid incorrectly the first time,
 	 * do check sum and retry to make sure read correct edid.
 	 */
@@ -1162,9 +1165,9 @@ void dm_helpers_override_panel_settings(
 		link->panel_config.dsc.disable_dsc_edp = true;
 
 	if (dc_get_edp_link_panel_inst(ctx->dc, link, &panel_inst) && panel_inst == 1) {
-			link->panel_config.psr.disable_psr = true;
-			link->panel_config.psr.disallow_psrsu = true;;
-			link->panel_config.psr.disallow_replay = true;
+		link->panel_config.psr.disable_psr = true;
+		link->panel_config.psr.disallow_psrsu = true;
+		link->panel_config.psr.disallow_replay = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index 127207e18dcb0..54ae1c371511f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 /*
- * Copyright 2022 Advanced Micro Devices, Inc.
+ * Copyright 2022-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -704,21 +704,21 @@ static void amdgpu_dm_plane_add_gfx12_modifiers(struct amdgpu_device *adev,
 	uint8_t max_comp_block[] = {2, 1, 0};
 	uint64_t max_comp_block_mod[ARRAY_SIZE(max_comp_block)] = {0};
 	uint8_t i = 0, j = 0;
-	uint64_t gfx12_modifiers[] = {mod_256k, mod_64k, mod_4k, mod_256b, DRM_FORMAT_MOD_LINEAR};
+	/* Note, linear (no DCC) gets added to the modifier list for all chips by the caller. */
+	uint64_t gfx12_modifiers[] = {mod_256k, mod_64k, mod_4k, mod_256b};
 
 	for (i = 0; i < ARRAY_SIZE(max_comp_block); i++)
 		max_comp_block_mod[i] = AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, max_comp_block[i]);
 
 	/* With DCC: Best choice should be kept first. Hence, add all 256k modifiers of different
 	 * max compressed blocks first and then move on to the next smaller sized layouts.
-	 * Do not add the linear modifier here, and hence the condition of size-1 for the loop
 	 */
-	for (j = 0; j < ARRAY_SIZE(gfx12_modifiers) - 1; j++)
+	for (j = 0; j < ARRAY_SIZE(gfx12_modifiers); j++)
 		for (i = 0; i < ARRAY_SIZE(max_comp_block); i++)
 			amdgpu_dm_plane_add_modifier(mods, size, capacity,
 						     ver | dcc | max_comp_block_mod[i] | gfx12_modifiers[j]);
 
-	/* Without DCC. Add all modifiers including linear at the end */
+	/* Without DCC. */
 	for (i = 0; i < ARRAY_SIZE(gfx12_modifiers); i++)
 		amdgpu_dm_plane_add_modifier(mods, size, capacity, gfx12_modifiers[i]);
 
@@ -759,6 +759,7 @@ static int amdgpu_dm_plane_get_plane_modifiers(struct amdgpu_device *adev, unsig
 	case AMDGPU_FAMILY_GC_11_0_0:
 	case AMDGPU_FAMILY_GC_11_0_1:
 	case AMDGPU_FAMILY_GC_11_5_0:
+	case AMDGPU_FAMILY_GC_11_5_4:
 		amdgpu_dm_plane_add_gfx11_modifiers(adev, mods, &size, &capacity);
 		break;
 	case AMDGPU_FAMILY_GC_12_0_0:
@@ -1907,7 +1908,8 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 		drm_plane_create_zpos_immutable_property(plane, 255);
 	}
 
-	if (plane->type == DRM_PLANE_TYPE_PRIMARY &&
+	if ((plane->type == DRM_PLANE_TYPE_PRIMARY ||
+	     plane->type == DRM_PLANE_TYPE_OVERLAY) &&
 	    plane_cap &&
 	    (plane_cap->pixel_format_support.nv12 ||
 	     plane_cap->pixel_format_support.p010)) {
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index 73e3c45eeeba6..f947f82013c66 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -780,28 +780,31 @@ static enum bp_result bios_parser_encoder_control(
 	return bp->cmd_tbl.dig_encoder_control(bp, cntl);
 }
 
+static enum bp_result bios_parser_external_encoder_control(
+	struct dc_bios *dcb,
+	struct bp_external_encoder_control *cntl)
+{
+	struct bios_parser *bp = BP_FROM_DCB(dcb);
+
+	if (!bp->cmd_tbl.external_encoder_control)
+		return BP_RESULT_UNSUPPORTED;
+
+	return bp->cmd_tbl.external_encoder_control(bp, cntl);
+}
+
 static enum bp_result bios_parser_dac_load_detection(
 	struct dc_bios *dcb,
-	enum engine_id engine_id,
-	enum dal_device_type device_type,
-	uint32_t enum_id)
+	enum engine_id engine_id)
 {
 	struct bios_parser *bp = BP_FROM_DCB(dcb);
 	struct dc_context *ctx = dcb->ctx;
 	struct bp_load_detection_parameters bp_params = {0};
-	enum bp_result bp_result;
+	enum bp_result bp_result = BP_RESULT_UNSUPPORTED;
 	uint32_t bios_0_scratch;
 	uint32_t device_id_mask = 0;
 
-	bp_params.engine_id = engine_id;
-	bp_params.device_id = get_support_mask_for_device_id(device_type, enum_id);
-
-	if (engine_id != ENGINE_ID_DACA &&
-	    engine_id != ENGINE_ID_DACB)
-		return BP_RESULT_UNSUPPORTED;
-
-	if (!bp->cmd_tbl.dac_load_detection)
-		return BP_RESULT_UNSUPPORTED;
+	bp_params.device_id = get_support_mask_for_device_id(
+		DEVICE_TYPE_CRT, engine_id == ENGINE_ID_DACB ? 2 : 1);
 
 	if (bp_params.device_id == ATOM_DEVICE_CRT1_SUPPORT)
 		device_id_mask = ATOM_S0_CRT1_MASK;
@@ -815,7 +818,13 @@ static enum bp_result bios_parser_dac_load_detection(
 	bios_0_scratch &= ~device_id_mask;
 	dm_write_reg(ctx, bp->base.regs->BIOS_SCRATCH_0, bios_0_scratch);
 
-	bp_result = bp->cmd_tbl.dac_load_detection(bp, &bp_params);
+	if (engine_id == ENGINE_ID_DACA || engine_id == ENGINE_ID_DACB) {
+		if (!bp->cmd_tbl.dac_load_detection)
+			return BP_RESULT_UNSUPPORTED;
+
+		bp_params.engine_id = engine_id;
+		bp_result = bp->cmd_tbl.dac_load_detection(bp, &bp_params);
+	}
 
 	if (bp_result != BP_RESULT_OK)
 		return bp_result;
@@ -2912,6 +2921,8 @@ static const struct dc_vbios_funcs vbios_funcs = {
 
 	.encoder_control = bios_parser_encoder_control,
 
+	.external_encoder_control = bios_parser_external_encoder_control,
+
 	.dac_load_detection = bios_parser_dac_load_detection,
 
 	.transmitter_control = bios_parser_transmitter_control,
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index b692fa37402d9..f6e22dcecf29e 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -226,6 +226,28 @@ static enum bp_result encoder_control_dig2_v1(
 	return result;
 }
 
+static uint8_t dc_color_depth_to_atom(enum dc_color_depth color_depth)
+{
+	switch (color_depth) {
+	case COLOR_DEPTH_UNDEFINED:
+		return PANEL_BPC_UNDEFINE;
+	case COLOR_DEPTH_666:
+		return PANEL_6BIT_PER_COLOR;
+	default:
+	case COLOR_DEPTH_888:
+		return PANEL_8BIT_PER_COLOR;
+	case COLOR_DEPTH_101010:
+		return PANEL_10BIT_PER_COLOR;
+	case COLOR_DEPTH_121212:
+		return PANEL_12BIT_PER_COLOR;
+	case COLOR_DEPTH_141414:
+		dm_error("14-bit color not supported by ATOMBIOS\n");
+		return PANEL_BPC_UNDEFINE;
+	case COLOR_DEPTH_161616:
+		return PANEL_16BIT_PER_COLOR;
+	}
+}
+
 static enum bp_result encoder_control_digx_v3(
 	struct bios_parser *bp,
 	struct bp_encoder_control *cntl)
@@ -248,23 +270,7 @@ static enum bp_result encoder_control_digx_v3(
 					cntl->signal,
 					cntl->enable_dp_audio);
 	params.ucLaneNum = (uint8_t)(cntl->lanes_number);
-
-	switch (cntl->color_depth) {
-	case COLOR_DEPTH_888:
-		params.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_101010:
-		params.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_121212:
-		params.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_161616:
-		params.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-		break;
-	default:
-		break;
-	}
+	params.ucBitPerColor = dc_color_depth_to_atom(cntl->color_depth);
 
 	if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params))
 		result = BP_RESULT_OK;
@@ -294,23 +300,7 @@ static enum bp_result encoder_control_digx_v4(
 					cntl->signal,
 					cntl->enable_dp_audio));
 	params.ucLaneNum = (uint8_t)(cntl->lanes_number);
-
-	switch (cntl->color_depth) {
-	case COLOR_DEPTH_888:
-		params.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_101010:
-		params.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_121212:
-		params.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_161616:
-		params.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-		break;
-	default:
-		break;
-	}
+	params.ucBitPerColor = dc_color_depth_to_atom(cntl->color_depth);
 
 	if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params))
 		result = BP_RESULT_OK;
@@ -334,23 +324,7 @@ static enum bp_result encoder_control_digx_v5(
 					cntl->signal,
 					cntl->enable_dp_audio));
 	params.ucLaneNum = (uint8_t)(cntl->lanes_number);
-
-	switch (cntl->color_depth) {
-	case COLOR_DEPTH_888:
-		params.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_101010:
-		params.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_121212:
-		params.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_161616:
-		params.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-		break;
-	default:
-		break;
-	}
+	params.ucBitPerColor = dc_color_depth_to_atom(cntl->color_depth);
 
 	if (cntl->signal == SIGNAL_TYPE_HDMI_TYPE_A)
 		switch (cntl->color_depth) {
@@ -1797,30 +1771,7 @@ static enum bp_result select_crtc_source_v3(
 		&params.ucEncodeMode))
 		return BP_RESULT_BADINPUT;
 
-	switch (bp_params->color_depth) {
-	case COLOR_DEPTH_UNDEFINED:
-		params.ucDstBpc = PANEL_BPC_UNDEFINE;
-		break;
-	case COLOR_DEPTH_666:
-		params.ucDstBpc = PANEL_6BIT_PER_COLOR;
-		break;
-	default:
-	case COLOR_DEPTH_888:
-		params.ucDstBpc = PANEL_8BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_101010:
-		params.ucDstBpc = PANEL_10BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_121212:
-		params.ucDstBpc = PANEL_12BIT_PER_COLOR;
-		break;
-	case COLOR_DEPTH_141414:
-		dm_error("14-bit color not supported by SelectCRTC_Source v3\n");
-		break;
-	case COLOR_DEPTH_161616:
-		params.ucDstBpc = PANEL_16BIT_PER_COLOR;
-		break;
-	}
+	params.ucDstBpc = dc_color_depth_to_atom(bp_params->color_depth);
 
 	if (EXEC_BIOS_CMD_TABLE(SelectCRTC_Source, params))
 		result = BP_RESULT_OK;
@@ -2569,6 +2520,7 @@ static enum bp_result external_encoder_control_v3(
 				cpu_to_le16((uint16_t)cntl->connector_obj_id.id);
 		break;
 	case EXTERNAL_ENCODER_CONTROL_SETUP:
+	case EXTERNAL_ENCODER_CONTROL_ENABLE:
 		/* EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3 pixel clock unit in
 		 * 10KHz
 		 * output display device pixel clock frequency in unit of 10KHz.
@@ -2585,26 +2537,24 @@ static enum bp_result external_encoder_control_v3(
 		if (is_input_signal_dp) {
 			/* Bit[0]: indicate link rate, =1: 2.7Ghz, =0: 1.62Ghz,
 			 * only valid in encoder setup with DP mode. */
-			if (LINK_RATE_HIGH == cntl->link_rate)
-				cntl_params->ucConfig |= 1;
+			if (cntl->link_rate == LINK_RATE_LOW)
+				cntl_params->ucConfig |=
+					EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_1_62GHZ;
+			else if (cntl->link_rate == LINK_RATE_HIGH)
+				cntl_params->ucConfig |=
+					EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
+			else
+				dm_error("Link rate not supported by external encoder");
+
 			/* output color depth Indicate encoder data bpc format
 			 * in DP mode, only valid in encoder setup in DP mode.
 			 */
-			cntl_params->ucBitPerColor =
-					(uint8_t)(cntl->color_depth);
+			cntl_params->ucBitPerColor = dc_color_depth_to_atom(cntl->color_depth);
 		}
 		/* Indicate how many lanes used by external encoder, only valid
 		 * in encoder setup and enableoutput. */
 		cntl_params->ucLaneNum = (uint8_t)(cntl->lanes_number);
 		break;
-	case EXTERNAL_ENCODER_CONTROL_ENABLE:
-		cntl_params->usPixelClock =
-				cpu_to_le16((uint16_t)(cntl->pixel_clock / 10));
-		cntl_params->ucEncoderMode =
-				(uint8_t)bp->cmd_helper->encoder_mode_bp_to_atom(
-						cntl->signal, false);
-		cntl_params->ucLaneNum = (uint8_t)cntl->lanes_number;
-		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
index 268e2414b34f0..a93a8860535a1 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
@@ -84,6 +84,7 @@ bool dal_bios_parser_init_cmd_tbl_helper2(
 	case DCN_VERSION_3_51:
 	case DCN_VERSION_3_6:
 	case DCN_VERSION_4_01:
+	case DCN_VERSION_4_2:
 		*h = dal_cmd_tbl_helper_dce112_get_table2();
 		return true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
index 60021671b386a..98911ef56c2ed 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2017 Advanced Micro Devices, Inc.
+# Copyright 2017-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -29,18 +29,6 @@ AMD_DAL_CLK_MGR = $(addprefix $(AMDDALPATH)/dc/clk_mgr/,$(CLK_MGR))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR)
 
-
-ifdef CONFIG_DRM_AMD_DC_SI
-###############################################################################
-# DCE 60
-###############################################################################
-CLK_MGR_DCE60 = dce60_clk_mgr.o
-
-AMD_DAL_CLK_MGR_DCE60 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce60/,$(CLK_MGR_DCE60))
-
-AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE60)
-endif
-
 ###############################################################################
 # DCE 100 and DCE8x
 ###############################################################################
@@ -188,4 +176,13 @@ CLK_MGR_DCN401 = dcn401_clk_mgr.o dcn401_clk_mgr_smu_msg.o
 AMD_DAL_CLK_MGR_DCN401 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn401/,$(CLK_MGR_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN401)
+
+###############################################################################
+# DCN42
+###############################################################################
+CLK_MGR_DCN42 = dcn42_smu.o dcn42_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCN42 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn42/,$(CLK_MGR_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
index 08d0e05a313ea..5a5249f3ffbda 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
@@ -34,7 +34,6 @@
 #include "dce110/dce110_clk_mgr.h"
 #include "dce112/dce112_clk_mgr.h"
 #include "dce120/dce120_clk_mgr.h"
-#include "dce60/dce60_clk_mgr.h"
 #include "dcn10/rv1_clk_mgr.h"
 #include "dcn10/rv2_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
@@ -49,6 +48,7 @@
 #include "dcn32/dcn32_clk_mgr.h"
 #include "dcn35/dcn35_clk_mgr.h"
 #include "dcn401/dcn401_clk_mgr.h"
+#include "dcn42/dcn42_clk_mgr.h"
 
 int clk_mgr_helper_get_active_display_cnt(
 		struct dc *dc,
@@ -149,18 +149,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 	struct hw_asic_id asic_id = ctx->asic_id;
 
 	switch (asic_id.chip_family) {
-#if defined(CONFIG_DRM_AMD_DC_SI)
-	case FAMILY_SI: {
-		struct clk_mgr_internal *clk_mgr = kzalloc_obj(*clk_mgr);
-
-		if (clk_mgr == NULL) {
-			BREAK_TO_DEBUGGER();
-			return NULL;
-		}
-		dce60_clk_mgr_construct(ctx, clk_mgr);
-		return &clk_mgr->base;
-	}
-#endif
+	case FAMILY_SI:
 	case FAMILY_CI:
 	case FAMILY_KV: {
 		struct clk_mgr_internal *clk_mgr = kzalloc_obj(*clk_mgr);
@@ -255,6 +244,10 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 			BREAK_TO_DEBUGGER();
 			return NULL;
 		}
+		if (ctx->dce_version == DCN_VERSION_2_01) {
+			dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
+			return &clk_mgr->base;
+		}
 		if (ASICREV_IS_SIENNA_CICHLID_P(asic_id.hw_internal_rev)) {
 			dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
 			return &clk_mgr->base;
@@ -267,10 +260,6 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 			dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
 			return &clk_mgr->base;
 		}
-		if (ctx->dce_version == DCN_VERSION_2_01) {
-			dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
-			return &clk_mgr->base;
-		}
 		dcn20_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
 		return &clk_mgr->base;
 	}
@@ -374,6 +363,18 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 		return &clk_mgr->base;
 	}
 	break;
+	case AMDGPU_FAMILY_GC_11_5_4: {
+		struct clk_mgr_dcn42 *clk_mgr = kzalloc(sizeof(*clk_mgr), GFP_KERNEL);
+
+		if (clk_mgr == NULL) {
+			BREAK_TO_DEBUGGER();
+			return NULL;
+		}
+
+		dcn42_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
+		return &clk_mgr->base.base;
+	}
+	break;
 #endif	/* CONFIG_DRM_AMD_DC_FP */
 	default:
 		ASSERT(0); /* Unknown Asic */
@@ -431,6 +432,9 @@ void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
 	case AMDGPU_FAMILY_GC_12_0_0:
 		dcn401_clk_mgr_destroy(clk_mgr);
 		break;
+	case AMDGPU_FAMILY_GC_11_5_4:
+		dcn42_clk_mgr_destroy(clk_mgr);
+		break;
 
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
index 6131ede2db7a2..6d41df52d7c9b 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
@@ -62,6 +62,18 @@ static const struct clk_mgr_mask disp_clk_mask = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
 };
 
+/* Max clock values for each state indexed by "enum clocks_state": */
+static const struct state_dependent_clocks dce60_max_clks_by_state[] = {
+/* ClocksStateInvalid - should not be used */
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/* ClocksStateUltraLow - not expected to be used for DCE 6.0 */
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/* ClocksStateLow */
+{ .display_clk_khz = 352000, .pixel_clk_khz = 330000},
+/* ClocksStateNominal */
+{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 },
+/* ClocksStatePerformance */
+{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 } };
 
 /* Max clock values for each state indexed by "enum clocks_state": */
 static const struct state_dependent_clocks dce80_max_clks_by_state[] = {
@@ -126,6 +138,20 @@ int dce_adjust_dp_ref_freq_for_ss(struct clk_mgr_internal *clk_mgr_dce, int dp_r
 	return dp_ref_clk_khz;
 }
 
+static int dce60_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct dc_context *ctx = clk_mgr_base->ctx;
+	int dp_ref_clk_khz = 0;
+
+	if (ASIC_REV_IS_TAHITI_P(ctx->asic_id.hw_internal_rev))
+		dp_ref_clk_khz = ctx->dc_bios->fw_info.default_display_engine_pll_frequency;
+	else
+		dp_ref_clk_khz = clk_mgr_base->clks.dispclk_khz;
+
+	return dce_adjust_dp_ref_freq_for_ss(clk_mgr, dp_ref_clk_khz);
+}
+
 int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
@@ -134,6 +160,9 @@ int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 	int dp_ref_clk_khz;
 	int target_div;
 
+	if (clk_mgr_base->ctx->dce_version <= DCE_VERSION_6_4)
+		return dce60_get_dp_ref_freq_khz(clk_mgr_base);
+
 	/* ASSERT DP Reference Clock source is from DFS*/
 	REG_GET(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, &dprefclk_src_sel);
 	ASSERT(dprefclk_src_sel == 0);
@@ -441,27 +470,37 @@ void dce_clk_mgr_construct(
 	struct clk_mgr *base = &clk_mgr->base;
 	struct dm_pp_static_clock_info static_clk_info = {0};
 
-	memcpy(clk_mgr->max_clks_by_state,
-		dce80_max_clks_by_state,
-		sizeof(dce80_max_clks_by_state));
+	if (ctx->dce_version <= DCE_VERSION_6_4)
+		memcpy(clk_mgr->max_clks_by_state,
+			dce60_max_clks_by_state,
+			sizeof(dce60_max_clks_by_state));
+	else
+		memcpy(clk_mgr->max_clks_by_state,
+			dce80_max_clks_by_state,
+			sizeof(dce80_max_clks_by_state));
+
 
 	base->ctx = ctx;
 	base->funcs = &dce_funcs;
 
-	clk_mgr->regs = &disp_clk_regs;
-	clk_mgr->clk_mgr_shift = &disp_clk_shift;
-	clk_mgr->clk_mgr_mask = &disp_clk_mask;
-	clk_mgr->dfs_bypass_disp_clk = 0;
+	if (ctx->dce_version >= DCE_VERSION_8_0) {
+		clk_mgr->regs = &disp_clk_regs;
+		clk_mgr->clk_mgr_shift = &disp_clk_shift;
+		clk_mgr->clk_mgr_mask = &disp_clk_mask;
+	}
 
+	clk_mgr->dfs_bypass_disp_clk = 0;
 	clk_mgr->dprefclk_ss_percentage = 0;
 	clk_mgr->dprefclk_ss_divider = 1000;
 	clk_mgr->ss_on_dprefclk = false;
 
-	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		clk_mgr->max_clks_state = static_clk_info.max_clocks_state;
-	else
-		clk_mgr->max_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
-	clk_mgr->cur_min_clks_state = DM_PP_CLOCKS_STATE_INVALID;
+	if (ctx->dce_version >= DCE_VERSION_8_0) {
+		if (dm_pp_get_static_clocks(ctx, &static_clk_info))
+			clk_mgr->max_clks_state = static_clk_info.max_clocks_state;
+		else
+			clk_mgr->max_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
+		clk_mgr->cur_min_clks_state = DM_PP_CLOCKS_STATE_INVALID;
+	}
 
 	base->clks.max_supported_dispclk_khz =
 		clk_mgr->max_clks_by_state[DM_PP_CLOCKS_STATE_PERFORMANCE].display_clk_khz;
@@ -469,4 +508,3 @@ void dce_clk_mgr_construct(
 	dce_clock_read_integrated_info(clk_mgr);
 	dce_clock_read_ss_info(clk_mgr);
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c
deleted file mode 100644
index 69dd80d9f7388..0000000000000
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.c
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dccg.h"
-#include "clk_mgr_internal.h"
-#include "dce100/dce_clk_mgr.h"
-#include "dce110/dce110_clk_mgr.h"
-#include "dce60_clk_mgr.h"
-#include "reg_helper.h"
-#include "dmcu.h"
-#include "core_types.h"
-#include "dal_asic_id.h"
-
-/*
- * Currently the register shifts and masks in this file are used for dce60
- * which has no DPREFCLK_CNTL register
- * TODO: remove this when DENTIST_DISPCLK_CNTL
- * is moved to dccg, where it belongs
- */
-#include "dce/dce_6_0_d.h"
-#include "dce/dce_6_0_sh_mask.h"
-
-#define REG(reg) \
-	(clk_mgr->regs->reg)
-
-#undef FN
-#define FN(reg_name, field_name) \
-	clk_mgr->clk_mgr_shift->field_name, clk_mgr->clk_mgr_mask->field_name
-
-/* set register offset */
-#define SR(reg_name)\
-	.reg_name = mm ## reg_name
-
-static const struct clk_mgr_registers disp_clk_regs = {
-		CLK_COMMON_REG_LIST_DCE60_BASE()
-};
-
-static const struct clk_mgr_shift disp_clk_shift = {
-		CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(__SHIFT)
-};
-
-static const struct clk_mgr_mask disp_clk_mask = {
-		CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(_MASK)
-};
-
-
-/* Max clock values for each state indexed by "enum clocks_state": */
-static const struct state_dependent_clocks dce60_max_clks_by_state[] = {
-/* ClocksStateInvalid - should not be used */
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/* ClocksStateUltraLow - not expected to be used for DCE 6.0 */
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/* ClocksStateLow */
-{ .display_clk_khz = 352000, .pixel_clk_khz = 330000},
-/* ClocksStateNominal */
-{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 },
-/* ClocksStatePerformance */
-{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 } };
-
-static int dce60_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	struct dc_context *ctx = clk_mgr_base->ctx;
-	int dp_ref_clk_khz = 0;
-
-	if (ASIC_REV_IS_TAHITI_P(ctx->asic_id.hw_internal_rev))
-		dp_ref_clk_khz = ctx->dc_bios->fw_info.default_display_engine_pll_frequency;
-	else
-		dp_ref_clk_khz = clk_mgr_base->clks.dispclk_khz;
-
-	return dce_adjust_dp_ref_freq_for_ss(clk_mgr, dp_ref_clk_khz);
-}
-
-static void dce60_pplib_apply_display_requirements(
-	struct dc *dc,
-	struct dc_state *context)
-{
-	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
-
-	dce110_fill_display_configs(context, pp_display_cfg);
-
-	if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) !=  0)
-		dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
-}
-
-static void dce60_update_clocks(struct clk_mgr *clk_mgr_base,
-			struct dc_state *context,
-			bool safe_to_lower)
-{
-	struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	struct dm_pp_power_level_change_request level_change_req;
-	const int max_disp_clk =
-		clk_mgr_dce->max_clks_by_state[DM_PP_CLOCKS_STATE_PERFORMANCE].display_clk_khz;
-	int patched_disp_clk = MIN(max_disp_clk, context->bw_ctx.bw.dce.dispclk_khz);
-
-	level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context);
-	/* get max clock state from PPLIB */
-	if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
-			|| level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
-		if (dm_pp_apply_power_level_change_request(clk_mgr_base->ctx, &level_change_req))
-			clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
-	}
-
-	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) {
-		patched_disp_clk = dce_set_clock(clk_mgr_base, patched_disp_clk);
-		clk_mgr_base->clks.dispclk_khz = patched_disp_clk;
-	}
-	dce60_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context);
-}
-
-
-
-
-
-
-
-
-static struct clk_mgr_funcs dce60_funcs = {
-	.get_dp_ref_clk_frequency = dce60_get_dp_ref_freq_khz,
-	.update_clocks = dce60_update_clocks
-};
-
-void dce60_clk_mgr_construct(
-		struct dc_context *ctx,
-		struct clk_mgr_internal *clk_mgr)
-{
-	struct clk_mgr *base = &clk_mgr->base;
-
-	dce_clk_mgr_construct(ctx, clk_mgr);
-
-	memcpy(clk_mgr->max_clks_by_state,
-		dce60_max_clks_by_state,
-		sizeof(dce60_max_clks_by_state));
-
-	clk_mgr->regs = &disp_clk_regs;
-	clk_mgr->clk_mgr_shift = &disp_clk_shift;
-	clk_mgr->clk_mgr_mask = &disp_clk_mask;
-	clk_mgr->base.funcs = &dce60_funcs;
-
-	base->clks.max_supported_dispclk_khz =
-		clk_mgr->max_clks_by_state[DM_PP_CLOCKS_STATE_PERFORMANCE].display_clk_khz;
-}
-
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h
deleted file mode 100644
index eca3e5168089b..0000000000000
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce60/dce60_clk_mgr.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2020 Mauro Rossi <issor.oruam@gmail.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#ifndef DAL_DC_DCE_DCE60_CLK_MGR_H_
-#define DAL_DC_DCE_DCE60_CLK_MGR_H_
-
-#include "dc.h"
-
-void dce60_clk_mgr_construct(
-		struct dc_context *ctx,
-		struct clk_mgr_internal *clk_mgr_dce);
-
-#endif /* DAL_DC_DCE_DCE60_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
index b0aba3a6f13c2..b48522480dfd9 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
@@ -547,6 +547,7 @@ void dcn3_clk_mgr_construct(
 	/* in case we don't get a value from the register, use default */
 	if (clk_mgr->base.dentist_vco_freq_khz == 0)
 		clk_mgr->base.dentist_vco_freq_khz = 3650000;
+
 	/* Convert dprefclk units from MHz to KHz */
 	/* Value already divided by 10, some resolution lost */
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
index 604d256cb47a6..9d8f81c3d3f05 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
@@ -204,7 +204,7 @@ int dcn35_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispcl
 			khz_to_mhz_ceil(requested_dispclk_khz));
 
 	smu_print("requested_dispclk_khz = %d, actual_dispclk_set_mhz: %d\n", requested_dispclk_khz, actual_dispclk_set_mhz);
-	return actual_dispclk_set_mhz * 1000;
+	return (int)((long long)actual_dispclk_set_mhz * 1000);
 }
 
 int dcn35_smu_set_dprefclk(struct clk_mgr_internal *clk_mgr)
@@ -221,7 +221,7 @@ int dcn35_smu_set_dprefclk(struct clk_mgr_internal *clk_mgr)
 
 	/* TODO: add code for programing DP DTO, currently this is down by command table */
 
-	return actual_dprefclk_set_mhz * 1000;
+	return (int)((long long)actual_dprefclk_set_mhz * 1000);
 }
 
 int dcn35_smu_set_hard_min_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_dcfclk_khz)
@@ -238,7 +238,7 @@ int dcn35_smu_set_hard_min_dcfclk(struct clk_mgr_internal *clk_mgr, int requeste
 
 	smu_print("requested_dcfclk_khz = %d, actual_dcfclk_set_mhz: %d\n", requested_dcfclk_khz, actual_dcfclk_set_mhz);
 
-	return actual_dcfclk_set_mhz * 1000;
+	return (int)((long long)actual_dcfclk_set_mhz * 1000);
 }
 
 int dcn35_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_min_ds_dcfclk_khz)
@@ -255,7 +255,7 @@ int dcn35_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, int re
 
 	smu_print("requested_min_ds_dcfclk_khz = %d, actual_min_ds_dcfclk_mhz: %d\n", requested_min_ds_dcfclk_khz, actual_min_ds_dcfclk_mhz);
 
-	return actual_min_ds_dcfclk_mhz * 1000;
+	return (int)((long long)actual_min_ds_dcfclk_mhz * 1000);
 }
 
 int dcn35_smu_set_dppclk(struct clk_mgr_internal *clk_mgr, int requested_dpp_khz)
@@ -272,7 +272,7 @@ int dcn35_smu_set_dppclk(struct clk_mgr_internal *clk_mgr, int requested_dpp_khz
 
 	smu_print("requested_dpp_khz = %d, actual_dppclk_set_mhz: %d\n", requested_dpp_khz, actual_dppclk_set_mhz);
 
-	return actual_dppclk_set_mhz * 1000;
+	return (int)((long long)actual_dppclk_set_mhz * 1000);
 }
 
 void dcn35_smu_set_display_idle_optimization(struct clk_mgr_internal *clk_mgr, uint32_t idle_info)
@@ -424,7 +424,7 @@ int dcn35_smu_get_dprefclk(struct clk_mgr_internal *clk_mgr)
 						 0);
 
 	smu_print("%s:  SMU DPREF clk  = %d mhz\n",  __func__, dprefclk);
-	return dprefclk * 1000;
+	return (int)((long long)dprefclk * 1000);
 }
 
 int dcn35_smu_get_dtbclk(struct clk_mgr_internal *clk_mgr)
@@ -439,7 +439,7 @@ int dcn35_smu_get_dtbclk(struct clk_mgr_internal *clk_mgr)
 					       0);
 
 	smu_print("%s: get_dtbclk  = %dmhz\n", __func__, dtbclk);
-	return dtbclk * 1000;
+	return (int)((long long)dtbclk * 1000);
 }
 /* Arg = 1: Turn DTB on; 0: Turn DTB CLK OFF. when it is on, it is 600MHZ */
 void dcn35_smu_set_dtbclk(struct clk_mgr_internal *clk_mgr, bool enable)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
new file mode 100644
index 0000000000000..97c9f0ce13e40
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
@@ -0,0 +1,1165 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dcn42_clk_mgr.h"
+
+#include "dccg.h"
+#include "clk_mgr_internal.h"
+
+// For dce12_get_dp_ref_freq_khz
+#include "dce100/dce_clk_mgr.h"
+
+// For dcn20_update_clocks_update_dpp_dto
+#include "dcn20/dcn20_clk_mgr.h"
+
+
+
+
+#include "reg_helper.h"
+#include "core_types.h"
+#include "dcn42_smu.h"
+#include "dm_helpers.h"
+
+/* TODO: remove this include once we ported over remaining clk mgr functions*/
+#include "dcn30/dcn30_clk_mgr.h"
+#include "dcn31/dcn31_clk_mgr.h"
+
+#include "dcn35/dcn35_clk_mgr.h"
+
+#include "dc_dmub_srv.h"
+#include "link_service.h"
+#include "logger_types.h"
+
+#include "clk/clk_15_0_0_offset.h"
+#include "clk/clk_15_0_0_sh_mask.h"
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+
+
+#undef DC_LOGGER
+#define DC_LOGGER \
+	dc_logger
+#define DC_LOGGER_INIT(logger) \
+	struct dal_logger *dc_logger = logger
+
+#define DCN42_CLKIP_REFCLK 48000
+
+#undef FN
+#define FN(reg_name, field_name) \
+	clk_mgr->clk_mgr_shift->field_name, clk_mgr->clk_mgr_mask->field_name
+
+#define REG(reg) \
+	(clk_mgr->regs->reg)
+
+// for DCN register access
+#define DCN_BASE__INST0_SEG0 0x00000012
+#define DCN_BASE__INST0_SEG1 0x000000C0
+
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name) \
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + reg ## reg_name
+
+// for CLKIP register access
+#define CLK_BASE__INST0_SEG0 0x00016C00
+
+#define CLK_BASE_INNER(seg) \
+	CLK_BASE__INST0_SEG ## seg
+
+#define CLK_SR_DCN42(reg_name) \
+	.reg_name = CLK_BASE(reg ## reg_name ## _BASE_IDX) + reg ## reg_name
+
+static const struct clk_mgr_registers clk_mgr_regs_dcn42 = {
+	CLK_REG_LIST_DCN42()
+};
+
+static const struct clk_mgr_shift clk_mgr_shift_dcn42 = {
+	CLK_COMMON_MASK_SH_LIST_DCN42(__SHIFT)
+};
+
+static const struct clk_mgr_mask clk_mgr_mask_dcn42 = {
+	CLK_COMMON_MASK_SH_LIST_DCN42(_MASK)
+};
+
+#define TO_CLK_MGR_DCN42(clk_mgr_int)\
+	container_of(clk_mgr_int, struct clk_mgr_dcn42, base)
+
+bool dcn42_has_active_display(struct dc *dc, const struct dc_state *context)
+{
+	int i, active_count = 0;
+
+	for (i = 0; i < context->stream_count; i++) {
+		const struct dc_stream_state *stream = context->streams[i];
+
+		/* Checking stream / link detection ensuring that PHY is active*/
+		if (dc_is_hdmi_signal(stream->signal) ||
+		    dc_is_dvi_signal(stream->signal) ||
+		    (dc_is_dp_signal(stream->signal) && !stream->dpms_off))
+			active_count++;
+	}
+
+	for (i = 0; i < dc->link_count; i++) {
+		const struct dc_link *link = dc->links[i];
+
+		/* abusing the fact that the dig and phy are coupled to see if the phy is enabled */
+		if (link->link_enc && link->link_enc->funcs->is_dig_enabled &&
+				link->link_enc->funcs->is_dig_enabled(link->link_enc))
+			active_count++;
+	}
+
+	return active_count > 0;
+}
+
+static uint32_t dcn42_get_clock_freq_from_clkip(struct clk_mgr *clk_mgr_base, enum clock_type clock)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	uint64_t clock_freq_mhz = 0;
+	uint32_t timer_threshold = 0;
+
+	// always safer to read the timer threshold instead of using cached value
+	REG_GET(CLK8_CLK_TICK_CNT_CONFIG_REG, TIMER_THRESHOLD, &timer_threshold);
+
+	if (timer_threshold == 0) {
+		BREAK_TO_DEBUGGER();
+		return 0;
+	}
+
+	switch (clock) {
+	case clock_type_dispclk:
+		clock_freq_mhz = REG_READ(CLK8_CLK0_CURRENT_CNT);
+		break;
+	case clock_type_dppclk:
+		clock_freq_mhz = REG_READ(CLK8_CLK1_CURRENT_CNT);
+		break;
+	case clock_type_dprefclk:
+		clock_freq_mhz = REG_READ(CLK8_CLK2_CURRENT_CNT);
+		break;
+	case clock_type_dcfclk:
+		clock_freq_mhz = REG_READ(CLK8_CLK3_CURRENT_CNT);
+		break;
+	case clock_type_dtbclk:
+		clock_freq_mhz = REG_READ(CLK8_CLK4_CURRENT_CNT);
+		break;
+	default:
+		break;
+	}
+
+	clock_freq_mhz *= DCN42_CLKIP_REFCLK;
+	clock_freq_mhz = div_u64(clock_freq_mhz, timer_threshold);
+
+	// there are no DCN clocks over 0xFFFFFFFF MHz
+	ASSERT(clock_freq_mhz <= 0xFFFFFFFF);
+
+	return (uint32_t)clock_freq_mhz;
+}
+
+void dcn42_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
+		struct dc_state *context,
+		int ref_dtbclk_khz)
+{
+	/* DCN42 does not implement set_dtbclk_dto function, so this is a no-op */
+}
+
+void dcn42_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
+		struct dc_state *context, bool safe_to_lower)
+{
+	int i;
+	bool dppclk_active[MAX_PIPES] = {0};
+
+
+	clk_mgr->dccg->ref_dppclk = clk_mgr->base.clks.dppclk_khz;
+	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
+		int dpp_inst = 0, dppclk_khz, prev_dppclk_khz;
+
+		dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;
+
+		if (context->res_ctx.pipe_ctx[i].plane_res.dpp)
+			dpp_inst = context->res_ctx.pipe_ctx[i].plane_res.dpp->inst;
+		else if (!context->res_ctx.pipe_ctx[i].plane_res.dpp && dppclk_khz == 0) {
+			/* dpp == NULL && dppclk_khz == 0 is valid because of pipe harvesting.
+			 * In this case just continue in loop
+			 */
+			continue;
+		} else if (!context->res_ctx.pipe_ctx[i].plane_res.dpp && dppclk_khz > 0) {
+			/* The software state is not valid if dpp resource is NULL and
+			 * dppclk_khz > 0.
+			 */
+			ASSERT(false);
+			continue;
+		}
+
+		prev_dppclk_khz = clk_mgr->dccg->pipe_dppclk_khz[i];
+
+		if (safe_to_lower || prev_dppclk_khz < dppclk_khz)
+			clk_mgr->dccg->funcs->update_dpp_dto(
+							clk_mgr->dccg, dpp_inst, dppclk_khz);
+		dppclk_active[dpp_inst] = true;
+	}
+	if (safe_to_lower)
+		for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
+			struct dpp *old_dpp = clk_mgr->base.ctx->dc->current_state->res_ctx.pipe_ctx[i].plane_res.dpp;
+
+			if (old_dpp && !dppclk_active[old_dpp->inst])
+				clk_mgr->dccg->funcs->update_dpp_dto(clk_mgr->dccg, old_dpp->inst, 0);
+		}
+}
+
+void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
+			struct dc_state *context,
+			bool safe_to_lower)
+{
+	union dmub_rb_cmd cmd;
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
+	struct dc *dc = clk_mgr_base->ctx->dc;
+	bool update_dppclk = false;
+	bool update_dispclk = false;
+	bool dpp_clock_lowered = false;
+	bool has_active_display;
+
+	if (dc->work_arounds.skip_clock_update)
+		return;
+
+	has_active_display = dcn42_has_active_display(dc, context);
+
+	if (new_clocks->dtbclk_en && new_clocks->ref_dtbclk_khz < 590000)
+		new_clocks->ref_dtbclk_khz = 600000;
+	/*
+	 * if it is safe to lower, but we are already in the lower state, we don't have to do anything
+	 * also if safe to lower is false, we just go in the higher state
+	 */
+	if (safe_to_lower) {
+		if (new_clocks->zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW &&
+				new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) {
+			dcn42_smu_set_zstate_support(clk_mgr, new_clocks->zstate_support);
+			clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
+		}
+
+		if (clk_mgr_base->clks.dtbclk_en && !new_clocks->dtbclk_en) {
+			if (clk_mgr->base.ctx->dc->config.allow_0_dtb_clk)
+				dcn42_smu_set_dtbclk(clk_mgr, false);
+			clk_mgr_base->clks.dtbclk_en = new_clocks->dtbclk_en;
+		}
+		/* check that we're not already in lower */
+		if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) {
+			/* if we can go lower, go lower */
+			if (has_active_display == false)
+				clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_LOW_POWER;
+		}
+	} else {
+		if (new_clocks->zstate_support == DCN_ZSTATE_SUPPORT_DISALLOW &&
+				new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) {
+			dcn42_smu_set_zstate_support(clk_mgr, DCN_ZSTATE_SUPPORT_DISALLOW);
+			clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
+		}
+		if (!clk_mgr_base->clks.dtbclk_en && new_clocks->dtbclk_en) {
+			int actual_dtbclk = 0;
+
+			dcn42_update_clocks_update_dtb_dto(clk_mgr, context, new_clocks->ref_dtbclk_khz);
+			dcn42_smu_set_dtbclk(clk_mgr, true);
+			actual_dtbclk = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dtbclk);
+
+			if (actual_dtbclk > 590000) {
+				clk_mgr_base->clks.ref_dtbclk_khz = new_clocks->ref_dtbclk_khz;
+				clk_mgr_base->clks.dtbclk_en = new_clocks->dtbclk_en;
+			}
+		}
+
+		/* check that we're not already in D0 */
+		if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_MISSION_MODE) {
+			union display_idle_optimization_u idle_info = { 0 };
+
+			dcn42_smu_set_display_idle_optimization(clk_mgr, idle_info.data);
+			/* update power state */
+			clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_MISSION_MODE;
+		}
+	}
+	if (dc->debug.force_min_dcfclk_mhz > 0)
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
+				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+
+	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
+		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+		clk_mgr_base->clks.fclk_khz = new_clocks->fclk_khz;
+		clk_mgr_base->clks.dramclk_khz = new_clocks->dramclk_khz;
+		dcn42_smu_set_hard_min_dcfclk(clk_mgr, clk_mgr_base->clks.dcfclk_khz);
+	}
+
+	if (should_set_clock(safe_to_lower,
+			new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz)) {
+		clk_mgr_base->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+		dcn42_smu_set_min_deep_sleep_dcfclk(clk_mgr, clk_mgr_base->clks.dcfclk_deep_sleep_khz);
+	}
+
+	// workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow.
+
+	if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) {
+		if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz)
+			dpp_clock_lowered = true;
+		clk_mgr_base->clks.dppclk_khz = new_clocks->dppclk_khz;
+		update_dppclk = true;
+	}
+
+	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
+	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && has_active_display == false))) {
+		int requested_dispclk_khz = new_clocks->dispclk_khz;
+
+		dcn35_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
+
+		/* Clamp the requested clock to PMFW based on their limit. */
+		if (dc->debug.min_disp_clk_khz > 0 && requested_dispclk_khz < dc->debug.min_disp_clk_khz)
+			requested_dispclk_khz = dc->debug.min_disp_clk_khz;
+
+		dcn42_smu_set_dispclk(clk_mgr, requested_dispclk_khz);
+		clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
+		dcn35_disable_otg_wa(clk_mgr_base, context, safe_to_lower, false);
+
+		update_dispclk = true;
+	}
+
+	/* clock limits are received with MHz precision, divide by 1000 to prevent setting clocks at every call */
+	if (!dc->debug.disable_dtb_ref_clk_switch &&
+	    should_set_clock(safe_to_lower, new_clocks->ref_dtbclk_khz / 1000,
+			     clk_mgr_base->clks.ref_dtbclk_khz / 1000)) {
+		dcn42_update_clocks_update_dtb_dto(clk_mgr, context, new_clocks->ref_dtbclk_khz);
+		clk_mgr_base->clks.ref_dtbclk_khz = new_clocks->ref_dtbclk_khz;
+	}
+
+	if (dpp_clock_lowered) {
+		// increase per DPP DTO before lowering global dppclk
+		dcn42_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
+		dcn42_smu_set_dppclk(clk_mgr, clk_mgr_base->clks.dppclk_khz);
+	} else {
+		// increase global DPPCLK before lowering per DPP DTO
+		if (update_dppclk || update_dispclk)
+			dcn42_smu_set_dppclk(clk_mgr, clk_mgr_base->clks.dppclk_khz);
+		dcn42_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
+	}
+	// notify DMCUB of latest clocks
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.notify_clocks.header.type = DMUB_CMD__CLK_MGR;
+	cmd.notify_clocks.header.sub_type = DMUB_CMD__CLK_MGR_NOTIFY_CLOCKS;
+	cmd.notify_clocks.clocks.dcfclk_khz = clk_mgr_base->clks.dcfclk_khz;
+	cmd.notify_clocks.clocks.dcfclk_deep_sleep_khz =
+		clk_mgr_base->clks.dcfclk_deep_sleep_khz;
+	cmd.notify_clocks.clocks.dispclk_khz = clk_mgr_base->clks.dispclk_khz;
+	cmd.notify_clocks.clocks.dppclk_khz = clk_mgr_base->clks.dppclk_khz;
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
+
+
+void dcn42_enable_pme_wa(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+	dcn42_smu_enable_pme_wa(clk_mgr);
+}
+
+
+bool dcn42_are_clock_states_equal(struct dc_clocks *a,
+		struct dc_clocks *b)
+{
+	if (a->dispclk_khz != b->dispclk_khz)
+		return false;
+	else if (a->dppclk_khz != b->dppclk_khz)
+		return false;
+	else if (a->dcfclk_khz != b->dcfclk_khz)
+		return false;
+	else if (a->dcfclk_deep_sleep_khz != b->dcfclk_deep_sleep_khz)
+		return false;
+	else if (a->zstate_support != b->zstate_support)
+		return false;
+	else if (a->dtbclk_en != b->dtbclk_en)
+		return false;
+
+	return true;
+}
+
+static void dcn42_dump_clk_registers_internal(struct dcn42_clk_internal *internal, struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+	REG_GET(CLK8_CLK_TICK_CNT_CONFIG_REG, TIMER_THRESHOLD, &internal->CLK8_CLK_TICK_CNT__TIMER_THRESHOLD);
+
+	// read dcf deep sleep divider
+	internal->CLK8_CLK0_DS_CNTL = REG_READ(CLK8_CLK0_DS_CNTL);
+	internal->CLK8_CLK3_DS_CNTL = REG_READ(CLK8_CLK3_DS_CNTL);
+	// read dispclk
+	internal->CLK8_CLK0_CURRENT_CNT = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dispclk);
+	internal->CLK8_CLK0_BYPASS_CNTL = REG_READ(CLK8_CLK0_BYPASS_CNTL);
+	// read dppclk
+	internal->CLK8_CLK1_CURRENT_CNT = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dppclk);
+	internal->CLK8_CLK1_BYPASS_CNTL = REG_READ(CLK8_CLK1_BYPASS_CNTL);
+	// read dprefclk
+	internal->CLK8_CLK2_CURRENT_CNT = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dprefclk);
+	internal->CLK8_CLK2_BYPASS_CNTL = REG_READ(CLK8_CLK2_BYPASS_CNTL);
+	// read dcfclk
+	internal->CLK8_CLK3_CURRENT_CNT = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dcfclk);
+	internal->CLK8_CLK3_BYPASS_CNTL = REG_READ(CLK8_CLK3_BYPASS_CNTL);
+	// read dtbclk
+	internal->CLK8_CLK4_CURRENT_CNT = dcn42_get_clock_freq_from_clkip(clk_mgr_base, clock_type_dtbclk);
+	internal->CLK8_CLK4_BYPASS_CNTL = REG_READ(CLK8_CLK4_BYPASS_CNTL);
+}
+
+static void dcn42_dump_clk_registers(struct clk_state_registers_and_bypass *regs_and_bypass,
+		struct clk_mgr_dcn42 *clk_mgr)
+{
+	struct dcn42_clk_internal internal = {0};
+	char *bypass_clks[5] = {"0x0 DFS", "0x1 REFCLK", "0x2 ERROR", "0x3 400 FCH", "0x4 600 FCH"};
+
+	DC_LOGGER_INIT(clk_mgr->base.base.ctx->logger);
+	(void)dc_logger;
+
+	dcn42_dump_clk_registers_internal(&internal, &clk_mgr->base.base);
+	regs_and_bypass->timer_threshold = internal.CLK8_CLK_TICK_CNT__TIMER_THRESHOLD;
+	regs_and_bypass->dcfclk = internal.CLK8_CLK3_CURRENT_CNT / 10;
+	regs_and_bypass->dcf_deep_sleep_divider = internal.CLK8_CLK3_DS_CNTL / 10;
+	regs_and_bypass->dcf_deep_sleep_allow = internal.CLK8_CLK3_DS_CNTL & 0x10; /*bit 4: CLK0_ALLOW_DS*/
+	regs_and_bypass->dprefclk = internal.CLK8_CLK2_CURRENT_CNT / 10;
+	regs_and_bypass->dispclk = internal.CLK8_CLK0_CURRENT_CNT / 10;
+	regs_and_bypass->dppclk = internal.CLK8_CLK1_CURRENT_CNT / 10;
+	regs_and_bypass->dtbclk = internal.CLK8_CLK4_CURRENT_CNT / 10;
+
+	regs_and_bypass->dispclk_bypass = get_reg_field_value(internal.CLK8_CLK0_BYPASS_CNTL, CLK8_CLK0_BYPASS_CNTL, CLK0_BYPASS_SEL);
+	regs_and_bypass->dppclk_bypass = get_reg_field_value(internal.CLK8_CLK1_BYPASS_CNTL, CLK8_CLK1_BYPASS_CNTL, CLK1_BYPASS_SEL);
+	regs_and_bypass->dprefclk_bypass = get_reg_field_value(internal.CLK8_CLK2_BYPASS_CNTL, CLK8_CLK2_BYPASS_CNTL, CLK2_BYPASS_SEL);
+	regs_and_bypass->dcfclk_bypass = get_reg_field_value(internal.CLK8_CLK3_BYPASS_CNTL, CLK8_CLK3_BYPASS_CNTL, CLK3_BYPASS_SEL);
+
+	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled) {
+		DC_LOG_SMU("clk_type,clk_value,deepsleep_cntl,deepsleep_allow,bypass\n");
+
+		DC_LOG_SMU("dcfclk,%d,%d,%d,%s\n",
+				   regs_and_bypass->dcfclk,
+				   regs_and_bypass->dcf_deep_sleep_divider,
+				   regs_and_bypass->dcf_deep_sleep_allow,
+				   bypass_clks[(int) regs_and_bypass->dcfclk_bypass]);
+
+		DC_LOG_SMU("dprefclk,%d,N/A,N/A,%s\n",
+			regs_and_bypass->dprefclk,
+			bypass_clks[(int) regs_and_bypass->dprefclk_bypass]);
+
+		DC_LOG_SMU("dispclk,%d,N/A,N/A,%s\n",
+			regs_and_bypass->dispclk,
+			bypass_clks[(int) regs_and_bypass->dispclk_bypass]);
+
+		//split
+		DC_LOG_SMU("SPLIT\n");
+
+		// REGISTER VALUES
+		DC_LOG_SMU("reg_name,value,clk_type\n");
+		DC_LOG_SMU("CLK1_CLK3_CURRENT_CNT,%d,dcfclk\n",
+				internal.CLK8_CLK3_CURRENT_CNT);
+
+		DC_LOG_SMU("CLK1_CLK3_DS_CNTL,%d,dcf_deep_sleep_divider\n",
+					internal.CLK8_CLK3_DS_CNTL);
+
+		DC_LOG_SMU("CLK1_CLK3_ALLOW_DS,%d,dcf_deep_sleep_allow\n",
+					(internal.CLK8_CLK3_DS_CNTL & 0x10));
+
+		DC_LOG_SMU("CLK1_CLK2_CURRENT_CNT,%d,dprefclk\n",
+					internal.CLK8_CLK2_CURRENT_CNT);
+
+		DC_LOG_SMU("CLK1_CLK0_CURRENT_CNT,%d,dispclk\n",
+					internal.CLK8_CLK0_CURRENT_CNT);
+
+		DC_LOG_SMU("CLK1_CLK1_CURRENT_CNT,%d,dppclk\n",
+					internal.CLK8_CLK1_CURRENT_CNT);
+
+		DC_LOG_SMU("CLK1_CLK3_BYPASS_CNTL,%d,dcfclk_bypass\n",
+					internal.CLK8_CLK3_BYPASS_CNTL);
+
+		DC_LOG_SMU("CLK1_CLK2_BYPASS_CNTL,%d,dprefclk_bypass\n",
+					internal.CLK8_CLK2_BYPASS_CNTL);
+
+		DC_LOG_SMU("CLK1_CLK0_BYPASS_CNTL,%d,dispclk_bypass\n",
+					internal.CLK8_CLK0_BYPASS_CNTL);
+
+		DC_LOG_SMU("CLK1_CLK1_BYPASS_CNTL,%d,dppclk_bypass\n",
+					internal.CLK8_CLK1_BYPASS_CNTL);
+	}
+}
+
+bool dcn42_is_spll_ssc_enabled(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct dc_context *ctx = clk_mgr->base.ctx;
+
+
+	if (ctx->dc->config.ignore_dpref_ss) {
+		/*revert bios's ss info for test only*/
+		return (clk_mgr->dprefclk_ss_percentage == 0);
+	}
+	/*need to update after BU*/
+	return false;
+}
+
+static void init_clk_states(struct clk_mgr *clk_mgr)
+{
+	uint32_t ref_dtbclk = clk_mgr->clks.ref_dtbclk_khz;
+
+	memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks));
+
+	clk_mgr->clks.dtbclk_en = true; // request DTBCLK disable on first commit
+	clk_mgr->clks.ref_dtbclk_khz = ref_dtbclk;      // restore ref_dtbclk
+	clk_mgr->clks.p_state_change_support = true;
+	clk_mgr->clks.prev_p_state_change_support = true;
+	clk_mgr->clks.pwr_state = DCN_PWR_STATE_UNKNOWN;
+	clk_mgr->clks.zstate_support = DCN_ZSTATE_SUPPORT_UNKNOWN;
+}
+
+static void dcn42_get_dpm_table_from_smu(struct clk_mgr_internal *clk_mgr,
+		struct dcn42_smu_dpm_clks *smu_dpm_clks)
+{
+	DpmClocks_t_dcn42 *table = smu_dpm_clks->dpm_clks;
+
+	if (!clk_mgr->smu_ver)
+		return;
+
+	if (!table || smu_dpm_clks->mc_address.quad_part == 0)
+		return;
+
+	memset(table, 0, sizeof(*table));
+
+	dcn42_smu_set_dram_addr_high(clk_mgr,
+			smu_dpm_clks->mc_address.high_part);
+	dcn42_smu_set_dram_addr_low(clk_mgr,
+			smu_dpm_clks->mc_address.low_part);
+	dcn42_smu_transfer_dpm_table_smu_2_dram(clk_mgr);
+}
+
+void dcn42_init_single_clock(unsigned int *entry_0,
+		uint32_t *smu_entry_0,
+		uint8_t num_levels)
+{
+	int i;
+	char *entry_i = (char *)entry_0;
+
+	ASSERT(num_levels <= MAX_NUM_DPM_LVL);
+	if (num_levels > MAX_NUM_DPM_LVL)
+		num_levels = MAX_NUM_DPM_LVL;
+
+
+	for (i = 0; i < num_levels; i++) {
+		*((unsigned int *)entry_i) = smu_entry_0[i];
+		entry_i += sizeof(struct clk_limit_table_entry);
+	}
+}
+
+unsigned int dcn42_convert_wck_ratio(uint8_t wck_ratio)
+{
+	switch (wck_ratio) {
+	case WCK_RATIO_1_2:
+		return 2;
+
+	case WCK_RATIO_1_4:
+		return 4;
+
+	default:
+			break;
+	}
+
+	return 1;
+}
+
+void dcn42_init_clocks(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct clk_mgr_dcn42 *clk_mgr = TO_CLK_MGR_DCN42(clk_mgr_int);
+	struct dcn42_smu_dpm_clks smu_dpm_clks = { 0 };
+
+	DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
+	(void)dc_logger;
+
+	init_clk_states(clk_mgr_base);
+
+	// to adjust dp_dto reference clock if ssc is enable otherwise to apply dprefclk
+	if (dcn42_is_spll_ssc_enabled(clk_mgr_base))
+		clk_mgr_base->dp_dto_source_clock_in_khz =
+			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr_base->dprefclk_khz);
+	else
+		clk_mgr_base->dp_dto_source_clock_in_khz = clk_mgr_base->dprefclk_khz;
+
+	DC_LOG_SMU("dp_dto_source_clock %d, dprefclk %d\n", clk_mgr_base->dp_dto_source_clock_in_khz, clk_mgr_base->dprefclk_khz);
+	dcn42_dump_clk_registers(&clk_mgr_base->boot_snapshot, clk_mgr);
+
+	clk_mgr_base->clks.ref_dtbclk_khz =  clk_mgr_base->boot_snapshot.dtbclk * 10;
+	if (clk_mgr_base->boot_snapshot.dtbclk > 59000) {
+		/*dtbclk enabled based on*/
+		clk_mgr_base->clks.dtbclk_en = true;
+	}
+
+	if (clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dcfclk_levels != 0) {
+		/*skip to get clock table and notify pmfw watermark range again*/
+		DC_LOG_SMU("skip to get dpm_clks from pmfw from resume and acr\n");
+		return;
+	}
+
+	smu_dpm_clks.dpm_clks = (DpmClocks_t_dcn42 *)dm_helpers_allocate_gpu_mem(
+				clk_mgr_base->ctx,
+				DC_MEM_ALLOC_TYPE_GART,
+				sizeof(DpmClocks_t_dcn42),
+				&smu_dpm_clks.mc_address.quad_part);
+
+	ASSERT(smu_dpm_clks.dpm_clks);
+	if (clk_mgr_base->ctx->dc->debug.pstate_enabled && clk_mgr_int->smu_present && smu_dpm_clks.mc_address.quad_part != 0) {
+		int i;
+		DpmClocks_t_dcn42 *dpm_clks = smu_dpm_clks.dpm_clks;
+
+		dcn42_get_dpm_table_from_smu(clk_mgr_int, &smu_dpm_clks);
+		DC_LOG_SMU("NumDcfClkLevelsEnabled: %d\n"
+				   "NumDispClkLevelsEnabled: %d\n"
+				   "NumSocClkLevelsEnabled: %d\n"
+				   "VcnClkLevelsEnabled: %d\n"
+				   "FClkLevelsEnabled: %d\n"
+				   "NumMemPstatesEnabled: %d\n"
+				   "MinGfxClk: %d\n"
+				   "MaxGfxClk: %d\n",
+				   dpm_clks->NumDcfClkLevelsEnabled,
+				   dpm_clks->NumDispClkLevelsEnabled,
+				   dpm_clks->NumSocClkLevelsEnabled,
+				   dpm_clks->VcnClkLevelsEnabled,
+				   dpm_clks->NumFclkLevelsEnabled,
+				   dpm_clks->NumMemPstatesEnabled,
+				   dpm_clks->MinGfxClk,
+				   dpm_clks->MaxGfxClk);
+
+		for (i = 0; i < NUM_DCFCLK_DPM_LEVELS; i++) {
+			DC_LOG_SMU("dpm_clks->DcfClocks[%d] = %d\n",
+					   i,
+					   dpm_clks->DcfClocks[i]);
+		}
+		for (i = 0; i < NUM_DISPCLK_DPM_LEVELS; i++) {
+			DC_LOG_SMU("dpm_clks->DispClocks[%d] = %d\n",
+					   i, dpm_clks->DispClocks[i]);
+		}
+		for (i = 0; i < NUM_SOCCLK_DPM_LEVELS; i++) {
+			DC_LOG_SMU("dpm_clks->SocClocks[%d] = %d\n",
+					   i, dpm_clks->SocClocks[i]);
+		}
+		for (i = 0; i < NUM_FCLK_DPM_LEVELS; i++) {
+			DC_LOG_SMU("dpm_clks->FclkClocks_Freq[%d] = %d\n",
+					   i, dpm_clks->FclkClocks_Freq[i]);
+			DC_LOG_SMU("dpm_clks->FclkClocks_Voltage[%d] = %d\n",
+					   i, dpm_clks->FclkClocks_Voltage[i]);
+		}
+		for (i = 0; i < NUM_SOCCLK_DPM_LEVELS; i++)
+			DC_LOG_SMU("dpm_clks->SocVoltage[%d] = %d\n",
+					   i, dpm_clks->SocVoltage[i]);
+
+		for (i = 0; i < NUM_MEM_PSTATE_LEVELS; i++) {
+			DC_LOG_SMU("dpm_clks.MemPstateTable[%d].UClk = %d\n"
+					   "dpm_clks->MemPstateTable[%d].MemClk= %d\n"
+					   "dpm_clks->MemPstateTable[%d].Voltage = %d\n",
+					   i, dpm_clks->MemPstateTable[i].UClk,
+					   i, dpm_clks->MemPstateTable[i].MemClk,
+					   i, dpm_clks->MemPstateTable[i].Voltage);
+		}
+
+		if (clk_mgr_base->ctx->dc_bios->integrated_info && clk_mgr_base->ctx->dc->config.use_default_clock_table == false) {
+			/* DCFCLK */
+			dcn42_init_single_clock(&clk_mgr_base->bw_params->clk_table.entries[0].dcfclk_mhz,
+					dpm_clks->DcfClocks,
+					dpm_clks->NumDcfClkLevelsEnabled);
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dcfclk_levels = dpm_clks->NumDcfClkLevelsEnabled;
+
+			/* SOCCLK */
+			dcn42_init_single_clock(&clk_mgr_base->bw_params->clk_table.entries[0].socclk_mhz,
+					dpm_clks->SocClocks,
+					dpm_clks->NumSocClkLevelsEnabled);
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_socclk_levels = dpm_clks->NumSocClkLevelsEnabled;
+
+			/* DISPCLK */
+			dcn42_init_single_clock(&clk_mgr_base->bw_params->clk_table.entries[0].dispclk_mhz,
+					dpm_clks->DispClocks,
+					dpm_clks->NumDispClkLevelsEnabled);
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dispclk_levels = dpm_clks->NumDispClkLevelsEnabled;
+
+			/* DPPCLK */
+			dcn42_init_single_clock(&clk_mgr_base->bw_params->clk_table.entries[0].dppclk_mhz,
+					dpm_clks->DppClocks,
+					dpm_clks->NumDispClkLevelsEnabled);
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dppclk_levels = dpm_clks->NumDispClkLevelsEnabled;
+
+			/* FCLK */
+			dcn42_init_single_clock(&clk_mgr_base->bw_params->clk_table.entries[0].fclk_mhz,
+					dpm_clks->FclkClocks_Freq,
+					NUM_FCLK_DPM_LEVELS);
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_fclk_levels = dpm_clks->NumFclkLevelsEnabled;
+			clk_mgr_base->bw_params->clk_table.num_entries = dpm_clks->NumFclkLevelsEnabled;
+
+			/* Memory Pstate table is in reverse order*/
+			ASSERT(dpm_clks->NumMemPstatesEnabled <= NUM_MEM_PSTATE_LEVELS);
+			if (dpm_clks->NumMemPstatesEnabled > NUM_MEM_PSTATE_LEVELS)
+				dpm_clks->NumMemPstatesEnabled = NUM_MEM_PSTATE_LEVELS;
+			for (i = 0; i < dpm_clks->NumMemPstatesEnabled; i++) {
+				clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].memclk_mhz = dpm_clks->MemPstateTable[i].UClk;
+				clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].wck_ratio = dcn42_convert_wck_ratio(dpm_clks->MemPstateTable[i].WckRatio)	;
+			}
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels = dpm_clks->NumMemPstatesEnabled;
+
+			/* DTBCLK*/
+			clk_mgr_base->bw_params->clk_table.entries[0].dtbclk_mhz = clk_mgr_base->clks.ref_dtbclk_khz / 1000;
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dtbclk_levels = 1;
+			/* Refresh bounding box */
+			clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box(
+				clk_mgr_base->ctx->dc, clk_mgr_base->bw_params);
+		}
+	}
+	if (smu_dpm_clks.dpm_clks && smu_dpm_clks.mc_address.quad_part != 0)
+		dm_helpers_free_gpu_mem(clk_mgr_base->ctx, DC_MEM_ALLOC_TYPE_GART,
+				smu_dpm_clks.dpm_clks);
+}
+
+static struct clk_bw_params dcn42_bw_params = {
+	.vram_type = Ddr4MemType,
+	.num_channels = 1,
+	.clk_table = {
+		.num_entries = 4,
+	},
+
+};
+
+static struct wm_table ddr5_wm_table = {
+	.entries = {
+		{
+			.wm_inst = WM_A,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.72,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_B,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.72,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_C,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.72,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_D,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.72,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+	}
+};
+
+static struct wm_table lpddr5_wm_table = {
+	.entries = {
+		{
+			.wm_inst = WM_A,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.65333,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_B,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.65333,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_C,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.65333,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+		{
+			.wm_inst = WM_D,
+			.wm_type = WM_TYPE_PSTATE_CHG,
+			.pstate_latency_us = 11.65333,
+			.sr_exit_time_us = 28.0,
+			.sr_enter_plus_exit_time_us = 30.0,
+			.valid = true,
+		},
+	}
+};
+
+struct dcn42_ss_info_table dcn42_ss_info_table = {
+	.ss_divider = 1000,
+	.ss_percentage = {0, 0, 375, 375, 375}
+};
+
+static void dcn42_read_ss_info_from_lut(struct clk_mgr_internal *clk_mgr)
+{
+	uint32_t clock_source;
+
+	clock_source = (REG_READ(CLK8_CLK2_BYPASS_CNTL) & CLK8_CLK2_BYPASS_CNTL__CLK2_BYPASS_SEL_MASK);
+	// If it's DFS mode, clock_source is 0.
+	if (dcn42_is_spll_ssc_enabled(&clk_mgr->base) && (clock_source < ARRAY_SIZE(dcn42_ss_info_table.ss_percentage))) {
+		clk_mgr->dprefclk_ss_percentage = dcn42_ss_info_table.ss_percentage[clock_source];
+
+		if (clk_mgr->dprefclk_ss_percentage != 0) {
+			clk_mgr->ss_on_dprefclk = true;
+			clk_mgr->dprefclk_ss_divider = dcn42_ss_info_table.ss_divider;
+		}
+	}
+}
+
+void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_watermarks *table)
+{
+	int i, num_valid_sets;
+
+	num_valid_sets = 0;
+
+	for (i = 0; i < WM_SET_COUNT; i++) {
+		/* skip empty entries, the smu array has no holes*/
+		if (!bw_params->wm_table.entries[i].valid)
+			continue;
+
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+		/* We will not select WM based on fclk, so leave it as unconstrained */
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
+
+		if (table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType == WM_TYPE_PSTATE_CHG) {
+			if (i == 0)
+				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk = 0;
+			else {
+				/* add 1 to make it non-overlapping with next lvl */
+				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
+						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+			}
+			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
+					bw_params->clk_table.entries[i].dcfclk_mhz;
+
+		} else {
+			/* unconstrained for memory retraining */
+			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
+			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
+
+			/* Modify previous watermark range to cover up to max */
+			if (num_valid_sets > 0)
+				table->WatermarkRow[WM_DCFCLK][num_valid_sets - 1].MaxClock = 0xFFFF;
+		}
+		num_valid_sets++;
+	}
+
+	ASSERT(num_valid_sets != 0); /* Must have at least one set of valid watermarks */
+
+	/* modify the min and max to make sure we cover the whole range*/
+	table->WatermarkRow[WM_DCFCLK][0].MinMclk = 0;
+	table->WatermarkRow[WM_DCFCLK][0].MinClock = 0;
+	table->WatermarkRow[WM_DCFCLK][num_valid_sets - 1].MaxMclk = 0xFFFF;
+	table->WatermarkRow[WM_DCFCLK][num_valid_sets - 1].MaxClock = 0xFFFF;
+
+	/* This is for writeback only, does not matter currently as no writeback support*/
+	table->WatermarkRow[WM_SOCCLK][0].WmSetting = WM_A;
+	table->WatermarkRow[WM_SOCCLK][0].MinClock = 0;
+	table->WatermarkRow[WM_SOCCLK][0].MaxClock = 0xFFFF;
+	table->WatermarkRow[WM_SOCCLK][0].MinMclk = 0;
+	table->WatermarkRow[WM_SOCCLK][0].MaxMclk = 0xFFFF;
+}
+
+void dcn42_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
+{
+	int i = 0;
+	struct dcn42_watermarks *table = NULL;
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct clk_mgr_dcn42 *clk_mgr_dcn42 = TO_CLK_MGR_DCN42(clk_mgr);
+
+	if (!clk_mgr->smu_ver)
+		return;
+	/*send once already skip*/
+	if (clk_mgr_base->bw_params->wm_table.entries[WM_A].valid == true)
+		return;
+	clk_mgr_dcn42->smu_wm_set.wm_set = (struct dcn42_watermarks *)dm_helpers_allocate_gpu_mem(
+				clk_mgr->base.ctx,
+				DC_MEM_ALLOC_TYPE_GART,
+				sizeof(struct dcn42_watermarks),
+				&clk_mgr_dcn42->smu_wm_set.mc_address.quad_part);
+
+	ASSERT(clk_mgr_dcn42->smu_wm_set.wm_set);
+
+	table = clk_mgr_dcn42->smu_wm_set.wm_set;
+
+	if (!table || clk_mgr_dcn42->smu_wm_set.mc_address.quad_part == 0)
+		return;
+
+	memset(table, 0, sizeof(*table));
+	/*same as previous asic, set wm valid before building watermark ranges*/
+	for (i = 0; i < WM_SET_COUNT; i++) {
+		clk_mgr_base->bw_params->wm_table.entries[i].wm_inst = i;
+
+		if (i >= clk_mgr_base->bw_params->clk_table.num_entries) {
+			clk_mgr_base->bw_params->wm_table.entries[i].valid = false;
+			continue;
+		}
+		clk_mgr_base->bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
+		clk_mgr_base->bw_params->wm_table.entries[i].valid = true;
+	}
+	/* build watermark_range will check this valid range*/
+	dcn42_build_watermark_ranges(clk_mgr_base->bw_params, table);
+
+	dcn42_smu_set_dram_addr_high(clk_mgr,
+			clk_mgr_dcn42->smu_wm_set.mc_address.high_part);
+	dcn42_smu_set_dram_addr_low(clk_mgr,
+			clk_mgr_dcn42->smu_wm_set.mc_address.low_part);
+	dcn42_smu_transfer_wm_table_dram_2_smu(clk_mgr);
+
+	if (clk_mgr_dcn42->smu_wm_set.wm_set && clk_mgr_dcn42->smu_wm_set.mc_address.quad_part != 0)
+		dm_helpers_free_gpu_mem(clk_mgr->base.ctx, DC_MEM_ALLOC_TYPE_GART,
+				clk_mgr_dcn42->smu_wm_set.wm_set);
+
+}
+
+void dcn42_set_low_power_state(struct clk_mgr *clk_mgr_base)
+{
+	struct dc *dc = clk_mgr_base->ctx->dc;
+	struct dc_state *context = dc->current_state;
+
+	if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) {
+		/* if we can go lower, go lower */
+		if (dcn42_has_active_display(dc, context) == false)
+			clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_LOW_POWER;
+	}
+
+	if (clk_mgr_base->clks.pwr_state == DCN_PWR_STATE_LOW_POWER) {
+		union display_idle_optimization_u idle_info = { 0 };
+		struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+		idle_info.idle_info.df_request_disabled = 1;
+		idle_info.idle_info.phy_ref_clk_off = 1;
+		idle_info.idle_info.s0i2_rdy = 1;
+		dcn42_smu_set_display_idle_optimization(clk_mgr, idle_info.data);
+	}
+}
+
+void dcn42_exit_low_power_state(struct clk_mgr *clk_mgr_base)
+{
+
+}
+
+static void dcn42_init_clocks_fpga(struct clk_mgr *clk_mgr)
+{
+	init_clk_states(clk_mgr);
+
+}
+
+static void dcn42_update_clocks_fpga(struct clk_mgr *clk_mgr,
+		struct dc_state *context,
+		bool safe_to_lower)
+{
+	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr);
+	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
+	int fclk_adj = new_clocks->fclk_khz;
+
+	/* TODO: remove this after correctly set by DML */
+	new_clocks->dcfclk_khz = 400000;
+	new_clocks->socclk_khz = 400000;
+
+	/* Min fclk = 1.2GHz since all the extra scemi logic seems to run off of it */
+	//int fclk_adj = new_clocks->fclk_khz > 1200000 ? new_clocks->fclk_khz : 1200000;
+	new_clocks->fclk_khz = 4320000;
+
+	if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr->clks.phyclk_khz))
+		clk_mgr->clks.phyclk_khz = new_clocks->phyclk_khz;
+
+	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr->clks.dcfclk_khz))
+		clk_mgr->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+
+	if (should_set_clock(safe_to_lower,
+			new_clocks->dcfclk_deep_sleep_khz, clk_mgr->clks.dcfclk_deep_sleep_khz))
+		clk_mgr->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+
+	if (should_set_clock(safe_to_lower, new_clocks->socclk_khz, clk_mgr->clks.socclk_khz))
+		clk_mgr->clks.socclk_khz = new_clocks->socclk_khz;
+
+	if (should_set_clock(safe_to_lower, new_clocks->dramclk_khz, clk_mgr->clks.dramclk_khz))
+		clk_mgr->clks.dramclk_khz = new_clocks->dramclk_khz;
+
+	if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->clks.dppclk_khz))
+		clk_mgr->clks.dppclk_khz = new_clocks->dppclk_khz;
+
+	if (should_set_clock(safe_to_lower, fclk_adj, clk_mgr->clks.fclk_khz))
+		clk_mgr->clks.fclk_khz = fclk_adj;
+
+	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr->clks.dispclk_khz))
+		clk_mgr->clks.dispclk_khz = new_clocks->dispclk_khz;
+
+	/* Both fclk and ref_dppclk run on the same scemi clock.
+	 * So take the higher value since the DPP DTO is typically programmed
+	 * such that max dppclk is 1:1 with ref_dppclk.
+	 */
+	if (clk_mgr->clks.fclk_khz > clk_mgr->clks.dppclk_khz)
+		clk_mgr->clks.dppclk_khz = clk_mgr->clks.fclk_khz;
+	if (clk_mgr->clks.dppclk_khz > clk_mgr->clks.fclk_khz)
+		clk_mgr->clks.fclk_khz = clk_mgr->clks.dppclk_khz;
+
+	// Both fclk and ref_dppclk run on the same scemi clock.
+	clk_mgr_int->dccg->ref_dppclk = clk_mgr->clks.fclk_khz;
+
+	/* TODO: set dtbclk in correct place */
+	clk_mgr->clks.dtbclk_en = true;
+
+	dm_set_dcn_clocks(clk_mgr->ctx, &clk_mgr->clks);
+	dcn42_update_clocks_update_dpp_dto(clk_mgr_int, context, safe_to_lower);
+
+	dcn42_update_clocks_update_dtb_dto(clk_mgr_int, context, clk_mgr->clks.ref_dtbclk_khz);
+}
+
+unsigned int dcn42_get_max_clock_khz(struct clk_mgr *clk_mgr_base, enum clk_type clk_type)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+	unsigned int num_clk_levels;
+
+	switch (clk_type) {
+	case CLK_TYPE_DISPCLK:
+		num_clk_levels = clk_mgr->base.bw_params->clk_table.num_entries_per_clk.num_dispclk_levels;
+		return num_clk_levels ?
+				clk_mgr->base.bw_params->clk_table.entries[num_clk_levels - 1].dispclk_mhz * 1000 :
+				clk_mgr->base.boot_snapshot.dispclk;
+	case CLK_TYPE_DPPCLK:
+		num_clk_levels = clk_mgr->base.bw_params->clk_table.num_entries_per_clk.num_dppclk_levels;
+		return num_clk_levels ?
+				clk_mgr->base.bw_params->clk_table.entries[num_clk_levels - 1].dppclk_mhz * 1000 :
+				clk_mgr->base.boot_snapshot.dppclk;
+	case CLK_TYPE_DSCCLK:
+		num_clk_levels = clk_mgr->base.bw_params->clk_table.num_entries_per_clk.num_dispclk_levels;
+		return num_clk_levels ?
+				clk_mgr->base.bw_params->clk_table.entries[num_clk_levels - 1].dispclk_mhz * 1000 / 3 :
+				clk_mgr->base.boot_snapshot.dispclk / 3;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int dcn42_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	uint32_t dispclk_wdivider;
+	int disp_divider;
+
+	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
+	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
+
+	/* Return DISPCLK freq in Khz */
+	if (disp_divider)
+		return (DENTIST_DIVIDER_RANGE_SCALE_FACTOR * clk_mgr->base.dentist_vco_freq_khz) / disp_divider;
+
+	return 0;
+}
+bool dcn42_is_smu_present(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	return clk_mgr->smu_present;
+}
+
+static struct clk_mgr_funcs dcn42_funcs = {
+	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
+	.update_clocks = dcn42_update_clocks,
+	.init_clocks = dcn42_init_clocks,
+	.enable_pme_wa = dcn42_enable_pme_wa,
+	.are_clock_states_equal = dcn42_are_clock_states_equal,
+	.notify_wm_ranges = dcn42_notify_wm_ranges,
+	.set_low_power_state = dcn42_set_low_power_state,
+	.exit_low_power_state = dcn42_exit_low_power_state,
+	.get_max_clock_khz = dcn42_get_max_clock_khz,
+	.get_dispclk_from_dentist = dcn42_get_dispclk_from_dentist,
+	.is_smu_present = dcn42_is_smu_present,
+};
+
+struct clk_mgr_funcs dcn42_fpga_funcs = {
+	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.update_clocks = dcn42_update_clocks_fpga,
+	.init_clocks = dcn42_init_clocks_fpga,
+	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
+};
+
+void dcn42_clk_mgr_construct(
+		struct dc_context *ctx,
+		struct clk_mgr_dcn42 *clk_mgr,
+		struct pp_smu_funcs *pp_smu,
+		struct dccg *dccg)
+{
+	clk_mgr->base.base.ctx = ctx;
+	clk_mgr->base.base.funcs = &dcn42_funcs;
+	clk_mgr->base.regs = &clk_mgr_regs_dcn42;
+	clk_mgr->base.clk_mgr_shift = &clk_mgr_shift_dcn42;
+	clk_mgr->base.clk_mgr_mask = &clk_mgr_mask_dcn42;
+
+	clk_mgr->base.pp_smu = pp_smu;
+
+	clk_mgr->base.dccg = dccg;
+	clk_mgr->base.dfs_bypass_disp_clk = 0;
+
+	clk_mgr->base.dprefclk_ss_percentage = 0;
+	clk_mgr->base.dprefclk_ss_divider = 1000;
+	clk_mgr->base.ss_on_dprefclk = false;
+	clk_mgr->base.dfs_ref_freq_khz = 48000; /*sync with pmfw*/
+	clk_mgr->base.base.clks.ref_dtbclk_khz = 600000;
+
+	/* Changed from DCN3.2_clock_frequency doc to match
+	 * dcn32_dump_clk_registers from 4 * dentist_vco_freq_khz /
+	 * dprefclk DID divider
+	 */
+	clk_mgr->base.base.dprefclk_khz = 600000;
+
+		clk_mgr->base.smu_present = false;
+		clk_mgr->base.smu_ver = dcn42_smu_get_pmfw_version(&clk_mgr->base);
+		if (clk_mgr->base.smu_ver && clk_mgr->base.smu_ver != -1)
+			clk_mgr->base.smu_present = true;
+
+		if (ctx->dc_bios->integrated_info) {
+			clk_mgr->base.base.dentist_vco_freq_khz = ctx->dc_bios->integrated_info->dentist_vco_freq;
+
+			if (ctx->dc_bios->integrated_info->memory_type == LpDdr5MemType)
+				dcn42_bw_params.wm_table = lpddr5_wm_table;
+			else
+				dcn42_bw_params.wm_table = ddr5_wm_table;
+			dcn42_bw_params.vram_type = ctx->dc_bios->integrated_info->memory_type;
+			dcn42_bw_params.dram_channel_width_bytes = ctx->dc_bios->integrated_info->memory_type == 0x22 ? 8 : 4;
+			dcn42_bw_params.num_channels = ctx->dc_bios->integrated_info->ma_channel_number ? ctx->dc_bios->integrated_info->ma_channel_number : 1;
+			clk_mgr->base.base.dprefclk_khz = dcn42_smu_get_dprefclk(&clk_mgr->base);
+			clk_mgr->base.base.clks.ref_dtbclk_khz = dcn42_smu_get_dtbclk(&clk_mgr->base);
+		}
+		/* in case we don't get a value from the BIOS, use default */
+		if (clk_mgr->base.base.dentist_vco_freq_khz == 0)
+			clk_mgr->base.base.dentist_vco_freq_khz = 3000000; /* 3000MHz */
+
+		/* Saved clocks configured at boot for debug purposes */
+		dcn42_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, clk_mgr);
+
+	dce_clock_read_ss_info(&clk_mgr->base);
+	/*when clk src is from FCH, it could have ss, same clock src as DPREF clk*/
+
+	dcn42_read_ss_info_from_lut(&clk_mgr->base);
+
+	clk_mgr->base.base.bw_params = &dcn42_bw_params;
+}
+
+void dcn42_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int)
+{
+	struct clk_mgr_dcn42 *clk_mgr = TO_CLK_MGR_DCN42(clk_mgr_int);
+
+	if (clk_mgr->smu_wm_set.wm_set && clk_mgr->smu_wm_set.mc_address.quad_part != 0)
+		dm_helpers_free_gpu_mem(clk_mgr_int->base.ctx, DC_MEM_ALLOC_TYPE_GART,
+				clk_mgr->smu_wm_set.wm_set);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
new file mode 100644
index 0000000000000..5ad027a9edaf0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DCN42_CLK_MGR_H__
+#define __DCN42_CLK_MGR_H__
+#include "clk_mgr_internal.h"
+
+#define NUM_CLOCK_SOURCES 5
+
+struct dcn42_watermarks;
+
+struct dcn42_smu_watermark_set {
+	struct dcn42_watermarks *wm_set;
+	union large_integer mc_address;
+};
+
+struct dcn42_ss_info_table {
+	uint32_t ss_divider;
+	uint32_t ss_percentage[NUM_CLOCK_SOURCES];
+};
+
+struct clk_mgr_dcn42 {
+	struct clk_mgr_internal base;
+	struct dcn42_smu_watermark_set smu_wm_set;
+};
+
+bool dcn42_are_clock_states_equal(struct dc_clocks *a,
+								  struct dc_clocks *b);
+void dcn42_init_clocks(struct clk_mgr *clk_mgr);
+void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
+						 struct dc_state *context,
+						 bool safe_to_lower);
+
+void dcn42_clk_mgr_construct(struct dc_context *ctx,
+							 struct clk_mgr_dcn42 *clk_mgr,
+							 struct pp_smu_funcs *pp_smu,
+							 struct dccg *dccg);
+
+void dcn42_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int);
+
+void dcn42_init_single_clock(unsigned int *entry_0,
+			      uint32_t *smu_entry_0,
+			      uint8_t num_levels);
+unsigned int dcn42_convert_wck_ratio(uint8_t wck_ratio);
+extern struct dcn42_ss_info_table dcn42_ss_info_table;
+void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_watermarks *table);
+void dcn42_enable_pme_wa(struct clk_mgr *clk_mgr_base);
+void dcn42_notify_wm_ranges(struct clk_mgr *clk_mgr_base);
+void dcn42_set_low_power_state(struct clk_mgr *clk_mgr_base);
+void dcn42_exit_low_power_state(struct clk_mgr *clk_mgr_base);
+unsigned int dcn42_get_max_clock_khz(struct clk_mgr *clk_mgr_base, enum clk_type clk_type);
+bool dcn42_is_smu_present(struct clk_mgr *clk_mgr_base);
+int dcn42_get_active_display_cnt_wa(struct dc *dc, struct dc_state *context, int *all_active_disps);
+void dcn42_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr, struct dc_state *context, bool safe_to_lower);
+void dcn42_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr, struct dc_state *context, int ref_dtbclk_khz);
+bool dcn42_is_spll_ssc_enabled(struct clk_mgr *clk_mgr_base);
+bool dcn42_has_active_display(struct dc *dc, const struct dc_state *context);
+#endif //__DCN42_CLK_MGR_H__
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
new file mode 100644
index 0000000000000..c791bb1edb47e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
@@ -0,0 +1,431 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+#include "reg_helper.h"
+#include "dm_helpers.h"
+#include "dcn42_smu.h"
+
+#include "mp/mp_15_0_0_offset.h"
+#include "mp/mp_15_0_0_sh_mask.h"
+
+#ifdef BASE_INNER
+#undef BASE_INNER
+#endif
+
+#define MP1_BASE__INST0_SEG0                       0x00016000
+#define MP1_BASE__INST0_SEG1                       0x00016200
+#define MP1_BASE__INST0_SEG2                       0x00E00000
+#define MP1_BASE__INST0_SEG3                       0x00E80000
+#define MP1_BASE__INST0_SEG4                       0x00EC0000
+#define MP1_BASE__INST0_SEG5                       0x00F00000
+#define MP1_BASE__INST0_SEG6                       0x02400400
+#define MP1_BASE__INST0_SEG7                       0x0243F400
+#define MP1_BASE__INST0_SEG8                       0x3C004000
+#define MP1_BASE__INST0_SEG9                       0x3C3F4000
+
+#define BASE_INNER(seg) MP1_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define REG(reg_name) (BASE(reg##reg_name##_BASE_IDX) + reg##reg_name)
+
+#define FN(reg_name, field) \
+	FD(reg_name##__##field)
+
+#include "logger_types.h"
+#undef DC_LOGGER
+#define DC_LOGGER \
+	CTX->logger
+#define smu_print(str, ...) {DC_LOG_SMU(str, ##__VA_ARGS__); }
+
+// VBIOS and DAL to PMFW Interface
+
+// DAL to PMFW interface registers
+#define DAL_MSG_REG                             MP1_SMN_C2PMSG_71
+#define DAL_RESP_REG                            MP1_SMN_C2PMSG_72
+#define DAL_ARG_REG                             MP1_SMN_C2PMSG_73
+
+/** @defgroup ResponseCodes PMFW Response Codes
+*  @{
+*/
+// SMU Response Codes:
+#define DALSMC_Result_OK                        0x01 ///< Message Response OK
+#define DALSMC_Result_Failed                    0xFF ///< Message Response Failed
+#define DALSMC_Result_UnknownCmd                0xFE ///< Message Response Unknown Command
+#define DALSMC_Result_CmdRejectedPrereq         0xFD ///< Message Response Command Failed Prerequisite
+#define DALSMC_Result_CmdRejectedBusy           0xFC ///< Message Response Command Rejected due to PMFW is busy. Sender should retry sending this message
+/** @}*/
+
+// Message Definitions:
+/** @defgroup definitions Message definitions
+*  @{
+*/
+#define DALSMC_MSG_TestMessage                  0x01 ///< To check if PMFW is alive and responding. Requirement specified by PMFW team
+#define DALSMC_MSG_GetPmfwVersion               0x02 ///< Get version
+#define DALSMC_MSG_SetDispclkFreq               0x03 ///< Set display clock frequency in MHZ
+#define DALSMC_MSG_SetDppclkFreq                0x04 ///< Set DPP clock frequency in MHZ
+#define DALSMC_MSG_SetHardMinDcfclkByFreq       0x05 ///< Set DCF clock frequency hard min in MHZ
+#define DALSMC_MSG_SetMinDeepSleepDcfclk        0x06 ///< Set DCF clock minimum frequency in deep sleep in MHZ
+#define DALSMC_MSG_UpdatePmeRestore             0x07 ///< To ask PMFW to write into Azalia for PME wake up event
+#define DALSMC_MSG_SetDramAddrHigh              0x08 ///< Set DRAM address high 32 bits for WM table transfer
+#define DALSMC_MSG_SetDramAddrLow               0x09 ///< Set DRAM address low 32 bits for WM table transfer
+
+#define DALSMC_MSG_TransferTableSmu2Dram        0x0A ///< Transfer table from PMFW SRAM to system DRAM
+#define DALSMC_MSG_TransferTableDram2Smu        0x0B ///< Transfer table from system DRAM to PMFW
+#define DALSMC_MSG_SetDisplayIdleOptimizations  0x0C ///< Set Idle state optimization for display off
+#define DALSMC_MSG_GetDprefclkFreq              0x0D ///< Get DPREF clock frequency. Return in MHZ
+#define DALSMC_MSG_GetDtbclkFreq                0x0E ///< Get DTB clock frequency. Return in MHZ
+#define DALSMC_MSG_AllowZstatesEntry            0x0F ///< Inform PMFW of display allowing Zstate entry
+#define DALSMC_MSG_SetDtbClk                    0x10 ///< Inform PMFW to turn of/off DTB cl0ck. arg = 1: turn DTB on with 600MHZ; arg = 0: turn DTB clk off
+#define DALSMC_MSG_DispIPS2Exit                 0x11 ///< Display IPS2 exit
+
+#define DALSMC_MSG_QueryIPS2Support             0x12 ///< Return 1: support; else not supported
+#define DALSMC_Message_Count                    0x13 ///< Total number of VBIS and DAL messages
+/** @}*/
+
+
+union dcn42_dpia_host_router_bw {
+	struct {
+		uint32_t hr_id : 16;
+		uint32_t bw_mbps : 16;
+	} bits;
+	uint32_t all;
+};
+
+/*
+ * Function to be used instead of REG_WAIT macro because the wait ends when
+ * the register is NOT EQUAL to zero, and because `the translation in msg_if.h
+ * won't work with REG_WAIT.
+ */
+static uint32_t dcn42_smu_wait_for_response(struct clk_mgr_internal *clk_mgr,
+		unsigned int delay_us, unsigned int max_retries)
+{
+	uint32_t res_val;
+
+	do {
+		res_val = REG_READ(DAL_RESP_REG);
+		if (res_val != DALSMC_Result_CmdRejectedBusy)
+			break;
+
+		if (delay_us >= 1000)
+			msleep(delay_us/1000);
+		else if (delay_us > 0)
+			udelay(delay_us);
+
+		if (clk_mgr->base.ctx->dc->debug.disable_timeout)
+			max_retries++;
+	} while (max_retries--);
+
+	return res_val;
+}
+
+static int dcn42_smu_send_msg_with_param(struct clk_mgr_internal *clk_mgr,
+					 unsigned int msg_id,
+					 unsigned int param)
+{
+	uint32_t result;
+
+	result = dcn42_smu_wait_for_response(clk_mgr, 10, 2000000);
+
+	if (result != DALSMC_Result_OK) {
+		DC_LOG_WARNING("SMU response after wait: %d, msg id = %d\n", result, msg_id);
+
+		if (result == DALSMC_Result_CmdRejectedBusy)
+			return -1;
+	}
+
+	/* First clear response register */
+	REG_WRITE(DAL_RESP_REG, DALSMC_Result_CmdRejectedBusy);
+
+	/* Set the parameter register for the SMU message, unit is Mhz */
+	REG_WRITE(DAL_ARG_REG, param);
+
+	/* Trigger the message transaction by writing the message ID */
+	REG_WRITE(DAL_MSG_REG, msg_id);
+
+	result = dcn42_smu_wait_for_response(clk_mgr, 10, 2000000);
+
+	if (result == DALSMC_Result_Failed) {
+		if (msg_id == DALSMC_MSG_TransferTableDram2Smu &&
+		    param == TABLE_WATERMARKS)
+			DC_LOG_WARNING("Watermarks table not configured properly by SMU");
+		REG_WRITE(DAL_RESP_REG, DALSMC_Result_OK);
+		DC_LOG_WARNING("SMU response after wait: %d, msg id = %d\n", result, msg_id);
+		return -1;
+	}
+
+	if (IS_SMU_TIMEOUT(result)) {
+		ASSERT(0);
+		result = dcn42_smu_wait_for_response(clk_mgr, 10, 2000000);
+		//dm_helpers_smu_timeout(CTX, msg_id, param, 10 * 200000);
+		DC_LOG_WARNING("SMU response after wait: %d, msg id = %d\n", result, msg_id);
+	}
+
+	return REG_READ(DAL_ARG_REG);
+}
+
+int dcn42_smu_get_pmfw_version(struct clk_mgr_internal *clk_mgr)
+{
+	return dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_GetPmfwVersion,
+			0);
+}
+
+
+int dcn42_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz)
+{
+	int actual_dispclk_set_mhz;
+
+	if (!clk_mgr->smu_present)
+		return requested_dispclk_khz;
+
+	/*  Unit of SMU msg parameter is Mhz */
+	actual_dispclk_set_mhz = dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetDispclkFreq,
+			khz_to_mhz_ceil(requested_dispclk_khz));
+
+	smu_print("requested_dispclk_khz = %d, actual_dispclk_set_mhz: %d\n",
+		requested_dispclk_khz, actual_dispclk_set_mhz);
+	return (int)((long long)actual_dispclk_set_mhz * 1000);
+}
+
+
+int dcn42_smu_set_hard_min_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_dcfclk_khz)
+{
+	int actual_dcfclk_set_mhz;
+
+	if (!clk_mgr->smu_present)
+		return requested_dcfclk_khz;
+
+	actual_dcfclk_set_mhz = dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetHardMinDcfclkByFreq,
+			khz_to_mhz_ceil(requested_dcfclk_khz));
+
+	smu_print("requested_dcfclk_khz = %d, actual_dcfclk_set_mhz: %d\n",
+		requested_dcfclk_khz, actual_dcfclk_set_mhz);
+
+	return (int)((long long)actual_dcfclk_set_mhz * 1000);
+}
+
+int dcn42_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_min_ds_dcfclk_khz)
+{
+	int actual_min_ds_dcfclk_mhz;
+
+	if (!clk_mgr->smu_present)
+		return requested_min_ds_dcfclk_khz;
+
+	actual_min_ds_dcfclk_mhz = dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetMinDeepSleepDcfclk,
+			khz_to_mhz_ceil(requested_min_ds_dcfclk_khz));
+
+	smu_print("requested_min_ds_dcfclk_khz = %d, actual_min_ds_dcfclk_mhz: %d\n",
+		requested_min_ds_dcfclk_khz, actual_min_ds_dcfclk_mhz);
+
+	return (int)((long long)actual_min_ds_dcfclk_mhz * 1000);
+}
+
+int dcn42_smu_set_dppclk(struct clk_mgr_internal *clk_mgr, int requested_dpp_khz)
+{
+	int actual_dppclk_set_mhz;
+
+	if (!clk_mgr->smu_present)
+		return requested_dpp_khz;
+
+	actual_dppclk_set_mhz = dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetDppclkFreq,
+			khz_to_mhz_ceil(requested_dpp_khz));
+
+	smu_print("requested_dpp_khz = %d, actual_dppclk_set_mhz: %d\n",
+		requested_dpp_khz, actual_dppclk_set_mhz);
+
+	return (int)((long long)actual_dppclk_set_mhz * 1000);
+}
+
+void dcn42_smu_set_display_idle_optimization(struct clk_mgr_internal *clk_mgr, uint32_t idle_info)
+{
+	if (!clk_mgr->base.ctx->dc->debug.pstate_enabled)
+		return;
+
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(
+		clk_mgr,
+		DALSMC_MSG_SetDisplayIdleOptimizations,
+		idle_info);
+	smu_print("%s: SMC_MSG_SetDisplayIdleOptimizations idle_info  = %x\n", __func__, idle_info);
+}
+
+void dcn42_smu_enable_phy_refclk_pwrdwn(struct clk_mgr_internal *clk_mgr, bool enable)
+{
+	union display_idle_optimization_u idle_info = { 0 };
+
+	if (!clk_mgr->smu_present)
+		return;
+
+	if (enable) {
+		idle_info.idle_info.df_request_disabled = 1;
+		idle_info.idle_info.phy_ref_clk_off = 1;
+	}
+
+	dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetDisplayIdleOptimizations,
+			idle_info.data);
+	smu_print("%s smu_enable_phy_refclk_pwrdwn  = %d\n", __func__, enable ? 1 : 0);
+}
+
+void dcn42_smu_enable_pme_wa(struct clk_mgr_internal *clk_mgr)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_UpdatePmeRestore,
+			0);
+	smu_print("%s: SMC_MSG_UpdatePmeRestore\n", __func__);
+}
+
+void dcn42_smu_set_dram_addr_high(struct clk_mgr_internal *clk_mgr, uint32_t addr_high)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(clk_mgr,
+			DALSMC_MSG_SetDramAddrHigh, addr_high);
+}
+
+void dcn42_smu_set_dram_addr_low(struct clk_mgr_internal *clk_mgr, uint32_t addr_low)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(clk_mgr,
+			DALSMC_MSG_SetDramAddrLow, addr_low);
+}
+
+void dcn42_smu_transfer_dpm_table_smu_2_dram(struct clk_mgr_internal *clk_mgr)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(clk_mgr,
+			DALSMC_MSG_TransferTableSmu2Dram, TABLE_DPMCLOCKS);
+}
+
+void dcn42_smu_transfer_wm_table_dram_2_smu(struct clk_mgr_internal *clk_mgr)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(clk_mgr,
+			DALSMC_MSG_TransferTableDram2Smu, TABLE_WATERMARKS);
+}
+
+void dcn42_smu_set_zstate_support(struct clk_mgr_internal *clk_mgr, enum dcn_zstate_support_state support)
+{
+	unsigned int msg_id, param, retv;
+
+	if (!clk_mgr->smu_present)
+		return;
+
+	switch (support) {
+
+	case DCN_ZSTATE_SUPPORT_ALLOW:
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = (1 << 10) | (1 << 9) | (1 << 8);
+		smu_print("%s: SMC_MSG_AllowZstatesEntry msg = ALLOW, param = 0x%x\n", __func__, param);
+		break;
+
+	case DCN_ZSTATE_SUPPORT_DISALLOW:
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = 0;
+		smu_print("%s: SMC_MSG_AllowZstatesEntry msg_id = DISALLOW, param = 0x%x\n",  __func__, param);
+		break;
+
+
+	case DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY:
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = (1 << 10);
+		smu_print("%s: SMC_MSG_AllowZstatesEntry msg = ALLOW_Z10_ONLY, param = 0x%x\n", __func__, param);
+		break;
+
+	case DCN_ZSTATE_SUPPORT_ALLOW_Z8_Z10_ONLY:
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = (1 << 10) | (1 << 8);
+		smu_print("%s: SMC_MSG_AllowZstatesEntry msg = ALLOW_Z8_Z10_ONLY, param = 0x%x\n", __func__, param);
+		break;
+
+	case DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY:
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = (1 << 8);
+		smu_print("%s: SMC_MSG_AllowZstatesEntry msg = ALLOW_Z8_ONLY, param = 0x%x\n", __func__, param);
+		break;
+
+	default: //DCN_ZSTATE_SUPPORT_UNKNOWN
+		msg_id = DALSMC_MSG_AllowZstatesEntry;
+		param = 0;
+		break;
+	}
+
+
+	retv = dcn42_smu_send_msg_with_param(
+		clk_mgr,
+		msg_id,
+		param);
+	smu_print("%s:  msg_id = %d, param = 0x%x, return = 0x%x\n", __func__, msg_id, param, retv);
+}
+
+int dcn42_smu_get_dprefclk(struct clk_mgr_internal *clk_mgr)
+{
+	int dprefclk;
+
+	if (!clk_mgr->smu_present)
+		return 0;
+
+	dprefclk = dcn42_smu_send_msg_with_param(clk_mgr,
+						 DALSMC_MSG_GetDprefclkFreq,
+						 0);
+
+	smu_print("%s:  SMU DPREF clk  = %d mhz\n",  __func__, dprefclk);
+	return (int)((long long)dprefclk * 1000);
+}
+
+int dcn42_smu_get_dtbclk(struct clk_mgr_internal *clk_mgr)
+{
+	int dtbclk;
+
+	if (!clk_mgr->smu_present)
+		return 0;
+
+	dtbclk = dcn42_smu_send_msg_with_param(clk_mgr,
+					       DALSMC_MSG_GetDtbclkFreq,
+					       0);
+
+	smu_print("%s: get_dtbclk  = %dmhz\n", __func__, dtbclk);
+	return (int)((long long)dtbclk * 1000);
+}
+/* Arg = 1: Turn DTB on; 0: Turn DTB CLK OFF. when it is on, it is 600MHZ */
+void dcn42_smu_set_dtbclk(struct clk_mgr_internal *clk_mgr, bool enable)
+{
+	if (!clk_mgr->smu_present)
+		return;
+
+	dcn42_smu_send_msg_with_param(
+			clk_mgr,
+			DALSMC_MSG_SetDtbClk,
+			enable);
+	smu_print("%s: smu_set_dtbclk = %d\n", __func__, enable ? 1 : 0);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.h
new file mode 100644
index 0000000000000..8ba7ff04dc05a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.h
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef DAL_DC_42_SMU_H_
+#define DAL_DC_42_SMU_H_
+
+#include "os_types.h"
+
+#ifndef PMFW_DRIVER_IF_H
+#define PMFW_DRIVER_IF_H
+#define PMFW_DRIVER_IF_VERSION 7
+
+typedef enum {
+	DSPCLK_DCFCLK = 0,
+	DSPCLK_DISPCLK,
+	DSPCLK_PIXCLK,
+	DSPCLK_PHYCLK,
+	DSPCLK_COUNT,
+} DSPCLK_e;
+
+typedef struct {
+	uint16_t Freq; // in MHz
+	uint16_t Vid;  // min voltage in SVI3 VID
+} DisplayClockTable_t;
+
+typedef struct {
+	uint16_t MinClock; // This is either DCFCLK or SOCCLK (in MHz)
+	uint16_t MaxClock; // This is either DCFCLK or SOCCLK (in MHz)
+	uint16_t MinMclk;
+	uint16_t MaxMclk;
+
+	uint8_t WmSetting;
+	uint8_t WmType; // Used for normal pstate change or memory retraining
+	uint8_t Padding[2];
+} WatermarkRowGeneric_t;
+
+#define NUM_WM_RANGES 4
+#define WM_PSTATE_CHG 0
+#define WM_RETRAINING 1
+
+typedef enum {
+	WM_SOCCLK = 0,
+	WM_DCFCLK,
+	WM_COUNT,
+} WM_CLOCK_e;
+
+typedef struct {
+	WatermarkRowGeneric_t WatermarkRow[WM_COUNT][NUM_WM_RANGES];
+
+	uint32_t MmHubPadding[7]; // SMU internal use
+} Watermarks_t;
+
+#define NUM_DCFCLK_DPM_LEVELS 8
+#define NUM_DISPCLK_DPM_LEVELS 8
+#define NUM_DPPCLK_DPM_LEVELS 8
+#define NUM_SOCCLK_DPM_LEVELS 8
+#define NUM_VCN_DPM_LEVELS 8
+#define NUM_SOC_VOLTAGE_LEVELS 8
+#define NUM_VPE_DPM_LEVELS 8
+#define NUM_FCLK_DPM_LEVELS 8
+#define NUM_MEM_PSTATE_LEVELS 4
+
+typedef enum {
+	WCK_RATIO_1_1 = 0, // DDR5, Wck:ck is always 1:1;
+	WCK_RATIO_1_2,
+	WCK_RATIO_1_4,
+	WCK_RATIO_MAX
+} WCK_RATIO_e;
+
+typedef struct {
+	uint32_t UClk;
+	uint32_t MemClk;
+	uint32_t Voltage;
+	uint8_t WckRatio;
+	uint8_t Spare[3];
+} MemPstateTable_t;
+
+
+	// Watermarks
+
+
+// Freq in MHz
+// Voltage in milli volts with 2 fractional bits
+typedef struct {
+	uint32_t DcfClocks[NUM_DCFCLK_DPM_LEVELS];
+	uint32_t DispClocks[NUM_DISPCLK_DPM_LEVELS];
+	uint32_t DppClocks[NUM_DPPCLK_DPM_LEVELS];
+	uint32_t SocClocks[NUM_SOCCLK_DPM_LEVELS];
+	uint32_t VClocks[NUM_VCN_DPM_LEVELS];
+	uint32_t DClocks[NUM_VCN_DPM_LEVELS];
+	uint32_t VPEClocks[NUM_VPE_DPM_LEVELS];
+	uint32_t FclkClocks_Freq[NUM_FCLK_DPM_LEVELS];
+	uint32_t FclkClocks_Voltage[NUM_FCLK_DPM_LEVELS];
+	uint32_t SocVoltage[NUM_SOC_VOLTAGE_LEVELS];
+	MemPstateTable_t MemPstateTable[NUM_MEM_PSTATE_LEVELS];
+
+	uint8_t  NumDcfClkLevelsEnabled;
+	uint8_t  NumDispClkLevelsEnabled; //Applies to both Dispclk and Dppclk
+	uint8_t  NumSocClkLevelsEnabled;
+	uint8_t  VcnClkLevelsEnabled;     //Applies to both Vclk and Dclk
+
+	uint8_t  VpeClkLevelsEnabled;
+	uint8_t  NumMemPstatesEnabled;
+	uint8_t  NumFclkLevelsEnabled;
+	uint8_t  spare;
+
+	uint32_t MinGfxClk;
+	uint32_t MaxGfxClk;
+} DpmClocks_t_dcn42;
+
+// Throttler Status Bitmask
+
+
+#define TABLE_BIOS_IF 0    // Called by BIOS
+#define TABLE_WATERMARKS 1 // Called by DAL through VBIOS
+#define TABLE_CUSTOM_DPM 2 // Called by Driver
+#define TABLE_SPARE1 3
+#define TABLE_DPMCLOCKS 4    // Called by Driver
+#define TABLE_MOMENTARY_PM 5 // Called by Tools
+#define TABLE_MODERN_STDBY 6 // Called by Tools for Modern Standby Log
+#define TABLE_SMU_METRICS 7  // Called by Driver
+#define TABLE_COUNT 8
+
+#endif
+
+struct dcn42_watermarks {
+	// Watermarks
+	WatermarkRowGeneric_t WatermarkRow[WM_COUNT][NUM_WM_RANGES];
+	uint32_t MmHubPadding[7]; // SMU internal use
+};
+
+struct dcn42_smu_dpm_clks {
+	DpmClocks_t_dcn42 *dpm_clks;
+	union large_integer mc_address;
+};
+
+struct display_idle_optimization {
+	unsigned int df_request_disabled : 1;
+	unsigned int phy_ref_clk_off : 1;
+	unsigned int s0i2_rdy : 1;
+	unsigned int reserved : 29;
+};
+
+union display_idle_optimization_u {
+	struct display_idle_optimization idle_info;
+	uint32_t data;
+};
+
+int dcn42_smu_get_pmfw_version(struct clk_mgr_internal *clk_mgr);
+int dcn42_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz);
+int dcn42_smu_set_hard_min_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_dcfclk_khz);
+int dcn42_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_min_ds_dcfclk_khz);
+int dcn42_smu_set_dppclk(struct clk_mgr_internal *clk_mgr, int requested_dpp_khz);
+void dcn42_smu_set_display_idle_optimization(struct clk_mgr_internal *clk_mgr, uint32_t idle_info);
+void dcn42_smu_enable_phy_refclk_pwrdwn(struct clk_mgr_internal *clk_mgr, bool enable);
+void dcn42_smu_enable_pme_wa(struct clk_mgr_internal *clk_mgr);
+void dcn42_smu_set_dram_addr_high(struct clk_mgr_internal *clk_mgr, uint32_t addr_high);
+void dcn42_smu_set_dram_addr_low(struct clk_mgr_internal *clk_mgr, uint32_t addr_low);
+void dcn42_smu_transfer_dpm_table_smu_2_dram(struct clk_mgr_internal *clk_mgr);
+void dcn42_smu_transfer_wm_table_dram_2_smu(struct clk_mgr_internal *clk_mgr);
+
+void dcn42_smu_set_zstate_support(struct clk_mgr_internal *clk_mgr, enum dcn_zstate_support_state support);
+void dcn42_smu_set_dtbclk(struct clk_mgr_internal *clk_mgr, bool enable);
+void dcn42_vbios_smu_enable_48mhz_tmdp_refclk_pwrdwn(struct clk_mgr_internal *clk_mgr, bool enable);
+
+int dcn42_smu_get_dtbclk(struct clk_mgr_internal *clk_mgr);
+int dcn42_smu_get_dprefclk(struct clk_mgr_internal *clk_mgr);
+
+#endif /* DAL_DC_42_SMU_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 3e87b6a553be9..8b9c686eefd25 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -442,75 +442,6 @@ static bool set_long_vtotal(struct dc *dc, struct dc_stream_state *stream, struc
 	return false;
 }
 
-/**
- *  dc_stream_adjust_vmin_vmax - look up pipe context & update parts of DRR
- *  @dc:     dc reference
- *  @stream: Initial dc stream state
- *  @adjust: Updated parameters for vertical_total_min and vertical_total_max
- *
- *  Looks up the pipe context of dc_stream_state and updates the
- *  vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh
- *  Rate, which is a power-saving feature that targets reducing panel
- *  refresh rate while the screen is static
- *
- *  Return: %true if the pipe context is found and adjusted;
- *          %false if the pipe context is not found.
- */
-bool dc_stream_adjust_vmin_vmax(struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_crtc_timing_adjust *adjust)
-{
-	int i;
-
-	/*
-	 * Don't adjust DRR while there's bandwidth optimizations pending to
-	 * avoid conflicting with firmware updates.
-	 */
-	if (dc->ctx->dce_version > DCE_VERSION_MAX) {
-		if (dc->optimized_required &&
-			(stream->adjust.v_total_max != adjust->v_total_max ||
-			stream->adjust.v_total_min != adjust->v_total_min)) {
-			stream->adjust.timing_adjust_pending = true;
-			return false;
-		}
-	}
-
-	dc_exit_ips_for_hw_access(dc);
-
-	stream->adjust.v_total_max = adjust->v_total_max;
-	stream->adjust.v_total_mid = adjust->v_total_mid;
-	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
-	stream->adjust.v_total_min = adjust->v_total_min;
-	stream->adjust.allow_otg_v_count_halt = adjust->allow_otg_v_count_halt;
-
-	if (dc->caps.max_v_total != 0 &&
-		(adjust->v_total_max > dc->caps.max_v_total || adjust->v_total_min > dc->caps.max_v_total)) {
-		stream->adjust.timing_adjust_pending = false;
-		if (adjust->allow_otg_v_count_halt)
-			return set_long_vtotal(dc, stream, adjust);
-		else
-			return false;
-	}
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
-
-		if (pipe->stream == stream && pipe->stream_res.tg) {
-			dc->hwss.set_drr(&pipe,
-					1,
-					*adjust);
-			stream->adjust.timing_adjust_pending = false;
-
-			if (dc->hwss.notify_cursor_offload_drr_update)
-				dc->hwss.notify_cursor_offload_drr_update(dc, dc->current_state, stream);
-
-			return true;
-		}
-	}
-
-	return false;
-}
-
 /**
  * dc_stream_get_last_used_drr_vtotal - Looks up the pipe context of
  * dc_stream_state and gets the last VTOTAL used by DRR (Dynamic Refresh Rate)
@@ -1274,6 +1205,8 @@ static void dc_update_visual_confirm_color(struct dc *dc, struct dc_state *conte
 				get_fams2_visual_confirm_color(dc, context, pipe_ctx, &(pipe_ctx->visual_confirm_color));
 			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_VABC)
 				get_vabc_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_BOOSTED_REFRESH_RATE)
+				get_refresh_rate_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
 		}
 	}
 }
@@ -1323,6 +1256,83 @@ void dc_get_visual_confirm_for_stream(
 	}
 }
 
+/**
+ *  dc_stream_adjust_vmin_vmax - look up pipe context & update parts of DRR
+ *  @dc:     dc reference
+ *  @stream: Initial dc stream state
+ *  @adjust: Updated parameters for vertical_total_min and vertical_total_max
+ *
+ *  Looks up the pipe context of dc_stream_state and updates the
+ *  vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh
+ *  Rate, which is a power-saving feature that targets reducing panel
+ *  refresh rate while the screen is static
+ *
+ *  Return: %true if the pipe context is found and adjusted;
+ *          %false if the pipe context is not found.
+ */
+bool dc_stream_adjust_vmin_vmax(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_crtc_timing_adjust *adjust)
+{
+	int i;
+
+	/*
+	 * Don't adjust DRR while there's bandwidth optimizations pending to
+	 * avoid conflicting with firmware updates.
+	 */
+	if (dc->ctx->dce_version > DCE_VERSION_MAX) {
+		if (dc->optimized_required &&
+			(stream->adjust.v_total_max != adjust->v_total_max ||
+			stream->adjust.v_total_min != adjust->v_total_min)) {
+			stream->adjust.timing_adjust_pending = true;
+			return false;
+		}
+	}
+
+	dc_exit_ips_for_hw_access(dc);
+
+	stream->adjust.v_total_max = adjust->v_total_max;
+	stream->adjust.v_total_mid = adjust->v_total_mid;
+	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
+	stream->adjust.v_total_min = adjust->v_total_min;
+	stream->adjust.allow_otg_v_count_halt = adjust->allow_otg_v_count_halt;
+
+	if (dc->caps.max_v_total != 0 &&
+		(adjust->v_total_max > dc->caps.max_v_total || adjust->v_total_min > dc->caps.max_v_total)) {
+		stream->adjust.timing_adjust_pending = false;
+		if (adjust->allow_otg_v_count_halt)
+			return set_long_vtotal(dc, stream, adjust);
+		else
+			return false;
+	}
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg) {
+			dc->hwss.set_drr(&pipe,
+					1,
+					*adjust);
+			stream->adjust.timing_adjust_pending = false;
+
+			if (dc->debug.visual_confirm == VISUAL_CONFIRM_BOOSTED_REFRESH_RATE) {
+				if (pipe->stream && pipe->plane_state) {
+					dc_update_visual_confirm_color(dc, dc->current_state, pipe);
+					dc->hwss.update_visual_confirm_color(dc, pipe, pipe->plane_res.hubp->mpcc_id);
+
+				}
+			}
+
+			if (dc->hwss.notify_cursor_offload_drr_update)
+				dc->hwss.notify_cursor_offload_drr_update(dc, dc->current_state, stream);
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
 static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
 {
 	int i, j;
@@ -2607,6 +2617,16 @@ void dc_post_update_surfaces_to_stream(struct dc *dc)
 	dc->optimized_required = false;
 }
 
+void dc_get_default_tiling_info(const struct dc *dc, struct dc_tiling_info *tiling_info)
+{
+	if (!dc || !tiling_info)
+		return;
+	if (dc->res_pool && dc->res_pool->funcs && dc->res_pool->funcs->get_default_tiling_info) {
+		dc->res_pool->funcs->get_default_tiling_info(tiling_info);
+		return;
+	}
+}
+
 bool dc_set_generic_gpio_for_stereo(bool enable,
 		struct gpio_service *gpio_service)
 {
@@ -2885,16 +2905,27 @@ static struct surface_update_descriptor det_surface_update(
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->blend_tf || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
+	if (u->cm || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
 		update_flags->bits.gamma_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->lut3d_func || u->func_shaper) {
+	if (u->cm && (u->cm->flags.bits.lut3d_enable || u->surface->cm.flags.bits.lut3d_enable)) {
 		update_flags->bits.lut_3d = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
+	if (u->cm && u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable &&
+			u->cm->flags.bits.lut3d_enable && u->surface->cm.flags.bits.lut3d_enable) {
+		/* Toggling 3DLUT loading between DMA and Host is illegal */
+		BREAK_TO_DEBUGGER();
+	}
+
+	if (u->cm && u->cm->flags.bits.lut3d_enable && !u->cm->flags.bits.lut3d_dma_enable) {
+		/* Host loading 3DLUT requires full update but only stream lock  */
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+	}
+
 	if (u->hdr_mult.value)
 		if (u->hdr_mult.value != u->surface->hdr_mult.value) {
 			// TODO: Should be fast?
@@ -2909,24 +2940,15 @@ static struct surface_update_descriptor det_surface_update(
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		}
 
-	if (u->cm2_params) {
-		if (u->cm2_params->component_settings.shaper_3dlut_setting != u->surface->mcm_shaper_3dlut_setting
-				|| u->cm2_params->component_settings.lut1d_enable != u->surface->mcm_lut1d_enable
-				|| u->cm2_params->cm2_luts.lut3d_data.lut3d_src != u->surface->mcm_luts.lut3d_data.lut3d_src) {
-			update_flags->bits.mcm_transfer_function_enable_change = 1;
-			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
-		}
-	}
-
-	if (update_flags->bits.lut_3d &&
-			u->surface->mcm_luts.lut3d_data.lut3d_src != DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
-		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	if (u->cm_hist_control) {
+		update_flags->bits.cm_hist_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
-
 	if (check_config->enable_legacy_fast_update &&
 			(update_flags->bits.gamma_change ||
 			update_flags->bits.gamut_remap_change ||
 			update_flags->bits.input_csc_change ||
+			update_flags->bits.cm_hist_change ||
 			update_flags->bits.coeff_reduction_change)) {
 		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
@@ -3064,7 +3086,7 @@ struct surface_update_descriptor dc_check_update_surfaces_for_stream(
 {
 	if (stream_update)
 		stream_update->stream->update_flags.raw = 0;
-	for (size_t i = 0; i < surface_count; i++)
+	for (int i = 0; i < surface_count; i++)
 		updates[i].surface->update_flags.raw = 0;
 
 	return check_update_surfaces_for_stream(check_config, updates, surface_count, stream_update);
@@ -3158,6 +3180,11 @@ static void copy_surface_update_to_plane(
 		surface->gamma_correction.type =
 			srf_update->gamma->type;
 	}
+	if (srf_update->cm_hist_control) {
+		memcpy(&surface->cm_hist_control,
+			srf_update->cm_hist_control,
+			sizeof(surface->cm_hist_control));
+	}
 
 	if (srf_update->in_transfer_func) {
 		surface->in_transfer_func.sdr_ref_white_level =
@@ -3171,24 +3198,12 @@ static void copy_surface_update_to_plane(
 			sizeof(struct dc_transfer_func_distributed_points));
 	}
 
-	if (srf_update->cm2_params) {
-		surface->mcm_shaper_3dlut_setting = srf_update->cm2_params->component_settings.shaper_3dlut_setting;
-		surface->mcm_lut1d_enable = srf_update->cm2_params->component_settings.lut1d_enable;
-		surface->mcm_luts = srf_update->cm2_params->cm2_luts;
-	}
-
-	if (srf_update->func_shaper) {
-		memcpy(&surface->in_shaper_func, srf_update->func_shaper,
-		sizeof(surface->in_shaper_func));
-
-		if (surface->mcm_shaper_3dlut_setting >= DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER)
-			surface->mcm_luts.shaper = &surface->in_shaper_func;
+	/* Shaper, 3DLUT, 1DLUT */
+	if (srf_update->cm) {
+		memcpy(&surface->cm, srf_update->cm,
+				sizeof(surface->cm));
 	}
 
-	if (srf_update->lut3d_func)
-		memcpy(&surface->lut3d_func, srf_update->lut3d_func,
-		sizeof(surface->lut3d_func));
-
 	if (srf_update->hdr_mult.value)
 		surface->hdr_mult =
 				srf_update->hdr_mult;
@@ -3197,17 +3212,6 @@ static void copy_surface_update_to_plane(
 		surface->sdr_white_level_nits =
 				srf_update->sdr_white_level_nits;
 
-	if (srf_update->blend_tf) {
-		memcpy(&surface->blend_tf, srf_update->blend_tf,
-		sizeof(surface->blend_tf));
-
-		if (surface->mcm_lut1d_enable)
-			surface->mcm_luts.lut1d_func = &surface->blend_tf;
-	}
-
-	if (srf_update->cm2_params || srf_update->blend_tf)
-		surface->lut_bank_a = !surface->lut_bank_a;
-
 	if (srf_update->input_csc_color_matrix)
 		surface->input_csc_color_matrix =
 			*srf_update->input_csc_color_matrix;
@@ -4491,11 +4495,9 @@ static void commit_planes_for_stream(struct dc *dc,
 				if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
 					continue;
 
-				if (srf_updates[i].cm2_params &&
-						srf_updates[i].cm2_params->cm2_luts.lut3d_data.lut3d_src ==
-								DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM &&
-						srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting ==
-								DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT &&
+				if (srf_updates[i].cm &&
+						srf_updates[i].cm->flags.bits.lut3d_enable &&
+						srf_updates[i].cm->flags.bits.lut3d_dma_enable &&
 						dc->hwss.trigger_3dlut_dma_load)
 					dc->hwss.trigger_3dlut_dma_load(dc, pipe_ctx);
 
@@ -5063,6 +5065,7 @@ void populate_fast_updates(struct dc_fast_update *fast_update,
 		fast_update[i].input_csc_color_matrix = srf_updates[i].input_csc_color_matrix;
 		fast_update[i].coeff_reduction_factor = srf_updates[i].coeff_reduction_factor;
 		fast_update[i].cursor_csc_color_matrix = srf_updates[i].cursor_csc_color_matrix;
+		fast_update[i].cm_hist_control = srf_updates[i].cm_hist_control;
 	}
 }
 
@@ -5080,6 +5083,7 @@ static bool fast_updates_exist(const struct dc_fast_update *fast_update, int sur
 				fast_update[i].gamut_remap_matrix ||
 				fast_update[i].input_csc_color_matrix ||
 				fast_update[i].cursor_csc_color_matrix ||
+				fast_update[i].cm_hist_control ||
 				fast_update[i].coeff_reduction_factor)
 			return true;
 	}
@@ -5100,6 +5104,7 @@ bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_
 				fast_update[i].gamma ||
 				fast_update[i].gamut_remap_matrix ||
 				fast_update[i].coeff_reduction_factor ||
+				fast_update[i].cm_hist_control ||
 				fast_update[i].cursor_csc_color_matrix)
 			return true;
 	}
@@ -5141,6 +5146,12 @@ static bool full_update_required(
 		const struct dc_stream_update *stream_update,
 		const struct dc_stream_state *stream)
 {
+	const union dc_plane_cm_flags blend_only_flags = {
+		.bits = {
+			.blend_enable = 1,
+		}
+	};
+
 	if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream))
 		return true;
 
@@ -5153,14 +5164,12 @@ static bool full_update_required(
 				(srf_updates[i].sdr_white_level_nits &&
 				srf_updates[i].sdr_white_level_nits != srf_updates->surface->sdr_white_level_nits) ||
 				srf_updates[i].in_transfer_func ||
-				srf_updates[i].func_shaper ||
-				srf_updates[i].lut3d_func ||
 				srf_updates[i].surface->force_full_update ||
 				(srf_updates[i].flip_addr &&
 				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface) ||
-				(srf_updates[i].cm2_params &&
-				 (srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting != srf_updates[i].surface->mcm_shaper_3dlut_setting ||
-				  srf_updates[i].cm2_params->component_settings.lut1d_enable != srf_updates[i].surface->mcm_lut1d_enable))))
+				(srf_updates[i].cm &&
+				((srf_updates[i].cm->flags.all != blend_only_flags.all && srf_updates[i].cm->flags.all != 0) ||
+				(srf_updates[i].surface->cm.flags.all != blend_only_flags.all && srf_updates[i].surface->cm.flags.all != 0)))))
 			return true;
 	}
 
@@ -5540,6 +5549,9 @@ void dc_power_down_on_boot(struct dc *dc)
 {
 	if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW &&
 	    dc->hwss.power_down_on_boot) {
+		if (dc->current_state->stream_count > 0)
+			return;
+
 		if (dc->caps.ips_support)
 			dc_exit_ips_for_hw_access(dc);
 		dc->hwss.power_down_on_boot(dc);
@@ -5551,12 +5563,12 @@ void dc_set_power_state(struct dc *dc, enum dc_acpi_cm_power_state power_state)
 	if (!dc->current_state)
 		return;
 
+	dc_exit_ips_for_hw_access(dc);
+
 	switch (power_state) {
 	case DC_ACPI_CM_POWER_STATE_D0:
 		dc_state_construct(dc, dc->current_state);
 
-		dc_exit_ips_for_hw_access(dc);
-
 		dc_z10_restore(dc);
 
 		dc_dmub_srv_notify_fw_dc_power_state(dc->ctx->dmub_srv, power_state);
@@ -5932,6 +5944,7 @@ bool dc_is_dmub_outbox_supported(struct dc *dc)
 
 	case AMDGPU_FAMILY_GC_11_0_1:
 	case AMDGPU_FAMILY_GC_11_5_0:
+	case AMDGPU_FAMILY_GC_11_5_4:
 		if (!dc->debug.dpia_debug.bits.disable_dpia)
 			return true;
 	break;
@@ -6884,7 +6897,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 			struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 
 			/* MPCC blending tree and mode control - capture actual blend configuration */
-			state->mpc.mpcc_mode[i] = (plane_state->blend_tf.type != TF_TYPE_BYPASS) ? 1 : 0;
+			state->mpc.mpcc_mode[i] = (plane_state->cm.blend_func.type != TF_TYPE_BYPASS) ? 1 : 0;
 			state->mpc.mpcc_alpha_blend_mode[i] = plane_state->per_pixel_alpha ? 1 : 0;
 			state->mpc.mpcc_alpha_multiplied_mode[i] = plane_state->pre_multiplied_alpha ? 1 : 0;
 			state->mpc.mpcc_blnd_active_overlap_only[i] = 0; /* Default - no overlap restriction */
@@ -7282,6 +7295,23 @@ static bool update_planes_and_stream_prepare_v3(
 	ASSERT(scratch->flow == UPDATE_V3_FLOW_INVALID);
 	dc_exit_ips_for_hw_access(scratch->dc);
 
+	/* HWSS path determination needs to be done prior to updating the surface and stream states. */
+	struct dc_fast_update fast_update[MAX_SURFACES] = { 0 };
+
+	populate_fast_updates(fast_update,
+			      scratch->surface_updates,
+			      scratch->surface_count,
+			      scratch->stream_update);
+
+	const bool is_hwss_fast_path_only =
+		fast_update_only(scratch->dc,
+				 fast_update,
+				 scratch->surface_updates,
+				 scratch->surface_count,
+				 scratch->stream_update,
+				 scratch->stream) &&
+		!scratch->dc->check_config.enable_legacy_fast_update;
+
 	if (!update_planes_and_stream_state(
 			scratch->dc,
 			scratch->surface_updates,
@@ -7297,26 +7327,7 @@ static bool update_planes_and_stream_prepare_v3(
 	if (scratch->new_context == scratch->dc->current_state) {
 		ASSERT(scratch->update_type < UPDATE_TYPE_FULL);
 
-		// TODO: Do we need this to be alive in execute?
-		struct dc_fast_update fast_update[MAX_SURFACES] = { 0 };
-
-		populate_fast_updates(
-				fast_update,
-				scratch->surface_updates,
-				scratch->surface_count,
-				scratch->stream_update
-		);
-		const bool fast = fast_update_only(
-				scratch->dc,
-				fast_update,
-				scratch->surface_updates,
-				scratch->surface_count,
-				scratch->stream_update,
-				scratch->stream
-		)
-		// TODO: Can this be used to skip `populate_fast_updates`?
-				&& !scratch->dc->check_config.enable_legacy_fast_update;
-		scratch->flow = fast
+		scratch->flow = is_hwss_fast_path_only
 				? UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST
 				: UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL;
 		return true;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 052d573408c3e..5b3695e72e196 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2015-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -846,6 +846,16 @@ void hwss_build_fast_sequence(struct dc *dc,
 					block_sequence[*num_steps].func = DPP_PROGRAM_BIAS_AND_SCALE;
 					(*num_steps)++;
 				}
+				if (current_mpc_pipe->plane_state->update_flags.bits.cm_hist_change) {
+					block_sequence[*num_steps].params.control_cm_hist_params.dpp
+						= current_mpc_pipe->plane_res.dpp;
+					block_sequence[*num_steps].params.control_cm_hist_params.cm_hist_control
+						= current_mpc_pipe->plane_state->cm_hist_control;
+					block_sequence[*num_steps].params.control_cm_hist_params.color_space
+						= current_mpc_pipe->plane_state->color_space;
+					block_sequence[*num_steps].func = DPP_PROGRAM_CM_HIST;
+					(*num_steps)++;
+				}
 			}
 			if (hws->funcs.set_output_transfer_func && current_mpc_pipe->stream->update_flags.bits.out_tf) {
 				block_sequence[*num_steps].params.set_output_transfer_func_params.dc = dc;
@@ -1029,6 +1039,9 @@ void hwss_execute_sequence(struct dc *dc,
 		case HUBP_PROGRAM_MCACHE_ID:
 			hwss_program_mcache_id_and_split_coordinate(params);
 			break;
+		case DPP_PROGRAM_CM_HIST:
+			hwss_program_cm_hist(params);
+			break;
 		case PROGRAM_CURSOR_UPDATE_NOW:
 			dc->hwss.program_cursor_offload_now(
 				params->program_cursor_update_now_params.dc,
@@ -2054,6 +2067,16 @@ void hwss_program_mcache_id_and_split_coordinate(union block_sequence_params *pa
 
 }
 
+void hwss_program_cm_hist(union block_sequence_params *params)
+{
+	struct dpp *dpp = params->control_cm_hist_params.dpp;
+
+	if (dpp && dpp->funcs->dpp_cm_hist_control)
+		dpp->funcs->dpp_cm_hist_control(dpp,
+			params->control_cm_hist_params.cm_hist_control,
+			params->control_cm_hist_params.color_space);
+}
+
 void get_surface_tile_visual_confirm_color(
 		struct pipe_ctx *pipe_ctx,
 		struct tg_color *color)
@@ -3370,6 +3393,20 @@ void hwss_add_opp_program_bit_depth_reduction(struct block_sequence_state *seq_s
 	}
 }
 
+void hwss_add_dpp_program_cm_hist(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		struct cm_hist_control cm_hist_control,
+		enum dc_color_space color_space)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DPP_PROGRAM_CM_HIST;
+		seq_state->steps[*seq_state->num_steps].params.control_cm_hist_params.dpp = dpp;
+		seq_state->steps[*seq_state->num_steps].params.control_cm_hist_params.cm_hist_control = cm_hist_control;
+		seq_state->steps[*seq_state->num_steps].params.control_cm_hist_params.color_space = color_space;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_dc_ip_request_cntl(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		bool enable)
@@ -4071,3 +4108,24 @@ void hwss_add_tg_get_frame_count(struct block_sequence_state *seq_state,
 		(*seq_state->num_steps)++;
 	}
 }
+
+
+void get_refresh_rate_confirm_color(struct pipe_ctx *pipe_ctx, struct tg_color *color)
+{
+	uint32_t color_value = MAX_TG_COLOR_VALUE;
+	unsigned int refresh_rate = 0;
+	uint32_t scaling_factor = 0;
+	if (pipe_ctx && pipe_ctx->stream && color) {
+		refresh_rate = (pipe_ctx->stream->timing.pix_clk_100hz * 100) / (pipe_ctx->stream->adjust.v_total_max * pipe_ctx->stream->timing.h_total);
+
+		uint32_t min_refresh_rate = pipe_ctx->stream->timing.min_refresh_in_uhz / 1000000;
+		uint32_t max_refresh_rate = pipe_ctx->stream->timing.max_refresh_in_uhz / 1000000;
+
+		if (max_refresh_rate - min_refresh_rate)
+			scaling_factor = MAX_TG_COLOR_VALUE * (refresh_rate - min_refresh_rate) / (max_refresh_rate - min_refresh_rate);
+
+		pipe_ctx->visual_confirm_color.color_r_cr = color_value;
+		pipe_ctx->visual_confirm_color.color_g_y = scaling_factor;
+		pipe_ctx->visual_confirm_color.color_b_cb = color_value;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 03d125f794b05..8271b12c1a66d 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -78,6 +78,7 @@
 #include "dcn351/dcn351_resource.h"
 #include "dcn36/dcn36_resource.h"
 #include "dcn401/dcn401_resource.h"
+#include "dcn42/dcn42_resource.h"
 #if defined(CONFIG_DRM_AMD_DC_FP)
 #include "dc_spl_translate.h"
 #endif
@@ -252,6 +253,9 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id)
 			ASICREV_IS_GC_12_0_0_A0(asic_id.hw_internal_rev))
 			dc_version = DCN_VERSION_4_01;
 		break;
+	case AMDGPU_FAMILY_GC_11_5_4:
+			dc_version = DCN_VERSION_4_2;
+	break;
 	default:
 		dc_version = DCE_VERSION_UNKNOWN;
 		break;
@@ -368,6 +372,9 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 	case DCN_VERSION_4_01:
 		res_pool = dcn401_create_resource_pool(init_data, dc);
 		break;
+	case DCN_VERSION_4_2:
+		res_pool = dcn42_create_resource_pool(init_data, dc);
+		break;
 #endif /* CONFIG_DRM_AMD_DC_FP */
 	default:
 		break;
@@ -2424,7 +2431,6 @@ static void resource_log_pipe_for_stream(struct dc *dc, struct dc_state *state,
 
 	int slice_idx, dpp_idx, plane_idx, slice_count, dpp_count;
 	bool is_primary;
-	DC_LOGGER_INIT(dc->ctx->logger);
 
 	slice_count = resource_get_opp_heads_for_otg_master(otg_master,
 			&state->res_ctx, opp_heads);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index baf820e6eae85..908f79b021020 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -33,6 +33,7 @@
 #include "dc_dmub_srv.h"
 #include "dc_state_priv.h"
 #include "dc_stream_priv.h"
+#include "dce/dmub_hw_lock_mgr.h"
 
 #define DC_LOGGER dc->ctx->logger
 #ifndef MIN
@@ -171,10 +172,12 @@ struct dc_stream_state *dc_create_stream_for_sink(
 		goto fail;
 
 	stream = kzalloc_obj(struct dc_stream_state, GFP_ATOMIC);
+
 	if (stream == NULL)
 		goto fail;
 
 	stream->update_scratch = kzalloc((int32_t) dc_update_scratch_space_size(), GFP_ATOMIC);
+
 	if (stream->update_scratch == NULL)
 		goto fail;
 
@@ -245,7 +248,6 @@ const struct dc_stream_status *dc_stream_get_status_const(
 	const struct dc_stream_state *stream)
 {
 	struct dc *dc = stream->ctx->dc;
-
 	return dc_state_get_stream_status(dc->current_state, stream);
 }
 
@@ -257,6 +259,7 @@ void program_cursor_attributes(
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+	bool unlock_dmub = false;
 
 	if (!stream)
 		return;
@@ -275,6 +278,12 @@ void program_cursor_attributes(
 			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
 				dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
 			} else {
+				if (dc->hwss.dmub_hw_control_lock && pipe_ctx->stream &&
+				    should_use_dmub_inbox0_lock_for_link(dc, pipe_ctx->stream->link)) {
+					dc->hwss.dmub_hw_control_lock(dc, dc->current_state, true);
+					unlock_dmub = true;
+				}
+
 				dc->hwss.cursor_lock(dc, pipe_to_program, true);
 				if (pipe_to_program->next_odm_pipe)
 					dc->hwss.cursor_lock(dc, pipe_to_program->next_odm_pipe, true);
@@ -297,6 +306,9 @@ void program_cursor_attributes(
 			dc->hwss.cursor_lock(dc, pipe_to_program, false);
 			if (pipe_to_program->next_odm_pipe)
 				dc->hwss.cursor_lock(dc, pipe_to_program->next_odm_pipe, false);
+
+			if (unlock_dmub)
+				dc->hwss.dmub_hw_control_lock(dc, dc->current_state, false);
 		}
 	}
 }
@@ -404,6 +416,7 @@ void program_cursor_position(
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+	bool unlock_dmub = false;
 
 	if (!stream)
 		return;
@@ -423,10 +436,16 @@ void program_cursor_position(
 		if (!pipe_to_program) {
 			pipe_to_program = pipe_ctx;
 
-			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update)
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
 				dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
-			else
+			} else {
+				if (dc->hwss.dmub_hw_control_lock && pipe_ctx->stream &&
+				    should_use_dmub_inbox0_lock_for_link(dc, pipe_ctx->stream->link)) {
+					dc->hwss.dmub_hw_control_lock(dc, dc->current_state, true);
+					unlock_dmub = true;
+				}
 				dc->hwss.cursor_lock(dc, pipe_to_program, true);
+			}
 		}
 
 		dc->hwss.set_cursor_position(pipe_ctx);
@@ -438,10 +457,14 @@ void program_cursor_position(
 	}
 
 	if (pipe_to_program) {
-		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update)
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
 			dc->hwss.commit_cursor_offload_update(dc, pipe_to_program);
-		else
+		} else {
 			dc->hwss.cursor_lock(dc, pipe_to_program, false);
+
+			if (unlock_dmub)
+				dc->hwss.dmub_hw_control_lock(dc, dc->current_state, false);
+		}
 	}
 }
 
@@ -523,8 +546,10 @@ bool dc_stream_program_cursor_position(
 				struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
 
 				/* trigger event on first pipe with current stream */
-				if (stream == pipe_ctx->stream) {
-					pipe_ctx->stream_res.tg->funcs->program_manual_trigger(pipe_ctx->stream_res.tg);
+				if (stream == pipe_ctx->stream &&
+				pipe_ctx->stream_res.tg->funcs->program_manual_trigger) {
+					pipe_ctx->stream_res.tg->funcs->program_manual_trigger(
+					pipe_ctx->stream_res.tg);
 					break;
 				}
 			}
@@ -984,7 +1009,6 @@ void dc_stream_release_3dlut_for_stream(
 	if (rmcm_3dlut) {
 		rmcm_3dlut->isInUse = false;
 		rmcm_3dlut->stream  = NULL;
-		rmcm_3dlut->protection_bits = 0;
 	}
 }
 
@@ -996,7 +1020,6 @@ void dc_stream_init_rmcm_3dlut(struct dc *dc)
 	for (int i = 0; i < num_rmcm; i++) {
 		dc->res_pool->rmcm_3dlut[i].isInUse = false;
 		dc->res_pool->rmcm_3dlut[i].stream = NULL;
-		dc->res_pool->rmcm_3dlut[i].protection_bits = 0;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index d4c40b44d9093..5f12dcca7f711 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -139,6 +139,9 @@ const struct dc_plane_status *dc_plane_get_status(
 
 		if (pipe_ctx->plane_state && flags.bits.address)
 			pipe_ctx->plane_state->status.is_flip_pending = false;
+		if (pipe_ctx->plane_state && flags.bits.histogram)
+			memset(&pipe_ctx->plane_state->status.cm_hist, 0,
+				sizeof(pipe_ctx->plane_state->status.cm_hist));
 
 		break;
 	}
@@ -154,6 +157,12 @@ const struct dc_plane_status *dc_plane_get_status(
 
 		if (flags.bits.address)
 			dc->hwss.update_pending_status(pipe_ctx);
+		if (flags.bits.histogram) {
+			struct dpp *dpp = pipe_ctx->plane_res.dpp;
+
+			if (dpp && dpp->funcs->dpp_cm_hist_read)
+				dpp->funcs->dpp_cm_hist_read(dpp, &pipe_ctx->plane_state->status.cm_hist);
+		}
 	}
 
 	return plane_status;
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 4c4239cac863d..0b48feaba131e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2023 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -63,7 +63,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.369"
+#define DC_VER "3.2.374"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -581,6 +581,7 @@ enum visual_confirm {
 	VISUAL_CONFIRM_HW_CURSOR = 20,
 	VISUAL_CONFIRM_VABC = 21,
 	VISUAL_CONFIRM_DCC = 22,
+	VISUAL_CONFIRM_BOOSTED_REFRESH_RATE = 23,
 	VISUAL_CONFIRM_EXPLICIT = 0x80000000,
 };
 
@@ -733,6 +734,8 @@ struct dc_clocks {
 	struct {
 		uint8_t base_efficiency; //LP1
 		uint8_t low_power_efficiency; //LP2
+		uint8_t z8_stutter_efficiency;
+		int z8_stutter_period;
 	} stutter_efficiency;
 };
 
@@ -1211,6 +1214,7 @@ struct dc_debug_options {
 	unsigned int num_fast_flips_to_steady_state_override;
 	bool enable_dmu_recovery;
 	unsigned int force_vmin_threshold;
+	bool enable_otg_frame_sync_pwa;
 };
 
 
@@ -1400,15 +1404,50 @@ struct lut_mem_mapping {
 struct dc_rmcm_3dlut {
 	bool isInUse;
 	const struct dc_stream_state *stream;
-	uint8_t protection_bits;
 };
 
 struct dc_3dlut {
 	struct kref refcount;
 	struct tetrahedral_params lut_3d;
-	struct fixed31_32 hdr_multiplier;
 	union dc_3dlut_state state;
 };
+
+/* 3DLUT DMA (Fast Load) params */
+struct dc_3dlut_dma {
+	struct dc_plane_address addr;
+	enum dc_cm_lut_swizzle swizzle;
+	enum dc_cm_lut_pixel_format format;
+	uint16_t bias; /* FP1.5.10 */
+	uint16_t scale; /* FP1.5.10 */
+	enum dc_cm_lut_size size;
+};
+
+/* color manager */
+union dc_plane_cm_flags {
+	unsigned int all;
+	struct {
+		unsigned int shaper_enable    : 1;
+		unsigned int lut3d_enable     : 1;
+		unsigned int blend_enable     : 1;
+		/* whether legacy (lut3d_func) or DMA is valid */
+		unsigned int lut3d_dma_enable : 1;
+		/* RMCM lut to be used instead of MCM */
+		unsigned int rmcm_enable	 : 1;
+		unsigned int reserved: 27;
+	} bits;
+};
+
+struct dc_plane_cm {
+	struct kref refcount;
+	struct dc_transfer_func shaper_func;
+	union {
+		struct dc_3dlut lut3d_func;
+		struct dc_3dlut_dma lut3d_dma;
+	};
+	struct dc_transfer_func blend_func;
+	union dc_plane_cm_flags flags;
+};
+
 /*
  * This structure is filled in by dc_surface_get_status and contains
  * the last requested address and the currently active address so the called
@@ -1419,6 +1458,7 @@ struct dc_plane_status {
 	struct dc_plane_address current_address;
 	bool is_flip_pending;
 	bool is_right_eye;
+	struct cm_hist cm_hist;
 };
 
 union surface_update_flags {
@@ -1455,6 +1495,7 @@ union surface_update_flags {
 		uint32_t mcm_transfer_function_enable_change:1; /* disable or enable MCM transfer func */
 		uint32_t full_update:1;
 		uint32_t sdr_white_level_nits:1;
+		uint32_t cm_hist_change:1;
 	} bits;
 
 	uint32_t raw;
@@ -1484,14 +1525,18 @@ struct dc_plane_state {
 	struct fixed31_32 hdr_mult;
 	struct colorspace_transform gamut_remap_matrix;
 
-	// TODO: No longer used, remove
-	struct dc_hdr_static_metadata hdr_static_ctx;
-
 	enum dc_color_space color_space;
 
+	bool lut_bank_a;
+	struct dc_hdr_static_metadata hdr_static_ctx;
 	struct dc_3dlut lut3d_func;
 	struct dc_transfer_func in_shaper_func;
 	struct dc_transfer_func blend_tf;
+	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
+	bool mcm_lut1d_enable;
+	struct dc_cm2_func_luts mcm_luts;
+	enum mpcc_movable_cm_location mcm_location;
+	struct dc_plane_cm cm;
 
 	struct dc_transfer_func *gamcor_tf;
 	enum surface_pixel_format format;
@@ -1528,17 +1573,13 @@ struct dc_plane_state {
 
 	bool is_statically_allocated;
 	enum chroma_cositing cositing;
-	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
-	bool mcm_lut1d_enable;
-	struct dc_cm2_func_luts mcm_luts;
-	bool lut_bank_a;
-	enum mpcc_movable_cm_location mcm_location;
 	struct dc_csc_transform cursor_csc_color_matrix;
 	bool adaptive_sharpness_en;
 	int adaptive_sharpness_policy;
 	int sharpness_level;
 	enum linear_light_scaling linear_light_scaling;
 	unsigned int sdr_white_level_nits;
+	struct cm_hist_control cm_hist_control;
 	struct spl_sharpness_range sharpness_range;
 	enum sharpness_range_source sharpness_source;
 };
@@ -1682,6 +1723,10 @@ struct dc_scratch_space {
 	struct panel_cntl *panel_cntl;
 	struct link_encoder *link_enc;
 	struct graphics_object_id link_id;
+
+	/* External encoder eg. NUTMEG or TRAVIS used on CIK APUs. */
+	struct graphics_object_id ext_enc_id;
+
 	/* Endpoint type distinguishes display endpoints which do not have entries
 	 * in the BIOS connector table from those that do. Helps when tracking link
 	 * encoder to display endpoint assignments.
@@ -1843,6 +1888,7 @@ struct dc_fast_update {
 	struct dc_transfer_func *out_transfer_func;
 	struct dc_csc_transform *output_csc_transform;
 	const struct dc_csc_transform *cursor_csc_color_matrix;
+	struct cm_hist_control *cm_hist_control;
 };
 
 struct dc_surface_update {
@@ -1872,9 +1918,11 @@ struct dc_surface_update {
 	 * change cm2_params.cm2_luts: Fast update
 	 */
 	const struct dc_cm2_parameters *cm2_params;
+	const struct dc_plane_cm *cm;
 	const struct dc_csc_transform *cursor_csc_color_matrix;
 	unsigned int sdr_white_level_nits;
 	struct dc_bias_and_scale bias_and_scale;
+	struct cm_hist_control *cm_hist_control;
 };
 
 struct dc_underflow_debug_data {
@@ -1915,9 +1963,22 @@ struct dc_3dlut *dc_create_3dlut_func(void);
 void dc_3dlut_func_release(struct dc_3dlut *lut);
 void dc_3dlut_func_retain(struct dc_3dlut *lut);
 
+struct dc_plane_cm *dc_plane_cm_create(void);
+void dc_plane_cm_release(struct dc_plane_cm *cm);
+void dc_plane_cm_retain(struct dc_plane_cm *cm);
+
 void dc_post_update_surfaces_to_stream(
 		struct dc *dc);
 
+/*
+ * dc_get_default_tiling_info() - Retrieve an ASIC-appropriate default tiling
+ * description for (typically) linear surfaces.
+ *
+ * This is used by OS/DM paths that need a valid, fully-initialized tiling
+ * description without hardcoding gfx-version specifics in the caller.
+ */
+void dc_get_default_tiling_info(const struct dc *dc, struct dc_tiling_info *tiling_info);
+
 /**
  * struct dc_validation_set - Struct to store surface/stream associations for validation
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
index 40d7a7d83c406..6f96c5cf39fe8 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
@@ -97,11 +97,12 @@ struct dc_vbios_funcs {
 	enum bp_result (*encoder_control)(
 		struct dc_bios *bios,
 		struct bp_encoder_control *cntl);
+	enum bp_result (*external_encoder_control)(
+		struct dc_bios *bios,
+		struct bp_external_encoder_control *cntl);
 	enum bp_result (*dac_load_detection)(
 		struct dc_bios *bios,
-		enum engine_id engine_id,
-		enum dal_device_type device_type,
-		uint32_t enum_id);
+		enum engine_id engine_id);
 	enum bp_result (*transmitter_control)(
 		struct dc_bios *bios,
 		struct bp_transmitter_control *cntl);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index b15360bcdacf7..e5a2224258145 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -2349,6 +2349,33 @@ bool dc_dmub_srv_is_cursor_offload_enabled(const struct dc *dc)
 	return dc->ctx->dmub_srv && dc->ctx->dmub_srv->cursor_offload_enabled;
 }
 
+void dc_dmub_srv_boot_time_crc_init(const struct dc *dc, uint64_t gpu_addr, uint32_t size)
+{
+	struct dc_dmub_srv *dc_dmub_srv;
+	struct dc_context *dc_ctx;
+	union dmub_rb_cmd cmd = {0};
+	bool result = false;
+
+	if (!dc || !dc->ctx || !dc->ctx->dmub_srv || size == 0)
+		return;
+
+	dc_dmub_srv = dc->ctx->dmub_srv;
+	dc_ctx = dc_dmub_srv->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.boot_time_crc_init.header.type = DMUB_CMD__BOOT_TIME_CRC;
+	cmd.boot_time_crc_init.header.sub_type = DMUB_CMD__BOOT_TIME_CRC_INIT_MEM;
+	cmd.boot_time_crc_init.header.payload_bytes =
+		sizeof(struct dmub_rb_cmd_boot_time_crc_init);
+	cmd.boot_time_crc_init.data.buffer_addr.quad_part = gpu_addr;
+	cmd.boot_time_crc_init.data.buffer_size = size;
+
+	result = dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+
+	if (!result)
+		DC_ERROR("Boot time crc init failed in DMUB");
+}
+
 void dc_dmub_srv_release_hw(const struct dc *dc)
 {
 	struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 72e0a41f39f04..6579cf9cce3c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -361,6 +361,15 @@ void dc_dmub_srv_program_cursor_now(struct dc *dc, const struct pipe_ctx *pipe);
  */
 bool dc_dmub_srv_is_cursor_offload_enabled(const struct dc *dc);
 
+/**
+ * dc_dmub_srv_boot_time_crc_init() - Initializes DMUB boot time CRC.
+ *
+ * @dc - pointer to DC object
+ * @gpu_addr - address for the boot time CRC buffer
+ * @size - size of the boot time CRC buffer
+ */
+void dc_dmub_srv_boot_time_crc_init(const struct dc *dc, uint64_t gpu_addr, uint32_t size);
+
 /**
  * dc_dmub_srv_release_hw() - Notifies DMUB service that HW access is no longer required.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 5e3646b7550c1..7fa336bf1115d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -947,8 +947,8 @@ union dp_tun_cap_support {
 /* DPCD[0xE000E] DP_IN_ADAPTER_INFO register. */
 union dpia_info {
 	struct {
-		uint8_t dpia_num :5;
-		uint8_t rsvd :3;
+		uint8_t dpia_num :6;
+		uint8_t rsvd :2;
 	} bits;
 	uint8_t raw;
 };
@@ -1374,7 +1374,7 @@ union dpcd_replay_configuration {
 		unsigned char DESYNC_ERROR_STATUS              : 1;
 		unsigned char SINK_DEVICE_REPLAY_STATUS        : 3;
 		unsigned char SINK_FRAME_LOCKED                : 2;
-		unsigned char RESERVED                         : 1;
+		unsigned char FRAME_SKIPPING_ERROR_STATUS      : 1;
 	} bits;
 	unsigned char raw;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 5a365bd19933f..04b8b798dfff6 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -751,6 +751,8 @@ char *dce_version_to_string(const int version)
 		return "DCN 3.6";
 	case DCN_VERSION_4_01:
 		return "DCN 4.0.1";
+	case DCN_VERSION_4_2:
+		return "DCN 4.2";
 	default:
 		return "Unknown";
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index cfa569a7bff1b..9bf853edc46fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -271,6 +271,15 @@ enum tile_split_values_new {
 	DC_SURF_TILE_SPLIT_1KB = 0x4,
 };
 
+enum otg_pwa_sync_mode {
+	DC_OTG_PWA_FRAME_SYNC_MODE_VSYNC = 0x0,
+	DC_OTG_PWA_FRAME_SYNC_MODE_VSTARTUP = 0x1,
+};
+struct otc_pwa_frame_sync {
+	enum otg_pwa_sync_mode pwa_sync_mode;
+	uint32_t pwa_frame_sync_line_offset;
+};
+
 /* TODO: These values come from hardware spec. We need to readdress this
  * if they ever change.
  */
@@ -1137,6 +1146,7 @@ struct mcif_buf_params {
 	unsigned int		warmup_pitch;
 	unsigned int		swlock;
 	unsigned int		p_vmid;
+	uint8_t				tmz_id;
 };
 
 
@@ -1162,5 +1172,68 @@ struct phy_state {
 	enum symclk_state symclk_state;
 };
 
+enum cm_hist_tap_point {
+	CM_HIST_TAP_POINT_1,
+	CM_HIST_TAP_POINT_2,
+	CM_HIST_TAP_POINT_3,
+	CM_HIST_TAP_POINT_4,
+};
+
+enum cm_hist_src {
+	CM_HIST_SRC1,
+	CM_HIST_SRC2,
+	CM_HIST_SRC3,
+};
+
+enum cm_hist_format {
+	CM_HIST_FORMAT_FIXED_POINT,
+	CM_HIST_FORMAT_FP16_POS,
+	CM_HIST_FORMAT_FP16_POS_AND_NEG,
+};
+
+enum cm_hist_read_channel_mask {
+	CM_HIST_READ_DISABLED,
+	CM_HIST_READ_CH1,
+	CM_HIST_READ_CH2,
+	CM_HIST_READ_CH1_CH2,
+	CM_HIST_READ_CH3,
+	CM_HIST_READ_CH1_CH3,
+	CM_HIST_READ_CH2_CH3,
+	CM_HIST_READ_ALL,
+};
+
+enum cm_hist_src1_mode {
+	CM_HIST_SRC1_MODE_R_OR_CR,
+	CM_HIST_SRC1_MODE_MAX_RGB,
+};
+
+enum cm_hist_src2_mode {
+	CM_HIST_SRC2_MODE_G_OR_Y,
+	CM_HIST_SRC2_MODE_RGB_TO_Y,
+};
+
+enum cm_hist_src3_mode {
+	CM_HIST_SRC3_MODE_B_OR_CB,
+	CM_HIST_SRC3_MODE_MIN_RGB,
+};
+
+struct cm_hist_control {
+	enum cm_hist_tap_point tap_point;
+	uint32_t channels_enabled;
+	enum cm_hist_src1_mode src_1_select;
+	enum cm_hist_src2_mode src_2_select;
+	enum cm_hist_src3_mode src_3_select;
+	enum cm_hist_src ch1_src;
+	enum cm_hist_src ch2_src;
+	enum cm_hist_src ch3_src;
+	enum cm_hist_format format;
+	enum cm_hist_read_channel_mask read_channel_mask;
+};
+
+struct cm_hist {
+	uint32_t ch1[256];
+	uint32_t ch2[256];
+	uint32_t ch3[256];
+};
 #endif /* DC_HW_TYPES_H */
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_plane.h b/drivers/gpu/drm/amd/display/dc/dc_plane.h
index 14feb843e6940..2ad6d9318566e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_plane.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_plane.h
@@ -31,6 +31,7 @@
 union dc_plane_status_update_flags {
 	struct {
 		uint32_t address : 1;
+		uint32_t histogram : 1;
 	} bits;
 	uint32_t raw;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c b/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
index 37d1a79e8241e..ba7bf23f2b2fd 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
@@ -89,6 +89,7 @@ void translate_SPL_in_params_from_pipe_ctx(struct pipe_ctx *pipe_ctx, struct spl
 		spl_in->callbacks = dcn32_spl_callbacks;
 		break;
 	case DCN_VERSION_4_01:
+	case DCN_VERSION_4_2:
 		spl_in->callbacks = dcn401_spl_callbacks;
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/display/dc/dc_trace.h b/drivers/gpu/drm/amd/display/dc/dc_trace.h
index bbec308a3a5e2..b7a011646d533 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_trace.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_trace.h
@@ -23,8 +23,8 @@
 
 #include "amdgpu_dm_trace.h"
 
-#define TRACE_DC_PIPE_STATE(pipe_ctx, index, max_pipes) \
-	for (index = 0; index < max_pipes; ++index) { \
+#define TRACE_DC_PIPE_STATE(pipe_ctx, max_pipes) \
+	for (int index = 0; index < max_pipes; ++index) { \
 		struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[index]; \
 		if (pipe_ctx->plane_state) \
 			trace_amdgpu_dm_dc_pipe_state(pipe_ctx->pipe_idx, pipe_ctx->plane_state, \
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index bddb16bb76d41..e224077c8902e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -1194,6 +1194,8 @@ struct replay_config {
 	union replay_optimization replay_optimization;
 	/* Replay sub feature Frame Skipping is supported */
 	bool frame_skip_supported;
+	/* Replay Received Frame Skipping Error HPD. */
+	bool received_frame_skipping_error_hpd;
 };
 
 /* Replay feature flags*/
@@ -1346,6 +1348,7 @@ enum dc_cm2_gpu_mem_layout {
 
 enum dc_cm2_gpu_mem_pixel_component_order {
 	DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA,
+	DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_BGRA
 };
 
 enum dc_cm2_gpu_mem_format {
@@ -1367,6 +1370,9 @@ struct dc_cm2_gpu_mem_format_parameters {
 
 enum dc_cm2_gpu_mem_size {
 	DC_CM2_GPU_MEM_SIZE_171717,
+	DC_CM2_GPU_MEM_SIZE_333333,
+	DC_CM2_GPU_MEM_SIZE_454545,
+	DC_CM2_GPU_MEM_SIZE_656565,
 	DC_CM2_GPU_MEM_SIZE_TRANSFORMED,
 };
 
@@ -1477,4 +1483,28 @@ struct dc_validation_dpia_set {
 	uint32_t required_bw;
 };
 
+enum dc_cm_lut_swizzle {
+	CM_LUT_3D_SWIZZLE_LINEAR_RGB,
+	CM_LUT_3D_SWIZZLE_LINEAR_BGR,
+	CM_LUT_1D_PACKED_LINEAR
+};
+
+enum dc_cm_lut_pixel_format {
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB,
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB,
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB,
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB,
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10,
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10
+};
+
+enum dc_cm_lut_size {
+	CM_LUT_SIZE_NONE,
+	CM_LUT_SIZE_999,
+	CM_LUT_SIZE_171717,
+	CM_LUT_SIZE_333333,
+	CM_LUT_SIZE_454545,
+	CM_LUT_SIZE_656565,
+};
+
 #endif /* DC_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/Makefile b/drivers/gpu/drm/amd/display/dc/dccg/Makefile
index 1d5cf0f8e79d6..bf7e71166b5ea 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dccg/Makefile
@@ -1,5 +1,5 @@
 
-# Copyright 2022 Advanced Micro Devices, Inc.
+# Copyright 2022-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -100,4 +100,13 @@ DCCG_DCN401 = dcn401_dccg.o
 AMD_DAL_DCCG_DCN401 = $(addprefix $(AMDDALPATH)/dc/dccg/dcn401/,$(DCCG_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_DCCG_DCN401)
+
+###############################################################################
+# DCN42
+###############################################################################
+DCCG_DCN42 = dcn42_dccg.o
+
+AMD_DAL_DCCG_DCN42 = $(addprefix $(AMDDALPATH)/dc/dccg/dcn42/,$(DCCG_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DCCG_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
index 4c4e61bc91b50..088cf305a772c 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018 Advanced Micro Devices, Inc.
+ * Copyright 2018-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -366,6 +366,17 @@
 	type SYMCLK32_LE3_EN;\
 	type DP_DTO_ENABLE[MAX_PIPES];
 
+#define DCCG42_REG_FIELD_LIST(type) \
+	type OTG0_ADD_PIXEL;\
+	type OTG1_ADD_PIXEL;\
+	type OTG2_ADD_PIXEL;\
+	type OTG0_DROP_PIXEL;\
+	type OTG1_DROP_PIXEL;\
+	type OTG2_DROP_PIXEL;\
+	type OTG3_ADD_PIXEL;\
+	type OTG3_DROP_PIXEL;\
+	type RESYNC_FIFO_LEVEL_ADJUST_EN;
+
 struct dccg_shift {
 	DCCG_REG_FIELD_LIST(uint8_t)
 	DCCG3_REG_FIELD_LIST(uint8_t)
@@ -374,6 +385,7 @@ struct dccg_shift {
 	DCCG32_REG_FIELD_LIST(uint8_t)
 	DCCG35_REG_FIELD_LIST(uint8_t)
 	DCCG401_REG_FIELD_LIST(uint8_t)
+	DCCG42_REG_FIELD_LIST(uint8_t)
 };
 
 struct dccg_mask {
@@ -384,6 +396,7 @@ struct dccg_mask {
 	DCCG32_REG_FIELD_LIST(uint32_t)
 	DCCG35_REG_FIELD_LIST(uint32_t)
 	DCCG401_REG_FIELD_LIST(uint32_t)
+	DCCG42_REG_FIELD_LIST(uint32_t)
 };
 
 #define DCCG_REG_VARIABLE_LIST \
@@ -497,6 +510,7 @@ struct dccg_mask {
 
 struct dccg_registers {
 	DCCG_REG_VARIABLE_LIST;
+	uint32_t OTG_ADD_DROP_PIXEL_CNTL;
 };
 
 struct dcn_dccg {
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
index a37f94dec6f2f..4b9a14c679d3f 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
@@ -827,6 +827,16 @@ void dccg401_enable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst, uint3
 		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
 			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, 1);
 		break;
+	case 4:
+		if (dccg_dcn->dccg_mask->SYMCLKE_FE_ROOT_GATE_DISABLE) {
+			REG_UPDATE_2(SYMCLKE_CLOCK_ENABLE,
+					SYMCLKE_FE_EN, 1,
+					SYMCLKE_FE_SRC_SEL, link_enc_inst);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, 1);
+		}
+		break;
+	default:
+		return;
 	}
 }
 
@@ -855,6 +865,16 @@ void dccg401_disable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst, uint
 				SYMCLKD_FE_EN, 0,
 				SYMCLKD_FE_SRC_SEL, 0);
 		break;
+	case 4:
+		if (dccg_dcn->dccg_mask->SYMCLKE_FE_ROOT_GATE_DISABLE) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, 0);
+			REG_UPDATE_2(SYMCLKE_CLOCK_ENABLE,
+					SYMCLKE_FE_EN, 0,
+					SYMCLKE_FE_SRC_SEL, 0);
+		}
+		break;
+	default:
+		return;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
new file mode 100644
index 0000000000000..19dfc3fe5c3a4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "reg_helper.h"
+#include "core_types.h"
+#include "dcn35/dcn35_dccg.h"
+#include "dcn42_dccg.h"
+
+#define TO_DCN_DCCG(dccg)\
+	container_of(dccg, struct dcn_dccg, base)
+
+#define REG(reg) \
+	(dccg_dcn->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	dccg_dcn->dccg_shift->field_name, dccg_dcn->dccg_mask->field_name
+
+#define CTX \
+	dccg_dcn->base.ctx
+#define DC_LOGGER \
+	dccg->ctx->logger
+
+void dccg42_otg_add_pixel(struct dccg *dccg,
+		uint32_t otg_inst)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	switch (otg_inst) {
+	case 0:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG0_ADD_PIXEL, 1);
+		break;
+	case 1:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG1_ADD_PIXEL, 1);
+		break;
+	case 2:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG2_ADD_PIXEL, 1);
+		break;
+	case 3:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG3_ADD_PIXEL, 1);
+		break;
+	default:
+		ASSERT(0);
+	}
+}
+
+void dccg42_otg_drop_pixel(struct dccg *dccg,
+		uint32_t otg_inst)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	switch (otg_inst) {
+	case 0:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG0_DROP_PIXEL, 1);
+		break;
+	case 1:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG1_DROP_PIXEL, 1);
+		break;
+	case 2:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG2_DROP_PIXEL, 1);
+		break;
+	case 3:
+		REG_UPDATE(OTG_ADD_DROP_PIXEL_CNTL,
+				OTG3_DROP_PIXEL, 1);
+		break;
+	default:
+		ASSERT(0);
+	}
+}
+
+void dccg42_enable_global_fgcg(struct dccg *dccg, bool value)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	if (dccg->ctx->dc->debug.disable_clock_gate)
+		value = false;
+	REG_UPDATE(DCCG_GLOBAL_FGCG_REP_CNTL, DCCG_GLOBAL_FGCG_REP_DIS, !value);
+}
+
+void dccg42_set_physymclk(
+		struct dccg *dccg,
+		int phy_inst,
+		enum physymclk_clock_source clk_src,
+		bool force_enable)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	switch (phy_inst) {
+	case 0:
+		if (force_enable) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYASYMCLK_ROOT_GATE_DISABLE, 1);
+			REG_UPDATE_2(PHYASYMCLK_CLOCK_CNTL,
+					PHYASYMCLK_EN, 1,
+					PHYASYMCLK_SRC_SEL, clk_src);
+		} else {
+			REG_UPDATE_2(PHYASYMCLK_CLOCK_CNTL,
+					PHYASYMCLK_EN, 0,
+					PHYASYMCLK_SRC_SEL, 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYASYMCLK_ROOT_GATE_DISABLE,
+					dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk ? 0 : 1);
+		}
+		break;
+	case 1:
+		if (force_enable) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYBSYMCLK_ROOT_GATE_DISABLE, 1);
+			REG_UPDATE_2(PHYBSYMCLK_CLOCK_CNTL,
+					PHYBSYMCLK_EN, 1,
+					PHYBSYMCLK_SRC_SEL, clk_src);
+		} else {
+			REG_UPDATE_2(PHYBSYMCLK_CLOCK_CNTL,
+					PHYBSYMCLK_EN, 0,
+					PHYBSYMCLK_SRC_SEL, 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYBSYMCLK_ROOT_GATE_DISABLE,
+					dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk ? 0 : 1);
+		}
+		break;
+	case 2:
+		if (force_enable) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYCSYMCLK_ROOT_GATE_DISABLE, 1);
+			REG_UPDATE_2(PHYCSYMCLK_CLOCK_CNTL,
+					PHYCSYMCLK_EN, 1,
+					PHYCSYMCLK_SRC_SEL, clk_src);
+		} else {
+			REG_UPDATE_2(PHYCSYMCLK_CLOCK_CNTL,
+					PHYCSYMCLK_EN, 0,
+					PHYCSYMCLK_SRC_SEL, 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYCSYMCLK_ROOT_GATE_DISABLE,
+					dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk ? 0 : 1);
+		}
+		break;
+	case 3:
+		if (force_enable) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYDSYMCLK_ROOT_GATE_DISABLE, 1);
+			REG_UPDATE_2(PHYDSYMCLK_CLOCK_CNTL,
+					PHYDSYMCLK_EN, 1,
+					PHYDSYMCLK_SRC_SEL, clk_src);
+		} else {
+			REG_UPDATE_2(PHYDSYMCLK_CLOCK_CNTL,
+					PHYDSYMCLK_EN, 0,
+					PHYDSYMCLK_SRC_SEL, 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYDSYMCLK_ROOT_GATE_DISABLE,
+					dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk ? 0 : 1);
+		}
+		break;
+	case 4:
+		if (force_enable) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYESYMCLK_ROOT_GATE_DISABLE, 1);
+			REG_UPDATE_2(PHYESYMCLK_CLOCK_CNTL,
+					PHYESYMCLK_EN, 1,
+					PHYESYMCLK_SRC_SEL, clk_src);
+		} else {
+			REG_UPDATE_2(PHYESYMCLK_CLOCK_CNTL,
+					PHYESYMCLK_EN, 0,
+					PHYESYMCLK_SRC_SEL, 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
+					PHYESYMCLK_ROOT_GATE_DISABLE,
+					dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk ? 0 : 1);
+		}
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+}
+
+void dccg42_set_pixel_rate_div(
+		struct dccg *dccg,
+		uint32_t otg_inst,
+		enum pixel_rate_div tmds_div,
+		enum pixel_rate_div unused)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	uint32_t cur_tmds_div = PIXEL_RATE_DIV_NA;
+	uint32_t dp_dto_int;
+	uint32_t reg_val;
+
+	// only 2 and 4 are valid on dcn401
+	if (tmds_div != PIXEL_RATE_DIV_BY_2 && tmds_div != PIXEL_RATE_DIV_BY_4) {
+		return;
+	}
+
+	dccg401_get_pixel_rate_div(dccg, otg_inst, &cur_tmds_div, &dp_dto_int);
+	if (tmds_div == cur_tmds_div)
+		return;
+
+	// encode enum to register value
+	reg_val = tmds_div == PIXEL_RATE_DIV_BY_4 ? 1 : 0;
+
+	switch (otg_inst) {
+	case 0:
+		REG_UPDATE(OTG_PIXEL_RATE_DIV,
+				OTG0_TMDS_PIXEL_RATE_DIV, reg_val);
+		break;
+	case 1:
+		REG_UPDATE(OTG_PIXEL_RATE_DIV,
+				OTG1_TMDS_PIXEL_RATE_DIV, reg_val);
+		break;
+	case 2:
+		REG_UPDATE(OTG_PIXEL_RATE_DIV,
+				OTG2_TMDS_PIXEL_RATE_DIV, reg_val);
+		break;
+	case 3:
+		REG_UPDATE(OTG_PIXEL_RATE_DIV,
+				OTG3_TMDS_PIXEL_RATE_DIV, reg_val);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+}
+
+void dccg42_trigger_dio_fifo_resync(struct dccg *dccg)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	REG_UPDATE(DISPCLK_FREQ_CHANGE_CNTL, RESYNC_FIFO_LEVEL_ADJUST_EN, 1);
+	REG_UPDATE(DISPCLK_FREQ_CHANGE_CNTL, RESYNC_FIFO_LEVEL_ADJUST_EN, 0);
+	REG_WAIT(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_FREQ_RAMP_DONE, 1, 50, 2000);
+}
+
+static void dccg42_init(struct dccg *dccg)
+{
+	int otg_inst;
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	/* Set HPO stream encoder to use refclk to avoid case where PHY is
+	 * disabled and SYMCLK32 for HPO SE is sourced from PHYD32CLK which
+	 * will cause DCN to hang.
+	 */
+	for (otg_inst = 0; otg_inst < 4; otg_inst++)
+		dccg35_disable_symclk32_se(dccg, otg_inst);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le) {
+		dccg401_disable_symclk32_le(dccg, 0);
+		dccg401_disable_symclk32_le(dccg, 1);
+		dccg401_disable_symclk32_le(dccg, 2);
+		dccg401_disable_symclk32_le(dccg, 3);
+	}
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
+		dccg401_disable_dpstreamclk(dccg, 0);
+		dccg401_disable_dpstreamclk(dccg, 1);
+		dccg401_disable_dpstreamclk(dccg, 2);
+		dccg401_disable_dpstreamclk(dccg, 3);
+	}
+	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk) {
+		REG_UPDATE_5(DCCG_GATE_DISABLE_CNTL2,
+			PHYASYMCLK_ROOT_GATE_DISABLE, 1,
+			PHYBSYMCLK_ROOT_GATE_DISABLE, 1,
+			PHYCSYMCLK_ROOT_GATE_DISABLE, 1,
+			PHYDSYMCLK_ROOT_GATE_DISABLE, 1,
+			PHYESYMCLK_ROOT_GATE_DISABLE, 1);
+	}
+}
+
+
+static const struct dccg_funcs dccg42_funcs = {
+	.update_dpp_dto = dccg35_update_dpp_dto,
+	.dpp_root_clock_control = dccg35_dpp_root_clock_control,
+	.get_dccg_ref_freq = dccg401_get_dccg_ref_freq,
+	.dccg_init = dccg42_init,
+	.set_dpstreamclk = dccg401_set_dpstreamclk,
+	/* Redundant with above^ */
+	/* .set_dpstreamclk_root_clock_gating = dccg35_set_dpstreamclk_root_clock_gating, */
+	.enable_symclk32_se = dccg31_enable_symclk32_se,
+	.disable_symclk32_se = dccg35_disable_symclk32_se,
+	.enable_symclk32_le = dccg401_enable_symclk32_le,
+	.disable_symclk32_le = dccg401_disable_symclk32_le,
+	.set_symclk32_le_root_clock_gating = dccg31_set_symclk32_le_root_clock_gating,
+	.set_physymclk = dccg42_set_physymclk,
+	.set_dtbclk_dto = NULL,
+	.set_dto_dscclk = dccg401_set_dto_dscclk,
+	.set_ref_dscclk = dccg401_set_ref_dscclk,
+	.set_valid_pixel_rate = NULL,
+	.set_fifo_errdet_ovr_en = dccg2_set_fifo_errdet_ovr_en,
+	.set_audio_dtbclk_dto = NULL,
+	.otg_add_pixel = dccg42_otg_add_pixel,
+	.otg_drop_pixel = dccg42_otg_drop_pixel,
+	.disable_dsc = dccg35_disable_dscclk,
+	.enable_dsc = dccg35_enable_dscclk,
+	.set_pixel_rate_div = dccg42_set_pixel_rate_div,
+	.get_pixel_rate_div = dccg401_get_pixel_rate_div,
+	.trigger_dio_fifo_resync = dccg42_trigger_dio_fifo_resync,
+	.set_dp_dto = dccg401_set_dp_dto,
+	.enable_symclk_se = dccg35_enable_symclk_se,
+	.disable_symclk_se = dccg35_disable_symclk_se,
+	.set_dtbclk_p_src = dccg401_set_dtbclk_p_src,
+	.dccg_root_gate_disable_control = dccg35_root_gate_disable_control,
+	.dccg_read_reg_state = dccg31_read_reg_state,
+	.dccg_enable_global_fgcg = dccg42_enable_global_fgcg,
+};
+
+struct dccg *dccg42_create(
+	struct dc_context *ctx,
+	const struct dccg_registers *regs,
+	const struct dccg_shift *dccg_shift,
+	const struct dccg_mask *dccg_mask)
+{
+	struct dcn_dccg *dccg_dcn = kzalloc(sizeof(*dccg_dcn), GFP_KERNEL);
+	struct dccg *base;
+
+	if (dccg_dcn == NULL) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	base = &dccg_dcn->base;
+	base->ctx = ctx;
+	base->funcs = &dccg42_funcs;
+
+	dccg_dcn->regs = regs;
+	dccg_dcn->dccg_shift = dccg_shift;
+	dccg_dcn->dccg_mask = dccg_mask;
+
+	return &dccg_dcn->base;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
new file mode 100644
index 0000000000000..d9831b0f82359
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024-2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DCN42_DCCG_H__
+#define __DCN42_DCCG_H__
+
+#include "dcn401/dcn401_dccg.h"
+
+#define DCCG_MASK_SH_LIST_DCN42_COMMON(mask_sh) \
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 0, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 1, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 2, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 3, mask_sh),\
+	DCCG_SF(DPPCLK_CTRL, DPPCLK0_EN, mask_sh),\
+	DCCG_SF(DPPCLK_CTRL, DPPCLK1_EN, mask_sh),\
+	DCCG_SF(DPPCLK_CTRL, DPPCLK2_EN, mask_sh),\
+	DCCG_SF(DPPCLK_CTRL, DPPCLK3_EN, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_STEP_DELAY, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_STEP_SIZE, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_FREQ_RAMP_DONE, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_MAX_ERRDET_CYCLES, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DCCG_FIFO_ERRDET_RESET, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DCCG_FIFO_ERRDET_STATE, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DCCG_FIFO_ERRDET_OVR_EN, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, DISPCLK_CHG_FWD_CORR_DISABLE, mask_sh),\
+	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\
+	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_EN, mask_sh),\
+	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_EN, mask_sh),\
+	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_EN, mask_sh),\
+	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_EN, mask_sh),\
+	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_SRC_SEL, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_EN, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_EN, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_EN, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_EN, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_SRC_SEL, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_SRC_SEL, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_SRC_SEL, mask_sh),\
+	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_SRC_SEL, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE2_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE2_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE3_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE0_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE1_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE0_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE1_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE2_EN, mask_sh),\
+	DCCG_SF(SYMCLK32_LE_CNTL, SYMCLK32_LE3_EN, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 0, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 1, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 2, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 3, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG0_ADD_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG1_ADD_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG2_ADD_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG3_ADD_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG0_DROP_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG1_DROP_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG2_DROP_PIXEL, mask_sh),\
+	DCCG_SF(OTG_ADD_DROP_PIXEL_CNTL, OTG3_DROP_PIXEL, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, OTG0_TMDS_PIXEL_RATE_DIV, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, DPDTO0_INT, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, OTG1_TMDS_PIXEL_RATE_DIV, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, DPDTO1_INT, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, OTG2_TMDS_PIXEL_RATE_DIV, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, DPDTO2_INT, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, OTG3_TMDS_PIXEL_RATE_DIV, mask_sh),\
+	DCCG_SF(OTG_PIXEL_RATE_DIV, DPDTO3_INT, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P0_SRC_SEL, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P0_EN, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P1_SRC_SEL, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P1_EN, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P2_SRC_SEL, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P2_EN, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_SRC_SEL, mask_sh),\
+	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_EN, mask_sh),\
+	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\
+	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_RDIVIDER, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_MODE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 0, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 1, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 2, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, DP_DTO, ENABLE, 3, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 0, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 1, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 2, mask_sh),\
+	DCCG_SFII(OTG, PIXEL_RATE_CNTL, PIPE, DTO_SRC_SEL, 3, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_EN, mask_sh),\
+	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_MODULO, mask_sh),\
+	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_MODULO, mask_sh),\
+	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_MODULO, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL, DISPCLK_DCCG_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, HDMICHARCLK0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, HDMISTREAMCLK0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_SE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_LE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, HDMICHARCLK0_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYA_REFCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYB_REFCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYC_REFCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYD_REFCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_FE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_FE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_FE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK0_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK1_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK2_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DPPCLK0_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DPPCLK1_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DPPCLK2_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DPPCLK3_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, HDMISTREAMCLK0_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_CLOCK_ENABLE, mask_sh),\
+	DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_CLOCK_ENABLE, mask_sh),\
+	DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_CLOCK_ENABLE, mask_sh),\
+	DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_CLOCK_ENABLE, mask_sh),\
+	DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_FE_EN, mask_sh),\
+	DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_FE_EN, mask_sh),\
+	DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_EN, mask_sh),\
+	DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_EN, mask_sh),\
+	DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_FE_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_FE_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_SRC_SEL, mask_sh)
+
+#define DCCG_MASK_SH_LIST_DCN42(mask_sh) \
+	DCCG_MASK_SH_LIST_DCN42_COMMON(mask_sh),\
+	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_EN, mask_sh),\
+	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_SRC_SEL, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_PHASE, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_MODULO, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_MODULO, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL4, PHYE_REFCLK_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL6, DSCCLK3_ROOT_GATE_DISABLE, mask_sh),\
+	DCCG_SF(SYMCLKA_CLOCK_ENABLE, SYMCLKA_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_SRC_SEL, mask_sh),\
+	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_CLOCK_ENABLE, mask_sh),\
+	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_EN, mask_sh),\
+	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_SRC_SEL, mask_sh),\
+	DCCG_SF(DISPCLK_FREQ_CHANGE_CNTL, RESYNC_FIFO_LEVEL_ADJUST_EN, mask_sh)
+
+
+void dccg42_otg_add_pixel(struct dccg *dccg,
+		uint32_t otg_inst);
+
+void dccg42_otg_drop_pixel(struct dccg *dccg,
+		uint32_t otg_inst);
+void dccg42_enable_global_fgcg(struct dccg *dccg, bool value);
+
+void dccg42_set_physymclk(
+		struct dccg *dccg,
+		int phy_inst,
+		enum physymclk_clock_source clk_src,
+		bool force_enable);
+
+void dccg42_set_pixel_rate_div(
+		struct dccg *dccg,
+		uint32_t otg_inst,
+		enum pixel_rate_div tmds_div,
+		enum pixel_rate_div unused);
+
+void dccg42_trigger_dio_fifo_resync(struct dccg *dccg);
+
+struct dccg *dccg42_create(
+	struct dc_context *ctx,
+	const struct dccg_registers *regs,
+	const struct dccg_shift *dccg_shift,
+	const struct dccg_mask *dccg_mask);
+
+#endif //__DCN42_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
index 3d819fc5654cd..1e9ab5ff3a31e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-16 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -300,6 +300,64 @@
 	ABM_SF(ABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX, \
 			ABM1_HG_BIN_57_64_SHIFT_INDEX, mask_sh)
 
+#define ABM_MASK_SH_LIST_DCN42(mask_sh) \
+	ABM_SF(ABM0_DC_ABM1_HG_MISC_CTRL, \
+			ABM1_HG_NUM_OF_BINS_SEL, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_MISC_CTRL, \
+			ABM1_HG_VMAX_SEL, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_MISC_CTRL, \
+			ABM1_HG_BIN_BITWIDTH_SIZE_SEL, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_IPCSC_COEFF_SEL, \
+			ABM1_IPCSC_COEFF_SEL_R, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_IPCSC_COEFF_SEL, \
+			ABM1_IPCSC_COEFF_SEL_G, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_IPCSC_COEFF_SEL, \
+			ABM1_IPCSC_COEFF_SEL_B, mask_sh), \
+	ABM_SF(ABM0_BL1_PWM_CURRENT_ABM_LEVEL, \
+			BL1_PWM_CURRENT_ABM_LEVEL, mask_sh), \
+	ABM_SF(ABM0_BL1_PWM_TARGET_ABM_LEVEL, \
+			BL1_PWM_TARGET_ABM_LEVEL, mask_sh), \
+	ABM_SF(ABM0_BL1_PWM_USER_LEVEL, \
+			BL1_PWM_USER_LEVEL, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, \
+			ABM1_LS_MIN_PIXEL_VALUE_THRES, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, \
+			ABM1_LS_MAX_PIXEL_VALUE_THRES, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS, \
+			ABM1_HG_REG_READ_MISSED_FRAME_CLEAR, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS, \
+			ABM1_LS_REG_READ_MISSED_FRAME_CLEAR, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS, \
+			ABM1_BL_REG_READ_MISSED_FRAME_CLEAR, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA, \
+			ABM1_ACE_SLOPE_DATA, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA, \
+			ABM1_ACE_OFFSET_DATA, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_PWL_CNTL, \
+			ABM1_ACE_OFFSET_SLOPE_INDEX, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_PWL_CNTL, \
+			ABM1_ACE_IGNORE_MASTER_LOCK_EN, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_PWL_CNTL, \
+			ABM1_ACE_READBACK_DB_REG_VALUE_EN, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_PWL_CNTL, \
+			ABM1_ACE_DBUF_REG_UPDATE_PENDING, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_ACE_PWL_CNTL, \
+			ABM1_ACE_LOCK, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_RESULT_DATA, \
+			ABM1_HG_RESULT_DATA, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_RESULT_INDEX, \
+			ABM1_HG_RESULT_INDEX, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX, \
+			ABM1_HG_BIN_33_40_SHIFT_INDEX, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG, \
+			ABM1_HG_BIN_33_64_SHIFT_FLAG, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX, \
+			ABM1_HG_BIN_41_48_SHIFT_INDEX, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX, \
+			ABM1_HG_BIN_49_56_SHIFT_INDEX, mask_sh), \
+	ABM_SF(ABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX, \
+			ABM1_HG_BIN_57_64_SHIFT_INDEX, mask_sh)
+
 #define ABM_REG_FIELD_LIST(type) \
 	type ABM1_HG_NUM_OF_BINS_SEL; \
 	type ABM1_HG_VMAX_SEL; \
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
index d18490cd0f1eb..0807d20985c7d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
@@ -1150,98 +1150,6 @@ void dce_aud_wall_dto_setup(
 	}
 }
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-static void dce60_aud_wall_dto_setup(
-	struct audio *audio,
-	enum signal_type signal,
-	const struct audio_crtc_info *crtc_info,
-	const struct audio_pll_info *pll_info)
-{
-	struct dce_audio *aud = DCE_AUD(audio);
-
-	struct azalia_clock_info clock_info = { 0 };
-
-	if (dc_is_hdmi_signal(signal)) {
-		uint32_t src_sel;
-
-		/*DTO0 Programming goal:
-		-generate 24MHz, 128*Fs from 24MHz
-		-use DTO0 when an active HDMI port is connected
-		(optionally a DP is connected) */
-
-		/* calculate DTO settings */
-		get_azalia_clock_info_hdmi(
-			crtc_info->requested_pixel_clock_100Hz,
-			crtc_info->calculated_pixel_clock_100Hz,
-			&clock_info);
-
-		DC_LOG_HW_AUDIO("\n%s:Input::requested_pixel_clock_100Hz = %d"\
-				"calculated_pixel_clock_100Hz =%d\n"\
-				"audio_dto_module = %d audio_dto_phase =%d \n\n", __func__,\
-				crtc_info->requested_pixel_clock_100Hz,\
-				crtc_info->calculated_pixel_clock_100Hz,\
-				clock_info.audio_dto_module,\
-				clock_info.audio_dto_phase);
-
-		/* On TN/SI, Program DTO source select and DTO select before
-		programming DTO modulo and DTO phase. These bits must be
-		programmed first, otherwise there will be no HDMI audio at boot
-		up. This is a HW sequence change (different from old ASICs).
-		Caution when changing this programming sequence.
-
-		HDMI enabled, using DTO0
-		program master CRTC for DTO0 */
-		src_sel = pll_info->dto_source - DTO_SOURCE_ID0;
-		REG_UPDATE_2(DCCG_AUDIO_DTO_SOURCE,
-			DCCG_AUDIO_DTO0_SOURCE_SEL, src_sel,
-			DCCG_AUDIO_DTO_SEL, 0);
-
-		/* module */
-		REG_UPDATE(DCCG_AUDIO_DTO0_MODULE,
-			DCCG_AUDIO_DTO0_MODULE, clock_info.audio_dto_module);
-
-		/* phase */
-		REG_UPDATE(DCCG_AUDIO_DTO0_PHASE,
-			DCCG_AUDIO_DTO0_PHASE, clock_info.audio_dto_phase);
-	} else {
-		/*DTO1 Programming goal:
-		-generate 24MHz, 128*Fs from 24MHz (DCE6 does not support 512*Fs)
-		-default is to used DTO1, and switch to DTO0 when an audio
-		master HDMI port is connected
-		-use as default for DP
-
-		calculate DTO settings */
-		get_azalia_clock_info_dp(
-			crtc_info->requested_pixel_clock_100Hz,
-			pll_info,
-			&clock_info);
-
-		/* Program DTO select before programming DTO modulo and DTO
-		phase. default to use DTO1 */
-
-		REG_UPDATE(DCCG_AUDIO_DTO_SOURCE,
-				DCCG_AUDIO_DTO_SEL, 1);
-
-			/* DCCG_AUDIO_DTO2_USE_512FBR_DTO, 1)
-			 * Cannot select 512fs for DP
-			 *
-			 * DCE6 has no DCCG_AUDIO_DTO2_USE_512FBR_DTO mask
-			*/
-
-		/* module */
-		REG_UPDATE(DCCG_AUDIO_DTO1_MODULE,
-				DCCG_AUDIO_DTO1_MODULE, clock_info.audio_dto_module);
-
-		/* phase */
-		REG_UPDATE(DCCG_AUDIO_DTO1_PHASE,
-				DCCG_AUDIO_DTO1_PHASE, clock_info.audio_dto_phase);
-
-		/* DCE6 has no DCCG_AUDIO_DTO2_USE_512FBR_DTO mask in DCCG_AUDIO_DTO_SOURCE reg */
-
-	}
-}
-#endif
-
 static bool dce_aud_endpoint_valid(struct audio *audio)
 {
 	uint32_t value;
@@ -1303,18 +1211,6 @@ static const struct audio_funcs funcs = {
 	.destroy = dce_aud_destroy,
 };
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-static const struct audio_funcs dce60_funcs = {
-	.endpoint_valid = dce_aud_endpoint_valid,
-	.hw_init = dce_aud_hw_init,
-	.wall_dto_setup = dce60_aud_wall_dto_setup,
-	.az_enable = dce_aud_az_enable,
-	.az_disable = dce_aud_az_disable,
-	.az_configure = dce_aud_az_configure,
-	.destroy = dce_aud_destroy,
-};
-#endif
-
 void dce_aud_destroy(struct audio **audio)
 {
 	struct dce_audio *aud = DCE_AUD(*audio);
@@ -1347,30 +1243,3 @@ struct audio *dce_audio_create(
 	audio->masks = masks;
 	return &audio->base;
 }
-
-#if defined(CONFIG_DRM_AMD_DC_SI)
-struct audio *dce60_audio_create(
-		struct dc_context *ctx,
-		unsigned int inst,
-		const struct dce_audio_registers *reg,
-		const struct dce_audio_shift *shifts,
-		const struct dce_audio_mask *masks
-		)
-{
-	struct dce_audio *audio = kzalloc_obj(*audio);
-
-	if (audio == NULL) {
-		ASSERT_CRITICAL(audio);
-		return NULL;
-	}
-
-	audio->base.ctx = ctx;
-	audio->base.inst = inst;
-	audio->base.funcs = &dce60_funcs;
-
-	audio->regs = reg;
-	audio->shifts = shifts;
-	audio->masks = masks;
-	return &audio->base;
-}
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
index 1b7b8b079af44..3f1d161a0045a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
@@ -151,15 +151,6 @@ struct audio *dce_audio_create(
 		const struct dce_audio_shift *shifts,
 		const struct dce_audio_mask *masks);
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-struct audio *dce60_audio_create(
-		struct dc_context *ctx,
-		unsigned int inst,
-		const struct dce_audio_registers *reg,
-		const struct dce_audio_shift *shifts,
-		const struct dce_audio_mask *masks);
-#endif
-
 void dce_aud_destroy(struct audio **audio);
 
 void dce_aud_hw_init(struct audio *audio);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
deleted file mode 100644
index dca025b5ff1f0..0000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
+++ /dev/null
@@ -1,966 +0,0 @@
-/*
- * Copyright 2012-16 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include <linux/slab.h>
-
-#include "dce_clk_mgr.h"
-
-#include "reg_helper.h"
-#include "dmcu.h"
-#include "core_types.h"
-#include "dal_asic_id.h"
-
-#define TO_DCE_CLK_MGR(clocks)\
-	container_of(clocks, struct dce_clk_mgr, base)
-
-#define REG(reg) \
-	(clk_mgr_dce->regs->reg)
-
-#undef FN
-#define FN(reg_name, field_name) \
-	clk_mgr_dce->clk_mgr_shift->field_name, clk_mgr_dce->clk_mgr_mask->field_name
-
-#define CTX \
-	clk_mgr_dce->base.ctx
-#define DC_LOGGER \
-	clk_mgr->ctx->logger
-
-/* Max clock values for each state indexed by "enum clocks_state": */
-static const struct state_dependent_clocks dce80_max_clks_by_state[] = {
-/* ClocksStateInvalid - should not be used */
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/* ClocksStateUltraLow - not expected to be used for DCE 8.0 */
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/* ClocksStateLow */
-{ .display_clk_khz = 352000, .pixel_clk_khz = 330000},
-/* ClocksStateNominal */
-{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 },
-/* ClocksStatePerformance */
-{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 } };
-
-static const struct state_dependent_clocks dce110_max_clks_by_state[] = {
-/*ClocksStateInvalid - should not be used*/
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
-{ .display_clk_khz = 352000, .pixel_clk_khz = 330000 },
-/*ClocksStateLow*/
-{ .display_clk_khz = 352000, .pixel_clk_khz = 330000 },
-/*ClocksStateNominal*/
-{ .display_clk_khz = 467000, .pixel_clk_khz = 400000 },
-/*ClocksStatePerformance*/
-{ .display_clk_khz = 643000, .pixel_clk_khz = 400000 } };
-
-static const struct state_dependent_clocks dce112_max_clks_by_state[] = {
-/*ClocksStateInvalid - should not be used*/
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
-{ .display_clk_khz = 389189, .pixel_clk_khz = 346672 },
-/*ClocksStateLow*/
-{ .display_clk_khz = 459000, .pixel_clk_khz = 400000 },
-/*ClocksStateNominal*/
-{ .display_clk_khz = 667000, .pixel_clk_khz = 600000 },
-/*ClocksStatePerformance*/
-{ .display_clk_khz = 1132000, .pixel_clk_khz = 600000 } };
-
-static const struct state_dependent_clocks dce120_max_clks_by_state[] = {
-/*ClocksStateInvalid - should not be used*/
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
-{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
-/*ClocksStateLow*/
-{ .display_clk_khz = 460000, .pixel_clk_khz = 400000 },
-/*ClocksStateNominal*/
-{ .display_clk_khz = 670000, .pixel_clk_khz = 600000 },
-/*ClocksStatePerformance*/
-{ .display_clk_khz = 1133000, .pixel_clk_khz = 600000 } };
-
-int dentist_get_divider_from_did(int did)
-{
-	if (did < DENTIST_BASE_DID_1)
-		did = DENTIST_BASE_DID_1;
-	if (did > DENTIST_MAX_DID)
-		did = DENTIST_MAX_DID;
-
-	if (did < DENTIST_BASE_DID_2) {
-		return DENTIST_DIVIDER_RANGE_1_START + DENTIST_DIVIDER_RANGE_1_STEP
-							* (did - DENTIST_BASE_DID_1);
-	} else if (did < DENTIST_BASE_DID_3) {
-		return DENTIST_DIVIDER_RANGE_2_START + DENTIST_DIVIDER_RANGE_2_STEP
-							* (did - DENTIST_BASE_DID_2);
-	} else if (did < DENTIST_BASE_DID_4) {
-		return DENTIST_DIVIDER_RANGE_3_START + DENTIST_DIVIDER_RANGE_3_STEP
-							* (did - DENTIST_BASE_DID_3);
-	} else {
-		return DENTIST_DIVIDER_RANGE_4_START + DENTIST_DIVIDER_RANGE_4_STEP
-							* (did - DENTIST_BASE_DID_4);
-	}
-}
-
-/* SW will adjust DP REF Clock average value for all purposes
- * (DP DTO / DP Audio DTO and DP GTC)
- if clock is spread for all cases:
- -if SS enabled on DP Ref clock and HW de-spreading enabled with SW
- calculations for DS_INCR/DS_MODULO (this is planned to be default case)
- -if SS enabled on DP Ref clock and HW de-spreading enabled with HW
- calculations (not planned to be used, but average clock should still
- be valid)
- -if SS enabled on DP Ref clock and HW de-spreading disabled
- (should not be case with CIK) then SW should program all rates
- generated according to average value (case as with previous ASICs)
-  */
-static int clk_mgr_adjust_dp_ref_freq_for_ss(struct dce_clk_mgr *clk_mgr_dce, int dp_ref_clk_khz)
-{
-	if (clk_mgr_dce->ss_on_dprefclk && clk_mgr_dce->dprefclk_ss_divider != 0) {
-		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
-				dc_fixpt_from_fraction(clk_mgr_dce->dprefclk_ss_percentage,
-							clk_mgr_dce->dprefclk_ss_divider), 200);
-		struct fixed31_32 adj_dp_ref_clk_khz;
-
-		ss_percentage = dc_fixpt_sub(dc_fixpt_one, ss_percentage);
-		adj_dp_ref_clk_khz = dc_fixpt_mul_int(ss_percentage, dp_ref_clk_khz);
-		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
-	}
-	return dp_ref_clk_khz;
-}
-
-static int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	int dprefclk_wdivider;
-	int dprefclk_src_sel;
-	int dp_ref_clk_khz = 600000;
-	int target_div;
-
-	/* ASSERT DP Reference Clock source is from DFS*/
-	REG_GET(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, &dprefclk_src_sel);
-	ASSERT(dprefclk_src_sel == 0);
-
-	/* Read the mmDENTIST_DISPCLK_CNTL to get the currently
-	 * programmed DID DENTIST_DPREFCLK_WDIVIDER*/
-	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, &dprefclk_wdivider);
-
-	/* Convert DENTIST_DPREFCLK_WDIVIDERto actual divider*/
-	target_div = dentist_get_divider_from_did(dprefclk_wdivider);
-
-	/* Calculate the current DFS clock, in kHz.*/
-	dp_ref_clk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
-		* clk_mgr_dce->dentist_vco_freq_khz) / target_div;
-
-	return clk_mgr_adjust_dp_ref_freq_for_ss(clk_mgr_dce, dp_ref_clk_khz);
-}
-
-int dce12_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-
-	return clk_mgr_adjust_dp_ref_freq_for_ss(clk_mgr_dce, clk_mgr_dce->dprefclk_khz);
-}
-
-/* unit: in_khz before mode set, get pixel clock from context. ASIC register
- * may not be programmed yet
- */
-static uint32_t get_max_pixel_clock_for_all_paths(struct dc_state *context)
-{
-	uint32_t max_pix_clk = 0;
-	int i;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe_ctx->stream == NULL)
-			continue;
-
-		/* do not check under lay */
-		if (pipe_ctx->top_pipe)
-			continue;
-
-		if (pipe_ctx->stream_res.pix_clk_params.requested_pix_clk_100hz / 10 > max_pix_clk)
-			max_pix_clk = pipe_ctx->stream_res.pix_clk_params.requested_pix_clk_100hz / 10;
-
-		/* raise clock state for HBR3/2 if required. Confirmed with HW DCE/DPCS
-		 * logic for HBR3 still needs Nominal (0.8V) on VDDC rail
-		 */
-		if (dc_is_dp_signal(pipe_ctx->stream->signal) &&
-				pipe_ctx->stream_res.pix_clk_params.requested_sym_clk > max_pix_clk)
-			max_pix_clk = pipe_ctx->stream_res.pix_clk_params.requested_sym_clk;
-	}
-
-	return max_pix_clk;
-}
-
-static enum dm_pp_clocks_state dce_get_required_clocks_state(
-	struct clk_mgr *clk_mgr,
-	struct dc_state *context)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	int i;
-	enum dm_pp_clocks_state low_req_clk;
-	int max_pix_clk = get_max_pixel_clock_for_all_paths(context);
-
-	/* Iterate from highest supported to lowest valid state, and update
-	 * lowest RequiredState with the lowest state that satisfies
-	 * all required clocks
-	 */
-	for (i = clk_mgr_dce->max_clks_state; i >= DM_PP_CLOCKS_STATE_ULTRA_LOW; i--)
-		if (context->bw_ctx.bw.dce.dispclk_khz >
-				clk_mgr_dce->max_clks_by_state[i].display_clk_khz
-			|| max_pix_clk >
-				clk_mgr_dce->max_clks_by_state[i].pixel_clk_khz)
-			break;
-
-	low_req_clk = i + 1;
-	if (low_req_clk > clk_mgr_dce->max_clks_state) {
-		/* set max clock state for high phyclock, invalid on exceeding display clock */
-		if (clk_mgr_dce->max_clks_by_state[clk_mgr_dce->max_clks_state].display_clk_khz
-				< context->bw_ctx.bw.dce.dispclk_khz)
-			low_req_clk = DM_PP_CLOCKS_STATE_INVALID;
-		else
-			low_req_clk = clk_mgr_dce->max_clks_state;
-	}
-
-	return low_req_clk;
-}
-
-static int dce_set_clock(
-	struct clk_mgr *clk_mgr,
-	int requested_clk_khz)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct bp_pixel_clock_parameters pxl_clk_params = { 0 };
-	struct dc_bios *bp = clk_mgr->ctx->dc_bios;
-	int actual_clock = requested_clk_khz;
-	struct dmcu *dmcu = clk_mgr_dce->base.ctx->dc->res_pool->dmcu;
-
-	/* Make sure requested clock isn't lower than minimum threshold*/
-	if (requested_clk_khz > 0)
-		requested_clk_khz = max(requested_clk_khz,
-				clk_mgr_dce->dentist_vco_freq_khz / 64);
-
-	/* Prepare to program display clock*/
-	pxl_clk_params.target_pixel_clock_100hz = requested_clk_khz * 10;
-	pxl_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
-
-	if (clk_mgr_dce->dfs_bypass_active)
-		pxl_clk_params.flags.SET_DISPCLK_DFS_BYPASS = true;
-
-	bp->funcs->program_display_engine_pll(bp, &pxl_clk_params);
-
-	if (clk_mgr_dce->dfs_bypass_active) {
-		/* Cache the fixed display clock*/
-		clk_mgr_dce->dfs_bypass_disp_clk =
-			pxl_clk_params.dfs_bypass_display_clock;
-		actual_clock = pxl_clk_params.dfs_bypass_display_clock;
-	}
-
-	/* from power down, we need mark the clock state as ClocksStateNominal
-	 * from HWReset, so when resume we will call pplib voltage regulator.*/
-	if (requested_clk_khz == 0)
-		clk_mgr_dce->cur_min_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
-
-	if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu))
-		dmcu->funcs->set_psr_wait_loop(dmcu, actual_clock / 1000 / 7);
-
-	return actual_clock;
-}
-
-int dce112_set_clock(struct clk_mgr *clk_mgr, int requested_clk_khz)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct bp_set_dce_clock_parameters dce_clk_params;
-	struct dc_bios *bp = clk_mgr->ctx->dc_bios;
-	struct dc *core_dc = clk_mgr->ctx->dc;
-	struct dmcu *dmcu = core_dc->res_pool->dmcu;
-	int actual_clock = requested_clk_khz;
-	/* Prepare to program display clock*/
-	memset(&dce_clk_params, 0, sizeof(dce_clk_params));
-
-	/* Make sure requested clock isn't lower than minimum threshold*/
-	if (requested_clk_khz > 0)
-		requested_clk_khz = max(requested_clk_khz,
-				clk_mgr_dce->dentist_vco_freq_khz / 62);
-
-	dce_clk_params.target_clock_frequency = requested_clk_khz;
-	dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
-	dce_clk_params.clock_type = DCECLOCK_TYPE_DISPLAY_CLOCK;
-
-	bp->funcs->set_dce_clock(bp, &dce_clk_params);
-	actual_clock = dce_clk_params.target_clock_frequency;
-
-	/* from power down, we need mark the clock state as ClocksStateNominal
-	 * from HWReset, so when resume we will call pplib voltage regulator.*/
-	if (requested_clk_khz == 0)
-		clk_mgr_dce->cur_min_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
-
-	/*Program DP ref Clock*/
-	/*VBIOS will determine DPREFCLK frequency, so we don't set it*/
-	dce_clk_params.target_clock_frequency = 0;
-	dce_clk_params.clock_type = DCECLOCK_TYPE_DPREFCLK;
-
-	if (!((clk_mgr->ctx->asic_id.chip_family == FAMILY_AI) &&
-	       ASICREV_IS_VEGA20_P(clk_mgr->ctx->asic_id.hw_internal_rev)))
-		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
-			(dce_clk_params.pll_id ==
-					CLOCK_SOURCE_COMBO_DISPLAY_PLL0);
-	else
-		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK = false;
-
-	bp->funcs->set_dce_clock(bp, &dce_clk_params);
-
-	if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment)) {
-		if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
-			if (clk_mgr_dce->dfs_bypass_disp_clk != actual_clock)
-				dmcu->funcs->set_psr_wait_loop(dmcu,
-						actual_clock / 1000 / 7);
-		}
-	}
-
-	clk_mgr_dce->dfs_bypass_disp_clk = actual_clock;
-	return actual_clock;
-}
-
-static void dce_clock_read_integrated_info(struct dce_clk_mgr *clk_mgr_dce)
-{
-	struct dc_debug_options *debug = &clk_mgr_dce->base.ctx->dc->debug;
-	struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
-	struct integrated_info info = { { { 0 } } };
-	struct dc_firmware_info fw_info = { { 0 } };
-	int i;
-
-	if (bp->integrated_info)
-		info = *bp->integrated_info;
-
-	clk_mgr_dce->dentist_vco_freq_khz = info.dentist_vco_freq;
-	if (clk_mgr_dce->dentist_vco_freq_khz == 0) {
-		bp->funcs->get_firmware_info(bp, &fw_info);
-		clk_mgr_dce->dentist_vco_freq_khz =
-			fw_info.smu_gpu_pll_output_freq;
-		if (clk_mgr_dce->dentist_vco_freq_khz == 0)
-			clk_mgr_dce->dentist_vco_freq_khz = 3600000;
-	}
-
-	/*update the maximum display clock for each power state*/
-	for (i = 0; i < NUMBER_OF_DISP_CLK_VOLTAGE; ++i) {
-		enum dm_pp_clocks_state clk_state = DM_PP_CLOCKS_STATE_INVALID;
-
-		switch (i) {
-		case 0:
-			clk_state = DM_PP_CLOCKS_STATE_ULTRA_LOW;
-			break;
-
-		case 1:
-			clk_state = DM_PP_CLOCKS_STATE_LOW;
-			break;
-
-		case 2:
-			clk_state = DM_PP_CLOCKS_STATE_NOMINAL;
-			break;
-
-		case 3:
-			clk_state = DM_PP_CLOCKS_STATE_PERFORMANCE;
-			break;
-
-		default:
-			clk_state = DM_PP_CLOCKS_STATE_INVALID;
-			break;
-		}
-
-		/*Do not allow bad VBIOS/SBIOS to override with invalid values,
-		 * check for > 100MHz*/
-		if (info.disp_clk_voltage[i].max_supported_clk >= 100000)
-			clk_mgr_dce->max_clks_by_state[clk_state].display_clk_khz =
-				info.disp_clk_voltage[i].max_supported_clk;
-	}
-
-	if (!debug->disable_dfs_bypass && bp->integrated_info)
-		if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
-			clk_mgr_dce->dfs_bypass_enabled = true;
-}
-
-void dce_clock_read_ss_info(struct dce_clk_mgr *clk_mgr_dce)
-{
-	struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
-	int ss_info_num = bp->funcs->get_ss_entry_number(
-			bp, AS_SIGNAL_TYPE_GPU_PLL);
-
-	if (ss_info_num) {
-		struct spread_spectrum_info info = { { 0 } };
-		enum bp_result result = bp->funcs->get_spread_spectrum_info(
-				bp, AS_SIGNAL_TYPE_GPU_PLL, 0, &info);
-
-		/* Based on VBIOS, VBIOS will keep entry for GPU PLL SS
-		 * even if SS not enabled and in that case
-		 * SSInfo.spreadSpectrumPercentage !=0 would be sign
-		 * that SS is enabled
-		 */
-		if (result == BP_RESULT_OK &&
-				info.spread_spectrum_percentage != 0) {
-			clk_mgr_dce->ss_on_dprefclk = true;
-			clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
-
-			if (info.type.CENTER_MODE == 0) {
-				/* TODO: Currently for DP Reference clock we
-				 * need only SS percentage for
-				 * downspread */
-				clk_mgr_dce->dprefclk_ss_percentage =
-						info.spread_spectrum_percentage;
-			}
-
-			return;
-		}
-
-		result = bp->funcs->get_spread_spectrum_info(
-				bp, AS_SIGNAL_TYPE_DISPLAY_PORT, 0, &info);
-
-		/* Based on VBIOS, VBIOS will keep entry for DPREFCLK SS
-		 * even if SS not enabled and in that case
-		 * SSInfo.spreadSpectrumPercentage !=0 would be sign
-		 * that SS is enabled
-		 */
-		if (result == BP_RESULT_OK &&
-				info.spread_spectrum_percentage != 0) {
-			clk_mgr_dce->ss_on_dprefclk = true;
-			clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
-
-			if (info.type.CENTER_MODE == 0) {
-				/* Currently for DP Reference clock we
-				 * need only SS percentage for
-				 * downspread */
-				clk_mgr_dce->dprefclk_ss_percentage =
-						info.spread_spectrum_percentage;
-			}
-			if (clk_mgr_dce->base.ctx->dc->debug.ignore_dpref_ss)
-				clk_mgr_dce->dprefclk_ss_percentage = 0;
-		}
-	}
-}
-
-/**
- * dce121_clock_patch_xgmi_ss_info() - Save XGMI spread spectrum info
- * @clk_mgr: clock manager base structure
- *
- * Reads from VBIOS the XGMI spread spectrum info and saves it within
- * the dce clock manager. This operation will overwrite the existing dprefclk
- * SS values if the vBIOS query succeeds. Otherwise, it does nothing. It also
- * sets the ->xgmi_enabled flag.
- */
-void dce121_clock_patch_xgmi_ss_info(struct clk_mgr *clk_mgr)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	enum bp_result result;
-	struct spread_spectrum_info info = { { 0 } };
-	struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
-
-	clk_mgr_dce->xgmi_enabled = false;
-
-	result = bp->funcs->get_spread_spectrum_info(bp, AS_SIGNAL_TYPE_XGMI,
-						     0, &info);
-	if (result == BP_RESULT_OK && info.spread_spectrum_percentage != 0) {
-		clk_mgr_dce->xgmi_enabled = true;
-		clk_mgr_dce->ss_on_dprefclk = true;
-		clk_mgr_dce->dprefclk_ss_divider =
-				info.spread_percentage_divider;
-
-		if (info.type.CENTER_MODE == 0) {
-			/* Currently for DP Reference clock we
-			 * need only SS percentage for
-			 * downspread */
-			clk_mgr_dce->dprefclk_ss_percentage =
-					info.spread_spectrum_percentage;
-		}
-	}
-}
-
-void dce110_fill_display_configs(
-	const struct dc_state *context,
-	struct dm_pp_display_configuration *pp_display_cfg)
-{
-	int j;
-	int num_cfgs = 0;
-
-	for (j = 0; j < context->stream_count; j++) {
-		int k;
-
-		const struct dc_stream_state *stream = context->streams[j];
-		struct dm_pp_single_disp_config *cfg =
-			&pp_display_cfg->disp_configs[num_cfgs];
-		const struct pipe_ctx *pipe_ctx = NULL;
-
-		for (k = 0; k < MAX_PIPES; k++)
-			if (stream == context->res_ctx.pipe_ctx[k].stream) {
-				pipe_ctx = &context->res_ctx.pipe_ctx[k];
-				break;
-			}
-
-		ASSERT(pipe_ctx != NULL);
-
-		/* only notify active stream */
-		if (stream->dpms_off)
-			continue;
-
-		num_cfgs++;
-		cfg->signal = pipe_ctx->stream->signal;
-		cfg->pipe_idx = pipe_ctx->stream_res.tg->inst;
-		cfg->src_height = stream->src.height;
-		cfg->src_width = stream->src.width;
-		cfg->ddi_channel_mapping =
-			stream->link->ddi_channel_mapping.raw;
-		cfg->transmitter =
-			stream->link->link_enc->transmitter;
-		cfg->link_settings.lane_count =
-			stream->link->cur_link_settings.lane_count;
-		cfg->link_settings.link_rate =
-			stream->link->cur_link_settings.link_rate;
-		cfg->link_settings.link_spread =
-			stream->link->cur_link_settings.link_spread;
-		cfg->sym_clock = stream->phy_pix_clk;
-		/* Round v_refresh*/
-		cfg->v_refresh = stream->timing.pix_clk_100hz * 100;
-		cfg->v_refresh /= stream->timing.h_total;
-		cfg->v_refresh = (cfg->v_refresh + stream->timing.v_total / 2)
-							/ stream->timing.v_total;
-	}
-
-	pp_display_cfg->display_count = num_cfgs;
-}
-
-static uint32_t dce110_get_min_vblank_time_us(const struct dc_state *context)
-{
-	uint8_t j;
-	uint32_t min_vertical_blank_time = -1;
-
-	for (j = 0; j < context->stream_count; j++) {
-		struct dc_stream_state *stream = context->streams[j];
-		uint32_t vertical_blank_in_pixels = 0;
-		uint32_t vertical_blank_time = 0;
-
-		vertical_blank_in_pixels = stream->timing.h_total *
-			(stream->timing.v_total
-			 - stream->timing.v_addressable);
-
-		vertical_blank_time = vertical_blank_in_pixels
-			* 10000 / stream->timing.pix_clk_100hz;
-
-		if (min_vertical_blank_time > vertical_blank_time)
-			min_vertical_blank_time = vertical_blank_time;
-	}
-
-	return min_vertical_blank_time;
-}
-
-static int determine_sclk_from_bounding_box(
-		const struct dc *dc,
-		int required_sclk)
-{
-	int i;
-
-	/*
-	 * Some asics do not give us sclk levels, so we just report the actual
-	 * required sclk
-	 */
-	if (dc->sclk_lvls.num_levels == 0)
-		return required_sclk;
-
-	for (i = 0; i < dc->sclk_lvls.num_levels; i++) {
-		if (dc->sclk_lvls.clocks_in_khz[i] >= required_sclk)
-			return dc->sclk_lvls.clocks_in_khz[i];
-	}
-	/*
-	 * even maximum level could not satisfy requirement, this
-	 * is unexpected at this stage, should have been caught at
-	 * validation time
-	 */
-	ASSERT(0);
-	return dc->sclk_lvls.clocks_in_khz[dc->sclk_lvls.num_levels - 1];
-}
-
-static void dce_pplib_apply_display_requirements(
-	struct dc *dc,
-	struct dc_state *context)
-{
-	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
-
-	pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context);
-
-	dce110_fill_display_configs(context, pp_display_cfg);
-
-	if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) !=  0)
-		dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
-}
-
-static void dce11_pplib_apply_display_requirements(
-	struct dc *dc,
-	struct dc_state *context)
-{
-	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
-
-	pp_display_cfg->all_displays_in_sync =
-		context->bw_ctx.bw.dce.all_displays_in_sync;
-	pp_display_cfg->nb_pstate_switch_disable =
-			context->bw_ctx.bw.dce.nbp_state_change_enable == false;
-	pp_display_cfg->cpu_cc6_disable =
-			context->bw_ctx.bw.dce.cpuc_state_change_enable == false;
-	pp_display_cfg->cpu_pstate_disable =
-			context->bw_ctx.bw.dce.cpup_state_change_enable == false;
-	pp_display_cfg->cpu_pstate_separation_time =
-			context->bw_ctx.bw.dce.blackout_recovery_time_us;
-
-	pp_display_cfg->min_memory_clock_khz = context->bw_ctx.bw.dce.yclk_khz
-		/ MEMORY_TYPE_MULTIPLIER_CZ;
-
-	pp_display_cfg->min_engine_clock_khz = determine_sclk_from_bounding_box(
-			dc,
-			context->bw_ctx.bw.dce.sclk_khz);
-
-	/*
-	 * As workaround for >4x4K lightup set dcfclock to min_engine_clock value.
-	 * This is not required for less than 5 displays,
-	 * thus don't request decfclk in dc to avoid impact
-	 * on power saving.
-	 *
-	 */
-	pp_display_cfg->min_dcfclock_khz = (context->stream_count > 4) ?
-			pp_display_cfg->min_engine_clock_khz : 0;
-
-	pp_display_cfg->min_engine_clock_deep_sleep_khz
-			= context->bw_ctx.bw.dce.sclk_deep_sleep_khz;
-
-	pp_display_cfg->avail_mclk_switch_time_us =
-						dce110_get_min_vblank_time_us(context);
-	/* TODO: dce11.2*/
-	pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0;
-
-	pp_display_cfg->disp_clk_khz = dc->res_pool->clk_mgr->clks.dispclk_khz;
-
-	dce110_fill_display_configs(context, pp_display_cfg);
-
-	/* TODO: is this still applicable?*/
-	if (pp_display_cfg->display_count == 1) {
-		const struct dc_crtc_timing *timing =
-			&context->streams[0]->timing;
-
-		pp_display_cfg->crtc_index =
-			pp_display_cfg->disp_configs[0].pipe_idx;
-		pp_display_cfg->line_time_in_us = timing->h_total * 10000 / timing->pix_clk_100hz;
-	}
-
-	if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) !=  0)
-		dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
-}
-
-static void dce_update_clocks(struct clk_mgr *clk_mgr,
-			struct dc_state *context,
-			bool safe_to_lower)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
-
-	/*TODO: W/A for dal3 linux, investigate why this works */
-	if (!clk_mgr_dce->dfs_bypass_active)
-		patched_disp_clk = patched_disp_clk * 115 / 100;
-
-	level_change_req.power_level = dce_get_required_clocks_state(clk_mgr, context);
-	/* get max clock state from PPLIB */
-	if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
-			|| level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
-		if (dm_pp_apply_power_level_change_request(clk_mgr->ctx, &level_change_req))
-			clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
-	}
-
-	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) {
-		patched_disp_clk = dce_set_clock(clk_mgr, patched_disp_clk);
-		clk_mgr->clks.dispclk_khz = patched_disp_clk;
-	}
-	dce_pplib_apply_display_requirements(clk_mgr->ctx->dc, context);
-}
-
-static void dce11_update_clocks(struct clk_mgr *clk_mgr,
-			struct dc_state *context,
-			bool safe_to_lower)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
-
-	/*TODO: W/A for dal3 linux, investigate why this works */
-	if (!clk_mgr_dce->dfs_bypass_active)
-		patched_disp_clk = patched_disp_clk * 115 / 100;
-
-	level_change_req.power_level = dce_get_required_clocks_state(clk_mgr, context);
-	/* get max clock state from PPLIB */
-	if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
-			|| level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
-		if (dm_pp_apply_power_level_change_request(clk_mgr->ctx, &level_change_req))
-			clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
-	}
-
-	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) {
-		context->bw_ctx.bw.dce.dispclk_khz = dce_set_clock(clk_mgr, patched_disp_clk);
-		clk_mgr->clks.dispclk_khz = patched_disp_clk;
-	}
-	dce11_pplib_apply_display_requirements(clk_mgr->ctx->dc, context);
-}
-
-static void dce112_update_clocks(struct clk_mgr *clk_mgr,
-			struct dc_state *context,
-			bool safe_to_lower)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
-
-	/*TODO: W/A for dal3 linux, investigate why this works */
-	if (!clk_mgr_dce->dfs_bypass_active)
-		patched_disp_clk = patched_disp_clk * 115 / 100;
-
-	level_change_req.power_level = dce_get_required_clocks_state(clk_mgr, context);
-	/* get max clock state from PPLIB */
-	if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
-			|| level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
-		if (dm_pp_apply_power_level_change_request(clk_mgr->ctx, &level_change_req))
-			clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
-	}
-
-	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) {
-		patched_disp_clk = dce112_set_clock(clk_mgr, patched_disp_clk);
-		clk_mgr->clks.dispclk_khz = patched_disp_clk;
-	}
-	dce11_pplib_apply_display_requirements(clk_mgr->ctx->dc, context);
-}
-
-static void dce12_update_clocks(struct clk_mgr *clk_mgr,
-			struct dc_state *context,
-			bool safe_to_lower)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
-	struct dm_pp_clock_for_voltage_req clock_voltage_req = {0};
-	int max_pix_clk = get_max_pixel_clock_for_all_paths(context);
-	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
-
-	/*TODO: W/A for dal3 linux, investigate why this works */
-	if (!clk_mgr_dce->dfs_bypass_active)
-		patched_disp_clk = patched_disp_clk * 115 / 100;
-
-	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) {
-		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAY_CLK;
-		/*
-		 * When xGMI is enabled, the display clk needs to be adjusted
-		 * with the WAFL link's SS percentage.
-		 */
-		if (clk_mgr_dce->xgmi_enabled)
-			patched_disp_clk = clk_mgr_adjust_dp_ref_freq_for_ss(
-					clk_mgr_dce, patched_disp_clk);
-		clock_voltage_req.clocks_in_khz = patched_disp_clk;
-		clk_mgr->clks.dispclk_khz = dce112_set_clock(clk_mgr, patched_disp_clk);
-
-		dm_pp_apply_clock_for_voltage_request(clk_mgr->ctx, &clock_voltage_req);
-	}
-
-	if (should_set_clock(safe_to_lower, max_pix_clk, clk_mgr->clks.phyclk_khz)) {
-		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAYPHYCLK;
-		clock_voltage_req.clocks_in_khz = max_pix_clk;
-		clk_mgr->clks.phyclk_khz = max_pix_clk;
-
-		dm_pp_apply_clock_for_voltage_request(clk_mgr->ctx, &clock_voltage_req);
-	}
-	dce11_pplib_apply_display_requirements(clk_mgr->ctx->dc, context);
-}
-
-static const struct clk_mgr_funcs dce120_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
-	.update_clocks = dce12_update_clocks
-};
-
-static const struct clk_mgr_funcs dce112_funcs = {
-	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
-	.update_clocks = dce112_update_clocks
-};
-
-static const struct clk_mgr_funcs dce110_funcs = {
-	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
-	.update_clocks = dce11_update_clocks,
-};
-
-static const struct clk_mgr_funcs dce_funcs = {
-	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
-	.update_clocks = dce_update_clocks
-};
-
-static void dce_clk_mgr_construct(
-	struct dce_clk_mgr *clk_mgr_dce,
-	struct dc_context *ctx,
-	const struct clk_mgr_registers *regs,
-	const struct clk_mgr_shift *clk_shift,
-	const struct clk_mgr_mask *clk_mask)
-{
-	struct clk_mgr *base = &clk_mgr_dce->base;
-	struct dm_pp_static_clock_info static_clk_info = {0};
-
-	base->ctx = ctx;
-	base->funcs = &dce_funcs;
-
-	clk_mgr_dce->regs = regs;
-	clk_mgr_dce->clk_mgr_shift = clk_shift;
-	clk_mgr_dce->clk_mgr_mask = clk_mask;
-
-	clk_mgr_dce->dfs_bypass_disp_clk = 0;
-
-	clk_mgr_dce->dprefclk_ss_percentage = 0;
-	clk_mgr_dce->dprefclk_ss_divider = 1000;
-	clk_mgr_dce->ss_on_dprefclk = false;
-
-
-	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		clk_mgr_dce->max_clks_state = static_clk_info.max_clocks_state;
-	else
-		clk_mgr_dce->max_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
-	clk_mgr_dce->cur_min_clks_state = DM_PP_CLOCKS_STATE_INVALID;
-
-	dce_clock_read_integrated_info(clk_mgr_dce);
-	dce_clock_read_ss_info(clk_mgr_dce);
-}
-
-struct clk_mgr *dce_clk_mgr_create(
-	struct dc_context *ctx,
-	const struct clk_mgr_registers *regs,
-	const struct clk_mgr_shift *clk_shift,
-	const struct clk_mgr_mask *clk_mask)
-{
-	struct dce_clk_mgr *clk_mgr_dce = kzalloc_obj(*clk_mgr_dce);
-
-	if (clk_mgr_dce == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	memcpy(clk_mgr_dce->max_clks_by_state,
-		dce80_max_clks_by_state,
-		sizeof(dce80_max_clks_by_state));
-
-	dce_clk_mgr_construct(
-		clk_mgr_dce, ctx, regs, clk_shift, clk_mask);
-
-	return &clk_mgr_dce->base;
-}
-
-struct clk_mgr *dce110_clk_mgr_create(
-	struct dc_context *ctx,
-	const struct clk_mgr_registers *regs,
-	const struct clk_mgr_shift *clk_shift,
-	const struct clk_mgr_mask *clk_mask)
-{
-	struct dce_clk_mgr *clk_mgr_dce = kzalloc_obj(*clk_mgr_dce);
-
-	if (clk_mgr_dce == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	memcpy(clk_mgr_dce->max_clks_by_state,
-		dce110_max_clks_by_state,
-		sizeof(dce110_max_clks_by_state));
-
-	dce_clk_mgr_construct(
-		clk_mgr_dce, ctx, regs, clk_shift, clk_mask);
-
-	clk_mgr_dce->base.funcs = &dce110_funcs;
-
-	return &clk_mgr_dce->base;
-}
-
-struct clk_mgr *dce112_clk_mgr_create(
-	struct dc_context *ctx,
-	const struct clk_mgr_registers *regs,
-	const struct clk_mgr_shift *clk_shift,
-	const struct clk_mgr_mask *clk_mask)
-{
-	struct dce_clk_mgr *clk_mgr_dce = kzalloc_obj(*clk_mgr_dce);
-
-	if (clk_mgr_dce == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	memcpy(clk_mgr_dce->max_clks_by_state,
-		dce112_max_clks_by_state,
-		sizeof(dce112_max_clks_by_state));
-
-	dce_clk_mgr_construct(
-		clk_mgr_dce, ctx, regs, clk_shift, clk_mask);
-
-	clk_mgr_dce->base.funcs = &dce112_funcs;
-
-	return &clk_mgr_dce->base;
-}
-
-struct clk_mgr *dce120_clk_mgr_create(struct dc_context *ctx)
-{
-	struct dce_clk_mgr *clk_mgr_dce = kzalloc_obj(*clk_mgr_dce);
-
-	if (clk_mgr_dce == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	memcpy(clk_mgr_dce->max_clks_by_state,
-		dce120_max_clks_by_state,
-		sizeof(dce120_max_clks_by_state));
-
-	dce_clk_mgr_construct(
-		clk_mgr_dce, ctx, NULL, NULL, NULL);
-
-	clk_mgr_dce->dprefclk_khz = 600000;
-	clk_mgr_dce->base.funcs = &dce120_funcs;
-
-	return &clk_mgr_dce->base;
-}
-
-struct clk_mgr *dce121_clk_mgr_create(struct dc_context *ctx)
-{
-	struct dce_clk_mgr *clk_mgr_dce = kzalloc_obj(*clk_mgr_dce);
-
-	if (clk_mgr_dce == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	memcpy(clk_mgr_dce->max_clks_by_state, dce120_max_clks_by_state,
-	       sizeof(dce120_max_clks_by_state));
-
-	dce_clk_mgr_construct(clk_mgr_dce, ctx, NULL, NULL, NULL);
-
-	clk_mgr_dce->dprefclk_khz = 625000;
-	clk_mgr_dce->base.funcs = &dce120_funcs;
-
-	return &clk_mgr_dce->base;
-}
-
-void dce_clk_mgr_destroy(struct clk_mgr **clk_mgr)
-{
-	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(*clk_mgr);
-
-	kfree(clk_mgr_dce);
-	*clk_mgr = NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
index 7116fdd4c7ecd..d0ffa99f1fe0d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
@@ -28,6 +28,14 @@
 #include "dc_types.h"
 #include "core_types.h"
 
+static bool dmub_hw_lock_has_inbox0_lock(const struct dc *dc)
+{
+	return dc->ctx && dc->ctx->dmub_srv &&
+	       dc->hwss.dmub_hw_control_lock &&
+	       dc->hwss.dmub_hw_control_lock_fast &&
+	       dc->ctx->dmub_srv->dmub->meta_info.feature_bits.bits.inbox0_lock_support;
+}
+
 void dmub_hw_lock_mgr_cmd(struct dc_dmub_srv *dmub_srv,
 				bool lock,
 				union dmub_hw_lock_flags *hw_locks,
@@ -105,5 +113,13 @@ bool should_use_dmub_inbox1_lock(const struct dc *dc, const struct dc_link *link
 	if (dc->ctx->dce_version >= DCN_VERSION_4_01)
 		return false;
 
+	if (dmub_hw_lock_has_inbox0_lock(dc))
+		return false;
+
 	return dmub_hw_lock_mgr_does_link_require_lock(dc, link);
 }
+
+bool should_use_dmub_inbox0_lock_for_link(const struct dc *dc, const struct dc_link *link)
+{
+	return dmub_hw_lock_has_inbox0_lock(dc) && dmub_hw_lock_mgr_does_link_require_lock(dc, link);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.h b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.h
index 4c80ca8484ad3..3e8caa9303906 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.h
@@ -46,7 +46,38 @@ void dmub_hw_lock_mgr_inbox0_cmd(struct dc_dmub_srv *dmub_srv,
  * Return: true if the inbox1 lock should be used, false otherwise
  */
 bool should_use_dmub_inbox1_lock(const struct dc *dc, const struct dc_link *link);
+
+/**
+ * dmub_hw_lock_mgr_does_link_require_lock() - Returns true if the link has a feature that needs the HW lock.
+ *
+ * @dc: Pointer to DC object
+ * @link: The link to check
+ *
+ * Return: true if the link has a feature that needs the HW lock, false otherwise
+ */
 bool dmub_hw_lock_mgr_does_link_require_lock(const struct dc *dc, const struct dc_link *link);
+
+/**
+ * dmub_hw_lock_mgr_does_context_require_lock() - Returns true if the context has any stream that needs the HW lock.
+ *
+ * @dc: Pointer to DC object
+ * @context: The context to check
+ *
+ * Return: true if the context has any stream that needs the HW lock, false otherwise
+ */
 bool dmub_hw_lock_mgr_does_context_require_lock(const struct dc *dc, const struct dc_state *context);
 
+/**
+ * should_use_dmub_inbox0_lock_for_link() - Checks if the inbox0 interlock with DMU should be used.
+ *
+ * Is not functionally equivalent to inbox1 as DMUB will not own programming of the relevant locking
+ * registers.
+ *
+ * @dc: pointer to DC object
+ * @link: optional pointer to the link object to check for enabled link features
+ *
+ * Return: true if the inbox0 lock should be used, false otherwise
+ */
+bool should_use_dmub_inbox0_lock_for_link(const struct dc *dc, const struct dc_link *link);
+
 #endif /*_DMUB_HW_LOCK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dio/Makefile b/drivers/gpu/drm/amd/display/dc/dio/Makefile
index 2f5619078e1f9..4832533b9514a 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dio/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Advanced Micro Devices, Inc.
+# Copyright 2020-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -123,4 +123,13 @@ DIO_DCN401 = dcn401_dio_link_encoder.o dcn401_dio_stream_encoder.o
 AMD_DAL_DIO_DCN401 = $(addprefix $(AMDDALPATH)/dc/dio/dcn401/,$(DIO_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_DIO_DCN401)
+
+###############################################################################
+# DCN42
+###############################################################################
+DIO_DCN42 = dcn42_dio_link_encoder.o dcn42_dio_stream_encoder.o
+
+AMD_DAL_DIO_DCN42 = $(addprefix $(AMDDALPATH)/dc/dio/dcn42/,$(DIO_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DIO_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.h
index 54a6a4ebd6363..7553675065563 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -190,6 +190,7 @@ struct dcn10_stream_enc_registers {
 	uint32_t DIG_FE_CLK_CNTL;
 	uint32_t DIG_FE_EN_CNTL;
 	uint32_t STREAM_MAPPER_CONTROL;
+	uint32_t DIG_FE_AUDIO_CNTL;
 };
 
 
@@ -598,6 +599,11 @@ struct dcn10_stream_enc_registers {
 	type DP_VID_N_INTERVAL;\
 	type DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE;\
 	type DP_STEER_FIFO_ENABLE
+
+#define SE_REG_FIELD_LIST_DCN_AUDIO_COMMON(type) \
+	type DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL;\
+	type APG_CLOCK_ENABLE
+
 struct dcn10_stream_encoder_shift {
 	SE_REG_FIELD_LIST_DCN1_0(uint8_t);
 	uint8_t HDMI_ACP_SEND;
@@ -606,6 +612,7 @@ struct dcn10_stream_encoder_shift {
 	SE_REG_FIELD_LIST_DCN3_1_COMMON(uint8_t);
 	SE_REG_FIELD_LIST_DCN3_5_COMMON(uint8_t);
 	SE_REG_FIELD_LIST_DCN4_01_COMMON(uint32_t);
+	SE_REG_FIELD_LIST_DCN_AUDIO_COMMON(uint32_t);
 };
 
 struct dcn10_stream_encoder_mask {
@@ -616,6 +623,7 @@ struct dcn10_stream_encoder_mask {
 	SE_REG_FIELD_LIST_DCN3_1_COMMON(uint32_t);
 	SE_REG_FIELD_LIST_DCN3_5_COMMON(uint32_t);
 	SE_REG_FIELD_LIST_DCN4_01_COMMON(uint32_t);
+	SE_REG_FIELD_LIST_DCN_AUDIO_COMMON(uint32_t);
 };
 
 struct dcn10_stream_encoder {
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
new file mode 100644
index 0000000000000..35dfe3bb0c555
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "reg_helper.h"
+
+#include "core_types.h"
+#include "link_encoder.h"
+#include "dcn35/dcn35_dio_link_encoder.h"
+#include "dcn42_dio_link_encoder.h"
+
+#ifndef MIN
+#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
+#endif
+
+#define CTX \
+	enc10->base.ctx
+#define DC_LOGGER \
+	enc10->base.ctx->logger
+
+#define REG(reg)\
+	(enc10->link_regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	enc10->link_shift->field_name, enc10->link_mask->field_name
+
+#define HPD_REG(reg)\
+	(enc10->hpd_regs->reg)
+
+#define HPD_REG_GET(reg_name, field, val)	\
+		generic_reg_get(CTX, HPD_REG(reg_name), \
+				FN(reg_name, field), val)
+
+#define HPD_REG_SET_N(reg_name, n, initial_val, ...)	\
+		generic_reg_set_ex(CTX, \
+				HPD_REG(reg_name), \
+				initial_val, \
+				n, __VA_ARGS__)
+
+#define HPD_REG_SET_2(reg, init_value, f1, v1, f2, v2)	\
+		HPD_REG_SET_N(reg, 2, init_value, \
+				FN(reg, f1), v1,\
+				FN(reg, f2), v2)
+
+bool dcn42_get_hpd_state(struct link_encoder *enc)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+	uint32_t state;
+
+	HPD_REG_GET(DC_HPD_INT_STATUS, DC_HPD_SENSE, &state);
+
+	return state;
+}
+
+bool dcn42_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms,	int delay_on_disconnect_in_ms)
+{
+	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+	HPD_REG_SET_2(DC_HPD_TOGGLE_FILT_CNTL, 0,
+			DC_HPD_CONNECT_INT_DELAY, delay_on_connect_in_ms,
+			DC_HPD_DISCONNECT_INT_DELAY, delay_on_disconnect_in_ms);
+
+	return true;
+}
+
+static const struct link_encoder_funcs dcn42_link_enc_funcs = {
+	.read_state = link_enc2_read_state,
+	.validate_output_with_stream =
+			dcn30_link_encoder_validate_output_with_stream,
+	.hw_init = dcn35_link_encoder_init,
+	.setup = dcn35_link_encoder_setup,
+	.enable_tmds_output = dcn10_link_encoder_enable_tmds_output,
+	.enable_dp_output = dcn35_link_encoder_enable_dp_output,
+	.enable_dp_mst_output = dcn35_link_encoder_enable_dp_mst_output,
+	.disable_output = dcn35_link_encoder_disable_output,
+	.dp_set_lane_settings = dcn10_link_encoder_dp_set_lane_settings,
+	.dp_set_phy_pattern = dcn10_link_encoder_dp_set_phy_pattern,
+	.update_mst_stream_allocation_table =
+		dcn10_link_encoder_update_mst_stream_allocation_table,
+	.psr_program_dp_dphy_fast_training =
+			dcn10_psr_program_dp_dphy_fast_training,
+	.psr_program_secondary_packet = dcn10_psr_program_secondary_packet,
+	.connect_dig_be_to_fe = dcn10_link_encoder_connect_dig_be_to_fe,
+	.enable_hpd = dcn10_link_encoder_enable_hpd,
+	.disable_hpd = dcn10_link_encoder_disable_hpd,
+	.is_dig_enabled = dcn35_is_dig_enabled,
+	.destroy = dcn10_link_encoder_destroy,
+	.fec_set_enable = enc2_fec_set_enable,
+	.fec_set_ready = enc2_fec_set_ready,
+	.fec_is_active = enc2_fec_is_active,
+	.get_dig_frontend = dcn10_get_dig_frontend,
+	.get_dig_mode = dcn401_get_dig_mode,
+	.is_in_alt_mode = dcn32_link_encoder_is_in_alt_mode,
+	.get_max_link_cap = dcn32_link_encoder_get_max_link_cap,
+	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
+	.enable_dpia_output = dcn35_link_encoder_enable_dpia_output,
+	.disable_dpia_output = dcn35_link_encoder_disable_dpia_output,
+	.get_hpd_state = dcn42_get_hpd_state,
+	.program_hpd_filter = dcn42_program_hpd_filter,
+};
+
+void dcn42_link_encoder_construct(
+	struct dcn20_link_encoder *enc20,
+	const struct encoder_init_data *init_data,
+	const struct encoder_feature_support *enc_features,
+	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_aux_registers *aux_regs,
+	const struct dcn10_link_enc_hpd_registers *hpd_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask)
+{
+	struct bp_connector_speed_cap_info bp_cap_info = {0};
+	const struct dc_vbios_funcs *bp_funcs = init_data->ctx->dc_bios->funcs;
+	enum bp_result result = BP_RESULT_OK;
+	struct dcn10_link_encoder *enc10 = &enc20->enc10;
+
+	enc10->base.funcs = &dcn42_link_enc_funcs;
+	enc10->base.ctx = init_data->ctx;
+	enc10->base.id = init_data->encoder;
+
+	enc10->base.hpd_source = init_data->hpd_source;
+	enc10->base.connector = init_data->connector;
+
+
+	enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+
+	enc10->base.features = *enc_features;
+	if (enc10->base.connector.id == CONNECTOR_ID_USBC)
+		enc10->base.features.flags.bits.DP_IS_USB_C = 1;
+
+	enc10->base.transmitter = init_data->transmitter;
+
+	/* set the flag to indicate whether driver poll the I2C data pin
+	 * while doing the DP sink detect
+	 */
+
+/*	if (dal_adapter_service_is_feature_supported(as,
+		FEATURE_DP_SINK_DETECT_POLL_DATA_PIN))
+		enc10->base.features.flags.bits.
+			DP_SINK_DETECT_POLL_DATA_PIN = true;*/
+
+	enc10->base.output_signals =
+		SIGNAL_TYPE_DVI_SINGLE_LINK |
+		SIGNAL_TYPE_DVI_DUAL_LINK |
+		SIGNAL_TYPE_LVDS |
+		SIGNAL_TYPE_DISPLAY_PORT |
+		SIGNAL_TYPE_DISPLAY_PORT_MST |
+		SIGNAL_TYPE_EDP |
+		SIGNAL_TYPE_HDMI_TYPE_A;
+
+	enc10->link_regs = link_regs;
+	enc10->aux_regs = aux_regs;
+	enc10->hpd_regs = hpd_regs;
+	enc10->link_shift = link_shift;
+	enc10->link_mask = link_mask;
+
+	switch (enc10->base.transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		enc10->base.preferred_engine = ENGINE_ID_DIGA;
+	break;
+	case TRANSMITTER_UNIPHY_B:
+		enc10->base.preferred_engine = ENGINE_ID_DIGB;
+	break;
+	case TRANSMITTER_UNIPHY_C:
+		enc10->base.preferred_engine = ENGINE_ID_DIGC;
+	break;
+	case TRANSMITTER_UNIPHY_D:
+		enc10->base.preferred_engine = ENGINE_ID_DIGD;
+	break;
+	case TRANSMITTER_UNIPHY_E:
+		enc10->base.preferred_engine = ENGINE_ID_DIGE;
+	break;
+	default:
+		ASSERT_CRITICAL(false);
+		enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+	}
+
+	/* default to one to mirror Windows behavior */
+	enc10->base.features.flags.bits.HDMI_6GB_EN = 1;
+
+	if (bp_funcs->get_connector_speed_cap_info)
+		result = bp_funcs->get_connector_speed_cap_info(enc10->base.ctx->dc_bios,
+						enc10->base.connector, &bp_cap_info);
+
+	/* Override features with DCE-specific values */
+	if (result == BP_RESULT_OK) {
+		enc10->base.features.flags.bits.IS_HBR2_CAPABLE =
+				bp_cap_info.DP_HBR2_EN;
+		enc10->base.features.flags.bits.IS_HBR3_CAPABLE =
+				bp_cap_info.DP_HBR3_EN;
+		enc10->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN;
+		enc10->base.features.flags.bits.IS_DP2_CAPABLE = 1;
+		enc10->base.features.flags.bits.IS_UHBR10_CAPABLE = bp_cap_info.DP_UHBR10_EN;
+		enc10->base.features.flags.bits.IS_UHBR13_5_CAPABLE = bp_cap_info.DP_UHBR13_5_EN;
+		enc10->base.features.flags.bits.IS_UHBR20_CAPABLE = bp_cap_info.DP_UHBR20_EN;
+	} else {
+		DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n",
+				__func__,
+				result);
+	}
+	if (enc10->base.ctx->dc->debug.hdmi20_disable) {
+		enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
new file mode 100644
index 0000000000000..4b5a9594f2790
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DC_LINK_ENCODER__DCN42_H__
+#define __DC_LINK_ENCODER__DCN42_H__
+
+#include "dcn401/dcn401_dio_link_encoder.h"
+
+#define LINK_ENCODER_MASK_SH_LIST_DCN42(mask_sh) \
+	LE_SF(DIG0_DIG_BE_EN_CNTL, DIG_BE_ENABLE, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_RB_SWITCH_EN, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_HPD_SELECT, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_MODE, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_CLK_EN, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SOFT_RESET, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, HDCP_SOFT_RESET, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_CLOCK_ON, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_HDCP_CLOCK_ON, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_TMDS_CLOCK_ON, mask_sh),\
+	LE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\
+	LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_BYPASS, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE0, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE1, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE2, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE3, mask_sh),\
+	LE_SF(DP0_DP_DPHY_PRBS_CNTL, DPHY_PRBS_EN, mask_sh),\
+	LE_SF(DP0_DP_DPHY_PRBS_CNTL, DPHY_PRBS_SEL, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM1, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM2, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM0, DPHY_SYM3, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM4, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM5, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM1, DPHY_SYM6, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM2, DPHY_SYM7, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SYM2, DPHY_SYM8, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_BS_COUNT, mask_sh),\
+	LE_SF(DP0_DP_DPHY_SCRAM_CNTL, DPHY_SCRAMBLER_ADVANCE, mask_sh),\
+	LE_SF(DP0_DP_DPHY_FAST_TRAINING, DPHY_RX_FAST_TRAINING_CAPABLE, mask_sh),\
+	LE_SF(DP0_DP_DPHY_BS_SR_SWAP_CNTL, DPHY_LOAD_BS_COUNT, mask_sh),\
+	LE_SF(DP0_DP_DPHY_TRAINING_PATTERN_SEL, DPHY_TRAINING_PATTERN_SEL, mask_sh),\
+	LE_SF(DP0_DP_DPHY_HBR2_PATTERN_CONTROL, DP_DPHY_HBR2_PATTERN_CONTROL, mask_sh),\
+	LE_SF(DP0_DP_LINK_CNTL, DP_LINK_TRAINING_COMPLETE, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_IDLE_BS_INTERVAL, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_VBID_DISABLE, mask_sh),\
+	LE_SF(DP0_DP_LINK_FRAMING_CNTL, DP_VID_ENHANCED_FRAME_MODE, mask_sh),\
+	LE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, mask_sh),\
+	LE_SF(DP0_DP_CONFIG, DP_UDI_LANES, mask_sh),\
+	LE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP0_LINE_NUM, mask_sh),\
+	LE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP0_PRIORITY, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SRC0, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SRC1, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SLOT_COUNT0, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT0, DP_MSE_SAT_SLOT_COUNT1, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SRC2, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SRC3, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SLOT_COUNT2, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT1, DP_MSE_SAT_SLOT_COUNT3, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT_UPDATE, DP_MSE_SAT_UPDATE, mask_sh),\
+	LE_SF(DP0_DP_MSE_SAT_UPDATE, DP_MSE_16_MTP_KEEPOUT, mask_sh),\
+	LE_SF(DP_AUX0_AUX_CONTROL, AUX_HPD_SEL, mask_sh),\
+	LE_SF(DP_AUX0_AUX_CONTROL, AUX_LS_READ_EN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_RECEIVE_WINDOW, mask_sh),\
+	LE_SF(HPD0_DC_HPD_CONTROL, DC_HPD_EN, mask_sh),\
+	LE_SF(HPD0_DC_HPD_INT_STATUS, DC_HPD_SENSE, mask_sh),\
+	LE_SF(HPD0_DC_HPD_TOGGLE_FILT_CNTL, DC_HPD_CONNECT_INT_DELAY, mask_sh),\
+	LE_SF(HPD0_DC_HPD_TOGGLE_FILT_CNTL, DC_HPD_DISCONNECT_INT_DELAY, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_EN, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_READY_SHADOW, mask_sh),\
+	LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_ACTIVE_STATUS, mask_sh),\
+	LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_START_WINDOW, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_HALF_SYM_DETECT_LEN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_TRANSITION_FILTER_EN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_START, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_STOP, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_PHASE_DETECT_LEN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_DETECTION_THRESHOLD, mask_sh), \
+	LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_TX_PRECHARGE_LEN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_TX_PRECHARGE_SYMBOLS, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_MODE_DET_CHECK_DELAY, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL1, AUX_RX_PRECHARGE_SKIP, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL1, AUX_RX_TIMEOUT_LEN, mask_sh),\
+	LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL1, AUX_RX_TIMEOUT_LEN_MUL, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, DISPCLK_R_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, DISPCLK_G_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, REFCLK_R_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, REFCLK_G_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SOCCLK_G_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLK_FE_R_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLK_FE_G_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLK_R_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLK_G_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, DIO_FGCG_REP_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, DISPCLK_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKA_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKB_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKC_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKD_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKE_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKF_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKG_G_HDCP_GATE_DIS, mask_sh)
+
+void dcn42_link_encoder_construct(
+	struct dcn20_link_encoder *enc20,
+	const struct encoder_init_data *init_data,
+	const struct encoder_feature_support *enc_features,
+	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_aux_registers *aux_regs,
+	const struct dcn10_link_enc_hpd_registers *hpd_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask);
+
+bool dcn42_get_hpd_state(struct link_encoder *enc);
+bool dcn42_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms,	int delay_on_disconnect_in_ms);
+
+#endif /* __DC_LINK_ENCODER__DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
new file mode 100644
index 0000000000000..65afbfcaa96b0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
@@ -0,0 +1,522 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dc_bios_types.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn32/dcn32_dio_stream_encoder.h"
+#include "dcn35/dcn35_dio_stream_encoder.h"
+#include "dcn401/dcn401_dio_stream_encoder.h"
+#include "dcn31/dcn31_apg.h"
+
+#include "dcn42_dio_stream_encoder.h"
+#include "reg_helper.h"
+#include "hw_shared.h"
+#include "link_service.h"
+#include "dpcd_defs.h"
+
+#define DC_LOGGER \
+		enc1->base.ctx->logger
+
+#define REG(reg)\
+	(enc1->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	enc1->se_shift->field_name, enc1->se_mask->field_name
+
+#define VBI_LINE_0 0
+#define HDMI_CLOCK_CHANNEL_RATE_MORE_340M 340000
+
+#define CTX \
+	enc1->base.ctx
+
+/* setup stream encoder in hdmi mode */
+static void enc42_stream_encoder_hdmi_set_stream_attribute(
+	struct stream_encoder *enc,
+	struct dc_crtc_timing *crtc_timing,
+	int actual_pix_clk_khz,
+	bool enable_audio)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	if (!enc->ctx->dc->debug.avoid_vbios_exec_table) {
+		struct bp_encoder_control cntl = {0};
+
+		cntl.action = ENCODER_CONTROL_SETUP;
+		cntl.engine_id = enc1->base.id;
+		cntl.signal = SIGNAL_TYPE_HDMI_TYPE_A;
+		cntl.enable_dp_audio = enable_audio;
+		cntl.pixel_clock = actual_pix_clk_khz;
+		cntl.lanes_number = LANE_COUNT_FOUR;
+
+		if (enc1->base.bp->funcs->encoder_control(
+				enc1->base.bp, &cntl) != BP_RESULT_OK)
+			return;
+
+	} else {
+
+		//Set pattern for clock channel, default vlue 0x63 does not work
+		REG_UPDATE(DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, 0x1F);
+
+		//DIG_BE_TMDS_HDMI_MODE : TMDS-HDMI mode is already set in link_encoder_setup
+
+		//DIG_SOURCE_SELECT is already set in dig_connect_to_otg
+
+		/* DIG_START is removed from the register spec */
+	}
+
+	/* Configure pixel encoding */
+	enc401_stream_encoder_set_stream_attribute_helper(enc1, crtc_timing);
+
+	/* setup HDMI engine */
+	REG_UPDATE_4(HDMI_CONTROL,
+		HDMI_DEEP_COLOR_ENABLE, 0,
+		HDMI_DATA_SCRAMBLE_EN, 0,
+		HDMI_NO_EXTRA_NULL_PACKET_FILLED, 1,
+		HDMI_CLOCK_CHANNEL_RATE, 0);
+
+	/* Configure color depth */
+	switch (crtc_timing->display_color_depth) {
+	case COLOR_DEPTH_888:
+		REG_UPDATE(HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, 0);
+		break;
+	case COLOR_DEPTH_101010:
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 1,
+					HDMI_DEEP_COLOR_ENABLE, 0);
+		} else {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 1,
+					HDMI_DEEP_COLOR_ENABLE, 1);
+			}
+		break;
+	case COLOR_DEPTH_121212:
+		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 2,
+					HDMI_DEEP_COLOR_ENABLE, 0);
+		} else {
+			REG_UPDATE_2(HDMI_CONTROL,
+					HDMI_DEEP_COLOR_DEPTH, 2,
+					HDMI_DEEP_COLOR_ENABLE, 1);
+			}
+		break;
+	case COLOR_DEPTH_161616:
+		REG_UPDATE_2(HDMI_CONTROL,
+				HDMI_DEEP_COLOR_DEPTH, 3,
+				HDMI_DEEP_COLOR_ENABLE, 1);
+		break;
+	default:
+		break;
+	}
+
+	if (actual_pix_clk_khz >= HDMI_CLOCK_CHANNEL_RATE_MORE_340M) {
+		/* enable HDMI data scrambler
+		 * HDMI_CLOCK_CHANNEL_RATE_MORE_340M
+		 * Clock channel frequency is 1/4 of character rate.
+		 */
+		REG_UPDATE_2(HDMI_CONTROL,
+			HDMI_DATA_SCRAMBLE_EN, 1,
+			HDMI_CLOCK_CHANNEL_RATE, 1);
+	} else if (crtc_timing->flags.LTE_340MCSC_SCRAMBLE) {
+
+		/* TODO: New feature for DCE11, still need to implement */
+
+		/* enable HDMI data scrambler
+		 * HDMI_CLOCK_CHANNEL_FREQ_EQUAL_TO_CHAR_RATE
+		 * Clock channel frequency is the same
+		 * as character rate
+		 */
+		REG_UPDATE_2(HDMI_CONTROL,
+			HDMI_DATA_SCRAMBLE_EN, 1,
+			HDMI_CLOCK_CHANNEL_RATE, 0);
+	}
+
+	/* Enable transmission of General Control packet on every frame */
+	REG_UPDATE_3(HDMI_VBI_PACKET_CONTROL,
+		HDMI_GC_CONT, 1,
+		HDMI_GC_SEND, 1,
+		HDMI_NULL_SEND, 1);
+
+	/* Disable Audio Content Protection packet transmission */
+	REG_UPDATE(HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, 0);
+	/* following belongs to audio */
+	/* Enable Audio InfoFrame packet transmission. */
+	REG_UPDATE(HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, 1);
+
+	/* Select line number on which to send Audio InfoFrame packets */
+	REG_UPDATE(HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_LINE,
+				VBI_LINE_0 + 2);
+
+	/* set HDMI GC AVMUTE */
+	REG_UPDATE(HDMI_GC, HDMI_GC_AVMUTE, 0);
+}
+
+static void enc42_stream_encoder_stop_dp_info_packets(
+	struct stream_encoder *enc)
+{
+	/* stop generic packets on DP */
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t value = 0;
+
+	REG_SET_9(DP_SEC_CNTL, 0,
+		DP_SEC_GSP0_ENABLE, 0,
+		DP_SEC_GSP1_ENABLE, 0,
+		DP_SEC_GSP2_ENABLE, 0,
+		DP_SEC_GSP3_ENABLE, 0,
+		DP_SEC_GSP4_ENABLE, 0,
+		DP_SEC_GSP5_ENABLE, 0,
+		DP_SEC_GSP6_ENABLE, 0,
+		DP_SEC_GSP7_ENABLE, 0,
+		DP_SEC_STREAM_ENABLE, 0);
+
+	/* this register shared with audio info frame.
+	 * therefore we need to keep master enabled
+	 * if at least one of the fields is not 0
+	 */
+	value = REG_READ(DP_SEC_CNTL);
+	if (value)
+		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+static void enc42_stream_encoder_update_hdmi_info_packets(
+	struct stream_encoder *enc,
+	const struct encoder_info_frame *info_frame)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	REG_UPDATE(HDMI_DB_CONTROL, HDMI_DB_DISABLE, 1);
+	REG_UPDATE(DIG_FE_AUDIO_CNTL, APG_CLOCK_ENABLE, 1);
+
+	/*Always add mandatory packets first followed by optional ones*/
+	enc3_update_hdmi_info_packet(enc1, 0, &info_frame->avi);
+	enc3_update_hdmi_info_packet(enc1, 5, &info_frame->hfvsif);
+	enc3_update_hdmi_info_packet(enc1, 2, &info_frame->gamut);
+	enc3_update_hdmi_info_packet(enc1, 1, &info_frame->vendor);
+	enc3_update_hdmi_info_packet(enc1, 3, &info_frame->spd);
+	enc3_update_hdmi_info_packet(enc1, 4, &info_frame->hdrsmd);
+	enc3_update_hdmi_info_packet(enc1, 6, &info_frame->vtem);
+}
+
+static void enc42_dp_set_dsc_pps_info_packet(struct stream_encoder *enc,
+					bool enable,
+					uint8_t *dsc_packed_pps,
+					bool immediate_update)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	if (enable) {
+		struct dc_info_packet pps_sdp;
+		int i;
+
+		/* Configure for PPS packet size (128 bytes) */
+		REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP11_PPS, 1);
+
+		/* Load PPS into infoframe (SDP) registers */
+		pps_sdp.valid = true;
+		pps_sdp.hb0 = 0;
+		pps_sdp.hb1 = DC_DP_INFOFRAME_TYPE_PPS;
+		pps_sdp.hb2 = 127;
+		pps_sdp.hb3 = 0;
+
+		for (i = 0; i < 4; i++) {
+			memcpy(pps_sdp.sb, &dsc_packed_pps[i * 32], 32);
+			enc1->base.vpg->funcs->update_generic_info_packet(
+							enc1->base.vpg,
+							11 + i,
+							&pps_sdp,
+							immediate_update);
+		}
+
+		/* SW should make sure VBID[6] update line number is bigger
+		 * than PPS transmit line number
+		 */
+		REG_UPDATE(DP_GSP11_CNTL,
+				DP_SEC_GSP11_LINE_NUM, 2);
+		REG_UPDATE_2(DP_MSA_VBID_MISC,
+				DP_VBID6_LINE_REFERENCE, 0,
+				DP_VBID6_LINE_NUM, 3);
+
+		/* Send PPS data at the line number specified above.
+		 * DP spec requires PPS to be sent only when it changes, however since
+		 * decoder has to be able to handle its change on every frame, we're
+		 * sending it always (i.e. on every frame) to reduce the chance it'd be
+		 * missed by decoder. If it turns out required to send PPS only when it
+		 * changes, we can use DP_SEC_GSP11_SEND register.
+		 */
+		REG_UPDATE(DP_GSP11_CNTL,
+			DP_SEC_GSP11_ENABLE, 1);
+		REG_UPDATE(DP_SEC_CNTL,
+			DP_SEC_STREAM_ENABLE, 1);
+	} else {
+		/* Disable Generic Stream Packet 11 (GSP) transmission */
+		REG_UPDATE(DP_GSP11_CNTL, DP_SEC_GSP11_ENABLE, 0);
+		REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP11_PPS, 0);
+	}
+}
+
+static void enc42_se_dp_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	enc42_se_enable_audio_clock(enc, true);
+
+	REG_UPDATE(DIG_FE_AUDIO_CNTL,
+			DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL, az_inst);
+
+	enc->apg->funcs->se_audio_setup(enc->apg, az_inst, info);
+}
+
+#define DP_SEC_AUD_N__DP_SEC_AUD_N__DEFAULT 0x8000
+#define DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__AUTO_CALC 1
+
+static void enc42_se_setup_dp_audio(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	/* --- DP Audio packet configurations --- */
+
+	/* ATP Configuration */
+	REG_SET(DP_SEC_AUD_N, 0,
+			DP_SEC_AUD_N, DP_SEC_AUD_N__DP_SEC_AUD_N__DEFAULT);
+
+	/* Async/auto-calc timestamp mode */
+	REG_SET(DP_SEC_TIMESTAMP, 0, DP_SEC_TIMESTAMP_MODE,
+			DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__AUTO_CALC);
+}
+
+static void enc42_se_dp_audio_enable(
+	struct stream_encoder *enc)
+{
+	ASSERT (enc->apg);
+	enc42_se_enable_audio_clock(enc, true);
+	/*APG_CONTROL2 for DP*/
+	enc42_se_setup_dp_audio(enc);
+	/*DP_SEC_CNTL programming*/
+	enc1_se_enable_dp_audio(enc);
+	/*APG_en*/
+	enc->apg->funcs->enable_apg(enc->apg);
+
+}
+
+static void enc42_se_disable_dp_audio(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+	uint32_t value = 0;
+
+	/* Disable Audio packets */
+	REG_UPDATE_5(DP_SEC_CNTL,
+			DP_SEC_ASP_ENABLE, 0,
+			DP_SEC_ATP_ENABLE, 0,
+			DP_SEC_AIP_ENABLE, 0,
+			DP_SEC_ACM_ENABLE, 0,
+			DP_SEC_STREAM_ENABLE, 0);
+
+	/* This register shared with encoder info frame. Therefore we need to
+	 * keep master enabled if at least on of the fields is not 0
+	 */
+	value = REG_READ(DP_SEC_CNTL);
+	if (value != 0)
+		REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+static void enc42_se_dp_audio_disable(
+	struct stream_encoder *enc)
+{
+
+	enc->apg->funcs->disable_apg(enc->apg);
+	enc42_se_disable_dp_audio(enc);
+	enc42_se_enable_audio_clock(enc, false);
+}
+
+static void enc42_se_setup_hdmi_audio(
+	struct stream_encoder *enc,
+	const struct audio_crtc_info *crtc_info)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	struct audio_clock_info audio_clock_info = {0};
+
+	/* HDMI_AUDIO_PACKET_CONTROL */
+	REG_UPDATE(HDMI_AUDIO_PACKET_CONTROL,
+			HDMI_AUDIO_DELAY_EN, 1);
+
+	/* HDMI_ACR_PACKET_CONTROL */
+	REG_UPDATE_2(HDMI_ACR_PACKET_CONTROL,
+			HDMI_ACR_AUTO_SEND, 1,
+			HDMI_ACR_SOURCE, 0);
+
+	/* Program audio clock sample/regeneration parameters */
+	get_audio_clock_info(crtc_info->color_depth,
+			     crtc_info->requested_pixel_clock_100Hz,
+			     crtc_info->calculated_pixel_clock_100Hz,
+			     &audio_clock_info);
+	DC_LOG_HW_AUDIO("\n%s:Input::requested_pixel_clock_100Hz = %d" \
+			"calculated_pixel_clock_100Hz = %d\n", __func__,\
+			crtc_info->requested_pixel_clock_100Hz, \
+			crtc_info->calculated_pixel_clock_100Hz);
+
+	/* HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK */
+	REG_UPDATE(HDMI_ACR_32_0, HDMI_ACR_CTS_32, audio_clock_info.cts_32khz);
+
+	/* HDMI_ACR_32_1__HDMI_ACR_N_32_MASK */
+	REG_UPDATE(HDMI_ACR_32_1, HDMI_ACR_N_32, audio_clock_info.n_32khz);
+
+	/* HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK */
+	REG_UPDATE(HDMI_ACR_44_0, HDMI_ACR_CTS_44, audio_clock_info.cts_44khz);
+
+	/* HDMI_ACR_44_1__HDMI_ACR_N_44_MASK */
+	REG_UPDATE(HDMI_ACR_44_1, HDMI_ACR_N_44, audio_clock_info.n_44khz);
+
+	/* HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK */
+	REG_UPDATE(HDMI_ACR_48_0, HDMI_ACR_CTS_48, audio_clock_info.cts_48khz);
+
+	/* HDMI_ACR_48_1__HDMI_ACR_N_48_MASK */
+	REG_UPDATE(HDMI_ACR_48_1, HDMI_ACR_N_48, audio_clock_info.n_48khz);
+
+	/* Video driver cannot know in advance which sample rate will
+	 * be used by HD Audio driver
+	 * HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE field is
+	 * programmed below in interruppt callback
+	 */
+}
+
+static void enc42_se_hdmi_audio_disable(
+	struct stream_encoder *enc)
+{
+	ASSERT (enc->apg);
+	enc->apg->funcs->disable_apg(enc->apg);
+	enc42_se_enable_audio_clock(enc, false);
+}
+
+void enc42_se_enable_audio_clock(
+	struct stream_encoder *enc,
+	bool enable)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	REG_UPDATE(DIG_FE_AUDIO_CNTL, APG_CLOCK_ENABLE, !!enable);
+}
+
+
+static void enc42_se_hdmi_audio_setup(
+	struct stream_encoder *enc,
+	unsigned int az_inst,
+	struct audio_info *info,
+	struct audio_crtc_info *audio_crtc_info)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	ASSERT (enc->apg);
+	enc42_se_enable_audio_clock(enc, true);
+	/*ACR setup*/
+	enc42_se_setup_hdmi_audio(enc, audio_crtc_info);
+
+	REG_UPDATE(DIG_FE_AUDIO_CNTL,
+				DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL, az_inst);
+
+	/*APG_DBG_GEN_CONTROL*/
+	enc->apg->funcs->se_audio_setup(enc->apg, az_inst, info);
+	/*APG enable*/
+	enc->apg->funcs->enable_apg(enc->apg);
+}
+
+static void enc42_audio_mute_control(
+	struct stream_encoder *enc,
+	bool mute)
+{
+	if (mute)
+		enc->apg->funcs->disable_apg(enc->apg);
+	else
+		enc->apg->funcs->enable_apg(enc->apg);
+}
+void enc42_reset_hdmi_stream_attribute(
+	struct stream_encoder *enc)
+{
+	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+	REG_UPDATE_3(HDMI_CONTROL,
+		HDMI_DEEP_COLOR_ENABLE, 0,
+		HDMI_DATA_SCRAMBLE_EN, 0,
+		HDMI_CLOCK_CHANNEL_RATE, 0);
+}
+
+static const struct stream_encoder_funcs dcn42_str_enc_funcs = {
+	.dp_set_stream_attribute =
+		enc401_stream_encoder_dp_set_stream_attribute,
+	.hdmi_set_stream_attribute =
+		enc42_stream_encoder_hdmi_set_stream_attribute,
+	.dvi_set_stream_attribute =
+		enc401_stream_encoder_dvi_set_stream_attribute,
+	.set_throttled_vcp_size =
+		enc1_stream_encoder_set_throttled_vcp_size,
+	.update_hdmi_info_packets =
+		enc42_stream_encoder_update_hdmi_info_packets,
+	.stop_hdmi_info_packets =
+		enc3_stream_encoder_stop_hdmi_info_packets,
+	.update_dp_info_packets_sdp_line_num =
+		enc3_stream_encoder_update_dp_info_packets_sdp_line_num,
+	.update_dp_info_packets =
+		enc3_stream_encoder_update_dp_info_packets,
+	.stop_dp_info_packets =
+		enc42_stream_encoder_stop_dp_info_packets,
+	.dp_blank =
+		enc1_stream_encoder_dp_blank,
+	.dp_unblank =
+		enc401_stream_encoder_dp_unblank,
+	.audio_mute_control = enc42_audio_mute_control,
+
+	.dp_audio_setup = enc42_se_dp_audio_setup,
+	.dp_audio_enable = enc42_se_dp_audio_enable,
+	.dp_audio_disable = enc42_se_dp_audio_disable,
+
+	.hdmi_audio_setup = enc42_se_hdmi_audio_setup,
+	.hdmi_audio_disable = enc42_se_hdmi_audio_disable,
+	.setup_stereo_sync  = enc1_setup_stereo_sync,
+	.set_avmute = enc1_stream_encoder_set_avmute,
+	.dig_connect_to_otg = enc1_dig_connect_to_otg,
+	.dig_source_otg = enc1_dig_source_otg,
+
+	.dp_get_pixel_format  = enc1_stream_encoder_dp_get_pixel_format,
+
+	.enc_read_state = enc401_read_state,
+	.dp_set_dsc_config = NULL,
+	.dp_set_dsc_pps_info_packet = enc42_dp_set_dsc_pps_info_packet,
+	.set_dynamic_metadata = enc401_set_dynamic_metadata,
+	.hdmi_reset_stream_attribute = enc42_reset_hdmi_stream_attribute,
+	.enable_stream = enc401_stream_encoder_enable,
+
+	.set_input_mode = enc401_set_dig_input_mode,
+	.enable_fifo = enc35_enable_fifo,
+	.disable_fifo = enc35_disable_fifo,
+	.map_stream_to_link = enc401_stream_encoder_map_to_link,
+};
+
+void dcn42_dio_stream_encoder_construct(
+	struct dcn10_stream_encoder *enc1,
+	struct dc_context *ctx,
+	struct dc_bios *bp,
+	enum engine_id eng_id,
+	struct vpg *vpg,
+	struct apg *apg,
+	const struct dcn10_stream_enc_registers *regs,
+	const struct dcn10_stream_encoder_shift *se_shift,
+	const struct dcn10_stream_encoder_mask *se_mask)
+{
+	enc1->base.funcs = &dcn42_str_enc_funcs;
+	enc1->base.ctx = ctx;
+	enc1->base.id = eng_id;
+	enc1->base.bp = bp;
+	enc1->base.vpg = vpg;
+	enc1->base.apg = apg;
+	enc1->regs = regs;
+	enc1->se_shift = se_shift;
+	enc1->se_mask = se_mask;
+	enc1->base.stream_enc_inst = vpg->inst;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.h
new file mode 100644
index 0000000000000..7a98fee46081f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.h
@@ -0,0 +1,206 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 - Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_DIO_STREAM_ENCODER_DCN42_H__
+#define __DC_DIO_STREAM_ENCODER_DCN42_H__
+
+#include "dcn30/dcn30_vpg.h"
+#include "stream_encoder.h"
+#include "dcn20/dcn20_stream_encoder.h"
+
+#define SE_COMMON_MASK_SH_LIST_DCN42(mask_sh)\
+	SE_SF(DP0_DP_PIXEL_FORMAT, PIXEL_ENCODING_TYPE, mask_sh),\
+	SE_SF(DP0_DP_PIXEL_FORMAT, UNCOMPRESSED_PIXEL_FORMAT, mask_sh),\
+	SE_SF(DP0_DP_PIXEL_FORMAT, UNCOMPRESSED_COMPONENT_DEPTH, mask_sh),\
+	SE_SF(DP0_DP_PIXEL_FORMAT, COMPRESSED_PIXEL_FORMAT, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DEEP_COLOR_ENABLE, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DEEP_COLOR_DEPTH, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_DATA_SCRAMBLE_EN, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_NO_EXTRA_NULL_PACKET_FILLED, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_NULL_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GC, HDMI_GC_AVMUTE, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_CNTL, DP_MSE_RATE_X, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_CNTL, DP_MSE_RATE_Y, mask_sh),\
+	SE_SF(DP0_DP_MSE_RATE_UPDATE, DP_MSE_RATE_UPDATE_PENDING, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP0_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP1_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP5_LINE_REFERENCE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND_PENDING, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL4, DP_SEC_GSP4_LINE_NUM, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL5, DP_SEC_GSP5_LINE_NUM, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND_ANY_LINE, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_DIS_DEFER, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_STATUS, mask_sh),\
+	SE_SF(DP0_DP_STEER_FIFO, DP_STEER_FIFO_RESET, mask_sh),\
+	SE_SF(DP0_DP_STEER_FIFO, DP_STEER_FIFO_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_TIMING, DP_VID_M_N_GEN_EN, mask_sh),\
+	SE_SF(DP0_DP_VID_N, DP_VID_N, mask_sh),\
+	SE_SF(DP0_DP_VID_M, DP_VID_M, mask_sh),\
+	SE_SF(DIG0_HDMI_AUDIO_PACKET_CONTROL, HDMI_AUDIO_DELAY_EN, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_PACKET_CONTROL, HDMI_ACR_AUTO_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_PACKET_CONTROL, HDMI_ACR_SOURCE, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_32_0, HDMI_ACR_CTS_32, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_32_1, HDMI_ACR_N_32, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_44_0, HDMI_ACR_CTS_44, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_44_1, HDMI_ACR_N_44, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_48_0, HDMI_ACR_CTS_48, mask_sh),\
+	SE_SF(DIG0_HDMI_ACR_48_1, HDMI_ACR_N_48, mask_sh),\
+	SE_SF(DP0_DP_SEC_AUD_N, DP_SEC_AUD_N, mask_sh),\
+	SE_SF(DP0_DP_SEC_TIMESTAMP, DP_SEC_TIMESTAMP_MODE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ASP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ATP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_AIP_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_ACM_ENABLE, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, HDMI_CLOCK_CHANNEL_RATE, mask_sh),\
+	SE_SF(DIG1_HDMI_CONTROL, TMDS_PIXEL_ENCODING, mask_sh),\
+	SE_SF(DIG1_HDMI_CONTROL, TMDS_COLOR_FORMAT, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_STEREOSYNC_SELECT, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_STEREOSYNC_GATE_EN, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP4_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP5_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP6_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP7_SEND, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL6, DP_SEC_GSP7_LINE_NUM, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP11_PPS, mask_sh),\
+	SE_SF(DP0_DP_GSP11_CNTL, DP_SEC_GSP11_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_GSP11_CNTL, DP_SEC_GSP11_LINE_NUM, mask_sh),\
+	SE_SF(DP0_DP_DB_CNTL, DP_DB_DISABLE, mask_sh),\
+	SE_SF(DP0_DP_MSA_COLORIMETRY, DP_MSA_MISC0, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM1, DP_MSA_HTOTAL, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM1, DP_MSA_VTOTAL, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM2, DP_MSA_HSTART, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM2, DP_MSA_VSTART, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_HSYNCWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_HSYNCPOLARITY, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_VSYNCWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM3, DP_MSA_VSYNCPOLARITY, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM4, DP_MSA_HWIDTH, mask_sh),\
+	SE_SF(DP0_DP_MSA_TIMING_PARAM4, DP_MSA_VHEIGHT, mask_sh),\
+	SE_SF(DIG0_HDMI_DB_CONTROL, HDMI_DB_DISABLE, mask_sh),\
+	SE_SF(DP0_DP_VID_TIMING, DP_VID_N_INTERVAL, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_SOURCE_SELECT, mask_sh), \
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC2_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC2_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC3_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC3_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC4_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC4_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC5_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC5_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC6_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC6_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC7_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC7_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC8_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC8_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC9_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC9_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC10_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC10_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC11_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC11_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC12_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC12_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC13_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC13_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC14_CONT, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL6, HDMI_GENERIC14_SEND, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL1, HDMI_GENERIC0_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL1, HDMI_GENERIC1_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL2, HDMI_GENERIC2_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL2, HDMI_GENERIC3_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL3, HDMI_GENERIC4_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL3, HDMI_GENERIC5_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL4, HDMI_GENERIC6_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL4, HDMI_GENERIC7_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL7, HDMI_GENERIC8_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL7, HDMI_GENERIC9_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL8, HDMI_GENERIC10_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL8, HDMI_GENERIC11_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL9, HDMI_GENERIC12_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL9, HDMI_GENERIC13_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL10, HDMI_GENERIC14_LINE, mask_sh),\
+	SE_SF(DP0_DP_MSA_VBID_MISC, DP_VBID6_LINE_REFERENCE, mask_sh),\
+	SE_SF(DP0_DP_MSA_VBID_MISC, DP_VBID6_LINE_NUM, mask_sh),\
+	SE_SF(DME0_DME_CONTROL, METADATA_ENGINE_EN, mask_sh),\
+	SE_SF(DME0_DME_CONTROL, METADATA_HUBP_REQUESTOR_ID, mask_sh),\
+	SE_SF(DME0_DME_CONTROL, METADATA_STREAM_TYPE, mask_sh),\
+	SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_LINE_REFERENCE, mask_sh),\
+	SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_ENABLE, mask_sh),\
+	SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_LINE_REFERENCE, mask_sh),\
+	SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_LINE, mask_sh),\
+	SE_SF(DIG0_HDMI_CONTROL, DOLBY_VISION_EN, mask_sh),\
+	SE_SF(DIG0_DIG_FE_EN_CNTL, DIG_FE_ENABLE, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CLK_CNTL, DIG_FE_MODE, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CLK_CNTL, DIG_FE_CLK_EN, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CLK_CNTL, DIG_FE_SOFT_RESET, mask_sh),\
+	SE_SF(DIG0_DIG_FE_CNTL, DIG_STEREOSYNC_GATE_EN, mask_sh),\
+	SE_SF(DP0_DP_SEC_FRAMING4, DP_SST_SDP_SPLITTING, mask_sh),\
+	SE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, mask_sh),\
+	SE_SF(DIG0_STREAM_MAPPER_CONTROL, DIG_STREAM_LINK_TARGET, mask_sh),\
+	SE_SF(DIG0_DIG_FE_AUDIO_CNTL, DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL, mask_sh),\
+	SE_SF(DIG0_DIG_FE_AUDIO_CNTL, APG_CLOCK_ENABLE, mask_sh)
+
+void dcn42_dio_stream_encoder_construct(
+	struct dcn10_stream_encoder *enc1,
+	struct dc_context *ctx,
+	struct dc_bios *bp,
+	enum engine_id eng_id,
+	struct vpg *vpg,
+	struct apg *apg,
+	const struct dcn10_stream_enc_registers *regs,
+	const struct dcn10_stream_encoder_shift *se_shift,
+	const struct dcn10_stream_encoder_mask *se_mask);
+
+void enc42_se_enable_audio_clock(
+	struct stream_encoder *enc,
+	bool enable);
+
+void enc42_reset_hdmi_stream_attribute(
+	struct stream_encoder *enc);
+#endif /* __DC_DIO_STREAM_ENCODER_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
index cd8cca6514196..48905ca39b706 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
@@ -5304,7 +5304,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 	double LinesInDETC;
 	unsigned int LinesInDETYRoundedDownToSwath[DC__NUM_DPP__MAX];
 	unsigned int LinesInDETCRoundedDownToSwath;
-	double FullDETBufferingTimeY[DC__NUM_DPP__MAX];
+	double FullDETBufferingTimeY[DC__NUM_DPP__MAX] = { 0 };
 	double FullDETBufferingTimeC;
 	double ActiveDRAMClockChangeLatencyMarginY;
 	double ActiveDRAMClockChangeLatencyMarginC;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
index 0748ef36a16a2..19b142412a84b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
@@ -459,8 +459,8 @@ void dml32_CalculateSwathAndDETConfiguration(
 {
 	unsigned int MaximumSwathHeightY[DC__NUM_DPP__MAX];
 	unsigned int MaximumSwathHeightC[DC__NUM_DPP__MAX];
-	unsigned int RoundedUpMaxSwathSizeBytesY[DC__NUM_DPP__MAX];
-	unsigned int RoundedUpMaxSwathSizeBytesC[DC__NUM_DPP__MAX];
+	unsigned int RoundedUpMaxSwathSizeBytesY[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int RoundedUpMaxSwathSizeBytesC[DC__NUM_DPP__MAX] = { 0 };
 	unsigned int RoundedUpSwathSizeBytesY;
 	unsigned int RoundedUpSwathSizeBytesC;
 	double SwathWidthdoubleDPP[DC__NUM_DPP__MAX];
@@ -2988,7 +2988,7 @@ void dml32_UseMinimumDCFCLK(
 		for  (j = 0; j <= 1; ++j) {
 			double PixelDCFCLKCyclesRequiredInPrefetch[DC__NUM_DPP__MAX];
 			double PrefetchPixelLinesTime[DC__NUM_DPP__MAX];
-			double DCFCLKRequiredForPeakBandwidthPerSurface[DC__NUM_DPP__MAX];
+			double DCFCLKRequiredForPeakBandwidthPerSurface[DC__NUM_DPP__MAX] = { 0 };
 			double DynamicMetadataVMExtraLatency[DC__NUM_DPP__MAX];
 			double MinimumTWait = 0.0;
 			double DPTEBandwidth;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
index 30cfc0848792e..70d9f2cd0b601 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
@@ -90,6 +90,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.o := $(dml2
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.o := $(dml2_ccflags)
+CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn3.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.o := $(dml2_ccflags)
@@ -107,6 +108,7 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_top/dml2_top_soc15.o := $(d
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.o := $(dml2_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn3.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.o := $(dml2_rcflags)
@@ -124,6 +126,7 @@ DML21 += src/dml2_core/dml2_core_dcn4_calcs.o
 DML21 += src/dml2_dpmm/dml2_dpmm_dcn4.o
 DML21 += src/dml2_dpmm/dml2_dpmm_factory.o
 DML21 += src/dml2_mcg/dml2_mcg_dcn4.o
+DML21 += src/dml2_mcg/dml2_mcg_dcn42.o
 DML21 += src/dml2_mcg/dml2_mcg_factory.o
 DML21 += src/dml2_pmo/dml2_pmo_dcn3.o
 DML21 += src/dml2_pmo/dml2_pmo_factory.o
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
index 09303c282495b..8e8935995fca4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
@@ -4089,8 +4089,8 @@ static void CalculateSwathAndDETConfiguration(struct display_mode_lib_scratch_st
 	dml_uint_t MaximumSwathHeightC[__DML_NUM_PLANES__];
 	dml_uint_t RoundedUpMaxSwathSizeBytesY[__DML_NUM_PLANES__];
 	dml_uint_t RoundedUpMaxSwathSizeBytesC[__DML_NUM_PLANES__];
-	dml_uint_t RoundedUpSwathSizeBytesY[__DML_NUM_PLANES__];
-	dml_uint_t RoundedUpSwathSizeBytesC[__DML_NUM_PLANES__];
+	dml_uint_t RoundedUpSwathSizeBytesY[__DML_NUM_PLANES__] = { 0 };
+	dml_uint_t RoundedUpSwathSizeBytesC[__DML_NUM_PLANES__] = { 0 };
 	dml_uint_t SwathWidthSingleDPP[__DML_NUM_PLANES__];
 	dml_uint_t SwathWidthSingleDPPChroma[__DML_NUM_PLANES__];
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
index bf5e7f4e04167..847fab5087503 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
@@ -45,6 +45,9 @@ static enum dml2_project_id dml21_dcn_revision_to_dml2_project_id(enum dce_versi
 	case DCN_VERSION_4_01:
 		project_id = dml2_project_dcn4x_stage2_auto_drr_svp;
 		break;
+	case DCN_VERSION_4_2:
+		project_id = dml2_project_dcn42;
+		break;
 	default:
 		project_id = dml2_project_invalid;
 		DC_ERR("unsupported dcn version for DML21!");
@@ -598,29 +601,31 @@ static void populate_dml21_plane_config_from_plane_state(struct dml2_context *dm
 
 	plane->composition.viewport.stationary = false;
 
-	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
+	if (plane_state->cm.flags.bits.lut3d_dma_enable) {
 		plane->tdlut.setup_for_tdlut = true;
 
-		switch (plane_state->mcm_luts.lut3d_data.gpu_mem_params.layout) {
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+		switch (plane_state->cm.lut3d_dma.swizzle) {
+		case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
+		case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
 			plane->tdlut.tdlut_addressing_mode = dml2_tdlut_sw_linear;
 			break;
-		case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+		case CM_LUT_1D_PACKED_LINEAR:
+		default:
 			plane->tdlut.tdlut_addressing_mode = dml2_tdlut_simple_linear;
 			break;
 		}
 
-		switch (plane_state->mcm_luts.lut3d_data.gpu_mem_params.size) {
-		case DC_CM2_GPU_MEM_SIZE_171717:
-			plane->tdlut.tdlut_width_mode = dml2_tdlut_width_17_cube;
+		switch (plane_state->cm.lut3d_dma.size) {
+		case CM_LUT_SIZE_333333:
+			plane->tdlut.tdlut_width_mode = dml2_tdlut_width_33_cube;
 			break;
-		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
+		case CM_LUT_SIZE_171717:
 		default:
-			//plane->tdlut.tdlut_width_mode = dml2_tdlut_width_flatten; // dml2_tdlut_width_flatten undefined
+			plane->tdlut.tdlut_width_mode = dml2_tdlut_width_17_cube;
 			break;
 		}
 	}
+
 	plane->tdlut.setup_for_tdlut |= dml_ctx->config.force_tdlut_enable;
 
 	plane->dynamic_meta_data.enable = false;
@@ -734,8 +739,12 @@ bool dml21_map_dc_state_into_dml_display_cfg(const struct dc *in_dc, struct dc_s
 	memset(&dml_ctx->v21.dml_to_dc_pipe_mapping, 0, sizeof(struct dml2_dml_to_dc_pipe_mapping));
 
 	dml_dispcfg->gpuvm_enable = dml_ctx->config.gpuvm_enable;
-	dml_dispcfg->gpuvm_max_page_table_levels = 4;
-	dml_dispcfg->hostvm_enable = false;
+	if (dml_ctx->v21.dml_init.soc_bb.gpuvm_max_page_table_levels)
+		dml_dispcfg->gpuvm_max_page_table_levels = dml_ctx->v21.dml_init.soc_bb.gpuvm_max_page_table_levels;
+	else
+		dml_dispcfg->gpuvm_max_page_table_levels = 4;
+	dml_dispcfg->hostvm_enable = dml_ctx->config.hostvm_enable;
+	dml_dispcfg->hostvm_max_non_cached_page_table_levels = dml_ctx->v21.dml_init.soc_bb.hostvm_max_non_cached_page_table_levels;
 	dml_dispcfg->minimize_det_reallocation = true;
 	dml_dispcfg->overrides.enable_subvp_implicit_pmo = true;
 
@@ -820,6 +829,9 @@ void dml21_copy_clocks_to_dc_state(struct dml2_context *in_ctx, struct dc_state
 	context->bw_ctx.bw.dcn.clk.subvp_prefetch_fclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.svp_prefetch_no_throttle.fclk_khz;
 	context->bw_ctx.bw.dcn.clk.stutter_efficiency.base_efficiency = in_ctx->v21.mode_programming.programming->stutter.base_percent_efficiency;
 	context->bw_ctx.bw.dcn.clk.stutter_efficiency.low_power_efficiency = in_ctx->v21.mode_programming.programming->stutter.low_power_percent_efficiency;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_efficiency = in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_efficiency;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_period = in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_period;
+	context->bw_ctx.bw.dcn.clk.zstate_support = in_ctx->v21.mode_programming.programming->z8_stutter.supported_in_blank; /*ignore meets_eco since it is not used*/
 }
 
 static struct dml2_dchub_watermark_regs *wm_set_index_to_dc_wm_set(union dcn_watermark_set *watermarks, const enum dml2_dchub_watermark_reg_set_index wm_index)
@@ -927,3 +939,31 @@ void dml21_set_dc_p_state_type(
 	}
 }
 
+void dml21_init_min_clocks_for_dc_state(struct dml2_context *in_ctx, struct dc_state *context)
+{
+	unsigned int lowest_dpm_state_index = 0;
+	struct dc_clocks *min_clocks = &context->bw_ctx.bw.dcn.clk;
+
+	min_clocks->dispclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->dppclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->dcfclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dcfclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->dramclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.uclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->fclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.fclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->idle_dramclk_khz = 0;
+	min_clocks->idle_fclk_khz = 0;
+	min_clocks->dcfclk_deep_sleep_khz = 0;
+	min_clocks->fclk_p_state_change_support = true;
+	min_clocks->p_state_change_support = true;
+	min_clocks->dtbclk_en = false;
+	min_clocks->ref_dtbclk_khz = 0;
+	min_clocks->socclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.socclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->subvp_prefetch_dramclk_khz = 0;
+	min_clocks->subvp_prefetch_fclk_khz = 0;
+	min_clocks->phyclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.phyclk.clk_values_khz[lowest_dpm_state_index];
+	min_clocks->stutter_efficiency.base_efficiency = 1;
+	min_clocks->stutter_efficiency.low_power_efficiency = 1;
+	min_clocks->stutter_efficiency.z8_stutter_efficiency = 1;
+	min_clocks->stutter_efficiency.z8_stutter_period = 100000;
+	min_clocks->zstate_support = DCN_ZSTATE_SUPPORT_ALLOW;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h
index 9880d3e0398e2..f51d3d8a52c3f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h
@@ -25,4 +25,5 @@ void dml21_map_hw_resources(struct dml2_context *dml_ctx);
 void dml21_get_pipe_mcache_config(struct dc_state *context, struct pipe_ctx *pipe_ctx, struct dml2_per_plane_programming *pln_prog, struct dml2_pipe_configuration_descriptor *mcache_pipe_config);
 void dml21_set_dc_p_state_type(struct pipe_ctx *pipe_ctx, struct dml2_per_stream_programming *stream_programming, bool sub_vp_enabled);
 unsigned int map_plane_to_dml21_display_cfg(const struct dml2_context *dml_ctx, unsigned int stream_id, const struct dc_plane_state *plane, const struct dc_state *context);
+void dml21_init_min_clocks_for_dc_state(struct dml2_context *in_ctx, struct dc_state *context);
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
index f667026cb43eb..ab7ec24268bea 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
@@ -374,128 +374,136 @@ void dml21_handle_phantom_streams_planes(const struct dc *dc, struct dc_state *c
 		dml2_map_dc_pipes(dml_ctx, context, NULL, &dml_ctx->v21.dml_to_dc_pipe_mapping, dc->current_state);
 }
 
-void dml21_build_fams2_programming(const struct dc *dc,
+
+static unsigned int dml21_build_fams2_stream_programming_v2(const struct dc *dc,
 		struct dc_state *context,
 		struct dml2_context *dml_ctx)
 {
-	int i, j, k;
+	int dc_stream_idx, dc_plane_idx, dc_pipe_idx;
 	unsigned int num_fams2_streams = 0;
 
-	/* reset fams2 data */
-	memset(&context->bw_ctx.bw.dcn.fams2_stream_base_params, 0, sizeof(union dmub_cmd_fams2_config) * DML2_MAX_PLANES);
-	memset(&context->bw_ctx.bw.dcn.fams2_stream_sub_params, 0, sizeof(union dmub_cmd_fams2_config) * DML2_MAX_PLANES);
-	memset(&context->bw_ctx.bw.dcn.fams2_stream_sub_params_v2, 0, sizeof(union dmub_fams2_stream_static_sub_state_v2) * DML2_MAX_PLANES);
-	memset(&context->bw_ctx.bw.dcn.fams2_global_config, 0, sizeof(struct dmub_cmd_fams2_global_config));
+	for (dc_stream_idx = 0; dc_stream_idx < context->stream_count; dc_stream_idx++) {
+		int dml_stream_idx;
+		struct dc_stream_state *phantom_stream;
+		struct dc_stream_status *phantom_status;
+		enum fams2_stream_type type = 0;
 
-	if ((dml_ctx->v21.mode_programming.programming->fams2_required) ||
-		(dml_ctx->v21.mode_programming.programming->legacy_pstate_info_for_dmu)) {
-		for (i = 0; i < context->stream_count; i++) {
-			int dml_stream_idx;
-			struct dc_stream_state *phantom_stream;
-			struct dc_stream_status *phantom_status;
-			enum fams2_stream_type type = 0;
+		union dmub_cmd_fams2_config *static_base_state = &context->bw_ctx.bw.dcn.fams2_stream_base_params[num_fams2_streams];
+		union dmub_cmd_fams2_config *static_sub_state = &context->bw_ctx.bw.dcn.fams2_stream_sub_params[num_fams2_streams];
 
-			union dmub_cmd_fams2_config *static_base_state = &context->bw_ctx.bw.dcn.fams2_stream_base_params[num_fams2_streams];
-			union dmub_cmd_fams2_config *static_sub_state = &context->bw_ctx.bw.dcn.fams2_stream_sub_params[num_fams2_streams];
+		struct dc_stream_state *stream = context->streams[dc_stream_idx];
 
-			struct dc_stream_state *stream = context->streams[i];
+		if (context->stream_status[dc_stream_idx].plane_count == 0 ||
+				dml_ctx->config.svp_pstate.callbacks.get_stream_subvp_type(context, stream) == SUBVP_PHANTOM) {
+			/* can ignore blanked or phantom streams */
+			continue;
+		}
 
-			if (context->stream_status[i].plane_count == 0 ||
-					dml_ctx->config.svp_pstate.callbacks.get_stream_subvp_type(context, stream) == SUBVP_PHANTOM) {
-				/* can ignore blanked or phantom streams */
-				continue;
-			}
+		dml_stream_idx = dml21_helper_find_dml_pipe_idx_by_stream_id(dml_ctx, stream->stream_id);
+		if (dml_stream_idx < 0) {
+			ASSERT(dml_stream_idx >= 0);
+			continue;
+		}
 
-			dml_stream_idx = dml21_helper_find_dml_pipe_idx_by_stream_id(dml_ctx, stream->stream_id);
-			if (dml_stream_idx < 0) {
-				ASSERT(dml_stream_idx >= 0);
-				continue;
+		/* copy static state from PMO */
+		memcpy(static_base_state,
+				&dml_ctx->v21.mode_programming.programming->stream_programming[dml_stream_idx].fams2_base_params,
+				sizeof(union dmub_cmd_fams2_config));
+
+		memcpy(static_sub_state,
+				&dml_ctx->v21.mode_programming.programming->stream_programming[dml_stream_idx].fams2_sub_params,
+				sizeof(union dmub_cmd_fams2_config));
+
+		switch (dc->debug.fams_version.minor) {
+		case 1:
+		default:
+			type = static_base_state->stream_v1.base.type;
+
+			/* get information from context */
+			static_base_state->stream_v1.base.num_planes = context->stream_status[dc_stream_idx].plane_count;
+			static_base_state->stream_v1.base.otg_inst = context->stream_status[dc_stream_idx].primary_otg_inst;
+
+			/* populate pipe masks for planes */
+			for (dc_plane_idx = 0; dc_plane_idx < context->stream_status[dc_stream_idx].plane_count; dc_plane_idx++) {
+				for (dc_pipe_idx = 0; dc_pipe_idx < dc->res_pool->pipe_count; dc_pipe_idx++) {
+					if (context->res_ctx.pipe_ctx[dc_pipe_idx].stream &&
+							context->res_ctx.pipe_ctx[dc_pipe_idx].stream->stream_id == stream->stream_id &&
+							context->res_ctx.pipe_ctx[dc_pipe_idx].plane_state == context->stream_status[dc_stream_idx].plane_states[dc_plane_idx]) {
+						static_base_state->stream_v1.base.pipe_mask |= (1 << dc_pipe_idx);
+						static_base_state->stream_v1.base.plane_pipe_masks[dc_plane_idx] |= (1 << dc_pipe_idx);
+					}
+				}
 			}
+		}
 
-			/* copy static state from PMO */
-			memcpy(static_base_state,
-					&dml_ctx->v21.mode_programming.programming->stream_programming[dml_stream_idx].fams2_base_params,
-					sizeof(union dmub_cmd_fams2_config));
-
-			if (dc->debug.fams_version.major == 3) {
-				memcpy(&context->bw_ctx.bw.dcn.fams2_stream_sub_params_v2[num_fams2_streams],
-						&dml_ctx->v21.mode_programming.programming->stream_programming[dml_stream_idx].fams2_sub_params_v2,
-						sizeof(union dmub_fams2_stream_static_sub_state_v2));
-			} else {
-				memcpy(static_sub_state,
-						&dml_ctx->v21.mode_programming.programming->stream_programming[dml_stream_idx].fams2_sub_params,
-						sizeof(union dmub_cmd_fams2_config));
-			}
+
+		/* get per method programming */
+		switch (type) {
+		case FAMS2_STREAM_TYPE_VBLANK:
+		case FAMS2_STREAM_TYPE_VACTIVE:
+		case FAMS2_STREAM_TYPE_DRR:
+			break;
+		case FAMS2_STREAM_TYPE_SUBVP:
+			phantom_stream = dml_ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, stream);
+			if (!phantom_stream)
+				break;
+
+			phantom_status = dml_ctx->config.callbacks.get_stream_status(context, phantom_stream);
+
+			/* phantom status should always be present */
+			ASSERT(phantom_status);
+			if (!phantom_status)
+				break;
 
 			switch (dc->debug.fams_version.minor) {
 			case 1:
 			default:
-				type = static_base_state->stream_v1.base.type;
-
-				/* get information from context */
-				static_base_state->stream_v1.base.num_planes = context->stream_status[i].plane_count;
-				static_base_state->stream_v1.base.otg_inst = context->stream_status[i].primary_otg_inst;
-
-				/* populate pipe masks for planes */
-				for (j = 0; j < context->stream_status[i].plane_count; j++) {
-					for (k = 0; k < dc->res_pool->pipe_count; k++) {
-						if (context->res_ctx.pipe_ctx[k].stream &&
-								context->res_ctx.pipe_ctx[k].stream->stream_id == stream->stream_id &&
-								context->res_ctx.pipe_ctx[k].plane_state == context->stream_status[i].plane_states[j]) {
-							static_base_state->stream_v1.base.pipe_mask |= (1 << k);
-							static_base_state->stream_v1.base.plane_pipe_masks[j] |= (1 << k);
+				static_sub_state->stream_v1.sub_state.subvp.phantom_otg_inst = phantom_status->primary_otg_inst;
+
+				/* populate pipe masks for phantom planes */
+				for (dc_plane_idx = 0; dc_plane_idx < phantom_status->plane_count; dc_plane_idx++) {
+					for (dc_pipe_idx = 0; dc_pipe_idx < dc->res_pool->pipe_count; dc_pipe_idx++) {
+						if (context->res_ctx.pipe_ctx[dc_pipe_idx].stream &&
+								context->res_ctx.pipe_ctx[dc_pipe_idx].stream->stream_id == phantom_stream->stream_id &&
+								context->res_ctx.pipe_ctx[dc_pipe_idx].plane_state == phantom_status->plane_states[dc_plane_idx]) {
+							switch (dc->debug.fams_version.minor) {
+							case 1:
+							default:
+								static_sub_state->stream_v1.sub_state.subvp.phantom_pipe_mask |= (1 << dc_pipe_idx);
+								static_sub_state->stream_v1.sub_state.subvp.phantom_plane_pipe_masks[dc_plane_idx] |= (1 << dc_pipe_idx);
+							}
 						}
 					}
 				}
 			}
+			break;
+		default:
+			ASSERT(false);
+			break;
+		}
 
+		num_fams2_streams++;
+	}
 
-			/* get per method programming */
-			switch (type) {
-			case FAMS2_STREAM_TYPE_VBLANK:
-			case FAMS2_STREAM_TYPE_VACTIVE:
-			case FAMS2_STREAM_TYPE_DRR:
-				break;
-			case FAMS2_STREAM_TYPE_SUBVP:
-				phantom_stream = dml_ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, stream);
-				if (!phantom_stream)
-					break;
-
-				phantom_status = dml_ctx->config.callbacks.get_stream_status(context, phantom_stream);
-
-				/* phantom status should always be present */
-				ASSERT(phantom_status);
-				if (!phantom_status)
-					break;
-
-				switch (dc->debug.fams_version.minor) {
-				case 1:
-				default:
-					static_sub_state->stream_v1.sub_state.subvp.phantom_otg_inst = phantom_status->primary_otg_inst;
-
-					/* populate pipe masks for phantom planes */
-					for (j = 0; j < phantom_status->plane_count; j++) {
-						for (k = 0; k < dc->res_pool->pipe_count; k++) {
-							if (context->res_ctx.pipe_ctx[k].stream &&
-									context->res_ctx.pipe_ctx[k].stream->stream_id == phantom_stream->stream_id &&
-									context->res_ctx.pipe_ctx[k].plane_state == phantom_status->plane_states[j]) {
-								switch (dc->debug.fams_version.minor) {
-								case 1:
-								default:
-									static_sub_state->stream_v1.sub_state.subvp.phantom_pipe_mask |= (1 << k);
-									static_sub_state->stream_v1.sub_state.subvp.phantom_plane_pipe_masks[j] |= (1 << k);
-								}
-							}
-						}
-					}
-				}
-				break;
-			default:
-				ASSERT(false);
-				break;
-			}
+	return num_fams2_streams;
+}
+
+void dml21_build_fams2_programming(const struct dc *dc,
+		struct dc_state *context,
+		struct dml2_context *dml_ctx)
+{
+	unsigned int num_fams2_streams = 0;
+
+	/* reset fams2 data */
+	memset(&context->bw_ctx.bw.dcn.fams2_stream_base_params, 0, sizeof(union dmub_cmd_fams2_config) * DML2_MAX_PLANES);
+	memset(&context->bw_ctx.bw.dcn.fams2_stream_sub_params, 0, sizeof(union dmub_cmd_fams2_config) * DML2_MAX_PLANES);
+	memset(&context->bw_ctx.bw.dcn.fams2_stream_sub_params_v2, 0, sizeof(union dmub_fams2_stream_static_sub_state_v2) * DML2_MAX_PLANES);
+	memset(&context->bw_ctx.bw.dcn.fams2_global_config, 0, sizeof(struct dmub_cmd_fams2_global_config));
 
-			num_fams2_streams++;
+	if (dml_ctx->v21.mode_programming.programming->fams2_required ||
+			dml_ctx->v21.mode_programming.programming->legacy_pstate_info_for_dmu) {
+		if (dc->debug.fams_version.major == 2) {
+			num_fams2_streams = dml21_build_fams2_stream_programming_v2(dc, context, dml_ctx);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h
index 4bff52eaaef8d..bff945a4ab3a9 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h
@@ -9,6 +9,7 @@
 struct dc_state;
 struct dc_plane_state;
 struct pipe_ctx;
+struct dc_dmub_srv;
 
 struct dml2_context;
 struct dml2_display_rq_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c
index 798abb2b2e676..2623e917ec28c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c
@@ -9,16 +9,21 @@
 #include "dml21_utils.h"
 #include "dml21_translation_helper.h"
 #include "dml2_dc_resource_mgmt.h"
+#include "dc_fpu.h"
+
+#if !defined(DC_RUN_WITH_PREEMPTION_ENABLED)
+#define DC_RUN_WITH_PREEMPTION_ENABLED(code) code
+#endif // !DC_RUN_WITH_PREEMPTION_ENABLED
 
 #define INVALID -1
 
 static bool dml21_allocate_memory(struct dml2_context **dml_ctx)
 {
-	*dml_ctx = vzalloc(sizeof(struct dml2_context));
+	DC_RUN_WITH_PREEMPTION_ENABLED(*dml_ctx = vzalloc(sizeof(struct dml2_context)));
 	if (!(*dml_ctx))
 		return false;
 
-	(*dml_ctx)->v21.dml_init.dml2_instance = vzalloc(sizeof(struct dml2_instance));
+	DC_RUN_WITH_PREEMPTION_ENABLED((*dml_ctx)->v21.dml_init.dml2_instance = vzalloc(sizeof(struct dml2_instance)));
 	if (!((*dml_ctx)->v21.dml_init.dml2_instance))
 		return false;
 
@@ -28,7 +33,7 @@ static bool dml21_allocate_memory(struct dml2_context **dml_ctx)
 	(*dml_ctx)->v21.mode_support.display_config = &(*dml_ctx)->v21.display_config;
 	(*dml_ctx)->v21.mode_programming.display_config = (*dml_ctx)->v21.mode_support.display_config;
 
-	(*dml_ctx)->v21.mode_programming.programming = vzalloc(sizeof(struct dml2_display_cfg_programming));
+	DC_RUN_WITH_PREEMPTION_ENABLED((*dml_ctx)->v21.mode_programming.programming = vzalloc(sizeof(struct dml2_display_cfg_programming)));
 	if (!((*dml_ctx)->v21.mode_programming.programming))
 		return false;
 
@@ -70,8 +75,9 @@ static void dml21_init(const struct dc *in_dc, struct dml2_context *dml_ctx, con
 bool dml21_create(const struct dc *in_dc, struct dml2_context **dml_ctx, const struct dml2_configuration_options *config)
 {
 	/* Allocate memory for initializing DML21 instance */
-	if (!dml21_allocate_memory(dml_ctx))
+	if (!dml21_allocate_memory(dml_ctx)) {
 		return false;
+	}
 
 	dml21_init(in_dc, *dml_ctx, config);
 
@@ -215,6 +221,7 @@ static bool dml21_mode_check_and_programming(const struct dc *in_dc, struct dc_s
 		return true;
 
 	if (context->stream_count == 0) {
+		dml21_init_min_clocks_for_dc_state(dml_ctx, context);
 		dml21_build_fams2_programming(in_dc, context, dml_ctx);
 		return true;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
new file mode 100644
index 0000000000000..c75778ea7a2c4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
@@ -0,0 +1,263 @@
+/* SPDX-License-Identifier: MIT */
+//
+// Copyright 2024 Advanced Micro Devices, Inc.
+
+#ifndef __DML_DML_DCN42_SOC_BB__
+#define __DML_DML_DCN42_SOC_BB__
+
+#include "dml_top_soc_parameter_types.h"
+
+static const struct dml2_soc_qos_parameters dml_dcn42_variant_a_soc_qos_params = {
+	.derate_table = {
+		.system_active_urgent = {
+			.dram_derate_percent_pixel = 65,
+			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_pixel_and_vm = 60,
+			.fclk_derate_percent = 80,
+			.dcfclk_derate_percent = 80,
+		},
+		.system_active_average = {
+			.dram_derate_percent_pixel = 30,
+			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_pixel_and_vm = 30,
+			.fclk_derate_percent = 60,
+			.dcfclk_derate_percent = 60,
+		},
+		.dcn_mall_prefetch_urgent = {
+			.dram_derate_percent_pixel = 65,
+			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_pixel_and_vm = 60,
+			.fclk_derate_percent = 80,
+			.dcfclk_derate_percent = 80,
+		},
+		.dcn_mall_prefetch_average = {
+			.dram_derate_percent_pixel = 30,
+			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_pixel_and_vm = 30,
+			.fclk_derate_percent = 60,
+			.dcfclk_derate_percent = 60,
+		},
+		.system_idle_average = {
+			.dram_derate_percent_pixel = 30,
+			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_pixel_and_vm = 30,
+			.fclk_derate_percent = 60,
+			.dcfclk_derate_percent = 60,
+		},
+	},
+	.writeback = {
+		.base_latency_us = 12,
+		.scaling_factor_us = 0,
+		.scaling_factor_mhz = 0,
+	},
+	.qos_params = {
+		.dcn32x = {
+			.loaded_round_trip_latency_fclk_cycles = 106,
+			.urgent_latency_us = {
+				.base_latency_us = 4,
+				.base_latency_pixel_vm_us = 4,
+				.base_latency_vm_us = 4,
+				.scaling_factor_fclk_us = 0,
+				.scaling_factor_mhz = 0,
+			},
+			.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
+			.urgent_out_of_order_return_per_channel_pixel_only_bytes = 4096,
+			.urgent_out_of_order_return_per_channel_vm_only_bytes = 4096,
+		},
+	},
+	.qos_type = dml2_qos_param_type_dcn3,
+};
+
+static const struct dml2_soc_bb dml2_socbb_dcn42 = {
+	.clk_table = {
+		.wck_ratio = {
+				.clk_values_khz = {2},
+		},
+		.uclk = {
+				.clk_values_khz = {400000},
+				.num_clk_values = 1,
+		},
+		.fclk = {
+				.clk_values_khz = {400000},
+				.num_clk_values = 1,
+		},
+		.dcfclk = {
+				.clk_values_khz = {200000},
+				.num_clk_values = 1,
+		},
+		.dispclk = {
+				.clk_values_khz = {1500000},
+				.num_clk_values = 1,
+		},
+		.dppclk = {
+				.clk_values_khz = {1500000},
+				.num_clk_values = 1,
+		},
+		.dtbclk = {
+				.clk_values_khz = {600000},
+				.num_clk_values = 1,
+		},
+		.phyclk = {
+				.clk_values_khz = {810000},
+				.num_clk_values = 1,
+		},
+		.socclk = {
+				.clk_values_khz = {600000},
+				.num_clk_values = 1,
+		},
+		.dscclk = {
+				.clk_values_khz = {500000},
+				.num_clk_values = 1,
+		},
+		.phyclk_d18 = {
+				.clk_values_khz = {667000},
+				.num_clk_values = 1,
+		},
+		.phyclk_d32 = {
+				.clk_values_khz = {625000},
+				.num_clk_values = 1,
+		},
+		.dram_config = {
+			.channel_width_bytes = 4,
+			.channel_count = 4,
+			.alt_clock_bw_conversion = true,
+		},
+	},
+
+	.qos_parameters = {
+		.derate_table = {
+			.system_active_urgent = {
+				.dram_derate_percent_pixel = 65,
+				.dram_derate_percent_vm = 30,
+				.dram_derate_percent_pixel_and_vm = 60,
+				.fclk_derate_percent = 80,
+				.dcfclk_derate_percent = 80,
+			},
+			.system_active_average = {
+				.dram_derate_percent_pixel = 30,
+				.dram_derate_percent_vm = 30,
+				.dram_derate_percent_pixel_and_vm = 30,
+				.fclk_derate_percent = 60,
+				.dcfclk_derate_percent = 60,
+			},
+			.dcn_mall_prefetch_urgent = {
+				.dram_derate_percent_pixel = 65,
+				.dram_derate_percent_vm = 30,
+				.dram_derate_percent_pixel_and_vm = 60,
+				.fclk_derate_percent = 80,
+				.dcfclk_derate_percent = 80,
+			},
+			.dcn_mall_prefetch_average = {
+				.dram_derate_percent_pixel = 30,
+				.dram_derate_percent_vm = 30,
+				.dram_derate_percent_pixel_and_vm = 30,
+				.fclk_derate_percent = 60,
+				.dcfclk_derate_percent = 60,
+			},
+			.system_idle_average = {
+				.dram_derate_percent_pixel = 30,
+				.dram_derate_percent_vm = 30,
+				.dram_derate_percent_pixel_and_vm = 30,
+				.fclk_derate_percent = 60,
+				.dcfclk_derate_percent = 60,
+			},
+		},
+		.writeback = {
+			.base_latency_us = 12,
+			.scaling_factor_us = 0,
+			.scaling_factor_mhz = 0,
+		},
+		.qos_params = {
+			.dcn32x = {
+				.loaded_round_trip_latency_fclk_cycles = 106,
+				.urgent_latency_us = {
+					.base_latency_us = 4,
+					.base_latency_pixel_vm_us = 4,
+					.base_latency_vm_us = 4,
+					.scaling_factor_fclk_us = 0,
+					.scaling_factor_mhz = 0,
+				},
+				.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
+				.urgent_out_of_order_return_per_channel_pixel_only_bytes = 4096,
+				.urgent_out_of_order_return_per_channel_vm_only_bytes = 4096,
+			},
+		},
+		.qos_type = dml2_qos_param_type_dcn3,
+	},
+
+	.power_management_parameters = {
+		.dram_clk_change_blackout_us = 29,
+		.fclk_change_blackout_us = 0,
+		.g7_ppt_blackout_us = 0,
+		.stutter_enter_plus_exit_latency_us = 11,
+		.stutter_exit_latency_us = 9,
+		.z8_stutter_enter_plus_exit_latency_us = 300,
+		.z8_stutter_exit_latency_us = 200,
+	},
+
+	.vmin_limit = {
+		.dispclk_khz = 632 * 1000,
+	},
+
+	.dprefclk_mhz = 600,
+	.xtalclk_mhz = 24,
+	.pcie_refclk_mhz = 100,
+	.dchub_refclk_mhz = 50,
+	.mall_allocated_for_dcn_mbytes = 64,
+	.max_outstanding_reqs = 256,
+	.fabric_datapath_to_dcn_data_return_bytes = 32,
+	.return_bus_width_bytes = 64,
+	.hostvm_min_page_size_kbytes = 4,
+	.gpuvm_min_page_size_kbytes = 256,
+	.gpuvm_max_page_table_levels = 1,
+	.hostvm_max_non_cached_page_table_levels = 2,
+	.phy_downspread_percent = 0.38,
+	.dcn_downspread_percent = 0.38,
+	.dispclk_dppclk_vco_speed_mhz = 3000,
+	.do_urgent_latency_adjustment = 0,
+	.mem_word_bytes = 32,
+	.num_dcc_mcaches = 8,
+	.mcache_size_bytes = 2048,
+	.mcache_line_size_bytes = 32,
+	.max_fclk_for_uclk_dpm_khz = 2200 * 1000,
+};
+
+static const struct dml2_ip_capabilities dml2_dcn42_max_ip_caps = {
+	.pipe_count = 4,
+	.otg_count = 4,
+	.num_dsc = 4,
+	.max_num_dp2p0_streams = 4,
+	.max_num_hdmi_frl_outputs = 1,
+	.max_num_dp2p0_outputs = 4,
+	.rob_buffer_size_kbytes = 64,
+	.config_return_buffer_size_in_kbytes = 1792,
+	.config_return_buffer_segment_size_in_kbytes = 64,
+	.meta_fifo_size_in_kentries = 32,
+	.compressed_buffer_segment_size_in_kbytes = 64,
+	.cursor_buffer_size = 24,
+	.max_flip_time_us = 110,
+	.max_flip_time_lines = 50,
+	.hostvm_mode = 0,
+	.subvp_drr_scheduling_margin_us = 100,
+	.subvp_prefetch_end_to_mall_start_us = 15,
+	.subvp_fw_processing_delay = 15,
+	.max_vactive_det_fill_delay_us = 400,
+
+	.fams2 = {
+		.max_allow_delay_us = 100 * 1000,
+		.scheduling_delay_us = 550,
+		.vertical_interrupt_ack_delay_us = 40,
+		.allow_programming_delay_us = 18,
+		.min_allow_width_us = 20,
+		.subvp_df_throttle_delay_us = 100,
+		.subvp_programming_delay_us = 200,
+		.subvp_prefetch_to_mall_delay_us = 18,
+		.drr_programming_delay_us = 35,
+
+		.lock_timeout_us = 5000,
+		.recovery_timeout_us = 5000,
+		.flip_programming_delay_us = 300,
+	},
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
index b44762e21550f..4e9abe1a568d6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
@@ -27,6 +27,19 @@ enum dml2_swizzle_mode {
 	dml2_gfx11_sw_256kb_d_x,
 	dml2_gfx11_sw_256kb_r_x,
 
+	dml2_sw_linear_256b, // GFX10 SW_LINEAR only accepts 256 byte aligned pitch
+	dml2_gfx10_sw_64kb_r_x,
+	dml2_gfx102_sw_64kb_s,
+	dml2_gfx102_sw_64kb_s_t,
+	dml2_gfx102_sw_64kb_s_x,
+	dml2_gfx102_sw_64kb_r_x,
+
+	dml2_linear_64elements, // GFX7 LINEAR_ALIGNED accepts pitch alignment of the maximum of 64 elements or 256 bytes
+	dml2_gfx7_1d_thin,
+	dml2_gfx7_2d_thin_gen_zero,
+	dml2_gfx7_2d_thin_gen_one,
+	dml2_gfx7_2d_thin_arlene,
+	dml2_gfx7_2d_thin_anubis
 };
 
 enum dml2_source_format_class {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
index c4cce870877a1..6152155d60735 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
@@ -160,6 +160,8 @@ struct dml2_soc_bb {
 	unsigned long return_bus_width_bytes;
 	unsigned long hostvm_min_page_size_kbytes;
 	unsigned long gpuvm_min_page_size_kbytes;
+	unsigned int hostvm_max_non_cached_page_table_levels;
+	unsigned int gpuvm_max_page_table_levels;
 	double phy_downspread_percent;
 	double dcn_downspread_percent;
 	double dispclk_dppclk_vco_speed_mhz;
@@ -190,6 +192,7 @@ struct dml2_ip_capabilities {
 	unsigned int max_flip_time_us;
 	unsigned int max_flip_time_lines;
 	unsigned int hostvm_mode;
+	bool dcn_mrq_present;
 	unsigned int subvp_drr_scheduling_margin_us;
 	unsigned int subvp_prefetch_end_to_mall_start_us;
 	unsigned int subvp_fw_processing_delay;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
index 943fd3f040c3a..98b26116cdc14 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
@@ -19,6 +19,8 @@ enum dml2_project_id {
 	dml2_project_dcn4x_stage1,
 	dml2_project_dcn4x_stage2,
 	dml2_project_dcn4x_stage2_auto_drr_svp,
+	dml2_project_dcn40,
+	dml2_project_dcn42,
 };
 
 enum dml2_pstate_change_support {
@@ -79,6 +81,7 @@ struct dml2_options {
 	struct dml2_pmo_options pmo_options;
 };
 
+
 struct dml2_initialize_instance_in_out {
 	struct dml2_instance *dml2_instance;
 	struct dml2_options options;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
index eba948e187c11..99fc2f0666e2b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
@@ -78,6 +78,86 @@ struct dml2_core_ip_params core_dcn4_ip_caps_base = {
 	.subvp_swath_height_margin_lines = 16,
 };
 
+struct dml2_core_ip_params core_dcn42_ip_caps_base = {
+	.vblank_nom_default_us = 668,
+	.remote_iommu_outstanding_translations = 256,
+	.rob_buffer_size_kbytes = 64,
+	.config_return_buffer_size_in_kbytes = 1792,
+	.config_return_buffer_segment_size_in_kbytes = 64,
+	.compressed_buffer_segment_size_in_kbytes = 64,
+	.dpte_buffer_size_in_pte_reqs_luma = 68,
+	.dpte_buffer_size_in_pte_reqs_chroma = 36,
+	.pixel_chunk_size_kbytes = 8,
+	.alpha_pixel_chunk_size_kbytes = 4,
+	.min_pixel_chunk_size_bytes = 1024,
+	.writeback_chunk_size_kbytes = 8,
+	.line_buffer_size_bits = 1171920,
+	.max_line_buffer_lines = 32,
+	.writeback_interface_buffer_size_kbytes = 90,
+
+	//Number of pipes after DCN Pipe harvesting
+	.max_num_dpp = 4,
+	.max_num_otg = 4,
+	.max_num_opp = 4,
+	.max_num_wb = 1,
+	.max_dchub_pscl_bw_pix_per_clk = 4,
+	.max_pscl_lb_bw_pix_per_clk = 2,
+	.max_lb_vscl_bw_pix_per_clk = 4,
+	.max_vscl_hscl_bw_pix_per_clk = 4,
+	.max_hscl_ratio = 6,
+	.max_vscl_ratio = 6,
+	.max_hscl_taps = 8,
+	.max_vscl_taps = 8,
+	.dispclk_ramp_margin_percent = 1,
+	.dppclk_delay_subtotal = 47,
+	.dppclk_delay_scl = 50,
+	.dppclk_delay_scl_lb_only = 16,
+	.dppclk_delay_cnvc_formatter = 28,
+	.dppclk_delay_cnvc_cursor = 6,
+	.cursor_buffer_size = 42,
+	.cursor_chunk_size = 2,
+	.dispclk_delay_subtotal = 125,
+	.max_inter_dcn_tile_repeaters = 8,
+	.writeback_max_hscl_ratio = 1,
+	.writeback_max_vscl_ratio = 1,
+	.writeback_min_hscl_ratio = 1,
+	.writeback_min_vscl_ratio = 1,
+	.writeback_max_hscl_taps = 1,
+	.writeback_max_vscl_taps = 1,
+	.writeback_line_buffer_buffer_size = 0,
+	.num_dsc = 4,
+	.maximum_dsc_bits_per_component = 12,
+	.maximum_pixels_per_line_per_dsc_unit = 5760,
+	.dsc422_native_support = true,
+	.dcc_supported = true,
+	.ptoi_supported = false,
+
+	.cursor_64bpp_support = true,
+	.dynamic_metadata_vm_enabled = false,
+
+	.max_num_hdmi_frl_outputs = 0,
+	.max_num_dp2p0_outputs = 2,
+	.max_num_dp2p0_streams = 4,
+	.imall_supported = 1,
+	.max_flip_time_us = 110,
+	.max_flip_time_lines = 50,
+	.words_per_channel = 16,
+
+	.subvp_fw_processing_delay_us = 15,
+	.subvp_pstate_allow_width_us = 20,
+	.subvp_swath_height_margin_lines = 16,
+
+	.dcn_mrq_present = 1,
+	.zero_size_buffer_entries = 512,
+	.compbuf_reserved_space_zs = 64,
+	.dcc_meta_buffer_size_bytes = 6272,
+	.meta_chunk_size_kbytes = 2,
+	.min_meta_chunk_size_bytes = 256,
+
+	.dchub_arb_to_ret_delay = 102,
+	.hostvm_mode = 1,
+};
+
 static void patch_ip_caps_with_explicit_ip_params(struct dml2_ip_capabilities *ip_caps, const struct dml2_core_ip_params *ip_params)
 {
 	ip_caps->pipe_count = ip_params->max_num_dpp;
@@ -107,6 +187,7 @@ static void patch_ip_params_with_ip_caps(struct dml2_core_ip_params *ip_params,
 {
 	ip_params->max_num_dpp = ip_caps->pipe_count;
 	ip_params->max_num_otg = ip_caps->otg_count;
+	ip_params->max_num_opp = ip_caps->otg_count;
 	ip_params->num_dsc = ip_caps->num_dsc;
 	ip_params->max_num_dp2p0_streams = ip_caps->max_num_dp2p0_streams;
 	ip_params->max_num_dp2p0_outputs = ip_caps->max_num_dp2p0_outputs;
@@ -153,6 +234,37 @@ bool core_dcn4_initialize(struct dml2_core_initialize_in_out *in_out)
 	return true;
 }
 
+bool core_dcn42_initialize(struct dml2_core_initialize_in_out *in_out)
+{
+	struct dml2_core_instance *core = in_out->instance;
+
+	if (!in_out->minimum_clock_table)
+		return false;
+	else
+		core->minimum_clock_table = in_out->minimum_clock_table;
+
+	if (in_out->explicit_ip_bb && in_out->explicit_ip_bb_size > 0) {
+		memcpy(&core->clean_me_up.mode_lib.ip, in_out->explicit_ip_bb, in_out->explicit_ip_bb_size);
+
+		// FIXME_STAGE2:
+		// DV still uses stage1 ip_param_st for each variant, need to patch the ip_caps with ip_param info
+		// Should move DV to use ip_caps but need move more overrides to ip_caps
+		patch_ip_caps_with_explicit_ip_params(in_out->ip_caps, in_out->explicit_ip_bb);
+		core->clean_me_up.mode_lib.ip.subvp_pstate_allow_width_us = core_dcn4_ip_caps_base.subvp_pstate_allow_width_us;
+		core->clean_me_up.mode_lib.ip.subvp_fw_processing_delay_us = core_dcn4_ip_caps_base.subvp_pstate_allow_width_us;
+		core->clean_me_up.mode_lib.ip.subvp_swath_height_margin_lines = core_dcn4_ip_caps_base.subvp_swath_height_margin_lines;
+	} else {
+		memcpy(&core->clean_me_up.mode_lib.ip, &core_dcn42_ip_caps_base, sizeof(struct dml2_core_ip_params));
+		patch_ip_params_with_ip_caps(&core->clean_me_up.mode_lib.ip, in_out->ip_caps);
+		core->clean_me_up.mode_lib.ip.imall_supported = false;
+	}
+
+	memcpy(&core->clean_me_up.mode_lib.soc, in_out->soc_bb, sizeof(struct dml2_soc_bb));
+	memcpy(&core->clean_me_up.mode_lib.ip_caps, in_out->ip_caps, sizeof(struct dml2_ip_capabilities));
+
+	return true;
+}
+
 static void create_phantom_stream_from_main_stream(struct dml2_stream_parameters *phantom, const struct dml2_stream_parameters *main,
 	const struct dml2_implicit_svp_meta *meta)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.h
index a68bb001a3465..5c26d819a6739 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.h
@@ -5,6 +5,7 @@
 #ifndef __DML2_CORE_DCN4_H__
 #define __DML2_CORE_DCN4_H__
 bool core_dcn4_initialize(struct dml2_core_initialize_in_out *in_out);
+bool core_dcn42_initialize(struct dml2_core_initialize_in_out *in_out);
 bool core_dcn4_mode_support(struct dml2_core_mode_support_in_out *in_out);
 bool core_dcn4_mode_programming(struct dml2_core_mode_programming_in_out *in_out);
 bool core_dcn4_populate_informative(struct dml2_core_populate_informative_in_out *in_out);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
index ca5ac3c0deb56..b30d16474ceb0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
@@ -12262,11 +12262,15 @@ static void rq_dlg_get_rq_reg(struct dml2_display_rq_regs *rq_regs,
 
 	unsigned int pixel_chunk_bytes = 0;
 	unsigned int min_pixel_chunk_bytes = 0;
+	unsigned int meta_chunk_bytes = 0;
+	unsigned int min_meta_chunk_bytes = 0;
 	unsigned int dpte_group_bytes = 0;
 	unsigned int mpte_group_bytes = 0;
 
 	unsigned int p1_pixel_chunk_bytes = 0;
 	unsigned int p1_min_pixel_chunk_bytes = 0;
+	unsigned int p1_meta_chunk_bytes = 0;
+	unsigned int p1_min_meta_chunk_bytes = 0;
 	unsigned int p1_dpte_group_bytes = 0;
 	unsigned int p1_mpte_group_bytes = 0;
 
@@ -12287,8 +12291,13 @@ static void rq_dlg_get_rq_reg(struct dml2_display_rq_regs *rq_regs,
 	dpte_group_bytes = (unsigned int)(dml_get_dpte_group_size_in_bytes(mode_lib, pipe_idx));
 	mpte_group_bytes = (unsigned int)(dml_get_vm_group_size_in_bytes(mode_lib, pipe_idx));
 
+	meta_chunk_bytes =  (unsigned int)(mode_lib->ip.meta_chunk_size_kbytes * 1024);
+	min_meta_chunk_bytes = (unsigned int)(mode_lib->ip.min_meta_chunk_size_bytes);
+
 	p1_pixel_chunk_bytes = pixel_chunk_bytes;
 	p1_min_pixel_chunk_bytes = min_pixel_chunk_bytes;
+	p1_meta_chunk_bytes =  meta_chunk_bytes;
+	p1_min_meta_chunk_bytes =  min_meta_chunk_bytes;
 	p1_dpte_group_bytes = dpte_group_bytes;
 	p1_mpte_group_bytes = mpte_group_bytes;
 
@@ -12309,6 +12318,19 @@ static void rq_dlg_get_rq_reg(struct dml2_display_rq_regs *rq_regs,
 	else
 		rq_regs->rq_regs_c.min_chunk_size = log_and_substract_if_non_zero(p1_min_pixel_chunk_bytes, 8 - 1);
 
+	rq_regs->rq_regs_l.meta_chunk_size = log_and_substract_if_non_zero(meta_chunk_bytes, 10);
+	rq_regs->rq_regs_c.meta_chunk_size = log_and_substract_if_non_zero(p1_meta_chunk_bytes, 10);
+
+	if (min_meta_chunk_bytes == 0)
+		rq_regs->rq_regs_l.min_meta_chunk_size = 0;
+	else
+		rq_regs->rq_regs_l.min_meta_chunk_size = log_and_substract_if_non_zero(min_meta_chunk_bytes, 6 - 1);
+
+	if (min_meta_chunk_bytes == 0)
+		rq_regs->rq_regs_c.min_meta_chunk_size = 0;
+	else
+		rq_regs->rq_regs_c.min_meta_chunk_size = log_and_substract_if_non_zero(p1_min_meta_chunk_bytes, 6 - 1);
+
 	rq_regs->rq_regs_l.dpte_group_size = log_and_substract_if_non_zero(dpte_group_bytes, 6);
 	rq_regs->rq_regs_l.mpte_group_size = log_and_substract_if_non_zero(mpte_group_bytes, 6);
 	rq_regs->rq_regs_c.dpte_group_size = log_and_substract_if_non_zero(p1_dpte_group_bytes, 6);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
index cc4f0663c6d67..6cad99c21139b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
@@ -21,6 +21,7 @@ bool dml2_core_create(enum dml2_project_id project_id, struct dml2_core_instance
 	case dml2_project_dcn4x_stage1:
 		result = false;
 		break;
+	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
 		out->initialize = &core_dcn4_initialize;
@@ -30,6 +31,14 @@ bool dml2_core_create(enum dml2_project_id project_id, struct dml2_core_instance
 		out->calculate_mcache_allocation = &core_dcn4_calculate_mcache_allocation;
 		result = true;
 		break;
+	case dml2_project_dcn42:
+		out->initialize = &core_dcn42_initialize;
+		out->mode_support = &core_dcn4_mode_support;
+		out->mode_programming = &core_dcn4_mode_programming;
+		out->populate_informative = &core_dcn4_populate_informative;
+		out->calculate_mcache_allocation = &core_dcn4_calculate_mcache_allocation;
+		result = true;
+		break;
 	case dml2_project_invalid:
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.c
index b57d0f6ea6a1c..6930ba7ce5b76 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.c
@@ -428,6 +428,9 @@ bool dml2_core_utils_is_phantom_pipe(const struct dml2_plane_parameters *plane_c
 
 unsigned int dml2_core_utils_get_tile_block_size_bytes(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel)
 {
+	if (dml2_core_utils_get_gfx_version(sw_mode) == 10 || dml2_core_utils_get_gfx_version(sw_mode) == 7) {
+		return dml2_core_utils_get_tile_block_size_bytes_backcompat(sw_mode, byte_per_pixel);
+	}
 
 	if (sw_mode == dml2_sw_linear)
 		return 256;
@@ -459,14 +462,56 @@ unsigned int dml2_core_utils_get_tile_block_size_bytes(enum dml2_swizzle_mode sw
 	};
 }
 
+unsigned int dml2_core_utils_get_tile_block_size_bytes_backcompat(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel)
+{
+	if (sw_mode == dml2_sw_linear_256b)
+		return 256;
+	else if (sw_mode == dml2_gfx10_sw_64kb_r_x)
+		return 65536;
+	else if (sw_mode == dml2_gfx102_sw_64kb_s)
+		return 65536;
+	else if (sw_mode == dml2_gfx102_sw_64kb_s_t)
+		return 65536;
+	else if (sw_mode == dml2_gfx102_sw_64kb_s_x)
+		return 65536;
+	else if (sw_mode == dml2_gfx102_sw_64kb_r_x)
+		return 65536;
+	else if (sw_mode == dml2_linear_64elements)
+		return 256;
+	else if (sw_mode == dml2_gfx7_1d_thin)
+		return 256;
+	else if (sw_mode == dml2_gfx7_2d_thin_gen_zero)
+		return (128 * 64 * byte_per_pixel);
+	else if (sw_mode == dml2_gfx7_2d_thin_gen_one)
+		return (128 * 128 * byte_per_pixel);
+	else if (sw_mode == dml2_gfx7_2d_thin_arlene)
+		return (64 * 32 * byte_per_pixel);
+	else if (sw_mode == dml2_gfx7_2d_thin_anubis)
+		return (128 * 128 * byte_per_pixel);
+	else {
+		DML_ASSERT(0);
+		return 256;
+	};
+}
+
 bool dml2_core_utils_get_segment_horizontal_contiguous(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel)
 {
-	return (byte_per_pixel != 2);
+	if (dml2_core_utils_get_gfx_version(sw_mode) == 10 || dml2_core_utils_get_gfx_version(sw_mode) == 7) {
+		return dml2_core_utils_get_segment_horizontal_contiguous_backcompat(sw_mode, byte_per_pixel);
+	} else {
+		return (byte_per_pixel != 2);
+	}
+}
+
+bool dml2_core_utils_get_segment_horizontal_contiguous_backcompat(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel)
+{
+	return !((byte_per_pixel == 4) &&
+		((sw_mode == dml2_gfx10_sw_64kb_r_x) || (sw_mode == dml2_gfx102_sw_64kb_s) || (sw_mode == dml2_gfx102_sw_64kb_s_t) || (sw_mode == dml2_gfx102_sw_64kb_s_x)));
 }
 
 bool dml2_core_utils_is_linear(enum dml2_swizzle_mode sw_mode)
 {
-	return sw_mode == dml2_sw_linear;
+	return (sw_mode == dml2_sw_linear || sw_mode == dml2_sw_linear_256b || sw_mode == dml2_linear_64elements);
 };
 
 
@@ -499,6 +544,20 @@ int unsigned dml2_core_utils_get_gfx_version(enum dml2_swizzle_mode sw_mode)
 		sw_mode == dml2_gfx11_sw_256kb_d_x ||
 		sw_mode == dml2_gfx11_sw_256kb_r_x)
 		version = 11;
+	else if (sw_mode == dml2_sw_linear_256b ||
+		sw_mode == dml2_gfx10_sw_64kb_r_x ||
+		sw_mode == dml2_gfx102_sw_64kb_s ||
+		sw_mode == dml2_gfx102_sw_64kb_s_t ||
+		sw_mode == dml2_gfx102_sw_64kb_s_x ||
+		sw_mode == dml2_gfx102_sw_64kb_r_x)
+		version = 10;
+	else if (sw_mode == dml2_linear_64elements ||
+		sw_mode == dml2_gfx7_1d_thin ||
+		sw_mode == dml2_gfx7_2d_thin_gen_zero ||
+		sw_mode == dml2_gfx7_2d_thin_gen_one ||
+		sw_mode == dml2_gfx7_2d_thin_arlene ||
+		sw_mode == dml2_gfx7_2d_thin_anubis)
+		version = 7;
 	else {
 		DML_LOG_VERBOSE("ERROR: Invalid sw_mode setting! val=%u\n", sw_mode);
 		DML_ASSERT(0);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.h
index 95f0d017add45..471e73ed671cb 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.h
@@ -22,6 +22,8 @@ void dml2_core_utils_pipe_plane_mapping(const struct core_display_cfg_support_in
 bool dml2_core_utils_is_phantom_pipe(const struct dml2_plane_parameters *plane_cfg);
 unsigned int dml2_core_utils_get_tile_block_size_bytes(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel);
 bool dml2_core_utils_get_segment_horizontal_contiguous(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel);
+unsigned int dml2_core_utils_get_tile_block_size_bytes_backcompat(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel);
+bool dml2_core_utils_get_segment_horizontal_contiguous_backcompat(enum dml2_swizzle_mode sw_mode, unsigned int byte_per_pixel);
 bool dml2_core_utils_is_vertical_rotation(enum dml2_rotation_angle Scan);
 bool dml2_core_utils_is_linear(enum dml2_swizzle_mode sw_mode);
 int unsigned dml2_core_utils_get_gfx_version(enum dml2_swizzle_mode sw_mode);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
index 9d7741fd0adb1..d17e59d684fd1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
@@ -802,3 +802,36 @@ bool dpmm_dcn4_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 
 	return true;
 }
+bool dpmm_dcn42_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_out)
+{
+	const struct dml2_display_cfg *display_cfg = &in_out->display_cfg->display_config;
+	const struct dml2_core_internal_display_mode_lib *mode_lib = &in_out->core->clean_me_up.mode_lib;
+	struct dml2_dchub_global_register_set *dchubbub_regs = &in_out->programming->global_regs;
+
+	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : mode_lib->soc.dchub_refclk_mhz;
+
+	/* set A */
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].sr_enter = (int unsigned)(mode_lib->mp.Watermark.StutterEnterPlusExitWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].sr_exit = (int unsigned)(mode_lib->mp.Watermark.StutterExitWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].sr_enter_z8 = (int unsigned)(mode_lib->mp.Watermark.Z8StutterEnterPlusExitWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].sr_exit_z8 = (int unsigned)(mode_lib->mp.Watermark.Z8StutterExitWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].temp_read_or_ppt = (int unsigned)(mode_lib->mp.Watermark.temp_read_or_ppt_watermark_us * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].uclk_pstate = (int unsigned)(mode_lib->mp.Watermark.DRAMClockChangeWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].urgent = (int unsigned)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+
+	/* set B */
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B] = dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A];
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_C] = dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A];
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_D] = dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A];
+
+	dchubbub_regs->num_watermark_sets = 4;
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.h
index e7b58f2efda4b..5fbd07e238a8c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.h
@@ -10,5 +10,6 @@
 bool dpmm_dcn3_map_mode_to_soc_dpm(struct dml2_dpmm_map_mode_to_soc_dpm_params_in_out *in_out);
 bool dpmm_dcn4_map_mode_to_soc_dpm(struct dml2_dpmm_map_mode_to_soc_dpm_params_in_out *in_out);
 bool dpmm_dcn4_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_out);
+bool dpmm_dcn42_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_out);
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
index dfd01440737df..1f2d9e97f5fdb 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
@@ -31,6 +31,7 @@ bool dml2_dpmm_create(enum dml2_project_id project_id, struct dml2_dpmm_instance
 		out->map_watermarks = &dummy_map_watermarks;
 		result = true;
 		break;
+	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 		out->map_mode_to_soc_dpm = &dpmm_dcn3_map_mode_to_soc_dpm;
 		out->map_watermarks = &dummy_map_watermarks;
@@ -41,6 +42,11 @@ bool dml2_dpmm_create(enum dml2_project_id project_id, struct dml2_dpmm_instance
 		out->map_watermarks = &dpmm_dcn4_map_watermarks;
 		result = true;
 		break;
+	case dml2_project_dcn42:
+		out->map_mode_to_soc_dpm = &dpmm_dcn4_map_mode_to_soc_dpm;
+		out->map_watermarks = &dpmm_dcn42_map_watermarks;
+		result = true;
+		break;
 	case dml2_project_invalid:
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.h
index f54fde8fba906..02da6f45cbf75 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn4.h
@@ -10,4 +10,4 @@
 bool mcg_dcn4_build_min_clock_table(struct dml2_mcg_build_min_clock_table_params_in_out *in_out);
 bool mcg_dcn4_unit_test(void);
 
-#endif
\ No newline at end of file
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
new file mode 100644
index 0000000000000..1f67cbc2c2366
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dml2_mcg_dcn42.h"
+#include "dml_top_soc_parameter_types.h"
+
+static unsigned long long uclk_to_dram_bw_kbps(unsigned long uclk_khz, const struct dml2_dram_params *dram_config, unsigned long wck_ratio)
+{
+	unsigned long long bw_kbps = 0;
+
+	bw_kbps = (unsigned long long) uclk_khz * dram_config->channel_count * dram_config->channel_width_bytes * wck_ratio * 2;
+	return bw_kbps;
+}
+
+static bool build_min_clk_table_coarse_grained(const struct dml2_soc_bb *soc_bb, struct dml2_mcg_min_clock_table *min_table)
+{
+	int i;
+
+	for (i = 0; i < soc_bb->clk_table.fclk.num_clk_values; i++) {
+		if (i < soc_bb->clk_table.uclk.num_clk_values) {
+			min_table->dram_bw_table.entries[i].pre_derate_dram_bw_kbps =
+					uclk_to_dram_bw_kbps(soc_bb->clk_table.uclk.clk_values_khz[i], &soc_bb->clk_table.dram_config, soc_bb->clk_table.wck_ratio.clk_values_khz[i]);
+			min_table->dram_bw_table.entries[i].min_uclk_khz = soc_bb->clk_table.uclk.clk_values_khz[i];
+		} else {
+			min_table->dram_bw_table.entries[i].pre_derate_dram_bw_kbps = min_table->dram_bw_table.entries[soc_bb->clk_table.uclk.num_clk_values - 1].pre_derate_dram_bw_kbps;
+			min_table->dram_bw_table.entries[i].min_uclk_khz = soc_bb->clk_table.uclk.clk_values_khz[soc_bb->clk_table.uclk.num_clk_values - 1];
+		}
+
+		min_table->dram_bw_table.entries[i].min_dcfclk_khz = soc_bb->clk_table.dcfclk.clk_values_khz[i];
+		min_table->dram_bw_table.entries[i].min_fclk_khz = soc_bb->clk_table.fclk.clk_values_khz[i];
+	}
+	min_table->dram_bw_table.num_entries = soc_bb->clk_table.fclk.num_clk_values;
+
+	return true;
+}
+
+static bool build_min_clock_table(const struct dml2_soc_bb *soc_bb, struct dml2_mcg_min_clock_table *min_table)
+{
+	bool result;
+
+	if (!soc_bb || !min_table)
+		return false;
+
+
+	if (soc_bb->clk_table.uclk.num_clk_values > DML_MCG_MAX_CLK_TABLE_SIZE)
+		return false;
+
+	min_table->fixed_clocks_khz.amclk = 0;
+	min_table->fixed_clocks_khz.dprefclk = soc_bb->dprefclk_mhz * 1000;
+	min_table->fixed_clocks_khz.pcierefclk = soc_bb->pcie_refclk_mhz * 1000;
+	min_table->fixed_clocks_khz.dchubrefclk = soc_bb->dchub_refclk_mhz * 1000;
+	min_table->fixed_clocks_khz.xtalclk = soc_bb->xtalclk_mhz * 1000;
+
+	min_table->max_clocks_khz.dispclk = soc_bb->clk_table.dispclk.clk_values_khz[soc_bb->clk_table.dispclk.num_clk_values - 1];
+	min_table->max_clocks_khz.dppclk = soc_bb->clk_table.dppclk.clk_values_khz[soc_bb->clk_table.dppclk.num_clk_values - 1];
+	min_table->max_clocks_khz.dscclk = soc_bb->clk_table.dscclk.clk_values_khz[soc_bb->clk_table.dscclk.num_clk_values - 1];
+	min_table->max_clocks_khz.dtbclk = soc_bb->clk_table.dtbclk.clk_values_khz[soc_bb->clk_table.dtbclk.num_clk_values - 1];
+	min_table->max_clocks_khz.phyclk = soc_bb->clk_table.phyclk.clk_values_khz[soc_bb->clk_table.phyclk.num_clk_values - 1];
+
+	min_table->max_ss_clocks_khz.dispclk = (unsigned int)((double)min_table->max_clocks_khz.dispclk / (1.0 + soc_bb->dcn_downspread_percent / 100.0));
+	min_table->max_ss_clocks_khz.dppclk = (unsigned int)((double)min_table->max_clocks_khz.dppclk / (1.0 + soc_bb->dcn_downspread_percent / 100.0));
+	min_table->max_ss_clocks_khz.dtbclk = (unsigned int)((double)min_table->max_clocks_khz.dtbclk / (1.0 + soc_bb->dcn_downspread_percent / 100.0));
+
+	min_table->max_clocks_khz.dcfclk = soc_bb->clk_table.dcfclk.clk_values_khz[soc_bb->clk_table.dcfclk.num_clk_values - 1];
+	min_table->max_clocks_khz.fclk = soc_bb->clk_table.fclk.clk_values_khz[soc_bb->clk_table.fclk.num_clk_values - 1];
+
+	result = build_min_clk_table_coarse_grained(soc_bb, min_table);
+
+	return result;
+}
+
+bool mcg_dcn42_build_min_clock_table(struct dml2_mcg_build_min_clock_table_params_in_out *in_out)
+{
+	return build_min_clock_table(in_out->soc_bb, in_out->min_clk_table);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.h
new file mode 100644
index 0000000000000..d4ea49e3e6744
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: MIT */
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#ifndef __DML2_MCG_DCN42_H__
+#define __DML2_MCG_DCN42_H__
+
+#include "dml2_internal_shared_types.h"
+
+bool mcg_dcn42_build_min_clock_table(struct dml2_mcg_build_min_clock_table_params_in_out *in_out);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
index c60b8fe90819d..3dcd2c250633a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
@@ -4,6 +4,7 @@
 
 #include "dml2_mcg_factory.h"
 #include "dml2_mcg_dcn4.h"
+#include "dml2_mcg_dcn42.h"
 #include "dml2_external_lib_deps.h"
 
 static bool dummy_build_min_clock_table(struct dml2_mcg_build_min_clock_table_params_in_out *in_out)
@@ -25,11 +26,16 @@ bool dml2_mcg_create(enum dml2_project_id project_id, struct dml2_mcg_instance *
 		out->build_min_clock_table = &dummy_build_min_clock_table;
 		result = true;
 		break;
+	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
 		out->build_min_clock_table = &mcg_dcn4_build_min_clock_table;
 		result = true;
 		break;
+	case dml2_project_dcn42:
+		out->build_min_clock_table = &mcg_dcn42_build_min_clock_table;
+		result = true;
+		break;
 	case dml2_project_invalid:
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
index c26e100fcaf2e..e8691983c0eba 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
@@ -23,6 +23,7 @@ static const struct dml2_pmo_pstate_strategy base_strategy_list_1_display[] = {
 		.allow_state_increase = true,
 	},
 
+
 	// Then VBlank
 	{
 		.per_stream_pstate_method = { dml2_pstate_method_vblank, dml2_pstate_method_na, dml2_pstate_method_na, dml2_pstate_method_na },
@@ -53,6 +54,7 @@ static const struct dml2_pmo_pstate_strategy base_strategy_list_2_display[] = {
 		.allow_state_increase = true,
 	},
 
+
 	// Then VActive + VBlank
 	{
 		.per_stream_pstate_method = { dml2_pstate_method_vactive, dml2_pstate_method_vblank, dml2_pstate_method_na, dml2_pstate_method_na },
@@ -113,6 +115,7 @@ static const struct dml2_pmo_pstate_strategy base_strategy_list_3_display[] = {
 		.allow_state_increase = true,
 	},
 
+
 	// VActive + 1 VBlank
 	{
 		.per_stream_pstate_method = { dml2_pstate_method_vactive, dml2_pstate_method_vactive, dml2_pstate_method_vblank, dml2_pstate_method_na },
@@ -149,6 +152,7 @@ static const struct dml2_pmo_pstate_strategy base_strategy_list_4_display[] = {
 		.allow_state_increase = true,
 	},
 
+
 	// VActive + 1 VBlank
 	{
 		.per_stream_pstate_method = { dml2_pstate_method_vactive, dml2_pstate_method_vactive, dml2_pstate_method_vactive, dml2_pstate_method_vblank },
@@ -1651,6 +1655,7 @@ static bool validate_pstate_support_strategy_cofunctionality(struct dml2_pmo_ins
 	if (svp_count > 0 && (pmo->options->disable_svp || !all_timings_support_svp(pmo, display_cfg, svp_stream_mask)))
 		return false;
 
+
 	return is_config_schedulable(pmo, display_cfg, pstate_strategy);
 }
 
@@ -1980,6 +1985,7 @@ static void reset_display_configuration(struct display_configuation_with_meta *d
 	}
 }
 
+
 static void setup_planes_for_drr_by_mask(struct display_configuation_with_meta *display_config,
 	struct dml2_pmo_instance *pmo,
 	int plane_mask)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
index 55d2464365d04..4d687fa86caab 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
@@ -3,8 +3,8 @@
 // Copyright 2024 Advanced Micro Devices, Inc.
 
 #include "dml2_pmo_factory.h"
-#include "dml2_pmo_dcn4_fams2.h"
 #include "dml2_pmo_dcn3.h"
+#include "dml2_pmo_dcn4_fams2.h"
 #include "dml2_external_lib_deps.h"
 
 static bool dummy_init_for_stutter(struct dml2_pmo_init_for_stutter_in_out *in_out)
@@ -37,6 +37,7 @@ bool dml2_pmo_create(enum dml2_project_id project_id, struct dml2_pmo_instance *
 		out->optimize_dcc_mcache = pmo_dcn4_fams2_optimize_dcc_mcache;
 		result = true;
 		break;
+	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 		out->initialize = pmo_dcn3_initialize;
 
@@ -56,6 +57,7 @@ bool dml2_pmo_create(enum dml2_project_id project_id, struct dml2_pmo_instance *
 
 		result = true;
 		break;
+	case dml2_project_dcn42:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
 		out->initialize = pmo_dcn4_fams2_initialize;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.h
index b90f6263cd85d..7218de1824cca 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.h
@@ -10,4 +10,4 @@
 
 bool dml2_pmo_create(enum dml2_project_id project_id, struct dml2_pmo_instance *out);
 
-#endif
\ No newline at end of file
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
index 5a33e2f357f45..a6c5031f69c19 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
@@ -17,6 +17,8 @@ bool dml2_initialize_instance(struct dml2_initialize_instance_in_out *in_out)
 	case dml2_project_dcn4x_stage1:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
+	case dml2_project_dcn40:
+	case dml2_project_dcn42:
 		return dml2_top_soc15_initialize_instance(in_out);
 	case dml2_project_invalid:
 	default:
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.c
index 5e14d85821e27..0e3177fe9d27e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.c
@@ -3,7 +3,6 @@
 // Copyright 2024 Advanced Micro Devices, Inc.
 
 #include "dml2_top_legacy.h"
-#include "dml2_top_soc15.h"
 #include "dml2_core_factory.h"
 #include "dml2_pmo_factory.h"
 #include "display_mode_core_structs.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
index a6bd75f30d202..d328d92240b4e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
@@ -410,6 +410,7 @@ struct dml2_core_mode_support_in_out {
 	} legacy;
 };
 
+
 struct dml2_core_mode_programming_in_out {
 	/*
 	* Inputs
@@ -501,6 +502,7 @@ struct dml2_core_instance {
 	bool (*populate_informative)(struct dml2_core_populate_informative_in_out *in_out);
 	bool (*calculate_mcache_allocation)(struct dml2_calculate_mcache_allocation_in_out *in_out);
 
+
 	struct {
 		struct dml2_core_internal_display_mode_lib mode_lib;
 	} clean_me_up;
@@ -753,6 +755,7 @@ struct dml2_pmo_instance {
 	bool (*test_for_stutter)(struct dml2_pmo_test_for_stutter_in_out *in_out);
 	bool (*optimize_for_stutter)(struct dml2_pmo_optimize_for_stutter_in_out *in_out);
 
+
 	struct dml2_pmo_init_data init_data;
 	struct dml2_pmo_scratch scratch;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c
index 4cfe64aa84927..74812a7d5e28c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c
@@ -909,10 +909,10 @@ static unsigned int get_source_mpc_factor(const struct dml2_context *ctx,
 		const struct dc_plane_state *plane)
 {
 	struct pipe_ctx *dpp_pipes[MAX_PIPES] = {0};
-	int dpp_pipe_count = ctx->config.callbacks.get_dpp_pipes_for_plane(plane,
-			&state->res_ctx, dpp_pipes);
 
-	ASSERT(dpp_pipe_count > 0);
+	if (ctx->config.callbacks.get_dpp_pipes_for_plane(plane, &state->res_ctx, dpp_pipes) <= 0)
+		ASSERT(false);
+
 	return ctx->config.callbacks.get_mpc_slice_count(dpp_pipes[0]);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
index 66040c877d68a..d56e58ce26c7a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
@@ -357,7 +357,7 @@ static bool enough_pipes_for_subvp(struct dml2_context *ctx, struct dc_state *st
  */
 static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *context)
 {
-	struct pipe_ctx *subvp_pipes[2];
+	struct pipe_ctx *subvp_pipes[2] = { NULL, NULL };
 	struct dc_stream_state *phantom = NULL;
 	uint32_t microschedule_lines = 0;
 	uint32_t index = 0;
@@ -369,6 +369,9 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 		uint32_t time_us = 0;
 
+		if (pipe == NULL || pipe->stream == NULL)
+			continue;
+
 		/* Loop to calculate the maximum microschedule time between the two SubVP pipes,
 		 * and also to store the two main SubVP pipe pointers in subvp_pipes[2].
 		 */
@@ -386,14 +389,19 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 			if (time_us > max_microschedule_us)
 				max_microschedule_us = time_us;
 
-			subvp_pipes[index] = pipe;
-			index++;
+			if (index < 2)
+				subvp_pipes[index++] = pipe;
 
 			// Maximum 2 SubVP pipes
 			if (index == 2)
 				break;
 		}
 	}
+
+	/* Minimal guard to avoid C6001 before subvp_pipes[0]/[1] dereference */
+	if (index < 2 || !subvp_pipes[0] || !subvp_pipes[1])
+		return false;
+
 	vactive1_us = ((subvp_pipes[0]->stream->timing.v_addressable * subvp_pipes[0]->stream->timing.h_total) /
 			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000;
 	vactive2_us = ((subvp_pipes[1]->stream->timing.v_addressable * subvp_pipes[1]->stream->timing.h_total) /
@@ -459,6 +467,11 @@ bool dml2_svp_drr_schedulable(struct dml2_context *ctx, struct dc_state *context
 			break;
 	}
 
+	if (pipe == NULL || pipe->stream == NULL) {
+		// Defensive: should never happen, try to catch in debug
+		ASSERT(0);
+		return false;
+	}
 	phantom_stream = ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, pipe->stream);
 	main_timing = &pipe->stream->timing;
 	phantom_timing = &phantom_stream->timing;
@@ -549,6 +562,13 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 		if (!subvp_pipe && pipe_mall_type == SUBVP_MAIN)
 			subvp_pipe = pipe;
 	}
+
+	if (subvp_pipe == NULL) {
+		// Defensive: should never happen, catch in debug
+		ASSERT(0);
+		return false;
+	}
+
 	// Use ignore_msa_timing_param flag to identify as DRR
 	if (found && context->res_ctx.pipe_ctx[vblank_index].stream->ignore_msa_timing_param) {
 		// SUBVP + DRR case
@@ -753,6 +773,12 @@ static void enable_phantom_plane(struct dml2_context *ctx,
 				return;
 		}
 
+		/* Minimal NULL guard for C6011 */
+		if (!phantom_plane) {
+			ASSERT(0);
+			continue;
+		}
+
 		memcpy(&phantom_plane->address, &curr_pipe->plane_state->address, sizeof(phantom_plane->address));
 		memcpy(&phantom_plane->scaling_quality, &curr_pipe->plane_state->scaling_quality,
 				sizeof(phantom_plane->scaling_quality));
@@ -880,6 +906,11 @@ bool dml2_svp_add_phantom_pipe_to_dc_state(struct dml2_context *ctx, struct dc_s
 	if (ctx->config.svp_pstate.force_disable_subvp)
 		return false;
 
+	if (!state) {
+		ASSERT(0);
+		return false;
+	}
+
 	if (!all_pipes_have_stream_and_plane(ctx, state))
 		return false;
 
@@ -898,6 +929,10 @@ bool dml2_svp_add_phantom_pipe_to_dc_state(struct dml2_context *ctx, struct dc_s
 	}
 
 	if (enough_pipes_for_subvp(ctx, state) && assign_subvp_pipe(ctx, state, &dc_pipe_idx)) {
+		if (state->res_ctx.pipe_ctx[dc_pipe_idx].stream == NULL) {
+			ASSERT(0);
+			return false;
+		}
 		dml_pipe_idx = dml2_helper_find_dml_pipe_idx_by_stream_id(ctx, state->res_ctx.pipe_ctx[dc_pipe_idx].stream->stream_id);
 		svp_height = mode_support_info->SubViewportLinesNeededInMALL[dml_pipe_idx];
 		vstartup = dml_get_vstartup_calculated(&ctx->v20.dml_core_ctx, dml_pipe_idx);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
index 307186eb6af08..408559d6fb2d6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
@@ -84,8 +84,9 @@ static void dml2_init(const struct dc *in_dc, const struct dml2_configuration_op
 bool dml2_create(const struct dc *in_dc, const struct dml2_configuration_options *config, struct dml2_context **dml2)
 {
 	// TODO : Temporarily add DCN_VERSION_3_2 for N-1 validation. Remove DCN_VERSION_3_2 after N-1 validation phase is complete.
-	if ((in_dc->debug.using_dml21) && (in_dc->ctx->dce_version >= DCN_VERSION_4_01))
+	if ((in_dc->debug.using_dml21) && (in_dc->ctx->dce_version >= DCN_VERSION_4_01)) {
 		return dml21_create(in_dc, dml2, config);
+	}
 
 	// Allocate Mode Lib Ctx
 	*dml2 = dml2_allocate_memory();
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
index 9a9c27962f681..5ee489682f2ea 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
@@ -239,6 +239,7 @@ struct dml2_configuration_options {
 
 	bool use_clock_dc_limits;
 	bool gpuvm_enable;
+	bool hostvm_enable;
 	bool force_tdlut_enable;
 	void *bb_from_dmub;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/Makefile b/drivers/gpu/drm/amd/display/dc/dpp/Makefile
index 8324a56fe7dbf..2595b2504f9d1 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dpp/Makefile
@@ -1,5 +1,5 @@
 
-# Copyright 2022 Advanced Micro Devices, Inc.
+# Copyright 2022-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -80,4 +80,12 @@ AMD_DAL_DPP_DCN401 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn401/,$(DPP_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN401)
 
+###############################################################################
+# DCN42
+###############################################################################
+DPP_DCN42 = dcn42_dpp.o
+
+AMD_DAL_DPP_DCN42 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn42/,$(DPP_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
index ce91e5d289567..0e70ffc784b19 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
@@ -172,10 +172,14 @@ bool dpp1_get_optimal_number_of_taps(
 		scl_data->taps.h_taps_c = in_taps->h_taps_c;
 
 	if (!dpp->ctx->dc->debug.always_scale) {
-		if (IDENTITY_RATIO(scl_data->ratios.horz))
+		if (IDENTITY_RATIO(scl_data->ratios.horz)) {
 			scl_data->taps.h_taps = 1;
-		if (IDENTITY_RATIO(scl_data->ratios.vert))
+			scl_data->taps.h_taps_c = 1;
+		}
+		if (IDENTITY_RATIO(scl_data->ratios.vert)) {
 			scl_data->taps.v_taps = 1;
+			scl_data->taps.v_taps_c = 1;
+		}
 		if (IDENTITY_RATIO(scl_data->ratios.horz_c))
 			scl_data->taps.h_taps_c = 1;
 		if (IDENTITY_RATIO(scl_data->ratios.vert_c))
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
index c7923531da83d..8a146968ee156 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
@@ -524,10 +524,14 @@ bool dpp3_get_optimal_number_of_taps(
 		scl_data->taps.v_taps_c = max_taps_c;
 
 	if (!dpp->ctx->dc->debug.always_scale) {
-		if (IDENTITY_RATIO(scl_data->ratios.horz))
+		if (IDENTITY_RATIO(scl_data->ratios.horz)) {
 			scl_data->taps.h_taps = 1;
-		if (IDENTITY_RATIO(scl_data->ratios.vert))
+			scl_data->taps.h_taps_c = 1;
+		}
+		if (IDENTITY_RATIO(scl_data->ratios.vert)) {
 			scl_data->taps.v_taps = 1;
+			scl_data->taps.v_taps_c = 1;
+		}
 		if (IDENTITY_RATIO(scl_data->ratios.horz_c))
 			scl_data->taps.h_taps_c = 1;
 		if (IDENTITY_RATIO(scl_data->ratios.vert_c))
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
index 82eca0e7b7d05..3284084ca7ad4 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
@@ -132,6 +132,8 @@ static void dpp3_power_on_gamcor_lut(
 	if (dpp_base->ctx->dc->debug.enable_mem_low_power.bits.cm) {
 		if (power_on) {
 			REG_UPDATE(CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_FORCE, 0);
+			if (dpp_base->ctx->dc->caps.ips_v2_support)
+				REG_UPDATE(CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, 1);
 			REG_WAIT(CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, 0, 1, 5);
 		} else {
 			dpp_base->ctx->dc->optimized_required = true;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
index 58b9f1cd08255..debcf308cb3b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
@@ -1,4 +1,5 @@
-/* Copyright 2023 Advanced Micro Devices, Inc.
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2023-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -653,6 +654,8 @@
 
 struct dcn401_dpp_registers {
 	DPP_REG_VARIABLE_LIST_DCN401;
+	uint32_t ALPHA_2BIT_LUT01;
+	uint32_t ALPHA_2BIT_LUT23;
 };
 
 struct dcn401_dpp_shift {
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
index a62c4733ed3be..8b6155f9122f8 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
@@ -155,7 +155,12 @@ static void dpp401_power_on_dscl(
 	if (dpp->tf_regs->DSCL_MEM_PWR_CTRL) {
 		if (power_on) {
 			REG_UPDATE(DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_FORCE, 0);
-			REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 5);
+			if (dpp->base.ctx->dc->caps.ips_v2_support) {
+				/*hw default changes to LS*/
+				REG_UPDATE(DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_DIS, 1);
+				REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 100);
+			} else
+				REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 5);
 		} else {
 			if (dpp->base.ctx->dc->debug.enable_mem_low_power.bits.dscl) {
 				dpp->base.ctx->dc->optimized_required = true;
@@ -956,6 +961,15 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 	*bs_coeffs_updated = false;
 
 	PERF_TRACE();
+	/*power on isharp_delta_mem first*/
+	if (dpp_base->ctx->dc->caps.ips_v2_support) {
+		/*HW default is LS, need to wake up*/
+		REG_UPDATE_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+					ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+					ISHARP_DELTA_LUT_MEM_PWR_DIS, 1);
+		REG_WAIT(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+			ISHARP_DELTA_LUT_MEM_PWR_STATE, 0, 1, 100);
+	}
 	/* ISHARP_MODE */
 	REG_SET_6(ISHARP_MODE, 0,
 		ISHARP_EN, scl_data->dscl_prog_data.isharp_en,
@@ -1033,6 +1047,13 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 		}
 	}
 
+	/*power on isharp_delta_mem first*/
+	if (dpp_base->ctx->dc->caps.ips_v2_support) {
+		/*HW default is LS, need to wake up*/
+		REG_UPDATE_SEQ_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+					ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+					ISHARP_DELTA_LUT_MEM_PWR_DIS, 0);
+	}
 	PERF_TRACE();
 } // dpp401_dscl_program_isharp
 /**
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
new file mode 100644
index 0000000000000..c126fb9d5bfa9
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "core_types.h"
+#include "reg_helper.h"
+#include "dcn42_dpp.h"
+#include "dcn35/dcn35_dpp.h"
+
+#define REG(reg)\
+	dpp->tf_regs->reg
+
+#define CTX \
+	dpp->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	dpp->tf_shift->field_name, dpp->tf_mask->field_name
+
+static const uint32_t *get_hist_rgb_luma_coefs(enum dc_color_space color_space)
+{
+	// Coefs in s.6.12.
+	// Y = 0.2126R + 0.7152G + 0.0722B
+	static const uint32_t luma_transform_bt_709[] = {0x1cb36, 0x1e6e2, 0x1b27b};
+	// Y = 0.2627R + 0.678G + 0.0593B
+	static const uint32_t luma_transform_bt_2020[] = {0x1d0d0, 0x1e5b2, 0x1ae5c};
+
+	switch (color_space) {
+	case COLOR_SPACE_2020_RGB_FULLRANGE:
+	case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
+		return luma_transform_bt_2020;
+	default:
+		return luma_transform_bt_709;
+	}
+}
+
+static void dpp42_dpp_cm_hist_control(
+	struct dpp *dpp_base,
+	struct cm_hist_control cntl,
+	enum dc_color_space color_space)
+{
+	struct dcn42_dpp *dpp = TO_DCN42_DPP(dpp_base);
+
+	REG_UPDATE_10(
+		CM_HIST_CNTL,
+		CM_HIST_SEL, cntl.tap_point,
+		CM_HIST_CH_EN, cntl.channels_enabled,
+		CM_HIST_SRC1_SEL, cntl.src_1_select,
+		CM_HIST_SRC2_SEL, cntl.src_2_select,
+		CM_HIST_SRC3_SEL, cntl.src_3_select,
+		CM_HIST_CH1_XBAR, cntl.ch1_src,
+		CM_HIST_CH2_XBAR, cntl.ch2_src,
+		CM_HIST_CH3_XBAR, cntl.ch3_src,
+		CM_HIST_FORMAT, cntl.format,
+		CM_HIST_READ_CHANNEL_MASK, cntl.read_channel_mask);
+
+	if (cntl.src_2_select == CM_HIST_SRC2_MODE_RGB_TO_Y) {
+		const uint32_t *luma_transform = get_hist_rgb_luma_coefs(color_space);
+
+		REG_UPDATE(CM_HIST_COEFA_SRC2, CM_HIST_COEFA_SRC2, luma_transform[0]);
+		REG_UPDATE(CM_HIST_COEFB_SRC2, CM_HIST_COEFB_SRC2, luma_transform[1]);
+		REG_UPDATE(CM_HIST_COEFC_SRC2, CM_HIST_COEFC_SRC2, luma_transform[2]);
+	} else {
+		REG_UPDATE(CM_HIST_COEFA_SRC2, CM_HIST_COEFA_SRC2, 0);
+		REG_UPDATE(CM_HIST_COEFB_SRC2, CM_HIST_COEFB_SRC2, 0x1f000); // 1 in s.6.12
+		REG_UPDATE(CM_HIST_COEFC_SRC2, CM_HIST_COEFC_SRC2, 0);
+	}
+}
+
+static bool dpp42_dpp_cm_hist_read(struct dpp *dpp_base, struct cm_hist *hist_out)
+{
+	struct dcn42_dpp *dpp = TO_DCN42_DPP(dpp_base);
+	uint32_t channel_mask = 0;
+	uint32_t rdy_status = 0, rdy_status_a = 0, rdy_status_b = 0;
+	bool ch1, ch2, ch3;
+
+	if (hist_out == NULL)
+		return false;
+
+
+	REG_GET(CM_HIST_CNTL, CM_HIST_READ_CHANNEL_MASK, &channel_mask);
+	ch1 = (channel_mask & 1) > 0;
+	ch2 = (channel_mask & 2) > 0;
+	ch3 = (channel_mask & 4) > 0;
+
+	REG_GET(CM_HIST_STATUS, CM_HIST_BUFA_RDY_STATUS, &rdy_status_a);
+	REG_GET(CM_HIST_STATUS, CM_HIST_BUFB_RDY_STATUS, &rdy_status_b);
+
+	rdy_status = rdy_status_a || rdy_status_b;
+
+	if (rdy_status) {
+		REG_UPDATE(CM_HIST_LOCK, CM_HIST_LOCK, 1);
+		REG_UPDATE(CM_HIST_INDEX, CM_HIST_INDEX, 0);
+		for (int i = 0; i < 256; i++) {
+			uint32_t temp;
+
+			if (ch1) {
+				REG_GET(CM_HIST_DATA, CM_HIST_DATA, &temp);
+				hist_out->ch1[i] += temp;
+			}
+			if (ch2) {
+				REG_GET(CM_HIST_DATA, CM_HIST_DATA, &temp);
+				hist_out->ch2[i] += temp;
+			}
+			if (ch3) {
+				REG_GET(CM_HIST_DATA, CM_HIST_DATA, &temp);
+				hist_out->ch3[i] += temp;
+			}
+		}
+		REG_UPDATE(CM_HIST_LOCK, CM_HIST_LOCK, 0);
+		return true;
+	} else {
+		return false;
+	}
+}
+
+static void dpp42_dpp_setup(
+	struct dpp *dpp_base,
+	enum surface_pixel_format format,
+	enum expansion_mode mode,
+	struct dc_csc_transform input_csc_color_matrix,
+	enum dc_color_space input_color_space,
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+{
+	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
+	uint32_t pixel_format = 0;
+	uint32_t alpha_en = 1;
+	enum dc_color_space color_space = COLOR_SPACE_SRGB;
+	enum dcn10_input_csc_select select = INPUT_CSC_SELECT_BYPASS;
+	uint32_t is_2bit = 0;
+	uint32_t alpha_plane_enable = 0;
+	uint32_t dealpha_en = 0, dealpha_ablnd_en = 0;
+	uint32_t realpha_en = 0, realpha_ablnd_en = 0;
+	struct out_csc_color_matrix tbl_entry;
+	int i;
+
+	REG_SET_2(FORMAT_CONTROL, 0,
+		CNVC_BYPASS, 0,
+		FORMAT_EXPANSION_MODE, mode);
+
+	REG_UPDATE(FORMAT_CONTROL, FORMAT_CNV16, 0);
+	REG_UPDATE(FORMAT_CONTROL, CNVC_BYPASS_MSB_ALIGN, 0);
+	REG_UPDATE(FORMAT_CONTROL, CLAMP_POSITIVE, 0);
+	REG_UPDATE(FORMAT_CONTROL, CLAMP_POSITIVE_C, 0);
+
+	REG_UPDATE(FORMAT_CONTROL, FORMAT_CROSSBAR_R, 0);
+	REG_UPDATE(FORMAT_CONTROL, FORMAT_CROSSBAR_G, 1);
+	REG_UPDATE(FORMAT_CONTROL, FORMAT_CROSSBAR_B, 2);
+
+	switch (format) {
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+		pixel_format = 1;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+		pixel_format = 3;
+		alpha_en = 0;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
+		pixel_format = 8;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
+		pixel_format = 10;
+		is_2bit = 1;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr:
+		pixel_format = 65;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb:
+		pixel_format = 64;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr:
+		pixel_format = 67;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb:
+		pixel_format = 66;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616:
+		pixel_format = 26; /* ARGB16161616_UNORM */
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+		pixel_format = 24;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+		pixel_format = 25;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888:
+		pixel_format = 12;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
+		pixel_format = 112;
+		alpha_en = 0;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
+		pixel_format = 113;
+		alpha_en = 0;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010:
+		pixel_format = 114;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		is_2bit = 1;
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_CrYCbA1010102:
+		pixel_format = 115;
+		color_space = COLOR_SPACE_YCBCR709;
+		select = INPUT_CSC_SELECT_ICSC;
+		is_2bit = 1;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGBE:
+		pixel_format = 116;
+		alpha_plane_enable = 0;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA:
+		pixel_format = 116;
+		alpha_plane_enable = 1;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
+		pixel_format = 118;
+		alpha_en = 0;
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
+		pixel_format = 119;
+		alpha_en = 0;
+		break;
+	default:
+		break;
+	}
+
+	/* Set default color space based on format if none is given. */
+	color_space = input_color_space ? input_color_space : color_space;
+
+	if (is_2bit == 1 && alpha_2bit_lut != NULL) {
+		REG_UPDATE(ALPHA_2BIT_LUT01, ALPHA_2BIT_LUT0, alpha_2bit_lut->lut0);
+		REG_UPDATE(ALPHA_2BIT_LUT01, ALPHA_2BIT_LUT1, alpha_2bit_lut->lut1);
+		REG_UPDATE(ALPHA_2BIT_LUT23, ALPHA_2BIT_LUT2, alpha_2bit_lut->lut2);
+		REG_UPDATE(ALPHA_2BIT_LUT23, ALPHA_2BIT_LUT3, alpha_2bit_lut->lut3);
+	}
+
+	REG_SET_2(CNVC_SURFACE_PIXEL_FORMAT, 0,
+		CNVC_SURFACE_PIXEL_FORMAT, pixel_format,
+		CNVC_ALPHA_PLANE_ENABLE, alpha_plane_enable);
+	REG_UPDATE(FORMAT_CONTROL, FORMAT_CONTROL__ALPHA_EN, alpha_en);
+
+	REG_SET_2(PRE_DEALPHA, 0,
+		PRE_DEALPHA_EN, dealpha_en,
+		PRE_DEALPHA_ABLND_EN, dealpha_ablnd_en);
+	REG_SET_2(PRE_REALPHA, 0,
+		PRE_REALPHA_EN, realpha_en,
+		PRE_REALPHA_ABLND_EN, realpha_ablnd_en);
+
+	/* If input adjustment exists, program the ICSC with those values. */
+	if (input_csc_color_matrix.enable_adjustment == true) {
+		for (i = 0; i < 12; i++)
+			tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
+
+		tbl_entry.color_space = input_color_space;
+
+		if (color_space >= COLOR_SPACE_YCBCR601)
+			select = INPUT_CSC_SELECT_ICSC;
+		else
+			select = INPUT_CSC_SELECT_BYPASS;
+
+		dpp3_program_post_csc(dpp_base, color_space, select,
+			&tbl_entry);
+	} else {
+		dpp3_program_post_csc(dpp_base, color_space, select, NULL);
+	}
+}
+
+static struct dpp_funcs dcn42_dpp_funcs = {
+	.dpp_program_gamcor_lut		= dpp3_program_gamcor_lut,
+	.dpp_read_state				= dpp401_read_state,
+	.dpp_reset					= dpp_reset,
+	.dpp_set_scaler				= dpp401_dscl_set_scaler_manual_scale,
+	.dpp_get_optimal_number_of_taps	= dpp3_get_optimal_number_of_taps,
+	.dpp_set_pre_degam			= dpp3_set_pre_degam,
+	.dpp_setup					= dpp42_dpp_setup,
+	.dpp_program_cm_dealpha		= dpp3_program_cm_dealpha,
+	.dpp_program_cm_bias		= dpp3_program_cm_bias,
+	.dpp_program_bias_and_scale	= dpp35_program_bias_and_scale_fcnv,
+	.dpp_cnv_set_alpha_keyer	= dpp2_cnv_set_alpha_keyer,
+	.set_cursor_attributes		= dpp401_set_cursor_attributes,
+	.set_cursor_position		= dpp401_set_cursor_position,
+	.set_optional_cursor_attributes	= dpp401_set_optional_cursor_attributes,
+	.dpp_dppclk_control			= dpp35_dppclk_control,
+	.dpp_set_hdr_multiplier		= dpp3_set_hdr_multiplier,
+	.set_cursor_matrix			= dpp401_set_cursor_matrix,
+	.dpp_cm_hist_control        = dpp42_dpp_cm_hist_control,
+	.dpp_cm_hist_read           = dpp42_dpp_cm_hist_read,
+	.dpp_read_reg_state			= dpp30_read_reg_state,
+};
+
+
+static struct dpp_caps dcn42_dpp_cap = {
+	.dscl_data_proc_format = DSCL_DATA_PRCESSING_FLOAT_FORMAT,
+	.max_lb_partitions = 63,
+	.dscl_calc_lb_num_partitions = dscl401_calc_lb_num_partitions,
+};
+
+bool dpp42_construct(
+	struct dcn42_dpp *dpp,
+	struct dc_context *ctx,
+	uint32_t inst,
+	const struct dcn42_dpp_registers *tf_regs,
+	const struct dcn42_dpp_shift *tf_shift,
+	const struct dcn42_dpp_mask *tf_mask)
+{
+	dpp->base.ctx = ctx;
+
+	dpp->base.inst = inst;
+	dpp->base.funcs = &dcn42_dpp_funcs;
+	dpp->base.caps = &dcn42_dpp_cap;
+
+	dpp->tf_regs = tf_regs;
+	dpp->tf_shift = tf_shift;
+	dpp->tf_mask = tf_mask;
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.h
new file mode 100644
index 0000000000000..f4ee419eabf77
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.h
@@ -0,0 +1,469 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef __DCN42_DPP_H__
+#define __DCN42_DPP_H__
+
+#include "dcn401/dcn401_dpp.h"
+
+#define TO_DCN42_DPP(dpp)\
+	 container_of(dpp, struct dcn42_dpp, base)
+
+#define DPP_REG_LIST_SH_MASK_DCN42_COMMON(mask_sh)\
+	TF_SF(CM0_CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, mask_sh),\
+	TF_SF(CM0_CM_DEALPHA, CM_DEALPHA_EN, mask_sh),\
+	TF_SF(CM0_CM_DEALPHA, CM_DEALPHA_ABLND, mask_sh),\
+	TF_SF(CM0_CM_BIAS_CR_R, CM_BIAS_CR_R, mask_sh),\
+	TF_SF(CM0_CM_BIAS_Y_G_CB_B, CM_BIAS_Y_G, mask_sh),\
+	TF_SF(CM0_CM_BIAS_Y_G_CB_B, CM_BIAS_CB_B, mask_sh),\
+	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, mask_sh),\
+	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, mask_sh),\
+	TF_SF(CNVC_CFG0_PRE_DEGAM, PRE_DEGAM_MODE, mask_sh),\
+	TF_SF(CNVC_CFG0_PRE_DEGAM, PRE_DEGAM_SELECT, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_MODE, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_SELECT, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_PWL_DISABLE, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_MODE_CURRENT, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_SELECT_CURRENT, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_INDEX, CM_GAMCOR_LUT_INDEX, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_DATA, CM_GAMCOR_LUT_DATA, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_CONTROL, CM_GAMCOR_LUT_WRITE_COLOR_MASK, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_CONTROL, CM_GAMCOR_LUT_READ_COLOR_SEL, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_CONTROL, CM_GAMCOR_LUT_READ_DBG, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_CONTROL, CM_GAMCOR_LUT_HOST_SEL, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_LUT_CONTROL, CM_GAMCOR_LUT_CONFIG_MODE, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_START_CNTL_B, CM_GAMCOR_RAMA_EXP_REGION_START_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_START_CNTL_B, CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B, CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B, CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_END_CNTL1_B, CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_END_CNTL2_B, CM_GAMCOR_RAMA_EXP_REGION_END_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_END_CNTL2_B, CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_OFFSET_B, CM_GAMCOR_RAMA_OFFSET_B, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_REGION_0_1, CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_REGION_0_1, CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_REGION_0_1, CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	TF_SF(CM0_CM_GAMCOR_RAMA_REGION_0_1, CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	TF_SF(DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT, EXT_OVERSCAN_LEFT, mask_sh),\
+	TF_SF(DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT, EXT_OVERSCAN_RIGHT, mask_sh),\
+	TF_SF(DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM, EXT_OVERSCAN_BOTTOM, mask_sh),\
+	TF_SF(DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM, EXT_OVERSCAN_TOP, mask_sh),\
+	TF_SF(DSCL0_OTG_H_BLANK, OTG_H_BLANK_START, mask_sh),\
+	TF_SF(DSCL0_OTG_H_BLANK, OTG_H_BLANK_END, mask_sh),\
+	TF_SF(DSCL0_OTG_V_BLANK, OTG_V_BLANK_START, mask_sh),\
+	TF_SF(DSCL0_OTG_V_BLANK, OTG_V_BLANK_END, mask_sh),\
+	TF_SF(DSCL0_LB_DATA_FORMAT, INTERLEAVE_EN, mask_sh),\
+	TF2_SF(DSCL0, LB_DATA_FORMAT__ALPHA_EN, mask_sh),\
+	TF_SF(DSCL0_LB_MEMORY_CTRL, MEMORY_CONFIG, mask_sh),\
+	TF_SF(DSCL0_LB_MEMORY_CTRL, LB_MAX_PARTITIONS, mask_sh),\
+	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_NUM_PIPE, mask_sh),\
+	TF_SF(DSCL0_DSCL_CONTROL, SCL_BOUNDARY_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_PIPE_ID, mask_sh),\
+	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS, mask_sh),\
+	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_H_NUM_TAPS, mask_sh),\
+	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS_C, mask_sh),\
+	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_H_NUM_TAPS_C, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_SELECT, SCL_COEF_RAM_TAP_PAIR_IDX, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_SELECT, SCL_COEF_RAM_PHASE, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_SELECT, SCL_COEF_RAM_FILTER_TYPE, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_DATA, SCL_COEF_RAM_EVEN_TAP_COEF, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_DATA, SCL_COEF_RAM_EVEN_TAP_COEF_EN, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_DATA, SCL_COEF_RAM_ODD_TAP_COEF, mask_sh),\
+	TF_SF(DSCL0_SCL_COEF_RAM_TAP_DATA, SCL_COEF_RAM_ODD_TAP_COEF_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_H_2TAP_HARDCODE_COEF_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_H_2TAP_SHARP_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_H_2TAP_SHARP_FACTOR, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_V_2TAP_HARDCODE_COEF_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_V_2TAP_SHARP_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_2TAP_CONTROL, SCL_V_2TAP_SHARP_FACTOR, mask_sh),\
+	TF_SF(DSCL0_SCL_MODE, SCL_COEF_RAM_SELECT, mask_sh),\
+	TF_SF(DSCL0_SCL_MODE, DSCL_MODE, mask_sh),\
+	TF_SF(DSCL0_RECOUT_START, RECOUT_START_X, mask_sh),\
+	TF_SF(DSCL0_RECOUT_START, RECOUT_START_Y, mask_sh),\
+	TF_SF(DSCL0_RECOUT_SIZE, RECOUT_WIDTH, mask_sh),\
+	TF_SF(DSCL0_RECOUT_SIZE, RECOUT_HEIGHT, mask_sh),\
+	TF_SF(DSCL0_MPC_SIZE, MPC_WIDTH, mask_sh),\
+	TF_SF(DSCL0_MPC_SIZE, MPC_HEIGHT, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_SCALE_RATIO, SCL_H_SCALE_RATIO, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_SCALE_RATIO, SCL_V_SCALE_RATIO, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C, SCL_H_SCALE_RATIO_C, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_SCALE_RATIO_C, SCL_V_SCALE_RATIO_C, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_INIT, SCL_H_INIT_FRAC, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_INIT, SCL_H_INIT_INT, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_INIT_C, SCL_H_INIT_FRAC_C, mask_sh),\
+	TF_SF(DSCL0_SCL_HORZ_FILTER_INIT_C, SCL_H_INIT_INT_C, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT, SCL_V_INIT_FRAC, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT, SCL_V_INIT_INT, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_C, SCL_V_INIT_FRAC_C, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_C, SCL_V_INIT_INT_C, mask_sh),\
+	TF_SF(DSCL0_SCL_MODE, SCL_CHROMA_COEF_MODE, mask_sh),\
+	TF_SF(DSCL0_SCL_MODE, SCL_COEF_RAM_SELECT_CURRENT, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_DEALPHA, PRE_DEALPHA_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_DEALPHA, PRE_DEALPHA_ABLND_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_REALPHA, PRE_REALPHA_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_REALPHA, PRE_REALPHA_ABLND_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_MODE, PRE_CSC_MODE, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_MODE, PRE_CSC_MODE_CURRENT, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_C11_C12, PRE_CSC_C11, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_C11_C12, PRE_CSC_C12, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_C33_C34, PRE_CSC_C33, mask_sh), \
+	TF_SF(CNVC_CFG0_PRE_CSC_C33_C34, PRE_CSC_C34, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_CONTROL, CM_POST_CSC_MODE, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_CONTROL, CM_POST_CSC_MODE_CURRENT, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_C11_C12, CM_POST_CSC_C11, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_C11_C12, CM_POST_CSC_C12, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_C33_C34, CM_POST_CSC_C33, mask_sh), \
+	TF_SF(CM0_CM_POST_CSC_C33_C34, CM_POST_CSC_C34, mask_sh), \
+	TF_SF(CM0_CM_TEST_DEBUG_INDEX, CM_TEST_DEBUG_INDEX, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CNVC_BYPASS, mask_sh), \
+	TF2_SF(CNVC_CFG0, FORMAT_CONTROL__ALPHA_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_EXPANSION_MODE, mask_sh), \
+	TF_SF(CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT, CNVC_SURFACE_PIXEL_FORMAT, mask_sh), \
+	TF_SF(CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT, CNVC_ALPHA_PLANE_ENABLE, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_MODE, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_EXPANSION_MODE, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_ENABLE, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_COLOR0, CUR0_COLOR0, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_COLOR1, CUR0_COLOR1, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y, CUR0_FP_BIAS_G_Y, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y, CUR0_FP_SCALE_G_Y, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB, CUR0_FP_BIAS_RB_CRCB, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB, CUR0_FP_SCALE_RB_CRCB, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_MODE, CUR0_MATRIX_MODE, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_MODE, CUR0_MATRIX_MODE_CURRENT, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_MODE, CUR0_MATRIX_COEF_FORMAT, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C11_C12_A, CUR0_MATRIX_C11_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C11_C12_A, CUR0_MATRIX_C12_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C13_C14_A, CUR0_MATRIX_C13_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C13_C14_A, CUR0_MATRIX_C14_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C21_C22_A, CUR0_MATRIX_C21_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C21_C22_A, CUR0_MATRIX_C22_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C23_C24_A, CUR0_MATRIX_C23_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C23_C24_A, CUR0_MATRIX_C24_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C31_C32_A, CUR0_MATRIX_C31_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C31_C32_A, CUR0_MATRIX_C32_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C33_C34_A, CUR0_MATRIX_C33_A, mask_sh), \
+	TF_SF(CM_CUR0_CUR0_MATRIX_C33_C34_A, CUR0_MATRIX_C34_A, mask_sh), \
+	TF_SF(DPP_TOP0_DPP_CONTROL, DPP_CLOCK_ENABLE, mask_sh), \
+	TF_SF(CM0_CM_HDR_MULT_COEF, CM_HDR_MULT_COEF, mask_sh), \
+	TF_SF(CM0_CM_CONTROL, CM_BYPASS, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_SEL, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_CH_EN, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_SRC1_SEL, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_SRC2_SEL, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_SRC3_SEL, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_CH1_XBAR, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_CH2_XBAR, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_CH3_XBAR, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_FORMAT, mask_sh), \
+	TF_SF(CM0_CM_HIST_CNTL, CM_HIST_READ_CHANNEL_MASK, mask_sh), \
+	TF_SF(CM0_CM_HIST_LOCK, CM_HIST_LOCK, mask_sh), \
+	TF_SF(CM0_CM_HIST_INDEX, CM_HIST_INDEX, mask_sh), \
+	TF_SF(CM0_CM_HIST_DATA, CM_HIST_DATA, mask_sh), \
+	TF_SF(CM0_CM_HIST_DATA, CM_HIST_DATA, mask_sh), \
+	TF_SF(CM0_CM_HIST_STATUS, CM_HIST_BUFA_RDY_STATUS, mask_sh), \
+	TF_SF(CM0_CM_HIST_STATUS, CM_HIST_BUFB_RDY_STATUS, mask_sh), \
+	TF_SF(CM0_CM_HIST_SCALE_SRC1, CM_HIST_SCALE_SRC1, mask_sh), \
+	TF_SF(CM0_CM_HIST_COEFA_SRC2, CM_HIST_COEFA_SRC2, mask_sh), \
+	TF_SF(CM0_CM_HIST_COEFB_SRC2, CM_HIST_COEFB_SRC2, mask_sh), \
+	TF_SF(CM0_CM_HIST_COEFC_SRC2, CM_HIST_COEFC_SRC2, mask_sh), \
+	TF_SF(CM0_CM_HIST_SCALE_SRC3, CM_HIST_SCALE_SRC3, mask_sh), \
+	TF_SF(CM0_CM_HIST_BIAS_SRC1, CM_HIST_BIAS_SRC1, mask_sh), \
+	TF_SF(CM0_CM_HIST_BIAS_SRC2, CM_HIST_BIAS_SRC2, mask_sh), \
+	TF_SF(CM0_CM_HIST_BIAS_SRC3, CM_HIST_BIAS_SRC3, mask_sh), \
+	TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
+	TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
+	TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
+	TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_CNV16, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CNVC_BYPASS_MSB_ALIGN, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CLAMP_POSITIVE, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CLAMP_POSITIVE_C, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_CROSSBAR_R, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_CROSSBAR_G, mask_sh), \
+	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_CROSSBAR_B, mask_sh), \
+	TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT01, ALPHA_2BIT_LUT0, mask_sh), \
+	TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT01, ALPHA_2BIT_LUT1, mask_sh), \
+	TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT23, ALPHA_2BIT_LUT2, mask_sh), \
+	TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT23, ALPHA_2BIT_LUT3, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_BIAS_R, FCNV_FP_BIAS_R, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_BIAS_G, FCNV_FP_BIAS_G, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_BIAS_B, FCNV_FP_BIAS_B, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_SCALE_R, FCNV_FP_SCALE_R, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_SCALE_G, FCNV_FP_SCALE_G, mask_sh), \
+	TF_SF(CNVC_CFG0_FCNV_FP_SCALE_B, FCNV_FP_SCALE_B, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_CONTROL, COLOR_KEYER_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_CONTROL, LUMA_KEYER_EN, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_CONTROL, COLOR_KEYER_MODE, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_LOW, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_HIGH, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_RED, COLOR_KEYER_RED_LOW, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_RED, COLOR_KEYER_RED_HIGH, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_LOW, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_HIGH, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_LOW, mask_sh), \
+	TF_SF(CNVC_CFG0_COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_HIGH, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_PIX_INV_MODE, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_PIXEL_ALPHA_MOD_EN, mask_sh), \
+	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_ROM_EN, mask_sh),\
+	TF_SF(DSCL0_OBUF_MEM_PWR_CTRL, OBUF_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(DSCL0_DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(DSCL0_DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_DIS, mask_sh),\
+	TF_SF(DSCL0_DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MODE, SCL_SC_MATRIX_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MODE, SCL_SC_LTONL_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_MODE, SCL_EASF_H_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_MODE, SCL_EASF_H_RINGEST_FORCE_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_MODE, SCL_EASF_H_2TAP_SHARP_FACTOR, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF1_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF2_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF3_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF2_FLAT1_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF2_FLAT2_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_CNTL, SCL_EASF_H_BF2_ROC_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE, SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE, SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN, SCL_EASF_H_RINGEST_EVENTAP_GAIN1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN, SCL_EASF_H_RINGEST_EVENTAP_GAIN2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN, SCL_EASF_H_BF_MAXA, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN, SCL_EASF_H_BF_MAXB, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN, SCL_EASF_H_BF_MINA, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN, SCL_EASF_H_BF_MINB, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG0, SCL_EASF_H_BF1_PWL_IN_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG0, SCL_EASF_H_BF1_PWL_BASE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG0, SCL_EASF_H_BF1_PWL_SLOPE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG1, SCL_EASF_H_BF1_PWL_IN_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG1, SCL_EASF_H_BF1_PWL_BASE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG1, SCL_EASF_H_BF1_PWL_SLOPE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG2, SCL_EASF_H_BF1_PWL_IN_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG2, SCL_EASF_H_BF1_PWL_BASE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG2, SCL_EASF_H_BF1_PWL_SLOPE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG3, SCL_EASF_H_BF1_PWL_IN_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG3, SCL_EASF_H_BF1_PWL_BASE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG3, SCL_EASF_H_BF1_PWL_SLOPE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG4, SCL_EASF_H_BF1_PWL_IN_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG4, SCL_EASF_H_BF1_PWL_BASE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG4, SCL_EASF_H_BF1_PWL_SLOPE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG5, SCL_EASF_H_BF1_PWL_IN_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG5, SCL_EASF_H_BF1_PWL_BASE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG5, SCL_EASF_H_BF1_PWL_SLOPE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG6, SCL_EASF_H_BF1_PWL_IN_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG6, SCL_EASF_H_BF1_PWL_BASE_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG6, SCL_EASF_H_BF1_PWL_SLOPE_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG7, SCL_EASF_H_BF1_PWL_IN_SEG7, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF1_PWL_SEG7, SCL_EASF_H_BF1_PWL_BASE_SEG7, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG0, SCL_EASF_H_BF3_PWL_IN_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG0, SCL_EASF_H_BF3_PWL_BASE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG0, SCL_EASF_H_BF3_PWL_SLOPE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG1, SCL_EASF_H_BF3_PWL_IN_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG1, SCL_EASF_H_BF3_PWL_BASE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG1, SCL_EASF_H_BF3_PWL_SLOPE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG2, SCL_EASF_H_BF3_PWL_IN_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG2, SCL_EASF_H_BF3_PWL_BASE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG2, SCL_EASF_H_BF3_PWL_SLOPE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG3, SCL_EASF_H_BF3_PWL_IN_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG3, SCL_EASF_H_BF3_PWL_BASE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG3, SCL_EASF_H_BF3_PWL_SLOPE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG4, SCL_EASF_H_BF3_PWL_IN_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG4, SCL_EASF_H_BF3_PWL_BASE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG4, SCL_EASF_H_BF3_PWL_SLOPE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG5, SCL_EASF_H_BF3_PWL_IN_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_H_BF3_PWL_SEG5, SCL_EASF_H_BF3_PWL_BASE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_MODE, SCL_EASF_V_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_MODE, SCL_EASF_V_RINGEST_FORCE_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_MODE, SCL_EASF_V_2TAP_SHARP_FACTOR, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF1_EN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF2_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF3_MODE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF2_FLAT1_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF2_FLAT2_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_CNTL, SCL_EASF_V_BF2_ROC_GAIN, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1, SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1, SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2, SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2, SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3, SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3, SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE, SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE, SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN, SCL_EASF_V_RINGEST_EVENTAP_GAIN1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN, SCL_EASF_V_RINGEST_EVENTAP_GAIN2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN, SCL_EASF_V_BF_MAXA, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN, SCL_EASF_V_BF_MAXB, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN, SCL_EASF_V_BF_MINA, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN, SCL_EASF_V_BF_MINB, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG0, SCL_EASF_V_BF1_PWL_IN_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG0, SCL_EASF_V_BF1_PWL_BASE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG0, SCL_EASF_V_BF1_PWL_SLOPE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG1, SCL_EASF_V_BF1_PWL_IN_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG1, SCL_EASF_V_BF1_PWL_BASE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG1, SCL_EASF_V_BF1_PWL_SLOPE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG2, SCL_EASF_V_BF1_PWL_IN_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG2, SCL_EASF_V_BF1_PWL_BASE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG2, SCL_EASF_V_BF1_PWL_SLOPE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG3, SCL_EASF_V_BF1_PWL_IN_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG3, SCL_EASF_V_BF1_PWL_BASE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG3, SCL_EASF_V_BF1_PWL_SLOPE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG4, SCL_EASF_V_BF1_PWL_IN_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG4, SCL_EASF_V_BF1_PWL_BASE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG4, SCL_EASF_V_BF1_PWL_SLOPE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG5, SCL_EASF_V_BF1_PWL_IN_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG5, SCL_EASF_V_BF1_PWL_BASE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG5, SCL_EASF_V_BF1_PWL_SLOPE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG6, SCL_EASF_V_BF1_PWL_IN_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG6, SCL_EASF_V_BF1_PWL_BASE_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG6, SCL_EASF_V_BF1_PWL_SLOPE_SEG6, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG7, SCL_EASF_V_BF1_PWL_IN_SEG7, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF1_PWL_SEG7, SCL_EASF_V_BF1_PWL_BASE_SEG7, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG0, SCL_EASF_V_BF3_PWL_IN_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG0, SCL_EASF_V_BF3_PWL_BASE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG0, SCL_EASF_V_BF3_PWL_SLOPE_SEG0, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG1, SCL_EASF_V_BF3_PWL_IN_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG1, SCL_EASF_V_BF3_PWL_BASE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG1, SCL_EASF_V_BF3_PWL_SLOPE_SEG1, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG2, SCL_EASF_V_BF3_PWL_IN_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG2, SCL_EASF_V_BF3_PWL_BASE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG2, SCL_EASF_V_BF3_PWL_SLOPE_SEG2, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG3, SCL_EASF_V_BF3_PWL_IN_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG3, SCL_EASF_V_BF3_PWL_BASE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG3, SCL_EASF_V_BF3_PWL_SLOPE_SEG3, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG4, SCL_EASF_V_BF3_PWL_IN_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG4, SCL_EASF_V_BF3_PWL_BASE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG4, SCL_EASF_V_BF3_PWL_SLOPE_SEG4, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG5, SCL_EASF_V_BF3_PWL_IN_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_EASF_V_BF3_PWL_SEG5, SCL_EASF_V_BF3_PWL_BASE_SEG5, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MATRIX_C0C1, SCL_SC_MATRIX_C0, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MATRIX_C0C1, SCL_SC_MATRIX_C1, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MATRIX_C2C3, SCL_SC_MATRIX_C2, mask_sh),\
+	TF_SF(DSCL0_DSCL_SC_MATRIX_C2C3, SCL_SC_MATRIX_C3, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_CTRL, ISHARP_DELTA_LUT_HOST_SELECT, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_DIS, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_STATE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_DATA, ISHARP_DELTA_DATA, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_INDEX, ISHARP_DELTA_INDEX, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_EN, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_NOISEDET_EN, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_NOISEDET_MODE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_LBA_MODE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_DELTA_LUT_SELECT, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_FMT_MODE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_FMT_NORM, mask_sh),\
+	TF_SF(DSCL0_ISHARP_MODE, ISHARP_DELTA_LUT_SELECT_CURRENT, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG0, ISHARP_LBA_PWL_IN_SEG0, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG0, ISHARP_LBA_PWL_BASE_SEG0, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG0, ISHARP_LBA_PWL_SLOPE_SEG0, mask_sh), \
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG1, ISHARP_LBA_PWL_IN_SEG1, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG1, ISHARP_LBA_PWL_BASE_SEG1, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG1, ISHARP_LBA_PWL_SLOPE_SEG1, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG2, ISHARP_LBA_PWL_IN_SEG2, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG2, ISHARP_LBA_PWL_BASE_SEG2, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG2, ISHARP_LBA_PWL_SLOPE_SEG2, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG3, ISHARP_LBA_PWL_IN_SEG3, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG3, ISHARP_LBA_PWL_BASE_SEG3, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG3, ISHARP_LBA_PWL_SLOPE_SEG3, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG4, ISHARP_LBA_PWL_IN_SEG4, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG4, ISHARP_LBA_PWL_BASE_SEG4, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG4, ISHARP_LBA_PWL_SLOPE_SEG4, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG5, ISHARP_LBA_PWL_IN_SEG5, mask_sh),\
+	TF_SF(DSCL0_ISHARP_LBA_PWL_SEG5, ISHARP_LBA_PWL_BASE_SEG5, mask_sh),\
+	TF_SF(DSCL0_ISHARP_NOISEDET_THRESHOLD, ISHARP_NOISEDET_UTHRE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_NOISEDET_THRESHOLD, ISHARP_NOISEDET_DTHRE, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NOISE_GAIN_PWL, ISHARP_NOISEDET_PWL_START_IN, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NOISE_GAIN_PWL, ISHARP_NOISEDET_PWL_END_IN, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NOISE_GAIN_PWL, ISHARP_NOISEDET_PWL_SLOPE, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_EN_P, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_PIVOT_P, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_SLOPE_P, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_EN_N, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_PIVOT_N, mask_sh), \
+	TF_SF(DSCL0_ISHARP_NLDELTA_SOFT_CLIP, ISHARP_NLDELTA_SCLIP_SLOPE_N, mask_sh), \
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_BOT, SCL_V_INIT_FRAC_BOT, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_BOT, SCL_V_INIT_INT_BOT, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_BOT_C, SCL_V_INIT_FRAC_BOT_C, mask_sh),\
+	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_BOT_C, SCL_V_INIT_INT_BOT_C, mask_sh)
+
+#define DPP_REG_FIELD_LIST_DCN42(type) \
+	DPP_REG_FIELD_LIST_DCN401(type); \
+	type CM_HIST_SEL; \
+	type CM_HIST_CH_EN; \
+	type CM_HIST_SRC1_SEL; \
+	type CM_HIST_SRC2_SEL; \
+	type CM_HIST_SRC3_SEL; \
+	type CM_HIST_CH1_XBAR; \
+	type CM_HIST_CH2_XBAR; \
+	type CM_HIST_CH3_XBAR; \
+	type CM_HIST_FORMAT; \
+	type CM_HIST_READ_CHANNEL_MASK; \
+	type CM_HIST_LOCK; \
+	type CM_HIST_INDEX; \
+	type CM_HIST_DATA; \
+	type CM_HIST_BUFA_RDY_STATUS; \
+	type CM_HIST_BUFB_RDY_STATUS; \
+	type CM_HIST_SCALE_SRC1; \
+	type CM_HIST_COEFA_SRC2; \
+	type CM_HIST_COEFB_SRC2; \
+	type CM_HIST_COEFC_SRC2; \
+	type CM_HIST_SCALE_SRC3; \
+	type CM_HIST_BIAS_SRC1;  \
+	type CM_HIST_BIAS_SRC2;  \
+	type CM_HIST_BIAS_SRC3
+
+
+#define DPP_REG_VARIABLE_LIST_DCN42 \
+	DPP_REG_VARIABLE_LIST_DCN401; \
+	uint32_t ALPHA_2BIT_LUT01; \
+	uint32_t ALPHA_2BIT_LUT23; \
+	uint32_t CM_HIST_CNTL; \
+	uint32_t CM_HIST_INDEX; \
+	uint32_t CM_HIST_LOCK; \
+	uint32_t CM_HIST_DATA; \
+	uint32_t CM_HIST_STATUS; \
+	uint32_t CM_HIST_SCALE_SRC1; \
+	uint32_t CM_HIST_COEFA_SRC2; \
+	uint32_t CM_HIST_COEFB_SRC2; \
+	uint32_t CM_HIST_COEFC_SRC2; \
+	uint32_t CM_HIST_SCALE_SRC3; \
+	uint32_t CM_HIST_BIAS_SRC1;  \
+	uint32_t CM_HIST_BIAS_SRC2;  \
+	uint32_t CM_HIST_BIAS_SRC3
+
+struct dcn42_dpp_registers {
+	DPP_REG_VARIABLE_LIST_DCN42;
+};
+
+struct dcn42_dpp_shift {
+	DPP_REG_FIELD_LIST_DCN42(uint8_t);
+};
+
+struct dcn42_dpp_mask {
+	DPP_REG_FIELD_LIST_DCN42(uint32_t);
+};
+struct dcn42_dpp {
+	struct dpp base;
+	const struct dcn42_dpp_registers *tf_regs;
+	const struct dcn42_dpp_shift *tf_shift;
+	const struct dcn42_dpp_mask *tf_mask;
+	const uint16_t *filter_v;
+	const uint16_t *filter_h;
+	const uint16_t *filter_v_c;
+	const uint16_t *filter_h_c;
+	int lb_pixel_depth_supported;
+	int lb_memory_size;
+	int lb_bits_per_entry;
+	bool is_write_to_ram_a_safe;
+	struct scaler_data scl_data;
+	struct pwl_params pwl_data;
+};
+
+bool dpp42_construct(struct dcn42_dpp *dpp42,
+	struct dc_context *ctx,
+	uint32_t inst,
+	const struct dcn42_dpp_registers *tf_regs,
+	const struct dcn42_dpp_shift *tf_shift,
+	const struct dcn42_dpp_mask *tf_mask);
+
+
+#endif /* __DCN42_DPP_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/Makefile b/drivers/gpu/drm/amd/display/dc/gpio/Makefile
index b72e2a9f9a28e..f8142ebf363f3 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/gpio/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2017 Advanced Micro Devices, Inc.
+# Copyright 2017-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -132,3 +132,14 @@ AMD_DAL_GPIO_DCN401 = $(addprefix $(AMDDALPATH)/dc/gpio/dcn401/,$(GPIO_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_GPIO_DCN401)
 
+###############################################################################
+# DCN 4.2
+###############################################################################
+
+GPIO_DCN42 = hw_translate_dcn42.o hw_factory_dcn42.o
+
+AMD_DAL_GPIO_DCN42 = $(addprefix $(AMDDALPATH)/dc/gpio/dcn42/,$(GPIO_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_GPIO_DCN42)
+
+
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.c
new file mode 100644
index 0000000000000..7d2090c7870e6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.c
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "include/gpio_types.h"
+#include "../hw_factory.h"
+
+
+#include "../hw_gpio.h"
+#include "../hw_ddc.h"
+#include "../hw_hpd.h"
+#include "../hw_generic.h"
+
+
+
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+#include "dpcs/dpcs_4_0_0_offset.h"
+#include "dpcs/dpcs_4_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "../hpd_regs.h"
+#include "hw_factory_dcn42.h"
+
+#define DCN_BASE__INST0_SEG2                       0x000034C0
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define block HPD
+#define reg_num 0
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+
+
+#define REG(reg_name)\
+		(BASE(reg ## reg_name ## _BASE_IDX) + reg ## reg_name)
+
+#define SF_HPD(reg_name, field_name, post_fix)\
+	.field_name = HPD0_ ## reg_name ## __ ## field_name ## post_fix
+
+#define REGI(reg_name, block, id)\
+	(BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+				reg ## block ## id ## _ ## reg_name)
+
+#define SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+// DC_GPIO_HPD_* registers are gone.
+#undef HPD_REG_LIST
+#define HPD_REG_LIST(id) \
+	.int_status = REGI(DC_HPD_INT_STATUS, HPD, id),\
+	.toggle_filt_cntl = REGI(DC_HPD_TOGGLE_FILT_CNTL, HPD, id)
+
+#define hpd_regs(id) \
+{\
+	HPD_REG_LIST(id)\
+}
+
+static const struct hpd_registers hpd_regs[] = {
+	hpd_regs(0),
+	hpd_regs(1),
+	hpd_regs(2),
+	hpd_regs(3),
+	hpd_regs(4),
+};
+
+static const struct hpd_sh_mask hpd_shift = {
+		HPD_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct hpd_sh_mask hpd_mask = {
+		HPD_MASK_SH_LIST(_MASK)
+};
+
+#include "../ddc_regs.h"
+
+ /* set field name */
+#define SF_DDC(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+static const struct ddc_registers ddc_data_regs_dcn[] = {
+	ddc_data_regs_dcn2(1),
+	ddc_data_regs_dcn2(2),
+	ddc_data_regs_dcn2(3),
+	ddc_data_regs_dcn2(4),
+	ddc_data_regs_dcn2(5),
+	{
+		// add a dummy entry for cases no such port
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,},
+		.ddc_setup = 0,
+		.phy_aux_cntl = 0,
+		.dc_gpio_aux_ctrl_5 = 0
+	},
+	{
+			DDC_GPIO_VGA_REG_LIST(DATA),
+			.ddc_setup = 0,
+			.phy_aux_cntl = 0,
+			.dc_gpio_aux_ctrl_5 = 0
+	}
+};
+
+static const struct ddc_registers ddc_clk_regs_dcn[] = {
+	ddc_clk_regs_dcn2(1),
+	ddc_clk_regs_dcn2(2),
+	ddc_clk_regs_dcn2(3),
+	ddc_clk_regs_dcn2(4),
+	ddc_clk_regs_dcn2(5),
+	{
+		// add a dummy entry for cases no such port
+		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,},
+		.ddc_setup = 0,
+		.phy_aux_cntl = 0,
+		.dc_gpio_aux_ctrl_5 = 0
+	},
+	{
+			DDC_GPIO_VGA_REG_LIST(CLK),
+			.ddc_setup = 0,
+			.phy_aux_cntl = 0,
+			.dc_gpio_aux_ctrl_5 = 0
+	}
+};
+
+static const struct ddc_sh_mask ddc_shift[] = {
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 1),
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 2),
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 3),
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 4),
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 5),
+	DDC_MASK_SH_LIST_DCN2(__SHIFT, 6),
+	DDC_MASK_SH_LIST_DCN2_VGA(__SHIFT)
+};
+
+static const struct ddc_sh_mask ddc_mask[] = {
+	DDC_MASK_SH_LIST_DCN2(_MASK, 1),
+	DDC_MASK_SH_LIST_DCN2(_MASK, 2),
+	DDC_MASK_SH_LIST_DCN2(_MASK, 3),
+	DDC_MASK_SH_LIST_DCN2(_MASK, 4),
+	DDC_MASK_SH_LIST_DCN2(_MASK, 5),
+	DDC_MASK_SH_LIST_DCN2(_MASK, 6),
+	DDC_MASK_SH_LIST_DCN2_VGA(_MASK)
+};
+
+#include "../generic_regs.h"
+
+/* set field name */
+#define SF_GENERIC(reg_name, field_name, post_fix)\
+	.field_name = 0
+
+static const struct generic_registers generic_regs[] = {
+	{{ 0 }},
+	{{ 0 }},
+};
+
+static const struct generic_sh_mask generic_shift[] = {
+	{ 0 },
+	{ 0 },
+};
+
+static const struct generic_sh_mask generic_mask[] = {
+	{ 0 },
+	{ 0 },
+};
+
+static void define_generic_registers(struct hw_gpio_pin *pin, uint32_t en)
+{
+	struct hw_generic *generic = HW_GENERIC_FROM_BASE(pin);
+
+	generic->regs = &generic_regs[en];
+	generic->shifts = &generic_shift[en];
+	generic->masks = &generic_mask[en];
+	generic->base.regs = &generic_regs[en].gpio;
+}
+
+static void define_ddc_registers(
+		struct hw_gpio_pin *pin,
+		uint32_t en)
+{
+	struct hw_ddc *ddc = HW_DDC_FROM_BASE(pin);
+
+	switch (pin->id) {
+	case GPIO_ID_DDC_DATA:
+		ddc->regs = &ddc_data_regs_dcn[en];
+		ddc->base.regs = &ddc_data_regs_dcn[en].gpio;
+		break;
+	case GPIO_ID_DDC_CLOCK:
+		ddc->regs = &ddc_clk_regs_dcn[en];
+		ddc->base.regs = &ddc_clk_regs_dcn[en].gpio;
+		break;
+	default:
+		ASSERT_CRITICAL(false);
+		return;
+	}
+
+	ddc->shifts = &ddc_shift[en];
+	ddc->masks = &ddc_mask[en];
+
+}
+
+static void define_hpd_registers(struct hw_gpio_pin *pin, uint32_t en)
+{
+	struct hw_hpd *hpd = HW_HPD_FROM_BASE(pin);
+
+	hpd->regs = &hpd_regs[en];
+	hpd->shifts = &hpd_shift;
+	hpd->masks = &hpd_mask;
+	hpd->base.regs = &hpd_regs[en].gpio;
+}
+
+
+/* function table */
+static const struct hw_factory_funcs funcs = {
+	.init_ddc_data = dal_hw_ddc_init,
+	.init_generic = dal_hw_generic_init,
+	.init_hpd = dal_hw_hpd_init,
+	.get_ddc_pin = dal_hw_ddc_get_pin,
+	.get_hpd_pin = dal_hw_hpd_get_pin,
+	.get_generic_pin = dal_hw_generic_get_pin,
+	.define_hpd_registers = define_hpd_registers,
+	.define_ddc_registers = define_ddc_registers,
+	.define_generic_registers = define_generic_registers
+};
+
+/*
+ * dal_hw_factory_dcn42_init
+ *
+ * @brief
+ * Initialize HW factory function pointers and pin info
+ *
+ * @param
+ * struct hw_factory *factory - [out] struct of function pointers
+ */
+void dal_hw_factory_dcn42_init(struct hw_factory *factory)
+{
+	factory->number_of_pins[GPIO_ID_DDC_DATA] = 8;
+	factory->number_of_pins[GPIO_ID_DDC_CLOCK] = 8;
+	factory->number_of_pins[GPIO_ID_GENERIC] = 4;
+	factory->number_of_pins[GPIO_ID_HPD] = 5;
+	factory->number_of_pins[GPIO_ID_GPIO_PAD] = 28;
+	factory->number_of_pins[GPIO_ID_VIP_PAD] = 0;
+	factory->number_of_pins[GPIO_ID_SYNC] = 0;
+	factory->number_of_pins[GPIO_ID_GSL] = 0;/*add this*/
+
+	factory->funcs = &funcs;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.h b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.h
new file mode 100644
index 0000000000000..b723380eb17f5
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_factory_dcn42.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DAL_HW_FACTORY_DCN42_H__
+#define __DAL_HW_FACTORY_DCN42_H__
+
+/* Initialize HW factory function pointers and pin info */
+void dal_hw_factory_dcn42_init(struct hw_factory *factory);
+
+#endif /* __DAL_HW_FACTORY_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
new file mode 100644
index 0000000000000..dcbcf6b85abff
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "hw_translate_dcn42.h"
+
+#include "dm_services.h"
+#include "include/gpio_types.h"
+#include "../hw_translate.h"
+
+
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+#include "dpcs/dpcs_4_0_0_offset.h"
+#include "dpcs/dpcs_4_0_0_sh_mask.h"
+
+#define DCN_BASE__INST0_SEG2                       0x000034C0
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define block HPD
+#define reg_num 0
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#undef REG
+#define REG(reg_name)\
+		(BASE(reg ## reg_name ## _BASE_IDX) + reg ## reg_name)
+#define SF_HPD(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+
+static bool offset_to_id(
+	uint32_t offset,
+	uint32_t mask,
+	enum gpio_id *id,
+	uint32_t *en)
+{
+	switch (offset) {
+	/* HPD */
+	case REG(HPD0_DC_HPD_INT_STATUS):
+		*id = GPIO_ID_HPD;
+		*en = GPIO_HPD_1;
+		return true;
+	case REG(HPD1_DC_HPD_INT_STATUS):
+		*id = GPIO_ID_HPD;
+		*en = GPIO_HPD_2;
+		return true;
+	case REG(HPD2_DC_HPD_INT_STATUS):
+		*id = GPIO_ID_HPD;
+		*en = GPIO_HPD_3;
+		return true;
+	case REG(HPD3_DC_HPD_INT_STATUS):
+		*id = GPIO_ID_HPD;
+		*en = GPIO_HPD_4;
+		return true;
+	case REG(HPD4_DC_HPD_INT_STATUS):
+		*id = GPIO_ID_HPD;
+		*en = GPIO_HPD_5;
+		return true;
+	/* DDC */
+	/* we don't care about the GPIO_ID for DDC
+	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
+	 * directly in the create method
+	 */
+	case REG(DC_GPIO_DDC1_A):
+		*en = GPIO_DDC_LINE_DDC1;
+		return true;
+	case REG(DC_GPIO_DDC2_A):
+		*en = GPIO_DDC_LINE_DDC2;
+		return true;
+	case REG(DC_GPIO_DDC3_A):
+		*en = GPIO_DDC_LINE_DDC3;
+		return true;
+	case REG(DC_GPIO_DDC4_A):
+		*en = GPIO_DDC_LINE_DDC4;
+		return true;
+	case REG(DC_GPIO_DDC5_A):
+		*en = GPIO_DDC_LINE_DDC5;
+		return true;
+	case REG(DC_GPIO_DDCVGA_A):
+		*en = GPIO_DDC_LINE_DDC_VGA;
+		return true;
+	default:
+		ASSERT_CRITICAL(false);
+		return false;
+	}
+}
+
+
+static bool id_to_offset(
+	enum gpio_id id,
+	uint32_t en,
+	struct gpio_pin_info *info)
+{
+	bool result = true;
+
+	switch (id) {
+	case GPIO_ID_DDC_DATA:
+		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK;
+		switch (en) {
+		case GPIO_DDC_LINE_DDC1:
+			info->offset = REG(DC_GPIO_DDC1_A);
+		break;
+		case GPIO_DDC_LINE_DDC2:
+			info->offset = REG(DC_GPIO_DDC2_A);
+		break;
+		case GPIO_DDC_LINE_DDC3:
+			info->offset = REG(DC_GPIO_DDC3_A);
+		break;
+		case GPIO_DDC_LINE_DDC4:
+			info->offset = REG(DC_GPIO_DDC4_A);
+		break;
+		case GPIO_DDC_LINE_DDC5:
+			info->offset = REG(DC_GPIO_DDC5_A);
+		break;
+		case GPIO_DDC_LINE_DDC_VGA:
+			info->offset = REG(DC_GPIO_DDCVGA_A);
+		break;
+		case GPIO_DDC_LINE_I2C_PAD:
+		default:
+			ASSERT_CRITICAL(false);
+			result = false;
+		}
+	break;
+	case GPIO_ID_DDC_CLOCK:
+		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK;
+		switch (en) {
+		case GPIO_DDC_LINE_DDC1:
+			info->offset = REG(DC_GPIO_DDC1_A);
+		break;
+		case GPIO_DDC_LINE_DDC2:
+			info->offset = REG(DC_GPIO_DDC2_A);
+		break;
+		case GPIO_DDC_LINE_DDC3:
+			info->offset = REG(DC_GPIO_DDC3_A);
+		break;
+		case GPIO_DDC_LINE_DDC4:
+			info->offset = REG(DC_GPIO_DDC4_A);
+		break;
+		case GPIO_DDC_LINE_DDC5:
+			info->offset = REG(DC_GPIO_DDC5_A);
+		break;
+		case GPIO_DDC_LINE_DDC_VGA:
+			info->offset = REG(DC_GPIO_DDCVGA_A);
+		break;
+		case GPIO_DDC_LINE_I2C_PAD:
+		default:
+			ASSERT_CRITICAL(false);
+			result = false;
+		}
+	break;
+	case GPIO_ID_SYNC:
+	case GPIO_ID_VIP_PAD:
+	default:
+		ASSERT_CRITICAL(false);
+		result = false;
+	}
+
+	if (result) {
+		info->offset_y = info->offset + 2;
+		info->offset_en = info->offset + 1;
+		info->offset_mask = info->offset - 1;
+
+		info->mask_y = info->mask;
+		info->mask_en = info->mask;
+		info->mask_mask = info->mask;
+	}
+
+	return result;
+}
+
+
+/* function table */
+static const struct hw_translate_funcs funcs = {
+	.offset_to_id = offset_to_id,
+	.id_to_offset = id_to_offset,
+};
+
+
+/*
+ * dal_hw_translate_dcn42_init
+ *
+ * @brief
+ * Initialize Hw translate function pointers.
+ *
+ * @param
+ * struct hw_translate *tr - [out] struct of function pointers
+ *
+ */
+void dal_hw_translate_dcn42_init(struct hw_translate *tr)
+{
+	tr->funcs = &funcs;
+}
+
+
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.h b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.h
new file mode 100644
index 0000000000000..007b4d55fef1f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __DAL_HW_TRANSLATE_DCN42_H__
+#define __DAL_HW_TRANSLATE_DCN42_H__
+
+struct hw_translate;
+
+/* Initialize Hw translate function pointers */
+void dal_hw_translate_dcn42_init(struct hw_translate *tr);
+#ifdef DAL_EMULATION_SUPPORTED
+void dal_emulated_hw_translate_dcn42_init(struct hw_translate *tr);
+#endif
+
+#endif /* __DAL_HW_TRANSLATE_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
index 8bc67ca421977..f3d562c8df4c9 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
@@ -53,6 +53,7 @@
 #include "dcn315/hw_factory_dcn315.h"
 #include "dcn32/hw_factory_dcn32.h"
 #include "dcn401/hw_factory_dcn401.h"
+#include "dcn42/hw_factory_dcn42.h"
 
 bool dal_hw_factory_init(
 	struct hw_factory *factory,
@@ -118,6 +119,9 @@ bool dal_hw_factory_init(
 	case DCN_VERSION_4_01:
 		dal_hw_factory_dcn401_init(factory);
 		return true;
+	case DCN_VERSION_4_2:
+		dal_hw_factory_dcn42_init(factory);
+		return true;
 	default:
 		ASSERT_CRITICAL(false);
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
index cb79a28322877..1c977fc4d0e36 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
@@ -53,6 +53,7 @@
 #include "dcn315/hw_translate_dcn315.h"
 #include "dcn32/hw_translate_dcn32.h"
 #include "dcn401/hw_translate_dcn401.h"
+#include "dcn42/hw_translate_dcn42.h"
 
 /*
  * This unit
@@ -119,6 +120,9 @@ bool dal_hw_translate_init(
 	case DCN_VERSION_4_01:
 		dal_hw_translate_dcn401_init(translate);
 		return true;
+	case DCN_VERSION_4_2:
+		dal_hw_translate_dcn42_init(translate);
+		return true;
 	default:
 		BREAK_TO_DEBUGGER();
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/Makefile b/drivers/gpu/drm/amd/display/dc/hpo/Makefile
index 7f2c9ee0dff1d..25b196714bd8c 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hpo/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Advanced Micro Devices, Inc.
+# Copyright 2020-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -47,4 +47,13 @@ HPO_DCN32 = dcn32_hpo_dp_link_encoder.o
 AMD_DAL_HPO_DCN32 = $(addprefix $(AMDDALPATH)/dc/hpo/dcn32/,$(HPO_DCN32))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HPO_DCN32)
+
+###############################################################################
+# DCN42
+###############################################################################
+HPO_DCN42 = dcn42_hpo_dp_link_encoder.o
+
+AMD_DAL_HPO_DCN42 = $(addprefix $(AMDDALPATH)/dc/hpo/dcn42/,$(HPO_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_HPO_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
index 759b453385c46..92ed130aeaec9 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
@@ -640,6 +640,11 @@ static void dcn31_hpo_dp_stream_enc_audio_setup(
 	REG_UPDATE(DP_STREAM_ENC_AUDIO_CONTROL,
 			DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL, az_inst);
 
+	if (enc3->hpo_se_mask->DP_STREAM_ENC_APG_CLOCK_EN) {
+		/*enable apg clk*/
+		REG_UPDATE(DP_STREAM_ENC_AUDIO_CONTROL,
+				DP_STREAM_ENC_APG_CLOCK_EN, 1);
+	}
 	ASSERT(enc->apg);
 	enc->apg->funcs->se_audio_setup(enc->apg, az_inst, info);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.h
index 82c3b3ac1f0d0..d96fcc59cc0ee 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Advanced Micro Devices, Inc.
+ * Copyright 2019-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -208,17 +208,24 @@
 	type CRC_CONT_MODE_ENABLE;\
 	type HBLANK_MINIMUM_SYMBOL_WIDTH
 
+#define DCN4_2_HPO_DP_STREAM_ENC_REG_FIELD_LIST(type) \
+	type DP_STREAM_ENC_APG_CLOCK_EN
 
+#define DCN4_2_HPO_DP_STREAM_ENC_MASK_SH_LIST(mask_sh)\
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(mask_sh),\
+	SE_SF(DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC_APG_CLOCK_EN, mask_sh)
 struct dcn31_hpo_dp_stream_encoder_registers {
 	DCN3_1_HPO_DP_STREAM_ENC_REGS;
 };
 
 struct dcn31_hpo_dp_stream_encoder_shift {
 	DCN3_1_HPO_DP_STREAM_ENC_REG_FIELD_LIST(uint8_t);
+	DCN4_2_HPO_DP_STREAM_ENC_REG_FIELD_LIST(uint8_t);
 };
 
 struct dcn31_hpo_dp_stream_encoder_mask {
 	DCN3_1_HPO_DP_STREAM_ENC_REG_FIELD_LIST(uint32_t);
+	DCN4_2_HPO_DP_STREAM_ENC_REG_FIELD_LIST(uint32_t);
 };
 
 struct dcn31_hpo_dp_stream_encoder {
@@ -241,5 +248,4 @@ void dcn31_hpo_dp_stream_encoder_construct(
 	const struct dcn31_hpo_dp_stream_encoder_shift *hpo_se_shift,
 	const struct dcn31_hpo_dp_stream_encoder_mask *hpo_se_mask);
 
-
 #endif   // __DAL_DCN31_HPO_STREAM_ENCODER_H__
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.c
new file mode 100644
index 0000000000000..4b86b1bf6503f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dc_bios_types.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "dcn42_hpo_dp_link_encoder.h"
+#include "reg_helper.h"
+#include "stream_encoder.h"
+
+#define DC_LOGGER \
+		enc3->base.ctx->logger
+
+#define REG(reg)\
+	(enc3->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	enc3->hpo_le_shift->field_name, enc3->hpo_le_mask->field_name
+
+
+#define CTX \
+	enc3->base.ctx
+
+
+static void dcn42_hpo_dp_link_enc_read_state(
+		struct hpo_dp_link_encoder *enc,
+		struct hpo_dp_link_enc_state *state)
+{
+	struct dcn31_hpo_dp_link_encoder *enc3 = DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(enc);
+
+	ASSERT(state);
+
+	REG_GET(DP_DPHY_SYM32_STATUS,
+			STATUS, &state->link_enc_enabled);
+	REG_GET(DP_DPHY_SYM32_CONTROL,
+			NUM_LANES, &state->lane_count);
+	REG_GET(DP_DPHY_SYM32_CONTROL,
+			MODE, (uint32_t *)&state->link_mode);
+
+	REG_GET_2(DP_DPHY_SYM32_SAT_VC0,
+			SAT_STREAM_SOURCE, &state->stream_src[0],
+			SAT_SLOT_COUNT, &state->slot_count[0]);
+	REG_GET_2(DP_DPHY_SYM32_SAT_VC1,
+			SAT_STREAM_SOURCE, &state->stream_src[1],
+			SAT_SLOT_COUNT, &state->slot_count[1]);
+	REG_GET_2(DP_DPHY_SYM32_SAT_VC2,
+			SAT_STREAM_SOURCE, &state->stream_src[2],
+			SAT_SLOT_COUNT, &state->slot_count[2]);
+
+	REG_GET_2(DP_DPHY_SYM32_VC_RATE_CNTL0,
+			STREAM_VC_RATE_X, &state->vc_rate_x[0],
+			STREAM_VC_RATE_Y, &state->vc_rate_y[0]);
+	REG_GET_2(DP_DPHY_SYM32_VC_RATE_CNTL1,
+			STREAM_VC_RATE_X, &state->vc_rate_x[1],
+			STREAM_VC_RATE_Y, &state->vc_rate_y[1]);
+	REG_GET_2(DP_DPHY_SYM32_VC_RATE_CNTL2,
+			STREAM_VC_RATE_X, &state->vc_rate_x[2],
+			STREAM_VC_RATE_Y, &state->vc_rate_y[2]);
+}
+
+static struct hpo_dp_link_encoder_funcs dcn42_hpo_dp_link_encoder_funcs = {
+	.enable_link_phy = dcn31_hpo_dp_link_enc_enable_dp_output,
+	.disable_link_phy = dcn31_hpo_dp_link_enc_disable_output,
+	.link_enable = dcn31_hpo_dp_link_enc_enable,
+	.link_disable = dcn31_hpo_dp_link_enc_disable,
+	.set_link_test_pattern = dcn31_hpo_dp_link_enc_set_link_test_pattern,
+	.update_stream_allocation_table = dcn31_hpo_dp_link_enc_update_stream_allocation_table,
+	.set_throttled_vcp_size = dcn31_hpo_dp_link_enc_set_throttled_vcp_size,
+	.is_in_alt_mode = dcn32_hpo_dp_link_enc_is_in_alt_mode,
+	.read_state = dcn42_hpo_dp_link_enc_read_state,
+	.set_ffe = dcn31_hpo_dp_link_enc_set_ffe,
+};
+
+void hpo_dp_link_encoder42_construct(struct dcn31_hpo_dp_link_encoder *enc31,
+		struct dc_context *ctx,
+		uint32_t inst,
+		const struct dcn31_hpo_dp_link_encoder_registers *hpo_le_regs,
+		const struct dcn31_hpo_dp_link_encoder_shift *hpo_le_shift,
+		const struct dcn31_hpo_dp_link_encoder_mask *hpo_le_mask)
+{
+	enc31->base.ctx = ctx;
+
+	enc31->base.inst = inst;
+	enc31->base.funcs = &dcn42_hpo_dp_link_encoder_funcs;
+	enc31->base.hpd_source = HPD_SOURCEID_UNKNOWN;
+	enc31->base.transmitter = TRANSMITTER_UNKNOWN;
+
+	enc31->regs = hpo_le_regs;
+	enc31->hpo_le_shift = hpo_le_shift;
+	enc31->hpo_le_mask = hpo_le_mask;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.h b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.h
new file mode 100644
index 0000000000000..3e84d016507cf
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_dp_link_encoder.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+
+#ifndef __DAL_DCN42_HPO_DP_LINK_ENCODER_H__
+#define __DAL_DCN42_HPO_DP_LINK_ENCODER_H__
+
+#include "link_encoder.h"
+
+void hpo_dp_link_encoder42_construct(struct dcn31_hpo_dp_link_encoder *enc31,
+	struct dc_context *ctx,
+	uint32_t inst,
+	const struct dcn31_hpo_dp_link_encoder_registers *hpo_le_regs,
+	const struct dcn31_hpo_dp_link_encoder_shift *hpo_le_shift,
+	const struct dcn31_hpo_dp_link_encoder_mask *hpo_le_mask);
+
+#endif   // __DAL_DCN32_HPO_DP_LINK_ENCODER_H__
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/Makefile b/drivers/gpu/drm/amd/display/dc/hubbub/Makefile
index 66ca5a6a0415e..29c8e2651442b 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/Makefile
@@ -1,5 +1,5 @@
 
-# Copyright 2022 Advanced Micro Devices, Inc.
+# Copyright 2022-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -101,4 +101,12 @@ AMD_DAL_HUBBUB_DCN401 = $(addprefix $(AMDDALPATH)/dc/hubbub/dcn401/,$(HUBBUB_DCN
 AMD_DISPLAY_FILES += $(AMD_DAL_HUBBUB_DCN401)
 
 ###############################################################################
+# DCN42
+###############################################################################
+HUBBUB_DCN42 = dcn42_hubbub.o
+
+AMD_DAL_HUBBUB_DCN42 = $(addprefix $(AMDDALPATH)/dc/hubbub/dcn42/,$(HUBBUB_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_HUBBUB_DCN42)
+
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
index 990d3cd8e0505..ff9a1ae29474f 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Advanced Micro Devices, Inc.
+ * Copyright 2016-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -441,6 +441,8 @@ struct dcn_hubbub_registers {
 		type DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE;\
 		type DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE
 
+#define HUBBUB_REG_FIELD_LIST_DCN4_2(type) \
+	type URGENT_ZERO_SIZE_REQ_EN
 struct dcn_hubbub_shift {
 	DCN_HUBBUB_REG_FIELD_LIST(uint8_t);
 	HUBBUB_STUTTER_REG_FIELD_LIST(uint8_t);
@@ -449,6 +451,7 @@ struct dcn_hubbub_shift {
 	HUBBUB_REG_FIELD_LIST_DCN32(uint8_t);
 	HUBBUB_REG_FIELD_LIST_DCN35(uint8_t);
 	HUBBUB_REG_FIELD_LIST_DCN4_01(uint8_t);
+	HUBBUB_REG_FIELD_LIST_DCN4_2(uint8_t);
 
 };
 
@@ -460,6 +463,7 @@ struct dcn_hubbub_mask {
 	HUBBUB_REG_FIELD_LIST_DCN32(uint32_t);
 	HUBBUB_REG_FIELD_LIST_DCN35(uint32_t);
 	HUBBUB_REG_FIELD_LIST_DCN4_01(uint32_t);
+	HUBBUB_REG_FIELD_LIST_DCN4_2(uint8_t);
 
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
index 941dce439e979..b0a4b68cf359a 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
@@ -1151,8 +1151,6 @@ void dcn401_program_compbuf_segments(struct hubbub *hubbub, unsigned compbuf_siz
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
 
-	unsigned int cur_compbuf_size_seg = 0;
-
 	if (safe_to_increase || compbuf_size_seg <= hubbub2->compbuf_size_segments) {
 		if (compbuf_size_seg > hubbub2->compbuf_size_segments) {
 			REG_WAIT(DCHUBBUB_DET0_CTRL, DET0_SIZE_CURRENT, hubbub2->det0_size, 1, 100);
@@ -1165,8 +1163,6 @@ void dcn401_program_compbuf_segments(struct hubbub *hubbub, unsigned compbuf_siz
 				+ hubbub2->det3_size + compbuf_size_seg <= hubbub2->crb_size_segs);
 		REG_UPDATE(DCHUBBUB_COMPBUF_CTRL, COMPBUF_SIZE, compbuf_size_seg);
 		hubbub2->compbuf_size_segments = compbuf_size_seg;
-
-		ASSERT(REG_GET(DCHUBBUB_COMPBUF_CTRL, CONFIG_ERROR, &cur_compbuf_size_seg) && !cur_compbuf_size_seg);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
new file mode 100644
index 0000000000000..d6e6fbaa041b3
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn32/dcn32_hubbub.h"
+#include "dcn35/dcn35_hubbub.h"
+#include "dcn42/dcn42_hubbub.h"
+#include "dm_services.h"
+#include "reg_helper.h"
+
+#define DCN42_CRB_SEGMENT_SIZE_KB 64
+
+#define CTX \
+	hubbub2->base.ctx
+#define DC_LOGGER \
+	hubbub2->base.ctx->logger
+#define REG(reg)\
+	hubbub2->regs->reg
+
+#undef FN
+#define FN(reg_name, field_name) \
+	hubbub2->shifts->field_name, hubbub2->masks->field_name
+
+static bool hubbub42_program_urgent_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		bool safe_to_lower)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	bool wm_pending = false;
+
+	/* Repeat for water mark set A, B, C and D. */
+	/* clock state A */
+	if (safe_to_lower || watermarks->dcn4x.a.urgent > hubbub2->watermarks.dcn4x.a.urgent) {
+		hubbub2->watermarks.dcn4x.a.urgent = watermarks->dcn4x.a.urgent;
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, watermarks->dcn4x.a.urgent);
+	} else if (watermarks->dcn4x.a.urgent < hubbub2->watermarks.dcn4x.a.urgent)
+		wm_pending = true;
+
+	/* determine the transfer time for a quantity of data for a particular requestor.*/
+	if (safe_to_lower || watermarks->dcn4x.a.frac_urg_bw_flip > hubbub2->watermarks.dcn4x.a.frac_urg_bw_flip) {
+		hubbub2->watermarks.dcn4x.a.frac_urg_bw_flip = watermarks->dcn4x.a.frac_urg_bw_flip;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A, watermarks->dcn4x.a.frac_urg_bw_flip);
+	} else if (watermarks->dcn4x.a.frac_urg_bw_flip < hubbub2->watermarks.dcn4x.a.frac_urg_bw_flip)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.a.frac_urg_bw_nom > hubbub2->watermarks.dcn4x.a.frac_urg_bw_nom) {
+		hubbub2->watermarks.dcn4x.a.frac_urg_bw_nom = watermarks->dcn4x.a.frac_urg_bw_nom;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_NOM_A, watermarks->dcn4x.a.frac_urg_bw_nom);
+	} else if (watermarks->dcn4x.a.frac_urg_bw_nom < hubbub2->watermarks.dcn4x.a.frac_urg_bw_nom)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.a.refcyc_per_trip_to_mem > hubbub2->watermarks.dcn4x.a.refcyc_per_trip_to_mem) {
+		hubbub2->watermarks.dcn4x.a.refcyc_per_trip_to_mem = watermarks->dcn4x.a.refcyc_per_trip_to_mem;
+		REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, 0,
+				DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, watermarks->dcn4x.a.refcyc_per_trip_to_mem);
+	} else if (watermarks->dcn4x.a.refcyc_per_trip_to_mem < hubbub2->watermarks.dcn4x.a.refcyc_per_trip_to_mem)
+		wm_pending = true;
+
+	/* clock state B */
+	if (safe_to_lower || watermarks->dcn4x.b.urgent > hubbub2->watermarks.dcn4x.b.urgent) {
+		hubbub2->watermarks.dcn4x.b.urgent = watermarks->dcn4x.b.urgent;
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, watermarks->dcn4x.b.urgent);
+	} else if (watermarks->dcn4x.b.urgent < hubbub2->watermarks.dcn4x.b.urgent)
+		wm_pending = true;
+
+	/* determine the transfer time for a quantity of data for a particular requestor.*/
+	if (safe_to_lower || watermarks->dcn4x.b.frac_urg_bw_flip > hubbub2->watermarks.dcn4x.b.frac_urg_bw_flip) {
+		hubbub2->watermarks.dcn4x.b.frac_urg_bw_flip = watermarks->dcn4x.b.frac_urg_bw_flip;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B, watermarks->dcn4x.b.frac_urg_bw_flip);
+	} else if (watermarks->dcn4x.b.frac_urg_bw_flip < hubbub2->watermarks.dcn4x.b.frac_urg_bw_flip)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.b.frac_urg_bw_nom > hubbub2->watermarks.dcn4x.b.frac_urg_bw_nom) {
+		hubbub2->watermarks.dcn4x.b.frac_urg_bw_nom = watermarks->dcn4x.b.frac_urg_bw_nom;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_NOM_B, watermarks->dcn4x.b.frac_urg_bw_nom);
+	} else if (watermarks->dcn4x.b.frac_urg_bw_nom < hubbub2->watermarks.dcn4x.b.frac_urg_bw_nom)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.b.refcyc_per_trip_to_mem > hubbub2->watermarks.dcn4x.b.refcyc_per_trip_to_mem) {
+		hubbub2->watermarks.dcn4x.b.refcyc_per_trip_to_mem = watermarks->dcn4x.b.refcyc_per_trip_to_mem;
+		REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, 0,
+				DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, watermarks->dcn4x.b.refcyc_per_trip_to_mem);
+	} else if (watermarks->dcn4x.b.refcyc_per_trip_to_mem < hubbub2->watermarks.dcn4x.b.refcyc_per_trip_to_mem)
+		wm_pending = true;
+
+	/* clock state C */
+	if (safe_to_lower || watermarks->dcn4x.c.urgent > hubbub2->watermarks.dcn4x.c.urgent) {
+		hubbub2->watermarks.dcn4x.c.urgent = watermarks->dcn4x.c.urgent;
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, watermarks->dcn4x.c.urgent);
+	} else if (watermarks->dcn4x.c.urgent < hubbub2->watermarks.dcn4x.c.urgent)
+		wm_pending = true;
+
+	/* determine the transfer time for a quantity of data for a particular requestor.*/
+	if (safe_to_lower || watermarks->dcn4x.c.frac_urg_bw_flip > hubbub2->watermarks.dcn4x.c.frac_urg_bw_flip) {
+		hubbub2->watermarks.dcn4x.c.frac_urg_bw_flip = watermarks->dcn4x.c.frac_urg_bw_flip;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C, watermarks->dcn4x.c.frac_urg_bw_flip);
+	} else if (watermarks->dcn4x.c.frac_urg_bw_flip < hubbub2->watermarks.dcn4x.c.frac_urg_bw_flip)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.c.frac_urg_bw_nom > hubbub2->watermarks.dcn4x.c.frac_urg_bw_nom) {
+		hubbub2->watermarks.dcn4x.c.frac_urg_bw_nom = watermarks->dcn4x.c.frac_urg_bw_nom;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_NOM_C, watermarks->dcn4x.c.frac_urg_bw_nom);
+	} else if (watermarks->dcn4x.c.frac_urg_bw_nom < hubbub2->watermarks.dcn4x.c.frac_urg_bw_nom)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.c.refcyc_per_trip_to_mem > hubbub2->watermarks.dcn4x.c.refcyc_per_trip_to_mem) {
+		hubbub2->watermarks.dcn4x.c.refcyc_per_trip_to_mem = watermarks->dcn4x.c.refcyc_per_trip_to_mem;
+		REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, 0,
+				DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, watermarks->dcn4x.c.refcyc_per_trip_to_mem);
+	} else if (watermarks->dcn4x.c.refcyc_per_trip_to_mem < hubbub2->watermarks.dcn4x.c.refcyc_per_trip_to_mem)
+		wm_pending = true;
+
+	/* clock state D */
+	if (safe_to_lower || watermarks->dcn4x.d.urgent > hubbub2->watermarks.dcn4x.d.urgent) {
+		hubbub2->watermarks.dcn4x.d.urgent = watermarks->dcn4x.d.urgent;
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, watermarks->dcn4x.d.urgent);
+	} else if (watermarks->dcn4x.d.urgent < hubbub2->watermarks.dcn4x.d.urgent)
+		wm_pending = true;
+
+	/* determine the transfer time for a quantity of data for a particular requestor.*/
+	if (safe_to_lower || watermarks->dcn4x.d.frac_urg_bw_flip > hubbub2->watermarks.dcn4x.d.frac_urg_bw_flip) {
+		hubbub2->watermarks.dcn4x.d.frac_urg_bw_flip = watermarks->dcn4x.d.frac_urg_bw_flip;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D, watermarks->dcn4x.d.frac_urg_bw_flip);
+	} else if (watermarks->dcn4x.d.frac_urg_bw_flip < hubbub2->watermarks.dcn4x.d.frac_urg_bw_flip)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.d.frac_urg_bw_nom > hubbub2->watermarks.dcn4x.d.frac_urg_bw_nom) {
+		hubbub2->watermarks.dcn4x.d.frac_urg_bw_nom = watermarks->dcn4x.d.frac_urg_bw_nom;
+		REG_SET(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D, 0,
+				DCHUBBUB_ARB_FRAC_URG_BW_NOM_D, watermarks->dcn4x.d.frac_urg_bw_nom);
+	} else if (watermarks->dcn4x.d.frac_urg_bw_nom < hubbub2->watermarks.dcn4x.d.frac_urg_bw_nom)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.d.refcyc_per_trip_to_mem > hubbub2->watermarks.dcn4x.d.refcyc_per_trip_to_mem) {
+		hubbub2->watermarks.dcn4x.d.refcyc_per_trip_to_mem = watermarks->dcn4x.d.refcyc_per_trip_to_mem;
+		REG_SET(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, 0,
+				DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, watermarks->dcn4x.d.refcyc_per_trip_to_mem);
+	} else if (watermarks->dcn4x.d.refcyc_per_trip_to_mem < hubbub2->watermarks.dcn4x.d.refcyc_per_trip_to_mem)
+		wm_pending = true;
+
+	return wm_pending;
+}
+
+static bool hubbub42_program_stutter_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		bool safe_to_lower)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	bool wm_pending = false;
+
+	/* clock state A */
+	if (safe_to_lower || watermarks->dcn4x.a.sr_enter > hubbub2->watermarks.dcn4x.a.sr_enter) {
+		hubbub2->watermarks.dcn4x.a.sr_enter =	watermarks->dcn4x.a.sr_enter;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, watermarks->dcn4x.a.sr_enter);
+	} else if (watermarks->dcn4x.a.sr_enter < hubbub2->watermarks.dcn4x.a.sr_enter)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.a.sr_exit > hubbub2->watermarks.dcn4x.a.sr_exit) {
+		hubbub2->watermarks.dcn4x.a.sr_exit = watermarks->dcn4x.a.sr_exit;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, watermarks->dcn4x.a.sr_exit);
+	} else if (watermarks->dcn4x.a.sr_exit < hubbub2->watermarks.dcn4x.a.sr_exit)
+		wm_pending = true;
+
+	/* clock state B */
+	if (safe_to_lower || watermarks->dcn4x.b.sr_enter > hubbub2->watermarks.dcn4x.b.sr_enter) {
+		hubbub2->watermarks.dcn4x.b.sr_enter =	watermarks->dcn4x.b.sr_enter;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, watermarks->dcn4x.b.sr_enter);
+	} else if (watermarks->dcn4x.b.sr_enter < hubbub2->watermarks.dcn4x.b.sr_enter)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.b.sr_exit > hubbub2->watermarks.dcn4x.b.sr_exit) {
+		hubbub2->watermarks.dcn4x.b.sr_exit = watermarks->dcn4x.b.sr_exit;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, watermarks->dcn4x.b.sr_exit);
+	} else if (watermarks->dcn4x.b.sr_exit < hubbub2->watermarks.dcn4x.b.sr_exit)
+		wm_pending = true;
+
+	/* clock state C */
+	if (safe_to_lower || watermarks->dcn4x.c.sr_enter > hubbub2->watermarks.dcn4x.c.sr_enter) {
+		hubbub2->watermarks.dcn4x.c.sr_enter =	watermarks->dcn4x.c.sr_enter;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, watermarks->dcn4x.c.sr_enter);
+	} else if (watermarks->dcn4x.c.sr_enter < hubbub2->watermarks.dcn4x.c.sr_enter)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.c.sr_exit > hubbub2->watermarks.dcn4x.c.sr_exit) {
+		hubbub2->watermarks.dcn4x.c.sr_exit = watermarks->dcn4x.c.sr_exit;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, watermarks->dcn4x.c.sr_exit);
+	} else if (watermarks->dcn4x.c.sr_exit < hubbub2->watermarks.dcn4x.c.sr_exit)
+		wm_pending = true;
+
+	/* clock state D */
+	if (safe_to_lower || watermarks->dcn4x.d.sr_enter > hubbub2->watermarks.dcn4x.d.sr_enter) {
+		hubbub2->watermarks.dcn4x.d.sr_enter =	watermarks->dcn4x.d.sr_enter;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, watermarks->dcn4x.d.sr_enter);
+	} else if (watermarks->dcn4x.d.sr_enter < hubbub2->watermarks.dcn4x.d.sr_enter)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.d.sr_exit > hubbub2->watermarks.dcn4x.d.sr_exit) {
+		hubbub2->watermarks.dcn4x.d.sr_exit = watermarks->dcn4x.d.sr_exit;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, watermarks->dcn4x.d.sr_exit);
+	} else if (watermarks->dcn4x.d.sr_exit < hubbub2->watermarks.dcn4x.d.sr_exit)
+		wm_pending = true;
+
+	return wm_pending;
+}
+
+static bool hubbub42_program_pstate_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		bool safe_to_lower)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	bool wm_pending = false;
+
+	/* Section for UCLK_PSTATE_CHANGE_WATERMARKS */
+	/* UCLK state A */
+	if (safe_to_lower || watermarks->dcn4x.a.uclk_pstate > hubbub2->watermarks.dcn4x.a.uclk_pstate) {
+		hubbub2->watermarks.dcn4x.a.uclk_pstate = watermarks->dcn4x.a.uclk_pstate;
+		REG_SET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, 0,
+				DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, watermarks->dcn4x.a.uclk_pstate);
+	} else if (watermarks->dcn4x.a.uclk_pstate < hubbub2->watermarks.dcn4x.a.uclk_pstate)
+		wm_pending = true;
+
+	/* UCLK state B */
+	if (safe_to_lower || watermarks->dcn4x.b.uclk_pstate > hubbub2->watermarks.dcn4x.b.uclk_pstate) {
+		hubbub2->watermarks.dcn4x.b.uclk_pstate = watermarks->dcn4x.b.uclk_pstate;
+		REG_SET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, 0,
+				DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, watermarks->dcn4x.b.uclk_pstate);
+	} else if (watermarks->dcn4x.b.uclk_pstate < hubbub2->watermarks.dcn4x.b.uclk_pstate)
+		wm_pending = true;
+
+	/* UCLK state C */
+	if (safe_to_lower || watermarks->dcn4x.c.uclk_pstate > hubbub2->watermarks.dcn4x.c.uclk_pstate) {
+		hubbub2->watermarks.dcn4x.c.uclk_pstate = watermarks->dcn4x.c.uclk_pstate;
+		REG_SET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, 0,
+				DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, watermarks->dcn4x.c.uclk_pstate);
+	} else if (watermarks->dcn4x.c.uclk_pstate < hubbub2->watermarks.dcn4x.c.uclk_pstate)
+		wm_pending = true;
+
+	/* UCLK state D */
+	if (safe_to_lower || watermarks->dcn4x.d.uclk_pstate > hubbub2->watermarks.dcn4x.d.uclk_pstate) {
+		hubbub2->watermarks.dcn4x.d.uclk_pstate = watermarks->dcn4x.d.uclk_pstate;
+		REG_SET(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, 0,
+				DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, watermarks->dcn4x.d.uclk_pstate);
+	} else if (watermarks->dcn4x.d.uclk_pstate < hubbub2->watermarks.dcn4x.d.uclk_pstate)
+		wm_pending = true;
+
+	/* Section for FCLK_PSTATE_CHANGE_WATERMARKS */
+	/* FCLK state A */
+	if (safe_to_lower || watermarks->dcn4x.a.fclk_pstate > hubbub2->watermarks.dcn4x.a.fclk_pstate) {
+		hubbub2->watermarks.dcn4x.a.fclk_pstate = watermarks->dcn4x.a.fclk_pstate;
+		REG_SET(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A, 0,
+				DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A, watermarks->dcn4x.a.fclk_pstate);
+	} else if (watermarks->dcn4x.a.fclk_pstate < hubbub2->watermarks.dcn4x.a.fclk_pstate)
+		wm_pending = true;
+
+	/* FCLK state B */
+	if (safe_to_lower || watermarks->dcn4x.b.fclk_pstate > hubbub2->watermarks.dcn4x.b.fclk_pstate) {
+		hubbub2->watermarks.dcn4x.b.fclk_pstate = watermarks->dcn4x.b.fclk_pstate;
+		REG_SET(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B, 0,
+				DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B, watermarks->dcn4x.b.fclk_pstate);
+	} else if (watermarks->dcn4x.b.fclk_pstate < hubbub2->watermarks.dcn4x.b.fclk_pstate)
+		wm_pending = true;
+
+	/* FCLK state C */
+	if (safe_to_lower || watermarks->dcn4x.c.fclk_pstate > hubbub2->watermarks.dcn4x.c.fclk_pstate) {
+		hubbub2->watermarks.dcn4x.c.fclk_pstate = watermarks->dcn4x.c.fclk_pstate;
+		REG_SET(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C, 0,
+				DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C, watermarks->dcn4x.c.fclk_pstate);
+	} else if (watermarks->dcn4x.c.fclk_pstate < hubbub2->watermarks.dcn4x.c.fclk_pstate)
+		wm_pending = true;
+
+	/* FCLK state D */
+	if (safe_to_lower || watermarks->dcn4x.d.fclk_pstate > hubbub2->watermarks.dcn4x.d.fclk_pstate) {
+		hubbub2->watermarks.dcn4x.d.fclk_pstate = watermarks->dcn4x.d.fclk_pstate;
+		REG_SET(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, 0,
+				DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, watermarks->dcn4x.d.fclk_pstate);
+	} else if (watermarks->dcn4x.d.fclk_pstate < hubbub2->watermarks.dcn4x.d.fclk_pstate)
+		wm_pending = true;
+
+	return wm_pending;
+}
+
+static bool hubbub42_program_usr_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		bool safe_to_lower)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	bool wm_pending = false;
+
+	/* clock state A */
+	if (safe_to_lower || watermarks->dcn4x.a.usr > hubbub2->watermarks.dcn4x.a.usr) {
+		hubbub2->watermarks.dcn4x.a.usr = watermarks->dcn4x.a.usr;
+		REG_SET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, 0,
+				DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, watermarks->dcn4x.a.usr);
+	} else if (watermarks->dcn4x.a.usr < hubbub2->watermarks.dcn4x.a.usr)
+		wm_pending = true;
+
+	/* clock state B */
+	if (safe_to_lower || watermarks->dcn4x.b.usr > hubbub2->watermarks.dcn4x.b.usr) {
+		hubbub2->watermarks.dcn4x.b.usr = watermarks->dcn4x.b.usr;
+		REG_SET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, 0,
+				DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, watermarks->dcn4x.b.usr);
+	} else if (watermarks->dcn4x.b.usr < hubbub2->watermarks.dcn4x.b.usr)
+		wm_pending = true;
+
+	/* clock state C */
+	if (safe_to_lower || watermarks->dcn4x.c.usr > hubbub2->watermarks.dcn4x.c.usr) {
+		hubbub2->watermarks.dcn4x.c.usr = watermarks->dcn4x.c.usr;
+		REG_SET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, 0,
+				DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, watermarks->dcn4x.c.usr);
+	} else if (watermarks->dcn4x.c.usr < hubbub2->watermarks.dcn4x.c.usr)
+		wm_pending = true;
+
+	/* clock state D */
+	if (safe_to_lower || watermarks->dcn4x.d.usr > hubbub2->watermarks.dcn4x.d.usr) {
+		hubbub2->watermarks.dcn4x.d.usr = watermarks->dcn4x.d.usr;
+		REG_SET(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, 0,
+				DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, watermarks->dcn4x.d.usr);
+	} else if (watermarks->dcn4x.d.usr < hubbub2->watermarks.dcn4x.d.usr)
+		wm_pending = true;
+
+	return wm_pending;
+}
+
+static bool hubbub42_program_stutter_z8_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		bool safe_to_lower)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	bool wm_pending = false;
+
+	/* clock state A */
+	if (safe_to_lower || watermarks->dcn4x.a.sr_enter_z8 > hubbub2->watermarks.dcn4x.a.sr_enter_z8) {
+		hubbub2->watermarks.dcn4x.a.sr_enter_z8 = watermarks->dcn4x.a.sr_enter_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, watermarks->dcn4x.a.sr_enter_z8);
+	} else if (watermarks->dcn4x.a.sr_enter_z8 < hubbub2->watermarks.dcn4x.a.sr_enter_z8)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.a.sr_exit_z8 > hubbub2->watermarks.dcn4x.a.sr_exit_z8) {
+		hubbub2->watermarks.dcn4x.a.sr_exit_z8 = watermarks->dcn4x.a.sr_exit_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, watermarks->dcn4x.a.sr_exit_z8);
+	} else if (watermarks->dcn4x.a.sr_exit_z8 < hubbub2->watermarks.dcn4x.a.sr_exit_z8)
+		wm_pending = true;
+
+	/* clock state B */
+	if (safe_to_lower || watermarks->dcn4x.b.sr_enter_z8 > hubbub2->watermarks.dcn4x.b.sr_enter_z8) {
+		hubbub2->watermarks.dcn4x.b.sr_enter_z8 = watermarks->dcn4x.b.sr_enter_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, watermarks->dcn4x.b.sr_enter_z8);
+	} else if (watermarks->dcn4x.b.sr_enter_z8 < hubbub2->watermarks.dcn4x.b.sr_enter_z8)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.b.sr_exit_z8 > hubbub2->watermarks.dcn4x.b.sr_exit_z8) {
+		hubbub2->watermarks.dcn4x.b.sr_exit_z8 = watermarks->dcn4x.b.sr_exit_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, watermarks->dcn4x.b.sr_exit_z8);
+	} else if (watermarks->dcn4x.b.sr_exit_z8 < hubbub2->watermarks.dcn4x.b.sr_exit_z8)
+		wm_pending = true;
+
+	/* clock state C */
+	if (safe_to_lower || watermarks->dcn4x.c.sr_enter_z8 > hubbub2->watermarks.dcn4x.c.sr_enter_z8) {
+		hubbub2->watermarks.dcn4x.c.sr_enter_z8 = watermarks->dcn4x.c.sr_enter_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, watermarks->dcn4x.c.sr_enter_z8);
+	} else if (watermarks->dcn4x.c.sr_enter_z8 < hubbub2->watermarks.dcn4x.c.sr_enter_z8)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.c.sr_exit_z8 > hubbub2->watermarks.dcn4x.c.sr_exit_z8) {
+		hubbub2->watermarks.dcn4x.c.sr_exit_z8 = watermarks->dcn4x.c.sr_exit_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, watermarks->dcn4x.c.sr_exit_z8);
+	} else if (watermarks->dcn4x.c.sr_exit_z8 < hubbub2->watermarks.dcn4x.c.sr_exit_z8)
+		wm_pending = true;
+
+	/* clock state D */
+	if (safe_to_lower || watermarks->dcn4x.d.sr_enter_z8 > hubbub2->watermarks.dcn4x.d.sr_enter_z8) {
+		hubbub2->watermarks.dcn4x.d.sr_enter_z8 = watermarks->dcn4x.d.sr_enter_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, 0,
+				DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, watermarks->dcn4x.d.sr_enter_z8);
+	} else if (watermarks->dcn4x.d.sr_enter_z8 < hubbub2->watermarks.dcn4x.d.sr_enter_z8)
+		wm_pending = true;
+
+	if (safe_to_lower || watermarks->dcn4x.d.sr_exit_z8 > hubbub2->watermarks.dcn4x.d.sr_exit_z8) {
+		hubbub2->watermarks.dcn4x.d.sr_exit_z8 = watermarks->dcn4x.d.sr_exit_z8;
+		REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, 0,
+				DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, watermarks->dcn4x.d.sr_exit_z8);
+	} else if (watermarks->dcn4x.d.sr_exit_z8 < hubbub2->watermarks.dcn4x.d.sr_exit_z8)
+		wm_pending = true;
+
+	return wm_pending;
+}
+
+static void hubbub42_allow_self_refresh_control(struct hubbub *hubbub, bool allow)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+
+	/*
+	 * DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE = 1 means do not allow stutter
+	 * DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE = 0 means allow stutter
+	 */
+
+	REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, 0,
+			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, !allow);
+
+	if (!allow && hubbub->ctx->dc->debug.disable_stutter) {/*controlled by registry key*/
+		REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+			DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE, 0,
+			DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE, 1);
+		REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+			DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 0,
+			DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 1);
+	}
+}
+static void hubbub42_set_sdp_control(struct hubbub *hubbub, bool dc_control)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+
+	REG_UPDATE(DCHUBBUB_SDPIF_CFG0,
+			SDPIF_PORT_CONTROL, dc_control);
+}
+
+static bool hubbub42_program_watermarks(
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower)
+{
+	bool wm_pending = false;
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+
+	if (!safe_to_lower && hubbub->ctx->dc->debug.disable_stutter_for_wm_program) {
+		/* before raising watermarks, SDP control give to DF, stutter must be disabled */
+		wm_pending = true;
+		hubbub42_set_sdp_control(hubbub, false);
+		hubbub42_allow_self_refresh_control(hubbub, false);
+	}
+	if (hubbub42_program_urgent_watermarks(hubbub, watermarks, safe_to_lower))
+		wm_pending = true;
+
+	if (hubbub42_program_stutter_watermarks(hubbub, watermarks, safe_to_lower))
+		wm_pending = true;
+
+	if (hubbub42_program_pstate_watermarks(hubbub, watermarks, safe_to_lower))
+		wm_pending = true;
+
+	if (hubbub42_program_usr_watermarks(hubbub, watermarks, safe_to_lower))
+		wm_pending = true;
+
+	if (hubbub42_program_stutter_z8_watermarks(hubbub, watermarks, safe_to_lower))
+		wm_pending = true;
+
+	REG_SET(DCHUBBUB_ARB_SAT_LEVEL, 0,
+			DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
+	REG_UPDATE_2(DCHUBBUB_ARB_DF_REQ_OUTSTAND,
+			DCHUBBUB_ARB_MIN_REQ_OUTSTAND, 0xFF,
+			DCHUBBUB_ARB_MIN_REQ_OUTSTAND_COMMIT_THRESHOLD, 0xA);/*hw delta*/
+	REG_UPDATE(DCHUBBUB_ARB_HOSTVM_CNTL, DCHUBBUB_ARB_MAX_QOS_COMMIT_THRESHOLD, 0xF);
+
+	if (safe_to_lower || hubbub->ctx->dc->debug.disable_stutter)
+		hubbub42_allow_self_refresh_control(hubbub, !hubbub->ctx->dc->debug.disable_stutter);
+	if (safe_to_lower && hubbub->ctx->dc->debug.disable_stutter_for_wm_program) {
+		hubbub42_set_sdp_control(hubbub, true);
+	}
+	hubbub32_force_usr_retraining_allow(hubbub, hubbub->ctx->dc->debug.force_usr_allow);
+
+	return wm_pending;
+}
+
+static const struct hubbub_funcs hubbub42_funcs = {
+	.update_dchub = hubbub2_update_dchub,
+	.init_dchub_sys_ctx = hubbub31_init_dchub_sys_ctx,
+	.init_vm_ctx = hubbub2_init_vm_ctx,
+	.dcc_support_swizzle = hubbub3_dcc_support_swizzle,
+	.dcc_support_pixel_format = hubbub2_dcc_support_pixel_format,
+	.get_dcc_compression_cap = hubbub3_get_dcc_compression_cap,
+	.wm_read_state = hubbub35_wm_read_state,
+	.get_dchub_ref_freq = hubbub35_get_dchub_ref_freq,
+	.program_watermarks = hubbub42_program_watermarks,
+	.allow_self_refresh_control = hubbub42_allow_self_refresh_control,
+	.is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
+	.force_wm_propagate_to_pipes = hubbub32_force_wm_propagate_to_pipes,
+	.force_pstate_change_control = hubbub3_force_pstate_change_control,
+	.init_watermarks = hubbub35_init_watermarks,
+	.program_det_size = dcn32_program_det_size,
+	.program_compbuf_size = dcn35_program_compbuf_size,
+	.init_crb = dcn35_init_crb,
+	.hubbub_read_state = hubbub2_read_state,
+	.force_usr_retraining_allow = hubbub32_force_usr_retraining_allow,
+	.dchubbub_init = hubbub35_init,
+	.dchvm_init = dcn35_dchvm_init,
+};
+
+void hubbub42_construct(struct dcn20_hubbub *hubbub2,
+	struct dc_context *ctx,
+	const struct dcn_hubbub_registers *hubbub_regs,
+	const struct dcn_hubbub_shift *hubbub_shift,
+	const struct dcn_hubbub_mask *hubbub_mask,
+	int det_size_kb,
+	int pixel_chunk_size_kb,
+	int config_return_buffer_size_kb)
+{
+	hubbub2->base.ctx = ctx;
+	hubbub2->base.funcs = &hubbub42_funcs;
+	hubbub2->regs = hubbub_regs;
+	hubbub2->shifts = hubbub_shift;
+	hubbub2->masks = hubbub_mask;
+
+	hubbub2->detile_buf_size = det_size_kb * 1024;
+	hubbub2->pixel_chunk_size = pixel_chunk_size_kb * 1024;
+	hubbub2->crb_size_segs = config_return_buffer_size_kb / DCN42_CRB_SEGMENT_SIZE_KB;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.h
new file mode 100644
index 0000000000000..926ddc156a2b3
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.h
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_HUBBUB_DCN42_H__
+#define __DC_HUBBUB_DCN42_H__
+
+#include "dcn32/dcn32_hubbub.h"
+
+#define DCN42_CRB_SIZE_KB 1792
+#define DCN42_DEFAULT_DET_SIZE 320
+#define DCN42_CRB_SEGMENT_SIZE_KB 64
+
+#define HUBBUB_REG_LIST_DCN42(id)\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),\
+	SR(DCHUBBUB_ARB_DRAM_STATE_CNTL),\
+	SR(DCHUBBUB_ARB_SAT_LEVEL),\
+	SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND),\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_SOFT_RESET),\
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DCN_VM_FB_LOCATION_BASE),\
+	SR(DCN_VM_FB_LOCATION_TOP),\
+	SR(DCN_VM_FB_OFFSET),\
+	SR(DCN_VM_AGP_BOT),\
+	SR(DCN_VM_AGP_TOP),\
+	SR(DCN_VM_AGP_BASE),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C),\
+	SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D),\
+	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A),\
+	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B),\
+	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C),\
+	SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D),\
+	SR(DCHUBBUB_DET0_CTRL),\
+	SR(DCHUBBUB_DET1_CTRL),\
+	SR(DCHUBBUB_DET2_CTRL),\
+	SR(DCHUBBUB_DET3_CTRL),\
+	SR(DCHUBBUB_COMPBUF_CTRL),\
+	SR(COMPBUF_RESERVED_SPACE),\
+	SR(DCHUBBUB_DEBUG_CTRL_0),\
+	SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL),\
+	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D),\
+	SR(DCN_VM_FAULT_ADDR_MSB),\
+	SR(DCN_VM_FAULT_ADDR_LSB),\
+	SR(DCN_VM_FAULT_CNTL),\
+	SR(DCN_VM_FAULT_STATUS),\
+	SR(SDPIF_REQUEST_RATE_LIMIT),\
+	SR(DCHUBBUB_CLOCK_CNTL),\
+	SR(DCHUBBUB_SDPIF_CFG0),\
+	SR(DCHUBBUB_SDPIF_CFG1),\
+	SR(DCHUBBUB_MEM_PWR_MODE_CTRL),\
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL),\
+	SR(DCHVM_CTRL0),\
+	SR(DCHVM_MEM_CTRL),\
+	SR(DCHVM_CLK_CTRL),\
+	SR(DCHVM_RIOMMU_CTRL0),\
+	SR(DCHVM_RIOMMU_STAT0),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D),\
+	SR(DCHUBBUB_ARB_QOS_FORCE)
+
+#define HUBBUB_MASK_SH_LIST_DCN4_2(mask_sh)\
+	HUBBUB_MASK_SH_LIST_DCN32(mask_sh), \
+	HUBBUB_SF(DCHVM_CTRL0, HOSTVM_INIT_REQ, mask_sh),\
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_PWR_REQ_DIS, mask_sh),\
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_FORCE_REQ, mask_sh),\
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_POWER_STATUS, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DISPCLK_R_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DISPCLK_G_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DCFCLK_R_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DCFCLK_G_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, TR_REQ_REQCLKREQ_MODE, mask_sh),\
+	HUBBUB_SF(DCHVM_CLK_CTRL, TW_RSP_COMPCLKREQ_MODE, mask_sh),\
+	HUBBUB_SF(DCHVM_RIOMMU_CTRL0, HOSTVM_PREFETCH_REQ, mask_sh),\
+	HUBBUB_SF(DCHVM_RIOMMU_CTRL0, HOSTVM_POWERSTATUS, mask_sh),\
+	HUBBUB_SF(DCHVM_RIOMMU_STAT0, RIOMMU_ACTIVE, mask_sh),\
+	HUBBUB_SF(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, COMPBUF_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, COMPBUF_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, CONFIG_ERROR, mask_sh),\
+	HUBBUB_SF(COMPBUF_RESERVED_SPACE, COMPBUF_RESERVED_SPACE_64B, mask_sh),\
+	HUBBUB_SF(COMPBUF_RESERVED_SPACE, COMPBUF_RESERVED_SPACE_ZS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CLOCK_CNTL, DCHUBBUB_FGCG_REP_DIS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MAX_REQ_OUTSTAND, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_CSTATE_DEEPSLEEP_LEGACY_MODE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_SOFT_RESET, DCHUBBUB_GLOBAL_SOFT_RESET, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_HOSTVM_CNTL, DCHUBBUB_ARB_MAX_QOS_COMMIT_THRESHOLD, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND_COMMIT_THRESHOLD, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_SAT_LEVEL, DCHUBBUB_ARB_SAT_LEVEL, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MAX_REQ_OUTSTAND, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HUBBUB_SF(DCN_VM_FB_LOCATION_BASE, FB_BASE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FB_LOCATION_TOP, FB_TOP, mask_sh), \
+	HUBBUB_SF(DCN_VM_FB_OFFSET, FB_OFFSET, mask_sh), \
+	HUBBUB_SF(DCN_VM_AGP_BOT, AGP_BOT, mask_sh), \
+	HUBBUB_SF(DCN_VM_AGP_TOP, AGP_TOP, mask_sh), \
+	HUBBUB_SF(DCN_VM_AGP_BASE, AGP_BASE, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A, DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B, DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A, DCHUBBUB_ARB_FRAC_URG_BW_NOM_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B, DCHUBBUB_ARB_FRAC_URG_BW_NOM_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C, DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C, DCHUBBUB_ARB_FRAC_URG_BW_NOM_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D, DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D, DCHUBBUB_ARB_FRAC_URG_BW_NOM_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_DEBUG_CTRL_0, DET_DEPTH, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET0_CTRL, DET0_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET0_CTRL, DET0_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET1_CTRL, DET1_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET1_CTRL, DET1_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET2_CTRL, DET2_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET2_CTRL, DET2_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET3_CTRL, DET3_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_DET3_CTRL, DET3_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, COMPBUF_SIZE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, COMPBUF_SIZE_CURRENT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_COMPBUF_CTRL, CONFIG_ERROR, mask_sh),\
+	HUBBUB_SF(COMPBUF_RESERVED_SPACE, COMPBUF_RESERVED_SPACE_64B, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_CNTL, DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_VALUE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_CNTL, DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_ENABLE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_CNTL, DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PSTATE_CHANGE_REQUEST, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_CNTL, DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PRE_CSTATE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A, DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B, DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C, DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C, mask_sh), \
+	HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_ADDR_MSB, DCN_VM_FAULT_ADDR_MSB, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_ADDR_LSB, DCN_VM_FAULT_ADDR_LSB, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_CNTL, DCN_VM_ERROR_STATUS_CLEAR, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_CNTL, DCN_VM_ERROR_STATUS_MODE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_CNTL, DCN_VM_ERROR_INTERRUPT_ENABLE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_CNTL, DCN_VM_RANGE_FAULT_DISABLE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_CNTL, DCN_VM_PRQ_FAULT_DISABLE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_STATUS, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_VMID, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_TABLE_LEVEL, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_PIPE, mask_sh), \
+	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_INTERRUPT_STATUS, mask_sh),\
+	HUBBUB_SF(SDPIF_REQUEST_RATE_LIMIT, SDPIF_REQUEST_RATE_LIMIT, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CLOCK_CNTL, DISPCLK_R_DCHUBBUB_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CLOCK_CNTL, DCFCLK_R_DCHUBBUB_GATE_DIS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_SDPIF_CFG0, SDPIF_PORT_CONTROL, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_SDPIF_CFG1, SDPIF_MAX_NUM_OUTSTANDING, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_MEM_PWR_MODE_CTRL, DET_MEM_PWR_LS_MODE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_TIMEOUT_DETECTION_CTRL1, DCHUBBUB_TIMEOUT_ERROR_STATUS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_TIMEOUT_DETECTION_CTRL1, DCHUBBUB_TIMEOUT_REQ_STALL_THRESHOLD, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_TIMEOUT_DETECTION_CTRL2, DCHUBBUB_TIMEOUT_PSTATE_STALL_THRESHOLD, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_TIMEOUT_DETECTION_CTRL2, DCHUBBUB_TIMEOUT_DETECTION_EN, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_TIMEOUT_DETECTION_CTRL2, DCHUBBUB_TIMEOUT_TIMER_RESET, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, ROB_OVERFLOW_STATUS, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, ROB_OVERFLOW_CLEAR, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, DCHUBBUB_HW_DEBUG, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, URGENT_ZERO_SIZE_REQ_EN, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, CSTATE_SWATH_CHK_GOOD_MODE, mask_sh)
+
+void hubbub42_construct(struct dcn20_hubbub *hubbub2,
+	struct dc_context *ctx,
+	const struct dcn_hubbub_registers *hubbub_regs,
+	const struct dcn_hubbub_shift *hubbub_shift,
+	const struct dcn_hubbub_mask *hubbub_mask,
+	int det_size_kb,
+	int pixel_chunk_size_kb,
+	int config_return_buffer_size_kb);
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/Makefile b/drivers/gpu/drm/amd/display/dc/hubp/Makefile
index a2d1128de7a18..e0799bb873e69 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hubp/Makefile
@@ -1,5 +1,5 @@
 
-# Copyright 2022 Advanced Micro Devices, Inc.
+# Copyright 2022-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -94,4 +94,12 @@ AMD_DAL_HUBP_DCN401 = $(addprefix $(AMDDALPATH)/dc/hubp/dcn401/,$(HUBP_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HUBP_DCN401)
 
+###############################################################################
+
+HUBP_DCN42 = dcn42_hubp.o
+
+AMD_DAL_HUBP_DCN42 = $(addprefix $(AMDDALPATH)/dc/hubp/dcn42/,$(HUBP_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_HUBP_DCN42)
+
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
index 7062e6653062a..80be394a8852d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-17 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -288,16 +288,22 @@
 	type MCACHEID_MALL_PREF_2H_P1;\
 	type HUBP_FGCG_REP_DIS
 
+#define DCN42_HUBP_REG_FIELD_VARIABLE_LIST(type) \
+	type HUBP_3DLUT_CROSSBAR_SEL_G;\
+	type HUBP_3DLUT_CROSSBAR_SEL_B;\
+	type HUBP_3DLUT_CROSSBAR_SEL_R
 struct dcn_hubp2_registers {
 	DCN401_HUBP_REG_COMMON_VARIABLE_LIST;
 };
 
 struct dcn_hubp2_shift {
 	DCN401_HUBP_REG_FIELD_VARIABLE_LIST(uint8_t);
+	DCN42_HUBP_REG_FIELD_VARIABLE_LIST(uint8_t);
 };
 
 struct dcn_hubp2_mask {
 	DCN401_HUBP_REG_FIELD_VARIABLE_LIST(uint32_t);
+	DCN42_HUBP_REG_FIELD_VARIABLE_LIST(uint32_t);
 };
 
 struct dcn20_hubp {
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
index c205500290ecd..263e0c4d34f66 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
@@ -41,12 +41,12 @@
 	hubp2->hubp_shift->field_name, hubp2->hubp_mask->field_name
 
 void hubp401_program_3dlut_fl_addr(struct hubp *hubp,
-	const struct dc_plane_address address)
+	const struct dc_plane_address *address)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address.lut3d.addr.high_part);
-	REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address.lut3d.addr.low_part);
+	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address->lut3d.addr.high_part);
+	REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address->lut3d.addr.low_part);
 }
 
 void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group)
@@ -72,33 +72,46 @@ int hubp401_get_3dlut_fl_done(struct hubp *hubp)
 	return ret;
 }
 
-void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode)
+static void hubp401_get_3dlut_fl_xbar_map(
+		const enum dc_cm_lut_pixel_format format,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r)
 {
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-
-	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, addr_mode);
+	switch (format) {
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
+		/* BGRA */
+		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*bit_slice_cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		break;
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
+	default:
+		/* RGBA */
+		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		break;
+	}
 }
 
-void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width)
+void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
+		const enum dc_cm_lut_pixel_format format)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_WIDTH, width);
-}
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r = 0;
 
-void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits)
-{
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-
-	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_TMZ, protection_bits);
-}
-
-void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r)
-{
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	hubp401_get_3dlut_fl_xbar_map(format,
+			&bit_slice_y_g,
+			&bit_slice_cb_b,
+			&bit_slice_cr_r);
 
 	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
 			HUBP_3DLUT_CROSSBAR_SELECT_Y_G, bit_slice_y_g,
@@ -106,62 +119,122 @@ void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
 			HUBP_3DLUT_CROSSBAR_SELECT_CR_R, bit_slice_cr_r);
 }
 
-void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale)
+static enum hubp_3dlut_fl_width hubp401_get_3dlut_fl_width(
+		const enum dc_cm_lut_size size,
+		const enum dc_cm_lut_swizzle swizzle)
 {
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	enum hubp_3dlut_fl_width width = 0;
+
+	switch (size) {
+	case CM_LUT_SIZE_333333:
+		ASSERT(swizzle != CM_LUT_1D_PACKED_LINEAR);
+		width = hubp_3dlut_fl_width_33;
+		break;
+	case CM_LUT_SIZE_171717:
+		if (swizzle != CM_LUT_1D_PACKED_LINEAR) {
+			width = hubp_3dlut_fl_width_17;
+		} else {
+			width = hubp_3dlut_fl_width_17_transformed;
+		}
+		break;
+	default:
+		width = 0;
+		break;
+	}
 
-	REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, bias, HUBP0_3DLUT_FL_SCALE, scale);
+	return width;
 }
 
-void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode)
+static enum hubp_3dlut_fl_format hubp401_get_3dlut_fl_format(
+		const enum dc_cm_lut_pixel_format dc_format)
 {
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	enum hubp_3dlut_fl_format hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
 
-	REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_MODE, mode);
+	switch (dc_format) {
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
+		hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+		break;
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
+		hubp_format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
+		break;
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
+		hubp_format = hubp_3dlut_fl_format_float_fp1_5_10;
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	return hubp_format;
 }
 
-void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format)
+static enum hubp_3dlut_fl_addressing_mode hubp401_get_3dlut_fl_addr_mode(
+		const enum dc_cm_lut_swizzle swizzle)
 {
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	enum hubp_3dlut_fl_addressing_mode addr_mode;
+
+	switch (swizzle) {
+	case CM_LUT_1D_PACKED_LINEAR:
+		addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
+		break;
+	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
+	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
+	default:
+		addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	}
 
-	REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, format);
+	return addr_mode;
 }
 
-void hubp401_program_3dlut_fl_config(
-	struct hubp *hubp,
-	struct hubp_fl_3dlut_config *cfg)
+static enum hubp_3dlut_fl_mode hubp401_get_3dlut_fl_mode(
+		const enum dc_cm_lut_swizzle swizzle)
 {
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	enum hubp_3dlut_fl_mode mode;
+
+	switch (swizzle) {
+	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
+		mode = hubp_3dlut_fl_mode_native_1;
+		break;
+	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
+		mode = hubp_3dlut_fl_mode_native_2;
+		break;
+	case CM_LUT_1D_PACKED_LINEAR:
+		mode = hubp_3dlut_fl_mode_transform;
+		break;
+	default:
+		mode = hubp_3dlut_fl_mode_disable;
+		break;
+	}
 
-	uint32_t mpc_width = {(cfg->width == 17) ? 0 : 1};
-	uint32_t width = {cfg->width};
+	return mode;
+}
+
+void hubp401_program_3dlut_fl_config(struct hubp *hubp,
+		const struct dc_3dlut_dma *config)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	if (cfg->layout == DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR)
-		width = (cfg->width == 17) ? 4916 : 35940;
+	enum hubp_3dlut_fl_width width = hubp401_get_3dlut_fl_width(config->size, config->swizzle);
+	enum hubp_3dlut_fl_format format = hubp401_get_3dlut_fl_format(config->format);
+	enum hubp_3dlut_fl_addressing_mode addr_mode = hubp401_get_3dlut_fl_addr_mode(config->swizzle);
+	enum hubp_3dlut_fl_mode mode = hubp401_get_3dlut_fl_mode(config->swizzle);
 
 	REG_UPDATE_2(_3DLUT_FL_CONFIG,
-		HUBP0_3DLUT_FL_MODE, cfg->mode,
-		HUBP0_3DLUT_FL_FORMAT, cfg->format);
+			HUBP0_3DLUT_FL_MODE, mode,
+			HUBP0_3DLUT_FL_FORMAT, format);
 
 	REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE,
-		HUBP0_3DLUT_FL_BIAS, cfg->bias,
-		HUBP0_3DLUT_FL_SCALE, cfg->scale);
-
-	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH,
-		HUBP_3DLUT_ADDRESS_HIGH, cfg->address.lut3d.addr.high_part);
-	REG_UPDATE(HUBP_3DLUT_ADDRESS_LOW,
-		HUBP_3DLUT_ADDRESS_LOW, cfg->address.lut3d.addr.low_part);
-
-	//cross bar
-	REG_UPDATE_8(HUBP_3DLUT_CONTROL,
-		HUBP_3DLUT_MPC_WIDTH, mpc_width,
-		HUBP_3DLUT_WIDTH, width,
-		HUBP_3DLUT_CROSSBAR_SELECT_CR_R, cfg->crossbar_bit_slice_cr_r,
-		HUBP_3DLUT_CROSSBAR_SELECT_Y_G, cfg->crossbar_bit_slice_y_g,
-		HUBP_3DLUT_CROSSBAR_SELECT_CB_B, cfg->crossbar_bit_slice_cb_b,
-		HUBP_3DLUT_ADDRESSING_MODE, cfg->addr_mode,
-		HUBP_3DLUT_TMZ, cfg->protection_bits,
-		HUBP_3DLUT_ENABLE, cfg->enabled ? 1 : 0);
+			HUBP0_3DLUT_FL_BIAS, config->bias,
+			HUBP0_3DLUT_FL_SCALE, config->scale);
+
+	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
+			HUBP_3DLUT_WIDTH, width,
+			HUBP_3DLUT_ADDRESSING_MODE, addr_mode,
+			HUBP_3DLUT_TMZ, config->addr.tmz_surface);
 }
 
 void hubp401_update_mall_sel(struct hubp *hubp, uint32_t mall_sel, bool c_cursor)
@@ -739,9 +812,8 @@ void hubp401_cursor_set_position(
 	int x_pos_viewport = 0;
 	int x_hot_viewport = 0;
 	uint32_t cur_en = pos->enable ? 1 : 0;
-
+	uint32_t x_hotspot_clamped = pos->x_hotspot;
 	hubp->curs_pos = *pos;
-
 	/* Recout is zero for pipes if the entire dst_rect is contained
 	 * within preceeding ODM slices.
 	 */
@@ -772,6 +844,8 @@ void hubp401_cursor_set_position(
 
 	ASSERT(param->h_scale_ratio.value);
 
+	if (x_hotspot_clamped > 0xFF)
+		x_hotspot_clamped = 0xFF;
 	if (param->h_scale_ratio.value)
 		dst_x_offset = dc_fixpt_floor(dc_fixpt_div(
 			dc_fixpt_from_int(dst_x_offset),
@@ -792,7 +866,7 @@ void hubp401_cursor_set_position(
 			CURSOR_Y_POSITION, pos->y);
 
 		REG_SET_2(CURSOR_HOT_SPOT, 0,
-			CURSOR_HOT_SPOT_X, pos->x_hotspot,
+			CURSOR_HOT_SPOT_X, x_hotspot_clamped,
 			CURSOR_HOT_SPOT_Y, pos->y_hotspot);
 
 		REG_SET(CURSOR_DST_OFFSET, 0,
@@ -1058,19 +1132,13 @@ static struct hubp_funcs dcn401_hubp_funcs = {
 	.hubp_update_mall_sel = hubp401_update_mall_sel,
 	.hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering,
 	.hubp_program_mcache_id_and_split_coordinate = hubp401_program_mcache_id_and_split_coordinate,
-	.hubp_update_3dlut_fl_bias_scale = hubp401_update_3dlut_fl_bias_scale,
-	.hubp_program_3dlut_fl_mode = hubp401_program_3dlut_fl_mode,
-	.hubp_program_3dlut_fl_format = hubp401_program_3dlut_fl_format,
 	.hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr,
+	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
 	.hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param,
 	.hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl,
-	.hubp_program_3dlut_fl_addressing_mode = hubp401_program_3dlut_fl_addressing_mode,
-	.hubp_program_3dlut_fl_width = hubp401_program_3dlut_fl_width,
-	.hubp_program_3dlut_fl_tmz_protected = hubp401_program_3dlut_fl_tmz_protected,
 	.hubp_program_3dlut_fl_crossbar = hubp401_program_3dlut_fl_crossbar,
 	.hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done,
 	.hubp_clear_tiling = hubp401_clear_tiling,
-	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
 	.hubp_read_reg_state = hubp3_read_reg_state
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
index 4570b8016de5a..043948f64b862 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
@@ -328,32 +328,17 @@ int hubp401_get_3dlut_fl_done(struct hubp *hubp);
 
 void hubp401_set_unbounded_requesting(struct hubp *hubp, bool enable);
 
-void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale);
-
 void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r);
-
-void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits);
-
-void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width);
-
-void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode);
+		const enum dc_cm_lut_pixel_format format);
 
 void hubp401_enable_3dlut_fl(struct hubp *hubp, bool enable);
 
 void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group);
 
-void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address address);
+void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address *address);
 
-void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format);
-
-void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode);
-
-void hubp401_program_3dlut_fl_config(
-	struct hubp *hubp,
-	struct hubp_fl_3dlut_config *cfg);
+void hubp401_program_3dlut_fl_config(struct hubp *hubp,
+		const struct dc_3dlut_dma *config);
 
 void hubp401_clear_tiling(struct hubp *hubp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
new file mode 100644
index 0000000000000..0e33c739f459f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
@@ -0,0 +1,708 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dcn401/dcn401_hubp.h"
+#include "dcn42_hubp.h"
+#include "reg_helper.h"
+
+#define REG(reg) \
+	hubp2->hubp_regs->reg
+
+#define CTX \
+	hubp2->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	hubp2->hubp_shift->field_name, hubp2->hubp_mask->field_name
+
+static void hubp42_set_fgcg(struct hubp *hubp, bool enable)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	REG_UPDATE(HUBP_CLK_CNTL, HUBP_FGCG_REP_DIS, !enable);
+}
+
+static void hubp42_init(struct hubp *hubp)
+{
+	hubp3_init(hubp);
+
+	hubp42_set_fgcg(hubp, hubp->ctx->dc->debug.enable_fine_grain_clock_gating.bits.dchub);
+
+	/*do nothing for now for dcn3.5 or later*/
+}
+
+static void hubp42_program_pixel_format(
+	struct hubp *hubp,
+	enum surface_pixel_format format)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	uint32_t green_bar = 1;
+	uint32_t red_bar = 3;
+	uint32_t blue_bar = 2;
+
+	/* swap for ABGR format */
+	if (format == SURFACE_PIXEL_FORMAT_GRPH_ABGR8888
+			|| format == SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010
+			|| format == SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010_XR_BIAS
+			|| format == SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616
+			|| format == SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F) {
+		red_bar = 2;
+		blue_bar = 3;
+	}
+
+	REG_UPDATE_3(HUBPRET_CONTROL,
+			CROSSBAR_SRC_Y_G, green_bar,
+			CROSSBAR_SRC_CB_B, blue_bar,
+			CROSSBAR_SRC_CR_R, red_bar);
+
+	/* Mapping is same as ipp programming (cnvc) */
+
+	switch (format)	{
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 1);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 3);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 8);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010_XR_BIAS:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 10);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616: /* we use crossbar already */
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 26); /* ARGB16161616_UNORM */
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:/*we use crossbar already*/
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 24);
+		break;
+
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 65);
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 64);
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 67);
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 66);
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 12);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 112);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 113);
+		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 114);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 118);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 119);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGBE:
+		REG_UPDATE_2(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 116,
+				ALPHA_PLANE_EN, 0);
+		break;
+	case SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA:
+		REG_UPDATE_2(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 116,
+				ALPHA_PLANE_EN, 1);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	/* don't see the need of program the xbar in DCN 1.0 */
+}
+
+static void hubp42_program_deadline(
+		struct hubp *hubp,
+		struct dml2_display_dlg_regs *dlg_attr,
+		struct dml2_display_ttu_regs *ttu_attr)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	/* put DLG in mission mode */
+	REG_WRITE(HUBPREQ_DEBUG_DB, 0);
+
+	/* DLG - Per hubp */
+	REG_SET_2(BLANK_OFFSET_0, 0,
+		REFCYC_H_BLANK_END, dlg_attr->refcyc_h_blank_end,
+		DLG_V_BLANK_END, dlg_attr->dlg_vblank_end);
+
+	REG_SET(BLANK_OFFSET_1, 0,
+		MIN_DST_Y_NEXT_START, dlg_attr->min_dst_y_next_start);
+
+	REG_SET(DST_DIMENSIONS, 0,
+		REFCYC_PER_HTOTAL, dlg_attr->refcyc_per_htotal);
+
+	REG_SET_2(DST_AFTER_SCALER, 0,
+		REFCYC_X_AFTER_SCALER, dlg_attr->refcyc_x_after_scaler,
+		DST_Y_AFTER_SCALER, dlg_attr->dst_y_after_scaler);
+
+	REG_SET(REF_FREQ_TO_PIX_FREQ, 0,
+		REF_FREQ_TO_PIX_FREQ, dlg_attr->ref_freq_to_pix_freq);
+
+	/* DLG - Per luma/chroma */
+	REG_SET(VBLANK_PARAMETERS_1, 0,
+		REFCYC_PER_PTE_GROUP_VBLANK_L, dlg_attr->refcyc_per_pte_group_vblank_l);
+
+	if (REG(NOM_PARAMETERS_0))
+		REG_SET(NOM_PARAMETERS_0, 0,
+			DST_Y_PER_PTE_ROW_NOM_L, dlg_attr->dst_y_per_pte_row_nom_l);
+
+	if (REG(NOM_PARAMETERS_1))
+		REG_SET(NOM_PARAMETERS_1, 0,
+			REFCYC_PER_PTE_GROUP_NOM_L, dlg_attr->refcyc_per_pte_group_nom_l);
+
+	REG_SET(NOM_PARAMETERS_4, 0,
+		DST_Y_PER_META_ROW_NOM_L, dlg_attr->dst_y_per_meta_row_nom_l);
+
+	REG_SET(NOM_PARAMETERS_5, 0,
+		REFCYC_PER_META_CHUNK_NOM_L, dlg_attr->refcyc_per_meta_chunk_nom_l);
+
+	REG_SET_2(PER_LINE_DELIVERY, 0,
+		REFCYC_PER_LINE_DELIVERY_L, dlg_attr->refcyc_per_line_delivery_l,
+		REFCYC_PER_LINE_DELIVERY_C, dlg_attr->refcyc_per_line_delivery_c);
+
+	REG_SET(VBLANK_PARAMETERS_2, 0,
+		REFCYC_PER_PTE_GROUP_VBLANK_C, dlg_attr->refcyc_per_pte_group_vblank_c);
+
+	if (REG(NOM_PARAMETERS_2))
+		REG_SET(NOM_PARAMETERS_2, 0,
+			DST_Y_PER_PTE_ROW_NOM_C, dlg_attr->dst_y_per_pte_row_nom_c);
+
+	if (REG(NOM_PARAMETERS_3))
+		REG_SET(NOM_PARAMETERS_3, 0,
+			REFCYC_PER_PTE_GROUP_NOM_C, dlg_attr->refcyc_per_pte_group_nom_c);
+
+	REG_SET(NOM_PARAMETERS_6, 0,
+		DST_Y_PER_META_ROW_NOM_C, dlg_attr->dst_y_per_meta_row_nom_c);
+
+	REG_SET(NOM_PARAMETERS_7, 0,
+		REFCYC_PER_META_CHUNK_NOM_C, dlg_attr->refcyc_per_meta_chunk_nom_c);
+
+	/* TTU - per hubp */
+	REG_SET_2(DCN_TTU_QOS_WM, 0,
+		QoS_LEVEL_LOW_WM, ttu_attr->qos_level_low_wm,
+		QoS_LEVEL_HIGH_WM, ttu_attr->qos_level_high_wm);
+
+	/* TTU - per luma/chroma */
+	/* Assumed surf0 is luma and 1 is chroma */
+
+	REG_SET_3(DCN_SURF0_TTU_CNTL0, 0,
+		REFCYC_PER_REQ_DELIVERY, ttu_attr->refcyc_per_req_delivery_l,
+		QoS_LEVEL_FIXED, ttu_attr->qos_level_fixed_l,
+		QoS_RAMP_DISABLE, ttu_attr->qos_ramp_disable_l);
+
+	REG_SET_3(DCN_SURF1_TTU_CNTL0, 0,
+		REFCYC_PER_REQ_DELIVERY, ttu_attr->refcyc_per_req_delivery_c,
+		QoS_LEVEL_FIXED, ttu_attr->qos_level_fixed_c,
+		QoS_RAMP_DISABLE, ttu_attr->qos_ramp_disable_c);
+
+	REG_SET_3(DCN_CUR0_TTU_CNTL0, 0,
+		REFCYC_PER_REQ_DELIVERY, ttu_attr->refcyc_per_req_delivery_cur0,
+		QoS_LEVEL_FIXED, ttu_attr->qos_level_fixed_cur0,
+		QoS_RAMP_DISABLE, ttu_attr->qos_ramp_disable_cur0);
+
+	REG_SET(FLIP_PARAMETERS_1, 0,
+		REFCYC_PER_PTE_GROUP_FLIP_L, dlg_attr->refcyc_per_pte_group_flip_l);
+	REG_SET(HUBP_3DLUT_DLG_PARAM, 0, REFCYC_PER_3DLUT_GROUP, dlg_attr->refcyc_per_tdlut_group);
+
+	REG_UPDATE(DCN_DMDATA_VM_CNTL,
+			REFCYC_PER_VM_DMDATA, dlg_attr->refcyc_per_vm_dmdata);
+}
+
+void hubp42_setup(
+		struct hubp *hubp,
+	    struct dml2_dchub_per_pipe_register_set *pipe_regs,
+		union dml2_global_sync_programming *pipe_global_sync,
+		struct dc_crtc_timing *timing)
+{
+	/* otg is locked when this func is called. Register are double buffered.
+	 * disable the requestors is not needed
+	 */
+	hubp401_vready_at_or_After_vsync(hubp, pipe_global_sync, timing);
+	hubp401_program_requestor(hubp, &pipe_regs->rq_regs);
+	hubp42_program_deadline(hubp, &pipe_regs->dlg_regs, &pipe_regs->ttu_regs);
+}
+static void hubp42_program_surface_config(
+	struct hubp *hubp,
+	enum surface_pixel_format format,
+	struct dc_tiling_info *tiling_info,
+	struct plane_size *plane_size,
+	enum dc_rotation_angle rotation,
+	struct dc_plane_dcc_param *dcc,
+	bool horizontal_mirror,
+	unsigned int compat_level)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	hubp3_dcc_control_sienna_cichlid(hubp, dcc);
+	hubp3_program_tiling(hubp2, tiling_info, format);
+	hubp2_program_size(hubp, format, plane_size, dcc);
+	hubp2_program_rotation(hubp, rotation, horizontal_mirror);
+	hubp42_program_pixel_format(hubp, format);
+}
+
+static void hubp42_get_3dlut_fl_xbar_map(
+		const enum dc_cm_lut_pixel_format format,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b,
+		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r)
+{
+	switch (format) {
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
+		/* BGRA */
+		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*bit_slice_cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		break;
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
+	default:
+		/* RGBA */
+		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		break;
+	}
+}
+
+void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp,
+		const enum dc_cm_lut_pixel_format format)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r = 0;
+
+	hubp42_get_3dlut_fl_xbar_map(format,
+			&bit_slice_g,
+			&bit_slice_b,
+			&bit_slice_r);
+
+	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
+		HUBP_3DLUT_CROSSBAR_SEL_R, bit_slice_r,
+		HUBP_3DLUT_CROSSBAR_SEL_G, bit_slice_g,
+		HUBP_3DLUT_CROSSBAR_SEL_B, bit_slice_b);
+}
+
+static uint32_t hubp42_get_3dlut_fl_mpc_width(
+		const enum dc_cm_lut_size size)
+{
+	uint32_t width = 0;
+
+	switch (size) {
+	case CM_LUT_SIZE_333333:
+		width = 1;
+		break;
+	case CM_LUT_SIZE_171717:
+	default:
+		width = 0;
+		break;
+	}
+
+	return width;
+}
+
+void hubp42_program_3dlut_fl_config(struct hubp *hubp,
+		const struct dc_3dlut_dma *config)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	uint32_t mpc_width = hubp42_get_3dlut_fl_mpc_width(config->size);
+
+	REG_UPDATE(HUBP_3DLUT_CONTROL,
+			HUBP_3DLUT_MPC_WIDTH, mpc_width);
+
+	hubp401_program_3dlut_fl_config(hubp, config);
+}
+
+static bool hubp42_program_surface_flip_and_addr(
+	struct hubp *hubp,
+	const struct dc_plane_address *address,
+	bool flip_immediate)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	//program flip type
+	REG_UPDATE(DCSURF_FLIP_CONTROL,
+			SURFACE_FLIP_TYPE, flip_immediate);
+
+	// Program VMID reg
+	if (flip_immediate == 0)
+		REG_UPDATE(VMID_SETTINGS_0,
+			VMID, address->vmid);
+
+	if (address->type == PLN_ADDR_TYPE_GRPH_STEREO) {
+		REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_MODE_FOR_STEREOSYNC, 0);
+		REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_IN_STEREOSYNC, 0x1);
+
+	} else {
+		// turn off stereo if not in stereo
+		REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_MODE_FOR_STEREOSYNC, 0x0);
+		REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_IN_STEREOSYNC, 0x0);
+	}
+
+	/* HW automatically latch rest of address register on write to
+	 * DCSURF_PRIMARY_SURFACE_ADDRESS if SURFACE_UPDATE_LOCK is not used
+	 *
+	 * program high first and then the low addr, order matters!
+	 */
+	switch (address->type) {
+	case PLN_ADDR_TYPE_GRAPHICS:
+		/* DCN1.0 does not support const color
+		 * TODO: program DCHUBBUB_RET_PATH_DCC_CFGx_0/1
+		 * base on address->grph.dcc_const_color
+		 * x = 0, 2, 4, 6 for pipe 0, 1, 2, 3 for rgb and luma
+		 * x = 1, 3, 5, 7 for pipe 0, 1, 2, 3 for chroma
+		 */
+
+		if (address->grph.addr.quad_part == 0)
+			break;
+
+		REG_UPDATE_2(DCSURF_SURFACE_CONTROL,
+				PRIMARY_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ, address->tmz_surface);
+
+		if (address->grph.meta_addr.quad_part != 0) {
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH,
+					address->grph.meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+					PRIMARY_META_SURFACE_ADDRESS,
+					address->grph.meta_addr.low_part);
+		}
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH,
+				address->grph.addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+				PRIMARY_SURFACE_ADDRESS,
+				address->grph.addr.low_part);
+		break;
+	case PLN_ADDR_TYPE_VIDEO_PROGRESSIVE:
+		if (address->video_progressive.luma_addr.quad_part == 0
+				|| address->video_progressive.chroma_addr.quad_part == 0)
+			break;
+
+		REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
+				PRIMARY_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ_C, address->tmz_surface);
+
+		if (address->video_progressive.luma_meta_addr.quad_part != 0) {
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH_C,
+					address->video_progressive.chroma_meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, 0,
+					PRIMARY_META_SURFACE_ADDRESS_C,
+					address->video_progressive.chroma_meta_addr.low_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH,
+					address->video_progressive.luma_meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+					PRIMARY_META_SURFACE_ADDRESS,
+					address->video_progressive.luma_meta_addr.low_part);
+		}
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH_C,
+				address->video_progressive.chroma_addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_C, 0,
+				PRIMARY_SURFACE_ADDRESS_C,
+				address->video_progressive.chroma_addr.low_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH,
+				address->video_progressive.luma_addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+				PRIMARY_SURFACE_ADDRESS,
+				address->video_progressive.luma_addr.low_part);
+		break;
+	case PLN_ADDR_TYPE_GRPH_STEREO:
+		if (address->grph_stereo.left_addr.quad_part == 0)
+			break;
+		if (address->grph_stereo.right_addr.quad_part == 0)
+			break;
+
+		REG_UPDATE_8(DCSURF_SURFACE_CONTROL,
+				PRIMARY_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ_C, address->tmz_surface,
+				SECONDARY_SURFACE_TMZ, address->tmz_surface,
+				SECONDARY_SURFACE_TMZ_C, address->tmz_surface,
+				SECONDARY_META_SURFACE_TMZ, address->tmz_surface,
+				SECONDARY_META_SURFACE_TMZ_C, address->tmz_surface);
+
+		if (address->grph_stereo.right_meta_addr.quad_part != 0) {
+
+			REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, 0,
+				SECONDARY_META_SURFACE_ADDRESS_HIGH_C,
+				address->grph_stereo.right_alpha_meta_addr.high_part);
+
+			REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, 0,
+				SECONDARY_META_SURFACE_ADDRESS_C,
+				address->grph_stereo.right_alpha_meta_addr.low_part);
+
+			REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, 0,
+					SECONDARY_META_SURFACE_ADDRESS_HIGH,
+					address->grph_stereo.right_meta_addr.high_part);
+
+			REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS, 0,
+					SECONDARY_META_SURFACE_ADDRESS,
+					address->grph_stereo.right_meta_addr.low_part);
+		}
+		if (address->grph_stereo.left_meta_addr.quad_part != 0) {
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, 0,
+				PRIMARY_META_SURFACE_ADDRESS_HIGH_C,
+				address->grph_stereo.left_alpha_meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, 0,
+				PRIMARY_META_SURFACE_ADDRESS_C,
+				address->grph_stereo.left_alpha_meta_addr.low_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH,
+					address->grph_stereo.left_meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+					PRIMARY_META_SURFACE_ADDRESS,
+					address->grph_stereo.left_meta_addr.low_part);
+		}
+
+		REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, 0,
+				SECONDARY_SURFACE_ADDRESS_HIGH_C,
+				address->grph_stereo.right_alpha_addr.high_part);
+
+		REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS_C, 0,
+				SECONDARY_SURFACE_ADDRESS_C,
+				address->grph_stereo.right_alpha_addr.low_part);
+
+		REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, 0,
+				SECONDARY_SURFACE_ADDRESS_HIGH,
+				address->grph_stereo.right_addr.high_part);
+
+		REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS, 0,
+				SECONDARY_SURFACE_ADDRESS,
+				address->grph_stereo.right_addr.low_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH_C,
+				address->grph_stereo.left_alpha_addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_C, 0,
+				PRIMARY_SURFACE_ADDRESS_C,
+				address->grph_stereo.left_alpha_addr.low_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH,
+				address->grph_stereo.left_addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+				PRIMARY_SURFACE_ADDRESS,
+				address->grph_stereo.left_addr.low_part);
+		break;
+	case PLN_ADDR_TYPE_RGBEA:
+		if (address->rgbea.addr.quad_part == 0
+				|| address->rgbea.alpha_addr.quad_part == 0)
+			break;
+
+		REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
+				PRIMARY_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_META_SURFACE_TMZ_C, address->tmz_surface);
+
+		if (address->rgbea.meta_addr.quad_part != 0) {
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH_C,
+					address->rgbea.alpha_meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, 0,
+					PRIMARY_META_SURFACE_ADDRESS_C,
+					address->rgbea.alpha_meta_addr.low_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+					PRIMARY_META_SURFACE_ADDRESS_HIGH,
+					address->rgbea.meta_addr.high_part);
+
+			REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+					PRIMARY_META_SURFACE_ADDRESS,
+					address->rgbea.meta_addr.low_part);
+		}
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH_C,
+				address->rgbea.alpha_addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_C, 0,
+				PRIMARY_SURFACE_ADDRESS_C,
+				address->rgbea.alpha_addr.low_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+				PRIMARY_SURFACE_ADDRESS_HIGH,
+				address->rgbea.addr.high_part);
+
+		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+				PRIMARY_SURFACE_ADDRESS,
+				address->rgbea.addr.low_part);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	hubp->request_address = *address;
+
+	return true;
+}
+
+struct hubp_funcs dcn42_hubp_funcs = {
+	.hubp_enable_tripleBuffer = hubp2_enable_triplebuffer,
+	.hubp_is_triplebuffer_enabled = hubp2_is_triplebuffer_enabled,
+	.hubp_program_surface_flip_and_addr = hubp42_program_surface_flip_and_addr,
+	.hubp_program_surface_config = hubp42_program_surface_config,
+	.hubp_is_flip_pending = hubp2_is_flip_pending,
+	.hubp_setup2 = hubp42_setup,
+	.hubp_setup_interdependent2 = hubp401_setup_interdependent,
+	.hubp_set_vm_system_aperture_settings = hubp3_set_vm_system_aperture_settings,
+	.set_blank = hubp2_set_blank,
+	.set_blank_regs = hubp2_set_blank_regs,
+	.dcc_control = hubp3_dcc_control,
+	.hubp_reset = hubp_reset,
+	.mem_program_viewport = min_set_viewport,
+	.set_cursor_attributes	= hubp32_cursor_set_attributes,
+	.set_cursor_position	= hubp401_cursor_set_position,
+	.hubp_clk_cntl = hubp2_clk_cntl,
+	.hubp_vtg_sel = hubp2_vtg_sel,
+	.dmdata_set_attributes = hubp3_dmdata_set_attributes,
+	.dmdata_load = hubp2_dmdata_load,
+	.dmdata_status_done = hubp2_dmdata_status_done,
+	.hubp_read_state = hubp42_read_state,
+	.hubp_clear_underflow = hubp2_clear_underflow,
+	.hubp_set_flip_control_surface_gsl = hubp2_set_flip_control_surface_gsl,
+	.hubp_init = hubp42_init,
+	.set_unbounded_requesting = hubp31_set_unbounded_requesting,
+	.hubp_soft_reset = hubp31_soft_reset,
+	.hubp_set_flip_int = hubp1_set_flip_int,
+	.hubp_in_blank = hubp1_in_blank,
+	.program_extended_blank = hubp31_program_extended_blank_value,
+	.hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr,
+	.hubp_program_3dlut_fl_config = hubp42_program_3dlut_fl_config,
+	.hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param,
+	.hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl,
+	.hubp_program_3dlut_fl_crossbar = hubp42_program_3dlut_fl_crossbar,
+	.hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done,
+	.hubp_clear_tiling = hubp3_clear_tiling,
+	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
+	.hubp_read_reg_state = hubp3_read_reg_state
+};
+
+bool hubp42_construct(
+	struct dcn20_hubp *hubp2,
+	struct dc_context *ctx,
+	uint32_t inst,
+	const struct dcn_hubp2_registers *hubp_regs,
+	const struct dcn_hubp2_shift *hubp_shift,
+	const struct dcn_hubp2_mask *hubp_mask)
+{
+	hubp2->base.funcs = &dcn42_hubp_funcs;
+	hubp2->base.ctx = ctx;
+	hubp2->hubp_regs = hubp_regs;
+	hubp2->hubp_shift = hubp_shift;
+	hubp2->hubp_mask = hubp_mask;
+	hubp2->base.inst = inst;
+	hubp2->base.opp_id = OPP_ID_INVALID;
+	hubp2->base.mpcc_id = 0xf;
+
+	return true;
+}
+
+
+void hubp42_read_state(struct hubp *hubp)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	struct dcn_hubp_state *s = &hubp2->state;
+	struct dcn_fl_regs_st *fl_regs = &s->fl_regs;
+
+	hubp401_read_state(hubp);
+
+	/* CONTROL */
+	REG_GET_5(HUBP_3DLUT_CONTROL,
+			HUBP_3DLUT_ENABLE, &fl_regs->lut_enable,
+			HUBP_3DLUT_DONE, &fl_regs->lut_done,
+			HUBP_3DLUT_ADDRESSING_MODE, &fl_regs->lut_addr_mode,
+			HUBP_3DLUT_WIDTH, &fl_regs->lut_width,
+			HUBP_3DLUT_MPC_WIDTH, &fl_regs->lut_mpc_width);
+
+	REG_GET_4(HUBP_3DLUT_CONTROL,
+			HUBP_3DLUT_TMZ, &fl_regs->lut_tmz,
+			HUBP_3DLUT_CROSSBAR_SEL_R, &fl_regs->lut_crossbar_sel_r,
+			HUBP_3DLUT_CROSSBAR_SEL_G, &fl_regs->lut_crossbar_sel_g,
+			HUBP_3DLUT_CROSSBAR_SEL_B, &fl_regs->lut_crossbar_sel_b);
+
+	/* ADDR */
+	REG_GET(HUBP_3DLUT_ADDRESS_HIGH,
+			HUBP_3DLUT_ADDRESS_HIGH, &fl_regs->lut_addr_hi);
+	REG_GET(HUBP_3DLUT_ADDRESS_LOW,
+			HUBP_3DLUT_ADDRESS_LOW, &fl_regs->lut_addr_lo);
+	/* CLK */
+	REG_GET(HUBP_3DLUT_DLG_PARAM,
+			REFCYC_PER_3DLUT_GROUP, &fl_regs->refcyc_3dlut_group);
+	/* FL */
+	REG_GET_2(_3DLUT_FL_BIAS_SCALE,
+		HUBP0_3DLUT_FL_BIAS, &fl_regs->lut_fl_bias,
+		HUBP0_3DLUT_FL_SCALE, &fl_regs->lut_fl_scale);
+	REG_GET_2(_3DLUT_FL_CONFIG,
+		HUBP0_3DLUT_FL_MODE, &fl_regs->lut_fl_mode,
+		HUBP0_3DLUT_FL_FORMAT, &fl_regs->lut_fl_format);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
new file mode 100644
index 0000000000000..486c8907413aa
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_HUBP_DCN42_H__
+#define __DC_HUBP_DCN42_H__
+
+#include "dcn35/dcn35_hubp.h"
+
+#define HUBP_MASK_SH_LIST_DCN42(mask_sh)\
+	HUBP_MASK_SH_LIST_DCN35(mask_sh),\
+	HUBP_SF(HUBP0_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_MODE, mask_sh),\
+	HUBP_SF(HUBP0_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, mask_sh),\
+	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, mask_sh),\
+	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_SCALE, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ENABLE, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_DONE, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_WIDTH, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_MPC_WIDTH, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_TMZ, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_CROSSBAR_SEL_B, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_CROSSBAR_SEL_G, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_CROSSBAR_SEL_R, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_ADDRESS_LOW, HUBP_3DLUT_ADDRESS_LOW, mask_sh),\
+	HUBP_SF(CURSOR0_0_HUBP_3DLUT_DLG_PARAM, REFCYC_PER_3DLUT_GROUP, mask_sh)
+
+bool hubp42_construct(
+	struct dcn20_hubp *hubp2,
+	struct dc_context *ctx,
+	uint32_t inst,
+	const struct dcn_hubp2_registers *hubp_regs,
+	const struct dcn_hubp2_shift *hubp_shift,
+	const struct dcn_hubp2_mask *hubp_mask);
+
+void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp,
+		const enum dc_cm_lut_pixel_format format);
+
+void hubp42_program_3dlut_fl_config(struct hubp *hubp,
+		const struct dc_3dlut_dma *config);
+
+void hubp42_read_state(struct hubp *hubp);
+
+void hubp42_setup(
+		struct hubp *hubp,
+	    struct dml2_dchub_per_pipe_register_set *pipe_regs,
+		union dml2_global_sync_programming *pipe_global_sync,
+		struct dc_crtc_timing *timing);
+
+#endif /* __DC_HUBP_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/Makefile b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
index bee617ca0838c..08cbbf87a19fa 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
@@ -1,5 +1,4 @@
-
-# Copyright 2022 Advanced Micro Devices, Inc.
+# Copyright 2022-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -199,4 +198,13 @@ HWSS_DCN401 = dcn401_hwseq.o dcn401_init.o
 AMD_DAL_HWSS_DCN401 = $(addprefix $(AMDDALPATH)/dc/hwss/dcn401/,$(HWSS_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCN401)
+
+###############################################################################
+
+HWSS_DCN42 = dcn42_hwseq.o dcn42_init.o
+
+AMD_DAL_HWSS_DCN42 = $(addprefix $(AMDDALPATH)/dc/hwss/dcn42/,$(HWSS_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCN42)
+
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
index f66a38f43a091..dc8feca578c5b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Advanced Micro Devices, Inc.
+ * Copyright 2016, 2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -700,6 +700,8 @@ struct dce_hwseq_registers {
 	uint32_t DOMAIN23_PG_STATUS;
 	uint32_t DOMAIN24_PG_STATUS;
 	uint32_t DOMAIN25_PG_STATUS;
+	uint32_t DOMAIN26_PG_CONFIG;
+	uint32_t DOMAIN26_PG_STATUS;
 };
  /* set field name */
 #define HWS_SF(blk_name, reg_name, field_name, post_fix)\
@@ -1245,6 +1247,13 @@ struct dce_hwseq_registers {
 	type DOMAIN25_PGFSM_PWR_STATUS; \
 	type DOMAIN_DESIRED_PWR_STATE;
 
+#define HWSEQ_DCN42_REG_FIELD_LIST(type) \
+	type DPIASYMCLK4_GATE_DISABLE;\
+	type DPIASYMCLK5_GATE_DISABLE;\
+	type DOMAIN26_POWER_FORCEON; \
+	type DOMAIN26_POWER_GATE; \
+	type DOMAIN26_PGFSM_PWR_STATUS;
+
 struct dce_hwseq_shift {
 	HWSEQ_REG_FIELD_LIST(uint8_t)
 	HWSEQ_DCN_REG_FIELD_LIST(uint8_t)
@@ -1253,6 +1262,7 @@ struct dce_hwseq_shift {
 	HWSEQ_DCN31_REG_FIELD_LIST(uint8_t)
 	HWSEQ_DCN35_REG_FIELD_LIST(uint8_t)
 	HWSEQ_DCN401_REG_FIELD_LIST(uint8_t)
+	HWSEQ_DCN42_REG_FIELD_LIST(uint8_t)
 };
 
 struct dce_hwseq_mask {
@@ -1263,6 +1273,7 @@ struct dce_hwseq_mask {
 	HWSEQ_DCN31_REG_FIELD_LIST(uint32_t)
 	HWSEQ_DCN35_REG_FIELD_LIST(uint32_t)
 	HWSEQ_DCN401_REG_FIELD_LIST(uint32_t)
+	HWSEQ_DCN42_REG_FIELD_LIST(uint32_t)
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 699a756bbc405..8a17cc036399a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -86,9 +86,9 @@
 	hws->ctx
 
 #define DC_LOGGER \
-	ctx->logger
-#define DC_LOGGER_INIT() \
-	struct dc_context *ctx = dc->ctx
+	dc_ctx->logger
+#define DC_LOGGER_INIT(ctx) \
+	struct dc_context *dc_ctx = ctx
 
 #define REG(reg)\
 	hws->regs->reg
@@ -660,6 +660,30 @@ void dce110_update_info_frame(struct pipe_ctx *pipe_ctx)
 	}
 }
 
+static void
+dce110_dac_encoder_control(struct pipe_ctx *pipe_ctx, bool enable)
+{
+	struct dc_link *link = pipe_ctx->stream->link;
+	struct dc_bios *bios = link->ctx->dc_bios;
+	struct bp_encoder_control encoder_control = {0};
+
+	encoder_control.action = enable ? ENCODER_CONTROL_ENABLE : ENCODER_CONTROL_DISABLE;
+	encoder_control.engine_id = link->link_enc->analog_engine;
+	encoder_control.pixel_clock = pipe_ctx->stream->timing.pix_clk_100hz / 10;
+	bios->funcs->encoder_control(bios, &encoder_control);
+}
+
+static bool
+dce110_dac_load_detect(struct dc_link *link)
+{
+	struct dc_bios *bios = link->ctx->dc_bios;
+	struct link_encoder *link_enc = link->link_enc;
+	enum bp_result bp_result;
+
+	bp_result = bios->funcs->dac_load_detection(bios, link_enc->analog_engine);
+	return bp_result == BP_RESULT_OK;
+}
+
 void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 {
 	enum dc_lane_count lane_count =
@@ -672,6 +696,7 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 	uint32_t early_control = 0;
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
 
+	link_hwss->setup_stream_attribute(pipe_ctx);
 	link_hwss->setup_stream_encoder(pipe_ctx);
 
 	dc->hwss.update_info_frame(pipe_ctx);
@@ -690,6 +715,8 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 
 	tg->funcs->set_early_control(tg, early_control);
 
+	if (dc_is_rgb_signal(pipe_ctx->stream->signal))
+		dce110_dac_encoder_control(pipe_ctx, true);
 }
 
 static enum bp_result link_transmitter_control(
@@ -711,13 +738,14 @@ void dce110_edp_wait_for_hpd_ready(
 		struct dc_link *link,
 		bool power_up)
 {
-	struct dc_context *ctx = link->ctx;
 	struct graphics_object_id connector = link->link_enc->connector;
 	bool edp_hpd_high = false;
 	uint32_t time_elapsed = 0;
 	uint32_t timeout = power_up ?
 		PANEL_POWER_UP_TIMEOUT : PANEL_POWER_DOWN_TIMEOUT;
 
+	DC_LOGGER_INIT(link->ctx);
+
 	if (dal_graphics_object_id_get_connector_id(connector)
 			!= CONNECTOR_ID_EDP) {
 		BREAK_TO_DEBUGGER();
@@ -769,6 +797,7 @@ void dce110_edp_power_control(
 	enum bp_result bp_result;
 	uint8_t pwrseq_instance;
 
+	DC_LOGGER_INIT(ctx);
 
 	if (dal_graphics_object_id_get_connector_id(link->link_enc->connector)
 			!= CONNECTOR_ID_EDP) {
@@ -937,6 +966,8 @@ void dce110_edp_backlight_control(
 	unsigned int pre_T11_delay = (link->dpcd_sink_ext_caps.bits.oled ? OLED_PRE_T11_DELAY : 0);
 	unsigned int post_T7_delay = (link->dpcd_sink_ext_caps.bits.oled ? OLED_POST_T7_DELAY : 0);
 
+	DC_LOGGER_INIT(ctx);
+
 	if (dal_graphics_object_id_get_connector_id(link->link_enc->connector)
 		!= CONNECTOR_ID_EDP) {
 		BREAK_TO_DEBUGGER();
@@ -1183,6 +1214,9 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
 		dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst,
 					       link_enc->transmitter - TRANSMITTER_UNIPHY_A);
 	}
+
+	if (dc_is_rgb_signal(pipe_ctx->stream->signal))
+		dce110_dac_encoder_control(pipe_ctx, false);
 }
 
 void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
@@ -1564,6 +1598,22 @@ static enum dc_status dce110_enable_stream_timing(
 
 	return DC_OK;
 }
+static void
+dce110_select_crtc_source(struct pipe_ctx *pipe_ctx)
+{
+	struct dc_link *link = pipe_ctx->stream->link;
+	struct dc_bios *bios = link->ctx->dc_bios;
+	struct bp_crtc_source_select crtc_source_select = {0};
+	enum engine_id engine_id = link->link_enc->preferred_engine;
+
+	if (dc_is_rgb_signal(pipe_ctx->stream->signal))
+		engine_id = link->link_enc->analog_engine;
+	crtc_source_select.controller_id = CONTROLLER_ID_D0 + pipe_ctx->stream_res.tg->inst;
+	crtc_source_select.color_depth = pipe_ctx->stream->timing.display_color_depth;
+	crtc_source_select.engine_id = engine_id;
+	crtc_source_select.sink_signal = pipe_ctx->stream->signal;
+	bios->funcs->select_crtc_source(bios, &crtc_source_select);
+}
 
 enum dc_status dce110_apply_single_controller_ctx_to_hw(
 		struct pipe_ctx *pipe_ctx,
@@ -1584,6 +1634,10 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 		hws->funcs.disable_stream_gating(dc, pipe_ctx);
 	}
 
+	if (pipe_ctx->stream->signal == SIGNAL_TYPE_RGB) {
+		dce110_select_crtc_source(pipe_ctx);
+	}
+
 	if (pipe_ctx->stream_res.audio != NULL) {
 		struct audio_output audio_output = {0};
 
@@ -1663,7 +1717,8 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 		pipe_ctx->stream_res.tg->funcs->set_static_screen_control(
 				pipe_ctx->stream_res.tg, event_triggers, 2);
 
-	if (!dc_is_virtual_signal(pipe_ctx->stream->signal))
+	if (!dc_is_virtual_signal(pipe_ctx->stream->signal) &&
+		!dc_is_rgb_signal(pipe_ctx->stream->signal))
 		pipe_ctx->stream_res.stream_enc->funcs->dig_connect_to_otg(
 			pipe_ctx->stream_res.stream_enc,
 			pipe_ctx->stream_res.tg->inst);
@@ -1885,6 +1940,35 @@ static void clean_up_dsc_blocks(struct dc *dc)
 	}
 }
 
+static void dc_hwss_enable_otg_pwa(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	struct timing_generator *tg = NULL;
+
+	if (dc->debug.enable_otg_frame_sync_pwa == 0)
+		return;
+
+	if (pipe_ctx == NULL || pipe_ctx->stream_res.tg == NULL)
+		return;
+	tg = pipe_ctx->stream_res.tg;
+
+	/*only enable this if one active*/
+	if (tg->funcs->enable_otg_pwa) {
+		struct otc_pwa_frame_sync pwa_param = {0};
+
+		DC_LOGGER_INIT(dc->ctx);
+		/* mode 1 to choose generate pwa sync signal on line 0 counting
+		 * from vstartup at very beginning of the frame
+		 */
+		pwa_param.pwa_frame_sync_line_offset = 0;
+		pwa_param.pwa_sync_mode = DC_OTG_PWA_FRAME_SYNC_MODE_VSTARTUP;
+		/*frame sync line for generating high frame sync*/
+		tg->funcs->enable_otg_pwa(tg, &pwa_param);
+		DC_LOG_DC("Enable OTG PWA frame sync on TG %d\n", tg->inst);
+	}
+}
+
 /*
  * When ASIC goes from VBIOS/VGA mode to driver/accelerated mode we need:
  *  1. Power down all DC HW blocks
@@ -1910,8 +1994,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 	bool keep_edp_vdd_on = false;
 	bool should_clean_dsc_block = true;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
-	DC_LOGGER_INIT();
-
+	DC_LOGGER_INIT(dc->ctx);
 
 	get_edp_links_with_sink(dc, edp_links_with_sink, &edp_with_sink_num);
 	dc_get_edp_links(dc, edp_links, &edp_num);
@@ -1962,6 +2045,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 						// If VBios supports it, we check it from reigster or other flags.
 						pipe_ctx->stream_res.pix_clk_params.dio_se_pix_per_cycle = 1;
 					}
+					dc_hwss_enable_otg_pwa(dc, pipe_ctx);
 				}
 				break;
 			}
@@ -2531,6 +2615,18 @@ enum dc_status dce110_apply_ctx_to_hw(
 #endif
 	}
 
+
+	if (dc->debug.enable_otg_frame_sync_pwa && context->stream_count == 1) {
+		/* only enable this on one OTG*/
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+			if (pipe_ctx && pipe_ctx->stream != NULL) {
+				dc_hwss_enable_otg_pwa(dc, pipe_ctx);
+				break;
+			}
+		}
+	}
 	if (dc->fbc_compressor)
 		enable_fbc(dc, dc->current_state);
 
@@ -2677,7 +2773,6 @@ static bool wait_for_reset_trigger_to_occur(
 	struct dc_context *dc_ctx,
 	struct timing_generator *tg)
 {
-	struct dc_context *ctx = dc_ctx;
 	bool rc = false;
 
 	/* To avoid endless loop we wait at most
@@ -2719,10 +2814,9 @@ static void dce110_enable_timing_synchronization(
 		int group_size,
 		struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	struct dcp_gsl_params gsl_params = { 0 };
 	int i;
-	DC_LOGGER_INIT();
+	DC_LOGGER_INIT(dc->ctx);
 
 	DC_SYNC_INFO("GSL: Setting-up...\n");
 
@@ -2765,10 +2859,9 @@ static void dce110_enable_per_frame_crtc_position_reset(
 		int group_size,
 		struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	struct dcp_gsl_params gsl_params = { 0 };
 	int i;
-	DC_LOGGER_INIT();
+	DC_LOGGER_INIT(dc->ctx);
 
 	gsl_params.gsl_group = 0;
 	gsl_params.gsl_master = 0;
@@ -3261,15 +3354,6 @@ void dce110_enable_tmds_link_output(struct dc_link *link,
 	link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
 }
 
-static void dce110_enable_analog_link_output(
-		struct dc_link *link,
-		uint32_t pix_clk_100hz)
-{
-	link->link_enc->funcs->enable_analog_output(
-			link->link_enc,
-			pix_clk_100hz);
-}
-
 void dce110_enable_dp_link_output(
 		struct dc_link *link,
 		const struct link_resource *link_res,
@@ -3407,8 +3491,8 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.enable_lvds_link_output = dce110_enable_lvds_link_output,
 	.enable_tmds_link_output = dce110_enable_tmds_link_output,
 	.enable_dp_link_output = dce110_enable_dp_link_output,
-	.enable_analog_link_output = dce110_enable_analog_link_output,
 	.disable_link_output = dce110_disable_link_output,
+	.dac_load_detect = dce110_dac_load_detect,
 };
 
 static const struct hwseq_private_funcs dce110_private_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index a2d28be480e81..17ff66d9a6172 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -60,9 +60,9 @@
 #include "dc_state_priv.h"
 
 #define DC_LOGGER \
-	dc_logger
-#define DC_LOGGER_INIT(logger) \
-	struct dal_logger *dc_logger = logger
+	dc_ctx->logger
+#define DC_LOGGER_INIT(ctx) \
+	struct dc_context *dc_ctx = ctx
 
 #define CTX \
 	hws->ctx
@@ -1009,7 +1009,7 @@ static void power_on_plane_resources(
 	struct dce_hwseq *hws,
 	int plane_id)
 {
-	DC_LOGGER_INIT(hws->ctx->logger);
+	DC_LOGGER_INIT(hws->ctx);
 
 	if (hws->funcs.dpp_root_clock_control)
 		hws->funcs.dpp_root_clock_control(hws, plane_id, true);
@@ -1286,7 +1286,7 @@ static void dcn10_reset_back_end_for_pipe(
 {
 	int i;
 	struct dc_link *link;
-	DC_LOGGER_INIT(dc->ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
 		pipe_ctx->stream = NULL;
 		return;
@@ -1422,12 +1422,10 @@ void dcn10_verify_allow_pstate_change_high(struct dc *dc)
 		return;
 
 	if (!hubbub->funcs->verify_allow_pstate_change_high(hubbub)) {
-		int i = 0;
-
 		if (should_log_hw_state)
 			dcn10_log_hw_state(dc, NULL);
 
-		TRACE_DC_PIPE_STATE(pipe_ctx, i, MAX_PIPES);
+		TRACE_DC_PIPE_STATE(pipe_ctx, MAX_PIPES);
 		BREAK_TO_DEBUGGER();
 		if (dcn10_hw_wa_force_recovery(dc)) {
 			/*check again*/
@@ -1490,7 +1488,7 @@ void dcn10_plane_atomic_power_down(struct dc *dc,
 		struct hubp *hubp)
 {
 	struct dce_hwseq *hws = dc->hwseq;
-	DC_LOGGER_INIT(dc->ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
@@ -1554,7 +1552,7 @@ void dcn10_plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 void dcn10_disable_plane(struct dc *dc, struct dc_state *state, struct pipe_ctx *pipe_ctx)
 {
 	struct dce_hwseq *hws = dc->hwseq;
-	DC_LOGGER_INIT(dc->ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	if (!pipe_ctx->plane_res.hubp || pipe_ctx->plane_res.hubp->power_gated)
 		return;
@@ -2268,8 +2266,6 @@ static bool wait_for_reset_trigger_to_occur(
 {
 	bool rc = false;
 
-	DC_LOGGER_INIT(dc_ctx->logger);
-
 	/* To avoid endless loop we wait at most
 	 * frames_to_wait_on_triggered_reset frames for the reset to occur. */
 	const uint32_t frames_to_wait_on_triggered_reset = 10;
@@ -2384,7 +2380,6 @@ static uint8_t get_clock_divider(struct pipe_ctx *pipe,
 static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
 				    struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	int i, master = -1, embedded = -1;
 	struct dc_crtc_timing *hw_crtc_timing;
 	uint64_t phase[MAX_PIPES];
@@ -2397,7 +2392,7 @@ static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
 	uint32_t dp_ref_clk_100hz =
 		dc->res_pool->dp_clock_source->ctx->dc->clk_mgr->dprefclk_khz*10;
 
-	DC_LOGGER_INIT(dc_ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	hw_crtc_timing = kzalloc_objs(*hw_crtc_timing, MAX_PIPES);
 	if (!hw_crtc_timing)
@@ -2477,12 +2472,11 @@ void dcn10_enable_vblanks_synchronization(
 	int group_size,
 	struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
 	int i, width = 0, height = 0, master;
 
-	DC_LOGGER_INIT(dc_ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	for (i = 1; i < group_size; i++) {
 		opp = grouped_pipes[i]->stream_res.opp;
@@ -2543,12 +2537,11 @@ void dcn10_enable_timing_synchronization(
 	int group_size,
 	struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
 	int i, width = 0, height = 0;
 
-	DC_LOGGER_INIT(dc_ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	DC_SYNC_INFO("Setting up OTG reset trigger\n");
 
@@ -2624,10 +2617,9 @@ void dcn10_enable_per_frame_crtc_position_reset(
 	int group_size,
 	struct pipe_ctx *grouped_pipes[])
 {
-	struct dc_context *dc_ctx = dc->ctx;
 	int i;
 
-	DC_LOGGER_INIT(dc_ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	DC_SYNC_INFO("Setting up\n");
 	for (i = 0; i < group_size; i++)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
index db2f7cbb12ff5..94f63fd54e3eb 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
@@ -513,7 +513,6 @@ static void dcn31_reset_back_end_for_pipe(
 {
 	struct dc_link *link;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
 		pipe_ctx->stream = NULL;
 		return;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index b5a4cefbd35f2..b5f60f59382ea 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -60,15 +60,15 @@
 #include "dcn20/dcn20_hwseq.h"
 #include "dc_state_priv.h"
 
-#define DC_LOGGER_INIT(logger) \
-	struct dal_logger *dc_logger = logger
+#define DC_LOGGER \
+	dc_ctx->logger
+#define DC_LOGGER_INIT(ctx) \
+	struct dc_context *dc_ctx = ctx
 
 #define CTX \
 	hws->ctx
 #define REG(reg)\
 	hws->regs->reg
-#define DC_LOGGER \
-	dc_logger
 
 
 #undef FN
@@ -331,7 +331,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 	struct pipe_ctx *odm_pipe;
 	int opp_cnt = 1;
 
-	DC_LOGGER_INIT(stream->ctx->logger);
+	DC_LOGGER_INIT(stream->ctx);
 
 	ASSERT(dsc);
 	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
@@ -897,7 +897,7 @@ void dcn35_disable_plane(struct dc *dc, struct dc_state *state, struct pipe_ctx
 	bool is_phantom = dc_state_get_pipe_subvp_type(state, pipe_ctx) == SUBVP_PHANTOM;
 	struct timing_generator *tg = is_phantom ? pipe_ctx->stream_res.tg : NULL;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
+	DC_LOGGER_INIT(dc->ctx);
 
 	if (!pipe_ctx->plane_res.hubp || pipe_ctx->plane_res.hubp->power_gated)
 		return;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index eb198d52a1155..357899116ecd5 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -95,10 +95,6 @@ void dcn401_program_gamut_remap(struct pipe_ctx *pipe_ctx)
 	unsigned int mpcc_id = pipe_ctx->plane_res.mpcc_inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
 
-	//For now assert if location is not pre-blend
-	if (pipe_ctx->plane_state)
-		ASSERT(pipe_ctx->plane_state->mcm_location == MPCC_MOVABLE_CM_LOCATION_BEFORE);
-
 	// program MPCC_MCM_FIRST_GAMUT_REMAP
 	memset(&mpc_adjust, 0, sizeof(mpc_adjust));
 	mpc_adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS;
@@ -304,6 +300,7 @@ void dcn401_init_hw(struct dc *dc)
 			}
 		}
 	}
+
 	for (i = 0; i < res_pool->audio_count; i++) {
 		struct audio *audio = res_pool->audios[i];
 
@@ -372,293 +369,179 @@ void dcn401_init_hw(struct dc *dc)
 	}
 }
 
-static void dcn401_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx *pipe_ctx,
-		enum MCM_LUT_XABLE *shaper_xable,
-		enum MCM_LUT_XABLE *lut3d_xable,
-		enum MCM_LUT_XABLE *lut1d_xable)
+void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
-	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
-	bool lut1d_enable = false;
-	struct mpc *mpc = dc->res_pool->mpc;
-	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
-
-	if (!pipe_ctx->plane_state)
-		return;
-	shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting;
-	lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable;
-	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
-	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
-
-	*lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
 
-	switch (shaper_3dlut_setting) {
-	case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL:
-		*lut3d_xable = *shaper_xable = MCM_LUT_DISABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER:
-		*lut3d_xable = MCM_LUT_DISABLE;
-		*shaper_xable = MCM_LUT_ENABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT:
-		*lut3d_xable = *shaper_xable = MCM_LUT_ENABLE;
-		break;
+	if (hubp->funcs->hubp_enable_3dlut_fl) {
+		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
 	}
 }
 
-void dcn401_populate_mcm_luts(struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		struct dc_cm2_func_luts mcm_luts,
-		bool lut_bank_a)
+bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
+				const struct dc_plane_state *plane_state)
 {
+	struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	const struct dc_plane_cm *cm = &plane_state->cm;
 	int mpcc_id = hubp->inst;
 	struct mpc *mpc = dc->res_pool->mpc;
 	union mcm_lut_params m_lut_params;
-	enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src;
-	enum hubp_3dlut_fl_format format = 0;
-	enum hubp_3dlut_fl_mode mode;
-	enum hubp_3dlut_fl_width width = 0;
-	enum hubp_3dlut_fl_addressing_mode addr_mode;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0;
-	enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE;
-	enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE;
-	enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE;
+	struct dc_3dlut_dma lut3d_dma;
+	bool lut_enable;
+	bool lut_bank_a;
 	bool rval;
+	bool result = true;
 
-	dcn401_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
+	/* decide LUT bank based on current in use */
+	mpc->funcs->get_lut_mode(mpc, MCM_LUT_1DLUT, mpcc_id, &lut_enable, &lut_bank_a);
+	if (!lut_enable) {
+		mpc->funcs->get_lut_mode(mpc, MCM_LUT_SHAPER, mpcc_id, &lut_enable, &lut_bank_a);
+	}
+	if (!lut_enable) {
+		mpc->funcs->get_lut_mode(mpc, MCM_LUT_3DLUT, mpcc_id, &lut_enable, &lut_bank_a);
+	}
+
+	/* switch to the next bank */
+	if (lut_enable) {
+		lut_bank_a = !lut_bank_a;
+	}
+
+	/* MCM location fixed to pre-blend */
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
 
 	/* 1D LUT */
-	if (mcm_luts.lut1d_func) {
-		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.lut1d_func->pwl;
-		else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.lut1d_func,
-					&dpp_base->regamma_params, false);
+	lut_enable = cm->flags.bits.blend_enable;
+	memset(&m_lut_params, 0, sizeof(m_lut_params));
+	if (lut_enable) {
+		if (cm->blend_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->blend_func.pwl;
+		else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+					&cm->blend_func,
+					&dpp_base->regamma_params,
+					false);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
-		if (m_lut_params.pwl) {
-			if (mpc->funcs->populate_lut)
-				mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, m_lut_params, lut_bank_a, mpcc_id);
+
+		if (!m_lut_params.pwl) {
+			lut_enable = false;
 		}
-		if (mpc->funcs->program_lut_mode)
-			mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut1d_xable && m_lut_params.pwl, lut_bank_a, mpcc_id);
+	} else {
+		lut_enable = false;
 	}
 
+	if (mpc->funcs->program_lut_mode)
+		mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id);
+	if (lut_enable && mpc->funcs->populate_lut)
+		mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, &m_lut_params, lut_bank_a, mpcc_id);
+
 	/* Shaper */
-	if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) {
+	lut_enable = cm->flags.bits.shaper_enable;
+	if (lut_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.shaper->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.shaper->pwl;
-		else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->shaper_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
-			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.shaper,
-					&dpp_base->regamma_params, true);
-			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
+			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+					&cm->shaper_func,
+					&dpp_base->shaper_params,
+					true);
+			m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL;
 		}
-		if (m_lut_params.pwl) {
-			if (mpc->funcs->mcm.populate_lut)
-				mpc->funcs->mcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id);
-			if (mpc->funcs->program_lut_mode)
-				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_ENABLE, lut_bank_a, mpcc_id);
+		if (!m_lut_params.pwl) {
+			lut_enable = false;
 		}
+	} else {
+		lut_enable = false;
 	}
 
-	/* 3DLUT */
-	switch (lut3d_src) {
-	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
-		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (hubp->funcs->hubp_enable_3dlut_fl)
-			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
-
-		if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) {
-			m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d;
-			if (mpc->funcs->populate_lut)
-				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id);
-			if (mpc->funcs->program_lut_mode)
-				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a,
-						mpcc_id);
-		}
-		break;
-		case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
-		switch (mcm_luts.lut3d_data.gpu_mem_params.size) {
-		case DC_CM2_GPU_MEM_SIZE_171717:
-			width = hubp_3dlut_fl_width_17;
-			break;
-		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
-			width = hubp_3dlut_fl_width_transformed;
-			break;
-		default:
-			//TODO: handle default case
-			break;
-		}
+	if (mpc->funcs->program_lut_mode)
+		mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id);
+	if (lut_enable && mpc->funcs->populate_lut)
+		mpc->funcs->populate_lut(mpc, MCM_LUT_SHAPER, &m_lut_params, lut_bank_a, mpcc_id);
 
-		//check for support
-		if (mpc->funcs->mcm.is_config_supported &&
-			!mpc->funcs->mcm.is_config_supported(width))
-			break;
+	/* NOTE: Toggling from DMA->Host is not supported atomically as hardware
+	 * blocks writes until 3DLUT FL mode is cleared from HUBP on VUpdate.
+	 * Expectation is either option is used consistently.
+	 */
 
-		if (mpc->funcs->program_lut_read_write_control)
-			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id);
+	/* 3DLUT */
+	lut_enable = cm->flags.bits.lut3d_enable;
+	if (lut_enable && cm->flags.bits.lut3d_dma_enable) {
+		/* Fast (DMA) Load Mode */
+		/* MPC */
 		if (mpc->funcs->program_lut_mode)
-			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id);
-
-		if (hubp->funcs->hubp_program_3dlut_fl_addr)
-			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr);
+			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut_enable, lut_bank_a, cm->lut3d_dma.size, mpcc_id);
 
-		if (mpc->funcs->mcm.program_bit_depth)
-			mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id);
-
-		switch (mcm_luts.lut3d_data.gpu_mem_params.layout) {
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
-			mode = hubp_3dlut_fl_mode_native_1;
-			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
-			break;
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
-			mode = hubp_3dlut_fl_mode_native_2;
-			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
-			break;
-		case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
-			mode = hubp_3dlut_fl_mode_transform;
-			addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
-			break;
-		default:
-			mode = hubp_3dlut_fl_mode_disable;
-			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
-			break;
-		}
-		if (hubp->funcs->hubp_program_3dlut_fl_mode)
-			hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode);
+		/* only supports 12 bit */
+		if (mpc->funcs->program_lut_read_write_control)
+			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, 12, mpcc_id);
 
-		if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode)
-			hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode);
+		if (mpc->funcs->update_3dlut_fast_load_select)
+			mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst);
 
-		switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) {
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
-			format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
-			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
-			format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
-			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
-			format = hubp_3dlut_fl_format_float_fp1_5_10;
-			break;
-		}
-		if (hubp->funcs->hubp_program_3dlut_fl_format)
-			hubp->funcs->hubp_program_3dlut_fl_format(hubp, format);
-		if (hubp->funcs->hubp_update_3dlut_fl_bias_scale &&
-				mpc->funcs->mcm.program_bias_scale) {
-			mpc->funcs->mcm.program_bias_scale(mpc,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale,
-				mpcc_id);
-			hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale);
-		}
-
-		//navi 4x has a bug and r and blue are swapped and need to be worked around here in
-		//TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x
-		switch (mcm_luts.lut3d_data.gpu_mem_params.component_order) {
-		case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA:
-		default:
-			crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
-			crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-			crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-			break;
-		}
+		/* HUBP */
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
 
 		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
-			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp,
-					crossbar_bit_slice_cr_r,
-					crossbar_bit_slice_y_g,
-					crossbar_bit_slice_cb_b);
-
-		if (mpc->funcs->mcm.program_lut_read_write_control)
-			mpc->funcs->mcm.program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, true, mpcc_id);
-
-		if (mpc->funcs->mcm.program_3dlut_size)
-			mpc->funcs->mcm.program_3dlut_size(mpc, width, mpcc_id);
+			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, cm->lut3d_dma.format);
 
-		if (mpc->funcs->update_3dlut_fast_load_select)
-			mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst);
+		if (hubp->funcs->hubp_program_3dlut_fl_addr)
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
 
-		if (hubp->funcs->hubp_enable_3dlut_fl)
+		if (hubp->funcs->hubp_enable_3dlut_fl) {
 			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
-		else {
-			if (mpc->funcs->program_lut_mode) {
-				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
-				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
-				mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
-			}
+		} else {
+			/* GPU memory only supports fast load path */
+			BREAK_TO_DEBUGGER();
+			lut_enable = false;
+			result = false;
 		}
-		break;
+	} else {
+		/* Legacy (Host) Load Mode */
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
 
-	}
-}
+		if (cm->flags.bits.lut3d_enable && cm->lut3d_func.state.bits.initialized) {
+			m_lut_params.lut3d = &cm->lut3d_func.lut_3d;
+		} else {
+			lut_enable = false;
+		}
 
-void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx)
-{
-	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+		/* MPC */
+		if (mpc->funcs->program_lut_mode)
+			mpc->funcs->program_lut_mode(mpc,
+					MCM_LUT_3DLUT,
+					lut_enable,
+					lut_bank_a,
+					cm->lut3d_func.lut_3d.use_tetrahedral_9 ? CM_LUT_SIZE_999 : CM_LUT_SIZE_171717,
+					mpcc_id);
+
+		if (lut_enable) {
+			if (mpc->funcs->program_lut_read_write_control)
+				mpc->funcs->program_lut_read_write_control(mpc,
+						MCM_LUT_3DLUT,
+						lut_bank_a,
+						cm->lut3d_func.lut_3d.use_12bits ? 12 : 10,
+						mpcc_id);
 
-	if (hubp->funcs->hubp_enable_3dlut_fl) {
-		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
-	}
-}
+			if (mpc->funcs->update_3dlut_fast_load_select)
+				mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, 0xf);
 
-bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
-{
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
-	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
-	struct dc *dc = pipe_ctx->stream_res.opp->ctx->dc;
-	struct mpc *mpc = dc->res_pool->mpc;
-	bool result;
-	const struct pwl_params *lut_params = NULL;
-	bool rval;
+			if (mpc->funcs->populate_lut)
+				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, &m_lut_params, lut_bank_a, mpcc_id);
+		}
 
-	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
-		dcn401_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a);
-		return true;
-	}
+		/* HUBP */
+		memset(&lut3d_dma, 0, sizeof(lut3d_dma));
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &lut3d_dma);
 
-	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
-	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
-	// 1D LUT
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
-		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							       &plane_state->blend_tf,
-							       &dpp_base->regamma_params, false);
-		lut_params = rval ? &dpp_base->regamma_params : NULL;
-	}
-	result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id);
-	lut_params = NULL;
-
-	// Shaper
-	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->in_shaper_func.pwl;
-	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
-		// TODO: dpp_base replace
-		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							       &plane_state->in_shaper_func,
-							       &dpp_base->shaper_params, true);
-		lut_params = rval ? &dpp_base->shaper_params : NULL;
-	}
-	result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
-
-	// 3D
-	if (mpc->funcs->program_3dlut) {
-		if (plane_state->lut3d_func.state.bits.initialized == 1)
-			result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
-		else
-			result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
 	}
 
 	return result;
@@ -982,6 +865,8 @@ void dcn401_enable_stream(struct pipe_ctx *pipe_ctx)
 		}
 	}
 
+	link_hwss->setup_stream_attribute(pipe_ctx);
+
 	if (dc->res_pool->dccg->funcs->set_pixel_rate_div) {
 		dc->res_pool->dccg->funcs->set_pixel_rate_div(
 			dc->res_pool->dccg,
@@ -1206,6 +1091,25 @@ void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	x_pos = pos_cpy.x - param.recout.x;
 	y_pos = pos_cpy.y - param.recout.y;
 
+	/**
+	 * If the cursor position is negative after recout adjustment, we need
+	 * to shift the hotspot to compensate and clamp position to 0. This
+	 * handles the case where cursor straddles the left/top edge of an
+	 * overlay plane - the cursor is partially visible and needs correct
+	 * hotspot adjustment to render the visible portion.
+	 */
+	if (x_pos < 0) {
+		pos_cpy.x_hotspot -= x_pos;
+		if (hubp->curs_attr.attribute_flags.bits.ENABLE_MAGNIFICATION)
+			adjust_hotspot_between_slices_for_2x_magnify(hubp->curs_attr.width, &pos_cpy);
+		x_pos = 0;
+	}
+
+	if (y_pos < 0) {
+		pos_cpy.y_hotspot -= y_pos;
+		y_pos = 0;
+	}
+
 	recout_x_pos = x_pos - pos_cpy.x_hotspot;
 	recout_y_pos = y_pos - pos_cpy.y_hotspot;
 
@@ -1468,7 +1372,7 @@ void dcn401_optimize_bandwidth(
 			dc->clk_mgr,
 			context,
 			true);
-	if (context->bw_ctx.bw.dcn.clk.zstate_support == DCN_ZSTATE_SUPPORT_ALLOW) {
+	if (context->bw_ctx.bw.dcn.clk.zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW) {
 		for (i = 0; i < dc->res_pool->pipe_count; ++i) {
 			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
@@ -1476,7 +1380,7 @@ void dcn401_optimize_bandwidth(
 				&& pipe_ctx->stream->adjust.v_total_min == pipe_ctx->stream->adjust.v_total_max
 				&& pipe_ctx->stream->adjust.v_total_max > pipe_ctx->stream->timing.v_total)
 					pipe_ctx->plane_res.hubp->funcs->program_extended_blank(pipe_ctx->plane_res.hubp,
-						pipe_ctx->dlg_regs.min_dst_y_next_start);
+						pipe_ctx->hubp_regs.dlg_regs.min_dst_y_next_start);
 		}
 	}
 }
@@ -1911,10 +1815,9 @@ void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx)
 
 	for (odm_pipe = pipe_ctx; odm_pipe != NULL; odm_pipe = odm_pipe->next_odm_pipe) {
 		for (mpc_pipe = odm_pipe; mpc_pipe != NULL; mpc_pipe = mpc_pipe->bottom_pipe) {
-			if (mpc_pipe->plane_state && mpc_pipe->plane_state->mcm_luts.lut3d_data.lut3d_src
-						== DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM
-					&& mpc_pipe->plane_state->mcm_shaper_3dlut_setting
-						== DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT) {
+			if (mpc_pipe->plane_state &&
+					mpc_pipe->plane_state->cm.flags.bits.lut3d_enable &&
+					mpc_pipe->plane_state->cm.flags.bits.lut3d_dma_enable) {
 				wa_pipes[wa_pipe_ct++] = mpc_pipe;
 			}
 		}
@@ -1963,7 +1866,6 @@ void dcn401_reset_back_end_for_pipe(
 	struct dc_link *link = pipe_ctx->stream->link;
 	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
 
-	DC_LOGGER_INIT(dc->ctx->logger);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
 		pipe_ctx->stream = NULL;
 		return;
@@ -2042,6 +1944,22 @@ void dcn401_reset_back_end_for_pipe(
 	DC_LOG_DEBUG("Reset back end for pipe %d, tg:%d\n",
 					pipe_ctx->pipe_idx, pipe_ctx->stream_res.tg->inst);
 }
+static void dc_hwss_disable_otg_pwa(struct dc *dc)
+{
+	if (dc->debug.enable_otg_frame_sync_pwa) {
+		int i;
+
+		/*reset all the otg*/
+		for (i = dc->res_pool->timing_generator_count - 1; i >= 0 ; i--) {
+			struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+			if (tg->funcs->disable_otg_pwa) {
+				tg->funcs->disable_otg_pwa(tg);
+				DC_LOG_DC("otg frame sync pwa disabled on otg%d\n", tg->inst);
+			}
+		}
+	}
+}
 
 void dcn401_reset_hw_ctx_wrap(
 		struct dc *dc,
@@ -2050,6 +1968,7 @@ void dcn401_reset_hw_ctx_wrap(
 	int i;
 	struct dce_hwseq *hws = dc->hwseq;
 
+	dc_hwss_disable_otg_pwa(dc);
 	/* Reset Back End*/
 	for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
 		struct pipe_ctx *pipe_ctx_old =
@@ -2240,6 +2159,10 @@ void dcn401_program_pipe(
 			pipe_ctx->stream_res.test_pattern_params.height,
 			pipe_ctx->stream_res.test_pattern_params.offset);
 	}
+	if (pipe_ctx->plane_state
+		&& pipe_ctx->plane_state->update_flags.bits.cm_hist_change
+		&& hws->funcs.program_cm_hist)
+		hws->funcs.program_cm_hist(dc, pipe_ctx, pipe_ctx->plane_state);
 }
 
 /*
@@ -2391,6 +2314,13 @@ void dcn401_program_pipe_sequence(
 			pipe_ctx->stream_res.test_pattern_params.offset);
 	}
 
+	if (pipe_ctx->plane_state
+			&& pipe_ctx->plane_state->update_flags.bits.cm_hist_change
+			&& hws->funcs.program_cm_hist) {
+
+		hwss_add_dpp_program_cm_hist(seq_state, pipe_ctx->plane_res.dpp,
+			pipe_ctx->plane_state->cm_hist_control, pipe_ctx->plane_state->color_space);
+	}
 }
 
 void dcn401_program_front_end_for_ctx(
@@ -2403,8 +2333,6 @@ void dcn401_program_front_end_for_ctx(
 	struct dce_hwseq *hws = dc->hwseq;
 	struct pipe_ctx *pipe = NULL;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
-
 	if (resource_is_pipe_topology_changed(dc->current_state, context))
 		resource_log_pipe_topology_update(dc, context);
 
@@ -2568,8 +2496,6 @@ void dcn401_post_unlock_program_front_end(
 	struct dce_hwseq *hwseq = dc->hwseq;
 	int i;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
-
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
 		if (resource_is_pipe_type(&dc->current_state->res_ctx.pipe_ctx[i], OPP_HEAD) &&
 			!resource_is_pipe_type(&context->res_ctx.pipe_ctx[i], OPP_HEAD))
@@ -2949,8 +2875,6 @@ void dcn401_plane_atomic_power_down(struct dc *dc,
 	struct dce_hwseq *hws = dc->hwseq;
 	uint32_t org_ip_request_cntl = 0;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
-
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 		if (org_ip_request_cntl == 0)
@@ -3042,8 +2966,6 @@ void dcn401_plane_atomic_power_down_sequence(struct dc *dc,
 	struct dce_hwseq *hws = dc->hwseq;
 	uint32_t org_ip_request_cntl = 0;
 
-	DC_LOGGER_INIT(dc->ctx->logger);
-
 	/* Check and set DC_IP_REQUEST_CNTL if needed */
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
new file mode 100644
index 0000000000000..8e12dc1297c4d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
@@ -0,0 +1,1091 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "dm_helpers.h"
+#include "core_types.h"
+#include "resource.h"
+#include "dccg.h"
+#include "dce/dce_hwseq.h"
+#include "dcn30/dcn30_cm_common.h"
+#include "reg_helper.h"
+#include "abm.h"
+#include "hubp.h"
+#include "dchubbub.h"
+#include "timing_generator.h"
+#include "opp.h"
+#include "ipp.h"
+#include "mpc.h"
+#include "mcif_wb.h"
+#include "dc_dmub_srv.h"
+#include "link_hwss.h"
+#include "dpcd_defs.h"
+#include "dcn401/dcn401_hwseq.h"
+#include "dcn42_hwseq.h"
+#include "clk_mgr.h"
+#include "dsc.h"
+#include "dcn20/dcn20_optc.h"
+#include "dce/dmub_hw_lock_mgr.h"
+#include "dcn42/dcn42_resource.h"
+#include "link_service.h"
+#include "../dcn10/dcn10_hwseq.h"
+#include "../dcn20/dcn20_hwseq.h"
+#include "dc_state_priv.h"
+#include "dc_stream_priv.h"
+#include "dcn35/dcn35_hwseq.h"
+#include "dcn42/dcn42_hwseq.h"
+#include "dce/dmub_hw_lock_mgr.h"
+#include "dio/dcn10/dcn10_dio.h"
+
+#define DC_LOGGER \
+	ctx->logger
+
+#define CTX \
+	hws->ctx
+
+#define REG(reg)\
+	hws->regs->reg
+
+#undef FN
+#define FN(reg_name, field_name) \
+	hws->shifts->field_name, hws->masks->field_name
+
+
+static void print_pg_status(struct dc *dc, const char *debug_func, const char *debug_log)
+{
+	if (dc->debug.enable_pg_cntl_debug_logs && dc->res_pool->pg_cntl) {
+		if (dc->res_pool->pg_cntl->funcs->print_pg_status)
+			dc->res_pool->pg_cntl->funcs->print_pg_status(dc->res_pool->pg_cntl, debug_func, debug_log);
+	}
+}
+void dcn42_init_hw(struct dc *dc)
+{
+	struct abm **abms = dc->res_pool->multiple_abms;
+	struct dce_hwseq *hws = dc->hwseq;
+	struct dc_bios *dcb = dc->ctx->dc_bios;
+	struct resource_pool *res_pool = dc->res_pool;
+	int i;
+	int edp_num;
+	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
+	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
+	int current_dchub_ref_freq = 0;
+
+	if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->init_clocks) {
+		dc->clk_mgr->funcs->init_clocks(dc->clk_mgr);
+
+		// mark dcmode limits present if any clock has distinct AC and DC values from SMU
+		dc->caps.dcmode_power_limits_present = dc->clk_mgr->funcs->is_dc_mode_present &&
+				dc->clk_mgr->funcs->is_dc_mode_present(dc->clk_mgr);
+	}
+
+	// Initialize the dccg
+	if (res_pool->dccg->funcs->dccg_init)
+		res_pool->dccg->funcs->dccg_init(res_pool->dccg);
+
+	// Disable DMUB Initialization until IPS state programming is finalized
+	//if (!dcb->funcs->is_accelerated_mode(dcb)) {
+	//	hws->funcs.bios_golden_init(dc);
+	//}
+
+	// Set default OPTC memory power states
+	if (dc->debug.enable_mem_low_power.bits.optc) {
+		// Shutdown when unassigned and light sleep in VBLANK
+		REG_SET_2(ODM_MEM_PWR_CTRL3, 0, ODM_MEM_UNASSIGNED_PWR_MODE, 3, ODM_MEM_VBLANK_PWR_MODE, 1);
+	}
+
+	if (dc->debug.enable_mem_low_power.bits.vga) {
+		// Power down VGA memory
+		REG_UPDATE(MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, 1);
+	}
+
+	if (dc->ctx->dc_bios->fw_info_valid) {
+		res_pool->ref_clocks.xtalin_clock_inKhz =
+				dc->ctx->dc_bios->fw_info.pll_info.crystal_frequency;
+
+		if (res_pool->hubbub) {
+			(res_pool->dccg->funcs->get_dccg_ref_freq)(res_pool->dccg,
+					dc->ctx->dc_bios->fw_info.pll_info.crystal_frequency,
+					&res_pool->ref_clocks.dccg_ref_clock_inKhz);
+
+			current_dchub_ref_freq = res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000;
+
+			(res_pool->hubbub->funcs->get_dchub_ref_freq)(res_pool->hubbub,
+					res_pool->ref_clocks.dccg_ref_clock_inKhz,
+					&res_pool->ref_clocks.dchub_ref_clock_inKhz);
+		} else {
+			// Not all ASICs have DCCG sw component
+			res_pool->ref_clocks.dccg_ref_clock_inKhz =
+					res_pool->ref_clocks.xtalin_clock_inKhz;
+			res_pool->ref_clocks.dchub_ref_clock_inKhz =
+					res_pool->ref_clocks.xtalin_clock_inKhz;
+		}
+	} else
+		ASSERT_CRITICAL(false);
+
+	for (i = 0; i < dc->link_count; i++) {
+		/* Power up AND update implementation according to the
+		 * required signal (which may be different from the
+		 * default signal on connector).
+		 */
+		struct dc_link *link = dc->links[i];
+
+		if (link->ep_type != DISPLAY_ENDPOINT_PHY)
+			continue;
+
+		link->link_enc->funcs->hw_init(link->link_enc);
+
+		/* Check for enabled DIG to identify enabled display */
+		if (link->link_enc->funcs->is_dig_enabled &&
+			link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
+			link->link_status.link_active = true;
+			link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+			if (link->link_enc->funcs->fec_is_active &&
+					link->link_enc->funcs->fec_is_active(link->link_enc))
+				link->fec_state = dc_link_fec_enabled;
+		}
+	}
+
+	/* enable_power_gating_plane before dsc_pg_control because
+	 * FORCEON = 1 with hw default value on bootup, resume from s3
+	 */
+	if (hws->funcs.enable_power_gating_plane)
+		hws->funcs.enable_power_gating_plane(dc->hwseq, true);
+
+	/* we want to turn off all dp displays before doing detection */
+	dc->link_srv->blank_all_dp_displays(dc);
+
+	/* If taking control over from VBIOS, we may want to optimize our first
+	 * mode set, so we need to skip powering down pipes until we know which
+	 * pipes we want to use.
+	 * Otherwise, if taking control is not possible, we need to power
+	 * everything down.
+	 */
+	if (dcb->funcs->is_accelerated_mode(dcb) || !dc->config.seamless_boot_edp_requested) {
+		/* Disable boot optimizations means power down everything including PHY, DIG,
+		 * and OTG (i.e. the boot is not optimized because we do a full power down).
+		 */
+		if (dc->hwss.enable_accelerated_mode && dc->debug.disable_boot_optimizations)
+			dc->hwss.enable_accelerated_mode(dc, dc->current_state);
+		else
+			hws->funcs.init_pipes(dc, dc->current_state);
+
+		if (dc->res_pool->hubbub->funcs->allow_self_refresh_control)
+			dc->res_pool->hubbub->funcs->allow_self_refresh_control(dc->res_pool->hubbub,
+					!dc->res_pool->hubbub->ctx->dc->debug.disable_stutter);
+
+	}
+
+	/* In headless boot cases, DIG may be turned
+	 * on which causes HW/SW discrepancies.
+	 * To avoid this, power down hardware on boot
+	 * if DIG is turned on and seamless boot not enabled
+	 */
+	if (!dc->config.seamless_boot_edp_requested) {
+		struct dc_link *edp_links[MAX_NUM_EDP];
+		struct dc_link *edp_link;
+
+		dc_get_edp_links(dc, edp_links, &edp_num);
+		if (edp_num) {
+			for (i = 0; i < edp_num; i++) {
+				edp_link = edp_links[i];
+				if (edp_link->link_enc->funcs->is_dig_enabled &&
+						edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc) &&
+						dc->hwss.edp_backlight_control &&
+						hws->funcs.power_down &&
+						dc->hwss.edp_power_control) {
+					dc->hwss.edp_backlight_control(edp_link, false);
+					hws->funcs.power_down(dc);
+					dc->hwss.edp_power_control(edp_link, false);
+				}
+			}
+		} else {
+			for (i = 0; i < dc->link_count; i++) {
+				struct dc_link *link = dc->links[i];
+
+				if (link->link_enc->funcs->is_dig_enabled &&
+						link->link_enc->funcs->is_dig_enabled(link->link_enc) &&
+						hws->funcs.power_down) {
+					hws->funcs.power_down(dc);
+					break;
+				}
+
+			}
+		}
+	}
+
+	for (i = 0; i < res_pool->audio_count; i++) {
+		struct audio *audio = res_pool->audios[i];
+
+		audio->funcs->hw_init(audio);
+	}
+
+	for (i = 0; i < dc->link_count; i++) {
+		struct dc_link *link = dc->links[i];
+
+		if (link->panel_cntl) {
+			backlight = link->panel_cntl->funcs->hw_init(link->panel_cntl);
+			user_level = link->panel_cntl->stored_backlight_registers.USER_LEVEL;
+		}
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (abms[i] != NULL && abms[i]->funcs != NULL)
+			abms[i]->funcs->abm_init(abms[i], backlight, user_level);
+	}
+
+	/* power AFMT HDMI memory TODO: may move to dis/en output save power*/
+	if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)
+		dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);
+
+	if (!dc->debug.disable_clock_gate) {
+		/* enable all DCN clock gating */
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
+
+		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+	}
+
+	dcn401_setup_hpo_hw_control(hws, true);
+
+	if (!dcb->funcs->is_accelerated_mode(dcb) && dc->res_pool->hubbub->funcs->init_watermarks)
+		dc->res_pool->hubbub->funcs->init_watermarks(dc->res_pool->hubbub);
+
+	if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->notify_wm_ranges)
+		dc->clk_mgr->funcs->notify_wm_ranges(dc->clk_mgr);
+
+	if (dc->res_pool->hubbub->funcs->force_pstate_change_control)
+		dc->res_pool->hubbub->funcs->force_pstate_change_control(
+				dc->res_pool->hubbub, false, false);
+
+	if (dc->res_pool->hubbub->funcs->init_crb)
+		dc->res_pool->hubbub->funcs->init_crb(dc->res_pool->hubbub);
+
+	if (dc->res_pool->hubbub->funcs->set_request_limit && dc->config.sdpif_request_limit_words_per_umc > 0)
+		dc->res_pool->hubbub->funcs->set_request_limit(dc->res_pool->hubbub, dc->clk_mgr->bw_params->num_channels, dc->config.sdpif_request_limit_words_per_umc);
+
+	// Get DMCUB capabilities
+	if (dc->ctx->dmub_srv) {
+		dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv);
+		dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
+		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver > 0;
+		dc->caps.dmub_caps.fams_ver = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
+		dc->debug.fams2_config.bits.enable &=
+				dc->caps.dmub_caps.fams_ver == dc->debug.fams_version.ver; // sw & fw fams versions must match for support
+		if ((!dc->debug.fams2_config.bits.enable && dc->res_pool->funcs->update_bw_bounding_box)
+			|| res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 != current_dchub_ref_freq) {
+			/* update bounding box if FAMS2 disabled, or if dchub clk has changed */
+			if (dc->clk_mgr)
+				dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params);
+		}
+	}
+	if (dc->res_pool->pg_cntl) {
+		if (dc->res_pool->pg_cntl->funcs->init_pg_status)
+			dc->res_pool->pg_cntl->funcs->init_pg_status(dc->res_pool->pg_cntl);
+	}
+	print_pg_status(dc, __func__, ": after init_pg_status");
+}
+
+void dcn42_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	struct mpcc_blnd_cfg blnd_cfg = {0};
+	bool per_pixel_alpha = pipe_ctx->plane_state->per_pixel_alpha;
+	int mpcc_id;
+	struct mpcc *new_mpcc;
+	struct mpc *mpc = dc->res_pool->mpc;
+	struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
+
+
+	blnd_cfg.overlap_only = false;
+	blnd_cfg.global_gain = 0xfff;
+
+	if (per_pixel_alpha) {
+		blnd_cfg.pre_multiplied_alpha = pipe_ctx->plane_state->pre_multiplied_alpha;
+		if (pipe_ctx->plane_state->global_alpha) {
+			blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN;
+			blnd_cfg.global_gain = pipe_ctx->plane_state->global_alpha_value;
+		} else {
+			blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
+		}
+	} else {
+		blnd_cfg.pre_multiplied_alpha = false;
+		blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
+	}
+
+	if (pipe_ctx->plane_state->global_alpha)
+		blnd_cfg.global_alpha = pipe_ctx->plane_state->global_alpha_value;
+	else
+		blnd_cfg.global_alpha = 0xfff;
+
+	blnd_cfg.background_color_bpc = 4;
+	blnd_cfg.bottom_gain_mode = 0;
+	blnd_cfg.top_gain = 0x1f000;
+	blnd_cfg.bottom_inside_gain = 0x1f000;
+	blnd_cfg.bottom_outside_gain = 0x1f000;
+
+	if (pipe_ctx->plane_state->format
+			== SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA)
+		blnd_cfg.pre_multiplied_alpha = false;
+
+	/*
+	 * TODO: remove hack
+	 * Note: currently there is a bug in init_hw such that
+	 * on resume from hibernate, BIOS sets up MPCC0, and
+	 * we do mpcc_remove but the mpcc cannot go to idle
+	 * after remove. This cause us to pick mpcc1 here,
+	 * which causes a pstate hang for yet unknown reason.
+	 */
+	mpcc_id = hubp->inst;
+
+	/* If there is no full update, don't need to touch MPC tree*/
+	if (!pipe_ctx->plane_state->update_flags.bits.full_update &&
+		!pipe_ctx->update_flags.bits.mpcc) {
+		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
+		dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
+		return;
+	}
+
+	/* check if this MPCC is already being used */
+	new_mpcc = mpc->funcs->get_mpcc_for_dpp(mpc_tree_params, mpcc_id);
+	/* remove MPCC if being used */
+	if (new_mpcc != NULL)
+		mpc->funcs->remove_mpcc(mpc, mpc_tree_params, new_mpcc);
+	else
+		if (dc->debug.sanity_checks)
+			mpc->funcs->assert_mpcc_idle_before_connect(
+					dc->res_pool->mpc, mpcc_id);
+
+	/* Call MPC to insert new plane */
+	new_mpcc = mpc->funcs->insert_plane(dc->res_pool->mpc,
+			mpc_tree_params,
+			&blnd_cfg,
+			NULL,
+			NULL,
+			hubp->inst,
+			mpcc_id);
+	dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
+
+	ASSERT(new_mpcc != NULL);
+	hubp->opp_id = pipe_ctx->stream_res.opp->inst;
+	hubp->mpcc_id = mpcc_id;
+}
+
+void dcn42_program_cm_hist(
+	struct dc *dc,
+	struct pipe_ctx *pipe_ctx,
+	const struct dc_plane_state *plane_state)
+{
+	struct dpp *dpp = pipe_ctx->plane_res.dpp;
+
+	if (dpp && dpp->funcs->dpp_cm_hist_control)
+		dpp->funcs->dpp_cm_hist_control(dpp,
+			plane_state->cm_hist_control, plane_state->color_space);
+}
+
+static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc)
+{
+	if (mpc->funcs->rmcm.power_on_shaper_3dlut &&
+		mpc->funcs->rmcm.fl_3dlut_configure &&
+		hubp->funcs->hubp_program_3dlut_fl_config)
+		return true;
+
+	return false;
+}
+
+bool dcn42_program_rmcm_luts(
+	struct hubp *hubp,
+	struct pipe_ctx *pipe_ctx,
+	const struct dc_plane_cm *cm,
+	struct mpc *mpc,
+	int mpcc_id)
+{
+	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	union mcm_lut_params m_lut_params = {0};
+
+	struct dc *dc = hubp->ctx->dc;
+	struct mpc_fl_3dlut_config mpc_fl_config;
+
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	bool bypass_rmcm_shaper = false;
+	// true->false when it can be allocated at DI time
+	struct dc_rmcm_3dlut *rmcm_3dlut = dc_stream_get_3dlut_for_stream(dc, stream, false);
+
+	bool lut_bank_a = true; // TODO get from HW
+
+	//check to see current pipe is part of a stream with allocated rmcm 3dlut
+	if (!rmcm_3dlut)
+		return false;
+
+	/* Shaper */
+	if (cm->flags.bits.shaper_enable) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+
+		if (cm->shaper_func.type == TF_TYPE_HWPWL) {
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		} else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+			ASSERT(false);
+			cm_helper_translate_curve_to_hw_format(
+					dc->ctx,
+					&cm->shaper_func,
+					&dpp_base->shaper_params, true);
+			m_lut_params.pwl = &dpp_base->shaper_params;
+		}
+		if (m_lut_params.pwl) {
+			if (mpc->funcs->rmcm.populate_lut)
+				mpc->funcs->rmcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id);
+			if (mpc->funcs->rmcm.program_lut_mode)
+				mpc->funcs->rmcm.program_lut_mode(mpc, !bypass_rmcm_shaper, lut_bank_a, mpcc_id);
+		} else {
+			//RMCM 3dlut won't work without its shaper
+			return false;
+		}
+	}
+
+	/* 3DLUT */
+	if (!cm->flags.bits.lut3d_dma_enable) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		// Don't know what to do in this case.
+	} else {
+		if (!dc_is_rmcm_3dlut_supported(hubp, mpc))
+			return false;
+
+		//seems to be only for the MCM
+		mpc_fl_config.enabled			= cm->flags.bits.lut3d_enable;
+		mpc_fl_config.size	            = cm->lut3d_dma.size;
+		mpc_fl_config.select_lut_bank_a = lut_bank_a;
+		mpc_fl_config.bit_depth		    = 0;
+		mpc_fl_config.hubp_index		= hubp->inst;
+		mpc_fl_config.bias	= cm->lut3d_dma.bias;
+		mpc_fl_config.scale	= cm->lut3d_dma.scale;
+
+		//1. power down the block
+		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, false);
+
+		//2. program RMCM - 3dlut reg programming
+		mpc->funcs->rmcm.fl_3dlut_configure(mpc, &mpc_fl_config, mpcc_id);
+
+		/* HUBP */
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_addr)
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
+
+		//3. power on the block
+		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, true);
+	}
+
+	return true;
+}
+
+bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
+				const struct dc_plane_state *plane_state)
+{
+	struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	const struct dc_plane_cm *cm = &plane_state->cm;
+	struct mpc *mpc = dc->res_pool->mpc;
+	int mpcc_id = hubp->inst;
+	bool result;
+
+	/* MCM */
+	result = dcn401_set_mcm_luts(pipe_ctx, plane_state);
+
+	/* RMCM */
+	if (cm->flags.bits.rmcm_enable && cm->flags.bits.lut3d_dma_enable) {
+		/* TODO - move RMCM to its own block */
+		dcn42_program_rmcm_luts(
+			hubp,
+			pipe_ctx,
+			cm,
+			mpc,
+			mpcc_id);
+	}
+
+	return result;
+}
+void dcn42_hardware_release(struct dc *dc)
+{
+	dcn35_hardware_release(dc);
+
+}
+static int count_active_streams(const struct dc *dc)
+{
+	int i, count = 0;
+
+	for (i = 0; i < dc->current_state->stream_count; ++i) {
+		struct dc_stream_state *stream = dc->current_state->streams[i];
+
+		if (stream && (!stream->dpms_off || dc->config.disable_ips_in_dpms_off))
+			count += 1;
+	}
+
+	return count;
+}
+
+void dcn42_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state)
+{
+	bool hpo_frl_stream_enc_acquired = false;
+	bool hpo_dp_stream_enc_acquired = false;
+	int i = 0, j = 0;
+
+	memset(update_state, 0, sizeof(struct pg_block_update));
+
+	update_state->pg_res_update[PG_DIO] = true;
+
+	for (i = 0; i < dc->res_pool->hpo_dp_stream_enc_count; i++) {
+		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[i] &&
+				dc->res_pool->hpo_dp_stream_enc[i]) {
+			hpo_dp_stream_enc_acquired = true;
+			break;
+		}
+	}
+
+	if (!hpo_frl_stream_enc_acquired && !hpo_dp_stream_enc_acquired)
+		update_state->pg_res_update[PG_HPO] = true;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+		for (j = 0; j < PG_HW_PIPE_RESOURCES_NUM_ELEMENT; j++)
+			update_state->pg_pipe_res_update[j][i] = true;
+
+		if (!pipe_ctx)
+			continue;
+
+		if (pipe_ctx->plane_res.hubp)
+			update_state->pg_pipe_res_update[PG_HUBP][pipe_ctx->plane_res.hubp->inst] = false;
+
+		if (pipe_ctx->plane_res.dpp && pipe_ctx->plane_res.hubp)
+			update_state->pg_pipe_res_update[PG_DPP][pipe_ctx->plane_res.hubp->inst] = false;
+
+		if (pipe_ctx->plane_res.dpp || pipe_ctx->stream_res.opp)
+			update_state->pg_pipe_res_update[PG_MPCC][pipe_ctx->plane_res.mpcc_inst] = false;
+
+		if (pipe_ctx->stream_res.dsc) {
+			update_state->pg_pipe_res_update[PG_DSC][pipe_ctx->stream_res.dsc->inst] = false;
+			if (dc->caps.sequential_ono) {
+				update_state->pg_pipe_res_update[PG_HUBP][pipe_ctx->stream_res.dsc->inst] = false;
+				update_state->pg_pipe_res_update[PG_DPP][pipe_ctx->stream_res.dsc->inst] = false;
+			}
+		}
+		if (pipe_ctx->stream_res.opp)
+			update_state->pg_pipe_res_update[PG_OPP][pipe_ctx->stream_res.opp->inst] = false;
+
+		if (pipe_ctx->stream_res.hpo_dp_stream_enc)
+			update_state->pg_pipe_res_update[PG_DPSTREAM][pipe_ctx->stream_res.hpo_dp_stream_enc->inst] = false;
+
+		if (pipe_ctx->link_res.dio_link_enc) {
+			update_state->pg_res_update[PG_DIO] = false;
+		}
+		if (pipe_ctx->link_res.hpo_dp_link_enc) {
+			update_state->pg_res_update[PG_HPO] = false;
+		}
+	}
+
+
+	for (i = 0; i < dc->link_count; i++) {
+		update_state->pg_pipe_res_update[PG_PHYSYMCLK][dc->links[i]->link_enc_hw_inst] = true;
+		if (dc->links[i]->type != dc_connection_none)
+			update_state->pg_pipe_res_update[PG_PHYSYMCLK][dc->links[i]->link_enc_hw_inst] = false;
+	}
+
+	/*domain24 controls all the otg, mpc, opp, as long as one otg is still up, avoid enabling OTG PG*/
+	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+		if (tg && tg->funcs->is_tg_enabled(tg)) {
+			update_state->pg_pipe_res_update[PG_OPTC][i] = false;
+		} else {
+			 // Update pg_pipe_res_enable state
+			if (dc->res_pool->pg_cntl->funcs->optc_pg_control)
+				dc->res_pool->pg_cntl->funcs->optc_pg_control(dc->res_pool->pg_cntl, i, false);
+		}
+	}
+
+}
+
+void dcn42_prepare_bandwidth(
+	struct dc *dc,
+	struct dc_state *context)
+{
+
+	struct pg_block_update pg_update_state;
+
+	if (dc->hwss.calc_blocks_to_ungate) {
+		dc->hwss.calc_blocks_to_ungate(dc, context, &pg_update_state);
+
+		if (dc->hwss.root_clock_control)
+			dc->hwss.root_clock_control(dc, &pg_update_state, true);
+		/*power up required HW block*/
+		if (dc->hwss.hw_block_power_up)
+			dc->hwss.hw_block_power_up(dc, &pg_update_state);
+	}
+
+	dcn401_prepare_bandwidth(dc, context);
+}
+
+void dcn42_optimize_bandwidth(struct dc *dc, struct dc_state *context)
+{
+		struct pg_block_update pg_update_state;
+
+	print_pg_status(dc, __func__, ": before rcg and power up");
+
+	dcn401_optimize_bandwidth(dc, context);
+
+	if (dc->hwss.calc_blocks_to_gate) {
+		dc->hwss.calc_blocks_to_gate(dc, context, &pg_update_state);
+		/*try to power down unused block*/
+		if (dc->hwss.hw_block_power_down)
+			dc->hwss.hw_block_power_down(dc, &pg_update_state);
+
+		if (dc->hwss.root_clock_control)
+			dc->hwss.root_clock_control(dc, &pg_update_state, false);
+	}
+
+	print_pg_status(dc, __func__, ": after rcg and power up");
+
+}
+
+void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state)
+{
+	bool hpo_dp_stream_enc_acquired = false;
+	int i = 0, j = 0;
+
+	memset(update_state, 0, sizeof(struct pg_block_update));
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *cur_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (cur_pipe == NULL || new_pipe == NULL)
+			continue;
+
+		if ((!cur_pipe->plane_state && new_pipe->plane_state) ||
+			(!cur_pipe->stream && new_pipe->stream) ||
+			(cur_pipe->stream != new_pipe->stream && new_pipe->stream)) {
+			// New pipe addition
+			for (j = 0; j < PG_HW_PIPE_RESOURCES_NUM_ELEMENT; j++) {
+				if (j == PG_HUBP && new_pipe->plane_res.hubp)
+					update_state->pg_pipe_res_update[j][new_pipe->plane_res.hubp->inst] = true;
+
+				if (j == PG_DPP && new_pipe->plane_res.dpp)
+					update_state->pg_pipe_res_update[j][new_pipe->plane_res.dpp->inst] = true;
+
+				if (j == PG_MPCC && new_pipe->plane_res.dpp)
+					update_state->pg_pipe_res_update[j][new_pipe->plane_res.mpcc_inst] = true;
+
+				if (j == PG_DSC && new_pipe->stream_res.dsc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.dsc->inst] = true;
+
+				if (j == PG_OPP && new_pipe->stream_res.opp)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.opp->inst] = true;
+
+				if (j == PG_OPTC && new_pipe->stream_res.tg)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.tg->inst] = true;
+
+				if (j == PG_DPSTREAM && new_pipe->stream_res.hpo_dp_stream_enc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.hpo_dp_stream_enc->inst] = true;
+			}
+		} else if (cur_pipe->plane_state == new_pipe->plane_state ||
+				cur_pipe == new_pipe) {
+			//unchanged pipes
+			for (j = 0; j < PG_HW_PIPE_RESOURCES_NUM_ELEMENT; j++) {
+				if (j == PG_HUBP &&
+					cur_pipe->plane_res.hubp != new_pipe->plane_res.hubp &&
+					new_pipe->plane_res.hubp)
+					update_state->pg_pipe_res_update[j][new_pipe->plane_res.hubp->inst] = true;
+
+				if (j == PG_DPP &&
+					cur_pipe->plane_res.dpp != new_pipe->plane_res.dpp &&
+					new_pipe->plane_res.dpp)
+					update_state->pg_pipe_res_update[j][new_pipe->plane_res.dpp->inst] = true;
+
+				if (j == PG_OPP &&
+					cur_pipe->stream_res.opp != new_pipe->stream_res.opp &&
+					new_pipe->stream_res.opp)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.opp->inst] = true;
+
+				if (j == PG_DSC &&
+					cur_pipe->stream_res.dsc != new_pipe->stream_res.dsc &&
+					new_pipe->stream_res.dsc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.dsc->inst] = true;
+
+				if (j == PG_OPTC &&
+					cur_pipe->stream_res.tg != new_pipe->stream_res.tg &&
+					new_pipe->stream_res.tg)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.tg->inst] = true;
+
+				if (j == PG_DPSTREAM &&
+					cur_pipe->stream_res.hpo_dp_stream_enc != new_pipe->stream_res.hpo_dp_stream_enc &&
+					new_pipe->stream_res.hpo_dp_stream_enc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.hpo_dp_stream_enc->inst] = true;
+			}
+		}
+	}
+
+	for (i = 0; i < dc->link_count; i++)
+		if (dc->links[i]->type != dc_connection_none)
+			update_state->pg_pipe_res_update[PG_PHYSYMCLK][dc->links[i]->link_enc_hw_inst] = true;
+
+	for (i = 0; i < dc->res_pool->stream_enc_count; i++) {
+		if (dc->current_state->res_ctx.is_stream_enc_acquired[i]) {
+			update_state->pg_res_update[PG_DIO] = true;
+			break;
+		}
+	}
+	for (i = 0; i < dc->res_pool->hpo_dp_stream_enc_count; i++) {
+		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[i] &&
+				dc->res_pool->hpo_dp_stream_enc[i]) {
+			hpo_dp_stream_enc_acquired = true;
+			break;
+		}
+	}
+
+	if (hpo_dp_stream_enc_acquired)
+		update_state->pg_res_update[PG_HPO] = true;
+
+	if (count_active_streams(dc) > 0) {
+		update_state->pg_res_update[PG_DCCG] = true;
+		update_state->pg_res_update[PG_DCIO] = true;
+		update_state->pg_res_update[PG_DCHUBBUB] = true;
+		update_state->pg_res_update[PG_DCHVM] = true;
+		update_state->pg_res_update[PG_DCOH] = true;
+	}
+}
+/**
+ * dcn42_hw_block_power_down() - power down sequence
+ *
+ * The following sequence describes the ON-OFF (ONO) for power down:
+ *
+ *	ONO Region 4, DCPG 25: hpo
+ *	ONO Region 11, DCPG 3: dchubp3, dpp3
+ *	ONO Region 9, DCPG 2: dchubp2, dpp2
+ *	ONO Region 7, DCPG 1: dchubp1, dpp1
+ *	ONO Region 5, DCPG 0: dchubp0, dpp0
+ *	ONO Region 2, DCPG 23: dchubbub, dchububmem, dchvm
+ *	ONO Region 12, DCPG 19: dsc3
+ *	ONO Region 10, DCPG 18: dsc2
+ *	ONO Region 8, DCPG 17: dsc1
+ *	ONO Region 6, DCPG 16: dsc0
+ *	ONO Region 3, DCPG 24: mpc, opp, optc, dwb
+ *	ONO Region 1, DCPG 26: dio
+ *	ONO Region 0, DCPG 22: dccg dcio dcoh - SKIPPED
+ *
+ *  No seuential ONO power up/down order for DCN42
+ *  Driver PG should only be limited to DCHUBP/DPP, DSC, HPO and DIO, so further optimization can be done
+ *
+ * @dc: Current DC state
+ * @update_state: update PG sequence states for HW block
+ */
+ void dcn42_hw_block_power_down(struct dc *dc,
+	struct pg_block_update *update_state)
+{
+	int i = 0;
+	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
+	bool block_disabled = true;
+
+	if (!pg_cntl)
+		return;
+	if (dc->debug.ignore_pg)
+		return;
+
+	if (update_state->pg_res_update[PG_HPO]) {
+		if (pg_cntl->funcs->hpo_pg_control)
+			pg_cntl->funcs->hpo_pg_control(pg_cntl, false);
+	}
+
+	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			update_state->pg_pipe_res_update[PG_DPP][i]) {
+			if (pg_cntl->funcs->hubp_dpp_pg_control)
+				pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, i, false);
+		}
+	}
+
+	if (update_state->pg_res_update[PG_DCHUBBUB]) {
+		if (pg_cntl->funcs->mem_pg_control)
+			pg_cntl->funcs->mem_pg_control(pg_cntl, false);
+	}
+
+	for (i = dc->res_pool->res_cap->num_dsc-1; i >= 0; i--) {
+		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
+			if (pg_cntl->funcs->dsc_pg_control)
+				pg_cntl->funcs->dsc_pg_control(pg_cntl, i, false);
+		}
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (!update_state->pg_pipe_res_update[PG_MPCC][i] ||
+			!update_state->pg_pipe_res_update[PG_OPP][i] ||
+			!update_state->pg_pipe_res_update[PG_OPTC][i]) {
+				block_disabled = false;
+				break;
+		}
+	}
+
+	if (block_disabled) {
+		if (pg_cntl->funcs->plane_otg_pg_control)
+			pg_cntl->funcs->plane_otg_pg_control(pg_cntl, false);
+	}
+
+	if (update_state->pg_res_update[PG_DIO]) {
+		if (pg_cntl->funcs->dio_pg_control)
+			pg_cntl->funcs->dio_pg_control(pg_cntl, false);
+	}
+
+	if (update_state->pg_res_update[PG_DCCG] &&
+		update_state->pg_res_update[PG_DCIO] &&
+		update_state->pg_res_update[PG_DCOH]) {
+		// Driver PG should not power down DCCG, DCIO, DCOH. This is handled by IPS.
+		if (pg_cntl->funcs->io_clk_pg_control)
+			pg_cntl->funcs->io_clk_pg_control(pg_cntl, false);
+		}
+}
+
+/**
+ * dcn42_hw_block_power_up() - power up sequence
+ *
+ * The following sequence describes the ON-OFF (ONO) for power up:
+ *
+ *	ONO Region 0, DCPG 22: dccg dcio dcoh
+ *  ONO Region 1, DCPG 26: dio
+ *	ONO Region 3, DCPG 24: mpc, opp, optc, dwb
+ *  ONO Region 6, DCPG 16: dsc0
+ *  ONO Region 8, DCPG 17: dsc1
+ *  ONO Region 10, DCPG 18: dsc2
+ *	ONO Region 12, DCPG 19: dsc3
+ *	ONO Region 2, DCPG 23: dchubbub, dchububmem, dchvm
+ *	ONO Region 5, DCPG 0: dchubp0, dpp0
+ *	ONO Region 7, DCPG 1: dchubp1, dpp1
+ *	ONO Region 9, DCPG 2: dchubp2, dpp2
+ *	ONO Region 11, DCPG 3: dchubp3, dpp3
+ *	ONO Region 4, DCPG 25: hpo
+ *
+ *  No sequential power up/down ordering for DCN42
+ *
+ * @dc: Current DC state
+ * @update_state: update PG sequence states for HW block
+ */
+void dcn42_hw_block_power_up(struct dc *dc,
+	struct pg_block_update *update_state)
+{
+	int i = 0;
+	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
+	bool block_enabled = false;
+
+	if (!pg_cntl)
+		return;
+	if (dc->debug.ignore_pg)
+		return;
+
+	if (update_state->pg_res_update[PG_DCCG] ||
+		update_state->pg_res_update[PG_DCIO] ||
+		update_state->pg_res_update[PG_DCOH]) {
+		if (pg_cntl->funcs->io_clk_pg_control)
+			pg_cntl->funcs->io_clk_pg_control(pg_cntl, true);
+	}
+
+	if (update_state->pg_res_update[PG_DIO]) {
+		if (pg_cntl->funcs->dio_pg_control)
+			pg_cntl->funcs->dio_pg_control(pg_cntl, true);
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (update_state->pg_pipe_res_update[PG_MPCC][i] ||
+			update_state->pg_pipe_res_update[PG_OPP][i] ||
+			update_state->pg_pipe_res_update[PG_OPTC][i]) {
+				block_enabled = true;
+				break;
+		}
+	}
+	if (block_enabled) {
+		if (pg_cntl->funcs->plane_otg_pg_control)
+			pg_cntl->funcs->plane_otg_pg_control(pg_cntl, true);
+	}
+
+	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
+			if (pg_cntl->funcs->dsc_pg_control)
+				pg_cntl->funcs->dsc_pg_control(pg_cntl, i, true);
+		}
+	}
+
+	if (update_state->pg_res_update[PG_DCHUBBUB]) {
+		if (pg_cntl->funcs->mem_pg_control)
+			pg_cntl->funcs->mem_pg_control(pg_cntl, true);
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			update_state->pg_pipe_res_update[PG_DPP][i]) {
+			if (pg_cntl->funcs->hubp_dpp_pg_control)
+				pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, i, true);
+		}
+	}
+	if (update_state->pg_res_update[PG_HPO]) {
+		if (pg_cntl->funcs->hpo_pg_control)
+			pg_cntl->funcs->hpo_pg_control(pg_cntl, true);
+	}
+}
+void dcn42_root_clock_control(struct dc *dc,
+	struct pg_block_update *update_state, bool power_on)
+{
+	int i = 0;
+	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
+
+	if (!pg_cntl)
+		return;
+	/*enable root clock first when power up*/
+	if (power_on) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+				update_state->pg_pipe_res_update[PG_DPP][i]) {
+				if (dc->hwseq->funcs.dpp_root_clock_control)
+					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
+			}
+			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
+				if (dc->hwseq->funcs.dpstream_root_clock_control)
+					dc->hwseq->funcs.dpstream_root_clock_control(dc->hwseq, i, power_on);
+		}
+
+		for (i = 0; i < dc->res_pool->dig_link_enc_count; i++)
+			if (update_state->pg_pipe_res_update[PG_PHYSYMCLK][i])
+				if (dc->hwseq->funcs.physymclk_root_clock_control)
+					dc->hwseq->funcs.physymclk_root_clock_control(dc->hwseq, i, power_on);
+
+	}
+	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
+			if (power_on) {
+				if (dc->res_pool->dccg->funcs->enable_dsc)
+					dc->res_pool->dccg->funcs->enable_dsc(dc->res_pool->dccg, i);
+			} else {
+				if (dc->res_pool->dccg->funcs->disable_dsc)
+					dc->res_pool->dccg->funcs->disable_dsc(dc->res_pool->dccg, i);
+			}
+		}
+	}
+	/*disable root clock first when power down*/
+	if (!power_on) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+				update_state->pg_pipe_res_update[PG_DPP][i]) {
+				if (dc->hwseq->funcs.dpp_root_clock_control)
+					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
+			}
+			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
+				if (dc->hwseq->funcs.dpstream_root_clock_control)
+					dc->hwseq->funcs.dpstream_root_clock_control(dc->hwseq, i, power_on);
+		}
+
+		for (i = 0; i < dc->res_pool->dig_link_enc_count; i++)
+			if (update_state->pg_pipe_res_update[PG_PHYSYMCLK][i])
+				if (dc->hwseq->funcs.physymclk_root_clock_control)
+					dc->hwseq->funcs.physymclk_root_clock_control(dc->hwseq, i, power_on);
+
+	}
+}
+void dcn42_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc)
+{
+	struct crtc_stereo_flags flags = { 0 };
+	struct dc_stream_state *stream = pipe_ctx->stream;
+
+	dcn10_config_stereo_parameters(stream, &flags);
+
+	pipe_ctx->stream_res.opp->funcs->opp_program_stereo(
+		pipe_ctx->stream_res.opp,
+		flags.PROGRAM_STEREO == 1,
+		&stream->timing);
+
+	pipe_ctx->stream_res.tg->funcs->program_stereo(
+		pipe_ctx->stream_res.tg,
+		&stream->timing,
+		&flags);
+
+	return;
+}
+void dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock)
+{
+
+	union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
+
+	if (!dc->ctx || !dc->ctx->dmub_srv)
+		return;
+
+	/* Use helper to check PSR/Replay for all streams in context */
+
+	if (!dc->debug.fams2_config.bits.enable && !dc_dmub_srv_is_cursor_offload_enabled(dc)
+		&& !dmub_hw_lock_mgr_does_context_require_lock(dc, context))
+		return;
+
+	hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
+	hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
+	hw_lock_cmd.bits.lock = lock;
+	hw_lock_cmd.bits.should_release = !lock;
+	dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+}
+
+void dcn42_dmub_hw_control_lock_fast(union block_sequence_params *params)
+{
+	struct dc *dc = params->dmub_hw_control_lock_fast_params.dc;
+	bool lock = params->dmub_hw_control_lock_fast_params.lock;
+
+	/* Use helper to check PSR/Replay for the given stream in fast path */
+	if (params->dmub_hw_control_lock_fast_params.is_required) {
+		union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
+		hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
+		hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
+		hw_lock_cmd.bits.lock = lock;
+		hw_lock_cmd.bits.should_release = !lock;
+		dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+	}
+}
+
+/* In headless boot cases, DIG may be turned
+ * on which causes HW/SW discrepancies.
+ * To avoid this, power down hardware on boot
+ * if DIG is turned on
+ */
+void dcn42_power_down_on_boot(struct dc *dc)
+{
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	struct dc_link *edp_link = NULL;
+	int edp_num;
+	int i = 0;
+
+	dc_get_edp_links(dc, edp_links, &edp_num);
+	if (edp_num)
+		edp_link = edp_links[0];
+
+	if (edp_link && edp_link->link_enc->funcs->is_dig_enabled &&
+			edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc) &&
+			dc->hwseq->funcs.edp_backlight_control &&
+			dc->hwseq->funcs.power_down &&
+			dc->hwss.edp_power_control) {
+		dc->hwseq->funcs.edp_backlight_control(edp_link, false);
+		dc->hwseq->funcs.power_down(dc);
+		dc->hwss.edp_power_control(edp_link, false);
+	} else {
+		for (i = 0; i < dc->link_count; i++) {
+			struct dc_link *link = dc->links[i];
+
+			if (link->link_enc && link->link_enc->funcs->is_dig_enabled &&
+					link->link_enc->funcs->is_dig_enabled(link->link_enc) &&
+					dc->hwseq->funcs.power_down) {
+				dc->hwseq->funcs.power_down(dc);
+				break;
+			}
+
+		}
+	}
+
+	/*
+	 * Call update_clocks with empty context
+	 * to send DISPLAY_OFF
+	 * Otherwise DISPLAY_OFF may not be asserted
+	 */
+	if (dc->clk_mgr->funcs->set_low_power_state)
+		dc->clk_mgr->funcs->set_low_power_state(dc->clk_mgr);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
new file mode 100644
index 0000000000000..c4cfeed45b190
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef __DC_HWSS_DCN42_H__
+#define __DC_HWSS_DCN42_H__
+
+#include "dc.h"
+#include "hw_sequencer_private.h"
+
+void dcn42_init_hw(struct dc *dc);
+void dcn42_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx);
+
+void dcn42_program_cm_hist(
+	struct dc *dc,
+	struct pipe_ctx *pipe_ctx,
+	const struct dc_plane_state *plane_state);
+
+bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
+	const struct dc_plane_state *plane_state);
+
+bool dcn42_program_rmcm_luts(
+	struct hubp *hubp,
+	struct pipe_ctx *pipe_ctx,
+	const struct dc_plane_cm *cm,
+	struct mpc *mpc,
+	int mpcc_id);
+void dcn42_hardware_release(struct dc *dc);
+
+void dcn42_prepare_bandwidth(
+	struct dc *dc,
+	struct dc_state *context);
+void dcn42_optimize_bandwidth(struct dc *dc, struct dc_state *context);
+void dcn42_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
+		struct pg_block_update *update_state);
+void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
+		struct pg_block_update *update_state);
+void dcn42_hw_block_power_down(struct dc *dc,
+	struct pg_block_update *update_state);
+void dcn42_hw_block_power_up(struct dc *dc,
+		struct pg_block_update *update_state);
+void dcn42_root_clock_control(struct dc *dc,
+		struct pg_block_update *update_state, bool power_on);
+void dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock);
+void dcn42_dmub_hw_control_lock_fast(union block_sequence_params *params);
+void dcn42_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc);
+void dcn42_power_down_on_boot(struct dc *dc);
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
new file mode 100644
index 0000000000000..b324a2195e8a6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dce110/dce110_hwseq.h"
+#include "dcn10/dcn10_hwseq.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn21/dcn21_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dcn31/dcn31_hwseq.h"
+#include "dcn32/dcn32_hwseq.h"
+#include "dcn314/dcn314_hwseq.h"
+#include "dcn35/dcn35_hwseq.h"
+#include "dcn401/dcn401_hwseq.h"
+#include "dcn42/dcn42_hwseq.h"
+#include "dcn42_init.h"
+
+static const struct hw_sequencer_funcs dcn42_funcs = {
+	.program_gamut_remap = dcn401_program_gamut_remap,
+	.init_hw = dcn42_init_hw,
+	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
+	.power_down_on_boot = dcn42_power_down_on_boot,
+	.apply_ctx_for_surface = NULL,
+	.program_front_end_for_ctx = dcn401_program_front_end_for_ctx,
+	.clear_surface_dcc_and_tiling = dcn10_reset_surface_dcc_and_tiling,
+	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
+	.post_unlock_program_front_end = dcn401_post_unlock_program_front_end,
+	.update_plane_addr = dcn20_update_plane_addr,
+	.update_dchub = dcn10_update_dchub,
+	.update_pending_status = dcn10_update_pending_status,
+	.program_output_csc = dcn20_program_output_csc,
+	.trigger_3dlut_dma_load = dcn401_trigger_3dlut_dma_load,
+	.enable_accelerated_mode = dce110_enable_accelerated_mode,
+	.enable_timing_synchronization = dcn10_enable_timing_synchronization,
+	.enable_per_frame_crtc_position_reset = dcn10_enable_per_frame_crtc_position_reset,
+	.update_info_frame = dcn31_update_info_frame,
+	.send_immediate_sdp_message = dcn10_send_immediate_sdp_message,
+	.enable_stream = dcn401_enable_stream,
+	.disable_stream = dce110_disable_stream,
+	.unblank_stream = dcn401_unblank_stream,
+	.blank_stream = dce110_blank_stream,
+	.enable_audio_stream = dce110_enable_audio_stream,
+	.disable_audio_stream = dce110_disable_audio_stream,
+	.disable_plane = dcn35_disable_plane,
+	.pipe_control_lock = dcn20_pipe_control_lock,
+	.interdependent_update_lock = dcn401_interdependent_update_lock,
+	.cursor_lock = dcn10_cursor_lock,
+	.prepare_bandwidth = dcn42_prepare_bandwidth,
+	.optimize_bandwidth = dcn42_optimize_bandwidth,
+	.update_bandwidth = dcn401_update_bandwidth,
+	.set_drr = dcn35_set_drr,
+	.get_position = dcn10_get_position,
+	.set_static_screen_control = dcn35_set_static_screen_control,
+	.setup_stereo = dcn42_setup_stereo,
+	.set_avmute = dcn30_set_avmute,
+	.log_hw_state = dcn10_log_hw_state,
+	.get_hw_state = dcn10_get_hw_state,
+	.clear_status_bits = dcn10_clear_status_bits,
+	.wait_for_mpcc_disconnect = dcn10_wait_for_mpcc_disconnect,
+	.edp_backlight_control = dce110_edp_backlight_control,
+	.edp_power_control = dce110_edp_power_control,
+	.edp_wait_for_hpd_ready = dce110_edp_wait_for_hpd_ready,
+	.edp_wait_for_T12 = dce110_edp_wait_for_T12,
+	.set_cursor_position = dcn401_set_cursor_position,
+	.set_cursor_attribute = dcn10_set_cursor_attribute,
+	.set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level,
+	.abort_cursor_offload_update = dcn35_abort_cursor_offload_update,
+	.begin_cursor_offload_update = dcn35_begin_cursor_offload_update,
+	.commit_cursor_offload_update = dcn35_commit_cursor_offload_update,
+	.update_cursor_offload_pipe = dcn401_update_cursor_offload_pipe,
+	.notify_cursor_offload_drr_update = dcn35_notify_cursor_offload_drr_update,
+	.program_cursor_offload_now = dcn35_program_cursor_offload_now,
+	.setup_periodic_interrupt = dcn10_setup_periodic_interrupt,
+	.set_clock = dcn10_set_clock,
+	.get_clock = dcn10_get_clock,
+	.program_triplebuffer = dcn20_program_triple_buffer,
+	.enable_writeback = dcn30_enable_writeback,
+	.disable_writeback = dcn30_disable_writeback,
+	.update_writeback = dcn30_update_writeback,
+	.dmdata_status_done = dcn20_dmdata_status_done,
+	.program_dmdata_engine = dcn30_program_dmdata_engine,
+	.set_dmdata_attributes = dcn20_set_dmdata_attributes,
+	.init_sys_ctx = dcn31_init_sys_ctx,
+	.init_vm_ctx = dcn20_init_vm_ctx,
+	.set_flip_control_gsl = dcn20_set_flip_control_gsl,
+	.get_vupdate_offset_from_vsync = dcn10_get_vupdate_offset_from_vsync,
+	.calc_vupdate_position = dcn10_calc_vupdate_position,
+	.apply_idle_power_optimizations = dcn35_apply_idle_power_optimizations,
+	.does_plane_fit_in_mall = NULL,
+	.set_backlight_level = dcn31_set_backlight_level,
+	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
+	.hardware_release = dcn42_hardware_release,
+	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dcn401_disable_link_output,
+	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
+	.optimize_pwr_state = dcn21_optimize_pwr_state,
+	.exit_optimized_pwr_state = dcn21_exit_optimized_pwr_state,
+	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
+	.enable_phantom_streams = dcn32_enable_phantom_streams,
+	.disable_phantom_streams = dcn32_disable_phantom_streams,
+	.update_visual_confirm_color = dcn10_update_visual_confirm_color,
+	.update_phantom_vp_position = dcn32_update_phantom_vp_position,
+	.apply_update_flags_for_phantom = dcn32_apply_update_flags_for_phantom,
+	.wait_for_dcc_meta_propagation = dcn401_wait_for_dcc_meta_propagation,
+	.is_pipe_topology_transition_seamless = dcn32_is_pipe_topology_transition_seamless,
+	.dmub_hw_control_lock = dcn42_dmub_hw_control_lock,
+	.fams2_update_config = dcn401_fams2_update_config,
+	.dmub_hw_control_lock_fast = dcn42_dmub_hw_control_lock_fast,
+	.program_outstanding_updates = dcn401_program_outstanding_updates,
+	.wait_for_all_pending_updates = dcn30_wait_for_all_pending_updates,
+	.detect_pipe_changes = dcn401_detect_pipe_changes,
+	.enable_plane = dcn35_enable_plane,
+	.update_dchubp_dpp = dcn20_update_dchubp_dpp,
+	.post_unlock_reset_opp = dcn20_post_unlock_reset_opp,
+	.get_underflow_debug_data = dcn30_get_underflow_debug_data,
+	// Driver PG
+	.hw_block_power_up = dcn42_hw_block_power_up,
+	.hw_block_power_down = dcn42_hw_block_power_down,
+	.root_clock_control = dcn42_root_clock_control,
+	.calc_blocks_to_gate = dcn42_calc_blocks_to_gate,
+	.calc_blocks_to_ungate = dcn42_calc_blocks_to_ungate,
+
+};
+
+static const struct hwseq_private_funcs dcn42_private_funcs = {
+	.init_pipes = dcn35_init_pipes,
+	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
+	.update_mpcc = dcn42_update_mpcc,
+	.set_input_transfer_func = dcn32_set_input_transfer_func,
+	.set_output_transfer_func = dcn401_set_output_transfer_func,
+	.power_down = dce110_power_down,
+	.enable_display_power_gating = dcn10_dummy_display_power_gating,
+	.blank_pixel_data = dcn20_blank_pixel_data,
+	.reset_hw_ctx_wrap = dcn401_reset_hw_ctx_wrap,
+	.enable_stream_timing = dcn401_enable_stream_timing,
+	.edp_backlight_control = dce110_edp_backlight_control,
+	.setup_vupdate_interrupt = dcn20_setup_vupdate_interrupt,
+	.did_underflow_occur = dcn10_did_underflow_occur,
+	.init_blank = dcn32_init_blank,
+	.disable_vga = dcn20_disable_vga,
+	.bios_golden_init = dcn10_bios_golden_init,
+	.plane_atomic_disable = dcn35_plane_atomic_disable,
+	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
+	.update_odm = dcn401_update_odm,
+	.dsc_pg_status = dcn32_dsc_pg_status,
+	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
+	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
+	.set_mcm_luts = dcn42_set_mcm_luts,
+	.program_mall_pipe_config = dcn32_program_mall_pipe_config,
+	.update_mall_sel = dcn32_update_mall_sel,
+	.calculate_dccg_k1_k2_values = NULL,
+	.apply_single_controller_ctx_to_hw = dce110_apply_single_controller_ctx_to_hw,
+	.reset_back_end_for_pipe = dcn401_reset_back_end_for_pipe,
+	.populate_mcm_luts = NULL,
+	.perform_3dlut_wa_unlock = dcn401_perform_3dlut_wa_unlock,
+	.program_cm_hist = dcn42_program_cm_hist,
+	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
+	.physymclk_root_clock_control = dcn35_physymclk_root_clock_control,
+	.resync_fifo_dccg_dio = dcn314_resync_fifo_dccg_dio,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
+};
+
+void dcn42_hw_sequencer_init_functions(struct dc *dc)
+{
+	dc->hwss = dcn42_funcs;
+	dc->hwseq->funcs = dcn42_private_funcs;
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.h
new file mode 100644
index 0000000000000..f66eb752eeebf
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef __DC_DCN42_INIT_H__
+#define __DC_DCN42_INIT_H__
+
+struct dc;
+
+void dcn42_hw_sequencer_init_functions(struct dc *dc);
+
+#endif /* __DC_DCN401_INIT_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index 4632a5761b169..d1dba7ffcd9b3 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2015-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -174,6 +174,12 @@ struct program_mcache_id_and_split_coordinate {
 	struct dml2_hubp_pipe_mcache_regs *mcache_regs;
 };
 
+struct control_cm_hist_params {
+	struct dpp *dpp;
+	struct cm_hist_control cm_hist_control;
+	enum dc_color_space color_space;
+};
+
 struct program_cursor_update_now_params {
 	struct dc *dc;
 	struct pipe_ctx *pipe_ctx;
@@ -755,6 +761,7 @@ union block_sequence_params {
 	struct dmub_hw_control_lock_fast_params dmub_hw_control_lock_fast_params;
 	struct program_surface_config_params program_surface_config_params;
 	struct program_mcache_id_and_split_coordinate program_mcache_id_and_split_coordinate;
+	struct control_cm_hist_params control_cm_hist_params;
 	struct program_cursor_update_now_params program_cursor_update_now_params;
 	struct hubp_wait_pipe_read_start_params hubp_wait_pipe_read_start_params;
 	struct apply_update_flags_for_phantom_params apply_update_flags_for_phantom_params;
@@ -879,6 +886,7 @@ enum block_sequence_func {
 	DMUB_HW_CONTROL_LOCK_FAST,
 	HUBP_PROGRAM_SURFACE_CONFIG,
 	HUBP_PROGRAM_MCACHE_ID,
+	DPP_PROGRAM_CM_HIST,
 	PROGRAM_CURSOR_UPDATE_NOW,
 	HUBP_WAIT_PIPE_READ_START,
 	HWS_APPLY_UPDATE_FLAGS_FOR_PHANTOM,
@@ -1189,6 +1197,8 @@ struct hw_sequencer_funcs {
 	void (*disable_link_output)(struct dc_link *link,
 			const struct link_resource *link_res,
 			enum signal_type signal);
+	bool (*dac_load_detect)(struct dc_link *link);
+	void (*prepare_ddc)(struct dc_link *link);
 
 	void (*get_dcc_en_bits)(struct dc *dc, int *dcc_en_bits);
 
@@ -1355,6 +1365,10 @@ void get_dcc_visual_confirm_color(
 	struct pipe_ctx *pipe_ctx,
 	struct tg_color *color);
 
+void get_refresh_rate_confirm_color(
+		struct pipe_ctx *pipe_ctx,
+		struct tg_color *color);
+
 void set_p_state_switch_method(
 		struct dc *dc,
 		struct dc_state *context,
@@ -1413,6 +1427,8 @@ void hwss_program_surface_config(union block_sequence_params *params);
 
 void hwss_program_mcache_id_and_split_coordinate(union block_sequence_params *params);
 
+void hwss_program_cm_hist(union block_sequence_params *params);
+
 void hwss_set_odm_combine(union block_sequence_params *params);
 
 void hwss_set_odm_bypass(union block_sequence_params *params);
@@ -1763,6 +1779,11 @@ void hwss_add_opp_program_bit_depth_reduction(struct block_sequence_state *seq_s
 		bool use_default_params,
 		struct pipe_ctx *pipe_ctx);
 
+void hwss_add_dpp_program_cm_hist(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		struct cm_hist_control cm_hist_control,
+		enum dc_color_space color_space);
+
 void hwss_add_dc_ip_request_cntl(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		bool enable);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
index 406db231bc72a..8e3f54fb53fd6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2015-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -222,6 +222,9 @@ struct hwseq_private_funcs {
 	void (*wait_for_pipe_update_if_needed)(struct dc *dc, struct pipe_ctx *pipe_ctx, bool is_surface_update_only);
 	void (*set_wait_for_update_needed_for_pipe)(struct dc *dc, struct pipe_ctx *pipe_ctx);
 	void (*dc_ip_request_cntl)(struct dc *dc, bool enable);
+	void (*program_cm_hist)(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		const struct dc_plane_state *plane_state);
 };
 
 struct dce_hwseq {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index 43579b0e14825..e960ca9062adc 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -214,6 +214,7 @@ struct resource_funcs {
             unsigned int index);
 
 	void (*get_panel_config_defaults)(struct dc_panel_config *panel_config);
+	void (*get_default_tiling_info)(struct dc_tiling_info *tiling_info);
 	void (*build_pipe_pix_clk_params)(struct pipe_ctx *pipe_ctx);
 	/*
 	 * Get indicator of power from a context that went through full validation
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 2c9a4a12bd8a7..21224fd6b36d6 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: MIT */
 /*
- * Copyright 2012-16 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -95,6 +96,26 @@ struct dcn301_clk_internal {
 	uint32_t CLK1_CLK3_BYPASS_CNTL; //dcfclk bypass
 };
 
+struct dcn42_clk_internal {
+	int dummy;
+	uint32_t CLK8_CLK0_CURRENT_CNT; //dispclk
+	uint32_t CLK8_CLK1_CURRENT_CNT; //dppclk
+	uint32_t CLK8_CLK2_CURRENT_CNT; //dprefclk
+	uint32_t CLK8_CLK3_CURRENT_CNT; //dcfclk
+	uint32_t CLK8_CLK4_CURRENT_CNT; //dtbclk
+	uint32_t CLK8_CLK0_DS_CNTL;	    //dispclk deep_sleep_divider
+	uint32_t CLK8_CLK1_DS_CNTL;	    //dppclk deep_sleep_divider
+	uint32_t CLK8_CLK2_DS_CNTL;	    //dprefclk deep_sleep_divider
+	uint32_t CLK8_CLK3_DS_CNTL;	    //dcfclk deep_sleep_divider
+	uint32_t CLK8_CLK4_DS_CNTL;	    //dtbclk deep_sleep_divider
+	uint32_t CLK8_CLK0_BYPASS_CNTL; //dispclk bypass
+	uint32_t CLK8_CLK1_BYPASS_CNTL; //dppclk bypass
+	uint32_t CLK8_CLK2_BYPASS_CNTL; //dprefclk bypass
+	uint32_t CLK8_CLK3_BYPASS_CNTL; //dcfclk bypass
+	uint32_t CLK8_CLK4_BYPASS_CNTL; //dtbclk bypass
+	uint32_t CLK8_CLK_TICK_CNT__TIMER_THRESHOLD;
+};
+
 /* Will these bw structures be ASIC specific? */
 
 #define MAX_NUM_DPM_LVL		8
@@ -194,6 +215,7 @@ struct clk_state_registers_and_bypass {
 	uint32_t dcfclk_bypass;
 	uint32_t dprefclk_bypass;
 	uint32_t dispclk_bypass;
+	uint32_t timer_threshold;
 };
 
 struct rv1_clk_internal {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
index bac8febad69a5..c69ccfcebeb5a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: MIT */
 /*
- * Copyright 2018 Advanced Micro Devices, Inc.
+ * Copyright 2018-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -89,11 +90,6 @@ enum dentist_divider_range {
 	.DPREFCLK_CNTL = mmDPREFCLK_CNTL, \
 	.DENTIST_DISPCLK_CNTL = mmDENTIST_DISPCLK_CNTL
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-#define CLK_COMMON_REG_LIST_DCE60_BASE() \
-	SR(DENTIST_DISPCLK_CNTL)
-#endif
-
 #define CLK_COMMON_REG_LIST_DCN_BASE() \
 	SR(DENTIST_DISPCLK_CNTL)
 
@@ -119,12 +115,6 @@ enum dentist_divider_range {
 	CLK_SF(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, mask_sh), \
 	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, mask_sh)
 
-#if defined(CONFIG_DRM_AMD_DC_SI)
-#define CLK_COMMON_MASK_SH_LIST_DCE60_COMMON_BASE(mask_sh) \
-	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\
-	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh)
-#endif
-
 #define CLK_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh) \
 	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\
 	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh)
@@ -234,6 +224,45 @@ enum dentist_divider_range {
 #define CLK_COMMON_MASK_SH_LIST_DCN401(mask_sh) \
 	CLK_COMMON_MASK_SH_LIST_DCN321(mask_sh)
 
+#define CLK_REG_LIST_DCN42()	  \
+	SR(DENTIST_DISPCLK_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK_TICK_CNT_CONFIG_REG), \
+	CLK_SR_DCN42(CLK8_CLK0_CURRENT_CNT), \
+	CLK_SR_DCN42(CLK8_CLK1_CURRENT_CNT), \
+	CLK_SR_DCN42(CLK8_CLK2_CURRENT_CNT), \
+	CLK_SR_DCN42(CLK8_CLK3_CURRENT_CNT), \
+	CLK_SR_DCN42(CLK8_CLK4_CURRENT_CNT), \
+	CLK_SR_DCN42(CLK8_CLK0_BYPASS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK1_BYPASS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK2_BYPASS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK3_BYPASS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK4_BYPASS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK0_DS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK1_DS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK2_DS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK3_DS_CNTL), \
+	CLK_SR_DCN42(CLK8_CLK4_DS_CNTL)
+
+#define CLK_COMMON_MASK_SH_LIST_DCN42(mask_sh) \
+	CLK_SF(CLK8_CLK_TICK_CNT_CONFIG_REG, TIMER_THRESHOLD, mask_sh), \
+	CLK_SF(CLK8_CLK0_BYPASS_CNTL, CLK0_BYPASS_SEL, mask_sh), \
+	CLK_SF(CLK8_CLK1_BYPASS_CNTL, CLK1_BYPASS_SEL, mask_sh), \
+	CLK_SF(CLK8_CLK2_BYPASS_CNTL, CLK2_BYPASS_SEL, mask_sh), \
+	CLK_SF(CLK8_CLK3_BYPASS_CNTL, CLK3_BYPASS_SEL, mask_sh), \
+	CLK_SF(CLK8_CLK4_BYPASS_CNTL, CLK4_BYPASS_SEL, mask_sh), \
+	CLK_SF(CLK8_CLK0_DS_CNTL, CLK0_DS_DIV_ID, mask_sh), \
+	CLK_SF(CLK8_CLK1_DS_CNTL, CLK1_DS_DIV_ID, mask_sh), \
+	CLK_SF(CLK8_CLK2_DS_CNTL, CLK2_DS_DIV_ID, mask_sh), \
+	CLK_SF(CLK8_CLK3_DS_CNTL, CLK3_DS_DIV_ID, mask_sh), \
+	CLK_SF(CLK8_CLK4_DS_CNTL, CLK4_DS_DIV_ID, mask_sh), \
+	CLK_SF(CLK8_CLK0_DS_CNTL, CLK0_ALLOW_DS, mask_sh), \
+	CLK_SF(CLK8_CLK1_DS_CNTL, CLK1_ALLOW_DS, mask_sh), \
+	CLK_SF(CLK8_CLK2_DS_CNTL, CLK2_ALLOW_DS, mask_sh), \
+	CLK_SF(CLK8_CLK3_DS_CNTL, CLK3_ALLOW_DS, mask_sh), \
+	CLK_SF(CLK8_CLK4_DS_CNTL, CLK4_ALLOW_DS, mask_sh), \
+
+
+
 #define CLK_REG_FIELD_LIST(type) \
 	type DPREFCLK_SRC_SEL; \
 	type DENTIST_DPREFCLK_WDIVIDER; \
@@ -246,6 +275,42 @@ enum dentist_divider_range {
 	type FbMult_int; \
 	type FbMult_frac;
 
+#define CLK42_REG_LIST(clkip_num, type) \
+	type CLK ## clkip_num ## _CLK_TICK_CNT_CONFIG_REG; \
+	type CLK ## clkip_num ## _CLK0_CURRENT_CNT; \
+	type CLK ## clkip_num ## _CLK1_CURRENT_CNT; \
+	type CLK ## clkip_num ## _CLK2_CURRENT_CNT; \
+	type CLK ## clkip_num ## _CLK3_CURRENT_CNT; \
+	type CLK ## clkip_num ## _CLK4_CURRENT_CNT; \
+	type CLK ## clkip_num ## _CLK0_BYPASS_CNTL; \
+	type CLK ## clkip_num ## _CLK1_BYPASS_CNTL; \
+	type CLK ## clkip_num ## _CLK2_BYPASS_CNTL; \
+	type CLK ## clkip_num ## _CLK3_BYPASS_CNTL; \
+	type CLK ## clkip_num ## _CLK4_BYPASS_CNTL; \
+	type CLK ## clkip_num ## _CLK0_DS_CNTL; \
+	type CLK ## clkip_num ## _CLK1_DS_CNTL; \
+	type CLK ## clkip_num ## _CLK2_DS_CNTL; \
+	type CLK ## clkip_num ## _CLK3_DS_CNTL; \
+	type CLK ## clkip_num ## _CLK4_DS_CNTL;
+
+#define CLK42_REG_FIELD_LIST(type) \
+	type TIMER_THRESHOLD; \
+	type CLK0_BYPASS_SEL; \
+	type CLK1_BYPASS_SEL; \
+	type CLK2_BYPASS_SEL; \
+	type CLK3_BYPASS_SEL; \
+	type CLK4_BYPASS_SEL; \
+	type CLK0_DS_DIV_ID; \
+	type CLK1_DS_DIV_ID; \
+	type CLK2_DS_DIV_ID; \
+	type CLK3_DS_DIV_ID; \
+	type CLK4_DS_DIV_ID; \
+	type CLK0_ALLOW_DS; \
+	type CLK1_ALLOW_DS; \
+	type CLK2_ALLOW_DS; \
+	type CLK3_ALLOW_DS; \
+	type CLK4_ALLOW_DS;
+
 /*
  ***************************************************************************************
  ****************** Clock Manager Private Structures ***********************************
@@ -277,10 +342,10 @@ struct clk_mgr_registers {
 	uint32_t CLK2_CLK2_DFS_CNTL;
 
 	uint32_t CLK1_CLK0_CURRENT_CNT;
-    uint32_t CLK1_CLK1_CURRENT_CNT;
-    uint32_t CLK1_CLK2_CURRENT_CNT;
-    uint32_t CLK1_CLK3_CURRENT_CNT;
-    uint32_t CLK1_CLK4_CURRENT_CNT;
+	uint32_t CLK1_CLK1_CURRENT_CNT;
+	uint32_t CLK1_CLK2_CURRENT_CNT;
+	uint32_t CLK1_CLK3_CURRENT_CNT;
+	uint32_t CLK1_CLK4_CURRENT_CNT;
 	uint32_t CLK1_CLK5_CURRENT_CNT;
 
 	uint32_t CLK0_CLK0_DFS_CNTL;
@@ -309,16 +374,19 @@ struct clk_mgr_registers {
 	uint32_t CLK1_CLK5_ALLOW_DS;
 	uint32_t CLK5_spll_field_8;
 	uint32_t CLK6_spll_field_8;
+	CLK42_REG_LIST(8, uint32_t)
 };
 
 struct clk_mgr_shift {
 	CLK_REG_FIELD_LIST(uint8_t)
 	CLK20_REG_FIELD_LIST(uint8_t)
+	CLK42_REG_FIELD_LIST(uint8_t)
 };
 
 struct clk_mgr_mask {
 	CLK_REG_FIELD_LIST(uint32_t)
 	CLK20_REG_FIELD_LIST(uint32_t)
+	CLK42_REG_FIELD_LIST(uint32_t)
 };
 
 enum clock_type {
@@ -333,6 +401,8 @@ enum clock_type {
 	clock_type_dscclk,
 	clock_type_uclk,
 	clock_type_dramclk,
+	clock_type_dprefclk,
+	clock_type_dtbclk,
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index d88b57d4f5125..39215ecb480e4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -367,6 +367,14 @@ struct dpp_funcs {
 
 	void (*dpp_force_disable_cursor)(struct dpp *dpp_base);
 
+	void (*dpp_cm_hist_control)(
+		struct dpp *dpp_base,
+		struct cm_hist_control cm_hist_control,
+		enum dc_color_space color_space);
+
+	bool (*dpp_cm_hist_read)(
+		struct dpp *dpp_base,
+		struct cm_hist *cm_hist);
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index a79019365af8c..2a5a81d159507 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -89,7 +89,7 @@ enum hubp_3dlut_fl_addressing_mode {
 enum hubp_3dlut_fl_width {
 	hubp_3dlut_fl_width_17 = 17,
 	hubp_3dlut_fl_width_33 = 33,
-	hubp_3dlut_fl_width_transformed    = 4916, //mpc default
+	hubp_3dlut_fl_width_17_transformed    = 4916, //mpc default
 };
 
 enum hubp_3dlut_fl_crossbar_bit_slice {
@@ -99,22 +99,6 @@ enum hubp_3dlut_fl_crossbar_bit_slice {
 	hubp_3dlut_fl_crossbar_bit_slice_48_63 = 3
 };
 
-struct hubp_fl_3dlut_config {
-	bool enabled;
-	enum hubp_3dlut_fl_width width;
-	enum hubp_3dlut_fl_mode mode;
-	enum hubp_3dlut_fl_format format;
-	uint16_t bias;
-	uint16_t scale;
-	struct dc_plane_address address;
-	enum hubp_3dlut_fl_addressing_mode addr_mode;
-	enum dc_cm2_gpu_mem_layout layout;
-	uint8_t protection_bits;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r;
-};
-
 struct hubp {
 	const struct hubp_funcs *funcs;
 	struct dc_context *ctx;
@@ -243,7 +227,6 @@ struct hubp_funcs {
 	void (*hubp_disable_control)(struct hubp *hubp, bool disable_hubp);
 	unsigned int (*hubp_get_underflow_status)(struct hubp *hubp);
 	void (*hubp_init)(struct hubp *hubp);
-
 	void (*dmdata_set_attributes)(
 			struct hubp *hubp,
 			const struct dc_dmdata_attributes *attr);
@@ -290,24 +273,15 @@ struct hubp_funcs {
 
 	void (*hubp_wait_pipe_read_start)(struct hubp *hubp);
 	void (*hubp_program_mcache_id_and_split_coordinate)(struct hubp *hubp, struct dml2_hubp_pipe_mcache_regs *mcache_regs);
-	void (*hubp_update_3dlut_fl_bias_scale)(struct hubp *hubp, uint16_t bias, uint16_t scale);
-	void (*hubp_program_3dlut_fl_mode)(struct hubp *hubp,
-			enum hubp_3dlut_fl_mode mode);
-	void (*hubp_program_3dlut_fl_format)(struct hubp *hubp,
-			enum hubp_3dlut_fl_format format);
 	void (*hubp_program_3dlut_fl_addr)(struct hubp *hubp,
-		const struct dc_plane_address address);
+		const struct dc_plane_address *address);
+	void (*hubp_program_3dlut_fl_config)(struct hubp *hubp,
+		const struct dc_3dlut_dma *config);
 	void (*hubp_program_3dlut_fl_dlg_param)(struct hubp *hubp, int refcyc_per_3dlut_group);
 	void (*hubp_enable_3dlut_fl)(struct hubp *hubp, bool enable);
-	void (*hubp_program_3dlut_fl_addressing_mode)(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode);
-	void (*hubp_program_3dlut_fl_width)(struct hubp *hubp, enum hubp_3dlut_fl_width width);
-	void (*hubp_program_3dlut_fl_tmz_protected)(struct hubp *hubp, uint8_t protection_bits);
 	void (*hubp_program_3dlut_fl_crossbar)(struct hubp *hubp,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
-			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r);
+			enum dc_cm_lut_pixel_format format);
 	int (*hubp_get_3dlut_fl_done)(struct hubp *hubp);
-	void (*hubp_program_3dlut_fl_config)(struct hubp *hubp, struct hubp_fl_3dlut_config *cfg);
 	void (*hubp_clear_tiling)(struct hubp *hubp);
 	uint32_t (*hubp_get_current_read_line)(struct hubp *hubp);
 	uint32_t (*hubp_get_det_config_error)(struct hubp *hubp);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index a61d12ec61bc4..b152f68794954 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -152,6 +152,13 @@ struct dc_rgb {
 	uint32_t blue;
 };
 
+struct tetrahedral_33x33x33 {
+	struct dc_rgb lut0[8985];
+	struct dc_rgb lut1[8984];
+	struct dc_rgb lut2[8984];
+	struct dc_rgb lut3[8984];
+};
+
 struct tetrahedral_17x17x17 {
 	struct dc_rgb lut0[1229];
 	struct dc_rgb lut1[1228];
@@ -165,14 +172,23 @@ struct tetrahedral_9x9x9 {
 	struct dc_rgb lut3[182];
 };
 
+enum lut_dimension {
+	LUT_DIM_INVALID = 0,
+	LUT_DIM_9 = 9,
+	LUT_DIM_17 = 17,
+	LUT_DIM_33 = 33,
+};
+
 struct tetrahedral_params {
 	union {
+//TODO: Uncomment when in use.
+//		struct tetrahedral_33x33x33 tetrahedral_33;
 		struct tetrahedral_17x17x17 tetrahedral_17;
 		struct tetrahedral_9x9x9 tetrahedral_9;
 	};
 	bool use_tetrahedral_9;
 	bool use_12bits;
-
+	enum lut_dimension lut_dim;
 };
 
 /* arr_curve_points - regamma regions/segments specification
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index a8d1abe20f62c..f5617674bea8e 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -54,6 +54,7 @@
 #include "dc_hw_types.h"
 #include "hw_shared.h"
 #include "transform.h"
+#include "dc_types.h"
 
 #define MAX_MPCC 6
 #define MAX_OPP 6
@@ -66,7 +67,6 @@ enum mpc_output_csc_mode {
 	MPC_OUTPUT_CSC_COEF_B
 };
 
-
 enum mpcc_blend_mode {
 	MPCC_BLEND_MODE_BYPASS,
 	MPCC_BLEND_MODE_TOP_LAYER_PASSTHROUGH,
@@ -102,13 +102,6 @@ enum mpcc_movable_cm_location {
 	MPCC_MOVABLE_CM_LOCATION_AFTER,
 };
 
-enum MCM_LUT_XABLE {
-	MCM_LUT_DISABLE,
-	MCM_LUT_DISABLED = MCM_LUT_DISABLE,
-	MCM_LUT_ENABLE,
-	MCM_LUT_ENABLED = MCM_LUT_ENABLE,
-};
-
 enum MCM_LUT_ID {
 	MCM_LUT_3DLUT,
 	MCM_LUT_1DLUT,
@@ -117,7 +110,7 @@ enum MCM_LUT_ID {
 
 struct mpc_fl_3dlut_config {
 	bool enabled;
-	uint16_t width;
+	enum dc_cm_lut_size size;
 	bool select_lut_bank_a;
 	uint16_t bit_depth;
 	int hubp_index;
@@ -1042,6 +1035,24 @@ struct mpc_funcs {
 
 	void (*update_3dlut_fast_load_select)(struct mpc *mpc, int mpcc_id, int hubp_idx);
 
+	/**
+	* @get_3dlut_fast_load_status:
+	*
+	* Get 3D LUT fast load status and reference them with done, soft_underflow and hard_underflow pointers.
+	*
+	* Parameters:
+	* - [in/out] mpc - MPC context.
+	* - [in] mpcc_id
+	* - [in/out] done
+	* - [in/out] soft_underflow
+	* - [in/out] hard_underflow
+	*
+	* Return:
+	*
+	* void
+	*/
+	void (*get_3dlut_fast_load_status)(struct mpc *mpc, int mpcc_id, uint32_t *done, uint32_t *soft_underflow, uint32_t *hard_underflow);
+
 	/**
 	* @populate_lut:
 	*
@@ -1058,8 +1069,11 @@ struct mpc_funcs {
 	*
 	* void
 	*/
-	void (*populate_lut)(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params,
-			bool lut_bank_a, int mpcc_id);
+	void (*populate_lut)(struct mpc *mpc,
+			const enum MCM_LUT_ID id,
+			const union mcm_lut_params *params,
+			const bool lut_bank_a,
+			const int mpcc_id);
 
 	/**
 	* @program_lut_read_write_control:
@@ -1070,13 +1084,18 @@ struct mpc_funcs {
 	* - [in/out] mpc - MPC context.
 	* - [in] id
 	* - [in] lut_bank_a
+	* - [in] bit_depth
 	* - [in] mpcc_id
 	*
 	* Return:
 	*
 	* void
 	*/
-	void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id);
+	void (*program_lut_read_write_control)(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const bool lut_bank_a,
+		const unsigned int bit_depth,
+		const int mpcc_id);
 
 	/**
 	* @program_lut_mode:
@@ -1086,33 +1105,44 @@ struct mpc_funcs {
 	* Parameters:
 	* - [in/out] mpc - MPC context.
 	* - [in] id
-	* - [in] xable
+	* - [in] enable
 	* - [in] lut_bank_a
+	* - [in] size
 	* - [in] mpcc_id
 	*
 	* Return:
 	*
 	* void
 	*/
-	void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_ID id, const enum MCM_LUT_XABLE xable,
-			bool lut_bank_a, int mpcc_id);
+	void (*program_lut_mode)(struct mpc *mpc,
+			const enum MCM_LUT_ID id,
+			const bool enable,
+			const bool lut_bank_a,
+			const enum dc_cm_lut_size size,
+			const int mpcc_id);
 
-	/**
-	 * @mcm:
-	 *
-	 * MPC MCM new HW sequential programming functions
-	 */
-	struct {
-		void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id);
-		void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id);
-		void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id);
-		bool (*is_config_supported)(uint32_t width);
-		void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id,
-			bool lut_bank_a, bool enabled, int mpcc_id);
 
-		void (*populate_lut)(struct mpc *mpc, const union mcm_lut_params params,
-			bool lut_bank_a, int mpcc_id);
-	} mcm;
+	/**
+	* @get_lut_mode:
+	*
+	* Obtains enablement and ram bank status.
+	*
+	* Parameters:
+	* - [in/out] mpc - MPC context.
+	* - [in] id
+	* - [in] mpcc_id
+	* - [out] enable
+	* - [out] lut_bank_a
+	*
+	* Return:
+	*
+	* void
+	*/
+	void (*get_lut_mode)(struct mpc *mpc,
+			const enum MCM_LUT_ID id,
+			const int mpcc_id,
+			bool *enable,
+			bool *lut_bank_a);
 
 	/**
 	 * @rmcm:
@@ -1125,9 +1155,11 @@ struct mpc_funcs {
 		void (*update_3dlut_fast_load_select)(struct mpc *mpc, int mpcc_id, int hubp_idx);
 		void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id,
 			bool lut_bank_a, bool enabled, int mpcc_id);
-		void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_XABLE xable,
-			bool lut_bank_a, int mpcc_id);
-		void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id);
+		void (*program_lut_mode)(struct mpc *mpc,
+			bool enable,
+			bool lut_bank_a,
+			int mpcc_id);
+		void (*program_3dlut_size)(struct mpc *mpc, const enum dc_cm_lut_size size, int mpcc_id);
 		void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id);
 		void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id);
 		bool (*is_config_supported)(uint32_t width);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 671ab1fc73207..2f70bb476c97b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -518,6 +518,8 @@ struct timing_generator_funcs {
 	bool (*wait_update_lock_status)(struct timing_generator *tg, bool locked);
 	void (*read_otg_state)(struct timing_generator *tg, struct dcn_otg_state *s);
 	void (*optc_read_reg_state)(struct timing_generator *tg, struct dcn_optc_reg_state *optc_reg_state);
+	void (*enable_otg_pwa)(struct timing_generator *tg,  struct otc_pwa_frame_sync *pwa_param);
+	void (*disable_otg_pwa)(struct timing_generator *tg);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 79746d931471a..cecd3282a29f7 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -60,6 +60,7 @@ struct resource_caps {
 	int num_hpo_dp_stream_encoder;
 	int num_hpo_dp_link_encoder;
 	int num_mpc_3dlut;
+	int num_rmcm;
 };
 
 struct resource_straps {
diff --git a/drivers/gpu/drm/amd/display/dc/irq/Makefile b/drivers/gpu/drm/amd/display/dc/irq/Makefile
index b5e14d7923784..807b73c2a758b 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/irq/Makefile
@@ -198,3 +198,12 @@ IRQ_DCN401 = irq_service_dcn401.o
 AMD_DAL_IRQ_DCN401= $(addprefix $(AMDDALPATH)/dc/irq/dcn401/,$(IRQ_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCN401)
+
+###############################################################################
+# DCN 42
+###############################################################################
+IRQ_DCN42 = irq_service_dcn42.o
+
+AMD_DAL_IRQ_DCN42= $(addprefix $(AMDDALPATH)/dc/irq/dcn42/,$(IRQ_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCN42)
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.c b/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.c
new file mode 100644
index 0000000000000..19e0741c62cda
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.c
@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "include/logger_interface.h"
+#include "../dce110/irq_service_dce110.h"
+
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+
+#include "irq_service_dcn42.h"
+
+#define DCN_BASE__INST0_SEG2 0x000034C0
+
+static enum dc_irq_source to_dal_irq_source_dcn42(
+	struct irq_service *irq_service,
+	uint32_t src_id,
+	uint32_t ext_id)
+{
+	switch (src_id) {
+	case DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK1;
+	case DCN_1_0__SRCID__DC_D2_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK2;
+	case DCN_1_0__SRCID__DC_D3_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK3;
+	case DCN_1_0__SRCID__DC_D4_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK4;
+	case DCN_1_0__SRCID__DC_D5_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK5;
+	case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK6;
+	case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC1_VLINE0;
+	case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC2_VLINE0;
+	case DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC3_VLINE0;
+	case DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC4_VLINE0;
+	case DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC5_VLINE0;
+	case DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL:
+		return DC_IRQ_SOURCE_DC6_VLINE0;
+	case DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP1;
+	case DCN_1_0__SRCID__HUBP1_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP2;
+	case DCN_1_0__SRCID__HUBP2_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP3;
+	case DCN_1_0__SRCID__HUBP3_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP4;
+	case DCN_1_0__SRCID__HUBP4_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP5;
+	case DCN_1_0__SRCID__HUBP5_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP6;
+	case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE1;
+	case DCN_1_0__SRCID__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE2;
+	case DCN_1_0__SRCID__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE3;
+	case DCN_1_0__SRCID__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE4;
+	case DCN_1_0__SRCID__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE5;
+	case DCN_1_0__SRCID__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE6;
+	case DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT:
+		return DC_IRQ_SOURCE_DMCUB_OUTBOX;
+	case DCN_1_0__SRCID__DC_HPD1_INT:
+		/* generic src_id for all HPD and HPDRX interrupts */
+		switch (ext_id) {
+		case DCN_1_0__CTXID__DC_HPD1_INT:
+			return DC_IRQ_SOURCE_HPD1;
+		case DCN_1_0__CTXID__DC_HPD2_INT:
+			return DC_IRQ_SOURCE_HPD2;
+		case DCN_1_0__CTXID__DC_HPD3_INT:
+			return DC_IRQ_SOURCE_HPD3;
+		case DCN_1_0__CTXID__DC_HPD4_INT:
+			return DC_IRQ_SOURCE_HPD4;
+		case DCN_1_0__CTXID__DC_HPD5_INT:
+			return DC_IRQ_SOURCE_HPD5;
+		case DCN_1_0__CTXID__DC_HPD6_INT:
+			return DC_IRQ_SOURCE_HPD6;
+		case DCN_1_0__CTXID__DC_HPD1_RX_INT:
+			return DC_IRQ_SOURCE_HPD1RX;
+		case DCN_1_0__CTXID__DC_HPD2_RX_INT:
+			return DC_IRQ_SOURCE_HPD2RX;
+		case DCN_1_0__CTXID__DC_HPD3_RX_INT:
+			return DC_IRQ_SOURCE_HPD3RX;
+		case DCN_1_0__CTXID__DC_HPD4_RX_INT:
+			return DC_IRQ_SOURCE_HPD4RX;
+		case DCN_1_0__CTXID__DC_HPD5_RX_INT:
+			return DC_IRQ_SOURCE_HPD5RX;
+		case DCN_1_0__CTXID__DC_HPD6_RX_INT:
+			return DC_IRQ_SOURCE_HPD6RX;
+		default:
+			return DC_IRQ_SOURCE_INVALID;
+		}
+		break;
+	default:
+		return DC_IRQ_SOURCE_INVALID;
+	}
+
+}
+
+static struct irq_source_info_funcs hpd_irq_info_funcs = {
+	.set = NULL,
+	.ack = hpd0_ack
+};
+
+static struct irq_source_info_funcs hpd_rx_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs pflip_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs vblank_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs outbox_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs vline0_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs vline1_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+static struct irq_source_info_funcs vline2_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+/* compile time expand base address. */
+#define BASE(seg) \
+	BASE_INNER(seg)
+
+#define SRI(reg_name, block, id)\
+	(BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+			reg ## block ## id ## _ ## reg_name)
+
+#define SRI_DMUB(reg_name)\
+	(BASE(reg ## reg_name ## _BASE_IDX) + \
+			reg ## reg_name)
+
+#define IRQ_REG_ENTRY(block, reg_num, reg1, mask1, reg2, mask2)\
+	.enable_reg = SRI(reg1, block, reg_num),\
+	.enable_mask = \
+		block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK,\
+	.enable_value = {\
+		block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK,\
+		~block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK \
+	},\
+	.ack_reg = SRI(reg2, block, reg_num),\
+	.ack_mask = \
+		block ## reg_num ## _ ## reg2 ## __ ## mask2 ## _MASK,\
+	.ack_value = \
+		block ## reg_num ## _ ## reg2 ## __ ## mask2 ## _MASK \
+
+#define IRQ_REG_ENTRY_DMUB(reg1, mask1, reg2, mask2)\
+	.enable_reg = SRI_DMUB(reg1),\
+	.enable_mask = \
+		reg1 ## __ ## mask1 ## _MASK,\
+	.enable_value = {\
+		reg1 ## __ ## mask1 ## _MASK,\
+		~reg1 ## __ ## mask1 ## _MASK \
+	},\
+	.ack_reg = SRI_DMUB(reg2),\
+	.ack_mask = \
+		reg2 ## __ ## mask2 ## _MASK,\
+	.ack_value = \
+		reg2 ## __ ## mask2 ## _MASK \
+
+#define hpd_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_HPD1 + reg_num] = {\
+		IRQ_REG_ENTRY(HPD, reg_num,\
+			DC_HPD_INT_CONTROL, DC_HPD_INT_EN,\
+			DC_HPD_INT_CONTROL, DC_HPD_INT_ACK),\
+		.status_reg = SRI(DC_HPD_INT_STATUS, HPD, reg_num),\
+		.funcs = &hpd_irq_info_funcs\
+	}
+
+#define hpd_rx_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_HPD1RX + reg_num] = {\
+		IRQ_REG_ENTRY(HPD, reg_num,\
+			DC_HPD_INT_CONTROL, DC_HPD_RX_INT_EN,\
+			DC_HPD_INT_CONTROL, DC_HPD_RX_INT_ACK),\
+		.status_reg = SRI(DC_HPD_INT_STATUS, HPD, reg_num),\
+		.funcs = &hpd_rx_irq_info_funcs\
+	}
+#define pflip_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_PFLIP1 + reg_num] = {\
+		IRQ_REG_ENTRY(HUBPREQ, reg_num,\
+			DCSURF_SURFACE_FLIP_INTERRUPT, SURFACE_FLIP_INT_MASK,\
+			DCSURF_SURFACE_FLIP_INTERRUPT, SURFACE_FLIP_CLEAR),\
+		.funcs = &pflip_irq_info_funcs\
+	}
+
+#define vblank_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\
+		IRQ_REG_ENTRY(OTG, reg_num,\
+			OTG_GLOBAL_SYNC_STATUS, VSTARTUP_INT_EN,\
+			OTG_GLOBAL_SYNC_STATUS, VSTARTUP_EVENT_CLEAR),\
+		.funcs = &vblank_irq_info_funcs\
+	}
+
+#define vupdate_no_lock_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\
+		IRQ_REG_ENTRY(OTG, reg_num,\
+			OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_INT_EN,\
+			OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR),\
+		.funcs = &vupdate_no_lock_irq_info_funcs\
+	}
+
+#define vline0_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_DC1_VLINE0 + reg_num] = {\
+		IRQ_REG_ENTRY(OTG, reg_num,\
+			OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_INT_ENABLE,\
+			OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_CLEAR),\
+		.funcs = &vline0_irq_info_funcs\
+	}
+#define vline1_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_DC1_VLINE1 + reg_num] = {\
+		IRQ_REG_ENTRY(OTG, reg_num,\
+			OTG_VERTICAL_INTERRUPT1_CONTROL, OTG_VERTICAL_INTERRUPT1_INT_ENABLE,\
+			OTG_VERTICAL_INTERRUPT1_CONTROL, OTG_VERTICAL_INTERRUPT1_CLEAR),\
+		.funcs = &vline1_irq_info_funcs\
+	}
+#define vline2_int_entry(reg_num)\
+	[DC_IRQ_SOURCE_DC1_VLINE2 + reg_num] = {\
+		IRQ_REG_ENTRY(OTG, reg_num,\
+			OTG_VERTICAL_INTERRUPT2_CONTROL, OTG_VERTICAL_INTERRUPT2_INT_ENABLE,\
+			OTG_VERTICAL_INTERRUPT2_CONTROL, OTG_VERTICAL_INTERRUPT2_CLEAR),\
+		.funcs = &vline2_irq_info_funcs\
+	}
+#define dmub_outbox_int_entry()\
+	[DC_IRQ_SOURCE_DMCUB_OUTBOX] = {\
+		IRQ_REG_ENTRY_DMUB(\
+			DMCUB_INTERRUPT_ENABLE, DMCUB_OUTBOX1_READY_INT_EN,\
+			DMCUB_INTERRUPT_ACK, DMCUB_OUTBOX1_READY_INT_ACK),\
+		.funcs = &outbox_irq_info_funcs\
+	}
+
+#define dummy_irq_entry() \
+	{\
+		.funcs = &dummy_irq_info_funcs\
+	}
+
+#define i2c_int_entry(reg_num) \
+	[DC_IRQ_SOURCE_I2C_DDC ## reg_num] = dummy_irq_entry()
+
+#define dp_sink_int_entry(reg_num) \
+	[DC_IRQ_SOURCE_DPSINK ## reg_num] = dummy_irq_entry()
+
+#define gpio_pad_int_entry(reg_num) \
+	[DC_IRQ_SOURCE_GPIOPAD ## reg_num] = dummy_irq_entry()
+
+#define dc_underflow_int_entry(reg_num) \
+	[DC_IRQ_SOURCE_DC ## reg_num ## UNDERFLOW] = dummy_irq_entry()
+
+static struct irq_source_info_funcs dummy_irq_info_funcs = {
+	.set = dal_irq_service_dummy_set,
+	.ack = dal_irq_service_dummy_ack
+};
+
+static const struct irq_source_info irq_source_info_dcn42[DAL_IRQ_SOURCES_NUMBER] = {
+	[DC_IRQ_SOURCE_INVALID] = dummy_irq_entry(),
+	/* HPD (1-4) */
+	hpd_int_entry(0),
+	hpd_int_entry(1),
+	hpd_int_entry(2),
+	hpd_int_entry(3),
+	/* HPD RX (1-4) */
+	hpd_rx_int_entry(0),
+	hpd_rx_int_entry(1),
+	hpd_rx_int_entry(2),
+	hpd_rx_int_entry(3),
+	/* I2C (1-6) */
+	i2c_int_entry(1),
+	i2c_int_entry(2),
+	i2c_int_entry(3),
+	i2c_int_entry(4),
+	i2c_int_entry(5),
+	i2c_int_entry(6),
+	/* DP sink (1-6) keep table shape */
+	dp_sink_int_entry(1),
+	dp_sink_int_entry(2),
+	dp_sink_int_entry(3),
+	dp_sink_int_entry(4),
+	dp_sink_int_entry(5),
+	dp_sink_int_entry(6),
+	[DC_IRQ_SOURCE_TIMER] = dummy_irq_entry(),
+	/* PFLIP (1-4) */
+	pflip_int_entry(0),
+	pflip_int_entry(1),
+	pflip_int_entry(2),
+	pflip_int_entry(3),
+	[DC_IRQ_SOURCE_PFLIP5] = dummy_irq_entry(),
+	[DC_IRQ_SOURCE_PFLIP6] = dummy_irq_entry(),
+	[DC_IRQ_SOURCE_PFLIP_UNDERLAY0] = dummy_irq_entry(),
+	/* GPIO pads */
+	gpio_pad_int_entry(0),
+	gpio_pad_int_entry(1),
+	gpio_pad_int_entry(2),
+	gpio_pad_int_entry(3),
+	gpio_pad_int_entry(4),
+	gpio_pad_int_entry(5),
+	gpio_pad_int_entry(6),
+	gpio_pad_int_entry(7),
+	gpio_pad_int_entry(8),
+	gpio_pad_int_entry(9),
+	gpio_pad_int_entry(10),
+	gpio_pad_int_entry(11),
+	gpio_pad_int_entry(12),
+	gpio_pad_int_entry(13),
+	gpio_pad_int_entry(14),
+	gpio_pad_int_entry(15),
+	gpio_pad_int_entry(16),
+	gpio_pad_int_entry(17),
+	gpio_pad_int_entry(18),
+	gpio_pad_int_entry(19),
+	gpio_pad_int_entry(20),
+	gpio_pad_int_entry(21),
+	gpio_pad_int_entry(22),
+	gpio_pad_int_entry(23),
+	gpio_pad_int_entry(24),
+	gpio_pad_int_entry(25),
+	gpio_pad_int_entry(26),
+	gpio_pad_int_entry(27),
+	gpio_pad_int_entry(28),
+	gpio_pad_int_entry(29),
+	gpio_pad_int_entry(30),
+	/* Underflow (1-4 active, keep 5-6 dummy) */
+	dc_underflow_int_entry(1),
+	dc_underflow_int_entry(2),
+	dc_underflow_int_entry(3),
+	dc_underflow_int_entry(4),
+	dc_underflow_int_entry(5),
+	dc_underflow_int_entry(6),
+	[DC_IRQ_SOURCE_DMCU_SCP] = dummy_irq_entry(),
+	[DC_IRQ_SOURCE_VBIOS_SW] = dummy_irq_entry(),
+	vblank_int_entry(0),
+	vblank_int_entry(1),
+	vblank_int_entry(2),
+	vblank_int_entry(3),
+	[DC_IRQ_SOURCE_DC5_VLINE1] = dummy_irq_entry(),
+	[DC_IRQ_SOURCE_DC6_VLINE1] = dummy_irq_entry(),
+	dmub_outbox_int_entry(),
+	vupdate_no_lock_int_entry(0),
+	vupdate_no_lock_int_entry(1),
+	vupdate_no_lock_int_entry(2),
+	vupdate_no_lock_int_entry(3),
+	vline0_int_entry(0),
+	vline0_int_entry(1),
+	vline0_int_entry(2),
+	vline0_int_entry(3),
+	vline1_int_entry(0),
+	vline1_int_entry(1),
+	vline1_int_entry(2),
+	vline1_int_entry(3),
+	vline2_int_entry(0),
+	vline2_int_entry(1),
+	vline2_int_entry(2),
+	vline2_int_entry(3),
+};
+
+static const struct irq_service_funcs irq_service_funcs_dcn42 = {
+	.to_dal_irq_source = to_dal_irq_source_dcn42
+};
+
+static void dcn42_irq_construct(struct irq_service *irq_service,
+				struct irq_service_init_data *init_data)
+{
+	dal_irq_service_construct(irq_service, init_data);
+	irq_service->info = irq_source_info_dcn42;
+	irq_service->funcs = &irq_service_funcs_dcn42;
+}
+
+struct irq_service *dal_irq_service_dcn42_create(struct irq_service_init_data *init_data)
+{
+	struct irq_service *irq_service = kzalloc(sizeof(*irq_service), GFP_KERNEL);
+
+	if (!irq_service)
+		return NULL;
+
+	dcn42_irq_construct(irq_service, init_data);
+	return irq_service;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.h b/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.h
new file mode 100644
index 0000000000000..a3f70bce3a618
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn42/irq_service_dcn42.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * DCN4.2 IRQ service interface (dal-dev only)
+ */
+#ifndef __IRQ_SERVICE_DCN42_H__
+#define __IRQ_SERVICE_DCN42_H__
+
+#include "../dce110/irq_service_dce110.h"
+
+struct irq_service *dal_irq_service_dcn42_create(
+	struct irq_service_init_data *init_data);
+
+#endif /* __IRQ_SERVICE_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index b2c020071cbfb..f992c2d167487 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -150,14 +150,6 @@ static enum signal_type get_basic_signal_type(struct graphics_object_id encoder,
 		default:
 			return SIGNAL_TYPE_NONE;
 		}
-	} else if (downstream.type == OBJECT_TYPE_ENCODER) {
-		switch (downstream.id) {
-		case ENCODER_ID_EXTERNAL_NUTMEG:
-		case ENCODER_ID_EXTERNAL_TRAVIS:
-			return SIGNAL_TYPE_DISPLAY_PORT;
-		default:
-			return SIGNAL_TYPE_NONE;
-		}
 	}
 
 	return SIGNAL_TYPE_NONE;
@@ -174,6 +166,10 @@ static enum signal_type link_detect_sink_signal_type(struct dc_link *link,
 	struct audio_support *aud_support;
 	struct graphics_object_id enc_id;
 
+	/* External DP bridges should use DP signal regardless of connector type. */
+	if (link->ext_enc_id.id)
+		return SIGNAL_TYPE_DISPLAY_PORT;
+
 	if (link->is_dig_mapping_flexible)
 		enc_id = (struct graphics_object_id){.id = ENCODER_ID_UNKNOWN};
 	else
@@ -356,7 +352,7 @@ static void query_dp_dual_mode_adaptor(
 			*dongle = DISPLAY_DONGLE_DP_DVI_DONGLE;
 			sink_cap->max_hdmi_pixel_clock = DP_ADAPTOR_DVI_MAX_TMDS_CLK;
 
-			CONN_DATA_DETECT(ddc->link, type2_dongle_buf, sizeof(type2_dongle_buf),
+			CONN_DATA_DETECT(link, type2_dongle_buf, sizeof(type2_dongle_buf),
 					"DP-DVI passive dongle %dMhz: ",
 					DP_ADAPTOR_DVI_MAX_TMDS_CLK / 1000);
 			return;
@@ -620,6 +616,14 @@ static bool detect_dp(struct dc_link *link,
 		link->dpcd_caps.usb4_dp_tun_info.dp_tun_cap.raw = 0;
 	}
 
+	if (link->ext_enc_id.id) {
+		/* Fix number of connected sinks reported by external DP bridge */
+		link->dpcd_caps.sink_count.bits.SINK_COUNT = 1;
+		/* NUTMEG requires that we use HBR, doesn't work with RBR. */
+		if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_00001A)
+			link->preferred_link_setting.link_rate = LINK_RATE_HIGH;
+	}
+
 	return true;
 }
 
@@ -653,8 +657,6 @@ static bool wait_for_entering_dp_alt_mode(struct dc_link *link)
 	unsigned long long time_taken_in_ns;
 	int tries_taken;
 
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	/**
 	 * this function will only exist if we are on dcn21 (is_in_alt_mode is a
 	 *  function pointer, so checking to see if it is equal to 0 is the same
@@ -725,8 +727,6 @@ static void revert_dpia_mst_dsc_always_on_wa(struct dc_link *link)
 
 static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason reason)
 {
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	LINK_INFO("link=%d, mst branch is now Connected\n",
 		  link->link_index);
 
@@ -746,8 +746,6 @@ static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason
 
 bool link_reset_cur_dp_mst_topology(struct dc_link *link)
 {
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	LINK_INFO("link=%d, mst branch is now Disconnected\n",
 		  link->link_index);
 
@@ -903,11 +901,21 @@ static bool link_detect_evaluate_edid_header(uint8_t edid_header[8])
  */
 static bool link_detect_ddc_probe(struct dc_link *link)
 {
+	enum signal_type signal = link_detect_sink_signal_type(link, DETECT_REASON_HPD);
+	enum ddc_transaction_type transaction_type = get_ddc_transaction_type(signal);
+	uint8_t edid_header[8] = {0};
+	uint8_t zero = 0;
+	bool ddc_probed;
+
 	if (!link->ddc)
 		return false;
 
-	uint8_t edid_header[8] = {0};
-	bool ddc_probed = i2c_read(link->ddc, 0x50, edid_header, sizeof(edid_header));
+	if (link->dc->hwss.prepare_ddc)
+		link->dc->hwss.prepare_ddc(link);
+
+	set_ddc_transaction_type(link->ddc, transaction_type);
+
+	ddc_probed = link_query_ddc_data(link->ddc, 0x50, &zero, 1, edid_header, sizeof(edid_header));
 
 	if (!ddc_probed)
 		return false;
@@ -932,28 +940,10 @@ static bool link_detect_ddc_probe(struct dc_link *link)
  */
 static bool link_detect_dac_load_detect(struct dc_link *link)
 {
-	struct dc_bios *bios = link->ctx->dc_bios;
-	struct link_encoder *link_enc = link->link_enc;
-	enum engine_id engine_id = link_enc->preferred_engine;
-	enum dal_device_type device_type = DEVICE_TYPE_CRT;
-	enum bp_result bp_result = BP_RESULT_UNSUPPORTED;
-	uint32_t enum_id;
-
-	switch (engine_id) {
-	case ENGINE_ID_DACB:
-		enum_id = 2;
-		break;
-	case ENGINE_ID_DACA:
-	default:
-		engine_id = ENGINE_ID_DACA;
-		enum_id = 1;
-		break;
-	}
-
-	if (bios->funcs->dac_load_detection)
-		bp_result = bios->funcs->dac_load_detection(bios, engine_id, device_type, enum_id);
+	if (!link->dc->hwss.dac_load_detect)
+		return false;
 
-	return bp_result == BP_RESULT_OK;
+	return link->dc->hwss.dac_load_detect(link);
 }
 
 /*
@@ -981,8 +971,6 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 	enum dc_connection_type new_connection_type = dc_connection_none;
 	const uint32_t post_oui_delay = 30; // 30ms
 
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	if (dc_is_virtual_signal(link->connector_signal))
 		return false;
 
@@ -1463,8 +1451,6 @@ bool link_detect(struct dc_link *link, enum dc_detect_reason reason)
 	bool is_delegated_to_mst_top_mgr = false;
 	enum dc_connection_type pre_link_type = link->type;
 
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	is_local_sink_detect_success = detect_link_and_local_sink(link, reason);
 
 	if (is_local_sink_detect_success && link->local_sink)
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index 9b1d34c3438b2..b4f46408a0005 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -223,16 +223,32 @@ void link_get_master_pipes_with_dpms_on(const struct dc_link *link,
 	}
 }
 
-static bool get_ext_hdmi_settings(struct pipe_ctx *pipe_ctx,
-		enum engine_id eng_id,
-		struct ext_hdmi_settings *settings)
+static struct ext_hdmi_settings create_ext_hdmi_settings(
+		uint8_t address,
+		uint8_t reg_num,
+		uint8_t reg_num_6g,
+		const struct i2c_reg_info *reg_settings,
+		const struct i2c_reg_info *reg_settings_6g
+)
 {
-	bool result = false;
-	int i = 0;
-	struct integrated_info *integrated_info =
-			pipe_ctx->stream->ctx->dc_bios->integrated_info;
+	struct ext_hdmi_settings result = {
+		.slv_addr = address,
+		.reg_num = reg_num,
+		.reg_num_6g = reg_num_6g,
+	};
+
+	memcpy(result.reg_settings, reg_settings, sizeof(result.reg_settings));
+	memcpy(result.reg_settings_6g, reg_settings_6g, sizeof(result.reg_settings_6g));
+	return result;
+}
 
-	if (integrated_info == NULL)
+static bool get_ext_hdmi_settings(
+		const struct integrated_info *info,
+		enum engine_id eng_id,
+		struct ext_hdmi_settings *settings
+)
+{
+	if (!settings || !info)
 		return false;
 
 	/*
@@ -242,422 +258,268 @@ static bool get_ext_hdmi_settings(struct pipe_ctx *pipe_ctx,
 	 */
 
 	// Check if current bios contains ext Hdmi settings
-	if (integrated_info->gpu_cap_info & 0x20) {
-		switch (eng_id) {
-		case ENGINE_ID_DIGA:
-			settings->slv_addr = integrated_info->dp0_ext_hdmi_slv_addr;
-			settings->reg_num = integrated_info->dp0_ext_hdmi_6g_reg_num;
-			settings->reg_num_6g = integrated_info->dp0_ext_hdmi_6g_reg_num;
-			memmove(settings->reg_settings,
-					integrated_info->dp0_ext_hdmi_reg_settings,
-					sizeof(integrated_info->dp0_ext_hdmi_reg_settings));
-			memmove(settings->reg_settings_6g,
-					integrated_info->dp0_ext_hdmi_6g_reg_settings,
-					sizeof(integrated_info->dp0_ext_hdmi_6g_reg_settings));
-			result = true;
-			break;
-		case ENGINE_ID_DIGB:
-			settings->slv_addr = integrated_info->dp1_ext_hdmi_slv_addr;
-			settings->reg_num = integrated_info->dp1_ext_hdmi_6g_reg_num;
-			settings->reg_num_6g = integrated_info->dp1_ext_hdmi_6g_reg_num;
-			memmove(settings->reg_settings,
-					integrated_info->dp1_ext_hdmi_reg_settings,
-					sizeof(integrated_info->dp1_ext_hdmi_reg_settings));
-			memmove(settings->reg_settings_6g,
-					integrated_info->dp1_ext_hdmi_6g_reg_settings,
-					sizeof(integrated_info->dp1_ext_hdmi_6g_reg_settings));
-			result = true;
-			break;
-		case ENGINE_ID_DIGC:
-			settings->slv_addr = integrated_info->dp2_ext_hdmi_slv_addr;
-			settings->reg_num = integrated_info->dp2_ext_hdmi_6g_reg_num;
-			settings->reg_num_6g = integrated_info->dp2_ext_hdmi_6g_reg_num;
-			memmove(settings->reg_settings,
-					integrated_info->dp2_ext_hdmi_reg_settings,
-					sizeof(integrated_info->dp2_ext_hdmi_reg_settings));
-			memmove(settings->reg_settings_6g,
-					integrated_info->dp2_ext_hdmi_6g_reg_settings,
-					sizeof(integrated_info->dp2_ext_hdmi_6g_reg_settings));
-			result = true;
-			break;
-		case ENGINE_ID_DIGD:
-			settings->slv_addr = integrated_info->dp3_ext_hdmi_slv_addr;
-			settings->reg_num = integrated_info->dp3_ext_hdmi_6g_reg_num;
-			settings->reg_num_6g = integrated_info->dp3_ext_hdmi_6g_reg_num;
-			memmove(settings->reg_settings,
-					integrated_info->dp3_ext_hdmi_reg_settings,
-					sizeof(integrated_info->dp3_ext_hdmi_reg_settings));
-			memmove(settings->reg_settings_6g,
-					integrated_info->dp3_ext_hdmi_6g_reg_settings,
-					sizeof(integrated_info->dp3_ext_hdmi_6g_reg_settings));
-			result = true;
-			break;
-		default:
-			break;
-		}
+	if (!(info->gpu_cap_info & 0x20))
+		return false;
 
-		if (result == true) {
-			// Validate settings from bios integrated info table
-			if (settings->slv_addr == 0)
-				return false;
-			if (settings->reg_num > 9)
-				return false;
-			if (settings->reg_num_6g > 3)
-				return false;
+	switch (eng_id) {
+	case ENGINE_ID_DIGA:
+		*settings = create_ext_hdmi_settings(
+				info->dp0_ext_hdmi_slv_addr,
+				info->dp0_ext_hdmi_reg_num,
+				info->dp0_ext_hdmi_6g_reg_num,
+				info->dp0_ext_hdmi_reg_settings,
+				info->dp0_ext_hdmi_6g_reg_settings
+		);
+		break;
+	case ENGINE_ID_DIGB:
+		*settings = create_ext_hdmi_settings(
+				info->dp1_ext_hdmi_slv_addr,
+				info->dp1_ext_hdmi_reg_num,
+				info->dp1_ext_hdmi_6g_reg_num,
+				info->dp1_ext_hdmi_reg_settings,
+				info->dp1_ext_hdmi_6g_reg_settings
+		);
+		break;
+	case ENGINE_ID_DIGC:
+		*settings = create_ext_hdmi_settings(
+				info->dp2_ext_hdmi_slv_addr,
+				info->dp2_ext_hdmi_reg_num,
+				info->dp2_ext_hdmi_6g_reg_num,
+				info->dp2_ext_hdmi_reg_settings,
+				info->dp2_ext_hdmi_6g_reg_settings
+		);
+		break;
+	case ENGINE_ID_DIGD:
+		*settings = create_ext_hdmi_settings(
+				info->dp3_ext_hdmi_slv_addr,
+				info->dp3_ext_hdmi_reg_num,
+				info->dp3_ext_hdmi_6g_reg_num,
+				info->dp3_ext_hdmi_reg_settings,
+				info->dp3_ext_hdmi_6g_reg_settings
+		);
+		break;
+	default:
+		return false;
+	}
 
-			for (i = 0; i < settings->reg_num; i++) {
-				if (settings->reg_settings[i].i2c_reg_index > 0x20)
-					return false;
-			}
+	// Validate settings from bios integrated info table
+	if (
+			!settings->slv_addr
+			|| settings->reg_num > ARRAY_SIZE(settings->reg_settings)
+			|| settings->reg_num_6g > ARRAY_SIZE(settings->reg_settings_6g)
+	) {
+		return false;
+	}
 
-			for (i = 0; i < settings->reg_num_6g; i++) {
-				if (settings->reg_settings_6g[i].i2c_reg_index > 0x20)
-					return false;
-			}
-		}
+	for (size_t i = 0; i < settings->reg_num; i++) {
+		if (settings->reg_settings[i].i2c_reg_index > 0x20)
+			return false;
 	}
 
-	return result;
+	for (size_t i = 0; i < settings->reg_num_6g; i++) {
+		if (settings->reg_settings_6g[i].i2c_reg_index > 0x20)
+			return false;
+	}
+	return true;
 }
 
-static bool write_i2c(struct pipe_ctx *pipe_ctx,
-		uint8_t address, uint8_t *buffer, uint32_t length)
+static bool write_i2c(
+		const struct dc_link *link,
+		uint8_t address,
+		uint8_t *buffer,
+		uint32_t length
+)
 {
-	struct i2c_command cmd = {0};
-	struct i2c_payload payload = {0};
+	struct i2c_payload payload = {
+		.write = true,
+		.address = address,
+		.length = length,
+		.data = buffer,
+	};
+	struct i2c_command cmd = {
+		.payloads = &payload,
+		.number_of_payloads = 1,
+		.engine = I2C_COMMAND_ENGINE_DEFAULT,
+		.speed = link->ctx->dc->caps.i2c_speed_in_khz,
+	};
+
+	return dm_helpers_submit_i2c(link->ctx, link, &cmd);
+}
 
-	memset(&payload, 0, sizeof(payload));
-	memset(&cmd, 0, sizeof(cmd));
+static bool write_i2c_retimer_offset_value(
+		const struct dc_link *link,
+		uint8_t address,
+		uint8_t offset,
+		uint8_t value
+)
+{
+	DC_LOGGER_INIT(link->ctx->logger);
+	uint8_t buffer[] = { offset, value };
+	const bool success = write_i2c(link, address, buffer, sizeof(buffer));
+
+	RETIMER_REDRIVER_INFO(
+			"Retimer write, address: 0x%x, offset: 0x%x, value: 0x%x, success: %d\n",
+			address, offset, value, success
+	);
+	return success;
+}
 
-	cmd.number_of_payloads = 1;
-	cmd.engine = I2C_COMMAND_ENGINE_DEFAULT;
-	cmd.speed = pipe_ctx->stream->ctx->dc->caps.i2c_speed_in_khz;
+static bool write_i2c_retimer_vga(
+		const struct dc_link *link,
+		uint8_t address
+)
+{
+	DC_LOGGER_INIT(link->ctx->logger);
+	const uint8_t vga_data[][2] = {
+		{ 0xFF, 0x01 },
+		{ 0x00, 0x23 },
+		{ 0xFF, 0x00 },
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(vga_data); i++) {
+		if (!write_i2c_retimer_offset_value(link, address, vga_data[i][0], vga_data[i][1])) {
+			DC_LOG_ERROR("Set retimer failed, vga index: %zu\n", i);
+			return false;
+		}
+	}
+	return true;
+}
 
-	payload.address = address;
-	payload.data = buffer;
-	payload.length = length;
-	payload.write = true;
-	cmd.payloads = &payload;
+static bool write_i2c_retimer_byte(
+		const struct dc_link *link,
+		uint8_t address,
+		uint8_t index,
+		uint8_t value
+)
+{
+	DC_LOGGER_INIT(link->ctx->logger);
+	const uint8_t apply_rx_tx_change = 0x4;
 
-	if (dm_helpers_submit_i2c(pipe_ctx->stream->ctx,
-			pipe_ctx->stream->link, &cmd))
+	if (index > 0x20)
 		return true;
 
-	return false;
+	if (!write_i2c_retimer_offset_value(link, address, index, value)) {
+		DC_LOG_ERROR("Set retimer failed, 3g index: 0x%x, value: 0x%x\n", index, value);
+		return false;
+	}
+
+	// Based on DP159 specs, APPLY_RX_TX_CHANGE bit in 0x0A
+	// needs to be set to 1 on every 0x0A-0x0C write.
+	if (0x0A <= index && index <= 0x0C) {
+		uint8_t offset = 0x0A;
+
+		// Query current value from offset 0x0A
+		if (index == 0x0A) {
+			// Just written correct value, so no need to read it back
+		} else {
+			if (!link_query_ddc_data(
+					link->ddc, address, &offset, 1, &value, 1
+			)) {
+				DC_LOG_ERROR("Set retimer failed, link_query_ddc_data\n");
+				return false;
+			}
+		}
+
+		value |= apply_rx_tx_change;
+		if (!write_i2c_retimer_offset_value(link, address, offset, value)) {
+			DC_LOG_ERROR("Set retimer failed, 3g offset: 0x%x, value: 0x%x\n", offset, value);
+			return false;
+		}
+	}
+	return true;
 }
 
-static void write_i2c_retimer_setting(
-		struct pipe_ctx *pipe_ctx,
+static bool write_i2c_retimer_setting(
+		const struct dc_link *link,
 		bool is_vga_mode,
 		bool is_over_340mhz,
 		struct ext_hdmi_settings *settings)
 {
-	uint8_t slave_address = (settings->slv_addr >> 1);
-	uint8_t buffer[2];
-	const uint8_t apply_rx_tx_change = 0x4;
-	uint8_t offset = 0xA;
-	uint8_t value = 0;
-	int i = 0;
-	bool i2c_success = false;
-	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
-
-	memset(&buffer, 0, sizeof(buffer));
-
-	/* Start Ext-Hdmi programming*/
-
-	for (i = 0; i < settings->reg_num; i++) {
-		/* Apply 3G settings */
-		if (settings->reg_settings[i].i2c_reg_index <= 0x20) {
-
-			buffer[0] = settings->reg_settings[i].i2c_reg_index;
-			buffer[1] = settings->reg_settings[i].i2c_reg_val;
-			i2c_success = write_i2c(pipe_ctx, slave_address,
-						buffer, sizeof(buffer));
-			RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-				offset = 0x%x, reg_val= 0x%x, i2c_success = %d\n",
-				slave_address, buffer[0], buffer[1], i2c_success?1:0);
-
-			if (!i2c_success)
-				goto i2c_write_fail;
+	DC_LOGGER_INIT(link->ctx->logger);
+	const uint8_t address = settings->slv_addr >> 1;
 
-			/* Based on DP159 specs, APPLY_RX_TX_CHANGE bit in 0x0A
-			 * needs to be set to 1 on every 0xA-0xC write.
-			 */
-			if (settings->reg_settings[i].i2c_reg_index == 0xA ||
-				settings->reg_settings[i].i2c_reg_index == 0xB ||
-				settings->reg_settings[i].i2c_reg_index == 0xC) {
-
-				/* Query current value from offset 0xA */
-				if (settings->reg_settings[i].i2c_reg_index == 0xA)
-					value = settings->reg_settings[i].i2c_reg_val;
-				else {
-					i2c_success =
-						link_query_ddc_data(
-						pipe_ctx->stream->link->ddc,
-						slave_address, &offset, 1, &value, 1);
-					if (!i2c_success)
-						goto i2c_write_fail;
-				}
+	for (size_t i = 0; i < settings->reg_num; i++) {
+		const uint8_t index = settings->reg_settings[i].i2c_reg_index;
+		uint8_t value = settings->reg_settings[i].i2c_reg_val;
 
-				buffer[0] = offset;
-				/* Set APPLY_RX_TX_CHANGE bit to 1 */
-				buffer[1] = value | apply_rx_tx_change;
-				i2c_success = write_i2c(pipe_ctx, slave_address,
-						buffer, sizeof(buffer));
-				RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-					offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-					slave_address, buffer[0], buffer[1], i2c_success?1:0);
-				if (!i2c_success)
-					goto i2c_write_fail;
-			}
+		if (!write_i2c_retimer_byte(link, address, index, value)) {
+			DC_LOG_ERROR("Set retimer failed, index: %zu\n", i);
+			return false;
 		}
 	}
 
-	/* Apply 3G settings */
 	if (is_over_340mhz) {
-		for (i = 0; i < settings->reg_num_6g; i++) {
-			/* Apply 3G settings */
-			if (settings->reg_settings[i].i2c_reg_index <= 0x20) {
-
-				buffer[0] = settings->reg_settings_6g[i].i2c_reg_index;
-				buffer[1] = settings->reg_settings_6g[i].i2c_reg_val;
-				i2c_success = write_i2c(pipe_ctx, slave_address,
-							buffer, sizeof(buffer));
-				RETIMER_REDRIVER_INFO("above 340Mhz: retimer write to slave_address = 0x%x,\
-					offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-					slave_address, buffer[0], buffer[1], i2c_success?1:0);
-
-				if (!i2c_success)
-					goto i2c_write_fail;
-
-				/* Based on DP159 specs, APPLY_RX_TX_CHANGE bit in 0x0A
-				 * needs to be set to 1 on every 0xA-0xC write.
-				 */
-				if (settings->reg_settings_6g[i].i2c_reg_index == 0xA ||
-					settings->reg_settings_6g[i].i2c_reg_index == 0xB ||
-					settings->reg_settings_6g[i].i2c_reg_index == 0xC) {
-
-					/* Query current value from offset 0xA */
-					if (settings->reg_settings_6g[i].i2c_reg_index == 0xA)
-						value = settings->reg_settings_6g[i].i2c_reg_val;
-					else {
-						i2c_success =
-								link_query_ddc_data(
-								pipe_ctx->stream->link->ddc,
-								slave_address, &offset, 1, &value, 1);
-						if (!i2c_success)
-							goto i2c_write_fail;
-					}
+		for (size_t i = 0; i < settings->reg_num_6g; i++) {
+			const uint8_t index = settings->reg_settings_6g[i].i2c_reg_index;
+			uint8_t value = settings->reg_settings_6g[i].i2c_reg_val;
 
-					buffer[0] = offset;
-					/* Set APPLY_RX_TX_CHANGE bit to 1 */
-					buffer[1] = value | apply_rx_tx_change;
-					i2c_success = write_i2c(pipe_ctx, slave_address,
-							buffer, sizeof(buffer));
-					RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-						offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-						slave_address, buffer[0], buffer[1], i2c_success?1:0);
-					if (!i2c_success)
-						goto i2c_write_fail;
-				}
+			if (!write_i2c_retimer_byte(link, address, index, value)) {
+				DC_LOG_ERROR("Set retimer failed, 6g index: %zu\n", i);
+				return false;
 			}
 		}
 	}
 
 	if (is_vga_mode) {
-		/* Program additional settings if using 640x480 resolution */
-
-		/* Write offset 0xFF to 0x01 */
-		buffer[0] = 0xff;
-		buffer[1] = 0x01;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-				offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-				slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-
-		/* Write offset 0x00 to 0x23 */
-		buffer[0] = 0x00;
-		buffer[1] = 0x23;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-			offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-			slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-
-		/* Write offset 0xff to 0x00 */
-		buffer[0] = 0xff;
-		buffer[1] = 0x00;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write to slave_address = 0x%x,\
-			offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-			slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-
-	}
-
-	return;
-
-i2c_write_fail:
-	DC_LOG_DEBUG("Set retimer failed");
+		return write_i2c_retimer_vga(link, address);
+	}
+	return true;
 }
 
-static void write_i2c_default_retimer_setting(
-		struct pipe_ctx *pipe_ctx,
+static bool write_i2c_default_retimer_setting(
+		const struct dc_link *link,
 		bool is_vga_mode,
 		bool is_over_340mhz)
 {
-	uint8_t slave_address = (0xBA >> 1);
-	uint8_t buffer[2];
-	bool i2c_success = false;
-	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
+	const uint8_t address = 0xBA >> 1;
 
-	memset(&buffer, 0, sizeof(buffer));
-
-	/* Program Slave Address for tuning single integrity */
-	/* Write offset 0x0A to 0x13 */
-	buffer[0] = 0x0A;
-	buffer[1] = 0x13;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer writes default setting to slave_address = 0x%x,\
-		offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
-
-	/* Write offset 0x0A to 0x17 */
-	buffer[0] = 0x0A;
-	buffer[1] = 0x17;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-		offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
-
-	/* Write offset 0x0B to 0xDA or 0xD8 */
-	buffer[0] = 0x0B;
-	buffer[1] = is_over_340mhz ? 0xDA : 0xD8;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-		offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
-
-	/* Write offset 0x0A to 0x17 */
-	buffer[0] = 0x0A;
-	buffer[1] = 0x17;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-		offset = 0x%x, reg_val= 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
-
-	/* Write offset 0x0C to 0x1D or 0x91 */
-	buffer[0] = 0x0C;
-	buffer[1] = is_over_340mhz ? 0x1D : 0x91;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-		offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
-
-	/* Write offset 0x0A to 0x17 */
-	buffer[0] = 0x0A;
-	buffer[1] = 0x17;
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-			buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-		offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-		slave_address, buffer[0], buffer[1], i2c_success?1:0);
-	if (!i2c_success)
-		goto i2c_write_fail;
+	DC_LOGGER_INIT(link->ctx->logger);
 
+	const uint8_t data[][2] = {
+		{ 0x0A, 0x13 },
+		{ 0x0A, 0x17 },
+		{ 0x0B, is_over_340mhz ? 0xDA : 0xD8 },
+		{ 0x0A, 0x17 },
+		{ 0x0C, is_over_340mhz ? 0x1D : 0x91 },
+		{ 0x0A, 0x17 },
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(data); i++) {
+		if (!write_i2c_retimer_offset_value(link, address, data[i][0], data[i][1])) {
+			DC_LOG_ERROR("Set default retimer failed, index: %zu\n", i);
+			return false;
+		}
+	}
 
 	if (is_vga_mode) {
-		/* Program additional settings if using 640x480 resolution */
-
-		/* Write offset 0xFF to 0x01 */
-		buffer[0] = 0xff;
-		buffer[1] = 0x01;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-			offset = 0x%x, reg_val = 0x%x, i2c_success = %d\n",
-			slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-
-		/* Write offset 0x00 to 0x23 */
-		buffer[0] = 0x00;
-		buffer[1] = 0x23;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write to slave_addr = 0x%x,\
-			offset = 0x%x, reg_val= 0x%x, i2c_success = %d\n",
-			slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-
-		/* Write offset 0xff to 0x00 */
-		buffer[0] = 0xff;
-		buffer[1] = 0x00;
-		i2c_success = write_i2c(pipe_ctx, slave_address,
-				buffer, sizeof(buffer));
-		RETIMER_REDRIVER_INFO("retimer write default setting to slave_addr = 0x%x,\
-			offset = 0x%x, reg_val= 0x%x, i2c_success = %d end here\n",
-			slave_address, buffer[0], buffer[1], i2c_success?1:0);
-		if (!i2c_success)
-			goto i2c_write_fail;
-	}
-
-	return;
-
-i2c_write_fail:
-	DC_LOG_DEBUG("Set default retimer failed");
+		return write_i2c_retimer_vga(link, address);
+	}
+	return true;
 }
 
-static void write_i2c_redriver_setting(
-		struct pipe_ctx *pipe_ctx,
+static bool write_i2c_redriver_setting(
+		const struct dc_link *link,
 		bool is_over_340mhz)
 {
-	uint8_t slave_address = (0xF0 >> 1);
-	uint8_t buffer[16];
-	bool i2c_success = false;
-	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
-
-	memset(&buffer, 0, sizeof(buffer));
-
-	// Program Slave Address for tuning single integrity
-	buffer[3] = 0x4E;
-	buffer[4] = 0x4E;
-	buffer[5] = 0x4E;
-	buffer[6] = is_over_340mhz ? 0x4E : 0x4A;
-
-	i2c_success = write_i2c(pipe_ctx, slave_address,
-					buffer, sizeof(buffer));
-	RETIMER_REDRIVER_INFO("redriver write 0 to all 16 reg offset expect following:\n\
-		\t slave_addr = 0x%x, offset[3] = 0x%x, offset[4] = 0x%x,\
-		offset[5] = 0x%x,offset[6] is_over_340mhz = 0x%x,\
-		i2c_success = %d\n",
-		slave_address, buffer[3], buffer[4], buffer[5], buffer[6], i2c_success?1:0);
-
-	if (!i2c_success)
-		DC_LOG_DEBUG("Set redriver failed");
+	DC_LOGGER_INIT(link->ctx->logger);
+	const uint8_t address = 0xF0 >> 1;
+	uint8_t buffer[16] = {
+		[3] = 0x4E,
+		[4] = 0x4E,
+		[5] = 0x4E,
+		[6] = is_over_340mhz ? 0x4E : 0x4A,
+	};
+
+	const bool success = write_i2c(link, address, buffer, sizeof(buffer));
+
+	RETIMER_REDRIVER_INFO(
+			"Redriver write, address: 0x%x, buffer: { [3]: 0x%x, 0x%x, 0x%x, 0x%x }, success: %d\n",
+			address, buffer[3], buffer[4], buffer[5], buffer[6], success
+	);
+
+	if (!success)
+		DC_LOG_ERROR("Set redriver failed");
+	return success;
 }
 
 static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
@@ -1933,16 +1795,14 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 			/* DP159, Retimer settings */
 			eng_id = pipe_ctx->stream_res.stream_enc->id;
 
-			if (get_ext_hdmi_settings(pipe_ctx, eng_id, &settings)) {
-				write_i2c_retimer_setting(pipe_ctx,
-						is_vga_mode, is_over_340mhz, &settings);
+			if (get_ext_hdmi_settings(stream->ctx->dc_bios->integrated_info, eng_id, &settings)) {
+				write_i2c_retimer_setting(link, is_vga_mode, is_over_340mhz, &settings);
 			} else {
-				write_i2c_default_retimer_setting(pipe_ctx,
-						is_vga_mode, is_over_340mhz);
+				write_i2c_default_retimer_setting(link, is_vga_mode, is_over_340mhz);
 			}
 		} else if (masked_chip_caps == AMD_EXT_DISPLAY_PATH_CAPS__HDMI20_PI3EQX1204) {
 			/* PI3EQX1204, Redriver settings */
-			write_i2c_redriver_setting(pipe_ctx, is_over_340mhz);
+			write_i2c_redriver_setting(link, is_over_340mhz);
 		}
 	}
 
@@ -2364,15 +2224,13 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 			false);
 		if (masked_chip_caps == AMD_EXT_DISPLAY_PATH_CAPS__HDMI20_TISN65DP159RSBT) {
 			/* DP159, Retimer settings */
-			if (get_ext_hdmi_settings(pipe_ctx, eng_id, &settings))
-				write_i2c_retimer_setting(pipe_ctx,
-						false, false, &settings);
+			if (get_ext_hdmi_settings(stream->ctx->dc_bios->integrated_info, eng_id, &settings))
+				write_i2c_retimer_setting(link, false, false, &settings);
 			else
-				write_i2c_default_retimer_setting(pipe_ctx,
-						false, false);
+				write_i2c_default_retimer_setting(link, false, false);
 		} else if (masked_chip_caps == AMD_EXT_DISPLAY_PATH_CAPS__HDMI20_PI3EQX1204) {
 			/* PI3EQX1204, Redriver settings */
-			write_i2c_redriver_setting(pipe_ctx, false);
+			write_i2c_redriver_setting(link, false);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index 21815ad01a295..7e7682d7dfc83 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -367,6 +367,18 @@ static enum transmitter translate_encoder_to_transmitter(
 	}
 }
 
+static bool encoder_is_external_dp(
+		struct graphics_object_id encoder)
+{
+	switch (encoder.id) {
+	case ENCODER_ID_EXTERNAL_NUTMEG:
+	case ENCODER_ID_EXTERNAL_TRAVIS:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static void link_destruct(struct dc_link *link)
 {
 	int i;
@@ -514,6 +526,13 @@ static bool construct_phy(struct dc_link *link,
 	 * so that we avoid initializing DDC and HPD, etc.
 	 */
 	bp_funcs->get_src_obj(bios, link->link_id, 0, &link_encoder);
+
+	if (encoder_is_external_dp(link_encoder)) {
+		/* External DP bridge encoders: find the actual link encoder and use that. */
+		link->ext_enc_id = link_encoder;
+		bp_funcs->get_src_obj(bios, link->ext_enc_id, 0, &link_encoder);
+	}
+
 	transmitter_from_encoder = translate_encoder_to_transmitter(link_encoder);
 	link_analog_engine = find_analog_engine(link, &enc_init_data.analog_encoder);
 
@@ -691,6 +710,13 @@ static bool construct_phy(struct dc_link *link,
 		goto create_fail;
 	}
 
+	/* For external DP bridge encoders:
+	 * Set the connector signal to DisplayPort so that they can work with
+	 * the pre-existing code paths for DP without a lot of code churn.
+	 */
+	if (link->ext_enc_id.id != ENCODER_ID_UNKNOWN)
+		link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
 	LINK_INFO("Connector[%d] description: signal: %s\n",
 		  init_params->connector_index,
 		  signal_type_to_string(link->connector_signal));
@@ -735,7 +761,8 @@ static bool construct_phy(struct dc_link *link,
 						      link->device_tag.dev_id))
 			continue;
 		if (link->device_tag.dev_id.device_type == DEVICE_TYPE_CRT &&
-		    link->connector_signal != SIGNAL_TYPE_RGB)
+		    link->connector_signal != SIGNAL_TYPE_RGB &&
+		    link->ext_enc_id.id == ENCODER_ID_UNKNOWN)
 			continue;
 		if (link->device_tag.dev_id.device_type == DEVICE_TYPE_LCD &&
 		    link->connector_signal == SIGNAL_TYPE_RGB)
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index cdc7587cf0b68..e12bf3dd3e464 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -743,6 +743,8 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting
 {
 	struct dc_link_settings initial_link_setting = {
 		LANE_COUNT_ONE, LINK_RATE_LOW, LINK_SPREAD_DISABLED, false, 0};
+	if (link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN)
+		initial_link_setting.link_rate = link->preferred_link_setting.link_rate;
 	struct dc_link_settings current_link_setting =
 			initial_link_setting;
 	uint32_t link_bw;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
index 9b2f1a7da1d12..766b54631c798 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
@@ -71,7 +71,7 @@ enum dc_status dpcd_get_tunneling_device_data(struct dc_link *link)
 	link->dpcd_caps.usb4_dp_tun_info.dpia_info.raw =
 			dpcd_dp_tun_data[DP_IN_ADAPTER_INFO - DP_TUNNELING_CAPABILITIES_SUPPORT];
 	link->dpcd_caps.usb4_dp_tun_info.usb4_driver_id =
-			dpcd_dp_tun_data[DP_USB4_DRIVER_ID - DP_TUNNELING_CAPABILITIES_SUPPORT];
+			dpcd_dp_tun_data[DP_USB4_DRIVER_ID - DP_TUNNELING_CAPABILITIES_SUPPORT] & 0x0F;
 
 	if (link->dpcd_caps.usb4_dp_tun_info.dp_tun_cap.bits.dpia_bw_alloc) {
 		status = core_link_read_dpcd(link, USB4_DRIVER_BW_CAPABILITY,
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
index cc18a3bebef2b..1860d44f63c1f 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
@@ -223,9 +223,10 @@ static void handle_hpd_irq_vesa_replay_sink(struct dc_link *link)
 	}
 }
 
-static void handle_hpd_irq_replay_sink(struct dc_link *link)
+static void handle_hpd_irq_replay_sink(struct dc_link *link, bool *need_re_enable)
 {
 	union dpcd_replay_configuration replay_configuration = {0};
+	union dpcd_replay_configuration replay_sink_status = {0};
 	/*AMD Replay version reuse DP_PSR_ERROR_STATUS for REPLAY_ERROR status.*/
 	union psr_error_status replay_error_status = {0};
 	bool ret = false;
@@ -265,9 +266,17 @@ static void handle_hpd_irq_replay_sink(struct dc_link *link)
 		&replay_error_status.raw,
 		sizeof(replay_error_status.raw));
 
+	dm_helpers_dp_read_dpcd(
+		link->ctx,
+		link,
+		DP_PR_REPLAY_SINK_STATUS,
+		&replay_sink_status.raw,
+		1);
+
 	if (replay_error_status.bits.LINK_CRC_ERROR ||
 		replay_configuration.bits.DESYNC_ERROR_STATUS ||
-		replay_configuration.bits.STATE_TRANSITION_ERROR_STATUS) {
+		replay_configuration.bits.STATE_TRANSITION_ERROR_STATUS ||
+		replay_sink_status.bits.SINK_DEVICE_REPLAY_STATUS == 0x7) {
 		bool allow_active;
 
 		link->replay_settings.config.replay_error_status.raw |= replay_error_status.raw;
@@ -299,8 +308,7 @@ static void handle_hpd_irq_replay_sink(struct dc_link *link)
 		if (link->replay_settings.replay_allow_active) {
 			allow_active = false;
 			edp_set_replay_allow_active(link, &allow_active, true, false, NULL);
-			allow_active = true;
-			edp_set_replay_allow_active(link, &allow_active, true, false, NULL);
+			*need_re_enable = true;
 		}
 	}
 }
@@ -460,6 +468,7 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 	union device_service_irq device_service_clear = {0};
 	enum dc_status result;
 	bool status = false;
+	bool replay_re_enable_needed = false;
 
 	if (out_link_loss)
 		*out_link_loss = false;
@@ -519,7 +528,7 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 		/* PSR-related error was detected and handled */
 		return true;
 
-	handle_hpd_irq_replay_sink(link);
+	handle_hpd_irq_replay_sink(link, &replay_re_enable_needed);
 
 	/* If PSR-related error handled, Main link may be off,
 	 * so do not handle as a normal sink status change interrupt.
@@ -538,16 +547,16 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 		return false;
 	}
 
-	/* For now we only handle 'Downstream port status' case.
+	/* Handle 'Downstream port status' case for all DP link types.
 	 * If we got sink count changed it means
 	 * Downstream port status changed,
 	 * then DM should call DC to do the detection.
-	 * NOTE: Do not handle link loss on eDP since it is internal link
+	 * NOTE: Now includes eDP link loss detection and retraining
 	 */
-	if ((link->connector_signal != SIGNAL_TYPE_EDP) &&
-			dp_parse_link_loss_status(
-					link,
-					&hpd_irq_dpcd_data)) {
+
+	if (dp_parse_link_loss_status(
+			link,
+			&hpd_irq_dpcd_data)) {
 		/* Connectivity log: link loss */
 		CONN_DATA_LINK_LOSS(link,
 					hpd_irq_dpcd_data.raw,
@@ -576,6 +585,11 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 			!= link->dpcd_sink_count)
 		status = true;
 
+	if (replay_re_enable_needed) {
+		bool allow_active = true;
+
+		edp_set_replay_allow_active(link, &allow_active, true, false, NULL);
+	}
 	/* reasons for HPD RX:
 	 * 1. Link Loss - ie Re-train the Link
 	 * 2. MST sideband message
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
index 7e45d1e767bb1..96afce4ffbfa2 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
@@ -252,23 +252,24 @@ bool dp_pr_enable(struct dc_link *link, bool enable)
 	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
+	if (link->replay_settings.replay_allow_active == enable)
+		return true;
+
 	if (enable && !dc_is_embedded_signal(link->connector_signal))
 		dp_pr_set_static_screen_param(link);
 
-	if (link->replay_settings.replay_allow_active != enable) {
-		//for sending PR enable commands to DMUB
-		memset(&cmd, 0, sizeof(cmd));
+	// for sending PR enable commands to DMUB
+	memset(&cmd, 0, sizeof(cmd));
 
-		cmd.pr_enable.header.type = DMUB_CMD__PR;
-		cmd.pr_enable.header.sub_type = DMUB_CMD__PR_ENABLE;
-		cmd.pr_enable.header.payload_bytes = sizeof(struct dmub_cmd_pr_enable_data);
-		cmd.pr_enable.data.panel_inst = panel_inst;
-		cmd.pr_enable.data.enable = enable ? 1 : 0;
+	cmd.pr_enable.header.type = DMUB_CMD__PR;
+	cmd.pr_enable.header.sub_type = DMUB_CMD__PR_ENABLE;
+	cmd.pr_enable.header.payload_bytes = sizeof(struct dmub_cmd_pr_enable_data);
+	cmd.pr_enable.data.panel_inst = panel_inst;
+	cmd.pr_enable.data.enable = enable ? 1 : 0;
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
-		dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+	link->replay_settings.replay_allow_active = enable;
 
-		link->replay_settings.replay_allow_active = enable;
-	}
 	return true;
 }
 
@@ -389,7 +390,7 @@ bool dp_pr_get_state(const struct dc_link *link, uint64_t *state)
 	const struct dc *dc = link->ctx->dc;
 	unsigned int panel_inst = 0;
 	uint32_t retry_count = 0;
-	uint32_t replay_state = 0;
+	uint32_t replay_state = PR_STATE_INVALID;
 
 	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index aa02b38e183aa..4a2699a374b7e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -100,11 +100,7 @@ void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode)
 
 enum dp_panel_mode dp_get_panel_mode(struct dc_link *link)
 {
-	/* We need to explicitly check that connector
-	 * is not DP. Some Travis_VGA get reported
-	 * by video bios as DP.
-	 */
-	if (link->connector_signal != SIGNAL_TYPE_DISPLAY_PORT) {
+	if (link->ext_enc_id.id) {
 
 		switch (link->dpcd_caps.branch_dev_id) {
 		case DP_BRANCH_DEVICE_ID_0022B9:
@@ -124,7 +120,7 @@ enum dp_panel_mode dp_get_panel_mode(struct dc_link *link)
 			}
 			break;
 		case DP_BRANCH_DEVICE_ID_00001A:
-			/* alternate scrambler reset is required for Travis
+			/* alternate scrambler reset is required for NUTMEG
 			 * for the case when external chip does not provide
 			 * sink device id, alternate scrambler scheme will
 			 * be overriden later by querying Encoder feature
@@ -851,6 +847,7 @@ bool edp_setup_psr(struct dc_link *link,
 		case FAMILY_YELLOW_CARP:
 		case AMDGPU_FAMILY_GC_10_3_6:
 		case AMDGPU_FAMILY_GC_11_0_1:
+		case AMDGPU_FAMILY_GC_11_5_4:
 			if (dc->debug.disable_z10 || dc->debug.psr_skip_crtc_disable)
 				psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true;
 			break;
@@ -1098,8 +1095,6 @@ bool edp_send_replay_cmd(struct dc_link *link,
 	if (!replay)
 		return false;
 
-	DC_LOGGER_INIT(link->ctx->logger);
-
 	if (dp_pr_get_panel_inst(dc, link, &panel_inst))
 		cmd_data->panel_inst = panel_inst;
 	else {
diff --git a/drivers/gpu/drm/amd/display/dc/mmhubbub/Makefile b/drivers/gpu/drm/amd/display/dc/mmhubbub/Makefile
index 2d4b7a85847de..afc376defa0b0 100644
--- a/drivers/gpu/drm/amd/display/dc/mmhubbub/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/mmhubbub/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Advanced Micro Devices, Inc.
+# Copyright 2020-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -52,3 +52,11 @@ AMD_DAL_MMHUBBUB_DCN35 = $(addprefix $(AMDDALPATH)/dc/mmhubbub/dcn35/,$(MMHUBBUB
 AMD_DISPLAY_FILES += $(AMD_DAL_MMHUBBUB_DCN35)
 endif
 
+###############################################################################
+# DCN42
+###############################################################################
+MMHUBBUB_DCN42 = dcn42_mmhubbub.o
+
+AMD_DAL_MMHUBBUB_DCN42 = $(addprefix $(AMDDALPATH)/dc/mmhubbub/dcn42/,$(MMHUBBUB_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_MMHUBBUB_DCN42)
diff --git a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.c
new file mode 100644
index 0000000000000..4700bdc8701df
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dcn35/dcn35_mmhubbub.h"
+#include "dcn42_mmhubbub.h"
+#include "reg_helper.h"
+
+#define REG(reg)                                                             \
+	((const struct dcn35_mmhubbub_registers *)(mcif_wb30->mcif_wb_regs)) \
+		->reg
+
+#define CTX mcif_wb30->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name)                                                \
+	((const struct dcn35_mmhubbub_shift *)(mcif_wb30->mcif_wb_shift))       \
+		->field_name,                                                   \
+		((const struct dcn35_mmhubbub_mask *)(mcif_wb30->mcif_wb_mask)) \
+			->field_name
+
+void dcn42_mmhubbub_set_fgcg(struct dcn30_mmhubbub *mcif_wb30, bool enabled)
+{
+	REG_UPDATE(MMHUBBUB_CLOCK_CNTL, MMHUBBUB_FGCG_REP_DIS, !enabled);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.h
new file mode 100644
index 0000000000000..0dd1f0d2eb889
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn42/dcn42_mmhubbub.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+
+#ifndef __DCN42_MMHUBBUB_H
+#define __DCN42_MMHUBBUB_H
+
+#include "mcif_wb.h"
+#include "dcn32/dcn32_mmhubbub.h"
+#include "dcn35/dcn35_mmhubbub.h"
+
+void dcn42_mmhubbub_set_fgcg(struct dcn30_mmhubbub *mcif_wb30, bool enabled);
+#endif // __DCN42_MMHUBBUB_H
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/Makefile b/drivers/gpu/drm/amd/display/dc/mpc/Makefile
index 5402c3529f5ee..cc23aae3728a1 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/mpc/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Advanced Micro Devices, Inc.
+# Copyright 2020-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -69,4 +69,12 @@ AMD_DAL_MPC_DCN401 = $(addprefix $(AMDDALPATH)/dc/mpc/dcn401/,$(MPC_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_MPC_DCN401)
 
+###############################################################################
+# DCN42
+###############################################################################
+MPC_DCN42 = dcn42_mpc.o
+
+AMD_DAL_MPC_DCN42 = $(addprefix $(AMDDALPATH)/dc/mpc/dcn42/,$(MPC_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_MPC_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
index b23c64004dd53..27e6532348509 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
@@ -118,9 +118,7 @@ void mpc1_assert_idle_mpcc(struct mpc *mpc, int id)
 
 struct mpcc *mpc1_get_mpcc(struct mpc *mpc, int mpcc_id)
 {
-	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-
-	ASSERT(mpcc_id < mpc10->num_mpcc);
+	ASSERT(mpcc_id < TO_DCN10_MPC(mpc)->num_mpcc);
 	return &(mpc->mpcc_array[mpcc_id]);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
index eeac13fdd6f55..e0617db2d0c1d 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Advanced Micro Devices, Inc.
+ * Copyright 2023-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -47,6 +47,16 @@ void mpc401_update_3dlut_fast_load_select(struct mpc *mpc, int mpcc_id, int hubp
 	REG_SET(MPCC_MCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0, MPCC_MCM_3DLUT_FL_SEL, hubp_idx);
 }
 
+void mpc401_get_3dlut_fast_load_status(struct mpc *mpc, int mpcc_id, uint32_t *done, uint32_t *soft_underflow, uint32_t *hard_underflow)
+{
+	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
+
+	REG_GET_3(MPCC_MCM_3DLUT_FAST_LOAD_STATUS[mpcc_id],
+			MPCC_MCM_3DLUT_FL_DONE, done,
+			MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW, soft_underflow,
+			MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW, hard_underflow);
+}
+
 void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_location location, int mpcc_id)
 {
 	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
@@ -63,56 +73,15 @@ void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_locati
 	}
 }
 
-static enum dc_lut_mode get3dlut_config(
-			struct mpc *mpc,
-			bool *is_17x17x17,
-			bool *is_12bits_color_channel,
-			int mpcc_id)
-{
-	uint32_t i_mode, i_enable_10bits, lut_size;
-	enum dc_lut_mode mode;
-	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
-
-	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id],
-			MPCC_MCM_3DLUT_MODE_CURRENT,  &i_mode);
-
-	REG_GET(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-			MPCC_MCM_3DLUT_30BIT_EN, &i_enable_10bits);
-
-	switch (i_mode) {
-	case 0:
-		mode = LUT_BYPASS;
-		break;
-	case 1:
-		mode = LUT_RAM_A;
-		break;
-	case 2:
-		mode = LUT_RAM_B;
-		break;
-	default:
-		mode = LUT_BYPASS;
-		break;
-	}
-	if (i_enable_10bits > 0)
-		*is_12bits_color_channel = false;
-	else
-		*is_12bits_color_channel = true;
-
-	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &lut_size);
-
-	if (lut_size == 0)
-		*is_17x17x17 = true;
-	else
-		*is_17x17x17 = false;
-
-	return mode;
-}
-
-void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params, bool lut_bank_a, int mpcc_id)
+void mpc401_populate_lut(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const union mcm_lut_params *params,
+		const bool lut_bank_a,
+		const int mpcc_id)
 {
 	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
-	const struct pwl_params *lut1d = params.pwl;
-	const struct pwl_params *lut_shaper = params.pwl;
+	const struct pwl_params *lut1d = params->pwl;
+	const struct pwl_params *lut_shaper = params->pwl;
 	bool is_17x17x17;
 	bool is_12bits_color_channel;
 	const struct dc_rgb *lut0;
@@ -121,7 +90,7 @@ void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union
 	const struct dc_rgb *lut3;
 	int lut_size0;
 	int lut_size;
-	const struct tetrahedral_params *lut3d = params.lut3d;
+	const struct tetrahedral_params *lut3d = params->lut3d;
 
 	switch (id) {
 	case MCM_LUT_1DLUT:
@@ -164,8 +133,6 @@ void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union
 
 		mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true);
 
-		get3dlut_config(mpc, &is_17x17x17, &is_12bits_color_channel, mpcc_id);
-
 		is_17x17x17 = !lut3d->use_tetrahedral_9;
 		is_12bits_color_channel = lut3d->use_12bits;
 		if (is_17x17x17) {
@@ -188,8 +155,6 @@ void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union
 					sizeof(lut3d->tetrahedral_9.lut1[0]);
 			}
 
-		mpc32_select_3dlut_ram(mpc, next_mode,
-					is_12bits_color_channel, mpcc_id);
 		mpc32_select_3dlut_ram_mask(mpc, 0x1, mpcc_id);
 		if (is_12bits_color_channel)
 			mpc32_set3dlut_ram12(mpc, lut0, lut_size0, mpcc_id);
@@ -222,46 +187,69 @@ void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union
 
 }
 
+static uint32_t mpc401_cm_lut_size_to_3dlut_size(const enum dc_cm_lut_size cm_size)
+{
+	uint32_t size = 0;
+
+	switch (cm_size) {
+	case CM_LUT_SIZE_999:
+		size = 1;
+		break;
+	case CM_LUT_SIZE_171717:
+		size = 0;
+		break;
+	default:
+		/* invalid LUT size */
+		ASSERT(false);
+		size = 0;
+		break;
+	}
+
+	return size;
+}
+
 void mpc401_program_lut_mode(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		const enum MCM_LUT_XABLE xable,
-		bool lut_bank_a,
-		int mpcc_id)
+		const bool enable,
+		const bool lut_bank_a,
+		const enum dc_cm_lut_size size,
+		const int mpcc_id)
 {
+	uint32_t lut_size;
 	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
 
 	switch (id) {
 	case MCM_LUT_3DLUT:
-		switch (xable) {
-		case MCM_LUT_DISABLE:
+		if (enable) {
+			lut_size = mpc401_cm_lut_size_to_3dlut_size(size);
+			REG_UPDATE_2(MPCC_MCM_3DLUT_MODE[mpcc_id],
+					MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2,
+					MPCC_MCM_3DLUT_SIZE, lut_size);
+		} else {
+			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false);
 			REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, 0);
-			break;
-		case MCM_LUT_ENABLE:
-			REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2);
-			break;
 		}
 		break;
 	case MCM_LUT_SHAPER:
-		switch (xable) {
-		case MCM_LUT_DISABLE:
-			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, 0);
-			break;
-		case MCM_LUT_ENABLE:
+		if (enable) {
 			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
-			break;
+		} else {
+			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false);
+			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, 0);
 		}
 		break;
 	case MCM_LUT_1DLUT:
-		switch (xable) {
-		case MCM_LUT_DISABLE:
-			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
-					MPCC_MCM_1DLUT_MODE, 0);
-			break;
-		case MCM_LUT_ENABLE:
+		if (enable) {
 			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
 					MPCC_MCM_1DLUT_MODE, 2);
-			break;
+		} else {
+			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				mpc32_power_on_blnd_lut(mpc, mpcc_id, false);
+			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
+					MPCC_MCM_1DLUT_MODE, 0);
 		}
 		REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
 				MPCC_MCM_1DLUT_SELECT, lut_bank_a ? 0 : 1);
@@ -269,14 +257,20 @@ void mpc401_program_lut_mode(
 	}
 }
 
-void mpc401_program_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id)
+void mpc401_program_lut_read_write_control(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const bool lut_bank_a,
+		const unsigned int bit_depth,
+		const int mpcc_id)
 {
 	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
 
 	switch (id) {
 	case MCM_LUT_3DLUT:
 		mpc32_select_3dlut_ram_mask(mpc, 0xf, mpcc_id);
-		REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1);
+		REG_UPDATE_2(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+				MPCC_MCM_3DLUT_30BIT_EN, (bit_depth == 10) ? 1 : 0,
+				MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1);
 		break;
 	case MCM_LUT_SHAPER:
 		mpc32_configure_shaper_lut(mpc, lut_bank_a, mpcc_id);
@@ -568,6 +562,44 @@ void mpc401_get_gamut_remap(struct mpc *mpc,
 		arr_reg_val, ARRAY_SIZE(arr_reg_val));
 }
 
+void mpc401_get_lut_mode(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const int mpcc_id,
+		bool *enable,
+		bool *lut_bank_a)
+{
+	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
+
+	uint32_t lut_mode = 0;
+	uint32_t lut_select = 0;
+
+	*enable = false;
+	*lut_bank_a = true;
+
+	switch (id) {
+	case MCM_LUT_SHAPER:
+		REG_GET(MPCC_MCM_SHAPER_CONTROL[mpcc_id],
+				MPCC_MCM_SHAPER_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		*lut_bank_a = lut_mode != 2;
+		break;
+	case MCM_LUT_1DLUT:
+		REG_GET_2(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
+				MPCC_MCM_1DLUT_MODE_CURRENT, &lut_mode,
+				MPCC_MCM_1DLUT_SELECT_CURRENT, &lut_select);
+		*enable = lut_mode != 0;
+		*lut_bank_a = lut_mode == 0 || lut_select == 0;
+		break;
+	case MCM_LUT_3DLUT:
+	default:
+		REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id],
+				MPCC_MCM_3DLUT_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		*lut_bank_a = lut_mode != 2;
+		break;
+	}
+}
+
 static const struct mpc_funcs dcn401_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
@@ -602,9 +634,11 @@ static const struct mpc_funcs dcn401_mpc_funcs = {
 	.set_bg_color = mpc1_set_bg_color,
 	.set_movable_cm_location = mpc401_set_movable_cm_location,
 	.update_3dlut_fast_load_select = mpc401_update_3dlut_fast_load_select,
+	.get_3dlut_fast_load_status = mpc401_get_3dlut_fast_load_status,
 	.populate_lut = mpc401_populate_lut,
 	.program_lut_read_write_control = mpc401_program_lut_read_write_control,
 	.program_lut_mode = mpc401_program_lut_mode,
+	.get_lut_mode = mpc401_get_lut_mode,
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
index fdc42f8ab3ff5..c16560c84453e 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Advanced Micro Devices, Inc.
+ * Copyright 2023-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -184,6 +184,7 @@ struct dcn401_mpc_mask {
 
 struct dcn401_mpc_registers {
 	MPC_REG_VARIABLE_LIST_DCN4_01
+	uint32_t MPCC_CONTROL2[MAX_MPCC];
 };
 
 struct dcn401_mpc {
@@ -205,21 +206,32 @@ void dcn401_mpc_construct(struct dcn401_mpc *mpc401,
 	int num_rmu);
 
 void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_location location, int mpcc_id);
-void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params,
-		bool lut_bank_a, int mpcc_id);
+void mpc401_populate_lut(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const union mcm_lut_params *params,
+		bool lut_bank_a,
+		int mpcc_id);
 
 void mpc401_program_lut_mode(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		const enum MCM_LUT_XABLE xable,
-		bool lut_bank_a,
-		int mpcc_id);
+		const bool enable,
+		const bool lut_bank_a,
+		const enum dc_cm_lut_size size,
+		const int mpcc_id);
+
+void mpc401_get_lut_mode(struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		const int mpcc_id,
+		bool *enable,
+		bool *lut_bank_a);
 
 void mpc401_program_lut_read_write_control(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		bool lut_bank_a,
-		int mpcc_id);
+		const bool lut_bank_a,
+		const unsigned int bit_depth,
+		const int mpcc_id);
 
 void mpc401_set_gamut_remap(
 	struct mpc *mpc,
@@ -249,6 +261,13 @@ void mpc_read_gamut_remap(struct mpc *mpc,
 	enum mpcc_gamut_remap_id gamut_remap_block_id,
 	uint32_t *mode_select);
 
+void mpc401_get_3dlut_fast_load_status(
+	struct mpc *mpc,
+	int mpcc_id,
+	uint32_t *done,
+	uint32_t *soft_underflow,
+	uint32_t *hard_underflow);
+
 void mpc401_update_3dlut_fast_load_select(
 	struct mpc *mpc,
 	int mpcc_id,
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
new file mode 100644
index 0000000000000..507dbdbea6008
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
@@ -0,0 +1,797 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "reg_helper.h"
+#include "dc.h"
+#include "dcn42_mpc.h"
+#include "dcn10/dcn10_cm_common.h"
+#include "basics/conversion.h"
+#include "mpc.h"
+
+#define REG(reg)\
+	mpc42->mpc_regs->reg
+
+#define CTX \
+	mpc42->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	mpc42->mpc_shift->field_name, mpc42->mpc_mask->field_name
+
+
+void mpc42_init_mpcc(struct mpcc *mpcc, int mpcc_inst)
+{
+	mpcc->mpcc_id = mpcc_inst;
+	mpcc->dpp_id = 0xf;
+	mpcc->mpcc_bot = NULL;
+	mpcc->blnd_cfg.overlap_only = false;
+	mpcc->blnd_cfg.global_alpha = 0xfff;
+	mpcc->blnd_cfg.global_gain = 0xfff;
+	mpcc->blnd_cfg.background_color_bpc = 4;
+	mpcc->blnd_cfg.bottom_gain_mode = 0;
+	mpcc->blnd_cfg.top_gain = 0x1f000;
+	mpcc->blnd_cfg.bottom_inside_gain = 0x1f000;
+	mpcc->blnd_cfg.bottom_outside_gain = 0x1f000;
+	mpcc->sm_cfg.enable = false;
+	mpcc->shared_bottom = false;
+}
+
+void mpc42_update_blending(
+	struct mpc *mpc,
+	struct mpcc_blnd_cfg *blnd_cfg,
+	int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	struct mpcc *mpcc = mpc1_get_mpcc(mpc, mpcc_id);
+
+	REG_UPDATE_5(MPCC_CONTROL[mpcc_id],
+			MPCC_ALPHA_BLND_MODE,		blnd_cfg->alpha_mode,
+			MPCC_ALPHA_MULTIPLIED_MODE,	blnd_cfg->pre_multiplied_alpha,
+			MPCC_BLND_ACTIVE_OVERLAP_ONLY,	blnd_cfg->overlap_only,
+			MPCC_BG_BPC,			blnd_cfg->background_color_bpc,
+			MPCC_BOT_GAIN_MODE,		blnd_cfg->bottom_gain_mode);
+	REG_UPDATE_2(MPCC_CONTROL2[mpcc_id],
+			MPCC_GLOBAL_ALPHA,		blnd_cfg->global_alpha,
+			MPCC_GLOBAL_GAIN,		blnd_cfg->global_gain);
+
+	REG_SET(MPCC_TOP_GAIN[mpcc_id], 0, MPCC_TOP_GAIN, blnd_cfg->top_gain);
+	REG_SET(MPCC_BOT_GAIN_INSIDE[mpcc_id], 0, MPCC_BOT_GAIN_INSIDE, blnd_cfg->bottom_inside_gain);
+	REG_SET(MPCC_BOT_GAIN_OUTSIDE[mpcc_id], 0, MPCC_BOT_GAIN_OUTSIDE, blnd_cfg->bottom_outside_gain);
+
+	mpcc->blnd_cfg = *blnd_cfg;
+}
+
+/* RMCM Shaper functions */
+void mpc42_power_on_rmcm_shaper_3dlut(
+	struct mpc *mpc,
+	uint32_t mpcc_id,
+	bool power_on)
+{
+	uint32_t power_status_shaper = 2;
+	uint32_t power_status_3dlut  = 2;
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+	int max_retries = 10;
+
+	REG_SET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], 0,
+		MPC_RMCM_3DLUT_MEM_PWR_DIS, power_on == true ? 0 : 1);
+	REG_SET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], 0,
+		MPC_RMCM_SHAPER_MEM_PWR_DIS, power_on == true ? 0 : 1);
+	/* wait for memory to fully power up */
+	if (power_on && mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) {
+		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, 0, 1, max_retries);
+		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, 0, 1, max_retries);
+	}
+
+	/*read status is not mandatory, it is just for debugging*/
+	REG_GET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, &power_status_shaper);
+	REG_GET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, &power_status_3dlut);
+
+	if (power_status_shaper != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+
+	if (power_status_3dlut != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+}
+
+void mpc42_configure_rmcm_shaper_lut(
+	struct mpc *mpc,
+	bool is_ram_a,
+	uint32_t mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_id],
+		MPC_RMCM_SHAPER_SCALE_B, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_id],
+		MPC_RMCM_SHAPER_SCALE_G, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_R[mpcc_id],
+		MPC_RMCM_SHAPER_SCALE_R, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
+	REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_INDEX, 0);
+}
+
+void mpc42_program_rmcm_shaper_luta_settings(
+	struct mpc *mpc,
+	const struct pwl_params *params,
+	uint32_t mpcc_id)
+{
+	const struct gamma_curve *curve;
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[mpcc_id], 0,
+		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_G[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_R[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_G[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_R[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
+
+	curve = params->arr_curve_points;
+	if (curve) {
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_0_1[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_2_3[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_4_5[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_6_7[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_8_9[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_10_11[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_12_13[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_14_15[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_16_17[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_18_19[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_20_21[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_22_23[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_24_25[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_26_27[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_28_29[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_30_31[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_32_33[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+	}
+}
+
+
+void mpc42_program_rmcm_shaper_lutb_settings(
+	struct mpc *mpc,
+	const struct pwl_params *params,
+	uint32_t mpcc_id)
+{
+	const struct gamma_curve *curve;
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_B[mpcc_id], 0,
+		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_G[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_R[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_B[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_G[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_R[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
+
+	curve = params->arr_curve_points;
+	if (curve) {
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_0_1[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_2_3[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_4_5[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_6_7[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_8_9[mpcc_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_10_11[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_12_13[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_14_15[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_16_17[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_18_19[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_20_21[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_22_23[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_24_25[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_26_27[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_28_29[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_30_31[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_32_33[mpcc_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+	}
+}
+
+void mpc42_program_rmcm_shaper_lut(
+	struct mpc *mpc,
+	const struct pwl_result_data *rgb,
+	uint32_t num,
+	uint32_t mpcc_id)
+{
+	uint32_t i, red, green, blue;
+	uint32_t  red_delta, green_delta, blue_delta;
+	uint32_t  red_value, green_value, blue_value;
+
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	for (i = 0; i < num; i++) {
+
+		red   = rgb[i].red_reg;
+		green = rgb[i].green_reg;
+		blue  = rgb[i].blue_reg;
+
+		red_delta   = rgb[i].delta_red_reg;
+		green_delta = rgb[i].delta_green_reg;
+		blue_delta  = rgb[i].delta_blue_reg;
+
+		red_value   = ((red_delta   & 0x3ff) << 14) | (red   & 0x3fff);
+		green_value = ((green_delta & 0x3ff) << 14) | (green & 0x3fff);
+		blue_value  = ((blue_delta  & 0x3ff) << 14) | (blue  & 0x3fff);
+
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, red_value);
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, green_value);
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, blue_value);
+	}
+}
+
+void mpc42_enable_3dlut_fl(struct mpc *mpc, bool enable, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	//if enabled cho0se mpc 0, else: off (default value)
+	REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, enable ? 0 : 0xF); //0xF is not connected
+
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPC_RMCM_3DLUT_WRITE_EN_MASK, 0);
+
+	REG_UPDATE(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_DIS, enable ? 0 : 3);
+}
+
+void mpc42_update_3dlut_fast_load_select(struct mpc *mpc, int mpcc_id, int hubp_idx)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_SET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
+		MPC_RMCM_3DLUT_FL_SEL,
+		hubp_idx);
+}
+
+void mpc42_populate_rmcm_lut(struct mpc *mpc, const union mcm_lut_params params,
+	bool lut_bank_a, int mpcc_id)
+{
+	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
+	const struct pwl_params *lut_shaper = params.pwl;
+
+	if (lut_shaper == NULL)
+		return;
+	if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+		mpc42_power_on_rmcm_shaper_3dlut(mpc, mpcc_id, true);
+
+	mpc42_configure_rmcm_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id);
+
+	if (next_mode == LUT_RAM_A)
+		mpc42_program_rmcm_shaper_luta_settings(mpc, lut_shaper, mpcc_id);
+	else
+		mpc42_program_rmcm_shaper_lutb_settings(mpc, lut_shaper, mpcc_id);
+
+	mpc42_program_rmcm_shaper_lut(
+			mpc, lut_shaper->rgb_resulted, lut_shaper->hw_points_num, mpcc_id);
+
+	mpc42_power_on_rmcm_shaper_3dlut(mpc, mpcc_id, false);
+}
+
+void mpc42_program_rmcm_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id,
+	bool lut_bank_a, bool enabled, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_MODE,
+			(!enabled) ? 0 :
+			(lut_bank_a) ? 1 : 2);
+
+		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+			MPC_RMCM_3DLUT_RAM_SEL,
+			(lut_bank_a) ? 0 : 1);
+		break;
+	case MCM_LUT_SHAPER:
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL,
+			lut_bank_a == true ? 0:1);
+
+		REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[mpcc_id], 0,
+			MPC_RMCM_SHAPER_LUT_INDEX, 0);
+		break;
+	default:
+		break;
+	}
+}
+
+void mpc42_program_lut_mode(struct mpc *mpc,
+	bool enable,
+	bool lut_bank_a,
+	int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	if (enable) {
+		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
+	} else {
+		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, 0);
+	}
+}
+
+static uint32_t mpc42_get_rmcm_3dlut_width(
+		const enum dc_cm_lut_size size)
+{
+	uint32_t width = 0;
+
+	switch (size) {
+	case CM_LUT_SIZE_333333:
+		width = 2;
+		break;
+	case CM_LUT_SIZE_171717:
+	default:
+		width = 0;
+		break;
+	}
+
+	return width;
+}
+
+void mpc42_program_rmcm_3dlut_size(struct mpc *mpc,
+		const enum dc_cm_lut_size size,
+		int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+	uint32_t width = mpc42_get_rmcm_3dlut_width(size);
+
+	REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id],
+			MPC_RMCM_3DLUT_SIZE, width);
+}
+
+void mpc42_program_rmcm_3dlut_fast_load_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_R, bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_R, scale);
+
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_G[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_G, bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_G, scale);
+
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_B[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_B, bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_B, scale);
+}
+
+void mpc42_program_rmcm_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
+
+	//program bit_depth
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+		MPC_RMCM_3DLUT_30BIT_EN,
+		(bit_depth == 10) ? 1 : 0);
+}
+
+void mpc42_set_fl_config(
+	struct mpc *mpc,
+	struct mpc_fl_3dlut_config *cfg,
+	int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	uint32_t width = mpc42_get_rmcm_3dlut_width(cfg->size);
+	/*
+	From: Jie Zhou
+
+		To program any of the memories content.  The following sequence is used.
+	Set the MPCC_OGAM/SHAPER/3DLUT/1DLUT_PWR_DIS to 1 (Only need to set the one
+	that is being programmed) Set DISPCLK_G_PIPE<i>_GATE_DISABLE to 1 for the
+	MPCC pipe that’s being used, so the memory’s clock is ungated. Program the
+	target memory. Set the MPCC_OGAM/SHAPER/3DLUT/1DLUT_PWR_DIS back to 0.
+	Set DISPCLK_G_PIPE<i>_GATE_DISABLE back to 0
+	*/
+
+	//disconnect fl from mpc
+	REG_SET(MPCC_MCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
+		MPCC_MCM_3DLUT_FL_SEL, 0xF);
+
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+		MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
+
+	//program bit_depth
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+		MPC_RMCM_3DLUT_30BIT_EN, (cfg->bit_depth == 10) ? 1 : 0);
+
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+		MPC_RMCM_3DLUT_RAM_SEL, (cfg->select_lut_bank_a) ? 0 : 1);
+
+	//bias and scale
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_R, cfg->bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_R, cfg->scale);
+
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_G[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_G, cfg->bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_G, cfg->scale);
+
+	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_B[mpcc_id],
+		MPC_RMCM_3DLUT_OUT_OFFSET_B, cfg->bias,
+		MPC_RMCM_3DLUT_OUT_SCALE_B, cfg->scale);
+
+	//width
+	REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[mpcc_id],
+		MPC_RMCM_3DLUT_SIZE, width,
+		MPC_RMCM_3DLUT_MODE, (!cfg->enabled) ? 0 : (cfg->select_lut_bank_a) ? 1 : 2);
+
+	//connect to hubp
+	REG_SET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
+		MPC_RMCM_3DLUT_FL_SEL, cfg->hubp_index);
+
+	//ENABLE
+	//if enabled pick mpc 0, else: off (0xF)
+	//in future we'll select specific MPC
+	REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, cfg->enabled ? 0 : 0xF);
+}
+
+void mpc42_read_mpcc_state(
+		struct mpc *mpc,
+		int mpcc_inst,
+		struct mpcc_state *s)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	mpc1_read_mpcc_state(mpc, mpcc_inst, s);
+
+	if (mpcc_inst < 2) {
+		/* RMCM 3DLUT Status */
+		REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[mpcc_inst], MPC_RMCM_3DLUT_MEM_PWR_FORCE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_force,
+				MPC_RMCM_3DLUT_MEM_PWR_DIS, &s->rmcm_regs.rmcm_3dlut_mem_pwr_dis,
+				MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_mode,
+				MPC_RMCM_3DLUT_MEM_PWR_STATE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_state);
+
+		REG_GET_3(MPC_RMCM_3DLUT_MODE[mpcc_inst], MPC_RMCM_3DLUT_SIZE, &s->rmcm_regs.rmcm_3dlut_size,
+				MPC_RMCM_3DLUT_MODE, &s->rmcm_regs.rmcm_3dlut_mode,
+				MPC_RMCM_3DLUT_MODE_CURRENT, &s->rmcm_regs.rmcm_3dlut_mode_cur);
+
+		REG_GET_4(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_inst], MPC_RMCM_3DLUT_READ_SEL, &s->rmcm_regs.rmcm_3dlut_read_sel,
+				MPC_RMCM_3DLUT_30BIT_EN, &s->rmcm_regs.rmcm_3dlut_30bit_en,
+				MPC_RMCM_3DLUT_WRITE_EN_MASK, &s->rmcm_regs.rmcm_3dlut_wr_en_mask,
+				MPC_RMCM_3DLUT_RAM_SEL, &s->rmcm_regs.rmcm_3dlut_ram_sel);
+
+		REG_GET(MPC_RMCM_3DLUT_OUT_NORM_FACTOR[mpcc_inst], MPC_RMCM_3DLUT_OUT_NORM_FACTOR, &s->rmcm_regs.rmcm_3dlut_out_norm_factor);
+
+		REG_GET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_inst], MPC_RMCM_3DLUT_FL_SEL, &s->rmcm_regs.rmcm_3dlut_fl_sel);
+
+		REG_GET_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_inst], MPC_RMCM_3DLUT_OUT_OFFSET_R, &s->rmcm_regs.rmcm_3dlut_out_offset_r,
+				MPC_RMCM_3DLUT_OUT_SCALE_R, &s->rmcm_regs.rmcm_3dlut_out_scale_r);
+
+		REG_GET_3(MPC_RMCM_3DLUT_FAST_LOAD_STATUS[mpcc_inst], MPC_RMCM_3DLUT_FL_DONE, &s->rmcm_regs.rmcm_3dlut_fl_done,
+				MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, &s->rmcm_regs.rmcm_3dlut_fl_soft_underflow,
+				MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, &s->rmcm_regs.rmcm_3dlut_fl_hard_underflow);
+
+		/* RMCM Shaper Status */
+		REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[mpcc_inst], MPC_RMCM_SHAPER_MEM_PWR_FORCE, &s->rmcm_regs.rmcm_shaper_mem_pwr_force,
+				MPC_RMCM_SHAPER_MEM_PWR_DIS, &s->rmcm_regs.rmcm_shaper_mem_pwr_dis,
+				MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, &s->rmcm_regs.rmcm_shaper_mem_pwr_mode,
+				MPC_RMCM_SHAPER_MEM_PWR_STATE, &s->rmcm_regs.rmcm_shaper_mem_pwr_state);
+
+		REG_GET_2(MPC_RMCM_SHAPER_CONTROL[mpcc_inst], MPC_RMCM_SHAPER_LUT_MODE, &s->rmcm_regs.rmcm_shaper_lut_mode,
+				MPC_RMCM_SHAPER_MODE_CURRENT, &s->rmcm_regs.rmcm_shaper_mode_cur);
+
+		REG_GET_2(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_inst], MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, &s->rmcm_regs.rmcm_shaper_lut_write_en_mask,
+				MPC_RMCM_SHAPER_LUT_WRITE_SEL, &s->rmcm_regs.rmcm_shaper_lut_write_sel);
+
+		REG_GET(MPC_RMCM_SHAPER_OFFSET_B[mpcc_inst], MPC_RMCM_SHAPER_OFFSET_B, &s->rmcm_regs.rmcm_shaper_offset_b);
+
+		REG_GET(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_inst], MPC_RMCM_SHAPER_SCALE_B, &s->rmcm_regs.rmcm_shaper_scale_b);
+
+		REG_GET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[mpcc_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_start_b,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_start_seg_b);
+
+		REG_GET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[mpcc_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_end_b,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_end_base_b);
+
+		REG_GET(MPC_RMCM_CNTL[mpcc_inst], MPC_RMCM_CNTL, &s->rmcm_regs.rmcm_cntl);
+	}
+}
+
+static const struct mpc_funcs dcn42_mpc_funcs = {
+	.read_mpcc_state = mpc42_read_mpcc_state,
+	.insert_plane = mpc1_insert_plane,
+	.remove_mpcc = mpc1_remove_mpcc,
+	.mpc_init = mpc32_mpc_init,
+	.mpc_init_single_inst = mpc3_mpc_init_single_inst,
+	.update_blending = mpc42_update_blending,
+	.cursor_lock = mpc1_cursor_lock,
+	.get_mpcc_for_dpp = mpc1_get_mpcc_for_dpp,
+	.wait_for_idle = mpc2_assert_idle_mpcc,
+	.assert_mpcc_idle_before_connect = mpc2_assert_mpcc_idle_before_connect,
+	.init_mpcc_list_from_hw = mpc1_init_mpcc_list_from_hw,
+	.set_denorm =  mpc3_set_denorm,
+	.set_denorm_clamp = mpc3_set_denorm_clamp,
+	.set_output_csc = mpc3_set_output_csc,
+	.set_ocsc_default = mpc3_set_ocsc_default,
+	.set_output_gamma = mpc3_set_output_gamma,
+	.set_dwb_mux = mpc3_set_dwb_mux,
+	.disable_dwb_mux = mpc3_disable_dwb_mux,
+	.is_dwb_idle = mpc3_is_dwb_idle,
+	.set_gamut_remap = mpc401_set_gamut_remap,
+	.program_shaper = mpc32_program_shaper,
+	.program_3dlut = mpc32_program_3dlut,
+	.program_1dlut = mpc32_program_post1dlut,
+	.power_on_mpc_mem_pwr = mpc3_power_on_ogam_lut,
+	.get_mpc_out_mux = mpc1_get_mpc_out_mux,
+	.mpc_read_reg_state = mpc3_read_reg_state,
+	.set_bg_color = mpc1_set_bg_color,
+	.set_movable_cm_location = mpc401_set_movable_cm_location,
+	.update_3dlut_fast_load_select = mpc401_update_3dlut_fast_load_select,
+	.get_3dlut_fast_load_status = mpc401_get_3dlut_fast_load_status,
+	.populate_lut = mpc401_populate_lut,
+	.program_lut_read_write_control = mpc401_program_lut_read_write_control,
+	.program_lut_mode = mpc401_program_lut_mode,
+	.get_lut_mode = mpc401_get_lut_mode,
+	.rmcm = {
+		.enable_3dlut_fl = mpc42_enable_3dlut_fl,
+		.update_3dlut_fast_load_select = mpc42_update_3dlut_fast_load_select,
+		.program_lut_read_write_control = mpc42_program_rmcm_lut_read_write_control,
+		.program_lut_mode = mpc42_program_lut_mode,
+		.program_3dlut_size = mpc42_program_rmcm_3dlut_size,
+		.program_bias_scale = mpc42_program_rmcm_3dlut_fast_load_bias_scale,
+		.program_bit_depth = mpc42_program_rmcm_bit_depth,
+		.power_on_shaper_3dlut = mpc42_power_on_rmcm_shaper_3dlut,
+		.populate_lut = mpc42_populate_rmcm_lut,
+		.fl_3dlut_configure = mpc42_set_fl_config,
+	},
+};
+
+void dcn42_mpc_construct(struct dcn42_mpc *mpc42,
+	struct dc_context *ctx,
+	const struct dcn42_mpc_registers *mpc_regs,
+	const struct dcn42_mpc_shift *mpc_shift,
+	const struct dcn42_mpc_mask *mpc_mask,
+	int num_mpcc,
+	int num_rmu)
+{
+	int i;
+
+	mpc42->base.ctx = ctx;
+
+	mpc42->base.funcs = &dcn42_mpc_funcs;
+
+	mpc42->mpc_regs = mpc_regs;
+	mpc42->mpc_shift = mpc_shift;
+	mpc42->mpc_mask = mpc_mask;
+
+	mpc42->mpcc_in_use_mask = 0;
+	mpc42->num_mpcc = num_mpcc;
+	mpc42->num_rmu = num_rmu;
+
+	for (i = 0; i < MAX_MPCC; i++)
+		mpc42_init_mpcc(&mpc42->base.mpcc_array[i], i);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
new file mode 100644
index 0000000000000..a5f7f4f2bb3bc
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
@@ -0,0 +1,962 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef __DC_MPCC_DCN42_H__
+#define __DC_MPCC_DCN42_H__
+
+#include "dcn401/dcn401_mpc.h"
+
+#define TO_DCN42_MPC(mpc_base) \
+	container_of(mpc_base, struct dcn42_mpc, base)
+
+#define MPC_COMMON_MASK_SH_LIST_DCN42(mask_sh) \
+	SF(MPCC0_MPCC_TOP_SEL, MPCC_TOP_SEL, mask_sh),\
+	SF(MPCC0_MPCC_BOT_SEL, MPCC_BOT_SEL, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_MODE, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_ALPHA_BLND_MODE, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_ALPHA_MULTIPLIED_MODE, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_BLND_ACTIVE_OVERLAP_ONLY, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL2, MPCC_GLOBAL_ALPHA, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL2, MPCC_GLOBAL_GAIN, mask_sh),\
+	SF(MPCC0_MPCC_STATUS, MPCC_IDLE, mask_sh),\
+	SF(MPCC0_MPCC_STATUS, MPCC_BUSY, mask_sh),\
+	SF(MPCC0_MPCC_OPP_ID, MPCC_OPP_ID, mask_sh),\
+	SF(MPCC0_MPCC_BG_G_Y, MPCC_BG_G_Y, mask_sh),\
+	SF(MPCC0_MPCC_BG_R_CR, MPCC_BG_R_CR, mask_sh),\
+	SF(MPCC0_MPCC_BG_B_CB, MPCC_BG_B_CB, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_EN, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_MODE, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FRAME_ALT, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FIELD_ALT, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FORCE_NEXT_FRAME_POL, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FORCE_NEXT_TOP_POL, mask_sh),\
+	SF(MPC_OUT0_MUX, MPC_OUT_MUX, mask_sh),\
+	SF(MPCC0_MPCC_UPDATE_LOCK_SEL, MPCC_UPDATE_LOCK_SEL, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_BG_BPC, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_BOT_GAIN_MODE, mask_sh),\
+	SF(MPCC0_MPCC_TOP_GAIN, MPCC_TOP_GAIN, mask_sh),\
+	SF(MPCC0_MPCC_BOT_GAIN_INSIDE, MPCC_BOT_GAIN_INSIDE, mask_sh),\
+	SF(MPCC0_MPCC_BOT_GAIN_OUTSIDE, MPCC_BOT_GAIN_OUTSIDE, mask_sh),\
+	SF(MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC_MOVABLE_CM_LOCATION_CNTL, mask_sh),\
+	SF(MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT, mask_sh),\
+	SF(MPC_OUT0_CSC_MODE, MPC_OCSC_MODE, mask_sh),\
+	SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C11_A, mask_sh),\
+	SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C12_A, mask_sh),\
+	SF(MPCC0_MPCC_STATUS, MPCC_DISABLED, mask_sh),\
+	SF(MPCC0_MPCC_MEM_PWR_CTRL, MPCC_OGAM_MEM_PWR_FORCE, mask_sh),\
+	SF(MPCC0_MPCC_MEM_PWR_CTRL, MPCC_OGAM_MEM_PWR_DIS, mask_sh),\
+	SF(MPCC0_MPCC_MEM_PWR_CTRL, MPCC_OGAM_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPCC0_MPCC_MEM_PWR_CTRL, MPCC_OGAM_MEM_PWR_STATE, mask_sh),\
+	SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_MODE, mask_sh),\
+	SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_CLAMP_MAX_R_CR, mask_sh),\
+	SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_CLAMP_MIN_R_CR, mask_sh),\
+	SF(MPC_OUT0_DENORM_CLAMP_G_Y, MPC_OUT_DENORM_CLAMP_MAX_G_Y, mask_sh),\
+	SF(MPC_OUT0_DENORM_CLAMP_G_Y, MPC_OUT_DENORM_CLAMP_MIN_G_Y, mask_sh),\
+	SF(MPC_OUT0_DENORM_CLAMP_B_CB, MPC_OUT_DENORM_CLAMP_MAX_B_CB, mask_sh),\
+	SF(MPC_OUT0_DENORM_CLAMP_B_CB, MPC_OUT_DENORM_CLAMP_MIN_B_CB, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE, MPCC_GAMUT_REMAP_MODE, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE, MPCC_GAMUT_REMAP_MODE_CURRENT, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT, MPCC_GAMUT_REMAP_COEF_FORMAT, mask_sh),\
+	SF(MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A, MPCC_GAMUT_REMAP_C11_A, mask_sh),\
+	SF(MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A, MPCC_GAMUT_REMAP_C12_A, mask_sh),\
+	SF(MPC_DWB0_MUX, MPC_DWB0_MUX, mask_sh),\
+	SF(MPC_DWB0_MUX, MPC_DWB0_MUX_STATUS, mask_sh),\
+	SF(MPC_OUT0_MUX, MPC_OUT_RATE_CONTROL, mask_sh),\
+	SF(MPC_OUT0_MUX, MPC_OUT_RATE_CONTROL_DISABLE, mask_sh),\
+	SF(MPC_OUT0_MUX, MPC_OUT_FLOW_CONTROL_MODE, mask_sh),\
+	SF(MPC_OUT0_MUX, MPC_OUT_FLOW_CONTROL_COUNT, mask_sh), \
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM_RAMA_EXP_REGION_END_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B, MPCC_OGAM_RAMA_OFFSET_B, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G, MPCC_OGAM_RAMA_OFFSET_G, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R, MPCC_OGAM_RAMA_OFFSET_R, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_INDEX, MPCC_OGAM_LUT_INDEX, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_CONTROL, MPCC_OGAM_MODE, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_CONTROL, MPCC_OGAM_SELECT, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_CONTROL, MPCC_OGAM_PWL_DISABLE, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_CONTROL, MPCC_OGAM_MODE_CURRENT, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_CONTROL, MPCC_OGAM_SELECT_CURRENT, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL, MPCC_OGAM_LUT_WRITE_COLOR_MASK, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL, MPCC_OGAM_LUT_READ_COLOR_SEL, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL, MPCC_OGAM_LUT_HOST_SEL, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL, MPCC_OGAM_LUT_CONFIG_MODE, mask_sh),\
+	SF(MPCC_OGAM0_MPCC_OGAM_LUT_DATA, MPCC_OGAM_LUT_DATA, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_MODE, MPCC_MCM_3DLUT_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_MODE, MPCC_MCM_3DLUT_SIZE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_MODE, MPCC_MCM_3DLUT_MODE_CURRENT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM_3DLUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM_3DLUT_RAM_SEL, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM_3DLUT_30BIT_EN, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM_3DLUT_READ_SEL, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_INDEX, MPCC_MCM_3DLUT_INDEX, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_DATA, MPCC_MCM_3DLUT_DATA0, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_DATA, MPCC_MCM_3DLUT_DATA1, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM_3DLUT_DATA_30BIT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL, MPCC_MCM_SHAPER_LUT_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL, MPCC_MCM_SHAPER_MODE_CURRENT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM_SHAPER_OFFSET_R, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM_SHAPER_OFFSET_G, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM_SHAPER_OFFSET_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM_SHAPER_SCALE_R, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM_SHAPER_SCALE_G, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM_SHAPER_SCALE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM_SHAPER_LUT_INDEX, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM_SHAPER_LUT_DATA, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM_SHAPER_LUT_WRITE_SEL, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL, MPCC_MCM_1DLUT_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL, MPCC_MCM_1DLUT_SELECT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL, MPCC_MCM_1DLUT_PWL_DISABLE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL, MPCC_MCM_1DLUT_MODE_CURRENT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL, MPCC_MCM_1DLUT_SELECT_CURRENT, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX, MPCC_MCM_1DLUT_LUT_INDEX, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA, MPCC_MCM_1DLUT_LUT_DATA, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM_1DLUT_LUT_HOST_SEL, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM_1DLUT_LUT_CONFIG_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B, MPCC_MCM_1DLUT_RAMA_OFFSET_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_SHAPER_MEM_PWR_FORCE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_SHAPER_MEM_PWR_DIS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_3DLUT_MEM_PWR_FORCE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_3DLUT_MEM_PWR_DIS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_1DLUT_MEM_PWR_FORCE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_1DLUT_MEM_PWR_DIS, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_SHAPER_MEM_PWR_STATE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_3DLUT_MEM_PWR_STATE, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM_1DLUT_MEM_PWR_STATE, mask_sh),\
+	SF(CUR_VUPDATE_LOCK_SET0, CUR_VUPDATE_LOCK_SET, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT, MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE, MPCC_MCM_FIRST_GAMUT_REMAP_MODE, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE, MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A, MPCC_MCM_FIRST_GAMUT_REMAP_C11_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A, MPCC_MCM_FIRST_GAMUT_REMAP_C12_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A, MPCC_MCM_FIRST_GAMUT_REMAP_C13_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A, MPCC_MCM_FIRST_GAMUT_REMAP_C14_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A, MPCC_MCM_FIRST_GAMUT_REMAP_C21_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A, MPCC_MCM_FIRST_GAMUT_REMAP_C22_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A, MPCC_MCM_FIRST_GAMUT_REMAP_C23_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A, MPCC_MCM_FIRST_GAMUT_REMAP_C24_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A, MPCC_MCM_FIRST_GAMUT_REMAP_C31_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A, MPCC_MCM_FIRST_GAMUT_REMAP_C32_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A, MPCC_MCM_FIRST_GAMUT_REMAP_C33_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A, MPCC_MCM_FIRST_GAMUT_REMAP_C34_A, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT, MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE, MPCC_MCM_SECOND_GAMUT_REMAP_MODE, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE, MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A, MPCC_MCM_SECOND_GAMUT_REMAP_C11_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A, MPCC_MCM_SECOND_GAMUT_REMAP_C12_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A, MPCC_MCM_SECOND_GAMUT_REMAP_C13_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A, MPCC_MCM_SECOND_GAMUT_REMAP_C14_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A, MPCC_MCM_SECOND_GAMUT_REMAP_C21_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A, MPCC_MCM_SECOND_GAMUT_REMAP_C22_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A, MPCC_MCM_SECOND_GAMUT_REMAP_C23_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A, MPCC_MCM_SECOND_GAMUT_REMAP_C24_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A, MPCC_MCM_SECOND_GAMUT_REMAP_C31_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A, MPCC_MCM_SECOND_GAMUT_REMAP_C32_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A, MPCC_MCM_SECOND_GAMUT_REMAP_C33_A, mask_sh), \
+	SF(MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A, MPCC_MCM_SECOND_GAMUT_REMAP_C34_A, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT, MPCC_MCM_3DLUT_FL_SEL, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS, MPCC_MCM_3DLUT_FL_DONE, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS, MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS, MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW, mask_sh), \
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM_3DLUT_OUT_OFFSET_R, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM_3DLUT_OUT_SCALE_R, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM_3DLUT_OUT_OFFSET_G, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM_3DLUT_OUT_SCALE_G, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_OFFSET_B, mask_sh),\
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_SCALE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_LUT_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM_SHAPER_OFFSET_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM_SHAPER_OFFSET_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM_SHAPER_OFFSET_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM_SHAPER_SCALE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM_SHAPER_LUT_INDEX, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM_SHAPER_LUT_DATA, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_SIZE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_INDEX, MPC_RMCM_3DLUT_INDEX, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA0, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA1, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM_3DLUT_DATA_30BIT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_RAM_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_30BIT_EN, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_READ_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM_3DLUT_OUT_NORM_FACTOR, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_OFFSET_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_SCALE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_OFFSET_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_SCALE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_OFFSET_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_SCALE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C11_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C12_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C13_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C14_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C21_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C22_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C23_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C24_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C31_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C32_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C33_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C34_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C11_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C12_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C13_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C14_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C21_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C22_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C23_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C24_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C31_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C32_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C33_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C34_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_FORCE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_DIS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_FORCE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_DIS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_STATE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_STATE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM_3DLUT_FL_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_DONE, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_CNTL, MPC_RMCM_CNTL, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_INDEX, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_WRITE_EN, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM_TEST_DEBUG_DATA, mask_sh)
+
+#define MPC_RMCM_REG_LIST_DCN42(inst)\
+	SRII(MPC_RMCM_SHAPER_CONTROL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_MODE, MPC_RMCM, inst), /*TODO: may need to add other 3DLUT regs*/\
+	SRII(MPC_RMCM_3DLUT_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_DATA, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_CNTL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM, inst)
+
+
+#define MPC_REG_LIST_DCN42(inst) \
+	MPC_REG_LIST_DCN4_01_RI(inst),\
+	SRII(MPCC_CONTROL2, MPCC, inst)
+
+#define MPC_REG_FIELD_LIST_DCN42(type) \
+	MPC_REG_FIELD_LIST_DCN4_01(type); \
+	type MPCC_MCM_3DLUT_OUT_OFFSET_R;\
+	type MPCC_MCM_3DLUT_OUT_SCALE_R;\
+	type MPCC_MCM_3DLUT_OUT_OFFSET_G;\
+	type MPCC_MCM_3DLUT_OUT_SCALE_G;\
+	type MPCC_MCM_3DLUT_OUT_OFFSET_B;\
+	type MPCC_MCM_3DLUT_OUT_SCALE_B;\
+	type MPC_RMCM_SHAPER_LUT_MODE;\
+	type MPC_RMCM_SHAPER_MODE_CURRENT;\
+	type MPC_RMCM_SHAPER_OFFSET_R;\
+	type MPC_RMCM_SHAPER_OFFSET_G;\
+	type MPC_RMCM_SHAPER_OFFSET_B;\
+	type MPC_RMCM_SHAPER_SCALE_R;\
+	type MPC_RMCM_SHAPER_SCALE_G;\
+	type MPC_RMCM_SHAPER_SCALE_B;\
+	type MPC_RMCM_SHAPER_LUT_INDEX;\
+	type MPC_RMCM_SHAPER_LUT_DATA;\
+	type MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK;\
+	type MPC_RMCM_SHAPER_LUT_WRITE_SEL; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS;\
+	type MPC_RMCM_3DLUT_MODE;\
+	type MPC_RMCM_3DLUT_SIZE;\
+	type MPC_RMCM_3DLUT_MODE_CURRENT;\
+	type MPC_RMCM_3DLUT_INDEX; \
+	type MPC_RMCM_3DLUT_DATA0; \
+	type MPC_RMCM_3DLUT_DATA1; \
+	type MPC_RMCM_3DLUT_DATA_30BIT; \
+	type MPC_RMCM_3DLUT_WRITE_EN_MASK;\
+	type MPC_RMCM_3DLUT_RAM_SEL;\
+	type MPC_RMCM_3DLUT_30BIT_EN;\
+	type MPC_RMCM_3DLUT_READ_SEL;\
+	type MPC_RMCM_3DLUT_OUT_NORM_FACTOR;\
+	type MPC_RMCM_3DLUT_OUT_OFFSET_R;\
+	type MPC_RMCM_3DLUT_OUT_SCALE_R; \
+	type MPC_RMCM_3DLUT_OUT_OFFSET_G; \
+	type MPC_RMCM_3DLUT_OUT_SCALE_G; \
+	type MPC_RMCM_3DLUT_OUT_OFFSET_B; \
+	type MPC_RMCM_3DLUT_OUT_SCALE_B;\
+	type MPC_RMCM_GAMUT_REMAP_COEF_FORMAT;\
+	type MPC_RMCM_GAMUT_REMAP_MODE;\
+	type MPC_RMCM_GAMUT_REMAP_MODE_CURRENT;\
+	type MPC_RMCM_GAMUT_REMAP_C11_A;\
+	type MPC_RMCM_GAMUT_REMAP_C12_A;\
+	type MPC_RMCM_GAMUT_REMAP_C13_A; \
+	type MPC_RMCM_GAMUT_REMAP_C14_A; \
+	type MPC_RMCM_GAMUT_REMAP_C21_A; \
+	type MPC_RMCM_GAMUT_REMAP_C22_A; \
+	type MPC_RMCM_GAMUT_REMAP_C23_A;\
+	type MPC_RMCM_GAMUT_REMAP_C24_A;\
+	type MPC_RMCM_GAMUT_REMAP_C31_A;\
+	type MPC_RMCM_GAMUT_REMAP_C32_A;\
+	type MPC_RMCM_GAMUT_REMAP_C33_A;\
+	type MPC_RMCM_GAMUT_REMAP_C34_A;\
+	type MPC_RMCM_GAMUT_REMAP_C11_B;\
+	type MPC_RMCM_GAMUT_REMAP_C12_B;\
+	type MPC_RMCM_GAMUT_REMAP_C13_B; \
+	type MPC_RMCM_GAMUT_REMAP_C14_B; \
+	type MPC_RMCM_GAMUT_REMAP_C21_B; \
+	type MPC_RMCM_GAMUT_REMAP_C22_B; \
+	type MPC_RMCM_GAMUT_REMAP_C23_B;\
+	type MPC_RMCM_GAMUT_REMAP_C24_B;\
+	type MPC_RMCM_GAMUT_REMAP_C31_B;\
+	type MPC_RMCM_GAMUT_REMAP_C32_B;\
+	type MPC_RMCM_GAMUT_REMAP_C33_B;\
+	type MPC_RMCM_GAMUT_REMAP_C34_B;\
+	type MPC_RMCM_SHAPER_MEM_PWR_FORCE; \
+	type MPC_RMCM_SHAPER_MEM_PWR_DIS; \
+	type MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE; \
+	type MPC_RMCM_3DLUT_MEM_PWR_FORCE;\
+	type MPC_RMCM_3DLUT_MEM_PWR_DIS;\
+	type MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE;\
+	type MPC_RMCM_SHAPER_MEM_PWR_STATE;\
+	type MPC_RMCM_3DLUT_MEM_PWR_STATE;\
+	type MPC_RMCM_3DLUT_FL_SEL;\
+	type MPC_RMCM_3DLUT_FL_DONE; \
+	type MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW; \
+	type MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW; \
+	type MPC_RMCM_CNTL; \
+	type MPC_RMCM_TEST_DEBUG_INDEX;\
+	type MPC_RMCM_TEST_DEBUG_WRITE_EN;\
+	type MPC_RMCM_TEST_DEBUG_DATA
+
+#define MPC_REG_VARIABLE_LIST_DCN42 \
+	MPC_REG_VARIABLE_LIST_DCN4_01 \
+	uint32_t MPCC_CONTROL2[MAX_MPCC];\
+	uint32_t MPC_RMCM_SHAPER_CONTROL[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_SCALE_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_SCALE_G_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_INDEX[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_DATA[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_0_1[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_2_3[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_4_5[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_6_7[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_8_9[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_10_11[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_12_13[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_14_15[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_16_17[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_18_19[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_20_21[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_22_23[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_24_25[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_26_27[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_28_29[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_30_31[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_32_33[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_0_1[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_2_3[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_4_5[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_6_7[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_8_9[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_10_11[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_12_13[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_14_15[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_16_17[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_18_19[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_20_21[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_22_23[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_24_25[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_26_27[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_28_29[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_30_31[MAX_MPCC]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_32_33[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_MODE[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_INDEX[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_DATA[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_DATA_30BIT[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_READ_WRITE_CONTROL[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_NORM_FACTOR[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_R[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_G[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_COEF_FORMAT[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_MODE[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_A[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_B[MAX_MPCC]; \
+	uint32_t MPC_RMCM_MEM_PWR_CTRL[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_SELECT[MAX_MPCC]; \
+	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_STATUS[MAX_MPCC]; \
+	uint32_t MPC_RMCM_CNTL[MAX_MPCC]; \
+	uint32_t MPC_RMCM_TEST_DEBUG_INDEX[MAX_MPCC]; \
+	uint32_t MPC_RMCM_TEST_DEBUG_DATA[MAX_MPCC]
+
+struct dcn42_mpc_shift {
+	MPC_REG_FIELD_LIST_DCN42(uint8_t);
+};
+
+struct dcn42_mpc_mask {
+	MPC_REG_FIELD_LIST_DCN42(uint32_t);
+};
+
+struct dcn42_mpc_registers {
+	MPC_REG_VARIABLE_LIST_DCN42;
+};
+
+struct dcn42_mpc {
+	struct mpc base;
+
+	int mpcc_in_use_mask;
+	int num_mpcc;
+	const struct dcn42_mpc_registers *mpc_regs;
+	const struct dcn42_mpc_shift *mpc_shift;
+	const struct dcn42_mpc_mask *mpc_mask;
+	int num_rmu;
+};
+void dcn42_mpc_construct(struct dcn42_mpc *mpc401,
+	struct dc_context *ctx,
+	const struct dcn42_mpc_registers *mpc_regs,
+	const struct dcn42_mpc_shift *mpc_shift,
+	const struct dcn42_mpc_mask *mpc_mask,
+	int num_mpcc,
+	int num_rmu);
+
+void mpc42_init_mpcc(struct mpcc *mpcc, int mpcc_inst);
+
+/* RMCM */
+void mpc42_program_rmcm_shaper_lut(
+		struct mpc *mpc,
+		const struct pwl_result_data *rgb,
+		uint32_t num,
+		uint32_t mpcc_id);
+void mpc42_program_rmcm_shaper_lutb_settings(
+		struct mpc *mpc,
+		const struct pwl_params *params,
+		uint32_t mpcc_id);
+void mpc42_program_rmcm_shaper_luta_settings(
+		struct mpc *mpc,
+		const struct pwl_params *params,
+		uint32_t mpcc_id);
+void mpc42_configure_rmcm_shaper_lut(
+		struct mpc *mpc,
+		bool is_ram_a,
+		uint32_t mpcc_id);
+void mpc42_power_on_rmcm_shaper_3dlut(
+		struct mpc *mpc,
+		uint32_t mpcc_id,
+		bool power_on);
+void mpc42_enable_3dlut_fl(
+		struct mpc *mpc,
+		bool enable,
+		int mpcc_id);
+void mpc42_update_3dlut_fast_load_select(
+		struct mpc *mpc,
+		int mpcc_id,
+		int hubp_idx);
+void mpc42_populate_rmcm_lut(
+		struct mpc *mpc,
+		const union mcm_lut_params params,
+		bool lut_bank_a,
+		int mpcc_id);
+void mpc42_program_rmcm_lut_read_write_control(
+		struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		bool lut_bank_a,
+		bool enabled,
+		int mpcc_id);
+void mpc42_program_lut_mode(
+		struct mpc *mpc,
+		bool enable,
+		bool lut_bank_a,
+		int mpcc_id);
+void mpc42_program_rmcm_3dlut_size(
+		struct mpc *mpc,
+		const enum dc_cm_lut_size size,
+		int mpcc_id);
+void mpc42_program_rmcm_3dlut_fast_load_bias_scale(
+		struct mpc *mpc,
+		uint16_t bias,
+		uint16_t scale,
+		int mpcc_id);
+void mpc42_program_rmcm_bit_depth(
+		struct mpc *mpc,
+		uint16_t bit_depth,
+		int mpcc_id);
+
+void mpc42_set_fl_config(
+	struct mpc *mpc,
+	struct mpc_fl_3dlut_config *cfg,
+	int mpcc_id);
+
+void mpc42_read_mpcc_state(
+		struct mpc *mpc,
+		int mpcc_inst,
+		struct mpcc_state *s);
+
+void mpc42_update_blending(
+	struct mpc *mpc,
+	struct mpcc_blnd_cfg *blnd_cfg,
+	int mpcc_id);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/optc/Makefile b/drivers/gpu/drm/amd/display/dc/optc/Makefile
index 29fb610c8660f..06c98d70eabcc 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/optc/Makefile
@@ -104,11 +104,19 @@ AMD_DISPLAY_FILES += $(AMD_DAL_OPTC_DCN35)
 
 ###############################################################################
 
-###############################################################################
 OPTC_DCN401 = dcn401_optc.o
 
 AMD_DAL_OPTC_DCN401 = $(addprefix $(AMDDALPATH)/dc/optc/dcn401/,$(OPTC_DCN401))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_OPTC_DCN401)
+
+###############################################################################
+
+OPTC_DCN42 = dcn42_optc.o
+
+AMD_DAL_OPTC_DCN42 = $(addprefix $(AMDDALPATH)/dc/optc/dcn42/,$(OPTC_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_OPTC_DCN42)
+
 endif
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
index 0b3f974f452e5..138081e6cc977 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2012-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -218,7 +218,7 @@
 	uint32_t OTG_CRC_SIG_BLUE_CONTROL_MASK; \
 	uint32_t OTG_CRC_SIG_RED_GREEN_MASK; \
 	uint32_t OTG_DLPC_CONTROL; \
-	uint32_t OTG_DRR_CONTROL2; \
+	uint32_t OTG_DRR_CONTOL2; \
 	uint32_t OTG_DRR_TIMING_INT_STATUS; \
 	uint32_t OTG_GLOBAL_CONTROL3; \
 	uint32_t OTG_GLOBAL_SYNC_STATUS; \
@@ -253,8 +253,20 @@
 	uint32_t OTG_CRC1_DATA_B32
 
 
+#define OPTC_REG_VARIABLE_LIST_DCN42 \
+	uint32_t OTG_PWA_FRAME_SYNC_CONTROL; \
+	uint32_t OTG_CRC0_DATA_R; \
+	uint32_t OTG_CRC1_DATA_R; \
+	uint32_t OTG_CRC2_DATA_R; \
+	uint32_t OTG_CRC3_DATA_R; \
+	uint32_t OTG_CRC0_DATA_G; \
+	uint32_t OTG_CRC1_DATA_G; \
+	uint32_t OTG_CRC2_DATA_G; \
+	uint32_t OTG_CRC3_DATA_G
+
 struct dcn_optc_registers {
 	OPTC_REG_VARIABLE_LIST_DCN;
+	OPTC_REG_VARIABLE_LIST_DCN42;
 };
 
 #define TG_COMMON_MASK_SH_LIST_DCN(mask_sh)\
@@ -664,6 +676,10 @@ struct dcn_optc_registers {
 	type OTG_V_COUNT_STOP_TIMER;
 
 #define TG_REG_FIELD_LIST_DCN3_6(type) \
+	type OPTC_RSMU_UNDERFLOW_CLEAR;\
+	type OPTC_RSMU_UNDERFLOW_OCCURRED_STATUS;\
+	type OPTC_RSMU_UNDERFLOW_INT_EN;\
+	type OPTC_RSMU_UNDERFLOW_INT_STATUS;\
 	type OTG_CRC_POLY_SEL; \
 	type CRC0_R_CR32; \
 	type CRC0_G_Y32; \
@@ -679,6 +695,10 @@ struct dcn_optc_registers {
 	type OTG_UNBLANK;\
 	type OTG_PSTATE_ALLOW_WIDTH_MIN;
 
+#define TG_REG_FIELD_LIST_DCN42(type) \
+	type OTG_PWA_FRAME_SYNC_EN;\
+	type OTG_PWA_FRAME_SYNC_VCOUNT_MODE;\
+	type OTG_PWA_FRAME_SYNC_LINE;
 
 struct dcn_optc_shift {
 	TG_REG_FIELD_LIST(uint8_t)
@@ -687,6 +707,7 @@ struct dcn_optc_shift {
 	TG_REG_FIELD_LIST_DCN3_5(uint8_t)
 	TG_REG_FIELD_LIST_DCN3_6(uint8_t)
 	TG_REG_FIELD_LIST_DCN401(uint8_t)
+		TG_REG_FIELD_LIST_DCN42(uint8_t)
 };
 
 struct dcn_optc_mask {
@@ -696,6 +717,7 @@ struct dcn_optc_mask {
 	TG_REG_FIELD_LIST_DCN3_5(uint32_t)
 	TG_REG_FIELD_LIST_DCN3_6(uint32_t)
 	TG_REG_FIELD_LIST_DCN401(uint32_t)
+	TG_REG_FIELD_LIST_DCN42(uint32_t)
 };
 
 void dcn10_timing_generator_init(struct optc *optc);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
index c6417538090f2..893d2aff1f826 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
@@ -363,7 +363,7 @@ void optc31_read_reg_state(struct timing_generator *optc, struct dcn_optc_reg_st
 	optc_reg_state->otg_crc3_data_rg = REG_READ(OTG_CRC3_DATA_RG);
 	optc_reg_state->otg_dlpc_control = REG_READ(OTG_DLPC_CONTROL);
 	optc_reg_state->otg_double_buffer_control = REG_READ(OTG_DOUBLE_BUFFER_CONTROL);
-	optc_reg_state->otg_drr_control2 = REG_READ(OTG_DRR_CONTROL2);
+	optc_reg_state->otg_drr_control2 = REG_READ(OTG_DRR_CONTOL2);
 	optc_reg_state->otg_drr_control = REG_READ(OTG_DRR_CONTROL);
 	optc_reg_state->otg_drr_timing_int_status = REG_READ(OTG_DRR_TIMING_INT_STATUS);
 	optc_reg_state->otg_drr_trigger_window = REG_READ(OTG_DRR_TRIGGER_WINDOW);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
new file mode 100644
index 0000000000000..ed66a2bbb8aeb
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dcn42_optc.h"
+#include "dcn30/dcn30_optc.h"
+#include "dcn31/dcn31_optc.h"
+#include "dcn32/dcn32_optc.h"
+#include "dcn35/dcn35_optc.h"
+#include "dcn401/dcn401_optc.h"
+#include "reg_helper.h"
+#include "dc.h"
+#include "dcn_calc_math.h"
+#include "dc_dmub_srv.h"
+#include "dc_trace.h"
+
+#define REG(reg)\
+	optc1->tg_regs->reg
+
+#define CTX \
+	optc1->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	optc1->tg_shift->field_name, optc1->tg_mask->field_name
+
+/*
+ * optc42_get_crc - Capture CRC result per component
+ *
+ * @optc: timing_generator instance.
+ * @r_cr: 16-bit primary CRC signature for red data.
+ * @g_y: 16-bit primary CRC signature for green data.
+ * @b_cb: 16-bit primary CRC signature for blue data.
+ *
+ * This function reads the CRC signature from the OPTC registers. Notice that
+ * we have three registers to keep the CRC result per color component (RGB).
+ *
+ * Returns:
+ * If CRC is disabled, return false; otherwise, return true, and the CRC
+ * results in the parameters.
+ */
+
+static bool optc42_get_crc(struct timing_generator *optc, uint8_t idx,
+		   uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
+{
+	uint32_t field = 0;
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_GET(OTG_CRC_CNTL, OTG_CRC_EN, &field);
+
+	/* Early return if CRC is not enabled for this CRTC */
+	if (!field)
+		return false;
+
+
+	switch (idx) {
+	case 0:
+		/* OTG_CRC0_DATA_RG has the CRC16 results for the red component */
+		REG_GET(OTG_CRC0_DATA_R,
+			CRC0_R_CR, r_cr);
+
+		/* OTG_CRC0_DATA_RG has the CRC16 results for the green component */
+		REG_GET(OTG_CRC0_DATA_G,
+			CRC0_G_Y, g_y);
+
+		/* OTG_CRC0_DATA_B has the CRC16 results for the blue component */
+		REG_GET(OTG_CRC0_DATA_B,
+			CRC0_B_CB, b_cb);
+		break;
+	case 1:
+		/* OTG_CRC1_DATA_RG has the CRC16 results for the red component */
+		REG_GET(OTG_CRC1_DATA_R,
+			CRC0_R_CR, r_cr);
+
+		/* OTG_CRC1_DATA_RG has the CRC16 results for the green component */
+		REG_GET(OTG_CRC1_DATA_G,
+			CRC0_G_Y, g_y);
+
+		/* OTG_CRC1_DATA_B has the CRC16 results for the blue component */
+		REG_GET(OTG_CRC1_DATA_B,
+			CRC0_B_CB, b_cb);
+		break;
+	default:
+		return false;
+	}
+	return true;
+}
+
+void optc42_enable_pwa(struct timing_generator *optc, struct otc_pwa_frame_sync *pwa_sync_param)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	/*VCOUNT_MODE :
+	00	OTG_PWA_FRAME_SYNC_VCOUNT_0	Vcount counting from VSYNC.
+	01	OTG_PWA_FRAME_SYNC_VCOUNT_1	Vcount counting from VSTARTUP.*/
+
+	if (pwa_sync_param == NULL)
+		return;
+	if (optc1->base.ctx->dc->debug.enable_otg_frame_sync_pwa) {
+		/*take mode 1, use line number from vstartup to get output frame as earlier as possible*/
+		REG_UPDATE_3(OTG_PWA_FRAME_SYNC_CONTROL,
+			OTG_PWA_FRAME_SYNC_EN, 1,
+			OTG_PWA_FRAME_SYNC_VCOUNT_MODE, pwa_sync_param->pwa_sync_mode,
+			OTG_PWA_FRAME_SYNC_LINE, pwa_sync_param->pwa_frame_sync_line_offset);
+	}
+}
+void  optc42_disable_pwa(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_UPDATE(OTG_PWA_FRAME_SYNC_CONTROL,
+			OTG_PWA_FRAME_SYNC_EN, 0);
+}
+void optc42_clear_optc_underflow(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_UPDATE(OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_CLEAR, 1);
+	REG_UPDATE(OPTC_RSMU_UNDERFLOW, OPTC_RSMU_UNDERFLOW_CLEAR, 1);
+}
+bool optc42_is_optc_underflow_occurred(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t underflow_occurred = 0, rsmu_underflow_occurred = 0;
+
+	REG_GET(OPTC_INPUT_GLOBAL_CONTROL,
+			OPTC_UNDERFLOW_OCCURRED_STATUS,
+			&underflow_occurred);
+
+	REG_GET(OPTC_RSMU_UNDERFLOW,
+			OPTC_RSMU_UNDERFLOW_OCCURRED_STATUS,
+			&rsmu_underflow_occurred);
+	return (underflow_occurred == 1 || rsmu_underflow_occurred);
+}
+/* disable_crtc */
+bool optc42_disable_crtc(struct timing_generator *optc)
+{
+	optc401_disable_crtc(optc);
+	optc42_clear_optc_underflow(optc);
+
+	return true;
+}
+static void optc42_set_timing_double_buffer(struct timing_generator *optc, bool enable)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t mode = enable ? 2 : 0;
+	/* actually we have 4 modes now, use as the same as previous dcn3x
+	 * 00	OTG_DOUBLE_BUFFER_CONTROL_OTG_DRR_TIMING_DBUF_UPDATE_MODE_0	Double buffer update occurs at any time in a frame.
+	 * 01	OTG_DOUBLE_BUFFER_CONTROL_OTG_DRR_TIMING_DBUF_UPDATE_MODE_1	Double buffer update occurs at OTG start of frame.
+	 * 02	OTG_DOUBLE_BUFFER_CONTROL_OTG_DRR_TIMING_DBUF_UPDATE_MODE_2	Double buffer occurs DP start of frame.
+	 * 03	OTG_DOUBLE_BUFFER_CONTROL_OTG_DRR_TIMING_DBUF_UPDATE_MODE_3	Reserved.
+	 */
+
+	REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL,
+		   OTG_DRR_TIMING_DBUF_UPDATE_MODE, mode);
+}
+void optc42_tg_init(struct timing_generator *optc)
+{
+	optc42_set_timing_double_buffer(optc, true);
+	optc42_clear_optc_underflow(optc);
+}
+
+void optc42_lock_doublebuffer_enable(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t v_blank_start = 0;
+	uint32_t v_blank_end = 0;
+	uint32_t h_blank_start = 0;
+	uint32_t h_blank_end = 0;
+
+	REG_GET_2(OTG_V_BLANK_START_END,
+		OTG_V_BLANK_START, &v_blank_start,
+		OTG_V_BLANK_END, &v_blank_end);
+	REG_GET_2(OTG_H_BLANK_START_END,
+		OTG_H_BLANK_START, &h_blank_start,
+		OTG_H_BLANK_END, &h_blank_end);
+
+	REG_UPDATE_2(OTG_GLOBAL_CONTROL1,
+		MASTER_UPDATE_LOCK_DB_START_Y, v_blank_start,
+		MASTER_UPDATE_LOCK_DB_END_Y, v_blank_start);
+	REG_UPDATE_2(OTG_GLOBAL_CONTROL4,
+		DIG_UPDATE_POSITION_X, 20,
+		DIG_UPDATE_POSITION_Y, v_blank_start);
+	REG_UPDATE_3(OTG_GLOBAL_CONTROL0,
+		MASTER_UPDATE_LOCK_DB_START_X, h_blank_start - 200 - 1,
+		MASTER_UPDATE_LOCK_DB_END_X, h_blank_end,
+		MASTER_UPDATE_LOCK_DB_EN, 1);
+	REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 1);
+
+	REG_SET_3(OTG_VUPDATE_KEEPOUT, 0,
+		MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET, 0,
+		MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET, 100,
+		OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, 1);
+
+	TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true);
+}
+
+static struct timing_generator_funcs dcn42_tg_funcs = {
+		.validate_timing = optc1_validate_timing,
+		.program_timing = optc1_program_timing,
+		.setup_vertical_interrupt0 = optc1_setup_vertical_interrupt0,
+		.setup_vertical_interrupt1 = optc1_setup_vertical_interrupt1,
+		.setup_vertical_interrupt2 = optc1_setup_vertical_interrupt2,
+		.program_global_sync = optc401_program_global_sync,
+		.enable_crtc = optc401_enable_crtc,
+		.disable_crtc = optc42_disable_crtc,
+		.phantom_crtc_post_enable = optc401_phantom_crtc_post_enable,
+		.disable_phantom_crtc = optc401_disable_phantom_otg,
+		/* used by enable_timing_synchronization. Not need for FPGA */
+		.is_counter_moving = optc1_is_counter_moving,
+		.get_position = optc1_get_position,
+		.get_frame_count = optc1_get_vblank_counter,
+		.get_scanoutpos = optc1_get_crtc_scanoutpos,
+		.get_otg_active_size = optc1_get_otg_active_size,
+		.set_early_control = optc1_set_early_control,
+		/* used by enable_timing_synchronization. Not need for FPGA */
+		.wait_for_state = optc1_wait_for_state,
+		.set_blank_color = optc3_program_blank_color,
+		.did_triggered_reset_occur = optc1_did_triggered_reset_occur,
+		.triplebuffer_lock = optc3_triplebuffer_lock,
+		.triplebuffer_unlock = optc2_triplebuffer_unlock,
+		.enable_reset_trigger = optc1_enable_reset_trigger,
+		.enable_crtc_reset = optc1_enable_crtc_reset,
+		.disable_reset_trigger = optc1_disable_reset_trigger,
+		.lock = optc3_lock,
+		.unlock = optc1_unlock,
+		.lock_doublebuffer_enable = optc42_lock_doublebuffer_enable,
+		.lock_doublebuffer_disable = optc3_lock_doublebuffer_disable,
+		.enable_optc_clock = optc1_enable_optc_clock,
+		.set_drr = optc401_set_drr,
+		.get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal,
+		.set_vtotal_min_max = optc401_set_vtotal_min_max,
+		.set_static_screen_control = optc1_set_static_screen_control,
+		.program_stereo = optc1_program_stereo,
+		.is_stereo_left_eye = optc1_is_stereo_left_eye,
+		.tg_init = optc42_tg_init,
+		.is_tg_enabled = optc1_is_tg_enabled,
+		.is_optc_underflow_occurred = optc42_is_optc_underflow_occurred,
+		.clear_optc_underflow = optc42_clear_optc_underflow,
+		.setup_global_swap_lock = NULL,
+		.get_crc = optc42_get_crc,
+		.configure_crc = optc35_configure_crc,
+		.set_dsc_config = optc3_set_dsc_config,
+		.get_dsc_status = optc2_get_dsc_status,
+		.set_dwb_source = NULL,
+		.set_odm_bypass = optc401_set_odm_bypass,
+		.set_odm_combine = optc401_set_odm_combine,
+		.wait_odm_doublebuffer_pending_clear = optc32_wait_odm_doublebuffer_pending_clear,
+		.set_h_timing_div_manual_mode = optc401_set_h_timing_div_manual_mode,
+		.get_optc_source = optc2_get_optc_source,
+		.wait_otg_disable = optc35_wait_otg_disable,
+		.set_out_mux = optc401_set_out_mux,
+		.set_drr_trigger_window = optc3_set_drr_trigger_window,
+		.set_vtotal_change_limit = optc3_set_vtotal_change_limit,
+		.set_gsl = optc2_set_gsl,
+		.set_gsl_source_select = optc2_set_gsl_source_select,
+		.set_vtg_params = optc1_set_vtg_params,
+		.program_manual_trigger = optc2_program_manual_trigger,
+		.setup_manual_trigger = optc2_setup_manual_trigger,
+		.get_hw_timing = optc1_get_hw_timing,
+		.init_odm = optc3_init_odm,
+		.set_long_vtotal = optc35_set_long_vtotal,
+		.is_two_pixels_per_container = optc1_is_two_pixels_per_container,
+		.get_optc_double_buffer_pending = optc3_get_optc_double_buffer_pending,
+		.get_otg_double_buffer_pending = optc3_get_otg_update_pending,
+		.get_pipe_update_pending = optc3_get_pipe_update_pending,
+		.set_vupdate_keepout = optc401_set_vupdate_keepout,
+		.wait_update_lock_status = optc401_wait_update_lock_status,
+		.optc_read_reg_state = optc31_read_reg_state,
+		.read_otg_state = optc31_read_otg_state,
+		.enable_otg_pwa = optc42_enable_pwa,
+		.disable_otg_pwa = optc42_disable_pwa,
+};
+
+void dcn42_timing_generator_init(struct optc *optc1)
+{
+	optc1->base.funcs = &dcn42_tg_funcs;
+
+	optc1->max_h_total = optc1->tg_mask->OTG_H_TOTAL + 1;
+	optc1->max_v_total = optc1->tg_mask->OTG_V_TOTAL + 1;
+
+	optc1->min_h_blank = 32;
+	optc1->min_v_blank = 3;
+	optc1->min_v_blank_interlace = 5;
+	optc1->min_h_sync_width = 4;
+	optc1->min_v_sync_width = 1;
+	optc1->max_frame_count = 0xFFFFFF;
+
+	dcn35_timing_generator_set_fgcg(
+		optc1, CTX->dc->debug.enable_fine_grain_clock_gating.bits.optc);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
new file mode 100644
index 0000000000000..fc7192f01b336
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_OPTC_DCN42_H__
+#define __DC_OPTC_DCN42_H__
+
+#include "dcn10/dcn10_optc.h"
+
+#define OPTC_COMMON_MASK_SH_LIST_DCN42(mask_sh)\
+	SF(OTG0_OTG_VSTARTUP_PARAM, VSTARTUP_START, mask_sh),\
+	SF(OTG0_OTG_VUPDATE_PARAM, VUPDATE_OFFSET, mask_sh),\
+	SF(OTG0_OTG_VUPDATE_PARAM, VUPDATE_WIDTH, mask_sh),\
+	SF(OTG0_OTG_VREADY_PARAM, VREADY_OFFSET, mask_sh),\
+	SF(OTG0_OTG_MASTER_UPDATE_LOCK, OTG_MASTER_UPDATE_LOCK, mask_sh),\
+	SF(OTG0_OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_START_X, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_END_X, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_START_Y, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_END_Y, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL2, OTG_MASTER_UPDATE_LOCK_SEL, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL4, DIG_UPDATE_POSITION_X, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL4, DIG_UPDATE_POSITION_Y, mask_sh),\
+	SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_UPDATE_PENDING, mask_sh),\
+	SF(OTG0_OTG_H_TOTAL, OTG_H_TOTAL, mask_sh),\
+	SF(OTG0_OTG_H_BLANK_START_END, OTG_H_BLANK_START, mask_sh),\
+	SF(OTG0_OTG_H_BLANK_START_END, OTG_H_BLANK_END, mask_sh),\
+	SF(OTG0_OTG_H_SYNC_A, OTG_H_SYNC_A_START, mask_sh),\
+	SF(OTG0_OTG_H_SYNC_A, OTG_H_SYNC_A_END, mask_sh),\
+	SF(OTG0_OTG_H_SYNC_A_CNTL, OTG_H_SYNC_A_POL, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL, OTG_V_TOTAL, mask_sh),\
+	SF(OTG0_OTG_V_BLANK_START_END, OTG_V_BLANK_START, mask_sh),\
+	SF(OTG0_OTG_V_BLANK_START_END, OTG_V_BLANK_END, mask_sh),\
+	SF(OTG0_OTG_V_SYNC_A, OTG_V_SYNC_A_START, mask_sh),\
+	SF(OTG0_OTG_V_SYNC_A, OTG_V_SYNC_A_END, mask_sh),\
+	SF(OTG0_OTG_V_SYNC_A_CNTL, OTG_V_SYNC_A_POL, mask_sh),\
+	SF(OTG0_OTG_V_SYNC_A_CNTL, OTG_V_SYNC_MODE, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_CURRENT_MASTER_EN_STATE, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_MASTER_EN, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_START_POINT_CNTL, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_DISABLE_POINT_CNTL, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_FIELD_NUMBER_CNTL, mask_sh),\
+	SF(OTG0_OTG_CONTROL, OTG_OUT_MUX, mask_sh),\
+	SF(OTG0_OTG_STEREO_CONTROL, OTG_STEREO_EN, mask_sh),\
+	SF(OTG0_OTG_STEREO_CONTROL, OTG_STEREO_SYNC_OUTPUT_LINE_NUM, mask_sh),\
+	SF(OTG0_OTG_STEREO_CONTROL, OTG_STEREO_SYNC_OUTPUT_POLARITY, mask_sh),\
+	SF(OTG0_OTG_STEREO_CONTROL, OTG_STEREO_EYE_FLAG_POLARITY, mask_sh),\
+	SF(OTG0_OTG_STEREO_CONTROL, OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP, mask_sh),\
+	SF(OTG0_OTG_STEREO_STATUS, OTG_STEREO_CURRENT_EYE, mask_sh),\
+	SF(OTG0_OTG_3D_STRUCTURE_CONTROL, OTG_3D_STRUCTURE_EN, mask_sh),\
+	SF(OTG0_OTG_3D_STRUCTURE_CONTROL, OTG_3D_STRUCTURE_V_UPDATE_MODE, mask_sh),\
+	SF(OTG0_OTG_3D_STRUCTURE_CONTROL, OTG_3D_STRUCTURE_STEREO_SEL_OVR, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_MAX, OTG_V_TOTAL_MAX, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_MIN, OTG_V_TOTAL_MIN, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_V_TOTAL_MIN_SEL, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_V_TOTAL_MAX_SEL, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_FORCE_LOCK_ON_EVENT, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_SET_V_TOTAL_MIN_MASK, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_VTOTAL_MID_REPLACING_MIN_EN, mask_sh),\
+	SF(OTG0_OTG_V_TOTAL_CONTROL, OTG_VTOTAL_MID_REPLACING_MAX_EN, mask_sh),\
+	SF(OTG0_OTG_FORCE_COUNT_NOW_CNTL, OTG_FORCE_COUNT_NOW_CLEAR, mask_sh),\
+	SF(OTG0_OTG_FORCE_COUNT_NOW_CNTL, OTG_FORCE_COUNT_NOW_MODE, mask_sh),\
+	SF(OTG0_OTG_FORCE_COUNT_NOW_CNTL, OTG_FORCE_COUNT_NOW_OCCURRED, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_SOURCE_SELECT, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_SOURCE_PIPE_SELECT, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_RISING_EDGE_DETECT_CNTL, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_FALLING_EDGE_DETECT_CNTL, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_POLARITY_SELECT, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_FREQUENCY_SELECT, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_DELAY, mask_sh),\
+	SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_CLEAR, mask_sh),\
+	SF(OTG0_OTG_STATIC_SCREEN_CONTROL, OTG_STATIC_SCREEN_EVENT_MASK, mask_sh),\
+	SF(OTG0_OTG_STATIC_SCREEN_CONTROL, OTG_STATIC_SCREEN_FRAME_COUNT, mask_sh),\
+	SF(OTG0_OTG_STATUS_FRAME_COUNT, OTG_FRAME_COUNT, mask_sh),\
+	SF(OTG0_OTG_STATUS, OTG_V_BLANK, mask_sh),\
+	SF(OTG0_OTG_STATUS, OTG_V_ACTIVE_DISP, mask_sh),\
+	SF(OTG0_OTG_STATUS_POSITION, OTG_HORZ_COUNT, mask_sh),\
+	SF(OTG0_OTG_STATUS_POSITION, OTG_VERT_COUNT, mask_sh),\
+	SF(OTG0_OTG_NOM_VERT_POSITION, OTG_VERT_COUNT_NOM, mask_sh),\
+	SF(OTG0_OTG_M_CONST_DTO0, OTG_M_CONST_DTO_PHASE, mask_sh),\
+	SF(OTG0_OTG_M_CONST_DTO1, OTG_M_CONST_DTO_MODULO, mask_sh),\
+	SF(OTG0_OTG_CLOCK_CONTROL, OTG_BUSY, mask_sh),\
+	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_EN, mask_sh),\
+	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_ON, mask_sh),\
+	SF(OTG0_OTG_CLOCK_CONTROL, OTG_CLOCK_GATE_DIS, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_INT_ENABLE, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_POSITION, OTG_VERTICAL_INTERRUPT0_LINE_START, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT0_POSITION, OTG_VERTICAL_INTERRUPT0_LINE_END, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL, OTG_VERTICAL_INTERRUPT1_INT_ENABLE, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT1_POSITION, OTG_VERTICAL_INTERRUPT1_LINE_START, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL, OTG_VERTICAL_INTERRUPT2_INT_ENABLE, mask_sh),\
+	SF(OTG0_OTG_VERTICAL_INTERRUPT2_POSITION, OTG_VERTICAL_INTERRUPT2_LINE_START, mask_sh),\
+	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_EN, mask_sh),\
+	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_ON, mask_sh),\
+	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_GATE_DIS, mask_sh),\
+	SF(ODM0_OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_OCCURRED_STATUS, mask_sh),\
+	SF(ODM0_OPTC_INPUT_GLOBAL_CONTROL, OPTC_DOUBLE_BUFFER_PENDING, mask_sh),\
+	SF(ODM0_OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_CLEAR, mask_sh),\
+	SF(ODM0_OPTC_RSMU_UNDERFLOW, OPTC_RSMU_UNDERFLOW_OCCURRED_STATUS, mask_sh),\
+	SF(ODM0_OPTC_RSMU_UNDERFLOW, OPTC_RSMU_UNDERFLOW_CLEAR, mask_sh),\
+	SF(ODM0_OPTC_RSMU_UNDERFLOW, OPTC_RSMU_UNDERFLOW_INT_EN, mask_sh),\
+	SF(ODM0_OPTC_RSMU_UNDERFLOW, OPTC_RSMU_UNDERFLOW_INT_STATUS, mask_sh),\
+	SF(VTG0_CONTROL, VTG0_ENABLE, mask_sh),\
+	SF(VTG0_CONTROL, VTG0_FP2, mask_sh),\
+	SF(VTG0_CONTROL, VTG0_VCOUNT_INIT, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_FORCE_VSYNC_NEXT_LINE_CLEAR, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_AUTO_FORCE_VSYNC_MODE, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL0_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL1_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL2_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_MASTER_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_FORCE_DELAY, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_CHECK_ALL_FIELDS, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC_CONT_EN, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC0_SELECT, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC_EN, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_R, CRC0_R_CR, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_G, CRC0_G_Y, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_B, CRC0_B_CB, mask_sh),\
+	SF(OTG0_OTG_CRC1_DATA_R, CRC1_R_CR, mask_sh),\
+	SF(OTG0_OTG_CRC1_DATA_G, CRC1_G_Y, mask_sh),\
+	SF(OTG0_OTG_CRC1_DATA_B, CRC1_B_CB, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_X_CONTROL, OTG_CRC0_WINDOWA_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_X_CONTROL, OTG_CRC0_WINDOWA_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_Y_CONTROL, OTG_CRC0_WINDOWA_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_Y_CONTROL, OTG_CRC0_WINDOWA_Y_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_END, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWA_X_CONTROL, OTG_CRC1_WINDOWA_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWA_X_CONTROL, OTG_CRC1_WINDOWA_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWA_Y_CONTROL, OTG_CRC1_WINDOWA_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWA_Y_CONTROL, OTG_CRC1_WINDOWA_Y_END, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWB_X_CONTROL, OTG_CRC1_WINDOWB_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWB_X_CONTROL, OTG_CRC1_WINDOWB_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWB_Y_CONTROL, OTG_CRC1_WINDOWB_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC1_WINDOWB_Y_CONTROL, OTG_CRC1_WINDOWB_Y_END, mask_sh),\
+	SF(OTG0_OTG_TRIGA_MANUAL_TRIG, OTG_TRIGA_MANUAL_TRIG, mask_sh),\
+	SF(GSL_SOURCE_SELECT, GSL0_READY_SOURCE_SEL, mask_sh),\
+	SF(GSL_SOURCE_SELECT, GSL1_READY_SOURCE_SEL, mask_sh),\
+	SF(GSL_SOURCE_SELECT, GSL2_READY_SOURCE_SEL, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL2, MANUAL_FLOW_CONTROL_SEL, mask_sh),\
+	SF(OTG0_OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_START_X, mask_sh),\
+	SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_END_X, mask_sh), \
+	SF(OTG0_OTG_GSL_WINDOW_Y, OTG_GSL_WINDOW_START_Y, mask_sh),\
+	SF(OTG0_OTG_GSL_WINDOW_Y, OTG_GSL_WINDOW_END_Y, mask_sh),\
+	SF(OTG0_OTG_VUPDATE_KEEPOUT, OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, mask_sh), \
+	SF(OTG0_OTG_VUPDATE_KEEPOUT, MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET, mask_sh), \
+	SF(OTG0_OTG_VUPDATE_KEEPOUT, MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET, mask_sh), \
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_MASTER_MODE, mask_sh), \
+	SF(OTG0_OTG_GSL_CONTROL, OTG_MASTER_UPDATE_LOCK_GSL_EN, mask_sh), \
+	SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG0_SRC_SEL, mask_sh),\
+	SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG1_SRC_SEL, mask_sh),\
+	SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG2_SRC_SEL, mask_sh),\
+	SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG3_SRC_SEL, mask_sh),\
+	SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_NUM_OF_INPUT_SEGMENT, mask_sh),\
+	SF(ODM0_OPTC_MEMORY_CONFIG, OPTC_MEM_SEL, mask_sh),\
+	SF(ODM0_OPTC_DATA_FORMAT_CONTROL, OPTC_DATA_FORMAT, mask_sh),\
+	SF(ODM0_OPTC_DATA_FORMAT_CONTROL, OPTC_DSC_MODE, mask_sh),\
+	SF(ODM0_OPTC_BYTES_PER_PIXEL, OPTC_DSC_BYTES_PER_PIXEL, mask_sh),\
+	SF(ODM0_OPTC_WIDTH_CONTROL, OPTC_DSC_SLICE_WIDTH, mask_sh),\
+	SF(ODM0_OPTC_WIDTH_CONTROL, OPTC_SEGMENT_WIDTH, mask_sh),\
+	SF(ODM0_OPTC_WIDTH_CONTROL2, OPTC_SEGMENT_WIDTH_LAST, mask_sh),\
+	SF(OTG0_OTG_DRR_TRIGGER_WINDOW, OTG_DRR_TRIGGER_WINDOW_START_X, mask_sh),\
+	SF(OTG0_OTG_DRR_TRIGGER_WINDOW, OTG_DRR_TRIGGER_WINDOW_END_X, mask_sh),\
+	SF(OTG0_OTG_DRR_V_TOTAL_CHANGE, OTG_DRR_V_TOTAL_CHANGE_LIMIT, mask_sh),\
+	SF(OTG0_OTG_H_TIMING_CNTL, OTG_H_TIMING_DIV_MODE, mask_sh),\
+	SF(OTG0_OTG_H_TIMING_CNTL, OTG_H_TIMING_DIV_MODE_MANUAL, mask_sh),\
+	SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_DRR_TIMING_DBUF_UPDATE_MODE, mask_sh),\
+	SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_DRR_TIMING_DBUF_UPDATE_PENDING, mask_sh),\
+	SF(OTG0_OTG_DRR_CONTROL, OTG_V_TOTAL_LAST_USED_BY_DRR, mask_sh),\
+	SF(OTG0_OTG_PSTATE_REGISTER, OTG_PSTATE_KEEPOUT_START, mask_sh),\
+	SF(OTG0_OTG_PSTATE_REGISTER, OTG_PSTATE_EXTEND, mask_sh),\
+	SF(OTG0_OTG_PSTATE_REGISTER, OTG_UNBLANK, mask_sh),\
+	SF(OTG0_OTG_PSTATE_REGISTER, OTG_PSTATE_ALLOW_WIDTH_MIN, mask_sh),\
+	SF(OTG0_OTG_PIPE_UPDATE_STATUS, OTG_FLIP_PENDING, mask_sh),\
+	SF(OTG0_OTG_PIPE_UPDATE_STATUS, OTG_DC_REG_UPDATE_PENDING, mask_sh),\
+	SF(OTG0_OTG_PIPE_UPDATE_STATUS, OTG_CURSOR_UPDATE_PENDING, mask_sh),\
+	SF(OTG0_OTG_PIPE_UPDATE_STATUS, OTG_VUPDATE_KEEPOUT_STATUS, mask_sh),\
+	SF(OTG0_OTG_PWA_FRAME_SYNC_CONTROL, OTG_PWA_FRAME_SYNC_EN, mask_sh),\
+	SF(OTG0_OTG_PWA_FRAME_SYNC_CONTROL, OTG_PWA_FRAME_SYNC_VCOUNT_MODE, mask_sh),\
+	SF(OTG0_OTG_PWA_FRAME_SYNC_CONTROL, OTG_PWA_FRAME_SYNC_LINE, mask_sh),\
+	SF(OTG0_INTERRUPT_DEST, OTG0_IHC_OTG_VERTICAL_INTERRUPT2_DEST, mask_sh),\
+	SF(OPTC_CLOCK_CONTROL, OPTC_FGCG_REP_DIS, mask_sh)
+
+void dcn42_timing_generator_init(struct optc *optc1);
+void optc42_enable_pwa(struct timing_generator *optc, struct otc_pwa_frame_sync *pwa_sync_param);
+void optc42_disable_pwa(struct timing_generator *optc);
+void optc42_tg_init(struct timing_generator *optc);
+void optc42_clear_optc_underflow(struct timing_generator *optc);
+bool optc42_is_optc_underflow_occurred(struct timing_generator *optc);
+bool optc42_disable_crtc(struct timing_generator *optc);
+void optc42_lock_doublebuffer_enable(struct timing_generator *optc);
+#endif /* __DC_OPTC_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/pg/Makefile b/drivers/gpu/drm/amd/display/dc/pg/Makefile
index ec11d3157a57f..56f610cc84065 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/pg/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Advanced Micro Devices, Inc.
+# Copyright 2020, 2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -32,4 +32,12 @@ PG_DCN35 = dcn35_pg_cntl.o
 AMD_DAL_PG_DCN35 = $(addprefix $(AMDDALPATH)/dc/pg/dcn35/,$(PG_DCN35))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_PG_DCN35)
+###############################################################################
+# DCN42
+###############################################################################
+PG_DCN42 = dcn42_pg_cntl.o
+
+AMD_DAL_PG_DCN42 = $(addprefix $(AMDDALPATH)/dc/pg/dcn42/,$(PG_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_PG_DCN42)
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
new file mode 100644
index 0000000000000..3685080ce9dc4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
@@ -0,0 +1,648 @@
+// SPDX-License-Identifier: MIT
+
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "reg_helper.h"
+#include "core_types.h"
+#include "dcn42_pg_cntl.h"
+#include "dccg.h"
+
+#define TO_DCN_PG_CNTL(pg_cntl)\
+	container_of(pg_cntl, struct dcn_pg_cntl, base)
+
+#define REG(reg) \
+	(pg_cntl_dcn->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+	pg_cntl_dcn->pg_cntl_shift->field_name, pg_cntl_dcn->pg_cntl_mask->field_name
+
+#define CTX \
+	pg_cntl_dcn->base.ctx
+#define DC_LOGGER \
+	pg_cntl->ctx->logger
+
+static bool pg_cntl42_dsc_pg_status(struct pg_cntl *pg_cntl, unsigned int dsc_inst)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg)
+		return true;
+
+	switch (dsc_inst) {
+	case 0: /* DSC0 */
+		REG_GET(DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 1: /* DSC1 */
+		REG_GET(DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 2: /* DSC2 */
+		REG_GET(DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 3: /* DSC3 */
+		REG_GET(DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl = 0;
+	bool block_enabled;
+
+	/*need to enable dscclk regardless DSC_PG*/
+	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc && power_on)
+		pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc(
+				pg_cntl->ctx->dc->res_pool->dccg, dsc_inst);
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->debug.disable_dsc_power_gate ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, dsc_inst);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
+	}
+	switch (dsc_inst) {
+	case 0: /* DSC0 */
+		REG_UPDATE(DOMAIN16_PG_CONFIG,
+				DOMAIN_POWER_GATE, power_gate);
+
+		REG_WAIT(DOMAIN16_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+				1, 1000);
+		break;
+	case 1: /* DSC1 */
+		REG_UPDATE(DOMAIN17_PG_CONFIG,
+				DOMAIN_POWER_GATE, power_gate);
+
+		REG_WAIT(DOMAIN17_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+				1, 1000);
+		break;
+	case 2: /* DSC2 */
+		REG_UPDATE(DOMAIN18_PG_CONFIG,
+				DOMAIN_POWER_GATE, power_gate);
+
+		REG_WAIT(DOMAIN18_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+				1, 1000);
+		break;
+	case 3: /* DSC3 */
+		REG_UPDATE(DOMAIN19_PG_CONFIG,
+				DOMAIN_POWER_GATE, power_gate);
+
+		REG_WAIT(DOMAIN19_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+				1, 1000);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
+	}
+
+	if (dsc_inst < MAX_PIPES)
+		pg_cntl->pg_pipe_res_enable[PG_DSC][dsc_inst] = power_on;
+
+	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->disable_dsc && !power_on) {
+		/*this is to disable dscclk*/
+		pg_cntl->ctx->dc->res_pool->dccg->funcs->disable_dsc(
+			pg_cntl->ctx->dc->res_pool->dccg, dsc_inst);
+	}
+}
+
+static bool pg_cntl42_hubp_dpp_pg_status(struct pg_cntl *pg_cntl, unsigned int hubp_dpp_inst)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	switch (hubp_dpp_inst) {
+	case 0:
+		/* DPP0 & HUBP0 */
+		REG_GET(DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 1:
+		/* DPP1 & HUBP1 */
+		REG_GET(DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 2:
+		/* DPP2 & HUBP2 */
+		REG_GET(DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 3:
+		/* DPP3 & HUBP3 */
+		REG_GET(DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_hubp_dpp_pg_control(struct pg_cntl *pg_cntl, unsigned int hubp_dpp_inst, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	bool block_enabled;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->debug.disable_hubp_power_gate ||
+		pg_cntl->ctx->dc->debug.disable_dpp_power_gate ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	block_enabled = pg_cntl42_hubp_dpp_pg_status(pg_cntl, hubp_dpp_inst);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
+	}
+
+	switch (hubp_dpp_inst) {
+	case 0:
+		/* DPP0 & HUBP0 */
+		REG_UPDATE(DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+		REG_WAIT(DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+		break;
+	case 1:
+		/* DPP1 & HUBP1 */
+		REG_UPDATE(DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+		REG_WAIT(DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+		break;
+	case 2:
+		/* DPP2 & HUBP2 */
+		REG_UPDATE(DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+		REG_WAIT(DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+		break;
+	case 3:
+		/* DPP3 & HUBP3 */
+		REG_UPDATE(DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+		REG_WAIT(DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
+	}
+	DC_LOG_DEBUG("HUBP DPP instance %d, power %s", hubp_dpp_inst,
+		power_on ? "ON" : "OFF");
+
+	if (hubp_dpp_inst < MAX_PIPES) {
+		pg_cntl->pg_pipe_res_enable[PG_HUBP][hubp_dpp_inst] = power_on;
+		pg_cntl->pg_pipe_res_enable[PG_DPP][hubp_dpp_inst] = power_on;
+	}
+}
+
+static bool pg_cntl42_hpo_pg_status(struct pg_cntl *pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	REG_GET(DOMAIN25_PG_STATUS,
+			DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	uint32_t power_forceon;
+	bool block_enabled;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->debug.disable_hpo_power_gate ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	block_enabled = pg_cntl42_hpo_pg_status(pg_cntl);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
+	if (power_forceon)
+		return;
+
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
+	}
+	REG_UPDATE(DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+	REG_WAIT(DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
+	}
+	pg_cntl->pg_res_enable[PG_HPO] = power_on;
+}
+
+static bool pg_cntl42_io_clk_status(struct pg_cntl *pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	REG_GET(DOMAIN22_PG_STATUS,
+		DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	uint32_t power_forceon;
+	bool block_enabled;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed ||
+		pg_cntl->ctx->dc->debug.disable_io_clk_power_gate)
+		return;
+
+	block_enabled = pg_cntl42_io_clk_status(pg_cntl);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
+	if (power_forceon)
+		return;
+
+	if (!power_on) {
+		if (!pg_cntl->pg_res_enable[PG_DCCG] || !pg_cntl->pg_res_enable[PG_DCIO] || !pg_cntl->pg_res_enable[PG_DCOH])
+			return;
+	}
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+	/* DCCG, DCOH, DCIO */
+	REG_UPDATE(DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+	REG_WAIT(DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+
+	pg_cntl->pg_res_enable[PG_DCCG] = power_on;
+	pg_cntl->pg_res_enable[PG_DCOH] = power_on;
+	pg_cntl->pg_res_enable[PG_DCIO] = power_on;
+}
+
+static bool pg_cntl42_plane_otg_status(struct pg_cntl *pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	REG_GET(DOMAIN24_PG_STATUS,
+		DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_mpcc_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int mpcc_inst, bool power_on)
+{
+	if (pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	if (mpcc_inst < MAX_PIPES)
+		pg_cntl->pg_pipe_res_enable[PG_MPCC][mpcc_inst] = power_on;
+}
+
+void pg_cntl42_opp_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int opp_inst, bool power_on)
+{
+	if (pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	if (opp_inst < MAX_PIPES)
+		pg_cntl->pg_pipe_res_enable[PG_OPP][opp_inst] = power_on;
+}
+
+void pg_cntl42_optc_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int optc_inst, bool power_on)
+{
+	// Defer the ONO domain power up/down to plane_otg_pg_control
+	if (pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	if (optc_inst < MAX_PIPES)
+		pg_cntl->pg_pipe_res_enable[PG_OPTC][optc_inst] = power_on;
+}
+static bool pg_cntl42_mem_status(struct pg_cntl *pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	REG_GET(DOMAIN23_PG_STATUS,
+		DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_mem_pg_control(struct pg_cntl *pg_cntl, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	uint32_t power_forceon;
+	bool block_enabled;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed ||
+		pg_cntl->ctx->dc->debug.disable_mem_power_gate)
+		return;
+
+	block_enabled = pg_cntl42_mem_status(pg_cntl);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
+	if (power_forceon)
+		return;
+
+	if (!power_on) {
+		if (!pg_cntl->pg_res_enable[PG_DCHUBBUB] || !pg_cntl->pg_res_enable[PG_DCHVM])
+			return;
+	}
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+	/* DCHUBBUB, DCHUBBUBMEMn, DCHVM */
+	REG_UPDATE(DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+	REG_WAIT(DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+
+	pg_cntl->pg_res_enable[PG_DCHUBBUB] = power_on;
+	pg_cntl->pg_res_enable[PG_DCHVM] = power_on;
+}
+static bool pg_cntl42_dio_pg_status(struct pg_cntl *pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t pwr_status = 0;
+
+	REG_GET(DOMAIN26_PG_STATUS,
+		DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+
+	return pwr_status == 0;
+}
+
+void pg_cntl42_dio_pg_control(struct pg_cntl *pg_cntl, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	bool block_enabled;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed ||
+		pg_cntl->ctx->dc->debug.disable_dio_power_gate)
+		return;
+
+	block_enabled = pg_cntl42_dio_pg_status(pg_cntl);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
+	}
+	/* DIO */
+	REG_UPDATE(DOMAIN26_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+	REG_WAIT(DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+
+	if (power_on) {
+		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
+	}
+	pg_cntl->pg_res_enable[PG_DIO] = power_on;
+
+}
+
+void pg_cntl42_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+	uint32_t power_gate = power_on ? 0 : 1;
+	uint32_t pwr_status = power_on ? 0 : 2;
+	uint32_t org_ip_request_cntl;
+	int i;
+	bool block_enabled;
+	bool all_mpcc_disabled = true, all_opp_disabled = true;
+	bool all_optc_disabled = true, all_stream_disabled = true;
+
+	if (pg_cntl->ctx->dc->debug.ignore_pg ||
+		pg_cntl->ctx->dc->debug.disable_optc_power_gate ||
+		pg_cntl->ctx->dc->idle_optimizations_allowed)
+		return;
+
+	block_enabled = pg_cntl42_plane_otg_status(pg_cntl);
+	if (power_on) {
+		if (block_enabled)
+			return;
+	} else {
+		if (!block_enabled)
+			return;
+	}
+
+	for (i = 0; i < pg_cntl->ctx->dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &pg_cntl->ctx->dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx) {
+			if (pipe_ctx->stream)
+				all_stream_disabled = false;
+		}
+
+		if (pg_cntl->pg_pipe_res_enable[PG_MPCC][i])
+			all_mpcc_disabled = false;
+
+		if (pg_cntl->pg_pipe_res_enable[PG_OPP][i])
+			all_opp_disabled = false;
+
+		if (pg_cntl->pg_pipe_res_enable[PG_OPTC][i])
+			all_optc_disabled = false;
+	}
+
+	if (!power_on) {
+		if (!all_mpcc_disabled || !all_opp_disabled || !all_optc_disabled
+			|| !all_stream_disabled)
+			return;
+	}
+
+	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+	if (org_ip_request_cntl == 0)
+		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+	/* MPC, OPP, OPTC, DWB */
+	REG_UPDATE(DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
+	REG_WAIT(DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
+
+	for (i = 0; i < pg_cntl->ctx->dc->res_pool->pipe_count; i++) {
+		pg_cntl->pg_pipe_res_enable[PG_MPCC][i] = power_on;
+		pg_cntl->pg_pipe_res_enable[PG_OPP][i] = power_on;
+		pg_cntl->pg_pipe_res_enable[PG_OPTC][i] = power_on;
+	}
+}
+
+
+void pg_cntl42_init_pg_status(struct pg_cntl *pg_cntl)
+{
+	int i = 0;
+	bool block_enabled;
+
+	pg_cntl->pg_res_enable[PG_HPO] = pg_cntl42_hpo_pg_status(pg_cntl);
+
+	block_enabled = pg_cntl42_io_clk_status(pg_cntl);
+	pg_cntl->pg_res_enable[PG_DCCG] = block_enabled;
+	pg_cntl->pg_res_enable[PG_DCIO] = block_enabled;
+	pg_cntl->pg_res_enable[PG_DCOH] = block_enabled;
+
+	block_enabled = pg_cntl42_dio_pg_status(pg_cntl);
+	pg_cntl->pg_res_enable[PG_DIO] = block_enabled;
+
+	block_enabled = pg_cntl42_mem_status(pg_cntl);
+	pg_cntl->pg_res_enable[PG_DCHUBBUB] = block_enabled;
+	pg_cntl->pg_res_enable[PG_DCHVM] = block_enabled;
+
+	for (i = 0; i < pg_cntl->ctx->dc->res_pool->pipe_count; i++) {
+		block_enabled = pg_cntl42_hubp_dpp_pg_status(pg_cntl, i);
+		pg_cntl->pg_pipe_res_enable[PG_HUBP][i] = block_enabled;
+		pg_cntl->pg_pipe_res_enable[PG_DPP][i] = block_enabled;
+
+		block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, i);
+		pg_cntl->pg_pipe_res_enable[PG_DSC][i] = block_enabled;
+	}
+
+	block_enabled = pg_cntl42_plane_otg_status(pg_cntl);
+	for (i = 0; i < pg_cntl->ctx->dc->res_pool->pipe_count; i++) {
+		pg_cntl->pg_pipe_res_enable[PG_MPCC][i] = block_enabled;
+		pg_cntl->pg_pipe_res_enable[PG_OPP][i] = block_enabled;
+		pg_cntl->pg_pipe_res_enable[PG_OPTC][i] = block_enabled;
+	}
+}
+
+static const struct pg_cntl_funcs pg_cntl42_funcs = {
+	.init_pg_status = pg_cntl42_init_pg_status,
+	.dsc_pg_control = pg_cntl42_dsc_pg_control,
+	.hubp_dpp_pg_control = pg_cntl42_hubp_dpp_pg_control,
+	.hpo_pg_control = pg_cntl42_hpo_pg_control,
+	.io_clk_pg_control = pg_cntl42_io_clk_pg_control,
+	.plane_otg_pg_control = pg_cntl42_plane_otg_pg_control,
+	.mpcc_pg_control = pg_cntl42_mpcc_pg_control,
+	.opp_pg_control = pg_cntl42_opp_pg_control,
+	.optc_pg_control = pg_cntl42_optc_pg_control,
+	.mem_pg_control = pg_cntl42_mem_pg_control,
+	.dio_pg_control = pg_cntl42_dio_pg_control
+};
+
+struct pg_cntl *pg_cntl42_create(
+	struct dc_context *ctx,
+	const struct pg_cntl_registers *regs,
+	const struct pg_cntl_shift *pg_cntl_shift,
+	const struct pg_cntl_mask *pg_cntl_mask)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = kzalloc(sizeof(*pg_cntl_dcn), GFP_KERNEL);
+	struct pg_cntl *base;
+
+	if (pg_cntl_dcn == NULL) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	base = &pg_cntl_dcn->base;
+	base->ctx = ctx;
+	base->funcs = &pg_cntl42_funcs;
+
+	pg_cntl_dcn->regs = regs;
+	pg_cntl_dcn->pg_cntl_shift = pg_cntl_shift;
+	pg_cntl_dcn->pg_cntl_mask = pg_cntl_mask;
+
+	memset(base->pg_pipe_res_enable, 0, PG_HW_PIPE_RESOURCES_NUM_ELEMENT * MAX_PIPES * sizeof(bool));
+	memset(base->pg_res_enable, 0, PG_HW_RESOURCES_NUM_ELEMENT * sizeof(bool));
+
+	return &pg_cntl_dcn->base;
+}
+
+void dcn42_pg_cntl_destroy(struct pg_cntl **pg_cntl)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(*pg_cntl);
+
+	kfree(pg_cntl_dcn);
+	*pg_cntl = NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
new file mode 100644
index 0000000000000..a0aa14a796f4c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
@@ -0,0 +1,175 @@
+/* SPDX-License-Identifier: MIT */
+
+/* Copyright 2026 Advanced Micro Devices, Inc.*/
+
+#ifndef __DCN42_PG_CNTL_H__
+#define __DCN42_PG_CNTL_H__
+
+#include "pg_cntl.h"
+
+#define PG_CNTL_REG_LIST_DCN42()\
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN19_PG_CONFIG), \
+	SR(DOMAIN22_PG_CONFIG), \
+	SR(DOMAIN23_PG_CONFIG), \
+	SR(DOMAIN24_PG_CONFIG), \
+	SR(DOMAIN25_PG_CONFIG), \
+	SR(DOMAIN26_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(DOMAIN19_PG_STATUS), \
+	SR(DOMAIN22_PG_STATUS), \
+	SR(DOMAIN23_PG_STATUS), \
+	SR(DOMAIN24_PG_STATUS), \
+	SR(DOMAIN25_PG_STATUS), \
+	SR(DOMAIN26_PG_STATUS), \
+	SR(DC_IP_REQUEST_CNTL)
+
+#define PG_CNTL_SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define PG_CNTL_MASK_SH_LIST_DCN42(mask_sh) \
+	PG_CNTL_SF(DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN24_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN26_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	PG_CNTL_SF(DOMAIN26_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN0_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN1_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN2_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN3_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN16_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN17_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN18_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN19_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN22_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN23_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN24_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN25_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	PG_CNTL_SF(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh)
+
+struct pg_cntl_shift {
+	uint8_t IP_REQUEST_EN;
+	uint8_t DOMAIN_POWER_FORCEON;
+	uint8_t DOMAIN_POWER_GATE;
+	uint8_t DOMAIN_DESIRED_PWR_STATE;
+	uint8_t DOMAIN_PGFSM_PWR_STATUS;
+};
+struct pg_cntl_mask {
+	uint32_t IP_REQUEST_EN;
+	uint32_t DOMAIN_POWER_FORCEON;
+	uint32_t DOMAIN_POWER_GATE;
+	uint32_t DOMAIN_DESIRED_PWR_STATE;
+	uint32_t DOMAIN_PGFSM_PWR_STATUS;
+};
+
+struct pg_cntl_registers {
+	uint32_t LONO_STATE;
+	uint32_t DC_IP_REQUEST_CNTL;
+	uint32_t DOMAIN0_PG_CONFIG;
+	uint32_t DOMAIN1_PG_CONFIG;
+	uint32_t DOMAIN2_PG_CONFIG;
+	uint32_t DOMAIN3_PG_CONFIG;
+	uint32_t DOMAIN16_PG_CONFIG;
+	uint32_t DOMAIN17_PG_CONFIG;
+	uint32_t DOMAIN18_PG_CONFIG;
+	uint32_t DOMAIN19_PG_CONFIG;
+	uint32_t DOMAIN22_PG_CONFIG;
+	uint32_t DOMAIN23_PG_CONFIG;
+	uint32_t DOMAIN24_PG_CONFIG;
+	uint32_t DOMAIN25_PG_CONFIG;
+	uint32_t DOMAIN26_PG_CONFIG;
+	uint32_t DOMAIN0_PG_STATUS;
+	uint32_t DOMAIN1_PG_STATUS;
+	uint32_t DOMAIN2_PG_STATUS;
+	uint32_t DOMAIN3_PG_STATUS;
+	uint32_t DOMAIN16_PG_STATUS;
+	uint32_t DOMAIN17_PG_STATUS;
+	uint32_t DOMAIN18_PG_STATUS;
+	uint32_t DOMAIN19_PG_STATUS;
+	uint32_t DOMAIN22_PG_STATUS;
+	uint32_t DOMAIN23_PG_STATUS;
+	uint32_t DOMAIN24_PG_STATUS;
+	uint32_t DOMAIN25_PG_STATUS;
+	uint32_t DOMAIN26_PG_STATUS;
+};
+
+struct dcn_pg_cntl {
+	struct pg_cntl base;
+	const struct pg_cntl_registers *regs;
+	const struct pg_cntl_shift *pg_cntl_shift;
+	const struct pg_cntl_mask *pg_cntl_mask;
+};
+
+void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bool power_on);
+void pg_cntl42_hubp_dpp_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int hubp_dpp_inst, bool power_on);
+void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on);
+void pg_cntl42_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on);
+void pg_cntl42_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on);
+void pg_cntl42_mpcc_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int mpcc_inst, bool power_on);
+void pg_cntl42_opp_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int opp_inst, bool power_on);
+void pg_cntl42_optc_pg_control(struct pg_cntl *pg_cntl,
+	unsigned int optc_inst, bool power_on);
+void pg_cntl42_mem_pg_control(struct pg_cntl *pg_cntl, bool power_on);
+void pg_cntl42_dio_pg_control(struct pg_cntl *pg_cntl, bool power_on);
+void dcn42_pg_cntl_destroy(struct pg_cntl **pg_cntl);
+void pg_cntl42_init_pg_status(struct pg_cntl *pg_cntl);
+
+struct pg_cntl *pg_cntl42_create(
+	struct dc_context *ctx,
+	const struct pg_cntl_registers *regs,
+	const struct pg_cntl_shift *pg_cntl_shift,
+	const struct pg_cntl_mask *pg_cntl_mask);
+
+void dcn_pg_cntl_destroy(struct pg_cntl **pg_cntl);
+
+#endif /* DCN42_PG_CNTL */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/Makefile b/drivers/gpu/drm/amd/display/dc/resource/Makefile
index 5b42da8b79c2e..1a17a885fe84b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/resource/Makefile
@@ -222,4 +222,22 @@ AMD_DAL_RESOURCE_DCN401 = $(addprefix $(AMDDALPATH)/dc/resource/dcn401/,$(RESOUR
 
 AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN401)
 
+###############################################################################
+# DCN42
+###############################################################################
+RESOURCE_DCN42 = dcn42_resource.o dcn42_resource_fpu.o
+
+AMD_DAL_RESOURCE_DCN42 = $(addprefix $(AMDDALPATH)/dc/resource/dcn42/,$(RESOURCE_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN42)
+
+# FPU Compile Flags for FPU files
+resource_ccflags := $(CC_FLAGS_FPU)
+resource_rcflags := $(CC_FLAGS_NO_FPU)
+
+CFLAGS_$(AMDDALPATH)/dc/resource/dcn42/dcn42_resource_fpu.o := $(resource_ccflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/resource/dcn42/dcn42_resource_fpu.o := $(resource_rcflags)
+
+###############################################################################
 endif
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
index c27645708286a..a57d684278122 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
@@ -497,7 +497,7 @@ static void read_dce_straps(
 static struct audio *create_audio(
 		struct dc_context *ctx, unsigned int inst)
 {
-	return dce60_audio_create(ctx, inst,
+	return dce_audio_create(ctx, inst,
 			&audio_regs[inst], &audio_shift, &audio_mask);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
index bbe185e15eb67..44178e915bdcb 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
@@ -1273,6 +1273,12 @@ static const struct dc_cap_funcs cap_funcs = {
 	.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
 };
 
+void dcn10_get_default_tiling_info(struct dc_tiling_info *tiling_info)
+{
+	tiling_info->gfxversion = DcGfxVersion9;
+	tiling_info->gfx9.swizzle = DC_SW_LINEAR;
+}
+
 static const struct resource_funcs dcn10_res_pool_funcs = {
 	.destroy = dcn10_destroy_resource_pool,
 	.link_enc_create = dcn10_link_encoder_create,
@@ -1284,7 +1290,8 @@ static const struct resource_funcs dcn10_res_pool_funcs = {
 	.add_stream_to_ctx = dcn10_add_stream_to_ctx,
 	.patch_unknown_plane_state = dcn10_patch_unknown_plane_state,
 	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static uint32_t read_pipe_fuses(struct dc_context *ctx)
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h
index 7bc1be53e8009..c7409298caa9a 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h
@@ -53,5 +53,7 @@ struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
 
 unsigned int dcn10_get_vstartup_for_pipe(struct pipe_ctx *pipe_ctx);
 
+void dcn10_get_default_tiling_info(struct dc_tiling_info *tiling_info);
+
 #endif /* __DC_RESOURCE_DCN10_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
index 8b555187ac753..8426d5f9f3771 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
@@ -2232,7 +2232,8 @@ static const struct resource_funcs dcn20_res_pool_funcs = {
 	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
 	.populate_dml_pipes = dcn20_populate_dml_pipes_from_context,
 	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
@@ -2341,8 +2342,6 @@ static bool init_soc_bounding_box(struct dc *dc,
 	struct _vcs_dpi_ip_params_st *loaded_ip =
 			get_asic_rev_ip_params(dc->ctx->asic_id.hw_internal_rev);
 
-	DC_LOGGER_INIT(dc->ctx->logger);
-
 	if (pool->base.pp_smu) {
 		struct pp_smu_nv_clock_table max_clocks = {0};
 		unsigned int uclk_states[8] = {0};
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
index 4ea76e46ab15d..491d8d3b1b68b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
@@ -1081,7 +1081,8 @@ static struct resource_funcs dcn201_res_pool_funcs = {
 	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
 	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
 	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn201_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
index 0f4307f8f3dd5..a1a529cabb930 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
@@ -1378,7 +1378,8 @@ static const struct resource_funcs dcn21_res_pool_funcs = {
 	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
 	.update_bw_bounding_box = dcn21_update_bw_bounding_box,
 	.get_panel_config_defaults = dcn21_get_panel_config_defaults,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn21_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
index 2fa86b9587ed0..566517b99a097 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
@@ -2248,7 +2248,8 @@ static const struct resource_funcs dcn30_res_pool_funcs = {
 	.update_bw_bounding_box = dcn30_update_bw_bounding_box,
 	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
 	.get_panel_config_defaults = dcn30_get_panel_config_defaults,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 #define CTX ctx
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
index 7842bee57e636..d21b928055e5c 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
@@ -1400,7 +1400,8 @@ static struct resource_funcs dcn301_res_pool_funcs = {
 	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
 	.update_bw_bounding_box = dcn301_update_bw_bounding_box,
 	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn10_get_default_tiling_info,
 };
 
 static bool dcn301_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
index 1874d5d6b7820..d24b9b81df77e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
@@ -1155,7 +1155,8 @@ static struct resource_funcs dcn302_res_pool_funcs = {
 		.update_bw_bounding_box = dcn302_update_bw_bounding_box,
 		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
 		.get_panel_config_defaults = dcn302_get_panel_config_defaults,
-		.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+		.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+		.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static struct dc_cap_funcs cap_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
index d52201cb359fd..0b44a33a0d326 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
@@ -1099,7 +1099,8 @@ static struct resource_funcs dcn303_res_pool_funcs = {
 		.update_bw_bounding_box = dcn303_update_bw_bounding_box,
 		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
 		.get_panel_config_defaults = dcn303_get_panel_config_defaults,
-		.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe
+		.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
+		.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static struct dc_cap_funcs cap_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
index 2055f1f8af652..046724c86c7ae 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
@@ -1851,6 +1851,7 @@ static struct resource_funcs dcn31_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
+	.get_default_tiling_info = dcn10_get_default_tiling_info,
 	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
index 1939f720ba295..66bf7725aeaf0 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
@@ -1782,7 +1782,8 @@ static struct resource_funcs dcn314_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static struct clock_source *dcn30_clock_source_create(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
index e8377c190f635..4d5fcd7a0b006 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
@@ -1846,7 +1846,8 @@ static struct resource_funcs dcn315_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn315_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
index 045ce01bd74eb..63675b53674af 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
@@ -1721,7 +1721,8 @@ static struct resource_funcs dcn316_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn316_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index 7ebb7d1193af2..2b9d8d2245723 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -1785,7 +1785,10 @@ static bool dml1_validate(struct dc *dc, struct dc_state *context, enum dc_valid
 
 	dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
 
+	DC_FP_START();
 	dcn32_override_min_req_memclk(dc, context);
+	DC_FP_END();
+
 	dcn32_override_min_req_dcfclk(dc, context);
 
 	BW_VAL_TRACE_END_WATERMARKS();
@@ -2109,6 +2112,7 @@ static struct resource_funcs dcn32_res_pool_funcs = {
 	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
 	.add_phantom_pipes = dcn32_add_phantom_pipes,
+	.get_default_tiling_info = dcn10_get_default_tiling_info,
 	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
 	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
index c1582c27ac872..e3dc4b1aacda8 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
@@ -1619,6 +1619,7 @@ static struct resource_funcs dcn321_res_pool_funcs = {
 	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.get_max_hw_cursor_size = dcn32_get_max_hw_cursor_size,
+	.get_default_tiling_info = dcn10_get_default_tiling_info,
 };
 
 static uint32_t read_pipe_fuses(struct dc_context *ctx)
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index 3494a40cea99f..ff557c4d594e4 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -1802,7 +1802,8 @@ static struct resource_funcs dcn35_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn35_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index 080bc7f24ffaa..0c39d0b179470 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -1775,7 +1775,8 @@ static struct resource_funcs dcn351_res_pool_funcs = {
 	.get_det_buffer_size = dcn31_get_det_buffer_size,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info
 };
 
 static bool dcn351_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
index af51ac4ea59e2..1ad44fb642130 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
@@ -1781,7 +1781,8 @@ static struct resource_funcs dcn36_res_pool_funcs = {
 	.get_preferred_eng_id_dpia = dcn36_get_preferred_eng_id_dpia,
 	.get_vstartup_for_pipe = dcn10_get_vstartup_for_pipe,
 	.update_dc_state_for_encoder_switch = dcn31_update_dc_state_for_encoder_switch,
-	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params
+	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.get_default_tiling_info = dcn10_get_default_tiling_info,
 };
 
 static bool dcn36_resource_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
index e37aab939a41b..60a0e7c95c735 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
@@ -1839,9 +1839,16 @@ static struct resource_funcs dcn401_res_pool_funcs = {
 	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 	.get_power_profile = dcn401_get_power_profile,
 	.get_vstartup_for_pipe = dcn401_get_vstartup_for_pipe,
-	.get_max_hw_cursor_size = dcn32_get_max_hw_cursor_size
+	.get_max_hw_cursor_size = dcn32_get_max_hw_cursor_size,
+	.get_default_tiling_info = dcn401_get_default_tiling_info
 };
 
+void dcn401_get_default_tiling_info(struct dc_tiling_info *tiling_info)
+{
+	tiling_info->gfxversion = DcGfxAddr3;
+	tiling_info->gfx_addr3.swizzle = DC_ADDR3_SW_LINEAR;
+}
+
 static uint32_t read_pipe_fuses(struct dc_context *ctx)
 {
 	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
index 08bec17556174..5f3b0319cb5bb 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
@@ -28,6 +28,8 @@ enum dc_status dcn401_validate_bandwidth(struct dc *dc,
 
 void dcn401_prepare_mcache_programming(struct dc *dc, struct dc_state *context);
 
+void dcn401_get_default_tiling_info(struct dc_tiling_info *tiling_info);
+
 unsigned int dcn401_get_vstartup_for_pipe(struct pipe_ctx *pipe_ctx);
 
 /* Following are definitions for run time init of reg offsets */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
new file mode 100644
index 0000000000000..aef187bcf5c34
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
@@ -0,0 +1,2337 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn32/dcn32_init.h"
+#include "dcn42/dcn42_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+
+#include "dcn42_resource.h"
+#include "dcn42_resource_fpu.h"
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+#include "dcn32/dcn32_resource.h"
+#include "dcn35/dcn35_resource.h"
+#include "dcn321/dcn321_resource.h"
+#include "dcn401/dcn401_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn35/dcn35_hubbub.h"
+#include "dcn42/dcn42_hubbub.h"
+#include "dcn401/dcn401_mpc.h"
+#include "dcn42/dcn42_mpc.h"
+#include "dcn35/dcn35_hubp.h"
+#include "dcn42/dcn42_hubp.h"
+#include "irq/dcn42/irq_service_dcn42.h"
+#include "dcn42/dcn42_dpp.h"
+#include "dcn401/dcn401_dsc.h"
+#include "dcn42/dcn42_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn35/dcn35_opp.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn42/dcn42_dio_stream_encoder.h"
+#include "dcn42/dcn42_pg_cntl.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "dcn42/dcn42_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn401/dcn401_dio_link_encoder.h"
+#include "dcn10/dcn10_link_encoder.h"
+#include "dcn321/dcn321_dio_link_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "dio/virtual/virtual_stream_encoder.h"
+#include "dml/display_mode_vba.h"
+#include "dcn42/dcn42_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "link_service.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn42/dcn42_mmhubbub.h"
+#include "dcn42/dcn42_dio_link_encoder.h"
+
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+#include "dpcs/dpcs_4_0_0_offset.h"
+#include "dpcs/dpcs_4_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dmub_replay.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "dc_state_priv.h"
+#include "link_enc_cfg.h"
+
+#include "dml2_0/dml2_wrapper.h"
+
+#define regBIF_BX0_BIOS_SCRATCH_3                            0x003b
+#define regBIF_BX0_BIOS_SCRATCH_3_BASE_IDX                   1
+#define regBIF_BX0_BIOS_SCRATCH_6                            0x003e
+#define regBIF_BX0_BIOS_SCRATCH_6_BASE_IDX                   1
+
+#define DC_LOGGER_INIT(logger)
+
+enum dcn401_clk_src_array_id {
+	DCN401_CLK_SRC_PLL0,
+	DCN401_CLK_SRC_PLL1,
+	DCN401_CLK_SRC_PLL2,
+	DCN401_CLK_SRC_PLL3,
+	DCN401_CLK_SRC_PLL4,
+	DCN401_CLK_SRC_TOTAL
+};
+
+/* begin
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)                                       \
+	REG_STRUCT.reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+						  reg##reg_name
+#define SR_ARR(reg_name, id)                                   \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+							  reg##reg_name
+#define SR_ARR_INIT(reg_name, id, value) \
+	REG_STRUCT[id].reg_name = value
+
+#define SRI(reg_name, block, id)                                         \
+	REG_STRUCT.reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+						  reg##block##id##_##reg_name
+
+#define SRI_ARR(reg_name, block, id)                                         \
+	REG_STRUCT[id].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							  reg##block##id##_##reg_name
+
+/*
+ * Used when a reg_name would otherwise begin with an integer
+ */
+#define SRI_ARR_US(reg_name, block, id)                                   \
+	REG_STRUCT[id].reg_name = BASE(reg##block##id##reg_name##_BASE_IDX) + \
+							  reg##block##id##reg_name
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id - 1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)                                         \
+	REG_STRUCT[id - 1].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+								  reg##block##id##_##reg_name
+
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)                            \
+	REG_STRUCT[index].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+								 reg##block##id##_##reg_name
+
+#define SRI2(reg_name, block, id)                \
+	.reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+				reg##reg_name
+#define SRI2_ARR(reg_name, block, id)                          \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+							  reg##reg_name
+
+#define SRIR(var_name, reg_name, block, id)                    \
+	.var_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+				reg##block##id##_##reg_name
+
+#define SRII(reg_name, block, id)                                            \
+	REG_STRUCT.reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							  reg##block##id##_##reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)                                      \
+	REG_STRUCT[inst].reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+									reg##block##id##_##reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)                                  \
+	.RMU##_##reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							reg##block##id##_##reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)                              \
+	REG_STRUCT.reg_name[id] = \
+		BASE(reg##block##id##_##temp_name##_BASE_IDX) + \
+							  reg##block##id##_##temp_name
+
+#define DCCG_SRII(reg_name, block, id)                                                 \
+	REG_STRUCT.block##_##reg_name[id] = \
+		BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+										reg##block##id##_##reg_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
+	.field_name = reg_name##__##field_name##post_fix
+
+#define VUPDATE_SRII(reg_name, block, id)                                    \
+	REG_STRUCT.reg_name[id] = BASE(reg##reg_name##_##block##id##_BASE_IDX) + \
+							  reg##reg_name##_##block##id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)                                               \
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_##reg_name##_BASE_IDX) + \
+						  regBIF_BX0_##reg_name
+#define NBIO_SR_ARR(reg_name, id)                                           \
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_##reg_name##_BASE_IDX) + \
+							  regBIF_BX0_##reg_name
+
+#define CTX ctx
+#define REG(reg_name) \
+	(ctx->dcn_reg_offsets[reg##reg_name##_BASE_IDX] + reg##reg_name)
+
+static struct bios_registers bios_regs;
+
+#define bios_regs_init()     \
+	NBIO_SR(BIOS_SCRATCH_3), \
+		NBIO_SR(BIOS_SCRATCH_6)
+
+#define clk_src_regs_init(index, pllid) \
+	CS_COMMON_REG_LIST_DCN42_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
+
+static const struct dce110_clk_src_shift cs_shift = {
+	CS_COMMON_MASK_SH_LIST_DCN3_1_4(__SHIFT)
+};
+static const struct dce110_clk_src_mask cs_mask = {
+	CS_COMMON_MASK_SH_LIST_DCN3_1_4(_MASK)
+};
+#define abm_regs_init(id) \
+	ABM_DCN42_REG_LIST_RI(id)
+
+static struct dce_abm_registers abm_regs[4];
+
+static const struct dce_abm_shift abm_shift = {
+	ABM_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dce_abm_mask abm_mask = {
+	ABM_MASK_SH_LIST_DCN42(_MASK)};
+
+#define audio_regs_init(id) \
+	AUD_COMMON_REG_LIST_RI(id)
+
+static struct dce_audio_registers audio_regs[5];
+
+static const struct dce_audio_shift audio_shift = {
+		DCN42_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCN42_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs_init(id) \
+	VPG_DCN401_REG_LIST_RI(id)
+
+static struct dcn31_vpg_registers vpg_regs[10];
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)};
+
+#define apg_regs_init(id) \
+	APG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_apg_registers apg_regs[10];
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_apg_mask apg_mask = {
+	DCN31_APG_MASK_SH_LIST(_MASK)};
+
+#define stream_enc_regs_init(id) \
+	SE_DCN42_REG_LIST_RI(id)
+
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+	SE_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+	SE_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define aux_regs_init(id) \
+	DCN2_AUX_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
+
+#define hpd_regs_init(id) \
+	HPD_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
+
+#define link_regs_init(id, phyid) \
+	LE_DCN401_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_registers link_enc_regs[5];
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN42(_MASK)};
+
+#define hpo_dp_stream_encoder_reg_init(id) \
+	DCN42_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN4_2_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN4_2_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)};
+
+#define hpo_dp_link_encoder_reg_init(id) \
+	DCN42_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[4];
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)};
+
+#define dpp_regs_init(id) \
+	DPP_REG_LIST_DCN42_COMMON_RI(id)
+
+static struct dcn42_dpp_registers dpp_regs[4];
+
+static const struct dcn42_dpp_shift tf_shift = {
+	DPP_REG_LIST_SH_MASK_DCN42_COMMON(__SHIFT)};
+
+static const struct dcn42_dpp_mask tf_mask = {
+	DPP_REG_LIST_SH_MASK_DCN42_COMMON(_MASK)};
+
+#define opp_regs_init(id) \
+	OPP_REG_LIST_DCN401_RI(id)
+
+static struct dcn20_opp_registers opp_regs[4];
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)};
+
+#define aux_engine_regs_init(id)                                              \
+	AUX_COMMON_REG_LIST0_RI(id), SR_ARR_INIT(AUXN_IMPCAL, id, 0),             \
+		SR_ARR_INIT(AUXP_IMPCAL, id, 0),                                      \
+		SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+		SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)
+
+static struct dce110_aux_registers aux_engine_regs[5];
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)};
+
+#define dwbc_regs_dcn401_init(id) \
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dcn30_dwbc_registers dwbc401_regs[1];
+
+static const struct dcn30_dwbc_shift dwbc401_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)};
+
+static const struct dcn30_dwbc_mask dwbc401_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)};
+
+#define mcif_wb_regs_dcn3_init(id) \
+	MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(id)
+
+static struct dcn35_mmhubbub_registers mcif_wb35_regs[1];
+
+static const struct dcn35_mmhubbub_shift mcif_wb35_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)};
+
+static const struct dcn35_mmhubbub_mask mcif_wb35_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(_MASK)};
+
+#define dsc_regs_init(id) \
+	DSC_REG_LIST_DCN401_RI(id)
+
+static struct dcn401_dsc_registers dsc_regs[4];
+
+static const struct dcn401_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN401(__SHIFT)};
+
+static const struct dcn401_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN401(_MASK)};
+
+static struct dcn42_mpc_registers mpc_regs;
+
+#define dcn_mpc_regs_init()                \
+	MPC_REG_LIST_DCN42(0),                 \
+		MPC_REG_LIST_DCN42(1),             \
+		MPC_REG_LIST_DCN42(2),             \
+		MPC_REG_LIST_DCN42(3),             \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3), \
+		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0), \
+		MPC_RMCM_REG_LIST_DCN42(0),		   \
+		MPC_RMCM_REG_LIST_DCN42(1)
+
+static const struct dcn42_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn42_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define optc_regs_init(id) \
+	OPTC_COMMON_REG_LIST_DCN42_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define hubp_regs_init(id) \
+	HUBP_REG_LIST_DCN42_RI(id)
+
+static struct dcn_hubp2_registers hubp_regs[4];
+
+static const struct dcn_hubp2_shift hubp_shift = {
+	HUBP_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+	HUBP_MASK_SH_LIST_DCN42(_MASK)};
+
+static struct dcn_hubbub_registers hubbub_reg;
+
+#define hubbub_reg_init() \
+	HUBBUB_REG_LIST_DCN42(0)
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+	HUBBUB_MASK_SH_LIST_DCN4_2(__SHIFT)};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+	HUBBUB_MASK_SH_LIST_DCN4_2(_MASK)};
+
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init() \
+	DCCG_REG_LIST_DCN42_RI()
+
+static const struct dccg_shift dccg_shift = {
+	DCCG_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dccg_mask dccg_mask = {
+	DCCG_MASK_SH_LIST_DCN42(_MASK)};
+
+static struct pg_cntl_registers pg_cntl_regs;
+
+#define pg_cntl_dcn42_regs_init() \
+	PG_CNTL_REG_LIST_DCN42()
+
+static const struct pg_cntl_shift pg_cntl_shift = {
+		PG_CNTL_MASK_SH_LIST_DCN42(__SHIFT)
+};
+
+static const struct pg_cntl_mask pg_cntl_mask = {
+		PG_CNTL_MASK_SH_LIST_DCN42(_MASK)
+};
+#define SRII2(reg_name_pre, reg_name_post, id)                                                       \
+	.reg_name_pre##_##reg_name_post[id] = \
+		BASE(reg##reg_name_pre##id##_##reg_name_post##_BASE_IDX) + \
+					reg##reg_name_pre##id##_##reg_name_post
+
+#define HWSEQ_DCN42_REG_LIST()                \
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL),           \
+		SR(DIO_MEM_PWR_CTRL),                 \
+		SR(ODM_MEM_PWR_CTRL3),                \
+		SR(MMHUBBUB_MEM_PWR_CNTL),            \
+		SR(DCCG_GATE_DISABLE_CNTL),           \
+		SR(DCCG_GATE_DISABLE_CNTL2),          \
+		SR(DCFCLK_CNTL),                      \
+		SR(DC_MEM_GLOBAL_PWR_REQ_CNTL),       \
+		SRII(PIXEL_RATE_CNTL, OTG, 0),        \
+		SRII(PIXEL_RATE_CNTL, OTG, 1),        \
+		SRII(PIXEL_RATE_CNTL, OTG, 2),        \
+		SRII(PIXEL_RATE_CNTL, OTG, 3),\
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0), \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1), \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2), \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+		SR(MICROSECOND_TIME_BASE_DIV),        \
+		SR(MILLISECOND_TIME_BASE_DIV),        \
+		SR(DISPCLK_FREQ_CHANGE_CNTL),         \
+		SR(RBBMIF_TIMEOUT_DIS),               \
+		SR(RBBMIF_TIMEOUT_DIS_2),             \
+		SR(DCHUBBUB_CRC_CTRL),                \
+		SR(DPP_TOP0_DPP_CRC_CTRL),            \
+		SR(DPP_TOP0_DPP_CRC_VAL_R),           \
+		SR(DPP_TOP0_DPP_CRC_VAL_G),           \
+		SR(DPP_TOP0_DPP_CRC_VAL_B),           \
+		SR(MPC_CRC_CTRL),                     \
+		SR(MPC_CRC_RESULT_R),                 \
+		SR(MPC_CRC_RESULT_G),                 \
+		SR(MPC_CRC_RESULT_B),                 \
+		SR(MPC_CRC_RESULT_A),                 \
+		SR(DOMAIN0_PG_CONFIG),                \
+		SR(DOMAIN1_PG_CONFIG),                \
+		SR(DOMAIN2_PG_CONFIG),                \
+		SR(DOMAIN3_PG_CONFIG),                \
+		SR(DOMAIN16_PG_CONFIG),               \
+		SR(DOMAIN17_PG_CONFIG),               \
+		SR(DOMAIN18_PG_CONFIG),               \
+		SR(DOMAIN19_PG_CONFIG), \
+		SR(DOMAIN22_PG_CONFIG),               \
+		SR(DOMAIN23_PG_CONFIG),               \
+		SR(DOMAIN24_PG_CONFIG),               \
+		SR(DOMAIN25_PG_CONFIG),               \
+		SR(DOMAIN26_PG_CONFIG),               \
+		SR(DOMAIN0_PG_STATUS),                \
+		SR(DOMAIN1_PG_STATUS),                \
+		SR(DOMAIN2_PG_STATUS),                \
+		SR(DOMAIN3_PG_STATUS),                \
+		SR(DOMAIN16_PG_STATUS),               \
+		SR(DOMAIN17_PG_STATUS),               \
+		SR(DOMAIN18_PG_STATUS),               \
+		SR(DOMAIN19_PG_STATUS),               \
+		SR(DOMAIN22_PG_STATUS),               \
+		SR(DOMAIN23_PG_STATUS),               \
+		SR(DOMAIN24_PG_STATUS),               \
+		SR(DOMAIN25_PG_STATUS),               \
+		SR(DOMAIN26_PG_STATUS),               \
+		SR(DC_IP_REQUEST_CNTL),               \
+		SR(AZALIA_AUDIO_DTO),                 \
+		SR(HPO_TOP_HW_CONTROL),               \
+		SR(AZALIA_CONTROLLER_CLOCK_GATING)
+
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init() \
+	HWSEQ_DCN42_REG_LIST()
+
+#define HWSEQ_DCN42_MASK_SH_LIST(mask_sh)                                            \
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh),                                                 \
+		HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+		HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+		HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+		HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+		HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN26_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN26_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh),                        \
+		HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh),                \
+		HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh),      \
+		HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh),                            \
+		HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh),           \
+		HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+		HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_R_DMU_GATE_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_G_RBBMIF_GATE_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, RBBMIF_FGCG_REP_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DPREFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DPPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DTBCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DCFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DPIACLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_DISPCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_SOCCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_DMCUBCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, HDMICHARCLK0_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK0_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK1_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK2_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK3_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK4_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK5_GATE_DISABLE, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+	HWSEQ_DCN42_MASK_SH_LIST(__SHIFT)};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+	HWSEQ_DCN42_MASK_SH_LIST(_MASK)};
+
+#define vmid_regs_init(id) \
+	DCN20_VMID_REG_LIST_RI(id)
+
+static struct dcn_vmid_registers vmid_regs[16];
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+	DCN20_VMID_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+	DCN20_VMID_MASK_SH_LIST(_MASK)};
+
+static const struct resource_caps res_cap_dcn42 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_dpp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_usb4_dpia = 6,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 4,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 4,
+	.num_rmcm = 2,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+		.argb8888 = true,
+		.nv12 = true,
+		.fp16 = true,
+		.p010 = true,
+		.ayuv = false,
+	},
+
+	.max_upscale_factor = {.argb8888 = 16000, .nv12 = 16000, .fp16 = 16000},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {.argb8888 = 167, .nv12 = 167, .fp16 = 167},
+
+	.min_width = 64,
+	.min_height = 64};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.disable_dpp_power_gate = true,
+	.disable_hubp_power_gate = true,
+	.disable_optc_power_gate = true,
+	.pipe_split_policy = MPC_SPLIT_AVOID,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096, /*up to 4K for APU*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = false,
+			.i2c = true,
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+		}},
+	.root_clock_optimization = {
+		.bits = {
+			.dpp = true,
+			.dsc = true,/*dscclk and dsc pg*/
+			.hdmistream = false,
+			.hdmichar = true,
+			.dpstream = true,
+			.symclk32_se = true,
+			.symclk32_le = true,
+			.symclk_fe = true,
+			.physymclk = false,
+			.dpiasymclk = true,
+		}
+	},
+	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
+	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
+	.minimum_z8_residency_time = 1, /* Always allow when other conditions are met */
+	.support_eDP1_5 = true,
+	.use_max_lb = true,
+	.force_disable_subvp = false,
+	.exit_idle_opt_for_cursor_updates = true,
+	.using_dml2 = true,
+	.using_dml21 = true,
+	.enable_single_display_2to1_odm_policy = true,
+
+	// must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions
+	.enable_double_buffered_dsc_pg_support = true,
+	.enable_dp_dig_pixel_rate_div_policy = 1,
+	.allow_sw_cursor_fallback = false,
+	.psp_disabled_wa = true,
+	.alloc_extra_way_for_cursor = true,
+	.min_prefetch_in_strobe_ns = 60000, // 60us
+	.disable_unbounded_requesting = false,
+	.dcc_meta_propagation_delay_us = 10,
+	.disable_timeout = true,
+	.min_disp_clk_khz = 50000,
+	.static_screen_wait_frames = 2,
+	.disable_z10 = false,
+	.ignore_pg = true,
+	.disable_stutter_for_wm_program = true,
+};
+
+static const struct dc_check_config config_defaults = {
+	.enable_legacy_fast_update = false,
+};
+
+static struct dce_aux *dcn42_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+		aux_engine_regs_init(1),
+		aux_engine_regs_init(2),
+		aux_engine_regs_init(3),
+		aux_engine_regs_init(4);
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+								SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+								&aux_engine_regs[inst],
+								&aux_mask,
+								&aux_shift,
+								ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs_init(id) \
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
+
+static const struct dce_i2c_shift i2c_shifts = {
+	I2C_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
+};
+static const struct dce_i2c_mask i2c_masks = {
+	I2C_COMMON_MASK_SH_LIST_DCN35(_MASK)
+};
+
+/* ========================================================== */
+
+/*
+ * DPIA index | Preferred Encoder     |    Host Router
+ *   0        |      C                |       0
+ *   1        |      First Available  |       0
+ *   2        |      D                |       1
+ *   3        |      First Available  |       1
+ *   4        |      E                |       2
+ *   5        |      First Available  |       2
+ */
+/* ========================================================== */
+static const enum engine_id dpia_to_preferred_enc_id_table[] = {
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGE,
+		ENGINE_ID_DIGE
+};
+
+static enum engine_id dcn42_get_preferred_eng_id_dpia(unsigned int dpia_index)
+{
+	return dpia_to_preferred_enc_id_table[dpia_index];
+}
+
+static struct dce_i2c_hw *dcn42_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+						  &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct clock_source *dcn42_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn401_clk_src_construct(clk_src, ctx, bios, id,
+								 regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct hubbub *dcn42_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+		vmid_regs_init(1),
+		vmid_regs_init(2),
+		vmid_regs_init(3),
+		vmid_regs_init(4),
+		vmid_regs_init(5),
+		vmid_regs_init(6),
+		vmid_regs_init(7),
+		vmid_regs_init(8),
+		vmid_regs_init(9),
+		vmid_regs_init(10),
+		vmid_regs_init(11),
+		vmid_regs_init(12),
+		vmid_regs_init(13),
+		vmid_regs_init(14),
+		vmid_regs_init(15);
+
+	hubbub42_construct(hubbub3, ctx,
+					   &hubbub_reg,
+					   &hubbub_shift,
+					   &hubbub_mask,
+					   DCN42_DEFAULT_DET_SIZE,
+					   8,
+					   DCN42_CRB_SIZE_KB);
+	for (i = 0; i < res_cap_dcn42.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct hubp *dcn42_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+		hubp_regs_init(1),
+		hubp_regs_init(2),
+		hubp_regs_init(3);
+
+	if (hubp42_construct(hubp2, ctx, inst,
+						 &hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+static const struct dc_panel_config dcn42_panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn42_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN42_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn42_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn42_dpp *dpp42 =
+		kzalloc(sizeof(struct dcn42_dpp), GFP_KERNEL);
+
+	if (!dpp42)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+		dpp_regs_init(1),
+		dpp_regs_init(2),
+		dpp_regs_init(3);
+
+	if (dpp42_construct(dpp42, ctx, inst,
+						&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp42->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp42);
+	return NULL;
+}
+
+static struct mpc *dcn42_mpc_create(
+	struct dc_context *ctx,
+	int num_mpcc,
+	int num_rmu)
+{
+	struct dcn42_mpc *mpc401 = kzalloc(sizeof(struct dcn42_mpc),
+										GFP_KERNEL);
+
+	if (!mpc401)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
+	dcn42_mpc_construct(mpc401, ctx,
+						 &mpc_regs,
+						 &mpc_shift,
+						 &mpc_mask,
+						 num_mpcc,
+						 num_rmu);
+
+	return &mpc401->base;
+}
+
+static struct output_pixel_processor *dcn42_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp4 =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp4) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+		opp_regs_init(1),
+		opp_regs_init(2),
+		opp_regs_init(3);
+	dcn20_opp_construct(opp4, ctx, inst,
+						&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp4->base;
+}
+
+static struct timing_generator *dcn42_timing_generator_create(
+	struct dc_context *ctx,
+	uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+		optc_regs_init(1),
+		optc_regs_init(2),
+		optc_regs_init(3);
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn42_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+	.max_hdmi_deep_color = COLOR_DEPTH_121212,
+	.max_hdmi_pixel_clock = 600000,
+	.hdmi_ycbcr420_supported = true,
+	.dp_ycbcr420_supported = true,
+	.fec_supported = true,
+	.flags.bits.IS_HBR2_CAPABLE = true,
+	.flags.bits.IS_HBR3_CAPABLE = true,
+	.flags.bits.IS_TPS3_CAPABLE = true,
+	.flags.bits.IS_TPS4_CAPABLE = true};
+
+static struct link_encoder *dcn42_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs))
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+		aux_regs_init(1),
+		aux_regs_init(2),
+		aux_regs_init(3),
+		aux_regs_init(4);
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+		hpd_regs_init(1),
+		hpd_regs_init(2),
+		hpd_regs_init(3),
+		hpd_regs_init(4);
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+		link_regs_init(1, B),
+		link_regs_init(2, C),
+		link_regs_init(3, D),
+		link_regs_init(4, E);
+
+	dcn42_link_encoder_construct(enc20,
+								  enc_init_data,
+								  &link_enc_feature,
+								  &link_enc_regs[enc_init_data->transmitter],
+								  &link_enc_aux_regs[enc_init_data->channel - 1],
+								  &link_enc_hpd_regs[enc_init_data->hpd_source],
+								  &le_shift,
+								  &le_mask);
+	return &enc20->enc10.base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *dcn42_create_audio(
+	struct dc_context *ctx, unsigned int inst)
+{
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+		audio_regs_init(1),
+		audio_regs_init(2),
+		audio_regs_init(3),
+		audio_regs_init(4);
+
+	return dce_audio_create(ctx, inst,
+							&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn42_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg4 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg4)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+		vpg_regs_init(1),
+		vpg_regs_init(2),
+		vpg_regs_init(3),
+		vpg_regs_init(4),
+		vpg_regs_init(5),
+		vpg_regs_init(6),
+		vpg_regs_init(7),
+		vpg_regs_init(8),
+		vpg_regs_init(9);
+	vpg31_construct(vpg4, ctx, inst,
+					&vpg_regs[inst],
+					&vpg_shift,
+					&vpg_mask);
+
+	return &vpg4->base;
+}
+
+static struct apg *dcn42_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+		apg_regs_init(1),
+		apg_regs_init(2),
+		apg_regs_init(3),
+		apg_regs_init(4),
+		apg_regs_init(5),
+		apg_regs_init(6),
+		apg_regs_init(7),
+		apg_regs_init(8),
+		apg_regs_init(9);
+
+	apg31_construct(apg31, ctx, inst,
+					&apg_regs[inst],
+					&apg_shift,
+					&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn42_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct apg *apg;
+
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	/* Mapping of VPG, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGE) {
+		vpg_inst = eng_id;
+		apg_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn42_vpg_create(ctx, vpg_inst);
+	apg = dcn42_apg_create(ctx, apg_inst);
+
+	if (!enc1 || !vpg || !apg) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+		stream_enc_regs_init(1),
+		stream_enc_regs_init(2),
+		stream_enc_regs_init(3),
+		stream_enc_regs_init(4);
+
+	dcn42_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+									eng_id, vpg, apg,
+									&stream_enc_regs[eng_id],
+									&se_shift, &se_mask);
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn42_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[5] -> HPO_DP[0]
+	 * VPG[6] -> HPO_DP[1]
+	 * VPG[7] -> HPO_DP[2]
+	 * VPG[8] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 5;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[6] -> HPO_DP[0]
+	 * APG[7] -> HPO_DP[1]
+	 * APG[8] -> HPO_DP[2]
+	 * APG[9] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst + 5;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn42_vpg_create(ctx, vpg_inst);
+	apg = dcn42_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+		hpo_dp_stream_encoder_reg_init(1),
+		hpo_dp_stream_encoder_reg_init(2),
+		hpo_dp_stream_encoder_reg_init(3);
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+				hpo_dp_inst, eng_id, vpg, apg,
+				&hpo_dp_stream_enc_regs[hpo_dp_inst],
+				&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn42_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1),
+	hpo_dp_link_encoder_reg_init(2),
+	hpo_dp_link_encoder_reg_init(3);
+
+	hpo_dp_link_encoder42_construct(hpo_dp_enc31, ctx, inst,
+				&hpo_dp_link_enc_regs[inst],
+				&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn42_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn42_create_audio,
+	.create_stream_encoder = dcn42_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn42_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn42_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn42_hwseq_create,
+};
+
+static void dcn42_dsc_destroy(struct display_stream_compressor **dsc)
+{
+	kfree(container_of(*dsc, struct dcn401_dsc, base));
+	*dsc = NULL;
+}
+
+static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN31_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.stream_enc[i]->apg));
+				pool->base.stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN31_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn42_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn42_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL)
+			dal_irq_service_destroy(&pool->base.irqs);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL) {
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.pg_cntl != NULL)
+		dcn_pg_cntl_destroy(&pool->base.pg_cntl);
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+static void dcn42_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
+{
+	const struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc_link *link = stream->link;
+	struct link_encoder *link_enc = pipe_ctx->link_res.dio_link_enc;
+	struct pixel_clk_params *pixel_clk_params = &pipe_ctx->stream_res.pix_clk_params;
+
+	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+
+	if (pipe_ctx->dsc_padding_params.dsc_hactive_padding != 0)
+		pixel_clk_params->requested_pix_clk_100hz = pipe_ctx->dsc_padding_params.dsc_pix_clk_100hz;
+
+	if (!pipe_ctx->stream->ctx->dc->config.unify_link_enc_assignment)
+		link_enc = link_enc_cfg_get_link_enc(link);
+	if (link_enc)
+		pixel_clk_params->encoder_object_id = link_enc->id;
+
+	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+	/* TODO: un-hardcode*/
+
+	/* TODO - DP2.0 HW: calculate requested_sym_clk for UHBR rates */
+
+	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+		LINK_RATE_REF_FREQ_IN_KHZ;
+	pixel_clk_params->flags.ENABLE_SS = 0;
+	pixel_clk_params->color_depth =
+		stream->timing.display_color_depth;
+	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
+		pixel_clk_params->color_depth = COLOR_DEPTH_888;
+
+	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+		pixel_clk_params->requested_pix_clk_100hz *= 2;
+	if (dc_is_tmds_signal(stream->signal) &&
+			stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		pixel_clk_params->requested_pix_clk_100hz /= 2;
+
+	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+			pipe_ctx->clock_source,
+			&pipe_ctx->stream_res.pix_clk_params,
+			&pipe_ctx->pll_settings);
+
+	pixel_clk_params->dio_se_pix_per_cycle = 1;
+	if (dc_is_tmds_signal(stream->signal) &&
+			stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+		pixel_clk_params->dio_se_pix_per_cycle = 2;
+	} else if (dc_is_dp_signal(stream->signal)) {
+		/* round up to nearest power of 2, or max at 8 pixels per cycle */
+		if (pixel_clk_params->requested_pix_clk_100hz > 4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+			pixel_clk_params->dio_se_pix_per_cycle = 8;
+		} else if (pixel_clk_params->requested_pix_clk_100hz > 2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+			pixel_clk_params->dio_se_pix_per_cycle = 4;
+		} else if (pixel_clk_params->requested_pix_clk_100hz > stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+			pixel_clk_params->dio_se_pix_per_cycle = 2;
+		} else {
+			pixel_clk_params->dio_se_pix_per_cycle = 1;
+		}
+	}
+}
+
+static bool dcn42_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_dwbc *dwbc42 = kzalloc(sizeof(struct dcn30_dwbc),
+											GFP_KERNEL);
+
+		if (!dwbc42) {
+			dm_error("DC: failed to create dwbc42!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT dwbc401_regs
+		dwbc_regs_dcn401_init(0);
+
+		dcn30_dwbc_construct(dwbc42, ctx,
+				&dwbc401_regs[i],
+				&dwbc401_shift,
+				&dwbc401_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc42->base;
+	}
+	return true;
+}
+
+static void dcn42_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
+								struct dc_context *ctx)
+{
+	dcn42_mmhubbub_set_fgcg(
+		mcif_wb30,
+		ctx->dc->debug.enable_fine_grain_clock_gating.bits.mmhubbub);
+}
+
+static bool dcn42_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+												   GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb35_regs
+		mcif_wb_regs_dcn3_init(0);
+
+		dcn35_mmhubbub_construct(mcif_wb30, ctx,
+					&mcif_wb35_regs[i],
+					&mcif_wb35_shift,
+					&mcif_wb35_mask,
+					i);
+
+		dcn42_mmhubbub_init(mcif_wb30, ctx);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn42_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn401_dsc *dsc =
+		kzalloc(sizeof(struct dcn401_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regs_init(0),
+		dsc_regs_init(1),
+		dsc_regs_init(2),
+		dsc_regs_init(3);
+
+	dsc401_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	dsc401_set_fgcg(dsc, ctx->dc->debug.enable_fine_grain_clock_gating.bits.dsc);
+
+	dsc->max_image_width = 5760;
+
+	return &dsc->base;
+}
+
+static void dcn42_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn42_resource_pool *dcn42_pool = TO_DCN42_RES_POOL(*pool);
+
+	dcn42_resource_destruct(dcn42_pool);
+	kfree(dcn42_pool);
+	*pool = NULL;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap};
+
+static void dcn42_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	if (dc->current_state && dc->current_state->bw_ctx.dml2)
+		dml2_reinit(dc, &dc->dml2_options, &dc->current_state->bw_ctx.dml2);
+	DC_FP_END();
+}
+
+enum dc_status dcn42_validate_bandwidth(struct dc *dc,
+							  struct dc_state *context,
+							  enum dc_validate_mode validate_mode)
+{
+	bool out = false;
+
+	out = dml2_validate(dc, context, context->bw_ctx.dml2,
+						validate_mode);
+	DC_FP_START();
+	if (validate_mode == DC_VALIDATE_MODE_AND_PROGRAMMING) {
+		/*not required for mode enumeration*/
+		dcn42_decide_zstate_support(dc, context);
+	}
+	DC_FP_END();
+	return out ? DC_OK : DC_FAIL_BANDWIDTH_VALIDATE;
+}
+void dcn42_prepare_mcache_programming(struct dc *dc,
+									  struct dc_state *context)
+{
+	if (dc->debug.using_dml21)
+		dml2_prepare_mcache_programming(dc, context,
+			context->power_source == DC_POWER_SOURCE_DC ?
+				context->bw_ctx.dml2_dc_power_source : context->bw_ctx.dml2);
+}
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn42_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+static void dcn42_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = dcn42_panel_config_defaults;
+}
+static unsigned int dcn42_get_max_hw_cursor_size(const struct dc *dc,
+			struct dc_state *state,
+			const struct dc_stream_state *stream)
+{
+	return dc->caps.max_cursor_size;
+}
+static struct resource_funcs dcn42_res_pool_funcs = {
+	.destroy = dcn42_destroy_resource_pool,
+	.link_enc_create = dcn42_link_encoder_create,
+	.link_enc_create_minimal = dcn42_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn32_panel_cntl_create,
+	.validate_bandwidth = dcn42_validate_bandwidth,
+	.calculate_wm_and_dlg = NULL,
+	.populate_dml_pipes = NULL,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
+	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn42_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn401_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn42_get_panel_config_defaults,
+	.get_preferred_eng_id_dpia = dcn42_get_preferred_eng_id_dpia,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.add_phantom_pipes = dcn32_add_phantom_pipes,
+	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
+	.prepare_mcache_programming = dcn42_prepare_mcache_programming,
+	.build_pipe_pix_clk_params = dcn42_build_pipe_pix_clk_params,
+	.get_vstartup_for_pipe = dcn401_get_vstartup_for_pipe,
+	.get_max_hw_cursor_size = dcn42_get_max_hw_cursor_size,
+};
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+
+	if (value == 0 && ctx->dce_environment == DCE_ENV_DIAG)
+		value = 0xF;
+	/* DCN401 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+static bool dcn42_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn42_resource_pool *pool)
+{
+	int i, j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	uint32_t pipe_fuses;
+	uint32_t num_pipes;
+
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+		abm_regs_init(1),
+		abm_regs_init(2),
+		abm_regs_init(3);
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+	dccg_regs_init();
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn42;
+
+	/* max number of pipes for ASIC before checking for pipe fuses */
+	num_pipes = pool->base.res_cap->num_dpp;
+	pipe_fuses = read_pipe_fuses(ctx);
+
+	for (i = 0; i < pool->base.res_cap->num_dpp; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+
+	if (pipe_fuses & 1)
+		ASSERT(0); // Unexpected - Pipe 0 should always be fully functional!
+
+	if (pipe_fuses & CC_DC_PIPE_DIS__DC_FULL_DIS_MASK)
+		ASSERT(0); // Entire DCN is harvested!
+
+	pool->base.funcs = &dcn42_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	pool->base.pipe_count = num_pipes;
+	pool->base.mpcc_count = num_pipes;
+	dc->caps.ips_v2_support = true;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
+	/* TODO: Bring max cursor size back to 256 after subvp cursor corruption is fixed*/
+	dc->caps.max_cursor_size = 64;
+	dc->caps.max_buffered_cursor_size = 64;
+	dc->caps.cursor_not_scaled = true;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
+		dc->ctx->dc_bios->vram_info.num_chans * 1048576;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+	dc->caps.cache_line_size = 64;
+	dc->caps.cache_num_ways = 16;
+
+	/* Calculate the available MALL space */
+	dc->caps.max_cab_allocation_bytes =
+		dcn32_calc_num_avail_chans_for_mall(dc, dc->ctx->dc_bios->vram_info.num_chans) *
+				dc->caps.mall_size_per_mem_channel * 1024 * 1024;
+	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
+
+	dc->caps.subvp_fw_processing_delay_us = 15;
+	dc->caps.subvp_drr_max_vblank_margin_us = 40;
+	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
+	dc->caps.subvp_swath_height_margin_lines = 16;
+	dc->caps.subvp_pstate_allow_width_us = 20;
+	dc->caps.subvp_vertical_int_margin_us = 30;
+	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
+
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.dp_hpo = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+	dc->caps.seamless_odm = true;
+	dc->caps.zstate_support = true;
+	dc->caps.ips_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+	dc->caps.vtotal_limited_by_fp2 = true;
+
+	dc->caps.seamless_odm = true;
+	dc->caps.zstate_support = true;
+	dc->caps.ips_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+	dc->caps.vtotal_limited_by_fp2 = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 0;
+	dc->caps.color.dpp.ogam_ram = 0;
+	// no OGAM ROM on DCN2 and later ASICs
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	//configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut;
+	dc->caps.color.mpc.num_rmcm_3dluts = 2;
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+	dc->caps.color.mpc.preblend = true;
+	dc->caps.color.mpc.mcm_3d_lut_caps.dma_3d_lut = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.lut_dim_caps.dim_9 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.lut_dim_caps.dim_17 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.linear_1d = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.swizzle_3d_bgr = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.swizzle_3d_rgb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.unorm_12msb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.unorm_12lsb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.float_fp1_5_10 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_pixel_order_support.order_rgba = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_pixel_order_support.order_bgra = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.linear_1d = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.swizzle_3d_bgr = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.swizzle_3d_rgb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.unorm_12msb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.unorm_12lsb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.float_fp1_5_10 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_pixel_order_support.order_rgba = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_pixel_order_support.order_bgra = 1;
+
+	dc->caps.num_of_host_routers = 3;
+	dc->caps.num_of_dpias_per_host_router = 2;
+
+	/* max_disp_clock_khz_at_vmin is slightly lower than the STA value in order
+	 * to provide some margin.
+	 * It's expected for furture ASIC to have equal or higher value, in order to
+	 * have determinstic power improvement from generate to genration.
+	 * (i.e., we should not expect new ASIC generation with lower vmin rate)
+	 */
+	dc->caps.max_disp_clock_khz_at_vmin = 650000;
+	dc->config.use_spl = true;
+	dc->config.prefer_easf = true;
+
+	dc->config.dcn_sharpness_range.sdr_rgb_min = 0;
+	dc->config.dcn_sharpness_range.sdr_rgb_max = 1750;
+	dc->config.dcn_sharpness_range.sdr_rgb_mid = 750;
+	dc->config.dcn_sharpness_range.sdr_yuv_min = 0;
+	dc->config.dcn_sharpness_range.sdr_yuv_max = 3500;
+	dc->config.dcn_sharpness_range.sdr_yuv_mid = 1500;
+	dc->config.dcn_sharpness_range.hdr_rgb_min = 0;
+	dc->config.dcn_sharpness_range.hdr_rgb_max = 2750;
+	dc->config.dcn_sharpness_range.hdr_rgb_mid = 1500;
+
+	dc->config.dcn_override_sharpness_range.sdr_rgb_min = 0;
+	dc->config.dcn_override_sharpness_range.sdr_rgb_max = 3250;
+	dc->config.dcn_override_sharpness_range.sdr_rgb_mid = 1250;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_min = 0;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_max = 3500;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_mid = 1500;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_min = 0;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_max = 2750;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_mid = 1500;
+
+	dc->config.use_pipe_ctx_sync_logic = true;
+	dc->config.dc_mode_clk_limit_support = true;
+	dc->config.enable_windowed_mpo_odm = true;
+	/* Use psp mailbox to enable assr */
+	dc->config.use_assr_psp_message = true;
+	/* dcn42 and afterward always support external panel replay */
+	dc->config.frame_update_cmd_version2 = true;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		dc->caps.vbios_lttpr_aware = true;
+	}
+	dc->check_config = config_defaults;
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	/*HW default is to have all the FGCG enabled, SW no need to program them*/
+	dc->debug.enable_fine_grain_clock_gating.u32All = 0xFFFF;
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL0] =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL0,
+								  &clk_src_regs[0], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL1] =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL1,
+								  &clk_src_regs[1], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL2] =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL2,
+								  &clk_src_regs[2], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL3] =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL3,
+								  &clk_src_regs[3], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL4] =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL4,
+								  &clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN401_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+		dcn42_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_ID_DP_DTO,
+								  &clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg42_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT pg_cntl_regs
+	pg_cntl_dcn42_regs_init();
+
+	pool->base.pg_cntl = pg_cntl42_create(ctx, &pg_cntl_regs, &pg_cntl_shift, &pg_cntl_mask);
+	if (pool->base.pg_cntl == NULL) {
+		dm_error("DC: failed to create power gate control!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+	/* IRQ Service */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn42_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn42_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs, TGs, ABMs */
+	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if (pipe_fuses & 1 << i)
+			continue;
+
+		pool->base.hubps[j] = dcn42_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[j] = dcn42_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+
+		pool->base.opps[j] = dcn42_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.timing_generators[j] = dcn42_timing_generator_create(
+			ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+
+		pool->base.multiple_abms[j] = dmub_abm_create(ctx,
+													  &abm_regs[i],
+													  &abm_shift,
+													  &abm_mask);
+		if (pool->base.multiple_abms[j] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+
+		/* index for resource pool arrays for next valid pipe */
+		j++;
+	}
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* Replay */
+	pool->base.replay = dmub_replay_create(ctx);
+	if (pool->base.replay == NULL) {
+		dm_error("DC: failed to create replay obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* MPCCs */
+	pool->base.mpc = dcn42_mpc_create(ctx, pool->base.res_cap->num_timing_generator,
+			pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	/* DSCs */
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn42_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB */
+	if (!dcn42_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	/* MMHUBBUB */
+	if (!dcn42_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn42_aux_engine_create(ctx, i);
+
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn42_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+	/* DCN4.2 has 6 DPIA */
+	pool->base.usb4_dpia_count = dc->caps.num_of_host_routers * dc->caps.num_of_dpias_per_host_router;
+	if (dc->debug.dpia_debug.bits.disable_dpia)
+		pool->base.usb4_dpia_count = 0;
+
+	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+							&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer init functions and Plane caps */
+	dcn42_hw_sequencer_init_functions(dc);
+
+	dc->caps.max_planes = pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.max_odm_combine_factor = 4;
+
+	dc->cap_funcs = cap_funcs;
+	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
+
+	// For now enable SDPIF_REQUEST_RATE_LIMIT on DCN4_01 when vram_info.num_chans provided
+	if (dc->config.sdpif_request_limit_words_per_umc == 0)
+		dc->config.sdpif_request_limit_words_per_umc = 16;
+
+	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
+	 /*this will use real soc clock table*/
+	dc->dml2_options.use_native_soc_bb_construction = true;
+	dc->dml2_options.minimize_dispclk_using_odm = false;
+	if (dc->config.EnableMinDispClkODM)
+		dc->dml2_options.minimize_dispclk_using_odm = true;
+	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
+	dc->dml2_options.map_dc_pipes_with_callbacks = true;
+	dc->dml2_options.force_tdlut_enable = true;
+
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
+	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch =
+			&dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
+	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
+	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes =
+		pool->base.funcs->calculate_mall_ways_from_bytes;
+
+	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
+	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
+	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
+	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
+
+	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
+	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
+
+	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
+	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
+	dc->dml2_options.mall_cfg.max_cab_allocation_bytes =
+				dc->caps.max_cab_allocation_bytes;
+	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
+	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
+	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
+	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
+
+	dc->dml2_options.max_segments_per_hubp = 24;
+	dc->dml2_options.det_segment_size = DCN42_CRB_SEGMENT_SIZE_KB;
+	dc->dml2_options.gpuvm_enable = true;
+	dc->dml2_options.hostvm_enable = true;
+
+	/* SPL */
+	dc->caps.scl_caps.sharpener_support = true;
+
+	return true;
+
+create_fail:
+
+	dcn42_resource_destruct(pool);
+
+	return false;
+}
+struct resource_pool *dcn42_create_resource_pool(
+	const struct dc_init_data *init_data,
+	struct dc *dc)
+{
+	struct dcn42_resource_pool *pool =
+		kzalloc(sizeof(struct dcn401_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn42_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
new file mode 100644
index 0000000000000..a9b26df14520a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
@@ -0,0 +1,588 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN42_RESOURCE_H_
+#define _DCN42_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN42_RES_POOL(pool) \
+	container_of(pool, struct dcn42_resource_pool, base)
+
+/* DPP */
+#define DPP_REG_LIST_DCN42_COMMON_RI(id)                                        \
+	SRI_ARR(CM_DEALPHA, CM, id), SRI_ARR(CM_MEM_PWR_STATUS, CM, id),            \
+		SRI_ARR(CM_BIAS_CR_R, CM, id), SRI_ARR(CM_BIAS_Y_G_CB_B, CM, id),       \
+		SRI_ARR(PRE_DEGAM, CNVC_CFG, id), SRI_ARR(CM_GAMCOR_CONTROL, CM, id),   \
+		SRI_ARR(CM_GAMCOR_LUT_CONTROL, CM, id),                                 \
+		SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                   \
+		SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                   \
+		SRI_ARR(CM_GAMCOR_LUT_DATA, CM, id),                                    \
+		SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_B, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_G, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_R, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_B, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_G, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_R, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_B, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_B, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_G, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_G, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_R, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_R, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMB_REGION_0_1, CM, id),                             \
+		SRI_ARR(CM_GAMCOR_RAMB_REGION_32_33, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMB_OFFSET_B, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMB_OFFSET_G, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMB_OFFSET_R, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_B, CM, id),                      \
+		SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_G, CM, id),                      \
+		SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_R, CM, id),                      \
+		SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_B, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_G, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_R, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_B, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_G, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_R, CM, id),                     \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_B, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_B, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_G, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_G, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_R, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_R, CM, id),                            \
+		SRI_ARR(CM_GAMCOR_RAMA_REGION_0_1, CM, id),                             \
+		SRI_ARR(CM_GAMCOR_RAMA_REGION_32_33, CM, id),                           \
+		SRI_ARR(CM_GAMCOR_RAMA_OFFSET_B, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMA_OFFSET_G, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMA_OFFSET_R, CM, id),                               \
+		SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_B, CM, id),                      \
+		SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_G, CM, id),                      \
+		SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_R, CM, id),                      \
+		SRI_ARR(CM_HIST_CNTL, CM, id),                                          \
+		SRI_ARR(CM_HIST_LOCK, CM, id),                                          \
+		SRI_ARR(CM_HIST_INDEX, CM, id),                                         \
+		SRI_ARR(CM_HIST_DATA, CM, id),                                          \
+		SRI_ARR(CM_HIST_STATUS, CM, id),                                        \
+		SRI_ARR(CM_HIST_SCALE_SRC1, CM, id),                                    \
+		SRI_ARR(CM_HIST_COEFA_SRC2, CM, id),                                    \
+		SRI_ARR(CM_HIST_COEFB_SRC2, CM, id),                                    \
+		SRI_ARR(CM_HIST_COEFC_SRC2, CM, id),                                    \
+		SRI_ARR(CM_HIST_SCALE_SRC3, CM, id),                                    \
+		SRI_ARR(CM_HIST_BIAS_SRC1, CM, id),                                     \
+		SRI_ARR(CM_HIST_BIAS_SRC2, CM, id),                                     \
+		SRI_ARR(CM_HIST_BIAS_SRC3, CM, id),                                     \
+		SRI_ARR(DSCL_EXT_OVERSCAN_LEFT_RIGHT, DSCL, id),                        \
+		SRI_ARR(DSCL_EXT_OVERSCAN_TOP_BOTTOM, DSCL, id),                        \
+		SRI_ARR(OTG_H_BLANK, DSCL, id), SRI_ARR(OTG_V_BLANK, DSCL, id),         \
+		SRI_ARR(SCL_MODE, DSCL, id), SRI_ARR(LB_DATA_FORMAT, DSCL, id),         \
+		SRI_ARR(LB_MEMORY_CTRL, DSCL, id), SRI_ARR(DSCL_AUTOCAL, DSCL, id),     \
+		SRI_ARR(SCL_TAP_CONTROL, DSCL, id),                                     \
+		SRI_ARR(SCL_COEF_RAM_TAP_SELECT, DSCL, id),                             \
+		SRI_ARR(SCL_COEF_RAM_TAP_DATA, DSCL, id),                               \
+		SRI_ARR(DSCL_2TAP_CONTROL, DSCL, id), SRI_ARR(MPC_SIZE, DSCL, id),      \
+		SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO, DSCL, id),                         \
+		SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO, DSCL, id),                         \
+		SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO_C, DSCL, id),                       \
+		SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO_C, DSCL, id),                       \
+		SRI_ARR(SCL_HORZ_FILTER_INIT, DSCL, id),                                \
+		SRI_ARR(SCL_HORZ_FILTER_INIT_C, DSCL, id),                              \
+		SRI_ARR(SCL_VERT_FILTER_INIT, DSCL, id),                                \
+		SRI_ARR(SCL_VERT_FILTER_INIT_C, DSCL, id),                              \
+		SRI_ARR(RECOUT_START, DSCL, id), SRI_ARR(RECOUT_SIZE, DSCL, id),        \
+		SRI_ARR(PRE_DEALPHA, CNVC_CFG, id), SRI_ARR(PRE_REALPHA, CNVC_CFG, id), \
+		SRI_ARR(PRE_CSC_MODE, CNVC_CFG, id),                                    \
+		SRI_ARR(PRE_CSC_C11_C12, CNVC_CFG, id),                                 \
+		SRI_ARR(PRE_CSC_C33_C34, CNVC_CFG, id),                                 \
+		SRI_ARR(PRE_CSC_B_C11_C12, CNVC_CFG, id),                               \
+		SRI_ARR(PRE_CSC_B_C33_C34, CNVC_CFG, id),                               \
+		SRI_ARR(CM_POST_CSC_CONTROL, CM, id),                                   \
+		SRI_ARR(CM_POST_CSC_C11_C12, CM, id),                                   \
+		SRI_ARR(CM_POST_CSC_C33_C34, CM, id),                                   \
+		SRI_ARR(CM_POST_CSC_B_C11_C12, CM, id),                                 \
+		SRI_ARR(CM_POST_CSC_B_C33_C34, CM, id),                                 \
+		SRI_ARR(CM_MEM_PWR_CTRL, CM, id), SRI_ARR(CM_CONTROL, CM, id),          \
+		SRI_ARR(CM_TEST_DEBUG_INDEX, CM, id),                                   \
+		SRI_ARR(CM_TEST_DEBUG_DATA, CM, id),                                    \
+		SRI_ARR(FORMAT_CONTROL, CNVC_CFG, id),                                  \
+		SRI_ARR(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id),                       \
+		SRI_ARR(CURSOR0_CONTROL, CM_CUR, id),                                   \
+		SRI_ARR(CURSOR0_COLOR0, CM_CUR, id),                                    \
+		SRI_ARR(CURSOR0_COLOR1, CM_CUR, id),                                    \
+		SRI_ARR(CURSOR0_FP_SCALE_BIAS_G_Y, CM_CUR, id),                         \
+		SRI_ARR(CURSOR0_FP_SCALE_BIAS_RB_CRCB, CM_CUR, id),                     \
+		SRI_ARR(CUR0_MATRIX_MODE, CM_CUR, id),                                  \
+		SRI_ARR(CUR0_MATRIX_C11_C12_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C13_C14_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C21_C22_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C23_C24_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C31_C32_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C33_C34_A, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C11_C12_B, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C13_C14_B, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C21_C22_B, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C23_C24_B, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C31_C32_B, CM_CUR, id),                             \
+		SRI_ARR(CUR0_MATRIX_C33_C34_B, CM_CUR, id),                             \
+		SRI_ARR(DPP_CONTROL, DPP_TOP, id), SRI_ARR(CM_HDR_MULT_COEF, CM, id),   \
+		SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                  \
+		SRI_ARR(ALPHA_2BIT_LUT01, CNVC_CFG, id),                                \
+		SRI_ARR(ALPHA_2BIT_LUT23, CNVC_CFG, id),                                \
+		SRI_ARR(FCNV_FP_BIAS_R, CNVC_CFG, id),                                  \
+		SRI_ARR(FCNV_FP_BIAS_G, CNVC_CFG, id),                                  \
+		SRI_ARR(FCNV_FP_BIAS_B, CNVC_CFG, id),                                  \
+		SRI_ARR(FCNV_FP_SCALE_R, CNVC_CFG, id),                                 \
+		SRI_ARR(FCNV_FP_SCALE_G, CNVC_CFG, id),                                 \
+		SRI_ARR(FCNV_FP_SCALE_B, CNVC_CFG, id),                                 \
+		SRI_ARR(COLOR_KEYER_CONTROL, CNVC_CFG, id),                             \
+		SRI_ARR(COLOR_KEYER_ALPHA, CNVC_CFG, id),                               \
+		SRI_ARR(COLOR_KEYER_RED, CNVC_CFG, id),                                 \
+		SRI_ARR(COLOR_KEYER_GREEN, CNVC_CFG, id),                               \
+		SRI_ARR(COLOR_KEYER_BLUE, CNVC_CFG, id),                                \
+		SRI_ARR(OBUF_MEM_PWR_CTRL, DSCL, id),                                   \
+		SRI_ARR(DSCL_MEM_PWR_STATUS, DSCL, id),                                 \
+		SRI_ARR(DSCL_MEM_PWR_CTRL, DSCL, id),                                   \
+		SRI_ARR(DSCL_CONTROL, DSCL, id),                                        \
+		SRI_ARR(DSCL_SC_MODE, DSCL, id),                                        \
+		SRI_ARR(DSCL_EASF_H_MODE, DSCL, id),                                    \
+		SRI_ARR(DSCL_EASF_H_BF_CNTL, DSCL, id),                                 \
+		SRI_ARR(DSCL_EASF_H_RINGEST_EVENTAP_REDUCE, DSCL, id),                  \
+		SRI_ARR(DSCL_EASF_H_RINGEST_EVENTAP_GAIN, DSCL, id),                    \
+		SRI_ARR(DSCL_EASF_H_BF_FINAL_MAX_MIN, DSCL, id),                        \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG0, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG1, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG2, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG3, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG4, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG5, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG6, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF1_PWL_SEG7, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG0, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG1, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG2, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG3, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG4, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_H_BF3_PWL_SEG5, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_MODE, DSCL, id),                                    \
+		SRI_ARR(DSCL_EASF_V_BF_CNTL, DSCL, id),                                 \
+		SRI_ARR(DSCL_EASF_V_RINGEST_3TAP_CNTL1, DSCL, id),                      \
+		SRI_ARR(DSCL_EASF_V_RINGEST_3TAP_CNTL2, DSCL, id),                      \
+		SRI_ARR(DSCL_EASF_V_RINGEST_3TAP_CNTL3, DSCL, id),                      \
+		SRI_ARR(DSCL_EASF_V_RINGEST_EVENTAP_REDUCE, DSCL, id),                  \
+		SRI_ARR(DSCL_EASF_V_RINGEST_EVENTAP_GAIN, DSCL, id),                    \
+		SRI_ARR(DSCL_EASF_V_BF_FINAL_MAX_MIN, DSCL, id),                        \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG0, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG1, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG2, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG3, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG4, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG5, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG6, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF1_PWL_SEG7, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG0, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG1, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG2, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG3, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG4, DSCL, id),                            \
+		SRI_ARR(DSCL_EASF_V_BF3_PWL_SEG5, DSCL, id),                            \
+		SRI_ARR(DSCL_SC_MATRIX_C0C1, DSCL, id),                                 \
+		SRI_ARR(DSCL_SC_MATRIX_C2C3, DSCL, id),                                 \
+		SRI_ARR(ISHARP_MODE, DSCL, id),                                         \
+		SRI_ARR(ISHARP_DELTA_LUT_MEM_PWR_CTRL, DSCL, id),                       \
+		SRI_ARR(ISHARP_NOISEDET_THRESHOLD, DSCL, id),                           \
+		SRI_ARR(ISHARP_NOISE_GAIN_PWL, DSCL, id),                               \
+		SRI_ARR(ISHARP_LBA_PWL_SEG0, DSCL, id),                                 \
+		SRI_ARR(ISHARP_LBA_PWL_SEG1, DSCL, id),                                 \
+		SRI_ARR(ISHARP_LBA_PWL_SEG2, DSCL, id),                                 \
+		SRI_ARR(ISHARP_LBA_PWL_SEG3, DSCL, id),                                 \
+		SRI_ARR(ISHARP_LBA_PWL_SEG4, DSCL, id),                                 \
+		SRI_ARR(ISHARP_LBA_PWL_SEG5, DSCL, id),                                 \
+		SRI_ARR(ISHARP_DELTA_CTRL, DSCL, id),                                   \
+		SRI_ARR(ISHARP_DELTA_DATA, DSCL, id),                                   \
+		SRI_ARR(ISHARP_DELTA_INDEX, DSCL, id),                                  \
+		SRI_ARR(ISHARP_NLDELTA_SOFT_CLIP, DSCL, id),                            \
+		SRI_ARR(SCL_VERT_FILTER_INIT_BOT, DSCL, id),                            \
+		SRI_ARR(SCL_VERT_FILTER_INIT_BOT_C, DSCL, id)
+
+/* Stream encoder */
+#define SE_DCN42_REG_LIST_RI(id)                                                \
+	SRI_ARR(HDMI_CONTROL, DIG, id), SRI_ARR(HDMI_DB_CONTROL, DIG, id),          \
+		SRI_ARR(HDMI_GC, DIG, id),                                              \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id),                         \
+		SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id),                        \
+		SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id),                              \
+		SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id),                              \
+		SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),                            \
+		SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),                              \
+		SRI_ARR(HDMI_ACR_32_0, DIG, id), SRI_ARR(HDMI_ACR_32_1, DIG, id),       \
+		SRI_ARR(HDMI_ACR_44_0, DIG, id), SRI_ARR(HDMI_ACR_44_1, DIG, id),       \
+		SRI_ARR(HDMI_ACR_48_0, DIG, id), SRI_ARR(HDMI_ACR_48_1, DIG, id),       \
+		SRI_ARR(DP_DB_CNTL, DP, id), SRI_ARR(DP_MSA_MISC, DP, id),              \
+		SRI_ARR(DP_MSA_VBID_MISC, DP, id), SRI_ARR(DP_MSA_COLORIMETRY, DP, id), \
+		SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id),                                  \
+		SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id),                                  \
+		SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id),                                  \
+		SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id),                                  \
+		SRI_ARR(DP_MSE_RATE_CNTL, DP, id), SRI_ARR(DP_MSE_RATE_UPDATE, DP, id), \
+		SRI_ARR(DP_PIXEL_FORMAT, DP, id), SRI_ARR(DP_SEC_CNTL, DP, id),         \
+		SRI_ARR(DP_SEC_CNTL1, DP, id), SRI_ARR(DP_SEC_CNTL2, DP, id),           \
+		SRI_ARR(DP_SEC_CNTL5, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id),           \
+		SRI_ARR(DP_STEER_FIFO, DP, id), SRI_ARR(DP_VID_M, DP, id),              \
+		SRI_ARR(DP_VID_N, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),         \
+		SRI_ARR(DP_VID_TIMING, DP, id), SRI_ARR(DP_SEC_AUD_N, DP, id),          \
+		SRI_ARR(DP_SEC_TIMESTAMP, DP, id),                                      \
+		SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                          \
+		SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                         \
+		SRI_ARR(DP_SEC_FRAMING4, DP, id), SRI_ARR(DP_GSP11_CNTL, DP, id),       \
+		SRI_ARR(DME_CONTROL, DME, id),                                          \
+		SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                          \
+		SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                         \
+		SRI_ARR(DIG_FE_CNTL, DIG, id),                                          \
+		SRI_ARR(DIG_FE_EN_CNTL, DIG, id),                                       \
+		SRI_ARR(DIG_FE_CLK_CNTL, DIG, id),                                      \
+		SRI_ARR(DIG_CLOCK_PATTERN, DIG, id),                                    \
+		SRI_ARR(DIG_FIFO_CTRL0, DIG, id),                                       \
+		SRI_ARR(STREAM_MAPPER_CONTROL, DIG, id),\
+		SRI_ARR(DIG_FE_AUDIO_CNTL, DIG, id)
+
+/* HPO DP stream encoder */
+#define DCN42_HPO_DP_STREAM_ENC_REG_LIST_RI(id)                                             \
+	SR_ARR(DP_STREAM_MAPPER_CONTROL0, id),                                                  \
+		SR_ARR(DP_STREAM_MAPPER_CONTROL1, id),                                              \
+		SR_ARR(DP_STREAM_MAPPER_CONTROL2, id),                                              \
+		SR_ARR(DP_STREAM_MAPPER_CONTROL3, id),                                              \
+		SRI_ARR(DP_STREAM_ENC_CLOCK_CONTROL, DP_STREAM_ENC, id),                            \
+		SRI_ARR(DP_STREAM_ENC_INPUT_MUX_CONTROL, DP_STREAM_ENC, id),                        \
+		SRI_ARR(DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC, id),                            \
+		SRI_ARR(DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0, DP_STREAM_ENC, id), \
+		SRI_ARR(DP_SYM32_ENC_CONTROL, DP_SYM32_ENC, id),                                    \
+		SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),     \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA0, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA1, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA2, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA3, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA4, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA5, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA6, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA7, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA8, DP_SYM32_ENC, id),                                   \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA_CONTROL, DP_SYM32_ENC, id),                            \
+		SRI_ARR(DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),              \
+		SRI_ARR(DP_SYM32_ENC_VID_FIFO_CONTROL, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_VID_STREAM_CONTROL, DP_SYM32_ENC, id),                         \
+		SRI_ARR(DP_SYM32_ENC_VID_VBID_CONTROL, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_SDP_CONTROL, DP_SYM32_ENC, id),                                \
+		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL0, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL2, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL3, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL5, DP_SYM32_ENC, id),                           \
+		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL11, DP_SYM32_ENC, id),                          \
+		SRI_ARR(DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL, DP_SYM32_ENC, id),                \
+		SRI_ARR(DP_SYM32_ENC_SDP_AUDIO_CONTROL0, DP_SYM32_ENC, id),                         \
+		SRI_ARR(DP_SYM32_ENC_VID_CRC_CONTROL, DP_SYM32_ENC, id),                            \
+		SRI_ARR(DP_SYM32_ENC_HBLANK_CONTROL, DP_SYM32_ENC, id)
+
+/*HPO DP link encoder regs */
+#define DCN42_HPO_DP_LINK_ENC_REG_LIST_RI(id)                    \
+	SRI_ARR(DP_LINK_ENC_CLOCK_CONTROL, DP_LINK_ENC, id),         \
+		SRI_ARR(DP_DPHY_SYM32_CONTROL, DP_DPHY_SYM32, id),       \
+		SRI_ARR(DP_DPHY_SYM32_STATUS, DP_DPHY_SYM32, id),        \
+		SRI_ARR(DP_DPHY_SYM32_TP_CONFIG, DP_DPHY_SYM32, id),     \
+		SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED0, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED1, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED2, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED3, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_TP_SQ_PULSE, DP_DPHY_SYM32, id),   \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM0, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM1, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM2, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM3, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM4, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM5, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM6, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM7, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM8, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM9, DP_DPHY_SYM32, id),    \
+		SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM10, DP_DPHY_SYM32, id),   \
+		SRI_ARR(DP_DPHY_SYM32_SAT_VC0, DP_DPHY_SYM32, id),       \
+		SRI_ARR(DP_DPHY_SYM32_SAT_VC1, DP_DPHY_SYM32, id),       \
+		SRI_ARR(DP_DPHY_SYM32_SAT_VC2, DP_DPHY_SYM32, id),       \
+		SRI_ARR(DP_DPHY_SYM32_SAT_VC3, DP_DPHY_SYM32, id),       \
+		SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL0, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL1, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL2, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL3, DP_DPHY_SYM32, id), \
+		SRI_ARR(DP_DPHY_SYM32_SAT_UPDATE, DP_DPHY_SYM32, id)
+
+#define VPG_DCN42_REG_LIST_RI(id) \
+	SRI(VPG_GENERIC_STATUS, VPG, id), \
+	SRI(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id), \
+	SRI(VPG_GENERIC_PACKET_DATA, VPG, id), \
+	SRI(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id), \
+	SRI(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id), \
+	SRI(VPG_MEM_PWR, VPG, id)
+
+/* DCCG */
+#define DCCG_REG_LIST_DCN42_RI()               \
+	SR(DPPCLK_DTO_CTRL),                       \
+	DCCG_SRII(DTO_PARAM, DPPCLK, 0),       \
+	DCCG_SRII(DTO_PARAM, DPPCLK, 1),       \
+	DCCG_SRII(DTO_PARAM, DPPCLK, 2),       \
+	DCCG_SRII(DTO_PARAM, DPPCLK, 3),       \
+	DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0), \
+	SR(HDMISTREAMCLK0_DTO_PARAM),\
+	SR(DCCG_GLOBAL_FGCG_REP_CNTL),\
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(PHYASYMCLK_CLOCK_CNTL),  \
+	SR(PHYBSYMCLK_CLOCK_CNTL), \
+	SR(PHYCSYMCLK_CLOCK_CNTL), \
+	SR(PHYDSYMCLK_CLOCK_CNTL), \
+	SR(PHYESYMCLK_CLOCK_CNTL), \
+	SR(DPSTREAMCLK_CNTL), \
+	SR(HDMISTREAMCLK_CNTL), \
+	SR(SYMCLK32_SE_CNTL), \
+	SR(SYMCLK32_LE_CNTL), \
+	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 1), \
+	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2), \
+	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3), \
+	SR(OTG_PIXEL_RATE_DIV), \
+	SR(DTBCLK_P_CNTL), \
+	SR(DCCG_AUDIO_DTO_SOURCE), \
+	SR(DENTIST_DISPCLK_CNTL), \
+	SR(DPPCLK_CTRL), \
+	DCCG_SRII(MODULO, DP_DTO, 0), \
+	DCCG_SRII(MODULO, DP_DTO, 1), \
+	DCCG_SRII(MODULO, DP_DTO, 2), \
+	DCCG_SRII(MODULO, DP_DTO, 3), \
+	DCCG_SRII(PHASE, DP_DTO, 0), \
+	DCCG_SRII(PHASE, DP_DTO, 1), \
+	DCCG_SRII(PHASE, DP_DTO, 2), \
+	DCCG_SRII(PHASE, DP_DTO, 3), \
+	SR(OTG_ADD_DROP_PIXEL_CNTL), \
+	SR(DSCCLK0_DTO_PARAM), \
+	SR(DSCCLK1_DTO_PARAM), \
+	SR(DSCCLK2_DTO_PARAM), \
+	SR(DSCCLK3_DTO_PARAM), \
+	SR(DSCCLK_DTO_CTRL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCCG_GATE_DISABLE_CNTL3), \
+	SR(DCCG_GATE_DISABLE_CNTL4), \
+	SR(DCCG_GATE_DISABLE_CNTL5), \
+	SR(DCCG_GATE_DISABLE_CNTL6), \
+	SR(SYMCLKA_CLOCK_ENABLE), \
+	SR(SYMCLKB_CLOCK_ENABLE), \
+	SR(SYMCLKC_CLOCK_ENABLE), \
+	SR(SYMCLKD_CLOCK_ENABLE), \
+	SR(SYMCLKE_CLOCK_ENABLE)
+
+#define DCN42_AUD_COMMON_MASK_SH_LIST(mask_sh)                                             \
+	SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),   \
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_USE_512FBR_DTO, mask_sh),\
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO1_USE_512FBR_DTO, mask_sh),\
+		SF(DCCG_AUDIO_DTO0_MODULE, DCCG_AUDIO_DTO0_MODULE, mask_sh),\
+		SF(DCCG_AUDIO_DTO0_PHASE, DCCG_AUDIO_DTO0_PHASE, mask_sh),\
+		SF(DCCG_AUDIO_DTO1_MODULE, DCCG_AUDIO_DTO1_MODULE, mask_sh),\
+		SF(DCCG_AUDIO_DTO1_PHASE, DCCG_AUDIO_DTO1_PHASE, mask_sh),\
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, AUDIO_RATE_CAPABILITIES, mask_sh),\
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, CLKSTOP, mask_sh),\
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, EPSS, mask_sh)
+
+	/* HPO FRL stream encoder */
+#define DCN42_HPO_FRL_STREAM_ENC_REG_LIST_RI(id)                                 \
+	DCN3_0_HPO_FRL_STREAM_ENC_REG_LIST_RI(id),                               \
+	SR_ARR(HDMI_STREAM_ENC_AUDIO_CONTROL, id),\
+	SR_ARR(HDMI_TB_ENC_MEM_CTRL, id),\
+	SR_ARR(HDMI_FRL_ENC_MEM_CTRL, id)
+/* OPTC */
+#define OPTC_COMMON_REG_LIST_DCN42_RI(inst)                                      \
+		SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),                                      \
+		SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),                                   \
+		SRI_ARR(OTG_VREADY_PARAM, OTG, inst),                                    \
+		SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),                              \
+		SRI_ARR(OTG_MASTER_UPDATE_MODE, OTG, inst),                              \
+		SRI_ARR(OTG_V_COUNT_STOP_CONTROL, OTG, inst),                            \
+		SRI_ARR(OTG_V_COUNT_STOP_CONTROL2, OTG, inst),                           \
+		SRI_ARR(OTG_GSL_CONTROL, OTG, inst),					 \
+		SRI2_ARR(OPTC_CLOCK_CONTROL, OPTC, inst),\
+		SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),                                 \
+		SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+		SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+		SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),                                 \
+		SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),                           \
+		SRI_ARR(OTG_H_TOTAL, OTG, inst),                                         \
+		SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),                               \
+		SRI_ARR(OTG_H_SYNC_A, OTG, inst),\
+		SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst), \
+		SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst), \
+		SRI_ARR(OTG_V_TOTAL, OTG, inst),  \
+		SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),                               \
+		SRI_ARR(OTG_V_SYNC_A, OTG, inst), \
+		SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \
+		SRI_ARR(OTG_CONTROL, OTG, inst), \
+		SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \
+		SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),                            \
+		SRI_ARR(OTG_STEREO_STATUS, OTG, inst),                                   \
+		SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),                                     \
+		SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),                                     \
+		SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),                                 \
+		SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),                                      \
+		SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),                            \
+		SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),                           \
+		SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),                              \
+		SRI_ARR(OTG_STATUS, OTG, inst), 										\
+		SRI_ARR(OTG_STATUS_POSITION, OTG, inst), \
+		SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),                               \
+		SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),                                    \
+		SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),                                    \
+		SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),                                   \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),                     \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),                    \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),                     \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),                    \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),                     \
+		SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),                    \
+		SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),                            \
+		SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),                             \
+		SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),                           \
+		SRI_ARR(CONTROL, VTG, inst), \
+		SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),  \
+		SRI_ARR(OTG_GSL_CONTROL, OTG, inst), \
+		SRI_ARR(OTG_CRC_CNTL, OTG, inst),   \
+		SRI_ARR(OTG_CRC0_DATA_R, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC0_DATA_G, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC1_DATA_R, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC1_DATA_G, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC1_DATA_B, OTG, inst),                                     \
+		SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL, OTG, inst),                          \
+		SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
+		SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
+		SR_ARR(GSL_SOURCE_SELECT, inst),                                         \
+		SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),                               \
+		SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+		SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+		SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),                                    \
+		SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),                                    \
+		SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),                                 \
+		SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),                              \
+		SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),                              \
+		SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),                            \
+		SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),                                \
+		SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),                                  \
+		SRI_ARR(OPTC_WIDTH_CONTROL2, ODM, inst),                                 \
+		SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),                                  \
+		SRI_ARR(OTG_DRR_CONTROL, OTG, inst),                                     \
+		SRI_ARR(OTG_PSTATE_REGISTER, OTG, inst),                                 \
+		SRI_ARR(OTG_PWA_FRAME_SYNC_CONTROL, OTG, inst),\
+		SRI_ARR(OTG_PIPE_UPDATE_STATUS, OTG, inst),\
+		SRI_ARR(INTERRUPT_DEST, OTG, inst)
+
+/* CLK SRC */
+#define CS_COMMON_REG_LIST_DCN42_RI(index, pllid)               \
+	SRI_ARR_ALPHABET(PIXCLK_RESYNC_CNTL, PHYPLL, index, pllid), \
+		SRII_ARR_2(PHASE, DP_DTO, 0, index),                    \
+		SRII_ARR_2(PHASE, DP_DTO, 1, index),                    \
+		SRII_ARR_2(PHASE, DP_DTO, 2, index),                    \
+		SRII_ARR_2(PHASE, DP_DTO, 3, index),                    \
+		SRII_ARR_2(MODULO, DP_DTO, 0, index),                   \
+		SRII_ARR_2(MODULO, DP_DTO, 1, index),                   \
+		SRII_ARR_2(MODULO, DP_DTO, 2, index),                   \
+		SRII_ARR_2(MODULO, DP_DTO, 3, index),                   \
+		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 0, index),             \
+		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 1, index),             \
+		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index),             \
+		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 3, index)
+
+/* ABM */
+#define ABM_DCN42_REG_LIST_RI(id)                               \
+	SRI_ARR(DC_ABM1_HG_SAMPLE_RATE, ABM, id),                   \
+		SRI_ARR(DC_ABM1_LS_SAMPLE_RATE, ABM, id),               \
+		SRI_ARR(DC_ABM1_HG_MISC_CTRL, ABM, id),                 \
+		SRI_ARR(DC_ABM1_IPCSC_COEFF_SEL, ABM, id),              \
+		SRI_ARR(BL1_PWM_BL_UPDATE_SAMPLE_RATE, ABM, id),        \
+		SRI_ARR(BL1_PWM_CURRENT_ABM_LEVEL, ABM, id),            \
+		SRI_ARR(BL1_PWM_TARGET_ABM_LEVEL, ABM, id),             \
+		SRI_ARR(BL1_PWM_USER_LEVEL, ABM, id),                   \
+		SRI_ARR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id), \
+		SRI_ARR(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id),       \
+		SRI_ARR(DC_ABM1_HG_BIN_33_40_SHIFT_INDEX, ABM, id),     \
+		SRI_ARR(DC_ABM1_HG_BIN_33_64_SHIFT_FLAG, ABM, id),      \
+		SRI_ARR(DC_ABM1_HG_BIN_41_48_SHIFT_INDEX, ABM, id),     \
+		SRI_ARR(DC_ABM1_HG_BIN_49_56_SHIFT_INDEX, ABM, id),     \
+		SRI_ARR(DC_ABM1_HG_BIN_57_64_SHIFT_INDEX, ABM, id),     \
+		SRI_ARR(DC_ABM1_HG_RESULT_DATA, ABM, id),               \
+		SRI_ARR(DC_ABM1_HG_RESULT_INDEX, ABM, id),              \
+		SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_DATA, ABM, id),        \
+		SRI_ARR(DC_ABM1_ACE_PWL_CNTL, ABM, id)
+
+/* HUBP */
+#define HUBP_REG_LIST_DCN42_RI(id)                                     \
+	HUBP_REG_LIST_DCN30_RI(id), SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id), \
+		SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                         \
+		SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id),                       \
+		SRI_ARR(HUBP_3DLUT_DLG_PARAM, CURSOR0_, id),                   \
+		HUBP_3DLUT_FL_REG_LIST_DCN401(id)
+struct dcn42_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn42_create_resource_pool(
+	const struct dc_init_data *init_data,
+	struct dc *dc);
+
+enum dc_status dcn42_validate_bandwidth(struct dc *dc,
+							  struct dc_state *context,
+							  enum dc_validate_mode validate_mode);
+
+void dcn42_prepare_mcache_programming(struct dc *dc, struct dc_state *context);
+
+#endif /* _DCN42_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
new file mode 100644
index 0000000000000..33b9775420d3d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+
+
+#include "dc.h"
+#include "dcn42_resource_fpu.h"
+
+#define DC_LOGGER_INIT(logger)
+
+
+void dcn42_decide_zstate_support(struct dc *dc, struct dc_state *context)
+{
+	enum dcn_zstate_support_state support = DCN_ZSTATE_SUPPORT_DISALLOW;
+	unsigned int i, plane_count = 0;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	dc_assert_fp_enabled();
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (context->res_ctx.pipe_ctx[i].plane_state)
+			plane_count++;
+	}
+	/*dcn42 has no z10*/
+	if (context->stream_count == 0 || plane_count == 0) {
+		support = DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY;
+	} else if (context->stream_count == 1 && context->streams[0]->signal == SIGNAL_TYPE_EDP) {
+		struct dc_link *link = context->streams[0]->sink->link;
+		bool is_psr = (link && (link->psr_settings.psr_version == DC_PSR_VERSION_1 ||
+								link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) && !link->panel_config.psr.disable_psr);
+		bool is_replay = link && link->replay_settings.replay_feature_enabled;
+
+		if (is_psr || is_replay)
+			support = DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY;
+		else {
+			/*here we allow z8 for eDP based on dml21 output*/
+			support = context->bw_ctx.bw.dcn.clk.zstate_support ? DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY : DCN_ZSTATE_SUPPORT_DISALLOW;
+		}
+
+		DC_LOG_SMU("zstate_support: %d, StutterPeriod: %d\n, z8_stutter_efficiency: %d\n",
+			 support, (int)context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_period,
+			 (int)context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_efficiency);
+	}
+	context->bw_ctx.bw.dcn.clk.zstate_support = support;
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h
new file mode 100644
index 0000000000000..e321032205073
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+
+#ifndef _DCN42_RESOURCE_FPU_H_
+#define _DCN42_RESOURCE_FPU_H_
+
+#include "core_types.h"
+
+void dcn42_decide_zstate_support(struct dc *dc, struct dc_state *context);
+
+#endif /* _DCN42_RESOURCE_FPU_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
index bc93356a0b5b9..d168fb1eacf76 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
@@ -8,11 +8,14 @@ soc_and_ip_translator_ccflags := $(CC_FLAGS_FPU)
 soc_and_ip_translator_rcflags := $(CC_FLAGS_NO_FPU)
 
 CFLAGS_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.o := $(soc_and_ip_translator_ccflags)
+CFLAGS_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.o := $(soc_and_ip_translator_ccflags)
 
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.o := $(soc_and_ip_translator_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.o := $(soc_and_ip_translator_rcflags)
 
 soc_and_ip_translator := soc_and_ip_translator.o
 soc_and_ip_translator += dcn401/dcn401_soc_and_ip_translator.o
+soc_and_ip_translator += dcn42/dcn42_soc_and_ip_translator.o
 
 AMD_DAL_soc_and_ip_translator := $(addprefix $(AMDDALPATH)/dc/soc_and_ip_translator/, $(soc_and_ip_translator))
 
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
index 3190c76eb4820..1b397fa7e05c9 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
@@ -102,6 +102,9 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 					}
 				} else {
 					dml_clk_table->uclk.clk_values_khz[i] = dc_clk_table->entries[i].memclk_mhz * 1000;
+					#ifdef ENABLE_WCK
+					dml_clk_table->wck_ratio.clk_values_khz[i] = dc_clk_table->entries[i].wck_ratio;
+					#endif
 				}
 			} else {
 				dml_clk_table->uclk.clk_values_khz[i] = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
index c9e224d262c90..fd9c24b5df535 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
@@ -3,7 +3,7 @@
 // Copyright 2025 Advanced Micro Devices, Inc.
 
 #include "dcn42_soc_and_ip_translator.h"
-#include "soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.h"
+#include "../dcn401/dcn401_soc_and_ip_translator.h"
 #include "bounding_boxes/dcn42_soc_bb.h"
 
 /* soc_and_ip_translator component used to get up-to-date values for bounding box.
@@ -11,13 +11,171 @@
  * This component provides an interface to get DCN-specific bounding box values.
  */
 
+static void get_default_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc)
+{
+	{
+		memcpy(soc_bb, &dml2_socbb_dcn42, sizeof(struct dml2_soc_bb));
+		memcpy(&soc_bb->qos_parameters, &dml_dcn42_variant_a_soc_qos_params, sizeof(struct dml2_soc_qos_parameters));
+	}
+}
+
+/*
+ * DC clock table is obtained from SMU during runtime.
+ * SMU stands for System Management Unit. It is a power management processor.
+ * It owns the initialization of dc's clock table and programming of clock values
+ * based on dc's requests.
+ * Our clock values in base soc bb is a dummy placeholder. The real clock values
+ * are retrieved from SMU firmware to dc clock table at runtime.
+ * This function overrides our dummy placeholder values with real values in dc
+ * clock table.
+ */
+static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
+		struct dml2_soc_state_table *dml_clk_table,
+		struct dml2_soc_vmin_clock_limits *vmin_limit,
+		const struct clk_bw_params *dc_bw_params)
+{
+	int i;
+	const struct clk_limit_table *dc_clk_table;
+
+	if (dc_bw_params == NULL)
+		/* skip if bw params could not be obtained from smu */
+		return;
+
+	dc_clk_table = &dc_bw_params->clk_table;
+
+	/* fclk/dcfclk - dcn42 pmfw table can have 0 entries for inactive dpm levels
+	 * for use with dml we need to fill in using an active value aiming for >= 2x DCFCLK
+	 */
+	if (dc_clk_table->num_entries_per_clk.num_fclk_levels && dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
+		dml_clk_table->fclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
+		dml_clk_table->dcfclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
+				int j, max_fclk = 0;
+
+				dml_clk_table->dcfclk.clk_values_khz[i] = dc_clk_table->entries[i].dcfclk_mhz * 1000;
+				for (j = 0; j < MAX_NUM_DPM_LVL; j++) {
+					if (dc_clk_table->entries[j].fclk_mhz * 1000 > max_fclk)
+						max_fclk = dc_clk_table->entries[j].fclk_mhz * 1000;
+					dml_clk_table->fclk.clk_values_khz[i] = max_fclk;
+					if (max_fclk >= 2 * dml_clk_table->dcfclk.clk_values_khz[i])
+						break;
+				}
+			} else {
+				dml_clk_table->dcfclk.clk_values_khz[i] = 0;
+				dml_clk_table->fclk.clk_values_khz[i] = 0;
+			}
+		}
+	}
+
+	/* uclk */
+	if (dc_clk_table->num_entries_per_clk.num_memclk_levels) {
+		dml_clk_table->uclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_memclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dml_clk_table->uclk.num_clk_values) {
+				dml_clk_table->uclk.clk_values_khz[i] = dc_clk_table->entries[i].memclk_mhz * 1000;
+				dml_clk_table->wck_ratio.clk_values_khz[i] = dc_clk_table->entries[i].wck_ratio;
+			} else {
+				dml_clk_table->uclk.clk_values_khz[i] = 0;
+				dml_clk_table->wck_ratio.clk_values_khz[i] = 0;
+			}
+		}
+	}
+
+	/* dispclk */
+	if (dc_clk_table->num_entries_per_clk.num_dispclk_levels) {
+		dml_clk_table->dispclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dispclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dml_clk_table->dispclk.num_clk_values) {
+				dml_clk_table->dispclk.clk_values_khz[i] = dc_clk_table->entries[i].dispclk_mhz * 1000;
+			} else {
+				dml_clk_table->dispclk.clk_values_khz[i] = 0;
+			}
+		}
+		vmin_limit->dispclk_khz = min(dc_clk_table->entries[0].dispclk_mhz * 1000, vmin_limit->dispclk_khz);
+	}
+
+	/* dppclk */
+	if (dc_clk_table->num_entries_per_clk.num_dppclk_levels) {
+		dml_clk_table->dppclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dppclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dml_clk_table->dppclk.num_clk_values) {
+				dml_clk_table->dppclk.clk_values_khz[i] = dc_clk_table->entries[i].dppclk_mhz * 1000;
+			} else {
+				dml_clk_table->dppclk.clk_values_khz[i] = 0;
+			}
+		}
+	}
+
+	/* dtbclk */
+	if (dc_clk_table->num_entries_per_clk.num_dtbclk_levels) {
+		dml_clk_table->dtbclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dtbclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dml_clk_table->dtbclk.num_clk_values) {
+				dml_clk_table->dtbclk.clk_values_khz[i] = dc_clk_table->entries[i].dtbclk_mhz * 1000;
+			} else {
+				dml_clk_table->dtbclk.clk_values_khz[i] = 0;
+			}
+		}
+	}
+
+	/* socclk */
+	if (dc_clk_table->num_entries_per_clk.num_socclk_levels) {
+		dml_clk_table->socclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_socclk_levels;
+		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
+			if (i < dml_clk_table->socclk.num_clk_values) {
+				dml_clk_table->socclk.clk_values_khz[i] = dc_clk_table->entries[i].socclk_mhz * 1000;
+			} else {
+				dml_clk_table->socclk.clk_values_khz[i] = 0;
+			}
+		}
+	}
+
+	/* dram config */
+	dml_clk_table->dram_config.channel_count = dc_bw_params->num_channels;
+	dml_clk_table->dram_config.channel_width_bytes = dc_bw_params->dram_channel_width_bytes;
+}
+
+static void dcn42_update_soc_bb_with_values_from_clk_mgr(struct dml2_soc_bb *soc_bb, const struct dc *dc)
+{
+	soc_bb->dprefclk_mhz = dc->clk_mgr->dprefclk_khz / 1000;
+	soc_bb->dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
+	soc_bb->mall_allocated_for_dcn_mbytes = dc->caps.mall_size_total / (1024 * 1024);
+
+	if (dc->clk_mgr->funcs->is_smu_present &&
+			dc->clk_mgr->funcs->is_smu_present(dc->clk_mgr)) {
+		dcn42_convert_dc_clock_table_to_soc_bb_clock_table(&soc_bb->clk_table, &soc_bb->vmin_limit,
+			dc->clk_mgr->bw_params);
+	}
+}
+
+static void apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
+{
+	/* Individual modification can be overwritten even if it was obtained by a previous function.
+	 * Modifications are acquired in order of priority (lowest to highest).
+	 */
+	dc_assert_fp_enabled();
+
+	dcn42_update_soc_bb_with_values_from_clk_mgr(soc_bb, dc);
+	dcn401_update_soc_bb_with_values_from_vbios(soc_bb, dc);
+	dcn401_update_soc_bb_with_values_from_software_policy(soc_bb, dc);
+}
+
+void dcn42_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
+{
+	//get default soc_bb with static values
+	get_default_soc_bb(soc_bb, dc);
+	//update soc_bb values with more accurate values
+	apply_soc_bb_updates(soc_bb, dc, config);
+}
+
 static void dcn42_get_ip_caps(struct dml2_ip_capabilities *ip_caps)
 {
 	*ip_caps = dml2_dcn42_max_ip_caps;
 }
 
 static struct soc_and_ip_translator_funcs dcn42_translator_funcs = {
-	.get_soc_bb = dcn401_get_soc_bb,
+	.get_soc_bb = dcn42_get_soc_bb,
 	.get_ip_caps = dcn42_get_ip_caps,
 };
 
@@ -25,3 +183,4 @@ void dcn42_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and
 {
 	soc_and_ip_translator->translator_funcs = &dcn42_translator_funcs;
 }
+
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
index 914dcbb369a75..1dded54261521 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
@@ -12,5 +12,6 @@
 #include "soc_and_ip_translator.h"
 
 void dcn42_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator);
+void dcn42_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config);
 
 #endif /* _DCN42_SOC_AND_IP_TRANSLATOR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
index 6617c9d2d5f84..bad0bd79fa940 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
@@ -4,6 +4,7 @@
 
 #include "soc_and_ip_translator.h"
 #include "soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.h"
+#include "soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h"
 
 static void dc_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator,
 		enum dce_version dc_version)
@@ -12,6 +13,9 @@ static void dc_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc
 	case DCN_VERSION_4_01:
 		dcn401_construct_soc_and_ip_translator(soc_and_ip_translator);
 		break;
+	case DCN_VERSION_4_2:
+		dcn42_construct_soc_and_ip_translator(soc_and_ip_translator);
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index 3b6bba0170406..c18ff8f00bb85 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: MIT */
 /*
- * Copyright 2019 Advanced Micro Devices, Inc.
+ * Copyright 2019-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -119,6 +120,7 @@ enum dmub_asic {
 	DMUB_ASIC_DCN351,
 	DMUB_ASIC_DCN36,
 	DMUB_ASIC_DCN401,
+	DMUB_ASIC_DCN42,
 	DMUB_ASIC_MAX,
 };
 
@@ -602,6 +604,7 @@ struct dmub_srv {
 	struct dmub_srv_dcn32_regs *regs_dcn32;
 	struct dmub_srv_dcn35_regs *regs_dcn35;
 	const struct dmub_srv_dcn401_regs *regs_dcn401;
+	struct dmub_srv_dcn42_regs *regs_dcn42;
 	struct dmub_srv_base_funcs funcs;
 	struct dmub_srv_hw_funcs hw_funcs;
 	struct dmub_srv_inbox inbox1;
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index 6f388c910e18a..e11e32afac6be 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -909,7 +909,8 @@ union dmub_fw_meta_feature_bits {
 	struct {
 		uint32_t shared_state_link_detection : 1; /**< 1 supports link detection via shared state */
 		uint32_t cursor_offload_v1_support: 1; /**< 1 supports cursor offload */
-		uint32_t reserved : 30;
+		uint32_t inbox0_lock_support: 1; /**< 1 supports inbox0 lock mechanism */
+		uint32_t reserved : 29;
 	} bits; /**< status bits */
 	uint32_t all; /**< 32-bit access to status bits */
 };
@@ -1535,14 +1536,12 @@ enum dmub_gpint_command {
 	 *       1 - Enable ips measurement
 	 */
 	DMUB_GPINT__IPS_RESIDENCY = 121,
-
 	/**
 	 * DESC: Enable measurements for various task duration
 	 * ARGS: 0 - Disable measurement
 	 *       1 - Enable measurement
 	 */
 	DMUB_GPINT__TRACE_DMUB_WAKE_ACTIVITY = 123,
-
 	/**
 	 * DESC: Gets IPS residency in microseconds
 	 * ARGS: 0 - Return IPS1 residency
@@ -1552,21 +1551,18 @@ enum dmub_gpint_command {
 	 * RETURN: Total residency in microseconds - lower 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_DURATION_US_LO = 124,
-
 	/**
 	 * DESC: Gets IPS1 histogram counts
 	 * ARGS: Bucket index
 	 * RETURN: Total count for the bucket
 	 */
 	DMUB_GPINT__GET_IPS1_HISTOGRAM_COUNTER = 125,
-
 	/**
 	 * DESC: Gets IPS2 histogram counts
 	 * ARGS: Bucket index
 	 * RETURN: Total count for the bucket
 	 */
 	DMUB_GPINT__GET_IPS2_HISTOGRAM_COUNTER = 126,
-
 	/**
 	 * DESC: Gets IPS residency
 	 * ARGS: 0 - Return IPS1 residency
@@ -1576,21 +1572,18 @@ enum dmub_gpint_command {
 	 * RETURN: Total residency in milli-percent.
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_PERCENT = 127,
-
 	/**
 	 * DESC: Gets IPS1_RCG histogram counts
 	 * ARGS: Bucket index
 	 * RETURN: Total count for the bucket
 	 */
 	DMUB_GPINT__GET_IPS1_RCG_HISTOGRAM_COUNTER = 128,
-
 	/**
 	 * DESC: Gets IPS1_ONO2_ON histogram counts
 	 * ARGS: Bucket index
 	 * RETURN: Total count for the bucket
 	 */
 	DMUB_GPINT__GET_IPS1_ONO2_ON_HISTOGRAM_COUNTER = 129,
-
 	/**
 	 * DESC: Gets IPS entry counter during residency measurement
 	 * ARGS: 0 - Return IPS1 entry counts
@@ -1600,7 +1593,6 @@ enum dmub_gpint_command {
 	 * RETURN: Entry counter for selected IPS mode
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_ENTRY_COUNTER = 130,
-
 	/**
 	 * DESC: Gets IPS inactive residency in microseconds
 	 * ARGS: 0 - Return IPS1_MAX residency
@@ -1610,7 +1602,6 @@ enum dmub_gpint_command {
 	 * RETURN: Total inactive residency in microseconds - lower 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_INACTIVE_RESIDENCY_DURATION_US_LO = 131,
-
 	/**
 	 * DESC: Gets IPS inactive residency in microseconds
 	 * ARGS: 0 - Return IPS1_MAX residency
@@ -1620,7 +1611,6 @@ enum dmub_gpint_command {
 	 * RETURN: Total inactive residency in microseconds - upper 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_INACTIVE_RESIDENCY_DURATION_US_HI = 132,
-
 	/**
 	 * DESC: Gets IPS residency in microseconds
 	 * ARGS: 0 - Return IPS1 residency
@@ -1679,7 +1669,7 @@ union dmub_inbox0_cmd_lock_hw {
 
 		uint32_t lock: 1;			/**< Lock */
 		uint32_t should_release: 1;		/**< Release */
-		uint32_t reserved: 7;			/**< Reserved for extending more clients, HW, etc. */
+		uint32_t reserved: 7; 			/**< Reserved for extending more clients, HW, etc. */
 	} bits;
 	uint32_t all;
 };
@@ -1897,6 +1887,15 @@ enum dmub_cmd_type {
 	 */
 	DMUB_CMD__PR = 94,
 
+	/**
+	 * Command type used for all IHC commands.
+	 */
+	DMUB_CMD__IHC = 95,
+
+	/**
+	 * Command type use for boot time crc commands.
+	 */
+	DMUB_CMD__BOOT_TIME_CRC = 96,
 
 	/**
 	 * Command type use for VBIOS shared commands.
@@ -2610,9 +2609,9 @@ struct dmub_fams2_stream_static_state {
 	uint8_t allow_to_target_delay_otg_vlines; // time from allow vline to target vline
 	union {
 		struct {
-			uint8_t is_drr: 1; // stream is DRR enabled
-			uint8_t clamp_vtotal_min: 1; // clamp vtotal to min instead of nominal
-			uint8_t min_ttu_vblank_usable: 1; // if min ttu vblank is above wm, no force pstate is needed in blank
+			uint8_t is_drr : 1; // stream is DRR enabled
+			uint8_t clamp_vtotal_min : 1; // clamp vtotal to min instead of nominal
+			uint8_t min_ttu_vblank_usable : 1; // if min ttu vblank is above wm, no force pstate is needed in blank
 		} bits;
 		uint8_t all;
 	} config;
@@ -4437,6 +4436,7 @@ enum dmub_cmd_replay_general_subtype {
 	REPLAY_GENERAL_CMD_SET_LOW_RR_ACTIVATE,
 	REPLAY_GENERAL_CMD_VIDEO_CONFERENCING,
 	REPLAY_GENERAL_CMD_SET_CONTINUOUSLY_RESYNC,
+	REPLAY_GENERAL_CMD_SET_COASTING_VTOTAL_WITHOUT_FRAME_UPDATE,
 };
 
 struct dmub_alpm_auxless_data {
@@ -4655,6 +4655,18 @@ struct dmub_rb_cmd_replay_enable_data {
 	 * This does not support HDMI/DP2 for now.
 	 */
 	uint8_t phy_rate;
+	/**
+	 * @hpo_stream_enc_inst: HPO stream encoder instance
+	 */
+	uint8_t hpo_stream_enc_inst;
+	/**
+	 * @hpo_link_enc_inst: HPO link encoder instance
+	 */
+	uint8_t hpo_link_enc_inst;
+	/**
+	 * @pad: Align structure to 4 byte boundary.
+	 */
+	uint8_t pad[2];
 };
 
 /**
@@ -4959,6 +4971,52 @@ union dmub_replay_cmd_set {
 	struct dmub_cmd_replay_set_general_cmd_data set_general_cmd_data;
 };
 
+/**
+ * IHC command sub-types.
+ */
+enum dmub_cmd_ihc_type {
+	/**
+	 * Set DIG HDCP interrupt destination.
+	 */
+	DMUB_CMD__IHC_SET_DIG_HDCP_INTERRUPT_DEST = 0,
+};
+
+/**
+ * Data passed from driver to FW in a DMUB_CMD__IHC command.
+ */
+struct dmub_cmd_ihc_data {
+	/**
+	 * DIG engine ID (0-3).
+	 */
+	uint8_t dig_id;
+	/**
+	 * 1 = route to DMU, 0 = route to CPU.
+	 */
+	uint8_t to_dmu : 1;
+	/**
+	 * Reserved bits.
+	 */
+	uint8_t reserved : 7;
+	/**
+	 * Padding.
+	 */
+	uint8_t pad[2];
+};
+
+/**
+ * Definition of a DMUB_CMD__IHC command.
+ */
+struct dmub_rb_cmd_ihc {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+	/**
+	 * IHC command data.
+	 */
+	struct dmub_cmd_ihc_data data;
+};
+
 /**
  * SMART POWER OLED command sub-types.
  */
@@ -5222,8 +5280,8 @@ enum dmub_cmd_lsdma_type {
 	 */
 	DMUB_CMD__LSDMA_LINEAR_COPY = 1,
 	/**
-	 * LSDMA copies data from source to destination linearly in sub window
-	 */
+	* LSDMA copies data from source to destination linearly in sub window
+	*/
 	DMUB_CMD__LSDMA_LINEAR_SUB_WINDOW_COPY = 2,
 	/**
 	 * Send the tiled-to-tiled copy command
@@ -6785,6 +6843,29 @@ struct dmub_rb_cmd_pr_general_cmd {
 	struct dmub_cmd_pr_general_cmd_data data;
 };
 
+/**
+ * Command type of a DMUB_CMD__BOOT_TIME_CRC command
+ */
+enum dmub_cmd_boot_time_crc_type {
+	DMUB_CMD__BOOT_TIME_CRC_INIT_MEM = 0
+};
+
+/**
+ * Data passed from driver to FW in a DMUB_CMD__BOOT_TIME_CRC_INIT command.
+ */
+struct dmub_cmd_boot_time_crc_init_data {
+	union dmub_addr buffer_addr;
+	uint32_t buffer_size;
+};
+
+/**
+ * Definition of a DMUB_CMD__BOOT_TIME_CRC_INIT command.
+ */
+struct dmub_rb_cmd_boot_time_crc_init {
+	struct dmub_cmd_header header;
+	struct dmub_cmd_boot_time_crc_init_data data;
+};
+
 /**
  * union dmub_rb_cmd - DMUB inbox command.
  */
@@ -7142,6 +7223,14 @@ union dmub_rb_cmd {
 	struct dmub_rb_cmd_pr_update_state pr_update_state;
 
 	struct dmub_rb_cmd_pr_general_cmd pr_general_cmd;
+	/**
+	 * Definition of a DMUB_CMD__IHC command.
+	 */
+	struct dmub_rb_cmd_ihc ihc;
+	/**
+	 * Definition of a DMUB_CMD__BOOT_TIME_CRC_INIT command.
+	 */
+	struct dmub_rb_cmd_boot_time_crc_init boot_time_crc_init;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/src/Makefile b/drivers/gpu/drm/amd/display/dmub/src/Makefile
index 468b768c11aeb..ac7b17d8fb0fd 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/Makefile
+++ b/drivers/gpu/drm/amd/display/dmub/src/Makefile
@@ -1,5 +1,5 @@
 #
-# Copyright 2019 Advanced Micro Devices, Inc.
+# Copyright 2019-2026 Advanced Micro Devices, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -28,6 +28,7 @@ DMUB += dmub_dcn35.o
 DMUB += dmub_dcn351.o
 DMUB += dmub_dcn36.o
 DMUB += dmub_dcn401.o
+DMUB += dmub_dcn42.o
 
 AMD_DAL_DMUB = $(addprefix $(AMDDALPATH)/dmub/src/,$(DMUB))
 
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
new file mode 100644
index 0000000000000..4d274b7034e89
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
@@ -0,0 +1,752 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "../dmub_srv.h"
+#include "dmub_reg.h"
+#include "dmub_dcn35.h"
+#include "dmub_dcn401.h"
+#include "dmub_dcn42.h"
+
+#include "dcn/dcn_4_2_0_offset.h"
+#include "dcn/dcn_4_2_0_sh_mask.h"
+
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+#define CTX dmub
+#define REGS dmub->regs_dcn42
+#define REG_OFFSET_EXP(reg_name) (BASE(reg##reg_name##_BASE_IDX) + reg##reg_name)
+
+void dmub_srv_dcn42_regs_init(struct dmub_srv *dmub, struct dc_context *ctx)
+{
+	struct dmub_srv_dcn42_regs *regs = dmub->regs_dcn42;
+#define REG_STRUCT regs
+
+#define DMUB_SR(reg) REG_STRUCT->offset.reg = REG_OFFSET_EXP(reg);
+	DMUB_DCN42_REGS()
+	DMCUB_INTERNAL_REGS()
+#undef DMUB_SR
+
+#define DMUB_SF(reg, field) REG_STRUCT->mask.reg##__##field = FD_MASK(reg, field);
+	DMUB_DCN42_FIELDS()
+#undef DMUB_SF
+
+#define DMUB_SF(reg, field) REG_STRUCT->shift.reg##__##field = FD_SHIFT(reg, field);
+	DMUB_DCN42_FIELDS()
+#undef DMUB_SF
+#undef REG_STRUCT
+}
+
+void dmub_dcn42_enable_dmub_boot_options(struct dmub_srv *dmub, const struct dmub_srv_hw_params *params)
+{
+	union dmub_fw_boot_options boot_options = {0};
+	union dmub_fw_boot_options cur_boot_options = {0};
+
+	cur_boot_options = dmub_dcn42_get_fw_boot_option(dmub);
+
+	boot_options.bits.z10_disable = params->disable_z10;
+	boot_options.bits.dpia_supported = params->dpia_supported;
+	boot_options.bits.enable_dpia = cur_boot_options.bits.enable_dpia && !params->disable_dpia;
+	boot_options.bits.usb4_cm_version = params->usb4_cm_version;
+	boot_options.bits.dpia_hpd_int_enable_supported = params->dpia_hpd_int_enable_supported;
+	boot_options.bits.power_optimization = params->power_optimization;
+	boot_options.bits.disable_clk_ds = params->disallow_dispclk_dppclk_ds;
+	boot_options.bits.disable_clk_gate = params->disable_clock_gate;
+	boot_options.bits.ips_disable = params->disable_ips;
+	boot_options.bits.ips_sequential_ono = params->ips_sequential_ono;
+	boot_options.bits.disable_sldo_opt = params->disable_sldo_opt;
+	boot_options.bits.enable_non_transparent_setconfig = params->enable_non_transparent_setconfig;
+	boot_options.bits.lower_hbr3_phy_ssc = params->lower_hbr3_phy_ssc;
+	boot_options.bits.skip_phy_access = params->disallow_phy_access;
+	boot_options.bits.disable_dpia_bw_allocation = params->disable_dpia_bw_allocation;
+
+	REG_WRITE(DMCUB_SCRATCH14, boot_options.all);
+}
+
+static void dmub_dcn42_get_fb_base_offset(struct dmub_srv *dmub,
+					  uint64_t *fb_base,
+					  uint64_t *fb_offset)
+{
+	uint32_t tmp;
+
+	/*
+	if (dmub->soc_fb_info.fb_base || dmub->soc_fb_info.fb_offset) {
+		*fb_base = dmub->soc_fb_info.fb_base;
+		*fb_offset = dmub->soc_fb_info.fb_offset;
+		return;
+	}
+	*/
+
+	REG_GET(DCN_VM_FB_LOCATION_BASE, FB_BASE, &tmp);
+	*fb_base = (uint64_t)tmp << 24;
+
+	REG_GET(DCN_VM_FB_OFFSET, FB_OFFSET, &tmp);
+	*fb_offset = (uint64_t)tmp << 24;
+}
+
+static inline void dmub_dcn42_translate_addr(const union dmub_addr *addr_in,
+					     uint64_t fb_base,
+					     uint64_t fb_offset,
+					     union dmub_addr *addr_out)
+{
+	addr_out->quad_part = addr_in->quad_part - fb_base + fb_offset;
+}
+
+void dmub_dcn42_reset(struct dmub_srv *dmub)
+{
+	union dmub_gpint_data_register cmd;
+	const uint32_t timeout = 100000;
+	uint32_t in_reset, is_enabled, scratch, i, pwait_mode;
+
+	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &in_reset);
+	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_enabled);
+
+	if (in_reset == 0 && is_enabled != 0) {
+		cmd.bits.status = 1;
+		cmd.bits.command_code = DMUB_GPINT__STOP_FW;
+		cmd.bits.param = 0;
+
+		dmub->hw_funcs.set_gpint(dmub, cmd);
+
+		for (i = 0; i < timeout; ++i) {
+			if (dmub->hw_funcs.is_gpint_acked(dmub, cmd))
+				break;
+
+			udelay(1);
+		}
+
+		for (i = 0; i < timeout; ++i) {
+			scratch = REG_READ(DMCUB_SCRATCH7);
+			if (scratch == DMUB_GPINT__STOP_FW_RESPONSE)
+				break;
+
+			udelay(1);
+		}
+
+		for (i = 0; i < timeout; ++i) {
+			REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &pwait_mode);
+			if (pwait_mode & (1 << 0))
+				break;
+
+			udelay(1);
+		}
+		/* Force reset in case we timed out, DMCUB is likely hung. */
+	}
+
+	if (is_enabled) {
+		REG_UPDATE(DMCUB_CNTL2, DMCUB_SOFT_RESET, 1);
+		udelay(1);
+		REG_UPDATE(DMCUB_CNTL, DMCUB_ENABLE, 0);
+	}
+
+	REG_WRITE(DMCUB_INBOX1_RPTR, 0);
+	REG_WRITE(DMCUB_INBOX1_WPTR, 0);
+	REG_WRITE(DMCUB_OUTBOX1_RPTR, 0);
+	REG_WRITE(DMCUB_OUTBOX1_WPTR, 0);
+	REG_WRITE(DMCUB_OUTBOX0_RPTR, 0);
+	REG_WRITE(DMCUB_OUTBOX0_WPTR, 0);
+	REG_WRITE(DMCUB_SCRATCH0, 0);
+
+	/* Clear the GPINT command manually so we don't send anything during boot. */
+	cmd.all = 0;
+	dmub->hw_funcs.set_gpint(dmub, cmd);
+}
+
+void dmub_dcn42_reset_release(struct dmub_srv *dmub)
+{
+	REG_WRITE(DMCUB_SCRATCH15, dmub->psp_version & 0x001100FF);
+
+	REG_UPDATE_3(DMU_CLK_CNTL,
+		     LONO_DISPCLK_GATE_DISABLE, 1,
+		     LONO_SOCCLK_GATE_DISABLE, 1,
+		     LONO_DMCUBCLK_GATE_DISABLE, 1);
+
+	REG_UPDATE_2(DMCUB_CNTL, DMCUB_ENABLE, 1, DMCUB_TRACEPORT_EN, 1);
+	REG_UPDATE(MMHUBBUB_SOFT_RESET, DMUIF_SOFT_RESET, 0);
+	REG_UPDATE(DMCUB_CNTL2, DMCUB_SOFT_RESET, 0);
+}
+
+void dmub_dcn42_backdoor_load(struct dmub_srv *dmub,
+			      const struct dmub_window *cw0,
+			      const struct dmub_window *cw1)
+{
+	union dmub_addr offset;
+	uint64_t fb_base, fb_offset;
+
+	dmub_dcn42_get_fb_base_offset(dmub, &fb_base, &fb_offset);
+
+	dmub_dcn42_translate_addr(&cw0->offset, fb_base, fb_offset, &offset);
+
+	REG_WRITE(DMCUB_REGION3_CW0_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW0_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW0_BASE_ADDRESS, cw0->region.base);
+	REG_SET_2(DMCUB_REGION3_CW0_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW0_TOP_ADDRESS, cw0->region.top,
+		  DMCUB_REGION3_CW0_ENABLE, 1);
+
+	dmub_dcn42_translate_addr(&cw1->offset, fb_base, fb_offset, &offset);
+
+	REG_WRITE(DMCUB_REGION3_CW1_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW1_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW1_BASE_ADDRESS, cw1->region.base);
+	REG_SET_2(DMCUB_REGION3_CW1_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW1_TOP_ADDRESS, cw1->region.top,
+		  DMCUB_REGION3_CW1_ENABLE, 1);
+
+	/* TODO: Do we need to set DMCUB_MEM_UNIT_ID? */
+	REG_UPDATE(DMCUB_SEC_CNTL, DMCUB_SEC_RESET, 0);
+}
+
+void dmub_dcn42_backdoor_load_zfb_mode(struct dmub_srv *dmub,
+		      const struct dmub_window *cw0,
+		      const struct dmub_window *cw1)
+{
+	union dmub_addr offset;
+
+	REG_UPDATE(DMCUB_SEC_CNTL, DMCUB_SEC_RESET, 1);
+	offset = cw0->offset;
+	REG_WRITE(DMCUB_REGION3_CW0_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW0_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW0_BASE_ADDRESS, cw0->region.base);
+	REG_SET_2(DMCUB_REGION3_CW0_TOP_ADDRESS, 0,
+			DMCUB_REGION3_CW0_TOP_ADDRESS, cw0->region.top,
+			DMCUB_REGION3_CW0_ENABLE, 1);
+	offset = cw1->offset;
+	REG_WRITE(DMCUB_REGION3_CW1_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW1_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW1_BASE_ADDRESS, cw1->region.base);
+	REG_SET_2(DMCUB_REGION3_CW1_TOP_ADDRESS, 0,
+			DMCUB_REGION3_CW1_TOP_ADDRESS, cw1->region.top,
+			DMCUB_REGION3_CW1_ENABLE, 1);
+	REG_UPDATE_2(DMCUB_SEC_CNTL, DMCUB_SEC_RESET, 0, DMCUB_MEM_UNIT_ID,
+			0x20);
+}
+void dmub_dcn42_setup_windows(struct dmub_srv *dmub,
+			      const struct dmub_window *cw2,
+			      const struct dmub_window *cw3,
+			      const struct dmub_window *cw4,
+			      const struct dmub_window *cw5,
+			      const struct dmub_window *cw6,
+			      const struct dmub_window *region6)
+{
+	union dmub_addr offset;
+
+	offset = cw3->offset;
+
+	REG_WRITE(DMCUB_REGION3_CW3_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW3_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW3_BASE_ADDRESS, cw3->region.base);
+	REG_SET_2(DMCUB_REGION3_CW3_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW3_TOP_ADDRESS, cw3->region.top,
+		  DMCUB_REGION3_CW3_ENABLE, 1);
+
+	offset = cw4->offset;
+
+	REG_WRITE(DMCUB_REGION3_CW4_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW4_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW4_BASE_ADDRESS, cw4->region.base);
+	REG_SET_2(DMCUB_REGION3_CW4_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW4_TOP_ADDRESS, cw4->region.top,
+		  DMCUB_REGION3_CW4_ENABLE, 1);
+
+	offset = cw5->offset;
+
+	REG_WRITE(DMCUB_REGION3_CW5_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW5_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW5_BASE_ADDRESS, cw5->region.base);
+	REG_SET_2(DMCUB_REGION3_CW5_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW5_TOP_ADDRESS, cw5->region.top,
+		  DMCUB_REGION3_CW5_ENABLE, 1);
+
+	REG_WRITE(DMCUB_REGION5_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION5_OFFSET_HIGH, offset.u.high_part);
+	REG_SET_2(DMCUB_REGION5_TOP_ADDRESS, 0,
+		  DMCUB_REGION5_TOP_ADDRESS,
+		  cw5->region.top - cw5->region.base - 1,
+		  DMCUB_REGION5_ENABLE, 1);
+
+	offset = cw6->offset;
+
+	REG_WRITE(DMCUB_REGION3_CW6_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION3_CW6_OFFSET_HIGH, offset.u.high_part);
+	REG_WRITE(DMCUB_REGION3_CW6_BASE_ADDRESS, cw6->region.base);
+	REG_SET_2(DMCUB_REGION3_CW6_TOP_ADDRESS, 0,
+		  DMCUB_REGION3_CW6_TOP_ADDRESS, cw6->region.top,
+		  DMCUB_REGION3_CW6_ENABLE, 1);
+
+	offset = region6->offset;
+
+	REG_WRITE(DMCUB_REGION6_OFFSET, offset.u.low_part);
+	REG_WRITE(DMCUB_REGION6_OFFSET_HIGH, offset.u.high_part);
+	REG_SET_2(DMCUB_REGION6_TOP_ADDRESS, 0,
+		  DMCUB_REGION6_TOP_ADDRESS,
+		  region6->region.top - region6->region.base - 1,
+		  DMCUB_REGION6_ENABLE, 1);
+}
+
+uint32_t dmub_dcn42_get_inbox1_wptr(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_INBOX1_WPTR);
+}
+
+uint32_t dmub_dcn42_get_inbox1_rptr(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_INBOX1_RPTR);
+}
+
+void dmub_dcn42_setup_out_mailbox(struct dmub_srv *dmub,
+			      const struct dmub_region *outbox1)
+{
+	REG_WRITE(DMCUB_OUTBOX1_BASE_ADDRESS, outbox1->base);
+	REG_WRITE(DMCUB_OUTBOX1_SIZE, outbox1->top - outbox1->base);
+}
+
+uint32_t dmub_dcn42_get_outbox1_wptr(struct dmub_srv *dmub)
+{
+	/**
+	 * outbox1 wptr register is accessed without locks (dal & dc)
+	 * and to be called only by dmub_srv_stat_get_notification()
+	 */
+	return REG_READ(DMCUB_OUTBOX1_WPTR);
+}
+
+void dmub_dcn42_set_outbox1_rptr(struct dmub_srv *dmub, uint32_t rptr_offset)
+{
+	/**
+	 * outbox1 rptr register is accessed without locks (dal & dc)
+	 * and to be called only by dmub_srv_stat_get_notification()
+	 */
+	REG_WRITE(DMCUB_OUTBOX1_RPTR, rptr_offset);
+}
+
+bool dmub_dcn42_is_supported(struct dmub_srv *dmub)
+{
+	uint32_t supported = 0;
+
+	REG_GET(CC_DC_PIPE_DIS, DC_DMCUB_ENABLE, &supported);
+
+	return supported;
+}
+
+union dmub_fw_boot_options dmub_dcn42_get_fw_boot_option(struct dmub_srv *dmub)
+{
+	union dmub_fw_boot_options option;
+
+	option.all = REG_READ(DMCUB_SCRATCH14);
+	return option;
+}
+
+void dmub_dcn42_setup_outbox0(struct dmub_srv *dmub,
+			      const struct dmub_region *outbox0)
+{
+	REG_WRITE(DMCUB_OUTBOX0_BASE_ADDRESS, outbox0->base);
+
+	REG_WRITE(DMCUB_OUTBOX0_SIZE, outbox0->top - outbox0->base);
+}
+
+bool dmub_dcn42_should_detect(struct dmub_srv *dmub)
+{
+	uint32_t fw_boot_status = REG_READ(DMCUB_SCRATCH0);
+	bool should_detect = (fw_boot_status & DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED) != 0;
+	return should_detect;
+}
+
+void dmub_dcn42_send_inbox0_cmd(struct dmub_srv *dmub, union dmub_inbox0_data_register data)
+{
+	REG_WRITE(DMCUB_INBOX0_WPTR, data.inbox0_cmd_common.all);
+}
+
+uint32_t dmub_dcn42_read_inbox0_ack_register(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_SCRATCH17);
+}
+
+bool dmub_dcn42_is_hw_powered_up(struct dmub_srv *dmub)
+{
+	union dmub_fw_boot_status status;
+	uint32_t is_enable;
+
+	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_enable);
+	if (is_enable == 0)
+		return false;
+
+	status.all = REG_READ(DMCUB_SCRATCH0);
+
+	return (status.bits.dal_fw && status.bits.hw_power_init_done && status.bits.mailbox_rdy) ||
+	       (!status.bits.dal_fw && status.bits.mailbox_rdy);
+}
+
+void dmub_dcn42_set_inbox1_wptr(struct dmub_srv *dmub, uint32_t wptr_offset)
+{
+	REG_WRITE(DMCUB_INBOX1_WPTR, wptr_offset);
+}
+
+bool dmub_dcn42_is_hw_init(struct dmub_srv *dmub)
+{
+	union dmub_fw_boot_status status;
+	uint32_t is_hw_init;
+
+	status.all = REG_READ(DMCUB_SCRATCH0);
+	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_hw_init);
+
+	return is_hw_init != 0 && status.bits.dal_fw;
+}
+
+union dmub_fw_boot_status dmub_dcn42_get_fw_boot_status(struct dmub_srv *dmub)
+{
+	union dmub_fw_boot_status status;
+
+	status.all = REG_READ(DMCUB_SCRATCH0);
+	return status;
+}
+
+void dmub_dcn42_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip)
+{
+	union dmub_fw_boot_options boot_options;
+	boot_options.all = REG_READ(DMCUB_SCRATCH14);
+	boot_options.bits.skip_phy_init_panel_sequence = skip;
+	REG_WRITE(DMCUB_SCRATCH14, boot_options.all);
+}
+
+void dmub_dcn42_configure_dmub_in_system_memory(struct dmub_srv *dmub)
+{
+	/* DMCUB_REGION3_TMR_AXI_SPACE values:
+	 * 0b011 (0x3) - FB physical address
+	 * 0b100 (0x4) - GPU virtual address
+	 *
+	 * Default value is 0x3 (FB Physical address for TMR). When programming
+	 * DMUB to be in system memory, change to 0x4. The system memory allocated
+	 * is accessible by both GPU and CPU, so we use GPU virtual address.
+	 */
+	REG_WRITE(DMCUB_REGION3_TMR_AXI_SPACE, 0x4);
+}
+
+void dmub_dcn42_clear_inbox0_ack_register(struct dmub_srv *dmub)
+{
+	REG_WRITE(DMCUB_SCRATCH17, 0);
+}
+
+void dmub_dcn42_send_reg_inbox0_cmd_msg(struct dmub_srv *dmub,
+		union dmub_rb_cmd *cmd)
+{
+	uint32_t *dwords = (uint32_t *)cmd;
+	int32_t payload_size_bytes = cmd->cmd_common.header.payload_bytes;
+	uint32_t msg_index;
+	static_assert(sizeof(*cmd) == 64, "DMUB command size mismatch");
+
+	/* read remaining data based on payload size */
+	for (msg_index = 0; msg_index < 15; msg_index++) {
+		if (payload_size_bytes <= msg_index * 4) {
+			break;
+		}
+
+		switch (msg_index) {
+		case 0:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG0, dwords[msg_index + 1]);
+			break;
+		case 1:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG1, dwords[msg_index + 1]);
+			break;
+		case 2:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG2, dwords[msg_index + 1]);
+			break;
+		case 3:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG3, dwords[msg_index + 1]);
+			break;
+		case 4:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG4, dwords[msg_index + 1]);
+			break;
+		case 5:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG5, dwords[msg_index + 1]);
+			break;
+		case 6:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG6, dwords[msg_index + 1]);
+			break;
+		case 7:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG7, dwords[msg_index + 1]);
+			break;
+		case 8:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG8, dwords[msg_index + 1]);
+			break;
+		case 9:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG9, dwords[msg_index + 1]);
+			break;
+		case 10:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG10, dwords[msg_index + 1]);
+			break;
+		case 11:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG11, dwords[msg_index + 1]);
+			break;
+		case 12:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG12, dwords[msg_index + 1]);
+			break;
+		case 13:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG13, dwords[msg_index + 1]);
+			break;
+		case 14:
+			REG_WRITE(DMCUB_REG_INBOX0_MSG14, dwords[msg_index + 1]);
+			break;
+		}
+	}
+
+	/* writing to INBOX RDY register will trigger DMUB REG INBOX0 RDY
+	 * interrupt.
+	 */
+	REG_WRITE(DMCUB_REG_INBOX0_RDY, dwords[0]);
+}
+
+uint32_t dmub_dcn42_read_reg_inbox0_rsp_int_status(struct dmub_srv *dmub)
+{
+	uint32_t status;
+
+	REG_GET(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_STAT, &status);
+	return status;
+}
+
+void dmub_dcn42_read_reg_inbox0_cmd_rsp(struct dmub_srv *dmub,
+		union dmub_rb_cmd *cmd)
+{
+	uint32_t *dwords = (uint32_t *)cmd;
+
+	static_assert(sizeof(*cmd) == 64, "DMUB command size mismatch");
+
+	dwords[0] = REG_READ(DMCUB_REG_INBOX0_RSP);
+	dwords[1] = REG_READ(DMCUB_REG_INBOX0_MSG0);
+	dwords[2] = REG_READ(DMCUB_REG_INBOX0_MSG1);
+	dwords[3] = REG_READ(DMCUB_REG_INBOX0_MSG2);
+	dwords[4] = REG_READ(DMCUB_REG_INBOX0_MSG3);
+	dwords[5] = REG_READ(DMCUB_REG_INBOX0_MSG4);
+	dwords[6] = REG_READ(DMCUB_REG_INBOX0_MSG5);
+	dwords[7] = REG_READ(DMCUB_REG_INBOX0_MSG6);
+	dwords[8] = REG_READ(DMCUB_REG_INBOX0_MSG7);
+	dwords[9] = REG_READ(DMCUB_REG_INBOX0_MSG8);
+	dwords[10] = REG_READ(DMCUB_REG_INBOX0_MSG9);
+	dwords[11] = REG_READ(DMCUB_REG_INBOX0_MSG10);
+	dwords[12] = REG_READ(DMCUB_REG_INBOX0_MSG11);
+	dwords[13] = REG_READ(DMCUB_REG_INBOX0_MSG12);
+	dwords[14] = REG_READ(DMCUB_REG_INBOX0_MSG13);
+	dwords[15] = REG_READ(DMCUB_REG_INBOX0_MSG14);
+}
+
+void dmub_dcn42_write_reg_inbox0_rsp_int_ack(struct dmub_srv *dmub)
+{
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_ACK, 1);
+}
+
+void dmub_dcn42_clear_reg_inbox0_rsp_int_ack(struct dmub_srv *dmub)
+{
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_ACK, 0);
+}
+
+void dmub_dcn42_enable_reg_inbox0_rsp_int(struct dmub_srv *dmub, bool enable)
+{
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_EN, enable ? 1:0);
+}
+
+void dmub_dcn42_write_reg_outbox0_rdy_int_ack(struct dmub_srv *dmub)
+{
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_ACK, 1);
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_ACK, 0);
+}
+
+void dmub_dcn42_read_reg_outbox0_msg(struct dmub_srv *dmub, uint32_t *msg)
+{
+	*msg = REG_READ(DMCUB_REG_OUTBOX0_MSG0);
+}
+
+
+void dmub_dcn42_enable_reg_outbox0_rdy_int(struct dmub_srv *dmub, bool enable)
+{
+	REG_UPDATE(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_EN, enable ? 1:0);
+}
+
+uint32_t dmub_dcn42_read_reg_outbox0_rdy_int_status(struct dmub_srv *dmub)
+{
+	uint32_t status;
+
+	REG_GET(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_STAT, &status);
+	return status;
+}
+
+void dmub_dcn42_setup_mailbox(struct dmub_srv *dmub,
+		const struct dmub_region *inbox1)
+{
+	REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, inbox1->base);
+	REG_WRITE(DMCUB_INBOX1_SIZE, inbox1->top - inbox1->base);
+}
+
+void dmub_dcn42_set_gpint(struct dmub_srv *dmub,
+		union dmub_gpint_data_register reg)
+{
+	REG_WRITE(DMCUB_GPINT_DATAIN1, reg.all);
+}
+
+bool dmub_dcn42_is_gpint_acked(struct dmub_srv *dmub,
+		union dmub_gpint_data_register reg)
+{
+	union dmub_gpint_data_register test;
+
+	reg.bits.status = 0;
+	test.all = REG_READ(DMCUB_GPINT_DATAIN1);
+
+	return test.all == reg.all;
+}
+
+uint32_t dmub_dcn42_get_gpint_response(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_SCRATCH7);
+}
+
+uint32_t dmub_dcn42_get_gpint_dataout(struct dmub_srv *dmub)
+{
+	uint32_t dataout = REG_READ(DMCUB_GPINT_DATAOUT);
+
+	REG_UPDATE(DMCUB_INTERRUPT_ENABLE, DMCUB_GPINT_IH_INT_EN, 0);
+
+	REG_WRITE(DMCUB_GPINT_DATAOUT, 0);
+	REG_UPDATE(DMCUB_INTERRUPT_ACK, DMCUB_GPINT_IH_INT_ACK, 1);
+	REG_UPDATE(DMCUB_INTERRUPT_ACK, DMCUB_GPINT_IH_INT_ACK, 0);
+
+	REG_UPDATE(DMCUB_INTERRUPT_ENABLE, DMCUB_GPINT_IH_INT_EN, 1);
+
+	return dataout;
+}
+
+uint32_t dmub_dcn42_get_outbox0_wptr(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_OUTBOX0_WPTR);
+}
+
+void dmub_dcn42_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset)
+{
+	REG_WRITE(DMCUB_OUTBOX0_RPTR, rptr_offset);
+}
+
+uint32_t dmub_dcn42_get_current_time(struct dmub_srv *dmub)
+{
+	return REG_READ(DMCUB_TIMER_CURRENT);
+}
+
+void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub)
+{
+	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset, is_pwait;
+	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
+	struct dmub_timeout_info timeout = {0};
+
+	if (!dmub)
+		return;
+
+	/* timeout data filled externally, cache before resetting memory */
+	timeout = dmub->debug.timeout_info;
+	memset(&dmub->debug, 0, sizeof(dmub->debug));
+	dmub->debug.timeout_info = timeout;
+
+	dmub->debug.dmcub_version = dmub->fw_version;
+
+	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+
+	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
+	dmub->debug.is_dmcub_enabled = is_dmub_enabled;
+
+	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
+	dmub->debug.is_pwait = is_pwait;
+
+	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
+	dmub->debug.is_dmcub_soft_reset = is_soft_reset;
+
+	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
+	dmub->debug.is_dmcub_secure_reset = is_sec_reset;
+
+	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
+	dmub->debug.is_traceport_en  = is_traceport_enabled;
+
+	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
+	dmub->debug.is_cw0_enabled = is_cw0_enabled;
+
+	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
+	dmub->debug.is_cw6_enabled = is_cw6_enabled;
+
+	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+}
+
+bool dmub_dcn42_get_preos_fw_info(struct dmub_srv *dmub)
+{
+	uint64_t region3_cw5_offset;
+	uint32_t top_addr, top_addr_enable, offset_low;
+	uint32_t offset_high, base_addr, fw_version;
+	bool is_vbios_fw = false;
+
+	memset(&dmub->preos_info, 0, sizeof(dmub->preos_info));
+
+	fw_version = REG_READ(DMCUB_SCRATCH1);
+	is_vbios_fw = ((fw_version >> 6) & 0x01) ? true : false;
+	if (!is_vbios_fw)
+		return false;
+
+	dmub->preos_info.boot_status = REG_READ(DMCUB_SCRATCH0);
+	dmub->preos_info.fw_version = REG_READ(DMCUB_SCRATCH1);
+	dmub->preos_info.boot_options = REG_READ(DMCUB_SCRATCH14);
+	REG_GET(DMCUB_REGION3_CW5_TOP_ADDRESS,
+		DMCUB_REGION3_CW5_ENABLE, &top_addr_enable);
+	if (top_addr_enable) {
+		dmub_dcn42_get_fb_base_offset(dmub,
+			&dmub->preos_info.fb_base, &dmub->preos_info.fb_offset);
+		offset_low = REG_READ(DMCUB_REGION3_CW5_OFFSET);
+		offset_high = REG_READ(DMCUB_REGION3_CW5_OFFSET_HIGH);
+		region3_cw5_offset = ((uint64_t)offset_high << 32) | offset_low;
+		dmub->preos_info.trace_buffer_phy_addr = region3_cw5_offset
+			- dmub->preos_info.fb_base + dmub->preos_info.fb_offset;
+
+		REG_GET(DMCUB_REGION3_CW5_TOP_ADDRESS,
+			DMCUB_REGION3_CW5_TOP_ADDRESS, &top_addr);
+		base_addr = REG_READ(DMCUB_REGION3_CW5_BASE_ADDRESS) & 0x1FFFFFFF;
+		dmub->preos_info.trace_buffer_size =
+			(top_addr > base_addr) ? (top_addr - base_addr + 1) : 0;
+	}
+
+	return true;
+}
+
+void dmub_dcn42_write_reg_outbox0_rsp(struct dmub_srv *dmub, uint32_t *rsp)
+{
+	REG_WRITE(DMCUB_REG_OUTBOX0_RSP, *rsp);
+}
+
+uint32_t dmub_dcn42_read_reg_outbox0_rsp_int_status(struct dmub_srv *dmub)
+{
+	uint32_t status;
+
+	REG_GET(DMCUB_INTERRUPT_STATUS, DMCUB_REG_OUTBOX0_RSP_INT_STAT, &status);
+	return status;
+}
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
new file mode 100644
index 0000000000000..a49d88ab0455f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _DMUB_DCN42_H_
+#define _DMUB_DCN42_H_
+
+#include "dmub_dcn35.h"
+#include "dmub_dcn401.h"
+
+
+struct dmub_srv;
+
+/* DCN42 register definitions. */
+
+#define DMUB_DCN42_REGS() \
+	DMUB_DCN35_REGS() \
+	DMUB_SR(DMCUB_INTERRUPT_STATUS) \
+	DMUB_SR(DMCUB_REG_INBOX0_RDY) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG0) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG1) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG2) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG3) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG4) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG5) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG6) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG7) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG8) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG9) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG10) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG11) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG12) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG13) \
+	DMUB_SR(DMCUB_REG_INBOX0_MSG14) \
+	DMUB_SR(DMCUB_REG_INBOX0_RSP) \
+	DMUB_SR(DMCUB_REG_OUTBOX0_RDY) \
+	DMUB_SR(DMCUB_REG_OUTBOX0_MSG0) \
+	DMUB_SR(DMCUB_REG_OUTBOX0_RSP) \
+	DMUB_SR(HOST_INTERRUPT_CSR)
+
+#define DMUB_DCN42_FIELDS() \
+	DMUB_DCN35_FIELDS() \
+	DMUB_SF(DMCUB_INTERRUPT_STATUS, DMCUB_REG_OUTBOX0_RSP_INT_STAT) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_ACK) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_STAT) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_INBOX0_RSP_INT_EN) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_ACK) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_STAT) \
+	DMUB_SF(HOST_INTERRUPT_CSR, HOST_REG_OUTBOX0_RDY_INT_EN)
+
+struct dmub_srv_dcn42_reg_offset {
+#define DMUB_SR(reg) uint32_t reg;
+	DMUB_DCN42_REGS()
+	DMCUB_INTERNAL_REGS()
+#undef DMUB_SR
+};
+
+struct dmub_srv_dcn42_reg_shift {
+#define DMUB_SF(reg, field) uint8_t reg##__##field;
+	DMUB_DCN42_FIELDS()
+#undef DMUB_SF
+};
+
+struct dmub_srv_dcn42_reg_mask {
+#define DMUB_SF(reg, field) uint32_t reg##__##field;
+	DMUB_DCN42_FIELDS()
+#undef DMUB_SF
+};
+
+struct dmub_srv_dcn42_regs {
+	struct dmub_srv_dcn42_reg_offset offset;
+	struct dmub_srv_dcn42_reg_mask mask;
+	struct dmub_srv_dcn42_reg_shift shift;
+};
+
+/* Function declarations */
+
+/* Initialization and configuration */
+void dmub_srv_dcn42_regs_init(struct dmub_srv *dmub, struct dc_context *ctx);
+void dmub_dcn42_enable_dmub_boot_options(struct dmub_srv *dmub, const struct dmub_srv_hw_params *params);
+void dmub_dcn42_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip);
+void dmub_dcn42_configure_dmub_in_system_memory(struct dmub_srv *dmub);
+
+/* Reset and control */
+void dmub_dcn42_reset(struct dmub_srv *dmub);
+void dmub_dcn42_reset_release(struct dmub_srv *dmub);
+
+/* Firmware loading */
+void dmub_dcn42_backdoor_load(struct dmub_srv *dmub, const struct dmub_window *cw0, const struct dmub_window *cw1);
+void dmub_dcn42_backdoor_load_zfb_mode(struct dmub_srv *dmub, const struct dmub_window *cw0, const struct dmub_window *cw1);
+void dmub_dcn42_setup_windows(struct dmub_srv *dmub, const struct dmub_window *cw2, const struct dmub_window *cw3, const struct dmub_window *cw4, const struct dmub_window *cw5, const struct dmub_window *cw6, const struct dmub_window *region6);
+
+/* Mailbox operations - Inbox1 */
+void dmub_dcn42_setup_mailbox(struct dmub_srv *dmub, const struct dmub_region *inbox1);
+uint32_t dmub_dcn42_get_inbox1_wptr(struct dmub_srv *dmub);
+uint32_t dmub_dcn42_get_inbox1_rptr(struct dmub_srv *dmub);
+void dmub_dcn42_set_inbox1_wptr(struct dmub_srv *dmub, uint32_t wptr_offset);
+
+/* Mailbox operations - Outbox1 */
+void dmub_dcn42_setup_out_mailbox(struct dmub_srv *dmub, const struct dmub_region *outbox1);
+uint32_t dmub_dcn42_get_outbox1_wptr(struct dmub_srv *dmub);
+void dmub_dcn42_set_outbox1_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
+
+/* Mailbox operations - Outbox0 */
+void dmub_dcn42_setup_outbox0(struct dmub_srv *dmub, const struct dmub_region *outbox0);
+uint32_t dmub_dcn42_get_outbox0_wptr(struct dmub_srv *dmub);
+void dmub_dcn42_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
+
+/* Mailbox operations - Inbox0 */
+void dmub_dcn42_send_inbox0_cmd(struct dmub_srv *dmub, union dmub_inbox0_data_register data);
+void dmub_dcn42_clear_inbox0_ack_register(struct dmub_srv *dmub);
+uint32_t dmub_dcn42_read_inbox0_ack_register(struct dmub_srv *dmub);
+
+/* REG Inbox0/Outbox0 operations */
+void dmub_dcn42_send_reg_inbox0_cmd_msg(struct dmub_srv *dmub, union dmub_rb_cmd *cmd);
+uint32_t dmub_dcn42_read_reg_inbox0_rsp_int_status(struct dmub_srv *dmub);
+void dmub_dcn42_read_reg_inbox0_cmd_rsp(struct dmub_srv *dmub, union dmub_rb_cmd *cmd);
+void dmub_dcn42_write_reg_inbox0_rsp_int_ack(struct dmub_srv *dmub);
+void dmub_dcn42_clear_reg_inbox0_rsp_int_ack(struct dmub_srv *dmub);
+void dmub_dcn42_enable_reg_inbox0_rsp_int(struct dmub_srv *dmub, bool enable);
+
+void dmub_dcn42_write_reg_outbox0_rdy_int_ack(struct dmub_srv *dmub);
+void dmub_dcn42_read_reg_outbox0_msg(struct dmub_srv *dmub, uint32_t *msg);
+void dmub_dcn42_write_reg_outbox0_rsp(struct dmub_srv *dmub, uint32_t *rsp);
+uint32_t dmub_dcn42_read_reg_outbox0_rsp_int_status(struct dmub_srv *dmub);
+void dmub_dcn42_enable_reg_outbox0_rdy_int(struct dmub_srv *dmub, bool enable);
+uint32_t dmub_dcn42_read_reg_outbox0_rdy_int_status(struct dmub_srv *dmub);
+
+/* GPINT operations */
+void dmub_dcn42_set_gpint(struct dmub_srv *dmub, union dmub_gpint_data_register reg);
+bool dmub_dcn42_is_gpint_acked(struct dmub_srv *dmub, union dmub_gpint_data_register reg);
+uint32_t dmub_dcn42_get_gpint_response(struct dmub_srv *dmub);
+uint32_t dmub_dcn42_get_gpint_dataout(struct dmub_srv *dmub);
+
+/* Status and detection */
+bool dmub_dcn42_is_hw_init(struct dmub_srv *dmub);
+bool dmub_dcn42_is_supported(struct dmub_srv *dmub);
+bool dmub_dcn42_is_hw_powered_up(struct dmub_srv *dmub);
+bool dmub_dcn42_should_detect(struct dmub_srv *dmub);
+
+
+/* Firmware boot status and options */
+union dmub_fw_boot_status dmub_dcn42_get_fw_boot_status(struct dmub_srv *dmub);
+union dmub_fw_boot_options dmub_dcn42_get_fw_boot_option(struct dmub_srv *dmub);
+
+/* Timing and diagnostics */
+uint32_t dmub_dcn42_get_current_time(struct dmub_srv *dmub);
+void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub);
+bool dmub_dcn42_get_preos_fw_info(struct dmub_srv *dmub);
+
+#endif /* _DMUB_DCN42_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
index 83cf4888fb540..3bba256a288d7 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: MIT
 /*
- * Copyright 2019 Advanced Micro Devices, Inc.
+ * Copyright 2019-2026 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -22,7 +23,6 @@
  * Authors: AMD
  *
  */
-
 #include "../dmub_srv.h"
 #include "dmub_dcn20.h"
 #include "dmub_dcn21.h"
@@ -40,6 +40,7 @@
 #include "dmub_dcn351.h"
 #include "dmub_dcn36.h"
 #include "dmub_dcn401.h"
+#include "dmub_dcn42.h"
 #include "os_types.h"
 /*
  * Note: the DMUB service is standalone. No additional headers should be
@@ -87,6 +88,7 @@
 
 static struct dmub_srv_dcn32_regs dmub_srv_dcn32_regs;
 static struct dmub_srv_dcn35_regs dmub_srv_dcn35_regs;
+struct dmub_srv_dcn42_regs dmub_srv_dcn42_regs;
 
 static inline uint32_t dmub_align(uint32_t val, uint32_t factor)
 {
@@ -410,6 +412,64 @@ static bool dmub_srv_hw_setup(struct dmub_srv *dmub, enum dmub_asic asic)
 			funcs->is_hw_powered_up = dmub_dcn35_is_hw_powered_up;
 			funcs->should_detect = dmub_dcn35_should_detect;
 			break;
+	case DMUB_ASIC_DCN42:
+			dmub->regs_dcn42 = &dmub_srv_dcn42_regs;
+			funcs->configure_dmub_in_system_memory = dmub_dcn42_configure_dmub_in_system_memory;
+			funcs->send_inbox0_cmd = dmub_dcn42_send_inbox0_cmd;
+			funcs->clear_inbox0_ack_register = dmub_dcn42_clear_inbox0_ack_register;
+			funcs->read_inbox0_ack_register = dmub_dcn42_read_inbox0_ack_register;
+			funcs->reset = dmub_dcn42_reset;
+			funcs->reset_release = dmub_dcn42_reset_release;
+			funcs->backdoor_load = dmub_dcn42_backdoor_load;
+			funcs->backdoor_load_zfb_mode = dmub_dcn42_backdoor_load_zfb_mode;
+			funcs->setup_windows = dmub_dcn42_setup_windows;
+			funcs->setup_mailbox = dmub_dcn42_setup_mailbox;
+			funcs->get_inbox1_wptr = dmub_dcn42_get_inbox1_wptr;
+			funcs->get_inbox1_rptr = dmub_dcn42_get_inbox1_rptr;
+			funcs->set_inbox1_wptr = dmub_dcn42_set_inbox1_wptr;
+			funcs->setup_out_mailbox = dmub_dcn42_setup_out_mailbox;
+			funcs->get_outbox1_wptr = dmub_dcn42_get_outbox1_wptr;
+			funcs->set_outbox1_rptr = dmub_dcn42_set_outbox1_rptr;
+			funcs->is_supported = dmub_dcn42_is_supported;
+			funcs->is_hw_init = dmub_dcn42_is_hw_init;
+			funcs->set_gpint = dmub_dcn42_set_gpint;
+			funcs->is_gpint_acked = dmub_dcn42_is_gpint_acked;
+			funcs->get_gpint_response = dmub_dcn42_get_gpint_response;
+			funcs->get_gpint_dataout = dmub_dcn42_get_gpint_dataout;
+			funcs->get_fw_status = dmub_dcn42_get_fw_boot_status;
+			funcs->get_fw_boot_option = dmub_dcn42_get_fw_boot_option;
+			funcs->enable_dmub_boot_options = dmub_dcn42_enable_dmub_boot_options;
+			funcs->skip_dmub_panel_power_sequence = dmub_dcn42_skip_dmub_panel_power_sequence;
+			/*outbox0 call stacks*/
+			funcs->setup_outbox0 = dmub_dcn42_setup_outbox0;
+			funcs->get_outbox0_wptr = dmub_dcn42_get_outbox0_wptr;
+			funcs->set_outbox0_rptr = dmub_dcn42_set_outbox0_rptr;
+
+			funcs->get_current_time = dmub_dcn42_get_current_time;
+			funcs->get_diagnostic_data = dmub_dcn42_get_diagnostic_data;
+			funcs->get_preos_fw_info = dmub_dcn42_get_preos_fw_info;
+
+			/*carried from dcn401*/
+			funcs->send_reg_inbox0_cmd_msg = dmub_dcn42_send_reg_inbox0_cmd_msg;
+			funcs->read_reg_inbox0_rsp_int_status = dmub_dcn42_read_reg_inbox0_rsp_int_status;
+			funcs->read_reg_inbox0_cmd_rsp = dmub_dcn42_read_reg_inbox0_cmd_rsp;
+			funcs->write_reg_inbox0_rsp_int_ack = dmub_dcn42_write_reg_inbox0_rsp_int_ack;
+			funcs->clear_reg_inbox0_rsp_int_ack = dmub_dcn42_clear_reg_inbox0_rsp_int_ack;
+			funcs->enable_reg_inbox0_rsp_int = dmub_dcn42_enable_reg_inbox0_rsp_int;
+			default_inbox_type = DMUB_CMD_INTERFACE_FB; /*still default to FB for now*/
+
+			funcs->write_reg_outbox0_rdy_int_ack = dmub_dcn42_write_reg_outbox0_rdy_int_ack;
+			funcs->read_reg_outbox0_msg = dmub_dcn42_read_reg_outbox0_msg;
+			funcs->write_reg_outbox0_rsp = dmub_dcn42_write_reg_outbox0_rsp;
+			funcs->read_reg_outbox0_rdy_int_status = dmub_dcn42_read_reg_outbox0_rdy_int_status;
+			funcs->read_reg_outbox0_rsp_int_status = dmub_dcn42_read_reg_outbox0_rsp_int_status;
+			funcs->enable_reg_inbox0_rsp_int = dmub_dcn42_enable_reg_inbox0_rsp_int;
+			funcs->enable_reg_outbox0_rdy_int = dmub_dcn42_enable_reg_outbox0_rdy_int;
+			funcs->init_reg_offsets = dmub_srv_dcn42_regs_init;
+
+			funcs->is_hw_powered_up = dmub_dcn42_is_hw_powered_up;
+			funcs->should_detect = dmub_dcn42_should_detect;
+			break;
 
 	case DMUB_ASIC_DCN401:
 		dmub->regs_dcn401 = &dmub_srv_dcn401_regs;
diff --git a/drivers/gpu/drm/amd/display/include/bios_parser_types.h b/drivers/gpu/drm/amd/display/include/bios_parser_types.h
index f40dc612ec73b..b5d97b394131b 100644
--- a/drivers/gpu/drm/amd/display/include/bios_parser_types.h
+++ b/drivers/gpu/drm/amd/display/include/bios_parser_types.h
@@ -93,6 +93,8 @@ enum bp_external_encoder_control_action {
 	EXTERNAL_ENCODER_CONTROL_SETUP = 0xf,
 	EXTERNAL_ENCODER_CONTROL_UNBLANK = 0x10,
 	EXTERNAL_ENCODER_CONTROL_BLANK = 0x11,
+	EXTERNAL_ENCODER_CONTROL_DAC_LOAD_DETECT = 0x12,
+	EXTERNAL_ENCODER_CONTROL_DDC_SETUP = 0x14,
 };
 
 enum bp_pipe_control_action {
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index 8aea50aa95330..92510af1bd653 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -252,6 +252,7 @@ enum {
 #define AMDGPU_FAMILY_GC_11_0_0 145
 #define AMDGPU_FAMILY_GC_11_0_1 148
 #define AMDGPU_FAMILY_GC_11_5_0 150
+#define AMDGPU_FAMILY_GC_11_5_4	154
 #define GC_11_0_0_A0 0x1
 #define GC_11_0_2_A0 0x10
 #define GC_11_0_3_A0 0x20
diff --git a/drivers/gpu/drm/amd/display/include/dal_types.h b/drivers/gpu/drm/amd/display/include/dal_types.h
index a021d12acd741..d30eb24cf11e3 100644
--- a/drivers/gpu/drm/amd/display/include/dal_types.h
+++ b/drivers/gpu/drm/amd/display/include/dal_types.h
@@ -65,6 +65,7 @@ enum dce_version {
 	DCN_VERSION_3_51,
 	DCN_VERSION_3_6,
 	DCN_VERSION_4_01,
+	DCN_VERSION_4_2,
 	DCN_VERSION_MAX
 };
 
diff --git a/drivers/gpu/drm/amd/display/include/dpcd_defs.h b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
index 1afa10e85eb59..4a8ca0ac12666 100644
--- a/drivers/gpu/drm/amd/display/include/dpcd_defs.h
+++ b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
@@ -64,6 +64,9 @@
 #ifndef DP_PR_ERROR_STATUS                  // can remove this once the define gets into linux drm_dp_helper.h
 #define DP_PR_ERROR_STATUS                  0x2020  /* DP 2.0 */
 #endif /* DP_PR_ERROR_STATUS */
+#ifndef DP_PR_REPLAY_SINK_STATUS            // can remove this once the define gets into linux drm_dp_helper.h
+#define DP_PR_REPLAY_SINK_STATUS            0x2022
+#endif /* DP_PR_REPLAY_SINK_STATUS */
 #ifndef DP_PR_LINK_CRC_ERROR               // can remove this once the define gets into linux drm_dp_helper.h
 #define DP_PR_LINK_CRC_ERROR               (1 << 0)
 #endif /* DP_PR_LINK_CRC_ERROR */
diff --git a/drivers/gpu/drm/amd/include/amd_acpi.h b/drivers/gpu/drm/amd/include/amd_acpi.h
index 84933c07f720d..4225640131f2e 100644
--- a/drivers/gpu/drm/amd/include/amd_acpi.h
+++ b/drivers/gpu/drm/amd/include/amd_acpi.h
@@ -26,8 +26,6 @@
 
 #include <linux/types.h>
 
-#define ACPI_AC_CLASS           "ac_adapter"
-
 struct atif_verify_interface {
 	u16 size;		/* structure size in bytes (includes size field) */
 	u16 version;		/* version */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_offset.h
new file mode 100644
index 0000000000000..aa3ef3a340134
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_offset.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _clk_15_0_0_OFFSET_HEADER
+#define _clk_15_0_0_OFFSET_HEADER
+
+// addressBlock: clk_clk8_0_SmuClkDec
+// base address: 0x6e000
+#define regCLK8_CLK0_DS_CNTL                                                                            0x4c14
+#define regCLK8_CLK0_DS_CNTL_BASE_IDX                                                                   0
+#define regCLK8_CLK1_DS_CNTL                                                                            0x4c1c
+#define regCLK8_CLK1_DS_CNTL_BASE_IDX                                                                   0
+#define regCLK8_CLK2_DS_CNTL                                                                            0x4c24
+#define regCLK8_CLK2_DS_CNTL_BASE_IDX                                                                   0
+#define regCLK8_CLK3_DS_CNTL                                                                            0x4c2c
+#define regCLK8_CLK3_DS_CNTL_BASE_IDX                                                                   0
+#define regCLK8_CLK4_DS_CNTL                                                                            0x4c34
+#define regCLK8_CLK4_DS_CNTL_BASE_IDX                                                                   0
+#define regCLK8_CLK0_BYPASS_CNTL                                                                        0x4c1a
+#define regCLK8_CLK0_BYPASS_CNTL_BASE_IDX                                                               0
+#define regCLK8_CLK1_BYPASS_CNTL                                                                        0x4c22
+#define regCLK8_CLK1_BYPASS_CNTL_BASE_IDX                                                               0
+#define regCLK8_CLK2_BYPASS_CNTL                                                                        0x4c2a
+#define regCLK8_CLK2_BYPASS_CNTL_BASE_IDX                                                               0
+#define regCLK8_CLK3_BYPASS_CNTL                                                                        0x4c32
+#define regCLK8_CLK3_BYPASS_CNTL_BASE_IDX                                                               0
+#define regCLK8_CLK4_BYPASS_CNTL                                                                        0x4c3a
+#define regCLK8_CLK4_BYPASS_CNTL_BASE_IDX                                                               0
+#define regCLK8_CLK_TICK_CNT_CONFIG_REG                                                                 0x4c51
+#define regCLK8_CLK_TICK_CNT_CONFIG_REG_BASE_IDX                                                        0
+#define regCLK8_CLK_TICK_CNT_STATUS                                                                     0x4c52
+#define regCLK8_CLK_TICK_CNT_STATUS_BASE_IDX                                                            0
+#define regCLK8_CLK0_CURRENT_CNT                                                                        0x4c53
+#define regCLK8_CLK0_CURRENT_CNT_BASE_IDX                                                               0
+#define regCLK8_CLK1_CURRENT_CNT                                                                        0x4c54
+#define regCLK8_CLK1_CURRENT_CNT_BASE_IDX                                                               0
+#define regCLK8_CLK2_CURRENT_CNT                                                                        0x4c55
+#define regCLK8_CLK2_CURRENT_CNT_BASE_IDX                                                               0
+#define regCLK8_CLK3_CURRENT_CNT                                                                        0x4c56
+#define regCLK8_CLK3_CURRENT_CNT_BASE_IDX                                                               0
+#define regCLK8_CLK4_CURRENT_CNT                                                                        0x4c57
+#define regCLK8_CLK4_CURRENT_CNT_BASE_IDX                                                               0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_sh_mask.h
new file mode 100644
index 0000000000000..c78622d06e1be
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_15_0_0_sh_mask.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _clk_15_0_0_SH_MASK_HEADER
+#define _clk_15_0_0_SH_MASK_HEADER
+
+// addressBlock: clk_clk8_0_SmuClkDec
+//CLK8_CLK_TICK_CNT_CONFIG_REG
+#define CLK8_CLK_TICK_CNT_CONFIG_REG__TIMER_THRESHOLD__SHIFT                                                   0x0
+#define CLK8_CLK_TICK_CNT_CONFIG_REG__TIMER_THRESHOLD_MASK                                                     0xFFFFL
+//CLK8_CLK0_BYPASS_CNTL
+#define CLK8_CLK0_BYPASS_CNTL__CLK0_BYPASS_SEL__SHIFT                                                         0x0
+#define CLK8_CLK0_BYPASS_CNTL__CLK0_BYPASS_SEL_MASK                                                           0x00000007L
+//CLK8_CLK1_BYPASS_CNTL
+#define CLK8_CLK1_BYPASS_CNTL__CLK1_BYPASS_SEL__SHIFT                                                         0x0
+#define CLK8_CLK1_BYPASS_CNTL__CLK1_BYPASS_SEL_MASK                                                           0x00000007L
+//CLK8_CLK2_BYPASS_CNTL
+#define CLK8_CLK2_BYPASS_CNTL__CLK2_BYPASS_SEL__SHIFT                                                         0x0
+#define CLK8_CLK2_BYPASS_CNTL__CLK2_BYPASS_SEL_MASK                                                           0x00000007L
+//CLK8_CLK3_BYPASS_CNTL
+#define CLK8_CLK3_BYPASS_CNTL__CLK3_BYPASS_SEL__SHIFT                                                         0x0
+#define CLK8_CLK3_BYPASS_CNTL__CLK3_BYPASS_SEL_MASK                                                           0x00000007L
+//CLK8_CLK4_BYPASS_CNTL
+#define CLK8_CLK4_BYPASS_CNTL__CLK4_BYPASS_SEL__SHIFT                                                         0x0
+#define CLK8_CLK4_BYPASS_CNTL__CLK4_BYPASS_SEL_MASK                                                           0x00000007L
+//CLK8_CLK0_DS_CNTL
+#define CLK8_CLK0_DS_CNTL__CLK0_DS_DIV_ID__SHIFT                                                              0x0
+#define CLK8_CLK0_DS_CNTL__CLK0_DS_DIV_ID_MASK                                                                0x0000000FL
+#define CLK8_CLK0_DS_CNTL__CLK0_ALLOW_DS__SHIFT                                                              0x4
+#define CLK8_CLK0_DS_CNTL__CLK0_ALLOW_DS_MASK                                                                0x00000010L
+//CLK8_CLK1_DS_CNTL
+#define CLK8_CLK1_DS_CNTL__CLK1_DS_DIV_ID__SHIFT                                                              0x0
+#define CLK8_CLK1_DS_CNTL__CLK1_DS_DIV_ID_MASK                                                                0x0000000FL
+#define CLK8_CLK1_DS_CNTL__CLK1_ALLOW_DS__SHIFT                                                              0x4
+#define CLK8_CLK1_DS_CNTL__CLK1_ALLOW_DS_MASK                                                                0x00000010L
+//CLK8_CLK2_DS_CNTL
+#define CLK8_CLK2_DS_CNTL__CLK2_DS_DIV_ID__SHIFT                                                              0x0
+#define CLK8_CLK2_DS_CNTL__CLK2_DS_DIV_ID_MASK                                                                0x0000000FL
+#define CLK8_CLK2_DS_CNTL__CLK2_ALLOW_DS__SHIFT                                                              0x4
+#define CLK8_CLK2_DS_CNTL__CLK2_ALLOW_DS_MASK                                                                0x00000010L
+//CLK8_CLK3_DS_CNTL
+#define CLK8_CLK3_DS_CNTL__CLK3_DS_DIV_ID__SHIFT                                                              0x0
+#define CLK8_CLK3_DS_CNTL__CLK3_DS_DIV_ID_MASK                                                                0x0000000FL
+#define CLK8_CLK3_DS_CNTL__CLK3_ALLOW_DS__SHIFT                                                              0x4
+#define CLK8_CLK3_DS_CNTL__CLK3_ALLOW_DS_MASK                                                                0x00000010L
+//CLK8_CLK4_DS_CNTL
+#define CLK8_CLK4_DS_CNTL__CLK4_DS_DIV_ID__SHIFT                                                              0x0
+#define CLK8_CLK4_DS_CNTL__CLK4_DS_DIV_ID_MASK                                                                0x0000000FL
+#define CLK8_CLK4_DS_CNTL__CLK4_ALLOW_DS__SHIFT                                                              0x4
+#define CLK8_CLK4_DS_CNTL__CLK4_ALLOW_DS_MASK                                                                0x00000010L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_offset.h
new file mode 100644
index 0000000000000..52fbf2dc18995
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_offset.h
@@ -0,0 +1,17874 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _dcn_4_2_0_OFFSET_HEADER
+#define _dcn_4_2_0_OFFSET_HEADER
+
+
+
+// addressBlock: dce_dc_hda_azcontroller_azdec
+// base address: 0x0
+#define regAZCONTROLLER0_CORB_WRITE_POINTER                                                             0x0000
+#define regAZCONTROLLER0_CORB_WRITE_POINTER_BASE_IDX                                                    0
+#define regAZCONTROLLER0_CORB_READ_POINTER                                                              0x0000
+#define regAZCONTROLLER0_CORB_READ_POINTER_BASE_IDX                                                     0
+#define regAZCONTROLLER0_CORB_CONTROL                                                                   0x0001
+#define regAZCONTROLLER0_CORB_CONTROL_BASE_IDX                                                          0
+#define regAZCONTROLLER0_CORB_STATUS                                                                    0x0001
+#define regAZCONTROLLER0_CORB_STATUS_BASE_IDX                                                           0
+#define regAZCONTROLLER0_CORB_SIZE                                                                      0x0001
+#define regAZCONTROLLER0_CORB_SIZE_BASE_IDX                                                             0
+#define regAZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS                                                        0x0002
+#define regAZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS_BASE_IDX                                               0
+#define regAZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS                                                        0x0003
+#define regAZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS_BASE_IDX                                               0
+#define regAZCONTROLLER0_RIRB_WRITE_POINTER                                                             0x0004
+#define regAZCONTROLLER0_RIRB_WRITE_POINTER_BASE_IDX                                                    0
+#define regAZCONTROLLER0_RESPONSE_INTERRUPT_COUNT                                                       0x0004
+#define regAZCONTROLLER0_RESPONSE_INTERRUPT_COUNT_BASE_IDX                                              0
+#define regAZCONTROLLER0_RIRB_CONTROL                                                                   0x0005
+#define regAZCONTROLLER0_RIRB_CONTROL_BASE_IDX                                                          0
+#define regAZCONTROLLER0_RIRB_STATUS                                                                    0x0005
+#define regAZCONTROLLER0_RIRB_STATUS_BASE_IDX                                                           0
+#define regAZCONTROLLER0_RIRB_SIZE                                                                      0x0005
+#define regAZCONTROLLER0_RIRB_SIZE_BASE_IDX                                                             0
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE                                             0x0006
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_BASE_IDX                                    0
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                                        0x0006
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                               0
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                                       0x0006
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                              0
+#define regAZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE                                             0x0007
+#define regAZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE_BASE_IDX                                    0
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_STATUS                                                       0x0008
+#define regAZCONTROLLER0_IMMEDIATE_COMMAND_STATUS_BASE_IDX                                              0
+#define regAZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS                                                0x000a
+#define regAZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS_BASE_IDX                                       0
+#define regAZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS                                                0x000b
+#define regAZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS_BASE_IDX                                       0
+#define regAZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS                                                       0x074c
+#define regAZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS_BASE_IDX                                              1
+
+
+// addressBlock: dc_perfmon_dc_perfmondebugind
+// base address: 0x0
+#define ixPERFMON_DEBUG_ID                                                                             0x0000
+#define ixPERFMON_DEBUG01                                                                              0x0001
+#define ixPERFMON_DEBUG02                                                                              0x0002
+#define ixPERFMON_DEBUG03                                                                              0x0003
+#define ixPERFMON_DEBUG04                                                                              0x0004
+#define ixPERFMON_DEBUG05                                                                              0x0005
+#define ixPERFMON_DEBUG06                                                                              0x0006
+#define ixPERFMON_DEBUG07                                                                              0x0007
+#define ixPERFMON_DEBUG08                                                                              0x0008
+#define ixPERFMON_DEBUG09                                                                              0x0009
+#define ixPERFMON_DEBUG0A                                                                              0x000a
+#define ixPERFMON_DEBUG0B                                                                              0x000b
+#define ixPERFMON_DEBUG0C                                                                              0x000c
+#define ixPERFMON_DEBUG0D                                                                              0x000d
+#define ixPERFMON_DEBUG0E                                                                              0x000e
+#define ixPERFMON_DEBUG0F                                                                              0x000f
+#define ixPERFMON_DEBUG10                                                                              0x0010
+#define ixPERFMON_DEBUG11                                                                              0x0011
+#define ixPERFMON_DEBUG12                                                                              0x0012
+
+
+// addressBlock: mcif_wb0_mcif_wbdebugind
+// base address: 0x0
+#define ixMCIF_WB_DEBUG_ID                                                                             0x0000
+#define ixID01_WB_FMT_DBG                                                                              0x0001
+#define ixID02_WB_FMT_DBG                                                                              0x0002
+#define ixID03_WB_FMT_DBG                                                                              0x0003
+#define ixID04_WB_MGR_DBG                                                                              0x0004
+#define ixID05_WB_MGR_DBG                                                                              0x0005
+#define ixID06_WB_MGR_DBG                                                                              0x0006
+#define ixID07_WB_MGR_DBG                                                                              0x0007
+#define ixID08_WB_ARB_DBG                                                                              0x0008
+#define ixID09_WB_ARB_DBG                                                                              0x0009
+#define ixID0A_WB_ARB_DBG                                                                              0x000a
+#define ixID0B_WB_ARB_DBG                                                                              0x000b
+#define ixID0C_WB_ARB_DBG                                                                              0x000c
+#define ixID0D_WB_ARB_DBG                                                                              0x000d
+#define ixID0E_WB_ARB_DBG                                                                              0x000e
+#define ixID0F_P010_WB_FMT_DBG_Y                                                                       0x000f
+#define ixID10_P010_WB_FMT_DBG_C                                                                       0x0010
+#define ixID11_WB_ARB_P010_DBG                                                                         0x0011
+#define ixID12_WB_ARB_P010_DBG                                                                         0x0012
+
+
+// addressBlock: hubbub_dchubbubdebugind
+// base address: 0x0
+#define ixDCHUBBUB_KEY_STATUS_DEBUG_1                                                                  0x005f
+#define ixDCHUBBUB_KEY_STATUS_DEBUG_2                                                                  0x0060
+
+
+// addressBlock: odm0_odmdebugind
+// base address: 0x0
+#define ixODM0_OPTC_INPUT_DEBUG_ID                                                                     0x0000
+#define ixODM0_OPTC_INPUT_DEBUG0                                                                       0x0001
+#define ixODM0_OPTC_INPUT_DEBUG1                                                                       0x0002
+#define ixODM0_OPTC_INPUT_DEBUG2                                                                       0x0003
+
+
+// addressBlock: odm1_odmdebugind
+// base address: 0x0
+#define ixODM1_OPTC_INPUT_DEBUG_ID                                                                     0x0000
+#define ixODM1_OPTC_INPUT_DEBUG0                                                                       0x0001
+#define ixODM1_OPTC_INPUT_DEBUG1                                                                       0x0002
+#define ixODM1_OPTC_INPUT_DEBUG2                                                                       0x0003
+
+
+// addressBlock: odm2_odmdebugind
+// base address: 0x0
+#define ixODM2_OPTC_INPUT_DEBUG_ID                                                                     0x0000
+#define ixODM2_OPTC_INPUT_DEBUG0                                                                       0x0001
+#define ixODM2_OPTC_INPUT_DEBUG1                                                                       0x0002
+#define ixODM2_OPTC_INPUT_DEBUG2                                                                       0x0003
+
+
+// addressBlock: odm3_odmdebugind
+// base address: 0x0
+#define ixODM3_OPTC_INPUT_DEBUG_ID                                                                     0x0000
+#define ixODM3_OPTC_INPUT_DEBUG0                                                                       0x0001
+#define ixODM3_OPTC_INPUT_DEBUG1                                                                       0x0002
+#define ixODM3_OPTC_INPUT_DEBUG2                                                                       0x0003
+
+
+// addressBlock: rbbmif_rbbmifdebugind
+// base address: 0x0
+#define ixRBBMIF_DEBUG_ID                                                                              0x0000
+#define ixRBBMIF_DEBUG_0                                                                               0x0001
+#define ixRBBMIF_DEBUG_1                                                                               0x0002
+#define ixRBBMIF_DEBUG_2                                                                               0x0003
+#define ixRBBMIF_DEBUG_3                                                                               0x0004
+#define ixRBBMIF_DEBUG_4                                                                               0x0005
+#define ixRBBMIF_DEBUG_5                                                                               0x0006
+#define ixRBBMIF_DEBUG_6                                                                               0x0007
+#define ixRBBMIF_DEBUG_7                                                                               0x0008
+#define ixRBBMIF_DEBUG_8                                                                               0x0009
+#define ixRBBMIF_DEBUG_9                                                                               0x000a
+#define ixRBBMIF_DEBUG_10                                                                              0x000b
+#define ixRBBMIF_DEBUG_11                                                                              0x000c
+#define ixRBBMIF_DEBUG_12                                                                              0x000d
+#define ixRBBMIF_DEBUG_13                                                                              0x000e
+#define ixRBBMIF_DEBUG_14                                                                              0x000f
+#define ixRBBMIF_DEBUG_15                                                                              0x0010
+#define ixRBBMIF_DEBUG_16                                                                              0x0011
+#define ixRBBMIF_DEBUG_17                                                                              0x0012
+#define ixRBBMIF_DEBUG_18                                                                              0x0013
+#define ixRBBMIF_DEBUG_19                                                                              0x0014
+#define ixRBBMIF_DEBUG_20                                                                              0x0015
+#define ixRBBMIF_DEBUG_21                                                                              0x0016
+#define ixRBBMIF_DEBUG_22                                                                              0x0017
+#define ixRBBMIF_DEBUG_23                                                                              0x0018
+#define ixRBBMIF_DEBUG_24                                                                              0x0019
+#define ixRBBMIF_DEBUG_25                                                                              0x001a
+#define ixRBBMIF_DEBUG_26                                                                              0x001b
+#define ixRBBMIF_DEBUG_27                                                                              0x001c
+#define ixRBBMIF_DEBUG_28                                                                              0x001d
+#define ixRBBMIF_DEBUG_29                                                                              0x001e
+#define ixRBBMIF_DEBUG_30                                                                              0x001f
+#define ixRBBMIF_DEBUG_31                                                                              0x0020
+#define ixRBBMIF_DEBUG_32                                                                              0x0021
+#define ixRBBMIF_DEBUG_33                                                                              0x0022
+#define ixRBBMIF_DEBUG_34                                                                              0x0023
+#define ixRBBMIF_DEBUG_35                                                                              0x0024
+#define ixRBBMIF_DEBUG_36                                                                              0x0025
+#define ixRBBMIF_DEBUG_37                                                                              0x0026
+#define ixRBBMIF_DEBUG_38                                                                              0x0027
+#define ixRBBMIF_DEBUG_39                                                                              0x0028
+#define ixRBBMIF_DEBUG_40                                                                              0x0029
+#define ixRBBMIF_DEBUG_41                                                                              0x002a
+#define ixRBBMIF_DEBUG_42                                                                              0x002b
+#define ixRBBMIF_DEBUG_43                                                                              0x002c
+#define ixRBBMIF_DEBUG_44                                                                              0x002d
+#define ixRBBMIF_DEBUG_45                                                                              0x002e
+#define ixRBBMIF_DEBUG_46                                                                              0x002f
+#define ixRBBMIF_DEBUG_47                                                                              0x0030
+#define ixRBBMIF_DEBUG_48                                                                              0x0031
+#define ixRBBMIF_DEBUG_49                                                                              0x0032
+#define ixRBBMIF_DEBUG_50                                                                              0x0033
+#define ixRBBMIF_DEBUG_51                                                                              0x0034
+#define ixRBBMIF_DEBUG_52                                                                              0x0035
+
+
+// addressBlock: ihc_ihcdebugind
+// base address: 0x0
+#define ixIHC_DEBUG_ID                                                                                 0x0000
+#define ixIHC_CLIENT_DEBUG_A                                                                           0x0001
+#define ixIHC_CLIENT_DEBUG_B                                                                           0x0002
+#define ixIHC_CLIENT_DEBUG_C                                                                           0x0003
+#define ixIHC_CLIENT_DEBUG_D                                                                           0x0004
+#define ixIHC_CLIENT_DEBUG_E                                                                           0x0005
+
+
+// addressBlock: dmu_misc_dmumiscdebugind
+// base address: 0x0
+#define ixDMU_MISC_DEBUG_ID                                                                            0x0000
+#define ixDMU_MISC_DEBUG_1                                                                             0x0001
+#define ixDMU_ZSC_TEST_DEBUG1_DATA                                                                     0x0002
+#define ixDMU_ZSC_TEST_DEBUG2_DATA                                                                     0x0003
+
+
+// addressBlock: dc_pg_dc_pgdebugind
+// base address: 0x0
+#define ixDCPG_DEBUG_ID                                                                                0x0000
+#define ixDCPG_CONTROL_DEBUG_BUS0                                                                      0x0001
+#define ixDCPG_CONTROL_DEBUG_BUS1                                                                      0x0002
+#define ixDCPG_CONTROL_DEBUG_BUS2                                                                      0x0003
+#define ixDCPG_CONTROL_DEBUG_BUS3                                                                      0x0004
+#define ixDCPG_CONTROL_DEBUG_BUS16                                                                     0x0011
+#define ixDCPG_CONTROL_DEBUG_BUS17                                                                     0x0012
+#define ixDCPG_CONTROL_DEBUG_BUS18                                                                     0x0013
+#define ixDCPG_CONTROL_DEBUG_BUS19                                                                     0x0014
+#define ixDCPG_CONTROL_DEBUG_BUS22                                                                     0x0017
+#define ixDCPG_CONTROL_DEBUG_BUS23                                                                     0x0018
+#define ixDCPG_CONTROL_DEBUG_BUS24                                                                     0x0019
+#define ixDCPG_CONTROL_DEBUG_BUS25                                                                     0x001a
+#define ixDCPG_CONTROL_DEBUG_BUS26                                                                     0x001b
+
+
+// addressBlock: dp0_dpdebugind
+// base address: 0x0
+#define ixDP0_DP_DEBUG_ID                                                                              0x0000
+#define ixDP0_DP_DEBUG_K                                                                               0x00a2
+#define ixDP0_DP_DEBUG_L                                                                               0x00a3
+#define ixDP0_DP_DEBUG_M                                                                               0x00a4
+#define ixDP0_DP_DEBUG_G                                                                               0x00b4
+#define ixDP0_DP_DEBUG_O                                                                               0x00fb
+#define ixDP0_DP_DEBUG_P                                                                               0x00fc
+#define ixDP0_DP_DEBUG_Q                                                                               0x00fd
+#define ixDP0_DP_DEBUG_R                                                                               0x00fe
+#define ixDP0_DP_DEBUG_S                                                                               0x00ff
+
+
+// addressBlock: dp0_dpfedebugind
+// base address: 0x0
+#define ixDP0_DP_FE_DEBUG_ID                                                                           0x0000
+#define ixDP0_DP_FE_SYMCLK_DEBUG_DATA_0                                                                0x0001
+#define ixDP0_DP_FE_SYMCLK_DEBUG_DATA_1                                                                0x0002
+#define ixDP0_DP_FE_SYMCLK_DEBUG_DATA_2                                                                0x0003
+#define ixDP0_DP_FE_SYMCLK_DEBUG_DATA_3                                                                0x0004
+#define ixDP0_DP_FE_SYMCLK_DEBUG_DATA_4                                                                0x0005
+#define ixDP0_DP_DEBUG_T                                                                               0x0090
+#define ixDP0_DP_DEBUG_U                                                                               0x0091
+#define ixDP0_DP_DEBUG_V                                                                               0x0092
+#define ixDP0_DP_DEBUG_W                                                                               0x0093
+#define ixDP0_DP_DEBUG_X                                                                               0x0094
+#define ixDP0_DP_DEBUG_Y                                                                               0x0095
+#define ixDP0_DP_DEBUG_Z                                                                               0x0096
+#define ixDP0_DP_DEBUG_1A                                                                              0x0097
+#define ixDP0_DP_DEBUG_1B                                                                              0x0098
+#define ixDP0_DP_DEBUG_I                                                                               0x00a0
+#define ixDP0_DP_DEBUG_J                                                                               0x00a1
+#define ixDP0_DP_DEBUG_N                                                                               0x00a2
+#define ixDP0_DP_DEBUG_H                                                                               0x00ac
+#define ixDP0_DP_DEBUG_A                                                                               0x00ad
+#define ixDP0_DP_DEBUG_B                                                                               0x00ae
+#define ixDP0_DP_DEBUG_C                                                                               0x00af
+#define ixDP0_DP_DEBUG_D                                                                               0x00b0
+#define ixDP0_DP_DEBUG_E                                                                               0x00b2
+#define ixDP0_DP_DEBUG_F                                                                               0x00b3
+
+
+// addressBlock: dp0_dpfe_dprefclk_debugind
+// base address: 0x0
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_ID                                                                  0x0000
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_0                                                              0x0001
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_1                                                              0x0002
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_2                                                              0x0003
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_3                                                              0x0004
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_4                                                              0x0005
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_5                                                              0x0006
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_6                                                              0x0007
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_7                                                              0x0008
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_8                                                              0x0009
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_9                                                              0x000a
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_10                                                             0x000b
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_11                                                             0x000c
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_12                                                             0x000d
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_13                                                             0x000e
+#define ixDP0_DP_FE_DPREFCLK_DEBUG_DATA_14                                                             0x000f
+
+
+// addressBlock: dig0_digdebugind
+// base address: 0x0
+#define ixDIG0_DIG_BE_CLKGEN_DEBUG0                                                                    0x0010
+
+
+// addressBlock: dig0_digfedebugind
+// base address: 0x0
+#define ixDIG0_DIG_FE_DEBUG_ID                                                                         0x0000
+#define ixDIG0_DIG_VPG_DEBUG0                                                                          0x000b
+#define ixDIG0_DIG_VPG_DEBUG1                                                                          0x000c
+#define ixDIG0_DIG_VPG_DEBUG2                                                                          0x000d
+#define ixDIG0_DIG_VPG_DEBUG3                                                                          0x000e
+
+
+// addressBlock: dp1_dpdebugind
+// base address: 0x0
+#define ixDP1_DP_DEBUG_ID                                                                              0x0000
+#define ixDP1_DP_DEBUG_K                                                                               0x00a2
+#define ixDP1_DP_DEBUG_L                                                                               0x00a3
+#define ixDP1_DP_DEBUG_M                                                                               0x00a4
+#define ixDP1_DP_DEBUG_G                                                                               0x00b4
+#define ixDP1_DP_DEBUG_O                                                                               0x00fb
+#define ixDP1_DP_DEBUG_P                                                                               0x00fc
+#define ixDP1_DP_DEBUG_Q                                                                               0x00fd
+#define ixDP1_DP_DEBUG_R                                                                               0x00fe
+#define ixDP1_DP_DEBUG_S                                                                               0x00ff
+
+
+// addressBlock: dp1_dpfedebugind
+// base address: 0x0
+#define ixDP1_DP_FE_DEBUG_ID                                                                           0x0000
+#define ixDP1_DP_FE_SYMCLK_DEBUG_DATA_0                                                                0x0001
+#define ixDP1_DP_FE_SYMCLK_DEBUG_DATA_1                                                                0x0002
+#define ixDP1_DP_FE_SYMCLK_DEBUG_DATA_2                                                                0x0003
+#define ixDP1_DP_FE_SYMCLK_DEBUG_DATA_3                                                                0x0004
+#define ixDP1_DP_FE_SYMCLK_DEBUG_DATA_4                                                                0x0005
+#define ixDP1_DP_DEBUG_T                                                                               0x0090
+#define ixDP1_DP_DEBUG_U                                                                               0x0091
+#define ixDP1_DP_DEBUG_V                                                                               0x0092
+#define ixDP1_DP_DEBUG_W                                                                               0x0093
+#define ixDP1_DP_DEBUG_X                                                                               0x0094
+#define ixDP1_DP_DEBUG_Y                                                                               0x0095
+#define ixDP1_DP_DEBUG_Z                                                                               0x0096
+#define ixDP1_DP_DEBUG_1A                                                                              0x0097
+#define ixDP1_DP_DEBUG_1B                                                                              0x0098
+#define ixDP1_DP_DEBUG_I                                                                               0x00a0
+#define ixDP1_DP_DEBUG_J                                                                               0x00a1
+#define ixDP1_DP_DEBUG_N                                                                               0x00a2
+#define ixDP1_DP_DEBUG_H                                                                               0x00ac
+#define ixDP1_DP_DEBUG_A                                                                               0x00ad
+#define ixDP1_DP_DEBUG_B                                                                               0x00ae
+#define ixDP1_DP_DEBUG_C                                                                               0x00af
+#define ixDP1_DP_DEBUG_D                                                                               0x00b0
+#define ixDP1_DP_DEBUG_E                                                                               0x00b2
+#define ixDP1_DP_DEBUG_F                                                                               0x00b3
+
+
+// addressBlock: dp1_dpfe_dprefclk_debugind
+// base address: 0x0
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_ID                                                                  0x0000
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_0                                                              0x0001
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_1                                                              0x0002
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_2                                                              0x0003
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_3                                                              0x0004
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_4                                                              0x0005
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_5                                                              0x0006
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_6                                                              0x0007
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_7                                                              0x0008
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_8                                                              0x0009
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_9                                                              0x000a
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_10                                                             0x000b
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_11                                                             0x000c
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_12                                                             0x000d
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_13                                                             0x000e
+#define ixDP1_DP_FE_DPREFCLK_DEBUG_DATA_14                                                             0x000f
+
+
+// addressBlock: dig1_digdebugind
+// base address: 0x0
+#define ixDIG1_DIG_BE_CLKGEN_DEBUG0                                                                    0x0010
+
+
+// addressBlock: dig1_digfedebugind
+// base address: 0x0
+#define ixDIG1_DIG_FE_DEBUG_ID                                                                         0x0000
+#define ixDIG1_DIG_VPG_DEBUG0                                                                          0x000b
+#define ixDIG1_DIG_VPG_DEBUG1                                                                          0x000c
+#define ixDIG1_DIG_VPG_DEBUG2                                                                          0x000d
+#define ixDIG1_DIG_VPG_DEBUG3                                                                          0x000e
+
+
+// addressBlock: dp2_dpdebugind
+// base address: 0x0
+#define ixDP2_DP_DEBUG_ID                                                                              0x0000
+#define ixDP2_DP_DEBUG_K                                                                               0x00a2
+#define ixDP2_DP_DEBUG_L                                                                               0x00a3
+#define ixDP2_DP_DEBUG_M                                                                               0x00a4
+#define ixDP2_DP_DEBUG_G                                                                               0x00b4
+#define ixDP2_DP_DEBUG_O                                                                               0x00fb
+#define ixDP2_DP_DEBUG_P                                                                               0x00fc
+#define ixDP2_DP_DEBUG_Q                                                                               0x00fd
+#define ixDP2_DP_DEBUG_R                                                                               0x00fe
+#define ixDP2_DP_DEBUG_S                                                                               0x00ff
+
+
+// addressBlock: dp2_dpfedebugind
+// base address: 0x0
+#define ixDP2_DP_FE_DEBUG_ID                                                                           0x0000
+#define ixDP2_DP_FE_SYMCLK_DEBUG_DATA_0                                                                0x0001
+#define ixDP2_DP_FE_SYMCLK_DEBUG_DATA_1                                                                0x0002
+#define ixDP2_DP_FE_SYMCLK_DEBUG_DATA_2                                                                0x0003
+#define ixDP2_DP_FE_SYMCLK_DEBUG_DATA_3                                                                0x0004
+#define ixDP2_DP_FE_SYMCLK_DEBUG_DATA_4                                                                0x0005
+#define ixDP2_DP_DEBUG_T                                                                               0x0090
+#define ixDP2_DP_DEBUG_U                                                                               0x0091
+#define ixDP2_DP_DEBUG_V                                                                               0x0092
+#define ixDP2_DP_DEBUG_W                                                                               0x0093
+#define ixDP2_DP_DEBUG_X                                                                               0x0094
+#define ixDP2_DP_DEBUG_Y                                                                               0x0095
+#define ixDP2_DP_DEBUG_Z                                                                               0x0096
+#define ixDP2_DP_DEBUG_1A                                                                              0x0097
+#define ixDP2_DP_DEBUG_1B                                                                              0x0098
+#define ixDP2_DP_DEBUG_I                                                                               0x00a0
+#define ixDP2_DP_DEBUG_J                                                                               0x00a1
+#define ixDP2_DP_DEBUG_N                                                                               0x00a2
+#define ixDP2_DP_DEBUG_H                                                                               0x00ac
+#define ixDP2_DP_DEBUG_A                                                                               0x00ad
+#define ixDP2_DP_DEBUG_B                                                                               0x00ae
+#define ixDP2_DP_DEBUG_C                                                                               0x00af
+#define ixDP2_DP_DEBUG_D                                                                               0x00b0
+#define ixDP2_DP_DEBUG_E                                                                               0x00b2
+#define ixDP2_DP_DEBUG_F                                                                               0x00b3
+
+
+// addressBlock: dp2_dpfe_dprefclk_debugind
+// base address: 0x0
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_ID                                                                  0x0000
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_0                                                              0x0001
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_1                                                              0x0002
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_2                                                              0x0003
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_3                                                              0x0004
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_4                                                              0x0005
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_5                                                              0x0006
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_6                                                              0x0007
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_7                                                              0x0008
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_8                                                              0x0009
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_9                                                              0x000a
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_10                                                             0x000b
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_11                                                             0x000c
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_12                                                             0x000d
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_13                                                             0x000e
+#define ixDP2_DP_FE_DPREFCLK_DEBUG_DATA_14                                                             0x000f
+
+
+// addressBlock: dig2_digdebugind
+// base address: 0x0
+#define ixDIG2_DIG_BE_CLKGEN_DEBUG0                                                                    0x0010
+
+
+// addressBlock: dig2_digfedebugind
+// base address: 0x0
+#define ixDIG2_DIG_FE_DEBUG_ID                                                                         0x0000
+#define ixDIG2_DIG_VPG_DEBUG0                                                                          0x000b
+#define ixDIG2_DIG_VPG_DEBUG1                                                                          0x000c
+#define ixDIG2_DIG_VPG_DEBUG2                                                                          0x000d
+#define ixDIG2_DIG_VPG_DEBUG3                                                                          0x000e
+
+
+// addressBlock: dp3_dpdebugind
+// base address: 0x0
+#define ixDP3_DP_DEBUG_ID                                                                              0x0000
+#define ixDP3_DP_DEBUG_K                                                                               0x00a2
+#define ixDP3_DP_DEBUG_L                                                                               0x00a3
+#define ixDP3_DP_DEBUG_M                                                                               0x00a4
+#define ixDP3_DP_DEBUG_G                                                                               0x00b4
+#define ixDP3_DP_DEBUG_O                                                                               0x00fb
+#define ixDP3_DP_DEBUG_P                                                                               0x00fc
+#define ixDP3_DP_DEBUG_Q                                                                               0x00fd
+#define ixDP3_DP_DEBUG_R                                                                               0x00fe
+#define ixDP3_DP_DEBUG_S                                                                               0x00ff
+
+
+// addressBlock: dp3_dpfedebugind
+// base address: 0x0
+#define ixDP3_DP_FE_DEBUG_ID                                                                           0x0000
+#define ixDP3_DP_FE_SYMCLK_DEBUG_DATA_0                                                                0x0001
+#define ixDP3_DP_FE_SYMCLK_DEBUG_DATA_1                                                                0x0002
+#define ixDP3_DP_FE_SYMCLK_DEBUG_DATA_2                                                                0x0003
+#define ixDP3_DP_FE_SYMCLK_DEBUG_DATA_3                                                                0x0004
+#define ixDP3_DP_FE_SYMCLK_DEBUG_DATA_4                                                                0x0005
+#define ixDP3_DP_DEBUG_T                                                                               0x0090
+#define ixDP3_DP_DEBUG_U                                                                               0x0091
+#define ixDP3_DP_DEBUG_V                                                                               0x0092
+#define ixDP3_DP_DEBUG_W                                                                               0x0093
+#define ixDP3_DP_DEBUG_X                                                                               0x0094
+#define ixDP3_DP_DEBUG_Y                                                                               0x0095
+#define ixDP3_DP_DEBUG_Z                                                                               0x0096
+#define ixDP3_DP_DEBUG_1A                                                                              0x0097
+#define ixDP3_DP_DEBUG_1B                                                                              0x0098
+#define ixDP3_DP_DEBUG_I                                                                               0x00a0
+#define ixDP3_DP_DEBUG_J                                                                               0x00a1
+#define ixDP3_DP_DEBUG_N                                                                               0x00a2
+#define ixDP3_DP_DEBUG_H                                                                               0x00ac
+#define ixDP3_DP_DEBUG_A                                                                               0x00ad
+#define ixDP3_DP_DEBUG_B                                                                               0x00ae
+#define ixDP3_DP_DEBUG_C                                                                               0x00af
+#define ixDP3_DP_DEBUG_D                                                                               0x00b0
+#define ixDP3_DP_DEBUG_E                                                                               0x00b2
+#define ixDP3_DP_DEBUG_F                                                                               0x00b3
+
+
+// addressBlock: dp3_dpfe_dprefclk_debugind
+// base address: 0x0
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_ID                                                                  0x0000
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_0                                                              0x0001
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_1                                                              0x0002
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_2                                                              0x0003
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_3                                                              0x0004
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_4                                                              0x0005
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_5                                                              0x0006
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_6                                                              0x0007
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_7                                                              0x0008
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_8                                                              0x0009
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_9                                                              0x000a
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_10                                                             0x000b
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_11                                                             0x000c
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_12                                                             0x000d
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_13                                                             0x000e
+#define ixDP3_DP_FE_DPREFCLK_DEBUG_DATA_14                                                             0x000f
+
+
+// addressBlock: dig3_digdebugind
+// base address: 0x0
+#define ixDIG3_DIG_BE_CLKGEN_DEBUG0                                                                    0x0010
+
+
+// addressBlock: dig3_digfedebugind
+// base address: 0x0
+#define ixDIG3_DIG_FE_DEBUG_ID                                                                         0x0000
+#define ixDIG3_DIG_VPG_DEBUG0                                                                          0x000b
+#define ixDIG3_DIG_VPG_DEBUG1                                                                          0x000c
+#define ixDIG3_DIG_VPG_DEBUG2                                                                          0x000d
+#define ixDIG3_DIG_VPG_DEBUG3                                                                          0x000e
+
+
+// addressBlock: dp4_dpdebugind
+// base address: 0x0
+#define ixDP4_DP_DEBUG_ID                                                                              0x0000
+#define ixDP4_DP_DEBUG_K                                                                               0x00a2
+#define ixDP4_DP_DEBUG_L                                                                               0x00a3
+#define ixDP4_DP_DEBUG_M                                                                               0x00a4
+#define ixDP4_DP_DEBUG_G                                                                               0x00b4
+#define ixDP4_DP_DEBUG_O                                                                               0x00fb
+#define ixDP4_DP_DEBUG_P                                                                               0x00fc
+#define ixDP4_DP_DEBUG_Q                                                                               0x00fd
+#define ixDP4_DP_DEBUG_R                                                                               0x00fe
+#define ixDP4_DP_DEBUG_S                                                                               0x00ff
+
+
+// addressBlock: dp4_dpfedebugind
+// base address: 0x0
+#define ixDP4_DP_FE_DEBUG_ID                                                                           0x0000
+#define ixDP4_DP_FE_SYMCLK_DEBUG_DATA_0                                                                0x0001
+#define ixDP4_DP_FE_SYMCLK_DEBUG_DATA_1                                                                0x0002
+#define ixDP4_DP_FE_SYMCLK_DEBUG_DATA_2                                                                0x0003
+#define ixDP4_DP_FE_SYMCLK_DEBUG_DATA_3                                                                0x0004
+#define ixDP4_DP_FE_SYMCLK_DEBUG_DATA_4                                                                0x0005
+#define ixDP4_DP_DEBUG_T                                                                               0x0090
+#define ixDP4_DP_DEBUG_U                                                                               0x0091
+#define ixDP4_DP_DEBUG_V                                                                               0x0092
+#define ixDP4_DP_DEBUG_W                                                                               0x0093
+#define ixDP4_DP_DEBUG_X                                                                               0x0094
+#define ixDP4_DP_DEBUG_Y                                                                               0x0095
+#define ixDP4_DP_DEBUG_Z                                                                               0x0096
+#define ixDP4_DP_DEBUG_1A                                                                              0x0097
+#define ixDP4_DP_DEBUG_1B                                                                              0x0098
+#define ixDP4_DP_DEBUG_I                                                                               0x00a0
+#define ixDP4_DP_DEBUG_J                                                                               0x00a1
+#define ixDP4_DP_DEBUG_N                                                                               0x00a2
+#define ixDP4_DP_DEBUG_H                                                                               0x00ac
+#define ixDP4_DP_DEBUG_A                                                                               0x00ad
+#define ixDP4_DP_DEBUG_B                                                                               0x00ae
+#define ixDP4_DP_DEBUG_C                                                                               0x00af
+#define ixDP4_DP_DEBUG_D                                                                               0x00b0
+#define ixDP4_DP_DEBUG_E                                                                               0x00b2
+#define ixDP4_DP_DEBUG_F                                                                               0x00b3
+
+
+// addressBlock: dp4_dpfe_dprefclk_debugind
+// base address: 0x0
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_ID                                                                  0x0000
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_0                                                              0x0001
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_1                                                              0x0002
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_2                                                              0x0003
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_3                                                              0x0004
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_4                                                              0x0005
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_5                                                              0x0006
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_6                                                              0x0007
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_7                                                              0x0008
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_8                                                              0x0009
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_9                                                              0x000a
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_10                                                             0x000b
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_11                                                             0x000c
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_12                                                             0x000d
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_13                                                             0x000e
+#define ixDP4_DP_FE_DPREFCLK_DEBUG_DATA_14                                                             0x000f
+
+
+// addressBlock: dig4_digdebugind
+// base address: 0x0
+#define ixDIG4_DIG_BE_CLKGEN_DEBUG0                                                                    0x0010
+
+
+// addressBlock: dig4_digfedebugind
+// base address: 0x0
+#define ixDIG4_DIG_FE_DEBUG_ID                                                                         0x0000
+#define ixDIG4_DIG_VPG_DEBUG0                                                                          0x000b
+#define ixDIG4_DIG_VPG_DEBUG1                                                                          0x000c
+#define ixDIG4_DIG_VPG_DEBUG2                                                                          0x000d
+#define ixDIG4_DIG_VPG_DEBUG3                                                                          0x000e
+
+
+// addressBlock: dio_misc_dio_miscdebugind
+// base address: 0x0
+#define ixI2C_DEBUG_BUS                                                                                0x0002
+#define ixDIO_RBBMIF_DEBUG_0                                                                           0x0007
+#define ixDIO_RBBMIF_DEBUG_1                                                                           0x0008
+#define ixDIO_RBBMIF_DEBUG_2                                                                           0x0009
+#define ixDIO_RBBMIF_DEBUG_3                                                                           0x000a
+#define ixDIGA_RBBMIF_DEBUG_0                                                                          0x000b
+#define ixDIGA_RBBMIF_DEBUG_1                                                                          0x000c
+#define ixDIGA_RBBMIF_DEBUG_2                                                                          0x000d
+#define ixDIGA_RBBMIF_DEBUG_3                                                                          0x000e
+#define ixDIGB_RBBMIF_DEBUG_0                                                                          0x000f
+#define ixDIGB_RBBMIF_DEBUG_1                                                                          0x0010
+#define ixDIGB_RBBMIF_DEBUG_2                                                                          0x0011
+#define ixDIGB_RBBMIF_DEBUG_3                                                                          0x0012
+#define ixDIGC_RBBMIF_DEBUG_0                                                                          0x0013
+#define ixDIGC_RBBMIF_DEBUG_1                                                                          0x0014
+#define ixDIGC_RBBMIF_DEBUG_2                                                                          0x0015
+#define ixDIGC_RBBMIF_DEBUG_3                                                                          0x0016
+#define ixDIGD_RBBMIF_DEBUG_0                                                                          0x0017
+#define ixDIGD_RBBMIF_DEBUG_1                                                                          0x0018
+#define ixDIGD_RBBMIF_DEBUG_2                                                                          0x0019
+#define ixDIGD_RBBMIF_DEBUG_3                                                                          0x001a
+#define ixDIGE_RBBMIF_DEBUG_0                                                                          0x001b
+#define ixDIGE_RBBMIF_DEBUG_1                                                                          0x001c
+#define ixDIGE_RBBMIF_DEBUG_2                                                                          0x001d
+#define ixDIGE_RBBMIF_DEBUG_3                                                                          0x001e
+#define ixDIGA_DME_DEBUG                                                                               0x0027
+#define ixDIGB_DME_DEBUG                                                                               0x0028
+#define ixDIGC_DME_DEBUG                                                                               0x0029
+#define ixDIGD_DME_DEBUG                                                                               0x002a
+#define ixDIGE_DME_DEBUG                                                                               0x002b
+#define ixDIGA_RESYNC_FIFO_DEBUG                                                                       0x0033
+#define ixDIGB_RESYNC_FIFO_DEBUG                                                                       0x0034
+#define ixDIGC_RESYNC_FIFO_DEBUG                                                                       0x0035
+#define ixDIGD_RESYNC_FIFO_DEBUG                                                                       0x0036
+#define ixDIGE_RESYNC_FIFO_DEBUG                                                                       0x0037
+
+
+// addressBlock: apg_apg_socclk_debugind
+// base address: 0x0
+#define ixAPG_SOCCLK_DEBUG_ID                                                                          0x0000
+#define ixAPG_SOCCLK_DEBUG0                                                                            0x0001
+
+
+// addressBlock: apg_apg_encclk_debugind
+// base address: 0x0
+#define ixAPG_ENCCLK_DEBUG_ID                                                                          0x0000
+#define ixAPG_ENCCLK_DEBUG0                                                                            0x0001
+#define ixAPG_ENCCLK_DEBUG1                                                                            0x0002
+#define ixAPG_ENCCLK_DEBUG2                                                                            0x0003
+#define ixAPG_ENCCLK_DEBUG3                                                                            0x0004
+
+
+// addressBlock: dcoh_top_dcoh_topdebugind
+// base address: 0x0
+#define ixDCOH_TOP_DEBUG_ID                                                                            0x0000
+
+
+// addressBlock: dp_aux0_auxdebugdispclkind
+// base address: 0x0
+#define ixDP_AUX0_AUX_DEBUG_DISPCLK_ID                                                                 0x0000
+#define ixDP_AUX0_DP_AUX_DEBUG_H                                                                       0x0001
+#define ixDP_AUX0_DP_AUX_DEBUG_I                                                                       0x0002
+
+
+// addressBlock: dp_aux0_auxdebugrefclkind
+// base address: 0x0
+#define ixDP_AUX0_AUX_DEBUG_REFCLK_ID                                                                  0x0000
+#define ixDP_AUX0_DP_AUX_DEBUG_A                                                                       0x0001
+#define ixDP_AUX0_DP_AUX_DEBUG_B                                                                       0x0002
+#define ixDP_AUX0_DP_AUX_DEBUG_C                                                                       0x0003
+#define ixDP_AUX0_DP_AUX_DEBUG_D                                                                       0x0004
+#define ixDP_AUX0_DP_AUX_DEBUG_E                                                                       0x0005
+#define ixDP_AUX0_DP_AUX_DEBUG_F                                                                       0x0006
+#define ixDP_AUX0_DP_AUX_DEBUG_G                                                                       0x0007
+#define ixDP_AUX0_DP_AUX_DEBUG_J                                                                       0x0008
+
+
+// addressBlock: dp_aux1_auxdebugdispclkind
+// base address: 0x0
+#define ixDP_AUX1_AUX_DEBUG_DISPCLK_ID                                                                 0x0000
+#define ixDP_AUX1_DP_AUX_DEBUG_H                                                                       0x0001
+#define ixDP_AUX1_DP_AUX_DEBUG_I                                                                       0x0002
+
+
+// addressBlock: dp_aux1_auxdebugrefclkind
+// base address: 0x0
+#define ixDP_AUX1_AUX_DEBUG_REFCLK_ID                                                                  0x0000
+#define ixDP_AUX1_DP_AUX_DEBUG_A                                                                       0x0001
+#define ixDP_AUX1_DP_AUX_DEBUG_B                                                                       0x0002
+#define ixDP_AUX1_DP_AUX_DEBUG_C                                                                       0x0003
+#define ixDP_AUX1_DP_AUX_DEBUG_D                                                                       0x0004
+#define ixDP_AUX1_DP_AUX_DEBUG_E                                                                       0x0005
+#define ixDP_AUX1_DP_AUX_DEBUG_F                                                                       0x0006
+#define ixDP_AUX1_DP_AUX_DEBUG_G                                                                       0x0007
+#define ixDP_AUX1_DP_AUX_DEBUG_J                                                                       0x0008
+
+
+// addressBlock: dp_aux2_auxdebugdispclkind
+// base address: 0x0
+#define ixDP_AUX2_AUX_DEBUG_DISPCLK_ID                                                                 0x0000
+#define ixDP_AUX2_DP_AUX_DEBUG_H                                                                       0x0001
+#define ixDP_AUX2_DP_AUX_DEBUG_I                                                                       0x0002
+
+
+// addressBlock: dp_aux2_auxdebugrefclkind
+// base address: 0x0
+#define ixDP_AUX2_AUX_DEBUG_REFCLK_ID                                                                  0x0000
+#define ixDP_AUX2_DP_AUX_DEBUG_A                                                                       0x0001
+#define ixDP_AUX2_DP_AUX_DEBUG_B                                                                       0x0002
+#define ixDP_AUX2_DP_AUX_DEBUG_C                                                                       0x0003
+#define ixDP_AUX2_DP_AUX_DEBUG_D                                                                       0x0004
+#define ixDP_AUX2_DP_AUX_DEBUG_E                                                                       0x0005
+#define ixDP_AUX2_DP_AUX_DEBUG_F                                                                       0x0006
+#define ixDP_AUX2_DP_AUX_DEBUG_G                                                                       0x0007
+#define ixDP_AUX2_DP_AUX_DEBUG_J                                                                       0x0008
+
+
+// addressBlock: dp_aux3_auxdebugdispclkind
+// base address: 0x0
+#define ixDP_AUX3_AUX_DEBUG_DISPCLK_ID                                                                 0x0000
+#define ixDP_AUX3_DP_AUX_DEBUG_H                                                                       0x0001
+#define ixDP_AUX3_DP_AUX_DEBUG_I                                                                       0x0002
+
+
+// addressBlock: dp_aux3_auxdebugrefclkind
+// base address: 0x0
+#define ixDP_AUX3_AUX_DEBUG_REFCLK_ID                                                                  0x0000
+#define ixDP_AUX3_DP_AUX_DEBUG_A                                                                       0x0001
+#define ixDP_AUX3_DP_AUX_DEBUG_B                                                                       0x0002
+#define ixDP_AUX3_DP_AUX_DEBUG_C                                                                       0x0003
+#define ixDP_AUX3_DP_AUX_DEBUG_D                                                                       0x0004
+#define ixDP_AUX3_DP_AUX_DEBUG_E                                                                       0x0005
+#define ixDP_AUX3_DP_AUX_DEBUG_F                                                                       0x0006
+#define ixDP_AUX3_DP_AUX_DEBUG_G                                                                       0x0007
+#define ixDP_AUX3_DP_AUX_DEBUG_J                                                                       0x0008
+
+
+// addressBlock: dp_aux4_auxdebugdispclkind
+// base address: 0x0
+#define ixDP_AUX4_AUX_DEBUG_DISPCLK_ID                                                                 0x0000
+#define ixDP_AUX4_DP_AUX_DEBUG_H                                                                       0x0001
+#define ixDP_AUX4_DP_AUX_DEBUG_I                                                                       0x0002
+
+
+// addressBlock: dp_aux4_auxdebugrefclkind
+// base address: 0x0
+#define ixDP_AUX4_AUX_DEBUG_REFCLK_ID                                                                  0x0000
+#define ixDP_AUX4_DP_AUX_DEBUG_A                                                                       0x0001
+#define ixDP_AUX4_DP_AUX_DEBUG_B                                                                       0x0002
+#define ixDP_AUX4_DP_AUX_DEBUG_C                                                                       0x0003
+#define ixDP_AUX4_DP_AUX_DEBUG_D                                                                       0x0004
+#define ixDP_AUX4_DP_AUX_DEBUG_E                                                                       0x0005
+#define ixDP_AUX4_DP_AUX_DEBUG_F                                                                       0x0006
+#define ixDP_AUX4_DP_AUX_DEBUG_G                                                                       0x0007
+#define ixDP_AUX4_DP_AUX_DEBUG_J                                                                       0x0008
+
+
+// addressBlock: hpd0_hpddebugind
+// base address: 0x0
+#define ixHPD0_HPD_DEBUG_ID                                                                            0x0000
+#define ixHPD0_HPD_DEBUG_0                                                                             0x0001
+
+
+// addressBlock: hpd1_hpddebugind
+// base address: 0x0
+#define ixHPD1_HPD_DEBUG_ID                                                                            0x0000
+#define ixHPD1_HPD_DEBUG_0                                                                             0x0001
+
+
+// addressBlock: hpd2_hpddebugind
+// base address: 0x0
+#define ixHPD2_HPD_DEBUG_ID                                                                            0x0000
+#define ixHPD2_HPD_DEBUG_0                                                                             0x0001
+
+
+// addressBlock: hpd3_hpddebugind
+// base address: 0x0
+#define ixHPD3_HPD_DEBUG_ID                                                                            0x0000
+#define ixHPD3_HPD_DEBUG_0                                                                             0x0001
+
+
+// addressBlock: hpd4_hpddebugind
+// base address: 0x0
+#define ixHPD4_HPD_DEBUG_ID                                                                            0x0000
+#define ixHPD4_HPD_DEBUG_0                                                                             0x0001
+
+
+// addressBlock: hpo_top_hpo_topdebugind
+// base address: 0x0
+#define ixHPO_TOP_DEBUG_ID                                                                             0x0000
+
+
+// addressBlock: hdmi_link_enc0_linkencdebugind
+// base address: 0x0
+#define ixHDMI_LINK_ENC_DEBUG_ID                                                                       0x0000
+
+
+// addressBlock: hdmi_frl_enc0_frldebugind
+// base address: 0x0
+#define ixHDMI_FRL_ENC_DEBUG_ID                                                                        0x0000
+
+
+// addressBlock: hdmi_stream_enc0_hdmi_stream_enc_hdmistreamclk_debugind
+// base address: 0x0
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID                                                       0x0000
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0                                                        0x0001
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1                                                        0x0002
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2                                                        0x0003
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3                                                        0x0004
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4                                                        0x0005
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5                                                        0x0006
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6                                                        0x0007
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7                                                        0x0008
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8                                                        0x0009
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9                                                        0x000a
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10                                                       0x000b
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11                                                       0x000c
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12                                                       0x000d
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13                                                       0x000e
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14                                                       0x000f
+#define ixHDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15                                                       0x0010
+
+
+// addressBlock: hdmi_stream_enc0_hdmi_stream_enc_dispclk_debugind
+// base address: 0x0
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_ID                                                             0x0000
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_0                                                              0x0001
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_1                                                              0x0002
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_2                                                              0x0003
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_3                                                              0x0004
+#define ixHDMI_STREAM_ENC_DISPCLK_DEBUG_4                                                              0x0005
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_dispclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC0_DP_STREAM_ENC_DISPCLK_DEBUG_ID                                                0x0000
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC0_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID                                            0x0000
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC0_DP_STREAM_ENC_SYMCLK32_DEBUG_ID                                               0x0000
+
+
+// addressBlock: dp_sym32_enc0_dp_sym32_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC0_DP_SYM32_ENC_SYMCLK32_DEBUG_ID                                                 0x0000
+
+
+// addressBlock: dp_sym32_enc0_dp_sym32_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC0_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID                                              0x0000
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_dispclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC1_DP_STREAM_ENC_DISPCLK_DEBUG_ID                                                0x0000
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC1_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID                                            0x0000
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC1_DP_STREAM_ENC_SYMCLK32_DEBUG_ID                                               0x0000
+
+
+// addressBlock: dp_sym32_enc1_dp_sym32_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC1_DP_SYM32_ENC_SYMCLK32_DEBUG_ID                                                 0x0000
+
+
+// addressBlock: dp_sym32_enc1_dp_sym32_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC1_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID                                              0x0000
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_dispclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC2_DP_STREAM_ENC_DISPCLK_DEBUG_ID                                                0x0000
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC2_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID                                            0x0000
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC2_DP_STREAM_ENC_SYMCLK32_DEBUG_ID                                               0x0000
+
+
+// addressBlock: dp_sym32_enc2_dp_sym32_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC2_DP_SYM32_ENC_SYMCLK32_DEBUG_ID                                                 0x0000
+
+
+// addressBlock: dp_sym32_enc2_dp_sym32_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC2_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID                                              0x0000
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_dispclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC3_DP_STREAM_ENC_DISPCLK_DEBUG_ID                                                0x0000
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC3_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID                                            0x0000
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_STREAM_ENC3_DP_STREAM_ENC_SYMCLK32_DEBUG_ID                                               0x0000
+
+
+// addressBlock: dp_sym32_enc3_dp_sym32_enc_symclk32_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC3_DP_SYM32_ENC_SYMCLK32_DEBUG_ID                                                 0x0000
+
+
+// addressBlock: dp_sym32_enc3_dp_sym32_enc_dpstreamclk_debugind
+// base address: 0x0
+#define ixDP_SYM32_ENC3_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID                                              0x0000
+
+
+// addressBlock: dp_link_enc0_dplinkencdebugind
+// base address: 0x0
+#define ixDP_LINK_ENC0_DP_LINK_ENC_DEBUG_ID                                                            0x0000
+#define ixDP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS0                                                          0x0001
+#define ixDP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS1                                                          0x0002
+#define ixDP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS2                                                          0x0003
+#define ixDP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS3                                                          0x0004
+
+
+// addressBlock: dp_dphy_sym320_dpdphysym32debugind
+// base address: 0x0
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_ID                                                        0x0000
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS0                                                      0x0001
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS1                                                      0x0002
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS2                                                      0x0003
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS3                                                      0x0004
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS4                                                      0x0005
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS5                                                      0x0006
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS6                                                      0x0007
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS7                                                      0x0008
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS8                                                      0x0009
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS9                                                      0x000a
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS10                                                     0x000b
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS11                                                     0x000c
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS12                                                     0x000d
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS13                                                     0x000e
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS14                                                     0x000f
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS15                                                     0x0010
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS16                                                     0x0011
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS17                                                     0x0012
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS18                                                     0x0013
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS19                                                     0x0014
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS20                                                     0x0015
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS21                                                     0x0016
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS22                                                     0x0017
+#define ixDP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS23                                                     0x0018
+
+
+// addressBlock: dp_link_enc1_dplinkencdebugind
+// base address: 0x0
+#define ixDP_LINK_ENC1_DP_LINK_ENC_DEBUG_ID                                                            0x0000
+#define ixDP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS0                                                          0x0001
+#define ixDP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS1                                                          0x0002
+#define ixDP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS2                                                          0x0003
+#define ixDP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS3                                                          0x0004
+
+
+// addressBlock: dp_dphy_sym321_dpdphysym32debugind
+// base address: 0x0
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_ID                                                        0x0000
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS0                                                      0x0001
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS1                                                      0x0002
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS2                                                      0x0003
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS3                                                      0x0004
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS4                                                      0x0005
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS5                                                      0x0006
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS6                                                      0x0007
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS7                                                      0x0008
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS8                                                      0x0009
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS9                                                      0x000a
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS10                                                     0x000b
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS11                                                     0x000c
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS12                                                     0x000d
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS13                                                     0x000e
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS14                                                     0x000f
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS15                                                     0x0010
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS16                                                     0x0011
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS17                                                     0x0012
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS18                                                     0x0013
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS19                                                     0x0014
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS20                                                     0x0015
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS21                                                     0x0016
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS22                                                     0x0017
+#define ixDP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS23                                                     0x0018
+
+
+// addressBlock: dp_link_enc2_dplinkencdebugind
+// base address: 0x0
+#define ixDP_LINK_ENC2_DP_LINK_ENC_DEBUG_ID                                                            0x0000
+#define ixDP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS0                                                          0x0001
+#define ixDP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS1                                                          0x0002
+#define ixDP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS2                                                          0x0003
+#define ixDP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS3                                                          0x0004
+
+
+// addressBlock: dp_dphy_sym322_dpdphysym32debugind
+// base address: 0x0
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_ID                                                        0x0000
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS0                                                      0x0001
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS1                                                      0x0002
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS2                                                      0x0003
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS3                                                      0x0004
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS4                                                      0x0005
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS5                                                      0x0006
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS6                                                      0x0007
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS7                                                      0x0008
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS8                                                      0x0009
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS9                                                      0x000a
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS10                                                     0x000b
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS11                                                     0x000c
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS12                                                     0x000d
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS13                                                     0x000e
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS14                                                     0x000f
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS15                                                     0x0010
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS16                                                     0x0011
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS17                                                     0x0012
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS18                                                     0x0013
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS19                                                     0x0014
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS20                                                     0x0015
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS21                                                     0x0016
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS22                                                     0x0017
+#define ixDP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS23                                                     0x0018
+
+
+// addressBlock: dp_link_enc3_dplinkencdebugind
+// base address: 0x0
+#define ixDP_LINK_ENC3_DP_LINK_ENC_DEBUG_ID                                                            0x0000
+#define ixDP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS0                                                          0x0001
+#define ixDP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS1                                                          0x0002
+#define ixDP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS2                                                          0x0003
+#define ixDP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS3                                                          0x0004
+
+
+// addressBlock: dp_dphy_sym323_dpdphysym32debugind
+// base address: 0x0
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_ID                                                        0x0000
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS0                                                      0x0001
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS1                                                      0x0002
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS2                                                      0x0003
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS3                                                      0x0004
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS4                                                      0x0005
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS5                                                      0x0006
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS6                                                      0x0007
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS7                                                      0x0008
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS8                                                      0x0009
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS9                                                      0x000a
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS10                                                     0x000b
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS11                                                     0x000c
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS12                                                     0x000d
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS13                                                     0x000e
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS14                                                     0x000f
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS15                                                     0x0010
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS16                                                     0x0011
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS17                                                     0x0012
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS18                                                     0x0013
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS19                                                     0x0014
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS20                                                     0x0015
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS21                                                     0x0016
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS22                                                     0x0017
+#define ixDP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS23                                                     0x0018
+
+
+// addressBlock: azendpoint_sinkinfoind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID                                                  0x0000
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID                                                       0x0001
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN                                             0x0002
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID0                                                          0x0003
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID1                                                          0x0004
+#define ixSINK_DESCRIPTION0                                                                            0x0005
+#define ixSINK_DESCRIPTION1                                                                            0x0006
+#define ixSINK_DESCRIPTION2                                                                            0x0007
+#define ixSINK_DESCRIPTION3                                                                            0x0008
+#define ixSINK_DESCRIPTION4                                                                            0x0009
+#define ixSINK_DESCRIPTION5                                                                            0x000a
+#define ixSINK_DESCRIPTION6                                                                            0x000b
+#define ixSINK_DESCRIPTION7                                                                            0x000c
+#define ixSINK_DESCRIPTION8                                                                            0x000d
+#define ixSINK_DESCRIPTION9                                                                            0x000e
+#define ixSINK_DESCRIPTION10                                                                           0x000f
+#define ixSINK_DESCRIPTION11                                                                           0x0010
+#define ixSINK_DESCRIPTION12                                                                           0x0011
+#define ixSINK_DESCRIPTION13                                                                           0x0012
+#define ixSINK_DESCRIPTION14                                                                           0x0013
+#define ixSINK_DESCRIPTION15                                                                           0x0014
+#define ixSINK_DESCRIPTION16                                                                           0x0015
+#define ixSINK_DESCRIPTION17                                                                           0x0016
+
+
+// addressBlock: azf0controller_azinputcrc0resultind
+// base address: 0x0
+#define ixAZALIA_INPUT_CRC0_CHANNEL0                                                                   0x0000
+#define ixAZALIA_INPUT_CRC0_CHANNEL1                                                                   0x0001
+#define ixAZALIA_INPUT_CRC0_CHANNEL2                                                                   0x0002
+#define ixAZALIA_INPUT_CRC0_CHANNEL3                                                                   0x0003
+#define ixAZALIA_INPUT_CRC0_CHANNEL4                                                                   0x0004
+#define ixAZALIA_INPUT_CRC0_CHANNEL5                                                                   0x0005
+#define ixAZALIA_INPUT_CRC0_CHANNEL6                                                                   0x0006
+#define ixAZALIA_INPUT_CRC0_CHANNEL7                                                                   0x0007
+
+
+// addressBlock: azf0controller_azinputcrc1resultind
+// base address: 0x0
+#define ixAZALIA_INPUT_CRC1_CHANNEL0                                                                   0x0000
+#define ixAZALIA_INPUT_CRC1_CHANNEL1                                                                   0x0001
+#define ixAZALIA_INPUT_CRC1_CHANNEL2                                                                   0x0002
+#define ixAZALIA_INPUT_CRC1_CHANNEL3                                                                   0x0003
+#define ixAZALIA_INPUT_CRC1_CHANNEL4                                                                   0x0004
+#define ixAZALIA_INPUT_CRC1_CHANNEL5                                                                   0x0005
+#define ixAZALIA_INPUT_CRC1_CHANNEL6                                                                   0x0006
+#define ixAZALIA_INPUT_CRC1_CHANNEL7                                                                   0x0007
+
+
+// addressBlock: azf0controller_azcrc0resultind
+// base address: 0x0
+#define ixAZALIA_CRC0_CHANNEL0                                                                         0x0000
+#define ixAZALIA_CRC0_CHANNEL1                                                                         0x0001
+#define ixAZALIA_CRC0_CHANNEL2                                                                         0x0002
+#define ixAZALIA_CRC0_CHANNEL3                                                                         0x0003
+#define ixAZALIA_CRC0_CHANNEL4                                                                         0x0004
+#define ixAZALIA_CRC0_CHANNEL5                                                                         0x0005
+#define ixAZALIA_CRC0_CHANNEL6                                                                         0x0006
+#define ixAZALIA_CRC0_CHANNEL7                                                                         0x0007
+
+
+// addressBlock: azf0controller_azcrc1resultind
+// base address: 0x0
+#define ixAZALIA_CRC1_CHANNEL0                                                                         0x0000
+#define ixAZALIA_CRC1_CHANNEL1                                                                         0x0001
+#define ixAZALIA_CRC1_CHANNEL2                                                                         0x0002
+#define ixAZALIA_CRC1_CHANNEL3                                                                         0x0003
+#define ixAZALIA_CRC1_CHANNEL4                                                                         0x0004
+#define ixAZALIA_CRC1_CHANNEL5                                                                         0x0005
+#define ixAZALIA_CRC1_CHANNEL6                                                                         0x0006
+#define ixAZALIA_CRC1_CHANNEL7                                                                         0x0007
+
+
+// addressBlock: azf0stream0_streamind
+// base address: 0x0
+#define ixAZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM0_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream1_streamind
+// base address: 0x0
+#define ixAZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM1_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream2_streamind
+// base address: 0x0
+#define ixAZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM2_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream3_streamind
+// base address: 0x0
+#define ixAZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM3_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream4_streamind
+// base address: 0x0
+#define ixAZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM4_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream5_streamind
+// base address: 0x0
+#define ixAZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM5_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream6_streamind
+// base address: 0x0
+#define ixAZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM6_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream7_streamind
+// base address: 0x0
+#define ixAZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM7_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream8_streamind
+// base address: 0x0
+#define ixAZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM8_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream9_streamind
+// base address: 0x0
+#define ixAZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL                                                         0x0000
+#define ixAZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL                                                   0x0001
+#define ixAZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT                                                   0x0002
+#define ixAZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT                                                  0x0003
+#define ixAZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT                                                  0x0004
+#define ixAZF0STREAM9_AZALIA_STREAM_DEBUG                                                              0x0005
+
+
+// addressBlock: azf0stream10_streamind
+// base address: 0x0
+#define ixAZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM10_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0stream11_streamind
+// base address: 0x0
+#define ixAZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM11_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0stream12_streamind
+// base address: 0x0
+#define ixAZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM12_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0stream13_streamind
+// base address: 0x0
+#define ixAZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM13_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0stream14_streamind
+// base address: 0x0
+#define ixAZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM14_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0stream15_streamind
+// base address: 0x0
+#define ixAZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL                                                        0x0000
+#define ixAZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL                                                  0x0001
+#define ixAZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT                                                  0x0002
+#define ixAZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT                                                 0x0003
+#define ixAZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT                                                 0x0004
+#define ixAZF0STREAM15_AZALIA_STREAM_DEBUG                                                             0x0005
+
+
+// addressBlock: azf0endpoint0_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT0_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint1_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT1_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint2_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT2_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint3_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT3_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint4_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT4_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint5_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT5_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint6_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT6_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0endpoint7_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG                                            0x0000
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                  0x0001
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                             0x0002
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                            0x0003
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                            0x0004
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                             0x0005
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                       0x0006
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL                                       0x0007
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE                                    0x0008
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                        0x0020
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES                                     0x0021
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                               0x0022
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                 0x0023
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL                                     0x0024
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER                                    0x0025
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA                                           0x0027
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0                                  0x0028
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1                                  0x0029
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2                                  0x002a
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3                                  0x002b
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4                                  0x002c
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5                                  0x002d
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6                                  0x002e
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7                                  0x002f
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8                                  0x0030
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9                                  0x0031
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10                                 0x0032
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11                                 0x0033
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12                                 0x0034
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13                                 0x0035
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE                                0x0036
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC                                   0x0037
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR                                       0x0038
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0                                         0x003a
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1                                         0x003b
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2                                         0x003c
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3                                         0x003d
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4                                         0x003e
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5                                         0x003f
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6                                         0x0040
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7                                         0x0041
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8                                         0x0042
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL                                   0x0054
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE                         0x0055
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                     0x0056
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2                               0x0057
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                  0x0058
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                      0x0059
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                      0x005a
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                      0x005b
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                      0x005c
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                      0x005d
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                      0x005e
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                      0x005f
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                      0x0060
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                      0x0061
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO                                           0x0062
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                              0x0063
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                              0x0064
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB                                               0x0065
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                0x0066
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE                                        0x0067
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED                                     0x0068
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                    0x0069
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                   0x006a
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS                                                  0x006b
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS                                             0x006c
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS                                            0x006d
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS                                      0x006e
+#define ixAZF0ENDPOINT7_AZALIA_F0_ENDPOINT_FGCG_REP_DIS                                                0x0070
+
+
+// addressBlock: azf0inputendpoint0_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint1_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint2_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint3_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint4_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint5_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint6_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: azf0inputendpoint7_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG                                 0x0000
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES       0x0001
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                  0x0002
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                 0x0003
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                 0x0004
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                  0x0005
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES            0x0006
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES             0x0020
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                          0x0021
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                    0x0022
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE                0x0023
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                          0x0024
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE                     0x0036
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2                    0x0037
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR                            0x0038
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                      0x0053
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL                        0x0054
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE              0x0055
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT          0x0056
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                   0x0064
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB                                    0x0065
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                     0x0066
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                    0x0067
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME                               0x0068
+
+
+// addressBlock: dscc_dsccdebugind0
+// base address: 0x0
+#define ixDSCC_DEBUG_ID                                                                                0x0000
+#define ixDSCC_DEBUG_0                                                                                 0x0001
+#define ixDSCC_DEBUG_1                                                                                 0x0002
+#define ixDSCC_DEBUG_2                                                                                 0x0003
+#define ixDSCC_DEBUG_3                                                                                 0x0004
+#define ixDSCC_DEBUG_4                                                                                 0x0005
+#define ixDSCC_DEBUG_5                                                                                 0x0006
+#define ixDSCC_DEBUG_6                                                                                 0x0007
+#define ixDSCC_DEBUG_7                                                                                 0x0008
+#define ixDSCC_DEBUG_8                                                                                 0x0009
+#define ixDSCC_DEBUG_9                                                                                 0x000a
+#define ixDSCC_DEBUG_10                                                                                0x000b
+#define ixDSCC_DEBUG_11                                                                                0x000c
+#define ixDSCC_DEBUG_12                                                                                0x000d
+#define ixDSCC_DEBUG_13                                                                                0x000e
+#define ixDSCC_DEBUG_14                                                                                0x000f
+#define ixDSCC_DEBUG_15                                                                                0x0010
+#define ixDSCC_DEBUG_16                                                                                0x0011
+#define ixDSCC_DEBUG_17                                                                                0x0012
+#define ixDSCC_DEBUG_18                                                                                0x0013
+#define ixDSCC_DEBUG_19                                                                                0x0014
+#define ixDSCC_DEBUG_20                                                                                0x0015
+#define ixDSCC_DEBUG_21                                                                                0x0016
+#define ixDSCC_DEBUG_22                                                                                0x0017
+#define ixDSCC_DEBUG_23                                                                                0x0018
+#define ixDSCC_DEBUG_24                                                                                0x0019
+#define ixDSCC_DEBUG_25                                                                                0x001a
+#define ixDSCC_DEBUG_26                                                                                0x001b
+#define ixDSCC_DEBUG_27                                                                                0x001c
+#define ixDSCC_DEBUG_28                                                                                0x001d
+
+
+// addressBlock: dscc_dscc_dispclk_debugind0
+// base address: 0x0
+#define ixDSCC_DISPCLK_DEBUG_ID                                                                        0x0000
+#define ixDSCC_DISPCLK_DEBUG_0                                                                         0x0001
+#define ixDSCC_DISPCLK_DEBUG_1                                                                         0x0002
+#define ixDSCC_DISPCLK_DEBUG_2                                                                         0x0003
+#define ixDSCC_DISPCLK_DEBUG_3                                                                         0x0004
+#define ixDSCC_DISPCLK_DEBUG_4                                                                         0x0005
+#define ixDSCC_DISPCLK_DEBUG_5                                                                         0x0006
+#define ixDSCC_DISPCLK_DEBUG_6                                                                         0x0007
+#define ixDSCC_DISPCLK_DEBUG_7                                                                         0x0008
+#define ixDSCC_DISPCLK_DEBUG_8                                                                         0x0009
+#define ixDSCC_DISPCLK_DEBUG_9                                                                         0x000a
+#define ixDSCC_DISPCLK_DEBUG_10                                                                        0x000b
+#define ixDSCC_DISPCLK_DEBUG_11                                                                        0x000c
+#define ixDSCC_DISPCLK_DEBUG_12                                                                        0x000d
+#define ixDSCC_DISPCLK_DEBUG_13                                                                        0x000e
+#define ixDSCC_DISPCLK_DEBUG_14                                                                        0x000f
+#define ixDSCC_DISPCLK_DEBUG_15                                                                        0x0010
+#define ixDSCC_DISPCLK_DEBUG_16                                                                        0x0011
+#define ixDSCC_DISPCLK_DEBUG_17                                                                        0x0012
+#define ixDSCC_DISPCLK_DEBUG_18                                                                        0x0013
+#define ixDSCC_DISPCLK_DEBUG_19                                                                        0x0014
+#define ixDSCC_DISPCLK_DEBUG_20                                                                        0x0015
+#define ixDSCC_DISPCLK_DEBUG_21                                                                        0x0016
+#define ixDSCC_DISPCLK_DEBUG_22                                                                        0x0017
+#define ixDSCC_DISPCLK_DEBUG_23                                                                        0x0018
+#define ixDSCC_DISPCLK_DEBUG_24                                                                        0x0019
+#define ixDSCC_DISPCLK_DEBUG_25                                                                        0x001a
+#define ixDSCC_DISPCLK_DEBUG_26                                                                        0x001b
+#define ixDSCC_DISPCLK_DEBUG_27                                                                        0x001c
+#define ixDSCC_DISPCLK_DEBUG_28                                                                        0x001d
+#define ixDSCC_DISPCLK_DEBUG_29                                                                        0x001e
+#define ixDSCC_DISPCLK_DEBUG_30                                                                        0x001f
+#define ixDSCC_DISPCLK_DEBUG_31                                                                        0x0020
+#define ixDSCC_DISPCLK_DEBUG_32                                                                        0x0021
+#define ixDSCC_DISPCLK_DEBUG_33                                                                        0x0022
+
+
+// addressBlock: dsccif_dsccifdebugind
+// base address: 0x0
+#define ixDSCCIF_DEBUG_ID                                                                              0x0000
+#define ixDSCCIF_DEBUG_0                                                                               0x0001
+#define ixDSCCIF_DEBUG_1                                                                               0x0002
+
+
+// addressBlock: dsc_top_dsc_topdebugind
+// base address: 0x0
+#define ixDSC_TOP_DEBUG_ID                                                                             0x0000
+#define ixDSC_TOP_DEBUG_0                                                                              0x0001
+#define ixDSC_TOP_DEBUG_1                                                                              0x0002
+#define ixDSC_TOP_DEBUG_2                                                                              0x0003
+
+
+// addressBlock: dpia_port0_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT0_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT0_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port1_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT1_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT1_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port2_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT2_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT2_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port3_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT3_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT3_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port4_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT4_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT4_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port5_dpiaportdebugind
+// base address: 0x0
+#define ixDPIA_PORT5_DPIA_PORT_DEBUG_ID                                                                0x0000
+#define ixDPIA_PORT5_DPIA_PORT_DEBUG_BUS0                                                              0x0001
+
+
+// addressBlock: dpia_port_ml0_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_ml1_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_ml2_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_ml3_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_ml4_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_ml5_dpiaportmldebugind
+// base address: 0x0
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_TEST_DEBUG_ID                                                     0x0000
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA0                                                       0x0001
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA1                                                       0x0002
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA2                                                       0x0003
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA3                                                       0x0004
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA4                                                       0x0005
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA5                                                       0x0006
+#define ixDPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA6                                                       0x0007
+
+
+// addressBlock: dpia_port_aux0_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_port_aux1_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_port_aux2_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_port_aux3_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_port_aux4_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_port_aux5_dpiaportauxdebugind
+// base address: 0x0
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG_ID                                                   0x0000
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG1                                                     0x0001
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG2                                                     0x0002
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG3                                                     0x0003
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG4                                                     0x0004
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG5                                                     0x0005
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG6                                                     0x0006
+#define ixDPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG7                                                     0x0007
+
+
+// addressBlock: dpia_mu0_dpia_mu_debugind
+// base address: 0x0
+#define ixDPIA_MU_DEBUG_ID                                                                             0x0000
+
+
+// addressBlock: azendpoint_descriptorind
+// base address: 0x0
+#define ixAUDIO_DESCRIPTOR0                                                                            0x0001
+#define ixAUDIO_DESCRIPTOR1                                                                            0x0002
+#define ixAUDIO_DESCRIPTOR2                                                                            0x0003
+#define ixAUDIO_DESCRIPTOR3                                                                            0x0004
+#define ixAUDIO_DESCRIPTOR4                                                                            0x0005
+#define ixAUDIO_DESCRIPTOR5                                                                            0x0006
+#define ixAUDIO_DESCRIPTOR6                                                                            0x0007
+#define ixAUDIO_DESCRIPTOR7                                                                            0x0008
+#define ixAUDIO_DESCRIPTOR8                                                                            0x0009
+#define ixAUDIO_DESCRIPTOR9                                                                            0x000a
+#define ixAUDIO_DESCRIPTOR10                                                                           0x000b
+#define ixAUDIO_DESCRIPTOR11                                                                           0x000c
+#define ixAUDIO_DESCRIPTOR12                                                                           0x000d
+#define ixAUDIO_DESCRIPTOR13                                                                           0x000e
+
+
+// addressBlock: dce_dc_hda_azendpoint_azdec
+// base address: 0x0
+#define regAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                               0x0006
+#define regAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                      0
+#define regAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                              0x0006
+#define regAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                     0
+
+
+// addressBlock: dce_dc_hda_azinputendpoint_azdec
+// base address: 0x0
+#define regAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA                           0x0006
+#define regAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_BASE_IDX                  0
+#define regAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX                          0x0006
+#define regAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_BASE_IDX                 0
+
+
+// addressBlock: dce_dc_hda_azroot_azdec
+// base address: 0x0
+#define regAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                                       0x0006
+#define regAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                              0
+#define regAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                                      0x0006
+#define regAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                             0
+
+
+// addressBlock: dce_dc_hda_azstream0_azdec
+// base address: 0x0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x000e
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x000f
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0010
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0011
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x0012
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x0012
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x0014
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x0015
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0761
+#define regAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream1_azdec
+// base address: 0x20
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x0016
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x0017
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0018
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0019
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x001a
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x001a
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x001c
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x001d
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0769
+#define regAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream2_azdec
+// base address: 0x40
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x001e
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x001f
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0020
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0021
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x0022
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x0022
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x0024
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x0025
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0771
+#define regAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream3_azdec
+// base address: 0x60
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x0026
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x0027
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0028
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0029
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x002a
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x002a
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x002c
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x002d
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0779
+#define regAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream4_azdec
+// base address: 0x80
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x002e
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x002f
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0030
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0031
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x0032
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x0032
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x0034
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x0035
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0781
+#define regAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream5_azdec
+// base address: 0xa0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x0036
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x0037
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0038
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0039
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x003a
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x003a
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x003c
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x003d
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0789
+#define regAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream6_azdec
+// base address: 0xc0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x003e
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x003f
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0040
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0041
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x0042
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x0042
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x0044
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x0045
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0791
+#define regAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_hda_azstream7_azdec
+// base address: 0xe0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x0046
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x0047
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x0048
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x0049
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x004a
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x004a
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x004c
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x004d
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 0
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x0799
+#define regAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          1
+
+
+// addressBlock: dce_dc_dccg_dccg_dispdec
+// base address: 0x0
+#define regPHYPLLA_PIXCLK_RESYNC_CNTL                                                                   0x0040
+#define regPHYPLLA_PIXCLK_RESYNC_CNTL_BASE_IDX                                                          1
+#define regPHYPLLB_PIXCLK_RESYNC_CNTL                                                                   0x0041
+#define regPHYPLLB_PIXCLK_RESYNC_CNTL_BASE_IDX                                                          1
+#define regPHYPLLC_PIXCLK_RESYNC_CNTL                                                                   0x0042
+#define regPHYPLLC_PIXCLK_RESYNC_CNTL_BASE_IDX                                                          1
+#define regPHYPLLD_PIXCLK_RESYNC_CNTL                                                                   0x0043
+#define regPHYPLLD_PIXCLK_RESYNC_CNTL_BASE_IDX                                                          1
+#define regDP_DTO_DBUF_EN                                                                               0x0044
+#define regDP_DTO_DBUF_EN_BASE_IDX                                                                      1
+#define regDSCCLK3_DTO_PARAM                                                                            0x0045
+#define regDSCCLK3_DTO_PARAM_BASE_IDX                                                                   1
+#define regDPREFCLK_CGTT_BLK_CTRL_REG                                                                   0x0048
+#define regDPREFCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                          1
+#define regDCCG_GATE_DISABLE_CNTL4                                                                      0x0049
+#define regDCCG_GATE_DISABLE_CNTL4_BASE_IDX                                                             1
+#define regDPSTREAMCLK_CNTL                                                                             0x004a
+#define regDPSTREAMCLK_CNTL_BASE_IDX                                                                    1
+#define regREFCLK_CGTT_BLK_CTRL_REG                                                                     0x004b
+#define regREFCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                            1
+#define regPHYPLLE_PIXCLK_RESYNC_CNTL                                                                   0x004c
+#define regPHYPLLE_PIXCLK_RESYNC_CNTL_BASE_IDX                                                          1
+#define regDCCG_PERFMON_CNTL2                                                                           0x004e
+#define regDCCG_PERFMON_CNTL2_BASE_IDX                                                                  1
+#define regDPIASYMCLK_CNTL                                                                              0x004f
+#define regDPIASYMCLK_CNTL_BASE_IDX                                                                     1
+#define regDCCG_GLOBAL_FGCG_REP_CNTL                                                                    0x0050
+#define regDCCG_GLOBAL_FGCG_REP_CNTL_BASE_IDX                                                           1
+#define regOTG_ADD_DROP_PIXEL_CNTL                                                                      0x0056
+#define regOTG_ADD_DROP_PIXEL_CNTL_BASE_IDX                                                             1
+#define regDPREFCLK_CNTL                                                                                0x0058
+#define regDPREFCLK_CNTL_BASE_IDX                                                                       1
+#define regDCE_VERSION                                                                                  0x005e
+#define regDCE_VERSION_BASE_IDX                                                                         1
+#define regSYMCLK32_SE_CNTL                                                                             0x0065
+#define regSYMCLK32_SE_CNTL_BASE_IDX                                                                    1
+#define regSYMCLK32_LE_CNTL                                                                             0x0066
+#define regSYMCLK32_LE_CNTL_BASE_IDX                                                                    1
+#define regDTBCLK_P_CNTL                                                                                0x0068
+#define regDTBCLK_P_CNTL_BASE_IDX                                                                       1
+#define regDCCG_GATE_DISABLE_CNTL5                                                                      0x0069
+#define regDCCG_GATE_DISABLE_CNTL5_BASE_IDX                                                             1
+#define regDSCCLK0_DTO_PARAM                                                                            0x006c
+#define regDSCCLK0_DTO_PARAM_BASE_IDX                                                                   1
+#define regDSCCLK1_DTO_PARAM                                                                            0x006d
+#define regDSCCLK1_DTO_PARAM_BASE_IDX                                                                   1
+#define regDSCCLK2_DTO_PARAM                                                                            0x006e
+#define regDSCCLK2_DTO_PARAM_BASE_IDX                                                                   1
+#define regOTG_PIXEL_RATE_DIV                                                                           0x006f
+#define regOTG_PIXEL_RATE_DIV_BASE_IDX                                                                  1
+#define regMILLISECOND_TIME_BASE_DIV                                                                    0x0070
+#define regMILLISECOND_TIME_BASE_DIV_BASE_IDX                                                           1
+#define regDISPCLK_FREQ_CHANGE_CNTL                                                                     0x0071
+#define regDISPCLK_FREQ_CHANGE_CNTL_BASE_IDX                                                            1
+#define regDC_MEM_GLOBAL_PWR_REQ_CNTL                                                                   0x0072
+#define regDC_MEM_GLOBAL_PWR_REQ_CNTL_BASE_IDX                                                          1
+#define regDCCG_PERFMON_CNTL                                                                            0x0073
+#define regDCCG_PERFMON_CNTL_BASE_IDX                                                                   1
+#define regDCCG_GATE_DISABLE_CNTL                                                                       0x0074
+#define regDCCG_GATE_DISABLE_CNTL_BASE_IDX                                                              1
+#define regDISPCLK_CGTT_BLK_CTRL_REG                                                                    0x0075
+#define regDISPCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                           1
+#define regSOCCLK_CGTT_BLK_CTRL_REG                                                                     0x0076
+#define regSOCCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                            1
+#define regDCCG_CAC_STATUS                                                                              0x0077
+#define regDCCG_CAC_STATUS_BASE_IDX                                                                     1
+#define regMICROSECOND_TIME_BASE_DIV                                                                    0x007b
+#define regMICROSECOND_TIME_BASE_DIV_BASE_IDX                                                           1
+#define regDCCG_GATE_DISABLE_CNTL2                                                                      0x007c
+#define regDCCG_GATE_DISABLE_CNTL2_BASE_IDX                                                             1
+#define regSYMCLK_CGTT_BLK_CTRL_REG                                                                     0x007d
+#define regSYMCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                            1
+#define regDCCG_DISP_CNTL_REG                                                                           0x007f
+#define regDCCG_DISP_CNTL_REG_BASE_IDX                                                                  1
+#define regOTG0_PIXEL_RATE_CNTL                                                                         0x0080
+#define regOTG0_PIXEL_RATE_CNTL_BASE_IDX                                                                1
+#define regDP_DTO0_PHASE                                                                                0x0081
+#define regDP_DTO0_PHASE_BASE_IDX                                                                       1
+#define regDP_DTO0_MODULO                                                                               0x0082
+#define regDP_DTO0_MODULO_BASE_IDX                                                                      1
+#define regOTG0_PHYPLL_PIXEL_RATE_CNTL                                                                  0x0083
+#define regOTG0_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX                                                         1
+#define regOTG1_PIXEL_RATE_CNTL                                                                         0x0084
+#define regOTG1_PIXEL_RATE_CNTL_BASE_IDX                                                                1
+#define regDP_DTO1_PHASE                                                                                0x0085
+#define regDP_DTO1_PHASE_BASE_IDX                                                                       1
+#define regDP_DTO1_MODULO                                                                               0x0086
+#define regDP_DTO1_MODULO_BASE_IDX                                                                      1
+#define regOTG1_PHYPLL_PIXEL_RATE_CNTL                                                                  0x0087
+#define regOTG1_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX                                                         1
+#define regOTG2_PIXEL_RATE_CNTL                                                                         0x0088
+#define regOTG2_PIXEL_RATE_CNTL_BASE_IDX                                                                1
+#define regDP_DTO2_PHASE                                                                                0x0089
+#define regDP_DTO2_PHASE_BASE_IDX                                                                       1
+#define regDP_DTO2_MODULO                                                                               0x008a
+#define regDP_DTO2_MODULO_BASE_IDX                                                                      1
+#define regOTG2_PHYPLL_PIXEL_RATE_CNTL                                                                  0x008b
+#define regOTG2_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX                                                         1
+#define regOTG3_PIXEL_RATE_CNTL                                                                         0x008c
+#define regOTG3_PIXEL_RATE_CNTL_BASE_IDX                                                                1
+#define regDP_DTO3_PHASE                                                                                0x008d
+#define regDP_DTO3_PHASE_BASE_IDX                                                                       1
+#define regDP_DTO3_MODULO                                                                               0x008e
+#define regDP_DTO3_MODULO_BASE_IDX                                                                      1
+#define regOTG3_PHYPLL_PIXEL_RATE_CNTL                                                                  0x008f
+#define regOTG3_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX                                                         1
+#define regDPPCLK_CGTT_BLK_CTRL_REG                                                                     0x0098
+#define regDPPCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                            1
+#define regDPPCLK0_DTO_PARAM                                                                            0x0099
+#define regDPPCLK0_DTO_PARAM_BASE_IDX                                                                   1
+#define regDPPCLK1_DTO_PARAM                                                                            0x009a
+#define regDPPCLK1_DTO_PARAM_BASE_IDX                                                                   1
+#define regDPPCLK2_DTO_PARAM                                                                            0x009b
+#define regDPPCLK2_DTO_PARAM_BASE_IDX                                                                   1
+#define regDPPCLK3_DTO_PARAM                                                                            0x009c
+#define regDPPCLK3_DTO_PARAM_BASE_IDX                                                                   1
+#define regDCCG_CAC_STATUS2                                                                             0x009f
+#define regDCCG_CAC_STATUS2_BASE_IDX                                                                    1
+#define regSYMCLKA_CLOCK_ENABLE                                                                         0x00a0
+#define regSYMCLKA_CLOCK_ENABLE_BASE_IDX                                                                1
+#define regSYMCLKB_CLOCK_ENABLE                                                                         0x00a1
+#define regSYMCLKB_CLOCK_ENABLE_BASE_IDX                                                                1
+#define regSYMCLKC_CLOCK_ENABLE                                                                         0x00a2
+#define regSYMCLKC_CLOCK_ENABLE_BASE_IDX                                                                1
+#define regSYMCLKD_CLOCK_ENABLE                                                                         0x00a3
+#define regSYMCLKD_CLOCK_ENABLE_BASE_IDX                                                                1
+#define regSYMCLKE_CLOCK_ENABLE                                                                         0x00a4
+#define regSYMCLKE_CLOCK_ENABLE_BASE_IDX                                                                1
+#define regDCCG_SOFT_RESET                                                                              0x00a6
+#define regDCCG_SOFT_RESET_BASE_IDX                                                                     1
+#define regDSCCLK_DTO_CTRL                                                                              0x00a7
+#define regDSCCLK_DTO_CTRL_BASE_IDX                                                                     1
+#define regDPPCLK_CTRL                                                                                  0x00a8
+#define regDPPCLK_CTRL_BASE_IDX                                                                         1
+#define regDCCG_GATE_DISABLE_CNTL6                                                                      0x00a9
+#define regDCCG_GATE_DISABLE_CNTL6_BASE_IDX                                                             1
+#define regSYMCLK_PSP_CNTL                                                                              0x00aa
+#define regSYMCLK_PSP_CNTL_BASE_IDX                                                                     1
+#define regDCCG_AUDIO_DTO_SOURCE                                                                        0x00ab
+#define regDCCG_AUDIO_DTO_SOURCE_BASE_IDX                                                               1
+#define regDCCG_AUDIO_DTO0_PHASE                                                                        0x00ac
+#define regDCCG_AUDIO_DTO0_PHASE_BASE_IDX                                                               1
+#define regDCCG_AUDIO_DTO0_MODULE                                                                       0x00ad
+#define regDCCG_AUDIO_DTO0_MODULE_BASE_IDX                                                              1
+#define regDCCG_AUDIO_DTO1_PHASE                                                                        0x00ae
+#define regDCCG_AUDIO_DTO1_PHASE_BASE_IDX                                                               1
+#define regDCCG_AUDIO_DTO1_MODULE                                                                       0x00af
+#define regDCCG_AUDIO_DTO1_MODULE_BASE_IDX                                                              1
+#define regDCCG_VSYNC_OTG0_LATCH_VALUE                                                                  0x00b0
+#define regDCCG_VSYNC_OTG0_LATCH_VALUE_BASE_IDX                                                         1
+#define regDCCG_VSYNC_OTG1_LATCH_VALUE                                                                  0x00b1
+#define regDCCG_VSYNC_OTG1_LATCH_VALUE_BASE_IDX                                                         1
+#define regDCCG_VSYNC_OTG2_LATCH_VALUE                                                                  0x00b2
+#define regDCCG_VSYNC_OTG2_LATCH_VALUE_BASE_IDX                                                         1
+#define regDCCG_VSYNC_OTG3_LATCH_VALUE                                                                  0x00b3
+#define regDCCG_VSYNC_OTG3_LATCH_VALUE_BASE_IDX                                                         1
+#define regDCCG_VSYNC_OTG4_LATCH_VALUE                                                                  0x00b4
+#define regDCCG_VSYNC_OTG4_LATCH_VALUE_BASE_IDX                                                         1
+#define regDCCG_VSYNC_OTG5_LATCH_VALUE                                                                  0x00b5
+#define regDCCG_VSYNC_OTG5_LATCH_VALUE_BASE_IDX                                                         1
+#define regDPPCLK_DTO_CTRL                                                                              0x00b6
+#define regDPPCLK_DTO_CTRL_BASE_IDX                                                                     1
+#define regDCCG_VSYNC_CNT_CTRL                                                                          0x00b8
+#define regDCCG_VSYNC_CNT_CTRL_BASE_IDX                                                                 1
+#define regDCCG_VSYNC_CNT_INT_CTRL                                                                      0x00b9
+#define regDCCG_VSYNC_CNT_INT_CTRL_BASE_IDX                                                             1
+#define regFORCE_SYMCLK_DISABLE                                                                         0x00ba
+#define regFORCE_SYMCLK_DISABLE_BASE_IDX                                                                1
+#define regDCCG_TEST_CLK_SEL                                                                            0x00be
+#define regDCCG_TEST_CLK_SEL_BASE_IDX                                                                   1
+#define regHDMICHARCLK0_CLOCK_CNTL                                                                      0x004a
+#define regHDMICHARCLK0_CLOCK_CNTL_BASE_IDX                                                             2
+#define regPHYASYMCLK_CLOCK_CNTL                                                                        0x0052
+#define regPHYASYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regPHYBSYMCLK_CLOCK_CNTL                                                                        0x0053
+#define regPHYBSYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regPHYCSYMCLK_CLOCK_CNTL                                                                        0x0054
+#define regPHYCSYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regPHYDSYMCLK_CLOCK_CNTL                                                                        0x0055
+#define regPHYDSYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regPHYESYMCLK_CLOCK_CNTL                                                                        0x0056
+#define regPHYESYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regHDMISTREAMCLK_CNTL                                                                           0x0059
+#define regHDMISTREAMCLK_CNTL_BASE_IDX                                                                  2
+#define regDCCG_GATE_DISABLE_CNTL3                                                                      0x005a
+#define regDCCG_GATE_DISABLE_CNTL3_BASE_IDX                                                             2
+#define regHDMISTREAMCLK0_DTO_PARAM                                                                     0x005b
+#define regHDMISTREAMCLK0_DTO_PARAM_BASE_IDX                                                            2
+#define regDPIACLK_810M_DTO_PHASE                                                                       0x0064
+#define regDPIACLK_810M_DTO_PHASE_BASE_IDX                                                              2
+#define regDPIACLK_540M_DTO_PHASE                                                                       0x0065
+#define regDPIACLK_540M_DTO_PHASE_BASE_IDX                                                              2
+#define regDPIACLK_DTO_CNTL                                                                             0x0066
+#define regDPIACLK_DTO_CNTL_BASE_IDX                                                                    2
+#define regDPIACLK_810M_DTO_MODULO                                                                      0x0068
+#define regDPIACLK_810M_DTO_MODULO_BASE_IDX                                                             2
+#define regDPIACLK_540M_DTO_MODULO                                                                      0x0069
+#define regDPIACLK_540M_DTO_MODULO_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dccg_dccg_dfs_dispdec
+// base address: 0x0
+#define regDENTIST_DISPCLK_CNTL                                                                         0x0064
+#define regDENTIST_DISPCLK_CNTL_BASE_IDX                                                                1
+
+
+// addressBlock: azroot_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID                                          0x0f00
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID                                                   0x0f02
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT                                        0x0f04
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE                                                 0x1705
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID                                       0x1720
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2                                     0x1721
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3                                     0x1722
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4                                     0x1723
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION                                   0x1770
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESET                                                       0x17ff
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT                                    0x1f04
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE                                                0x1f05
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES                                      0x1f0a
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS                                            0x1f0b
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES                                              0x1f0f
+
+
+// addressBlock: azendpoint_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT                                           0x2200
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID                                          0x2706
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER                                          0x270d
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2                                        0x270e
+#define ixAZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL                                                     0x2724
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3                                        0x273e
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE                                                  0x2770
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                                0x2f09
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                                     0x2f0a
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS                                           0x2f0b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY                                   0x3702
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL                                                   0x3707
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE                                             0x3708
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE                                               0x3709
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                                   0x371c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2                                 0x371d
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3                                 0x371e
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4                                 0x371f
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION                                      0x3770
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION                                               0x3771
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO                                                    0x3772
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX                                                        0x3774
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_ACP_DATA                                                         0x3775
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR                                                 0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA                                            0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE                                            0x3777
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE                                            0x3778
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE                                            0x3779
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE                                            0x377a
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC                                                          0x377b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_HBR                                                              0x377c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX                                            0x3780
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA                                             0x3781
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE                                             0x3785
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE                                             0x3786
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE                                             0x3787
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE                                             0x3788
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE                                                0x3789
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0                                                    0x378a
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1                                                    0x378b
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2                                                    0x378c
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3                                                    0x378d
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4                                                    0x378e
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5                                                    0x378f
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6                                                    0x3790
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7                                                    0x3791
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8                                                    0x3792
+#define ixAZALIA_F2_CODEC_PIN_ASSOCIATION_INFO                                                         0x3793
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS                                            0x3797
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                                            0x3798
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB                                                             0x3799
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                              0x379a
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE                                                      0x379b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED                                                   0x379c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION                                  0x379d
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE                                                 0x379e
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                                      0x3f09
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES                                                   0x3f0c
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH                                         0x3f0e
+
+
+// addressBlock: azinputendpoint_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT                                     0x6200
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID                                    0x6706
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER                                    0x670d
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES                          0x6f09
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES                               0x6f0a
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS                                     0x6f0b
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL                                             0x7707
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE                                       0x7708
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE                                         0x7709
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT                             0x771c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2                           0x771d
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3                           0x771e
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4                           0x771f
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION                                         0x7771
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE                                       0x7777
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE                                       0x7778
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE                                       0x7779
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE                                       0x777a
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR                                                        0x777c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE                                       0x7785
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE                                       0x7786
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE                                       0x7787
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE                                       0x7788
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL                                      0x7798
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB                                                       0x7799
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT                                        0x779a
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL                                       0x779b
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME                                                  0x779c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L                                           0x779d
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H                                           0x779e
+#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES                                0x7f09
+#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES                                             0x7f0c
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon0_dc_perfmon_dispdec
+// base address: 0x0
+#define regDC_PERFMON0_PERFCOUNTER_CNTL                                                                 0x0000
+#define regDC_PERFMON0_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON0_PERFCOUNTER_CNTL2                                                                0x0001
+#define regDC_PERFMON0_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON0_PERFCOUNTER_STATE                                                                0x0002
+#define regDC_PERFMON0_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON0_PERFMON_CNTL                                                                     0x0003
+#define regDC_PERFMON0_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON0_PERFMON_CNTL2                                                                    0x0004
+#define regDC_PERFMON0_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON0_PERFMON_CVALUE_INT_MISC                                                          0x0005
+#define regDC_PERFMON0_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON0_PERFMON_CVALUE_LOW                                                               0x0006
+#define regDC_PERFMON0_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON0_PERFMON_HI                                                                       0x0007
+#define regDC_PERFMON0_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON0_PERFMON_LOW                                                                      0x0008
+#define regDC_PERFMON0_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon1_dc_perfmon_dispdec
+// base address: 0x30
+#define regDC_PERFMON1_PERFCOUNTER_CNTL                                                                 0x000c
+#define regDC_PERFMON1_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON1_PERFCOUNTER_CNTL2                                                                0x000d
+#define regDC_PERFMON1_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON1_PERFCOUNTER_STATE                                                                0x000e
+#define regDC_PERFMON1_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON1_PERFMON_CNTL                                                                     0x000f
+#define regDC_PERFMON1_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON1_PERFMON_CNTL2                                                                    0x0010
+#define regDC_PERFMON1_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON1_PERFMON_CVALUE_INT_MISC                                                          0x0011
+#define regDC_PERFMON1_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON1_PERFMON_CVALUE_LOW                                                               0x0012
+#define regDC_PERFMON1_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON1_PERFMON_HI                                                                       0x0013
+#define regDC_PERFMON1_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON1_PERFMON_LOW                                                                      0x0014
+#define regDC_PERFMON1_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dmu_dc_pg_dispdec
+// base address: 0x0
+#define regDOMAIN0_PG_CONFIG                                                                            0x0080
+#define regDOMAIN0_PG_CONFIG_BASE_IDX                                                                   2
+#define regDOMAIN0_PG_STATUS                                                                            0x0081
+#define regDOMAIN0_PG_STATUS_BASE_IDX                                                                   2
+#define regDOMAIN1_PG_CONFIG                                                                            0x0082
+#define regDOMAIN1_PG_CONFIG_BASE_IDX                                                                   2
+#define regDOMAIN1_PG_STATUS                                                                            0x0083
+#define regDOMAIN1_PG_STATUS_BASE_IDX                                                                   2
+#define regDOMAIN2_PG_CONFIG                                                                            0x0084
+#define regDOMAIN2_PG_CONFIG_BASE_IDX                                                                   2
+#define regDOMAIN2_PG_STATUS                                                                            0x0085
+#define regDOMAIN2_PG_STATUS_BASE_IDX                                                                   2
+#define regDOMAIN3_PG_CONFIG                                                                            0x0086
+#define regDOMAIN3_PG_CONFIG_BASE_IDX                                                                   2
+#define regDOMAIN3_PG_STATUS                                                                            0x0087
+#define regDOMAIN3_PG_STATUS_BASE_IDX                                                                   2
+#define regDOMAIN16_PG_CONFIG                                                                           0x0089
+#define regDOMAIN16_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN16_PG_STATUS                                                                           0x008a
+#define regDOMAIN16_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN17_PG_CONFIG                                                                           0x008b
+#define regDOMAIN17_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN17_PG_STATUS                                                                           0x008c
+#define regDOMAIN17_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN18_PG_CONFIG                                                                           0x008d
+#define regDOMAIN18_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN18_PG_STATUS                                                                           0x008e
+#define regDOMAIN18_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN19_PG_CONFIG                                                                           0x008f
+#define regDOMAIN19_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN19_PG_STATUS                                                                           0x0090
+#define regDOMAIN19_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN22_PG_CONFIG                                                                           0x0092
+#define regDOMAIN22_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN22_PG_STATUS                                                                           0x0093
+#define regDOMAIN22_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN23_PG_CONFIG                                                                           0x0094
+#define regDOMAIN23_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN23_PG_STATUS                                                                           0x0095
+#define regDOMAIN23_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN24_PG_CONFIG                                                                           0x0096
+#define regDOMAIN24_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN24_PG_STATUS                                                                           0x0097
+#define regDOMAIN24_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN25_PG_CONFIG                                                                           0x0098
+#define regDOMAIN25_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN25_PG_STATUS                                                                           0x0099
+#define regDOMAIN25_PG_STATUS_BASE_IDX                                                                  2
+#define regDOMAIN26_PG_CONFIG                                                                           0x009a
+#define regDOMAIN26_PG_CONFIG_BASE_IDX                                                                  2
+#define regDOMAIN26_PG_STATUS                                                                           0x009b
+#define regDOMAIN26_PG_STATUS_BASE_IDX                                                                  2
+#define regDCPG_INTERRUPT_STATUS                                                                        0x009c
+#define regDCPG_INTERRUPT_STATUS_BASE_IDX                                                               2
+#define regDCPG_INTERRUPT_STATUS_2                                                                      0x009d
+#define regDCPG_INTERRUPT_STATUS_2_BASE_IDX                                                             2
+#define regDCPG_INTERRUPT_STATUS_3                                                                      0x009e
+#define regDCPG_INTERRUPT_STATUS_3_BASE_IDX                                                             2
+#define regDCPG_INTERRUPT_CONTROL_1                                                                     0x009f
+#define regDCPG_INTERRUPT_CONTROL_1_BASE_IDX                                                            2
+#define regDCPG_INTERRUPT_CONTROL_2                                                                     0x00a0
+#define regDCPG_INTERRUPT_CONTROL_2_BASE_IDX                                                            2
+#define regDCPG_INTERRUPT_CONTROL_3                                                                     0x00a1
+#define regDCPG_INTERRUPT_CONTROL_3_BASE_IDX                                                            2
+#define regDC_IP_REQUEST_CNTL                                                                           0x00a2
+#define regDC_IP_REQUEST_CNTL_BASE_IDX                                                                  2
+#define regLONO_MEM_PWR_REQ_CNTL                                                                        0x00a6
+#define regLONO_MEM_PWR_REQ_CNTL_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dmu_dmu_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2f8
+#define regDC_PERFMON2_PERFCOUNTER_CNTL                                                                 0x00be
+#define regDC_PERFMON2_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON2_PERFCOUNTER_CNTL2                                                                0x00bf
+#define regDC_PERFMON2_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON2_PERFCOUNTER_STATE                                                                0x00c0
+#define regDC_PERFMON2_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON2_PERFMON_CNTL                                                                     0x00c1
+#define regDC_PERFMON2_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON2_PERFMON_CNTL2                                                                    0x00c2
+#define regDC_PERFMON2_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON2_PERFMON_CVALUE_INT_MISC                                                          0x00c3
+#define regDC_PERFMON2_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON2_PERFMON_CVALUE_LOW                                                               0x00c4
+#define regDC_PERFMON2_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON2_PERFMON_HI                                                                       0x00c5
+#define regDC_PERFMON2_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON2_PERFMON_LOW                                                                      0x00c6
+#define regDC_PERFMON2_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dmu_dmu_misc_dispdec
+// base address: 0x0
+#define regCC_DC_PIPE_DIS                                                                               0x00ca
+#define regCC_DC_PIPE_DIS_BASE_IDX                                                                      2
+#define regDMU_CLK_CNTL                                                                                 0x00cb
+#define regDMU_CLK_CNTL_BASE_IDX                                                                        2
+#define regDMCUB_SMU_INTERRUPT_CNTL                                                                     0x00cd
+#define regDMCUB_SMU_INTERRUPT_CNTL_BASE_IDX                                                            2
+#define regSMU_INTERRUPT_CONTROL                                                                        0x00ce
+#define regSMU_INTERRUPT_CONTROL_BASE_IDX                                                               2
+#define regZSC_CNTL                                                                                     0x00cf
+#define regZSC_CNTL_BASE_IDX                                                                            2
+#define regZSC_CNTL2                                                                                    0x00d0
+#define regZSC_CNTL2_BASE_IDX                                                                           2
+#define regDMU_MISC_ALLOW_DS_FORCE                                                                      0x00d6
+#define regDMU_MISC_ALLOW_DS_FORCE_BASE_IDX                                                             2
+#define regZSC_STATUS                                                                                   0x00d7
+#define regZSC_STATUS_BASE_IDX                                                                          2
+#define regDMU_DISPCLK_CGTT_BLK_CTRL_REG                                                                0x00d8
+#define regDMU_DISPCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                       2
+#define regDMU_SOCCLK_CGTT_BLK_CTRL_REG                                                                 0x00d9
+#define regDMU_SOCCLK_CGTT_BLK_CTRL_REG_BASE_IDX                                                        2
+#define regZPR_CLK_UNGATE_DELAY                                                                         0x00da
+#define regZPR_CLK_UNGATE_DELAY_BASE_IDX                                                                2
+
+
+// addressBlock: dce_dc_dmu_ihc_dispdec
+// base address: 0x0
+#define regDC_GPU_TIMER_START_POSITION_V_UPDATE                                                         0x0126
+#define regDC_GPU_TIMER_START_POSITION_V_UPDATE_BASE_IDX                                                2
+#define regDC_GPU_TIMER_START_POSITION_VSTARTUP                                                         0x0127
+#define regDC_GPU_TIMER_START_POSITION_VSTARTUP_BASE_IDX                                                2
+#define regDC_GPU_TIMER_READ                                                                            0x0128
+#define regDC_GPU_TIMER_READ_BASE_IDX                                                                   2
+#define regDC_GPU_TIMER_READ_CNTL                                                                       0x0129
+#define regDC_GPU_TIMER_READ_CNTL_BASE_IDX                                                              2
+#define regDISP_INTERRUPT_STATUS                                                                        0x012a
+#define regDISP_INTERRUPT_STATUS_BASE_IDX                                                               2
+#define regDISP_INTERRUPT_STATUS_CONTINUE                                                               0x012b
+#define regDISP_INTERRUPT_STATUS_CONTINUE_BASE_IDX                                                      2
+#define regDISP_INTERRUPT_STATUS_CONTINUE2                                                              0x012c
+#define regDISP_INTERRUPT_STATUS_CONTINUE2_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE3                                                              0x012d
+#define regDISP_INTERRUPT_STATUS_CONTINUE3_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE4                                                              0x012e
+#define regDISP_INTERRUPT_STATUS_CONTINUE4_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE5                                                              0x012f
+#define regDISP_INTERRUPT_STATUS_CONTINUE5_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE6                                                              0x0130
+#define regDISP_INTERRUPT_STATUS_CONTINUE6_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE7                                                              0x0131
+#define regDISP_INTERRUPT_STATUS_CONTINUE7_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE8                                                              0x0132
+#define regDISP_INTERRUPT_STATUS_CONTINUE8_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE9                                                              0x0133
+#define regDISP_INTERRUPT_STATUS_CONTINUE9_BASE_IDX                                                     2
+#define regDISP_INTERRUPT_STATUS_CONTINUE10                                                             0x0134
+#define regDISP_INTERRUPT_STATUS_CONTINUE10_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE11                                                             0x0135
+#define regDISP_INTERRUPT_STATUS_CONTINUE11_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE12                                                             0x0136
+#define regDISP_INTERRUPT_STATUS_CONTINUE12_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE13                                                             0x0137
+#define regDISP_INTERRUPT_STATUS_CONTINUE13_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE14                                                             0x0138
+#define regDISP_INTERRUPT_STATUS_CONTINUE14_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE15                                                             0x0139
+#define regDISP_INTERRUPT_STATUS_CONTINUE15_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE16                                                             0x013a
+#define regDISP_INTERRUPT_STATUS_CONTINUE16_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE17                                                             0x013b
+#define regDISP_INTERRUPT_STATUS_CONTINUE17_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE18                                                             0x013c
+#define regDISP_INTERRUPT_STATUS_CONTINUE18_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE19                                                             0x013d
+#define regDISP_INTERRUPT_STATUS_CONTINUE19_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE20                                                             0x013e
+#define regDISP_INTERRUPT_STATUS_CONTINUE20_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE21                                                             0x013f
+#define regDISP_INTERRUPT_STATUS_CONTINUE21_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE22                                                             0x0140
+#define regDISP_INTERRUPT_STATUS_CONTINUE22_BASE_IDX                                                    2
+#define regDC_GPU_TIMER_START_POSITION_VREADY                                                           0x0141
+#define regDC_GPU_TIMER_START_POSITION_VREADY_BASE_IDX                                                  2
+#define regDC_GPU_TIMER_START_POSITION_FLIP                                                             0x0142
+#define regDC_GPU_TIMER_START_POSITION_FLIP_BASE_IDX                                                    2
+#define regDC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK                                                 0x0143
+#define regDC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK_BASE_IDX                                        2
+#define regDC_GPU_TIMER_START_POSITION_FLIP_AWAY                                                        0x0144
+#define regDC_GPU_TIMER_START_POSITION_FLIP_AWAY_BASE_IDX                                               2
+#define regDISP_INTERRUPT_STATUS_CONTINUE23                                                             0x0145
+#define regDISP_INTERRUPT_STATUS_CONTINUE23_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE24                                                             0x0146
+#define regDISP_INTERRUPT_STATUS_CONTINUE24_BASE_IDX                                                    2
+#define regDISP_INTERRUPT_STATUS_CONTINUE25                                                             0x0147
+#define regDISP_INTERRUPT_STATUS_CONTINUE25_BASE_IDX                                                    2
+#define regDCCG_INTERRUPT_DEST                                                                          0x0148
+#define regDCCG_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regDMU_INTERRUPT_DEST                                                                           0x0149
+#define regDMU_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regDMU_INTERRUPT_DEST2                                                                          0x014a
+#define regDMU_INTERRUPT_DEST2_BASE_IDX                                                                 2
+#define regDCPG_INTERRUPT_DEST                                                                          0x014b
+#define regDCPG_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regDCPG_INTERRUPT_DEST2                                                                         0x014c
+#define regDCPG_INTERRUPT_DEST2_BASE_IDX                                                                2
+#define regMMHUBBUB_INTERRUPT_DEST                                                                      0x014d
+#define regMMHUBBUB_INTERRUPT_DEST_BASE_IDX                                                             2
+#define regWB_INTERRUPT_DEST                                                                            0x014e
+#define regWB_INTERRUPT_DEST_BASE_IDX                                                                   2
+#define regDCHUB_INTERRUPT_DEST                                                                         0x014f
+#define regDCHUB_INTERRUPT_DEST_BASE_IDX                                                                2
+#define regDCHUB_PERFCOUNTER_INTERRUPT_DEST                                                             0x0150
+#define regDCHUB_PERFCOUNTER_INTERRUPT_DEST_BASE_IDX                                                    2
+#define regDCHUB_INTERRUPT_DEST2                                                                        0x0151
+#define regDCHUB_INTERRUPT_DEST2_BASE_IDX                                                               2
+#define regDPP_PERFCOUNTER_INTERRUPT_DEST                                                               0x0152
+#define regDPP_PERFCOUNTER_INTERRUPT_DEST_BASE_IDX                                                      2
+#define regMPC_INTERRUPT_DEST                                                                           0x0153
+#define regMPC_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regOPP_INTERRUPT_DEST                                                                           0x0154
+#define regOPP_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regOPTC_INTERRUPT_DEST                                                                          0x0155
+#define regOPTC_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG0_INTERRUPT_DEST                                                                          0x0156
+#define regOTG0_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG1_INTERRUPT_DEST                                                                          0x0157
+#define regOTG1_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG2_INTERRUPT_DEST                                                                          0x0158
+#define regOTG2_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG3_INTERRUPT_DEST                                                                          0x0159
+#define regOTG3_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG4_INTERRUPT_DEST                                                                          0x015a
+#define regOTG4_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regOTG5_INTERRUPT_DEST                                                                          0x015b
+#define regOTG5_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regDIG_INTERRUPT_DEST                                                                           0x015c
+#define regDIG_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regI2C_DDC_HPD_INTERRUPT_DEST                                                                   0x015d
+#define regI2C_DDC_HPD_INTERRUPT_DEST_BASE_IDX                                                          2
+#define regDIO_INTERRUPT_DEST                                                                           0x015f
+#define regDIO_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regDCIO_INTERRUPT_DEST                                                                          0x0160
+#define regDCIO_INTERRUPT_DEST_BASE_IDX                                                                 2
+#define regHPD_INTERRUPT_DEST                                                                           0x0161
+#define regHPD_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regAZ_INTERRUPT_DEST                                                                            0x0162
+#define regAZ_INTERRUPT_DEST_BASE_IDX                                                                   2
+#define regAUX_INTERRUPT_DEST                                                                           0x0163
+#define regAUX_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regDSC_INTERRUPT_DEST                                                                           0x0164
+#define regDSC_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regHPO_INTERRUPT_DEST                                                                           0x0165
+#define regHPO_INTERRUPT_DEST_BASE_IDX                                                                  2
+#define regDPP_INTERRUPT_DEST                                                                           0x0166
+#define regDPP_INTERRUPT_DEST_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dmu_fgsec_dispdec
+// base address: 0x0
+#define regDMCUB_RBBMIF_SEC_CNTL                                                                        0x017a
+#define regDMCUB_RBBMIF_SEC_CNTL_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dmu_rbbmif_dispdec
+// base address: 0x0
+#define regRBBMIF_TIMEOUT                                                                               0x017f
+#define regRBBMIF_TIMEOUT_BASE_IDX                                                                      2
+#define regRBBMIF_STATUS                                                                                0x0180
+#define regRBBMIF_STATUS_BASE_IDX                                                                       2
+#define regRBBMIF_STATUS_2                                                                              0x0181
+#define regRBBMIF_STATUS_2_BASE_IDX                                                                     2
+#define regRBBMIF_INT_STATUS                                                                            0x0182
+#define regRBBMIF_INT_STATUS_BASE_IDX                                                                   2
+#define regRBBMIF_TIMEOUT_DIS                                                                           0x0183
+#define regRBBMIF_TIMEOUT_DIS_BASE_IDX                                                                  2
+#define regRBBMIF_TIMEOUT_DIS_2                                                                         0x0184
+#define regRBBMIF_TIMEOUT_DIS_2_BASE_IDX                                                                2
+#define regRBBMIF_STATUS_FLAG                                                                           0x0185
+#define regRBBMIF_STATUS_FLAG_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dmu_dmcub_dispdec
+// base address: 0x0
+#define regDMCUB_REGION0_OFFSET                                                                         0x018e
+#define regDMCUB_REGION0_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION0_OFFSET_HIGH                                                                    0x018f
+#define regDMCUB_REGION0_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION1_OFFSET                                                                         0x0190
+#define regDMCUB_REGION1_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION1_OFFSET_HIGH                                                                    0x0191
+#define regDMCUB_REGION1_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION2_OFFSET                                                                         0x0192
+#define regDMCUB_REGION2_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION2_OFFSET_HIGH                                                                    0x0193
+#define regDMCUB_REGION2_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION4_OFFSET                                                                         0x0196
+#define regDMCUB_REGION4_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION4_OFFSET_HIGH                                                                    0x0197
+#define regDMCUB_REGION4_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION5_OFFSET                                                                         0x0198
+#define regDMCUB_REGION5_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION5_OFFSET_HIGH                                                                    0x0199
+#define regDMCUB_REGION5_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION6_OFFSET                                                                         0x019a
+#define regDMCUB_REGION6_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION6_OFFSET_HIGH                                                                    0x019b
+#define regDMCUB_REGION6_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION7_OFFSET                                                                         0x019c
+#define regDMCUB_REGION7_OFFSET_BASE_IDX                                                                2
+#define regDMCUB_REGION7_OFFSET_HIGH                                                                    0x019d
+#define regDMCUB_REGION7_OFFSET_HIGH_BASE_IDX                                                           2
+#define regDMCUB_REGION0_TOP_ADDRESS                                                                    0x019e
+#define regDMCUB_REGION0_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION1_TOP_ADDRESS                                                                    0x019f
+#define regDMCUB_REGION1_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION2_TOP_ADDRESS                                                                    0x01a0
+#define regDMCUB_REGION2_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION4_TOP_ADDRESS                                                                    0x01a1
+#define regDMCUB_REGION4_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION5_TOP_ADDRESS                                                                    0x01a2
+#define regDMCUB_REGION5_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION6_TOP_ADDRESS                                                                    0x01a3
+#define regDMCUB_REGION6_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION7_TOP_ADDRESS                                                                    0x01a4
+#define regDMCUB_REGION7_TOP_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_REGION3_CW0_BASE_ADDRESS                                                               0x01a5
+#define regDMCUB_REGION3_CW0_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW1_BASE_ADDRESS                                                               0x01a6
+#define regDMCUB_REGION3_CW1_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW2_BASE_ADDRESS                                                               0x01a7
+#define regDMCUB_REGION3_CW2_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW3_BASE_ADDRESS                                                               0x01a8
+#define regDMCUB_REGION3_CW3_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW4_BASE_ADDRESS                                                               0x01a9
+#define regDMCUB_REGION3_CW4_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW5_BASE_ADDRESS                                                               0x01aa
+#define regDMCUB_REGION3_CW5_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW6_BASE_ADDRESS                                                               0x01ab
+#define regDMCUB_REGION3_CW6_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW7_BASE_ADDRESS                                                               0x01ac
+#define regDMCUB_REGION3_CW7_BASE_ADDRESS_BASE_IDX                                                      2
+#define regDMCUB_REGION3_CW0_TOP_ADDRESS                                                                0x01ad
+#define regDMCUB_REGION3_CW0_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW1_TOP_ADDRESS                                                                0x01ae
+#define regDMCUB_REGION3_CW1_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW2_TOP_ADDRESS                                                                0x01af
+#define regDMCUB_REGION3_CW2_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW3_TOP_ADDRESS                                                                0x01b0
+#define regDMCUB_REGION3_CW3_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW4_TOP_ADDRESS                                                                0x01b1
+#define regDMCUB_REGION3_CW4_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW5_TOP_ADDRESS                                                                0x01b2
+#define regDMCUB_REGION3_CW5_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW6_TOP_ADDRESS                                                                0x01b3
+#define regDMCUB_REGION3_CW6_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW7_TOP_ADDRESS                                                                0x01b4
+#define regDMCUB_REGION3_CW7_TOP_ADDRESS_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW0_OFFSET                                                                     0x01b5
+#define regDMCUB_REGION3_CW0_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW0_OFFSET_HIGH                                                                0x01b6
+#define regDMCUB_REGION3_CW0_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW1_OFFSET                                                                     0x01b7
+#define regDMCUB_REGION3_CW1_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW1_OFFSET_HIGH                                                                0x01b8
+#define regDMCUB_REGION3_CW1_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW2_OFFSET                                                                     0x01b9
+#define regDMCUB_REGION3_CW2_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW2_OFFSET_HIGH                                                                0x01ba
+#define regDMCUB_REGION3_CW2_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW3_OFFSET                                                                     0x01bb
+#define regDMCUB_REGION3_CW3_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW3_OFFSET_HIGH                                                                0x01bc
+#define regDMCUB_REGION3_CW3_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW4_OFFSET                                                                     0x01bd
+#define regDMCUB_REGION3_CW4_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW4_OFFSET_HIGH                                                                0x01be
+#define regDMCUB_REGION3_CW4_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW5_OFFSET                                                                     0x01bf
+#define regDMCUB_REGION3_CW5_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW5_OFFSET_HIGH                                                                0x01c0
+#define regDMCUB_REGION3_CW5_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW6_OFFSET                                                                     0x01c1
+#define regDMCUB_REGION3_CW6_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW6_OFFSET_HIGH                                                                0x01c2
+#define regDMCUB_REGION3_CW6_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_REGION3_CW7_OFFSET                                                                     0x01c3
+#define regDMCUB_REGION3_CW7_OFFSET_BASE_IDX                                                            2
+#define regDMCUB_REGION3_CW7_OFFSET_HIGH                                                                0x01c4
+#define regDMCUB_REGION3_CW7_OFFSET_HIGH_BASE_IDX                                                       2
+#define regDMCUB_INTERRUPT_ENABLE                                                                       0x01c5
+#define regDMCUB_INTERRUPT_ENABLE_BASE_IDX                                                              2
+#define regDMCUB_INTERRUPT_ACK                                                                          0x01c6
+#define regDMCUB_INTERRUPT_ACK_BASE_IDX                                                                 2
+#define regDMCUB_INTERRUPT_STATUS                                                                       0x01c7
+#define regDMCUB_INTERRUPT_STATUS_BASE_IDX                                                              2
+#define regDMCUB_INTERRUPT_TYPE                                                                         0x01c8
+#define regDMCUB_INTERRUPT_TYPE_BASE_IDX                                                                2
+#define regDMCUB_EXT_INTERRUPT_STATUS                                                                   0x01c9
+#define regDMCUB_EXT_INTERRUPT_STATUS_BASE_IDX                                                          2
+#define regDMCUB_EXT_INTERRUPT_CTXID                                                                    0x01ca
+#define regDMCUB_EXT_INTERRUPT_CTXID_BASE_IDX                                                           2
+#define regDMCUB_EXT_INTERRUPT_ACK                                                                      0x01cb
+#define regDMCUB_EXT_INTERRUPT_ACK_BASE_IDX                                                             2
+#define regDMCUB_INST_FETCH_FAULT_ADDR                                                                  0x01cc
+#define regDMCUB_INST_FETCH_FAULT_ADDR_BASE_IDX                                                         2
+#define regDMCUB_DATA_WRITE_FAULT_ADDR                                                                  0x01cd
+#define regDMCUB_DATA_WRITE_FAULT_ADDR_BASE_IDX                                                         2
+#define regDMCUB_SEC_CNTL                                                                               0x01ce
+#define regDMCUB_SEC_CNTL_BASE_IDX                                                                      2
+#define regDMCUB_MEM_CNTL                                                                               0x01cf
+#define regDMCUB_MEM_CNTL_BASE_IDX                                                                      2
+#define regDMCUB_INBOX0_BASE_ADDRESS                                                                    0x01d0
+#define regDMCUB_INBOX0_BASE_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_INBOX0_SIZE                                                                            0x01d1
+#define regDMCUB_INBOX0_SIZE_BASE_IDX                                                                   2
+#define regDMCUB_INBOX0_WPTR                                                                            0x01d2
+#define regDMCUB_INBOX0_WPTR_BASE_IDX                                                                   2
+#define regDMCUB_INBOX0_RPTR                                                                            0x01d3
+#define regDMCUB_INBOX0_RPTR_BASE_IDX                                                                   2
+#define regDMCUB_INBOX1_BASE_ADDRESS                                                                    0x01d4
+#define regDMCUB_INBOX1_BASE_ADDRESS_BASE_IDX                                                           2
+#define regDMCUB_INBOX1_SIZE                                                                            0x01d5
+#define regDMCUB_INBOX1_SIZE_BASE_IDX                                                                   2
+#define regDMCUB_INBOX1_WPTR                                                                            0x01d6
+#define regDMCUB_INBOX1_WPTR_BASE_IDX                                                                   2
+#define regDMCUB_INBOX1_RPTR                                                                            0x01d7
+#define regDMCUB_INBOX1_RPTR_BASE_IDX                                                                   2
+#define regDMCUB_OUTBOX0_BASE_ADDRESS                                                                   0x01d8
+#define regDMCUB_OUTBOX0_BASE_ADDRESS_BASE_IDX                                                          2
+#define regDMCUB_OUTBOX0_SIZE                                                                           0x01d9
+#define regDMCUB_OUTBOX0_SIZE_BASE_IDX                                                                  2
+#define regDMCUB_OUTBOX0_WPTR                                                                           0x01da
+#define regDMCUB_OUTBOX0_WPTR_BASE_IDX                                                                  2
+#define regDMCUB_OUTBOX0_RPTR                                                                           0x01db
+#define regDMCUB_OUTBOX0_RPTR_BASE_IDX                                                                  2
+#define regDMCUB_OUTBOX1_BASE_ADDRESS                                                                   0x01dc
+#define regDMCUB_OUTBOX1_BASE_ADDRESS_BASE_IDX                                                          2
+#define regDMCUB_OUTBOX1_SIZE                                                                           0x01dd
+#define regDMCUB_OUTBOX1_SIZE_BASE_IDX                                                                  2
+#define regDMCUB_OUTBOX1_WPTR                                                                           0x01de
+#define regDMCUB_OUTBOX1_WPTR_BASE_IDX                                                                  2
+#define regDMCUB_OUTBOX1_RPTR                                                                           0x01df
+#define regDMCUB_OUTBOX1_RPTR_BASE_IDX                                                                  2
+#define regDMCUB_TIMER_TRIGGER0                                                                         0x01e0
+#define regDMCUB_TIMER_TRIGGER0_BASE_IDX                                                                2
+#define regDMCUB_TIMER_TRIGGER1                                                                         0x01e1
+#define regDMCUB_TIMER_TRIGGER1_BASE_IDX                                                                2
+#define regDMCUB_TIMER_WINDOW                                                                           0x01e2
+#define regDMCUB_TIMER_WINDOW_BASE_IDX                                                                  2
+#define regDMCUB_SCRATCH0                                                                               0x01e3
+#define regDMCUB_SCRATCH0_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH1                                                                               0x01e4
+#define regDMCUB_SCRATCH1_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH2                                                                               0x01e5
+#define regDMCUB_SCRATCH2_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH3                                                                               0x01e6
+#define regDMCUB_SCRATCH3_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH4                                                                               0x01e7
+#define regDMCUB_SCRATCH4_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH5                                                                               0x01e8
+#define regDMCUB_SCRATCH5_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH6                                                                               0x01e9
+#define regDMCUB_SCRATCH6_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH7                                                                               0x01ea
+#define regDMCUB_SCRATCH7_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH8                                                                               0x01eb
+#define regDMCUB_SCRATCH8_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH9                                                                               0x01ec
+#define regDMCUB_SCRATCH9_BASE_IDX                                                                      2
+#define regDMCUB_SCRATCH10                                                                              0x01ed
+#define regDMCUB_SCRATCH10_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH11                                                                              0x01ee
+#define regDMCUB_SCRATCH11_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH12                                                                              0x01ef
+#define regDMCUB_SCRATCH12_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH13                                                                              0x01f0
+#define regDMCUB_SCRATCH13_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH14                                                                              0x01f1
+#define regDMCUB_SCRATCH14_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH15                                                                              0x01f2
+#define regDMCUB_SCRATCH15_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH16                                                                              0x01f3
+#define regDMCUB_SCRATCH16_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH17                                                                              0x01f4
+#define regDMCUB_SCRATCH17_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH18                                                                              0x01f5
+#define regDMCUB_SCRATCH18_BASE_IDX                                                                     2
+#define regDMCUB_CNTL                                                                                   0x01f6
+#define regDMCUB_CNTL_BASE_IDX                                                                          2
+#define regDMCUB_GPINT_DATAIN0                                                                          0x01f7
+#define regDMCUB_GPINT_DATAIN0_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAIN1                                                                          0x01f8
+#define regDMCUB_GPINT_DATAIN1_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAOUT                                                                          0x01f9
+#define regDMCUB_GPINT_DATAOUT_BASE_IDX                                                                 2
+#define regDMCUB_UNDEFINED_ADDRESS_FAULT_ADDR                                                           0x01fa
+#define regDMCUB_UNDEFINED_ADDRESS_FAULT_ADDR_BASE_IDX                                                  2
+#define regDMCUB_LS_WAKE_INT_ENABLE                                                                     0x01fb
+#define regDMCUB_LS_WAKE_INT_ENABLE_BASE_IDX                                                            2
+#define regDMCUB_MEM_PWR_CNTL                                                                           0x01fc
+#define regDMCUB_MEM_PWR_CNTL_BASE_IDX                                                                  2
+#define regDMCUB_TIMER_CURRENT                                                                          0x01fd
+#define regDMCUB_TIMER_CURRENT_BASE_IDX                                                                 2
+#define regDMCUB_PROC_ID                                                                                0x01ff
+#define regDMCUB_PROC_ID_BASE_IDX                                                                       2
+#define regDMCUB_CNTL2                                                                                  0x0200
+#define regDMCUB_CNTL2_BASE_IDX                                                                         2
+#define regDMCUB_GPINT_DATAIN2                                                                          0x0215
+#define regDMCUB_GPINT_DATAIN2_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAIN3                                                                          0x0216
+#define regDMCUB_GPINT_DATAIN3_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAIN4                                                                          0x0217
+#define regDMCUB_GPINT_DATAIN4_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAIN5                                                                          0x0218
+#define regDMCUB_GPINT_DATAIN5_BASE_IDX                                                                 2
+#define regDMCUB_GPINT_DATAIN6                                                                          0x0219
+#define regDMCUB_GPINT_DATAIN6_BASE_IDX                                                                 2
+#define regDMCUB_REGION3_TMR_AXI_SPACE                                                                  0x021a
+#define regDMCUB_REGION3_TMR_AXI_SPACE_BASE_IDX                                                         2
+#define regDMCUB_SCRATCH19                                                                              0x021b
+#define regDMCUB_SCRATCH19_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH20                                                                              0x021c
+#define regDMCUB_SCRATCH20_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH21                                                                              0x021d
+#define regDMCUB_SCRATCH21_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH22                                                                              0x021e
+#define regDMCUB_SCRATCH22_BASE_IDX                                                                     2
+#define regDMCUB_SCRATCH23                                                                              0x021f
+#define regDMCUB_SCRATCH23_BASE_IDX                                                                     2
+#define regHOST_INTERRUPT_CSR                                                                           0x0222
+#define regHOST_INTERRUPT_CSR_BASE_IDX                                                                  2
+#define regDMCUB_REG_INBOX0_RDY                                                                         0x0223
+#define regDMCUB_REG_INBOX0_RDY_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX0_MSG0                                                                        0x0224
+#define regDMCUB_REG_INBOX0_MSG0_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG1                                                                        0x0225
+#define regDMCUB_REG_INBOX0_MSG1_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG2                                                                        0x0226
+#define regDMCUB_REG_INBOX0_MSG2_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG3                                                                        0x0227
+#define regDMCUB_REG_INBOX0_MSG3_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG4                                                                        0x0228
+#define regDMCUB_REG_INBOX0_MSG4_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG5                                                                        0x0229
+#define regDMCUB_REG_INBOX0_MSG5_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG6                                                                        0x022a
+#define regDMCUB_REG_INBOX0_MSG6_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG7                                                                        0x022b
+#define regDMCUB_REG_INBOX0_MSG7_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG8                                                                        0x022c
+#define regDMCUB_REG_INBOX0_MSG8_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG9                                                                        0x022d
+#define regDMCUB_REG_INBOX0_MSG9_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX0_MSG10                                                                       0x022e
+#define regDMCUB_REG_INBOX0_MSG10_BASE_IDX                                                              2
+#define regDMCUB_REG_INBOX0_MSG11                                                                       0x022f
+#define regDMCUB_REG_INBOX0_MSG11_BASE_IDX                                                              2
+#define regDMCUB_REG_INBOX0_MSG12                                                                       0x0230
+#define regDMCUB_REG_INBOX0_MSG12_BASE_IDX                                                              2
+#define regDMCUB_REG_INBOX0_MSG13                                                                       0x0231
+#define regDMCUB_REG_INBOX0_MSG13_BASE_IDX                                                              2
+#define regDMCUB_REG_INBOX0_MSG14                                                                       0x0232
+#define regDMCUB_REG_INBOX0_MSG14_BASE_IDX                                                              2
+#define regDMCUB_REG_INBOX0_RSP                                                                         0x0233
+#define regDMCUB_REG_INBOX0_RSP_BASE_IDX                                                                2
+#define regDMCUB_REG_OUTBOX0_RDY                                                                        0x0234
+#define regDMCUB_REG_OUTBOX0_RDY_BASE_IDX                                                               2
+#define regDMCUB_REG_OUTBOX0_MSG0                                                                       0x0235
+#define regDMCUB_REG_OUTBOX0_MSG0_BASE_IDX                                                              2
+#define regDMCUB_REG_OUTBOX0_RSP                                                                        0x0236
+#define regDMCUB_REG_OUTBOX0_RSP_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX1_RDY                                                                         0x0237
+#define regDMCUB_REG_INBOX1_RDY_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX1_MSG0                                                                        0x0238
+#define regDMCUB_REG_INBOX1_MSG0_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX1_MSG1                                                                        0x0239
+#define regDMCUB_REG_INBOX1_MSG1_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX1_RSP                                                                         0x023a
+#define regDMCUB_REG_INBOX1_RSP_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX1_INTERRUPT_CSR                                                               0x023b
+#define regDMCUB_REG_INBOX1_INTERRUPT_CSR_BASE_IDX                                                      2
+#define regDMCUB_REG_INBOX2_RDY                                                                         0x023c
+#define regDMCUB_REG_INBOX2_RDY_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX2_MSG0                                                                        0x023d
+#define regDMCUB_REG_INBOX2_MSG0_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX2_MSG1                                                                        0x023e
+#define regDMCUB_REG_INBOX2_MSG1_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX2_RSP                                                                         0x023f
+#define regDMCUB_REG_INBOX2_RSP_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX3_RDY                                                                         0x0240
+#define regDMCUB_REG_INBOX3_RDY_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX3_MSG0                                                                        0x0241
+#define regDMCUB_REG_INBOX3_MSG0_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX3_MSG1                                                                        0x0242
+#define regDMCUB_REG_INBOX3_MSG1_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX3_RSP                                                                         0x0243
+#define regDMCUB_REG_INBOX3_RSP_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX4_RDY                                                                         0x0244
+#define regDMCUB_REG_INBOX4_RDY_BASE_IDX                                                                2
+#define regDMCUB_REG_INBOX4_MSG0                                                                        0x0245
+#define regDMCUB_REG_INBOX4_MSG0_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX4_MSG1                                                                        0x0246
+#define regDMCUB_REG_INBOX4_MSG1_BASE_IDX                                                               2
+#define regDMCUB_REG_INBOX4_RSP                                                                         0x0247
+#define regDMCUB_REG_INBOX4_RSP_BASE_IDX                                                                2
+#define regDMCUB_IDLE_HYSTERESIS_CSR                                                                    0x0249
+#define regDMCUB_IDLE_HYSTERESIS_CSR_BASE_IDX                                                           2
+
+
+// addressBlock: dce_dc_mmhubbub_mcif_wb0_dispdec
+// base address: 0x0
+#define regMCIF_WB_BUFMGR_SW_CONTROL                                                                    0x0272
+#define regMCIF_WB_BUFMGR_SW_CONTROL_BASE_IDX                                                           2
+#define regMCIF_WB_BUFMGR_STATUS                                                                        0x0274
+#define regMCIF_WB_BUFMGR_STATUS_BASE_IDX                                                               2
+#define regMCIF_WB_BUF_PITCH                                                                            0x0275
+#define regMCIF_WB_BUF_PITCH_BASE_IDX                                                                   2
+#define regMCIF_WB_BUF_1_STATUS                                                                         0x0276
+#define regMCIF_WB_BUF_1_STATUS_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_1_STATUS2                                                                        0x0277
+#define regMCIF_WB_BUF_1_STATUS2_BASE_IDX                                                               2
+#define regMCIF_WB_BUF_2_STATUS                                                                         0x0278
+#define regMCIF_WB_BUF_2_STATUS_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_2_STATUS2                                                                        0x0279
+#define regMCIF_WB_BUF_2_STATUS2_BASE_IDX                                                               2
+#define regMCIF_WB_BUF_3_STATUS                                                                         0x027a
+#define regMCIF_WB_BUF_3_STATUS_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_3_STATUS2                                                                        0x027b
+#define regMCIF_WB_BUF_3_STATUS2_BASE_IDX                                                               2
+#define regMCIF_WB_BUF_4_STATUS                                                                         0x027c
+#define regMCIF_WB_BUF_4_STATUS_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_4_STATUS2                                                                        0x027d
+#define regMCIF_WB_BUF_4_STATUS2_BASE_IDX                                                               2
+#define regMCIF_WB_ARBITRATION_CONTROL                                                                  0x027e
+#define regMCIF_WB_ARBITRATION_CONTROL_BASE_IDX                                                         2
+#define regMCIF_WB_SCLK_CHANGE                                                                          0x027f
+#define regMCIF_WB_SCLK_CHANGE_BASE_IDX                                                                 2
+#define regMCIF_WB_TEST_DEBUG_INDEX                                                                     0x0280
+#define regMCIF_WB_TEST_DEBUG_INDEX_BASE_IDX                                                            2
+#define regMCIF_WB_TEST_DEBUG_DATA                                                                      0x0281
+#define regMCIF_WB_TEST_DEBUG_DATA_BASE_IDX                                                             2
+#define regMCIF_WB_BUF_1_ADDR_Y                                                                         0x0282
+#define regMCIF_WB_BUF_1_ADDR_Y_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_1_ADDR_C                                                                         0x0284
+#define regMCIF_WB_BUF_1_ADDR_C_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_2_ADDR_Y                                                                         0x0286
+#define regMCIF_WB_BUF_2_ADDR_Y_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_2_ADDR_C                                                                         0x0288
+#define regMCIF_WB_BUF_2_ADDR_C_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_3_ADDR_Y                                                                         0x028a
+#define regMCIF_WB_BUF_3_ADDR_Y_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_3_ADDR_C                                                                         0x028c
+#define regMCIF_WB_BUF_3_ADDR_C_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_4_ADDR_Y                                                                         0x028e
+#define regMCIF_WB_BUF_4_ADDR_Y_BASE_IDX                                                                2
+#define regMCIF_WB_BUF_4_ADDR_C                                                                         0x0290
+#define regMCIF_WB_BUF_4_ADDR_C_BASE_IDX                                                                2
+#define regMCIF_WB_BUFMGR_VCE_CONTROL                                                                   0x0292
+#define regMCIF_WB_BUFMGR_VCE_CONTROL_BASE_IDX                                                          2
+#define regMCIF_WB_NB_PSTATE_CONTROL                                                                    0x0293
+#define regMCIF_WB_NB_PSTATE_CONTROL_BASE_IDX                                                           2
+#define regMCIF_WB_CLOCK_GATER_CONTROL                                                                  0x0294
+#define regMCIF_WB_CLOCK_GATER_CONTROL_BASE_IDX                                                         2
+#define regMCIF_WB_SELF_REFRESH_CONTROL                                                                 0x0296
+#define regMCIF_WB_SELF_REFRESH_CONTROL_BASE_IDX                                                        2
+#define regMULTI_LEVEL_QOS_CTRL                                                                         0x0297
+#define regMULTI_LEVEL_QOS_CTRL_BASE_IDX                                                                2
+#define regMCIF_WB_SECURITY_LEVEL                                                                       0x0298
+#define regMCIF_WB_SECURITY_LEVEL_BASE_IDX                                                              2
+#define regMCIF_WB_BUF_LUMA_SIZE                                                                        0x0299
+#define regMCIF_WB_BUF_LUMA_SIZE_BASE_IDX                                                               2
+#define regMCIF_WB_BUF_CHROMA_SIZE                                                                      0x029a
+#define regMCIF_WB_BUF_CHROMA_SIZE_BASE_IDX                                                             2
+#define regMCIF_WB_BUF_1_ADDR_Y_HIGH                                                                    0x029b
+#define regMCIF_WB_BUF_1_ADDR_Y_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_1_ADDR_C_HIGH                                                                    0x029c
+#define regMCIF_WB_BUF_1_ADDR_C_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_2_ADDR_Y_HIGH                                                                    0x029d
+#define regMCIF_WB_BUF_2_ADDR_Y_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_2_ADDR_C_HIGH                                                                    0x029e
+#define regMCIF_WB_BUF_2_ADDR_C_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_3_ADDR_Y_HIGH                                                                    0x029f
+#define regMCIF_WB_BUF_3_ADDR_Y_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_3_ADDR_C_HIGH                                                                    0x02a0
+#define regMCIF_WB_BUF_3_ADDR_C_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_4_ADDR_Y_HIGH                                                                    0x02a1
+#define regMCIF_WB_BUF_4_ADDR_Y_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_4_ADDR_C_HIGH                                                                    0x02a2
+#define regMCIF_WB_BUF_4_ADDR_C_HIGH_BASE_IDX                                                           2
+#define regMCIF_WB_BUF_1_RESOLUTION                                                                     0x02a3
+#define regMCIF_WB_BUF_1_RESOLUTION_BASE_IDX                                                            2
+#define regMCIF_WB_BUF_2_RESOLUTION                                                                     0x02a4
+#define regMCIF_WB_BUF_2_RESOLUTION_BASE_IDX                                                            2
+#define regMCIF_WB_BUF_3_RESOLUTION                                                                     0x02a5
+#define regMCIF_WB_BUF_3_RESOLUTION_BASE_IDX                                                            2
+#define regMCIF_WB_BUF_4_RESOLUTION                                                                     0x02a6
+#define regMCIF_WB_BUF_4_RESOLUTION_BASE_IDX                                                            2
+#define regMCIF_WB_PSTATE_CHANGE_DURATION_VBI                                                           0x02a7
+#define regMCIF_WB_PSTATE_CHANGE_DURATION_VBI_BASE_IDX                                                  2
+#define regMCIF_WB_VMID_CONTROL                                                                         0x02a8
+#define regMCIF_WB_VMID_CONTROL_BASE_IDX                                                                2
+#define regMCIF_WB_MIN_TTO                                                                              0x02a9
+#define regMCIF_WB_MIN_TTO_BASE_IDX                                                                     2
+#define regMCIF_WB_TMZ                                                                                  0x02b2
+#define regMCIF_WB_TMZ_BASE_IDX                                                                         2
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dispdec
+// base address: 0x0
+#define regMCIF_WB_NB_PSTATE_LATENCY_WATERMARK                                                          0x02aa
+#define regMCIF_WB_NB_PSTATE_LATENCY_WATERMARK_BASE_IDX                                                 2
+#define regMCIF_WB_WATERMARK                                                                            0x02ab
+#define regMCIF_WB_WATERMARK_BASE_IDX                                                                   2
+#define regMMHUBBUB_WARMUP_CONFIG                                                                       0x02ac
+#define regMMHUBBUB_WARMUP_CONFIG_BASE_IDX                                                              2
+#define regMMHUBBUB_WARMUP_CONTROL_STATUS                                                               0x02ad
+#define regMMHUBBUB_WARMUP_CONTROL_STATUS_BASE_IDX                                                      2
+#define regMMHUBBUB_WARMUP_BASE_ADDR_LOW                                                                0x02ae
+#define regMMHUBBUB_WARMUP_BASE_ADDR_LOW_BASE_IDX                                                       2
+#define regMMHUBBUB_WARMUP_BASE_ADDR_HIGH                                                               0x02af
+#define regMMHUBBUB_WARMUP_BASE_ADDR_HIGH_BASE_IDX                                                      2
+#define regMMHUBBUB_WARMUP_ADDR_REGION                                                                  0x02b0
+#define regMMHUBBUB_WARMUP_ADDR_REGION_BASE_IDX                                                         2
+#define regMMHUBBUB_MIN_TTO                                                                             0x02b1
+#define regMMHUBBUB_MIN_TTO_BASE_IDX                                                                    2
+#define regMMHUBBUB_CTRL                                                                                0x0333
+#define regMMHUBBUB_CTRL_BASE_IDX                                                                       2
+#define regWBIF_SMU_WM_CONTROL                                                                          0x0334
+#define regWBIF_SMU_WM_CONTROL_BASE_IDX                                                                 2
+#define regWBIF0_MISC_CTRL                                                                              0x0335
+#define regWBIF0_MISC_CTRL_BASE_IDX                                                                     2
+#define regWBIF0_PHASE0_OUTSTANDING_COUNTER                                                             0x0336
+#define regWBIF0_PHASE0_OUTSTANDING_COUNTER_BASE_IDX                                                    2
+#define regWBIF0_PHASE1_OUTSTANDING_COUNTER                                                             0x0337
+#define regWBIF0_PHASE1_OUTSTANDING_COUNTER_BASE_IDX                                                    2
+#define regMMHUBBUB_MEM_PWR_STATUS                                                                      0x033e
+#define regMMHUBBUB_MEM_PWR_STATUS_BASE_IDX                                                             2
+#define regMMHUBBUB_MEM_PWR_CNTL                                                                        0x033f
+#define regMMHUBBUB_MEM_PWR_CNTL_BASE_IDX                                                               2
+#define regMMHUBBUB_CLOCK_CNTL                                                                          0x0340
+#define regMMHUBBUB_CLOCK_CNTL_BASE_IDX                                                                 2
+#define regMMHUBBUB_SOFT_RESET                                                                          0x0341
+#define regMMHUBBUB_SOFT_RESET_BASE_IDX                                                                 2
+#define regDMU_IF_ERR_STATUS                                                                            0x0345
+#define regDMU_IF_ERR_STATUS_BASE_IDX                                                                   2
+#define regMMHUBBUB_CLIENT_UNIT_ID                                                                      0x0346
+#define regMMHUBBUB_CLIENT_UNIT_ID_BASE_IDX                                                             2
+#define regMMHUBBUB_WARMUP_SECURITY_CFG                                                                 0x0347
+#define regMMHUBBUB_WARMUP_SECURITY_CFG_BASE_IDX                                                        2
+#define regMMHUBBUB_WARMUP_VMID_CONTROL                                                                 0x0348
+#define regMMHUBBUB_WARMUP_VMID_CONTROL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dcperfmon_dc_perfmon_dispdec
+// base address: 0xd48
+#define regDC_PERFMON3_PERFCOUNTER_CNTL                                                                 0x0352
+#define regDC_PERFMON3_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON3_PERFCOUNTER_CNTL2                                                                0x0353
+#define regDC_PERFMON3_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON3_PERFCOUNTER_STATE                                                                0x0354
+#define regDC_PERFMON3_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON3_PERFMON_CNTL                                                                     0x0355
+#define regDC_PERFMON3_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON3_PERFMON_CNTL2                                                                    0x0356
+#define regDC_PERFMON3_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON3_PERFMON_CVALUE_INT_MISC                                                          0x0357
+#define regDC_PERFMON3_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON3_PERFMON_CVALUE_LOW                                                               0x0358
+#define regDC_PERFMON3_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON3_PERFMON_HI                                                                       0x0359
+#define regDC_PERFMON3_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON3_PERFMON_LOW                                                                      0x035a
+#define regDC_PERFMON3_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hda_azf0stream0_dispdec
+// base address: 0x0
+#define regAZF0STREAM0_AZALIA_STREAM_INDEX                                                              0x035e
+#define regAZF0STREAM0_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM0_AZALIA_STREAM_DATA                                                               0x035f
+#define regAZF0STREAM0_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream1_dispdec
+// base address: 0x8
+#define regAZF0STREAM1_AZALIA_STREAM_INDEX                                                              0x0360
+#define regAZF0STREAM1_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM1_AZALIA_STREAM_DATA                                                               0x0361
+#define regAZF0STREAM1_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream2_dispdec
+// base address: 0x10
+#define regAZF0STREAM2_AZALIA_STREAM_INDEX                                                              0x0362
+#define regAZF0STREAM2_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM2_AZALIA_STREAM_DATA                                                               0x0363
+#define regAZF0STREAM2_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream3_dispdec
+// base address: 0x18
+#define regAZF0STREAM3_AZALIA_STREAM_INDEX                                                              0x0364
+#define regAZF0STREAM3_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM3_AZALIA_STREAM_DATA                                                               0x0365
+#define regAZF0STREAM3_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream4_dispdec
+// base address: 0x20
+#define regAZF0STREAM4_AZALIA_STREAM_INDEX                                                              0x0366
+#define regAZF0STREAM4_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM4_AZALIA_STREAM_DATA                                                               0x0367
+#define regAZF0STREAM4_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream5_dispdec
+// base address: 0x28
+#define regAZF0STREAM5_AZALIA_STREAM_INDEX                                                              0x0368
+#define regAZF0STREAM5_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM5_AZALIA_STREAM_DATA                                                               0x0369
+#define regAZF0STREAM5_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream6_dispdec
+// base address: 0x30
+#define regAZF0STREAM6_AZALIA_STREAM_INDEX                                                              0x036a
+#define regAZF0STREAM6_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM6_AZALIA_STREAM_DATA                                                               0x036b
+#define regAZF0STREAM6_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream7_dispdec
+// base address: 0x38
+#define regAZF0STREAM7_AZALIA_STREAM_INDEX                                                              0x036c
+#define regAZF0STREAM7_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM7_AZALIA_STREAM_DATA                                                               0x036d
+#define regAZF0STREAM7_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_az_misc_dispdec
+// base address: 0x0
+#define regAZ_CLOCK_CNTL                                                                                0x0372
+#define regAZ_CLOCK_CNTL_BASE_IDX                                                                       2
+#define regAZ_MEM_GLOBAL_PWR_REQ_CNTL                                                                   0x0373
+#define regAZ_MEM_GLOBAL_PWR_REQ_CNTL_BASE_IDX                                                          2
+#define regAZ_STRAPS                                                                                    0x0374
+#define regAZ_STRAPS_BASE_IDX                                                                           2
+
+
+// addressBlock: dce_dc_hda_az_dcperfmon_dc_perfmon_dispdec
+// base address: 0xde8
+#define regDC_PERFMON4_PERFCOUNTER_CNTL                                                                 0x037a
+#define regDC_PERFMON4_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON4_PERFCOUNTER_CNTL2                                                                0x037b
+#define regDC_PERFMON4_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON4_PERFCOUNTER_STATE                                                                0x037c
+#define regDC_PERFMON4_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON4_PERFMON_CNTL                                                                     0x037d
+#define regDC_PERFMON4_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON4_PERFMON_CNTL2                                                                    0x037e
+#define regDC_PERFMON4_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON4_PERFMON_CVALUE_INT_MISC                                                          0x037f
+#define regDC_PERFMON4_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON4_PERFMON_CVALUE_LOW                                                               0x0380
+#define regDC_PERFMON4_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON4_PERFMON_HI                                                                       0x0381
+#define regDC_PERFMON4_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON4_PERFMON_LOW                                                                      0x0382
+#define regDC_PERFMON4_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint0_dispdec
+// base address: 0x0
+#define regAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x0386
+#define regAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x0387
+#define regAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint1_dispdec
+// base address: 0x18
+#define regAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x038c
+#define regAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x038d
+#define regAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint2_dispdec
+// base address: 0x30
+#define regAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x0392
+#define regAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x0393
+#define regAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint3_dispdec
+// base address: 0x48
+#define regAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x0398
+#define regAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x0399
+#define regAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint4_dispdec
+// base address: 0x60
+#define regAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x039e
+#define regAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x039f
+#define regAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint5_dispdec
+// base address: 0x78
+#define regAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x03a4
+#define regAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x03a5
+#define regAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint6_dispdec
+// base address: 0x90
+#define regAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x03aa
+#define regAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x03ab
+#define regAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint7_dispdec
+// base address: 0xa8
+#define regAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX                                                 0x03b0
+#define regAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX                                        2
+#define regAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA                                                  0x03b1
+#define regAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX                                         2
+
+
+// addressBlock: dce_dc_hda_azf0controller_dispdec
+// base address: 0x0
+#define regAZALIA_CONTROLLER_CLOCK_GATING                                                               0x03c2
+#define regAZALIA_CONTROLLER_CLOCK_GATING_BASE_IDX                                                      2
+#define regAZALIA_AUDIO_DTO                                                                             0x03c3
+#define regAZALIA_AUDIO_DTO_BASE_IDX                                                                    2
+#define regAZALIA_AUDIO_DTO_CONTROL                                                                     0x03c4
+#define regAZALIA_AUDIO_DTO_CONTROL_BASE_IDX                                                            2
+#define regAZALIA_SOCCLK_CONTROL                                                                        0x03c5
+#define regAZALIA_SOCCLK_CONTROL_BASE_IDX                                                               2
+#define regAZALIA_UNDERFLOW_FILLER_SAMPLE                                                               0x03c6
+#define regAZALIA_UNDERFLOW_FILLER_SAMPLE_BASE_IDX                                                      2
+#define regAZALIA_DATA_DMA_CONTROL                                                                      0x03c7
+#define regAZALIA_DATA_DMA_CONTROL_BASE_IDX                                                             2
+#define regAZALIA_BDL_DMA_CONTROL                                                                       0x03c8
+#define regAZALIA_BDL_DMA_CONTROL_BASE_IDX                                                              2
+#define regAZALIA_RIRB_AND_DP_CONTROL                                                                   0x03c9
+#define regAZALIA_RIRB_AND_DP_CONTROL_BASE_IDX                                                          2
+#define regAZALIA_CORB_DMA_CONTROL                                                                      0x03ca
+#define regAZALIA_CORB_DMA_CONTROL_BASE_IDX                                                             2
+#define regAZALIA_GLOBAL_CAPABILITIES                                                                   0x03d3
+#define regAZALIA_GLOBAL_CAPABILITIES_BASE_IDX                                                          2
+#define regAZALIA_OUTPUT_PAYLOAD_CAPABILITY                                                             0x03d4
+#define regAZALIA_OUTPUT_PAYLOAD_CAPABILITY_BASE_IDX                                                    2
+#define regAZALIA_OUTPUT_STREAM_ARBITER_CONTROL                                                         0x03d5
+#define regAZALIA_OUTPUT_STREAM_ARBITER_CONTROL_BASE_IDX                                                2
+#define regAZALIA_INPUT_PAYLOAD_CAPABILITY                                                              0x03d6
+#define regAZALIA_INPUT_PAYLOAD_CAPABILITY_BASE_IDX                                                     2
+#define regAZALIA_INPUT_CRC0_CONTROL0                                                                   0x03d9
+#define regAZALIA_INPUT_CRC0_CONTROL0_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC0_CONTROL1                                                                   0x03da
+#define regAZALIA_INPUT_CRC0_CONTROL1_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC0_CONTROL2                                                                   0x03db
+#define regAZALIA_INPUT_CRC0_CONTROL2_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC0_CONTROL3                                                                   0x03dc
+#define regAZALIA_INPUT_CRC0_CONTROL3_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC0_RESULT                                                                     0x03dd
+#define regAZALIA_INPUT_CRC0_RESULT_BASE_IDX                                                            2
+#define regAZALIA_INPUT_CRC1_CONTROL0                                                                   0x03de
+#define regAZALIA_INPUT_CRC1_CONTROL0_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC1_CONTROL1                                                                   0x03df
+#define regAZALIA_INPUT_CRC1_CONTROL1_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC1_CONTROL2                                                                   0x03e0
+#define regAZALIA_INPUT_CRC1_CONTROL2_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC1_CONTROL3                                                                   0x03e1
+#define regAZALIA_INPUT_CRC1_CONTROL3_BASE_IDX                                                          2
+#define regAZALIA_INPUT_CRC1_RESULT                                                                     0x03e2
+#define regAZALIA_INPUT_CRC1_RESULT_BASE_IDX                                                            2
+#define regAZALIA_CRC0_CONTROL0                                                                         0x03e3
+#define regAZALIA_CRC0_CONTROL0_BASE_IDX                                                                2
+#define regAZALIA_CRC0_CONTROL1                                                                         0x03e4
+#define regAZALIA_CRC0_CONTROL1_BASE_IDX                                                                2
+#define regAZALIA_CRC0_CONTROL2                                                                         0x03e5
+#define regAZALIA_CRC0_CONTROL2_BASE_IDX                                                                2
+#define regAZALIA_CRC0_CONTROL3                                                                         0x03e6
+#define regAZALIA_CRC0_CONTROL3_BASE_IDX                                                                2
+#define regAZALIA_CRC0_RESULT                                                                           0x03e7
+#define regAZALIA_CRC0_RESULT_BASE_IDX                                                                  2
+#define regAZALIA_CRC1_CONTROL0                                                                         0x03e8
+#define regAZALIA_CRC1_CONTROL0_BASE_IDX                                                                2
+#define regAZALIA_CRC1_CONTROL1                                                                         0x03e9
+#define regAZALIA_CRC1_CONTROL1_BASE_IDX                                                                2
+#define regAZALIA_CRC1_CONTROL2                                                                         0x03ea
+#define regAZALIA_CRC1_CONTROL2_BASE_IDX                                                                2
+#define regAZALIA_CRC1_CONTROL3                                                                         0x03eb
+#define regAZALIA_CRC1_CONTROL3_BASE_IDX                                                                2
+#define regAZALIA_CRC1_RESULT                                                                           0x03ec
+#define regAZALIA_CRC1_RESULT_BASE_IDX                                                                  2
+#define regAZALIA_MEM_PWR_CTRL                                                                          0x03ee
+#define regAZALIA_MEM_PWR_CTRL_BASE_IDX                                                                 2
+#define regAZALIA_MEM_PWR_STATUS                                                                        0x03ef
+#define regAZALIA_MEM_PWR_STATUS_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_hda_azf0root_dispdec
+// base address: 0x0
+#define regAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID                                          0x0406
+#define regAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_BASE_IDX                                 2
+#define regAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID                                                   0x0407
+#define regAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID_BASE_IDX                                          2
+#define regAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL                                                        0x0408
+#define regAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL_BASE_IDX                                               2
+#define regAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL                                                          0x0409
+#define regAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL_BASE_IDX                                                 2
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE                                                0x040a
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_BASE_IDX                                       2
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES                                      0x040b
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_BASE_IDX                             2
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS                                            0x040c
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_BASE_IDX                                   2
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES                                              0x040d
+#define regAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES_BASE_IDX                                     2
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE                                                 0x040e
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE_BASE_IDX                                        2
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET                                                       0x040f
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET_BASE_IDX                                              2
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID                                       0x0410
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_BASE_IDX                              2
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION                                   0x0411
+#define regAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_BASE_IDX                          2
+#define regCC_RCU_DC_AUDIO_PORT_CONNECTIVITY                                                            0x0412
+#define regCC_RCU_DC_AUDIO_PORT_CONNECTIVITY_BASE_IDX                                                   2
+#define regCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY                                                      0x0413
+#define regCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY_BASE_IDX                                             2
+#define regREG_DC_AUDIO_PORT_CONNECTIVITY                                                               0x041c
+#define regREG_DC_AUDIO_PORT_CONNECTIVITY_BASE_IDX                                                      2
+#define regREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY                                                         0x041d
+#define regREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY_BASE_IDX                                                2
+
+
+// addressBlock: dce_dc_hda_azf0stream8_dispdec
+// base address: 0x320
+#define regAZF0STREAM8_AZALIA_STREAM_INDEX                                                              0x0426
+#define regAZF0STREAM8_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM8_AZALIA_STREAM_DATA                                                               0x0427
+#define regAZF0STREAM8_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream9_dispdec
+// base address: 0x328
+#define regAZF0STREAM9_AZALIA_STREAM_INDEX                                                              0x0428
+#define regAZF0STREAM9_AZALIA_STREAM_INDEX_BASE_IDX                                                     2
+#define regAZF0STREAM9_AZALIA_STREAM_DATA                                                               0x0429
+#define regAZF0STREAM9_AZALIA_STREAM_DATA_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hda_azf0stream10_dispdec
+// base address: 0x330
+#define regAZF0STREAM10_AZALIA_STREAM_INDEX                                                             0x042a
+#define regAZF0STREAM10_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM10_AZALIA_STREAM_DATA                                                              0x042b
+#define regAZF0STREAM10_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0stream11_dispdec
+// base address: 0x338
+#define regAZF0STREAM11_AZALIA_STREAM_INDEX                                                             0x042c
+#define regAZF0STREAM11_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM11_AZALIA_STREAM_DATA                                                              0x042d
+#define regAZF0STREAM11_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0stream12_dispdec
+// base address: 0x340
+#define regAZF0STREAM12_AZALIA_STREAM_INDEX                                                             0x042e
+#define regAZF0STREAM12_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM12_AZALIA_STREAM_DATA                                                              0x042f
+#define regAZF0STREAM12_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0stream13_dispdec
+// base address: 0x348
+#define regAZF0STREAM13_AZALIA_STREAM_INDEX                                                             0x0430
+#define regAZF0STREAM13_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM13_AZALIA_STREAM_DATA                                                              0x0431
+#define regAZF0STREAM13_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0stream14_dispdec
+// base address: 0x350
+#define regAZF0STREAM14_AZALIA_STREAM_INDEX                                                             0x0432
+#define regAZF0STREAM14_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM14_AZALIA_STREAM_DATA                                                              0x0433
+#define regAZF0STREAM14_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0stream15_dispdec
+// base address: 0x358
+#define regAZF0STREAM15_AZALIA_STREAM_INDEX                                                             0x0434
+#define regAZF0STREAM15_AZALIA_STREAM_INDEX_BASE_IDX                                                    2
+#define regAZF0STREAM15_AZALIA_STREAM_DATA                                                              0x0435
+#define regAZF0STREAM15_AZALIA_STREAM_DATA_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint0_dispdec
+// base address: 0x0
+#define regAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x043a
+#define regAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x043b
+#define regAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint1_dispdec
+// base address: 0x10
+#define regAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x043e
+#define regAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x043f
+#define regAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint2_dispdec
+// base address: 0x20
+#define regAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x0442
+#define regAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x0443
+#define regAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint3_dispdec
+// base address: 0x30
+#define regAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x0446
+#define regAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x0447
+#define regAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint4_dispdec
+// base address: 0x40
+#define regAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x044a
+#define regAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x044b
+#define regAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint5_dispdec
+// base address: 0x50
+#define regAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x044e
+#define regAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x044f
+#define regAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint6_dispdec
+// base address: 0x60
+#define regAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x0452
+#define regAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x0453
+#define regAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint7_dispdec
+// base address: 0x70
+#define regAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX                                      0x0456
+#define regAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX                             2
+#define regAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA                                       0x0457
+#define regAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX                              2
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_sdpif_dispdec
+// base address: 0x0
+#define regDCHUBBUB_SDPIF_CFG0                                                                          0x046f
+#define regDCHUBBUB_SDPIF_CFG0_BASE_IDX                                                                 2
+#define regDCHUBBUB_SDPIF_CFG1                                                                          0x0470
+#define regDCHUBBUB_SDPIF_CFG1_BASE_IDX                                                                 2
+#define regDCHUBBUB_SDPIF_CFG2                                                                          0x0471
+#define regDCHUBBUB_SDPIF_CFG2_BASE_IDX                                                                 2
+#define regVM_REQUEST_PHYSICAL                                                                          0x0472
+#define regVM_REQUEST_PHYSICAL_BASE_IDX                                                                 2
+#define regDCHUBBUB_FORCE_IO_STATUS_0                                                                   0x0473
+#define regDCHUBBUB_FORCE_IO_STATUS_0_BASE_IDX                                                          2
+#define regDCHUBBUB_FORCE_IO_STATUS_1                                                                   0x0474
+#define regDCHUBBUB_FORCE_IO_STATUS_1_BASE_IDX                                                          2
+#define regDCN_VM_FB_LOCATION_BASE                                                                      0x0475
+#define regDCN_VM_FB_LOCATION_BASE_BASE_IDX                                                             2
+#define regDCN_VM_FB_LOCATION_TOP                                                                       0x0476
+#define regDCN_VM_FB_LOCATION_TOP_BASE_IDX                                                              2
+#define regDCN_VM_FB_OFFSET                                                                             0x0477
+#define regDCN_VM_FB_OFFSET_BASE_IDX                                                                    2
+#define regDCN_VM_AGP_BOT                                                                               0x0478
+#define regDCN_VM_AGP_BOT_BASE_IDX                                                                      2
+#define regDCN_VM_AGP_TOP                                                                               0x0479
+#define regDCN_VM_AGP_TOP_BASE_IDX                                                                      2
+#define regDCN_VM_AGP_BASE                                                                              0x047a
+#define regDCN_VM_AGP_BASE_BASE_IDX                                                                     2
+#define regDCN_VM_LOCAL_HBM_ADDRESS_START                                                               0x047b
+#define regDCN_VM_LOCAL_HBM_ADDRESS_START_BASE_IDX                                                      2
+#define regDCN_VM_LOCAL_HBM_ADDRESS_END                                                                 0x047c
+#define regDCN_VM_LOCAL_HBM_ADDRESS_END_BASE_IDX                                                        2
+#define regDCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL                                                           0x047d
+#define regDCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_BASE_IDX                                                  2
+#define regDCHUBBUB_SDPIF_PIPE_SEC_LVL                                                                  0x047e
+#define regDCHUBBUB_SDPIF_PIPE_SEC_LVL_BASE_IDX                                                         2
+#define regDCHUBBUB_SDPIF_PIPE_NOALLOC                                                                  0x047f
+#define regDCHUBBUB_SDPIF_PIPE_NOALLOC_BASE_IDX                                                         2
+#define regDCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL                                                           0x0480
+#define regDCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL_BASE_IDX                                                  2
+#define regDCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL                                                          0x0481
+#define regDCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL_BASE_IDX                                                 2
+#define regDCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL                                                          0x0482
+#define regDCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL_BASE_IDX                                                 2
+#define regDCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL                                                            0x0483
+#define regDCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL_BASE_IDX                                                   2
+#define regDCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL                                                            0x0484
+#define regDCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL_BASE_IDX                                                   2
+#define regSDPIF_REQUEST_RATE_LIMIT                                                                     0x0485
+#define regSDPIF_REQUEST_RATE_LIMIT_BASE_IDX                                                            2
+#define regDCHUBBUB_SDPIF_MEM_PWR_CTRL                                                                  0x0486
+#define regDCHUBBUB_SDPIF_MEM_PWR_CTRL_BASE_IDX                                                         2
+#define regDCHUBBUB_SDPIF_MEM_PWR_STATUS                                                                0x0487
+#define regDCHUBBUB_SDPIF_MEM_PWR_STATUS_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_ret_path_dispdec
+// base address: 0x0
+#define regDCHUBBUB_RET_PATH_MEM_PWR_CTRL                                                               0x04af
+#define regDCHUBBUB_RET_PATH_MEM_PWR_CTRL_BASE_IDX                                                      2
+#define regDCHUBBUB_RET_PATH_MEM_PWR_STATUS                                                             0x04b0
+#define regDCHUBBUB_RET_PATH_MEM_PWR_STATUS_BASE_IDX                                                    2
+#define regDCHUBBUB_CRC_CTRL                                                                            0x04b1
+#define regDCHUBBUB_CRC_CTRL_BASE_IDX                                                                   2
+#define regDCHUBBUB_CRC0_VAL_R                                                                          0x04b2
+#define regDCHUBBUB_CRC0_VAL_R_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC0_VAL_G                                                                          0x04b3
+#define regDCHUBBUB_CRC0_VAL_G_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC0_VAL_B                                                                          0x04b4
+#define regDCHUBBUB_CRC0_VAL_B_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC0_VAL_A                                                                          0x04b5
+#define regDCHUBBUB_CRC0_VAL_A_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC1_VAL_R                                                                          0x04b6
+#define regDCHUBBUB_CRC1_VAL_R_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC1_VAL_G                                                                          0x04b7
+#define regDCHUBBUB_CRC1_VAL_G_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC1_VAL_B                                                                          0x04b8
+#define regDCHUBBUB_CRC1_VAL_B_BASE_IDX                                                                 2
+#define regDCHUBBUB_CRC1_VAL_A                                                                          0x04b9
+#define regDCHUBBUB_CRC1_VAL_A_BASE_IDX                                                                 2
+#define regDCHUBBUB_DCC_STAT_CNTL                                                                       0x04ba
+#define regDCHUBBUB_DCC_STAT_CNTL_BASE_IDX                                                              2
+#define regDCHUBBUB_DCC_STAT0                                                                           0x04bb
+#define regDCHUBBUB_DCC_STAT0_BASE_IDX                                                                  2
+#define regDCHUBBUB_DCC_STAT1                                                                           0x04bc
+#define regDCHUBBUB_DCC_STAT1_BASE_IDX                                                                  2
+#define regDCHUBBUB_DCC_STAT2                                                                           0x04bd
+#define regDCHUBBUB_DCC_STAT2_BASE_IDX                                                                  2
+#define regDCHUBBUB_COMPBUF_CTRL                                                                        0x04be
+#define regDCHUBBUB_COMPBUF_CTRL_BASE_IDX                                                               2
+#define regDCHUBBUB_DET0_CTRL                                                                           0x04bf
+#define regDCHUBBUB_DET0_CTRL_BASE_IDX                                                                  2
+#define regDCHUBBUB_DET1_CTRL                                                                           0x04c0
+#define regDCHUBBUB_DET1_CTRL_BASE_IDX                                                                  2
+#define regDCHUBBUB_DET2_CTRL                                                                           0x04c1
+#define regDCHUBBUB_DET2_CTRL_BASE_IDX                                                                  2
+#define regDCHUBBUB_DET3_CTRL                                                                           0x04c2
+#define regDCHUBBUB_DET3_CTRL_BASE_IDX                                                                  2
+#define regDCHUBBUB_MEM_PWR_MODE_CTRL                                                                   0x04c4
+#define regDCHUBBUB_MEM_PWR_MODE_CTRL_BASE_IDX                                                          2
+#define regCOMPBUF_MEM_PWR_CTRL_1                                                                       0x04c5
+#define regCOMPBUF_MEM_PWR_CTRL_1_BASE_IDX                                                              2
+#define regCOMPBUF_MEM_PWR_CTRL_2                                                                       0x04c6
+#define regCOMPBUF_MEM_PWR_CTRL_2_BASE_IDX                                                              2
+#define regDCHUBBUB_MEM_PWR_STATUS                                                                      0x04c7
+#define regDCHUBBUB_MEM_PWR_STATUS_BASE_IDX                                                             2
+#define regCOMPBUF_RESERVED_SPACE                                                                       0x04c8
+#define regCOMPBUF_RESERVED_SPACE_BASE_IDX                                                              2
+#define regDCHUBBUB_DEBUG_CTRL_0                                                                        0x04c9
+#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                                               2
+#define regDCHUBBUB_RET_PATH_TEST_DEBUG_INDEX                                                           0x04cc
+#define regDCHUBBUB_RET_PATH_TEST_DEBUG_INDEX_BASE_IDX                                                  2
+#define regDCHUBBUB_RET_PATH_TEST_DEBUG_DATA                                                            0x04cd
+#define regDCHUBBUB_RET_PATH_TEST_DEBUG_DATA_BASE_IDX                                                   2
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_dispdec
+// base address: 0x0
+#define regDCHUBBUB_ARB_DF_REQ_OUTSTAND                                                                 0x04ef
+#define regDCHUBBUB_ARB_DF_REQ_OUTSTAND_BASE_IDX                                                        2
+#define regDCHUBBUB_ARB_SAT_LEVEL                                                                       0x04f0
+#define regDCHUBBUB_ARB_SAT_LEVEL_BASE_IDX                                                              2
+#define regDCHUBBUB_ARB_QOS_FORCE                                                                       0x04f1
+#define regDCHUBBUB_ARB_QOS_FORCE_BASE_IDX                                                              2
+#define regDCHUBBUB_ARB_DRAM_STATE_CNTL                                                                 0x04f2
+#define regDCHUBBUB_ARB_DRAM_STATE_CNTL_BASE_IDX                                                        2
+#define regDCHUBBUB_ARB_DRAM_HVM_STATE_CNTL                                                             0x04f3
+#define regDCHUBBUB_ARB_DRAM_HVM_STATE_CNTL_BASE_IDX                                                    2
+#define regDCHUBBUB_ARB_USR_RETRAINING_CNTL                                                             0x04f4
+#define regDCHUBBUB_ARB_USR_RETRAINING_CNTL_BASE_IDX                                                    2
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A                                                        0x04f5
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A_BASE_IDX                                               2
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A                                                      0x04f6
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A                                                     0x04f7
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A_BASE_IDX                                            2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A                                                      0x04f8
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A                                                   0x04f9
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A_BASE_IDX                                          2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A                                                       0x04fa
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A_BASE_IDX                                              2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A                                                    0x04fb
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A_BASE_IDX                                           2
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A                                                  0x04fc
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A                                                  0x04fd
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_A                                                               0x04fe
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_A_BASE_IDX                                                      2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_A                                                              0x04ff
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_A_BASE_IDX                                                     2
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B                                                        0x0500
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B_BASE_IDX                                               2
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B                                                      0x0501
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B                                                     0x0502
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B_BASE_IDX                                            2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B                                                      0x0503
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B                                                   0x0504
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B_BASE_IDX                                          2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B                                                       0x0505
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B_BASE_IDX                                              2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B                                                    0x0506
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B_BASE_IDX                                           2
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B                                                  0x0507
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B                                                  0x0508
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_B                                                               0x0509
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_B_BASE_IDX                                                      2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_B                                                              0x050a
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_B_BASE_IDX                                                     2
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C                                                        0x050b
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C_BASE_IDX                                               2
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C                                                      0x050c
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C                                                     0x050d
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C_BASE_IDX                                            2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C                                                      0x050e
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C                                                   0x050f
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C_BASE_IDX                                          2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C                                                       0x0510
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C_BASE_IDX                                              2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C                                                    0x0511
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C_BASE_IDX                                           2
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C                                                  0x0512
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C                                                  0x0513
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_C                                                               0x0514
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_C_BASE_IDX                                                      2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_C                                                              0x0515
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_C_BASE_IDX                                                     2
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D                                                        0x0516
+#define regDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D_BASE_IDX                                               2
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D                                                      0x0517
+#define regDCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D                                                     0x0518
+#define regDCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D_BASE_IDX                                            2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D                                                      0x0519
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D_BASE_IDX                                             2
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D                                                   0x051a
+#define regDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D_BASE_IDX                                          2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D                                                       0x051b
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D_BASE_IDX                                              2
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D                                                    0x051c
+#define regDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D_BASE_IDX                                           2
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D                                                  0x051d
+#define regDCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D                                                  0x051e
+#define regDCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D_BASE_IDX                                         2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_D                                                               0x051f
+#define regDCHUBBUB_ARB_FRAC_URG_BW_NOM_D_BASE_IDX                                                      2
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_D                                                              0x0520
+#define regDCHUBBUB_ARB_FRAC_URG_BW_FLIP_D_BASE_IDX                                                     2
+#define regDCHUBBUB_ARB_HOSTVM_CNTL                                                                     0x0521
+#define regDCHUBBUB_ARB_HOSTVM_CNTL_BASE_IDX                                                            2
+#define regDCHUBBUB_ARB_WATERMARK_CHANGE_CNTL                                                           0x0522
+#define regDCHUBBUB_ARB_WATERMARK_CHANGE_CNTL_BASE_IDX                                                  2
+#define regDCHUBBUB_ARB_MALL_CNTL                                                                       0x0523
+#define regDCHUBBUB_ARB_MALL_CNTL_BASE_IDX                                                              2
+#define regDCHUBBUB_ARB_TIMEOUT_ENABLE                                                                  0x0524
+#define regDCHUBBUB_ARB_TIMEOUT_ENABLE_BASE_IDX                                                         2
+#define regDCHUBBUB_GLOBAL_TIMER_CNTL                                                                   0x0525
+#define regDCHUBBUB_GLOBAL_TIMER_CNTL_BASE_IDX                                                          2
+#define regSURFACE_CHECK0_ADDRESS_LSB                                                                   0x0526
+#define regSURFACE_CHECK0_ADDRESS_LSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK0_ADDRESS_MSB                                                                   0x0527
+#define regSURFACE_CHECK0_ADDRESS_MSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK1_ADDRESS_LSB                                                                   0x0528
+#define regSURFACE_CHECK1_ADDRESS_LSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK1_ADDRESS_MSB                                                                   0x0529
+#define regSURFACE_CHECK1_ADDRESS_MSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK2_ADDRESS_LSB                                                                   0x052a
+#define regSURFACE_CHECK2_ADDRESS_LSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK2_ADDRESS_MSB                                                                   0x052b
+#define regSURFACE_CHECK2_ADDRESS_MSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK3_ADDRESS_LSB                                                                   0x052c
+#define regSURFACE_CHECK3_ADDRESS_LSB_BASE_IDX                                                          2
+#define regSURFACE_CHECK3_ADDRESS_MSB                                                                   0x052d
+#define regSURFACE_CHECK3_ADDRESS_MSB_BASE_IDX                                                          2
+#define regVTG0_CONTROL                                                                                 0x052e
+#define regVTG0_CONTROL_BASE_IDX                                                                        2
+#define regVTG1_CONTROL                                                                                 0x052f
+#define regVTG1_CONTROL_BASE_IDX                                                                        2
+#define regVTG2_CONTROL                                                                                 0x0530
+#define regVTG2_CONTROL_BASE_IDX                                                                        2
+#define regVTG3_CONTROL                                                                                 0x0531
+#define regVTG3_CONTROL_BASE_IDX                                                                        2
+#define regDCHUBBUB_SOFT_RESET                                                                          0x0532
+#define regDCHUBBUB_SOFT_RESET_BASE_IDX                                                                 2
+#define regDCHUBBUB_CLOCK_CNTL                                                                          0x0533
+#define regDCHUBBUB_CLOCK_CNTL_BASE_IDX                                                                 2
+#define regDCFCLK_CNTL                                                                                  0x0534
+#define regDCFCLK_CNTL_BASE_IDX                                                                         2
+#define regDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL                                                        0x0535
+#define regDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL_BASE_IDX                                               2
+#define regDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2                                                       0x0536
+#define regDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2_BASE_IDX                                              2
+#define regDCHUBBUB_VLINE_SNAPSHOT                                                                      0x0537
+#define regDCHUBBUB_VLINE_SNAPSHOT_BASE_IDX                                                             2
+#define regDCHUBBUB_CTRL_STATUS                                                                         0x0538
+#define regDCHUBBUB_CTRL_STATUS_BASE_IDX                                                                2
+#define regDCHUBBUB_TIMEOUT_DETECTION_CTRL1                                                             0x053e
+#define regDCHUBBUB_TIMEOUT_DETECTION_CTRL1_BASE_IDX                                                    2
+#define regDCHUBBUB_TIMEOUT_DETECTION_CTRL2                                                             0x053f
+#define regDCHUBBUB_TIMEOUT_DETECTION_CTRL2_BASE_IDX                                                    2
+#define regDCHUBBUB_TIMEOUT_INTERRUPT_STATUS                                                            0x0540
+#define regDCHUBBUB_TIMEOUT_INTERRUPT_STATUS_BASE_IDX                                                   2
+#define regFMON_CTRL                                                                                    0x0541
+#define regFMON_CTRL_BASE_IDX                                                                           2
+#define regDCHUBBUB_TEST_DEBUG_INDEX                                                                    0x0542
+#define regDCHUBBUB_TEST_DEBUG_INDEX_BASE_IDX                                                           2
+#define regDCHUBBUB_TEST_DEBUG_DATA                                                                     0x0543
+#define regDCHUBBUB_TEST_DEBUG_DATA_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dchubbubl_dchubbub_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1534
+#define regDC_PERFMON5_PERFCOUNTER_CNTL                                                                 0x054d
+#define regDC_PERFMON5_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON5_PERFCOUNTER_CNTL2                                                                0x054e
+#define regDC_PERFMON5_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON5_PERFCOUNTER_STATE                                                                0x054f
+#define regDC_PERFMON5_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON5_PERFMON_CNTL                                                                     0x0550
+#define regDC_PERFMON5_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON5_PERFMON_CNTL2                                                                    0x0551
+#define regDC_PERFMON5_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON5_PERFMON_CVALUE_INT_MISC                                                          0x0552
+#define regDC_PERFMON5_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON5_PERFMON_CVALUE_LOW                                                               0x0553
+#define regDC_PERFMON5_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON5_PERFMON_HI                                                                       0x0554
+#define regDC_PERFMON5_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON5_PERFMON_LOW                                                                      0x0555
+#define regDC_PERFMON5_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_vmrq_if_dispdec
+// base address: 0x0
+#define regDCN_VM_CONTEXT0_CNTL                                                                         0x0559
+#define regDCN_VM_CONTEXT0_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                    0x055a
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                    0x055b
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                   0x055c
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                   0x055d
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                     0x055e
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                     0x055f
+#define regDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT1_CNTL                                                                         0x0560
+#define regDCN_VM_CONTEXT1_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0561
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0562
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                   0x0563
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                   0x0564
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                     0x0565
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                     0x0566
+#define regDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT2_CNTL                                                                         0x0567
+#define regDCN_VM_CONTEXT2_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0568
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0569
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                   0x056a
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                   0x056b
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                     0x056c
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                     0x056d
+#define regDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT3_CNTL                                                                         0x056e
+#define regDCN_VM_CONTEXT3_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                    0x056f
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0570
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                   0x0571
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                   0x0572
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                     0x0573
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                     0x0574
+#define regDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT4_CNTL                                                                         0x0575
+#define regDCN_VM_CONTEXT4_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0576
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0577
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                   0x0578
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                   0x0579
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                     0x057a
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                     0x057b
+#define regDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT5_CNTL                                                                         0x057c
+#define regDCN_VM_CONTEXT5_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                    0x057d
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                    0x057e
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                   0x057f
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                   0x0580
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                     0x0581
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                     0x0582
+#define regDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT6_CNTL                                                                         0x0583
+#define regDCN_VM_CONTEXT6_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0584
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0585
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                   0x0586
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                   0x0587
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                     0x0588
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                     0x0589
+#define regDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT7_CNTL                                                                         0x058a
+#define regDCN_VM_CONTEXT7_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                    0x058b
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                    0x058c
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                   0x058d
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                   0x058e
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                     0x058f
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                     0x0590
+#define regDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT8_CNTL                                                                         0x0591
+#define regDCN_VM_CONTEXT8_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0592
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                    0x0593
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                   0x0594
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                   0x0595
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                     0x0596
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                     0x0597
+#define regDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT9_CNTL                                                                         0x0598
+#define regDCN_VM_CONTEXT9_CNTL_BASE_IDX                                                                2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                    0x0599
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                    0x059a
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                   0x059b
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                   0x059c
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                     0x059d
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                     0x059e
+#define regDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                            2
+#define regDCN_VM_CONTEXT10_CNTL                                                                        0x059f
+#define regDCN_VM_CONTEXT10_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05a0
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05a1
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                  0x05a2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                  0x05a3
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                    0x05a4
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                    0x05a5
+#define regDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT11_CNTL                                                                        0x05a6
+#define regDCN_VM_CONTEXT11_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05a7
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05a8
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                  0x05a9
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                  0x05aa
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                    0x05ab
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                    0x05ac
+#define regDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT12_CNTL                                                                        0x05ad
+#define regDCN_VM_CONTEXT12_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05ae
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05af
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                  0x05b0
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                  0x05b1
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                    0x05b2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                    0x05b3
+#define regDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT13_CNTL                                                                        0x05b4
+#define regDCN_VM_CONTEXT13_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05b5
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05b6
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                  0x05b7
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                  0x05b8
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                    0x05b9
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                    0x05ba
+#define regDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT14_CNTL                                                                        0x05bb
+#define regDCN_VM_CONTEXT14_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05bc
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05bd
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                  0x05be
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                  0x05bf
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                    0x05c0
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                    0x05c1
+#define regDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT15_CNTL                                                                        0x05c2
+#define regDCN_VM_CONTEXT15_CNTL_BASE_IDX                                                               2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                   0x05c3
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                   0x05c4
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                          2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                  0x05c5
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                  0x05c6
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                         2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                    0x05c7
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                           2
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                    0x05c8
+#define regDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                           2
+#define regDCN_VM_DEFAULT_ADDR_MSB                                                                      0x05c9
+#define regDCN_VM_DEFAULT_ADDR_MSB_BASE_IDX                                                             2
+#define regDCN_VM_DEFAULT_ADDR_LSB                                                                      0x05ca
+#define regDCN_VM_DEFAULT_ADDR_LSB_BASE_IDX                                                             2
+#define regDCN_VM_FAULT_CNTL                                                                            0x05cb
+#define regDCN_VM_FAULT_CNTL_BASE_IDX                                                                   2
+#define regDCN_VM_FAULT_STATUS                                                                          0x05cc
+#define regDCN_VM_FAULT_STATUS_BASE_IDX                                                                 2
+#define regDCN_VM_FAULT_ADDR_MSB                                                                        0x05cd
+#define regDCN_VM_FAULT_ADDR_MSB_BASE_IDX                                                               2
+#define regDCN_VM_FAULT_ADDR_LSB                                                                        0x05ce
+#define regDCN_VM_FAULT_ADDR_LSB_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dispdec
+// base address: 0x0
+#define regHUBP0_DCSURF_SURFACE_CONFIG                                                                  0x05e5
+#define regHUBP0_DCSURF_SURFACE_CONFIG_BASE_IDX                                                         2
+#define regHUBP0_DCSURF_ADDR_CONFIG                                                                     0x05e6
+#define regHUBP0_DCSURF_ADDR_CONFIG_BASE_IDX                                                            2
+#define regHUBP0_DCSURF_TILING_CONFIG                                                                   0x05e7
+#define regHUBP0_DCSURF_TILING_CONFIG_BASE_IDX                                                          2
+#define regHUBP0_DCSURF_PRI_VIEWPORT_START                                                              0x05e9
+#define regHUBP0_DCSURF_PRI_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x05ea
+#define regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP0_DCSURF_PRI_VIEWPORT_START_C                                                            0x05eb
+#define regHUBP0_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C                                                        0x05ec
+#define regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP0_DCSURF_SEC_VIEWPORT_START                                                              0x05ed
+#define regHUBP0_DCSURF_SEC_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION                                                          0x05ee
+#define regHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP0_DCSURF_SEC_VIEWPORT_START_C                                                            0x05ef
+#define regHUBP0_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C                                                        0x05f0
+#define regHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP0_DCHUBP_REQ_SIZE_CONFIG                                                                 0x05f1
+#define regHUBP0_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX                                                        2
+#define regHUBP0_DCHUBP_REQ_SIZE_CONFIG_C                                                               0x05f2
+#define regHUBP0_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX                                                      2
+#define regHUBP0_DCHUBP_CNTL                                                                            0x05f3
+#define regHUBP0_DCHUBP_CNTL_BASE_IDX                                                                   2
+#define regHUBP0_HUBP_CLK_CNTL                                                                          0x05f4
+#define regHUBP0_HUBP_CLK_CNTL_BASE_IDX                                                                 2
+#define regHUBP0_DCHUBP_VMPG_CONFIG                                                                     0x05f5
+#define regHUBP0_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
+#define regHUBP0_DCHUBP_MALL_CONFIG                                                                     0x05f6
+#define regHUBP0_DCHUBP_MALL_CONFIG_BASE_IDX                                                            2
+#define regHUBP0_DCHUBP_MALL_SUB_VP                                                                     0x05f7
+#define regHUBP0_DCHUBP_MALL_SUB_VP_BASE_IDX                                                            2
+#define regHUBP0_HUBPREQ_DEBUG_DB                                                                       0x05f8
+#define regHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define regHUBP0_HUBPREQ_DEBUG                                                                          0x05f9
+#define regHUBP0_HUBPREQ_DEBUG_BASE_IDX                                                                 2
+#define regHUBP0_HUBP_DEBUG_CTRL                                                                        0x05fa
+#define regHUBP0_HUBP_DEBUG_CTRL_BASE_IDX                                                               2
+#define regHUBP0_HUBP_DEBUG_MUX_DCFCLK                                                                  0x05fb
+#define regHUBP0_HUBP_DEBUG_MUX_DCFCLK_BASE_IDX                                                         2
+#define regHUBP0_HUBP_DEBUG_MUX_DPPCLK                                                                  0x05fc
+#define regHUBP0_HUBP_DEBUG_MUX_DPPCLK_BASE_IDX                                                         2
+#define regHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x05fd
+#define regHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
+#define regHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x05fe
+#define regHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX                                                  2
+#define regHUBP0_HUBP_MALL_STATUS                                                                       0x05ff
+#define regHUBP0_HUBP_MALL_STATUS_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpreq_dispdec
+// base address: 0x0
+#define regHUBPREQ0_DCSURF_SURFACE_PITCH                                                                0x0607
+#define regHUBPREQ0_DCSURF_SURFACE_PITCH_BASE_IDX                                                       2
+#define regHUBPREQ0_DCSURF_SURFACE_PITCH_C                                                              0x0608
+#define regHUBPREQ0_DCSURF_SURFACE_PITCH_C_BASE_IDX                                                     2
+#define regHUBPREQ0_VMID_SETTINGS_0                                                                     0x0609
+#define regHUBPREQ0_VMID_SETTINGS_0_BASE_IDX                                                            2
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS                                                      0x060a
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX                                             2
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH                                                 0x060b
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                        2
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C                                                    0x060c
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX                                           2
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C                                               0x060d
+#define regHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                      2
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS                                                    0x060e
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX                                           2
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH                                               0x060f
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                      2
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C                                                  0x0610
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX                                         2
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C                                             0x0611
+#define regHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                    2
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS                                                 0x0612
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX                                        2
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH                                            0x0613
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                   2
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C                                               0x0614
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX                                      2
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C                                          0x0615
+#define regHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                 2
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS                                               0x0616
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX                                      2
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH                                          0x0617
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                 2
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C                                             0x0618
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX                                    2
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C                                        0x0619
+#define regHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                               2
+#define regHUBPREQ0_DCSURF_SURFACE_CONTROL                                                              0x061a
+#define regHUBPREQ0_DCSURF_SURFACE_CONTROL_BASE_IDX                                                     2
+#define regHUBPREQ0_DCSURF_FLIP_CONTROL                                                                 0x061b
+#define regHUBPREQ0_DCSURF_FLIP_CONTROL_BASE_IDX                                                        2
+#define regHUBPREQ0_DCSURF_FLIP_CONTROL2                                                                0x061c
+#define regHUBPREQ0_DCSURF_FLIP_CONTROL2_BASE_IDX                                                       2
+#define regHUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT                                                       0x061f
+#define regHUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX                                              2
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE                                                                0x0620
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_BASE_IDX                                                       2
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH                                                           0x0621
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX                                                  2
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_C                                                              0x0622
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_C_BASE_IDX                                                     2
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C                                                         0x0623
+#define regHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX                                                2
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE                                                       0x0624
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX                                              2
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH                                                  0x0625
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX                                         2
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C                                                     0x0626
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX                                            2
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C                                                0x0627
+#define regHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX                                       2
+#define regHUBPREQ0_DCN_EXPANSION_MODE                                                                  0x0628
+#define regHUBPREQ0_DCN_EXPANSION_MODE_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_TTU_QOS_WM                                                                      0x0629
+#define regHUBPREQ0_DCN_TTU_QOS_WM_BASE_IDX                                                             2
+#define regHUBPREQ0_DCN_GLOBAL_TTU_CNTL                                                                 0x062a
+#define regHUBPREQ0_DCN_GLOBAL_TTU_CNTL_BASE_IDX                                                        2
+#define regHUBPREQ0_DCN_SURF0_TTU_CNTL0                                                                 0x062b
+#define regHUBPREQ0_DCN_SURF0_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ0_DCN_SURF0_TTU_CNTL1                                                                 0x062c
+#define regHUBPREQ0_DCN_SURF0_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ0_DCN_SURF1_TTU_CNTL0                                                                 0x062d
+#define regHUBPREQ0_DCN_SURF1_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ0_DCN_SURF1_TTU_CNTL1                                                                 0x062e
+#define regHUBPREQ0_DCN_SURF1_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ0_DCN_CUR0_TTU_CNTL0                                                                  0x062f
+#define regHUBPREQ0_DCN_CUR0_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_CUR0_TTU_CNTL1                                                                  0x0630
+#define regHUBPREQ0_DCN_CUR0_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_CUR1_TTU_CNTL0                                                                  0x0631
+#define regHUBPREQ0_DCN_CUR1_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_CUR1_TTU_CNTL1                                                                  0x0632
+#define regHUBPREQ0_DCN_CUR1_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_DMDATA_VM_CNTL                                                                  0x0633
+#define regHUBPREQ0_DCN_DMDATA_VM_CNTL_BASE_IDX                                                         2
+#define regHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR                                                     0x0634
+#define regHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                            2
+#define regHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR                                                    0x0635
+#define regHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                           2
+#define regHUBPREQ0_DCN_VM_MX_L1_TLB_CNTL                                                               0x0642
+#define regHUBPREQ0_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX                                                      2
+#define regHUBPREQ0_BLANK_OFFSET_0                                                                      0x0643
+#define regHUBPREQ0_BLANK_OFFSET_0_BASE_IDX                                                             2
+#define regHUBPREQ0_BLANK_OFFSET_1                                                                      0x0644
+#define regHUBPREQ0_BLANK_OFFSET_1_BASE_IDX                                                             2
+#define regHUBPREQ0_DST_DIMENSIONS                                                                      0x0645
+#define regHUBPREQ0_DST_DIMENSIONS_BASE_IDX                                                             2
+#define regHUBPREQ0_DST_AFTER_SCALER                                                                    0x0646
+#define regHUBPREQ0_DST_AFTER_SCALER_BASE_IDX                                                           2
+#define regHUBPREQ0_PREFETCH_SETTINGS                                                                   0x0647
+#define regHUBPREQ0_PREFETCH_SETTINGS_BASE_IDX                                                          2
+#define regHUBPREQ0_PREFETCH_SETTINGS_C                                                                 0x0648
+#define regHUBPREQ0_PREFETCH_SETTINGS_C_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_0                                                                 0x0649
+#define regHUBPREQ0_VBLANK_PARAMETERS_0_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_1                                                                 0x064a
+#define regHUBPREQ0_VBLANK_PARAMETERS_1_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_2                                                                 0x064b
+#define regHUBPREQ0_VBLANK_PARAMETERS_2_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_3                                                                 0x064c
+#define regHUBPREQ0_VBLANK_PARAMETERS_3_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_4                                                                 0x064d
+#define regHUBPREQ0_VBLANK_PARAMETERS_4_BASE_IDX                                                        2
+#define regHUBPREQ0_FLIP_PARAMETERS_0                                                                   0x064e
+#define regHUBPREQ0_FLIP_PARAMETERS_0_BASE_IDX                                                          2
+#define regHUBPREQ0_FLIP_PARAMETERS_1                                                                   0x064f
+#define regHUBPREQ0_FLIP_PARAMETERS_1_BASE_IDX                                                          2
+#define regHUBPREQ0_FLIP_PARAMETERS_2                                                                   0x0650
+#define regHUBPREQ0_FLIP_PARAMETERS_2_BASE_IDX                                                          2
+#define regHUBPREQ0_NOM_PARAMETERS_0                                                                    0x0651
+#define regHUBPREQ0_NOM_PARAMETERS_0_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_1                                                                    0x0652
+#define regHUBPREQ0_NOM_PARAMETERS_1_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_2                                                                    0x0653
+#define regHUBPREQ0_NOM_PARAMETERS_2_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_3                                                                    0x0654
+#define regHUBPREQ0_NOM_PARAMETERS_3_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_4                                                                    0x0655
+#define regHUBPREQ0_NOM_PARAMETERS_4_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_5                                                                    0x0656
+#define regHUBPREQ0_NOM_PARAMETERS_5_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_6                                                                    0x0657
+#define regHUBPREQ0_NOM_PARAMETERS_6_BASE_IDX                                                           2
+#define regHUBPREQ0_NOM_PARAMETERS_7                                                                    0x0658
+#define regHUBPREQ0_NOM_PARAMETERS_7_BASE_IDX                                                           2
+#define regHUBPREQ0_PER_LINE_DELIVERY_PRE                                                               0x0659
+#define regHUBPREQ0_PER_LINE_DELIVERY_PRE_BASE_IDX                                                      2
+#define regHUBPREQ0_PER_LINE_DELIVERY                                                                   0x065a
+#define regHUBPREQ0_PER_LINE_DELIVERY_BASE_IDX                                                          2
+#define regHUBPREQ0_CURSOR_SETTINGS                                                                     0x065b
+#define regHUBPREQ0_CURSOR_SETTINGS_BASE_IDX                                                            2
+#define regHUBPREQ0_REF_FREQ_TO_PIX_FREQ                                                                0x065c
+#define regHUBPREQ0_REF_FREQ_TO_PIX_FREQ_BASE_IDX                                                       2
+#define regHUBPREQ0_DST_Y_DELTA_DRQ_LIMIT                                                               0x065d
+#define regHUBPREQ0_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX                                                      2
+#define regHUBPREQ0_HUBPREQ_MEM_PWR_CTRL                                                                0x065e
+#define regHUBPREQ0_HUBPREQ_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPREQ0_HUBPREQ_MEM_PWR_STATUS                                                              0x065f
+#define regHUBPREQ0_HUBPREQ_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPREQ0_VBLANK_PARAMETERS_5                                                                 0x0662
+#define regHUBPREQ0_VBLANK_PARAMETERS_5_BASE_IDX                                                        2
+#define regHUBPREQ0_VBLANK_PARAMETERS_6                                                                 0x0663
+#define regHUBPREQ0_VBLANK_PARAMETERS_6_BASE_IDX                                                        2
+#define regHUBPREQ0_FLIP_PARAMETERS_3                                                                   0x0664
+#define regHUBPREQ0_FLIP_PARAMETERS_3_BASE_IDX                                                          2
+#define regHUBPREQ0_FLIP_PARAMETERS_4                                                                   0x0665
+#define regHUBPREQ0_FLIP_PARAMETERS_4_BASE_IDX                                                          2
+#define regHUBPREQ0_FLIP_PARAMETERS_5                                                                   0x0666
+#define regHUBPREQ0_FLIP_PARAMETERS_5_BASE_IDX                                                          2
+#define regHUBPREQ0_FLIP_PARAMETERS_6                                                                   0x0667
+#define regHUBPREQ0_FLIP_PARAMETERS_6_BASE_IDX                                                          2
+#define regHUBPREQ0_UCLK_PSTATE_FORCE                                                                   0x0668
+#define regHUBPREQ0_UCLK_PSTATE_FORCE_BASE_IDX                                                          2
+#define regHUBPREQ0_HUBPREQ_STATUS_REG0                                                                 0x0669
+#define regHUBPREQ0_HUBPREQ_STATUS_REG0_BASE_IDX                                                        2
+#define regHUBPREQ0_HUBPREQ_STATUS_REG1                                                                 0x066a
+#define regHUBPREQ0_HUBPREQ_STATUS_REG1_BASE_IDX                                                        2
+#define regHUBPREQ0_HUBPREQ_STATUS_REG2                                                                 0x066b
+#define regHUBPREQ0_HUBPREQ_STATUS_REG2_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpret_dispdec
+// base address: 0x0
+#define regHUBPRET0_HUBPRET_CONTROL                                                                     0x066c
+#define regHUBPRET0_HUBPRET_CONTROL_BASE_IDX                                                            2
+#define regHUBPRET0_HUBPRET_MEM_PWR_CTRL                                                                0x066d
+#define regHUBPRET0_HUBPRET_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPRET0_HUBPRET_MEM_PWR_STATUS                                                              0x066e
+#define regHUBPRET0_HUBPRET_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPRET0_HUBPRET_READ_LINE_CTRL0                                                             0x066f
+#define regHUBPRET0_HUBPRET_READ_LINE_CTRL0_BASE_IDX                                                    2
+#define regHUBPRET0_HUBPRET_READ_LINE_CTRL1                                                             0x0670
+#define regHUBPRET0_HUBPRET_READ_LINE_CTRL1_BASE_IDX                                                    2
+#define regHUBPRET0_HUBPRET_READ_LINE0                                                                  0x0671
+#define regHUBPRET0_HUBPRET_READ_LINE0_BASE_IDX                                                         2
+#define regHUBPRET0_HUBPRET_READ_LINE1                                                                  0x0672
+#define regHUBPRET0_HUBPRET_READ_LINE1_BASE_IDX                                                         2
+#define regHUBPRET0_HUBPRET_INTERRUPT                                                                   0x0673
+#define regHUBPRET0_HUBPRET_INTERRUPT_BASE_IDX                                                          2
+#define regHUBPRET0_HUBPRET_READ_LINE_VALUE                                                             0x0674
+#define regHUBPRET0_HUBPRET_READ_LINE_VALUE_BASE_IDX                                                    2
+#define regHUBPRET0_HUBPRET_READ_LINE_STATUS                                                            0x0675
+#define regHUBPRET0_HUBPRET_READ_LINE_STATUS_BASE_IDX                                                   2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_cursor0_dispdec
+// base address: 0x0
+#define regCURSOR0_0_CURSOR_CONTROL                                                                     0x0678
+#define regCURSOR0_0_CURSOR_CONTROL_BASE_IDX                                                            2
+#define regCURSOR0_0_CURSOR_SURFACE_ADDRESS                                                             0x0679
+#define regCURSOR0_0_CURSOR_SURFACE_ADDRESS_BASE_IDX                                                    2
+#define regCURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH                                                        0x067a
+#define regCURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX                                               2
+#define regCURSOR0_0_CURSOR_SIZE                                                                        0x067b
+#define regCURSOR0_0_CURSOR_SIZE_BASE_IDX                                                               2
+#define regCURSOR0_0_CURSOR_POSITION                                                                    0x067c
+#define regCURSOR0_0_CURSOR_POSITION_BASE_IDX                                                           2
+#define regCURSOR0_0_CURSOR_HOT_SPOT                                                                    0x067d
+#define regCURSOR0_0_CURSOR_HOT_SPOT_BASE_IDX                                                           2
+#define regCURSOR0_0_CURSOR_STEREO_CONTROL                                                              0x067e
+#define regCURSOR0_0_CURSOR_STEREO_CONTROL_BASE_IDX                                                     2
+#define regCURSOR0_0_CURSOR_DST_OFFSET                                                                  0x067f
+#define regCURSOR0_0_CURSOR_DST_OFFSET_BASE_IDX                                                         2
+#define regCURSOR0_0_CURSOR_MEM_PWR_CTRL                                                                0x0680
+#define regCURSOR0_0_CURSOR_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regCURSOR0_0_CURSOR_MEM_PWR_STATUS                                                              0x0681
+#define regCURSOR0_0_CURSOR_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regCURSOR0_0_DMDATA_ADDRESS_HIGH                                                                0x0682
+#define regCURSOR0_0_DMDATA_ADDRESS_HIGH_BASE_IDX                                                       2
+#define regCURSOR0_0_DMDATA_ADDRESS_LOW                                                                 0x0683
+#define regCURSOR0_0_DMDATA_ADDRESS_LOW_BASE_IDX                                                        2
+#define regCURSOR0_0_DMDATA_CNTL                                                                        0x0684
+#define regCURSOR0_0_DMDATA_CNTL_BASE_IDX                                                               2
+#define regCURSOR0_0_DMDATA_QOS_CNTL                                                                    0x0685
+#define regCURSOR0_0_DMDATA_QOS_CNTL_BASE_IDX                                                           2
+#define regCURSOR0_0_DMDATA_STATUS                                                                      0x0686
+#define regCURSOR0_0_DMDATA_STATUS_BASE_IDX                                                             2
+#define regCURSOR0_0_DMDATA_SW_CNTL                                                                     0x0687
+#define regCURSOR0_0_DMDATA_SW_CNTL_BASE_IDX                                                            2
+#define regCURSOR0_0_DMDATA_SW_DATA                                                                     0x0688
+#define regCURSOR0_0_DMDATA_SW_DATA_BASE_IDX                                                            2
+#define regCURSOR0_0_HUBP_3DLUT_CONTROL                                                                 0x0689
+#define regCURSOR0_0_HUBP_3DLUT_CONTROL_BASE_IDX                                                        2
+#define regCURSOR0_0_HUBP_3DLUT_ADDRESS_LOW                                                             0x068a
+#define regCURSOR0_0_HUBP_3DLUT_ADDRESS_LOW_BASE_IDX                                                    2
+#define regCURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH                                                            0x068b
+#define regCURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH_BASE_IDX                                                   2
+#define regCURSOR0_0_HUBP_3DLUT_DLG_PARAM                                                               0x068c
+#define regCURSOR0_0_HUBP_3DLUT_DLG_PARAM_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1a74
+#define regDC_PERFMON6_PERFCOUNTER_CNTL                                                                 0x069d
+#define regDC_PERFMON6_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON6_PERFCOUNTER_CNTL2                                                                0x069e
+#define regDC_PERFMON6_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON6_PERFCOUNTER_STATE                                                                0x069f
+#define regDC_PERFMON6_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON6_PERFMON_CNTL                                                                     0x06a0
+#define regDC_PERFMON6_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON6_PERFMON_CNTL2                                                                    0x06a1
+#define regDC_PERFMON6_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON6_PERFMON_CVALUE_INT_MISC                                                          0x06a2
+#define regDC_PERFMON6_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON6_PERFMON_CVALUE_LOW                                                               0x06a3
+#define regDC_PERFMON6_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON6_PERFMON_HI                                                                       0x06a4
+#define regDC_PERFMON6_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON6_PERFMON_LOW                                                                      0x06a5
+#define regDC_PERFMON6_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dispdec
+// base address: 0x370
+#define regHUBP1_DCSURF_SURFACE_CONFIG                                                                  0x06c1
+#define regHUBP1_DCSURF_SURFACE_CONFIG_BASE_IDX                                                         2
+#define regHUBP1_DCSURF_ADDR_CONFIG                                                                     0x06c2
+#define regHUBP1_DCSURF_ADDR_CONFIG_BASE_IDX                                                            2
+#define regHUBP1_DCSURF_TILING_CONFIG                                                                   0x06c3
+#define regHUBP1_DCSURF_TILING_CONFIG_BASE_IDX                                                          2
+#define regHUBP1_DCSURF_PRI_VIEWPORT_START                                                              0x06c5
+#define regHUBP1_DCSURF_PRI_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x06c6
+#define regHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP1_DCSURF_PRI_VIEWPORT_START_C                                                            0x06c7
+#define regHUBP1_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C                                                        0x06c8
+#define regHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP1_DCSURF_SEC_VIEWPORT_START                                                              0x06c9
+#define regHUBP1_DCSURF_SEC_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION                                                          0x06ca
+#define regHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP1_DCSURF_SEC_VIEWPORT_START_C                                                            0x06cb
+#define regHUBP1_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C                                                        0x06cc
+#define regHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP1_DCHUBP_REQ_SIZE_CONFIG                                                                 0x06cd
+#define regHUBP1_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX                                                        2
+#define regHUBP1_DCHUBP_REQ_SIZE_CONFIG_C                                                               0x06ce
+#define regHUBP1_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX                                                      2
+#define regHUBP1_DCHUBP_CNTL                                                                            0x06cf
+#define regHUBP1_DCHUBP_CNTL_BASE_IDX                                                                   2
+#define regHUBP1_HUBP_CLK_CNTL                                                                          0x06d0
+#define regHUBP1_HUBP_CLK_CNTL_BASE_IDX                                                                 2
+#define regHUBP1_DCHUBP_VMPG_CONFIG                                                                     0x06d1
+#define regHUBP1_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
+#define regHUBP1_DCHUBP_MALL_CONFIG                                                                     0x06d2
+#define regHUBP1_DCHUBP_MALL_CONFIG_BASE_IDX                                                            2
+#define regHUBP1_DCHUBP_MALL_SUB_VP                                                                     0x06d3
+#define regHUBP1_DCHUBP_MALL_SUB_VP_BASE_IDX                                                            2
+#define regHUBP1_HUBPREQ_DEBUG_DB                                                                       0x06d4
+#define regHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define regHUBP1_HUBPREQ_DEBUG                                                                          0x06d5
+#define regHUBP1_HUBPREQ_DEBUG_BASE_IDX                                                                 2
+#define regHUBP1_HUBP_DEBUG_CTRL                                                                        0x06d6
+#define regHUBP1_HUBP_DEBUG_CTRL_BASE_IDX                                                               2
+#define regHUBP1_HUBP_DEBUG_MUX_DCFCLK                                                                  0x06d7
+#define regHUBP1_HUBP_DEBUG_MUX_DCFCLK_BASE_IDX                                                         2
+#define regHUBP1_HUBP_DEBUG_MUX_DPPCLK                                                                  0x06d8
+#define regHUBP1_HUBP_DEBUG_MUX_DPPCLK_BASE_IDX                                                         2
+#define regHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x06d9
+#define regHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
+#define regHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x06da
+#define regHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX                                                  2
+#define regHUBP1_HUBP_MALL_STATUS                                                                       0x06db
+#define regHUBP1_HUBP_MALL_STATUS_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpreq_dispdec
+// base address: 0x370
+#define regHUBPREQ1_DCSURF_SURFACE_PITCH                                                                0x06e3
+#define regHUBPREQ1_DCSURF_SURFACE_PITCH_BASE_IDX                                                       2
+#define regHUBPREQ1_DCSURF_SURFACE_PITCH_C                                                              0x06e4
+#define regHUBPREQ1_DCSURF_SURFACE_PITCH_C_BASE_IDX                                                     2
+#define regHUBPREQ1_VMID_SETTINGS_0                                                                     0x06e5
+#define regHUBPREQ1_VMID_SETTINGS_0_BASE_IDX                                                            2
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS                                                      0x06e6
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX                                             2
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH                                                 0x06e7
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                        2
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C                                                    0x06e8
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX                                           2
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C                                               0x06e9
+#define regHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                      2
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS                                                    0x06ea
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX                                           2
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH                                               0x06eb
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                      2
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C                                                  0x06ec
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX                                         2
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C                                             0x06ed
+#define regHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                    2
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS                                                 0x06ee
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX                                        2
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH                                            0x06ef
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                   2
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C                                               0x06f0
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX                                      2
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C                                          0x06f1
+#define regHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                 2
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS                                               0x06f2
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX                                      2
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH                                          0x06f3
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                 2
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C                                             0x06f4
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX                                    2
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C                                        0x06f5
+#define regHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                               2
+#define regHUBPREQ1_DCSURF_SURFACE_CONTROL                                                              0x06f6
+#define regHUBPREQ1_DCSURF_SURFACE_CONTROL_BASE_IDX                                                     2
+#define regHUBPREQ1_DCSURF_FLIP_CONTROL                                                                 0x06f7
+#define regHUBPREQ1_DCSURF_FLIP_CONTROL_BASE_IDX                                                        2
+#define regHUBPREQ1_DCSURF_FLIP_CONTROL2                                                                0x06f8
+#define regHUBPREQ1_DCSURF_FLIP_CONTROL2_BASE_IDX                                                       2
+#define regHUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT                                                       0x06fb
+#define regHUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX                                              2
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE                                                                0x06fc
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_BASE_IDX                                                       2
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH                                                           0x06fd
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX                                                  2
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_C                                                              0x06fe
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_C_BASE_IDX                                                     2
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C                                                         0x06ff
+#define regHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX                                                2
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE                                                       0x0700
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX                                              2
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH                                                  0x0701
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX                                         2
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C                                                     0x0702
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX                                            2
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C                                                0x0703
+#define regHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX                                       2
+#define regHUBPREQ1_DCN_EXPANSION_MODE                                                                  0x0704
+#define regHUBPREQ1_DCN_EXPANSION_MODE_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_TTU_QOS_WM                                                                      0x0705
+#define regHUBPREQ1_DCN_TTU_QOS_WM_BASE_IDX                                                             2
+#define regHUBPREQ1_DCN_GLOBAL_TTU_CNTL                                                                 0x0706
+#define regHUBPREQ1_DCN_GLOBAL_TTU_CNTL_BASE_IDX                                                        2
+#define regHUBPREQ1_DCN_SURF0_TTU_CNTL0                                                                 0x0707
+#define regHUBPREQ1_DCN_SURF0_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ1_DCN_SURF0_TTU_CNTL1                                                                 0x0708
+#define regHUBPREQ1_DCN_SURF0_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ1_DCN_SURF1_TTU_CNTL0                                                                 0x0709
+#define regHUBPREQ1_DCN_SURF1_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ1_DCN_SURF1_TTU_CNTL1                                                                 0x070a
+#define regHUBPREQ1_DCN_SURF1_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ1_DCN_CUR0_TTU_CNTL0                                                                  0x070b
+#define regHUBPREQ1_DCN_CUR0_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_CUR0_TTU_CNTL1                                                                  0x070c
+#define regHUBPREQ1_DCN_CUR0_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_CUR1_TTU_CNTL0                                                                  0x070d
+#define regHUBPREQ1_DCN_CUR1_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_CUR1_TTU_CNTL1                                                                  0x070e
+#define regHUBPREQ1_DCN_CUR1_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_DMDATA_VM_CNTL                                                                  0x070f
+#define regHUBPREQ1_DCN_DMDATA_VM_CNTL_BASE_IDX                                                         2
+#define regHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR                                                     0x0710
+#define regHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                            2
+#define regHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR                                                    0x0711
+#define regHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                           2
+#define regHUBPREQ1_DCN_VM_MX_L1_TLB_CNTL                                                               0x071e
+#define regHUBPREQ1_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX                                                      2
+#define regHUBPREQ1_BLANK_OFFSET_0                                                                      0x071f
+#define regHUBPREQ1_BLANK_OFFSET_0_BASE_IDX                                                             2
+#define regHUBPREQ1_BLANK_OFFSET_1                                                                      0x0720
+#define regHUBPREQ1_BLANK_OFFSET_1_BASE_IDX                                                             2
+#define regHUBPREQ1_DST_DIMENSIONS                                                                      0x0721
+#define regHUBPREQ1_DST_DIMENSIONS_BASE_IDX                                                             2
+#define regHUBPREQ1_DST_AFTER_SCALER                                                                    0x0722
+#define regHUBPREQ1_DST_AFTER_SCALER_BASE_IDX                                                           2
+#define regHUBPREQ1_PREFETCH_SETTINGS                                                                   0x0723
+#define regHUBPREQ1_PREFETCH_SETTINGS_BASE_IDX                                                          2
+#define regHUBPREQ1_PREFETCH_SETTINGS_C                                                                 0x0724
+#define regHUBPREQ1_PREFETCH_SETTINGS_C_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_0                                                                 0x0725
+#define regHUBPREQ1_VBLANK_PARAMETERS_0_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_1                                                                 0x0726
+#define regHUBPREQ1_VBLANK_PARAMETERS_1_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_2                                                                 0x0727
+#define regHUBPREQ1_VBLANK_PARAMETERS_2_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_3                                                                 0x0728
+#define regHUBPREQ1_VBLANK_PARAMETERS_3_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_4                                                                 0x0729
+#define regHUBPREQ1_VBLANK_PARAMETERS_4_BASE_IDX                                                        2
+#define regHUBPREQ1_FLIP_PARAMETERS_0                                                                   0x072a
+#define regHUBPREQ1_FLIP_PARAMETERS_0_BASE_IDX                                                          2
+#define regHUBPREQ1_FLIP_PARAMETERS_1                                                                   0x072b
+#define regHUBPREQ1_FLIP_PARAMETERS_1_BASE_IDX                                                          2
+#define regHUBPREQ1_FLIP_PARAMETERS_2                                                                   0x072c
+#define regHUBPREQ1_FLIP_PARAMETERS_2_BASE_IDX                                                          2
+#define regHUBPREQ1_NOM_PARAMETERS_0                                                                    0x072d
+#define regHUBPREQ1_NOM_PARAMETERS_0_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_1                                                                    0x072e
+#define regHUBPREQ1_NOM_PARAMETERS_1_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_2                                                                    0x072f
+#define regHUBPREQ1_NOM_PARAMETERS_2_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_3                                                                    0x0730
+#define regHUBPREQ1_NOM_PARAMETERS_3_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_4                                                                    0x0731
+#define regHUBPREQ1_NOM_PARAMETERS_4_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_5                                                                    0x0732
+#define regHUBPREQ1_NOM_PARAMETERS_5_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_6                                                                    0x0733
+#define regHUBPREQ1_NOM_PARAMETERS_6_BASE_IDX                                                           2
+#define regHUBPREQ1_NOM_PARAMETERS_7                                                                    0x0734
+#define regHUBPREQ1_NOM_PARAMETERS_7_BASE_IDX                                                           2
+#define regHUBPREQ1_PER_LINE_DELIVERY_PRE                                                               0x0735
+#define regHUBPREQ1_PER_LINE_DELIVERY_PRE_BASE_IDX                                                      2
+#define regHUBPREQ1_PER_LINE_DELIVERY                                                                   0x0736
+#define regHUBPREQ1_PER_LINE_DELIVERY_BASE_IDX                                                          2
+#define regHUBPREQ1_CURSOR_SETTINGS                                                                     0x0737
+#define regHUBPREQ1_CURSOR_SETTINGS_BASE_IDX                                                            2
+#define regHUBPREQ1_REF_FREQ_TO_PIX_FREQ                                                                0x0738
+#define regHUBPREQ1_REF_FREQ_TO_PIX_FREQ_BASE_IDX                                                       2
+#define regHUBPREQ1_DST_Y_DELTA_DRQ_LIMIT                                                               0x0739
+#define regHUBPREQ1_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX                                                      2
+#define regHUBPREQ1_HUBPREQ_MEM_PWR_CTRL                                                                0x073a
+#define regHUBPREQ1_HUBPREQ_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPREQ1_HUBPREQ_MEM_PWR_STATUS                                                              0x073b
+#define regHUBPREQ1_HUBPREQ_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPREQ1_VBLANK_PARAMETERS_5                                                                 0x073e
+#define regHUBPREQ1_VBLANK_PARAMETERS_5_BASE_IDX                                                        2
+#define regHUBPREQ1_VBLANK_PARAMETERS_6                                                                 0x073f
+#define regHUBPREQ1_VBLANK_PARAMETERS_6_BASE_IDX                                                        2
+#define regHUBPREQ1_FLIP_PARAMETERS_3                                                                   0x0740
+#define regHUBPREQ1_FLIP_PARAMETERS_3_BASE_IDX                                                          2
+#define regHUBPREQ1_FLIP_PARAMETERS_4                                                                   0x0741
+#define regHUBPREQ1_FLIP_PARAMETERS_4_BASE_IDX                                                          2
+#define regHUBPREQ1_FLIP_PARAMETERS_5                                                                   0x0742
+#define regHUBPREQ1_FLIP_PARAMETERS_5_BASE_IDX                                                          2
+#define regHUBPREQ1_FLIP_PARAMETERS_6                                                                   0x0743
+#define regHUBPREQ1_FLIP_PARAMETERS_6_BASE_IDX                                                          2
+#define regHUBPREQ1_UCLK_PSTATE_FORCE                                                                   0x0744
+#define regHUBPREQ1_UCLK_PSTATE_FORCE_BASE_IDX                                                          2
+#define regHUBPREQ1_HUBPREQ_STATUS_REG0                                                                 0x0745
+#define regHUBPREQ1_HUBPREQ_STATUS_REG0_BASE_IDX                                                        2
+#define regHUBPREQ1_HUBPREQ_STATUS_REG1                                                                 0x0746
+#define regHUBPREQ1_HUBPREQ_STATUS_REG1_BASE_IDX                                                        2
+#define regHUBPREQ1_HUBPREQ_STATUS_REG2                                                                 0x0747
+#define regHUBPREQ1_HUBPREQ_STATUS_REG2_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpret_dispdec
+// base address: 0x370
+#define regHUBPRET1_HUBPRET_CONTROL                                                                     0x0748
+#define regHUBPRET1_HUBPRET_CONTROL_BASE_IDX                                                            2
+#define regHUBPRET1_HUBPRET_MEM_PWR_CTRL                                                                0x0749
+#define regHUBPRET1_HUBPRET_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPRET1_HUBPRET_MEM_PWR_STATUS                                                              0x074a
+#define regHUBPRET1_HUBPRET_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPRET1_HUBPRET_READ_LINE_CTRL0                                                             0x074b
+#define regHUBPRET1_HUBPRET_READ_LINE_CTRL0_BASE_IDX                                                    2
+#define regHUBPRET1_HUBPRET_READ_LINE_CTRL1                                                             0x074c
+#define regHUBPRET1_HUBPRET_READ_LINE_CTRL1_BASE_IDX                                                    2
+#define regHUBPRET1_HUBPRET_READ_LINE0                                                                  0x074d
+#define regHUBPRET1_HUBPRET_READ_LINE0_BASE_IDX                                                         2
+#define regHUBPRET1_HUBPRET_READ_LINE1                                                                  0x074e
+#define regHUBPRET1_HUBPRET_READ_LINE1_BASE_IDX                                                         2
+#define regHUBPRET1_HUBPRET_INTERRUPT                                                                   0x074f
+#define regHUBPRET1_HUBPRET_INTERRUPT_BASE_IDX                                                          2
+#define regHUBPRET1_HUBPRET_READ_LINE_VALUE                                                             0x0750
+#define regHUBPRET1_HUBPRET_READ_LINE_VALUE_BASE_IDX                                                    2
+#define regHUBPRET1_HUBPRET_READ_LINE_STATUS                                                            0x0751
+#define regHUBPRET1_HUBPRET_READ_LINE_STATUS_BASE_IDX                                                   2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_cursor0_dispdec
+// base address: 0x370
+#define regCURSOR0_1_CURSOR_CONTROL                                                                     0x0754
+#define regCURSOR0_1_CURSOR_CONTROL_BASE_IDX                                                            2
+#define regCURSOR0_1_CURSOR_SURFACE_ADDRESS                                                             0x0755
+#define regCURSOR0_1_CURSOR_SURFACE_ADDRESS_BASE_IDX                                                    2
+#define regCURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH                                                        0x0756
+#define regCURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX                                               2
+#define regCURSOR0_1_CURSOR_SIZE                                                                        0x0757
+#define regCURSOR0_1_CURSOR_SIZE_BASE_IDX                                                               2
+#define regCURSOR0_1_CURSOR_POSITION                                                                    0x0758
+#define regCURSOR0_1_CURSOR_POSITION_BASE_IDX                                                           2
+#define regCURSOR0_1_CURSOR_HOT_SPOT                                                                    0x0759
+#define regCURSOR0_1_CURSOR_HOT_SPOT_BASE_IDX                                                           2
+#define regCURSOR0_1_CURSOR_STEREO_CONTROL                                                              0x075a
+#define regCURSOR0_1_CURSOR_STEREO_CONTROL_BASE_IDX                                                     2
+#define regCURSOR0_1_CURSOR_DST_OFFSET                                                                  0x075b
+#define regCURSOR0_1_CURSOR_DST_OFFSET_BASE_IDX                                                         2
+#define regCURSOR0_1_CURSOR_MEM_PWR_CTRL                                                                0x075c
+#define regCURSOR0_1_CURSOR_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regCURSOR0_1_CURSOR_MEM_PWR_STATUS                                                              0x075d
+#define regCURSOR0_1_CURSOR_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regCURSOR0_1_DMDATA_ADDRESS_HIGH                                                                0x075e
+#define regCURSOR0_1_DMDATA_ADDRESS_HIGH_BASE_IDX                                                       2
+#define regCURSOR0_1_DMDATA_ADDRESS_LOW                                                                 0x075f
+#define regCURSOR0_1_DMDATA_ADDRESS_LOW_BASE_IDX                                                        2
+#define regCURSOR0_1_DMDATA_CNTL                                                                        0x0760
+#define regCURSOR0_1_DMDATA_CNTL_BASE_IDX                                                               2
+#define regCURSOR0_1_DMDATA_QOS_CNTL                                                                    0x0761
+#define regCURSOR0_1_DMDATA_QOS_CNTL_BASE_IDX                                                           2
+#define regCURSOR0_1_DMDATA_STATUS                                                                      0x0762
+#define regCURSOR0_1_DMDATA_STATUS_BASE_IDX                                                             2
+#define regCURSOR0_1_DMDATA_SW_CNTL                                                                     0x0763
+#define regCURSOR0_1_DMDATA_SW_CNTL_BASE_IDX                                                            2
+#define regCURSOR0_1_DMDATA_SW_DATA                                                                     0x0764
+#define regCURSOR0_1_DMDATA_SW_DATA_BASE_IDX                                                            2
+#define regCURSOR0_1_HUBP_3DLUT_CONTROL                                                                 0x0765
+#define regCURSOR0_1_HUBP_3DLUT_CONTROL_BASE_IDX                                                        2
+#define regCURSOR0_1_HUBP_3DLUT_ADDRESS_LOW                                                             0x0766
+#define regCURSOR0_1_HUBP_3DLUT_ADDRESS_LOW_BASE_IDX                                                    2
+#define regCURSOR0_1_HUBP_3DLUT_ADDRESS_HIGH                                                            0x0767
+#define regCURSOR0_1_HUBP_3DLUT_ADDRESS_HIGH_BASE_IDX                                                   2
+#define regCURSOR0_1_HUBP_3DLUT_DLG_PARAM                                                               0x0768
+#define regCURSOR0_1_HUBP_3DLUT_DLG_PARAM_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1de4
+#define regDC_PERFMON7_PERFCOUNTER_CNTL                                                                 0x0779
+#define regDC_PERFMON7_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON7_PERFCOUNTER_CNTL2                                                                0x077a
+#define regDC_PERFMON7_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON7_PERFCOUNTER_STATE                                                                0x077b
+#define regDC_PERFMON7_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON7_PERFMON_CNTL                                                                     0x077c
+#define regDC_PERFMON7_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON7_PERFMON_CNTL2                                                                    0x077d
+#define regDC_PERFMON7_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON7_PERFMON_CVALUE_INT_MISC                                                          0x077e
+#define regDC_PERFMON7_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON7_PERFMON_CVALUE_LOW                                                               0x077f
+#define regDC_PERFMON7_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON7_PERFMON_HI                                                                       0x0780
+#define regDC_PERFMON7_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON7_PERFMON_LOW                                                                      0x0781
+#define regDC_PERFMON7_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dispdec
+// base address: 0x6e0
+#define regHUBP2_DCSURF_SURFACE_CONFIG                                                                  0x079d
+#define regHUBP2_DCSURF_SURFACE_CONFIG_BASE_IDX                                                         2
+#define regHUBP2_DCSURF_ADDR_CONFIG                                                                     0x079e
+#define regHUBP2_DCSURF_ADDR_CONFIG_BASE_IDX                                                            2
+#define regHUBP2_DCSURF_TILING_CONFIG                                                                   0x079f
+#define regHUBP2_DCSURF_TILING_CONFIG_BASE_IDX                                                          2
+#define regHUBP2_DCSURF_PRI_VIEWPORT_START                                                              0x07a1
+#define regHUBP2_DCSURF_PRI_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x07a2
+#define regHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP2_DCSURF_PRI_VIEWPORT_START_C                                                            0x07a3
+#define regHUBP2_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C                                                        0x07a4
+#define regHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP2_DCSURF_SEC_VIEWPORT_START                                                              0x07a5
+#define regHUBP2_DCSURF_SEC_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION                                                          0x07a6
+#define regHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP2_DCSURF_SEC_VIEWPORT_START_C                                                            0x07a7
+#define regHUBP2_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C                                                        0x07a8
+#define regHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP2_DCHUBP_REQ_SIZE_CONFIG                                                                 0x07a9
+#define regHUBP2_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX                                                        2
+#define regHUBP2_DCHUBP_REQ_SIZE_CONFIG_C                                                               0x07aa
+#define regHUBP2_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX                                                      2
+#define regHUBP2_DCHUBP_CNTL                                                                            0x07ab
+#define regHUBP2_DCHUBP_CNTL_BASE_IDX                                                                   2
+#define regHUBP2_HUBP_CLK_CNTL                                                                          0x07ac
+#define regHUBP2_HUBP_CLK_CNTL_BASE_IDX                                                                 2
+#define regHUBP2_DCHUBP_VMPG_CONFIG                                                                     0x07ad
+#define regHUBP2_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
+#define regHUBP2_DCHUBP_MALL_CONFIG                                                                     0x07ae
+#define regHUBP2_DCHUBP_MALL_CONFIG_BASE_IDX                                                            2
+#define regHUBP2_DCHUBP_MALL_SUB_VP                                                                     0x07af
+#define regHUBP2_DCHUBP_MALL_SUB_VP_BASE_IDX                                                            2
+#define regHUBP2_HUBPREQ_DEBUG_DB                                                                       0x07b0
+#define regHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define regHUBP2_HUBPREQ_DEBUG                                                                          0x07b1
+#define regHUBP2_HUBPREQ_DEBUG_BASE_IDX                                                                 2
+#define regHUBP2_HUBP_DEBUG_CTRL                                                                        0x07b2
+#define regHUBP2_HUBP_DEBUG_CTRL_BASE_IDX                                                               2
+#define regHUBP2_HUBP_DEBUG_MUX_DCFCLK                                                                  0x07b3
+#define regHUBP2_HUBP_DEBUG_MUX_DCFCLK_BASE_IDX                                                         2
+#define regHUBP2_HUBP_DEBUG_MUX_DPPCLK                                                                  0x07b4
+#define regHUBP2_HUBP_DEBUG_MUX_DPPCLK_BASE_IDX                                                         2
+#define regHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x07b5
+#define regHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
+#define regHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x07b6
+#define regHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX                                                  2
+#define regHUBP2_HUBP_MALL_STATUS                                                                       0x07b7
+#define regHUBP2_HUBP_MALL_STATUS_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpreq_dispdec
+// base address: 0x6e0
+#define regHUBPREQ2_DCSURF_SURFACE_PITCH                                                                0x07bf
+#define regHUBPREQ2_DCSURF_SURFACE_PITCH_BASE_IDX                                                       2
+#define regHUBPREQ2_DCSURF_SURFACE_PITCH_C                                                              0x07c0
+#define regHUBPREQ2_DCSURF_SURFACE_PITCH_C_BASE_IDX                                                     2
+#define regHUBPREQ2_VMID_SETTINGS_0                                                                     0x07c1
+#define regHUBPREQ2_VMID_SETTINGS_0_BASE_IDX                                                            2
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS                                                      0x07c2
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX                                             2
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH                                                 0x07c3
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                        2
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C                                                    0x07c4
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX                                           2
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C                                               0x07c5
+#define regHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                      2
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS                                                    0x07c6
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX                                           2
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH                                               0x07c7
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                      2
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C                                                  0x07c8
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX                                         2
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C                                             0x07c9
+#define regHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                    2
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS                                                 0x07ca
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX                                        2
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH                                            0x07cb
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                   2
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C                                               0x07cc
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX                                      2
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C                                          0x07cd
+#define regHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                 2
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS                                               0x07ce
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX                                      2
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH                                          0x07cf
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                 2
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C                                             0x07d0
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX                                    2
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C                                        0x07d1
+#define regHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                               2
+#define regHUBPREQ2_DCSURF_SURFACE_CONTROL                                                              0x07d2
+#define regHUBPREQ2_DCSURF_SURFACE_CONTROL_BASE_IDX                                                     2
+#define regHUBPREQ2_DCSURF_FLIP_CONTROL                                                                 0x07d3
+#define regHUBPREQ2_DCSURF_FLIP_CONTROL_BASE_IDX                                                        2
+#define regHUBPREQ2_DCSURF_FLIP_CONTROL2                                                                0x07d4
+#define regHUBPREQ2_DCSURF_FLIP_CONTROL2_BASE_IDX                                                       2
+#define regHUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT                                                       0x07d7
+#define regHUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX                                              2
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE                                                                0x07d8
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_BASE_IDX                                                       2
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH                                                           0x07d9
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX                                                  2
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_C                                                              0x07da
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_C_BASE_IDX                                                     2
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C                                                         0x07db
+#define regHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX                                                2
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE                                                       0x07dc
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX                                              2
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH                                                  0x07dd
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX                                         2
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C                                                     0x07de
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX                                            2
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C                                                0x07df
+#define regHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX                                       2
+#define regHUBPREQ2_DCN_EXPANSION_MODE                                                                  0x07e0
+#define regHUBPREQ2_DCN_EXPANSION_MODE_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_TTU_QOS_WM                                                                      0x07e1
+#define regHUBPREQ2_DCN_TTU_QOS_WM_BASE_IDX                                                             2
+#define regHUBPREQ2_DCN_GLOBAL_TTU_CNTL                                                                 0x07e2
+#define regHUBPREQ2_DCN_GLOBAL_TTU_CNTL_BASE_IDX                                                        2
+#define regHUBPREQ2_DCN_SURF0_TTU_CNTL0                                                                 0x07e3
+#define regHUBPREQ2_DCN_SURF0_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ2_DCN_SURF0_TTU_CNTL1                                                                 0x07e4
+#define regHUBPREQ2_DCN_SURF0_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ2_DCN_SURF1_TTU_CNTL0                                                                 0x07e5
+#define regHUBPREQ2_DCN_SURF1_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ2_DCN_SURF1_TTU_CNTL1                                                                 0x07e6
+#define regHUBPREQ2_DCN_SURF1_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ2_DCN_CUR0_TTU_CNTL0                                                                  0x07e7
+#define regHUBPREQ2_DCN_CUR0_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_CUR0_TTU_CNTL1                                                                  0x07e8
+#define regHUBPREQ2_DCN_CUR0_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_CUR1_TTU_CNTL0                                                                  0x07e9
+#define regHUBPREQ2_DCN_CUR1_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_CUR1_TTU_CNTL1                                                                  0x07ea
+#define regHUBPREQ2_DCN_CUR1_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_DMDATA_VM_CNTL                                                                  0x07eb
+#define regHUBPREQ2_DCN_DMDATA_VM_CNTL_BASE_IDX                                                         2
+#define regHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR                                                     0x07ec
+#define regHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                            2
+#define regHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR                                                    0x07ed
+#define regHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                           2
+#define regHUBPREQ2_DCN_VM_MX_L1_TLB_CNTL                                                               0x07fa
+#define regHUBPREQ2_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX                                                      2
+#define regHUBPREQ2_BLANK_OFFSET_0                                                                      0x07fb
+#define regHUBPREQ2_BLANK_OFFSET_0_BASE_IDX                                                             2
+#define regHUBPREQ2_BLANK_OFFSET_1                                                                      0x07fc
+#define regHUBPREQ2_BLANK_OFFSET_1_BASE_IDX                                                             2
+#define regHUBPREQ2_DST_DIMENSIONS                                                                      0x07fd
+#define regHUBPREQ2_DST_DIMENSIONS_BASE_IDX                                                             2
+#define regHUBPREQ2_DST_AFTER_SCALER                                                                    0x07fe
+#define regHUBPREQ2_DST_AFTER_SCALER_BASE_IDX                                                           2
+#define regHUBPREQ2_PREFETCH_SETTINGS                                                                   0x07ff
+#define regHUBPREQ2_PREFETCH_SETTINGS_BASE_IDX                                                          2
+#define regHUBPREQ2_PREFETCH_SETTINGS_C                                                                 0x0800
+#define regHUBPREQ2_PREFETCH_SETTINGS_C_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_0                                                                 0x0801
+#define regHUBPREQ2_VBLANK_PARAMETERS_0_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_1                                                                 0x0802
+#define regHUBPREQ2_VBLANK_PARAMETERS_1_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_2                                                                 0x0803
+#define regHUBPREQ2_VBLANK_PARAMETERS_2_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_3                                                                 0x0804
+#define regHUBPREQ2_VBLANK_PARAMETERS_3_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_4                                                                 0x0805
+#define regHUBPREQ2_VBLANK_PARAMETERS_4_BASE_IDX                                                        2
+#define regHUBPREQ2_FLIP_PARAMETERS_0                                                                   0x0806
+#define regHUBPREQ2_FLIP_PARAMETERS_0_BASE_IDX                                                          2
+#define regHUBPREQ2_FLIP_PARAMETERS_1                                                                   0x0807
+#define regHUBPREQ2_FLIP_PARAMETERS_1_BASE_IDX                                                          2
+#define regHUBPREQ2_FLIP_PARAMETERS_2                                                                   0x0808
+#define regHUBPREQ2_FLIP_PARAMETERS_2_BASE_IDX                                                          2
+#define regHUBPREQ2_NOM_PARAMETERS_0                                                                    0x0809
+#define regHUBPREQ2_NOM_PARAMETERS_0_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_1                                                                    0x080a
+#define regHUBPREQ2_NOM_PARAMETERS_1_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_2                                                                    0x080b
+#define regHUBPREQ2_NOM_PARAMETERS_2_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_3                                                                    0x080c
+#define regHUBPREQ2_NOM_PARAMETERS_3_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_4                                                                    0x080d
+#define regHUBPREQ2_NOM_PARAMETERS_4_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_5                                                                    0x080e
+#define regHUBPREQ2_NOM_PARAMETERS_5_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_6                                                                    0x080f
+#define regHUBPREQ2_NOM_PARAMETERS_6_BASE_IDX                                                           2
+#define regHUBPREQ2_NOM_PARAMETERS_7                                                                    0x0810
+#define regHUBPREQ2_NOM_PARAMETERS_7_BASE_IDX                                                           2
+#define regHUBPREQ2_PER_LINE_DELIVERY_PRE                                                               0x0811
+#define regHUBPREQ2_PER_LINE_DELIVERY_PRE_BASE_IDX                                                      2
+#define regHUBPREQ2_PER_LINE_DELIVERY                                                                   0x0812
+#define regHUBPREQ2_PER_LINE_DELIVERY_BASE_IDX                                                          2
+#define regHUBPREQ2_CURSOR_SETTINGS                                                                     0x0813
+#define regHUBPREQ2_CURSOR_SETTINGS_BASE_IDX                                                            2
+#define regHUBPREQ2_REF_FREQ_TO_PIX_FREQ                                                                0x0814
+#define regHUBPREQ2_REF_FREQ_TO_PIX_FREQ_BASE_IDX                                                       2
+#define regHUBPREQ2_DST_Y_DELTA_DRQ_LIMIT                                                               0x0815
+#define regHUBPREQ2_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX                                                      2
+#define regHUBPREQ2_HUBPREQ_MEM_PWR_CTRL                                                                0x0816
+#define regHUBPREQ2_HUBPREQ_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPREQ2_HUBPREQ_MEM_PWR_STATUS                                                              0x0817
+#define regHUBPREQ2_HUBPREQ_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPREQ2_VBLANK_PARAMETERS_5                                                                 0x081a
+#define regHUBPREQ2_VBLANK_PARAMETERS_5_BASE_IDX                                                        2
+#define regHUBPREQ2_VBLANK_PARAMETERS_6                                                                 0x081b
+#define regHUBPREQ2_VBLANK_PARAMETERS_6_BASE_IDX                                                        2
+#define regHUBPREQ2_FLIP_PARAMETERS_3                                                                   0x081c
+#define regHUBPREQ2_FLIP_PARAMETERS_3_BASE_IDX                                                          2
+#define regHUBPREQ2_FLIP_PARAMETERS_4                                                                   0x081d
+#define regHUBPREQ2_FLIP_PARAMETERS_4_BASE_IDX                                                          2
+#define regHUBPREQ2_FLIP_PARAMETERS_5                                                                   0x081e
+#define regHUBPREQ2_FLIP_PARAMETERS_5_BASE_IDX                                                          2
+#define regHUBPREQ2_FLIP_PARAMETERS_6                                                                   0x081f
+#define regHUBPREQ2_FLIP_PARAMETERS_6_BASE_IDX                                                          2
+#define regHUBPREQ2_UCLK_PSTATE_FORCE                                                                   0x0820
+#define regHUBPREQ2_UCLK_PSTATE_FORCE_BASE_IDX                                                          2
+#define regHUBPREQ2_HUBPREQ_STATUS_REG0                                                                 0x0821
+#define regHUBPREQ2_HUBPREQ_STATUS_REG0_BASE_IDX                                                        2
+#define regHUBPREQ2_HUBPREQ_STATUS_REG1                                                                 0x0822
+#define regHUBPREQ2_HUBPREQ_STATUS_REG1_BASE_IDX                                                        2
+#define regHUBPREQ2_HUBPREQ_STATUS_REG2                                                                 0x0823
+#define regHUBPREQ2_HUBPREQ_STATUS_REG2_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpret_dispdec
+// base address: 0x6e0
+#define regHUBPRET2_HUBPRET_CONTROL                                                                     0x0824
+#define regHUBPRET2_HUBPRET_CONTROL_BASE_IDX                                                            2
+#define regHUBPRET2_HUBPRET_MEM_PWR_CTRL                                                                0x0825
+#define regHUBPRET2_HUBPRET_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPRET2_HUBPRET_MEM_PWR_STATUS                                                              0x0826
+#define regHUBPRET2_HUBPRET_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPRET2_HUBPRET_READ_LINE_CTRL0                                                             0x0827
+#define regHUBPRET2_HUBPRET_READ_LINE_CTRL0_BASE_IDX                                                    2
+#define regHUBPRET2_HUBPRET_READ_LINE_CTRL1                                                             0x0828
+#define regHUBPRET2_HUBPRET_READ_LINE_CTRL1_BASE_IDX                                                    2
+#define regHUBPRET2_HUBPRET_READ_LINE0                                                                  0x0829
+#define regHUBPRET2_HUBPRET_READ_LINE0_BASE_IDX                                                         2
+#define regHUBPRET2_HUBPRET_READ_LINE1                                                                  0x082a
+#define regHUBPRET2_HUBPRET_READ_LINE1_BASE_IDX                                                         2
+#define regHUBPRET2_HUBPRET_INTERRUPT                                                                   0x082b
+#define regHUBPRET2_HUBPRET_INTERRUPT_BASE_IDX                                                          2
+#define regHUBPRET2_HUBPRET_READ_LINE_VALUE                                                             0x082c
+#define regHUBPRET2_HUBPRET_READ_LINE_VALUE_BASE_IDX                                                    2
+#define regHUBPRET2_HUBPRET_READ_LINE_STATUS                                                            0x082d
+#define regHUBPRET2_HUBPRET_READ_LINE_STATUS_BASE_IDX                                                   2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_cursor0_dispdec
+// base address: 0x6e0
+#define regCURSOR0_2_CURSOR_CONTROL                                                                     0x0830
+#define regCURSOR0_2_CURSOR_CONTROL_BASE_IDX                                                            2
+#define regCURSOR0_2_CURSOR_SURFACE_ADDRESS                                                             0x0831
+#define regCURSOR0_2_CURSOR_SURFACE_ADDRESS_BASE_IDX                                                    2
+#define regCURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH                                                        0x0832
+#define regCURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX                                               2
+#define regCURSOR0_2_CURSOR_SIZE                                                                        0x0833
+#define regCURSOR0_2_CURSOR_SIZE_BASE_IDX                                                               2
+#define regCURSOR0_2_CURSOR_POSITION                                                                    0x0834
+#define regCURSOR0_2_CURSOR_POSITION_BASE_IDX                                                           2
+#define regCURSOR0_2_CURSOR_HOT_SPOT                                                                    0x0835
+#define regCURSOR0_2_CURSOR_HOT_SPOT_BASE_IDX                                                           2
+#define regCURSOR0_2_CURSOR_STEREO_CONTROL                                                              0x0836
+#define regCURSOR0_2_CURSOR_STEREO_CONTROL_BASE_IDX                                                     2
+#define regCURSOR0_2_CURSOR_DST_OFFSET                                                                  0x0837
+#define regCURSOR0_2_CURSOR_DST_OFFSET_BASE_IDX                                                         2
+#define regCURSOR0_2_CURSOR_MEM_PWR_CTRL                                                                0x0838
+#define regCURSOR0_2_CURSOR_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regCURSOR0_2_CURSOR_MEM_PWR_STATUS                                                              0x0839
+#define regCURSOR0_2_CURSOR_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regCURSOR0_2_DMDATA_ADDRESS_HIGH                                                                0x083a
+#define regCURSOR0_2_DMDATA_ADDRESS_HIGH_BASE_IDX                                                       2
+#define regCURSOR0_2_DMDATA_ADDRESS_LOW                                                                 0x083b
+#define regCURSOR0_2_DMDATA_ADDRESS_LOW_BASE_IDX                                                        2
+#define regCURSOR0_2_DMDATA_CNTL                                                                        0x083c
+#define regCURSOR0_2_DMDATA_CNTL_BASE_IDX                                                               2
+#define regCURSOR0_2_DMDATA_QOS_CNTL                                                                    0x083d
+#define regCURSOR0_2_DMDATA_QOS_CNTL_BASE_IDX                                                           2
+#define regCURSOR0_2_DMDATA_STATUS                                                                      0x083e
+#define regCURSOR0_2_DMDATA_STATUS_BASE_IDX                                                             2
+#define regCURSOR0_2_DMDATA_SW_CNTL                                                                     0x083f
+#define regCURSOR0_2_DMDATA_SW_CNTL_BASE_IDX                                                            2
+#define regCURSOR0_2_DMDATA_SW_DATA                                                                     0x0840
+#define regCURSOR0_2_DMDATA_SW_DATA_BASE_IDX                                                            2
+#define regCURSOR0_2_HUBP_3DLUT_CONTROL                                                                 0x0841
+#define regCURSOR0_2_HUBP_3DLUT_CONTROL_BASE_IDX                                                        2
+#define regCURSOR0_2_HUBP_3DLUT_ADDRESS_LOW                                                             0x0842
+#define regCURSOR0_2_HUBP_3DLUT_ADDRESS_LOW_BASE_IDX                                                    2
+#define regCURSOR0_2_HUBP_3DLUT_ADDRESS_HIGH                                                            0x0843
+#define regCURSOR0_2_HUBP_3DLUT_ADDRESS_HIGH_BASE_IDX                                                   2
+#define regCURSOR0_2_HUBP_3DLUT_DLG_PARAM                                                               0x0844
+#define regCURSOR0_2_HUBP_3DLUT_DLG_PARAM_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2154
+#define regDC_PERFMON8_PERFCOUNTER_CNTL                                                                 0x0855
+#define regDC_PERFMON8_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON8_PERFCOUNTER_CNTL2                                                                0x0856
+#define regDC_PERFMON8_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON8_PERFCOUNTER_STATE                                                                0x0857
+#define regDC_PERFMON8_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON8_PERFMON_CNTL                                                                     0x0858
+#define regDC_PERFMON8_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON8_PERFMON_CNTL2                                                                    0x0859
+#define regDC_PERFMON8_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON8_PERFMON_CVALUE_INT_MISC                                                          0x085a
+#define regDC_PERFMON8_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON8_PERFMON_CVALUE_LOW                                                               0x085b
+#define regDC_PERFMON8_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON8_PERFMON_HI                                                                       0x085c
+#define regDC_PERFMON8_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON8_PERFMON_LOW                                                                      0x085d
+#define regDC_PERFMON8_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dispdec
+// base address: 0xa50
+#define regHUBP3_DCSURF_SURFACE_CONFIG                                                                  0x0879
+#define regHUBP3_DCSURF_SURFACE_CONFIG_BASE_IDX                                                         2
+#define regHUBP3_DCSURF_ADDR_CONFIG                                                                     0x087a
+#define regHUBP3_DCSURF_ADDR_CONFIG_BASE_IDX                                                            2
+#define regHUBP3_DCSURF_TILING_CONFIG                                                                   0x087b
+#define regHUBP3_DCSURF_TILING_CONFIG_BASE_IDX                                                          2
+#define regHUBP3_DCSURF_PRI_VIEWPORT_START                                                              0x087d
+#define regHUBP3_DCSURF_PRI_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x087e
+#define regHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP3_DCSURF_PRI_VIEWPORT_START_C                                                            0x087f
+#define regHUBP3_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C                                                        0x0880
+#define regHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP3_DCSURF_SEC_VIEWPORT_START                                                              0x0881
+#define regHUBP3_DCSURF_SEC_VIEWPORT_START_BASE_IDX                                                     2
+#define regHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION                                                          0x0882
+#define regHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX                                                 2
+#define regHUBP3_DCSURF_SEC_VIEWPORT_START_C                                                            0x0883
+#define regHUBP3_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX                                                   2
+#define regHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C                                                        0x0884
+#define regHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX                                               2
+#define regHUBP3_DCHUBP_REQ_SIZE_CONFIG                                                                 0x0885
+#define regHUBP3_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX                                                        2
+#define regHUBP3_DCHUBP_REQ_SIZE_CONFIG_C                                                               0x0886
+#define regHUBP3_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX                                                      2
+#define regHUBP3_DCHUBP_CNTL                                                                            0x0887
+#define regHUBP3_DCHUBP_CNTL_BASE_IDX                                                                   2
+#define regHUBP3_HUBP_CLK_CNTL                                                                          0x0888
+#define regHUBP3_HUBP_CLK_CNTL_BASE_IDX                                                                 2
+#define regHUBP3_DCHUBP_VMPG_CONFIG                                                                     0x0889
+#define regHUBP3_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
+#define regHUBP3_DCHUBP_MALL_CONFIG                                                                     0x088a
+#define regHUBP3_DCHUBP_MALL_CONFIG_BASE_IDX                                                            2
+#define regHUBP3_DCHUBP_MALL_SUB_VP                                                                     0x088b
+#define regHUBP3_DCHUBP_MALL_SUB_VP_BASE_IDX                                                            2
+#define regHUBP3_HUBPREQ_DEBUG_DB                                                                       0x088c
+#define regHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define regHUBP3_HUBPREQ_DEBUG                                                                          0x088d
+#define regHUBP3_HUBPREQ_DEBUG_BASE_IDX                                                                 2
+#define regHUBP3_HUBP_DEBUG_CTRL                                                                        0x088e
+#define regHUBP3_HUBP_DEBUG_CTRL_BASE_IDX                                                               2
+#define regHUBP3_HUBP_DEBUG_MUX_DCFCLK                                                                  0x088f
+#define regHUBP3_HUBP_DEBUG_MUX_DCFCLK_BASE_IDX                                                         2
+#define regHUBP3_HUBP_DEBUG_MUX_DPPCLK                                                                  0x0890
+#define regHUBP3_HUBP_DEBUG_MUX_DPPCLK_BASE_IDX                                                         2
+#define regHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x0891
+#define regHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
+#define regHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x0892
+#define regHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX                                                  2
+#define regHUBP3_HUBP_MALL_STATUS                                                                       0x0893
+#define regHUBP3_HUBP_MALL_STATUS_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpreq_dispdec
+// base address: 0xa50
+#define regHUBPREQ3_DCSURF_SURFACE_PITCH                                                                0x089b
+#define regHUBPREQ3_DCSURF_SURFACE_PITCH_BASE_IDX                                                       2
+#define regHUBPREQ3_DCSURF_SURFACE_PITCH_C                                                              0x089c
+#define regHUBPREQ3_DCSURF_SURFACE_PITCH_C_BASE_IDX                                                     2
+#define regHUBPREQ3_VMID_SETTINGS_0                                                                     0x089d
+#define regHUBPREQ3_VMID_SETTINGS_0_BASE_IDX                                                            2
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS                                                      0x089e
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX                                             2
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH                                                 0x089f
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                        2
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C                                                    0x08a0
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX                                           2
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C                                               0x08a1
+#define regHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                      2
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS                                                    0x08a2
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX                                           2
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH                                               0x08a3
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX                                      2
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C                                                  0x08a4
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX                                         2
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C                                             0x08a5
+#define regHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                    2
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS                                                 0x08a6
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX                                        2
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH                                            0x08a7
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                   2
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C                                               0x08a8
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX                                      2
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C                                          0x08a9
+#define regHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                                 2
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS                                               0x08aa
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX                                      2
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH                                          0x08ab
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX                                 2
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C                                             0x08ac
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX                                    2
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C                                        0x08ad
+#define regHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX                               2
+#define regHUBPREQ3_DCSURF_SURFACE_CONTROL                                                              0x08ae
+#define regHUBPREQ3_DCSURF_SURFACE_CONTROL_BASE_IDX                                                     2
+#define regHUBPREQ3_DCSURF_FLIP_CONTROL                                                                 0x08af
+#define regHUBPREQ3_DCSURF_FLIP_CONTROL_BASE_IDX                                                        2
+#define regHUBPREQ3_DCSURF_FLIP_CONTROL2                                                                0x08b0
+#define regHUBPREQ3_DCSURF_FLIP_CONTROL2_BASE_IDX                                                       2
+#define regHUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT                                                       0x08b3
+#define regHUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX                                              2
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE                                                                0x08b4
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_BASE_IDX                                                       2
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH                                                           0x08b5
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX                                                  2
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_C                                                              0x08b6
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_C_BASE_IDX                                                     2
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C                                                         0x08b7
+#define regHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX                                                2
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE                                                       0x08b8
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX                                              2
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH                                                  0x08b9
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX                                         2
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C                                                     0x08ba
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX                                            2
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C                                                0x08bb
+#define regHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX                                       2
+#define regHUBPREQ3_DCN_EXPANSION_MODE                                                                  0x08bc
+#define regHUBPREQ3_DCN_EXPANSION_MODE_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_TTU_QOS_WM                                                                      0x08bd
+#define regHUBPREQ3_DCN_TTU_QOS_WM_BASE_IDX                                                             2
+#define regHUBPREQ3_DCN_GLOBAL_TTU_CNTL                                                                 0x08be
+#define regHUBPREQ3_DCN_GLOBAL_TTU_CNTL_BASE_IDX                                                        2
+#define regHUBPREQ3_DCN_SURF0_TTU_CNTL0                                                                 0x08bf
+#define regHUBPREQ3_DCN_SURF0_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ3_DCN_SURF0_TTU_CNTL1                                                                 0x08c0
+#define regHUBPREQ3_DCN_SURF0_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ3_DCN_SURF1_TTU_CNTL0                                                                 0x08c1
+#define regHUBPREQ3_DCN_SURF1_TTU_CNTL0_BASE_IDX                                                        2
+#define regHUBPREQ3_DCN_SURF1_TTU_CNTL1                                                                 0x08c2
+#define regHUBPREQ3_DCN_SURF1_TTU_CNTL1_BASE_IDX                                                        2
+#define regHUBPREQ3_DCN_CUR0_TTU_CNTL0                                                                  0x08c3
+#define regHUBPREQ3_DCN_CUR0_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_CUR0_TTU_CNTL1                                                                  0x08c4
+#define regHUBPREQ3_DCN_CUR0_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_CUR1_TTU_CNTL0                                                                  0x08c5
+#define regHUBPREQ3_DCN_CUR1_TTU_CNTL0_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_CUR1_TTU_CNTL1                                                                  0x08c6
+#define regHUBPREQ3_DCN_CUR1_TTU_CNTL1_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_DMDATA_VM_CNTL                                                                  0x08c7
+#define regHUBPREQ3_DCN_DMDATA_VM_CNTL_BASE_IDX                                                         2
+#define regHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR                                                     0x08c8
+#define regHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                            2
+#define regHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR                                                    0x08c9
+#define regHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                           2
+#define regHUBPREQ3_DCN_VM_MX_L1_TLB_CNTL                                                               0x08d6
+#define regHUBPREQ3_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX                                                      2
+#define regHUBPREQ3_BLANK_OFFSET_0                                                                      0x08d7
+#define regHUBPREQ3_BLANK_OFFSET_0_BASE_IDX                                                             2
+#define regHUBPREQ3_BLANK_OFFSET_1                                                                      0x08d8
+#define regHUBPREQ3_BLANK_OFFSET_1_BASE_IDX                                                             2
+#define regHUBPREQ3_DST_DIMENSIONS                                                                      0x08d9
+#define regHUBPREQ3_DST_DIMENSIONS_BASE_IDX                                                             2
+#define regHUBPREQ3_DST_AFTER_SCALER                                                                    0x08da
+#define regHUBPREQ3_DST_AFTER_SCALER_BASE_IDX                                                           2
+#define regHUBPREQ3_PREFETCH_SETTINGS                                                                   0x08db
+#define regHUBPREQ3_PREFETCH_SETTINGS_BASE_IDX                                                          2
+#define regHUBPREQ3_PREFETCH_SETTINGS_C                                                                 0x08dc
+#define regHUBPREQ3_PREFETCH_SETTINGS_C_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_0                                                                 0x08dd
+#define regHUBPREQ3_VBLANK_PARAMETERS_0_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_1                                                                 0x08de
+#define regHUBPREQ3_VBLANK_PARAMETERS_1_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_2                                                                 0x08df
+#define regHUBPREQ3_VBLANK_PARAMETERS_2_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_3                                                                 0x08e0
+#define regHUBPREQ3_VBLANK_PARAMETERS_3_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_4                                                                 0x08e1
+#define regHUBPREQ3_VBLANK_PARAMETERS_4_BASE_IDX                                                        2
+#define regHUBPREQ3_FLIP_PARAMETERS_0                                                                   0x08e2
+#define regHUBPREQ3_FLIP_PARAMETERS_0_BASE_IDX                                                          2
+#define regHUBPREQ3_FLIP_PARAMETERS_1                                                                   0x08e3
+#define regHUBPREQ3_FLIP_PARAMETERS_1_BASE_IDX                                                          2
+#define regHUBPREQ3_FLIP_PARAMETERS_2                                                                   0x08e4
+#define regHUBPREQ3_FLIP_PARAMETERS_2_BASE_IDX                                                          2
+#define regHUBPREQ3_NOM_PARAMETERS_0                                                                    0x08e5
+#define regHUBPREQ3_NOM_PARAMETERS_0_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_1                                                                    0x08e6
+#define regHUBPREQ3_NOM_PARAMETERS_1_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_2                                                                    0x08e7
+#define regHUBPREQ3_NOM_PARAMETERS_2_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_3                                                                    0x08e8
+#define regHUBPREQ3_NOM_PARAMETERS_3_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_4                                                                    0x08e9
+#define regHUBPREQ3_NOM_PARAMETERS_4_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_5                                                                    0x08ea
+#define regHUBPREQ3_NOM_PARAMETERS_5_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_6                                                                    0x08eb
+#define regHUBPREQ3_NOM_PARAMETERS_6_BASE_IDX                                                           2
+#define regHUBPREQ3_NOM_PARAMETERS_7                                                                    0x08ec
+#define regHUBPREQ3_NOM_PARAMETERS_7_BASE_IDX                                                           2
+#define regHUBPREQ3_PER_LINE_DELIVERY_PRE                                                               0x08ed
+#define regHUBPREQ3_PER_LINE_DELIVERY_PRE_BASE_IDX                                                      2
+#define regHUBPREQ3_PER_LINE_DELIVERY                                                                   0x08ee
+#define regHUBPREQ3_PER_LINE_DELIVERY_BASE_IDX                                                          2
+#define regHUBPREQ3_CURSOR_SETTINGS                                                                     0x08ef
+#define regHUBPREQ3_CURSOR_SETTINGS_BASE_IDX                                                            2
+#define regHUBPREQ3_REF_FREQ_TO_PIX_FREQ                                                                0x08f0
+#define regHUBPREQ3_REF_FREQ_TO_PIX_FREQ_BASE_IDX                                                       2
+#define regHUBPREQ3_DST_Y_DELTA_DRQ_LIMIT                                                               0x08f1
+#define regHUBPREQ3_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX                                                      2
+#define regHUBPREQ3_HUBPREQ_MEM_PWR_CTRL                                                                0x08f2
+#define regHUBPREQ3_HUBPREQ_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPREQ3_HUBPREQ_MEM_PWR_STATUS                                                              0x08f3
+#define regHUBPREQ3_HUBPREQ_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPREQ3_VBLANK_PARAMETERS_5                                                                 0x08f6
+#define regHUBPREQ3_VBLANK_PARAMETERS_5_BASE_IDX                                                        2
+#define regHUBPREQ3_VBLANK_PARAMETERS_6                                                                 0x08f7
+#define regHUBPREQ3_VBLANK_PARAMETERS_6_BASE_IDX                                                        2
+#define regHUBPREQ3_FLIP_PARAMETERS_3                                                                   0x08f8
+#define regHUBPREQ3_FLIP_PARAMETERS_3_BASE_IDX                                                          2
+#define regHUBPREQ3_FLIP_PARAMETERS_4                                                                   0x08f9
+#define regHUBPREQ3_FLIP_PARAMETERS_4_BASE_IDX                                                          2
+#define regHUBPREQ3_FLIP_PARAMETERS_5                                                                   0x08fa
+#define regHUBPREQ3_FLIP_PARAMETERS_5_BASE_IDX                                                          2
+#define regHUBPREQ3_FLIP_PARAMETERS_6                                                                   0x08fb
+#define regHUBPREQ3_FLIP_PARAMETERS_6_BASE_IDX                                                          2
+#define regHUBPREQ3_UCLK_PSTATE_FORCE                                                                   0x08fc
+#define regHUBPREQ3_UCLK_PSTATE_FORCE_BASE_IDX                                                          2
+#define regHUBPREQ3_HUBPREQ_STATUS_REG0                                                                 0x08fd
+#define regHUBPREQ3_HUBPREQ_STATUS_REG0_BASE_IDX                                                        2
+#define regHUBPREQ3_HUBPREQ_STATUS_REG1                                                                 0x08fe
+#define regHUBPREQ3_HUBPREQ_STATUS_REG1_BASE_IDX                                                        2
+#define regHUBPREQ3_HUBPREQ_STATUS_REG2                                                                 0x08ff
+#define regHUBPREQ3_HUBPREQ_STATUS_REG2_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpret_dispdec
+// base address: 0xa50
+#define regHUBPRET3_HUBPRET_CONTROL                                                                     0x0900
+#define regHUBPRET3_HUBPRET_CONTROL_BASE_IDX                                                            2
+#define regHUBPRET3_HUBPRET_MEM_PWR_CTRL                                                                0x0901
+#define regHUBPRET3_HUBPRET_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regHUBPRET3_HUBPRET_MEM_PWR_STATUS                                                              0x0902
+#define regHUBPRET3_HUBPRET_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regHUBPRET3_HUBPRET_READ_LINE_CTRL0                                                             0x0903
+#define regHUBPRET3_HUBPRET_READ_LINE_CTRL0_BASE_IDX                                                    2
+#define regHUBPRET3_HUBPRET_READ_LINE_CTRL1                                                             0x0904
+#define regHUBPRET3_HUBPRET_READ_LINE_CTRL1_BASE_IDX                                                    2
+#define regHUBPRET3_HUBPRET_READ_LINE0                                                                  0x0905
+#define regHUBPRET3_HUBPRET_READ_LINE0_BASE_IDX                                                         2
+#define regHUBPRET3_HUBPRET_READ_LINE1                                                                  0x0906
+#define regHUBPRET3_HUBPRET_READ_LINE1_BASE_IDX                                                         2
+#define regHUBPRET3_HUBPRET_INTERRUPT                                                                   0x0907
+#define regHUBPRET3_HUBPRET_INTERRUPT_BASE_IDX                                                          2
+#define regHUBPRET3_HUBPRET_READ_LINE_VALUE                                                             0x0908
+#define regHUBPRET3_HUBPRET_READ_LINE_VALUE_BASE_IDX                                                    2
+#define regHUBPRET3_HUBPRET_READ_LINE_STATUS                                                            0x0909
+#define regHUBPRET3_HUBPRET_READ_LINE_STATUS_BASE_IDX                                                   2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_cursor0_dispdec
+// base address: 0xa50
+#define regCURSOR0_3_CURSOR_CONTROL                                                                     0x090c
+#define regCURSOR0_3_CURSOR_CONTROL_BASE_IDX                                                            2
+#define regCURSOR0_3_CURSOR_SURFACE_ADDRESS                                                             0x090d
+#define regCURSOR0_3_CURSOR_SURFACE_ADDRESS_BASE_IDX                                                    2
+#define regCURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH                                                        0x090e
+#define regCURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX                                               2
+#define regCURSOR0_3_CURSOR_SIZE                                                                        0x090f
+#define regCURSOR0_3_CURSOR_SIZE_BASE_IDX                                                               2
+#define regCURSOR0_3_CURSOR_POSITION                                                                    0x0910
+#define regCURSOR0_3_CURSOR_POSITION_BASE_IDX                                                           2
+#define regCURSOR0_3_CURSOR_HOT_SPOT                                                                    0x0911
+#define regCURSOR0_3_CURSOR_HOT_SPOT_BASE_IDX                                                           2
+#define regCURSOR0_3_CURSOR_STEREO_CONTROL                                                              0x0912
+#define regCURSOR0_3_CURSOR_STEREO_CONTROL_BASE_IDX                                                     2
+#define regCURSOR0_3_CURSOR_DST_OFFSET                                                                  0x0913
+#define regCURSOR0_3_CURSOR_DST_OFFSET_BASE_IDX                                                         2
+#define regCURSOR0_3_CURSOR_MEM_PWR_CTRL                                                                0x0914
+#define regCURSOR0_3_CURSOR_MEM_PWR_CTRL_BASE_IDX                                                       2
+#define regCURSOR0_3_CURSOR_MEM_PWR_STATUS                                                              0x0915
+#define regCURSOR0_3_CURSOR_MEM_PWR_STATUS_BASE_IDX                                                     2
+#define regCURSOR0_3_DMDATA_ADDRESS_HIGH                                                                0x0916
+#define regCURSOR0_3_DMDATA_ADDRESS_HIGH_BASE_IDX                                                       2
+#define regCURSOR0_3_DMDATA_ADDRESS_LOW                                                                 0x0917
+#define regCURSOR0_3_DMDATA_ADDRESS_LOW_BASE_IDX                                                        2
+#define regCURSOR0_3_DMDATA_CNTL                                                                        0x0918
+#define regCURSOR0_3_DMDATA_CNTL_BASE_IDX                                                               2
+#define regCURSOR0_3_DMDATA_QOS_CNTL                                                                    0x0919
+#define regCURSOR0_3_DMDATA_QOS_CNTL_BASE_IDX                                                           2
+#define regCURSOR0_3_DMDATA_STATUS                                                                      0x091a
+#define regCURSOR0_3_DMDATA_STATUS_BASE_IDX                                                             2
+#define regCURSOR0_3_DMDATA_SW_CNTL                                                                     0x091b
+#define regCURSOR0_3_DMDATA_SW_CNTL_BASE_IDX                                                            2
+#define regCURSOR0_3_DMDATA_SW_DATA                                                                     0x091c
+#define regCURSOR0_3_DMDATA_SW_DATA_BASE_IDX                                                            2
+#define regCURSOR0_3_HUBP_3DLUT_CONTROL                                                                 0x091d
+#define regCURSOR0_3_HUBP_3DLUT_CONTROL_BASE_IDX                                                        2
+#define regCURSOR0_3_HUBP_3DLUT_ADDRESS_LOW                                                             0x091e
+#define regCURSOR0_3_HUBP_3DLUT_ADDRESS_LOW_BASE_IDX                                                    2
+#define regCURSOR0_3_HUBP_3DLUT_ADDRESS_HIGH                                                            0x091f
+#define regCURSOR0_3_HUBP_3DLUT_ADDRESS_HIGH_BASE_IDX                                                   2
+#define regCURSOR0_3_HUBP_3DLUT_DLG_PARAM                                                               0x0920
+#define regCURSOR0_3_HUBP_3DLUT_DLG_PARAM_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x24c4
+#define regDC_PERFMON9_PERFCOUNTER_CNTL                                                                 0x0931
+#define regDC_PERFMON9_PERFCOUNTER_CNTL_BASE_IDX                                                        2
+#define regDC_PERFMON9_PERFCOUNTER_CNTL2                                                                0x0932
+#define regDC_PERFMON9_PERFCOUNTER_CNTL2_BASE_IDX                                                       2
+#define regDC_PERFMON9_PERFCOUNTER_STATE                                                                0x0933
+#define regDC_PERFMON9_PERFCOUNTER_STATE_BASE_IDX                                                       2
+#define regDC_PERFMON9_PERFMON_CNTL                                                                     0x0934
+#define regDC_PERFMON9_PERFMON_CNTL_BASE_IDX                                                            2
+#define regDC_PERFMON9_PERFMON_CNTL2                                                                    0x0935
+#define regDC_PERFMON9_PERFMON_CNTL2_BASE_IDX                                                           2
+#define regDC_PERFMON9_PERFMON_CVALUE_INT_MISC                                                          0x0936
+#define regDC_PERFMON9_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                 2
+#define regDC_PERFMON9_PERFMON_CVALUE_LOW                                                               0x0937
+#define regDC_PERFMON9_PERFMON_CVALUE_LOW_BASE_IDX                                                      2
+#define regDC_PERFMON9_PERFMON_HI                                                                       0x0938
+#define regDC_PERFMON9_PERFMON_HI_BASE_IDX                                                              2
+#define regDC_PERFMON9_PERFMON_LOW                                                                      0x0939
+#define regDC_PERFMON9_PERFMON_LOW_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
+// base address: 0x0
+#define regDPP_TOP0_DPP_CONTROL                                                                         0x0cc5
+#define regDPP_TOP0_DPP_CONTROL_BASE_IDX                                                                2
+#define regDPP_TOP0_DPP_SOFT_RESET                                                                      0x0cc6
+#define regDPP_TOP0_DPP_SOFT_RESET_BASE_IDX                                                             2
+#define regDPP_TOP0_DPP_CRC_VAL_R                                                                       0x0cc7
+#define regDPP_TOP0_DPP_CRC_VAL_R_BASE_IDX                                                              2
+#define regDPP_TOP0_DPP_CRC_VAL_G                                                                       0x0cc8
+#define regDPP_TOP0_DPP_CRC_VAL_G_BASE_IDX                                                              2
+#define regDPP_TOP0_DPP_CRC_VAL_B                                                                       0x0cc9
+#define regDPP_TOP0_DPP_CRC_VAL_B_BASE_IDX                                                              2
+#define regDPP_TOP0_DPP_CRC_VAL_A                                                                       0x0cca
+#define regDPP_TOP0_DPP_CRC_VAL_A_BASE_IDX                                                              2
+#define regDPP_TOP0_DPP_CRC_CTRL                                                                        0x0ccb
+#define regDPP_TOP0_DPP_CRC_CTRL_BASE_IDX                                                               2
+#define regDPP_TOP0_HOST_READ_CONTROL                                                                   0x0ccc
+#define regDPP_TOP0_HOST_READ_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cnvc_cfg_dispdec
+// base address: 0x0
+#define regCNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT                                                          0x0cd1
+#define regCNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX                                                 2
+#define regCNVC_CFG0_FORMAT_CONTROL                                                                     0x0cd2
+#define regCNVC_CFG0_FORMAT_CONTROL_BASE_IDX                                                            2
+#define regCNVC_CFG0_FCNV_FP_BIAS_R                                                                     0x0cd3
+#define regCNVC_CFG0_FCNV_FP_BIAS_R_BASE_IDX                                                            2
+#define regCNVC_CFG0_FCNV_FP_BIAS_G                                                                     0x0cd4
+#define regCNVC_CFG0_FCNV_FP_BIAS_G_BASE_IDX                                                            2
+#define regCNVC_CFG0_FCNV_FP_BIAS_B                                                                     0x0cd5
+#define regCNVC_CFG0_FCNV_FP_BIAS_B_BASE_IDX                                                            2
+#define regCNVC_CFG0_FCNV_FP_SCALE_R                                                                    0x0cd6
+#define regCNVC_CFG0_FCNV_FP_SCALE_R_BASE_IDX                                                           2
+#define regCNVC_CFG0_FCNV_FP_SCALE_G                                                                    0x0cd7
+#define regCNVC_CFG0_FCNV_FP_SCALE_G_BASE_IDX                                                           2
+#define regCNVC_CFG0_FCNV_FP_SCALE_B                                                                    0x0cd8
+#define regCNVC_CFG0_FCNV_FP_SCALE_B_BASE_IDX                                                           2
+#define regCNVC_CFG0_COLOR_KEYER_CONTROL                                                                0x0cd9
+#define regCNVC_CFG0_COLOR_KEYER_CONTROL_BASE_IDX                                                       2
+#define regCNVC_CFG0_COLOR_KEYER_ALPHA                                                                  0x0cda
+#define regCNVC_CFG0_COLOR_KEYER_ALPHA_BASE_IDX                                                         2
+#define regCNVC_CFG0_COLOR_KEYER_RED                                                                    0x0cdb
+#define regCNVC_CFG0_COLOR_KEYER_RED_BASE_IDX                                                           2
+#define regCNVC_CFG0_COLOR_KEYER_GREEN                                                                  0x0cdc
+#define regCNVC_CFG0_COLOR_KEYER_GREEN_BASE_IDX                                                         2
+#define regCNVC_CFG0_COLOR_KEYER_BLUE                                                                   0x0cdd
+#define regCNVC_CFG0_COLOR_KEYER_BLUE_BASE_IDX                                                          2
+#define regCNVC_CFG0_ALPHA_2BIT_LUT01                                                                   0x0cdf
+#define regCNVC_CFG0_ALPHA_2BIT_LUT01_BASE_IDX                                                          2
+#define regCNVC_CFG0_ALPHA_2BIT_LUT23                                                                   0x0ce0
+#define regCNVC_CFG0_ALPHA_2BIT_LUT23_BASE_IDX                                                          2
+#define regCNVC_CFG0_PRE_DEALPHA                                                                        0x0ce1
+#define regCNVC_CFG0_PRE_DEALPHA_BASE_IDX                                                               2
+#define regCNVC_CFG0_PRE_CSC_MODE                                                                       0x0ce2
+#define regCNVC_CFG0_PRE_CSC_MODE_BASE_IDX                                                              2
+#define regCNVC_CFG0_PRE_CSC_C11_C12                                                                    0x0ce3
+#define regCNVC_CFG0_PRE_CSC_C11_C12_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_C13_C14                                                                    0x0ce4
+#define regCNVC_CFG0_PRE_CSC_C13_C14_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_C21_C22                                                                    0x0ce5
+#define regCNVC_CFG0_PRE_CSC_C21_C22_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_C23_C24                                                                    0x0ce6
+#define regCNVC_CFG0_PRE_CSC_C23_C24_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_C31_C32                                                                    0x0ce7
+#define regCNVC_CFG0_PRE_CSC_C31_C32_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_C33_C34                                                                    0x0ce8
+#define regCNVC_CFG0_PRE_CSC_C33_C34_BASE_IDX                                                           2
+#define regCNVC_CFG0_PRE_CSC_B_C11_C12                                                                  0x0ce9
+#define regCNVC_CFG0_PRE_CSC_B_C11_C12_BASE_IDX                                                         2
+#define regCNVC_CFG0_PRE_CSC_B_C13_C14                                                                  0x0cea
+#define regCNVC_CFG0_PRE_CSC_B_C13_C14_BASE_IDX                                                         2
+#define regCNVC_CFG0_PRE_CSC_B_C21_C22                                                                  0x0ceb
+#define regCNVC_CFG0_PRE_CSC_B_C21_C22_BASE_IDX                                                         2
+#define regCNVC_CFG0_PRE_CSC_B_C23_C24                                                                  0x0cec
+#define regCNVC_CFG0_PRE_CSC_B_C23_C24_BASE_IDX                                                         2
+#define regCNVC_CFG0_PRE_CSC_B_C31_C32                                                                  0x0ced
+#define regCNVC_CFG0_PRE_CSC_B_C31_C32_BASE_IDX                                                         2
+#define regCNVC_CFG0_PRE_CSC_B_C33_C34                                                                  0x0cee
+#define regCNVC_CFG0_PRE_CSC_B_C33_C34_BASE_IDX                                                         2
+#define regCNVC_CFG0_CNVC_COEF_FORMAT                                                                   0x0cef
+#define regCNVC_CFG0_CNVC_COEF_FORMAT_BASE_IDX                                                          2
+#define regCNVC_CFG0_PRE_DEGAM                                                                          0x0cf0
+#define regCNVC_CFG0_PRE_DEGAM_BASE_IDX                                                                 2
+#define regCNVC_CFG0_PRE_REALPHA                                                                        0x0cf1
+#define regCNVC_CFG0_PRE_REALPHA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cm_cur_dispdec
+// base address: 0x0
+#define regCM_CUR0_CURSOR0_CONTROL                                                                      0x0cf4
+#define regCM_CUR0_CURSOR0_CONTROL_BASE_IDX                                                             2
+#define regCM_CUR0_CURSOR0_COLOR0                                                                       0x0cf5
+#define regCM_CUR0_CURSOR0_COLOR0_BASE_IDX                                                              2
+#define regCM_CUR0_CURSOR0_COLOR1                                                                       0x0cf6
+#define regCM_CUR0_CURSOR0_COLOR1_BASE_IDX                                                              2
+#define regCM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y                                                            0x0cf7
+#define regCM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y_BASE_IDX                                                   2
+#define regCM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB                                                        0x0cf8
+#define regCM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB_BASE_IDX                                               2
+#define regCM_CUR0_CUR0_MATRIX_MODE                                                                     0x0cf9
+#define regCM_CUR0_CUR0_MATRIX_MODE_BASE_IDX                                                            2
+#define regCM_CUR0_CUR0_MATRIX_C11_C12_A                                                                0x0cfa
+#define regCM_CUR0_CUR0_MATRIX_C11_C12_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C13_C14_A                                                                0x0cfb
+#define regCM_CUR0_CUR0_MATRIX_C13_C14_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C21_C22_A                                                                0x0cfc
+#define regCM_CUR0_CUR0_MATRIX_C21_C22_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C23_C24_A                                                                0x0cfd
+#define regCM_CUR0_CUR0_MATRIX_C23_C24_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C31_C32_A                                                                0x0cfe
+#define regCM_CUR0_CUR0_MATRIX_C31_C32_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C33_C34_A                                                                0x0cff
+#define regCM_CUR0_CUR0_MATRIX_C33_C34_A_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C11_C12_B                                                                0x0d00
+#define regCM_CUR0_CUR0_MATRIX_C11_C12_B_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C13_C14_B                                                                0x0d01
+#define regCM_CUR0_CUR0_MATRIX_C13_C14_B_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C21_C22_B                                                                0x0d02
+#define regCM_CUR0_CUR0_MATRIX_C21_C22_B_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C23_C24_B                                                                0x0d03
+#define regCM_CUR0_CUR0_MATRIX_C23_C24_B_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C31_C32_B                                                                0x0d04
+#define regCM_CUR0_CUR0_MATRIX_C31_C32_B_BASE_IDX                                                       2
+#define regCM_CUR0_CUR0_MATRIX_C33_C34_B                                                                0x0d05
+#define regCM_CUR0_CUR0_MATRIX_C33_C34_B_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dscl_dispdec
+// base address: 0x0
+#define regDSCL0_SCL_COEF_RAM_TAP_SELECT                                                                0x0d08
+#define regDSCL0_SCL_COEF_RAM_TAP_SELECT_BASE_IDX                                                       2
+#define regDSCL0_SCL_COEF_RAM_TAP_DATA                                                                  0x0d09
+#define regDSCL0_SCL_COEF_RAM_TAP_DATA_BASE_IDX                                                         2
+#define regDSCL0_SCL_MODE                                                                               0x0d0a
+#define regDSCL0_SCL_MODE_BASE_IDX                                                                      2
+#define regDSCL0_SCL_TAP_CONTROL                                                                        0x0d0b
+#define regDSCL0_SCL_TAP_CONTROL_BASE_IDX                                                               2
+#define regDSCL0_DSCL_CONTROL                                                                           0x0d0c
+#define regDSCL0_DSCL_CONTROL_BASE_IDX                                                                  2
+#define regDSCL0_DSCL_2TAP_CONTROL                                                                      0x0d0d
+#define regDSCL0_DSCL_2TAP_CONTROL_BASE_IDX                                                             2
+#define regDSCL0_SCL_MANUAL_REPLICATE_CONTROL                                                           0x0d0e
+#define regDSCL0_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX                                                  2
+#define regDSCL0_SCL_HORZ_FILTER_SCALE_RATIO                                                            0x0d0f
+#define regDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL0_SCL_HORZ_FILTER_INIT                                                                   0x0d10
+#define regDSCL0_SCL_HORZ_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C                                                          0x0d11
+#define regDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL0_SCL_HORZ_FILTER_INIT_C                                                                 0x0d12
+#define regDSCL0_SCL_HORZ_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL0_SCL_VERT_FILTER_SCALE_RATIO                                                            0x0d13
+#define regDSCL0_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL0_SCL_VERT_FILTER_INIT                                                                   0x0d14
+#define regDSCL0_SCL_VERT_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL0_SCL_VERT_FILTER_INIT_BOT                                                               0x0d15
+#define regDSCL0_SCL_VERT_FILTER_INIT_BOT_BASE_IDX                                                      2
+#define regDSCL0_SCL_VERT_FILTER_SCALE_RATIO_C                                                          0x0d16
+#define regDSCL0_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL0_SCL_VERT_FILTER_INIT_C                                                                 0x0d17
+#define regDSCL0_SCL_VERT_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL0_SCL_VERT_FILTER_INIT_BOT_C                                                             0x0d18
+#define regDSCL0_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX                                                    2
+#define regDSCL0_SCL_BLACK_COLOR                                                                        0x0d19
+#define regDSCL0_SCL_BLACK_COLOR_BASE_IDX                                                               2
+#define regDSCL0_DSCL_UPDATE                                                                            0x0d1a
+#define regDSCL0_DSCL_UPDATE_BASE_IDX                                                                   2
+#define regDSCL0_DSCL_AUTOCAL                                                                           0x0d1b
+#define regDSCL0_DSCL_AUTOCAL_BASE_IDX                                                                  2
+#define regDSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT                                                           0x0d1c
+#define regDSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX                                                  2
+#define regDSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM                                                           0x0d1d
+#define regDSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX                                                  2
+#define regDSCL0_OTG_H_BLANK                                                                            0x0d1e
+#define regDSCL0_OTG_H_BLANK_BASE_IDX                                                                   2
+#define regDSCL0_OTG_V_BLANK                                                                            0x0d1f
+#define regDSCL0_OTG_V_BLANK_BASE_IDX                                                                   2
+#define regDSCL0_RECOUT_START                                                                           0x0d20
+#define regDSCL0_RECOUT_START_BASE_IDX                                                                  2
+#define regDSCL0_RECOUT_SIZE                                                                            0x0d21
+#define regDSCL0_RECOUT_SIZE_BASE_IDX                                                                   2
+#define regDSCL0_MPC_SIZE                                                                               0x0d22
+#define regDSCL0_MPC_SIZE_BASE_IDX                                                                      2
+#define regDSCL0_LB_DATA_FORMAT                                                                         0x0d23
+#define regDSCL0_LB_DATA_FORMAT_BASE_IDX                                                                2
+#define regDSCL0_LB_MEMORY_CTRL                                                                         0x0d24
+#define regDSCL0_LB_MEMORY_CTRL_BASE_IDX                                                                2
+#define regDSCL0_LB_V_COUNTER                                                                           0x0d25
+#define regDSCL0_LB_V_COUNTER_BASE_IDX                                                                  2
+#define regDSCL0_DSCL_MEM_PWR_CTRL                                                                      0x0d26
+#define regDSCL0_DSCL_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL0_DSCL_MEM_PWR_STATUS                                                                    0x0d27
+#define regDSCL0_DSCL_MEM_PWR_STATUS_BASE_IDX                                                           2
+#define regDSCL0_OBUF_CONTROL                                                                           0x0d28
+#define regDSCL0_OBUF_CONTROL_BASE_IDX                                                                  2
+#define regDSCL0_OBUF_MEM_PWR_CTRL                                                                      0x0d29
+#define regDSCL0_OBUF_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL0_DSCL_EASF_H_MODE                                                                       0x0d2a
+#define regDSCL0_DSCL_EASF_H_MODE_BASE_IDX                                                              2
+#define regDSCL0_DSCL_EASF_V_MODE                                                                       0x0d2b
+#define regDSCL0_DSCL_EASF_V_MODE_BASE_IDX                                                              2
+#define regDSCL0_DSCL_SC_MODE                                                                           0x0d2c
+#define regDSCL0_DSCL_SC_MODE_BASE_IDX                                                                  2
+#define regDSCL0_DSCL_SC_MATRIX_C0C1                                                                    0x0d2d
+#define regDSCL0_DSCL_SC_MATRIX_C0C1_BASE_IDX                                                           2
+#define regDSCL0_DSCL_SC_MATRIX_C2C3                                                                    0x0d2e
+#define regDSCL0_DSCL_SC_MATRIX_C2C3_BASE_IDX                                                           2
+#define regDSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN                                                       0x0d2f
+#define regDSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE                                                     0x0d30
+#define regDSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN                                                       0x0d31
+#define regDSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE                                                     0x0d32
+#define regDSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1                                                         0x0d33
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1_BASE_IDX                                                2
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2                                                         0x0d34
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2_BASE_IDX                                                2
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3                                                         0x0d35
+#define regDSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3_BASE_IDX                                                2
+#define regDSCL0_DSCL_EASF_RINGEST_FORCE                                                                0x0d36
+#define regDSCL0_DSCL_EASF_RINGEST_FORCE_BASE_IDX                                                       2
+#define regDSCL0_DSCL_EASF_H_BF_CNTL                                                                    0x0d37
+#define regDSCL0_DSCL_EASF_H_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN                                                           0x0d38
+#define regDSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL0_DSCL_EASF_V_BF_CNTL                                                                    0x0d39
+#define regDSCL0_DSCL_EASF_V_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN                                                           0x0d3a
+#define regDSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG0                                                               0x0d3b
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG1                                                               0x0d3c
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG2                                                               0x0d3d
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG3                                                               0x0d3e
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG4                                                               0x0d3f
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG5                                                               0x0d40
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG6                                                               0x0d41
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG7                                                               0x0d42
+#define regDSCL0_DSCL_EASF_H_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG0                                                               0x0d43
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG1                                                               0x0d44
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG2                                                               0x0d45
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG3                                                               0x0d46
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG4                                                               0x0d47
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG5                                                               0x0d48
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG6                                                               0x0d49
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG7                                                               0x0d4a
+#define regDSCL0_DSCL_EASF_V_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG0                                                               0x0d4b
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG1                                                               0x0d4c
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG2                                                               0x0d4d
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG3                                                               0x0d4e
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG4                                                               0x0d4f
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG5                                                               0x0d50
+#define regDSCL0_DSCL_EASF_H_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG0                                                               0x0d51
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG1                                                               0x0d52
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG2                                                               0x0d53
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG3                                                               0x0d54
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG4                                                               0x0d55
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG5                                                               0x0d56
+#define regDSCL0_DSCL_EASF_V_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL0_ISHARP_MODE                                                                            0x0d57
+#define regDSCL0_ISHARP_MODE_BASE_IDX                                                                   2
+#define regDSCL0_ISHARP_DELTA_CTRL                                                                      0x0d58
+#define regDSCL0_ISHARP_DELTA_CTRL_BASE_IDX                                                             2
+#define regDSCL0_ISHARP_DELTA_INDEX                                                                     0x0d59
+#define regDSCL0_ISHARP_DELTA_INDEX_BASE_IDX                                                            2
+#define regDSCL0_ISHARP_DELTA_DATA                                                                      0x0d5a
+#define regDSCL0_ISHARP_DELTA_DATA_BASE_IDX                                                             2
+#define regDSCL0_ISHARP_NLDELTA_SOFT_CLIP                                                               0x0d5b
+#define regDSCL0_ISHARP_NLDELTA_SOFT_CLIP_BASE_IDX                                                      2
+#define regDSCL0_ISHARP_NOISEDET_THRESHOLD                                                              0x0d5c
+#define regDSCL0_ISHARP_NOISEDET_THRESHOLD_BASE_IDX                                                     2
+#define regDSCL0_ISHARP_NOISE_GAIN_PWL                                                                  0x0d5d
+#define regDSCL0_ISHARP_NOISE_GAIN_PWL_BASE_IDX                                                         2
+#define regDSCL0_ISHARP_LBA_PWL_SEG0                                                                    0x0d5e
+#define regDSCL0_ISHARP_LBA_PWL_SEG0_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_LBA_PWL_SEG1                                                                    0x0d5f
+#define regDSCL0_ISHARP_LBA_PWL_SEG1_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_LBA_PWL_SEG2                                                                    0x0d60
+#define regDSCL0_ISHARP_LBA_PWL_SEG2_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_LBA_PWL_SEG3                                                                    0x0d61
+#define regDSCL0_ISHARP_LBA_PWL_SEG3_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_LBA_PWL_SEG4                                                                    0x0d62
+#define regDSCL0_ISHARP_LBA_PWL_SEG4_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_LBA_PWL_SEG5                                                                    0x0d63
+#define regDSCL0_ISHARP_LBA_PWL_SEG5_BASE_IDX                                                           2
+#define regDSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL                                                          0x0d64
+#define regDSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cm_dispdec
+// base address: 0x0
+#define regCM0_CM_CONTROL                                                                               0x0d69
+#define regCM0_CM_CONTROL_BASE_IDX                                                                      2
+#define regCM0_CM_POST_CSC_CONTROL                                                                      0x0d6a
+#define regCM0_CM_POST_CSC_CONTROL_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C11_C12                                                                      0x0d6b
+#define regCM0_CM_POST_CSC_C11_C12_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C13_C14                                                                      0x0d6c
+#define regCM0_CM_POST_CSC_C13_C14_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C21_C22                                                                      0x0d6d
+#define regCM0_CM_POST_CSC_C21_C22_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C23_C24                                                                      0x0d6e
+#define regCM0_CM_POST_CSC_C23_C24_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C31_C32                                                                      0x0d6f
+#define regCM0_CM_POST_CSC_C31_C32_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_C33_C34                                                                      0x0d70
+#define regCM0_CM_POST_CSC_C33_C34_BASE_IDX                                                             2
+#define regCM0_CM_POST_CSC_B_C11_C12                                                                    0x0d71
+#define regCM0_CM_POST_CSC_B_C11_C12_BASE_IDX                                                           2
+#define regCM0_CM_POST_CSC_B_C13_C14                                                                    0x0d72
+#define regCM0_CM_POST_CSC_B_C13_C14_BASE_IDX                                                           2
+#define regCM0_CM_POST_CSC_B_C21_C22                                                                    0x0d73
+#define regCM0_CM_POST_CSC_B_C21_C22_BASE_IDX                                                           2
+#define regCM0_CM_POST_CSC_B_C23_C24                                                                    0x0d74
+#define regCM0_CM_POST_CSC_B_C23_C24_BASE_IDX                                                           2
+#define regCM0_CM_POST_CSC_B_C31_C32                                                                    0x0d75
+#define regCM0_CM_POST_CSC_B_C31_C32_BASE_IDX                                                           2
+#define regCM0_CM_POST_CSC_B_C33_C34                                                                    0x0d76
+#define regCM0_CM_POST_CSC_B_C33_C34_BASE_IDX                                                           2
+#define regCM0_CM_BIAS_CR_R                                                                             0x0d77
+#define regCM0_CM_BIAS_CR_R_BASE_IDX                                                                    2
+#define regCM0_CM_BIAS_Y_G_CB_B                                                                         0x0d78
+#define regCM0_CM_BIAS_Y_G_CB_B_BASE_IDX                                                                2
+#define regCM0_CM_GAMCOR_CONTROL                                                                        0x0d79
+#define regCM0_CM_GAMCOR_CONTROL_BASE_IDX                                                               2
+#define regCM0_CM_GAMCOR_LUT_INDEX                                                                      0x0d7a
+#define regCM0_CM_GAMCOR_LUT_INDEX_BASE_IDX                                                             2
+#define regCM0_CM_GAMCOR_LUT_DATA                                                                       0x0d7b
+#define regCM0_CM_GAMCOR_LUT_DATA_BASE_IDX                                                              2
+#define regCM0_CM_GAMCOR_LUT_CONTROL                                                                    0x0d7c
+#define regCM0_CM_GAMCOR_LUT_CONTROL_BASE_IDX                                                           2
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_B                                                              0x0d7d
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_B_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_G                                                              0x0d7e
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_G_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_R                                                              0x0d7f
+#define regCM0_CM_GAMCOR_RAMA_START_CNTL_R_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B                                                        0x0d80
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G                                                        0x0d81
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R                                                        0x0d82
+#define regCM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B                                                         0x0d83
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_G                                                         0x0d84
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_R                                                         0x0d85
+#define regCM0_CM_GAMCOR_RAMA_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_B                                                               0x0d86
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_B                                                               0x0d87
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_G                                                               0x0d88
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_G                                                               0x0d89
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_R                                                               0x0d8a
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_R                                                               0x0d8b
+#define regCM0_CM_GAMCOR_RAMA_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_B                                                                  0x0d8c
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_B_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_G                                                                  0x0d8d
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_G_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_R                                                                  0x0d8e
+#define regCM0_CM_GAMCOR_RAMA_OFFSET_R_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMA_REGION_0_1                                                                0x0d8f
+#define regCM0_CM_GAMCOR_RAMA_REGION_0_1_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMA_REGION_2_3                                                                0x0d90
+#define regCM0_CM_GAMCOR_RAMA_REGION_2_3_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMA_REGION_4_5                                                                0x0d91
+#define regCM0_CM_GAMCOR_RAMA_REGION_4_5_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMA_REGION_6_7                                                                0x0d92
+#define regCM0_CM_GAMCOR_RAMA_REGION_6_7_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMA_REGION_8_9                                                                0x0d93
+#define regCM0_CM_GAMCOR_RAMA_REGION_8_9_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMA_REGION_10_11                                                              0x0d94
+#define regCM0_CM_GAMCOR_RAMA_REGION_10_11_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_12_13                                                              0x0d95
+#define regCM0_CM_GAMCOR_RAMA_REGION_12_13_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_14_15                                                              0x0d96
+#define regCM0_CM_GAMCOR_RAMA_REGION_14_15_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_16_17                                                              0x0d97
+#define regCM0_CM_GAMCOR_RAMA_REGION_16_17_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_18_19                                                              0x0d98
+#define regCM0_CM_GAMCOR_RAMA_REGION_18_19_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_20_21                                                              0x0d99
+#define regCM0_CM_GAMCOR_RAMA_REGION_20_21_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_22_23                                                              0x0d9a
+#define regCM0_CM_GAMCOR_RAMA_REGION_22_23_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_24_25                                                              0x0d9b
+#define regCM0_CM_GAMCOR_RAMA_REGION_24_25_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_26_27                                                              0x0d9c
+#define regCM0_CM_GAMCOR_RAMA_REGION_26_27_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_28_29                                                              0x0d9d
+#define regCM0_CM_GAMCOR_RAMA_REGION_28_29_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_30_31                                                              0x0d9e
+#define regCM0_CM_GAMCOR_RAMA_REGION_30_31_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMA_REGION_32_33                                                              0x0d9f
+#define regCM0_CM_GAMCOR_RAMA_REGION_32_33_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_B                                                              0x0da0
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_B_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_G                                                              0x0da1
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_G_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_R                                                              0x0da2
+#define regCM0_CM_GAMCOR_RAMB_START_CNTL_R_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B                                                        0x0da3
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G                                                        0x0da4
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R                                                        0x0da5
+#define regCM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_B                                                         0x0da6
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_G                                                         0x0da7
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_R                                                         0x0da8
+#define regCM0_CM_GAMCOR_RAMB_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_B                                                               0x0da9
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_B                                                               0x0daa
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_G                                                               0x0dab
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_G                                                               0x0dac
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_R                                                               0x0dad
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_R                                                               0x0dae
+#define regCM0_CM_GAMCOR_RAMB_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_B                                                                  0x0daf
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_B_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_G                                                                  0x0db0
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_G_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_R                                                                  0x0db1
+#define regCM0_CM_GAMCOR_RAMB_OFFSET_R_BASE_IDX                                                         2
+#define regCM0_CM_GAMCOR_RAMB_REGION_0_1                                                                0x0db2
+#define regCM0_CM_GAMCOR_RAMB_REGION_0_1_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMB_REGION_2_3                                                                0x0db3
+#define regCM0_CM_GAMCOR_RAMB_REGION_2_3_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMB_REGION_4_5                                                                0x0db4
+#define regCM0_CM_GAMCOR_RAMB_REGION_4_5_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMB_REGION_6_7                                                                0x0db5
+#define regCM0_CM_GAMCOR_RAMB_REGION_6_7_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMB_REGION_8_9                                                                0x0db6
+#define regCM0_CM_GAMCOR_RAMB_REGION_8_9_BASE_IDX                                                       2
+#define regCM0_CM_GAMCOR_RAMB_REGION_10_11                                                              0x0db7
+#define regCM0_CM_GAMCOR_RAMB_REGION_10_11_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_12_13                                                              0x0db8
+#define regCM0_CM_GAMCOR_RAMB_REGION_12_13_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_14_15                                                              0x0db9
+#define regCM0_CM_GAMCOR_RAMB_REGION_14_15_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_16_17                                                              0x0dba
+#define regCM0_CM_GAMCOR_RAMB_REGION_16_17_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_18_19                                                              0x0dbb
+#define regCM0_CM_GAMCOR_RAMB_REGION_18_19_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_20_21                                                              0x0dbc
+#define regCM0_CM_GAMCOR_RAMB_REGION_20_21_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_22_23                                                              0x0dbd
+#define regCM0_CM_GAMCOR_RAMB_REGION_22_23_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_24_25                                                              0x0dbe
+#define regCM0_CM_GAMCOR_RAMB_REGION_24_25_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_26_27                                                              0x0dbf
+#define regCM0_CM_GAMCOR_RAMB_REGION_26_27_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_28_29                                                              0x0dc0
+#define regCM0_CM_GAMCOR_RAMB_REGION_28_29_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_30_31                                                              0x0dc1
+#define regCM0_CM_GAMCOR_RAMB_REGION_30_31_BASE_IDX                                                     2
+#define regCM0_CM_GAMCOR_RAMB_REGION_32_33                                                              0x0dc2
+#define regCM0_CM_GAMCOR_RAMB_REGION_32_33_BASE_IDX                                                     2
+#define regCM0_CM_HDR_MULT_COEF                                                                         0x0dc3
+#define regCM0_CM_HDR_MULT_COEF_BASE_IDX                                                                2
+#define regCM0_CM_MEM_PWR_CTRL                                                                          0x0dc4
+#define regCM0_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
+#define regCM0_CM_MEM_PWR_STATUS                                                                        0x0dc5
+#define regCM0_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
+#define regCM0_CM_DEALPHA                                                                               0x0dc7
+#define regCM0_CM_DEALPHA_BASE_IDX                                                                      2
+#define regCM0_CM_COEF_FORMAT                                                                           0x0dc8
+#define regCM0_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM0_CM_TEST_DEBUG_INDEX                                                                      0x0dc9
+#define regCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM0_CM_TEST_DEBUG_DATA                                                                       0x0dca
+#define regCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
+#define regCM0_CM_HIST_CNTL                                                                             0x0dcb
+#define regCM0_CM_HIST_CNTL_BASE_IDX                                                                    2
+#define regCM0_CM_HIST_SCALE_SRC1                                                                       0x0dcc
+#define regCM0_CM_HIST_SCALE_SRC1_BASE_IDX                                                              2
+#define regCM0_CM_HIST_COEFA_SRC2                                                                       0x0dcd
+#define regCM0_CM_HIST_COEFA_SRC2_BASE_IDX                                                              2
+#define regCM0_CM_HIST_COEFB_SRC2                                                                       0x0dce
+#define regCM0_CM_HIST_COEFB_SRC2_BASE_IDX                                                              2
+#define regCM0_CM_HIST_COEFC_SRC2                                                                       0x0dcf
+#define regCM0_CM_HIST_COEFC_SRC2_BASE_IDX                                                              2
+#define regCM0_CM_HIST_SCALE_SRC3                                                                       0x0dd0
+#define regCM0_CM_HIST_SCALE_SRC3_BASE_IDX                                                              2
+#define regCM0_CM_HIST_BIAS_SRC1                                                                        0x0dd1
+#define regCM0_CM_HIST_BIAS_SRC1_BASE_IDX                                                               2
+#define regCM0_CM_HIST_BIAS_SRC2                                                                        0x0dd2
+#define regCM0_CM_HIST_BIAS_SRC2_BASE_IDX                                                               2
+#define regCM0_CM_HIST_BIAS_SRC3                                                                        0x0dd3
+#define regCM0_CM_HIST_BIAS_SRC3_BASE_IDX                                                               2
+#define regCM0_CM_HIST_LOCK                                                                             0x0dd4
+#define regCM0_CM_HIST_LOCK_BASE_IDX                                                                    2
+#define regCM0_CM_HIST_INDEX                                                                            0x0dd5
+#define regCM0_CM_HIST_INDEX_BASE_IDX                                                                   2
+#define regCM0_CM_HIST_DATA                                                                             0x0dd6
+#define regCM0_CM_HIST_DATA_BASE_IDX                                                                    2
+#define regCM0_CM_HIST_STATUS                                                                           0x0dd7
+#define regCM0_CM_HIST_STATUS_BASE_IDX                                                                  2
+#define regCM0_CM_HIST_INT_CONTROL                                                                      0x0dd8
+#define regCM0_CM_HIST_INT_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x3890
+#define regDC_PERFMON10_PERFCOUNTER_CNTL                                                                0x0e24
+#define regDC_PERFMON10_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON10_PERFCOUNTER_CNTL2                                                               0x0e25
+#define regDC_PERFMON10_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON10_PERFCOUNTER_STATE                                                               0x0e26
+#define regDC_PERFMON10_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON10_PERFMON_CNTL                                                                    0x0e27
+#define regDC_PERFMON10_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON10_PERFMON_CNTL2                                                                   0x0e28
+#define regDC_PERFMON10_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON10_PERFMON_CVALUE_INT_MISC                                                         0x0e29
+#define regDC_PERFMON10_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON10_PERFMON_CVALUE_LOW                                                              0x0e2a
+#define regDC_PERFMON10_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON10_PERFMON_HI                                                                      0x0e2b
+#define regDC_PERFMON10_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON10_PERFMON_LOW                                                                     0x0e2c
+#define regDC_PERFMON10_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_top_dispdec
+// base address: 0x5ac
+#define regDPP_TOP1_DPP_CONTROL                                                                         0x0e30
+#define regDPP_TOP1_DPP_CONTROL_BASE_IDX                                                                2
+#define regDPP_TOP1_DPP_SOFT_RESET                                                                      0x0e31
+#define regDPP_TOP1_DPP_SOFT_RESET_BASE_IDX                                                             2
+#define regDPP_TOP1_DPP_CRC_VAL_R                                                                       0x0e32
+#define regDPP_TOP1_DPP_CRC_VAL_R_BASE_IDX                                                              2
+#define regDPP_TOP1_DPP_CRC_VAL_G                                                                       0x0e33
+#define regDPP_TOP1_DPP_CRC_VAL_G_BASE_IDX                                                              2
+#define regDPP_TOP1_DPP_CRC_VAL_B                                                                       0x0e34
+#define regDPP_TOP1_DPP_CRC_VAL_B_BASE_IDX                                                              2
+#define regDPP_TOP1_DPP_CRC_VAL_A                                                                       0x0e35
+#define regDPP_TOP1_DPP_CRC_VAL_A_BASE_IDX                                                              2
+#define regDPP_TOP1_DPP_CRC_CTRL                                                                        0x0e36
+#define regDPP_TOP1_DPP_CRC_CTRL_BASE_IDX                                                               2
+#define regDPP_TOP1_HOST_READ_CONTROL                                                                   0x0e37
+#define regDPP_TOP1_HOST_READ_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cnvc_cfg_dispdec
+// base address: 0x5ac
+#define regCNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT                                                          0x0e3c
+#define regCNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX                                                 2
+#define regCNVC_CFG1_FORMAT_CONTROL                                                                     0x0e3d
+#define regCNVC_CFG1_FORMAT_CONTROL_BASE_IDX                                                            2
+#define regCNVC_CFG1_FCNV_FP_BIAS_R                                                                     0x0e3e
+#define regCNVC_CFG1_FCNV_FP_BIAS_R_BASE_IDX                                                            2
+#define regCNVC_CFG1_FCNV_FP_BIAS_G                                                                     0x0e3f
+#define regCNVC_CFG1_FCNV_FP_BIAS_G_BASE_IDX                                                            2
+#define regCNVC_CFG1_FCNV_FP_BIAS_B                                                                     0x0e40
+#define regCNVC_CFG1_FCNV_FP_BIAS_B_BASE_IDX                                                            2
+#define regCNVC_CFG1_FCNV_FP_SCALE_R                                                                    0x0e41
+#define regCNVC_CFG1_FCNV_FP_SCALE_R_BASE_IDX                                                           2
+#define regCNVC_CFG1_FCNV_FP_SCALE_G                                                                    0x0e42
+#define regCNVC_CFG1_FCNV_FP_SCALE_G_BASE_IDX                                                           2
+#define regCNVC_CFG1_FCNV_FP_SCALE_B                                                                    0x0e43
+#define regCNVC_CFG1_FCNV_FP_SCALE_B_BASE_IDX                                                           2
+#define regCNVC_CFG1_COLOR_KEYER_CONTROL                                                                0x0e44
+#define regCNVC_CFG1_COLOR_KEYER_CONTROL_BASE_IDX                                                       2
+#define regCNVC_CFG1_COLOR_KEYER_ALPHA                                                                  0x0e45
+#define regCNVC_CFG1_COLOR_KEYER_ALPHA_BASE_IDX                                                         2
+#define regCNVC_CFG1_COLOR_KEYER_RED                                                                    0x0e46
+#define regCNVC_CFG1_COLOR_KEYER_RED_BASE_IDX                                                           2
+#define regCNVC_CFG1_COLOR_KEYER_GREEN                                                                  0x0e47
+#define regCNVC_CFG1_COLOR_KEYER_GREEN_BASE_IDX                                                         2
+#define regCNVC_CFG1_COLOR_KEYER_BLUE                                                                   0x0e48
+#define regCNVC_CFG1_COLOR_KEYER_BLUE_BASE_IDX                                                          2
+#define regCNVC_CFG1_ALPHA_2BIT_LUT01                                                                   0x0e4a
+#define regCNVC_CFG1_ALPHA_2BIT_LUT01_BASE_IDX                                                          2
+#define regCNVC_CFG1_ALPHA_2BIT_LUT23                                                                   0x0e4b
+#define regCNVC_CFG1_ALPHA_2BIT_LUT23_BASE_IDX                                                          2
+#define regCNVC_CFG1_PRE_DEALPHA                                                                        0x0e4c
+#define regCNVC_CFG1_PRE_DEALPHA_BASE_IDX                                                               2
+#define regCNVC_CFG1_PRE_CSC_MODE                                                                       0x0e4d
+#define regCNVC_CFG1_PRE_CSC_MODE_BASE_IDX                                                              2
+#define regCNVC_CFG1_PRE_CSC_C11_C12                                                                    0x0e4e
+#define regCNVC_CFG1_PRE_CSC_C11_C12_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_C13_C14                                                                    0x0e4f
+#define regCNVC_CFG1_PRE_CSC_C13_C14_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_C21_C22                                                                    0x0e50
+#define regCNVC_CFG1_PRE_CSC_C21_C22_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_C23_C24                                                                    0x0e51
+#define regCNVC_CFG1_PRE_CSC_C23_C24_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_C31_C32                                                                    0x0e52
+#define regCNVC_CFG1_PRE_CSC_C31_C32_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_C33_C34                                                                    0x0e53
+#define regCNVC_CFG1_PRE_CSC_C33_C34_BASE_IDX                                                           2
+#define regCNVC_CFG1_PRE_CSC_B_C11_C12                                                                  0x0e54
+#define regCNVC_CFG1_PRE_CSC_B_C11_C12_BASE_IDX                                                         2
+#define regCNVC_CFG1_PRE_CSC_B_C13_C14                                                                  0x0e55
+#define regCNVC_CFG1_PRE_CSC_B_C13_C14_BASE_IDX                                                         2
+#define regCNVC_CFG1_PRE_CSC_B_C21_C22                                                                  0x0e56
+#define regCNVC_CFG1_PRE_CSC_B_C21_C22_BASE_IDX                                                         2
+#define regCNVC_CFG1_PRE_CSC_B_C23_C24                                                                  0x0e57
+#define regCNVC_CFG1_PRE_CSC_B_C23_C24_BASE_IDX                                                         2
+#define regCNVC_CFG1_PRE_CSC_B_C31_C32                                                                  0x0e58
+#define regCNVC_CFG1_PRE_CSC_B_C31_C32_BASE_IDX                                                         2
+#define regCNVC_CFG1_PRE_CSC_B_C33_C34                                                                  0x0e59
+#define regCNVC_CFG1_PRE_CSC_B_C33_C34_BASE_IDX                                                         2
+#define regCNVC_CFG1_CNVC_COEF_FORMAT                                                                   0x0e5a
+#define regCNVC_CFG1_CNVC_COEF_FORMAT_BASE_IDX                                                          2
+#define regCNVC_CFG1_PRE_DEGAM                                                                          0x0e5b
+#define regCNVC_CFG1_PRE_DEGAM_BASE_IDX                                                                 2
+#define regCNVC_CFG1_PRE_REALPHA                                                                        0x0e5c
+#define regCNVC_CFG1_PRE_REALPHA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cm_cur_dispdec
+// base address: 0x5ac
+#define regCM_CUR1_CURSOR0_CONTROL                                                                      0x0e5f
+#define regCM_CUR1_CURSOR0_CONTROL_BASE_IDX                                                             2
+#define regCM_CUR1_CURSOR0_COLOR0                                                                       0x0e60
+#define regCM_CUR1_CURSOR0_COLOR0_BASE_IDX                                                              2
+#define regCM_CUR1_CURSOR0_COLOR1                                                                       0x0e61
+#define regCM_CUR1_CURSOR0_COLOR1_BASE_IDX                                                              2
+#define regCM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y                                                            0x0e62
+#define regCM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y_BASE_IDX                                                   2
+#define regCM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB                                                        0x0e63
+#define regCM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB_BASE_IDX                                               2
+#define regCM_CUR1_CUR0_MATRIX_MODE                                                                     0x0e64
+#define regCM_CUR1_CUR0_MATRIX_MODE_BASE_IDX                                                            2
+#define regCM_CUR1_CUR0_MATRIX_C11_C12_A                                                                0x0e65
+#define regCM_CUR1_CUR0_MATRIX_C11_C12_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C13_C14_A                                                                0x0e66
+#define regCM_CUR1_CUR0_MATRIX_C13_C14_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C21_C22_A                                                                0x0e67
+#define regCM_CUR1_CUR0_MATRIX_C21_C22_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C23_C24_A                                                                0x0e68
+#define regCM_CUR1_CUR0_MATRIX_C23_C24_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C31_C32_A                                                                0x0e69
+#define regCM_CUR1_CUR0_MATRIX_C31_C32_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C33_C34_A                                                                0x0e6a
+#define regCM_CUR1_CUR0_MATRIX_C33_C34_A_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C11_C12_B                                                                0x0e6b
+#define regCM_CUR1_CUR0_MATRIX_C11_C12_B_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C13_C14_B                                                                0x0e6c
+#define regCM_CUR1_CUR0_MATRIX_C13_C14_B_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C21_C22_B                                                                0x0e6d
+#define regCM_CUR1_CUR0_MATRIX_C21_C22_B_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C23_C24_B                                                                0x0e6e
+#define regCM_CUR1_CUR0_MATRIX_C23_C24_B_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C31_C32_B                                                                0x0e6f
+#define regCM_CUR1_CUR0_MATRIX_C31_C32_B_BASE_IDX                                                       2
+#define regCM_CUR1_CUR0_MATRIX_C33_C34_B                                                                0x0e70
+#define regCM_CUR1_CUR0_MATRIX_C33_C34_B_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dscl_dispdec
+// base address: 0x5ac
+#define regDSCL1_SCL_COEF_RAM_TAP_SELECT                                                                0x0e73
+#define regDSCL1_SCL_COEF_RAM_TAP_SELECT_BASE_IDX                                                       2
+#define regDSCL1_SCL_COEF_RAM_TAP_DATA                                                                  0x0e74
+#define regDSCL1_SCL_COEF_RAM_TAP_DATA_BASE_IDX                                                         2
+#define regDSCL1_SCL_MODE                                                                               0x0e75
+#define regDSCL1_SCL_MODE_BASE_IDX                                                                      2
+#define regDSCL1_SCL_TAP_CONTROL                                                                        0x0e76
+#define regDSCL1_SCL_TAP_CONTROL_BASE_IDX                                                               2
+#define regDSCL1_DSCL_CONTROL                                                                           0x0e77
+#define regDSCL1_DSCL_CONTROL_BASE_IDX                                                                  2
+#define regDSCL1_DSCL_2TAP_CONTROL                                                                      0x0e78
+#define regDSCL1_DSCL_2TAP_CONTROL_BASE_IDX                                                             2
+#define regDSCL1_SCL_MANUAL_REPLICATE_CONTROL                                                           0x0e79
+#define regDSCL1_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX                                                  2
+#define regDSCL1_SCL_HORZ_FILTER_SCALE_RATIO                                                            0x0e7a
+#define regDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL1_SCL_HORZ_FILTER_INIT                                                                   0x0e7b
+#define regDSCL1_SCL_HORZ_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C                                                          0x0e7c
+#define regDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL1_SCL_HORZ_FILTER_INIT_C                                                                 0x0e7d
+#define regDSCL1_SCL_HORZ_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL1_SCL_VERT_FILTER_SCALE_RATIO                                                            0x0e7e
+#define regDSCL1_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL1_SCL_VERT_FILTER_INIT                                                                   0x0e7f
+#define regDSCL1_SCL_VERT_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL1_SCL_VERT_FILTER_INIT_BOT                                                               0x0e80
+#define regDSCL1_SCL_VERT_FILTER_INIT_BOT_BASE_IDX                                                      2
+#define regDSCL1_SCL_VERT_FILTER_SCALE_RATIO_C                                                          0x0e81
+#define regDSCL1_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL1_SCL_VERT_FILTER_INIT_C                                                                 0x0e82
+#define regDSCL1_SCL_VERT_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL1_SCL_VERT_FILTER_INIT_BOT_C                                                             0x0e83
+#define regDSCL1_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX                                                    2
+#define regDSCL1_SCL_BLACK_COLOR                                                                        0x0e84
+#define regDSCL1_SCL_BLACK_COLOR_BASE_IDX                                                               2
+#define regDSCL1_DSCL_UPDATE                                                                            0x0e85
+#define regDSCL1_DSCL_UPDATE_BASE_IDX                                                                   2
+#define regDSCL1_DSCL_AUTOCAL                                                                           0x0e86
+#define regDSCL1_DSCL_AUTOCAL_BASE_IDX                                                                  2
+#define regDSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT                                                           0x0e87
+#define regDSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX                                                  2
+#define regDSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM                                                           0x0e88
+#define regDSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX                                                  2
+#define regDSCL1_OTG_H_BLANK                                                                            0x0e89
+#define regDSCL1_OTG_H_BLANK_BASE_IDX                                                                   2
+#define regDSCL1_OTG_V_BLANK                                                                            0x0e8a
+#define regDSCL1_OTG_V_BLANK_BASE_IDX                                                                   2
+#define regDSCL1_RECOUT_START                                                                           0x0e8b
+#define regDSCL1_RECOUT_START_BASE_IDX                                                                  2
+#define regDSCL1_RECOUT_SIZE                                                                            0x0e8c
+#define regDSCL1_RECOUT_SIZE_BASE_IDX                                                                   2
+#define regDSCL1_MPC_SIZE                                                                               0x0e8d
+#define regDSCL1_MPC_SIZE_BASE_IDX                                                                      2
+#define regDSCL1_LB_DATA_FORMAT                                                                         0x0e8e
+#define regDSCL1_LB_DATA_FORMAT_BASE_IDX                                                                2
+#define regDSCL1_LB_MEMORY_CTRL                                                                         0x0e8f
+#define regDSCL1_LB_MEMORY_CTRL_BASE_IDX                                                                2
+#define regDSCL1_LB_V_COUNTER                                                                           0x0e90
+#define regDSCL1_LB_V_COUNTER_BASE_IDX                                                                  2
+#define regDSCL1_DSCL_MEM_PWR_CTRL                                                                      0x0e91
+#define regDSCL1_DSCL_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL1_DSCL_MEM_PWR_STATUS                                                                    0x0e92
+#define regDSCL1_DSCL_MEM_PWR_STATUS_BASE_IDX                                                           2
+#define regDSCL1_OBUF_CONTROL                                                                           0x0e93
+#define regDSCL1_OBUF_CONTROL_BASE_IDX                                                                  2
+#define regDSCL1_OBUF_MEM_PWR_CTRL                                                                      0x0e94
+#define regDSCL1_OBUF_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL1_DSCL_EASF_H_MODE                                                                       0x0e95
+#define regDSCL1_DSCL_EASF_H_MODE_BASE_IDX                                                              2
+#define regDSCL1_DSCL_EASF_V_MODE                                                                       0x0e96
+#define regDSCL1_DSCL_EASF_V_MODE_BASE_IDX                                                              2
+#define regDSCL1_DSCL_SC_MODE                                                                           0x0e97
+#define regDSCL1_DSCL_SC_MODE_BASE_IDX                                                                  2
+#define regDSCL1_DSCL_SC_MATRIX_C0C1                                                                    0x0e98
+#define regDSCL1_DSCL_SC_MATRIX_C0C1_BASE_IDX                                                           2
+#define regDSCL1_DSCL_SC_MATRIX_C2C3                                                                    0x0e99
+#define regDSCL1_DSCL_SC_MATRIX_C2C3_BASE_IDX                                                           2
+#define regDSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN                                                       0x0e9a
+#define regDSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE                                                     0x0e9b
+#define regDSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN                                                       0x0e9c
+#define regDSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE                                                     0x0e9d
+#define regDSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1                                                         0x0e9e
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1_BASE_IDX                                                2
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2                                                         0x0e9f
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2_BASE_IDX                                                2
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3                                                         0x0ea0
+#define regDSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3_BASE_IDX                                                2
+#define regDSCL1_DSCL_EASF_RINGEST_FORCE                                                                0x0ea1
+#define regDSCL1_DSCL_EASF_RINGEST_FORCE_BASE_IDX                                                       2
+#define regDSCL1_DSCL_EASF_H_BF_CNTL                                                                    0x0ea2
+#define regDSCL1_DSCL_EASF_H_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN                                                           0x0ea3
+#define regDSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL1_DSCL_EASF_V_BF_CNTL                                                                    0x0ea4
+#define regDSCL1_DSCL_EASF_V_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN                                                           0x0ea5
+#define regDSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG0                                                               0x0ea6
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG1                                                               0x0ea7
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG2                                                               0x0ea8
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG3                                                               0x0ea9
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG4                                                               0x0eaa
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG5                                                               0x0eab
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG6                                                               0x0eac
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG7                                                               0x0ead
+#define regDSCL1_DSCL_EASF_H_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG0                                                               0x0eae
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG1                                                               0x0eaf
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG2                                                               0x0eb0
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG3                                                               0x0eb1
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG4                                                               0x0eb2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG5                                                               0x0eb3
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG6                                                               0x0eb4
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG7                                                               0x0eb5
+#define regDSCL1_DSCL_EASF_V_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG0                                                               0x0eb6
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG1                                                               0x0eb7
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG2                                                               0x0eb8
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG3                                                               0x0eb9
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG4                                                               0x0eba
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG5                                                               0x0ebb
+#define regDSCL1_DSCL_EASF_H_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG0                                                               0x0ebc
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG1                                                               0x0ebd
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG2                                                               0x0ebe
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG3                                                               0x0ebf
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG4                                                               0x0ec0
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG5                                                               0x0ec1
+#define regDSCL1_DSCL_EASF_V_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL1_ISHARP_MODE                                                                            0x0ec2
+#define regDSCL1_ISHARP_MODE_BASE_IDX                                                                   2
+#define regDSCL1_ISHARP_DELTA_CTRL                                                                      0x0ec3
+#define regDSCL1_ISHARP_DELTA_CTRL_BASE_IDX                                                             2
+#define regDSCL1_ISHARP_DELTA_INDEX                                                                     0x0ec4
+#define regDSCL1_ISHARP_DELTA_INDEX_BASE_IDX                                                            2
+#define regDSCL1_ISHARP_DELTA_DATA                                                                      0x0ec5
+#define regDSCL1_ISHARP_DELTA_DATA_BASE_IDX                                                             2
+#define regDSCL1_ISHARP_NLDELTA_SOFT_CLIP                                                               0x0ec6
+#define regDSCL1_ISHARP_NLDELTA_SOFT_CLIP_BASE_IDX                                                      2
+#define regDSCL1_ISHARP_NOISEDET_THRESHOLD                                                              0x0ec7
+#define regDSCL1_ISHARP_NOISEDET_THRESHOLD_BASE_IDX                                                     2
+#define regDSCL1_ISHARP_NOISE_GAIN_PWL                                                                  0x0ec8
+#define regDSCL1_ISHARP_NOISE_GAIN_PWL_BASE_IDX                                                         2
+#define regDSCL1_ISHARP_LBA_PWL_SEG0                                                                    0x0ec9
+#define regDSCL1_ISHARP_LBA_PWL_SEG0_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_LBA_PWL_SEG1                                                                    0x0eca
+#define regDSCL1_ISHARP_LBA_PWL_SEG1_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_LBA_PWL_SEG2                                                                    0x0ecb
+#define regDSCL1_ISHARP_LBA_PWL_SEG2_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_LBA_PWL_SEG3                                                                    0x0ecc
+#define regDSCL1_ISHARP_LBA_PWL_SEG3_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_LBA_PWL_SEG4                                                                    0x0ecd
+#define regDSCL1_ISHARP_LBA_PWL_SEG4_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_LBA_PWL_SEG5                                                                    0x0ece
+#define regDSCL1_ISHARP_LBA_PWL_SEG5_BASE_IDX                                                           2
+#define regDSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL                                                          0x0ecf
+#define regDSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cm_dispdec
+// base address: 0x5ac
+#define regCM1_CM_CONTROL                                                                               0x0ed4
+#define regCM1_CM_CONTROL_BASE_IDX                                                                      2
+#define regCM1_CM_POST_CSC_CONTROL                                                                      0x0ed5
+#define regCM1_CM_POST_CSC_CONTROL_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C11_C12                                                                      0x0ed6
+#define regCM1_CM_POST_CSC_C11_C12_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C13_C14                                                                      0x0ed7
+#define regCM1_CM_POST_CSC_C13_C14_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C21_C22                                                                      0x0ed8
+#define regCM1_CM_POST_CSC_C21_C22_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C23_C24                                                                      0x0ed9
+#define regCM1_CM_POST_CSC_C23_C24_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C31_C32                                                                      0x0eda
+#define regCM1_CM_POST_CSC_C31_C32_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_C33_C34                                                                      0x0edb
+#define regCM1_CM_POST_CSC_C33_C34_BASE_IDX                                                             2
+#define regCM1_CM_POST_CSC_B_C11_C12                                                                    0x0edc
+#define regCM1_CM_POST_CSC_B_C11_C12_BASE_IDX                                                           2
+#define regCM1_CM_POST_CSC_B_C13_C14                                                                    0x0edd
+#define regCM1_CM_POST_CSC_B_C13_C14_BASE_IDX                                                           2
+#define regCM1_CM_POST_CSC_B_C21_C22                                                                    0x0ede
+#define regCM1_CM_POST_CSC_B_C21_C22_BASE_IDX                                                           2
+#define regCM1_CM_POST_CSC_B_C23_C24                                                                    0x0edf
+#define regCM1_CM_POST_CSC_B_C23_C24_BASE_IDX                                                           2
+#define regCM1_CM_POST_CSC_B_C31_C32                                                                    0x0ee0
+#define regCM1_CM_POST_CSC_B_C31_C32_BASE_IDX                                                           2
+#define regCM1_CM_POST_CSC_B_C33_C34                                                                    0x0ee1
+#define regCM1_CM_POST_CSC_B_C33_C34_BASE_IDX                                                           2
+#define regCM1_CM_BIAS_CR_R                                                                             0x0ee2
+#define regCM1_CM_BIAS_CR_R_BASE_IDX                                                                    2
+#define regCM1_CM_BIAS_Y_G_CB_B                                                                         0x0ee3
+#define regCM1_CM_BIAS_Y_G_CB_B_BASE_IDX                                                                2
+#define regCM1_CM_GAMCOR_CONTROL                                                                        0x0ee4
+#define regCM1_CM_GAMCOR_CONTROL_BASE_IDX                                                               2
+#define regCM1_CM_GAMCOR_LUT_INDEX                                                                      0x0ee5
+#define regCM1_CM_GAMCOR_LUT_INDEX_BASE_IDX                                                             2
+#define regCM1_CM_GAMCOR_LUT_DATA                                                                       0x0ee6
+#define regCM1_CM_GAMCOR_LUT_DATA_BASE_IDX                                                              2
+#define regCM1_CM_GAMCOR_LUT_CONTROL                                                                    0x0ee7
+#define regCM1_CM_GAMCOR_LUT_CONTROL_BASE_IDX                                                           2
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_B                                                              0x0ee8
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_B_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_G                                                              0x0ee9
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_G_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_R                                                              0x0eea
+#define regCM1_CM_GAMCOR_RAMA_START_CNTL_R_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B                                                        0x0eeb
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G                                                        0x0eec
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R                                                        0x0eed
+#define regCM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_B                                                         0x0eee
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_G                                                         0x0eef
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_R                                                         0x0ef0
+#define regCM1_CM_GAMCOR_RAMA_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_B                                                               0x0ef1
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_B                                                               0x0ef2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_G                                                               0x0ef3
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_G                                                               0x0ef4
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_R                                                               0x0ef5
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_R                                                               0x0ef6
+#define regCM1_CM_GAMCOR_RAMA_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_B                                                                  0x0ef7
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_B_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_G                                                                  0x0ef8
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_G_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_R                                                                  0x0ef9
+#define regCM1_CM_GAMCOR_RAMA_OFFSET_R_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMA_REGION_0_1                                                                0x0efa
+#define regCM1_CM_GAMCOR_RAMA_REGION_0_1_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMA_REGION_2_3                                                                0x0efb
+#define regCM1_CM_GAMCOR_RAMA_REGION_2_3_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMA_REGION_4_5                                                                0x0efc
+#define regCM1_CM_GAMCOR_RAMA_REGION_4_5_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMA_REGION_6_7                                                                0x0efd
+#define regCM1_CM_GAMCOR_RAMA_REGION_6_7_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMA_REGION_8_9                                                                0x0efe
+#define regCM1_CM_GAMCOR_RAMA_REGION_8_9_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMA_REGION_10_11                                                              0x0eff
+#define regCM1_CM_GAMCOR_RAMA_REGION_10_11_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_12_13                                                              0x0f00
+#define regCM1_CM_GAMCOR_RAMA_REGION_12_13_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_14_15                                                              0x0f01
+#define regCM1_CM_GAMCOR_RAMA_REGION_14_15_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_16_17                                                              0x0f02
+#define regCM1_CM_GAMCOR_RAMA_REGION_16_17_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_18_19                                                              0x0f03
+#define regCM1_CM_GAMCOR_RAMA_REGION_18_19_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_20_21                                                              0x0f04
+#define regCM1_CM_GAMCOR_RAMA_REGION_20_21_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_22_23                                                              0x0f05
+#define regCM1_CM_GAMCOR_RAMA_REGION_22_23_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_24_25                                                              0x0f06
+#define regCM1_CM_GAMCOR_RAMA_REGION_24_25_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_26_27                                                              0x0f07
+#define regCM1_CM_GAMCOR_RAMA_REGION_26_27_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_28_29                                                              0x0f08
+#define regCM1_CM_GAMCOR_RAMA_REGION_28_29_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_30_31                                                              0x0f09
+#define regCM1_CM_GAMCOR_RAMA_REGION_30_31_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMA_REGION_32_33                                                              0x0f0a
+#define regCM1_CM_GAMCOR_RAMA_REGION_32_33_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_B                                                              0x0f0b
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_B_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_G                                                              0x0f0c
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_G_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_R                                                              0x0f0d
+#define regCM1_CM_GAMCOR_RAMB_START_CNTL_R_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B                                                        0x0f0e
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G                                                        0x0f0f
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R                                                        0x0f10
+#define regCM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_B                                                         0x0f11
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_G                                                         0x0f12
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_R                                                         0x0f13
+#define regCM1_CM_GAMCOR_RAMB_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_B                                                               0x0f14
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_B                                                               0x0f15
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_G                                                               0x0f16
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_G                                                               0x0f17
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_R                                                               0x0f18
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_R                                                               0x0f19
+#define regCM1_CM_GAMCOR_RAMB_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_B                                                                  0x0f1a
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_B_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_G                                                                  0x0f1b
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_G_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_R                                                                  0x0f1c
+#define regCM1_CM_GAMCOR_RAMB_OFFSET_R_BASE_IDX                                                         2
+#define regCM1_CM_GAMCOR_RAMB_REGION_0_1                                                                0x0f1d
+#define regCM1_CM_GAMCOR_RAMB_REGION_0_1_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMB_REGION_2_3                                                                0x0f1e
+#define regCM1_CM_GAMCOR_RAMB_REGION_2_3_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMB_REGION_4_5                                                                0x0f1f
+#define regCM1_CM_GAMCOR_RAMB_REGION_4_5_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMB_REGION_6_7                                                                0x0f20
+#define regCM1_CM_GAMCOR_RAMB_REGION_6_7_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMB_REGION_8_9                                                                0x0f21
+#define regCM1_CM_GAMCOR_RAMB_REGION_8_9_BASE_IDX                                                       2
+#define regCM1_CM_GAMCOR_RAMB_REGION_10_11                                                              0x0f22
+#define regCM1_CM_GAMCOR_RAMB_REGION_10_11_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_12_13                                                              0x0f23
+#define regCM1_CM_GAMCOR_RAMB_REGION_12_13_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_14_15                                                              0x0f24
+#define regCM1_CM_GAMCOR_RAMB_REGION_14_15_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_16_17                                                              0x0f25
+#define regCM1_CM_GAMCOR_RAMB_REGION_16_17_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_18_19                                                              0x0f26
+#define regCM1_CM_GAMCOR_RAMB_REGION_18_19_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_20_21                                                              0x0f27
+#define regCM1_CM_GAMCOR_RAMB_REGION_20_21_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_22_23                                                              0x0f28
+#define regCM1_CM_GAMCOR_RAMB_REGION_22_23_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_24_25                                                              0x0f29
+#define regCM1_CM_GAMCOR_RAMB_REGION_24_25_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_26_27                                                              0x0f2a
+#define regCM1_CM_GAMCOR_RAMB_REGION_26_27_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_28_29                                                              0x0f2b
+#define regCM1_CM_GAMCOR_RAMB_REGION_28_29_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_30_31                                                              0x0f2c
+#define regCM1_CM_GAMCOR_RAMB_REGION_30_31_BASE_IDX                                                     2
+#define regCM1_CM_GAMCOR_RAMB_REGION_32_33                                                              0x0f2d
+#define regCM1_CM_GAMCOR_RAMB_REGION_32_33_BASE_IDX                                                     2
+#define regCM1_CM_HDR_MULT_COEF                                                                         0x0f2e
+#define regCM1_CM_HDR_MULT_COEF_BASE_IDX                                                                2
+#define regCM1_CM_MEM_PWR_CTRL                                                                          0x0f2f
+#define regCM1_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
+#define regCM1_CM_MEM_PWR_STATUS                                                                        0x0f30
+#define regCM1_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
+#define regCM1_CM_DEALPHA                                                                               0x0f32
+#define regCM1_CM_DEALPHA_BASE_IDX                                                                      2
+#define regCM1_CM_COEF_FORMAT                                                                           0x0f33
+#define regCM1_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM1_CM_TEST_DEBUG_INDEX                                                                      0x0f34
+#define regCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM1_CM_TEST_DEBUG_DATA                                                                       0x0f35
+#define regCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
+#define regCM1_CM_HIST_CNTL                                                                             0x0f36
+#define regCM1_CM_HIST_CNTL_BASE_IDX                                                                    2
+#define regCM1_CM_HIST_SCALE_SRC1                                                                       0x0f37
+#define regCM1_CM_HIST_SCALE_SRC1_BASE_IDX                                                              2
+#define regCM1_CM_HIST_COEFA_SRC2                                                                       0x0f38
+#define regCM1_CM_HIST_COEFA_SRC2_BASE_IDX                                                              2
+#define regCM1_CM_HIST_COEFB_SRC2                                                                       0x0f39
+#define regCM1_CM_HIST_COEFB_SRC2_BASE_IDX                                                              2
+#define regCM1_CM_HIST_COEFC_SRC2                                                                       0x0f3a
+#define regCM1_CM_HIST_COEFC_SRC2_BASE_IDX                                                              2
+#define regCM1_CM_HIST_SCALE_SRC3                                                                       0x0f3b
+#define regCM1_CM_HIST_SCALE_SRC3_BASE_IDX                                                              2
+#define regCM1_CM_HIST_BIAS_SRC1                                                                        0x0f3c
+#define regCM1_CM_HIST_BIAS_SRC1_BASE_IDX                                                               2
+#define regCM1_CM_HIST_BIAS_SRC2                                                                        0x0f3d
+#define regCM1_CM_HIST_BIAS_SRC2_BASE_IDX                                                               2
+#define regCM1_CM_HIST_BIAS_SRC3                                                                        0x0f3e
+#define regCM1_CM_HIST_BIAS_SRC3_BASE_IDX                                                               2
+#define regCM1_CM_HIST_LOCK                                                                             0x0f3f
+#define regCM1_CM_HIST_LOCK_BASE_IDX                                                                    2
+#define regCM1_CM_HIST_INDEX                                                                            0x0f40
+#define regCM1_CM_HIST_INDEX_BASE_IDX                                                                   2
+#define regCM1_CM_HIST_DATA                                                                             0x0f41
+#define regCM1_CM_HIST_DATA_BASE_IDX                                                                    2
+#define regCM1_CM_HIST_STATUS                                                                           0x0f42
+#define regCM1_CM_HIST_STATUS_BASE_IDX                                                                  2
+#define regCM1_CM_HIST_INT_CONTROL                                                                      0x0f43
+#define regCM1_CM_HIST_INT_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x3e3c
+#define regDC_PERFMON11_PERFCOUNTER_CNTL                                                                0x0f8f
+#define regDC_PERFMON11_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON11_PERFCOUNTER_CNTL2                                                               0x0f90
+#define regDC_PERFMON11_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON11_PERFCOUNTER_STATE                                                               0x0f91
+#define regDC_PERFMON11_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON11_PERFMON_CNTL                                                                    0x0f92
+#define regDC_PERFMON11_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON11_PERFMON_CNTL2                                                                   0x0f93
+#define regDC_PERFMON11_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON11_PERFMON_CVALUE_INT_MISC                                                         0x0f94
+#define regDC_PERFMON11_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON11_PERFMON_CVALUE_LOW                                                              0x0f95
+#define regDC_PERFMON11_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON11_PERFMON_HI                                                                      0x0f96
+#define regDC_PERFMON11_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON11_PERFMON_LOW                                                                     0x0f97
+#define regDC_PERFMON11_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_top_dispdec
+// base address: 0xb58
+#define regDPP_TOP2_DPP_CONTROL                                                                         0x0f9b
+#define regDPP_TOP2_DPP_CONTROL_BASE_IDX                                                                2
+#define regDPP_TOP2_DPP_SOFT_RESET                                                                      0x0f9c
+#define regDPP_TOP2_DPP_SOFT_RESET_BASE_IDX                                                             2
+#define regDPP_TOP2_DPP_CRC_VAL_R                                                                       0x0f9d
+#define regDPP_TOP2_DPP_CRC_VAL_R_BASE_IDX                                                              2
+#define regDPP_TOP2_DPP_CRC_VAL_G                                                                       0x0f9e
+#define regDPP_TOP2_DPP_CRC_VAL_G_BASE_IDX                                                              2
+#define regDPP_TOP2_DPP_CRC_VAL_B                                                                       0x0f9f
+#define regDPP_TOP2_DPP_CRC_VAL_B_BASE_IDX                                                              2
+#define regDPP_TOP2_DPP_CRC_VAL_A                                                                       0x0fa0
+#define regDPP_TOP2_DPP_CRC_VAL_A_BASE_IDX                                                              2
+#define regDPP_TOP2_DPP_CRC_CTRL                                                                        0x0fa1
+#define regDPP_TOP2_DPP_CRC_CTRL_BASE_IDX                                                               2
+#define regDPP_TOP2_HOST_READ_CONTROL                                                                   0x0fa2
+#define regDPP_TOP2_HOST_READ_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cnvc_cfg_dispdec
+// base address: 0xb58
+#define regCNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT                                                          0x0fa7
+#define regCNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX                                                 2
+#define regCNVC_CFG2_FORMAT_CONTROL                                                                     0x0fa8
+#define regCNVC_CFG2_FORMAT_CONTROL_BASE_IDX                                                            2
+#define regCNVC_CFG2_FCNV_FP_BIAS_R                                                                     0x0fa9
+#define regCNVC_CFG2_FCNV_FP_BIAS_R_BASE_IDX                                                            2
+#define regCNVC_CFG2_FCNV_FP_BIAS_G                                                                     0x0faa
+#define regCNVC_CFG2_FCNV_FP_BIAS_G_BASE_IDX                                                            2
+#define regCNVC_CFG2_FCNV_FP_BIAS_B                                                                     0x0fab
+#define regCNVC_CFG2_FCNV_FP_BIAS_B_BASE_IDX                                                            2
+#define regCNVC_CFG2_FCNV_FP_SCALE_R                                                                    0x0fac
+#define regCNVC_CFG2_FCNV_FP_SCALE_R_BASE_IDX                                                           2
+#define regCNVC_CFG2_FCNV_FP_SCALE_G                                                                    0x0fad
+#define regCNVC_CFG2_FCNV_FP_SCALE_G_BASE_IDX                                                           2
+#define regCNVC_CFG2_FCNV_FP_SCALE_B                                                                    0x0fae
+#define regCNVC_CFG2_FCNV_FP_SCALE_B_BASE_IDX                                                           2
+#define regCNVC_CFG2_COLOR_KEYER_CONTROL                                                                0x0faf
+#define regCNVC_CFG2_COLOR_KEYER_CONTROL_BASE_IDX                                                       2
+#define regCNVC_CFG2_COLOR_KEYER_ALPHA                                                                  0x0fb0
+#define regCNVC_CFG2_COLOR_KEYER_ALPHA_BASE_IDX                                                         2
+#define regCNVC_CFG2_COLOR_KEYER_RED                                                                    0x0fb1
+#define regCNVC_CFG2_COLOR_KEYER_RED_BASE_IDX                                                           2
+#define regCNVC_CFG2_COLOR_KEYER_GREEN                                                                  0x0fb2
+#define regCNVC_CFG2_COLOR_KEYER_GREEN_BASE_IDX                                                         2
+#define regCNVC_CFG2_COLOR_KEYER_BLUE                                                                   0x0fb3
+#define regCNVC_CFG2_COLOR_KEYER_BLUE_BASE_IDX                                                          2
+#define regCNVC_CFG2_ALPHA_2BIT_LUT01                                                                   0x0fb5
+#define regCNVC_CFG2_ALPHA_2BIT_LUT01_BASE_IDX                                                          2
+#define regCNVC_CFG2_ALPHA_2BIT_LUT23                                                                   0x0fb6
+#define regCNVC_CFG2_ALPHA_2BIT_LUT23_BASE_IDX                                                          2
+#define regCNVC_CFG2_PRE_DEALPHA                                                                        0x0fb7
+#define regCNVC_CFG2_PRE_DEALPHA_BASE_IDX                                                               2
+#define regCNVC_CFG2_PRE_CSC_MODE                                                                       0x0fb8
+#define regCNVC_CFG2_PRE_CSC_MODE_BASE_IDX                                                              2
+#define regCNVC_CFG2_PRE_CSC_C11_C12                                                                    0x0fb9
+#define regCNVC_CFG2_PRE_CSC_C11_C12_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_C13_C14                                                                    0x0fba
+#define regCNVC_CFG2_PRE_CSC_C13_C14_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_C21_C22                                                                    0x0fbb
+#define regCNVC_CFG2_PRE_CSC_C21_C22_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_C23_C24                                                                    0x0fbc
+#define regCNVC_CFG2_PRE_CSC_C23_C24_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_C31_C32                                                                    0x0fbd
+#define regCNVC_CFG2_PRE_CSC_C31_C32_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_C33_C34                                                                    0x0fbe
+#define regCNVC_CFG2_PRE_CSC_C33_C34_BASE_IDX                                                           2
+#define regCNVC_CFG2_PRE_CSC_B_C11_C12                                                                  0x0fbf
+#define regCNVC_CFG2_PRE_CSC_B_C11_C12_BASE_IDX                                                         2
+#define regCNVC_CFG2_PRE_CSC_B_C13_C14                                                                  0x0fc0
+#define regCNVC_CFG2_PRE_CSC_B_C13_C14_BASE_IDX                                                         2
+#define regCNVC_CFG2_PRE_CSC_B_C21_C22                                                                  0x0fc1
+#define regCNVC_CFG2_PRE_CSC_B_C21_C22_BASE_IDX                                                         2
+#define regCNVC_CFG2_PRE_CSC_B_C23_C24                                                                  0x0fc2
+#define regCNVC_CFG2_PRE_CSC_B_C23_C24_BASE_IDX                                                         2
+#define regCNVC_CFG2_PRE_CSC_B_C31_C32                                                                  0x0fc3
+#define regCNVC_CFG2_PRE_CSC_B_C31_C32_BASE_IDX                                                         2
+#define regCNVC_CFG2_PRE_CSC_B_C33_C34                                                                  0x0fc4
+#define regCNVC_CFG2_PRE_CSC_B_C33_C34_BASE_IDX                                                         2
+#define regCNVC_CFG2_CNVC_COEF_FORMAT                                                                   0x0fc5
+#define regCNVC_CFG2_CNVC_COEF_FORMAT_BASE_IDX                                                          2
+#define regCNVC_CFG2_PRE_DEGAM                                                                          0x0fc6
+#define regCNVC_CFG2_PRE_DEGAM_BASE_IDX                                                                 2
+#define regCNVC_CFG2_PRE_REALPHA                                                                        0x0fc7
+#define regCNVC_CFG2_PRE_REALPHA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cm_cur_dispdec
+// base address: 0xb58
+#define regCM_CUR2_CURSOR0_CONTROL                                                                      0x0fca
+#define regCM_CUR2_CURSOR0_CONTROL_BASE_IDX                                                             2
+#define regCM_CUR2_CURSOR0_COLOR0                                                                       0x0fcb
+#define regCM_CUR2_CURSOR0_COLOR0_BASE_IDX                                                              2
+#define regCM_CUR2_CURSOR0_COLOR1                                                                       0x0fcc
+#define regCM_CUR2_CURSOR0_COLOR1_BASE_IDX                                                              2
+#define regCM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y                                                            0x0fcd
+#define regCM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y_BASE_IDX                                                   2
+#define regCM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB                                                        0x0fce
+#define regCM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB_BASE_IDX                                               2
+#define regCM_CUR2_CUR0_MATRIX_MODE                                                                     0x0fcf
+#define regCM_CUR2_CUR0_MATRIX_MODE_BASE_IDX                                                            2
+#define regCM_CUR2_CUR0_MATRIX_C11_C12_A                                                                0x0fd0
+#define regCM_CUR2_CUR0_MATRIX_C11_C12_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C13_C14_A                                                                0x0fd1
+#define regCM_CUR2_CUR0_MATRIX_C13_C14_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C21_C22_A                                                                0x0fd2
+#define regCM_CUR2_CUR0_MATRIX_C21_C22_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C23_C24_A                                                                0x0fd3
+#define regCM_CUR2_CUR0_MATRIX_C23_C24_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C31_C32_A                                                                0x0fd4
+#define regCM_CUR2_CUR0_MATRIX_C31_C32_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C33_C34_A                                                                0x0fd5
+#define regCM_CUR2_CUR0_MATRIX_C33_C34_A_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C11_C12_B                                                                0x0fd6
+#define regCM_CUR2_CUR0_MATRIX_C11_C12_B_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C13_C14_B                                                                0x0fd7
+#define regCM_CUR2_CUR0_MATRIX_C13_C14_B_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C21_C22_B                                                                0x0fd8
+#define regCM_CUR2_CUR0_MATRIX_C21_C22_B_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C23_C24_B                                                                0x0fd9
+#define regCM_CUR2_CUR0_MATRIX_C23_C24_B_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C31_C32_B                                                                0x0fda
+#define regCM_CUR2_CUR0_MATRIX_C31_C32_B_BASE_IDX                                                       2
+#define regCM_CUR2_CUR0_MATRIX_C33_C34_B                                                                0x0fdb
+#define regCM_CUR2_CUR0_MATRIX_C33_C34_B_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dscl_dispdec
+// base address: 0xb58
+#define regDSCL2_SCL_COEF_RAM_TAP_SELECT                                                                0x0fde
+#define regDSCL2_SCL_COEF_RAM_TAP_SELECT_BASE_IDX                                                       2
+#define regDSCL2_SCL_COEF_RAM_TAP_DATA                                                                  0x0fdf
+#define regDSCL2_SCL_COEF_RAM_TAP_DATA_BASE_IDX                                                         2
+#define regDSCL2_SCL_MODE                                                                               0x0fe0
+#define regDSCL2_SCL_MODE_BASE_IDX                                                                      2
+#define regDSCL2_SCL_TAP_CONTROL                                                                        0x0fe1
+#define regDSCL2_SCL_TAP_CONTROL_BASE_IDX                                                               2
+#define regDSCL2_DSCL_CONTROL                                                                           0x0fe2
+#define regDSCL2_DSCL_CONTROL_BASE_IDX                                                                  2
+#define regDSCL2_DSCL_2TAP_CONTROL                                                                      0x0fe3
+#define regDSCL2_DSCL_2TAP_CONTROL_BASE_IDX                                                             2
+#define regDSCL2_SCL_MANUAL_REPLICATE_CONTROL                                                           0x0fe4
+#define regDSCL2_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX                                                  2
+#define regDSCL2_SCL_HORZ_FILTER_SCALE_RATIO                                                            0x0fe5
+#define regDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL2_SCL_HORZ_FILTER_INIT                                                                   0x0fe6
+#define regDSCL2_SCL_HORZ_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C                                                          0x0fe7
+#define regDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL2_SCL_HORZ_FILTER_INIT_C                                                                 0x0fe8
+#define regDSCL2_SCL_HORZ_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL2_SCL_VERT_FILTER_SCALE_RATIO                                                            0x0fe9
+#define regDSCL2_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL2_SCL_VERT_FILTER_INIT                                                                   0x0fea
+#define regDSCL2_SCL_VERT_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL2_SCL_VERT_FILTER_INIT_BOT                                                               0x0feb
+#define regDSCL2_SCL_VERT_FILTER_INIT_BOT_BASE_IDX                                                      2
+#define regDSCL2_SCL_VERT_FILTER_SCALE_RATIO_C                                                          0x0fec
+#define regDSCL2_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL2_SCL_VERT_FILTER_INIT_C                                                                 0x0fed
+#define regDSCL2_SCL_VERT_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL2_SCL_VERT_FILTER_INIT_BOT_C                                                             0x0fee
+#define regDSCL2_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX                                                    2
+#define regDSCL2_SCL_BLACK_COLOR                                                                        0x0fef
+#define regDSCL2_SCL_BLACK_COLOR_BASE_IDX                                                               2
+#define regDSCL2_DSCL_UPDATE                                                                            0x0ff0
+#define regDSCL2_DSCL_UPDATE_BASE_IDX                                                                   2
+#define regDSCL2_DSCL_AUTOCAL                                                                           0x0ff1
+#define regDSCL2_DSCL_AUTOCAL_BASE_IDX                                                                  2
+#define regDSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT                                                           0x0ff2
+#define regDSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX                                                  2
+#define regDSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM                                                           0x0ff3
+#define regDSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX                                                  2
+#define regDSCL2_OTG_H_BLANK                                                                            0x0ff4
+#define regDSCL2_OTG_H_BLANK_BASE_IDX                                                                   2
+#define regDSCL2_OTG_V_BLANK                                                                            0x0ff5
+#define regDSCL2_OTG_V_BLANK_BASE_IDX                                                                   2
+#define regDSCL2_RECOUT_START                                                                           0x0ff6
+#define regDSCL2_RECOUT_START_BASE_IDX                                                                  2
+#define regDSCL2_RECOUT_SIZE                                                                            0x0ff7
+#define regDSCL2_RECOUT_SIZE_BASE_IDX                                                                   2
+#define regDSCL2_MPC_SIZE                                                                               0x0ff8
+#define regDSCL2_MPC_SIZE_BASE_IDX                                                                      2
+#define regDSCL2_LB_DATA_FORMAT                                                                         0x0ff9
+#define regDSCL2_LB_DATA_FORMAT_BASE_IDX                                                                2
+#define regDSCL2_LB_MEMORY_CTRL                                                                         0x0ffa
+#define regDSCL2_LB_MEMORY_CTRL_BASE_IDX                                                                2
+#define regDSCL2_LB_V_COUNTER                                                                           0x0ffb
+#define regDSCL2_LB_V_COUNTER_BASE_IDX                                                                  2
+#define regDSCL2_DSCL_MEM_PWR_CTRL                                                                      0x0ffc
+#define regDSCL2_DSCL_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL2_DSCL_MEM_PWR_STATUS                                                                    0x0ffd
+#define regDSCL2_DSCL_MEM_PWR_STATUS_BASE_IDX                                                           2
+#define regDSCL2_OBUF_CONTROL                                                                           0x0ffe
+#define regDSCL2_OBUF_CONTROL_BASE_IDX                                                                  2
+#define regDSCL2_OBUF_MEM_PWR_CTRL                                                                      0x0fff
+#define regDSCL2_OBUF_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL2_DSCL_EASF_H_MODE                                                                       0x1000
+#define regDSCL2_DSCL_EASF_H_MODE_BASE_IDX                                                              2
+#define regDSCL2_DSCL_EASF_V_MODE                                                                       0x1001
+#define regDSCL2_DSCL_EASF_V_MODE_BASE_IDX                                                              2
+#define regDSCL2_DSCL_SC_MODE                                                                           0x1002
+#define regDSCL2_DSCL_SC_MODE_BASE_IDX                                                                  2
+#define regDSCL2_DSCL_SC_MATRIX_C0C1                                                                    0x1003
+#define regDSCL2_DSCL_SC_MATRIX_C0C1_BASE_IDX                                                           2
+#define regDSCL2_DSCL_SC_MATRIX_C2C3                                                                    0x1004
+#define regDSCL2_DSCL_SC_MATRIX_C2C3_BASE_IDX                                                           2
+#define regDSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN                                                       0x1005
+#define regDSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE                                                     0x1006
+#define regDSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN                                                       0x1007
+#define regDSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE                                                     0x1008
+#define regDSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1                                                         0x1009
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1_BASE_IDX                                                2
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2                                                         0x100a
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2_BASE_IDX                                                2
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3                                                         0x100b
+#define regDSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3_BASE_IDX                                                2
+#define regDSCL2_DSCL_EASF_RINGEST_FORCE                                                                0x100c
+#define regDSCL2_DSCL_EASF_RINGEST_FORCE_BASE_IDX                                                       2
+#define regDSCL2_DSCL_EASF_H_BF_CNTL                                                                    0x100d
+#define regDSCL2_DSCL_EASF_H_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN                                                           0x100e
+#define regDSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL2_DSCL_EASF_V_BF_CNTL                                                                    0x100f
+#define regDSCL2_DSCL_EASF_V_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN                                                           0x1010
+#define regDSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG0                                                               0x1011
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG1                                                               0x1012
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG2                                                               0x1013
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG3                                                               0x1014
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG4                                                               0x1015
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG5                                                               0x1016
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG6                                                               0x1017
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG7                                                               0x1018
+#define regDSCL2_DSCL_EASF_H_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG0                                                               0x1019
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG1                                                               0x101a
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG2                                                               0x101b
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG3                                                               0x101c
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG4                                                               0x101d
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG5                                                               0x101e
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG6                                                               0x101f
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG7                                                               0x1020
+#define regDSCL2_DSCL_EASF_V_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG0                                                               0x1021
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG1                                                               0x1022
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG2                                                               0x1023
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG3                                                               0x1024
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG4                                                               0x1025
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG5                                                               0x1026
+#define regDSCL2_DSCL_EASF_H_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG0                                                               0x1027
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG1                                                               0x1028
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG2                                                               0x1029
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG3                                                               0x102a
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG4                                                               0x102b
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG5                                                               0x102c
+#define regDSCL2_DSCL_EASF_V_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL2_ISHARP_MODE                                                                            0x102d
+#define regDSCL2_ISHARP_MODE_BASE_IDX                                                                   2
+#define regDSCL2_ISHARP_DELTA_CTRL                                                                      0x102e
+#define regDSCL2_ISHARP_DELTA_CTRL_BASE_IDX                                                             2
+#define regDSCL2_ISHARP_DELTA_INDEX                                                                     0x102f
+#define regDSCL2_ISHARP_DELTA_INDEX_BASE_IDX                                                            2
+#define regDSCL2_ISHARP_DELTA_DATA                                                                      0x1030
+#define regDSCL2_ISHARP_DELTA_DATA_BASE_IDX                                                             2
+#define regDSCL2_ISHARP_NLDELTA_SOFT_CLIP                                                               0x1031
+#define regDSCL2_ISHARP_NLDELTA_SOFT_CLIP_BASE_IDX                                                      2
+#define regDSCL2_ISHARP_NOISEDET_THRESHOLD                                                              0x1032
+#define regDSCL2_ISHARP_NOISEDET_THRESHOLD_BASE_IDX                                                     2
+#define regDSCL2_ISHARP_NOISE_GAIN_PWL                                                                  0x1033
+#define regDSCL2_ISHARP_NOISE_GAIN_PWL_BASE_IDX                                                         2
+#define regDSCL2_ISHARP_LBA_PWL_SEG0                                                                    0x1034
+#define regDSCL2_ISHARP_LBA_PWL_SEG0_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_LBA_PWL_SEG1                                                                    0x1035
+#define regDSCL2_ISHARP_LBA_PWL_SEG1_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_LBA_PWL_SEG2                                                                    0x1036
+#define regDSCL2_ISHARP_LBA_PWL_SEG2_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_LBA_PWL_SEG3                                                                    0x1037
+#define regDSCL2_ISHARP_LBA_PWL_SEG3_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_LBA_PWL_SEG4                                                                    0x1038
+#define regDSCL2_ISHARP_LBA_PWL_SEG4_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_LBA_PWL_SEG5                                                                    0x1039
+#define regDSCL2_ISHARP_LBA_PWL_SEG5_BASE_IDX                                                           2
+#define regDSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL                                                          0x103a
+#define regDSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cm_dispdec
+// base address: 0xb58
+#define regCM2_CM_CONTROL                                                                               0x103f
+#define regCM2_CM_CONTROL_BASE_IDX                                                                      2
+#define regCM2_CM_POST_CSC_CONTROL                                                                      0x1040
+#define regCM2_CM_POST_CSC_CONTROL_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C11_C12                                                                      0x1041
+#define regCM2_CM_POST_CSC_C11_C12_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C13_C14                                                                      0x1042
+#define regCM2_CM_POST_CSC_C13_C14_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C21_C22                                                                      0x1043
+#define regCM2_CM_POST_CSC_C21_C22_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C23_C24                                                                      0x1044
+#define regCM2_CM_POST_CSC_C23_C24_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C31_C32                                                                      0x1045
+#define regCM2_CM_POST_CSC_C31_C32_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_C33_C34                                                                      0x1046
+#define regCM2_CM_POST_CSC_C33_C34_BASE_IDX                                                             2
+#define regCM2_CM_POST_CSC_B_C11_C12                                                                    0x1047
+#define regCM2_CM_POST_CSC_B_C11_C12_BASE_IDX                                                           2
+#define regCM2_CM_POST_CSC_B_C13_C14                                                                    0x1048
+#define regCM2_CM_POST_CSC_B_C13_C14_BASE_IDX                                                           2
+#define regCM2_CM_POST_CSC_B_C21_C22                                                                    0x1049
+#define regCM2_CM_POST_CSC_B_C21_C22_BASE_IDX                                                           2
+#define regCM2_CM_POST_CSC_B_C23_C24                                                                    0x104a
+#define regCM2_CM_POST_CSC_B_C23_C24_BASE_IDX                                                           2
+#define regCM2_CM_POST_CSC_B_C31_C32                                                                    0x104b
+#define regCM2_CM_POST_CSC_B_C31_C32_BASE_IDX                                                           2
+#define regCM2_CM_POST_CSC_B_C33_C34                                                                    0x104c
+#define regCM2_CM_POST_CSC_B_C33_C34_BASE_IDX                                                           2
+#define regCM2_CM_BIAS_CR_R                                                                             0x104d
+#define regCM2_CM_BIAS_CR_R_BASE_IDX                                                                    2
+#define regCM2_CM_BIAS_Y_G_CB_B                                                                         0x104e
+#define regCM2_CM_BIAS_Y_G_CB_B_BASE_IDX                                                                2
+#define regCM2_CM_GAMCOR_CONTROL                                                                        0x104f
+#define regCM2_CM_GAMCOR_CONTROL_BASE_IDX                                                               2
+#define regCM2_CM_GAMCOR_LUT_INDEX                                                                      0x1050
+#define regCM2_CM_GAMCOR_LUT_INDEX_BASE_IDX                                                             2
+#define regCM2_CM_GAMCOR_LUT_DATA                                                                       0x1051
+#define regCM2_CM_GAMCOR_LUT_DATA_BASE_IDX                                                              2
+#define regCM2_CM_GAMCOR_LUT_CONTROL                                                                    0x1052
+#define regCM2_CM_GAMCOR_LUT_CONTROL_BASE_IDX                                                           2
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_B                                                              0x1053
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_B_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_G                                                              0x1054
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_G_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_R                                                              0x1055
+#define regCM2_CM_GAMCOR_RAMA_START_CNTL_R_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B                                                        0x1056
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G                                                        0x1057
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R                                                        0x1058
+#define regCM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_B                                                         0x1059
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_G                                                         0x105a
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_R                                                         0x105b
+#define regCM2_CM_GAMCOR_RAMA_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_B                                                               0x105c
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_B                                                               0x105d
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_G                                                               0x105e
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_G                                                               0x105f
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_R                                                               0x1060
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_R                                                               0x1061
+#define regCM2_CM_GAMCOR_RAMA_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_B                                                                  0x1062
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_B_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_G                                                                  0x1063
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_G_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_R                                                                  0x1064
+#define regCM2_CM_GAMCOR_RAMA_OFFSET_R_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMA_REGION_0_1                                                                0x1065
+#define regCM2_CM_GAMCOR_RAMA_REGION_0_1_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMA_REGION_2_3                                                                0x1066
+#define regCM2_CM_GAMCOR_RAMA_REGION_2_3_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMA_REGION_4_5                                                                0x1067
+#define regCM2_CM_GAMCOR_RAMA_REGION_4_5_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMA_REGION_6_7                                                                0x1068
+#define regCM2_CM_GAMCOR_RAMA_REGION_6_7_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMA_REGION_8_9                                                                0x1069
+#define regCM2_CM_GAMCOR_RAMA_REGION_8_9_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMA_REGION_10_11                                                              0x106a
+#define regCM2_CM_GAMCOR_RAMA_REGION_10_11_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_12_13                                                              0x106b
+#define regCM2_CM_GAMCOR_RAMA_REGION_12_13_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_14_15                                                              0x106c
+#define regCM2_CM_GAMCOR_RAMA_REGION_14_15_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_16_17                                                              0x106d
+#define regCM2_CM_GAMCOR_RAMA_REGION_16_17_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_18_19                                                              0x106e
+#define regCM2_CM_GAMCOR_RAMA_REGION_18_19_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_20_21                                                              0x106f
+#define regCM2_CM_GAMCOR_RAMA_REGION_20_21_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_22_23                                                              0x1070
+#define regCM2_CM_GAMCOR_RAMA_REGION_22_23_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_24_25                                                              0x1071
+#define regCM2_CM_GAMCOR_RAMA_REGION_24_25_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_26_27                                                              0x1072
+#define regCM2_CM_GAMCOR_RAMA_REGION_26_27_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_28_29                                                              0x1073
+#define regCM2_CM_GAMCOR_RAMA_REGION_28_29_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_30_31                                                              0x1074
+#define regCM2_CM_GAMCOR_RAMA_REGION_30_31_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMA_REGION_32_33                                                              0x1075
+#define regCM2_CM_GAMCOR_RAMA_REGION_32_33_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_B                                                              0x1076
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_B_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_G                                                              0x1077
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_G_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_R                                                              0x1078
+#define regCM2_CM_GAMCOR_RAMB_START_CNTL_R_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B                                                        0x1079
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G                                                        0x107a
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R                                                        0x107b
+#define regCM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_B                                                         0x107c
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_G                                                         0x107d
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_R                                                         0x107e
+#define regCM2_CM_GAMCOR_RAMB_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_B                                                               0x107f
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_B                                                               0x1080
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_G                                                               0x1081
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_G                                                               0x1082
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_R                                                               0x1083
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_R                                                               0x1084
+#define regCM2_CM_GAMCOR_RAMB_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_B                                                                  0x1085
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_B_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_G                                                                  0x1086
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_G_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_R                                                                  0x1087
+#define regCM2_CM_GAMCOR_RAMB_OFFSET_R_BASE_IDX                                                         2
+#define regCM2_CM_GAMCOR_RAMB_REGION_0_1                                                                0x1088
+#define regCM2_CM_GAMCOR_RAMB_REGION_0_1_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMB_REGION_2_3                                                                0x1089
+#define regCM2_CM_GAMCOR_RAMB_REGION_2_3_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMB_REGION_4_5                                                                0x108a
+#define regCM2_CM_GAMCOR_RAMB_REGION_4_5_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMB_REGION_6_7                                                                0x108b
+#define regCM2_CM_GAMCOR_RAMB_REGION_6_7_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMB_REGION_8_9                                                                0x108c
+#define regCM2_CM_GAMCOR_RAMB_REGION_8_9_BASE_IDX                                                       2
+#define regCM2_CM_GAMCOR_RAMB_REGION_10_11                                                              0x108d
+#define regCM2_CM_GAMCOR_RAMB_REGION_10_11_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_12_13                                                              0x108e
+#define regCM2_CM_GAMCOR_RAMB_REGION_12_13_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_14_15                                                              0x108f
+#define regCM2_CM_GAMCOR_RAMB_REGION_14_15_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_16_17                                                              0x1090
+#define regCM2_CM_GAMCOR_RAMB_REGION_16_17_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_18_19                                                              0x1091
+#define regCM2_CM_GAMCOR_RAMB_REGION_18_19_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_20_21                                                              0x1092
+#define regCM2_CM_GAMCOR_RAMB_REGION_20_21_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_22_23                                                              0x1093
+#define regCM2_CM_GAMCOR_RAMB_REGION_22_23_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_24_25                                                              0x1094
+#define regCM2_CM_GAMCOR_RAMB_REGION_24_25_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_26_27                                                              0x1095
+#define regCM2_CM_GAMCOR_RAMB_REGION_26_27_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_28_29                                                              0x1096
+#define regCM2_CM_GAMCOR_RAMB_REGION_28_29_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_30_31                                                              0x1097
+#define regCM2_CM_GAMCOR_RAMB_REGION_30_31_BASE_IDX                                                     2
+#define regCM2_CM_GAMCOR_RAMB_REGION_32_33                                                              0x1098
+#define regCM2_CM_GAMCOR_RAMB_REGION_32_33_BASE_IDX                                                     2
+#define regCM2_CM_HDR_MULT_COEF                                                                         0x1099
+#define regCM2_CM_HDR_MULT_COEF_BASE_IDX                                                                2
+#define regCM2_CM_MEM_PWR_CTRL                                                                          0x109a
+#define regCM2_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
+#define regCM2_CM_MEM_PWR_STATUS                                                                        0x109b
+#define regCM2_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
+#define regCM2_CM_DEALPHA                                                                               0x109d
+#define regCM2_CM_DEALPHA_BASE_IDX                                                                      2
+#define regCM2_CM_COEF_FORMAT                                                                           0x109e
+#define regCM2_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM2_CM_TEST_DEBUG_INDEX                                                                      0x109f
+#define regCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM2_CM_TEST_DEBUG_DATA                                                                       0x10a0
+#define regCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
+#define regCM2_CM_HIST_CNTL                                                                             0x10a1
+#define regCM2_CM_HIST_CNTL_BASE_IDX                                                                    2
+#define regCM2_CM_HIST_SCALE_SRC1                                                                       0x10a2
+#define regCM2_CM_HIST_SCALE_SRC1_BASE_IDX                                                              2
+#define regCM2_CM_HIST_COEFA_SRC2                                                                       0x10a3
+#define regCM2_CM_HIST_COEFA_SRC2_BASE_IDX                                                              2
+#define regCM2_CM_HIST_COEFB_SRC2                                                                       0x10a4
+#define regCM2_CM_HIST_COEFB_SRC2_BASE_IDX                                                              2
+#define regCM2_CM_HIST_COEFC_SRC2                                                                       0x10a5
+#define regCM2_CM_HIST_COEFC_SRC2_BASE_IDX                                                              2
+#define regCM2_CM_HIST_SCALE_SRC3                                                                       0x10a6
+#define regCM2_CM_HIST_SCALE_SRC3_BASE_IDX                                                              2
+#define regCM2_CM_HIST_BIAS_SRC1                                                                        0x10a7
+#define regCM2_CM_HIST_BIAS_SRC1_BASE_IDX                                                               2
+#define regCM2_CM_HIST_BIAS_SRC2                                                                        0x10a8
+#define regCM2_CM_HIST_BIAS_SRC2_BASE_IDX                                                               2
+#define regCM2_CM_HIST_BIAS_SRC3                                                                        0x10a9
+#define regCM2_CM_HIST_BIAS_SRC3_BASE_IDX                                                               2
+#define regCM2_CM_HIST_LOCK                                                                             0x10aa
+#define regCM2_CM_HIST_LOCK_BASE_IDX                                                                    2
+#define regCM2_CM_HIST_INDEX                                                                            0x10ab
+#define regCM2_CM_HIST_INDEX_BASE_IDX                                                                   2
+#define regCM2_CM_HIST_DATA                                                                             0x10ac
+#define regCM2_CM_HIST_DATA_BASE_IDX                                                                    2
+#define regCM2_CM_HIST_STATUS                                                                           0x10ad
+#define regCM2_CM_HIST_STATUS_BASE_IDX                                                                  2
+#define regCM2_CM_HIST_INT_CONTROL                                                                      0x10ae
+#define regCM2_CM_HIST_INT_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x43e8
+#define regDC_PERFMON12_PERFCOUNTER_CNTL                                                                0x10fa
+#define regDC_PERFMON12_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON12_PERFCOUNTER_CNTL2                                                               0x10fb
+#define regDC_PERFMON12_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON12_PERFCOUNTER_STATE                                                               0x10fc
+#define regDC_PERFMON12_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON12_PERFMON_CNTL                                                                    0x10fd
+#define regDC_PERFMON12_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON12_PERFMON_CNTL2                                                                   0x10fe
+#define regDC_PERFMON12_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON12_PERFMON_CVALUE_INT_MISC                                                         0x10ff
+#define regDC_PERFMON12_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON12_PERFMON_CVALUE_LOW                                                              0x1100
+#define regDC_PERFMON12_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON12_PERFMON_HI                                                                      0x1101
+#define regDC_PERFMON12_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON12_PERFMON_LOW                                                                     0x1102
+#define regDC_PERFMON12_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_top_dispdec
+// base address: 0x1104
+#define regDPP_TOP3_DPP_CONTROL                                                                         0x1106
+#define regDPP_TOP3_DPP_CONTROL_BASE_IDX                                                                2
+#define regDPP_TOP3_DPP_SOFT_RESET                                                                      0x1107
+#define regDPP_TOP3_DPP_SOFT_RESET_BASE_IDX                                                             2
+#define regDPP_TOP3_DPP_CRC_VAL_R                                                                       0x1108
+#define regDPP_TOP3_DPP_CRC_VAL_R_BASE_IDX                                                              2
+#define regDPP_TOP3_DPP_CRC_VAL_G                                                                       0x1109
+#define regDPP_TOP3_DPP_CRC_VAL_G_BASE_IDX                                                              2
+#define regDPP_TOP3_DPP_CRC_VAL_B                                                                       0x110a
+#define regDPP_TOP3_DPP_CRC_VAL_B_BASE_IDX                                                              2
+#define regDPP_TOP3_DPP_CRC_VAL_A                                                                       0x110b
+#define regDPP_TOP3_DPP_CRC_VAL_A_BASE_IDX                                                              2
+#define regDPP_TOP3_DPP_CRC_CTRL                                                                        0x110c
+#define regDPP_TOP3_DPP_CRC_CTRL_BASE_IDX                                                               2
+#define regDPP_TOP3_HOST_READ_CONTROL                                                                   0x110d
+#define regDPP_TOP3_HOST_READ_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cnvc_cfg_dispdec
+// base address: 0x1104
+#define regCNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT                                                          0x1112
+#define regCNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX                                                 2
+#define regCNVC_CFG3_FORMAT_CONTROL                                                                     0x1113
+#define regCNVC_CFG3_FORMAT_CONTROL_BASE_IDX                                                            2
+#define regCNVC_CFG3_FCNV_FP_BIAS_R                                                                     0x1114
+#define regCNVC_CFG3_FCNV_FP_BIAS_R_BASE_IDX                                                            2
+#define regCNVC_CFG3_FCNV_FP_BIAS_G                                                                     0x1115
+#define regCNVC_CFG3_FCNV_FP_BIAS_G_BASE_IDX                                                            2
+#define regCNVC_CFG3_FCNV_FP_BIAS_B                                                                     0x1116
+#define regCNVC_CFG3_FCNV_FP_BIAS_B_BASE_IDX                                                            2
+#define regCNVC_CFG3_FCNV_FP_SCALE_R                                                                    0x1117
+#define regCNVC_CFG3_FCNV_FP_SCALE_R_BASE_IDX                                                           2
+#define regCNVC_CFG3_FCNV_FP_SCALE_G                                                                    0x1118
+#define regCNVC_CFG3_FCNV_FP_SCALE_G_BASE_IDX                                                           2
+#define regCNVC_CFG3_FCNV_FP_SCALE_B                                                                    0x1119
+#define regCNVC_CFG3_FCNV_FP_SCALE_B_BASE_IDX                                                           2
+#define regCNVC_CFG3_COLOR_KEYER_CONTROL                                                                0x111a
+#define regCNVC_CFG3_COLOR_KEYER_CONTROL_BASE_IDX                                                       2
+#define regCNVC_CFG3_COLOR_KEYER_ALPHA                                                                  0x111b
+#define regCNVC_CFG3_COLOR_KEYER_ALPHA_BASE_IDX                                                         2
+#define regCNVC_CFG3_COLOR_KEYER_RED                                                                    0x111c
+#define regCNVC_CFG3_COLOR_KEYER_RED_BASE_IDX                                                           2
+#define regCNVC_CFG3_COLOR_KEYER_GREEN                                                                  0x111d
+#define regCNVC_CFG3_COLOR_KEYER_GREEN_BASE_IDX                                                         2
+#define regCNVC_CFG3_COLOR_KEYER_BLUE                                                                   0x111e
+#define regCNVC_CFG3_COLOR_KEYER_BLUE_BASE_IDX                                                          2
+#define regCNVC_CFG3_ALPHA_2BIT_LUT01                                                                   0x1120
+#define regCNVC_CFG3_ALPHA_2BIT_LUT01_BASE_IDX                                                          2
+#define regCNVC_CFG3_ALPHA_2BIT_LUT23                                                                   0x1121
+#define regCNVC_CFG3_ALPHA_2BIT_LUT23_BASE_IDX                                                          2
+#define regCNVC_CFG3_PRE_DEALPHA                                                                        0x1122
+#define regCNVC_CFG3_PRE_DEALPHA_BASE_IDX                                                               2
+#define regCNVC_CFG3_PRE_CSC_MODE                                                                       0x1123
+#define regCNVC_CFG3_PRE_CSC_MODE_BASE_IDX                                                              2
+#define regCNVC_CFG3_PRE_CSC_C11_C12                                                                    0x1124
+#define regCNVC_CFG3_PRE_CSC_C11_C12_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_C13_C14                                                                    0x1125
+#define regCNVC_CFG3_PRE_CSC_C13_C14_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_C21_C22                                                                    0x1126
+#define regCNVC_CFG3_PRE_CSC_C21_C22_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_C23_C24                                                                    0x1127
+#define regCNVC_CFG3_PRE_CSC_C23_C24_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_C31_C32                                                                    0x1128
+#define regCNVC_CFG3_PRE_CSC_C31_C32_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_C33_C34                                                                    0x1129
+#define regCNVC_CFG3_PRE_CSC_C33_C34_BASE_IDX                                                           2
+#define regCNVC_CFG3_PRE_CSC_B_C11_C12                                                                  0x112a
+#define regCNVC_CFG3_PRE_CSC_B_C11_C12_BASE_IDX                                                         2
+#define regCNVC_CFG3_PRE_CSC_B_C13_C14                                                                  0x112b
+#define regCNVC_CFG3_PRE_CSC_B_C13_C14_BASE_IDX                                                         2
+#define regCNVC_CFG3_PRE_CSC_B_C21_C22                                                                  0x112c
+#define regCNVC_CFG3_PRE_CSC_B_C21_C22_BASE_IDX                                                         2
+#define regCNVC_CFG3_PRE_CSC_B_C23_C24                                                                  0x112d
+#define regCNVC_CFG3_PRE_CSC_B_C23_C24_BASE_IDX                                                         2
+#define regCNVC_CFG3_PRE_CSC_B_C31_C32                                                                  0x112e
+#define regCNVC_CFG3_PRE_CSC_B_C31_C32_BASE_IDX                                                         2
+#define regCNVC_CFG3_PRE_CSC_B_C33_C34                                                                  0x112f
+#define regCNVC_CFG3_PRE_CSC_B_C33_C34_BASE_IDX                                                         2
+#define regCNVC_CFG3_CNVC_COEF_FORMAT                                                                   0x1130
+#define regCNVC_CFG3_CNVC_COEF_FORMAT_BASE_IDX                                                          2
+#define regCNVC_CFG3_PRE_DEGAM                                                                          0x1131
+#define regCNVC_CFG3_PRE_DEGAM_BASE_IDX                                                                 2
+#define regCNVC_CFG3_PRE_REALPHA                                                                        0x1132
+#define regCNVC_CFG3_PRE_REALPHA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cm_cur_dispdec
+// base address: 0x1104
+#define regCM_CUR3_CURSOR0_CONTROL                                                                      0x1135
+#define regCM_CUR3_CURSOR0_CONTROL_BASE_IDX                                                             2
+#define regCM_CUR3_CURSOR0_COLOR0                                                                       0x1136
+#define regCM_CUR3_CURSOR0_COLOR0_BASE_IDX                                                              2
+#define regCM_CUR3_CURSOR0_COLOR1                                                                       0x1137
+#define regCM_CUR3_CURSOR0_COLOR1_BASE_IDX                                                              2
+#define regCM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y                                                            0x1138
+#define regCM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y_BASE_IDX                                                   2
+#define regCM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB                                                        0x1139
+#define regCM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB_BASE_IDX                                               2
+#define regCM_CUR3_CUR0_MATRIX_MODE                                                                     0x113a
+#define regCM_CUR3_CUR0_MATRIX_MODE_BASE_IDX                                                            2
+#define regCM_CUR3_CUR0_MATRIX_C11_C12_A                                                                0x113b
+#define regCM_CUR3_CUR0_MATRIX_C11_C12_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C13_C14_A                                                                0x113c
+#define regCM_CUR3_CUR0_MATRIX_C13_C14_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C21_C22_A                                                                0x113d
+#define regCM_CUR3_CUR0_MATRIX_C21_C22_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C23_C24_A                                                                0x113e
+#define regCM_CUR3_CUR0_MATRIX_C23_C24_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C31_C32_A                                                                0x113f
+#define regCM_CUR3_CUR0_MATRIX_C31_C32_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C33_C34_A                                                                0x1140
+#define regCM_CUR3_CUR0_MATRIX_C33_C34_A_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C11_C12_B                                                                0x1141
+#define regCM_CUR3_CUR0_MATRIX_C11_C12_B_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C13_C14_B                                                                0x1142
+#define regCM_CUR3_CUR0_MATRIX_C13_C14_B_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C21_C22_B                                                                0x1143
+#define regCM_CUR3_CUR0_MATRIX_C21_C22_B_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C23_C24_B                                                                0x1144
+#define regCM_CUR3_CUR0_MATRIX_C23_C24_B_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C31_C32_B                                                                0x1145
+#define regCM_CUR3_CUR0_MATRIX_C31_C32_B_BASE_IDX                                                       2
+#define regCM_CUR3_CUR0_MATRIX_C33_C34_B                                                                0x1146
+#define regCM_CUR3_CUR0_MATRIX_C33_C34_B_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dscl_dispdec
+// base address: 0x1104
+#define regDSCL3_SCL_COEF_RAM_TAP_SELECT                                                                0x1149
+#define regDSCL3_SCL_COEF_RAM_TAP_SELECT_BASE_IDX                                                       2
+#define regDSCL3_SCL_COEF_RAM_TAP_DATA                                                                  0x114a
+#define regDSCL3_SCL_COEF_RAM_TAP_DATA_BASE_IDX                                                         2
+#define regDSCL3_SCL_MODE                                                                               0x114b
+#define regDSCL3_SCL_MODE_BASE_IDX                                                                      2
+#define regDSCL3_SCL_TAP_CONTROL                                                                        0x114c
+#define regDSCL3_SCL_TAP_CONTROL_BASE_IDX                                                               2
+#define regDSCL3_DSCL_CONTROL                                                                           0x114d
+#define regDSCL3_DSCL_CONTROL_BASE_IDX                                                                  2
+#define regDSCL3_DSCL_2TAP_CONTROL                                                                      0x114e
+#define regDSCL3_DSCL_2TAP_CONTROL_BASE_IDX                                                             2
+#define regDSCL3_SCL_MANUAL_REPLICATE_CONTROL                                                           0x114f
+#define regDSCL3_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX                                                  2
+#define regDSCL3_SCL_HORZ_FILTER_SCALE_RATIO                                                            0x1150
+#define regDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL3_SCL_HORZ_FILTER_INIT                                                                   0x1151
+#define regDSCL3_SCL_HORZ_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C                                                          0x1152
+#define regDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL3_SCL_HORZ_FILTER_INIT_C                                                                 0x1153
+#define regDSCL3_SCL_HORZ_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL3_SCL_VERT_FILTER_SCALE_RATIO                                                            0x1154
+#define regDSCL3_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX                                                   2
+#define regDSCL3_SCL_VERT_FILTER_INIT                                                                   0x1155
+#define regDSCL3_SCL_VERT_FILTER_INIT_BASE_IDX                                                          2
+#define regDSCL3_SCL_VERT_FILTER_INIT_BOT                                                               0x1156
+#define regDSCL3_SCL_VERT_FILTER_INIT_BOT_BASE_IDX                                                      2
+#define regDSCL3_SCL_VERT_FILTER_SCALE_RATIO_C                                                          0x1157
+#define regDSCL3_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX                                                 2
+#define regDSCL3_SCL_VERT_FILTER_INIT_C                                                                 0x1158
+#define regDSCL3_SCL_VERT_FILTER_INIT_C_BASE_IDX                                                        2
+#define regDSCL3_SCL_VERT_FILTER_INIT_BOT_C                                                             0x1159
+#define regDSCL3_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX                                                    2
+#define regDSCL3_SCL_BLACK_COLOR                                                                        0x115a
+#define regDSCL3_SCL_BLACK_COLOR_BASE_IDX                                                               2
+#define regDSCL3_DSCL_UPDATE                                                                            0x115b
+#define regDSCL3_DSCL_UPDATE_BASE_IDX                                                                   2
+#define regDSCL3_DSCL_AUTOCAL                                                                           0x115c
+#define regDSCL3_DSCL_AUTOCAL_BASE_IDX                                                                  2
+#define regDSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT                                                           0x115d
+#define regDSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX                                                  2
+#define regDSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM                                                           0x115e
+#define regDSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX                                                  2
+#define regDSCL3_OTG_H_BLANK                                                                            0x115f
+#define regDSCL3_OTG_H_BLANK_BASE_IDX                                                                   2
+#define regDSCL3_OTG_V_BLANK                                                                            0x1160
+#define regDSCL3_OTG_V_BLANK_BASE_IDX                                                                   2
+#define regDSCL3_RECOUT_START                                                                           0x1161
+#define regDSCL3_RECOUT_START_BASE_IDX                                                                  2
+#define regDSCL3_RECOUT_SIZE                                                                            0x1162
+#define regDSCL3_RECOUT_SIZE_BASE_IDX                                                                   2
+#define regDSCL3_MPC_SIZE                                                                               0x1163
+#define regDSCL3_MPC_SIZE_BASE_IDX                                                                      2
+#define regDSCL3_LB_DATA_FORMAT                                                                         0x1164
+#define regDSCL3_LB_DATA_FORMAT_BASE_IDX                                                                2
+#define regDSCL3_LB_MEMORY_CTRL                                                                         0x1165
+#define regDSCL3_LB_MEMORY_CTRL_BASE_IDX                                                                2
+#define regDSCL3_LB_V_COUNTER                                                                           0x1166
+#define regDSCL3_LB_V_COUNTER_BASE_IDX                                                                  2
+#define regDSCL3_DSCL_MEM_PWR_CTRL                                                                      0x1167
+#define regDSCL3_DSCL_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL3_DSCL_MEM_PWR_STATUS                                                                    0x1168
+#define regDSCL3_DSCL_MEM_PWR_STATUS_BASE_IDX                                                           2
+#define regDSCL3_OBUF_CONTROL                                                                           0x1169
+#define regDSCL3_OBUF_CONTROL_BASE_IDX                                                                  2
+#define regDSCL3_OBUF_MEM_PWR_CTRL                                                                      0x116a
+#define regDSCL3_OBUF_MEM_PWR_CTRL_BASE_IDX                                                             2
+#define regDSCL3_DSCL_EASF_H_MODE                                                                       0x116b
+#define regDSCL3_DSCL_EASF_H_MODE_BASE_IDX                                                              2
+#define regDSCL3_DSCL_EASF_V_MODE                                                                       0x116c
+#define regDSCL3_DSCL_EASF_V_MODE_BASE_IDX                                                              2
+#define regDSCL3_DSCL_SC_MODE                                                                           0x116d
+#define regDSCL3_DSCL_SC_MODE_BASE_IDX                                                                  2
+#define regDSCL3_DSCL_SC_MATRIX_C0C1                                                                    0x116e
+#define regDSCL3_DSCL_SC_MATRIX_C0C1_BASE_IDX                                                           2
+#define regDSCL3_DSCL_SC_MATRIX_C2C3                                                                    0x116f
+#define regDSCL3_DSCL_SC_MATRIX_C2C3_BASE_IDX                                                           2
+#define regDSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN                                                       0x1170
+#define regDSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE                                                     0x1171
+#define regDSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN                                                       0x1172
+#define regDSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN_BASE_IDX                                              2
+#define regDSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE                                                     0x1173
+#define regDSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE_BASE_IDX                                            2
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1                                                         0x1174
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1_BASE_IDX                                                2
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2                                                         0x1175
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2_BASE_IDX                                                2
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3                                                         0x1176
+#define regDSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3_BASE_IDX                                                2
+#define regDSCL3_DSCL_EASF_RINGEST_FORCE                                                                0x1177
+#define regDSCL3_DSCL_EASF_RINGEST_FORCE_BASE_IDX                                                       2
+#define regDSCL3_DSCL_EASF_H_BF_CNTL                                                                    0x1178
+#define regDSCL3_DSCL_EASF_H_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN                                                           0x1179
+#define regDSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL3_DSCL_EASF_V_BF_CNTL                                                                    0x117a
+#define regDSCL3_DSCL_EASF_V_BF_CNTL_BASE_IDX                                                           2
+#define regDSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN                                                           0x117b
+#define regDSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN_BASE_IDX                                                  2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG0                                                               0x117c
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG1                                                               0x117d
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG2                                                               0x117e
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG3                                                               0x117f
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG4                                                               0x1180
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG5                                                               0x1181
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG6                                                               0x1182
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG7                                                               0x1183
+#define regDSCL3_DSCL_EASF_H_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG0                                                               0x1184
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG1                                                               0x1185
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG2                                                               0x1186
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG3                                                               0x1187
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG4                                                               0x1188
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG5                                                               0x1189
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG6                                                               0x118a
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG6_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG7                                                               0x118b
+#define regDSCL3_DSCL_EASF_V_BF1_PWL_SEG7_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG0                                                               0x118c
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG1                                                               0x118d
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG2                                                               0x118e
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG3                                                               0x118f
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG4                                                               0x1190
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG5                                                               0x1191
+#define regDSCL3_DSCL_EASF_H_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG0                                                               0x1192
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG0_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG1                                                               0x1193
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG1_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG2                                                               0x1194
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG2_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG3                                                               0x1195
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG3_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG4                                                               0x1196
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG4_BASE_IDX                                                      2
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG5                                                               0x1197
+#define regDSCL3_DSCL_EASF_V_BF3_PWL_SEG5_BASE_IDX                                                      2
+#define regDSCL3_ISHARP_MODE                                                                            0x1198
+#define regDSCL3_ISHARP_MODE_BASE_IDX                                                                   2
+#define regDSCL3_ISHARP_DELTA_CTRL                                                                      0x1199
+#define regDSCL3_ISHARP_DELTA_CTRL_BASE_IDX                                                             2
+#define regDSCL3_ISHARP_DELTA_INDEX                                                                     0x119a
+#define regDSCL3_ISHARP_DELTA_INDEX_BASE_IDX                                                            2
+#define regDSCL3_ISHARP_DELTA_DATA                                                                      0x119b
+#define regDSCL3_ISHARP_DELTA_DATA_BASE_IDX                                                             2
+#define regDSCL3_ISHARP_NLDELTA_SOFT_CLIP                                                               0x119c
+#define regDSCL3_ISHARP_NLDELTA_SOFT_CLIP_BASE_IDX                                                      2
+#define regDSCL3_ISHARP_NOISEDET_THRESHOLD                                                              0x119d
+#define regDSCL3_ISHARP_NOISEDET_THRESHOLD_BASE_IDX                                                     2
+#define regDSCL3_ISHARP_NOISE_GAIN_PWL                                                                  0x119e
+#define regDSCL3_ISHARP_NOISE_GAIN_PWL_BASE_IDX                                                         2
+#define regDSCL3_ISHARP_LBA_PWL_SEG0                                                                    0x119f
+#define regDSCL3_ISHARP_LBA_PWL_SEG0_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_LBA_PWL_SEG1                                                                    0x11a0
+#define regDSCL3_ISHARP_LBA_PWL_SEG1_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_LBA_PWL_SEG2                                                                    0x11a1
+#define regDSCL3_ISHARP_LBA_PWL_SEG2_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_LBA_PWL_SEG3                                                                    0x11a2
+#define regDSCL3_ISHARP_LBA_PWL_SEG3_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_LBA_PWL_SEG4                                                                    0x11a3
+#define regDSCL3_ISHARP_LBA_PWL_SEG4_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_LBA_PWL_SEG5                                                                    0x11a4
+#define regDSCL3_ISHARP_LBA_PWL_SEG5_BASE_IDX                                                           2
+#define regDSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL                                                          0x11a5
+#define regDSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cm_dispdec
+// base address: 0x1104
+#define regCM3_CM_CONTROL                                                                               0x11aa
+#define regCM3_CM_CONTROL_BASE_IDX                                                                      2
+#define regCM3_CM_POST_CSC_CONTROL                                                                      0x11ab
+#define regCM3_CM_POST_CSC_CONTROL_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C11_C12                                                                      0x11ac
+#define regCM3_CM_POST_CSC_C11_C12_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C13_C14                                                                      0x11ad
+#define regCM3_CM_POST_CSC_C13_C14_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C21_C22                                                                      0x11ae
+#define regCM3_CM_POST_CSC_C21_C22_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C23_C24                                                                      0x11af
+#define regCM3_CM_POST_CSC_C23_C24_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C31_C32                                                                      0x11b0
+#define regCM3_CM_POST_CSC_C31_C32_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_C33_C34                                                                      0x11b1
+#define regCM3_CM_POST_CSC_C33_C34_BASE_IDX                                                             2
+#define regCM3_CM_POST_CSC_B_C11_C12                                                                    0x11b2
+#define regCM3_CM_POST_CSC_B_C11_C12_BASE_IDX                                                           2
+#define regCM3_CM_POST_CSC_B_C13_C14                                                                    0x11b3
+#define regCM3_CM_POST_CSC_B_C13_C14_BASE_IDX                                                           2
+#define regCM3_CM_POST_CSC_B_C21_C22                                                                    0x11b4
+#define regCM3_CM_POST_CSC_B_C21_C22_BASE_IDX                                                           2
+#define regCM3_CM_POST_CSC_B_C23_C24                                                                    0x11b5
+#define regCM3_CM_POST_CSC_B_C23_C24_BASE_IDX                                                           2
+#define regCM3_CM_POST_CSC_B_C31_C32                                                                    0x11b6
+#define regCM3_CM_POST_CSC_B_C31_C32_BASE_IDX                                                           2
+#define regCM3_CM_POST_CSC_B_C33_C34                                                                    0x11b7
+#define regCM3_CM_POST_CSC_B_C33_C34_BASE_IDX                                                           2
+#define regCM3_CM_BIAS_CR_R                                                                             0x11b8
+#define regCM3_CM_BIAS_CR_R_BASE_IDX                                                                    2
+#define regCM3_CM_BIAS_Y_G_CB_B                                                                         0x11b9
+#define regCM3_CM_BIAS_Y_G_CB_B_BASE_IDX                                                                2
+#define regCM3_CM_GAMCOR_CONTROL                                                                        0x11ba
+#define regCM3_CM_GAMCOR_CONTROL_BASE_IDX                                                               2
+#define regCM3_CM_GAMCOR_LUT_INDEX                                                                      0x11bb
+#define regCM3_CM_GAMCOR_LUT_INDEX_BASE_IDX                                                             2
+#define regCM3_CM_GAMCOR_LUT_DATA                                                                       0x11bc
+#define regCM3_CM_GAMCOR_LUT_DATA_BASE_IDX                                                              2
+#define regCM3_CM_GAMCOR_LUT_CONTROL                                                                    0x11bd
+#define regCM3_CM_GAMCOR_LUT_CONTROL_BASE_IDX                                                           2
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_B                                                              0x11be
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_B_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_G                                                              0x11bf
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_G_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_R                                                              0x11c0
+#define regCM3_CM_GAMCOR_RAMA_START_CNTL_R_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B                                                        0x11c1
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G                                                        0x11c2
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R                                                        0x11c3
+#define regCM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_B                                                         0x11c4
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_G                                                         0x11c5
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_R                                                         0x11c6
+#define regCM3_CM_GAMCOR_RAMA_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_B                                                               0x11c7
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_B                                                               0x11c8
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_G                                                               0x11c9
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_G                                                               0x11ca
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_R                                                               0x11cb
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_R                                                               0x11cc
+#define regCM3_CM_GAMCOR_RAMA_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_B                                                                  0x11cd
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_B_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_G                                                                  0x11ce
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_G_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_R                                                                  0x11cf
+#define regCM3_CM_GAMCOR_RAMA_OFFSET_R_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMA_REGION_0_1                                                                0x11d0
+#define regCM3_CM_GAMCOR_RAMA_REGION_0_1_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMA_REGION_2_3                                                                0x11d1
+#define regCM3_CM_GAMCOR_RAMA_REGION_2_3_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMA_REGION_4_5                                                                0x11d2
+#define regCM3_CM_GAMCOR_RAMA_REGION_4_5_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMA_REGION_6_7                                                                0x11d3
+#define regCM3_CM_GAMCOR_RAMA_REGION_6_7_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMA_REGION_8_9                                                                0x11d4
+#define regCM3_CM_GAMCOR_RAMA_REGION_8_9_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMA_REGION_10_11                                                              0x11d5
+#define regCM3_CM_GAMCOR_RAMA_REGION_10_11_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_12_13                                                              0x11d6
+#define regCM3_CM_GAMCOR_RAMA_REGION_12_13_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_14_15                                                              0x11d7
+#define regCM3_CM_GAMCOR_RAMA_REGION_14_15_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_16_17                                                              0x11d8
+#define regCM3_CM_GAMCOR_RAMA_REGION_16_17_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_18_19                                                              0x11d9
+#define regCM3_CM_GAMCOR_RAMA_REGION_18_19_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_20_21                                                              0x11da
+#define regCM3_CM_GAMCOR_RAMA_REGION_20_21_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_22_23                                                              0x11db
+#define regCM3_CM_GAMCOR_RAMA_REGION_22_23_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_24_25                                                              0x11dc
+#define regCM3_CM_GAMCOR_RAMA_REGION_24_25_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_26_27                                                              0x11dd
+#define regCM3_CM_GAMCOR_RAMA_REGION_26_27_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_28_29                                                              0x11de
+#define regCM3_CM_GAMCOR_RAMA_REGION_28_29_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_30_31                                                              0x11df
+#define regCM3_CM_GAMCOR_RAMA_REGION_30_31_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMA_REGION_32_33                                                              0x11e0
+#define regCM3_CM_GAMCOR_RAMA_REGION_32_33_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_B                                                              0x11e1
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_B_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_G                                                              0x11e2
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_G_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_R                                                              0x11e3
+#define regCM3_CM_GAMCOR_RAMB_START_CNTL_R_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B                                                        0x11e4
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G                                                        0x11e5
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R                                                        0x11e6
+#define regCM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                               2
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_B                                                         0x11e7
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_B_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_G                                                         0x11e8
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_G_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_R                                                         0x11e9
+#define regCM3_CM_GAMCOR_RAMB_START_BASE_CNTL_R_BASE_IDX                                                2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_B                                                               0x11ea
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_B_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_B                                                               0x11eb
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_B_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_G                                                               0x11ec
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_G_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_G                                                               0x11ed
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_G_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_R                                                               0x11ee
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL1_R_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_R                                                               0x11ef
+#define regCM3_CM_GAMCOR_RAMB_END_CNTL2_R_BASE_IDX                                                      2
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_B                                                                  0x11f0
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_B_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_G                                                                  0x11f1
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_G_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_R                                                                  0x11f2
+#define regCM3_CM_GAMCOR_RAMB_OFFSET_R_BASE_IDX                                                         2
+#define regCM3_CM_GAMCOR_RAMB_REGION_0_1                                                                0x11f3
+#define regCM3_CM_GAMCOR_RAMB_REGION_0_1_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMB_REGION_2_3                                                                0x11f4
+#define regCM3_CM_GAMCOR_RAMB_REGION_2_3_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMB_REGION_4_5                                                                0x11f5
+#define regCM3_CM_GAMCOR_RAMB_REGION_4_5_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMB_REGION_6_7                                                                0x11f6
+#define regCM3_CM_GAMCOR_RAMB_REGION_6_7_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMB_REGION_8_9                                                                0x11f7
+#define regCM3_CM_GAMCOR_RAMB_REGION_8_9_BASE_IDX                                                       2
+#define regCM3_CM_GAMCOR_RAMB_REGION_10_11                                                              0x11f8
+#define regCM3_CM_GAMCOR_RAMB_REGION_10_11_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_12_13                                                              0x11f9
+#define regCM3_CM_GAMCOR_RAMB_REGION_12_13_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_14_15                                                              0x11fa
+#define regCM3_CM_GAMCOR_RAMB_REGION_14_15_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_16_17                                                              0x11fb
+#define regCM3_CM_GAMCOR_RAMB_REGION_16_17_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_18_19                                                              0x11fc
+#define regCM3_CM_GAMCOR_RAMB_REGION_18_19_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_20_21                                                              0x11fd
+#define regCM3_CM_GAMCOR_RAMB_REGION_20_21_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_22_23                                                              0x11fe
+#define regCM3_CM_GAMCOR_RAMB_REGION_22_23_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_24_25                                                              0x11ff
+#define regCM3_CM_GAMCOR_RAMB_REGION_24_25_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_26_27                                                              0x1200
+#define regCM3_CM_GAMCOR_RAMB_REGION_26_27_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_28_29                                                              0x1201
+#define regCM3_CM_GAMCOR_RAMB_REGION_28_29_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_30_31                                                              0x1202
+#define regCM3_CM_GAMCOR_RAMB_REGION_30_31_BASE_IDX                                                     2
+#define regCM3_CM_GAMCOR_RAMB_REGION_32_33                                                              0x1203
+#define regCM3_CM_GAMCOR_RAMB_REGION_32_33_BASE_IDX                                                     2
+#define regCM3_CM_HDR_MULT_COEF                                                                         0x1204
+#define regCM3_CM_HDR_MULT_COEF_BASE_IDX                                                                2
+#define regCM3_CM_MEM_PWR_CTRL                                                                          0x1205
+#define regCM3_CM_MEM_PWR_CTRL_BASE_IDX                                                                 2
+#define regCM3_CM_MEM_PWR_STATUS                                                                        0x1206
+#define regCM3_CM_MEM_PWR_STATUS_BASE_IDX                                                               2
+#define regCM3_CM_DEALPHA                                                                               0x1208
+#define regCM3_CM_DEALPHA_BASE_IDX                                                                      2
+#define regCM3_CM_COEF_FORMAT                                                                           0x1209
+#define regCM3_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM3_CM_TEST_DEBUG_INDEX                                                                      0x120a
+#define regCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM3_CM_TEST_DEBUG_DATA                                                                       0x120b
+#define regCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
+#define regCM3_CM_HIST_CNTL                                                                             0x120c
+#define regCM3_CM_HIST_CNTL_BASE_IDX                                                                    2
+#define regCM3_CM_HIST_SCALE_SRC1                                                                       0x120d
+#define regCM3_CM_HIST_SCALE_SRC1_BASE_IDX                                                              2
+#define regCM3_CM_HIST_COEFA_SRC2                                                                       0x120e
+#define regCM3_CM_HIST_COEFA_SRC2_BASE_IDX                                                              2
+#define regCM3_CM_HIST_COEFB_SRC2                                                                       0x120f
+#define regCM3_CM_HIST_COEFB_SRC2_BASE_IDX                                                              2
+#define regCM3_CM_HIST_COEFC_SRC2                                                                       0x1210
+#define regCM3_CM_HIST_COEFC_SRC2_BASE_IDX                                                              2
+#define regCM3_CM_HIST_SCALE_SRC3                                                                       0x1211
+#define regCM3_CM_HIST_SCALE_SRC3_BASE_IDX                                                              2
+#define regCM3_CM_HIST_BIAS_SRC1                                                                        0x1212
+#define regCM3_CM_HIST_BIAS_SRC1_BASE_IDX                                                               2
+#define regCM3_CM_HIST_BIAS_SRC2                                                                        0x1213
+#define regCM3_CM_HIST_BIAS_SRC2_BASE_IDX                                                               2
+#define regCM3_CM_HIST_BIAS_SRC3                                                                        0x1214
+#define regCM3_CM_HIST_BIAS_SRC3_BASE_IDX                                                               2
+#define regCM3_CM_HIST_LOCK                                                                             0x1215
+#define regCM3_CM_HIST_LOCK_BASE_IDX                                                                    2
+#define regCM3_CM_HIST_INDEX                                                                            0x1216
+#define regCM3_CM_HIST_INDEX_BASE_IDX                                                                   2
+#define regCM3_CM_HIST_DATA                                                                             0x1217
+#define regCM3_CM_HIST_DATA_BASE_IDX                                                                    2
+#define regCM3_CM_HIST_STATUS                                                                           0x1218
+#define regCM3_CM_HIST_STATUS_BASE_IDX                                                                  2
+#define regCM3_CM_HIST_INT_CONTROL                                                                      0x1219
+#define regCM3_CM_HIST_INT_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x4994
+#define regDC_PERFMON13_PERFCOUNTER_CNTL                                                                0x1265
+#define regDC_PERFMON13_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON13_PERFCOUNTER_CNTL2                                                               0x1266
+#define regDC_PERFMON13_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON13_PERFCOUNTER_STATE                                                               0x1267
+#define regDC_PERFMON13_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON13_PERFMON_CNTL                                                                    0x1268
+#define regDC_PERFMON13_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON13_PERFMON_CNTL2                                                                   0x1269
+#define regDC_PERFMON13_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON13_PERFMON_CVALUE_INT_MISC                                                         0x126a
+#define regDC_PERFMON13_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON13_PERFMON_CVALUE_LOW                                                              0x126b
+#define regDC_PERFMON13_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON13_PERFMON_HI                                                                      0x126c
+#define regDC_PERFMON13_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON13_PERFMON_LOW                                                                     0x126d
+#define regDC_PERFMON13_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dcoh_dp_aux0_dispdec
+// base address: 0x0
+#define regDP_AUX0_AUX_CONTROL                                                                          0x16b2
+#define regDP_AUX0_AUX_CONTROL_BASE_IDX                                                                 2
+#define regDP_AUX0_AUX_SW_CONTROL                                                                       0x16b3
+#define regDP_AUX0_AUX_SW_CONTROL_BASE_IDX                                                              2
+#define regDP_AUX0_AUX_ARB_CONTROL                                                                      0x16b4
+#define regDP_AUX0_AUX_ARB_CONTROL_BASE_IDX                                                             2
+#define regDP_AUX0_AUX_INTERRUPT_CONTROL                                                                0x16b5
+#define regDP_AUX0_AUX_INTERRUPT_CONTROL_BASE_IDX                                                       2
+#define regDP_AUX0_AUX_SW_STATUS                                                                        0x16b6
+#define regDP_AUX0_AUX_SW_STATUS_BASE_IDX                                                               2
+#define regDP_AUX0_AUX_LS_STATUS                                                                        0x16b7
+#define regDP_AUX0_AUX_LS_STATUS_BASE_IDX                                                               2
+#define regDP_AUX0_AUX_SW_DATA                                                                          0x16b8
+#define regDP_AUX0_AUX_SW_DATA_BASE_IDX                                                                 2
+#define regDP_AUX0_AUX_LS_DATA                                                                          0x16b9
+#define regDP_AUX0_AUX_LS_DATA_BASE_IDX                                                                 2
+#define regDP_AUX0_AUX_DPHY_TX_REF_CONTROL                                                              0x16ba
+#define regDP_AUX0_AUX_DPHY_TX_REF_CONTROL_BASE_IDX                                                     2
+#define regDP_AUX0_AUX_DPHY_TX_CONTROL                                                                  0x16bb
+#define regDP_AUX0_AUX_DPHY_TX_CONTROL_BASE_IDX                                                         2
+#define regDP_AUX0_AUX_DPHY_RX_CONTROL0                                                                 0x16bc
+#define regDP_AUX0_AUX_DPHY_RX_CONTROL0_BASE_IDX                                                        2
+#define regDP_AUX0_AUX_DPHY_RX_CONTROL1                                                                 0x16bd
+#define regDP_AUX0_AUX_DPHY_RX_CONTROL1_BASE_IDX                                                        2
+#define regDP_AUX0_AUX_DPHY_TX_STATUS                                                                   0x16be
+#define regDP_AUX0_AUX_DPHY_TX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX0_AUX_DPHY_RX_STATUS                                                                   0x16bf
+#define regDP_AUX0_AUX_DPHY_RX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX0_AUX_PHY_WAKE_CNTL                                                                    0x16c1
+#define regDP_AUX0_AUX_PHY_WAKE_CNTL_BASE_IDX                                                           2
+#define regDP_AUX0_AUX_PHY_WAKE_STATUS                                                                  0x16c2
+#define regDP_AUX0_AUX_PHY_WAKE_STATUS_BASE_IDX                                                         2
+
+
+// addressBlock: dce_dc_dcoh_dp_aux1_dispdec
+// base address: 0x70
+#define regDP_AUX1_AUX_CONTROL                                                                          0x16ce
+#define regDP_AUX1_AUX_CONTROL_BASE_IDX                                                                 2
+#define regDP_AUX1_AUX_SW_CONTROL                                                                       0x16cf
+#define regDP_AUX1_AUX_SW_CONTROL_BASE_IDX                                                              2
+#define regDP_AUX1_AUX_ARB_CONTROL                                                                      0x16d0
+#define regDP_AUX1_AUX_ARB_CONTROL_BASE_IDX                                                             2
+#define regDP_AUX1_AUX_INTERRUPT_CONTROL                                                                0x16d1
+#define regDP_AUX1_AUX_INTERRUPT_CONTROL_BASE_IDX                                                       2
+#define regDP_AUX1_AUX_SW_STATUS                                                                        0x16d2
+#define regDP_AUX1_AUX_SW_STATUS_BASE_IDX                                                               2
+#define regDP_AUX1_AUX_LS_STATUS                                                                        0x16d3
+#define regDP_AUX1_AUX_LS_STATUS_BASE_IDX                                                               2
+#define regDP_AUX1_AUX_SW_DATA                                                                          0x16d4
+#define regDP_AUX1_AUX_SW_DATA_BASE_IDX                                                                 2
+#define regDP_AUX1_AUX_LS_DATA                                                                          0x16d5
+#define regDP_AUX1_AUX_LS_DATA_BASE_IDX                                                                 2
+#define regDP_AUX1_AUX_DPHY_TX_REF_CONTROL                                                              0x16d6
+#define regDP_AUX1_AUX_DPHY_TX_REF_CONTROL_BASE_IDX                                                     2
+#define regDP_AUX1_AUX_DPHY_TX_CONTROL                                                                  0x16d7
+#define regDP_AUX1_AUX_DPHY_TX_CONTROL_BASE_IDX                                                         2
+#define regDP_AUX1_AUX_DPHY_RX_CONTROL0                                                                 0x16d8
+#define regDP_AUX1_AUX_DPHY_RX_CONTROL0_BASE_IDX                                                        2
+#define regDP_AUX1_AUX_DPHY_RX_CONTROL1                                                                 0x16d9
+#define regDP_AUX1_AUX_DPHY_RX_CONTROL1_BASE_IDX                                                        2
+#define regDP_AUX1_AUX_DPHY_TX_STATUS                                                                   0x16da
+#define regDP_AUX1_AUX_DPHY_TX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX1_AUX_DPHY_RX_STATUS                                                                   0x16db
+#define regDP_AUX1_AUX_DPHY_RX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX1_AUX_PHY_WAKE_CNTL                                                                    0x16dd
+#define regDP_AUX1_AUX_PHY_WAKE_CNTL_BASE_IDX                                                           2
+#define regDP_AUX1_AUX_PHY_WAKE_STATUS                                                                  0x16de
+#define regDP_AUX1_AUX_PHY_WAKE_STATUS_BASE_IDX                                                         2
+
+
+// addressBlock: dce_dc_dcoh_dp_aux2_dispdec
+// base address: 0xe0
+#define regDP_AUX2_AUX_CONTROL                                                                          0x16ea
+#define regDP_AUX2_AUX_CONTROL_BASE_IDX                                                                 2
+#define regDP_AUX2_AUX_SW_CONTROL                                                                       0x16eb
+#define regDP_AUX2_AUX_SW_CONTROL_BASE_IDX                                                              2
+#define regDP_AUX2_AUX_ARB_CONTROL                                                                      0x16ec
+#define regDP_AUX2_AUX_ARB_CONTROL_BASE_IDX                                                             2
+#define regDP_AUX2_AUX_INTERRUPT_CONTROL                                                                0x16ed
+#define regDP_AUX2_AUX_INTERRUPT_CONTROL_BASE_IDX                                                       2
+#define regDP_AUX2_AUX_SW_STATUS                                                                        0x16ee
+#define regDP_AUX2_AUX_SW_STATUS_BASE_IDX                                                               2
+#define regDP_AUX2_AUX_LS_STATUS                                                                        0x16ef
+#define regDP_AUX2_AUX_LS_STATUS_BASE_IDX                                                               2
+#define regDP_AUX2_AUX_SW_DATA                                                                          0x16f0
+#define regDP_AUX2_AUX_SW_DATA_BASE_IDX                                                                 2
+#define regDP_AUX2_AUX_LS_DATA                                                                          0x16f1
+#define regDP_AUX2_AUX_LS_DATA_BASE_IDX                                                                 2
+#define regDP_AUX2_AUX_DPHY_TX_REF_CONTROL                                                              0x16f2
+#define regDP_AUX2_AUX_DPHY_TX_REF_CONTROL_BASE_IDX                                                     2
+#define regDP_AUX2_AUX_DPHY_TX_CONTROL                                                                  0x16f3
+#define regDP_AUX2_AUX_DPHY_TX_CONTROL_BASE_IDX                                                         2
+#define regDP_AUX2_AUX_DPHY_RX_CONTROL0                                                                 0x16f4
+#define regDP_AUX2_AUX_DPHY_RX_CONTROL0_BASE_IDX                                                        2
+#define regDP_AUX2_AUX_DPHY_RX_CONTROL1                                                                 0x16f5
+#define regDP_AUX2_AUX_DPHY_RX_CONTROL1_BASE_IDX                                                        2
+#define regDP_AUX2_AUX_DPHY_TX_STATUS                                                                   0x16f6
+#define regDP_AUX2_AUX_DPHY_TX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX2_AUX_DPHY_RX_STATUS                                                                   0x16f7
+#define regDP_AUX2_AUX_DPHY_RX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX2_AUX_PHY_WAKE_CNTL                                                                    0x16f9
+#define regDP_AUX2_AUX_PHY_WAKE_CNTL_BASE_IDX                                                           2
+#define regDP_AUX2_AUX_PHY_WAKE_STATUS                                                                  0x16fa
+#define regDP_AUX2_AUX_PHY_WAKE_STATUS_BASE_IDX                                                         2
+
+
+// addressBlock: dce_dc_dcoh_dp_aux3_dispdec
+// base address: 0x150
+#define regDP_AUX3_AUX_CONTROL                                                                          0x1706
+#define regDP_AUX3_AUX_CONTROL_BASE_IDX                                                                 2
+#define regDP_AUX3_AUX_SW_CONTROL                                                                       0x1707
+#define regDP_AUX3_AUX_SW_CONTROL_BASE_IDX                                                              2
+#define regDP_AUX3_AUX_ARB_CONTROL                                                                      0x1708
+#define regDP_AUX3_AUX_ARB_CONTROL_BASE_IDX                                                             2
+#define regDP_AUX3_AUX_INTERRUPT_CONTROL                                                                0x1709
+#define regDP_AUX3_AUX_INTERRUPT_CONTROL_BASE_IDX                                                       2
+#define regDP_AUX3_AUX_SW_STATUS                                                                        0x170a
+#define regDP_AUX3_AUX_SW_STATUS_BASE_IDX                                                               2
+#define regDP_AUX3_AUX_LS_STATUS                                                                        0x170b
+#define regDP_AUX3_AUX_LS_STATUS_BASE_IDX                                                               2
+#define regDP_AUX3_AUX_SW_DATA                                                                          0x170c
+#define regDP_AUX3_AUX_SW_DATA_BASE_IDX                                                                 2
+#define regDP_AUX3_AUX_LS_DATA                                                                          0x170d
+#define regDP_AUX3_AUX_LS_DATA_BASE_IDX                                                                 2
+#define regDP_AUX3_AUX_DPHY_TX_REF_CONTROL                                                              0x170e
+#define regDP_AUX3_AUX_DPHY_TX_REF_CONTROL_BASE_IDX                                                     2
+#define regDP_AUX3_AUX_DPHY_TX_CONTROL                                                                  0x170f
+#define regDP_AUX3_AUX_DPHY_TX_CONTROL_BASE_IDX                                                         2
+#define regDP_AUX3_AUX_DPHY_RX_CONTROL0                                                                 0x1710
+#define regDP_AUX3_AUX_DPHY_RX_CONTROL0_BASE_IDX                                                        2
+#define regDP_AUX3_AUX_DPHY_RX_CONTROL1                                                                 0x1711
+#define regDP_AUX3_AUX_DPHY_RX_CONTROL1_BASE_IDX                                                        2
+#define regDP_AUX3_AUX_DPHY_TX_STATUS                                                                   0x1712
+#define regDP_AUX3_AUX_DPHY_TX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX3_AUX_DPHY_RX_STATUS                                                                   0x1713
+#define regDP_AUX3_AUX_DPHY_RX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX3_AUX_PHY_WAKE_CNTL                                                                    0x1715
+#define regDP_AUX3_AUX_PHY_WAKE_CNTL_BASE_IDX                                                           2
+#define regDP_AUX3_AUX_PHY_WAKE_STATUS                                                                  0x1716
+#define regDP_AUX3_AUX_PHY_WAKE_STATUS_BASE_IDX                                                         2
+
+
+// addressBlock: dce_dc_dcoh_dp_aux4_dispdec
+// base address: 0x1c0
+#define regDP_AUX4_AUX_CONTROL                                                                          0x1722
+#define regDP_AUX4_AUX_CONTROL_BASE_IDX                                                                 2
+#define regDP_AUX4_AUX_SW_CONTROL                                                                       0x1723
+#define regDP_AUX4_AUX_SW_CONTROL_BASE_IDX                                                              2
+#define regDP_AUX4_AUX_ARB_CONTROL                                                                      0x1724
+#define regDP_AUX4_AUX_ARB_CONTROL_BASE_IDX                                                             2
+#define regDP_AUX4_AUX_INTERRUPT_CONTROL                                                                0x1725
+#define regDP_AUX4_AUX_INTERRUPT_CONTROL_BASE_IDX                                                       2
+#define regDP_AUX4_AUX_SW_STATUS                                                                        0x1726
+#define regDP_AUX4_AUX_SW_STATUS_BASE_IDX                                                               2
+#define regDP_AUX4_AUX_LS_STATUS                                                                        0x1727
+#define regDP_AUX4_AUX_LS_STATUS_BASE_IDX                                                               2
+#define regDP_AUX4_AUX_SW_DATA                                                                          0x1728
+#define regDP_AUX4_AUX_SW_DATA_BASE_IDX                                                                 2
+#define regDP_AUX4_AUX_LS_DATA                                                                          0x1729
+#define regDP_AUX4_AUX_LS_DATA_BASE_IDX                                                                 2
+#define regDP_AUX4_AUX_DPHY_TX_REF_CONTROL                                                              0x172a
+#define regDP_AUX4_AUX_DPHY_TX_REF_CONTROL_BASE_IDX                                                     2
+#define regDP_AUX4_AUX_DPHY_TX_CONTROL                                                                  0x172b
+#define regDP_AUX4_AUX_DPHY_TX_CONTROL_BASE_IDX                                                         2
+#define regDP_AUX4_AUX_DPHY_RX_CONTROL0                                                                 0x172c
+#define regDP_AUX4_AUX_DPHY_RX_CONTROL0_BASE_IDX                                                        2
+#define regDP_AUX4_AUX_DPHY_RX_CONTROL1                                                                 0x172d
+#define regDP_AUX4_AUX_DPHY_RX_CONTROL1_BASE_IDX                                                        2
+#define regDP_AUX4_AUX_DPHY_TX_STATUS                                                                   0x172e
+#define regDP_AUX4_AUX_DPHY_TX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX4_AUX_DPHY_RX_STATUS                                                                   0x172f
+#define regDP_AUX4_AUX_DPHY_RX_STATUS_BASE_IDX                                                          2
+#define regDP_AUX4_AUX_PHY_WAKE_CNTL                                                                    0x1731
+#define regDP_AUX4_AUX_PHY_WAKE_CNTL_BASE_IDX                                                           2
+#define regDP_AUX4_AUX_PHY_WAKE_STATUS                                                                  0x1732
+#define regDP_AUX4_AUX_PHY_WAKE_STATUS_BASE_IDX                                                         2
+
+
+// addressBlock: dce_dc_dcoh_hpd0_dispdec
+// base address: 0x0
+#define regHPD0_DC_HPD_INT_STATUS                                                                       0x175a
+#define regHPD0_DC_HPD_INT_STATUS_BASE_IDX                                                              2
+#define regHPD0_DC_HPD_INT_CONTROL                                                                      0x175b
+#define regHPD0_DC_HPD_INT_CONTROL_BASE_IDX                                                             2
+#define regHPD0_DC_HPD_CONTROL                                                                          0x175c
+#define regHPD0_DC_HPD_CONTROL_BASE_IDX                                                                 2
+#define regHPD0_DC_HPD_FAST_TRAIN_CNTL                                                                  0x175d
+#define regHPD0_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX                                                         2
+#define regHPD0_DC_HPD_TOGGLE_FILT_CNTL                                                                 0x175e
+#define regHPD0_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcoh_hpd1_dispdec
+// base address: 0x20
+#define regHPD1_DC_HPD_INT_STATUS                                                                       0x1762
+#define regHPD1_DC_HPD_INT_STATUS_BASE_IDX                                                              2
+#define regHPD1_DC_HPD_INT_CONTROL                                                                      0x1763
+#define regHPD1_DC_HPD_INT_CONTROL_BASE_IDX                                                             2
+#define regHPD1_DC_HPD_CONTROL                                                                          0x1764
+#define regHPD1_DC_HPD_CONTROL_BASE_IDX                                                                 2
+#define regHPD1_DC_HPD_FAST_TRAIN_CNTL                                                                  0x1765
+#define regHPD1_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX                                                         2
+#define regHPD1_DC_HPD_TOGGLE_FILT_CNTL                                                                 0x1766
+#define regHPD1_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcoh_hpd2_dispdec
+// base address: 0x40
+#define regHPD2_DC_HPD_INT_STATUS                                                                       0x176a
+#define regHPD2_DC_HPD_INT_STATUS_BASE_IDX                                                              2
+#define regHPD2_DC_HPD_INT_CONTROL                                                                      0x176b
+#define regHPD2_DC_HPD_INT_CONTROL_BASE_IDX                                                             2
+#define regHPD2_DC_HPD_CONTROL                                                                          0x176c
+#define regHPD2_DC_HPD_CONTROL_BASE_IDX                                                                 2
+#define regHPD2_DC_HPD_FAST_TRAIN_CNTL                                                                  0x176d
+#define regHPD2_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX                                                         2
+#define regHPD2_DC_HPD_TOGGLE_FILT_CNTL                                                                 0x176e
+#define regHPD2_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcoh_hpd3_dispdec
+// base address: 0x60
+#define regHPD3_DC_HPD_INT_STATUS                                                                       0x1772
+#define regHPD3_DC_HPD_INT_STATUS_BASE_IDX                                                              2
+#define regHPD3_DC_HPD_INT_CONTROL                                                                      0x1773
+#define regHPD3_DC_HPD_INT_CONTROL_BASE_IDX                                                             2
+#define regHPD3_DC_HPD_CONTROL                                                                          0x1774
+#define regHPD3_DC_HPD_CONTROL_BASE_IDX                                                                 2
+#define regHPD3_DC_HPD_FAST_TRAIN_CNTL                                                                  0x1775
+#define regHPD3_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX                                                         2
+#define regHPD3_DC_HPD_TOGGLE_FILT_CNTL                                                                 0x1776
+#define regHPD3_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcoh_hpd4_dispdec
+// base address: 0x80
+#define regHPD4_DC_HPD_INT_STATUS                                                                       0x177a
+#define regHPD4_DC_HPD_INT_STATUS_BASE_IDX                                                              2
+#define regHPD4_DC_HPD_INT_CONTROL                                                                      0x177b
+#define regHPD4_DC_HPD_INT_CONTROL_BASE_IDX                                                             2
+#define regHPD4_DC_HPD_CONTROL                                                                          0x177c
+#define regHPD4_DC_HPD_CONTROL_BASE_IDX                                                                 2
+#define regHPD4_DC_HPD_FAST_TRAIN_CNTL                                                                  0x177d
+#define regHPD4_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX                                                         2
+#define regHPD4_DC_HPD_TOGGLE_FILT_CNTL                                                                 0x177e
+#define regHPD4_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX                                                        2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux0_dispdec
+// base address: 0x13128
+#define regDPIA_MUX0_DPIA_MUX_CONTROL                                                                   0x178a
+#define regDPIA_MUX0_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux1_dispdec
+// base address: 0x13130
+#define regDPIA_MUX1_DPIA_MUX_CONTROL                                                                   0x178c
+#define regDPIA_MUX1_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux2_dispdec
+// base address: 0x13138
+#define regDPIA_MUX2_DPIA_MUX_CONTROL                                                                   0x178e
+#define regDPIA_MUX2_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux3_dispdec
+// base address: 0x13140
+#define regDPIA_MUX3_DPIA_MUX_CONTROL                                                                   0x1790
+#define regDPIA_MUX3_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux4_dispdec
+// base address: 0x13148
+#define regDPIA_MUX4_DPIA_MUX_CONTROL                                                                   0x1792
+#define regDPIA_MUX4_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux5_dispdec
+// base address: 0x13150
+#define regDPIA_MUX5_DPIA_MUX_CONTROL                                                                   0x1794
+#define regDPIA_MUX5_DPIA_MUX_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dcoh_phy_mux0_dispdec
+// base address: 0x13168
+#define regPHY_MUX0_PHY_MUX_CONTROL                                                                     0x179a
+#define regPHY_MUX0_PHY_MUX_CONTROL_BASE_IDX                                                            2
+#define regPHY_MUX0_PORT_TYPE                                                                           0x179b
+#define regPHY_MUX0_PORT_TYPE_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dcoh_phy_mux1_dispdec
+// base address: 0x13174
+#define regPHY_MUX1_PHY_MUX_CONTROL                                                                     0x179d
+#define regPHY_MUX1_PHY_MUX_CONTROL_BASE_IDX                                                            2
+#define regPHY_MUX1_PORT_TYPE                                                                           0x179e
+#define regPHY_MUX1_PORT_TYPE_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dcoh_phy_mux2_dispdec
+// base address: 0x13180
+#define regPHY_MUX2_PHY_MUX_CONTROL                                                                     0x17a0
+#define regPHY_MUX2_PHY_MUX_CONTROL_BASE_IDX                                                            2
+#define regPHY_MUX2_PORT_TYPE                                                                           0x17a1
+#define regPHY_MUX2_PORT_TYPE_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dcoh_phy_mux3_dispdec
+// base address: 0x1318c
+#define regPHY_MUX3_PHY_MUX_CONTROL                                                                     0x17a3
+#define regPHY_MUX3_PHY_MUX_CONTROL_BASE_IDX                                                            2
+#define regPHY_MUX3_PORT_TYPE                                                                           0x17a4
+#define regPHY_MUX3_PORT_TYPE_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dcoh_phy_mux4_dispdec
+// base address: 0x13198
+#define regPHY_MUX4_PHY_MUX_CONTROL                                                                     0x17a6
+#define regPHY_MUX4_PHY_MUX_CONTROL_BASE_IDX                                                            2
+#define regPHY_MUX4_PORT_TYPE                                                                           0x17a7
+#define regPHY_MUX4_PORT_TYPE_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_dcoh_dcoh_top_dispdec
+// base address: 0x0
+#define regDCOH_TOP_CLOCK_CONTROL                                                                       0x17af
+#define regDCOH_TOP_CLOCK_CONTROL_BASE_IDX                                                              2
+#define regDCOH_DCN_STATUS                                                                              0x17b0
+#define regDCOH_DCN_STATUS_BASE_IDX                                                                     2
+#define regDCOH_TOP_SPARE                                                                               0x17b4
+#define regDCOH_TOP_SPARE_BASE_IDX                                                                      2
+
+
+// addressBlock: dce_dc_opp_fmt0_dispdec
+// base address: 0x0
+#define regFMT0_FMT_CLAMP_COMPONENT_R                                                                   0x183c
+#define regFMT0_FMT_CLAMP_COMPONENT_R_BASE_IDX                                                          2
+#define regFMT0_FMT_CLAMP_COMPONENT_G                                                                   0x183d
+#define regFMT0_FMT_CLAMP_COMPONENT_G_BASE_IDX                                                          2
+#define regFMT0_FMT_CLAMP_COMPONENT_B                                                                   0x183e
+#define regFMT0_FMT_CLAMP_COMPONENT_B_BASE_IDX                                                          2
+#define regFMT0_FMT_DYNAMIC_EXP_CNTL                                                                    0x183f
+#define regFMT0_FMT_DYNAMIC_EXP_CNTL_BASE_IDX                                                           2
+#define regFMT0_FMT_CONTROL                                                                             0x1840
+#define regFMT0_FMT_CONTROL_BASE_IDX                                                                    2
+#define regFMT0_FMT_BIT_DEPTH_CONTROL                                                                   0x1841
+#define regFMT0_FMT_BIT_DEPTH_CONTROL_BASE_IDX                                                          2
+#define regFMT0_FMT_DITHER_RAND_R_SEED                                                                  0x1842
+#define regFMT0_FMT_DITHER_RAND_R_SEED_BASE_IDX                                                         2
+#define regFMT0_FMT_DITHER_RAND_G_SEED                                                                  0x1843
+#define regFMT0_FMT_DITHER_RAND_G_SEED_BASE_IDX                                                         2
+#define regFMT0_FMT_DITHER_RAND_B_SEED                                                                  0x1844
+#define regFMT0_FMT_DITHER_RAND_B_SEED_BASE_IDX                                                         2
+#define regFMT0_FMT_CLAMP_CNTL                                                                          0x1845
+#define regFMT0_FMT_CLAMP_CNTL_BASE_IDX                                                                 2
+#define regFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL                                                         0x1846
+#define regFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX                                                2
+#define regFMT0_FMT_MAP420_MEMORY_CONTROL                                                               0x1847
+#define regFMT0_FMT_MAP420_MEMORY_CONTROL_BASE_IDX                                                      2
+#define regFMT0_FMT_422_CONTROL                                                                         0x1849
+#define regFMT0_FMT_422_CONTROL_BASE_IDX                                                                2
+
+
+// addressBlock: dce_dc_opp_dpg0_dispdec
+// base address: 0x0
+#define regDPG0_DPG_CONTROL                                                                             0x1854
+#define regDPG0_DPG_CONTROL_BASE_IDX                                                                    2
+#define regDPG0_DPG_RAMP_CONTROL                                                                        0x1855
+#define regDPG0_DPG_RAMP_CONTROL_BASE_IDX                                                               2
+#define regDPG0_DPG_DIMENSIONS                                                                          0x1856
+#define regDPG0_DPG_DIMENSIONS_BASE_IDX                                                                 2
+#define regDPG0_DPG_COLOUR_R_CR                                                                         0x1857
+#define regDPG0_DPG_COLOUR_R_CR_BASE_IDX                                                                2
+#define regDPG0_DPG_COLOUR_G_Y                                                                          0x1858
+#define regDPG0_DPG_COLOUR_G_Y_BASE_IDX                                                                 2
+#define regDPG0_DPG_COLOUR_B_CB                                                                         0x1859
+#define regDPG0_DPG_COLOUR_B_CB_BASE_IDX                                                                2
+#define regDPG0_DPG_OFFSET_SEGMENT                                                                      0x185a
+#define regDPG0_DPG_OFFSET_SEGMENT_BASE_IDX                                                             2
+#define regDPG0_DPG_STATUS                                                                              0x185b
+#define regDPG0_DPG_STATUS_BASE_IDX                                                                     2
+
+
+// addressBlock: dce_dc_opp_oppbuf0_dispdec
+// base address: 0x0
+#define regOPPBUF0_OPPBUF_CONTROL                                                                       0x1884
+#define regOPPBUF0_OPPBUF_CONTROL_BASE_IDX                                                              2
+#define regOPPBUF0_OPPBUF_3D_PARAMETERS_0                                                               0x1885
+#define regOPPBUF0_OPPBUF_3D_PARAMETERS_0_BASE_IDX                                                      2
+#define regOPPBUF0_OPPBUF_3D_PARAMETERS_1                                                               0x1886
+#define regOPPBUF0_OPPBUF_3D_PARAMETERS_1_BASE_IDX                                                      2
+#define regOPPBUF0_OPPBUF_CONTROL1                                                                      0x1889
+#define regOPPBUF0_OPPBUF_CONTROL1_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_opp_opp_pipe0_dispdec
+// base address: 0x0
+#define regOPP_PIPE0_OPP_PIPE_CONTROL                                                                   0x188b
+#define regOPP_PIPE0_OPP_PIPE_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc0_dispdec
+// base address: 0x0
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL                                                           0x188f
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_MASK                                                              0x1890
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_MASK_BASE_IDX                                                     2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTA                                                           0x1891
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTA_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTR                                                           0x1892
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTR_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTG                                                           0x1893
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTG_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTB                                                           0x1894
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTB_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTC                                                           0x1895
+#define regOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTC_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_opp_fmt1_dispdec
+// base address: 0x168
+#define regFMT1_FMT_CLAMP_COMPONENT_R                                                                   0x1896
+#define regFMT1_FMT_CLAMP_COMPONENT_R_BASE_IDX                                                          2
+#define regFMT1_FMT_CLAMP_COMPONENT_G                                                                   0x1897
+#define regFMT1_FMT_CLAMP_COMPONENT_G_BASE_IDX                                                          2
+#define regFMT1_FMT_CLAMP_COMPONENT_B                                                                   0x1898
+#define regFMT1_FMT_CLAMP_COMPONENT_B_BASE_IDX                                                          2
+#define regFMT1_FMT_DYNAMIC_EXP_CNTL                                                                    0x1899
+#define regFMT1_FMT_DYNAMIC_EXP_CNTL_BASE_IDX                                                           2
+#define regFMT1_FMT_CONTROL                                                                             0x189a
+#define regFMT1_FMT_CONTROL_BASE_IDX                                                                    2
+#define regFMT1_FMT_BIT_DEPTH_CONTROL                                                                   0x189b
+#define regFMT1_FMT_BIT_DEPTH_CONTROL_BASE_IDX                                                          2
+#define regFMT1_FMT_DITHER_RAND_R_SEED                                                                  0x189c
+#define regFMT1_FMT_DITHER_RAND_R_SEED_BASE_IDX                                                         2
+#define regFMT1_FMT_DITHER_RAND_G_SEED                                                                  0x189d
+#define regFMT1_FMT_DITHER_RAND_G_SEED_BASE_IDX                                                         2
+#define regFMT1_FMT_DITHER_RAND_B_SEED                                                                  0x189e
+#define regFMT1_FMT_DITHER_RAND_B_SEED_BASE_IDX                                                         2
+#define regFMT1_FMT_CLAMP_CNTL                                                                          0x189f
+#define regFMT1_FMT_CLAMP_CNTL_BASE_IDX                                                                 2
+#define regFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL                                                         0x18a0
+#define regFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX                                                2
+#define regFMT1_FMT_MAP420_MEMORY_CONTROL                                                               0x18a1
+#define regFMT1_FMT_MAP420_MEMORY_CONTROL_BASE_IDX                                                      2
+#define regFMT1_FMT_422_CONTROL                                                                         0x18a3
+#define regFMT1_FMT_422_CONTROL_BASE_IDX                                                                2
+
+
+// addressBlock: dce_dc_opp_dpg1_dispdec
+// base address: 0x168
+#define regDPG1_DPG_CONTROL                                                                             0x18ae
+#define regDPG1_DPG_CONTROL_BASE_IDX                                                                    2
+#define regDPG1_DPG_RAMP_CONTROL                                                                        0x18af
+#define regDPG1_DPG_RAMP_CONTROL_BASE_IDX                                                               2
+#define regDPG1_DPG_DIMENSIONS                                                                          0x18b0
+#define regDPG1_DPG_DIMENSIONS_BASE_IDX                                                                 2
+#define regDPG1_DPG_COLOUR_R_CR                                                                         0x18b1
+#define regDPG1_DPG_COLOUR_R_CR_BASE_IDX                                                                2
+#define regDPG1_DPG_COLOUR_G_Y                                                                          0x18b2
+#define regDPG1_DPG_COLOUR_G_Y_BASE_IDX                                                                 2
+#define regDPG1_DPG_COLOUR_B_CB                                                                         0x18b3
+#define regDPG1_DPG_COLOUR_B_CB_BASE_IDX                                                                2
+#define regDPG1_DPG_OFFSET_SEGMENT                                                                      0x18b4
+#define regDPG1_DPG_OFFSET_SEGMENT_BASE_IDX                                                             2
+#define regDPG1_DPG_STATUS                                                                              0x18b5
+#define regDPG1_DPG_STATUS_BASE_IDX                                                                     2
+
+
+// addressBlock: dce_dc_opp_oppbuf1_dispdec
+// base address: 0x168
+#define regOPPBUF1_OPPBUF_CONTROL                                                                       0x18de
+#define regOPPBUF1_OPPBUF_CONTROL_BASE_IDX                                                              2
+#define regOPPBUF1_OPPBUF_3D_PARAMETERS_0                                                               0x18df
+#define regOPPBUF1_OPPBUF_3D_PARAMETERS_0_BASE_IDX                                                      2
+#define regOPPBUF1_OPPBUF_3D_PARAMETERS_1                                                               0x18e0
+#define regOPPBUF1_OPPBUF_3D_PARAMETERS_1_BASE_IDX                                                      2
+#define regOPPBUF1_OPPBUF_CONTROL1                                                                      0x18e3
+#define regOPPBUF1_OPPBUF_CONTROL1_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_opp_opp_pipe1_dispdec
+// base address: 0x168
+#define regOPP_PIPE1_OPP_PIPE_CONTROL                                                                   0x18e5
+#define regOPP_PIPE1_OPP_PIPE_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc1_dispdec
+// base address: 0x168
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL                                                           0x18e9
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_MASK                                                              0x18ea
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_MASK_BASE_IDX                                                     2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTA                                                           0x18eb
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTA_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTR                                                           0x18ec
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTR_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTG                                                           0x18ed
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTG_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTB                                                           0x18ee
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTB_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTC                                                           0x18ef
+#define regOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTC_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_opp_fmt2_dispdec
+// base address: 0x2d0
+#define regFMT2_FMT_CLAMP_COMPONENT_R                                                                   0x18f0
+#define regFMT2_FMT_CLAMP_COMPONENT_R_BASE_IDX                                                          2
+#define regFMT2_FMT_CLAMP_COMPONENT_G                                                                   0x18f1
+#define regFMT2_FMT_CLAMP_COMPONENT_G_BASE_IDX                                                          2
+#define regFMT2_FMT_CLAMP_COMPONENT_B                                                                   0x18f2
+#define regFMT2_FMT_CLAMP_COMPONENT_B_BASE_IDX                                                          2
+#define regFMT2_FMT_DYNAMIC_EXP_CNTL                                                                    0x18f3
+#define regFMT2_FMT_DYNAMIC_EXP_CNTL_BASE_IDX                                                           2
+#define regFMT2_FMT_CONTROL                                                                             0x18f4
+#define regFMT2_FMT_CONTROL_BASE_IDX                                                                    2
+#define regFMT2_FMT_BIT_DEPTH_CONTROL                                                                   0x18f5
+#define regFMT2_FMT_BIT_DEPTH_CONTROL_BASE_IDX                                                          2
+#define regFMT2_FMT_DITHER_RAND_R_SEED                                                                  0x18f6
+#define regFMT2_FMT_DITHER_RAND_R_SEED_BASE_IDX                                                         2
+#define regFMT2_FMT_DITHER_RAND_G_SEED                                                                  0x18f7
+#define regFMT2_FMT_DITHER_RAND_G_SEED_BASE_IDX                                                         2
+#define regFMT2_FMT_DITHER_RAND_B_SEED                                                                  0x18f8
+#define regFMT2_FMT_DITHER_RAND_B_SEED_BASE_IDX                                                         2
+#define regFMT2_FMT_CLAMP_CNTL                                                                          0x18f9
+#define regFMT2_FMT_CLAMP_CNTL_BASE_IDX                                                                 2
+#define regFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL                                                         0x18fa
+#define regFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX                                                2
+#define regFMT2_FMT_MAP420_MEMORY_CONTROL                                                               0x18fb
+#define regFMT2_FMT_MAP420_MEMORY_CONTROL_BASE_IDX                                                      2
+#define regFMT2_FMT_422_CONTROL                                                                         0x18fd
+#define regFMT2_FMT_422_CONTROL_BASE_IDX                                                                2
+
+
+// addressBlock: dce_dc_opp_dpg2_dispdec
+// base address: 0x2d0
+#define regDPG2_DPG_CONTROL                                                                             0x1908
+#define regDPG2_DPG_CONTROL_BASE_IDX                                                                    2
+#define regDPG2_DPG_RAMP_CONTROL                                                                        0x1909
+#define regDPG2_DPG_RAMP_CONTROL_BASE_IDX                                                               2
+#define regDPG2_DPG_DIMENSIONS                                                                          0x190a
+#define regDPG2_DPG_DIMENSIONS_BASE_IDX                                                                 2
+#define regDPG2_DPG_COLOUR_R_CR                                                                         0x190b
+#define regDPG2_DPG_COLOUR_R_CR_BASE_IDX                                                                2
+#define regDPG2_DPG_COLOUR_G_Y                                                                          0x190c
+#define regDPG2_DPG_COLOUR_G_Y_BASE_IDX                                                                 2
+#define regDPG2_DPG_COLOUR_B_CB                                                                         0x190d
+#define regDPG2_DPG_COLOUR_B_CB_BASE_IDX                                                                2
+#define regDPG2_DPG_OFFSET_SEGMENT                                                                      0x190e
+#define regDPG2_DPG_OFFSET_SEGMENT_BASE_IDX                                                             2
+#define regDPG2_DPG_STATUS                                                                              0x190f
+#define regDPG2_DPG_STATUS_BASE_IDX                                                                     2
+
+
+// addressBlock: dce_dc_opp_oppbuf2_dispdec
+// base address: 0x2d0
+#define regOPPBUF2_OPPBUF_CONTROL                                                                       0x1938
+#define regOPPBUF2_OPPBUF_CONTROL_BASE_IDX                                                              2
+#define regOPPBUF2_OPPBUF_3D_PARAMETERS_0                                                               0x1939
+#define regOPPBUF2_OPPBUF_3D_PARAMETERS_0_BASE_IDX                                                      2
+#define regOPPBUF2_OPPBUF_3D_PARAMETERS_1                                                               0x193a
+#define regOPPBUF2_OPPBUF_3D_PARAMETERS_1_BASE_IDX                                                      2
+#define regOPPBUF2_OPPBUF_CONTROL1                                                                      0x193d
+#define regOPPBUF2_OPPBUF_CONTROL1_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_opp_opp_pipe2_dispdec
+// base address: 0x2d0
+#define regOPP_PIPE2_OPP_PIPE_CONTROL                                                                   0x193f
+#define regOPP_PIPE2_OPP_PIPE_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc2_dispdec
+// base address: 0x2d0
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL                                                           0x1943
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_MASK                                                              0x1944
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_MASK_BASE_IDX                                                     2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTA                                                           0x1945
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTA_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTR                                                           0x1946
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTR_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTG                                                           0x1947
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTG_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTB                                                           0x1948
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTB_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTC                                                           0x1949
+#define regOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTC_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_opp_fmt3_dispdec
+// base address: 0x438
+#define regFMT3_FMT_CLAMP_COMPONENT_R                                                                   0x194a
+#define regFMT3_FMT_CLAMP_COMPONENT_R_BASE_IDX                                                          2
+#define regFMT3_FMT_CLAMP_COMPONENT_G                                                                   0x194b
+#define regFMT3_FMT_CLAMP_COMPONENT_G_BASE_IDX                                                          2
+#define regFMT3_FMT_CLAMP_COMPONENT_B                                                                   0x194c
+#define regFMT3_FMT_CLAMP_COMPONENT_B_BASE_IDX                                                          2
+#define regFMT3_FMT_DYNAMIC_EXP_CNTL                                                                    0x194d
+#define regFMT3_FMT_DYNAMIC_EXP_CNTL_BASE_IDX                                                           2
+#define regFMT3_FMT_CONTROL                                                                             0x194e
+#define regFMT3_FMT_CONTROL_BASE_IDX                                                                    2
+#define regFMT3_FMT_BIT_DEPTH_CONTROL                                                                   0x194f
+#define regFMT3_FMT_BIT_DEPTH_CONTROL_BASE_IDX                                                          2
+#define regFMT3_FMT_DITHER_RAND_R_SEED                                                                  0x1950
+#define regFMT3_FMT_DITHER_RAND_R_SEED_BASE_IDX                                                         2
+#define regFMT3_FMT_DITHER_RAND_G_SEED                                                                  0x1951
+#define regFMT3_FMT_DITHER_RAND_G_SEED_BASE_IDX                                                         2
+#define regFMT3_FMT_DITHER_RAND_B_SEED                                                                  0x1952
+#define regFMT3_FMT_DITHER_RAND_B_SEED_BASE_IDX                                                         2
+#define regFMT3_FMT_CLAMP_CNTL                                                                          0x1953
+#define regFMT3_FMT_CLAMP_CNTL_BASE_IDX                                                                 2
+#define regFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL                                                         0x1954
+#define regFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX                                                2
+#define regFMT3_FMT_MAP420_MEMORY_CONTROL                                                               0x1955
+#define regFMT3_FMT_MAP420_MEMORY_CONTROL_BASE_IDX                                                      2
+#define regFMT3_FMT_422_CONTROL                                                                         0x1957
+#define regFMT3_FMT_422_CONTROL_BASE_IDX                                                                2
+
+
+// addressBlock: dce_dc_opp_dpg3_dispdec
+// base address: 0x438
+#define regDPG3_DPG_CONTROL                                                                             0x1962
+#define regDPG3_DPG_CONTROL_BASE_IDX                                                                    2
+#define regDPG3_DPG_RAMP_CONTROL                                                                        0x1963
+#define regDPG3_DPG_RAMP_CONTROL_BASE_IDX                                                               2
+#define regDPG3_DPG_DIMENSIONS                                                                          0x1964
+#define regDPG3_DPG_DIMENSIONS_BASE_IDX                                                                 2
+#define regDPG3_DPG_COLOUR_R_CR                                                                         0x1965
+#define regDPG3_DPG_COLOUR_R_CR_BASE_IDX                                                                2
+#define regDPG3_DPG_COLOUR_G_Y                                                                          0x1966
+#define regDPG3_DPG_COLOUR_G_Y_BASE_IDX                                                                 2
+#define regDPG3_DPG_COLOUR_B_CB                                                                         0x1967
+#define regDPG3_DPG_COLOUR_B_CB_BASE_IDX                                                                2
+#define regDPG3_DPG_OFFSET_SEGMENT                                                                      0x1968
+#define regDPG3_DPG_OFFSET_SEGMENT_BASE_IDX                                                             2
+#define regDPG3_DPG_STATUS                                                                              0x1969
+#define regDPG3_DPG_STATUS_BASE_IDX                                                                     2
+
+
+// addressBlock: dce_dc_opp_oppbuf3_dispdec
+// base address: 0x438
+#define regOPPBUF3_OPPBUF_CONTROL                                                                       0x1992
+#define regOPPBUF3_OPPBUF_CONTROL_BASE_IDX                                                              2
+#define regOPPBUF3_OPPBUF_3D_PARAMETERS_0                                                               0x1993
+#define regOPPBUF3_OPPBUF_3D_PARAMETERS_0_BASE_IDX                                                      2
+#define regOPPBUF3_OPPBUF_3D_PARAMETERS_1                                                               0x1994
+#define regOPPBUF3_OPPBUF_3D_PARAMETERS_1_BASE_IDX                                                      2
+#define regOPPBUF3_OPPBUF_CONTROL1                                                                      0x1997
+#define regOPPBUF3_OPPBUF_CONTROL1_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_opp_opp_pipe3_dispdec
+// base address: 0x438
+#define regOPP_PIPE3_OPP_PIPE_CONTROL                                                                   0x1999
+#define regOPP_PIPE3_OPP_PIPE_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc3_dispdec
+// base address: 0x438
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL                                                           0x199d
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_MASK                                                              0x199e
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_MASK_BASE_IDX                                                     2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTA                                                           0x199f
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTA_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTR                                                           0x19a0
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTR_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTG                                                           0x19a1
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTG_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTB                                                           0x19a2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTB_BASE_IDX                                                  2
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTC                                                           0x19a3
+#define regOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTC_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_opp_opp_top_dispdec
+// base address: 0x0
+#define regOPP_TOP_CLK_CONTROL                                                                          0x1a5e
+#define regOPP_TOP_CLK_CONTROL_BASE_IDX                                                                 2
+#define regOPP_ABM_CONTROL                                                                              0x1a60
+#define regOPP_ABM_CONTROL_BASE_IDX                                                                     2
+
+
+// addressBlock: dce_dc_opp_dscrm0_dispdec
+// base address: 0x0
+#define regDSCRM0_DSCRM_DSC_FORWARD_CONFIG                                                              0x1a64
+#define regDSCRM0_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_opp_dscrm1_dispdec
+// base address: 0x4
+#define regDSCRM1_DSCRM_DSC_FORWARD_CONFIG                                                              0x1a65
+#define regDSCRM1_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_opp_dscrm2_dispdec
+// base address: 0x8
+#define regDSCRM2_DSCRM_DSC_FORWARD_CONFIG                                                              0x1a66
+#define regDSCRM2_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_opp_dscrm3_dispdec
+// base address: 0xc
+#define regDSCRM3_DSCRM_DSC_FORWARD_CONFIG                                                              0x1a67
+#define regDSCRM3_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX                                                     2
+
+
+// addressBlock: dce_dc_opp_opp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x6af8
+#define regDC_PERFMON14_PERFCOUNTER_CNTL                                                                0x1abe
+#define regDC_PERFMON14_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON14_PERFCOUNTER_CNTL2                                                               0x1abf
+#define regDC_PERFMON14_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON14_PERFCOUNTER_STATE                                                               0x1ac0
+#define regDC_PERFMON14_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON14_PERFMON_CNTL                                                                    0x1ac1
+#define regDC_PERFMON14_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON14_PERFMON_CNTL2                                                                   0x1ac2
+#define regDC_PERFMON14_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON14_PERFMON_CVALUE_INT_MISC                                                         0x1ac3
+#define regDC_PERFMON14_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON14_PERFMON_CVALUE_LOW                                                              0x1ac4
+#define regDC_PERFMON14_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON14_PERFMON_HI                                                                      0x1ac5
+#define regDC_PERFMON14_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON14_PERFMON_LOW                                                                     0x1ac6
+#define regDC_PERFMON14_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_optc_odm0_dispdec
+// base address: 0x0
+#define regODM0_OPTC_INPUT_GLOBAL_CONTROL                                                               0x1aca
+#define regODM0_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX                                                      2
+#define regODM0_OPTC_RSMU_UNDERFLOW                                                                     0x1acb
+#define regODM0_OPTC_RSMU_UNDERFLOW_BASE_IDX                                                            2
+#define regODM0_OPTC_UNDERFLOW_THRESHOLD                                                                0x1acc
+#define regODM0_OPTC_UNDERFLOW_THRESHOLD_BASE_IDX                                                       2
+#define regODM0_OPTC_DATA_SOURCE_SELECT                                                                 0x1acd
+#define regODM0_OPTC_DATA_SOURCE_SELECT_BASE_IDX                                                        2
+#define regODM0_OPTC_DATA_FORMAT_CONTROL                                                                0x1ace
+#define regODM0_OPTC_DATA_FORMAT_CONTROL_BASE_IDX                                                       2
+#define regODM0_OPTC_BYTES_PER_PIXEL                                                                    0x1acf
+#define regODM0_OPTC_BYTES_PER_PIXEL_BASE_IDX                                                           2
+#define regODM0_OPTC_WIDTH_CONTROL                                                                      0x1ad0
+#define regODM0_OPTC_WIDTH_CONTROL_BASE_IDX                                                             2
+#define regODM0_OPTC_WIDTH_CONTROL2                                                                     0x1ad1
+#define regODM0_OPTC_WIDTH_CONTROL2_BASE_IDX                                                            2
+#define regODM0_OPTC_INPUT_CLOCK_CONTROL                                                                0x1ad2
+#define regODM0_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX                                                       2
+#define regODM0_OPTC_MEMORY_CONFIG                                                                      0x1ad3
+#define regODM0_OPTC_MEMORY_CONFIG_BASE_IDX                                                             2
+#define regODM0_OPTC_INPUT_SPARE_REGISTER                                                               0x1ad4
+#define regODM0_OPTC_INPUT_SPARE_REGISTER_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_optc_odm1_dispdec
+// base address: 0x40
+#define regODM1_OPTC_INPUT_GLOBAL_CONTROL                                                               0x1ada
+#define regODM1_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX                                                      2
+#define regODM1_OPTC_UNDERFLOW_THRESHOLD                                                                0x1adc
+#define regODM1_OPTC_UNDERFLOW_THRESHOLD_BASE_IDX                                                       2
+#define regODM1_OPTC_DATA_SOURCE_SELECT                                                                 0x1add
+#define regODM1_OPTC_DATA_SOURCE_SELECT_BASE_IDX                                                        2
+#define regODM1_OPTC_DATA_FORMAT_CONTROL                                                                0x1ade
+#define regODM1_OPTC_DATA_FORMAT_CONTROL_BASE_IDX                                                       2
+#define regODM1_OPTC_BYTES_PER_PIXEL                                                                    0x1adf
+#define regODM1_OPTC_BYTES_PER_PIXEL_BASE_IDX                                                           2
+#define regODM1_OPTC_WIDTH_CONTROL                                                                      0x1ae0
+#define regODM1_OPTC_WIDTH_CONTROL_BASE_IDX                                                             2
+#define regODM1_OPTC_WIDTH_CONTROL2                                                                     0x1ae1
+#define regODM1_OPTC_WIDTH_CONTROL2_BASE_IDX                                                            2
+#define regODM1_OPTC_INPUT_CLOCK_CONTROL                                                                0x1ae2
+#define regODM1_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX                                                       2
+#define regODM1_OPTC_MEMORY_CONFIG                                                                      0x1ae3
+#define regODM1_OPTC_MEMORY_CONFIG_BASE_IDX                                                             2
+#define regODM1_OPTC_INPUT_SPARE_REGISTER                                                               0x1ae4
+#define regODM1_OPTC_INPUT_SPARE_REGISTER_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_optc_odm2_dispdec
+// base address: 0x80
+#define regODM2_OPTC_INPUT_GLOBAL_CONTROL                                                               0x1aea
+#define regODM2_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX                                                      2
+#define regODM2_OPTC_UNDERFLOW_THRESHOLD                                                                0x1aec
+#define regODM2_OPTC_UNDERFLOW_THRESHOLD_BASE_IDX                                                       2
+#define regODM2_OPTC_DATA_SOURCE_SELECT                                                                 0x1aed
+#define regODM2_OPTC_DATA_SOURCE_SELECT_BASE_IDX                                                        2
+#define regODM2_OPTC_DATA_FORMAT_CONTROL                                                                0x1aee
+#define regODM2_OPTC_DATA_FORMAT_CONTROL_BASE_IDX                                                       2
+#define regODM2_OPTC_BYTES_PER_PIXEL                                                                    0x1aef
+#define regODM2_OPTC_BYTES_PER_PIXEL_BASE_IDX                                                           2
+#define regODM2_OPTC_WIDTH_CONTROL                                                                      0x1af0
+#define regODM2_OPTC_WIDTH_CONTROL_BASE_IDX                                                             2
+#define regODM2_OPTC_WIDTH_CONTROL2                                                                     0x1af1
+#define regODM2_OPTC_WIDTH_CONTROL2_BASE_IDX                                                            2
+#define regODM2_OPTC_INPUT_CLOCK_CONTROL                                                                0x1af2
+#define regODM2_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX                                                       2
+#define regODM2_OPTC_MEMORY_CONFIG                                                                      0x1af3
+#define regODM2_OPTC_MEMORY_CONFIG_BASE_IDX                                                             2
+#define regODM2_OPTC_INPUT_SPARE_REGISTER                                                               0x1af4
+#define regODM2_OPTC_INPUT_SPARE_REGISTER_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_optc_odm3_dispdec
+// base address: 0xc0
+#define regODM3_OPTC_INPUT_GLOBAL_CONTROL                                                               0x1afa
+#define regODM3_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX                                                      2
+#define regODM3_OPTC_UNDERFLOW_THRESHOLD                                                                0x1afc
+#define regODM3_OPTC_UNDERFLOW_THRESHOLD_BASE_IDX                                                       2
+#define regODM3_OPTC_DATA_SOURCE_SELECT                                                                 0x1afd
+#define regODM3_OPTC_DATA_SOURCE_SELECT_BASE_IDX                                                        2
+#define regODM3_OPTC_DATA_FORMAT_CONTROL                                                                0x1afe
+#define regODM3_OPTC_DATA_FORMAT_CONTROL_BASE_IDX                                                       2
+#define regODM3_OPTC_BYTES_PER_PIXEL                                                                    0x1aff
+#define regODM3_OPTC_BYTES_PER_PIXEL_BASE_IDX                                                           2
+#define regODM3_OPTC_WIDTH_CONTROL                                                                      0x1b00
+#define regODM3_OPTC_WIDTH_CONTROL_BASE_IDX                                                             2
+#define regODM3_OPTC_WIDTH_CONTROL2                                                                     0x1b01
+#define regODM3_OPTC_WIDTH_CONTROL2_BASE_IDX                                                            2
+#define regODM3_OPTC_INPUT_CLOCK_CONTROL                                                                0x1b02
+#define regODM3_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX                                                       2
+#define regODM3_OPTC_MEMORY_CONFIG                                                                      0x1b03
+#define regODM3_OPTC_MEMORY_CONFIG_BASE_IDX                                                             2
+#define regODM3_OPTC_INPUT_SPARE_REGISTER                                                               0x1b04
+#define regODM3_OPTC_INPUT_SPARE_REGISTER_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_optc_otg0_dispdec
+// base address: 0x0
+#define regOTG0_OTG_H_TOTAL                                                                             0x1b2a
+#define regOTG0_OTG_H_TOTAL_BASE_IDX                                                                    2
+#define regOTG0_OTG_H_BLANK_START_END                                                                   0x1b2b
+#define regOTG0_OTG_H_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG0_OTG_H_SYNC_A                                                                            0x1b2c
+#define regOTG0_OTG_H_SYNC_A_BASE_IDX                                                                   2
+#define regOTG0_OTG_H_SYNC_A_CNTL                                                                       0x1b2d
+#define regOTG0_OTG_H_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG0_OTG_H_TIMING_CNTL                                                                       0x1b2e
+#define regOTG0_OTG_H_TIMING_CNTL_BASE_IDX                                                              2
+#define regOTG0_OTG_V_TOTAL                                                                             0x1b2f
+#define regOTG0_OTG_V_TOTAL_BASE_IDX                                                                    2
+#define regOTG0_OTG_V_TOTAL_MIN                                                                         0x1b30
+#define regOTG0_OTG_V_TOTAL_MIN_BASE_IDX                                                                2
+#define regOTG0_OTG_V_TOTAL_MAX                                                                         0x1b31
+#define regOTG0_OTG_V_TOTAL_MAX_BASE_IDX                                                                2
+#define regOTG0_OTG_V_TOTAL_MID                                                                         0x1b32
+#define regOTG0_OTG_V_TOTAL_MID_BASE_IDX                                                                2
+#define regOTG0_OTG_V_TOTAL_CONTROL                                                                     0x1b33
+#define regOTG0_OTG_V_TOTAL_CONTROL_BASE_IDX                                                            2
+#define regOTG0_OTG_V_COUNT_STOP_CONTROL                                                                0x1b34
+#define regOTG0_OTG_V_COUNT_STOP_CONTROL_BASE_IDX                                                       2
+#define regOTG0_OTG_V_COUNT_STOP_CONTROL2                                                               0x1b35
+#define regOTG0_OTG_V_COUNT_STOP_CONTROL2_BASE_IDX                                                      2
+#define regOTG0_OTG_V_TOTAL_INT_STATUS                                                                  0x1b36
+#define regOTG0_OTG_V_TOTAL_INT_STATUS_BASE_IDX                                                         2
+#define regOTG0_OTG_VSYNC_NOM_INT_STATUS                                                                0x1b37
+#define regOTG0_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX                                                       2
+#define regOTG0_OTG_V_BLANK_START_END                                                                   0x1b38
+#define regOTG0_OTG_V_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG0_OTG_V_SYNC_A                                                                            0x1b39
+#define regOTG0_OTG_V_SYNC_A_BASE_IDX                                                                   2
+#define regOTG0_OTG_V_SYNC_A_CNTL                                                                       0x1b3a
+#define regOTG0_OTG_V_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG0_OTG_TRIGA_CNTL                                                                          0x1b3b
+#define regOTG0_OTG_TRIGA_CNTL_BASE_IDX                                                                 2
+#define regOTG0_OTG_TRIGA_MANUAL_TRIG                                                                   0x1b3c
+#define regOTG0_OTG_TRIGA_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG0_OTG_TRIGB_CNTL                                                                          0x1b3d
+#define regOTG0_OTG_TRIGB_CNTL_BASE_IDX                                                                 2
+#define regOTG0_OTG_TRIGB_MANUAL_TRIG                                                                   0x1b3e
+#define regOTG0_OTG_TRIGB_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG0_OTG_FORCE_COUNT_NOW_CNTL                                                                0x1b3f
+#define regOTG0_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX                                                       2
+#define regOTG0_OTG_STEREO_FORCE_NEXT_EYE                                                               0x1b41
+#define regOTG0_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX                                                      2
+#define regOTG0_OTG_CONTROL                                                                             0x1b43
+#define regOTG0_OTG_CONTROL_BASE_IDX                                                                    2
+#define regOTG0_OTG_DLPC_CONTROL                                                                        0x1b44
+#define regOTG0_OTG_DLPC_CONTROL_BASE_IDX                                                               2
+#define regOTG0_OTG_PWA_FRAME_SYNC_CONTROL                                                              0x1b45
+#define regOTG0_OTG_PWA_FRAME_SYNC_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_INTERLACE_CONTROL                                                                   0x1b46
+#define regOTG0_OTG_INTERLACE_CONTROL_BASE_IDX                                                          2
+#define regOTG0_OTG_INTERLACE_STATUS                                                                    0x1b47
+#define regOTG0_OTG_INTERLACE_STATUS_BASE_IDX                                                           2
+#define regOTG0_OTG_PIXEL_DATA_READBACK0                                                                0x1b48
+#define regOTG0_OTG_PIXEL_DATA_READBACK0_BASE_IDX                                                       2
+#define regOTG0_OTG_PIXEL_DATA_READBACK1                                                                0x1b49
+#define regOTG0_OTG_PIXEL_DATA_READBACK1_BASE_IDX                                                       2
+#define regOTG0_OTG_STATUS                                                                              0x1b4a
+#define regOTG0_OTG_STATUS_BASE_IDX                                                                     2
+#define regOTG0_OTG_STATUS_POSITION                                                                     0x1b4b
+#define regOTG0_OTG_STATUS_POSITION_BASE_IDX                                                            2
+#define regOTG0_OTG_LONG_VBLANK_STATUS                                                                  0x1b4c
+#define regOTG0_OTG_LONG_VBLANK_STATUS_BASE_IDX                                                         2
+#define regOTG0_OTG_NOM_VERT_POSITION                                                                   0x1b4d
+#define regOTG0_OTG_NOM_VERT_POSITION_BASE_IDX                                                          2
+#define regOTG0_OTG_STATUS_FRAME_COUNT                                                                  0x1b4e
+#define regOTG0_OTG_STATUS_FRAME_COUNT_BASE_IDX                                                         2
+#define regOTG0_OTG_STATUS_VF_COUNT                                                                     0x1b4f
+#define regOTG0_OTG_STATUS_VF_COUNT_BASE_IDX                                                            2
+#define regOTG0_OTG_STATUS_HV_COUNT                                                                     0x1b50
+#define regOTG0_OTG_STATUS_HV_COUNT_BASE_IDX                                                            2
+#define regOTG0_OTG_COUNT_CONTROL                                                                       0x1b51
+#define regOTG0_OTG_COUNT_CONTROL_BASE_IDX                                                              2
+#define regOTG0_OTG_COUNT_RESET                                                                         0x1b52
+#define regOTG0_OTG_COUNT_RESET_BASE_IDX                                                                2
+#define regOTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE                                                        0x1b53
+#define regOTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX                                               2
+#define regOTG0_OTG_VERT_SYNC_CONTROL                                                                   0x1b54
+#define regOTG0_OTG_VERT_SYNC_CONTROL_BASE_IDX                                                          2
+#define regOTG0_OTG_STEREO_STATUS                                                                       0x1b55
+#define regOTG0_OTG_STEREO_STATUS_BASE_IDX                                                              2
+#define regOTG0_OTG_STEREO_CONTROL                                                                      0x1b56
+#define regOTG0_OTG_STEREO_CONTROL_BASE_IDX                                                             2
+#define regOTG0_OTG_SNAPSHOT_STATUS                                                                     0x1b57
+#define regOTG0_OTG_SNAPSHOT_STATUS_BASE_IDX                                                            2
+#define regOTG0_OTG_SNAPSHOT_CONTROL                                                                    0x1b58
+#define regOTG0_OTG_SNAPSHOT_CONTROL_BASE_IDX                                                           2
+#define regOTG0_OTG_SNAPSHOT_POSITION                                                                   0x1b59
+#define regOTG0_OTG_SNAPSHOT_POSITION_BASE_IDX                                                          2
+#define regOTG0_OTG_SNAPSHOT_FRAME                                                                      0x1b5a
+#define regOTG0_OTG_SNAPSHOT_FRAME_BASE_IDX                                                             2
+#define regOTG0_OTG_INTERRUPT_CONTROL                                                                   0x1b5b
+#define regOTG0_OTG_INTERRUPT_CONTROL_BASE_IDX                                                          2
+#define regOTG0_OTG_UPDATE_LOCK                                                                         0x1b5c
+#define regOTG0_OTG_UPDATE_LOCK_BASE_IDX                                                                2
+#define regOTG0_OTG_DOUBLE_BUFFER_CONTROL                                                               0x1b5d
+#define regOTG0_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX                                                      2
+#define regOTG0_OTG_MASTER_EN                                                                           0x1b5e
+#define regOTG0_OTG_MASTER_EN_BASE_IDX                                                                  2
+#define regOTG0_OTG_VERTICAL_INTERRUPT0_POSITION                                                        0x1b60
+#define regOTG0_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX                                               2
+#define regOTG0_OTG_VERTICAL_INTERRUPT0_CONTROL                                                         0x1b61
+#define regOTG0_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX                                                2
+#define regOTG0_OTG_VERTICAL_INTERRUPT1_POSITION                                                        0x1b62
+#define regOTG0_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX                                               2
+#define regOTG0_OTG_VERTICAL_INTERRUPT1_CONTROL                                                         0x1b63
+#define regOTG0_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX                                                2
+#define regOTG0_OTG_VERTICAL_INTERRUPT2_POSITION                                                        0x1b64
+#define regOTG0_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX                                               2
+#define regOTG0_OTG_VERTICAL_INTERRUPT2_CONTROL                                                         0x1b65
+#define regOTG0_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX                                                2
+#define regOTG0_OTG_CRC_CNTL                                                                            0x1b66
+#define regOTG0_OTG_CRC_CNTL_BASE_IDX                                                                   2
+#define regOTG0_OTG_CRC0_WINDOWA_X_CONTROL                                                              0x1b67
+#define regOTG0_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC0_WINDOWA_Y_CONTROL                                                              0x1b68
+#define regOTG0_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC0_WINDOWB_X_CONTROL                                                              0x1b69
+#define regOTG0_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC0_WINDOWB_Y_CONTROL                                                              0x1b6a
+#define regOTG0_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC0_DATA_R                                                                         0x1b6b
+#define regOTG0_OTG_CRC0_DATA_R_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC0_DATA_G                                                                         0x1b6c
+#define regOTG0_OTG_CRC0_DATA_G_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC0_DATA_B                                                                         0x1b6d
+#define regOTG0_OTG_CRC0_DATA_B_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC0_DATA_C                                                                         0x1b6e
+#define regOTG0_OTG_CRC0_DATA_C_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC1_WINDOWA_X_CONTROL                                                              0x1b6f
+#define regOTG0_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC1_WINDOWA_Y_CONTROL                                                              0x1b70
+#define regOTG0_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC1_WINDOWB_X_CONTROL                                                              0x1b71
+#define regOTG0_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC1_WINDOWB_Y_CONTROL                                                              0x1b72
+#define regOTG0_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC1_DATA_R                                                                         0x1b73
+#define regOTG0_OTG_CRC1_DATA_R_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC1_DATA_G                                                                         0x1b74
+#define regOTG0_OTG_CRC1_DATA_G_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC1_DATA_B                                                                         0x1b75
+#define regOTG0_OTG_CRC1_DATA_B_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC1_DATA_C                                                                         0x1b76
+#define regOTG0_OTG_CRC1_DATA_C_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC2_DATA_R                                                                         0x1b77
+#define regOTG0_OTG_CRC2_DATA_R_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC2_DATA_G                                                                         0x1b78
+#define regOTG0_OTG_CRC2_DATA_G_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC2_DATA_B                                                                         0x1b79
+#define regOTG0_OTG_CRC2_DATA_B_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC2_DATA_C                                                                         0x1b7a
+#define regOTG0_OTG_CRC2_DATA_C_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC3_DATA_R                                                                         0x1b7b
+#define regOTG0_OTG_CRC3_DATA_R_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC3_DATA_G                                                                         0x1b7c
+#define regOTG0_OTG_CRC3_DATA_G_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC3_DATA_B                                                                         0x1b7d
+#define regOTG0_OTG_CRC3_DATA_B_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC3_DATA_C                                                                         0x1b7e
+#define regOTG0_OTG_CRC3_DATA_C_BASE_IDX                                                                2
+#define regOTG0_OTG_CRC0_DATA_AES                                                                       0x1b7f
+#define regOTG0_OTG_CRC0_DATA_AES_BASE_IDX                                                              2
+#define regOTG0_OTG_CRC_SIG_RED_GREEN_MASK                                                              0x1b80
+#define regOTG0_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX                                                     2
+#define regOTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK                                                           0x1b81
+#define regOTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX                                                  2
+#define regOTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK                                                     0x1b82
+#define regOTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK                                                     0x1b83
+#define regOTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK                                                     0x1b84
+#define regOTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK                                                     0x1b85
+#define regOTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK                                                     0x1b86
+#define regOTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK                                                     0x1b87
+#define regOTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK                                                     0x1b88
+#define regOTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK                                                     0x1b89
+#define regOTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG0_OTG_STATIC_SCREEN_CONTROL                                                               0x1b8a
+#define regOTG0_OTG_STATIC_SCREEN_CONTROL_BASE_IDX                                                      2
+#define regOTG0_OTG_3D_STRUCTURE_CONTROL                                                                0x1b8b
+#define regOTG0_OTG_3D_STRUCTURE_CONTROL_BASE_IDX                                                       2
+#define regOTG0_OTG_GSL_VSYNC_GAP                                                                       0x1b8c
+#define regOTG0_OTG_GSL_VSYNC_GAP_BASE_IDX                                                              2
+#define regOTG0_OTG_MASTER_UPDATE_MODE                                                                  0x1b8d
+#define regOTG0_OTG_MASTER_UPDATE_MODE_BASE_IDX                                                         2
+#define regOTG0_OTG_CLOCK_CONTROL                                                                       0x1b8e
+#define regOTG0_OTG_CLOCK_CONTROL_BASE_IDX                                                              2
+#define regOTG0_OTG_VSTARTUP_PARAM                                                                      0x1b8f
+#define regOTG0_OTG_VSTARTUP_PARAM_BASE_IDX                                                             2
+#define regOTG0_OTG_VUPDATE_PARAM                                                                       0x1b90
+#define regOTG0_OTG_VUPDATE_PARAM_BASE_IDX                                                              2
+#define regOTG0_OTG_VREADY_PARAM                                                                        0x1b91
+#define regOTG0_OTG_VREADY_PARAM_BASE_IDX                                                               2
+#define regOTG0_OTG_GLOBAL_SYNC_STATUS                                                                  0x1b92
+#define regOTG0_OTG_GLOBAL_SYNC_STATUS_BASE_IDX                                                         2
+#define regOTG0_OTG_MASTER_UPDATE_LOCK                                                                  0x1b93
+#define regOTG0_OTG_MASTER_UPDATE_LOCK_BASE_IDX                                                         2
+#define regOTG0_OTG_GSL_CONTROL                                                                         0x1b94
+#define regOTG0_OTG_GSL_CONTROL_BASE_IDX                                                                2
+#define regOTG0_OTG_GSL_WINDOW_X                                                                        0x1b95
+#define regOTG0_OTG_GSL_WINDOW_X_BASE_IDX                                                               2
+#define regOTG0_OTG_GSL_WINDOW_Y                                                                        0x1b96
+#define regOTG0_OTG_GSL_WINDOW_Y_BASE_IDX                                                               2
+#define regOTG0_OTG_VUPDATE_KEEPOUT                                                                     0x1b97
+#define regOTG0_OTG_VUPDATE_KEEPOUT_BASE_IDX                                                            2
+#define regOTG0_OTG_GLOBAL_CONTROL0                                                                     0x1b98
+#define regOTG0_OTG_GLOBAL_CONTROL0_BASE_IDX                                                            2
+#define regOTG0_OTG_GLOBAL_CONTROL1                                                                     0x1b99
+#define regOTG0_OTG_GLOBAL_CONTROL1_BASE_IDX                                                            2
+#define regOTG0_OTG_GLOBAL_CONTROL2                                                                     0x1b9a
+#define regOTG0_OTG_GLOBAL_CONTROL2_BASE_IDX                                                            2
+#define regOTG0_OTG_GLOBAL_CONTROL3                                                                     0x1b9b
+#define regOTG0_OTG_GLOBAL_CONTROL3_BASE_IDX                                                            2
+#define regOTG0_OTG_GLOBAL_CONTROL4                                                                     0x1b9c
+#define regOTG0_OTG_GLOBAL_CONTROL4_BASE_IDX                                                            2
+#define regOTG0_OTG_TRIG_MANUAL_CONTROL                                                                 0x1b9d
+#define regOTG0_OTG_TRIG_MANUAL_CONTROL_BASE_IDX                                                        2
+#define regOTG0_OTG_DRR_TIMING_INT_STATUS                                                               0x1b9f
+#define regOTG0_OTG_DRR_TIMING_INT_STATUS_BASE_IDX                                                      2
+#define regOTG0_OTG_DRR_V_TOTAL_REACH_RANGE                                                             0x1ba0
+#define regOTG0_OTG_DRR_V_TOTAL_REACH_RANGE_BASE_IDX                                                    2
+#define regOTG0_OTG_DRR_V_TOTAL_CHANGE                                                                  0x1ba1
+#define regOTG0_OTG_DRR_V_TOTAL_CHANGE_BASE_IDX                                                         2
+#define regOTG0_OTG_DRR_TRIGGER_WINDOW                                                                  0x1ba2
+#define regOTG0_OTG_DRR_TRIGGER_WINDOW_BASE_IDX                                                         2
+#define regOTG0_OTG_DRR_CONTROL                                                                         0x1ba3
+#define regOTG0_OTG_DRR_CONTROL_BASE_IDX                                                                2
+#define regOTG0_OTG_DRR_CONTOL2                                                                         0x1ba4
+#define regOTG0_OTG_DRR_CONTOL2_BASE_IDX                                                                2
+#define regOTG0_OTG_M_CONST_DTO0                                                                        0x1ba5
+#define regOTG0_OTG_M_CONST_DTO0_BASE_IDX                                                               2
+#define regOTG0_OTG_M_CONST_DTO1                                                                        0x1ba6
+#define regOTG0_OTG_M_CONST_DTO1_BASE_IDX                                                               2
+#define regOTG0_OTG_REQUEST_CONTROL                                                                     0x1ba7
+#define regOTG0_OTG_REQUEST_CONTROL_BASE_IDX                                                            2
+#define regOTG0_OTG_PIPE_UPDATE_STATUS                                                                  0x1ba8
+#define regOTG0_OTG_PIPE_UPDATE_STATUS_BASE_IDX                                                         2
+#define regOTG0_OTG_PSTATE_REGISTER                                                                     0x1ba9
+#define regOTG0_OTG_PSTATE_REGISTER_BASE_IDX                                                            2
+#define regOTG0_OTG_ENCRYPTION                                                                          0x1baa
+#define regOTG0_OTG_ENCRYPTION_BASE_IDX                                                                 2
+#define regOTG0_OTG_SPARE_REGISTER                                                                      0x1bab
+#define regOTG0_OTG_SPARE_REGISTER_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_optc_otg1_dispdec
+// base address: 0x220
+#define regOTG1_OTG_H_TOTAL                                                                             0x1bb2
+#define regOTG1_OTG_H_TOTAL_BASE_IDX                                                                    2
+#define regOTG1_OTG_H_BLANK_START_END                                                                   0x1bb3
+#define regOTG1_OTG_H_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG1_OTG_H_SYNC_A                                                                            0x1bb4
+#define regOTG1_OTG_H_SYNC_A_BASE_IDX                                                                   2
+#define regOTG1_OTG_H_SYNC_A_CNTL                                                                       0x1bb5
+#define regOTG1_OTG_H_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG1_OTG_H_TIMING_CNTL                                                                       0x1bb6
+#define regOTG1_OTG_H_TIMING_CNTL_BASE_IDX                                                              2
+#define regOTG1_OTG_V_TOTAL                                                                             0x1bb7
+#define regOTG1_OTG_V_TOTAL_BASE_IDX                                                                    2
+#define regOTG1_OTG_V_TOTAL_MIN                                                                         0x1bb8
+#define regOTG1_OTG_V_TOTAL_MIN_BASE_IDX                                                                2
+#define regOTG1_OTG_V_TOTAL_MAX                                                                         0x1bb9
+#define regOTG1_OTG_V_TOTAL_MAX_BASE_IDX                                                                2
+#define regOTG1_OTG_V_TOTAL_MID                                                                         0x1bba
+#define regOTG1_OTG_V_TOTAL_MID_BASE_IDX                                                                2
+#define regOTG1_OTG_V_TOTAL_CONTROL                                                                     0x1bbb
+#define regOTG1_OTG_V_TOTAL_CONTROL_BASE_IDX                                                            2
+#define regOTG1_OTG_V_COUNT_STOP_CONTROL                                                                0x1bbc
+#define regOTG1_OTG_V_COUNT_STOP_CONTROL_BASE_IDX                                                       2
+#define regOTG1_OTG_V_COUNT_STOP_CONTROL2                                                               0x1bbd
+#define regOTG1_OTG_V_COUNT_STOP_CONTROL2_BASE_IDX                                                      2
+#define regOTG1_OTG_V_TOTAL_INT_STATUS                                                                  0x1bbe
+#define regOTG1_OTG_V_TOTAL_INT_STATUS_BASE_IDX                                                         2
+#define regOTG1_OTG_VSYNC_NOM_INT_STATUS                                                                0x1bbf
+#define regOTG1_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX                                                       2
+#define regOTG1_OTG_V_BLANK_START_END                                                                   0x1bc0
+#define regOTG1_OTG_V_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG1_OTG_V_SYNC_A                                                                            0x1bc1
+#define regOTG1_OTG_V_SYNC_A_BASE_IDX                                                                   2
+#define regOTG1_OTG_V_SYNC_A_CNTL                                                                       0x1bc2
+#define regOTG1_OTG_V_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG1_OTG_TRIGA_CNTL                                                                          0x1bc3
+#define regOTG1_OTG_TRIGA_CNTL_BASE_IDX                                                                 2
+#define regOTG1_OTG_TRIGA_MANUAL_TRIG                                                                   0x1bc4
+#define regOTG1_OTG_TRIGA_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG1_OTG_TRIGB_CNTL                                                                          0x1bc5
+#define regOTG1_OTG_TRIGB_CNTL_BASE_IDX                                                                 2
+#define regOTG1_OTG_TRIGB_MANUAL_TRIG                                                                   0x1bc6
+#define regOTG1_OTG_TRIGB_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG1_OTG_FORCE_COUNT_NOW_CNTL                                                                0x1bc7
+#define regOTG1_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX                                                       2
+#define regOTG1_OTG_STEREO_FORCE_NEXT_EYE                                                               0x1bc9
+#define regOTG1_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX                                                      2
+#define regOTG1_OTG_CONTROL                                                                             0x1bcb
+#define regOTG1_OTG_CONTROL_BASE_IDX                                                                    2
+#define regOTG1_OTG_DLPC_CONTROL                                                                        0x1bcc
+#define regOTG1_OTG_DLPC_CONTROL_BASE_IDX                                                               2
+#define regOTG1_OTG_PWA_FRAME_SYNC_CONTROL                                                              0x1bcd
+#define regOTG1_OTG_PWA_FRAME_SYNC_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_INTERLACE_CONTROL                                                                   0x1bce
+#define regOTG1_OTG_INTERLACE_CONTROL_BASE_IDX                                                          2
+#define regOTG1_OTG_INTERLACE_STATUS                                                                    0x1bcf
+#define regOTG1_OTG_INTERLACE_STATUS_BASE_IDX                                                           2
+#define regOTG1_OTG_PIXEL_DATA_READBACK0                                                                0x1bd0
+#define regOTG1_OTG_PIXEL_DATA_READBACK0_BASE_IDX                                                       2
+#define regOTG1_OTG_PIXEL_DATA_READBACK1                                                                0x1bd1
+#define regOTG1_OTG_PIXEL_DATA_READBACK1_BASE_IDX                                                       2
+#define regOTG1_OTG_STATUS                                                                              0x1bd2
+#define regOTG1_OTG_STATUS_BASE_IDX                                                                     2
+#define regOTG1_OTG_STATUS_POSITION                                                                     0x1bd3
+#define regOTG1_OTG_STATUS_POSITION_BASE_IDX                                                            2
+#define regOTG1_OTG_LONG_VBLANK_STATUS                                                                  0x1bd4
+#define regOTG1_OTG_LONG_VBLANK_STATUS_BASE_IDX                                                         2
+#define regOTG1_OTG_NOM_VERT_POSITION                                                                   0x1bd5
+#define regOTG1_OTG_NOM_VERT_POSITION_BASE_IDX                                                          2
+#define regOTG1_OTG_STATUS_FRAME_COUNT                                                                  0x1bd6
+#define regOTG1_OTG_STATUS_FRAME_COUNT_BASE_IDX                                                         2
+#define regOTG1_OTG_STATUS_VF_COUNT                                                                     0x1bd7
+#define regOTG1_OTG_STATUS_VF_COUNT_BASE_IDX                                                            2
+#define regOTG1_OTG_STATUS_HV_COUNT                                                                     0x1bd8
+#define regOTG1_OTG_STATUS_HV_COUNT_BASE_IDX                                                            2
+#define regOTG1_OTG_COUNT_CONTROL                                                                       0x1bd9
+#define regOTG1_OTG_COUNT_CONTROL_BASE_IDX                                                              2
+#define regOTG1_OTG_COUNT_RESET                                                                         0x1bda
+#define regOTG1_OTG_COUNT_RESET_BASE_IDX                                                                2
+#define regOTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE                                                        0x1bdb
+#define regOTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX                                               2
+#define regOTG1_OTG_VERT_SYNC_CONTROL                                                                   0x1bdc
+#define regOTG1_OTG_VERT_SYNC_CONTROL_BASE_IDX                                                          2
+#define regOTG1_OTG_STEREO_STATUS                                                                       0x1bdd
+#define regOTG1_OTG_STEREO_STATUS_BASE_IDX                                                              2
+#define regOTG1_OTG_STEREO_CONTROL                                                                      0x1bde
+#define regOTG1_OTG_STEREO_CONTROL_BASE_IDX                                                             2
+#define regOTG1_OTG_SNAPSHOT_STATUS                                                                     0x1bdf
+#define regOTG1_OTG_SNAPSHOT_STATUS_BASE_IDX                                                            2
+#define regOTG1_OTG_SNAPSHOT_CONTROL                                                                    0x1be0
+#define regOTG1_OTG_SNAPSHOT_CONTROL_BASE_IDX                                                           2
+#define regOTG1_OTG_SNAPSHOT_POSITION                                                                   0x1be1
+#define regOTG1_OTG_SNAPSHOT_POSITION_BASE_IDX                                                          2
+#define regOTG1_OTG_SNAPSHOT_FRAME                                                                      0x1be2
+#define regOTG1_OTG_SNAPSHOT_FRAME_BASE_IDX                                                             2
+#define regOTG1_OTG_INTERRUPT_CONTROL                                                                   0x1be3
+#define regOTG1_OTG_INTERRUPT_CONTROL_BASE_IDX                                                          2
+#define regOTG1_OTG_UPDATE_LOCK                                                                         0x1be4
+#define regOTG1_OTG_UPDATE_LOCK_BASE_IDX                                                                2
+#define regOTG1_OTG_DOUBLE_BUFFER_CONTROL                                                               0x1be5
+#define regOTG1_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX                                                      2
+#define regOTG1_OTG_MASTER_EN                                                                           0x1be6
+#define regOTG1_OTG_MASTER_EN_BASE_IDX                                                                  2
+#define regOTG1_OTG_VERTICAL_INTERRUPT0_POSITION                                                        0x1be8
+#define regOTG1_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX                                               2
+#define regOTG1_OTG_VERTICAL_INTERRUPT0_CONTROL                                                         0x1be9
+#define regOTG1_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX                                                2
+#define regOTG1_OTG_VERTICAL_INTERRUPT1_POSITION                                                        0x1bea
+#define regOTG1_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX                                               2
+#define regOTG1_OTG_VERTICAL_INTERRUPT1_CONTROL                                                         0x1beb
+#define regOTG1_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX                                                2
+#define regOTG1_OTG_VERTICAL_INTERRUPT2_POSITION                                                        0x1bec
+#define regOTG1_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX                                               2
+#define regOTG1_OTG_VERTICAL_INTERRUPT2_CONTROL                                                         0x1bed
+#define regOTG1_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX                                                2
+#define regOTG1_OTG_CRC_CNTL                                                                            0x1bee
+#define regOTG1_OTG_CRC_CNTL_BASE_IDX                                                                   2
+#define regOTG1_OTG_CRC0_WINDOWA_X_CONTROL                                                              0x1bef
+#define regOTG1_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC0_WINDOWA_Y_CONTROL                                                              0x1bf0
+#define regOTG1_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC0_WINDOWB_X_CONTROL                                                              0x1bf1
+#define regOTG1_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC0_WINDOWB_Y_CONTROL                                                              0x1bf2
+#define regOTG1_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC0_DATA_R                                                                         0x1bf3
+#define regOTG1_OTG_CRC0_DATA_R_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC0_DATA_G                                                                         0x1bf4
+#define regOTG1_OTG_CRC0_DATA_G_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC0_DATA_B                                                                         0x1bf5
+#define regOTG1_OTG_CRC0_DATA_B_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC0_DATA_C                                                                         0x1bf6
+#define regOTG1_OTG_CRC0_DATA_C_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC1_WINDOWA_X_CONTROL                                                              0x1bf7
+#define regOTG1_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC1_WINDOWA_Y_CONTROL                                                              0x1bf8
+#define regOTG1_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC1_WINDOWB_X_CONTROL                                                              0x1bf9
+#define regOTG1_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC1_WINDOWB_Y_CONTROL                                                              0x1bfa
+#define regOTG1_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC1_DATA_R                                                                         0x1bfb
+#define regOTG1_OTG_CRC1_DATA_R_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC1_DATA_G                                                                         0x1bfc
+#define regOTG1_OTG_CRC1_DATA_G_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC1_DATA_B                                                                         0x1bfd
+#define regOTG1_OTG_CRC1_DATA_B_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC1_DATA_C                                                                         0x1bfe
+#define regOTG1_OTG_CRC1_DATA_C_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC2_DATA_R                                                                         0x1bff
+#define regOTG1_OTG_CRC2_DATA_R_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC2_DATA_G                                                                         0x1c00
+#define regOTG1_OTG_CRC2_DATA_G_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC2_DATA_B                                                                         0x1c01
+#define regOTG1_OTG_CRC2_DATA_B_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC2_DATA_C                                                                         0x1c02
+#define regOTG1_OTG_CRC2_DATA_C_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC3_DATA_R                                                                         0x1c03
+#define regOTG1_OTG_CRC3_DATA_R_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC3_DATA_G                                                                         0x1c04
+#define regOTG1_OTG_CRC3_DATA_G_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC3_DATA_B                                                                         0x1c05
+#define regOTG1_OTG_CRC3_DATA_B_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC3_DATA_C                                                                         0x1c06
+#define regOTG1_OTG_CRC3_DATA_C_BASE_IDX                                                                2
+#define regOTG1_OTG_CRC0_DATA_AES                                                                       0x1c07
+#define regOTG1_OTG_CRC0_DATA_AES_BASE_IDX                                                              2
+#define regOTG1_OTG_CRC_SIG_RED_GREEN_MASK                                                              0x1c08
+#define regOTG1_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX                                                     2
+#define regOTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK                                                           0x1c09
+#define regOTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX                                                  2
+#define regOTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK                                                     0x1c0a
+#define regOTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK                                                     0x1c0b
+#define regOTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK                                                     0x1c0c
+#define regOTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK                                                     0x1c0d
+#define regOTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK                                                     0x1c0e
+#define regOTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK                                                     0x1c0f
+#define regOTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK                                                     0x1c10
+#define regOTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK                                                     0x1c11
+#define regOTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG1_OTG_STATIC_SCREEN_CONTROL                                                               0x1c12
+#define regOTG1_OTG_STATIC_SCREEN_CONTROL_BASE_IDX                                                      2
+#define regOTG1_OTG_3D_STRUCTURE_CONTROL                                                                0x1c13
+#define regOTG1_OTG_3D_STRUCTURE_CONTROL_BASE_IDX                                                       2
+#define regOTG1_OTG_GSL_VSYNC_GAP                                                                       0x1c14
+#define regOTG1_OTG_GSL_VSYNC_GAP_BASE_IDX                                                              2
+#define regOTG1_OTG_MASTER_UPDATE_MODE                                                                  0x1c15
+#define regOTG1_OTG_MASTER_UPDATE_MODE_BASE_IDX                                                         2
+#define regOTG1_OTG_CLOCK_CONTROL                                                                       0x1c16
+#define regOTG1_OTG_CLOCK_CONTROL_BASE_IDX                                                              2
+#define regOTG1_OTG_VSTARTUP_PARAM                                                                      0x1c17
+#define regOTG1_OTG_VSTARTUP_PARAM_BASE_IDX                                                             2
+#define regOTG1_OTG_VUPDATE_PARAM                                                                       0x1c18
+#define regOTG1_OTG_VUPDATE_PARAM_BASE_IDX                                                              2
+#define regOTG1_OTG_VREADY_PARAM                                                                        0x1c19
+#define regOTG1_OTG_VREADY_PARAM_BASE_IDX                                                               2
+#define regOTG1_OTG_GLOBAL_SYNC_STATUS                                                                  0x1c1a
+#define regOTG1_OTG_GLOBAL_SYNC_STATUS_BASE_IDX                                                         2
+#define regOTG1_OTG_MASTER_UPDATE_LOCK                                                                  0x1c1b
+#define regOTG1_OTG_MASTER_UPDATE_LOCK_BASE_IDX                                                         2
+#define regOTG1_OTG_GSL_CONTROL                                                                         0x1c1c
+#define regOTG1_OTG_GSL_CONTROL_BASE_IDX                                                                2
+#define regOTG1_OTG_GSL_WINDOW_X                                                                        0x1c1d
+#define regOTG1_OTG_GSL_WINDOW_X_BASE_IDX                                                               2
+#define regOTG1_OTG_GSL_WINDOW_Y                                                                        0x1c1e
+#define regOTG1_OTG_GSL_WINDOW_Y_BASE_IDX                                                               2
+#define regOTG1_OTG_VUPDATE_KEEPOUT                                                                     0x1c1f
+#define regOTG1_OTG_VUPDATE_KEEPOUT_BASE_IDX                                                            2
+#define regOTG1_OTG_GLOBAL_CONTROL0                                                                     0x1c20
+#define regOTG1_OTG_GLOBAL_CONTROL0_BASE_IDX                                                            2
+#define regOTG1_OTG_GLOBAL_CONTROL1                                                                     0x1c21
+#define regOTG1_OTG_GLOBAL_CONTROL1_BASE_IDX                                                            2
+#define regOTG1_OTG_GLOBAL_CONTROL2                                                                     0x1c22
+#define regOTG1_OTG_GLOBAL_CONTROL2_BASE_IDX                                                            2
+#define regOTG1_OTG_GLOBAL_CONTROL3                                                                     0x1c23
+#define regOTG1_OTG_GLOBAL_CONTROL3_BASE_IDX                                                            2
+#define regOTG1_OTG_GLOBAL_CONTROL4                                                                     0x1c24
+#define regOTG1_OTG_GLOBAL_CONTROL4_BASE_IDX                                                            2
+#define regOTG1_OTG_TRIG_MANUAL_CONTROL                                                                 0x1c25
+#define regOTG1_OTG_TRIG_MANUAL_CONTROL_BASE_IDX                                                        2
+#define regOTG1_OTG_DRR_TIMING_INT_STATUS                                                               0x1c27
+#define regOTG1_OTG_DRR_TIMING_INT_STATUS_BASE_IDX                                                      2
+#define regOTG1_OTG_DRR_V_TOTAL_REACH_RANGE                                                             0x1c28
+#define regOTG1_OTG_DRR_V_TOTAL_REACH_RANGE_BASE_IDX                                                    2
+#define regOTG1_OTG_DRR_V_TOTAL_CHANGE                                                                  0x1c29
+#define regOTG1_OTG_DRR_V_TOTAL_CHANGE_BASE_IDX                                                         2
+#define regOTG1_OTG_DRR_TRIGGER_WINDOW                                                                  0x1c2a
+#define regOTG1_OTG_DRR_TRIGGER_WINDOW_BASE_IDX                                                         2
+#define regOTG1_OTG_DRR_CONTROL                                                                         0x1c2b
+#define regOTG1_OTG_DRR_CONTROL_BASE_IDX                                                                2
+#define regOTG1_OTG_DRR_CONTOL2                                                                         0x1c2c
+#define regOTG1_OTG_DRR_CONTOL2_BASE_IDX                                                                2
+#define regOTG1_OTG_M_CONST_DTO0                                                                        0x1c2d
+#define regOTG1_OTG_M_CONST_DTO0_BASE_IDX                                                               2
+#define regOTG1_OTG_M_CONST_DTO1                                                                        0x1c2e
+#define regOTG1_OTG_M_CONST_DTO1_BASE_IDX                                                               2
+#define regOTG1_OTG_REQUEST_CONTROL                                                                     0x1c2f
+#define regOTG1_OTG_REQUEST_CONTROL_BASE_IDX                                                            2
+#define regOTG1_OTG_PIPE_UPDATE_STATUS                                                                  0x1c30
+#define regOTG1_OTG_PIPE_UPDATE_STATUS_BASE_IDX                                                         2
+#define regOTG1_OTG_PSTATE_REGISTER                                                                     0x1c31
+#define regOTG1_OTG_PSTATE_REGISTER_BASE_IDX                                                            2
+#define regOTG1_OTG_ENCRYPTION                                                                          0x1c32
+#define regOTG1_OTG_ENCRYPTION_BASE_IDX                                                                 2
+#define regOTG1_OTG_SPARE_REGISTER                                                                      0x1c33
+#define regOTG1_OTG_SPARE_REGISTER_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_optc_otg2_dispdec
+// base address: 0x440
+#define regOTG2_OTG_H_TOTAL                                                                             0x1c3a
+#define regOTG2_OTG_H_TOTAL_BASE_IDX                                                                    2
+#define regOTG2_OTG_H_BLANK_START_END                                                                   0x1c3b
+#define regOTG2_OTG_H_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG2_OTG_H_SYNC_A                                                                            0x1c3c
+#define regOTG2_OTG_H_SYNC_A_BASE_IDX                                                                   2
+#define regOTG2_OTG_H_SYNC_A_CNTL                                                                       0x1c3d
+#define regOTG2_OTG_H_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG2_OTG_H_TIMING_CNTL                                                                       0x1c3e
+#define regOTG2_OTG_H_TIMING_CNTL_BASE_IDX                                                              2
+#define regOTG2_OTG_V_TOTAL                                                                             0x1c3f
+#define regOTG2_OTG_V_TOTAL_BASE_IDX                                                                    2
+#define regOTG2_OTG_V_TOTAL_MIN                                                                         0x1c40
+#define regOTG2_OTG_V_TOTAL_MIN_BASE_IDX                                                                2
+#define regOTG2_OTG_V_TOTAL_MAX                                                                         0x1c41
+#define regOTG2_OTG_V_TOTAL_MAX_BASE_IDX                                                                2
+#define regOTG2_OTG_V_TOTAL_MID                                                                         0x1c42
+#define regOTG2_OTG_V_TOTAL_MID_BASE_IDX                                                                2
+#define regOTG2_OTG_V_TOTAL_CONTROL                                                                     0x1c43
+#define regOTG2_OTG_V_TOTAL_CONTROL_BASE_IDX                                                            2
+#define regOTG2_OTG_V_COUNT_STOP_CONTROL                                                                0x1c44
+#define regOTG2_OTG_V_COUNT_STOP_CONTROL_BASE_IDX                                                       2
+#define regOTG2_OTG_V_COUNT_STOP_CONTROL2                                                               0x1c45
+#define regOTG2_OTG_V_COUNT_STOP_CONTROL2_BASE_IDX                                                      2
+#define regOTG2_OTG_V_TOTAL_INT_STATUS                                                                  0x1c46
+#define regOTG2_OTG_V_TOTAL_INT_STATUS_BASE_IDX                                                         2
+#define regOTG2_OTG_VSYNC_NOM_INT_STATUS                                                                0x1c47
+#define regOTG2_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX                                                       2
+#define regOTG2_OTG_V_BLANK_START_END                                                                   0x1c48
+#define regOTG2_OTG_V_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG2_OTG_V_SYNC_A                                                                            0x1c49
+#define regOTG2_OTG_V_SYNC_A_BASE_IDX                                                                   2
+#define regOTG2_OTG_V_SYNC_A_CNTL                                                                       0x1c4a
+#define regOTG2_OTG_V_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG2_OTG_TRIGA_CNTL                                                                          0x1c4b
+#define regOTG2_OTG_TRIGA_CNTL_BASE_IDX                                                                 2
+#define regOTG2_OTG_TRIGA_MANUAL_TRIG                                                                   0x1c4c
+#define regOTG2_OTG_TRIGA_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG2_OTG_TRIGB_CNTL                                                                          0x1c4d
+#define regOTG2_OTG_TRIGB_CNTL_BASE_IDX                                                                 2
+#define regOTG2_OTG_TRIGB_MANUAL_TRIG                                                                   0x1c4e
+#define regOTG2_OTG_TRIGB_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG2_OTG_FORCE_COUNT_NOW_CNTL                                                                0x1c4f
+#define regOTG2_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX                                                       2
+#define regOTG2_OTG_STEREO_FORCE_NEXT_EYE                                                               0x1c51
+#define regOTG2_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX                                                      2
+#define regOTG2_OTG_CONTROL                                                                             0x1c53
+#define regOTG2_OTG_CONTROL_BASE_IDX                                                                    2
+#define regOTG2_OTG_DLPC_CONTROL                                                                        0x1c54
+#define regOTG2_OTG_DLPC_CONTROL_BASE_IDX                                                               2
+#define regOTG2_OTG_PWA_FRAME_SYNC_CONTROL                                                              0x1c55
+#define regOTG2_OTG_PWA_FRAME_SYNC_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_INTERLACE_CONTROL                                                                   0x1c56
+#define regOTG2_OTG_INTERLACE_CONTROL_BASE_IDX                                                          2
+#define regOTG2_OTG_INTERLACE_STATUS                                                                    0x1c57
+#define regOTG2_OTG_INTERLACE_STATUS_BASE_IDX                                                           2
+#define regOTG2_OTG_PIXEL_DATA_READBACK0                                                                0x1c58
+#define regOTG2_OTG_PIXEL_DATA_READBACK0_BASE_IDX                                                       2
+#define regOTG2_OTG_PIXEL_DATA_READBACK1                                                                0x1c59
+#define regOTG2_OTG_PIXEL_DATA_READBACK1_BASE_IDX                                                       2
+#define regOTG2_OTG_STATUS                                                                              0x1c5a
+#define regOTG2_OTG_STATUS_BASE_IDX                                                                     2
+#define regOTG2_OTG_STATUS_POSITION                                                                     0x1c5b
+#define regOTG2_OTG_STATUS_POSITION_BASE_IDX                                                            2
+#define regOTG2_OTG_LONG_VBLANK_STATUS                                                                  0x1c5c
+#define regOTG2_OTG_LONG_VBLANK_STATUS_BASE_IDX                                                         2
+#define regOTG2_OTG_NOM_VERT_POSITION                                                                   0x1c5d
+#define regOTG2_OTG_NOM_VERT_POSITION_BASE_IDX                                                          2
+#define regOTG2_OTG_STATUS_FRAME_COUNT                                                                  0x1c5e
+#define regOTG2_OTG_STATUS_FRAME_COUNT_BASE_IDX                                                         2
+#define regOTG2_OTG_STATUS_VF_COUNT                                                                     0x1c5f
+#define regOTG2_OTG_STATUS_VF_COUNT_BASE_IDX                                                            2
+#define regOTG2_OTG_STATUS_HV_COUNT                                                                     0x1c60
+#define regOTG2_OTG_STATUS_HV_COUNT_BASE_IDX                                                            2
+#define regOTG2_OTG_COUNT_CONTROL                                                                       0x1c61
+#define regOTG2_OTG_COUNT_CONTROL_BASE_IDX                                                              2
+#define regOTG2_OTG_COUNT_RESET                                                                         0x1c62
+#define regOTG2_OTG_COUNT_RESET_BASE_IDX                                                                2
+#define regOTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE                                                        0x1c63
+#define regOTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX                                               2
+#define regOTG2_OTG_VERT_SYNC_CONTROL                                                                   0x1c64
+#define regOTG2_OTG_VERT_SYNC_CONTROL_BASE_IDX                                                          2
+#define regOTG2_OTG_STEREO_STATUS                                                                       0x1c65
+#define regOTG2_OTG_STEREO_STATUS_BASE_IDX                                                              2
+#define regOTG2_OTG_STEREO_CONTROL                                                                      0x1c66
+#define regOTG2_OTG_STEREO_CONTROL_BASE_IDX                                                             2
+#define regOTG2_OTG_SNAPSHOT_STATUS                                                                     0x1c67
+#define regOTG2_OTG_SNAPSHOT_STATUS_BASE_IDX                                                            2
+#define regOTG2_OTG_SNAPSHOT_CONTROL                                                                    0x1c68
+#define regOTG2_OTG_SNAPSHOT_CONTROL_BASE_IDX                                                           2
+#define regOTG2_OTG_SNAPSHOT_POSITION                                                                   0x1c69
+#define regOTG2_OTG_SNAPSHOT_POSITION_BASE_IDX                                                          2
+#define regOTG2_OTG_SNAPSHOT_FRAME                                                                      0x1c6a
+#define regOTG2_OTG_SNAPSHOT_FRAME_BASE_IDX                                                             2
+#define regOTG2_OTG_INTERRUPT_CONTROL                                                                   0x1c6b
+#define regOTG2_OTG_INTERRUPT_CONTROL_BASE_IDX                                                          2
+#define regOTG2_OTG_UPDATE_LOCK                                                                         0x1c6c
+#define regOTG2_OTG_UPDATE_LOCK_BASE_IDX                                                                2
+#define regOTG2_OTG_DOUBLE_BUFFER_CONTROL                                                               0x1c6d
+#define regOTG2_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX                                                      2
+#define regOTG2_OTG_MASTER_EN                                                                           0x1c6e
+#define regOTG2_OTG_MASTER_EN_BASE_IDX                                                                  2
+#define regOTG2_OTG_VERTICAL_INTERRUPT0_POSITION                                                        0x1c70
+#define regOTG2_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX                                               2
+#define regOTG2_OTG_VERTICAL_INTERRUPT0_CONTROL                                                         0x1c71
+#define regOTG2_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX                                                2
+#define regOTG2_OTG_VERTICAL_INTERRUPT1_POSITION                                                        0x1c72
+#define regOTG2_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX                                               2
+#define regOTG2_OTG_VERTICAL_INTERRUPT1_CONTROL                                                         0x1c73
+#define regOTG2_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX                                                2
+#define regOTG2_OTG_VERTICAL_INTERRUPT2_POSITION                                                        0x1c74
+#define regOTG2_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX                                               2
+#define regOTG2_OTG_VERTICAL_INTERRUPT2_CONTROL                                                         0x1c75
+#define regOTG2_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX                                                2
+#define regOTG2_OTG_CRC_CNTL                                                                            0x1c76
+#define regOTG2_OTG_CRC_CNTL_BASE_IDX                                                                   2
+#define regOTG2_OTG_CRC0_WINDOWA_X_CONTROL                                                              0x1c77
+#define regOTG2_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC0_WINDOWA_Y_CONTROL                                                              0x1c78
+#define regOTG2_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC0_WINDOWB_X_CONTROL                                                              0x1c79
+#define regOTG2_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC0_WINDOWB_Y_CONTROL                                                              0x1c7a
+#define regOTG2_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC0_DATA_R                                                                         0x1c7b
+#define regOTG2_OTG_CRC0_DATA_R_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC0_DATA_G                                                                         0x1c7c
+#define regOTG2_OTG_CRC0_DATA_G_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC0_DATA_B                                                                         0x1c7d
+#define regOTG2_OTG_CRC0_DATA_B_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC0_DATA_C                                                                         0x1c7e
+#define regOTG2_OTG_CRC0_DATA_C_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC1_WINDOWA_X_CONTROL                                                              0x1c7f
+#define regOTG2_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC1_WINDOWA_Y_CONTROL                                                              0x1c80
+#define regOTG2_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC1_WINDOWB_X_CONTROL                                                              0x1c81
+#define regOTG2_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC1_WINDOWB_Y_CONTROL                                                              0x1c82
+#define regOTG2_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC1_DATA_R                                                                         0x1c83
+#define regOTG2_OTG_CRC1_DATA_R_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC1_DATA_G                                                                         0x1c84
+#define regOTG2_OTG_CRC1_DATA_G_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC1_DATA_B                                                                         0x1c85
+#define regOTG2_OTG_CRC1_DATA_B_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC1_DATA_C                                                                         0x1c86
+#define regOTG2_OTG_CRC1_DATA_C_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC2_DATA_R                                                                         0x1c87
+#define regOTG2_OTG_CRC2_DATA_R_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC2_DATA_G                                                                         0x1c88
+#define regOTG2_OTG_CRC2_DATA_G_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC2_DATA_B                                                                         0x1c89
+#define regOTG2_OTG_CRC2_DATA_B_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC2_DATA_C                                                                         0x1c8a
+#define regOTG2_OTG_CRC2_DATA_C_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC3_DATA_R                                                                         0x1c8b
+#define regOTG2_OTG_CRC3_DATA_R_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC3_DATA_G                                                                         0x1c8c
+#define regOTG2_OTG_CRC3_DATA_G_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC3_DATA_B                                                                         0x1c8d
+#define regOTG2_OTG_CRC3_DATA_B_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC3_DATA_C                                                                         0x1c8e
+#define regOTG2_OTG_CRC3_DATA_C_BASE_IDX                                                                2
+#define regOTG2_OTG_CRC0_DATA_AES                                                                       0x1c8f
+#define regOTG2_OTG_CRC0_DATA_AES_BASE_IDX                                                              2
+#define regOTG2_OTG_CRC_SIG_RED_GREEN_MASK                                                              0x1c90
+#define regOTG2_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX                                                     2
+#define regOTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK                                                           0x1c91
+#define regOTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX                                                  2
+#define regOTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK                                                     0x1c92
+#define regOTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK                                                     0x1c93
+#define regOTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK                                                     0x1c94
+#define regOTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK                                                     0x1c95
+#define regOTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK                                                     0x1c96
+#define regOTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK                                                     0x1c97
+#define regOTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK                                                     0x1c98
+#define regOTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK                                                     0x1c99
+#define regOTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG2_OTG_STATIC_SCREEN_CONTROL                                                               0x1c9a
+#define regOTG2_OTG_STATIC_SCREEN_CONTROL_BASE_IDX                                                      2
+#define regOTG2_OTG_3D_STRUCTURE_CONTROL                                                                0x1c9b
+#define regOTG2_OTG_3D_STRUCTURE_CONTROL_BASE_IDX                                                       2
+#define regOTG2_OTG_GSL_VSYNC_GAP                                                                       0x1c9c
+#define regOTG2_OTG_GSL_VSYNC_GAP_BASE_IDX                                                              2
+#define regOTG2_OTG_MASTER_UPDATE_MODE                                                                  0x1c9d
+#define regOTG2_OTG_MASTER_UPDATE_MODE_BASE_IDX                                                         2
+#define regOTG2_OTG_CLOCK_CONTROL                                                                       0x1c9e
+#define regOTG2_OTG_CLOCK_CONTROL_BASE_IDX                                                              2
+#define regOTG2_OTG_VSTARTUP_PARAM                                                                      0x1c9f
+#define regOTG2_OTG_VSTARTUP_PARAM_BASE_IDX                                                             2
+#define regOTG2_OTG_VUPDATE_PARAM                                                                       0x1ca0
+#define regOTG2_OTG_VUPDATE_PARAM_BASE_IDX                                                              2
+#define regOTG2_OTG_VREADY_PARAM                                                                        0x1ca1
+#define regOTG2_OTG_VREADY_PARAM_BASE_IDX                                                               2
+#define regOTG2_OTG_GLOBAL_SYNC_STATUS                                                                  0x1ca2
+#define regOTG2_OTG_GLOBAL_SYNC_STATUS_BASE_IDX                                                         2
+#define regOTG2_OTG_MASTER_UPDATE_LOCK                                                                  0x1ca3
+#define regOTG2_OTG_MASTER_UPDATE_LOCK_BASE_IDX                                                         2
+#define regOTG2_OTG_GSL_CONTROL                                                                         0x1ca4
+#define regOTG2_OTG_GSL_CONTROL_BASE_IDX                                                                2
+#define regOTG2_OTG_GSL_WINDOW_X                                                                        0x1ca5
+#define regOTG2_OTG_GSL_WINDOW_X_BASE_IDX                                                               2
+#define regOTG2_OTG_GSL_WINDOW_Y                                                                        0x1ca6
+#define regOTG2_OTG_GSL_WINDOW_Y_BASE_IDX                                                               2
+#define regOTG2_OTG_VUPDATE_KEEPOUT                                                                     0x1ca7
+#define regOTG2_OTG_VUPDATE_KEEPOUT_BASE_IDX                                                            2
+#define regOTG2_OTG_GLOBAL_CONTROL0                                                                     0x1ca8
+#define regOTG2_OTG_GLOBAL_CONTROL0_BASE_IDX                                                            2
+#define regOTG2_OTG_GLOBAL_CONTROL1                                                                     0x1ca9
+#define regOTG2_OTG_GLOBAL_CONTROL1_BASE_IDX                                                            2
+#define regOTG2_OTG_GLOBAL_CONTROL2                                                                     0x1caa
+#define regOTG2_OTG_GLOBAL_CONTROL2_BASE_IDX                                                            2
+#define regOTG2_OTG_GLOBAL_CONTROL3                                                                     0x1cab
+#define regOTG2_OTG_GLOBAL_CONTROL3_BASE_IDX                                                            2
+#define regOTG2_OTG_GLOBAL_CONTROL4                                                                     0x1cac
+#define regOTG2_OTG_GLOBAL_CONTROL4_BASE_IDX                                                            2
+#define regOTG2_OTG_TRIG_MANUAL_CONTROL                                                                 0x1cad
+#define regOTG2_OTG_TRIG_MANUAL_CONTROL_BASE_IDX                                                        2
+#define regOTG2_OTG_DRR_TIMING_INT_STATUS                                                               0x1caf
+#define regOTG2_OTG_DRR_TIMING_INT_STATUS_BASE_IDX                                                      2
+#define regOTG2_OTG_DRR_V_TOTAL_REACH_RANGE                                                             0x1cb0
+#define regOTG2_OTG_DRR_V_TOTAL_REACH_RANGE_BASE_IDX                                                    2
+#define regOTG2_OTG_DRR_V_TOTAL_CHANGE                                                                  0x1cb1
+#define regOTG2_OTG_DRR_V_TOTAL_CHANGE_BASE_IDX                                                         2
+#define regOTG2_OTG_DRR_TRIGGER_WINDOW                                                                  0x1cb2
+#define regOTG2_OTG_DRR_TRIGGER_WINDOW_BASE_IDX                                                         2
+#define regOTG2_OTG_DRR_CONTROL                                                                         0x1cb3
+#define regOTG2_OTG_DRR_CONTROL_BASE_IDX                                                                2
+#define regOTG2_OTG_DRR_CONTOL2                                                                         0x1cb4
+#define regOTG2_OTG_DRR_CONTOL2_BASE_IDX                                                                2
+#define regOTG2_OTG_M_CONST_DTO0                                                                        0x1cb5
+#define regOTG2_OTG_M_CONST_DTO0_BASE_IDX                                                               2
+#define regOTG2_OTG_M_CONST_DTO1                                                                        0x1cb6
+#define regOTG2_OTG_M_CONST_DTO1_BASE_IDX                                                               2
+#define regOTG2_OTG_REQUEST_CONTROL                                                                     0x1cb7
+#define regOTG2_OTG_REQUEST_CONTROL_BASE_IDX                                                            2
+#define regOTG2_OTG_PIPE_UPDATE_STATUS                                                                  0x1cb8
+#define regOTG2_OTG_PIPE_UPDATE_STATUS_BASE_IDX                                                         2
+#define regOTG2_OTG_PSTATE_REGISTER                                                                     0x1cb9
+#define regOTG2_OTG_PSTATE_REGISTER_BASE_IDX                                                            2
+#define regOTG2_OTG_ENCRYPTION                                                                          0x1cba
+#define regOTG2_OTG_ENCRYPTION_BASE_IDX                                                                 2
+#define regOTG2_OTG_SPARE_REGISTER                                                                      0x1cbb
+#define regOTG2_OTG_SPARE_REGISTER_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_optc_otg3_dispdec
+// base address: 0x660
+#define regOTG3_OTG_H_TOTAL                                                                             0x1cc2
+#define regOTG3_OTG_H_TOTAL_BASE_IDX                                                                    2
+#define regOTG3_OTG_H_BLANK_START_END                                                                   0x1cc3
+#define regOTG3_OTG_H_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG3_OTG_H_SYNC_A                                                                            0x1cc4
+#define regOTG3_OTG_H_SYNC_A_BASE_IDX                                                                   2
+#define regOTG3_OTG_H_SYNC_A_CNTL                                                                       0x1cc5
+#define regOTG3_OTG_H_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG3_OTG_H_TIMING_CNTL                                                                       0x1cc6
+#define regOTG3_OTG_H_TIMING_CNTL_BASE_IDX                                                              2
+#define regOTG3_OTG_V_TOTAL                                                                             0x1cc7
+#define regOTG3_OTG_V_TOTAL_BASE_IDX                                                                    2
+#define regOTG3_OTG_V_TOTAL_MIN                                                                         0x1cc8
+#define regOTG3_OTG_V_TOTAL_MIN_BASE_IDX                                                                2
+#define regOTG3_OTG_V_TOTAL_MAX                                                                         0x1cc9
+#define regOTG3_OTG_V_TOTAL_MAX_BASE_IDX                                                                2
+#define regOTG3_OTG_V_TOTAL_MID                                                                         0x1cca
+#define regOTG3_OTG_V_TOTAL_MID_BASE_IDX                                                                2
+#define regOTG3_OTG_V_TOTAL_CONTROL                                                                     0x1ccb
+#define regOTG3_OTG_V_TOTAL_CONTROL_BASE_IDX                                                            2
+#define regOTG3_OTG_V_COUNT_STOP_CONTROL                                                                0x1ccc
+#define regOTG3_OTG_V_COUNT_STOP_CONTROL_BASE_IDX                                                       2
+#define regOTG3_OTG_V_COUNT_STOP_CONTROL2                                                               0x1ccd
+#define regOTG3_OTG_V_COUNT_STOP_CONTROL2_BASE_IDX                                                      2
+#define regOTG3_OTG_V_TOTAL_INT_STATUS                                                                  0x1cce
+#define regOTG3_OTG_V_TOTAL_INT_STATUS_BASE_IDX                                                         2
+#define regOTG3_OTG_VSYNC_NOM_INT_STATUS                                                                0x1ccf
+#define regOTG3_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX                                                       2
+#define regOTG3_OTG_V_BLANK_START_END                                                                   0x1cd0
+#define regOTG3_OTG_V_BLANK_START_END_BASE_IDX                                                          2
+#define regOTG3_OTG_V_SYNC_A                                                                            0x1cd1
+#define regOTG3_OTG_V_SYNC_A_BASE_IDX                                                                   2
+#define regOTG3_OTG_V_SYNC_A_CNTL                                                                       0x1cd2
+#define regOTG3_OTG_V_SYNC_A_CNTL_BASE_IDX                                                              2
+#define regOTG3_OTG_TRIGA_CNTL                                                                          0x1cd3
+#define regOTG3_OTG_TRIGA_CNTL_BASE_IDX                                                                 2
+#define regOTG3_OTG_TRIGA_MANUAL_TRIG                                                                   0x1cd4
+#define regOTG3_OTG_TRIGA_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG3_OTG_TRIGB_CNTL                                                                          0x1cd5
+#define regOTG3_OTG_TRIGB_CNTL_BASE_IDX                                                                 2
+#define regOTG3_OTG_TRIGB_MANUAL_TRIG                                                                   0x1cd6
+#define regOTG3_OTG_TRIGB_MANUAL_TRIG_BASE_IDX                                                          2
+#define regOTG3_OTG_FORCE_COUNT_NOW_CNTL                                                                0x1cd7
+#define regOTG3_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX                                                       2
+#define regOTG3_OTG_STEREO_FORCE_NEXT_EYE                                                               0x1cd9
+#define regOTG3_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX                                                      2
+#define regOTG3_OTG_CONTROL                                                                             0x1cdb
+#define regOTG3_OTG_CONTROL_BASE_IDX                                                                    2
+#define regOTG3_OTG_DLPC_CONTROL                                                                        0x1cdc
+#define regOTG3_OTG_DLPC_CONTROL_BASE_IDX                                                               2
+#define regOTG3_OTG_PWA_FRAME_SYNC_CONTROL                                                              0x1cdd
+#define regOTG3_OTG_PWA_FRAME_SYNC_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_INTERLACE_CONTROL                                                                   0x1cde
+#define regOTG3_OTG_INTERLACE_CONTROL_BASE_IDX                                                          2
+#define regOTG3_OTG_INTERLACE_STATUS                                                                    0x1cdf
+#define regOTG3_OTG_INTERLACE_STATUS_BASE_IDX                                                           2
+#define regOTG3_OTG_PIXEL_DATA_READBACK0                                                                0x1ce0
+#define regOTG3_OTG_PIXEL_DATA_READBACK0_BASE_IDX                                                       2
+#define regOTG3_OTG_PIXEL_DATA_READBACK1                                                                0x1ce1
+#define regOTG3_OTG_PIXEL_DATA_READBACK1_BASE_IDX                                                       2
+#define regOTG3_OTG_STATUS                                                                              0x1ce2
+#define regOTG3_OTG_STATUS_BASE_IDX                                                                     2
+#define regOTG3_OTG_STATUS_POSITION                                                                     0x1ce3
+#define regOTG3_OTG_STATUS_POSITION_BASE_IDX                                                            2
+#define regOTG3_OTG_LONG_VBLANK_STATUS                                                                  0x1ce4
+#define regOTG3_OTG_LONG_VBLANK_STATUS_BASE_IDX                                                         2
+#define regOTG3_OTG_NOM_VERT_POSITION                                                                   0x1ce5
+#define regOTG3_OTG_NOM_VERT_POSITION_BASE_IDX                                                          2
+#define regOTG3_OTG_STATUS_FRAME_COUNT                                                                  0x1ce6
+#define regOTG3_OTG_STATUS_FRAME_COUNT_BASE_IDX                                                         2
+#define regOTG3_OTG_STATUS_VF_COUNT                                                                     0x1ce7
+#define regOTG3_OTG_STATUS_VF_COUNT_BASE_IDX                                                            2
+#define regOTG3_OTG_STATUS_HV_COUNT                                                                     0x1ce8
+#define regOTG3_OTG_STATUS_HV_COUNT_BASE_IDX                                                            2
+#define regOTG3_OTG_COUNT_CONTROL                                                                       0x1ce9
+#define regOTG3_OTG_COUNT_CONTROL_BASE_IDX                                                              2
+#define regOTG3_OTG_COUNT_RESET                                                                         0x1cea
+#define regOTG3_OTG_COUNT_RESET_BASE_IDX                                                                2
+#define regOTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE                                                        0x1ceb
+#define regOTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX                                               2
+#define regOTG3_OTG_VERT_SYNC_CONTROL                                                                   0x1cec
+#define regOTG3_OTG_VERT_SYNC_CONTROL_BASE_IDX                                                          2
+#define regOTG3_OTG_STEREO_STATUS                                                                       0x1ced
+#define regOTG3_OTG_STEREO_STATUS_BASE_IDX                                                              2
+#define regOTG3_OTG_STEREO_CONTROL                                                                      0x1cee
+#define regOTG3_OTG_STEREO_CONTROL_BASE_IDX                                                             2
+#define regOTG3_OTG_SNAPSHOT_STATUS                                                                     0x1cef
+#define regOTG3_OTG_SNAPSHOT_STATUS_BASE_IDX                                                            2
+#define regOTG3_OTG_SNAPSHOT_CONTROL                                                                    0x1cf0
+#define regOTG3_OTG_SNAPSHOT_CONTROL_BASE_IDX                                                           2
+#define regOTG3_OTG_SNAPSHOT_POSITION                                                                   0x1cf1
+#define regOTG3_OTG_SNAPSHOT_POSITION_BASE_IDX                                                          2
+#define regOTG3_OTG_SNAPSHOT_FRAME                                                                      0x1cf2
+#define regOTG3_OTG_SNAPSHOT_FRAME_BASE_IDX                                                             2
+#define regOTG3_OTG_INTERRUPT_CONTROL                                                                   0x1cf3
+#define regOTG3_OTG_INTERRUPT_CONTROL_BASE_IDX                                                          2
+#define regOTG3_OTG_UPDATE_LOCK                                                                         0x1cf4
+#define regOTG3_OTG_UPDATE_LOCK_BASE_IDX                                                                2
+#define regOTG3_OTG_DOUBLE_BUFFER_CONTROL                                                               0x1cf5
+#define regOTG3_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX                                                      2
+#define regOTG3_OTG_MASTER_EN                                                                           0x1cf6
+#define regOTG3_OTG_MASTER_EN_BASE_IDX                                                                  2
+#define regOTG3_OTG_VERTICAL_INTERRUPT0_POSITION                                                        0x1cf8
+#define regOTG3_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX                                               2
+#define regOTG3_OTG_VERTICAL_INTERRUPT0_CONTROL                                                         0x1cf9
+#define regOTG3_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX                                                2
+#define regOTG3_OTG_VERTICAL_INTERRUPT1_POSITION                                                        0x1cfa
+#define regOTG3_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX                                               2
+#define regOTG3_OTG_VERTICAL_INTERRUPT1_CONTROL                                                         0x1cfb
+#define regOTG3_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX                                                2
+#define regOTG3_OTG_VERTICAL_INTERRUPT2_POSITION                                                        0x1cfc
+#define regOTG3_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX                                               2
+#define regOTG3_OTG_VERTICAL_INTERRUPT2_CONTROL                                                         0x1cfd
+#define regOTG3_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX                                                2
+#define regOTG3_OTG_CRC_CNTL                                                                            0x1cfe
+#define regOTG3_OTG_CRC_CNTL_BASE_IDX                                                                   2
+#define regOTG3_OTG_CRC0_WINDOWA_X_CONTROL                                                              0x1cff
+#define regOTG3_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC0_WINDOWA_Y_CONTROL                                                              0x1d00
+#define regOTG3_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC0_WINDOWB_X_CONTROL                                                              0x1d01
+#define regOTG3_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC0_WINDOWB_Y_CONTROL                                                              0x1d02
+#define regOTG3_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC0_DATA_R                                                                         0x1d03
+#define regOTG3_OTG_CRC0_DATA_R_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC0_DATA_G                                                                         0x1d04
+#define regOTG3_OTG_CRC0_DATA_G_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC0_DATA_B                                                                         0x1d05
+#define regOTG3_OTG_CRC0_DATA_B_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC0_DATA_C                                                                         0x1d06
+#define regOTG3_OTG_CRC0_DATA_C_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC1_WINDOWA_X_CONTROL                                                              0x1d07
+#define regOTG3_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC1_WINDOWA_Y_CONTROL                                                              0x1d08
+#define regOTG3_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC1_WINDOWB_X_CONTROL                                                              0x1d09
+#define regOTG3_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC1_WINDOWB_Y_CONTROL                                                              0x1d0a
+#define regOTG3_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC1_DATA_R                                                                         0x1d0b
+#define regOTG3_OTG_CRC1_DATA_R_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC1_DATA_G                                                                         0x1d0c
+#define regOTG3_OTG_CRC1_DATA_G_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC1_DATA_B                                                                         0x1d0d
+#define regOTG3_OTG_CRC1_DATA_B_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC1_DATA_C                                                                         0x1d0e
+#define regOTG3_OTG_CRC1_DATA_C_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC2_DATA_R                                                                         0x1d0f
+#define regOTG3_OTG_CRC2_DATA_R_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC2_DATA_G                                                                         0x1d10
+#define regOTG3_OTG_CRC2_DATA_G_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC2_DATA_B                                                                         0x1d11
+#define regOTG3_OTG_CRC2_DATA_B_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC2_DATA_C                                                                         0x1d12
+#define regOTG3_OTG_CRC2_DATA_C_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC3_DATA_R                                                                         0x1d13
+#define regOTG3_OTG_CRC3_DATA_R_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC3_DATA_G                                                                         0x1d14
+#define regOTG3_OTG_CRC3_DATA_G_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC3_DATA_B                                                                         0x1d15
+#define regOTG3_OTG_CRC3_DATA_B_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC3_DATA_C                                                                         0x1d16
+#define regOTG3_OTG_CRC3_DATA_C_BASE_IDX                                                                2
+#define regOTG3_OTG_CRC0_DATA_AES                                                                       0x1d17
+#define regOTG3_OTG_CRC0_DATA_AES_BASE_IDX                                                              2
+#define regOTG3_OTG_CRC_SIG_RED_GREEN_MASK                                                              0x1d18
+#define regOTG3_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX                                                     2
+#define regOTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK                                                           0x1d19
+#define regOTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX                                                  2
+#define regOTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK                                                     0x1d1a
+#define regOTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK                                                     0x1d1b
+#define regOTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK                                                     0x1d1c
+#define regOTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK                                                     0x1d1d
+#define regOTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK                                                     0x1d1e
+#define regOTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK                                                     0x1d1f
+#define regOTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK                                                     0x1d20
+#define regOTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK                                                     0x1d21
+#define regOTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK_BASE_IDX                                            2
+#define regOTG3_OTG_STATIC_SCREEN_CONTROL                                                               0x1d22
+#define regOTG3_OTG_STATIC_SCREEN_CONTROL_BASE_IDX                                                      2
+#define regOTG3_OTG_3D_STRUCTURE_CONTROL                                                                0x1d23
+#define regOTG3_OTG_3D_STRUCTURE_CONTROL_BASE_IDX                                                       2
+#define regOTG3_OTG_GSL_VSYNC_GAP                                                                       0x1d24
+#define regOTG3_OTG_GSL_VSYNC_GAP_BASE_IDX                                                              2
+#define regOTG3_OTG_MASTER_UPDATE_MODE                                                                  0x1d25
+#define regOTG3_OTG_MASTER_UPDATE_MODE_BASE_IDX                                                         2
+#define regOTG3_OTG_CLOCK_CONTROL                                                                       0x1d26
+#define regOTG3_OTG_CLOCK_CONTROL_BASE_IDX                                                              2
+#define regOTG3_OTG_VSTARTUP_PARAM                                                                      0x1d27
+#define regOTG3_OTG_VSTARTUP_PARAM_BASE_IDX                                                             2
+#define regOTG3_OTG_VUPDATE_PARAM                                                                       0x1d28
+#define regOTG3_OTG_VUPDATE_PARAM_BASE_IDX                                                              2
+#define regOTG3_OTG_VREADY_PARAM                                                                        0x1d29
+#define regOTG3_OTG_VREADY_PARAM_BASE_IDX                                                               2
+#define regOTG3_OTG_GLOBAL_SYNC_STATUS                                                                  0x1d2a
+#define regOTG3_OTG_GLOBAL_SYNC_STATUS_BASE_IDX                                                         2
+#define regOTG3_OTG_MASTER_UPDATE_LOCK                                                                  0x1d2b
+#define regOTG3_OTG_MASTER_UPDATE_LOCK_BASE_IDX                                                         2
+#define regOTG3_OTG_GSL_CONTROL                                                                         0x1d2c
+#define regOTG3_OTG_GSL_CONTROL_BASE_IDX                                                                2
+#define regOTG3_OTG_GSL_WINDOW_X                                                                        0x1d2d
+#define regOTG3_OTG_GSL_WINDOW_X_BASE_IDX                                                               2
+#define regOTG3_OTG_GSL_WINDOW_Y                                                                        0x1d2e
+#define regOTG3_OTG_GSL_WINDOW_Y_BASE_IDX                                                               2
+#define regOTG3_OTG_VUPDATE_KEEPOUT                                                                     0x1d2f
+#define regOTG3_OTG_VUPDATE_KEEPOUT_BASE_IDX                                                            2
+#define regOTG3_OTG_GLOBAL_CONTROL0                                                                     0x1d30
+#define regOTG3_OTG_GLOBAL_CONTROL0_BASE_IDX                                                            2
+#define regOTG3_OTG_GLOBAL_CONTROL1                                                                     0x1d31
+#define regOTG3_OTG_GLOBAL_CONTROL1_BASE_IDX                                                            2
+#define regOTG3_OTG_GLOBAL_CONTROL2                                                                     0x1d32
+#define regOTG3_OTG_GLOBAL_CONTROL2_BASE_IDX                                                            2
+#define regOTG3_OTG_GLOBAL_CONTROL3                                                                     0x1d33
+#define regOTG3_OTG_GLOBAL_CONTROL3_BASE_IDX                                                            2
+#define regOTG3_OTG_GLOBAL_CONTROL4                                                                     0x1d34
+#define regOTG3_OTG_GLOBAL_CONTROL4_BASE_IDX                                                            2
+#define regOTG3_OTG_TRIG_MANUAL_CONTROL                                                                 0x1d35
+#define regOTG3_OTG_TRIG_MANUAL_CONTROL_BASE_IDX                                                        2
+#define regOTG3_OTG_DRR_TIMING_INT_STATUS                                                               0x1d37
+#define regOTG3_OTG_DRR_TIMING_INT_STATUS_BASE_IDX                                                      2
+#define regOTG3_OTG_DRR_V_TOTAL_REACH_RANGE                                                             0x1d38
+#define regOTG3_OTG_DRR_V_TOTAL_REACH_RANGE_BASE_IDX                                                    2
+#define regOTG3_OTG_DRR_V_TOTAL_CHANGE                                                                  0x1d39
+#define regOTG3_OTG_DRR_V_TOTAL_CHANGE_BASE_IDX                                                         2
+#define regOTG3_OTG_DRR_TRIGGER_WINDOW                                                                  0x1d3a
+#define regOTG3_OTG_DRR_TRIGGER_WINDOW_BASE_IDX                                                         2
+#define regOTG3_OTG_DRR_CONTROL                                                                         0x1d3b
+#define regOTG3_OTG_DRR_CONTROL_BASE_IDX                                                                2
+#define regOTG3_OTG_DRR_CONTOL2                                                                         0x1d3c
+#define regOTG3_OTG_DRR_CONTOL2_BASE_IDX                                                                2
+#define regOTG3_OTG_M_CONST_DTO0                                                                        0x1d3d
+#define regOTG3_OTG_M_CONST_DTO0_BASE_IDX                                                               2
+#define regOTG3_OTG_M_CONST_DTO1                                                                        0x1d3e
+#define regOTG3_OTG_M_CONST_DTO1_BASE_IDX                                                               2
+#define regOTG3_OTG_REQUEST_CONTROL                                                                     0x1d3f
+#define regOTG3_OTG_REQUEST_CONTROL_BASE_IDX                                                            2
+#define regOTG3_OTG_PIPE_UPDATE_STATUS                                                                  0x1d40
+#define regOTG3_OTG_PIPE_UPDATE_STATUS_BASE_IDX                                                         2
+#define regOTG3_OTG_PSTATE_REGISTER                                                                     0x1d41
+#define regOTG3_OTG_PSTATE_REGISTER_BASE_IDX                                                            2
+#define regOTG3_OTG_ENCRYPTION                                                                          0x1d42
+#define regOTG3_OTG_ENCRYPTION_BASE_IDX                                                                 2
+#define regOTG3_OTG_SPARE_REGISTER                                                                      0x1d43
+#define regOTG3_OTG_SPARE_REGISTER_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_optc_optc_misc_dispdec
+// base address: 0x0
+#define regGSL_SOURCE_SELECT                                                                            0x1e2b
+#define regGSL_SOURCE_SELECT_BASE_IDX                                                                   2
+#define regOPTC_DLPC_CONTROL                                                                            0x1e2c
+#define regOPTC_DLPC_CONTROL_BASE_IDX                                                                   2
+#define regOPTC_CLOCK_CONTROL                                                                           0x1e2d
+#define regOPTC_CLOCK_CONTROL_BASE_IDX                                                                  2
+#define regODM_MEM_PWR_CTRL                                                                             0x1e2e
+#define regODM_MEM_PWR_CTRL_BASE_IDX                                                                    2
+#define regODM_MEM_PWR_CTRL3                                                                            0x1e30
+#define regODM_MEM_PWR_CTRL3_BASE_IDX                                                                   2
+#define regODM_MEM_PWR_STATUS                                                                           0x1e31
+#define regODM_MEM_PWR_STATUS_BASE_IDX                                                                  2
+#define regOPTC_MISC_SPARE_REGISTER                                                                     0x1e32
+#define regOPTC_MISC_SPARE_REGISTER_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_optc_optc_dcperfmon_dc_perfmon_dispdec
+// base address: 0x79a8
+#define regDC_PERFMON15_PERFCOUNTER_CNTL                                                                0x1e6a
+#define regDC_PERFMON15_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON15_PERFCOUNTER_CNTL2                                                               0x1e6b
+#define regDC_PERFMON15_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON15_PERFCOUNTER_STATE                                                               0x1e6c
+#define regDC_PERFMON15_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON15_PERFMON_CNTL                                                                    0x1e6d
+#define regDC_PERFMON15_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON15_PERFMON_CNTL2                                                                   0x1e6e
+#define regDC_PERFMON15_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON15_PERFMON_CVALUE_INT_MISC                                                         0x1e6f
+#define regDC_PERFMON15_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON15_PERFMON_CVALUE_LOW                                                              0x1e70
+#define regDC_PERFMON15_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON15_PERFMON_HI                                                                      0x1e71
+#define regDC_PERFMON15_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON15_PERFMON_LOW                                                                     0x1e72
+#define regDC_PERFMON15_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dio_dout_i2c_dispdec
+// base address: 0x0
+#define regDC_I2C_CONTROL                                                                               0x1e98
+#define regDC_I2C_CONTROL_BASE_IDX                                                                      2
+#define regDC_I2C_ARBITRATION                                                                           0x1e99
+#define regDC_I2C_ARBITRATION_BASE_IDX                                                                  2
+#define regDC_I2C_INTERRUPT_CONTROL                                                                     0x1e9a
+#define regDC_I2C_INTERRUPT_CONTROL_BASE_IDX                                                            2
+#define regDC_I2C_SW_STATUS                                                                             0x1e9b
+#define regDC_I2C_SW_STATUS_BASE_IDX                                                                    2
+#define regDC_I2C_DDC1_HW_STATUS                                                                        0x1e9c
+#define regDC_I2C_DDC1_HW_STATUS_BASE_IDX                                                               2
+#define regDC_I2C_DDC2_HW_STATUS                                                                        0x1e9d
+#define regDC_I2C_DDC2_HW_STATUS_BASE_IDX                                                               2
+#define regDC_I2C_DDC3_HW_STATUS                                                                        0x1e9e
+#define regDC_I2C_DDC3_HW_STATUS_BASE_IDX                                                               2
+#define regDC_I2C_DDC4_HW_STATUS                                                                        0x1e9f
+#define regDC_I2C_DDC4_HW_STATUS_BASE_IDX                                                               2
+#define regDC_I2C_DDC5_HW_STATUS                                                                        0x1ea0
+#define regDC_I2C_DDC5_HW_STATUS_BASE_IDX                                                               2
+#define regDC_I2C_DDC1_SPEED                                                                            0x1ea2
+#define regDC_I2C_DDC1_SPEED_BASE_IDX                                                                   2
+#define regDC_I2C_DDC1_SETUP                                                                            0x1ea3
+#define regDC_I2C_DDC1_SETUP_BASE_IDX                                                                   2
+#define regDC_I2C_DDC2_SPEED                                                                            0x1ea4
+#define regDC_I2C_DDC2_SPEED_BASE_IDX                                                                   2
+#define regDC_I2C_DDC2_SETUP                                                                            0x1ea5
+#define regDC_I2C_DDC2_SETUP_BASE_IDX                                                                   2
+#define regDC_I2C_DDC3_SPEED                                                                            0x1ea6
+#define regDC_I2C_DDC3_SPEED_BASE_IDX                                                                   2
+#define regDC_I2C_DDC3_SETUP                                                                            0x1ea7
+#define regDC_I2C_DDC3_SETUP_BASE_IDX                                                                   2
+#define regDC_I2C_DDC4_SPEED                                                                            0x1ea8
+#define regDC_I2C_DDC4_SPEED_BASE_IDX                                                                   2
+#define regDC_I2C_DDC4_SETUP                                                                            0x1ea9
+#define regDC_I2C_DDC4_SETUP_BASE_IDX                                                                   2
+#define regDC_I2C_DDC5_SPEED                                                                            0x1eaa
+#define regDC_I2C_DDC5_SPEED_BASE_IDX                                                                   2
+#define regDC_I2C_DDC5_SETUP                                                                            0x1eab
+#define regDC_I2C_DDC5_SETUP_BASE_IDX                                                                   2
+#define regDC_I2C_TRANSACTION0                                                                          0x1eae
+#define regDC_I2C_TRANSACTION0_BASE_IDX                                                                 2
+#define regDC_I2C_TRANSACTION1                                                                          0x1eaf
+#define regDC_I2C_TRANSACTION1_BASE_IDX                                                                 2
+#define regDC_I2C_TRANSACTION2                                                                          0x1eb0
+#define regDC_I2C_TRANSACTION2_BASE_IDX                                                                 2
+#define regDC_I2C_TRANSACTION3                                                                          0x1eb1
+#define regDC_I2C_TRANSACTION3_BASE_IDX                                                                 2
+#define regDC_I2C_DATA                                                                                  0x1eb2
+#define regDC_I2C_DATA_BASE_IDX                                                                         2
+#define regDC_I2C_EDID_DETECT_CTRL                                                                      0x1eb6
+#define regDC_I2C_EDID_DETECT_CTRL_BASE_IDX                                                             2
+#define regDC_I2C_READ_REQUEST_INTERRUPT                                                                0x1eb7
+#define regDC_I2C_READ_REQUEST_INTERRUPT_BASE_IDX                                                       2
+
+
+// addressBlock: dce_dc_dio_dio_misc_dispdec
+// base address: 0x0
+#define regDIO_SCRATCH0                                                                                 0x1eca
+#define regDIO_SCRATCH0_BASE_IDX                                                                        2
+#define regDIO_SCRATCH1                                                                                 0x1ecb
+#define regDIO_SCRATCH1_BASE_IDX                                                                        2
+#define regDIO_SCRATCH2                                                                                 0x1ecc
+#define regDIO_SCRATCH2_BASE_IDX                                                                        2
+#define regDIO_SCRATCH3                                                                                 0x1ecd
+#define regDIO_SCRATCH3_BASE_IDX                                                                        2
+#define regDIO_SCRATCH4                                                                                 0x1ece
+#define regDIO_SCRATCH4_BASE_IDX                                                                        2
+#define regDIO_SCRATCH5                                                                                 0x1ecf
+#define regDIO_SCRATCH5_BASE_IDX                                                                        2
+#define regDIO_SCRATCH6                                                                                 0x1ed0
+#define regDIO_SCRATCH6_BASE_IDX                                                                        2
+#define regDIO_SCRATCH7                                                                                 0x1ed1
+#define regDIO_SCRATCH7_BASE_IDX                                                                        2
+#define regDIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS                                                          0x1ed3
+#define regDIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS_BASE_IDX                                                 2
+#define regDIO_MEM_PWR_STATUS                                                                           0x1edd
+#define regDIO_MEM_PWR_STATUS_BASE_IDX                                                                  2
+#define regDIO_MEM_PWR_CTRL                                                                             0x1ede
+#define regDIO_MEM_PWR_CTRL_BASE_IDX                                                                    2
+#define regDIO_MEM_PWR_CTRL2                                                                            0x1edf
+#define regDIO_MEM_PWR_CTRL2_BASE_IDX                                                                   2
+#define regDIO_CLK_CNTL                                                                                 0x1ee0
+#define regDIO_CLK_CNTL_BASE_IDX                                                                        2
+#define regDIO_POWER_MANAGEMENT_CNTL                                                                    0x1ee4
+#define regDIO_POWER_MANAGEMENT_CNTL_BASE_IDX                                                           2
+#define regDIO_HDMI_RXSTATUS_TIMER_CONTROL                                                              0x1eff
+#define regDIO_HDMI_RXSTATUS_TIMER_CONTROL_BASE_IDX                                                     2
+#define regDIO_PSP_INTERRUPT_STATUS                                                                     0x1f00
+#define regDIO_PSP_INTERRUPT_STATUS_BASE_IDX                                                            2
+#define regDIO_PSP_INTERRUPT_CLEAR                                                                      0x1f01
+#define regDIO_PSP_INTERRUPT_CLEAR_BASE_IDX                                                             2
+#define regDIO_STATUS                                                                                   0x1f02
+#define regDIO_STATUS_BASE_IDX                                                                          2
+
+
+// addressBlock: dce_dc_dio_dig_stream_mapper_dispdec
+// base address: 0x0
+#define regDIG0_STREAM_MAPPER_CONTROL                                                                   0x1f0d
+#define regDIG0_STREAM_MAPPER_CONTROL_BASE_IDX                                                          2
+#define regDIG1_STREAM_MAPPER_CONTROL                                                                   0x1f0e
+#define regDIG1_STREAM_MAPPER_CONTROL_BASE_IDX                                                          2
+#define regDIG2_STREAM_MAPPER_CONTROL                                                                   0x1f0f
+#define regDIG2_STREAM_MAPPER_CONTROL_BASE_IDX                                                          2
+#define regDIG3_STREAM_MAPPER_CONTROL                                                                   0x1f10
+#define regDIG3_STREAM_MAPPER_CONTROL_BASE_IDX                                                          2
+#define regDIG4_STREAM_MAPPER_CONTROL                                                                   0x1f11
+#define regDIG4_STREAM_MAPPER_CONTROL_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dio_dio_dcperfmon_dc_perfmon_dispdec
+// base address: 0x7d10
+#define regDC_PERFMON16_PERFCOUNTER_CNTL                                                                0x1f44
+#define regDC_PERFMON16_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON16_PERFCOUNTER_CNTL2                                                               0x1f45
+#define regDC_PERFMON16_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON16_PERFCOUNTER_STATE                                                               0x1f46
+#define regDC_PERFMON16_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON16_PERFMON_CNTL                                                                    0x1f47
+#define regDC_PERFMON16_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON16_PERFMON_CNTL2                                                                   0x1f48
+#define regDC_PERFMON16_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON16_PERFMON_CVALUE_INT_MISC                                                         0x1f49
+#define regDC_PERFMON16_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON16_PERFMON_CVALUE_LOW                                                              0x1f4a
+#define regDC_PERFMON16_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON16_PERFMON_HI                                                                      0x1f4b
+#define regDC_PERFMON16_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON16_PERFMON_LOW                                                                     0x1f4c
+#define regDC_PERFMON16_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dio_dig0_vpg_vpg_dispdec
+// base address: 0x154a0
+#define regVPG0_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x2068
+#define regVPG0_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG0_VPG_GENERIC_PACKET_DATA                                                                 0x2069
+#define regVPG0_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG0_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x206a
+#define regVPG0_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x206b
+#define regVPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG0_VPG_GENERIC_STATUS                                                                      0x206c
+#define regVPG0_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG0_VPG_MEM_PWR                                                                             0x206d
+#define regVPG0_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG0_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x206e
+#define regVPG0_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG0_VPG_ISRC1_2_DATA                                                                        0x206f
+#define regVPG0_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dio_dig0_apg_apg_dispdec
+// base address: 0x154cc
+#define regAPG0_APG_CONTROL                                                                             0x2073
+#define regAPG0_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG0_APG_CONTROL2                                                                            0x2074
+#define regAPG0_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG0_APG_DBG_GEN_CONTROL                                                                     0x2075
+#define regAPG0_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG0_APG_PACKET_CONTROL                                                                      0x2076
+#define regAPG0_APG_PACKET_CONTROL_BASE_IDX                                                             2
+#define regAPG0_APG_DBG_ACP                                                                             0x2077
+#define regAPG0_APG_DBG_ACP_BASE_IDX                                                                    2
+#define regAPG0_APG_DBG_AUDIO_INFO                                                                      0x2079
+#define regAPG0_APG_DBG_AUDIO_INFO_BASE_IDX                                                             2
+#define regAPG0_APG_DBG_60958_0                                                                         0x207a
+#define regAPG0_APG_DBG_60958_0_BASE_IDX                                                                2
+#define regAPG0_APG_DBG_60958_1                                                                         0x207b
+#define regAPG0_APG_DBG_60958_1_BASE_IDX                                                                2
+#define regAPG0_APG_DBG_60958_2                                                                         0x207c
+#define regAPG0_APG_DBG_60958_2_BASE_IDX                                                                2
+#define regAPG0_APG_AUDIO_CRC_CONTROL                                                                   0x207d
+#define regAPG0_APG_AUDIO_CRC_CONTROL_BASE_IDX                                                          2
+#define regAPG0_APG_AUDIO_CRC_CONTROL2                                                                  0x207e
+#define regAPG0_APG_AUDIO_CRC_CONTROL2_BASE_IDX                                                         2
+#define regAPG0_APG_AUDIO_CRC_RESULT                                                                    0x207f
+#define regAPG0_APG_AUDIO_CRC_RESULT_BASE_IDX                                                           2
+#define regAPG0_APG_DBG_RAMP_CONTROL0                                                                   0x2080
+#define regAPG0_APG_DBG_RAMP_CONTROL0_BASE_IDX                                                          2
+#define regAPG0_APG_DBG_RAMP_CONTROL1                                                                   0x2081
+#define regAPG0_APG_DBG_RAMP_CONTROL1_BASE_IDX                                                          2
+#define regAPG0_APG_DBG_RAMP_CONTROL2                                                                   0x2082
+#define regAPG0_APG_DBG_RAMP_CONTROL2_BASE_IDX                                                          2
+#define regAPG0_APG_DBG_RAMP_CONTROL3                                                                   0x2083
+#define regAPG0_APG_DBG_RAMP_CONTROL3_BASE_IDX                                                          2
+#define regAPG0_APG_STATUS                                                                              0x2084
+#define regAPG0_APG_STATUS_BASE_IDX                                                                     2
+#define regAPG0_APG_STATUS2                                                                             0x2085
+#define regAPG0_APG_STATUS2_BASE_IDX                                                                    2
+#define regAPG0_APG_DBG_AUDIO_DTO_CNTL                                                                  0x2086
+#define regAPG0_APG_DBG_AUDIO_DTO_CNTL_BASE_IDX                                                         2
+#define regAPG0_APG_MEM_PWR                                                                             0x2087
+#define regAPG0_APG_MEM_PWR_BASE_IDX                                                                    2
+#define regAPG0_APG_SPARE                                                                               0x2089
+#define regAPG0_APG_SPARE_BASE_IDX                                                                      2
+
+
+// addressBlock: dce_dc_dio_dig0_dme_dme_dispdec
+// base address: 0x15544
+#define regDME0_DME_CONTROL                                                                             0x2091
+#define regDME0_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME0_DME_MEMORY_CONTROL                                                                      0x2092
+#define regDME0_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dio_dig0_dispdec
+// base address: 0x0
+#define regDIG0_DIG_FE_CNTL                                                                             0x2093
+#define regDIG0_DIG_FE_CNTL_BASE_IDX                                                                    2
+#define regDIG0_DIG_FE_CLK_CNTL                                                                         0x2094
+#define regDIG0_DIG_FE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG0_DIG_FE_EN_CNTL                                                                          0x2095
+#define regDIG0_DIG_FE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG0_DIG_OUTPUT_CRC_CNTL                                                                     0x2096
+#define regDIG0_DIG_OUTPUT_CRC_CNTL_BASE_IDX                                                            2
+#define regDIG0_DIG_OUTPUT_CRC_RESULT                                                                   0x2097
+#define regDIG0_DIG_OUTPUT_CRC_RESULT_BASE_IDX                                                          2
+#define regDIG0_DIG_CLOCK_PATTERN                                                                       0x2098
+#define regDIG0_DIG_CLOCK_PATTERN_BASE_IDX                                                              2
+#define regDIG0_DIG_TEST_PATTERN                                                                        0x2099
+#define regDIG0_DIG_TEST_PATTERN_BASE_IDX                                                               2
+#define regDIG0_DIG_RANDOM_PATTERN_SEED                                                                 0x209a
+#define regDIG0_DIG_RANDOM_PATTERN_SEED_BASE_IDX                                                        2
+#define regDIG0_DIG_FIFO_CTRL0                                                                          0x209b
+#define regDIG0_DIG_FIFO_CTRL0_BASE_IDX                                                                 2
+#define regDIG0_DIG_FIFO_CTRL1                                                                          0x209c
+#define regDIG0_DIG_FIFO_CTRL1_BASE_IDX                                                                 2
+#define regDIG0_HDMI_METADATA_PACKET_CONTROL                                                            0x209d
+#define regDIG0_HDMI_METADATA_PACKET_CONTROL_BASE_IDX                                                   2
+#define regDIG0_HDMI_CONTROL                                                                            0x209e
+#define regDIG0_HDMI_CONTROL_BASE_IDX                                                                   2
+#define regDIG0_HDMI_STATUS                                                                             0x209f
+#define regDIG0_HDMI_STATUS_BASE_IDX                                                                    2
+#define regDIG0_HDMI_AUDIO_PACKET_CONTROL                                                               0x20a0
+#define regDIG0_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX                                                      2
+#define regDIG0_HDMI_ACR_PACKET_CONTROL                                                                 0x20a1
+#define regDIG0_HDMI_ACR_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG0_HDMI_VBI_PACKET_CONTROL                                                                 0x20a2
+#define regDIG0_HDMI_VBI_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG0_HDMI_INFOFRAME_CONTROL0                                                                 0x20a3
+#define regDIG0_HDMI_INFOFRAME_CONTROL0_BASE_IDX                                                        2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL0                                                            0x20a4
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL6                                                            0x20a5
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL6_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL5                                                            0x20a6
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX                                                   2
+#define regDIG0_HDMI_GC                                                                                 0x20a7
+#define regDIG0_HDMI_GC_BASE_IDX                                                                        2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL1                                                            0x20a8
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL2                                                            0x20a9
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL3                                                            0x20aa
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL4                                                            0x20ab
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL7                                                            0x20ac
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL7_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL8                                                            0x20ad
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL8_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL9                                                            0x20ae
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL9_BASE_IDX                                                   2
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL10                                                           0x20af
+#define regDIG0_HDMI_GENERIC_PACKET_CONTROL10_BASE_IDX                                                  2
+#define regDIG0_HDMI_DB_CONTROL                                                                         0x20b0
+#define regDIG0_HDMI_DB_CONTROL_BASE_IDX                                                                2
+#define regDIG0_HDMI_ACR_32_0                                                                           0x20b1
+#define regDIG0_HDMI_ACR_32_0_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_32_1                                                                           0x20b2
+#define regDIG0_HDMI_ACR_32_1_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_44_0                                                                           0x20b3
+#define regDIG0_HDMI_ACR_44_0_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_44_1                                                                           0x20b4
+#define regDIG0_HDMI_ACR_44_1_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_48_0                                                                           0x20b5
+#define regDIG0_HDMI_ACR_48_0_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_48_1                                                                           0x20b6
+#define regDIG0_HDMI_ACR_48_1_BASE_IDX                                                                  2
+#define regDIG0_HDMI_ACR_STATUS_0                                                                       0x20b7
+#define regDIG0_HDMI_ACR_STATUS_0_BASE_IDX                                                              2
+#define regDIG0_HDMI_ACR_STATUS_1                                                                       0x20b8
+#define regDIG0_HDMI_ACR_STATUS_1_BASE_IDX                                                              2
+#define regDIG0_DIG_FE_AUDIO_CNTL                                                                       0x20b9
+#define regDIG0_DIG_FE_AUDIO_CNTL_BASE_IDX                                                              2
+#define regDIG0_DIG_BE_CLK_CNTL                                                                         0x20ba
+#define regDIG0_DIG_BE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG0_DIG_BE_CNTL                                                                             0x20bb
+#define regDIG0_DIG_BE_CNTL_BASE_IDX                                                                    2
+#define regDIG0_DIG_BE_EN_CNTL                                                                          0x20bc
+#define regDIG0_DIG_BE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG0_TMDS_CNTL                                                                               0x20e3
+#define regDIG0_TMDS_CNTL_BASE_IDX                                                                      2
+#define regDIG0_TMDS_CONTROL_CHAR                                                                       0x20e4
+#define regDIG0_TMDS_CONTROL_CHAR_BASE_IDX                                                              2
+#define regDIG0_TMDS_CONTROL0_FEEDBACK                                                                  0x20e5
+#define regDIG0_TMDS_CONTROL0_FEEDBACK_BASE_IDX                                                         2
+#define regDIG0_TMDS_STEREOSYNC_CTL_SEL                                                                 0x20e6
+#define regDIG0_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX                                                        2
+#define regDIG0_TMDS_SYNC_CHAR_PATTERN_0_1                                                              0x20e7
+#define regDIG0_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX                                                     2
+#define regDIG0_TMDS_SYNC_CHAR_PATTERN_2_3                                                              0x20e8
+#define regDIG0_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX                                                     2
+#define regDIG0_TMDS_CTL_BITS                                                                           0x20ea
+#define regDIG0_TMDS_CTL_BITS_BASE_IDX                                                                  2
+#define regDIG0_TMDS_DCBALANCER_CONTROL                                                                 0x20eb
+#define regDIG0_TMDS_DCBALANCER_CONTROL_BASE_IDX                                                        2
+#define regDIG0_TMDS_SYNC_DCBALANCE_CHAR                                                                0x20ec
+#define regDIG0_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX                                                       2
+#define regDIG0_TMDS_CTL0_1_GEN_CNTL                                                                    0x20ed
+#define regDIG0_TMDS_CTL0_1_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG0_TMDS_CTL2_3_GEN_CNTL                                                                    0x20ee
+#define regDIG0_TMDS_CTL2_3_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG0_DIG_VERSION                                                                             0x20f0
+#define regDIG0_DIG_VERSION_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_dio_dp0_dispdec
+// base address: 0x0
+#define regDP0_DP_LINK_CNTL                                                                             0x211e
+#define regDP0_DP_LINK_CNTL_BASE_IDX                                                                    2
+#define regDP0_DP_PIXEL_FORMAT                                                                          0x211f
+#define regDP0_DP_PIXEL_FORMAT_BASE_IDX                                                                 2
+#define regDP0_DP_MSA_COLORIMETRY                                                                       0x2120
+#define regDP0_DP_MSA_COLORIMETRY_BASE_IDX                                                              2
+#define regDP0_DP_CONFIG                                                                                0x2121
+#define regDP0_DP_CONFIG_BASE_IDX                                                                       2
+#define regDP0_DP_VID_STREAM_CNTL                                                                       0x2122
+#define regDP0_DP_VID_STREAM_CNTL_BASE_IDX                                                              2
+#define regDP0_DP_STEER_FIFO                                                                            0x2123
+#define regDP0_DP_STEER_FIFO_BASE_IDX                                                                   2
+#define regDP0_DP_MSA_MISC                                                                              0x2124
+#define regDP0_DP_MSA_MISC_BASE_IDX                                                                     2
+#define regDP0_DP_DPHY_INTERNAL_CTRL                                                                    0x2125
+#define regDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX                                                           2
+#define regDP0_DP_VID_TIMING                                                                            0x2126
+#define regDP0_DP_VID_TIMING_BASE_IDX                                                                   2
+#define regDP0_DP_VID_N                                                                                 0x2127
+#define regDP0_DP_VID_N_BASE_IDX                                                                        2
+#define regDP0_DP_VID_M                                                                                 0x2128
+#define regDP0_DP_VID_M_BASE_IDX                                                                        2
+#define regDP0_DP_LINK_FRAMING_CNTL                                                                     0x2129
+#define regDP0_DP_LINK_FRAMING_CNTL_BASE_IDX                                                            2
+#define regDP0_DP_HBR2_EYE_PATTERN                                                                      0x212a
+#define regDP0_DP_HBR2_EYE_PATTERN_BASE_IDX                                                             2
+#define regDP0_DP_VID_MSA_VBID                                                                          0x212b
+#define regDP0_DP_VID_MSA_VBID_BASE_IDX                                                                 2
+#define regDP0_DP_VID_INTERRUPT_CNTL                                                                    0x212c
+#define regDP0_DP_VID_INTERRUPT_CNTL_BASE_IDX                                                           2
+#define regDP0_DP_DPHY_CNTL                                                                             0x212d
+#define regDP0_DP_DPHY_CNTL_BASE_IDX                                                                    2
+#define regDP0_DP_DPHY_TRAINING_PATTERN_SEL                                                             0x212e
+#define regDP0_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX                                                    2
+#define regDP0_DP_DPHY_SYM0                                                                             0x212f
+#define regDP0_DP_DPHY_SYM0_BASE_IDX                                                                    2
+#define regDP0_DP_DPHY_SYM1                                                                             0x2130
+#define regDP0_DP_DPHY_SYM1_BASE_IDX                                                                    2
+#define regDP0_DP_DPHY_SYM2                                                                             0x2131
+#define regDP0_DP_DPHY_SYM2_BASE_IDX                                                                    2
+#define regDP0_DP_DPHY_8B10B_CNTL                                                                       0x2132
+#define regDP0_DP_DPHY_8B10B_CNTL_BASE_IDX                                                              2
+#define regDP0_DP_DPHY_PRBS_CNTL                                                                        0x2133
+#define regDP0_DP_DPHY_PRBS_CNTL_BASE_IDX                                                               2
+#define regDP0_DP_DPHY_SCRAM_CNTL                                                                       0x2134
+#define regDP0_DP_DPHY_SCRAM_CNTL_BASE_IDX                                                              2
+#define regDP0_DP_DPHY_CRC_CONTROL0                                                                     0x2135
+#define regDP0_DP_DPHY_CRC_CONTROL0_BASE_IDX                                                            2
+#define regDP0_DP_DPHY_CRC_CONTROL1                                                                     0x2136
+#define regDP0_DP_DPHY_CRC_CONTROL1_BASE_IDX                                                            2
+#define regDP0_DP_DPHY_CRC_RESULT0                                                                      0x2137
+#define regDP0_DP_DPHY_CRC_RESULT0_BASE_IDX                                                             2
+#define regDP0_DP_DPHY_CRC_RESULT1                                                                      0x2138
+#define regDP0_DP_DPHY_CRC_RESULT1_BASE_IDX                                                             2
+#define regDP0_DP_DPHY_CRC_RESULT2                                                                      0x2139
+#define regDP0_DP_DPHY_CRC_RESULT2_BASE_IDX                                                             2
+#define regDP0_DP_DPHY_CRC_RESULT3                                                                      0x213a
+#define regDP0_DP_DPHY_CRC_RESULT3_BASE_IDX                                                             2
+#define regDP0_DP_DPHY_CRC_STATUS                                                                       0x213b
+#define regDP0_DP_DPHY_CRC_STATUS_BASE_IDX                                                              2
+#define regDP0_DP_TU_CNTL                                                                               0x213c
+#define regDP0_DP_TU_CNTL_BASE_IDX                                                                      2
+#define regDP0_DP_PIXEL_FORMAT_DB_CNTL                                                                  0x213d
+#define regDP0_DP_PIXEL_FORMAT_DB_CNTL_BASE_IDX                                                         2
+#define regDP0_DP_SEC_CNTL                                                                              0x2141
+#define regDP0_DP_SEC_CNTL_BASE_IDX                                                                     2
+#define regDP0_DP_SEC_CNTL1                                                                             0x2142
+#define regDP0_DP_SEC_CNTL1_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_FRAMING1                                                                          0x2143
+#define regDP0_DP_SEC_FRAMING1_BASE_IDX                                                                 2
+#define regDP0_DP_SEC_FRAMING2                                                                          0x2144
+#define regDP0_DP_SEC_FRAMING2_BASE_IDX                                                                 2
+#define regDP0_DP_SEC_FRAMING3                                                                          0x2145
+#define regDP0_DP_SEC_FRAMING3_BASE_IDX                                                                 2
+#define regDP0_DP_SEC_FRAMING4                                                                          0x2146
+#define regDP0_DP_SEC_FRAMING4_BASE_IDX                                                                 2
+#define regDP0_DP_SEC_AUD_N                                                                             0x2147
+#define regDP0_DP_SEC_AUD_N_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_AUD_N_READBACK                                                                    0x2148
+#define regDP0_DP_SEC_AUD_N_READBACK_BASE_IDX                                                           2
+#define regDP0_DP_SEC_AUD_M                                                                             0x2149
+#define regDP0_DP_SEC_AUD_M_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_AUD_M_READBACK                                                                    0x214a
+#define regDP0_DP_SEC_AUD_M_READBACK_BASE_IDX                                                           2
+#define regDP0_DP_SEC_TIMESTAMP                                                                         0x214b
+#define regDP0_DP_SEC_TIMESTAMP_BASE_IDX                                                                2
+#define regDP0_DP_SEC_PACKET_CNTL                                                                       0x214c
+#define regDP0_DP_SEC_PACKET_CNTL_BASE_IDX                                                              2
+#define regDP0_DP_MSE_RATE_CNTL                                                                         0x214d
+#define regDP0_DP_MSE_RATE_CNTL_BASE_IDX                                                                2
+#define regDP0_DP_MSE_RATE_UPDATE                                                                       0x214f
+#define regDP0_DP_MSE_RATE_UPDATE_BASE_IDX                                                              2
+#define regDP0_DP_MSE_SAT0                                                                              0x2150
+#define regDP0_DP_MSE_SAT0_BASE_IDX                                                                     2
+#define regDP0_DP_MSE_SAT1                                                                              0x2151
+#define regDP0_DP_MSE_SAT1_BASE_IDX                                                                     2
+#define regDP0_DP_MSE_SAT2                                                                              0x2152
+#define regDP0_DP_MSE_SAT2_BASE_IDX                                                                     2
+#define regDP0_DP_MSE_SAT_UPDATE                                                                        0x2153
+#define regDP0_DP_MSE_SAT_UPDATE_BASE_IDX                                                               2
+#define regDP0_DP_MSE_LINK_TIMING                                                                       0x2154
+#define regDP0_DP_MSE_LINK_TIMING_BASE_IDX                                                              2
+#define regDP0_DP_MSE_MISC_CNTL                                                                         0x2155
+#define regDP0_DP_MSE_MISC_CNTL_BASE_IDX                                                                2
+#define regDP0_DP_DPHY_BS_SR_SWAP_CNTL                                                                  0x215a
+#define regDP0_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX                                                         2
+#define regDP0_DP_DPHY_HBR2_PATTERN_CONTROL                                                             0x215b
+#define regDP0_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX                                                    2
+#define regDP0_DP_MSE_SAT0_STATUS                                                                       0x215d
+#define regDP0_DP_MSE_SAT0_STATUS_BASE_IDX                                                              2
+#define regDP0_DP_MSE_SAT1_STATUS                                                                       0x215e
+#define regDP0_DP_MSE_SAT1_STATUS_BASE_IDX                                                              2
+#define regDP0_DP_MSE_SAT2_STATUS                                                                       0x215f
+#define regDP0_DP_MSE_SAT2_STATUS_BASE_IDX                                                              2
+#define regDP0_DP_HBLANK_CONTROL                                                                        0x2161
+#define regDP0_DP_HBLANK_CONTROL_BASE_IDX                                                               2
+#define regDP0_DP_MSA_TIMING_PARAM1                                                                     0x2162
+#define regDP0_DP_MSA_TIMING_PARAM1_BASE_IDX                                                            2
+#define regDP0_DP_MSA_TIMING_PARAM2                                                                     0x2163
+#define regDP0_DP_MSA_TIMING_PARAM2_BASE_IDX                                                            2
+#define regDP0_DP_MSA_TIMING_PARAM3                                                                     0x2164
+#define regDP0_DP_MSA_TIMING_PARAM3_BASE_IDX                                                            2
+#define regDP0_DP_MSA_TIMING_PARAM4                                                                     0x2165
+#define regDP0_DP_MSA_TIMING_PARAM4_BASE_IDX                                                            2
+#define regDP0_DP_MSO_CNTL                                                                              0x2166
+#define regDP0_DP_MSO_CNTL_BASE_IDX                                                                     2
+#define regDP0_DP_MSO_CNTL1                                                                             0x2167
+#define regDP0_DP_MSO_CNTL1_BASE_IDX                                                                    2
+#define regDP0_DP_STEER_FIFO_CNTL                                                                       0x2168
+#define regDP0_DP_STEER_FIFO_CNTL_BASE_IDX                                                              2
+#define regDP0_DP_SEC_CNTL2                                                                             0x2169
+#define regDP0_DP_SEC_CNTL2_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_CNTL3                                                                             0x216a
+#define regDP0_DP_SEC_CNTL3_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_CNTL4                                                                             0x216b
+#define regDP0_DP_SEC_CNTL4_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_CNTL5                                                                             0x216c
+#define regDP0_DP_SEC_CNTL5_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_CNTL6                                                                             0x216d
+#define regDP0_DP_SEC_CNTL6_BASE_IDX                                                                    2
+#define regDP0_DP_SEC_CNTL7                                                                             0x216e
+#define regDP0_DP_SEC_CNTL7_BASE_IDX                                                                    2
+#define regDP0_DP_DB_CNTL                                                                               0x216f
+#define regDP0_DP_DB_CNTL_BASE_IDX                                                                      2
+#define regDP0_DP_MSA_VBID_MISC                                                                         0x2170
+#define regDP0_DP_MSA_VBID_MISC_BASE_IDX                                                                2
+#define regDP0_DP_SEC_METADATA_TRANSMISSION                                                             0x2171
+#define regDP0_DP_SEC_METADATA_TRANSMISSION_BASE_IDX                                                    2
+#define regDP0_DP_ALPM_CNTL                                                                             0x2173
+#define regDP0_DP_ALPM_CNTL_BASE_IDX                                                                    2
+#define regDP0_DP_GSP8_CNTL                                                                             0x2174
+#define regDP0_DP_GSP8_CNTL_BASE_IDX                                                                    2
+#define regDP0_DP_GSP9_CNTL                                                                             0x2175
+#define regDP0_DP_GSP9_CNTL_BASE_IDX                                                                    2
+#define regDP0_DP_GSP10_CNTL                                                                            0x2176
+#define regDP0_DP_GSP10_CNTL_BASE_IDX                                                                   2
+#define regDP0_DP_GSP11_CNTL                                                                            0x2177
+#define regDP0_DP_GSP11_CNTL_BASE_IDX                                                                   2
+#define regDP0_DP_GSP_EN_DB_STATUS                                                                      0x2178
+#define regDP0_DP_GSP_EN_DB_STATUS_BASE_IDX                                                             2
+#define regDP0_DP_AUXLESS_ALPM_CNTL1                                                                    0x2179
+#define regDP0_DP_AUXLESS_ALPM_CNTL1_BASE_IDX                                                           2
+#define regDP0_DP_AUXLESS_ALPM_CNTL2                                                                    0x217a
+#define regDP0_DP_AUXLESS_ALPM_CNTL2_BASE_IDX                                                           2
+#define regDP0_DP_AUXLESS_ALPM_CNTL3                                                                    0x217b
+#define regDP0_DP_AUXLESS_ALPM_CNTL3_BASE_IDX                                                           2
+#define regDP0_DP_AUXLESS_ALPM_CNTL4                                                                    0x217c
+#define regDP0_DP_AUXLESS_ALPM_CNTL4_BASE_IDX                                                           2
+#define regDP0_DP_AUXLESS_ALPM_CNTL5                                                                    0x217d
+#define regDP0_DP_AUXLESS_ALPM_CNTL5_BASE_IDX                                                           2
+#define regDP0_DP_STREAM_SYMBOL_COUNT_STATUS                                                            0x217e
+#define regDP0_DP_STREAM_SYMBOL_COUNT_STATUS_BASE_IDX                                                   2
+#define regDP0_DP_STREAM_SYMBOL_COUNT_CONTROL                                                           0x217f
+#define regDP0_DP_STREAM_SYMBOL_COUNT_CONTROL_BASE_IDX                                                  2
+#define regDP0_DP_LINK_SYMBOL_COUNT_STATUS0                                                             0x2180
+#define regDP0_DP_LINK_SYMBOL_COUNT_STATUS0_BASE_IDX                                                    2
+#define regDP0_DP_LINK_SYMBOL_COUNT_STATUS1                                                             0x2181
+#define regDP0_DP_LINK_SYMBOL_COUNT_STATUS1_BASE_IDX                                                    2
+#define regDP0_DP_LINK_SYMBOL_COUNT_CONTROL                                                             0x2182
+#define regDP0_DP_LINK_SYMBOL_COUNT_CONTROL_BASE_IDX                                                    2
+#define regDP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL                                                     0x2183
+#define regDP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                            2
+#define regDP0_DP_DPHY_FAST_TRAINING                                                                    0x2186
+#define regDP0_DP_DPHY_FAST_TRAINING_BASE_IDX                                                           2
+#define regDP0_DP_DPHY_FAST_TRAINING_STATUS                                                             0x2187
+#define regDP0_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dio_dig1_vpg_vpg_dispdec
+// base address: 0x15930
+#define regVPG1_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x218c
+#define regVPG1_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG1_VPG_GENERIC_PACKET_DATA                                                                 0x218d
+#define regVPG1_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG1_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x218e
+#define regVPG1_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x218f
+#define regVPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG1_VPG_GENERIC_STATUS                                                                      0x2190
+#define regVPG1_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG1_VPG_MEM_PWR                                                                             0x2191
+#define regVPG1_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG1_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x2192
+#define regVPG1_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG1_VPG_ISRC1_2_DATA                                                                        0x2193
+#define regVPG1_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dio_dig1_apg_apg_dispdec
+// base address: 0x1595c
+#define regAPG1_APG_CONTROL                                                                             0x2197
+#define regAPG1_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG1_APG_CONTROL2                                                                            0x2198
+#define regAPG1_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG1_APG_DBG_GEN_CONTROL                                                                     0x2199
+#define regAPG1_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG1_APG_PACKET_CONTROL                                                                      0x219a
+#define regAPG1_APG_PACKET_CONTROL_BASE_IDX                                                             2
+#define regAPG1_APG_DBG_ACP                                                                             0x219b
+#define regAPG1_APG_DBG_ACP_BASE_IDX                                                                    2
+#define regAPG1_APG_DBG_AUDIO_INFO                                                                      0x219d
+#define regAPG1_APG_DBG_AUDIO_INFO_BASE_IDX                                                             2
+#define regAPG1_APG_DBG_60958_0                                                                         0x219e
+#define regAPG1_APG_DBG_60958_0_BASE_IDX                                                                2
+#define regAPG1_APG_DBG_60958_1                                                                         0x219f
+#define regAPG1_APG_DBG_60958_1_BASE_IDX                                                                2
+#define regAPG1_APG_DBG_60958_2                                                                         0x21a0
+#define regAPG1_APG_DBG_60958_2_BASE_IDX                                                                2
+#define regAPG1_APG_AUDIO_CRC_CONTROL                                                                   0x21a1
+#define regAPG1_APG_AUDIO_CRC_CONTROL_BASE_IDX                                                          2
+#define regAPG1_APG_AUDIO_CRC_CONTROL2                                                                  0x21a2
+#define regAPG1_APG_AUDIO_CRC_CONTROL2_BASE_IDX                                                         2
+#define regAPG1_APG_AUDIO_CRC_RESULT                                                                    0x21a3
+#define regAPG1_APG_AUDIO_CRC_RESULT_BASE_IDX                                                           2
+#define regAPG1_APG_DBG_RAMP_CONTROL0                                                                   0x21a4
+#define regAPG1_APG_DBG_RAMP_CONTROL0_BASE_IDX                                                          2
+#define regAPG1_APG_DBG_RAMP_CONTROL1                                                                   0x21a5
+#define regAPG1_APG_DBG_RAMP_CONTROL1_BASE_IDX                                                          2
+#define regAPG1_APG_DBG_RAMP_CONTROL2                                                                   0x21a6
+#define regAPG1_APG_DBG_RAMP_CONTROL2_BASE_IDX                                                          2
+#define regAPG1_APG_DBG_RAMP_CONTROL3                                                                   0x21a7
+#define regAPG1_APG_DBG_RAMP_CONTROL3_BASE_IDX                                                          2
+#define regAPG1_APG_STATUS                                                                              0x21a8
+#define regAPG1_APG_STATUS_BASE_IDX                                                                     2
+#define regAPG1_APG_STATUS2                                                                             0x21a9
+#define regAPG1_APG_STATUS2_BASE_IDX                                                                    2
+#define regAPG1_APG_DBG_AUDIO_DTO_CNTL                                                                  0x21aa
+#define regAPG1_APG_DBG_AUDIO_DTO_CNTL_BASE_IDX                                                         2
+#define regAPG1_APG_MEM_PWR                                                                             0x21ab
+#define regAPG1_APG_MEM_PWR_BASE_IDX                                                                    2
+#define regAPG1_APG_SPARE                                                                               0x21ad
+#define regAPG1_APG_SPARE_BASE_IDX                                                                      2
+
+
+// addressBlock: dce_dc_dio_dig1_dme_dme_dispdec
+// base address: 0x159d4
+#define regDME1_DME_CONTROL                                                                             0x21b5
+#define regDME1_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME1_DME_MEMORY_CONTROL                                                                      0x21b6
+#define regDME1_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dio_dig1_dispdec
+// base address: 0x490
+#define regDIG1_DIG_FE_CNTL                                                                             0x21b7
+#define regDIG1_DIG_FE_CNTL_BASE_IDX                                                                    2
+#define regDIG1_DIG_FE_CLK_CNTL                                                                         0x21b8
+#define regDIG1_DIG_FE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG1_DIG_FE_EN_CNTL                                                                          0x21b9
+#define regDIG1_DIG_FE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG1_DIG_OUTPUT_CRC_CNTL                                                                     0x21ba
+#define regDIG1_DIG_OUTPUT_CRC_CNTL_BASE_IDX                                                            2
+#define regDIG1_DIG_OUTPUT_CRC_RESULT                                                                   0x21bb
+#define regDIG1_DIG_OUTPUT_CRC_RESULT_BASE_IDX                                                          2
+#define regDIG1_DIG_CLOCK_PATTERN                                                                       0x21bc
+#define regDIG1_DIG_CLOCK_PATTERN_BASE_IDX                                                              2
+#define regDIG1_DIG_TEST_PATTERN                                                                        0x21bd
+#define regDIG1_DIG_TEST_PATTERN_BASE_IDX                                                               2
+#define regDIG1_DIG_RANDOM_PATTERN_SEED                                                                 0x21be
+#define regDIG1_DIG_RANDOM_PATTERN_SEED_BASE_IDX                                                        2
+#define regDIG1_DIG_FIFO_CTRL0                                                                          0x21bf
+#define regDIG1_DIG_FIFO_CTRL0_BASE_IDX                                                                 2
+#define regDIG1_DIG_FIFO_CTRL1                                                                          0x21c0
+#define regDIG1_DIG_FIFO_CTRL1_BASE_IDX                                                                 2
+#define regDIG1_HDMI_METADATA_PACKET_CONTROL                                                            0x21c1
+#define regDIG1_HDMI_METADATA_PACKET_CONTROL_BASE_IDX                                                   2
+#define regDIG1_HDMI_CONTROL                                                                            0x21c2
+#define regDIG1_HDMI_CONTROL_BASE_IDX                                                                   2
+#define regDIG1_HDMI_STATUS                                                                             0x21c3
+#define regDIG1_HDMI_STATUS_BASE_IDX                                                                    2
+#define regDIG1_HDMI_AUDIO_PACKET_CONTROL                                                               0x21c4
+#define regDIG1_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX                                                      2
+#define regDIG1_HDMI_ACR_PACKET_CONTROL                                                                 0x21c5
+#define regDIG1_HDMI_ACR_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG1_HDMI_VBI_PACKET_CONTROL                                                                 0x21c6
+#define regDIG1_HDMI_VBI_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG1_HDMI_INFOFRAME_CONTROL0                                                                 0x21c7
+#define regDIG1_HDMI_INFOFRAME_CONTROL0_BASE_IDX                                                        2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL0                                                            0x21c8
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL6                                                            0x21c9
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL6_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL5                                                            0x21ca
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX                                                   2
+#define regDIG1_HDMI_GC                                                                                 0x21cb
+#define regDIG1_HDMI_GC_BASE_IDX                                                                        2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL1                                                            0x21cc
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL2                                                            0x21cd
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL3                                                            0x21ce
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL4                                                            0x21cf
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL7                                                            0x21d0
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL7_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL8                                                            0x21d1
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL8_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL9                                                            0x21d2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL9_BASE_IDX                                                   2
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL10                                                           0x21d3
+#define regDIG1_HDMI_GENERIC_PACKET_CONTROL10_BASE_IDX                                                  2
+#define regDIG1_HDMI_DB_CONTROL                                                                         0x21d4
+#define regDIG1_HDMI_DB_CONTROL_BASE_IDX                                                                2
+#define regDIG1_HDMI_ACR_32_0                                                                           0x21d5
+#define regDIG1_HDMI_ACR_32_0_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_32_1                                                                           0x21d6
+#define regDIG1_HDMI_ACR_32_1_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_44_0                                                                           0x21d7
+#define regDIG1_HDMI_ACR_44_0_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_44_1                                                                           0x21d8
+#define regDIG1_HDMI_ACR_44_1_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_48_0                                                                           0x21d9
+#define regDIG1_HDMI_ACR_48_0_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_48_1                                                                           0x21da
+#define regDIG1_HDMI_ACR_48_1_BASE_IDX                                                                  2
+#define regDIG1_HDMI_ACR_STATUS_0                                                                       0x21db
+#define regDIG1_HDMI_ACR_STATUS_0_BASE_IDX                                                              2
+#define regDIG1_HDMI_ACR_STATUS_1                                                                       0x21dc
+#define regDIG1_HDMI_ACR_STATUS_1_BASE_IDX                                                              2
+#define regDIG1_DIG_FE_AUDIO_CNTL                                                                       0x21dd
+#define regDIG1_DIG_FE_AUDIO_CNTL_BASE_IDX                                                              2
+#define regDIG1_DIG_BE_CLK_CNTL                                                                         0x21de
+#define regDIG1_DIG_BE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG1_DIG_BE_CNTL                                                                             0x21df
+#define regDIG1_DIG_BE_CNTL_BASE_IDX                                                                    2
+#define regDIG1_DIG_BE_EN_CNTL                                                                          0x21e0
+#define regDIG1_DIG_BE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG1_TMDS_CNTL                                                                               0x2207
+#define regDIG1_TMDS_CNTL_BASE_IDX                                                                      2
+#define regDIG1_TMDS_CONTROL_CHAR                                                                       0x2208
+#define regDIG1_TMDS_CONTROL_CHAR_BASE_IDX                                                              2
+#define regDIG1_TMDS_CONTROL0_FEEDBACK                                                                  0x2209
+#define regDIG1_TMDS_CONTROL0_FEEDBACK_BASE_IDX                                                         2
+#define regDIG1_TMDS_STEREOSYNC_CTL_SEL                                                                 0x220a
+#define regDIG1_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX                                                        2
+#define regDIG1_TMDS_SYNC_CHAR_PATTERN_0_1                                                              0x220b
+#define regDIG1_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX                                                     2
+#define regDIG1_TMDS_SYNC_CHAR_PATTERN_2_3                                                              0x220c
+#define regDIG1_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX                                                     2
+#define regDIG1_TMDS_CTL_BITS                                                                           0x220e
+#define regDIG1_TMDS_CTL_BITS_BASE_IDX                                                                  2
+#define regDIG1_TMDS_DCBALANCER_CONTROL                                                                 0x220f
+#define regDIG1_TMDS_DCBALANCER_CONTROL_BASE_IDX                                                        2
+#define regDIG1_TMDS_SYNC_DCBALANCE_CHAR                                                                0x2210
+#define regDIG1_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX                                                       2
+#define regDIG1_TMDS_CTL0_1_GEN_CNTL                                                                    0x2211
+#define regDIG1_TMDS_CTL0_1_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG1_TMDS_CTL2_3_GEN_CNTL                                                                    0x2212
+#define regDIG1_TMDS_CTL2_3_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG1_DIG_VERSION                                                                             0x2214
+#define regDIG1_DIG_VERSION_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_dio_dp1_dispdec
+// base address: 0x490
+#define regDP1_DP_LINK_CNTL                                                                             0x2242
+#define regDP1_DP_LINK_CNTL_BASE_IDX                                                                    2
+#define regDP1_DP_PIXEL_FORMAT                                                                          0x2243
+#define regDP1_DP_PIXEL_FORMAT_BASE_IDX                                                                 2
+#define regDP1_DP_MSA_COLORIMETRY                                                                       0x2244
+#define regDP1_DP_MSA_COLORIMETRY_BASE_IDX                                                              2
+#define regDP1_DP_CONFIG                                                                                0x2245
+#define regDP1_DP_CONFIG_BASE_IDX                                                                       2
+#define regDP1_DP_VID_STREAM_CNTL                                                                       0x2246
+#define regDP1_DP_VID_STREAM_CNTL_BASE_IDX                                                              2
+#define regDP1_DP_STEER_FIFO                                                                            0x2247
+#define regDP1_DP_STEER_FIFO_BASE_IDX                                                                   2
+#define regDP1_DP_MSA_MISC                                                                              0x2248
+#define regDP1_DP_MSA_MISC_BASE_IDX                                                                     2
+#define regDP1_DP_DPHY_INTERNAL_CTRL                                                                    0x2249
+#define regDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX                                                           2
+#define regDP1_DP_VID_TIMING                                                                            0x224a
+#define regDP1_DP_VID_TIMING_BASE_IDX                                                                   2
+#define regDP1_DP_VID_N                                                                                 0x224b
+#define regDP1_DP_VID_N_BASE_IDX                                                                        2
+#define regDP1_DP_VID_M                                                                                 0x224c
+#define regDP1_DP_VID_M_BASE_IDX                                                                        2
+#define regDP1_DP_LINK_FRAMING_CNTL                                                                     0x224d
+#define regDP1_DP_LINK_FRAMING_CNTL_BASE_IDX                                                            2
+#define regDP1_DP_HBR2_EYE_PATTERN                                                                      0x224e
+#define regDP1_DP_HBR2_EYE_PATTERN_BASE_IDX                                                             2
+#define regDP1_DP_VID_MSA_VBID                                                                          0x224f
+#define regDP1_DP_VID_MSA_VBID_BASE_IDX                                                                 2
+#define regDP1_DP_VID_INTERRUPT_CNTL                                                                    0x2250
+#define regDP1_DP_VID_INTERRUPT_CNTL_BASE_IDX                                                           2
+#define regDP1_DP_DPHY_CNTL                                                                             0x2251
+#define regDP1_DP_DPHY_CNTL_BASE_IDX                                                                    2
+#define regDP1_DP_DPHY_TRAINING_PATTERN_SEL                                                             0x2252
+#define regDP1_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX                                                    2
+#define regDP1_DP_DPHY_SYM0                                                                             0x2253
+#define regDP1_DP_DPHY_SYM0_BASE_IDX                                                                    2
+#define regDP1_DP_DPHY_SYM1                                                                             0x2254
+#define regDP1_DP_DPHY_SYM1_BASE_IDX                                                                    2
+#define regDP1_DP_DPHY_SYM2                                                                             0x2255
+#define regDP1_DP_DPHY_SYM2_BASE_IDX                                                                    2
+#define regDP1_DP_DPHY_8B10B_CNTL                                                                       0x2256
+#define regDP1_DP_DPHY_8B10B_CNTL_BASE_IDX                                                              2
+#define regDP1_DP_DPHY_PRBS_CNTL                                                                        0x2257
+#define regDP1_DP_DPHY_PRBS_CNTL_BASE_IDX                                                               2
+#define regDP1_DP_DPHY_SCRAM_CNTL                                                                       0x2258
+#define regDP1_DP_DPHY_SCRAM_CNTL_BASE_IDX                                                              2
+#define regDP1_DP_DPHY_CRC_CONTROL0                                                                     0x2259
+#define regDP1_DP_DPHY_CRC_CONTROL0_BASE_IDX                                                            2
+#define regDP1_DP_DPHY_CRC_CONTROL1                                                                     0x225a
+#define regDP1_DP_DPHY_CRC_CONTROL1_BASE_IDX                                                            2
+#define regDP1_DP_DPHY_CRC_RESULT0                                                                      0x225b
+#define regDP1_DP_DPHY_CRC_RESULT0_BASE_IDX                                                             2
+#define regDP1_DP_DPHY_CRC_RESULT1                                                                      0x225c
+#define regDP1_DP_DPHY_CRC_RESULT1_BASE_IDX                                                             2
+#define regDP1_DP_DPHY_CRC_RESULT2                                                                      0x225d
+#define regDP1_DP_DPHY_CRC_RESULT2_BASE_IDX                                                             2
+#define regDP1_DP_DPHY_CRC_RESULT3                                                                      0x225e
+#define regDP1_DP_DPHY_CRC_RESULT3_BASE_IDX                                                             2
+#define regDP1_DP_DPHY_CRC_STATUS                                                                       0x225f
+#define regDP1_DP_DPHY_CRC_STATUS_BASE_IDX                                                              2
+#define regDP1_DP_TU_CNTL                                                                               0x2260
+#define regDP1_DP_TU_CNTL_BASE_IDX                                                                      2
+#define regDP1_DP_PIXEL_FORMAT_DB_CNTL                                                                  0x2261
+#define regDP1_DP_PIXEL_FORMAT_DB_CNTL_BASE_IDX                                                         2
+#define regDP1_DP_SEC_CNTL                                                                              0x2265
+#define regDP1_DP_SEC_CNTL_BASE_IDX                                                                     2
+#define regDP1_DP_SEC_CNTL1                                                                             0x2266
+#define regDP1_DP_SEC_CNTL1_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_FRAMING1                                                                          0x2267
+#define regDP1_DP_SEC_FRAMING1_BASE_IDX                                                                 2
+#define regDP1_DP_SEC_FRAMING2                                                                          0x2268
+#define regDP1_DP_SEC_FRAMING2_BASE_IDX                                                                 2
+#define regDP1_DP_SEC_FRAMING3                                                                          0x2269
+#define regDP1_DP_SEC_FRAMING3_BASE_IDX                                                                 2
+#define regDP1_DP_SEC_FRAMING4                                                                          0x226a
+#define regDP1_DP_SEC_FRAMING4_BASE_IDX                                                                 2
+#define regDP1_DP_SEC_AUD_N                                                                             0x226b
+#define regDP1_DP_SEC_AUD_N_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_AUD_N_READBACK                                                                    0x226c
+#define regDP1_DP_SEC_AUD_N_READBACK_BASE_IDX                                                           2
+#define regDP1_DP_SEC_AUD_M                                                                             0x226d
+#define regDP1_DP_SEC_AUD_M_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_AUD_M_READBACK                                                                    0x226e
+#define regDP1_DP_SEC_AUD_M_READBACK_BASE_IDX                                                           2
+#define regDP1_DP_SEC_TIMESTAMP                                                                         0x226f
+#define regDP1_DP_SEC_TIMESTAMP_BASE_IDX                                                                2
+#define regDP1_DP_SEC_PACKET_CNTL                                                                       0x2270
+#define regDP1_DP_SEC_PACKET_CNTL_BASE_IDX                                                              2
+#define regDP1_DP_MSE_RATE_CNTL                                                                         0x2271
+#define regDP1_DP_MSE_RATE_CNTL_BASE_IDX                                                                2
+#define regDP1_DP_MSE_RATE_UPDATE                                                                       0x2273
+#define regDP1_DP_MSE_RATE_UPDATE_BASE_IDX                                                              2
+#define regDP1_DP_MSE_SAT0                                                                              0x2274
+#define regDP1_DP_MSE_SAT0_BASE_IDX                                                                     2
+#define regDP1_DP_MSE_SAT1                                                                              0x2275
+#define regDP1_DP_MSE_SAT1_BASE_IDX                                                                     2
+#define regDP1_DP_MSE_SAT2                                                                              0x2276
+#define regDP1_DP_MSE_SAT2_BASE_IDX                                                                     2
+#define regDP1_DP_MSE_SAT_UPDATE                                                                        0x2277
+#define regDP1_DP_MSE_SAT_UPDATE_BASE_IDX                                                               2
+#define regDP1_DP_MSE_LINK_TIMING                                                                       0x2278
+#define regDP1_DP_MSE_LINK_TIMING_BASE_IDX                                                              2
+#define regDP1_DP_MSE_MISC_CNTL                                                                         0x2279
+#define regDP1_DP_MSE_MISC_CNTL_BASE_IDX                                                                2
+#define regDP1_DP_DPHY_BS_SR_SWAP_CNTL                                                                  0x227e
+#define regDP1_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX                                                         2
+#define regDP1_DP_DPHY_HBR2_PATTERN_CONTROL                                                             0x227f
+#define regDP1_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX                                                    2
+#define regDP1_DP_MSE_SAT0_STATUS                                                                       0x2281
+#define regDP1_DP_MSE_SAT0_STATUS_BASE_IDX                                                              2
+#define regDP1_DP_MSE_SAT1_STATUS                                                                       0x2282
+#define regDP1_DP_MSE_SAT1_STATUS_BASE_IDX                                                              2
+#define regDP1_DP_MSE_SAT2_STATUS                                                                       0x2283
+#define regDP1_DP_MSE_SAT2_STATUS_BASE_IDX                                                              2
+#define regDP1_DP_HBLANK_CONTROL                                                                        0x2285
+#define regDP1_DP_HBLANK_CONTROL_BASE_IDX                                                               2
+#define regDP1_DP_MSA_TIMING_PARAM1                                                                     0x2286
+#define regDP1_DP_MSA_TIMING_PARAM1_BASE_IDX                                                            2
+#define regDP1_DP_MSA_TIMING_PARAM2                                                                     0x2287
+#define regDP1_DP_MSA_TIMING_PARAM2_BASE_IDX                                                            2
+#define regDP1_DP_MSA_TIMING_PARAM3                                                                     0x2288
+#define regDP1_DP_MSA_TIMING_PARAM3_BASE_IDX                                                            2
+#define regDP1_DP_MSA_TIMING_PARAM4                                                                     0x2289
+#define regDP1_DP_MSA_TIMING_PARAM4_BASE_IDX                                                            2
+#define regDP1_DP_MSO_CNTL                                                                              0x228a
+#define regDP1_DP_MSO_CNTL_BASE_IDX                                                                     2
+#define regDP1_DP_MSO_CNTL1                                                                             0x228b
+#define regDP1_DP_MSO_CNTL1_BASE_IDX                                                                    2
+#define regDP1_DP_STEER_FIFO_CNTL                                                                       0x228c
+#define regDP1_DP_STEER_FIFO_CNTL_BASE_IDX                                                              2
+#define regDP1_DP_SEC_CNTL2                                                                             0x228d
+#define regDP1_DP_SEC_CNTL2_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_CNTL3                                                                             0x228e
+#define regDP1_DP_SEC_CNTL3_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_CNTL4                                                                             0x228f
+#define regDP1_DP_SEC_CNTL4_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_CNTL5                                                                             0x2290
+#define regDP1_DP_SEC_CNTL5_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_CNTL6                                                                             0x2291
+#define regDP1_DP_SEC_CNTL6_BASE_IDX                                                                    2
+#define regDP1_DP_SEC_CNTL7                                                                             0x2292
+#define regDP1_DP_SEC_CNTL7_BASE_IDX                                                                    2
+#define regDP1_DP_DB_CNTL                                                                               0x2293
+#define regDP1_DP_DB_CNTL_BASE_IDX                                                                      2
+#define regDP1_DP_MSA_VBID_MISC                                                                         0x2294
+#define regDP1_DP_MSA_VBID_MISC_BASE_IDX                                                                2
+#define regDP1_DP_SEC_METADATA_TRANSMISSION                                                             0x2295
+#define regDP1_DP_SEC_METADATA_TRANSMISSION_BASE_IDX                                                    2
+#define regDP1_DP_ALPM_CNTL                                                                             0x2297
+#define regDP1_DP_ALPM_CNTL_BASE_IDX                                                                    2
+#define regDP1_DP_GSP8_CNTL                                                                             0x2298
+#define regDP1_DP_GSP8_CNTL_BASE_IDX                                                                    2
+#define regDP1_DP_GSP9_CNTL                                                                             0x2299
+#define regDP1_DP_GSP9_CNTL_BASE_IDX                                                                    2
+#define regDP1_DP_GSP10_CNTL                                                                            0x229a
+#define regDP1_DP_GSP10_CNTL_BASE_IDX                                                                   2
+#define regDP1_DP_GSP11_CNTL                                                                            0x229b
+#define regDP1_DP_GSP11_CNTL_BASE_IDX                                                                   2
+#define regDP1_DP_GSP_EN_DB_STATUS                                                                      0x229c
+#define regDP1_DP_GSP_EN_DB_STATUS_BASE_IDX                                                             2
+#define regDP1_DP_AUXLESS_ALPM_CNTL1                                                                    0x229d
+#define regDP1_DP_AUXLESS_ALPM_CNTL1_BASE_IDX                                                           2
+#define regDP1_DP_AUXLESS_ALPM_CNTL2                                                                    0x229e
+#define regDP1_DP_AUXLESS_ALPM_CNTL2_BASE_IDX                                                           2
+#define regDP1_DP_AUXLESS_ALPM_CNTL3                                                                    0x229f
+#define regDP1_DP_AUXLESS_ALPM_CNTL3_BASE_IDX                                                           2
+#define regDP1_DP_AUXLESS_ALPM_CNTL4                                                                    0x22a0
+#define regDP1_DP_AUXLESS_ALPM_CNTL4_BASE_IDX                                                           2
+#define regDP1_DP_AUXLESS_ALPM_CNTL5                                                                    0x22a1
+#define regDP1_DP_AUXLESS_ALPM_CNTL5_BASE_IDX                                                           2
+#define regDP1_DP_STREAM_SYMBOL_COUNT_STATUS                                                            0x22a2
+#define regDP1_DP_STREAM_SYMBOL_COUNT_STATUS_BASE_IDX                                                   2
+#define regDP1_DP_STREAM_SYMBOL_COUNT_CONTROL                                                           0x22a3
+#define regDP1_DP_STREAM_SYMBOL_COUNT_CONTROL_BASE_IDX                                                  2
+#define regDP1_DP_LINK_SYMBOL_COUNT_STATUS0                                                             0x22a4
+#define regDP1_DP_LINK_SYMBOL_COUNT_STATUS0_BASE_IDX                                                    2
+#define regDP1_DP_LINK_SYMBOL_COUNT_STATUS1                                                             0x22a5
+#define regDP1_DP_LINK_SYMBOL_COUNT_STATUS1_BASE_IDX                                                    2
+#define regDP1_DP_LINK_SYMBOL_COUNT_CONTROL                                                             0x22a6
+#define regDP1_DP_LINK_SYMBOL_COUNT_CONTROL_BASE_IDX                                                    2
+#define regDP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL                                                     0x22a7
+#define regDP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                            2
+#define regDP1_DP_DPHY_FAST_TRAINING                                                                    0x22aa
+#define regDP1_DP_DPHY_FAST_TRAINING_BASE_IDX                                                           2
+#define regDP1_DP_DPHY_FAST_TRAINING_STATUS                                                             0x22ab
+#define regDP1_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dio_dig2_vpg_vpg_dispdec
+// base address: 0x15dc0
+#define regVPG2_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x22b0
+#define regVPG2_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG2_VPG_GENERIC_PACKET_DATA                                                                 0x22b1
+#define regVPG2_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG2_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x22b2
+#define regVPG2_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x22b3
+#define regVPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG2_VPG_GENERIC_STATUS                                                                      0x22b4
+#define regVPG2_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG2_VPG_MEM_PWR                                                                             0x22b5
+#define regVPG2_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG2_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x22b6
+#define regVPG2_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG2_VPG_ISRC1_2_DATA                                                                        0x22b7
+#define regVPG2_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dio_dig2_apg_apg_dispdec
+// base address: 0x15dec
+#define regAPG2_APG_CONTROL                                                                             0x22bb
+#define regAPG2_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG2_APG_CONTROL2                                                                            0x22bc
+#define regAPG2_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG2_APG_DBG_GEN_CONTROL                                                                     0x22bd
+#define regAPG2_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG2_APG_PACKET_CONTROL                                                                      0x22be
+#define regAPG2_APG_PACKET_CONTROL_BASE_IDX                                                             2
+#define regAPG2_APG_DBG_ACP                                                                             0x22bf
+#define regAPG2_APG_DBG_ACP_BASE_IDX                                                                    2
+#define regAPG2_APG_DBG_AUDIO_INFO                                                                      0x22c1
+#define regAPG2_APG_DBG_AUDIO_INFO_BASE_IDX                                                             2
+#define regAPG2_APG_DBG_60958_0                                                                         0x22c2
+#define regAPG2_APG_DBG_60958_0_BASE_IDX                                                                2
+#define regAPG2_APG_DBG_60958_1                                                                         0x22c3
+#define regAPG2_APG_DBG_60958_1_BASE_IDX                                                                2
+#define regAPG2_APG_DBG_60958_2                                                                         0x22c4
+#define regAPG2_APG_DBG_60958_2_BASE_IDX                                                                2
+#define regAPG2_APG_AUDIO_CRC_CONTROL                                                                   0x22c5
+#define regAPG2_APG_AUDIO_CRC_CONTROL_BASE_IDX                                                          2
+#define regAPG2_APG_AUDIO_CRC_CONTROL2                                                                  0x22c6
+#define regAPG2_APG_AUDIO_CRC_CONTROL2_BASE_IDX                                                         2
+#define regAPG2_APG_AUDIO_CRC_RESULT                                                                    0x22c7
+#define regAPG2_APG_AUDIO_CRC_RESULT_BASE_IDX                                                           2
+#define regAPG2_APG_DBG_RAMP_CONTROL0                                                                   0x22c8
+#define regAPG2_APG_DBG_RAMP_CONTROL0_BASE_IDX                                                          2
+#define regAPG2_APG_DBG_RAMP_CONTROL1                                                                   0x22c9
+#define regAPG2_APG_DBG_RAMP_CONTROL1_BASE_IDX                                                          2
+#define regAPG2_APG_DBG_RAMP_CONTROL2                                                                   0x22ca
+#define regAPG2_APG_DBG_RAMP_CONTROL2_BASE_IDX                                                          2
+#define regAPG2_APG_DBG_RAMP_CONTROL3                                                                   0x22cb
+#define regAPG2_APG_DBG_RAMP_CONTROL3_BASE_IDX                                                          2
+#define regAPG2_APG_STATUS                                                                              0x22cc
+#define regAPG2_APG_STATUS_BASE_IDX                                                                     2
+#define regAPG2_APG_STATUS2                                                                             0x22cd
+#define regAPG2_APG_STATUS2_BASE_IDX                                                                    2
+#define regAPG2_APG_DBG_AUDIO_DTO_CNTL                                                                  0x22ce
+#define regAPG2_APG_DBG_AUDIO_DTO_CNTL_BASE_IDX                                                         2
+#define regAPG2_APG_MEM_PWR                                                                             0x22cf
+#define regAPG2_APG_MEM_PWR_BASE_IDX                                                                    2
+#define regAPG2_APG_SPARE                                                                               0x22d1
+#define regAPG2_APG_SPARE_BASE_IDX                                                                      2
+
+
+// addressBlock: dce_dc_dio_dig2_dme_dme_dispdec
+// base address: 0x15e64
+#define regDME2_DME_CONTROL                                                                             0x22d9
+#define regDME2_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME2_DME_MEMORY_CONTROL                                                                      0x22da
+#define regDME2_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dio_dig2_dispdec
+// base address: 0x920
+#define regDIG2_DIG_FE_CNTL                                                                             0x22db
+#define regDIG2_DIG_FE_CNTL_BASE_IDX                                                                    2
+#define regDIG2_DIG_FE_CLK_CNTL                                                                         0x22dc
+#define regDIG2_DIG_FE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG2_DIG_FE_EN_CNTL                                                                          0x22dd
+#define regDIG2_DIG_FE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG2_DIG_OUTPUT_CRC_CNTL                                                                     0x22de
+#define regDIG2_DIG_OUTPUT_CRC_CNTL_BASE_IDX                                                            2
+#define regDIG2_DIG_OUTPUT_CRC_RESULT                                                                   0x22df
+#define regDIG2_DIG_OUTPUT_CRC_RESULT_BASE_IDX                                                          2
+#define regDIG2_DIG_CLOCK_PATTERN                                                                       0x22e0
+#define regDIG2_DIG_CLOCK_PATTERN_BASE_IDX                                                              2
+#define regDIG2_DIG_TEST_PATTERN                                                                        0x22e1
+#define regDIG2_DIG_TEST_PATTERN_BASE_IDX                                                               2
+#define regDIG2_DIG_RANDOM_PATTERN_SEED                                                                 0x22e2
+#define regDIG2_DIG_RANDOM_PATTERN_SEED_BASE_IDX                                                        2
+#define regDIG2_DIG_FIFO_CTRL0                                                                          0x22e3
+#define regDIG2_DIG_FIFO_CTRL0_BASE_IDX                                                                 2
+#define regDIG2_DIG_FIFO_CTRL1                                                                          0x22e4
+#define regDIG2_DIG_FIFO_CTRL1_BASE_IDX                                                                 2
+#define regDIG2_HDMI_METADATA_PACKET_CONTROL                                                            0x22e5
+#define regDIG2_HDMI_METADATA_PACKET_CONTROL_BASE_IDX                                                   2
+#define regDIG2_HDMI_CONTROL                                                                            0x22e6
+#define regDIG2_HDMI_CONTROL_BASE_IDX                                                                   2
+#define regDIG2_HDMI_STATUS                                                                             0x22e7
+#define regDIG2_HDMI_STATUS_BASE_IDX                                                                    2
+#define regDIG2_HDMI_AUDIO_PACKET_CONTROL                                                               0x22e8
+#define regDIG2_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX                                                      2
+#define regDIG2_HDMI_ACR_PACKET_CONTROL                                                                 0x22e9
+#define regDIG2_HDMI_ACR_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG2_HDMI_VBI_PACKET_CONTROL                                                                 0x22ea
+#define regDIG2_HDMI_VBI_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG2_HDMI_INFOFRAME_CONTROL0                                                                 0x22eb
+#define regDIG2_HDMI_INFOFRAME_CONTROL0_BASE_IDX                                                        2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL0                                                            0x22ec
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL6                                                            0x22ed
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL6_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL5                                                            0x22ee
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX                                                   2
+#define regDIG2_HDMI_GC                                                                                 0x22ef
+#define regDIG2_HDMI_GC_BASE_IDX                                                                        2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL1                                                            0x22f0
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL2                                                            0x22f1
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL3                                                            0x22f2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL4                                                            0x22f3
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL7                                                            0x22f4
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL7_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL8                                                            0x22f5
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL8_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL9                                                            0x22f6
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL9_BASE_IDX                                                   2
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL10                                                           0x22f7
+#define regDIG2_HDMI_GENERIC_PACKET_CONTROL10_BASE_IDX                                                  2
+#define regDIG2_HDMI_DB_CONTROL                                                                         0x22f8
+#define regDIG2_HDMI_DB_CONTROL_BASE_IDX                                                                2
+#define regDIG2_HDMI_ACR_32_0                                                                           0x22f9
+#define regDIG2_HDMI_ACR_32_0_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_32_1                                                                           0x22fa
+#define regDIG2_HDMI_ACR_32_1_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_44_0                                                                           0x22fb
+#define regDIG2_HDMI_ACR_44_0_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_44_1                                                                           0x22fc
+#define regDIG2_HDMI_ACR_44_1_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_48_0                                                                           0x22fd
+#define regDIG2_HDMI_ACR_48_0_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_48_1                                                                           0x22fe
+#define regDIG2_HDMI_ACR_48_1_BASE_IDX                                                                  2
+#define regDIG2_HDMI_ACR_STATUS_0                                                                       0x22ff
+#define regDIG2_HDMI_ACR_STATUS_0_BASE_IDX                                                              2
+#define regDIG2_HDMI_ACR_STATUS_1                                                                       0x2300
+#define regDIG2_HDMI_ACR_STATUS_1_BASE_IDX                                                              2
+#define regDIG2_DIG_FE_AUDIO_CNTL                                                                       0x2301
+#define regDIG2_DIG_FE_AUDIO_CNTL_BASE_IDX                                                              2
+#define regDIG2_DIG_BE_CLK_CNTL                                                                         0x2302
+#define regDIG2_DIG_BE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG2_DIG_BE_CNTL                                                                             0x2303
+#define regDIG2_DIG_BE_CNTL_BASE_IDX                                                                    2
+#define regDIG2_DIG_BE_EN_CNTL                                                                          0x2304
+#define regDIG2_DIG_BE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG2_TMDS_CNTL                                                                               0x232b
+#define regDIG2_TMDS_CNTL_BASE_IDX                                                                      2
+#define regDIG2_TMDS_CONTROL_CHAR                                                                       0x232c
+#define regDIG2_TMDS_CONTROL_CHAR_BASE_IDX                                                              2
+#define regDIG2_TMDS_CONTROL0_FEEDBACK                                                                  0x232d
+#define regDIG2_TMDS_CONTROL0_FEEDBACK_BASE_IDX                                                         2
+#define regDIG2_TMDS_STEREOSYNC_CTL_SEL                                                                 0x232e
+#define regDIG2_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX                                                        2
+#define regDIG2_TMDS_SYNC_CHAR_PATTERN_0_1                                                              0x232f
+#define regDIG2_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX                                                     2
+#define regDIG2_TMDS_SYNC_CHAR_PATTERN_2_3                                                              0x2330
+#define regDIG2_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX                                                     2
+#define regDIG2_TMDS_CTL_BITS                                                                           0x2332
+#define regDIG2_TMDS_CTL_BITS_BASE_IDX                                                                  2
+#define regDIG2_TMDS_DCBALANCER_CONTROL                                                                 0x2333
+#define regDIG2_TMDS_DCBALANCER_CONTROL_BASE_IDX                                                        2
+#define regDIG2_TMDS_SYNC_DCBALANCE_CHAR                                                                0x2334
+#define regDIG2_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX                                                       2
+#define regDIG2_TMDS_CTL0_1_GEN_CNTL                                                                    0x2335
+#define regDIG2_TMDS_CTL0_1_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG2_TMDS_CTL2_3_GEN_CNTL                                                                    0x2336
+#define regDIG2_TMDS_CTL2_3_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG2_DIG_VERSION                                                                             0x2338
+#define regDIG2_DIG_VERSION_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_dio_dp2_dispdec
+// base address: 0x920
+#define regDP2_DP_LINK_CNTL                                                                             0x2366
+#define regDP2_DP_LINK_CNTL_BASE_IDX                                                                    2
+#define regDP2_DP_PIXEL_FORMAT                                                                          0x2367
+#define regDP2_DP_PIXEL_FORMAT_BASE_IDX                                                                 2
+#define regDP2_DP_MSA_COLORIMETRY                                                                       0x2368
+#define regDP2_DP_MSA_COLORIMETRY_BASE_IDX                                                              2
+#define regDP2_DP_CONFIG                                                                                0x2369
+#define regDP2_DP_CONFIG_BASE_IDX                                                                       2
+#define regDP2_DP_VID_STREAM_CNTL                                                                       0x236a
+#define regDP2_DP_VID_STREAM_CNTL_BASE_IDX                                                              2
+#define regDP2_DP_STEER_FIFO                                                                            0x236b
+#define regDP2_DP_STEER_FIFO_BASE_IDX                                                                   2
+#define regDP2_DP_MSA_MISC                                                                              0x236c
+#define regDP2_DP_MSA_MISC_BASE_IDX                                                                     2
+#define regDP2_DP_DPHY_INTERNAL_CTRL                                                                    0x236d
+#define regDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX                                                           2
+#define regDP2_DP_VID_TIMING                                                                            0x236e
+#define regDP2_DP_VID_TIMING_BASE_IDX                                                                   2
+#define regDP2_DP_VID_N                                                                                 0x236f
+#define regDP2_DP_VID_N_BASE_IDX                                                                        2
+#define regDP2_DP_VID_M                                                                                 0x2370
+#define regDP2_DP_VID_M_BASE_IDX                                                                        2
+#define regDP2_DP_LINK_FRAMING_CNTL                                                                     0x2371
+#define regDP2_DP_LINK_FRAMING_CNTL_BASE_IDX                                                            2
+#define regDP2_DP_HBR2_EYE_PATTERN                                                                      0x2372
+#define regDP2_DP_HBR2_EYE_PATTERN_BASE_IDX                                                             2
+#define regDP2_DP_VID_MSA_VBID                                                                          0x2373
+#define regDP2_DP_VID_MSA_VBID_BASE_IDX                                                                 2
+#define regDP2_DP_VID_INTERRUPT_CNTL                                                                    0x2374
+#define regDP2_DP_VID_INTERRUPT_CNTL_BASE_IDX                                                           2
+#define regDP2_DP_DPHY_CNTL                                                                             0x2375
+#define regDP2_DP_DPHY_CNTL_BASE_IDX                                                                    2
+#define regDP2_DP_DPHY_TRAINING_PATTERN_SEL                                                             0x2376
+#define regDP2_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX                                                    2
+#define regDP2_DP_DPHY_SYM0                                                                             0x2377
+#define regDP2_DP_DPHY_SYM0_BASE_IDX                                                                    2
+#define regDP2_DP_DPHY_SYM1                                                                             0x2378
+#define regDP2_DP_DPHY_SYM1_BASE_IDX                                                                    2
+#define regDP2_DP_DPHY_SYM2                                                                             0x2379
+#define regDP2_DP_DPHY_SYM2_BASE_IDX                                                                    2
+#define regDP2_DP_DPHY_8B10B_CNTL                                                                       0x237a
+#define regDP2_DP_DPHY_8B10B_CNTL_BASE_IDX                                                              2
+#define regDP2_DP_DPHY_PRBS_CNTL                                                                        0x237b
+#define regDP2_DP_DPHY_PRBS_CNTL_BASE_IDX                                                               2
+#define regDP2_DP_DPHY_SCRAM_CNTL                                                                       0x237c
+#define regDP2_DP_DPHY_SCRAM_CNTL_BASE_IDX                                                              2
+#define regDP2_DP_DPHY_CRC_CONTROL0                                                                     0x237d
+#define regDP2_DP_DPHY_CRC_CONTROL0_BASE_IDX                                                            2
+#define regDP2_DP_DPHY_CRC_CONTROL1                                                                     0x237e
+#define regDP2_DP_DPHY_CRC_CONTROL1_BASE_IDX                                                            2
+#define regDP2_DP_DPHY_CRC_RESULT0                                                                      0x237f
+#define regDP2_DP_DPHY_CRC_RESULT0_BASE_IDX                                                             2
+#define regDP2_DP_DPHY_CRC_RESULT1                                                                      0x2380
+#define regDP2_DP_DPHY_CRC_RESULT1_BASE_IDX                                                             2
+#define regDP2_DP_DPHY_CRC_RESULT2                                                                      0x2381
+#define regDP2_DP_DPHY_CRC_RESULT2_BASE_IDX                                                             2
+#define regDP2_DP_DPHY_CRC_RESULT3                                                                      0x2382
+#define regDP2_DP_DPHY_CRC_RESULT3_BASE_IDX                                                             2
+#define regDP2_DP_DPHY_CRC_STATUS                                                                       0x2383
+#define regDP2_DP_DPHY_CRC_STATUS_BASE_IDX                                                              2
+#define regDP2_DP_TU_CNTL                                                                               0x2384
+#define regDP2_DP_TU_CNTL_BASE_IDX                                                                      2
+#define regDP2_DP_PIXEL_FORMAT_DB_CNTL                                                                  0x2385
+#define regDP2_DP_PIXEL_FORMAT_DB_CNTL_BASE_IDX                                                         2
+#define regDP2_DP_SEC_CNTL                                                                              0x2389
+#define regDP2_DP_SEC_CNTL_BASE_IDX                                                                     2
+#define regDP2_DP_SEC_CNTL1                                                                             0x238a
+#define regDP2_DP_SEC_CNTL1_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_FRAMING1                                                                          0x238b
+#define regDP2_DP_SEC_FRAMING1_BASE_IDX                                                                 2
+#define regDP2_DP_SEC_FRAMING2                                                                          0x238c
+#define regDP2_DP_SEC_FRAMING2_BASE_IDX                                                                 2
+#define regDP2_DP_SEC_FRAMING3                                                                          0x238d
+#define regDP2_DP_SEC_FRAMING3_BASE_IDX                                                                 2
+#define regDP2_DP_SEC_FRAMING4                                                                          0x238e
+#define regDP2_DP_SEC_FRAMING4_BASE_IDX                                                                 2
+#define regDP2_DP_SEC_AUD_N                                                                             0x238f
+#define regDP2_DP_SEC_AUD_N_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_AUD_N_READBACK                                                                    0x2390
+#define regDP2_DP_SEC_AUD_N_READBACK_BASE_IDX                                                           2
+#define regDP2_DP_SEC_AUD_M                                                                             0x2391
+#define regDP2_DP_SEC_AUD_M_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_AUD_M_READBACK                                                                    0x2392
+#define regDP2_DP_SEC_AUD_M_READBACK_BASE_IDX                                                           2
+#define regDP2_DP_SEC_TIMESTAMP                                                                         0x2393
+#define regDP2_DP_SEC_TIMESTAMP_BASE_IDX                                                                2
+#define regDP2_DP_SEC_PACKET_CNTL                                                                       0x2394
+#define regDP2_DP_SEC_PACKET_CNTL_BASE_IDX                                                              2
+#define regDP2_DP_MSE_RATE_CNTL                                                                         0x2395
+#define regDP2_DP_MSE_RATE_CNTL_BASE_IDX                                                                2
+#define regDP2_DP_MSE_RATE_UPDATE                                                                       0x2397
+#define regDP2_DP_MSE_RATE_UPDATE_BASE_IDX                                                              2
+#define regDP2_DP_MSE_SAT0                                                                              0x2398
+#define regDP2_DP_MSE_SAT0_BASE_IDX                                                                     2
+#define regDP2_DP_MSE_SAT1                                                                              0x2399
+#define regDP2_DP_MSE_SAT1_BASE_IDX                                                                     2
+#define regDP2_DP_MSE_SAT2                                                                              0x239a
+#define regDP2_DP_MSE_SAT2_BASE_IDX                                                                     2
+#define regDP2_DP_MSE_SAT_UPDATE                                                                        0x239b
+#define regDP2_DP_MSE_SAT_UPDATE_BASE_IDX                                                               2
+#define regDP2_DP_MSE_LINK_TIMING                                                                       0x239c
+#define regDP2_DP_MSE_LINK_TIMING_BASE_IDX                                                              2
+#define regDP2_DP_MSE_MISC_CNTL                                                                         0x239d
+#define regDP2_DP_MSE_MISC_CNTL_BASE_IDX                                                                2
+#define regDP2_DP_DPHY_BS_SR_SWAP_CNTL                                                                  0x23a2
+#define regDP2_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX                                                         2
+#define regDP2_DP_DPHY_HBR2_PATTERN_CONTROL                                                             0x23a3
+#define regDP2_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX                                                    2
+#define regDP2_DP_MSE_SAT0_STATUS                                                                       0x23a5
+#define regDP2_DP_MSE_SAT0_STATUS_BASE_IDX                                                              2
+#define regDP2_DP_MSE_SAT1_STATUS                                                                       0x23a6
+#define regDP2_DP_MSE_SAT1_STATUS_BASE_IDX                                                              2
+#define regDP2_DP_MSE_SAT2_STATUS                                                                       0x23a7
+#define regDP2_DP_MSE_SAT2_STATUS_BASE_IDX                                                              2
+#define regDP2_DP_HBLANK_CONTROL                                                                        0x23a9
+#define regDP2_DP_HBLANK_CONTROL_BASE_IDX                                                               2
+#define regDP2_DP_MSA_TIMING_PARAM1                                                                     0x23aa
+#define regDP2_DP_MSA_TIMING_PARAM1_BASE_IDX                                                            2
+#define regDP2_DP_MSA_TIMING_PARAM2                                                                     0x23ab
+#define regDP2_DP_MSA_TIMING_PARAM2_BASE_IDX                                                            2
+#define regDP2_DP_MSA_TIMING_PARAM3                                                                     0x23ac
+#define regDP2_DP_MSA_TIMING_PARAM3_BASE_IDX                                                            2
+#define regDP2_DP_MSA_TIMING_PARAM4                                                                     0x23ad
+#define regDP2_DP_MSA_TIMING_PARAM4_BASE_IDX                                                            2
+#define regDP2_DP_MSO_CNTL                                                                              0x23ae
+#define regDP2_DP_MSO_CNTL_BASE_IDX                                                                     2
+#define regDP2_DP_MSO_CNTL1                                                                             0x23af
+#define regDP2_DP_MSO_CNTL1_BASE_IDX                                                                    2
+#define regDP2_DP_STEER_FIFO_CNTL                                                                       0x23b0
+#define regDP2_DP_STEER_FIFO_CNTL_BASE_IDX                                                              2
+#define regDP2_DP_SEC_CNTL2                                                                             0x23b1
+#define regDP2_DP_SEC_CNTL2_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_CNTL3                                                                             0x23b2
+#define regDP2_DP_SEC_CNTL3_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_CNTL4                                                                             0x23b3
+#define regDP2_DP_SEC_CNTL4_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_CNTL5                                                                             0x23b4
+#define regDP2_DP_SEC_CNTL5_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_CNTL6                                                                             0x23b5
+#define regDP2_DP_SEC_CNTL6_BASE_IDX                                                                    2
+#define regDP2_DP_SEC_CNTL7                                                                             0x23b6
+#define regDP2_DP_SEC_CNTL7_BASE_IDX                                                                    2
+#define regDP2_DP_DB_CNTL                                                                               0x23b7
+#define regDP2_DP_DB_CNTL_BASE_IDX                                                                      2
+#define regDP2_DP_MSA_VBID_MISC                                                                         0x23b8
+#define regDP2_DP_MSA_VBID_MISC_BASE_IDX                                                                2
+#define regDP2_DP_SEC_METADATA_TRANSMISSION                                                             0x23b9
+#define regDP2_DP_SEC_METADATA_TRANSMISSION_BASE_IDX                                                    2
+#define regDP2_DP_ALPM_CNTL                                                                             0x23bb
+#define regDP2_DP_ALPM_CNTL_BASE_IDX                                                                    2
+#define regDP2_DP_GSP8_CNTL                                                                             0x23bc
+#define regDP2_DP_GSP8_CNTL_BASE_IDX                                                                    2
+#define regDP2_DP_GSP9_CNTL                                                                             0x23bd
+#define regDP2_DP_GSP9_CNTL_BASE_IDX                                                                    2
+#define regDP2_DP_GSP10_CNTL                                                                            0x23be
+#define regDP2_DP_GSP10_CNTL_BASE_IDX                                                                   2
+#define regDP2_DP_GSP11_CNTL                                                                            0x23bf
+#define regDP2_DP_GSP11_CNTL_BASE_IDX                                                                   2
+#define regDP2_DP_GSP_EN_DB_STATUS                                                                      0x23c0
+#define regDP2_DP_GSP_EN_DB_STATUS_BASE_IDX                                                             2
+#define regDP2_DP_AUXLESS_ALPM_CNTL1                                                                    0x23c1
+#define regDP2_DP_AUXLESS_ALPM_CNTL1_BASE_IDX                                                           2
+#define regDP2_DP_AUXLESS_ALPM_CNTL2                                                                    0x23c2
+#define regDP2_DP_AUXLESS_ALPM_CNTL2_BASE_IDX                                                           2
+#define regDP2_DP_AUXLESS_ALPM_CNTL3                                                                    0x23c3
+#define regDP2_DP_AUXLESS_ALPM_CNTL3_BASE_IDX                                                           2
+#define regDP2_DP_AUXLESS_ALPM_CNTL4                                                                    0x23c4
+#define regDP2_DP_AUXLESS_ALPM_CNTL4_BASE_IDX                                                           2
+#define regDP2_DP_AUXLESS_ALPM_CNTL5                                                                    0x23c5
+#define regDP2_DP_AUXLESS_ALPM_CNTL5_BASE_IDX                                                           2
+#define regDP2_DP_STREAM_SYMBOL_COUNT_STATUS                                                            0x23c6
+#define regDP2_DP_STREAM_SYMBOL_COUNT_STATUS_BASE_IDX                                                   2
+#define regDP2_DP_STREAM_SYMBOL_COUNT_CONTROL                                                           0x23c7
+#define regDP2_DP_STREAM_SYMBOL_COUNT_CONTROL_BASE_IDX                                                  2
+#define regDP2_DP_LINK_SYMBOL_COUNT_STATUS0                                                             0x23c8
+#define regDP2_DP_LINK_SYMBOL_COUNT_STATUS0_BASE_IDX                                                    2
+#define regDP2_DP_LINK_SYMBOL_COUNT_STATUS1                                                             0x23c9
+#define regDP2_DP_LINK_SYMBOL_COUNT_STATUS1_BASE_IDX                                                    2
+#define regDP2_DP_LINK_SYMBOL_COUNT_CONTROL                                                             0x23ca
+#define regDP2_DP_LINK_SYMBOL_COUNT_CONTROL_BASE_IDX                                                    2
+#define regDP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL                                                     0x23cb
+#define regDP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                            2
+#define regDP2_DP_DPHY_FAST_TRAINING                                                                    0x23ce
+#define regDP2_DP_DPHY_FAST_TRAINING_BASE_IDX                                                           2
+#define regDP2_DP_DPHY_FAST_TRAINING_STATUS                                                             0x23cf
+#define regDP2_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dio_dig3_vpg_vpg_dispdec
+// base address: 0x16250
+#define regVPG3_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x23d4
+#define regVPG3_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG3_VPG_GENERIC_PACKET_DATA                                                                 0x23d5
+#define regVPG3_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG3_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x23d6
+#define regVPG3_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x23d7
+#define regVPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG3_VPG_GENERIC_STATUS                                                                      0x23d8
+#define regVPG3_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG3_VPG_MEM_PWR                                                                             0x23d9
+#define regVPG3_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG3_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x23da
+#define regVPG3_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG3_VPG_ISRC1_2_DATA                                                                        0x23db
+#define regVPG3_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dio_dig3_apg_apg_dispdec
+// base address: 0x1627c
+#define regAPG3_APG_CONTROL                                                                             0x23df
+#define regAPG3_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG3_APG_CONTROL2                                                                            0x23e0
+#define regAPG3_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG3_APG_DBG_GEN_CONTROL                                                                     0x23e1
+#define regAPG3_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG3_APG_PACKET_CONTROL                                                                      0x23e2
+#define regAPG3_APG_PACKET_CONTROL_BASE_IDX                                                             2
+#define regAPG3_APG_DBG_ACP                                                                             0x23e3
+#define regAPG3_APG_DBG_ACP_BASE_IDX                                                                    2
+#define regAPG3_APG_DBG_AUDIO_INFO                                                                      0x23e5
+#define regAPG3_APG_DBG_AUDIO_INFO_BASE_IDX                                                             2
+#define regAPG3_APG_DBG_60958_0                                                                         0x23e6
+#define regAPG3_APG_DBG_60958_0_BASE_IDX                                                                2
+#define regAPG3_APG_DBG_60958_1                                                                         0x23e7
+#define regAPG3_APG_DBG_60958_1_BASE_IDX                                                                2
+#define regAPG3_APG_DBG_60958_2                                                                         0x23e8
+#define regAPG3_APG_DBG_60958_2_BASE_IDX                                                                2
+#define regAPG3_APG_AUDIO_CRC_CONTROL                                                                   0x23e9
+#define regAPG3_APG_AUDIO_CRC_CONTROL_BASE_IDX                                                          2
+#define regAPG3_APG_AUDIO_CRC_CONTROL2                                                                  0x23ea
+#define regAPG3_APG_AUDIO_CRC_CONTROL2_BASE_IDX                                                         2
+#define regAPG3_APG_AUDIO_CRC_RESULT                                                                    0x23eb
+#define regAPG3_APG_AUDIO_CRC_RESULT_BASE_IDX                                                           2
+#define regAPG3_APG_DBG_RAMP_CONTROL0                                                                   0x23ec
+#define regAPG3_APG_DBG_RAMP_CONTROL0_BASE_IDX                                                          2
+#define regAPG3_APG_DBG_RAMP_CONTROL1                                                                   0x23ed
+#define regAPG3_APG_DBG_RAMP_CONTROL1_BASE_IDX                                                          2
+#define regAPG3_APG_DBG_RAMP_CONTROL2                                                                   0x23ee
+#define regAPG3_APG_DBG_RAMP_CONTROL2_BASE_IDX                                                          2
+#define regAPG3_APG_DBG_RAMP_CONTROL3                                                                   0x23ef
+#define regAPG3_APG_DBG_RAMP_CONTROL3_BASE_IDX                                                          2
+#define regAPG3_APG_STATUS                                                                              0x23f0
+#define regAPG3_APG_STATUS_BASE_IDX                                                                     2
+#define regAPG3_APG_STATUS2                                                                             0x23f1
+#define regAPG3_APG_STATUS2_BASE_IDX                                                                    2
+#define regAPG3_APG_DBG_AUDIO_DTO_CNTL                                                                  0x23f2
+#define regAPG3_APG_DBG_AUDIO_DTO_CNTL_BASE_IDX                                                         2
+#define regAPG3_APG_MEM_PWR                                                                             0x23f3
+#define regAPG3_APG_MEM_PWR_BASE_IDX                                                                    2
+#define regAPG3_APG_SPARE                                                                               0x23f5
+#define regAPG3_APG_SPARE_BASE_IDX                                                                      2
+
+
+// addressBlock: dce_dc_dio_dig3_dme_dme_dispdec
+// base address: 0x162f4
+#define regDME3_DME_CONTROL                                                                             0x23fd
+#define regDME3_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME3_DME_MEMORY_CONTROL                                                                      0x23fe
+#define regDME3_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dio_dig3_dispdec
+// base address: 0xdb0
+#define regDIG3_DIG_FE_CNTL                                                                             0x23ff
+#define regDIG3_DIG_FE_CNTL_BASE_IDX                                                                    2
+#define regDIG3_DIG_FE_CLK_CNTL                                                                         0x2400
+#define regDIG3_DIG_FE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG3_DIG_FE_EN_CNTL                                                                          0x2401
+#define regDIG3_DIG_FE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG3_DIG_OUTPUT_CRC_CNTL                                                                     0x2402
+#define regDIG3_DIG_OUTPUT_CRC_CNTL_BASE_IDX                                                            2
+#define regDIG3_DIG_OUTPUT_CRC_RESULT                                                                   0x2403
+#define regDIG3_DIG_OUTPUT_CRC_RESULT_BASE_IDX                                                          2
+#define regDIG3_DIG_CLOCK_PATTERN                                                                       0x2404
+#define regDIG3_DIG_CLOCK_PATTERN_BASE_IDX                                                              2
+#define regDIG3_DIG_TEST_PATTERN                                                                        0x2405
+#define regDIG3_DIG_TEST_PATTERN_BASE_IDX                                                               2
+#define regDIG3_DIG_RANDOM_PATTERN_SEED                                                                 0x2406
+#define regDIG3_DIG_RANDOM_PATTERN_SEED_BASE_IDX                                                        2
+#define regDIG3_DIG_FIFO_CTRL0                                                                          0x2407
+#define regDIG3_DIG_FIFO_CTRL0_BASE_IDX                                                                 2
+#define regDIG3_DIG_FIFO_CTRL1                                                                          0x2408
+#define regDIG3_DIG_FIFO_CTRL1_BASE_IDX                                                                 2
+#define regDIG3_HDMI_METADATA_PACKET_CONTROL                                                            0x2409
+#define regDIG3_HDMI_METADATA_PACKET_CONTROL_BASE_IDX                                                   2
+#define regDIG3_HDMI_CONTROL                                                                            0x240a
+#define regDIG3_HDMI_CONTROL_BASE_IDX                                                                   2
+#define regDIG3_HDMI_STATUS                                                                             0x240b
+#define regDIG3_HDMI_STATUS_BASE_IDX                                                                    2
+#define regDIG3_HDMI_AUDIO_PACKET_CONTROL                                                               0x240c
+#define regDIG3_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX                                                      2
+#define regDIG3_HDMI_ACR_PACKET_CONTROL                                                                 0x240d
+#define regDIG3_HDMI_ACR_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG3_HDMI_VBI_PACKET_CONTROL                                                                 0x240e
+#define regDIG3_HDMI_VBI_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG3_HDMI_INFOFRAME_CONTROL0                                                                 0x240f
+#define regDIG3_HDMI_INFOFRAME_CONTROL0_BASE_IDX                                                        2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL0                                                            0x2410
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL6                                                            0x2411
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL6_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL5                                                            0x2412
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX                                                   2
+#define regDIG3_HDMI_GC                                                                                 0x2413
+#define regDIG3_HDMI_GC_BASE_IDX                                                                        2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL1                                                            0x2414
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL2                                                            0x2415
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL3                                                            0x2416
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL4                                                            0x2417
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL7                                                            0x2418
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL7_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL8                                                            0x2419
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL8_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL9                                                            0x241a
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL9_BASE_IDX                                                   2
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL10                                                           0x241b
+#define regDIG3_HDMI_GENERIC_PACKET_CONTROL10_BASE_IDX                                                  2
+#define regDIG3_HDMI_DB_CONTROL                                                                         0x241c
+#define regDIG3_HDMI_DB_CONTROL_BASE_IDX                                                                2
+#define regDIG3_HDMI_ACR_32_0                                                                           0x241d
+#define regDIG3_HDMI_ACR_32_0_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_32_1                                                                           0x241e
+#define regDIG3_HDMI_ACR_32_1_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_44_0                                                                           0x241f
+#define regDIG3_HDMI_ACR_44_0_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_44_1                                                                           0x2420
+#define regDIG3_HDMI_ACR_44_1_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_48_0                                                                           0x2421
+#define regDIG3_HDMI_ACR_48_0_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_48_1                                                                           0x2422
+#define regDIG3_HDMI_ACR_48_1_BASE_IDX                                                                  2
+#define regDIG3_HDMI_ACR_STATUS_0                                                                       0x2423
+#define regDIG3_HDMI_ACR_STATUS_0_BASE_IDX                                                              2
+#define regDIG3_HDMI_ACR_STATUS_1                                                                       0x2424
+#define regDIG3_HDMI_ACR_STATUS_1_BASE_IDX                                                              2
+#define regDIG3_DIG_FE_AUDIO_CNTL                                                                       0x2425
+#define regDIG3_DIG_FE_AUDIO_CNTL_BASE_IDX                                                              2
+#define regDIG3_DIG_BE_CLK_CNTL                                                                         0x2426
+#define regDIG3_DIG_BE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG3_DIG_BE_CNTL                                                                             0x2427
+#define regDIG3_DIG_BE_CNTL_BASE_IDX                                                                    2
+#define regDIG3_DIG_BE_EN_CNTL                                                                          0x2428
+#define regDIG3_DIG_BE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG3_TMDS_CNTL                                                                               0x244f
+#define regDIG3_TMDS_CNTL_BASE_IDX                                                                      2
+#define regDIG3_TMDS_CONTROL_CHAR                                                                       0x2450
+#define regDIG3_TMDS_CONTROL_CHAR_BASE_IDX                                                              2
+#define regDIG3_TMDS_CONTROL0_FEEDBACK                                                                  0x2451
+#define regDIG3_TMDS_CONTROL0_FEEDBACK_BASE_IDX                                                         2
+#define regDIG3_TMDS_STEREOSYNC_CTL_SEL                                                                 0x2452
+#define regDIG3_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX                                                        2
+#define regDIG3_TMDS_SYNC_CHAR_PATTERN_0_1                                                              0x2453
+#define regDIG3_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX                                                     2
+#define regDIG3_TMDS_SYNC_CHAR_PATTERN_2_3                                                              0x2454
+#define regDIG3_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX                                                     2
+#define regDIG3_TMDS_CTL_BITS                                                                           0x2456
+#define regDIG3_TMDS_CTL_BITS_BASE_IDX                                                                  2
+#define regDIG3_TMDS_DCBALANCER_CONTROL                                                                 0x2457
+#define regDIG3_TMDS_DCBALANCER_CONTROL_BASE_IDX                                                        2
+#define regDIG3_TMDS_SYNC_DCBALANCE_CHAR                                                                0x2458
+#define regDIG3_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX                                                       2
+#define regDIG3_TMDS_CTL0_1_GEN_CNTL                                                                    0x2459
+#define regDIG3_TMDS_CTL0_1_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG3_TMDS_CTL2_3_GEN_CNTL                                                                    0x245a
+#define regDIG3_TMDS_CTL2_3_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG3_DIG_VERSION                                                                             0x245c
+#define regDIG3_DIG_VERSION_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_dio_dp3_dispdec
+// base address: 0xdb0
+#define regDP3_DP_LINK_CNTL                                                                             0x248a
+#define regDP3_DP_LINK_CNTL_BASE_IDX                                                                    2
+#define regDP3_DP_PIXEL_FORMAT                                                                          0x248b
+#define regDP3_DP_PIXEL_FORMAT_BASE_IDX                                                                 2
+#define regDP3_DP_MSA_COLORIMETRY                                                                       0x248c
+#define regDP3_DP_MSA_COLORIMETRY_BASE_IDX                                                              2
+#define regDP3_DP_CONFIG                                                                                0x248d
+#define regDP3_DP_CONFIG_BASE_IDX                                                                       2
+#define regDP3_DP_VID_STREAM_CNTL                                                                       0x248e
+#define regDP3_DP_VID_STREAM_CNTL_BASE_IDX                                                              2
+#define regDP3_DP_STEER_FIFO                                                                            0x248f
+#define regDP3_DP_STEER_FIFO_BASE_IDX                                                                   2
+#define regDP3_DP_MSA_MISC                                                                              0x2490
+#define regDP3_DP_MSA_MISC_BASE_IDX                                                                     2
+#define regDP3_DP_DPHY_INTERNAL_CTRL                                                                    0x2491
+#define regDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX                                                           2
+#define regDP3_DP_VID_TIMING                                                                            0x2492
+#define regDP3_DP_VID_TIMING_BASE_IDX                                                                   2
+#define regDP3_DP_VID_N                                                                                 0x2493
+#define regDP3_DP_VID_N_BASE_IDX                                                                        2
+#define regDP3_DP_VID_M                                                                                 0x2494
+#define regDP3_DP_VID_M_BASE_IDX                                                                        2
+#define regDP3_DP_LINK_FRAMING_CNTL                                                                     0x2495
+#define regDP3_DP_LINK_FRAMING_CNTL_BASE_IDX                                                            2
+#define regDP3_DP_HBR2_EYE_PATTERN                                                                      0x2496
+#define regDP3_DP_HBR2_EYE_PATTERN_BASE_IDX                                                             2
+#define regDP3_DP_VID_MSA_VBID                                                                          0x2497
+#define regDP3_DP_VID_MSA_VBID_BASE_IDX                                                                 2
+#define regDP3_DP_VID_INTERRUPT_CNTL                                                                    0x2498
+#define regDP3_DP_VID_INTERRUPT_CNTL_BASE_IDX                                                           2
+#define regDP3_DP_DPHY_CNTL                                                                             0x2499
+#define regDP3_DP_DPHY_CNTL_BASE_IDX                                                                    2
+#define regDP3_DP_DPHY_TRAINING_PATTERN_SEL                                                             0x249a
+#define regDP3_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX                                                    2
+#define regDP3_DP_DPHY_SYM0                                                                             0x249b
+#define regDP3_DP_DPHY_SYM0_BASE_IDX                                                                    2
+#define regDP3_DP_DPHY_SYM1                                                                             0x249c
+#define regDP3_DP_DPHY_SYM1_BASE_IDX                                                                    2
+#define regDP3_DP_DPHY_SYM2                                                                             0x249d
+#define regDP3_DP_DPHY_SYM2_BASE_IDX                                                                    2
+#define regDP3_DP_DPHY_8B10B_CNTL                                                                       0x249e
+#define regDP3_DP_DPHY_8B10B_CNTL_BASE_IDX                                                              2
+#define regDP3_DP_DPHY_PRBS_CNTL                                                                        0x249f
+#define regDP3_DP_DPHY_PRBS_CNTL_BASE_IDX                                                               2
+#define regDP3_DP_DPHY_SCRAM_CNTL                                                                       0x24a0
+#define regDP3_DP_DPHY_SCRAM_CNTL_BASE_IDX                                                              2
+#define regDP3_DP_DPHY_CRC_CONTROL0                                                                     0x24a1
+#define regDP3_DP_DPHY_CRC_CONTROL0_BASE_IDX                                                            2
+#define regDP3_DP_DPHY_CRC_CONTROL1                                                                     0x24a2
+#define regDP3_DP_DPHY_CRC_CONTROL1_BASE_IDX                                                            2
+#define regDP3_DP_DPHY_CRC_RESULT0                                                                      0x24a3
+#define regDP3_DP_DPHY_CRC_RESULT0_BASE_IDX                                                             2
+#define regDP3_DP_DPHY_CRC_RESULT1                                                                      0x24a4
+#define regDP3_DP_DPHY_CRC_RESULT1_BASE_IDX                                                             2
+#define regDP3_DP_DPHY_CRC_RESULT2                                                                      0x24a5
+#define regDP3_DP_DPHY_CRC_RESULT2_BASE_IDX                                                             2
+#define regDP3_DP_DPHY_CRC_RESULT3                                                                      0x24a6
+#define regDP3_DP_DPHY_CRC_RESULT3_BASE_IDX                                                             2
+#define regDP3_DP_DPHY_CRC_STATUS                                                                       0x24a7
+#define regDP3_DP_DPHY_CRC_STATUS_BASE_IDX                                                              2
+#define regDP3_DP_TU_CNTL                                                                               0x24a8
+#define regDP3_DP_TU_CNTL_BASE_IDX                                                                      2
+#define regDP3_DP_PIXEL_FORMAT_DB_CNTL                                                                  0x24a9
+#define regDP3_DP_PIXEL_FORMAT_DB_CNTL_BASE_IDX                                                         2
+#define regDP3_DP_SEC_CNTL                                                                              0x24ad
+#define regDP3_DP_SEC_CNTL_BASE_IDX                                                                     2
+#define regDP3_DP_SEC_CNTL1                                                                             0x24ae
+#define regDP3_DP_SEC_CNTL1_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_FRAMING1                                                                          0x24af
+#define regDP3_DP_SEC_FRAMING1_BASE_IDX                                                                 2
+#define regDP3_DP_SEC_FRAMING2                                                                          0x24b0
+#define regDP3_DP_SEC_FRAMING2_BASE_IDX                                                                 2
+#define regDP3_DP_SEC_FRAMING3                                                                          0x24b1
+#define regDP3_DP_SEC_FRAMING3_BASE_IDX                                                                 2
+#define regDP3_DP_SEC_FRAMING4                                                                          0x24b2
+#define regDP3_DP_SEC_FRAMING4_BASE_IDX                                                                 2
+#define regDP3_DP_SEC_AUD_N                                                                             0x24b3
+#define regDP3_DP_SEC_AUD_N_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_AUD_N_READBACK                                                                    0x24b4
+#define regDP3_DP_SEC_AUD_N_READBACK_BASE_IDX                                                           2
+#define regDP3_DP_SEC_AUD_M                                                                             0x24b5
+#define regDP3_DP_SEC_AUD_M_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_AUD_M_READBACK                                                                    0x24b6
+#define regDP3_DP_SEC_AUD_M_READBACK_BASE_IDX                                                           2
+#define regDP3_DP_SEC_TIMESTAMP                                                                         0x24b7
+#define regDP3_DP_SEC_TIMESTAMP_BASE_IDX                                                                2
+#define regDP3_DP_SEC_PACKET_CNTL                                                                       0x24b8
+#define regDP3_DP_SEC_PACKET_CNTL_BASE_IDX                                                              2
+#define regDP3_DP_MSE_RATE_CNTL                                                                         0x24b9
+#define regDP3_DP_MSE_RATE_CNTL_BASE_IDX                                                                2
+#define regDP3_DP_MSE_RATE_UPDATE                                                                       0x24bb
+#define regDP3_DP_MSE_RATE_UPDATE_BASE_IDX                                                              2
+#define regDP3_DP_MSE_SAT0                                                                              0x24bc
+#define regDP3_DP_MSE_SAT0_BASE_IDX                                                                     2
+#define regDP3_DP_MSE_SAT1                                                                              0x24bd
+#define regDP3_DP_MSE_SAT1_BASE_IDX                                                                     2
+#define regDP3_DP_MSE_SAT2                                                                              0x24be
+#define regDP3_DP_MSE_SAT2_BASE_IDX                                                                     2
+#define regDP3_DP_MSE_SAT_UPDATE                                                                        0x24bf
+#define regDP3_DP_MSE_SAT_UPDATE_BASE_IDX                                                               2
+#define regDP3_DP_MSE_LINK_TIMING                                                                       0x24c0
+#define regDP3_DP_MSE_LINK_TIMING_BASE_IDX                                                              2
+#define regDP3_DP_MSE_MISC_CNTL                                                                         0x24c1
+#define regDP3_DP_MSE_MISC_CNTL_BASE_IDX                                                                2
+#define regDP3_DP_DPHY_BS_SR_SWAP_CNTL                                                                  0x24c6
+#define regDP3_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX                                                         2
+#define regDP3_DP_DPHY_HBR2_PATTERN_CONTROL                                                             0x24c7
+#define regDP3_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX                                                    2
+#define regDP3_DP_MSE_SAT0_STATUS                                                                       0x24c9
+#define regDP3_DP_MSE_SAT0_STATUS_BASE_IDX                                                              2
+#define regDP3_DP_MSE_SAT1_STATUS                                                                       0x24ca
+#define regDP3_DP_MSE_SAT1_STATUS_BASE_IDX                                                              2
+#define regDP3_DP_MSE_SAT2_STATUS                                                                       0x24cb
+#define regDP3_DP_MSE_SAT2_STATUS_BASE_IDX                                                              2
+#define regDP3_DP_HBLANK_CONTROL                                                                        0x24cd
+#define regDP3_DP_HBLANK_CONTROL_BASE_IDX                                                               2
+#define regDP3_DP_MSA_TIMING_PARAM1                                                                     0x24ce
+#define regDP3_DP_MSA_TIMING_PARAM1_BASE_IDX                                                            2
+#define regDP3_DP_MSA_TIMING_PARAM2                                                                     0x24cf
+#define regDP3_DP_MSA_TIMING_PARAM2_BASE_IDX                                                            2
+#define regDP3_DP_MSA_TIMING_PARAM3                                                                     0x24d0
+#define regDP3_DP_MSA_TIMING_PARAM3_BASE_IDX                                                            2
+#define regDP3_DP_MSA_TIMING_PARAM4                                                                     0x24d1
+#define regDP3_DP_MSA_TIMING_PARAM4_BASE_IDX                                                            2
+#define regDP3_DP_MSO_CNTL                                                                              0x24d2
+#define regDP3_DP_MSO_CNTL_BASE_IDX                                                                     2
+#define regDP3_DP_MSO_CNTL1                                                                             0x24d3
+#define regDP3_DP_MSO_CNTL1_BASE_IDX                                                                    2
+#define regDP3_DP_STEER_FIFO_CNTL                                                                       0x24d4
+#define regDP3_DP_STEER_FIFO_CNTL_BASE_IDX                                                              2
+#define regDP3_DP_SEC_CNTL2                                                                             0x24d5
+#define regDP3_DP_SEC_CNTL2_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_CNTL3                                                                             0x24d6
+#define regDP3_DP_SEC_CNTL3_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_CNTL4                                                                             0x24d7
+#define regDP3_DP_SEC_CNTL4_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_CNTL5                                                                             0x24d8
+#define regDP3_DP_SEC_CNTL5_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_CNTL6                                                                             0x24d9
+#define regDP3_DP_SEC_CNTL6_BASE_IDX                                                                    2
+#define regDP3_DP_SEC_CNTL7                                                                             0x24da
+#define regDP3_DP_SEC_CNTL7_BASE_IDX                                                                    2
+#define regDP3_DP_DB_CNTL                                                                               0x24db
+#define regDP3_DP_DB_CNTL_BASE_IDX                                                                      2
+#define regDP3_DP_MSA_VBID_MISC                                                                         0x24dc
+#define regDP3_DP_MSA_VBID_MISC_BASE_IDX                                                                2
+#define regDP3_DP_SEC_METADATA_TRANSMISSION                                                             0x24dd
+#define regDP3_DP_SEC_METADATA_TRANSMISSION_BASE_IDX                                                    2
+#define regDP3_DP_ALPM_CNTL                                                                             0x24df
+#define regDP3_DP_ALPM_CNTL_BASE_IDX                                                                    2
+#define regDP3_DP_GSP8_CNTL                                                                             0x24e0
+#define regDP3_DP_GSP8_CNTL_BASE_IDX                                                                    2
+#define regDP3_DP_GSP9_CNTL                                                                             0x24e1
+#define regDP3_DP_GSP9_CNTL_BASE_IDX                                                                    2
+#define regDP3_DP_GSP10_CNTL                                                                            0x24e2
+#define regDP3_DP_GSP10_CNTL_BASE_IDX                                                                   2
+#define regDP3_DP_GSP11_CNTL                                                                            0x24e3
+#define regDP3_DP_GSP11_CNTL_BASE_IDX                                                                   2
+#define regDP3_DP_GSP_EN_DB_STATUS                                                                      0x24e4
+#define regDP3_DP_GSP_EN_DB_STATUS_BASE_IDX                                                             2
+#define regDP3_DP_AUXLESS_ALPM_CNTL1                                                                    0x24e5
+#define regDP3_DP_AUXLESS_ALPM_CNTL1_BASE_IDX                                                           2
+#define regDP3_DP_AUXLESS_ALPM_CNTL2                                                                    0x24e6
+#define regDP3_DP_AUXLESS_ALPM_CNTL2_BASE_IDX                                                           2
+#define regDP3_DP_AUXLESS_ALPM_CNTL3                                                                    0x24e7
+#define regDP3_DP_AUXLESS_ALPM_CNTL3_BASE_IDX                                                           2
+#define regDP3_DP_AUXLESS_ALPM_CNTL4                                                                    0x24e8
+#define regDP3_DP_AUXLESS_ALPM_CNTL4_BASE_IDX                                                           2
+#define regDP3_DP_AUXLESS_ALPM_CNTL5                                                                    0x24e9
+#define regDP3_DP_AUXLESS_ALPM_CNTL5_BASE_IDX                                                           2
+#define regDP3_DP_STREAM_SYMBOL_COUNT_STATUS                                                            0x24ea
+#define regDP3_DP_STREAM_SYMBOL_COUNT_STATUS_BASE_IDX                                                   2
+#define regDP3_DP_STREAM_SYMBOL_COUNT_CONTROL                                                           0x24eb
+#define regDP3_DP_STREAM_SYMBOL_COUNT_CONTROL_BASE_IDX                                                  2
+#define regDP3_DP_LINK_SYMBOL_COUNT_STATUS0                                                             0x24ec
+#define regDP3_DP_LINK_SYMBOL_COUNT_STATUS0_BASE_IDX                                                    2
+#define regDP3_DP_LINK_SYMBOL_COUNT_STATUS1                                                             0x24ed
+#define regDP3_DP_LINK_SYMBOL_COUNT_STATUS1_BASE_IDX                                                    2
+#define regDP3_DP_LINK_SYMBOL_COUNT_CONTROL                                                             0x24ee
+#define regDP3_DP_LINK_SYMBOL_COUNT_CONTROL_BASE_IDX                                                    2
+#define regDP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL                                                     0x24ef
+#define regDP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                            2
+#define regDP3_DP_DPHY_FAST_TRAINING                                                                    0x24f2
+#define regDP3_DP_DPHY_FAST_TRAINING_BASE_IDX                                                           2
+#define regDP3_DP_DPHY_FAST_TRAINING_STATUS                                                             0x24f3
+#define regDP3_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dio_dig4_vpg_vpg_dispdec
+// base address: 0x166e0
+#define regVPG4_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x24f8
+#define regVPG4_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG4_VPG_GENERIC_PACKET_DATA                                                                 0x24f9
+#define regVPG4_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG4_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x24fa
+#define regVPG4_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x24fb
+#define regVPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG4_VPG_GENERIC_STATUS                                                                      0x24fc
+#define regVPG4_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG4_VPG_MEM_PWR                                                                             0x24fd
+#define regVPG4_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG4_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x24fe
+#define regVPG4_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG4_VPG_ISRC1_2_DATA                                                                        0x24ff
+#define regVPG4_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_dio_dig4_apg_apg_dispdec
+// base address: 0x1670c
+#define regAPG4_APG_CONTROL                                                                             0x2503
+#define regAPG4_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG4_APG_CONTROL2                                                                            0x2504
+#define regAPG4_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG4_APG_DBG_GEN_CONTROL                                                                     0x2505
+#define regAPG4_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG4_APG_MEM_PWR                                                                             0x2517
+#define regAPG4_APG_MEM_PWR_BASE_IDX                                                                    2
+
+// addressBlock: dce_dc_dio_dig4_dme_dme_dispdec
+// base address: 0x16784
+#define regDME4_DME_CONTROL                                                                             0x2521
+#define regDME4_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME4_DME_MEMORY_CONTROL                                                                      0x2522
+#define regDME4_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_dio_dig4_dispdec
+// base address: 0x1240
+#define regDIG4_DIG_FE_CNTL                                                                             0x2523
+#define regDIG4_DIG_FE_CNTL_BASE_IDX                                                                    2
+#define regDIG4_DIG_FE_CLK_CNTL                                                                         0x2524
+#define regDIG4_DIG_FE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG4_DIG_FE_EN_CNTL                                                                          0x2525
+#define regDIG4_DIG_FE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG4_DIG_OUTPUT_CRC_CNTL                                                                     0x2526
+#define regDIG4_DIG_OUTPUT_CRC_CNTL_BASE_IDX                                                            2
+#define regDIG4_DIG_OUTPUT_CRC_RESULT                                                                   0x2527
+#define regDIG4_DIG_OUTPUT_CRC_RESULT_BASE_IDX                                                          2
+#define regDIG4_DIG_CLOCK_PATTERN                                                                       0x2528
+#define regDIG4_DIG_CLOCK_PATTERN_BASE_IDX                                                              2
+#define regDIG4_DIG_TEST_PATTERN                                                                        0x2529
+#define regDIG4_DIG_TEST_PATTERN_BASE_IDX                                                               2
+#define regDIG4_DIG_RANDOM_PATTERN_SEED                                                                 0x252a
+#define regDIG4_DIG_RANDOM_PATTERN_SEED_BASE_IDX                                                        2
+#define regDIG4_DIG_FIFO_CTRL0                                                                          0x252b
+#define regDIG4_DIG_FIFO_CTRL0_BASE_IDX                                                                 2
+#define regDIG4_DIG_FIFO_CTRL1                                                                          0x252c
+#define regDIG4_DIG_FIFO_CTRL1_BASE_IDX                                                                 2
+#define regDIG4_HDMI_METADATA_PACKET_CONTROL                                                            0x252d
+#define regDIG4_HDMI_METADATA_PACKET_CONTROL_BASE_IDX                                                   2
+#define regDIG4_HDMI_CONTROL                                                                            0x252e
+#define regDIG4_HDMI_CONTROL_BASE_IDX                                                                   2
+#define regDIG4_HDMI_STATUS                                                                             0x252f
+#define regDIG4_HDMI_STATUS_BASE_IDX                                                                    2
+#define regDIG4_HDMI_AUDIO_PACKET_CONTROL                                                               0x2530
+#define regDIG4_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX                                                      2
+#define regDIG4_HDMI_ACR_PACKET_CONTROL                                                                 0x2531
+#define regDIG4_HDMI_ACR_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG4_HDMI_VBI_PACKET_CONTROL                                                                 0x2532
+#define regDIG4_HDMI_VBI_PACKET_CONTROL_BASE_IDX                                                        2
+#define regDIG4_HDMI_INFOFRAME_CONTROL0                                                                 0x2533
+#define regDIG4_HDMI_INFOFRAME_CONTROL0_BASE_IDX                                                        2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL0                                                            0x2534
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL6                                                            0x2535
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL6_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL5                                                            0x2536
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX                                                   2
+#define regDIG4_HDMI_GC                                                                                 0x2537
+#define regDIG4_HDMI_GC_BASE_IDX                                                                        2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL1                                                            0x2538
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL2                                                            0x2539
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL3                                                            0x253a
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL4                                                            0x253b
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL7                                                            0x253c
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL7_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL8                                                            0x253d
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL8_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL9                                                            0x253e
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL9_BASE_IDX                                                   2
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL10                                                           0x253f
+#define regDIG4_HDMI_GENERIC_PACKET_CONTROL10_BASE_IDX                                                  2
+#define regDIG4_HDMI_DB_CONTROL                                                                         0x2540
+#define regDIG4_HDMI_DB_CONTROL_BASE_IDX                                                                2
+#define regDIG4_HDMI_ACR_32_0                                                                           0x2541
+#define regDIG4_HDMI_ACR_32_0_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_32_1                                                                           0x2542
+#define regDIG4_HDMI_ACR_32_1_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_44_0                                                                           0x2543
+#define regDIG4_HDMI_ACR_44_0_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_44_1                                                                           0x2544
+#define regDIG4_HDMI_ACR_44_1_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_48_0                                                                           0x2545
+#define regDIG4_HDMI_ACR_48_0_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_48_1                                                                           0x2546
+#define regDIG4_HDMI_ACR_48_1_BASE_IDX                                                                  2
+#define regDIG4_HDMI_ACR_STATUS_0                                                                       0x2547
+#define regDIG4_HDMI_ACR_STATUS_0_BASE_IDX                                                              2
+#define regDIG4_HDMI_ACR_STATUS_1                                                                       0x2548
+#define regDIG4_HDMI_ACR_STATUS_1_BASE_IDX                                                              2
+#define regDIG4_DIG_FE_AUDIO_CNTL                                                                       0x2549
+#define regDIG4_DIG_FE_AUDIO_CNTL_BASE_IDX                                                              2
+#define regDIG4_DIG_BE_CLK_CNTL                                                                         0x254a
+#define regDIG4_DIG_BE_CLK_CNTL_BASE_IDX                                                                2
+#define regDIG4_DIG_BE_CNTL                                                                             0x254b
+#define regDIG4_DIG_BE_CNTL_BASE_IDX                                                                    2
+#define regDIG4_DIG_BE_EN_CNTL                                                                          0x254c
+#define regDIG4_DIG_BE_EN_CNTL_BASE_IDX                                                                 2
+#define regDIG4_TMDS_CNTL                                                                               0x2573
+#define regDIG4_TMDS_CNTL_BASE_IDX                                                                      2
+#define regDIG4_TMDS_CONTROL_CHAR                                                                       0x2574
+#define regDIG4_TMDS_CONTROL_CHAR_BASE_IDX                                                              2
+#define regDIG4_TMDS_CONTROL0_FEEDBACK                                                                  0x2575
+#define regDIG4_TMDS_CONTROL0_FEEDBACK_BASE_IDX                                                         2
+#define regDIG4_TMDS_STEREOSYNC_CTL_SEL                                                                 0x2576
+#define regDIG4_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX                                                        2
+#define regDIG4_TMDS_SYNC_CHAR_PATTERN_0_1                                                              0x2577
+#define regDIG4_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX                                                     2
+#define regDIG4_TMDS_SYNC_CHAR_PATTERN_2_3                                                              0x2578
+#define regDIG4_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX                                                     2
+#define regDIG4_TMDS_CTL_BITS                                                                           0x257a
+#define regDIG4_TMDS_CTL_BITS_BASE_IDX                                                                  2
+#define regDIG4_TMDS_DCBALANCER_CONTROL                                                                 0x257b
+#define regDIG4_TMDS_DCBALANCER_CONTROL_BASE_IDX                                                        2
+#define regDIG4_TMDS_SYNC_DCBALANCE_CHAR                                                                0x257c
+#define regDIG4_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX                                                       2
+#define regDIG4_TMDS_CTL0_1_GEN_CNTL                                                                    0x257d
+#define regDIG4_TMDS_CTL0_1_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG4_TMDS_CTL2_3_GEN_CNTL                                                                    0x257e
+#define regDIG4_TMDS_CTL2_3_GEN_CNTL_BASE_IDX                                                           2
+#define regDIG4_DIG_VERSION                                                                             0x2580
+#define regDIG4_DIG_VERSION_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_dio_dp4_dispdec
+// base address: 0x1240
+#define regDP4_DP_LINK_CNTL                                                                             0x25ae
+#define regDP4_DP_LINK_CNTL_BASE_IDX                                                                    2
+#define regDP4_DP_PIXEL_FORMAT                                                                          0x25af
+#define regDP4_DP_PIXEL_FORMAT_BASE_IDX                                                                 2
+#define regDP4_DP_MSA_COLORIMETRY                                                                       0x25b0
+#define regDP4_DP_MSA_COLORIMETRY_BASE_IDX                                                              2
+#define regDP4_DP_CONFIG                                                                                0x25b1
+#define regDP4_DP_CONFIG_BASE_IDX                                                                       2
+#define regDP4_DP_VID_STREAM_CNTL                                                                       0x25b2
+#define regDP4_DP_VID_STREAM_CNTL_BASE_IDX                                                              2
+#define regDP4_DP_STEER_FIFO                                                                            0x25b3
+#define regDP4_DP_STEER_FIFO_BASE_IDX                                                                   2
+#define regDP4_DP_MSA_MISC                                                                              0x25b4
+#define regDP4_DP_MSA_MISC_BASE_IDX                                                                     2
+#define regDP4_DP_DPHY_INTERNAL_CTRL                                                                    0x25b5
+#define regDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX                                                           2
+#define regDP4_DP_VID_TIMING                                                                            0x25b6
+#define regDP4_DP_VID_TIMING_BASE_IDX                                                                   2
+#define regDP4_DP_VID_N                                                                                 0x25b7
+#define regDP4_DP_VID_N_BASE_IDX                                                                        2
+#define regDP4_DP_VID_M                                                                                 0x25b8
+#define regDP4_DP_VID_M_BASE_IDX                                                                        2
+#define regDP4_DP_LINK_FRAMING_CNTL                                                                     0x25b9
+#define regDP4_DP_LINK_FRAMING_CNTL_BASE_IDX                                                            2
+#define regDP4_DP_HBR2_EYE_PATTERN                                                                      0x25ba
+#define regDP4_DP_HBR2_EYE_PATTERN_BASE_IDX                                                             2
+#define regDP4_DP_VID_MSA_VBID                                                                          0x25bb
+#define regDP4_DP_VID_MSA_VBID_BASE_IDX                                                                 2
+#define regDP4_DP_VID_INTERRUPT_CNTL                                                                    0x25bc
+#define regDP4_DP_VID_INTERRUPT_CNTL_BASE_IDX                                                           2
+#define regDP4_DP_DPHY_CNTL                                                                             0x25bd
+#define regDP4_DP_DPHY_CNTL_BASE_IDX                                                                    2
+#define regDP4_DP_DPHY_TRAINING_PATTERN_SEL                                                             0x25be
+#define regDP4_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX                                                    2
+#define regDP4_DP_DPHY_SYM0                                                                             0x25bf
+#define regDP4_DP_DPHY_SYM0_BASE_IDX                                                                    2
+#define regDP4_DP_DPHY_SYM1                                                                             0x25c0
+#define regDP4_DP_DPHY_SYM1_BASE_IDX                                                                    2
+#define regDP4_DP_DPHY_SYM2                                                                             0x25c1
+#define regDP4_DP_DPHY_SYM2_BASE_IDX                                                                    2
+#define regDP4_DP_DPHY_8B10B_CNTL                                                                       0x25c2
+#define regDP4_DP_DPHY_8B10B_CNTL_BASE_IDX                                                              2
+#define regDP4_DP_DPHY_PRBS_CNTL                                                                        0x25c3
+#define regDP4_DP_DPHY_PRBS_CNTL_BASE_IDX                                                               2
+#define regDP4_DP_DPHY_SCRAM_CNTL                                                                       0x25c4
+#define regDP4_DP_DPHY_SCRAM_CNTL_BASE_IDX                                                              2
+#define regDP4_DP_DPHY_CRC_CONTROL0                                                                     0x25c5
+#define regDP4_DP_DPHY_CRC_CONTROL0_BASE_IDX                                                            2
+#define regDP4_DP_DPHY_CRC_CONTROL1                                                                     0x25c6
+#define regDP4_DP_DPHY_CRC_CONTROL1_BASE_IDX                                                            2
+#define regDP4_DP_DPHY_CRC_RESULT0                                                                      0x25c7
+#define regDP4_DP_DPHY_CRC_RESULT0_BASE_IDX                                                             2
+#define regDP4_DP_DPHY_CRC_RESULT1                                                                      0x25c8
+#define regDP4_DP_DPHY_CRC_RESULT1_BASE_IDX                                                             2
+#define regDP4_DP_DPHY_CRC_RESULT2                                                                      0x25c9
+#define regDP4_DP_DPHY_CRC_RESULT2_BASE_IDX                                                             2
+#define regDP4_DP_DPHY_CRC_RESULT3                                                                      0x25ca
+#define regDP4_DP_DPHY_CRC_RESULT3_BASE_IDX                                                             2
+#define regDP4_DP_DPHY_CRC_STATUS                                                                       0x25cb
+#define regDP4_DP_DPHY_CRC_STATUS_BASE_IDX                                                              2
+#define regDP4_DP_TU_CNTL                                                                               0x25cc
+#define regDP4_DP_TU_CNTL_BASE_IDX                                                                      2
+#define regDP4_DP_PIXEL_FORMAT_DB_CNTL                                                                  0x25cd
+#define regDP4_DP_PIXEL_FORMAT_DB_CNTL_BASE_IDX                                                         2
+#define regDP4_DP_SEC_CNTL                                                                              0x25d1
+#define regDP4_DP_SEC_CNTL_BASE_IDX                                                                     2
+#define regDP4_DP_SEC_CNTL1                                                                             0x25d2
+#define regDP4_DP_SEC_CNTL1_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_FRAMING1                                                                          0x25d3
+#define regDP4_DP_SEC_FRAMING1_BASE_IDX                                                                 2
+#define regDP4_DP_SEC_FRAMING2                                                                          0x25d4
+#define regDP4_DP_SEC_FRAMING2_BASE_IDX                                                                 2
+#define regDP4_DP_SEC_FRAMING3                                                                          0x25d5
+#define regDP4_DP_SEC_FRAMING3_BASE_IDX                                                                 2
+#define regDP4_DP_SEC_FRAMING4                                                                          0x25d6
+#define regDP4_DP_SEC_FRAMING4_BASE_IDX                                                                 2
+#define regDP4_DP_SEC_AUD_N                                                                             0x25d7
+#define regDP4_DP_SEC_AUD_N_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_AUD_N_READBACK                                                                    0x25d8
+#define regDP4_DP_SEC_AUD_N_READBACK_BASE_IDX                                                           2
+#define regDP4_DP_SEC_AUD_M                                                                             0x25d9
+#define regDP4_DP_SEC_AUD_M_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_AUD_M_READBACK                                                                    0x25da
+#define regDP4_DP_SEC_AUD_M_READBACK_BASE_IDX                                                           2
+#define regDP4_DP_SEC_TIMESTAMP                                                                         0x25db
+#define regDP4_DP_SEC_TIMESTAMP_BASE_IDX                                                                2
+#define regDP4_DP_SEC_PACKET_CNTL                                                                       0x25dc
+#define regDP4_DP_SEC_PACKET_CNTL_BASE_IDX                                                              2
+#define regDP4_DP_MSE_RATE_CNTL                                                                         0x25dd
+#define regDP4_DP_MSE_RATE_CNTL_BASE_IDX                                                                2
+#define regDP4_DP_MSE_RATE_UPDATE                                                                       0x25df
+#define regDP4_DP_MSE_RATE_UPDATE_BASE_IDX                                                              2
+#define regDP4_DP_MSE_SAT0                                                                              0x25e0
+#define regDP4_DP_MSE_SAT0_BASE_IDX                                                                     2
+#define regDP4_DP_MSE_SAT1                                                                              0x25e1
+#define regDP4_DP_MSE_SAT1_BASE_IDX                                                                     2
+#define regDP4_DP_MSE_SAT2                                                                              0x25e2
+#define regDP4_DP_MSE_SAT2_BASE_IDX                                                                     2
+#define regDP4_DP_MSE_SAT_UPDATE                                                                        0x25e3
+#define regDP4_DP_MSE_SAT_UPDATE_BASE_IDX                                                               2
+#define regDP4_DP_MSE_LINK_TIMING                                                                       0x25e4
+#define regDP4_DP_MSE_LINK_TIMING_BASE_IDX                                                              2
+#define regDP4_DP_MSE_MISC_CNTL                                                                         0x25e5
+#define regDP4_DP_MSE_MISC_CNTL_BASE_IDX                                                                2
+#define regDP4_DP_DPHY_BS_SR_SWAP_CNTL                                                                  0x25ea
+#define regDP4_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX                                                         2
+#define regDP4_DP_DPHY_HBR2_PATTERN_CONTROL                                                             0x25eb
+#define regDP4_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX                                                    2
+#define regDP4_DP_MSE_SAT0_STATUS                                                                       0x25ed
+#define regDP4_DP_MSE_SAT0_STATUS_BASE_IDX                                                              2
+#define regDP4_DP_MSE_SAT1_STATUS                                                                       0x25ee
+#define regDP4_DP_MSE_SAT1_STATUS_BASE_IDX                                                              2
+#define regDP4_DP_MSE_SAT2_STATUS                                                                       0x25ef
+#define regDP4_DP_MSE_SAT2_STATUS_BASE_IDX                                                              2
+#define regDP4_DP_HBLANK_CONTROL                                                                        0x25f1
+#define regDP4_DP_HBLANK_CONTROL_BASE_IDX                                                               2
+#define regDP4_DP_MSA_TIMING_PARAM1                                                                     0x25f2
+#define regDP4_DP_MSA_TIMING_PARAM1_BASE_IDX                                                            2
+#define regDP4_DP_MSA_TIMING_PARAM2                                                                     0x25f3
+#define regDP4_DP_MSA_TIMING_PARAM2_BASE_IDX                                                            2
+#define regDP4_DP_MSA_TIMING_PARAM3                                                                     0x25f4
+#define regDP4_DP_MSA_TIMING_PARAM3_BASE_IDX                                                            2
+#define regDP4_DP_MSA_TIMING_PARAM4                                                                     0x25f5
+#define regDP4_DP_MSA_TIMING_PARAM4_BASE_IDX                                                            2
+#define regDP4_DP_MSO_CNTL                                                                              0x25f6
+#define regDP4_DP_MSO_CNTL_BASE_IDX                                                                     2
+#define regDP4_DP_MSO_CNTL1                                                                             0x25f7
+#define regDP4_DP_MSO_CNTL1_BASE_IDX                                                                    2
+#define regDP4_DP_STEER_FIFO_CNTL                                                                       0x25f8
+#define regDP4_DP_STEER_FIFO_CNTL_BASE_IDX                                                              2
+#define regDP4_DP_SEC_CNTL2                                                                             0x25f9
+#define regDP4_DP_SEC_CNTL2_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_CNTL3                                                                             0x25fa
+#define regDP4_DP_SEC_CNTL3_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_CNTL4                                                                             0x25fb
+#define regDP4_DP_SEC_CNTL4_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_CNTL5                                                                             0x25fc
+#define regDP4_DP_SEC_CNTL5_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_CNTL6                                                                             0x25fd
+#define regDP4_DP_SEC_CNTL6_BASE_IDX                                                                    2
+#define regDP4_DP_SEC_CNTL7                                                                             0x25fe
+#define regDP4_DP_SEC_CNTL7_BASE_IDX                                                                    2
+#define regDP4_DP_DB_CNTL                                                                               0x25ff
+#define regDP4_DP_DB_CNTL_BASE_IDX                                                                      2
+#define regDP4_DP_MSA_VBID_MISC                                                                         0x2600
+#define regDP4_DP_MSA_VBID_MISC_BASE_IDX                                                                2
+#define regDP4_DP_SEC_METADATA_TRANSMISSION                                                             0x2601
+#define regDP4_DP_SEC_METADATA_TRANSMISSION_BASE_IDX                                                    2
+#define regDP4_DP_ALPM_CNTL                                                                             0x2603
+#define regDP4_DP_ALPM_CNTL_BASE_IDX                                                                    2
+#define regDP4_DP_GSP8_CNTL                                                                             0x2604
+#define regDP4_DP_GSP8_CNTL_BASE_IDX                                                                    2
+#define regDP4_DP_GSP9_CNTL                                                                             0x2605
+#define regDP4_DP_GSP9_CNTL_BASE_IDX                                                                    2
+#define regDP4_DP_GSP10_CNTL                                                                            0x2606
+#define regDP4_DP_GSP10_CNTL_BASE_IDX                                                                   2
+#define regDP4_DP_GSP11_CNTL                                                                            0x2607
+#define regDP4_DP_GSP11_CNTL_BASE_IDX                                                                   2
+#define regDP4_DP_GSP_EN_DB_STATUS                                                                      0x2608
+#define regDP4_DP_GSP_EN_DB_STATUS_BASE_IDX                                                             2
+#define regDP4_DP_AUXLESS_ALPM_CNTL1                                                                    0x2609
+#define regDP4_DP_AUXLESS_ALPM_CNTL1_BASE_IDX                                                           2
+#define regDP4_DP_AUXLESS_ALPM_CNTL2                                                                    0x260a
+#define regDP4_DP_AUXLESS_ALPM_CNTL2_BASE_IDX                                                           2
+#define regDP4_DP_AUXLESS_ALPM_CNTL3                                                                    0x260b
+#define regDP4_DP_AUXLESS_ALPM_CNTL3_BASE_IDX                                                           2
+#define regDP4_DP_AUXLESS_ALPM_CNTL4                                                                    0x260c
+#define regDP4_DP_AUXLESS_ALPM_CNTL4_BASE_IDX                                                           2
+#define regDP4_DP_AUXLESS_ALPM_CNTL5                                                                    0x260d
+#define regDP4_DP_AUXLESS_ALPM_CNTL5_BASE_IDX                                                           2
+#define regDP4_DP_STREAM_SYMBOL_COUNT_STATUS                                                            0x260e
+#define regDP4_DP_STREAM_SYMBOL_COUNT_STATUS_BASE_IDX                                                   2
+#define regDP4_DP_STREAM_SYMBOL_COUNT_CONTROL                                                           0x260f
+#define regDP4_DP_STREAM_SYMBOL_COUNT_CONTROL_BASE_IDX                                                  2
+#define regDP4_DP_LINK_SYMBOL_COUNT_STATUS0                                                             0x2610
+#define regDP4_DP_LINK_SYMBOL_COUNT_STATUS0_BASE_IDX                                                    2
+#define regDP4_DP_LINK_SYMBOL_COUNT_STATUS1                                                             0x2611
+#define regDP4_DP_LINK_SYMBOL_COUNT_STATUS1_BASE_IDX                                                    2
+#define regDP4_DP_LINK_SYMBOL_COUNT_CONTROL                                                             0x2612
+#define regDP4_DP_LINK_SYMBOL_COUNT_CONTROL_BASE_IDX                                                    2
+#define regDP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL                                                     0x2613
+#define regDP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                            2
+#define regDP4_DP_DPHY_FAST_TRAINING                                                                    0x2616
+#define regDP4_DP_DPHY_FAST_TRAINING_BASE_IDX                                                           2
+#define regDP4_DP_DPHY_FAST_TRAINING_STATUS                                                             0x2617
+#define regDP4_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_top_dispdec
+// base address: 0x0
+#define regDSC_TOP0_DSC_TOP_CONTROL                                                                     0x3000
+#define regDSC_TOP0_DSC_TOP_CONTROL_BASE_IDX                                                            2
+#define regDSC_TOP0_DSC_DEBUG_CONTROL                                                                   0x3001
+#define regDSC_TOP0_DSC_DEBUG_CONTROL_BASE_IDX                                                          2
+#define regDSC_TOP0_DSC_SPARE_DEBUG                                                                     0x3002
+#define regDSC_TOP0_DSC_SPARE_DEBUG_BASE_IDX                                                            2
+#define regDSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX                                                            0x3003
+#define regDSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX_BASE_IDX                                                   2
+#define regDSC_TOP0_DSC_TOP_TEST_DEBUG_DATA                                                             0x3004
+#define regDSC_TOP0_DSC_TOP_TEST_DEBUG_DATA_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
+// base address: 0x0
+#define regDSCCIF0_DSCCIF_CONFIG0                                                                       0x3005
+#define regDSCCIF0_DSCCIF_CONFIG0_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dscc_dispdec
+// base address: 0x0
+#define regDSCC0_DSCC_CONFIG0                                                                           0x300a
+#define regDSCC0_DSCC_CONFIG0_BASE_IDX                                                                  2
+#define regDSCC0_DSCC_CONFIG1                                                                           0x300b
+#define regDSCC0_DSCC_CONFIG1_BASE_IDX                                                                  2
+#define regDSCC0_DSCC_CONFIG2                                                                           0x300c
+#define regDSCC0_DSCC_CONFIG2_BASE_IDX                                                                  2
+#define regDSCC0_DSCC_STATUS                                                                            0x300d
+#define regDSCC0_DSCC_STATUS_BASE_IDX                                                                   2
+#define regDSCC0_DSCC_INTERRUPT_CONTROL0                                                                0x300e
+#define regDSCC0_DSCC_INTERRUPT_CONTROL0_BASE_IDX                                                       2
+#define regDSCC0_DSCC_INTERRUPT_CONTROL1                                                                0x300f
+#define regDSCC0_DSCC_INTERRUPT_CONTROL1_BASE_IDX                                                       2
+#define regDSCC0_DSCC_INTERRUPT_STATUS0                                                                 0x3010
+#define regDSCC0_DSCC_INTERRUPT_STATUS0_BASE_IDX                                                        2
+#define regDSCC0_DSCC_INTERRUPT_STATUS1                                                                 0x3011
+#define regDSCC0_DSCC_INTERRUPT_STATUS1_BASE_IDX                                                        2
+#define regDSCC0_DSCC_PPS_CONFIG0                                                                       0x3012
+#define regDSCC0_DSCC_PPS_CONFIG0_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG1                                                                       0x3013
+#define regDSCC0_DSCC_PPS_CONFIG1_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG2                                                                       0x3014
+#define regDSCC0_DSCC_PPS_CONFIG2_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG3                                                                       0x3015
+#define regDSCC0_DSCC_PPS_CONFIG3_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG4                                                                       0x3016
+#define regDSCC0_DSCC_PPS_CONFIG4_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG5                                                                       0x3017
+#define regDSCC0_DSCC_PPS_CONFIG5_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG6                                                                       0x3018
+#define regDSCC0_DSCC_PPS_CONFIG6_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG7                                                                       0x3019
+#define regDSCC0_DSCC_PPS_CONFIG7_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG8                                                                       0x301a
+#define regDSCC0_DSCC_PPS_CONFIG8_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG9                                                                       0x301b
+#define regDSCC0_DSCC_PPS_CONFIG9_BASE_IDX                                                              2
+#define regDSCC0_DSCC_PPS_CONFIG10                                                                      0x301c
+#define regDSCC0_DSCC_PPS_CONFIG10_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG11                                                                      0x301d
+#define regDSCC0_DSCC_PPS_CONFIG11_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG12                                                                      0x301e
+#define regDSCC0_DSCC_PPS_CONFIG12_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG13                                                                      0x301f
+#define regDSCC0_DSCC_PPS_CONFIG13_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG14                                                                      0x3020
+#define regDSCC0_DSCC_PPS_CONFIG14_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG15                                                                      0x3021
+#define regDSCC0_DSCC_PPS_CONFIG15_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG16                                                                      0x3022
+#define regDSCC0_DSCC_PPS_CONFIG16_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG17                                                                      0x3023
+#define regDSCC0_DSCC_PPS_CONFIG17_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG18                                                                      0x3024
+#define regDSCC0_DSCC_PPS_CONFIG18_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG19                                                                      0x3025
+#define regDSCC0_DSCC_PPS_CONFIG19_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG20                                                                      0x3026
+#define regDSCC0_DSCC_PPS_CONFIG20_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG21                                                                      0x3027
+#define regDSCC0_DSCC_PPS_CONFIG21_BASE_IDX                                                             2
+#define regDSCC0_DSCC_PPS_CONFIG22                                                                      0x3028
+#define regDSCC0_DSCC_PPS_CONFIG22_BASE_IDX                                                             2
+#define regDSCC0_DSCC_MEM_POWER_CONTROL0                                                                0x3029
+#define regDSCC0_DSCC_MEM_POWER_CONTROL0_BASE_IDX                                                       2
+#define regDSCC0_DSCC_MEM_POWER_CONTROL1                                                                0x302a
+#define regDSCC0_DSCC_MEM_POWER_CONTROL1_BASE_IDX                                                       2
+#define regDSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER                                                           0x302b
+#define regDSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX                                                  2
+#define regDSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER                                                           0x302c
+#define regDSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX                                                  2
+#define regDSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER                                                          0x302d
+#define regDSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER                                                          0x302e
+#define regDSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER                                                          0x302f
+#define regDSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER                                                          0x3030
+#define regDSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC0_DSCC_MAX_ABS_ERROR0                                                                    0x3031
+#define regDSCC0_DSCC_MAX_ABS_ERROR0_BASE_IDX                                                           2
+#define regDSCC0_DSCC_MAX_ABS_ERROR1                                                                    0x3032
+#define regDSCC0_DSCC_MAX_ABS_ERROR1_BASE_IDX                                                           2
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0                                                 0x3033
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_BASE_IDX                                        2
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1                                                 0x3034
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_BASE_IDX                                        2
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2                                                 0x3035
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_BASE_IDX                                        2
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3                                                 0x3036
+#define regDSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_BASE_IDX                                        2
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0                                             0x3037
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_BASE_IDX                                    2
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1                                             0x3038
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_BASE_IDX                                    2
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2                                             0x3039
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_BASE_IDX                                    2
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3                                             0x303a
+#define regDSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_BASE_IDX                                    2
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX0                                                                 0x303b
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX0_BASE_IDX                                                        2
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX1                                                                 0x303c
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX1_BASE_IDX                                                        2
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX2                                                                 0x303d
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX2_BASE_IDX                                                        2
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX3                                                                 0x303e
+#define regDSCC0_DSCC_TEST_DEBUG_INDEX3_BASE_IDX                                                        2
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x303f
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0                                                                  0x3040
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA1                                                                  0x3041
+#define regDSCC0_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA2                                                                  0x3042
+#define regDSCC0_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA3                                                                  0x3043
+#define regDSCC0_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
+#define regDSCC0_DSCC_DISPCLK_TEST_DEBUG_INDEX0                                                         0x3044
+#define regDSCC0_DSCC_DISPCLK_TEST_DEBUG_INDEX0_BASE_IDX                                                2
+#define regDSCC0_DSCC_DISPCLK_TEST_DEBUG_DATA0                                                          0x3045
+#define regDSCC0_DSCC_DISPCLK_TEST_DEBUG_DATA0_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc140
+#define regDC_PERFMON17_PERFCOUNTER_CNTL                                                                0x3050
+#define regDC_PERFMON17_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON17_PERFCOUNTER_CNTL2                                                               0x3051
+#define regDC_PERFMON17_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON17_PERFCOUNTER_STATE                                                               0x3052
+#define regDC_PERFMON17_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON17_PERFMON_CNTL                                                                    0x3053
+#define regDC_PERFMON17_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON17_PERFMON_CNTL2                                                                   0x3054
+#define regDC_PERFMON17_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON17_PERFMON_CVALUE_INT_MISC                                                         0x3055
+#define regDC_PERFMON17_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON17_PERFMON_CVALUE_LOW                                                              0x3056
+#define regDC_PERFMON17_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON17_PERFMON_HI                                                                      0x3057
+#define regDC_PERFMON17_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON17_PERFMON_LOW                                                                     0x3058
+#define regDC_PERFMON17_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_top_dispdec
+// base address: 0x170
+#define regDSC_TOP1_DSC_TOP_CONTROL                                                                     0x305c
+#define regDSC_TOP1_DSC_TOP_CONTROL_BASE_IDX                                                            2
+#define regDSC_TOP1_DSC_DEBUG_CONTROL                                                                   0x305d
+#define regDSC_TOP1_DSC_DEBUG_CONTROL_BASE_IDX                                                          2
+#define regDSC_TOP1_DSC_SPARE_DEBUG                                                                     0x305e
+#define regDSC_TOP1_DSC_SPARE_DEBUG_BASE_IDX                                                            2
+#define regDSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX                                                            0x305f
+#define regDSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX_BASE_IDX                                                   2
+#define regDSC_TOP1_DSC_TOP_TEST_DEBUG_DATA                                                             0x3060
+#define regDSC_TOP1_DSC_TOP_TEST_DEBUG_DATA_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsccif_dispdec
+// base address: 0x170
+#define regDSCCIF1_DSCCIF_CONFIG0                                                                       0x3061
+#define regDSCCIF1_DSCCIF_CONFIG0_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dscc_dispdec
+// base address: 0x170
+#define regDSCC1_DSCC_CONFIG0                                                                           0x3066
+#define regDSCC1_DSCC_CONFIG0_BASE_IDX                                                                  2
+#define regDSCC1_DSCC_CONFIG1                                                                           0x3067
+#define regDSCC1_DSCC_CONFIG1_BASE_IDX                                                                  2
+#define regDSCC1_DSCC_CONFIG2                                                                           0x3068
+#define regDSCC1_DSCC_CONFIG2_BASE_IDX                                                                  2
+#define regDSCC1_DSCC_STATUS                                                                            0x3069
+#define regDSCC1_DSCC_STATUS_BASE_IDX                                                                   2
+#define regDSCC1_DSCC_INTERRUPT_CONTROL0                                                                0x306a
+#define regDSCC1_DSCC_INTERRUPT_CONTROL0_BASE_IDX                                                       2
+#define regDSCC1_DSCC_INTERRUPT_CONTROL1                                                                0x306b
+#define regDSCC1_DSCC_INTERRUPT_CONTROL1_BASE_IDX                                                       2
+#define regDSCC1_DSCC_INTERRUPT_STATUS0                                                                 0x306c
+#define regDSCC1_DSCC_INTERRUPT_STATUS0_BASE_IDX                                                        2
+#define regDSCC1_DSCC_INTERRUPT_STATUS1                                                                 0x306d
+#define regDSCC1_DSCC_INTERRUPT_STATUS1_BASE_IDX                                                        2
+#define regDSCC1_DSCC_PPS_CONFIG0                                                                       0x306e
+#define regDSCC1_DSCC_PPS_CONFIG0_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG1                                                                       0x306f
+#define regDSCC1_DSCC_PPS_CONFIG1_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG2                                                                       0x3070
+#define regDSCC1_DSCC_PPS_CONFIG2_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG3                                                                       0x3071
+#define regDSCC1_DSCC_PPS_CONFIG3_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG4                                                                       0x3072
+#define regDSCC1_DSCC_PPS_CONFIG4_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG5                                                                       0x3073
+#define regDSCC1_DSCC_PPS_CONFIG5_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG6                                                                       0x3074
+#define regDSCC1_DSCC_PPS_CONFIG6_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG7                                                                       0x3075
+#define regDSCC1_DSCC_PPS_CONFIG7_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG8                                                                       0x3076
+#define regDSCC1_DSCC_PPS_CONFIG8_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG9                                                                       0x3077
+#define regDSCC1_DSCC_PPS_CONFIG9_BASE_IDX                                                              2
+#define regDSCC1_DSCC_PPS_CONFIG10                                                                      0x3078
+#define regDSCC1_DSCC_PPS_CONFIG10_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG11                                                                      0x3079
+#define regDSCC1_DSCC_PPS_CONFIG11_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG12                                                                      0x307a
+#define regDSCC1_DSCC_PPS_CONFIG12_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG13                                                                      0x307b
+#define regDSCC1_DSCC_PPS_CONFIG13_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG14                                                                      0x307c
+#define regDSCC1_DSCC_PPS_CONFIG14_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG15                                                                      0x307d
+#define regDSCC1_DSCC_PPS_CONFIG15_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG16                                                                      0x307e
+#define regDSCC1_DSCC_PPS_CONFIG16_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG17                                                                      0x307f
+#define regDSCC1_DSCC_PPS_CONFIG17_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG18                                                                      0x3080
+#define regDSCC1_DSCC_PPS_CONFIG18_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG19                                                                      0x3081
+#define regDSCC1_DSCC_PPS_CONFIG19_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG20                                                                      0x3082
+#define regDSCC1_DSCC_PPS_CONFIG20_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG21                                                                      0x3083
+#define regDSCC1_DSCC_PPS_CONFIG21_BASE_IDX                                                             2
+#define regDSCC1_DSCC_PPS_CONFIG22                                                                      0x3084
+#define regDSCC1_DSCC_PPS_CONFIG22_BASE_IDX                                                             2
+#define regDSCC1_DSCC_MEM_POWER_CONTROL0                                                                0x3085
+#define regDSCC1_DSCC_MEM_POWER_CONTROL0_BASE_IDX                                                       2
+#define regDSCC1_DSCC_MEM_POWER_CONTROL1                                                                0x3086
+#define regDSCC1_DSCC_MEM_POWER_CONTROL1_BASE_IDX                                                       2
+#define regDSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER                                                           0x3087
+#define regDSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX                                                  2
+#define regDSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER                                                           0x3088
+#define regDSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX                                                  2
+#define regDSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER                                                          0x3089
+#define regDSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER                                                          0x308a
+#define regDSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER                                                          0x308b
+#define regDSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER                                                          0x308c
+#define regDSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC1_DSCC_MAX_ABS_ERROR0                                                                    0x308d
+#define regDSCC1_DSCC_MAX_ABS_ERROR0_BASE_IDX                                                           2
+#define regDSCC1_DSCC_MAX_ABS_ERROR1                                                                    0x308e
+#define regDSCC1_DSCC_MAX_ABS_ERROR1_BASE_IDX                                                           2
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0                                                 0x308f
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_BASE_IDX                                        2
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1                                                 0x3090
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_BASE_IDX                                        2
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2                                                 0x3091
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_BASE_IDX                                        2
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3                                                 0x3092
+#define regDSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_BASE_IDX                                        2
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0                                             0x3093
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_BASE_IDX                                    2
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1                                             0x3094
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_BASE_IDX                                    2
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2                                             0x3095
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_BASE_IDX                                    2
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3                                             0x3096
+#define regDSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_BASE_IDX                                    2
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX0                                                                 0x3097
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX0_BASE_IDX                                                        2
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX1                                                                 0x3098
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX1_BASE_IDX                                                        2
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX2                                                                 0x3099
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX2_BASE_IDX                                                        2
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX3                                                                 0x309a
+#define regDSCC1_DSCC_TEST_DEBUG_INDEX3_BASE_IDX                                                        2
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x309b
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0                                                                  0x309c
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA1                                                                  0x309d
+#define regDSCC1_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA2                                                                  0x309e
+#define regDSCC1_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA3                                                                  0x309f
+#define regDSCC1_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
+#define regDSCC1_DSCC_DISPCLK_TEST_DEBUG_INDEX0                                                         0x30a0
+#define regDSCC1_DSCC_DISPCLK_TEST_DEBUG_INDEX0_BASE_IDX                                                2
+#define regDSCC1_DSCC_DISPCLK_TEST_DEBUG_DATA0                                                          0x30a1
+#define regDSCC1_DSCC_DISPCLK_TEST_DEBUG_DATA0_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc2b0
+#define regDC_PERFMON18_PERFCOUNTER_CNTL                                                                0x30ac
+#define regDC_PERFMON18_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON18_PERFCOUNTER_CNTL2                                                               0x30ad
+#define regDC_PERFMON18_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON18_PERFCOUNTER_STATE                                                               0x30ae
+#define regDC_PERFMON18_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON18_PERFMON_CNTL                                                                    0x30af
+#define regDC_PERFMON18_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON18_PERFMON_CNTL2                                                                   0x30b0
+#define regDC_PERFMON18_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON18_PERFMON_CVALUE_INT_MISC                                                         0x30b1
+#define regDC_PERFMON18_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON18_PERFMON_CVALUE_LOW                                                              0x30b2
+#define regDC_PERFMON18_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON18_PERFMON_HI                                                                      0x30b3
+#define regDC_PERFMON18_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON18_PERFMON_LOW                                                                     0x30b4
+#define regDC_PERFMON18_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_top_dispdec
+// base address: 0x2e0
+#define regDSC_TOP2_DSC_TOP_CONTROL                                                                     0x30b8
+#define regDSC_TOP2_DSC_TOP_CONTROL_BASE_IDX                                                            2
+#define regDSC_TOP2_DSC_DEBUG_CONTROL                                                                   0x30b9
+#define regDSC_TOP2_DSC_DEBUG_CONTROL_BASE_IDX                                                          2
+#define regDSC_TOP2_DSC_SPARE_DEBUG                                                                     0x30ba
+#define regDSC_TOP2_DSC_SPARE_DEBUG_BASE_IDX                                                            2
+#define regDSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX                                                            0x30bb
+#define regDSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX_BASE_IDX                                                   2
+#define regDSC_TOP2_DSC_TOP_TEST_DEBUG_DATA                                                             0x30bc
+#define regDSC_TOP2_DSC_TOP_TEST_DEBUG_DATA_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsccif_dispdec
+// base address: 0x2e0
+#define regDSCCIF2_DSCCIF_CONFIG0                                                                       0x30bd
+#define regDSCCIF2_DSCCIF_CONFIG0_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dscc_dispdec
+// base address: 0x2e0
+#define regDSCC2_DSCC_CONFIG0                                                                           0x30c2
+#define regDSCC2_DSCC_CONFIG0_BASE_IDX                                                                  2
+#define regDSCC2_DSCC_CONFIG1                                                                           0x30c3
+#define regDSCC2_DSCC_CONFIG1_BASE_IDX                                                                  2
+#define regDSCC2_DSCC_CONFIG2                                                                           0x30c4
+#define regDSCC2_DSCC_CONFIG2_BASE_IDX                                                                  2
+#define regDSCC2_DSCC_STATUS                                                                            0x30c5
+#define regDSCC2_DSCC_STATUS_BASE_IDX                                                                   2
+#define regDSCC2_DSCC_INTERRUPT_CONTROL0                                                                0x30c6
+#define regDSCC2_DSCC_INTERRUPT_CONTROL0_BASE_IDX                                                       2
+#define regDSCC2_DSCC_INTERRUPT_CONTROL1                                                                0x30c7
+#define regDSCC2_DSCC_INTERRUPT_CONTROL1_BASE_IDX                                                       2
+#define regDSCC2_DSCC_INTERRUPT_STATUS0                                                                 0x30c8
+#define regDSCC2_DSCC_INTERRUPT_STATUS0_BASE_IDX                                                        2
+#define regDSCC2_DSCC_INTERRUPT_STATUS1                                                                 0x30c9
+#define regDSCC2_DSCC_INTERRUPT_STATUS1_BASE_IDX                                                        2
+#define regDSCC2_DSCC_PPS_CONFIG0                                                                       0x30ca
+#define regDSCC2_DSCC_PPS_CONFIG0_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG1                                                                       0x30cb
+#define regDSCC2_DSCC_PPS_CONFIG1_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG2                                                                       0x30cc
+#define regDSCC2_DSCC_PPS_CONFIG2_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG3                                                                       0x30cd
+#define regDSCC2_DSCC_PPS_CONFIG3_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG4                                                                       0x30ce
+#define regDSCC2_DSCC_PPS_CONFIG4_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG5                                                                       0x30cf
+#define regDSCC2_DSCC_PPS_CONFIG5_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG6                                                                       0x30d0
+#define regDSCC2_DSCC_PPS_CONFIG6_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG7                                                                       0x30d1
+#define regDSCC2_DSCC_PPS_CONFIG7_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG8                                                                       0x30d2
+#define regDSCC2_DSCC_PPS_CONFIG8_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG9                                                                       0x30d3
+#define regDSCC2_DSCC_PPS_CONFIG9_BASE_IDX                                                              2
+#define regDSCC2_DSCC_PPS_CONFIG10                                                                      0x30d4
+#define regDSCC2_DSCC_PPS_CONFIG10_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG11                                                                      0x30d5
+#define regDSCC2_DSCC_PPS_CONFIG11_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG12                                                                      0x30d6
+#define regDSCC2_DSCC_PPS_CONFIG12_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG13                                                                      0x30d7
+#define regDSCC2_DSCC_PPS_CONFIG13_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG14                                                                      0x30d8
+#define regDSCC2_DSCC_PPS_CONFIG14_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG15                                                                      0x30d9
+#define regDSCC2_DSCC_PPS_CONFIG15_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG16                                                                      0x30da
+#define regDSCC2_DSCC_PPS_CONFIG16_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG17                                                                      0x30db
+#define regDSCC2_DSCC_PPS_CONFIG17_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG18                                                                      0x30dc
+#define regDSCC2_DSCC_PPS_CONFIG18_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG19                                                                      0x30dd
+#define regDSCC2_DSCC_PPS_CONFIG19_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG20                                                                      0x30de
+#define regDSCC2_DSCC_PPS_CONFIG20_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG21                                                                      0x30df
+#define regDSCC2_DSCC_PPS_CONFIG21_BASE_IDX                                                             2
+#define regDSCC2_DSCC_PPS_CONFIG22                                                                      0x30e0
+#define regDSCC2_DSCC_PPS_CONFIG22_BASE_IDX                                                             2
+#define regDSCC2_DSCC_MEM_POWER_CONTROL0                                                                0x30e1
+#define regDSCC2_DSCC_MEM_POWER_CONTROL0_BASE_IDX                                                       2
+#define regDSCC2_DSCC_MEM_POWER_CONTROL1                                                                0x30e2
+#define regDSCC2_DSCC_MEM_POWER_CONTROL1_BASE_IDX                                                       2
+#define regDSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER                                                           0x30e3
+#define regDSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX                                                  2
+#define regDSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER                                                           0x30e4
+#define regDSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX                                                  2
+#define regDSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER                                                          0x30e5
+#define regDSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER                                                          0x30e6
+#define regDSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER                                                          0x30e7
+#define regDSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER                                                          0x30e8
+#define regDSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC2_DSCC_MAX_ABS_ERROR0                                                                    0x30e9
+#define regDSCC2_DSCC_MAX_ABS_ERROR0_BASE_IDX                                                           2
+#define regDSCC2_DSCC_MAX_ABS_ERROR1                                                                    0x30ea
+#define regDSCC2_DSCC_MAX_ABS_ERROR1_BASE_IDX                                                           2
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0                                                 0x30eb
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_BASE_IDX                                        2
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1                                                 0x30ec
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_BASE_IDX                                        2
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2                                                 0x30ed
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_BASE_IDX                                        2
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3                                                 0x30ee
+#define regDSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_BASE_IDX                                        2
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0                                             0x30ef
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_BASE_IDX                                    2
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1                                             0x30f0
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_BASE_IDX                                    2
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2                                             0x30f1
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_BASE_IDX                                    2
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3                                             0x30f2
+#define regDSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_BASE_IDX                                    2
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX0                                                                 0x30f3
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX0_BASE_IDX                                                        2
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX1                                                                 0x30f4
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX1_BASE_IDX                                                        2
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX2                                                                 0x30f5
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX2_BASE_IDX                                                        2
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX3                                                                 0x30f6
+#define regDSCC2_DSCC_TEST_DEBUG_INDEX3_BASE_IDX                                                        2
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x30f7
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0                                                                  0x30f8
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA1                                                                  0x30f9
+#define regDSCC2_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA2                                                                  0x30fa
+#define regDSCC2_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA3                                                                  0x30fb
+#define regDSCC2_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
+#define regDSCC2_DSCC_DISPCLK_TEST_DEBUG_INDEX0                                                         0x30fc
+#define regDSCC2_DSCC_DISPCLK_TEST_DEBUG_INDEX0_BASE_IDX                                                2
+#define regDSCC2_DSCC_DISPCLK_TEST_DEBUG_DATA0                                                          0x30fd
+#define regDSCC2_DSCC_DISPCLK_TEST_DEBUG_DATA0_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc420
+#define regDC_PERFMON19_PERFCOUNTER_CNTL                                                                0x3108
+#define regDC_PERFMON19_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON19_PERFCOUNTER_CNTL2                                                               0x3109
+#define regDC_PERFMON19_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON19_PERFCOUNTER_STATE                                                               0x310a
+#define regDC_PERFMON19_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON19_PERFMON_CNTL                                                                    0x310b
+#define regDC_PERFMON19_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON19_PERFMON_CNTL2                                                                   0x310c
+#define regDC_PERFMON19_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON19_PERFMON_CVALUE_INT_MISC                                                         0x310d
+#define regDC_PERFMON19_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON19_PERFMON_CVALUE_LOW                                                              0x310e
+#define regDC_PERFMON19_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON19_PERFMON_HI                                                                      0x310f
+#define regDC_PERFMON19_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON19_PERFMON_LOW                                                                     0x3110
+#define regDC_PERFMON19_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_top_dispdec
+// base address: 0x450
+#define regDSC_TOP3_DSC_TOP_CONTROL                                                                     0x3114
+#define regDSC_TOP3_DSC_TOP_CONTROL_BASE_IDX                                                            2
+#define regDSC_TOP3_DSC_DEBUG_CONTROL                                                                   0x3115
+#define regDSC_TOP3_DSC_DEBUG_CONTROL_BASE_IDX                                                          2
+#define regDSC_TOP3_DSC_SPARE_DEBUG                                                                     0x3116
+#define regDSC_TOP3_DSC_SPARE_DEBUG_BASE_IDX                                                            2
+#define regDSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX                                                            0x3117
+#define regDSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX_BASE_IDX                                                   2
+#define regDSC_TOP3_DSC_TOP_TEST_DEBUG_DATA                                                             0x3118
+#define regDSC_TOP3_DSC_TOP_TEST_DEBUG_DATA_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsccif_dispdec
+// base address: 0x450
+#define regDSCCIF3_DSCCIF_CONFIG0                                                                       0x3119
+#define regDSCCIF3_DSCCIF_CONFIG0_BASE_IDX                                                              2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dscc_dispdec
+// base address: 0x450
+#define regDSCC3_DSCC_CONFIG0                                                                           0x311e
+#define regDSCC3_DSCC_CONFIG0_BASE_IDX                                                                  2
+#define regDSCC3_DSCC_CONFIG1                                                                           0x311f
+#define regDSCC3_DSCC_CONFIG1_BASE_IDX                                                                  2
+#define regDSCC3_DSCC_CONFIG2                                                                           0x3120
+#define regDSCC3_DSCC_CONFIG2_BASE_IDX                                                                  2
+#define regDSCC3_DSCC_STATUS                                                                            0x3121
+#define regDSCC3_DSCC_STATUS_BASE_IDX                                                                   2
+#define regDSCC3_DSCC_INTERRUPT_CONTROL0                                                                0x3122
+#define regDSCC3_DSCC_INTERRUPT_CONTROL0_BASE_IDX                                                       2
+#define regDSCC3_DSCC_INTERRUPT_CONTROL1                                                                0x3123
+#define regDSCC3_DSCC_INTERRUPT_CONTROL1_BASE_IDX                                                       2
+#define regDSCC3_DSCC_INTERRUPT_STATUS0                                                                 0x3124
+#define regDSCC3_DSCC_INTERRUPT_STATUS0_BASE_IDX                                                        2
+#define regDSCC3_DSCC_INTERRUPT_STATUS1                                                                 0x3125
+#define regDSCC3_DSCC_INTERRUPT_STATUS1_BASE_IDX                                                        2
+#define regDSCC3_DSCC_PPS_CONFIG0                                                                       0x3126
+#define regDSCC3_DSCC_PPS_CONFIG0_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG1                                                                       0x3127
+#define regDSCC3_DSCC_PPS_CONFIG1_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG2                                                                       0x3128
+#define regDSCC3_DSCC_PPS_CONFIG2_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG3                                                                       0x3129
+#define regDSCC3_DSCC_PPS_CONFIG3_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG4                                                                       0x312a
+#define regDSCC3_DSCC_PPS_CONFIG4_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG5                                                                       0x312b
+#define regDSCC3_DSCC_PPS_CONFIG5_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG6                                                                       0x312c
+#define regDSCC3_DSCC_PPS_CONFIG6_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG7                                                                       0x312d
+#define regDSCC3_DSCC_PPS_CONFIG7_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG8                                                                       0x312e
+#define regDSCC3_DSCC_PPS_CONFIG8_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG9                                                                       0x312f
+#define regDSCC3_DSCC_PPS_CONFIG9_BASE_IDX                                                              2
+#define regDSCC3_DSCC_PPS_CONFIG10                                                                      0x3130
+#define regDSCC3_DSCC_PPS_CONFIG10_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG11                                                                      0x3131
+#define regDSCC3_DSCC_PPS_CONFIG11_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG12                                                                      0x3132
+#define regDSCC3_DSCC_PPS_CONFIG12_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG13                                                                      0x3133
+#define regDSCC3_DSCC_PPS_CONFIG13_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG14                                                                      0x3134
+#define regDSCC3_DSCC_PPS_CONFIG14_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG15                                                                      0x3135
+#define regDSCC3_DSCC_PPS_CONFIG15_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG16                                                                      0x3136
+#define regDSCC3_DSCC_PPS_CONFIG16_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG17                                                                      0x3137
+#define regDSCC3_DSCC_PPS_CONFIG17_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG18                                                                      0x3138
+#define regDSCC3_DSCC_PPS_CONFIG18_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG19                                                                      0x3139
+#define regDSCC3_DSCC_PPS_CONFIG19_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG20                                                                      0x313a
+#define regDSCC3_DSCC_PPS_CONFIG20_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG21                                                                      0x313b
+#define regDSCC3_DSCC_PPS_CONFIG21_BASE_IDX                                                             2
+#define regDSCC3_DSCC_PPS_CONFIG22                                                                      0x313c
+#define regDSCC3_DSCC_PPS_CONFIG22_BASE_IDX                                                             2
+#define regDSCC3_DSCC_MEM_POWER_CONTROL0                                                                0x313d
+#define regDSCC3_DSCC_MEM_POWER_CONTROL0_BASE_IDX                                                       2
+#define regDSCC3_DSCC_MEM_POWER_CONTROL1                                                                0x313e
+#define regDSCC3_DSCC_MEM_POWER_CONTROL1_BASE_IDX                                                       2
+#define regDSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER                                                           0x313f
+#define regDSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX                                                  2
+#define regDSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER                                                           0x3140
+#define regDSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX                                                  2
+#define regDSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER                                                          0x3141
+#define regDSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER                                                          0x3142
+#define regDSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER                                                          0x3143
+#define regDSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX                                                 2
+#define regDSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER                                                          0x3144
+#define regDSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX                                                 2
+#define regDSCC3_DSCC_MAX_ABS_ERROR0                                                                    0x3145
+#define regDSCC3_DSCC_MAX_ABS_ERROR0_BASE_IDX                                                           2
+#define regDSCC3_DSCC_MAX_ABS_ERROR1                                                                    0x3146
+#define regDSCC3_DSCC_MAX_ABS_ERROR1_BASE_IDX                                                           2
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0                                                 0x3147
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_BASE_IDX                                        2
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1                                                 0x3148
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_BASE_IDX                                        2
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2                                                 0x3149
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_BASE_IDX                                        2
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3                                                 0x314a
+#define regDSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_BASE_IDX                                        2
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0                                             0x314b
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_BASE_IDX                                    2
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1                                             0x314c
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_BASE_IDX                                    2
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2                                             0x314d
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_BASE_IDX                                    2
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3                                             0x314e
+#define regDSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_BASE_IDX                                    2
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX0                                                                 0x314f
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX0_BASE_IDX                                                        2
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX1                                                                 0x3150
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX1_BASE_IDX                                                        2
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX2                                                                 0x3151
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX2_BASE_IDX                                                        2
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX3                                                                 0x3152
+#define regDSCC3_DSCC_TEST_DEBUG_INDEX3_BASE_IDX                                                        2
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x3153
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0                                                                  0x3154
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA1                                                                  0x3155
+#define regDSCC3_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA2                                                                  0x3156
+#define regDSCC3_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA3                                                                  0x3157
+#define regDSCC3_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
+#define regDSCC3_DSCC_DISPCLK_TEST_DEBUG_INDEX0                                                         0x3158
+#define regDSCC3_DSCC_DISPCLK_TEST_DEBUG_INDEX0_BASE_IDX                                                2
+#define regDSCC3_DSCC_DISPCLK_TEST_DEBUG_DATA0                                                          0x3159
+#define regDSCC3_DSCC_DISPCLK_TEST_DEBUG_DATA0_BASE_IDX                                                 2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc590
+#define regDC_PERFMON20_PERFCOUNTER_CNTL                                                                0x3164
+#define regDC_PERFMON20_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON20_PERFCOUNTER_CNTL2                                                               0x3165
+#define regDC_PERFMON20_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON20_PERFCOUNTER_STATE                                                               0x3166
+#define regDC_PERFMON20_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON20_PERFMON_CNTL                                                                    0x3167
+#define regDC_PERFMON20_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON20_PERFMON_CNTL2                                                                   0x3168
+#define regDC_PERFMON20_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON20_PERFMON_CVALUE_INT_MISC                                                         0x3169
+#define regDC_PERFMON20_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON20_PERFMON_CVALUE_LOW                                                              0x316a
+#define regDC_PERFMON20_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON20_PERFMON_HI                                                                      0x316b
+#define regDC_PERFMON20_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON20_PERFMON_LOW                                                                     0x316c
+#define regDC_PERFMON20_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_wb0_dispdec_dwb_top_dispdec
+// base address: 0x0
+#define regDWB_ENABLE_CLK_CTRL                                                                          0x3228
+#define regDWB_ENABLE_CLK_CTRL_BASE_IDX                                                                 2
+#define regDWB_MEM_PWR_CTRL                                                                             0x3229
+#define regDWB_MEM_PWR_CTRL_BASE_IDX                                                                    2
+#define regFC_MODE_CTRL                                                                                 0x322a
+#define regFC_MODE_CTRL_BASE_IDX                                                                        2
+#define regFC_FLOW_CTRL                                                                                 0x322b
+#define regFC_FLOW_CTRL_BASE_IDX                                                                        2
+#define regFC_WINDOW_START                                                                              0x322c
+#define regFC_WINDOW_START_BASE_IDX                                                                     2
+#define regFC_WINDOW_SIZE                                                                               0x322d
+#define regFC_WINDOW_SIZE_BASE_IDX                                                                      2
+#define regFC_SOURCE_SIZE                                                                               0x322e
+#define regFC_SOURCE_SIZE_BASE_IDX                                                                      2
+#define regDWB_UPDATE_CTRL                                                                              0x322f
+#define regDWB_UPDATE_CTRL_BASE_IDX                                                                     2
+#define regDWB_CRC_CTRL                                                                                 0x3230
+#define regDWB_CRC_CTRL_BASE_IDX                                                                        2
+#define regDWB_CRC_MASK_R_G                                                                             0x3231
+#define regDWB_CRC_MASK_R_G_BASE_IDX                                                                    2
+#define regDWB_CRC_MASK_B_A                                                                             0x3232
+#define regDWB_CRC_MASK_B_A_BASE_IDX                                                                    2
+#define regDWB_CRC_VAL_R_G                                                                              0x3233
+#define regDWB_CRC_VAL_R_G_BASE_IDX                                                                     2
+#define regDWB_CRC_VAL_B_A                                                                              0x3234
+#define regDWB_CRC_VAL_B_A_BASE_IDX                                                                     2
+#define regDWB_OUT_CTRL                                                                                 0x3235
+#define regDWB_OUT_CTRL_BASE_IDX                                                                        2
+#define regDWB_MMHUBBUB_BACKPRESSURE_CNT_EN                                                             0x3236
+#define regDWB_MMHUBBUB_BACKPRESSURE_CNT_EN_BASE_IDX                                                    2
+#define regDWB_MMHUBBUB_BACKPRESSURE_CNT                                                                0x3237
+#define regDWB_MMHUBBUB_BACKPRESSURE_CNT_BASE_IDX                                                       2
+#define regDWB_HOST_READ_CONTROL                                                                        0x3238
+#define regDWB_HOST_READ_CONTROL_BASE_IDX                                                               2
+#define regDWB_OVERFLOW_STATUS                                                                          0x3239
+#define regDWB_OVERFLOW_STATUS_BASE_IDX                                                                 2
+#define regDWB_OVERFLOW_COUNTER                                                                         0x323a
+#define regDWB_OVERFLOW_COUNTER_BASE_IDX                                                                2
+#define regDWB_SOFT_RESET                                                                               0x323b
+#define regDWB_SOFT_RESET_BASE_IDX                                                                      2
+#define regDWB_DEBUG_CTRL                                                                               0x323c
+#define regDWB_DEBUG_CTRL_BASE_IDX                                                                      2
+#define regDWB_DEBUG                                                                                    0x323d
+#define regDWB_DEBUG_BASE_IDX                                                                           2
+
+
+// addressBlock: dce_dc_wb0_dispdec_wb_dcperfmon_dc_perfmon_dispdec
+// base address: 0xca20
+#define regDC_PERFMON21_PERFCOUNTER_CNTL                                                                0x3288
+#define regDC_PERFMON21_PERFCOUNTER_CNTL_BASE_IDX                                                       2
+#define regDC_PERFMON21_PERFCOUNTER_CNTL2                                                               0x3289
+#define regDC_PERFMON21_PERFCOUNTER_CNTL2_BASE_IDX                                                      2
+#define regDC_PERFMON21_PERFCOUNTER_STATE                                                               0x328a
+#define regDC_PERFMON21_PERFCOUNTER_STATE_BASE_IDX                                                      2
+#define regDC_PERFMON21_PERFMON_CNTL                                                                    0x328b
+#define regDC_PERFMON21_PERFMON_CNTL_BASE_IDX                                                           2
+#define regDC_PERFMON21_PERFMON_CNTL2                                                                   0x328c
+#define regDC_PERFMON21_PERFMON_CNTL2_BASE_IDX                                                          2
+#define regDC_PERFMON21_PERFMON_CVALUE_INT_MISC                                                         0x328d
+#define regDC_PERFMON21_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                2
+#define regDC_PERFMON21_PERFMON_CVALUE_LOW                                                              0x328e
+#define regDC_PERFMON21_PERFMON_CVALUE_LOW_BASE_IDX                                                     2
+#define regDC_PERFMON21_PERFMON_HI                                                                      0x328f
+#define regDC_PERFMON21_PERFMON_HI_BASE_IDX                                                             2
+#define regDC_PERFMON21_PERFMON_LOW                                                                     0x3290
+#define regDC_PERFMON21_PERFMON_LOW_BASE_IDX                                                            2
+
+
+// addressBlock: dce_dc_wb0_dispdec_dwbcp_dispdec
+// base address: 0x0
+#define regDWB_HDR_MULT_COEF                                                                            0x3294
+#define regDWB_HDR_MULT_COEF_BASE_IDX                                                                   2
+#define regDWB_GAMUT_REMAP_MODE                                                                         0x3295
+#define regDWB_GAMUT_REMAP_MODE_BASE_IDX                                                                2
+#define regDWB_GAMUT_REMAP_COEF_FORMAT                                                                  0x3296
+#define regDWB_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                                         2
+#define regDWB_GAMUT_REMAPA_C11_C12                                                                     0x3297
+#define regDWB_GAMUT_REMAPA_C11_C12_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPA_C13_C14                                                                     0x3298
+#define regDWB_GAMUT_REMAPA_C13_C14_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPA_C21_C22                                                                     0x3299
+#define regDWB_GAMUT_REMAPA_C21_C22_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPA_C23_C24                                                                     0x329a
+#define regDWB_GAMUT_REMAPA_C23_C24_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPA_C31_C32                                                                     0x329b
+#define regDWB_GAMUT_REMAPA_C31_C32_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPA_C33_C34                                                                     0x329c
+#define regDWB_GAMUT_REMAPA_C33_C34_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C11_C12                                                                     0x329d
+#define regDWB_GAMUT_REMAPB_C11_C12_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C13_C14                                                                     0x329e
+#define regDWB_GAMUT_REMAPB_C13_C14_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C21_C22                                                                     0x329f
+#define regDWB_GAMUT_REMAPB_C21_C22_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C23_C24                                                                     0x32a0
+#define regDWB_GAMUT_REMAPB_C23_C24_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C31_C32                                                                     0x32a1
+#define regDWB_GAMUT_REMAPB_C31_C32_BASE_IDX                                                            2
+#define regDWB_GAMUT_REMAPB_C33_C34                                                                     0x32a2
+#define regDWB_GAMUT_REMAPB_C33_C34_BASE_IDX                                                            2
+#define regDWB_OGAM_CONTROL                                                                             0x32a3
+#define regDWB_OGAM_CONTROL_BASE_IDX                                                                    2
+#define regDWB_OGAM_LUT_INDEX                                                                           0x32a4
+#define regDWB_OGAM_LUT_INDEX_BASE_IDX                                                                  2
+#define regDWB_OGAM_LUT_DATA                                                                            0x32a5
+#define regDWB_OGAM_LUT_DATA_BASE_IDX                                                                   2
+#define regDWB_OGAM_LUT_CONTROL                                                                         0x32a6
+#define regDWB_OGAM_LUT_CONTROL_BASE_IDX                                                                2
+#define regDWB_OGAM_RAMA_START_CNTL_B                                                                   0x32a7
+#define regDWB_OGAM_RAMA_START_CNTL_B_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_START_CNTL_G                                                                   0x32a8
+#define regDWB_OGAM_RAMA_START_CNTL_G_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_START_CNTL_R                                                                   0x32a9
+#define regDWB_OGAM_RAMA_START_CNTL_R_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_B                                                              0x32aa
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_B_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_B                                                             0x32ab
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_G                                                              0x32ac
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_G_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_G                                                             0x32ad
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_R                                                              0x32ae
+#define regDWB_OGAM_RAMA_START_BASE_CNTL_R_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_R                                                             0x32af
+#define regDWB_OGAM_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMA_END_CNTL1_B                                                                    0x32b0
+#define regDWB_OGAM_RAMA_END_CNTL1_B_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_END_CNTL2_B                                                                    0x32b1
+#define regDWB_OGAM_RAMA_END_CNTL2_B_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_END_CNTL1_G                                                                    0x32b2
+#define regDWB_OGAM_RAMA_END_CNTL1_G_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_END_CNTL2_G                                                                    0x32b3
+#define regDWB_OGAM_RAMA_END_CNTL2_G_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_END_CNTL1_R                                                                    0x32b4
+#define regDWB_OGAM_RAMA_END_CNTL1_R_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_END_CNTL2_R                                                                    0x32b5
+#define regDWB_OGAM_RAMA_END_CNTL2_R_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMA_OFFSET_B                                                                       0x32b6
+#define regDWB_OGAM_RAMA_OFFSET_B_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMA_OFFSET_G                                                                       0x32b7
+#define regDWB_OGAM_RAMA_OFFSET_G_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMA_OFFSET_R                                                                       0x32b8
+#define regDWB_OGAM_RAMA_OFFSET_R_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMA_REGION_0_1                                                                     0x32b9
+#define regDWB_OGAM_RAMA_REGION_0_1_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMA_REGION_2_3                                                                     0x32ba
+#define regDWB_OGAM_RAMA_REGION_2_3_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMA_REGION_4_5                                                                     0x32bb
+#define regDWB_OGAM_RAMA_REGION_4_5_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMA_REGION_6_7                                                                     0x32bc
+#define regDWB_OGAM_RAMA_REGION_6_7_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMA_REGION_8_9                                                                     0x32bd
+#define regDWB_OGAM_RAMA_REGION_8_9_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMA_REGION_10_11                                                                   0x32be
+#define regDWB_OGAM_RAMA_REGION_10_11_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_12_13                                                                   0x32bf
+#define regDWB_OGAM_RAMA_REGION_12_13_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_14_15                                                                   0x32c0
+#define regDWB_OGAM_RAMA_REGION_14_15_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_16_17                                                                   0x32c1
+#define regDWB_OGAM_RAMA_REGION_16_17_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_18_19                                                                   0x32c2
+#define regDWB_OGAM_RAMA_REGION_18_19_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_20_21                                                                   0x32c3
+#define regDWB_OGAM_RAMA_REGION_20_21_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_22_23                                                                   0x32c4
+#define regDWB_OGAM_RAMA_REGION_22_23_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_24_25                                                                   0x32c5
+#define regDWB_OGAM_RAMA_REGION_24_25_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_26_27                                                                   0x32c6
+#define regDWB_OGAM_RAMA_REGION_26_27_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_28_29                                                                   0x32c7
+#define regDWB_OGAM_RAMA_REGION_28_29_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_30_31                                                                   0x32c8
+#define regDWB_OGAM_RAMA_REGION_30_31_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMA_REGION_32_33                                                                   0x32c9
+#define regDWB_OGAM_RAMA_REGION_32_33_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_START_CNTL_B                                                                   0x32ca
+#define regDWB_OGAM_RAMB_START_CNTL_B_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_START_CNTL_G                                                                   0x32cb
+#define regDWB_OGAM_RAMB_START_CNTL_G_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_START_CNTL_R                                                                   0x32cc
+#define regDWB_OGAM_RAMB_START_CNTL_R_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_B                                                              0x32cd
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_B_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_B                                                             0x32ce
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_G                                                              0x32cf
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_G_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_G                                                             0x32d0
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_R                                                              0x32d1
+#define regDWB_OGAM_RAMB_START_BASE_CNTL_R_BASE_IDX                                                     2
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_R                                                             0x32d2
+#define regDWB_OGAM_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                                    2
+#define regDWB_OGAM_RAMB_END_CNTL1_B                                                                    0x32d3
+#define regDWB_OGAM_RAMB_END_CNTL1_B_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_END_CNTL2_B                                                                    0x32d4
+#define regDWB_OGAM_RAMB_END_CNTL2_B_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_END_CNTL1_G                                                                    0x32d5
+#define regDWB_OGAM_RAMB_END_CNTL1_G_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_END_CNTL2_G                                                                    0x32d6
+#define regDWB_OGAM_RAMB_END_CNTL2_G_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_END_CNTL1_R                                                                    0x32d7
+#define regDWB_OGAM_RAMB_END_CNTL1_R_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_END_CNTL2_R                                                                    0x32d8
+#define regDWB_OGAM_RAMB_END_CNTL2_R_BASE_IDX                                                           2
+#define regDWB_OGAM_RAMB_OFFSET_B                                                                       0x32d9
+#define regDWB_OGAM_RAMB_OFFSET_B_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMB_OFFSET_G                                                                       0x32da
+#define regDWB_OGAM_RAMB_OFFSET_G_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMB_OFFSET_R                                                                       0x32db
+#define regDWB_OGAM_RAMB_OFFSET_R_BASE_IDX                                                              2
+#define regDWB_OGAM_RAMB_REGION_0_1                                                                     0x32dc
+#define regDWB_OGAM_RAMB_REGION_0_1_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMB_REGION_2_3                                                                     0x32dd
+#define regDWB_OGAM_RAMB_REGION_2_3_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMB_REGION_4_5                                                                     0x32de
+#define regDWB_OGAM_RAMB_REGION_4_5_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMB_REGION_6_7                                                                     0x32df
+#define regDWB_OGAM_RAMB_REGION_6_7_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMB_REGION_8_9                                                                     0x32e0
+#define regDWB_OGAM_RAMB_REGION_8_9_BASE_IDX                                                            2
+#define regDWB_OGAM_RAMB_REGION_10_11                                                                   0x32e1
+#define regDWB_OGAM_RAMB_REGION_10_11_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_12_13                                                                   0x32e2
+#define regDWB_OGAM_RAMB_REGION_12_13_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_14_15                                                                   0x32e3
+#define regDWB_OGAM_RAMB_REGION_14_15_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_16_17                                                                   0x32e4
+#define regDWB_OGAM_RAMB_REGION_16_17_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_18_19                                                                   0x32e5
+#define regDWB_OGAM_RAMB_REGION_18_19_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_20_21                                                                   0x32e6
+#define regDWB_OGAM_RAMB_REGION_20_21_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_22_23                                                                   0x32e7
+#define regDWB_OGAM_RAMB_REGION_22_23_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_24_25                                                                   0x32e8
+#define regDWB_OGAM_RAMB_REGION_24_25_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_26_27                                                                   0x32e9
+#define regDWB_OGAM_RAMB_REGION_26_27_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_28_29                                                                   0x32ea
+#define regDWB_OGAM_RAMB_REGION_28_29_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_30_31                                                                   0x32eb
+#define regDWB_OGAM_RAMB_REGION_30_31_BASE_IDX                                                          2
+#define regDWB_OGAM_RAMB_REGION_32_33                                                                   0x32ec
+#define regDWB_OGAM_RAMB_REGION_32_33_BASE_IDX                                                          2
+
+
+// addressBlock: dce_dc_dchvm_hvm_dispdec
+// base address: 0x0
+#define regDCHVM_CTRL0                                                                                  0x3603
+#define regDCHVM_CTRL0_BASE_IDX                                                                         2
+#define regDCHVM_CTRL1                                                                                  0x3604
+#define regDCHVM_CTRL1_BASE_IDX                                                                         2
+#define regDCHVM_CLK_CTRL                                                                               0x3605
+#define regDCHVM_CLK_CTRL_BASE_IDX                                                                      2
+#define regDCHVM_MEM_CTRL                                                                               0x3606
+#define regDCHVM_MEM_CTRL_BASE_IDX                                                                      2
+#define regDCHVM_RIOMMU_CTRL0                                                                           0x3607
+#define regDCHVM_RIOMMU_CTRL0_BASE_IDX                                                                  2
+#define regDCHVM_RIOMMU_STAT0                                                                           0x3608
+#define regDCHVM_RIOMMU_STAT0_BASE_IDX                                                                  2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
+// base address: 0x1ab8c
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x3623
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_INPUT_MUX_CONTROL                                               0x3624
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                      2
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL                                                   0x3625
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                        0x3626
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX               2
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                        0x3627
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX               2
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_SPARE                                                           0x3628
+#define regDP_STREAM_ENC0_DP_STREAM_ENC_SPARE_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_apg_apg_dispdec
+// base address: 0x1abc0
+#define regAPG5_APG_CONTROL                                                                             0x3630
+#define regAPG5_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG5_APG_CONTROL2                                                                            0x3631
+#define regAPG5_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG5_APG_DBG_GEN_CONTROL                                                                     0x3632
+#define regAPG5_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG5_APG_MEM_PWR                                                                             0x3644
+#define regAPG5_APG_MEM_PWR_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_dme_dme_dispdec
+// base address: 0x1ac38
+#define regDME5_DME_CONTROL                                                                             0x364e
+#define regDME5_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME5_DME_MEMORY_CONTROL                                                                      0x364f
+#define regDME5_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_vpg_vpg_dispdec
+// base address: 0x1ac44
+#define regVPG5_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x3651
+#define regVPG5_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG5_VPG_GENERIC_PACKET_DATA                                                                 0x3652
+#define regVPG5_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG5_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x3653
+#define regVPG5_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x3654
+#define regVPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG5_VPG_GENERIC_STATUS                                                                      0x3655
+#define regVPG5_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG5_VPG_MEM_PWR                                                                             0x3656
+#define regVPG5_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG5_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x3657
+#define regVPG5_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG5_VPG_ISRC1_2_DATA                                                                        0x3658
+#define regVPG5_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc0_dispdec
+// base address: 0x1ac74
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_CONTROL                                                           0x365d
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_CONTROL_BASE_IDX                                                  2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL                                                  0x365e
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL                                     0x365f
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL_BASE_IDX                            2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL                            0x3660
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL_BASE_IDX                   2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT                                                  0x3661
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA0                                                          0x3662
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA0_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA1                                                          0x3663
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA1_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA2                                                          0x3664
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA2_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA3                                                          0x3665
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA3_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA4                                                          0x3666
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA4_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA5                                                          0x3667
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA5_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA6                                                          0x3668
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA6_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA7                                                          0x3669
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA7_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA8                                                          0x366a
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA8_BASE_IDX                                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x366b
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0                                                  0x366c
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1                                                  0x366d
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2                                                  0x366e
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3                                                  0x366f
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4                                                  0x3670
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5                                                  0x3671
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6                                                  0x3672
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7                                                  0x3673
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8                                                  0x3674
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9                                                  0x3675
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10                                                 0x3676
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11                                                 0x3677
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12                                                 0x3678
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13                                                 0x3679
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14                                                 0x367a
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL                                                       0x367b
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL_BASE_IDX                                              2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0                                                0x367c
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1                                                0x367d
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL                                       0x367e
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL                                                   0x3683
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL                                                  0x3684
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL                                                0x3685
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL                                          0x3686
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                 2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x3687
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT0                                                   0x3688
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT0_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT1                                                   0x3689
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT1_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT2                                                   0x368a
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT2_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT3                                                   0x368b
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT3_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS                                                    0x368c
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS_BASE_IDX                                           2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_STATUS                                               0x368d
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_STATUS_BASE_IDX                                      2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL                                              0x368e
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL_BASE_IDX                                     2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL                                                0x368f
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL                                                 0x3690
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET                                               0x3691
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET_BASE_IDX                                      2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_READY_CONTROL                                                0x3692
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_READY_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL                                        0x3693
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL_BASE_IDX                               2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS                                                       0x3694
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS_BASE_IDX                                              2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_START_CONTROL                                                0x3695
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_START_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL                                       0x3696
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS                                        0x3697
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS_BASE_IDX                               2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL                                                 0x3698
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SPARE                                                             0x3699
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_SPARE_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc0_dispdec
+// base address: 0x1ad7c
+#define regDP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL                                                       0x369f
+#define regDP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL_BASE_IDX                                              2
+#define regDP_LINK_ENC0_DP_LINK_ENC_SPARE                                                               0x36a0
+#define regDP_LINK_ENC0_DP_LINK_ENC_SPARE_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym320_dispdec
+// base address: 0x1add0
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL                                                         0x36b4
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL_BASE_IDX                                                2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_STATUS                                                          0x36b5
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_STATUS_BASE_IDX                                                 2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_UPDATE                                                      0x36b8
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_UPDATE_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0                                                   0x36b9
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1                                                   0x36ba
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2                                                   0x36bb
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3                                                   0x36bc
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0                                                         0x36bf
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0_BASE_IDX                                                2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1                                                         0x36c0
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1_BASE_IDX                                                2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2                                                         0x36c1
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2_BASE_IDX                                                2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3                                                         0x36c2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3_BASE_IDX                                                2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0                                                  0x36c5
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0_BASE_IDX                                         2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1                                                  0x36c6
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1_BASE_IDX                                         2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2                                                  0x36c7
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2_BASE_IDX                                         2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3                                                  0x36c8
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3_BASE_IDX                                         2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0                                                     0x36cb
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0_BASE_IDX                                            2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR0                                                       0x36cc
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR0_BASE_IDX                                              2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR1                                                       0x36cd
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR1_BASE_IDX                                              2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR2                                                       0x36ce
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR2_BASE_IDX                                              2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR3                                                       0x36cf
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR3_BASE_IDX                                              2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0                                              0x36d0
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0_BASE_IDX                                     2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0                                               0x36d1
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0_BASE_IDX                                      2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL                                                    0x36d3
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL_BASE_IDX                                           2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG                                                       0x36d4
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG_BASE_IDX                                              2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED0                                                   0x36d5
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED0_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED1                                                   0x36d6
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED1_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED2                                                   0x36d7
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED2_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED3                                                   0x36d8
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED3_BASE_IDX                                          2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE                                                     0x36d9
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE_BASE_IDX                                            2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM0                                                      0x36da
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM0_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM1                                                      0x36db
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM1_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM2                                                      0x36dc
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM2_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM3                                                      0x36dd
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM3_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM4                                                      0x36de
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM4_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM5                                                      0x36df
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM5_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM6                                                      0x36e0
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM6_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM7                                                      0x36e1
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM7_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM8                                                      0x36e2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM8_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM9                                                      0x36e3
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM9_BASE_IDX                                             2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM10                                                     0x36e4
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM10_BASE_IDX                                            2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS                                                    0x36e5
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS_BASE_IDX                                           2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0                                            0x36e8
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0_BASE_IDX                                   2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1                                            0x36e9
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1_BASE_IDX                                   2
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL                                            0x36ea
+#define regDP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL_BASE_IDX                                   2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_dispdec
+// base address: 0x1aedc
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x36f7
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_INPUT_MUX_CONTROL                                               0x36f8
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                      2
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL                                                   0x36f9
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                        0x36fa
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX               2
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                        0x36fb
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX               2
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_SPARE                                                           0x36fc
+#define regDP_STREAM_ENC1_DP_STREAM_ENC_SPARE_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_apg_apg_dispdec
+// base address: 0x1af10
+#define regAPG6_APG_CONTROL                                                                             0x3704
+#define regAPG6_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG6_APG_CONTROL2                                                                            0x3705
+#define regAPG6_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG6_APG_DBG_GEN_CONTROL                                                                     0x3706
+#define regAPG6_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG6_APG_MEM_PWR                                                                             0x3718
+#define regAPG6_APG_MEM_PWR_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_dme_dme_dispdec
+// base address: 0x1af88
+#define regDME6_DME_CONTROL                                                                             0x3722
+#define regDME6_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME6_DME_MEMORY_CONTROL                                                                      0x3723
+#define regDME6_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_vpg_vpg_dispdec
+// base address: 0x1af94
+#define regVPG6_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x3725
+#define regVPG6_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG6_VPG_GENERIC_PACKET_DATA                                                                 0x3726
+#define regVPG6_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG6_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x3727
+#define regVPG6_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x3728
+#define regVPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG6_VPG_GENERIC_STATUS                                                                      0x3729
+#define regVPG6_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG6_VPG_MEM_PWR                                                                             0x372a
+#define regVPG6_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG6_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x372b
+#define regVPG6_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG6_VPG_ISRC1_2_DATA                                                                        0x372c
+#define regVPG6_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc1_dispdec
+// base address: 0x1afc4
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_CONTROL                                                           0x3731
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_CONTROL_BASE_IDX                                                  2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL                                                  0x3732
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL                                     0x3733
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL_BASE_IDX                            2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL                            0x3734
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL_BASE_IDX                   2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT                                                  0x3735
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA0                                                          0x3736
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA0_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA1                                                          0x3737
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA1_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA2                                                          0x3738
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA2_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA3                                                          0x3739
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA3_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA4                                                          0x373a
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA4_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA5                                                          0x373b
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA5_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA6                                                          0x373c
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA6_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA7                                                          0x373d
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA7_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA8                                                          0x373e
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA8_BASE_IDX                                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x373f
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0                                                  0x3740
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1                                                  0x3741
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2                                                  0x3742
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3                                                  0x3743
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4                                                  0x3744
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5                                                  0x3745
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6                                                  0x3746
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7                                                  0x3747
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8                                                  0x3748
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9                                                  0x3749
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10                                                 0x374a
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11                                                 0x374b
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12                                                 0x374c
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13                                                 0x374d
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14                                                 0x374e
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL                                                       0x374f
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL_BASE_IDX                                              2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0                                                0x3750
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1                                                0x3751
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL                                       0x3752
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL                                                   0x3757
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL                                                  0x3758
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL                                                0x3759
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL                                          0x375a
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                 2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x375b
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT0                                                   0x375c
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT0_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT1                                                   0x375d
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT1_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT2                                                   0x375e
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT2_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT3                                                   0x375f
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT3_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS                                                    0x3760
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS_BASE_IDX                                           2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_STATUS                                               0x3761
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_STATUS_BASE_IDX                                      2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL                                              0x3762
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL_BASE_IDX                                     2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL                                                0x3763
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL                                                 0x3764
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET                                               0x3765
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET_BASE_IDX                                      2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_READY_CONTROL                                                0x3766
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_READY_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL                                        0x3767
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL_BASE_IDX                               2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS                                                       0x3768
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS_BASE_IDX                                              2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_START_CONTROL                                                0x3769
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_START_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL                                       0x376a
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS                                        0x376b
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS_BASE_IDX                               2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL                                                 0x376c
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SPARE                                                             0x376d
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_SPARE_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc1_dispdec
+// base address: 0x1b0cc
+#define regDP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL                                                       0x3773
+#define regDP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL_BASE_IDX                                              2
+#define regDP_LINK_ENC1_DP_LINK_ENC_SPARE                                                               0x3774
+#define regDP_LINK_ENC1_DP_LINK_ENC_SPARE_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym321_dispdec
+// base address: 0x1b120
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL                                                         0x3788
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL_BASE_IDX                                                2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_STATUS                                                          0x3789
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_STATUS_BASE_IDX                                                 2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_UPDATE                                                      0x378c
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_UPDATE_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0                                                   0x378d
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1                                                   0x378e
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2                                                   0x378f
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3                                                   0x3790
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0                                                         0x3793
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0_BASE_IDX                                                2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1                                                         0x3794
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1_BASE_IDX                                                2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2                                                         0x3795
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2_BASE_IDX                                                2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3                                                         0x3796
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3_BASE_IDX                                                2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0                                                  0x3799
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0_BASE_IDX                                         2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1                                                  0x379a
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1_BASE_IDX                                         2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2                                                  0x379b
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2_BASE_IDX                                         2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3                                                  0x379c
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3_BASE_IDX                                         2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0                                                     0x379f
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0_BASE_IDX                                            2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR0                                                       0x37a0
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR0_BASE_IDX                                              2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR1                                                       0x37a1
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR1_BASE_IDX                                              2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR2                                                       0x37a2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR2_BASE_IDX                                              2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR3                                                       0x37a3
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR3_BASE_IDX                                              2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0                                              0x37a4
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0_BASE_IDX                                     2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0                                               0x37a5
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0_BASE_IDX                                      2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL                                                    0x37a7
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL_BASE_IDX                                           2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG                                                       0x37a8
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG_BASE_IDX                                              2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED0                                                   0x37a9
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED0_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED1                                                   0x37aa
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED1_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED2                                                   0x37ab
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED2_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED3                                                   0x37ac
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED3_BASE_IDX                                          2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_SQ_PULSE                                                     0x37ad
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_SQ_PULSE_BASE_IDX                                            2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM0                                                      0x37ae
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM0_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM1                                                      0x37af
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM1_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM2                                                      0x37b0
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM2_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM3                                                      0x37b1
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM3_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM4                                                      0x37b2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM4_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM5                                                      0x37b3
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM5_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM6                                                      0x37b4
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM6_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM7                                                      0x37b5
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM7_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM8                                                      0x37b6
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM8_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM9                                                      0x37b7
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM9_BASE_IDX                                             2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM10                                                     0x37b8
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM10_BASE_IDX                                            2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS                                                    0x37b9
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS_BASE_IDX                                           2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0                                            0x37bc
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0_BASE_IDX                                   2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1                                            0x37bd
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1_BASE_IDX                                   2
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL                                            0x37be
+#define regDP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL_BASE_IDX                                   2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_dispdec
+// base address: 0x1b22c
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x37cb
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_INPUT_MUX_CONTROL                                               0x37cc
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                      2
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL                                                   0x37cd
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                        0x37ce
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX               2
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                        0x37cf
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX               2
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_SPARE                                                           0x37d0
+#define regDP_STREAM_ENC2_DP_STREAM_ENC_SPARE_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_apg_apg_dispdec
+// base address: 0x1b260
+#define regAPG7_APG_CONTROL                                                                             0x37d8
+#define regAPG7_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG7_APG_CONTROL2                                                                            0x37d9
+#define regAPG7_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG7_APG_DBG_GEN_CONTROL                                                                     0x37da
+#define regAPG7_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG7_APG_MEM_PWR                                                                             0x37ec
+#define regAPG7_APG_MEM_PWR_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_dme_dme_dispdec
+// base address: 0x1b2d8
+#define regDME7_DME_CONTROL                                                                             0x37f6
+#define regDME7_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME7_DME_MEMORY_CONTROL                                                                      0x37f7
+#define regDME7_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_vpg_vpg_dispdec
+// base address: 0x1b2e4
+#define regVPG7_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x37f9
+#define regVPG7_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG7_VPG_GENERIC_PACKET_DATA                                                                 0x37fa
+#define regVPG7_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG7_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x37fb
+#define regVPG7_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x37fc
+#define regVPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG7_VPG_GENERIC_STATUS                                                                      0x37fd
+#define regVPG7_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG7_VPG_MEM_PWR                                                                             0x37fe
+#define regVPG7_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG7_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x37ff
+#define regVPG7_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG7_VPG_ISRC1_2_DATA                                                                        0x3800
+#define regVPG7_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc2_dispdec
+// base address: 0x1b314
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_CONTROL                                                           0x3805
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_CONTROL_BASE_IDX                                                  2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL                                                  0x3806
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL                                     0x3807
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL_BASE_IDX                            2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL                            0x3808
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL_BASE_IDX                   2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT                                                  0x3809
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA0                                                          0x380a
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA0_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA1                                                          0x380b
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA1_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA2                                                          0x380c
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA2_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA3                                                          0x380d
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA3_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA4                                                          0x380e
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA4_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA5                                                          0x380f
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA5_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA6                                                          0x3810
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA6_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA7                                                          0x3811
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA7_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA8                                                          0x3812
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA8_BASE_IDX                                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x3813
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0                                                  0x3814
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1                                                  0x3815
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2                                                  0x3816
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3                                                  0x3817
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4                                                  0x3818
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5                                                  0x3819
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6                                                  0x381a
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7                                                  0x381b
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8                                                  0x381c
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9                                                  0x381d
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10                                                 0x381e
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11                                                 0x381f
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12                                                 0x3820
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13                                                 0x3821
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14                                                 0x3822
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL                                                       0x3823
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL_BASE_IDX                                              2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0                                                0x3824
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1                                                0x3825
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL                                       0x3826
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL                                                   0x382b
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL                                                  0x382c
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL                                                0x382d
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL                                          0x382e
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                 2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x382f
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT0                                                   0x3830
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT0_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT1                                                   0x3831
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT1_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT2                                                   0x3832
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT2_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT3                                                   0x3833
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT3_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS                                                    0x3834
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS_BASE_IDX                                           2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_STATUS                                               0x3835
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_STATUS_BASE_IDX                                      2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL                                              0x3836
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL_BASE_IDX                                     2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL                                                0x3837
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL                                                 0x3838
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET                                               0x3839
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET_BASE_IDX                                      2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_READY_CONTROL                                                0x383a
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_READY_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL                                        0x383b
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL_BASE_IDX                               2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS                                                       0x383c
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS_BASE_IDX                                              2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_START_CONTROL                                                0x383d
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_START_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL                                       0x383e
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS                                        0x383f
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS_BASE_IDX                               2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL                                                 0x3840
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SPARE                                                             0x3841
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_SPARE_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc2_dispdec
+// base address: 0x1b41c
+#define regDP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL                                                       0x3847
+#define regDP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL_BASE_IDX                                              2
+#define regDP_LINK_ENC2_DP_LINK_ENC_SPARE                                                               0x3848
+#define regDP_LINK_ENC2_DP_LINK_ENC_SPARE_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym322_dispdec
+// base address: 0x1b470
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL                                                         0x385c
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL_BASE_IDX                                                2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_STATUS                                                          0x385d
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_STATUS_BASE_IDX                                                 2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_UPDATE                                                      0x3860
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_UPDATE_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0                                                   0x3861
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1                                                   0x3862
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2                                                   0x3863
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3                                                   0x3864
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0                                                         0x3867
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0_BASE_IDX                                                2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1                                                         0x3868
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1_BASE_IDX                                                2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2                                                         0x3869
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2_BASE_IDX                                                2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3                                                         0x386a
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3_BASE_IDX                                                2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0                                                  0x386d
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0_BASE_IDX                                         2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1                                                  0x386e
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1_BASE_IDX                                         2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2                                                  0x386f
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2_BASE_IDX                                         2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3                                                  0x3870
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3_BASE_IDX                                         2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0                                                     0x3873
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0_BASE_IDX                                            2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR0                                                       0x3874
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR0_BASE_IDX                                              2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR1                                                       0x3875
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR1_BASE_IDX                                              2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR2                                                       0x3876
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR2_BASE_IDX                                              2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR3                                                       0x3877
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR3_BASE_IDX                                              2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0                                              0x3878
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0_BASE_IDX                                     2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0                                               0x3879
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0_BASE_IDX                                      2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL                                                    0x387b
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL_BASE_IDX                                           2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG                                                       0x387c
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG_BASE_IDX                                              2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED0                                                   0x387d
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED0_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED1                                                   0x387e
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED1_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED2                                                   0x387f
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED2_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED3                                                   0x3880
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED3_BASE_IDX                                          2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_SQ_PULSE                                                     0x3881
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_SQ_PULSE_BASE_IDX                                            2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM0                                                      0x3882
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM0_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM1                                                      0x3883
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM1_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM2                                                      0x3884
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM2_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM3                                                      0x3885
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM3_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM4                                                      0x3886
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM4_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM5                                                      0x3887
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM5_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM6                                                      0x3888
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM6_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM7                                                      0x3889
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM7_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM8                                                      0x388a
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM8_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM9                                                      0x388b
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM9_BASE_IDX                                             2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM10                                                     0x388c
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM10_BASE_IDX                                            2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS                                                    0x388d
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS_BASE_IDX                                           2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0                                            0x3890
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0_BASE_IDX                                   2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1                                            0x3891
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1_BASE_IDX                                   2
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL                                            0x3892
+#define regDP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL_BASE_IDX                                   2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_dispdec
+// base address: 0x1b57c
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x389f
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_INPUT_MUX_CONTROL                                               0x38a0
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                      2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL                                                   0x38a1
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL_BASE_IDX                                          2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                        0x38a2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX               2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                        0x38a3
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX               2
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_SPARE                                                           0x38a4
+#define regDP_STREAM_ENC3_DP_STREAM_ENC_SPARE_BASE_IDX                                                  2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_apg_apg_dispdec
+// base address: 0x1b5b0
+#define regAPG8_APG_CONTROL                                                                             0x38ac
+#define regAPG8_APG_CONTROL_BASE_IDX                                                                    2
+#define regAPG8_APG_CONTROL2                                                                            0x38ad
+#define regAPG8_APG_CONTROL2_BASE_IDX                                                                   2
+#define regAPG8_APG_DBG_GEN_CONTROL                                                                     0x38ae
+#define regAPG8_APG_DBG_GEN_CONTROL_BASE_IDX                                                            2
+#define regAPG8_APG_MEM_PWR                                                                             0x38c0
+#define regAPG8_APG_MEM_PWR_BASE_IDX                                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_dme_dme_dispdec
+// base address: 0x1b628
+#define regDME8_DME_CONTROL                                                                             0x38ca
+#define regDME8_DME_CONTROL_BASE_IDX                                                                    2
+#define regDME8_DME_MEMORY_CONTROL                                                                      0x38cb
+#define regDME8_DME_MEMORY_CONTROL_BASE_IDX                                                             2
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_vpg_vpg_dispdec
+// base address: 0x1b634
+#define regVPG8_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x38cd
+#define regVPG8_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 2
+#define regVPG8_VPG_GENERIC_PACKET_DATA                                                                 0x38ce
+#define regVPG8_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        2
+#define regVPG8_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x38cf
+#define regVPG8_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      2
+#define regVPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x38d0
+#define regVPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  2
+#define regVPG8_VPG_GENERIC_STATUS                                                                      0x38d1
+#define regVPG8_VPG_GENERIC_STATUS_BASE_IDX                                                             2
+#define regVPG8_VPG_MEM_PWR                                                                             0x38d2
+#define regVPG8_VPG_MEM_PWR_BASE_IDX                                                                    2
+#define regVPG8_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x38d3
+#define regVPG8_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        2
+#define regVPG8_VPG_ISRC1_2_DATA                                                                        0x38d4
+#define regVPG8_VPG_ISRC1_2_DATA_BASE_IDX                                                               2
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc3_dispdec
+// base address: 0x1b664
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_CONTROL                                                           0x38d9
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_CONTROL_BASE_IDX                                                  2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL                                                  0x38da
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL                                     0x38db
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL_BASE_IDX                            2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL                            0x38dc
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL_BASE_IDX                   2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT                                                  0x38dd
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA0                                                          0x38de
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA0_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA1                                                          0x38df
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA1_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA2                                                          0x38e0
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA2_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA3                                                          0x38e1
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA3_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA4                                                          0x38e2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA4_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA5                                                          0x38e3
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA5_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA6                                                          0x38e4
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA6_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA7                                                          0x38e5
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA7_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA8                                                          0x38e6
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA8_BASE_IDX                                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x38e7
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0                                                  0x38e8
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1                                                  0x38e9
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2                                                  0x38ea
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3                                                  0x38eb
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4                                                  0x38ec
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5                                                  0x38ed
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6                                                  0x38ee
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7                                                  0x38ef
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8                                                  0x38f0
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9                                                  0x38f1
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10                                                 0x38f2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11                                                 0x38f3
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12                                                 0x38f4
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13                                                 0x38f5
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14                                                 0x38f6
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL                                                       0x38f7
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL_BASE_IDX                                              2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0                                                0x38f8
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1                                                0x38f9
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL                                       0x38fa
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL                                                   0x38ff
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL                                                  0x3900
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL_BASE_IDX                                         2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL                                                0x3901
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL                                          0x3902
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL_BASE_IDX                                 2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x3903
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT0                                                   0x3904
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT0_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT1                                                   0x3905
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT1_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT2                                                   0x3906
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT2_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT3                                                   0x3907
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT3_BASE_IDX                                          2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS                                                    0x3908
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS_BASE_IDX                                           2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_STATUS                                               0x3909
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_STATUS_BASE_IDX                                      2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL                                              0x390a
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL_BASE_IDX                                     2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL                                                0x390b
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL                                                 0x390c
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET                                               0x390d
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET_BASE_IDX                                      2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_READY_CONTROL                                                0x390e
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_READY_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL                                        0x390f
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL_BASE_IDX                               2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS                                                       0x3910
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS_BASE_IDX                                              2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_START_CONTROL                                                0x3911
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_START_CONTROL_BASE_IDX                                       2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL                                       0x3912
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL_BASE_IDX                              2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS                                        0x3913
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS_BASE_IDX                               2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL                                                 0x3914
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL_BASE_IDX                                        2
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SPARE                                                             0x3915
+#define regDP_SYM32_ENC3_DP_SYM32_ENC_SPARE_BASE_IDX                                                    2
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc3_dispdec
+// base address: 0x1b76c
+#define regDP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL                                                       0x391b
+#define regDP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL_BASE_IDX                                              2
+#define regDP_LINK_ENC3_DP_LINK_ENC_SPARE                                                               0x391c
+#define regDP_LINK_ENC3_DP_LINK_ENC_SPARE_BASE_IDX                                                      2
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym323_dispdec
+// base address: 0x1b7c0
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL                                                         0x3930
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL_BASE_IDX                                                2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_STATUS                                                          0x3931
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_STATUS_BASE_IDX                                                 2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_UPDATE                                                      0x3934
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_UPDATE_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0                                                   0x3935
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1                                                   0x3936
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2                                                   0x3937
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3                                                   0x3938
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0                                                         0x393b
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0_BASE_IDX                                                2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1                                                         0x393c
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1_BASE_IDX                                                2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2                                                         0x393d
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2_BASE_IDX                                                2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3                                                         0x393e
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3_BASE_IDX                                                2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0                                                  0x3941
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0_BASE_IDX                                         2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1                                                  0x3942
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1_BASE_IDX                                         2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2                                                  0x3943
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2_BASE_IDX                                         2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3                                                  0x3944
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3_BASE_IDX                                         2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0                                                     0x3947
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0_BASE_IDX                                            2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR0                                                       0x3948
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR0_BASE_IDX                                              2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR1                                                       0x3949
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR1_BASE_IDX                                              2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR2                                                       0x394a
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR2_BASE_IDX                                              2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR3                                                       0x394b
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR3_BASE_IDX                                              2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0                                              0x394c
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0_BASE_IDX                                     2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0                                               0x394d
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0_BASE_IDX                                      2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL                                                    0x394f
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL_BASE_IDX                                           2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG                                                       0x3950
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG_BASE_IDX                                              2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED0                                                   0x3951
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED0_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED1                                                   0x3952
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED1_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED2                                                   0x3953
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED2_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED3                                                   0x3954
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED3_BASE_IDX                                          2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_SQ_PULSE                                                     0x3955
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_SQ_PULSE_BASE_IDX                                            2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM0                                                      0x3956
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM0_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM1                                                      0x3957
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM1_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM2                                                      0x3958
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM2_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM3                                                      0x3959
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM3_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM4                                                      0x395a
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM4_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM5                                                      0x395b
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM5_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM6                                                      0x395c
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM6_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM7                                                      0x395d
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM7_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM8                                                      0x395e
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM8_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM9                                                      0x395f
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM9_BASE_IDX                                             2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM10                                                     0x3960
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM10_BASE_IDX                                            2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS                                                    0x3961
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS_BASE_IDX                                           2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0                                            0x3964
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0_BASE_IDX                                   2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1                                            0x3965
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1_BASE_IDX                                   2
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL                                            0x3966
+#define regDP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL_BASE_IDX                                   2
+
+
+// addressBlock: dce_dc_mpc_mpcc0_dispdec
+// base address: 0x0
+#define regMPCC0_MPCC_TOP_SEL                                                                           0x0000
+#define regMPCC0_MPCC_TOP_SEL_BASE_IDX                                                                  3
+#define regMPCC0_MPCC_BOT_SEL                                                                           0x0001
+#define regMPCC0_MPCC_BOT_SEL_BASE_IDX                                                                  3
+#define regMPCC0_MPCC_OPP_ID                                                                            0x0002
+#define regMPCC0_MPCC_OPP_ID_BASE_IDX                                                                   3
+#define regMPCC0_MPCC_CONTROL                                                                           0x0003
+#define regMPCC0_MPCC_CONTROL_BASE_IDX                                                                  3
+#define regMPCC0_MPCC_CONTROL2                                                                          0x0004
+#define regMPCC0_MPCC_CONTROL2_BASE_IDX                                                                 3
+#define regMPCC0_MPCC_SM_CONTROL                                                                        0x0005
+#define regMPCC0_MPCC_SM_CONTROL_BASE_IDX                                                               3
+#define regMPCC0_MPCC_UPDATE_LOCK_SEL                                                                   0x0006
+#define regMPCC0_MPCC_UPDATE_LOCK_SEL_BASE_IDX                                                          3
+#define regMPCC0_MPCC_TOP_GAIN                                                                          0x0007
+#define regMPCC0_MPCC_TOP_GAIN_BASE_IDX                                                                 3
+#define regMPCC0_MPCC_BOT_GAIN_INSIDE                                                                   0x0008
+#define regMPCC0_MPCC_BOT_GAIN_INSIDE_BASE_IDX                                                          3
+#define regMPCC0_MPCC_BOT_GAIN_OUTSIDE                                                                  0x0009
+#define regMPCC0_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX                                                         3
+#define regMPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL                                                       0x000a
+#define regMPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL_BASE_IDX                                              3
+#define regMPCC0_MPCC_BG_R_CR                                                                           0x000b
+#define regMPCC0_MPCC_BG_R_CR_BASE_IDX                                                                  3
+#define regMPCC0_MPCC_BG_G_Y                                                                            0x000c
+#define regMPCC0_MPCC_BG_G_Y_BASE_IDX                                                                   3
+#define regMPCC0_MPCC_BG_B_CB                                                                           0x000d
+#define regMPCC0_MPCC_BG_B_CB_BASE_IDX                                                                  3
+#define regMPCC0_MPCC_MEM_PWR_CTRL                                                                      0x000e
+#define regMPCC0_MPCC_MEM_PWR_CTRL_BASE_IDX                                                             3
+#define regMPCC0_MPCC_STATUS                                                                            0x000f
+#define regMPCC0_MPCC_STATUS_BASE_IDX                                                                   3
+
+
+// addressBlock: dce_dc_mpc_mpcc1_dispdec
+// base address: 0x54
+#define regMPCC1_MPCC_TOP_SEL                                                                           0x0015
+#define regMPCC1_MPCC_TOP_SEL_BASE_IDX                                                                  3
+#define regMPCC1_MPCC_BOT_SEL                                                                           0x0016
+#define regMPCC1_MPCC_BOT_SEL_BASE_IDX                                                                  3
+#define regMPCC1_MPCC_OPP_ID                                                                            0x0017
+#define regMPCC1_MPCC_OPP_ID_BASE_IDX                                                                   3
+#define regMPCC1_MPCC_CONTROL                                                                           0x0018
+#define regMPCC1_MPCC_CONTROL_BASE_IDX                                                                  3
+#define regMPCC1_MPCC_CONTROL2                                                                          0x0019
+#define regMPCC1_MPCC_CONTROL2_BASE_IDX                                                                 3
+#define regMPCC1_MPCC_SM_CONTROL                                                                        0x001a
+#define regMPCC1_MPCC_SM_CONTROL_BASE_IDX                                                               3
+#define regMPCC1_MPCC_UPDATE_LOCK_SEL                                                                   0x001b
+#define regMPCC1_MPCC_UPDATE_LOCK_SEL_BASE_IDX                                                          3
+#define regMPCC1_MPCC_TOP_GAIN                                                                          0x001c
+#define regMPCC1_MPCC_TOP_GAIN_BASE_IDX                                                                 3
+#define regMPCC1_MPCC_BOT_GAIN_INSIDE                                                                   0x001d
+#define regMPCC1_MPCC_BOT_GAIN_INSIDE_BASE_IDX                                                          3
+#define regMPCC1_MPCC_BOT_GAIN_OUTSIDE                                                                  0x001e
+#define regMPCC1_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX                                                         3
+#define regMPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL                                                       0x001f
+#define regMPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL_BASE_IDX                                              3
+#define regMPCC1_MPCC_BG_R_CR                                                                           0x0020
+#define regMPCC1_MPCC_BG_R_CR_BASE_IDX                                                                  3
+#define regMPCC1_MPCC_BG_G_Y                                                                            0x0021
+#define regMPCC1_MPCC_BG_G_Y_BASE_IDX                                                                   3
+#define regMPCC1_MPCC_BG_B_CB                                                                           0x0022
+#define regMPCC1_MPCC_BG_B_CB_BASE_IDX                                                                  3
+#define regMPCC1_MPCC_MEM_PWR_CTRL                                                                      0x0023
+#define regMPCC1_MPCC_MEM_PWR_CTRL_BASE_IDX                                                             3
+#define regMPCC1_MPCC_STATUS                                                                            0x0024
+#define regMPCC1_MPCC_STATUS_BASE_IDX                                                                   3
+
+
+// addressBlock: dce_dc_mpc_mpcc2_dispdec
+// base address: 0xa8
+#define regMPCC2_MPCC_TOP_SEL                                                                           0x002a
+#define regMPCC2_MPCC_TOP_SEL_BASE_IDX                                                                  3
+#define regMPCC2_MPCC_BOT_SEL                                                                           0x002b
+#define regMPCC2_MPCC_BOT_SEL_BASE_IDX                                                                  3
+#define regMPCC2_MPCC_OPP_ID                                                                            0x002c
+#define regMPCC2_MPCC_OPP_ID_BASE_IDX                                                                   3
+#define regMPCC2_MPCC_CONTROL                                                                           0x002d
+#define regMPCC2_MPCC_CONTROL_BASE_IDX                                                                  3
+#define regMPCC2_MPCC_CONTROL2                                                                          0x002e
+#define regMPCC2_MPCC_CONTROL2_BASE_IDX                                                                 3
+#define regMPCC2_MPCC_SM_CONTROL                                                                        0x002f
+#define regMPCC2_MPCC_SM_CONTROL_BASE_IDX                                                               3
+#define regMPCC2_MPCC_UPDATE_LOCK_SEL                                                                   0x0030
+#define regMPCC2_MPCC_UPDATE_LOCK_SEL_BASE_IDX                                                          3
+#define regMPCC2_MPCC_TOP_GAIN                                                                          0x0031
+#define regMPCC2_MPCC_TOP_GAIN_BASE_IDX                                                                 3
+#define regMPCC2_MPCC_BOT_GAIN_INSIDE                                                                   0x0032
+#define regMPCC2_MPCC_BOT_GAIN_INSIDE_BASE_IDX                                                          3
+#define regMPCC2_MPCC_BOT_GAIN_OUTSIDE                                                                  0x0033
+#define regMPCC2_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX                                                         3
+#define regMPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL                                                       0x0034
+#define regMPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL_BASE_IDX                                              3
+#define regMPCC2_MPCC_BG_R_CR                                                                           0x0035
+#define regMPCC2_MPCC_BG_R_CR_BASE_IDX                                                                  3
+#define regMPCC2_MPCC_BG_G_Y                                                                            0x0036
+#define regMPCC2_MPCC_BG_G_Y_BASE_IDX                                                                   3
+#define regMPCC2_MPCC_BG_B_CB                                                                           0x0037
+#define regMPCC2_MPCC_BG_B_CB_BASE_IDX                                                                  3
+#define regMPCC2_MPCC_MEM_PWR_CTRL                                                                      0x0038
+#define regMPCC2_MPCC_MEM_PWR_CTRL_BASE_IDX                                                             3
+#define regMPCC2_MPCC_STATUS                                                                            0x0039
+#define regMPCC2_MPCC_STATUS_BASE_IDX                                                                   3
+
+
+// addressBlock: dce_dc_mpc_mpcc3_dispdec
+// base address: 0xfc
+#define regMPCC3_MPCC_TOP_SEL                                                                           0x003f
+#define regMPCC3_MPCC_TOP_SEL_BASE_IDX                                                                  3
+#define regMPCC3_MPCC_BOT_SEL                                                                           0x0040
+#define regMPCC3_MPCC_BOT_SEL_BASE_IDX                                                                  3
+#define regMPCC3_MPCC_OPP_ID                                                                            0x0041
+#define regMPCC3_MPCC_OPP_ID_BASE_IDX                                                                   3
+#define regMPCC3_MPCC_CONTROL                                                                           0x0042
+#define regMPCC3_MPCC_CONTROL_BASE_IDX                                                                  3
+#define regMPCC3_MPCC_CONTROL2                                                                          0x0043
+#define regMPCC3_MPCC_CONTROL2_BASE_IDX                                                                 3
+#define regMPCC3_MPCC_SM_CONTROL                                                                        0x0044
+#define regMPCC3_MPCC_SM_CONTROL_BASE_IDX                                                               3
+#define regMPCC3_MPCC_UPDATE_LOCK_SEL                                                                   0x0045
+#define regMPCC3_MPCC_UPDATE_LOCK_SEL_BASE_IDX                                                          3
+#define regMPCC3_MPCC_TOP_GAIN                                                                          0x0046
+#define regMPCC3_MPCC_TOP_GAIN_BASE_IDX                                                                 3
+#define regMPCC3_MPCC_BOT_GAIN_INSIDE                                                                   0x0047
+#define regMPCC3_MPCC_BOT_GAIN_INSIDE_BASE_IDX                                                          3
+#define regMPCC3_MPCC_BOT_GAIN_OUTSIDE                                                                  0x0048
+#define regMPCC3_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX                                                         3
+#define regMPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL                                                       0x0049
+#define regMPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL_BASE_IDX                                              3
+#define regMPCC3_MPCC_BG_R_CR                                                                           0x004a
+#define regMPCC3_MPCC_BG_R_CR_BASE_IDX                                                                  3
+#define regMPCC3_MPCC_BG_G_Y                                                                            0x004b
+#define regMPCC3_MPCC_BG_G_Y_BASE_IDX                                                                   3
+#define regMPCC3_MPCC_BG_B_CB                                                                           0x004c
+#define regMPCC3_MPCC_BG_B_CB_BASE_IDX                                                                  3
+#define regMPCC3_MPCC_MEM_PWR_CTRL                                                                      0x004d
+#define regMPCC3_MPCC_MEM_PWR_CTRL_BASE_IDX                                                             3
+#define regMPCC3_MPCC_STATUS                                                                            0x004e
+#define regMPCC3_MPCC_STATUS_BASE_IDX                                                                   3
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam0_dispdec
+// base address: 0x0
+#define regMPCC_OGAM0_MPCC_OGAM_CONTROL                                                                 0x007e
+#define regMPCC_OGAM0_MPCC_OGAM_CONTROL_BASE_IDX                                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_INDEX                                                               0x007f
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_DATA                                                                0x0080
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_DATA_BASE_IDX                                                       3
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_CONTROL                                                             0x0081
+#define regMPCC_OGAM0_MPCC_OGAM_LUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B                                                       0x0082
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G                                                       0x0083
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R                                                       0x0084
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B                                                 0x0085
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G                                                 0x0086
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R                                                 0x0087
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B                                                  0x0088
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_G                                                  0x0089
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_R                                                  0x008a
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B                                                        0x008b
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B                                                        0x008c
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G                                                        0x008d
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G                                                        0x008e
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R                                                        0x008f
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R                                                        0x0090
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B                                                           0x0091
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G                                                           0x0092
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R                                                           0x0093
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1                                                         0x0094
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3                                                         0x0095
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5                                                         0x0096
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7                                                         0x0097
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9                                                         0x0098
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11                                                       0x0099
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13                                                       0x009a
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15                                                       0x009b
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17                                                       0x009c
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19                                                       0x009d
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21                                                       0x009e
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23                                                       0x009f
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25                                                       0x00a0
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27                                                       0x00a1
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29                                                       0x00a2
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31                                                       0x00a3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33                                                       0x00a4
+#define regMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B                                                       0x00a5
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G                                                       0x00a6
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R                                                       0x00a7
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B                                                 0x00a8
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G                                                 0x00a9
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R                                                 0x00aa
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_B                                                  0x00ab
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_G                                                  0x00ac
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_R                                                  0x00ad
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B                                                        0x00ae
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B                                                        0x00af
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G                                                        0x00b0
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G                                                        0x00b1
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R                                                        0x00b2
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R                                                        0x00b3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_B                                                           0x00b4
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_G                                                           0x00b5
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_R                                                           0x00b6
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1                                                         0x00b7
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3                                                         0x00b8
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5                                                         0x00b9
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7                                                         0x00ba
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9                                                         0x00bb
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11                                                       0x00bc
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13                                                       0x00bd
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15                                                       0x00be
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17                                                       0x00bf
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19                                                       0x00c0
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21                                                       0x00c1
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23                                                       0x00c2
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25                                                       0x00c3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27                                                       0x00c4
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29                                                       0x00c5
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31                                                       0x00c6
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33                                                       0x00c7
+#define regMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT                                                      0x00c8
+#define regMPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                             3
+#define regMPCC_OGAM0_MPCC_GAMUT_REMAP_MODE                                                             0x00c9
+#define regMPCC_OGAM0_MPCC_GAMUT_REMAP_MODE_BASE_IDX                                                    3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A                                                         0x00ca
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A                                                         0x00cb
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A                                                         0x00cc
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A                                                         0x00cd
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A                                                         0x00ce
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A                                                         0x00cf
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B                                                         0x00d0
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B                                                         0x00d1
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B                                                         0x00d2
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B                                                         0x00d3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B                                                         0x00d4
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B_BASE_IDX                                                3
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B                                                         0x00d5
+#define regMPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B_BASE_IDX                                                3
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam1_dispdec
+// base address: 0x178
+#define regMPCC_OGAM1_MPCC_OGAM_CONTROL                                                                 0x00dc
+#define regMPCC_OGAM1_MPCC_OGAM_CONTROL_BASE_IDX                                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_INDEX                                                               0x00dd
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_DATA                                                                0x00de
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_DATA_BASE_IDX                                                       3
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_CONTROL                                                             0x00df
+#define regMPCC_OGAM1_MPCC_OGAM_LUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B                                                       0x00e0
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G                                                       0x00e1
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R                                                       0x00e2
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B                                                 0x00e3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G                                                 0x00e4
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R                                                 0x00e5
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_B                                                  0x00e6
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_G                                                  0x00e7
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_R                                                  0x00e8
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B                                                        0x00e9
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B                                                        0x00ea
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G                                                        0x00eb
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G                                                        0x00ec
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R                                                        0x00ed
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R                                                        0x00ee
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_B                                                           0x00ef
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_G                                                           0x00f0
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_R                                                           0x00f1
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1                                                         0x00f2
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3                                                         0x00f3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5                                                         0x00f4
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7                                                         0x00f5
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9                                                         0x00f6
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11                                                       0x00f7
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13                                                       0x00f8
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15                                                       0x00f9
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17                                                       0x00fa
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19                                                       0x00fb
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21                                                       0x00fc
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23                                                       0x00fd
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25                                                       0x00fe
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27                                                       0x00ff
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29                                                       0x0100
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31                                                       0x0101
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33                                                       0x0102
+#define regMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B                                                       0x0103
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G                                                       0x0104
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R                                                       0x0105
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B                                                 0x0106
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G                                                 0x0107
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R                                                 0x0108
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_B                                                  0x0109
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_G                                                  0x010a
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_R                                                  0x010b
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B                                                        0x010c
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B                                                        0x010d
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G                                                        0x010e
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G                                                        0x010f
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R                                                        0x0110
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R                                                        0x0111
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_B                                                           0x0112
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_G                                                           0x0113
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_R                                                           0x0114
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1                                                         0x0115
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3                                                         0x0116
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5                                                         0x0117
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7                                                         0x0118
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9                                                         0x0119
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11                                                       0x011a
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13                                                       0x011b
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15                                                       0x011c
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17                                                       0x011d
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19                                                       0x011e
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21                                                       0x011f
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23                                                       0x0120
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25                                                       0x0121
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27                                                       0x0122
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29                                                       0x0123
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31                                                       0x0124
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33                                                       0x0125
+#define regMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM1_MPCC_GAMUT_REMAP_COEF_FORMAT                                                      0x0126
+#define regMPCC_OGAM1_MPCC_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                             3
+#define regMPCC_OGAM1_MPCC_GAMUT_REMAP_MODE                                                             0x0127
+#define regMPCC_OGAM1_MPCC_GAMUT_REMAP_MODE_BASE_IDX                                                    3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A                                                         0x0128
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A                                                         0x0129
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A                                                         0x012a
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A                                                         0x012b
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A                                                         0x012c
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A                                                         0x012d
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B                                                         0x012e
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B                                                         0x012f
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B                                                         0x0130
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B                                                         0x0131
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B                                                         0x0132
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B_BASE_IDX                                                3
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B                                                         0x0133
+#define regMPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B_BASE_IDX                                                3
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam2_dispdec
+// base address: 0x2f0
+#define regMPCC_OGAM2_MPCC_OGAM_CONTROL                                                                 0x013a
+#define regMPCC_OGAM2_MPCC_OGAM_CONTROL_BASE_IDX                                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_INDEX                                                               0x013b
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_DATA                                                                0x013c
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_DATA_BASE_IDX                                                       3
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_CONTROL                                                             0x013d
+#define regMPCC_OGAM2_MPCC_OGAM_LUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B                                                       0x013e
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G                                                       0x013f
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R                                                       0x0140
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B                                                 0x0141
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G                                                 0x0142
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R                                                 0x0143
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_B                                                  0x0144
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_G                                                  0x0145
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_R                                                  0x0146
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B                                                        0x0147
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B                                                        0x0148
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G                                                        0x0149
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G                                                        0x014a
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R                                                        0x014b
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R                                                        0x014c
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_B                                                           0x014d
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_G                                                           0x014e
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_R                                                           0x014f
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1                                                         0x0150
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3                                                         0x0151
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5                                                         0x0152
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7                                                         0x0153
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9                                                         0x0154
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11                                                       0x0155
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13                                                       0x0156
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15                                                       0x0157
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17                                                       0x0158
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19                                                       0x0159
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21                                                       0x015a
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23                                                       0x015b
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25                                                       0x015c
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27                                                       0x015d
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29                                                       0x015e
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31                                                       0x015f
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33                                                       0x0160
+#define regMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B                                                       0x0161
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G                                                       0x0162
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R                                                       0x0163
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B                                                 0x0164
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G                                                 0x0165
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R                                                 0x0166
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_B                                                  0x0167
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_G                                                  0x0168
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_R                                                  0x0169
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B                                                        0x016a
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B                                                        0x016b
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G                                                        0x016c
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G                                                        0x016d
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R                                                        0x016e
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R                                                        0x016f
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_B                                                           0x0170
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_G                                                           0x0171
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_R                                                           0x0172
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1                                                         0x0173
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3                                                         0x0174
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5                                                         0x0175
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7                                                         0x0176
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9                                                         0x0177
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11                                                       0x0178
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13                                                       0x0179
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15                                                       0x017a
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17                                                       0x017b
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19                                                       0x017c
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21                                                       0x017d
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23                                                       0x017e
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25                                                       0x017f
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27                                                       0x0180
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29                                                       0x0181
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31                                                       0x0182
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33                                                       0x0183
+#define regMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM2_MPCC_GAMUT_REMAP_COEF_FORMAT                                                      0x0184
+#define regMPCC_OGAM2_MPCC_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                             3
+#define regMPCC_OGAM2_MPCC_GAMUT_REMAP_MODE                                                             0x0185
+#define regMPCC_OGAM2_MPCC_GAMUT_REMAP_MODE_BASE_IDX                                                    3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A                                                         0x0186
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A                                                         0x0187
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A                                                         0x0188
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A                                                         0x0189
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A                                                         0x018a
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A                                                         0x018b
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B                                                         0x018c
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B                                                         0x018d
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B                                                         0x018e
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B                                                         0x018f
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B                                                         0x0190
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B_BASE_IDX                                                3
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B                                                         0x0191
+#define regMPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B_BASE_IDX                                                3
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam3_dispdec
+// base address: 0x468
+#define regMPCC_OGAM3_MPCC_OGAM_CONTROL                                                                 0x0198
+#define regMPCC_OGAM3_MPCC_OGAM_CONTROL_BASE_IDX                                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_INDEX                                                               0x0199
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_DATA                                                                0x019a
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_DATA_BASE_IDX                                                       3
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_CONTROL                                                             0x019b
+#define regMPCC_OGAM3_MPCC_OGAM_LUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B                                                       0x019c
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G                                                       0x019d
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R                                                       0x019e
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B                                                 0x019f
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G                                                 0x01a0
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R                                                 0x01a1
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_B                                                  0x01a2
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_G                                                  0x01a3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_R                                                  0x01a4
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B                                                        0x01a5
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B                                                        0x01a6
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G                                                        0x01a7
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G                                                        0x01a8
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R                                                        0x01a9
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R                                                        0x01aa
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_B                                                           0x01ab
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_G                                                           0x01ac
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_R                                                           0x01ad
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1                                                         0x01ae
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3                                                         0x01af
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5                                                         0x01b0
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7                                                         0x01b1
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9                                                         0x01b2
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11                                                       0x01b3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13                                                       0x01b4
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15                                                       0x01b5
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17                                                       0x01b6
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19                                                       0x01b7
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21                                                       0x01b8
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23                                                       0x01b9
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25                                                       0x01ba
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27                                                       0x01bb
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29                                                       0x01bc
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31                                                       0x01bd
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33                                                       0x01be
+#define regMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B                                                       0x01bf
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G                                                       0x01c0
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R                                                       0x01c1
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B                                                 0x01c2
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G                                                 0x01c3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R                                                 0x01c4
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                        3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_B                                                  0x01c5
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_B_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_G                                                  0x01c6
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_G_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_R                                                  0x01c7
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_R_BASE_IDX                                         3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B                                                        0x01c8
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B                                                        0x01c9
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G                                                        0x01ca
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G                                                        0x01cb
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R                                                        0x01cc
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R                                                        0x01cd
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX                                               3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_B                                                           0x01ce
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_G                                                           0x01cf
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_R                                                           0x01d0
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1                                                         0x01d1
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3                                                         0x01d2
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5                                                         0x01d3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7                                                         0x01d4
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9                                                         0x01d5
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11                                                       0x01d6
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13                                                       0x01d7
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15                                                       0x01d8
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17                                                       0x01d9
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19                                                       0x01da
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21                                                       0x01db
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23                                                       0x01dc
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25                                                       0x01dd
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27                                                       0x01de
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29                                                       0x01df
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31                                                       0x01e0
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33                                                       0x01e1
+#define regMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX                                              3
+#define regMPCC_OGAM3_MPCC_GAMUT_REMAP_COEF_FORMAT                                                      0x01e2
+#define regMPCC_OGAM3_MPCC_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                             3
+#define regMPCC_OGAM3_MPCC_GAMUT_REMAP_MODE                                                             0x01e3
+#define regMPCC_OGAM3_MPCC_GAMUT_REMAP_MODE_BASE_IDX                                                    3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A                                                         0x01e4
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A                                                         0x01e5
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A                                                         0x01e6
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A                                                         0x01e7
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A                                                         0x01e8
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A                                                         0x01e9
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B                                                         0x01ea
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B                                                         0x01eb
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B                                                         0x01ec
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B                                                         0x01ed
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B                                                         0x01ee
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B_BASE_IDX                                                3
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B                                                         0x01ef
+#define regMPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B_BASE_IDX                                                3
+
+
+// addressBlock: dce_dc_mpc_mpc_ocsc_dispdec
+// base address: 0x0
+#define regMPC_OUT0_MUX                                                                                 0x02b2
+#define regMPC_OUT0_MUX_BASE_IDX                                                                        3
+#define regMPC_OUT0_DENORM_CONTROL                                                                      0x02b3
+#define regMPC_OUT0_DENORM_CONTROL_BASE_IDX                                                             3
+#define regMPC_OUT0_DENORM_CLAMP_G_Y                                                                    0x02b4
+#define regMPC_OUT0_DENORM_CLAMP_G_Y_BASE_IDX                                                           3
+#define regMPC_OUT0_DENORM_CLAMP_B_CB                                                                   0x02b5
+#define regMPC_OUT0_DENORM_CLAMP_B_CB_BASE_IDX                                                          3
+#define regMPC_OUT1_MUX                                                                                 0x02b6
+#define regMPC_OUT1_MUX_BASE_IDX                                                                        3
+#define regMPC_OUT1_DENORM_CONTROL                                                                      0x02b7
+#define regMPC_OUT1_DENORM_CONTROL_BASE_IDX                                                             3
+#define regMPC_OUT1_DENORM_CLAMP_G_Y                                                                    0x02b8
+#define regMPC_OUT1_DENORM_CLAMP_G_Y_BASE_IDX                                                           3
+#define regMPC_OUT1_DENORM_CLAMP_B_CB                                                                   0x02b9
+#define regMPC_OUT1_DENORM_CLAMP_B_CB_BASE_IDX                                                          3
+#define regMPC_OUT2_MUX                                                                                 0x02ba
+#define regMPC_OUT2_MUX_BASE_IDX                                                                        3
+#define regMPC_OUT2_DENORM_CONTROL                                                                      0x02bb
+#define regMPC_OUT2_DENORM_CONTROL_BASE_IDX                                                             3
+#define regMPC_OUT2_DENORM_CLAMP_G_Y                                                                    0x02bc
+#define regMPC_OUT2_DENORM_CLAMP_G_Y_BASE_IDX                                                           3
+#define regMPC_OUT2_DENORM_CLAMP_B_CB                                                                   0x02bd
+#define regMPC_OUT2_DENORM_CLAMP_B_CB_BASE_IDX                                                          3
+#define regMPC_OUT3_MUX                                                                                 0x02be
+#define regMPC_OUT3_MUX_BASE_IDX                                                                        3
+#define regMPC_OUT3_DENORM_CONTROL                                                                      0x02bf
+#define regMPC_OUT3_DENORM_CONTROL_BASE_IDX                                                             3
+#define regMPC_OUT3_DENORM_CLAMP_G_Y                                                                    0x02c0
+#define regMPC_OUT3_DENORM_CLAMP_G_Y_BASE_IDX                                                           3
+#define regMPC_OUT3_DENORM_CLAMP_B_CB                                                                   0x02c1
+#define regMPC_OUT3_DENORM_CLAMP_B_CB_BASE_IDX                                                          3
+#define regMPC_OUT_CSC_COEF_FORMAT                                                                      0x02ca
+#define regMPC_OUT_CSC_COEF_FORMAT_BASE_IDX                                                             3
+#define regMPC_OUT0_CSC_MODE                                                                            0x02cb
+#define regMPC_OUT0_CSC_MODE_BASE_IDX                                                                   3
+#define regMPC_OUT0_CSC_C11_C12_A                                                                       0x02cc
+#define regMPC_OUT0_CSC_C11_C12_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C13_C14_A                                                                       0x02cd
+#define regMPC_OUT0_CSC_C13_C14_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C21_C22_A                                                                       0x02ce
+#define regMPC_OUT0_CSC_C21_C22_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C23_C24_A                                                                       0x02cf
+#define regMPC_OUT0_CSC_C23_C24_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C31_C32_A                                                                       0x02d0
+#define regMPC_OUT0_CSC_C31_C32_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C33_C34_A                                                                       0x02d1
+#define regMPC_OUT0_CSC_C33_C34_A_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C11_C12_B                                                                       0x02d2
+#define regMPC_OUT0_CSC_C11_C12_B_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C13_C14_B                                                                       0x02d3
+#define regMPC_OUT0_CSC_C13_C14_B_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C21_C22_B                                                                       0x02d4
+#define regMPC_OUT0_CSC_C21_C22_B_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C23_C24_B                                                                       0x02d5
+#define regMPC_OUT0_CSC_C23_C24_B_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C31_C32_B                                                                       0x02d6
+#define regMPC_OUT0_CSC_C31_C32_B_BASE_IDX                                                              3
+#define regMPC_OUT0_CSC_C33_C34_B                                                                       0x02d7
+#define regMPC_OUT0_CSC_C33_C34_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_MODE                                                                            0x02d8
+#define regMPC_OUT1_CSC_MODE_BASE_IDX                                                                   3
+#define regMPC_OUT1_CSC_C11_C12_A                                                                       0x02d9
+#define regMPC_OUT1_CSC_C11_C12_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C13_C14_A                                                                       0x02da
+#define regMPC_OUT1_CSC_C13_C14_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C21_C22_A                                                                       0x02db
+#define regMPC_OUT1_CSC_C21_C22_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C23_C24_A                                                                       0x02dc
+#define regMPC_OUT1_CSC_C23_C24_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C31_C32_A                                                                       0x02dd
+#define regMPC_OUT1_CSC_C31_C32_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C33_C34_A                                                                       0x02de
+#define regMPC_OUT1_CSC_C33_C34_A_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C11_C12_B                                                                       0x02df
+#define regMPC_OUT1_CSC_C11_C12_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C13_C14_B                                                                       0x02e0
+#define regMPC_OUT1_CSC_C13_C14_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C21_C22_B                                                                       0x02e1
+#define regMPC_OUT1_CSC_C21_C22_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C23_C24_B                                                                       0x02e2
+#define regMPC_OUT1_CSC_C23_C24_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C31_C32_B                                                                       0x02e3
+#define regMPC_OUT1_CSC_C31_C32_B_BASE_IDX                                                              3
+#define regMPC_OUT1_CSC_C33_C34_B                                                                       0x02e4
+#define regMPC_OUT1_CSC_C33_C34_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_MODE                                                                            0x02e5
+#define regMPC_OUT2_CSC_MODE_BASE_IDX                                                                   3
+#define regMPC_OUT2_CSC_C11_C12_A                                                                       0x02e6
+#define regMPC_OUT2_CSC_C11_C12_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C13_C14_A                                                                       0x02e7
+#define regMPC_OUT2_CSC_C13_C14_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C21_C22_A                                                                       0x02e8
+#define regMPC_OUT2_CSC_C21_C22_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C23_C24_A                                                                       0x02e9
+#define regMPC_OUT2_CSC_C23_C24_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C31_C32_A                                                                       0x02ea
+#define regMPC_OUT2_CSC_C31_C32_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C33_C34_A                                                                       0x02eb
+#define regMPC_OUT2_CSC_C33_C34_A_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C11_C12_B                                                                       0x02ec
+#define regMPC_OUT2_CSC_C11_C12_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C13_C14_B                                                                       0x02ed
+#define regMPC_OUT2_CSC_C13_C14_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C21_C22_B                                                                       0x02ee
+#define regMPC_OUT2_CSC_C21_C22_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C23_C24_B                                                                       0x02ef
+#define regMPC_OUT2_CSC_C23_C24_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C31_C32_B                                                                       0x02f0
+#define regMPC_OUT2_CSC_C31_C32_B_BASE_IDX                                                              3
+#define regMPC_OUT2_CSC_C33_C34_B                                                                       0x02f1
+#define regMPC_OUT2_CSC_C33_C34_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_MODE                                                                            0x02f2
+#define regMPC_OUT3_CSC_MODE_BASE_IDX                                                                   3
+#define regMPC_OUT3_CSC_C11_C12_A                                                                       0x02f3
+#define regMPC_OUT3_CSC_C11_C12_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C13_C14_A                                                                       0x02f4
+#define regMPC_OUT3_CSC_C13_C14_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C21_C22_A                                                                       0x02f5
+#define regMPC_OUT3_CSC_C21_C22_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C23_C24_A                                                                       0x02f6
+#define regMPC_OUT3_CSC_C23_C24_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C31_C32_A                                                                       0x02f7
+#define regMPC_OUT3_CSC_C31_C32_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C33_C34_A                                                                       0x02f8
+#define regMPC_OUT3_CSC_C33_C34_A_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C11_C12_B                                                                       0x02f9
+#define regMPC_OUT3_CSC_C11_C12_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C13_C14_B                                                                       0x02fa
+#define regMPC_OUT3_CSC_C13_C14_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C21_C22_B                                                                       0x02fb
+#define regMPC_OUT3_CSC_C21_C22_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C23_C24_B                                                                       0x02fc
+#define regMPC_OUT3_CSC_C23_C24_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C31_C32_B                                                                       0x02fd
+#define regMPC_OUT3_CSC_C31_C32_B_BASE_IDX                                                              3
+#define regMPC_OUT3_CSC_C33_C34_B                                                                       0x02fe
+#define regMPC_OUT3_CSC_C33_C34_B_BASE_IDX                                                              3
+#define regMPC_OCSC_TEST_DEBUG_INDEX                                                                    0x031b
+#define regMPC_OCSC_TEST_DEBUG_INDEX_BASE_IDX                                                           3
+#define regMPC_OCSC_TEST_DEBUG_DATA                                                                     0x031c
+#define regMPC_OCSC_TEST_DEBUG_DATA_BASE_IDX                                                            3
+
+
+// addressBlock: dce_dc_mpc_mpc_rmcm0_dispdec
+// base address: 0x0
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_CONTROL                                                            0x031f
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R                                                           0x0320
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G                                                           0x0321
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B                                                           0x0322
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R                                                            0x0323
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B                                                          0x0324
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX                                                          0x0325
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA                                                           0x0326
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x0327
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B                                                  0x0328
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G                                                  0x0329
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R                                                  0x032a
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B                                                    0x032b
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G                                                    0x032c
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R                                                    0x032d
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1                                                    0x032e
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3                                                    0x032f
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5                                                    0x0330
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7                                                    0x0331
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9                                                    0x0332
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11                                                  0x0333
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13                                                  0x0334
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15                                                  0x0335
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17                                                  0x0336
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19                                                  0x0337
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21                                                  0x0338
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23                                                  0x0339
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25                                                  0x033a
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27                                                  0x033b
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29                                                  0x033c
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31                                                  0x033d
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33                                                  0x033e
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B                                                  0x033f
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G                                                  0x0340
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R                                                  0x0341
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B                                                    0x0342
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G                                                    0x0343
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R                                                    0x0344
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1                                                    0x0345
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3                                                    0x0346
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5                                                    0x0347
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7                                                    0x0348
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9                                                    0x0349
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11                                                  0x034a
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13                                                  0x034b
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15                                                  0x034c
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17                                                  0x034d
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19                                                  0x034e
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21                                                  0x034f
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23                                                  0x0350
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25                                                  0x0351
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27                                                  0x0352
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29                                                  0x0353
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31                                                  0x0354
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33                                                  0x0355
+#define regMPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_MODE                                                                0x0356
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_INDEX                                                               0x0357
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_DATA                                                                0x0358
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT                                                          0x0359
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL                                                  0x035a
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR                                                     0x035b
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R                                                        0x035c
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G                                                        0x035d
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B                                                        0x035e
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT                                                   0x035f
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                          3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE                                                          0x0360
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE_BASE_IDX                                                 3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A                                                     0x0361
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A                                                     0x0362
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A                                                     0x0363
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A                                                     0x0364
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A                                                     0x0365
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A                                                     0x0366
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B                                                     0x0367
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B                                                     0x0368
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B                                                     0x0369
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B                                                     0x036a
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B                                                     0x036b
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B                                                     0x036c
+#define regMPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B_BASE_IDX                                            3
+#define regMPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL                                                              0x036d
+#define regMPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT                                                    0x036e
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS                                                    0x036f
+#define regMPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+#define regMPC_RMCM0_MPC_RMCM_CNTL                                                                      0x0370
+#define regMPC_RMCM0_MPC_RMCM_CNTL_BASE_IDX                                                             3
+#define regMPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX                                                          0x0372
+#define regMPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX_BASE_IDX                                                 3
+#define regMPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA                                                           0x0373
+#define regMPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA_BASE_IDX                                                  3
+
+
+// addressBlock: dce_dc_mpc_mpc_rmcm1_dispdec
+// base address: 0x168
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_CONTROL                                                            0x0379
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_R                                                           0x037a
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_G                                                           0x037b
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_B                                                           0x037c
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_SCALE_R                                                            0x037d
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B                                                          0x037e
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_INDEX                                                          0x037f
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_DATA                                                           0x0380
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x0381
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B                                                  0x0382
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G                                                  0x0383
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R                                                  0x0384
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B                                                    0x0385
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G                                                    0x0386
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R                                                    0x0387
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1                                                    0x0388
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3                                                    0x0389
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5                                                    0x038a
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7                                                    0x038b
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9                                                    0x038c
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11                                                  0x038d
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13                                                  0x038e
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15                                                  0x038f
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17                                                  0x0390
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19                                                  0x0391
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21                                                  0x0392
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23                                                  0x0393
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25                                                  0x0394
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27                                                  0x0395
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29                                                  0x0396
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31                                                  0x0397
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33                                                  0x0398
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B                                                  0x0399
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G                                                  0x039a
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R                                                  0x039b
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B                                                    0x039c
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G                                                    0x039d
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R                                                    0x039e
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1                                                    0x039f
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3                                                    0x03a0
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5                                                    0x03a1
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7                                                    0x03a2
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9                                                    0x03a3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11                                                  0x03a4
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13                                                  0x03a5
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15                                                  0x03a6
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17                                                  0x03a7
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19                                                  0x03a8
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21                                                  0x03a9
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23                                                  0x03aa
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25                                                  0x03ab
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27                                                  0x03ac
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29                                                  0x03ad
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31                                                  0x03ae
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33                                                  0x03af
+#define regMPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_MODE                                                                0x03b0
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_INDEX                                                               0x03b1
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_DATA                                                                0x03b2
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_DATA_30BIT                                                          0x03b3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL                                                  0x03b4
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_NORM_FACTOR                                                     0x03b5
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R                                                        0x03b6
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G                                                        0x03b7
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B                                                        0x03b8
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT                                                   0x03b9
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                          3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE                                                          0x03ba
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE_BASE_IDX                                                 3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A                                                     0x03bb
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A                                                     0x03bc
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A                                                     0x03bd
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A                                                     0x03be
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A                                                     0x03bf
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A                                                     0x03c0
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B                                                     0x03c1
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B                                                     0x03c2
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B                                                     0x03c3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B                                                     0x03c4
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B                                                     0x03c5
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B                                                     0x03c6
+#define regMPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B_BASE_IDX                                            3
+#define regMPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL                                                              0x03c7
+#define regMPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_SELECT                                                    0x03c8
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS                                                    0x03c9
+#define regMPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+#define regMPC_RMCM1_MPC_RMCM_CNTL                                                                      0x03ca
+#define regMPC_RMCM1_MPC_RMCM_CNTL_BASE_IDX                                                             3
+#define regMPC_RMCM1_MPC_RMCM_TEST_DEBUG_INDEX                                                          0x03cc
+#define regMPC_RMCM1_MPC_RMCM_TEST_DEBUG_INDEX_BASE_IDX                                                 3
+#define regMPC_RMCM1_MPC_RMCM_TEST_DEBUG_DATA                                                           0x03cd
+#define regMPC_RMCM1_MPC_RMCM_TEST_DEBUG_DATA_BASE_IDX                                                  3
+
+
+// addressBlock: dce_dc_mpc_mpc_dcperfmon_dc_perfmon_dispdec
+// base address: 0x17e1c
+#define regDC_PERFMON22_PERFCOUNTER_CNTL                                                                0x0447
+#define regDC_PERFMON22_PERFCOUNTER_CNTL_BASE_IDX                                                       3
+#define regDC_PERFMON22_PERFCOUNTER_CNTL2                                                               0x0448
+#define regDC_PERFMON22_PERFCOUNTER_CNTL2_BASE_IDX                                                      3
+#define regDC_PERFMON22_PERFCOUNTER_STATE                                                               0x0449
+#define regDC_PERFMON22_PERFCOUNTER_STATE_BASE_IDX                                                      3
+#define regDC_PERFMON22_PERFMON_CNTL                                                                    0x044a
+#define regDC_PERFMON22_PERFMON_CNTL_BASE_IDX                                                           3
+#define regDC_PERFMON22_PERFMON_CNTL2                                                                   0x044b
+#define regDC_PERFMON22_PERFMON_CNTL2_BASE_IDX                                                          3
+#define regDC_PERFMON22_PERFMON_CVALUE_INT_MISC                                                         0x044c
+#define regDC_PERFMON22_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                3
+#define regDC_PERFMON22_PERFMON_CVALUE_LOW                                                              0x044d
+#define regDC_PERFMON22_PERFMON_CVALUE_LOW_BASE_IDX                                                     3
+#define regDC_PERFMON22_PERFMON_HI                                                                      0x044e
+#define regDC_PERFMON22_PERFMON_HI_BASE_IDX                                                             3
+#define regDC_PERFMON22_PERFMON_LOW                                                                     0x044f
+#define regDC_PERFMON22_PERFMON_LOW_BASE_IDX                                                            3
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm0_dispdec
+// base address: 0x0
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_CONTROL                                                            0x0453
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R                                                           0x0454
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G                                                           0x0455
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B                                                           0x0456
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R                                                            0x0457
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B                                                          0x0458
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX                                                          0x0459
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA                                                           0x045a
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x045b
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B                                                  0x045c
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G                                                  0x045d
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R                                                  0x045e
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B                                                    0x045f
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G                                                    0x0460
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R                                                    0x0461
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1                                                    0x0462
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3                                                    0x0463
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5                                                    0x0464
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7                                                    0x0465
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9                                                    0x0466
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11                                                  0x0467
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13                                                  0x0468
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15                                                  0x0469
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17                                                  0x046a
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19                                                  0x046b
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21                                                  0x046c
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23                                                  0x046d
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25                                                  0x046e
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27                                                  0x046f
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29                                                  0x0470
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31                                                  0x0471
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33                                                  0x0472
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B                                                  0x0473
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G                                                  0x0474
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R                                                  0x0475
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B                                                    0x0476
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G                                                    0x0477
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R                                                    0x0478
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1                                                    0x0479
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3                                                    0x047a
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5                                                    0x047b
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7                                                    0x047c
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9                                                    0x047d
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11                                                  0x047e
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13                                                  0x047f
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15                                                  0x0480
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17                                                  0x0481
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19                                                  0x0482
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21                                                  0x0483
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23                                                  0x0484
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25                                                  0x0485
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27                                                  0x0486
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29                                                  0x0487
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31                                                  0x0488
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33                                                  0x0489
+#define regMPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_MODE                                                                0x048a
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_INDEX                                                               0x048b
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_DATA                                                                0x048c
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT                                                          0x048d
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL                                                  0x048e
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_NORM_FACTOR                                                     0x048f
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R                                                        0x0490
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G                                                        0x0491
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B                                                        0x0492
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_CONTROL                                                             0x0493
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX                                                           0x0494
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX_BASE_IDX                                                  3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA                                                            0x0495
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA_BASE_IDX                                                   3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL                                                         0x0496
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL_BASE_IDX                                                3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B                                                   0x0497
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G                                                   0x0498
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R                                                   0x0499
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B                                             0x049a
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G                                             0x049b
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R                                             0x049c
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B                                              0x049d
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G                                              0x049e
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R                                              0x049f
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B                                                    0x04a0
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B                                                    0x04a1
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G                                                    0x04a2
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G                                                    0x04a3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R                                                    0x04a4
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R                                                    0x04a5
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B                                                       0x04a6
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_G                                                       0x04a7
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_R                                                       0x04a8
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1                                                     0x04a9
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3                                                     0x04aa
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5                                                     0x04ab
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7                                                     0x04ac
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9                                                     0x04ad
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11                                                   0x04ae
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13                                                   0x04af
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15                                                   0x04b0
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17                                                   0x04b1
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19                                                   0x04b2
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21                                                   0x04b3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23                                                   0x04b4
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25                                                   0x04b5
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27                                                   0x04b6
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29                                                   0x04b7
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31                                                   0x04b8
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33                                                   0x04b9
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B                                                   0x04ba
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G                                                   0x04bb
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R                                                   0x04bc
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B                                             0x04bd
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G                                             0x04be
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R                                             0x04bf
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B                                              0x04c0
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G                                              0x04c1
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R                                              0x04c2
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B                                                    0x04c3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B                                                    0x04c4
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G                                                    0x04c5
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G                                                    0x04c6
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R                                                    0x04c7
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R                                                    0x04c8
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_B                                                       0x04c9
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_G                                                       0x04ca
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_R                                                       0x04cb
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1                                                     0x04cc
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3                                                     0x04cd
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5                                                     0x04ce
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7                                                     0x04cf
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9                                                     0x04d0
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11                                                   0x04d1
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13                                                   0x04d2
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15                                                   0x04d3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17                                                   0x04d4
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19                                                   0x04d5
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21                                                   0x04d6
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23                                                   0x04d7
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25                                                   0x04d8
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27                                                   0x04d9
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29                                                   0x04da
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31                                                   0x04db
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33                                                   0x04dc
+#define regMPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT                                             0x04dd
+#define regMPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                    3
+#define regMPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE                                                    0x04de
+#define regMPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE_BASE_IDX                                           3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A                                                0x04df
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A                                                0x04e0
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A                                                0x04e1
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A                                                0x04e2
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A                                                0x04e3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A                                                0x04e4
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B                                                0x04e5
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B                                                0x04e6
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B                                                0x04e7
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B                                                0x04e8
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B                                                0x04e9
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B                                                0x04ea
+#define regMPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B_BASE_IDX                                       3
+#define regMPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT                                            0x04eb
+#define regMPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                   3
+#define regMPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE                                                   0x04ec
+#define regMPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE_BASE_IDX                                          3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A                                               0x04ed
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A                                               0x04ee
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A                                               0x04ef
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A                                               0x04f0
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A                                               0x04f1
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A                                               0x04f2
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B                                               0x04f3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B                                               0x04f4
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B                                               0x04f5
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B                                               0x04f6
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B                                               0x04f7
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B                                               0x04f8
+#define regMPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B_BASE_IDX                                      3
+#define regMPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL                                                              0x04f9
+#define regMPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT                                                    0x04fa
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS                                                    0x04fb
+#define regMPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm1_dispdec
+// base address: 0x2c0
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_CONTROL                                                            0x0503
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_R                                                           0x0504
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_G                                                           0x0505
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_B                                                           0x0506
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_SCALE_R                                                            0x0507
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B                                                          0x0508
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_INDEX                                                          0x0509
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_DATA                                                           0x050a
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x050b
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B                                                  0x050c
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G                                                  0x050d
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R                                                  0x050e
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B                                                    0x050f
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G                                                    0x0510
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R                                                    0x0511
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1                                                    0x0512
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3                                                    0x0513
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5                                                    0x0514
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7                                                    0x0515
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9                                                    0x0516
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11                                                  0x0517
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13                                                  0x0518
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15                                                  0x0519
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17                                                  0x051a
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19                                                  0x051b
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21                                                  0x051c
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23                                                  0x051d
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25                                                  0x051e
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27                                                  0x051f
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29                                                  0x0520
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31                                                  0x0521
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33                                                  0x0522
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B                                                  0x0523
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G                                                  0x0524
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R                                                  0x0525
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B                                                    0x0526
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G                                                    0x0527
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R                                                    0x0528
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1                                                    0x0529
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3                                                    0x052a
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5                                                    0x052b
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7                                                    0x052c
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9                                                    0x052d
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11                                                  0x052e
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13                                                  0x052f
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15                                                  0x0530
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17                                                  0x0531
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19                                                  0x0532
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21                                                  0x0533
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23                                                  0x0534
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25                                                  0x0535
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27                                                  0x0536
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29                                                  0x0537
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31                                                  0x0538
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33                                                  0x0539
+#define regMPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_MODE                                                                0x053a
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_INDEX                                                               0x053b
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_DATA                                                                0x053c
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_DATA_30BIT                                                          0x053d
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL                                                  0x053e
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_NORM_FACTOR                                                     0x053f
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R                                                        0x0540
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G                                                        0x0541
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B                                                        0x0542
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_CONTROL                                                             0x0543
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_INDEX                                                           0x0544
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_INDEX_BASE_IDX                                                  3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_DATA                                                            0x0545
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_DATA_BASE_IDX                                                   3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL                                                         0x0546
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL_BASE_IDX                                                3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B                                                   0x0547
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G                                                   0x0548
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R                                                   0x0549
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B                                             0x054a
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G                                             0x054b
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R                                             0x054c
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B                                              0x054d
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G                                              0x054e
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R                                              0x054f
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B                                                    0x0550
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B                                                    0x0551
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G                                                    0x0552
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G                                                    0x0553
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R                                                    0x0554
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R                                                    0x0555
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_B                                                       0x0556
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_G                                                       0x0557
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_R                                                       0x0558
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1                                                     0x0559
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3                                                     0x055a
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5                                                     0x055b
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7                                                     0x055c
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9                                                     0x055d
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11                                                   0x055e
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13                                                   0x055f
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15                                                   0x0560
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17                                                   0x0561
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19                                                   0x0562
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21                                                   0x0563
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23                                                   0x0564
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25                                                   0x0565
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27                                                   0x0566
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29                                                   0x0567
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31                                                   0x0568
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33                                                   0x0569
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B                                                   0x056a
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G                                                   0x056b
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R                                                   0x056c
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B                                             0x056d
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G                                             0x056e
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R                                             0x056f
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B                                              0x0570
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G                                              0x0571
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R                                              0x0572
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B                                                    0x0573
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B                                                    0x0574
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G                                                    0x0575
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G                                                    0x0576
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R                                                    0x0577
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R                                                    0x0578
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_B                                                       0x0579
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_G                                                       0x057a
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_R                                                       0x057b
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1                                                     0x057c
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3                                                     0x057d
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5                                                     0x057e
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7                                                     0x057f
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9                                                     0x0580
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11                                                   0x0581
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13                                                   0x0582
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15                                                   0x0583
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17                                                   0x0584
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19                                                   0x0585
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21                                                   0x0586
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23                                                   0x0587
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25                                                   0x0588
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27                                                   0x0589
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29                                                   0x058a
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31                                                   0x058b
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33                                                   0x058c
+#define regMPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT                                             0x058d
+#define regMPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                    3
+#define regMPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE                                                    0x058e
+#define regMPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE_BASE_IDX                                           3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A                                                0x058f
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A                                                0x0590
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A                                                0x0591
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A                                                0x0592
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A                                                0x0593
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A                                                0x0594
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B                                                0x0595
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B                                                0x0596
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B                                                0x0597
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B                                                0x0598
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B                                                0x0599
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B                                                0x059a
+#define regMPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B_BASE_IDX                                       3
+#define regMPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT                                            0x059b
+#define regMPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                   3
+#define regMPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE                                                   0x059c
+#define regMPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE_BASE_IDX                                          3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A                                               0x059d
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A                                               0x059e
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A                                               0x059f
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A                                               0x05a0
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A                                               0x05a1
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A                                               0x05a2
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B                                               0x05a3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B                                               0x05a4
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B                                               0x05a5
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B                                               0x05a6
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B                                               0x05a7
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B                                               0x05a8
+#define regMPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B_BASE_IDX                                      3
+#define regMPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL                                                              0x05a9
+#define regMPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_SELECT                                                    0x05aa
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS                                                    0x05ab
+#define regMPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm2_dispdec
+// base address: 0x580
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_CONTROL                                                            0x05b3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_R                                                           0x05b4
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_G                                                           0x05b5
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_B                                                           0x05b6
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_SCALE_R                                                            0x05b7
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B                                                          0x05b8
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_INDEX                                                          0x05b9
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_DATA                                                           0x05ba
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x05bb
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B                                                  0x05bc
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G                                                  0x05bd
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R                                                  0x05be
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B                                                    0x05bf
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G                                                    0x05c0
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R                                                    0x05c1
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1                                                    0x05c2
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3                                                    0x05c3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5                                                    0x05c4
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7                                                    0x05c5
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9                                                    0x05c6
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11                                                  0x05c7
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13                                                  0x05c8
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15                                                  0x05c9
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17                                                  0x05ca
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19                                                  0x05cb
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21                                                  0x05cc
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23                                                  0x05cd
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25                                                  0x05ce
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27                                                  0x05cf
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29                                                  0x05d0
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31                                                  0x05d1
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33                                                  0x05d2
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B                                                  0x05d3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G                                                  0x05d4
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R                                                  0x05d5
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B                                                    0x05d6
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G                                                    0x05d7
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R                                                    0x05d8
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1                                                    0x05d9
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3                                                    0x05da
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5                                                    0x05db
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7                                                    0x05dc
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9                                                    0x05dd
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11                                                  0x05de
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13                                                  0x05df
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15                                                  0x05e0
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17                                                  0x05e1
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19                                                  0x05e2
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21                                                  0x05e3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23                                                  0x05e4
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25                                                  0x05e5
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27                                                  0x05e6
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29                                                  0x05e7
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31                                                  0x05e8
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33                                                  0x05e9
+#define regMPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_MODE                                                                0x05ea
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_INDEX                                                               0x05eb
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_DATA                                                                0x05ec
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_DATA_30BIT                                                          0x05ed
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL                                                  0x05ee
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_NORM_FACTOR                                                     0x05ef
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R                                                        0x05f0
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G                                                        0x05f1
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B                                                        0x05f2
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_CONTROL                                                             0x05f3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_INDEX                                                           0x05f4
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_INDEX_BASE_IDX                                                  3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_DATA                                                            0x05f5
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_DATA_BASE_IDX                                                   3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL                                                         0x05f6
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL_BASE_IDX                                                3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B                                                   0x05f7
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G                                                   0x05f8
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R                                                   0x05f9
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B                                             0x05fa
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G                                             0x05fb
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R                                             0x05fc
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B                                              0x05fd
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G                                              0x05fe
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R                                              0x05ff
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B                                                    0x0600
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B                                                    0x0601
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G                                                    0x0602
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G                                                    0x0603
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R                                                    0x0604
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R                                                    0x0605
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_B                                                       0x0606
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_G                                                       0x0607
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_R                                                       0x0608
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1                                                     0x0609
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3                                                     0x060a
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5                                                     0x060b
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7                                                     0x060c
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9                                                     0x060d
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11                                                   0x060e
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13                                                   0x060f
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15                                                   0x0610
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17                                                   0x0611
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19                                                   0x0612
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21                                                   0x0613
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23                                                   0x0614
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25                                                   0x0615
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27                                                   0x0616
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29                                                   0x0617
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31                                                   0x0618
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33                                                   0x0619
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B                                                   0x061a
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G                                                   0x061b
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R                                                   0x061c
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B                                             0x061d
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G                                             0x061e
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R                                             0x061f
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B                                              0x0620
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G                                              0x0621
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R                                              0x0622
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B                                                    0x0623
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B                                                    0x0624
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G                                                    0x0625
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G                                                    0x0626
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R                                                    0x0627
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R                                                    0x0628
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_B                                                       0x0629
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_G                                                       0x062a
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_R                                                       0x062b
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1                                                     0x062c
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3                                                     0x062d
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5                                                     0x062e
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7                                                     0x062f
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9                                                     0x0630
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11                                                   0x0631
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13                                                   0x0632
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15                                                   0x0633
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17                                                   0x0634
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19                                                   0x0635
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21                                                   0x0636
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23                                                   0x0637
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25                                                   0x0638
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27                                                   0x0639
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29                                                   0x063a
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31                                                   0x063b
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33                                                   0x063c
+#define regMPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT                                             0x063d
+#define regMPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                    3
+#define regMPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE                                                    0x063e
+#define regMPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE_BASE_IDX                                           3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A                                                0x063f
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A                                                0x0640
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A                                                0x0641
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A                                                0x0642
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A                                                0x0643
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A                                                0x0644
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B                                                0x0645
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B                                                0x0646
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B                                                0x0647
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B                                                0x0648
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B                                                0x0649
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B                                                0x064a
+#define regMPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B_BASE_IDX                                       3
+#define regMPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT                                            0x064b
+#define regMPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                   3
+#define regMPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE                                                   0x064c
+#define regMPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE_BASE_IDX                                          3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A                                               0x064d
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A                                               0x064e
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A                                               0x064f
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A                                               0x0650
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A                                               0x0651
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A                                               0x0652
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B                                               0x0653
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B                                               0x0654
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B                                               0x0655
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B                                               0x0656
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B                                               0x0657
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B                                               0x0658
+#define regMPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B_BASE_IDX                                      3
+#define regMPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL                                                              0x0659
+#define regMPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_SELECT                                                    0x065a
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS                                                    0x065b
+#define regMPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm3_dispdec
+// base address: 0x840
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_CONTROL                                                            0x0663
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_CONTROL_BASE_IDX                                                   3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_R                                                           0x0664
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_R_BASE_IDX                                                  3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_G                                                           0x0665
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_G_BASE_IDX                                                  3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_B                                                           0x0666
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_B_BASE_IDX                                                  3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_SCALE_R                                                            0x0667
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_SCALE_R_BASE_IDX                                                   3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B                                                          0x0668
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B_BASE_IDX                                                 3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_INDEX                                                          0x0669
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_INDEX_BASE_IDX                                                 3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_DATA                                                           0x066a
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_DATA_BASE_IDX                                                  3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK                                                  0x066b
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B                                                  0x066c
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G                                                  0x066d
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R                                                  0x066e
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B                                                    0x066f
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G                                                    0x0670
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R                                                    0x0671
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1                                                    0x0672
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3                                                    0x0673
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5                                                    0x0674
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7                                                    0x0675
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9                                                    0x0676
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11                                                  0x0677
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13                                                  0x0678
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15                                                  0x0679
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17                                                  0x067a
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19                                                  0x067b
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21                                                  0x067c
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23                                                  0x067d
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25                                                  0x067e
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27                                                  0x067f
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29                                                  0x0680
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31                                                  0x0681
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33                                                  0x0682
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B                                                  0x0683
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G                                                  0x0684
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R                                                  0x0685
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B                                                    0x0686
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G                                                    0x0687
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R                                                    0x0688
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1                                                    0x0689
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3                                                    0x068a
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5                                                    0x068b
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7                                                    0x068c
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9                                                    0x068d
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11                                                  0x068e
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13                                                  0x068f
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15                                                  0x0690
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17                                                  0x0691
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19                                                  0x0692
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21                                                  0x0693
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23                                                  0x0694
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25                                                  0x0695
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27                                                  0x0696
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29                                                  0x0697
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31                                                  0x0698
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33                                                  0x0699
+#define regMPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_MODE                                                                0x069a
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_MODE_BASE_IDX                                                       3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_INDEX                                                               0x069b
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_INDEX_BASE_IDX                                                      3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_DATA                                                                0x069c
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_DATA_BASE_IDX                                                       3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_DATA_30BIT                                                          0x069d
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_DATA_30BIT_BASE_IDX                                                 3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL                                                  0x069e
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL_BASE_IDX                                         3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_NORM_FACTOR                                                     0x069f
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_NORM_FACTOR_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R                                                        0x06a0
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R_BASE_IDX                                               3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G                                                        0x06a1
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G_BASE_IDX                                               3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B                                                        0x06a2
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B_BASE_IDX                                               3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_CONTROL                                                             0x06a3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_CONTROL_BASE_IDX                                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_INDEX                                                           0x06a4
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_INDEX_BASE_IDX                                                  3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_DATA                                                            0x06a5
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_DATA_BASE_IDX                                                   3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL                                                         0x06a6
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL_BASE_IDX                                                3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B                                                   0x06a7
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G                                                   0x06a8
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R                                                   0x06a9
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B                                             0x06aa
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G                                             0x06ab
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R                                             0x06ac
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B                                              0x06ad
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G                                              0x06ae
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R                                              0x06af
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B                                                    0x06b0
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B                                                    0x06b1
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G                                                    0x06b2
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G                                                    0x06b3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R                                                    0x06b4
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R                                                    0x06b5
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_B                                                       0x06b6
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_G                                                       0x06b7
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_R                                                       0x06b8
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1                                                     0x06b9
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3                                                     0x06ba
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5                                                     0x06bb
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7                                                     0x06bc
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9                                                     0x06bd
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11                                                   0x06be
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13                                                   0x06bf
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15                                                   0x06c0
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17                                                   0x06c1
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19                                                   0x06c2
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21                                                   0x06c3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23                                                   0x06c4
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25                                                   0x06c5
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27                                                   0x06c6
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29                                                   0x06c7
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31                                                   0x06c8
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33                                                   0x06c9
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B                                                   0x06ca
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G                                                   0x06cb
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R                                                   0x06cc
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B                                             0x06cd
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G                                             0x06ce
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R                                             0x06cf
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B                                              0x06d0
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G                                              0x06d1
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R                                              0x06d2
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R_BASE_IDX                                     3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B                                                    0x06d3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B                                                    0x06d4
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G                                                    0x06d5
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G                                                    0x06d6
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R                                                    0x06d7
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R                                                    0x06d8
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_B                                                       0x06d9
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_B_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_G                                                       0x06da
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_G_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_R                                                       0x06db
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_R_BASE_IDX                                              3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1                                                     0x06dc
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3                                                     0x06dd
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5                                                     0x06de
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7                                                     0x06df
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9                                                     0x06e0
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9_BASE_IDX                                            3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11                                                   0x06e1
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13                                                   0x06e2
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15                                                   0x06e3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17                                                   0x06e4
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19                                                   0x06e5
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21                                                   0x06e6
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23                                                   0x06e7
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25                                                   0x06e8
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27                                                   0x06e9
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29                                                   0x06ea
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31                                                   0x06eb
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33                                                   0x06ec
+#define regMPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33_BASE_IDX                                          3
+#define regMPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT                                             0x06ed
+#define regMPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                    3
+#define regMPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE                                                    0x06ee
+#define regMPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE_BASE_IDX                                           3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A                                                0x06ef
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A                                                0x06f0
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A                                                0x06f1
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A                                                0x06f2
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A                                                0x06f3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A                                                0x06f4
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B                                                0x06f5
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B                                                0x06f6
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B                                                0x06f7
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B                                                0x06f8
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B                                                0x06f9
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B                                                0x06fa
+#define regMPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B_BASE_IDX                                       3
+#define regMPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT                                            0x06fb
+#define regMPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_BASE_IDX                                   3
+#define regMPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE                                                   0x06fc
+#define regMPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE_BASE_IDX                                          3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A                                               0x06fd
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A                                               0x06fe
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A                                               0x06ff
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A                                               0x0700
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A                                               0x0701
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A                                               0x0702
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B                                               0x0703
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B                                               0x0704
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B                                               0x0705
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B                                               0x0706
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B                                               0x0707
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B                                               0x0708
+#define regMPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B_BASE_IDX                                      3
+#define regMPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL                                                              0x0709
+#define regMPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL_BASE_IDX                                                     3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_SELECT                                                    0x070a
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_SELECT_BASE_IDX                                           3
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS                                                    0x070b
+#define regMPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS_BASE_IDX                                           3
+
+
+// addressBlock: dce_dc_mpc_mpc_cfg_dispdec
+// base address: 0x0
+#define regMPC_CLOCK_CONTROL                                                                            0x0873
+#define regMPC_CLOCK_CONTROL_BASE_IDX                                                                   3
+#define regMPC_SOFT_RESET                                                                               0x0874
+#define regMPC_SOFT_RESET_BASE_IDX                                                                      3
+#define regMPC_CRC_CTRL                                                                                 0x0875
+#define regMPC_CRC_CTRL_BASE_IDX                                                                        3
+#define regMPC_CRC_SEL_CONTROL                                                                          0x0876
+#define regMPC_CRC_SEL_CONTROL_BASE_IDX                                                                 3
+#define regMPC_CRC_RESULT_A                                                                             0x0877
+#define regMPC_CRC_RESULT_A_BASE_IDX                                                                    3
+#define regMPC_CRC_RESULT_R                                                                             0x0878
+#define regMPC_CRC_RESULT_R_BASE_IDX                                                                    3
+#define regMPC_CRC_RESULT_G                                                                             0x0879
+#define regMPC_CRC_RESULT_G_BASE_IDX                                                                    3
+#define regMPC_CRC_RESULT_B                                                                             0x087a
+#define regMPC_CRC_RESULT_B_BASE_IDX                                                                    3
+#define regMPC_PERFMON_EVENT_CTRL                                                                       0x087d
+#define regMPC_PERFMON_EVENT_CTRL_BASE_IDX                                                              3
+#define regMPC_BYPASS_BG_AR                                                                             0x087e
+#define regMPC_BYPASS_BG_AR_BASE_IDX                                                                    3
+#define regMPC_BYPASS_BG_GB                                                                             0x087f
+#define regMPC_BYPASS_BG_GB_BASE_IDX                                                                    3
+#define regMPC_HOST_READ_CONTROL                                                                        0x0880
+#define regMPC_HOST_READ_CONTROL_BASE_IDX                                                               3
+#define regMPC_DPP_PENDING_STATUS                                                                       0x0881
+#define regMPC_DPP_PENDING_STATUS_BASE_IDX                                                              3
+#define regMPC_PENDING_STATUS_MISC                                                                      0x0882
+#define regMPC_PENDING_STATUS_MISC_BASE_IDX                                                             3
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET0                                                                0x0883
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET0_BASE_IDX                                                       3
+#define regADR_CFG_VUPDATE_LOCK_SET0                                                                    0x0884
+#define regADR_CFG_VUPDATE_LOCK_SET0_BASE_IDX                                                           3
+#define regADR_VUPDATE_LOCK_SET0                                                                        0x0885
+#define regADR_VUPDATE_LOCK_SET0_BASE_IDX                                                               3
+#define regCFG_VUPDATE_LOCK_SET0                                                                        0x0886
+#define regCFG_VUPDATE_LOCK_SET0_BASE_IDX                                                               3
+#define regCUR_VUPDATE_LOCK_SET0                                                                        0x0887
+#define regCUR_VUPDATE_LOCK_SET0_BASE_IDX                                                               3
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET1                                                                0x0888
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET1_BASE_IDX                                                       3
+#define regADR_CFG_VUPDATE_LOCK_SET1                                                                    0x0889
+#define regADR_CFG_VUPDATE_LOCK_SET1_BASE_IDX                                                           3
+#define regADR_VUPDATE_LOCK_SET1                                                                        0x088a
+#define regADR_VUPDATE_LOCK_SET1_BASE_IDX                                                               3
+#define regCFG_VUPDATE_LOCK_SET1                                                                        0x088b
+#define regCFG_VUPDATE_LOCK_SET1_BASE_IDX                                                               3
+#define regCUR_VUPDATE_LOCK_SET1                                                                        0x088c
+#define regCUR_VUPDATE_LOCK_SET1_BASE_IDX                                                               3
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET2                                                                0x088d
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET2_BASE_IDX                                                       3
+#define regADR_CFG_VUPDATE_LOCK_SET2                                                                    0x088e
+#define regADR_CFG_VUPDATE_LOCK_SET2_BASE_IDX                                                           3
+#define regADR_VUPDATE_LOCK_SET2                                                                        0x088f
+#define regADR_VUPDATE_LOCK_SET2_BASE_IDX                                                               3
+#define regCFG_VUPDATE_LOCK_SET2                                                                        0x0890
+#define regCFG_VUPDATE_LOCK_SET2_BASE_IDX                                                               3
+#define regCUR_VUPDATE_LOCK_SET2                                                                        0x0891
+#define regCUR_VUPDATE_LOCK_SET2_BASE_IDX                                                               3
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET3                                                                0x0892
+#define regADR_CFG_CUR_VUPDATE_LOCK_SET3_BASE_IDX                                                       3
+#define regADR_CFG_VUPDATE_LOCK_SET3                                                                    0x0893
+#define regADR_CFG_VUPDATE_LOCK_SET3_BASE_IDX                                                           3
+#define regADR_VUPDATE_LOCK_SET3                                                                        0x0894
+#define regADR_VUPDATE_LOCK_SET3_BASE_IDX                                                               3
+#define regCFG_VUPDATE_LOCK_SET3                                                                        0x0895
+#define regCFG_VUPDATE_LOCK_SET3_BASE_IDX                                                               3
+#define regCUR_VUPDATE_LOCK_SET3                                                                        0x0896
+#define regCUR_VUPDATE_LOCK_SET3_BASE_IDX                                                               3
+#define regHUBP0_3DLUT_FL_CONFIG                                                                        0x0897
+#define regHUBP0_3DLUT_FL_CONFIG_BASE_IDX                                                               3
+#define regHUBP0_3DLUT_FL_BIAS_SCALE                                                                    0x0898
+#define regHUBP0_3DLUT_FL_BIAS_SCALE_BASE_IDX                                                           3
+#define regHUBP1_3DLUT_FL_CONFIG                                                                        0x0899
+#define regHUBP1_3DLUT_FL_CONFIG_BASE_IDX                                                               3
+#define regHUBP1_3DLUT_FL_BIAS_SCALE                                                                    0x089a
+#define regHUBP1_3DLUT_FL_BIAS_SCALE_BASE_IDX                                                           3
+#define regHUBP2_3DLUT_FL_CONFIG                                                                        0x089b
+#define regHUBP2_3DLUT_FL_CONFIG_BASE_IDX                                                               3
+#define regHUBP2_3DLUT_FL_BIAS_SCALE                                                                    0x089c
+#define regHUBP2_3DLUT_FL_BIAS_SCALE_BASE_IDX                                                           3
+#define regHUBP3_3DLUT_FL_CONFIG                                                                        0x089d
+#define regHUBP3_3DLUT_FL_CONFIG_BASE_IDX                                                               3
+#define regHUBP3_3DLUT_FL_BIAS_SCALE                                                                    0x089e
+#define regHUBP3_3DLUT_FL_BIAS_SCALE_BASE_IDX                                                           3
+#define regMPC_DWB0_MUX                                                                                 0x08ad
+#define regMPC_DWB0_MUX_BASE_IDX                                                                        3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d4
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d5
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_AUDIO_CONTROL                                                                0x08d8
+#define regHDMI_STREAM_ENC_AUDIO_CONTROL_BASE_IDX                                                       3
+
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ENCRYPTION_CONTROL                                                               0x0907
+#define regHDMI_TB_ENC_ENCRYPTION_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_apg_apg_dispdec
+// base address: 0x2644c
+#define regAPG9_APG_CONTROL                                                                             0x0913
+#define regAPG9_APG_CONTROL_BASE_IDX                                                                    3
+#define regAPG9_APG_CONTROL2                                                                            0x0914
+#define regAPG9_APG_CONTROL2_BASE_IDX                                                                   3
+#define regAPG9_APG_DBG_GEN_CONTROL                                                                     0x0915
+#define regAPG9_APG_DBG_GEN_CONTROL_BASE_IDX                                                            3
+#define regAPG9_APG_MEM_PWR                                                                             0x0927
+#define regAPG9_APG_MEM_PWR_BASE_IDX                                                                    3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_vpg_vpg_dispdec
+// base address: 0x264c4
+#define regVPG9_VPG_GENERIC_PACKET_ACCESS_CTRL                                                          0x0931
+#define regVPG9_VPG_GENERIC_PACKET_ACCESS_CTRL_BASE_IDX                                                 3
+#define regVPG9_VPG_GENERIC_PACKET_DATA                                                                 0x0932
+#define regVPG9_VPG_GENERIC_PACKET_DATA_BASE_IDX                                                        3
+#define regVPG9_VPG_GSP_FRAME_UPDATE_CTRL                                                               0x0933
+#define regVPG9_VPG_GSP_FRAME_UPDATE_CTRL_BASE_IDX                                                      3
+#define regVPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL                                                           0x0934
+#define regVPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL_BASE_IDX                                                  3
+#define regVPG9_VPG_GENERIC_STATUS                                                                      0x0935
+#define regVPG9_VPG_GENERIC_STATUS_BASE_IDX                                                             3
+#define regVPG9_VPG_MEM_PWR                                                                             0x0936
+#define regVPG9_VPG_MEM_PWR_BASE_IDX                                                                    3
+#define regVPG9_VPG_ISRC1_2_ACCESS_CTRL                                                                 0x0937
+#define regVPG9_VPG_ISRC1_2_ACCESS_CTRL_BASE_IDX                                                        3
+#define regVPG9_VPG_ISRC1_2_DATA                                                                        0x0938
+#define regVPG9_VPG_ISRC1_2_DATA_BASE_IDX                                                               3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dme_dme_dispdec
+// base address: 0x264f0
+#define regDME9_DME_CONTROL                                                                             0x093c
+#define regDME9_DME_CONTROL_BASE_IDX                                                                    3
+#define regDME9_DME_MEMORY_CONTROL                                                                      0x093d
+#define regDME9_DME_MEMORY_CONTROL_BASE_IDX                                                             3
+
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+
+// addressBlock: dce_dc_hpo_hpo_top_dispdec
+// base address: 0x2790c
+#define regHPO_TOP_CLOCK_CONTROL                                                                        0x0e43
+#define regHPO_TOP_CLOCK_CONTROL_BASE_IDX                                                               3
+#define regHPO_TOP_HW_CONTROL                                                                           0x0e44
+#define regHPO_TOP_HW_CONTROL_BASE_IDX                                                                  3
+
+
+// addressBlock: dce_dc_hpo_dp_stream_mapper_dispdec
+// base address: 0x27958
+#define regDP_STREAM_MAPPER_CONTROL0                                                                    0x0e56
+#define regDP_STREAM_MAPPER_CONTROL0_BASE_IDX                                                           3
+#define regDP_STREAM_MAPPER_CONTROL1                                                                    0x0e57
+#define regDP_STREAM_MAPPER_CONTROL1_BASE_IDX                                                           3
+#define regDP_STREAM_MAPPER_CONTROL2                                                                    0x0e58
+#define regDP_STREAM_MAPPER_CONTROL2_BASE_IDX                                                           3
+#define regDP_STREAM_MAPPER_CONTROL3                                                                    0x0e59
+#define regDP_STREAM_MAPPER_CONTROL3_BASE_IDX                                                           3
+
+
+// addressBlock: dce_dc_hpo_hpo_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1a698
+#define regDC_PERFMON23_PERFCOUNTER_CNTL                                                                0x0e66
+#define regDC_PERFMON23_PERFCOUNTER_CNTL_BASE_IDX                                                       3
+#define regDC_PERFMON23_PERFCOUNTER_CNTL2                                                               0x0e67
+#define regDC_PERFMON23_PERFCOUNTER_CNTL2_BASE_IDX                                                      3
+#define regDC_PERFMON23_PERFCOUNTER_STATE                                                               0x0e68
+#define regDC_PERFMON23_PERFCOUNTER_STATE_BASE_IDX                                                      3
+#define regDC_PERFMON23_PERFMON_CNTL                                                                    0x0e69
+#define regDC_PERFMON23_PERFMON_CNTL_BASE_IDX                                                           3
+#define regDC_PERFMON23_PERFMON_CNTL2                                                                   0x0e6a
+#define regDC_PERFMON23_PERFMON_CNTL2_BASE_IDX                                                          3
+#define regDC_PERFMON23_PERFMON_CVALUE_INT_MISC                                                         0x0e6b
+#define regDC_PERFMON23_PERFMON_CVALUE_INT_MISC_BASE_IDX                                                3
+#define regDC_PERFMON23_PERFMON_CVALUE_LOW                                                              0x0e6c
+#define regDC_PERFMON23_PERFMON_CVALUE_LOW_BASE_IDX                                                     3
+#define regDC_PERFMON23_PERFMON_HI                                                                      0x0e6d
+#define regDC_PERFMON23_PERFMON_HI_BASE_IDX                                                             3
+#define regDC_PERFMON23_PERFMON_LOW                                                                     0x0e6e
+#define regDC_PERFMON23_PERFMON_LOW_BASE_IDX                                                            3
+
+
+// addressBlock: dce_dc_opp_abm0_dispdec
+// base address: 0x0
+#define regABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL                                                             0x0e7a
+#define regABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL_BASE_IDX                                                    3
+#define regABM0_BL1_PWM_USER_LEVEL                                                                      0x0e7b
+#define regABM0_BL1_PWM_USER_LEVEL_BASE_IDX                                                             3
+#define regABM0_BL1_PWM_TARGET_ABM_LEVEL                                                                0x0e7c
+#define regABM0_BL1_PWM_TARGET_ABM_LEVEL_BASE_IDX                                                       3
+#define regABM0_BL1_PWM_CURRENT_ABM_LEVEL                                                               0x0e7d
+#define regABM0_BL1_PWM_CURRENT_ABM_LEVEL_BASE_IDX                                                      3
+#define regABM0_BL1_PWM_FINAL_DUTY_CYCLE                                                                0x0e7e
+#define regABM0_BL1_PWM_FINAL_DUTY_CYCLE_BASE_IDX                                                       3
+#define regABM0_BL1_PWM_MINIMUM_DUTY_CYCLE                                                              0x0e7f
+#define regABM0_BL1_PWM_MINIMUM_DUTY_CYCLE_BASE_IDX                                                     3
+#define regABM0_BL1_PWM_ABM_CNTL                                                                        0x0e80
+#define regABM0_BL1_PWM_ABM_CNTL_BASE_IDX                                                               3
+#define regABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE                                                           0x0e81
+#define regABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE_BASE_IDX                                                  3
+#define regABM0_DC_ABM1_CNTL                                                                            0x0e82
+#define regABM0_DC_ABM1_CNTL_BASE_IDX                                                                   3
+#define regABM0_DC_ABM1_IPCSC_COEFF_SEL                                                                 0x0e83
+#define regABM0_DC_ABM1_IPCSC_COEFF_SEL_BASE_IDX                                                        3
+#define regABM0_DC_ABM1_ACE_PWL_CNTL                                                                    0x0e84
+#define regABM0_DC_ABM1_ACE_PWL_CNTL_BASE_IDX                                                           3
+#define regABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA                                                           0x0e85
+#define regABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA_BASE_IDX                                                  3
+#define regABM0_DC_ABM1_ACE_CNTL_MISC                                                                   0x0e86
+#define regABM0_DC_ABM1_ACE_CNTL_MISC_BASE_IDX                                                          3
+#define regABM0_DC_ABM1_HGLS_REG_READ_PROGRESS                                                          0x0e88
+#define regABM0_DC_ABM1_HGLS_REG_READ_PROGRESS_BASE_IDX                                                 3
+#define regABM0_DC_ABM1_HG_MISC_CTRL                                                                    0x0e89
+#define regABM0_DC_ABM1_HG_MISC_CTRL_BASE_IDX                                                           3
+#define regABM0_DC_ABM1_LS_SUM_OF_LUMA                                                                  0x0e8a
+#define regABM0_DC_ABM1_LS_SUM_OF_LUMA_BASE_IDX                                                         3
+#define regABM0_DC_ABM1_LS_SUM_OF_LUMA_MSB                                                              0x0e8b
+#define regABM0_DC_ABM1_LS_SUM_OF_LUMA_MSB_BASE_IDX                                                     3
+#define regABM0_DC_ABM1_LS_MIN_MAX_LUMA                                                                 0x0e8c
+#define regABM0_DC_ABM1_LS_MIN_MAX_LUMA_BASE_IDX                                                        3
+#define regABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA                                                        0x0e8d
+#define regABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_BASE_IDX                                               3
+#define regABM0_DC_ABM1_LS_PIXEL_COUNT                                                                  0x0e8e
+#define regABM0_DC_ABM1_LS_PIXEL_COUNT_BASE_IDX                                                         3
+#define regABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES                                                    0x0e8f
+#define regABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_BASE_IDX                                           3
+#define regABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT                                                        0x0e90
+#define regABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT                                                        0x0e91
+#define regABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_SAMPLE_RATE                                                                  0x0e92
+#define regABM0_DC_ABM1_HG_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM0_DC_ABM1_LS_SAMPLE_RATE                                                                  0x0e93
+#define regABM0_DC_ABM1_LS_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG                                                          0x0e94
+#define regABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_BASE_IDX                                                 3
+#define regABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG                                                         0x0e95
+#define regABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG_BASE_IDX                                                3
+#define regABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX                                                          0x0e96
+#define regABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_BASE_IDX                                                 3
+#define regABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX                                                         0x0e97
+#define regABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_BASE_IDX                                                3
+#define regABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX                                                        0x0e98
+#define regABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX                                                        0x0e99
+#define regABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX                                                        0x0e9a
+#define regABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX                                                        0x0e9b
+#define regABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX                                                        0x0e9c
+#define regABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX                                                        0x0e9d
+#define regABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM0_DC_ABM1_HG_RESULT_INDEX                                                                 0x0e9e
+#define regABM0_DC_ABM1_HG_RESULT_INDEX_BASE_IDX                                                        3
+#define regABM0_DC_ABM1_HG_RESULT_DATA                                                                  0x0e9f
+#define regABM0_DC_ABM1_HG_RESULT_DATA_BASE_IDX                                                         3
+#define regABM0_DC_ABM1_BL_MASTER_LOCK                                                                  0x0ea0
+#define regABM0_DC_ABM1_BL_MASTER_LOCK_BASE_IDX                                                         3
+
+
+// addressBlock: dce_dc_opp_abm1_dispdec
+// base address: 0x104
+#define regABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL                                                             0x0ebb
+#define regABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL_BASE_IDX                                                    3
+#define regABM1_BL1_PWM_USER_LEVEL                                                                      0x0ebc
+#define regABM1_BL1_PWM_USER_LEVEL_BASE_IDX                                                             3
+#define regABM1_BL1_PWM_TARGET_ABM_LEVEL                                                                0x0ebd
+#define regABM1_BL1_PWM_TARGET_ABM_LEVEL_BASE_IDX                                                       3
+#define regABM1_BL1_PWM_CURRENT_ABM_LEVEL                                                               0x0ebe
+#define regABM1_BL1_PWM_CURRENT_ABM_LEVEL_BASE_IDX                                                      3
+#define regABM1_BL1_PWM_FINAL_DUTY_CYCLE                                                                0x0ebf
+#define regABM1_BL1_PWM_FINAL_DUTY_CYCLE_BASE_IDX                                                       3
+#define regABM1_BL1_PWM_MINIMUM_DUTY_CYCLE                                                              0x0ec0
+#define regABM1_BL1_PWM_MINIMUM_DUTY_CYCLE_BASE_IDX                                                     3
+#define regABM1_BL1_PWM_ABM_CNTL                                                                        0x0ec1
+#define regABM1_BL1_PWM_ABM_CNTL_BASE_IDX                                                               3
+#define regABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE                                                           0x0ec2
+#define regABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE_BASE_IDX                                                  3
+#define regABM1_DC_ABM1_CNTL                                                                            0x0ec3
+#define regABM1_DC_ABM1_CNTL_BASE_IDX                                                                   3
+#define regABM1_DC_ABM1_IPCSC_COEFF_SEL                                                                 0x0ec4
+#define regABM1_DC_ABM1_IPCSC_COEFF_SEL_BASE_IDX                                                        3
+#define regABM1_DC_ABM1_ACE_PWL_CNTL                                                                    0x0ec5
+#define regABM1_DC_ABM1_ACE_PWL_CNTL_BASE_IDX                                                           3
+#define regABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA                                                           0x0ec6
+#define regABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA_BASE_IDX                                                  3
+#define regABM1_DC_ABM1_ACE_CNTL_MISC                                                                   0x0ec7
+#define regABM1_DC_ABM1_ACE_CNTL_MISC_BASE_IDX                                                          3
+#define regABM1_DC_ABM1_HGLS_REG_READ_PROGRESS                                                          0x0ec9
+#define regABM1_DC_ABM1_HGLS_REG_READ_PROGRESS_BASE_IDX                                                 3
+#define regABM1_DC_ABM1_HG_MISC_CTRL                                                                    0x0eca
+#define regABM1_DC_ABM1_HG_MISC_CTRL_BASE_IDX                                                           3
+#define regABM1_DC_ABM1_LS_SUM_OF_LUMA                                                                  0x0ecb
+#define regABM1_DC_ABM1_LS_SUM_OF_LUMA_BASE_IDX                                                         3
+#define regABM1_DC_ABM1_LS_SUM_OF_LUMA_MSB                                                              0x0ecc
+#define regABM1_DC_ABM1_LS_SUM_OF_LUMA_MSB_BASE_IDX                                                     3
+#define regABM1_DC_ABM1_LS_MIN_MAX_LUMA                                                                 0x0ecd
+#define regABM1_DC_ABM1_LS_MIN_MAX_LUMA_BASE_IDX                                                        3
+#define regABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA                                                        0x0ece
+#define regABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_BASE_IDX                                               3
+#define regABM1_DC_ABM1_LS_PIXEL_COUNT                                                                  0x0ecf
+#define regABM1_DC_ABM1_LS_PIXEL_COUNT_BASE_IDX                                                         3
+#define regABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES                                                    0x0ed0
+#define regABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_BASE_IDX                                           3
+#define regABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT                                                        0x0ed1
+#define regABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT                                                        0x0ed2
+#define regABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_SAMPLE_RATE                                                                  0x0ed3
+#define regABM1_DC_ABM1_HG_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM1_DC_ABM1_LS_SAMPLE_RATE                                                                  0x0ed4
+#define regABM1_DC_ABM1_LS_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG                                                          0x0ed5
+#define regABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_BASE_IDX                                                 3
+#define regABM1_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG                                                         0x0ed6
+#define regABM1_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG_BASE_IDX                                                3
+#define regABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX                                                          0x0ed7
+#define regABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_BASE_IDX                                                 3
+#define regABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX                                                         0x0ed8
+#define regABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_BASE_IDX                                                3
+#define regABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX                                                        0x0ed9
+#define regABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX                                                        0x0eda
+#define regABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX                                                        0x0edb
+#define regABM1_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX                                                        0x0edc
+#define regABM1_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX                                                        0x0edd
+#define regABM1_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX                                                        0x0ede
+#define regABM1_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM1_DC_ABM1_HG_RESULT_INDEX                                                                 0x0edf
+#define regABM1_DC_ABM1_HG_RESULT_INDEX_BASE_IDX                                                        3
+#define regABM1_DC_ABM1_HG_RESULT_DATA                                                                  0x0ee0
+#define regABM1_DC_ABM1_HG_RESULT_DATA_BASE_IDX                                                         3
+#define regABM1_DC_ABM1_BL_MASTER_LOCK                                                                  0x0ee1
+#define regABM1_DC_ABM1_BL_MASTER_LOCK_BASE_IDX                                                         3
+
+
+// addressBlock: dce_dc_opp_abm2_dispdec
+// base address: 0x208
+#define regABM2_BL1_PWM_AMBIENT_LIGHT_LEVEL                                                             0x0efc
+#define regABM2_BL1_PWM_AMBIENT_LIGHT_LEVEL_BASE_IDX                                                    3
+#define regABM2_BL1_PWM_USER_LEVEL                                                                      0x0efd
+#define regABM2_BL1_PWM_USER_LEVEL_BASE_IDX                                                             3
+#define regABM2_BL1_PWM_TARGET_ABM_LEVEL                                                                0x0efe
+#define regABM2_BL1_PWM_TARGET_ABM_LEVEL_BASE_IDX                                                       3
+#define regABM2_BL1_PWM_CURRENT_ABM_LEVEL                                                               0x0eff
+#define regABM2_BL1_PWM_CURRENT_ABM_LEVEL_BASE_IDX                                                      3
+#define regABM2_BL1_PWM_FINAL_DUTY_CYCLE                                                                0x0f00
+#define regABM2_BL1_PWM_FINAL_DUTY_CYCLE_BASE_IDX                                                       3
+#define regABM2_BL1_PWM_MINIMUM_DUTY_CYCLE                                                              0x0f01
+#define regABM2_BL1_PWM_MINIMUM_DUTY_CYCLE_BASE_IDX                                                     3
+#define regABM2_BL1_PWM_ABM_CNTL                                                                        0x0f02
+#define regABM2_BL1_PWM_ABM_CNTL_BASE_IDX                                                               3
+#define regABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE                                                           0x0f03
+#define regABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE_BASE_IDX                                                  3
+#define regABM2_DC_ABM1_CNTL                                                                            0x0f04
+#define regABM2_DC_ABM1_CNTL_BASE_IDX                                                                   3
+#define regABM2_DC_ABM1_IPCSC_COEFF_SEL                                                                 0x0f05
+#define regABM2_DC_ABM1_IPCSC_COEFF_SEL_BASE_IDX                                                        3
+#define regABM2_DC_ABM1_ACE_PWL_CNTL                                                                    0x0f06
+#define regABM2_DC_ABM1_ACE_PWL_CNTL_BASE_IDX                                                           3
+#define regABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA                                                           0x0f07
+#define regABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA_BASE_IDX                                                  3
+#define regABM2_DC_ABM1_ACE_CNTL_MISC                                                                   0x0f08
+#define regABM2_DC_ABM1_ACE_CNTL_MISC_BASE_IDX                                                          3
+#define regABM2_DC_ABM1_HGLS_REG_READ_PROGRESS                                                          0x0f0a
+#define regABM2_DC_ABM1_HGLS_REG_READ_PROGRESS_BASE_IDX                                                 3
+#define regABM2_DC_ABM1_HG_MISC_CTRL                                                                    0x0f0b
+#define regABM2_DC_ABM1_HG_MISC_CTRL_BASE_IDX                                                           3
+#define regABM2_DC_ABM1_LS_SUM_OF_LUMA                                                                  0x0f0c
+#define regABM2_DC_ABM1_LS_SUM_OF_LUMA_BASE_IDX                                                         3
+#define regABM2_DC_ABM1_LS_SUM_OF_LUMA_MSB                                                              0x0f0d
+#define regABM2_DC_ABM1_LS_SUM_OF_LUMA_MSB_BASE_IDX                                                     3
+#define regABM2_DC_ABM1_LS_MIN_MAX_LUMA                                                                 0x0f0e
+#define regABM2_DC_ABM1_LS_MIN_MAX_LUMA_BASE_IDX                                                        3
+#define regABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA                                                        0x0f0f
+#define regABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_BASE_IDX                                               3
+#define regABM2_DC_ABM1_LS_PIXEL_COUNT                                                                  0x0f10
+#define regABM2_DC_ABM1_LS_PIXEL_COUNT_BASE_IDX                                                         3
+#define regABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES                                                    0x0f11
+#define regABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_BASE_IDX                                           3
+#define regABM2_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT                                                        0x0f12
+#define regABM2_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM2_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT                                                        0x0f13
+#define regABM2_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_SAMPLE_RATE                                                                  0x0f14
+#define regABM2_DC_ABM1_HG_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM2_DC_ABM1_LS_SAMPLE_RATE                                                                  0x0f15
+#define regABM2_DC_ABM1_LS_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM2_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG                                                          0x0f16
+#define regABM2_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_BASE_IDX                                                 3
+#define regABM2_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG                                                         0x0f17
+#define regABM2_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG_BASE_IDX                                                3
+#define regABM2_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX                                                          0x0f18
+#define regABM2_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_BASE_IDX                                                 3
+#define regABM2_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX                                                         0x0f19
+#define regABM2_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_BASE_IDX                                                3
+#define regABM2_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX                                                        0x0f1a
+#define regABM2_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX                                                        0x0f1b
+#define regABM2_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX                                                        0x0f1c
+#define regABM2_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX                                                        0x0f1d
+#define regABM2_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX                                                        0x0f1e
+#define regABM2_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX                                                        0x0f1f
+#define regABM2_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM2_DC_ABM1_HG_RESULT_INDEX                                                                 0x0f20
+#define regABM2_DC_ABM1_HG_RESULT_INDEX_BASE_IDX                                                        3
+#define regABM2_DC_ABM1_HG_RESULT_DATA                                                                  0x0f21
+#define regABM2_DC_ABM1_HG_RESULT_DATA_BASE_IDX                                                         3
+#define regABM2_DC_ABM1_BL_MASTER_LOCK                                                                  0x0f22
+#define regABM2_DC_ABM1_BL_MASTER_LOCK_BASE_IDX                                                         3
+
+
+// addressBlock: dce_dc_opp_abm3_dispdec
+// base address: 0x30c
+#define regABM3_BL1_PWM_AMBIENT_LIGHT_LEVEL                                                             0x0f3d
+#define regABM3_BL1_PWM_AMBIENT_LIGHT_LEVEL_BASE_IDX                                                    3
+#define regABM3_BL1_PWM_USER_LEVEL                                                                      0x0f3e
+#define regABM3_BL1_PWM_USER_LEVEL_BASE_IDX                                                             3
+#define regABM3_BL1_PWM_TARGET_ABM_LEVEL                                                                0x0f3f
+#define regABM3_BL1_PWM_TARGET_ABM_LEVEL_BASE_IDX                                                       3
+#define regABM3_BL1_PWM_CURRENT_ABM_LEVEL                                                               0x0f40
+#define regABM3_BL1_PWM_CURRENT_ABM_LEVEL_BASE_IDX                                                      3
+#define regABM3_BL1_PWM_FINAL_DUTY_CYCLE                                                                0x0f41
+#define regABM3_BL1_PWM_FINAL_DUTY_CYCLE_BASE_IDX                                                       3
+#define regABM3_BL1_PWM_MINIMUM_DUTY_CYCLE                                                              0x0f42
+#define regABM3_BL1_PWM_MINIMUM_DUTY_CYCLE_BASE_IDX                                                     3
+#define regABM3_BL1_PWM_ABM_CNTL                                                                        0x0f43
+#define regABM3_BL1_PWM_ABM_CNTL_BASE_IDX                                                               3
+#define regABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE                                                           0x0f44
+#define regABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE_BASE_IDX                                                  3
+#define regABM3_DC_ABM1_CNTL                                                                            0x0f45
+#define regABM3_DC_ABM1_CNTL_BASE_IDX                                                                   3
+#define regABM3_DC_ABM1_IPCSC_COEFF_SEL                                                                 0x0f46
+#define regABM3_DC_ABM1_IPCSC_COEFF_SEL_BASE_IDX                                                        3
+#define regABM3_DC_ABM1_ACE_PWL_CNTL                                                                    0x0f47
+#define regABM3_DC_ABM1_ACE_PWL_CNTL_BASE_IDX                                                           3
+#define regABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA                                                           0x0f48
+#define regABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA_BASE_IDX                                                  3
+#define regABM3_DC_ABM1_ACE_CNTL_MISC                                                                   0x0f49
+#define regABM3_DC_ABM1_ACE_CNTL_MISC_BASE_IDX                                                          3
+#define regABM3_DC_ABM1_HGLS_REG_READ_PROGRESS                                                          0x0f4b
+#define regABM3_DC_ABM1_HGLS_REG_READ_PROGRESS_BASE_IDX                                                 3
+#define regABM3_DC_ABM1_HG_MISC_CTRL                                                                    0x0f4c
+#define regABM3_DC_ABM1_HG_MISC_CTRL_BASE_IDX                                                           3
+#define regABM3_DC_ABM1_LS_SUM_OF_LUMA                                                                  0x0f4d
+#define regABM3_DC_ABM1_LS_SUM_OF_LUMA_BASE_IDX                                                         3
+#define regABM3_DC_ABM1_LS_SUM_OF_LUMA_MSB                                                              0x0f4e
+#define regABM3_DC_ABM1_LS_SUM_OF_LUMA_MSB_BASE_IDX                                                     3
+#define regABM3_DC_ABM1_LS_MIN_MAX_LUMA                                                                 0x0f4f
+#define regABM3_DC_ABM1_LS_MIN_MAX_LUMA_BASE_IDX                                                        3
+#define regABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA                                                        0x0f50
+#define regABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_BASE_IDX                                               3
+#define regABM3_DC_ABM1_LS_PIXEL_COUNT                                                                  0x0f51
+#define regABM3_DC_ABM1_LS_PIXEL_COUNT_BASE_IDX                                                         3
+#define regABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES                                                    0x0f52
+#define regABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_BASE_IDX                                           3
+#define regABM3_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT                                                        0x0f53
+#define regABM3_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM3_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT                                                        0x0f54
+#define regABM3_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_SAMPLE_RATE                                                                  0x0f55
+#define regABM3_DC_ABM1_HG_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM3_DC_ABM1_LS_SAMPLE_RATE                                                                  0x0f56
+#define regABM3_DC_ABM1_LS_SAMPLE_RATE_BASE_IDX                                                         3
+#define regABM3_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG                                                          0x0f57
+#define regABM3_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_BASE_IDX                                                 3
+#define regABM3_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG                                                         0x0f58
+#define regABM3_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG_BASE_IDX                                                3
+#define regABM3_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX                                                          0x0f59
+#define regABM3_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_BASE_IDX                                                 3
+#define regABM3_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX                                                         0x0f5a
+#define regABM3_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_BASE_IDX                                                3
+#define regABM3_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX                                                        0x0f5b
+#define regABM3_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX                                                        0x0f5c
+#define regABM3_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX                                                        0x0f5d
+#define regABM3_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX                                                        0x0f5e
+#define regABM3_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX                                                        0x0f5f
+#define regABM3_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX                                                        0x0f60
+#define regABM3_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX_BASE_IDX                                               3
+#define regABM3_DC_ABM1_HG_RESULT_INDEX                                                                 0x0f61
+#define regABM3_DC_ABM1_HG_RESULT_INDEX_BASE_IDX                                                        3
+#define regABM3_DC_ABM1_HG_RESULT_DATA                                                                  0x0f62
+#define regABM3_DC_ABM1_HG_RESULT_DATA_BASE_IDX                                                         3
+#define regABM3_DC_ABM1_BL_MASTER_LOCK                                                                  0x0f63
+#define regABM3_DC_ABM1_BL_MASTER_LOCK_BASE_IDX                                                         3
+
+
+// addressBlock: dce_dpia_dpia_mu0_dpiadec
+// base address: 0x72000
+#define regDPIA_MU_CLOCK_CTRL                                                                           0x13800
+#define regDPIA_MU_CLOCK_CTRL_BASE_IDX                                                                  3
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT0                                                                0x13801
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT0_BASE_IDX                                                       3
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT0                                                                0x13802
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT0_BASE_IDX                                                       3
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT1                                                                0x13803
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT1_BASE_IDX                                                       3
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT1                                                                0x13804
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT1_BASE_IDX                                                       3
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT2                                                                0x13805
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT2_BASE_IDX                                                       3
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT2                                                                0x13806
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT2_BASE_IDX                                                       3
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT3                                                                0x13807
+#define regDPIA_MU_CLOCK_CTRL_DPIA_PORT3_BASE_IDX                                                       3
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT3                                                                0x13808
+#define regDPIA_MU_RESET_CTRL_DPIA_PORT3_BASE_IDX                                                       3
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT0                                                                0x13811
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT0_BASE_IDX                                                       3
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT1                                                                0x13812
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT1_BASE_IDX                                                       3
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT2                                                                0x13813
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT2_BASE_IDX                                                       3
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT3                                                                0x13814
+#define regDPIA_MU_TPI_STATUS_DPIA_PORT3_BASE_IDX                                                       3
+#define regDPIA_MU_TPI_MAX_CREDIT_COUNT                                                                 0x13819
+#define regDPIA_MU_TPI_MAX_CREDIT_COUNT_BASE_IDX                                                        3
+#define regDPIA_MU_INTERRUPT_STATUS                                                                     0x1381a
+#define regDPIA_MU_INTERRUPT_STATUS_BASE_IDX                                                            3
+#define regDPIA_MU_INTERRUPT_CTRL                                                                       0x1381b
+#define regDPIA_MU_INTERRUPT_CTRL_BASE_IDX                                                              3
+#define regDPIA_MU_LOCAL_INTERRUPT_CTRL                                                                 0x1381c
+#define regDPIA_MU_LOCAL_INTERRUPT_CTRL_BASE_IDX                                                        3
+#define regDPIA_MU_LOCAL_INTERRUPT_ACK                                                                  0x1381d
+#define regDPIA_MU_LOCAL_INTERRUPT_ACK_BASE_IDX                                                         3
+#define regDPIA_MU_RBBMIF_TIMEOUT_CTRL                                                                  0x1381e
+#define regDPIA_MU_RBBMIF_TIMEOUT_CTRL_BASE_IDX                                                         3
+#define regDPIA_MU_RBBMIF_TIMEOUT_CTRL2                                                                 0x1381f
+#define regDPIA_MU_RBBMIF_TIMEOUT_CTRL2_BASE_IDX                                                        3
+#define regDPIA_MU_RBBMIF_STATUS                                                                        0x13820
+#define regDPIA_MU_RBBMIF_STATUS_BASE_IDX                                                               3
+#define regDPIA_MU_MICROSECOND_REF_CTRL                                                                 0x13821
+#define regDPIA_MU_MICROSECOND_REF_CTRL_BASE_IDX                                                        3
+#define regDPIA_MU_PORT_ADP_STATUS                                                                      0x13822
+#define regDPIA_MU_PORT_ADP_STATUS_BASE_IDX                                                             3
+#define regDPIA_GLUE_CTRL                                                                               0x13823
+#define regDPIA_GLUE_CTRL_BASE_IDX                                                                      3
+#define regDPIA_PERF_COUNT_CONTROL0                                                                     0x13825
+#define regDPIA_PERF_COUNT_CONTROL0_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_CONTROL1                                                                     0x13826
+#define regDPIA_PERF_COUNT_CONTROL1_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_CONTROL2                                                                     0x13827
+#define regDPIA_PERF_COUNT_CONTROL2_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_CONTROL3                                                                     0x13828
+#define regDPIA_PERF_COUNT_CONTROL3_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_CONTROL4                                                                     0x13829
+#define regDPIA_PERF_COUNT_CONTROL4_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_CONTROL5                                                                     0x1382a
+#define regDPIA_PERF_COUNT_CONTROL5_BASE_IDX                                                            3
+#define regDPIA_PERF_COUNT_INDEX                                                                        0x1382b
+#define regDPIA_PERF_COUNT_INDEX_BASE_IDX                                                               3
+#define regDPIA_PERF_COUNT_DATA_LO                                                                      0x1382c
+#define regDPIA_PERF_COUNT_DATA_LO_BASE_IDX                                                             3
+#define regDPIA_MU_SPARE                                                                                0x1382d
+#define regDPIA_MU_SPARE_BASE_IDX                                                                       3
+
+// addressBlock: dce_dc_hda_azcontroller_azdec
+// base address: 0x1300000
+#define regGLOBAL_CAPABILITIES                                                                          0x4b7000
+#define regGLOBAL_CAPABILITIES_BASE_IDX                                                                 3
+#define regMINOR_VERSION                                                                                0x4b7000
+#define regMINOR_VERSION_BASE_IDX                                                                       3
+#define regMAJOR_VERSION                                                                                0x4b7000
+#define regMAJOR_VERSION_BASE_IDX                                                                       3
+#define regOUTPUT_PAYLOAD_CAPABILITY                                                                    0x4b7001
+#define regOUTPUT_PAYLOAD_CAPABILITY_BASE_IDX                                                           3
+#define regINPUT_PAYLOAD_CAPABILITY                                                                     0x4b7001
+#define regINPUT_PAYLOAD_CAPABILITY_BASE_IDX                                                            3
+#define regGLOBAL_CONTROL                                                                               0x4b7002
+#define regGLOBAL_CONTROL_BASE_IDX                                                                      3
+#define regWAKE_ENABLE                                                                                  0x4b7003
+#define regWAKE_ENABLE_BASE_IDX                                                                         3
+#define regSTATE_CHANGE_STATUS                                                                          0x4b7003
+#define regSTATE_CHANGE_STATUS_BASE_IDX                                                                 3
+#define regGLOBAL_STATUS                                                                                0x4b7004
+#define regGLOBAL_STATUS_BASE_IDX                                                                       3
+#define regOUTPUT_STREAM_PAYLOAD_CAPABILITY                                                             0x4b7006
+#define regOUTPUT_STREAM_PAYLOAD_CAPABILITY_BASE_IDX                                                    3
+#define regINPUT_STREAM_PAYLOAD_CAPABILITY                                                              0x4b7006
+#define regINPUT_STREAM_PAYLOAD_CAPABILITY_BASE_IDX                                                     3
+#define regINTERRUPT_CONTROL                                                                            0x4b7008
+#define regINTERRUPT_CONTROL_BASE_IDX                                                                   3
+#define regINTERRUPT_STATUS                                                                             0x4b7009
+#define regINTERRUPT_STATUS_BASE_IDX                                                                    3
+#define regWALL_CLOCK_COUNTER                                                                           0x4b700c
+#define regWALL_CLOCK_COUNTER_BASE_IDX                                                                  3
+#define regSTREAM_SYNCHRONIZATION                                                                       0x4b700e
+#define regSTREAM_SYNCHRONIZATION_BASE_IDX                                                              3
+#define regCORB_LOWER_BASE_ADDRESS                                                                      0x4b7010
+#define regCORB_LOWER_BASE_ADDRESS_BASE_IDX                                                             3
+#define regCORB_UPPER_BASE_ADDRESS                                                                      0x4b7011
+#define regCORB_UPPER_BASE_ADDRESS_BASE_IDX                                                             3
+#define regAZCONTROLLER1_CORB_WRITE_POINTER                                                             0x4b7012
+#define regAZCONTROLLER1_CORB_WRITE_POINTER_BASE_IDX                                                    3
+#define regAZCONTROLLER1_CORB_READ_POINTER                                                              0x4b7012
+#define regAZCONTROLLER1_CORB_READ_POINTER_BASE_IDX                                                     3
+#define regAZCONTROLLER1_CORB_CONTROL                                                                   0x4b7013
+#define regAZCONTROLLER1_CORB_CONTROL_BASE_IDX                                                          3
+#define regAZCONTROLLER1_CORB_STATUS                                                                    0x4b7013
+#define regAZCONTROLLER1_CORB_STATUS_BASE_IDX                                                           3
+#define regAZCONTROLLER1_CORB_SIZE                                                                      0x4b7013
+#define regAZCONTROLLER1_CORB_SIZE_BASE_IDX                                                             3
+#define regAZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS                                                        0x4b7014
+#define regAZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS_BASE_IDX                                               3
+#define regAZCONTROLLER1_RIRB_UPPER_BASE_ADDRESS                                                        0x4b7015
+#define regAZCONTROLLER1_RIRB_UPPER_BASE_ADDRESS_BASE_IDX                                               3
+#define regAZCONTROLLER1_RIRB_WRITE_POINTER                                                             0x4b7016
+#define regAZCONTROLLER1_RIRB_WRITE_POINTER_BASE_IDX                                                    3
+#define regAZCONTROLLER1_RESPONSE_INTERRUPT_COUNT                                                       0x4b7016
+#define regAZCONTROLLER1_RESPONSE_INTERRUPT_COUNT_BASE_IDX                                              3
+#define regAZCONTROLLER1_RIRB_CONTROL                                                                   0x4b7017
+#define regAZCONTROLLER1_RIRB_CONTROL_BASE_IDX                                                          3
+#define regAZCONTROLLER1_RIRB_STATUS                                                                    0x4b7017
+#define regAZCONTROLLER1_RIRB_STATUS_BASE_IDX                                                           3
+#define regAZCONTROLLER1_RIRB_SIZE                                                                      0x4b7017
+#define regAZCONTROLLER1_RIRB_SIZE_BASE_IDX                                                             3
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE                                             0x4b7018
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_BASE_IDX                                    3
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                                        0x4b7018
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                               3
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                                       0x4b7018
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                              3
+#define regAZCONTROLLER1_IMMEDIATE_RESPONSE_INPUT_INTERFACE                                             0x4b7019
+#define regAZCONTROLLER1_IMMEDIATE_RESPONSE_INPUT_INTERFACE_BASE_IDX                                    3
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_STATUS                                                       0x4b701a
+#define regAZCONTROLLER1_IMMEDIATE_COMMAND_STATUS_BASE_IDX                                              3
+#define regAZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS                                                0x4b701c
+#define regAZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS_BASE_IDX                                       3
+#define regAZCONTROLLER1_DMA_POSITION_UPPER_BASE_ADDRESS                                                0x4b701d
+#define regAZCONTROLLER1_DMA_POSITION_UPPER_BASE_ADDRESS_BASE_IDX                                       3
+#define regAZCONTROLLER1_WALL_CLOCK_COUNTER_ALIAS                                                       0x4b780c
+#define regAZCONTROLLER1_WALL_CLOCK_COUNTER_ALIAS_BASE_IDX                                              3
+
+
+// addressBlock: dce_dc_hda_azendpoint_azdec
+// base address: 0x1300000
+#define regAZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                               0x4b7018
+#define regAZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                      3
+#define regAZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                              0x4b7018
+#define regAZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                     3
+
+
+// addressBlock: dce_dc_hda_azinputendpoint_azdec
+// base address: 0x1300000
+#define regAZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA                           0x4b7018
+#define regAZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_BASE_IDX                  3
+#define regAZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX                          0x4b7018
+#define regAZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_BASE_IDX                 3
+
+
+// addressBlock: dce_dc_hda_azroot_azdec
+// base address: 0x1300000
+#define regAZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA                                       0x4b7018
+#define regAZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX                              3
+#define regAZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX                                      0x4b7018
+#define regAZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX                             3
+
+
+// addressBlock: dce_dc_hda_azstream0_azdec
+// base address: 0x1300000
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7020
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7021
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b7022
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b7023
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b7024
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b7024
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b7026
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b7027
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7821
+#define regAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream1_azdec
+// base address: 0x1300020
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7028
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7029
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b702a
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b702b
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b702c
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b702c
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b702e
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b702f
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7829
+#define regAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream2_azdec
+// base address: 0x1300040
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7030
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7031
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b7032
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b7033
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b7034
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b7034
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b7036
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b7037
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7831
+#define regAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream3_azdec
+// base address: 0x1300060
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7038
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7039
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b703a
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b703b
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b703c
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b703c
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b703e
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b703f
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7839
+#define regAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream4_azdec
+// base address: 0x1300080
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7040
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7041
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b7042
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b7043
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b7044
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b7044
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b7046
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b7047
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7841
+#define regAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream5_azdec
+// base address: 0x13000a0
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7048
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7049
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b704a
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b704b
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b704c
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b704c
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b704e
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b704f
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7849
+#define regAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream6_azdec
+// base address: 0x13000c0
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7050
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7051
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b7052
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b7053
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b7054
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b7054
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b7056
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b7057
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7851
+#define regAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+
+// addressBlock: dce_dc_hda_azstream7_azdec
+// base address: 0x13000e0
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS                                      0x4b7058
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX                             3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER                         0x4b7059
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX                3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH                                    0x4b705a
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX                           3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX                                        0x4b705b
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX                               3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE                                               0x4b705c
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX                                      3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT                                                  0x4b705c
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX                                         3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS                          0x4b705e
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS                          0x4b705f
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX                 3
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS                   0x4b7859
+#define regAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX          3
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_sh_mask.h
new file mode 100644
index 0000000000000..01fb530933698
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_4_2_0_sh_mask.h
@@ -0,0 +1,67286 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _dcn_4_2_0_SH_MASK_HEADER
+#define _dcn_4_2_0_SH_MASK_HEADER
+
+
+// addressBlock: dce_dc_hda_azcontroller_azdec
+//AZCONTROLLER0_CORB_WRITE_POINTER
+#define AZCONTROLLER0_CORB_WRITE_POINTER__CORB_WRITE_POINTER__SHIFT                                           0x0
+#define AZCONTROLLER0_CORB_WRITE_POINTER__CORB_WRITE_POINTER_MASK                                             0x00FFL
+//AZCONTROLLER0_CORB_READ_POINTER
+#define AZCONTROLLER0_CORB_READ_POINTER__CORB_READ_POINTER__SHIFT                                             0x0
+#define AZCONTROLLER0_CORB_READ_POINTER__CORB_READ_POINTER_RESET__SHIFT                                       0xf
+#define AZCONTROLLER0_CORB_READ_POINTER__CORB_READ_POINTER_MASK                                               0x00FFL
+#define AZCONTROLLER0_CORB_READ_POINTER__CORB_READ_POINTER_RESET_MASK                                         0x8000L
+//AZCONTROLLER0_CORB_CONTROL
+#define AZCONTROLLER0_CORB_CONTROL__CORB_MEMORY_ERROR_INTERRUPT_ENABLE__SHIFT                                 0x0
+#define AZCONTROLLER0_CORB_CONTROL__ENABLE_CORB_DMA_ENGINE__SHIFT                                             0x1
+#define AZCONTROLLER0_CORB_CONTROL__CORB_MEMORY_ERROR_INTERRUPT_ENABLE_MASK                                   0x01L
+#define AZCONTROLLER0_CORB_CONTROL__ENABLE_CORB_DMA_ENGINE_MASK                                               0x02L
+//AZCONTROLLER0_CORB_STATUS
+#define AZCONTROLLER0_CORB_STATUS__CORB_MEMORY_ERROR_INDICATION__SHIFT                                        0x0
+#define AZCONTROLLER0_CORB_STATUS__CORB_MEMORY_ERROR_INDICATION_MASK                                          0x01L
+//AZCONTROLLER0_CORB_SIZE
+#define AZCONTROLLER0_CORB_SIZE__CORB_SIZE__SHIFT                                                             0x0
+#define AZCONTROLLER0_CORB_SIZE__CORB_SIZE_CAPABILITY__SHIFT                                                  0x4
+#define AZCONTROLLER0_CORB_SIZE__CORB_SIZE_MASK                                                               0x0003L
+#define AZCONTROLLER0_CORB_SIZE__CORB_SIZE_CAPABILITY_MASK                                                    0x00F0L
+//AZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS
+#define AZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_UNIMPLEMENTED_BITS__SHIFT                      0x0
+#define AZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_ADDRESS__SHIFT                                 0x7
+#define AZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_UNIMPLEMENTED_BITS_MASK                        0x0000007FL
+#define AZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_ADDRESS_MASK                                   0xFFFFFF80L
+//AZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS
+#define AZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS__RIRB_UPPER_BASE_ADDRESS__SHIFT                                 0x0
+#define AZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS__RIRB_UPPER_BASE_ADDRESS_MASK                                   0xFFFFFFFFL
+//AZCONTROLLER0_RIRB_WRITE_POINTER
+#define AZCONTROLLER0_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER__SHIFT                                           0x0
+#define AZCONTROLLER0_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_RESET__SHIFT                                     0xf
+#define AZCONTROLLER0_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_MASK                                             0x00FFL
+#define AZCONTROLLER0_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_RESET_MASK                                       0x8000L
+//AZCONTROLLER0_RESPONSE_INTERRUPT_COUNT
+#define AZCONTROLLER0_RESPONSE_INTERRUPT_COUNT__N_RESPONSE_INTERRUPT_COUNT__SHIFT                             0x0
+#define AZCONTROLLER0_RESPONSE_INTERRUPT_COUNT__N_RESPONSE_INTERRUPT_COUNT_MASK                               0x00FFL
+//AZCONTROLLER0_RIRB_CONTROL
+#define AZCONTROLLER0_RIRB_CONTROL__RESPONSE_INTERRUPT_CONTROL__SHIFT                                         0x0
+#define AZCONTROLLER0_RIRB_CONTROL__RIRB_DMA_ENABLE__SHIFT                                                    0x1
+#define AZCONTROLLER0_RIRB_CONTROL__RESPONSE_OVERRUN_INTERRUPT_CONTROL__SHIFT                                 0x2
+#define AZCONTROLLER0_RIRB_CONTROL__RESPONSE_INTERRUPT_CONTROL_MASK                                           0x01L
+#define AZCONTROLLER0_RIRB_CONTROL__RIRB_DMA_ENABLE_MASK                                                      0x02L
+#define AZCONTROLLER0_RIRB_CONTROL__RESPONSE_OVERRUN_INTERRUPT_CONTROL_MASK                                   0x04L
+//AZCONTROLLER0_RIRB_STATUS
+#define AZCONTROLLER0_RIRB_STATUS__RESPONSE_INTERRUPT__SHIFT                                                  0x0
+#define AZCONTROLLER0_RIRB_STATUS__RESPONSE_OVERRUN_INTERRUPT_STATUS__SHIFT                                   0x2
+#define AZCONTROLLER0_RIRB_STATUS__RESPONSE_INTERRUPT_MASK                                                    0x01L
+#define AZCONTROLLER0_RIRB_STATUS__RESPONSE_OVERRUN_INTERRUPT_STATUS_MASK                                     0x04L
+//AZCONTROLLER0_RIRB_SIZE
+#define AZCONTROLLER0_RIRB_SIZE__RIRB_SIZE__SHIFT                                                             0x0
+#define AZCONTROLLER0_RIRB_SIZE__RIRB_SIZE_CAPABILITY__SHIFT                                                  0x4
+#define AZCONTROLLER0_RIRB_SIZE__RIRB_SIZE_MASK                                                               0x0003L
+#define AZCONTROLLER0_RIRB_SIZE__RIRB_SIZE_CAPABILITY_MASK                                                    0x00F0L
+//AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_VERB_AND_PAYLOAD__SHIFT     0x0
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_CODEC_ADDRESS__SHIFT        0x1c
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_VERB_AND_PAYLOAD_MASK       0x0FFFFFFFL
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_CODEC_ADDRESS_MASK          0xF0000000L
+//AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT                 0x0
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK                   0xFFFFFFFFL
+//AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT                0x0
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK                  0x0000FFFFL
+//AZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE
+#define AZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE__IMMEDIATE_RESPONSE_READ__SHIFT                      0x0
+#define AZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE__IMMEDIATE_RESPONSE_READ_MASK                        0xFFFFFFFFL
+//AZCONTROLLER0_IMMEDIATE_COMMAND_STATUS
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_COMMAND_BUSY__SHIFT                                 0x0
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_RESULT_VALID__SHIFT                                 0x1
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_COMMAND_BUSY_MASK                                   0x00000001L
+#define AZCONTROLLER0_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_RESULT_VALID_MASK                                   0x00000002L
+//AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_BUFFER_ENABLE__SHIFT                      0x0
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_UNIMPLEMENTED_BITS__SHIFT      0x1
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_ADDRESS__SHIFT                 0x7
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_BUFFER_ENABLE_MASK                        0x00000001L
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_UNIMPLEMENTED_BITS_MASK        0x0000007EL
+#define AZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_ADDRESS_MASK                   0xFFFFFF80L
+//AZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS
+#define AZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS__DMA_POSITION_UPPER_BASE_ADDRESS__SHIFT                 0x0
+#define AZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS__DMA_POSITION_UPPER_BASE_ADDRESS_MASK                   0xFFFFFFFFL
+//AZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS
+#define AZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS__WALL_CLOCK_COUNTER_ALIAS__SHIFT                               0x0
+#define AZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS__WALL_CLOCK_COUNTER_ALIAS_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dc_perfmon_dc_perfmondebugind
+//PERFMON_DEBUG_ID
+#define PERFMON_DEBUG_ID__PERFMON_DEBUG_ID__SHIFT                                                             0x0
+#define PERFMON_DEBUG_ID__PERFMON_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//PERFMON_DEBUG01
+#define PERFMON_DEBUG01__CLK0_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG01__CLK0_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG02
+#define PERFMON_DEBUG02__CLK0_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG02__CLK0_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG02__CLK0_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG02__CLK0_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG03
+#define PERFMON_DEBUG03__CLK1_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG03__CLK1_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG04
+#define PERFMON_DEBUG04__CLK1_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG04__CLK1_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG04__CLK1_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG04__CLK1_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG05
+#define PERFMON_DEBUG05__CLK2_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG05__CLK2_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG06
+#define PERFMON_DEBUG06__CLK2_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG06__CLK2_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG06__CLK2_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG06__CLK2_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG07
+#define PERFMON_DEBUG07__CLK3_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG07__CLK3_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG08
+#define PERFMON_DEBUG08__CLK3_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG08__CLK3_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG08__CLK3_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG08__CLK3_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG09
+#define PERFMON_DEBUG09__PERFMON_EN_EVENT_START__SHIFT                                                        0x0
+#define PERFMON_DEBUG09__PERFMON_EN_EVENT_STOP__SHIFT                                                         0x1
+#define PERFMON_DEBUG09__PERFMON_EN_EVENT_START_MASK                                                          0x00000001L
+#define PERFMON_DEBUG09__PERFMON_EN_EVENT_STOP_MASK                                                           0x00000002L
+//PERFMON_DEBUG0A
+#define PERFMON_DEBUG0A__CLK4_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG0A__CLK4_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG0B
+#define PERFMON_DEBUG0B__CLK4_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG0B__CLK4_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG0B__CLK4_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG0B__CLK4_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG0C
+#define PERFMON_DEBUG0C__CLK5_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG0C__CLK5_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG0D
+#define PERFMON_DEBUG0D__CLK5_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG0D__CLK5_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG0D__CLK5_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG0D__CLK5_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG0E
+#define PERFMON_DEBUG0E__CLK6_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG0E__CLK6_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG0F
+#define PERFMON_DEBUG0F__CLK6_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG0F__CLK6_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG0F__CLK6_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG0F__CLK6_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG10
+#define PERFMON_DEBUG10__CLK7_PERFCOUNTER_LOW__SHIFT                                                          0x0
+#define PERFMON_DEBUG10__CLK7_PERFCOUNTER_LOW_MASK                                                            0xFFFFFFFFL
+//PERFMON_DEBUG11
+#define PERFMON_DEBUG11__CLK7_PERFCOUNTER_HI__SHIFT                                                           0x0
+#define PERFMON_DEBUG11__CLK7_PERFCOUNTER_EVENT__SHIFT                                                        0x10
+#define PERFMON_DEBUG11__CLK7_PERFCOUNTER_HI_MASK                                                             0x0000FFFFL
+#define PERFMON_DEBUG11__CLK7_PERFCOUNTER_EVENT_MASK                                                          0x001F0000L
+//PERFMON_DEBUG12
+#define PERFMON_DEBUG12__CLK0_PERFCOUNTER_OFF__SHIFT                                                          0x0
+#define PERFMON_DEBUG12__CLK1_PERFCOUNTER_OFF__SHIFT                                                          0x1
+#define PERFMON_DEBUG12__CLK2_PERFCOUNTER_OFF__SHIFT                                                          0x2
+#define PERFMON_DEBUG12__CLK3_PERFCOUNTER_OFF__SHIFT                                                          0x3
+#define PERFMON_DEBUG12__CLK4_PERFCOUNTER_OFF__SHIFT                                                          0x4
+#define PERFMON_DEBUG12__CLK5_PERFCOUNTER_OFF__SHIFT                                                          0x5
+#define PERFMON_DEBUG12__CLK6_PERFCOUNTER_OFF__SHIFT                                                          0x6
+#define PERFMON_DEBUG12__CLK7_PERFCOUNTER_OFF__SHIFT                                                          0x7
+#define PERFMON_DEBUG12__PERFCOUNTER_OFF__SHIFT                                                               0x8
+#define PERFMON_DEBUG12__CLK0_PERFCOUNTER_OFF_MASK                                                            0x00000001L
+#define PERFMON_DEBUG12__CLK1_PERFCOUNTER_OFF_MASK                                                            0x00000002L
+#define PERFMON_DEBUG12__CLK2_PERFCOUNTER_OFF_MASK                                                            0x00000004L
+#define PERFMON_DEBUG12__CLK3_PERFCOUNTER_OFF_MASK                                                            0x00000008L
+#define PERFMON_DEBUG12__CLK4_PERFCOUNTER_OFF_MASK                                                            0x00000010L
+#define PERFMON_DEBUG12__CLK5_PERFCOUNTER_OFF_MASK                                                            0x00000020L
+#define PERFMON_DEBUG12__CLK6_PERFCOUNTER_OFF_MASK                                                            0x00000040L
+#define PERFMON_DEBUG12__CLK7_PERFCOUNTER_OFF_MASK                                                            0x00000080L
+#define PERFMON_DEBUG12__PERFCOUNTER_OFF_MASK                                                                 0x00000100L
+
+
+// addressBlock: mcif_wb0_mcif_wbdebugind
+//MCIF_WB_DEBUG_ID
+#define MCIF_WB_DEBUG_ID__MCIF_WB_DEBUG_ID__SHIFT                                                             0x0
+#define MCIF_WB_DEBUG_ID__MCIF_WB_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//ID01_WB_FMT_DBG
+#define ID01_WB_FMT_DBG__ID01_WB_FMT_DBG__SHIFT                                                               0x0
+#define ID01_WB_FMT_DBG__ID01_WB_FMT_DBG_MASK                                                                 0xFFFFFFFFL
+//ID02_WB_FMT_DBG
+#define ID02_WB_FMT_DBG__ID02_WB_FMT_DBG__SHIFT                                                               0x0
+#define ID02_WB_FMT_DBG__ID02_WB_FMT_DBG_MASK                                                                 0xFFFFFFFFL
+//ID03_WB_FMT_DBG
+#define ID03_WB_FMT_DBG__ID03_WB_FMT_DBG__SHIFT                                                               0x0
+#define ID03_WB_FMT_DBG__ID03_WB_FMT_DBG_MASK                                                                 0xFFFFFFFFL
+//ID04_WB_MGR_DBG
+#define ID04_WB_MGR_DBG__ID04_WB_MGR_DBG__SHIFT                                                               0x0
+#define ID04_WB_MGR_DBG__ID04_WB_MGR_DBG_MASK                                                                 0xFFFFFFFFL
+//ID05_WB_MGR_DBG
+#define ID05_WB_MGR_DBG__ID05_WB_MGR_DBG__SHIFT                                                               0x0
+#define ID05_WB_MGR_DBG__ID05_WB_MGR_DBG_MASK                                                                 0xFFFFFFFFL
+//ID06_WB_MGR_DBG
+#define ID06_WB_MGR_DBG__ID06_WB_MGR_DBG__SHIFT                                                               0x0
+#define ID06_WB_MGR_DBG__ID06_WB_MGR_DBG_MASK                                                                 0xFFFFFFFFL
+//ID07_WB_MGR_DBG
+#define ID07_WB_MGR_DBG__ID07_WB_MGR_DBG__SHIFT                                                               0x0
+#define ID07_WB_MGR_DBG__ID07_WB_MGR_DBG_MASK                                                                 0xFFFFFFFFL
+//ID08_WB_ARB_DBG
+#define ID08_WB_ARB_DBG__ID08_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID08_WB_ARB_DBG__ID08_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID09_WB_ARB_DBG
+#define ID09_WB_ARB_DBG__ID09_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID09_WB_ARB_DBG__ID09_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0A_WB_ARB_DBG
+#define ID0A_WB_ARB_DBG__ID0A_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID0A_WB_ARB_DBG__ID0A_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0B_WB_ARB_DBG
+#define ID0B_WB_ARB_DBG__ID0B_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID0B_WB_ARB_DBG__ID0B_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0C_WB_ARB_DBG
+#define ID0C_WB_ARB_DBG__ID0C_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID0C_WB_ARB_DBG__ID0C_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0D_WB_ARB_DBG
+#define ID0D_WB_ARB_DBG__ID0D_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID0D_WB_ARB_DBG__ID0D_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0E_WB_ARB_DBG
+#define ID0E_WB_ARB_DBG__ID0E_WB_ARB_DBG__SHIFT                                                               0x0
+#define ID0E_WB_ARB_DBG__ID0E_WB_ARB_DBG_MASK                                                                 0xFFFFFFFFL
+//ID0F_P010_WB_FMT_DBG_Y
+#define ID0F_P010_WB_FMT_DBG_Y__ID0F_P010_WB_FMT_DBG_Y__SHIFT                                                 0x0
+#define ID0F_P010_WB_FMT_DBG_Y__ID0F_P010_WB_FMT_DBG_Y_MASK                                                   0xFFFFFFFFL
+//ID10_P010_WB_FMT_DBG_C
+#define ID10_P010_WB_FMT_DBG_C__ID10_P010_WB_FMT_DBG_C__SHIFT                                                 0x0
+#define ID10_P010_WB_FMT_DBG_C__ID10_P010_WB_FMT_DBG_C_MASK                                                   0xFFFFFFFFL
+//ID11_WB_ARB_P010_DBG
+#define ID11_WB_ARB_P010_DBG__ID11_WB_ARB_P010_DBG__SHIFT                                                     0x0
+#define ID11_WB_ARB_P010_DBG__ID11_WB_ARB_P010_DBG_MASK                                                       0xFFFFFFFFL
+//ID12_WB_ARB_P010_DBG
+#define ID12_WB_ARB_P010_DBG__ID12_WB_ARB_P010_DBG__SHIFT                                                     0x0
+#define ID12_WB_ARB_P010_DBG__ID12_WB_ARB_P010_DBG_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: hubbub_dchubbubdebugind
+//DCHUBBUB_KEY_STATUS_DEBUG_1
+#define DCHUBBUB_KEY_STATUS_DEBUG_1__DCHUBBUB_KEY_STATUS_DEBUG_1__SHIFT                                       0x0
+#define DCHUBBUB_KEY_STATUS_DEBUG_1__DCHUBBUB_KEY_STATUS_DEBUG_1_MASK                                         0xFFFFFFFFL
+//DCHUBBUB_KEY_STATUS_DEBUG_2
+#define DCHUBBUB_KEY_STATUS_DEBUG_2__DCHUBBUB_KEY_STATUS_DEBUG_2__SHIFT                                       0x0
+#define DCHUBBUB_KEY_STATUS_DEBUG_2__DCHUBBUB_KEY_STATUS_DEBUG_2_MASK                                         0xFFFFFFFFL
+
+
+// addressBlock: odm0_odmdebugind
+//ODM0_OPTC_INPUT_DEBUG_ID
+#define ODM0_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID__SHIFT                                                  0x0
+#define ODM0_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID_MASK                                                    0xFFFFFFFFL
+//ODM0_OPTC_INPUT_DEBUG0
+#define ODM0_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0__SHIFT                                                      0x0
+#define ODM0_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0_MASK                                                        0xFFFFFFFFL
+//ODM0_OPTC_INPUT_DEBUG1
+#define ODM0_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1__SHIFT                                                      0x0
+#define ODM0_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1_MASK                                                        0xFFFFFFFFL
+//ODM0_OPTC_INPUT_DEBUG2
+#define ODM0_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2__SHIFT                                                      0x0
+#define ODM0_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2_MASK                                                        0xFFFFFFFFL
+
+
+// addressBlock: odm1_odmdebugind
+//ODM1_OPTC_INPUT_DEBUG_ID
+#define ODM1_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID__SHIFT                                                  0x0
+#define ODM1_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID_MASK                                                    0xFFFFFFFFL
+//ODM1_OPTC_INPUT_DEBUG0
+#define ODM1_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0__SHIFT                                                      0x0
+#define ODM1_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0_MASK                                                        0xFFFFFFFFL
+//ODM1_OPTC_INPUT_DEBUG1
+#define ODM1_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1__SHIFT                                                      0x0
+#define ODM1_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1_MASK                                                        0xFFFFFFFFL
+//ODM1_OPTC_INPUT_DEBUG2
+#define ODM1_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2__SHIFT                                                      0x0
+#define ODM1_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2_MASK                                                        0xFFFFFFFFL
+
+
+// addressBlock: odm2_odmdebugind
+//ODM2_OPTC_INPUT_DEBUG_ID
+#define ODM2_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID__SHIFT                                                  0x0
+#define ODM2_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID_MASK                                                    0xFFFFFFFFL
+//ODM2_OPTC_INPUT_DEBUG0
+#define ODM2_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0__SHIFT                                                      0x0
+#define ODM2_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0_MASK                                                        0xFFFFFFFFL
+//ODM2_OPTC_INPUT_DEBUG1
+#define ODM2_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1__SHIFT                                                      0x0
+#define ODM2_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1_MASK                                                        0xFFFFFFFFL
+//ODM2_OPTC_INPUT_DEBUG2
+#define ODM2_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2__SHIFT                                                      0x0
+#define ODM2_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2_MASK                                                        0xFFFFFFFFL
+
+
+// addressBlock: odm3_odmdebugind
+//ODM3_OPTC_INPUT_DEBUG_ID
+#define ODM3_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID__SHIFT                                                  0x0
+#define ODM3_OPTC_INPUT_DEBUG_ID__OPTC_INPUT_DEBUG_ID_MASK                                                    0xFFFFFFFFL
+//ODM3_OPTC_INPUT_DEBUG0
+#define ODM3_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0__SHIFT                                                      0x0
+#define ODM3_OPTC_INPUT_DEBUG0__OPTC_INPUT_DEBUG0_MASK                                                        0xFFFFFFFFL
+//ODM3_OPTC_INPUT_DEBUG1
+#define ODM3_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1__SHIFT                                                      0x0
+#define ODM3_OPTC_INPUT_DEBUG1__OPTC_INPUT_DEBUG1_MASK                                                        0xFFFFFFFFL
+//ODM3_OPTC_INPUT_DEBUG2
+#define ODM3_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2__SHIFT                                                      0x0
+#define ODM3_OPTC_INPUT_DEBUG2__OPTC_INPUT_DEBUG2_MASK                                                        0xFFFFFFFFL
+
+
+// addressBlock: rbbmif_rbbmifdebugind
+//RBBMIF_DEBUG_ID
+#define RBBMIF_DEBUG_ID__RBBMIF_DEBUG_ID__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_ID__RBBMIF_DEBUG_ID_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_0
+#define RBBMIF_DEBUG_0__RBBMIF_DEBUG_0__SHIFT                                                                 0x0
+#define RBBMIF_DEBUG_0__RBBMIF_DEBUG_0_MASK                                                                   0xFFFFFFFFL
+//RBBMIF_DEBUG_1
+#define RBBMIF_DEBUG_1__RBBMIF_DEBUG_1__SHIFT                                                                 0x0
+#define RBBMIF_DEBUG_1__RBBMIF_DEBUG_1_MASK                                                                   0xFFFFFFFFL
+//RBBMIF_DEBUG_2
+#define RBBMIF_DEBUG_2__RBBMIF_DEBUG_2__SHIFT                                                                 0x0
+#define RBBMIF_DEBUG_2__RBBMIF_DEBUG_2_MASK                                                                   0xFFFFFFFFL
+//RBBMIF_DEBUG_3
+#define RBBMIF_DEBUG_3__RBBMIF_DEBUG_3__SHIFT                                                                 0x0
+#define RBBMIF_DEBUG_3__RBBMIF_DEBUG_3_MASK                                                                   0xFFFFFFFFL
+//RBBMIF_DEBUG_4
+#define RBBMIF_DEBUG_4__RBBMIF_DEBUG_4__SHIFT                                                                 0x0
+#define RBBMIF_DEBUG_4__RBBMIF_DEBUG_4_MASK                                                                   0xFFFFFFFFL
+//RBBMIF_DEBUG_5
+#define RBBMIF_DEBUG_5__DBG_RBBMIF_DC_0__SHIFT                                                                0x0
+#define RBBMIF_DEBUG_5__DBG_RBBMIF_DC_0_MASK                                                                  0xFFFFFFFFL
+//RBBMIF_DEBUG_6
+#define RBBMIF_DEBUG_6__DBG_RBBMIF_DC_1__SHIFT                                                                0x0
+#define RBBMIF_DEBUG_6__DBG_RBBMIF_DC_1_MASK                                                                  0xFFFFFFFFL
+//RBBMIF_DEBUG_7
+#define RBBMIF_DEBUG_7__DBG_RBBMIF_DC_2__SHIFT                                                                0x0
+#define RBBMIF_DEBUG_7__DBG_RBBMIF_DC_2_MASK                                                                  0xFFFFFFFFL
+//RBBMIF_DEBUG_8
+#define RBBMIF_DEBUG_8__DBG_RBBMIF_DC_3__SHIFT                                                                0x0
+#define RBBMIF_DEBUG_8__DBG_RBBMIF_DC_3_MASK                                                                  0xFFFFFFFFL
+//RBBMIF_DEBUG_9
+#define RBBMIF_DEBUG_9__DBG_RBBMIF_DC_4__SHIFT                                                                0x0
+#define RBBMIF_DEBUG_9__DBG_RBBMIF_DC_4_MASK                                                                  0xFFFFFFFFL
+//RBBMIF_DEBUG_10
+#define RBBMIF_DEBUG_10__DBG_RBBMIF_DC_5__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_10__DBG_RBBMIF_DC_5_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_11
+#define RBBMIF_DEBUG_11__DBG_RBBMIF_DC_6__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_11__DBG_RBBMIF_DC_6_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_12
+#define RBBMIF_DEBUG_12__DBG_RBBMIF_DC_7__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_12__DBG_RBBMIF_DC_7_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_13
+#define RBBMIF_DEBUG_13__DBG_RBBMIF_DC_8__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_13__DBG_RBBMIF_DC_8_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_14
+#define RBBMIF_DEBUG_14__DBG_RBBMIF_DC_9__SHIFT                                                               0x0
+#define RBBMIF_DEBUG_14__DBG_RBBMIF_DC_9_MASK                                                                 0xFFFFFFFFL
+//RBBMIF_DEBUG_15
+#define RBBMIF_DEBUG_15__DBG_RBBMIF_DC_10__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_15__DBG_RBBMIF_DC_10_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_16
+#define RBBMIF_DEBUG_16__DBG_RBBMIF_DC_11__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_16__DBG_RBBMIF_DC_11_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_17
+#define RBBMIF_DEBUG_17__DBG_RBBMIF_DC_12__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_17__DBG_RBBMIF_DC_12_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_18
+#define RBBMIF_DEBUG_18__DBG_RBBMIF_DC_13__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_18__DBG_RBBMIF_DC_13_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_19
+#define RBBMIF_DEBUG_19__DBG_RBBMIF_DC_14__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_19__DBG_RBBMIF_DC_14_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_20
+#define RBBMIF_DEBUG_20__DBG_RBBMIF_DC_15__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_20__DBG_RBBMIF_DC_15_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_21
+#define RBBMIF_DEBUG_21__DBG_RBBMIF_DC_16__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_21__DBG_RBBMIF_DC_16_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_22
+#define RBBMIF_DEBUG_22__DBG_RBBMIF_DC_17__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_22__DBG_RBBMIF_DC_17_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_23
+#define RBBMIF_DEBUG_23__DBG_RBBMIF_DC_18__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_23__DBG_RBBMIF_DC_18_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_24
+#define RBBMIF_DEBUG_24__DBG_RBBMIF_DC_19__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_24__DBG_RBBMIF_DC_19_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_25
+#define RBBMIF_DEBUG_25__DBG_RBBMIF_DC_20__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_25__DBG_RBBMIF_DC_20_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_26
+#define RBBMIF_DEBUG_26__DBG_RBBMIF_DC_21__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_26__DBG_RBBMIF_DC_21_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_27
+#define RBBMIF_DEBUG_27__DBG_RBBMIF_DC_22__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_27__DBG_RBBMIF_DC_22_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_28
+#define RBBMIF_DEBUG_28__DBG_RBBMIF_DC_23__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_28__DBG_RBBMIF_DC_23_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_29
+#define RBBMIF_DEBUG_29__DBG_RBBMIF_DC_24__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_29__DBG_RBBMIF_DC_24_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_30
+#define RBBMIF_DEBUG_30__DBG_RBBMIF_DC_25__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_30__DBG_RBBMIF_DC_25_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_31
+#define RBBMIF_DEBUG_31__DBG_RBBMIF_DC_26__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_31__DBG_RBBMIF_DC_26_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_32
+#define RBBMIF_DEBUG_32__DBG_RBBMIF_DC_27__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_32__DBG_RBBMIF_DC_27_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_33
+#define RBBMIF_DEBUG_33__DBG_RBBMIF_DC_28__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_33__DBG_RBBMIF_DC_28_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_34
+#define RBBMIF_DEBUG_34__DBG_RBBMIF_DC_29__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_34__DBG_RBBMIF_DC_29_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_35
+#define RBBMIF_DEBUG_35__DBG_RBBMIF_DC_30__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_35__DBG_RBBMIF_DC_30_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_36
+#define RBBMIF_DEBUG_36__DBG_RBBMIF_DC_31__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_36__DBG_RBBMIF_DC_31_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_37
+#define RBBMIF_DEBUG_37__DBG_RBBMIF_DC_32__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_37__DBG_RBBMIF_DC_32_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_38
+#define RBBMIF_DEBUG_38__DBG_RBBMIF_DC_33__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_38__DBG_RBBMIF_DC_33_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_39
+#define RBBMIF_DEBUG_39__DBG_RBBMIF_DC_34__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_39__DBG_RBBMIF_DC_34_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_40
+#define RBBMIF_DEBUG_40__DBG_RBBMIF_DC_35__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_40__DBG_RBBMIF_DC_35_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_41
+#define RBBMIF_DEBUG_41__DBG_RBBMIF_DC_36__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_41__DBG_RBBMIF_DC_36_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_42
+#define RBBMIF_DEBUG_42__DBG_RBBMIF_DC_37__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_42__DBG_RBBMIF_DC_37_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_43
+#define RBBMIF_DEBUG_43__DBG_RBBMIF_DC_38__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_43__DBG_RBBMIF_DC_38_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_44
+#define RBBMIF_DEBUG_44__DBG_RBBMIF_DC_39__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_44__DBG_RBBMIF_DC_39_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_45
+#define RBBMIF_DEBUG_45__DBG_RBBMIF_DC_40__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_45__DBG_RBBMIF_DC_40_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_46
+#define RBBMIF_DEBUG_46__DBG_RBBMIF_DC_41__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_46__DBG_RBBMIF_DC_41_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_47
+#define RBBMIF_DEBUG_47__DBG_RBBMIF_DC_42__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_47__DBG_RBBMIF_DC_42_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_48
+#define RBBMIF_DEBUG_48__DBG_RBBMIF_DC_43__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_48__DBG_RBBMIF_DC_43_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_49
+#define RBBMIF_DEBUG_49__DBG_RBBMIF_DC_44__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_49__DBG_RBBMIF_DC_44_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_50
+#define RBBMIF_DEBUG_50__DBG_RBBMIF_DC_45__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_50__DBG_RBBMIF_DC_45_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_51
+#define RBBMIF_DEBUG_51__DBG_RBBMIF_DC_46__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_51__DBG_RBBMIF_DC_46_MASK                                                                0xFFFFFFFFL
+//RBBMIF_DEBUG_52
+#define RBBMIF_DEBUG_52__DBG_RBBMIF_DC_47__SHIFT                                                              0x0
+#define RBBMIF_DEBUG_52__DBG_RBBMIF_DC_47_MASK                                                                0xFFFFFFFFL
+
+
+// addressBlock: ihc_ihcdebugind
+//IHC_DEBUG_ID
+#define IHC_DEBUG_ID__IHC_DEBUG_ID__SHIFT                                                                     0x0
+#define IHC_DEBUG_ID__IHC_DEBUG_ID_MASK                                                                       0xFFFFFFFFL
+//IHC_CLIENT_DEBUG_A
+#define IHC_CLIENT_DEBUG_A__IHC_TEST_DEBUG_A__SHIFT                                                           0x0
+#define IHC_CLIENT_DEBUG_A__IHC_TEST_DEBUG_A_MASK                                                             0xFFFFFFFFL
+//IHC_CLIENT_DEBUG_B
+#define IHC_CLIENT_DEBUG_B__IHC_TEST_DEBUG_B__SHIFT                                                           0x0
+#define IHC_CLIENT_DEBUG_B__IHC_TEST_DEBUG_B_MASK                                                             0xFFFFFFFFL
+//IHC_CLIENT_DEBUG_C
+#define IHC_CLIENT_DEBUG_C__IHC_TEST_DEBUG_C__SHIFT                                                           0x0
+#define IHC_CLIENT_DEBUG_C__IHC_TEST_DEBUG_C_MASK                                                             0xFFFFFFFFL
+//IHC_CLIENT_DEBUG_D
+#define IHC_CLIENT_DEBUG_D__IHC_TEST_DEBUG_D__SHIFT                                                           0x0
+#define IHC_CLIENT_DEBUG_D__IHC_TEST_DEBUG_D_MASK                                                             0xFFFFFFFFL
+//IHC_CLIENT_DEBUG_E
+#define IHC_CLIENT_DEBUG_E__IHC_TEST_DEBUG_E__SHIFT                                                           0x0
+#define IHC_CLIENT_DEBUG_E__IHC_TEST_DEBUG_E_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dmu_misc_dmumiscdebugind
+//DMU_MISC_DEBUG_ID
+#define DMU_MISC_DEBUG_ID__DMU_MISC_DEBUG_ID__SHIFT                                                           0x0
+#define DMU_MISC_DEBUG_ID__DMU_MISC_DEBUG_ID_MASK                                                             0xFFFFFFFFL
+//DMU_MISC_DEBUG_1
+#define DMU_MISC_DEBUG_1__DMU_MISC_DEBUG_1__SHIFT                                                             0x0
+#define DMU_MISC_DEBUG_1__DMU_MISC_DEBUG_1_MASK                                                               0xFFFFFFFFL
+//DMU_ZSC_TEST_DEBUG1_DATA
+#define DMU_ZSC_TEST_DEBUG1_DATA__DMU_ZSC_TEST_DEBUG1_DATA__SHIFT                                             0x0
+#define DMU_ZSC_TEST_DEBUG1_DATA__DMU_ZSC_TEST_DEBUG1_DATA_MASK                                               0xFFFFFFFFL
+//DMU_ZSC_TEST_DEBUG2_DATA
+#define DMU_ZSC_TEST_DEBUG2_DATA__DMU_ZSC_TEST_DEBUG2_DATA__SHIFT                                             0x0
+#define DMU_ZSC_TEST_DEBUG2_DATA__DMU_ZSC_TEST_DEBUG2_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: dc_pg_dc_pgdebugind
+//DCPG_DEBUG_ID
+#define DCPG_DEBUG_ID__DCPG_DEBUG_ID__SHIFT                                                                   0x0
+#define DCPG_DEBUG_ID__DCPG_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS0
+#define DCPG_CONTROL_DEBUG_BUS0__DCPG_CONTROL_DEBUG_BUS0__SHIFT                                               0x0
+#define DCPG_CONTROL_DEBUG_BUS0__DCPG_CONTROL_DEBUG_BUS0_MASK                                                 0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS1
+#define DCPG_CONTROL_DEBUG_BUS1__DCPG_CONTROL_DEBUG_BUS1__SHIFT                                               0x0
+#define DCPG_CONTROL_DEBUG_BUS1__DCPG_CONTROL_DEBUG_BUS1_MASK                                                 0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS2
+#define DCPG_CONTROL_DEBUG_BUS2__DCPG_CONTROL_DEBUG_BUS2__SHIFT                                               0x0
+#define DCPG_CONTROL_DEBUG_BUS2__DCPG_CONTROL_DEBUG_BUS2_MASK                                                 0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS3
+#define DCPG_CONTROL_DEBUG_BUS3__DCPG_CONTROL_DEBUG_BUS3__SHIFT                                               0x0
+#define DCPG_CONTROL_DEBUG_BUS3__DCPG_CONTROL_DEBUG_BUS3_MASK                                                 0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS16
+#define DCPG_CONTROL_DEBUG_BUS16__DCPG_CONTROL_DEBUG_BUS16__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS16__DCPG_CONTROL_DEBUG_BUS16_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS17
+#define DCPG_CONTROL_DEBUG_BUS17__DCPG_CONTROL_DEBUG_BUS17__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS17__DCPG_CONTROL_DEBUG_BUS17_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS18
+#define DCPG_CONTROL_DEBUG_BUS18__DCPG_CONTROL_DEBUG_BUS18__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS18__DCPG_CONTROL_DEBUG_BUS18_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS19
+#define DCPG_CONTROL_DEBUG_BUS19__DCPG_CONTROL_DEBUG_BUS19__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS19__DCPG_CONTROL_DEBUG_BUS19_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS22
+#define DCPG_CONTROL_DEBUG_BUS22__DCPG_CONTROL_DEBUG_BUS22__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS22__DCPG_CONTROL_DEBUG_BUS22_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS23
+#define DCPG_CONTROL_DEBUG_BUS23__DCPG_CONTROL_DEBUG_BUS23__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS23__DCPG_CONTROL_DEBUG_BUS23_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS24
+#define DCPG_CONTROL_DEBUG_BUS24__DCPG_CONTROL_DEBUG_BUS24__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS24__DCPG_CONTROL_DEBUG_BUS24_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS25
+#define DCPG_CONTROL_DEBUG_BUS25__DCPG_CONTROL_DEBUG_BUS25__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS25__DCPG_CONTROL_DEBUG_BUS25_MASK                                               0xFFFFFFFFL
+//DCPG_CONTROL_DEBUG_BUS26
+#define DCPG_CONTROL_DEBUG_BUS26__DCPG_CONTROL_DEBUG_BUS26__SHIFT                                             0x0
+#define DCPG_CONTROL_DEBUG_BUS26__DCPG_CONTROL_DEBUG_BUS26_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: dp0_dpdebugind
+//DP0_DP_DEBUG_ID
+#define DP0_DP_DEBUG_ID__DP_DEBUG_ID__SHIFT                                                                   0x0
+#define DP0_DP_DEBUG_ID__DP_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DP0_DP_DEBUG_K
+#define DP0_DP_DEBUG_K__DP_DEBUG_K__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_K__DP_DEBUG_K_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_L
+#define DP0_DP_DEBUG_L__DP_DEBUG_L__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_L__DP_DEBUG_L_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_M
+#define DP0_DP_DEBUG_M__DP_DEBUG_M__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_M__DP_DEBUG_M_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_G
+#define DP0_DP_DEBUG_G__DP_DEBUG_G__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_G__DP_DEBUG_G_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_O
+#define DP0_DP_DEBUG_O__DP_DEBUG_O__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_O__DP_DEBUG_O_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_P
+#define DP0_DP_DEBUG_P__DP_DEBUG_P__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_P__DP_DEBUG_P_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_Q
+#define DP0_DP_DEBUG_Q__DP_DEBUG_Q__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_Q__DP_DEBUG_Q_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_R
+#define DP0_DP_DEBUG_R__DP_DEBUG_R__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_R__DP_DEBUG_R_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_S
+#define DP0_DP_DEBUG_S__DP_DEBUG_S__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_S__DP_DEBUG_S_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp0_dpfedebugind
+//DP0_DP_FE_DEBUG_ID
+#define DP0_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID__SHIFT                                                             0x0
+#define DP0_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DP0_DP_FE_SYMCLK_DEBUG_DATA_0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0__SHIFT                                       0x0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0_MASK                                         0xFFFFFFFFL
+//DP0_DP_FE_SYMCLK_DEBUG_DATA_1
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1__SHIFT                                       0x0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1_MASK                                         0xFFFFFFFFL
+//DP0_DP_FE_SYMCLK_DEBUG_DATA_2
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2__SHIFT                                       0x0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2_MASK                                         0xFFFFFFFFL
+//DP0_DP_FE_SYMCLK_DEBUG_DATA_3
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3__SHIFT                                       0x0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3_MASK                                         0xFFFFFFFFL
+//DP0_DP_FE_SYMCLK_DEBUG_DATA_4
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4__SHIFT                                       0x0
+#define DP0_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4_MASK                                         0xFFFFFFFFL
+//DP0_DP_DEBUG_T
+#define DP0_DP_DEBUG_T__DP_DEBUG_T__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_T__DP_DEBUG_T_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_U
+#define DP0_DP_DEBUG_U__DP_DEBUG_U__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_U__DP_DEBUG_U_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_V
+#define DP0_DP_DEBUG_V__DP_DEBUG_V__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_V__DP_DEBUG_V_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_W
+#define DP0_DP_DEBUG_W__DP_DEBUG_W__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_W__DP_DEBUG_W_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_X
+#define DP0_DP_DEBUG_X__DP_DEBUG_X__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_X__DP_DEBUG_X_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_Y
+#define DP0_DP_DEBUG_Y__DP_DEBUG_Y__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_Y__DP_DEBUG_Y_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_Z
+#define DP0_DP_DEBUG_Z__DP_DEBUG_Z__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_Z__DP_DEBUG_Z_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_1A
+#define DP0_DP_DEBUG_1A__DP_DEBUG_1A__SHIFT                                                                   0x0
+#define DP0_DP_DEBUG_1A__DP_DEBUG_1A_MASK                                                                     0xFFFFFFFFL
+//DP0_DP_DEBUG_1B
+#define DP0_DP_DEBUG_1B__DP_DEBUG_1B__SHIFT                                                                   0x0
+#define DP0_DP_DEBUG_1B__DP_DEBUG_1B_MASK                                                                     0xFFFFFFFFL
+//DP0_DP_DEBUG_I
+#define DP0_DP_DEBUG_I__DP_DEBUG_I__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_I__DP_DEBUG_I_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_J
+#define DP0_DP_DEBUG_J__DP_DEBUG_J__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_J__DP_DEBUG_J_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_N
+#define DP0_DP_DEBUG_N__DP_DEBUG_N__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_N__DP_DEBUG_N_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_H
+#define DP0_DP_DEBUG_H__DP_DEBUG_H__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_H__DP_DEBUG_H_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_A
+#define DP0_DP_DEBUG_A__DP_DEBUG_A__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_A__DP_DEBUG_A_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_B
+#define DP0_DP_DEBUG_B__DP_DEBUG_B__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_B__DP_DEBUG_B_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_C
+#define DP0_DP_DEBUG_C__DP_DEBUG_C__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_C__DP_DEBUG_C_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_D
+#define DP0_DP_DEBUG_D__DP_DEBUG_D__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_D__DP_DEBUG_D_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_E
+#define DP0_DP_DEBUG_E__DP_DEBUG_E__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_E__DP_DEBUG_E_MASK                                                                       0xFFFFFFFFL
+//DP0_DP_DEBUG_F
+#define DP0_DP_DEBUG_F__DP_DEBUG_F__SHIFT                                                                     0x0
+#define DP0_DP_DEBUG_F__DP_DEBUG_F_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp0_dpfe_dprefclk_debugind
+//DP0_DP_FE_DPREFCLK_DEBUG_ID
+#define DP0_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID__SHIFT                                           0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID_MASK                                             0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_1
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_2
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_3
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_4
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_5
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_6
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_7
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_8
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_9
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9__SHIFT                                   0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9_MASK                                     0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_10
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10__SHIFT                                 0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10_MASK                                   0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_11
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11__SHIFT                                 0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11_MASK                                   0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_12
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12__SHIFT                                 0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12_MASK                                   0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_13
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13__SHIFT                                 0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13_MASK                                   0xFFFFFFFFL
+//DP0_DP_FE_DPREFCLK_DEBUG_DATA_14
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14__SHIFT                                 0x0
+#define DP0_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: dig0_digdebugind
+//DIG0_DIG_BE_CLKGEN_DEBUG0
+#define DIG0_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0__SHIFT                                                0x0
+#define DIG0_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0_MASK                                                  0xFFFFFFFFL
+
+
+// addressBlock: dig0_digfedebugind
+//DIG0_DIG_FE_DEBUG_ID
+#define DIG0_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID__SHIFT                                                          0x0
+#define DIG0_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DIG0_DIG_VPG_DEBUG0
+#define DIG0_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0__SHIFT                                                            0x0
+#define DIG0_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0_MASK                                                              0xFFFFFFFFL
+//DIG0_DIG_VPG_DEBUG1
+#define DIG0_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1__SHIFT                                                            0x0
+#define DIG0_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1_MASK                                                              0xFFFFFFFFL
+//DIG0_DIG_VPG_DEBUG2
+#define DIG0_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2__SHIFT                                                            0x0
+#define DIG0_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2_MASK                                                              0xFFFFFFFFL
+//DIG0_DIG_VPG_DEBUG3
+#define DIG0_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3__SHIFT                                                            0x0
+#define DIG0_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: dp1_dpdebugind
+//DP1_DP_DEBUG_ID
+#define DP1_DP_DEBUG_ID__DP_DEBUG_ID__SHIFT                                                                   0x0
+#define DP1_DP_DEBUG_ID__DP_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DP1_DP_DEBUG_K
+#define DP1_DP_DEBUG_K__DP_DEBUG_K__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_K__DP_DEBUG_K_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_L
+#define DP1_DP_DEBUG_L__DP_DEBUG_L__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_L__DP_DEBUG_L_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_M
+#define DP1_DP_DEBUG_M__DP_DEBUG_M__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_M__DP_DEBUG_M_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_G
+#define DP1_DP_DEBUG_G__DP_DEBUG_G__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_G__DP_DEBUG_G_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_O
+#define DP1_DP_DEBUG_O__DP_DEBUG_O__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_O__DP_DEBUG_O_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_P
+#define DP1_DP_DEBUG_P__DP_DEBUG_P__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_P__DP_DEBUG_P_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_Q
+#define DP1_DP_DEBUG_Q__DP_DEBUG_Q__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_Q__DP_DEBUG_Q_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_R
+#define DP1_DP_DEBUG_R__DP_DEBUG_R__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_R__DP_DEBUG_R_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_S
+#define DP1_DP_DEBUG_S__DP_DEBUG_S__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_S__DP_DEBUG_S_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp1_dpfedebugind
+//DP1_DP_FE_DEBUG_ID
+#define DP1_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID__SHIFT                                                             0x0
+#define DP1_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DP1_DP_FE_SYMCLK_DEBUG_DATA_0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0__SHIFT                                       0x0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0_MASK                                         0xFFFFFFFFL
+//DP1_DP_FE_SYMCLK_DEBUG_DATA_1
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1__SHIFT                                       0x0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1_MASK                                         0xFFFFFFFFL
+//DP1_DP_FE_SYMCLK_DEBUG_DATA_2
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2__SHIFT                                       0x0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2_MASK                                         0xFFFFFFFFL
+//DP1_DP_FE_SYMCLK_DEBUG_DATA_3
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3__SHIFT                                       0x0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3_MASK                                         0xFFFFFFFFL
+//DP1_DP_FE_SYMCLK_DEBUG_DATA_4
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4__SHIFT                                       0x0
+#define DP1_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4_MASK                                         0xFFFFFFFFL
+//DP1_DP_DEBUG_T
+#define DP1_DP_DEBUG_T__DP_DEBUG_T__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_T__DP_DEBUG_T_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_U
+#define DP1_DP_DEBUG_U__DP_DEBUG_U__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_U__DP_DEBUG_U_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_V
+#define DP1_DP_DEBUG_V__DP_DEBUG_V__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_V__DP_DEBUG_V_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_W
+#define DP1_DP_DEBUG_W__DP_DEBUG_W__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_W__DP_DEBUG_W_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_X
+#define DP1_DP_DEBUG_X__DP_DEBUG_X__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_X__DP_DEBUG_X_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_Y
+#define DP1_DP_DEBUG_Y__DP_DEBUG_Y__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_Y__DP_DEBUG_Y_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_Z
+#define DP1_DP_DEBUG_Z__DP_DEBUG_Z__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_Z__DP_DEBUG_Z_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_1A
+#define DP1_DP_DEBUG_1A__DP_DEBUG_1A__SHIFT                                                                   0x0
+#define DP1_DP_DEBUG_1A__DP_DEBUG_1A_MASK                                                                     0xFFFFFFFFL
+//DP1_DP_DEBUG_1B
+#define DP1_DP_DEBUG_1B__DP_DEBUG_1B__SHIFT                                                                   0x0
+#define DP1_DP_DEBUG_1B__DP_DEBUG_1B_MASK                                                                     0xFFFFFFFFL
+//DP1_DP_DEBUG_I
+#define DP1_DP_DEBUG_I__DP_DEBUG_I__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_I__DP_DEBUG_I_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_J
+#define DP1_DP_DEBUG_J__DP_DEBUG_J__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_J__DP_DEBUG_J_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_N
+#define DP1_DP_DEBUG_N__DP_DEBUG_N__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_N__DP_DEBUG_N_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_H
+#define DP1_DP_DEBUG_H__DP_DEBUG_H__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_H__DP_DEBUG_H_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_A
+#define DP1_DP_DEBUG_A__DP_DEBUG_A__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_A__DP_DEBUG_A_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_B
+#define DP1_DP_DEBUG_B__DP_DEBUG_B__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_B__DP_DEBUG_B_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_C
+#define DP1_DP_DEBUG_C__DP_DEBUG_C__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_C__DP_DEBUG_C_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_D
+#define DP1_DP_DEBUG_D__DP_DEBUG_D__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_D__DP_DEBUG_D_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_E
+#define DP1_DP_DEBUG_E__DP_DEBUG_E__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_E__DP_DEBUG_E_MASK                                                                       0xFFFFFFFFL
+//DP1_DP_DEBUG_F
+#define DP1_DP_DEBUG_F__DP_DEBUG_F__SHIFT                                                                     0x0
+#define DP1_DP_DEBUG_F__DP_DEBUG_F_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp1_dpfe_dprefclk_debugind
+//DP1_DP_FE_DPREFCLK_DEBUG_ID
+#define DP1_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID__SHIFT                                           0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID_MASK                                             0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_1
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_2
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_3
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_4
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_5
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_6
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_7
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_8
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_9
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9__SHIFT                                   0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9_MASK                                     0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_10
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10__SHIFT                                 0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10_MASK                                   0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_11
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11__SHIFT                                 0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11_MASK                                   0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_12
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12__SHIFT                                 0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12_MASK                                   0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_13
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13__SHIFT                                 0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13_MASK                                   0xFFFFFFFFL
+//DP1_DP_FE_DPREFCLK_DEBUG_DATA_14
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14__SHIFT                                 0x0
+#define DP1_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: dig1_digdebugind
+//DIG1_DIG_BE_CLKGEN_DEBUG0
+#define DIG1_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0__SHIFT                                                0x0
+#define DIG1_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0_MASK                                                  0xFFFFFFFFL
+
+
+// addressBlock: dig1_digfedebugind
+//DIG1_DIG_FE_DEBUG_ID
+#define DIG1_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID__SHIFT                                                          0x0
+#define DIG1_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DIG1_DIG_VPG_DEBUG0
+#define DIG1_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0__SHIFT                                                            0x0
+#define DIG1_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0_MASK                                                              0xFFFFFFFFL
+//DIG1_DIG_VPG_DEBUG1
+#define DIG1_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1__SHIFT                                                            0x0
+#define DIG1_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1_MASK                                                              0xFFFFFFFFL
+//DIG1_DIG_VPG_DEBUG2
+#define DIG1_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2__SHIFT                                                            0x0
+#define DIG1_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2_MASK                                                              0xFFFFFFFFL
+//DIG1_DIG_VPG_DEBUG3
+#define DIG1_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3__SHIFT                                                            0x0
+#define DIG1_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: dp2_dpdebugind
+//DP2_DP_DEBUG_ID
+#define DP2_DP_DEBUG_ID__DP_DEBUG_ID__SHIFT                                                                   0x0
+#define DP2_DP_DEBUG_ID__DP_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DP2_DP_DEBUG_K
+#define DP2_DP_DEBUG_K__DP_DEBUG_K__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_K__DP_DEBUG_K_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_L
+#define DP2_DP_DEBUG_L__DP_DEBUG_L__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_L__DP_DEBUG_L_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_M
+#define DP2_DP_DEBUG_M__DP_DEBUG_M__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_M__DP_DEBUG_M_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_G
+#define DP2_DP_DEBUG_G__DP_DEBUG_G__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_G__DP_DEBUG_G_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_O
+#define DP2_DP_DEBUG_O__DP_DEBUG_O__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_O__DP_DEBUG_O_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_P
+#define DP2_DP_DEBUG_P__DP_DEBUG_P__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_P__DP_DEBUG_P_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_Q
+#define DP2_DP_DEBUG_Q__DP_DEBUG_Q__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_Q__DP_DEBUG_Q_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_R
+#define DP2_DP_DEBUG_R__DP_DEBUG_R__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_R__DP_DEBUG_R_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_S
+#define DP2_DP_DEBUG_S__DP_DEBUG_S__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_S__DP_DEBUG_S_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp2_dpfedebugind
+//DP2_DP_FE_DEBUG_ID
+#define DP2_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID__SHIFT                                                             0x0
+#define DP2_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DP2_DP_FE_SYMCLK_DEBUG_DATA_0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0__SHIFT                                       0x0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0_MASK                                         0xFFFFFFFFL
+//DP2_DP_FE_SYMCLK_DEBUG_DATA_1
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1__SHIFT                                       0x0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1_MASK                                         0xFFFFFFFFL
+//DP2_DP_FE_SYMCLK_DEBUG_DATA_2
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2__SHIFT                                       0x0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2_MASK                                         0xFFFFFFFFL
+//DP2_DP_FE_SYMCLK_DEBUG_DATA_3
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3__SHIFT                                       0x0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3_MASK                                         0xFFFFFFFFL
+//DP2_DP_FE_SYMCLK_DEBUG_DATA_4
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4__SHIFT                                       0x0
+#define DP2_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4_MASK                                         0xFFFFFFFFL
+//DP2_DP_DEBUG_T
+#define DP2_DP_DEBUG_T__DP_DEBUG_T__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_T__DP_DEBUG_T_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_U
+#define DP2_DP_DEBUG_U__DP_DEBUG_U__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_U__DP_DEBUG_U_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_V
+#define DP2_DP_DEBUG_V__DP_DEBUG_V__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_V__DP_DEBUG_V_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_W
+#define DP2_DP_DEBUG_W__DP_DEBUG_W__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_W__DP_DEBUG_W_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_X
+#define DP2_DP_DEBUG_X__DP_DEBUG_X__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_X__DP_DEBUG_X_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_Y
+#define DP2_DP_DEBUG_Y__DP_DEBUG_Y__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_Y__DP_DEBUG_Y_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_Z
+#define DP2_DP_DEBUG_Z__DP_DEBUG_Z__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_Z__DP_DEBUG_Z_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_1A
+#define DP2_DP_DEBUG_1A__DP_DEBUG_1A__SHIFT                                                                   0x0
+#define DP2_DP_DEBUG_1A__DP_DEBUG_1A_MASK                                                                     0xFFFFFFFFL
+//DP2_DP_DEBUG_1B
+#define DP2_DP_DEBUG_1B__DP_DEBUG_1B__SHIFT                                                                   0x0
+#define DP2_DP_DEBUG_1B__DP_DEBUG_1B_MASK                                                                     0xFFFFFFFFL
+//DP2_DP_DEBUG_I
+#define DP2_DP_DEBUG_I__DP_DEBUG_I__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_I__DP_DEBUG_I_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_J
+#define DP2_DP_DEBUG_J__DP_DEBUG_J__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_J__DP_DEBUG_J_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_N
+#define DP2_DP_DEBUG_N__DP_DEBUG_N__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_N__DP_DEBUG_N_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_H
+#define DP2_DP_DEBUG_H__DP_DEBUG_H__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_H__DP_DEBUG_H_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_A
+#define DP2_DP_DEBUG_A__DP_DEBUG_A__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_A__DP_DEBUG_A_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_B
+#define DP2_DP_DEBUG_B__DP_DEBUG_B__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_B__DP_DEBUG_B_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_C
+#define DP2_DP_DEBUG_C__DP_DEBUG_C__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_C__DP_DEBUG_C_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_D
+#define DP2_DP_DEBUG_D__DP_DEBUG_D__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_D__DP_DEBUG_D_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_E
+#define DP2_DP_DEBUG_E__DP_DEBUG_E__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_E__DP_DEBUG_E_MASK                                                                       0xFFFFFFFFL
+//DP2_DP_DEBUG_F
+#define DP2_DP_DEBUG_F__DP_DEBUG_F__SHIFT                                                                     0x0
+#define DP2_DP_DEBUG_F__DP_DEBUG_F_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp2_dpfe_dprefclk_debugind
+//DP2_DP_FE_DPREFCLK_DEBUG_ID
+#define DP2_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID__SHIFT                                           0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID_MASK                                             0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_1
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_2
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_3
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_4
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_5
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_6
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_7
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_8
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_9
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9__SHIFT                                   0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9_MASK                                     0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_10
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10__SHIFT                                 0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10_MASK                                   0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_11
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11__SHIFT                                 0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11_MASK                                   0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_12
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12__SHIFT                                 0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12_MASK                                   0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_13
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13__SHIFT                                 0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13_MASK                                   0xFFFFFFFFL
+//DP2_DP_FE_DPREFCLK_DEBUG_DATA_14
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14__SHIFT                                 0x0
+#define DP2_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: dig2_digdebugind
+//DIG2_DIG_BE_CLKGEN_DEBUG0
+#define DIG2_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0__SHIFT                                                0x0
+#define DIG2_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0_MASK                                                  0xFFFFFFFFL
+
+
+// addressBlock: dig2_digfedebugind
+//DIG2_DIG_FE_DEBUG_ID
+#define DIG2_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID__SHIFT                                                          0x0
+#define DIG2_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DIG2_DIG_VPG_DEBUG0
+#define DIG2_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0__SHIFT                                                            0x0
+#define DIG2_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0_MASK                                                              0xFFFFFFFFL
+//DIG2_DIG_VPG_DEBUG1
+#define DIG2_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1__SHIFT                                                            0x0
+#define DIG2_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1_MASK                                                              0xFFFFFFFFL
+//DIG2_DIG_VPG_DEBUG2
+#define DIG2_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2__SHIFT                                                            0x0
+#define DIG2_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2_MASK                                                              0xFFFFFFFFL
+//DIG2_DIG_VPG_DEBUG3
+#define DIG2_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3__SHIFT                                                            0x0
+#define DIG2_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: dp3_dpdebugind
+//DP3_DP_DEBUG_ID
+#define DP3_DP_DEBUG_ID__DP_DEBUG_ID__SHIFT                                                                   0x0
+#define DP3_DP_DEBUG_ID__DP_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DP3_DP_DEBUG_K
+#define DP3_DP_DEBUG_K__DP_DEBUG_K__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_K__DP_DEBUG_K_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_L
+#define DP3_DP_DEBUG_L__DP_DEBUG_L__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_L__DP_DEBUG_L_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_M
+#define DP3_DP_DEBUG_M__DP_DEBUG_M__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_M__DP_DEBUG_M_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_G
+#define DP3_DP_DEBUG_G__DP_DEBUG_G__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_G__DP_DEBUG_G_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_O
+#define DP3_DP_DEBUG_O__DP_DEBUG_O__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_O__DP_DEBUG_O_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_P
+#define DP3_DP_DEBUG_P__DP_DEBUG_P__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_P__DP_DEBUG_P_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_Q
+#define DP3_DP_DEBUG_Q__DP_DEBUG_Q__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_Q__DP_DEBUG_Q_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_R
+#define DP3_DP_DEBUG_R__DP_DEBUG_R__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_R__DP_DEBUG_R_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_S
+#define DP3_DP_DEBUG_S__DP_DEBUG_S__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_S__DP_DEBUG_S_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp3_dpfedebugind
+//DP3_DP_FE_DEBUG_ID
+#define DP3_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID__SHIFT                                                             0x0
+#define DP3_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DP3_DP_FE_SYMCLK_DEBUG_DATA_0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0__SHIFT                                       0x0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0_MASK                                         0xFFFFFFFFL
+//DP3_DP_FE_SYMCLK_DEBUG_DATA_1
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1__SHIFT                                       0x0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1_MASK                                         0xFFFFFFFFL
+//DP3_DP_FE_SYMCLK_DEBUG_DATA_2
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2__SHIFT                                       0x0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2_MASK                                         0xFFFFFFFFL
+//DP3_DP_FE_SYMCLK_DEBUG_DATA_3
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3__SHIFT                                       0x0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3_MASK                                         0xFFFFFFFFL
+//DP3_DP_FE_SYMCLK_DEBUG_DATA_4
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4__SHIFT                                       0x0
+#define DP3_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4_MASK                                         0xFFFFFFFFL
+//DP3_DP_DEBUG_T
+#define DP3_DP_DEBUG_T__DP_DEBUG_T__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_T__DP_DEBUG_T_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_U
+#define DP3_DP_DEBUG_U__DP_DEBUG_U__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_U__DP_DEBUG_U_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_V
+#define DP3_DP_DEBUG_V__DP_DEBUG_V__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_V__DP_DEBUG_V_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_W
+#define DP3_DP_DEBUG_W__DP_DEBUG_W__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_W__DP_DEBUG_W_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_X
+#define DP3_DP_DEBUG_X__DP_DEBUG_X__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_X__DP_DEBUG_X_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_Y
+#define DP3_DP_DEBUG_Y__DP_DEBUG_Y__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_Y__DP_DEBUG_Y_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_Z
+#define DP3_DP_DEBUG_Z__DP_DEBUG_Z__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_Z__DP_DEBUG_Z_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_1A
+#define DP3_DP_DEBUG_1A__DP_DEBUG_1A__SHIFT                                                                   0x0
+#define DP3_DP_DEBUG_1A__DP_DEBUG_1A_MASK                                                                     0xFFFFFFFFL
+//DP3_DP_DEBUG_1B
+#define DP3_DP_DEBUG_1B__DP_DEBUG_1B__SHIFT                                                                   0x0
+#define DP3_DP_DEBUG_1B__DP_DEBUG_1B_MASK                                                                     0xFFFFFFFFL
+//DP3_DP_DEBUG_I
+#define DP3_DP_DEBUG_I__DP_DEBUG_I__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_I__DP_DEBUG_I_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_J
+#define DP3_DP_DEBUG_J__DP_DEBUG_J__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_J__DP_DEBUG_J_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_N
+#define DP3_DP_DEBUG_N__DP_DEBUG_N__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_N__DP_DEBUG_N_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_H
+#define DP3_DP_DEBUG_H__DP_DEBUG_H__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_H__DP_DEBUG_H_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_A
+#define DP3_DP_DEBUG_A__DP_DEBUG_A__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_A__DP_DEBUG_A_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_B
+#define DP3_DP_DEBUG_B__DP_DEBUG_B__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_B__DP_DEBUG_B_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_C
+#define DP3_DP_DEBUG_C__DP_DEBUG_C__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_C__DP_DEBUG_C_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_D
+#define DP3_DP_DEBUG_D__DP_DEBUG_D__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_D__DP_DEBUG_D_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_E
+#define DP3_DP_DEBUG_E__DP_DEBUG_E__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_E__DP_DEBUG_E_MASK                                                                       0xFFFFFFFFL
+//DP3_DP_DEBUG_F
+#define DP3_DP_DEBUG_F__DP_DEBUG_F__SHIFT                                                                     0x0
+#define DP3_DP_DEBUG_F__DP_DEBUG_F_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp3_dpfe_dprefclk_debugind
+//DP3_DP_FE_DPREFCLK_DEBUG_ID
+#define DP3_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID__SHIFT                                           0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID_MASK                                             0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_1
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_2
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_3
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_4
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_5
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_6
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_7
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_8
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_9
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9__SHIFT                                   0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9_MASK                                     0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_10
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10__SHIFT                                 0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10_MASK                                   0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_11
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11__SHIFT                                 0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11_MASK                                   0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_12
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12__SHIFT                                 0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12_MASK                                   0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_13
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13__SHIFT                                 0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13_MASK                                   0xFFFFFFFFL
+//DP3_DP_FE_DPREFCLK_DEBUG_DATA_14
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14__SHIFT                                 0x0
+#define DP3_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: dig3_digdebugind
+//DIG3_DIG_BE_CLKGEN_DEBUG0
+#define DIG3_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0__SHIFT                                                0x0
+#define DIG3_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0_MASK                                                  0xFFFFFFFFL
+
+
+// addressBlock: dig3_digfedebugind
+//DIG3_DIG_FE_DEBUG_ID
+#define DIG3_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID__SHIFT                                                          0x0
+#define DIG3_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DIG3_DIG_VPG_DEBUG0
+#define DIG3_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0__SHIFT                                                            0x0
+#define DIG3_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0_MASK                                                              0xFFFFFFFFL
+//DIG3_DIG_VPG_DEBUG1
+#define DIG3_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1__SHIFT                                                            0x0
+#define DIG3_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1_MASK                                                              0xFFFFFFFFL
+//DIG3_DIG_VPG_DEBUG2
+#define DIG3_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2__SHIFT                                                            0x0
+#define DIG3_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2_MASK                                                              0xFFFFFFFFL
+//DIG3_DIG_VPG_DEBUG3
+#define DIG3_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3__SHIFT                                                            0x0
+#define DIG3_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: dp4_dpdebugind
+//DP4_DP_DEBUG_ID
+#define DP4_DP_DEBUG_ID__DP_DEBUG_ID__SHIFT                                                                   0x0
+#define DP4_DP_DEBUG_ID__DP_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DP4_DP_DEBUG_K
+#define DP4_DP_DEBUG_K__DP_DEBUG_K__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_K__DP_DEBUG_K_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_L
+#define DP4_DP_DEBUG_L__DP_DEBUG_L__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_L__DP_DEBUG_L_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_M
+#define DP4_DP_DEBUG_M__DP_DEBUG_M__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_M__DP_DEBUG_M_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_G
+#define DP4_DP_DEBUG_G__DP_DEBUG_G__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_G__DP_DEBUG_G_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_O
+#define DP4_DP_DEBUG_O__DP_DEBUG_O__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_O__DP_DEBUG_O_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_P
+#define DP4_DP_DEBUG_P__DP_DEBUG_P__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_P__DP_DEBUG_P_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_Q
+#define DP4_DP_DEBUG_Q__DP_DEBUG_Q__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_Q__DP_DEBUG_Q_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_R
+#define DP4_DP_DEBUG_R__DP_DEBUG_R__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_R__DP_DEBUG_R_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_S
+#define DP4_DP_DEBUG_S__DP_DEBUG_S__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_S__DP_DEBUG_S_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp4_dpfedebugind
+//DP4_DP_FE_DEBUG_ID
+#define DP4_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID__SHIFT                                                             0x0
+#define DP4_DP_FE_DEBUG_ID__DP_FE_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DP4_DP_FE_SYMCLK_DEBUG_DATA_0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0__SHIFT                                       0x0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_0__DP_FE_SYMCLK_DEBUG_DATA_0_MASK                                         0xFFFFFFFFL
+//DP4_DP_FE_SYMCLK_DEBUG_DATA_1
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1__SHIFT                                       0x0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_1__DP_FE_SYMCLK_DEBUG_DATA_1_MASK                                         0xFFFFFFFFL
+//DP4_DP_FE_SYMCLK_DEBUG_DATA_2
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2__SHIFT                                       0x0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_2__DP_FE_SYMCLK_DEBUG_DATA_2_MASK                                         0xFFFFFFFFL
+//DP4_DP_FE_SYMCLK_DEBUG_DATA_3
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3__SHIFT                                       0x0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_3__DP_FE_SYMCLK_DEBUG_DATA_3_MASK                                         0xFFFFFFFFL
+//DP4_DP_FE_SYMCLK_DEBUG_DATA_4
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4__SHIFT                                       0x0
+#define DP4_DP_FE_SYMCLK_DEBUG_DATA_4__DP_FE_SYMCLK_DEBUG_DATA_4_MASK                                         0xFFFFFFFFL
+//DP4_DP_DEBUG_T
+#define DP4_DP_DEBUG_T__DP_DEBUG_T__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_T__DP_DEBUG_T_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_U
+#define DP4_DP_DEBUG_U__DP_DEBUG_U__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_U__DP_DEBUG_U_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_V
+#define DP4_DP_DEBUG_V__DP_DEBUG_V__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_V__DP_DEBUG_V_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_W
+#define DP4_DP_DEBUG_W__DP_DEBUG_W__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_W__DP_DEBUG_W_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_X
+#define DP4_DP_DEBUG_X__DP_DEBUG_X__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_X__DP_DEBUG_X_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_Y
+#define DP4_DP_DEBUG_Y__DP_DEBUG_Y__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_Y__DP_DEBUG_Y_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_Z
+#define DP4_DP_DEBUG_Z__DP_DEBUG_Z__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_Z__DP_DEBUG_Z_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_1A
+#define DP4_DP_DEBUG_1A__DP_DEBUG_1A__SHIFT                                                                   0x0
+#define DP4_DP_DEBUG_1A__DP_DEBUG_1A_MASK                                                                     0xFFFFFFFFL
+//DP4_DP_DEBUG_1B
+#define DP4_DP_DEBUG_1B__DP_DEBUG_1B__SHIFT                                                                   0x0
+#define DP4_DP_DEBUG_1B__DP_DEBUG_1B_MASK                                                                     0xFFFFFFFFL
+//DP4_DP_DEBUG_I
+#define DP4_DP_DEBUG_I__DP_DEBUG_I__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_I__DP_DEBUG_I_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_J
+#define DP4_DP_DEBUG_J__DP_DEBUG_J__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_J__DP_DEBUG_J_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_N
+#define DP4_DP_DEBUG_N__DP_DEBUG_N__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_N__DP_DEBUG_N_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_H
+#define DP4_DP_DEBUG_H__DP_DEBUG_H__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_H__DP_DEBUG_H_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_A
+#define DP4_DP_DEBUG_A__DP_DEBUG_A__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_A__DP_DEBUG_A_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_B
+#define DP4_DP_DEBUG_B__DP_DEBUG_B__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_B__DP_DEBUG_B_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_C
+#define DP4_DP_DEBUG_C__DP_DEBUG_C__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_C__DP_DEBUG_C_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_D
+#define DP4_DP_DEBUG_D__DP_DEBUG_D__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_D__DP_DEBUG_D_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_E
+#define DP4_DP_DEBUG_E__DP_DEBUG_E__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_E__DP_DEBUG_E_MASK                                                                       0xFFFFFFFFL
+//DP4_DP_DEBUG_F
+#define DP4_DP_DEBUG_F__DP_DEBUG_F__SHIFT                                                                     0x0
+#define DP4_DP_DEBUG_F__DP_DEBUG_F_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: dp4_dpfe_dprefclk_debugind
+//DP4_DP_FE_DPREFCLK_DEBUG_ID
+#define DP4_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID__SHIFT                                           0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_ID__DP_FE_DPREFCLK_DEBUG_ID_MASK                                             0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_0__DP_FE_DPREFCLK_DEBUG_DATA_0_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_1
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_1__DP_FE_DPREFCLK_DEBUG_DATA_1_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_2
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_2__DP_FE_DPREFCLK_DEBUG_DATA_2_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_3
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_3__DP_FE_DPREFCLK_DEBUG_DATA_3_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_4
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_4__DP_FE_DPREFCLK_DEBUG_DATA_4_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_5
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_5__DP_FE_DPREFCLK_DEBUG_DATA_5_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_6
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_6__DP_FE_DPREFCLK_DEBUG_DATA_6_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_7
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_7__DP_FE_DPREFCLK_DEBUG_DATA_7_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_8
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_8__DP_FE_DPREFCLK_DEBUG_DATA_8_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_9
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9__SHIFT                                   0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_9__DP_FE_DPREFCLK_DEBUG_DATA_9_MASK                                     0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_10
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10__SHIFT                                 0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_10__DP_FE_DPREFCLK_DEBUG_DATA_10_MASK                                   0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_11
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11__SHIFT                                 0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_11__DP_FE_DPREFCLK_DEBUG_DATA_11_MASK                                   0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_12
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12__SHIFT                                 0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_12__DP_FE_DPREFCLK_DEBUG_DATA_12_MASK                                   0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_13
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13__SHIFT                                 0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_13__DP_FE_DPREFCLK_DEBUG_DATA_13_MASK                                   0xFFFFFFFFL
+//DP4_DP_FE_DPREFCLK_DEBUG_DATA_14
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14__SHIFT                                 0x0
+#define DP4_DP_FE_DPREFCLK_DEBUG_DATA_14__DP_FE_DPREFCLK_DEBUG_DATA_14_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: dig4_digdebugind
+//DIG4_DIG_BE_CLKGEN_DEBUG0
+#define DIG4_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0__SHIFT                                                0x0
+#define DIG4_DIG_BE_CLKGEN_DEBUG0__DIG_BE_CLKGEN_DEBUG0_MASK                                                  0xFFFFFFFFL
+
+
+// addressBlock: dig4_digfedebugind
+//DIG4_DIG_FE_DEBUG_ID
+#define DIG4_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID__SHIFT                                                          0x0
+#define DIG4_DIG_FE_DEBUG_ID__DIG_FE_DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DIG4_DIG_VPG_DEBUG0
+#define DIG4_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0__SHIFT                                                            0x0
+#define DIG4_DIG_VPG_DEBUG0__DIG_VPG_DEBUG0_MASK                                                              0xFFFFFFFFL
+//DIG4_DIG_VPG_DEBUG1
+#define DIG4_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1__SHIFT                                                            0x0
+#define DIG4_DIG_VPG_DEBUG1__DIG_VPG_DEBUG1_MASK                                                              0xFFFFFFFFL
+//DIG4_DIG_VPG_DEBUG2
+#define DIG4_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2__SHIFT                                                            0x0
+#define DIG4_DIG_VPG_DEBUG2__DIG_VPG_DEBUG2_MASK                                                              0xFFFFFFFFL
+//DIG4_DIG_VPG_DEBUG3
+#define DIG4_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3__SHIFT                                                            0x0
+#define DIG4_DIG_VPG_DEBUG3__DIG_VPG_DEBUG3_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: dio_misc_dio_miscdebugind
+//I2C_DEBUG_BUS
+#define I2C_DEBUG_BUS__I2C_DEBUG_BUS__SHIFT                                                                   0x12
+#define I2C_DEBUG_BUS__I2C_DEBUG_BUS_MASK                                                                     0xFFFC0000L
+//DIO_RBBMIF_DEBUG_0
+#define DIO_RBBMIF_DEBUG_0__DIO_RBBMIF_DEBUG_0__SHIFT                                                         0x0
+#define DIO_RBBMIF_DEBUG_0__DIO_RBBMIF_DEBUG_0_MASK                                                           0xFFFFFFFFL
+//DIO_RBBMIF_DEBUG_1
+#define DIO_RBBMIF_DEBUG_1__DIO_RBBMIF_DEBUG_1__SHIFT                                                         0x0
+#define DIO_RBBMIF_DEBUG_1__DIO_RBBMIF_DEBUG_1_MASK                                                           0xFFFFFFFFL
+//DIO_RBBMIF_DEBUG_2
+#define DIO_RBBMIF_DEBUG_2__DIO_RBBMIF_DEBUG_2__SHIFT                                                         0x0
+#define DIO_RBBMIF_DEBUG_2__DIO_RBBMIF_DEBUG_2_MASK                                                           0xFFFFFFFFL
+//DIO_RBBMIF_DEBUG_3
+#define DIO_RBBMIF_DEBUG_3__DIO_RBBMIF_DEBUG_3__SHIFT                                                         0x0
+#define DIO_RBBMIF_DEBUG_3__DIO_RBBMIF_DEBUG_3_MASK                                                           0xFFFFFFFFL
+//DIGA_RBBMIF_DEBUG_0
+#define DIGA_RBBMIF_DEBUG_0__DIGA_RBBMIF_DEBUG_0__SHIFT                                                       0x0
+#define DIGA_RBBMIF_DEBUG_0__DIGA_RBBMIF_DEBUG_0_MASK                                                         0xFFFFFFFFL
+//DIGA_RBBMIF_DEBUG_1
+#define DIGA_RBBMIF_DEBUG_1__DIGA_RBBMIF_DEBUG_1__SHIFT                                                       0x0
+#define DIGA_RBBMIF_DEBUG_1__DIGA_RBBMIF_DEBUG_1_MASK                                                         0xFFFFFFFFL
+//DIGA_RBBMIF_DEBUG_2
+#define DIGA_RBBMIF_DEBUG_2__DIGA_RBBMIF_DEBUG_2__SHIFT                                                       0x0
+#define DIGA_RBBMIF_DEBUG_2__DIGA_RBBMIF_DEBUG_2_MASK                                                         0xFFFFFFFFL
+//DIGA_RBBMIF_DEBUG_3
+#define DIGA_RBBMIF_DEBUG_3__DIGA_RBBMIF_DEBUG_3__SHIFT                                                       0x0
+#define DIGA_RBBMIF_DEBUG_3__DIGA_RBBMIF_DEBUG_3_MASK                                                         0xFFFFFFFFL
+//DIGB_RBBMIF_DEBUG_0
+#define DIGB_RBBMIF_DEBUG_0__DIGB_RBBMIF_DEBUG_0__SHIFT                                                       0x0
+#define DIGB_RBBMIF_DEBUG_0__DIGB_RBBMIF_DEBUG_0_MASK                                                         0xFFFFFFFFL
+//DIGB_RBBMIF_DEBUG_1
+#define DIGB_RBBMIF_DEBUG_1__DIGB_RBBMIF_DEBUG_1__SHIFT                                                       0x0
+#define DIGB_RBBMIF_DEBUG_1__DIGB_RBBMIF_DEBUG_1_MASK                                                         0xFFFFFFFFL
+//DIGB_RBBMIF_DEBUG_2
+#define DIGB_RBBMIF_DEBUG_2__DIGB_RBBMIF_DEBUG_2__SHIFT                                                       0x0
+#define DIGB_RBBMIF_DEBUG_2__DIGB_RBBMIF_DEBUG_2_MASK                                                         0xFFFFFFFFL
+//DIGB_RBBMIF_DEBUG_3
+#define DIGB_RBBMIF_DEBUG_3__DIGB_RBBMIF_DEBUG_3__SHIFT                                                       0x0
+#define DIGB_RBBMIF_DEBUG_3__DIGB_RBBMIF_DEBUG_3_MASK                                                         0xFFFFFFFFL
+//DIGC_RBBMIF_DEBUG_0
+#define DIGC_RBBMIF_DEBUG_0__DIGC_RBBMIF_DEBUG_0__SHIFT                                                       0x0
+#define DIGC_RBBMIF_DEBUG_0__DIGC_RBBMIF_DEBUG_0_MASK                                                         0xFFFFFFFFL
+//DIGC_RBBMIF_DEBUG_1
+#define DIGC_RBBMIF_DEBUG_1__DIGC_RBBMIF_DEBUG_1__SHIFT                                                       0x0
+#define DIGC_RBBMIF_DEBUG_1__DIGC_RBBMIF_DEBUG_1_MASK                                                         0xFFFFFFFFL
+//DIGC_RBBMIF_DEBUG_2
+#define DIGC_RBBMIF_DEBUG_2__DIGC_RBBMIF_DEBUG_2__SHIFT                                                       0x0
+#define DIGC_RBBMIF_DEBUG_2__DIGC_RBBMIF_DEBUG_2_MASK                                                         0xFFFFFFFFL
+//DIGC_RBBMIF_DEBUG_3
+#define DIGC_RBBMIF_DEBUG_3__DIGC_RBBMIF_DEBUG_3__SHIFT                                                       0x0
+#define DIGC_RBBMIF_DEBUG_3__DIGC_RBBMIF_DEBUG_3_MASK                                                         0xFFFFFFFFL
+//DIGD_RBBMIF_DEBUG_0
+#define DIGD_RBBMIF_DEBUG_0__DIGD_RBBMIF_DEBUG_0__SHIFT                                                       0x0
+#define DIGD_RBBMIF_DEBUG_0__DIGD_RBBMIF_DEBUG_0_MASK                                                         0xFFFFFFFFL
+//DIGD_RBBMIF_DEBUG_1
+#define DIGD_RBBMIF_DEBUG_1__DIGD_RBBMIF_DEBUG_1__SHIFT                                                       0x0
+#define DIGD_RBBMIF_DEBUG_1__DIGD_RBBMIF_DEBUG_1_MASK                                                         0xFFFFFFFFL
+//DIGD_RBBMIF_DEBUG_2
+#define DIGD_RBBMIF_DEBUG_2__DIGD_RBBMIF_DEBUG_2__SHIFT                                                       0x0
+#define DIGD_RBBMIF_DEBUG_2__DIGD_RBBMIF_DEBUG_2_MASK                                                         0xFFFFFFFFL
+//DIGD_RBBMIF_DEBUG_3
+#define DIGD_RBBMIF_DEBUG_3__DIGD_RBBMIF_DEBUG_3__SHIFT                                                       0x0
+#define DIGD_RBBMIF_DEBUG_3__DIGD_RBBMIF_DEBUG_3_MASK                                                         0xFFFFFFFFL
+//DIGE_RBBMIF_DEBUG_0
+#define DIGE_RBBMIF_DEBUG_0__DIGE_RBBMIF_DEBUG_0__SHIFT                                                       0x0
+#define DIGE_RBBMIF_DEBUG_0__DIGE_RBBMIF_DEBUG_0_MASK                                                         0xFFFFFFFFL
+//DIGE_RBBMIF_DEBUG_1
+#define DIGE_RBBMIF_DEBUG_1__DIGE_RBBMIF_DEBUG_1__SHIFT                                                       0x0
+#define DIGE_RBBMIF_DEBUG_1__DIGE_RBBMIF_DEBUG_1_MASK                                                         0xFFFFFFFFL
+//DIGE_RBBMIF_DEBUG_2
+#define DIGE_RBBMIF_DEBUG_2__DIGE_RBBMIF_DEBUG_2__SHIFT                                                       0x0
+#define DIGE_RBBMIF_DEBUG_2__DIGE_RBBMIF_DEBUG_2_MASK                                                         0xFFFFFFFFL
+//DIGE_RBBMIF_DEBUG_3
+#define DIGE_RBBMIF_DEBUG_3__DIGE_RBBMIF_DEBUG_3__SHIFT                                                       0x0
+#define DIGE_RBBMIF_DEBUG_3__DIGE_RBBMIF_DEBUG_3_MASK                                                         0xFFFFFFFFL
+//DIGA_DME_DEBUG
+#define DIGA_DME_DEBUG__DIGA_DME_DEBUG__SHIFT                                                                 0x0
+#define DIGA_DME_DEBUG__DIGA_DME_DEBUG_MASK                                                                   0xFFFFFFFFL
+//DIGB_DME_DEBUG
+#define DIGB_DME_DEBUG__DIGB_DME_DEBUG__SHIFT                                                                 0x0
+#define DIGB_DME_DEBUG__DIGB_DME_DEBUG_MASK                                                                   0xFFFFFFFFL
+//DIGC_DME_DEBUG
+#define DIGC_DME_DEBUG__DIGC_DME_DEBUG__SHIFT                                                                 0x0
+#define DIGC_DME_DEBUG__DIGC_DME_DEBUG_MASK                                                                   0xFFFFFFFFL
+//DIGD_DME_DEBUG
+#define DIGD_DME_DEBUG__DIGD_DME_DEBUG__SHIFT                                                                 0x0
+#define DIGD_DME_DEBUG__DIGD_DME_DEBUG_MASK                                                                   0xFFFFFFFFL
+//DIGE_DME_DEBUG
+#define DIGE_DME_DEBUG__DIGE_DME_DEBUG__SHIFT                                                                 0x0
+#define DIGE_DME_DEBUG__DIGE_DME_DEBUG_MASK                                                                   0xFFFFFFFFL
+//DIGA_RESYNC_FIFO_DEBUG
+#define DIGA_RESYNC_FIFO_DEBUG__DIGA_RESYNC_FIFO_DEBUG__SHIFT                                                 0x0
+#define DIGA_RESYNC_FIFO_DEBUG__DIGA_RESYNC_FIFO_DEBUG_MASK                                                   0xFFFFFFFFL
+//DIGB_RESYNC_FIFO_DEBUG
+#define DIGB_RESYNC_FIFO_DEBUG__DIGB_RESYNC_FIFO_DEBUG__SHIFT                                                 0x0
+#define DIGB_RESYNC_FIFO_DEBUG__DIGB_RESYNC_FIFO_DEBUG_MASK                                                   0xFFFFFFFFL
+//DIGC_RESYNC_FIFO_DEBUG
+#define DIGC_RESYNC_FIFO_DEBUG__DIGC_RESYNC_FIFO_DEBUG__SHIFT                                                 0x0
+#define DIGC_RESYNC_FIFO_DEBUG__DIGC_RESYNC_FIFO_DEBUG_MASK                                                   0xFFFFFFFFL
+//DIGD_RESYNC_FIFO_DEBUG
+#define DIGD_RESYNC_FIFO_DEBUG__DIGD_RESYNC_FIFO_DEBUG__SHIFT                                                 0x0
+#define DIGD_RESYNC_FIFO_DEBUG__DIGD_RESYNC_FIFO_DEBUG_MASK                                                   0xFFFFFFFFL
+//DIGE_RESYNC_FIFO_DEBUG
+#define DIGE_RESYNC_FIFO_DEBUG__DIGE_RESYNC_FIFO_DEBUG__SHIFT                                                 0x0
+#define DIGE_RESYNC_FIFO_DEBUG__DIGE_RESYNC_FIFO_DEBUG_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: apg_apg_socclk_debugind
+//APG_SOCCLK_DEBUG_ID
+#define APG_SOCCLK_DEBUG_ID__APG_SOCCLK_DEBUG_ID__SHIFT                                                       0x0
+#define APG_SOCCLK_DEBUG_ID__APG_SOCCLK_DEBUG_ID_MASK                                                         0xFFFFFFFFL
+//APG_SOCCLK_DEBUG0
+#define APG_SOCCLK_DEBUG0__APG_SOCCLK_DEBUG0__SHIFT                                                           0x0
+#define APG_SOCCLK_DEBUG0__APG_SOCCLK_DEBUG0_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: apg_apg_encclk_debugind
+//APG_ENCCLK_DEBUG_ID
+#define APG_ENCCLK_DEBUG_ID__APG_ENCCLK_DEBUG_ID__SHIFT                                                       0x0
+#define APG_ENCCLK_DEBUG_ID__APG_ENCCLK_DEBUG_ID_MASK                                                         0xFFFFFFFFL
+//APG_ENCCLK_DEBUG0
+#define APG_ENCCLK_DEBUG0__APG_ENCCLK_DEBUG0__SHIFT                                                           0x0
+#define APG_ENCCLK_DEBUG0__APG_ENCCLK_DEBUG0_MASK                                                             0xFFFFFFFFL
+//APG_ENCCLK_DEBUG1
+#define APG_ENCCLK_DEBUG1__APG_ENCCLK_DEBUG1__SHIFT                                                           0x0
+#define APG_ENCCLK_DEBUG1__APG_ENCCLK_DEBUG1_MASK                                                             0xFFFFFFFFL
+//APG_ENCCLK_DEBUG2
+#define APG_ENCCLK_DEBUG2__APG_ENCCLK_DEBUG2__SHIFT                                                           0x0
+#define APG_ENCCLK_DEBUG2__APG_ENCCLK_DEBUG2_MASK                                                             0xFFFFFFFFL
+//APG_ENCCLK_DEBUG3
+#define APG_ENCCLK_DEBUG3__APG_ENCCLK_DEBUG3__SHIFT                                                           0x0
+#define APG_ENCCLK_DEBUG3__APG_ENCCLK_DEBUG3_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dcoh_top_dcoh_topdebugind
+//DCOH_TOP_DEBUG_ID
+#define DCOH_TOP_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define DCOH_TOP_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: dp_aux0_auxdebugdispclkind
+//DP_AUX0_AUX_DEBUG_DISPCLK_ID
+#define DP_AUX0_AUX_DEBUG_DISPCLK_ID__DEBUG_ID__SHIFT                                                         0x0
+#define DP_AUX0_AUX_DEBUG_DISPCLK_ID__DEBUG_ID_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_H
+#define DP_AUX0_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_I
+#define DP_AUX0_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux0_auxdebugrefclkind
+//DP_AUX0_AUX_DEBUG_REFCLK_ID
+#define DP_AUX0_AUX_DEBUG_REFCLK_ID__DEBUG_ID__SHIFT                                                          0x0
+#define DP_AUX0_AUX_DEBUG_REFCLK_ID__DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_A
+#define DP_AUX0_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_B
+#define DP_AUX0_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_C
+#define DP_AUX0_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_D
+#define DP_AUX0_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_E
+#define DP_AUX0_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_F
+#define DP_AUX0_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_G
+#define DP_AUX0_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G_MASK                                                           0xFFFFFFFFL
+//DP_AUX0_DP_AUX_DEBUG_J
+#define DP_AUX0_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J__SHIFT                                                         0x0
+#define DP_AUX0_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux1_auxdebugdispclkind
+//DP_AUX1_AUX_DEBUG_DISPCLK_ID
+#define DP_AUX1_AUX_DEBUG_DISPCLK_ID__DEBUG_ID__SHIFT                                                         0x0
+#define DP_AUX1_AUX_DEBUG_DISPCLK_ID__DEBUG_ID_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_H
+#define DP_AUX1_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_I
+#define DP_AUX1_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux1_auxdebugrefclkind
+//DP_AUX1_AUX_DEBUG_REFCLK_ID
+#define DP_AUX1_AUX_DEBUG_REFCLK_ID__DEBUG_ID__SHIFT                                                          0x0
+#define DP_AUX1_AUX_DEBUG_REFCLK_ID__DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_A
+#define DP_AUX1_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_B
+#define DP_AUX1_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_C
+#define DP_AUX1_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_D
+#define DP_AUX1_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_E
+#define DP_AUX1_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_F
+#define DP_AUX1_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_G
+#define DP_AUX1_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G_MASK                                                           0xFFFFFFFFL
+//DP_AUX1_DP_AUX_DEBUG_J
+#define DP_AUX1_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J__SHIFT                                                         0x0
+#define DP_AUX1_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux2_auxdebugdispclkind
+//DP_AUX2_AUX_DEBUG_DISPCLK_ID
+#define DP_AUX2_AUX_DEBUG_DISPCLK_ID__DEBUG_ID__SHIFT                                                         0x0
+#define DP_AUX2_AUX_DEBUG_DISPCLK_ID__DEBUG_ID_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_H
+#define DP_AUX2_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_I
+#define DP_AUX2_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux2_auxdebugrefclkind
+//DP_AUX2_AUX_DEBUG_REFCLK_ID
+#define DP_AUX2_AUX_DEBUG_REFCLK_ID__DEBUG_ID__SHIFT                                                          0x0
+#define DP_AUX2_AUX_DEBUG_REFCLK_ID__DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_A
+#define DP_AUX2_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_B
+#define DP_AUX2_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_C
+#define DP_AUX2_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_D
+#define DP_AUX2_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_E
+#define DP_AUX2_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_F
+#define DP_AUX2_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_G
+#define DP_AUX2_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G_MASK                                                           0xFFFFFFFFL
+//DP_AUX2_DP_AUX_DEBUG_J
+#define DP_AUX2_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J__SHIFT                                                         0x0
+#define DP_AUX2_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux3_auxdebugdispclkind
+//DP_AUX3_AUX_DEBUG_DISPCLK_ID
+#define DP_AUX3_AUX_DEBUG_DISPCLK_ID__DEBUG_ID__SHIFT                                                         0x0
+#define DP_AUX3_AUX_DEBUG_DISPCLK_ID__DEBUG_ID_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_H
+#define DP_AUX3_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_I
+#define DP_AUX3_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux3_auxdebugrefclkind
+//DP_AUX3_AUX_DEBUG_REFCLK_ID
+#define DP_AUX3_AUX_DEBUG_REFCLK_ID__DEBUG_ID__SHIFT                                                          0x0
+#define DP_AUX3_AUX_DEBUG_REFCLK_ID__DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_A
+#define DP_AUX3_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_B
+#define DP_AUX3_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_C
+#define DP_AUX3_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_D
+#define DP_AUX3_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_E
+#define DP_AUX3_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_F
+#define DP_AUX3_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_G
+#define DP_AUX3_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G_MASK                                                           0xFFFFFFFFL
+//DP_AUX3_DP_AUX_DEBUG_J
+#define DP_AUX3_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J__SHIFT                                                         0x0
+#define DP_AUX3_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux4_auxdebugdispclkind
+//DP_AUX4_AUX_DEBUG_DISPCLK_ID
+#define DP_AUX4_AUX_DEBUG_DISPCLK_ID__DEBUG_ID__SHIFT                                                         0x0
+#define DP_AUX4_AUX_DEBUG_DISPCLK_ID__DEBUG_ID_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_H
+#define DP_AUX4_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_H__DP_AUX_DEBUG_H_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_I
+#define DP_AUX4_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_I__DP_AUX_DEBUG_I_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dp_aux4_auxdebugrefclkind
+//DP_AUX4_AUX_DEBUG_REFCLK_ID
+#define DP_AUX4_AUX_DEBUG_REFCLK_ID__DEBUG_ID__SHIFT                                                          0x0
+#define DP_AUX4_AUX_DEBUG_REFCLK_ID__DEBUG_ID_MASK                                                            0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_A
+#define DP_AUX4_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_A__DP_AUX_DEBUG_A_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_B
+#define DP_AUX4_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_B__DP_AUX_DEBUG_B_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_C
+#define DP_AUX4_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_C__DP_AUX_DEBUG_C_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_D
+#define DP_AUX4_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_D__DP_AUX_DEBUG_D_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_E
+#define DP_AUX4_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_E__DP_AUX_DEBUG_E_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_F
+#define DP_AUX4_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_F__DP_AUX_DEBUG_F_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_G
+#define DP_AUX4_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_G__DP_AUX_DEBUG_G_MASK                                                           0xFFFFFFFFL
+//DP_AUX4_DP_AUX_DEBUG_J
+#define DP_AUX4_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J__SHIFT                                                         0x0
+#define DP_AUX4_DP_AUX_DEBUG_J__DP_AUX_DEBUG_J_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: hpd0_hpddebugind
+//HPD0_HPD_DEBUG_ID
+#define HPD0_HPD_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define HPD0_HPD_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+//HPD0_HPD_DEBUG_0
+#define HPD0_HPD_DEBUG_0__HPD_DEBUG_0__SHIFT                                                                  0x0
+#define HPD0_HPD_DEBUG_0__HPD_DEBUG_0_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: hpd1_hpddebugind
+//HPD1_HPD_DEBUG_ID
+#define HPD1_HPD_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define HPD1_HPD_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+//HPD1_HPD_DEBUG_0
+#define HPD1_HPD_DEBUG_0__HPD_DEBUG_0__SHIFT                                                                  0x0
+#define HPD1_HPD_DEBUG_0__HPD_DEBUG_0_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: hpd2_hpddebugind
+//HPD2_HPD_DEBUG_ID
+#define HPD2_HPD_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define HPD2_HPD_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+//HPD2_HPD_DEBUG_0
+#define HPD2_HPD_DEBUG_0__HPD_DEBUG_0__SHIFT                                                                  0x0
+#define HPD2_HPD_DEBUG_0__HPD_DEBUG_0_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: hpd3_hpddebugind
+//HPD3_HPD_DEBUG_ID
+#define HPD3_HPD_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define HPD3_HPD_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+//HPD3_HPD_DEBUG_0
+#define HPD3_HPD_DEBUG_0__HPD_DEBUG_0__SHIFT                                                                  0x0
+#define HPD3_HPD_DEBUG_0__HPD_DEBUG_0_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: hpd4_hpddebugind
+//HPD4_HPD_DEBUG_ID
+#define HPD4_HPD_DEBUG_ID__DEBUG_ID__SHIFT                                                                    0x0
+#define HPD4_HPD_DEBUG_ID__DEBUG_ID_MASK                                                                      0xFFFFFFFFL
+//HPD4_HPD_DEBUG_0
+#define HPD4_HPD_DEBUG_0__HPD_DEBUG_0__SHIFT                                                                  0x0
+#define HPD4_HPD_DEBUG_0__HPD_DEBUG_0_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: hpo_top_hpo_topdebugind
+//HPO_TOP_DEBUG_ID
+#define HPO_TOP_DEBUG_ID__DEBUG_ID__SHIFT                                                                     0x0
+#define HPO_TOP_DEBUG_ID__DEBUG_ID_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: hdmi_link_enc0_linkencdebugind
+//HDMI_LINK_ENC_DEBUG_ID
+#define HDMI_LINK_ENC_DEBUG_ID__HDMI_LINK_ENC_DEBUG_ID__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_DEBUG_ID__HDMI_LINK_ENC_DEBUG_ID_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: hdmi_frl_enc0_frldebugind
+//HDMI_FRL_ENC_DEBUG_ID
+#define HDMI_FRL_ENC_DEBUG_ID__HDMI_FRL_ENC_DEBUG_ID__SHIFT                                                   0x0
+#define HDMI_FRL_ENC_DEBUG_ID__HDMI_FRL_ENC_DEBUG_ID_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: hdmi_stream_enc0_hdmi_stream_enc_hdmistreamclk_debugind
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_ID_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_0_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_1_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_2_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_3_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_4_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_5_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_6_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_7_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_8_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9__SHIFT                   0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_9_MASK                     0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_10_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_11_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_12_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_13_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_14_MASK                   0xFFFFFFFFL
+//HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15__HDMI_STREAM_ENC_HDMISTREAMCLK_DEBUG_15_MASK                   0xFFFFFFFFL
+
+
+// addressBlock: hdmi_stream_enc0_hdmi_stream_enc_dispclk_debugind
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_ID
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_ID__HDMI_STREAM_ENC_DISPCLK_DEBUG_ID__SHIFT                             0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_ID__HDMI_STREAM_ENC_DISPCLK_DEBUG_ID_MASK                               0xFFFFFFFFL
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_0__HDMI_STREAM_ENC_DISPCLK_DEBUG_0__SHIFT                               0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_0__HDMI_STREAM_ENC_DISPCLK_DEBUG_0_MASK                                 0xFFFFFFFFL
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_1
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_1__HDMI_STREAM_ENC_DISPCLK_DEBUG_1__SHIFT                               0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_1__HDMI_STREAM_ENC_DISPCLK_DEBUG_1_MASK                                 0xFFFFFFFFL
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_2
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_2__HDMI_STREAM_ENC_DISPCLK_DEBUG_2__SHIFT                               0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_2__HDMI_STREAM_ENC_DISPCLK_DEBUG_2_MASK                                 0xFFFFFFFFL
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_3
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_3__HDMI_STREAM_ENC_DISPCLK_DEBUG_3__SHIFT                               0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_3__HDMI_STREAM_ENC_DISPCLK_DEBUG_3_MASK                                 0xFFFFFFFFL
+//HDMI_STREAM_ENC_DISPCLK_DEBUG_4
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_4__HDMI_STREAM_ENC_DISPCLK_DEBUG_4__SHIFT                               0x0
+#define HDMI_STREAM_ENC_DISPCLK_DEBUG_4__HDMI_STREAM_ENC_DISPCLK_DEBUG_4_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_dispclk_debugind
+//DP_STREAM_ENC0_DP_STREAM_ENC_DISPCLK_DEBUG_ID
+#define DP_STREAM_ENC0_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID__SHIFT                  0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID_MASK                    0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_dpstreamclk_debugind
+//DP_STREAM_ENC0_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_STREAM_ENC0_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT          0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID_MASK            0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc0_dp_stream_enc_symclk32_debugind
+//DP_STREAM_ENC0_DP_STREAM_ENC_SYMCLK32_DEBUG_ID
+#define DP_STREAM_ENC0_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID__SHIFT                0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID_MASK                  0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc0_dp_sym32_enc_symclk32_debugind
+//DP_SYM32_ENC0_DP_SYM32_ENC_SYMCLK32_DEBUG_ID
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID__SHIFT                   0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID_MASK                     0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc0_dp_sym32_enc_dpstreamclk_debugind
+//DP_SYM32_ENC0_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_SYM32_ENC0_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT             0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID_MASK               0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_dispclk_debugind
+//DP_STREAM_ENC1_DP_STREAM_ENC_DISPCLK_DEBUG_ID
+#define DP_STREAM_ENC1_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID__SHIFT                  0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID_MASK                    0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_dpstreamclk_debugind
+//DP_STREAM_ENC1_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_STREAM_ENC1_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT          0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID_MASK            0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc1_dp_stream_enc_symclk32_debugind
+//DP_STREAM_ENC1_DP_STREAM_ENC_SYMCLK32_DEBUG_ID
+#define DP_STREAM_ENC1_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID__SHIFT                0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID_MASK                  0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc1_dp_sym32_enc_symclk32_debugind
+//DP_SYM32_ENC1_DP_SYM32_ENC_SYMCLK32_DEBUG_ID
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID__SHIFT                   0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID_MASK                     0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc1_dp_sym32_enc_dpstreamclk_debugind
+//DP_SYM32_ENC1_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_SYM32_ENC1_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT             0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID_MASK               0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_dispclk_debugind
+//DP_STREAM_ENC2_DP_STREAM_ENC_DISPCLK_DEBUG_ID
+#define DP_STREAM_ENC2_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID__SHIFT                  0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID_MASK                    0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_dpstreamclk_debugind
+//DP_STREAM_ENC2_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_STREAM_ENC2_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT          0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID_MASK            0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc2_dp_stream_enc_symclk32_debugind
+//DP_STREAM_ENC2_DP_STREAM_ENC_SYMCLK32_DEBUG_ID
+#define DP_STREAM_ENC2_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID__SHIFT                0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID_MASK                  0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc2_dp_sym32_enc_symclk32_debugind
+//DP_SYM32_ENC2_DP_SYM32_ENC_SYMCLK32_DEBUG_ID
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID__SHIFT                   0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID_MASK                     0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc2_dp_sym32_enc_dpstreamclk_debugind
+//DP_SYM32_ENC2_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_SYM32_ENC2_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT             0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID_MASK               0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_dispclk_debugind
+//DP_STREAM_ENC3_DP_STREAM_ENC_DISPCLK_DEBUG_ID
+#define DP_STREAM_ENC3_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID__SHIFT                  0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_DISPCLK_DEBUG_ID__DP_STREAM_ENC_DISPCLK_DEBUG_ID_MASK                    0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_dpstreamclk_debugind
+//DP_STREAM_ENC3_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_STREAM_ENC3_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT          0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID__DP_STREAM_ENC_DPSTREAMCLK_DEBUG_ID_MASK            0xFFFFFFFFL
+
+
+// addressBlock: dp_stream_enc3_dp_stream_enc_symclk32_debugind
+//DP_STREAM_ENC3_DP_STREAM_ENC_SYMCLK32_DEBUG_ID
+#define DP_STREAM_ENC3_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID__SHIFT                0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_SYMCLK32_DEBUG_ID__DP_STREAM_ENC_SYMCLK32_DEBUG_ID_MASK                  0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc3_dp_sym32_enc_symclk32_debugind
+//DP_SYM32_ENC3_DP_SYM32_ENC_SYMCLK32_DEBUG_ID
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID__SHIFT                   0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMCLK32_DEBUG_ID__DP_SYM32_ENC_SYMCLK32_DEBUG_ID_MASK                     0xFFFFFFFFL
+
+
+// addressBlock: dp_sym32_enc3_dp_sym32_enc_dpstreamclk_debugind
+//DP_SYM32_ENC3_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID
+#define DP_SYM32_ENC3_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__SHIFT             0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID__DP_SYM32_ENC_DPSTREAMCLK_DEBUG_ID_MASK               0xFFFFFFFFL
+
+
+// addressBlock: dp_link_enc0_dplinkencdebugind
+//DP_LINK_ENC0_DP_LINK_ENC_DEBUG_ID
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID__SHIFT                                        0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID_MASK                                          0xFFFFFFFFL
+//DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0__SHIFT                                    0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS1
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1__SHIFT                                    0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS2
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2__SHIFT                                    0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS3
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3__SHIFT                                    0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3_MASK                                      0xFFFFFFFFL
+
+
+// addressBlock: dp_dphy_sym320_dpdphysym32debugind
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_ID
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID__SHIFT                                  0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID_MASK                                    0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS6
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS7
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS9
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS10
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS11
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS12
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS13
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS14
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS15
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS16
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS17
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS18
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS19
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS20
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS21
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS22
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS23
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23__SHIFT                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dp_link_enc1_dplinkencdebugind
+//DP_LINK_ENC1_DP_LINK_ENC_DEBUG_ID
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID__SHIFT                                        0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID_MASK                                          0xFFFFFFFFL
+//DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0__SHIFT                                    0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS1
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1__SHIFT                                    0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS2
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2__SHIFT                                    0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS3
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3__SHIFT                                    0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3_MASK                                      0xFFFFFFFFL
+
+
+// addressBlock: dp_dphy_sym321_dpdphysym32debugind
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_ID
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID__SHIFT                                  0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID_MASK                                    0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS6
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS7
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS9
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS10
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS11
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS12
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS13
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS14
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS15
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS16
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS17
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS18
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS19
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS20
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS21
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS22
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS23
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23__SHIFT                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dp_link_enc2_dplinkencdebugind
+//DP_LINK_ENC2_DP_LINK_ENC_DEBUG_ID
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID__SHIFT                                        0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID_MASK                                          0xFFFFFFFFL
+//DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0__SHIFT                                    0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS1
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1__SHIFT                                    0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS2
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2__SHIFT                                    0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS3
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3__SHIFT                                    0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3_MASK                                      0xFFFFFFFFL
+
+
+// addressBlock: dp_dphy_sym322_dpdphysym32debugind
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_ID
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID__SHIFT                                  0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID_MASK                                    0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS6
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS7
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS9
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS10
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS11
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS12
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS13
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS14
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS15
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS16
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS17
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS18
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS19
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS20
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS21
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS22
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS23
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23__SHIFT                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dp_link_enc3_dplinkencdebugind
+//DP_LINK_ENC3_DP_LINK_ENC_DEBUG_ID
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID__SHIFT                                        0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_ID__DP_LINK_ENC_DEBUG_ID_MASK                                          0xFFFFFFFFL
+//DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0__SHIFT                                    0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS0__DP_LINK_ENC_DEBUG_BUS0_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS1
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1__SHIFT                                    0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS1__DP_LINK_ENC_DEBUG_BUS1_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS2
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2__SHIFT                                    0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS2__DP_LINK_ENC_DEBUG_BUS2_MASK                                      0xFFFFFFFFL
+//DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS3
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3__SHIFT                                    0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_DEBUG_BUS3__DP_LINK_ENC_DEBUG_BUS3_MASK                                      0xFFFFFFFFL
+
+
+// addressBlock: dp_dphy_sym323_dpdphysym32debugind
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_ID
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID__SHIFT                                  0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_ID__DP_DPHY_SYM32_DEBUG_ID_MASK                                    0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS0__DP_DPHY_SYM32_DEBUG_BUS0_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS1__DP_DPHY_SYM32_DEBUG_BUS1_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS2__DP_DPHY_SYM32_DEBUG_BUS2_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS3__DP_DPHY_SYM32_DEBUG_BUS3_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS4__DP_DPHY_SYM32_DEBUG_BUS4_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS5__DP_DPHY_SYM32_DEBUG_BUS5_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS6
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS6__DP_DPHY_SYM32_DEBUG_BUS6_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS7
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS7__DP_DPHY_SYM32_DEBUG_BUS7_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS8__DP_DPHY_SYM32_DEBUG_BUS8_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS9
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS9__DP_DPHY_SYM32_DEBUG_BUS9_MASK                                0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS10
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS10__DP_DPHY_SYM32_DEBUG_BUS10_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS11
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS11__DP_DPHY_SYM32_DEBUG_BUS11_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS12
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS12__DP_DPHY_SYM32_DEBUG_BUS12_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS13
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS13__DP_DPHY_SYM32_DEBUG_BUS13_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS14
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS14__DP_DPHY_SYM32_DEBUG_BUS14_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS15
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS15__DP_DPHY_SYM32_DEBUG_BUS15_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS16
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS16__DP_DPHY_SYM32_DEBUG_BUS16_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS17
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS17__DP_DPHY_SYM32_DEBUG_BUS17_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS18
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS18__DP_DPHY_SYM32_DEBUG_BUS18_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS19
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS19__DP_DPHY_SYM32_DEBUG_BUS19_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS20
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS20__DP_DPHY_SYM32_DEBUG_BUS20_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS21
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS21__DP_DPHY_SYM32_DEBUG_BUS21_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS22
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS22__DP_DPHY_SYM32_DEBUG_BUS22_MASK                              0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS23
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23__SHIFT                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_DEBUG_BUS23__DP_DPHY_SYM32_DEBUG_BUS23_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: azendpoint_sinkinfoind
+//AZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID
+#define AZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID__MANUFACTURER_ID__SHIFT                                   0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID__MANUFACTURER_ID_MASK                                     0x0000FFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID
+#define AZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID__PRODUCT_ID__SHIFT                                             0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID__PRODUCT_ID_MASK                                               0x0000FFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN
+#define AZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN__SINK_DESCRIPTION_LEN__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN__SINK_DESCRIPTION_LEN_MASK                           0x000000FFL
+//AZALIA_F2_CODEC_PIN_CONTROL_PORTID0
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID0__PORTID__SHIFT                                                    0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID0__PORTID_MASK                                                      0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_PORTID1
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID1__PORTID__SHIFT                                                    0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID1__PORTID_MASK                                                      0xFFFFFFFFL
+//SINK_DESCRIPTION0
+#define SINK_DESCRIPTION0__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION0__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION1
+#define SINK_DESCRIPTION1__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION1__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION2
+#define SINK_DESCRIPTION2__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION2__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION3
+#define SINK_DESCRIPTION3__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION3__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION4
+#define SINK_DESCRIPTION4__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION4__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION5
+#define SINK_DESCRIPTION5__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION5__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION6
+#define SINK_DESCRIPTION6__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION6__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION7
+#define SINK_DESCRIPTION7__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION7__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION8
+#define SINK_DESCRIPTION8__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION8__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION9
+#define SINK_DESCRIPTION9__DESCRIPTION__SHIFT                                                                 0x0
+#define SINK_DESCRIPTION9__DESCRIPTION_MASK                                                                   0x000000FFL
+//SINK_DESCRIPTION10
+#define SINK_DESCRIPTION10__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION10__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION11
+#define SINK_DESCRIPTION11__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION11__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION12
+#define SINK_DESCRIPTION12__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION12__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION13
+#define SINK_DESCRIPTION13__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION13__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION14
+#define SINK_DESCRIPTION14__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION14__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION15
+#define SINK_DESCRIPTION15__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION15__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION16
+#define SINK_DESCRIPTION16__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION16__DESCRIPTION_MASK                                                                  0x000000FFL
+//SINK_DESCRIPTION17
+#define SINK_DESCRIPTION17__DESCRIPTION__SHIFT                                                                0x0
+#define SINK_DESCRIPTION17__DESCRIPTION_MASK                                                                  0x000000FFL
+
+
+// addressBlock: azf0controller_azinputcrc0resultind
+//AZALIA_INPUT_CRC0_CHANNEL0
+#define AZALIA_INPUT_CRC0_CHANNEL0__INPUT_CRC_CHANNEL0__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL0__INPUT_CRC_CHANNEL0_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL1
+#define AZALIA_INPUT_CRC0_CHANNEL1__INPUT_CRC_CHANNEL1__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL1__INPUT_CRC_CHANNEL1_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL2
+#define AZALIA_INPUT_CRC0_CHANNEL2__INPUT_CRC_CHANNEL2__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL2__INPUT_CRC_CHANNEL2_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL3
+#define AZALIA_INPUT_CRC0_CHANNEL3__INPUT_CRC_CHANNEL3__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL3__INPUT_CRC_CHANNEL3_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL4
+#define AZALIA_INPUT_CRC0_CHANNEL4__INPUT_CRC_CHANNEL4__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL4__INPUT_CRC_CHANNEL4_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL5
+#define AZALIA_INPUT_CRC0_CHANNEL5__INPUT_CRC_CHANNEL5__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL5__INPUT_CRC_CHANNEL5_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL6
+#define AZALIA_INPUT_CRC0_CHANNEL6__INPUT_CRC_CHANNEL6__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL6__INPUT_CRC_CHANNEL6_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CHANNEL7
+#define AZALIA_INPUT_CRC0_CHANNEL7__INPUT_CRC_CHANNEL7__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CHANNEL7__INPUT_CRC_CHANNEL7_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: azf0controller_azinputcrc1resultind
+//AZALIA_INPUT_CRC1_CHANNEL0
+#define AZALIA_INPUT_CRC1_CHANNEL0__INPUT_CRC_CHANNEL0__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL0__INPUT_CRC_CHANNEL0_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL1
+#define AZALIA_INPUT_CRC1_CHANNEL1__INPUT_CRC_CHANNEL1__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL1__INPUT_CRC_CHANNEL1_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL2
+#define AZALIA_INPUT_CRC1_CHANNEL2__INPUT_CRC_CHANNEL2__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL2__INPUT_CRC_CHANNEL2_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL3
+#define AZALIA_INPUT_CRC1_CHANNEL3__INPUT_CRC_CHANNEL3__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL3__INPUT_CRC_CHANNEL3_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL4
+#define AZALIA_INPUT_CRC1_CHANNEL4__INPUT_CRC_CHANNEL4__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL4__INPUT_CRC_CHANNEL4_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL5
+#define AZALIA_INPUT_CRC1_CHANNEL5__INPUT_CRC_CHANNEL5__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL5__INPUT_CRC_CHANNEL5_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL6
+#define AZALIA_INPUT_CRC1_CHANNEL6__INPUT_CRC_CHANNEL6__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL6__INPUT_CRC_CHANNEL6_MASK                                                   0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CHANNEL7
+#define AZALIA_INPUT_CRC1_CHANNEL7__INPUT_CRC_CHANNEL7__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CHANNEL7__INPUT_CRC_CHANNEL7_MASK                                                   0xFFFFFFFFL
+
+
+// addressBlock: azf0controller_azcrc0resultind
+//AZALIA_CRC0_CHANNEL0
+#define AZALIA_CRC0_CHANNEL0__CRC_CHANNEL0__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL0__CRC_CHANNEL0_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL1
+#define AZALIA_CRC0_CHANNEL1__CRC_CHANNEL1__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL1__CRC_CHANNEL1_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL2
+#define AZALIA_CRC0_CHANNEL2__CRC_CHANNEL2__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL2__CRC_CHANNEL2_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL3
+#define AZALIA_CRC0_CHANNEL3__CRC_CHANNEL3__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL3__CRC_CHANNEL3_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL4
+#define AZALIA_CRC0_CHANNEL4__CRC_CHANNEL4__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL4__CRC_CHANNEL4_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL5
+#define AZALIA_CRC0_CHANNEL5__CRC_CHANNEL5__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL5__CRC_CHANNEL5_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL6
+#define AZALIA_CRC0_CHANNEL6__CRC_CHANNEL6__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL6__CRC_CHANNEL6_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC0_CHANNEL7
+#define AZALIA_CRC0_CHANNEL7__CRC_CHANNEL7__SHIFT                                                             0x0
+#define AZALIA_CRC0_CHANNEL7__CRC_CHANNEL7_MASK                                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0controller_azcrc1resultind
+//AZALIA_CRC1_CHANNEL0
+#define AZALIA_CRC1_CHANNEL0__CRC_CHANNEL0__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL0__CRC_CHANNEL0_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL1
+#define AZALIA_CRC1_CHANNEL1__CRC_CHANNEL1__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL1__CRC_CHANNEL1_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL2
+#define AZALIA_CRC1_CHANNEL2__CRC_CHANNEL2__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL2__CRC_CHANNEL2_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL3
+#define AZALIA_CRC1_CHANNEL3__CRC_CHANNEL3__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL3__CRC_CHANNEL3_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL4
+#define AZALIA_CRC1_CHANNEL4__CRC_CHANNEL4__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL4__CRC_CHANNEL4_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL5
+#define AZALIA_CRC1_CHANNEL5__CRC_CHANNEL5__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL5__CRC_CHANNEL5_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL6
+#define AZALIA_CRC1_CHANNEL6__CRC_CHANNEL6__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL6__CRC_CHANNEL6_MASK                                                               0xFFFFFFFFL
+//AZALIA_CRC1_CHANNEL7
+#define AZALIA_CRC1_CHANNEL7__CRC_CHANNEL7__SHIFT                                                             0x0
+#define AZALIA_CRC1_CHANNEL7__CRC_CHANNEL7_MASK                                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream0_streamind
+//AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM0_AZALIA_STREAM_DEBUG
+#define AZF0STREAM0_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM0_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream1_streamind
+//AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM1_AZALIA_STREAM_DEBUG
+#define AZF0STREAM1_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM1_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream2_streamind
+//AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM2_AZALIA_STREAM_DEBUG
+#define AZF0STREAM2_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM2_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream3_streamind
+//AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM3_AZALIA_STREAM_DEBUG
+#define AZF0STREAM3_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM3_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream4_streamind
+//AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM4_AZALIA_STREAM_DEBUG
+#define AZF0STREAM4_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM4_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream5_streamind
+//AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM5_AZALIA_STREAM_DEBUG
+#define AZF0STREAM5_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM5_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream6_streamind
+//AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM6_AZALIA_STREAM_DEBUG
+#define AZF0STREAM6_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM6_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream7_streamind
+//AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM7_AZALIA_STREAM_DEBUG
+#define AZF0STREAM7_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM7_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream8_streamind
+//AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM8_AZALIA_STREAM_DEBUG
+#define AZF0STREAM8_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM8_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream9_streamind
+//AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                            0x0
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                            0x8
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                      0x10
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                              0x0000007FL
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                              0x00007F00L
+#define AZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                        0x00FF0000L
+//AZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                       0x0
+#define AZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                         0x00000001L
+//AZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                     0x0
+#define AZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                       0xFFFFFFFFL
+//AZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                   0x0
+#define AZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                   0x0
+#define AZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                     0xFFFFFFFFL
+//AZF0STREAM9_AZALIA_STREAM_DEBUG
+#define AZF0STREAM9_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                             0x0
+#define AZF0STREAM9_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: azf0stream10_streamind
+//AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM10_AZALIA_STREAM_DEBUG
+#define AZF0STREAM10_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM10_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0stream11_streamind
+//AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM11_AZALIA_STREAM_DEBUG
+#define AZF0STREAM11_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM11_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0stream12_streamind
+//AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM12_AZALIA_STREAM_DEBUG
+#define AZF0STREAM12_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM12_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0stream13_streamind
+//AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM13_AZALIA_STREAM_DEBUG
+#define AZF0STREAM13_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM13_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0stream14_streamind
+//AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM14_AZALIA_STREAM_DEBUG
+#define AZF0STREAM14_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM14_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0stream15_streamind
+//AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT                                           0x0
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT                                           0x8
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT                                     0x10
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK                                             0x0000007FL
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK                                             0x00007F00L
+#define AZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK                                       0x00FF0000L
+//AZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL
+#define AZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT                      0x0
+#define AZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK                        0x00000001L
+//AZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT
+#define AZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT                    0x0
+#define AZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK                      0xFFFFFFFFL
+//AZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT
+#define AZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT                  0x0
+#define AZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT
+#define AZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT                  0x0
+#define AZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK                    0xFFFFFFFFL
+//AZF0STREAM15_AZALIA_STREAM_DEBUG
+#define AZF0STREAM15_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT                                            0x0
+#define AZF0STREAM15_AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK                                              0xFFFFFFFFL
+
+
+// addressBlock: azf0endpoint0_endpointind
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT0_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT0_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT0_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint1_endpointind
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT1_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT1_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT1_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint2_endpointind
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT2_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT2_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT2_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint3_endpointind
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT3_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT3_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT3_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint4_endpointind
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT4_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT4_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT4_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint5_endpointind
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT5_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT5_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT5_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint6_endpointind
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT6_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT6_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT6_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0endpoint7_endpointind
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT                                0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG_MASK                                  0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT   0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT            0x5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT   0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT           0x9
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT     0xa
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT           0xb
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT              0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK     0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK              0x00000020L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK   0x00000040L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK     0x00000100L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK             0x00000200L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK       0x00000400L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK             0x00000800L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                0x00F00000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT              0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT          0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT         0xb
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT             0xe
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                  0xf
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK             0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                0x00000070L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK            0x00000700L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK           0x00003800L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK               0x00004000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                    0x00008000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                   0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                    0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                     0x000000F0L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                       0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                           0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                        0x2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                         0x3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                        0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                   0x5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                         0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                           0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                          0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                   0x17
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                         0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                             0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                          0x00000004L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                           0x00000008L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                          0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                     0x00000020L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                           0x00000040L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                             0x00000080L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                            0x00007F00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                     0x00800000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT               0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                 0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK   0x001F0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                      0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                           0x00000003L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                        0x00700000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                             0x000000FFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                  0x5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT       0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT         0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                 0x9
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT           0xa
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                 0xb
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                    0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK   0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                    0x00000020L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK         0x00000040L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK           0x00000100L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                   0x00000200L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK             0x00000400L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                   0x00000800L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                      0x00F00000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT              0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                     0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT            0x2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT              0x3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                       0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                        0x5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                    0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                 0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                         0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                         0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                   0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                       0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK              0x00000004L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                0x00000008L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                         0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                          0x00000020L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                      0x00000040L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                   0x00000080L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                           0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                           0x00010000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                     0x01000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                            0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                         0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                              0x0000003FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                           0x00000080L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                    0x7FFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                           0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                             0x00000040L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT                  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT                  0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT                     0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT                       0x11
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT               0x12
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL__SHIFT                  0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT                         0x1b
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT                    0x1f
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK                    0x0000007FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK                    0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK                       0x00010000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK                         0x00020000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK                 0x00FC0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LFE_PLAYBACK_LEVEL_MASK                    0x03000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK                           0x78000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK                      0x80000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX__SHIFT                                  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI__SHIFT                                0x6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE__SHIFT                          0x7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE__SHIFT                                   0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0__SHIFT                   0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1__SHIFT                   0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_INDEX_MASK                                    0x0000003FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__SUPPORTS_AI_MASK                                  0x00000040L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_PACKET_ENABLE_MASK                            0x00000080L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_MASK                                     0x00000300L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE0_MASK                     0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_ACP_DATA__ACP_TYPE_DEPENDENT_BYTE1_MASK                     0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT      0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK        0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                 0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                        0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                   0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                       0x00000007L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK              0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                  0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT                      0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT                      0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK                        0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK                        0x0000FF00L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                            0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                             0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                              0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                               0x00000010L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT                          0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT                               0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK                            0x0000FFFFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK                                 0xFFFF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK                       0x000000FFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT                                 0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT                                 0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK                                   0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT                             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT                             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT                             0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT                             0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK                               0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK                               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK                               0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK                               0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT                             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT                             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT                             0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT                             0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK                               0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK                               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK                               0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK                               0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT                             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT                             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT                            0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT                            0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK                               0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK                               0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK                              0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK                              0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT                            0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT                            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT                            0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT                            0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK                              0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK                              0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK                              0x00FF0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK                              0xFF000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT                            0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT                            0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK                              0x000000FFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK                              0x0000FF00L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT               0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT                     0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT                      0x1f
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK                 0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK                       0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK                        0x80000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                 0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                0xc
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT      0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT       0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT             0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT    0x1e
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK               0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK    0x000000F0L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                   0x00000F00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                  0x0000F000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK        0x000F0000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK         0x00F00000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK               0x3F000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK      0xC0000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT             0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT       0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT           0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT             0x9
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT       0xc
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT           0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT             0x11
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT       0x14
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT           0x18
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT             0x19
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT       0x1c
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK               0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK         0x000000F0L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK             0x00000100L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK               0x00000200L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK         0x0000F000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK             0x00010000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK               0x00020000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK         0x00F00000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK             0x01000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK               0x02000000L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK         0xF0000000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                 0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                   0x00000001L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                     0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT            0x2
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                       0x00000003L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK              0x0000003CL
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT  0x2
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT              0x3
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT     0x7
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK             0x00000003L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK    0x00000004L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                0x00000078L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK       0x00000080L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT       0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x6
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK         0x0000003FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000040L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT  0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK  0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK  0x00000010L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT       0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                   0x5
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT             0x7
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK   0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK         0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                     0x00000060L
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK               0x00000080L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT         0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK           0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK           0x000000F0L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT         0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK           0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK           0x000000F0L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT         0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK           0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK           0x000000F0L
+//AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT         0x4
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK           0x0000000FL
+#define AZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK           0x000000F0L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                             0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                 0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                   0x00000001L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT            0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT      0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK              0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK        0x0000FF00L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                             0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK               0xFFFFFFFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                               0x000000FFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                       0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT         0x1
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                 0x8
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT               0x10
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                         0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK           0x00000002L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                   0x0000FF00L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                 0x00FF0000L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT  0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK  0x00000003L
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT       0x4
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK             0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK         0x00000010L
+//AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS__SHIFT                               0x0
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS__AUDIO_ENABLE_STATUS_MASK                                 0x00000001L
+//AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG__SHIFT                           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK__SHIFT                           0x4
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE__SHIFT                           0x8
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_FLAG_MASK                             0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_MASK_MASK                             0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS__AUDIO_ENABLED_TYPE_MASK                             0x00000100L
+//AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG__SHIFT                         0x0
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK__SHIFT                         0x4
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE__SHIFT                         0x8
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_FLAG_MASK                           0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_MASK_MASK                           0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS__AUDIO_DISABLED_TYPE_MASK                           0x00000100L
+//AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG__SHIFT             0x0
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK__SHIFT             0x4
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE__SHIFT             0x8
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_FLAG_MASK               0x00000001L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_MASK_MASK               0x00000010L
+#define AZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS__AUDIO_FORMAT_CHANGED_TYPE_MASK               0x00000100L
+//AZF0ENDPOINT7_AZALIA_F0_ENDPOINT_FGCG_REP_DIS
+#define AZF0ENDPOINT7_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS__SHIFT                           0x0
+#define AZF0ENDPOINT7_AZALIA_F0_ENDPOINT_FGCG_REP_DIS__ENDPOINT_FGCG_REP_DIS_MASK                             0x00000001L
+
+
+// addressBlock: azf0inputendpoint0_inputendpointind
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint1_inputendpointind
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint2_inputendpointind
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint3_inputendpointind
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint4_inputendpointind
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint5_inputendpointind
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint6_inputendpointind
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: azf0inputendpoint7_inputendpointind
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG__SHIFT               0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PIN_DEBUG__AZALIA_INPUT_DEBUG_MASK                 0xFFFFFFFFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT  0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT  0x9
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT  0xb
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT   0x14
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK   0x00000020L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK  0x00000200L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK  0x00000800L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK     0x00F00000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT   0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT  0xb
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT  0xe
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT       0xf
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK  0x0000000FL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK     0x00000070L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK  0x00000700L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK  0x00003800L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK    0x00004000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK         0x00008000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT       0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT        0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK         0x0000000FL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK          0x000000F0L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT            0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT             0x2
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT              0x3
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT             0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT        0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT              0x6
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                0x7
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT               0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT        0x17
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK              0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                  0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK               0x00000004L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                0x00000008L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK               0x00000010L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK          0x00000020L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                0x00000040L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                  0x00000080L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                 0x00007F00L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK          0x00800000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT    0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK      0xFFFFFFFFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK  0x00000FFFL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK  0x001F0000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT  0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT       0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT  0x6
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT      0x9
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT  0xa
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT      0xb
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT         0x14
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK  0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK  0x00000004L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK         0x00000020L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK  0x00000040L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK  0x00000100L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK        0x00000200L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK  0x00000400L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK        0x00000800L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK           0x00F00000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT   0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT          0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT  0x2
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT   0x3
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT            0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT             0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT         0x6
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                      0x7
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT              0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT              0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                        0x18
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK     0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK            0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK   0x00000004L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK     0x00000008L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK              0x00000010L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK               0x00000020L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK           0x00000040L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                        0x00000080L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                0x0000FF00L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                0x00010000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                          0x01000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                 0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT              0x7
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                   0x0000003FL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                0x00000080L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT__SHIFT  0x1f
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__IMPEDANCE_SENSE_MASK   0x7FFFFFFFL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE__PRESENCE_DETECT_MASK   0x80000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                 0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                   0x00000020L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE__SHIFT   0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE__SHIFT   0x9
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE__SHIFT   0x11
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE__SHIFT   0x19
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_ENABLE_MASK   0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_MUTE_MASK     0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_ENABLE_MASK   0x00000100L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_MUTE_MASK     0x00000200L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_ENABLE_MASK   0x00010000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_MUTE_MASK     0x00020000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_ENABLE_MASK   0x01000000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_MUTE_MASK     0x02000000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE__SHIFT  0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID__SHIFT  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT  0x9
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT  0xc
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE__SHIFT  0x11
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID__SHIFT  0x14
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT  0x18
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT  0x19
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT  0x1c
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_ENABLE_MASK  0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_MUTE_MASK    0x00000002L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL4_CHANNEL_ID_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK  0x00000100L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK    0x00000200L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK  0x0000F000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_ENABLE_MASK  0x00010000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_MUTE_MASK    0x00020000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL6_CHANNEL_ID_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK  0x01000000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK    0x02000000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK  0xF0000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT                 0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT                  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK                   0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK                    0x00000010L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT    0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK      0x000000FFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE__SHIFT    0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE__SHIFT          0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT           0x1f
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_GATING_DISABLE_MASK      0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__CLOCK_ON_STATE_MASK            0x00000010L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK             0x80000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT  0x1c
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK  0x03FFFFFFL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK  0x10000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT      0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT     0xc
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT  0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT  0x14
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT  0x18
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT  0x1e
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK    0x0000000FL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK  0x000000F0L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK        0x00000F00L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK       0x0000F000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK  0x000F0000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK  0x00F00000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK    0x3F000000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK  0xC0000000L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT  0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK   0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK  0x0000FF00L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                  0xFFFFFFFFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK    0xFFFFFFFFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT      0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT      0x1
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT  0x4
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK        0x00000001L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK        0x00000006L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK  0x00000010L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK  0x00000020L
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                  0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT             0x8
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT               0x10
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                0x1f
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                    0x00000007L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK               0x0000FF00L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                 0x00FF0000L
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                  0x80000000L
+
+
+// addressBlock: dscc_dsccdebugind0
+//DSCC_DEBUG_ID
+#define DSCC_DEBUG_ID__DSCC_DEBUG_ID__SHIFT                                                                   0x0
+#define DSCC_DEBUG_ID__DSCC_DEBUG_ID_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_0
+#define DSCC_DEBUG_0__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL0__SHIFT                                           0x0
+#define DSCC_DEBUG_0__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL0_MASK                                             0x0003FFFFL
+//DSCC_DEBUG_1
+#define DSCC_DEBUG_1__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL1__SHIFT                                           0x0
+#define DSCC_DEBUG_1__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL1_MASK                                             0x0003FFFFL
+//DSCC_DEBUG_2
+#define DSCC_DEBUG_2__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL2__SHIFT                                           0x0
+#define DSCC_DEBUG_2__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL2_MASK                                             0x0003FFFFL
+//DSCC_DEBUG_3
+#define DSCC_DEBUG_3__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL3__SHIFT                                           0x0
+#define DSCC_DEBUG_3__DSCC_RATE_BUFFER_MODEL_FULLNESS_LEVEL3_MASK                                             0x0003FFFFL
+//DSCC_DEBUG_4
+#define DSCC_DEBUG_4__DSCC_DEBUG_4__SHIFT                                                                     0x0
+#define DSCC_DEBUG_4__DSCC_DEBUG_4_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_5
+#define DSCC_DEBUG_5__DSCC_DEBUG_5__SHIFT                                                                     0x0
+#define DSCC_DEBUG_5__DSCC_DEBUG_5_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_6
+#define DSCC_DEBUG_6__DSCC_DEBUG_6__SHIFT                                                                     0x0
+#define DSCC_DEBUG_6__DSCC_DEBUG_6_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_7
+#define DSCC_DEBUG_7__DSCC_DEBUG_7__SHIFT                                                                     0x0
+#define DSCC_DEBUG_7__DSCC_DEBUG_7_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_8
+#define DSCC_DEBUG_8__DSCC_DEBUG_8__SHIFT                                                                     0x0
+#define DSCC_DEBUG_8__DSCC_DEBUG_8_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_9
+#define DSCC_DEBUG_9__DSCC_DEBUG_9__SHIFT                                                                     0x0
+#define DSCC_DEBUG_9__DSCC_DEBUG_9_MASK                                                                       0xFFFFFFFFL
+//DSCC_DEBUG_10
+#define DSCC_DEBUG_10__DSCC_DEBUG_10__SHIFT                                                                   0x0
+#define DSCC_DEBUG_10__DSCC_DEBUG_10_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_11
+#define DSCC_DEBUG_11__DSCC_DEBUG_11__SHIFT                                                                   0x0
+#define DSCC_DEBUG_11__DSCC_DEBUG_11_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_12
+#define DSCC_DEBUG_12__DSCC_DEBUG_12__SHIFT                                                                   0x0
+#define DSCC_DEBUG_12__DSCC_DEBUG_12_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_13
+#define DSCC_DEBUG_13__DSCC_DEBUG_13__SHIFT                                                                   0x0
+#define DSCC_DEBUG_13__DSCC_DEBUG_13_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_14
+#define DSCC_DEBUG_14__DSCC_DEBUG_14__SHIFT                                                                   0x0
+#define DSCC_DEBUG_14__DSCC_DEBUG_14_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_15
+#define DSCC_DEBUG_15__DSCC_DEBUG_15__SHIFT                                                                   0x0
+#define DSCC_DEBUG_15__DSCC_DEBUG_15_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_16
+#define DSCC_DEBUG_16__DSCC_DEBUG_16__SHIFT                                                                   0x0
+#define DSCC_DEBUG_16__DSCC_DEBUG_16_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_17
+#define DSCC_DEBUG_17__DSCC_DEBUG_17__SHIFT                                                                   0x0
+#define DSCC_DEBUG_17__DSCC_DEBUG_17_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_18
+#define DSCC_DEBUG_18__DSCC_DEBUG_18__SHIFT                                                                   0x0
+#define DSCC_DEBUG_18__DSCC_DEBUG_18_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_19
+#define DSCC_DEBUG_19__DSCC_DEBUG_19__SHIFT                                                                   0x0
+#define DSCC_DEBUG_19__DSCC_DEBUG_19_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_20
+#define DSCC_DEBUG_20__DSCC_DEBUG_20__SHIFT                                                                   0x0
+#define DSCC_DEBUG_20__DSCC_DEBUG_20_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_21
+#define DSCC_DEBUG_21__DSCC_DEBUG_21__SHIFT                                                                   0x0
+#define DSCC_DEBUG_21__DSCC_DEBUG_21_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_22
+#define DSCC_DEBUG_22__DSCC_DEBUG_22__SHIFT                                                                   0x0
+#define DSCC_DEBUG_22__DSCC_DEBUG_22_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_23
+#define DSCC_DEBUG_23__DSCC_DEBUG_23__SHIFT                                                                   0x0
+#define DSCC_DEBUG_23__DSCC_DEBUG_23_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_24
+#define DSCC_DEBUG_24__DSCC_DEBUG_24__SHIFT                                                                   0x0
+#define DSCC_DEBUG_24__DSCC_DEBUG_24_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_25
+#define DSCC_DEBUG_25__DSCC_DEBUG_25__SHIFT                                                                   0x0
+#define DSCC_DEBUG_25__DSCC_DEBUG_25_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_26
+#define DSCC_DEBUG_26__DSCC_DEBUG_26__SHIFT                                                                   0x0
+#define DSCC_DEBUG_26__DSCC_DEBUG_26_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_27
+#define DSCC_DEBUG_27__DSCC_DEBUG_27__SHIFT                                                                   0x0
+#define DSCC_DEBUG_27__DSCC_DEBUG_27_MASK                                                                     0xFFFFFFFFL
+//DSCC_DEBUG_28
+#define DSCC_DEBUG_28__DSCC_DEBUG_28__SHIFT                                                                   0x0
+#define DSCC_DEBUG_28__DSCC_DEBUG_28_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: dscc_dscc_dispclk_debugind0
+//DSCC_DISPCLK_DEBUG_ID
+#define DSCC_DISPCLK_DEBUG_ID__DSCC_DISPCLK_DEBUG_ID__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_ID__DSCC_DISPCLK_DEBUG_ID_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_0
+#define DSCC_DISPCLK_DEBUG_0__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL0__SHIFT                                       0x0
+#define DSCC_DISPCLK_DEBUG_0__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL0_MASK                                         0x00007FFFL
+//DSCC_DISPCLK_DEBUG_1
+#define DSCC_DISPCLK_DEBUG_1__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL1__SHIFT                                       0x0
+#define DSCC_DISPCLK_DEBUG_1__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL1_MASK                                         0x00007FFFL
+//DSCC_DISPCLK_DEBUG_2
+#define DSCC_DISPCLK_DEBUG_2__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL2__SHIFT                                       0x0
+#define DSCC_DISPCLK_DEBUG_2__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL2_MASK                                         0x00007FFFL
+//DSCC_DISPCLK_DEBUG_3
+#define DSCC_DISPCLK_DEBUG_3__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL3__SHIFT                                       0x0
+#define DSCC_DISPCLK_DEBUG_3__DSCC_OUTPUT_BUFFER_FULLNESS_LEVEL3_MASK                                         0x00007FFFL
+//DSCC_DISPCLK_DEBUG_4
+#define DSCC_DISPCLK_DEBUG_4__DSCC_INITIAL_XMIT_DELAY_REACHED__SHIFT                                          0x0
+#define DSCC_DISPCLK_DEBUG_4__DSCC_LAST_SLICE_PIXEL_COUNT_AT_INITIAL_XMIT_DELAY__SHIFT                        0x4
+#define DSCC_DISPCLK_DEBUG_4__DSCC_OUTPUT_BUFFER_PIXEL_THRESHOLD_AT_INITIAL_XMIT_DELAY__SHIFT                 0x10
+#define DSCC_DISPCLK_DEBUG_4__DSCC_INITIAL_XMIT_DELAY_REACHED_MASK                                            0x00000001L
+#define DSCC_DISPCLK_DEBUG_4__DSCC_LAST_SLICE_PIXEL_COUNT_AT_INITIAL_XMIT_DELAY_MASK                          0x00007FF0L
+#define DSCC_DISPCLK_DEBUG_4__DSCC_OUTPUT_BUFFER_PIXEL_THRESHOLD_AT_INITIAL_XMIT_DELAY_MASK                   0xFFFF0000L
+//DSCC_DISPCLK_DEBUG_5
+#define DSCC_DISPCLK_DEBUG_5__DSCC_TOTAL_PIXEL_COUNT_IN_OUTPUT_BUFFER__SHIFT                                  0x0
+#define DSCC_DISPCLK_DEBUG_5__DSCC_TOTAL_PIXEL_COUNT_IN_OUTPUT_BUFFER_MASK                                    0x0000FFFFL
+//DSCC_DISPCLK_DEBUG_6
+#define DSCC_DISPCLK_DEBUG_6__DSCC_DISPCLK_DEBUG_6__SHIFT                                                     0x0
+#define DSCC_DISPCLK_DEBUG_6__DSCC_DISPCLK_DEBUG_6_MASK                                                       0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_7
+#define DSCC_DISPCLK_DEBUG_7__DSCC_DISPCLK_DEBUG_7__SHIFT                                                     0x0
+#define DSCC_DISPCLK_DEBUG_7__DSCC_DISPCLK_DEBUG_7_MASK                                                       0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_8
+#define DSCC_DISPCLK_DEBUG_8__DSCC_DISPCLK_DEBUG_8__SHIFT                                                     0x0
+#define DSCC_DISPCLK_DEBUG_8__DSCC_DISPCLK_DEBUG_8_MASK                                                       0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_9
+#define DSCC_DISPCLK_DEBUG_9__DSCC_DISPCLK_DEBUG_9__SHIFT                                                     0x0
+#define DSCC_DISPCLK_DEBUG_9__DSCC_DISPCLK_DEBUG_9_MASK                                                       0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_10
+#define DSCC_DISPCLK_DEBUG_10__DSCC_DISPCLK_DEBUG_10__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_10__DSCC_DISPCLK_DEBUG_10_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_11
+#define DSCC_DISPCLK_DEBUG_11__DSCC_DISPCLK_DEBUG_11__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_11__DSCC_DISPCLK_DEBUG_11_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_12
+#define DSCC_DISPCLK_DEBUG_12__DSCC_DISPCLK_DEBUG_12__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_12__DSCC_DISPCLK_DEBUG_12_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_13
+#define DSCC_DISPCLK_DEBUG_13__DSCC_DISPCLK_DEBUG_13__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_13__DSCC_DISPCLK_DEBUG_13_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_14
+#define DSCC_DISPCLK_DEBUG_14__DSCC_DISPCLK_DEBUG_14__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_14__DSCC_DISPCLK_DEBUG_14_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_15
+#define DSCC_DISPCLK_DEBUG_15__DSCC_DISPCLK_DEBUG_15__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_15__DSCC_DISPCLK_DEBUG_15_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_16
+#define DSCC_DISPCLK_DEBUG_16__DSCC_DISPCLK_DEBUG_16__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_16__DSCC_DISPCLK_DEBUG_16_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_17
+#define DSCC_DISPCLK_DEBUG_17__DSCC_DISPCLK_DEBUG_17__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_17__DSCC_DISPCLK_DEBUG_17_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_18
+#define DSCC_DISPCLK_DEBUG_18__DSCC_DISPCLK_DEBUG_18__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_18__DSCC_DISPCLK_DEBUG_18_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_19
+#define DSCC_DISPCLK_DEBUG_19__DSCC_DISPCLK_DEBUG_19__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_19__DSCC_DISPCLK_DEBUG_19_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_20
+#define DSCC_DISPCLK_DEBUG_20__DSCC_DISPCLK_DEBUG_20__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_20__DSCC_DISPCLK_DEBUG_20_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_21
+#define DSCC_DISPCLK_DEBUG_21__DSCC_DISPCLK_DEBUG_21__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_21__DSCC_DISPCLK_DEBUG_21_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_22
+#define DSCC_DISPCLK_DEBUG_22__DSCC_DISPCLK_DEBUG_22__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_22__DSCC_DISPCLK_DEBUG_22_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_23
+#define DSCC_DISPCLK_DEBUG_23__DSCC_DISPCLK_DEBUG_23__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_23__DSCC_DISPCLK_DEBUG_23_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_24
+#define DSCC_DISPCLK_DEBUG_24__DSCC_DISPCLK_DEBUG_24__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_24__DSCC_DISPCLK_DEBUG_24_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_25
+#define DSCC_DISPCLK_DEBUG_25__DSCC_DISPCLK_DEBUG_25__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_25__DSCC_DISPCLK_DEBUG_25_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_26
+#define DSCC_DISPCLK_DEBUG_26__DSCC_DISPCLK_DEBUG_26__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_26__DSCC_DISPCLK_DEBUG_26_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_27
+#define DSCC_DISPCLK_DEBUG_27__DSCC_DISPCLK_DEBUG_27__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_27__DSCC_DISPCLK_DEBUG_27_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_28
+#define DSCC_DISPCLK_DEBUG_28__DSCC_DISPCLK_DEBUG_28__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_28__DSCC_DISPCLK_DEBUG_28_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_29
+#define DSCC_DISPCLK_DEBUG_29__DSCC_DISPCLK_DEBUG_29__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_29__DSCC_DISPCLK_DEBUG_29_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_30
+#define DSCC_DISPCLK_DEBUG_30__DSCC_DISPCLK_DEBUG_30__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_30__DSCC_DISPCLK_DEBUG_30_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_31
+#define DSCC_DISPCLK_DEBUG_31__DSCC_DISPCLK_DEBUG_31__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_31__DSCC_DISPCLK_DEBUG_31_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_32
+#define DSCC_DISPCLK_DEBUG_32__DSCC_DISPCLK_DEBUG_32__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_32__DSCC_DISPCLK_DEBUG_32_MASK                                                     0xFFFFFFFFL
+//DSCC_DISPCLK_DEBUG_33
+#define DSCC_DISPCLK_DEBUG_33__DSCC_DISPCLK_DEBUG_33__SHIFT                                                   0x0
+#define DSCC_DISPCLK_DEBUG_33__DSCC_DISPCLK_DEBUG_33_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: dsccif_dsccifdebugind
+//DSCCIF_DEBUG_ID
+#define DSCCIF_DEBUG_ID__DSCCIF_DEBUG_ID__SHIFT                                                               0x0
+#define DSCCIF_DEBUG_ID__DSCCIF_DEBUG_ID_MASK                                                                 0xFFFFFFFFL
+//DSCCIF_DEBUG_0
+#define DSCCIF_DEBUG_0__DSCCIF_DEBUG_0__SHIFT                                                                 0x0
+#define DSCCIF_DEBUG_0__DSCCIF_DEBUG_0_MASK                                                                   0xFFFFFFFFL
+//DSCCIF_DEBUG_1
+#define DSCCIF_DEBUG_1__DSCCIF_DEBUG_1__SHIFT                                                                 0x0
+#define DSCCIF_DEBUG_1__DSCCIF_DEBUG_1_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: dsc_top_dsc_topdebugind
+//DSC_TOP_DEBUG_ID
+#define DSC_TOP_DEBUG_ID__DSC_TOP_DEBUG_ID__SHIFT                                                             0x0
+#define DSC_TOP_DEBUG_ID__DSC_TOP_DEBUG_ID_MASK                                                               0xFFFFFFFFL
+//DSC_TOP_DEBUG_0
+#define DSC_TOP_DEBUG_0__DSC_TOP_DEBUG_0__SHIFT                                                               0x0
+#define DSC_TOP_DEBUG_0__DSC_TOP_DEBUG_0_MASK                                                                 0xFFFFFFFFL
+//DSC_TOP_DEBUG_1
+#define DSC_TOP_DEBUG_1__DSC_TOP_DEBUG_1__SHIFT                                                               0x0
+#define DSC_TOP_DEBUG_1__DSC_TOP_DEBUG_1_MASK                                                                 0xFFFFFFFFL
+//DSC_TOP_DEBUG_2
+#define DSC_TOP_DEBUG_2__DSC_TOP_DEBUG_2__SHIFT                                                               0x0
+#define DSC_TOP_DEBUG_2__DSC_TOP_DEBUG_2_MASK                                                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port0_dpiaportdebugind
+//DPIA_PORT0_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT0_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT0_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT0_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT0_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT0_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port1_dpiaportdebugind
+//DPIA_PORT1_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT1_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT1_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT1_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT1_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT1_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port2_dpiaportdebugind
+//DPIA_PORT2_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT2_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT2_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT2_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT2_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT2_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port3_dpiaportdebugind
+//DPIA_PORT3_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT3_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT3_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT3_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT3_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT3_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port4_dpiaportdebugind
+//DPIA_PORT4_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT4_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT4_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT4_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT4_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT4_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port5_dpiaportdebugind
+//DPIA_PORT5_DPIA_PORT_DEBUG_ID
+#define DPIA_PORT5_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID__SHIFT                                              0x0
+#define DPIA_PORT5_DPIA_PORT_DEBUG_ID__DPIA_PORT_DEBUG_ID_MASK                                                0xFFFFFFFFL
+//DPIA_PORT5_DPIA_PORT_DEBUG_BUS0
+#define DPIA_PORT5_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0__SHIFT                                          0x0
+#define DPIA_PORT5_DPIA_PORT_DEBUG_BUS0__DPIA_PORT_DEBUG_BUS0_MASK                                            0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml0_dpiaportmldebugind
+//DPIA_PORT_ML0_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML0_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML0_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml1_dpiaportmldebugind
+//DPIA_PORT_ML1_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML1_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML1_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml2_dpiaportmldebugind
+//DPIA_PORT_ML2_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML2_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML2_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml3_dpiaportmldebugind
+//DPIA_PORT_ML3_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML3_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML3_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml4_dpiaportmldebugind
+//DPIA_PORT_ML4_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML4_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML4_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_ml5_dpiaportmldebugind
+//DPIA_PORT_ML5_DPIA_PORT_ML_TEST_DEBUG_ID
+#define DPIA_PORT_ML5_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID__SHIFT                           0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_TEST_DEBUG_ID__DPIA_PORT_ML_TEST_DEBUG_ID_MASK                             0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA0__DPIA_PORT_ML_DEBUG_DATA0_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA1
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA1__DPIA_PORT_ML_DEBUG_DATA1_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA2
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA2__DPIA_PORT_ML_DEBUG_DATA2_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA3
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA3__DPIA_PORT_ML_DEBUG_DATA3_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA4
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA4__DPIA_PORT_ML_DEBUG_DATA4_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA5
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA5__DPIA_PORT_ML_DEBUG_DATA5_MASK                                 0xFFFFFFFFL
+//DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA6
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6__SHIFT                               0x0
+#define DPIA_PORT_ML5_DPIA_PORT_ML_DEBUG_DATA6__DPIA_PORT_ML_DEBUG_DATA6_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux0_dpiaportauxdebugind
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX0_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux1_dpiaportauxdebugind
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX1_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux2_dpiaportauxdebugind
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX2_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux3_dpiaportauxdebugind
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX3_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux4_dpiaportauxdebugind
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX4_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_port_aux5_dpiaportauxdebugind
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG_ID
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID__SHIFT                        0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG_ID__DPIA_PORT_AUX_TEST_DEBUG_ID_MASK                          0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG1
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG1__DPIA_PORT_AUX_TEST_DEBUG1_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG2
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG2__DPIA_PORT_AUX_TEST_DEBUG2_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG3
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG3__DPIA_PORT_AUX_TEST_DEBUG3_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG4
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG4__DPIA_PORT_AUX_TEST_DEBUG4_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG5
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG5__DPIA_PORT_AUX_TEST_DEBUG5_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG6
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG6__DPIA_PORT_AUX_TEST_DEBUG6_MASK                              0xFFFFFFFFL
+//DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG7
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7__SHIFT                            0x0
+#define DPIA_PORT_AUX5_DPIA_PORT_AUX_TEST_DEBUG7__DPIA_PORT_AUX_TEST_DEBUG7_MASK                              0xFFFFFFFFL
+
+
+// addressBlock: dpia_mu0_dpia_mu_debugind
+//DPIA_MU_DEBUG_ID
+#define DPIA_MU_DEBUG_ID__DEBUG_ID__SHIFT                                                                     0x0
+#define DPIA_MU_DEBUG_ID__DEBUG_ID_MASK                                                                       0xFFFFFFFFL
+
+
+// addressBlock: azendpoint_descriptorind
+//AUDIO_DESCRIPTOR0
+#define AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR1
+#define AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR2
+#define AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR3
+#define AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR4
+#define AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR5
+#define AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR6
+#define AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR7
+#define AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR8
+#define AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR9
+#define AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT                                                                0x0
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT                                                       0x8
+#define AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT                                                           0x10
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                                0x18
+#define AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK                                                                  0x00000007L
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK                                                         0x0000FF00L
+#define AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK                                                             0x00FF0000L
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_STEREO_MASK                                                  0xFF000000L
+//AUDIO_DESCRIPTOR10
+#define AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT                                                               0x0
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT                                                      0x8
+#define AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT                                                          0x10
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                               0x18
+#define AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK                                                                 0x00000007L
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK                                                        0x0000FF00L
+#define AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK                                                            0x00FF0000L
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_STEREO_MASK                                                 0xFF000000L
+//AUDIO_DESCRIPTOR11
+#define AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT                                                               0x0
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT                                                      0x8
+#define AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT                                                          0x10
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                               0x18
+#define AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK                                                                 0x00000007L
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK                                                        0x0000FF00L
+#define AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK                                                            0x00FF0000L
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_STEREO_MASK                                                 0xFF000000L
+//AUDIO_DESCRIPTOR12
+#define AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT                                                               0x0
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT                                                      0x8
+#define AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT                                                          0x10
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                               0x18
+#define AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK                                                                 0x00000007L
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK                                                        0x0000FF00L
+#define AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK                                                            0x00FF0000L
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_STEREO_MASK                                                 0xFF000000L
+//AUDIO_DESCRIPTOR13
+#define AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT                                                               0x0
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT                                                      0x8
+#define AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT                                                          0x10
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_STEREO__SHIFT                                               0x18
+#define AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK                                                                 0x00000007L
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK                                                        0x0000FF00L
+#define AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK                                                            0x00FF0000L
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_STEREO_MASK                                                 0xFF000000L
+
+
+// addressBlock: dce_dc_hda_azendpoint_azdec
+//AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT        0x0
+#define AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK          0xFFFFFFFFL
+//AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT       0x0
+#define AZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK         0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azinputendpoint_azdec
+//AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA
+#define AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT    0x0
+#define AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK      0xFFFFFFFFL
+//AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX
+#define AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT   0x0
+#define AZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK     0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azroot_azdec
+//AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT                0x0
+#define AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK                  0xFFFFFFFFL
+//AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT               0x0
+#define AZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK                 0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azstream0_azdec
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream1_azdec
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream2_azdec
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream3_azdec
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream4_azdec
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream5_azdec
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream6_azdec
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream7_azdec
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dccg_dccg_dispdec
+//PHYPLLA_PIXCLK_RESYNC_CNTL
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_PIXCLK_RESYNC_ENABLE__SHIFT                                       0x0
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT                               0x1
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_DCCG_DEEP_COLOR_CNTL__SHIFT                                       0x4
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_PIXCLK_ENABLE__SHIFT                                              0x8
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_PIXCLK_RESYNC_ENABLE_MASK                                         0x00000001L
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_DEEP_COLOR_DTO_ENABLE_STATUS_MASK                                 0x00000002L
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_DCCG_DEEP_COLOR_CNTL_MASK                                         0x00000030L
+#define PHYPLLA_PIXCLK_RESYNC_CNTL__PHYPLLA_PIXCLK_ENABLE_MASK                                                0x00000100L
+//PHYPLLB_PIXCLK_RESYNC_CNTL
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_PIXCLK_RESYNC_ENABLE__SHIFT                                       0x0
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT                               0x1
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_DCCG_DEEP_COLOR_CNTL__SHIFT                                       0x4
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_PIXCLK_ENABLE__SHIFT                                              0x8
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_PIXCLK_RESYNC_ENABLE_MASK                                         0x00000001L
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_DEEP_COLOR_DTO_ENABLE_STATUS_MASK                                 0x00000002L
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_DCCG_DEEP_COLOR_CNTL_MASK                                         0x00000030L
+#define PHYPLLB_PIXCLK_RESYNC_CNTL__PHYPLLB_PIXCLK_ENABLE_MASK                                                0x00000100L
+//PHYPLLC_PIXCLK_RESYNC_CNTL
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_PIXCLK_RESYNC_ENABLE__SHIFT                                       0x0
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT                               0x1
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_DCCG_DEEP_COLOR_CNTL__SHIFT                                       0x4
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_PIXCLK_ENABLE__SHIFT                                              0x8
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_PIXCLK_RESYNC_ENABLE_MASK                                         0x00000001L
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_DEEP_COLOR_DTO_ENABLE_STATUS_MASK                                 0x00000002L
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_DCCG_DEEP_COLOR_CNTL_MASK                                         0x00000030L
+#define PHYPLLC_PIXCLK_RESYNC_CNTL__PHYPLLC_PIXCLK_ENABLE_MASK                                                0x00000100L
+//PHYPLLD_PIXCLK_RESYNC_CNTL
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_PIXCLK_RESYNC_ENABLE__SHIFT                                       0x0
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT                               0x1
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_DCCG_DEEP_COLOR_CNTL__SHIFT                                       0x4
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_PIXCLK_ENABLE__SHIFT                                              0x8
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_PIXCLK_RESYNC_ENABLE_MASK                                         0x00000001L
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_DEEP_COLOR_DTO_ENABLE_STATUS_MASK                                 0x00000002L
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_DCCG_DEEP_COLOR_CNTL_MASK                                         0x00000030L
+#define PHYPLLD_PIXCLK_RESYNC_CNTL__PHYPLLD_PIXCLK_ENABLE_MASK                                                0x00000100L
+//DP_DTO_DBUF_EN
+#define DP_DTO_DBUF_EN__DP_DTO0_DBUF_EN__SHIFT                                                                0x0
+#define DP_DTO_DBUF_EN__DP_DTO1_DBUF_EN__SHIFT                                                                0x1
+#define DP_DTO_DBUF_EN__DP_DTO2_DBUF_EN__SHIFT                                                                0x2
+#define DP_DTO_DBUF_EN__DP_DTO3_DBUF_EN__SHIFT                                                                0x3
+#define DP_DTO_DBUF_EN__DP_DTO4_DBUF_EN__SHIFT                                                                0x4
+#define DP_DTO_DBUF_EN__DP_DTO5_DBUF_EN__SHIFT                                                                0x5
+#define DP_DTO_DBUF_EN__DP_DTO6_DBUF_EN__SHIFT                                                                0x6
+#define DP_DTO_DBUF_EN__DP_DTO7_DBUF_EN__SHIFT                                                                0x7
+#define DP_DTO_DBUF_EN__DP_DTO0_DBUF_EN_MASK                                                                  0x00000001L
+#define DP_DTO_DBUF_EN__DP_DTO1_DBUF_EN_MASK                                                                  0x00000002L
+#define DP_DTO_DBUF_EN__DP_DTO2_DBUF_EN_MASK                                                                  0x00000004L
+#define DP_DTO_DBUF_EN__DP_DTO3_DBUF_EN_MASK                                                                  0x00000008L
+#define DP_DTO_DBUF_EN__DP_DTO4_DBUF_EN_MASK                                                                  0x00000010L
+#define DP_DTO_DBUF_EN__DP_DTO5_DBUF_EN_MASK                                                                  0x00000020L
+#define DP_DTO_DBUF_EN__DP_DTO6_DBUF_EN_MASK                                                                  0x00000040L
+#define DP_DTO_DBUF_EN__DP_DTO7_DBUF_EN_MASK                                                                  0x00000080L
+//DSCCLK3_DTO_PARAM
+#define DSCCLK3_DTO_PARAM__DSCCLK3_DTO_PHASE__SHIFT                                                           0x0
+#define DSCCLK3_DTO_PARAM__DSCCLK3_DTO_MODULO__SHIFT                                                          0x10
+#define DSCCLK3_DTO_PARAM__DSCCLK3_DTO_PHASE_MASK                                                             0x000000FFL
+#define DSCCLK3_DTO_PARAM__DSCCLK3_DTO_MODULO_MASK                                                            0x00FF0000L
+//DPREFCLK_CGTT_BLK_CTRL_REG
+#define DPREFCLK_CGTT_BLK_CTRL_REG__DPREFCLK_TURN_ON_DELAY__SHIFT                                             0x0
+#define DPREFCLK_CGTT_BLK_CTRL_REG__DPREFCLK_TURN_OFF_DELAY__SHIFT                                            0x4
+#define DPREFCLK_CGTT_BLK_CTRL_REG__DPREFCLK_TURN_ON_DELAY_MASK                                               0x0000000FL
+#define DPREFCLK_CGTT_BLK_CTRL_REG__DPREFCLK_TURN_OFF_DELAY_MASK                                              0x00000FF0L
+//DCCG_GATE_DISABLE_CNTL4
+#define DCCG_GATE_DISABLE_CNTL4__PHYA_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x0
+#define DCCG_GATE_DISABLE_CNTL4__PHYB_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x1
+#define DCCG_GATE_DISABLE_CNTL4__PHYC_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x2
+#define DCCG_GATE_DISABLE_CNTL4__PHYD_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x3
+#define DCCG_GATE_DISABLE_CNTL4__PHYE_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x4
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_ROOT_GATE_DISABLE__SHIFT                                         0x7
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_ROOT_GATE_DISABLE__SHIFT                                         0x8
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_ROOT_GATE_DISABLE__SHIFT                                         0x9
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_ROOT_GATE_DISABLE__SHIFT                                         0xa
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK4_ROOT_GATE_DISABLE__SHIFT                                         0xb
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK5_ROOT_GATE_DISABLE__SHIFT                                         0xc
+#define DCCG_GATE_DISABLE_CNTL4__HDMICHARCLK0_ROOT_GATE_DISABLE__SHIFT                                        0x11
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_GATE_DISABLE__SHIFT                                              0x17
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_GATE_DISABLE__SHIFT                                              0x18
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_GATE_DISABLE__SHIFT                                              0x19
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_GATE_DISABLE__SHIFT                                              0x1a
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK4_GATE_DISABLE__SHIFT                                              0x1b
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK5_GATE_DISABLE__SHIFT                                              0x1c
+#define DCCG_GATE_DISABLE_CNTL4__PHYA_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000001L
+#define DCCG_GATE_DISABLE_CNTL4__PHYB_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000002L
+#define DCCG_GATE_DISABLE_CNTL4__PHYC_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000004L
+#define DCCG_GATE_DISABLE_CNTL4__PHYD_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000008L
+#define DCCG_GATE_DISABLE_CNTL4__PHYE_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000010L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_ROOT_GATE_DISABLE_MASK                                           0x00000080L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_ROOT_GATE_DISABLE_MASK                                           0x00000100L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_ROOT_GATE_DISABLE_MASK                                           0x00000200L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_ROOT_GATE_DISABLE_MASK                                           0x00000400L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK4_ROOT_GATE_DISABLE_MASK                                           0x00000800L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK5_ROOT_GATE_DISABLE_MASK                                           0x00001000L
+#define DCCG_GATE_DISABLE_CNTL4__HDMICHARCLK0_ROOT_GATE_DISABLE_MASK                                          0x00020000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_GATE_DISABLE_MASK                                                0x00800000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_GATE_DISABLE_MASK                                                0x01000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_GATE_DISABLE_MASK                                                0x02000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_GATE_DISABLE_MASK                                                0x04000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK4_GATE_DISABLE_MASK                                                0x08000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK5_GATE_DISABLE_MASK                                                0x10000000L
+//DPSTREAMCLK_CNTL
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK0_SRC_SEL__SHIFT                                                         0x0
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK0_EN__SHIFT                                                              0x3
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK1_SRC_SEL__SHIFT                                                         0x4
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK1_EN__SHIFT                                                              0x7
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK2_SRC_SEL__SHIFT                                                         0x8
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK2_EN__SHIFT                                                              0xb
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK3_SRC_SEL__SHIFT                                                         0xc
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK3_EN__SHIFT                                                              0xf
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK0_SRC_SEL_MASK                                                           0x00000007L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK0_EN_MASK                                                                0x00000008L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK1_SRC_SEL_MASK                                                           0x00000070L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK1_EN_MASK                                                                0x00000080L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK2_SRC_SEL_MASK                                                           0x00000700L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK2_EN_MASK                                                                0x00000800L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK3_SRC_SEL_MASK                                                           0x00007000L
+#define DPSTREAMCLK_CNTL__DPSTREAMCLK3_EN_MASK                                                                0x00008000L
+//REFCLK_CGTT_BLK_CTRL_REG
+#define REFCLK_CGTT_BLK_CTRL_REG__REFCLK_TURN_ON_DELAY__SHIFT                                                 0x0
+#define REFCLK_CGTT_BLK_CTRL_REG__REFCLK_TURN_OFF_DELAY__SHIFT                                                0x4
+#define REFCLK_CGTT_BLK_CTRL_REG__REFCLK_TURN_ON_DELAY_MASK                                                   0x0000000FL
+#define REFCLK_CGTT_BLK_CTRL_REG__REFCLK_TURN_OFF_DELAY_MASK                                                  0x00000FF0L
+//PHYPLLE_PIXCLK_RESYNC_CNTL
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_PIXCLK_RESYNC_ENABLE__SHIFT                                       0x0
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT                               0x1
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_DCCG_DEEP_COLOR_CNTL__SHIFT                                       0x4
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_PIXCLK_ENABLE__SHIFT                                              0x8
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_PIXCLK_RESYNC_ENABLE_MASK                                         0x00000001L
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_DEEP_COLOR_DTO_ENABLE_STATUS_MASK                                 0x00000002L
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_DCCG_DEEP_COLOR_CNTL_MASK                                         0x00000030L
+#define PHYPLLE_PIXCLK_RESYNC_CNTL__PHYPLLE_PIXCLK_ENABLE_MASK                                                0x00000100L
+//DCCG_PERFMON_CNTL2
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_DSICLK_ENABLE__SHIFT                                                    0x0
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_REFCLK_ENABLE__SHIFT                                                    0x1
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_PIXCLK1_ENABLE__SHIFT                                                   0x2
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_PIXCLK2_ENABLE__SHIFT                                                   0x3
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYC_PIXCLK_ENABLE__SHIFT                                            0x4
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYD_PIXCLK_ENABLE__SHIFT                                            0x5
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYE_PIXCLK_ENABLE__SHIFT                                            0x6
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYF_PIXCLK_ENABLE__SHIFT                                            0x7
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYG_PIXCLK_ENABLE__SHIFT                                            0x8
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_DTBCLK0_ENABLE__SHIFT                                                   0x9
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_DSICLK_ENABLE_MASK                                                      0x00000001L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_REFCLK_ENABLE_MASK                                                      0x00000002L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_PIXCLK1_ENABLE_MASK                                                     0x00000004L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_PIXCLK2_ENABLE_MASK                                                     0x00000008L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYC_PIXCLK_ENABLE_MASK                                              0x00000010L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYD_PIXCLK_ENABLE_MASK                                              0x00000020L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYE_PIXCLK_ENABLE_MASK                                              0x00000040L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYF_PIXCLK_ENABLE_MASK                                              0x00000080L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_UNIPHYG_PIXCLK_ENABLE_MASK                                              0x00000100L
+#define DCCG_PERFMON_CNTL2__DCCG_PERF_DTBCLK0_ENABLE_MASK                                                     0x00000200L
+//DPIASYMCLK_CNTL
+#define DPIASYMCLK_CNTL__DPIASYMCLK0_SRC_SEL__SHIFT                                                           0x0
+#define DPIASYMCLK_CNTL__DPIASYMCLK0_EN__SHIFT                                                                0x3
+#define DPIASYMCLK_CNTL__DPIASYMCLK1_SRC_SEL__SHIFT                                                           0x4
+#define DPIASYMCLK_CNTL__DPIASYMCLK1_EN__SHIFT                                                                0x7
+#define DPIASYMCLK_CNTL__DPIASYMCLK2_SRC_SEL__SHIFT                                                           0x8
+#define DPIASYMCLK_CNTL__DPIASYMCLK2_EN__SHIFT                                                                0xb
+#define DPIASYMCLK_CNTL__DPIASYMCLK3_SRC_SEL__SHIFT                                                           0xc
+#define DPIASYMCLK_CNTL__DPIASYMCLK3_EN__SHIFT                                                                0xf
+#define DPIASYMCLK_CNTL__DPIASYMCLK4_SRC_SEL__SHIFT                                                           0x10
+#define DPIASYMCLK_CNTL__DPIASYMCLK4_EN__SHIFT                                                                0x13
+#define DPIASYMCLK_CNTL__DPIASYMCLK5_SRC_SEL__SHIFT                                                           0x14
+#define DPIASYMCLK_CNTL__DPIASYMCLK5_EN__SHIFT                                                                0x17
+#define DPIASYMCLK_CNTL__DPIASYMCLK0_SRC_SEL_MASK                                                             0x00000007L
+#define DPIASYMCLK_CNTL__DPIASYMCLK0_EN_MASK                                                                  0x00000008L
+#define DPIASYMCLK_CNTL__DPIASYMCLK1_SRC_SEL_MASK                                                             0x00000070L
+#define DPIASYMCLK_CNTL__DPIASYMCLK1_EN_MASK                                                                  0x00000080L
+#define DPIASYMCLK_CNTL__DPIASYMCLK2_SRC_SEL_MASK                                                             0x00000700L
+#define DPIASYMCLK_CNTL__DPIASYMCLK2_EN_MASK                                                                  0x00000800L
+#define DPIASYMCLK_CNTL__DPIASYMCLK3_SRC_SEL_MASK                                                             0x00007000L
+#define DPIASYMCLK_CNTL__DPIASYMCLK3_EN_MASK                                                                  0x00008000L
+#define DPIASYMCLK_CNTL__DPIASYMCLK4_SRC_SEL_MASK                                                             0x00070000L
+#define DPIASYMCLK_CNTL__DPIASYMCLK4_EN_MASK                                                                  0x00080000L
+#define DPIASYMCLK_CNTL__DPIASYMCLK5_SRC_SEL_MASK                                                             0x00700000L
+#define DPIASYMCLK_CNTL__DPIASYMCLK5_EN_MASK                                                                  0x00800000L
+//DCCG_GLOBAL_FGCG_REP_CNTL
+#define DCCG_GLOBAL_FGCG_REP_CNTL__DCCG_GLOBAL_FGCG_REP_DIS__SHIFT                                            0x0
+#define DCCG_GLOBAL_FGCG_REP_CNTL__DCCG_GLOBAL_FGCG_REP_DIS_MASK                                              0x00000001L
+//OTG_ADD_DROP_PIXEL_CNTL
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG0_ADD_PIXEL__SHIFT                                                        0x0
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG1_ADD_PIXEL__SHIFT                                                        0x1
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG2_ADD_PIXEL__SHIFT                                                        0x2
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG3_ADD_PIXEL__SHIFT                                                        0x3
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG0_DROP_PIXEL__SHIFT                                                       0x8
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG1_DROP_PIXEL__SHIFT                                                       0x9
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG2_DROP_PIXEL__SHIFT                                                       0xa
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG3_DROP_PIXEL__SHIFT                                                       0xb
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG_ADD_DROP_PIXEL_DONE__SHIFT                                               0x1f
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG0_ADD_PIXEL_MASK                                                          0x00000001L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG1_ADD_PIXEL_MASK                                                          0x00000002L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG2_ADD_PIXEL_MASK                                                          0x00000004L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG3_ADD_PIXEL_MASK                                                          0x00000008L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG0_DROP_PIXEL_MASK                                                         0x00000100L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG1_DROP_PIXEL_MASK                                                         0x00000200L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG2_DROP_PIXEL_MASK                                                         0x00000400L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG3_DROP_PIXEL_MASK                                                         0x00000800L
+#define OTG_ADD_DROP_PIXEL_CNTL__OTG_ADD_DROP_PIXEL_DONE_MASK                                                 0x80000000L
+//DPREFCLK_CNTL
+#define DPREFCLK_CNTL__DPREFCLK_SRC_SEL__SHIFT                                                                0x0
+#define DPREFCLK_CNTL__DPREFCLK_SRC_SEL_MASK                                                                  0x00000007L
+//DCE_VERSION
+#define DCE_VERSION__MAJOR_VERSION__SHIFT                                                                     0x0
+#define DCE_VERSION__MINOR_VERSION__SHIFT                                                                     0x8
+#define DCE_VERSION__MAJOR_VERSION_MASK                                                                       0x000000FFL
+#define DCE_VERSION__MINOR_VERSION_MASK                                                                       0x0000FF00L
+//SYMCLK32_SE_CNTL
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE0_SRC_SEL__SHIFT                                                         0x0
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE0_EN__SHIFT                                                              0x3
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE1_SRC_SEL__SHIFT                                                         0x4
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE1_EN__SHIFT                                                              0x7
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE2_SRC_SEL__SHIFT                                                         0x8
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE2_EN__SHIFT                                                              0xb
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE3_SRC_SEL__SHIFT                                                         0xc
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE3_EN__SHIFT                                                              0xf
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE0_SRC_SEL_MASK                                                           0x00000007L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE0_EN_MASK                                                                0x00000008L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE1_SRC_SEL_MASK                                                           0x00000070L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE1_EN_MASK                                                                0x00000080L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE2_SRC_SEL_MASK                                                           0x00000700L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE2_EN_MASK                                                                0x00000800L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE3_SRC_SEL_MASK                                                           0x00007000L
+#define SYMCLK32_SE_CNTL__SYMCLK32_SE3_EN_MASK                                                                0x00008000L
+//SYMCLK32_LE_CNTL
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE0_SRC_SEL__SHIFT                                                         0x0
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE0_EN__SHIFT                                                              0x3
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE1_SRC_SEL__SHIFT                                                         0x4
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE1_EN__SHIFT                                                              0x7
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE2_SRC_SEL__SHIFT                                                         0x8
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE2_EN__SHIFT                                                              0xb
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE3_SRC_SEL__SHIFT                                                         0xc
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE3_EN__SHIFT                                                              0xf
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE0_SRC_SEL_MASK                                                           0x00000007L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE0_EN_MASK                                                                0x00000008L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE1_SRC_SEL_MASK                                                           0x00000070L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE1_EN_MASK                                                                0x00000080L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE2_SRC_SEL_MASK                                                           0x00000700L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE2_EN_MASK                                                                0x00000800L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE3_SRC_SEL_MASK                                                           0x00007000L
+#define SYMCLK32_LE_CNTL__SYMCLK32_LE3_EN_MASK                                                                0x00008000L
+//DTBCLK_P_CNTL
+#define DTBCLK_P_CNTL__DTBCLK_P0_SRC_SEL__SHIFT                                                               0x0
+#define DTBCLK_P_CNTL__DTBCLK_P0_EN__SHIFT                                                                    0x2
+#define DTBCLK_P_CNTL__DTBCLK_P1_SRC_SEL__SHIFT                                                               0x3
+#define DTBCLK_P_CNTL__DTBCLK_P1_EN__SHIFT                                                                    0x5
+#define DTBCLK_P_CNTL__DTBCLK_P2_SRC_SEL__SHIFT                                                               0x6
+#define DTBCLK_P_CNTL__DTBCLK_P2_EN__SHIFT                                                                    0x8
+#define DTBCLK_P_CNTL__DTBCLK_P3_SRC_SEL__SHIFT                                                               0x9
+#define DTBCLK_P_CNTL__DTBCLK_P3_EN__SHIFT                                                                    0xb
+#define DTBCLK_P_CNTL__DTBCLK_P0_SRC_SEL_MASK                                                                 0x00000003L
+#define DTBCLK_P_CNTL__DTBCLK_P0_EN_MASK                                                                      0x00000004L
+#define DTBCLK_P_CNTL__DTBCLK_P1_SRC_SEL_MASK                                                                 0x00000018L
+#define DTBCLK_P_CNTL__DTBCLK_P1_EN_MASK                                                                      0x00000020L
+#define DTBCLK_P_CNTL__DTBCLK_P2_SRC_SEL_MASK                                                                 0x000000C0L
+#define DTBCLK_P_CNTL__DTBCLK_P2_EN_MASK                                                                      0x00000100L
+#define DTBCLK_P_CNTL__DTBCLK_P3_SRC_SEL_MASK                                                                 0x00000600L
+#define DTBCLK_P_CNTL__DTBCLK_P3_EN_MASK                                                                      0x00000800L
+//DCCG_GATE_DISABLE_CNTL5
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P0_GATE_DISABLE__SHIFT                                                0x0
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P1_GATE_DISABLE__SHIFT                                                0x1
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P2_GATE_DISABLE__SHIFT                                                0x2
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P3_GATE_DISABLE__SHIFT                                                0x3
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK0_ROOT_GATE_DISABLE__SHIFT                                        0x6
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK0_GATE_DISABLE__SHIFT                                             0x7
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK1_ROOT_GATE_DISABLE__SHIFT                                        0x8
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK1_GATE_DISABLE__SHIFT                                             0x9
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK2_ROOT_GATE_DISABLE__SHIFT                                        0xa
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK2_GATE_DISABLE__SHIFT                                             0xb
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK3_ROOT_GATE_DISABLE__SHIFT                                        0xc
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK3_GATE_DISABLE__SHIFT                                             0xd
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKA_ROOT_GATE_DISABLE__SHIFT                                             0x12
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKB_ROOT_GATE_DISABLE__SHIFT                                             0x13
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKC_ROOT_GATE_DISABLE__SHIFT                                             0x14
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKD_ROOT_GATE_DISABLE__SHIFT                                             0x15
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKE_ROOT_GATE_DISABLE__SHIFT                                             0x16
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKA_FE_ROOT_GATE_DISABLE__SHIFT                                          0x18
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKB_FE_ROOT_GATE_DISABLE__SHIFT                                          0x19
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKC_FE_ROOT_GATE_DISABLE__SHIFT                                          0x1a
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKD_FE_ROOT_GATE_DISABLE__SHIFT                                          0x1b
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKE_FE_ROOT_GATE_DISABLE__SHIFT                                          0x1c
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P0_GATE_DISABLE_MASK                                                  0x00000001L
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P1_GATE_DISABLE_MASK                                                  0x00000002L
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P2_GATE_DISABLE_MASK                                                  0x00000004L
+#define DCCG_GATE_DISABLE_CNTL5__DTBCLK_P3_GATE_DISABLE_MASK                                                  0x00000008L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK0_ROOT_GATE_DISABLE_MASK                                          0x00000040L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK0_GATE_DISABLE_MASK                                               0x00000080L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK1_ROOT_GATE_DISABLE_MASK                                          0x00000100L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK1_GATE_DISABLE_MASK                                               0x00000200L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK2_ROOT_GATE_DISABLE_MASK                                          0x00000400L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK2_GATE_DISABLE_MASK                                               0x00000800L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK3_ROOT_GATE_DISABLE_MASK                                          0x00001000L
+#define DCCG_GATE_DISABLE_CNTL5__DPSTREAMCLK3_GATE_DISABLE_MASK                                               0x00002000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKA_ROOT_GATE_DISABLE_MASK                                               0x00040000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKB_ROOT_GATE_DISABLE_MASK                                               0x00080000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKC_ROOT_GATE_DISABLE_MASK                                               0x00100000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKD_ROOT_GATE_DISABLE_MASK                                               0x00200000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKE_ROOT_GATE_DISABLE_MASK                                               0x00400000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKA_FE_ROOT_GATE_DISABLE_MASK                                            0x01000000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKB_FE_ROOT_GATE_DISABLE_MASK                                            0x02000000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKC_FE_ROOT_GATE_DISABLE_MASK                                            0x04000000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKD_FE_ROOT_GATE_DISABLE_MASK                                            0x08000000L
+#define DCCG_GATE_DISABLE_CNTL5__SYMCLKE_FE_ROOT_GATE_DISABLE_MASK                                            0x10000000L
+//DSCCLK0_DTO_PARAM
+#define DSCCLK0_DTO_PARAM__DSCCLK0_DTO_PHASE__SHIFT                                                           0x0
+#define DSCCLK0_DTO_PARAM__DSCCLK0_DTO_MODULO__SHIFT                                                          0x10
+#define DSCCLK0_DTO_PARAM__DSCCLK0_DTO_PHASE_MASK                                                             0x000000FFL
+#define DSCCLK0_DTO_PARAM__DSCCLK0_DTO_MODULO_MASK                                                            0x00FF0000L
+//DSCCLK1_DTO_PARAM
+#define DSCCLK1_DTO_PARAM__DSCCLK1_DTO_PHASE__SHIFT                                                           0x0
+#define DSCCLK1_DTO_PARAM__DSCCLK1_DTO_MODULO__SHIFT                                                          0x10
+#define DSCCLK1_DTO_PARAM__DSCCLK1_DTO_PHASE_MASK                                                             0x000000FFL
+#define DSCCLK1_DTO_PARAM__DSCCLK1_DTO_MODULO_MASK                                                            0x00FF0000L
+//DSCCLK2_DTO_PARAM
+#define DSCCLK2_DTO_PARAM__DSCCLK2_DTO_PHASE__SHIFT                                                           0x0
+#define DSCCLK2_DTO_PARAM__DSCCLK2_DTO_MODULO__SHIFT                                                          0x10
+#define DSCCLK2_DTO_PARAM__DSCCLK2_DTO_PHASE_MASK                                                             0x000000FFL
+#define DSCCLK2_DTO_PARAM__DSCCLK2_DTO_MODULO_MASK                                                            0x00FF0000L
+//OTG_PIXEL_RATE_DIV
+#define OTG_PIXEL_RATE_DIV__OTG0_TMDS_PIXEL_RATE_DIV__SHIFT                                                   0x0
+#define OTG_PIXEL_RATE_DIV__DPDTO0_INT__SHIFT                                                                 0x1
+#define OTG_PIXEL_RATE_DIV__OTG1_TMDS_PIXEL_RATE_DIV__SHIFT                                                   0x5
+#define OTG_PIXEL_RATE_DIV__DPDTO1_INT__SHIFT                                                                 0x6
+#define OTG_PIXEL_RATE_DIV__OTG2_TMDS_PIXEL_RATE_DIV__SHIFT                                                   0xa
+#define OTG_PIXEL_RATE_DIV__DPDTO2_INT__SHIFT                                                                 0xb
+#define OTG_PIXEL_RATE_DIV__OTG3_TMDS_PIXEL_RATE_DIV__SHIFT                                                   0xf
+#define OTG_PIXEL_RATE_DIV__DPDTO3_INT__SHIFT                                                                 0x10
+#define OTG_PIXEL_RATE_DIV__OTG0_TMDS_PIXEL_RATE_DIV_MASK                                                     0x00000001L
+#define OTG_PIXEL_RATE_DIV__DPDTO0_INT_MASK                                                                   0x0000001EL
+#define OTG_PIXEL_RATE_DIV__OTG1_TMDS_PIXEL_RATE_DIV_MASK                                                     0x00000020L
+#define OTG_PIXEL_RATE_DIV__DPDTO1_INT_MASK                                                                   0x000003C0L
+#define OTG_PIXEL_RATE_DIV__OTG2_TMDS_PIXEL_RATE_DIV_MASK                                                     0x00000400L
+#define OTG_PIXEL_RATE_DIV__DPDTO2_INT_MASK                                                                   0x00007800L
+#define OTG_PIXEL_RATE_DIV__OTG3_TMDS_PIXEL_RATE_DIV_MASK                                                     0x00008000L
+#define OTG_PIXEL_RATE_DIV__DPDTO3_INT_MASK                                                                   0x000F0000L
+//MILLISECOND_TIME_BASE_DIV
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_DIV__SHIFT                                           0x0
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_CLOCK_SOURCE_SEL__SHIFT                              0x14
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_DIV_MASK                                             0x0001FFFFL
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_CLOCK_SOURCE_SEL_MASK                                0x00100000L
+//DISPCLK_FREQ_CHANGE_CNTL
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_DELAY__SHIFT                                                   0x0
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_SIZE__SHIFT                                                    0x10
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_FREQ_RAMP_DONE__SHIFT                                               0x14
+#define DISPCLK_FREQ_CHANGE_CNTL__RESYNC_FIFO_LEVEL_ADJUST_EN__SHIFT                                          0x15
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_RESET__SHIFT                                               0x16
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_MAX_ERRDET_CYCLES__SHIFT                                            0x18
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_STATE__SHIFT                                               0x1d
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_OVR_EN__SHIFT                                              0x1e
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_CHG_FWD_CORR_DISABLE__SHIFT                                         0x1f
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_DELAY_MASK                                                     0x00003FFFL
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_SIZE_MASK                                                      0x000F0000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_FREQ_RAMP_DONE_MASK                                                 0x00100000L
+#define DISPCLK_FREQ_CHANGE_CNTL__RESYNC_FIFO_LEVEL_ADJUST_EN_MASK                                            0x00200000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_RESET_MASK                                                 0x00400000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_MAX_ERRDET_CYCLES_MASK                                              0x1F000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_STATE_MASK                                                 0x20000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_OVR_EN_MASK                                                0x40000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_CHG_FWD_CORR_DISABLE_MASK                                           0x80000000L
+//DC_MEM_GLOBAL_PWR_REQ_CNTL
+#define DC_MEM_GLOBAL_PWR_REQ_CNTL__DC_MEM_GLOBAL_PWR_REQ_DIS__SHIFT                                          0x0
+#define DC_MEM_GLOBAL_PWR_REQ_CNTL__DC_MEM_GLOBAL_PWR_REQ_DIS_MASK                                            0x00000001L
+//DCCG_PERFMON_CNTL
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DISPCLK_ENABLE__SHIFT                                                    0x0
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DPREFCLK_ENABLE__SHIFT                                                   0x1
+#define DCCG_PERFMON_CNTL__DCCG_PERF_UNIPHYA_PIXCLK_ENABLE__SHIFT                                             0x2
+#define DCCG_PERFMON_CNTL__DCCG_PERF_UNIPHYB_PIXCLK_ENABLE__SHIFT                                             0x3
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK0_ENABLE__SHIFT                                                    0x4
+#define DCCG_PERFMON_CNTL__DCCG_PERF_RUN__SHIFT                                                               0x5
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_VSYNC__SHIFT                                                        0x6
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_HSYNC__SHIFT                                                        0x7
+#define DCCG_PERFMON_CNTL__DCCG_PERF_OTG_SEL__SHIFT                                                           0x8
+#define DCCG_PERFMON_CNTL__DCCG_PERF_TEST_CLK_ENABLE__SHIFT                                                   0xb
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DISPCLK_ENABLE_MASK                                                      0x00000001L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DPREFCLK_ENABLE_MASK                                                     0x00000002L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_UNIPHYA_PIXCLK_ENABLE_MASK                                               0x00000004L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_UNIPHYB_PIXCLK_ENABLE_MASK                                               0x00000008L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK0_ENABLE_MASK                                                      0x00000010L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_RUN_MASK                                                                 0x00000020L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_VSYNC_MASK                                                          0x00000040L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_HSYNC_MASK                                                          0x00000080L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_OTG_SEL_MASK                                                             0x00000700L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_TEST_CLK_ENABLE_MASK                                                     0x00000800L
+//DCCG_GATE_DISABLE_CNTL
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_DCCG_GATE_DISABLE__SHIFT                                              0x0
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_GATE_DISABLE__SHIFT                                            0x1
+#define DCCG_GATE_DISABLE_CNTL__SOCCLK_GATE_DISABLE__SHIFT                                                    0x2
+#define DCCG_GATE_DISABLE_CNTL__DPREFCLK_GATE_DISABLE__SHIFT                                                  0x3
+#define DCCG_GATE_DISABLE_CNTL__DACACLK_GATE_DISABLE__SHIFT                                                   0x4
+#define DCCG_GATE_DISABLE_CNTL__DVOACLK_GATE_DISABLE__SHIFT                                                   0x6
+#define DCCG_GATE_DISABLE_CNTL__DPREFCLK_R_DCCG_GATE_DISABLE__SHIFT                                           0x8
+#define DCCG_GATE_DISABLE_CNTL__DPPCLK_GATE_DISABLE__SHIFT                                                    0x9
+#define DCCG_GATE_DISABLE_CNTL__DPPCLK_R_DCCG_GATE_DISABLE__SHIFT                                             0xa
+#define DCCG_GATE_DISABLE_CNTL__DSCCLK_GATE_DISABLE__SHIFT                                                    0xb
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK0_GATE_DISABLE__SHIFT                                                   0x11
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK1_GATE_DISABLE__SHIFT                                                   0x12
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK2_GATE_DISABLE__SHIFT                                                   0x13
+#define DCCG_GATE_DISABLE_CNTL__REFCLK_GATE_DISABLE__SHIFT                                                    0x1a
+#define DCCG_GATE_DISABLE_CNTL__REFCLK_R_DIG_GATE_DISABLE__SHIFT                                              0x1b
+#define DCCG_GATE_DISABLE_CNTL__DSICLK_GATE_DISABLE__SHIFT                                                    0x1c
+#define DCCG_GATE_DISABLE_CNTL__BYTECLK_GATE_DISABLE__SHIFT                                                   0x1d
+#define DCCG_GATE_DISABLE_CNTL__ESCCLK_GATE_DISABLE__SHIFT                                                    0x1e
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_DCCG_GATE_DISABLE_MASK                                                0x00000001L
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_GATE_DISABLE_MASK                                              0x00000002L
+#define DCCG_GATE_DISABLE_CNTL__SOCCLK_GATE_DISABLE_MASK                                                      0x00000004L
+#define DCCG_GATE_DISABLE_CNTL__DPREFCLK_GATE_DISABLE_MASK                                                    0x00000008L
+#define DCCG_GATE_DISABLE_CNTL__DACACLK_GATE_DISABLE_MASK                                                     0x00000010L
+#define DCCG_GATE_DISABLE_CNTL__DVOACLK_GATE_DISABLE_MASK                                                     0x00000040L
+#define DCCG_GATE_DISABLE_CNTL__DPREFCLK_R_DCCG_GATE_DISABLE_MASK                                             0x00000100L
+#define DCCG_GATE_DISABLE_CNTL__DPPCLK_GATE_DISABLE_MASK                                                      0x00000200L
+#define DCCG_GATE_DISABLE_CNTL__DPPCLK_R_DCCG_GATE_DISABLE_MASK                                               0x00000400L
+#define DCCG_GATE_DISABLE_CNTL__DSCCLK_GATE_DISABLE_MASK                                                      0x00000800L
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK0_GATE_DISABLE_MASK                                                     0x00020000L
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK1_GATE_DISABLE_MASK                                                     0x00040000L
+#define DCCG_GATE_DISABLE_CNTL__AOMCLK2_GATE_DISABLE_MASK                                                     0x00080000L
+#define DCCG_GATE_DISABLE_CNTL__REFCLK_GATE_DISABLE_MASK                                                      0x04000000L
+#define DCCG_GATE_DISABLE_CNTL__REFCLK_R_DIG_GATE_DISABLE_MASK                                                0x08000000L
+#define DCCG_GATE_DISABLE_CNTL__DSICLK_GATE_DISABLE_MASK                                                      0x10000000L
+#define DCCG_GATE_DISABLE_CNTL__BYTECLK_GATE_DISABLE_MASK                                                     0x20000000L
+#define DCCG_GATE_DISABLE_CNTL__ESCCLK_GATE_DISABLE_MASK                                                      0x40000000L
+//DISPCLK_CGTT_BLK_CTRL_REG
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_ON_DELAY__SHIFT                                               0x0
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_OFF_DELAY__SHIFT                                              0x4
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_ON_DELAY_MASK                                                 0x0000000FL
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_OFF_DELAY_MASK                                                0x00000FF0L
+//SOCCLK_CGTT_BLK_CTRL_REG
+#define SOCCLK_CGTT_BLK_CTRL_REG__SOCCLK_TURN_ON_DELAY__SHIFT                                                 0x0
+#define SOCCLK_CGTT_BLK_CTRL_REG__SOCCLK_TURN_OFF_DELAY__SHIFT                                                0x4
+#define SOCCLK_CGTT_BLK_CTRL_REG__SOCCLK_TURN_ON_DELAY_MASK                                                   0x0000000FL
+#define SOCCLK_CGTT_BLK_CTRL_REG__SOCCLK_TURN_OFF_DELAY_MASK                                                  0x00000FF0L
+//DCCG_CAC_STATUS
+#define DCCG_CAC_STATUS__CAC_STATUS_RDDATA__SHIFT                                                             0x0
+#define DCCG_CAC_STATUS__CAC_STATUS_RDDATA_MASK                                                               0xFFFFFFFFL
+//MICROSECOND_TIME_BASE_DIV
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_DIV__SHIFT                                           0x0
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_DIV__SHIFT                                                        0x8
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_SEL__SHIFT                                                        0x10
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_CLOCK_SOURCE_SEL__SHIFT                                           0x11
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_CLOCK_SOURCE_SEL__SHIFT                              0x14
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_DIV_MASK                                             0x0000007FL
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_DIV_MASK                                                          0x00007F00L
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_SEL_MASK                                                          0x00010000L
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_CLOCK_SOURCE_SEL_MASK                                             0x00020000L
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_CLOCK_SOURCE_SEL_MASK                                0x00100000L
+//DCCG_GATE_DISABLE_CNTL2
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKA_FE_GATE_DISABLE__SHIFT                                               0x0
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKB_FE_GATE_DISABLE__SHIFT                                               0x1
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKC_FE_GATE_DISABLE__SHIFT                                               0x2
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKD_FE_GATE_DISABLE__SHIFT                                               0x3
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKE_FE_GATE_DISABLE__SHIFT                                               0x4
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKF_FE_GATE_DISABLE__SHIFT                                               0x5
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKG_FE_GATE_DISABLE__SHIFT                                               0x6
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK0_GATE_DISABLE__SHIFT                                             0x8
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK1_GATE_DISABLE__SHIFT                                             0x9
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK2_GATE_DISABLE__SHIFT                                             0xa
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK3_GATE_DISABLE__SHIFT                                             0xb
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK4_GATE_DISABLE__SHIFT                                             0xc
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK5_GATE_DISABLE__SHIFT                                             0xd
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKA_GATE_DISABLE__SHIFT                                                  0x10
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKB_GATE_DISABLE__SHIFT                                                  0x11
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKC_GATE_DISABLE__SHIFT                                                  0x12
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKD_GATE_DISABLE__SHIFT                                                  0x13
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKE_GATE_DISABLE__SHIFT                                                  0x14
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKF_GATE_DISABLE__SHIFT                                                  0x15
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKG_GATE_DISABLE__SHIFT                                                  0x16
+#define DCCG_GATE_DISABLE_CNTL2__PHYASYMCLK_ROOT_GATE_DISABLE__SHIFT                                          0x18
+#define DCCG_GATE_DISABLE_CNTL2__PHYBSYMCLK_ROOT_GATE_DISABLE__SHIFT                                          0x19
+#define DCCG_GATE_DISABLE_CNTL2__PHYCSYMCLK_ROOT_GATE_DISABLE__SHIFT                                          0x1a
+#define DCCG_GATE_DISABLE_CNTL2__PHYDSYMCLK_ROOT_GATE_DISABLE__SHIFT                                          0x1b
+#define DCCG_GATE_DISABLE_CNTL2__PHYESYMCLK_ROOT_GATE_DISABLE__SHIFT                                          0x1c
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKA_FE_GATE_DISABLE_MASK                                                 0x00000001L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKB_FE_GATE_DISABLE_MASK                                                 0x00000002L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKC_FE_GATE_DISABLE_MASK                                                 0x00000004L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKD_FE_GATE_DISABLE_MASK                                                 0x00000008L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKE_FE_GATE_DISABLE_MASK                                                 0x00000010L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKF_FE_GATE_DISABLE_MASK                                                 0x00000020L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKG_FE_GATE_DISABLE_MASK                                                 0x00000040L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK0_GATE_DISABLE_MASK                                               0x00000100L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK1_GATE_DISABLE_MASK                                               0x00000200L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK2_GATE_DISABLE_MASK                                               0x00000400L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK3_GATE_DISABLE_MASK                                               0x00000800L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK4_GATE_DISABLE_MASK                                               0x00001000L
+#define DCCG_GATE_DISABLE_CNTL2__HDMICHARCLK5_GATE_DISABLE_MASK                                               0x00002000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKA_GATE_DISABLE_MASK                                                    0x00010000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKB_GATE_DISABLE_MASK                                                    0x00020000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKC_GATE_DISABLE_MASK                                                    0x00040000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKD_GATE_DISABLE_MASK                                                    0x00080000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKE_GATE_DISABLE_MASK                                                    0x00100000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKF_GATE_DISABLE_MASK                                                    0x00200000L
+#define DCCG_GATE_DISABLE_CNTL2__SYMCLKG_GATE_DISABLE_MASK                                                    0x00400000L
+#define DCCG_GATE_DISABLE_CNTL2__PHYASYMCLK_ROOT_GATE_DISABLE_MASK                                            0x01000000L
+#define DCCG_GATE_DISABLE_CNTL2__PHYBSYMCLK_ROOT_GATE_DISABLE_MASK                                            0x02000000L
+#define DCCG_GATE_DISABLE_CNTL2__PHYCSYMCLK_ROOT_GATE_DISABLE_MASK                                            0x04000000L
+#define DCCG_GATE_DISABLE_CNTL2__PHYDSYMCLK_ROOT_GATE_DISABLE_MASK                                            0x08000000L
+#define DCCG_GATE_DISABLE_CNTL2__PHYESYMCLK_ROOT_GATE_DISABLE_MASK                                            0x10000000L
+//SYMCLK_CGTT_BLK_CTRL_REG
+#define SYMCLK_CGTT_BLK_CTRL_REG__SYMCLK_TURN_ON_DELAY__SHIFT                                                 0x0
+#define SYMCLK_CGTT_BLK_CTRL_REG__SYMCLK_TURN_OFF_DELAY__SHIFT                                                0x4
+#define SYMCLK_CGTT_BLK_CTRL_REG__SYMCLK_TURN_ON_DELAY_MASK                                                   0x0000000FL
+#define SYMCLK_CGTT_BLK_CTRL_REG__SYMCLK_TURN_OFF_DELAY_MASK                                                  0x00000FF0L
+//DCCG_DISP_CNTL_REG
+#define DCCG_DISP_CNTL_REG__ALLOW_SR_ON_TRANS_REQ__SHIFT                                                      0x8
+#define DCCG_DISP_CNTL_REG__ALLOW_SR_ON_TRANS_REQ_MASK                                                        0x00000100L
+//OTG0_PIXEL_RATE_CNTL
+#define OTG0_PIXEL_RATE_CNTL__OTG0_PIXEL_RATE_SOURCE__SHIFT                                                   0x0
+#define OTG0_PIXEL_RATE_CNTL__DP_DTO0_ENABLE__SHIFT                                                           0x4
+#define OTG0_PIXEL_RATE_CNTL__DP_DTO0_DS_DISABLE__SHIFT                                                       0x5
+#define OTG0_PIXEL_RATE_CNTL__DPDTO0_ENABLE_STATUS__SHIFT                                                     0x7
+#define OTG0_PIXEL_RATE_CNTL__PIPE0_DTO_SRC_SEL__SHIFT                                                        0xc
+#define OTG0_PIXEL_RATE_CNTL__OTG0_DIO_FIFO_ERROR__SHIFT                                                      0xe
+#define OTG0_PIXEL_RATE_CNTL__OTG0_DIO_ERROR_COUNT__SHIFT                                                     0x10
+#define OTG0_PIXEL_RATE_CNTL__OTG0_PIXEL_RATE_SOURCE_MASK                                                     0x00000003L
+#define OTG0_PIXEL_RATE_CNTL__DP_DTO0_ENABLE_MASK                                                             0x00000010L
+#define OTG0_PIXEL_RATE_CNTL__DP_DTO0_DS_DISABLE_MASK                                                         0x00000020L
+#define OTG0_PIXEL_RATE_CNTL__DPDTO0_ENABLE_STATUS_MASK                                                       0x00000080L
+#define OTG0_PIXEL_RATE_CNTL__PIPE0_DTO_SRC_SEL_MASK                                                          0x00001000L
+#define OTG0_PIXEL_RATE_CNTL__OTG0_DIO_FIFO_ERROR_MASK                                                        0x0000C000L
+#define OTG0_PIXEL_RATE_CNTL__OTG0_DIO_ERROR_COUNT_MASK                                                       0x0FFF0000L
+//DP_DTO0_PHASE
+#define DP_DTO0_PHASE__DP_DTO0_PHASE__SHIFT                                                                   0x0
+#define DP_DTO0_PHASE__DP_DTO0_PHASE_MASK                                                                     0xFFFFFFFFL
+//DP_DTO0_MODULO
+#define DP_DTO0_MODULO__DP_DTO0_MODULO__SHIFT                                                                 0x0
+#define DP_DTO0_MODULO__DP_DTO0_MODULO_MASK                                                                   0xFFFFFFFFL
+//OTG0_PHYPLL_PIXEL_RATE_CNTL
+#define OTG0_PHYPLL_PIXEL_RATE_CNTL__OTG0_PHYPLL_PIXEL_RATE_SOURCE__SHIFT                                     0x0
+#define OTG0_PHYPLL_PIXEL_RATE_CNTL__OTG0_PIXEL_RATE_PLL_SOURCE__SHIFT                                        0x4
+#define OTG0_PHYPLL_PIXEL_RATE_CNTL__OTG0_PHYPLL_PIXEL_RATE_SOURCE_MASK                                       0x00000007L
+#define OTG0_PHYPLL_PIXEL_RATE_CNTL__OTG0_PIXEL_RATE_PLL_SOURCE_MASK                                          0x00000010L
+//OTG1_PIXEL_RATE_CNTL
+#define OTG1_PIXEL_RATE_CNTL__OTG1_PIXEL_RATE_SOURCE__SHIFT                                                   0x0
+#define OTG1_PIXEL_RATE_CNTL__DP_DTO1_ENABLE__SHIFT                                                           0x4
+#define OTG1_PIXEL_RATE_CNTL__DP_DTO1_DS_DISABLE__SHIFT                                                       0x5
+#define OTG1_PIXEL_RATE_CNTL__DPDTO1_ENABLE_STATUS__SHIFT                                                     0x7
+#define OTG1_PIXEL_RATE_CNTL__PIPE1_DTO_SRC_SEL__SHIFT                                                        0xc
+#define OTG1_PIXEL_RATE_CNTL__OTG1_DIO_FIFO_ERROR__SHIFT                                                      0xe
+#define OTG1_PIXEL_RATE_CNTL__OTG1_DIO_ERROR_COUNT__SHIFT                                                     0x10
+#define OTG1_PIXEL_RATE_CNTL__OTG1_PIXEL_RATE_SOURCE_MASK                                                     0x00000003L
+#define OTG1_PIXEL_RATE_CNTL__DP_DTO1_ENABLE_MASK                                                             0x00000010L
+#define OTG1_PIXEL_RATE_CNTL__DP_DTO1_DS_DISABLE_MASK                                                         0x00000020L
+#define OTG1_PIXEL_RATE_CNTL__DPDTO1_ENABLE_STATUS_MASK                                                       0x00000080L
+#define OTG1_PIXEL_RATE_CNTL__PIPE1_DTO_SRC_SEL_MASK                                                          0x00001000L
+#define OTG1_PIXEL_RATE_CNTL__OTG1_DIO_FIFO_ERROR_MASK                                                        0x0000C000L
+#define OTG1_PIXEL_RATE_CNTL__OTG1_DIO_ERROR_COUNT_MASK                                                       0x0FFF0000L
+//DP_DTO1_PHASE
+#define DP_DTO1_PHASE__DP_DTO1_PHASE__SHIFT                                                                   0x0
+#define DP_DTO1_PHASE__DP_DTO1_PHASE_MASK                                                                     0xFFFFFFFFL
+//DP_DTO1_MODULO
+#define DP_DTO1_MODULO__DP_DTO1_MODULO__SHIFT                                                                 0x0
+#define DP_DTO1_MODULO__DP_DTO1_MODULO_MASK                                                                   0xFFFFFFFFL
+//OTG1_PHYPLL_PIXEL_RATE_CNTL
+#define OTG1_PHYPLL_PIXEL_RATE_CNTL__OTG1_PHYPLL_PIXEL_RATE_SOURCE__SHIFT                                     0x0
+#define OTG1_PHYPLL_PIXEL_RATE_CNTL__OTG1_PIXEL_RATE_PLL_SOURCE__SHIFT                                        0x4
+#define OTG1_PHYPLL_PIXEL_RATE_CNTL__OTG1_PHYPLL_PIXEL_RATE_SOURCE_MASK                                       0x00000007L
+#define OTG1_PHYPLL_PIXEL_RATE_CNTL__OTG1_PIXEL_RATE_PLL_SOURCE_MASK                                          0x00000010L
+//OTG2_PIXEL_RATE_CNTL
+#define OTG2_PIXEL_RATE_CNTL__OTG2_PIXEL_RATE_SOURCE__SHIFT                                                   0x0
+#define OTG2_PIXEL_RATE_CNTL__DP_DTO2_ENABLE__SHIFT                                                           0x4
+#define OTG2_PIXEL_RATE_CNTL__DP_DTO2_DS_DISABLE__SHIFT                                                       0x5
+#define OTG2_PIXEL_RATE_CNTL__DPDTO2_ENABLE_STATUS__SHIFT                                                     0x7
+#define OTG2_PIXEL_RATE_CNTL__PIPE2_DTO_SRC_SEL__SHIFT                                                        0xc
+#define OTG2_PIXEL_RATE_CNTL__OTG2_DIO_FIFO_ERROR__SHIFT                                                      0xe
+#define OTG2_PIXEL_RATE_CNTL__OTG2_DIO_ERROR_COUNT__SHIFT                                                     0x10
+#define OTG2_PIXEL_RATE_CNTL__OTG2_PIXEL_RATE_SOURCE_MASK                                                     0x00000003L
+#define OTG2_PIXEL_RATE_CNTL__DP_DTO2_ENABLE_MASK                                                             0x00000010L
+#define OTG2_PIXEL_RATE_CNTL__DP_DTO2_DS_DISABLE_MASK                                                         0x00000020L
+#define OTG2_PIXEL_RATE_CNTL__DPDTO2_ENABLE_STATUS_MASK                                                       0x00000080L
+#define OTG2_PIXEL_RATE_CNTL__PIPE2_DTO_SRC_SEL_MASK                                                          0x00001000L
+#define OTG2_PIXEL_RATE_CNTL__OTG2_DIO_FIFO_ERROR_MASK                                                        0x0000C000L
+#define OTG2_PIXEL_RATE_CNTL__OTG2_DIO_ERROR_COUNT_MASK                                                       0x0FFF0000L
+//DP_DTO2_PHASE
+#define DP_DTO2_PHASE__DP_DTO2_PHASE__SHIFT                                                                   0x0
+#define DP_DTO2_PHASE__DP_DTO2_PHASE_MASK                                                                     0xFFFFFFFFL
+//DP_DTO2_MODULO
+#define DP_DTO2_MODULO__DP_DTO2_MODULO__SHIFT                                                                 0x0
+#define DP_DTO2_MODULO__DP_DTO2_MODULO_MASK                                                                   0xFFFFFFFFL
+//OTG2_PHYPLL_PIXEL_RATE_CNTL
+#define OTG2_PHYPLL_PIXEL_RATE_CNTL__OTG2_PHYPLL_PIXEL_RATE_SOURCE__SHIFT                                     0x0
+#define OTG2_PHYPLL_PIXEL_RATE_CNTL__OTG2_PIXEL_RATE_PLL_SOURCE__SHIFT                                        0x4
+#define OTG2_PHYPLL_PIXEL_RATE_CNTL__OTG2_PHYPLL_PIXEL_RATE_SOURCE_MASK                                       0x00000007L
+#define OTG2_PHYPLL_PIXEL_RATE_CNTL__OTG2_PIXEL_RATE_PLL_SOURCE_MASK                                          0x00000010L
+//OTG3_PIXEL_RATE_CNTL
+#define OTG3_PIXEL_RATE_CNTL__OTG3_PIXEL_RATE_SOURCE__SHIFT                                                   0x0
+#define OTG3_PIXEL_RATE_CNTL__DP_DTO3_ENABLE__SHIFT                                                           0x4
+#define OTG3_PIXEL_RATE_CNTL__DP_DTO3_DS_DISABLE__SHIFT                                                       0x5
+#define OTG3_PIXEL_RATE_CNTL__DPDTO3_ENABLE_STATUS__SHIFT                                                     0x7
+#define OTG3_PIXEL_RATE_CNTL__PIPE3_DTO_SRC_SEL__SHIFT                                                        0xc
+#define OTG3_PIXEL_RATE_CNTL__OTG3_DIO_FIFO_ERROR__SHIFT                                                      0xe
+#define OTG3_PIXEL_RATE_CNTL__OTG3_DIO_ERROR_COUNT__SHIFT                                                     0x10
+#define OTG3_PIXEL_RATE_CNTL__OTG3_PIXEL_RATE_SOURCE_MASK                                                     0x00000003L
+#define OTG3_PIXEL_RATE_CNTL__DP_DTO3_ENABLE_MASK                                                             0x00000010L
+#define OTG3_PIXEL_RATE_CNTL__DP_DTO3_DS_DISABLE_MASK                                                         0x00000020L
+#define OTG3_PIXEL_RATE_CNTL__DPDTO3_ENABLE_STATUS_MASK                                                       0x00000080L
+#define OTG3_PIXEL_RATE_CNTL__PIPE3_DTO_SRC_SEL_MASK                                                          0x00001000L
+#define OTG3_PIXEL_RATE_CNTL__OTG3_DIO_FIFO_ERROR_MASK                                                        0x0000C000L
+#define OTG3_PIXEL_RATE_CNTL__OTG3_DIO_ERROR_COUNT_MASK                                                       0x0FFF0000L
+//DP_DTO3_PHASE
+#define DP_DTO3_PHASE__DP_DTO3_PHASE__SHIFT                                                                   0x0
+#define DP_DTO3_PHASE__DP_DTO3_PHASE_MASK                                                                     0xFFFFFFFFL
+//DP_DTO3_MODULO
+#define DP_DTO3_MODULO__DP_DTO3_MODULO__SHIFT                                                                 0x0
+#define DP_DTO3_MODULO__DP_DTO3_MODULO_MASK                                                                   0xFFFFFFFFL
+//OTG3_PHYPLL_PIXEL_RATE_CNTL
+#define OTG3_PHYPLL_PIXEL_RATE_CNTL__OTG3_PHYPLL_PIXEL_RATE_SOURCE__SHIFT                                     0x0
+#define OTG3_PHYPLL_PIXEL_RATE_CNTL__OTG3_PIXEL_RATE_PLL_SOURCE__SHIFT                                        0x4
+#define OTG3_PHYPLL_PIXEL_RATE_CNTL__OTG3_PHYPLL_PIXEL_RATE_SOURCE_MASK                                       0x00000007L
+#define OTG3_PHYPLL_PIXEL_RATE_CNTL__OTG3_PIXEL_RATE_PLL_SOURCE_MASK                                          0x00000010L
+//DPPCLK_CGTT_BLK_CTRL_REG
+#define DPPCLK_CGTT_BLK_CTRL_REG__DPPCLK_TURN_ON_DELAY__SHIFT                                                 0x0
+#define DPPCLK_CGTT_BLK_CTRL_REG__DPPCLK_TURN_OFF_DELAY__SHIFT                                                0x4
+#define DPPCLK_CGTT_BLK_CTRL_REG__DPPCLK_TURN_ON_DELAY_MASK                                                   0x0000000FL
+#define DPPCLK_CGTT_BLK_CTRL_REG__DPPCLK_TURN_OFF_DELAY_MASK                                                  0x00000FF0L
+//DPPCLK0_DTO_PARAM
+#define DPPCLK0_DTO_PARAM__DPPCLK0_DTO_PHASE__SHIFT                                                           0x0
+#define DPPCLK0_DTO_PARAM__DPPCLK0_DTO_MODULO__SHIFT                                                          0x10
+#define DPPCLK0_DTO_PARAM__DPPCLK0_DTO_PHASE_MASK                                                             0x000000FFL
+#define DPPCLK0_DTO_PARAM__DPPCLK0_DTO_MODULO_MASK                                                            0x00FF0000L
+//DPPCLK1_DTO_PARAM
+#define DPPCLK1_DTO_PARAM__DPPCLK1_DTO_PHASE__SHIFT                                                           0x0
+#define DPPCLK1_DTO_PARAM__DPPCLK1_DTO_MODULO__SHIFT                                                          0x10
+#define DPPCLK1_DTO_PARAM__DPPCLK1_DTO_PHASE_MASK                                                             0x000000FFL
+#define DPPCLK1_DTO_PARAM__DPPCLK1_DTO_MODULO_MASK                                                            0x00FF0000L
+//DPPCLK2_DTO_PARAM
+#define DPPCLK2_DTO_PARAM__DPPCLK2_DTO_PHASE__SHIFT                                                           0x0
+#define DPPCLK2_DTO_PARAM__DPPCLK2_DTO_MODULO__SHIFT                                                          0x10
+#define DPPCLK2_DTO_PARAM__DPPCLK2_DTO_PHASE_MASK                                                             0x000000FFL
+#define DPPCLK2_DTO_PARAM__DPPCLK2_DTO_MODULO_MASK                                                            0x00FF0000L
+//DPPCLK3_DTO_PARAM
+#define DPPCLK3_DTO_PARAM__DPPCLK3_DTO_PHASE__SHIFT                                                           0x0
+#define DPPCLK3_DTO_PARAM__DPPCLK3_DTO_MODULO__SHIFT                                                          0x10
+#define DPPCLK3_DTO_PARAM__DPPCLK3_DTO_PHASE_MASK                                                             0x000000FFL
+#define DPPCLK3_DTO_PARAM__DPPCLK3_DTO_MODULO_MASK                                                            0x00FF0000L
+//DCCG_CAC_STATUS2
+#define DCCG_CAC_STATUS2__CAC_STATUS_RDDATA2__SHIFT                                                           0x0
+#define DCCG_CAC_STATUS2__CAC_STATUS_RDDATA2_MASK                                                             0x0007FFFFL
+//SYMCLKA_CLOCK_ENABLE
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_CLOCK_ENABLE__SHIFT                                                     0x0
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_EN__SHIFT                                                            0x4
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_SRC_SEL__SHIFT                                                          0x5
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_SRC_SEL__SHIFT                                                       0x8
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_CLOCK_ENABLE_MASK                                                       0x00000001L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_EN_MASK                                                              0x00000010L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_SRC_SEL_MASK                                                            0x000000E0L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_SRC_SEL_MASK                                                         0x00000700L
+//SYMCLKB_CLOCK_ENABLE
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_CLOCK_ENABLE__SHIFT                                                     0x0
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_EN__SHIFT                                                            0x4
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_SRC_SEL__SHIFT                                                          0x5
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_SRC_SEL__SHIFT                                                       0x8
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_CLOCK_ENABLE_MASK                                                       0x00000001L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_EN_MASK                                                              0x00000010L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_SRC_SEL_MASK                                                            0x000000E0L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_SRC_SEL_MASK                                                         0x00000700L
+//SYMCLKC_CLOCK_ENABLE
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_CLOCK_ENABLE__SHIFT                                                     0x0
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_EN__SHIFT                                                            0x4
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_SRC_SEL__SHIFT                                                          0x5
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_SRC_SEL__SHIFT                                                       0x8
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_CLOCK_ENABLE_MASK                                                       0x00000001L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_EN_MASK                                                              0x00000010L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_SRC_SEL_MASK                                                            0x000000E0L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_SRC_SEL_MASK                                                         0x00000700L
+//SYMCLKD_CLOCK_ENABLE
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_CLOCK_ENABLE__SHIFT                                                     0x0
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_EN__SHIFT                                                            0x4
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_SRC_SEL__SHIFT                                                          0x5
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_SRC_SEL__SHIFT                                                       0x8
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_CLOCK_ENABLE_MASK                                                       0x00000001L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_EN_MASK                                                              0x00000010L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_SRC_SEL_MASK                                                            0x000000E0L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_SRC_SEL_MASK                                                         0x00000700L
+//SYMCLKE_CLOCK_ENABLE
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_CLOCK_ENABLE__SHIFT                                                     0x0
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_EN__SHIFT                                                            0x4
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_SRC_SEL__SHIFT                                                          0x5
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_SRC_SEL__SHIFT                                                       0x8
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_CLOCK_ENABLE_MASK                                                       0x00000001L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_EN_MASK                                                              0x00000010L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_SRC_SEL_MASK                                                            0x000000E0L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_SRC_SEL_MASK                                                         0x00000700L
+//DCCG_SOFT_RESET
+#define DCCG_SOFT_RESET__REFCLK_SOFT_RESET__SHIFT                                                             0x0
+#define DCCG_SOFT_RESET__SOFT_RESET_DVO__SHIFT                                                                0x2
+#define DCCG_SOFT_RESET__DVO_ENABLE_RST__SHIFT                                                                0x3
+#define DCCG_SOFT_RESET__AUDIO_DTO2_CLK_SOFT_RESET__SHIFT                                                     0x4
+#define DCCG_SOFT_RESET__DPREFCLK_SOFT_RESET__SHIFT                                                           0x8
+#define DCCG_SOFT_RESET__AMCLK0_SOFT_RESET__SHIFT                                                             0xc
+#define DCCG_SOFT_RESET__AMCLK1_SOFT_RESET__SHIFT                                                             0xd
+#define DCCG_SOFT_RESET__P0PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0xe
+#define DCCG_SOFT_RESET__P1PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0xf
+#define DCCG_SOFT_RESET__P2PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x10
+#define DCCG_SOFT_RESET__A0PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x11
+#define DCCG_SOFT_RESET__A1PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x12
+#define DCCG_SOFT_RESET__C0PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x13
+#define DCCG_SOFT_RESET__C1PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x14
+#define DCCG_SOFT_RESET__C2PLL_CFG_IF_SOFT_RESET__SHIFT                                                       0x15
+#define DCCG_SOFT_RESET__REFCLK_SOFT_RESET_MASK                                                               0x00000001L
+#define DCCG_SOFT_RESET__SOFT_RESET_DVO_MASK                                                                  0x00000004L
+#define DCCG_SOFT_RESET__DVO_ENABLE_RST_MASK                                                                  0x00000008L
+#define DCCG_SOFT_RESET__AUDIO_DTO2_CLK_SOFT_RESET_MASK                                                       0x00000010L
+#define DCCG_SOFT_RESET__DPREFCLK_SOFT_RESET_MASK                                                             0x00000100L
+#define DCCG_SOFT_RESET__AMCLK0_SOFT_RESET_MASK                                                               0x00001000L
+#define DCCG_SOFT_RESET__AMCLK1_SOFT_RESET_MASK                                                               0x00002000L
+#define DCCG_SOFT_RESET__P0PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00004000L
+#define DCCG_SOFT_RESET__P1PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00008000L
+#define DCCG_SOFT_RESET__P2PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00010000L
+#define DCCG_SOFT_RESET__A0PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00020000L
+#define DCCG_SOFT_RESET__A1PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00040000L
+#define DCCG_SOFT_RESET__C0PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00080000L
+#define DCCG_SOFT_RESET__C1PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00100000L
+#define DCCG_SOFT_RESET__C2PLL_CFG_IF_SOFT_RESET_MASK                                                         0x00200000L
+//DSCCLK_DTO_CTRL
+#define DSCCLK_DTO_CTRL__DSCCLK0_EN__SHIFT                                                                    0x0
+#define DSCCLK_DTO_CTRL__DSCCLK1_EN__SHIFT                                                                    0x1
+#define DSCCLK_DTO_CTRL__DSCCLK2_EN__SHIFT                                                                    0x2
+#define DSCCLK_DTO_CTRL__DSCCLK3_EN__SHIFT                                                                    0x3
+#define DSCCLK_DTO_CTRL__DSCCLK4_EN__SHIFT                                                                    0x4
+#define DSCCLK_DTO_CTRL__DSCCLK5_EN__SHIFT                                                                    0x5
+#define DSCCLK_DTO_CTRL__DSCCLK0_DTO_DB_EN__SHIFT                                                             0x8
+#define DSCCLK_DTO_CTRL__DSCCLK1_DTO_DB_EN__SHIFT                                                             0x9
+#define DSCCLK_DTO_CTRL__DSCCLK2_DTO_DB_EN__SHIFT                                                             0xa
+#define DSCCLK_DTO_CTRL__DSCCLK3_DTO_DB_EN__SHIFT                                                             0xb
+#define DSCCLK_DTO_CTRL__DSCCLK4_DTO_DB_EN__SHIFT                                                             0xc
+#define DSCCLK_DTO_CTRL__DSCCLK5_DTO_DB_EN__SHIFT                                                             0xd
+#define DSCCLK_DTO_CTRL__DSCCLK0_EN_MASK                                                                      0x00000001L
+#define DSCCLK_DTO_CTRL__DSCCLK1_EN_MASK                                                                      0x00000002L
+#define DSCCLK_DTO_CTRL__DSCCLK2_EN_MASK                                                                      0x00000004L
+#define DSCCLK_DTO_CTRL__DSCCLK3_EN_MASK                                                                      0x00000008L
+#define DSCCLK_DTO_CTRL__DSCCLK4_EN_MASK                                                                      0x00000010L
+#define DSCCLK_DTO_CTRL__DSCCLK5_EN_MASK                                                                      0x00000020L
+#define DSCCLK_DTO_CTRL__DSCCLK0_DTO_DB_EN_MASK                                                               0x00000100L
+#define DSCCLK_DTO_CTRL__DSCCLK1_DTO_DB_EN_MASK                                                               0x00000200L
+#define DSCCLK_DTO_CTRL__DSCCLK2_DTO_DB_EN_MASK                                                               0x00000400L
+#define DSCCLK_DTO_CTRL__DSCCLK3_DTO_DB_EN_MASK                                                               0x00000800L
+#define DSCCLK_DTO_CTRL__DSCCLK4_DTO_DB_EN_MASK                                                               0x00001000L
+#define DSCCLK_DTO_CTRL__DSCCLK5_DTO_DB_EN_MASK                                                               0x00002000L
+//DPPCLK_CTRL
+#define DPPCLK_CTRL__DPPCLK0_EN__SHIFT                                                                        0x0
+#define DPPCLK_CTRL__DPPCLK1_EN__SHIFT                                                                        0x3
+#define DPPCLK_CTRL__DPPCLK2_EN__SHIFT                                                                        0x6
+#define DPPCLK_CTRL__DPPCLK3_EN__SHIFT                                                                        0x9
+#define DPPCLK_CTRL__DPPCLK0_EN_MASK                                                                          0x00000001L
+#define DPPCLK_CTRL__DPPCLK1_EN_MASK                                                                          0x00000008L
+#define DPPCLK_CTRL__DPPCLK2_EN_MASK                                                                          0x00000040L
+#define DPPCLK_CTRL__DPPCLK3_EN_MASK                                                                          0x00000200L
+//DCCG_GATE_DISABLE_CNTL6
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK0_ROOT_GATE_DISABLE__SHIFT                                             0x0
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK1_ROOT_GATE_DISABLE__SHIFT                                             0x1
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK2_ROOT_GATE_DISABLE__SHIFT                                             0x2
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK3_ROOT_GATE_DISABLE__SHIFT                                             0x3
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK0_ROOT_GATE_DISABLE__SHIFT                                             0x8
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK1_ROOT_GATE_DISABLE__SHIFT                                             0x9
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK2_ROOT_GATE_DISABLE__SHIFT                                             0xa
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK3_ROOT_GATE_DISABLE__SHIFT                                             0xb
+#define DCCG_GATE_DISABLE_CNTL6__HDMISTREAMCLK0_ROOT_GATE_DISABLE__SHIFT                                      0xf
+#define DCCG_GATE_DISABLE_CNTL6__DPREFCLK_DIO_GATE_DISABLE__SHIFT                                             0x18
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK0_ROOT_GATE_DISABLE_MASK                                               0x00000001L
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK1_ROOT_GATE_DISABLE_MASK                                               0x00000002L
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK2_ROOT_GATE_DISABLE_MASK                                               0x00000004L
+#define DCCG_GATE_DISABLE_CNTL6__DPPCLK3_ROOT_GATE_DISABLE_MASK                                               0x00000008L
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK0_ROOT_GATE_DISABLE_MASK                                               0x00000100L
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK1_ROOT_GATE_DISABLE_MASK                                               0x00000200L
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK2_ROOT_GATE_DISABLE_MASK                                               0x00000400L
+#define DCCG_GATE_DISABLE_CNTL6__DSCCLK3_ROOT_GATE_DISABLE_MASK                                               0x00000800L
+#define DCCG_GATE_DISABLE_CNTL6__HDMISTREAMCLK0_ROOT_GATE_DISABLE_MASK                                        0x00008000L
+#define DCCG_GATE_DISABLE_CNTL6__DPREFCLK_DIO_GATE_DISABLE_MASK                                               0x01000000L
+//SYMCLK_PSP_CNTL
+#define SYMCLK_PSP_CNTL__SYMCLK_PSP_FORCE_ON__SHIFT                                                           0x0
+#define SYMCLK_PSP_CNTL__SYMCLK_PSP_FORCE_ON_MASK                                                             0x00000001L
+//DCCG_AUDIO_DTO_SOURCE
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_SOURCE_SEL__SHIFT                                              0x0
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO_SEL__SHIFT                                                      0x4
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO2_USE_512FBR_DTO__SHIFT                                          0x14
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_USE_512FBR_DTO__SHIFT                                          0x18
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO1_USE_512FBR_DTO__SHIFT                                          0x1c
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_SOURCE_SEL_MASK                                                0x00000007L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO_SEL_MASK                                                        0x00000030L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO2_USE_512FBR_DTO_MASK                                            0x00100000L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_USE_512FBR_DTO_MASK                                            0x01000000L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO1_USE_512FBR_DTO_MASK                                            0x10000000L
+//DCCG_AUDIO_DTO0_PHASE
+#define DCCG_AUDIO_DTO0_PHASE__DCCG_AUDIO_DTO0_PHASE__SHIFT                                                   0x0
+#define DCCG_AUDIO_DTO0_PHASE__DCCG_AUDIO_DTO0_PHASE_MASK                                                     0xFFFFFFFFL
+//DCCG_AUDIO_DTO0_MODULE
+#define DCCG_AUDIO_DTO0_MODULE__DCCG_AUDIO_DTO0_MODULE__SHIFT                                                 0x0
+#define DCCG_AUDIO_DTO0_MODULE__DCCG_AUDIO_DTO0_MODULE_MASK                                                   0xFFFFFFFFL
+//DCCG_AUDIO_DTO1_PHASE
+#define DCCG_AUDIO_DTO1_PHASE__DCCG_AUDIO_DTO1_PHASE__SHIFT                                                   0x0
+#define DCCG_AUDIO_DTO1_PHASE__DCCG_AUDIO_DTO1_PHASE_MASK                                                     0xFFFFFFFFL
+//DCCG_AUDIO_DTO1_MODULE
+#define DCCG_AUDIO_DTO1_MODULE__DCCG_AUDIO_DTO1_MODULE__SHIFT                                                 0x0
+#define DCCG_AUDIO_DTO1_MODULE__DCCG_AUDIO_DTO1_MODULE_MASK                                                   0xFFFFFFFFL
+//DCCG_VSYNC_OTG0_LATCH_VALUE
+#define DCCG_VSYNC_OTG0_LATCH_VALUE__DCCG_VSYNC_CNT_OTG0_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG0_LATCH_VALUE__DCCG_VSYNC_CNT_OTG0_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DCCG_VSYNC_OTG1_LATCH_VALUE
+#define DCCG_VSYNC_OTG1_LATCH_VALUE__DCCG_VSYNC_CNT_OTG1_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG1_LATCH_VALUE__DCCG_VSYNC_CNT_OTG1_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DCCG_VSYNC_OTG2_LATCH_VALUE
+#define DCCG_VSYNC_OTG2_LATCH_VALUE__DCCG_VSYNC_CNT_OTG2_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG2_LATCH_VALUE__DCCG_VSYNC_CNT_OTG2_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DCCG_VSYNC_OTG3_LATCH_VALUE
+#define DCCG_VSYNC_OTG3_LATCH_VALUE__DCCG_VSYNC_CNT_OTG3_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG3_LATCH_VALUE__DCCG_VSYNC_CNT_OTG3_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DCCG_VSYNC_OTG4_LATCH_VALUE
+#define DCCG_VSYNC_OTG4_LATCH_VALUE__DCCG_VSYNC_CNT_OTG4_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG4_LATCH_VALUE__DCCG_VSYNC_CNT_OTG4_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DCCG_VSYNC_OTG5_LATCH_VALUE
+#define DCCG_VSYNC_OTG5_LATCH_VALUE__DCCG_VSYNC_CNT_OTG5_LATCH_VALUE__SHIFT                                   0x0
+#define DCCG_VSYNC_OTG5_LATCH_VALUE__DCCG_VSYNC_CNT_OTG5_LATCH_VALUE_MASK                                     0xFFFFFFFFL
+//DPPCLK_DTO_CTRL
+#define DPPCLK_DTO_CTRL__DPPCLK0_DTO_DB_EN__SHIFT                                                             0x1
+#define DPPCLK_DTO_CTRL__DPPCLK1_DTO_DB_EN__SHIFT                                                             0x5
+#define DPPCLK_DTO_CTRL__DPPCLK2_DTO_DB_EN__SHIFT                                                             0x9
+#define DPPCLK_DTO_CTRL__DPPCLK3_DTO_DB_EN__SHIFT                                                             0xd
+#define DPPCLK_DTO_CTRL__DPPCLK4_DTO_DB_EN__SHIFT                                                             0x11
+#define DPPCLK_DTO_CTRL__DPPCLK5_DTO_DB_EN__SHIFT                                                             0x15
+#define DPPCLK_DTO_CTRL__DPPCLK0_DTO_DB_EN_MASK                                                               0x00000002L
+#define DPPCLK_DTO_CTRL__DPPCLK1_DTO_DB_EN_MASK                                                               0x00000020L
+#define DPPCLK_DTO_CTRL__DPPCLK2_DTO_DB_EN_MASK                                                               0x00000200L
+#define DPPCLK_DTO_CTRL__DPPCLK3_DTO_DB_EN_MASK                                                               0x00002000L
+#define DPPCLK_DTO_CTRL__DPPCLK4_DTO_DB_EN_MASK                                                               0x00020000L
+#define DPPCLK_DTO_CTRL__DPPCLK5_DTO_DB_EN_MASK                                                               0x00200000L
+//DCCG_VSYNC_CNT_CTRL
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_ENABLE__SHIFT                                                     0x0
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_SW_RESET__SHIFT                                                   0x2
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_RESET_SEL__SHIFT                                                  0x3
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_EXT_TRIG_SEL__SHIFT                                               0x4
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_FRAME_CNT__SHIFT                                                  0x8
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG0_LATCH_EN__SHIFT                                                  0x10
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG1_LATCH_EN__SHIFT                                                  0x11
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG2_LATCH_EN__SHIFT                                                  0x12
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG3_LATCH_EN__SHIFT                                                  0x13
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG4_LATCH_EN__SHIFT                                                  0x14
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG5_LATCH_EN__SHIFT                                                  0x15
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG0_VSYNC_TRIG_SEL__SHIFT                                            0x18
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG1_VSYNC_TRIG_SEL__SHIFT                                            0x19
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG2_VSYNC_TRIG_SEL__SHIFT                                            0x1a
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG3_VSYNC_TRIG_SEL__SHIFT                                            0x1b
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG4_VSYNC_TRIG_SEL__SHIFT                                            0x1c
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG5_VSYNC_TRIG_SEL__SHIFT                                            0x1d
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_ENABLE_MASK                                                       0x00000001L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_SW_RESET_MASK                                                     0x00000004L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_RESET_SEL_MASK                                                    0x00000008L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_EXT_TRIG_SEL_MASK                                                 0x000000F0L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_CNT_FRAME_CNT_MASK                                                    0x00000F00L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG0_LATCH_EN_MASK                                                    0x00010000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG1_LATCH_EN_MASK                                                    0x00020000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG2_LATCH_EN_MASK                                                    0x00040000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG3_LATCH_EN_MASK                                                    0x00080000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG4_LATCH_EN_MASK                                                    0x00100000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG5_LATCH_EN_MASK                                                    0x00200000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG0_VSYNC_TRIG_SEL_MASK                                              0x01000000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG1_VSYNC_TRIG_SEL_MASK                                              0x02000000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG2_VSYNC_TRIG_SEL_MASK                                              0x04000000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG3_VSYNC_TRIG_SEL_MASK                                              0x08000000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG4_VSYNC_TRIG_SEL_MASK                                              0x10000000L
+#define DCCG_VSYNC_CNT_CTRL__DCCG_VSYNC_OTG5_VSYNC_TRIG_SEL_MASK                                              0x20000000L
+//DCCG_VSYNC_CNT_INT_CTRL
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_INTERRUPT__SHIFT                                   0x0
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_INTERRUPT__SHIFT                                   0x1
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_INTERRUPT__SHIFT                                   0x2
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_INTERRUPT__SHIFT                                   0x3
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_INTERRUPT__SHIFT                                   0x4
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_INTERRUPT__SHIFT                                   0x5
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_INTERRUPT_CLEAR__SHIFT                             0x8
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_INTERRUPT_CLEAR__SHIFT                             0x9
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_INTERRUPT_CLEAR__SHIFT                             0xa
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_INTERRUPT_CLEAR__SHIFT                             0xb
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_INTERRUPT_CLEAR__SHIFT                             0xc
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_INTERRUPT_CLEAR__SHIFT                             0xd
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_MASK__SHIFT                                        0x10
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_MASK__SHIFT                                        0x11
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_MASK__SHIFT                                        0x12
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_MASK__SHIFT                                        0x13
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_MASK__SHIFT                                        0x14
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_MASK__SHIFT                                        0x15
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_INTERRUPT_MASK                                     0x00000001L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_INTERRUPT_MASK                                     0x00000002L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_INTERRUPT_MASK                                     0x00000004L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_INTERRUPT_MASK                                     0x00000008L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_INTERRUPT_MASK                                     0x00000010L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_INTERRUPT_MASK                                     0x00000020L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_INTERRUPT_CLEAR_MASK                               0x00000100L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_INTERRUPT_CLEAR_MASK                               0x00000200L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_INTERRUPT_CLEAR_MASK                               0x00000400L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_INTERRUPT_CLEAR_MASK                               0x00000800L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_INTERRUPT_CLEAR_MASK                               0x00001000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_INTERRUPT_CLEAR_MASK                               0x00002000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG0_LATCH_MASK_MASK                                          0x00010000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG1_LATCH_MASK_MASK                                          0x00020000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG2_LATCH_MASK_MASK                                          0x00040000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG3_LATCH_MASK_MASK                                          0x00080000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG4_LATCH_MASK_MASK                                          0x00100000L
+#define DCCG_VSYNC_CNT_INT_CTRL__DCCG_VSYNC_CNT_OTG5_LATCH_MASK_MASK                                          0x00200000L
+//FORCE_SYMCLK_DISABLE
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKA_DISABLE__SHIFT                                                    0x0
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKB_DISABLE__SHIFT                                                    0x1
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKC_DISABLE__SHIFT                                                    0x2
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKD_DISABLE__SHIFT                                                    0x3
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKE_DISABLE__SHIFT                                                    0x4
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKF_DISABLE__SHIFT                                                    0x5
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKG_DISABLE__SHIFT                                                    0x6
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKA_DISABLE_MASK                                                      0x00000001L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKB_DISABLE_MASK                                                      0x00000002L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKC_DISABLE_MASK                                                      0x00000004L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKD_DISABLE_MASK                                                      0x00000008L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKE_DISABLE_MASK                                                      0x00000010L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKF_DISABLE_MASK                                                      0x00000020L
+#define FORCE_SYMCLK_DISABLE__FORCE_SYMCLKG_DISABLE_MASK                                                      0x00000040L
+//DCCG_TEST_CLK_SEL
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_SEL__SHIFT                                                  0x0
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_OR_DATA_GENERICA__SHIFT                                              0x9
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_INV__SHIFT                                                  0xc
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_DIV_SEL__SHIFT                                              0xe
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_SEL_MASK                                                    0x000001FFL
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_OR_DATA_GENERICA_MASK                                                0x00000200L
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_INV_MASK                                                    0x00001000L
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_DIV_SEL_MASK                                                0x0000C000L
+//HDMICHARCLK0_CLOCK_CNTL
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN__SHIFT                                                       0x0
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL__SHIFT                                                  0x4
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN_MASK                                                         0x00000001L
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL_MASK                                                    0x00000070L
+//PHYASYMCLK_CLOCK_CNTL
+#define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_EN__SHIFT                                                           0x0
+#define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_SRC_SEL__SHIFT                                                      0x4
+#define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_EN_MASK                                                             0x00000001L
+#define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_SRC_SEL_MASK                                                        0x00000030L
+//PHYBSYMCLK_CLOCK_CNTL
+#define PHYBSYMCLK_CLOCK_CNTL__PHYBSYMCLK_EN__SHIFT                                                           0x0
+#define PHYBSYMCLK_CLOCK_CNTL__PHYBSYMCLK_SRC_SEL__SHIFT                                                      0x4
+#define PHYBSYMCLK_CLOCK_CNTL__PHYBSYMCLK_EN_MASK                                                             0x00000001L
+#define PHYBSYMCLK_CLOCK_CNTL__PHYBSYMCLK_SRC_SEL_MASK                                                        0x00000030L
+//PHYCSYMCLK_CLOCK_CNTL
+#define PHYCSYMCLK_CLOCK_CNTL__PHYCSYMCLK_EN__SHIFT                                                           0x0
+#define PHYCSYMCLK_CLOCK_CNTL__PHYCSYMCLK_SRC_SEL__SHIFT                                                      0x4
+#define PHYCSYMCLK_CLOCK_CNTL__PHYCSYMCLK_EN_MASK                                                             0x00000001L
+#define PHYCSYMCLK_CLOCK_CNTL__PHYCSYMCLK_SRC_SEL_MASK                                                        0x00000030L
+//PHYDSYMCLK_CLOCK_CNTL
+#define PHYDSYMCLK_CLOCK_CNTL__PHYDSYMCLK_EN__SHIFT                                                           0x0
+#define PHYDSYMCLK_CLOCK_CNTL__PHYDSYMCLK_SRC_SEL__SHIFT                                                      0x4
+#define PHYDSYMCLK_CLOCK_CNTL__PHYDSYMCLK_EN_MASK                                                             0x00000001L
+#define PHYDSYMCLK_CLOCK_CNTL__PHYDSYMCLK_SRC_SEL_MASK                                                        0x00000030L
+//PHYESYMCLK_CLOCK_CNTL
+#define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_EN__SHIFT                                                           0x0
+#define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_SRC_SEL__SHIFT                                                      0x4
+#define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_EN_MASK                                                             0x00000001L
+#define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_SRC_SEL_MASK                                                        0x00000030L
+//HDMISTREAMCLK_CNTL
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL__SHIFT                                                     0x0
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_EN__SHIFT                                                          0x3
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL_MASK                                                       0x00000007L
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_EN_MASK                                                            0x00000008L
+//DCCG_GATE_DISABLE_CNTL3
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK0_GATE_DISABLE__SHIFT                                           0x0
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK1_GATE_DISABLE__SHIFT                                           0x1
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK2_GATE_DISABLE__SHIFT                                           0x2
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK3_GATE_DISABLE__SHIFT                                           0x3
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK4_GATE_DISABLE__SHIFT                                           0x4
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK5_GATE_DISABLE__SHIFT                                           0x5
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE0_GATE_DISABLE__SHIFT                                        0x8
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE0_GATE_DISABLE__SHIFT                                             0x9
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE1_GATE_DISABLE__SHIFT                                        0xa
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE1_GATE_DISABLE__SHIFT                                             0xb
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE2_GATE_DISABLE__SHIFT                                        0xc
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE2_GATE_DISABLE__SHIFT                                             0xd
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE3_GATE_DISABLE__SHIFT                                        0xe
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE3_GATE_DISABLE__SHIFT                                             0xf
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE0_GATE_DISABLE__SHIFT                                        0x14
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE0_GATE_DISABLE__SHIFT                                             0x15
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE1_GATE_DISABLE__SHIFT                                        0x16
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE1_GATE_DISABLE__SHIFT                                             0x17
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE2_GATE_DISABLE__SHIFT                                        0x18
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE2_GATE_DISABLE__SHIFT                                             0x19
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE3_GATE_DISABLE__SHIFT                                        0x1a
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE3_GATE_DISABLE__SHIFT                                             0x1b
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK0_GATE_DISABLE_MASK                                             0x00000001L
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK1_GATE_DISABLE_MASK                                             0x00000002L
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK2_GATE_DISABLE_MASK                                             0x00000004L
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK3_GATE_DISABLE_MASK                                             0x00000008L
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK4_GATE_DISABLE_MASK                                             0x00000010L
+#define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK5_GATE_DISABLE_MASK                                             0x00000020L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE0_GATE_DISABLE_MASK                                          0x00000100L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE0_GATE_DISABLE_MASK                                               0x00000200L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE1_GATE_DISABLE_MASK                                          0x00000400L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE1_GATE_DISABLE_MASK                                               0x00000800L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE2_GATE_DISABLE_MASK                                          0x00001000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE2_GATE_DISABLE_MASK                                               0x00002000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_SE3_GATE_DISABLE_MASK                                          0x00004000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_SE3_GATE_DISABLE_MASK                                               0x00008000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE0_GATE_DISABLE_MASK                                          0x00100000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE0_GATE_DISABLE_MASK                                               0x00200000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE1_GATE_DISABLE_MASK                                          0x00400000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE1_GATE_DISABLE_MASK                                               0x00800000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE2_GATE_DISABLE_MASK                                          0x01000000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE2_GATE_DISABLE_MASK                                               0x02000000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_ROOT_LE3_GATE_DISABLE_MASK                                          0x04000000L
+#define DCCG_GATE_DISABLE_CNTL3__SYMCLK32_LE3_GATE_DISABLE_MASK                                               0x08000000L
+//HDMISTREAMCLK0_DTO_PARAM
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_PHASE__SHIFT                                             0x0
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_MODULO__SHIFT                                            0x8
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_EN__SHIFT                                                0x10
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_PHASE_MASK                                               0x000000FFL
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_MODULO_MASK                                              0x0000FF00L
+#define HDMISTREAMCLK0_DTO_PARAM__HDMISTREAMCLK0_DTO_EN_MASK                                                  0x00010000L
+//DPIACLK_810M_DTO_PHASE
+#define DPIACLK_810M_DTO_PHASE__DPIACLK_810M_DTO_PHASE__SHIFT                                                 0x0
+#define DPIACLK_810M_DTO_PHASE__DPIACLK_810M_DTO_PHASE_MASK                                                   0xFFFFFFFFL
+//DPIACLK_540M_DTO_PHASE
+#define DPIACLK_540M_DTO_PHASE__DPIACLK_540M_DTO_PHASE__SHIFT                                                 0x0
+#define DPIACLK_540M_DTO_PHASE__DPIACLK_540M_DTO_PHASE_MASK                                                   0xFFFFFFFFL
+//DPIACLK_DTO_CNTL
+#define DPIACLK_DTO_CNTL__DPIACLK_810M_DTO_ENABLE__SHIFT                                                      0x0
+#define DPIACLK_DTO_CNTL__DPIACLK_540M_DTO_ENABLE__SHIFT                                                      0x1
+#define DPIACLK_DTO_CNTL__DPIACLK_810M_DTO_ENABLE_MASK                                                        0x00000001L
+#define DPIACLK_DTO_CNTL__DPIACLK_540M_DTO_ENABLE_MASK                                                        0x00000002L
+//DPIACLK_810M_DTO_MODULO
+#define DPIACLK_810M_DTO_MODULO__DPIACLK_810M_DTO_MODULO__SHIFT                                               0x0
+#define DPIACLK_810M_DTO_MODULO__DPIACLK_810M_DTO_MODULO_MASK                                                 0xFFFFFFFFL
+//DPIACLK_540M_DTO_MODULO
+#define DPIACLK_540M_DTO_MODULO__DPIACLK_540M_DTO_MODULO__SHIFT                                               0x0
+#define DPIACLK_540M_DTO_MODULO__DPIACLK_540M_DTO_MODULO_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dccg_dccg_dfs_dispdec
+//DENTIST_DISPCLK_CNTL
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_WDIVIDER__SHIFT                                                 0x0
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_RDIVIDER__SHIFT                                                 0x8
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_MODE__SHIFT                                                 0xf
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHGTOG__SHIFT                                                   0x11
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_DONETOG__SHIFT                                                  0x12
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_DONE__SHIFT                                                 0x13
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_CHG_DONE__SHIFT                                                  0x14
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_CHGTOG__SHIFT                                                    0x15
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_DONETOG__SHIFT                                                   0x16
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_WDIVIDER__SHIFT                                                  0x18
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_WDIVIDER_MASK                                                   0x0000007FL
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_RDIVIDER_MASK                                                   0x00007F00L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_MODE_MASK                                                   0x00018000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHGTOG_MASK                                                     0x00020000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_DONETOG_MASK                                                    0x00040000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_DONE_MASK                                                   0x00080000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_CHG_DONE_MASK                                                    0x00100000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_CHGTOG_MASK                                                      0x00200000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_DONETOG_MASK                                                     0x00400000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPPCLK_WDIVIDER_MASK                                                    0x7F000000L
+
+
+// addressBlock: azroot_f2codecind
+//AZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__SHIFT  0x0
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_MASK  0xFFFFFFFFL
+//AZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID__SHIFT            0x0
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID_MASK              0xFFFFFFFFL
+//AZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__SHIFT  0x0
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT_MASK  0xFFFFFFFFL
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET__SHIFT                                  0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT__SHIFT                                  0x4
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK__SHIFT                                        0x9
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET__SHIFT                       0xa
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET_MASK                                    0x0000000FL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT_MASK                                    0x000000F0L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK_MASK                                          0x00000200L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET_MASK                         0x00000400L
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0__SHIFT                     0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1__SHIFT                     0x8
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2__SHIFT                     0x10
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3__SHIFT                     0x18
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0_MASK                       0x000000FFL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1_MASK                       0x0000FF00L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2_MASK                       0x00FF0000L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3_MASK                       0xFF000000L
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2__SUBSYSTEM_ID_BYTE1__SHIFT                   0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2__SUBSYSTEM_ID_BYTE1_MASK                     0x000000FFL
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3__SUBSYSTEM_ID_BYTE2__SHIFT                   0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3__SUBSYSTEM_ID_BYTE2_MASK                     0x000000FFL
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4__SUBSYSTEM_ID_BYTE3__SHIFT                   0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4__SUBSYSTEM_ID_BYTE3_MASK                     0x000000FFL
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION__SHIFT          0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION_MASK            0x000000FFL
+//AZALIA_F2_CODEC_FUNCTION_CONTROL_RESET
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET__SHIFT                                            0x0
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET_MASK                                              0x00000001L
+//AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__SHIFT  0x0
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT_MASK  0xFFFFFFFFL
+//AZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__SHIFT      0x0
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_MASK        0xFFFFFFFFL
+//AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT               0x0
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT                0x10
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK                 0x00000FFFL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK                  0x001F0000L
+//AZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__SHIFT  0x0
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_MASK  0xFFFFFFFFL
+//AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES__SHIFT  0x0
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP__SHIFT                                       0x1e
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS__SHIFT                                          0x1f
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES_MASK    0x3FFFFFFFL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP_MASK                                         0x40000000L
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS_MASK                                            0x80000000L
+
+
+// addressBlock: azendpoint_f2codecind
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT                         0x0
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT                            0x4
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                        0x8
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                       0xb
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT                           0xe
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                                0xf
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_R__SHIFT                              0xf
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK                           0x0000000FL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                              0x00000070L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK                          0x00000700L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                         0x00003800L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK                             0x00004000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                                  0x00008000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_R_MASK                                0x00008000L
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                                0x0
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                                 0x4
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                                  0x0000000FL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                                   0x000000F0L
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                                     0x0
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                                         0x1
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                                      0x2
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                                       0x3
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                                      0x4
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                                 0x5
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                                       0x6
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                                         0x7
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                                        0x8
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                                 0x17
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                                       0x00000001L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                                           0x00000002L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                                        0x00000004L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                                         0x00000008L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                                        0x00000010L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                                   0x00000020L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                                         0x00000040L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                                           0x00000080L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                                          0x00007F00L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                                   0x00800000L
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2__CC__SHIFT                                      0x0
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2__CC_MASK                                        0x0000007FL
+//AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT                                       0x0
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT                                    0x14
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK                                         0x00000003L
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK                                      0x00700000L
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3__KEEPALIVE__SHIFT                               0x7
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3__KEEPALIVE_MASK                                 0x00000080L
+//AZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT                                         0x0
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK                                           0x000000FFL
+//AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT      0x0
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT         0x1
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT        0x2
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT    0x3
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT                 0x4
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                          0x5
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT               0x6
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT                 0x8
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                         0x9
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT                   0xa
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                         0xb
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                            0x14
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK        0x00000001L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK           0x00000002L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK          0x00000004L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK      0x00000008L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK                   0x00000010L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                            0x00000020L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK                 0x00000040L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK   0x00000080L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK                   0x00000100L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                           0x00000200L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK                     0x00000400L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                           0x00000800L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK   0x000F0000L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                              0x00F00000L
+//AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT              0x0
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT               0x10
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK                0x00000FFFL
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK                 0x001F0000L
+//AZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT                             0x0
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                               0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY__CONNECTION_LIST_ENTRY__SHIFT              0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY__CONNECTION_LIST_ENTRY_MASK                0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT                                         0x6
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK                                           0x00000040L
+//AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                                          0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                                       0x7
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                                            0x0000003FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                                         0x00000080L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                                0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT__SHIFT                                0x1f
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                                  0x7FFFFFFFL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT_MASK                                  0x80000000L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT                           0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT                0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                               0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                              0xc
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT                    0x10
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT                     0x14
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT                           0x18
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT                  0x1e
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK                             0x0000000FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK                  0x000000F0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                                 0x00000F00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                                0x0000F000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK                      0x000F0000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK                       0x00F00000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK                             0x3F000000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK                    0xC0000000L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC__SHIFT                             0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR__SHIFT                            0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC_MASK                               0x0000000FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR_MASK                              0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE__SHIFT                  0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE__SHIFT                   0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE_MASK                    0x0000000FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE_MASK                     0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY__SHIFT                0x6
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION_MASK                           0x0000003FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY_MASK                  0x000000C0L
+//AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__SPEAKER_ALLOCATION__SHIFT                    0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__HDMI_CONNECTION__SHIFT                       0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__DP_CONNECTION__SHIFT                         0x9
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__EXTRA_CONNECTION_INFO__SHIFT                 0xa
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__SPEAKER_ALLOCATION_MASK                      0x0000007FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__HDMI_CONNECTION_MASK                         0x00000100L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__DP_CONNECTION_MASK                           0x00000200L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__EXTRA_CONNECTION_INFO_MASK                   0x0000FC00L
+//AZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT                             0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK                               0x000000FFL
+//AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LFE_PLAYBACK_LEVEL__SHIFT                                  0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LEVEL_SHIFT__SHIFT                                         0x3
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__DOWN_MIX_INHIBIT__SHIFT                                    0x7
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LFE_PLAYBACK_LEVEL_MASK                                    0x00000003L
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LEVEL_SHIFT_MASK                                           0x00000078L
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__DOWN_MIX_INHIBIT_MASK                                      0x00000080L
+//AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__ACP_INDEX__SHIFT                                               0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__SUPPORTS_AI__SHIFT                                             0x6
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__ACP_PACKET_ENABLE__SHIFT                                       0x7
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__ACP_INDEX_MASK                                                 0x0000003FL
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__SUPPORTS_AI_MASK                                               0x00000040L
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_INDEX__ACP_PACKET_ENABLE_MASK                                         0x00000080L
+//AZALIA_F2_CODEC_PIN_CONTROL_ACP_DATA
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_DATA__ACP_DATA__SHIFT                                                 0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_ACP_DATA__ACP_DATA_MASK                                                   0x000000FFL
+//AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__MAX_CHANNELS__SHIFT                                     0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__FORMAT_CODE__SHIFT                                      0x3
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES__SHIFT                            0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__DESCRIPTOR_BYTE_2__SHIFT                                0x10
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_STEREO__SHIFT                     0x18
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__MAX_CHANNELS_MASK                                       0x00000007L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__FORMAT_CODE_MASK                                        0x00000078L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_MASK                              0x0000FF00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__DESCRIPTOR_BYTE_2_MASK                                  0x00FF0000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_STEREO_MASK                       0xFF000000L
+//AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA__DESCRIPTOR__SHIFT                                  0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA__DESCRIPTOR_MASK                                    0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_ENABLE__SHIFT                       0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_MUTE__SHIFT                         0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT                   0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_ENABLE_MASK                         0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_MUTE_MASK                           0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK                     0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_ENABLE__SHIFT                       0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_MUTE__SHIFT                         0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT                   0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_ENABLE_MASK                         0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_MUTE_MASK                           0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK                     0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_ENABLE__SHIFT                       0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_MUTE__SHIFT                         0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT                   0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_ENABLE_MASK                         0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_MUTE_MASK                           0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK                     0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_ENABLE__SHIFT                       0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_MUTE__SHIFT                         0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT                   0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_ENABLE_MASK                         0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_MUTE_MASK                           0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK                     0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__VIDEO_LIPSYNC__SHIFT                                             0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__AUDIO_LIPSYNC__SHIFT                                             0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__VIDEO_LIPSYNC_MASK                                               0x000000FFL
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__AUDIO_LIPSYNC_MASK                                               0x0000FF00L
+//AZALIA_F2_CODEC_PIN_CONTROL_HBR
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_CAPABLE__SHIFT                                                   0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_ENABLE__SHIFT                                                    0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_CAPABLE_MASK                                                     0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_ENABLE_MASK                                                      0x00000010L
+//AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX__SINK_INFO_INDEX__SHIFT                             0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX__SINK_INFO_INDEX_MASK                               0x000000FFL
+//AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA__SINK_DATA__SHIFT                                    0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA__SINK_DATA_MASK                                      0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE__SHIFT                           0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT                     0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE_MASK                             0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK                       0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE__SHIFT                           0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT                     0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE_MASK                             0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK                       0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE__SHIFT                           0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID__SHIFT                     0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE_MASK                             0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID_MASK                       0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE__SHIFT                           0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID__SHIFT                     0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE_MASK                             0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID_MASK                       0x000000F0L
+//AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT                               0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK                                 0x00000001L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT                                   0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT                          0x2
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK                                     0x00000003L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK                            0x0000003CL
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT                         0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT                0x2
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT                            0x3
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT                   0x7
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK                           0x00000003L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK                  0x00000004L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK                              0x00000078L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK                     0x00000080L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT                     0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT            0x6
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK                       0x0000003FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK              0x00000040L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT            0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT   0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK              0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK     0x00000010L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT               0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT                     0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT                                 0x5
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT                           0x7
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK                 0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK                       0x00000010L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK                                   0x00000060L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK                             0x00000080L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT                       0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT                       0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK                         0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK                         0x000000F0L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT                       0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT                       0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK                         0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK                         0x000000F0L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT                       0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT                       0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK                         0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK                         0x000000F0L
+//AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT                       0x0
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT                       0x4
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK                         0x0000000FL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK                         0x000000F0L
+//AZALIA_F2_CODEC_PIN_ASSOCIATION_INFO
+#define AZALIA_F2_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT                                         0x0
+#define AZALIA_F2_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK                                           0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS
+#define AZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE__SHIFT                               0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS__OUTPUT_ACTIVE_MASK                                 0x00000001L
+//AZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT                          0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT                    0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK                            0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK                      0x0000FF00L
+//AZALIA_F2_CODEC_PIN_CONTROL_LPIB
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB__LPIB__SHIFT                                                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB__LPIB_MASK                                                           0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT                           0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK                             0xFFFFFFFFL
+//AZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE
+#define AZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE__SHIFT                                           0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE__CODING_TYPE_MASK                                             0x000000FFL
+//AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED__SHIFT                                     0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE__SHIFT                       0x1
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON__SHIFT                               0x8
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE__SHIFT                             0x10
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_MASK                                       0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGED_ACK_UR_ENABLE_MASK                         0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_REASON_MASK                                 0x0000FF00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED__FORMAT_CHANGE_RESPONSE_MASK                               0x00FF0000L
+//AZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION__SHIFT   0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION__WIRELESS_DISPLAY_IDENTIFICATION_MASK     0x00000003L
+//AZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE
+#define AZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE__SHIFT                         0x0
+#define AZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY__SHIFT                     0x4
+#define AZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_ENABLE_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE__REMOTE_KEEP_ALIVE_CAPABILITY_MASK                       0x00000010L
+//AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT            0x0
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT               0x1
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT              0x2
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT          0x3
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                                0x5
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT                     0x6
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT       0x7
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT                       0x8
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                               0x9
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT                         0xa
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                               0xb
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT       0x10
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                                  0x14
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK              0x00000001L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK                 0x00000002L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK                0x00000004L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK            0x00000008L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                                  0x00000020L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK                       0x00000040L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK         0x00000080L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK                         0x00000100L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                                 0x00000200L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK                           0x00000400L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                                 0x00000800L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK         0x000F0000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                                    0x00F00000L
+//AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT                            0x0
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                                   0x1
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT                          0x2
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT                            0x3
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                                     0x4
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                                      0x5
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                                  0x6
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                               0x7
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                                       0x8
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                                       0x10
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                                 0x18
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                              0x00000001L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                                     0x00000002L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK                            0x00000004L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                              0x00000008L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                                       0x00000010L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                                        0x00000020L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                                    0x00000040L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                                 0x00000080L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                                         0x0000FF00L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                                         0x00010000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK                                                   0x01000000L
+//AZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH__CONNECTION_LIST_LENGTH__SHIFT                   0x0
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH__CONNECTION_LIST_LENGTH_MASK                     0xFFFFFFFFL
+
+
+// addressBlock: azinputendpoint_f2codecind
+//AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT                      0x4
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                  0x8
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                 0xb
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT                     0xe
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT                          0xf
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK                     0x0000000FL
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK                        0x00000070L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK                    0x00000700L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                   0x00003800L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK                       0x00004000L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK                            0x00008000L
+//AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT                          0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT                           0x4
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK                            0x0000000FL
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK                             0x000000F0L
+//AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT                               0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT                                   0x1
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT                                0x2
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT                                 0x3
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT                                0x4
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT                           0x5
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT                                 0x6
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT                                   0x7
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT                                  0x8
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT                           0x17
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK                                 0x00000001L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK                                     0x00000002L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK                                  0x00000004L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK                                   0x00000008L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK                                  0x00000010L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK                             0x00000020L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK                                   0x00000040L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK                                     0x00000080L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK                                    0x00007F00L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK                             0x00800000L
+//AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT  0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT   0x1
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT  0x2
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT  0x3
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT           0x4
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                    0x5
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT         0x6
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT           0x8
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                   0x9
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT             0xa
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                   0xb
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                      0x14
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK  0x00000001L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK     0x00000002L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK    0x00000004L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK  0x00000008L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK             0x00000010L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                      0x00000020L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK           0x00000040L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK  0x00000080L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK             0x00000100L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                     0x00000200L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK               0x00000400L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                     0x00000800L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK  0x000F0000L
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                        0x00F00000L
+//AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT        0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT         0x10
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK          0x00000FFFL
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK           0x001F0000L
+//AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT                       0x0
+#define AZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK                         0xFFFFFFFFL
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE__SHIFT                                    0x5
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL__IN_ENABLE_MASK                                      0x00000020L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT                                    0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT                                 0x7
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK                                      0x0000003FL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK                                   0x00000080L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT                          0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT__SHIFT                          0x1f
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK                            0x7FFFFFFFL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT_MASK                            0x80000000L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT                     0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT          0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT                         0x8
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT                        0xc
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT              0x10
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT               0x14
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT                     0x18
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT            0x1e
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK                       0x0000000FL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK            0x000000F0L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK                           0x00000F00L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK                          0x0000F000L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK                0x000F0000L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK                 0x00F00000L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK                       0x3F000000L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK              0xC0000000L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC__SHIFT                       0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR__SHIFT                      0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC_MASK                         0x0000000FL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR_MASK                        0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE__SHIFT            0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE__SHIFT             0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE_MASK              0x0000000FL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE_MASK               0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY__SHIFT          0x6
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION_MASK                     0x0000003FL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY_MASK            0x000000C0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT                       0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK                         0x000000FFL
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE__MULTICHANNEL0_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE__MULTICHANNEL2_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE__MULTICHANNEL4_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE__MULTICHANNEL6_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR__HBR_CAPABLE__SHIFT                                             0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR__HBR_ENABLE__SHIFT                                              0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR__HBR_CAPABLE_MASK                                               0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR__HBR_ENABLE_MASK                                                0x00000010L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE__SHIFT                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE__SHIFT                     0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE_MASK                     0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE_MASK                       0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID_MASK                 0x000000F0L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK__SHIFT                    0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT__SHIFT              0x8
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__LPIB_SNAPSHOT_LOCK_MASK                      0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL__CYCLIC_BUFFER_WRAP_COUNT_MASK                0x0000FF00L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB__SHIFT                                                   0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB__LPIB_MASK                                                     0xFFFFFFFFL
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT__SHIFT                     0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT__LPIB_TIMER_SNAPSHOT_MASK                       0xFFFFFFFFL
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY__SHIFT                         0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT__SHIFT                         0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE__SHIFT               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE__SHIFT  0x5
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_MASK                           0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__CHANNEL_LAYOUT_MASK                           0x00000006L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_ACTIVITY_UR_ENABLE_MASK                 0x00000010L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL__INPUT_CL_CS_INFOFRAME_CHANGE_UR_ENABLE_MASK   0x00000020L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT__SHIFT                                     0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION__SHIFT                                0x8
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5__SHIFT                                  0x10
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID__SHIFT                                   0x1f
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_COUNT_MASK                                       0x00000007L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__CHANNEL_ALLOCATION_MASK                                  0x0000FF00L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_BYTE_5_MASK                                    0x00FF0000L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME__INFOFRAME_VALID_MASK                                     0x80000000L
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L__CHANNEL_STATUS_L__SHIFT                           0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L__CHANNEL_STATUS_L_MASK                             0xFFFFFFFFL
+//AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H__CHANNEL_STATUS_H__SHIFT                           0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H__CHANNEL_STATUS_H_MASK                             0xFFFFFFFFL
+//AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT      0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT         0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT        0x2
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT    0x3
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT                          0x5
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT               0x6
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT  0x7
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT                 0x8
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT                         0x9
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT                   0xa
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT                         0xb
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT  0x10
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT                            0x14
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK        0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK           0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK          0x00000004L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK      0x00000008L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK                            0x00000020L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK                 0x00000040L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK   0x00000080L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK                   0x00000100L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK                           0x00000200L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK                     0x00000400L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK                           0x00000800L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK   0x000F0000L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK                              0x00F00000L
+//AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT                      0x0
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT                             0x1
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT                    0x2
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT                      0x3
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT                               0x4
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT                                0x5
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT                            0x6
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT                                         0x7
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT                                 0x8
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT                                 0x10
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP__SHIFT                                           0x18
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK                        0x00000001L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK                               0x00000002L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK                      0x00000004L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK                        0x00000008L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK                                 0x00000010L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK                                  0x00000020L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK                              0x00000040L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__HDMI_MASK                                           0x00000080L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK                                   0x0000FF00L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK                                   0x00010000L
+#define AZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES__DP_MASK                                             0x01000000L
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon0_dc_perfmon_dispdec
+//DC_PERFMON0_PERFCOUNTER_CNTL
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON0_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON0_PERFCOUNTER_CNTL2
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON0_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON0_PERFCOUNTER_STATE
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON0_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON0_PERFMON_CNTL
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON0_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON0_PERFMON_CNTL2
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON0_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON0_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON0_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON0_PERFMON_CVALUE_LOW
+#define DC_PERFMON0_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON0_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON0_PERFMON_HI
+#define DC_PERFMON0_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON0_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON0_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON0_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON0_PERFMON_LOW
+#define DC_PERFMON0_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON0_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon1_dc_perfmon_dispdec
+//DC_PERFMON1_PERFCOUNTER_CNTL
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON1_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON1_PERFCOUNTER_CNTL2
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON1_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON1_PERFCOUNTER_STATE
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON1_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON1_PERFMON_CNTL
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON1_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON1_PERFMON_CNTL2
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON1_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON1_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON1_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON1_PERFMON_CVALUE_LOW
+#define DC_PERFMON1_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON1_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON1_PERFMON_HI
+#define DC_PERFMON1_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON1_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON1_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON1_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON1_PERFMON_LOW
+#define DC_PERFMON1_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON1_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dmu_dc_pg_dispdec
+//DOMAIN0_PG_CONFIG
+#define DOMAIN0_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                        0x0
+#define DOMAIN0_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                           0x8
+#define DOMAIN0_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                          0x00000001L
+#define DOMAIN0_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                             0x00000100L
+//DOMAIN0_PG_STATUS
+#define DOMAIN0_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                    0x1c
+#define DOMAIN0_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                     0x1e
+#define DOMAIN0_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                      0x10000000L
+#define DOMAIN0_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                       0xC0000000L
+//DOMAIN1_PG_CONFIG
+#define DOMAIN1_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                        0x0
+#define DOMAIN1_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                           0x8
+#define DOMAIN1_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                          0x00000001L
+#define DOMAIN1_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                             0x00000100L
+//DOMAIN1_PG_STATUS
+#define DOMAIN1_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                    0x1c
+#define DOMAIN1_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                     0x1e
+#define DOMAIN1_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                      0x10000000L
+#define DOMAIN1_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                       0xC0000000L
+//DOMAIN2_PG_CONFIG
+#define DOMAIN2_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                        0x0
+#define DOMAIN2_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                           0x8
+#define DOMAIN2_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                          0x00000001L
+#define DOMAIN2_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                             0x00000100L
+//DOMAIN2_PG_STATUS
+#define DOMAIN2_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                    0x1c
+#define DOMAIN2_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                     0x1e
+#define DOMAIN2_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                      0x10000000L
+#define DOMAIN2_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                       0xC0000000L
+//DOMAIN3_PG_CONFIG
+#define DOMAIN3_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                        0x0
+#define DOMAIN3_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                           0x8
+#define DOMAIN3_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                          0x00000001L
+#define DOMAIN3_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                             0x00000100L
+//DOMAIN3_PG_STATUS
+#define DOMAIN3_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                    0x1c
+#define DOMAIN3_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                     0x1e
+#define DOMAIN3_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                      0x10000000L
+#define DOMAIN3_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                       0xC0000000L
+//DOMAIN16_PG_CONFIG
+#define DOMAIN16_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN16_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN16_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN16_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN16_PG_STATUS
+#define DOMAIN16_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN16_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN16_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN16_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN17_PG_CONFIG
+#define DOMAIN17_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN17_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN17_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN17_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN17_PG_STATUS
+#define DOMAIN17_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN17_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN17_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN17_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN18_PG_CONFIG
+#define DOMAIN18_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN18_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN18_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN18_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN18_PG_STATUS
+#define DOMAIN18_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN18_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN18_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN18_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN19_PG_CONFIG
+#define DOMAIN19_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN19_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN19_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN19_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN19_PG_STATUS
+#define DOMAIN19_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN19_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN19_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN19_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN22_PG_CONFIG
+#define DOMAIN22_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN22_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN22_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN22_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN22_PG_STATUS
+#define DOMAIN22_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN22_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN22_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN22_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN23_PG_CONFIG
+#define DOMAIN23_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN23_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN23_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN23_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN23_PG_STATUS
+#define DOMAIN23_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN23_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN23_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN23_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN24_PG_CONFIG
+#define DOMAIN24_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN24_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN24_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN24_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN24_PG_STATUS
+#define DOMAIN24_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN24_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN24_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN24_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN25_PG_CONFIG
+#define DOMAIN25_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN25_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN25_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN25_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN25_PG_STATUS
+#define DOMAIN25_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN25_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN25_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN25_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DOMAIN26_PG_CONFIG
+#define DOMAIN26_PG_CONFIG__DOMAIN_POWER_FORCEON__SHIFT                                                       0x0
+#define DOMAIN26_PG_CONFIG__DOMAIN_POWER_GATE__SHIFT                                                          0x8
+#define DOMAIN26_PG_CONFIG__DOMAIN_POWER_FORCEON_MASK                                                         0x00000001L
+#define DOMAIN26_PG_CONFIG__DOMAIN_POWER_GATE_MASK                                                            0x00000100L
+//DOMAIN26_PG_STATUS
+#define DOMAIN26_PG_STATUS__DOMAIN_DESIRED_PWR_STATE__SHIFT                                                   0x1c
+#define DOMAIN26_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS__SHIFT                                                    0x1e
+#define DOMAIN26_PG_STATUS__DOMAIN_DESIRED_PWR_STATE_MASK                                                     0x10000000L
+#define DOMAIN26_PG_STATUS__DOMAIN_PGFSM_PWR_STATUS_MASK                                                      0xC0000000L
+//DCPG_INTERRUPT_STATUS
+#define DCPG_INTERRUPT_STATUS__DOMAIN0_POWER_UP_INT_OCCURRED__SHIFT                                           0x0
+#define DCPG_INTERRUPT_STATUS__DOMAIN0_POWER_DOWN_INT_OCCURRED__SHIFT                                         0x1
+#define DCPG_INTERRUPT_STATUS__DOMAIN1_POWER_UP_INT_OCCURRED__SHIFT                                           0x2
+#define DCPG_INTERRUPT_STATUS__DOMAIN1_POWER_DOWN_INT_OCCURRED__SHIFT                                         0x3
+#define DCPG_INTERRUPT_STATUS__DOMAIN2_POWER_UP_INT_OCCURRED__SHIFT                                           0x4
+#define DCPG_INTERRUPT_STATUS__DOMAIN2_POWER_DOWN_INT_OCCURRED__SHIFT                                         0x5
+#define DCPG_INTERRUPT_STATUS__DOMAIN3_POWER_UP_INT_OCCURRED__SHIFT                                           0x6
+#define DCPG_INTERRUPT_STATUS__DOMAIN3_POWER_DOWN_INT_OCCURRED__SHIFT                                         0x7
+#define DCPG_INTERRUPT_STATUS__DOMAIN0_POWER_UP_INT_OCCURRED_MASK                                             0x00000001L
+#define DCPG_INTERRUPT_STATUS__DOMAIN0_POWER_DOWN_INT_OCCURRED_MASK                                           0x00000002L
+#define DCPG_INTERRUPT_STATUS__DOMAIN1_POWER_UP_INT_OCCURRED_MASK                                             0x00000004L
+#define DCPG_INTERRUPT_STATUS__DOMAIN1_POWER_DOWN_INT_OCCURRED_MASK                                           0x00000008L
+#define DCPG_INTERRUPT_STATUS__DOMAIN2_POWER_UP_INT_OCCURRED_MASK                                             0x00000010L
+#define DCPG_INTERRUPT_STATUS__DOMAIN2_POWER_DOWN_INT_OCCURRED_MASK                                           0x00000020L
+#define DCPG_INTERRUPT_STATUS__DOMAIN3_POWER_UP_INT_OCCURRED_MASK                                             0x00000040L
+#define DCPG_INTERRUPT_STATUS__DOMAIN3_POWER_DOWN_INT_OCCURRED_MASK                                           0x00000080L
+//DCPG_INTERRUPT_STATUS_2
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN16_POWER_UP_INT_OCCURRED__SHIFT                                        0x0
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN16_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x1
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN17_POWER_UP_INT_OCCURRED__SHIFT                                        0x2
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN17_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x3
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN18_POWER_UP_INT_OCCURRED__SHIFT                                        0x4
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN18_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x5
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN19_POWER_UP_INT_OCCURRED__SHIFT                                        0x6
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN19_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x7
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN16_POWER_UP_INT_OCCURRED_MASK                                          0x00000001L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN16_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000002L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN17_POWER_UP_INT_OCCURRED_MASK                                          0x00000004L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN17_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000008L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN18_POWER_UP_INT_OCCURRED_MASK                                          0x00000010L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN18_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000020L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN19_POWER_UP_INT_OCCURRED_MASK                                          0x00000040L
+#define DCPG_INTERRUPT_STATUS_2__DOMAIN19_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000080L
+//DCPG_INTERRUPT_STATUS_3
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN22_POWER_UP_INT_OCCURRED__SHIFT                                        0x0
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN22_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x1
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN23_POWER_UP_INT_OCCURRED__SHIFT                                        0x2
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN23_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x3
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN24_POWER_UP_INT_OCCURRED__SHIFT                                        0x4
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN24_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x5
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN25_POWER_UP_INT_OCCURRED__SHIFT                                        0x6
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN25_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x7
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN26_POWER_UP_INT_OCCURRED__SHIFT                                        0x8
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN26_POWER_DOWN_INT_OCCURRED__SHIFT                                      0x9
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN22_POWER_UP_INT_OCCURRED_MASK                                          0x00000001L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN22_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000002L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN23_POWER_UP_INT_OCCURRED_MASK                                          0x00000004L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN23_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000008L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN24_POWER_UP_INT_OCCURRED_MASK                                          0x00000010L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN24_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000020L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN25_POWER_UP_INT_OCCURRED_MASK                                          0x00000040L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN25_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000080L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN26_POWER_UP_INT_OCCURRED_MASK                                          0x00000100L
+#define DCPG_INTERRUPT_STATUS_3__DOMAIN26_POWER_DOWN_INT_OCCURRED_MASK                                        0x00000200L
+//DCPG_INTERRUPT_CONTROL_1
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_UP_INT_MASK__SHIFT                                            0x0
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_UP_INT_CLEAR__SHIFT                                           0x1
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_DOWN_INT_MASK__SHIFT                                          0x2
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_DOWN_INT_CLEAR__SHIFT                                         0x3
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_UP_INT_MASK__SHIFT                                            0x4
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_UP_INT_CLEAR__SHIFT                                           0x5
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_DOWN_INT_MASK__SHIFT                                          0x6
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_DOWN_INT_CLEAR__SHIFT                                         0x7
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_UP_INT_MASK__SHIFT                                            0x8
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_UP_INT_CLEAR__SHIFT                                           0x9
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_DOWN_INT_MASK__SHIFT                                          0xa
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_DOWN_INT_CLEAR__SHIFT                                         0xb
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_UP_INT_MASK__SHIFT                                            0xc
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_UP_INT_CLEAR__SHIFT                                           0xd
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_DOWN_INT_MASK__SHIFT                                          0xe
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_DOWN_INT_CLEAR__SHIFT                                         0xf
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_UP_INT_MASK_MASK                                              0x00000001L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_UP_INT_CLEAR_MASK                                             0x00000002L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_DOWN_INT_MASK_MASK                                            0x00000004L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN0_POWER_DOWN_INT_CLEAR_MASK                                           0x00000008L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_UP_INT_MASK_MASK                                              0x00000010L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_UP_INT_CLEAR_MASK                                             0x00000020L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_DOWN_INT_MASK_MASK                                            0x00000040L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN1_POWER_DOWN_INT_CLEAR_MASK                                           0x00000080L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_UP_INT_MASK_MASK                                              0x00000100L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_UP_INT_CLEAR_MASK                                             0x00000200L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_DOWN_INT_MASK_MASK                                            0x00000400L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN2_POWER_DOWN_INT_CLEAR_MASK                                           0x00000800L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_UP_INT_MASK_MASK                                              0x00001000L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_UP_INT_CLEAR_MASK                                             0x00002000L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_DOWN_INT_MASK_MASK                                            0x00004000L
+#define DCPG_INTERRUPT_CONTROL_1__DOMAIN3_POWER_DOWN_INT_CLEAR_MASK                                           0x00008000L
+//DCPG_INTERRUPT_CONTROL_2
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_UP_INT_MASK__SHIFT                                           0x0
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_UP_INT_CLEAR__SHIFT                                          0x1
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_DOWN_INT_MASK__SHIFT                                         0x2
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_DOWN_INT_CLEAR__SHIFT                                        0x3
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_UP_INT_MASK__SHIFT                                           0x4
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_UP_INT_CLEAR__SHIFT                                          0x5
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_DOWN_INT_MASK__SHIFT                                         0x6
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_DOWN_INT_CLEAR__SHIFT                                        0x7
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_UP_INT_MASK__SHIFT                                           0x8
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_UP_INT_CLEAR__SHIFT                                          0x9
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_DOWN_INT_MASK__SHIFT                                         0xa
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_DOWN_INT_CLEAR__SHIFT                                        0xb
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_UP_INT_MASK__SHIFT                                           0xc
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_UP_INT_CLEAR__SHIFT                                          0xd
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_DOWN_INT_MASK__SHIFT                                         0xe
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_DOWN_INT_CLEAR__SHIFT                                        0xf
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_UP_INT_MASK_MASK                                             0x00000001L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_UP_INT_CLEAR_MASK                                            0x00000002L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_DOWN_INT_MASK_MASK                                           0x00000004L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN16_POWER_DOWN_INT_CLEAR_MASK                                          0x00000008L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_UP_INT_MASK_MASK                                             0x00000010L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_UP_INT_CLEAR_MASK                                            0x00000020L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_DOWN_INT_MASK_MASK                                           0x00000040L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN17_POWER_DOWN_INT_CLEAR_MASK                                          0x00000080L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_UP_INT_MASK_MASK                                             0x00000100L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_UP_INT_CLEAR_MASK                                            0x00000200L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_DOWN_INT_MASK_MASK                                           0x00000400L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN18_POWER_DOWN_INT_CLEAR_MASK                                          0x00000800L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_UP_INT_MASK_MASK                                             0x00001000L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_UP_INT_CLEAR_MASK                                            0x00002000L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_DOWN_INT_MASK_MASK                                           0x00004000L
+#define DCPG_INTERRUPT_CONTROL_2__DOMAIN19_POWER_DOWN_INT_CLEAR_MASK                                          0x00008000L
+//DCPG_INTERRUPT_CONTROL_3
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_UP_INT_MASK__SHIFT                                           0x0
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_UP_INT_CLEAR__SHIFT                                          0x1
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_DOWN_INT_MASK__SHIFT                                         0x2
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_DOWN_INT_CLEAR__SHIFT                                        0x3
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_UP_INT_MASK__SHIFT                                           0x4
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_UP_INT_CLEAR__SHIFT                                          0x5
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_DOWN_INT_MASK__SHIFT                                         0x6
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_DOWN_INT_CLEAR__SHIFT                                        0x7
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_UP_INT_MASK__SHIFT                                           0x8
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_UP_INT_CLEAR__SHIFT                                          0x9
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_DOWN_INT_MASK__SHIFT                                         0xa
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_DOWN_INT_CLEAR__SHIFT                                        0xb
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_UP_INT_MASK__SHIFT                                           0xc
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_UP_INT_CLEAR__SHIFT                                          0xd
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_DOWN_INT_MASK__SHIFT                                         0xe
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_DOWN_INT_CLEAR__SHIFT                                        0xf
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_UP_INT_MASK__SHIFT                                           0x10
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_UP_INT_CLEAR__SHIFT                                          0x11
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_DOWN_INT_MASK__SHIFT                                         0x12
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_DOWN_INT_CLEAR__SHIFT                                        0x13
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_UP_INT_MASK_MASK                                             0x00000001L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_UP_INT_CLEAR_MASK                                            0x00000002L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_DOWN_INT_MASK_MASK                                           0x00000004L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN22_POWER_DOWN_INT_CLEAR_MASK                                          0x00000008L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_UP_INT_MASK_MASK                                             0x00000010L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_UP_INT_CLEAR_MASK                                            0x00000020L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_DOWN_INT_MASK_MASK                                           0x00000040L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN23_POWER_DOWN_INT_CLEAR_MASK                                          0x00000080L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_UP_INT_MASK_MASK                                             0x00000100L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_UP_INT_CLEAR_MASK                                            0x00000200L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_DOWN_INT_MASK_MASK                                           0x00000400L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN24_POWER_DOWN_INT_CLEAR_MASK                                          0x00000800L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_UP_INT_MASK_MASK                                             0x00001000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_UP_INT_CLEAR_MASK                                            0x00002000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_DOWN_INT_MASK_MASK                                           0x00004000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN25_POWER_DOWN_INT_CLEAR_MASK                                          0x00008000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_UP_INT_MASK_MASK                                             0x00010000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_UP_INT_CLEAR_MASK                                            0x00020000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_DOWN_INT_MASK_MASK                                           0x00040000L
+#define DCPG_INTERRUPT_CONTROL_3__DOMAIN26_POWER_DOWN_INT_CLEAR_MASK                                          0x00080000L
+//DC_IP_REQUEST_CNTL
+#define DC_IP_REQUEST_CNTL__IP_REQUEST_EN__SHIFT                                                              0x0
+#define DC_IP_REQUEST_CNTL__IP_REQUEST_EN_MASK                                                                0x00000001L
+//LONO_MEM_PWR_REQ_CNTL
+#define LONO_MEM_PWR_REQ_CNTL__LONO_MEM_PWR_REQ_DIS__SHIFT                                                    0x0
+#define LONO_MEM_PWR_REQ_CNTL__LONO_MEM_PWR_REQ_DIS_MASK                                                      0x00000001L
+
+
+// addressBlock: dce_dc_dmu_dmu_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON2_PERFCOUNTER_CNTL
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON2_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON2_PERFCOUNTER_CNTL2
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON2_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON2_PERFCOUNTER_STATE
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON2_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON2_PERFMON_CNTL
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON2_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON2_PERFMON_CNTL2
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON2_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON2_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON2_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON2_PERFMON_CVALUE_LOW
+#define DC_PERFMON2_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON2_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON2_PERFMON_HI
+#define DC_PERFMON2_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON2_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON2_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON2_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON2_PERFMON_LOW
+#define DC_PERFMON2_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON2_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dmu_dmu_misc_dispdec
+//CC_DC_PIPE_DIS
+#define CC_DC_PIPE_DIS__DC_PIPE_DIS__SHIFT                                                                    0x0
+#define CC_DC_PIPE_DIS__DC_FULL_DIS__SHIFT                                                                    0xc
+#define CC_DC_PIPE_DIS__DC_DMCUB_ENABLE__SHIFT                                                                0x10
+#define CC_DC_PIPE_DIS__DC_PIPE_DIS_MASK                                                                      0x000000FFL
+#define CC_DC_PIPE_DIS__DC_FULL_DIS_MASK                                                                      0x00001000L
+#define CC_DC_PIPE_DIS__DC_DMCUB_ENABLE_MASK                                                                  0x00010000L
+//DMU_CLK_CNTL
+#define DMU_CLK_CNTL__DMU_TEST_CLK_SEL__SHIFT                                                                 0x0
+#define DMU_CLK_CNTL__DISPCLK_R_DMU_GATE_DIS__SHIFT                                                           0x4
+#define DMU_CLK_CNTL__DISPCLK_G_RBBMIF_GATE_DIS__SHIFT                                                        0x5
+#define DMU_CLK_CNTL__DISPCLK_R_CLOCK_ON__SHIFT                                                               0x6
+#define DMU_CLK_CNTL__DISPCLK_G_RBBMIF_CLOCK_ON__SHIFT                                                        0x7
+#define DMU_CLK_CNTL__RIOMMU_CLK_SEL__SHIFT                                                                   0x8
+#define DMU_CLK_CNTL__RBBMIF_FGCG_REP_DIS__SHIFT                                                              0xc
+#define DMU_CLK_CNTL__IHC_FGCG_REP_DIS__SHIFT                                                                 0xd
+#define DMU_CLK_CNTL__DMCUB_DMCUBCLK_SRC_SEL__SHIFT                                                           0xe
+#define DMU_CLK_CNTL__DPREFCLK_ALLOW_DS_CLKSTOP__SHIFT                                                        0x10
+#define DMU_CLK_CNTL__DISPCLK_ALLOW_DS_CLKSTOP__SHIFT                                                         0x12
+#define DMU_CLK_CNTL__DPPCLK_ALLOW_DS_CLKSTOP__SHIFT                                                          0x14
+#define DMU_CLK_CNTL__DTBCLK_ALLOW_DS_CLKSTOP__SHIFT                                                          0x16
+#define DMU_CLK_CNTL__DCFCLK_ALLOW_DS_CLKSTOP__SHIFT                                                          0x18
+#define DMU_CLK_CNTL__DPIACLK_ALLOW_DS_CLKSTOP__SHIFT                                                         0x1a
+#define DMU_CLK_CNTL__LONO_FGCG_REP_DIS__SHIFT                                                                0x1c
+#define DMU_CLK_CNTL__LONO_DISPCLK_GATE_DISABLE__SHIFT                                                        0x1d
+#define DMU_CLK_CNTL__LONO_SOCCLK_GATE_DISABLE__SHIFT                                                         0x1e
+#define DMU_CLK_CNTL__LONO_DMCUBCLK_GATE_DISABLE__SHIFT                                                       0x1f
+#define DMU_CLK_CNTL__DMU_TEST_CLK_SEL_MASK                                                                   0x0000000FL
+#define DMU_CLK_CNTL__DISPCLK_R_DMU_GATE_DIS_MASK                                                             0x00000010L
+#define DMU_CLK_CNTL__DISPCLK_G_RBBMIF_GATE_DIS_MASK                                                          0x00000020L
+#define DMU_CLK_CNTL__DISPCLK_R_CLOCK_ON_MASK                                                                 0x00000040L
+#define DMU_CLK_CNTL__DISPCLK_G_RBBMIF_CLOCK_ON_MASK                                                          0x00000080L
+#define DMU_CLK_CNTL__RIOMMU_CLK_SEL_MASK                                                                     0x00000100L
+#define DMU_CLK_CNTL__RBBMIF_FGCG_REP_DIS_MASK                                                                0x00001000L
+#define DMU_CLK_CNTL__IHC_FGCG_REP_DIS_MASK                                                                   0x00002000L
+#define DMU_CLK_CNTL__DMCUB_DMCUBCLK_SRC_SEL_MASK                                                             0x0000C000L
+#define DMU_CLK_CNTL__DPREFCLK_ALLOW_DS_CLKSTOP_MASK                                                          0x00030000L
+#define DMU_CLK_CNTL__DISPCLK_ALLOW_DS_CLKSTOP_MASK                                                           0x000C0000L
+#define DMU_CLK_CNTL__DPPCLK_ALLOW_DS_CLKSTOP_MASK                                                            0x00300000L
+#define DMU_CLK_CNTL__DTBCLK_ALLOW_DS_CLKSTOP_MASK                                                            0x00C00000L
+#define DMU_CLK_CNTL__DCFCLK_ALLOW_DS_CLKSTOP_MASK                                                            0x03000000L
+#define DMU_CLK_CNTL__DPIACLK_ALLOW_DS_CLKSTOP_MASK                                                           0x0C000000L
+#define DMU_CLK_CNTL__LONO_FGCG_REP_DIS_MASK                                                                  0x10000000L
+#define DMU_CLK_CNTL__LONO_DISPCLK_GATE_DISABLE_MASK                                                          0x20000000L
+#define DMU_CLK_CNTL__LONO_SOCCLK_GATE_DISABLE_MASK                                                           0x40000000L
+#define DMU_CLK_CNTL__LONO_DMCUBCLK_GATE_DISABLE_MASK                                                         0x80000000L
+//DMCUB_SMU_INTERRUPT_CNTL
+#define DMCUB_SMU_INTERRUPT_CNTL__DMCUB_SMU_MSG_INT__SHIFT                                                    0x0
+#define DMCUB_SMU_INTERRUPT_CNTL__DMCUB_SMU_MSG__SHIFT                                                        0x10
+#define DMCUB_SMU_INTERRUPT_CNTL__DMCUB_SMU_MSG_INT_MASK                                                      0x00000001L
+#define DMCUB_SMU_INTERRUPT_CNTL__DMCUB_SMU_MSG_MASK                                                          0xFFFF0000L
+//SMU_INTERRUPT_CONTROL
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_ENABLE__SHIFT                                                       0x0
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_STATUS__SHIFT                                                       0x4
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_EVENT__SHIFT                                                        0x10
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_ENABLE_MASK                                                         0x00000001L
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_STATUS_MASK                                                         0x00000010L
+#define SMU_INTERRUPT_CONTROL__DC_SMU_INT_EVENT_MASK                                                          0xFFFF0000L
+//ZSC_CNTL
+#define ZSC_CNTL__FORCE_SOC_ACCESS__SHIFT                                                                     0x0
+#define ZSC_CNTL__LONO_PWR_DN__SHIFT                                                                          0x8
+#define ZSC_CNTL__FORCE_SOC_ACCESS_MASK                                                                       0x00000003L
+#define ZSC_CNTL__LONO_PWR_DN_MASK                                                                            0x00000100L
+//ZSC_CNTL2
+#define ZSC_CNTL2__ALLOW_Z10__SHIFT                                                                           0x0
+#define ZSC_CNTL2__ALLOW_Z10_MASK                                                                             0x00000001L
+//DMU_MISC_ALLOW_DS_FORCE
+#define DMU_MISC_ALLOW_DS_FORCE__DMU_MISC_ALLOW_DS_FORCE_EN__SHIFT                                            0x0
+#define DMU_MISC_ALLOW_DS_FORCE__DMU_MISC_ALLOW_DS_FORCE_VALUE__SHIFT                                         0x4
+#define DMU_MISC_ALLOW_DS_FORCE__DMU_MISC_ALLOW_DS_FORCE_EN_MASK                                              0x00000001L
+#define DMU_MISC_ALLOW_DS_FORCE__DMU_MISC_ALLOW_DS_FORCE_VALUE_MASK                                           0x00000010L
+//ZSC_STATUS
+#define ZSC_STATUS__SOC_ACCESS_TRIGGER_STATUS__SHIFT                                                          0x0
+#define ZSC_STATUS__SOC_ACCESS_STICKY_TRIGGER_STATUS__SHIFT                                                   0x4
+#define ZSC_STATUS__FENCE_REQ_STATUS__SHIFT                                                                   0x8
+#define ZSC_STATUS__FENCE_ACK_STATUS__SHIFT                                                                   0x9
+#define ZSC_STATUS__FENCE_STATUS__SHIFT                                                                       0xa
+#define ZSC_STATUS__SOC_ACCESS_TRIGGER_STATUS_MASK                                                            0x00000007L
+#define ZSC_STATUS__SOC_ACCESS_STICKY_TRIGGER_STATUS_MASK                                                     0x00000070L
+#define ZSC_STATUS__FENCE_REQ_STATUS_MASK                                                                     0x00000100L
+#define ZSC_STATUS__FENCE_ACK_STATUS_MASK                                                                     0x00000200L
+#define ZSC_STATUS__FENCE_STATUS_MASK                                                                         0x00000C00L
+//DMU_DISPCLK_CGTT_BLK_CTRL_REG
+#define DMU_DISPCLK_CGTT_BLK_CTRL_REG__LONO_DISPCLK_TURN_ON_DELAY__SHIFT                                      0x0
+#define DMU_DISPCLK_CGTT_BLK_CTRL_REG__LONO_DISPCLK_TURN_OFF_DELAY__SHIFT                                     0x4
+#define DMU_DISPCLK_CGTT_BLK_CTRL_REG__LONO_DISPCLK_TURN_ON_DELAY_MASK                                        0x0000000FL
+#define DMU_DISPCLK_CGTT_BLK_CTRL_REG__LONO_DISPCLK_TURN_OFF_DELAY_MASK                                       0x00000FF0L
+//DMU_SOCCLK_CGTT_BLK_CTRL_REG
+#define DMU_SOCCLK_CGTT_BLK_CTRL_REG__LONO_SOCCLK_TURN_ON_DELAY__SHIFT                                        0x0
+#define DMU_SOCCLK_CGTT_BLK_CTRL_REG__LONO_SOCCLK_TURN_OFF_DELAY__SHIFT                                       0x4
+#define DMU_SOCCLK_CGTT_BLK_CTRL_REG__LONO_SOCCLK_TURN_ON_DELAY_MASK                                          0x0000000FL
+#define DMU_SOCCLK_CGTT_BLK_CTRL_REG__LONO_SOCCLK_TURN_OFF_DELAY_MASK                                         0x00000FF0L
+//ZPR_CLK_UNGATE_DELAY
+#define ZPR_CLK_UNGATE_DELAY__ZPR_CLK_UNGATE_DELAY__SHIFT                                                     0x0
+#define ZPR_CLK_UNGATE_DELAY__ZPR_CLK_UNGATE_DELAY_MASK                                                       0x000000FFL
+
+
+// addressBlock: dce_dc_dmu_ihc_dispdec
+//DC_GPU_TIMER_START_POSITION_V_UPDATE
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE__SHIFT                  0x0
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE__SHIFT                  0x4
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE__SHIFT                  0x8
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE__SHIFT                  0xc
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE__SHIFT                  0x10
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE__SHIFT                  0x14
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE_MASK                    0x00000007L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE_MASK                    0x00000070L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE_MASK                    0x00000700L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE_MASK                    0x00007000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE_MASK                    0x00070000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE_MASK                    0x00700000L
+//DC_GPU_TIMER_START_POSITION_VSTARTUP
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D1_VSTARTUP__SHIFT                  0x0
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D2_VSTARTUP__SHIFT                  0x4
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D3_VSTARTUP__SHIFT                  0x8
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D4_VSTARTUP__SHIFT                  0xc
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D5_VSTARTUP__SHIFT                  0x10
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D6_VSTARTUP__SHIFT                  0x14
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D1_VSTARTUP_MASK                    0x00000007L
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D2_VSTARTUP_MASK                    0x00000070L
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D3_VSTARTUP_MASK                    0x00000700L
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D4_VSTARTUP_MASK                    0x00007000L
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D5_VSTARTUP_MASK                    0x00070000L
+#define DC_GPU_TIMER_START_POSITION_VSTARTUP__DC_GPU_TIMER_START_POSITION_D6_VSTARTUP_MASK                    0x00700000L
+//DC_GPU_TIMER_READ
+#define DC_GPU_TIMER_READ__DC_GPU_TIMER_READ__SHIFT                                                           0x0
+#define DC_GPU_TIMER_READ__DC_GPU_TIMER_READ_MASK                                                             0xFFFFFFFFL
+//DC_GPU_TIMER_READ_CNTL
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_READ_SELECT__SHIFT                                               0x0
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D1_VSYNC_NOM__SHIFT                               0x8
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D2_VSYNC_NOM__SHIFT                               0xb
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D3_VSYNC_NOM__SHIFT                               0xe
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D4_VSYNC_NOM__SHIFT                               0x11
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D5_VSYNC_NOM__SHIFT                               0x14
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D6_VSYNC_NOM__SHIFT                               0x17
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_READ_SELECT_MASK                                                 0x0000007FL
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D1_VSYNC_NOM_MASK                                 0x00000700L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D2_VSYNC_NOM_MASK                                 0x00003800L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D3_VSYNC_NOM_MASK                                 0x0001C000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D4_VSYNC_NOM_MASK                                 0x000E0000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D5_VSYNC_NOM_MASK                                 0x00700000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D6_VSYNC_NOM_MASK                                 0x03800000L
+//DISP_INTERRUPT_STATUS
+#define DISP_INTERRUPT_STATUS__OPTC1_DATA_UNDERFLOW_INTERRUPT__SHIFT                                          0x1
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_SNAPSHOT_INTERRUPT__SHIFT                                             0x4
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                                0x5
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                                      0x6
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_TRIGA_INTERRUPT__SHIFT                                                0x7
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_TRIGB_INTERRUPT__SHIFT                                                0x8
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                            0x9
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                              0xa
+#define DISP_INTERRUPT_STATUS__DIGA_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                                0xf
+#define DISP_INTERRUPT_STATUS__DIGA_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                                    0x10
+#define DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT__SHIFT                                                       0x11
+#define DISP_INTERRUPT_STATUS__DC_HPD1_RX_INTERRUPT__SHIFT                                                    0x12
+#define DISP_INTERRUPT_STATUS__AUX1_SW_DONE_INTERRUPT__SHIFT                                                  0x13
+#define DISP_INTERRUPT_STATUS__AUX1_LS_DONE_INTERRUPT__SHIFT                                                  0x14
+#define DISP_INTERRUPT_STATUS__DIO_ALPM_INTERRUPT__SHIFT                                                      0x15
+#define DISP_INTERRUPT_STATUS__RBBMIF_IHC_TIMEOUT_INTERRUPT__SHIFT                                            0x17
+#define DISP_INTERRUPT_STATUS__DC_I2C_SW_DONE_INTERRUPT__SHIFT                                                0x18
+#define DISP_INTERRUPT_STATUS__DISP_INTERRUPT_STATUS_CONTINUE__SHIFT                                          0x1f
+#define DISP_INTERRUPT_STATUS__OPTC1_DATA_UNDERFLOW_INTERRUPT_MASK                                            0x00000002L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_SNAPSHOT_INTERRUPT_MASK                                               0x00000010L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                                  0x00000020L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                                        0x00000040L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_TRIGA_INTERRUPT_MASK                                                  0x00000080L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_TRIGB_INTERRUPT_MASK                                                  0x00000100L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_VSYNC_NOM_INTERRUPT_MASK                                              0x00000200L
+#define DISP_INTERRUPT_STATUS__OTG1_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                                0x00000400L
+#define DISP_INTERRUPT_STATUS__DIGA_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                                  0x00008000L
+#define DISP_INTERRUPT_STATUS__DIGA_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                                      0x00010000L
+#define DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT_MASK                                                         0x00020000L
+#define DISP_INTERRUPT_STATUS__DC_HPD1_RX_INTERRUPT_MASK                                                      0x00040000L
+#define DISP_INTERRUPT_STATUS__AUX1_SW_DONE_INTERRUPT_MASK                                                    0x00080000L
+#define DISP_INTERRUPT_STATUS__AUX1_LS_DONE_INTERRUPT_MASK                                                    0x00100000L
+#define DISP_INTERRUPT_STATUS__DIO_ALPM_INTERRUPT_MASK                                                        0x00200000L
+#define DISP_INTERRUPT_STATUS__RBBMIF_IHC_TIMEOUT_INTERRUPT_MASK                                              0x00800000L
+#define DISP_INTERRUPT_STATUS__DC_I2C_SW_DONE_INTERRUPT_MASK                                                  0x01000000L
+#define DISP_INTERRUPT_STATUS__DISP_INTERRUPT_STATUS_CONTINUE_MASK                                            0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE
+#define DISP_INTERRUPT_STATUS_CONTINUE__OPTC2_DATA_UNDERFLOW_INTERRUPT__SHIFT                                 0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_SNAPSHOT_INTERRUPT__SHIFT                                    0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                       0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                             0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_TRIGA_INTERRUPT__SHIFT                                       0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_TRIGB_INTERRUPT__SHIFT                                       0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                   0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                     0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                       0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                           0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT__SHIFT                                              0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_RX_INTERRUPT__SHIFT                                           0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_SW_DONE_INTERRUPT__SHIFT                                         0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_LS_DONE_INTERRUPT__SHIFT                                         0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT0__SHIFT                                   0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT1__SHIFT                                   0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT2__SHIFT                                   0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_INTERRUPT_STATUS_CONTINUE2__SHIFT                                0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE__OPTC2_DATA_UNDERFLOW_INTERRUPT_MASK                                   0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_SNAPSHOT_INTERRUPT_MASK                                      0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                         0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                               0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_TRIGA_INTERRUPT_MASK                                         0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_TRIGB_INTERRUPT_MASK                                         0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_VSYNC_NOM_INTERRUPT_MASK                                     0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG2_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                       0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                         0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                             0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT_MASK                                                0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_RX_INTERRUPT_MASK                                             0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_SW_DONE_INTERRUPT_MASK                                           0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_LS_DONE_INTERRUPT_MASK                                           0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT0_MASK                                     0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT1_MASK                                     0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__OTG1_IHC_VERTICAL_INTERRUPT2_MASK                                     0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_INTERRUPT_STATUS_CONTINUE2_MASK                                  0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE2
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OPTC3_DATA_UNDERFLOW_INTERRUPT__SHIFT                                0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_SNAPSHOT_INTERRUPT__SHIFT                                   0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                      0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                            0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_TRIGA_INTERRUPT__SHIFT                                      0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_TRIGB_INTERRUPT__SHIFT                                      0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                  0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                    0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                      0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT__SHIFT                                             0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_RX_INTERRUPT__SHIFT                                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_SW_DONE_INTERRUPT__SHIFT                                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_LS_DONE_INTERRUPT__SHIFT                                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT0__SHIFT                                  0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT1__SHIFT                                  0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT2__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DISP_INTERRUPT_STATUS_CONTINUE3__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OPTC3_DATA_UNDERFLOW_INTERRUPT_MASK                                  0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_SNAPSHOT_INTERRUPT_MASK                                     0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                        0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                              0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_TRIGA_INTERRUPT_MASK                                        0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_TRIGB_INTERRUPT_MASK                                        0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_VSYNC_NOM_INTERRUPT_MASK                                    0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG3_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                      0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                        0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT_MASK                                               0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_RX_INTERRUPT_MASK                                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_SW_DONE_INTERRUPT_MASK                                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_LS_DONE_INTERRUPT_MASK                                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT0_MASK                                    0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT1_MASK                                    0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__OTG2_IHC_VERTICAL_INTERRUPT2_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DISP_INTERRUPT_STATUS_CONTINUE3_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE3
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OPTC4_DATA_UNDERFLOW_INTERRUPT__SHIFT                                0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_SNAPSHOT_INTERRUPT__SHIFT                                   0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                      0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                            0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_TRIGA_INTERRUPT__SHIFT                                      0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_TRIGB_INTERRUPT__SHIFT                                      0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                  0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                    0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                      0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT__SHIFT                                             0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_RX_INTERRUPT__SHIFT                                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_SW_DONE_INTERRUPT__SHIFT                                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_LS_DONE_INTERRUPT__SHIFT                                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE3__WBSCL0_DATA_OVERFLOW_INTERRUPT__SHIFT                                0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT0__SHIFT                                  0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT1__SHIFT                                  0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT2__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DISP_INTERRUPT_STATUS_CONTINUE4__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OPTC4_DATA_UNDERFLOW_INTERRUPT_MASK                                  0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_SNAPSHOT_INTERRUPT_MASK                                     0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                        0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                              0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_TRIGA_INTERRUPT_MASK                                        0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_TRIGB_INTERRUPT_MASK                                        0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_VSYNC_NOM_INTERRUPT_MASK                                    0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG4_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                      0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                        0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT_MASK                                               0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_RX_INTERRUPT_MASK                                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_SW_DONE_INTERRUPT_MASK                                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_LS_DONE_INTERRUPT_MASK                                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__WBSCL0_DATA_OVERFLOW_INTERRUPT_MASK                                  0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT0_MASK                                    0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT1_MASK                                    0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__OTG3_IHC_VERTICAL_INTERRUPT2_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DISP_INTERRUPT_STATUS_CONTINUE4_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE4
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OPTC5_DATA_UNDERFLOW_INTERRUPT__SHIFT                                0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OPTC6_DATA_UNDERFLOW_INTERRUPT__SHIFT                                0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_SNAPSHOT_INTERRUPT__SHIFT                                   0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                      0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                            0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_TRIGA_INTERRUPT__SHIFT                                      0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_TRIGB_INTERRUPT__SHIFT                                      0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                  0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                    0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                      0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT__SHIFT                                             0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_RX_INTERRUPT__SHIFT                                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_SW_DONE_INTERRUPT__SHIFT                                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_LS_DONE_INTERRUPT__SHIFT                                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT0__SHIFT                                  0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT1__SHIFT                                  0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT2__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DISP_INTERRUPT_STATUS_CONTINUE5__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OPTC5_DATA_UNDERFLOW_INTERRUPT_MASK                                  0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OPTC6_DATA_UNDERFLOW_INTERRUPT_MASK                                  0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_SNAPSHOT_INTERRUPT_MASK                                     0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                        0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                              0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_TRIGA_INTERRUPT_MASK                                        0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_TRIGB_INTERRUPT_MASK                                        0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_VSYNC_NOM_INTERRUPT_MASK                                    0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG5_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                      0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                        0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT_MASK                                               0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_RX_INTERRUPT_MASK                                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_SW_DONE_INTERRUPT_MASK                                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_LS_DONE_INTERRUPT_MASK                                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT0_MASK                                    0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT1_MASK                                    0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__OTG4_IHC_VERTICAL_INTERRUPT2_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DISP_INTERRUPT_STATUS_CONTINUE5_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE5
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_SNAPSHOT_INTERRUPT__SHIFT                                   0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT                      0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_FORCE_COUNT_NOW_INTERRUPT__SHIFT                            0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_TRIGA_INTERRUPT__SHIFT                                      0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_TRIGB_INTERRUPT__SHIFT                                      0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VSYNC_NOM_INTERRUPT__SHIFT                                  0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT                    0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                      0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT__SHIFT                                             0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_RX_INTERRUPT__SHIFT                                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_SW_DONE_INTERRUPT__SHIFT                                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_LS_DONE_INTERRUPT__SHIFT                                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT0__SHIFT                                  0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT1__SHIFT                                  0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT2__SHIFT                                  0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT0__SHIFT                                  0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT1__SHIFT                                  0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT2__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DISP_INTERRUPT_STATUS_CONTINUE6__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_SNAPSHOT_INTERRUPT_MASK                                     0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK                        0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_FORCE_COUNT_NOW_INTERRUPT_MASK                              0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_TRIGA_INTERRUPT_MASK                                        0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_TRIGB_INTERRUPT_MASK                                        0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VSYNC_NOM_INTERRUPT_MASK                                    0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK                      0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                        0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK                                               0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_RX_INTERRUPT_MASK                                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_SW_DONE_INTERRUPT_MASK                                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_LS_DONE_INTERRUPT_MASK                                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT0_MASK                                    0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT1_MASK                                    0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG5_IHC_VERTICAL_INTERRUPT2_MASK                                    0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT0_MASK                                    0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT1_MASK                                    0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__OTG6_IHC_VERTICAL_INTERRUPT2_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DISP_INTERRUPT_STATUS_CONTINUE6_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE6
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE0_ALPM_WAKE_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE1_ALPM_WAKE_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE2_ALPM_WAKE_INTERRUPT__SHIFT                               0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE3_ALPM_WAKE_INTERRUPT__SHIFT                               0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE4_ALPM_WAKE_INTERRUPT__SHIFT                               0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE5_ALPM_WAKE_INTERRUPT__SHIFT                               0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_CWB0_IHIF_INTERRUPT__SHIFT                                      0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_CWB1_IHIF_INTERRUPT__SHIFT                                      0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB0_IHIF_INTERRUPT__SHIFT                                      0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB1_IHIF_INTERRUPT__SHIFT                                      0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB2_IHIF_INTERRUPT__SHIFT                                      0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE6__DISP_INTERRUPT_STATUS_CONTINUE7__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE0_ALPM_WAKE_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE1_ALPM_WAKE_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE2_ALPM_WAKE_INTERRUPT_MASK                                 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE3_ALPM_WAKE_INTERRUPT_MASK                                 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE4_ALPM_WAKE_INTERRUPT_MASK                                 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__HPO_IHC_SE5_ALPM_WAKE_INTERRUPT_MASK                                 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_CWB0_IHIF_INTERRUPT_MASK                                        0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_CWB1_IHIF_INTERRUPT_MASK                                        0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB0_IHIF_INTERRUPT_MASK                                        0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB1_IHIF_INTERRUPT_MASK                                        0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__MCIF_DWB2_IHIF_INTERRUPT_MASK                                        0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE6__DISP_INTERRUPT_STATUS_CONTINUE7_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE7
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DCCG_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DCCG_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DMU_PERFMON_COUNTER0_INTERRUPT__SHIFT                                0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DMU_PERFMON_COUNTER1_INTERRUPT__SHIFT                                0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DIO_PERFMON_COUNTER0_INTERRUPT__SHIFT                                0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DIO_PERFMON_COUNTER1_INTERRUPT__SHIFT                                0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE7__WB0_PERFMON_COUNTER0_INTERRUPT__SHIFT                                0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE7__WB0_PERFMON_COUNTER1_INTERRUPT__SHIFT                                0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DISP_INTERRUPT_STATUS_CONTINUE8__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DCCG_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DCCG_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DMU_PERFMON_COUNTER0_INTERRUPT_MASK                                  0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DMU_PERFMON_COUNTER1_INTERRUPT_MASK                                  0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DIO_PERFMON_COUNTER0_INTERRUPT_MASK                                  0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DIO_PERFMON_COUNTER1_INTERRUPT_MASK                                  0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__WB0_PERFMON_COUNTER0_INTERRUPT_MASK                                  0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__WB0_PERFMON_COUNTER1_INTERRUPT_MASK                                  0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE7__DISP_INTERRUPT_STATUS_CONTINUE8_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE8
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP0_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP0_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP1_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP1_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP2_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP2_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DISP_INTERRUPT_STATUS_CONTINUE9__SHIFT                               0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP0_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP0_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP1_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP1_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP2_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DPP2_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE8__DISP_INTERRUPT_STATUS_CONTINUE9_MASK                                 0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE9
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP0_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP1_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP2_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP6_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP7_HIST_RDY_IHC_INTERRUPT__SHIFT                                   0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE9__WBSCL1_DATA_OVERFLOW_INTERRUPT__SHIFT                                0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE9__WBSCL2_DATA_OVERFLOW_INTERRUPT__SHIFT                                0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DISP_INTERRUPT_STATUS_CONTINUE10__SHIFT                              0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP0_HIST_RDY_IHC_INTERRUPT_MASK                                     0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP1_HIST_RDY_IHC_INTERRUPT_MASK                                     0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP2_HIST_RDY_IHC_INTERRUPT_MASK                                     0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP3_HIST_RDY_IHC_INTERRUPT_MASK                                     0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP4_HIST_RDY_IHC_INTERRUPT_MASK                                     0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP5_HIST_RDY_IHC_INTERRUPT_MASK                                     0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP6_HIST_RDY_IHC_INTERRUPT_MASK                                     0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DPP7_HIST_RDY_IHC_INTERRUPT_MASK                                     0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__WBSCL1_DATA_OVERFLOW_INTERRUPT_MASK                                  0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__WBSCL2_DATA_OVERFLOW_INTERRUPT_MASK                                  0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE9__DISP_INTERRUPT_STATUS_CONTINUE10_MASK                                0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE10
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG0_LATCH_INT__SHIFT                                0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG1_LATCH_INT__SHIFT                                0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG2_LATCH_INT__SHIFT                                0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG3_LATCH_INT__SHIFT                                0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG4_LATCH_INT__SHIFT                                0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG5_LATCH_INT__SHIFT                                0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_PERFMON2_COUNTER0_INTERRUPT__SHIFT                             0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_PERFMON2_COUNTER1_INTERRUPT__SHIFT                             0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG1_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG2_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG3_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG4_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG5_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG6_IHC_DRR_TIMING_UPDATE__SHIFT                                   0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DISP_INTERRUPT_STATUS_CONTINUE11__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG0_LATCH_INT_MASK                                  0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG1_LATCH_INT_MASK                                  0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG2_LATCH_INT_MASK                                  0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG3_LATCH_INT_MASK                                  0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG4_LATCH_INT_MASK                                  0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_IHC_VSYNC_OTG5_LATCH_INT_MASK                                  0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_PERFMON2_COUNTER0_INTERRUPT_MASK                               0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DCCG_PERFMON2_COUNTER1_INTERRUPT_MASK                               0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG1_IHC_DRR_TIMING_UPDATE_MASK                                     0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG2_IHC_DRR_TIMING_UPDATE_MASK                                     0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG3_IHC_DRR_TIMING_UPDATE_MASK                                     0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG4_IHC_DRR_TIMING_UPDATE_MASK                                     0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG5_IHC_DRR_TIMING_UPDATE_MASK                                     0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__OTG6_IHC_DRR_TIMING_UPDATE_MASK                                     0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE10__DISP_INTERRUPT_STATUS_CONTINUE11_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE11
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB1_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB1_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB2_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB2_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC0_STALL_INTERRUPT__SHIFT                                        0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC1_STALL_INTERRUPT__SHIFT                                        0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC2_STALL_INTERRUPT__SHIFT                                        0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC3_STALL_INTERRUPT__SHIFT                                        0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC4_STALL_INTERRUPT__SHIFT                                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC5_STALL_INTERRUPT__SHIFT                                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC6_STALL_INTERRUPT__SHIFT                                        0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC7_STALL_INTERRUPT__SHIFT                                        0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE11__DISP_INTERRUPT_STATUS_CONTINUE12__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB1_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB1_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB2_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__WB2_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC0_STALL_INTERRUPT_MASK                                          0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC1_STALL_INTERRUPT_MASK                                          0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC2_STALL_INTERRUPT_MASK                                          0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC3_STALL_INTERRUPT_MASK                                          0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC4_STALL_INTERRUPT_MASK                                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC5_STALL_INTERRUPT_MASK                                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC6_STALL_INTERRUPT_MASK                                          0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__MPCC7_STALL_INTERRUPT_MASK                                          0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE11__DISP_INTERRUPT_STATUS_CONTINUE12_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE12
+#define DISP_INTERRUPT_STATUS_CONTINUE12__MPC_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE12__MPC_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP6_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP6_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP7_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP7_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DISP_INTERRUPT_STATUS_CONTINUE13__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE12__MPC_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__MPC_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP6_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP6_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP7_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DPP7_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE12__DISP_INTERRUPT_STATUS_CONTINUE13_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE13
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_PERFMON_COUNTER0_INTERRUPT__SHIFT                            0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_PERFMON_COUNTER1_INTERRUPT__SHIFT                            0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_VM_FAULT_INTERRUPT__SHIFT                                0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_TIMEOUT_INTERRUPT__SHIFT                                 0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_COMPBUF_SIZE_CHANGE_INTERRUPT__SHIFT                     0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN0_POWER_UP_INTERRUPT__SHIFT                          0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN1_POWER_UP_INTERRUPT__SHIFT                          0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN2_POWER_UP_INTERRUPT__SHIFT                          0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN3_POWER_UP_INTERRUPT__SHIFT                          0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN4_POWER_UP_INTERRUPT__SHIFT                          0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN5_POWER_UP_INTERRUPT__SHIFT                          0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN6_POWER_UP_INTERRUPT__SHIFT                          0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN7_POWER_UP_INTERRUPT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VBLANK_INTERRUPT__SHIFT                                   0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VLINE_INTERRUPT__SHIFT                                    0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VLINE2_INTERRUPT__SHIFT                                   0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DISP_INTERRUPT_STATUS_CONTINUE14__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_PERFMON_COUNTER0_INTERRUPT_MASK                              0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_PERFMON_COUNTER1_INTERRUPT_MASK                              0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_VM_FAULT_INTERRUPT_MASK                                  0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_TIMEOUT_INTERRUPT_MASK                                   0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBBUB_IHC_COMPBUF_SIZE_CHANGE_INTERRUPT_MASK                       0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN0_POWER_UP_INTERRUPT_MASK                            0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN1_POWER_UP_INTERRUPT_MASK                            0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN2_POWER_UP_INTERRUPT_MASK                            0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN3_POWER_UP_INTERRUPT_MASK                            0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN4_POWER_UP_INTERRUPT_MASK                            0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN5_POWER_UP_INTERRUPT_MASK                            0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN6_POWER_UP_INTERRUPT_MASK                            0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DCPG_IHC_DOMAIN7_POWER_UP_INTERRUPT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VBLANK_INTERRUPT_MASK                                     0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VLINE_INTERRUPT_MASK                                      0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_VLINE2_INTERRUPT_MASK                                     0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__HUBP0_IHC_TIMEOUT_INTERRUPT_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE13__DISP_INTERRUPT_STATUS_CONTINUE14_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE14
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP2_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP2_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP3_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP3_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VBLANK_INTERRUPT__SHIFT                                   0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VLINE_INTERRUPT__SHIFT                                    0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VLINE2_INTERRUPT__SHIFT                                   0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE14__DISP_INTERRUPT_STATUS_CONTINUE15__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP2_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP2_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP3_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP3_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VBLANK_INTERRUPT_MASK                                     0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VLINE_INTERRUPT_MASK                                      0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_VLINE2_INTERRUPT_MASK                                     0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__HUBP1_IHC_TIMEOUT_INTERRUPT_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE14__DISP_INTERRUPT_STATUS_CONTINUE15_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE15
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP4_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP4_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP5_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP5_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP6_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP6_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VBLANK_INTERRUPT__SHIFT                                   0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VLINE_INTERRUPT__SHIFT                                    0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VLINE2_INTERRUPT__SHIFT                                   0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE15__DISP_INTERRUPT_STATUS_CONTINUE16__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP4_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP4_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP5_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP5_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP6_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP6_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VBLANK_INTERRUPT_MASK                                     0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VLINE_INTERRUPT_MASK                                      0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_VLINE2_INTERRUPT_MASK                                     0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__HUBP2_IHC_TIMEOUT_INTERRUPT_MASK                                    0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE15__DISP_INTERRUPT_STATUS_CONTINUE16_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE16
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_PERFMON_COUNTER0_INTERRUPT__SHIFT                             0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_PERFMON_COUNTER1_INTERRUPT__SHIFT                             0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VBLANK_INTERRUPT__SHIFT                                   0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VLINE_INTERRUPT__SHIFT                                    0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VLINE2_INTERRUPT__SHIFT                                   0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VBLANK_INTERRUPT__SHIFT                                   0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VLINE_INTERRUPT__SHIFT                                    0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VLINE2_INTERRUPT__SHIFT                                   0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VBLANK_INTERRUPT__SHIFT                                   0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VLINE_INTERRUPT__SHIFT                                    0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VLINE2_INTERRUPT__SHIFT                                   0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VBLANK_INTERRUPT__SHIFT                                   0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VLINE_INTERRUPT__SHIFT                                    0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VLINE2_INTERRUPT__SHIFT                                   0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VBLANK_INTERRUPT__SHIFT                                   0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VLINE_INTERRUPT__SHIFT                                    0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VLINE2_INTERRUPT__SHIFT                                   0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_TIMEOUT_INTERRUPT__SHIFT                                  0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE16__DISP_INTERRUPT_STATUS_CONTINUE17__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_PERFMON_COUNTER0_INTERRUPT_MASK                               0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_PERFMON_COUNTER1_INTERRUPT_MASK                               0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VBLANK_INTERRUPT_MASK                                     0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VLINE_INTERRUPT_MASK                                      0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_VLINE2_INTERRUPT_MASK                                     0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VBLANK_INTERRUPT_MASK                                     0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VLINE_INTERRUPT_MASK                                      0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_VLINE2_INTERRUPT_MASK                                     0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VBLANK_INTERRUPT_MASK                                     0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VLINE_INTERRUPT_MASK                                      0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_VLINE2_INTERRUPT_MASK                                     0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VBLANK_INTERRUPT_MASK                                     0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VLINE_INTERRUPT_MASK                                      0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_VLINE2_INTERRUPT_MASK                                     0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VBLANK_INTERRUPT_MASK                                     0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VLINE_INTERRUPT_MASK                                      0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_VLINE2_INTERRUPT_MASK                                     0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP3_IHC_TIMEOUT_INTERRUPT_MASK                                    0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP4_IHC_TIMEOUT_INTERRUPT_MASK                                    0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP5_IHC_TIMEOUT_INTERRUPT_MASK                                    0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP6_IHC_TIMEOUT_INTERRUPT_MASK                                    0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__HUBP7_IHC_TIMEOUT_INTERRUPT_MASK                                    0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE16__DISP_INTERRUPT_STATUS_CONTINUE17_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE17
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPP_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPP_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP0_IHC_FLIP_INTERRUPT__SHIFT                                     0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP1_IHC_FLIP_INTERRUPT__SHIFT                                     0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP2_IHC_FLIP_INTERRUPT__SHIFT                                     0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP3_IHC_FLIP_INTERRUPT__SHIFT                                     0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP4_IHC_FLIP_INTERRUPT__SHIFT                                     0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP5_IHC_FLIP_INTERRUPT__SHIFT                                     0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP6_IHC_FLIP_INTERRUPT__SHIFT                                     0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP7_IHC_FLIP_INTERRUPT__SHIFT                                     0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPTC_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPTC_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE17__MMHUBBUB_PERFMON_COUNTER0_INTERRUPT__SHIFT                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE17__MMHUBBUB_PERFMON_COUNTER1_INTERRUPT__SHIFT                          0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP0_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP1_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP2_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP3_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP4_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP5_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP6_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP7_IHC_FLIP_AWAY_INTERRUPT__SHIFT                                0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE17__DISP_INTERRUPT_STATUS_CONTINUE18__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPP_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPP_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP0_IHC_FLIP_INTERRUPT_MASK                                       0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP1_IHC_FLIP_INTERRUPT_MASK                                       0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP2_IHC_FLIP_INTERRUPT_MASK                                       0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP3_IHC_FLIP_INTERRUPT_MASK                                       0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP4_IHC_FLIP_INTERRUPT_MASK                                       0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP5_IHC_FLIP_INTERRUPT_MASK                                       0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP6_IHC_FLIP_INTERRUPT_MASK                                       0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP7_IHC_FLIP_INTERRUPT_MASK                                       0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPTC_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__OPTC_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__MMHUBBUB_PERFMON_COUNTER0_INTERRUPT_MASK                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__MMHUBBUB_PERFMON_COUNTER1_INTERRUPT_MASK                            0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP0_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP1_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP2_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP3_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP4_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP5_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP6_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__HUBP7_IHC_FLIP_AWAY_INTERRUPT_MASK                                  0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE17__DISP_INTERRUPT_STATUS_CONTINUE18_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE18
+#define DISP_INTERRUPT_STATUS_CONTINUE18__AZ_PERFMON_COUNTER0_INTERRUPT__SHIFT                                0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE18__AZ_PERFMON_COUNTER1_INTERRUPT__SHIFT                                0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXA_IHC_ERROR_INTERRUPT__SHIFT                            0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXB_IHC_ERROR_INTERRUPT__SHIFT                            0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXC_IHC_ERROR_INTERRUPT__SHIFT                            0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXD_IHC_ERROR_INTERRUPT__SHIFT                            0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXE_IHC_ERROR_INTERRUPT__SHIFT                            0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXF_IHC_ERROR_INTERRUPT__SHIFT                            0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXG_IHC_ERROR_INTERRUPT__SHIFT                            0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_RXA_IHC_ERROR_INTERRUPT__SHIFT                            0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN0_POWER_DOWN_INTERRUPT__SHIFT                        0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN1_POWER_DOWN_INTERRUPT__SHIFT                        0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN2_POWER_DOWN_INTERRUPT__SHIFT                        0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN3_POWER_DOWN_INTERRUPT__SHIFT                        0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN4_POWER_DOWN_INTERRUPT__SHIFT                        0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN5_POWER_DOWN_INTERRUPT__SHIFT                        0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN6_POWER_DOWN_INTERRUPT__SHIFT                        0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN7_POWER_DOWN_INTERRUPT__SHIFT                        0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DISP_INTERRUPT_STATUS_CONTINUE19__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE18__AZ_PERFMON_COUNTER0_INTERRUPT_MASK                                  0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__AZ_PERFMON_COUNTER1_INTERRUPT_MASK                                  0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXA_IHC_ERROR_INTERRUPT_MASK                              0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXB_IHC_ERROR_INTERRUPT_MASK                              0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXC_IHC_ERROR_INTERRUPT_MASK                              0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXD_IHC_ERROR_INTERRUPT_MASK                              0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXE_IHC_ERROR_INTERRUPT_MASK                              0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXF_IHC_ERROR_INTERRUPT_MASK                              0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_TXG_IHC_ERROR_INTERRUPT_MASK                              0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCIO_DPCS_RXA_IHC_ERROR_INTERRUPT_MASK                              0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN0_POWER_DOWN_INTERRUPT_MASK                          0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN1_POWER_DOWN_INTERRUPT_MASK                          0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN2_POWER_DOWN_INTERRUPT_MASK                          0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN3_POWER_DOWN_INTERRUPT_MASK                          0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN4_POWER_DOWN_INTERRUPT_MASK                          0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN5_POWER_DOWN_INTERRUPT_MASK                          0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN6_POWER_DOWN_INTERRUPT_MASK                          0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DCPG_IHC_DOMAIN7_POWER_DOWN_INTERRUPT_MASK                          0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE18__DISP_INTERRUPT_STATUS_CONTINUE19_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE19
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_FORMAT_CHANGED_INT__SHIFT                    0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_ENABLED_INT__SHIFT                           0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_ENABLED_INT__SHIFT                           0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_ENABLED_INT__SHIFT                           0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_ENABLED_INT__SHIFT                           0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_ENABLED_INT__SHIFT                           0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_ENABLED_INT__SHIFT                           0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_ENABLED_INT__SHIFT                           0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_ENABLED_INT__SHIFT                           0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_DISABLED_INT__SHIFT                          0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_DISABLED_INT__SHIFT                          0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_DISABLED_INT__SHIFT                          0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_DISABLED_INT__SHIFT                          0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_DISABLED_INT__SHIFT                          0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_DISABLED_INT__SHIFT                          0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_DISABLED_INT__SHIFT                          0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_DISABLED_INT__SHIFT                          0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DIGG_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                     0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DIGG_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                         0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DISP_INTERRUPT_STATUS_CONTINUE20__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_FORMAT_CHANGED_INT_MASK                      0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_ENABLED_INT_MASK                             0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_ENABLED_INT_MASK                             0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_ENABLED_INT_MASK                             0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_ENABLED_INT_MASK                             0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_ENABLED_INT_MASK                             0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_ENABLED_INT_MASK                             0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_ENABLED_INT_MASK                             0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_ENABLED_INT_MASK                             0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT0_AUDIO_DISABLED_INT_MASK                            0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT1_AUDIO_DISABLED_INT_MASK                            0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT2_AUDIO_DISABLED_INT_MASK                            0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT3_AUDIO_DISABLED_INT_MASK                            0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT4_AUDIO_DISABLED_INT_MASK                            0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT5_AUDIO_DISABLED_INT_MASK                            0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT6_AUDIO_DISABLED_INT_MASK                            0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__AZ_IHC_ENDPOINT7_AUDIO_DISABLED_INT_MASK                            0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DIGG_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                       0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DIGG_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                           0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE19__DISP_INTERRUPT_STATUS_CONTINUE20_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE20
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_CPU_SS_INTERRUPT__SHIFT                                    0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_V_UPDATE_INTERRUPT__SHIFT                                  0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_GSL_VSYNC_GAP_INTERRUPT__SHIFT                             0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_VSTARTUP_INTERRUPT__SHIFT                                  0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_VREADY_INTERRUPT__SHIFT                                    0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_VREADY_INTERRUPT__SHIFT                                    0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_VREADY_INTERRUPT__SHIFT                                    0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_VREADY_INTERRUPT__SHIFT                                    0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_VREADY_INTERRUPT__SHIFT                                    0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_VREADY_INTERRUPT__SHIFT                                    0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE20__DISP_INTERRUPT_STATUS_CONTINUE21__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_CPU_SS_INTERRUPT_MASK                                      0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_V_UPDATE_INTERRUPT_MASK                                    0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_GSL_VSYNC_GAP_INTERRUPT_MASK                               0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_VSTARTUP_INTERRUPT_MASK                                    0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG1_IHC_VREADY_INTERRUPT_MASK                                      0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG2_IHC_VREADY_INTERRUPT_MASK                                      0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG3_IHC_VREADY_INTERRUPT_MASK                                      0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG4_IHC_VREADY_INTERRUPT_MASK                                      0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG5_IHC_VREADY_INTERRUPT_MASK                                      0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__OTG6_IHC_VREADY_INTERRUPT_MASK                                      0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE20__DISP_INTERRUPT_STATUS_CONTINUE21_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE21
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC1_HW_DONE_INTERRUPT__SHIFT                          0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC2_HW_DONE_INTERRUPT__SHIFT                          0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC3_HW_DONE_INTERRUPT__SHIFT                          0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC4_HW_DONE_INTERRUPT__SHIFT                          0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC5_HW_DONE_INTERRUPT__SHIFT                          0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC6_HW_DONE_INTERRUPT__SHIFT                          0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDCVGA_HW_DONE_INTERRUPT__SHIFT                        0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC1_READ_REQUEST_INTERRUPT__SHIFT                           0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC2_READ_REQUEST_INTERRUPT__SHIFT                           0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC3_READ_REQUEST_INTERRUPT__SHIFT                           0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC4_READ_REQUEST_INTERRUPT__SHIFT                           0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC5_READ_REQUEST_INTERRUPT__SHIFT                           0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC6_READ_REQUEST_INTERRUPT__SHIFT                           0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_VGA_READ_REQUEST_INTERRUPT__SHIFT                            0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DIGH_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT                     0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DIGH_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT                         0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DISP_INTERRUPT_STATUS_CONTINUE22__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC1_HW_DONE_INTERRUPT_MASK                            0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC2_HW_DONE_INTERRUPT_MASK                            0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC3_HW_DONE_INTERRUPT_MASK                            0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC4_HW_DONE_INTERRUPT_MASK                            0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC5_HW_DONE_INTERRUPT_MASK                            0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDC6_HW_DONE_INTERRUPT_MASK                            0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DOUT_IHC_I2C_DDCVGA_HW_DONE_INTERRUPT_MASK                          0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC1_READ_REQUEST_INTERRUPT_MASK                             0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC2_READ_REQUEST_INTERRUPT_MASK                             0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC3_READ_REQUEST_INTERRUPT_MASK                             0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC4_READ_REQUEST_INTERRUPT_MASK                             0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC5_READ_REQUEST_INTERRUPT_MASK                             0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_DDC6_READ_REQUEST_INTERRUPT_MASK                             0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DC_I2C_VGA_READ_REQUEST_INTERRUPT_MASK                              0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DIGH_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK                       0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DIGH_DP_VID_STREAM_DISABLE_INTERRUPT_MASK                           0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE21__DISP_INTERRUPT_STATUS_CONTINUE22_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE22
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT__SHIFT                          0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG0_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG1_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG2_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG3_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG4_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG5_DRR_V_TOTAL_REACH_INTERRUPT__SHIFT                             0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE22__DISP_INTERRUPT_STATUS_CONTINUE23__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT_MASK                            0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG0_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG1_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG2_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG3_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG4_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__OTG5_DRR_V_TOTAL_REACH_INTERRUPT_MASK                               0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE22__DISP_INTERRUPT_STATUS_CONTINUE23_MASK                               0x80000000L
+//DC_GPU_TIMER_START_POSITION_VREADY
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D1_VREADY__SHIFT                      0x0
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D2_VREADY__SHIFT                      0x4
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D3_VREADY__SHIFT                      0x8
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D4_VREADY__SHIFT                      0xc
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D5_VREADY__SHIFT                      0x10
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D6_VREADY__SHIFT                      0x14
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D1_VREADY_MASK                        0x00000007L
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D2_VREADY_MASK                        0x00000070L
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D3_VREADY_MASK                        0x00000700L
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D4_VREADY_MASK                        0x00007000L
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D5_VREADY_MASK                        0x00070000L
+#define DC_GPU_TIMER_START_POSITION_VREADY__DC_GPU_TIMER_START_POSITION_D6_VREADY_MASK                        0x00700000L
+//DC_GPU_TIMER_START_POSITION_FLIP
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D1_FLIP__SHIFT                          0x0
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D2_FLIP__SHIFT                          0x4
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D3_FLIP__SHIFT                          0x8
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D4_FLIP__SHIFT                          0xc
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D5_FLIP__SHIFT                          0x10
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D6_FLIP__SHIFT                          0x14
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D7_FLIP__SHIFT                          0x18
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D8_FLIP__SHIFT                          0x1c
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D1_FLIP_MASK                            0x00000007L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D2_FLIP_MASK                            0x00000070L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D3_FLIP_MASK                            0x00000700L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D4_FLIP_MASK                            0x00007000L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D5_FLIP_MASK                            0x00070000L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D6_FLIP_MASK                            0x00700000L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D7_FLIP_MASK                            0x07000000L
+#define DC_GPU_TIMER_START_POSITION_FLIP__DC_GPU_TIMER_START_POSITION_D8_FLIP_MASK                            0x70000000L
+//DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE_NO_LOCK__SHIFT  0x0
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE_NO_LOCK__SHIFT  0x4
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE_NO_LOCK__SHIFT  0x8
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE_NO_LOCK__SHIFT  0xc
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE_NO_LOCK__SHIFT  0x10
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE_NO_LOCK__SHIFT  0x14
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE_NO_LOCK_MASK    0x00000007L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE_NO_LOCK_MASK    0x00000070L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE_NO_LOCK_MASK    0x00000700L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE_NO_LOCK_MASK    0x00007000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE_NO_LOCK_MASK    0x00070000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE_NO_LOCK_MASK    0x00700000L
+//DC_GPU_TIMER_START_POSITION_FLIP_AWAY
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D1_FLIP_AWAY__SHIFT                0x0
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D2_FLIP_AWAY__SHIFT                0x4
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D3_FLIP_AWAY__SHIFT                0x8
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D4_FLIP_AWAY__SHIFT                0xc
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D5_FLIP_AWAY__SHIFT                0x10
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D6_FLIP_AWAY__SHIFT                0x14
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D7_FLIP_AWAY__SHIFT                0x18
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D8_FLIP_AWAY__SHIFT                0x1c
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D1_FLIP_AWAY_MASK                  0x00000007L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D2_FLIP_AWAY_MASK                  0x00000070L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D3_FLIP_AWAY_MASK                  0x00000700L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D4_FLIP_AWAY_MASK                  0x00007000L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D5_FLIP_AWAY_MASK                  0x00070000L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D6_FLIP_AWAY_MASK                  0x00700000L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D7_FLIP_AWAY_MASK                  0x07000000L
+#define DC_GPU_TIMER_START_POSITION_FLIP_AWAY__DC_GPU_TIMER_START_POSITION_D8_FLIP_AWAY_MASK                  0x70000000L
+//DISP_INTERRUPT_STATUS_CONTINUE23
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN16_POWER_UP_INTERRUPT__SHIFT                         0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN17_POWER_UP_INTERRUPT__SHIFT                         0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN18_POWER_UP_INTERRUPT__SHIFT                         0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN19_POWER_UP_INTERRUPT__SHIFT                         0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN20_POWER_UP_INTERRUPT__SHIFT                         0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN21_POWER_UP_INTERRUPT__SHIFT                         0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN22_POWER_UP_INTERRUPT__SHIFT                         0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN23_POWER_UP_INTERRUPT__SHIFT                         0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN24_POWER_UP_INTERRUPT__SHIFT                         0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN25_POWER_UP_INTERRUPT__SHIFT                         0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN26_POWER_UP_INTERRUPT__SHIFT                         0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN16_POWER_DOWN_INTERRUPT__SHIFT                       0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN17_POWER_DOWN_INTERRUPT__SHIFT                       0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN18_POWER_DOWN_INTERRUPT__SHIFT                       0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN19_POWER_DOWN_INTERRUPT__SHIFT                       0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN20_POWER_DOWN_INTERRUPT__SHIFT                       0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN21_POWER_DOWN_INTERRUPT__SHIFT                       0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN22_POWER_DOWN_INTERRUPT__SHIFT                       0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN23_POWER_DOWN_INTERRUPT__SHIFT                       0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN24_POWER_DOWN_INTERRUPT__SHIFT                       0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN25_POWER_DOWN_INTERRUPT__SHIFT                       0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN26_POWER_DOWN_INTERRUPT__SHIFT                       0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DISP_INTERRUPT_STATUS_CONTINUE24__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN16_POWER_UP_INTERRUPT_MASK                           0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN17_POWER_UP_INTERRUPT_MASK                           0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN18_POWER_UP_INTERRUPT_MASK                           0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN19_POWER_UP_INTERRUPT_MASK                           0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN20_POWER_UP_INTERRUPT_MASK                           0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN21_POWER_UP_INTERRUPT_MASK                           0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN22_POWER_UP_INTERRUPT_MASK                           0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN23_POWER_UP_INTERRUPT_MASK                           0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN24_POWER_UP_INTERRUPT_MASK                           0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN25_POWER_UP_INTERRUPT_MASK                           0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN26_POWER_UP_INTERRUPT_MASK                           0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN16_POWER_DOWN_INTERRUPT_MASK                         0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN17_POWER_DOWN_INTERRUPT_MASK                         0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN18_POWER_DOWN_INTERRUPT_MASK                         0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN19_POWER_DOWN_INTERRUPT_MASK                         0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN20_POWER_DOWN_INTERRUPT_MASK                         0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN21_POWER_DOWN_INTERRUPT_MASK                         0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN22_POWER_DOWN_INTERRUPT_MASK                         0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN23_POWER_DOWN_INTERRUPT_MASK                         0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN24_POWER_DOWN_INTERRUPT_MASK                         0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN25_POWER_DOWN_INTERRUPT_MASK                         0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DCPG_IHC_DOMAIN26_POWER_DOWN_INTERRUPT_MASK                         0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE23__DISP_INTERRUPT_STATUS_CONTINUE24_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE24
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC0_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x0
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC0_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x1
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC1_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x2
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC1_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x3
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC2_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC2_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x5
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC3_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC3_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC4_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC4_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC5_PERFMON_COUNTER0_INTERRUPT__SHIFT                              0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC5_PERFMON_COUNTER1_INTERRUPT__SHIFT                              0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_TIMER_HIGH_PRIORITY_INTERRUPT__SHIFT                          0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_TIMER_LOW_PRIORITY_INTERRUPT__SHIFT                           0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_HIGH_PRIORITY_READY_INTERRUPT__SHIFT                    0xe
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_HIGH_PRIORITY_DONE_INTERRUPT__SHIFT                     0xf
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_LOW_PRIORITY_READY_INTERRUPT__SHIFT                     0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_LOW_PRIORITY_DONE_INTERRUPT__SHIFT                      0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INTERRUPT__SHIFT                   0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_HIGH_PRIORITY_DONE_INTERRUPT__SHIFT                    0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_LOW_PRIORITY_READY_INTERRUPT__SHIFT                    0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_LOW_PRIORITY_DONE_INTERRUPT__SHIFT                     0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN0_INTERRUPT__SHIFT                              0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN1_INTERRUPT__SHIFT                              0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN2_INTERRUPT__SHIFT                              0x18
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN3_INTERRUPT__SHIFT                              0x19
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN4_INTERRUPT__SHIFT                              0x1a
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN5_INTERRUPT__SHIFT                              0x1b
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN6_INTERRUPT__SHIFT                              0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAOUT_INTERRUPT__SHIFT                              0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_UNDEFINED_ADDRESS_FAULT_INTERRUPT__SHIFT                      0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DISP_INTERRUPT_STATUS_CONTINUE25__SHIFT                             0x1f
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC0_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC0_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000002L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC1_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC1_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC2_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC2_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC3_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC3_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC4_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC4_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC5_PERFMON_COUNTER0_INTERRUPT_MASK                                0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DSC5_PERFMON_COUNTER1_INTERRUPT_MASK                                0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_TIMER_HIGH_PRIORITY_INTERRUPT_MASK                            0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_TIMER_LOW_PRIORITY_INTERRUPT_MASK                             0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_HIGH_PRIORITY_READY_INTERRUPT_MASK                      0x00004000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_HIGH_PRIORITY_DONE_INTERRUPT_MASK                       0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_LOW_PRIORITY_READY_INTERRUPT_MASK                       0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_INBOX_LOW_PRIORITY_DONE_INTERRUPT_MASK                        0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INTERRUPT_MASK                     0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_HIGH_PRIORITY_DONE_INTERRUPT_MASK                      0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_LOW_PRIORITY_READY_INTERRUPT_MASK                      0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_OUTBOX_LOW_PRIORITY_DONE_INTERRUPT_MASK                       0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN0_INTERRUPT_MASK                                0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN1_INTERRUPT_MASK                                0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN2_INTERRUPT_MASK                                0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN3_INTERRUPT_MASK                                0x02000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN4_INTERRUPT_MASK                                0x04000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN5_INTERRUPT_MASK                                0x08000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAIN6_INTERRUPT_MASK                                0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_GENERAL_DATAOUT_INTERRUPT_MASK                                0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DMCUB_UNDEFINED_ADDRESS_FAULT_INTERRUPT_MASK                        0x40000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE24__DISP_INTERRUPT_STATUS_CONTINUE25_MASK                               0x80000000L
+//DISP_INTERRUPT_STATUS_CONTINUE25
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC0_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0x6
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC1_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0x7
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC2_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC3_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0x9
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC4_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0xa
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC5_IHC_CORE_ERROR_INTERRUPT__SHIFT                                0xb
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DPIA_INTERRUPT__SHIFT                                               0xc
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_WHITELIST_INVALID_ACCESS_INTERRUPT__SHIFT                     0xd
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX0_RDY_INTERRUPT__SHIFT                               0x10
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX1_RDY_INTERRUPT__SHIFT                               0x11
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX2_RDY_INTERRUPT__SHIFT                               0x12
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX3_RDY_INTERRUPT__SHIFT                               0x13
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX4_RDY_INTERRUPT__SHIFT                               0x14
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_OUTBOX0_RSP_INTERRUPT__SHIFT                              0x15
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_HOST_REG_INBOX0_RSP_INTERRUPT__SHIFT                          0x16
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_HOST_REG_OUTBOX0_RDY_INTERRUPT__SHIFT                         0x17
+#define DISP_INTERRUPT_STATUS_CONTINUE25__HPO_PERFMON_COUNTER0_INTERRUPT__SHIFT                               0x1c
+#define DISP_INTERRUPT_STATUS_CONTINUE25__HPO_PERFMON_COUNTER1_INTERRUPT__SHIFT                               0x1d
+#define DISP_INTERRUPT_STATUS_CONTINUE25__MMHUBBUB_WARMUP_INTERRUPT__SHIFT                                    0x1e
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC0_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC1_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC2_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC3_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC4_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DSC5_IHC_CORE_ERROR_INTERRUPT_MASK                                  0x00000800L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DPIA_INTERRUPT_MASK                                                 0x00001000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_WHITELIST_INVALID_ACCESS_INTERRUPT_MASK                       0x00002000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX0_RDY_INTERRUPT_MASK                                 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX1_RDY_INTERRUPT_MASK                                 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX2_RDY_INTERRUPT_MASK                                 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX3_RDY_INTERRUPT_MASK                                 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_INBOX4_RDY_INTERRUPT_MASK                                 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_REG_OUTBOX0_RSP_INTERRUPT_MASK                                0x00200000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_HOST_REG_INBOX0_RSP_INTERRUPT_MASK                            0x00400000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__DMCUB_HOST_REG_OUTBOX0_RDY_INTERRUPT_MASK                           0x00800000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__HPO_PERFMON_COUNTER0_INTERRUPT_MASK                                 0x10000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__HPO_PERFMON_COUNTER1_INTERRUPT_MASK                                 0x20000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE25__MMHUBBUB_WARMUP_INTERRUPT_MASK                                      0x40000000L
+//DCCG_INTERRUPT_DEST
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG0_LATCH_INT_DEST__SHIFT                                        0x0
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG1_LATCH_INT_DEST__SHIFT                                        0x1
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG2_LATCH_INT_DEST__SHIFT                                        0x2
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG3_LATCH_INT_DEST__SHIFT                                        0x3
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG4_LATCH_INT_DEST__SHIFT                                        0x4
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG5_LATCH_INT_DEST__SHIFT                                        0x5
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                  0xc
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                  0xd
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON2_COUNTER0_INTERRUPT_DEST__SHIFT                                 0xe
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON2_COUNTER1_INTERRUPT_DEST__SHIFT                                 0xf
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG0_LATCH_INT_DEST_MASK                                          0x00000001L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG1_LATCH_INT_DEST_MASK                                          0x00000002L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG2_LATCH_INT_DEST_MASK                                          0x00000004L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG3_LATCH_INT_DEST_MASK                                          0x00000008L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG4_LATCH_INT_DEST_MASK                                          0x00000010L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_VSYNC_OTG5_LATCH_INT_DEST_MASK                                          0x00000020L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                    0x00001000L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                    0x00002000L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON2_COUNTER0_INTERRUPT_DEST_MASK                                   0x00004000L
+#define DCCG_INTERRUPT_DEST__DCCG_IHC_PERFMON2_COUNTER1_INTERRUPT_DEST_MASK                                   0x00008000L
+//DMU_INTERRUPT_DEST
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_TIMER0_INT_DEST__SHIFT                                                  0x0
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_TIMER1_INT_DEST__SHIFT                                                  0x1
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT0_INT_DEST__SHIFT                                                  0x2
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT1_INT_DEST__SHIFT                                                  0x3
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT2_INT_DEST__SHIFT                                                  0x4
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT3_INT_DEST__SHIFT                                                  0x5
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT4_INT_DEST__SHIFT                                                  0x6
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT5_INT_DEST__SHIFT                                                  0x7
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT6_INT_DEST__SHIFT                                                  0x8
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT_IH_INT_DEST__SHIFT                                                0x9
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX0_READY_INT_DEST__SHIFT                                            0xa
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX0_DONE_INT_DEST__SHIFT                                             0xb
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX1_READY_INT_DEST__SHIFT                                            0xc
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX1_DONE_INT_DEST__SHIFT                                             0xd
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX0_READY_INT_DEST__SHIFT                                           0xe
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX0_DONE_INT_DEST__SHIFT                                            0xf
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX1_READY_INT_DEST__SHIFT                                           0x10
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX1_DONE_INT_DEST__SHIFT                                            0x11
+#define DMU_INTERRUPT_DEST__DMU_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                    0x12
+#define DMU_INTERRUPT_DEST__DMU_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                    0x13
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_UNDEFINED_ADDRESS_FAULT_INT_DEST__SHIFT                                 0x1a
+#define DMU_INTERRUPT_DEST__RBBMIF_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                          0x1b
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_TIMER0_INT_DEST_MASK                                                    0x00000001L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_TIMER1_INT_DEST_MASK                                                    0x00000002L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT0_INT_DEST_MASK                                                    0x00000004L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT1_INT_DEST_MASK                                                    0x00000008L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT2_INT_DEST_MASK                                                    0x00000010L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT3_INT_DEST_MASK                                                    0x00000020L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT4_INT_DEST_MASK                                                    0x00000040L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT5_INT_DEST_MASK                                                    0x00000080L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT6_INT_DEST_MASK                                                    0x00000100L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_GPINT_IH_INT_DEST_MASK                                                  0x00000200L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX0_READY_INT_DEST_MASK                                              0x00000400L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX0_DONE_INT_DEST_MASK                                               0x00000800L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX1_READY_INT_DEST_MASK                                              0x00001000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_INBOX1_DONE_INT_DEST_MASK                                               0x00002000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX0_READY_INT_DEST_MASK                                             0x00004000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX0_DONE_INT_DEST_MASK                                              0x00008000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX1_READY_INT_DEST_MASK                                             0x00010000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_OUTBOX1_DONE_INT_DEST_MASK                                              0x00020000L
+#define DMU_INTERRUPT_DEST__DMU_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                      0x00040000L
+#define DMU_INTERRUPT_DEST__DMU_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                      0x00080000L
+#define DMU_INTERRUPT_DEST__DMCUB_IHC_UNDEFINED_ADDRESS_FAULT_INT_DEST_MASK                                   0x04000000L
+#define DMU_INTERRUPT_DEST__RBBMIF_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                            0x08000000L
+//DMU_INTERRUPT_DEST2
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX0_RDY_INT_DEST__SHIFT                                         0x0
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX1_RDY_INT_DEST__SHIFT                                         0x1
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX2_RDY_INT_DEST__SHIFT                                         0x2
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX3_RDY_INT_DEST__SHIFT                                         0x3
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX4_RDY_INT_DEST__SHIFT                                         0x4
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_OUTBOX0_RSP_INT_DEST__SHIFT                                        0x5
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_HOST_REG_INBOX0_RSP_INT_DEST__SHIFT                                    0x6
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_HOST_REG_OUTBOX0_RDY_INT_DEST__SHIFT                                   0x7
+#define DMU_INTERRUPT_DEST2__DPIA_IHC_INTERRUPT_DEST__SHIFT                                                   0xc
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_WHITELIST_INVALID_ACCESS_INTERRUPT_DEST__SHIFT                         0xd
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX0_RDY_INT_DEST_MASK                                           0x00000001L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX1_RDY_INT_DEST_MASK                                           0x00000002L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX2_RDY_INT_DEST_MASK                                           0x00000004L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX3_RDY_INT_DEST_MASK                                           0x00000008L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_INBOX4_RDY_INT_DEST_MASK                                           0x00000010L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_REG_OUTBOX0_RSP_INT_DEST_MASK                                          0x00000020L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_HOST_REG_INBOX0_RSP_INT_DEST_MASK                                      0x00000040L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_HOST_REG_OUTBOX0_RDY_INT_DEST_MASK                                     0x00000080L
+#define DMU_INTERRUPT_DEST2__DPIA_IHC_INTERRUPT_DEST_MASK                                                     0x00001000L
+#define DMU_INTERRUPT_DEST2__DMCUB_IHC_WHITELIST_INVALID_ACCESS_INTERRUPT_DEST_MASK                           0x00002000L
+//DCPG_INTERRUPT_DEST
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN0_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x0
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN1_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x1
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN2_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x2
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN3_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x3
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN4_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x4
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN5_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x5
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN6_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x6
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN7_POWER_UP_INTERRUPT_DEST__SHIFT                                  0x7
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN0_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x10
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN1_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x11
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN2_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x12
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN3_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x13
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN4_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x14
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN5_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x15
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN6_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x16
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN7_POWER_DOWN_INTERRUPT_DEST__SHIFT                                0x17
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN0_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000001L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN1_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000002L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN2_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000004L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN3_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000008L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN4_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000010L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN5_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000020L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN6_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000040L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN7_POWER_UP_INTERRUPT_DEST_MASK                                    0x00000080L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN0_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00010000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN1_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00020000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN2_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00040000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN3_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00080000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN4_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00100000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN5_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00200000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN6_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00400000L
+#define DCPG_INTERRUPT_DEST__DCPG_IHC_DOMAIN7_POWER_DOWN_INTERRUPT_DEST_MASK                                  0x00800000L
+//DCPG_INTERRUPT_DEST2
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN16_POWER_UP_INTERRUPT_DEST__SHIFT                                0x0
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN17_POWER_UP_INTERRUPT_DEST__SHIFT                                0x1
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN18_POWER_UP_INTERRUPT_DEST__SHIFT                                0x2
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN19_POWER_UP_INTERRUPT_DEST__SHIFT                                0x3
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN20_POWER_UP_INTERRUPT_DEST__SHIFT                                0x4
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN21_POWER_UP_INTERRUPT_DEST__SHIFT                                0x5
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN22_POWER_UP_INTERRUPT_DEST__SHIFT                                0x6
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN23_POWER_UP_INTERRUPT_DEST__SHIFT                                0x7
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN24_POWER_UP_INTERRUPT_DEST__SHIFT                                0x8
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN25_POWER_UP_INTERRUPT_DEST__SHIFT                                0x9
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN26_POWER_UP_INTERRUPT_DEST__SHIFT                                0xa
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN16_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x10
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN17_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x11
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN18_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x12
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN19_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x13
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN20_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x14
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN21_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x15
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN22_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x16
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN23_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x17
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN24_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x18
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN25_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x19
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN26_POWER_DOWN_INTERRUPT_DEST__SHIFT                              0x1a
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN16_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000001L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN17_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000002L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN18_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000004L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN19_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000008L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN20_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000010L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN21_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000020L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN22_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000040L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN23_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000080L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN24_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000100L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN25_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000200L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN26_POWER_UP_INTERRUPT_DEST_MASK                                  0x00000400L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN16_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00010000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN17_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00020000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN18_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00040000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN19_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00080000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN20_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00100000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN21_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00200000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN22_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00400000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN23_POWER_DOWN_INTERRUPT_DEST_MASK                                0x00800000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN24_POWER_DOWN_INTERRUPT_DEST_MASK                                0x01000000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN25_POWER_DOWN_INTERRUPT_DEST_MASK                                0x02000000L
+#define DCPG_INTERRUPT_DEST2__DCPG_IHC_DOMAIN26_POWER_DOWN_INTERRUPT_DEST_MASK                                0x04000000L
+//MMHUBBUB_INTERRUPT_DEST
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_CWB0_IHIF_INTERRUPT_DEST__SHIFT                                       0x1
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_CWB1_IHIF_INTERRUPT_DEST__SHIFT                                       0x2
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB0_IHIF_INTERRUPT_DEST__SHIFT                                       0x3
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB1_IHIF_INTERRUPT_DEST__SHIFT                                       0x4
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB2_IHIF_INTERRUPT_DEST__SHIFT                                       0x5
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_WARMUP_INTERRUPT_DEST__SHIFT                                    0x8
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                          0xc
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                          0xd
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_CWB0_IHIF_INTERRUPT_DEST_MASK                                         0x00000002L
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_CWB1_IHIF_INTERRUPT_DEST_MASK                                         0x00000004L
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB0_IHIF_INTERRUPT_DEST_MASK                                         0x00000008L
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB1_IHIF_INTERRUPT_DEST_MASK                                         0x00000010L
+#define MMHUBBUB_INTERRUPT_DEST__BUFMGR_DWB2_IHIF_INTERRUPT_DEST_MASK                                         0x00000020L
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_WARMUP_INTERRUPT_DEST_MASK                                      0x00000100L
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                            0x00001000L
+#define MMHUBBUB_INTERRUPT_DEST__MMHUBBUB_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                            0x00002000L
+//WB_INTERRUPT_DEST
+#define WB_INTERRUPT_DEST__WBSCL0_IHIF_DATA_OVERFLOW_INTERRUPT_DEST__SHIFT                                    0x1
+#define WB_INTERRUPT_DEST__WBSCL1_IHIF_DATA_OVERFLOW_INTERRUPT_DEST__SHIFT                                    0x9
+#define WB_INTERRUPT_DEST__WBSCL2_IHIF_DATA_OVERFLOW_INTERRUPT_DEST__SHIFT                                    0xb
+#define WB_INTERRUPT_DEST__WB0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                     0xc
+#define WB_INTERRUPT_DEST__WB0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                     0xd
+#define WB_INTERRUPT_DEST__WB1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                     0xe
+#define WB_INTERRUPT_DEST__WB1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                     0xf
+#define WB_INTERRUPT_DEST__WB2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                     0x10
+#define WB_INTERRUPT_DEST__WB2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                     0x11
+#define WB_INTERRUPT_DEST__WBSCL0_IHIF_DATA_OVERFLOW_INTERRUPT_DEST_MASK                                      0x00000002L
+#define WB_INTERRUPT_DEST__WBSCL1_IHIF_DATA_OVERFLOW_INTERRUPT_DEST_MASK                                      0x00000200L
+#define WB_INTERRUPT_DEST__WBSCL2_IHIF_DATA_OVERFLOW_INTERRUPT_DEST_MASK                                      0x00000800L
+#define WB_INTERRUPT_DEST__WB0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                       0x00001000L
+#define WB_INTERRUPT_DEST__WB0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                       0x00002000L
+#define WB_INTERRUPT_DEST__WB1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                       0x00004000L
+#define WB_INTERRUPT_DEST__WB1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                       0x00008000L
+#define WB_INTERRUPT_DEST__WB2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                       0x00010000L
+#define WB_INTERRUPT_DEST__WB2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                       0x00020000L
+//DCHUB_INTERRUPT_DEST
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x0
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x1
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x2
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x3
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x4
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x5
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x6
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x7
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x8
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x9
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0xa
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0xb
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0xc
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0xd
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0xe
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0xf
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x10
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x11
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x12
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x13
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x14
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x15
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x16
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x17
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x18
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x19
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x1a
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x1b
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VBLANK_INTERRUPT_DEST__SHIFT                                          0x1c
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VLINE_INTERRUPT_DEST__SHIFT                                           0x1d
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VLINE2_INTERRUPT_DEST__SHIFT                                          0x1e
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                         0x1f
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00000001L
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00000002L
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00000004L
+#define DCHUB_INTERRUPT_DEST__HUBP0_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00000008L
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00000010L
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00000020L
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00000040L
+#define DCHUB_INTERRUPT_DEST__HUBP1_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00000080L
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00000100L
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00000200L
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00000400L
+#define DCHUB_INTERRUPT_DEST__HUBP2_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00000800L
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00001000L
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00002000L
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00004000L
+#define DCHUB_INTERRUPT_DEST__HUBP3_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00008000L
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00010000L
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00020000L
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00040000L
+#define DCHUB_INTERRUPT_DEST__HUBP4_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00080000L
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x00100000L
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x00200000L
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x00400000L
+#define DCHUB_INTERRUPT_DEST__HUBP5_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x00800000L
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x01000000L
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x02000000L
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x04000000L
+#define DCHUB_INTERRUPT_DEST__HUBP6_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x08000000L
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VBLANK_INTERRUPT_DEST_MASK                                            0x10000000L
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VLINE_INTERRUPT_DEST_MASK                                             0x20000000L
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_VLINE2_INTERRUPT_DEST_MASK                                            0x40000000L
+#define DCHUB_INTERRUPT_DEST__HUBP7_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                           0x80000000L
+//DCHUB_PERFCOUNTER_INTERRUPT_DEST
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBBUB_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                   0xc
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBBUB_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                   0xd
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0xe
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0xf
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x10
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x11
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x12
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x13
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x14
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x15
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x16
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x17
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x18
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x19
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP6_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x1a
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP6_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x1b
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP7_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                    0x1c
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP7_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                    0x1d
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBBUB_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                     0x00001000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBBUB_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                     0x00002000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x00004000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x00008000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x00010000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x00020000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x00040000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x00080000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x00100000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x00200000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x00400000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x00800000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x01000000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x02000000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP6_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x04000000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP6_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x08000000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP7_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                      0x10000000L
+#define DCHUB_PERFCOUNTER_INTERRUPT_DEST__HUBP7_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                      0x20000000L
+//DCHUB_INTERRUPT_DEST2
+#define DCHUB_INTERRUPT_DEST2__HUBP0_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0x0
+#define DCHUB_INTERRUPT_DEST2__HUBP0_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0x1
+#define DCHUB_INTERRUPT_DEST2__HUBP1_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0x2
+#define DCHUB_INTERRUPT_DEST2__HUBP1_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0x3
+#define DCHUB_INTERRUPT_DEST2__HUBP2_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0x4
+#define DCHUB_INTERRUPT_DEST2__HUBP2_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0x5
+#define DCHUB_INTERRUPT_DEST2__HUBP3_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0x6
+#define DCHUB_INTERRUPT_DEST2__HUBP3_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0x7
+#define DCHUB_INTERRUPT_DEST2__HUBP4_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0x8
+#define DCHUB_INTERRUPT_DEST2__HUBP4_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0x9
+#define DCHUB_INTERRUPT_DEST2__HUBP5_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0xa
+#define DCHUB_INTERRUPT_DEST2__HUBP5_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0xb
+#define DCHUB_INTERRUPT_DEST2__HUBP6_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0xc
+#define DCHUB_INTERRUPT_DEST2__HUBP6_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0xd
+#define DCHUB_INTERRUPT_DEST2__HUBP7_IHC_FLIP_INTERRUPT_DEST__SHIFT                                           0xe
+#define DCHUB_INTERRUPT_DEST2__HUBP7_IHC_FLIP_AWAY_INTERRUPT_DEST__SHIFT                                      0xf
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_VM_FAULT_INTERRUPT_DEST__SHIFT                                      0x18
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_TIMEOUT_INTERRUPT_DEST__SHIFT                                       0x19
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_COMPBUF_SIZE_CHANGE_INTERRUPT_DEST__SHIFT                           0x1a
+#define DCHUB_INTERRUPT_DEST2__HUBP0_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000001L
+#define DCHUB_INTERRUPT_DEST2__HUBP0_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000002L
+#define DCHUB_INTERRUPT_DEST2__HUBP1_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000004L
+#define DCHUB_INTERRUPT_DEST2__HUBP1_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000008L
+#define DCHUB_INTERRUPT_DEST2__HUBP2_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000010L
+#define DCHUB_INTERRUPT_DEST2__HUBP2_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000020L
+#define DCHUB_INTERRUPT_DEST2__HUBP3_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000040L
+#define DCHUB_INTERRUPT_DEST2__HUBP3_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000080L
+#define DCHUB_INTERRUPT_DEST2__HUBP4_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000100L
+#define DCHUB_INTERRUPT_DEST2__HUBP4_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000200L
+#define DCHUB_INTERRUPT_DEST2__HUBP5_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00000400L
+#define DCHUB_INTERRUPT_DEST2__HUBP5_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00000800L
+#define DCHUB_INTERRUPT_DEST2__HUBP6_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00001000L
+#define DCHUB_INTERRUPT_DEST2__HUBP6_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00002000L
+#define DCHUB_INTERRUPT_DEST2__HUBP7_IHC_FLIP_INTERRUPT_DEST_MASK                                             0x00004000L
+#define DCHUB_INTERRUPT_DEST2__HUBP7_IHC_FLIP_AWAY_INTERRUPT_DEST_MASK                                        0x00008000L
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_VM_FAULT_INTERRUPT_DEST_MASK                                        0x01000000L
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_TIMEOUT_INTERRUPT_DEST_MASK                                         0x02000000L
+#define DCHUB_INTERRUPT_DEST2__HUBBUB_IHC_COMPBUF_SIZE_CHANGE_INTERRUPT_DEST_MASK                             0x04000000L
+//DPP_PERFCOUNTER_INTERRUPT_DEST
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0xc
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0xd
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0xe
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0xf
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x10
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x11
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x12
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x13
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x14
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x15
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x16
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x17
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP6_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x18
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP6_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x19
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP7_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                       0x1a
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP7_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                       0x1b
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00001000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00002000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00004000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00008000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00010000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00020000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00040000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00080000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00100000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00200000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x00400000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x00800000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP6_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x01000000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP6_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x02000000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP7_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                         0x04000000L
+#define DPP_PERFCOUNTER_INTERRUPT_DEST__DPP7_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                         0x08000000L
+//MPC_INTERRUPT_DEST
+#define MPC_INTERRUPT_DEST__MPCC0_STALL_INTERRUPT_DEST__SHIFT                                                 0x0
+#define MPC_INTERRUPT_DEST__MPCC1_STALL_INTERRUPT_DEST__SHIFT                                                 0x1
+#define MPC_INTERRUPT_DEST__MPCC2_STALL_INTERRUPT_DEST__SHIFT                                                 0x2
+#define MPC_INTERRUPT_DEST__MPCC3_STALL_INTERRUPT_DEST__SHIFT                                                 0x3
+#define MPC_INTERRUPT_DEST__MPCC4_STALL_INTERRUPT_DEST__SHIFT                                                 0x4
+#define MPC_INTERRUPT_DEST__MPCC5_STALL_INTERRUPT_DEST__SHIFT                                                 0x5
+#define MPC_INTERRUPT_DEST__MPCC6_STALL_INTERRUPT_DEST__SHIFT                                                 0x6
+#define MPC_INTERRUPT_DEST__MPCC7_STALL_INTERRUPT_DEST__SHIFT                                                 0x7
+#define MPC_INTERRUPT_DEST__MPC_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                    0xc
+#define MPC_INTERRUPT_DEST__MPC_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                    0xd
+#define MPC_INTERRUPT_DEST__MPCC0_STALL_INTERRUPT_DEST_MASK                                                   0x00000001L
+#define MPC_INTERRUPT_DEST__MPCC1_STALL_INTERRUPT_DEST_MASK                                                   0x00000002L
+#define MPC_INTERRUPT_DEST__MPCC2_STALL_INTERRUPT_DEST_MASK                                                   0x00000004L
+#define MPC_INTERRUPT_DEST__MPCC3_STALL_INTERRUPT_DEST_MASK                                                   0x00000008L
+#define MPC_INTERRUPT_DEST__MPCC4_STALL_INTERRUPT_DEST_MASK                                                   0x00000010L
+#define MPC_INTERRUPT_DEST__MPCC5_STALL_INTERRUPT_DEST_MASK                                                   0x00000020L
+#define MPC_INTERRUPT_DEST__MPCC6_STALL_INTERRUPT_DEST_MASK                                                   0x00000040L
+#define MPC_INTERRUPT_DEST__MPCC7_STALL_INTERRUPT_DEST_MASK                                                   0x00000080L
+#define MPC_INTERRUPT_DEST__MPC_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                      0x00001000L
+#define MPC_INTERRUPT_DEST__MPC_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                      0x00002000L
+//OPP_INTERRUPT_DEST
+#define OPP_INTERRUPT_DEST__OPP_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                    0xc
+#define OPP_INTERRUPT_DEST__OPP_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                    0xd
+#define OPP_INTERRUPT_DEST__OPP_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                      0x00001000L
+#define OPP_INTERRUPT_DEST__OPP_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                      0x00002000L
+//OPTC_INTERRUPT_DEST
+#define OPTC_INTERRUPT_DEST__OPTC_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                  0xc
+#define OPTC_INTERRUPT_DEST__OPTC_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                  0xd
+#define OPTC_INTERRUPT_DEST__OPTC0_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x18
+#define OPTC_INTERRUPT_DEST__OPTC1_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x19
+#define OPTC_INTERRUPT_DEST__OPTC2_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x1a
+#define OPTC_INTERRUPT_DEST__OPTC3_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x1b
+#define OPTC_INTERRUPT_DEST__OPTC4_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x1c
+#define OPTC_INTERRUPT_DEST__OPTC5_IHC_DATA_UNDERFLOW_INTERRUPT_DEST__SHIFT                                   0x1d
+#define OPTC_INTERRUPT_DEST__OPTC_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                    0x00001000L
+#define OPTC_INTERRUPT_DEST__OPTC_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                    0x00002000L
+#define OPTC_INTERRUPT_DEST__OPTC0_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x01000000L
+#define OPTC_INTERRUPT_DEST__OPTC1_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x02000000L
+#define OPTC_INTERRUPT_DEST__OPTC2_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x04000000L
+#define OPTC_INTERRUPT_DEST__OPTC3_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x08000000L
+#define OPTC_INTERRUPT_DEST__OPTC4_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x10000000L
+#define OPTC_INTERRUPT_DEST__OPTC5_IHC_DATA_UNDERFLOW_INTERRUPT_DEST_MASK                                     0x20000000L
+//OTG0_INTERRUPT_DEST
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG0_INTERRUPT_DEST__OTG0_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG0_INTERRUPT_DEST__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG0_INTERRUPT_DEST__OTG0_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//OTG1_INTERRUPT_DEST
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG1_INTERRUPT_DEST__OTG1_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG1_INTERRUPT_DEST__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG1_INTERRUPT_DEST__OTG1_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//OTG2_INTERRUPT_DEST
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG2_INTERRUPT_DEST__OTG2_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG2_INTERRUPT_DEST__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG2_INTERRUPT_DEST__OTG2_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//OTG3_INTERRUPT_DEST
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG3_INTERRUPT_DEST__OTG3_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG3_INTERRUPT_DEST__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG3_INTERRUPT_DEST__OTG3_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//OTG4_INTERRUPT_DEST
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG4_INTERRUPT_DEST__OTG4_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG4_INTERRUPT_DEST__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG4_INTERRUPT_DEST__OTG4_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//OTG5_INTERRUPT_DEST
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_CPU_SS_INTERRUPT_DEST__SHIFT                                            0x0
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_DRR_TIMING_INTERRUPT_DEST__SHIFT                                        0x1
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_V_UPDATE_INTERRUPT_DEST__SHIFT                                          0x2
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_SNAPSHOT_INTERRUPT_DEST__SHIFT                                          0x3
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST__SHIFT                               0x4
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST__SHIFT                             0x5
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_TRIGA_INTERRUPT_DEST__SHIFT                                         0x6
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_TRIGB_INTERRUPT_DEST__SHIFT                                         0x7
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST__SHIFT                                     0x8
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT0_DEST__SHIFT                                     0x9
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT1_DEST__SHIFT                                     0xa
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT2_DEST__SHIFT                                     0xb
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST__SHIFT                     0xf
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VSTARTUP_INTERRUPT_DEST__SHIFT                                          0x10
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VREADY_INTERRUPT_DEST__SHIFT                                            0x11
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VSYNC_NOM_INTERRUPT_DEST__SHIFT                                         0x12
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST__SHIFT                                  0x13
+#define OTG5_INTERRUPT_DEST__OTG5_DRR_V_TOTAL_REACH_INTERRUPT_DEST__SHIFT                                     0x14
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_CPU_SS_INTERRUPT_DEST_MASK                                              0x00000001L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_DRR_TIMING_INTERRUPT_DEST_MASK                                          0x00000002L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_V_UPDATE_INTERRUPT_DEST_MASK                                            0x00000004L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_SNAPSHOT_INTERRUPT_DEST_MASK                                            0x00000008L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_FORCE_COUNT_NOW_INTERRUPT_DEST_MASK                                 0x00000010L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_FORCE_VSYNC_NEXT_LINE_INTERRUPT_DEST_MASK                               0x00000020L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_TRIGA_INTERRUPT_DEST_MASK                                           0x00000040L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_TRIGB_INTERRUPT_DEST_MASK                                           0x00000080L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_GSL_VSYNC_GAP_INTERRUPT_DEST_MASK                                       0x00000100L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT0_DEST_MASK                                       0x00000200L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT1_DEST_MASK                                       0x00000400L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_OTG_VERTICAL_INTERRUPT2_DEST_MASK                                       0x00000800L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_SET_V_TOTAL_MIN_EVENT_OCCURED_INTERRUPT_DEST_MASK                       0x00008000L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VSTARTUP_INTERRUPT_DEST_MASK                                            0x00010000L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VREADY_INTERRUPT_DEST_MASK                                              0x00020000L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_VSYNC_NOM_INTERRUPT_DEST_MASK                                           0x00040000L
+#define OTG5_INTERRUPT_DEST__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT_DEST_MASK                                    0x00080000L
+#define OTG5_INTERRUPT_DEST__OTG5_DRR_V_TOTAL_REACH_INTERRUPT_DEST_MASK                                       0x00100000L
+//DIG_INTERRUPT_DEST
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGA_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x0
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGB_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x1
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGC_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x2
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGD_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x3
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGE_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x4
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGF_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x5
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGG_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x6
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGH_VID_STREAM_DISABLE_INTERRUPT_DEST__SHIFT                            0x7
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGA_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0x8
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGB_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0x9
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGC_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xa
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGD_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xb
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGE_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xc
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGF_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xd
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGG_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xe
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGH_FAST_TRAINING_COMPLETE_INTERRUPT_DEST__SHIFT                        0xf
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGA_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000001L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGB_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000002L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGC_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000004L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGD_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000008L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGE_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000010L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGF_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000020L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGG_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000040L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGH_VID_STREAM_DISABLE_INTERRUPT_DEST_MASK                              0x00000080L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGA_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00000100L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGB_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00000200L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGC_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00000400L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGD_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00000800L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGE_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00001000L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGF_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00002000L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGG_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00004000L
+#define DIG_INTERRUPT_DEST__DOUT_IHC_DIGH_FAST_TRAINING_COMPLETE_INTERRUPT_DEST_MASK                          0x00008000L
+//I2C_DDC_HPD_INTERRUPT_DEST
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_SW_DONE_INTERRUPT_DEST__SHIFT                                0x0
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC1_HW_DONE_INTERRUPT_DEST__SHIFT                           0x1
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC2_HW_DONE_INTERRUPT_DEST__SHIFT                           0x2
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC3_HW_DONE_INTERRUPT_DEST__SHIFT                           0x3
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC4_HW_DONE_INTERRUPT_DEST__SHIFT                           0x4
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC5_HW_DONE_INTERRUPT_DEST__SHIFT                           0x5
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC6_HW_DONE_INTERRUPT_DEST__SHIFT                           0x6
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDCVGA_HW_DONE_INTERRUPT_DEST__SHIFT                         0x7
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC1_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x10
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC2_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x11
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC3_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x12
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC4_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x13
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC5_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x14
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC6_READ_REQUEST_INTERRUPT_DEST__SHIFT                            0x15
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDCVGA_READ_REQUEST_INTERRPUT_DEST__SHIFT                          0x16
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_SW_DONE_INTERRUPT_DEST_MASK                                  0x00000001L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC1_HW_DONE_INTERRUPT_DEST_MASK                             0x00000002L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC2_HW_DONE_INTERRUPT_DEST_MASK                             0x00000004L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC3_HW_DONE_INTERRUPT_DEST_MASK                             0x00000008L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC4_HW_DONE_INTERRUPT_DEST_MASK                             0x00000010L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC5_HW_DONE_INTERRUPT_DEST_MASK                             0x00000020L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDC6_HW_DONE_INTERRUPT_DEST_MASK                             0x00000040L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DOUT_IHC_I2C_DDCVGA_HW_DONE_INTERRUPT_DEST_MASK                           0x00000080L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC1_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00010000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC2_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00020000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC3_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00040000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC4_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00080000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC5_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00100000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDC6_READ_REQUEST_INTERRUPT_DEST_MASK                              0x00200000L
+#define I2C_DDC_HPD_INTERRUPT_DEST__DC_I2C_DDCVGA_READ_REQUEST_INTERRPUT_DEST_MASK                            0x00400000L
+//DIO_INTERRUPT_DEST
+#define DIO_INTERRUPT_DEST__DIO_ALPM_INTERRUPT_DEST__SHIFT                                                    0x4
+#define DIO_INTERRUPT_DEST__DIO_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                    0xc
+#define DIO_INTERRUPT_DEST__DIO_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                    0xd
+#define DIO_INTERRUPT_DEST__DIO_ALPM_INTERRUPT_DEST_MASK                                                      0x00000010L
+#define DIO_INTERRUPT_DEST__DIO_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                      0x00001000L
+#define DIO_INTERRUPT_DEST__DIO_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                      0x00002000L
+//DCIO_INTERRUPT_DEST
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXA_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x0
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXB_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x1
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXC_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x2
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXD_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x3
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXE_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x4
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXF_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x5
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXG_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x6
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_RXA_IHC_ERROR_INTERRUPT_DEST__SHIFT                                    0x10
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXA_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000001L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXB_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000002L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXC_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000004L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXD_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000008L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXE_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000010L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXF_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000020L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_TXG_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00000040L
+#define DCIO_INTERRUPT_DEST__DCIO_DPCS_RXA_IHC_ERROR_INTERRUPT_DEST_MASK                                      0x00010000L
+//HPD_INTERRUPT_DEST
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD1_INTERRUPT_DEST__SHIFT                                               0x0
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD2_INTERRUPT_DEST__SHIFT                                               0x1
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD3_INTERRUPT_DEST__SHIFT                                               0x2
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD4_INTERRUPT_DEST__SHIFT                                               0x3
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD5_INTERRUPT_DEST__SHIFT                                               0x4
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD6_INTERRUPT_DEST__SHIFT                                               0x5
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD1_RX_INTERRUPT_DEST__SHIFT                                            0x8
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD2_RX_INTERRUPT_DEST__SHIFT                                            0x9
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD3_RX_INTERRUPT_DEST__SHIFT                                            0xa
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD4_RX_INTERRUPT_DEST__SHIFT                                            0xb
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD5_RX_INTERRUPT_DEST__SHIFT                                            0xc
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD6_RX_INTERRUPT_DEST__SHIFT                                            0xd
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD1_INTERRUPT_DEST_MASK                                                 0x00000001L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD2_INTERRUPT_DEST_MASK                                                 0x00000002L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD3_INTERRUPT_DEST_MASK                                                 0x00000004L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD4_INTERRUPT_DEST_MASK                                                 0x00000008L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD5_INTERRUPT_DEST_MASK                                                 0x00000010L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD6_INTERRUPT_DEST_MASK                                                 0x00000020L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD1_RX_INTERRUPT_DEST_MASK                                              0x00000100L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD2_RX_INTERRUPT_DEST_MASK                                              0x00000200L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD3_RX_INTERRUPT_DEST_MASK                                              0x00000400L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD4_RX_INTERRUPT_DEST_MASK                                              0x00000800L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD5_RX_INTERRUPT_DEST_MASK                                              0x00001000L
+#define HPD_INTERRUPT_DEST__DOUT_IHC_HPD6_RX_INTERRUPT_DEST_MASK                                              0x00002000L
+//AZ_INTERRUPT_DEST
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x0
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x1
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x2
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x3
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x4
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x5
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x6
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_FORMAT_CHANGED_INT_DEST__SHIFT                              0x7
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_ENABLED_INT_DEST__SHIFT                                     0x8
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_ENABLED_INT_DEST__SHIFT                                     0x9
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xa
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xb
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xc
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xd
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xe
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_ENABLED_INT_DEST__SHIFT                                     0xf
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x10
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x11
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x12
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x13
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x14
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x15
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x16
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_DISABLED_INT_DEST__SHIFT                                    0x17
+#define AZ_INTERRUPT_DEST__AZ_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                      0x1e
+#define AZ_INTERRUPT_DEST__AZ_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                      0x1f
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000001L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000002L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000004L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000008L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000010L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000020L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000040L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_FORMAT_CHANGED_INT_DEST_MASK                                0x00000080L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_ENABLED_INT_DEST_MASK                                       0x00000100L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_ENABLED_INT_DEST_MASK                                       0x00000200L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_ENABLED_INT_DEST_MASK                                       0x00000400L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_ENABLED_INT_DEST_MASK                                       0x00000800L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_ENABLED_INT_DEST_MASK                                       0x00001000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_ENABLED_INT_DEST_MASK                                       0x00002000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_ENABLED_INT_DEST_MASK                                       0x00004000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_ENABLED_INT_DEST_MASK                                       0x00008000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT0_AUDIO_DISABLED_INT_DEST_MASK                                      0x00010000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT1_AUDIO_DISABLED_INT_DEST_MASK                                      0x00020000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT2_AUDIO_DISABLED_INT_DEST_MASK                                      0x00040000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT3_AUDIO_DISABLED_INT_DEST_MASK                                      0x00080000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT4_AUDIO_DISABLED_INT_DEST_MASK                                      0x00100000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT5_AUDIO_DISABLED_INT_DEST_MASK                                      0x00200000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT6_AUDIO_DISABLED_INT_DEST_MASK                                      0x00400000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_ENDPOINT7_AUDIO_DISABLED_INT_DEST_MASK                                      0x00800000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                        0x40000000L
+#define AZ_INTERRUPT_DEST__AZ_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                        0x80000000L
+//AUX_INTERRUPT_DEST
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX1_SW_DONE_INTERRUPT_DEST__SHIFT                                       0x0
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX1_LS_DONE_INTERRUPT_DEST__SHIFT                                       0x1
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX2_SW_DONE_INTERRUPT_DEST__SHIFT                                       0x2
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX2_LS_DONE_INTERRUPT_DEST__SHIFT                                       0x3
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX3_SW_DONE_INTERRUPT_DEST__SHIFT                                       0x4
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX3_LS_DONE_INTERRUPT_DEST__SHIFT                                       0x5
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX4_SW_DONE_INTERRUPT_DEST__SHIFT                                       0x6
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX4_LS_DONE_INTERRUPT_DEST__SHIFT                                       0x7
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX5_SW_DONE_INTERRUPT_DEST__SHIFT                                       0x8
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX5_LS_DONE_INTERRUPT_DEST__SHIFT                                       0x9
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX6_SW_DONE_INTERRUPT_DEST__SHIFT                                       0xa
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX6_LS_DONE_INTERRUPT_DEST__SHIFT                                       0xb
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX1_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000001L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX1_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000002L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX2_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000004L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX2_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000008L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX3_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000010L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX3_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000020L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX4_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000040L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX4_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000080L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX5_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000100L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX5_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000200L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX6_SW_DONE_INTERRUPT_DEST_MASK                                         0x00000400L
+#define AUX_INTERRUPT_DEST__DOUT_IHC_AUX6_LS_DONE_INTERRUPT_DEST_MASK                                         0x00000800L
+//DSC_INTERRUPT_DEST
+#define DSC_INTERRUPT_DEST__DSC0_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0x1
+#define DSC_INTERRUPT_DEST__DSC0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0x2
+#define DSC_INTERRUPT_DEST__DSC0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0x3
+#define DSC_INTERRUPT_DEST__DSC1_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0x5
+#define DSC_INTERRUPT_DEST__DSC1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0x6
+#define DSC_INTERRUPT_DEST__DSC1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0x7
+#define DSC_INTERRUPT_DEST__DSC2_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0x9
+#define DSC_INTERRUPT_DEST__DSC2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0xa
+#define DSC_INTERRUPT_DEST__DSC2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0xb
+#define DSC_INTERRUPT_DEST__DSC3_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0xd
+#define DSC_INTERRUPT_DEST__DSC3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0xe
+#define DSC_INTERRUPT_DEST__DSC3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0xf
+#define DSC_INTERRUPT_DEST__DSC4_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0x11
+#define DSC_INTERRUPT_DEST__DSC4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0x12
+#define DSC_INTERRUPT_DEST__DSC4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0x13
+#define DSC_INTERRUPT_DEST__DSC5_IHC_CORE_ERROR_INTERRUPT_DEST__SHIFT                                         0x15
+#define DSC_INTERRUPT_DEST__DSC5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                   0x16
+#define DSC_INTERRUPT_DEST__DSC5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                   0x17
+#define DSC_INTERRUPT_DEST__DSC0_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00000002L
+#define DSC_INTERRUPT_DEST__DSC0_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00000004L
+#define DSC_INTERRUPT_DEST__DSC0_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00000008L
+#define DSC_INTERRUPT_DEST__DSC1_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00000020L
+#define DSC_INTERRUPT_DEST__DSC1_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00000040L
+#define DSC_INTERRUPT_DEST__DSC1_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00000080L
+#define DSC_INTERRUPT_DEST__DSC2_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00000200L
+#define DSC_INTERRUPT_DEST__DSC2_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00000400L
+#define DSC_INTERRUPT_DEST__DSC2_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00000800L
+#define DSC_INTERRUPT_DEST__DSC3_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00002000L
+#define DSC_INTERRUPT_DEST__DSC3_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00004000L
+#define DSC_INTERRUPT_DEST__DSC3_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00008000L
+#define DSC_INTERRUPT_DEST__DSC4_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00020000L
+#define DSC_INTERRUPT_DEST__DSC4_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00040000L
+#define DSC_INTERRUPT_DEST__DSC4_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00080000L
+#define DSC_INTERRUPT_DEST__DSC5_IHC_CORE_ERROR_INTERRUPT_DEST_MASK                                           0x00200000L
+#define DSC_INTERRUPT_DEST__DSC5_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                     0x00400000L
+#define DSC_INTERRUPT_DEST__DSC5_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                     0x00800000L
+//HPO_INTERRUPT_DEST
+#define HPO_INTERRUPT_DEST__HPO_IHC_PERFMON_COUNTER0_INTERRUPT_DEST__SHIFT                                    0x2
+#define HPO_INTERRUPT_DEST__HPO_IHC_PERFMON_COUNTER1_INTERRUPT_DEST__SHIFT                                    0x3
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE0_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x4
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE1_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x5
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE2_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x6
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE3_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x7
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE4_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x8
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE5_ALPM_WAKE_INTERRUPT_DEST__SHIFT                                       0x9
+#define HPO_INTERRUPT_DEST__HPO_IHC_PERFMON_COUNTER0_INTERRUPT_DEST_MASK                                      0x00000004L
+#define HPO_INTERRUPT_DEST__HPO_IHC_PERFMON_COUNTER1_INTERRUPT_DEST_MASK                                      0x00000008L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE0_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000010L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE1_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000020L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE2_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000040L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE3_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000080L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE4_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000100L
+#define HPO_INTERRUPT_DEST__HPO_IHC_SE5_ALPM_WAKE_INTERRUPT_DEST_MASK                                         0x00000200L
+//DPP_INTERRUPT_DEST
+#define DPP_INTERRUPT_DEST__DPP0_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x0
+#define DPP_INTERRUPT_DEST__DPP1_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x1
+#define DPP_INTERRUPT_DEST__DPP2_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x2
+#define DPP_INTERRUPT_DEST__DPP3_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x3
+#define DPP_INTERRUPT_DEST__DPP4_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x4
+#define DPP_INTERRUPT_DEST__DPP5_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x5
+#define DPP_INTERRUPT_DEST__DPP6_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x6
+#define DPP_INTERRUPT_DEST__DPP7_HIST_RDY_IHC_INTERRUPT_DEST__SHIFT                                           0x7
+#define DPP_INTERRUPT_DEST__DPP0_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000001L
+#define DPP_INTERRUPT_DEST__DPP1_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000002L
+#define DPP_INTERRUPT_DEST__DPP2_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000004L
+#define DPP_INTERRUPT_DEST__DPP3_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000008L
+#define DPP_INTERRUPT_DEST__DPP4_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000010L
+#define DPP_INTERRUPT_DEST__DPP5_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000020L
+#define DPP_INTERRUPT_DEST__DPP6_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000040L
+#define DPP_INTERRUPT_DEST__DPP7_HIST_RDY_IHC_INTERRUPT_DEST_MASK                                             0x00000080L
+
+
+// addressBlock: dce_dc_dmu_fgsec_dispdec
+//DMCUB_RBBMIF_SEC_CNTL
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_SEC_LVL__SHIFT                                                    0x0
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_TRUST_LVL__SHIFT                                                  0x4
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_SOURCE_ID__SHIFT                                                  0x8
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_SEC_LVL_MASK                                                      0x00000007L
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_TRUST_LVL_MASK                                                    0x000000F0L
+#define DMCUB_RBBMIF_SEC_CNTL__DMCUB_RBBMIF_SOURCE_ID_MASK                                                    0x01FFFF00L
+
+
+// addressBlock: dce_dc_dmu_rbbmif_dispdec
+//RBBMIF_TIMEOUT
+#define RBBMIF_TIMEOUT__RBBMIF_TIMEOUT_DELAY__SHIFT                                                           0x0
+#define RBBMIF_TIMEOUT__RBBMIF_TIMEOUT_TO_REQ_HOLD__SHIFT                                                     0x14
+#define RBBMIF_TIMEOUT__RBBMIF_TIMEOUT_DELAY_MASK                                                             0x000FFFFFL
+#define RBBMIF_TIMEOUT__RBBMIF_TIMEOUT_TO_REQ_HOLD_MASK                                                       0xFFF00000L
+//RBBMIF_STATUS
+#define RBBMIF_STATUS__RBBMIF_TIMEOUT_CLIENTS_DEC__SHIFT                                                      0x0
+#define RBBMIF_STATUS__RBBMIF_TIMEOUT_CLIENTS_DEC_MASK                                                        0xFFFFFFFFL
+//RBBMIF_STATUS_2
+#define RBBMIF_STATUS_2__RBBMIF_TIMEOUT_CLIENTS_DEC_2__SHIFT                                                  0x0
+#define RBBMIF_STATUS_2__RBBMIF_TIMEOUT_CLIENTS_DEC_2_MASK                                                    0x000000FFL
+//RBBMIF_INT_STATUS
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_ADDR__SHIFT                                                         0x2
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_OP__SHIFT                                                           0x1c
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_RDWR_STATUS__SHIFT                                                  0x1d
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_ACK__SHIFT                                                          0x1e
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_MASK__SHIFT                                                         0x1f
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_ADDR_MASK                                                           0x0003FFFCL
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_OP_MASK                                                             0x10000000L
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_RDWR_STATUS_MASK                                                    0x20000000L
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_ACK_MASK                                                            0x40000000L
+#define RBBMIF_INT_STATUS__RBBMIF_TIMEOUT_MASK_MASK                                                           0x80000000L
+//RBBMIF_TIMEOUT_DIS
+#define RBBMIF_TIMEOUT_DIS__CLIENT0_TIMEOUT_DIS__SHIFT                                                        0x0
+#define RBBMIF_TIMEOUT_DIS__CLIENT1_TIMEOUT_DIS__SHIFT                                                        0x1
+#define RBBMIF_TIMEOUT_DIS__CLIENT2_TIMEOUT_DIS__SHIFT                                                        0x2
+#define RBBMIF_TIMEOUT_DIS__CLIENT3_TIMEOUT_DIS__SHIFT                                                        0x3
+#define RBBMIF_TIMEOUT_DIS__CLIENT4_TIMEOUT_DIS__SHIFT                                                        0x4
+#define RBBMIF_TIMEOUT_DIS__CLIENT5_TIMEOUT_DIS__SHIFT                                                        0x5
+#define RBBMIF_TIMEOUT_DIS__CLIENT6_TIMEOUT_DIS__SHIFT                                                        0x6
+#define RBBMIF_TIMEOUT_DIS__CLIENT7_TIMEOUT_DIS__SHIFT                                                        0x7
+#define RBBMIF_TIMEOUT_DIS__CLIENT8_TIMEOUT_DIS__SHIFT                                                        0x8
+#define RBBMIF_TIMEOUT_DIS__CLIENT9_TIMEOUT_DIS__SHIFT                                                        0x9
+#define RBBMIF_TIMEOUT_DIS__CLIENT10_TIMEOUT_DIS__SHIFT                                                       0xa
+#define RBBMIF_TIMEOUT_DIS__CLIENT11_TIMEOUT_DIS__SHIFT                                                       0xb
+#define RBBMIF_TIMEOUT_DIS__CLIENT12_TIMEOUT_DIS__SHIFT                                                       0xc
+#define RBBMIF_TIMEOUT_DIS__CLIENT13_TIMEOUT_DIS__SHIFT                                                       0xd
+#define RBBMIF_TIMEOUT_DIS__CLIENT14_TIMEOUT_DIS__SHIFT                                                       0xe
+#define RBBMIF_TIMEOUT_DIS__CLIENT15_TIMEOUT_DIS__SHIFT                                                       0xf
+#define RBBMIF_TIMEOUT_DIS__CLIENT16_TIMEOUT_DIS__SHIFT                                                       0x10
+#define RBBMIF_TIMEOUT_DIS__CLIENT17_TIMEOUT_DIS__SHIFT                                                       0x11
+#define RBBMIF_TIMEOUT_DIS__CLIENT18_TIMEOUT_DIS__SHIFT                                                       0x12
+#define RBBMIF_TIMEOUT_DIS__CLIENT19_TIMEOUT_DIS__SHIFT                                                       0x13
+#define RBBMIF_TIMEOUT_DIS__CLIENT20_TIMEOUT_DIS__SHIFT                                                       0x14
+#define RBBMIF_TIMEOUT_DIS__CLIENT21_TIMEOUT_DIS__SHIFT                                                       0x15
+#define RBBMIF_TIMEOUT_DIS__CLIENT22_TIMEOUT_DIS__SHIFT                                                       0x16
+#define RBBMIF_TIMEOUT_DIS__CLIENT23_TIMEOUT_DIS__SHIFT                                                       0x17
+#define RBBMIF_TIMEOUT_DIS__CLIENT24_TIMEOUT_DIS__SHIFT                                                       0x18
+#define RBBMIF_TIMEOUT_DIS__CLIENT25_TIMEOUT_DIS__SHIFT                                                       0x19
+#define RBBMIF_TIMEOUT_DIS__CLIENT26_TIMEOUT_DIS__SHIFT                                                       0x1a
+#define RBBMIF_TIMEOUT_DIS__CLIENT27_TIMEOUT_DIS__SHIFT                                                       0x1b
+#define RBBMIF_TIMEOUT_DIS__CLIENT28_TIMEOUT_DIS__SHIFT                                                       0x1c
+#define RBBMIF_TIMEOUT_DIS__CLIENT29_TIMEOUT_DIS__SHIFT                                                       0x1d
+#define RBBMIF_TIMEOUT_DIS__CLIENT30_TIMEOUT_DIS__SHIFT                                                       0x1e
+#define RBBMIF_TIMEOUT_DIS__CLIENT31_TIMEOUT_DIS__SHIFT                                                       0x1f
+#define RBBMIF_TIMEOUT_DIS__CLIENT0_TIMEOUT_DIS_MASK                                                          0x00000001L
+#define RBBMIF_TIMEOUT_DIS__CLIENT1_TIMEOUT_DIS_MASK                                                          0x00000002L
+#define RBBMIF_TIMEOUT_DIS__CLIENT2_TIMEOUT_DIS_MASK                                                          0x00000004L
+#define RBBMIF_TIMEOUT_DIS__CLIENT3_TIMEOUT_DIS_MASK                                                          0x00000008L
+#define RBBMIF_TIMEOUT_DIS__CLIENT4_TIMEOUT_DIS_MASK                                                          0x00000010L
+#define RBBMIF_TIMEOUT_DIS__CLIENT5_TIMEOUT_DIS_MASK                                                          0x00000020L
+#define RBBMIF_TIMEOUT_DIS__CLIENT6_TIMEOUT_DIS_MASK                                                          0x00000040L
+#define RBBMIF_TIMEOUT_DIS__CLIENT7_TIMEOUT_DIS_MASK                                                          0x00000080L
+#define RBBMIF_TIMEOUT_DIS__CLIENT8_TIMEOUT_DIS_MASK                                                          0x00000100L
+#define RBBMIF_TIMEOUT_DIS__CLIENT9_TIMEOUT_DIS_MASK                                                          0x00000200L
+#define RBBMIF_TIMEOUT_DIS__CLIENT10_TIMEOUT_DIS_MASK                                                         0x00000400L
+#define RBBMIF_TIMEOUT_DIS__CLIENT11_TIMEOUT_DIS_MASK                                                         0x00000800L
+#define RBBMIF_TIMEOUT_DIS__CLIENT12_TIMEOUT_DIS_MASK                                                         0x00001000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT13_TIMEOUT_DIS_MASK                                                         0x00002000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT14_TIMEOUT_DIS_MASK                                                         0x00004000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT15_TIMEOUT_DIS_MASK                                                         0x00008000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT16_TIMEOUT_DIS_MASK                                                         0x00010000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT17_TIMEOUT_DIS_MASK                                                         0x00020000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT18_TIMEOUT_DIS_MASK                                                         0x00040000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT19_TIMEOUT_DIS_MASK                                                         0x00080000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT20_TIMEOUT_DIS_MASK                                                         0x00100000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT21_TIMEOUT_DIS_MASK                                                         0x00200000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT22_TIMEOUT_DIS_MASK                                                         0x00400000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT23_TIMEOUT_DIS_MASK                                                         0x00800000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT24_TIMEOUT_DIS_MASK                                                         0x01000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT25_TIMEOUT_DIS_MASK                                                         0x02000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT26_TIMEOUT_DIS_MASK                                                         0x04000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT27_TIMEOUT_DIS_MASK                                                         0x08000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT28_TIMEOUT_DIS_MASK                                                         0x10000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT29_TIMEOUT_DIS_MASK                                                         0x20000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT30_TIMEOUT_DIS_MASK                                                         0x40000000L
+#define RBBMIF_TIMEOUT_DIS__CLIENT31_TIMEOUT_DIS_MASK                                                         0x80000000L
+//RBBMIF_TIMEOUT_DIS_2
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT32_TIMEOUT_DIS__SHIFT                                                     0x0
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT33_TIMEOUT_DIS__SHIFT                                                     0x1
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT34_TIMEOUT_DIS__SHIFT                                                     0x2
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT35_TIMEOUT_DIS__SHIFT                                                     0x3
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT36_TIMEOUT_DIS__SHIFT                                                     0x4
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT37_TIMEOUT_DIS__SHIFT                                                     0x5
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT38_TIMEOUT_DIS__SHIFT                                                     0x6
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT39_TIMEOUT_DIS__SHIFT                                                     0x7
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT32_TIMEOUT_DIS_MASK                                                       0x00000001L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT33_TIMEOUT_DIS_MASK                                                       0x00000002L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT34_TIMEOUT_DIS_MASK                                                       0x00000004L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT35_TIMEOUT_DIS_MASK                                                       0x00000008L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT36_TIMEOUT_DIS_MASK                                                       0x00000010L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT37_TIMEOUT_DIS_MASK                                                       0x00000020L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT38_TIMEOUT_DIS_MASK                                                       0x00000040L
+#define RBBMIF_TIMEOUT_DIS_2__CLIENT39_TIMEOUT_DIS_MASK                                                       0x00000080L
+//RBBMIF_STATUS_FLAG
+#define RBBMIF_STATUS_FLAG__RBBMIF_STATE__SHIFT                                                               0x0
+#define RBBMIF_STATUS_FLAG__RBBMIF_READ_TIMEOUT__SHIFT                                                        0x4
+#define RBBMIF_STATUS_FLAG__RBBMIF_FIFO_EMPTY__SHIFT                                                          0x5
+#define RBBMIF_STATUS_FLAG__RBBMIF_FIFO_FULL__SHIFT                                                           0x6
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_FLAG__SHIFT                                                 0x8
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_TYPE__SHIFT                                                 0x9
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_ADDR__SHIFT                                                 0x10
+#define RBBMIF_STATUS_FLAG__RBBMIF_STATE_MASK                                                                 0x00000003L
+#define RBBMIF_STATUS_FLAG__RBBMIF_READ_TIMEOUT_MASK                                                          0x00000010L
+#define RBBMIF_STATUS_FLAG__RBBMIF_FIFO_EMPTY_MASK                                                            0x00000020L
+#define RBBMIF_STATUS_FLAG__RBBMIF_FIFO_FULL_MASK                                                             0x00000040L
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_FLAG_MASK                                                   0x00000100L
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_TYPE_MASK                                                   0x00000E00L
+#define RBBMIF_STATUS_FLAG__RBBMIF_INVALID_ACCESS_ADDR_MASK                                                   0xFFFF0000L
+
+
+// addressBlock: dce_dc_dmu_dmcub_dispdec
+//DMCUB_REGION0_OFFSET
+#define DMCUB_REGION0_OFFSET__DMCUB_REGION0_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION0_OFFSET__DMCUB_REGION0_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION0_OFFSET_HIGH
+#define DMCUB_REGION0_OFFSET_HIGH__DMCUB_REGION0_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION0_OFFSET_HIGH__DMCUB_REGION0_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION1_OFFSET
+#define DMCUB_REGION1_OFFSET__DMCUB_REGION1_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION1_OFFSET__DMCUB_REGION1_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION1_OFFSET_HIGH
+#define DMCUB_REGION1_OFFSET_HIGH__DMCUB_REGION1_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION1_OFFSET_HIGH__DMCUB_REGION1_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION2_OFFSET
+#define DMCUB_REGION2_OFFSET__DMCUB_REGION2_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION2_OFFSET__DMCUB_REGION2_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION2_OFFSET_HIGH
+#define DMCUB_REGION2_OFFSET_HIGH__DMCUB_REGION2_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION2_OFFSET_HIGH__DMCUB_REGION2_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION4_OFFSET
+#define DMCUB_REGION4_OFFSET__DMCUB_REGION4_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION4_OFFSET__DMCUB_REGION4_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION4_OFFSET_HIGH
+#define DMCUB_REGION4_OFFSET_HIGH__DMCUB_REGION4_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION4_OFFSET_HIGH__DMCUB_REGION4_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION5_OFFSET
+#define DMCUB_REGION5_OFFSET__DMCUB_REGION5_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION5_OFFSET__DMCUB_REGION5_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION5_OFFSET_HIGH
+#define DMCUB_REGION5_OFFSET_HIGH__DMCUB_REGION5_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION5_OFFSET_HIGH__DMCUB_REGION5_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION6_OFFSET
+#define DMCUB_REGION6_OFFSET__DMCUB_REGION6_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION6_OFFSET__DMCUB_REGION6_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION6_OFFSET_HIGH
+#define DMCUB_REGION6_OFFSET_HIGH__DMCUB_REGION6_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION6_OFFSET_HIGH__DMCUB_REGION6_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION7_OFFSET
+#define DMCUB_REGION7_OFFSET__DMCUB_REGION7_OFFSET__SHIFT                                                     0x8
+#define DMCUB_REGION7_OFFSET__DMCUB_REGION7_OFFSET_MASK                                                       0xFFFFFF00L
+//DMCUB_REGION7_OFFSET_HIGH
+#define DMCUB_REGION7_OFFSET_HIGH__DMCUB_REGION7_OFFSET_HIGH__SHIFT                                           0x0
+#define DMCUB_REGION7_OFFSET_HIGH__DMCUB_REGION7_OFFSET_HIGH_MASK                                             0x0000FFFFL
+//DMCUB_REGION0_TOP_ADDRESS
+#define DMCUB_REGION0_TOP_ADDRESS__DMCUB_REGION0_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION0_TOP_ADDRESS__DMCUB_REGION0_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION0_TOP_ADDRESS__DMCUB_REGION0_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION0_TOP_ADDRESS__DMCUB_REGION0_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION1_TOP_ADDRESS
+#define DMCUB_REGION1_TOP_ADDRESS__DMCUB_REGION1_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION1_TOP_ADDRESS__DMCUB_REGION1_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION1_TOP_ADDRESS__DMCUB_REGION1_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION1_TOP_ADDRESS__DMCUB_REGION1_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION2_TOP_ADDRESS
+#define DMCUB_REGION2_TOP_ADDRESS__DMCUB_REGION2_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION2_TOP_ADDRESS__DMCUB_REGION2_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION2_TOP_ADDRESS__DMCUB_REGION2_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION2_TOP_ADDRESS__DMCUB_REGION2_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION4_TOP_ADDRESS
+#define DMCUB_REGION4_TOP_ADDRESS__DMCUB_REGION4_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION4_TOP_ADDRESS__DMCUB_REGION4_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION4_TOP_ADDRESS__DMCUB_REGION4_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION4_TOP_ADDRESS__DMCUB_REGION4_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION5_TOP_ADDRESS
+#define DMCUB_REGION5_TOP_ADDRESS__DMCUB_REGION5_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION5_TOP_ADDRESS__DMCUB_REGION5_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION5_TOP_ADDRESS__DMCUB_REGION5_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION5_TOP_ADDRESS__DMCUB_REGION5_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION6_TOP_ADDRESS
+#define DMCUB_REGION6_TOP_ADDRESS__DMCUB_REGION6_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION6_TOP_ADDRESS__DMCUB_REGION6_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION6_TOP_ADDRESS__DMCUB_REGION6_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION6_TOP_ADDRESS__DMCUB_REGION6_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION7_TOP_ADDRESS
+#define DMCUB_REGION7_TOP_ADDRESS__DMCUB_REGION7_TOP_ADDRESS__SHIFT                                           0x0
+#define DMCUB_REGION7_TOP_ADDRESS__DMCUB_REGION7_ENABLE__SHIFT                                                0x1f
+#define DMCUB_REGION7_TOP_ADDRESS__DMCUB_REGION7_TOP_ADDRESS_MASK                                             0x1FFFFFFFL
+#define DMCUB_REGION7_TOP_ADDRESS__DMCUB_REGION7_ENABLE_MASK                                                  0x80000000L
+//DMCUB_REGION3_CW0_BASE_ADDRESS
+#define DMCUB_REGION3_CW0_BASE_ADDRESS__DMCUB_REGION3_CW0_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW0_BASE_ADDRESS__DMCUB_REGION3_CW0_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW1_BASE_ADDRESS
+#define DMCUB_REGION3_CW1_BASE_ADDRESS__DMCUB_REGION3_CW1_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW1_BASE_ADDRESS__DMCUB_REGION3_CW1_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW2_BASE_ADDRESS
+#define DMCUB_REGION3_CW2_BASE_ADDRESS__DMCUB_REGION3_CW2_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW2_BASE_ADDRESS__DMCUB_REGION3_CW2_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW3_BASE_ADDRESS
+#define DMCUB_REGION3_CW3_BASE_ADDRESS__DMCUB_REGION3_CW3_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW3_BASE_ADDRESS__DMCUB_REGION3_CW3_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW4_BASE_ADDRESS
+#define DMCUB_REGION3_CW4_BASE_ADDRESS__DMCUB_REGION3_CW4_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW4_BASE_ADDRESS__DMCUB_REGION3_CW4_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW5_BASE_ADDRESS
+#define DMCUB_REGION3_CW5_BASE_ADDRESS__DMCUB_REGION3_CW5_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW5_BASE_ADDRESS__DMCUB_REGION3_CW5_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW6_BASE_ADDRESS
+#define DMCUB_REGION3_CW6_BASE_ADDRESS__DMCUB_REGION3_CW6_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW6_BASE_ADDRESS__DMCUB_REGION3_CW6_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW7_BASE_ADDRESS
+#define DMCUB_REGION3_CW7_BASE_ADDRESS__DMCUB_REGION3_CW7_BASE_ADDRESS__SHIFT                                 0x0
+#define DMCUB_REGION3_CW7_BASE_ADDRESS__DMCUB_REGION3_CW7_BASE_ADDRESS_MASK                                   0x1FFFFFFFL
+//DMCUB_REGION3_CW0_TOP_ADDRESS
+#define DMCUB_REGION3_CW0_TOP_ADDRESS__DMCUB_REGION3_CW0_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW0_TOP_ADDRESS__DMCUB_REGION3_CW0_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW0_TOP_ADDRESS__DMCUB_REGION3_CW0_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW0_TOP_ADDRESS__DMCUB_REGION3_CW0_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW1_TOP_ADDRESS
+#define DMCUB_REGION3_CW1_TOP_ADDRESS__DMCUB_REGION3_CW1_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW1_TOP_ADDRESS__DMCUB_REGION3_CW1_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW1_TOP_ADDRESS__DMCUB_REGION3_CW1_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW1_TOP_ADDRESS__DMCUB_REGION3_CW1_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW2_TOP_ADDRESS
+#define DMCUB_REGION3_CW2_TOP_ADDRESS__DMCUB_REGION3_CW2_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW2_TOP_ADDRESS__DMCUB_REGION3_CW2_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW2_TOP_ADDRESS__DMCUB_REGION3_CW2_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW2_TOP_ADDRESS__DMCUB_REGION3_CW2_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW3_TOP_ADDRESS
+#define DMCUB_REGION3_CW3_TOP_ADDRESS__DMCUB_REGION3_CW3_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW3_TOP_ADDRESS__DMCUB_REGION3_CW3_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW3_TOP_ADDRESS__DMCUB_REGION3_CW3_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW3_TOP_ADDRESS__DMCUB_REGION3_CW3_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW4_TOP_ADDRESS
+#define DMCUB_REGION3_CW4_TOP_ADDRESS__DMCUB_REGION3_CW4_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW4_TOP_ADDRESS__DMCUB_REGION3_CW4_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW4_TOP_ADDRESS__DMCUB_REGION3_CW4_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW4_TOP_ADDRESS__DMCUB_REGION3_CW4_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW5_TOP_ADDRESS
+#define DMCUB_REGION3_CW5_TOP_ADDRESS__DMCUB_REGION3_CW5_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW5_TOP_ADDRESS__DMCUB_REGION3_CW5_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW5_TOP_ADDRESS__DMCUB_REGION3_CW5_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW5_TOP_ADDRESS__DMCUB_REGION3_CW5_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW6_TOP_ADDRESS
+#define DMCUB_REGION3_CW6_TOP_ADDRESS__DMCUB_REGION3_CW6_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW6_TOP_ADDRESS__DMCUB_REGION3_CW6_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW6_TOP_ADDRESS__DMCUB_REGION3_CW6_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW6_TOP_ADDRESS__DMCUB_REGION3_CW6_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW7_TOP_ADDRESS
+#define DMCUB_REGION3_CW7_TOP_ADDRESS__DMCUB_REGION3_CW7_TOP_ADDRESS__SHIFT                                   0x0
+#define DMCUB_REGION3_CW7_TOP_ADDRESS__DMCUB_REGION3_CW7_ENABLE__SHIFT                                        0x1f
+#define DMCUB_REGION3_CW7_TOP_ADDRESS__DMCUB_REGION3_CW7_TOP_ADDRESS_MASK                                     0x1FFFFFFFL
+#define DMCUB_REGION3_CW7_TOP_ADDRESS__DMCUB_REGION3_CW7_ENABLE_MASK                                          0x80000000L
+//DMCUB_REGION3_CW0_OFFSET
+#define DMCUB_REGION3_CW0_OFFSET__DMCUB_REGION3_CW0_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW0_OFFSET__DMCUB_REGION3_CW0_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW0_OFFSET_HIGH
+#define DMCUB_REGION3_CW0_OFFSET_HIGH__DMCUB_REGION3_CW0_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW0_OFFSET_HIGH__DMCUB_REGION3_CW0_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW1_OFFSET
+#define DMCUB_REGION3_CW1_OFFSET__DMCUB_REGION3_CW1_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW1_OFFSET__DMCUB_REGION3_CW1_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW1_OFFSET_HIGH
+#define DMCUB_REGION3_CW1_OFFSET_HIGH__DMCUB_REGION3_CW1_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW1_OFFSET_HIGH__DMCUB_REGION3_CW1_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW2_OFFSET
+#define DMCUB_REGION3_CW2_OFFSET__DMCUB_REGION3_CW2_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW2_OFFSET__DMCUB_REGION3_CW2_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW2_OFFSET_HIGH
+#define DMCUB_REGION3_CW2_OFFSET_HIGH__DMCUB_REGION3_CW2_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW2_OFFSET_HIGH__DMCUB_REGION3_CW2_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW3_OFFSET
+#define DMCUB_REGION3_CW3_OFFSET__DMCUB_REGION3_CW3_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW3_OFFSET__DMCUB_REGION3_CW3_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW3_OFFSET_HIGH
+#define DMCUB_REGION3_CW3_OFFSET_HIGH__DMCUB_REGION3_CW3_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW3_OFFSET_HIGH__DMCUB_REGION3_CW3_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW4_OFFSET
+#define DMCUB_REGION3_CW4_OFFSET__DMCUB_REGION3_CW4_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW4_OFFSET__DMCUB_REGION3_CW4_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW4_OFFSET_HIGH
+#define DMCUB_REGION3_CW4_OFFSET_HIGH__DMCUB_REGION3_CW4_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW4_OFFSET_HIGH__DMCUB_REGION3_CW4_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW5_OFFSET
+#define DMCUB_REGION3_CW5_OFFSET__DMCUB_REGION3_CW5_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW5_OFFSET__DMCUB_REGION3_CW5_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW5_OFFSET_HIGH
+#define DMCUB_REGION3_CW5_OFFSET_HIGH__DMCUB_REGION3_CW5_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW5_OFFSET_HIGH__DMCUB_REGION3_CW5_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW6_OFFSET
+#define DMCUB_REGION3_CW6_OFFSET__DMCUB_REGION3_CW6_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW6_OFFSET__DMCUB_REGION3_CW6_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW6_OFFSET_HIGH
+#define DMCUB_REGION3_CW6_OFFSET_HIGH__DMCUB_REGION3_CW6_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW6_OFFSET_HIGH__DMCUB_REGION3_CW6_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_REGION3_CW7_OFFSET
+#define DMCUB_REGION3_CW7_OFFSET__DMCUB_REGION3_CW7_OFFSET__SHIFT                                             0x8
+#define DMCUB_REGION3_CW7_OFFSET__DMCUB_REGION3_CW7_OFFSET_MASK                                               0xFFFFFF00L
+//DMCUB_REGION3_CW7_OFFSET_HIGH
+#define DMCUB_REGION3_CW7_OFFSET_HIGH__DMCUB_REGION3_CW7_OFFSET_HIGH__SHIFT                                   0x0
+#define DMCUB_REGION3_CW7_OFFSET_HIGH__DMCUB_REGION3_CW7_OFFSET_HIGH_MASK                                     0x0000FFFFL
+//DMCUB_INTERRUPT_ENABLE
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_TIMER0_INT_EN__SHIFT                                                    0x0
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_TIMER1_INT_EN__SHIFT                                                    0x1
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX0_READY_INT_EN__SHIFT                                              0x2
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX0_DONE_INT_EN__SHIFT                                               0x3
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX1_READY_INT_EN__SHIFT                                              0x4
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX1_DONE_INT_EN__SHIFT                                               0x5
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX0_READY_INT_EN__SHIFT                                             0x6
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX0_DONE_INT_EN__SHIFT                                              0x7
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX1_READY_INT_EN__SHIFT                                             0x8
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX1_DONE_INT_EN__SHIFT                                              0x9
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT0_INT_EN__SHIFT                                                    0xa
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT1_INT_EN__SHIFT                                                    0xb
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT2_INT_EN__SHIFT                                                    0xc
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT3_INT_EN__SHIFT                                                    0xd
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT4_INT_EN__SHIFT                                                    0xe
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT5_INT_EN__SHIFT                                                    0xf
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT6_INT_EN__SHIFT                                                    0x10
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT_IH_INT_EN__SHIFT                                                  0x11
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_UNDEFINED_ADDRESS_FAULT_INT_EN__SHIFT                                   0x12
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX0_RDY_INT_EN__SHIFT                                            0x13
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX2_RDY_INT_EN__SHIFT                                            0x15
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX3_RDY_INT_EN__SHIFT                                            0x16
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX4_RDY_INT_EN__SHIFT                                            0x17
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_OUTBOX0_RSP_INT_EN__SHIFT                                           0x18
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_TIMER0_INT_EN_MASK                                                      0x00000001L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_TIMER1_INT_EN_MASK                                                      0x00000002L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX0_READY_INT_EN_MASK                                                0x00000004L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX0_DONE_INT_EN_MASK                                                 0x00000008L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX1_READY_INT_EN_MASK                                                0x00000010L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_INBOX1_DONE_INT_EN_MASK                                                 0x00000020L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX0_READY_INT_EN_MASK                                               0x00000040L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX0_DONE_INT_EN_MASK                                                0x00000080L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX1_READY_INT_EN_MASK                                               0x00000100L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_OUTBOX1_DONE_INT_EN_MASK                                                0x00000200L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT0_INT_EN_MASK                                                      0x00000400L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT1_INT_EN_MASK                                                      0x00000800L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT2_INT_EN_MASK                                                      0x00001000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT3_INT_EN_MASK                                                      0x00002000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT4_INT_EN_MASK                                                      0x00004000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT5_INT_EN_MASK                                                      0x00008000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT6_INT_EN_MASK                                                      0x00010000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_GPINT_IH_INT_EN_MASK                                                    0x00020000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_UNDEFINED_ADDRESS_FAULT_INT_EN_MASK                                     0x00040000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX0_RDY_INT_EN_MASK                                              0x00080000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX2_RDY_INT_EN_MASK                                              0x00200000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX3_RDY_INT_EN_MASK                                              0x00400000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_INBOX4_RDY_INT_EN_MASK                                              0x00800000L
+#define DMCUB_INTERRUPT_ENABLE__DMCUB_REG_OUTBOX0_RSP_INT_EN_MASK                                             0x01000000L
+//DMCUB_INTERRUPT_ACK
+#define DMCUB_INTERRUPT_ACK__DMCUB_TIMER0_INT_ACK__SHIFT                                                      0x0
+#define DMCUB_INTERRUPT_ACK__DMCUB_TIMER1_INT_ACK__SHIFT                                                      0x1
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX0_READY_INT_ACK__SHIFT                                                0x2
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX0_DONE_INT_ACK__SHIFT                                                 0x3
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX1_READY_INT_ACK__SHIFT                                                0x4
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX1_DONE_INT_ACK__SHIFT                                                 0x5
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX0_READY_INT_ACK__SHIFT                                               0x6
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX0_DONE_INT_ACK__SHIFT                                                0x7
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX1_READY_INT_ACK__SHIFT                                               0x8
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX1_DONE_INT_ACK__SHIFT                                                0x9
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT0_INT_ACK__SHIFT                                                      0xa
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT1_INT_ACK__SHIFT                                                      0xb
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT2_INT_ACK__SHIFT                                                      0xc
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT3_INT_ACK__SHIFT                                                      0xd
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT4_INT_ACK__SHIFT                                                      0xe
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT5_INT_ACK__SHIFT                                                      0xf
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT6_INT_ACK__SHIFT                                                      0x10
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT_IH_INT_ACK__SHIFT                                                    0x11
+#define DMCUB_INTERRUPT_ACK__DMCUB_UNDEFINED_ADDRESS_FAULT_ACK__SHIFT                                         0x12
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX0_RDY_INT_ACK__SHIFT                                              0x13
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX2_RDY_INT_ACK__SHIFT                                              0x15
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX3_RDY_INT_ACK__SHIFT                                              0x16
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX4_RDY_INT_ACK__SHIFT                                              0x17
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_OUTBOX0_RSP_INT_ACK__SHIFT                                             0x18
+#define DMCUB_INTERRUPT_ACK__DMCUB_TIMER0_INT_ACK_MASK                                                        0x00000001L
+#define DMCUB_INTERRUPT_ACK__DMCUB_TIMER1_INT_ACK_MASK                                                        0x00000002L
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX0_READY_INT_ACK_MASK                                                  0x00000004L
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX0_DONE_INT_ACK_MASK                                                   0x00000008L
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX1_READY_INT_ACK_MASK                                                  0x00000010L
+#define DMCUB_INTERRUPT_ACK__DMCUB_INBOX1_DONE_INT_ACK_MASK                                                   0x00000020L
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX0_READY_INT_ACK_MASK                                                 0x00000040L
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX0_DONE_INT_ACK_MASK                                                  0x00000080L
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX1_READY_INT_ACK_MASK                                                 0x00000100L
+#define DMCUB_INTERRUPT_ACK__DMCUB_OUTBOX1_DONE_INT_ACK_MASK                                                  0x00000200L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT0_INT_ACK_MASK                                                        0x00000400L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT1_INT_ACK_MASK                                                        0x00000800L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT2_INT_ACK_MASK                                                        0x00001000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT3_INT_ACK_MASK                                                        0x00002000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT4_INT_ACK_MASK                                                        0x00004000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT5_INT_ACK_MASK                                                        0x00008000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT6_INT_ACK_MASK                                                        0x00010000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_GPINT_IH_INT_ACK_MASK                                                      0x00020000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_UNDEFINED_ADDRESS_FAULT_ACK_MASK                                           0x00040000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX0_RDY_INT_ACK_MASK                                                0x00080000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX2_RDY_INT_ACK_MASK                                                0x00200000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX3_RDY_INT_ACK_MASK                                                0x00400000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_INBOX4_RDY_INT_ACK_MASK                                                0x00800000L
+#define DMCUB_INTERRUPT_ACK__DMCUB_REG_OUTBOX0_RSP_INT_ACK_MASK                                               0x01000000L
+//DMCUB_INTERRUPT_STATUS
+#define DMCUB_INTERRUPT_STATUS__DMCUB_TIMER0_INT_STAT__SHIFT                                                  0x0
+#define DMCUB_INTERRUPT_STATUS__DMCUB_TIMER1_INT_STAT__SHIFT                                                  0x1
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX0_READY_INT_STAT__SHIFT                                            0x2
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX0_DONE_INT_STAT__SHIFT                                             0x3
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX1_READY_INT_STAT__SHIFT                                            0x4
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX1_DONE_INT_STAT__SHIFT                                             0x5
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX0_READY_INT_STAT__SHIFT                                           0x6
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX0_DONE_INT_STAT__SHIFT                                            0x7
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX1_READY_INT_STAT__SHIFT                                           0x8
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX1_DONE_INT_STAT__SHIFT                                            0x9
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT0_INT_STAT__SHIFT                                                  0xa
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT1_INT_STAT__SHIFT                                                  0xb
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT2_INT_STAT__SHIFT                                                  0xc
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT3_INT_STAT__SHIFT                                                  0xd
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT4_INT_STAT__SHIFT                                                  0xe
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT5_INT_STAT__SHIFT                                                  0xf
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT6_INT_STAT__SHIFT                                                  0x10
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT_IH_INT_STAT__SHIFT                                                0x11
+#define DMCUB_INTERRUPT_STATUS__DMCUB_UNDEFINED_ADDRESS_FAULT__SHIFT                                          0x12
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INST_FETCH_FAULT__SHIFT                                                 0x13
+#define DMCUB_INTERRUPT_STATUS__DMCUB_DATA_WRITE_FAULT__SHIFT                                                 0x14
+#define DMCUB_INTERRUPT_STATUS__DMCUB_PWR_UP_TRIG_INT_STAT__SHIFT                                             0x15
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OTG_RESYNC_TRIG_INT_STAT__SHIFT                                         0x16
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX0_RDY_INT_STAT__SHIFT                                          0x17
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX2_RDY_INT_STAT__SHIFT                                          0x19
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX3_RDY_INT_STAT__SHIFT                                          0x1a
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX4_RDY_INT_STAT__SHIFT                                          0x1b
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_OUTBOX0_RSP_INT_STAT__SHIFT                                         0x1c
+#define DMCUB_INTERRUPT_STATUS__DMCUB_TIMER0_INT_STAT_MASK                                                    0x00000001L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_TIMER1_INT_STAT_MASK                                                    0x00000002L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX0_READY_INT_STAT_MASK                                              0x00000004L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX0_DONE_INT_STAT_MASK                                               0x00000008L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX1_READY_INT_STAT_MASK                                              0x00000010L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INBOX1_DONE_INT_STAT_MASK                                               0x00000020L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX0_READY_INT_STAT_MASK                                             0x00000040L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX0_DONE_INT_STAT_MASK                                              0x00000080L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX1_READY_INT_STAT_MASK                                             0x00000100L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OUTBOX1_DONE_INT_STAT_MASK                                              0x00000200L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT0_INT_STAT_MASK                                                    0x00000400L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT1_INT_STAT_MASK                                                    0x00000800L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT2_INT_STAT_MASK                                                    0x00001000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT3_INT_STAT_MASK                                                    0x00002000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT4_INT_STAT_MASK                                                    0x00004000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT5_INT_STAT_MASK                                                    0x00008000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT6_INT_STAT_MASK                                                    0x00010000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_GPINT_IH_INT_STAT_MASK                                                  0x00020000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_UNDEFINED_ADDRESS_FAULT_MASK                                            0x00040000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_INST_FETCH_FAULT_MASK                                                   0x00080000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_DATA_WRITE_FAULT_MASK                                                   0x00100000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_PWR_UP_TRIG_INT_STAT_MASK                                               0x00200000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_OTG_RESYNC_TRIG_INT_STAT_MASK                                           0x00400000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX0_RDY_INT_STAT_MASK                                            0x00800000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX2_RDY_INT_STAT_MASK                                            0x02000000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX3_RDY_INT_STAT_MASK                                            0x04000000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_INBOX4_RDY_INT_STAT_MASK                                            0x08000000L
+#define DMCUB_INTERRUPT_STATUS__DMCUB_REG_OUTBOX0_RSP_INT_STAT_MASK                                           0x10000000L
+//DMCUB_INTERRUPT_TYPE
+#define DMCUB_INTERRUPT_TYPE__DMCUB_TIMER0_INT_TYPE__SHIFT                                                    0x0
+#define DMCUB_INTERRUPT_TYPE__DMCUB_TIMER1_INT_TYPE__SHIFT                                                    0x1
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX0_READY_INT_TYPE__SHIFT                                              0x2
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX0_DONE_INT_TYPE__SHIFT                                               0x3
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX1_READY_INT_TYPE__SHIFT                                              0x4
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX1_DONE_INT_TYPE__SHIFT                                               0x5
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX0_READY_INT_TYPE__SHIFT                                             0x6
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX0_DONE_INT_TYPE__SHIFT                                              0x7
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX1_READY_INT_TYPE__SHIFT                                             0x8
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX1_DONE_INT_TYPE__SHIFT                                              0x9
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT0_INT_TYPE__SHIFT                                                    0xa
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT1_INT_TYPE__SHIFT                                                    0xb
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT2_INT_TYPE__SHIFT                                                    0xc
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT3_INT_TYPE__SHIFT                                                    0xd
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT4_INT_TYPE__SHIFT                                                    0xe
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT5_INT_TYPE__SHIFT                                                    0xf
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT6_INT_TYPE__SHIFT                                                    0x10
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT_IH_INT_TYPE__SHIFT                                                  0x11
+#define DMCUB_INTERRUPT_TYPE__DMCUB_UNDEFINED_ADDRESS_FAULT_INT_TYPE__SHIFT                                   0x12
+#define DMCUB_INTERRUPT_TYPE__DMCUB_TIMER0_INT_TYPE_MASK                                                      0x00000001L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_TIMER1_INT_TYPE_MASK                                                      0x00000002L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX0_READY_INT_TYPE_MASK                                                0x00000004L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX0_DONE_INT_TYPE_MASK                                                 0x00000008L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX1_READY_INT_TYPE_MASK                                                0x00000010L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_INBOX1_DONE_INT_TYPE_MASK                                                 0x00000020L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX0_READY_INT_TYPE_MASK                                               0x00000040L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX0_DONE_INT_TYPE_MASK                                                0x00000080L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX1_READY_INT_TYPE_MASK                                               0x00000100L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_OUTBOX1_DONE_INT_TYPE_MASK                                                0x00000200L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT0_INT_TYPE_MASK                                                      0x00000400L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT1_INT_TYPE_MASK                                                      0x00000800L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT2_INT_TYPE_MASK                                                      0x00001000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT3_INT_TYPE_MASK                                                      0x00002000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT4_INT_TYPE_MASK                                                      0x00004000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT5_INT_TYPE_MASK                                                      0x00008000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT6_INT_TYPE_MASK                                                      0x00010000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_GPINT_IH_INT_TYPE_MASK                                                    0x00020000L
+#define DMCUB_INTERRUPT_TYPE__DMCUB_UNDEFINED_ADDRESS_FAULT_INT_TYPE_MASK                                     0x00040000L
+//DMCUB_EXT_INTERRUPT_STATUS
+#define DMCUB_EXT_INTERRUPT_STATUS__DMCUB_EXT_INTERRUPT_COUNT__SHIFT                                          0x0
+#define DMCUB_EXT_INTERRUPT_STATUS__DMCUB_EXT_INTERRUPT_ID__SHIFT                                             0x8
+#define DMCUB_EXT_INTERRUPT_STATUS__DMCUB_EXT_INTERRUPT_COUNT_MASK                                            0x000000FFL
+#define DMCUB_EXT_INTERRUPT_STATUS__DMCUB_EXT_INTERRUPT_ID_MASK                                               0x0000FF00L
+//DMCUB_EXT_INTERRUPT_CTXID
+#define DMCUB_EXT_INTERRUPT_CTXID__DMCUB_EXT_INTERRUPT_CTXID__SHIFT                                           0x0
+#define DMCUB_EXT_INTERRUPT_CTXID__DMCUB_EXT_INTERRUPT_CTXID_MASK                                             0x0FFFFFFFL
+//DMCUB_EXT_INTERRUPT_ACK
+#define DMCUB_EXT_INTERRUPT_ACK__DMCUB_EXT_INTERRUPT_ACK__SHIFT                                               0x0
+#define DMCUB_EXT_INTERRUPT_ACK__DMCUB_EXT_INTERRUPT_ACK_MASK                                                 0x00000001L
+//DMCUB_INST_FETCH_FAULT_ADDR
+#define DMCUB_INST_FETCH_FAULT_ADDR__DMCUB_INST_FETCH_FAULT_ADDR__SHIFT                                       0x0
+#define DMCUB_INST_FETCH_FAULT_ADDR__DMCUB_INST_FETCH_FAULT_ADDR_MASK                                         0xFFFFFFFFL
+//DMCUB_DATA_WRITE_FAULT_ADDR
+#define DMCUB_DATA_WRITE_FAULT_ADDR__DMCUB_DATA_WRITE_FAULT_ADDR__SHIFT                                       0x0
+#define DMCUB_DATA_WRITE_FAULT_ADDR__DMCUB_DATA_WRITE_FAULT_ADDR_MASK                                         0xFFFFFFFFL
+//DMCUB_SEC_CNTL
+#define DMCUB_SEC_CNTL__DMCUB_MEM_SEC_LVL__SHIFT                                                              0x0
+#define DMCUB_SEC_CNTL__DMCUB_MEM_UNIT_ID__SHIFT                                                              0x8
+#define DMCUB_SEC_CNTL__DMCUB_SEC_RESET__SHIFT                                                                0x10
+#define DMCUB_SEC_CNTL__DMCUB_DATA_FAULT_INT_DISABLE__SHIFT                                                   0x11
+#define DMCUB_SEC_CNTL__DMCUB_AUTO_RESET_STATUS__SHIFT                                                        0x14
+#define DMCUB_SEC_CNTL__DMCUB_SEC_RESET_STATUS__SHIFT                                                         0x15
+#define DMCUB_SEC_CNTL__DMCUB_INST_FETCH_FAULT_CLEAR__SHIFT                                                   0x18
+#define DMCUB_SEC_CNTL__DMCUB_DATA_WRITE_FAULT_CLEAR__SHIFT                                                   0x19
+#define DMCUB_SEC_CNTL__DMCUB_MEM_SEC_LVL_MASK                                                                0x00000007L
+#define DMCUB_SEC_CNTL__DMCUB_MEM_UNIT_ID_MASK                                                                0x00003F00L
+#define DMCUB_SEC_CNTL__DMCUB_SEC_RESET_MASK                                                                  0x00010000L
+#define DMCUB_SEC_CNTL__DMCUB_DATA_FAULT_INT_DISABLE_MASK                                                     0x00020000L
+#define DMCUB_SEC_CNTL__DMCUB_AUTO_RESET_STATUS_MASK                                                          0x00100000L
+#define DMCUB_SEC_CNTL__DMCUB_SEC_RESET_STATUS_MASK                                                           0x00200000L
+#define DMCUB_SEC_CNTL__DMCUB_INST_FETCH_FAULT_CLEAR_MASK                                                     0x01000000L
+#define DMCUB_SEC_CNTL__DMCUB_DATA_WRITE_FAULT_CLEAR_MASK                                                     0x02000000L
+//DMCUB_MEM_CNTL
+#define DMCUB_MEM_CNTL__DMCUB_MEM_WRITE_QOS__SHIFT                                                            0x0
+#define DMCUB_MEM_CNTL__DMCUB_MEM_READ_QOS__SHIFT                                                             0x4
+#define DMCUB_MEM_CNTL__DMCUB_MEM_WRITE_QOS_MASK                                                              0x0000000FL
+#define DMCUB_MEM_CNTL__DMCUB_MEM_READ_QOS_MASK                                                               0x000000F0L
+//DMCUB_INBOX0_BASE_ADDRESS
+#define DMCUB_INBOX0_BASE_ADDRESS__DMCUB_INBOX0_BASE_ADDRESS__SHIFT                                           0x0
+#define DMCUB_INBOX0_BASE_ADDRESS__DMCUB_INBOX0_BASE_ADDRESS_MASK                                             0xFFFFFFFFL
+//DMCUB_INBOX0_SIZE
+#define DMCUB_INBOX0_SIZE__DMCUB_INBOX0_SIZE__SHIFT                                                           0x0
+#define DMCUB_INBOX0_SIZE__DMCUB_INBOX0_SIZE_MASK                                                             0xFFFFFFFFL
+//DMCUB_INBOX0_WPTR
+#define DMCUB_INBOX0_WPTR__DMCUB_INBOX0_WPTR__SHIFT                                                           0x0
+#define DMCUB_INBOX0_WPTR__DMCUB_INBOX0_WPTR_MASK                                                             0xFFFFFFFFL
+//DMCUB_INBOX0_RPTR
+#define DMCUB_INBOX0_RPTR__DMCUB_INBOX0_RPTR__SHIFT                                                           0x0
+#define DMCUB_INBOX0_RPTR__DMCUB_INBOX0_RPTR_MASK                                                             0xFFFFFFFFL
+//DMCUB_INBOX1_BASE_ADDRESS
+#define DMCUB_INBOX1_BASE_ADDRESS__DMCUB_INBOX1_BASE_ADDRESS__SHIFT                                           0x0
+#define DMCUB_INBOX1_BASE_ADDRESS__DMCUB_INBOX1_BASE_ADDRESS_MASK                                             0xFFFFFFFFL
+//DMCUB_INBOX1_SIZE
+#define DMCUB_INBOX1_SIZE__DMCUB_INBOX1_SIZE__SHIFT                                                           0x0
+#define DMCUB_INBOX1_SIZE__DMCUB_INBOX1_SIZE_MASK                                                             0xFFFFFFFFL
+//DMCUB_INBOX1_WPTR
+#define DMCUB_INBOX1_WPTR__DMCUB_INBOX1_WPTR__SHIFT                                                           0x0
+#define DMCUB_INBOX1_WPTR__DMCUB_INBOX1_WPTR_MASK                                                             0xFFFFFFFFL
+//DMCUB_INBOX1_RPTR
+#define DMCUB_INBOX1_RPTR__DMCUB_INBOX1_RPTR__SHIFT                                                           0x0
+#define DMCUB_INBOX1_RPTR__DMCUB_INBOX1_RPTR_MASK                                                             0xFFFFFFFFL
+//DMCUB_OUTBOX0_BASE_ADDRESS
+#define DMCUB_OUTBOX0_BASE_ADDRESS__DMCUB_OUTBOX0_BASE_ADDRESS__SHIFT                                         0x0
+#define DMCUB_OUTBOX0_BASE_ADDRESS__DMCUB_OUTBOX0_BASE_ADDRESS_MASK                                           0xFFFFFFFFL
+//DMCUB_OUTBOX0_SIZE
+#define DMCUB_OUTBOX0_SIZE__DMCUB_OUTBOX0_SIZE__SHIFT                                                         0x0
+#define DMCUB_OUTBOX0_SIZE__DMCUB_OUTBOX0_SIZE_MASK                                                           0xFFFFFFFFL
+//DMCUB_OUTBOX0_WPTR
+#define DMCUB_OUTBOX0_WPTR__DMCUB_OUTBOX0_WPTR__SHIFT                                                         0x0
+#define DMCUB_OUTBOX0_WPTR__DMCUB_OUTBOX0_WPTR_MASK                                                           0xFFFFFFFFL
+//DMCUB_OUTBOX0_RPTR
+#define DMCUB_OUTBOX0_RPTR__DMCUB_OUTBOX0_RPTR__SHIFT                                                         0x0
+#define DMCUB_OUTBOX0_RPTR__DMCUB_OUTBOX0_RPTR_MASK                                                           0xFFFFFFFFL
+//DMCUB_OUTBOX1_BASE_ADDRESS
+#define DMCUB_OUTBOX1_BASE_ADDRESS__DMCUB_OUTBOX1_BASE_ADDRESS__SHIFT                                         0x0
+#define DMCUB_OUTBOX1_BASE_ADDRESS__DMCUB_OUTBOX1_BASE_ADDRESS_MASK                                           0xFFFFFFFFL
+//DMCUB_OUTBOX1_SIZE
+#define DMCUB_OUTBOX1_SIZE__DMCUB_OUTBOX1_SIZE__SHIFT                                                         0x0
+#define DMCUB_OUTBOX1_SIZE__DMCUB_OUTBOX1_SIZE_MASK                                                           0xFFFFFFFFL
+//DMCUB_OUTBOX1_WPTR
+#define DMCUB_OUTBOX1_WPTR__DMCUB_OUTBOX1_WPTR__SHIFT                                                         0x0
+#define DMCUB_OUTBOX1_WPTR__DMCUB_OUTBOX1_WPTR_MASK                                                           0xFFFFFFFFL
+//DMCUB_OUTBOX1_RPTR
+#define DMCUB_OUTBOX1_RPTR__DMCUB_OUTBOX1_RPTR__SHIFT                                                         0x0
+#define DMCUB_OUTBOX1_RPTR__DMCUB_OUTBOX1_RPTR_MASK                                                           0xFFFFFFFFL
+//DMCUB_TIMER_TRIGGER0
+#define DMCUB_TIMER_TRIGGER0__DMCUB_TIMER_TRIGGER0__SHIFT                                                     0x0
+#define DMCUB_TIMER_TRIGGER0__DMCUB_TIMER_TRIGGER0_MASK                                                       0xFFFFFFFFL
+//DMCUB_TIMER_TRIGGER1
+#define DMCUB_TIMER_TRIGGER1__DMCUB_TIMER_TRIGGER1__SHIFT                                                     0x0
+#define DMCUB_TIMER_TRIGGER1__DMCUB_TIMER_TRIGGER1_MASK                                                       0xFFFFFFFFL
+//DMCUB_TIMER_WINDOW
+#define DMCUB_TIMER_WINDOW__DMCUB_TIMER_WINDOW__SHIFT                                                         0x0
+#define DMCUB_TIMER_WINDOW__DMCUB_TIMER_WINDOW_MASK                                                           0x00000007L
+//DMCUB_SCRATCH0
+#define DMCUB_SCRATCH0__DMCUB_SCRATCH0__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH0__DMCUB_SCRATCH0_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH1
+#define DMCUB_SCRATCH1__DMCUB_SCRATCH1__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH1__DMCUB_SCRATCH1_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH2
+#define DMCUB_SCRATCH2__DMCUB_SCRATCH2__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH2__DMCUB_SCRATCH2_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH3
+#define DMCUB_SCRATCH3__DMCUB_SCRATCH3__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH3__DMCUB_SCRATCH3_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH4
+#define DMCUB_SCRATCH4__DMCUB_SCRATCH4__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH4__DMCUB_SCRATCH4_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH5
+#define DMCUB_SCRATCH5__DMCUB_SCRATCH5__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH5__DMCUB_SCRATCH5_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH6
+#define DMCUB_SCRATCH6__DMCUB_SCRATCH6__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH6__DMCUB_SCRATCH6_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH7
+#define DMCUB_SCRATCH7__DMCUB_SCRATCH7__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH7__DMCUB_SCRATCH7_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH8
+#define DMCUB_SCRATCH8__DMCUB_SCRATCH8__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH8__DMCUB_SCRATCH8_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH9
+#define DMCUB_SCRATCH9__DMCUB_SCRATCH9__SHIFT                                                                 0x0
+#define DMCUB_SCRATCH9__DMCUB_SCRATCH9_MASK                                                                   0xFFFFFFFFL
+//DMCUB_SCRATCH10
+#define DMCUB_SCRATCH10__DMCUB_SCRATCH10__SHIFT                                                               0x0
+#define DMCUB_SCRATCH10__DMCUB_SCRATCH10_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH11
+#define DMCUB_SCRATCH11__DMCUB_SCRATCH11__SHIFT                                                               0x0
+#define DMCUB_SCRATCH11__DMCUB_SCRATCH11_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH12
+#define DMCUB_SCRATCH12__DMCUB_SCRATCH12__SHIFT                                                               0x0
+#define DMCUB_SCRATCH12__DMCUB_SCRATCH12_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH13
+#define DMCUB_SCRATCH13__DMCUB_SCRATCH13__SHIFT                                                               0x0
+#define DMCUB_SCRATCH13__DMCUB_SCRATCH13_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH14
+#define DMCUB_SCRATCH14__DMCUB_SCRATCH14__SHIFT                                                               0x0
+#define DMCUB_SCRATCH14__DMCUB_SCRATCH14_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH15
+#define DMCUB_SCRATCH15__DMCUB_SCRATCH15__SHIFT                                                               0x0
+#define DMCUB_SCRATCH15__DMCUB_SCRATCH15_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH16
+#define DMCUB_SCRATCH16__DMCUB_SCRATCH16__SHIFT                                                               0x0
+#define DMCUB_SCRATCH16__DMCUB_SCRATCH16_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH17
+#define DMCUB_SCRATCH17__DMCUB_SCRATCH17__SHIFT                                                               0x0
+#define DMCUB_SCRATCH17__DMCUB_SCRATCH17_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH18
+#define DMCUB_SCRATCH18__DMCUB_SCRATCH18__SHIFT                                                               0x0
+#define DMCUB_SCRATCH18__DMCUB_SCRATCH18_MASK                                                                 0xFFFFFFFFL
+//DMCUB_CNTL
+#define DMCUB_CNTL__DMCUB_LS_WAKE_DELAY__SHIFT                                                                0x0
+#define DMCUB_CNTL__DMCUB_DMCUBCLK_R_GATE_DIS__SHIFT                                                          0x8
+#define DMCUB_CNTL__DMCUB_ENABLE__SHIFT                                                                       0x10
+#define DMCUB_CNTL__DMCUB_MEM_LIGHT_SLEEP_DISABLE__SHIFT                                                      0x12
+#define DMCUB_CNTL__DMCUB_TRACEPORT_EN__SHIFT                                                                 0x13
+#define DMCUB_CNTL__DMCUB_PWAIT_MODE_STATUS__SHIFT                                                            0x14
+#define DMCUB_CNTL__DMCUB_LS_WAKE_DELAY_MASK                                                                  0x000000FFL
+#define DMCUB_CNTL__DMCUB_DMCUBCLK_R_GATE_DIS_MASK                                                            0x00000100L
+#define DMCUB_CNTL__DMCUB_ENABLE_MASK                                                                         0x00010000L
+#define DMCUB_CNTL__DMCUB_MEM_LIGHT_SLEEP_DISABLE_MASK                                                        0x00040000L
+#define DMCUB_CNTL__DMCUB_TRACEPORT_EN_MASK                                                                   0x00080000L
+#define DMCUB_CNTL__DMCUB_PWAIT_MODE_STATUS_MASK                                                              0x00100000L
+//DMCUB_GPINT_DATAIN0
+#define DMCUB_GPINT_DATAIN0__DMCUB_GPINT_DATAIN0__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN0__DMCUB_GPINT_DATAIN0_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAIN1
+#define DMCUB_GPINT_DATAIN1__DMCUB_GPINT_DATAIN1__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN1__DMCUB_GPINT_DATAIN1_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAOUT
+#define DMCUB_GPINT_DATAOUT__DMCUB_GPINT_DATAOUT__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAOUT__DMCUB_GPINT_DATAOUT_MASK                                                         0xFFFFFFFFL
+//DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR
+#define DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR__DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR__SHIFT                         0x0
+#define DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR__DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR_MASK                           0xFFFFFFFFL
+//DMCUB_LS_WAKE_INT_ENABLE
+#define DMCUB_LS_WAKE_INT_ENABLE__DMCUB_LS_WAKE_INT_ENABLE__SHIFT                                             0x0
+#define DMCUB_LS_WAKE_INT_ENABLE__DMCUB_LS_WAKE_INT_ENABLE_MASK                                               0xFFFFFFFFL
+//DMCUB_MEM_PWR_CNTL
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_FORCE__SHIFT                                                        0x1
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_DIS__SHIFT                                                          0x3
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_STATE__SHIFT                                                        0x4
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_FORCE_MASK                                                          0x00000006L
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_DIS_MASK                                                            0x00000008L
+#define DMCUB_MEM_PWR_CNTL__DMCUB_MEM_PWR_STATE_MASK                                                          0x00000030L
+//DMCUB_TIMER_CURRENT
+#define DMCUB_TIMER_CURRENT__DMCUB_TIMER_CURRENT__SHIFT                                                       0x0
+#define DMCUB_TIMER_CURRENT__DMCUB_TIMER_CURRENT_MASK                                                         0xFFFFFFFFL
+//DMCUB_PROC_ID
+#define DMCUB_PROC_ID__DMCUB_PROC_ID__SHIFT                                                                   0x0
+#define DMCUB_PROC_ID__DMCUB_PROC_ID_MASK                                                                     0x0000FFFFL
+//DMCUB_CNTL2
+#define DMCUB_CNTL2__DMCUB_SOFT_RESET__SHIFT                                                                  0x0
+#define DMCUB_CNTL2__DMCUB_SOFT_RESET_MASK                                                                    0x00000001L
+//DMCUB_GPINT_DATAIN2
+#define DMCUB_GPINT_DATAIN2__DMCUB_GPINT_DATAIN2__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN2__DMCUB_GPINT_DATAIN2_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAIN3
+#define DMCUB_GPINT_DATAIN3__DMCUB_GPINT_DATAIN3__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN3__DMCUB_GPINT_DATAIN3_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAIN4
+#define DMCUB_GPINT_DATAIN4__DMCUB_GPINT_DATAIN4__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN4__DMCUB_GPINT_DATAIN4_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAIN5
+#define DMCUB_GPINT_DATAIN5__DMCUB_GPINT_DATAIN5__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN5__DMCUB_GPINT_DATAIN5_MASK                                                         0xFFFFFFFFL
+//DMCUB_GPINT_DATAIN6
+#define DMCUB_GPINT_DATAIN6__DMCUB_GPINT_DATAIN6__SHIFT                                                       0x0
+#define DMCUB_GPINT_DATAIN6__DMCUB_GPINT_DATAIN6_MASK                                                         0xFFFFFFFFL
+//DMCUB_REGION3_TMR_AXI_SPACE
+#define DMCUB_REGION3_TMR_AXI_SPACE__DMCUB_REGION3_TMR_AXI_SPACE__SHIFT                                       0x0
+#define DMCUB_REGION3_TMR_AXI_SPACE__DMCUB_REGION3_TMR_AXI_SPACE_MASK                                         0x07L
+//DMCUB_SCRATCH19
+#define DMCUB_SCRATCH19__DMCUB_SCRATCH19__SHIFT                                                               0x0
+#define DMCUB_SCRATCH19__DMCUB_SCRATCH19_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH20
+#define DMCUB_SCRATCH20__DMCUB_SCRATCH20__SHIFT                                                               0x0
+#define DMCUB_SCRATCH20__DMCUB_SCRATCH20_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH21
+#define DMCUB_SCRATCH21__DMCUB_SCRATCH21__SHIFT                                                               0x0
+#define DMCUB_SCRATCH21__DMCUB_SCRATCH21_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH22
+#define DMCUB_SCRATCH22__DMCUB_SCRATCH22__SHIFT                                                               0x0
+#define DMCUB_SCRATCH22__DMCUB_SCRATCH22_MASK                                                                 0xFFFFFFFFL
+//DMCUB_SCRATCH23
+#define DMCUB_SCRATCH23__DMCUB_SCRATCH23__SHIFT                                                               0x0
+#define DMCUB_SCRATCH23__DMCUB_SCRATCH23_MASK                                                                 0xFFFFFFFFL
+//HOST_INTERRUPT_CSR
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_EN__SHIFT                                                 0x0
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_EN__SHIFT                                                0x1
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_ACK__SHIFT                                                0x8
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_ACK__SHIFT                                               0x9
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_STAT__SHIFT                                               0xc
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_STAT__SHIFT                                              0xd
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_EN_MASK                                                   0x00000001L
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_EN_MASK                                                  0x00000002L
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_ACK_MASK                                                  0x00000100L
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_ACK_MASK                                                 0x00000200L
+#define HOST_INTERRUPT_CSR__HOST_REG_INBOX0_RSP_INT_STAT_MASK                                                 0x00001000L
+#define HOST_INTERRUPT_CSR__HOST_REG_OUTBOX0_RDY_INT_STAT_MASK                                                0x00002000L
+//DMCUB_REG_INBOX0_RDY
+#define DMCUB_REG_INBOX0_RDY__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX0_RDY__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG0
+#define DMCUB_REG_INBOX0_MSG0__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG0__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG1
+#define DMCUB_REG_INBOX0_MSG1__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG1__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG2
+#define DMCUB_REG_INBOX0_MSG2__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG2__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG3
+#define DMCUB_REG_INBOX0_MSG3__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG3__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG4
+#define DMCUB_REG_INBOX0_MSG4__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG4__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG5
+#define DMCUB_REG_INBOX0_MSG5__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG5__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG6
+#define DMCUB_REG_INBOX0_MSG6__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG6__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG7
+#define DMCUB_REG_INBOX0_MSG7__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG7__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG8
+#define DMCUB_REG_INBOX0_MSG8__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG8__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG9
+#define DMCUB_REG_INBOX0_MSG9__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX0_MSG9__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG10
+#define DMCUB_REG_INBOX0_MSG10__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_INBOX0_MSG10__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG11
+#define DMCUB_REG_INBOX0_MSG11__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_INBOX0_MSG11__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG12
+#define DMCUB_REG_INBOX0_MSG12__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_INBOX0_MSG12__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG13
+#define DMCUB_REG_INBOX0_MSG13__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_INBOX0_MSG13__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_INBOX0_MSG14
+#define DMCUB_REG_INBOX0_MSG14__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_INBOX0_MSG14__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_INBOX0_RSP
+#define DMCUB_REG_INBOX0_RSP__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX0_RSP__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_OUTBOX0_RDY
+#define DMCUB_REG_OUTBOX0_RDY__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_OUTBOX0_RDY__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_OUTBOX0_MSG0
+#define DMCUB_REG_OUTBOX0_MSG0__DATA__SHIFT                                                                   0x0
+#define DMCUB_REG_OUTBOX0_MSG0__DATA_MASK                                                                     0xFFFFFFFFL
+//DMCUB_REG_OUTBOX0_RSP
+#define DMCUB_REG_OUTBOX0_RSP__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_OUTBOX0_RSP__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX1_RDY
+#define DMCUB_REG_INBOX1_RDY__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX1_RDY__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX1_MSG0
+#define DMCUB_REG_INBOX1_MSG0__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX1_MSG0__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX1_MSG1
+#define DMCUB_REG_INBOX1_MSG1__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX1_MSG1__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX1_RSP
+#define DMCUB_REG_INBOX1_RSP__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX1_RSP__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX1_INTERRUPT_CSR
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_EN__SHIFT                                    0x0
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_ACK__SHIFT                                   0x1
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_STAT__SHIFT                                  0x2
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_EN_MASK                                      0x00000001L
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_ACK_MASK                                     0x00000002L
+#define DMCUB_REG_INBOX1_INTERRUPT_CSR__DMCUB_REG_INBOX1_RDY_INT_STAT_MASK                                    0x00000004L
+//DMCUB_REG_INBOX2_RDY
+#define DMCUB_REG_INBOX2_RDY__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX2_RDY__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX2_MSG0
+#define DMCUB_REG_INBOX2_MSG0__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX2_MSG0__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX2_MSG1
+#define DMCUB_REG_INBOX2_MSG1__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX2_MSG1__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX2_RSP
+#define DMCUB_REG_INBOX2_RSP__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX2_RSP__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX3_RDY
+#define DMCUB_REG_INBOX3_RDY__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX3_RDY__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX3_MSG0
+#define DMCUB_REG_INBOX3_MSG0__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX3_MSG0__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX3_MSG1
+#define DMCUB_REG_INBOX3_MSG1__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX3_MSG1__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX3_RSP
+#define DMCUB_REG_INBOX3_RSP__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX3_RSP__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX4_RDY
+#define DMCUB_REG_INBOX4_RDY__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX4_RDY__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_REG_INBOX4_MSG0
+#define DMCUB_REG_INBOX4_MSG0__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX4_MSG0__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX4_MSG1
+#define DMCUB_REG_INBOX4_MSG1__DATA__SHIFT                                                                    0x0
+#define DMCUB_REG_INBOX4_MSG1__DATA_MASK                                                                      0xFFFFFFFFL
+//DMCUB_REG_INBOX4_RSP
+#define DMCUB_REG_INBOX4_RSP__DATA__SHIFT                                                                     0x0
+#define DMCUB_REG_INBOX4_RSP__DATA_MASK                                                                       0xFFFFFFFFL
+//DMCUB_IDLE_HYSTERESIS_CSR
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_ENABLE__SHIFT                                              0x0
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_STATE__SHIFT                                               0x1
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_THRESHOLD__SHIFT                                           0x10
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_ENABLE_MASK                                                0x00000001L
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_STATE_MASK                                                 0x00000002L
+#define DMCUB_IDLE_HYSTERESIS_CSR__IDLE_HYSTERESIS_THRESHOLD_MASK                                             0xFFFF0000L
+
+
+// addressBlock: dce_dc_mmhubbub_mcif_wb0_dispdec
+//MCIF_WB_BUFMGR_SW_CONTROL
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_ENABLE__SHIFT                                               0x0
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_INT_EN__SHIFT                                            0x4
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_INT_ACK__SHIFT                                           0x5
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_SLICE_INT_EN__SHIFT                                      0x6
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN__SHIFT                                    0x7
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_LOCK__SHIFT                                              0x8
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUF_ADDR_FENCE_EN__SHIFT                                           0x18
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_ENABLE_MASK                                                 0x00000001L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_INT_EN_MASK                                              0x00000010L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_INT_ACK_MASK                                             0x00000020L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_SLICE_INT_EN_MASK                                        0x00000040L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN_MASK                                      0x00000080L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUFMGR_SW_LOCK_MASK                                                0x00000F00L
+#define MCIF_WB_BUFMGR_SW_CONTROL__MCIF_WB_BUF_ADDR_FENCE_EN_MASK                                             0x01000000L
+//MCIF_WB_BUFMGR_STATUS
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_SW_INT_STATUS__SHIFT                                            0x0
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS__SHIFT                                    0x1
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_CUR_BUF__SHIFT                                                  0x4
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUF_DUALSIZE_STATUS__SHIFT                                             0x7
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_BUFTAG__SHIFT                                                   0x8
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_CUR_LINE_L__SHIFT                                               0xc
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_NEXT_BUF__SHIFT                                                 0x1c
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_SW_INT_STATUS_MASK                                              0x00000001L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS_MASK                                      0x00000002L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_CUR_BUF_MASK                                                    0x00000070L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUF_DUALSIZE_STATUS_MASK                                               0x00000080L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_BUFTAG_MASK                                                     0x00000F00L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_CUR_LINE_L_MASK                                                 0x01FFF000L
+#define MCIF_WB_BUFMGR_STATUS__MCIF_WB_BUFMGR_NEXT_BUF_MASK                                                   0x70000000L
+//MCIF_WB_BUF_PITCH
+#define MCIF_WB_BUF_PITCH__MCIF_WB_BUF_LUMA_PITCH__SHIFT                                                      0x8
+#define MCIF_WB_BUF_PITCH__MCIF_WB_BUF_CHROMA_PITCH__SHIFT                                                    0x18
+#define MCIF_WB_BUF_PITCH__MCIF_WB_BUF_LUMA_PITCH_MASK                                                        0x0000FF00L
+#define MCIF_WB_BUF_PITCH__MCIF_WB_BUF_CHROMA_PITCH_MASK                                                      0xFF000000L
+//MCIF_WB_BUF_1_STATUS
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_ACTIVE__SHIFT                                                     0x0
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_SW_LOCKED__SHIFT                                                  0x1
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_VCE_LOCKED__SHIFT                                                 0x2
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_OVERFLOW__SHIFT                                                   0x3
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_DISABLE__SHIFT                                                    0x4
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_MODE__SHIFT                                                       0x5
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_BUFTAG__SHIFT                                                     0x8
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_NXT_BUF__SHIFT                                                    0xc
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_CUR_LINE_L__SHIFT                                                 0x10
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_ACTIVE_MASK                                                       0x00000001L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_SW_LOCKED_MASK                                                    0x00000002L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_VCE_LOCKED_MASK                                                   0x00000004L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_OVERFLOW_MASK                                                     0x00000008L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_DISABLE_MASK                                                      0x00000010L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_MODE_MASK                                                         0x000000E0L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_BUFTAG_MASK                                                       0x00000F00L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_NXT_BUF_MASK                                                      0x00007000L
+#define MCIF_WB_BUF_1_STATUS__MCIF_WB_BUF_1_CUR_LINE_L_MASK                                                   0x1FFF0000L
+//MCIF_WB_BUF_1_STATUS2
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_NEW_CONTENT__SHIFT                                               0xd
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_COLOR_DEPTH__SHIFT                                               0xe
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_TMZ_BLACK_PIXEL__SHIFT                                           0xf
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_TMZ__SHIFT                                                       0x10
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_Y_OVERRUN__SHIFT                                                 0x14
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_C_OVERRUN__SHIFT                                                 0x15
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_EYE_FLAG__SHIFT                                                  0x16
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_NEW_CONTENT_MASK                                                 0x00002000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_COLOR_DEPTH_MASK                                                 0x00004000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_TMZ_BLACK_PIXEL_MASK                                             0x00008000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_TMZ_MASK                                                         0x000F0000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_Y_OVERRUN_MASK                                                   0x00100000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_C_OVERRUN_MASK                                                   0x00200000L
+#define MCIF_WB_BUF_1_STATUS2__MCIF_WB_BUF_1_EYE_FLAG_MASK                                                    0x00400000L
+//MCIF_WB_BUF_2_STATUS
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_ACTIVE__SHIFT                                                     0x0
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_SW_LOCKED__SHIFT                                                  0x1
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_VCE_LOCKED__SHIFT                                                 0x2
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_OVERFLOW__SHIFT                                                   0x3
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_DISABLE__SHIFT                                                    0x4
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_MODE__SHIFT                                                       0x5
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_BUFTAG__SHIFT                                                     0x8
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_NXT_BUF__SHIFT                                                    0xc
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_CUR_LINE_L__SHIFT                                                 0x10
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_ACTIVE_MASK                                                       0x00000001L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_SW_LOCKED_MASK                                                    0x00000002L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_VCE_LOCKED_MASK                                                   0x00000004L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_OVERFLOW_MASK                                                     0x00000008L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_DISABLE_MASK                                                      0x00000010L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_MODE_MASK                                                         0x000000E0L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_BUFTAG_MASK                                                       0x00000F00L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_NXT_BUF_MASK                                                      0x00007000L
+#define MCIF_WB_BUF_2_STATUS__MCIF_WB_BUF_2_CUR_LINE_L_MASK                                                   0x1FFF0000L
+//MCIF_WB_BUF_2_STATUS2
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_NEW_CONTENT__SHIFT                                               0xd
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_COLOR_DEPTH__SHIFT                                               0xe
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_TMZ_BLACK_PIXEL__SHIFT                                           0xf
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_TMZ__SHIFT                                                       0x10
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_Y_OVERRUN__SHIFT                                                 0x14
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_C_OVERRUN__SHIFT                                                 0x15
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_EYE_FLAG__SHIFT                                                  0x16
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_NEW_CONTENT_MASK                                                 0x00002000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_COLOR_DEPTH_MASK                                                 0x00004000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_TMZ_BLACK_PIXEL_MASK                                             0x00008000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_TMZ_MASK                                                         0x000F0000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_Y_OVERRUN_MASK                                                   0x00100000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_C_OVERRUN_MASK                                                   0x00200000L
+#define MCIF_WB_BUF_2_STATUS2__MCIF_WB_BUF_2_EYE_FLAG_MASK                                                    0x00400000L
+//MCIF_WB_BUF_3_STATUS
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_ACTIVE__SHIFT                                                     0x0
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_SW_LOCKED__SHIFT                                                  0x1
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_VCE_LOCKED__SHIFT                                                 0x2
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_OVERFLOW__SHIFT                                                   0x3
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_DISABLE__SHIFT                                                    0x4
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_MODE__SHIFT                                                       0x5
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_BUFTAG__SHIFT                                                     0x8
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_NXT_BUF__SHIFT                                                    0xc
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_CUR_LINE_L__SHIFT                                                 0x10
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_ACTIVE_MASK                                                       0x00000001L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_SW_LOCKED_MASK                                                    0x00000002L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_VCE_LOCKED_MASK                                                   0x00000004L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_OVERFLOW_MASK                                                     0x00000008L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_DISABLE_MASK                                                      0x00000010L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_MODE_MASK                                                         0x000000E0L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_BUFTAG_MASK                                                       0x00000F00L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_NXT_BUF_MASK                                                      0x00007000L
+#define MCIF_WB_BUF_3_STATUS__MCIF_WB_BUF_3_CUR_LINE_L_MASK                                                   0x1FFF0000L
+//MCIF_WB_BUF_3_STATUS2
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_NEW_CONTENT__SHIFT                                               0xd
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_COLOR_DEPTH__SHIFT                                               0xe
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_TMZ_BLACK_PIXEL__SHIFT                                           0xf
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_TMZ__SHIFT                                                       0x10
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_Y_OVERRUN__SHIFT                                                 0x14
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_C_OVERRUN__SHIFT                                                 0x15
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_EYE_FLAG__SHIFT                                                  0x16
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_NEW_CONTENT_MASK                                                 0x00002000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_COLOR_DEPTH_MASK                                                 0x00004000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_TMZ_BLACK_PIXEL_MASK                                             0x00008000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_TMZ_MASK                                                         0x000F0000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_Y_OVERRUN_MASK                                                   0x00100000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_C_OVERRUN_MASK                                                   0x00200000L
+#define MCIF_WB_BUF_3_STATUS2__MCIF_WB_BUF_3_EYE_FLAG_MASK                                                    0x00400000L
+//MCIF_WB_BUF_4_STATUS
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_ACTIVE__SHIFT                                                     0x0
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_SW_LOCKED__SHIFT                                                  0x1
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_VCE_LOCKED__SHIFT                                                 0x2
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_OVERFLOW__SHIFT                                                   0x3
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_DISABLE__SHIFT                                                    0x4
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_MODE__SHIFT                                                       0x5
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_BUFTAG__SHIFT                                                     0x8
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_NXT_BUF__SHIFT                                                    0xc
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_CUR_LINE_L__SHIFT                                                 0x10
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_ACTIVE_MASK                                                       0x00000001L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_SW_LOCKED_MASK                                                    0x00000002L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_VCE_LOCKED_MASK                                                   0x00000004L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_OVERFLOW_MASK                                                     0x00000008L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_DISABLE_MASK                                                      0x00000010L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_MODE_MASK                                                         0x000000E0L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_BUFTAG_MASK                                                       0x00000F00L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_NXT_BUF_MASK                                                      0x00007000L
+#define MCIF_WB_BUF_4_STATUS__MCIF_WB_BUF_4_CUR_LINE_L_MASK                                                   0x1FFF0000L
+//MCIF_WB_BUF_4_STATUS2
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_NEW_CONTENT__SHIFT                                               0xd
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_COLOR_DEPTH__SHIFT                                               0xe
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_TMZ_BLACK_PIXEL__SHIFT                                           0xf
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_TMZ__SHIFT                                                       0x10
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_Y_OVERRUN__SHIFT                                                 0x14
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_C_OVERRUN__SHIFT                                                 0x15
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_EYE_FLAG__SHIFT                                                  0x16
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_NEW_CONTENT_MASK                                                 0x00002000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_COLOR_DEPTH_MASK                                                 0x00004000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_TMZ_BLACK_PIXEL_MASK                                             0x00008000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_TMZ_MASK                                                         0x000F0000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_Y_OVERRUN_MASK                                                   0x00100000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_C_OVERRUN_MASK                                                   0x00200000L
+#define MCIF_WB_BUF_4_STATUS2__MCIF_WB_BUF_4_EYE_FLAG_MASK                                                    0x00400000L
+//MCIF_WB_ARBITRATION_CONTROL
+#define MCIF_WB_ARBITRATION_CONTROL__MCIF_WB_CLIENT_ARBITRATION_SLICE__SHIFT                                  0x0
+#define MCIF_WB_ARBITRATION_CONTROL__MCIF_WB_TIME_PER_PIXEL__SHIFT                                            0x14
+#define MCIF_WB_ARBITRATION_CONTROL__MCIF_WB_CLIENT_ARBITRATION_SLICE_MASK                                    0x00000003L
+#define MCIF_WB_ARBITRATION_CONTROL__MCIF_WB_TIME_PER_PIXEL_MASK                                              0xFFF00000L
+//MCIF_WB_SCLK_CHANGE
+#define MCIF_WB_SCLK_CHANGE__WM_CHANGE_ACK_FORCE_ON__SHIFT                                                    0x0
+#define MCIF_WB_SCLK_CHANGE__WM_CHANGE_ACK_FORCE_ON_MASK                                                      0x00000001L
+//MCIF_WB_TEST_DEBUG_INDEX
+#define MCIF_WB_TEST_DEBUG_INDEX__MCIF_WB_TEST_DEBUG_INDEX__SHIFT                                             0x0
+#define MCIF_WB_TEST_DEBUG_INDEX__MCIF_WB_TEST_DEBUG_WRITE_EN__SHIFT                                          0x8
+#define MCIF_WB_TEST_DEBUG_INDEX__MCIF_WB_TEST_DEBUG_INDEX_MASK                                               0x000000FFL
+#define MCIF_WB_TEST_DEBUG_INDEX__MCIF_WB_TEST_DEBUG_WRITE_EN_MASK                                            0x00000100L
+//MCIF_WB_TEST_DEBUG_DATA
+#define MCIF_WB_TEST_DEBUG_DATA__MCIF_WB_TEST_DEBUG_DATA__SHIFT                                               0x0
+#define MCIF_WB_TEST_DEBUG_DATA__MCIF_WB_TEST_DEBUG_DATA_MASK                                                 0xFFFFFFFFL
+//MCIF_WB_BUF_1_ADDR_Y
+#define MCIF_WB_BUF_1_ADDR_Y__MCIF_WB_BUF_1_ADDR_Y__SHIFT                                                     0x0
+#define MCIF_WB_BUF_1_ADDR_Y__MCIF_WB_BUF_1_ADDR_Y_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_1_ADDR_C
+#define MCIF_WB_BUF_1_ADDR_C__MCIF_WB_BUF_1_ADDR_C__SHIFT                                                     0x0
+#define MCIF_WB_BUF_1_ADDR_C__MCIF_WB_BUF_1_ADDR_C_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_2_ADDR_Y
+#define MCIF_WB_BUF_2_ADDR_Y__MCIF_WB_BUF_2_ADDR_Y__SHIFT                                                     0x0
+#define MCIF_WB_BUF_2_ADDR_Y__MCIF_WB_BUF_2_ADDR_Y_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_2_ADDR_C
+#define MCIF_WB_BUF_2_ADDR_C__MCIF_WB_BUF_2_ADDR_C__SHIFT                                                     0x0
+#define MCIF_WB_BUF_2_ADDR_C__MCIF_WB_BUF_2_ADDR_C_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_3_ADDR_Y
+#define MCIF_WB_BUF_3_ADDR_Y__MCIF_WB_BUF_3_ADDR_Y__SHIFT                                                     0x0
+#define MCIF_WB_BUF_3_ADDR_Y__MCIF_WB_BUF_3_ADDR_Y_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_3_ADDR_C
+#define MCIF_WB_BUF_3_ADDR_C__MCIF_WB_BUF_3_ADDR_C__SHIFT                                                     0x0
+#define MCIF_WB_BUF_3_ADDR_C__MCIF_WB_BUF_3_ADDR_C_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_4_ADDR_Y
+#define MCIF_WB_BUF_4_ADDR_Y__MCIF_WB_BUF_4_ADDR_Y__SHIFT                                                     0x0
+#define MCIF_WB_BUF_4_ADDR_Y__MCIF_WB_BUF_4_ADDR_Y_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUF_4_ADDR_C
+#define MCIF_WB_BUF_4_ADDR_C__MCIF_WB_BUF_4_ADDR_C__SHIFT                                                     0x0
+#define MCIF_WB_BUF_4_ADDR_C__MCIF_WB_BUF_4_ADDR_C_MASK                                                       0xFFFFFFFFL
+//MCIF_WB_BUFMGR_VCE_CONTROL
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_VCE_LOCK_IGNORE__SHIFT                                     0x0
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_VCE_LOCK__SHIFT                                            0x8
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_SLICE_SIZE__SHIFT                                          0x10
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_VCE_LOCK_IGNORE_MASK                                       0x00000001L
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_VCE_LOCK_MASK                                              0x00000F00L
+#define MCIF_WB_BUFMGR_VCE_CONTROL__MCIF_WB_BUFMGR_SLICE_SIZE_MASK                                            0x1FFF0000L
+//MCIF_WB_NB_PSTATE_CONTROL
+#define MCIF_WB_NB_PSTATE_CONTROL__NB_PSTATE_CHANGE_FORCE_ON__SHIFT                                           0x1
+#define MCIF_WB_NB_PSTATE_CONTROL__NB_PSTATE_CHANGE_FORCE_ON_MASK                                             0x00000002L
+//MCIF_WB_CLOCK_GATER_CONTROL
+#define MCIF_WB_CLOCK_GATER_CONTROL__MCIF_WB_CLI_CLOCK_GATER_OVERRIDE__SHIFT                                  0x0
+#define MCIF_WB_CLOCK_GATER_CONTROL__MCIF_WB_CLI_CLOCK_GATER_OVERRIDE_MASK                                    0x00000001L
+//MCIF_WB_SELF_REFRESH_CONTROL
+#define MCIF_WB_SELF_REFRESH_CONTROL__PERFRAME_SELF_REFRESH__SHIFT                                            0x1
+#define MCIF_WB_SELF_REFRESH_CONTROL__PERFRAME_SELF_REFRESH_MASK                                              0x00000002L
+//MULTI_LEVEL_QOS_CTRL
+#define MULTI_LEVEL_QOS_CTRL__MAX_SCALED_TIME_TO_URGENT__SHIFT                                                0x0
+#define MULTI_LEVEL_QOS_CTRL__MAX_SCALED_TIME_TO_URGENT_MASK                                                  0x003FFFFFL
+//MCIF_WB_SECURITY_LEVEL
+#define MCIF_WB_SECURITY_LEVEL__MCIF_WB_SECURITY_LEVEL__SHIFT                                                 0x0
+#define MCIF_WB_SECURITY_LEVEL__MCIF_WB_SPACE__SHIFT                                                          0x4
+#define MCIF_WB_SECURITY_LEVEL__MCIF_WB_SECURITY_LEVEL_MASK                                                   0x00000007L
+#define MCIF_WB_SECURITY_LEVEL__MCIF_WB_SPACE_MASK                                                            0x00000070L
+//MCIF_WB_BUF_LUMA_SIZE
+#define MCIF_WB_BUF_LUMA_SIZE__MCIF_WB_BUF_LUMA_SIZE__SHIFT                                                   0x0
+#define MCIF_WB_BUF_LUMA_SIZE__MCIF_WB_BUF_LUMA_SIZE_MASK                                                     0x000FFFFFL
+//MCIF_WB_BUF_CHROMA_SIZE
+#define MCIF_WB_BUF_CHROMA_SIZE__MCIF_WB_BUF_CHROMA_SIZE__SHIFT                                               0x0
+#define MCIF_WB_BUF_CHROMA_SIZE__MCIF_WB_BUF_CHROMA_SIZE_MASK                                                 0x000FFFFFL
+//MCIF_WB_BUF_1_ADDR_Y_HIGH
+#define MCIF_WB_BUF_1_ADDR_Y_HIGH__MCIF_WB_BUF_1_ADDR_Y_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_1_ADDR_Y_HIGH__MCIF_WB_BUF_1_ADDR_Y_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_1_ADDR_C_HIGH
+#define MCIF_WB_BUF_1_ADDR_C_HIGH__MCIF_WB_BUF_1_ADDR_C_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_1_ADDR_C_HIGH__MCIF_WB_BUF_1_ADDR_C_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_2_ADDR_Y_HIGH
+#define MCIF_WB_BUF_2_ADDR_Y_HIGH__MCIF_WB_BUF_2_ADDR_Y_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_2_ADDR_Y_HIGH__MCIF_WB_BUF_2_ADDR_Y_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_2_ADDR_C_HIGH
+#define MCIF_WB_BUF_2_ADDR_C_HIGH__MCIF_WB_BUF_2_ADDR_C_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_2_ADDR_C_HIGH__MCIF_WB_BUF_2_ADDR_C_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_3_ADDR_Y_HIGH
+#define MCIF_WB_BUF_3_ADDR_Y_HIGH__MCIF_WB_BUF_3_ADDR_Y_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_3_ADDR_Y_HIGH__MCIF_WB_BUF_3_ADDR_Y_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_3_ADDR_C_HIGH
+#define MCIF_WB_BUF_3_ADDR_C_HIGH__MCIF_WB_BUF_3_ADDR_C_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_3_ADDR_C_HIGH__MCIF_WB_BUF_3_ADDR_C_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_4_ADDR_Y_HIGH
+#define MCIF_WB_BUF_4_ADDR_Y_HIGH__MCIF_WB_BUF_4_ADDR_Y_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_4_ADDR_Y_HIGH__MCIF_WB_BUF_4_ADDR_Y_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_4_ADDR_C_HIGH
+#define MCIF_WB_BUF_4_ADDR_C_HIGH__MCIF_WB_BUF_4_ADDR_C_HIGH__SHIFT                                           0x0
+#define MCIF_WB_BUF_4_ADDR_C_HIGH__MCIF_WB_BUF_4_ADDR_C_HIGH_MASK                                             0x000000FFL
+//MCIF_WB_BUF_1_RESOLUTION
+#define MCIF_WB_BUF_1_RESOLUTION__MCIF_WB_BUF_1_RESOLUTION_WIDTH__SHIFT                                       0x0
+#define MCIF_WB_BUF_1_RESOLUTION__MCIF_WB_BUF_1_RESOLUTION_HEIGHT__SHIFT                                      0x10
+#define MCIF_WB_BUF_1_RESOLUTION__MCIF_WB_BUF_1_RESOLUTION_WIDTH_MASK                                         0x00001FFFL
+#define MCIF_WB_BUF_1_RESOLUTION__MCIF_WB_BUF_1_RESOLUTION_HEIGHT_MASK                                        0x1FFF0000L
+//MCIF_WB_BUF_2_RESOLUTION
+#define MCIF_WB_BUF_2_RESOLUTION__MCIF_WB_BUF_2_RESOLUTION_WIDTH__SHIFT                                       0x0
+#define MCIF_WB_BUF_2_RESOLUTION__MCIF_WB_BUF_2_RESOLUTION_HEIGHT__SHIFT                                      0x10
+#define MCIF_WB_BUF_2_RESOLUTION__MCIF_WB_BUF_2_RESOLUTION_WIDTH_MASK                                         0x00001FFFL
+#define MCIF_WB_BUF_2_RESOLUTION__MCIF_WB_BUF_2_RESOLUTION_HEIGHT_MASK                                        0x1FFF0000L
+//MCIF_WB_BUF_3_RESOLUTION
+#define MCIF_WB_BUF_3_RESOLUTION__MCIF_WB_BUF_3_RESOLUTION_WIDTH__SHIFT                                       0x0
+#define MCIF_WB_BUF_3_RESOLUTION__MCIF_WB_BUF_3_RESOLUTION_HEIGHT__SHIFT                                      0x10
+#define MCIF_WB_BUF_3_RESOLUTION__MCIF_WB_BUF_3_RESOLUTION_WIDTH_MASK                                         0x00001FFFL
+#define MCIF_WB_BUF_3_RESOLUTION__MCIF_WB_BUF_3_RESOLUTION_HEIGHT_MASK                                        0x1FFF0000L
+//MCIF_WB_BUF_4_RESOLUTION
+#define MCIF_WB_BUF_4_RESOLUTION__MCIF_WB_BUF_4_RESOLUTION_WIDTH__SHIFT                                       0x0
+#define MCIF_WB_BUF_4_RESOLUTION__MCIF_WB_BUF_4_RESOLUTION_HEIGHT__SHIFT                                      0x10
+#define MCIF_WB_BUF_4_RESOLUTION__MCIF_WB_BUF_4_RESOLUTION_WIDTH_MASK                                         0x00001FFFL
+#define MCIF_WB_BUF_4_RESOLUTION__MCIF_WB_BUF_4_RESOLUTION_HEIGHT_MASK                                        0x1FFF0000L
+//MCIF_WB_PSTATE_CHANGE_DURATION_VBI
+#define MCIF_WB_PSTATE_CHANGE_DURATION_VBI__MCIF_WB_UCLK_PSTATE_CHANGE_DURATION_VBI__SHIFT                    0x0
+#define MCIF_WB_PSTATE_CHANGE_DURATION_VBI__MCIF_WB_FCLK_PSTATE_CHANGE_DURATION_VBI__SHIFT                    0x10
+#define MCIF_WB_PSTATE_CHANGE_DURATION_VBI__MCIF_WB_UCLK_PSTATE_CHANGE_DURATION_VBI_MASK                      0x0000FFFFL
+#define MCIF_WB_PSTATE_CHANGE_DURATION_VBI__MCIF_WB_FCLK_PSTATE_CHANGE_DURATION_VBI_MASK                      0xFFFF0000L
+//MCIF_WB_VMID_CONTROL
+#define MCIF_WB_VMID_CONTROL__MCIF_WB_P_VMID__SHIFT                                                           0x0
+#define MCIF_WB_VMID_CONTROL__MCIF_WB_P_VMID_MASK                                                             0x0000000FL
+//MCIF_WB_MIN_TTO
+#define MCIF_WB_MIN_TTO__MCIF_WB_MIN_TTO__SHIFT                                                               0x0
+#define MCIF_WB_MIN_TTO__MCIF_WB_MIN_TTO_MASK                                                                 0x0007FFFFL
+//MCIF_WB_TMZ
+#define MCIF_WB_TMZ__MCIF_WB_TMZ__SHIFT                                                                       0x0
+#define MCIF_WB_TMZ__MCIF_WB_TMZ_MASK                                                                         0x0000000FL
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dispdec
+//MCIF_WB_NB_PSTATE_LATENCY_WATERMARK
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_REFRESH_WATERMARK__SHIFT                        0x0
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_WATERMARK_MASK__SHIFT                           0x18
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_WATERMARK_TYPE__SHIFT                           0x1f
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_REFRESH_WATERMARK_MASK                          0x001FFFFFL
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_WATERMARK_MASK_MASK                             0x07000000L
+#define MCIF_WB_NB_PSTATE_LATENCY_WATERMARK__NB_PSTATE_CHANGE_WATERMARK_TYPE_MASK                             0x80000000L
+//MCIF_WB_WATERMARK
+#define MCIF_WB_WATERMARK__MCIF_WB_CLI_WATERMARK__SHIFT                                                       0x0
+#define MCIF_WB_WATERMARK__MCIF_WB_CLI_WATERMARK_MASK__SHIFT                                                  0x18
+#define MCIF_WB_WATERMARK__MCIF_WB_CLI_WATERMARK_MASK                                                         0x001FFFFFL
+#define MCIF_WB_WATERMARK__MCIF_WB_CLI_WATERMARK_MASK_MASK                                                    0x07000000L
+//MMHUBBUB_WARMUP_CONFIG
+#define MMHUBBUB_WARMUP_CONFIG__MMHUBBUB_WARMUP_QOS__SHIFT                                                    0x10
+#define MMHUBBUB_WARMUP_CONFIG__MMHUBBUB_WARMUP_AWID__SHIFT                                                   0x14
+#define MMHUBBUB_WARMUP_CONFIG__MMHUBBUB_WARMUP_QOS_MASK                                                      0x000F0000L
+#define MMHUBBUB_WARMUP_CONFIG__MMHUBBUB_WARMUP_AWID_MASK                                                     0x00F00000L
+//MMHUBBUB_WARMUP_CONTROL_STATUS
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_EN__SHIFT                                             0x0
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_EN__SHIFT                                      0x4
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_STATUS__SHIFT                                  0x5
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_ACK__SHIFT                                     0x6
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_INC_ADDR__SHIFT                                       0x8
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_EN_MASK                                               0x00000001L
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_EN_MASK                                        0x00000010L
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_STATUS_MASK                                    0x00000020L
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_SW_INT_ACK_MASK                                       0x00000040L
+#define MMHUBBUB_WARMUP_CONTROL_STATUS__MMHUBBUB_WARMUP_INC_ADDR_MASK                                         0x03FFFF00L
+//MMHUBBUB_WARMUP_BASE_ADDR_LOW
+#define MMHUBBUB_WARMUP_BASE_ADDR_LOW__MMHUBBUB_WARMUP_BASE_ADDR_LOW__SHIFT                                   0x0
+#define MMHUBBUB_WARMUP_BASE_ADDR_LOW__MMHUBBUB_WARMUP_BASE_ADDR_LOW_MASK                                     0xFFFFFFFFL
+//MMHUBBUB_WARMUP_BASE_ADDR_HIGH
+#define MMHUBBUB_WARMUP_BASE_ADDR_HIGH__MMHUBBUB_WARMUP_BASE_ADDR_HIGH__SHIFT                                 0x0
+#define MMHUBBUB_WARMUP_BASE_ADDR_HIGH__MMHUBBUB_WARMUP_BASE_ADDR_HIGH_MASK                                   0x000007FFL
+//MMHUBBUB_WARMUP_ADDR_REGION
+#define MMHUBBUB_WARMUP_ADDR_REGION__MMHUBBUB_WARMUP_ADDR_REGION__SHIFT                                       0x0
+#define MMHUBBUB_WARMUP_ADDR_REGION__MMHUBBUB_WARMUP_ADDR_REGION_MASK                                         0x07FFFFFFL
+//MMHUBBUB_MIN_TTO
+#define MMHUBBUB_MIN_TTO__MMHUBBUB_MIN_TTO__SHIFT                                                             0x0
+#define MMHUBBUB_MIN_TTO__MMHUBBUB_MIN_TTO_MASK                                                               0x0007FFFFL
+//MMHUBBUB_CTRL
+#define MMHUBBUB_CTRL__MMHUB_SOCCLK_DS_MODE__SHIFT                                                            0x0
+#define MMHUBBUB_CTRL__MMHUB_SOCCLK_DS_MODE_MASK                                                              0x00000003L
+//WBIF_SMU_WM_CONTROL
+#define WBIF_SMU_WM_CONTROL__MCIF_WB_WM_CHG_SEL__SHIFT                                                        0x14
+#define WBIF_SMU_WM_CONTROL__MCIF_WB_WM_CHG_REQ__SHIFT                                                        0x16
+#define WBIF_SMU_WM_CONTROL__MCIF_WB_WM_CHG_SEL_MASK                                                          0x00300000L
+#define WBIF_SMU_WM_CONTROL__MCIF_WB_WM_CHG_REQ_MASK                                                          0x00400000L
+//WBIF0_MISC_CTRL
+#define WBIF0_MISC_CTRL__MCIFWB0_WR_COMBINE_TIMEOUT_THRESH__SHIFT                                             0x0
+#define WBIF0_MISC_CTRL__MCIF_WB0_SOCCLK_DS_ENABLE__SHIFT                                                     0x10
+#define WBIF0_MISC_CTRL__MCIF_WB0_WM_CHG_ACK_INT_DIS__SHIFT                                                   0x18
+#define WBIF0_MISC_CTRL__MCIF_WB0_WM_CHG_ACK_INT_STATUS__SHIFT                                                0x19
+#define WBIF0_MISC_CTRL__MCIFWB0_WR_COMBINE_TIMEOUT_THRESH_MASK                                               0x000003FFL
+#define WBIF0_MISC_CTRL__MCIF_WB0_SOCCLK_DS_ENABLE_MASK                                                       0x00010000L
+#define WBIF0_MISC_CTRL__MCIF_WB0_WM_CHG_ACK_INT_DIS_MASK                                                     0x01000000L
+#define WBIF0_MISC_CTRL__MCIF_WB0_WM_CHG_ACK_INT_STATUS_MASK                                                  0x02000000L
+//WBIF0_PHASE0_OUTSTANDING_COUNTER
+#define WBIF0_PHASE0_OUTSTANDING_COUNTER__MCIF_WB0_PHASE0_OUTSTANDING_COUNTER__SHIFT                          0x0
+#define WBIF0_PHASE0_OUTSTANDING_COUNTER__MCIF_WB0_PHASE0_OUTSTANDING_COUNTER_MASK                            0x07FFFFFFL
+//WBIF0_PHASE1_OUTSTANDING_COUNTER
+#define WBIF0_PHASE1_OUTSTANDING_COUNTER__MCIF_WB0_PHASE1_OUTSTANDING_COUNTER__SHIFT                          0x0
+#define WBIF0_PHASE1_OUTSTANDING_COUNTER__MCIF_WB0_PHASE1_OUTSTANDING_COUNTER_MASK                            0x07FFFFFFL
+//MMHUBBUB_MEM_PWR_STATUS
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_LUMA_MEM0_PWR_STATE__SHIFT                                         0x0
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_LUMA_MEM1_PWR_STATE__SHIFT                                         0x2
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_CHROMA_MEM0_PWR_STATE__SHIFT                                       0x4
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_CHROMA_MEM1_PWR_STATE__SHIFT                                       0x6
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_LUMA_MEM0_PWR_STATE_MASK                                           0x00000003L
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_LUMA_MEM1_PWR_STATE_MASK                                           0x0000000CL
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_CHROMA_MEM0_PWR_STATE_MASK                                         0x00000030L
+#define MMHUBBUB_MEM_PWR_STATUS__MCIF_DWB0_CHROMA_MEM1_PWR_STATE_MASK                                         0x000000C0L
+//MMHUBBUB_MEM_PWR_CNTL
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_FORCE__SHIFT                                                 0x2
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_DIS__SHIFT                                                   0x4
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_MODE_SEL__SHIFT                                              0x5
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_LUMA_MEM_EN_NUM__SHIFT                                               0x7
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_CHROMA_MEM_EN_NUM__SHIFT                                             0x8
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_FORCE_MASK                                                   0x0000000CL
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_DIS_MASK                                                     0x00000010L
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_MEM_PWR_MODE_SEL_MASK                                                0x00000060L
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_LUMA_MEM_EN_NUM_MASK                                                 0x00000080L
+#define MMHUBBUB_MEM_PWR_CNTL__MCIF_DWB0_CHROMA_MEM_EN_NUM_MASK                                               0x00000100L
+//MMHUBBUB_CLOCK_CNTL
+#define MMHUBBUB_CLOCK_CNTL__MMHUBBUB_TEST_CLK_SEL__SHIFT                                                     0x0
+#define MMHUBBUB_CLOCK_CNTL__DISPCLK_R_MMHUBBUB_GATE_DIS__SHIFT                                               0x5
+#define MMHUBBUB_CLOCK_CNTL__DISPCLK_G_WBIF0_GATE_DIS__SHIFT                                                  0x9
+#define MMHUBBUB_CLOCK_CNTL__SOCCLK_G_WBIF0_GATE_DIS__SHIFT                                                   0xa
+#define MMHUBBUB_CLOCK_CNTL__MMHUBBUB_FGCG_REP_DIS__SHIFT                                                     0x11
+#define MMHUBBUB_CLOCK_CNTL__MMHUBBUB_TEST_CLK_SEL_MASK                                                       0x0000001FL
+#define MMHUBBUB_CLOCK_CNTL__DISPCLK_R_MMHUBBUB_GATE_DIS_MASK                                                 0x00000020L
+#define MMHUBBUB_CLOCK_CNTL__DISPCLK_G_WBIF0_GATE_DIS_MASK                                                    0x00000200L
+#define MMHUBBUB_CLOCK_CNTL__SOCCLK_G_WBIF0_GATE_DIS_MASK                                                     0x00000400L
+#define MMHUBBUB_CLOCK_CNTL__MMHUBBUB_FGCG_REP_DIS_MASK                                                       0x00020000L
+//MMHUBBUB_SOFT_RESET
+#define MMHUBBUB_SOFT_RESET__WBIF0_SOFT_RESET__SHIFT                                                          0x2
+#define MMHUBBUB_SOFT_RESET__DMUIF_SOFT_RESET__SHIFT                                                          0x8
+#define MMHUBBUB_SOFT_RESET__WBIF0_SOFT_RESET_MASK                                                            0x00000004L
+#define MMHUBBUB_SOFT_RESET__DMUIF_SOFT_RESET_MASK                                                            0x00000100L
+//DMU_IF_ERR_STATUS
+#define DMU_IF_ERR_STATUS__DMU_RD_OUTSTANDING_ERR__SHIFT                                                      0x0
+#define DMU_IF_ERR_STATUS__DMU_RD_OUTSTANDING_ERR_CLR__SHIFT                                                  0x4
+#define DMU_IF_ERR_STATUS__DMU_RD_OUTSTANDING_ERR_MASK                                                        0x00000001L
+#define DMU_IF_ERR_STATUS__DMU_RD_OUTSTANDING_ERR_CLR_MASK                                                    0x00000010L
+//MMHUBBUB_CLIENT_UNIT_ID
+#define MMHUBBUB_CLIENT_UNIT_ID__WBIF0_UNIT_ID__SHIFT                                                         0x8
+#define MMHUBBUB_CLIENT_UNIT_ID__WBIF0_UNIT_ID_MASK                                                           0x00003F00L
+//MMHUBBUB_WARMUP_SECURITY_CFG
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_UNITID__SHIFT                                           0x0
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_SECURE_LVL__SHIFT                                       0x8
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_SPACE__SHIFT                                            0x10
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_UNITID_MASK                                             0x0000003FL
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_SECURE_LVL_MASK                                         0x00000700L
+#define MMHUBBUB_WARMUP_SECURITY_CFG__MMHUBBUB_WARMUP_SPACE_MASK                                              0x00070000L
+//MMHUBBUB_WARMUP_VMID_CONTROL
+#define MMHUBBUB_WARMUP_VMID_CONTROL__MMHUBBUB_WARMUP_P_VMID__SHIFT                                           0x0
+#define MMHUBBUB_WARMUP_VMID_CONTROL__MMHUBBUB_WARMUP_P_VMID_MASK                                             0x0000000FL
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON3_PERFCOUNTER_CNTL
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON3_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON3_PERFCOUNTER_CNTL2
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON3_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON3_PERFCOUNTER_STATE
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON3_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON3_PERFMON_CNTL
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON3_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON3_PERFMON_CNTL2
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON3_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON3_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON3_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON3_PERFMON_CVALUE_LOW
+#define DC_PERFMON3_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON3_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON3_PERFMON_HI
+#define DC_PERFMON3_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON3_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON3_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON3_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON3_PERFMON_LOW
+#define DC_PERFMON3_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON3_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream0_dispdec
+//AZF0STREAM0_AZALIA_STREAM_INDEX
+#define AZF0STREAM0_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM0_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM0_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM0_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM0_AZALIA_STREAM_DATA
+#define AZF0STREAM0_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM0_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream1_dispdec
+//AZF0STREAM1_AZALIA_STREAM_INDEX
+#define AZF0STREAM1_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM1_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM1_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM1_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM1_AZALIA_STREAM_DATA
+#define AZF0STREAM1_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM1_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream2_dispdec
+//AZF0STREAM2_AZALIA_STREAM_INDEX
+#define AZF0STREAM2_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM2_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM2_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM2_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM2_AZALIA_STREAM_DATA
+#define AZF0STREAM2_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM2_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream3_dispdec
+//AZF0STREAM3_AZALIA_STREAM_INDEX
+#define AZF0STREAM3_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM3_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM3_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM3_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM3_AZALIA_STREAM_DATA
+#define AZF0STREAM3_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM3_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream4_dispdec
+//AZF0STREAM4_AZALIA_STREAM_INDEX
+#define AZF0STREAM4_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM4_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM4_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM4_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM4_AZALIA_STREAM_DATA
+#define AZF0STREAM4_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM4_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream5_dispdec
+//AZF0STREAM5_AZALIA_STREAM_INDEX
+#define AZF0STREAM5_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM5_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM5_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM5_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM5_AZALIA_STREAM_DATA
+#define AZF0STREAM5_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM5_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream6_dispdec
+//AZF0STREAM6_AZALIA_STREAM_INDEX
+#define AZF0STREAM6_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM6_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM6_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM6_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM6_AZALIA_STREAM_DATA
+#define AZF0STREAM6_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM6_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream7_dispdec
+//AZF0STREAM7_AZALIA_STREAM_INDEX
+#define AZF0STREAM7_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM7_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM7_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM7_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM7_AZALIA_STREAM_DATA
+#define AZF0STREAM7_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM7_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_az_misc_dispdec
+//AZ_CLOCK_CNTL
+#define AZ_CLOCK_CNTL__SCLK_G_STREAM_AZ_GATE_DIS__SHIFT                                                       0x0
+#define AZ_CLOCK_CNTL__AZ_GLOBAL_FGCG_REP_DIS__SHIFT                                                          0x1
+#define AZ_CLOCK_CNTL__SCLK_R_AZ_GATE_DIS__SHIFT                                                              0x4
+#define AZ_CLOCK_CNTL__SCLK_G_CNTL_AZ_GATE_DIS__SHIFT                                                         0x8
+#define AZ_CLOCK_CNTL__DCIPG_TEST_CLK_SEL__SHIFT                                                              0xc
+#define AZ_CLOCK_CNTL__SCLK_GATE_DIS__SHIFT                                                                   0x10
+#define AZ_CLOCK_CNTL__SCLK_TURN_ON_DELAY__SHIFT                                                              0x14
+#define AZ_CLOCK_CNTL__SCLK_TURN_OFF_DELAY__SHIFT                                                             0x18
+#define AZ_CLOCK_CNTL__SCLK_G_STREAM_AZ_GATE_DIS_MASK                                                         0x00000001L
+#define AZ_CLOCK_CNTL__AZ_GLOBAL_FGCG_REP_DIS_MASK                                                            0x00000002L
+#define AZ_CLOCK_CNTL__SCLK_R_AZ_GATE_DIS_MASK                                                                0x00000010L
+#define AZ_CLOCK_CNTL__SCLK_G_CNTL_AZ_GATE_DIS_MASK                                                           0x00000100L
+#define AZ_CLOCK_CNTL__DCIPG_TEST_CLK_SEL_MASK                                                                0x0000F000L
+#define AZ_CLOCK_CNTL__SCLK_GATE_DIS_MASK                                                                     0x00010000L
+#define AZ_CLOCK_CNTL__SCLK_TURN_ON_DELAY_MASK                                                                0x00F00000L
+#define AZ_CLOCK_CNTL__SCLK_TURN_OFF_DELAY_MASK                                                               0xFF000000L
+//AZ_MEM_GLOBAL_PWR_REQ_CNTL
+#define AZ_MEM_GLOBAL_PWR_REQ_CNTL__AZ_MEM_GLOBAL_PWR_REQ_DIS__SHIFT                                          0x0
+#define AZ_MEM_GLOBAL_PWR_REQ_CNTL__AZ_MEM_GLOBAL_PWR_REQ_DIS_MASK                                            0x00000001L
+//AZ_STRAPS
+#define AZ_STRAPS__AUDIO_PORT_CONNECTIVITY__SHIFT                                                             0x0
+#define AZ_STRAPS__AUDIO_STREAM_NUMBER__SHIFT                                                                 0x4
+#define AZ_STRAPS__AUDIO_PORT_CONNECTIVITY_MASK                                                               0x00000007L
+#define AZ_STRAPS__AUDIO_STREAM_NUMBER_MASK                                                                   0x00000070L
+
+
+// addressBlock: dce_dc_hda_az_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON4_PERFCOUNTER_CNTL
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON4_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON4_PERFCOUNTER_CNTL2
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON4_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON4_PERFCOUNTER_STATE
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON4_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON4_PERFMON_CNTL
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON4_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON4_PERFMON_CNTL2
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON4_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON4_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON4_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON4_PERFMON_CVALUE_LOW
+#define DC_PERFMON4_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON4_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON4_PERFMON_HI
+#define DC_PERFMON4_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON4_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON4_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON4_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON4_PERFMON_LOW
+#define DC_PERFMON4_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON4_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint0_dispdec
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint1_dispdec
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint2_dispdec
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint3_dispdec
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint4_dispdec
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint5_dispdec
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint6_dispdec
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0endpoint7_dispdec
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT                        0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK                          0x00003FFFL
+//AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT                          0x0
+#define AZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0controller_dispdec
+//AZALIA_CONTROLLER_CLOCK_GATING
+#define AZALIA_CONTROLLER_CLOCK_GATING__ENABLE_CLOCK_GATING__SHIFT                                            0x0
+#define AZALIA_CONTROLLER_CLOCK_GATING__CLOCK_ON_STATE__SHIFT                                                 0x4
+#define AZALIA_CONTROLLER_CLOCK_GATING__ENABLE_CLOCK_GATING_MASK                                              0x00000001L
+#define AZALIA_CONTROLLER_CLOCK_GATING__CLOCK_ON_STATE_MASK                                                   0x00000010L
+//AZALIA_AUDIO_DTO
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_PHASE__SHIFT                                                       0x0
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_MODULE__SHIFT                                                      0x10
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_PHASE_MASK                                                         0x0000FFFFL
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_MODULE_MASK                                                        0xFFFF0000L
+//AZALIA_AUDIO_DTO_CONTROL
+#define AZALIA_AUDIO_DTO_CONTROL__AZALIA_AUDIO_FORCE_DTO__SHIFT                                               0x8
+#define AZALIA_AUDIO_DTO_CONTROL__AZALIA_AUDIO_FORCE_DTO_MASK                                                 0x00000300L
+//AZALIA_SOCCLK_CONTROL
+#define AZALIA_SOCCLK_CONTROL__AUDIO_STREAM_SOCCLK_DEEP_SLEEP_EXIT_EN__SHIFT                                  0x1
+#define AZALIA_SOCCLK_CONTROL__AUDIO_STREAM_SOCCLK_DEEP_SLEEP_EXIT_EN_MASK                                    0x00000002L
+//AZALIA_UNDERFLOW_FILLER_SAMPLE
+#define AZALIA_UNDERFLOW_FILLER_SAMPLE__AZALIA_UNDERFLOW_FILLER_SAMPLE__SHIFT                                 0x0
+#define AZALIA_UNDERFLOW_FILLER_SAMPLE__AZALIA_UNDERFLOW_FILLER_SAMPLE_MASK                                   0xFFFFFFFFL
+//AZALIA_DATA_DMA_CONTROL
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_NON_SNOOP__SHIFT                                                    0x0
+#define AZALIA_DATA_DMA_CONTROL__INPUT_DATA_DMA_NON_SNOOP__SHIFT                                              0x2
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_ISOCHRONOUS__SHIFT                                                  0x4
+#define AZALIA_DATA_DMA_CONTROL__INPUT_DATA_DMA_ISOCHRONOUS__SHIFT                                            0x6
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_IOC_GENERATION_METHOD__SHIFT                                          0x10
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_UNDERFLOW_CONTROL__SHIFT                                              0x11
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_NON_SNOOP_MASK                                                      0x00000003L
+#define AZALIA_DATA_DMA_CONTROL__INPUT_DATA_DMA_NON_SNOOP_MASK                                                0x0000000CL
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_ISOCHRONOUS_MASK                                                    0x00000030L
+#define AZALIA_DATA_DMA_CONTROL__INPUT_DATA_DMA_ISOCHRONOUS_MASK                                              0x000000C0L
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_IOC_GENERATION_METHOD_MASK                                            0x00010000L
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_UNDERFLOW_CONTROL_MASK                                                0x00020000L
+//AZALIA_BDL_DMA_CONTROL
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_NON_SNOOP__SHIFT                                                      0x0
+#define AZALIA_BDL_DMA_CONTROL__INPUT_BDL_DMA_NON_SNOOP__SHIFT                                                0x2
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_ISOCHRONOUS__SHIFT                                                    0x4
+#define AZALIA_BDL_DMA_CONTROL__INPUT_BDL_DMA_ISOCHRONOUS__SHIFT                                              0x6
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_NON_SNOOP_MASK                                                        0x00000003L
+#define AZALIA_BDL_DMA_CONTROL__INPUT_BDL_DMA_NON_SNOOP_MASK                                                  0x0000000CL
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_ISOCHRONOUS_MASK                                                      0x00000030L
+#define AZALIA_BDL_DMA_CONTROL__INPUT_BDL_DMA_ISOCHRONOUS_MASK                                                0x000000C0L
+//AZALIA_RIRB_AND_DP_CONTROL
+#define AZALIA_RIRB_AND_DP_CONTROL__RIRB_NON_SNOOP__SHIFT                                                     0x0
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_DMA_NON_SNOOP__SHIFT                                                   0x4
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_UPDATE_FREQ_DIVIDER__SHIFT                                             0x5
+#define AZALIA_RIRB_AND_DP_CONTROL__RIRB_NON_SNOOP_MASK                                                       0x00000001L
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_DMA_NON_SNOOP_MASK                                                     0x00000010L
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_UPDATE_FREQ_DIVIDER_MASK                                               0x000001E0L
+//AZALIA_CORB_DMA_CONTROL
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_NON_SNOOP__SHIFT                                                    0x0
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_ISOCHRONOUS__SHIFT                                                  0x4
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_NON_SNOOP_MASK                                                      0x00000001L
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_ISOCHRONOUS_MASK                                                    0x00000010L
+//AZALIA_GLOBAL_CAPABILITIES
+#define AZALIA_GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS__SHIFT                               0x1
+#define AZALIA_GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS_MASK                                 0x00000006L
+//AZALIA_OUTPUT_PAYLOAD_CAPABILITY
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY__SHIFT                                    0x0
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTSTRMPAY__SHIFT                                                   0x10
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY_MASK                                      0x0000FFFFL
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTSTRMPAY_MASK                                                     0xFFFF0000L
+//AZALIA_OUTPUT_STREAM_ARBITER_CONTROL
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__LATENCY_HIDING_LEVEL__SHIFT                                     0x0
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__SYS_MEM_ACTIVE_ENABLE__SHIFT                                    0x8
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__INPUT_LATENCY_HIDING_LEVEL__SHIFT                               0x10
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__LATENCY_HIDING_LEVEL_MASK                                       0x000000FFL
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__SYS_MEM_ACTIVE_ENABLE_MASK                                      0x00000100L
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__INPUT_LATENCY_HIDING_LEVEL_MASK                                 0x00FF0000L
+//AZALIA_INPUT_PAYLOAD_CAPABILITY
+#define AZALIA_INPUT_PAYLOAD_CAPABILITY__INPUT_PAYLOAD_CAPABILITY__SHIFT                                      0x0
+#define AZALIA_INPUT_PAYLOAD_CAPABILITY__INSTRMPAY__SHIFT                                                     0x10
+#define AZALIA_INPUT_PAYLOAD_CAPABILITY__INPUT_PAYLOAD_CAPABILITY_MASK                                        0x0000FFFFL
+#define AZALIA_INPUT_PAYLOAD_CAPABILITY__INSTRMPAY_MASK                                                       0xFFFF0000L
+//AZALIA_INPUT_CRC0_CONTROL0
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_EN__SHIFT                                                       0x0
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_BLOCK_MODE__SHIFT                                               0x4
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_INSTANCE_SEL__SHIFT                                             0x8
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_EN_MASK                                                         0x00000001L
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_BLOCK_MODE_MASK                                                 0x00000010L
+#define AZALIA_INPUT_CRC0_CONTROL0__INPUT_CRC_INSTANCE_SEL_MASK                                               0x00000700L
+//AZALIA_INPUT_CRC0_CONTROL1
+#define AZALIA_INPUT_CRC0_CONTROL1__INPUT_CRC_BLOCK_SIZE__SHIFT                                               0x0
+#define AZALIA_INPUT_CRC0_CONTROL1__INPUT_CRC_BLOCK_SIZE_MASK                                                 0xFFFFFFFFL
+//AZALIA_INPUT_CRC0_CONTROL2
+#define AZALIA_INPUT_CRC0_CONTROL2__INPUT_CRC_BLOCK_ITERATION__SHIFT                                          0x0
+#define AZALIA_INPUT_CRC0_CONTROL2__INPUT_CRC_BLOCK_ITERATION_MASK                                            0x0000FFFFL
+//AZALIA_INPUT_CRC0_CONTROL3
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_COMPLETE__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_BLOCK_COMPLETE_PHASE__SHIFT                                     0x4
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_CHANNEL_RESULT_SEL__SHIFT                                       0x8
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_COMPLETE_MASK                                                   0x00000001L
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_BLOCK_COMPLETE_PHASE_MASK                                       0x00000010L
+#define AZALIA_INPUT_CRC0_CONTROL3__INPUT_CRC_CHANNEL_RESULT_SEL_MASK                                         0x00000700L
+//AZALIA_INPUT_CRC0_RESULT
+#define AZALIA_INPUT_CRC0_RESULT__INPUT_CRC_RESULT__SHIFT                                                     0x0
+#define AZALIA_INPUT_CRC0_RESULT__INPUT_CRC_RESULT_MASK                                                       0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CONTROL0
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_EN__SHIFT                                                       0x0
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_BLOCK_MODE__SHIFT                                               0x4
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_INSTANCE_SEL__SHIFT                                             0x8
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_EN_MASK                                                         0x00000001L
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_BLOCK_MODE_MASK                                                 0x00000010L
+#define AZALIA_INPUT_CRC1_CONTROL0__INPUT_CRC_INSTANCE_SEL_MASK                                               0x00000700L
+//AZALIA_INPUT_CRC1_CONTROL1
+#define AZALIA_INPUT_CRC1_CONTROL1__INPUT_CRC_BLOCK_SIZE__SHIFT                                               0x0
+#define AZALIA_INPUT_CRC1_CONTROL1__INPUT_CRC_BLOCK_SIZE_MASK                                                 0xFFFFFFFFL
+//AZALIA_INPUT_CRC1_CONTROL2
+#define AZALIA_INPUT_CRC1_CONTROL2__INPUT_CRC_BLOCK_ITERATION__SHIFT                                          0x0
+#define AZALIA_INPUT_CRC1_CONTROL2__INPUT_CRC_BLOCK_ITERATION_MASK                                            0x0000FFFFL
+//AZALIA_INPUT_CRC1_CONTROL3
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_COMPLETE__SHIFT                                                 0x0
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_BLOCK_COMPLETE_PHASE__SHIFT                                     0x4
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_CHANNEL_RESULT_SEL__SHIFT                                       0x8
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_COMPLETE_MASK                                                   0x00000001L
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_BLOCK_COMPLETE_PHASE_MASK                                       0x00000010L
+#define AZALIA_INPUT_CRC1_CONTROL3__INPUT_CRC_CHANNEL_RESULT_SEL_MASK                                         0x00000700L
+//AZALIA_INPUT_CRC1_RESULT
+#define AZALIA_INPUT_CRC1_RESULT__INPUT_CRC_RESULT__SHIFT                                                     0x0
+#define AZALIA_INPUT_CRC1_RESULT__INPUT_CRC_RESULT_MASK                                                       0xFFFFFFFFL
+//AZALIA_CRC0_CONTROL0
+#define AZALIA_CRC0_CONTROL0__CRC_EN__SHIFT                                                                   0x0
+#define AZALIA_CRC0_CONTROL0__CRC_BLOCK_MODE__SHIFT                                                           0x4
+#define AZALIA_CRC0_CONTROL0__CRC_INSTANCE_SEL__SHIFT                                                         0x8
+#define AZALIA_CRC0_CONTROL0__CRC_SOURCE_SEL__SHIFT                                                           0xc
+#define AZALIA_CRC0_CONTROL0__CRC_EN_MASK                                                                     0x00000001L
+#define AZALIA_CRC0_CONTROL0__CRC_BLOCK_MODE_MASK                                                             0x00000010L
+#define AZALIA_CRC0_CONTROL0__CRC_INSTANCE_SEL_MASK                                                           0x00000700L
+#define AZALIA_CRC0_CONTROL0__CRC_SOURCE_SEL_MASK                                                             0x00001000L
+//AZALIA_CRC0_CONTROL1
+#define AZALIA_CRC0_CONTROL1__CRC_BLOCK_SIZE__SHIFT                                                           0x0
+#define AZALIA_CRC0_CONTROL1__CRC_BLOCK_SIZE_MASK                                                             0xFFFFFFFFL
+//AZALIA_CRC0_CONTROL2
+#define AZALIA_CRC0_CONTROL2__CRC_BLOCK_ITERATION__SHIFT                                                      0x0
+#define AZALIA_CRC0_CONTROL2__CRC_BLOCK_ITERATION_MASK                                                        0x0000FFFFL
+//AZALIA_CRC0_CONTROL3
+#define AZALIA_CRC0_CONTROL3__CRC_COMPLETE__SHIFT                                                             0x0
+#define AZALIA_CRC0_CONTROL3__CRC_BLOCK_COMPLETE_PHASE__SHIFT                                                 0x4
+#define AZALIA_CRC0_CONTROL3__CRC_CHANNEL_RESULT_SEL__SHIFT                                                   0x8
+#define AZALIA_CRC0_CONTROL3__CRC_COMPLETE_MASK                                                               0x00000001L
+#define AZALIA_CRC0_CONTROL3__CRC_BLOCK_COMPLETE_PHASE_MASK                                                   0x00000010L
+#define AZALIA_CRC0_CONTROL3__CRC_CHANNEL_RESULT_SEL_MASK                                                     0x00000700L
+//AZALIA_CRC0_RESULT
+#define AZALIA_CRC0_RESULT__CRC_RESULT__SHIFT                                                                 0x0
+#define AZALIA_CRC0_RESULT__CRC_RESULT_MASK                                                                   0xFFFFFFFFL
+//AZALIA_CRC1_CONTROL0
+#define AZALIA_CRC1_CONTROL0__CRC_EN__SHIFT                                                                   0x0
+#define AZALIA_CRC1_CONTROL0__CRC_BLOCK_MODE__SHIFT                                                           0x4
+#define AZALIA_CRC1_CONTROL0__CRC_INSTANCE_SEL__SHIFT                                                         0x8
+#define AZALIA_CRC1_CONTROL0__CRC_SOURCE_SEL__SHIFT                                                           0xc
+#define AZALIA_CRC1_CONTROL0__CRC_EN_MASK                                                                     0x00000001L
+#define AZALIA_CRC1_CONTROL0__CRC_BLOCK_MODE_MASK                                                             0x00000010L
+#define AZALIA_CRC1_CONTROL0__CRC_INSTANCE_SEL_MASK                                                           0x00000700L
+#define AZALIA_CRC1_CONTROL0__CRC_SOURCE_SEL_MASK                                                             0x00001000L
+//AZALIA_CRC1_CONTROL1
+#define AZALIA_CRC1_CONTROL1__CRC_BLOCK_SIZE__SHIFT                                                           0x0
+#define AZALIA_CRC1_CONTROL1__CRC_BLOCK_SIZE_MASK                                                             0xFFFFFFFFL
+//AZALIA_CRC1_CONTROL2
+#define AZALIA_CRC1_CONTROL2__CRC_BLOCK_ITERATION__SHIFT                                                      0x0
+#define AZALIA_CRC1_CONTROL2__CRC_BLOCK_ITERATION_MASK                                                        0x0000FFFFL
+//AZALIA_CRC1_CONTROL3
+#define AZALIA_CRC1_CONTROL3__CRC_COMPLETE__SHIFT                                                             0x0
+#define AZALIA_CRC1_CONTROL3__CRC_BLOCK_COMPLETE_PHASE__SHIFT                                                 0x4
+#define AZALIA_CRC1_CONTROL3__CRC_CHANNEL_RESULT_SEL__SHIFT                                                   0x8
+#define AZALIA_CRC1_CONTROL3__CRC_COMPLETE_MASK                                                               0x00000001L
+#define AZALIA_CRC1_CONTROL3__CRC_BLOCK_COMPLETE_PHASE_MASK                                                   0x00000010L
+#define AZALIA_CRC1_CONTROL3__CRC_CHANNEL_RESULT_SEL_MASK                                                     0x00000700L
+//AZALIA_CRC1_RESULT
+#define AZALIA_CRC1_RESULT__CRC_RESULT__SHIFT                                                                 0x0
+#define AZALIA_CRC1_RESULT__CRC_RESULT_MASK                                                                   0xFFFFFFFFL
+//AZALIA_MEM_PWR_CTRL
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_FORCE__SHIFT                                                          0x0
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_DIS__SHIFT                                                            0x2
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM0_MEM_PWR_FORCE__SHIFT                                            0x3
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM0_MEM_PWR_DIS__SHIFT                                              0x5
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM1_MEM_PWR_FORCE__SHIFT                                            0x6
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM1_MEM_PWR_DIS__SHIFT                                              0x8
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM2_MEM_PWR_FORCE__SHIFT                                            0x9
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM2_MEM_PWR_DIS__SHIFT                                              0xb
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM3_MEM_PWR_FORCE__SHIFT                                            0xc
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM3_MEM_PWR_DIS__SHIFT                                              0xe
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM4_MEM_PWR_FORCE__SHIFT                                            0xf
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM4_MEM_PWR_DIS__SHIFT                                              0x11
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM5_MEM_PWR_FORCE__SHIFT                                            0x12
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM5_MEM_PWR_DIS__SHIFT                                              0x14
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_MODE_SEL__SHIFT                                                       0x1c
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_FORCE_MASK                                                            0x00000003L
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_DIS_MASK                                                              0x00000004L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM0_MEM_PWR_FORCE_MASK                                              0x00000018L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM0_MEM_PWR_DIS_MASK                                                0x00000020L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM1_MEM_PWR_FORCE_MASK                                              0x000000C0L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM1_MEM_PWR_DIS_MASK                                                0x00000100L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM2_MEM_PWR_FORCE_MASK                                              0x00000600L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM2_MEM_PWR_DIS_MASK                                                0x00000800L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM3_MEM_PWR_FORCE_MASK                                              0x00003000L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM3_MEM_PWR_DIS_MASK                                                0x00004000L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM4_MEM_PWR_FORCE_MASK                                              0x00018000L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM4_MEM_PWR_DIS_MASK                                                0x00020000L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM5_MEM_PWR_FORCE_MASK                                              0x000C0000L
+#define AZALIA_MEM_PWR_CTRL__AZ_INPUT_STREAM5_MEM_PWR_DIS_MASK                                                0x00100000L
+#define AZALIA_MEM_PWR_CTRL__AZ_MEM_PWR_MODE_SEL_MASK                                                         0x30000000L
+//AZALIA_MEM_PWR_STATUS
+#define AZALIA_MEM_PWR_STATUS__AZ_MEM_PWR_STATE__SHIFT                                                        0x0
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM0_MEM_PWR_STATE__SHIFT                                          0x2
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM1_MEM_PWR_STATE__SHIFT                                          0x4
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM2_MEM_PWR_STATE__SHIFT                                          0x6
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM3_MEM_PWR_STATE__SHIFT                                          0x8
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM4_MEM_PWR_STATE__SHIFT                                          0xa
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM5_MEM_PWR_STATE__SHIFT                                          0xc
+#define AZALIA_MEM_PWR_STATUS__AZ_MEM_PWR_STATE_MASK                                                          0x00000003L
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM0_MEM_PWR_STATE_MASK                                            0x0000000CL
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM1_MEM_PWR_STATE_MASK                                            0x00000030L
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM2_MEM_PWR_STATE_MASK                                            0x000000C0L
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM3_MEM_PWR_STATE_MASK                                            0x00000300L
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM4_MEM_PWR_STATE_MASK                                            0x00000C00L
+#define AZALIA_MEM_PWR_STATUS__AZ_INPUT_STREAM5_MEM_PWR_STATE_MASK                                            0x00003000L
+
+
+// addressBlock: dce_dc_hda_azf0root_dispdec
+//AZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__SHIFT  0x0
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_MASK  0xFFFFFFFFL
+//AZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID__SHIFT            0x0
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID_MASK              0xFFFFFFFFL
+//AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__HBR_CHANNEL_COUNT__SHIFT                                       0x0
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__COMPRESSED_CHANNEL_COUNT__SHIFT                                0x4
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__HBR_CHANNEL_COUNT_MASK                                         0x00000007L
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__COMPRESSED_CHANNEL_COUNT_MASK                                  0x00000070L
+//AZALIA_F0_CODEC_RESYNC_FIFO_CONTROL
+#define AZALIA_F0_CODEC_RESYNC_FIFO_CONTROL__RESYNC_FIFO_STARTUP_KEEPOUT_WINDOW__SHIFT                        0x0
+#define AZALIA_F0_CODEC_RESYNC_FIFO_CONTROL__RESYNC_FIFO_STARTUP_KEEPOUT_WINDOW_MASK                          0x0000003FL
+//AZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__SHIFT      0x0
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_MASK        0xFFFFFFFFL
+//AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT               0x0
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT                0x10
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK                 0x00000FFFL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK                  0x001F0000L
+//AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__SHIFT  0x0
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_MASK  0xFFFFFFFFL
+//AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES__SHIFT  0x0
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP__SHIFT                                       0x1e
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS__SHIFT                                          0x1f
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES_MASK    0x3FFFFFFFL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP_MASK                                         0x40000000L
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS_MASK                                            0x80000000L
+//AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET__SHIFT                                  0x0
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT__SHIFT                                  0x4
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK__SHIFT                                        0x9
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET__SHIFT                       0xa
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET_MASK                                    0x0000000FL
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT_MASK                                    0x000000F0L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK_MASK                                          0x00000200L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET_MASK                         0x00000400L
+//AZALIA_F0_CODEC_FUNCTION_CONTROL_RESET
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET__SHIFT                                            0x0
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET_MASK                                              0x00000001L
+//AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0__SHIFT                     0x0
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1__SHIFT                     0x8
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2__SHIFT                     0x10
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3__SHIFT                     0x18
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0_MASK                       0x000000FFL
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1_MASK                       0x0000FF00L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2_MASK                       0x00FF0000L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3_MASK                       0xFF000000L
+//AZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION__SHIFT          0x0
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION_MASK            0x000000FFL
+//CC_RCU_DC_AUDIO_PORT_CONNECTIVITY
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY__SHIFT                                           0x0
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_OVERRIDE_ENABLE__SHIFT                           0x4
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_MASK                                             0x00000007L
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_OVERRIDE_ENABLE_MASK                             0x00000010L
+//CC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY
+#define CC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY__INPUT_PORT_CONNECTIVITY__SHIFT                               0x0
+#define CC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY__INPUT_PORT_CONNECTIVITY_OVERRIDE_ENABLE__SHIFT               0x4
+#define CC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY__INPUT_PORT_CONNECTIVITY_MASK                                 0x00000007L
+#define CC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY__INPUT_PORT_CONNECTIVITY_OVERRIDE_ENABLE_MASK                 0x00000010L
+//REG_DC_AUDIO_PORT_CONNECTIVITY
+#define REG_DC_AUDIO_PORT_CONNECTIVITY__REG_PORT_CONNECTIVITY__SHIFT                                          0x0
+#define REG_DC_AUDIO_PORT_CONNECTIVITY__REG_PORT_CONNECTIVITY_OVERRIDE_ENABLE__SHIFT                          0x4
+#define REG_DC_AUDIO_PORT_CONNECTIVITY__REG_PORT_CONNECTIVITY_MASK                                            0x00000007L
+#define REG_DC_AUDIO_PORT_CONNECTIVITY__REG_PORT_CONNECTIVITY_OVERRIDE_ENABLE_MASK                            0x00000010L
+//REG_DC_AUDIO_INPUT_PORT_CONNECTIVITY
+#define REG_DC_AUDIO_INPUT_PORT_CONNECTIVITY__REG_INPUT_PORT_CONNECTIVITY__SHIFT                              0x0
+#define REG_DC_AUDIO_INPUT_PORT_CONNECTIVITY__REG_INPUT_PORT_CONNECTIVITY_OVERRIDE_ENABLE__SHIFT              0x4
+#define REG_DC_AUDIO_INPUT_PORT_CONNECTIVITY__REG_INPUT_PORT_CONNECTIVITY_MASK                                0x00000007L
+#define REG_DC_AUDIO_INPUT_PORT_CONNECTIVITY__REG_INPUT_PORT_CONNECTIVITY_OVERRIDE_ENABLE_MASK                0x00000010L
+
+
+// addressBlock: dce_dc_hda_azf0stream8_dispdec
+//AZF0STREAM8_AZALIA_STREAM_INDEX
+#define AZF0STREAM8_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM8_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM8_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM8_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM8_AZALIA_STREAM_DATA
+#define AZF0STREAM8_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM8_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream9_dispdec
+//AZF0STREAM9_AZALIA_STREAM_INDEX
+#define AZF0STREAM9_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                       0x0
+#define AZF0STREAM9_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                    0x8
+#define AZF0STREAM9_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                         0x000000FFL
+#define AZF0STREAM9_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                      0x00000100L
+//AZF0STREAM9_AZALIA_STREAM_DATA
+#define AZF0STREAM9_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                         0x0
+#define AZF0STREAM9_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream10_dispdec
+//AZF0STREAM10_AZALIA_STREAM_INDEX
+#define AZF0STREAM10_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM10_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM10_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM10_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM10_AZALIA_STREAM_DATA
+#define AZF0STREAM10_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM10_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream11_dispdec
+//AZF0STREAM11_AZALIA_STREAM_INDEX
+#define AZF0STREAM11_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM11_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM11_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM11_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM11_AZALIA_STREAM_DATA
+#define AZF0STREAM11_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM11_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream12_dispdec
+//AZF0STREAM12_AZALIA_STREAM_INDEX
+#define AZF0STREAM12_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM12_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM12_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM12_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM12_AZALIA_STREAM_DATA
+#define AZF0STREAM12_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM12_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream13_dispdec
+//AZF0STREAM13_AZALIA_STREAM_INDEX
+#define AZF0STREAM13_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM13_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM13_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM13_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM13_AZALIA_STREAM_DATA
+#define AZF0STREAM13_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM13_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream14_dispdec
+//AZF0STREAM14_AZALIA_STREAM_INDEX
+#define AZF0STREAM14_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM14_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM14_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM14_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM14_AZALIA_STREAM_DATA
+#define AZF0STREAM14_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM14_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0stream15_dispdec
+//AZF0STREAM15_AZALIA_STREAM_INDEX
+#define AZF0STREAM15_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT                                      0x0
+#define AZF0STREAM15_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT                                   0x8
+#define AZF0STREAM15_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK                                        0x000000FFL
+#define AZF0STREAM15_AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK                                     0x00000100L
+//AZF0STREAM15_AZALIA_STREAM_DATA
+#define AZF0STREAM15_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT                                        0x0
+#define AZF0STREAM15_AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint0_dispdec
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint1_dispdec
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint2_dispdec
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint3_dispdec
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint4_dispdec
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint5_dispdec
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint6_dispdec
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint7_dispdec
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX__SHIFT       0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX__AZALIA_INPUT_ENDPOINT_REG_INDEX_MASK         0x00003FFFL
+//AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA__SHIFT         0x0
+#define AZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA__AZALIA_INPUT_ENDPOINT_REG_DATA_MASK           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_sdpif_dispdec
+//DCHUBBUB_SDPIF_CFG0
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_NO_OUTSTANDING_REQ__SHIFT                                                  0x0
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_PORT_STATUS__SHIFT                                                         0x1
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_DATA_RESPONSE_STATUS__SHIFT                                                0x3
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_RESPONSE_STATUS__SHIFT                                                     0x6
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_ERROR__SHIFT                                                    0xa
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_RESPONSE_STATUS_CLEAR__SHIFT                                               0xb
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_ERROR_CLEAR__SHIFT                                              0xc
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_FLUSH_REQ_CREDIT_EN__SHIFT                                                 0xd
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_EN__SHIFT                                                       0xe
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_PORT_CONTROL__SHIFT                                                        0xf
+#define DCHUBBUB_SDPIF_CFG0__DF_CSTATE_DISALLOW__SHIFT                                                        0x10
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_CREDIT_DISCONNECT_DELAY__SHIFT                                             0x19
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_NO_OUTSTANDING_REQ_MASK                                                    0x00000001L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_PORT_STATUS_MASK                                                           0x00000006L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_DATA_RESPONSE_STATUS_MASK                                                  0x00000038L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_RESPONSE_STATUS_MASK                                                       0x000003C0L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_ERROR_MASK                                                      0x00000400L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_RESPONSE_STATUS_CLEAR_MASK                                                 0x00000800L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_ERROR_CLEAR_MASK                                                0x00001000L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_FLUSH_REQ_CREDIT_EN_MASK                                                   0x00002000L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_REQ_CREDIT_EN_MASK                                                         0x00004000L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_PORT_CONTROL_MASK                                                          0x00008000L
+#define DCHUBBUB_SDPIF_CFG0__DF_CSTATE_DISALLOW_MASK                                                          0x00010000L
+#define DCHUBBUB_SDPIF_CFG0__SDPIF_CREDIT_DISCONNECT_DELAY_MASK                                               0x7E000000L
+//DCHUBBUB_SDPIF_CFG1
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_DETECT_EN__SHIFT                                                 0x0
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_STATUS__SHIFT                                                    0x1
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_STATUS_CLEAR__SHIFT                                              0x2
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_FORCE_SNOOP__SHIFT                                                         0x8
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_MAX_NUM_OUTSTANDING__SHIFT                                                 0x9
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_DETECT_EN_MASK                                                   0x00000001L
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_STATUS_MASK                                                      0x00000002L
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_PRQ_ERROR_STATUS_CLEAR_MASK                                                0x00000004L
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_FORCE_SNOOP_MASK                                                           0x00000100L
+#define DCHUBBUB_SDPIF_CFG1__SDPIF_MAX_NUM_OUTSTANDING_MASK                                                   0x00000200L
+//DCHUBBUB_SDPIF_CFG2
+#define DCHUBBUB_SDPIF_CFG2__dGPU_ADDR_PRESENT__SHIFT                                                         0x0
+#define DCHUBBUB_SDPIF_CFG2__SDPIF_HOSTVM_SEC_LVL__SHIFT                                                      0x8
+#define DCHUBBUB_SDPIF_CFG2__SDPIF_UNIT_ID_BITMASK__SHIFT                                                     0x10
+#define DCHUBBUB_SDPIF_CFG2__dGPU_ADDR_PRESENT_MASK                                                           0x00000001L
+#define DCHUBBUB_SDPIF_CFG2__SDPIF_HOSTVM_SEC_LVL_MASK                                                        0x00000F00L
+#define DCHUBBUB_SDPIF_CFG2__SDPIF_UNIT_ID_BITMASK_MASK                                                       0x01FF0000L
+//VM_REQUEST_PHYSICAL
+#define VM_REQUEST_PHYSICAL__PDE_REQUEST_PHYSICAL__SHIFT                                                      0x0
+#define VM_REQUEST_PHYSICAL__PTE_REQUEST_PHYSICAL__SHIFT                                                      0x3
+#define VM_REQUEST_PHYSICAL__PDE_REQUEST_PHYSICAL_MASK                                                        0x00000001L
+#define VM_REQUEST_PHYSICAL__PTE_REQUEST_PHYSICAL_MASK                                                        0x00000008L
+//DCHUBBUB_FORCE_IO_STATUS_0
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS__SHIFT                                              0x0
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_STICKY__SHIFT                                       0x1
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_CLEAR__SHIFT                                        0x2
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_PIPE_ID__SHIFT                                      0x3
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_REQUEST_TYPE__SHIFT                                 0x7
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_ADDR_LO__SHIFT                                      0xa
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_MASK                                                0x00000001L
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_STICKY_MASK                                         0x00000002L
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_CLEAR_MASK                                          0x00000004L
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_PIPE_ID_MASK                                        0x00000078L
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_REQUEST_TYPE_MASK                                   0x00000380L
+#define DCHUBBUB_FORCE_IO_STATUS_0__SDPIF_FORCE_IO_STATUS_ADDR_LO_MASK                                        0xFFFFFC00L
+//DCHUBBUB_FORCE_IO_STATUS_1
+#define DCHUBBUB_FORCE_IO_STATUS_1__SDPIF_FORCE_IO_STATUS_ADDR_HI__SHIFT                                      0x0
+#define DCHUBBUB_FORCE_IO_STATUS_1__SDPIF_FORCE_IO_STATUS_ADDR_HI_MASK                                        0x001FFFFFL
+//DCN_VM_FB_LOCATION_BASE
+#define DCN_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                               0x0
+#define DCN_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                                 0x00FFFFFFL
+//DCN_VM_FB_LOCATION_TOP
+#define DCN_VM_FB_LOCATION_TOP__FB_TOP__SHIFT                                                                 0x0
+#define DCN_VM_FB_LOCATION_TOP__FB_TOP_MASK                                                                   0x00FFFFFFL
+//DCN_VM_FB_OFFSET
+#define DCN_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                    0x0
+#define DCN_VM_FB_OFFSET__FB_OFFSET_MASK                                                                      0x00FFFFFFL
+//DCN_VM_AGP_BOT
+#define DCN_VM_AGP_BOT__AGP_BOT__SHIFT                                                                        0x0
+#define DCN_VM_AGP_BOT__AGP_BOT_MASK                                                                          0x00FFFFFFL
+//DCN_VM_AGP_TOP
+#define DCN_VM_AGP_TOP__AGP_TOP__SHIFT                                                                        0x0
+#define DCN_VM_AGP_TOP__AGP_TOP_MASK                                                                          0x00FFFFFFL
+//DCN_VM_AGP_BASE
+#define DCN_VM_AGP_BASE__AGP_BASE__SHIFT                                                                      0x0
+#define DCN_VM_AGP_BASE__AGP_BASE_MASK                                                                        0x00FFFFFFL
+//DCN_VM_LOCAL_HBM_ADDRESS_START
+#define DCN_VM_LOCAL_HBM_ADDRESS_START__ADDRESS_START__SHIFT                                                  0x0
+#define DCN_VM_LOCAL_HBM_ADDRESS_START__ADDRESS_START_MASK                                                    0x000FFFFFL
+//DCN_VM_LOCAL_HBM_ADDRESS_END
+#define DCN_VM_LOCAL_HBM_ADDRESS_END__ADDRESS_END__SHIFT                                                      0x0
+#define DCN_VM_LOCAL_HBM_ADDRESS_END__ADDRESS_END_MASK                                                        0x000FFFFFL
+//DCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL
+#define DCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                       0x0
+#define DCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK_MASK                                                         0x00000001L
+//DCHUBBUB_SDPIF_PIPE_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE0_SEC_LVL__SHIFT                                               0x0
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE1_SEC_LVL__SHIFT                                               0x4
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE2_SEC_LVL__SHIFT                                               0x8
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE3_SEC_LVL__SHIFT                                               0xc
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE0_SEC_LVL_MASK                                                 0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE1_SEC_LVL_MASK                                                 0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE2_SEC_LVL_MASK                                                 0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_SEC_LVL__SDPIF_PIPE3_SEC_LVL_MASK                                                 0x0000F000L
+//DCHUBBUB_SDPIF_PIPE_NOALLOC
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE0_NOALLOC__SHIFT                                               0x0
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE1_NOALLOC__SHIFT                                               0x1
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE2_NOALLOC__SHIFT                                               0x2
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE3_NOALLOC__SHIFT                                               0x3
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE0_NOALLOC_MASK                                                 0x00000001L
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE1_NOALLOC_MASK                                                 0x00000002L
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE2_NOALLOC_MASK                                                 0x00000004L
+#define DCHUBBUB_SDPIF_PIPE_NOALLOC__SDPIF_PIPE3_NOALLOC_MASK                                                 0x00000008L
+//DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE0_DMDATA_SEC_LVL__SHIFT                                 0x0
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE1_DMDATA_SEC_LVL__SHIFT                                 0x4
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE2_DMDATA_SEC_LVL__SHIFT                                 0x8
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE3_DMDATA_SEC_LVL__SHIFT                                 0xc
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE0_DMDATA_SEC_LVL_MASK                                   0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE1_DMDATA_SEC_LVL_MASK                                   0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE2_DMDATA_SEC_LVL_MASK                                   0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL__SDPIF_PIPE3_DMDATA_SEC_LVL_MASK                                   0x0000F000L
+//DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE0_DCCMETA_SEC_LVL__SHIFT                               0x0
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE1_DCCMETA_SEC_LVL__SHIFT                               0x4
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE2_DCCMETA_SEC_LVL__SHIFT                               0x8
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE3_DCCMETA_SEC_LVL__SHIFT                               0xc
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE0_DCCMETA_SEC_LVL_MASK                                 0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE1_DCCMETA_SEC_LVL_MASK                                 0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE2_DCCMETA_SEC_LVL_MASK                                 0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_DCCMETA_SEC_LVL__SDPIF_PIPE3_DCCMETA_SEC_LVL_MASK                                 0x0000F000L
+//DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE0_CURSOR0_SEC_LVL__SHIFT                               0x0
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE1_CURSOR0_SEC_LVL__SHIFT                               0x4
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE2_CURSOR0_SEC_LVL__SHIFT                               0x8
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE3_CURSOR0_SEC_LVL__SHIFT                               0xc
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE0_CURSOR0_SEC_LVL_MASK                                 0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE1_CURSOR0_SEC_LVL_MASK                                 0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE2_CURSOR0_SEC_LVL_MASK                                 0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_CURSOR0_SEC_LVL__SDPIF_PIPE3_CURSOR0_SEC_LVL_MASK                                 0x0000F000L
+//DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE0_3DLUT_SEC_LVL__SHIFT                                   0x0
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE1_3DLUT_SEC_LVL__SHIFT                                   0x4
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE2_3DLUT_SEC_LVL__SHIFT                                   0x8
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE3_3DLUT_SEC_LVL__SHIFT                                   0xc
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE0_3DLUT_SEC_LVL_MASK                                     0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE1_3DLUT_SEC_LVL_MASK                                     0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE2_3DLUT_SEC_LVL_MASK                                     0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_3DLUT_SEC_LVL__SDPIF_PIPE3_3DLUT_SEC_LVL_MASK                                     0x0000F000L
+//DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE0_GPUVM_SEC_LVL__SHIFT                                   0x0
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE1_GPUVM_SEC_LVL__SHIFT                                   0x4
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE2_GPUVM_SEC_LVL__SHIFT                                   0x8
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE3_GPUVM_SEC_LVL__SHIFT                                   0xc
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE0_GPUVM_SEC_LVL_MASK                                     0x0000000FL
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE1_GPUVM_SEC_LVL_MASK                                     0x000000F0L
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE2_GPUVM_SEC_LVL_MASK                                     0x00000F00L
+#define DCHUBBUB_SDPIF_PIPE_GPUVM_SEC_LVL__SDPIF_PIPE3_GPUVM_SEC_LVL_MASK                                     0x0000F000L
+//SDPIF_REQUEST_RATE_LIMIT
+#define SDPIF_REQUEST_RATE_LIMIT__SDPIF_REQUEST_RATE_LIMIT__SHIFT                                             0x0
+#define SDPIF_REQUEST_RATE_LIMIT__SDPIF_REQUEST_RATE_LIMIT_MASK                                               0x00000FFFL
+//DCHUBBUB_SDPIF_MEM_PWR_CTRL
+#define DCHUBBUB_SDPIF_MEM_PWR_CTRL__DCHUBBUB_SDPIF_MEM_PWR_FORCE__SHIFT                                      0x0
+#define DCHUBBUB_SDPIF_MEM_PWR_CTRL__DCHUBBUB_SDPIF_MEM_PWR_DIS__SHIFT                                        0x2
+#define DCHUBBUB_SDPIF_MEM_PWR_CTRL__DCHUBBUB_SDPIF_MEM_PWR_FORCE_MASK                                        0x00000003L
+#define DCHUBBUB_SDPIF_MEM_PWR_CTRL__DCHUBBUB_SDPIF_MEM_PWR_DIS_MASK                                          0x00000004L
+//DCHUBBUB_SDPIF_MEM_PWR_STATUS
+#define DCHUBBUB_SDPIF_MEM_PWR_STATUS__DCHUBBUB_SDPIF_MEM_PWR_STATE__SHIFT                                    0x0
+#define DCHUBBUB_SDPIF_MEM_PWR_STATUS__DCHUBBUB_SDPIF_MEM_PWR_STATE_MASK                                      0x00000003L
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_ret_path_dispdec
+//DCHUBBUB_RET_PATH_MEM_PWR_CTRL
+#define DCHUBBUB_RET_PATH_MEM_PWR_CTRL__DCHUBBUB_RET_PATH_MEM_PWR_FORCE__SHIFT                                0x0
+#define DCHUBBUB_RET_PATH_MEM_PWR_CTRL__DCHUBBUB_RET_PATH_MEM_PWR_DIS__SHIFT                                  0x2
+#define DCHUBBUB_RET_PATH_MEM_PWR_CTRL__DCHUBBUB_RET_PATH_MEM_PWR_FORCE_MASK                                  0x00000003L
+#define DCHUBBUB_RET_PATH_MEM_PWR_CTRL__DCHUBBUB_RET_PATH_MEM_PWR_DIS_MASK                                    0x00000004L
+//DCHUBBUB_RET_PATH_MEM_PWR_STATUS
+#define DCHUBBUB_RET_PATH_MEM_PWR_STATUS__DCHUBBUB_RET_PATH_MEM_PWR_STATE__SHIFT                              0x0
+#define DCHUBBUB_RET_PATH_MEM_PWR_STATUS__DCHUBBUB_RET_PATH_MEM_PWR_STATE_MASK                                0x00000003L
+//DCHUBBUB_CRC_CTRL
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_EN__SHIFT                                                             0x0
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_CONT_EN__SHIFT                                                        0x1
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC0_ONE_SHOT_PENDING__SHIFT                                              0x2
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC1_ONE_SHOT_PENDING__SHIFT                                              0x3
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC0_SRC_SEL__SHIFT                                                       0x4
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC1_SRC_SEL__SHIFT                                                       0x6
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_PIPE_SEL__SHIFT                                                       0x8
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_SURF_SEL__SHIFT                                                       0xc
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_DATA_SRC_SEL__SHIFT                                                   0x14
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_EN_MASK                                                               0x00000001L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_CONT_EN_MASK                                                          0x00000002L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC0_ONE_SHOT_PENDING_MASK                                                0x00000004L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC1_ONE_SHOT_PENDING_MASK                                                0x00000008L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC0_SRC_SEL_MASK                                                         0x00000030L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC1_SRC_SEL_MASK                                                         0x000000C0L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_PIPE_SEL_MASK                                                         0x00000F00L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_SURF_SEL_MASK                                                         0x00001000L
+#define DCHUBBUB_CRC_CTRL__DCHUBBUB_CRC_DATA_SRC_SEL_MASK                                                     0x00100000L
+//DCHUBBUB_CRC0_VAL_R
+#define DCHUBBUB_CRC0_VAL_R__DCHUBBUB_CRC0_R_CR__SHIFT                                                        0x0
+#define DCHUBBUB_CRC0_VAL_R__DCHUBBUB_CRC0_R_CR_MASK                                                          0xFFFFFFFFL
+//DCHUBBUB_CRC0_VAL_G
+#define DCHUBBUB_CRC0_VAL_G__DCHUBBUB_CRC0_G_Y__SHIFT                                                         0x0
+#define DCHUBBUB_CRC0_VAL_G__DCHUBBUB_CRC0_G_Y_MASK                                                           0xFFFFFFFFL
+//DCHUBBUB_CRC0_VAL_B
+#define DCHUBBUB_CRC0_VAL_B__DCHUBBUB_CRC0_B_CB__SHIFT                                                        0x0
+#define DCHUBBUB_CRC0_VAL_B__DCHUBBUB_CRC0_B_CB_MASK                                                          0xFFFFFFFFL
+//DCHUBBUB_CRC0_VAL_A
+#define DCHUBBUB_CRC0_VAL_A__DCHUBBUB_CRC0_ALPHA__SHIFT                                                       0x0
+#define DCHUBBUB_CRC0_VAL_A__DCHUBBUB_CRC0_ALPHA_MASK                                                         0xFFFFFFFFL
+//DCHUBBUB_CRC1_VAL_R
+#define DCHUBBUB_CRC1_VAL_R__DCHUBBUB_CRC1_R_CR__SHIFT                                                        0x0
+#define DCHUBBUB_CRC1_VAL_R__DCHUBBUB_CRC1_R_CR_MASK                                                          0xFFFFFFFFL
+//DCHUBBUB_CRC1_VAL_G
+#define DCHUBBUB_CRC1_VAL_G__DCHUBBUB_CRC1_G_Y__SHIFT                                                         0x0
+#define DCHUBBUB_CRC1_VAL_G__DCHUBBUB_CRC1_G_Y_MASK                                                           0xFFFFFFFFL
+//DCHUBBUB_CRC1_VAL_B
+#define DCHUBBUB_CRC1_VAL_B__DCHUBBUB_CRC1_B_CB__SHIFT                                                        0x0
+#define DCHUBBUB_CRC1_VAL_B__DCHUBBUB_CRC1_B_CB_MASK                                                          0xFFFFFFFFL
+//DCHUBBUB_CRC1_VAL_A
+#define DCHUBBUB_CRC1_VAL_A__DCHUBBUB_CRC1_ALPHA__SHIFT                                                       0x0
+#define DCHUBBUB_CRC1_VAL_A__DCHUBBUB_CRC1_ALPHA_MASK                                                         0xFFFFFFFFL
+//DCHUBBUB_DCC_STAT_CNTL
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_MODE__SHIFT                                                 0x0
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_EN__SHIFT                                                   0x1
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_DONE__SHIFT                                                 0x2
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_PIPE_SEL__SHIFT                                             0x4
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_FRAME_CNT__SHIFT                                            0x10
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_MODE_MASK                                                   0x00000001L
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_EN_MASK                                                     0x00000002L
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_DONE_MASK                                                   0x00000004L
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_PIPE_SEL_MASK                                               0x000000F0L
+#define DCHUBBUB_DCC_STAT_CNTL__DCHUBBUB_DCC_STAT_FRAME_CNT_MASK                                              0xFFFF0000L
+//DCHUBBUB_DCC_STAT0
+#define DCHUBBUB_DCC_STAT0__DCHUBBUB_DCC_STAT_TOTAL_REQ__SHIFT                                                0x0
+#define DCHUBBUB_DCC_STAT0__DCHUBBUB_DCC_STAT_TOTAL_REQ_MASK                                                  0xFFFFFFFFL
+//DCHUBBUB_DCC_STAT1
+#define DCHUBBUB_DCC_STAT1__DCHUBBUB_DCC_STAT_ZS_REQ__SHIFT                                                   0x0
+#define DCHUBBUB_DCC_STAT1__DCHUBBUB_DCC_STAT_ZS_REQ_MASK                                                     0xFFFFFFFFL
+//DCHUBBUB_DCC_STAT2
+#define DCHUBBUB_DCC_STAT2__DCHUBBUB_DCC_STAT_DCC_REQ__SHIFT                                                  0x0
+#define DCHUBBUB_DCC_STAT2__DCHUBBUB_DCC_STAT_DCC_REQ_MASK                                                    0xFFFFFFFFL
+//DCHUBBUB_COMPBUF_CTRL
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE__SHIFT                                                            0x0
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CURRENT__SHIFT                                                    0x8
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_ENABLE__SHIFT                                     0x10
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_STATUS__SHIFT                                     0x12
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_CLEAR__SHIFT                                      0x13
+#define DCHUBBUB_COMPBUF_CTRL__CONFIG_ERROR__SHIFT                                                            0x1f
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_MASK                                                              0x0000001FL
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CURRENT_MASK                                                      0x00001F00L
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_ENABLE_MASK                                       0x00010000L
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_STATUS_MASK                                       0x00040000L
+#define DCHUBBUB_COMPBUF_CTRL__COMPBUF_SIZE_CHANGE_DONE_INT_CLEAR_MASK                                        0x00080000L
+#define DCHUBBUB_COMPBUF_CTRL__CONFIG_ERROR_MASK                                                              0x80000000L
+//DCHUBBUB_DET0_CTRL
+#define DCHUBBUB_DET0_CTRL__DET0_SIZE__SHIFT                                                                  0x0
+#define DCHUBBUB_DET0_CTRL__DET0_SIZE_CURRENT__SHIFT                                                          0x8
+#define DCHUBBUB_DET0_CTRL__DET0_SIZE_MASK                                                                    0x0000001FL
+#define DCHUBBUB_DET0_CTRL__DET0_SIZE_CURRENT_MASK                                                            0x00001F00L
+//DCHUBBUB_DET1_CTRL
+#define DCHUBBUB_DET1_CTRL__DET1_SIZE__SHIFT                                                                  0x0
+#define DCHUBBUB_DET1_CTRL__DET1_SIZE_CURRENT__SHIFT                                                          0x8
+#define DCHUBBUB_DET1_CTRL__DET1_SIZE_MASK                                                                    0x0000001FL
+#define DCHUBBUB_DET1_CTRL__DET1_SIZE_CURRENT_MASK                                                            0x00001F00L
+//DCHUBBUB_DET2_CTRL
+#define DCHUBBUB_DET2_CTRL__DET2_SIZE__SHIFT                                                                  0x0
+#define DCHUBBUB_DET2_CTRL__DET2_SIZE_CURRENT__SHIFT                                                          0x8
+#define DCHUBBUB_DET2_CTRL__DET2_SIZE_MASK                                                                    0x0000001FL
+#define DCHUBBUB_DET2_CTRL__DET2_SIZE_CURRENT_MASK                                                            0x00001F00L
+//DCHUBBUB_DET3_CTRL
+#define DCHUBBUB_DET3_CTRL__DET3_SIZE__SHIFT                                                                  0x0
+#define DCHUBBUB_DET3_CTRL__DET3_SIZE_CURRENT__SHIFT                                                          0x8
+#define DCHUBBUB_DET3_CTRL__DET3_SIZE_MASK                                                                    0x0000001FL
+#define DCHUBBUB_DET3_CTRL__DET3_SIZE_CURRENT_MASK                                                            0x00001F00L
+//DCHUBBUB_MEM_PWR_MODE_CTRL
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_ACCESS_MEM_PWR_MODE__SHIFT                                        0x0
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_ACTIVE_MEM_PWR_MODE__SHIFT                                        0x2
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_IDLE_MEM_PWR_MODE__SHIFT                                          0x4
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__METAFIFO_MEM_PWR_FORCE__SHIFT                                             0x6
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DCC_SKID_MEM_PWR_FORCE__SHIFT                                             0x8
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__UNALLOCATED_MEM_PWR_MODE__SHIFT                                           0xa
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_MEM_PWR_FORCE__SHIFT                                                  0x10
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_IDLE_MEM_PWR_MODE__SHIFT                                              0x12
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_MEM_PWR_LS_MODE__SHIFT                                                0x14
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__SEGMENT_MEM_PWR_DIS__SHIFT                                                0x18
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__METAFIFO_MEM_PWR_DIS__SHIFT                                               0x19
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DCC_SKID_MEM_PWR_DIS__SHIFT                                               0x1a
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_ACCESS_MEM_PWR_MODE_MASK                                          0x00000003L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_ACTIVE_MEM_PWR_MODE_MASK                                          0x0000000CL
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__COMPBUF_IDLE_MEM_PWR_MODE_MASK                                            0x00000030L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__METAFIFO_MEM_PWR_FORCE_MASK                                               0x000000C0L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DCC_SKID_MEM_PWR_FORCE_MASK                                               0x00000300L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__UNALLOCATED_MEM_PWR_MODE_MASK                                             0x00000C00L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_MEM_PWR_FORCE_MASK                                                    0x00030000L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_IDLE_MEM_PWR_MODE_MASK                                                0x000C0000L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DET_MEM_PWR_LS_MODE_MASK                                                  0x00300000L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__SEGMENT_MEM_PWR_DIS_MASK                                                  0x01000000L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__METAFIFO_MEM_PWR_DIS_MASK                                                 0x02000000L
+#define DCHUBBUB_MEM_PWR_MODE_CTRL__DCC_SKID_MEM_PWR_DIS_MASK                                                 0x04000000L
+//COMPBUF_MEM_PWR_CTRL_1
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_ACTIVE_WAKE_LATENCY__SHIFT                                            0x0
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_ACTIVE_SLEEP_LATENCY__SHIFT                                           0x8
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_IDLE_WAKE_LATENCY__SHIFT                                              0x10
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_IDLE_SLEEP_LATENCY__SHIFT                                             0x18
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_ACTIVE_WAKE_LATENCY_MASK                                              0x000000FFL
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_ACTIVE_SLEEP_LATENCY_MASK                                             0x0000FF00L
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_IDLE_WAKE_LATENCY_MASK                                                0x00FF0000L
+#define COMPBUF_MEM_PWR_CTRL_1__COMPBUF_IDLE_SLEEP_LATENCY_MASK                                               0xFF000000L
+//COMPBUF_MEM_PWR_CTRL_2
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_UNALLOCATED_WAKE_LATENCY__SHIFT                                       0x0
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_ACTIVE_ENTER_LATENCY__SHIFT                                           0x8
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_IDLE_ENTER_LATENCY__SHIFT                                             0xc
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_UNALLOCATED_WAKE_LATENCY_MASK                                         0x000000FFL
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_ACTIVE_ENTER_LATENCY_MASK                                             0x00000F00L
+#define COMPBUF_MEM_PWR_CTRL_2__COMPBUF_IDLE_ENTER_LATENCY_MASK                                               0x0000F000L
+//DCHUBBUB_MEM_PWR_STATUS
+#define DCHUBBUB_MEM_PWR_STATUS__COMPBUF_MEM_PWR_STATE__SHIFT                                                 0x0
+#define DCHUBBUB_MEM_PWR_STATUS__METAFIFO_MEM_PWR_STATE__SHIFT                                                0x2
+#define DCHUBBUB_MEM_PWR_STATUS__UNALLOCATED_MEM_PWR_STATE__SHIFT                                             0x4
+#define DCHUBBUB_MEM_PWR_STATUS__DCC_SKID_MEM_PWR_STATE__SHIFT                                                0x6
+#define DCHUBBUB_MEM_PWR_STATUS__DET0_MEM_PWR_STATE__SHIFT                                                    0x8
+#define DCHUBBUB_MEM_PWR_STATUS__DET1_MEM_PWR_STATE__SHIFT                                                    0xa
+#define DCHUBBUB_MEM_PWR_STATUS__DET2_MEM_PWR_STATE__SHIFT                                                    0xc
+#define DCHUBBUB_MEM_PWR_STATUS__DET3_MEM_PWR_STATE__SHIFT                                                    0xe
+#define DCHUBBUB_MEM_PWR_STATUS__COMPBUF_MEM_PWR_STATE_MASK                                                   0x00000003L
+#define DCHUBBUB_MEM_PWR_STATUS__METAFIFO_MEM_PWR_STATE_MASK                                                  0x0000000CL
+#define DCHUBBUB_MEM_PWR_STATUS__UNALLOCATED_MEM_PWR_STATE_MASK                                               0x00000030L
+#define DCHUBBUB_MEM_PWR_STATUS__DCC_SKID_MEM_PWR_STATE_MASK                                                  0x000000C0L
+#define DCHUBBUB_MEM_PWR_STATUS__DET0_MEM_PWR_STATE_MASK                                                      0x00000300L
+#define DCHUBBUB_MEM_PWR_STATUS__DET1_MEM_PWR_STATE_MASK                                                      0x00000C00L
+#define DCHUBBUB_MEM_PWR_STATUS__DET2_MEM_PWR_STATE_MASK                                                      0x00003000L
+#define DCHUBBUB_MEM_PWR_STATUS__DET3_MEM_PWR_STATE_MASK                                                      0x0000C000L
+//COMPBUF_RESERVED_SPACE
+#define COMPBUF_RESERVED_SPACE__COMPBUF_RESERVED_SPACE_64B__SHIFT                                             0x0
+#define COMPBUF_RESERVED_SPACE__COMPBUF_RESERVED_SPACE_ZS__SHIFT                                              0x10
+#define COMPBUF_RESERVED_SPACE__COMPBUF_RESERVED_SPACE_64B_MASK                                               0x00000FFFL
+#define COMPBUF_RESERVED_SPACE__COMPBUF_RESERVED_SPACE_ZS_MASK                                                0x0FFF0000L
+//DCHUBBUB_DEBUG_CTRL_0
+#define DCHUBBUB_DEBUG_CTRL_0__METAFIFO_DEPTH__SHIFT                                                          0x0
+#define DCHUBBUB_DEBUG_CTRL_0__COMPBUF_SEG_DEPTH__SHIFT                                                       0x8
+#define DCHUBBUB_DEBUG_CTRL_0__DET_SEG_DEPTH__SHIFT                                                           0xc
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                                                               0x10
+#define DCHUBBUB_DEBUG_CTRL_0__DELAY_COMPBUF_DEALLOC_ON_DRQ_STOP_DISABLE__SHIFT                               0x1b
+#define DCHUBBUB_DEBUG_CTRL_0__SEG_ALLOC_ERR_PIPE_BLANK_ENABLE__SHIFT                                         0x1c
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_RESET_OPTIMIZATION_DISABLE__SHIFT                                     0x1d
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_STALL_FOR_ALLOC_ENABLE__SHIFT                                         0x1e
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_STALL_FOR_DEALLOC_ENABLE__SHIFT                                       0x1f
+#define DCHUBBUB_DEBUG_CTRL_0__METAFIFO_DEPTH_MASK                                                            0x000000FFL
+#define DCHUBBUB_DEBUG_CTRL_0__COMPBUF_SEG_DEPTH_MASK                                                         0x00000F00L
+#define DCHUBBUB_DEBUG_CTRL_0__DET_SEG_DEPTH_MASK                                                             0x0000F000L
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                                                 0x07FF0000L
+#define DCHUBBUB_DEBUG_CTRL_0__DELAY_COMPBUF_DEALLOC_ON_DRQ_STOP_DISABLE_MASK                                 0x08000000L
+#define DCHUBBUB_DEBUG_CTRL_0__SEG_ALLOC_ERR_PIPE_BLANK_ENABLE_MASK                                           0x10000000L
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_RESET_OPTIMIZATION_DISABLE_MASK                                       0x20000000L
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_STALL_FOR_ALLOC_ENABLE_MASK                                           0x40000000L
+#define DCHUBBUB_DEBUG_CTRL_0__DATAFIFO_STALL_FOR_DEALLOC_ENABLE_MASK                                         0x80000000L
+//DCHUBBUB_RET_PATH_TEST_DEBUG_INDEX
+#define DCHUBBUB_RET_PATH_TEST_DEBUG_INDEX__DCHUBBUB_RET_PATH_TEST_DEBUG_INDEX__SHIFT                         0x0
+#define DCHUBBUB_RET_PATH_TEST_DEBUG_INDEX__DCHUBBUB_RET_PATH_TEST_DEBUG_INDEX_MASK                           0x000000FFL
+//DCHUBBUB_RET_PATH_TEST_DEBUG_DATA
+#define DCHUBBUB_RET_PATH_TEST_DEBUG_DATA__DCHUBBUB_RET_PATH_TEST_DEBUG_DATA__SHIFT                           0x0
+#define DCHUBBUB_RET_PATH_TEST_DEBUG_DATA__DCHUBBUB_RET_PATH_TEST_DEBUG_DATA_MASK                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_dispdec
+//DCHUBBUB_ARB_DF_REQ_OUTSTAND
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MAX_REQ_OUTSTAND__SHIFT                                    0x0
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MIN_REQ_OUTSTAND__SHIFT                                    0xa
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MIN_REQ_OUTSTAND_COMMIT_THRESHOLD__SHIFT                   0x16
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MAX_REQ_OUTSTAND_MASK                                      0x000001FFL
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MIN_REQ_OUTSTAND_MASK                                      0x0007FC00L
+#define DCHUBBUB_ARB_DF_REQ_OUTSTAND__DCHUBBUB_ARB_MIN_REQ_OUTSTAND_COMMIT_THRESHOLD_MASK                     0x7FC00000L
+//DCHUBBUB_ARB_SAT_LEVEL
+#define DCHUBBUB_ARB_SAT_LEVEL__DCHUBBUB_ARB_SAT_LEVEL__SHIFT                                                 0x0
+#define DCHUBBUB_ARB_SAT_LEVEL__DCHUBBUB_ARB_SAT_LEVEL_MASK                                                   0xFFFFFFFFL
+//DCHUBBUB_ARB_QOS_FORCE
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_QOS_FORCE_VALUE__SHIFT                                           0x0
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_QOS_FORCE_ENABLE__SHIFT                                          0x8
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_DO_NOT_FORCE_URGENCY_DURING_PSTATE_CHANGE_REQUEST__SHIFT         0x9
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_HOSTVM_STALL_QOS__SHIFT                                          0xc
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_QOS_FORCE_VALUE_MASK                                             0x0000000FL
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_QOS_FORCE_ENABLE_MASK                                            0x00000100L
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_DO_NOT_FORCE_URGENCY_DURING_PSTATE_CHANGE_REQUEST_MASK           0x00000200L
+#define DCHUBBUB_ARB_QOS_FORCE__DCHUBBUB_ARB_HOSTVM_STALL_QOS_MASK                                            0x0000F000L
+//DCHUBBUB_ARB_DRAM_STATE_CNTL
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE__SHIFT                      0x0
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE__SHIFT                     0x1
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_CSTATE_DURING_PSTATE_CHANGE_REQUEST__SHIFT  0x2
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE__SHIFT                     0x4
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE__SHIFT                    0x5
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__ENABLE_QOS_FORCE_PSTATE__SHIFT                                          0x7
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE__SHIFT                                 0xc
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_CSTATE_DEEPSLEEP_LEGACY_MODE__SHIFT                  0xd
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DISABL_EXIT_DEEPSLEEP_WHEN_PREFETCH_START_PENDING__SHIFT   0xe
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DISABLE_HOSTVM_FORCE_DCFCLK_DEEP_SLEEP__SHIFT                           0xf
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE__SHIFT                 0x10
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE__SHIFT                0x11
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_LEGACY__SHIFT                0x12
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DCFCLK_DEEP_SLEEP_HYSTERESIS__SHIFT                        0x18
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE_MASK                        0x00000001L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE_MASK                       0x00000002L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_CSTATE_DURING_PSTATE_CHANGE_REQUEST_MASK  0x00000004L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE_MASK                       0x00000010L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE_MASK                      0x00000020L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__ENABLE_QOS_FORCE_PSTATE_MASK                                            0x00000080L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE_MASK                                   0x00001000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_CSTATE_DEEPSLEEP_LEGACY_MODE_MASK                    0x00002000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DISABL_EXIT_DEEPSLEEP_WHEN_PREFETCH_START_PENDING_MASK     0x00004000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DISABLE_HOSTVM_FORCE_DCFCLK_DEEP_SLEEP_MASK                             0x00008000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE_MASK                   0x00010000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE_MASK                  0x00020000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_LEGACY_MASK                  0x00040000L
+#define DCHUBBUB_ARB_DRAM_STATE_CNTL__DCHUBBUB_ARB_DCFCLK_DEEP_SLEEP_HYSTERESIS_MASK                          0xFF000000L
+//DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL
+#define DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL__DCHUBBUB_ARB_NO_PENDING_HVM_REQ_BEFORE_ENTER_CSTATE_HYSTERESIS__SHIFT  0x0
+#define DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL__DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL_RESERVED__SHIFT                    0x10
+#define DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL__DCHUBBUB_ARB_NO_PENDING_HVM_REQ_BEFORE_ENTER_CSTATE_HYSTERESIS_MASK  0x0000FFFFL
+#define DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL__DCHUBBUB_ARB_DRAM_HVM_STATE_CNTL_RESERVED_MASK                      0xFFFF0000L
+//DCHUBBUB_ARB_USR_RETRAINING_CNTL
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__USR_RETRAINING_REQUEST__SHIFT                                       0x0
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__ALLOW_USR_RETRAINING__SHIFT                                         0x1
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_VALUE__SHIFT                0x8
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_ENABLE__SHIFT               0x9
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PSTATE_CHANGE_REQUEST__SHIFT  0xa
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PRE_CSTATE__SHIFT  0xb
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__USR_RETRAINING_REQUEST_MASK                                         0x00000001L
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__ALLOW_USR_RETRAINING_MASK                                           0x00000002L
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_VALUE_MASK                  0x00000100L
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_ALLOW_USR_RETRAINING_FORCE_ENABLE_MASK                 0x00000200L
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PSTATE_CHANGE_REQUEST_MASK  0x00000400L
+#define DCHUBBUB_ARB_USR_RETRAINING_CNTL__DCHUBBUB_ARB_DO_NOT_FORCE_ALLOW_USR_RETRAINING_DURING_PRE_CSTATE_MASK  0x00000800L
+//DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A__SHIFT                   0x0
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A_MASK                     0x00003FFFL
+//DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A__SHIFT               0x0
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A_MASK                 0x00003FFFL
+//DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A__SHIFT             0x0
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A_MASK               0x00003FFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A__SHIFT               0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A_MASK                 0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A__SHIFT         0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_A_MASK           0x000FFFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A__SHIFT                 0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A_MASK                   0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A__SHIFT           0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_A_MASK             0x000FFFFFL
+//DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A__SHIFT       0x0
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A__SHIFT       0x0
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FRAC_URG_BW_NOM_A
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_A__DCHUBBUB_ARB_FRAC_URG_BW_NOM_A__SHIFT                                 0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_A__DCHUBBUB_ARB_FRAC_URG_BW_NOM_A_MASK                                   0x000003FFL
+//DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A__SHIFT                               0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A_MASK                                 0x000003FFL
+//DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B__SHIFT                   0x0
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B_MASK                     0x00003FFFL
+//DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B__SHIFT               0x0
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B_MASK                 0x00003FFFL
+//DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B__SHIFT             0x0
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B_MASK               0x00003FFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B__SHIFT               0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B_MASK                 0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B__SHIFT         0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_B_MASK           0x000FFFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B__SHIFT                 0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B_MASK                   0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B__SHIFT           0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_B_MASK             0x000FFFFFL
+//DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B__SHIFT       0x0
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B__SHIFT       0x0
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FRAC_URG_BW_NOM_B
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_B__DCHUBBUB_ARB_FRAC_URG_BW_NOM_B__SHIFT                                 0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_B__DCHUBBUB_ARB_FRAC_URG_BW_NOM_B_MASK                                   0x000003FFL
+//DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B__SHIFT                               0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B_MASK                                 0x000003FFL
+//DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C__SHIFT                   0x0
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C_MASK                     0x00003FFFL
+//DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C__SHIFT               0x0
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C_MASK                 0x00003FFFL
+//DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C__SHIFT             0x0
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C_MASK               0x00003FFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C__SHIFT               0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C_MASK                 0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C__SHIFT         0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_C_MASK           0x000FFFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C__SHIFT                 0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C_MASK                   0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C__SHIFT           0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_C_MASK             0x000FFFFFL
+//DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C__SHIFT       0x0
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C__SHIFT       0x0
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FRAC_URG_BW_NOM_C
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_C__DCHUBBUB_ARB_FRAC_URG_BW_NOM_C__SHIFT                                 0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_C__DCHUBBUB_ARB_FRAC_URG_BW_NOM_C_MASK                                   0x000003FFL
+//DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C__SHIFT                               0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C_MASK                                 0x000003FFL
+//DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D__SHIFT                   0x0
+#define DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D__DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D_MASK                     0x00003FFFL
+//DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D__SHIFT               0x0
+#define DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D__DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D_MASK                 0x00003FFFL
+//DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D__SHIFT             0x0
+#define DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D__DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D_MASK               0x00003FFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D__SHIFT               0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D_MASK                 0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D__SHIFT         0x0
+#define DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D__DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_Z8_D_MASK           0x000FFFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D__SHIFT                 0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D_MASK                   0x0000FFFFL
+//DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D__SHIFT           0x0
+#define DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D__DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_Z8_D_MASK             0x000FFFFFL
+//DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D__SHIFT       0x0
+#define DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D__DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D__SHIFT       0x0
+#define DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D__DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D_MASK         0x0000FFFFL
+//DCHUBBUB_ARB_FRAC_URG_BW_NOM_D
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_D__DCHUBBUB_ARB_FRAC_URG_BW_NOM_D__SHIFT                                 0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_NOM_D__DCHUBBUB_ARB_FRAC_URG_BW_NOM_D_MASK                                   0x000003FFL
+//DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D__SHIFT                               0x0
+#define DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D__DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D_MASK                                 0x000003FFL
+//DCHUBBUB_ARB_HOSTVM_CNTL
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_CSTATE__SHIFT                                          0x0
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_ALLOW_PSTATE__SHIFT                                    0x1
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_ALLOW_USR_RETRAINING__SHIFT                            0x2
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_SLACK_MASK__SHIFT                                                       0x3
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_SPACE_OK_STATUS__SHIFT                                                  0x4
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_GID_FREE_STATUS__SHIFT                                                  0x5
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DCHVM_RET_FIFO_FREE_STATUS__SHIFT                                           0x6
+#define DCHUBBUB_ARB_HOSTVM_CNTL__NON_PRQ_CLIENT_WINNER_STATUS__SHIFT                                         0x7
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_MAX_ALLOCATED_GROUPS__SHIFT                                          0x8
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_MAX_RD_FIFO_ENTRIES__SHIFT                                           0x10
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_QOS__SHIFT                                                           0x18
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DCHUBBUB_ARB_MAX_QOS_COMMIT_THRESHOLD__SHIFT                                0x1c
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_CSTATE_MASK                                            0x00000001L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_ALLOW_PSTATE_MASK                                      0x00000002L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DISABLE_HOSTVM_FORCE_ALLOW_USR_RETRAINING_MASK                              0x00000004L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_SLACK_MASK_MASK                                                         0x00000008L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_SPACE_OK_STATUS_MASK                                                    0x00000010L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__PRQ_GID_FREE_STATUS_MASK                                                    0x00000020L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DCHVM_RET_FIFO_FREE_STATUS_MASK                                             0x00000040L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__NON_PRQ_CLIENT_WINNER_STATUS_MASK                                           0x00000080L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_MAX_ALLOCATED_GROUPS_MASK                                            0x00003F00L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_MAX_RD_FIFO_ENTRIES_MASK                                             0x00FF0000L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__HOSTVM_QOS_MASK                                                             0x0F000000L
+#define DCHUBBUB_ARB_HOSTVM_CNTL__DCHUBBUB_ARB_MAX_QOS_COMMIT_THRESHOLD_MASK                                  0xF0000000L
+//DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_SELECT__SHIFT                       0x0
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE__SHIFT       0x4
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_STATUS__SHIFT        0x5
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST__SHIFT                      0x8
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_SELECT_Z8__SHIFT                    0x10
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__PSTATE_CHANGE_TYPE__SHIFT                                         0x18
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_SELECT_MASK                         0x00000003L
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE_MASK         0x00000010L
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_STATUS_MASK          0x00000020L
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST_MASK                        0x00000100L
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__DCHUBBUB_ARB_WATERMARK_CHANGE_SELECT_Z8_MASK                      0x00010000L
+#define DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL__PSTATE_CHANGE_TYPE_MASK                                           0x01000000L
+//DCHUBBUB_ARB_MALL_CNTL
+#define DCHUBBUB_ARB_MALL_CNTL__GLOBAL_USE_MALL_FOR_SS__SHIFT                                                 0x0
+#define DCHUBBUB_ARB_MALL_CNTL__MALL_IN_USE__SHIFT                                                            0x4
+#define DCHUBBUB_ARB_MALL_CNTL__MALL_PREFETCH_COMPLETE__SHIFT                                                 0x5
+#define DCHUBBUB_ARB_MALL_CNTL__GLOBAL_USE_MALL_FOR_SS_MASK                                                   0x00000001L
+#define DCHUBBUB_ARB_MALL_CNTL__MALL_IN_USE_MASK                                                              0x00000010L
+#define DCHUBBUB_ARB_MALL_CNTL__MALL_PREFETCH_COMPLETE_MASK                                                   0x00000020L
+//DCHUBBUB_ARB_TIMEOUT_ENABLE
+#define DCHUBBUB_ARB_TIMEOUT_ENABLE__DCHUBBUB_ARB_TIMEOUT_ENABLE__SHIFT                                       0x0
+#define DCHUBBUB_ARB_TIMEOUT_ENABLE__DCHUBBUB_ARB_TIMEOUT_ENABLE_MASK                                         0x00000001L
+//DCHUBBUB_GLOBAL_TIMER_CNTL
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_REFDIV__SHIFT                                       0x0
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_ENABLE__SHIFT                                       0xc
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_INIT__SHIFT                                         0x10
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_REFDIV_MASK                                         0x0000000FL
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_ENABLE_MASK                                         0x00001000L
+#define DCHUBBUB_GLOBAL_TIMER_CNTL__DCHUBBUB_GLOBAL_TIMER_INIT_MASK                                           0xFFFF0000L
+//SURFACE_CHECK0_ADDRESS_LSB
+#define SURFACE_CHECK0_ADDRESS_LSB__SURFACE_CHECK0_ADDRESS_LSB__SHIFT                                         0x0
+#define SURFACE_CHECK0_ADDRESS_LSB__SURFACE_CHECK0_ADDRESS_LSB_MASK                                           0xFFFFFFFFL
+//SURFACE_CHECK0_ADDRESS_MSB
+#define SURFACE_CHECK0_ADDRESS_MSB__SURFACE_CHECK0_ADDRESS_MSB__SHIFT                                         0x0
+#define SURFACE_CHECK0_ADDRESS_MSB__CHECKER0_SURFACE_INUSE__SHIFT                                             0x1f
+#define SURFACE_CHECK0_ADDRESS_MSB__SURFACE_CHECK0_ADDRESS_MSB_MASK                                           0x0000FFFFL
+#define SURFACE_CHECK0_ADDRESS_MSB__CHECKER0_SURFACE_INUSE_MASK                                               0x80000000L
+//SURFACE_CHECK1_ADDRESS_LSB
+#define SURFACE_CHECK1_ADDRESS_LSB__SURFACE_CHECK1_ADDRESS_LSB__SHIFT                                         0x0
+#define SURFACE_CHECK1_ADDRESS_LSB__SURFACE_CHECK1_ADDRESS_LSB_MASK                                           0xFFFFFFFFL
+//SURFACE_CHECK1_ADDRESS_MSB
+#define SURFACE_CHECK1_ADDRESS_MSB__SURFACE_CHECK1_ADDRESS_MSB__SHIFT                                         0x0
+#define SURFACE_CHECK1_ADDRESS_MSB__CHECKER1_SURFACE_INUSE__SHIFT                                             0x1f
+#define SURFACE_CHECK1_ADDRESS_MSB__SURFACE_CHECK1_ADDRESS_MSB_MASK                                           0x0000FFFFL
+#define SURFACE_CHECK1_ADDRESS_MSB__CHECKER1_SURFACE_INUSE_MASK                                               0x80000000L
+//SURFACE_CHECK2_ADDRESS_LSB
+#define SURFACE_CHECK2_ADDRESS_LSB__SURFACE_CHECK2_ADDRESS_LSB__SHIFT                                         0x0
+#define SURFACE_CHECK2_ADDRESS_LSB__SURFACE_CHECK2_ADDRESS_LSB_MASK                                           0xFFFFFFFFL
+//SURFACE_CHECK2_ADDRESS_MSB
+#define SURFACE_CHECK2_ADDRESS_MSB__SURFACE_CHECK2_ADDRESS_MSB__SHIFT                                         0x0
+#define SURFACE_CHECK2_ADDRESS_MSB__CHECKER2_SURFACE_INUSE__SHIFT                                             0x1f
+#define SURFACE_CHECK2_ADDRESS_MSB__SURFACE_CHECK2_ADDRESS_MSB_MASK                                           0x0000FFFFL
+#define SURFACE_CHECK2_ADDRESS_MSB__CHECKER2_SURFACE_INUSE_MASK                                               0x80000000L
+//SURFACE_CHECK3_ADDRESS_LSB
+#define SURFACE_CHECK3_ADDRESS_LSB__SURFACE_CHECK3_ADDRESS_LSB__SHIFT                                         0x0
+#define SURFACE_CHECK3_ADDRESS_LSB__SURFACE_CHECK3_ADDRESS_LSB_MASK                                           0xFFFFFFFFL
+//SURFACE_CHECK3_ADDRESS_MSB
+#define SURFACE_CHECK3_ADDRESS_MSB__SURFACE_CHECK3_ADDRESS_MSB__SHIFT                                         0x0
+#define SURFACE_CHECK3_ADDRESS_MSB__CHECKER3_SURFACE_INUSE__SHIFT                                             0x1f
+#define SURFACE_CHECK3_ADDRESS_MSB__SURFACE_CHECK3_ADDRESS_MSB_MASK                                           0x0000FFFFL
+#define SURFACE_CHECK3_ADDRESS_MSB__CHECKER3_SURFACE_INUSE_MASK                                               0x80000000L
+//VTG0_CONTROL
+#define VTG0_CONTROL__VTG0_FP2__SHIFT                                                                         0x0
+#define VTG0_CONTROL__VTG0_VCOUNT_INIT__SHIFT                                                                 0x10
+#define VTG0_CONTROL__VTG0_ENABLE__SHIFT                                                                      0x1f
+#define VTG0_CONTROL__VTG0_FP2_MASK                                                                           0x00007FFFL
+#define VTG0_CONTROL__VTG0_VCOUNT_INIT_MASK                                                                   0x7FFF0000L
+#define VTG0_CONTROL__VTG0_ENABLE_MASK                                                                        0x80000000L
+//VTG1_CONTROL
+#define VTG1_CONTROL__VTG1_FP2__SHIFT                                                                         0x0
+#define VTG1_CONTROL__VTG1_VCOUNT_INIT__SHIFT                                                                 0x10
+#define VTG1_CONTROL__VTG1_ENABLE__SHIFT                                                                      0x1f
+#define VTG1_CONTROL__VTG1_FP2_MASK                                                                           0x00007FFFL
+#define VTG1_CONTROL__VTG1_VCOUNT_INIT_MASK                                                                   0x7FFF0000L
+#define VTG1_CONTROL__VTG1_ENABLE_MASK                                                                        0x80000000L
+//VTG2_CONTROL
+#define VTG2_CONTROL__VTG2_FP2__SHIFT                                                                         0x0
+#define VTG2_CONTROL__VTG2_VCOUNT_INIT__SHIFT                                                                 0x10
+#define VTG2_CONTROL__VTG2_ENABLE__SHIFT                                                                      0x1f
+#define VTG2_CONTROL__VTG2_FP2_MASK                                                                           0x00007FFFL
+#define VTG2_CONTROL__VTG2_VCOUNT_INIT_MASK                                                                   0x7FFF0000L
+#define VTG2_CONTROL__VTG2_ENABLE_MASK                                                                        0x80000000L
+//VTG3_CONTROL
+#define VTG3_CONTROL__VTG3_FP2__SHIFT                                                                         0x0
+#define VTG3_CONTROL__VTG3_VCOUNT_INIT__SHIFT                                                                 0x10
+#define VTG3_CONTROL__VTG3_ENABLE__SHIFT                                                                      0x1f
+#define VTG3_CONTROL__VTG3_FP2_MASK                                                                           0x00007FFFL
+#define VTG3_CONTROL__VTG3_VCOUNT_INIT_MASK                                                                   0x7FFF0000L
+#define VTG3_CONTROL__VTG3_ENABLE_MASK                                                                        0x80000000L
+//DCHUBBUB_SOFT_RESET
+#define DCHUBBUB_SOFT_RESET__DCHUBBUB_GLOBAL_SOFT_RESET__SHIFT                                                0x0
+#define DCHUBBUB_SOFT_RESET__ALLOW_CSTATE_SOFT_RESET__SHIFT                                                   0x1
+#define DCHUBBUB_SOFT_RESET__GLBFLIP_SOFT_RESET__SHIFT                                                        0x4
+#define DCHUBBUB_SOFT_RESET__DCHUBBUB_GLOBAL_SOFT_RESET_MASK                                                  0x00000001L
+#define DCHUBBUB_SOFT_RESET__ALLOW_CSTATE_SOFT_RESET_MASK                                                     0x00000002L
+#define DCHUBBUB_SOFT_RESET__GLBFLIP_SOFT_RESET_MASK                                                          0x00000010L
+//DCHUBBUB_CLOCK_CNTL
+#define DCHUBBUB_CLOCK_CNTL__DCHUBBUB_TEST_CLK_SEL__SHIFT                                                     0x0
+#define DCHUBBUB_CLOCK_CNTL__DISPCLK_R_DCHUBBUB_GATE_DIS__SHIFT                                               0x5
+#define DCHUBBUB_CLOCK_CNTL__DCFCLK_R_DCHUBBUB_GATE_DIS__SHIFT                                                0x6
+#define DCHUBBUB_CLOCK_CNTL__DCHUBBUB_FGCG_REP_DIS__SHIFT                                                     0x7
+#define DCHUBBUB_CLOCK_CNTL__DCHUBBUB_TEST_CLK_SEL_MASK                                                       0x0000001FL
+#define DCHUBBUB_CLOCK_CNTL__DISPCLK_R_DCHUBBUB_GATE_DIS_MASK                                                 0x00000020L
+#define DCHUBBUB_CLOCK_CNTL__DCFCLK_R_DCHUBBUB_GATE_DIS_MASK                                                  0x00000040L
+#define DCHUBBUB_CLOCK_CNTL__DCHUBBUB_FGCG_REP_DIS_MASK                                                       0x00000080L
+//DCFCLK_CNTL
+#define DCFCLK_CNTL__DCFCLK_TURN_ON_DELAY__SHIFT                                                              0x0
+#define DCFCLK_CNTL__DCFCLK_TURN_OFF_DELAY__SHIFT                                                             0x4
+#define DCFCLK_CNTL__DCFCLK_GATE_DIS__SHIFT                                                                   0x1f
+#define DCFCLK_CNTL__DCFCLK_TURN_ON_DELAY_MASK                                                                0x0000000FL
+#define DCFCLK_CNTL__DCFCLK_TURN_OFF_DELAY_MASK                                                               0x00000FF0L
+#define DCFCLK_CNTL__DCFCLK_GATE_DIS_MASK                                                                     0x80000000L
+//DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_LATENCY_CNT_EN__SHIFT                                 0x0
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_LATENCY_EVENT_SHORT_PULSE_FILTER_EN__SHIFT            0x1
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_DF_REQ_CMD_LATENCY_SEL__SHIFT                         0x2
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ARB_LATENCY_PIPE_SEL__SHIFT                                    0x3
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ARB_LATENCY_REQ_TYPE_SEL__SHIFT                                0x7
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DF_LATENCY_URGENT_ONLY__SHIFT                                  0xa
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ROB_FIFO_LEVEL__SHIFT                                          0xb
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_LATENCY_CNT_EN_MASK                                   0x00000001L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_LATENCY_EVENT_SHORT_PULSE_FILTER_EN_MASK              0x00000002L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DCHUBBUB_DF_REQ_CMD_LATENCY_SEL_MASK                           0x00000004L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ARB_LATENCY_PIPE_SEL_MASK                                      0x00000078L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ARB_LATENCY_REQ_TYPE_SEL_MASK                                  0x00000380L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__DF_LATENCY_URGENT_ONLY_MASK                                    0x00000400L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL__ROB_FIFO_LEVEL_MASK                                            0x003FF800L
+//DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_EN__SHIFT                          0x0
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_SRC_SEL__SHIFT                     0x1
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_DUR__SHIFT                         0x4
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__LATENCY_SOURCE_SEL__SHIFT                                     0xc
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__LATENCY_DEBUG_SEL__SHIFT                                      0xf
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__ROB_MAX_FIFO_LEVEL__SHIFT                                     0x14
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__ROB_MAX_FIFO_LEVEL_RESET__SHIFT                               0x1f
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_EN_MASK                            0x00000001L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_SRC_SEL_MASK                       0x0000000EL
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__DCHUBBUB_LATENCY_FRAME_WIN_DUR_MASK                           0x00000FF0L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__LATENCY_SOURCE_SEL_MASK                                       0x00007000L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__LATENCY_DEBUG_SEL_MASK                                        0x00008000L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__ROB_MAX_FIFO_LEVEL_MASK                                       0x7FF00000L
+#define DCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2__ROB_MAX_FIFO_LEVEL_RESET_MASK                                 0x80000000L
+//DCHUBBUB_VLINE_SNAPSHOT
+#define DCHUBBUB_VLINE_SNAPSHOT__DCHUBBUB_VLINE_SNAPSHOT__SHIFT                                               0x0
+#define DCHUBBUB_VLINE_SNAPSHOT__DCHUBBUB_VLINE_SNAPSHOT_MASK                                                 0x00000001L
+//DCHUBBUB_CTRL_STATUS
+#define DCHUBBUB_CTRL_STATUS__URGENT_ZERO_SIZE_REQ_EN__SHIFT                                                  0x0
+#define DCHUBBUB_CTRL_STATUS__ROB_OVERFLOW_STATUS__SHIFT                                                      0x2
+#define DCHUBBUB_CTRL_STATUS__ROB_OVERFLOW_CLEAR__SHIFT                                                       0x3
+#define DCHUBBUB_CTRL_STATUS__DCHUBBUB_HW_DEBUG__SHIFT                                                        0x4
+#define DCHUBBUB_CTRL_STATUS__CSTATE_SWATH_CHK_GOOD_MODE__SHIFT                                               0x1f
+#define DCHUBBUB_CTRL_STATUS__URGENT_ZERO_SIZE_REQ_EN_MASK                                                    0x00000001L
+#define DCHUBBUB_CTRL_STATUS__ROB_OVERFLOW_STATUS_MASK                                                        0x00000004L
+#define DCHUBBUB_CTRL_STATUS__ROB_OVERFLOW_CLEAR_MASK                                                         0x00000008L
+#define DCHUBBUB_CTRL_STATUS__DCHUBBUB_HW_DEBUG_MASK                                                          0x3FFFFFF0L
+#define DCHUBBUB_CTRL_STATUS__CSTATE_SWATH_CHK_GOOD_MODE_MASK                                                 0x80000000L
+//DCHUBBUB_TIMEOUT_DETECTION_CTRL1
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL1__DCHUBBUB_TIMEOUT_ERROR_STATUS__SHIFT                                0x0
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL1__DCHUBBUB_TIMEOUT_REQ_STALL_THRESHOLD__SHIFT                         0x6
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL1__DCHUBBUB_TIMEOUT_ERROR_STATUS_MASK                                  0x0000003FL
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL1__DCHUBBUB_TIMEOUT_REQ_STALL_THRESHOLD_MASK                           0xFFFFFFC0L
+//DCHUBBUB_TIMEOUT_DETECTION_CTRL2
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_PSTATE_STALL_THRESHOLD__SHIFT                      0x0
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_DETECTION_EN__SHIFT                                0x1b
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_TIMER_RESET__SHIFT                                 0x1c
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_PSTATE_STALL_THRESHOLD_MASK                        0x07FFFFFFL
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_DETECTION_EN_MASK                                  0x08000000L
+#define DCHUBBUB_TIMEOUT_DETECTION_CTRL2__DCHUBBUB_TIMEOUT_TIMER_RESET_MASK                                   0x10000000L
+//DCHUBBUB_TIMEOUT_INTERRUPT_STATUS
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_ENABLE__SHIFT                                 0x0
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_STATUS__SHIFT                                 0x1
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_CLEAR__SHIFT                                  0x2
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_MASK__SHIFT                                   0x3
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_ENABLE_MASK                                   0x00000001L
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_STATUS_MASK                                   0x00000002L
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_CLEAR_MASK                                    0x00000004L
+#define DCHUBBUB_TIMEOUT_INTERRUPT_STATUS__DCHUBBUB_TIMEOUT_INT_MASK_MASK                                     0x000000F8L
+//FMON_CTRL
+#define FMON_CTRL__FMON_START__SHIFT                                                                          0x0
+#define FMON_CTRL__FMON_MODE__SHIFT                                                                           0x1
+#define FMON_CTRL__FMON_PSTATE_IGNORE__SHIFT                                                                  0x4
+#define FMON_CTRL__FMON_STATUS_IGNORE__SHIFT                                                                  0x5
+#define FMON_CTRL__FMON_URG_MODE_GREATER__SHIFT                                                               0x6
+#define FMON_CTRL__FMON_FILTER_UID_EN__SHIFT                                                                  0x7
+#define FMON_CTRL__FMON_STATE__SHIFT                                                                          0x9
+#define FMON_CTRL__FMON_URG_FILTER__SHIFT                                                                     0xc
+#define FMON_CTRL__FMON_URG_THRESHOLD__SHIFT                                                                  0xd
+#define FMON_CTRL__FMON_FILTER_UID_1__SHIFT                                                                   0x11
+#define FMON_CTRL__FMON_FILTER_UID_2__SHIFT                                                                   0x16
+#define FMON_CTRL__FMON_SOF_SEL__SHIFT                                                                        0x1b
+#define FMON_CTRL__FMON_START_MASK                                                                            0x00000001L
+#define FMON_CTRL__FMON_MODE_MASK                                                                             0x00000006L
+#define FMON_CTRL__FMON_PSTATE_IGNORE_MASK                                                                    0x00000010L
+#define FMON_CTRL__FMON_STATUS_IGNORE_MASK                                                                    0x00000020L
+#define FMON_CTRL__FMON_URG_MODE_GREATER_MASK                                                                 0x00000040L
+#define FMON_CTRL__FMON_FILTER_UID_EN_MASK                                                                    0x00000180L
+#define FMON_CTRL__FMON_STATE_MASK                                                                            0x00000600L
+#define FMON_CTRL__FMON_URG_FILTER_MASK                                                                       0x00001000L
+#define FMON_CTRL__FMON_URG_THRESHOLD_MASK                                                                    0x0001E000L
+#define FMON_CTRL__FMON_FILTER_UID_1_MASK                                                                     0x003E0000L
+#define FMON_CTRL__FMON_FILTER_UID_2_MASK                                                                     0x07C00000L
+#define FMON_CTRL__FMON_SOF_SEL_MASK                                                                          0x38000000L
+//DCHUBBUB_TEST_DEBUG_INDEX
+#define DCHUBBUB_TEST_DEBUG_INDEX__DCHUBBUB_TEST_DEBUG_INDEX__SHIFT                                           0x0
+#define DCHUBBUB_TEST_DEBUG_INDEX__DCHUBBUB_TEST_DEBUG_INDEX_MASK                                             0x000000FFL
+//DCHUBBUB_TEST_DEBUG_DATA
+#define DCHUBBUB_TEST_DEBUG_DATA__DCHUBBUB_TEST_DEBUG_DATA__SHIFT                                             0x0
+#define DCHUBBUB_TEST_DEBUG_DATA__DCHUBBUB_TEST_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dchubbubl_dchubbub_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON5_PERFCOUNTER_CNTL
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON5_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON5_PERFCOUNTER_CNTL2
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON5_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON5_PERFCOUNTER_STATE
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON5_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON5_PERFMON_CNTL
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON5_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON5_PERFMON_CNTL2
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON5_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON5_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON5_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON5_PERFMON_CVALUE_LOW
+#define DC_PERFMON5_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON5_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON5_PERFMON_HI
+#define DC_PERFMON5_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON5_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON5_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON5_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON5_PERFMON_LOW
+#define DC_PERFMON5_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON5_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dchubbubl_hubbub_vmrq_if_dispdec
+//DCN_VM_CONTEXT0_CNTL
+#define DCN_VM_CONTEXT0_CNTL__VM_CONTEXT0_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT0_CNTL__VM_CONTEXT0_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT0_CNTL__VM_CONTEXT0_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT0_CNTL__VM_CONTEXT0_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT1_CNTL
+#define DCN_VM_CONTEXT1_CNTL__VM_CONTEXT1_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT1_CNTL__VM_CONTEXT1_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT1_CNTL__VM_CONTEXT1_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT1_CNTL__VM_CONTEXT1_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT1_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT1_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT1_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT1_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT1_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT1_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT1_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT1_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT1_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT1_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT1_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT1_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT2_CNTL
+#define DCN_VM_CONTEXT2_CNTL__VM_CONTEXT2_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT2_CNTL__VM_CONTEXT2_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT2_CNTL__VM_CONTEXT2_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT2_CNTL__VM_CONTEXT2_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT2_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT2_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT2_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT2_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT2_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT2_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT2_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT2_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT2_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT2_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT2_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT2_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT3_CNTL
+#define DCN_VM_CONTEXT3_CNTL__VM_CONTEXT3_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT3_CNTL__VM_CONTEXT3_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT3_CNTL__VM_CONTEXT3_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT3_CNTL__VM_CONTEXT3_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT3_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT3_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT3_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT3_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT3_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT3_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT3_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT3_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT3_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT3_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT3_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT3_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT4_CNTL
+#define DCN_VM_CONTEXT4_CNTL__VM_CONTEXT4_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT4_CNTL__VM_CONTEXT4_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT4_CNTL__VM_CONTEXT4_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT4_CNTL__VM_CONTEXT4_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT4_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT4_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT4_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT4_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT4_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT4_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT4_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT4_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT4_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT4_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT4_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT4_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT5_CNTL
+#define DCN_VM_CONTEXT5_CNTL__VM_CONTEXT5_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT5_CNTL__VM_CONTEXT5_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT5_CNTL__VM_CONTEXT5_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT5_CNTL__VM_CONTEXT5_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT5_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT5_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT5_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT5_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT5_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT5_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT5_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT5_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT5_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT5_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT5_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT5_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT6_CNTL
+#define DCN_VM_CONTEXT6_CNTL__VM_CONTEXT6_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT6_CNTL__VM_CONTEXT6_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT6_CNTL__VM_CONTEXT6_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT6_CNTL__VM_CONTEXT6_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT6_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT6_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT6_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT6_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT6_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT6_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT6_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT6_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT6_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT6_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT6_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT6_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT7_CNTL
+#define DCN_VM_CONTEXT7_CNTL__VM_CONTEXT7_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT7_CNTL__VM_CONTEXT7_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT7_CNTL__VM_CONTEXT7_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT7_CNTL__VM_CONTEXT7_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT7_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT7_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT7_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT7_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT7_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT7_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT7_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT7_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT7_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT7_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT7_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT7_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT8_CNTL
+#define DCN_VM_CONTEXT8_CNTL__VM_CONTEXT8_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT8_CNTL__VM_CONTEXT8_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT8_CNTL__VM_CONTEXT8_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT8_CNTL__VM_CONTEXT8_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT8_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT8_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT8_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT8_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT8_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT8_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT8_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT8_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT8_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT8_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT8_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT8_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT9_CNTL
+#define DCN_VM_CONTEXT9_CNTL__VM_CONTEXT9_PAGE_TABLE_DEPTH__SHIFT                                             0x1
+#define DCN_VM_CONTEXT9_CNTL__VM_CONTEXT9_PAGE_TABLE_BLOCK_SIZE__SHIFT                                        0x3
+#define DCN_VM_CONTEXT9_CNTL__VM_CONTEXT9_PAGE_TABLE_DEPTH_MASK                                               0x00000006L
+#define DCN_VM_CONTEXT9_CNTL__VM_CONTEXT9_PAGE_TABLE_BLOCK_SIZE_MASK                                          0x00000078L
+//DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT9_PAGE_DIRECTORY_ENTRY_HI32__SHIFT               0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT9_PAGE_DIRECTORY_ENTRY_HI32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT9_PAGE_DIRECTORY_ENTRY_LO32__SHIFT               0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT9_PAGE_DIRECTORY_ENTRY_LO32_MASK                 0xFFFFFFFFL
+//DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT9_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT          0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT9_START_LOGICAL_PAGE_NUMBER_HI4_MASK            0x0000000FL
+//DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT9_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT         0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT9_START_LOGICAL_PAGE_NUMBER_LO32_MASK           0xFFFFFFFFL
+//DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT9_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT              0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT9_END_LOGICAL_PAGE_NUMBER_HI4_MASK                0x0000000FL
+//DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT9_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT9_END_LOGICAL_PAGE_NUMBER_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT10_CNTL
+#define DCN_VM_CONTEXT10_CNTL__VM_CONTEXT10_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT10_CNTL__VM_CONTEXT10_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT10_CNTL__VM_CONTEXT10_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT10_CNTL__VM_CONTEXT10_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT10_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT10_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT10_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT10_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT10_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT10_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT10_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT10_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT10_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT10_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT10_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT10_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_CONTEXT11_CNTL
+#define DCN_VM_CONTEXT11_CNTL__VM_CONTEXT11_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT11_CNTL__VM_CONTEXT11_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT11_CNTL__VM_CONTEXT11_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT11_CNTL__VM_CONTEXT11_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT11_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT11_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT11_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT11_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT11_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT11_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT11_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT11_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT11_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT11_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT11_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT11_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_CONTEXT12_CNTL
+#define DCN_VM_CONTEXT12_CNTL__VM_CONTEXT12_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT12_CNTL__VM_CONTEXT12_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT12_CNTL__VM_CONTEXT12_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT12_CNTL__VM_CONTEXT12_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT12_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT12_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT12_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT12_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT12_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT12_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT12_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT12_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT12_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT12_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT12_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT12_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_CONTEXT13_CNTL
+#define DCN_VM_CONTEXT13_CNTL__VM_CONTEXT13_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT13_CNTL__VM_CONTEXT13_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT13_CNTL__VM_CONTEXT13_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT13_CNTL__VM_CONTEXT13_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT13_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT13_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT13_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT13_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT13_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT13_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT13_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT13_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT13_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT13_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT13_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT13_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_CONTEXT14_CNTL
+#define DCN_VM_CONTEXT14_CNTL__VM_CONTEXT14_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT14_CNTL__VM_CONTEXT14_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT14_CNTL__VM_CONTEXT14_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT14_CNTL__VM_CONTEXT14_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT14_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT14_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT14_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT14_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT14_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT14_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT14_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT14_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT14_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT14_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT14_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT14_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_CONTEXT15_CNTL
+#define DCN_VM_CONTEXT15_CNTL__VM_CONTEXT15_PAGE_TABLE_DEPTH__SHIFT                                           0x1
+#define DCN_VM_CONTEXT15_CNTL__VM_CONTEXT15_PAGE_TABLE_BLOCK_SIZE__SHIFT                                      0x3
+#define DCN_VM_CONTEXT15_CNTL__VM_CONTEXT15_PAGE_TABLE_DEPTH_MASK                                             0x00000006L
+#define DCN_VM_CONTEXT15_CNTL__VM_CONTEXT15_PAGE_TABLE_BLOCK_SIZE_MASK                                        0x00000078L
+//DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT15_PAGE_DIRECTORY_ENTRY_HI32__SHIFT             0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__VM_CONTEXT15_PAGE_DIRECTORY_ENTRY_HI32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT15_PAGE_DIRECTORY_ENTRY_LO32__SHIFT             0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__VM_CONTEXT15_PAGE_DIRECTORY_ENTRY_LO32_MASK               0xFFFFFFFFL
+//DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT15_START_LOGICAL_PAGE_NUMBER_HI4__SHIFT        0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__VM_CONTEXT15_START_LOGICAL_PAGE_NUMBER_HI4_MASK          0x0000000FL
+//DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT15_START_LOGICAL_PAGE_NUMBER_LO32__SHIFT       0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__VM_CONTEXT15_START_LOGICAL_PAGE_NUMBER_LO32_MASK         0xFFFFFFFFL
+//DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT15_END_LOGICAL_PAGE_NUMBER_HI4__SHIFT            0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__VM_CONTEXT15_END_LOGICAL_PAGE_NUMBER_HI4_MASK              0x0000000FL
+//DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT15_END_LOGICAL_PAGE_NUMBER_LO32__SHIFT           0x0
+#define DCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__VM_CONTEXT15_END_LOGICAL_PAGE_NUMBER_LO32_MASK             0xFFFFFFFFL
+//DCN_VM_DEFAULT_ADDR_MSB
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_ADDR_MSB__SHIFT                                               0x0
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_SPA__SHIFT                                                    0x1c
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_SNOOP__SHIFT                                                  0x1d
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_ADDR_MSB_MASK                                                 0x0000000FL
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_SPA_MASK                                                      0x10000000L
+#define DCN_VM_DEFAULT_ADDR_MSB__DCN_VM_DEFAULT_SNOOP_MASK                                                    0x20000000L
+//DCN_VM_DEFAULT_ADDR_LSB
+#define DCN_VM_DEFAULT_ADDR_LSB__DCN_VM_DEFAULT_ADDR_LSB__SHIFT                                               0x0
+#define DCN_VM_DEFAULT_ADDR_LSB__DCN_VM_DEFAULT_ADDR_LSB_MASK                                                 0xFFFFFFFFL
+//DCN_VM_FAULT_CNTL
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_STATUS_CLEAR__SHIFT                                                   0x0
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_STATUS_MODE__SHIFT                                                    0x1
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_INTERRUPT_ENABLE__SHIFT                                               0x2
+#define DCN_VM_FAULT_CNTL__DCN_VM_RANGE_FAULT_DISABLE__SHIFT                                                  0x8
+#define DCN_VM_FAULT_CNTL__DCN_VM_PRQ_FAULT_DISABLE__SHIFT                                                    0x9
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_STATUS_CLEAR_MASK                                                     0x00000001L
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_STATUS_MODE_MASK                                                      0x00000002L
+#define DCN_VM_FAULT_CNTL__DCN_VM_ERROR_INTERRUPT_ENABLE_MASK                                                 0x00000004L
+#define DCN_VM_FAULT_CNTL__DCN_VM_RANGE_FAULT_DISABLE_MASK                                                    0x00000100L
+#define DCN_VM_FAULT_CNTL__DCN_VM_PRQ_FAULT_DISABLE_MASK                                                      0x00000200L
+//DCN_VM_FAULT_STATUS
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_STATUS__SHIFT                                                       0x0
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_VMID__SHIFT                                                         0x10
+#define DCN_VM_FAULT_STATUS__DCN_VM_TR_RESP_ERROR_VMID__SHIFT                                                 0x14
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_TABLE_LEVEL__SHIFT                                                  0x18
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_PIPE__SHIFT                                                         0x1a
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_INTERRUPT_STATUS__SHIFT                                             0x1f
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_STATUS_MASK                                                         0x0000FFFFL
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_VMID_MASK                                                           0x000F0000L
+#define DCN_VM_FAULT_STATUS__DCN_VM_TR_RESP_ERROR_VMID_MASK                                                   0x00F00000L
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_TABLE_LEVEL_MASK                                                    0x03000000L
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_PIPE_MASK                                                           0x3C000000L
+#define DCN_VM_FAULT_STATUS__DCN_VM_ERROR_INTERRUPT_STATUS_MASK                                               0x80000000L
+//DCN_VM_FAULT_ADDR_MSB
+#define DCN_VM_FAULT_ADDR_MSB__DCN_VM_FAULT_ADDR_MSB__SHIFT                                                   0x0
+#define DCN_VM_FAULT_ADDR_MSB__DCN_VM_FAULT_ADDR_MSB_MASK                                                     0x0000000FL
+//DCN_VM_FAULT_ADDR_LSB
+#define DCN_VM_FAULT_ADDR_LSB__DCN_VM_FAULT_ADDR_LSB__SHIFT                                                   0x0
+#define DCN_VM_FAULT_ADDR_LSB__DCN_VM_FAULT_ADDR_LSB_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dispdec
+//HUBP0_DCSURF_SURFACE_CONFIG
+#define HUBP0_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT__SHIFT                                              0x0
+#define HUBP0_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE__SHIFT                                                    0x8
+#define HUBP0_DCSURF_SURFACE_CONFIG__H_MIRROR_EN__SHIFT                                                       0xa
+#define HUBP0_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN__SHIFT                                                    0xb
+#define HUBP0_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT_MASK                                                0x0000007FL
+#define HUBP0_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE_MASK                                                      0x00000300L
+#define HUBP0_DCSURF_SURFACE_CONFIG__H_MIRROR_EN_MASK                                                         0x00000400L
+#define HUBP0_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN_MASK                                                      0x00000800L
+//HUBP0_DCSURF_ADDR_CONFIG
+#define HUBP0_DCSURF_ADDR_CONFIG__NUM_PIPES__SHIFT                                                            0x0
+#define HUBP0_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE__SHIFT                                                      0x6
+#define HUBP0_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                 0xc
+#define HUBP0_DCSURF_ADDR_CONFIG__NUM_PKRS__SHIFT                                                             0x10
+#define HUBP0_DCSURF_ADDR_CONFIG__NUM_PIPES_MASK                                                              0x00000007L
+#define HUBP0_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE_MASK                                                        0x000000C0L
+#define HUBP0_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                   0x00003000L
+#define HUBP0_DCSURF_ADDR_CONFIG__NUM_PKRS_MASK                                                               0x00070000L
+//HUBP0_DCSURF_TILING_CONFIG
+#define HUBP0_DCSURF_TILING_CONFIG__SW_MODE__SHIFT                                                            0x0
+#define HUBP0_DCSURF_TILING_CONFIG__DIM_TYPE__SHIFT                                                           0x7
+#define HUBP0_DCSURF_TILING_CONFIG__META_LINEAR__SHIFT                                                        0x9
+#define HUBP0_DCSURF_TILING_CONFIG__PIPE_ALIGNED__SHIFT                                                       0xb
+#define HUBP0_DCSURF_TILING_CONFIG__SW_MODE_MASK                                                              0x0000001FL
+#define HUBP0_DCSURF_TILING_CONFIG__DIM_TYPE_MASK                                                             0x00000180L
+#define HUBP0_DCSURF_TILING_CONFIG__META_LINEAR_MASK                                                          0x00000200L
+#define HUBP0_DCSURF_TILING_CONFIG__PIPE_ALIGNED_MASK                                                         0x00000800L
+//HUBP0_DCSURF_PRI_VIEWPORT_START
+#define HUBP0_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP0_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP0_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP0_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP0_DCSURF_PRI_VIEWPORT_START_C
+#define HUBP0_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP0_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP0_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP0_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP0_DCSURF_SEC_VIEWPORT_START
+#define HUBP0_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP0_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP0_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP0_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP0_DCSURF_SEC_VIEWPORT_START_C
+#define HUBP0_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP0_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP0_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP0_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP0_DCHUBP_REQ_SIZE_CONFIG
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT__SHIFT                                                     0x0
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR__SHIFT                                            0x4
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE__SHIFT                                                       0x8
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE__SHIFT                                                   0xb
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE__SHIFT                                                  0x10
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE__SHIFT                                              0x12
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE__SHIFT                                                  0x14
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE__SHIFT                                                    0x18
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT_MASK                                                       0x00000007L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR_MASK                                              0x00000070L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE_MASK                                                         0x00000700L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE_MASK                                                     0x00001800L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE_MASK                                                    0x00030000L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE_MASK                                                0x000C0000L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE_MASK                                                    0x00700000L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE_MASK                                                      0x07000000L
+//HUBP0_DCHUBP_REQ_SIZE_CONFIG_C
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C__SHIFT                                                 0x0
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C__SHIFT                                        0x4
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C__SHIFT                                                   0x8
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C__SHIFT                                               0xb
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C__SHIFT                                              0x10
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C__SHIFT                                          0x12
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C__SHIFT                                              0x14
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C_MASK                                                   0x00000007L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C_MASK                                          0x00000070L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C_MASK                                                     0x00000700L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C_MASK                                                 0x00001800L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C_MASK                                                0x00030000L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C_MASK                                            0x000C0000L
+#define HUBP0_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C_MASK                                                0x00700000L
+//HUBP0_DCHUBP_CNTL
+#define HUBP0_DCHUBP_CNTL__HUBP_BLANK_EN__SHIFT                                                               0x0
+#define HUBP0_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ__SHIFT                                                     0x1
+#define HUBP0_DCHUBP_CNTL__HUBP_SOFT_RESET__SHIFT                                                             0x2
+#define HUBP0_DCHUBP_CNTL__HUBP_IN_BLANK__SHIFT                                                               0x3
+#define HUBP0_DCHUBP_CNTL__HUBP_VTG_SEL__SHIFT                                                                0x4
+#define HUBP0_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC__SHIFT                                               0x8
+#define HUBP0_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM__SHIFT                                            0x9
+#define HUBP0_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE__SHIFT                                                     0xa
+#define HUBP0_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS__SHIFT                                                   0xb
+#define HUBP0_DCHUBP_CNTL__HUBP_TTU_DISABLE__SHIFT                                                            0xc
+#define HUBP0_DCHUBP_CNTL__HUBP_TTU_MODE__SHIFT                                                               0xd
+#define HUBP0_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ__SHIFT                                                 0x10
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS__SHIFT                                                         0x14
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD__SHIFT                                                      0x18
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR__SHIFT                                                   0x1a
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN__SHIFT                                                   0x1b
+#define HUBP0_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS__SHIFT                                                       0x1c
+#define HUBP0_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR__SHIFT                                                        0x1f
+#define HUBP0_DCHUBP_CNTL__HUBP_BLANK_EN_MASK                                                                 0x00000001L
+#define HUBP0_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ_MASK                                                       0x00000002L
+#define HUBP0_DCHUBP_CNTL__HUBP_SOFT_RESET_MASK                                                               0x00000004L
+#define HUBP0_DCHUBP_CNTL__HUBP_IN_BLANK_MASK                                                                 0x00000008L
+#define HUBP0_DCHUBP_CNTL__HUBP_VTG_SEL_MASK                                                                  0x000000F0L
+#define HUBP0_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC_MASK                                                 0x00000100L
+#define HUBP0_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM_MASK                                              0x00000200L
+#define HUBP0_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE_MASK                                                       0x00000400L
+#define HUBP0_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS_MASK                                                     0x00000800L
+#define HUBP0_DCHUBP_CNTL__HUBP_TTU_DISABLE_MASK                                                              0x00001000L
+#define HUBP0_DCHUBP_CNTL__HUBP_TTU_MODE_MASK                                                                 0x0000E000L
+#define HUBP0_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ_MASK                                                   0x000F0000L
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_MASK                                                           0x00F00000L
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD_MASK                                                        0x03000000L
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR_MASK                                                     0x04000000L
+#define HUBP0_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN_MASK                                                     0x08000000L
+#define HUBP0_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS_MASK                                                         0x70000000L
+#define HUBP0_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR_MASK                                                          0x80000000L
+//HUBP0_HUBP_CLK_CNTL
+#define HUBP0_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE__SHIFT                                                         0x0
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS__SHIFT                                                   0x5
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS__SHIFT                                                    0x8
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS__SHIFT                                                    0xc
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS__SHIFT                                                    0x10
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON__SHIFT                                                   0x14
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON__SHIFT                                                    0x15
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON__SHIFT                                                    0x16
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON__SHIFT                                                    0x17
+#define HUBP0_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON__SHIFT                                                   0x19
+#define HUBP0_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define HUBP0_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE_MASK                                                           0x00000001L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS_MASK                                                     0x00000020L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS_MASK                                                      0x00000100L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS_MASK                                                      0x00001000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS_MASK                                                      0x00010000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON_MASK                                                     0x00100000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON_MASK                                                      0x00200000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON_MASK                                                      0x00400000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON_MASK                                                      0x00800000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON_MASK                                                     0x02000000L
+#define HUBP0_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL_MASK                                                           0xF0000000L
+//HUBP0_DCHUBP_VMPG_CONFIG
+#define HUBP0_DCHUBP_VMPG_CONFIG__VMPG_SIZE__SHIFT                                                            0x0
+#define HUBP0_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE__SHIFT                                                      0x1
+#define HUBP0_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE__SHIFT                                                   0x2
+#define HUBP0_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME__SHIFT                                              0x7
+#define HUBP0_DCHUBP_VMPG_CONFIG__VMPG_SIZE_MASK                                                              0x00000001L
+#define HUBP0_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE_MASK                                                        0x00000002L
+#define HUBP0_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE_MASK                                                     0x0000007CL
+#define HUBP0_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME_MASK                                                0x00000080L
+//HUBP0_DCHUBP_MALL_CONFIG
+#define HUBP0_DCHUBP_MALL_CONFIG__USE_MALL_SEL__SHIFT                                                         0x0
+#define HUBP0_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR__SHIFT                                                  0x2
+#define HUBP0_DCHUBP_MALL_CONFIG__USE_MALL_SEL_MASK                                                           0x00000003L
+#define HUBP0_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR_MASK                                                    0x00000004L
+//HUBP0_DCHUBP_MALL_SUB_VP
+#define HUBP0_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE__SHIFT                                               0x0
+#define HUBP0_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0__SHIFT                                                 0x1
+#define HUBP0_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1__SHIFT                                                 0xf
+#define HUBP0_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE_MASK                                                 0x00000001L
+#define HUBP0_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0_MASK                                                   0x00007FFEL
+#define HUBP0_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1_MASK                                                   0x1FFF8000L
+//HUBP0_HUBPREQ_DEBUG_DB
+#define HUBP0_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG__SHIFT                                                          0x0
+#define HUBP0_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG_MASK                                                            0xFFFFFFFFL
+//HUBP0_HUBPREQ_DEBUG
+#define HUBP0_HUBPREQ_DEBUG__HUBPREQ_DEBUG__SHIFT                                                             0x0
+#define HUBP0_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS__SHIFT                                 0x1f
+#define HUBP0_HUBPREQ_DEBUG__HUBPREQ_DEBUG_MASK                                                               0x7FFFFFFFL
+#define HUBP0_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS_MASK                                   0x80000000L
+//HUBP0_HUBP_DEBUG_CTRL
+#define HUBP0_HUBP_DEBUG_CTRL__HUBP_DBG_EN__SHIFT                                                             0x0
+#define HUBP0_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS__SHIFT                                              0x4
+#define HUBP0_HUBP_DEBUG_CTRL__HUBP_DBG_EN_MASK                                                               0x00000001L
+#define HUBP0_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS_MASK                                                0x00000010L
+//HUBP0_HUBP_DEBUG_MUX_DCFCLK
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK__SHIFT                                         0x0
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK__SHIFT                                         0x8
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK__SHIFT                                         0x10
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK__SHIFT                                         0x18
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK_MASK                                           0x000000FFL
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK_MASK                                           0x0000FF00L
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK_MASK                                           0x00FF0000L
+#define HUBP0_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK_MASK                                           0xFF000000L
+//HUBP0_HUBP_DEBUG_MUX_DPPCLK
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK__SHIFT                                         0x0
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK__SHIFT                                         0x4
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK__SHIFT                                        0x8
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK_MASK                                           0x00000007L
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK_MASK                                           0x00000070L
+#define HUBP0_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK_MASK                                          0x00000700L
+//HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK__SHIFT                                 0x0
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK__SHIFT                          0x4
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK__SHIFT                              0xc
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK__SHIFT                               0x14
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK__SHIFT                               0x1c
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK_MASK                                   0x00000001L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK_MASK                            0x00000FF0L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK_MASK                                0x0001F000L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK_MASK                                 0x01F00000L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK_MASK                                 0x30000000L
+//HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK__SHIFT                                 0x0
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK__SHIFT                            0x1
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK__SHIFT                          0x4
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK__SHIFT                              0xc
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK__SHIFT                               0x14
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK_MASK                                   0x00000001L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK_MASK                              0x00000002L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK_MASK                            0x00000FF0L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK_MASK                                0x0001F000L
+#define HUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK_MASK                                 0x01F00000L
+//HUBP0_HUBP_MALL_STATUS
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN__SHIFT                                         0x0
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE__SHIFT                                         0x1
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQUEST__SHIFT                                                       0x2
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_RESPONSE__SHIFT                                                      0x3
+#define HUBP0_HUBP_MALL_STATUS__MALL_IN_USE__SHIFT                                                            0x4
+#define HUBP0_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE__SHIFT                                                 0x5
+#define HUBP0_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE__SHIFT                                                   0x6
+#define HUBP0_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE__SHIFT                                                  0x7
+#define HUBP0_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE__SHIFT                                                  0x8
+#define HUBP0_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH__SHIFT                                                  0x9
+#define HUBP0_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME__SHIFT                                                    0xa
+#define HUBP0_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME__SHIFT                                                    0xb
+#define HUBP0_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL__SHIFT                                                     0xc
+#define HUBP0_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL__SHIFT                                                     0xd
+#define HUBP0_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL__SHIFT                                                     0xe
+#define HUBP0_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME__SHIFT                                                  0xf
+#define HUBP0_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS__SHIFT                                   0x10
+#define HUBP0_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING__SHIFT                                            0x11
+#define HUBP0_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING__SHIFT                                                   0x12
+#define HUBP0_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO__SHIFT                                                      0x13
+#define HUBP0_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x14
+#define HUBP0_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x15
+#define HUBP0_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING__SHIFT                                                   0x16
+#define HUBP0_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP__SHIFT                                              0x17
+#define HUBP0_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING__SHIFT                                                   0x18
+#define HUBP0_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING__SHIFT                                                  0x19
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN_MASK                                           0x00000001L
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE_MASK                                           0x00000002L
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_REQUEST_MASK                                                         0x00000004L
+#define HUBP0_HUBP_MALL_STATUS__MALL_USE_RESPONSE_MASK                                                        0x00000008L
+#define HUBP0_HUBP_MALL_STATUS__MALL_IN_USE_MASK                                                              0x00000010L
+#define HUBP0_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE_MASK                                                   0x00000020L
+#define HUBP0_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE_MASK                                                     0x00000040L
+#define HUBP0_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE_MASK                                                    0x00000080L
+#define HUBP0_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE_MASK                                                    0x00000100L
+#define HUBP0_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH_MASK                                                    0x00000200L
+#define HUBP0_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME_MASK                                                      0x00000400L
+#define HUBP0_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME_MASK                                                      0x00000800L
+#define HUBP0_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL_MASK                                                       0x00001000L
+#define HUBP0_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_MASK                                                       0x00002000L
+#define HUBP0_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL_MASK                                                       0x00004000L
+#define HUBP0_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME_MASK                                                    0x00008000L
+#define HUBP0_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS_MASK                                     0x00010000L
+#define HUBP0_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING_MASK                                              0x00020000L
+#define HUBP0_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING_MASK                                                     0x00040000L
+#define HUBP0_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO_MASK                                                        0x00080000L
+#define HUBP0_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP_MASK                                              0x00100000L
+#define HUBP0_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP_MASK                                              0x00200000L
+#define HUBP0_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING_MASK                                                     0x00400000L
+#define HUBP0_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP_MASK                                                0x00800000L
+#define HUBP0_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING_MASK                                                     0x01000000L
+#define HUBP0_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING_MASK                                                    0x02000000L
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpreq_dispdec
+//HUBPREQ0_DCSURF_SURFACE_PITCH
+#define HUBPREQ0_DCSURF_SURFACE_PITCH__PITCH__SHIFT                                                           0x0
+#define HUBPREQ0_DCSURF_SURFACE_PITCH__META_PITCH__SHIFT                                                      0x10
+#define HUBPREQ0_DCSURF_SURFACE_PITCH__PITCH_MASK                                                             0x00003FFFL
+#define HUBPREQ0_DCSURF_SURFACE_PITCH__META_PITCH_MASK                                                        0x3FFF0000L
+//HUBPREQ0_DCSURF_SURFACE_PITCH_C
+#define HUBPREQ0_DCSURF_SURFACE_PITCH_C__PITCH_C__SHIFT                                                       0x0
+#define HUBPREQ0_DCSURF_SURFACE_PITCH_C__META_PITCH_C__SHIFT                                                  0x10
+#define HUBPREQ0_DCSURF_SURFACE_PITCH_C__PITCH_C_MASK                                                         0x00003FFFL
+#define HUBPREQ0_DCSURF_SURFACE_PITCH_C__META_PITCH_C_MASK                                                    0x3FFF0000L
+//HUBPREQ0_VMID_SETTINGS_0
+#define HUBPREQ0_VMID_SETTINGS_0__VMID__SHIFT                                                                 0x0
+#define HUBPREQ0_VMID_SETTINGS_0__VMID_MASK                                                                   0x0000000FL
+//HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS__SHIFT                               0x0
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS_MASK                                 0xFFFFFFFFL
+//HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH__SHIFT                     0x0
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH_MASK                       0x0000FFFFL
+//HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C__SHIFT                           0x0
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C_MASK                             0xFFFFFFFFL
+//HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C__SHIFT                 0x0
+#define HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C_MASK                   0x0000FFFFL
+//HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS__SHIFT                           0x0
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS_MASK                             0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT                 0x0
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH_MASK                   0x0000FFFFL
+//HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C__SHIFT                       0x0
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C_MASK                         0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C__SHIFT             0x0
+#define HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C_MASK               0x0000FFFFL
+//HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS__SHIFT                     0x0
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS_MASK                       0xFFFFFFFFL
+//HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH__SHIFT           0x0
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH_MASK             0x0000FFFFL
+//HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C__SHIFT                 0x0
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C_MASK                   0xFFFFFFFFL
+//HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT       0x0
+#define HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C_MASK         0x0000FFFFL
+//HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS__SHIFT                 0x0
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS_MASK                   0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH__SHIFT       0x0
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH_MASK         0x0000FFFFL
+//HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C__SHIFT             0x0
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C_MASK               0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT   0x0
+#define HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C_MASK     0x0000FFFFL
+//HUBPREQ0_DCSURF_SURFACE_CONTROL
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ__SHIFT                                           0x0
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN__SHIFT                                        0x4
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK__SHIFT                                   0x5
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C__SHIFT                                         0x7
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C__SHIFT                                 0xb
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ__SHIFT                                         0xd
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN__SHIFT                                      0x11
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK__SHIFT                                 0x12
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C__SHIFT                                       0x14
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C__SHIFT                               0x18
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ__SHIFT                                      0x1a
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C__SHIFT                                    0x1b
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ__SHIFT                                    0x1c
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C__SHIFT                                  0x1d
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_MASK                                             0x0000000FL
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN_MASK                                          0x00000010L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_MASK                                     0x00000060L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C_MASK                                           0x00000780L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C_MASK                                   0x00001800L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_MASK                                           0x0001E000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN_MASK                                        0x00020000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_MASK                                   0x000C0000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C_MASK                                         0x00F00000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C_MASK                                 0x03000000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_MASK                                        0x04000000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C_MASK                                      0x08000000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_MASK                                      0x10000000L
+#define HUBPREQ0_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C_MASK                                    0x20000000L
+//HUBPREQ0_DCSURF_FLIP_CONTROL
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK__SHIFT                                              0x0
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE__SHIFT                                                0x1
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM__SHIFT                                    0x4
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING__SHIFT                                             0x8
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS__SHIFT                                0x9
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC__SHIFT                                 0xc
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC__SHIFT                                       0x10
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE__SHIFT                               0x11
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY__SHIFT                              0x12
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY__SHIFT                                       0x14
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK_MASK                                                0x00000001L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE_MASK                                                  0x00000002L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM_MASK                                      0x000000F0L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_MASK                                               0x00000100L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS_MASK                                  0x00000200L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC_MASK                                   0x00003000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC_MASK                                         0x00010000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE_MASK                                 0x00020000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY_MASK                                0x00040000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY_MASK                                         0x3FF00000L
+//HUBPREQ0_DCSURF_FLIP_CONTROL2
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME__SHIFT                                   0x0
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE__SHIFT                                              0x8
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK__SHIFT                                                0x9
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE__SHIFT                                    0xa
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH__SHIFT                                     0xc
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE__SHIFT                                        0x1c
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE__SHIFT                                    0x1f
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME_MASK                                     0x000000FFL
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE_MASK                                                0x00000100L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK_MASK                                                  0x00000200L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE_MASK                                      0x00000400L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH_MASK                                       0x00001000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE_MASK                                          0x10000000L
+#define HUBPREQ0_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE_MASK                                      0x80000000L
+//HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK__SHIFT                                  0x0
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE__SHIFT                                  0x1
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK__SHIFT                             0x2
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE__SHIFT                             0x3
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR__SHIFT                                     0x8
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR__SHIFT                                0x9
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED__SHIFT                                  0x10
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS__SHIFT                                0x11
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED__SHIFT                             0x12
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS__SHIFT                           0x13
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK_MASK                                    0x00000001L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE_MASK                                    0x00000002L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK_MASK                               0x00000004L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE_MASK                               0x00000008L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR_MASK                                       0x00000100L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR_MASK                                  0x00000200L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED_MASK                                    0x00010000L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS_MASK                                  0x00020000L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED_MASK                               0x00040000L
+#define HUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS_MASK                             0x00080000L
+//HUBPREQ0_DCSURF_SURFACE_INUSE
+#define HUBPREQ0_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS__SHIFT                                           0x0
+#define HUBPREQ0_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS_MASK                                             0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH__SHIFT                                 0x0
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID__SHIFT                                         0x1c
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH_MASK                                   0x0000FFFFL
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID_MASK                                           0xF0000000L
+//HUBPREQ0_DCSURF_SURFACE_INUSE_C
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C__SHIFT                                       0x0
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C_MASK                                         0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C__SHIFT                             0x0
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C__SHIFT                                     0x1c
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C_MASK                               0x0000FFFFL
+#define HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C_MASK                                       0xF0000000L
+//HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS__SHIFT                         0x0
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS_MASK                           0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH__SHIFT               0x0
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID__SHIFT                       0x1c
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_MASK                 0x0000FFFFL
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID_MASK                         0xF0000000L
+//HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C__SHIFT                     0x0
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C_MASK                       0xFFFFFFFFL
+//HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C__SHIFT           0x0
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C__SHIFT                   0x1c
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C_MASK             0x0000FFFFL
+#define HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C_MASK                     0xF0000000L
+//HUBPREQ0_DCN_EXPANSION_MODE
+#define HUBPREQ0_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE__SHIFT                                                0x0
+#define HUBPREQ0_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE__SHIFT                                                0x2
+#define HUBPREQ0_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE__SHIFT                                                0x4
+#define HUBPREQ0_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE__SHIFT                                                0x6
+#define HUBPREQ0_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE_MASK                                                  0x00000003L
+#define HUBPREQ0_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE_MASK                                                  0x0000000CL
+#define HUBPREQ0_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE_MASK                                                  0x00000030L
+#define HUBPREQ0_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE_MASK                                                  0x000000C0L
+//HUBPREQ0_DCN_TTU_QOS_WM
+#define HUBPREQ0_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM__SHIFT                                                      0x0
+#define HUBPREQ0_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM__SHIFT                                                     0x10
+#define HUBPREQ0_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM_MASK                                                        0x00003FFFL
+#define HUBPREQ0_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM_MASK                                                       0x3FFF0000L
+//HUBPREQ0_DCN_GLOBAL_TTU_CNTL
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK__SHIFT                                                   0x0
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT__SHIFT                                             0x18
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT__SHIFT                                             0x19
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE__SHIFT                                                     0x1b
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP__SHIFT                                                   0x1c
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK_MASK                                                     0x00FFFFFFL
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT_MASK                                               0x01000000L
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT_MASK                                               0x02000000L
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE_MASK                                                       0x08000000L
+#define HUBPREQ0_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP_MASK                                                     0xF0000000L
+//HUBPREQ0_DCN_SURF0_TTU_CNTL0
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ0_DCN_SURF0_TTU_CNTL1
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ0_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ0_DCN_SURF1_TTU_CNTL0
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ0_DCN_SURF1_TTU_CNTL1
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ0_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ0_DCN_CUR0_TTU_CNTL0
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ0_DCN_CUR0_TTU_CNTL1
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ0_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ0_DCN_CUR1_TTU_CNTL0
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ0_DCN_CUR1_TTU_CNTL1
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ0_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ0_DCN_DMDATA_VM_CNTL
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA__SHIFT                                              0x0
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS__SHIFT                                            0x10
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR__SHIFT                                      0x14
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS__SHIFT                                        0x18
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS__SHIFT                                             0x19
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR__SHIFT                                  0x1a
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE__SHIFT                                                    0x1f
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA_MASK                                                0x0000FFFFL
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_MASK                                              0x000F0000L
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR_MASK                                        0x00100000L
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_MASK                                          0x01000000L
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS_MASK                                               0x02000000L
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR_MASK                                    0x04000000L
+#define HUBPREQ0_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE_MASK                                                      0x80000000L
+//HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR
+#define HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR__SHIFT                       0x0
+#define HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR_MASK                         0x3FFFFFFFL
+//HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR__SHIFT                     0x0
+#define HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MASK                       0x3FFFFFFFL
+//HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                  0x0
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                             0x3
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                0x5
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                   0x6
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                    0x00000001L
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                               0x00000018L
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                  0x00000020L
+#define HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                     0x00000040L
+//HUBPREQ0_BLANK_OFFSET_0
+#define HUBPREQ0_BLANK_OFFSET_0__REFCYC_H_BLANK_END__SHIFT                                                    0x0
+#define HUBPREQ0_BLANK_OFFSET_0__DLG_V_BLANK_END__SHIFT                                                       0x10
+#define HUBPREQ0_BLANK_OFFSET_0__REFCYC_H_BLANK_END_MASK                                                      0x00001FFFL
+#define HUBPREQ0_BLANK_OFFSET_0__DLG_V_BLANK_END_MASK                                                         0x7FFF0000L
+//HUBPREQ0_BLANK_OFFSET_1
+#define HUBPREQ0_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START__SHIFT                                                  0x0
+#define HUBPREQ0_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START_MASK                                                    0x0003FFFFL
+//HUBPREQ0_DST_DIMENSIONS
+#define HUBPREQ0_DST_DIMENSIONS__REFCYC_PER_HTOTAL__SHIFT                                                     0x0
+#define HUBPREQ0_DST_DIMENSIONS__REFCYC_PER_HTOTAL_MASK                                                       0x001FFFFFL
+//HUBPREQ0_DST_AFTER_SCALER
+#define HUBPREQ0_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER__SHIFT                                               0x0
+#define HUBPREQ0_DST_AFTER_SCALER__DST_Y_AFTER_SCALER__SHIFT                                                  0x10
+#define HUBPREQ0_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER_MASK                                                 0x00001FFFL
+#define HUBPREQ0_DST_AFTER_SCALER__DST_Y_AFTER_SCALER_MASK                                                    0x00070000L
+//HUBPREQ0_PREFETCH_SETTINGS
+#define HUBPREQ0_PREFETCH_SETTINGS__VRATIO_PREFETCH__SHIFT                                                    0x0
+#define HUBPREQ0_PREFETCH_SETTINGS__DST_Y_PREFETCH__SHIFT                                                     0x18
+#define HUBPREQ0_PREFETCH_SETTINGS__VRATIO_PREFETCH_MASK                                                      0x003FFFFFL
+#define HUBPREQ0_PREFETCH_SETTINGS__DST_Y_PREFETCH_MASK                                                       0xFF000000L
+//HUBPREQ0_PREFETCH_SETTINGS_C
+#define HUBPREQ0_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C__SHIFT                                                0x0
+#define HUBPREQ0_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C_MASK                                                  0x003FFFFFL
+//HUBPREQ0_VBLANK_PARAMETERS_0
+#define HUBPREQ0_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK__SHIFT                                              0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK__SHIFT                                             0x8
+#define HUBPREQ0_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK_MASK                                                0x0000007FL
+#define HUBPREQ0_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK_MASK                                               0x00003F00L
+//HUBPREQ0_VBLANK_PARAMETERS_1
+#define HUBPREQ0_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L__SHIFT                                    0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L_MASK                                      0x007FFFFFL
+//HUBPREQ0_VBLANK_PARAMETERS_2
+#define HUBPREQ0_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C__SHIFT                                    0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C_MASK                                      0x007FFFFFL
+//HUBPREQ0_VBLANK_PARAMETERS_3
+#define HUBPREQ0_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L__SHIFT                                   0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L_MASK                                     0x007FFFFFL
+//HUBPREQ0_VBLANK_PARAMETERS_4
+#define HUBPREQ0_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C__SHIFT                                   0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C_MASK                                     0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_0
+#define HUBPREQ0_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP__SHIFT                                                  0x0
+#define HUBPREQ0_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP__SHIFT                                                 0x8
+#define HUBPREQ0_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP_MASK                                                    0x0000007FL
+#define HUBPREQ0_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP_MASK                                                   0x00003F00L
+//HUBPREQ0_FLIP_PARAMETERS_1
+#define HUBPREQ0_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L__SHIFT                                        0x0
+#define HUBPREQ0_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L_MASK                                          0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_2
+#define HUBPREQ0_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L__SHIFT                                       0x0
+#define HUBPREQ0_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L_MASK                                         0x007FFFFFL
+//HUBPREQ0_NOM_PARAMETERS_0
+#define HUBPREQ0_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L__SHIFT                                             0x0
+#define HUBPREQ0_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L_MASK                                               0x0001FFFFL
+//HUBPREQ0_NOM_PARAMETERS_1
+#define HUBPREQ0_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L__SHIFT                                          0x0
+#define HUBPREQ0_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L_MASK                                            0x007FFFFFL
+//HUBPREQ0_NOM_PARAMETERS_2
+#define HUBPREQ0_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C__SHIFT                                             0x0
+#define HUBPREQ0_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C_MASK                                               0x0001FFFFL
+//HUBPREQ0_NOM_PARAMETERS_3
+#define HUBPREQ0_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C__SHIFT                                          0x0
+#define HUBPREQ0_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C_MASK                                            0x007FFFFFL
+//HUBPREQ0_NOM_PARAMETERS_4
+#define HUBPREQ0_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L__SHIFT                                            0x0
+#define HUBPREQ0_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L_MASK                                              0x0001FFFFL
+//HUBPREQ0_NOM_PARAMETERS_5
+#define HUBPREQ0_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L__SHIFT                                         0x0
+#define HUBPREQ0_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L_MASK                                           0x007FFFFFL
+//HUBPREQ0_NOM_PARAMETERS_6
+#define HUBPREQ0_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C__SHIFT                                            0x0
+#define HUBPREQ0_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C_MASK                                              0x0001FFFFL
+//HUBPREQ0_NOM_PARAMETERS_7
+#define HUBPREQ0_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C__SHIFT                                         0x0
+#define HUBPREQ0_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C_MASK                                           0x007FFFFFL
+//HUBPREQ0_PER_LINE_DELIVERY_PRE
+#define HUBPREQ0_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L__SHIFT                                 0x0
+#define HUBPREQ0_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C__SHIFT                                 0x10
+#define HUBPREQ0_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L_MASK                                   0x00001FFFL
+#define HUBPREQ0_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C_MASK                                   0x1FFF0000L
+//HUBPREQ0_PER_LINE_DELIVERY
+#define HUBPREQ0_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L__SHIFT                                         0x0
+#define HUBPREQ0_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C__SHIFT                                         0x10
+#define HUBPREQ0_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L_MASK                                           0x00001FFFL
+#define HUBPREQ0_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C_MASK                                           0x1FFF0000L
+//HUBPREQ0_CURSOR_SETTINGS
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET__SHIFT                                                 0x0
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST__SHIFT                                             0x8
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET__SHIFT                                                 0x10
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST__SHIFT                                             0x18
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET_MASK                                                   0x000000FFL
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST_MASK                                               0x00000300L
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET_MASK                                                   0x00FF0000L
+#define HUBPREQ0_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST_MASK                                               0x03000000L
+//HUBPREQ0_REF_FREQ_TO_PIX_FREQ
+#define HUBPREQ0_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ__SHIFT                                            0x0
+#define HUBPREQ0_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ_MASK                                              0x001FFFFFL
+//HUBPREQ0_DST_Y_DELTA_DRQ_LIMIT
+#define HUBPREQ0_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT__SHIFT                                          0x0
+#define HUBPREQ0_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT_MASK                                            0x00007FFFL
+//HUBPREQ0_HUBPREQ_MEM_PWR_CTRL
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE__SHIFT                                          0x0
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS__SHIFT                                            0x2
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE__SHIFT                                          0x4
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS__SHIFT                                            0x6
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE__SHIFT                                          0x8
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS__SHIFT                                            0xa
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE__SHIFT                                           0xc
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS__SHIFT                                             0xe
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE__SHIFT                                          0xf
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS__SHIFT                                            0x11
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE_MASK                                            0x00000003L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS_MASK                                              0x00000004L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE_MASK                                            0x00000030L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS_MASK                                              0x00000040L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE_MASK                                            0x00000300L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS_MASK                                              0x00000400L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE_MASK                                             0x00003000L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS_MASK                                               0x00004000L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE_MASK                                            0x00018000L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS_MASK                                              0x00020000L
+//HUBPREQ0_HUBPREQ_MEM_PWR_STATUS
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE__SHIFT                                        0x0
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE__SHIFT                                        0x2
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE__SHIFT                                        0x4
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE__SHIFT                                         0x6
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE__SHIFT                                        0x8
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE_MASK                                          0x00000003L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE_MASK                                          0x0000000CL
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE_MASK                                          0x00000030L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE_MASK                                           0x000000C0L
+#define HUBPREQ0_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE_MASK                                          0x00000300L
+//HUBPREQ0_VBLANK_PARAMETERS_5
+#define HUBPREQ0_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK__SHIFT                                       0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK_MASK                                         0x007FFFFFL
+//HUBPREQ0_VBLANK_PARAMETERS_6
+#define HUBPREQ0_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK__SHIFT                                         0x0
+#define HUBPREQ0_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK_MASK                                           0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_3
+#define HUBPREQ0_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP__SHIFT                                           0x0
+#define HUBPREQ0_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP_MASK                                             0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_4
+#define HUBPREQ0_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP__SHIFT                                             0x0
+#define HUBPREQ0_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP_MASK                                               0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_5
+#define HUBPREQ0_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C__SHIFT                                        0x0
+#define HUBPREQ0_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C_MASK                                          0x007FFFFFL
+//HUBPREQ0_FLIP_PARAMETERS_6
+#define HUBPREQ0_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C__SHIFT                                       0x0
+#define HUBPREQ0_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C_MASK                                         0x007FFFFFL
+//HUBPREQ0_UCLK_PSTATE_FORCE
+#define HUBPREQ0_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN__SHIFT                                          0x0
+#define HUBPREQ0_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE__SHIFT                                       0x1
+#define HUBPREQ0_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN__SHIFT                                        0x2
+#define HUBPREQ0_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE__SHIFT                                     0x3
+#define HUBPREQ0_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN_MASK                                            0x00000001L
+#define HUBPREQ0_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE_MASK                                         0x00000002L
+#define HUBPREQ0_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN_MASK                                          0x00000004L
+#define HUBPREQ0_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE_MASK                                       0x00000008L
+//HUBPREQ0_HUBPREQ_STATUS_REG0
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0__SHIFT                                         0x0
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1__SHIFT                                         0x8
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT__SHIFT                                                 0x10
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0_MASK                                           0x0000001FL
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1_MASK                                           0x00001F00L
+#define HUBPREQ0_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT_MASK                                                   0x7FFF0000L
+//HUBPREQ0_HUBPREQ_STATUS_REG1
+#define HUBPREQ0_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0__SHIFT                                       0x0
+#define HUBPREQ0_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1__SHIFT                                       0x10
+#define HUBPREQ0_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0_MASK                                         0x00003FFFL
+#define HUBPREQ0_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1_MASK                                         0x3FFF0000L
+//HUBPREQ0_HUBPREQ_STATUS_REG2
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0__SHIFT                                      0x0
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0__SHIFT                                     0x1
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0__SHIFT                               0x2
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0__SHIFT                               0x3
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0__SHIFT                                   0x4
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0__SHIFT                                             0x5
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1__SHIFT                                      0x8
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1__SHIFT                                     0x9
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1__SHIFT                               0xa
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1__SHIFT                               0xb
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1__SHIFT                                   0xc
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1__SHIFT                                             0xd
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR__SHIFT                                     0x10
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR__SHIFT                                    0x11
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR__SHIFT                              0x12
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR__SHIFT                              0x13
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR__SHIFT                                  0x14
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR__SHIFT                                            0x15
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_VBLANK__SHIFT                                                    0x1a
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN__SHIFT                                                   0x1b
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY__SHIFT                                          0x1c
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH__SHIFT                                             0x1d
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0__SHIFT                                            0x1e
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1__SHIFT                                            0x1f
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0_MASK                                        0x00000001L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0_MASK                                       0x00000002L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0_MASK                                 0x00000004L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0_MASK                                 0x00000008L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0_MASK                                     0x00000010L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0_MASK                                               0x00000020L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1_MASK                                        0x00000100L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1_MASK                                       0x00000200L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1_MASK                                 0x00000400L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1_MASK                                 0x00000800L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1_MASK                                     0x00001000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1_MASK                                               0x00002000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR_MASK                                       0x00010000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR_MASK                                      0x00020000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR_MASK                                0x00040000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR_MASK                                0x00080000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR_MASK                                    0x00100000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR_MASK                                              0x00200000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_VBLANK_MASK                                                      0x04000000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN_MASK                                                     0x08000000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY_MASK                                            0x10000000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH_MASK                                               0x20000000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0_MASK                                              0x40000000L
+#define HUBPREQ0_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1_MASK                                              0x80000000L
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpret_dispdec
+//HUBPRET0_HUBPRET_CONTROL
+#define HUBPRET0_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS__SHIFT                                          0x4
+#define HUBPRET0_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE__SHIFT                                            0xf
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA__SHIFT                                                   0x10
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G__SHIFT                                                     0x12
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B__SHIFT                                                    0x14
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R__SHIFT                                                    0x16
+#define HUBPRET0_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE__SHIFT                                                0x18
+#define HUBPRET0_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS_MASK                                            0x00007FF0L
+#define HUBPRET0_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE_MASK                                              0x00008000L
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA_MASK                                                     0x00030000L
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G_MASK                                                       0x000C0000L
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B_MASK                                                      0x00300000L
+#define HUBPRET0_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R_MASK                                                      0x00C00000L
+#define HUBPRET0_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE_MASK                                                  0xFF000000L
+//HUBPRET0_HUBPRET_MEM_PWR_CTRL
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE__SHIFT                                             0x8
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS__SHIFT                                               0xa
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE__SHIFT                                            0x10
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS__SHIFT                                              0x12
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE__SHIFT                                          0x14
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE_MASK                                               0x00000300L
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS_MASK                                                 0x00000400L
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE_MASK                                              0x00030000L
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS_MASK                                                0x00040000L
+#define HUBPRET0_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE_MASK                                            0x00300000L
+//HUBPRET0_HUBPRET_MEM_PWR_STATUS
+#define HUBPRET0_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE__SHIFT                                           0x2
+#define HUBPRET0_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE__SHIFT                                          0x4
+#define HUBPRET0_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE_MASK                                             0x0000000CL
+#define HUBPRET0_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE_MASK                                            0x00000030L
+//HUBPRET0_HUBPRET_READ_LINE_CTRL0
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE__SHIFT                         0x0
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM__SHIFT                                0x10
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE_MASK                           0x0000FFFFL
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM_MASK                                  0x3FFF0000L
+//HUBPRET0_HUBPRET_READ_LINE_CTRL1
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED__SHIFT                    0x0
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE__SHIFT                                0x10
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED_MASK                      0x00003FFFL
+#define HUBPRET0_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE_MASK                                  0xFFFF0000L
+//HUBPRET0_HUBPRET_READ_LINE0
+#define HUBPRET0_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START__SHIFT                                             0x0
+#define HUBPRET0_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END__SHIFT                                               0x10
+#define HUBPRET0_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START_MASK                                               0x00003FFFL
+#define HUBPRET0_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END_MASK                                                 0x3FFF0000L
+//HUBPRET0_HUBPRET_READ_LINE1
+#define HUBPRET0_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START__SHIFT                                             0x0
+#define HUBPRET0_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END__SHIFT                                               0x10
+#define HUBPRET0_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START_MASK                                               0x00003FFFL
+#define HUBPRET0_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END_MASK                                                 0x3FFF0000L
+//HUBPRET0_HUBPRET_INTERRUPT
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK__SHIFT                                               0x0
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK__SHIFT                                           0x1
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK__SHIFT                                           0x2
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE__SHIFT                                               0x4
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE__SHIFT                                           0x5
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE__SHIFT                                           0x6
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR__SHIFT                                              0x8
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR__SHIFT                                          0x9
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR__SHIFT                                          0xa
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS__SHIFT                                                 0xc
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS__SHIFT                                             0xd
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS__SHIFT                                             0xe
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS__SHIFT                                             0x10
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS__SHIFT                                         0x11
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS__SHIFT                                         0x12
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK_MASK                                                 0x00000001L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK_MASK                                             0x00000002L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK_MASK                                             0x00000004L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE_MASK                                                 0x00000010L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE_MASK                                             0x00000020L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE_MASK                                             0x00000040L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR_MASK                                                0x00000100L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR_MASK                                            0x00000200L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR_MASK                                            0x00000400L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS_MASK                                                   0x00001000L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS_MASK                                               0x00002000L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS_MASK                                               0x00004000L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS_MASK                                               0x00010000L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS_MASK                                           0x00020000L
+#define HUBPRET0_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS_MASK                                           0x00040000L
+//HUBPRET0_HUBPRET_READ_LINE_VALUE
+#define HUBPRET0_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE__SHIFT                                               0x0
+#define HUBPRET0_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT__SHIFT                                      0x10
+#define HUBPRET0_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_MASK                                                 0x00003FFFL
+#define HUBPRET0_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT_MASK                                        0x3FFF0000L
+//HUBPRET0_HUBPRET_READ_LINE_STATUS
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK__SHIFT                                            0x0
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE__SHIFT                                      0x4
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE__SHIFT                                     0x5
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE__SHIFT                                      0x8
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE__SHIFT                                     0xa
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK_MASK                                              0x00000001L
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE_MASK                                        0x00000010L
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE_MASK                                       0x00000020L
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE_MASK                                        0x00000100L
+#define HUBPRET0_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE_MASK                                       0x00000400L
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_cursor0_dispdec
+//CURSOR0_0_CURSOR_CONTROL
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_ENABLE__SHIFT                                                        0x0
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_REQ_MODE__SHIFT                                                      0x2
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT                                                    0x4
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_MODE__SHIFT                                                          0x8
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_TMZ__SHIFT                                                           0xc
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PITCH__SHIFT                                                         0x10
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS__SHIFT                     0x14
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK__SHIFT                                               0x18
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN__SHIFT                                    0x1e
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL__SHIFT                                   0x1f
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_ENABLE_MASK                                                          0x00000001L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_REQ_MODE_MASK                                                        0x00000004L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_2X_MAGNIFY_MASK                                                      0x00000010L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_MODE_MASK                                                            0x00000700L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_TMZ_MASK                                                             0x00001000L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PITCH_MASK                                                           0x00030000L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS_MASK                       0x00100000L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK_MASK                                                 0x1F000000L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN_MASK                                      0x40000000L
+#define CURSOR0_0_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL_MASK                                     0x80000000L
+//CURSOR0_0_CURSOR_SURFACE_ADDRESS
+#define CURSOR0_0_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS__SHIFT                                       0x0
+#define CURSOR0_0_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS_MASK                                         0xFFFFFFFFL
+//CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH
+#define CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH__SHIFT                             0x0
+#define CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH_MASK                               0x0000FFFFL
+//CURSOR0_0_CURSOR_SIZE
+#define CURSOR0_0_CURSOR_SIZE__CURSOR_HEIGHT__SHIFT                                                           0x0
+#define CURSOR0_0_CURSOR_SIZE__CURSOR_WIDTH__SHIFT                                                            0x10
+#define CURSOR0_0_CURSOR_SIZE__CURSOR_HEIGHT_MASK                                                             0x000001FFL
+#define CURSOR0_0_CURSOR_SIZE__CURSOR_WIDTH_MASK                                                              0x01FF0000L
+//CURSOR0_0_CURSOR_POSITION
+#define CURSOR0_0_CURSOR_POSITION__CURSOR_Y_POSITION__SHIFT                                                   0x0
+#define CURSOR0_0_CURSOR_POSITION__CURSOR_X_POSITION__SHIFT                                                   0x10
+#define CURSOR0_0_CURSOR_POSITION__CURSOR_Y_POSITION_MASK                                                     0x00003FFFL
+#define CURSOR0_0_CURSOR_POSITION__CURSOR_X_POSITION_MASK                                                     0x3FFF0000L
+//CURSOR0_0_CURSOR_HOT_SPOT
+#define CURSOR0_0_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y__SHIFT                                                   0x0
+#define CURSOR0_0_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X__SHIFT                                                   0x10
+#define CURSOR0_0_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y_MASK                                                     0x000000FFL
+#define CURSOR0_0_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X_MASK                                                     0x00FF0000L
+//CURSOR0_0_CURSOR_STEREO_CONTROL
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN__SHIFT                                              0x0
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET__SHIFT                                         0x4
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET__SHIFT                                       0x12
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN_MASK                                                0x00000001L
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET_MASK                                           0x0003FFF0L
+#define CURSOR0_0_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET_MASK                                         0xFFFC0000L
+//CURSOR0_0_CURSOR_DST_OFFSET
+#define CURSOR0_0_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET__SHIFT                                               0x0
+#define CURSOR0_0_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET_MASK                                                 0x00001FFFL
+//CURSOR0_0_CURSOR_MEM_PWR_CTRL
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE__SHIFT                                              0x0
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS__SHIFT                                                0x2
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE__SHIFT                                            0x4
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE_MASK                                                0x00000003L
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS_MASK                                                  0x00000004L
+#define CURSOR0_0_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE_MASK                                              0x00000030L
+//CURSOR0_0_CURSOR_MEM_PWR_STATUS
+#define CURSOR0_0_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE__SHIFT                                            0x0
+#define CURSOR0_0_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE_MASK                                              0x00000003L
+//CURSOR0_0_DMDATA_ADDRESS_HIGH
+#define CURSOR0_0_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH__SHIFT                                             0x0
+#define CURSOR0_0_DMDATA_ADDRESS_HIGH__DMDATA_TMZ__SHIFT                                                      0x1e
+#define CURSOR0_0_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH_MASK                                               0x0000FFFFL
+#define CURSOR0_0_DMDATA_ADDRESS_HIGH__DMDATA_TMZ_MASK                                                        0x40000000L
+//CURSOR0_0_DMDATA_ADDRESS_LOW
+#define CURSOR0_0_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW__SHIFT                                               0x0
+#define CURSOR0_0_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW_MASK                                                 0xFFFFFFFFL
+//CURSOR0_0_DMDATA_CNTL
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_UPDATED__SHIFT                                                          0x0
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_REPEAT__SHIFT                                                           0x1
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_MODE__SHIFT                                                             0x2
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_SIZE__SHIFT                                                             0x10
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_UPDATED_MASK                                                            0x00000001L
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_REPEAT_MASK                                                             0x00000002L
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_MODE_MASK                                                               0x00000004L
+#define CURSOR0_0_DMDATA_CNTL__DMDATA_SIZE_MASK                                                               0x0FFF0000L
+//CURSOR0_0_DMDATA_QOS_CNTL
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_QOS_MODE__SHIFT                                                     0x0
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL__SHIFT                                                    0x4
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_DL_DELTA__SHIFT                                                     0x10
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_QOS_MODE_MASK                                                       0x00000001L
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL_MASK                                                      0x000000F0L
+#define CURSOR0_0_DMDATA_QOS_CNTL__DMDATA_DL_DELTA_MASK                                                       0xFFFF0000L
+//CURSOR0_0_DMDATA_STATUS
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_DONE__SHIFT                                                           0x0
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_UNDERFLOW__SHIFT                                                      0x2
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR__SHIFT                                                0x4
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_DONE_MASK                                                             0x00000001L
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_UNDERFLOW_MASK                                                        0x00000004L
+#define CURSOR0_0_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR_MASK                                                  0x00000010L
+//CURSOR0_0_DMDATA_SW_CNTL
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_UPDATED__SHIFT                                                    0x0
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_REPEAT__SHIFT                                                     0x1
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_SIZE__SHIFT                                                       0x10
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_UPDATED_MASK                                                      0x00000001L
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_REPEAT_MASK                                                       0x00000002L
+#define CURSOR0_0_DMDATA_SW_CNTL__DMDATA_SW_SIZE_MASK                                                         0x0FFF0000L
+//CURSOR0_0_DMDATA_SW_DATA
+#define CURSOR0_0_DMDATA_SW_DATA__DMDATA_SW_DATA__SHIFT                                                       0x0
+#define CURSOR0_0_DMDATA_SW_DATA__DMDATA_SW_DATA_MASK                                                         0xFFFFFFFFL
+//CURSOR0_0_HUBP_3DLUT_CONTROL
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE__SHIFT                                                0x0
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE__SHIFT                                       0x1
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH__SHIFT                                                 0x2
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ__SHIFT                                                   0x12
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B__SHIFT                                        0x16
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G__SHIFT                                        0x18
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R__SHIFT                                        0x1a
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS__SHIFT                                         0x1d
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH__SHIFT                                             0x1e
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE__SHIFT                                                  0x1f
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE_MASK                                                  0x00000001L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE_MASK                                         0x00000002L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH_MASK                                                   0x0003FFFCL
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ_MASK                                                     0x003C0000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B_MASK                                          0x00C00000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G_MASK                                          0x03000000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R_MASK                                          0x0C000000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS_MASK                                           0x20000000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH_MASK                                               0x40000000L
+#define CURSOR0_0_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE_MASK                                                    0x80000000L
+//CURSOR0_0_HUBP_3DLUT_ADDRESS_LOW
+#define CURSOR0_0_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW__SHIFT                                       0x0
+#define CURSOR0_0_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW_MASK                                         0xFFFFFFFFL
+//CURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH
+#define CURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH__SHIFT                                     0x0
+#define CURSOR0_0_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH_MASK                                       0x0000FFFFL
+//CURSOR0_0_HUBP_3DLUT_DLG_PARAM
+#define CURSOR0_0_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP__SHIFT                                         0x0
+#define CURSOR0_0_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP_MASK                                           0x007FFFFFL
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON6_PERFCOUNTER_CNTL
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON6_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON6_PERFCOUNTER_CNTL2
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON6_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON6_PERFCOUNTER_STATE
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON6_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON6_PERFMON_CNTL
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON6_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON6_PERFMON_CNTL2
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON6_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON6_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON6_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON6_PERFMON_CVALUE_LOW
+#define DC_PERFMON6_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON6_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON6_PERFMON_HI
+#define DC_PERFMON6_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON6_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON6_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON6_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON6_PERFMON_LOW
+#define DC_PERFMON6_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON6_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dispdec
+//HUBP1_DCSURF_SURFACE_CONFIG
+#define HUBP1_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT__SHIFT                                              0x0
+#define HUBP1_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE__SHIFT                                                    0x8
+#define HUBP1_DCSURF_SURFACE_CONFIG__H_MIRROR_EN__SHIFT                                                       0xa
+#define HUBP1_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN__SHIFT                                                    0xb
+#define HUBP1_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT_MASK                                                0x0000007FL
+#define HUBP1_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE_MASK                                                      0x00000300L
+#define HUBP1_DCSURF_SURFACE_CONFIG__H_MIRROR_EN_MASK                                                         0x00000400L
+#define HUBP1_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN_MASK                                                      0x00000800L
+//HUBP1_DCSURF_ADDR_CONFIG
+#define HUBP1_DCSURF_ADDR_CONFIG__NUM_PIPES__SHIFT                                                            0x0
+#define HUBP1_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE__SHIFT                                                      0x6
+#define HUBP1_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                 0xc
+#define HUBP1_DCSURF_ADDR_CONFIG__NUM_PKRS__SHIFT                                                             0x10
+#define HUBP1_DCSURF_ADDR_CONFIG__NUM_PIPES_MASK                                                              0x00000007L
+#define HUBP1_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE_MASK                                                        0x000000C0L
+#define HUBP1_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                   0x00003000L
+#define HUBP1_DCSURF_ADDR_CONFIG__NUM_PKRS_MASK                                                               0x00070000L
+//HUBP1_DCSURF_TILING_CONFIG
+#define HUBP1_DCSURF_TILING_CONFIG__SW_MODE__SHIFT                                                            0x0
+#define HUBP1_DCSURF_TILING_CONFIG__DIM_TYPE__SHIFT                                                           0x7
+#define HUBP1_DCSURF_TILING_CONFIG__META_LINEAR__SHIFT                                                        0x9
+#define HUBP1_DCSURF_TILING_CONFIG__PIPE_ALIGNED__SHIFT                                                       0xb
+#define HUBP1_DCSURF_TILING_CONFIG__SW_MODE_MASK                                                              0x0000001FL
+#define HUBP1_DCSURF_TILING_CONFIG__DIM_TYPE_MASK                                                             0x00000180L
+#define HUBP1_DCSURF_TILING_CONFIG__META_LINEAR_MASK                                                          0x00000200L
+#define HUBP1_DCSURF_TILING_CONFIG__PIPE_ALIGNED_MASK                                                         0x00000800L
+//HUBP1_DCSURF_PRI_VIEWPORT_START
+#define HUBP1_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP1_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP1_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP1_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP1_DCSURF_PRI_VIEWPORT_START_C
+#define HUBP1_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP1_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP1_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP1_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP1_DCSURF_SEC_VIEWPORT_START
+#define HUBP1_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP1_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP1_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP1_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP1_DCSURF_SEC_VIEWPORT_START_C
+#define HUBP1_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP1_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP1_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP1_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP1_DCHUBP_REQ_SIZE_CONFIG
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT__SHIFT                                                     0x0
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR__SHIFT                                            0x4
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE__SHIFT                                                       0x8
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE__SHIFT                                                   0xb
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE__SHIFT                                                  0x10
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE__SHIFT                                              0x12
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE__SHIFT                                                  0x14
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE__SHIFT                                                    0x18
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT_MASK                                                       0x00000007L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR_MASK                                              0x00000070L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE_MASK                                                         0x00000700L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE_MASK                                                     0x00001800L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE_MASK                                                    0x00030000L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE_MASK                                                0x000C0000L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE_MASK                                                    0x00700000L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE_MASK                                                      0x07000000L
+//HUBP1_DCHUBP_REQ_SIZE_CONFIG_C
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C__SHIFT                                                 0x0
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C__SHIFT                                        0x4
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C__SHIFT                                                   0x8
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C__SHIFT                                               0xb
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C__SHIFT                                              0x10
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C__SHIFT                                          0x12
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C__SHIFT                                              0x14
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C_MASK                                                   0x00000007L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C_MASK                                          0x00000070L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C_MASK                                                     0x00000700L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C_MASK                                                 0x00001800L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C_MASK                                                0x00030000L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C_MASK                                            0x000C0000L
+#define HUBP1_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C_MASK                                                0x00700000L
+//HUBP1_DCHUBP_CNTL
+#define HUBP1_DCHUBP_CNTL__HUBP_BLANK_EN__SHIFT                                                               0x0
+#define HUBP1_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ__SHIFT                                                     0x1
+#define HUBP1_DCHUBP_CNTL__HUBP_SOFT_RESET__SHIFT                                                             0x2
+#define HUBP1_DCHUBP_CNTL__HUBP_IN_BLANK__SHIFT                                                               0x3
+#define HUBP1_DCHUBP_CNTL__HUBP_VTG_SEL__SHIFT                                                                0x4
+#define HUBP1_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC__SHIFT                                               0x8
+#define HUBP1_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM__SHIFT                                            0x9
+#define HUBP1_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE__SHIFT                                                     0xa
+#define HUBP1_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS__SHIFT                                                   0xb
+#define HUBP1_DCHUBP_CNTL__HUBP_TTU_DISABLE__SHIFT                                                            0xc
+#define HUBP1_DCHUBP_CNTL__HUBP_TTU_MODE__SHIFT                                                               0xd
+#define HUBP1_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ__SHIFT                                                 0x10
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS__SHIFT                                                         0x14
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD__SHIFT                                                      0x18
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR__SHIFT                                                   0x1a
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN__SHIFT                                                   0x1b
+#define HUBP1_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS__SHIFT                                                       0x1c
+#define HUBP1_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR__SHIFT                                                        0x1f
+#define HUBP1_DCHUBP_CNTL__HUBP_BLANK_EN_MASK                                                                 0x00000001L
+#define HUBP1_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ_MASK                                                       0x00000002L
+#define HUBP1_DCHUBP_CNTL__HUBP_SOFT_RESET_MASK                                                               0x00000004L
+#define HUBP1_DCHUBP_CNTL__HUBP_IN_BLANK_MASK                                                                 0x00000008L
+#define HUBP1_DCHUBP_CNTL__HUBP_VTG_SEL_MASK                                                                  0x000000F0L
+#define HUBP1_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC_MASK                                                 0x00000100L
+#define HUBP1_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM_MASK                                              0x00000200L
+#define HUBP1_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE_MASK                                                       0x00000400L
+#define HUBP1_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS_MASK                                                     0x00000800L
+#define HUBP1_DCHUBP_CNTL__HUBP_TTU_DISABLE_MASK                                                              0x00001000L
+#define HUBP1_DCHUBP_CNTL__HUBP_TTU_MODE_MASK                                                                 0x0000E000L
+#define HUBP1_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ_MASK                                                   0x000F0000L
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_MASK                                                           0x00F00000L
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD_MASK                                                        0x03000000L
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR_MASK                                                     0x04000000L
+#define HUBP1_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN_MASK                                                     0x08000000L
+#define HUBP1_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS_MASK                                                         0x70000000L
+#define HUBP1_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR_MASK                                                          0x80000000L
+//HUBP1_HUBP_CLK_CNTL
+#define HUBP1_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE__SHIFT                                                         0x0
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS__SHIFT                                                   0x5
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS__SHIFT                                                    0x8
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS__SHIFT                                                    0xc
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS__SHIFT                                                    0x10
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON__SHIFT                                                   0x14
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON__SHIFT                                                    0x15
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON__SHIFT                                                    0x16
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON__SHIFT                                                    0x17
+#define HUBP1_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON__SHIFT                                                   0x19
+#define HUBP1_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define HUBP1_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE_MASK                                                           0x00000001L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS_MASK                                                     0x00000020L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS_MASK                                                      0x00000100L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS_MASK                                                      0x00001000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS_MASK                                                      0x00010000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON_MASK                                                     0x00100000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON_MASK                                                      0x00200000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON_MASK                                                      0x00400000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON_MASK                                                      0x00800000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON_MASK                                                     0x02000000L
+#define HUBP1_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL_MASK                                                           0xF0000000L
+//HUBP1_DCHUBP_VMPG_CONFIG
+#define HUBP1_DCHUBP_VMPG_CONFIG__VMPG_SIZE__SHIFT                                                            0x0
+#define HUBP1_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE__SHIFT                                                      0x1
+#define HUBP1_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE__SHIFT                                                   0x2
+#define HUBP1_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME__SHIFT                                              0x7
+#define HUBP1_DCHUBP_VMPG_CONFIG__VMPG_SIZE_MASK                                                              0x00000001L
+#define HUBP1_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE_MASK                                                        0x00000002L
+#define HUBP1_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE_MASK                                                     0x0000007CL
+#define HUBP1_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME_MASK                                                0x00000080L
+//HUBP1_DCHUBP_MALL_CONFIG
+#define HUBP1_DCHUBP_MALL_CONFIG__USE_MALL_SEL__SHIFT                                                         0x0
+#define HUBP1_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR__SHIFT                                                  0x2
+#define HUBP1_DCHUBP_MALL_CONFIG__USE_MALL_SEL_MASK                                                           0x00000003L
+#define HUBP1_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR_MASK                                                    0x00000004L
+//HUBP1_DCHUBP_MALL_SUB_VP
+#define HUBP1_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE__SHIFT                                               0x0
+#define HUBP1_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0__SHIFT                                                 0x1
+#define HUBP1_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1__SHIFT                                                 0xf
+#define HUBP1_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE_MASK                                                 0x00000001L
+#define HUBP1_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0_MASK                                                   0x00007FFEL
+#define HUBP1_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1_MASK                                                   0x1FFF8000L
+//HUBP1_HUBPREQ_DEBUG_DB
+#define HUBP1_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG__SHIFT                                                          0x0
+#define HUBP1_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG_MASK                                                            0xFFFFFFFFL
+//HUBP1_HUBPREQ_DEBUG
+#define HUBP1_HUBPREQ_DEBUG__HUBPREQ_DEBUG__SHIFT                                                             0x0
+#define HUBP1_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS__SHIFT                                 0x1f
+#define HUBP1_HUBPREQ_DEBUG__HUBPREQ_DEBUG_MASK                                                               0x7FFFFFFFL
+#define HUBP1_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS_MASK                                   0x80000000L
+//HUBP1_HUBP_DEBUG_CTRL
+#define HUBP1_HUBP_DEBUG_CTRL__HUBP_DBG_EN__SHIFT                                                             0x0
+#define HUBP1_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS__SHIFT                                              0x4
+#define HUBP1_HUBP_DEBUG_CTRL__HUBP_DBG_EN_MASK                                                               0x00000001L
+#define HUBP1_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS_MASK                                                0x00000010L
+//HUBP1_HUBP_DEBUG_MUX_DCFCLK
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK__SHIFT                                         0x0
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK__SHIFT                                         0x8
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK__SHIFT                                         0x10
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK__SHIFT                                         0x18
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK_MASK                                           0x000000FFL
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK_MASK                                           0x0000FF00L
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK_MASK                                           0x00FF0000L
+#define HUBP1_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK_MASK                                           0xFF000000L
+//HUBP1_HUBP_DEBUG_MUX_DPPCLK
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK__SHIFT                                         0x0
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK__SHIFT                                         0x4
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK__SHIFT                                        0x8
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK_MASK                                           0x00000007L
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK_MASK                                           0x00000070L
+#define HUBP1_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK_MASK                                          0x00000700L
+//HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK__SHIFT                                 0x0
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK__SHIFT                          0x4
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK__SHIFT                              0xc
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK__SHIFT                               0x14
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK__SHIFT                               0x1c
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK_MASK                                   0x00000001L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK_MASK                            0x00000FF0L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK_MASK                                0x0001F000L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK_MASK                                 0x01F00000L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK_MASK                                 0x30000000L
+//HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK__SHIFT                                 0x0
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK__SHIFT                            0x1
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK__SHIFT                          0x4
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK__SHIFT                              0xc
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK__SHIFT                               0x14
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK_MASK                                   0x00000001L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK_MASK                              0x00000002L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK_MASK                            0x00000FF0L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK_MASK                                0x0001F000L
+#define HUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK_MASK                                 0x01F00000L
+//HUBP1_HUBP_MALL_STATUS
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN__SHIFT                                         0x0
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE__SHIFT                                         0x1
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQUEST__SHIFT                                                       0x2
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_RESPONSE__SHIFT                                                      0x3
+#define HUBP1_HUBP_MALL_STATUS__MALL_IN_USE__SHIFT                                                            0x4
+#define HUBP1_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE__SHIFT                                                 0x5
+#define HUBP1_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE__SHIFT                                                   0x6
+#define HUBP1_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE__SHIFT                                                  0x7
+#define HUBP1_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE__SHIFT                                                  0x8
+#define HUBP1_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH__SHIFT                                                  0x9
+#define HUBP1_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME__SHIFT                                                    0xa
+#define HUBP1_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME__SHIFT                                                    0xb
+#define HUBP1_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL__SHIFT                                                     0xc
+#define HUBP1_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL__SHIFT                                                     0xd
+#define HUBP1_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL__SHIFT                                                     0xe
+#define HUBP1_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME__SHIFT                                                  0xf
+#define HUBP1_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS__SHIFT                                   0x10
+#define HUBP1_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING__SHIFT                                            0x11
+#define HUBP1_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING__SHIFT                                                   0x12
+#define HUBP1_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO__SHIFT                                                      0x13
+#define HUBP1_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x14
+#define HUBP1_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x15
+#define HUBP1_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING__SHIFT                                                   0x16
+#define HUBP1_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP__SHIFT                                              0x17
+#define HUBP1_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING__SHIFT                                                   0x18
+#define HUBP1_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING__SHIFT                                                  0x19
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN_MASK                                           0x00000001L
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE_MASK                                           0x00000002L
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_REQUEST_MASK                                                         0x00000004L
+#define HUBP1_HUBP_MALL_STATUS__MALL_USE_RESPONSE_MASK                                                        0x00000008L
+#define HUBP1_HUBP_MALL_STATUS__MALL_IN_USE_MASK                                                              0x00000010L
+#define HUBP1_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE_MASK                                                   0x00000020L
+#define HUBP1_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE_MASK                                                     0x00000040L
+#define HUBP1_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE_MASK                                                    0x00000080L
+#define HUBP1_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE_MASK                                                    0x00000100L
+#define HUBP1_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH_MASK                                                    0x00000200L
+#define HUBP1_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME_MASK                                                      0x00000400L
+#define HUBP1_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME_MASK                                                      0x00000800L
+#define HUBP1_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL_MASK                                                       0x00001000L
+#define HUBP1_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_MASK                                                       0x00002000L
+#define HUBP1_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL_MASK                                                       0x00004000L
+#define HUBP1_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME_MASK                                                    0x00008000L
+#define HUBP1_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS_MASK                                     0x00010000L
+#define HUBP1_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING_MASK                                              0x00020000L
+#define HUBP1_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING_MASK                                                     0x00040000L
+#define HUBP1_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO_MASK                                                        0x00080000L
+#define HUBP1_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP_MASK                                              0x00100000L
+#define HUBP1_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP_MASK                                              0x00200000L
+#define HUBP1_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING_MASK                                                     0x00400000L
+#define HUBP1_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP_MASK                                                0x00800000L
+#define HUBP1_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING_MASK                                                     0x01000000L
+#define HUBP1_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING_MASK                                                    0x02000000L
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpreq_dispdec
+//HUBPREQ1_DCSURF_SURFACE_PITCH
+#define HUBPREQ1_DCSURF_SURFACE_PITCH__PITCH__SHIFT                                                           0x0
+#define HUBPREQ1_DCSURF_SURFACE_PITCH__META_PITCH__SHIFT                                                      0x10
+#define HUBPREQ1_DCSURF_SURFACE_PITCH__PITCH_MASK                                                             0x00003FFFL
+#define HUBPREQ1_DCSURF_SURFACE_PITCH__META_PITCH_MASK                                                        0x3FFF0000L
+//HUBPREQ1_DCSURF_SURFACE_PITCH_C
+#define HUBPREQ1_DCSURF_SURFACE_PITCH_C__PITCH_C__SHIFT                                                       0x0
+#define HUBPREQ1_DCSURF_SURFACE_PITCH_C__META_PITCH_C__SHIFT                                                  0x10
+#define HUBPREQ1_DCSURF_SURFACE_PITCH_C__PITCH_C_MASK                                                         0x00003FFFL
+#define HUBPREQ1_DCSURF_SURFACE_PITCH_C__META_PITCH_C_MASK                                                    0x3FFF0000L
+//HUBPREQ1_VMID_SETTINGS_0
+#define HUBPREQ1_VMID_SETTINGS_0__VMID__SHIFT                                                                 0x0
+#define HUBPREQ1_VMID_SETTINGS_0__VMID_MASK                                                                   0x0000000FL
+//HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS__SHIFT                               0x0
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS_MASK                                 0xFFFFFFFFL
+//HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH__SHIFT                     0x0
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH_MASK                       0x0000FFFFL
+//HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C__SHIFT                           0x0
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C_MASK                             0xFFFFFFFFL
+//HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C__SHIFT                 0x0
+#define HUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C_MASK                   0x0000FFFFL
+//HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS__SHIFT                           0x0
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS_MASK                             0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT                 0x0
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH_MASK                   0x0000FFFFL
+//HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C__SHIFT                       0x0
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C_MASK                         0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C__SHIFT             0x0
+#define HUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C_MASK               0x0000FFFFL
+//HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS__SHIFT                     0x0
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS_MASK                       0xFFFFFFFFL
+//HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH__SHIFT           0x0
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH_MASK             0x0000FFFFL
+//HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C__SHIFT                 0x0
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C_MASK                   0xFFFFFFFFL
+//HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT       0x0
+#define HUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C_MASK         0x0000FFFFL
+//HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS__SHIFT                 0x0
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS_MASK                   0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH__SHIFT       0x0
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH_MASK         0x0000FFFFL
+//HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C__SHIFT             0x0
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C_MASK               0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT   0x0
+#define HUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C_MASK     0x0000FFFFL
+//HUBPREQ1_DCSURF_SURFACE_CONTROL
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ__SHIFT                                           0x0
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN__SHIFT                                        0x4
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK__SHIFT                                   0x5
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C__SHIFT                                         0x7
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C__SHIFT                                 0xb
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ__SHIFT                                         0xd
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN__SHIFT                                      0x11
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK__SHIFT                                 0x12
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C__SHIFT                                       0x14
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C__SHIFT                               0x18
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ__SHIFT                                      0x1a
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C__SHIFT                                    0x1b
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ__SHIFT                                    0x1c
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C__SHIFT                                  0x1d
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_MASK                                             0x0000000FL
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN_MASK                                          0x00000010L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_MASK                                     0x00000060L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C_MASK                                           0x00000780L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C_MASK                                   0x00001800L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_MASK                                           0x0001E000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN_MASK                                        0x00020000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_MASK                                   0x000C0000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C_MASK                                         0x00F00000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C_MASK                                 0x03000000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_MASK                                        0x04000000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C_MASK                                      0x08000000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_MASK                                      0x10000000L
+#define HUBPREQ1_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C_MASK                                    0x20000000L
+//HUBPREQ1_DCSURF_FLIP_CONTROL
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK__SHIFT                                              0x0
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE__SHIFT                                                0x1
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM__SHIFT                                    0x4
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING__SHIFT                                             0x8
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS__SHIFT                                0x9
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC__SHIFT                                 0xc
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC__SHIFT                                       0x10
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE__SHIFT                               0x11
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY__SHIFT                              0x12
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY__SHIFT                                       0x14
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK_MASK                                                0x00000001L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE_MASK                                                  0x00000002L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM_MASK                                      0x000000F0L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_MASK                                               0x00000100L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS_MASK                                  0x00000200L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC_MASK                                   0x00003000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC_MASK                                         0x00010000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE_MASK                                 0x00020000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY_MASK                                0x00040000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY_MASK                                         0x3FF00000L
+//HUBPREQ1_DCSURF_FLIP_CONTROL2
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME__SHIFT                                   0x0
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE__SHIFT                                              0x8
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK__SHIFT                                                0x9
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE__SHIFT                                    0xa
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH__SHIFT                                     0xc
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE__SHIFT                                        0x1c
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE__SHIFT                                    0x1f
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME_MASK                                     0x000000FFL
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE_MASK                                                0x00000100L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK_MASK                                                  0x00000200L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE_MASK                                      0x00000400L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH_MASK                                       0x00001000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE_MASK                                          0x10000000L
+#define HUBPREQ1_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE_MASK                                      0x80000000L
+//HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK__SHIFT                                  0x0
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE__SHIFT                                  0x1
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK__SHIFT                             0x2
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE__SHIFT                             0x3
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR__SHIFT                                     0x8
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR__SHIFT                                0x9
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED__SHIFT                                  0x10
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS__SHIFT                                0x11
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED__SHIFT                             0x12
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS__SHIFT                           0x13
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK_MASK                                    0x00000001L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE_MASK                                    0x00000002L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK_MASK                               0x00000004L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE_MASK                               0x00000008L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR_MASK                                       0x00000100L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR_MASK                                  0x00000200L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED_MASK                                    0x00010000L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS_MASK                                  0x00020000L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED_MASK                               0x00040000L
+#define HUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS_MASK                             0x00080000L
+//HUBPREQ1_DCSURF_SURFACE_INUSE
+#define HUBPREQ1_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS__SHIFT                                           0x0
+#define HUBPREQ1_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS_MASK                                             0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH__SHIFT                                 0x0
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID__SHIFT                                         0x1c
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH_MASK                                   0x0000FFFFL
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID_MASK                                           0xF0000000L
+//HUBPREQ1_DCSURF_SURFACE_INUSE_C
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C__SHIFT                                       0x0
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C_MASK                                         0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C__SHIFT                             0x0
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C__SHIFT                                     0x1c
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C_MASK                               0x0000FFFFL
+#define HUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C_MASK                                       0xF0000000L
+//HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS__SHIFT                         0x0
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS_MASK                           0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH__SHIFT               0x0
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID__SHIFT                       0x1c
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_MASK                 0x0000FFFFL
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID_MASK                         0xF0000000L
+//HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C__SHIFT                     0x0
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C_MASK                       0xFFFFFFFFL
+//HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C__SHIFT           0x0
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C__SHIFT                   0x1c
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C_MASK             0x0000FFFFL
+#define HUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C_MASK                     0xF0000000L
+//HUBPREQ1_DCN_EXPANSION_MODE
+#define HUBPREQ1_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE__SHIFT                                                0x0
+#define HUBPREQ1_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE__SHIFT                                                0x2
+#define HUBPREQ1_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE__SHIFT                                                0x4
+#define HUBPREQ1_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE__SHIFT                                                0x6
+#define HUBPREQ1_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE_MASK                                                  0x00000003L
+#define HUBPREQ1_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE_MASK                                                  0x0000000CL
+#define HUBPREQ1_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE_MASK                                                  0x00000030L
+#define HUBPREQ1_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE_MASK                                                  0x000000C0L
+//HUBPREQ1_DCN_TTU_QOS_WM
+#define HUBPREQ1_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM__SHIFT                                                      0x0
+#define HUBPREQ1_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM__SHIFT                                                     0x10
+#define HUBPREQ1_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM_MASK                                                        0x00003FFFL
+#define HUBPREQ1_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM_MASK                                                       0x3FFF0000L
+//HUBPREQ1_DCN_GLOBAL_TTU_CNTL
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK__SHIFT                                                   0x0
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT__SHIFT                                             0x18
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT__SHIFT                                             0x19
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE__SHIFT                                                     0x1b
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP__SHIFT                                                   0x1c
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK_MASK                                                     0x00FFFFFFL
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT_MASK                                               0x01000000L
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT_MASK                                               0x02000000L
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE_MASK                                                       0x08000000L
+#define HUBPREQ1_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP_MASK                                                     0xF0000000L
+//HUBPREQ1_DCN_SURF0_TTU_CNTL0
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ1_DCN_SURF0_TTU_CNTL1
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ1_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ1_DCN_SURF1_TTU_CNTL0
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ1_DCN_SURF1_TTU_CNTL1
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ1_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ1_DCN_CUR0_TTU_CNTL0
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ1_DCN_CUR0_TTU_CNTL1
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ1_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ1_DCN_CUR1_TTU_CNTL0
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ1_DCN_CUR1_TTU_CNTL1
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ1_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ1_DCN_DMDATA_VM_CNTL
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA__SHIFT                                              0x0
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS__SHIFT                                            0x10
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR__SHIFT                                      0x14
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS__SHIFT                                        0x18
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS__SHIFT                                             0x19
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR__SHIFT                                  0x1a
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE__SHIFT                                                    0x1f
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA_MASK                                                0x0000FFFFL
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_MASK                                              0x000F0000L
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR_MASK                                        0x00100000L
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_MASK                                          0x01000000L
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS_MASK                                               0x02000000L
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR_MASK                                    0x04000000L
+#define HUBPREQ1_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE_MASK                                                      0x80000000L
+//HUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR
+#define HUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR__SHIFT                       0x0
+#define HUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR_MASK                         0x3FFFFFFFL
+//HUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define HUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR__SHIFT                     0x0
+#define HUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MASK                       0x3FFFFFFFL
+//HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                  0x0
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                             0x3
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                0x5
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                   0x6
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                    0x00000001L
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                               0x00000018L
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                  0x00000020L
+#define HUBPREQ1_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                     0x00000040L
+//HUBPREQ1_BLANK_OFFSET_0
+#define HUBPREQ1_BLANK_OFFSET_0__REFCYC_H_BLANK_END__SHIFT                                                    0x0
+#define HUBPREQ1_BLANK_OFFSET_0__DLG_V_BLANK_END__SHIFT                                                       0x10
+#define HUBPREQ1_BLANK_OFFSET_0__REFCYC_H_BLANK_END_MASK                                                      0x00001FFFL
+#define HUBPREQ1_BLANK_OFFSET_0__DLG_V_BLANK_END_MASK                                                         0x7FFF0000L
+//HUBPREQ1_BLANK_OFFSET_1
+#define HUBPREQ1_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START__SHIFT                                                  0x0
+#define HUBPREQ1_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START_MASK                                                    0x0003FFFFL
+//HUBPREQ1_DST_DIMENSIONS
+#define HUBPREQ1_DST_DIMENSIONS__REFCYC_PER_HTOTAL__SHIFT                                                     0x0
+#define HUBPREQ1_DST_DIMENSIONS__REFCYC_PER_HTOTAL_MASK                                                       0x001FFFFFL
+//HUBPREQ1_DST_AFTER_SCALER
+#define HUBPREQ1_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER__SHIFT                                               0x0
+#define HUBPREQ1_DST_AFTER_SCALER__DST_Y_AFTER_SCALER__SHIFT                                                  0x10
+#define HUBPREQ1_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER_MASK                                                 0x00001FFFL
+#define HUBPREQ1_DST_AFTER_SCALER__DST_Y_AFTER_SCALER_MASK                                                    0x00070000L
+//HUBPREQ1_PREFETCH_SETTINGS
+#define HUBPREQ1_PREFETCH_SETTINGS__VRATIO_PREFETCH__SHIFT                                                    0x0
+#define HUBPREQ1_PREFETCH_SETTINGS__DST_Y_PREFETCH__SHIFT                                                     0x18
+#define HUBPREQ1_PREFETCH_SETTINGS__VRATIO_PREFETCH_MASK                                                      0x003FFFFFL
+#define HUBPREQ1_PREFETCH_SETTINGS__DST_Y_PREFETCH_MASK                                                       0xFF000000L
+//HUBPREQ1_PREFETCH_SETTINGS_C
+#define HUBPREQ1_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C__SHIFT                                                0x0
+#define HUBPREQ1_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C_MASK                                                  0x003FFFFFL
+//HUBPREQ1_VBLANK_PARAMETERS_0
+#define HUBPREQ1_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK__SHIFT                                              0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK__SHIFT                                             0x8
+#define HUBPREQ1_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK_MASK                                                0x0000007FL
+#define HUBPREQ1_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK_MASK                                               0x00003F00L
+//HUBPREQ1_VBLANK_PARAMETERS_1
+#define HUBPREQ1_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L__SHIFT                                    0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L_MASK                                      0x007FFFFFL
+//HUBPREQ1_VBLANK_PARAMETERS_2
+#define HUBPREQ1_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C__SHIFT                                    0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C_MASK                                      0x007FFFFFL
+//HUBPREQ1_VBLANK_PARAMETERS_3
+#define HUBPREQ1_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L__SHIFT                                   0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L_MASK                                     0x007FFFFFL
+//HUBPREQ1_VBLANK_PARAMETERS_4
+#define HUBPREQ1_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C__SHIFT                                   0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C_MASK                                     0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_0
+#define HUBPREQ1_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP__SHIFT                                                  0x0
+#define HUBPREQ1_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP__SHIFT                                                 0x8
+#define HUBPREQ1_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP_MASK                                                    0x0000007FL
+#define HUBPREQ1_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP_MASK                                                   0x00003F00L
+//HUBPREQ1_FLIP_PARAMETERS_1
+#define HUBPREQ1_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L__SHIFT                                        0x0
+#define HUBPREQ1_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L_MASK                                          0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_2
+#define HUBPREQ1_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L__SHIFT                                       0x0
+#define HUBPREQ1_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L_MASK                                         0x007FFFFFL
+//HUBPREQ1_NOM_PARAMETERS_0
+#define HUBPREQ1_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L__SHIFT                                             0x0
+#define HUBPREQ1_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L_MASK                                               0x0001FFFFL
+//HUBPREQ1_NOM_PARAMETERS_1
+#define HUBPREQ1_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L__SHIFT                                          0x0
+#define HUBPREQ1_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L_MASK                                            0x007FFFFFL
+//HUBPREQ1_NOM_PARAMETERS_2
+#define HUBPREQ1_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C__SHIFT                                             0x0
+#define HUBPREQ1_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C_MASK                                               0x0001FFFFL
+//HUBPREQ1_NOM_PARAMETERS_3
+#define HUBPREQ1_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C__SHIFT                                          0x0
+#define HUBPREQ1_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C_MASK                                            0x007FFFFFL
+//HUBPREQ1_NOM_PARAMETERS_4
+#define HUBPREQ1_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L__SHIFT                                            0x0
+#define HUBPREQ1_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L_MASK                                              0x0001FFFFL
+//HUBPREQ1_NOM_PARAMETERS_5
+#define HUBPREQ1_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L__SHIFT                                         0x0
+#define HUBPREQ1_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L_MASK                                           0x007FFFFFL
+//HUBPREQ1_NOM_PARAMETERS_6
+#define HUBPREQ1_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C__SHIFT                                            0x0
+#define HUBPREQ1_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C_MASK                                              0x0001FFFFL
+//HUBPREQ1_NOM_PARAMETERS_7
+#define HUBPREQ1_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C__SHIFT                                         0x0
+#define HUBPREQ1_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C_MASK                                           0x007FFFFFL
+//HUBPREQ1_PER_LINE_DELIVERY_PRE
+#define HUBPREQ1_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L__SHIFT                                 0x0
+#define HUBPREQ1_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C__SHIFT                                 0x10
+#define HUBPREQ1_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L_MASK                                   0x00001FFFL
+#define HUBPREQ1_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C_MASK                                   0x1FFF0000L
+//HUBPREQ1_PER_LINE_DELIVERY
+#define HUBPREQ1_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L__SHIFT                                         0x0
+#define HUBPREQ1_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C__SHIFT                                         0x10
+#define HUBPREQ1_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L_MASK                                           0x00001FFFL
+#define HUBPREQ1_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C_MASK                                           0x1FFF0000L
+//HUBPREQ1_CURSOR_SETTINGS
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET__SHIFT                                                 0x0
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST__SHIFT                                             0x8
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET__SHIFT                                                 0x10
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST__SHIFT                                             0x18
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET_MASK                                                   0x000000FFL
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST_MASK                                               0x00000300L
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET_MASK                                                   0x00FF0000L
+#define HUBPREQ1_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST_MASK                                               0x03000000L
+//HUBPREQ1_REF_FREQ_TO_PIX_FREQ
+#define HUBPREQ1_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ__SHIFT                                            0x0
+#define HUBPREQ1_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ_MASK                                              0x001FFFFFL
+//HUBPREQ1_DST_Y_DELTA_DRQ_LIMIT
+#define HUBPREQ1_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT__SHIFT                                          0x0
+#define HUBPREQ1_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT_MASK                                            0x00007FFFL
+//HUBPREQ1_HUBPREQ_MEM_PWR_CTRL
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE__SHIFT                                          0x0
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS__SHIFT                                            0x2
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE__SHIFT                                          0x4
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS__SHIFT                                            0x6
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE__SHIFT                                          0x8
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS__SHIFT                                            0xa
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE__SHIFT                                           0xc
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS__SHIFT                                             0xe
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE__SHIFT                                          0xf
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS__SHIFT                                            0x11
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE_MASK                                            0x00000003L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS_MASK                                              0x00000004L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE_MASK                                            0x00000030L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS_MASK                                              0x00000040L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE_MASK                                            0x00000300L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS_MASK                                              0x00000400L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE_MASK                                             0x00003000L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS_MASK                                               0x00004000L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE_MASK                                            0x00018000L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS_MASK                                              0x00020000L
+//HUBPREQ1_HUBPREQ_MEM_PWR_STATUS
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE__SHIFT                                        0x0
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE__SHIFT                                        0x2
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE__SHIFT                                        0x4
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE__SHIFT                                         0x6
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE__SHIFT                                        0x8
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE_MASK                                          0x00000003L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE_MASK                                          0x0000000CL
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE_MASK                                          0x00000030L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE_MASK                                           0x000000C0L
+#define HUBPREQ1_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE_MASK                                          0x00000300L
+//HUBPREQ1_VBLANK_PARAMETERS_5
+#define HUBPREQ1_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK__SHIFT                                       0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK_MASK                                         0x007FFFFFL
+//HUBPREQ1_VBLANK_PARAMETERS_6
+#define HUBPREQ1_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK__SHIFT                                         0x0
+#define HUBPREQ1_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK_MASK                                           0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_3
+#define HUBPREQ1_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP__SHIFT                                           0x0
+#define HUBPREQ1_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP_MASK                                             0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_4
+#define HUBPREQ1_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP__SHIFT                                             0x0
+#define HUBPREQ1_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP_MASK                                               0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_5
+#define HUBPREQ1_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C__SHIFT                                        0x0
+#define HUBPREQ1_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C_MASK                                          0x007FFFFFL
+//HUBPREQ1_FLIP_PARAMETERS_6
+#define HUBPREQ1_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C__SHIFT                                       0x0
+#define HUBPREQ1_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C_MASK                                         0x007FFFFFL
+//HUBPREQ1_UCLK_PSTATE_FORCE
+#define HUBPREQ1_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN__SHIFT                                          0x0
+#define HUBPREQ1_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE__SHIFT                                       0x1
+#define HUBPREQ1_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN__SHIFT                                        0x2
+#define HUBPREQ1_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE__SHIFT                                     0x3
+#define HUBPREQ1_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN_MASK                                            0x00000001L
+#define HUBPREQ1_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE_MASK                                         0x00000002L
+#define HUBPREQ1_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN_MASK                                          0x00000004L
+#define HUBPREQ1_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE_MASK                                       0x00000008L
+//HUBPREQ1_HUBPREQ_STATUS_REG0
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0__SHIFT                                         0x0
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1__SHIFT                                         0x8
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT__SHIFT                                                 0x10
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0_MASK                                           0x0000001FL
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1_MASK                                           0x00001F00L
+#define HUBPREQ1_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT_MASK                                                   0x7FFF0000L
+//HUBPREQ1_HUBPREQ_STATUS_REG1
+#define HUBPREQ1_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0__SHIFT                                       0x0
+#define HUBPREQ1_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1__SHIFT                                       0x10
+#define HUBPREQ1_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0_MASK                                         0x00003FFFL
+#define HUBPREQ1_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1_MASK                                         0x3FFF0000L
+//HUBPREQ1_HUBPREQ_STATUS_REG2
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0__SHIFT                                      0x0
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0__SHIFT                                     0x1
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0__SHIFT                               0x2
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0__SHIFT                               0x3
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0__SHIFT                                   0x4
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0__SHIFT                                             0x5
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1__SHIFT                                      0x8
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1__SHIFT                                     0x9
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1__SHIFT                               0xa
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1__SHIFT                               0xb
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1__SHIFT                                   0xc
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1__SHIFT                                             0xd
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR__SHIFT                                     0x10
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR__SHIFT                                    0x11
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR__SHIFT                              0x12
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR__SHIFT                              0x13
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR__SHIFT                                  0x14
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR__SHIFT                                            0x15
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_VBLANK__SHIFT                                                    0x1a
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN__SHIFT                                                   0x1b
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY__SHIFT                                          0x1c
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH__SHIFT                                             0x1d
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0__SHIFT                                            0x1e
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1__SHIFT                                            0x1f
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0_MASK                                        0x00000001L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0_MASK                                       0x00000002L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0_MASK                                 0x00000004L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0_MASK                                 0x00000008L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0_MASK                                     0x00000010L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0_MASK                                               0x00000020L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1_MASK                                        0x00000100L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1_MASK                                       0x00000200L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1_MASK                                 0x00000400L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1_MASK                                 0x00000800L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1_MASK                                     0x00001000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1_MASK                                               0x00002000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR_MASK                                       0x00010000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR_MASK                                      0x00020000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR_MASK                                0x00040000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR_MASK                                0x00080000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR_MASK                                    0x00100000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR_MASK                                              0x00200000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_VBLANK_MASK                                                      0x04000000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN_MASK                                                     0x08000000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY_MASK                                            0x10000000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH_MASK                                               0x20000000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0_MASK                                              0x40000000L
+#define HUBPREQ1_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1_MASK                                              0x80000000L
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpret_dispdec
+//HUBPRET1_HUBPRET_CONTROL
+#define HUBPRET1_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS__SHIFT                                          0x4
+#define HUBPRET1_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE__SHIFT                                            0xf
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA__SHIFT                                                   0x10
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G__SHIFT                                                     0x12
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B__SHIFT                                                    0x14
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R__SHIFT                                                    0x16
+#define HUBPRET1_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE__SHIFT                                                0x18
+#define HUBPRET1_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS_MASK                                            0x00007FF0L
+#define HUBPRET1_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE_MASK                                              0x00008000L
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA_MASK                                                     0x00030000L
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G_MASK                                                       0x000C0000L
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B_MASK                                                      0x00300000L
+#define HUBPRET1_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R_MASK                                                      0x00C00000L
+#define HUBPRET1_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE_MASK                                                  0xFF000000L
+//HUBPRET1_HUBPRET_MEM_PWR_CTRL
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE__SHIFT                                             0x8
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS__SHIFT                                               0xa
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE__SHIFT                                            0x10
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS__SHIFT                                              0x12
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE__SHIFT                                          0x14
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE_MASK                                               0x00000300L
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS_MASK                                                 0x00000400L
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE_MASK                                              0x00030000L
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS_MASK                                                0x00040000L
+#define HUBPRET1_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE_MASK                                            0x00300000L
+//HUBPRET1_HUBPRET_MEM_PWR_STATUS
+#define HUBPRET1_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE__SHIFT                                           0x2
+#define HUBPRET1_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE__SHIFT                                          0x4
+#define HUBPRET1_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE_MASK                                             0x0000000CL
+#define HUBPRET1_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE_MASK                                            0x00000030L
+//HUBPRET1_HUBPRET_READ_LINE_CTRL0
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE__SHIFT                         0x0
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM__SHIFT                                0x10
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE_MASK                           0x0000FFFFL
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM_MASK                                  0x3FFF0000L
+//HUBPRET1_HUBPRET_READ_LINE_CTRL1
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED__SHIFT                    0x0
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE__SHIFT                                0x10
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED_MASK                      0x00003FFFL
+#define HUBPRET1_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE_MASK                                  0xFFFF0000L
+//HUBPRET1_HUBPRET_READ_LINE0
+#define HUBPRET1_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START__SHIFT                                             0x0
+#define HUBPRET1_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END__SHIFT                                               0x10
+#define HUBPRET1_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START_MASK                                               0x00003FFFL
+#define HUBPRET1_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END_MASK                                                 0x3FFF0000L
+//HUBPRET1_HUBPRET_READ_LINE1
+#define HUBPRET1_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START__SHIFT                                             0x0
+#define HUBPRET1_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END__SHIFT                                               0x10
+#define HUBPRET1_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START_MASK                                               0x00003FFFL
+#define HUBPRET1_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END_MASK                                                 0x3FFF0000L
+//HUBPRET1_HUBPRET_INTERRUPT
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK__SHIFT                                               0x0
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK__SHIFT                                           0x1
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK__SHIFT                                           0x2
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE__SHIFT                                               0x4
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE__SHIFT                                           0x5
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE__SHIFT                                           0x6
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR__SHIFT                                              0x8
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR__SHIFT                                          0x9
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR__SHIFT                                          0xa
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS__SHIFT                                                 0xc
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS__SHIFT                                             0xd
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS__SHIFT                                             0xe
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS__SHIFT                                             0x10
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS__SHIFT                                         0x11
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS__SHIFT                                         0x12
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK_MASK                                                 0x00000001L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK_MASK                                             0x00000002L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK_MASK                                             0x00000004L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE_MASK                                                 0x00000010L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE_MASK                                             0x00000020L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE_MASK                                             0x00000040L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR_MASK                                                0x00000100L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR_MASK                                            0x00000200L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR_MASK                                            0x00000400L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS_MASK                                                   0x00001000L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS_MASK                                               0x00002000L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS_MASK                                               0x00004000L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS_MASK                                               0x00010000L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS_MASK                                           0x00020000L
+#define HUBPRET1_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS_MASK                                           0x00040000L
+//HUBPRET1_HUBPRET_READ_LINE_VALUE
+#define HUBPRET1_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE__SHIFT                                               0x0
+#define HUBPRET1_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT__SHIFT                                      0x10
+#define HUBPRET1_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_MASK                                                 0x00003FFFL
+#define HUBPRET1_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT_MASK                                        0x3FFF0000L
+//HUBPRET1_HUBPRET_READ_LINE_STATUS
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK__SHIFT                                            0x0
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE__SHIFT                                      0x4
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE__SHIFT                                     0x5
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE__SHIFT                                      0x8
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE__SHIFT                                     0xa
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK_MASK                                              0x00000001L
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE_MASK                                        0x00000010L
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE_MASK                                       0x00000020L
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE_MASK                                        0x00000100L
+#define HUBPRET1_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE_MASK                                       0x00000400L
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_cursor0_dispdec
+//CURSOR0_1_CURSOR_CONTROL
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_ENABLE__SHIFT                                                        0x0
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_REQ_MODE__SHIFT                                                      0x2
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT                                                    0x4
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_MODE__SHIFT                                                          0x8
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_TMZ__SHIFT                                                           0xc
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PITCH__SHIFT                                                         0x10
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS__SHIFT                     0x14
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK__SHIFT                                               0x18
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN__SHIFT                                    0x1e
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL__SHIFT                                   0x1f
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_ENABLE_MASK                                                          0x00000001L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_REQ_MODE_MASK                                                        0x00000004L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_2X_MAGNIFY_MASK                                                      0x00000010L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_MODE_MASK                                                            0x00000700L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_TMZ_MASK                                                             0x00001000L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PITCH_MASK                                                           0x00030000L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS_MASK                       0x00100000L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK_MASK                                                 0x1F000000L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN_MASK                                      0x40000000L
+#define CURSOR0_1_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL_MASK                                     0x80000000L
+//CURSOR0_1_CURSOR_SURFACE_ADDRESS
+#define CURSOR0_1_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS__SHIFT                                       0x0
+#define CURSOR0_1_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS_MASK                                         0xFFFFFFFFL
+//CURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH
+#define CURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH__SHIFT                             0x0
+#define CURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH_MASK                               0x0000FFFFL
+//CURSOR0_1_CURSOR_SIZE
+#define CURSOR0_1_CURSOR_SIZE__CURSOR_HEIGHT__SHIFT                                                           0x0
+#define CURSOR0_1_CURSOR_SIZE__CURSOR_WIDTH__SHIFT                                                            0x10
+#define CURSOR0_1_CURSOR_SIZE__CURSOR_HEIGHT_MASK                                                             0x000001FFL
+#define CURSOR0_1_CURSOR_SIZE__CURSOR_WIDTH_MASK                                                              0x01FF0000L
+//CURSOR0_1_CURSOR_POSITION
+#define CURSOR0_1_CURSOR_POSITION__CURSOR_Y_POSITION__SHIFT                                                   0x0
+#define CURSOR0_1_CURSOR_POSITION__CURSOR_X_POSITION__SHIFT                                                   0x10
+#define CURSOR0_1_CURSOR_POSITION__CURSOR_Y_POSITION_MASK                                                     0x00003FFFL
+#define CURSOR0_1_CURSOR_POSITION__CURSOR_X_POSITION_MASK                                                     0x3FFF0000L
+//CURSOR0_1_CURSOR_HOT_SPOT
+#define CURSOR0_1_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y__SHIFT                                                   0x0
+#define CURSOR0_1_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X__SHIFT                                                   0x10
+#define CURSOR0_1_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y_MASK                                                     0x000000FFL
+#define CURSOR0_1_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X_MASK                                                     0x00FF0000L
+//CURSOR0_1_CURSOR_STEREO_CONTROL
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN__SHIFT                                              0x0
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET__SHIFT                                         0x4
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET__SHIFT                                       0x12
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN_MASK                                                0x00000001L
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET_MASK                                           0x0003FFF0L
+#define CURSOR0_1_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET_MASK                                         0xFFFC0000L
+//CURSOR0_1_CURSOR_DST_OFFSET
+#define CURSOR0_1_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET__SHIFT                                               0x0
+#define CURSOR0_1_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET_MASK                                                 0x00001FFFL
+//CURSOR0_1_CURSOR_MEM_PWR_CTRL
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE__SHIFT                                              0x0
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS__SHIFT                                                0x2
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE__SHIFT                                            0x4
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE_MASK                                                0x00000003L
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS_MASK                                                  0x00000004L
+#define CURSOR0_1_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE_MASK                                              0x00000030L
+//CURSOR0_1_CURSOR_MEM_PWR_STATUS
+#define CURSOR0_1_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE__SHIFT                                            0x0
+#define CURSOR0_1_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE_MASK                                              0x00000003L
+//CURSOR0_1_DMDATA_ADDRESS_HIGH
+#define CURSOR0_1_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH__SHIFT                                             0x0
+#define CURSOR0_1_DMDATA_ADDRESS_HIGH__DMDATA_TMZ__SHIFT                                                      0x1e
+#define CURSOR0_1_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH_MASK                                               0x0000FFFFL
+#define CURSOR0_1_DMDATA_ADDRESS_HIGH__DMDATA_TMZ_MASK                                                        0x40000000L
+//CURSOR0_1_DMDATA_ADDRESS_LOW
+#define CURSOR0_1_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW__SHIFT                                               0x0
+#define CURSOR0_1_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW_MASK                                                 0xFFFFFFFFL
+//CURSOR0_1_DMDATA_CNTL
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_UPDATED__SHIFT                                                          0x0
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_REPEAT__SHIFT                                                           0x1
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_MODE__SHIFT                                                             0x2
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_SIZE__SHIFT                                                             0x10
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_UPDATED_MASK                                                            0x00000001L
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_REPEAT_MASK                                                             0x00000002L
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_MODE_MASK                                                               0x00000004L
+#define CURSOR0_1_DMDATA_CNTL__DMDATA_SIZE_MASK                                                               0x0FFF0000L
+//CURSOR0_1_DMDATA_QOS_CNTL
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_QOS_MODE__SHIFT                                                     0x0
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL__SHIFT                                                    0x4
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_DL_DELTA__SHIFT                                                     0x10
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_QOS_MODE_MASK                                                       0x00000001L
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL_MASK                                                      0x000000F0L
+#define CURSOR0_1_DMDATA_QOS_CNTL__DMDATA_DL_DELTA_MASK                                                       0xFFFF0000L
+//CURSOR0_1_DMDATA_STATUS
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_DONE__SHIFT                                                           0x0
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_UNDERFLOW__SHIFT                                                      0x2
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR__SHIFT                                                0x4
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_DONE_MASK                                                             0x00000001L
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_UNDERFLOW_MASK                                                        0x00000004L
+#define CURSOR0_1_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR_MASK                                                  0x00000010L
+//CURSOR0_1_DMDATA_SW_CNTL
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_UPDATED__SHIFT                                                    0x0
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_REPEAT__SHIFT                                                     0x1
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_SIZE__SHIFT                                                       0x10
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_UPDATED_MASK                                                      0x00000001L
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_REPEAT_MASK                                                       0x00000002L
+#define CURSOR0_1_DMDATA_SW_CNTL__DMDATA_SW_SIZE_MASK                                                         0x0FFF0000L
+//CURSOR0_1_DMDATA_SW_DATA
+#define CURSOR0_1_DMDATA_SW_DATA__DMDATA_SW_DATA__SHIFT                                                       0x0
+#define CURSOR0_1_DMDATA_SW_DATA__DMDATA_SW_DATA_MASK                                                         0xFFFFFFFFL
+//CURSOR0_1_HUBP_3DLUT_CONTROL
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE__SHIFT                                                0x0
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE__SHIFT                                       0x1
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH__SHIFT                                                 0x2
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ__SHIFT                                                   0x12
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B__SHIFT                                        0x16
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G__SHIFT                                        0x18
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R__SHIFT                                        0x1a
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS__SHIFT                                         0x1d
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH__SHIFT                                             0x1e
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE__SHIFT                                                  0x1f
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE_MASK                                                  0x00000001L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE_MASK                                         0x00000002L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH_MASK                                                   0x0003FFFCL
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ_MASK                                                     0x003C0000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B_MASK                                          0x00C00000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G_MASK                                          0x03000000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R_MASK                                          0x0C000000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS_MASK                                           0x20000000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH_MASK                                               0x40000000L
+#define CURSOR0_1_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE_MASK                                                    0x80000000L
+//CURSOR0_1_HUBP_3DLUT_ADDRESS_LOW
+#define CURSOR0_1_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW__SHIFT                                       0x0
+#define CURSOR0_1_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW_MASK                                         0xFFFFFFFFL
+//CURSOR0_1_HUBP_3DLUT_ADDRESS_HIGH
+#define CURSOR0_1_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH__SHIFT                                     0x0
+#define CURSOR0_1_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH_MASK                                       0x0000FFFFL
+//CURSOR0_1_HUBP_3DLUT_DLG_PARAM
+#define CURSOR0_1_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP__SHIFT                                         0x0
+#define CURSOR0_1_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP_MASK                                           0x007FFFFFL
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON7_PERFCOUNTER_CNTL
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON7_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON7_PERFCOUNTER_CNTL2
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON7_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON7_PERFCOUNTER_STATE
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON7_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON7_PERFMON_CNTL
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON7_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON7_PERFMON_CNTL2
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON7_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON7_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON7_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON7_PERFMON_CVALUE_LOW
+#define DC_PERFMON7_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON7_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON7_PERFMON_HI
+#define DC_PERFMON7_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON7_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON7_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON7_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON7_PERFMON_LOW
+#define DC_PERFMON7_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON7_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dispdec
+//HUBP2_DCSURF_SURFACE_CONFIG
+#define HUBP2_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT__SHIFT                                              0x0
+#define HUBP2_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE__SHIFT                                                    0x8
+#define HUBP2_DCSURF_SURFACE_CONFIG__H_MIRROR_EN__SHIFT                                                       0xa
+#define HUBP2_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN__SHIFT                                                    0xb
+#define HUBP2_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT_MASK                                                0x0000007FL
+#define HUBP2_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE_MASK                                                      0x00000300L
+#define HUBP2_DCSURF_SURFACE_CONFIG__H_MIRROR_EN_MASK                                                         0x00000400L
+#define HUBP2_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN_MASK                                                      0x00000800L
+//HUBP2_DCSURF_ADDR_CONFIG
+#define HUBP2_DCSURF_ADDR_CONFIG__NUM_PIPES__SHIFT                                                            0x0
+#define HUBP2_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE__SHIFT                                                      0x6
+#define HUBP2_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                 0xc
+#define HUBP2_DCSURF_ADDR_CONFIG__NUM_PKRS__SHIFT                                                             0x10
+#define HUBP2_DCSURF_ADDR_CONFIG__NUM_PIPES_MASK                                                              0x00000007L
+#define HUBP2_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE_MASK                                                        0x000000C0L
+#define HUBP2_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                   0x00003000L
+#define HUBP2_DCSURF_ADDR_CONFIG__NUM_PKRS_MASK                                                               0x00070000L
+//HUBP2_DCSURF_TILING_CONFIG
+#define HUBP2_DCSURF_TILING_CONFIG__SW_MODE__SHIFT                                                            0x0
+#define HUBP2_DCSURF_TILING_CONFIG__DIM_TYPE__SHIFT                                                           0x7
+#define HUBP2_DCSURF_TILING_CONFIG__META_LINEAR__SHIFT                                                        0x9
+#define HUBP2_DCSURF_TILING_CONFIG__PIPE_ALIGNED__SHIFT                                                       0xb
+#define HUBP2_DCSURF_TILING_CONFIG__SW_MODE_MASK                                                              0x0000001FL
+#define HUBP2_DCSURF_TILING_CONFIG__DIM_TYPE_MASK                                                             0x00000180L
+#define HUBP2_DCSURF_TILING_CONFIG__META_LINEAR_MASK                                                          0x00000200L
+#define HUBP2_DCSURF_TILING_CONFIG__PIPE_ALIGNED_MASK                                                         0x00000800L
+//HUBP2_DCSURF_PRI_VIEWPORT_START
+#define HUBP2_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP2_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP2_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP2_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP2_DCSURF_PRI_VIEWPORT_START_C
+#define HUBP2_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP2_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP2_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP2_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP2_DCSURF_SEC_VIEWPORT_START
+#define HUBP2_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP2_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP2_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP2_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP2_DCSURF_SEC_VIEWPORT_START_C
+#define HUBP2_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP2_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP2_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP2_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP2_DCHUBP_REQ_SIZE_CONFIG
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT__SHIFT                                                     0x0
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR__SHIFT                                            0x4
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE__SHIFT                                                       0x8
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE__SHIFT                                                   0xb
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE__SHIFT                                                  0x10
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE__SHIFT                                              0x12
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE__SHIFT                                                  0x14
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE__SHIFT                                                    0x18
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT_MASK                                                       0x00000007L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR_MASK                                              0x00000070L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE_MASK                                                         0x00000700L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE_MASK                                                     0x00001800L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE_MASK                                                    0x00030000L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE_MASK                                                0x000C0000L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE_MASK                                                    0x00700000L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE_MASK                                                      0x07000000L
+//HUBP2_DCHUBP_REQ_SIZE_CONFIG_C
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C__SHIFT                                                 0x0
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C__SHIFT                                        0x4
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C__SHIFT                                                   0x8
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C__SHIFT                                               0xb
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C__SHIFT                                              0x10
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C__SHIFT                                          0x12
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C__SHIFT                                              0x14
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C_MASK                                                   0x00000007L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C_MASK                                          0x00000070L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C_MASK                                                     0x00000700L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C_MASK                                                 0x00001800L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C_MASK                                                0x00030000L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C_MASK                                            0x000C0000L
+#define HUBP2_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C_MASK                                                0x00700000L
+//HUBP2_DCHUBP_CNTL
+#define HUBP2_DCHUBP_CNTL__HUBP_BLANK_EN__SHIFT                                                               0x0
+#define HUBP2_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ__SHIFT                                                     0x1
+#define HUBP2_DCHUBP_CNTL__HUBP_SOFT_RESET__SHIFT                                                             0x2
+#define HUBP2_DCHUBP_CNTL__HUBP_IN_BLANK__SHIFT                                                               0x3
+#define HUBP2_DCHUBP_CNTL__HUBP_VTG_SEL__SHIFT                                                                0x4
+#define HUBP2_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC__SHIFT                                               0x8
+#define HUBP2_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM__SHIFT                                            0x9
+#define HUBP2_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE__SHIFT                                                     0xa
+#define HUBP2_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS__SHIFT                                                   0xb
+#define HUBP2_DCHUBP_CNTL__HUBP_TTU_DISABLE__SHIFT                                                            0xc
+#define HUBP2_DCHUBP_CNTL__HUBP_TTU_MODE__SHIFT                                                               0xd
+#define HUBP2_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ__SHIFT                                                 0x10
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS__SHIFT                                                         0x14
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD__SHIFT                                                      0x18
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR__SHIFT                                                   0x1a
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN__SHIFT                                                   0x1b
+#define HUBP2_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS__SHIFT                                                       0x1c
+#define HUBP2_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR__SHIFT                                                        0x1f
+#define HUBP2_DCHUBP_CNTL__HUBP_BLANK_EN_MASK                                                                 0x00000001L
+#define HUBP2_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ_MASK                                                       0x00000002L
+#define HUBP2_DCHUBP_CNTL__HUBP_SOFT_RESET_MASK                                                               0x00000004L
+#define HUBP2_DCHUBP_CNTL__HUBP_IN_BLANK_MASK                                                                 0x00000008L
+#define HUBP2_DCHUBP_CNTL__HUBP_VTG_SEL_MASK                                                                  0x000000F0L
+#define HUBP2_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC_MASK                                                 0x00000100L
+#define HUBP2_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM_MASK                                              0x00000200L
+#define HUBP2_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE_MASK                                                       0x00000400L
+#define HUBP2_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS_MASK                                                     0x00000800L
+#define HUBP2_DCHUBP_CNTL__HUBP_TTU_DISABLE_MASK                                                              0x00001000L
+#define HUBP2_DCHUBP_CNTL__HUBP_TTU_MODE_MASK                                                                 0x0000E000L
+#define HUBP2_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ_MASK                                                   0x000F0000L
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_MASK                                                           0x00F00000L
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD_MASK                                                        0x03000000L
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR_MASK                                                     0x04000000L
+#define HUBP2_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN_MASK                                                     0x08000000L
+#define HUBP2_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS_MASK                                                         0x70000000L
+#define HUBP2_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR_MASK                                                          0x80000000L
+//HUBP2_HUBP_CLK_CNTL
+#define HUBP2_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE__SHIFT                                                         0x0
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS__SHIFT                                                   0x5
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS__SHIFT                                                    0x8
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS__SHIFT                                                    0xc
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS__SHIFT                                                    0x10
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON__SHIFT                                                   0x14
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON__SHIFT                                                    0x15
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON__SHIFT                                                    0x16
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON__SHIFT                                                    0x17
+#define HUBP2_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON__SHIFT                                                   0x19
+#define HUBP2_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define HUBP2_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE_MASK                                                           0x00000001L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS_MASK                                                     0x00000020L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS_MASK                                                      0x00000100L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS_MASK                                                      0x00001000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS_MASK                                                      0x00010000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON_MASK                                                     0x00100000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON_MASK                                                      0x00200000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON_MASK                                                      0x00400000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON_MASK                                                      0x00800000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON_MASK                                                     0x02000000L
+#define HUBP2_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL_MASK                                                           0xF0000000L
+//HUBP2_DCHUBP_VMPG_CONFIG
+#define HUBP2_DCHUBP_VMPG_CONFIG__VMPG_SIZE__SHIFT                                                            0x0
+#define HUBP2_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE__SHIFT                                                      0x1
+#define HUBP2_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE__SHIFT                                                   0x2
+#define HUBP2_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME__SHIFT                                              0x7
+#define HUBP2_DCHUBP_VMPG_CONFIG__VMPG_SIZE_MASK                                                              0x00000001L
+#define HUBP2_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE_MASK                                                        0x00000002L
+#define HUBP2_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE_MASK                                                     0x0000007CL
+#define HUBP2_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME_MASK                                                0x00000080L
+//HUBP2_DCHUBP_MALL_CONFIG
+#define HUBP2_DCHUBP_MALL_CONFIG__USE_MALL_SEL__SHIFT                                                         0x0
+#define HUBP2_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR__SHIFT                                                  0x2
+#define HUBP2_DCHUBP_MALL_CONFIG__USE_MALL_SEL_MASK                                                           0x00000003L
+#define HUBP2_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR_MASK                                                    0x00000004L
+//HUBP2_DCHUBP_MALL_SUB_VP
+#define HUBP2_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE__SHIFT                                               0x0
+#define HUBP2_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0__SHIFT                                                 0x1
+#define HUBP2_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1__SHIFT                                                 0xf
+#define HUBP2_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE_MASK                                                 0x00000001L
+#define HUBP2_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0_MASK                                                   0x00007FFEL
+#define HUBP2_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1_MASK                                                   0x1FFF8000L
+//HUBP2_HUBPREQ_DEBUG_DB
+#define HUBP2_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG__SHIFT                                                          0x0
+#define HUBP2_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG_MASK                                                            0xFFFFFFFFL
+//HUBP2_HUBPREQ_DEBUG
+#define HUBP2_HUBPREQ_DEBUG__HUBPREQ_DEBUG__SHIFT                                                             0x0
+#define HUBP2_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS__SHIFT                                 0x1f
+#define HUBP2_HUBPREQ_DEBUG__HUBPREQ_DEBUG_MASK                                                               0x7FFFFFFFL
+#define HUBP2_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS_MASK                                   0x80000000L
+//HUBP2_HUBP_DEBUG_CTRL
+#define HUBP2_HUBP_DEBUG_CTRL__HUBP_DBG_EN__SHIFT                                                             0x0
+#define HUBP2_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS__SHIFT                                              0x4
+#define HUBP2_HUBP_DEBUG_CTRL__HUBP_DBG_EN_MASK                                                               0x00000001L
+#define HUBP2_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS_MASK                                                0x00000010L
+//HUBP2_HUBP_DEBUG_MUX_DCFCLK
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK__SHIFT                                         0x0
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK__SHIFT                                         0x8
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK__SHIFT                                         0x10
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK__SHIFT                                         0x18
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK_MASK                                           0x000000FFL
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK_MASK                                           0x0000FF00L
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK_MASK                                           0x00FF0000L
+#define HUBP2_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK_MASK                                           0xFF000000L
+//HUBP2_HUBP_DEBUG_MUX_DPPCLK
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK__SHIFT                                         0x0
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK__SHIFT                                         0x4
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK__SHIFT                                        0x8
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK_MASK                                           0x00000007L
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK_MASK                                           0x00000070L
+#define HUBP2_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK_MASK                                          0x00000700L
+//HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK__SHIFT                                 0x0
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK__SHIFT                          0x4
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK__SHIFT                              0xc
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK__SHIFT                               0x14
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK__SHIFT                               0x1c
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK_MASK                                   0x00000001L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK_MASK                            0x00000FF0L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK_MASK                                0x0001F000L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK_MASK                                 0x01F00000L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK_MASK                                 0x30000000L
+//HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK__SHIFT                                 0x0
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK__SHIFT                            0x1
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK__SHIFT                          0x4
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK__SHIFT                              0xc
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK__SHIFT                               0x14
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK_MASK                                   0x00000001L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK_MASK                              0x00000002L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK_MASK                            0x00000FF0L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK_MASK                                0x0001F000L
+#define HUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK_MASK                                 0x01F00000L
+//HUBP2_HUBP_MALL_STATUS
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN__SHIFT                                         0x0
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE__SHIFT                                         0x1
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQUEST__SHIFT                                                       0x2
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_RESPONSE__SHIFT                                                      0x3
+#define HUBP2_HUBP_MALL_STATUS__MALL_IN_USE__SHIFT                                                            0x4
+#define HUBP2_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE__SHIFT                                                 0x5
+#define HUBP2_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE__SHIFT                                                   0x6
+#define HUBP2_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE__SHIFT                                                  0x7
+#define HUBP2_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE__SHIFT                                                  0x8
+#define HUBP2_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH__SHIFT                                                  0x9
+#define HUBP2_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME__SHIFT                                                    0xa
+#define HUBP2_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME__SHIFT                                                    0xb
+#define HUBP2_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL__SHIFT                                                     0xc
+#define HUBP2_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL__SHIFT                                                     0xd
+#define HUBP2_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL__SHIFT                                                     0xe
+#define HUBP2_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME__SHIFT                                                  0xf
+#define HUBP2_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS__SHIFT                                   0x10
+#define HUBP2_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING__SHIFT                                            0x11
+#define HUBP2_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING__SHIFT                                                   0x12
+#define HUBP2_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO__SHIFT                                                      0x13
+#define HUBP2_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x14
+#define HUBP2_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x15
+#define HUBP2_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING__SHIFT                                                   0x16
+#define HUBP2_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP__SHIFT                                              0x17
+#define HUBP2_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING__SHIFT                                                   0x18
+#define HUBP2_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING__SHIFT                                                  0x19
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN_MASK                                           0x00000001L
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE_MASK                                           0x00000002L
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_REQUEST_MASK                                                         0x00000004L
+#define HUBP2_HUBP_MALL_STATUS__MALL_USE_RESPONSE_MASK                                                        0x00000008L
+#define HUBP2_HUBP_MALL_STATUS__MALL_IN_USE_MASK                                                              0x00000010L
+#define HUBP2_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE_MASK                                                   0x00000020L
+#define HUBP2_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE_MASK                                                     0x00000040L
+#define HUBP2_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE_MASK                                                    0x00000080L
+#define HUBP2_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE_MASK                                                    0x00000100L
+#define HUBP2_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH_MASK                                                    0x00000200L
+#define HUBP2_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME_MASK                                                      0x00000400L
+#define HUBP2_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME_MASK                                                      0x00000800L
+#define HUBP2_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL_MASK                                                       0x00001000L
+#define HUBP2_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_MASK                                                       0x00002000L
+#define HUBP2_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL_MASK                                                       0x00004000L
+#define HUBP2_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME_MASK                                                    0x00008000L
+#define HUBP2_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS_MASK                                     0x00010000L
+#define HUBP2_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING_MASK                                              0x00020000L
+#define HUBP2_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING_MASK                                                     0x00040000L
+#define HUBP2_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO_MASK                                                        0x00080000L
+#define HUBP2_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP_MASK                                              0x00100000L
+#define HUBP2_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP_MASK                                              0x00200000L
+#define HUBP2_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING_MASK                                                     0x00400000L
+#define HUBP2_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP_MASK                                                0x00800000L
+#define HUBP2_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING_MASK                                                     0x01000000L
+#define HUBP2_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING_MASK                                                    0x02000000L
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpreq_dispdec
+//HUBPREQ2_DCSURF_SURFACE_PITCH
+#define HUBPREQ2_DCSURF_SURFACE_PITCH__PITCH__SHIFT                                                           0x0
+#define HUBPREQ2_DCSURF_SURFACE_PITCH__META_PITCH__SHIFT                                                      0x10
+#define HUBPREQ2_DCSURF_SURFACE_PITCH__PITCH_MASK                                                             0x00003FFFL
+#define HUBPREQ2_DCSURF_SURFACE_PITCH__META_PITCH_MASK                                                        0x3FFF0000L
+//HUBPREQ2_DCSURF_SURFACE_PITCH_C
+#define HUBPREQ2_DCSURF_SURFACE_PITCH_C__PITCH_C__SHIFT                                                       0x0
+#define HUBPREQ2_DCSURF_SURFACE_PITCH_C__META_PITCH_C__SHIFT                                                  0x10
+#define HUBPREQ2_DCSURF_SURFACE_PITCH_C__PITCH_C_MASK                                                         0x00003FFFL
+#define HUBPREQ2_DCSURF_SURFACE_PITCH_C__META_PITCH_C_MASK                                                    0x3FFF0000L
+//HUBPREQ2_VMID_SETTINGS_0
+#define HUBPREQ2_VMID_SETTINGS_0__VMID__SHIFT                                                                 0x0
+#define HUBPREQ2_VMID_SETTINGS_0__VMID_MASK                                                                   0x0000000FL
+//HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS__SHIFT                               0x0
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS_MASK                                 0xFFFFFFFFL
+//HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH__SHIFT                     0x0
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH_MASK                       0x0000FFFFL
+//HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C__SHIFT                           0x0
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C_MASK                             0xFFFFFFFFL
+//HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C__SHIFT                 0x0
+#define HUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C_MASK                   0x0000FFFFL
+//HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS__SHIFT                           0x0
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS_MASK                             0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT                 0x0
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH_MASK                   0x0000FFFFL
+//HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C__SHIFT                       0x0
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C_MASK                         0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C__SHIFT             0x0
+#define HUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C_MASK               0x0000FFFFL
+//HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS__SHIFT                     0x0
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS_MASK                       0xFFFFFFFFL
+//HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH__SHIFT           0x0
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH_MASK             0x0000FFFFL
+//HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C__SHIFT                 0x0
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C_MASK                   0xFFFFFFFFL
+//HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT       0x0
+#define HUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C_MASK         0x0000FFFFL
+//HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS__SHIFT                 0x0
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS_MASK                   0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH__SHIFT       0x0
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH_MASK         0x0000FFFFL
+//HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C__SHIFT             0x0
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C_MASK               0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT   0x0
+#define HUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C_MASK     0x0000FFFFL
+//HUBPREQ2_DCSURF_SURFACE_CONTROL
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ__SHIFT                                           0x0
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN__SHIFT                                        0x4
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK__SHIFT                                   0x5
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C__SHIFT                                         0x7
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C__SHIFT                                 0xb
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ__SHIFT                                         0xd
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN__SHIFT                                      0x11
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK__SHIFT                                 0x12
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C__SHIFT                                       0x14
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C__SHIFT                               0x18
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ__SHIFT                                      0x1a
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C__SHIFT                                    0x1b
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ__SHIFT                                    0x1c
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C__SHIFT                                  0x1d
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_MASK                                             0x0000000FL
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN_MASK                                          0x00000010L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_MASK                                     0x00000060L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C_MASK                                           0x00000780L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C_MASK                                   0x00001800L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_MASK                                           0x0001E000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN_MASK                                        0x00020000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_MASK                                   0x000C0000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C_MASK                                         0x00F00000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C_MASK                                 0x03000000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_MASK                                        0x04000000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C_MASK                                      0x08000000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_MASK                                      0x10000000L
+#define HUBPREQ2_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C_MASK                                    0x20000000L
+//HUBPREQ2_DCSURF_FLIP_CONTROL
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK__SHIFT                                              0x0
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE__SHIFT                                                0x1
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM__SHIFT                                    0x4
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING__SHIFT                                             0x8
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS__SHIFT                                0x9
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC__SHIFT                                 0xc
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC__SHIFT                                       0x10
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE__SHIFT                               0x11
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY__SHIFT                              0x12
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY__SHIFT                                       0x14
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK_MASK                                                0x00000001L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE_MASK                                                  0x00000002L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM_MASK                                      0x000000F0L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_MASK                                               0x00000100L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS_MASK                                  0x00000200L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC_MASK                                   0x00003000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC_MASK                                         0x00010000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE_MASK                                 0x00020000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY_MASK                                0x00040000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY_MASK                                         0x3FF00000L
+//HUBPREQ2_DCSURF_FLIP_CONTROL2
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME__SHIFT                                   0x0
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE__SHIFT                                              0x8
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK__SHIFT                                                0x9
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE__SHIFT                                    0xa
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH__SHIFT                                     0xc
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE__SHIFT                                        0x1c
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE__SHIFT                                    0x1f
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME_MASK                                     0x000000FFL
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE_MASK                                                0x00000100L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK_MASK                                                  0x00000200L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE_MASK                                      0x00000400L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH_MASK                                       0x00001000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE_MASK                                          0x10000000L
+#define HUBPREQ2_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE_MASK                                      0x80000000L
+//HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK__SHIFT                                  0x0
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE__SHIFT                                  0x1
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK__SHIFT                             0x2
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE__SHIFT                             0x3
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR__SHIFT                                     0x8
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR__SHIFT                                0x9
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED__SHIFT                                  0x10
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS__SHIFT                                0x11
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED__SHIFT                             0x12
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS__SHIFT                           0x13
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK_MASK                                    0x00000001L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE_MASK                                    0x00000002L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK_MASK                               0x00000004L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE_MASK                               0x00000008L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR_MASK                                       0x00000100L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR_MASK                                  0x00000200L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED_MASK                                    0x00010000L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS_MASK                                  0x00020000L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED_MASK                               0x00040000L
+#define HUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS_MASK                             0x00080000L
+//HUBPREQ2_DCSURF_SURFACE_INUSE
+#define HUBPREQ2_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS__SHIFT                                           0x0
+#define HUBPREQ2_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS_MASK                                             0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH__SHIFT                                 0x0
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID__SHIFT                                         0x1c
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH_MASK                                   0x0000FFFFL
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID_MASK                                           0xF0000000L
+//HUBPREQ2_DCSURF_SURFACE_INUSE_C
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C__SHIFT                                       0x0
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C_MASK                                         0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C__SHIFT                             0x0
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C__SHIFT                                     0x1c
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C_MASK                               0x0000FFFFL
+#define HUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C_MASK                                       0xF0000000L
+//HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS__SHIFT                         0x0
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS_MASK                           0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH__SHIFT               0x0
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID__SHIFT                       0x1c
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_MASK                 0x0000FFFFL
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID_MASK                         0xF0000000L
+//HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C__SHIFT                     0x0
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C_MASK                       0xFFFFFFFFL
+//HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C__SHIFT           0x0
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C__SHIFT                   0x1c
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C_MASK             0x0000FFFFL
+#define HUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C_MASK                     0xF0000000L
+//HUBPREQ2_DCN_EXPANSION_MODE
+#define HUBPREQ2_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE__SHIFT                                                0x0
+#define HUBPREQ2_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE__SHIFT                                                0x2
+#define HUBPREQ2_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE__SHIFT                                                0x4
+#define HUBPREQ2_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE__SHIFT                                                0x6
+#define HUBPREQ2_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE_MASK                                                  0x00000003L
+#define HUBPREQ2_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE_MASK                                                  0x0000000CL
+#define HUBPREQ2_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE_MASK                                                  0x00000030L
+#define HUBPREQ2_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE_MASK                                                  0x000000C0L
+//HUBPREQ2_DCN_TTU_QOS_WM
+#define HUBPREQ2_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM__SHIFT                                                      0x0
+#define HUBPREQ2_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM__SHIFT                                                     0x10
+#define HUBPREQ2_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM_MASK                                                        0x00003FFFL
+#define HUBPREQ2_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM_MASK                                                       0x3FFF0000L
+//HUBPREQ2_DCN_GLOBAL_TTU_CNTL
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK__SHIFT                                                   0x0
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT__SHIFT                                             0x18
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT__SHIFT                                             0x19
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE__SHIFT                                                     0x1b
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP__SHIFT                                                   0x1c
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK_MASK                                                     0x00FFFFFFL
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT_MASK                                               0x01000000L
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT_MASK                                               0x02000000L
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE_MASK                                                       0x08000000L
+#define HUBPREQ2_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP_MASK                                                     0xF0000000L
+//HUBPREQ2_DCN_SURF0_TTU_CNTL0
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ2_DCN_SURF0_TTU_CNTL1
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ2_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ2_DCN_SURF1_TTU_CNTL0
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ2_DCN_SURF1_TTU_CNTL1
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ2_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ2_DCN_CUR0_TTU_CNTL0
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ2_DCN_CUR0_TTU_CNTL1
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ2_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ2_DCN_CUR1_TTU_CNTL0
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ2_DCN_CUR1_TTU_CNTL1
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ2_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ2_DCN_DMDATA_VM_CNTL
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA__SHIFT                                              0x0
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS__SHIFT                                            0x10
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR__SHIFT                                      0x14
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS__SHIFT                                        0x18
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS__SHIFT                                             0x19
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR__SHIFT                                  0x1a
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE__SHIFT                                                    0x1f
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA_MASK                                                0x0000FFFFL
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_MASK                                              0x000F0000L
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR_MASK                                        0x00100000L
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_MASK                                          0x01000000L
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS_MASK                                               0x02000000L
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR_MASK                                    0x04000000L
+#define HUBPREQ2_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE_MASK                                                      0x80000000L
+//HUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR
+#define HUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR__SHIFT                       0x0
+#define HUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR_MASK                         0x3FFFFFFFL
+//HUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define HUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR__SHIFT                     0x0
+#define HUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MASK                       0x3FFFFFFFL
+//HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                  0x0
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                             0x3
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                0x5
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                   0x6
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                    0x00000001L
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                               0x00000018L
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                  0x00000020L
+#define HUBPREQ2_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                     0x00000040L
+//HUBPREQ2_BLANK_OFFSET_0
+#define HUBPREQ2_BLANK_OFFSET_0__REFCYC_H_BLANK_END__SHIFT                                                    0x0
+#define HUBPREQ2_BLANK_OFFSET_0__DLG_V_BLANK_END__SHIFT                                                       0x10
+#define HUBPREQ2_BLANK_OFFSET_0__REFCYC_H_BLANK_END_MASK                                                      0x00001FFFL
+#define HUBPREQ2_BLANK_OFFSET_0__DLG_V_BLANK_END_MASK                                                         0x7FFF0000L
+//HUBPREQ2_BLANK_OFFSET_1
+#define HUBPREQ2_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START__SHIFT                                                  0x0
+#define HUBPREQ2_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START_MASK                                                    0x0003FFFFL
+//HUBPREQ2_DST_DIMENSIONS
+#define HUBPREQ2_DST_DIMENSIONS__REFCYC_PER_HTOTAL__SHIFT                                                     0x0
+#define HUBPREQ2_DST_DIMENSIONS__REFCYC_PER_HTOTAL_MASK                                                       0x001FFFFFL
+//HUBPREQ2_DST_AFTER_SCALER
+#define HUBPREQ2_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER__SHIFT                                               0x0
+#define HUBPREQ2_DST_AFTER_SCALER__DST_Y_AFTER_SCALER__SHIFT                                                  0x10
+#define HUBPREQ2_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER_MASK                                                 0x00001FFFL
+#define HUBPREQ2_DST_AFTER_SCALER__DST_Y_AFTER_SCALER_MASK                                                    0x00070000L
+//HUBPREQ2_PREFETCH_SETTINGS
+#define HUBPREQ2_PREFETCH_SETTINGS__VRATIO_PREFETCH__SHIFT                                                    0x0
+#define HUBPREQ2_PREFETCH_SETTINGS__DST_Y_PREFETCH__SHIFT                                                     0x18
+#define HUBPREQ2_PREFETCH_SETTINGS__VRATIO_PREFETCH_MASK                                                      0x003FFFFFL
+#define HUBPREQ2_PREFETCH_SETTINGS__DST_Y_PREFETCH_MASK                                                       0xFF000000L
+//HUBPREQ2_PREFETCH_SETTINGS_C
+#define HUBPREQ2_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C__SHIFT                                                0x0
+#define HUBPREQ2_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C_MASK                                                  0x003FFFFFL
+//HUBPREQ2_VBLANK_PARAMETERS_0
+#define HUBPREQ2_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK__SHIFT                                              0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK__SHIFT                                             0x8
+#define HUBPREQ2_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK_MASK                                                0x0000007FL
+#define HUBPREQ2_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK_MASK                                               0x00003F00L
+//HUBPREQ2_VBLANK_PARAMETERS_1
+#define HUBPREQ2_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L__SHIFT                                    0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L_MASK                                      0x007FFFFFL
+//HUBPREQ2_VBLANK_PARAMETERS_2
+#define HUBPREQ2_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C__SHIFT                                    0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C_MASK                                      0x007FFFFFL
+//HUBPREQ2_VBLANK_PARAMETERS_3
+#define HUBPREQ2_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L__SHIFT                                   0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L_MASK                                     0x007FFFFFL
+//HUBPREQ2_VBLANK_PARAMETERS_4
+#define HUBPREQ2_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C__SHIFT                                   0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C_MASK                                     0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_0
+#define HUBPREQ2_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP__SHIFT                                                  0x0
+#define HUBPREQ2_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP__SHIFT                                                 0x8
+#define HUBPREQ2_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP_MASK                                                    0x0000007FL
+#define HUBPREQ2_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP_MASK                                                   0x00003F00L
+//HUBPREQ2_FLIP_PARAMETERS_1
+#define HUBPREQ2_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L__SHIFT                                        0x0
+#define HUBPREQ2_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L_MASK                                          0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_2
+#define HUBPREQ2_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L__SHIFT                                       0x0
+#define HUBPREQ2_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L_MASK                                         0x007FFFFFL
+//HUBPREQ2_NOM_PARAMETERS_0
+#define HUBPREQ2_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L__SHIFT                                             0x0
+#define HUBPREQ2_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L_MASK                                               0x0001FFFFL
+//HUBPREQ2_NOM_PARAMETERS_1
+#define HUBPREQ2_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L__SHIFT                                          0x0
+#define HUBPREQ2_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L_MASK                                            0x007FFFFFL
+//HUBPREQ2_NOM_PARAMETERS_2
+#define HUBPREQ2_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C__SHIFT                                             0x0
+#define HUBPREQ2_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C_MASK                                               0x0001FFFFL
+//HUBPREQ2_NOM_PARAMETERS_3
+#define HUBPREQ2_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C__SHIFT                                          0x0
+#define HUBPREQ2_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C_MASK                                            0x007FFFFFL
+//HUBPREQ2_NOM_PARAMETERS_4
+#define HUBPREQ2_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L__SHIFT                                            0x0
+#define HUBPREQ2_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L_MASK                                              0x0001FFFFL
+//HUBPREQ2_NOM_PARAMETERS_5
+#define HUBPREQ2_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L__SHIFT                                         0x0
+#define HUBPREQ2_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L_MASK                                           0x007FFFFFL
+//HUBPREQ2_NOM_PARAMETERS_6
+#define HUBPREQ2_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C__SHIFT                                            0x0
+#define HUBPREQ2_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C_MASK                                              0x0001FFFFL
+//HUBPREQ2_NOM_PARAMETERS_7
+#define HUBPREQ2_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C__SHIFT                                         0x0
+#define HUBPREQ2_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C_MASK                                           0x007FFFFFL
+//HUBPREQ2_PER_LINE_DELIVERY_PRE
+#define HUBPREQ2_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L__SHIFT                                 0x0
+#define HUBPREQ2_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C__SHIFT                                 0x10
+#define HUBPREQ2_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L_MASK                                   0x00001FFFL
+#define HUBPREQ2_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C_MASK                                   0x1FFF0000L
+//HUBPREQ2_PER_LINE_DELIVERY
+#define HUBPREQ2_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L__SHIFT                                         0x0
+#define HUBPREQ2_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C__SHIFT                                         0x10
+#define HUBPREQ2_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L_MASK                                           0x00001FFFL
+#define HUBPREQ2_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C_MASK                                           0x1FFF0000L
+//HUBPREQ2_CURSOR_SETTINGS
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET__SHIFT                                                 0x0
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST__SHIFT                                             0x8
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET__SHIFT                                                 0x10
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST__SHIFT                                             0x18
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET_MASK                                                   0x000000FFL
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST_MASK                                               0x00000300L
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET_MASK                                                   0x00FF0000L
+#define HUBPREQ2_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST_MASK                                               0x03000000L
+//HUBPREQ2_REF_FREQ_TO_PIX_FREQ
+#define HUBPREQ2_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ__SHIFT                                            0x0
+#define HUBPREQ2_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ_MASK                                              0x001FFFFFL
+//HUBPREQ2_DST_Y_DELTA_DRQ_LIMIT
+#define HUBPREQ2_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT__SHIFT                                          0x0
+#define HUBPREQ2_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT_MASK                                            0x00007FFFL
+//HUBPREQ2_HUBPREQ_MEM_PWR_CTRL
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE__SHIFT                                          0x0
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS__SHIFT                                            0x2
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE__SHIFT                                          0x4
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS__SHIFT                                            0x6
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE__SHIFT                                          0x8
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS__SHIFT                                            0xa
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE__SHIFT                                           0xc
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS__SHIFT                                             0xe
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE__SHIFT                                          0xf
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS__SHIFT                                            0x11
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE_MASK                                            0x00000003L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS_MASK                                              0x00000004L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE_MASK                                            0x00000030L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS_MASK                                              0x00000040L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE_MASK                                            0x00000300L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS_MASK                                              0x00000400L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE_MASK                                             0x00003000L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS_MASK                                               0x00004000L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE_MASK                                            0x00018000L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS_MASK                                              0x00020000L
+//HUBPREQ2_HUBPREQ_MEM_PWR_STATUS
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE__SHIFT                                        0x0
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE__SHIFT                                        0x2
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE__SHIFT                                        0x4
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE__SHIFT                                         0x6
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE__SHIFT                                        0x8
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE_MASK                                          0x00000003L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE_MASK                                          0x0000000CL
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE_MASK                                          0x00000030L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE_MASK                                           0x000000C0L
+#define HUBPREQ2_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE_MASK                                          0x00000300L
+//HUBPREQ2_VBLANK_PARAMETERS_5
+#define HUBPREQ2_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK__SHIFT                                       0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK_MASK                                         0x007FFFFFL
+//HUBPREQ2_VBLANK_PARAMETERS_6
+#define HUBPREQ2_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK__SHIFT                                         0x0
+#define HUBPREQ2_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK_MASK                                           0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_3
+#define HUBPREQ2_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP__SHIFT                                           0x0
+#define HUBPREQ2_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP_MASK                                             0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_4
+#define HUBPREQ2_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP__SHIFT                                             0x0
+#define HUBPREQ2_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP_MASK                                               0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_5
+#define HUBPREQ2_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C__SHIFT                                        0x0
+#define HUBPREQ2_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C_MASK                                          0x007FFFFFL
+//HUBPREQ2_FLIP_PARAMETERS_6
+#define HUBPREQ2_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C__SHIFT                                       0x0
+#define HUBPREQ2_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C_MASK                                         0x007FFFFFL
+//HUBPREQ2_UCLK_PSTATE_FORCE
+#define HUBPREQ2_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN__SHIFT                                          0x0
+#define HUBPREQ2_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE__SHIFT                                       0x1
+#define HUBPREQ2_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN__SHIFT                                        0x2
+#define HUBPREQ2_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE__SHIFT                                     0x3
+#define HUBPREQ2_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN_MASK                                            0x00000001L
+#define HUBPREQ2_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE_MASK                                         0x00000002L
+#define HUBPREQ2_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN_MASK                                          0x00000004L
+#define HUBPREQ2_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE_MASK                                       0x00000008L
+//HUBPREQ2_HUBPREQ_STATUS_REG0
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0__SHIFT                                         0x0
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1__SHIFT                                         0x8
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT__SHIFT                                                 0x10
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0_MASK                                           0x0000001FL
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1_MASK                                           0x00001F00L
+#define HUBPREQ2_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT_MASK                                                   0x7FFF0000L
+//HUBPREQ2_HUBPREQ_STATUS_REG1
+#define HUBPREQ2_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0__SHIFT                                       0x0
+#define HUBPREQ2_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1__SHIFT                                       0x10
+#define HUBPREQ2_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0_MASK                                         0x00003FFFL
+#define HUBPREQ2_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1_MASK                                         0x3FFF0000L
+//HUBPREQ2_HUBPREQ_STATUS_REG2
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0__SHIFT                                      0x0
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0__SHIFT                                     0x1
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0__SHIFT                               0x2
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0__SHIFT                               0x3
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0__SHIFT                                   0x4
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0__SHIFT                                             0x5
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1__SHIFT                                      0x8
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1__SHIFT                                     0x9
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1__SHIFT                               0xa
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1__SHIFT                               0xb
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1__SHIFT                                   0xc
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1__SHIFT                                             0xd
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR__SHIFT                                     0x10
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR__SHIFT                                    0x11
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR__SHIFT                              0x12
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR__SHIFT                              0x13
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR__SHIFT                                  0x14
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR__SHIFT                                            0x15
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_VBLANK__SHIFT                                                    0x1a
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN__SHIFT                                                   0x1b
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY__SHIFT                                          0x1c
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH__SHIFT                                             0x1d
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0__SHIFT                                            0x1e
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1__SHIFT                                            0x1f
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0_MASK                                        0x00000001L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0_MASK                                       0x00000002L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0_MASK                                 0x00000004L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0_MASK                                 0x00000008L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0_MASK                                     0x00000010L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0_MASK                                               0x00000020L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1_MASK                                        0x00000100L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1_MASK                                       0x00000200L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1_MASK                                 0x00000400L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1_MASK                                 0x00000800L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1_MASK                                     0x00001000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1_MASK                                               0x00002000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR_MASK                                       0x00010000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR_MASK                                      0x00020000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR_MASK                                0x00040000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR_MASK                                0x00080000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR_MASK                                    0x00100000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR_MASK                                              0x00200000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_VBLANK_MASK                                                      0x04000000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN_MASK                                                     0x08000000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY_MASK                                            0x10000000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH_MASK                                               0x20000000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0_MASK                                              0x40000000L
+#define HUBPREQ2_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1_MASK                                              0x80000000L
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpret_dispdec
+//HUBPRET2_HUBPRET_CONTROL
+#define HUBPRET2_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS__SHIFT                                          0x4
+#define HUBPRET2_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE__SHIFT                                            0xf
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA__SHIFT                                                   0x10
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G__SHIFT                                                     0x12
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B__SHIFT                                                    0x14
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R__SHIFT                                                    0x16
+#define HUBPRET2_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE__SHIFT                                                0x18
+#define HUBPRET2_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS_MASK                                            0x00007FF0L
+#define HUBPRET2_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE_MASK                                              0x00008000L
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA_MASK                                                     0x00030000L
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G_MASK                                                       0x000C0000L
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B_MASK                                                      0x00300000L
+#define HUBPRET2_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R_MASK                                                      0x00C00000L
+#define HUBPRET2_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE_MASK                                                  0xFF000000L
+//HUBPRET2_HUBPRET_MEM_PWR_CTRL
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE__SHIFT                                             0x8
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS__SHIFT                                               0xa
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE__SHIFT                                            0x10
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS__SHIFT                                              0x12
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE__SHIFT                                          0x14
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE_MASK                                               0x00000300L
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS_MASK                                                 0x00000400L
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE_MASK                                              0x00030000L
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS_MASK                                                0x00040000L
+#define HUBPRET2_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE_MASK                                            0x00300000L
+//HUBPRET2_HUBPRET_MEM_PWR_STATUS
+#define HUBPRET2_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE__SHIFT                                           0x2
+#define HUBPRET2_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE__SHIFT                                          0x4
+#define HUBPRET2_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE_MASK                                             0x0000000CL
+#define HUBPRET2_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE_MASK                                            0x00000030L
+//HUBPRET2_HUBPRET_READ_LINE_CTRL0
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE__SHIFT                         0x0
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM__SHIFT                                0x10
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE_MASK                           0x0000FFFFL
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM_MASK                                  0x3FFF0000L
+//HUBPRET2_HUBPRET_READ_LINE_CTRL1
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED__SHIFT                    0x0
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE__SHIFT                                0x10
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED_MASK                      0x00003FFFL
+#define HUBPRET2_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE_MASK                                  0xFFFF0000L
+//HUBPRET2_HUBPRET_READ_LINE0
+#define HUBPRET2_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START__SHIFT                                             0x0
+#define HUBPRET2_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END__SHIFT                                               0x10
+#define HUBPRET2_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START_MASK                                               0x00003FFFL
+#define HUBPRET2_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END_MASK                                                 0x3FFF0000L
+//HUBPRET2_HUBPRET_READ_LINE1
+#define HUBPRET2_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START__SHIFT                                             0x0
+#define HUBPRET2_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END__SHIFT                                               0x10
+#define HUBPRET2_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START_MASK                                               0x00003FFFL
+#define HUBPRET2_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END_MASK                                                 0x3FFF0000L
+//HUBPRET2_HUBPRET_INTERRUPT
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK__SHIFT                                               0x0
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK__SHIFT                                           0x1
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK__SHIFT                                           0x2
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE__SHIFT                                               0x4
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE__SHIFT                                           0x5
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE__SHIFT                                           0x6
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR__SHIFT                                              0x8
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR__SHIFT                                          0x9
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR__SHIFT                                          0xa
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS__SHIFT                                                 0xc
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS__SHIFT                                             0xd
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS__SHIFT                                             0xe
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS__SHIFT                                             0x10
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS__SHIFT                                         0x11
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS__SHIFT                                         0x12
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK_MASK                                                 0x00000001L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK_MASK                                             0x00000002L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK_MASK                                             0x00000004L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE_MASK                                                 0x00000010L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE_MASK                                             0x00000020L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE_MASK                                             0x00000040L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR_MASK                                                0x00000100L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR_MASK                                            0x00000200L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR_MASK                                            0x00000400L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS_MASK                                                   0x00001000L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS_MASK                                               0x00002000L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS_MASK                                               0x00004000L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS_MASK                                               0x00010000L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS_MASK                                           0x00020000L
+#define HUBPRET2_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS_MASK                                           0x00040000L
+//HUBPRET2_HUBPRET_READ_LINE_VALUE
+#define HUBPRET2_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE__SHIFT                                               0x0
+#define HUBPRET2_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT__SHIFT                                      0x10
+#define HUBPRET2_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_MASK                                                 0x00003FFFL
+#define HUBPRET2_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT_MASK                                        0x3FFF0000L
+//HUBPRET2_HUBPRET_READ_LINE_STATUS
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK__SHIFT                                            0x0
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE__SHIFT                                      0x4
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE__SHIFT                                     0x5
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE__SHIFT                                      0x8
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE__SHIFT                                     0xa
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK_MASK                                              0x00000001L
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE_MASK                                        0x00000010L
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE_MASK                                       0x00000020L
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE_MASK                                        0x00000100L
+#define HUBPRET2_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE_MASK                                       0x00000400L
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_cursor0_dispdec
+//CURSOR0_2_CURSOR_CONTROL
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_ENABLE__SHIFT                                                        0x0
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_REQ_MODE__SHIFT                                                      0x2
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT                                                    0x4
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_MODE__SHIFT                                                          0x8
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_TMZ__SHIFT                                                           0xc
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PITCH__SHIFT                                                         0x10
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS__SHIFT                     0x14
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK__SHIFT                                               0x18
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN__SHIFT                                    0x1e
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL__SHIFT                                   0x1f
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_ENABLE_MASK                                                          0x00000001L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_REQ_MODE_MASK                                                        0x00000004L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_2X_MAGNIFY_MASK                                                      0x00000010L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_MODE_MASK                                                            0x00000700L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_TMZ_MASK                                                             0x00001000L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PITCH_MASK                                                           0x00030000L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS_MASK                       0x00100000L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK_MASK                                                 0x1F000000L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN_MASK                                      0x40000000L
+#define CURSOR0_2_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL_MASK                                     0x80000000L
+//CURSOR0_2_CURSOR_SURFACE_ADDRESS
+#define CURSOR0_2_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS__SHIFT                                       0x0
+#define CURSOR0_2_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS_MASK                                         0xFFFFFFFFL
+//CURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH
+#define CURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH__SHIFT                             0x0
+#define CURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH_MASK                               0x0000FFFFL
+//CURSOR0_2_CURSOR_SIZE
+#define CURSOR0_2_CURSOR_SIZE__CURSOR_HEIGHT__SHIFT                                                           0x0
+#define CURSOR0_2_CURSOR_SIZE__CURSOR_WIDTH__SHIFT                                                            0x10
+#define CURSOR0_2_CURSOR_SIZE__CURSOR_HEIGHT_MASK                                                             0x000001FFL
+#define CURSOR0_2_CURSOR_SIZE__CURSOR_WIDTH_MASK                                                              0x01FF0000L
+//CURSOR0_2_CURSOR_POSITION
+#define CURSOR0_2_CURSOR_POSITION__CURSOR_Y_POSITION__SHIFT                                                   0x0
+#define CURSOR0_2_CURSOR_POSITION__CURSOR_X_POSITION__SHIFT                                                   0x10
+#define CURSOR0_2_CURSOR_POSITION__CURSOR_Y_POSITION_MASK                                                     0x00003FFFL
+#define CURSOR0_2_CURSOR_POSITION__CURSOR_X_POSITION_MASK                                                     0x3FFF0000L
+//CURSOR0_2_CURSOR_HOT_SPOT
+#define CURSOR0_2_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y__SHIFT                                                   0x0
+#define CURSOR0_2_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X__SHIFT                                                   0x10
+#define CURSOR0_2_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y_MASK                                                     0x000000FFL
+#define CURSOR0_2_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X_MASK                                                     0x00FF0000L
+//CURSOR0_2_CURSOR_STEREO_CONTROL
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN__SHIFT                                              0x0
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET__SHIFT                                         0x4
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET__SHIFT                                       0x12
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN_MASK                                                0x00000001L
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET_MASK                                           0x0003FFF0L
+#define CURSOR0_2_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET_MASK                                         0xFFFC0000L
+//CURSOR0_2_CURSOR_DST_OFFSET
+#define CURSOR0_2_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET__SHIFT                                               0x0
+#define CURSOR0_2_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET_MASK                                                 0x00001FFFL
+//CURSOR0_2_CURSOR_MEM_PWR_CTRL
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE__SHIFT                                              0x0
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS__SHIFT                                                0x2
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE__SHIFT                                            0x4
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE_MASK                                                0x00000003L
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS_MASK                                                  0x00000004L
+#define CURSOR0_2_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE_MASK                                              0x00000030L
+//CURSOR0_2_CURSOR_MEM_PWR_STATUS
+#define CURSOR0_2_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE__SHIFT                                            0x0
+#define CURSOR0_2_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE_MASK                                              0x00000003L
+//CURSOR0_2_DMDATA_ADDRESS_HIGH
+#define CURSOR0_2_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH__SHIFT                                             0x0
+#define CURSOR0_2_DMDATA_ADDRESS_HIGH__DMDATA_TMZ__SHIFT                                                      0x1e
+#define CURSOR0_2_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH_MASK                                               0x0000FFFFL
+#define CURSOR0_2_DMDATA_ADDRESS_HIGH__DMDATA_TMZ_MASK                                                        0x40000000L
+//CURSOR0_2_DMDATA_ADDRESS_LOW
+#define CURSOR0_2_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW__SHIFT                                               0x0
+#define CURSOR0_2_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW_MASK                                                 0xFFFFFFFFL
+//CURSOR0_2_DMDATA_CNTL
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_UPDATED__SHIFT                                                          0x0
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_REPEAT__SHIFT                                                           0x1
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_MODE__SHIFT                                                             0x2
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_SIZE__SHIFT                                                             0x10
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_UPDATED_MASK                                                            0x00000001L
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_REPEAT_MASK                                                             0x00000002L
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_MODE_MASK                                                               0x00000004L
+#define CURSOR0_2_DMDATA_CNTL__DMDATA_SIZE_MASK                                                               0x0FFF0000L
+//CURSOR0_2_DMDATA_QOS_CNTL
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_QOS_MODE__SHIFT                                                     0x0
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL__SHIFT                                                    0x4
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_DL_DELTA__SHIFT                                                     0x10
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_QOS_MODE_MASK                                                       0x00000001L
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL_MASK                                                      0x000000F0L
+#define CURSOR0_2_DMDATA_QOS_CNTL__DMDATA_DL_DELTA_MASK                                                       0xFFFF0000L
+//CURSOR0_2_DMDATA_STATUS
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_DONE__SHIFT                                                           0x0
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_UNDERFLOW__SHIFT                                                      0x2
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR__SHIFT                                                0x4
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_DONE_MASK                                                             0x00000001L
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_UNDERFLOW_MASK                                                        0x00000004L
+#define CURSOR0_2_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR_MASK                                                  0x00000010L
+//CURSOR0_2_DMDATA_SW_CNTL
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_UPDATED__SHIFT                                                    0x0
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_REPEAT__SHIFT                                                     0x1
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_SIZE__SHIFT                                                       0x10
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_UPDATED_MASK                                                      0x00000001L
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_REPEAT_MASK                                                       0x00000002L
+#define CURSOR0_2_DMDATA_SW_CNTL__DMDATA_SW_SIZE_MASK                                                         0x0FFF0000L
+//CURSOR0_2_DMDATA_SW_DATA
+#define CURSOR0_2_DMDATA_SW_DATA__DMDATA_SW_DATA__SHIFT                                                       0x0
+#define CURSOR0_2_DMDATA_SW_DATA__DMDATA_SW_DATA_MASK                                                         0xFFFFFFFFL
+//CURSOR0_2_HUBP_3DLUT_CONTROL
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE__SHIFT                                                0x0
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE__SHIFT                                       0x1
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH__SHIFT                                                 0x2
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ__SHIFT                                                   0x12
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B__SHIFT                                        0x16
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G__SHIFT                                        0x18
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R__SHIFT                                        0x1a
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS__SHIFT                                         0x1d
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH__SHIFT                                             0x1e
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE__SHIFT                                                  0x1f
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE_MASK                                                  0x00000001L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE_MASK                                         0x00000002L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH_MASK                                                   0x0003FFFCL
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ_MASK                                                     0x003C0000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B_MASK                                          0x00C00000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G_MASK                                          0x03000000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R_MASK                                          0x0C000000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS_MASK                                           0x20000000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH_MASK                                               0x40000000L
+#define CURSOR0_2_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE_MASK                                                    0x80000000L
+//CURSOR0_2_HUBP_3DLUT_ADDRESS_LOW
+#define CURSOR0_2_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW__SHIFT                                       0x0
+#define CURSOR0_2_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW_MASK                                         0xFFFFFFFFL
+//CURSOR0_2_HUBP_3DLUT_ADDRESS_HIGH
+#define CURSOR0_2_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH__SHIFT                                     0x0
+#define CURSOR0_2_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH_MASK                                       0x0000FFFFL
+//CURSOR0_2_HUBP_3DLUT_DLG_PARAM
+#define CURSOR0_2_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP__SHIFT                                         0x0
+#define CURSOR0_2_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP_MASK                                           0x007FFFFFL
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON8_PERFCOUNTER_CNTL
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON8_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON8_PERFCOUNTER_CNTL2
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON8_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON8_PERFCOUNTER_STATE
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON8_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON8_PERFMON_CNTL
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON8_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON8_PERFMON_CNTL2
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON8_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON8_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON8_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON8_PERFMON_CVALUE_LOW
+#define DC_PERFMON8_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON8_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON8_PERFMON_HI
+#define DC_PERFMON8_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON8_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON8_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON8_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON8_PERFMON_LOW
+#define DC_PERFMON8_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON8_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dispdec
+//HUBP3_DCSURF_SURFACE_CONFIG
+#define HUBP3_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT__SHIFT                                              0x0
+#define HUBP3_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE__SHIFT                                                    0x8
+#define HUBP3_DCSURF_SURFACE_CONFIG__H_MIRROR_EN__SHIFT                                                       0xa
+#define HUBP3_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN__SHIFT                                                    0xb
+#define HUBP3_DCSURF_SURFACE_CONFIG__SURFACE_PIXEL_FORMAT_MASK                                                0x0000007FL
+#define HUBP3_DCSURF_SURFACE_CONFIG__ROTATION_ANGLE_MASK                                                      0x00000300L
+#define HUBP3_DCSURF_SURFACE_CONFIG__H_MIRROR_EN_MASK                                                         0x00000400L
+#define HUBP3_DCSURF_SURFACE_CONFIG__ALPHA_PLANE_EN_MASK                                                      0x00000800L
+//HUBP3_DCSURF_ADDR_CONFIG
+#define HUBP3_DCSURF_ADDR_CONFIG__NUM_PIPES__SHIFT                                                            0x0
+#define HUBP3_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE__SHIFT                                                      0x6
+#define HUBP3_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                 0xc
+#define HUBP3_DCSURF_ADDR_CONFIG__NUM_PKRS__SHIFT                                                             0x10
+#define HUBP3_DCSURF_ADDR_CONFIG__NUM_PIPES_MASK                                                              0x00000007L
+#define HUBP3_DCSURF_ADDR_CONFIG__PIPE_INTERLEAVE_MASK                                                        0x000000C0L
+#define HUBP3_DCSURF_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                   0x00003000L
+#define HUBP3_DCSURF_ADDR_CONFIG__NUM_PKRS_MASK                                                               0x00070000L
+//HUBP3_DCSURF_TILING_CONFIG
+#define HUBP3_DCSURF_TILING_CONFIG__SW_MODE__SHIFT                                                            0x0
+#define HUBP3_DCSURF_TILING_CONFIG__DIM_TYPE__SHIFT                                                           0x7
+#define HUBP3_DCSURF_TILING_CONFIG__META_LINEAR__SHIFT                                                        0x9
+#define HUBP3_DCSURF_TILING_CONFIG__PIPE_ALIGNED__SHIFT                                                       0xb
+#define HUBP3_DCSURF_TILING_CONFIG__SW_MODE_MASK                                                              0x0000001FL
+#define HUBP3_DCSURF_TILING_CONFIG__DIM_TYPE_MASK                                                             0x00000180L
+#define HUBP3_DCSURF_TILING_CONFIG__META_LINEAR_MASK                                                          0x00000200L
+#define HUBP3_DCSURF_TILING_CONFIG__PIPE_ALIGNED_MASK                                                         0x00000800L
+//HUBP3_DCSURF_PRI_VIEWPORT_START
+#define HUBP3_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP3_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP3_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP3_DCSURF_PRI_VIEWPORT_START__PRI_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP3_DCSURF_PRI_VIEWPORT_START_C
+#define HUBP3_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP3_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP3_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP3_DCSURF_PRI_VIEWPORT_START_C__PRI_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C__PRI_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP3_DCSURF_SEC_VIEWPORT_START
+#define HUBP3_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START__SHIFT                                          0x0
+#define HUBP3_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START__SHIFT                                          0x10
+#define HUBP3_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_X_START_MASK                                            0x00003FFFL
+#define HUBP3_DCSURF_SEC_VIEWPORT_START__SEC_VIEWPORT_Y_START_MASK                                            0x3FFF0000L
+//HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH__SHIFT                                        0x0
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT__SHIFT                                       0x10
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION__SEC_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
+//HUBP3_DCSURF_SEC_VIEWPORT_START_C
+#define HUBP3_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C__SHIFT                                      0x0
+#define HUBP3_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C__SHIFT                                      0x10
+#define HUBP3_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_X_START_C_MASK                                        0x00003FFFL
+#define HUBP3_DCSURF_SEC_VIEWPORT_START_C__SEC_VIEWPORT_Y_START_C_MASK                                        0x3FFF0000L
+//HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C__SHIFT                                    0x0
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C__SHIFT                                   0x10
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_WIDTH_C_MASK                                      0x00003FFFL
+#define HUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C__SEC_VIEWPORT_HEIGHT_C_MASK                                     0x3FFF0000L
+//HUBP3_DCHUBP_REQ_SIZE_CONFIG
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT__SHIFT                                                     0x0
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR__SHIFT                                            0x4
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE__SHIFT                                                       0x8
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE__SHIFT                                                   0xb
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE__SHIFT                                                  0x10
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE__SHIFT                                              0x12
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE__SHIFT                                                  0x14
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE__SHIFT                                                    0x18
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__SWATH_HEIGHT_MASK                                                       0x00000007L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__PTE_ROW_HEIGHT_LINEAR_MASK                                              0x00000070L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__CHUNK_SIZE_MASK                                                         0x00000700L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__MIN_CHUNK_SIZE_MASK                                                     0x00001800L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__META_CHUNK_SIZE_MASK                                                    0x00030000L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__MIN_META_CHUNK_SIZE_MASK                                                0x000C0000L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__DPTE_GROUP_SIZE_MASK                                                    0x00700000L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG__VM_GROUP_SIZE_MASK                                                      0x07000000L
+//HUBP3_DCHUBP_REQ_SIZE_CONFIG_C
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C__SHIFT                                                 0x0
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C__SHIFT                                        0x4
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C__SHIFT                                                   0x8
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C__SHIFT                                               0xb
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C__SHIFT                                              0x10
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C__SHIFT                                          0x12
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C__SHIFT                                              0x14
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__SWATH_HEIGHT_C_MASK                                                   0x00000007L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__PTE_ROW_HEIGHT_LINEAR_C_MASK                                          0x00000070L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__CHUNK_SIZE_C_MASK                                                     0x00000700L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__MIN_CHUNK_SIZE_C_MASK                                                 0x00001800L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__META_CHUNK_SIZE_C_MASK                                                0x00030000L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__MIN_META_CHUNK_SIZE_C_MASK                                            0x000C0000L
+#define HUBP3_DCHUBP_REQ_SIZE_CONFIG_C__DPTE_GROUP_SIZE_C_MASK                                                0x00700000L
+//HUBP3_DCHUBP_CNTL
+#define HUBP3_DCHUBP_CNTL__HUBP_BLANK_EN__SHIFT                                                               0x0
+#define HUBP3_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ__SHIFT                                                     0x1
+#define HUBP3_DCHUBP_CNTL__HUBP_SOFT_RESET__SHIFT                                                             0x2
+#define HUBP3_DCHUBP_CNTL__HUBP_IN_BLANK__SHIFT                                                               0x3
+#define HUBP3_DCHUBP_CNTL__HUBP_VTG_SEL__SHIFT                                                                0x4
+#define HUBP3_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC__SHIFT                                               0x8
+#define HUBP3_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM__SHIFT                                            0x9
+#define HUBP3_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE__SHIFT                                                     0xa
+#define HUBP3_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS__SHIFT                                                   0xb
+#define HUBP3_DCHUBP_CNTL__HUBP_TTU_DISABLE__SHIFT                                                            0xc
+#define HUBP3_DCHUBP_CNTL__HUBP_TTU_MODE__SHIFT                                                               0xd
+#define HUBP3_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ__SHIFT                                                 0x10
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS__SHIFT                                                         0x14
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD__SHIFT                                                      0x18
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR__SHIFT                                                   0x1a
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN__SHIFT                                                   0x1b
+#define HUBP3_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS__SHIFT                                                       0x1c
+#define HUBP3_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR__SHIFT                                                        0x1f
+#define HUBP3_DCHUBP_CNTL__HUBP_BLANK_EN_MASK                                                                 0x00000001L
+#define HUBP3_DCHUBP_CNTL__HUBP_NO_OUTSTANDING_REQ_MASK                                                       0x00000002L
+#define HUBP3_DCHUBP_CNTL__HUBP_SOFT_RESET_MASK                                                               0x00000004L
+#define HUBP3_DCHUBP_CNTL__HUBP_IN_BLANK_MASK                                                                 0x00000008L
+#define HUBP3_DCHUBP_CNTL__HUBP_VTG_SEL_MASK                                                                  0x000000F0L
+#define HUBP3_DCHUBP_CNTL__HUBP_VREADY_AT_OR_AFTER_VSYNC_MASK                                                 0x00000100L
+#define HUBP3_DCHUBP_CNTL__HUBP_DISABLE_STOP_DATA_DURING_VM_MASK                                              0x00000200L
+#define HUBP3_DCHUBP_CNTL__HUBP_UNBOUNDED_REQ_MODE_MASK                                                       0x00000400L
+#define HUBP3_DCHUBP_CNTL__HUBP_SEG_ALLOC_ERR_STATUS_MASK                                                     0x00000800L
+#define HUBP3_DCHUBP_CNTL__HUBP_TTU_DISABLE_MASK                                                              0x00001000L
+#define HUBP3_DCHUBP_CNTL__HUBP_TTU_MODE_MASK                                                                 0x0000E000L
+#define HUBP3_DCHUBP_CNTL__HUBP_XRQ_NO_OUTSTANDING_REQ_MASK                                                   0x000F0000L
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_MASK                                                           0x00F00000L
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_THRESHOLD_MASK                                                        0x03000000L
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_STATUS_CLEAR_MASK                                                     0x04000000L
+#define HUBP3_DCHUBP_CNTL__HUBP_TIMEOUT_INTERRUPT_EN_MASK                                                     0x08000000L
+#define HUBP3_DCHUBP_CNTL__HUBP_UNDERFLOW_STATUS_MASK                                                         0x70000000L
+#define HUBP3_DCHUBP_CNTL__HUBP_UNDERFLOW_CLEAR_MASK                                                          0x80000000L
+//HUBP3_HUBP_CLK_CNTL
+#define HUBP3_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE__SHIFT                                                         0x0
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS__SHIFT                                                   0x5
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS__SHIFT                                                    0x8
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS__SHIFT                                                    0xc
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS__SHIFT                                                    0x10
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON__SHIFT                                                   0x14
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON__SHIFT                                                    0x15
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON__SHIFT                                                    0x16
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON__SHIFT                                                    0x17
+#define HUBP3_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON__SHIFT                                                   0x19
+#define HUBP3_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define HUBP3_HUBP_CLK_CNTL__HUBP_CLOCK_ENABLE_MASK                                                           0x00000001L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_G_GATE_DIS_MASK                                                     0x00000020L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DPPCLK_G_GATE_DIS_MASK                                                      0x00000100L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_R_GATE_DIS_MASK                                                      0x00001000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_G_GATE_DIS_MASK                                                      0x00010000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_R_CLOCK_ON_MASK                                                     0x00100000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DPPCLK_G_CLOCK_ON_MASK                                                      0x00200000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_R_CLOCK_ON_MASK                                                      0x00400000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DCFCLK_G_CLOCK_ON_MASK                                                      0x00800000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_DISPCLK_G_CLOCK_ON_MASK                                                     0x02000000L
+#define HUBP3_HUBP_CLK_CNTL__HUBP_TEST_CLK_SEL_MASK                                                           0xF0000000L
+//HUBP3_DCHUBP_VMPG_CONFIG
+#define HUBP3_DCHUBP_VMPG_CONFIG__VMPG_SIZE__SHIFT                                                            0x0
+#define HUBP3_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE__SHIFT                                                      0x1
+#define HUBP3_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE__SHIFT                                                   0x2
+#define HUBP3_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME__SHIFT                                              0x7
+#define HUBP3_DCHUBP_VMPG_CONFIG__VMPG_SIZE_MASK                                                              0x00000001L
+#define HUBP3_DCHUBP_VMPG_CONFIG__PTE_BUFFER_MODE_MASK                                                        0x00000002L
+#define HUBP3_DCHUBP_VMPG_CONFIG__BIGK_FRAGMENT_SIZE_MASK                                                     0x0000007CL
+#define HUBP3_DCHUBP_VMPG_CONFIG__FORCE_ONE_ROW_FOR_FRAME_MASK                                                0x00000080L
+//HUBP3_DCHUBP_MALL_CONFIG
+#define HUBP3_DCHUBP_MALL_CONFIG__USE_MALL_SEL__SHIFT                                                         0x0
+#define HUBP3_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR__SHIFT                                                  0x2
+#define HUBP3_DCHUBP_MALL_CONFIG__USE_MALL_SEL_MASK                                                           0x00000003L
+#define HUBP3_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR_MASK                                                    0x00000004L
+//HUBP3_DCHUBP_MALL_SUB_VP
+#define HUBP3_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE__SHIFT                                               0x0
+#define HUBP3_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0__SHIFT                                                 0x1
+#define HUBP3_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1__SHIFT                                                 0xf
+#define HUBP3_DCHUBP_MALL_SUB_VP__USE_MALL_AT_START_LINE_MASK                                                 0x00000001L
+#define HUBP3_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S0_MASK                                                   0x00007FFEL
+#define HUBP3_DCHUBP_MALL_SUB_VP__SUB_VP_START_LINE_S1_MASK                                                   0x1FFF8000L
+//HUBP3_HUBPREQ_DEBUG_DB
+#define HUBP3_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG__SHIFT                                                          0x0
+#define HUBP3_HUBPREQ_DEBUG_DB__HUBPREQ_DEBUG_MASK                                                            0xFFFFFFFFL
+//HUBP3_HUBPREQ_DEBUG
+#define HUBP3_HUBPREQ_DEBUG__HUBPREQ_DEBUG__SHIFT                                                             0x0
+#define HUBP3_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS__SHIFT                                 0x1f
+#define HUBP3_HUBPREQ_DEBUG__HUBPREQ_DEBUG_MASK                                                               0x7FFFFFFFL
+#define HUBP3_HUBPREQ_DEBUG__HUBPREQ_DEBUG_FLIP_REQ_DURING_MALL_STATUS_MASK                                   0x80000000L
+//HUBP3_HUBP_DEBUG_CTRL
+#define HUBP3_HUBP_DEBUG_CTRL__HUBP_DBG_EN__SHIFT                                                             0x0
+#define HUBP3_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS__SHIFT                                              0x4
+#define HUBP3_HUBP_DEBUG_CTRL__HUBP_DBG_EN_MASK                                                               0x00000001L
+#define HUBP3_HUBP_DEBUG_CTRL__HUBP_DBG_HUBP_DCFCLK_G_DIS_MASK                                                0x00000010L
+//HUBP3_HUBP_DEBUG_MUX_DCFCLK
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK__SHIFT                                         0x0
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK__SHIFT                                         0x8
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK__SHIFT                                         0x10
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK__SHIFT                                         0x18
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL0_DCFCLK_MASK                                           0x000000FFL
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL1_DCFCLK_MASK                                           0x0000FF00L
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL2_DCFCLK_MASK                                           0x00FF0000L
+#define HUBP3_HUBP_DEBUG_MUX_DCFCLK__HUBP_DBG_DATA_SEL3_DCFCLK_MASK                                           0xFF000000L
+//HUBP3_HUBP_DEBUG_MUX_DPPCLK
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK__SHIFT                                         0x0
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK__SHIFT                                         0x4
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK__SHIFT                                        0x8
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DPPCLK_MASK                                           0x00000007L
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL1_DPPCLK_MASK                                           0x00000070L
+#define HUBP3_HUBP_DEBUG_MUX_DPPCLK__HUBP_DBG_DATA_SEL0_DISPCLK_MASK                                          0x00000700L
+//HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK__SHIFT                                 0x0
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK__SHIFT                          0x4
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK__SHIFT                              0xc
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK__SHIFT                               0x14
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK__SHIFT                               0x1c
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_EN_DCFCLK_MASK                                   0x00000001L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_PERIOD_M1_DCFCLK_MASK                            0x00000FF0L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_START_SEL_DCFCLK_MASK                                0x0001F000L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_PERFMON_STOP_SEL_DCFCLK_MASK                                 0x01F00000L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK__HUBP_MEASURE_WIN_MODE_DCFCLK_MASK                                 0x30000000L
+//HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK__SHIFT                                 0x0
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK__SHIFT                            0x1
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK__SHIFT                          0x4
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK__SHIFT                              0xc
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK__SHIFT                               0x14
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_EN_DPPCLK_MASK                                   0x00000001L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_SRC_SEL_DPPCLK_MASK                              0x00000002L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_MEASURE_WIN_PERIOD_M1_DPPCLK_MASK                            0x00000FF0L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_START_SEL_DPPCLK_MASK                                0x0001F000L
+#define HUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK__HUBP_PERFMON_STOP_SEL_DPPCLK_MASK                                 0x01F00000L
+//HUBP3_HUBP_MALL_STATUS
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN__SHIFT                                         0x0
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE__SHIFT                                         0x1
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQUEST__SHIFT                                                       0x2
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_RESPONSE__SHIFT                                                      0x3
+#define HUBP3_HUBP_MALL_STATUS__MALL_IN_USE__SHIFT                                                            0x4
+#define HUBP3_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE__SHIFT                                                 0x5
+#define HUBP3_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE__SHIFT                                                   0x6
+#define HUBP3_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE__SHIFT                                                  0x7
+#define HUBP3_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE__SHIFT                                                  0x8
+#define HUBP3_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH__SHIFT                                                  0x9
+#define HUBP3_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME__SHIFT                                                    0xa
+#define HUBP3_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME__SHIFT                                                    0xb
+#define HUBP3_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL__SHIFT                                                     0xc
+#define HUBP3_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL__SHIFT                                                     0xd
+#define HUBP3_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL__SHIFT                                                     0xe
+#define HUBP3_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME__SHIFT                                                  0xf
+#define HUBP3_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS__SHIFT                                   0x10
+#define HUBP3_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING__SHIFT                                            0x11
+#define HUBP3_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING__SHIFT                                                   0x12
+#define HUBP3_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO__SHIFT                                                      0x13
+#define HUBP3_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x14
+#define HUBP3_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP__SHIFT                                            0x15
+#define HUBP3_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING__SHIFT                                                   0x16
+#define HUBP3_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP__SHIFT                                              0x17
+#define HUBP3_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING__SHIFT                                                   0x18
+#define HUBP3_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING__SHIFT                                                  0x19
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_STATIC_SCREEN_MASK                                           0x00000001L
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQ_FOR_PSTATE_CHANGE_MASK                                           0x00000002L
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_REQUEST_MASK                                                         0x00000004L
+#define HUBP3_HUBP_MALL_STATUS__MALL_USE_RESPONSE_MASK                                                        0x00000008L
+#define HUBP3_HUBP_MALL_STATUS__MALL_IN_USE_MASK                                                              0x00000010L
+#define HUBP3_HUBP_MALL_STATUS__MALL_PREFETCH_COMPLETE_MASK                                                   0x00000020L
+#define HUBP3_HUBP_MALL_STATUS__SUB_VP_MALL_RETRIEVE_MASK                                                     0x00000040L
+#define HUBP3_HUBP_MALL_STATUS__MCB_MALL_USE_RESPONSE_MASK                                                    0x00000080L
+#define HUBP3_HUBP_MALL_STATUS__CURSOR_LOCAL_RETRIEVE_MASK                                                    0x00000100L
+#define HUBP3_HUBP_MALL_STATUS__CURSOR_LOCAL_PREFETCH_MASK                                                    0x00000200L
+#define HUBP3_HUBP_MALL_STATUS__MALL_RETRIEVE_FRAME_MASK                                                      0x00000400L
+#define HUBP3_HUBP_MALL_STATUS__MALL_PREFETCH_FRAME_MASK                                                      0x00000800L
+#define HUBP3_HUBP_MALL_STATUS__CRQ_BUSY_WITH_MALL_MASK                                                       0x00001000L
+#define HUBP3_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_MASK                                                       0x00002000L
+#define HUBP3_HUBP_MALL_STATUS__DRQ_BUSY_WITH_MALL_MASK                                                       0x00004000L
+#define HUBP3_HUBP_MALL_STATUS__USE_ONE_ROW_FOR_FRAME_MASK                                                    0x00008000L
+#define HUBP3_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_RETRIEVE_IN_PROGRESS_MASK                                     0x00010000L
+#define HUBP3_HUBP_MALL_STATUS__DRQ_SUB_VP_MALL_OUTSTANDING_MASK                                              0x00020000L
+#define HUBP3_HUBP_MALL_STATUS__DRQ_MALL_OUTSTANDING_MASK                                                     0x00040000L
+#define HUBP3_HUBP_MALL_STATUS__DRQ_MALL_CNT_ZERO_MASK                                                        0x00080000L
+#define HUBP3_HUBP_MALL_STATUS__MRQ_S1_MALL_RETRIEVE_SUB_VP_MASK                                              0x00100000L
+#define HUBP3_HUBP_MALL_STATUS__MRQ_S0_MALL_RETRIEVE_SUB_VP_MASK                                              0x00200000L
+#define HUBP3_HUBP_MALL_STATUS__MRQ_MALL_OUTSTANDING_MASK                                                     0x00400000L
+#define HUBP3_HUBP_MALL_STATUS__MRQ_BUSY_WITH_MALL_SUB_VP_MASK                                                0x00800000L
+#define HUBP3_HUBP_MALL_STATUS__CRQ_MALL_OUTSTANDING_MASK                                                     0x01000000L
+#define HUBP3_HUBP_MALL_STATUS__CRQ_LOCAL_OUTSTANDING_MASK                                                    0x02000000L
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpreq_dispdec
+//HUBPREQ3_DCSURF_SURFACE_PITCH
+#define HUBPREQ3_DCSURF_SURFACE_PITCH__PITCH__SHIFT                                                           0x0
+#define HUBPREQ3_DCSURF_SURFACE_PITCH__META_PITCH__SHIFT                                                      0x10
+#define HUBPREQ3_DCSURF_SURFACE_PITCH__PITCH_MASK                                                             0x00003FFFL
+#define HUBPREQ3_DCSURF_SURFACE_PITCH__META_PITCH_MASK                                                        0x3FFF0000L
+//HUBPREQ3_DCSURF_SURFACE_PITCH_C
+#define HUBPREQ3_DCSURF_SURFACE_PITCH_C__PITCH_C__SHIFT                                                       0x0
+#define HUBPREQ3_DCSURF_SURFACE_PITCH_C__META_PITCH_C__SHIFT                                                  0x10
+#define HUBPREQ3_DCSURF_SURFACE_PITCH_C__PITCH_C_MASK                                                         0x00003FFFL
+#define HUBPREQ3_DCSURF_SURFACE_PITCH_C__META_PITCH_C_MASK                                                    0x3FFF0000L
+//HUBPREQ3_VMID_SETTINGS_0
+#define HUBPREQ3_VMID_SETTINGS_0__VMID__SHIFT                                                                 0x0
+#define HUBPREQ3_VMID_SETTINGS_0__VMID_MASK                                                                   0x0000000FL
+//HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS__SHIFT                               0x0
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS__PRIMARY_SURFACE_ADDRESS_MASK                                 0xFFFFFFFFL
+//HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH__SHIFT                     0x0
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH__PRIMARY_SURFACE_ADDRESS_HIGH_MASK                       0x0000FFFFL
+//HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C__SHIFT                           0x0
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C__PRIMARY_SURFACE_ADDRESS_C_MASK                             0xFFFFFFFFL
+//HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C__SHIFT                 0x0
+#define HUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C__PRIMARY_SURFACE_ADDRESS_HIGH_C_MASK                   0x0000FFFFL
+//HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS__SHIFT                           0x0
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS__SECONDARY_SURFACE_ADDRESS_MASK                             0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT                 0x0
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH__SECONDARY_SURFACE_ADDRESS_HIGH_MASK                   0x0000FFFFL
+//HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C__SHIFT                       0x0
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C__SECONDARY_SURFACE_ADDRESS_C_MASK                         0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C__SHIFT             0x0
+#define HUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C__SECONDARY_SURFACE_ADDRESS_HIGH_C_MASK               0x0000FFFFL
+//HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS__SHIFT                     0x0
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS__PRIMARY_META_SURFACE_ADDRESS_MASK                       0xFFFFFFFFL
+//HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH__SHIFT           0x0
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH__PRIMARY_META_SURFACE_ADDRESS_HIGH_MASK             0x0000FFFFL
+//HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C__SHIFT                 0x0
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C__PRIMARY_META_SURFACE_ADDRESS_C_MASK                   0xFFFFFFFFL
+//HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT       0x0
+#define HUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C__PRIMARY_META_SURFACE_ADDRESS_HIGH_C_MASK         0x0000FFFFL
+//HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS__SHIFT                 0x0
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS__SECONDARY_META_SURFACE_ADDRESS_MASK                   0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH__SHIFT       0x0
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH__SECONDARY_META_SURFACE_ADDRESS_HIGH_MASK         0x0000FFFFL
+//HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C__SHIFT             0x0
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C__SECONDARY_META_SURFACE_ADDRESS_C_MASK               0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SHIFT   0x0
+#define HUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C__SECONDARY_META_SURFACE_ADDRESS_HIGH_C_MASK     0x0000FFFFL
+//HUBPREQ3_DCSURF_SURFACE_CONTROL
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ__SHIFT                                           0x0
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN__SHIFT                                        0x4
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK__SHIFT                                   0x5
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C__SHIFT                                         0x7
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C__SHIFT                                 0xb
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ__SHIFT                                         0xd
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN__SHIFT                                      0x11
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK__SHIFT                                 0x12
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C__SHIFT                                       0x14
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C__SHIFT                               0x18
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ__SHIFT                                      0x1a
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C__SHIFT                                    0x1b
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ__SHIFT                                    0x1c
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C__SHIFT                                  0x1d
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_MASK                                             0x0000000FL
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_EN_MASK                                          0x00000010L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_MASK                                     0x00000060L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_TMZ_C_MASK                                           0x00000780L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_SURFACE_DCC_IND_BLK_C_MASK                                   0x00001800L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_MASK                                           0x0001E000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_EN_MASK                                        0x00020000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_MASK                                   0x000C0000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_TMZ_C_MASK                                         0x00F00000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_SURFACE_DCC_IND_BLK_C_MASK                                 0x03000000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_MASK                                        0x04000000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__PRIMARY_META_SURFACE_TMZ_C_MASK                                      0x08000000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_MASK                                      0x10000000L
+#define HUBPREQ3_DCSURF_SURFACE_CONTROL__SECONDARY_META_SURFACE_TMZ_C_MASK                                    0x20000000L
+//HUBPREQ3_DCSURF_FLIP_CONTROL
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK__SHIFT                                              0x0
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE__SHIFT                                                0x1
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM__SHIFT                                    0x4
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING__SHIFT                                             0x8
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS__SHIFT                                0x9
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC__SHIFT                                 0xc
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC__SHIFT                                       0x10
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE__SHIFT                               0x11
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY__SHIFT                              0x12
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY__SHIFT                                       0x14
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_UPDATE_LOCK_MASK                                                0x00000001L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_TYPE_MASK                                                  0x00000002L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_VUPDATE_SKIP_NUM_MASK                                      0x000000F0L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_MASK                                               0x00000100L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__HUBPREQ_MASTER_UPDATE_LOCK_STATUS_MASK                                  0x00000200L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_MODE_FOR_STEREOSYNC_MASK                                   0x00003000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_IN_STEREOSYNC_MASK                                         0x00010000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_DISABLE_MASK                                 0x00020000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_STEREO_SELECT_POLARITY_MASK                                0x00040000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL__SURFACE_FLIP_PENDING_DELAY_MASK                                         0x3FF00000L
+//HUBPREQ3_DCSURF_FLIP_CONTROL2
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME__SHIFT                                   0x0
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE__SHIFT                                              0x8
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK__SHIFT                                                0x9
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE__SHIFT                                    0xa
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH__SHIFT                                     0xc
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE__SHIFT                                        0x1c
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE__SHIFT                                    0x1f
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_PENDING_MIN_TIME_MASK                                     0x000000FFL
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_GSL_ENABLE_MASK                                                0x00000100L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_GSL_MASK_MASK                                                  0x00000200L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_TRIPLE_BUFFER_ENABLE_MASK                                      0x00000400L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_INUSE_RAED_NO_LATCH_MASK                                       0x00001000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__IMME_FLIP_BUF_TRACK_MODE_MASK                                          0x10000000L
+#define HUBPREQ3_DCSURF_FLIP_CONTROL2__SURFACE_FLIP_EXEC_DEBUG_MODE_MASK                                      0x80000000L
+//HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK__SHIFT                                  0x0
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE__SHIFT                                  0x1
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK__SHIFT                             0x2
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE__SHIFT                             0x3
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR__SHIFT                                     0x8
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR__SHIFT                                0x9
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED__SHIFT                                  0x10
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS__SHIFT                                0x11
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED__SHIFT                             0x12
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS__SHIFT                           0x13
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_MASK_MASK                                    0x00000001L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_TYPE_MASK                                    0x00000002L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_MASK_MASK                               0x00000004L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_TYPE_MASK                               0x00000008L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_CLEAR_MASK                                       0x00000100L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_CLEAR_MASK                                  0x00000200L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_OCCURRED_MASK                                    0x00010000L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_INT_STATUS_MASK                                  0x00020000L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_OCCURRED_MASK                               0x00040000L
+#define HUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT__SURFACE_FLIP_AWAY_INT_STATUS_MASK                             0x00080000L
+//HUBPREQ3_DCSURF_SURFACE_INUSE
+#define HUBPREQ3_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS__SHIFT                                           0x0
+#define HUBPREQ3_DCSURF_SURFACE_INUSE__SURFACE_INUSE_ADDRESS_MASK                                             0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH__SHIFT                                 0x0
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID__SHIFT                                         0x1c
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_ADDRESS_HIGH_MASK                                   0x0000FFFFL
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH__SURFACE_INUSE_VMID_MASK                                           0xF0000000L
+//HUBPREQ3_DCSURF_SURFACE_INUSE_C
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C__SHIFT                                       0x0
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_C__SURFACE_INUSE_ADDRESS_C_MASK                                         0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C__SHIFT                             0x0
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C__SHIFT                                     0x1c
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_ADDRESS_HIGH_C_MASK                               0x0000FFFFL
+#define HUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C__SURFACE_INUSE_VMID_C_MASK                                       0xF0000000L
+//HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS__SHIFT                         0x0
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE__SURFACE_EARLIEST_INUSE_ADDRESS_MASK                           0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH__SHIFT               0x0
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID__SHIFT                       0x1c
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_MASK                 0x0000FFFFL
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH__SURFACE_EARLIEST_INUSE_VMID_MASK                         0xF0000000L
+//HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C__SHIFT                     0x0
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C__SURFACE_EARLIEST_INUSE_ADDRESS_C_MASK                       0xFFFFFFFFL
+//HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C__SHIFT           0x0
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C__SHIFT                   0x1c
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C_MASK             0x0000FFFFL
+#define HUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C__SURFACE_EARLIEST_INUSE_VMID_C_MASK                     0xF0000000L
+//HUBPREQ3_DCN_EXPANSION_MODE
+#define HUBPREQ3_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE__SHIFT                                                0x0
+#define HUBPREQ3_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE__SHIFT                                                0x2
+#define HUBPREQ3_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE__SHIFT                                                0x4
+#define HUBPREQ3_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE__SHIFT                                                0x6
+#define HUBPREQ3_DCN_EXPANSION_MODE__DRQ_EXPANSION_MODE_MASK                                                  0x00000003L
+#define HUBPREQ3_DCN_EXPANSION_MODE__CRQ_EXPANSION_MODE_MASK                                                  0x0000000CL
+#define HUBPREQ3_DCN_EXPANSION_MODE__MRQ_EXPANSION_MODE_MASK                                                  0x00000030L
+#define HUBPREQ3_DCN_EXPANSION_MODE__PRQ_EXPANSION_MODE_MASK                                                  0x000000C0L
+//HUBPREQ3_DCN_TTU_QOS_WM
+#define HUBPREQ3_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM__SHIFT                                                      0x0
+#define HUBPREQ3_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM__SHIFT                                                     0x10
+#define HUBPREQ3_DCN_TTU_QOS_WM__QoS_LEVEL_LOW_WM_MASK                                                        0x00003FFFL
+#define HUBPREQ3_DCN_TTU_QOS_WM__QoS_LEVEL_HIGH_WM_MASK                                                       0x3FFF0000L
+//HUBPREQ3_DCN_GLOBAL_TTU_CNTL
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK__SHIFT                                                   0x0
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT__SHIFT                                             0x18
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT__SHIFT                                             0x19
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE__SHIFT                                                     0x1b
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP__SHIFT                                                   0x1c
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__MIN_TTU_VBLANK_MASK                                                     0x00FFFFFFL
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__PIPE_IN_FLUSH_URGENT_MASK                                               0x01000000L
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__PRQ_MRQ_FLUSH_URGENT_MASK                                               0x02000000L
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__ROW_TTU_MODE_MASK                                                       0x08000000L
+#define HUBPREQ3_DCN_GLOBAL_TTU_CNTL__QoS_LEVEL_FLIP_MASK                                                     0xF0000000L
+//HUBPREQ3_DCN_SURF0_TTU_CNTL0
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ3_DCN_SURF0_TTU_CNTL1
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ3_DCN_SURF0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ3_DCN_SURF1_TTU_CNTL0
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                          0x0
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                  0x18
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                 0x1c
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                            0x007FFFFFL
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                    0x0F000000L
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                   0x10000000L
+//HUBPREQ3_DCN_SURF1_TTU_CNTL1
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                      0x0
+#define HUBPREQ3_DCN_SURF1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                        0x007FFFFFL
+//HUBPREQ3_DCN_CUR0_TTU_CNTL0
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ3_DCN_CUR0_TTU_CNTL1
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ3_DCN_CUR0_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ3_DCN_CUR1_TTU_CNTL0
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY__SHIFT                                           0x0
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED__SHIFT                                                   0x18
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE__SHIFT                                                  0x1c
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__REFCYC_PER_REQ_DELIVERY_MASK                                             0x007FFFFFL
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__QoS_LEVEL_FIXED_MASK                                                     0x0F000000L
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL0__QoS_RAMP_DISABLE_MASK                                                    0x10000000L
+//HUBPREQ3_DCN_CUR1_TTU_CNTL1
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE__SHIFT                                       0x0
+#define HUBPREQ3_DCN_CUR1_TTU_CNTL1__REFCYC_PER_REQ_DELIVERY_PRE_MASK                                         0x007FFFFFL
+//HUBPREQ3_DCN_DMDATA_VM_CNTL
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA__SHIFT                                              0x0
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS__SHIFT                                            0x10
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR__SHIFT                                      0x14
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS__SHIFT                                        0x18
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS__SHIFT                                             0x19
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR__SHIFT                                  0x1a
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE__SHIFT                                                    0x1f
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__REFCYC_PER_VM_DMDATA_MASK                                                0x0000FFFFL
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_MASK                                              0x000F0000L
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_FAULT_STATUS_CLEAR_MASK                                        0x00100000L
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_MASK                                          0x01000000L
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_LATE_STATUS_MASK                                               0x02000000L
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_UNDERFLOW_STATUS_CLEAR_MASK                                    0x04000000L
+#define HUBPREQ3_DCN_DMDATA_VM_CNTL__DMDATA_VM_DONE_MASK                                                      0x80000000L
+//HUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR
+#define HUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR__SHIFT                       0x0
+#define HUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR__MC_VM_SYSTEM_APERTURE_LOW_ADDR_MASK                         0x3FFFFFFFL
+//HUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define HUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR__SHIFT                     0x0
+#define HUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR__MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MASK                       0x3FFFFFFFL
+//HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                  0x0
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                             0x3
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                0x5
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                   0x6
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                    0x00000001L
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                               0x00000018L
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                  0x00000020L
+#define HUBPREQ3_DCN_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                     0x00000040L
+//HUBPREQ3_BLANK_OFFSET_0
+#define HUBPREQ3_BLANK_OFFSET_0__REFCYC_H_BLANK_END__SHIFT                                                    0x0
+#define HUBPREQ3_BLANK_OFFSET_0__DLG_V_BLANK_END__SHIFT                                                       0x10
+#define HUBPREQ3_BLANK_OFFSET_0__REFCYC_H_BLANK_END_MASK                                                      0x00001FFFL
+#define HUBPREQ3_BLANK_OFFSET_0__DLG_V_BLANK_END_MASK                                                         0x7FFF0000L
+//HUBPREQ3_BLANK_OFFSET_1
+#define HUBPREQ3_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START__SHIFT                                                  0x0
+#define HUBPREQ3_BLANK_OFFSET_1__MIN_DST_Y_NEXT_START_MASK                                                    0x0003FFFFL
+//HUBPREQ3_DST_DIMENSIONS
+#define HUBPREQ3_DST_DIMENSIONS__REFCYC_PER_HTOTAL__SHIFT                                                     0x0
+#define HUBPREQ3_DST_DIMENSIONS__REFCYC_PER_HTOTAL_MASK                                                       0x001FFFFFL
+//HUBPREQ3_DST_AFTER_SCALER
+#define HUBPREQ3_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER__SHIFT                                               0x0
+#define HUBPREQ3_DST_AFTER_SCALER__DST_Y_AFTER_SCALER__SHIFT                                                  0x10
+#define HUBPREQ3_DST_AFTER_SCALER__REFCYC_X_AFTER_SCALER_MASK                                                 0x00001FFFL
+#define HUBPREQ3_DST_AFTER_SCALER__DST_Y_AFTER_SCALER_MASK                                                    0x00070000L
+//HUBPREQ3_PREFETCH_SETTINGS
+#define HUBPREQ3_PREFETCH_SETTINGS__VRATIO_PREFETCH__SHIFT                                                    0x0
+#define HUBPREQ3_PREFETCH_SETTINGS__DST_Y_PREFETCH__SHIFT                                                     0x18
+#define HUBPREQ3_PREFETCH_SETTINGS__VRATIO_PREFETCH_MASK                                                      0x003FFFFFL
+#define HUBPREQ3_PREFETCH_SETTINGS__DST_Y_PREFETCH_MASK                                                       0xFF000000L
+//HUBPREQ3_PREFETCH_SETTINGS_C
+#define HUBPREQ3_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C__SHIFT                                                0x0
+#define HUBPREQ3_PREFETCH_SETTINGS_C__VRATIO_PREFETCH_C_MASK                                                  0x003FFFFFL
+//HUBPREQ3_VBLANK_PARAMETERS_0
+#define HUBPREQ3_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK__SHIFT                                              0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK__SHIFT                                             0x8
+#define HUBPREQ3_VBLANK_PARAMETERS_0__DST_Y_PER_VM_VBLANK_MASK                                                0x0000007FL
+#define HUBPREQ3_VBLANK_PARAMETERS_0__DST_Y_PER_ROW_VBLANK_MASK                                               0x00003F00L
+//HUBPREQ3_VBLANK_PARAMETERS_1
+#define HUBPREQ3_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L__SHIFT                                    0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_1__REFCYC_PER_PTE_GROUP_VBLANK_L_MASK                                      0x007FFFFFL
+//HUBPREQ3_VBLANK_PARAMETERS_2
+#define HUBPREQ3_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C__SHIFT                                    0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_2__REFCYC_PER_PTE_GROUP_VBLANK_C_MASK                                      0x007FFFFFL
+//HUBPREQ3_VBLANK_PARAMETERS_3
+#define HUBPREQ3_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L__SHIFT                                   0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_3__REFCYC_PER_META_CHUNK_VBLANK_L_MASK                                     0x007FFFFFL
+//HUBPREQ3_VBLANK_PARAMETERS_4
+#define HUBPREQ3_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C__SHIFT                                   0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_4__REFCYC_PER_META_CHUNK_VBLANK_C_MASK                                     0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_0
+#define HUBPREQ3_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP__SHIFT                                                  0x0
+#define HUBPREQ3_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP__SHIFT                                                 0x8
+#define HUBPREQ3_FLIP_PARAMETERS_0__DST_Y_PER_VM_FLIP_MASK                                                    0x0000007FL
+#define HUBPREQ3_FLIP_PARAMETERS_0__DST_Y_PER_ROW_FLIP_MASK                                                   0x00003F00L
+//HUBPREQ3_FLIP_PARAMETERS_1
+#define HUBPREQ3_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L__SHIFT                                        0x0
+#define HUBPREQ3_FLIP_PARAMETERS_1__REFCYC_PER_PTE_GROUP_FLIP_L_MASK                                          0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_2
+#define HUBPREQ3_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L__SHIFT                                       0x0
+#define HUBPREQ3_FLIP_PARAMETERS_2__REFCYC_PER_META_CHUNK_FLIP_L_MASK                                         0x007FFFFFL
+//HUBPREQ3_NOM_PARAMETERS_0
+#define HUBPREQ3_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L__SHIFT                                             0x0
+#define HUBPREQ3_NOM_PARAMETERS_0__DST_Y_PER_PTE_ROW_NOM_L_MASK                                               0x0001FFFFL
+//HUBPREQ3_NOM_PARAMETERS_1
+#define HUBPREQ3_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L__SHIFT                                          0x0
+#define HUBPREQ3_NOM_PARAMETERS_1__REFCYC_PER_PTE_GROUP_NOM_L_MASK                                            0x007FFFFFL
+//HUBPREQ3_NOM_PARAMETERS_2
+#define HUBPREQ3_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C__SHIFT                                             0x0
+#define HUBPREQ3_NOM_PARAMETERS_2__DST_Y_PER_PTE_ROW_NOM_C_MASK                                               0x0001FFFFL
+//HUBPREQ3_NOM_PARAMETERS_3
+#define HUBPREQ3_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C__SHIFT                                          0x0
+#define HUBPREQ3_NOM_PARAMETERS_3__REFCYC_PER_PTE_GROUP_NOM_C_MASK                                            0x007FFFFFL
+//HUBPREQ3_NOM_PARAMETERS_4
+#define HUBPREQ3_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L__SHIFT                                            0x0
+#define HUBPREQ3_NOM_PARAMETERS_4__DST_Y_PER_META_ROW_NOM_L_MASK                                              0x0001FFFFL
+//HUBPREQ3_NOM_PARAMETERS_5
+#define HUBPREQ3_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L__SHIFT                                         0x0
+#define HUBPREQ3_NOM_PARAMETERS_5__REFCYC_PER_META_CHUNK_NOM_L_MASK                                           0x007FFFFFL
+//HUBPREQ3_NOM_PARAMETERS_6
+#define HUBPREQ3_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C__SHIFT                                            0x0
+#define HUBPREQ3_NOM_PARAMETERS_6__DST_Y_PER_META_ROW_NOM_C_MASK                                              0x0001FFFFL
+//HUBPREQ3_NOM_PARAMETERS_7
+#define HUBPREQ3_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C__SHIFT                                         0x0
+#define HUBPREQ3_NOM_PARAMETERS_7__REFCYC_PER_META_CHUNK_NOM_C_MASK                                           0x007FFFFFL
+//HUBPREQ3_PER_LINE_DELIVERY_PRE
+#define HUBPREQ3_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L__SHIFT                                 0x0
+#define HUBPREQ3_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C__SHIFT                                 0x10
+#define HUBPREQ3_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_L_MASK                                   0x00001FFFL
+#define HUBPREQ3_PER_LINE_DELIVERY_PRE__REFCYC_PER_LINE_DELIVERY_PRE_C_MASK                                   0x1FFF0000L
+//HUBPREQ3_PER_LINE_DELIVERY
+#define HUBPREQ3_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L__SHIFT                                         0x0
+#define HUBPREQ3_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C__SHIFT                                         0x10
+#define HUBPREQ3_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_L_MASK                                           0x00001FFFL
+#define HUBPREQ3_PER_LINE_DELIVERY__REFCYC_PER_LINE_DELIVERY_C_MASK                                           0x1FFF0000L
+//HUBPREQ3_CURSOR_SETTINGS
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET__SHIFT                                                 0x0
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST__SHIFT                                             0x8
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET__SHIFT                                                 0x10
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST__SHIFT                                             0x18
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET_MASK                                                   0x000000FFL
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST_MASK                                               0x00000300L
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR1_DST_Y_OFFSET_MASK                                                   0x00FF0000L
+#define HUBPREQ3_CURSOR_SETTINGS__CURSOR1_CHUNK_HDL_ADJUST_MASK                                               0x03000000L
+//HUBPREQ3_REF_FREQ_TO_PIX_FREQ
+#define HUBPREQ3_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ__SHIFT                                            0x0
+#define HUBPREQ3_REF_FREQ_TO_PIX_FREQ__REF_FREQ_TO_PIX_FREQ_MASK                                              0x001FFFFFL
+//HUBPREQ3_DST_Y_DELTA_DRQ_LIMIT
+#define HUBPREQ3_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT__SHIFT                                          0x0
+#define HUBPREQ3_DST_Y_DELTA_DRQ_LIMIT__DST_Y_DELTA_DRQ_LIMIT_MASK                                            0x00007FFFL
+//HUBPREQ3_HUBPREQ_MEM_PWR_CTRL
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE__SHIFT                                          0x0
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS__SHIFT                                            0x2
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE__SHIFT                                          0x4
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS__SHIFT                                            0x6
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE__SHIFT                                          0x8
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS__SHIFT                                            0xa
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE__SHIFT                                           0xc
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS__SHIFT                                             0xe
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE__SHIFT                                          0xf
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS__SHIFT                                            0x11
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_FORCE_MASK                                            0x00000003L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_DPTE_MEM_PWR_DIS_MASK                                              0x00000004L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_FORCE_MASK                                            0x00000030L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_MPTE_MEM_PWR_DIS_MASK                                              0x00000040L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_FORCE_MASK                                            0x00000300L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_META_MEM_PWR_DIS_MASK                                              0x00000400L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_FORCE_MASK                                             0x00003000L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_PDE_MEM_PWR_DIS_MASK                                               0x00004000L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_FORCE_MASK                                            0x00018000L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_CTRL__REQ_TPTE_MEM_PWR_DIS_MASK                                              0x00020000L
+//HUBPREQ3_HUBPREQ_MEM_PWR_STATUS
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE__SHIFT                                        0x0
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE__SHIFT                                        0x2
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE__SHIFT                                        0x4
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE__SHIFT                                         0x6
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE__SHIFT                                        0x8
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_DPTE_MEM_PWR_STATE_MASK                                          0x00000003L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_MPTE_MEM_PWR_STATE_MASK                                          0x0000000CL
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_META_MEM_PWR_STATE_MASK                                          0x00000030L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_PDE_MEM_PWR_STATE_MASK                                           0x000000C0L
+#define HUBPREQ3_HUBPREQ_MEM_PWR_STATUS__REQ_TPTE_MEM_PWR_STATE_MASK                                          0x00000300L
+//HUBPREQ3_VBLANK_PARAMETERS_5
+#define HUBPREQ3_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK__SHIFT                                       0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_5__REFCYC_PER_VM_GROUP_VBLANK_MASK                                         0x007FFFFFL
+//HUBPREQ3_VBLANK_PARAMETERS_6
+#define HUBPREQ3_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK__SHIFT                                         0x0
+#define HUBPREQ3_VBLANK_PARAMETERS_6__REFCYC_PER_VM_REQ_VBLANK_MASK                                           0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_3
+#define HUBPREQ3_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP__SHIFT                                           0x0
+#define HUBPREQ3_FLIP_PARAMETERS_3__REFCYC_PER_VM_GROUP_FLIP_MASK                                             0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_4
+#define HUBPREQ3_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP__SHIFT                                             0x0
+#define HUBPREQ3_FLIP_PARAMETERS_4__REFCYC_PER_VM_REQ_FLIP_MASK                                               0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_5
+#define HUBPREQ3_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C__SHIFT                                        0x0
+#define HUBPREQ3_FLIP_PARAMETERS_5__REFCYC_PER_PTE_GROUP_FLIP_C_MASK                                          0x007FFFFFL
+//HUBPREQ3_FLIP_PARAMETERS_6
+#define HUBPREQ3_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C__SHIFT                                       0x0
+#define HUBPREQ3_FLIP_PARAMETERS_6__REFCYC_PER_META_CHUNK_FLIP_C_MASK                                         0x007FFFFFL
+//HUBPREQ3_UCLK_PSTATE_FORCE
+#define HUBPREQ3_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN__SHIFT                                          0x0
+#define HUBPREQ3_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE__SHIFT                                       0x1
+#define HUBPREQ3_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN__SHIFT                                        0x2
+#define HUBPREQ3_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE__SHIFT                                     0x3
+#define HUBPREQ3_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_EN_MASK                                            0x00000001L
+#define HUBPREQ3_UCLK_PSTATE_FORCE__DATA_UCLK_PSTATE_FORCE_VALUE_MASK                                         0x00000002L
+#define HUBPREQ3_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_EN_MASK                                          0x00000004L
+#define HUBPREQ3_UCLK_PSTATE_FORCE__CURSOR_UCLK_PSTATE_FORCE_VALUE_MASK                                       0x00000008L
+//HUBPREQ3_HUBPREQ_STATUS_REG0
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0__SHIFT                                         0x0
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1__SHIFT                                         0x8
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT__SHIFT                                                 0x10
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S0_MASK                                           0x0000001FL
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_MPTE_ROW_READY_S1_MASK                                           0x00001F00L
+#define HUBPREQ3_HUBPREQ_STATUS_REG0__STATUS_VTG_COUNT_MASK                                                   0x7FFF0000L
+//HUBPREQ3_HUBPREQ_STATUS_REG1
+#define HUBPREQ3_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0__SHIFT                                       0x0
+#define HUBPREQ3_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1__SHIFT                                       0x10
+#define HUBPREQ3_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S0_MASK                                         0x00003FFFL
+#define HUBPREQ3_HUBPREQ_STATUS_REG1__STATUS_CHUNK_REQ_X_OR_Y_S1_MASK                                         0x3FFF0000L
+//HUBPREQ3_HUBPREQ_STATUS_REG2
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0__SHIFT                                      0x0
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0__SHIFT                                     0x1
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0__SHIFT                               0x2
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0__SHIFT                               0x3
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0__SHIFT                                   0x4
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0__SHIFT                                             0x5
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1__SHIFT                                      0x8
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1__SHIFT                                     0x9
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1__SHIFT                               0xa
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1__SHIFT                               0xb
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1__SHIFT                                   0xc
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1__SHIFT                                             0xd
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR__SHIFT                                     0x10
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR__SHIFT                                    0x11
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR__SHIFT                              0x12
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR__SHIFT                              0x13
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR__SHIFT                                  0x14
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR__SHIFT                                            0x15
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_VBLANK__SHIFT                                                    0x1a
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN__SHIFT                                                   0x1b
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY__SHIFT                                          0x1c
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH__SHIFT                                             0x1d
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0__SHIFT                                            0x1e
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1__SHIFT                                            0x1f
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S0_MASK                                        0x00000001L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S0_MASK                                       0x00000002L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S0_MASK                                 0x00000004L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S0_MASK                                 0x00000008L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S0_MASK                                     0x00000010L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S0_MASK                                               0x00000020L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_S1_MASK                                        0x00000100L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_S1_MASK                                       0x00000200L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_S1_MASK                                 0x00000400L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_S1_MASK                                 0x00000800L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_S1_MASK                                     0x00001000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_S1_MASK                                               0x00002000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_EXIT_SELF_REFRESH_CUR_MASK                                       0x00010000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ENTER_SELF_REFRESH_CUR_MASK                                      0x00020000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_UCLK_PSTATE_CHANGE_CUR_MASK                                0x00040000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_FCLK_PSTATE_CHANGE_CUR_MASK                                0x00080000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_ALLOW_USR_RETRAINING_CUR_MASK                                    0x00100000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_QOS_URGENT_CUR_MASK                                              0x00200000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_VBLANK_MASK                                                      0x04000000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_HUBP_EN_MASK                                                     0x08000000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_RECOVERY_MASK                                            0x10000000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_PIPE_IN_FLUSH_MASK                                               0x20000000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S0_MASK                                              0x40000000L
+#define HUBPREQ3_HUBPREQ_STATUS_REG2__STATUS_FLIP_ACTIVE_S1_MASK                                              0x80000000L
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpret_dispdec
+//HUBPRET3_HUBPRET_CONTROL
+#define HUBPRET3_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS__SHIFT                                          0x4
+#define HUBPRET3_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE__SHIFT                                            0xf
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA__SHIFT                                                   0x10
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G__SHIFT                                                     0x12
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B__SHIFT                                                    0x14
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R__SHIFT                                                    0x16
+#define HUBPRET3_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE__SHIFT                                                0x18
+#define HUBPRET3_HUBPRET_CONTROL__DET_BUF_PLANE1_BASE_ADDRESS_MASK                                            0x00007FF0L
+#define HUBPRET3_HUBPRET_CONTROL__PACK_3TO2_ELEMENT_DISABLE_MASK                                              0x00008000L
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_ALPHA_MASK                                                     0x00030000L
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_Y_G_MASK                                                       0x000C0000L
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_CB_B_MASK                                                      0x00300000L
+#define HUBPRET3_HUBPRET_CONTROL__CROSSBAR_SRC_CR_R_MASK                                                      0x00C00000L
+#define HUBPRET3_HUBPRET_CONTROL__HUBPRET_CONTROL_SPARE_MASK                                                  0xFF000000L
+//HUBPRET3_HUBPRET_MEM_PWR_CTRL
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE__SHIFT                                             0x8
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS__SHIFT                                               0xa
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE__SHIFT                                            0x10
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS__SHIFT                                              0x12
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE__SHIFT                                          0x14
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_FORCE_MASK                                               0x00000300L
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__DMROB_MEM_PWR_DIS_MASK                                                 0x00000400L
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_FORCE_MASK                                              0x00030000L
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_DIS_MASK                                                0x00040000L
+#define HUBPRET3_HUBPRET_MEM_PWR_CTRL__PIXCDC_MEM_PWR_LS_MODE_MASK                                            0x00300000L
+//HUBPRET3_HUBPRET_MEM_PWR_STATUS
+#define HUBPRET3_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE__SHIFT                                           0x2
+#define HUBPRET3_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE__SHIFT                                          0x4
+#define HUBPRET3_HUBPRET_MEM_PWR_STATUS__DMROB_MEM_PWR_STATE_MASK                                             0x0000000CL
+#define HUBPRET3_HUBPRET_MEM_PWR_STATUS__PIXCDC_MEM_PWR_STATE_MASK                                            0x00000030L
+//HUBPRET3_HUBPRET_READ_LINE_CTRL0
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE__SHIFT                         0x0
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM__SHIFT                                0x10
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_INTERVAL_IN_NONACTIVE_MASK                           0x0000FFFFL
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL0__PIPE_READ_LINE_VBLANK_MAXIMUM_MASK                                  0x3FFF0000L
+//HUBPRET3_HUBPRET_READ_LINE_CTRL1
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED__SHIFT                    0x0
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE__SHIFT                                0x10
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL1__PIPE_READ_LINE_REPORTED_WHEN_REQ_DISABLED_MASK                      0x00003FFFL
+#define HUBPRET3_HUBPRET_READ_LINE_CTRL1__HUBPRET_READ_LINE_CTRL1_SPARE_MASK                                  0xFFFF0000L
+//HUBPRET3_HUBPRET_READ_LINE0
+#define HUBPRET3_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START__SHIFT                                             0x0
+#define HUBPRET3_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END__SHIFT                                               0x10
+#define HUBPRET3_HUBPRET_READ_LINE0__PIPE_READ_LINE0_START_MASK                                               0x00003FFFL
+#define HUBPRET3_HUBPRET_READ_LINE0__PIPE_READ_LINE0_END_MASK                                                 0x3FFF0000L
+//HUBPRET3_HUBPRET_READ_LINE1
+#define HUBPRET3_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START__SHIFT                                             0x0
+#define HUBPRET3_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END__SHIFT                                               0x10
+#define HUBPRET3_HUBPRET_READ_LINE1__PIPE_READ_LINE1_START_MASK                                               0x00003FFFL
+#define HUBPRET3_HUBPRET_READ_LINE1__PIPE_READ_LINE1_END_MASK                                                 0x3FFF0000L
+//HUBPRET3_HUBPRET_INTERRUPT
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK__SHIFT                                               0x0
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK__SHIFT                                           0x1
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK__SHIFT                                           0x2
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE__SHIFT                                               0x4
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE__SHIFT                                           0x5
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE__SHIFT                                           0x6
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR__SHIFT                                              0x8
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR__SHIFT                                          0x9
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR__SHIFT                                          0xa
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS__SHIFT                                                 0xc
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS__SHIFT                                             0xd
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS__SHIFT                                             0xe
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS__SHIFT                                             0x10
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS__SHIFT                                         0x11
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS__SHIFT                                         0x12
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_MASK_MASK                                                 0x00000001L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_MASK_MASK                                             0x00000002L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_MASK_MASK                                             0x00000004L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_TYPE_MASK                                                 0x00000010L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_TYPE_MASK                                             0x00000020L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_TYPE_MASK                                             0x00000040L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_CLEAR_MASK                                                0x00000100L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_CLEAR_MASK                                            0x00000200L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_CLEAR_MASK                                            0x00000400L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_STATUS_MASK                                                   0x00001000L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_STATUS_MASK                                               0x00002000L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_STATUS_MASK                                               0x00004000L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_VBLANK_INT_STATUS_MASK                                               0x00010000L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE0_INT_STATUS_MASK                                           0x00020000L
+#define HUBPRET3_HUBPRET_INTERRUPT__PIPE_READ_LINE1_INT_STATUS_MASK                                           0x00040000L
+//HUBPRET3_HUBPRET_READ_LINE_VALUE
+#define HUBPRET3_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE__SHIFT                                               0x0
+#define HUBPRET3_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT__SHIFT                                      0x10
+#define HUBPRET3_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_MASK                                                 0x00003FFFL
+#define HUBPRET3_HUBPRET_READ_LINE_VALUE__PIPE_READ_LINE_SNAPSHOT_MASK                                        0x3FFF0000L
+//HUBPRET3_HUBPRET_READ_LINE_STATUS
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK__SHIFT                                            0x0
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE__SHIFT                                      0x4
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE__SHIFT                                     0x5
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE__SHIFT                                      0x8
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE__SHIFT                                     0xa
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_VBLANK_MASK                                              0x00000001L
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_INSIDE_MASK                                        0x00000010L
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE0_OUTSIDE_MASK                                       0x00000020L
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_INSIDE_MASK                                        0x00000100L
+#define HUBPRET3_HUBPRET_READ_LINE_STATUS__PIPE_READ_LINE1_OUTSIDE_MASK                                       0x00000400L
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_cursor0_dispdec
+//CURSOR0_3_CURSOR_CONTROL
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_ENABLE__SHIFT                                                        0x0
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_REQ_MODE__SHIFT                                                      0x2
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT                                                    0x4
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_MODE__SHIFT                                                          0x8
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_TMZ__SHIFT                                                           0xc
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PITCH__SHIFT                                                         0x10
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS__SHIFT                     0x14
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK__SHIFT                                               0x18
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN__SHIFT                                    0x1e
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL__SHIFT                                   0x1f
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_ENABLE_MASK                                                          0x00000001L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_REQ_MODE_MASK                                                        0x00000004L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_2X_MAGNIFY_MASK                                                      0x00000010L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_MODE_MASK                                                            0x00000700L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_TMZ_MASK                                                             0x00001000L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PITCH_MASK                                                           0x00030000L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_XY_POSITION_ROTATION_AND_MIRRORING_BYPASS_MASK                       0x00100000L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_LINES_PER_CHUNK_MASK                                                 0x1F000000L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_EN_MASK                                      0x40000000L
+#define CURSOR0_3_CURSOR_CONTROL__CURSOR_PERFMON_LATENCY_MEASURE_SEL_MASK                                     0x80000000L
+//CURSOR0_3_CURSOR_SURFACE_ADDRESS
+#define CURSOR0_3_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS__SHIFT                                       0x0
+#define CURSOR0_3_CURSOR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS_MASK                                         0xFFFFFFFFL
+//CURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH
+#define CURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH__SHIFT                             0x0
+#define CURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH_MASK                               0x0000FFFFL
+//CURSOR0_3_CURSOR_SIZE
+#define CURSOR0_3_CURSOR_SIZE__CURSOR_HEIGHT__SHIFT                                                           0x0
+#define CURSOR0_3_CURSOR_SIZE__CURSOR_WIDTH__SHIFT                                                            0x10
+#define CURSOR0_3_CURSOR_SIZE__CURSOR_HEIGHT_MASK                                                             0x000001FFL
+#define CURSOR0_3_CURSOR_SIZE__CURSOR_WIDTH_MASK                                                              0x01FF0000L
+//CURSOR0_3_CURSOR_POSITION
+#define CURSOR0_3_CURSOR_POSITION__CURSOR_Y_POSITION__SHIFT                                                   0x0
+#define CURSOR0_3_CURSOR_POSITION__CURSOR_X_POSITION__SHIFT                                                   0x10
+#define CURSOR0_3_CURSOR_POSITION__CURSOR_Y_POSITION_MASK                                                     0x00003FFFL
+#define CURSOR0_3_CURSOR_POSITION__CURSOR_X_POSITION_MASK                                                     0x3FFF0000L
+//CURSOR0_3_CURSOR_HOT_SPOT
+#define CURSOR0_3_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y__SHIFT                                                   0x0
+#define CURSOR0_3_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X__SHIFT                                                   0x10
+#define CURSOR0_3_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_Y_MASK                                                     0x000000FFL
+#define CURSOR0_3_CURSOR_HOT_SPOT__CURSOR_HOT_SPOT_X_MASK                                                     0x00FF0000L
+//CURSOR0_3_CURSOR_STEREO_CONTROL
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN__SHIFT                                              0x0
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET__SHIFT                                         0x4
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET__SHIFT                                       0x12
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_STEREO_EN_MASK                                                0x00000001L
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_PRIMARY_OFFSET_MASK                                           0x0003FFF0L
+#define CURSOR0_3_CURSOR_STEREO_CONTROL__CURSOR_SECONDARY_OFFSET_MASK                                         0xFFFC0000L
+//CURSOR0_3_CURSOR_DST_OFFSET
+#define CURSOR0_3_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET__SHIFT                                               0x0
+#define CURSOR0_3_CURSOR_DST_OFFSET__CURSOR_DST_X_OFFSET_MASK                                                 0x00001FFFL
+//CURSOR0_3_CURSOR_MEM_PWR_CTRL
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE__SHIFT                                              0x0
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS__SHIFT                                                0x2
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE__SHIFT                                            0x4
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_FORCE_MASK                                                0x00000003L
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_DIS_MASK                                                  0x00000004L
+#define CURSOR0_3_CURSOR_MEM_PWR_CTRL__CROB_MEM_PWR_LS_MODE_MASK                                              0x00000030L
+//CURSOR0_3_CURSOR_MEM_PWR_STATUS
+#define CURSOR0_3_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE__SHIFT                                            0x0
+#define CURSOR0_3_CURSOR_MEM_PWR_STATUS__CROB_MEM_PWR_STATE_MASK                                              0x00000003L
+//CURSOR0_3_DMDATA_ADDRESS_HIGH
+#define CURSOR0_3_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH__SHIFT                                             0x0
+#define CURSOR0_3_DMDATA_ADDRESS_HIGH__DMDATA_TMZ__SHIFT                                                      0x1e
+#define CURSOR0_3_DMDATA_ADDRESS_HIGH__DMDATA_ADDRESS_HIGH_MASK                                               0x0000FFFFL
+#define CURSOR0_3_DMDATA_ADDRESS_HIGH__DMDATA_TMZ_MASK                                                        0x40000000L
+//CURSOR0_3_DMDATA_ADDRESS_LOW
+#define CURSOR0_3_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW__SHIFT                                               0x0
+#define CURSOR0_3_DMDATA_ADDRESS_LOW__DMDATA_ADDRESS_LOW_MASK                                                 0xFFFFFFFFL
+//CURSOR0_3_DMDATA_CNTL
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_UPDATED__SHIFT                                                          0x0
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_REPEAT__SHIFT                                                           0x1
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_MODE__SHIFT                                                             0x2
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_SIZE__SHIFT                                                             0x10
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_UPDATED_MASK                                                            0x00000001L
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_REPEAT_MASK                                                             0x00000002L
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_MODE_MASK                                                               0x00000004L
+#define CURSOR0_3_DMDATA_CNTL__DMDATA_SIZE_MASK                                                               0x0FFF0000L
+//CURSOR0_3_DMDATA_QOS_CNTL
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_QOS_MODE__SHIFT                                                     0x0
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL__SHIFT                                                    0x4
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_DL_DELTA__SHIFT                                                     0x10
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_QOS_MODE_MASK                                                       0x00000001L
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_QOS_LEVEL_MASK                                                      0x000000F0L
+#define CURSOR0_3_DMDATA_QOS_CNTL__DMDATA_DL_DELTA_MASK                                                       0xFFFF0000L
+//CURSOR0_3_DMDATA_STATUS
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_DONE__SHIFT                                                           0x0
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_UNDERFLOW__SHIFT                                                      0x2
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR__SHIFT                                                0x4
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_DONE_MASK                                                             0x00000001L
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_UNDERFLOW_MASK                                                        0x00000004L
+#define CURSOR0_3_DMDATA_STATUS__DMDATA_UNDERFLOW_CLEAR_MASK                                                  0x00000010L
+//CURSOR0_3_DMDATA_SW_CNTL
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_UPDATED__SHIFT                                                    0x0
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_REPEAT__SHIFT                                                     0x1
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_SIZE__SHIFT                                                       0x10
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_UPDATED_MASK                                                      0x00000001L
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_REPEAT_MASK                                                       0x00000002L
+#define CURSOR0_3_DMDATA_SW_CNTL__DMDATA_SW_SIZE_MASK                                                         0x0FFF0000L
+//CURSOR0_3_DMDATA_SW_DATA
+#define CURSOR0_3_DMDATA_SW_DATA__DMDATA_SW_DATA__SHIFT                                                       0x0
+#define CURSOR0_3_DMDATA_SW_DATA__DMDATA_SW_DATA_MASK                                                         0xFFFFFFFFL
+//CURSOR0_3_HUBP_3DLUT_CONTROL
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE__SHIFT                                                0x0
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE__SHIFT                                       0x1
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH__SHIFT                                                 0x2
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ__SHIFT                                                   0x12
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B__SHIFT                                        0x16
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G__SHIFT                                        0x18
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R__SHIFT                                        0x1a
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS__SHIFT                                         0x1d
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH__SHIFT                                             0x1e
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE__SHIFT                                                  0x1f
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ENABLE_MASK                                                  0x00000001L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_ADDRESSING_MODE_MASK                                         0x00000002L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WIDTH_MASK                                                   0x0003FFFCL
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_TMZ_MASK                                                     0x003C0000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_B_MASK                                          0x00C00000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_G_MASK                                          0x03000000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_CROSSBAR_SEL_R_MASK                                          0x0C000000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_WTRIG_FIX_DIS_MASK                                           0x20000000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_MPC_WIDTH_MASK                                               0x40000000L
+#define CURSOR0_3_HUBP_3DLUT_CONTROL__HUBP_3DLUT_DONE_MASK                                                    0x80000000L
+//CURSOR0_3_HUBP_3DLUT_ADDRESS_LOW
+#define CURSOR0_3_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW__SHIFT                                       0x0
+#define CURSOR0_3_HUBP_3DLUT_ADDRESS_LOW__HUBP_3DLUT_ADDRESS_LOW_MASK                                         0xFFFFFFFFL
+//CURSOR0_3_HUBP_3DLUT_ADDRESS_HIGH
+#define CURSOR0_3_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH__SHIFT                                     0x0
+#define CURSOR0_3_HUBP_3DLUT_ADDRESS_HIGH__HUBP_3DLUT_ADDRESS_HIGH_MASK                                       0x0000FFFFL
+//CURSOR0_3_HUBP_3DLUT_DLG_PARAM
+#define CURSOR0_3_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP__SHIFT                                         0x0
+#define CURSOR0_3_HUBP_3DLUT_DLG_PARAM__REFCYC_PER_3DLUT_GROUP_MASK                                           0x007FFFFFL
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON9_PERFCOUNTER_CNTL
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                            0x0
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                           0x9
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                             0xc
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                          0xf
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                           0x10
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                     0x16
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                           0x17
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                               0x18
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                             0x19
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                               0x1a
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                             0x1d
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                              0x000001FFL
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                             0x00000E00L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                               0x00007000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                            0x00008000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                             0x00010000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                       0x00400000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                             0x00800000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                 0x01000000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                               0x02000000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                 0x04000000L
+#define DC_PERFMON9_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                               0xE0000000L
+//DC_PERFMON9_PERFCOUNTER_CNTL2
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                  0x0
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                        0x2
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                        0x3
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                          0x8
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                           0x1d
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                    0x00000003L
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                          0x00000004L
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                          0x00000008L
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                            0x00003F00L
+#define DC_PERFMON9_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                             0xE0000000L
+//DC_PERFMON9_PERFCOUNTER_STATE
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                          0x0
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                          0x2
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                          0x4
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                          0x6
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                          0x8
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                          0xa
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                          0xc
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                          0xe
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                          0x10
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                          0x12
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                          0x14
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                          0x16
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                          0x18
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                          0x1a
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                          0x1c
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                          0x1e
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                            0x00000003L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                            0x00000004L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                            0x00000030L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                            0x00000040L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                            0x00000300L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                            0x00000400L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                            0x00003000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                            0x00004000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                            0x00030000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                            0x00040000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                            0x00300000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                            0x00400000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                            0x03000000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                            0x04000000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                            0x30000000L
+#define DC_PERFMON9_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                            0x40000000L
+//DC_PERFMON9_PERFMON_CNTL
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                        0x0
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                    0x8
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                                0x1c
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                                0x1d
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                            0x1e
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                               0x1f
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_STATE_MASK                                                          0x00000003L
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                      0x0FFFFF00L
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                  0x10000000L
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                  0x20000000L
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                              0x40000000L
+#define DC_PERFMON9_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                 0x80000000L
+//DC_PERFMON9_PERFMON_CNTL2
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                             0x0
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                  0x1
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                        0x2
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                         0xa
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                               0x00000001L
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                    0x00000002L
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                          0x000003FCL
+#define DC_PERFMON9_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                           0x0003FC00L
+//DC_PERFMON9_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                   0x0
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                   0x1
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                   0x2
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                   0x3
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                   0x4
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                   0x5
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                   0x6
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                   0x7
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                      0x8
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                      0x9
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                      0xa
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                      0xb
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                      0xc
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                      0xd
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                      0xe
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                      0xf
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                         0x10
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                     0x00000001L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                     0x00000002L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                     0x00000004L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                     0x00000008L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                     0x00000010L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                     0x00000020L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                     0x00000040L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                     0x00000080L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                        0x00000100L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                        0x00000200L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                        0x00000400L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                        0x00000800L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                        0x00001000L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                        0x00002000L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                        0x00004000L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                        0x00008000L
+#define DC_PERFMON9_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                           0xFFFF0000L
+//DC_PERFMON9_PERFMON_CVALUE_LOW
+#define DC_PERFMON9_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                             0x0
+#define DC_PERFMON9_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                               0xFFFFFFFFL
+//DC_PERFMON9_PERFMON_HI
+#define DC_PERFMON9_PERFMON_HI__PERFMON_HI__SHIFT                                                             0x0
+#define DC_PERFMON9_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                       0x1d
+#define DC_PERFMON9_PERFMON_HI__PERFMON_HI_MASK                                                               0x0000FFFFL
+#define DC_PERFMON9_PERFMON_HI__PERFMON_READ_SEL_MASK                                                         0xE0000000L
+//DC_PERFMON9_PERFMON_LOW
+#define DC_PERFMON9_PERFMON_LOW__PERFMON_LOW__SHIFT                                                           0x0
+#define DC_PERFMON9_PERFMON_LOW__PERFMON_LOW_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
+//DPP_TOP0_DPP_CONTROL
+#define DPP_TOP0_DPP_CONTROL__DPP_CLOCK_ENABLE__SHIFT                                                         0x4
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_GATE_DISABLE__SHIFT                                                    0x8
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE__SHIFT                                                0xa
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE__SHIFT                                               0xc
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_R_GATE_DISABLE__SHIFT                                                    0xe
+#define DPP_TOP0_DPP_CONTROL__DISPCLK_R_GATE_DISABLE__SHIFT                                                   0x10
+#define DPP_TOP0_DPP_CONTROL__DISPCLK_G_GATE_DISABLE__SHIFT                                                   0x12
+#define DPP_TOP0_DPP_CONTROL__DPP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define DPP_TOP0_DPP_CONTROL__DPP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define DPP_TOP0_DPP_CONTROL__DPP_CLOCK_ENABLE_MASK                                                           0x00000010L
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_GATE_DISABLE_MASK                                                      0x00000100L
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE_MASK                                                  0x00000400L
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE_MASK                                                 0x00001000L
+#define DPP_TOP0_DPP_CONTROL__DPPCLK_R_GATE_DISABLE_MASK                                                      0x00004000L
+#define DPP_TOP0_DPP_CONTROL__DISPCLK_R_GATE_DISABLE_MASK                                                     0x00010000L
+#define DPP_TOP0_DPP_CONTROL__DISPCLK_G_GATE_DISABLE_MASK                                                     0x00040000L
+#define DPP_TOP0_DPP_CONTROL__DPP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define DPP_TOP0_DPP_CONTROL__DPP_TEST_CLK_SEL_MASK                                                           0x70000000L
+//DPP_TOP0_DPP_SOFT_RESET
+#define DPP_TOP0_DPP_SOFT_RESET__CNVC_SOFT_RESET__SHIFT                                                       0x0
+#define DPP_TOP0_DPP_SOFT_RESET__DSCL_SOFT_RESET__SHIFT                                                       0x4
+#define DPP_TOP0_DPP_SOFT_RESET__CM_SOFT_RESET__SHIFT                                                         0x8
+#define DPP_TOP0_DPP_SOFT_RESET__OBUF_SOFT_RESET__SHIFT                                                       0xc
+#define DPP_TOP0_DPP_SOFT_RESET__HIST_SOFT_RESET__SHIFT                                                       0x10
+#define DPP_TOP0_DPP_SOFT_RESET__CNVC_SOFT_RESET_MASK                                                         0x00000001L
+#define DPP_TOP0_DPP_SOFT_RESET__DSCL_SOFT_RESET_MASK                                                         0x00000010L
+#define DPP_TOP0_DPP_SOFT_RESET__CM_SOFT_RESET_MASK                                                           0x00000100L
+#define DPP_TOP0_DPP_SOFT_RESET__OBUF_SOFT_RESET_MASK                                                         0x00001000L
+#define DPP_TOP0_DPP_SOFT_RESET__HIST_SOFT_RESET_MASK                                                         0x00010000L
+//DPP_TOP0_DPP_CRC_VAL_R
+#define DPP_TOP0_DPP_CRC_VAL_R__DPP_CRC_R_CR__SHIFT                                                           0x0
+#define DPP_TOP0_DPP_CRC_VAL_R__DPP_CRC_R_CR_MASK                                                             0xFFFFFFFFL
+//DPP_TOP0_DPP_CRC_VAL_G
+#define DPP_TOP0_DPP_CRC_VAL_G__DPP_CRC_G_Y__SHIFT                                                            0x0
+#define DPP_TOP0_DPP_CRC_VAL_G__DPP_CRC_G_Y_MASK                                                              0xFFFFFFFFL
+//DPP_TOP0_DPP_CRC_VAL_B
+#define DPP_TOP0_DPP_CRC_VAL_B__DPP_CRC_B_CB__SHIFT                                                           0x0
+#define DPP_TOP0_DPP_CRC_VAL_B__DPP_CRC_B_CB_MASK                                                             0xFFFFFFFFL
+//DPP_TOP0_DPP_CRC_VAL_A
+#define DPP_TOP0_DPP_CRC_VAL_A__DPP_CRC_ALPHA__SHIFT                                                          0x0
+#define DPP_TOP0_DPP_CRC_VAL_A__DPP_CRC_ALPHA_MASK                                                            0xFFFFFFFFL
+//DPP_TOP0_DPP_CRC_CTRL
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_EN__SHIFT                                                              0x0
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_CONT_EN__SHIFT                                                         0x1
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING__SHIFT                                                0x2
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL__SHIFT                                                    0x3
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_SRC_SEL__SHIFT                                                         0x4
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_STEREO_EN__SHIFT                                                       0x6
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE__SHIFT                                                     0x7
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE__SHIFT                                                  0x9
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL__SHIFT                                                  0xb
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_MASK__SHIFT                                                            0x10
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_EN_MASK                                                                0x00000001L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_CONT_EN_MASK                                                           0x00000002L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING_MASK                                                  0x00000004L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL_MASK                                                      0x00000008L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_SRC_SEL_MASK                                                           0x00000030L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_STEREO_EN_MASK                                                         0x00000040L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE_MASK                                                       0x00000180L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE_MASK                                                    0x00000600L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL_MASK                                                    0x00003800L
+#define DPP_TOP0_DPP_CRC_CTRL__DPP_CRC_MASK_MASK                                                              0xFFFF0000L
+//DPP_TOP0_HOST_READ_CONTROL
+#define DPP_TOP0_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL__SHIFT                                             0x0
+#define DPP_TOP0_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL_MASK                                               0x000000FFL
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cnvc_cfg_dispdec
+//CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT
+#define CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT__SHIFT                                 0x0
+#define CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE__SHIFT                                   0x8
+#define CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT_MASK                                   0x0000007FL
+#define CNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE_MASK                                     0x00000100L
+//CNVC_CFG0_FORMAT_CONTROL
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_EXPANSION_MODE__SHIFT                                                0x0
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CNV16__SHIFT                                                         0x4
+#define CNVC_CFG0_FORMAT_CONTROL__ALPHA_EN__SHIFT                                                             0x8
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_BYPASS__SHIFT                                                          0xc
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN__SHIFT                                                0xd
+#define CNVC_CFG0_FORMAT_CONTROL__CLAMP_POSITIVE__SHIFT                                                       0x10
+#define CNVC_CFG0_FORMAT_CONTROL__CLAMP_POSITIVE_C__SHIFT                                                     0x11
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_UPDATE_PENDING__SHIFT                                                  0x14
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_R__SHIFT                                                    0x18
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_G__SHIFT                                                    0x1a
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_B__SHIFT                                                    0x1c
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_EXPANSION_MODE_MASK                                                  0x00000001L
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CNV16_MASK                                                           0x00000010L
+#define CNVC_CFG0_FORMAT_CONTROL__ALPHA_EN_MASK                                                               0x00000100L
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_BYPASS_MASK                                                            0x00001000L
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN_MASK                                                  0x00002000L
+#define CNVC_CFG0_FORMAT_CONTROL__CLAMP_POSITIVE_MASK                                                         0x00010000L
+#define CNVC_CFG0_FORMAT_CONTROL__CLAMP_POSITIVE_C_MASK                                                       0x00020000L
+#define CNVC_CFG0_FORMAT_CONTROL__CNVC_UPDATE_PENDING_MASK                                                    0x00100000L
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_R_MASK                                                      0x03000000L
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_G_MASK                                                      0x0C000000L
+#define CNVC_CFG0_FORMAT_CONTROL__FORMAT_CROSSBAR_B_MASK                                                      0x30000000L
+//CNVC_CFG0_FCNV_FP_BIAS_R
+#define CNVC_CFG0_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R__SHIFT                                                       0x0
+#define CNVC_CFG0_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R_MASK                                                         0x0007FFFFL
+//CNVC_CFG0_FCNV_FP_BIAS_G
+#define CNVC_CFG0_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G__SHIFT                                                       0x0
+#define CNVC_CFG0_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G_MASK                                                         0x0007FFFFL
+//CNVC_CFG0_FCNV_FP_BIAS_B
+#define CNVC_CFG0_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B__SHIFT                                                       0x0
+#define CNVC_CFG0_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B_MASK                                                         0x0007FFFFL
+//CNVC_CFG0_FCNV_FP_SCALE_R
+#define CNVC_CFG0_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R__SHIFT                                                     0x0
+#define CNVC_CFG0_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R_MASK                                                       0x0007FFFFL
+//CNVC_CFG0_FCNV_FP_SCALE_G
+#define CNVC_CFG0_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G__SHIFT                                                     0x0
+#define CNVC_CFG0_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G_MASK                                                       0x0007FFFFL
+//CNVC_CFG0_FCNV_FP_SCALE_B
+#define CNVC_CFG0_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B__SHIFT                                                     0x0
+#define CNVC_CFG0_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B_MASK                                                       0x0007FFFFL
+//CNVC_CFG0_COLOR_KEYER_CONTROL
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__COLOR_KEYER_EN__SHIFT                                                  0x0
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__LUMA_KEYER_EN__SHIFT                                                   0x1
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE__SHIFT                                                0x4
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__COLOR_KEYER_EN_MASK                                                    0x00000001L
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__LUMA_KEYER_EN_MASK                                                     0x00000002L
+#define CNVC_CFG0_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE_MASK                                                  0x00000030L
+//CNVC_CFG0_COLOR_KEYER_ALPHA
+#define CNVC_CFG0_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW__SHIFT                                             0x0
+#define CNVC_CFG0_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH__SHIFT                                            0x10
+#define CNVC_CFG0_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG0_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG0_COLOR_KEYER_RED
+#define CNVC_CFG0_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW__SHIFT                                                 0x0
+#define CNVC_CFG0_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH__SHIFT                                                0x10
+#define CNVC_CFG0_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW_MASK                                                   0x0000FFFFL
+#define CNVC_CFG0_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH_MASK                                                  0xFFFF0000L
+//CNVC_CFG0_COLOR_KEYER_GREEN
+#define CNVC_CFG0_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW__SHIFT                                             0x0
+#define CNVC_CFG0_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH__SHIFT                                            0x10
+#define CNVC_CFG0_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG0_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG0_COLOR_KEYER_BLUE
+#define CNVC_CFG0_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW__SHIFT                                               0x0
+#define CNVC_CFG0_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH__SHIFT                                              0x10
+#define CNVC_CFG0_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW_MASK                                                 0x0000FFFFL
+#define CNVC_CFG0_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH_MASK                                                0xFFFF0000L
+//CNVC_CFG0_ALPHA_2BIT_LUT01
+#define CNVC_CFG0_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0__SHIFT                                                    0x0
+#define CNVC_CFG0_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1__SHIFT                                                    0x10
+#define CNVC_CFG0_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0_MASK                                                      0x00000FFFL
+#define CNVC_CFG0_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1_MASK                                                      0x0FFF0000L
+//CNVC_CFG0_ALPHA_2BIT_LUT23
+#define CNVC_CFG0_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2__SHIFT                                                    0x0
+#define CNVC_CFG0_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3__SHIFT                                                    0x10
+#define CNVC_CFG0_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2_MASK                                                      0x00000FFFL
+#define CNVC_CFG0_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3_MASK                                                      0x0FFF0000L
+//CNVC_CFG0_PRE_DEALPHA
+#define CNVC_CFG0_PRE_DEALPHA__PRE_DEALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG0_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG0_PRE_DEALPHA__PRE_DEALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG0_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN_MASK                                                      0x00000010L
+//CNVC_CFG0_PRE_CSC_MODE
+#define CNVC_CFG0_PRE_CSC_MODE__PRE_CSC_MODE__SHIFT                                                           0x0
+#define CNVC_CFG0_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT__SHIFT                                                   0x2
+#define CNVC_CFG0_PRE_CSC_MODE__PRE_CSC_MODE_MASK                                                             0x00000003L
+#define CNVC_CFG0_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT_MASK                                                     0x0000000CL
+//CNVC_CFG0_PRE_CSC_C11_C12
+#define CNVC_CFG0_PRE_CSC_C11_C12__PRE_CSC_C11__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C11_C12__PRE_CSC_C12__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C11_C12__PRE_CSC_C11_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C11_C12__PRE_CSC_C12_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_C13_C14
+#define CNVC_CFG0_PRE_CSC_C13_C14__PRE_CSC_C13__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C13_C14__PRE_CSC_C14__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C13_C14__PRE_CSC_C13_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C13_C14__PRE_CSC_C14_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_C21_C22
+#define CNVC_CFG0_PRE_CSC_C21_C22__PRE_CSC_C21__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C21_C22__PRE_CSC_C22__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C21_C22__PRE_CSC_C21_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C21_C22__PRE_CSC_C22_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_C23_C24
+#define CNVC_CFG0_PRE_CSC_C23_C24__PRE_CSC_C23__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C23_C24__PRE_CSC_C24__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C23_C24__PRE_CSC_C23_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C23_C24__PRE_CSC_C24_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_C31_C32
+#define CNVC_CFG0_PRE_CSC_C31_C32__PRE_CSC_C31__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C31_C32__PRE_CSC_C32__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C31_C32__PRE_CSC_C31_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C31_C32__PRE_CSC_C32_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_C33_C34
+#define CNVC_CFG0_PRE_CSC_C33_C34__PRE_CSC_C33__SHIFT                                                         0x0
+#define CNVC_CFG0_PRE_CSC_C33_C34__PRE_CSC_C34__SHIFT                                                         0x10
+#define CNVC_CFG0_PRE_CSC_C33_C34__PRE_CSC_C33_MASK                                                           0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_C33_C34__PRE_CSC_C34_MASK                                                           0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C11_C12
+#define CNVC_CFG0_PRE_CSC_B_C11_C12__PRE_CSC_B_C11__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C11_C12__PRE_CSC_B_C12__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C11_C12__PRE_CSC_B_C11_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C11_C12__PRE_CSC_B_C12_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C13_C14
+#define CNVC_CFG0_PRE_CSC_B_C13_C14__PRE_CSC_B_C13__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C13_C14__PRE_CSC_B_C14__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C13_C14__PRE_CSC_B_C13_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C13_C14__PRE_CSC_B_C14_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C21_C22
+#define CNVC_CFG0_PRE_CSC_B_C21_C22__PRE_CSC_B_C21__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C21_C22__PRE_CSC_B_C22__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C21_C22__PRE_CSC_B_C21_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C21_C22__PRE_CSC_B_C22_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C23_C24
+#define CNVC_CFG0_PRE_CSC_B_C23_C24__PRE_CSC_B_C23__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C23_C24__PRE_CSC_B_C24__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C23_C24__PRE_CSC_B_C23_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C23_C24__PRE_CSC_B_C24_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C31_C32
+#define CNVC_CFG0_PRE_CSC_B_C31_C32__PRE_CSC_B_C31__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C31_C32__PRE_CSC_B_C32__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C31_C32__PRE_CSC_B_C31_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C31_C32__PRE_CSC_B_C32_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_PRE_CSC_B_C33_C34
+#define CNVC_CFG0_PRE_CSC_B_C33_C34__PRE_CSC_B_C33__SHIFT                                                     0x0
+#define CNVC_CFG0_PRE_CSC_B_C33_C34__PRE_CSC_B_C34__SHIFT                                                     0x10
+#define CNVC_CFG0_PRE_CSC_B_C33_C34__PRE_CSC_B_C33_MASK                                                       0x0000FFFFL
+#define CNVC_CFG0_PRE_CSC_B_C33_C34__PRE_CSC_B_C34_MASK                                                       0xFFFF0000L
+//CNVC_CFG0_CNVC_COEF_FORMAT
+#define CNVC_CFG0_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT__SHIFT                                                0x0
+#define CNVC_CFG0_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT_MASK                                                  0x00000001L
+//CNVC_CFG0_PRE_DEGAM
+#define CNVC_CFG0_PRE_DEGAM__PRE_DEGAM_MODE__SHIFT                                                            0x0
+#define CNVC_CFG0_PRE_DEGAM__PRE_DEGAM_SELECT__SHIFT                                                          0x4
+#define CNVC_CFG0_PRE_DEGAM__PRE_DEGAM_MODE_MASK                                                              0x00000003L
+#define CNVC_CFG0_PRE_DEGAM__PRE_DEGAM_SELECT_MASK                                                            0x00000070L
+//CNVC_CFG0_PRE_REALPHA
+#define CNVC_CFG0_PRE_REALPHA__PRE_REALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG0_PRE_REALPHA__PRE_REALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG0_PRE_REALPHA__PRE_REALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG0_PRE_REALPHA__PRE_REALPHA_ABLND_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cm_cur_dispdec
+//CM_CUR0_CURSOR0_CONTROL
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_ENABLE__SHIFT                                                           0x0
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_EXPANSION_MODE__SHIFT                                                   0x1
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_PIX_INV_MODE__SHIFT                                                     0x2
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_ROM_EN__SHIFT                                                           0x3
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_MODE__SHIFT                                                             0x4
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN__SHIFT                                               0x7
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_UPDATE_PENDING__SHIFT                                                   0x10
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_ENABLE_MASK                                                             0x00000001L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_EXPANSION_MODE_MASK                                                     0x00000002L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_PIX_INV_MODE_MASK                                                       0x00000004L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_ROM_EN_MASK                                                             0x00000008L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_MODE_MASK                                                               0x00000070L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN_MASK                                                 0x00000080L
+#define CM_CUR0_CURSOR0_CONTROL__CUR0_UPDATE_PENDING_MASK                                                     0x00010000L
+//CM_CUR0_CURSOR0_COLOR0
+#define CM_CUR0_CURSOR0_COLOR0__CUR0_COLOR0__SHIFT                                                            0x0
+#define CM_CUR0_CURSOR0_COLOR0__CUR0_COLOR0_MASK                                                              0x00FFFFFFL
+//CM_CUR0_CURSOR0_COLOR1
+#define CM_CUR0_CURSOR0_COLOR1__CUR0_COLOR1__SHIFT                                                            0x0
+#define CM_CUR0_CURSOR0_COLOR1__CUR0_COLOR1_MASK                                                              0x00FFFFFFL
+//CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y__SHIFT                                           0x0
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y__SHIFT                                            0x10
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y_MASK                                             0x0000FFFFL
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y_MASK                                              0xFFFF0000L
+//CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB__SHIFT                                   0x0
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB__SHIFT                                    0x10
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB_MASK                                     0x0000FFFFL
+#define CM_CUR0_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB_MASK                                      0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_MODE
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE__SHIFT                                                     0x0
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT__SHIFT                                             0x2
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT__SHIFT                                              0x4
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_MASK                                                       0x00000003L
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT_MASK                                               0x0000000CL
+#define CM_CUR0_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT_MASK                                                0x00000010L
+//CM_CUR0_CUR0_MATRIX_C11_C12_A
+#define CM_CUR0_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C13_C14_A
+#define CM_CUR0_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C21_C22_A
+#define CM_CUR0_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C23_C24_A
+#define CM_CUR0_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C31_C32_A
+#define CM_CUR0_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C33_C34_A
+#define CM_CUR0_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C11_C12_B
+#define CM_CUR0_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C13_C14_B
+#define CM_CUR0_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C21_C22_B
+#define CM_CUR0_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C23_C24_B
+#define CM_CUR0_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C31_C32_B
+#define CM_CUR0_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B_MASK                                                 0xFFFF0000L
+//CM_CUR0_CUR0_MATRIX_C33_C34_B
+#define CM_CUR0_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B__SHIFT                                               0x0
+#define CM_CUR0_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B__SHIFT                                               0x10
+#define CM_CUR0_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B_MASK                                                 0x0000FFFFL
+#define CM_CUR0_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B_MASK                                                 0xFFFF0000L
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dscl_dispdec
+//DSCL0_SCL_COEF_RAM_TAP_SELECT
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX__SHIFT                                       0x0
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE__SHIFT                                              0x8
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE__SHIFT                                        0x10
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX_MASK                                         0x00000003L
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE_MASK                                                0x00003F00L
+#define DSCL0_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE_MASK                                          0x00070000L
+//DSCL0_SCL_COEF_RAM_TAP_DATA
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF__SHIFT                                        0x0
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN__SHIFT                                     0xf
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF__SHIFT                                         0x10
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN__SHIFT                                      0x1f
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_MASK                                          0x00003FFFL
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN_MASK                                       0x00008000L
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_MASK                                           0x3FFF0000L
+#define DSCL0_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN_MASK                                        0x80000000L
+//DSCL0_SCL_MODE
+#define DSCL0_SCL_MODE__DSCL_MODE__SHIFT                                                                      0x0
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT__SHIFT                                                            0x8
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT__SHIFT                                                    0xc
+#define DSCL0_SCL_MODE__SCL_CHROMA_COEF_MODE__SHIFT                                                           0x10
+#define DSCL0_SCL_MODE__SCL_ALPHA_COEF_MODE__SHIFT                                                            0x14
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT_RD__SHIFT                                                         0x18
+#define DSCL0_SCL_MODE__DSCL_MODE_MASK                                                                        0x00000007L
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT_MASK                                                              0x00000100L
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT_MASK                                                      0x00001000L
+#define DSCL0_SCL_MODE__SCL_CHROMA_COEF_MODE_MASK                                                             0x00010000L
+#define DSCL0_SCL_MODE__SCL_ALPHA_COEF_MODE_MASK                                                              0x00100000L
+#define DSCL0_SCL_MODE__SCL_COEF_RAM_SELECT_RD_MASK                                                           0x01000000L
+//DSCL0_SCL_TAP_CONTROL
+#define DSCL0_SCL_TAP_CONTROL__SCL_V_NUM_TAPS__SHIFT                                                          0x0
+#define DSCL0_SCL_TAP_CONTROL__SCL_H_NUM_TAPS__SHIFT                                                          0x4
+#define DSCL0_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C__SHIFT                                                        0x8
+#define DSCL0_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C__SHIFT                                                        0xc
+#define DSCL0_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_MASK                                                            0x00000007L
+#define DSCL0_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_MASK                                                            0x00000070L
+#define DSCL0_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C_MASK                                                          0x00000700L
+#define DSCL0_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C_MASK                                                          0x00007000L
+//DSCL0_DSCL_CONTROL
+#define DSCL0_DSCL_CONTROL__SCL_BOUNDARY_MODE__SHIFT                                                          0x0
+#define DSCL0_DSCL_CONTROL__SCL_BOUNDARY_MODE_MASK                                                            0x00000001L
+//DSCL0_DSCL_2TAP_CONTROL
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x0
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN__SHIFT                                                   0x4
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR__SHIFT                                               0x8
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x10
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN__SHIFT                                                   0x14
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR__SHIFT                                               0x18
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN_MASK                                             0x00000001L
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN_MASK                                                     0x00000010L
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR_MASK                                                 0x00000700L
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN_MASK                                             0x00010000L
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN_MASK                                                     0x00100000L
+#define DSCL0_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR_MASK                                                 0x07000000L
+//DSCL0_SCL_MANUAL_REPLICATE_CONTROL
+#define DSCL0_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR__SHIFT                              0x0
+#define DSCL0_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR__SHIFT                              0x8
+#define DSCL0_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR_MASK                                0x0000000FL
+#define DSCL0_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR_MASK                                0x00000F00L
+//DSCL0_SCL_HORZ_FILTER_SCALE_RATIO
+#define DSCL0_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL0_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL0_SCL_HORZ_FILTER_INIT
+#define DSCL0_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL0_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT__SHIFT                                                     0x18
+#define DSCL0_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL0_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT_MASK                                                       0x0F000000L
+//DSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C
+#define DSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL0_SCL_HORZ_FILTER_INIT_C
+#define DSCL0_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL0_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL0_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL0_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL0_SCL_VERT_FILTER_SCALE_RATIO
+#define DSCL0_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL0_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL0_SCL_VERT_FILTER_INIT
+#define DSCL0_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL0_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT__SHIFT                                                     0x18
+#define DSCL0_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL0_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT_MASK                                                       0x0F000000L
+//DSCL0_SCL_VERT_FILTER_INIT_BOT
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT__SHIFT                                            0x0
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT__SHIFT                                             0x18
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT_MASK                                              0x00FFFFFFL
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT_MASK                                               0x0F000000L
+//DSCL0_SCL_VERT_FILTER_SCALE_RATIO_C
+#define DSCL0_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL0_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL0_SCL_VERT_FILTER_INIT_C
+#define DSCL0_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL0_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL0_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL0_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL0_SCL_VERT_FILTER_INIT_BOT_C
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C__SHIFT                                        0x0
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C__SHIFT                                         0x18
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C_MASK                                          0x00FFFFFFL
+#define DSCL0_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C_MASK                                           0x0F000000L
+//DSCL0_SCL_BLACK_COLOR
+#define DSCL0_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y__SHIFT                                                   0x0
+#define DSCL0_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR__SHIFT                                                    0x10
+#define DSCL0_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y_MASK                                                     0x0000FFFFL
+#define DSCL0_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR_MASK                                                      0xFFFF0000L
+//DSCL0_DSCL_UPDATE
+#define DSCL0_DSCL_UPDATE__SCL_UPDATE_PENDING__SHIFT                                                          0x0
+#define DSCL0_DSCL_UPDATE__SCL_UPDATE_PENDING_MASK                                                            0x00000001L
+//DSCL0_DSCL_AUTOCAL
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_MODE__SHIFT                                                               0x0
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE__SHIFT                                                           0x8
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_PIPE_ID__SHIFT                                                            0xc
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_MODE_MASK                                                                 0x00000003L
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE_MASK                                                             0x00000300L
+#define DSCL0_DSCL_AUTOCAL__AUTOCAL_PIPE_ID_MASK                                                              0x00003000L
+//DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT
+#define DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT__SHIFT                                         0x0
+#define DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT__SHIFT                                          0x10
+#define DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT_MASK                                           0x00003FFFL
+#define DSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT_MASK                                            0x3FFF0000L
+//DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM
+#define DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM__SHIFT                                        0x0
+#define DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP__SHIFT                                           0x10
+#define DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM_MASK                                          0x00003FFFL
+#define DSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP_MASK                                             0x3FFF0000L
+//DSCL0_OTG_H_BLANK
+#define DSCL0_OTG_H_BLANK__OTG_H_BLANK_START__SHIFT                                                           0x0
+#define DSCL0_OTG_H_BLANK__OTG_H_BLANK_END__SHIFT                                                             0x10
+#define DSCL0_OTG_H_BLANK__OTG_H_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL0_OTG_H_BLANK__OTG_H_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL0_OTG_V_BLANK
+#define DSCL0_OTG_V_BLANK__OTG_V_BLANK_START__SHIFT                                                           0x0
+#define DSCL0_OTG_V_BLANK__OTG_V_BLANK_END__SHIFT                                                             0x10
+#define DSCL0_OTG_V_BLANK__OTG_V_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL0_OTG_V_BLANK__OTG_V_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL0_RECOUT_START
+#define DSCL0_RECOUT_START__RECOUT_START_X__SHIFT                                                             0x0
+#define DSCL0_RECOUT_START__RECOUT_START_Y__SHIFT                                                             0x10
+#define DSCL0_RECOUT_START__RECOUT_START_X_MASK                                                               0x00003FFFL
+#define DSCL0_RECOUT_START__RECOUT_START_Y_MASK                                                               0x3FFF0000L
+//DSCL0_RECOUT_SIZE
+#define DSCL0_RECOUT_SIZE__RECOUT_WIDTH__SHIFT                                                                0x0
+#define DSCL0_RECOUT_SIZE__RECOUT_HEIGHT__SHIFT                                                               0x10
+#define DSCL0_RECOUT_SIZE__RECOUT_WIDTH_MASK                                                                  0x00003FFFL
+#define DSCL0_RECOUT_SIZE__RECOUT_HEIGHT_MASK                                                                 0x3FFF0000L
+//DSCL0_MPC_SIZE
+#define DSCL0_MPC_SIZE__MPC_WIDTH__SHIFT                                                                      0x0
+#define DSCL0_MPC_SIZE__MPC_HEIGHT__SHIFT                                                                     0x10
+#define DSCL0_MPC_SIZE__MPC_WIDTH_MASK                                                                        0x00003FFFL
+#define DSCL0_MPC_SIZE__MPC_HEIGHT_MASK                                                                       0x3FFF0000L
+//DSCL0_LB_DATA_FORMAT
+#define DSCL0_LB_DATA_FORMAT__INTERLEAVE_EN__SHIFT                                                            0x0
+#define DSCL0_LB_DATA_FORMAT__ALPHA_EN__SHIFT                                                                 0x4
+#define DSCL0_LB_DATA_FORMAT__INTERLEAVE_EN_MASK                                                              0x00000001L
+#define DSCL0_LB_DATA_FORMAT__ALPHA_EN_MASK                                                                   0x00000010L
+//DSCL0_LB_MEMORY_CTRL
+#define DSCL0_LB_MEMORY_CTRL__MEMORY_CONFIG__SHIFT                                                            0x0
+#define DSCL0_LB_MEMORY_CTRL__LB_MAX_PARTITIONS__SHIFT                                                        0x8
+#define DSCL0_LB_MEMORY_CTRL__LB_NUM_PARTITIONS__SHIFT                                                        0x10
+#define DSCL0_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C__SHIFT                                                      0x18
+#define DSCL0_LB_MEMORY_CTRL__MEMORY_CONFIG_MASK                                                              0x00000003L
+#define DSCL0_LB_MEMORY_CTRL__LB_MAX_PARTITIONS_MASK                                                          0x00003F00L
+#define DSCL0_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_MASK                                                          0x007F0000L
+#define DSCL0_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C_MASK                                                        0x7F000000L
+//DSCL0_LB_V_COUNTER
+#define DSCL0_LB_V_COUNTER__V_COUNTER__SHIFT                                                                  0x0
+#define DSCL0_LB_V_COUNTER__V_COUNTER_C__SHIFT                                                                0x10
+#define DSCL0_LB_V_COUNTER__V_COUNTER_MASK                                                                    0x00001FFFL
+#define DSCL0_LB_V_COUNTER__V_COUNTER_C_MASK                                                                  0x1FFF0000L
+//DSCL0_DSCL_MEM_PWR_CTRL
+#define DSCL0_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DSCL0_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS__SHIFT                                                       0x2
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE__SHIFT                                                   0x4
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS__SHIFT                                                     0x6
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE__SHIFT                                                   0x8
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS__SHIFT                                                     0xa
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE__SHIFT                                                   0xc
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS__SHIFT                                                     0xe
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE__SHIFT                                                   0x10
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS__SHIFT                                                     0x12
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE__SHIFT                                                   0x14
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS__SHIFT                                                     0x16
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE__SHIFT                                                   0x18
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS__SHIFT                                                     0x1a
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE__SHIFT                                                       0x1c
+#define DSCL0_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS_MASK                                                         0x00000004L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE_MASK                                                     0x00000030L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS_MASK                                                       0x00000040L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE_MASK                                                     0x00000300L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS_MASK                                                       0x00000400L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE_MASK                                                     0x00003000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS_MASK                                                       0x00004000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE_MASK                                                     0x00030000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS_MASK                                                       0x00040000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE_MASK                                                     0x00300000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS_MASK                                                       0x00400000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE_MASK                                                     0x03000000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS_MASK                                                       0x04000000L
+#define DSCL0_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE_MASK                                                         0x10000000L
+//DSCL0_DSCL_MEM_PWR_STATUS
+#define DSCL0_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE__SHIFT                                                   0x0
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE__SHIFT                                                 0x2
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE__SHIFT                                                 0x4
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE__SHIFT                                                 0x6
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE__SHIFT                                                 0x8
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE__SHIFT                                                 0xa
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE__SHIFT                                                 0xc
+#define DSCL0_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE_MASK                                                     0x00000003L
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE_MASK                                                   0x0000000CL
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE_MASK                                                   0x00000030L
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE_MASK                                                   0x000000C0L
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE_MASK                                                   0x00000300L
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE_MASK                                                   0x00000C00L
+#define DSCL0_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE_MASK                                                   0x00003000L
+//DSCL0_OBUF_CONTROL
+#define DSCL0_OBUF_CONTROL__OBUF_BYPASS__SHIFT                                                                0x0
+#define DSCL0_OBUF_CONTROL__OBUF_USE_FULL_BUFFER__SHIFT                                                       0x1
+#define DSCL0_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH__SHIFT                                                  0x2
+#define DSCL0_OBUF_CONTROL__OBUF_OUT_HOLD_CNT__SHIFT                                                          0x4
+#define DSCL0_OBUF_CONTROL__OBUF_BYPASS_MASK                                                                  0x00000001L
+#define DSCL0_OBUF_CONTROL__OBUF_USE_FULL_BUFFER_MASK                                                         0x00000002L
+#define DSCL0_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH_MASK                                                    0x00000004L
+#define DSCL0_OBUF_CONTROL__OBUF_OUT_HOLD_CNT_MASK                                                            0x000000F0L
+//DSCL0_OBUF_MEM_PWR_CTRL
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE__SHIFT                                                    0x0
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS__SHIFT                                                      0x2
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE__SHIFT                                                     0x8
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE__SHIFT                                                    0x10
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE_MASK                                                      0x00000003L
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS_MASK                                                        0x00000004L
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE_MASK                                                       0x00000100L
+#define DSCL0_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE_MASK                                                      0x00030000L
+//DSCL0_DSCL_EASF_H_MODE
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_EN__SHIFT                                                          0x0
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_EN_MASK                                                            0x00000001L
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL0_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL0_DSCL_EASF_V_MODE
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_EN__SHIFT                                                          0x0
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_EN_MASK                                                            0x00000001L
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL0_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL0_DSCL_SC_MODE
+#define DSCL0_DSCL_SC_MODE__SCL_SC_MATRIX_MODE__SHIFT                                                         0x0
+#define DSCL0_DSCL_SC_MODE__SCL_SC_LTONL_EN__SHIFT                                                            0x8
+#define DSCL0_DSCL_SC_MODE__SCL_SC_MATRIX_MODE_MASK                                                           0x00000001L
+#define DSCL0_DSCL_SC_MODE__SCL_SC_LTONL_EN_MASK                                                              0x00000100L
+//DSCL0_DSCL_SC_MATRIX_C0C1
+#define DSCL0_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0__SHIFT                                                    0x0
+#define DSCL0_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1__SHIFT                                                    0x10
+#define DSCL0_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0_MASK                                                      0x0000FFFFL
+#define DSCL0_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1_MASK                                                      0xFFFF0000L
+//DSCL0_DSCL_SC_MATRIX_C2C3
+#define DSCL0_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2__SHIFT                                                    0x0
+#define DSCL0_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3__SHIFT                                                    0x10
+#define DSCL0_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2_MASK                                                      0x0000FFFFL
+#define DSCL0_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3_MASK                                                      0xFFFF0000L
+//DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL0_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL0_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT__SHIFT                    0x0
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL__SHIFT                    0x10
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT_MASK                      0x0000FFFFL
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL_MASK                      0xFFFF0000L
+//DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE__SHIFT                     0x0
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE__SHIFT                    0x10
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE_MASK                       0x0000FFFFL
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE_MASK                      0xFFFF0000L
+//DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE__SHIFT                    0x0
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET__SHIFT                   0x10
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE_MASK                      0x0000FFFFL
+#define DSCL0_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET_MASK                     0xFFFF0000L
+//DSCL0_DSCL_EASF_RINGEST_FORCE
+#define DSCL0_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE__SHIFT                                        0x0
+#define DSCL0_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE__SHIFT                                        0x10
+#define DSCL0_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE_MASK                                          0x0000FFFFL
+#define DSCL0_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE_MASK                                          0xFFFF0000L
+//DSCL0_DSCL_EASF_H_BF_CNTL
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN__SHIFT                                                   0x0
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE__SHIFT                                                 0x8
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE__SHIFT                                                 0x10
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN_MASK                                                     0x00000001L
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL0_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA__SHIFT                                         0x0
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB__SHIFT                                         0x8
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA__SHIFT                                         0x10
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB__SHIFT                                         0x18
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA_MASK                                           0x003F0000L
+#define DSCL0_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB_MASK                                           0x3F000000L
+//DSCL0_DSCL_EASF_V_BF_CNTL
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN__SHIFT                                                   0x0
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE__SHIFT                                                 0x8
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE__SHIFT                                                 0x10
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN_MASK                                                     0x00000001L
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL0_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA__SHIFT                                         0x0
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB__SHIFT                                         0x8
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA__SHIFT                                         0x10
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB__SHIFT                                         0x18
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA_MASK                                           0x003F0000L
+#define DSCL0_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB_MASK                                           0x3F000000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG1
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG2
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG3
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG4
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG5
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG6
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_H_BF1_PWL_SEG7
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG1
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG2
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG3
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG4
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG5
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG6
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL0_DSCL_EASF_V_BF1_PWL_SEG7
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL0_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG1
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG2
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG3
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG4
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_H_BF3_PWL_SEG5
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG1
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG2
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG3
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG4
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL0_DSCL_EASF_V_BF3_PWL_SEG5
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL0_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL0_ISHARP_MODE
+#define DSCL0_ISHARP_MODE__ISHARP_EN__SHIFT                                                                   0x0
+#define DSCL0_ISHARP_MODE__ISHARP_NOISEDET_EN__SHIFT                                                          0x4
+#define DSCL0_ISHARP_MODE__ISHARP_NOISEDET_MODE__SHIFT                                                        0x5
+#define DSCL0_ISHARP_MODE__ISHARP_LBA_MODE__SHIFT                                                             0x9
+#define DSCL0_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT__SHIFT                                                     0xa
+#define DSCL0_ISHARP_MODE__ISHARP_FMT_MODE__SHIFT                                                             0xb
+#define DSCL0_ISHARP_MODE__ISHARP_FMT_NORM__SHIFT                                                             0xc
+#define DSCL0_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT__SHIFT                                             0x1c
+#define DSCL0_ISHARP_MODE__ISHARP_EN_MASK                                                                     0x00000001L
+#define DSCL0_ISHARP_MODE__ISHARP_NOISEDET_EN_MASK                                                            0x00000010L
+#define DSCL0_ISHARP_MODE__ISHARP_NOISEDET_MODE_MASK                                                          0x00000060L
+#define DSCL0_ISHARP_MODE__ISHARP_LBA_MODE_MASK                                                               0x00000200L
+#define DSCL0_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_MASK                                                       0x00000400L
+#define DSCL0_ISHARP_MODE__ISHARP_FMT_MODE_MASK                                                               0x00000800L
+#define DSCL0_ISHARP_MODE__ISHARP_FMT_NORM_MASK                                                               0x0FFFF000L
+#define DSCL0_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT_MASK                                               0x10000000L
+//DSCL0_ISHARP_DELTA_CTRL
+#define DSCL0_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT__SHIFT                                          0x0
+#define DSCL0_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT_MASK                                            0x00000001L
+//DSCL0_ISHARP_DELTA_INDEX
+#define DSCL0_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX__SHIFT                                                   0x0
+#define DSCL0_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX_MASK                                                     0x0000001FL
+//DSCL0_ISHARP_DELTA_DATA
+#define DSCL0_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA__SHIFT                                                     0x0
+#define DSCL0_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA_MASK                                                       0xFFFFFFFFL
+//DSCL0_ISHARP_NLDELTA_SOFT_CLIP
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P__SHIFT                                      0x0
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P__SHIFT                                   0x1
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P__SHIFT                                   0x8
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N__SHIFT                                      0x10
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N__SHIFT                                   0x11
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N__SHIFT                                   0x18
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P_MASK                                        0x00000001L
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P_MASK                                     0x000000FEL
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P_MASK                                     0x0000FF00L
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N_MASK                                        0x00010000L
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N_MASK                                     0x00FE0000L
+#define DSCL0_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N_MASK                                     0xFF000000L
+//DSCL0_ISHARP_NOISEDET_THRESHOLD
+#define DSCL0_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE__SHIFT                                         0x0
+#define DSCL0_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE__SHIFT                                         0x10
+#define DSCL0_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE_MASK                                           0x000003FFL
+#define DSCL0_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE_MASK                                           0x03FF0000L
+//DSCL0_ISHARP_NOISE_GAIN_PWL
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN__SHIFT                                      0x0
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN__SHIFT                                        0x8
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE__SHIFT                                         0x10
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN_MASK                                        0x0000001FL
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN_MASK                                          0x00001F00L
+#define DSCL0_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE_MASK                                           0x3FFF0000L
+//DSCL0_ISHARP_LBA_PWL_SEG0
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0__SHIFT                                           0x14
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0_MASK                                              0x0003F000L
+#define DSCL0_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0_MASK                                             0x1FF00000L
+//DSCL0_ISHARP_LBA_PWL_SEG1
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1__SHIFT                                           0x14
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1_MASK                                              0x0003F000L
+#define DSCL0_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1_MASK                                             0x1FF00000L
+//DSCL0_ISHARP_LBA_PWL_SEG2
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2__SHIFT                                           0x14
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2_MASK                                              0x0003F000L
+#define DSCL0_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2_MASK                                             0x1FF00000L
+//DSCL0_ISHARP_LBA_PWL_SEG3
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3__SHIFT                                           0x14
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3_MASK                                              0x0003F000L
+#define DSCL0_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3_MASK                                             0x1FF00000L
+//DSCL0_ISHARP_LBA_PWL_SEG4
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4__SHIFT                                           0x14
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4_MASK                                              0x0003F000L
+#define DSCL0_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4_MASK                                             0x1FF00000L
+//DSCL0_ISHARP_LBA_PWL_SEG5
+#define DSCL0_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5__SHIFT                                              0x0
+#define DSCL0_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5__SHIFT                                            0xc
+#define DSCL0_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5_MASK                                                0x000003FFL
+#define DSCL0_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5_MASK                                              0x0003F000L
+//DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE__SHIFT                            0x0
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS__SHIFT                              0x2
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE__SHIFT                            0x4
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE_MASK                              0x00000003L
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS_MASK                                0x00000004L
+#define DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE_MASK                              0x00000030L
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cm_dispdec
+//CM0_CM_CONTROL
+#define CM0_CM_CONTROL__CM_BYPASS__SHIFT                                                                      0x0
+#define CM0_CM_CONTROL__CM_UPDATE_PENDING__SHIFT                                                              0x8
+#define CM0_CM_CONTROL__CM_BYPASS_MASK                                                                        0x00000001L
+#define CM0_CM_CONTROL__CM_UPDATE_PENDING_MASK                                                                0x00000100L
+//CM0_CM_POST_CSC_CONTROL
+#define CM0_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE__SHIFT                                                      0x0
+#define CM0_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT__SHIFT                                              0x2
+#define CM0_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_MASK                                                        0x00000003L
+#define CM0_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT_MASK                                                0x0000000CL
+//CM0_CM_POST_CSC_C11_C12
+#define CM0_CM_POST_CSC_C11_C12__CM_POST_CSC_C11__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C11_C12__CM_POST_CSC_C12__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C11_C12__CM_POST_CSC_C11_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C11_C12__CM_POST_CSC_C12_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_C13_C14
+#define CM0_CM_POST_CSC_C13_C14__CM_POST_CSC_C13__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C13_C14__CM_POST_CSC_C14__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C13_C14__CM_POST_CSC_C13_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C13_C14__CM_POST_CSC_C14_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_C21_C22
+#define CM0_CM_POST_CSC_C21_C22__CM_POST_CSC_C21__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C21_C22__CM_POST_CSC_C22__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C21_C22__CM_POST_CSC_C21_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C21_C22__CM_POST_CSC_C22_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_C23_C24
+#define CM0_CM_POST_CSC_C23_C24__CM_POST_CSC_C23__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C23_C24__CM_POST_CSC_C24__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C23_C24__CM_POST_CSC_C23_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C23_C24__CM_POST_CSC_C24_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_C31_C32
+#define CM0_CM_POST_CSC_C31_C32__CM_POST_CSC_C31__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C31_C32__CM_POST_CSC_C32__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C31_C32__CM_POST_CSC_C31_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C31_C32__CM_POST_CSC_C32_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_C33_C34
+#define CM0_CM_POST_CSC_C33_C34__CM_POST_CSC_C33__SHIFT                                                       0x0
+#define CM0_CM_POST_CSC_C33_C34__CM_POST_CSC_C34__SHIFT                                                       0x10
+#define CM0_CM_POST_CSC_C33_C34__CM_POST_CSC_C33_MASK                                                         0x0000FFFFL
+#define CM0_CM_POST_CSC_C33_C34__CM_POST_CSC_C34_MASK                                                         0xFFFF0000L
+//CM0_CM_POST_CSC_B_C11_C12
+#define CM0_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12_MASK                                                     0xFFFF0000L
+//CM0_CM_POST_CSC_B_C13_C14
+#define CM0_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14_MASK                                                     0xFFFF0000L
+//CM0_CM_POST_CSC_B_C21_C22
+#define CM0_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22_MASK                                                     0xFFFF0000L
+//CM0_CM_POST_CSC_B_C23_C24
+#define CM0_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24_MASK                                                     0xFFFF0000L
+//CM0_CM_POST_CSC_B_C31_C32
+#define CM0_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32_MASK                                                     0xFFFF0000L
+//CM0_CM_POST_CSC_B_C33_C34
+#define CM0_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33__SHIFT                                                   0x0
+#define CM0_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34__SHIFT                                                   0x10
+#define CM0_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33_MASK                                                     0x0000FFFFL
+#define CM0_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34_MASK                                                     0xFFFF0000L
+//CM0_CM_BIAS_CR_R
+#define CM0_CM_BIAS_CR_R__CM_BIAS_CR_R__SHIFT                                                                 0x0
+#define CM0_CM_BIAS_CR_R__CM_BIAS_CR_R_MASK                                                                   0x0000FFFFL
+//CM0_CM_BIAS_Y_G_CB_B
+#define CM0_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G__SHIFT                                                              0x0
+#define CM0_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B__SHIFT                                                             0x10
+#define CM0_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G_MASK                                                                0x0000FFFFL
+#define CM0_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B_MASK                                                               0xFFFF0000L
+//CM0_CM_GAMCOR_CONTROL
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE__SHIFT                                                          0x0
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT__SHIFT                                                        0x2
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE__SHIFT                                                   0x3
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT__SHIFT                                                  0x4
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT__SHIFT                                                0x6
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_MASK                                                            0x00000003L
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_MASK                                                          0x00000004L
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE_MASK                                                     0x00000008L
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT_MASK                                                    0x00000030L
+#define CM0_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT_MASK                                                  0x00000040L
+//CM0_CM_GAMCOR_LUT_INDEX
+#define CM0_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX__SHIFT                                                   0x0
+#define CM0_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX_MASK                                                     0x000001FFL
+//CM0_CM_GAMCOR_LUT_DATA
+#define CM0_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA__SHIFT                                                     0x0
+#define CM0_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA_MASK                                                       0x0003FFFFL
+//CM0_CM_GAMCOR_LUT_CONTROL
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK__SHIFT                                      0x0
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL__SHIFT                                        0x3
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG__SHIFT                                              0x5
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL__SHIFT                                              0x6
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE__SHIFT                                           0x7
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK_MASK                                        0x00000007L
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL_MASK                                          0x00000018L
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG_MASK                                                0x00000020L
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL_MASK                                                0x00000040L
+#define CM0_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE_MASK                                             0x00000080L
+//CM0_CM_GAMCOR_RAMA_START_CNTL_B
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMA_START_CNTL_G
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMA_START_CNTL_R
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_G
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_R
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_END_CNTL1_B
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_END_CNTL2_B
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMA_END_CNTL1_G
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_END_CNTL2_G
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMA_END_CNTL1_R
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMA_END_CNTL2_R
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMA_OFFSET_B
+#define CM0_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMA_OFFSET_G
+#define CM0_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMA_OFFSET_R
+#define CM0_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMA_REGION_0_1
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_2_3
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_4_5
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_6_7
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_8_9
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_10_11
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_12_13
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_14_15
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_16_17
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_18_19
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_20_21
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_22_23
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_24_25
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_26_27
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_28_29
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_30_31
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMA_REGION_32_33
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_START_CNTL_B
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMB_START_CNTL_G
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMB_START_CNTL_R
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R__SHIFT                             0x0
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM0_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM0_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_B
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_G
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_R
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM0_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_END_CNTL1_B
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_END_CNTL2_B
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMB_END_CNTL1_G
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_END_CNTL2_G
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMB_END_CNTL1_R
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM0_CM_GAMCOR_RAMB_END_CNTL2_R
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R__SHIFT                                0x0
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM0_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM0_CM_GAMCOR_RAMB_OFFSET_B
+#define CM0_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMB_OFFSET_G
+#define CM0_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMB_OFFSET_R
+#define CM0_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R__SHIFT                                           0x0
+#define CM0_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R_MASK                                             0x0007FFFFL
+//CM0_CM_GAMCOR_RAMB_REGION_0_1
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_2_3
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_4_5
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_6_7
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_8_9
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_10_11
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_12_13
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_14_15
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_16_17
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_18_19
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_20_21
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_22_23
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_24_25
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_26_27
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_28_29
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_30_31
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_GAMCOR_RAMB_REGION_32_33
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM0_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM0_CM_HDR_MULT_COEF
+#define CM0_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF__SHIFT                                                         0x0
+#define CM0_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF_MASK                                                           0x0007FFFFL
+//CM0_CM_MEM_PWR_CTRL
+#define CM0_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE__SHIFT                                                      0x0
+#define CM0_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS__SHIFT                                                        0x2
+#define CM0_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE__SHIFT                                                        0x8
+#define CM0_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS__SHIFT                                                          0x9
+#define CM0_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE_MASK                                                        0x00000003L
+#define CM0_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS_MASK                                                          0x00000004L
+#define CM0_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE_MASK                                                          0x00000100L
+#define CM0_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS_MASK                                                            0x00000200L
+//CM0_CM_MEM_PWR_STATUS
+#define CM0_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE__SHIFT                                                    0x0
+#define CM0_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE__SHIFT                                                      0x8
+#define CM0_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE_MASK                                                      0x00000003L
+#define CM0_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE_MASK                                                        0x00000100L
+//CM0_CM_DEALPHA
+#define CM0_CM_DEALPHA__CM_DEALPHA_EN__SHIFT                                                                  0x0
+#define CM0_CM_DEALPHA__CM_DEALPHA_ABLND__SHIFT                                                               0x1
+#define CM0_CM_DEALPHA__CM_DEALPHA_EN_MASK                                                                    0x00000001L
+#define CM0_CM_DEALPHA__CM_DEALPHA_ABLND_MASK                                                                 0x00000002L
+//CM0_CM_COEF_FORMAT
+#define CM0_CM_COEF_FORMAT__CM_BIAS_FORMAT__SHIFT                                                             0x0
+#define CM0_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT__SHIFT                                                    0x4
+#define CM0_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
+#define CM0_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM0_CM_TEST_DEBUG_DATA
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+//CM0_CM_HIST_CNTL
+#define CM0_CM_HIST_CNTL__CM_HIST_SEL__SHIFT                                                                  0x0
+#define CM0_CM_HIST_CNTL__CM_HIST_CH_EN__SHIFT                                                                0x2
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC1_SEL__SHIFT                                                             0x4
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC2_SEL__SHIFT                                                             0x5
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC3_SEL__SHIFT                                                             0x6
+#define CM0_CM_HIST_CNTL__CM_HIST_CH1_XBAR__SHIFT                                                             0x7
+#define CM0_CM_HIST_CNTL__CM_HIST_CH2_XBAR__SHIFT                                                             0x9
+#define CM0_CM_HIST_CNTL__CM_HIST_CH3_XBAR__SHIFT                                                             0xb
+#define CM0_CM_HIST_CNTL__CM_HIST_FORMAT__SHIFT                                                               0xd
+#define CM0_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK__SHIFT                                                    0xf
+#define CM0_CM_HIST_CNTL__CM_HIST_SEL_MASK                                                                    0x00000003L
+#define CM0_CM_HIST_CNTL__CM_HIST_CH_EN_MASK                                                                  0x0000000CL
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC1_SEL_MASK                                                               0x00000010L
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC2_SEL_MASK                                                               0x00000020L
+#define CM0_CM_HIST_CNTL__CM_HIST_SRC3_SEL_MASK                                                               0x00000040L
+#define CM0_CM_HIST_CNTL__CM_HIST_CH1_XBAR_MASK                                                               0x00000180L
+#define CM0_CM_HIST_CNTL__CM_HIST_CH2_XBAR_MASK                                                               0x00000600L
+#define CM0_CM_HIST_CNTL__CM_HIST_CH3_XBAR_MASK                                                               0x00001800L
+#define CM0_CM_HIST_CNTL__CM_HIST_FORMAT_MASK                                                                 0x00006000L
+#define CM0_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK_MASK                                                      0x00038000L
+//CM0_CM_HIST_SCALE_SRC1
+#define CM0_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1__SHIFT                                                     0x0
+#define CM0_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1_MASK                                                       0x0007FFFFL
+//CM0_CM_HIST_COEFA_SRC2
+#define CM0_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2__SHIFT                                                     0x0
+#define CM0_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2_MASK                                                       0x0007FFFFL
+//CM0_CM_HIST_COEFB_SRC2
+#define CM0_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2__SHIFT                                                     0x0
+#define CM0_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2_MASK                                                       0x0007FFFFL
+//CM0_CM_HIST_COEFC_SRC2
+#define CM0_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2__SHIFT                                                     0x0
+#define CM0_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2_MASK                                                       0x0007FFFFL
+//CM0_CM_HIST_SCALE_SRC3
+#define CM0_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3__SHIFT                                                     0x0
+#define CM0_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3_MASK                                                       0x0007FFFFL
+//CM0_CM_HIST_BIAS_SRC1
+#define CM0_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1__SHIFT                                                       0x0
+#define CM0_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1_MASK                                                         0x0007FFFFL
+//CM0_CM_HIST_BIAS_SRC2
+#define CM0_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2__SHIFT                                                       0x0
+#define CM0_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2_MASK                                                         0x0007FFFFL
+//CM0_CM_HIST_BIAS_SRC3
+#define CM0_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3__SHIFT                                                       0x0
+#define CM0_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3_MASK                                                         0x0007FFFFL
+//CM0_CM_HIST_LOCK
+#define CM0_CM_HIST_LOCK__CM_HIST_LOCK__SHIFT                                                                 0x0
+#define CM0_CM_HIST_LOCK__CM_HIST_LOCK_MASK                                                                   0x00000001L
+//CM0_CM_HIST_INDEX
+#define CM0_CM_HIST_INDEX__CM_HIST_INDEX__SHIFT                                                               0x0
+#define CM0_CM_HIST_INDEX__CM_HIST_INDEX_MASK                                                                 0x000000FFL
+//CM0_CM_HIST_DATA
+#define CM0_CM_HIST_DATA__CM_HIST_DATA__SHIFT                                                                 0x0
+#define CM0_CM_HIST_DATA__CM_HIST_DATA_MASK                                                                   0x01FFFFFFL
+//CM0_CM_HIST_STATUS
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS__SHIFT                                                    0x0
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS__SHIFT                                                    0x1
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT__SHIFT                                                 0x2
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ__SHIFT                                                    0x3
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED__SHIFT                                              0x4
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT__SHIFT                                      0x5
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT__SHIFT                                          0x6
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED__SHIFT                                            0xc
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0xd
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED__SHIFT                                            0x10
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0x11
+#define CM0_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW__SHIFT                                                     0x18
+#define CM0_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT__SHIFT                                             0x19
+#define CM0_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE__SHIFT                                                 0x1a
+#define CM0_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT__SHIFT                                         0x1b
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS_MASK                                                      0x00000001L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS_MASK                                                      0x00000002L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT_MASK                                                   0x00000004L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ_MASK                                                      0x00000008L
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_MASK                                                0x00000010L
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT_MASK                                        0x00000020L
+#define CM0_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT_MASK                                            0x000001C0L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_MASK                                              0x00001000L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00002000L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_MASK                                              0x00010000L
+#define CM0_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00020000L
+#define CM0_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_MASK                                                       0x01000000L
+#define CM0_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT_MASK                                               0x02000000L
+#define CM0_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_MASK                                                   0x04000000L
+#define CM0_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT_MASK                                           0x08000000L
+//CM0_CM_HIST_INT_CONTROL
+#define CM0_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT__SHIFT                                                       0x0
+#define CM0_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN__SHIFT                                                    0x4
+#define CM0_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_MASK                                                         0x00000001L
+#define CM0_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON10_PERFCOUNTER_CNTL
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON10_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON10_PERFCOUNTER_CNTL2
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON10_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON10_PERFCOUNTER_STATE
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON10_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON10_PERFMON_CNTL
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON10_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON10_PERFMON_CNTL2
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON10_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON10_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON10_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON10_PERFMON_CVALUE_LOW
+#define DC_PERFMON10_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON10_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON10_PERFMON_HI
+#define DC_PERFMON10_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON10_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON10_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON10_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON10_PERFMON_LOW
+#define DC_PERFMON10_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON10_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_top_dispdec
+//DPP_TOP1_DPP_CONTROL
+#define DPP_TOP1_DPP_CONTROL__DPP_CLOCK_ENABLE__SHIFT                                                         0x4
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_GATE_DISABLE__SHIFT                                                    0x8
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE__SHIFT                                                0xa
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE__SHIFT                                               0xc
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_R_GATE_DISABLE__SHIFT                                                    0xe
+#define DPP_TOP1_DPP_CONTROL__DISPCLK_R_GATE_DISABLE__SHIFT                                                   0x10
+#define DPP_TOP1_DPP_CONTROL__DISPCLK_G_GATE_DISABLE__SHIFT                                                   0x12
+#define DPP_TOP1_DPP_CONTROL__DPP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define DPP_TOP1_DPP_CONTROL__DPP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define DPP_TOP1_DPP_CONTROL__DPP_CLOCK_ENABLE_MASK                                                           0x00000010L
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_GATE_DISABLE_MASK                                                      0x00000100L
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE_MASK                                                  0x00000400L
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE_MASK                                                 0x00001000L
+#define DPP_TOP1_DPP_CONTROL__DPPCLK_R_GATE_DISABLE_MASK                                                      0x00004000L
+#define DPP_TOP1_DPP_CONTROL__DISPCLK_R_GATE_DISABLE_MASK                                                     0x00010000L
+#define DPP_TOP1_DPP_CONTROL__DISPCLK_G_GATE_DISABLE_MASK                                                     0x00040000L
+#define DPP_TOP1_DPP_CONTROL__DPP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define DPP_TOP1_DPP_CONTROL__DPP_TEST_CLK_SEL_MASK                                                           0x70000000L
+//DPP_TOP1_DPP_SOFT_RESET
+#define DPP_TOP1_DPP_SOFT_RESET__CNVC_SOFT_RESET__SHIFT                                                       0x0
+#define DPP_TOP1_DPP_SOFT_RESET__DSCL_SOFT_RESET__SHIFT                                                       0x4
+#define DPP_TOP1_DPP_SOFT_RESET__CM_SOFT_RESET__SHIFT                                                         0x8
+#define DPP_TOP1_DPP_SOFT_RESET__OBUF_SOFT_RESET__SHIFT                                                       0xc
+#define DPP_TOP1_DPP_SOFT_RESET__HIST_SOFT_RESET__SHIFT                                                       0x10
+#define DPP_TOP1_DPP_SOFT_RESET__CNVC_SOFT_RESET_MASK                                                         0x00000001L
+#define DPP_TOP1_DPP_SOFT_RESET__DSCL_SOFT_RESET_MASK                                                         0x00000010L
+#define DPP_TOP1_DPP_SOFT_RESET__CM_SOFT_RESET_MASK                                                           0x00000100L
+#define DPP_TOP1_DPP_SOFT_RESET__OBUF_SOFT_RESET_MASK                                                         0x00001000L
+#define DPP_TOP1_DPP_SOFT_RESET__HIST_SOFT_RESET_MASK                                                         0x00010000L
+//DPP_TOP1_DPP_CRC_VAL_R
+#define DPP_TOP1_DPP_CRC_VAL_R__DPP_CRC_R_CR__SHIFT                                                           0x0
+#define DPP_TOP1_DPP_CRC_VAL_R__DPP_CRC_R_CR_MASK                                                             0xFFFFFFFFL
+//DPP_TOP1_DPP_CRC_VAL_G
+#define DPP_TOP1_DPP_CRC_VAL_G__DPP_CRC_G_Y__SHIFT                                                            0x0
+#define DPP_TOP1_DPP_CRC_VAL_G__DPP_CRC_G_Y_MASK                                                              0xFFFFFFFFL
+//DPP_TOP1_DPP_CRC_VAL_B
+#define DPP_TOP1_DPP_CRC_VAL_B__DPP_CRC_B_CB__SHIFT                                                           0x0
+#define DPP_TOP1_DPP_CRC_VAL_B__DPP_CRC_B_CB_MASK                                                             0xFFFFFFFFL
+//DPP_TOP1_DPP_CRC_VAL_A
+#define DPP_TOP1_DPP_CRC_VAL_A__DPP_CRC_ALPHA__SHIFT                                                          0x0
+#define DPP_TOP1_DPP_CRC_VAL_A__DPP_CRC_ALPHA_MASK                                                            0xFFFFFFFFL
+//DPP_TOP1_DPP_CRC_CTRL
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_EN__SHIFT                                                              0x0
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_CONT_EN__SHIFT                                                         0x1
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING__SHIFT                                                0x2
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL__SHIFT                                                    0x3
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_SRC_SEL__SHIFT                                                         0x4
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_STEREO_EN__SHIFT                                                       0x6
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE__SHIFT                                                     0x7
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE__SHIFT                                                  0x9
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL__SHIFT                                                  0xb
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_MASK__SHIFT                                                            0x10
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_EN_MASK                                                                0x00000001L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_CONT_EN_MASK                                                           0x00000002L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING_MASK                                                  0x00000004L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL_MASK                                                      0x00000008L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_SRC_SEL_MASK                                                           0x00000030L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_STEREO_EN_MASK                                                         0x00000040L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE_MASK                                                       0x00000180L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE_MASK                                                    0x00000600L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL_MASK                                                    0x00003800L
+#define DPP_TOP1_DPP_CRC_CTRL__DPP_CRC_MASK_MASK                                                              0xFFFF0000L
+//DPP_TOP1_HOST_READ_CONTROL
+#define DPP_TOP1_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL__SHIFT                                             0x0
+#define DPP_TOP1_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL_MASK                                               0x000000FFL
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cnvc_cfg_dispdec
+//CNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT
+#define CNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT__SHIFT                                 0x0
+#define CNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE__SHIFT                                   0x8
+#define CNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT_MASK                                   0x0000007FL
+#define CNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE_MASK                                     0x00000100L
+//CNVC_CFG1_FORMAT_CONTROL
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_EXPANSION_MODE__SHIFT                                                0x0
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CNV16__SHIFT                                                         0x4
+#define CNVC_CFG1_FORMAT_CONTROL__ALPHA_EN__SHIFT                                                             0x8
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_BYPASS__SHIFT                                                          0xc
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN__SHIFT                                                0xd
+#define CNVC_CFG1_FORMAT_CONTROL__CLAMP_POSITIVE__SHIFT                                                       0x10
+#define CNVC_CFG1_FORMAT_CONTROL__CLAMP_POSITIVE_C__SHIFT                                                     0x11
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_UPDATE_PENDING__SHIFT                                                  0x14
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_R__SHIFT                                                    0x18
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_G__SHIFT                                                    0x1a
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_B__SHIFT                                                    0x1c
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_EXPANSION_MODE_MASK                                                  0x00000001L
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CNV16_MASK                                                           0x00000010L
+#define CNVC_CFG1_FORMAT_CONTROL__ALPHA_EN_MASK                                                               0x00000100L
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_BYPASS_MASK                                                            0x00001000L
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN_MASK                                                  0x00002000L
+#define CNVC_CFG1_FORMAT_CONTROL__CLAMP_POSITIVE_MASK                                                         0x00010000L
+#define CNVC_CFG1_FORMAT_CONTROL__CLAMP_POSITIVE_C_MASK                                                       0x00020000L
+#define CNVC_CFG1_FORMAT_CONTROL__CNVC_UPDATE_PENDING_MASK                                                    0x00100000L
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_R_MASK                                                      0x03000000L
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_G_MASK                                                      0x0C000000L
+#define CNVC_CFG1_FORMAT_CONTROL__FORMAT_CROSSBAR_B_MASK                                                      0x30000000L
+//CNVC_CFG1_FCNV_FP_BIAS_R
+#define CNVC_CFG1_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R__SHIFT                                                       0x0
+#define CNVC_CFG1_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R_MASK                                                         0x0007FFFFL
+//CNVC_CFG1_FCNV_FP_BIAS_G
+#define CNVC_CFG1_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G__SHIFT                                                       0x0
+#define CNVC_CFG1_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G_MASK                                                         0x0007FFFFL
+//CNVC_CFG1_FCNV_FP_BIAS_B
+#define CNVC_CFG1_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B__SHIFT                                                       0x0
+#define CNVC_CFG1_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B_MASK                                                         0x0007FFFFL
+//CNVC_CFG1_FCNV_FP_SCALE_R
+#define CNVC_CFG1_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R__SHIFT                                                     0x0
+#define CNVC_CFG1_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R_MASK                                                       0x0007FFFFL
+//CNVC_CFG1_FCNV_FP_SCALE_G
+#define CNVC_CFG1_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G__SHIFT                                                     0x0
+#define CNVC_CFG1_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G_MASK                                                       0x0007FFFFL
+//CNVC_CFG1_FCNV_FP_SCALE_B
+#define CNVC_CFG1_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B__SHIFT                                                     0x0
+#define CNVC_CFG1_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B_MASK                                                       0x0007FFFFL
+//CNVC_CFG1_COLOR_KEYER_CONTROL
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__COLOR_KEYER_EN__SHIFT                                                  0x0
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__LUMA_KEYER_EN__SHIFT                                                   0x1
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE__SHIFT                                                0x4
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__COLOR_KEYER_EN_MASK                                                    0x00000001L
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__LUMA_KEYER_EN_MASK                                                     0x00000002L
+#define CNVC_CFG1_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE_MASK                                                  0x00000030L
+//CNVC_CFG1_COLOR_KEYER_ALPHA
+#define CNVC_CFG1_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW__SHIFT                                             0x0
+#define CNVC_CFG1_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH__SHIFT                                            0x10
+#define CNVC_CFG1_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG1_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG1_COLOR_KEYER_RED
+#define CNVC_CFG1_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW__SHIFT                                                 0x0
+#define CNVC_CFG1_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH__SHIFT                                                0x10
+#define CNVC_CFG1_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW_MASK                                                   0x0000FFFFL
+#define CNVC_CFG1_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH_MASK                                                  0xFFFF0000L
+//CNVC_CFG1_COLOR_KEYER_GREEN
+#define CNVC_CFG1_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW__SHIFT                                             0x0
+#define CNVC_CFG1_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH__SHIFT                                            0x10
+#define CNVC_CFG1_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG1_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG1_COLOR_KEYER_BLUE
+#define CNVC_CFG1_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW__SHIFT                                               0x0
+#define CNVC_CFG1_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH__SHIFT                                              0x10
+#define CNVC_CFG1_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW_MASK                                                 0x0000FFFFL
+#define CNVC_CFG1_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH_MASK                                                0xFFFF0000L
+//CNVC_CFG1_ALPHA_2BIT_LUT01
+#define CNVC_CFG1_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0__SHIFT                                                    0x0
+#define CNVC_CFG1_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1__SHIFT                                                    0x10
+#define CNVC_CFG1_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0_MASK                                                      0x00000FFFL
+#define CNVC_CFG1_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1_MASK                                                      0x0FFF0000L
+//CNVC_CFG1_ALPHA_2BIT_LUT23
+#define CNVC_CFG1_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2__SHIFT                                                    0x0
+#define CNVC_CFG1_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3__SHIFT                                                    0x10
+#define CNVC_CFG1_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2_MASK                                                      0x00000FFFL
+#define CNVC_CFG1_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3_MASK                                                      0x0FFF0000L
+//CNVC_CFG1_PRE_DEALPHA
+#define CNVC_CFG1_PRE_DEALPHA__PRE_DEALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG1_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG1_PRE_DEALPHA__PRE_DEALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG1_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN_MASK                                                      0x00000010L
+//CNVC_CFG1_PRE_CSC_MODE
+#define CNVC_CFG1_PRE_CSC_MODE__PRE_CSC_MODE__SHIFT                                                           0x0
+#define CNVC_CFG1_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT__SHIFT                                                   0x2
+#define CNVC_CFG1_PRE_CSC_MODE__PRE_CSC_MODE_MASK                                                             0x00000003L
+#define CNVC_CFG1_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT_MASK                                                     0x0000000CL
+//CNVC_CFG1_PRE_CSC_C11_C12
+#define CNVC_CFG1_PRE_CSC_C11_C12__PRE_CSC_C11__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C11_C12__PRE_CSC_C12__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C11_C12__PRE_CSC_C11_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C11_C12__PRE_CSC_C12_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_C13_C14
+#define CNVC_CFG1_PRE_CSC_C13_C14__PRE_CSC_C13__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C13_C14__PRE_CSC_C14__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C13_C14__PRE_CSC_C13_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C13_C14__PRE_CSC_C14_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_C21_C22
+#define CNVC_CFG1_PRE_CSC_C21_C22__PRE_CSC_C21__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C21_C22__PRE_CSC_C22__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C21_C22__PRE_CSC_C21_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C21_C22__PRE_CSC_C22_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_C23_C24
+#define CNVC_CFG1_PRE_CSC_C23_C24__PRE_CSC_C23__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C23_C24__PRE_CSC_C24__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C23_C24__PRE_CSC_C23_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C23_C24__PRE_CSC_C24_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_C31_C32
+#define CNVC_CFG1_PRE_CSC_C31_C32__PRE_CSC_C31__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C31_C32__PRE_CSC_C32__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C31_C32__PRE_CSC_C31_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C31_C32__PRE_CSC_C32_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_C33_C34
+#define CNVC_CFG1_PRE_CSC_C33_C34__PRE_CSC_C33__SHIFT                                                         0x0
+#define CNVC_CFG1_PRE_CSC_C33_C34__PRE_CSC_C34__SHIFT                                                         0x10
+#define CNVC_CFG1_PRE_CSC_C33_C34__PRE_CSC_C33_MASK                                                           0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_C33_C34__PRE_CSC_C34_MASK                                                           0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C11_C12
+#define CNVC_CFG1_PRE_CSC_B_C11_C12__PRE_CSC_B_C11__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C11_C12__PRE_CSC_B_C12__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C11_C12__PRE_CSC_B_C11_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C11_C12__PRE_CSC_B_C12_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C13_C14
+#define CNVC_CFG1_PRE_CSC_B_C13_C14__PRE_CSC_B_C13__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C13_C14__PRE_CSC_B_C14__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C13_C14__PRE_CSC_B_C13_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C13_C14__PRE_CSC_B_C14_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C21_C22
+#define CNVC_CFG1_PRE_CSC_B_C21_C22__PRE_CSC_B_C21__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C21_C22__PRE_CSC_B_C22__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C21_C22__PRE_CSC_B_C21_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C21_C22__PRE_CSC_B_C22_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C23_C24
+#define CNVC_CFG1_PRE_CSC_B_C23_C24__PRE_CSC_B_C23__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C23_C24__PRE_CSC_B_C24__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C23_C24__PRE_CSC_B_C23_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C23_C24__PRE_CSC_B_C24_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C31_C32
+#define CNVC_CFG1_PRE_CSC_B_C31_C32__PRE_CSC_B_C31__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C31_C32__PRE_CSC_B_C32__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C31_C32__PRE_CSC_B_C31_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C31_C32__PRE_CSC_B_C32_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_PRE_CSC_B_C33_C34
+#define CNVC_CFG1_PRE_CSC_B_C33_C34__PRE_CSC_B_C33__SHIFT                                                     0x0
+#define CNVC_CFG1_PRE_CSC_B_C33_C34__PRE_CSC_B_C34__SHIFT                                                     0x10
+#define CNVC_CFG1_PRE_CSC_B_C33_C34__PRE_CSC_B_C33_MASK                                                       0x0000FFFFL
+#define CNVC_CFG1_PRE_CSC_B_C33_C34__PRE_CSC_B_C34_MASK                                                       0xFFFF0000L
+//CNVC_CFG1_CNVC_COEF_FORMAT
+#define CNVC_CFG1_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT__SHIFT                                                0x0
+#define CNVC_CFG1_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT_MASK                                                  0x00000001L
+//CNVC_CFG1_PRE_DEGAM
+#define CNVC_CFG1_PRE_DEGAM__PRE_DEGAM_MODE__SHIFT                                                            0x0
+#define CNVC_CFG1_PRE_DEGAM__PRE_DEGAM_SELECT__SHIFT                                                          0x4
+#define CNVC_CFG1_PRE_DEGAM__PRE_DEGAM_MODE_MASK                                                              0x00000003L
+#define CNVC_CFG1_PRE_DEGAM__PRE_DEGAM_SELECT_MASK                                                            0x00000070L
+//CNVC_CFG1_PRE_REALPHA
+#define CNVC_CFG1_PRE_REALPHA__PRE_REALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG1_PRE_REALPHA__PRE_REALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG1_PRE_REALPHA__PRE_REALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG1_PRE_REALPHA__PRE_REALPHA_ABLND_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cm_cur_dispdec
+//CM_CUR1_CURSOR0_CONTROL
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_ENABLE__SHIFT                                                           0x0
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_EXPANSION_MODE__SHIFT                                                   0x1
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_PIX_INV_MODE__SHIFT                                                     0x2
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_ROM_EN__SHIFT                                                           0x3
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_MODE__SHIFT                                                             0x4
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN__SHIFT                                               0x7
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_UPDATE_PENDING__SHIFT                                                   0x10
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_ENABLE_MASK                                                             0x00000001L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_EXPANSION_MODE_MASK                                                     0x00000002L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_PIX_INV_MODE_MASK                                                       0x00000004L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_ROM_EN_MASK                                                             0x00000008L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_MODE_MASK                                                               0x00000070L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN_MASK                                                 0x00000080L
+#define CM_CUR1_CURSOR0_CONTROL__CUR0_UPDATE_PENDING_MASK                                                     0x00010000L
+//CM_CUR1_CURSOR0_COLOR0
+#define CM_CUR1_CURSOR0_COLOR0__CUR0_COLOR0__SHIFT                                                            0x0
+#define CM_CUR1_CURSOR0_COLOR0__CUR0_COLOR0_MASK                                                              0x00FFFFFFL
+//CM_CUR1_CURSOR0_COLOR1
+#define CM_CUR1_CURSOR0_COLOR1__CUR0_COLOR1__SHIFT                                                            0x0
+#define CM_CUR1_CURSOR0_COLOR1__CUR0_COLOR1_MASK                                                              0x00FFFFFFL
+//CM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y__SHIFT                                           0x0
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y__SHIFT                                            0x10
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y_MASK                                             0x0000FFFFL
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y_MASK                                              0xFFFF0000L
+//CM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB__SHIFT                                   0x0
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB__SHIFT                                    0x10
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB_MASK                                     0x0000FFFFL
+#define CM_CUR1_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB_MASK                                      0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_MODE
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE__SHIFT                                                     0x0
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT__SHIFT                                             0x2
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT__SHIFT                                              0x4
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_MASK                                                       0x00000003L
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT_MASK                                               0x0000000CL
+#define CM_CUR1_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT_MASK                                                0x00000010L
+//CM_CUR1_CUR0_MATRIX_C11_C12_A
+#define CM_CUR1_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C13_C14_A
+#define CM_CUR1_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C21_C22_A
+#define CM_CUR1_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C23_C24_A
+#define CM_CUR1_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C31_C32_A
+#define CM_CUR1_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C33_C34_A
+#define CM_CUR1_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C11_C12_B
+#define CM_CUR1_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C13_C14_B
+#define CM_CUR1_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C21_C22_B
+#define CM_CUR1_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C23_C24_B
+#define CM_CUR1_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C31_C32_B
+#define CM_CUR1_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B_MASK                                                 0xFFFF0000L
+//CM_CUR1_CUR0_MATRIX_C33_C34_B
+#define CM_CUR1_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B__SHIFT                                               0x0
+#define CM_CUR1_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B__SHIFT                                               0x10
+#define CM_CUR1_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B_MASK                                                 0x0000FFFFL
+#define CM_CUR1_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B_MASK                                                 0xFFFF0000L
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dscl_dispdec
+//DSCL1_SCL_COEF_RAM_TAP_SELECT
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX__SHIFT                                       0x0
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE__SHIFT                                              0x8
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE__SHIFT                                        0x10
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX_MASK                                         0x00000003L
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE_MASK                                                0x00003F00L
+#define DSCL1_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE_MASK                                          0x00070000L
+//DSCL1_SCL_COEF_RAM_TAP_DATA
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF__SHIFT                                        0x0
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN__SHIFT                                     0xf
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF__SHIFT                                         0x10
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN__SHIFT                                      0x1f
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_MASK                                          0x00003FFFL
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN_MASK                                       0x00008000L
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_MASK                                           0x3FFF0000L
+#define DSCL1_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN_MASK                                        0x80000000L
+//DSCL1_SCL_MODE
+#define DSCL1_SCL_MODE__DSCL_MODE__SHIFT                                                                      0x0
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT__SHIFT                                                            0x8
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT__SHIFT                                                    0xc
+#define DSCL1_SCL_MODE__SCL_CHROMA_COEF_MODE__SHIFT                                                           0x10
+#define DSCL1_SCL_MODE__SCL_ALPHA_COEF_MODE__SHIFT                                                            0x14
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT_RD__SHIFT                                                         0x18
+#define DSCL1_SCL_MODE__DSCL_MODE_MASK                                                                        0x00000007L
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT_MASK                                                              0x00000100L
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT_MASK                                                      0x00001000L
+#define DSCL1_SCL_MODE__SCL_CHROMA_COEF_MODE_MASK                                                             0x00010000L
+#define DSCL1_SCL_MODE__SCL_ALPHA_COEF_MODE_MASK                                                              0x00100000L
+#define DSCL1_SCL_MODE__SCL_COEF_RAM_SELECT_RD_MASK                                                           0x01000000L
+//DSCL1_SCL_TAP_CONTROL
+#define DSCL1_SCL_TAP_CONTROL__SCL_V_NUM_TAPS__SHIFT                                                          0x0
+#define DSCL1_SCL_TAP_CONTROL__SCL_H_NUM_TAPS__SHIFT                                                          0x4
+#define DSCL1_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C__SHIFT                                                        0x8
+#define DSCL1_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C__SHIFT                                                        0xc
+#define DSCL1_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_MASK                                                            0x00000007L
+#define DSCL1_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_MASK                                                            0x00000070L
+#define DSCL1_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C_MASK                                                          0x00000700L
+#define DSCL1_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C_MASK                                                          0x00007000L
+//DSCL1_DSCL_CONTROL
+#define DSCL1_DSCL_CONTROL__SCL_BOUNDARY_MODE__SHIFT                                                          0x0
+#define DSCL1_DSCL_CONTROL__SCL_BOUNDARY_MODE_MASK                                                            0x00000001L
+//DSCL1_DSCL_2TAP_CONTROL
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x0
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN__SHIFT                                                   0x4
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR__SHIFT                                               0x8
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x10
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN__SHIFT                                                   0x14
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR__SHIFT                                               0x18
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN_MASK                                             0x00000001L
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN_MASK                                                     0x00000010L
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR_MASK                                                 0x00000700L
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN_MASK                                             0x00010000L
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN_MASK                                                     0x00100000L
+#define DSCL1_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR_MASK                                                 0x07000000L
+//DSCL1_SCL_MANUAL_REPLICATE_CONTROL
+#define DSCL1_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR__SHIFT                              0x0
+#define DSCL1_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR__SHIFT                              0x8
+#define DSCL1_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR_MASK                                0x0000000FL
+#define DSCL1_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR_MASK                                0x00000F00L
+//DSCL1_SCL_HORZ_FILTER_SCALE_RATIO
+#define DSCL1_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL1_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL1_SCL_HORZ_FILTER_INIT
+#define DSCL1_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL1_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT__SHIFT                                                     0x18
+#define DSCL1_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL1_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT_MASK                                                       0x0F000000L
+//DSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C
+#define DSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL1_SCL_HORZ_FILTER_INIT_C
+#define DSCL1_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL1_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL1_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL1_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL1_SCL_VERT_FILTER_SCALE_RATIO
+#define DSCL1_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL1_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL1_SCL_VERT_FILTER_INIT
+#define DSCL1_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL1_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT__SHIFT                                                     0x18
+#define DSCL1_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL1_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT_MASK                                                       0x0F000000L
+//DSCL1_SCL_VERT_FILTER_INIT_BOT
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT__SHIFT                                            0x0
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT__SHIFT                                             0x18
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT_MASK                                              0x00FFFFFFL
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT_MASK                                               0x0F000000L
+//DSCL1_SCL_VERT_FILTER_SCALE_RATIO_C
+#define DSCL1_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL1_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL1_SCL_VERT_FILTER_INIT_C
+#define DSCL1_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL1_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL1_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL1_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL1_SCL_VERT_FILTER_INIT_BOT_C
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C__SHIFT                                        0x0
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C__SHIFT                                         0x18
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C_MASK                                          0x00FFFFFFL
+#define DSCL1_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C_MASK                                           0x0F000000L
+//DSCL1_SCL_BLACK_COLOR
+#define DSCL1_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y__SHIFT                                                   0x0
+#define DSCL1_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR__SHIFT                                                    0x10
+#define DSCL1_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y_MASK                                                     0x0000FFFFL
+#define DSCL1_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR_MASK                                                      0xFFFF0000L
+//DSCL1_DSCL_UPDATE
+#define DSCL1_DSCL_UPDATE__SCL_UPDATE_PENDING__SHIFT                                                          0x0
+#define DSCL1_DSCL_UPDATE__SCL_UPDATE_PENDING_MASK                                                            0x00000001L
+//DSCL1_DSCL_AUTOCAL
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_MODE__SHIFT                                                               0x0
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE__SHIFT                                                           0x8
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_PIPE_ID__SHIFT                                                            0xc
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_MODE_MASK                                                                 0x00000003L
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE_MASK                                                             0x00000300L
+#define DSCL1_DSCL_AUTOCAL__AUTOCAL_PIPE_ID_MASK                                                              0x00003000L
+//DSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT
+#define DSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT__SHIFT                                         0x0
+#define DSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT__SHIFT                                          0x10
+#define DSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT_MASK                                           0x00003FFFL
+#define DSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT_MASK                                            0x3FFF0000L
+//DSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM
+#define DSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM__SHIFT                                        0x0
+#define DSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP__SHIFT                                           0x10
+#define DSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM_MASK                                          0x00003FFFL
+#define DSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP_MASK                                             0x3FFF0000L
+//DSCL1_OTG_H_BLANK
+#define DSCL1_OTG_H_BLANK__OTG_H_BLANK_START__SHIFT                                                           0x0
+#define DSCL1_OTG_H_BLANK__OTG_H_BLANK_END__SHIFT                                                             0x10
+#define DSCL1_OTG_H_BLANK__OTG_H_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL1_OTG_H_BLANK__OTG_H_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL1_OTG_V_BLANK
+#define DSCL1_OTG_V_BLANK__OTG_V_BLANK_START__SHIFT                                                           0x0
+#define DSCL1_OTG_V_BLANK__OTG_V_BLANK_END__SHIFT                                                             0x10
+#define DSCL1_OTG_V_BLANK__OTG_V_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL1_OTG_V_BLANK__OTG_V_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL1_RECOUT_START
+#define DSCL1_RECOUT_START__RECOUT_START_X__SHIFT                                                             0x0
+#define DSCL1_RECOUT_START__RECOUT_START_Y__SHIFT                                                             0x10
+#define DSCL1_RECOUT_START__RECOUT_START_X_MASK                                                               0x00003FFFL
+#define DSCL1_RECOUT_START__RECOUT_START_Y_MASK                                                               0x3FFF0000L
+//DSCL1_RECOUT_SIZE
+#define DSCL1_RECOUT_SIZE__RECOUT_WIDTH__SHIFT                                                                0x0
+#define DSCL1_RECOUT_SIZE__RECOUT_HEIGHT__SHIFT                                                               0x10
+#define DSCL1_RECOUT_SIZE__RECOUT_WIDTH_MASK                                                                  0x00003FFFL
+#define DSCL1_RECOUT_SIZE__RECOUT_HEIGHT_MASK                                                                 0x3FFF0000L
+//DSCL1_MPC_SIZE
+#define DSCL1_MPC_SIZE__MPC_WIDTH__SHIFT                                                                      0x0
+#define DSCL1_MPC_SIZE__MPC_HEIGHT__SHIFT                                                                     0x10
+#define DSCL1_MPC_SIZE__MPC_WIDTH_MASK                                                                        0x00003FFFL
+#define DSCL1_MPC_SIZE__MPC_HEIGHT_MASK                                                                       0x3FFF0000L
+//DSCL1_LB_DATA_FORMAT
+#define DSCL1_LB_DATA_FORMAT__INTERLEAVE_EN__SHIFT                                                            0x0
+#define DSCL1_LB_DATA_FORMAT__ALPHA_EN__SHIFT                                                                 0x4
+#define DSCL1_LB_DATA_FORMAT__INTERLEAVE_EN_MASK                                                              0x00000001L
+#define DSCL1_LB_DATA_FORMAT__ALPHA_EN_MASK                                                                   0x00000010L
+//DSCL1_LB_MEMORY_CTRL
+#define DSCL1_LB_MEMORY_CTRL__MEMORY_CONFIG__SHIFT                                                            0x0
+#define DSCL1_LB_MEMORY_CTRL__LB_MAX_PARTITIONS__SHIFT                                                        0x8
+#define DSCL1_LB_MEMORY_CTRL__LB_NUM_PARTITIONS__SHIFT                                                        0x10
+#define DSCL1_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C__SHIFT                                                      0x18
+#define DSCL1_LB_MEMORY_CTRL__MEMORY_CONFIG_MASK                                                              0x00000003L
+#define DSCL1_LB_MEMORY_CTRL__LB_MAX_PARTITIONS_MASK                                                          0x00003F00L
+#define DSCL1_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_MASK                                                          0x007F0000L
+#define DSCL1_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C_MASK                                                        0x7F000000L
+//DSCL1_LB_V_COUNTER
+#define DSCL1_LB_V_COUNTER__V_COUNTER__SHIFT                                                                  0x0
+#define DSCL1_LB_V_COUNTER__V_COUNTER_C__SHIFT                                                                0x10
+#define DSCL1_LB_V_COUNTER__V_COUNTER_MASK                                                                    0x00001FFFL
+#define DSCL1_LB_V_COUNTER__V_COUNTER_C_MASK                                                                  0x1FFF0000L
+//DSCL1_DSCL_MEM_PWR_CTRL
+#define DSCL1_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DSCL1_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS__SHIFT                                                       0x2
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE__SHIFT                                                   0x4
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS__SHIFT                                                     0x6
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE__SHIFT                                                   0x8
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS__SHIFT                                                     0xa
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE__SHIFT                                                   0xc
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS__SHIFT                                                     0xe
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE__SHIFT                                                   0x10
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS__SHIFT                                                     0x12
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE__SHIFT                                                   0x14
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS__SHIFT                                                     0x16
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE__SHIFT                                                   0x18
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS__SHIFT                                                     0x1a
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE__SHIFT                                                       0x1c
+#define DSCL1_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS_MASK                                                         0x00000004L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE_MASK                                                     0x00000030L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS_MASK                                                       0x00000040L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE_MASK                                                     0x00000300L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS_MASK                                                       0x00000400L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE_MASK                                                     0x00003000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS_MASK                                                       0x00004000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE_MASK                                                     0x00030000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS_MASK                                                       0x00040000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE_MASK                                                     0x00300000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS_MASK                                                       0x00400000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE_MASK                                                     0x03000000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS_MASK                                                       0x04000000L
+#define DSCL1_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE_MASK                                                         0x10000000L
+//DSCL1_DSCL_MEM_PWR_STATUS
+#define DSCL1_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE__SHIFT                                                   0x0
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE__SHIFT                                                 0x2
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE__SHIFT                                                 0x4
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE__SHIFT                                                 0x6
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE__SHIFT                                                 0x8
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE__SHIFT                                                 0xa
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE__SHIFT                                                 0xc
+#define DSCL1_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE_MASK                                                     0x00000003L
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE_MASK                                                   0x0000000CL
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE_MASK                                                   0x00000030L
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE_MASK                                                   0x000000C0L
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE_MASK                                                   0x00000300L
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE_MASK                                                   0x00000C00L
+#define DSCL1_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE_MASK                                                   0x00003000L
+//DSCL1_OBUF_CONTROL
+#define DSCL1_OBUF_CONTROL__OBUF_BYPASS__SHIFT                                                                0x0
+#define DSCL1_OBUF_CONTROL__OBUF_USE_FULL_BUFFER__SHIFT                                                       0x1
+#define DSCL1_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH__SHIFT                                                  0x2
+#define DSCL1_OBUF_CONTROL__OBUF_OUT_HOLD_CNT__SHIFT                                                          0x4
+#define DSCL1_OBUF_CONTROL__OBUF_BYPASS_MASK                                                                  0x00000001L
+#define DSCL1_OBUF_CONTROL__OBUF_USE_FULL_BUFFER_MASK                                                         0x00000002L
+#define DSCL1_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH_MASK                                                    0x00000004L
+#define DSCL1_OBUF_CONTROL__OBUF_OUT_HOLD_CNT_MASK                                                            0x000000F0L
+//DSCL1_OBUF_MEM_PWR_CTRL
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE__SHIFT                                                    0x0
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS__SHIFT                                                      0x2
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE__SHIFT                                                     0x8
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE__SHIFT                                                    0x10
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE_MASK                                                      0x00000003L
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS_MASK                                                        0x00000004L
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE_MASK                                                       0x00000100L
+#define DSCL1_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE_MASK                                                      0x00030000L
+//DSCL1_DSCL_EASF_H_MODE
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_EN__SHIFT                                                          0x0
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_EN_MASK                                                            0x00000001L
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL1_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL1_DSCL_EASF_V_MODE
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_EN__SHIFT                                                          0x0
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_EN_MASK                                                            0x00000001L
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL1_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL1_DSCL_SC_MODE
+#define DSCL1_DSCL_SC_MODE__SCL_SC_MATRIX_MODE__SHIFT                                                         0x0
+#define DSCL1_DSCL_SC_MODE__SCL_SC_LTONL_EN__SHIFT                                                            0x8
+#define DSCL1_DSCL_SC_MODE__SCL_SC_MATRIX_MODE_MASK                                                           0x00000001L
+#define DSCL1_DSCL_SC_MODE__SCL_SC_LTONL_EN_MASK                                                              0x00000100L
+//DSCL1_DSCL_SC_MATRIX_C0C1
+#define DSCL1_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0__SHIFT                                                    0x0
+#define DSCL1_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1__SHIFT                                                    0x10
+#define DSCL1_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0_MASK                                                      0x0000FFFFL
+#define DSCL1_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1_MASK                                                      0xFFFF0000L
+//DSCL1_DSCL_SC_MATRIX_C2C3
+#define DSCL1_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2__SHIFT                                                    0x0
+#define DSCL1_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3__SHIFT                                                    0x10
+#define DSCL1_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2_MASK                                                      0x0000FFFFL
+#define DSCL1_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3_MASK                                                      0xFFFF0000L
+//DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL1_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL1_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT__SHIFT                    0x0
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL__SHIFT                    0x10
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT_MASK                      0x0000FFFFL
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL_MASK                      0xFFFF0000L
+//DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE__SHIFT                     0x0
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE__SHIFT                    0x10
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE_MASK                       0x0000FFFFL
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE_MASK                      0xFFFF0000L
+//DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE__SHIFT                    0x0
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET__SHIFT                   0x10
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE_MASK                      0x0000FFFFL
+#define DSCL1_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET_MASK                     0xFFFF0000L
+//DSCL1_DSCL_EASF_RINGEST_FORCE
+#define DSCL1_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE__SHIFT                                        0x0
+#define DSCL1_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE__SHIFT                                        0x10
+#define DSCL1_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE_MASK                                          0x0000FFFFL
+#define DSCL1_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE_MASK                                          0xFFFF0000L
+//DSCL1_DSCL_EASF_H_BF_CNTL
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN__SHIFT                                                   0x0
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE__SHIFT                                                 0x8
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE__SHIFT                                                 0x10
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN_MASK                                                     0x00000001L
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL1_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA__SHIFT                                         0x0
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB__SHIFT                                         0x8
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA__SHIFT                                         0x10
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB__SHIFT                                         0x18
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA_MASK                                           0x003F0000L
+#define DSCL1_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB_MASK                                           0x3F000000L
+//DSCL1_DSCL_EASF_V_BF_CNTL
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN__SHIFT                                                   0x0
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE__SHIFT                                                 0x8
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE__SHIFT                                                 0x10
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN_MASK                                                     0x00000001L
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL1_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA__SHIFT                                         0x0
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB__SHIFT                                         0x8
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA__SHIFT                                         0x10
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB__SHIFT                                         0x18
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA_MASK                                           0x003F0000L
+#define DSCL1_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB_MASK                                           0x3F000000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG1
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG2
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG3
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG4
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG5
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG6
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_H_BF1_PWL_SEG7
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG1
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG2
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG3
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG4
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG5
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG6
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL1_DSCL_EASF_V_BF1_PWL_SEG7
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL1_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG1
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG2
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG3
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG4
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_H_BF3_PWL_SEG5
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG1
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG2
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG3
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG4
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL1_DSCL_EASF_V_BF3_PWL_SEG5
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL1_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL1_ISHARP_MODE
+#define DSCL1_ISHARP_MODE__ISHARP_EN__SHIFT                                                                   0x0
+#define DSCL1_ISHARP_MODE__ISHARP_NOISEDET_EN__SHIFT                                                          0x4
+#define DSCL1_ISHARP_MODE__ISHARP_NOISEDET_MODE__SHIFT                                                        0x5
+#define DSCL1_ISHARP_MODE__ISHARP_LBA_MODE__SHIFT                                                             0x9
+#define DSCL1_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT__SHIFT                                                     0xa
+#define DSCL1_ISHARP_MODE__ISHARP_FMT_MODE__SHIFT                                                             0xb
+#define DSCL1_ISHARP_MODE__ISHARP_FMT_NORM__SHIFT                                                             0xc
+#define DSCL1_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT__SHIFT                                             0x1c
+#define DSCL1_ISHARP_MODE__ISHARP_EN_MASK                                                                     0x00000001L
+#define DSCL1_ISHARP_MODE__ISHARP_NOISEDET_EN_MASK                                                            0x00000010L
+#define DSCL1_ISHARP_MODE__ISHARP_NOISEDET_MODE_MASK                                                          0x00000060L
+#define DSCL1_ISHARP_MODE__ISHARP_LBA_MODE_MASK                                                               0x00000200L
+#define DSCL1_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_MASK                                                       0x00000400L
+#define DSCL1_ISHARP_MODE__ISHARP_FMT_MODE_MASK                                                               0x00000800L
+#define DSCL1_ISHARP_MODE__ISHARP_FMT_NORM_MASK                                                               0x0FFFF000L
+#define DSCL1_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT_MASK                                               0x10000000L
+//DSCL1_ISHARP_DELTA_CTRL
+#define DSCL1_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT__SHIFT                                          0x0
+#define DSCL1_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT_MASK                                            0x00000001L
+//DSCL1_ISHARP_DELTA_INDEX
+#define DSCL1_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX__SHIFT                                                   0x0
+#define DSCL1_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX_MASK                                                     0x0000001FL
+//DSCL1_ISHARP_DELTA_DATA
+#define DSCL1_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA__SHIFT                                                     0x0
+#define DSCL1_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA_MASK                                                       0xFFFFFFFFL
+//DSCL1_ISHARP_NLDELTA_SOFT_CLIP
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P__SHIFT                                      0x0
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P__SHIFT                                   0x1
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P__SHIFT                                   0x8
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N__SHIFT                                      0x10
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N__SHIFT                                   0x11
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N__SHIFT                                   0x18
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P_MASK                                        0x00000001L
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P_MASK                                     0x000000FEL
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P_MASK                                     0x0000FF00L
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N_MASK                                        0x00010000L
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N_MASK                                     0x00FE0000L
+#define DSCL1_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N_MASK                                     0xFF000000L
+//DSCL1_ISHARP_NOISEDET_THRESHOLD
+#define DSCL1_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE__SHIFT                                         0x0
+#define DSCL1_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE__SHIFT                                         0x10
+#define DSCL1_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE_MASK                                           0x000003FFL
+#define DSCL1_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE_MASK                                           0x03FF0000L
+//DSCL1_ISHARP_NOISE_GAIN_PWL
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN__SHIFT                                      0x0
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN__SHIFT                                        0x8
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE__SHIFT                                         0x10
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN_MASK                                        0x0000001FL
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN_MASK                                          0x00001F00L
+#define DSCL1_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE_MASK                                           0x3FFF0000L
+//DSCL1_ISHARP_LBA_PWL_SEG0
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0__SHIFT                                           0x14
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0_MASK                                              0x0003F000L
+#define DSCL1_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0_MASK                                             0x1FF00000L
+//DSCL1_ISHARP_LBA_PWL_SEG1
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1__SHIFT                                           0x14
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1_MASK                                              0x0003F000L
+#define DSCL1_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1_MASK                                             0x1FF00000L
+//DSCL1_ISHARP_LBA_PWL_SEG2
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2__SHIFT                                           0x14
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2_MASK                                              0x0003F000L
+#define DSCL1_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2_MASK                                             0x1FF00000L
+//DSCL1_ISHARP_LBA_PWL_SEG3
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3__SHIFT                                           0x14
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3_MASK                                              0x0003F000L
+#define DSCL1_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3_MASK                                             0x1FF00000L
+//DSCL1_ISHARP_LBA_PWL_SEG4
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4__SHIFT                                           0x14
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4_MASK                                              0x0003F000L
+#define DSCL1_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4_MASK                                             0x1FF00000L
+//DSCL1_ISHARP_LBA_PWL_SEG5
+#define DSCL1_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5__SHIFT                                              0x0
+#define DSCL1_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5__SHIFT                                            0xc
+#define DSCL1_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5_MASK                                                0x000003FFL
+#define DSCL1_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5_MASK                                              0x0003F000L
+//DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE__SHIFT                            0x0
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS__SHIFT                              0x2
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE__SHIFT                            0x4
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE_MASK                              0x00000003L
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS_MASK                                0x00000004L
+#define DSCL1_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE_MASK                              0x00000030L
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cm_dispdec
+//CM1_CM_CONTROL
+#define CM1_CM_CONTROL__CM_BYPASS__SHIFT                                                                      0x0
+#define CM1_CM_CONTROL__CM_UPDATE_PENDING__SHIFT                                                              0x8
+#define CM1_CM_CONTROL__CM_BYPASS_MASK                                                                        0x00000001L
+#define CM1_CM_CONTROL__CM_UPDATE_PENDING_MASK                                                                0x00000100L
+//CM1_CM_POST_CSC_CONTROL
+#define CM1_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE__SHIFT                                                      0x0
+#define CM1_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT__SHIFT                                              0x2
+#define CM1_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_MASK                                                        0x00000003L
+#define CM1_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT_MASK                                                0x0000000CL
+//CM1_CM_POST_CSC_C11_C12
+#define CM1_CM_POST_CSC_C11_C12__CM_POST_CSC_C11__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C11_C12__CM_POST_CSC_C12__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C11_C12__CM_POST_CSC_C11_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C11_C12__CM_POST_CSC_C12_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_C13_C14
+#define CM1_CM_POST_CSC_C13_C14__CM_POST_CSC_C13__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C13_C14__CM_POST_CSC_C14__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C13_C14__CM_POST_CSC_C13_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C13_C14__CM_POST_CSC_C14_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_C21_C22
+#define CM1_CM_POST_CSC_C21_C22__CM_POST_CSC_C21__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C21_C22__CM_POST_CSC_C22__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C21_C22__CM_POST_CSC_C21_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C21_C22__CM_POST_CSC_C22_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_C23_C24
+#define CM1_CM_POST_CSC_C23_C24__CM_POST_CSC_C23__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C23_C24__CM_POST_CSC_C24__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C23_C24__CM_POST_CSC_C23_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C23_C24__CM_POST_CSC_C24_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_C31_C32
+#define CM1_CM_POST_CSC_C31_C32__CM_POST_CSC_C31__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C31_C32__CM_POST_CSC_C32__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C31_C32__CM_POST_CSC_C31_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C31_C32__CM_POST_CSC_C32_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_C33_C34
+#define CM1_CM_POST_CSC_C33_C34__CM_POST_CSC_C33__SHIFT                                                       0x0
+#define CM1_CM_POST_CSC_C33_C34__CM_POST_CSC_C34__SHIFT                                                       0x10
+#define CM1_CM_POST_CSC_C33_C34__CM_POST_CSC_C33_MASK                                                         0x0000FFFFL
+#define CM1_CM_POST_CSC_C33_C34__CM_POST_CSC_C34_MASK                                                         0xFFFF0000L
+//CM1_CM_POST_CSC_B_C11_C12
+#define CM1_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12_MASK                                                     0xFFFF0000L
+//CM1_CM_POST_CSC_B_C13_C14
+#define CM1_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14_MASK                                                     0xFFFF0000L
+//CM1_CM_POST_CSC_B_C21_C22
+#define CM1_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22_MASK                                                     0xFFFF0000L
+//CM1_CM_POST_CSC_B_C23_C24
+#define CM1_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24_MASK                                                     0xFFFF0000L
+//CM1_CM_POST_CSC_B_C31_C32
+#define CM1_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32_MASK                                                     0xFFFF0000L
+//CM1_CM_POST_CSC_B_C33_C34
+#define CM1_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33__SHIFT                                                   0x0
+#define CM1_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34__SHIFT                                                   0x10
+#define CM1_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33_MASK                                                     0x0000FFFFL
+#define CM1_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34_MASK                                                     0xFFFF0000L
+//CM1_CM_BIAS_CR_R
+#define CM1_CM_BIAS_CR_R__CM_BIAS_CR_R__SHIFT                                                                 0x0
+#define CM1_CM_BIAS_CR_R__CM_BIAS_CR_R_MASK                                                                   0x0000FFFFL
+//CM1_CM_BIAS_Y_G_CB_B
+#define CM1_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G__SHIFT                                                              0x0
+#define CM1_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B__SHIFT                                                             0x10
+#define CM1_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G_MASK                                                                0x0000FFFFL
+#define CM1_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B_MASK                                                               0xFFFF0000L
+//CM1_CM_GAMCOR_CONTROL
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE__SHIFT                                                          0x0
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT__SHIFT                                                        0x2
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE__SHIFT                                                   0x3
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT__SHIFT                                                  0x4
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT__SHIFT                                                0x6
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_MASK                                                            0x00000003L
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_MASK                                                          0x00000004L
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE_MASK                                                     0x00000008L
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT_MASK                                                    0x00000030L
+#define CM1_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT_MASK                                                  0x00000040L
+//CM1_CM_GAMCOR_LUT_INDEX
+#define CM1_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX__SHIFT                                                   0x0
+#define CM1_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX_MASK                                                     0x000001FFL
+//CM1_CM_GAMCOR_LUT_DATA
+#define CM1_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA__SHIFT                                                     0x0
+#define CM1_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA_MASK                                                       0x0003FFFFL
+//CM1_CM_GAMCOR_LUT_CONTROL
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK__SHIFT                                      0x0
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL__SHIFT                                        0x3
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG__SHIFT                                              0x5
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL__SHIFT                                              0x6
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE__SHIFT                                           0x7
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK_MASK                                        0x00000007L
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL_MASK                                          0x00000018L
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG_MASK                                                0x00000020L
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL_MASK                                                0x00000040L
+#define CM1_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE_MASK                                             0x00000080L
+//CM1_CM_GAMCOR_RAMA_START_CNTL_B
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMA_START_CNTL_G
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMA_START_CNTL_R
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_B
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_G
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_R
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_END_CNTL1_B
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_END_CNTL2_B
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMA_END_CNTL1_G
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_END_CNTL2_G
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMA_END_CNTL1_R
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMA_END_CNTL2_R
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMA_OFFSET_B
+#define CM1_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMA_OFFSET_G
+#define CM1_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMA_OFFSET_R
+#define CM1_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMA_REGION_0_1
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_2_3
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_4_5
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_6_7
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_8_9
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_10_11
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_12_13
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_14_15
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_16_17
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_18_19
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_20_21
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_22_23
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_24_25
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_26_27
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_28_29
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_30_31
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMA_REGION_32_33
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_START_CNTL_B
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMB_START_CNTL_G
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMB_START_CNTL_R
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R__SHIFT                             0x0
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM1_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM1_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_B
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_G
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_R
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM1_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_END_CNTL1_B
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_END_CNTL2_B
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMB_END_CNTL1_G
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_END_CNTL2_G
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMB_END_CNTL1_R
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM1_CM_GAMCOR_RAMB_END_CNTL2_R
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R__SHIFT                                0x0
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM1_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM1_CM_GAMCOR_RAMB_OFFSET_B
+#define CM1_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMB_OFFSET_G
+#define CM1_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMB_OFFSET_R
+#define CM1_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R__SHIFT                                           0x0
+#define CM1_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R_MASK                                             0x0007FFFFL
+//CM1_CM_GAMCOR_RAMB_REGION_0_1
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_2_3
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_4_5
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_6_7
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_8_9
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_10_11
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_12_13
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_14_15
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_16_17
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_18_19
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_20_21
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_22_23
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_24_25
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_26_27
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_28_29
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_30_31
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_GAMCOR_RAMB_REGION_32_33
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM1_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM1_CM_HDR_MULT_COEF
+#define CM1_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF__SHIFT                                                         0x0
+#define CM1_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF_MASK                                                           0x0007FFFFL
+//CM1_CM_MEM_PWR_CTRL
+#define CM1_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE__SHIFT                                                      0x0
+#define CM1_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS__SHIFT                                                        0x2
+#define CM1_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE__SHIFT                                                        0x8
+#define CM1_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS__SHIFT                                                          0x9
+#define CM1_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE_MASK                                                        0x00000003L
+#define CM1_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS_MASK                                                          0x00000004L
+#define CM1_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE_MASK                                                          0x00000100L
+#define CM1_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS_MASK                                                            0x00000200L
+//CM1_CM_MEM_PWR_STATUS
+#define CM1_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE__SHIFT                                                    0x0
+#define CM1_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE__SHIFT                                                      0x8
+#define CM1_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE_MASK                                                      0x00000003L
+#define CM1_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE_MASK                                                        0x00000100L
+//CM1_CM_DEALPHA
+#define CM1_CM_DEALPHA__CM_DEALPHA_EN__SHIFT                                                                  0x0
+#define CM1_CM_DEALPHA__CM_DEALPHA_ABLND__SHIFT                                                               0x1
+#define CM1_CM_DEALPHA__CM_DEALPHA_EN_MASK                                                                    0x00000001L
+#define CM1_CM_DEALPHA__CM_DEALPHA_ABLND_MASK                                                                 0x00000002L
+//CM1_CM_COEF_FORMAT
+#define CM1_CM_COEF_FORMAT__CM_BIAS_FORMAT__SHIFT                                                             0x0
+#define CM1_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT__SHIFT                                                    0x4
+#define CM1_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
+#define CM1_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
+//CM1_CM_TEST_DEBUG_INDEX
+#define CM1_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM1_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM1_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM1_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM1_CM_TEST_DEBUG_DATA
+#define CM1_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM1_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+//CM1_CM_HIST_CNTL
+#define CM1_CM_HIST_CNTL__CM_HIST_SEL__SHIFT                                                                  0x0
+#define CM1_CM_HIST_CNTL__CM_HIST_CH_EN__SHIFT                                                                0x2
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC1_SEL__SHIFT                                                             0x4
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC2_SEL__SHIFT                                                             0x5
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC3_SEL__SHIFT                                                             0x6
+#define CM1_CM_HIST_CNTL__CM_HIST_CH1_XBAR__SHIFT                                                             0x7
+#define CM1_CM_HIST_CNTL__CM_HIST_CH2_XBAR__SHIFT                                                             0x9
+#define CM1_CM_HIST_CNTL__CM_HIST_CH3_XBAR__SHIFT                                                             0xb
+#define CM1_CM_HIST_CNTL__CM_HIST_FORMAT__SHIFT                                                               0xd
+#define CM1_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK__SHIFT                                                    0xf
+#define CM1_CM_HIST_CNTL__CM_HIST_SEL_MASK                                                                    0x00000003L
+#define CM1_CM_HIST_CNTL__CM_HIST_CH_EN_MASK                                                                  0x0000000CL
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC1_SEL_MASK                                                               0x00000010L
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC2_SEL_MASK                                                               0x00000020L
+#define CM1_CM_HIST_CNTL__CM_HIST_SRC3_SEL_MASK                                                               0x00000040L
+#define CM1_CM_HIST_CNTL__CM_HIST_CH1_XBAR_MASK                                                               0x00000180L
+#define CM1_CM_HIST_CNTL__CM_HIST_CH2_XBAR_MASK                                                               0x00000600L
+#define CM1_CM_HIST_CNTL__CM_HIST_CH3_XBAR_MASK                                                               0x00001800L
+#define CM1_CM_HIST_CNTL__CM_HIST_FORMAT_MASK                                                                 0x00006000L
+#define CM1_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK_MASK                                                      0x00038000L
+//CM1_CM_HIST_SCALE_SRC1
+#define CM1_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1__SHIFT                                                     0x0
+#define CM1_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1_MASK                                                       0x0007FFFFL
+//CM1_CM_HIST_COEFA_SRC2
+#define CM1_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2__SHIFT                                                     0x0
+#define CM1_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2_MASK                                                       0x0007FFFFL
+//CM1_CM_HIST_COEFB_SRC2
+#define CM1_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2__SHIFT                                                     0x0
+#define CM1_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2_MASK                                                       0x0007FFFFL
+//CM1_CM_HIST_COEFC_SRC2
+#define CM1_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2__SHIFT                                                     0x0
+#define CM1_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2_MASK                                                       0x0007FFFFL
+//CM1_CM_HIST_SCALE_SRC3
+#define CM1_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3__SHIFT                                                     0x0
+#define CM1_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3_MASK                                                       0x0007FFFFL
+//CM1_CM_HIST_BIAS_SRC1
+#define CM1_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1__SHIFT                                                       0x0
+#define CM1_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1_MASK                                                         0x0007FFFFL
+//CM1_CM_HIST_BIAS_SRC2
+#define CM1_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2__SHIFT                                                       0x0
+#define CM1_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2_MASK                                                         0x0007FFFFL
+//CM1_CM_HIST_BIAS_SRC3
+#define CM1_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3__SHIFT                                                       0x0
+#define CM1_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3_MASK                                                         0x0007FFFFL
+//CM1_CM_HIST_LOCK
+#define CM1_CM_HIST_LOCK__CM_HIST_LOCK__SHIFT                                                                 0x0
+#define CM1_CM_HIST_LOCK__CM_HIST_LOCK_MASK                                                                   0x00000001L
+//CM1_CM_HIST_INDEX
+#define CM1_CM_HIST_INDEX__CM_HIST_INDEX__SHIFT                                                               0x0
+#define CM1_CM_HIST_INDEX__CM_HIST_INDEX_MASK                                                                 0x000000FFL
+//CM1_CM_HIST_DATA
+#define CM1_CM_HIST_DATA__CM_HIST_DATA__SHIFT                                                                 0x0
+#define CM1_CM_HIST_DATA__CM_HIST_DATA_MASK                                                                   0x01FFFFFFL
+//CM1_CM_HIST_STATUS
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS__SHIFT                                                    0x0
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS__SHIFT                                                    0x1
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT__SHIFT                                                 0x2
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ__SHIFT                                                    0x3
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED__SHIFT                                              0x4
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT__SHIFT                                      0x5
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT__SHIFT                                          0x6
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED__SHIFT                                            0xc
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0xd
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED__SHIFT                                            0x10
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0x11
+#define CM1_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW__SHIFT                                                     0x18
+#define CM1_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT__SHIFT                                             0x19
+#define CM1_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE__SHIFT                                                 0x1a
+#define CM1_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT__SHIFT                                         0x1b
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS_MASK                                                      0x00000001L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS_MASK                                                      0x00000002L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT_MASK                                                   0x00000004L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ_MASK                                                      0x00000008L
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_MASK                                                0x00000010L
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT_MASK                                        0x00000020L
+#define CM1_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT_MASK                                            0x000001C0L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_MASK                                              0x00001000L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00002000L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_MASK                                              0x00010000L
+#define CM1_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00020000L
+#define CM1_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_MASK                                                       0x01000000L
+#define CM1_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT_MASK                                               0x02000000L
+#define CM1_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_MASK                                                   0x04000000L
+#define CM1_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT_MASK                                           0x08000000L
+//CM1_CM_HIST_INT_CONTROL
+#define CM1_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT__SHIFT                                                       0x0
+#define CM1_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN__SHIFT                                                    0x4
+#define CM1_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_MASK                                                         0x00000001L
+#define CM1_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON11_PERFCOUNTER_CNTL
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON11_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON11_PERFCOUNTER_CNTL2
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON11_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON11_PERFCOUNTER_STATE
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON11_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON11_PERFMON_CNTL
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON11_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON11_PERFMON_CNTL2
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON11_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON11_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON11_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON11_PERFMON_CVALUE_LOW
+#define DC_PERFMON11_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON11_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON11_PERFMON_HI
+#define DC_PERFMON11_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON11_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON11_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON11_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON11_PERFMON_LOW
+#define DC_PERFMON11_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON11_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_top_dispdec
+//DPP_TOP2_DPP_CONTROL
+#define DPP_TOP2_DPP_CONTROL__DPP_CLOCK_ENABLE__SHIFT                                                         0x4
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_GATE_DISABLE__SHIFT                                                    0x8
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE__SHIFT                                                0xa
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE__SHIFT                                               0xc
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_R_GATE_DISABLE__SHIFT                                                    0xe
+#define DPP_TOP2_DPP_CONTROL__DISPCLK_R_GATE_DISABLE__SHIFT                                                   0x10
+#define DPP_TOP2_DPP_CONTROL__DISPCLK_G_GATE_DISABLE__SHIFT                                                   0x12
+#define DPP_TOP2_DPP_CONTROL__DPP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define DPP_TOP2_DPP_CONTROL__DPP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define DPP_TOP2_DPP_CONTROL__DPP_CLOCK_ENABLE_MASK                                                           0x00000010L
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_GATE_DISABLE_MASK                                                      0x00000100L
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE_MASK                                                  0x00000400L
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE_MASK                                                 0x00001000L
+#define DPP_TOP2_DPP_CONTROL__DPPCLK_R_GATE_DISABLE_MASK                                                      0x00004000L
+#define DPP_TOP2_DPP_CONTROL__DISPCLK_R_GATE_DISABLE_MASK                                                     0x00010000L
+#define DPP_TOP2_DPP_CONTROL__DISPCLK_G_GATE_DISABLE_MASK                                                     0x00040000L
+#define DPP_TOP2_DPP_CONTROL__DPP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define DPP_TOP2_DPP_CONTROL__DPP_TEST_CLK_SEL_MASK                                                           0x70000000L
+//DPP_TOP2_DPP_SOFT_RESET
+#define DPP_TOP2_DPP_SOFT_RESET__CNVC_SOFT_RESET__SHIFT                                                       0x0
+#define DPP_TOP2_DPP_SOFT_RESET__DSCL_SOFT_RESET__SHIFT                                                       0x4
+#define DPP_TOP2_DPP_SOFT_RESET__CM_SOFT_RESET__SHIFT                                                         0x8
+#define DPP_TOP2_DPP_SOFT_RESET__OBUF_SOFT_RESET__SHIFT                                                       0xc
+#define DPP_TOP2_DPP_SOFT_RESET__HIST_SOFT_RESET__SHIFT                                                       0x10
+#define DPP_TOP2_DPP_SOFT_RESET__CNVC_SOFT_RESET_MASK                                                         0x00000001L
+#define DPP_TOP2_DPP_SOFT_RESET__DSCL_SOFT_RESET_MASK                                                         0x00000010L
+#define DPP_TOP2_DPP_SOFT_RESET__CM_SOFT_RESET_MASK                                                           0x00000100L
+#define DPP_TOP2_DPP_SOFT_RESET__OBUF_SOFT_RESET_MASK                                                         0x00001000L
+#define DPP_TOP2_DPP_SOFT_RESET__HIST_SOFT_RESET_MASK                                                         0x00010000L
+//DPP_TOP2_DPP_CRC_VAL_R
+#define DPP_TOP2_DPP_CRC_VAL_R__DPP_CRC_R_CR__SHIFT                                                           0x0
+#define DPP_TOP2_DPP_CRC_VAL_R__DPP_CRC_R_CR_MASK                                                             0xFFFFFFFFL
+//DPP_TOP2_DPP_CRC_VAL_G
+#define DPP_TOP2_DPP_CRC_VAL_G__DPP_CRC_G_Y__SHIFT                                                            0x0
+#define DPP_TOP2_DPP_CRC_VAL_G__DPP_CRC_G_Y_MASK                                                              0xFFFFFFFFL
+//DPP_TOP2_DPP_CRC_VAL_B
+#define DPP_TOP2_DPP_CRC_VAL_B__DPP_CRC_B_CB__SHIFT                                                           0x0
+#define DPP_TOP2_DPP_CRC_VAL_B__DPP_CRC_B_CB_MASK                                                             0xFFFFFFFFL
+//DPP_TOP2_DPP_CRC_VAL_A
+#define DPP_TOP2_DPP_CRC_VAL_A__DPP_CRC_ALPHA__SHIFT                                                          0x0
+#define DPP_TOP2_DPP_CRC_VAL_A__DPP_CRC_ALPHA_MASK                                                            0xFFFFFFFFL
+//DPP_TOP2_DPP_CRC_CTRL
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_EN__SHIFT                                                              0x0
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_CONT_EN__SHIFT                                                         0x1
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING__SHIFT                                                0x2
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL__SHIFT                                                    0x3
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_SRC_SEL__SHIFT                                                         0x4
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_STEREO_EN__SHIFT                                                       0x6
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE__SHIFT                                                     0x7
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE__SHIFT                                                  0x9
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL__SHIFT                                                  0xb
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_MASK__SHIFT                                                            0x10
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_EN_MASK                                                                0x00000001L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_CONT_EN_MASK                                                           0x00000002L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING_MASK                                                  0x00000004L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL_MASK                                                      0x00000008L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_SRC_SEL_MASK                                                           0x00000030L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_STEREO_EN_MASK                                                         0x00000040L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE_MASK                                                       0x00000180L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE_MASK                                                    0x00000600L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL_MASK                                                    0x00003800L
+#define DPP_TOP2_DPP_CRC_CTRL__DPP_CRC_MASK_MASK                                                              0xFFFF0000L
+//DPP_TOP2_HOST_READ_CONTROL
+#define DPP_TOP2_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL__SHIFT                                             0x0
+#define DPP_TOP2_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL_MASK                                               0x000000FFL
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cnvc_cfg_dispdec
+//CNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT
+#define CNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT__SHIFT                                 0x0
+#define CNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE__SHIFT                                   0x8
+#define CNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT_MASK                                   0x0000007FL
+#define CNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE_MASK                                     0x00000100L
+//CNVC_CFG2_FORMAT_CONTROL
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_EXPANSION_MODE__SHIFT                                                0x0
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CNV16__SHIFT                                                         0x4
+#define CNVC_CFG2_FORMAT_CONTROL__ALPHA_EN__SHIFT                                                             0x8
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_BYPASS__SHIFT                                                          0xc
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN__SHIFT                                                0xd
+#define CNVC_CFG2_FORMAT_CONTROL__CLAMP_POSITIVE__SHIFT                                                       0x10
+#define CNVC_CFG2_FORMAT_CONTROL__CLAMP_POSITIVE_C__SHIFT                                                     0x11
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_UPDATE_PENDING__SHIFT                                                  0x14
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_R__SHIFT                                                    0x18
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_G__SHIFT                                                    0x1a
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_B__SHIFT                                                    0x1c
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_EXPANSION_MODE_MASK                                                  0x00000001L
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CNV16_MASK                                                           0x00000010L
+#define CNVC_CFG2_FORMAT_CONTROL__ALPHA_EN_MASK                                                               0x00000100L
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_BYPASS_MASK                                                            0x00001000L
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN_MASK                                                  0x00002000L
+#define CNVC_CFG2_FORMAT_CONTROL__CLAMP_POSITIVE_MASK                                                         0x00010000L
+#define CNVC_CFG2_FORMAT_CONTROL__CLAMP_POSITIVE_C_MASK                                                       0x00020000L
+#define CNVC_CFG2_FORMAT_CONTROL__CNVC_UPDATE_PENDING_MASK                                                    0x00100000L
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_R_MASK                                                      0x03000000L
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_G_MASK                                                      0x0C000000L
+#define CNVC_CFG2_FORMAT_CONTROL__FORMAT_CROSSBAR_B_MASK                                                      0x30000000L
+//CNVC_CFG2_FCNV_FP_BIAS_R
+#define CNVC_CFG2_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R__SHIFT                                                       0x0
+#define CNVC_CFG2_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R_MASK                                                         0x0007FFFFL
+//CNVC_CFG2_FCNV_FP_BIAS_G
+#define CNVC_CFG2_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G__SHIFT                                                       0x0
+#define CNVC_CFG2_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G_MASK                                                         0x0007FFFFL
+//CNVC_CFG2_FCNV_FP_BIAS_B
+#define CNVC_CFG2_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B__SHIFT                                                       0x0
+#define CNVC_CFG2_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B_MASK                                                         0x0007FFFFL
+//CNVC_CFG2_FCNV_FP_SCALE_R
+#define CNVC_CFG2_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R__SHIFT                                                     0x0
+#define CNVC_CFG2_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R_MASK                                                       0x0007FFFFL
+//CNVC_CFG2_FCNV_FP_SCALE_G
+#define CNVC_CFG2_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G__SHIFT                                                     0x0
+#define CNVC_CFG2_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G_MASK                                                       0x0007FFFFL
+//CNVC_CFG2_FCNV_FP_SCALE_B
+#define CNVC_CFG2_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B__SHIFT                                                     0x0
+#define CNVC_CFG2_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B_MASK                                                       0x0007FFFFL
+//CNVC_CFG2_COLOR_KEYER_CONTROL
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__COLOR_KEYER_EN__SHIFT                                                  0x0
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__LUMA_KEYER_EN__SHIFT                                                   0x1
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE__SHIFT                                                0x4
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__COLOR_KEYER_EN_MASK                                                    0x00000001L
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__LUMA_KEYER_EN_MASK                                                     0x00000002L
+#define CNVC_CFG2_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE_MASK                                                  0x00000030L
+//CNVC_CFG2_COLOR_KEYER_ALPHA
+#define CNVC_CFG2_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW__SHIFT                                             0x0
+#define CNVC_CFG2_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH__SHIFT                                            0x10
+#define CNVC_CFG2_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG2_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG2_COLOR_KEYER_RED
+#define CNVC_CFG2_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW__SHIFT                                                 0x0
+#define CNVC_CFG2_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH__SHIFT                                                0x10
+#define CNVC_CFG2_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW_MASK                                                   0x0000FFFFL
+#define CNVC_CFG2_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH_MASK                                                  0xFFFF0000L
+//CNVC_CFG2_COLOR_KEYER_GREEN
+#define CNVC_CFG2_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW__SHIFT                                             0x0
+#define CNVC_CFG2_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH__SHIFT                                            0x10
+#define CNVC_CFG2_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG2_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG2_COLOR_KEYER_BLUE
+#define CNVC_CFG2_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW__SHIFT                                               0x0
+#define CNVC_CFG2_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH__SHIFT                                              0x10
+#define CNVC_CFG2_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW_MASK                                                 0x0000FFFFL
+#define CNVC_CFG2_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH_MASK                                                0xFFFF0000L
+//CNVC_CFG2_ALPHA_2BIT_LUT01
+#define CNVC_CFG2_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0__SHIFT                                                    0x0
+#define CNVC_CFG2_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1__SHIFT                                                    0x10
+#define CNVC_CFG2_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0_MASK                                                      0x00000FFFL
+#define CNVC_CFG2_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1_MASK                                                      0x0FFF0000L
+//CNVC_CFG2_ALPHA_2BIT_LUT23
+#define CNVC_CFG2_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2__SHIFT                                                    0x0
+#define CNVC_CFG2_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3__SHIFT                                                    0x10
+#define CNVC_CFG2_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2_MASK                                                      0x00000FFFL
+#define CNVC_CFG2_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3_MASK                                                      0x0FFF0000L
+//CNVC_CFG2_PRE_DEALPHA
+#define CNVC_CFG2_PRE_DEALPHA__PRE_DEALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG2_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG2_PRE_DEALPHA__PRE_DEALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG2_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN_MASK                                                      0x00000010L
+//CNVC_CFG2_PRE_CSC_MODE
+#define CNVC_CFG2_PRE_CSC_MODE__PRE_CSC_MODE__SHIFT                                                           0x0
+#define CNVC_CFG2_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT__SHIFT                                                   0x2
+#define CNVC_CFG2_PRE_CSC_MODE__PRE_CSC_MODE_MASK                                                             0x00000003L
+#define CNVC_CFG2_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT_MASK                                                     0x0000000CL
+//CNVC_CFG2_PRE_CSC_C11_C12
+#define CNVC_CFG2_PRE_CSC_C11_C12__PRE_CSC_C11__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C11_C12__PRE_CSC_C12__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C11_C12__PRE_CSC_C11_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C11_C12__PRE_CSC_C12_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_C13_C14
+#define CNVC_CFG2_PRE_CSC_C13_C14__PRE_CSC_C13__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C13_C14__PRE_CSC_C14__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C13_C14__PRE_CSC_C13_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C13_C14__PRE_CSC_C14_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_C21_C22
+#define CNVC_CFG2_PRE_CSC_C21_C22__PRE_CSC_C21__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C21_C22__PRE_CSC_C22__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C21_C22__PRE_CSC_C21_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C21_C22__PRE_CSC_C22_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_C23_C24
+#define CNVC_CFG2_PRE_CSC_C23_C24__PRE_CSC_C23__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C23_C24__PRE_CSC_C24__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C23_C24__PRE_CSC_C23_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C23_C24__PRE_CSC_C24_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_C31_C32
+#define CNVC_CFG2_PRE_CSC_C31_C32__PRE_CSC_C31__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C31_C32__PRE_CSC_C32__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C31_C32__PRE_CSC_C31_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C31_C32__PRE_CSC_C32_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_C33_C34
+#define CNVC_CFG2_PRE_CSC_C33_C34__PRE_CSC_C33__SHIFT                                                         0x0
+#define CNVC_CFG2_PRE_CSC_C33_C34__PRE_CSC_C34__SHIFT                                                         0x10
+#define CNVC_CFG2_PRE_CSC_C33_C34__PRE_CSC_C33_MASK                                                           0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_C33_C34__PRE_CSC_C34_MASK                                                           0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C11_C12
+#define CNVC_CFG2_PRE_CSC_B_C11_C12__PRE_CSC_B_C11__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C11_C12__PRE_CSC_B_C12__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C11_C12__PRE_CSC_B_C11_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C11_C12__PRE_CSC_B_C12_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C13_C14
+#define CNVC_CFG2_PRE_CSC_B_C13_C14__PRE_CSC_B_C13__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C13_C14__PRE_CSC_B_C14__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C13_C14__PRE_CSC_B_C13_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C13_C14__PRE_CSC_B_C14_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C21_C22
+#define CNVC_CFG2_PRE_CSC_B_C21_C22__PRE_CSC_B_C21__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C21_C22__PRE_CSC_B_C22__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C21_C22__PRE_CSC_B_C21_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C21_C22__PRE_CSC_B_C22_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C23_C24
+#define CNVC_CFG2_PRE_CSC_B_C23_C24__PRE_CSC_B_C23__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C23_C24__PRE_CSC_B_C24__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C23_C24__PRE_CSC_B_C23_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C23_C24__PRE_CSC_B_C24_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C31_C32
+#define CNVC_CFG2_PRE_CSC_B_C31_C32__PRE_CSC_B_C31__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C31_C32__PRE_CSC_B_C32__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C31_C32__PRE_CSC_B_C31_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C31_C32__PRE_CSC_B_C32_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_PRE_CSC_B_C33_C34
+#define CNVC_CFG2_PRE_CSC_B_C33_C34__PRE_CSC_B_C33__SHIFT                                                     0x0
+#define CNVC_CFG2_PRE_CSC_B_C33_C34__PRE_CSC_B_C34__SHIFT                                                     0x10
+#define CNVC_CFG2_PRE_CSC_B_C33_C34__PRE_CSC_B_C33_MASK                                                       0x0000FFFFL
+#define CNVC_CFG2_PRE_CSC_B_C33_C34__PRE_CSC_B_C34_MASK                                                       0xFFFF0000L
+//CNVC_CFG2_CNVC_COEF_FORMAT
+#define CNVC_CFG2_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT__SHIFT                                                0x0
+#define CNVC_CFG2_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT_MASK                                                  0x00000001L
+//CNVC_CFG2_PRE_DEGAM
+#define CNVC_CFG2_PRE_DEGAM__PRE_DEGAM_MODE__SHIFT                                                            0x0
+#define CNVC_CFG2_PRE_DEGAM__PRE_DEGAM_SELECT__SHIFT                                                          0x4
+#define CNVC_CFG2_PRE_DEGAM__PRE_DEGAM_MODE_MASK                                                              0x00000003L
+#define CNVC_CFG2_PRE_DEGAM__PRE_DEGAM_SELECT_MASK                                                            0x00000070L
+//CNVC_CFG2_PRE_REALPHA
+#define CNVC_CFG2_PRE_REALPHA__PRE_REALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG2_PRE_REALPHA__PRE_REALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG2_PRE_REALPHA__PRE_REALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG2_PRE_REALPHA__PRE_REALPHA_ABLND_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cm_cur_dispdec
+//CM_CUR2_CURSOR0_CONTROL
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_ENABLE__SHIFT                                                           0x0
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_EXPANSION_MODE__SHIFT                                                   0x1
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_PIX_INV_MODE__SHIFT                                                     0x2
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_ROM_EN__SHIFT                                                           0x3
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_MODE__SHIFT                                                             0x4
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN__SHIFT                                               0x7
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_UPDATE_PENDING__SHIFT                                                   0x10
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_ENABLE_MASK                                                             0x00000001L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_EXPANSION_MODE_MASK                                                     0x00000002L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_PIX_INV_MODE_MASK                                                       0x00000004L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_ROM_EN_MASK                                                             0x00000008L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_MODE_MASK                                                               0x00000070L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN_MASK                                                 0x00000080L
+#define CM_CUR2_CURSOR0_CONTROL__CUR0_UPDATE_PENDING_MASK                                                     0x00010000L
+//CM_CUR2_CURSOR0_COLOR0
+#define CM_CUR2_CURSOR0_COLOR0__CUR0_COLOR0__SHIFT                                                            0x0
+#define CM_CUR2_CURSOR0_COLOR0__CUR0_COLOR0_MASK                                                              0x00FFFFFFL
+//CM_CUR2_CURSOR0_COLOR1
+#define CM_CUR2_CURSOR0_COLOR1__CUR0_COLOR1__SHIFT                                                            0x0
+#define CM_CUR2_CURSOR0_COLOR1__CUR0_COLOR1_MASK                                                              0x00FFFFFFL
+//CM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y__SHIFT                                           0x0
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y__SHIFT                                            0x10
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y_MASK                                             0x0000FFFFL
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y_MASK                                              0xFFFF0000L
+//CM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB__SHIFT                                   0x0
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB__SHIFT                                    0x10
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB_MASK                                     0x0000FFFFL
+#define CM_CUR2_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB_MASK                                      0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_MODE
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE__SHIFT                                                     0x0
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT__SHIFT                                             0x2
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT__SHIFT                                              0x4
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_MASK                                                       0x00000003L
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT_MASK                                               0x0000000CL
+#define CM_CUR2_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT_MASK                                                0x00000010L
+//CM_CUR2_CUR0_MATRIX_C11_C12_A
+#define CM_CUR2_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C13_C14_A
+#define CM_CUR2_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C21_C22_A
+#define CM_CUR2_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C23_C24_A
+#define CM_CUR2_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C31_C32_A
+#define CM_CUR2_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C33_C34_A
+#define CM_CUR2_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C11_C12_B
+#define CM_CUR2_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C13_C14_B
+#define CM_CUR2_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C21_C22_B
+#define CM_CUR2_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C23_C24_B
+#define CM_CUR2_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C31_C32_B
+#define CM_CUR2_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B_MASK                                                 0xFFFF0000L
+//CM_CUR2_CUR0_MATRIX_C33_C34_B
+#define CM_CUR2_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B__SHIFT                                               0x0
+#define CM_CUR2_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B__SHIFT                                               0x10
+#define CM_CUR2_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B_MASK                                                 0x0000FFFFL
+#define CM_CUR2_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B_MASK                                                 0xFFFF0000L
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dscl_dispdec
+//DSCL2_SCL_COEF_RAM_TAP_SELECT
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX__SHIFT                                       0x0
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE__SHIFT                                              0x8
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE__SHIFT                                        0x10
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX_MASK                                         0x00000003L
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE_MASK                                                0x00003F00L
+#define DSCL2_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE_MASK                                          0x00070000L
+//DSCL2_SCL_COEF_RAM_TAP_DATA
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF__SHIFT                                        0x0
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN__SHIFT                                     0xf
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF__SHIFT                                         0x10
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN__SHIFT                                      0x1f
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_MASK                                          0x00003FFFL
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN_MASK                                       0x00008000L
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_MASK                                           0x3FFF0000L
+#define DSCL2_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN_MASK                                        0x80000000L
+//DSCL2_SCL_MODE
+#define DSCL2_SCL_MODE__DSCL_MODE__SHIFT                                                                      0x0
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT__SHIFT                                                            0x8
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT__SHIFT                                                    0xc
+#define DSCL2_SCL_MODE__SCL_CHROMA_COEF_MODE__SHIFT                                                           0x10
+#define DSCL2_SCL_MODE__SCL_ALPHA_COEF_MODE__SHIFT                                                            0x14
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT_RD__SHIFT                                                         0x18
+#define DSCL2_SCL_MODE__DSCL_MODE_MASK                                                                        0x00000007L
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT_MASK                                                              0x00000100L
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT_MASK                                                      0x00001000L
+#define DSCL2_SCL_MODE__SCL_CHROMA_COEF_MODE_MASK                                                             0x00010000L
+#define DSCL2_SCL_MODE__SCL_ALPHA_COEF_MODE_MASK                                                              0x00100000L
+#define DSCL2_SCL_MODE__SCL_COEF_RAM_SELECT_RD_MASK                                                           0x01000000L
+//DSCL2_SCL_TAP_CONTROL
+#define DSCL2_SCL_TAP_CONTROL__SCL_V_NUM_TAPS__SHIFT                                                          0x0
+#define DSCL2_SCL_TAP_CONTROL__SCL_H_NUM_TAPS__SHIFT                                                          0x4
+#define DSCL2_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C__SHIFT                                                        0x8
+#define DSCL2_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C__SHIFT                                                        0xc
+#define DSCL2_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_MASK                                                            0x00000007L
+#define DSCL2_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_MASK                                                            0x00000070L
+#define DSCL2_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C_MASK                                                          0x00000700L
+#define DSCL2_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C_MASK                                                          0x00007000L
+//DSCL2_DSCL_CONTROL
+#define DSCL2_DSCL_CONTROL__SCL_BOUNDARY_MODE__SHIFT                                                          0x0
+#define DSCL2_DSCL_CONTROL__SCL_BOUNDARY_MODE_MASK                                                            0x00000001L
+//DSCL2_DSCL_2TAP_CONTROL
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x0
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN__SHIFT                                                   0x4
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR__SHIFT                                               0x8
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x10
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN__SHIFT                                                   0x14
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR__SHIFT                                               0x18
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN_MASK                                             0x00000001L
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN_MASK                                                     0x00000010L
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR_MASK                                                 0x00000700L
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN_MASK                                             0x00010000L
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN_MASK                                                     0x00100000L
+#define DSCL2_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR_MASK                                                 0x07000000L
+//DSCL2_SCL_MANUAL_REPLICATE_CONTROL
+#define DSCL2_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR__SHIFT                              0x0
+#define DSCL2_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR__SHIFT                              0x8
+#define DSCL2_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR_MASK                                0x0000000FL
+#define DSCL2_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR_MASK                                0x00000F00L
+//DSCL2_SCL_HORZ_FILTER_SCALE_RATIO
+#define DSCL2_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL2_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL2_SCL_HORZ_FILTER_INIT
+#define DSCL2_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL2_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT__SHIFT                                                     0x18
+#define DSCL2_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL2_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT_MASK                                                       0x0F000000L
+//DSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C
+#define DSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL2_SCL_HORZ_FILTER_INIT_C
+#define DSCL2_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL2_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL2_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL2_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL2_SCL_VERT_FILTER_SCALE_RATIO
+#define DSCL2_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL2_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL2_SCL_VERT_FILTER_INIT
+#define DSCL2_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL2_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT__SHIFT                                                     0x18
+#define DSCL2_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL2_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT_MASK                                                       0x0F000000L
+//DSCL2_SCL_VERT_FILTER_INIT_BOT
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT__SHIFT                                            0x0
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT__SHIFT                                             0x18
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT_MASK                                              0x00FFFFFFL
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT_MASK                                               0x0F000000L
+//DSCL2_SCL_VERT_FILTER_SCALE_RATIO_C
+#define DSCL2_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL2_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL2_SCL_VERT_FILTER_INIT_C
+#define DSCL2_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL2_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL2_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL2_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL2_SCL_VERT_FILTER_INIT_BOT_C
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C__SHIFT                                        0x0
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C__SHIFT                                         0x18
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C_MASK                                          0x00FFFFFFL
+#define DSCL2_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C_MASK                                           0x0F000000L
+//DSCL2_SCL_BLACK_COLOR
+#define DSCL2_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y__SHIFT                                                   0x0
+#define DSCL2_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR__SHIFT                                                    0x10
+#define DSCL2_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y_MASK                                                     0x0000FFFFL
+#define DSCL2_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR_MASK                                                      0xFFFF0000L
+//DSCL2_DSCL_UPDATE
+#define DSCL2_DSCL_UPDATE__SCL_UPDATE_PENDING__SHIFT                                                          0x0
+#define DSCL2_DSCL_UPDATE__SCL_UPDATE_PENDING_MASK                                                            0x00000001L
+//DSCL2_DSCL_AUTOCAL
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_MODE__SHIFT                                                               0x0
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE__SHIFT                                                           0x8
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_PIPE_ID__SHIFT                                                            0xc
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_MODE_MASK                                                                 0x00000003L
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE_MASK                                                             0x00000300L
+#define DSCL2_DSCL_AUTOCAL__AUTOCAL_PIPE_ID_MASK                                                              0x00003000L
+//DSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT
+#define DSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT__SHIFT                                         0x0
+#define DSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT__SHIFT                                          0x10
+#define DSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT_MASK                                           0x00003FFFL
+#define DSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT_MASK                                            0x3FFF0000L
+//DSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM
+#define DSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM__SHIFT                                        0x0
+#define DSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP__SHIFT                                           0x10
+#define DSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM_MASK                                          0x00003FFFL
+#define DSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP_MASK                                             0x3FFF0000L
+//DSCL2_OTG_H_BLANK
+#define DSCL2_OTG_H_BLANK__OTG_H_BLANK_START__SHIFT                                                           0x0
+#define DSCL2_OTG_H_BLANK__OTG_H_BLANK_END__SHIFT                                                             0x10
+#define DSCL2_OTG_H_BLANK__OTG_H_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL2_OTG_H_BLANK__OTG_H_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL2_OTG_V_BLANK
+#define DSCL2_OTG_V_BLANK__OTG_V_BLANK_START__SHIFT                                                           0x0
+#define DSCL2_OTG_V_BLANK__OTG_V_BLANK_END__SHIFT                                                             0x10
+#define DSCL2_OTG_V_BLANK__OTG_V_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL2_OTG_V_BLANK__OTG_V_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL2_RECOUT_START
+#define DSCL2_RECOUT_START__RECOUT_START_X__SHIFT                                                             0x0
+#define DSCL2_RECOUT_START__RECOUT_START_Y__SHIFT                                                             0x10
+#define DSCL2_RECOUT_START__RECOUT_START_X_MASK                                                               0x00003FFFL
+#define DSCL2_RECOUT_START__RECOUT_START_Y_MASK                                                               0x3FFF0000L
+//DSCL2_RECOUT_SIZE
+#define DSCL2_RECOUT_SIZE__RECOUT_WIDTH__SHIFT                                                                0x0
+#define DSCL2_RECOUT_SIZE__RECOUT_HEIGHT__SHIFT                                                               0x10
+#define DSCL2_RECOUT_SIZE__RECOUT_WIDTH_MASK                                                                  0x00003FFFL
+#define DSCL2_RECOUT_SIZE__RECOUT_HEIGHT_MASK                                                                 0x3FFF0000L
+//DSCL2_MPC_SIZE
+#define DSCL2_MPC_SIZE__MPC_WIDTH__SHIFT                                                                      0x0
+#define DSCL2_MPC_SIZE__MPC_HEIGHT__SHIFT                                                                     0x10
+#define DSCL2_MPC_SIZE__MPC_WIDTH_MASK                                                                        0x00003FFFL
+#define DSCL2_MPC_SIZE__MPC_HEIGHT_MASK                                                                       0x3FFF0000L
+//DSCL2_LB_DATA_FORMAT
+#define DSCL2_LB_DATA_FORMAT__INTERLEAVE_EN__SHIFT                                                            0x0
+#define DSCL2_LB_DATA_FORMAT__ALPHA_EN__SHIFT                                                                 0x4
+#define DSCL2_LB_DATA_FORMAT__INTERLEAVE_EN_MASK                                                              0x00000001L
+#define DSCL2_LB_DATA_FORMAT__ALPHA_EN_MASK                                                                   0x00000010L
+//DSCL2_LB_MEMORY_CTRL
+#define DSCL2_LB_MEMORY_CTRL__MEMORY_CONFIG__SHIFT                                                            0x0
+#define DSCL2_LB_MEMORY_CTRL__LB_MAX_PARTITIONS__SHIFT                                                        0x8
+#define DSCL2_LB_MEMORY_CTRL__LB_NUM_PARTITIONS__SHIFT                                                        0x10
+#define DSCL2_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C__SHIFT                                                      0x18
+#define DSCL2_LB_MEMORY_CTRL__MEMORY_CONFIG_MASK                                                              0x00000003L
+#define DSCL2_LB_MEMORY_CTRL__LB_MAX_PARTITIONS_MASK                                                          0x00003F00L
+#define DSCL2_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_MASK                                                          0x007F0000L
+#define DSCL2_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C_MASK                                                        0x7F000000L
+//DSCL2_LB_V_COUNTER
+#define DSCL2_LB_V_COUNTER__V_COUNTER__SHIFT                                                                  0x0
+#define DSCL2_LB_V_COUNTER__V_COUNTER_C__SHIFT                                                                0x10
+#define DSCL2_LB_V_COUNTER__V_COUNTER_MASK                                                                    0x00001FFFL
+#define DSCL2_LB_V_COUNTER__V_COUNTER_C_MASK                                                                  0x1FFF0000L
+//DSCL2_DSCL_MEM_PWR_CTRL
+#define DSCL2_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DSCL2_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS__SHIFT                                                       0x2
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE__SHIFT                                                   0x4
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS__SHIFT                                                     0x6
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE__SHIFT                                                   0x8
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS__SHIFT                                                     0xa
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE__SHIFT                                                   0xc
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS__SHIFT                                                     0xe
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE__SHIFT                                                   0x10
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS__SHIFT                                                     0x12
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE__SHIFT                                                   0x14
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS__SHIFT                                                     0x16
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE__SHIFT                                                   0x18
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS__SHIFT                                                     0x1a
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE__SHIFT                                                       0x1c
+#define DSCL2_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS_MASK                                                         0x00000004L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE_MASK                                                     0x00000030L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS_MASK                                                       0x00000040L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE_MASK                                                     0x00000300L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS_MASK                                                       0x00000400L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE_MASK                                                     0x00003000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS_MASK                                                       0x00004000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE_MASK                                                     0x00030000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS_MASK                                                       0x00040000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE_MASK                                                     0x00300000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS_MASK                                                       0x00400000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE_MASK                                                     0x03000000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS_MASK                                                       0x04000000L
+#define DSCL2_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE_MASK                                                         0x10000000L
+//DSCL2_DSCL_MEM_PWR_STATUS
+#define DSCL2_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE__SHIFT                                                   0x0
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE__SHIFT                                                 0x2
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE__SHIFT                                                 0x4
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE__SHIFT                                                 0x6
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE__SHIFT                                                 0x8
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE__SHIFT                                                 0xa
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE__SHIFT                                                 0xc
+#define DSCL2_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE_MASK                                                     0x00000003L
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE_MASK                                                   0x0000000CL
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE_MASK                                                   0x00000030L
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE_MASK                                                   0x000000C0L
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE_MASK                                                   0x00000300L
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE_MASK                                                   0x00000C00L
+#define DSCL2_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE_MASK                                                   0x00003000L
+//DSCL2_OBUF_CONTROL
+#define DSCL2_OBUF_CONTROL__OBUF_BYPASS__SHIFT                                                                0x0
+#define DSCL2_OBUF_CONTROL__OBUF_USE_FULL_BUFFER__SHIFT                                                       0x1
+#define DSCL2_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH__SHIFT                                                  0x2
+#define DSCL2_OBUF_CONTROL__OBUF_OUT_HOLD_CNT__SHIFT                                                          0x4
+#define DSCL2_OBUF_CONTROL__OBUF_BYPASS_MASK                                                                  0x00000001L
+#define DSCL2_OBUF_CONTROL__OBUF_USE_FULL_BUFFER_MASK                                                         0x00000002L
+#define DSCL2_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH_MASK                                                    0x00000004L
+#define DSCL2_OBUF_CONTROL__OBUF_OUT_HOLD_CNT_MASK                                                            0x000000F0L
+//DSCL2_OBUF_MEM_PWR_CTRL
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE__SHIFT                                                    0x0
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS__SHIFT                                                      0x2
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE__SHIFT                                                     0x8
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE__SHIFT                                                    0x10
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE_MASK                                                      0x00000003L
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS_MASK                                                        0x00000004L
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE_MASK                                                       0x00000100L
+#define DSCL2_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE_MASK                                                      0x00030000L
+//DSCL2_DSCL_EASF_H_MODE
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_EN__SHIFT                                                          0x0
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_EN_MASK                                                            0x00000001L
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL2_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL2_DSCL_EASF_V_MODE
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_EN__SHIFT                                                          0x0
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_EN_MASK                                                            0x00000001L
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL2_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL2_DSCL_SC_MODE
+#define DSCL2_DSCL_SC_MODE__SCL_SC_MATRIX_MODE__SHIFT                                                         0x0
+#define DSCL2_DSCL_SC_MODE__SCL_SC_LTONL_EN__SHIFT                                                            0x8
+#define DSCL2_DSCL_SC_MODE__SCL_SC_MATRIX_MODE_MASK                                                           0x00000001L
+#define DSCL2_DSCL_SC_MODE__SCL_SC_LTONL_EN_MASK                                                              0x00000100L
+//DSCL2_DSCL_SC_MATRIX_C0C1
+#define DSCL2_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0__SHIFT                                                    0x0
+#define DSCL2_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1__SHIFT                                                    0x10
+#define DSCL2_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0_MASK                                                      0x0000FFFFL
+#define DSCL2_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1_MASK                                                      0xFFFF0000L
+//DSCL2_DSCL_SC_MATRIX_C2C3
+#define DSCL2_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2__SHIFT                                                    0x0
+#define DSCL2_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3__SHIFT                                                    0x10
+#define DSCL2_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2_MASK                                                      0x0000FFFFL
+#define DSCL2_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3_MASK                                                      0xFFFF0000L
+//DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL2_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL2_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT__SHIFT                    0x0
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL__SHIFT                    0x10
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT_MASK                      0x0000FFFFL
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL_MASK                      0xFFFF0000L
+//DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE__SHIFT                     0x0
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE__SHIFT                    0x10
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE_MASK                       0x0000FFFFL
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE_MASK                      0xFFFF0000L
+//DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE__SHIFT                    0x0
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET__SHIFT                   0x10
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE_MASK                      0x0000FFFFL
+#define DSCL2_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET_MASK                     0xFFFF0000L
+//DSCL2_DSCL_EASF_RINGEST_FORCE
+#define DSCL2_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE__SHIFT                                        0x0
+#define DSCL2_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE__SHIFT                                        0x10
+#define DSCL2_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE_MASK                                          0x0000FFFFL
+#define DSCL2_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE_MASK                                          0xFFFF0000L
+//DSCL2_DSCL_EASF_H_BF_CNTL
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN__SHIFT                                                   0x0
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE__SHIFT                                                 0x8
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE__SHIFT                                                 0x10
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN_MASK                                                     0x00000001L
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL2_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA__SHIFT                                         0x0
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB__SHIFT                                         0x8
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA__SHIFT                                         0x10
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB__SHIFT                                         0x18
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA_MASK                                           0x003F0000L
+#define DSCL2_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB_MASK                                           0x3F000000L
+//DSCL2_DSCL_EASF_V_BF_CNTL
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN__SHIFT                                                   0x0
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE__SHIFT                                                 0x8
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE__SHIFT                                                 0x10
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN_MASK                                                     0x00000001L
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL2_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA__SHIFT                                         0x0
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB__SHIFT                                         0x8
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA__SHIFT                                         0x10
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB__SHIFT                                         0x18
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA_MASK                                           0x003F0000L
+#define DSCL2_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB_MASK                                           0x3F000000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG1
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG2
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG3
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG4
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG5
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG6
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_H_BF1_PWL_SEG7
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG1
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG2
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG3
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG4
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG5
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG6
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL2_DSCL_EASF_V_BF1_PWL_SEG7
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL2_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG1
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG2
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG3
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG4
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_H_BF3_PWL_SEG5
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG1
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG2
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG3
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG4
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL2_DSCL_EASF_V_BF3_PWL_SEG5
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL2_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL2_ISHARP_MODE
+#define DSCL2_ISHARP_MODE__ISHARP_EN__SHIFT                                                                   0x0
+#define DSCL2_ISHARP_MODE__ISHARP_NOISEDET_EN__SHIFT                                                          0x4
+#define DSCL2_ISHARP_MODE__ISHARP_NOISEDET_MODE__SHIFT                                                        0x5
+#define DSCL2_ISHARP_MODE__ISHARP_LBA_MODE__SHIFT                                                             0x9
+#define DSCL2_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT__SHIFT                                                     0xa
+#define DSCL2_ISHARP_MODE__ISHARP_FMT_MODE__SHIFT                                                             0xb
+#define DSCL2_ISHARP_MODE__ISHARP_FMT_NORM__SHIFT                                                             0xc
+#define DSCL2_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT__SHIFT                                             0x1c
+#define DSCL2_ISHARP_MODE__ISHARP_EN_MASK                                                                     0x00000001L
+#define DSCL2_ISHARP_MODE__ISHARP_NOISEDET_EN_MASK                                                            0x00000010L
+#define DSCL2_ISHARP_MODE__ISHARP_NOISEDET_MODE_MASK                                                          0x00000060L
+#define DSCL2_ISHARP_MODE__ISHARP_LBA_MODE_MASK                                                               0x00000200L
+#define DSCL2_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_MASK                                                       0x00000400L
+#define DSCL2_ISHARP_MODE__ISHARP_FMT_MODE_MASK                                                               0x00000800L
+#define DSCL2_ISHARP_MODE__ISHARP_FMT_NORM_MASK                                                               0x0FFFF000L
+#define DSCL2_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT_MASK                                               0x10000000L
+//DSCL2_ISHARP_DELTA_CTRL
+#define DSCL2_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT__SHIFT                                          0x0
+#define DSCL2_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT_MASK                                            0x00000001L
+//DSCL2_ISHARP_DELTA_INDEX
+#define DSCL2_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX__SHIFT                                                   0x0
+#define DSCL2_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX_MASK                                                     0x0000001FL
+//DSCL2_ISHARP_DELTA_DATA
+#define DSCL2_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA__SHIFT                                                     0x0
+#define DSCL2_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA_MASK                                                       0xFFFFFFFFL
+//DSCL2_ISHARP_NLDELTA_SOFT_CLIP
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P__SHIFT                                      0x0
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P__SHIFT                                   0x1
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P__SHIFT                                   0x8
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N__SHIFT                                      0x10
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N__SHIFT                                   0x11
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N__SHIFT                                   0x18
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P_MASK                                        0x00000001L
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P_MASK                                     0x000000FEL
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P_MASK                                     0x0000FF00L
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N_MASK                                        0x00010000L
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N_MASK                                     0x00FE0000L
+#define DSCL2_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N_MASK                                     0xFF000000L
+//DSCL2_ISHARP_NOISEDET_THRESHOLD
+#define DSCL2_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE__SHIFT                                         0x0
+#define DSCL2_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE__SHIFT                                         0x10
+#define DSCL2_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE_MASK                                           0x000003FFL
+#define DSCL2_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE_MASK                                           0x03FF0000L
+//DSCL2_ISHARP_NOISE_GAIN_PWL
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN__SHIFT                                      0x0
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN__SHIFT                                        0x8
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE__SHIFT                                         0x10
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN_MASK                                        0x0000001FL
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN_MASK                                          0x00001F00L
+#define DSCL2_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE_MASK                                           0x3FFF0000L
+//DSCL2_ISHARP_LBA_PWL_SEG0
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0__SHIFT                                           0x14
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0_MASK                                              0x0003F000L
+#define DSCL2_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0_MASK                                             0x1FF00000L
+//DSCL2_ISHARP_LBA_PWL_SEG1
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1__SHIFT                                           0x14
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1_MASK                                              0x0003F000L
+#define DSCL2_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1_MASK                                             0x1FF00000L
+//DSCL2_ISHARP_LBA_PWL_SEG2
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2__SHIFT                                           0x14
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2_MASK                                              0x0003F000L
+#define DSCL2_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2_MASK                                             0x1FF00000L
+//DSCL2_ISHARP_LBA_PWL_SEG3
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3__SHIFT                                           0x14
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3_MASK                                              0x0003F000L
+#define DSCL2_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3_MASK                                             0x1FF00000L
+//DSCL2_ISHARP_LBA_PWL_SEG4
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4__SHIFT                                           0x14
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4_MASK                                              0x0003F000L
+#define DSCL2_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4_MASK                                             0x1FF00000L
+//DSCL2_ISHARP_LBA_PWL_SEG5
+#define DSCL2_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5__SHIFT                                              0x0
+#define DSCL2_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5__SHIFT                                            0xc
+#define DSCL2_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5_MASK                                                0x000003FFL
+#define DSCL2_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5_MASK                                              0x0003F000L
+//DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE__SHIFT                            0x0
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS__SHIFT                              0x2
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE__SHIFT                            0x4
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE_MASK                              0x00000003L
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS_MASK                                0x00000004L
+#define DSCL2_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE_MASK                              0x00000030L
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cm_dispdec
+//CM2_CM_CONTROL
+#define CM2_CM_CONTROL__CM_BYPASS__SHIFT                                                                      0x0
+#define CM2_CM_CONTROL__CM_UPDATE_PENDING__SHIFT                                                              0x8
+#define CM2_CM_CONTROL__CM_BYPASS_MASK                                                                        0x00000001L
+#define CM2_CM_CONTROL__CM_UPDATE_PENDING_MASK                                                                0x00000100L
+//CM2_CM_POST_CSC_CONTROL
+#define CM2_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE__SHIFT                                                      0x0
+#define CM2_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT__SHIFT                                              0x2
+#define CM2_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_MASK                                                        0x00000003L
+#define CM2_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT_MASK                                                0x0000000CL
+//CM2_CM_POST_CSC_C11_C12
+#define CM2_CM_POST_CSC_C11_C12__CM_POST_CSC_C11__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C11_C12__CM_POST_CSC_C12__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C11_C12__CM_POST_CSC_C11_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C11_C12__CM_POST_CSC_C12_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_C13_C14
+#define CM2_CM_POST_CSC_C13_C14__CM_POST_CSC_C13__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C13_C14__CM_POST_CSC_C14__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C13_C14__CM_POST_CSC_C13_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C13_C14__CM_POST_CSC_C14_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_C21_C22
+#define CM2_CM_POST_CSC_C21_C22__CM_POST_CSC_C21__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C21_C22__CM_POST_CSC_C22__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C21_C22__CM_POST_CSC_C21_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C21_C22__CM_POST_CSC_C22_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_C23_C24
+#define CM2_CM_POST_CSC_C23_C24__CM_POST_CSC_C23__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C23_C24__CM_POST_CSC_C24__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C23_C24__CM_POST_CSC_C23_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C23_C24__CM_POST_CSC_C24_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_C31_C32
+#define CM2_CM_POST_CSC_C31_C32__CM_POST_CSC_C31__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C31_C32__CM_POST_CSC_C32__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C31_C32__CM_POST_CSC_C31_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C31_C32__CM_POST_CSC_C32_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_C33_C34
+#define CM2_CM_POST_CSC_C33_C34__CM_POST_CSC_C33__SHIFT                                                       0x0
+#define CM2_CM_POST_CSC_C33_C34__CM_POST_CSC_C34__SHIFT                                                       0x10
+#define CM2_CM_POST_CSC_C33_C34__CM_POST_CSC_C33_MASK                                                         0x0000FFFFL
+#define CM2_CM_POST_CSC_C33_C34__CM_POST_CSC_C34_MASK                                                         0xFFFF0000L
+//CM2_CM_POST_CSC_B_C11_C12
+#define CM2_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12_MASK                                                     0xFFFF0000L
+//CM2_CM_POST_CSC_B_C13_C14
+#define CM2_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14_MASK                                                     0xFFFF0000L
+//CM2_CM_POST_CSC_B_C21_C22
+#define CM2_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22_MASK                                                     0xFFFF0000L
+//CM2_CM_POST_CSC_B_C23_C24
+#define CM2_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24_MASK                                                     0xFFFF0000L
+//CM2_CM_POST_CSC_B_C31_C32
+#define CM2_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32_MASK                                                     0xFFFF0000L
+//CM2_CM_POST_CSC_B_C33_C34
+#define CM2_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33__SHIFT                                                   0x0
+#define CM2_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34__SHIFT                                                   0x10
+#define CM2_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33_MASK                                                     0x0000FFFFL
+#define CM2_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34_MASK                                                     0xFFFF0000L
+//CM2_CM_BIAS_CR_R
+#define CM2_CM_BIAS_CR_R__CM_BIAS_CR_R__SHIFT                                                                 0x0
+#define CM2_CM_BIAS_CR_R__CM_BIAS_CR_R_MASK                                                                   0x0000FFFFL
+//CM2_CM_BIAS_Y_G_CB_B
+#define CM2_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G__SHIFT                                                              0x0
+#define CM2_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B__SHIFT                                                             0x10
+#define CM2_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G_MASK                                                                0x0000FFFFL
+#define CM2_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B_MASK                                                               0xFFFF0000L
+//CM2_CM_GAMCOR_CONTROL
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE__SHIFT                                                          0x0
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT__SHIFT                                                        0x2
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE__SHIFT                                                   0x3
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT__SHIFT                                                  0x4
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT__SHIFT                                                0x6
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_MASK                                                            0x00000003L
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_MASK                                                          0x00000004L
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE_MASK                                                     0x00000008L
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT_MASK                                                    0x00000030L
+#define CM2_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT_MASK                                                  0x00000040L
+//CM2_CM_GAMCOR_LUT_INDEX
+#define CM2_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX__SHIFT                                                   0x0
+#define CM2_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX_MASK                                                     0x000001FFL
+//CM2_CM_GAMCOR_LUT_DATA
+#define CM2_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA__SHIFT                                                     0x0
+#define CM2_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA_MASK                                                       0x0003FFFFL
+//CM2_CM_GAMCOR_LUT_CONTROL
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK__SHIFT                                      0x0
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL__SHIFT                                        0x3
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG__SHIFT                                              0x5
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL__SHIFT                                              0x6
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE__SHIFT                                           0x7
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK_MASK                                        0x00000007L
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL_MASK                                          0x00000018L
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG_MASK                                                0x00000020L
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL_MASK                                                0x00000040L
+#define CM2_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE_MASK                                             0x00000080L
+//CM2_CM_GAMCOR_RAMA_START_CNTL_B
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMA_START_CNTL_G
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMA_START_CNTL_R
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_B
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_G
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_R
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_END_CNTL1_B
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_END_CNTL2_B
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMA_END_CNTL1_G
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_END_CNTL2_G
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMA_END_CNTL1_R
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMA_END_CNTL2_R
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMA_OFFSET_B
+#define CM2_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMA_OFFSET_G
+#define CM2_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMA_OFFSET_R
+#define CM2_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMA_REGION_0_1
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_2_3
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_4_5
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_6_7
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_8_9
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_10_11
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_12_13
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_14_15
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_16_17
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_18_19
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_20_21
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_22_23
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_24_25
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_26_27
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_28_29
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_30_31
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMA_REGION_32_33
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_START_CNTL_B
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMB_START_CNTL_G
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMB_START_CNTL_R
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R__SHIFT                             0x0
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM2_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM2_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_B
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_G
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_R
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM2_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_END_CNTL1_B
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_END_CNTL2_B
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMB_END_CNTL1_G
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_END_CNTL2_G
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMB_END_CNTL1_R
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM2_CM_GAMCOR_RAMB_END_CNTL2_R
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R__SHIFT                                0x0
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM2_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM2_CM_GAMCOR_RAMB_OFFSET_B
+#define CM2_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMB_OFFSET_G
+#define CM2_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMB_OFFSET_R
+#define CM2_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R__SHIFT                                           0x0
+#define CM2_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R_MASK                                             0x0007FFFFL
+//CM2_CM_GAMCOR_RAMB_REGION_0_1
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_2_3
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_4_5
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_6_7
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_8_9
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_10_11
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_12_13
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_14_15
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_16_17
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_18_19
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_20_21
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_22_23
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_24_25
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_26_27
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_28_29
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_30_31
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_GAMCOR_RAMB_REGION_32_33
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM2_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM2_CM_HDR_MULT_COEF
+#define CM2_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF__SHIFT                                                         0x0
+#define CM2_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF_MASK                                                           0x0007FFFFL
+//CM2_CM_MEM_PWR_CTRL
+#define CM2_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE__SHIFT                                                      0x0
+#define CM2_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS__SHIFT                                                        0x2
+#define CM2_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE__SHIFT                                                        0x8
+#define CM2_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS__SHIFT                                                          0x9
+#define CM2_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE_MASK                                                        0x00000003L
+#define CM2_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS_MASK                                                          0x00000004L
+#define CM2_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE_MASK                                                          0x00000100L
+#define CM2_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS_MASK                                                            0x00000200L
+//CM2_CM_MEM_PWR_STATUS
+#define CM2_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE__SHIFT                                                    0x0
+#define CM2_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE__SHIFT                                                      0x8
+#define CM2_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE_MASK                                                      0x00000003L
+#define CM2_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE_MASK                                                        0x00000100L
+//CM2_CM_DEALPHA
+#define CM2_CM_DEALPHA__CM_DEALPHA_EN__SHIFT                                                                  0x0
+#define CM2_CM_DEALPHA__CM_DEALPHA_ABLND__SHIFT                                                               0x1
+#define CM2_CM_DEALPHA__CM_DEALPHA_EN_MASK                                                                    0x00000001L
+#define CM2_CM_DEALPHA__CM_DEALPHA_ABLND_MASK                                                                 0x00000002L
+//CM2_CM_COEF_FORMAT
+#define CM2_CM_COEF_FORMAT__CM_BIAS_FORMAT__SHIFT                                                             0x0
+#define CM2_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT__SHIFT                                                    0x4
+#define CM2_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
+#define CM2_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
+//CM2_CM_TEST_DEBUG_INDEX
+#define CM2_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM2_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM2_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM2_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM2_CM_TEST_DEBUG_DATA
+#define CM2_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM2_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+//CM2_CM_HIST_CNTL
+#define CM2_CM_HIST_CNTL__CM_HIST_SEL__SHIFT                                                                  0x0
+#define CM2_CM_HIST_CNTL__CM_HIST_CH_EN__SHIFT                                                                0x2
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC1_SEL__SHIFT                                                             0x4
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC2_SEL__SHIFT                                                             0x5
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC3_SEL__SHIFT                                                             0x6
+#define CM2_CM_HIST_CNTL__CM_HIST_CH1_XBAR__SHIFT                                                             0x7
+#define CM2_CM_HIST_CNTL__CM_HIST_CH2_XBAR__SHIFT                                                             0x9
+#define CM2_CM_HIST_CNTL__CM_HIST_CH3_XBAR__SHIFT                                                             0xb
+#define CM2_CM_HIST_CNTL__CM_HIST_FORMAT__SHIFT                                                               0xd
+#define CM2_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK__SHIFT                                                    0xf
+#define CM2_CM_HIST_CNTL__CM_HIST_SEL_MASK                                                                    0x00000003L
+#define CM2_CM_HIST_CNTL__CM_HIST_CH_EN_MASK                                                                  0x0000000CL
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC1_SEL_MASK                                                               0x00000010L
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC2_SEL_MASK                                                               0x00000020L
+#define CM2_CM_HIST_CNTL__CM_HIST_SRC3_SEL_MASK                                                               0x00000040L
+#define CM2_CM_HIST_CNTL__CM_HIST_CH1_XBAR_MASK                                                               0x00000180L
+#define CM2_CM_HIST_CNTL__CM_HIST_CH2_XBAR_MASK                                                               0x00000600L
+#define CM2_CM_HIST_CNTL__CM_HIST_CH3_XBAR_MASK                                                               0x00001800L
+#define CM2_CM_HIST_CNTL__CM_HIST_FORMAT_MASK                                                                 0x00006000L
+#define CM2_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK_MASK                                                      0x00038000L
+//CM2_CM_HIST_SCALE_SRC1
+#define CM2_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1__SHIFT                                                     0x0
+#define CM2_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1_MASK                                                       0x0007FFFFL
+//CM2_CM_HIST_COEFA_SRC2
+#define CM2_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2__SHIFT                                                     0x0
+#define CM2_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2_MASK                                                       0x0007FFFFL
+//CM2_CM_HIST_COEFB_SRC2
+#define CM2_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2__SHIFT                                                     0x0
+#define CM2_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2_MASK                                                       0x0007FFFFL
+//CM2_CM_HIST_COEFC_SRC2
+#define CM2_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2__SHIFT                                                     0x0
+#define CM2_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2_MASK                                                       0x0007FFFFL
+//CM2_CM_HIST_SCALE_SRC3
+#define CM2_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3__SHIFT                                                     0x0
+#define CM2_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3_MASK                                                       0x0007FFFFL
+//CM2_CM_HIST_BIAS_SRC1
+#define CM2_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1__SHIFT                                                       0x0
+#define CM2_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1_MASK                                                         0x0007FFFFL
+//CM2_CM_HIST_BIAS_SRC2
+#define CM2_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2__SHIFT                                                       0x0
+#define CM2_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2_MASK                                                         0x0007FFFFL
+//CM2_CM_HIST_BIAS_SRC3
+#define CM2_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3__SHIFT                                                       0x0
+#define CM2_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3_MASK                                                         0x0007FFFFL
+//CM2_CM_HIST_LOCK
+#define CM2_CM_HIST_LOCK__CM_HIST_LOCK__SHIFT                                                                 0x0
+#define CM2_CM_HIST_LOCK__CM_HIST_LOCK_MASK                                                                   0x00000001L
+//CM2_CM_HIST_INDEX
+#define CM2_CM_HIST_INDEX__CM_HIST_INDEX__SHIFT                                                               0x0
+#define CM2_CM_HIST_INDEX__CM_HIST_INDEX_MASK                                                                 0x000000FFL
+//CM2_CM_HIST_DATA
+#define CM2_CM_HIST_DATA__CM_HIST_DATA__SHIFT                                                                 0x0
+#define CM2_CM_HIST_DATA__CM_HIST_DATA_MASK                                                                   0x01FFFFFFL
+//CM2_CM_HIST_STATUS
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS__SHIFT                                                    0x0
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS__SHIFT                                                    0x1
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT__SHIFT                                                 0x2
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ__SHIFT                                                    0x3
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED__SHIFT                                              0x4
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT__SHIFT                                      0x5
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT__SHIFT                                          0x6
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED__SHIFT                                            0xc
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0xd
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED__SHIFT                                            0x10
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0x11
+#define CM2_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW__SHIFT                                                     0x18
+#define CM2_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT__SHIFT                                             0x19
+#define CM2_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE__SHIFT                                                 0x1a
+#define CM2_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT__SHIFT                                         0x1b
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS_MASK                                                      0x00000001L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS_MASK                                                      0x00000002L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT_MASK                                                   0x00000004L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ_MASK                                                      0x00000008L
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_MASK                                                0x00000010L
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT_MASK                                        0x00000020L
+#define CM2_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT_MASK                                            0x000001C0L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_MASK                                              0x00001000L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00002000L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_MASK                                              0x00010000L
+#define CM2_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00020000L
+#define CM2_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_MASK                                                       0x01000000L
+#define CM2_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT_MASK                                               0x02000000L
+#define CM2_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_MASK                                                   0x04000000L
+#define CM2_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT_MASK                                           0x08000000L
+//CM2_CM_HIST_INT_CONTROL
+#define CM2_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT__SHIFT                                                       0x0
+#define CM2_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN__SHIFT                                                    0x4
+#define CM2_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_MASK                                                         0x00000001L
+#define CM2_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON12_PERFCOUNTER_CNTL
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON12_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON12_PERFCOUNTER_CNTL2
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON12_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON12_PERFCOUNTER_STATE
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON12_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON12_PERFMON_CNTL
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON12_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON12_PERFMON_CNTL2
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON12_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON12_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON12_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON12_PERFMON_CVALUE_LOW
+#define DC_PERFMON12_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON12_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON12_PERFMON_HI
+#define DC_PERFMON12_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON12_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON12_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON12_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON12_PERFMON_LOW
+#define DC_PERFMON12_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON12_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_top_dispdec
+//DPP_TOP3_DPP_CONTROL
+#define DPP_TOP3_DPP_CONTROL__DPP_CLOCK_ENABLE__SHIFT                                                         0x4
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_GATE_DISABLE__SHIFT                                                    0x8
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE__SHIFT                                                0xa
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE__SHIFT                                               0xc
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_R_GATE_DISABLE__SHIFT                                                    0xe
+#define DPP_TOP3_DPP_CONTROL__DISPCLK_R_GATE_DISABLE__SHIFT                                                   0x10
+#define DPP_TOP3_DPP_CONTROL__DISPCLK_G_GATE_DISABLE__SHIFT                                                   0x12
+#define DPP_TOP3_DPP_CONTROL__DPP_FGCG_REP_DIS__SHIFT                                                         0x18
+#define DPP_TOP3_DPP_CONTROL__DPP_TEST_CLK_SEL__SHIFT                                                         0x1c
+#define DPP_TOP3_DPP_CONTROL__DPP_CLOCK_ENABLE_MASK                                                           0x00000010L
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_GATE_DISABLE_MASK                                                      0x00000100L
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_DYN_GATE_DISABLE_MASK                                                  0x00000400L
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_G_DSCL_GATE_DISABLE_MASK                                                 0x00001000L
+#define DPP_TOP3_DPP_CONTROL__DPPCLK_R_GATE_DISABLE_MASK                                                      0x00004000L
+#define DPP_TOP3_DPP_CONTROL__DISPCLK_R_GATE_DISABLE_MASK                                                     0x00010000L
+#define DPP_TOP3_DPP_CONTROL__DISPCLK_G_GATE_DISABLE_MASK                                                     0x00040000L
+#define DPP_TOP3_DPP_CONTROL__DPP_FGCG_REP_DIS_MASK                                                           0x01000000L
+#define DPP_TOP3_DPP_CONTROL__DPP_TEST_CLK_SEL_MASK                                                           0x70000000L
+//DPP_TOP3_DPP_SOFT_RESET
+#define DPP_TOP3_DPP_SOFT_RESET__CNVC_SOFT_RESET__SHIFT                                                       0x0
+#define DPP_TOP3_DPP_SOFT_RESET__DSCL_SOFT_RESET__SHIFT                                                       0x4
+#define DPP_TOP3_DPP_SOFT_RESET__CM_SOFT_RESET__SHIFT                                                         0x8
+#define DPP_TOP3_DPP_SOFT_RESET__OBUF_SOFT_RESET__SHIFT                                                       0xc
+#define DPP_TOP3_DPP_SOFT_RESET__HIST_SOFT_RESET__SHIFT                                                       0x10
+#define DPP_TOP3_DPP_SOFT_RESET__CNVC_SOFT_RESET_MASK                                                         0x00000001L
+#define DPP_TOP3_DPP_SOFT_RESET__DSCL_SOFT_RESET_MASK                                                         0x00000010L
+#define DPP_TOP3_DPP_SOFT_RESET__CM_SOFT_RESET_MASK                                                           0x00000100L
+#define DPP_TOP3_DPP_SOFT_RESET__OBUF_SOFT_RESET_MASK                                                         0x00001000L
+#define DPP_TOP3_DPP_SOFT_RESET__HIST_SOFT_RESET_MASK                                                         0x00010000L
+//DPP_TOP3_DPP_CRC_VAL_R
+#define DPP_TOP3_DPP_CRC_VAL_R__DPP_CRC_R_CR__SHIFT                                                           0x0
+#define DPP_TOP3_DPP_CRC_VAL_R__DPP_CRC_R_CR_MASK                                                             0xFFFFFFFFL
+//DPP_TOP3_DPP_CRC_VAL_G
+#define DPP_TOP3_DPP_CRC_VAL_G__DPP_CRC_G_Y__SHIFT                                                            0x0
+#define DPP_TOP3_DPP_CRC_VAL_G__DPP_CRC_G_Y_MASK                                                              0xFFFFFFFFL
+//DPP_TOP3_DPP_CRC_VAL_B
+#define DPP_TOP3_DPP_CRC_VAL_B__DPP_CRC_B_CB__SHIFT                                                           0x0
+#define DPP_TOP3_DPP_CRC_VAL_B__DPP_CRC_B_CB_MASK                                                             0xFFFFFFFFL
+//DPP_TOP3_DPP_CRC_VAL_A
+#define DPP_TOP3_DPP_CRC_VAL_A__DPP_CRC_ALPHA__SHIFT                                                          0x0
+#define DPP_TOP3_DPP_CRC_VAL_A__DPP_CRC_ALPHA_MASK                                                            0xFFFFFFFFL
+//DPP_TOP3_DPP_CRC_CTRL
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_EN__SHIFT                                                              0x0
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_CONT_EN__SHIFT                                                         0x1
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING__SHIFT                                                0x2
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL__SHIFT                                                    0x3
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_SRC_SEL__SHIFT                                                         0x4
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_STEREO_EN__SHIFT                                                       0x6
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE__SHIFT                                                     0x7
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE__SHIFT                                                  0x9
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL__SHIFT                                                  0xb
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_MASK__SHIFT                                                            0x10
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_EN_MASK                                                                0x00000001L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_CONT_EN_MASK                                                           0x00000002L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_ONE_SHOT_PENDING_MASK                                                  0x00000004L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_420_COMP_SEL_MASK                                                      0x00000008L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_SRC_SEL_MASK                                                           0x00000030L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_STEREO_EN_MASK                                                         0x00000040L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_STEREO_MODE_MASK                                                       0x00000180L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_INTERLACE_MODE_MASK                                                    0x00000600L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_PIX_FORMAT_SEL_MASK                                                    0x00003800L
+#define DPP_TOP3_DPP_CRC_CTRL__DPP_CRC_MASK_MASK                                                              0xFFFF0000L
+//DPP_TOP3_HOST_READ_CONTROL
+#define DPP_TOP3_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL__SHIFT                                             0x0
+#define DPP_TOP3_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL_MASK                                               0x000000FFL
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cnvc_cfg_dispdec
+//CNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT
+#define CNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT__SHIFT                                 0x0
+#define CNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE__SHIFT                                   0x8
+#define CNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT__CNVC_SURFACE_PIXEL_FORMAT_MASK                                   0x0000007FL
+#define CNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT__CNVC_ALPHA_PLANE_ENABLE_MASK                                     0x00000100L
+//CNVC_CFG3_FORMAT_CONTROL
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_EXPANSION_MODE__SHIFT                                                0x0
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CNV16__SHIFT                                                         0x4
+#define CNVC_CFG3_FORMAT_CONTROL__ALPHA_EN__SHIFT                                                             0x8
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_BYPASS__SHIFT                                                          0xc
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN__SHIFT                                                0xd
+#define CNVC_CFG3_FORMAT_CONTROL__CLAMP_POSITIVE__SHIFT                                                       0x10
+#define CNVC_CFG3_FORMAT_CONTROL__CLAMP_POSITIVE_C__SHIFT                                                     0x11
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_UPDATE_PENDING__SHIFT                                                  0x14
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_R__SHIFT                                                    0x18
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_G__SHIFT                                                    0x1a
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_B__SHIFT                                                    0x1c
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_EXPANSION_MODE_MASK                                                  0x00000001L
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CNV16_MASK                                                           0x00000010L
+#define CNVC_CFG3_FORMAT_CONTROL__ALPHA_EN_MASK                                                               0x00000100L
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_BYPASS_MASK                                                            0x00001000L
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_BYPASS_MSB_ALIGN_MASK                                                  0x00002000L
+#define CNVC_CFG3_FORMAT_CONTROL__CLAMP_POSITIVE_MASK                                                         0x00010000L
+#define CNVC_CFG3_FORMAT_CONTROL__CLAMP_POSITIVE_C_MASK                                                       0x00020000L
+#define CNVC_CFG3_FORMAT_CONTROL__CNVC_UPDATE_PENDING_MASK                                                    0x00100000L
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_R_MASK                                                      0x03000000L
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_G_MASK                                                      0x0C000000L
+#define CNVC_CFG3_FORMAT_CONTROL__FORMAT_CROSSBAR_B_MASK                                                      0x30000000L
+//CNVC_CFG3_FCNV_FP_BIAS_R
+#define CNVC_CFG3_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R__SHIFT                                                       0x0
+#define CNVC_CFG3_FCNV_FP_BIAS_R__FCNV_FP_BIAS_R_MASK                                                         0x0007FFFFL
+//CNVC_CFG3_FCNV_FP_BIAS_G
+#define CNVC_CFG3_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G__SHIFT                                                       0x0
+#define CNVC_CFG3_FCNV_FP_BIAS_G__FCNV_FP_BIAS_G_MASK                                                         0x0007FFFFL
+//CNVC_CFG3_FCNV_FP_BIAS_B
+#define CNVC_CFG3_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B__SHIFT                                                       0x0
+#define CNVC_CFG3_FCNV_FP_BIAS_B__FCNV_FP_BIAS_B_MASK                                                         0x0007FFFFL
+//CNVC_CFG3_FCNV_FP_SCALE_R
+#define CNVC_CFG3_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R__SHIFT                                                     0x0
+#define CNVC_CFG3_FCNV_FP_SCALE_R__FCNV_FP_SCALE_R_MASK                                                       0x0007FFFFL
+//CNVC_CFG3_FCNV_FP_SCALE_G
+#define CNVC_CFG3_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G__SHIFT                                                     0x0
+#define CNVC_CFG3_FCNV_FP_SCALE_G__FCNV_FP_SCALE_G_MASK                                                       0x0007FFFFL
+//CNVC_CFG3_FCNV_FP_SCALE_B
+#define CNVC_CFG3_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B__SHIFT                                                     0x0
+#define CNVC_CFG3_FCNV_FP_SCALE_B__FCNV_FP_SCALE_B_MASK                                                       0x0007FFFFL
+//CNVC_CFG3_COLOR_KEYER_CONTROL
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__COLOR_KEYER_EN__SHIFT                                                  0x0
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__LUMA_KEYER_EN__SHIFT                                                   0x1
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE__SHIFT                                                0x4
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__COLOR_KEYER_EN_MASK                                                    0x00000001L
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__LUMA_KEYER_EN_MASK                                                     0x00000002L
+#define CNVC_CFG3_COLOR_KEYER_CONTROL__COLOR_KEYER_MODE_MASK                                                  0x00000030L
+//CNVC_CFG3_COLOR_KEYER_ALPHA
+#define CNVC_CFG3_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW__SHIFT                                             0x0
+#define CNVC_CFG3_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH__SHIFT                                            0x10
+#define CNVC_CFG3_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG3_COLOR_KEYER_ALPHA__COLOR_KEYER_ALPHA_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG3_COLOR_KEYER_RED
+#define CNVC_CFG3_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW__SHIFT                                                 0x0
+#define CNVC_CFG3_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH__SHIFT                                                0x10
+#define CNVC_CFG3_COLOR_KEYER_RED__COLOR_KEYER_RED_LOW_MASK                                                   0x0000FFFFL
+#define CNVC_CFG3_COLOR_KEYER_RED__COLOR_KEYER_RED_HIGH_MASK                                                  0xFFFF0000L
+//CNVC_CFG3_COLOR_KEYER_GREEN
+#define CNVC_CFG3_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW__SHIFT                                             0x0
+#define CNVC_CFG3_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH__SHIFT                                            0x10
+#define CNVC_CFG3_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_LOW_MASK                                               0x0000FFFFL
+#define CNVC_CFG3_COLOR_KEYER_GREEN__COLOR_KEYER_GREEN_HIGH_MASK                                              0xFFFF0000L
+//CNVC_CFG3_COLOR_KEYER_BLUE
+#define CNVC_CFG3_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW__SHIFT                                               0x0
+#define CNVC_CFG3_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH__SHIFT                                              0x10
+#define CNVC_CFG3_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_LOW_MASK                                                 0x0000FFFFL
+#define CNVC_CFG3_COLOR_KEYER_BLUE__COLOR_KEYER_BLUE_HIGH_MASK                                                0xFFFF0000L
+//CNVC_CFG3_ALPHA_2BIT_LUT01
+#define CNVC_CFG3_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0__SHIFT                                                    0x0
+#define CNVC_CFG3_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1__SHIFT                                                    0x10
+#define CNVC_CFG3_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT0_MASK                                                      0x00000FFFL
+#define CNVC_CFG3_ALPHA_2BIT_LUT01__ALPHA_2BIT_LUT1_MASK                                                      0x0FFF0000L
+//CNVC_CFG3_ALPHA_2BIT_LUT23
+#define CNVC_CFG3_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2__SHIFT                                                    0x0
+#define CNVC_CFG3_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3__SHIFT                                                    0x10
+#define CNVC_CFG3_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT2_MASK                                                      0x00000FFFL
+#define CNVC_CFG3_ALPHA_2BIT_LUT23__ALPHA_2BIT_LUT3_MASK                                                      0x0FFF0000L
+//CNVC_CFG3_PRE_DEALPHA
+#define CNVC_CFG3_PRE_DEALPHA__PRE_DEALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG3_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG3_PRE_DEALPHA__PRE_DEALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG3_PRE_DEALPHA__PRE_DEALPHA_ABLND_EN_MASK                                                      0x00000010L
+//CNVC_CFG3_PRE_CSC_MODE
+#define CNVC_CFG3_PRE_CSC_MODE__PRE_CSC_MODE__SHIFT                                                           0x0
+#define CNVC_CFG3_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT__SHIFT                                                   0x2
+#define CNVC_CFG3_PRE_CSC_MODE__PRE_CSC_MODE_MASK                                                             0x00000003L
+#define CNVC_CFG3_PRE_CSC_MODE__PRE_CSC_MODE_CURRENT_MASK                                                     0x0000000CL
+//CNVC_CFG3_PRE_CSC_C11_C12
+#define CNVC_CFG3_PRE_CSC_C11_C12__PRE_CSC_C11__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C11_C12__PRE_CSC_C12__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C11_C12__PRE_CSC_C11_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C11_C12__PRE_CSC_C12_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_C13_C14
+#define CNVC_CFG3_PRE_CSC_C13_C14__PRE_CSC_C13__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C13_C14__PRE_CSC_C14__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C13_C14__PRE_CSC_C13_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C13_C14__PRE_CSC_C14_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_C21_C22
+#define CNVC_CFG3_PRE_CSC_C21_C22__PRE_CSC_C21__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C21_C22__PRE_CSC_C22__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C21_C22__PRE_CSC_C21_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C21_C22__PRE_CSC_C22_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_C23_C24
+#define CNVC_CFG3_PRE_CSC_C23_C24__PRE_CSC_C23__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C23_C24__PRE_CSC_C24__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C23_C24__PRE_CSC_C23_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C23_C24__PRE_CSC_C24_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_C31_C32
+#define CNVC_CFG3_PRE_CSC_C31_C32__PRE_CSC_C31__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C31_C32__PRE_CSC_C32__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C31_C32__PRE_CSC_C31_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C31_C32__PRE_CSC_C32_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_C33_C34
+#define CNVC_CFG3_PRE_CSC_C33_C34__PRE_CSC_C33__SHIFT                                                         0x0
+#define CNVC_CFG3_PRE_CSC_C33_C34__PRE_CSC_C34__SHIFT                                                         0x10
+#define CNVC_CFG3_PRE_CSC_C33_C34__PRE_CSC_C33_MASK                                                           0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_C33_C34__PRE_CSC_C34_MASK                                                           0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C11_C12
+#define CNVC_CFG3_PRE_CSC_B_C11_C12__PRE_CSC_B_C11__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C11_C12__PRE_CSC_B_C12__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C11_C12__PRE_CSC_B_C11_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C11_C12__PRE_CSC_B_C12_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C13_C14
+#define CNVC_CFG3_PRE_CSC_B_C13_C14__PRE_CSC_B_C13__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C13_C14__PRE_CSC_B_C14__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C13_C14__PRE_CSC_B_C13_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C13_C14__PRE_CSC_B_C14_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C21_C22
+#define CNVC_CFG3_PRE_CSC_B_C21_C22__PRE_CSC_B_C21__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C21_C22__PRE_CSC_B_C22__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C21_C22__PRE_CSC_B_C21_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C21_C22__PRE_CSC_B_C22_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C23_C24
+#define CNVC_CFG3_PRE_CSC_B_C23_C24__PRE_CSC_B_C23__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C23_C24__PRE_CSC_B_C24__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C23_C24__PRE_CSC_B_C23_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C23_C24__PRE_CSC_B_C24_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C31_C32
+#define CNVC_CFG3_PRE_CSC_B_C31_C32__PRE_CSC_B_C31__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C31_C32__PRE_CSC_B_C32__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C31_C32__PRE_CSC_B_C31_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C31_C32__PRE_CSC_B_C32_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_PRE_CSC_B_C33_C34
+#define CNVC_CFG3_PRE_CSC_B_C33_C34__PRE_CSC_B_C33__SHIFT                                                     0x0
+#define CNVC_CFG3_PRE_CSC_B_C33_C34__PRE_CSC_B_C34__SHIFT                                                     0x10
+#define CNVC_CFG3_PRE_CSC_B_C33_C34__PRE_CSC_B_C33_MASK                                                       0x0000FFFFL
+#define CNVC_CFG3_PRE_CSC_B_C33_C34__PRE_CSC_B_C34_MASK                                                       0xFFFF0000L
+//CNVC_CFG3_CNVC_COEF_FORMAT
+#define CNVC_CFG3_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT__SHIFT                                                0x0
+#define CNVC_CFG3_CNVC_COEF_FORMAT__PRE_CSC_COEF_FORMAT_MASK                                                  0x00000001L
+//CNVC_CFG3_PRE_DEGAM
+#define CNVC_CFG3_PRE_DEGAM__PRE_DEGAM_MODE__SHIFT                                                            0x0
+#define CNVC_CFG3_PRE_DEGAM__PRE_DEGAM_SELECT__SHIFT                                                          0x4
+#define CNVC_CFG3_PRE_DEGAM__PRE_DEGAM_MODE_MASK                                                              0x00000003L
+#define CNVC_CFG3_PRE_DEGAM__PRE_DEGAM_SELECT_MASK                                                            0x00000070L
+//CNVC_CFG3_PRE_REALPHA
+#define CNVC_CFG3_PRE_REALPHA__PRE_REALPHA_EN__SHIFT                                                          0x0
+#define CNVC_CFG3_PRE_REALPHA__PRE_REALPHA_ABLND_EN__SHIFT                                                    0x4
+#define CNVC_CFG3_PRE_REALPHA__PRE_REALPHA_EN_MASK                                                            0x00000001L
+#define CNVC_CFG3_PRE_REALPHA__PRE_REALPHA_ABLND_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cm_cur_dispdec
+//CM_CUR3_CURSOR0_CONTROL
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_ENABLE__SHIFT                                                           0x0
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_EXPANSION_MODE__SHIFT                                                   0x1
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_PIX_INV_MODE__SHIFT                                                     0x2
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_ROM_EN__SHIFT                                                           0x3
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_MODE__SHIFT                                                             0x4
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN__SHIFT                                               0x7
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_UPDATE_PENDING__SHIFT                                                   0x10
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_ENABLE_MASK                                                             0x00000001L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_EXPANSION_MODE_MASK                                                     0x00000002L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_PIX_INV_MODE_MASK                                                       0x00000004L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_ROM_EN_MASK                                                             0x00000008L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_MODE_MASK                                                               0x00000070L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_PIXEL_ALPHA_MOD_EN_MASK                                                 0x00000080L
+#define CM_CUR3_CURSOR0_CONTROL__CUR0_UPDATE_PENDING_MASK                                                     0x00010000L
+//CM_CUR3_CURSOR0_COLOR0
+#define CM_CUR3_CURSOR0_COLOR0__CUR0_COLOR0__SHIFT                                                            0x0
+#define CM_CUR3_CURSOR0_COLOR0__CUR0_COLOR0_MASK                                                              0x00FFFFFFL
+//CM_CUR3_CURSOR0_COLOR1
+#define CM_CUR3_CURSOR0_COLOR1__CUR0_COLOR1__SHIFT                                                            0x0
+#define CM_CUR3_CURSOR0_COLOR1__CUR0_COLOR1_MASK                                                              0x00FFFFFFL
+//CM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y__SHIFT                                           0x0
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y__SHIFT                                            0x10
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_SCALE_G_Y_MASK                                             0x0000FFFFL
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_G_Y__CUR0_FP_BIAS_G_Y_MASK                                              0xFFFF0000L
+//CM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB__SHIFT                                   0x0
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB__SHIFT                                    0x10
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_SCALE_RB_CRCB_MASK                                     0x0000FFFFL
+#define CM_CUR3_CURSOR0_FP_SCALE_BIAS_RB_CRCB__CUR0_FP_BIAS_RB_CRCB_MASK                                      0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_MODE
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE__SHIFT                                                     0x0
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT__SHIFT                                             0x2
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT__SHIFT                                              0x4
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_MASK                                                       0x00000003L
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_MODE_CURRENT_MASK                                               0x0000000CL
+#define CM_CUR3_CUR0_MATRIX_MODE__CUR0_MATRIX_COEF_FORMAT_MASK                                                0x00000010L
+//CM_CUR3_CUR0_MATRIX_C11_C12_A
+#define CM_CUR3_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C11_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C11_C12_A__CUR0_MATRIX_C12_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C13_C14_A
+#define CM_CUR3_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C13_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C13_C14_A__CUR0_MATRIX_C14_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C21_C22_A
+#define CM_CUR3_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C21_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C21_C22_A__CUR0_MATRIX_C22_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C23_C24_A
+#define CM_CUR3_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C23_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C23_C24_A__CUR0_MATRIX_C24_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C31_C32_A
+#define CM_CUR3_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C31_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C31_C32_A__CUR0_MATRIX_C32_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C33_C34_A
+#define CM_CUR3_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C33_A_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C33_C34_A__CUR0_MATRIX_C34_A_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C11_C12_B
+#define CM_CUR3_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C11_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C11_C12_B__CUR0_MATRIX_C12_B_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C13_C14_B
+#define CM_CUR3_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C13_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C13_C14_B__CUR0_MATRIX_C14_B_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C21_C22_B
+#define CM_CUR3_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C21_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C21_C22_B__CUR0_MATRIX_C22_B_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C23_C24_B
+#define CM_CUR3_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C23_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C23_C24_B__CUR0_MATRIX_C24_B_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C31_C32_B
+#define CM_CUR3_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C31_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C31_C32_B__CUR0_MATRIX_C32_B_MASK                                                 0xFFFF0000L
+//CM_CUR3_CUR0_MATRIX_C33_C34_B
+#define CM_CUR3_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B__SHIFT                                               0x0
+#define CM_CUR3_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B__SHIFT                                               0x10
+#define CM_CUR3_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C33_B_MASK                                                 0x0000FFFFL
+#define CM_CUR3_CUR0_MATRIX_C33_C34_B__CUR0_MATRIX_C34_B_MASK                                                 0xFFFF0000L
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dscl_dispdec
+//DSCL3_SCL_COEF_RAM_TAP_SELECT
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX__SHIFT                                       0x0
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE__SHIFT                                              0x8
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE__SHIFT                                        0x10
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_TAP_PAIR_IDX_MASK                                         0x00000003L
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_PHASE_MASK                                                0x00003F00L
+#define DSCL3_SCL_COEF_RAM_TAP_SELECT__SCL_COEF_RAM_FILTER_TYPE_MASK                                          0x00070000L
+//DSCL3_SCL_COEF_RAM_TAP_DATA
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF__SHIFT                                        0x0
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN__SHIFT                                     0xf
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF__SHIFT                                         0x10
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN__SHIFT                                      0x1f
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_MASK                                          0x00003FFFL
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_EVEN_TAP_COEF_EN_MASK                                       0x00008000L
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_MASK                                           0x3FFF0000L
+#define DSCL3_SCL_COEF_RAM_TAP_DATA__SCL_COEF_RAM_ODD_TAP_COEF_EN_MASK                                        0x80000000L
+//DSCL3_SCL_MODE
+#define DSCL3_SCL_MODE__DSCL_MODE__SHIFT                                                                      0x0
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT__SHIFT                                                            0x8
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT__SHIFT                                                    0xc
+#define DSCL3_SCL_MODE__SCL_CHROMA_COEF_MODE__SHIFT                                                           0x10
+#define DSCL3_SCL_MODE__SCL_ALPHA_COEF_MODE__SHIFT                                                            0x14
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT_RD__SHIFT                                                         0x18
+#define DSCL3_SCL_MODE__DSCL_MODE_MASK                                                                        0x00000007L
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT_MASK                                                              0x00000100L
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT_CURRENT_MASK                                                      0x00001000L
+#define DSCL3_SCL_MODE__SCL_CHROMA_COEF_MODE_MASK                                                             0x00010000L
+#define DSCL3_SCL_MODE__SCL_ALPHA_COEF_MODE_MASK                                                              0x00100000L
+#define DSCL3_SCL_MODE__SCL_COEF_RAM_SELECT_RD_MASK                                                           0x01000000L
+//DSCL3_SCL_TAP_CONTROL
+#define DSCL3_SCL_TAP_CONTROL__SCL_V_NUM_TAPS__SHIFT                                                          0x0
+#define DSCL3_SCL_TAP_CONTROL__SCL_H_NUM_TAPS__SHIFT                                                          0x4
+#define DSCL3_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C__SHIFT                                                        0x8
+#define DSCL3_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C__SHIFT                                                        0xc
+#define DSCL3_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_MASK                                                            0x00000007L
+#define DSCL3_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_MASK                                                            0x00000070L
+#define DSCL3_SCL_TAP_CONTROL__SCL_V_NUM_TAPS_C_MASK                                                          0x00000700L
+#define DSCL3_SCL_TAP_CONTROL__SCL_H_NUM_TAPS_C_MASK                                                          0x00007000L
+//DSCL3_DSCL_CONTROL
+#define DSCL3_DSCL_CONTROL__SCL_BOUNDARY_MODE__SHIFT                                                          0x0
+#define DSCL3_DSCL_CONTROL__SCL_BOUNDARY_MODE_MASK                                                            0x00000001L
+//DSCL3_DSCL_2TAP_CONTROL
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x0
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN__SHIFT                                                   0x4
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR__SHIFT                                               0x8
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN__SHIFT                                           0x10
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN__SHIFT                                                   0x14
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR__SHIFT                                               0x18
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_HARDCODE_COEF_EN_MASK                                             0x00000001L
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_EN_MASK                                                     0x00000010L
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_H_2TAP_SHARP_FACTOR_MASK                                                 0x00000700L
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_HARDCODE_COEF_EN_MASK                                             0x00010000L
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_EN_MASK                                                     0x00100000L
+#define DSCL3_DSCL_2TAP_CONTROL__SCL_V_2TAP_SHARP_FACTOR_MASK                                                 0x07000000L
+//DSCL3_SCL_MANUAL_REPLICATE_CONTROL
+#define DSCL3_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR__SHIFT                              0x0
+#define DSCL3_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR__SHIFT                              0x8
+#define DSCL3_SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR_MASK                                0x0000000FL
+#define DSCL3_SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR_MASK                                0x00000F00L
+//DSCL3_SCL_HORZ_FILTER_SCALE_RATIO
+#define DSCL3_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL3_SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL3_SCL_HORZ_FILTER_INIT
+#define DSCL3_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL3_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT__SHIFT                                                     0x18
+#define DSCL3_SCL_HORZ_FILTER_INIT__SCL_H_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL3_SCL_HORZ_FILTER_INIT__SCL_H_INIT_INT_MASK                                                       0x0F000000L
+//DSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C
+#define DSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C__SCL_H_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL3_SCL_HORZ_FILTER_INIT_C
+#define DSCL3_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL3_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL3_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL3_SCL_HORZ_FILTER_INIT_C__SCL_H_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL3_SCL_VERT_FILTER_SCALE_RATIO
+#define DSCL3_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO__SHIFT                                           0x0
+#define DSCL3_SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO_MASK                                             0x07FFFFFFL
+//DSCL3_SCL_VERT_FILTER_INIT
+#define DSCL3_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC__SHIFT                                                    0x0
+#define DSCL3_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT__SHIFT                                                     0x18
+#define DSCL3_SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC_MASK                                                      0x00FFFFFFL
+#define DSCL3_SCL_VERT_FILTER_INIT__SCL_V_INIT_INT_MASK                                                       0x0F000000L
+//DSCL3_SCL_VERT_FILTER_INIT_BOT
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT__SHIFT                                            0x0
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT__SHIFT                                             0x18
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT_MASK                                              0x00FFFFFFL
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT_MASK                                               0x0F000000L
+//DSCL3_SCL_VERT_FILTER_SCALE_RATIO_C
+#define DSCL3_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C__SHIFT                                       0x0
+#define DSCL3_SCL_VERT_FILTER_SCALE_RATIO_C__SCL_V_SCALE_RATIO_C_MASK                                         0x07FFFFFFL
+//DSCL3_SCL_VERT_FILTER_INIT_C
+#define DSCL3_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C__SHIFT                                                0x0
+#define DSCL3_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C__SHIFT                                                 0x18
+#define DSCL3_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_FRAC_C_MASK                                                  0x00FFFFFFL
+#define DSCL3_SCL_VERT_FILTER_INIT_C__SCL_V_INIT_INT_C_MASK                                                   0x0F000000L
+//DSCL3_SCL_VERT_FILTER_INIT_BOT_C
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C__SHIFT                                        0x0
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C__SHIFT                                         0x18
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_FRAC_BOT_C_MASK                                          0x00FFFFFFL
+#define DSCL3_SCL_VERT_FILTER_INIT_BOT_C__SCL_V_INIT_INT_BOT_C_MASK                                           0x0F000000L
+//DSCL3_SCL_BLACK_COLOR
+#define DSCL3_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y__SHIFT                                                   0x0
+#define DSCL3_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR__SHIFT                                                    0x10
+#define DSCL3_SCL_BLACK_COLOR__SCL_BLACK_COLOR_RGB_Y_MASK                                                     0x0000FFFFL
+#define DSCL3_SCL_BLACK_COLOR__SCL_BLACK_COLOR_CBCR_MASK                                                      0xFFFF0000L
+//DSCL3_DSCL_UPDATE
+#define DSCL3_DSCL_UPDATE__SCL_UPDATE_PENDING__SHIFT                                                          0x0
+#define DSCL3_DSCL_UPDATE__SCL_UPDATE_PENDING_MASK                                                            0x00000001L
+//DSCL3_DSCL_AUTOCAL
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_MODE__SHIFT                                                               0x0
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE__SHIFT                                                           0x8
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_PIPE_ID__SHIFT                                                            0xc
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_MODE_MASK                                                                 0x00000003L
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_NUM_PIPE_MASK                                                             0x00000300L
+#define DSCL3_DSCL_AUTOCAL__AUTOCAL_PIPE_ID_MASK                                                              0x00003000L
+//DSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT
+#define DSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT__SHIFT                                         0x0
+#define DSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT__SHIFT                                          0x10
+#define DSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT_MASK                                           0x00003FFFL
+#define DSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT_MASK                                            0x3FFF0000L
+//DSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM
+#define DSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM__SHIFT                                        0x0
+#define DSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP__SHIFT                                           0x10
+#define DSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM_MASK                                          0x00003FFFL
+#define DSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP_MASK                                             0x3FFF0000L
+//DSCL3_OTG_H_BLANK
+#define DSCL3_OTG_H_BLANK__OTG_H_BLANK_START__SHIFT                                                           0x0
+#define DSCL3_OTG_H_BLANK__OTG_H_BLANK_END__SHIFT                                                             0x10
+#define DSCL3_OTG_H_BLANK__OTG_H_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL3_OTG_H_BLANK__OTG_H_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL3_OTG_V_BLANK
+#define DSCL3_OTG_V_BLANK__OTG_V_BLANK_START__SHIFT                                                           0x0
+#define DSCL3_OTG_V_BLANK__OTG_V_BLANK_END__SHIFT                                                             0x10
+#define DSCL3_OTG_V_BLANK__OTG_V_BLANK_START_MASK                                                             0x00003FFFL
+#define DSCL3_OTG_V_BLANK__OTG_V_BLANK_END_MASK                                                               0x3FFF0000L
+//DSCL3_RECOUT_START
+#define DSCL3_RECOUT_START__RECOUT_START_X__SHIFT                                                             0x0
+#define DSCL3_RECOUT_START__RECOUT_START_Y__SHIFT                                                             0x10
+#define DSCL3_RECOUT_START__RECOUT_START_X_MASK                                                               0x00003FFFL
+#define DSCL3_RECOUT_START__RECOUT_START_Y_MASK                                                               0x3FFF0000L
+//DSCL3_RECOUT_SIZE
+#define DSCL3_RECOUT_SIZE__RECOUT_WIDTH__SHIFT                                                                0x0
+#define DSCL3_RECOUT_SIZE__RECOUT_HEIGHT__SHIFT                                                               0x10
+#define DSCL3_RECOUT_SIZE__RECOUT_WIDTH_MASK                                                                  0x00003FFFL
+#define DSCL3_RECOUT_SIZE__RECOUT_HEIGHT_MASK                                                                 0x3FFF0000L
+//DSCL3_MPC_SIZE
+#define DSCL3_MPC_SIZE__MPC_WIDTH__SHIFT                                                                      0x0
+#define DSCL3_MPC_SIZE__MPC_HEIGHT__SHIFT                                                                     0x10
+#define DSCL3_MPC_SIZE__MPC_WIDTH_MASK                                                                        0x00003FFFL
+#define DSCL3_MPC_SIZE__MPC_HEIGHT_MASK                                                                       0x3FFF0000L
+//DSCL3_LB_DATA_FORMAT
+#define DSCL3_LB_DATA_FORMAT__INTERLEAVE_EN__SHIFT                                                            0x0
+#define DSCL3_LB_DATA_FORMAT__ALPHA_EN__SHIFT                                                                 0x4
+#define DSCL3_LB_DATA_FORMAT__INTERLEAVE_EN_MASK                                                              0x00000001L
+#define DSCL3_LB_DATA_FORMAT__ALPHA_EN_MASK                                                                   0x00000010L
+//DSCL3_LB_MEMORY_CTRL
+#define DSCL3_LB_MEMORY_CTRL__MEMORY_CONFIG__SHIFT                                                            0x0
+#define DSCL3_LB_MEMORY_CTRL__LB_MAX_PARTITIONS__SHIFT                                                        0x8
+#define DSCL3_LB_MEMORY_CTRL__LB_NUM_PARTITIONS__SHIFT                                                        0x10
+#define DSCL3_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C__SHIFT                                                      0x18
+#define DSCL3_LB_MEMORY_CTRL__MEMORY_CONFIG_MASK                                                              0x00000003L
+#define DSCL3_LB_MEMORY_CTRL__LB_MAX_PARTITIONS_MASK                                                          0x00003F00L
+#define DSCL3_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_MASK                                                          0x007F0000L
+#define DSCL3_LB_MEMORY_CTRL__LB_NUM_PARTITIONS_C_MASK                                                        0x7F000000L
+//DSCL3_LB_V_COUNTER
+#define DSCL3_LB_V_COUNTER__V_COUNTER__SHIFT                                                                  0x0
+#define DSCL3_LB_V_COUNTER__V_COUNTER_C__SHIFT                                                                0x10
+#define DSCL3_LB_V_COUNTER__V_COUNTER_MASK                                                                    0x00001FFFL
+#define DSCL3_LB_V_COUNTER__V_COUNTER_C_MASK                                                                  0x1FFF0000L
+//DSCL3_DSCL_MEM_PWR_CTRL
+#define DSCL3_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DSCL3_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS__SHIFT                                                       0x2
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE__SHIFT                                                   0x4
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS__SHIFT                                                     0x6
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE__SHIFT                                                   0x8
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS__SHIFT                                                     0xa
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE__SHIFT                                                   0xc
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS__SHIFT                                                     0xe
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE__SHIFT                                                   0x10
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS__SHIFT                                                     0x12
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE__SHIFT                                                   0x14
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS__SHIFT                                                     0x16
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE__SHIFT                                                   0x18
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS__SHIFT                                                     0x1a
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE__SHIFT                                                       0x1c
+#define DSCL3_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LUT_MEM_PWR_DIS_MASK                                                         0x00000004L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_FORCE_MASK                                                     0x00000030L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G1_MEM_PWR_DIS_MASK                                                       0x00000040L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_FORCE_MASK                                                     0x00000300L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G2_MEM_PWR_DIS_MASK                                                       0x00000400L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_FORCE_MASK                                                     0x00003000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G3_MEM_PWR_DIS_MASK                                                       0x00004000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_FORCE_MASK                                                     0x00030000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G4_MEM_PWR_DIS_MASK                                                       0x00040000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_FORCE_MASK                                                     0x00300000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G5_MEM_PWR_DIS_MASK                                                       0x00400000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_FORCE_MASK                                                     0x03000000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_G6_MEM_PWR_DIS_MASK                                                       0x04000000L
+#define DSCL3_DSCL_MEM_PWR_CTRL__LB_MEM_PWR_MODE_MASK                                                         0x10000000L
+//DSCL3_DSCL_MEM_PWR_STATUS
+#define DSCL3_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE__SHIFT                                                   0x0
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE__SHIFT                                                 0x2
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE__SHIFT                                                 0x4
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE__SHIFT                                                 0x6
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE__SHIFT                                                 0x8
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE__SHIFT                                                 0xa
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE__SHIFT                                                 0xc
+#define DSCL3_DSCL_MEM_PWR_STATUS__LUT_MEM_PWR_STATE_MASK                                                     0x00000003L
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G1_MEM_PWR_STATE_MASK                                                   0x0000000CL
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G2_MEM_PWR_STATE_MASK                                                   0x00000030L
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G3_MEM_PWR_STATE_MASK                                                   0x000000C0L
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G4_MEM_PWR_STATE_MASK                                                   0x00000300L
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G5_MEM_PWR_STATE_MASK                                                   0x00000C00L
+#define DSCL3_DSCL_MEM_PWR_STATUS__LB_G6_MEM_PWR_STATE_MASK                                                   0x00003000L
+//DSCL3_OBUF_CONTROL
+#define DSCL3_OBUF_CONTROL__OBUF_BYPASS__SHIFT                                                                0x0
+#define DSCL3_OBUF_CONTROL__OBUF_USE_FULL_BUFFER__SHIFT                                                       0x1
+#define DSCL3_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH__SHIFT                                                  0x2
+#define DSCL3_OBUF_CONTROL__OBUF_OUT_HOLD_CNT__SHIFT                                                          0x4
+#define DSCL3_OBUF_CONTROL__OBUF_BYPASS_MASK                                                                  0x00000001L
+#define DSCL3_OBUF_CONTROL__OBUF_USE_FULL_BUFFER_MASK                                                         0x00000002L
+#define DSCL3_OBUF_CONTROL__OBUF_IS_HALF_RECOUT_WIDTH_MASK                                                    0x00000004L
+#define DSCL3_OBUF_CONTROL__OBUF_OUT_HOLD_CNT_MASK                                                            0x000000F0L
+//DSCL3_OBUF_MEM_PWR_CTRL
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE__SHIFT                                                    0x0
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS__SHIFT                                                      0x2
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE__SHIFT                                                     0x8
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE__SHIFT                                                    0x10
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_FORCE_MASK                                                      0x00000003L
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_DIS_MASK                                                        0x00000004L
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_MODE_MASK                                                       0x00000100L
+#define DSCL3_OBUF_MEM_PWR_CTRL__OBUF_MEM_PWR_STATE_MASK                                                      0x00030000L
+//DSCL3_DSCL_EASF_H_MODE
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_EN__SHIFT                                                          0x0
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_EN_MASK                                                            0x00000001L
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL3_DSCL_EASF_H_MODE__SCL_EASF_H_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL3_DSCL_EASF_V_MODE
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_EN__SHIFT                                                          0x0
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN__SHIFT                                            0x4
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR__SHIFT                                           0x8
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_EN_MASK                                                            0x00000001L
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_RINGEST_FORCE_EN_MASK                                              0x00000010L
+#define DSCL3_DSCL_EASF_V_MODE__SCL_EASF_V_2TAP_SHARP_FACTOR_MASK                                             0x00003F00L
+//DSCL3_DSCL_SC_MODE
+#define DSCL3_DSCL_SC_MODE__SCL_SC_MATRIX_MODE__SHIFT                                                         0x0
+#define DSCL3_DSCL_SC_MODE__SCL_SC_LTONL_EN__SHIFT                                                            0x8
+#define DSCL3_DSCL_SC_MODE__SCL_SC_MATRIX_MODE_MASK                                                           0x00000001L
+#define DSCL3_DSCL_SC_MODE__SCL_SC_LTONL_EN_MASK                                                              0x00000100L
+//DSCL3_DSCL_SC_MATRIX_C0C1
+#define DSCL3_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0__SHIFT                                                    0x0
+#define DSCL3_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1__SHIFT                                                    0x10
+#define DSCL3_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C0_MASK                                                      0x0000FFFFL
+#define DSCL3_DSCL_SC_MATRIX_C0C1__SCL_SC_MATRIX_C1_MASK                                                      0xFFFF0000L
+//DSCL3_DSCL_SC_MATRIX_C2C3
+#define DSCL3_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2__SHIFT                                                    0x0
+#define DSCL3_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3__SHIFT                                                    0x10
+#define DSCL3_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C2_MASK                                                      0x0000FFFFL
+#define DSCL3_DSCL_SC_MATRIX_C2C3__SCL_SC_MATRIX_C3_MASK                                                      0xFFFF0000L
+//DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_GAIN__SCL_EASF_H_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL3_DSCL_EASF_H_RINGEST_EVENTAP_REDUCE__SCL_EASF_H_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1__SHIFT                       0x0
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2__SHIFT                       0x10
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN1_MASK                         0x0000FFFFL
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_GAIN__SCL_EASF_V_RINGEST_EVENTAP_GAIN2_MASK                         0xFFFF0000L
+//DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1__SHIFT                  0x0
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2__SHIFT                  0x10
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG1_MASK                    0x0000FFFFL
+#define DSCL3_DSCL_EASF_V_RINGEST_EVENTAP_REDUCE__SCL_EASF_V_RINGEST_EVENTAP_REDUCEG2_MASK                    0xFFFF0000L
+//DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT__SHIFT                    0x0
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL__SHIFT                    0x10
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_DNTILT_UPTILT_MASK                      0x0000FFFFL
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL1__SCL_EASF_V_RINGEST_3TAP_UPTILT_MAXVAL_MASK                      0xFFFF0000L
+//DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE__SHIFT                     0x0
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE__SHIFT                    0x10
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_DNTILT_SLOPE_MASK                       0x0000FFFFL
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL2__SCL_EASF_V_RINGEST_3TAP_UPTILT1_SLOPE_MASK                      0xFFFF0000L
+//DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE__SHIFT                    0x0
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET__SHIFT                   0x10
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_SLOPE_MASK                      0x0000FFFFL
+#define DSCL3_DSCL_EASF_V_RINGEST_3TAP_CNTL3__SCL_EASF_V_RINGEST_3TAP_UPTILT2_OFFSET_MASK                     0xFFFF0000L
+//DSCL3_DSCL_EASF_RINGEST_FORCE
+#define DSCL3_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE__SHIFT                                        0x0
+#define DSCL3_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE__SHIFT                                        0x10
+#define DSCL3_DSCL_EASF_RINGEST_FORCE__SCL_EASF_H_RINGEST_FORCE_MASK                                          0x0000FFFFL
+#define DSCL3_DSCL_EASF_RINGEST_FORCE__SCL_EASF_V_RINGEST_FORCE_MASK                                          0xFFFF0000L
+//DSCL3_DSCL_EASF_H_BF_CNTL
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN__SHIFT                                                   0x0
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE__SHIFT                                                 0x8
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE__SHIFT                                                 0x10
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF1_EN_MASK                                                     0x00000001L
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL3_DSCL_EASF_H_BF_CNTL__SCL_EASF_H_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA__SHIFT                                         0x0
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB__SHIFT                                         0x8
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA__SHIFT                                         0x10
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB__SHIFT                                         0x18
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINA_MASK                                           0x003F0000L
+#define DSCL3_DSCL_EASF_H_BF_FINAL_MAX_MIN__SCL_EASF_H_BF_MINB_MASK                                           0x3F000000L
+//DSCL3_DSCL_EASF_V_BF_CNTL
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN__SHIFT                                                   0x0
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE__SHIFT                                                 0x8
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE__SHIFT                                                 0x10
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN__SHIFT                                           0x14
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN__SHIFT                                           0x18
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN__SHIFT                                             0x1c
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF1_EN_MASK                                                     0x00000001L
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_MODE_MASK                                                   0x00000F00L
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF3_MODE_MASK                                                   0x00030000L
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT1_GAIN_MASK                                             0x00F00000L
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_FLAT2_GAIN_MASK                                             0x0F000000L
+#define DSCL3_DSCL_EASF_V_BF_CNTL__SCL_EASF_V_BF2_ROC_GAIN_MASK                                               0xF0000000L
+//DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA__SHIFT                                         0x0
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB__SHIFT                                         0x8
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA__SHIFT                                         0x10
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB__SHIFT                                         0x18
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXA_MASK                                           0x0000003FL
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MAXB_MASK                                           0x00003F00L
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINA_MASK                                           0x003F0000L
+#define DSCL3_DSCL_EASF_V_BF_FINAL_MAX_MIN__SCL_EASF_V_BF_MINB_MASK                                           0x3F000000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG0__SCL_EASF_H_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG1
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG1__SCL_EASF_H_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG2
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG2__SCL_EASF_H_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG3
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG3__SCL_EASF_H_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG4
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG4__SCL_EASF_H_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG5
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG5__SCL_EASF_H_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG6
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG6__SCL_EASF_H_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_H_BF1_PWL_SEG7
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_H_BF1_PWL_SEG7__SCL_EASF_H_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_IN_SEG0_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_BASE_SEG0_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG0__SCL_EASF_V_BF1_PWL_SLOPE_SEG0_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG1
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_IN_SEG1_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_BASE_SEG1_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG1__SCL_EASF_V_BF1_PWL_SLOPE_SEG1_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG2
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_IN_SEG2_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_BASE_SEG2_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG2__SCL_EASF_V_BF1_PWL_SLOPE_SEG2_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG3
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_IN_SEG3_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_BASE_SEG3_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG3__SCL_EASF_V_BF1_PWL_SLOPE_SEG3_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG4
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_IN_SEG4_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_BASE_SEG4_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG4__SCL_EASF_V_BF1_PWL_SLOPE_SEG4_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG5
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_IN_SEG5_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_BASE_SEG5_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG5__SCL_EASF_V_BF1_PWL_SLOPE_SEG5_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG6
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6__SHIFT                                  0x14
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_IN_SEG6_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_BASE_SEG6_MASK                                     0x0003F000L
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG6__SCL_EASF_V_BF1_PWL_SLOPE_SEG6_MASK                                    0x7FF00000L
+//DSCL3_DSCL_EASF_V_BF1_PWL_SEG7
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_IN_SEG7_MASK                                       0x000007FFL
+#define DSCL3_DSCL_EASF_V_BF1_PWL_SEG7__SCL_EASF_V_BF1_PWL_BASE_SEG7_MASK                                     0x0003F000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG0__SCL_EASF_H_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG1
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG1__SCL_EASF_H_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG2
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG2__SCL_EASF_H_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG3
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG3__SCL_EASF_H_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG4
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG4__SCL_EASF_H_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_H_BF3_PWL_SEG5
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_H_BF3_PWL_SEG5__SCL_EASF_H_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_IN_SEG0_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_BASE_SEG0_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG0__SCL_EASF_V_BF3_PWL_SLOPE_SEG0_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG1
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_IN_SEG1_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_BASE_SEG1_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG1__SCL_EASF_V_BF3_PWL_SLOPE_SEG1_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG2
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_IN_SEG2_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_BASE_SEG2_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG2__SCL_EASF_V_BF3_PWL_SLOPE_SEG2_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG3
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_IN_SEG3_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_BASE_SEG3_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG3__SCL_EASF_V_BF3_PWL_SLOPE_SEG3_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG4
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4__SHIFT                                  0x13
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_IN_SEG4_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_BASE_SEG4_MASK                                     0x0007F000L
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG4__SCL_EASF_V_BF3_PWL_SLOPE_SEG4_MASK                                    0xFFF80000L
+//DSCL3_DSCL_EASF_V_BF3_PWL_SEG5
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5__SHIFT                                     0x0
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5__SHIFT                                   0xc
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_IN_SEG5_MASK                                       0x00000FFFL
+#define DSCL3_DSCL_EASF_V_BF3_PWL_SEG5__SCL_EASF_V_BF3_PWL_BASE_SEG5_MASK                                     0x0007F000L
+//DSCL3_ISHARP_MODE
+#define DSCL3_ISHARP_MODE__ISHARP_EN__SHIFT                                                                   0x0
+#define DSCL3_ISHARP_MODE__ISHARP_NOISEDET_EN__SHIFT                                                          0x4
+#define DSCL3_ISHARP_MODE__ISHARP_NOISEDET_MODE__SHIFT                                                        0x5
+#define DSCL3_ISHARP_MODE__ISHARP_LBA_MODE__SHIFT                                                             0x9
+#define DSCL3_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT__SHIFT                                                     0xa
+#define DSCL3_ISHARP_MODE__ISHARP_FMT_MODE__SHIFT                                                             0xb
+#define DSCL3_ISHARP_MODE__ISHARP_FMT_NORM__SHIFT                                                             0xc
+#define DSCL3_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT__SHIFT                                             0x1c
+#define DSCL3_ISHARP_MODE__ISHARP_EN_MASK                                                                     0x00000001L
+#define DSCL3_ISHARP_MODE__ISHARP_NOISEDET_EN_MASK                                                            0x00000010L
+#define DSCL3_ISHARP_MODE__ISHARP_NOISEDET_MODE_MASK                                                          0x00000060L
+#define DSCL3_ISHARP_MODE__ISHARP_LBA_MODE_MASK                                                               0x00000200L
+#define DSCL3_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_MASK                                                       0x00000400L
+#define DSCL3_ISHARP_MODE__ISHARP_FMT_MODE_MASK                                                               0x00000800L
+#define DSCL3_ISHARP_MODE__ISHARP_FMT_NORM_MASK                                                               0x0FFFF000L
+#define DSCL3_ISHARP_MODE__ISHARP_DELTA_LUT_SELECT_CURRENT_MASK                                               0x10000000L
+//DSCL3_ISHARP_DELTA_CTRL
+#define DSCL3_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT__SHIFT                                          0x0
+#define DSCL3_ISHARP_DELTA_CTRL__ISHARP_DELTA_LUT_HOST_SELECT_MASK                                            0x00000001L
+//DSCL3_ISHARP_DELTA_INDEX
+#define DSCL3_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX__SHIFT                                                   0x0
+#define DSCL3_ISHARP_DELTA_INDEX__ISHARP_DELTA_INDEX_MASK                                                     0x0000001FL
+//DSCL3_ISHARP_DELTA_DATA
+#define DSCL3_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA__SHIFT                                                     0x0
+#define DSCL3_ISHARP_DELTA_DATA__ISHARP_DELTA_DATA_MASK                                                       0xFFFFFFFFL
+//DSCL3_ISHARP_NLDELTA_SOFT_CLIP
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P__SHIFT                                      0x0
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P__SHIFT                                   0x1
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P__SHIFT                                   0x8
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N__SHIFT                                      0x10
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N__SHIFT                                   0x11
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N__SHIFT                                   0x18
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_P_MASK                                        0x00000001L
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_P_MASK                                     0x000000FEL
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_P_MASK                                     0x0000FF00L
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_EN_N_MASK                                        0x00010000L
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_PIVOT_N_MASK                                     0x00FE0000L
+#define DSCL3_ISHARP_NLDELTA_SOFT_CLIP__ISHARP_NLDELTA_SCLIP_SLOPE_N_MASK                                     0xFF000000L
+//DSCL3_ISHARP_NOISEDET_THRESHOLD
+#define DSCL3_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE__SHIFT                                         0x0
+#define DSCL3_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE__SHIFT                                         0x10
+#define DSCL3_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_UTHRE_MASK                                           0x000003FFL
+#define DSCL3_ISHARP_NOISEDET_THRESHOLD__ISHARP_NOISEDET_DTHRE_MASK                                           0x03FF0000L
+//DSCL3_ISHARP_NOISE_GAIN_PWL
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN__SHIFT                                      0x0
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN__SHIFT                                        0x8
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE__SHIFT                                         0x10
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_START_IN_MASK                                        0x0000001FL
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_END_IN_MASK                                          0x00001F00L
+#define DSCL3_ISHARP_NOISE_GAIN_PWL__ISHARP_NOISEDET_PWL_SLOPE_MASK                                           0x3FFF0000L
+//DSCL3_ISHARP_LBA_PWL_SEG0
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0__SHIFT                                           0x14
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_IN_SEG0_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_BASE_SEG0_MASK                                              0x0003F000L
+#define DSCL3_ISHARP_LBA_PWL_SEG0__ISHARP_LBA_PWL_SLOPE_SEG0_MASK                                             0x1FF00000L
+//DSCL3_ISHARP_LBA_PWL_SEG1
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1__SHIFT                                           0x14
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_IN_SEG1_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_BASE_SEG1_MASK                                              0x0003F000L
+#define DSCL3_ISHARP_LBA_PWL_SEG1__ISHARP_LBA_PWL_SLOPE_SEG1_MASK                                             0x1FF00000L
+//DSCL3_ISHARP_LBA_PWL_SEG2
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2__SHIFT                                           0x14
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_IN_SEG2_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_BASE_SEG2_MASK                                              0x0003F000L
+#define DSCL3_ISHARP_LBA_PWL_SEG2__ISHARP_LBA_PWL_SLOPE_SEG2_MASK                                             0x1FF00000L
+//DSCL3_ISHARP_LBA_PWL_SEG3
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3__SHIFT                                           0x14
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_IN_SEG3_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_BASE_SEG3_MASK                                              0x0003F000L
+#define DSCL3_ISHARP_LBA_PWL_SEG3__ISHARP_LBA_PWL_SLOPE_SEG3_MASK                                             0x1FF00000L
+//DSCL3_ISHARP_LBA_PWL_SEG4
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4__SHIFT                                           0x14
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_IN_SEG4_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_BASE_SEG4_MASK                                              0x0003F000L
+#define DSCL3_ISHARP_LBA_PWL_SEG4__ISHARP_LBA_PWL_SLOPE_SEG4_MASK                                             0x1FF00000L
+//DSCL3_ISHARP_LBA_PWL_SEG5
+#define DSCL3_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5__SHIFT                                              0x0
+#define DSCL3_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5__SHIFT                                            0xc
+#define DSCL3_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_IN_SEG5_MASK                                                0x000003FFL
+#define DSCL3_ISHARP_LBA_PWL_SEG5__ISHARP_LBA_PWL_BASE_SEG5_MASK                                              0x0003F000L
+//DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE__SHIFT                            0x0
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS__SHIFT                              0x2
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE__SHIFT                            0x4
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_FORCE_MASK                              0x00000003L
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_DIS_MASK                                0x00000004L
+#define DSCL3_ISHARP_DELTA_LUT_MEM_PWR_CTRL__ISHARP_DELTA_LUT_MEM_PWR_STATE_MASK                              0x00000030L
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cm_dispdec
+//CM3_CM_CONTROL
+#define CM3_CM_CONTROL__CM_BYPASS__SHIFT                                                                      0x0
+#define CM3_CM_CONTROL__CM_UPDATE_PENDING__SHIFT                                                              0x8
+#define CM3_CM_CONTROL__CM_BYPASS_MASK                                                                        0x00000001L
+#define CM3_CM_CONTROL__CM_UPDATE_PENDING_MASK                                                                0x00000100L
+//CM3_CM_POST_CSC_CONTROL
+#define CM3_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE__SHIFT                                                      0x0
+#define CM3_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT__SHIFT                                              0x2
+#define CM3_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_MASK                                                        0x00000003L
+#define CM3_CM_POST_CSC_CONTROL__CM_POST_CSC_MODE_CURRENT_MASK                                                0x0000000CL
+//CM3_CM_POST_CSC_C11_C12
+#define CM3_CM_POST_CSC_C11_C12__CM_POST_CSC_C11__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C11_C12__CM_POST_CSC_C12__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C11_C12__CM_POST_CSC_C11_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C11_C12__CM_POST_CSC_C12_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_C13_C14
+#define CM3_CM_POST_CSC_C13_C14__CM_POST_CSC_C13__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C13_C14__CM_POST_CSC_C14__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C13_C14__CM_POST_CSC_C13_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C13_C14__CM_POST_CSC_C14_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_C21_C22
+#define CM3_CM_POST_CSC_C21_C22__CM_POST_CSC_C21__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C21_C22__CM_POST_CSC_C22__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C21_C22__CM_POST_CSC_C21_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C21_C22__CM_POST_CSC_C22_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_C23_C24
+#define CM3_CM_POST_CSC_C23_C24__CM_POST_CSC_C23__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C23_C24__CM_POST_CSC_C24__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C23_C24__CM_POST_CSC_C23_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C23_C24__CM_POST_CSC_C24_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_C31_C32
+#define CM3_CM_POST_CSC_C31_C32__CM_POST_CSC_C31__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C31_C32__CM_POST_CSC_C32__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C31_C32__CM_POST_CSC_C31_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C31_C32__CM_POST_CSC_C32_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_C33_C34
+#define CM3_CM_POST_CSC_C33_C34__CM_POST_CSC_C33__SHIFT                                                       0x0
+#define CM3_CM_POST_CSC_C33_C34__CM_POST_CSC_C34__SHIFT                                                       0x10
+#define CM3_CM_POST_CSC_C33_C34__CM_POST_CSC_C33_MASK                                                         0x0000FFFFL
+#define CM3_CM_POST_CSC_C33_C34__CM_POST_CSC_C34_MASK                                                         0xFFFF0000L
+//CM3_CM_POST_CSC_B_C11_C12
+#define CM3_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C11_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C11_C12__CM_POST_CSC_B_C12_MASK                                                     0xFFFF0000L
+//CM3_CM_POST_CSC_B_C13_C14
+#define CM3_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C13_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C13_C14__CM_POST_CSC_B_C14_MASK                                                     0xFFFF0000L
+//CM3_CM_POST_CSC_B_C21_C22
+#define CM3_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C21_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C21_C22__CM_POST_CSC_B_C22_MASK                                                     0xFFFF0000L
+//CM3_CM_POST_CSC_B_C23_C24
+#define CM3_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C23_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C23_C24__CM_POST_CSC_B_C24_MASK                                                     0xFFFF0000L
+//CM3_CM_POST_CSC_B_C31_C32
+#define CM3_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C31_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C31_C32__CM_POST_CSC_B_C32_MASK                                                     0xFFFF0000L
+//CM3_CM_POST_CSC_B_C33_C34
+#define CM3_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33__SHIFT                                                   0x0
+#define CM3_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34__SHIFT                                                   0x10
+#define CM3_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C33_MASK                                                     0x0000FFFFL
+#define CM3_CM_POST_CSC_B_C33_C34__CM_POST_CSC_B_C34_MASK                                                     0xFFFF0000L
+//CM3_CM_BIAS_CR_R
+#define CM3_CM_BIAS_CR_R__CM_BIAS_CR_R__SHIFT                                                                 0x0
+#define CM3_CM_BIAS_CR_R__CM_BIAS_CR_R_MASK                                                                   0x0000FFFFL
+//CM3_CM_BIAS_Y_G_CB_B
+#define CM3_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G__SHIFT                                                              0x0
+#define CM3_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B__SHIFT                                                             0x10
+#define CM3_CM_BIAS_Y_G_CB_B__CM_BIAS_Y_G_MASK                                                                0x0000FFFFL
+#define CM3_CM_BIAS_Y_G_CB_B__CM_BIAS_CB_B_MASK                                                               0xFFFF0000L
+//CM3_CM_GAMCOR_CONTROL
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE__SHIFT                                                          0x0
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT__SHIFT                                                        0x2
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE__SHIFT                                                   0x3
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT__SHIFT                                                  0x4
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT__SHIFT                                                0x6
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_MASK                                                            0x00000003L
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_MASK                                                          0x00000004L
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_PWL_DISABLE_MASK                                                     0x00000008L
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_MODE_CURRENT_MASK                                                    0x00000030L
+#define CM3_CM_GAMCOR_CONTROL__CM_GAMCOR_SELECT_CURRENT_MASK                                                  0x00000040L
+//CM3_CM_GAMCOR_LUT_INDEX
+#define CM3_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX__SHIFT                                                   0x0
+#define CM3_CM_GAMCOR_LUT_INDEX__CM_GAMCOR_LUT_INDEX_MASK                                                     0x000001FFL
+//CM3_CM_GAMCOR_LUT_DATA
+#define CM3_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA__SHIFT                                                     0x0
+#define CM3_CM_GAMCOR_LUT_DATA__CM_GAMCOR_LUT_DATA_MASK                                                       0x0003FFFFL
+//CM3_CM_GAMCOR_LUT_CONTROL
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK__SHIFT                                      0x0
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL__SHIFT                                        0x3
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG__SHIFT                                              0x5
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL__SHIFT                                              0x6
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE__SHIFT                                           0x7
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_WRITE_COLOR_MASK_MASK                                        0x00000007L
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_COLOR_SEL_MASK                                          0x00000018L
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_READ_DBG_MASK                                                0x00000020L
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_HOST_SEL_MASK                                                0x00000040L
+#define CM3_CM_GAMCOR_LUT_CONTROL__CM_GAMCOR_LUT_CONFIG_MODE_MASK                                             0x00000080L
+//CM3_CM_GAMCOR_RAMA_START_CNTL_B
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMA_START_CNTL_G
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMA_START_CNTL_R
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMA_START_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMA_START_SLOPE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_B
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_B__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_G
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_G__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_R
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMA_START_BASE_CNTL_R__CM_GAMCOR_RAMA_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_END_CNTL1_B
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_B__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_END_CNTL2_B
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_B__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMA_END_CNTL1_G
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_G__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_END_CNTL2_G
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_G__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMA_END_CNTL1_R
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL1_R__CM_GAMCOR_RAMA_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMA_END_CNTL2_R
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMA_END_CNTL2_R__CM_GAMCOR_RAMA_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMA_OFFSET_B
+#define CM3_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMA_OFFSET_B__CM_GAMCOR_RAMA_OFFSET_B_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMA_OFFSET_G
+#define CM3_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMA_OFFSET_G__CM_GAMCOR_RAMA_OFFSET_G_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMA_OFFSET_R
+#define CM3_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMA_OFFSET_R__CM_GAMCOR_RAMA_OFFSET_R_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMA_REGION_0_1
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_0_1__CM_GAMCOR_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_2_3
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_2_3__CM_GAMCOR_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_4_5
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_4_5__CM_GAMCOR_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_6_7
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_6_7__CM_GAMCOR_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_8_9
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_8_9__CM_GAMCOR_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_10_11
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_10_11__CM_GAMCOR_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_12_13
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_12_13__CM_GAMCOR_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_14_15
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_14_15__CM_GAMCOR_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_16_17
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_16_17__CM_GAMCOR_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_18_19
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_18_19__CM_GAMCOR_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_20_21
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_20_21__CM_GAMCOR_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_22_23
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_22_23__CM_GAMCOR_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_24_25
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_24_25__CM_GAMCOR_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_26_27
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_26_27__CM_GAMCOR_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_28_29
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_28_29__CM_GAMCOR_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_30_31
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_30_31__CM_GAMCOR_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMA_REGION_32_33
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMA_REGION_32_33__CM_GAMCOR_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_START_CNTL_B
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_B_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_B_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMB_START_CNTL_G
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_G_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_G_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMB_START_CNTL_R
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R__SHIFT                             0x0
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT                     0x14
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_R_MASK                               0x0003FFFFL
+#define CM3_CM_GAMCOR_RAMB_START_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SEGMENT_R_MASK                       0x07F00000L
+//CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_B_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_G_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R__SHIFT                 0x0
+#define CM3_CM_GAMCOR_RAMB_START_SLOPE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_SLOPE_R_MASK                   0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_B
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_B__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_B_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_G
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_G__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_G_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_R
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R__SHIFT                   0x0
+#define CM3_CM_GAMCOR_RAMB_START_BASE_CNTL_R__CM_GAMCOR_RAMB_EXP_REGION_START_BASE_R_MASK                     0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_END_CNTL1_B
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_B__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_B_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_END_CNTL2_B
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_B_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_B__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_B_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMB_END_CNTL1_G
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_G__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_G_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_END_CNTL2_G
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_G_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_G__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_G_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMB_END_CNTL1_R
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL1_R__CM_GAMCOR_RAMB_EXP_REGION_END_BASE_R_MASK                             0x0003FFFFL
+//CM3_CM_GAMCOR_RAMB_END_CNTL2_R
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R__SHIFT                                0x0
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                          0x10
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_R_MASK                                  0x0000FFFFL
+#define CM3_CM_GAMCOR_RAMB_END_CNTL2_R__CM_GAMCOR_RAMB_EXP_REGION_END_SLOPE_R_MASK                            0xFFFF0000L
+//CM3_CM_GAMCOR_RAMB_OFFSET_B
+#define CM3_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMB_OFFSET_B__CM_GAMCOR_RAMB_OFFSET_B_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMB_OFFSET_G
+#define CM3_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMB_OFFSET_G__CM_GAMCOR_RAMB_OFFSET_G_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMB_OFFSET_R
+#define CM3_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R__SHIFT                                           0x0
+#define CM3_CM_GAMCOR_RAMB_OFFSET_R__CM_GAMCOR_RAMB_OFFSET_R_MASK                                             0x0007FFFFL
+//CM3_CM_GAMCOR_RAMB_REGION_0_1
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_0_1__CM_GAMCOR_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_2_3
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_2_3__CM_GAMCOR_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_4_5
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_4_5__CM_GAMCOR_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_6_7
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_6_7__CM_GAMCOR_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_8_9
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                           0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                         0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                           0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                         0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_LUT_OFFSET_MASK                             0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                           0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_LUT_OFFSET_MASK                             0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_8_9__CM_GAMCOR_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                           0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_10_11
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_10_11__CM_GAMCOR_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_12_13
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_12_13__CM_GAMCOR_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_14_15
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_14_15__CM_GAMCOR_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_16_17
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_16_17__CM_GAMCOR_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_18_19
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_18_19__CM_GAMCOR_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_20_21
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_20_21__CM_GAMCOR_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_22_23
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_22_23__CM_GAMCOR_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_24_25
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_24_25__CM_GAMCOR_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_26_27
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_26_27__CM_GAMCOR_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_28_29
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_28_29__CM_GAMCOR_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_30_31
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_30_31__CM_GAMCOR_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_GAMCOR_RAMB_REGION_32_33
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                        0x0
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT                      0xc
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                        0x10
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT                      0x1c
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_LUT_OFFSET_MASK                          0x000001FFL
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                        0x00007000L
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_LUT_OFFSET_MASK                          0x01FF0000L
+#define CM3_CM_GAMCOR_RAMB_REGION_32_33__CM_GAMCOR_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                        0x70000000L
+//CM3_CM_HDR_MULT_COEF
+#define CM3_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF__SHIFT                                                         0x0
+#define CM3_CM_HDR_MULT_COEF__CM_HDR_MULT_COEF_MASK                                                           0x0007FFFFL
+//CM3_CM_MEM_PWR_CTRL
+#define CM3_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE__SHIFT                                                      0x0
+#define CM3_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS__SHIFT                                                        0x2
+#define CM3_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE__SHIFT                                                        0x8
+#define CM3_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS__SHIFT                                                          0x9
+#define CM3_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_FORCE_MASK                                                        0x00000003L
+#define CM3_CM_MEM_PWR_CTRL__GAMCOR_MEM_PWR_DIS_MASK                                                          0x00000004L
+#define CM3_CM_MEM_PWR_CTRL__HIST_MEM_PWR_FORCE_MASK                                                          0x00000100L
+#define CM3_CM_MEM_PWR_CTRL__HIST_MEM_PWR_DIS_MASK                                                            0x00000200L
+//CM3_CM_MEM_PWR_STATUS
+#define CM3_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE__SHIFT                                                    0x0
+#define CM3_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE__SHIFT                                                      0x8
+#define CM3_CM_MEM_PWR_STATUS__GAMCOR_MEM_PWR_STATE_MASK                                                      0x00000003L
+#define CM3_CM_MEM_PWR_STATUS__HIST_MEM_PWR_STATE_MASK                                                        0x00000100L
+//CM3_CM_DEALPHA
+#define CM3_CM_DEALPHA__CM_DEALPHA_EN__SHIFT                                                                  0x0
+#define CM3_CM_DEALPHA__CM_DEALPHA_ABLND__SHIFT                                                               0x1
+#define CM3_CM_DEALPHA__CM_DEALPHA_EN_MASK                                                                    0x00000001L
+#define CM3_CM_DEALPHA__CM_DEALPHA_ABLND_MASK                                                                 0x00000002L
+//CM3_CM_COEF_FORMAT
+#define CM3_CM_COEF_FORMAT__CM_BIAS_FORMAT__SHIFT                                                             0x0
+#define CM3_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT__SHIFT                                                    0x4
+#define CM3_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
+#define CM3_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
+//CM3_CM_TEST_DEBUG_INDEX
+#define CM3_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM3_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM3_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM3_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM3_CM_TEST_DEBUG_DATA
+#define CM3_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM3_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+//CM3_CM_HIST_CNTL
+#define CM3_CM_HIST_CNTL__CM_HIST_SEL__SHIFT                                                                  0x0
+#define CM3_CM_HIST_CNTL__CM_HIST_CH_EN__SHIFT                                                                0x2
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC1_SEL__SHIFT                                                             0x4
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC2_SEL__SHIFT                                                             0x5
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC3_SEL__SHIFT                                                             0x6
+#define CM3_CM_HIST_CNTL__CM_HIST_CH1_XBAR__SHIFT                                                             0x7
+#define CM3_CM_HIST_CNTL__CM_HIST_CH2_XBAR__SHIFT                                                             0x9
+#define CM3_CM_HIST_CNTL__CM_HIST_CH3_XBAR__SHIFT                                                             0xb
+#define CM3_CM_HIST_CNTL__CM_HIST_FORMAT__SHIFT                                                               0xd
+#define CM3_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK__SHIFT                                                    0xf
+#define CM3_CM_HIST_CNTL__CM_HIST_SEL_MASK                                                                    0x00000003L
+#define CM3_CM_HIST_CNTL__CM_HIST_CH_EN_MASK                                                                  0x0000000CL
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC1_SEL_MASK                                                               0x00000010L
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC2_SEL_MASK                                                               0x00000020L
+#define CM3_CM_HIST_CNTL__CM_HIST_SRC3_SEL_MASK                                                               0x00000040L
+#define CM3_CM_HIST_CNTL__CM_HIST_CH1_XBAR_MASK                                                               0x00000180L
+#define CM3_CM_HIST_CNTL__CM_HIST_CH2_XBAR_MASK                                                               0x00000600L
+#define CM3_CM_HIST_CNTL__CM_HIST_CH3_XBAR_MASK                                                               0x00001800L
+#define CM3_CM_HIST_CNTL__CM_HIST_FORMAT_MASK                                                                 0x00006000L
+#define CM3_CM_HIST_CNTL__CM_HIST_READ_CHANNEL_MASK_MASK                                                      0x00038000L
+//CM3_CM_HIST_SCALE_SRC1
+#define CM3_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1__SHIFT                                                     0x0
+#define CM3_CM_HIST_SCALE_SRC1__CM_HIST_SCALE_SRC1_MASK                                                       0x0007FFFFL
+//CM3_CM_HIST_COEFA_SRC2
+#define CM3_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2__SHIFT                                                     0x0
+#define CM3_CM_HIST_COEFA_SRC2__CM_HIST_COEFA_SRC2_MASK                                                       0x0007FFFFL
+//CM3_CM_HIST_COEFB_SRC2
+#define CM3_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2__SHIFT                                                     0x0
+#define CM3_CM_HIST_COEFB_SRC2__CM_HIST_COEFB_SRC2_MASK                                                       0x0007FFFFL
+//CM3_CM_HIST_COEFC_SRC2
+#define CM3_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2__SHIFT                                                     0x0
+#define CM3_CM_HIST_COEFC_SRC2__CM_HIST_COEFC_SRC2_MASK                                                       0x0007FFFFL
+//CM3_CM_HIST_SCALE_SRC3
+#define CM3_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3__SHIFT                                                     0x0
+#define CM3_CM_HIST_SCALE_SRC3__CM_HIST_SCALE_SRC3_MASK                                                       0x0007FFFFL
+//CM3_CM_HIST_BIAS_SRC1
+#define CM3_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1__SHIFT                                                       0x0
+#define CM3_CM_HIST_BIAS_SRC1__CM_HIST_BIAS_SRC1_MASK                                                         0x0007FFFFL
+//CM3_CM_HIST_BIAS_SRC2
+#define CM3_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2__SHIFT                                                       0x0
+#define CM3_CM_HIST_BIAS_SRC2__CM_HIST_BIAS_SRC2_MASK                                                         0x0007FFFFL
+//CM3_CM_HIST_BIAS_SRC3
+#define CM3_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3__SHIFT                                                       0x0
+#define CM3_CM_HIST_BIAS_SRC3__CM_HIST_BIAS_SRC3_MASK                                                         0x0007FFFFL
+//CM3_CM_HIST_LOCK
+#define CM3_CM_HIST_LOCK__CM_HIST_LOCK__SHIFT                                                                 0x0
+#define CM3_CM_HIST_LOCK__CM_HIST_LOCK_MASK                                                                   0x00000001L
+//CM3_CM_HIST_INDEX
+#define CM3_CM_HIST_INDEX__CM_HIST_INDEX__SHIFT                                                               0x0
+#define CM3_CM_HIST_INDEX__CM_HIST_INDEX_MASK                                                                 0x000000FFL
+//CM3_CM_HIST_DATA
+#define CM3_CM_HIST_DATA__CM_HIST_DATA__SHIFT                                                                 0x0
+#define CM3_CM_HIST_DATA__CM_HIST_DATA_MASK                                                                   0x01FFFFFFL
+//CM3_CM_HIST_STATUS
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS__SHIFT                                                    0x0
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS__SHIFT                                                    0x1
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT__SHIFT                                                 0x2
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ__SHIFT                                                    0x3
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED__SHIFT                                              0x4
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT__SHIFT                                      0x5
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT__SHIFT                                          0x6
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED__SHIFT                                            0xc
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0xd
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED__SHIFT                                            0x10
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT__SHIFT                                    0x11
+#define CM3_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW__SHIFT                                                     0x18
+#define CM3_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT__SHIFT                                             0x19
+#define CM3_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE__SHIFT                                                 0x1a
+#define CM3_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT__SHIFT                                         0x1b
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_RDY_STATUS_MASK                                                      0x00000001L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_RDY_STATUS_MASK                                                      0x00000002L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_COLLECT_MASK                                                   0x00000004L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFF_INUSE_READ_MASK                                                      0x00000008L
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_MASK                                                0x00000010L
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CURRENT_MASK                                        0x00000020L
+#define CM3_CM_HIST_STATUS__CM_HIST_FRAME_COLLECT_SKIPPED_CNT_MASK                                            0x000001C0L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_MASK                                              0x00001000L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFA_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00002000L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_MASK                                              0x00010000L
+#define CM3_CM_HIST_STATUS__CM_HIST_BUFB_FRAME_READ_SKIPPED_CURRENT_MASK                                      0x00020000L
+#define CM3_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_MASK                                                       0x01000000L
+#define CM3_CM_HIST_STATUS__CM_HIST_COUNT_OVERFLOW_CURRENT_MASK                                               0x02000000L
+#define CM3_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_MASK                                                   0x04000000L
+#define CM3_CM_HIST_STATUS__CM_HIST_COLLECT_INCOMPLETE_CURRENT_MASK                                           0x08000000L
+//CM3_CM_HIST_INT_CONTROL
+#define CM3_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT__SHIFT                                                       0x0
+#define CM3_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN__SHIFT                                                    0x4
+#define CM3_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_MASK                                                         0x00000001L
+#define CM3_CM_HIST_INT_CONTROL__CM_HIST_RDY_INT_EN_MASK                                                      0x00000010L
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON13_PERFCOUNTER_CNTL
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON13_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON13_PERFCOUNTER_CNTL2
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON13_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON13_PERFCOUNTER_STATE
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON13_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON13_PERFMON_CNTL
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON13_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON13_PERFMON_CNTL2
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON13_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON13_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON13_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON13_PERFMON_CVALUE_LOW
+#define DC_PERFMON13_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON13_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON13_PERFMON_HI
+#define DC_PERFMON13_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON13_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON13_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON13_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON13_PERFMON_LOW
+#define DC_PERFMON13_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON13_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dcoh_dp_aux0_dispdec
+//DP_AUX0_AUX_CONTROL
+#define DP_AUX0_AUX_CONTROL__AUX_EN__SHIFT                                                                    0x0
+#define DP_AUX0_AUX_CONTROL__AUX_RESET__SHIFT                                                                 0x4
+#define DP_AUX0_AUX_CONTROL__AUX_RESET_DONE__SHIFT                                                            0x5
+#define DP_AUX0_AUX_CONTROL__AUX_LS_READ_EN__SHIFT                                                            0x8
+#define DP_AUX0_AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT                                                     0xc
+#define DP_AUX0_AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT                                                     0x10
+#define DP_AUX0_AUX_CONTROL__AUX_MODE_DET_EN__SHIFT                                                           0x12
+#define DP_AUX0_AUX_CONTROL__AUX_HPD_SEL__SHIFT                                                               0x14
+#define DP_AUX0_AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT                                                         0x18
+#define DP_AUX0_AUX_CONTROL__AUX_TEST_MODE__SHIFT                                                             0x1c
+#define DP_AUX0_AUX_CONTROL__AUX_EN_MASK                                                                      0x00000001L
+#define DP_AUX0_AUX_CONTROL__AUX_RESET_MASK                                                                   0x00000010L
+#define DP_AUX0_AUX_CONTROL__AUX_RESET_DONE_MASK                                                              0x00000020L
+#define DP_AUX0_AUX_CONTROL__AUX_LS_READ_EN_MASK                                                              0x00000100L
+#define DP_AUX0_AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK                                                       0x00001000L
+#define DP_AUX0_AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK                                                       0x00010000L
+#define DP_AUX0_AUX_CONTROL__AUX_MODE_DET_EN_MASK                                                             0x00040000L
+#define DP_AUX0_AUX_CONTROL__AUX_HPD_SEL_MASK                                                                 0x00700000L
+#define DP_AUX0_AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK                                                           0x01000000L
+#define DP_AUX0_AUX_CONTROL__AUX_TEST_MODE_MASK                                                               0x10000000L
+//DP_AUX0_AUX_SW_CONTROL
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_GO__SHIFT                                                              0x0
+#define DP_AUX0_AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT                                                       0x2
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT                                                     0x4
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT                                                        0x10
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_GO_MASK                                                                0x00000001L
+#define DP_AUX0_AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK                                                         0x00000004L
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK                                                       0x000000F0L
+#define DP_AUX0_AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK                                                          0x001F0000L
+//DP_AUX0_AUX_ARB_CONTROL
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT                                                      0x0
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT                                                0x2
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT                                                   0x8
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT                                                   0xa
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT                                                0x10
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT                                        0x10
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT                                             0x11
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT                                              0x18
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT                                      0x18
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT                                           0x19
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK                                                        0x00000001L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK                                                  0x0000000CL
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK                                                     0x00000100L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK                                                     0x00000400L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK                                                  0x00010000L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK                                          0x00010000L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK                                               0x00020000L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK                                                0x01000000L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK                                        0x01000000L
+#define DP_AUX0_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK                                             0x02000000L
+//DP_AUX0_AUX_INTERRUPT_CONTROL
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT                                                 0x0
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT                                                 0x1
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT                                                0x2
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT                                                 0x4
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT                                                 0x5
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT                                                0x6
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK                                                   0x00000001L
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK                                                   0x00000002L
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK                                                  0x00000004L
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK                                                   0x00000010L
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK                                                   0x00000020L
+#define DP_AUX0_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK                                                  0x00000040L
+//DP_AUX0_AUX_SW_STATUS
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_DONE__SHIFT                                                             0x0
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_REQ__SHIFT                                                              0x1
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX0_AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT                                                          0x1d
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_DONE_MASK                                                               0x00000001L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_REQ_MASK                                                                0x00000002L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX0_AUX_SW_STATUS__AUX_ARB_STATUS_MASK                                                            0x60000000L
+//DP_AUX0_AUX_LS_STATUS
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_DONE__SHIFT                                                             0x0
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_REQ__SHIFT                                                              0x1
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT                                                           0x1d
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT                                                          0x1e
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT                                                      0x1f
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_DONE_MASK                                                               0x00000001L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_REQ_MASK                                                                0x00000002L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK                                                             0x20000000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_UPDATED_MASK                                                            0x40000000L
+#define DP_AUX0_AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK                                                        0x80000000L
+//DP_AUX0_AUX_SW_DATA
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT                                                            0x0
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_DATA__SHIFT                                                               0x8
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_INDEX__SHIFT                                                              0x10
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT                                              0x1f
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_DATA_RW_MASK                                                              0x00000001L
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_INDEX_MASK                                                                0x001F0000L
+#define DP_AUX0_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK                                                0x80000000L
+//DP_AUX0_AUX_LS_DATA
+#define DP_AUX0_AUX_LS_DATA__AUX_LS_DATA__SHIFT                                                               0x8
+#define DP_AUX0_AUX_LS_DATA__AUX_LS_INDEX__SHIFT                                                              0x10
+#define DP_AUX0_AUX_LS_DATA__AUX_LS_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX0_AUX_LS_DATA__AUX_LS_INDEX_MASK                                                                0x001F0000L
+//DP_AUX0_AUX_DPHY_TX_REF_CONTROL
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT                                                0x0
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT                                                   0x4
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT                                                0x10
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK                                                  0x00000001L
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK                                                     0x00000030L
+#define DP_AUX0_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK                                                  0x01FF0000L
+//DP_AUX0_AUX_DPHY_TX_CONTROL
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT                                              0x0
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL__SHIFT                                          0x4
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME__SHIFT                                             0x6
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT                                          0x8
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY__SHIFT                                          0x10
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK                                                0x0000000FL
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL_MASK                                            0x00000030L
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME_MASK                                               0x00000040L
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK                                            0x00003F00L
+#define DP_AUX0_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY_MASK                                            0x00070000L
+//DP_AUX0_AUX_DPHY_RX_CONTROL0
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT                                              0x4
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT                                            0x8
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT                                       0xc
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT                                      0x10
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT                        0x11
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT                               0x12
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT                                0x13
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT                                          0x14
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT                                       0x1c
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK                                                0x00000070L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK                                              0x00000700L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK                                         0x00003000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK                                        0x00010000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK                          0x00020000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK                                 0x00040000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK                                  0x00080000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK                                            0x00300000L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK                                         0x70000000L
+//DP_AUX0_AUX_DPHY_RX_CONTROL1
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT                                            0x0
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN__SHIFT                                               0x8
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL__SHIFT                                           0xf
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK                                              0x000000FFL
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MASK                                                 0x00007F00L
+#define DP_AUX0_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL_MASK                                             0x00018000L
+//DP_AUX0_AUX_DPHY_TX_STATUS
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT                                                      0x0
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT                                                       0x4
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT                                             0x10
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK                                                        0x00000001L
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK                                                         0x00000070L
+#define DP_AUX0_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK                                               0x01FF0000L
+//DP_AUX0_AUX_DPHY_RX_STATUS
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT                                                       0x0
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT                                            0x8
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT                                       0x10
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT                                             0x15
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK                                                         0x00000007L
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK                                              0x00001F00L
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK                                         0x001F0000L
+#define DP_AUX0_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK                                               0x3FE00000L
+//DP_AUX0_AUX_PHY_WAKE_CNTL
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO__SHIFT                                                  0x0
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING__SHIFT                                             0x1
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY__SHIFT                                            0x2
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK__SHIFT                                                 0x3
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO_MASK                                                    0x00000001L
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING_MASK                                               0x00000002L
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY_MASK                                              0x00000004L
+#define DP_AUX0_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK_MASK                                                   0x00000008L
+//DP_AUX0_AUX_PHY_WAKE_STATUS
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE__SHIFT                                    0x4
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT__SHIFT                                          0x7
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON__SHIFT                                           0x9
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE__SHIFT                                         0xb
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL__SHIFT                                   0xc
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP__SHIFT                                     0xe
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L__SHIFT                                   0x11
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H__SHIFT                                   0x12
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET__SHIFT                                      0x14
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H__SHIFT                                   0x16
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L__SHIFT                                   0x17
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE_MASK                                      0x00000070L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_MASK                                            0x00000080L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON_MASK                                             0x00000200L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE_MASK                                           0x00000800L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL_MASK                                     0x00001000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP_MASK                                       0x00004000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L_MASK                                     0x00020000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H_MASK                                     0x00040000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET_MASK                                        0x00100000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H_MASK                                     0x00400000L
+#define DP_AUX0_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L_MASK                                     0x00800000L
+
+
+// addressBlock: dce_dc_dcoh_dp_aux1_dispdec
+//DP_AUX1_AUX_CONTROL
+#define DP_AUX1_AUX_CONTROL__AUX_EN__SHIFT                                                                    0x0
+#define DP_AUX1_AUX_CONTROL__AUX_RESET__SHIFT                                                                 0x4
+#define DP_AUX1_AUX_CONTROL__AUX_RESET_DONE__SHIFT                                                            0x5
+#define DP_AUX1_AUX_CONTROL__AUX_LS_READ_EN__SHIFT                                                            0x8
+#define DP_AUX1_AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT                                                     0xc
+#define DP_AUX1_AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT                                                     0x10
+#define DP_AUX1_AUX_CONTROL__AUX_MODE_DET_EN__SHIFT                                                           0x12
+#define DP_AUX1_AUX_CONTROL__AUX_HPD_SEL__SHIFT                                                               0x14
+#define DP_AUX1_AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT                                                         0x18
+#define DP_AUX1_AUX_CONTROL__AUX_TEST_MODE__SHIFT                                                             0x1c
+#define DP_AUX1_AUX_CONTROL__AUX_EN_MASK                                                                      0x00000001L
+#define DP_AUX1_AUX_CONTROL__AUX_RESET_MASK                                                                   0x00000010L
+#define DP_AUX1_AUX_CONTROL__AUX_RESET_DONE_MASK                                                              0x00000020L
+#define DP_AUX1_AUX_CONTROL__AUX_LS_READ_EN_MASK                                                              0x00000100L
+#define DP_AUX1_AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK                                                       0x00001000L
+#define DP_AUX1_AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK                                                       0x00010000L
+#define DP_AUX1_AUX_CONTROL__AUX_MODE_DET_EN_MASK                                                             0x00040000L
+#define DP_AUX1_AUX_CONTROL__AUX_HPD_SEL_MASK                                                                 0x00700000L
+#define DP_AUX1_AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK                                                           0x01000000L
+#define DP_AUX1_AUX_CONTROL__AUX_TEST_MODE_MASK                                                               0x10000000L
+//DP_AUX1_AUX_SW_CONTROL
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_GO__SHIFT                                                              0x0
+#define DP_AUX1_AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT                                                       0x2
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT                                                     0x4
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT                                                        0x10
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_GO_MASK                                                                0x00000001L
+#define DP_AUX1_AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK                                                         0x00000004L
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK                                                       0x000000F0L
+#define DP_AUX1_AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK                                                          0x001F0000L
+//DP_AUX1_AUX_ARB_CONTROL
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT                                                      0x0
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT                                                0x2
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT                                                   0x8
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT                                                   0xa
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT                                                0x10
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT                                        0x10
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT                                             0x11
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT                                              0x18
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT                                      0x18
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT                                           0x19
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK                                                        0x00000001L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK                                                  0x0000000CL
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK                                                     0x00000100L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK                                                     0x00000400L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK                                                  0x00010000L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK                                          0x00010000L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK                                               0x00020000L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK                                                0x01000000L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK                                        0x01000000L
+#define DP_AUX1_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK                                             0x02000000L
+//DP_AUX1_AUX_INTERRUPT_CONTROL
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT                                                 0x0
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT                                                 0x1
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT                                                0x2
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT                                                 0x4
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT                                                 0x5
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT                                                0x6
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK                                                   0x00000001L
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK                                                   0x00000002L
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK                                                  0x00000004L
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK                                                   0x00000010L
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK                                                   0x00000020L
+#define DP_AUX1_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK                                                  0x00000040L
+//DP_AUX1_AUX_SW_STATUS
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_DONE__SHIFT                                                             0x0
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_REQ__SHIFT                                                              0x1
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX1_AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT                                                          0x1d
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_DONE_MASK                                                               0x00000001L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_REQ_MASK                                                                0x00000002L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX1_AUX_SW_STATUS__AUX_ARB_STATUS_MASK                                                            0x60000000L
+//DP_AUX1_AUX_LS_STATUS
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_DONE__SHIFT                                                             0x0
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_REQ__SHIFT                                                              0x1
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT                                                           0x1d
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT                                                          0x1e
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT                                                      0x1f
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_DONE_MASK                                                               0x00000001L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_REQ_MASK                                                                0x00000002L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK                                                             0x20000000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_UPDATED_MASK                                                            0x40000000L
+#define DP_AUX1_AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK                                                        0x80000000L
+//DP_AUX1_AUX_SW_DATA
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT                                                            0x0
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_DATA__SHIFT                                                               0x8
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_INDEX__SHIFT                                                              0x10
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT                                              0x1f
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_DATA_RW_MASK                                                              0x00000001L
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_INDEX_MASK                                                                0x001F0000L
+#define DP_AUX1_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK                                                0x80000000L
+//DP_AUX1_AUX_LS_DATA
+#define DP_AUX1_AUX_LS_DATA__AUX_LS_DATA__SHIFT                                                               0x8
+#define DP_AUX1_AUX_LS_DATA__AUX_LS_INDEX__SHIFT                                                              0x10
+#define DP_AUX1_AUX_LS_DATA__AUX_LS_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX1_AUX_LS_DATA__AUX_LS_INDEX_MASK                                                                0x001F0000L
+//DP_AUX1_AUX_DPHY_TX_REF_CONTROL
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT                                                0x0
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT                                                   0x4
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT                                                0x10
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK                                                  0x00000001L
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK                                                     0x00000030L
+#define DP_AUX1_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK                                                  0x01FF0000L
+//DP_AUX1_AUX_DPHY_TX_CONTROL
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT                                              0x0
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL__SHIFT                                          0x4
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME__SHIFT                                             0x6
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT                                          0x8
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY__SHIFT                                          0x10
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK                                                0x0000000FL
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL_MASK                                            0x00000030L
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME_MASK                                               0x00000040L
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK                                            0x00003F00L
+#define DP_AUX1_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY_MASK                                            0x00070000L
+//DP_AUX1_AUX_DPHY_RX_CONTROL0
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT                                              0x4
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT                                            0x8
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT                                       0xc
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT                                      0x10
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT                        0x11
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT                               0x12
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT                                0x13
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT                                          0x14
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT                                       0x1c
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK                                                0x00000070L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK                                              0x00000700L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK                                         0x00003000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK                                        0x00010000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK                          0x00020000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK                                 0x00040000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK                                  0x00080000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK                                            0x00300000L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK                                         0x70000000L
+//DP_AUX1_AUX_DPHY_RX_CONTROL1
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT                                            0x0
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN__SHIFT                                               0x8
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL__SHIFT                                           0xf
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK                                              0x000000FFL
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MASK                                                 0x00007F00L
+#define DP_AUX1_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL_MASK                                             0x00018000L
+//DP_AUX1_AUX_DPHY_TX_STATUS
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT                                                      0x0
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT                                                       0x4
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT                                             0x10
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK                                                        0x00000001L
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK                                                         0x00000070L
+#define DP_AUX1_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK                                               0x01FF0000L
+//DP_AUX1_AUX_DPHY_RX_STATUS
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT                                                       0x0
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT                                            0x8
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT                                       0x10
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT                                             0x15
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK                                                         0x00000007L
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK                                              0x00001F00L
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK                                         0x001F0000L
+#define DP_AUX1_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK                                               0x3FE00000L
+//DP_AUX1_AUX_PHY_WAKE_CNTL
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO__SHIFT                                                  0x0
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING__SHIFT                                             0x1
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY__SHIFT                                            0x2
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK__SHIFT                                                 0x3
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO_MASK                                                    0x00000001L
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING_MASK                                               0x00000002L
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY_MASK                                              0x00000004L
+#define DP_AUX1_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK_MASK                                                   0x00000008L
+//DP_AUX1_AUX_PHY_WAKE_STATUS
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE__SHIFT                                    0x4
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT__SHIFT                                          0x7
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON__SHIFT                                           0x9
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE__SHIFT                                         0xb
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL__SHIFT                                   0xc
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP__SHIFT                                     0xe
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L__SHIFT                                   0x11
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H__SHIFT                                   0x12
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET__SHIFT                                      0x14
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H__SHIFT                                   0x16
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L__SHIFT                                   0x17
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE_MASK                                      0x00000070L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_MASK                                            0x00000080L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON_MASK                                             0x00000200L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE_MASK                                           0x00000800L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL_MASK                                     0x00001000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP_MASK                                       0x00004000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L_MASK                                     0x00020000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H_MASK                                     0x00040000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET_MASK                                        0x00100000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H_MASK                                     0x00400000L
+#define DP_AUX1_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L_MASK                                     0x00800000L
+
+
+// addressBlock: dce_dc_dcoh_dp_aux2_dispdec
+//DP_AUX2_AUX_CONTROL
+#define DP_AUX2_AUX_CONTROL__AUX_EN__SHIFT                                                                    0x0
+#define DP_AUX2_AUX_CONTROL__AUX_RESET__SHIFT                                                                 0x4
+#define DP_AUX2_AUX_CONTROL__AUX_RESET_DONE__SHIFT                                                            0x5
+#define DP_AUX2_AUX_CONTROL__AUX_LS_READ_EN__SHIFT                                                            0x8
+#define DP_AUX2_AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT                                                     0xc
+#define DP_AUX2_AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT                                                     0x10
+#define DP_AUX2_AUX_CONTROL__AUX_MODE_DET_EN__SHIFT                                                           0x12
+#define DP_AUX2_AUX_CONTROL__AUX_HPD_SEL__SHIFT                                                               0x14
+#define DP_AUX2_AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT                                                         0x18
+#define DP_AUX2_AUX_CONTROL__AUX_TEST_MODE__SHIFT                                                             0x1c
+#define DP_AUX2_AUX_CONTROL__AUX_EN_MASK                                                                      0x00000001L
+#define DP_AUX2_AUX_CONTROL__AUX_RESET_MASK                                                                   0x00000010L
+#define DP_AUX2_AUX_CONTROL__AUX_RESET_DONE_MASK                                                              0x00000020L
+#define DP_AUX2_AUX_CONTROL__AUX_LS_READ_EN_MASK                                                              0x00000100L
+#define DP_AUX2_AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK                                                       0x00001000L
+#define DP_AUX2_AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK                                                       0x00010000L
+#define DP_AUX2_AUX_CONTROL__AUX_MODE_DET_EN_MASK                                                             0x00040000L
+#define DP_AUX2_AUX_CONTROL__AUX_HPD_SEL_MASK                                                                 0x00700000L
+#define DP_AUX2_AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK                                                           0x01000000L
+#define DP_AUX2_AUX_CONTROL__AUX_TEST_MODE_MASK                                                               0x10000000L
+//DP_AUX2_AUX_SW_CONTROL
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_GO__SHIFT                                                              0x0
+#define DP_AUX2_AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT                                                       0x2
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT                                                     0x4
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT                                                        0x10
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_GO_MASK                                                                0x00000001L
+#define DP_AUX2_AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK                                                         0x00000004L
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK                                                       0x000000F0L
+#define DP_AUX2_AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK                                                          0x001F0000L
+//DP_AUX2_AUX_ARB_CONTROL
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT                                                      0x0
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT                                                0x2
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT                                                   0x8
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT                                                   0xa
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT                                                0x10
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT                                        0x10
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT                                             0x11
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT                                              0x18
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT                                      0x18
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT                                           0x19
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK                                                        0x00000001L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK                                                  0x0000000CL
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK                                                     0x00000100L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK                                                     0x00000400L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK                                                  0x00010000L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK                                          0x00010000L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK                                               0x00020000L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK                                                0x01000000L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK                                        0x01000000L
+#define DP_AUX2_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK                                             0x02000000L
+//DP_AUX2_AUX_INTERRUPT_CONTROL
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT                                                 0x0
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT                                                 0x1
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT                                                0x2
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT                                                 0x4
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT                                                 0x5
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT                                                0x6
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK                                                   0x00000001L
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK                                                   0x00000002L
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK                                                  0x00000004L
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK                                                   0x00000010L
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK                                                   0x00000020L
+#define DP_AUX2_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK                                                  0x00000040L
+//DP_AUX2_AUX_SW_STATUS
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_DONE__SHIFT                                                             0x0
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_REQ__SHIFT                                                              0x1
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX2_AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT                                                          0x1d
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_DONE_MASK                                                               0x00000001L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_REQ_MASK                                                                0x00000002L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX2_AUX_SW_STATUS__AUX_ARB_STATUS_MASK                                                            0x60000000L
+//DP_AUX2_AUX_LS_STATUS
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_DONE__SHIFT                                                             0x0
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_REQ__SHIFT                                                              0x1
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT                                                           0x1d
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT                                                          0x1e
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT                                                      0x1f
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_DONE_MASK                                                               0x00000001L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_REQ_MASK                                                                0x00000002L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK                                                             0x20000000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_UPDATED_MASK                                                            0x40000000L
+#define DP_AUX2_AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK                                                        0x80000000L
+//DP_AUX2_AUX_SW_DATA
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT                                                            0x0
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_DATA__SHIFT                                                               0x8
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_INDEX__SHIFT                                                              0x10
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT                                              0x1f
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_DATA_RW_MASK                                                              0x00000001L
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_INDEX_MASK                                                                0x001F0000L
+#define DP_AUX2_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK                                                0x80000000L
+//DP_AUX2_AUX_LS_DATA
+#define DP_AUX2_AUX_LS_DATA__AUX_LS_DATA__SHIFT                                                               0x8
+#define DP_AUX2_AUX_LS_DATA__AUX_LS_INDEX__SHIFT                                                              0x10
+#define DP_AUX2_AUX_LS_DATA__AUX_LS_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX2_AUX_LS_DATA__AUX_LS_INDEX_MASK                                                                0x001F0000L
+//DP_AUX2_AUX_DPHY_TX_REF_CONTROL
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT                                                0x0
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT                                                   0x4
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT                                                0x10
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK                                                  0x00000001L
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK                                                     0x00000030L
+#define DP_AUX2_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK                                                  0x01FF0000L
+//DP_AUX2_AUX_DPHY_TX_CONTROL
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT                                              0x0
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL__SHIFT                                          0x4
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME__SHIFT                                             0x6
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT                                          0x8
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY__SHIFT                                          0x10
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK                                                0x0000000FL
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL_MASK                                            0x00000030L
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME_MASK                                               0x00000040L
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK                                            0x00003F00L
+#define DP_AUX2_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY_MASK                                            0x00070000L
+//DP_AUX2_AUX_DPHY_RX_CONTROL0
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT                                              0x4
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT                                            0x8
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT                                       0xc
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT                                      0x10
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT                        0x11
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT                               0x12
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT                                0x13
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT                                          0x14
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT                                       0x1c
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK                                                0x00000070L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK                                              0x00000700L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK                                         0x00003000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK                                        0x00010000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK                          0x00020000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK                                 0x00040000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK                                  0x00080000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK                                            0x00300000L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK                                         0x70000000L
+//DP_AUX2_AUX_DPHY_RX_CONTROL1
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT                                            0x0
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN__SHIFT                                               0x8
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL__SHIFT                                           0xf
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK                                              0x000000FFL
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MASK                                                 0x00007F00L
+#define DP_AUX2_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL_MASK                                             0x00018000L
+//DP_AUX2_AUX_DPHY_TX_STATUS
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT                                                      0x0
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT                                                       0x4
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT                                             0x10
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK                                                        0x00000001L
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK                                                         0x00000070L
+#define DP_AUX2_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK                                               0x01FF0000L
+//DP_AUX2_AUX_DPHY_RX_STATUS
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT                                                       0x0
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT                                            0x8
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT                                       0x10
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT                                             0x15
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK                                                         0x00000007L
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK                                              0x00001F00L
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK                                         0x001F0000L
+#define DP_AUX2_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK                                               0x3FE00000L
+//DP_AUX2_AUX_PHY_WAKE_CNTL
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO__SHIFT                                                  0x0
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING__SHIFT                                             0x1
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY__SHIFT                                            0x2
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK__SHIFT                                                 0x3
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO_MASK                                                    0x00000001L
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING_MASK                                               0x00000002L
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY_MASK                                              0x00000004L
+#define DP_AUX2_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK_MASK                                                   0x00000008L
+//DP_AUX2_AUX_PHY_WAKE_STATUS
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE__SHIFT                                    0x4
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT__SHIFT                                          0x7
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON__SHIFT                                           0x9
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE__SHIFT                                         0xb
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL__SHIFT                                   0xc
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP__SHIFT                                     0xe
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L__SHIFT                                   0x11
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H__SHIFT                                   0x12
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET__SHIFT                                      0x14
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H__SHIFT                                   0x16
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L__SHIFT                                   0x17
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE_MASK                                      0x00000070L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_MASK                                            0x00000080L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON_MASK                                             0x00000200L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE_MASK                                           0x00000800L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL_MASK                                     0x00001000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP_MASK                                       0x00004000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L_MASK                                     0x00020000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H_MASK                                     0x00040000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET_MASK                                        0x00100000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H_MASK                                     0x00400000L
+#define DP_AUX2_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L_MASK                                     0x00800000L
+
+
+// addressBlock: dce_dc_dcoh_dp_aux3_dispdec
+//DP_AUX3_AUX_CONTROL
+#define DP_AUX3_AUX_CONTROL__AUX_EN__SHIFT                                                                    0x0
+#define DP_AUX3_AUX_CONTROL__AUX_RESET__SHIFT                                                                 0x4
+#define DP_AUX3_AUX_CONTROL__AUX_RESET_DONE__SHIFT                                                            0x5
+#define DP_AUX3_AUX_CONTROL__AUX_LS_READ_EN__SHIFT                                                            0x8
+#define DP_AUX3_AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT                                                     0xc
+#define DP_AUX3_AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT                                                     0x10
+#define DP_AUX3_AUX_CONTROL__AUX_MODE_DET_EN__SHIFT                                                           0x12
+#define DP_AUX3_AUX_CONTROL__AUX_HPD_SEL__SHIFT                                                               0x14
+#define DP_AUX3_AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT                                                         0x18
+#define DP_AUX3_AUX_CONTROL__AUX_TEST_MODE__SHIFT                                                             0x1c
+#define DP_AUX3_AUX_CONTROL__AUX_EN_MASK                                                                      0x00000001L
+#define DP_AUX3_AUX_CONTROL__AUX_RESET_MASK                                                                   0x00000010L
+#define DP_AUX3_AUX_CONTROL__AUX_RESET_DONE_MASK                                                              0x00000020L
+#define DP_AUX3_AUX_CONTROL__AUX_LS_READ_EN_MASK                                                              0x00000100L
+#define DP_AUX3_AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK                                                       0x00001000L
+#define DP_AUX3_AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK                                                       0x00010000L
+#define DP_AUX3_AUX_CONTROL__AUX_MODE_DET_EN_MASK                                                             0x00040000L
+#define DP_AUX3_AUX_CONTROL__AUX_HPD_SEL_MASK                                                                 0x00700000L
+#define DP_AUX3_AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK                                                           0x01000000L
+#define DP_AUX3_AUX_CONTROL__AUX_TEST_MODE_MASK                                                               0x10000000L
+//DP_AUX3_AUX_SW_CONTROL
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_GO__SHIFT                                                              0x0
+#define DP_AUX3_AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT                                                       0x2
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT                                                     0x4
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT                                                        0x10
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_GO_MASK                                                                0x00000001L
+#define DP_AUX3_AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK                                                         0x00000004L
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK                                                       0x000000F0L
+#define DP_AUX3_AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK                                                          0x001F0000L
+//DP_AUX3_AUX_ARB_CONTROL
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT                                                      0x0
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT                                                0x2
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT                                                   0x8
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT                                                   0xa
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT                                                0x10
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT                                        0x10
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT                                             0x11
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT                                              0x18
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT                                      0x18
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT                                           0x19
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK                                                        0x00000001L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK                                                  0x0000000CL
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK                                                     0x00000100L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK                                                     0x00000400L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK                                                  0x00010000L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK                                          0x00010000L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK                                               0x00020000L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK                                                0x01000000L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK                                        0x01000000L
+#define DP_AUX3_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK                                             0x02000000L
+//DP_AUX3_AUX_INTERRUPT_CONTROL
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT                                                 0x0
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT                                                 0x1
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT                                                0x2
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT                                                 0x4
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT                                                 0x5
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT                                                0x6
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK                                                   0x00000001L
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK                                                   0x00000002L
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK                                                  0x00000004L
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK                                                   0x00000010L
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK                                                   0x00000020L
+#define DP_AUX3_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK                                                  0x00000040L
+//DP_AUX3_AUX_SW_STATUS
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_DONE__SHIFT                                                             0x0
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_REQ__SHIFT                                                              0x1
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX3_AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT                                                          0x1d
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_DONE_MASK                                                               0x00000001L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_REQ_MASK                                                                0x00000002L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX3_AUX_SW_STATUS__AUX_ARB_STATUS_MASK                                                            0x60000000L
+//DP_AUX3_AUX_LS_STATUS
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_DONE__SHIFT                                                             0x0
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_REQ__SHIFT                                                              0x1
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT                                                           0x1d
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT                                                          0x1e
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT                                                      0x1f
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_DONE_MASK                                                               0x00000001L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_REQ_MASK                                                                0x00000002L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK                                                             0x20000000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_UPDATED_MASK                                                            0x40000000L
+#define DP_AUX3_AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK                                                        0x80000000L
+//DP_AUX3_AUX_SW_DATA
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT                                                            0x0
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_DATA__SHIFT                                                               0x8
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_INDEX__SHIFT                                                              0x10
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT                                              0x1f
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_DATA_RW_MASK                                                              0x00000001L
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_INDEX_MASK                                                                0x001F0000L
+#define DP_AUX3_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK                                                0x80000000L
+//DP_AUX3_AUX_LS_DATA
+#define DP_AUX3_AUX_LS_DATA__AUX_LS_DATA__SHIFT                                                               0x8
+#define DP_AUX3_AUX_LS_DATA__AUX_LS_INDEX__SHIFT                                                              0x10
+#define DP_AUX3_AUX_LS_DATA__AUX_LS_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX3_AUX_LS_DATA__AUX_LS_INDEX_MASK                                                                0x001F0000L
+//DP_AUX3_AUX_DPHY_TX_REF_CONTROL
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT                                                0x0
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT                                                   0x4
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT                                                0x10
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK                                                  0x00000001L
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK                                                     0x00000030L
+#define DP_AUX3_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK                                                  0x01FF0000L
+//DP_AUX3_AUX_DPHY_TX_CONTROL
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT                                              0x0
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL__SHIFT                                          0x4
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME__SHIFT                                             0x6
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT                                          0x8
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY__SHIFT                                          0x10
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK                                                0x0000000FL
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL_MASK                                            0x00000030L
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME_MASK                                               0x00000040L
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK                                            0x00003F00L
+#define DP_AUX3_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY_MASK                                            0x00070000L
+//DP_AUX3_AUX_DPHY_RX_CONTROL0
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT                                              0x4
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT                                            0x8
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT                                       0xc
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT                                      0x10
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT                        0x11
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT                               0x12
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT                                0x13
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT                                          0x14
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT                                       0x1c
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK                                                0x00000070L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK                                              0x00000700L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK                                         0x00003000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK                                        0x00010000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK                          0x00020000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK                                 0x00040000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK                                  0x00080000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK                                            0x00300000L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK                                         0x70000000L
+//DP_AUX3_AUX_DPHY_RX_CONTROL1
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT                                            0x0
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN__SHIFT                                               0x8
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL__SHIFT                                           0xf
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK                                              0x000000FFL
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MASK                                                 0x00007F00L
+#define DP_AUX3_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL_MASK                                             0x00018000L
+//DP_AUX3_AUX_DPHY_TX_STATUS
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT                                                      0x0
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT                                                       0x4
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT                                             0x10
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK                                                        0x00000001L
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK                                                         0x00000070L
+#define DP_AUX3_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK                                               0x01FF0000L
+//DP_AUX3_AUX_DPHY_RX_STATUS
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT                                                       0x0
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT                                            0x8
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT                                       0x10
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT                                             0x15
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK                                                         0x00000007L
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK                                              0x00001F00L
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK                                         0x001F0000L
+#define DP_AUX3_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK                                               0x3FE00000L
+//DP_AUX3_AUX_PHY_WAKE_CNTL
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO__SHIFT                                                  0x0
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING__SHIFT                                             0x1
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY__SHIFT                                            0x2
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK__SHIFT                                                 0x3
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO_MASK                                                    0x00000001L
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING_MASK                                               0x00000002L
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY_MASK                                              0x00000004L
+#define DP_AUX3_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK_MASK                                                   0x00000008L
+//DP_AUX3_AUX_PHY_WAKE_STATUS
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE__SHIFT                                    0x4
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT__SHIFT                                          0x7
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON__SHIFT                                           0x9
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE__SHIFT                                         0xb
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL__SHIFT                                   0xc
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP__SHIFT                                     0xe
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L__SHIFT                                   0x11
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H__SHIFT                                   0x12
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET__SHIFT                                      0x14
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H__SHIFT                                   0x16
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L__SHIFT                                   0x17
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE_MASK                                      0x00000070L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_MASK                                            0x00000080L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON_MASK                                             0x00000200L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE_MASK                                           0x00000800L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL_MASK                                     0x00001000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP_MASK                                       0x00004000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L_MASK                                     0x00020000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H_MASK                                     0x00040000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET_MASK                                        0x00100000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H_MASK                                     0x00400000L
+#define DP_AUX3_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L_MASK                                     0x00800000L
+
+
+// addressBlock: dce_dc_dcoh_dp_aux4_dispdec
+//DP_AUX4_AUX_CONTROL
+#define DP_AUX4_AUX_CONTROL__AUX_EN__SHIFT                                                                    0x0
+#define DP_AUX4_AUX_CONTROL__AUX_RESET__SHIFT                                                                 0x4
+#define DP_AUX4_AUX_CONTROL__AUX_RESET_DONE__SHIFT                                                            0x5
+#define DP_AUX4_AUX_CONTROL__AUX_LS_READ_EN__SHIFT                                                            0x8
+#define DP_AUX4_AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT                                                     0xc
+#define DP_AUX4_AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT                                                     0x10
+#define DP_AUX4_AUX_CONTROL__AUX_MODE_DET_EN__SHIFT                                                           0x12
+#define DP_AUX4_AUX_CONTROL__AUX_HPD_SEL__SHIFT                                                               0x14
+#define DP_AUX4_AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT                                                         0x18
+#define DP_AUX4_AUX_CONTROL__AUX_TEST_MODE__SHIFT                                                             0x1c
+#define DP_AUX4_AUX_CONTROL__AUX_EN_MASK                                                                      0x00000001L
+#define DP_AUX4_AUX_CONTROL__AUX_RESET_MASK                                                                   0x00000010L
+#define DP_AUX4_AUX_CONTROL__AUX_RESET_DONE_MASK                                                              0x00000020L
+#define DP_AUX4_AUX_CONTROL__AUX_LS_READ_EN_MASK                                                              0x00000100L
+#define DP_AUX4_AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK                                                       0x00001000L
+#define DP_AUX4_AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK                                                       0x00010000L
+#define DP_AUX4_AUX_CONTROL__AUX_MODE_DET_EN_MASK                                                             0x00040000L
+#define DP_AUX4_AUX_CONTROL__AUX_HPD_SEL_MASK                                                                 0x00700000L
+#define DP_AUX4_AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK                                                           0x01000000L
+#define DP_AUX4_AUX_CONTROL__AUX_TEST_MODE_MASK                                                               0x10000000L
+//DP_AUX4_AUX_SW_CONTROL
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_GO__SHIFT                                                              0x0
+#define DP_AUX4_AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT                                                       0x2
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT                                                     0x4
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT                                                        0x10
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_GO_MASK                                                                0x00000001L
+#define DP_AUX4_AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK                                                         0x00000004L
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK                                                       0x000000F0L
+#define DP_AUX4_AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK                                                          0x001F0000L
+//DP_AUX4_AUX_ARB_CONTROL
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT                                                      0x0
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT                                                0x2
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT                                                   0x8
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT                                                   0xa
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT                                                0x10
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT                                        0x10
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT                                             0x11
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT                                              0x18
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT                                      0x18
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT                                           0x19
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK                                                        0x00000001L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK                                                  0x0000000CL
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK                                                     0x00000100L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK                                                     0x00000400L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK                                                  0x00010000L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK                                          0x00010000L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK                                               0x00020000L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK                                                0x01000000L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK                                        0x01000000L
+#define DP_AUX4_AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK                                             0x02000000L
+//DP_AUX4_AUX_INTERRUPT_CONTROL
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT                                                 0x0
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT                                                 0x1
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT                                                0x2
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT                                                 0x4
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT                                                 0x5
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT                                                0x6
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK                                                   0x00000001L
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK                                                   0x00000002L
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK                                                  0x00000004L
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK                                                   0x00000010L
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK                                                   0x00000020L
+#define DP_AUX4_AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK                                                  0x00000040L
+//DP_AUX4_AUX_SW_STATUS
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_DONE__SHIFT                                                             0x0
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_REQ__SHIFT                                                              0x1
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX4_AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT                                                          0x1d
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_DONE_MASK                                                               0x00000001L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_REQ_MASK                                                                0x00000002L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX4_AUX_SW_STATUS__AUX_ARB_STATUS_MASK                                                            0x60000000L
+//DP_AUX4_AUX_LS_STATUS
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_DONE__SHIFT                                                             0x0
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_REQ__SHIFT                                                              0x1
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT                                                 0x4
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT                                                       0x7
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT                                                      0x8
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT                                                       0x9
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT                                                  0xa
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT                                                     0xb
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT                                                0xc
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT                                                  0xe
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT                                                0x11
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT                                                0x12
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT                                                 0x13
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT                                                   0x14
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT                                                0x16
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT                                                0x17
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT                                                 0x18
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT                                                           0x1d
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT                                                          0x1e
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT                                                      0x1f
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_DONE_MASK                                                               0x00000001L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_REQ_MASK                                                                0x00000002L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK                                                   0x00000070L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK                                                         0x00000080L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK                                                        0x00000100L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK                                                         0x00000200L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK                                                    0x00000400L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK                                                       0x00000800L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK                                                  0x00001000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK                                                    0x00004000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK                                                  0x00020000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK                                                  0x00040000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK                                                   0x00080000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK                                                     0x00100000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK                                                  0x00400000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK                                                  0x00800000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK                                                   0x1F000000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK                                                             0x20000000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_UPDATED_MASK                                                            0x40000000L
+#define DP_AUX4_AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK                                                        0x80000000L
+//DP_AUX4_AUX_SW_DATA
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT                                                            0x0
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_DATA__SHIFT                                                               0x8
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_INDEX__SHIFT                                                              0x10
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT                                              0x1f
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_DATA_RW_MASK                                                              0x00000001L
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_INDEX_MASK                                                                0x001F0000L
+#define DP_AUX4_AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK                                                0x80000000L
+//DP_AUX4_AUX_LS_DATA
+#define DP_AUX4_AUX_LS_DATA__AUX_LS_DATA__SHIFT                                                               0x8
+#define DP_AUX4_AUX_LS_DATA__AUX_LS_INDEX__SHIFT                                                              0x10
+#define DP_AUX4_AUX_LS_DATA__AUX_LS_DATA_MASK                                                                 0x0000FF00L
+#define DP_AUX4_AUX_LS_DATA__AUX_LS_INDEX_MASK                                                                0x001F0000L
+//DP_AUX4_AUX_DPHY_TX_REF_CONTROL
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT                                                0x0
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT                                                   0x4
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT                                                0x10
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK                                                  0x00000001L
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK                                                     0x00000030L
+#define DP_AUX4_AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK                                                  0x01FF0000L
+//DP_AUX4_AUX_DPHY_TX_CONTROL
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT                                              0x0
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL__SHIFT                                          0x4
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME__SHIFT                                             0x6
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT                                          0x8
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY__SHIFT                                          0x10
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK                                                0x0000000FL
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MUL_MASK                                            0x00000030L
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_OE_ASSERT_TIME_MASK                                               0x00000040L
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK                                            0x00003F00L
+#define DP_AUX4_AUX_DPHY_TX_CONTROL__AUX_MODE_DET_CHECK_DELAY_MASK                                            0x00070000L
+//DP_AUX4_AUX_DPHY_RX_CONTROL0
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT                                              0x4
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT                                            0x8
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT                                       0xc
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT                                      0x10
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT                        0x11
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT                               0x12
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT                                0x13
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT                                          0x14
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT                                       0x1c
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK                                                0x00000070L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK                                              0x00000700L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK                                         0x00003000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK                                        0x00010000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK                          0x00020000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK                                 0x00040000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK                                  0x00080000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK                                            0x00300000L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK                                         0x70000000L
+//DP_AUX4_AUX_DPHY_RX_CONTROL1
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT                                            0x0
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN__SHIFT                                               0x8
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL__SHIFT                                           0xf
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK                                              0x000000FFL
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MASK                                                 0x00007F00L
+#define DP_AUX4_AUX_DPHY_RX_CONTROL1__AUX_RX_TIMEOUT_LEN_MUL_MASK                                             0x00018000L
+//DP_AUX4_AUX_DPHY_TX_STATUS
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT                                                      0x0
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT                                                       0x4
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT                                             0x10
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK                                                        0x00000001L
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK                                                         0x00000070L
+#define DP_AUX4_AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK                                               0x01FF0000L
+//DP_AUX4_AUX_DPHY_RX_STATUS
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT                                                       0x0
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT                                            0x8
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT                                       0x10
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT                                             0x15
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK                                                         0x00000007L
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK                                              0x00001F00L
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK                                         0x001F0000L
+#define DP_AUX4_AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK                                               0x3FE00000L
+//DP_AUX4_AUX_PHY_WAKE_CNTL
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO__SHIFT                                                  0x0
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING__SHIFT                                             0x1
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY__SHIFT                                            0x2
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK__SHIFT                                                 0x3
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_GO_MASK                                                    0x00000001L
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PENDING_MASK                                               0x00000002L
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_PRIORITY_MASK                                              0x00000004L
+#define DP_AUX4_AUX_PHY_WAKE_CNTL__DP_AUX_PHY_WAKE_ACK_MASK                                                   0x00000008L
+//DP_AUX4_AUX_PHY_WAKE_STATUS
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE__SHIFT                                    0x4
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT__SHIFT                                          0x7
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON__SHIFT                                           0x9
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE__SHIFT                                         0xb
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL__SHIFT                                   0xc
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP__SHIFT                                     0xe
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L__SHIFT                                   0x11
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H__SHIFT                                   0x12
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET__SHIFT                                      0x14
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H__SHIFT                                   0x16
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L__SHIFT                                   0x17
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_STATE_MASK                                      0x00000070L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_TIMEOUT_MASK                                            0x00000080L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_HPD_DISCON_MASK                                             0x00000200L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_NON_AUX_MODE_MASK                                           0x00000800L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_MIN_COUNT_VIOL_MASK                                     0x00001000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_INVALID_STOP_MASK                                       0x00004000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_L_MASK                                     0x00020000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_SYNC_INVALID_H_MASK                                     0x00040000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_NO_DET_MASK                                        0x00100000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_H_MASK                                     0x00400000L
+#define DP_AUX4_AUX_PHY_WAKE_STATUS__AUX_PHY_WAKE_ACK_RECV_INVALID_L_MASK                                     0x00800000L
+
+
+// addressBlock: dce_dc_dcoh_hpd0_dispdec
+//HPD0_DC_HPD_INT_STATUS
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS__SHIFT                                                      0x0
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_SENSE__SHIFT                                                           0x1
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED__SHIFT                                                   0x4
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS__SHIFT                                                   0x8
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL__SHIFT                                       0xc
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT                                    0x18
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS_MASK                                                        0x00000001L
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK                                                             0x00000002L
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED_MASK                                                     0x00000010L
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS_MASK                                                     0x00000100L
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL_MASK                                         0x000FF000L
+#define HPD0_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL_MASK                                      0xFF000000L
+//HPD0_DC_HPD_INT_CONTROL
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK__SHIFT                                                        0x0
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY__SHIFT                                                   0x8
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_EN__SHIFT                                                         0x10
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK__SHIFT                                                     0x14
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN__SHIFT                                                      0x18
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK_MASK                                                          0x00000001L
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY_MASK                                                     0x00000100L
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_INT_EN_MASK                                                           0x00010000L
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK_MASK                                                       0x00100000L
+#define HPD0_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN_MASK                                                        0x01000000L
+//HPD0_DC_HPD_CONTROL
+#define HPD0_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER__SHIFT                                                   0x0
+#define HPD0_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER__SHIFT                                                       0x10
+#define HPD0_DC_HPD_CONTROL__DC_HPD_EN__SHIFT                                                                 0x1c
+#define HPD0_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER_MASK                                                     0x00001FFFL
+#define HPD0_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER_MASK                                                         0x03FF0000L
+#define HPD0_DC_HPD_CONTROL__DC_HPD_EN_MASK                                                                   0x10000000L
+//HPD0_DC_HPD_FAST_TRAIN_CNTL
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY__SHIFT                                       0x0
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY__SHIFT                                   0xc
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN__SHIFT                                          0x18
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN__SHIFT                                      0x1c
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY_MASK                                         0x000000FFL
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY_MASK                                     0x000FF000L
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN_MASK                                            0x01000000L
+#define HPD0_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN_MASK                                        0x10000000L
+//HPD0_DC_HPD_TOGGLE_FILT_CNTL
+#define HPD0_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY__SHIFT                                         0x0
+#define HPD0_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY__SHIFT                                      0x14
+#define HPD0_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY_MASK                                           0x000000FFL
+#define HPD0_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY_MASK                                        0x0FF00000L
+
+
+// addressBlock: dce_dc_dcoh_hpd1_dispdec
+//HPD1_DC_HPD_INT_STATUS
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS__SHIFT                                                      0x0
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_SENSE__SHIFT                                                           0x1
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED__SHIFT                                                   0x4
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS__SHIFT                                                   0x8
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL__SHIFT                                       0xc
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT                                    0x18
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS_MASK                                                        0x00000001L
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK                                                             0x00000002L
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED_MASK                                                     0x00000010L
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS_MASK                                                     0x00000100L
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL_MASK                                         0x000FF000L
+#define HPD1_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL_MASK                                      0xFF000000L
+//HPD1_DC_HPD_INT_CONTROL
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK__SHIFT                                                        0x0
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY__SHIFT                                                   0x8
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_EN__SHIFT                                                         0x10
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK__SHIFT                                                     0x14
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN__SHIFT                                                      0x18
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK_MASK                                                          0x00000001L
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY_MASK                                                     0x00000100L
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_INT_EN_MASK                                                           0x00010000L
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK_MASK                                                       0x00100000L
+#define HPD1_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN_MASK                                                        0x01000000L
+//HPD1_DC_HPD_CONTROL
+#define HPD1_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER__SHIFT                                                   0x0
+#define HPD1_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER__SHIFT                                                       0x10
+#define HPD1_DC_HPD_CONTROL__DC_HPD_EN__SHIFT                                                                 0x1c
+#define HPD1_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER_MASK                                                     0x00001FFFL
+#define HPD1_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER_MASK                                                         0x03FF0000L
+#define HPD1_DC_HPD_CONTROL__DC_HPD_EN_MASK                                                                   0x10000000L
+//HPD1_DC_HPD_FAST_TRAIN_CNTL
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY__SHIFT                                       0x0
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY__SHIFT                                   0xc
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN__SHIFT                                          0x18
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN__SHIFT                                      0x1c
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY_MASK                                         0x000000FFL
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY_MASK                                     0x000FF000L
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN_MASK                                            0x01000000L
+#define HPD1_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN_MASK                                        0x10000000L
+//HPD1_DC_HPD_TOGGLE_FILT_CNTL
+#define HPD1_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY__SHIFT                                         0x0
+#define HPD1_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY__SHIFT                                      0x14
+#define HPD1_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY_MASK                                           0x000000FFL
+#define HPD1_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY_MASK                                        0x0FF00000L
+
+
+// addressBlock: dce_dc_dcoh_hpd2_dispdec
+//HPD2_DC_HPD_INT_STATUS
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS__SHIFT                                                      0x0
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_SENSE__SHIFT                                                           0x1
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED__SHIFT                                                   0x4
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS__SHIFT                                                   0x8
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL__SHIFT                                       0xc
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT                                    0x18
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS_MASK                                                        0x00000001L
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK                                                             0x00000002L
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED_MASK                                                     0x00000010L
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS_MASK                                                     0x00000100L
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL_MASK                                         0x000FF000L
+#define HPD2_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL_MASK                                      0xFF000000L
+//HPD2_DC_HPD_INT_CONTROL
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK__SHIFT                                                        0x0
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY__SHIFT                                                   0x8
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_EN__SHIFT                                                         0x10
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK__SHIFT                                                     0x14
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN__SHIFT                                                      0x18
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK_MASK                                                          0x00000001L
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY_MASK                                                     0x00000100L
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_INT_EN_MASK                                                           0x00010000L
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK_MASK                                                       0x00100000L
+#define HPD2_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN_MASK                                                        0x01000000L
+//HPD2_DC_HPD_CONTROL
+#define HPD2_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER__SHIFT                                                   0x0
+#define HPD2_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER__SHIFT                                                       0x10
+#define HPD2_DC_HPD_CONTROL__DC_HPD_EN__SHIFT                                                                 0x1c
+#define HPD2_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER_MASK                                                     0x00001FFFL
+#define HPD2_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER_MASK                                                         0x03FF0000L
+#define HPD2_DC_HPD_CONTROL__DC_HPD_EN_MASK                                                                   0x10000000L
+//HPD2_DC_HPD_FAST_TRAIN_CNTL
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY__SHIFT                                       0x0
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY__SHIFT                                   0xc
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN__SHIFT                                          0x18
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN__SHIFT                                      0x1c
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY_MASK                                         0x000000FFL
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY_MASK                                     0x000FF000L
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN_MASK                                            0x01000000L
+#define HPD2_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN_MASK                                        0x10000000L
+//HPD2_DC_HPD_TOGGLE_FILT_CNTL
+#define HPD2_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY__SHIFT                                         0x0
+#define HPD2_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY__SHIFT                                      0x14
+#define HPD2_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY_MASK                                           0x000000FFL
+#define HPD2_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY_MASK                                        0x0FF00000L
+
+
+// addressBlock: dce_dc_dcoh_hpd3_dispdec
+//HPD3_DC_HPD_INT_STATUS
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS__SHIFT                                                      0x0
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_SENSE__SHIFT                                                           0x1
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED__SHIFT                                                   0x4
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS__SHIFT                                                   0x8
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL__SHIFT                                       0xc
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT                                    0x18
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS_MASK                                                        0x00000001L
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK                                                             0x00000002L
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED_MASK                                                     0x00000010L
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS_MASK                                                     0x00000100L
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL_MASK                                         0x000FF000L
+#define HPD3_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL_MASK                                      0xFF000000L
+//HPD3_DC_HPD_INT_CONTROL
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK__SHIFT                                                        0x0
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY__SHIFT                                                   0x8
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_EN__SHIFT                                                         0x10
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK__SHIFT                                                     0x14
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN__SHIFT                                                      0x18
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK_MASK                                                          0x00000001L
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY_MASK                                                     0x00000100L
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_INT_EN_MASK                                                           0x00010000L
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK_MASK                                                       0x00100000L
+#define HPD3_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN_MASK                                                        0x01000000L
+//HPD3_DC_HPD_CONTROL
+#define HPD3_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER__SHIFT                                                   0x0
+#define HPD3_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER__SHIFT                                                       0x10
+#define HPD3_DC_HPD_CONTROL__DC_HPD_EN__SHIFT                                                                 0x1c
+#define HPD3_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER_MASK                                                     0x00001FFFL
+#define HPD3_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER_MASK                                                         0x03FF0000L
+#define HPD3_DC_HPD_CONTROL__DC_HPD_EN_MASK                                                                   0x10000000L
+//HPD3_DC_HPD_FAST_TRAIN_CNTL
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY__SHIFT                                       0x0
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY__SHIFT                                   0xc
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN__SHIFT                                          0x18
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN__SHIFT                                      0x1c
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY_MASK                                         0x000000FFL
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY_MASK                                     0x000FF000L
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN_MASK                                            0x01000000L
+#define HPD3_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN_MASK                                        0x10000000L
+//HPD3_DC_HPD_TOGGLE_FILT_CNTL
+#define HPD3_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY__SHIFT                                         0x0
+#define HPD3_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY__SHIFT                                      0x14
+#define HPD3_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY_MASK                                           0x000000FFL
+#define HPD3_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY_MASK                                        0x0FF00000L
+
+
+// addressBlock: dce_dc_dcoh_hpd4_dispdec
+//HPD4_DC_HPD_INT_STATUS
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS__SHIFT                                                      0x0
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_SENSE__SHIFT                                                           0x1
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED__SHIFT                                                   0x4
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS__SHIFT                                                   0x8
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL__SHIFT                                       0xc
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT                                    0x18
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_INT_STATUS_MASK                                                        0x00000001L
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK                                                             0x00000002L
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_SENSE_DELAYED_MASK                                                     0x00000010L
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_RX_INT_STATUS_MASK                                                     0x00000100L
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_CON_TIMER_VAL_MASK                                         0x000FF000L
+#define HPD4_DC_HPD_INT_STATUS__DC_HPD_TOGGLE_FILT_DISCON_TIMER_VAL_MASK                                      0xFF000000L
+//HPD4_DC_HPD_INT_CONTROL
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK__SHIFT                                                        0x0
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY__SHIFT                                                   0x8
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_EN__SHIFT                                                         0x10
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK__SHIFT                                                     0x14
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN__SHIFT                                                      0x18
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_ACK_MASK                                                          0x00000001L
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_POLARITY_MASK                                                     0x00000100L
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_INT_EN_MASK                                                           0x00010000L
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_ACK_MASK                                                       0x00100000L
+#define HPD4_DC_HPD_INT_CONTROL__DC_HPD_RX_INT_EN_MASK                                                        0x01000000L
+//HPD4_DC_HPD_CONTROL
+#define HPD4_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER__SHIFT                                                   0x0
+#define HPD4_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER__SHIFT                                                       0x10
+#define HPD4_DC_HPD_CONTROL__DC_HPD_EN__SHIFT                                                                 0x1c
+#define HPD4_DC_HPD_CONTROL__DC_HPD_CONNECTION_TIMER_MASK                                                     0x00001FFFL
+#define HPD4_DC_HPD_CONTROL__DC_HPD_RX_INT_TIMER_MASK                                                         0x03FF0000L
+#define HPD4_DC_HPD_CONTROL__DC_HPD_EN_MASK                                                                   0x10000000L
+//HPD4_DC_HPD_FAST_TRAIN_CNTL
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY__SHIFT                                       0x0
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY__SHIFT                                   0xc
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN__SHIFT                                          0x18
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN__SHIFT                                      0x1c
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_DELAY_MASK                                         0x000000FFL
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_DELAY_MASK                                     0x000FF000L
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_AUX_TX_EN_MASK                                            0x01000000L
+#define HPD4_DC_HPD_FAST_TRAIN_CNTL__DC_HPD_CONNECT_FAST_TRAIN_EN_MASK                                        0x10000000L
+//HPD4_DC_HPD_TOGGLE_FILT_CNTL
+#define HPD4_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY__SHIFT                                         0x0
+#define HPD4_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY__SHIFT                                      0x14
+#define HPD4_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_CONNECT_INT_DELAY_MASK                                           0x000000FFL
+#define HPD4_DC_HPD_TOGGLE_FILT_CNTL__DC_HPD_DISCONNECT_INT_DELAY_MASK                                        0x0FF00000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux0_dispdec
+//DPIA_MUX0_DPIA_MUX_CONTROL
+#define DPIA_MUX0_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX0_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX0_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX0_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX0_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX0_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX0_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX0_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX0_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX0_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX0_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX0_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux1_dispdec
+//DPIA_MUX1_DPIA_MUX_CONTROL
+#define DPIA_MUX1_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX1_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX1_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX1_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX1_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX1_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX1_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX1_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX1_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX1_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX1_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX1_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux2_dispdec
+//DPIA_MUX2_DPIA_MUX_CONTROL
+#define DPIA_MUX2_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX2_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX2_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX2_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX2_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX2_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX2_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX2_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX2_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX2_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX2_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX2_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux3_dispdec
+//DPIA_MUX3_DPIA_MUX_CONTROL
+#define DPIA_MUX3_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX3_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX3_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX3_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX3_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX3_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX3_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX3_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX3_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX3_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX3_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX3_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux4_dispdec
+//DPIA_MUX4_DPIA_MUX_CONTROL
+#define DPIA_MUX4_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX4_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX4_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX4_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX4_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX4_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX4_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX4_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX4_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX4_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX4_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX4_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_dpia_mux5_dispdec
+//DPIA_MUX5_DPIA_MUX_CONTROL
+#define DPIA_MUX5_DPIA_MUX_CONTROL__ENABLE__SHIFT                                                             0x0
+#define DPIA_MUX5_DPIA_MUX_CONTROL__RESET__SHIFT                                                              0x4
+#define DPIA_MUX5_DPIA_MUX_CONTROL__RESET_DONE__SHIFT                                                         0x5
+#define DPIA_MUX5_DPIA_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                       0x6
+#define DPIA_MUX5_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT__SHIFT                                               0x8
+#define DPIA_MUX5_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT__SHIFT                                               0x10
+#define DPIA_MUX5_DPIA_MUX_CONTROL__ENABLE_MASK                                                               0x00000001L
+#define DPIA_MUX5_DPIA_MUX_CONTROL__RESET_MASK                                                                0x00000010L
+#define DPIA_MUX5_DPIA_MUX_CONTROL__RESET_DONE_MASK                                                           0x00000020L
+#define DPIA_MUX5_DPIA_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                         0x00000040L
+#define DPIA_MUX5_DPIA_MUX_CONTROL__DIG_DP_SOURCE_SELECT_MASK                                                 0x00000F00L
+#define DPIA_MUX5_DPIA_MUX_CONTROL__HPO_DP_SOURCE_SELECT_MASK                                                 0x00070000L
+
+
+// addressBlock: dce_dc_dcoh_phy_mux0_dispdec
+//PHY_MUX0_PHY_MUX_CONTROL
+#define PHY_MUX0_PHY_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                         0x0
+#define PHY_MUX0_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL__SHIFT                                                     0x4
+#define PHY_MUX0_PHY_MUX_CONTROL__HPO_DP_ENC_SEL__SHIFT                                                       0x8
+#define PHY_MUX0_PHY_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                           0x00000003L
+#define PHY_MUX0_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL_MASK                                                       0x00000070L
+#define PHY_MUX0_PHY_MUX_CONTROL__HPO_DP_ENC_SEL_MASK                                                         0x00000700L
+//PHY_MUX0_PORT_TYPE
+#define PHY_MUX0_PORT_TYPE__PORT_TYPE__SHIFT                                                                  0x0
+#define PHY_MUX0_PORT_TYPE__PORT_TYPE_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_dcoh_phy_mux1_dispdec
+//PHY_MUX1_PHY_MUX_CONTROL
+#define PHY_MUX1_PHY_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                         0x0
+#define PHY_MUX1_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL__SHIFT                                                     0x4
+#define PHY_MUX1_PHY_MUX_CONTROL__HPO_DP_ENC_SEL__SHIFT                                                       0x8
+#define PHY_MUX1_PHY_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                           0x00000003L
+#define PHY_MUX1_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL_MASK                                                       0x00000070L
+#define PHY_MUX1_PHY_MUX_CONTROL__HPO_DP_ENC_SEL_MASK                                                         0x00000700L
+//PHY_MUX1_PORT_TYPE
+#define PHY_MUX1_PORT_TYPE__PORT_TYPE__SHIFT                                                                  0x0
+#define PHY_MUX1_PORT_TYPE__PORT_TYPE_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_dcoh_phy_mux2_dispdec
+//PHY_MUX2_PHY_MUX_CONTROL
+#define PHY_MUX2_PHY_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                         0x0
+#define PHY_MUX2_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL__SHIFT                                                     0x4
+#define PHY_MUX2_PHY_MUX_CONTROL__HPO_DP_ENC_SEL__SHIFT                                                       0x8
+#define PHY_MUX2_PHY_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                           0x00000003L
+#define PHY_MUX2_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL_MASK                                                       0x00000070L
+#define PHY_MUX2_PHY_MUX_CONTROL__HPO_DP_ENC_SEL_MASK                                                         0x00000700L
+//PHY_MUX2_PORT_TYPE
+#define PHY_MUX2_PORT_TYPE__PORT_TYPE__SHIFT                                                                  0x0
+#define PHY_MUX2_PORT_TYPE__PORT_TYPE_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_dcoh_phy_mux3_dispdec
+//PHY_MUX3_PHY_MUX_CONTROL
+#define PHY_MUX3_PHY_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                         0x0
+#define PHY_MUX3_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL__SHIFT                                                     0x4
+#define PHY_MUX3_PHY_MUX_CONTROL__HPO_DP_ENC_SEL__SHIFT                                                       0x8
+#define PHY_MUX3_PHY_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                           0x00000003L
+#define PHY_MUX3_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL_MASK                                                       0x00000070L
+#define PHY_MUX3_PHY_MUX_CONTROL__HPO_DP_ENC_SEL_MASK                                                         0x00000700L
+//PHY_MUX3_PORT_TYPE
+#define PHY_MUX3_PORT_TYPE__PORT_TYPE__SHIFT                                                                  0x0
+#define PHY_MUX3_PORT_TYPE__PORT_TYPE_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_dcoh_phy_mux4_dispdec
+//PHY_MUX4_PHY_MUX_CONTROL
+#define PHY_MUX4_PHY_MUX_CONTROL__ENC_TYPE_SEL__SHIFT                                                         0x0
+#define PHY_MUX4_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL__SHIFT                                                     0x4
+#define PHY_MUX4_PHY_MUX_CONTROL__HPO_DP_ENC_SEL__SHIFT                                                       0x8
+#define PHY_MUX4_PHY_MUX_CONTROL__ENC_TYPE_SEL_MASK                                                           0x00000003L
+#define PHY_MUX4_PHY_MUX_CONTROL__HPO_HDMI_ENC_SEL_MASK                                                       0x00000070L
+#define PHY_MUX4_PHY_MUX_CONTROL__HPO_DP_ENC_SEL_MASK                                                         0x00000700L
+//PHY_MUX4_PORT_TYPE
+#define PHY_MUX4_PORT_TYPE__PORT_TYPE__SHIFT                                                                  0x0
+#define PHY_MUX4_PORT_TYPE__PORT_TYPE_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_dcoh_dcoh_top_dispdec
+//DCOH_TOP_CLOCK_CONTROL
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_DISPCLK_G_GATE_DIS__SHIFT                                                0x0
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_DISPCLK_R_GATE_DIS__SHIFT                                                0x4
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_REFCLK_G_GATE_DIS__SHIFT                                                 0x8
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_REFCLK_R_GATE_DIS__SHIFT                                                 0xc
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_FGCG_REP_DIS__SHIFT                                                      0x10
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_DISPCLK_G_GATE_DIS_MASK                                                  0x00000001L
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_DISPCLK_R_GATE_DIS_MASK                                                  0x00000010L
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_REFCLK_G_GATE_DIS_MASK                                                   0x00000100L
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_REFCLK_R_GATE_DIS_MASK                                                   0x00001000L
+#define DCOH_TOP_CLOCK_CONTROL__DCOH_FGCG_REP_DIS_MASK                                                        0x00010000L
+//DCOH_DCN_STATUS
+#define DCOH_DCN_STATUS__DCN_ACTIVE__SHIFT                                                                    0x0
+#define DCOH_DCN_STATUS__DCN_ACTIVE_MASK                                                                      0x00000001L
+//DCOH_TOP_SPARE
+#define DCOH_TOP_SPARE__DCOH_TOP_SPARE__SHIFT                                                                 0x0
+#define DCOH_TOP_SPARE__DCOH_TOP_SPARE_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_fmt0_dispdec
+//FMT0_FMT_CLAMP_COMPONENT_R
+#define FMT0_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R__SHIFT                                                  0x0
+#define FMT0_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R__SHIFT                                                  0x10
+#define FMT0_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R_MASK                                                    0x0000FFFFL
+#define FMT0_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R_MASK                                                    0xFFFF0000L
+//FMT0_FMT_CLAMP_COMPONENT_G
+#define FMT0_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G__SHIFT                                                  0x0
+#define FMT0_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G__SHIFT                                                  0x10
+#define FMT0_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G_MASK                                                    0x0000FFFFL
+#define FMT0_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G_MASK                                                    0xFFFF0000L
+//FMT0_FMT_CLAMP_COMPONENT_B
+#define FMT0_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B__SHIFT                                                  0x0
+#define FMT0_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B__SHIFT                                                  0x10
+#define FMT0_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B_MASK                                                    0x0000FFFFL
+#define FMT0_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B_MASK                                                    0xFFFF0000L
+//FMT0_FMT_DYNAMIC_EXP_CNTL
+#define FMT0_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN__SHIFT                                                  0x0
+#define FMT0_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE__SHIFT                                                0x4
+#define FMT0_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN_MASK                                                    0x00000001L
+#define FMT0_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE_MASK                                                  0x00000010L
+//FMT0_FMT_CONTROL
+#define FMT0_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE__SHIFT                                                      0x0
+#define FMT0_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX__SHIFT                                         0x8
+#define FMT0_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP__SHIFT                                    0xc
+#define FMT0_FMT_CONTROL__FMT_PIXEL_ENCODING__SHIFT                                                           0x10
+#define FMT0_FMT_CONTROL__FMT_SUBSAMPLING_MODE__SHIFT                                                         0x12
+#define FMT0_FMT_CONTROL__FMT_SUBSAMPLING_ORDER__SHIFT                                                        0x14
+#define FMT0_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS__SHIFT                                                0x15
+#define FMT0_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                         0x18
+#define FMT0_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE_MASK                                                        0x00000001L
+#define FMT0_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX_MASK                                           0x00000F00L
+#define FMT0_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP_MASK                                      0x00003000L
+#define FMT0_FMT_CONTROL__FMT_PIXEL_ENCODING_MASK                                                             0x00030000L
+#define FMT0_FMT_CONTROL__FMT_SUBSAMPLING_MODE_MASK                                                           0x000C0000L
+#define FMT0_FMT_CONTROL__FMT_SUBSAMPLING_ORDER_MASK                                                          0x00100000L
+#define FMT0_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS_MASK                                                  0x00200000L
+#define FMT0_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                           0x01000000L
+//FMT0_FMT_BIT_DEPTH_CONTROL
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN__SHIFT                                                    0x0
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE__SHIFT                                                  0x1
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH__SHIFT                                                 0x4
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN__SHIFT                                              0x8
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE__SHIFT                                            0x9
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH__SHIFT                                           0xb
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE__SHIFT                                            0xd
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE__SHIFT                                              0xe
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE__SHIFT                                         0xf
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN__SHIFT                                             0x10
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH__SHIFT                                          0x11
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET__SHIFT                                         0x15
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL__SHIFT                                                 0x18
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET__SHIFT                                          0x19
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL__SHIFT                                                      0x1a
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL__SHIFT                                                      0x1c
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL__SHIFT                                                      0x1e
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK                                                      0x00000001L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE_MASK                                                    0x00000002L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK                                                   0x00000030L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK                                                0x00000100L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE_MASK                                              0x00000600L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK                                             0x00001800L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK                                              0x00002000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK                                                0x00004000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK                                           0x00008000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN_MASK                                               0x00010000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH_MASK                                            0x00060000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET_MASK                                           0x00600000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL_MASK                                                   0x01000000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET_MASK                                            0x02000000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL_MASK                                                        0x0C000000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL_MASK                                                        0x30000000L
+#define FMT0_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL_MASK                                                        0xC0000000L
+//FMT0_FMT_DITHER_RAND_R_SEED
+#define FMT0_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED__SHIFT                                                   0x0
+#define FMT0_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR__SHIFT                                                   0x10
+#define FMT0_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED_MASK                                                     0x000000FFL
+#define FMT0_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR_MASK                                                     0xFFFF0000L
+//FMT0_FMT_DITHER_RAND_G_SEED
+#define FMT0_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED__SHIFT                                                   0x0
+#define FMT0_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y__SHIFT                                                    0x10
+#define FMT0_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED_MASK                                                     0x000000FFL
+#define FMT0_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y_MASK                                                      0xFFFF0000L
+//FMT0_FMT_DITHER_RAND_B_SEED
+#define FMT0_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED__SHIFT                                                   0x0
+#define FMT0_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB__SHIFT                                                   0x10
+#define FMT0_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED_MASK                                                     0x000000FFL
+#define FMT0_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB_MASK                                                     0xFFFF0000L
+//FMT0_FMT_CLAMP_CNTL
+#define FMT0_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN__SHIFT                                                         0x0
+#define FMT0_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT__SHIFT                                                    0x10
+#define FMT0_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN_MASK                                                           0x00000001L
+#define FMT0_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT_MASK                                                      0x00070000L
+//FMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL
+#define FMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH__SHIFT                     0x0
+#define FMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH_MASK                       0x00001FFFL
+//FMT0_FMT_MAP420_MEMORY_CONTROL
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE__SHIFT                                        0x0
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS__SHIFT                                          0x4
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE__SHIFT                                        0x8
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0xc
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE_MASK                                          0x00000003L
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS_MASK                                            0x00000010L
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE_MASK                                          0x00000300L
+#define FMT0_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00003000L
+//FMT0_FMT_422_CONTROL
+#define FMT0_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT__SHIFT                                          0x0
+#define FMT0_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT_MASK                                            0x00000001L
+
+
+// addressBlock: dce_dc_opp_dpg0_dispdec
+//DPG0_DPG_CONTROL
+#define DPG0_DPG_CONTROL__DPG_EN__SHIFT                                                                       0x0
+#define DPG0_DPG_CONTROL__DPG_MODE__SHIFT                                                                     0x4
+#define DPG0_DPG_CONTROL__DPG_DYNAMIC_RANGE__SHIFT                                                            0x8
+#define DPG0_DPG_CONTROL__DPG_BIT_DEPTH__SHIFT                                                                0xc
+#define DPG0_DPG_CONTROL__DPG_VRES__SHIFT                                                                     0x10
+#define DPG0_DPG_CONTROL__DPG_HRES__SHIFT                                                                     0x14
+#define DPG0_DPG_CONTROL__DPG_FIELD_POLARITY__SHIFT                                                           0x18
+#define DPG0_DPG_CONTROL__DPG_EN_MASK                                                                         0x00000001L
+#define DPG0_DPG_CONTROL__DPG_MODE_MASK                                                                       0x00000070L
+#define DPG0_DPG_CONTROL__DPG_DYNAMIC_RANGE_MASK                                                              0x00000100L
+#define DPG0_DPG_CONTROL__DPG_BIT_DEPTH_MASK                                                                  0x00003000L
+#define DPG0_DPG_CONTROL__DPG_VRES_MASK                                                                       0x000F0000L
+#define DPG0_DPG_CONTROL__DPG_HRES_MASK                                                                       0x00F00000L
+#define DPG0_DPG_CONTROL__DPG_FIELD_POLARITY_MASK                                                             0x01000000L
+//DPG0_DPG_RAMP_CONTROL
+#define DPG0_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET__SHIFT                                                        0x0
+#define DPG0_DPG_RAMP_CONTROL__DPG_INC0__SHIFT                                                                0x18
+#define DPG0_DPG_RAMP_CONTROL__DPG_INC1__SHIFT                                                                0x1c
+#define DPG0_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET_MASK                                                          0x0000FFFFL
+#define DPG0_DPG_RAMP_CONTROL__DPG_INC0_MASK                                                                  0x0F000000L
+#define DPG0_DPG_RAMP_CONTROL__DPG_INC1_MASK                                                                  0xF0000000L
+//DPG0_DPG_DIMENSIONS
+#define DPG0_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT__SHIFT                                                         0x0
+#define DPG0_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH__SHIFT                                                          0x10
+#define DPG0_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT_MASK                                                           0x00003FFFL
+#define DPG0_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH_MASK                                                            0x3FFF0000L
+//DPG0_DPG_COLOUR_R_CR
+#define DPG0_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR__SHIFT                                                         0x0
+#define DPG0_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR__SHIFT                                                         0x10
+#define DPG0_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR_MASK                                                           0x0000FFFFL
+#define DPG0_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR_MASK                                                           0xFFFF0000L
+//DPG0_DPG_COLOUR_G_Y
+#define DPG0_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y__SHIFT                                                           0x0
+#define DPG0_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y__SHIFT                                                           0x10
+#define DPG0_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y_MASK                                                             0x0000FFFFL
+#define DPG0_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y_MASK                                                             0xFFFF0000L
+//DPG0_DPG_COLOUR_B_CB
+#define DPG0_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB__SHIFT                                                         0x0
+#define DPG0_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB__SHIFT                                                         0x10
+#define DPG0_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB_MASK                                                           0x0000FFFFL
+#define DPG0_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB_MASK                                                           0xFFFF0000L
+//DPG0_DPG_OFFSET_SEGMENT
+#define DPG0_DPG_OFFSET_SEGMENT__DPG_X_OFFSET__SHIFT                                                          0x0
+#define DPG0_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH__SHIFT                                                     0x10
+#define DPG0_DPG_OFFSET_SEGMENT__DPG_X_OFFSET_MASK                                                            0x00003FFFL
+#define DPG0_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH_MASK                                                       0x3FFF0000L
+//DPG0_DPG_STATUS
+#define DPG0_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING__SHIFT                                                     0x0
+#define DPG0_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING_MASK                                                       0x00000001L
+
+
+// addressBlock: dce_dc_opp_oppbuf0_dispdec
+//OPPBUF0_OPPBUF_CONTROL
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH__SHIFT                                                    0x0
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION__SHIFT                                            0x10
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM__SHIFT                                               0x14
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION__SHIFT                                                0x18
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING__SHIFT                                           0x1c
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH_MASK                                                      0x00003FFFL
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION_MASK                                              0x00070000L
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM_MASK                                                 0x00F00000L
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION_MASK                                                  0x0F000000L
+#define OPPBUF0_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING_MASK                                             0x10000000L
+//OPPBUF0_OPPBUF_3D_PARAMETERS_0
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE__SHIFT                                     0x0
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE__SHIFT                                     0xa
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R__SHIFT                                            0x14
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE_MASK                                       0x000003FFL
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE_MASK                                       0x000FFC00L
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R_MASK                                              0xFFF00000L
+//OPPBUF0_OPPBUF_3D_PARAMETERS_1
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G__SHIFT                                            0x0
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B__SHIFT                                            0x10
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G_MASK                                              0x00000FFFL
+#define OPPBUF0_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B_MASK                                              0x0FFF0000L
+//OPPBUF0_OPPBUF_CONTROL1
+#define OPPBUF0_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS__SHIFT                                      0x0
+#define OPPBUF0_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS_MASK                                        0x00000007L
+
+
+// addressBlock: dce_dc_opp_opp_pipe0_dispdec
+//OPP_PIPE0_OPP_PIPE_CONTROL
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN__SHIFT                                                  0x0
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON__SHIFT                                                  0x1
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN__SHIFT                                         0x4
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN_MASK                                                    0x00000001L
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON_MASK                                                    0x00000002L
+#define OPP_PIPE0_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN_MASK                                           0x00000010L
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc0_dispdec
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN__SHIFT                                            0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN__SHIFT                                       0x4
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE__SHIFT                                   0x8
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN__SHIFT                                     0xa
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE__SHIFT                                0xc
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN__SHIFT                                  0xe
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT__SHIFT                                  0x14
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT__SHIFT                                 0x18
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING__SHIFT                              0x1c
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN_MASK                                              0x00000001L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN_MASK                                         0x00000010L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE_MASK                                     0x00000300L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN_MASK                                       0x00000400L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE_MASK                                  0x00003000L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN_MASK                                    0x00004000L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT_MASK                                    0x00300000L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT_MASK                                   0x01000000L
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING_MASK                                0x10000000L
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_MASK
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK__SHIFT                                             0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK_MASK                                               0x0000FFFFL
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTA
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A__SHIFT                                      0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTR
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R__SHIFT                                      0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTG
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G__SHIFT                                      0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTB
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B__SHIFT                                      0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTC
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C__SHIFT                                      0x0
+#define OPP_PIPE_CRC0_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_fmt1_dispdec
+//FMT1_FMT_CLAMP_COMPONENT_R
+#define FMT1_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R__SHIFT                                                  0x0
+#define FMT1_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R__SHIFT                                                  0x10
+#define FMT1_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R_MASK                                                    0x0000FFFFL
+#define FMT1_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R_MASK                                                    0xFFFF0000L
+//FMT1_FMT_CLAMP_COMPONENT_G
+#define FMT1_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G__SHIFT                                                  0x0
+#define FMT1_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G__SHIFT                                                  0x10
+#define FMT1_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G_MASK                                                    0x0000FFFFL
+#define FMT1_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G_MASK                                                    0xFFFF0000L
+//FMT1_FMT_CLAMP_COMPONENT_B
+#define FMT1_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B__SHIFT                                                  0x0
+#define FMT1_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B__SHIFT                                                  0x10
+#define FMT1_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B_MASK                                                    0x0000FFFFL
+#define FMT1_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B_MASK                                                    0xFFFF0000L
+//FMT1_FMT_DYNAMIC_EXP_CNTL
+#define FMT1_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN__SHIFT                                                  0x0
+#define FMT1_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE__SHIFT                                                0x4
+#define FMT1_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN_MASK                                                    0x00000001L
+#define FMT1_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE_MASK                                                  0x00000010L
+//FMT1_FMT_CONTROL
+#define FMT1_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE__SHIFT                                                      0x0
+#define FMT1_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX__SHIFT                                         0x8
+#define FMT1_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP__SHIFT                                    0xc
+#define FMT1_FMT_CONTROL__FMT_PIXEL_ENCODING__SHIFT                                                           0x10
+#define FMT1_FMT_CONTROL__FMT_SUBSAMPLING_MODE__SHIFT                                                         0x12
+#define FMT1_FMT_CONTROL__FMT_SUBSAMPLING_ORDER__SHIFT                                                        0x14
+#define FMT1_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS__SHIFT                                                0x15
+#define FMT1_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                         0x18
+#define FMT1_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE_MASK                                                        0x00000001L
+#define FMT1_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX_MASK                                           0x00000F00L
+#define FMT1_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP_MASK                                      0x00003000L
+#define FMT1_FMT_CONTROL__FMT_PIXEL_ENCODING_MASK                                                             0x00030000L
+#define FMT1_FMT_CONTROL__FMT_SUBSAMPLING_MODE_MASK                                                           0x000C0000L
+#define FMT1_FMT_CONTROL__FMT_SUBSAMPLING_ORDER_MASK                                                          0x00100000L
+#define FMT1_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS_MASK                                                  0x00200000L
+#define FMT1_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                           0x01000000L
+//FMT1_FMT_BIT_DEPTH_CONTROL
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN__SHIFT                                                    0x0
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE__SHIFT                                                  0x1
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH__SHIFT                                                 0x4
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN__SHIFT                                              0x8
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE__SHIFT                                            0x9
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH__SHIFT                                           0xb
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE__SHIFT                                            0xd
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE__SHIFT                                              0xe
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE__SHIFT                                         0xf
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN__SHIFT                                             0x10
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH__SHIFT                                          0x11
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET__SHIFT                                         0x15
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL__SHIFT                                                 0x18
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET__SHIFT                                          0x19
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL__SHIFT                                                      0x1a
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL__SHIFT                                                      0x1c
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL__SHIFT                                                      0x1e
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK                                                      0x00000001L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE_MASK                                                    0x00000002L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK                                                   0x00000030L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK                                                0x00000100L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE_MASK                                              0x00000600L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK                                             0x00001800L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK                                              0x00002000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK                                                0x00004000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK                                           0x00008000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN_MASK                                               0x00010000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH_MASK                                            0x00060000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET_MASK                                           0x00600000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL_MASK                                                   0x01000000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET_MASK                                            0x02000000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL_MASK                                                        0x0C000000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL_MASK                                                        0x30000000L
+#define FMT1_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL_MASK                                                        0xC0000000L
+//FMT1_FMT_DITHER_RAND_R_SEED
+#define FMT1_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED__SHIFT                                                   0x0
+#define FMT1_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR__SHIFT                                                   0x10
+#define FMT1_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED_MASK                                                     0x000000FFL
+#define FMT1_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR_MASK                                                     0xFFFF0000L
+//FMT1_FMT_DITHER_RAND_G_SEED
+#define FMT1_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED__SHIFT                                                   0x0
+#define FMT1_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y__SHIFT                                                    0x10
+#define FMT1_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED_MASK                                                     0x000000FFL
+#define FMT1_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y_MASK                                                      0xFFFF0000L
+//FMT1_FMT_DITHER_RAND_B_SEED
+#define FMT1_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED__SHIFT                                                   0x0
+#define FMT1_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB__SHIFT                                                   0x10
+#define FMT1_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED_MASK                                                     0x000000FFL
+#define FMT1_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB_MASK                                                     0xFFFF0000L
+//FMT1_FMT_CLAMP_CNTL
+#define FMT1_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN__SHIFT                                                         0x0
+#define FMT1_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT__SHIFT                                                    0x10
+#define FMT1_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN_MASK                                                           0x00000001L
+#define FMT1_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT_MASK                                                      0x00070000L
+//FMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL
+#define FMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH__SHIFT                     0x0
+#define FMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH_MASK                       0x00001FFFL
+//FMT1_FMT_MAP420_MEMORY_CONTROL
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE__SHIFT                                        0x0
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS__SHIFT                                          0x4
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE__SHIFT                                        0x8
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0xc
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE_MASK                                          0x00000003L
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS_MASK                                            0x00000010L
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE_MASK                                          0x00000300L
+#define FMT1_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00003000L
+//FMT1_FMT_422_CONTROL
+#define FMT1_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT__SHIFT                                          0x0
+#define FMT1_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT_MASK                                            0x00000001L
+
+
+// addressBlock: dce_dc_opp_dpg1_dispdec
+//DPG1_DPG_CONTROL
+#define DPG1_DPG_CONTROL__DPG_EN__SHIFT                                                                       0x0
+#define DPG1_DPG_CONTROL__DPG_MODE__SHIFT                                                                     0x4
+#define DPG1_DPG_CONTROL__DPG_DYNAMIC_RANGE__SHIFT                                                            0x8
+#define DPG1_DPG_CONTROL__DPG_BIT_DEPTH__SHIFT                                                                0xc
+#define DPG1_DPG_CONTROL__DPG_VRES__SHIFT                                                                     0x10
+#define DPG1_DPG_CONTROL__DPG_HRES__SHIFT                                                                     0x14
+#define DPG1_DPG_CONTROL__DPG_FIELD_POLARITY__SHIFT                                                           0x18
+#define DPG1_DPG_CONTROL__DPG_EN_MASK                                                                         0x00000001L
+#define DPG1_DPG_CONTROL__DPG_MODE_MASK                                                                       0x00000070L
+#define DPG1_DPG_CONTROL__DPG_DYNAMIC_RANGE_MASK                                                              0x00000100L
+#define DPG1_DPG_CONTROL__DPG_BIT_DEPTH_MASK                                                                  0x00003000L
+#define DPG1_DPG_CONTROL__DPG_VRES_MASK                                                                       0x000F0000L
+#define DPG1_DPG_CONTROL__DPG_HRES_MASK                                                                       0x00F00000L
+#define DPG1_DPG_CONTROL__DPG_FIELD_POLARITY_MASK                                                             0x01000000L
+//DPG1_DPG_RAMP_CONTROL
+#define DPG1_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET__SHIFT                                                        0x0
+#define DPG1_DPG_RAMP_CONTROL__DPG_INC0__SHIFT                                                                0x18
+#define DPG1_DPG_RAMP_CONTROL__DPG_INC1__SHIFT                                                                0x1c
+#define DPG1_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET_MASK                                                          0x0000FFFFL
+#define DPG1_DPG_RAMP_CONTROL__DPG_INC0_MASK                                                                  0x0F000000L
+#define DPG1_DPG_RAMP_CONTROL__DPG_INC1_MASK                                                                  0xF0000000L
+//DPG1_DPG_DIMENSIONS
+#define DPG1_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT__SHIFT                                                         0x0
+#define DPG1_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH__SHIFT                                                          0x10
+#define DPG1_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT_MASK                                                           0x00003FFFL
+#define DPG1_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH_MASK                                                            0x3FFF0000L
+//DPG1_DPG_COLOUR_R_CR
+#define DPG1_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR__SHIFT                                                         0x0
+#define DPG1_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR__SHIFT                                                         0x10
+#define DPG1_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR_MASK                                                           0x0000FFFFL
+#define DPG1_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR_MASK                                                           0xFFFF0000L
+//DPG1_DPG_COLOUR_G_Y
+#define DPG1_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y__SHIFT                                                           0x0
+#define DPG1_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y__SHIFT                                                           0x10
+#define DPG1_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y_MASK                                                             0x0000FFFFL
+#define DPG1_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y_MASK                                                             0xFFFF0000L
+//DPG1_DPG_COLOUR_B_CB
+#define DPG1_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB__SHIFT                                                         0x0
+#define DPG1_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB__SHIFT                                                         0x10
+#define DPG1_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB_MASK                                                           0x0000FFFFL
+#define DPG1_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB_MASK                                                           0xFFFF0000L
+//DPG1_DPG_OFFSET_SEGMENT
+#define DPG1_DPG_OFFSET_SEGMENT__DPG_X_OFFSET__SHIFT                                                          0x0
+#define DPG1_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH__SHIFT                                                     0x10
+#define DPG1_DPG_OFFSET_SEGMENT__DPG_X_OFFSET_MASK                                                            0x00003FFFL
+#define DPG1_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH_MASK                                                       0x3FFF0000L
+//DPG1_DPG_STATUS
+#define DPG1_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING__SHIFT                                                     0x0
+#define DPG1_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING_MASK                                                       0x00000001L
+
+
+// addressBlock: dce_dc_opp_oppbuf1_dispdec
+//OPPBUF1_OPPBUF_CONTROL
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH__SHIFT                                                    0x0
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION__SHIFT                                            0x10
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM__SHIFT                                               0x14
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION__SHIFT                                                0x18
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING__SHIFT                                           0x1c
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH_MASK                                                      0x00003FFFL
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION_MASK                                              0x00070000L
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM_MASK                                                 0x00F00000L
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION_MASK                                                  0x0F000000L
+#define OPPBUF1_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING_MASK                                             0x10000000L
+//OPPBUF1_OPPBUF_3D_PARAMETERS_0
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE__SHIFT                                     0x0
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE__SHIFT                                     0xa
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R__SHIFT                                            0x14
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE_MASK                                       0x000003FFL
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE_MASK                                       0x000FFC00L
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R_MASK                                              0xFFF00000L
+//OPPBUF1_OPPBUF_3D_PARAMETERS_1
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G__SHIFT                                            0x0
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B__SHIFT                                            0x10
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G_MASK                                              0x00000FFFL
+#define OPPBUF1_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B_MASK                                              0x0FFF0000L
+//OPPBUF1_OPPBUF_CONTROL1
+#define OPPBUF1_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS__SHIFT                                      0x0
+#define OPPBUF1_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS_MASK                                        0x00000007L
+
+
+// addressBlock: dce_dc_opp_opp_pipe1_dispdec
+//OPP_PIPE1_OPP_PIPE_CONTROL
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN__SHIFT                                                  0x0
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON__SHIFT                                                  0x1
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN__SHIFT                                         0x4
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN_MASK                                                    0x00000001L
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON_MASK                                                    0x00000002L
+#define OPP_PIPE1_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN_MASK                                           0x00000010L
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc1_dispdec
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN__SHIFT                                            0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN__SHIFT                                       0x4
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE__SHIFT                                   0x8
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN__SHIFT                                     0xa
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE__SHIFT                                0xc
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN__SHIFT                                  0xe
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT__SHIFT                                  0x14
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT__SHIFT                                 0x18
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING__SHIFT                              0x1c
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN_MASK                                              0x00000001L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN_MASK                                         0x00000010L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE_MASK                                     0x00000300L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN_MASK                                       0x00000400L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE_MASK                                  0x00003000L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN_MASK                                    0x00004000L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT_MASK                                    0x00300000L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT_MASK                                   0x01000000L
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING_MASK                                0x10000000L
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_MASK
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK__SHIFT                                             0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK_MASK                                               0x0000FFFFL
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTA
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A__SHIFT                                      0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTR
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R__SHIFT                                      0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTG
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G__SHIFT                                      0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTB
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B__SHIFT                                      0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTC
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C__SHIFT                                      0x0
+#define OPP_PIPE_CRC1_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_fmt2_dispdec
+//FMT2_FMT_CLAMP_COMPONENT_R
+#define FMT2_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R__SHIFT                                                  0x0
+#define FMT2_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R__SHIFT                                                  0x10
+#define FMT2_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R_MASK                                                    0x0000FFFFL
+#define FMT2_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R_MASK                                                    0xFFFF0000L
+//FMT2_FMT_CLAMP_COMPONENT_G
+#define FMT2_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G__SHIFT                                                  0x0
+#define FMT2_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G__SHIFT                                                  0x10
+#define FMT2_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G_MASK                                                    0x0000FFFFL
+#define FMT2_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G_MASK                                                    0xFFFF0000L
+//FMT2_FMT_CLAMP_COMPONENT_B
+#define FMT2_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B__SHIFT                                                  0x0
+#define FMT2_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B__SHIFT                                                  0x10
+#define FMT2_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B_MASK                                                    0x0000FFFFL
+#define FMT2_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B_MASK                                                    0xFFFF0000L
+//FMT2_FMT_DYNAMIC_EXP_CNTL
+#define FMT2_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN__SHIFT                                                  0x0
+#define FMT2_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE__SHIFT                                                0x4
+#define FMT2_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN_MASK                                                    0x00000001L
+#define FMT2_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE_MASK                                                  0x00000010L
+//FMT2_FMT_CONTROL
+#define FMT2_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE__SHIFT                                                      0x0
+#define FMT2_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX__SHIFT                                         0x8
+#define FMT2_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP__SHIFT                                    0xc
+#define FMT2_FMT_CONTROL__FMT_PIXEL_ENCODING__SHIFT                                                           0x10
+#define FMT2_FMT_CONTROL__FMT_SUBSAMPLING_MODE__SHIFT                                                         0x12
+#define FMT2_FMT_CONTROL__FMT_SUBSAMPLING_ORDER__SHIFT                                                        0x14
+#define FMT2_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS__SHIFT                                                0x15
+#define FMT2_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                         0x18
+#define FMT2_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE_MASK                                                        0x00000001L
+#define FMT2_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX_MASK                                           0x00000F00L
+#define FMT2_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP_MASK                                      0x00003000L
+#define FMT2_FMT_CONTROL__FMT_PIXEL_ENCODING_MASK                                                             0x00030000L
+#define FMT2_FMT_CONTROL__FMT_SUBSAMPLING_MODE_MASK                                                           0x000C0000L
+#define FMT2_FMT_CONTROL__FMT_SUBSAMPLING_ORDER_MASK                                                          0x00100000L
+#define FMT2_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS_MASK                                                  0x00200000L
+#define FMT2_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                           0x01000000L
+//FMT2_FMT_BIT_DEPTH_CONTROL
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN__SHIFT                                                    0x0
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE__SHIFT                                                  0x1
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH__SHIFT                                                 0x4
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN__SHIFT                                              0x8
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE__SHIFT                                            0x9
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH__SHIFT                                           0xb
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE__SHIFT                                            0xd
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE__SHIFT                                              0xe
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE__SHIFT                                         0xf
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN__SHIFT                                             0x10
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH__SHIFT                                          0x11
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET__SHIFT                                         0x15
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL__SHIFT                                                 0x18
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET__SHIFT                                          0x19
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL__SHIFT                                                      0x1a
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL__SHIFT                                                      0x1c
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL__SHIFT                                                      0x1e
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK                                                      0x00000001L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE_MASK                                                    0x00000002L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK                                                   0x00000030L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK                                                0x00000100L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE_MASK                                              0x00000600L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK                                             0x00001800L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK                                              0x00002000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK                                                0x00004000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK                                           0x00008000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN_MASK                                               0x00010000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH_MASK                                            0x00060000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET_MASK                                           0x00600000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL_MASK                                                   0x01000000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET_MASK                                            0x02000000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL_MASK                                                        0x0C000000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL_MASK                                                        0x30000000L
+#define FMT2_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL_MASK                                                        0xC0000000L
+//FMT2_FMT_DITHER_RAND_R_SEED
+#define FMT2_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED__SHIFT                                                   0x0
+#define FMT2_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR__SHIFT                                                   0x10
+#define FMT2_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED_MASK                                                     0x000000FFL
+#define FMT2_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR_MASK                                                     0xFFFF0000L
+//FMT2_FMT_DITHER_RAND_G_SEED
+#define FMT2_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED__SHIFT                                                   0x0
+#define FMT2_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y__SHIFT                                                    0x10
+#define FMT2_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED_MASK                                                     0x000000FFL
+#define FMT2_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y_MASK                                                      0xFFFF0000L
+//FMT2_FMT_DITHER_RAND_B_SEED
+#define FMT2_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED__SHIFT                                                   0x0
+#define FMT2_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB__SHIFT                                                   0x10
+#define FMT2_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED_MASK                                                     0x000000FFL
+#define FMT2_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB_MASK                                                     0xFFFF0000L
+//FMT2_FMT_CLAMP_CNTL
+#define FMT2_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN__SHIFT                                                         0x0
+#define FMT2_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT__SHIFT                                                    0x10
+#define FMT2_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN_MASK                                                           0x00000001L
+#define FMT2_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT_MASK                                                      0x00070000L
+//FMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL
+#define FMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH__SHIFT                     0x0
+#define FMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH_MASK                       0x00001FFFL
+//FMT2_FMT_MAP420_MEMORY_CONTROL
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE__SHIFT                                        0x0
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS__SHIFT                                          0x4
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE__SHIFT                                        0x8
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0xc
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE_MASK                                          0x00000003L
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS_MASK                                            0x00000010L
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE_MASK                                          0x00000300L
+#define FMT2_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00003000L
+//FMT2_FMT_422_CONTROL
+#define FMT2_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT__SHIFT                                          0x0
+#define FMT2_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT_MASK                                            0x00000001L
+
+
+// addressBlock: dce_dc_opp_dpg2_dispdec
+//DPG2_DPG_CONTROL
+#define DPG2_DPG_CONTROL__DPG_EN__SHIFT                                                                       0x0
+#define DPG2_DPG_CONTROL__DPG_MODE__SHIFT                                                                     0x4
+#define DPG2_DPG_CONTROL__DPG_DYNAMIC_RANGE__SHIFT                                                            0x8
+#define DPG2_DPG_CONTROL__DPG_BIT_DEPTH__SHIFT                                                                0xc
+#define DPG2_DPG_CONTROL__DPG_VRES__SHIFT                                                                     0x10
+#define DPG2_DPG_CONTROL__DPG_HRES__SHIFT                                                                     0x14
+#define DPG2_DPG_CONTROL__DPG_FIELD_POLARITY__SHIFT                                                           0x18
+#define DPG2_DPG_CONTROL__DPG_EN_MASK                                                                         0x00000001L
+#define DPG2_DPG_CONTROL__DPG_MODE_MASK                                                                       0x00000070L
+#define DPG2_DPG_CONTROL__DPG_DYNAMIC_RANGE_MASK                                                              0x00000100L
+#define DPG2_DPG_CONTROL__DPG_BIT_DEPTH_MASK                                                                  0x00003000L
+#define DPG2_DPG_CONTROL__DPG_VRES_MASK                                                                       0x000F0000L
+#define DPG2_DPG_CONTROL__DPG_HRES_MASK                                                                       0x00F00000L
+#define DPG2_DPG_CONTROL__DPG_FIELD_POLARITY_MASK                                                             0x01000000L
+//DPG2_DPG_RAMP_CONTROL
+#define DPG2_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET__SHIFT                                                        0x0
+#define DPG2_DPG_RAMP_CONTROL__DPG_INC0__SHIFT                                                                0x18
+#define DPG2_DPG_RAMP_CONTROL__DPG_INC1__SHIFT                                                                0x1c
+#define DPG2_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET_MASK                                                          0x0000FFFFL
+#define DPG2_DPG_RAMP_CONTROL__DPG_INC0_MASK                                                                  0x0F000000L
+#define DPG2_DPG_RAMP_CONTROL__DPG_INC1_MASK                                                                  0xF0000000L
+//DPG2_DPG_DIMENSIONS
+#define DPG2_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT__SHIFT                                                         0x0
+#define DPG2_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH__SHIFT                                                          0x10
+#define DPG2_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT_MASK                                                           0x00003FFFL
+#define DPG2_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH_MASK                                                            0x3FFF0000L
+//DPG2_DPG_COLOUR_R_CR
+#define DPG2_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR__SHIFT                                                         0x0
+#define DPG2_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR__SHIFT                                                         0x10
+#define DPG2_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR_MASK                                                           0x0000FFFFL
+#define DPG2_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR_MASK                                                           0xFFFF0000L
+//DPG2_DPG_COLOUR_G_Y
+#define DPG2_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y__SHIFT                                                           0x0
+#define DPG2_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y__SHIFT                                                           0x10
+#define DPG2_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y_MASK                                                             0x0000FFFFL
+#define DPG2_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y_MASK                                                             0xFFFF0000L
+//DPG2_DPG_COLOUR_B_CB
+#define DPG2_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB__SHIFT                                                         0x0
+#define DPG2_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB__SHIFT                                                         0x10
+#define DPG2_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB_MASK                                                           0x0000FFFFL
+#define DPG2_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB_MASK                                                           0xFFFF0000L
+//DPG2_DPG_OFFSET_SEGMENT
+#define DPG2_DPG_OFFSET_SEGMENT__DPG_X_OFFSET__SHIFT                                                          0x0
+#define DPG2_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH__SHIFT                                                     0x10
+#define DPG2_DPG_OFFSET_SEGMENT__DPG_X_OFFSET_MASK                                                            0x00003FFFL
+#define DPG2_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH_MASK                                                       0x3FFF0000L
+//DPG2_DPG_STATUS
+#define DPG2_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING__SHIFT                                                     0x0
+#define DPG2_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING_MASK                                                       0x00000001L
+
+
+// addressBlock: dce_dc_opp_oppbuf2_dispdec
+//OPPBUF2_OPPBUF_CONTROL
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH__SHIFT                                                    0x0
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION__SHIFT                                            0x10
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM__SHIFT                                               0x14
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION__SHIFT                                                0x18
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING__SHIFT                                           0x1c
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH_MASK                                                      0x00003FFFL
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION_MASK                                              0x00070000L
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM_MASK                                                 0x00F00000L
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION_MASK                                                  0x0F000000L
+#define OPPBUF2_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING_MASK                                             0x10000000L
+//OPPBUF2_OPPBUF_3D_PARAMETERS_0
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE__SHIFT                                     0x0
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE__SHIFT                                     0xa
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R__SHIFT                                            0x14
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE_MASK                                       0x000003FFL
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE_MASK                                       0x000FFC00L
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R_MASK                                              0xFFF00000L
+//OPPBUF2_OPPBUF_3D_PARAMETERS_1
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G__SHIFT                                            0x0
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B__SHIFT                                            0x10
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G_MASK                                              0x00000FFFL
+#define OPPBUF2_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B_MASK                                              0x0FFF0000L
+//OPPBUF2_OPPBUF_CONTROL1
+#define OPPBUF2_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS__SHIFT                                      0x0
+#define OPPBUF2_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS_MASK                                        0x00000007L
+
+
+// addressBlock: dce_dc_opp_opp_pipe2_dispdec
+//OPP_PIPE2_OPP_PIPE_CONTROL
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN__SHIFT                                                  0x0
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON__SHIFT                                                  0x1
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN__SHIFT                                         0x4
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN_MASK                                                    0x00000001L
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON_MASK                                                    0x00000002L
+#define OPP_PIPE2_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN_MASK                                           0x00000010L
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc2_dispdec
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN__SHIFT                                            0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN__SHIFT                                       0x4
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE__SHIFT                                   0x8
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN__SHIFT                                     0xa
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE__SHIFT                                0xc
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN__SHIFT                                  0xe
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT__SHIFT                                  0x14
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT__SHIFT                                 0x18
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING__SHIFT                              0x1c
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN_MASK                                              0x00000001L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN_MASK                                         0x00000010L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE_MASK                                     0x00000300L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN_MASK                                       0x00000400L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE_MASK                                  0x00003000L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN_MASK                                    0x00004000L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT_MASK                                    0x00300000L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT_MASK                                   0x01000000L
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING_MASK                                0x10000000L
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_MASK
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK__SHIFT                                             0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK_MASK                                               0x0000FFFFL
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTA
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A__SHIFT                                      0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTR
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R__SHIFT                                      0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTG
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G__SHIFT                                      0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTB
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B__SHIFT                                      0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTC
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C__SHIFT                                      0x0
+#define OPP_PIPE_CRC2_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_fmt3_dispdec
+//FMT3_FMT_CLAMP_COMPONENT_R
+#define FMT3_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R__SHIFT                                                  0x0
+#define FMT3_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R__SHIFT                                                  0x10
+#define FMT3_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_LOWER_R_MASK                                                    0x0000FFFFL
+#define FMT3_FMT_CLAMP_COMPONENT_R__FMT_CLAMP_UPPER_R_MASK                                                    0xFFFF0000L
+//FMT3_FMT_CLAMP_COMPONENT_G
+#define FMT3_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G__SHIFT                                                  0x0
+#define FMT3_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G__SHIFT                                                  0x10
+#define FMT3_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_LOWER_G_MASK                                                    0x0000FFFFL
+#define FMT3_FMT_CLAMP_COMPONENT_G__FMT_CLAMP_UPPER_G_MASK                                                    0xFFFF0000L
+//FMT3_FMT_CLAMP_COMPONENT_B
+#define FMT3_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B__SHIFT                                                  0x0
+#define FMT3_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B__SHIFT                                                  0x10
+#define FMT3_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_LOWER_B_MASK                                                    0x0000FFFFL
+#define FMT3_FMT_CLAMP_COMPONENT_B__FMT_CLAMP_UPPER_B_MASK                                                    0xFFFF0000L
+//FMT3_FMT_DYNAMIC_EXP_CNTL
+#define FMT3_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN__SHIFT                                                  0x0
+#define FMT3_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE__SHIFT                                                0x4
+#define FMT3_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN_MASK                                                    0x00000001L
+#define FMT3_FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE_MASK                                                  0x00000010L
+//FMT3_FMT_CONTROL
+#define FMT3_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE__SHIFT                                                      0x0
+#define FMT3_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX__SHIFT                                         0x8
+#define FMT3_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP__SHIFT                                    0xc
+#define FMT3_FMT_CONTROL__FMT_PIXEL_ENCODING__SHIFT                                                           0x10
+#define FMT3_FMT_CONTROL__FMT_SUBSAMPLING_MODE__SHIFT                                                         0x12
+#define FMT3_FMT_CONTROL__FMT_SUBSAMPLING_ORDER__SHIFT                                                        0x14
+#define FMT3_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS__SHIFT                                                0x15
+#define FMT3_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                         0x18
+#define FMT3_FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE_MASK                                                        0x00000001L
+#define FMT3_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX_MASK                                           0x00000F00L
+#define FMT3_FMT_CONTROL__FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP_MASK                                      0x00003000L
+#define FMT3_FMT_CONTROL__FMT_PIXEL_ENCODING_MASK                                                             0x00030000L
+#define FMT3_FMT_CONTROL__FMT_SUBSAMPLING_MODE_MASK                                                           0x000C0000L
+#define FMT3_FMT_CONTROL__FMT_SUBSAMPLING_ORDER_MASK                                                          0x00100000L
+#define FMT3_FMT_CONTROL__FMT_CBCR_BIT_REDUCTION_BYPASS_MASK                                                  0x00200000L
+#define FMT3_FMT_CONTROL__FMT_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                           0x01000000L
+//FMT3_FMT_BIT_DEPTH_CONTROL
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN__SHIFT                                                    0x0
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE__SHIFT                                                  0x1
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH__SHIFT                                                 0x4
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN__SHIFT                                              0x8
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE__SHIFT                                            0x9
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH__SHIFT                                           0xb
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE__SHIFT                                            0xd
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE__SHIFT                                              0xe
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE__SHIFT                                         0xf
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN__SHIFT                                             0x10
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH__SHIFT                                          0x11
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET__SHIFT                                         0x15
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL__SHIFT                                                 0x18
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET__SHIFT                                          0x19
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL__SHIFT                                                      0x1a
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL__SHIFT                                                      0x1c
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL__SHIFT                                                      0x1e
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK                                                      0x00000001L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_MODE_MASK                                                    0x00000002L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK                                                   0x00000030L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK                                                0x00000100L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE_MASK                                              0x00000600L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK                                             0x00001800L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK                                              0x00002000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK                                                0x00004000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK                                           0x00008000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN_MASK                                               0x00010000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH_MASK                                            0x00060000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET_MASK                                           0x00600000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL_MASK                                                   0x01000000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET_MASK                                            0x02000000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL_MASK                                                        0x0C000000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL_MASK                                                        0x30000000L
+#define FMT3_FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL_MASK                                                        0xC0000000L
+//FMT3_FMT_DITHER_RAND_R_SEED
+#define FMT3_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED__SHIFT                                                   0x0
+#define FMT3_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR__SHIFT                                                   0x10
+#define FMT3_FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED_MASK                                                     0x000000FFL
+#define FMT3_FMT_DITHER_RAND_R_SEED__FMT_OFFSET_R_CR_MASK                                                     0xFFFF0000L
+//FMT3_FMT_DITHER_RAND_G_SEED
+#define FMT3_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED__SHIFT                                                   0x0
+#define FMT3_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y__SHIFT                                                    0x10
+#define FMT3_FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED_MASK                                                     0x000000FFL
+#define FMT3_FMT_DITHER_RAND_G_SEED__FMT_OFFSET_G_Y_MASK                                                      0xFFFF0000L
+//FMT3_FMT_DITHER_RAND_B_SEED
+#define FMT3_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED__SHIFT                                                   0x0
+#define FMT3_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB__SHIFT                                                   0x10
+#define FMT3_FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED_MASK                                                     0x000000FFL
+#define FMT3_FMT_DITHER_RAND_B_SEED__FMT_OFFSET_B_CB_MASK                                                     0xFFFF0000L
+//FMT3_FMT_CLAMP_CNTL
+#define FMT3_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN__SHIFT                                                         0x0
+#define FMT3_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT__SHIFT                                                    0x10
+#define FMT3_FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN_MASK                                                           0x00000001L
+#define FMT3_FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT_MASK                                                      0x00070000L
+//FMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL
+#define FMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH__SHIFT                     0x0
+#define FMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL__FMT_SIDE_BY_SIDE_STEREO_ACTIVE_WIDTH_MASK                       0x00001FFFL
+//FMT3_FMT_MAP420_MEMORY_CONTROL
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE__SHIFT                                        0x0
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS__SHIFT                                          0x4
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE__SHIFT                                        0x8
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0xc
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_FORCE_MASK                                          0x00000003L
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_DIS_MASK                                            0x00000010L
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_MAP420MEM_PWR_STATE_MASK                                          0x00000300L
+#define FMT3_FMT_MAP420_MEMORY_CONTROL__FMT_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00003000L
+//FMT3_FMT_422_CONTROL
+#define FMT3_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT__SHIFT                                          0x0
+#define FMT3_FMT_422_CONTROL__FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT_MASK                                            0x00000001L
+
+
+// addressBlock: dce_dc_opp_dpg3_dispdec
+//DPG3_DPG_CONTROL
+#define DPG3_DPG_CONTROL__DPG_EN__SHIFT                                                                       0x0
+#define DPG3_DPG_CONTROL__DPG_MODE__SHIFT                                                                     0x4
+#define DPG3_DPG_CONTROL__DPG_DYNAMIC_RANGE__SHIFT                                                            0x8
+#define DPG3_DPG_CONTROL__DPG_BIT_DEPTH__SHIFT                                                                0xc
+#define DPG3_DPG_CONTROL__DPG_VRES__SHIFT                                                                     0x10
+#define DPG3_DPG_CONTROL__DPG_HRES__SHIFT                                                                     0x14
+#define DPG3_DPG_CONTROL__DPG_FIELD_POLARITY__SHIFT                                                           0x18
+#define DPG3_DPG_CONTROL__DPG_EN_MASK                                                                         0x00000001L
+#define DPG3_DPG_CONTROL__DPG_MODE_MASK                                                                       0x00000070L
+#define DPG3_DPG_CONTROL__DPG_DYNAMIC_RANGE_MASK                                                              0x00000100L
+#define DPG3_DPG_CONTROL__DPG_BIT_DEPTH_MASK                                                                  0x00003000L
+#define DPG3_DPG_CONTROL__DPG_VRES_MASK                                                                       0x000F0000L
+#define DPG3_DPG_CONTROL__DPG_HRES_MASK                                                                       0x00F00000L
+#define DPG3_DPG_CONTROL__DPG_FIELD_POLARITY_MASK                                                             0x01000000L
+//DPG3_DPG_RAMP_CONTROL
+#define DPG3_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET__SHIFT                                                        0x0
+#define DPG3_DPG_RAMP_CONTROL__DPG_INC0__SHIFT                                                                0x18
+#define DPG3_DPG_RAMP_CONTROL__DPG_INC1__SHIFT                                                                0x1c
+#define DPG3_DPG_RAMP_CONTROL__DPG_RAMP0_OFFSET_MASK                                                          0x0000FFFFL
+#define DPG3_DPG_RAMP_CONTROL__DPG_INC0_MASK                                                                  0x0F000000L
+#define DPG3_DPG_RAMP_CONTROL__DPG_INC1_MASK                                                                  0xF0000000L
+//DPG3_DPG_DIMENSIONS
+#define DPG3_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT__SHIFT                                                         0x0
+#define DPG3_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH__SHIFT                                                          0x10
+#define DPG3_DPG_DIMENSIONS__DPG_ACTIVE_HEIGHT_MASK                                                           0x00003FFFL
+#define DPG3_DPG_DIMENSIONS__DPG_ACTIVE_WIDTH_MASK                                                            0x3FFF0000L
+//DPG3_DPG_COLOUR_R_CR
+#define DPG3_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR__SHIFT                                                         0x0
+#define DPG3_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR__SHIFT                                                         0x10
+#define DPG3_DPG_COLOUR_R_CR__DPG_COLOUR0_R_CR_MASK                                                           0x0000FFFFL
+#define DPG3_DPG_COLOUR_R_CR__DPG_COLOUR1_R_CR_MASK                                                           0xFFFF0000L
+//DPG3_DPG_COLOUR_G_Y
+#define DPG3_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y__SHIFT                                                           0x0
+#define DPG3_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y__SHIFT                                                           0x10
+#define DPG3_DPG_COLOUR_G_Y__DPG_COLOUR0_G_Y_MASK                                                             0x0000FFFFL
+#define DPG3_DPG_COLOUR_G_Y__DPG_COLOUR1_G_Y_MASK                                                             0xFFFF0000L
+//DPG3_DPG_COLOUR_B_CB
+#define DPG3_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB__SHIFT                                                         0x0
+#define DPG3_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB__SHIFT                                                         0x10
+#define DPG3_DPG_COLOUR_B_CB__DPG_COLOUR0_B_CB_MASK                                                           0x0000FFFFL
+#define DPG3_DPG_COLOUR_B_CB__DPG_COLOUR1_B_CB_MASK                                                           0xFFFF0000L
+//DPG3_DPG_OFFSET_SEGMENT
+#define DPG3_DPG_OFFSET_SEGMENT__DPG_X_OFFSET__SHIFT                                                          0x0
+#define DPG3_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH__SHIFT                                                     0x10
+#define DPG3_DPG_OFFSET_SEGMENT__DPG_X_OFFSET_MASK                                                            0x00003FFFL
+#define DPG3_DPG_OFFSET_SEGMENT__DPG_SEGMENT_WIDTH_MASK                                                       0x3FFF0000L
+//DPG3_DPG_STATUS
+#define DPG3_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING__SHIFT                                                     0x0
+#define DPG3_DPG_STATUS__DPG_DOUBLE_BUFFER_PENDING_MASK                                                       0x00000001L
+
+
+// addressBlock: dce_dc_opp_oppbuf3_dispdec
+//OPPBUF3_OPPBUF_CONTROL
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH__SHIFT                                                    0x0
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION__SHIFT                                            0x10
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM__SHIFT                                               0x14
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION__SHIFT                                                0x18
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING__SHIFT                                           0x1c
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_ACTIVE_WIDTH_MASK                                                      0x00003FFFL
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_DISPLAY_SEGMENTATION_MASK                                              0x00070000L
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_OVERLAP_PIXEL_NUM_MASK                                                 0x00F00000L
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_PIXEL_REPETITION_MASK                                                  0x0F000000L
+#define OPPBUF3_OPPBUF_CONTROL__OPPBUF_DOUBLE_BUFFER_PENDING_MASK                                             0x10000000L
+//OPPBUF3_OPPBUF_3D_PARAMETERS_0
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE__SHIFT                                     0x0
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE__SHIFT                                     0xa
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R__SHIFT                                            0x14
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE1_SIZE_MASK                                       0x000003FFL
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_3D_VACT_SPACE2_SIZE_MASK                                       0x000FFC00L
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_0__OPPBUF_DUMMY_DATA_R_MASK                                              0xFFF00000L
+//OPPBUF3_OPPBUF_3D_PARAMETERS_1
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G__SHIFT                                            0x0
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B__SHIFT                                            0x10
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_G_MASK                                              0x00000FFFL
+#define OPPBUF3_OPPBUF_3D_PARAMETERS_1__OPPBUF_DUMMY_DATA_B_MASK                                              0x0FFF0000L
+//OPPBUF3_OPPBUF_CONTROL1
+#define OPPBUF3_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS__SHIFT                                      0x0
+#define OPPBUF3_OPPBUF_CONTROL1__OPPBUF_NUM_SEGMENT_PADDED_PIXELS_MASK                                        0x00000007L
+
+
+// addressBlock: dce_dc_opp_opp_pipe3_dispdec
+//OPP_PIPE3_OPP_PIPE_CONTROL
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN__SHIFT                                                  0x0
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON__SHIFT                                                  0x1
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN__SHIFT                                         0x4
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_EN_MASK                                                    0x00000001L
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_CLOCK_ON_MASK                                                    0x00000002L
+#define OPP_PIPE3_OPP_PIPE_CONTROL__OPP_PIPE_DIGITAL_BYPASS_EN_MASK                                           0x00000010L
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc3_dispdec
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN__SHIFT                                            0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN__SHIFT                                       0x4
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE__SHIFT                                   0x8
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN__SHIFT                                     0xa
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE__SHIFT                                0xc
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN__SHIFT                                  0xe
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT__SHIFT                                  0x14
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT__SHIFT                                 0x18
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING__SHIFT                              0x1c
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_EN_MASK                                              0x00000001L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_CONT_EN_MASK                                         0x00000010L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_MODE_MASK                                     0x00000300L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_STEREO_EN_MASK                                       0x00000400L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_MODE_MASK                                  0x00003000L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_INTERLACE_EN_MASK                                    0x00004000L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_PIXEL_SELECT_MASK                                    0x00300000L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_SOURCE_SELECT_MASK                                   0x01000000L
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL__OPP_PIPE_CRC_ONE_SHOT_PENDING_MASK                                0x10000000L
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_MASK
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK__SHIFT                                             0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_MASK__OPP_PIPE_CRC_MASK_MASK                                               0x0000FFFFL
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTA
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A__SHIFT                                      0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTA__OPP_PIPE_CRC_RESULT_A_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTR
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R__SHIFT                                      0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTR__OPP_PIPE_CRC_RESULT_R_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTG
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G__SHIFT                                      0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTG__OPP_PIPE_CRC_RESULT_G_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTB
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B__SHIFT                                      0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTB__OPP_PIPE_CRC_RESULT_B_MASK                                        0xFFFFFFFFL
+//OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTC
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C__SHIFT                                      0x0
+#define OPP_PIPE_CRC3_OPP_PIPE_CRC_RESULTC__OPP_PIPE_CRC_RESULT_C_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_opp_top_dispdec
+//OPP_TOP_CLK_CONTROL
+#define OPP_TOP_CLK_CONTROL__OPP_DISPCLK_R_GATE_DIS__SHIFT                                                    0x0
+#define OPP_TOP_CLK_CONTROL__OPP_DISPCLK_G_ABM_GATE_DIS__SHIFT                                                0x4
+#define OPP_TOP_CLK_CONTROL__OPP_TEST_CLK_SEL__SHIFT                                                          0x8
+#define OPP_TOP_CLK_CONTROL__OPP_ABM0_CLOCK_ON__SHIFT                                                         0xc
+#define OPP_TOP_CLK_CONTROL__OPP_ABM1_CLOCK_ON__SHIFT                                                         0xd
+#define OPP_TOP_CLK_CONTROL__OPP_ABM2_CLOCK_ON__SHIFT                                                         0xe
+#define OPP_TOP_CLK_CONTROL__OPP_ABM3_CLOCK_ON__SHIFT                                                         0xf
+#define OPP_TOP_CLK_CONTROL__OPP_FGCG_REP_DIS__SHIFT                                                          0x18
+#define OPP_TOP_CLK_CONTROL__OPP_DISPCLK_R_GATE_DIS_MASK                                                      0x00000001L
+#define OPP_TOP_CLK_CONTROL__OPP_DISPCLK_G_ABM_GATE_DIS_MASK                                                  0x00000010L
+#define OPP_TOP_CLK_CONTROL__OPP_TEST_CLK_SEL_MASK                                                            0x00000F00L
+#define OPP_TOP_CLK_CONTROL__OPP_ABM0_CLOCK_ON_MASK                                                           0x00001000L
+#define OPP_TOP_CLK_CONTROL__OPP_ABM1_CLOCK_ON_MASK                                                           0x00002000L
+#define OPP_TOP_CLK_CONTROL__OPP_ABM2_CLOCK_ON_MASK                                                           0x00004000L
+#define OPP_TOP_CLK_CONTROL__OPP_ABM3_CLOCK_ON_MASK                                                           0x00008000L
+#define OPP_TOP_CLK_CONTROL__OPP_FGCG_REP_DIS_MASK                                                            0x01000000L
+//OPP_ABM_CONTROL
+#define OPP_ABM_CONTROL__OPP_ABM_BLPWM_SEL__SHIFT                                                             0x0
+#define OPP_ABM_CONTROL__OPP_ABM_BLPWM_SEL_MASK                                                               0x00000007L
+
+
+// addressBlock: dce_dc_opp_dscrm0_dispdec
+//DSCRM0_DSCRM_DSC_FORWARD_CONFIG
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN__SHIFT                                          0x0
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE__SHIFT                                     0x4
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                    0x8
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS__SHIFT                                   0xc
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_MASK                                            0x00000001L
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE_MASK                                       0x00000070L
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                      0x00000100L
+#define DSCRM0_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS_MASK                                     0x00001000L
+
+
+// addressBlock: dce_dc_opp_dscrm1_dispdec
+//DSCRM1_DSCRM_DSC_FORWARD_CONFIG
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN__SHIFT                                          0x0
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE__SHIFT                                     0x4
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                    0x8
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS__SHIFT                                   0xc
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_MASK                                            0x00000001L
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE_MASK                                       0x00000070L
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                      0x00000100L
+#define DSCRM1_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS_MASK                                     0x00001000L
+
+
+// addressBlock: dce_dc_opp_dscrm2_dispdec
+//DSCRM2_DSCRM_DSC_FORWARD_CONFIG
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN__SHIFT                                          0x0
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE__SHIFT                                     0x4
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                    0x8
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS__SHIFT                                   0xc
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_MASK                                            0x00000001L
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE_MASK                                       0x00000070L
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                      0x00000100L
+#define DSCRM2_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS_MASK                                     0x00001000L
+
+
+// addressBlock: dce_dc_opp_dscrm3_dispdec
+//DSCRM3_DSCRM_DSC_FORWARD_CONFIG
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN__SHIFT                                          0x0
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE__SHIFT                                     0x4
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                    0x8
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS__SHIFT                                   0xc
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_MASK                                            0x00000001L
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_OPP_PIPE_SOURCE_MASK                                       0x00000070L
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                      0x00000100L
+#define DSCRM3_DSCRM_DSC_FORWARD_CONFIG__DSCRM_DSC_FORWARD_EN_STATUS_MASK                                     0x00001000L
+
+
+// addressBlock: dce_dc_opp_opp_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON14_PERFCOUNTER_CNTL
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON14_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON14_PERFCOUNTER_CNTL2
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON14_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON14_PERFCOUNTER_STATE
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON14_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON14_PERFMON_CNTL
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON14_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON14_PERFMON_CNTL2
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON14_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON14_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON14_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON14_PERFMON_CVALUE_LOW
+#define DC_PERFMON14_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON14_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON14_PERFMON_HI
+#define DC_PERFMON14_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON14_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON14_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON14_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON14_PERFMON_LOW
+#define DC_PERFMON14_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON14_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_odm0_dispdec
+//ODM0_OPTC_INPUT_GLOBAL_CONTROL
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET__SHIFT                                          0x0
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN__SHIFT                                          0x8
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS__SHIFT                                 0xa
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS__SHIFT                                      0xb
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR__SHIFT                                           0xc
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT__SHIFT                                0xd
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING__SHIFT                                     0x1f
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET_MASK                                            0x00000001L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN_MASK                                            0x00000100L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS_MASK                                   0x00000400L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS_MASK                                        0x00000800L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR_MASK                                             0x00001000L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT_MASK                                  0x00002000L
+#define ODM0_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING_MASK                                       0x80000000L
+//ODM0_OPTC_RSMU_UNDERFLOW
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_INT_EN__SHIFT                                           0x0
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_OCCURRED_STATUS__SHIFT                                  0x1
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_INT_STATUS__SHIFT                                       0x2
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_CLEAR__SHIFT                                            0x3
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_INT_EN_MASK                                             0x00000001L
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_OCCURRED_STATUS_MASK                                    0x00000002L
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_INT_STATUS_MASK                                         0x00000004L
+#define ODM0_OPTC_RSMU_UNDERFLOW__OPTC_RSMU_UNDERFLOW_CLEAR_MASK                                              0x00000008L
+//ODM0_OPTC_UNDERFLOW_THRESHOLD
+#define ODM0_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD__SHIFT                                        0x0
+#define ODM0_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD_MASK                                          0x01FFFFFFL
+//ODM0_OPTC_DATA_SOURCE_SELECT
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT__SHIFT                                        0x0
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT__SHIFT                                       0x8
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL__SHIFT                                                0x10
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL__SHIFT                                                0x14
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL__SHIFT                                                0x18
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL__SHIFT                                                0x1c
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT_MASK                                          0x00000003L
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT_MASK                                         0x00000300L
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL_MASK                                                  0x000F0000L
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL_MASK                                                  0x00F00000L
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL_MASK                                                  0x0F000000L
+#define ODM0_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL_MASK                                                  0xF0000000L
+//ODM0_OPTC_DATA_FORMAT_CONTROL
+#define ODM0_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT__SHIFT                                                0x0
+#define ODM0_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE__SHIFT                                                   0x4
+#define ODM0_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT_MASK                                                  0x00000003L
+#define ODM0_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE_MASK                                                     0x00000030L
+//ODM0_OPTC_BYTES_PER_PIXEL
+#define ODM0_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL__SHIFT                                            0x0
+#define ODM0_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL_MASK                                              0x7FFFFFFFL
+//ODM0_OPTC_WIDTH_CONTROL
+#define ODM0_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH__SHIFT                                                    0x0
+#define ODM0_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH__SHIFT                                                  0x10
+#define ODM0_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH_MASK                                                      0x00001FFFL
+#define ODM0_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH_MASK                                                    0x1FFF0000L
+//ODM0_OPTC_WIDTH_CONTROL2
+#define ODM0_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST__SHIFT                                              0x0
+#define ODM0_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST_MASK                                                0x00001FFFL
+//ODM0_OPTC_INPUT_CLOCK_CONTROL
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS__SHIFT                                         0x0
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN__SHIFT                                               0x1
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON__SHIFT                                               0x2
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS_MASK                                           0x00000001L
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN_MASK                                                 0x00000002L
+#define ODM0_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON_MASK                                                 0x00000004L
+//ODM0_OPTC_MEMORY_CONFIG
+#define ODM0_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL__SHIFT                                                          0x0
+#define ODM0_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS__SHIFT                                                   0x10
+#define ODM0_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_MASK                                                            0x0000FFFFL
+#define ODM0_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS_MASK                                                     0xFFFF0000L
+//ODM0_OPTC_INPUT_SPARE_REGISTER
+#define ODM0_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG__SHIFT                                           0x0
+#define ODM0_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_odm1_dispdec
+//ODM1_OPTC_INPUT_GLOBAL_CONTROL
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET__SHIFT                                          0x0
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN__SHIFT                                          0x8
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS__SHIFT                                 0xa
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS__SHIFT                                      0xb
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR__SHIFT                                           0xc
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT__SHIFT                                0xd
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING__SHIFT                                     0x1f
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET_MASK                                            0x00000001L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN_MASK                                            0x00000100L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS_MASK                                   0x00000400L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS_MASK                                        0x00000800L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR_MASK                                             0x00001000L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT_MASK                                  0x00002000L
+#define ODM1_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING_MASK                                       0x80000000L
+//ODM1_OPTC_UNDERFLOW_THRESHOLD
+#define ODM1_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD__SHIFT                                        0x0
+#define ODM1_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD_MASK                                          0x01FFFFFFL
+//ODM1_OPTC_DATA_SOURCE_SELECT
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT__SHIFT                                        0x0
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT__SHIFT                                       0x8
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL__SHIFT                                                0x10
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL__SHIFT                                                0x14
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL__SHIFT                                                0x18
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL__SHIFT                                                0x1c
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT_MASK                                          0x00000003L
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT_MASK                                         0x00000300L
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL_MASK                                                  0x000F0000L
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL_MASK                                                  0x00F00000L
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL_MASK                                                  0x0F000000L
+#define ODM1_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL_MASK                                                  0xF0000000L
+//ODM1_OPTC_DATA_FORMAT_CONTROL
+#define ODM1_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT__SHIFT                                                0x0
+#define ODM1_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE__SHIFT                                                   0x4
+#define ODM1_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT_MASK                                                  0x00000003L
+#define ODM1_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE_MASK                                                     0x00000030L
+//ODM1_OPTC_BYTES_PER_PIXEL
+#define ODM1_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL__SHIFT                                            0x0
+#define ODM1_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL_MASK                                              0x7FFFFFFFL
+//ODM1_OPTC_WIDTH_CONTROL
+#define ODM1_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH__SHIFT                                                    0x0
+#define ODM1_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH__SHIFT                                                  0x10
+#define ODM1_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH_MASK                                                      0x00001FFFL
+#define ODM1_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH_MASK                                                    0x1FFF0000L
+//ODM1_OPTC_WIDTH_CONTROL2
+#define ODM1_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST__SHIFT                                              0x0
+#define ODM1_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST_MASK                                                0x00001FFFL
+//ODM1_OPTC_INPUT_CLOCK_CONTROL
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS__SHIFT                                         0x0
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN__SHIFT                                               0x1
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON__SHIFT                                               0x2
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS_MASK                                           0x00000001L
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN_MASK                                                 0x00000002L
+#define ODM1_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON_MASK                                                 0x00000004L
+//ODM1_OPTC_MEMORY_CONFIG
+#define ODM1_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL__SHIFT                                                          0x0
+#define ODM1_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS__SHIFT                                                   0x10
+#define ODM1_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_MASK                                                            0x0000FFFFL
+#define ODM1_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS_MASK                                                     0xFFFF0000L
+//ODM1_OPTC_INPUT_SPARE_REGISTER
+#define ODM1_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG__SHIFT                                           0x0
+#define ODM1_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_odm2_dispdec
+//ODM2_OPTC_INPUT_GLOBAL_CONTROL
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET__SHIFT                                          0x0
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN__SHIFT                                          0x8
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS__SHIFT                                 0xa
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS__SHIFT                                      0xb
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR__SHIFT                                           0xc
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT__SHIFT                                0xd
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING__SHIFT                                     0x1f
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET_MASK                                            0x00000001L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN_MASK                                            0x00000100L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS_MASK                                   0x00000400L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS_MASK                                        0x00000800L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR_MASK                                             0x00001000L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT_MASK                                  0x00002000L
+#define ODM2_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING_MASK                                       0x80000000L
+//ODM2_OPTC_UNDERFLOW_THRESHOLD
+#define ODM2_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD__SHIFT                                        0x0
+#define ODM2_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD_MASK                                          0x01FFFFFFL
+//ODM2_OPTC_DATA_SOURCE_SELECT
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT__SHIFT                                        0x0
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT__SHIFT                                       0x8
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL__SHIFT                                                0x10
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL__SHIFT                                                0x14
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL__SHIFT                                                0x18
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL__SHIFT                                                0x1c
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT_MASK                                          0x00000003L
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT_MASK                                         0x00000300L
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL_MASK                                                  0x000F0000L
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL_MASK                                                  0x00F00000L
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL_MASK                                                  0x0F000000L
+#define ODM2_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL_MASK                                                  0xF0000000L
+//ODM2_OPTC_DATA_FORMAT_CONTROL
+#define ODM2_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT__SHIFT                                                0x0
+#define ODM2_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE__SHIFT                                                   0x4
+#define ODM2_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT_MASK                                                  0x00000003L
+#define ODM2_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE_MASK                                                     0x00000030L
+//ODM2_OPTC_BYTES_PER_PIXEL
+#define ODM2_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL__SHIFT                                            0x0
+#define ODM2_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL_MASK                                              0x7FFFFFFFL
+//ODM2_OPTC_WIDTH_CONTROL
+#define ODM2_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH__SHIFT                                                    0x0
+#define ODM2_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH__SHIFT                                                  0x10
+#define ODM2_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH_MASK                                                      0x00001FFFL
+#define ODM2_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH_MASK                                                    0x1FFF0000L
+//ODM2_OPTC_WIDTH_CONTROL2
+#define ODM2_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST__SHIFT                                              0x0
+#define ODM2_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST_MASK                                                0x00001FFFL
+//ODM2_OPTC_INPUT_CLOCK_CONTROL
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS__SHIFT                                         0x0
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN__SHIFT                                               0x1
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON__SHIFT                                               0x2
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS_MASK                                           0x00000001L
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN_MASK                                                 0x00000002L
+#define ODM2_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON_MASK                                                 0x00000004L
+//ODM2_OPTC_MEMORY_CONFIG
+#define ODM2_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL__SHIFT                                                          0x0
+#define ODM2_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS__SHIFT                                                   0x10
+#define ODM2_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_MASK                                                            0x0000FFFFL
+#define ODM2_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS_MASK                                                     0xFFFF0000L
+//ODM2_OPTC_INPUT_SPARE_REGISTER
+#define ODM2_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG__SHIFT                                           0x0
+#define ODM2_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_odm3_dispdec
+//ODM3_OPTC_INPUT_GLOBAL_CONTROL
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET__SHIFT                                          0x0
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN__SHIFT                                          0x8
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS__SHIFT                                 0xa
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS__SHIFT                                      0xb
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR__SHIFT                                           0xc
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT__SHIFT                                0xd
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING__SHIFT                                     0x1f
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_INPUT_SOFT_RESET_MASK                                            0x00000001L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_EN_MASK                                            0x00000100L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_STATUS_MASK                                   0x00000400L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_INT_STATUS_MASK                                        0x00000800L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_CLEAR_MASK                                             0x00001000L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_UNDERFLOW_OCCURRED_CURRENT_MASK                                  0x00002000L
+#define ODM3_OPTC_INPUT_GLOBAL_CONTROL__OPTC_DOUBLE_BUFFER_PENDING_MASK                                       0x80000000L
+//ODM3_OPTC_UNDERFLOW_THRESHOLD
+#define ODM3_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD__SHIFT                                        0x0
+#define ODM3_OPTC_UNDERFLOW_THRESHOLD__OPTC_UNDERFLOW_THRESHOLD_MASK                                          0x01FFFFFFL
+//ODM3_OPTC_DATA_SOURCE_SELECT
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT__SHIFT                                        0x0
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT__SHIFT                                       0x8
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL__SHIFT                                                0x10
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL__SHIFT                                                0x14
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL__SHIFT                                                0x18
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL__SHIFT                                                0x1c
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_INPUT_SEGMENT_MASK                                          0x00000003L
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_NUM_OF_OUTPUT_SEGMENT_MASK                                         0x00000300L
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG0_SRC_SEL_MASK                                                  0x000F0000L
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG1_SRC_SEL_MASK                                                  0x00F00000L
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG2_SRC_SEL_MASK                                                  0x0F000000L
+#define ODM3_OPTC_DATA_SOURCE_SELECT__OPTC_SEG3_SRC_SEL_MASK                                                  0xF0000000L
+//ODM3_OPTC_DATA_FORMAT_CONTROL
+#define ODM3_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT__SHIFT                                                0x0
+#define ODM3_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE__SHIFT                                                   0x4
+#define ODM3_OPTC_DATA_FORMAT_CONTROL__OPTC_DATA_FORMAT_MASK                                                  0x00000003L
+#define ODM3_OPTC_DATA_FORMAT_CONTROL__OPTC_DSC_MODE_MASK                                                     0x00000030L
+//ODM3_OPTC_BYTES_PER_PIXEL
+#define ODM3_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL__SHIFT                                            0x0
+#define ODM3_OPTC_BYTES_PER_PIXEL__OPTC_DSC_BYTES_PER_PIXEL_MASK                                              0x7FFFFFFFL
+//ODM3_OPTC_WIDTH_CONTROL
+#define ODM3_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH__SHIFT                                                    0x0
+#define ODM3_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH__SHIFT                                                  0x10
+#define ODM3_OPTC_WIDTH_CONTROL__OPTC_SEGMENT_WIDTH_MASK                                                      0x00001FFFL
+#define ODM3_OPTC_WIDTH_CONTROL__OPTC_DSC_SLICE_WIDTH_MASK                                                    0x1FFF0000L
+//ODM3_OPTC_WIDTH_CONTROL2
+#define ODM3_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST__SHIFT                                              0x0
+#define ODM3_OPTC_WIDTH_CONTROL2__OPTC_SEGMENT_WIDTH_LAST_MASK                                                0x00001FFFL
+//ODM3_OPTC_INPUT_CLOCK_CONTROL
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS__SHIFT                                         0x0
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN__SHIFT                                               0x1
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON__SHIFT                                               0x2
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_GATE_DIS_MASK                                           0x00000001L
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_EN_MASK                                                 0x00000002L
+#define ODM3_OPTC_INPUT_CLOCK_CONTROL__OPTC_INPUT_CLK_ON_MASK                                                 0x00000004L
+//ODM3_OPTC_MEMORY_CONFIG
+#define ODM3_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL__SHIFT                                                          0x0
+#define ODM3_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS__SHIFT                                                   0x10
+#define ODM3_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_MASK                                                            0x0000FFFFL
+#define ODM3_OPTC_MEMORY_CONFIG__OPTC_MEM_SEL_STATUS_MASK                                                     0xFFFF0000L
+//ODM3_OPTC_INPUT_SPARE_REGISTER
+#define ODM3_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG__SHIFT                                           0x0
+#define ODM3_OPTC_INPUT_SPARE_REGISTER__OPTC_INPUT_SPARE_REG_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_otg0_dispdec
+//OTG0_OTG_H_TOTAL
+#define OTG0_OTG_H_TOTAL__OTG_H_TOTAL__SHIFT                                                                  0x0
+#define OTG0_OTG_H_TOTAL__OTG_H_TOTAL_MASK                                                                    0x00007FFFL
+//OTG0_OTG_H_BLANK_START_END
+#define OTG0_OTG_H_BLANK_START_END__OTG_H_BLANK_START__SHIFT                                                  0x0
+#define OTG0_OTG_H_BLANK_START_END__OTG_H_BLANK_END__SHIFT                                                    0x10
+#define OTG0_OTG_H_BLANK_START_END__OTG_H_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG0_OTG_H_BLANK_START_END__OTG_H_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG0_OTG_H_SYNC_A
+#define OTG0_OTG_H_SYNC_A__OTG_H_SYNC_A_START__SHIFT                                                          0x0
+#define OTG0_OTG_H_SYNC_A__OTG_H_SYNC_A_END__SHIFT                                                            0x10
+#define OTG0_OTG_H_SYNC_A__OTG_H_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG0_OTG_H_SYNC_A__OTG_H_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG0_OTG_H_SYNC_A_CNTL
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN__SHIFT                                                     0x10
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF__SHIFT                                                    0x11
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN_MASK                                                       0x00010000L
+#define OTG0_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF_MASK                                                      0x00020000L
+//OTG0_OTG_H_TIMING_CNTL
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE__SHIFT                                                  0x0
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL__SHIFT                                           0x8
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR__SHIFT                                             0x10
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MASK                                                    0x00000003L
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL_MASK                                             0x00000100L
+#define OTG0_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR_MASK                                               0x00030000L
+//OTG0_OTG_V_TOTAL
+#define OTG0_OTG_V_TOTAL__OTG_V_TOTAL__SHIFT                                                                  0x0
+#define OTG0_OTG_V_TOTAL__OTG_V_TOTAL_MASK                                                                    0x00007FFFL
+//OTG0_OTG_V_TOTAL_MIN
+#define OTG0_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN__SHIFT                                                          0x0
+#define OTG0_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN_MASK                                                            0x00007FFFL
+//OTG0_OTG_V_TOTAL_MAX
+#define OTG0_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX__SHIFT                                                          0x0
+#define OTG0_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX_MASK                                                            0x00007FFFL
+//OTG0_OTG_V_TOTAL_MID
+#define OTG0_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID__SHIFT                                                          0x0
+#define OTG0_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID_MASK                                                            0x00007FFFL
+//OTG0_OTG_V_TOTAL_CONTROL
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL__SHIFT                                                  0x0
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL__SHIFT                                                  0x1
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN__SHIFT                                      0x2
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN__SHIFT                                      0x3
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT__SHIFT                                              0x4
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD__SHIFT                                          0x5
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM__SHIFT                                             0x8
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK__SHIFT                                             0x10
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL_MASK                                                    0x00000001L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL_MASK                                                    0x00000002L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN_MASK                                        0x00000004L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN_MASK                                        0x00000008L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT_MASK                                                0x00000010L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD_MASK                                            0x00000020L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM_MASK                                               0x0000FF00L
+#define OTG0_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK_MASK                                               0xFFFF0000L
+//OTG0_OTG_V_COUNT_STOP_CONTROL
+#define OTG0_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP__SHIFT                                                0x0
+#define OTG0_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP_MASK                                                  0x00007FFFL
+//OTG0_OTG_V_COUNT_STOP_CONTROL2
+#define OTG0_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER__SHIFT                                         0x0
+#define OTG0_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER_MASK                                           0xFFFFFFFFL
+//OTG0_OTG_V_TOTAL_INT_STATUS
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED__SHIFT                                0x0
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT__SHIFT                            0x4
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK__SHIFT                            0x8
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK__SHIFT                            0xc
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MASK                                  0x00000001L
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT_MASK                              0x00000010L
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK_MASK                              0x00000100L
+#define OTG0_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK_MASK                              0x00001000L
+//OTG0_OTG_VSYNC_NOM_INT_STATUS
+#define OTG0_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM__SHIFT                                                   0x0
+#define OTG0_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR__SHIFT                                         0x4
+#define OTG0_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_MASK                                                     0x00000001L
+#define OTG0_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR_MASK                                           0x00000010L
+//OTG0_OTG_V_BLANK_START_END
+#define OTG0_OTG_V_BLANK_START_END__OTG_V_BLANK_START__SHIFT                                                  0x0
+#define OTG0_OTG_V_BLANK_START_END__OTG_V_BLANK_END__SHIFT                                                    0x10
+#define OTG0_OTG_V_BLANK_START_END__OTG_V_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG0_OTG_V_BLANK_START_END__OTG_V_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG0_OTG_V_SYNC_A
+#define OTG0_OTG_V_SYNC_A__OTG_V_SYNC_A_START__SHIFT                                                          0x0
+#define OTG0_OTG_V_SYNC_A__OTG_V_SYNC_A_END__SHIFT                                                            0x10
+#define OTG0_OTG_V_SYNC_A__OTG_V_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG0_OTG_V_SYNC_A__OTG_V_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG0_OTG_V_SYNC_A_CNTL
+#define OTG0_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG0_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE__SHIFT                                                        0x8
+#define OTG0_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG0_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE_MASK                                                          0x00000100L
+//OTG0_OTG_TRIGA_CNTL
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED__SHIFT                                                        0xe
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY__SHIFT                                                           0x18
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR__SHIFT                                                           0x1f
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED_MASK                                                          0x00004000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY_MASK                                                             0x1F000000L
+#define OTG0_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR_MASK                                                             0x80000000L
+//OTG0_OTG_TRIGA_MANUAL_TRIG
+#define OTG0_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG0_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG0_OTG_TRIGB_CNTL
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED__SHIFT                                                        0xe
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY__SHIFT                                                           0x18
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR__SHIFT                                                           0x1f
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED_MASK                                                          0x00004000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY_MASK                                                             0x1F000000L
+#define OTG0_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR_MASK                                                             0x80000000L
+//OTG0_OTG_TRIGB_MANUAL_TRIG
+#define OTG0_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG0_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG0_OTG_FORCE_COUNT_NOW_CNTL
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE__SHIFT                                        0x0
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK__SHIFT                                       0x4
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL__SHIFT                                    0x8
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED__SHIFT                                    0x10
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR__SHIFT                                       0x18
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE_MASK                                          0x00000003L
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK_MASK                                         0x00000010L
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL_MASK                                      0x00000100L
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED_MASK                                      0x00010000L
+#define OTG0_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR_MASK                                         0x01000000L
+//OTG0_OTG_STEREO_FORCE_NEXT_EYE
+#define OTG0_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE__SHIFT                                      0x0
+#define OTG0_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE_MASK                                        0x00000003L
+//OTG0_OTG_CONTROL
+#define OTG0_OTG_CONTROL__OTG_MASTER_EN__SHIFT                                                                0x0
+#define OTG0_OTG_CONTROL__OTG_DISABLE_POINT_CNTL__SHIFT                                                       0x8
+#define OTG0_OTG_CONTROL__OTG_START_POINT_CNTL__SHIFT                                                         0xc
+#define OTG0_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL__SHIFT                                                        0xd
+#define OTG0_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY__SHIFT                                                    0xe
+#define OTG0_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE__SHIFT                                                  0x10
+#define OTG0_OTG_CONTROL__OTG_OUT_MUX__SHIFT                                                                  0x14
+#define OTG0_OTG_CONTROL__OTG_MASTER_EN_MASK                                                                  0x00000001L
+#define OTG0_OTG_CONTROL__OTG_DISABLE_POINT_CNTL_MASK                                                         0x00000300L
+#define OTG0_OTG_CONTROL__OTG_START_POINT_CNTL_MASK                                                           0x00001000L
+#define OTG0_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL_MASK                                                          0x00002000L
+#define OTG0_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY_MASK                                                      0x00004000L
+#define OTG0_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE_MASK                                                    0x00010000L
+#define OTG0_OTG_CONTROL__OTG_OUT_MUX_MASK                                                                    0x00300000L
+//OTG0_OTG_DLPC_CONTROL
+#define OTG0_OTG_DLPC_CONTROL__OTG_RESYNC_MODE__SHIFT                                                         0x0
+#define OTG0_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION__SHIFT                                              0x10
+#define OTG0_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT__SHIFT                                               0x1f
+#define OTG0_OTG_DLPC_CONTROL__OTG_RESYNC_MODE_MASK                                                           0x00000001L
+#define OTG0_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION_MASK                                                0x7FFF0000L
+#define OTG0_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT_MASK                                                 0x80000000L
+//OTG0_OTG_PWA_FRAME_SYNC_CONTROL
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN__SHIFT                                         0x0
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE__SHIFT                                0x1
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE__SHIFT                                       0x10
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN_MASK                                           0x00000001L
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE_MASK                                  0x00000002L
+#define OTG0_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE_MASK                                         0x03FF0000L
+//OTG0_OTG_INTERLACE_CONTROL
+#define OTG0_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE__SHIFT                                               0x0
+#define OTG0_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD__SHIFT                                     0x10
+#define OTG0_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE_MASK                                                 0x00000001L
+#define OTG0_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD_MASK                                       0x00030000L
+//OTG0_OTG_INTERLACE_STATUS
+#define OTG0_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD__SHIFT                                         0x0
+#define OTG0_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD__SHIFT                                            0x1
+#define OTG0_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD_MASK                                           0x00000001L
+#define OTG0_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD_MASK                                              0x00000002L
+//OTG0_OTG_PIXEL_DATA_READBACK0
+#define OTG0_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB__SHIFT                                          0x0
+#define OTG0_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y__SHIFT                                          0x10
+#define OTG0_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB_MASK                                            0x0000FFFFL
+#define OTG0_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y_MASK                                            0xFFFF0000L
+//OTG0_OTG_PIXEL_DATA_READBACK1
+#define OTG0_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR__SHIFT                                           0x0
+#define OTG0_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR_MASK                                             0x0000FFFFL
+//OTG0_OTG_STATUS
+#define OTG0_OTG_STATUS__OTG_V_BLANK__SHIFT                                                                   0x0
+#define OTG0_OTG_STATUS__OTG_V_ACTIVE_DISP__SHIFT                                                             0x1
+#define OTG0_OTG_STATUS__OTG_V_SYNC_A__SHIFT                                                                  0x2
+#define OTG0_OTG_STATUS__OTG_V_UPDATE__SHIFT                                                                  0x3
+#define OTG0_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE__SHIFT                                                      0x5
+#define OTG0_OTG_STATUS__OTG_H_BLANK__SHIFT                                                                   0x10
+#define OTG0_OTG_STATUS__OTG_H_ACTIVE_DISP__SHIFT                                                             0x11
+#define OTG0_OTG_STATUS__OTG_H_SYNC_A__SHIFT                                                                  0x12
+#define OTG0_OTG_STATUS__OTG_V_BLANK_MASK                                                                     0x00000001L
+#define OTG0_OTG_STATUS__OTG_V_ACTIVE_DISP_MASK                                                               0x00000002L
+#define OTG0_OTG_STATUS__OTG_V_SYNC_A_MASK                                                                    0x00000004L
+#define OTG0_OTG_STATUS__OTG_V_UPDATE_MASK                                                                    0x00000008L
+#define OTG0_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE_MASK                                                        0x00000020L
+#define OTG0_OTG_STATUS__OTG_H_BLANK_MASK                                                                     0x00010000L
+#define OTG0_OTG_STATUS__OTG_H_ACTIVE_DISP_MASK                                                               0x00020000L
+#define OTG0_OTG_STATUS__OTG_H_SYNC_A_MASK                                                                    0x00040000L
+//OTG0_OTG_STATUS_POSITION
+#define OTG0_OTG_STATUS_POSITION__OTG_VERT_COUNT__SHIFT                                                       0x0
+#define OTG0_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK__SHIFT                                                 0xf
+#define OTG0_OTG_STATUS_POSITION__OTG_HORZ_COUNT__SHIFT                                                       0x10
+#define OTG0_OTG_STATUS_POSITION__OTG_VERT_COUNT_MASK                                                         0x00007FFFL
+#define OTG0_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK_MASK                                                   0x00008000L
+#define OTG0_OTG_STATUS_POSITION__OTG_HORZ_COUNT_MASK                                                         0x7FFF0000L
+//OTG0_OTG_LONG_VBLANK_STATUS
+#define OTG0_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT__SHIFT                                            0x0
+#define OTG0_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT_MASK                                              0xFFFFFFFFL
+//OTG0_OTG_NOM_VERT_POSITION
+#define OTG0_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM__SHIFT                                                 0x0
+#define OTG0_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM_MASK                                                   0x00007FFFL
+//OTG0_OTG_STATUS_FRAME_COUNT
+#define OTG0_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT__SHIFT                                                   0x0
+#define OTG0_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT_MASK                                                     0x00FFFFFFL
+//OTG0_OTG_STATUS_VF_COUNT
+#define OTG0_OTG_STATUS_VF_COUNT__OTG_VF_COUNT__SHIFT                                                         0x0
+#define OTG0_OTG_STATUS_VF_COUNT__OTG_VF_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG0_OTG_STATUS_HV_COUNT
+#define OTG0_OTG_STATUS_HV_COUNT__OTG_HV_COUNT__SHIFT                                                         0x0
+#define OTG0_OTG_STATUS_HV_COUNT__OTG_HV_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG0_OTG_COUNT_CONTROL
+#define OTG0_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN__SHIFT                                                  0x0
+#define OTG0_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT__SHIFT                                              0x1
+#define OTG0_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN_MASK                                                    0x00000001L
+#define OTG0_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT_MASK                                                0x0000001EL
+//OTG0_OTG_COUNT_RESET
+#define OTG0_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT__SHIFT                                                    0x0
+#define OTG0_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT_MASK                                                      0x00000001L
+//OTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE
+#define OTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__SHIFT                        0x0
+#define OTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_MASK                          0x00000001L
+//OTG0_OTG_VERT_SYNC_CONTROL
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED__SHIFT                                 0x0
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR__SHIFT                                    0x8
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE__SHIFT                                          0x10
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED_MASK                                   0x00000001L
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR_MASK                                      0x00000100L
+#define OTG0_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE_MASK                                            0x00030000L
+//OTG0_OTG_STEREO_STATUS
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE__SHIFT                                                 0x0
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT__SHIFT                                                 0x8
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT__SHIFT                                                 0x10
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG__SHIFT                                                    0x14
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING__SHIFT                                      0x18
+#define OTG0_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE__SHIFT                                         0x1e
+#define OTG0_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE__SHIFT                                        0x1f
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE_MASK                                                   0x00000001L
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT_MASK                                                   0x00000100L
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT_MASK                                                   0x00010000L
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG_MASK                                                      0x00100000L
+#define OTG0_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING_MASK                                        0x03000000L
+#define OTG0_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE_MASK                                           0x40000000L
+#define OTG0_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE_MASK                                          0x80000000L
+//OTG0_OTG_STEREO_CONTROL
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM__SHIFT                                       0x0
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY__SHIFT                                       0xf
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY__SHIFT                                          0x11
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP__SHIFT                                  0x12
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM__SHIFT                                                 0x13
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX__SHIFT                                        0x14
+#define OTG0_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL__SHIFT                                                     0x15
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_EN__SHIFT                                                         0x18
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM_MASK                                         0x00007FFFL
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY_MASK                                         0x00008000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY_MASK                                            0x00020000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP_MASK                                    0x00040000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM_MASK                                                   0x00080000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX_MASK                                          0x00100000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL_MASK                                                       0x00200000L
+#define OTG0_OTG_STEREO_CONTROL__OTG_STEREO_EN_MASK                                                           0x01000000L
+//OTG0_OTG_SNAPSHOT_STATUS
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED__SHIFT                                                0x0
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR__SHIFT                                                   0x1
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER__SHIFT                                          0x2
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED_MASK                                                  0x00000001L
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR_MASK                                                     0x00000002L
+#define OTG0_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER_MASK                                            0x00000004L
+//OTG0_OTG_SNAPSHOT_CONTROL
+#define OTG0_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL__SHIFT                                          0x0
+#define OTG0_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL_MASK                                            0x00000003L
+//OTG0_OTG_SNAPSHOT_POSITION
+#define OTG0_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT__SHIFT                                            0x0
+#define OTG0_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT__SHIFT                                            0x10
+#define OTG0_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT_MASK                                              0x00007FFFL
+#define OTG0_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT_MASK                                              0x7FFF0000L
+//OTG0_OTG_SNAPSHOT_FRAME
+#define OTG0_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT__SHIFT                                              0x0
+#define OTG0_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT_MASK                                                0x00FFFFFFL
+//OTG0_OTG_INTERRUPT_CONTROL
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK__SHIFT                                               0x0
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE__SHIFT                                              0x1
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK__SHIFT                                        0x8
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE__SHIFT                                       0x9
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK__SHIFT                                  0x10
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE__SHIFT                                 0x11
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK__SHIFT                                                  0x18
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK__SHIFT                                                  0x19
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE__SHIFT                                                 0x1a
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE__SHIFT                                                 0x1b
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK__SHIFT                                              0x1c
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE__SHIFT                                             0x1d
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK__SHIFT                                          0x1e
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE__SHIFT                                         0x1f
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK_MASK                                                 0x00000001L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE_MASK                                                0x00000002L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK_MASK                                          0x00000100L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE_MASK                                         0x00000200L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK_MASK                                    0x00010000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE_MASK                                   0x00020000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK_MASK                                                    0x01000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK_MASK                                                    0x02000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE_MASK                                                   0x04000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE_MASK                                                   0x08000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK_MASK                                                0x10000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE_MASK                                               0x20000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK_MASK                                            0x40000000L
+#define OTG0_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE_MASK                                           0x80000000L
+//OTG0_OTG_UPDATE_LOCK
+#define OTG0_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK__SHIFT                                                          0x0
+#define OTG0_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK_MASK                                                            0x00000001L
+//OTG0_OTG_DOUBLE_BUFFER_CONTROL
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING__SHIFT                                             0x0
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING__SHIFT                             0x4
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING__SHIFT                                   0x5
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING__SHIFT                                  0x6
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING__SHIFT                          0x7
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY__SHIFT                                           0x8
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING__SHIFT                                 0x9
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE__SHIFT                                0x18
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING_MASK                                               0x00000001L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING_MASK                               0x00000010L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING_MASK                                     0x00000020L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING_MASK                                    0x00000040L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING_MASK                            0x00000080L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY_MASK                                             0x00000100L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING_MASK                                   0x00000200L
+#define OTG0_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE_MASK                                  0x03000000L
+//OTG0_OTG_MASTER_EN
+#define OTG0_OTG_MASTER_EN__OTG_MASTER_EN__SHIFT                                                              0x0
+#define OTG0_OTG_MASTER_EN__OTG_MASTER_EN_MASK                                                                0x00000001L
+//OTG0_OTG_VERTICAL_INTERRUPT0_POSITION
+#define OTG0_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START__SHIFT                      0x0
+#define OTG0_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END__SHIFT                        0x10
+#define OTG0_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START_MASK                        0x00007FFFL
+#define OTG0_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END_MASK                          0x7FFF0000L
+//OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY__SHIFT                  0x4
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE__SHIFT                       0x8
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS__SHIFT                           0xc
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS__SHIFT                       0x10
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR__SHIFT                            0x14
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE__SHIFT                         0x18
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS__SHIFT                                         0x1c
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY_MASK                    0x00000010L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE_MASK                         0x00000100L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS_MASK                             0x00001000L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS_MASK                         0x00010000L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR_MASK                              0x00100000L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE_MASK                           0x01000000L
+#define OTG0_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS_MASK                                           0x10000000L
+//OTG0_OTG_VERTICAL_INTERRUPT1_POSITION
+#define OTG0_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START__SHIFT                      0x0
+#define OTG0_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START_MASK                        0x00007FFFL
+//OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE__SHIFT                       0x8
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS__SHIFT                           0xc
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS__SHIFT                       0x10
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR__SHIFT                            0x14
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE__SHIFT                         0x18
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE_MASK                         0x00000100L
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS_MASK                             0x00001000L
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS_MASK                         0x00010000L
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR_MASK                              0x00100000L
+#define OTG0_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE_MASK                           0x01000000L
+//OTG0_OTG_VERTICAL_INTERRUPT2_POSITION
+#define OTG0_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START__SHIFT                      0x0
+#define OTG0_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START_MASK                        0x00007FFFL
+//OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE__SHIFT                       0x8
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS__SHIFT                           0xc
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS__SHIFT                       0x10
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR__SHIFT                            0x14
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE__SHIFT                         0x18
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE_MASK                         0x00000100L
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS_MASK                             0x00001000L
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS_MASK                         0x00010000L
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR_MASK                              0x00100000L
+#define OTG0_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE_MASK                           0x01000000L
+//OTG0_OTG_CRC_CNTL
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_EN__SHIFT                                                                  0x0
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN__SHIFT                                                        0x1
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_POLY_SEL__SHIFT                                                            0x2
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY__SHIFT                                                          0x3
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CONT_EN__SHIFT                                                             0x4
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL__SHIFT                                                   0x5
+#define OTG0_OTG_CRC_CNTL__OTG_CRC1_EN__SHIFT                                                                 0x7
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE__SHIFT                                                         0x8
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CONT_MODE__SHIFT                                                           0xa
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE__SHIFT                                                      0xc
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_AES_EN__SHIFT                                                              0xe
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS__SHIFT                                         0x13
+#define OTG0_OTG_CRC_CNTL__OTG_CRC0_SELECT__SHIFT                                                             0x14
+#define OTG0_OTG_CRC_CNTL__OTG_CRC1_SELECT__SHIFT                                                             0x18
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING__SHIFT                                                   0x1c
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING__SHIFT                                                   0x1d
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING__SHIFT                                                   0x1e
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING__SHIFT                                                   0x1f
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_EN_MASK                                                                    0x00000001L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN_MASK                                                          0x00000002L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_POLY_SEL_MASK                                                              0x00000004L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY_MASK                                                            0x00000008L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CONT_EN_MASK                                                               0x00000010L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL_MASK                                                     0x00000060L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC1_EN_MASK                                                                   0x00000080L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE_MASK                                                           0x00000300L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_CONT_MODE_MASK                                                             0x00000400L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE_MASK                                                        0x00003000L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_AES_EN_MASK                                                                0x00004000L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS_MASK                                           0x00080000L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC0_SELECT_MASK                                                               0x00700000L
+#define OTG0_OTG_CRC_CNTL__OTG_CRC1_SELECT_MASK                                                               0x07000000L
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING_MASK                                                     0x10000000L
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING_MASK                                                     0x20000000L
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING_MASK                                                     0x40000000L
+#define OTG0_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING_MASK                                                     0x80000000L
+//OTG0_OTG_CRC0_WINDOWA_X_CONTROL
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWA_Y_CONTROL
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWB_X_CONTROL
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWB_Y_CONTROL
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC0_DATA_R
+#define OTG0_OTG_CRC0_DATA_R__CRC0_R_CR__SHIFT                                                                0x0
+#define OTG0_OTG_CRC0_DATA_R__CRC0_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC0_DATA_G
+#define OTG0_OTG_CRC0_DATA_G__CRC0_G_Y__SHIFT                                                                 0x0
+#define OTG0_OTG_CRC0_DATA_G__CRC0_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG0_OTG_CRC0_DATA_B
+#define OTG0_OTG_CRC0_DATA_B__CRC0_B_CB__SHIFT                                                                0x0
+#define OTG0_OTG_CRC0_DATA_B__CRC0_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC0_DATA_C
+#define OTG0_OTG_CRC0_DATA_C__CRC0_C__SHIFT                                                                   0x0
+#define OTG0_OTG_CRC0_DATA_C__CRC0_C_MASK                                                                     0xFFFFFFFFL
+//OTG0_OTG_CRC1_WINDOWA_X_CONTROL
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWA_Y_CONTROL
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWB_X_CONTROL
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWB_Y_CONTROL
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG0_OTG_CRC1_DATA_R
+#define OTG0_OTG_CRC1_DATA_R__CRC1_R_CR__SHIFT                                                                0x0
+#define OTG0_OTG_CRC1_DATA_R__CRC1_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC1_DATA_G
+#define OTG0_OTG_CRC1_DATA_G__CRC1_G_Y__SHIFT                                                                 0x0
+#define OTG0_OTG_CRC1_DATA_G__CRC1_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG0_OTG_CRC1_DATA_B
+#define OTG0_OTG_CRC1_DATA_B__CRC1_B_CB__SHIFT                                                                0x0
+#define OTG0_OTG_CRC1_DATA_B__CRC1_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC1_DATA_C
+#define OTG0_OTG_CRC1_DATA_C__CRC1_C__SHIFT                                                                   0x0
+#define OTG0_OTG_CRC1_DATA_C__CRC1_C_MASK                                                                     0xFFFFFFFFL
+//OTG0_OTG_CRC2_DATA_R
+#define OTG0_OTG_CRC2_DATA_R__CRC2_R_CR__SHIFT                                                                0x0
+#define OTG0_OTG_CRC2_DATA_R__CRC2_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC2_DATA_G
+#define OTG0_OTG_CRC2_DATA_G__CRC2_G_Y__SHIFT                                                                 0x0
+#define OTG0_OTG_CRC2_DATA_G__CRC2_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG0_OTG_CRC2_DATA_B
+#define OTG0_OTG_CRC2_DATA_B__CRC2_B_CB__SHIFT                                                                0x0
+#define OTG0_OTG_CRC2_DATA_B__CRC2_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC2_DATA_C
+#define OTG0_OTG_CRC2_DATA_C__CRC2_C__SHIFT                                                                   0x0
+#define OTG0_OTG_CRC2_DATA_C__CRC2_C_MASK                                                                     0xFFFFFFFFL
+//OTG0_OTG_CRC3_DATA_R
+#define OTG0_OTG_CRC3_DATA_R__CRC3_R_CR__SHIFT                                                                0x0
+#define OTG0_OTG_CRC3_DATA_R__CRC3_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC3_DATA_G
+#define OTG0_OTG_CRC3_DATA_G__CRC3_G_Y__SHIFT                                                                 0x0
+#define OTG0_OTG_CRC3_DATA_G__CRC3_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG0_OTG_CRC3_DATA_B
+#define OTG0_OTG_CRC3_DATA_B__CRC3_B_CB__SHIFT                                                                0x0
+#define OTG0_OTG_CRC3_DATA_B__CRC3_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG0_OTG_CRC3_DATA_C
+#define OTG0_OTG_CRC3_DATA_C__CRC3_C__SHIFT                                                                   0x0
+#define OTG0_OTG_CRC3_DATA_C__CRC3_C_MASK                                                                     0xFFFFFFFFL
+//OTG0_OTG_CRC0_DATA_AES
+#define OTG0_OTG_CRC0_DATA_AES__CRC0_AES__SHIFT                                                               0x0
+#define OTG0_OTG_CRC0_DATA_AES__CRC0_AES_MASK                                                                 0xFFFFFFFFL
+//OTG0_OTG_CRC_SIG_RED_GREEN_MASK
+#define OTG0_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK__SHIFT                                          0x0
+#define OTG0_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK__SHIFT                                        0x10
+#define OTG0_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK_MASK                                            0x0000FFFFL
+#define OTG0_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK_MASK                                          0xFFFF0000L
+//OTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK
+#define OTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK__SHIFT                                      0x0
+#define OTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK__SHIFT                                   0x10
+#define OTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK_MASK                                        0x0000FFFFL
+#define OTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK_MASK                                     0xFFFF0000L
+//OTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG0_OTG_STATIC_SCREEN_CONTROL
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK__SHIFT                                   0x0
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT__SHIFT                                  0x10
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE__SHIFT                                          0x18
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS__SHIFT                                                  0x19
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS__SHIFT                                          0x1a
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR__SHIFT                                           0x1b
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE__SHIFT                                            0x1c
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE__SHIFT                                     0x1e
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE__SHIFT                               0x1f
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK_MASK                                     0x0000FFFFL
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT_MASK                                    0x00FF0000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE_MASK                                            0x01000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS_MASK                                                    0x02000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS_MASK                                            0x04000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR_MASK                                             0x08000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE_MASK                                              0x10000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_MASK                                       0x40000000L
+#define OTG0_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE_MASK                                 0x80000000L
+//OTG0_OTG_3D_STRUCTURE_CONTROL
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN__SHIFT                                             0x0
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE__SHIFT                                  0x8
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR__SHIFT                                 0xc
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET__SHIFT                                  0x10
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING__SHIFT                          0x11
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT__SHIFT                                        0x12
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN_MASK                                               0x00000001L
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE_MASK                                    0x00000300L
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR_MASK                                   0x00001000L
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_MASK                                    0x00010000L
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING_MASK                            0x00020000L
+#define OTG0_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_MASK                                          0x000C0000L
+//OTG0_OTG_GSL_VSYNC_GAP
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT__SHIFT                                                0x0
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY__SHIFT                                                0x8
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL__SHIFT                                           0x10
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE__SHIFT                                                 0x11
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR__SHIFT                                                0x13
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED__SHIFT                                             0x14
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER__SHIFT                                        0x17
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP__SHIFT                                                      0x18
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT_MASK                                                  0x000000FFL
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY_MASK                                                  0x0000FF00L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL_MASK                                             0x00010000L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE_MASK                                                   0x00060000L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR_MASK                                                  0x00080000L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED_MASK                                               0x00100000L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER_MASK                                          0x00800000L
+#define OTG0_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASK                                                        0xFF000000L
+//OTG0_OTG_MASTER_UPDATE_MODE
+#define OTG0_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE__SHIFT                                     0x0
+#define OTG0_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE_MASK                                       0x00000003L
+//OTG0_OTG_CLOCK_CONTROL
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_EN__SHIFT                                                           0x0
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS__SHIFT                                                     0x1
+#define OTG0_OTG_CLOCK_CONTROL__OTG_SOFT_RESET__SHIFT                                                         0x4
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_ON__SHIFT                                                           0x8
+#define OTG0_OTG_CLOCK_CONTROL__OTG_BUSY__SHIFT                                                               0x10
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_EN_MASK                                                             0x00000001L
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS_MASK                                                       0x00000002L
+#define OTG0_OTG_CLOCK_CONTROL__OTG_SOFT_RESET_MASK                                                           0x00000010L
+#define OTG0_OTG_CLOCK_CONTROL__OTG_CLOCK_ON_MASK                                                             0x00000100L
+#define OTG0_OTG_CLOCK_CONTROL__OTG_BUSY_MASK                                                                 0x00010000L
+//OTG0_OTG_VSTARTUP_PARAM
+#define OTG0_OTG_VSTARTUP_PARAM__VSTARTUP_START__SHIFT                                                        0x0
+#define OTG0_OTG_VSTARTUP_PARAM__VSTARTUP_START_MASK                                                          0x000003FFL
+//OTG0_OTG_VUPDATE_PARAM
+#define OTG0_OTG_VUPDATE_PARAM__VUPDATE_OFFSET__SHIFT                                                         0x0
+#define OTG0_OTG_VUPDATE_PARAM__VUPDATE_WIDTH__SHIFT                                                          0x10
+#define OTG0_OTG_VUPDATE_PARAM__VUPDATE_OFFSET_MASK                                                           0x0000FFFFL
+#define OTG0_OTG_VUPDATE_PARAM__VUPDATE_WIDTH_MASK                                                            0x03FF0000L
+//OTG0_OTG_VREADY_PARAM
+#define OTG0_OTG_VREADY_PARAM__VREADY_OFFSET__SHIFT                                                           0x0
+#define OTG0_OTG_VREADY_PARAM__VREADY_OFFSET_MASK                                                             0x0000FFFFL
+//OTG0_OTG_GLOBAL_SYNC_STATUS
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN__SHIFT                                                   0x0
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE__SHIFT                                                 0x1
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED__SHIFT                                           0x2
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS__SHIFT                                               0x3
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR__SHIFT                                              0x4
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN__SHIFT                                                    0x5
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE__SHIFT                                                  0x6
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL__SHIFT                                          0x7
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED__SHIFT                                            0x8
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS__SHIFT                                                0x9
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR__SHIFT                                               0xa
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS__SHIFT                                                    0xb
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN__SHIFT                                            0xc
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE__SHIFT                                          0xd
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED__SHIFT                                    0xe
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS__SHIFT                                        0xf
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR__SHIFT                                       0x10
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS__SHIFT                                            0x11
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN__SHIFT                                                     0x12
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE__SHIFT                                                   0x13
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED__SHIFT                                             0x14
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS__SHIFT                                                 0x15
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR__SHIFT                                                0x16
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS__SHIFT                                              0x18
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS__SHIFT                                               0x19
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN_MASK                                                     0x00000001L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE_MASK                                                   0x00000002L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED_MASK                                             0x00000004L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS_MASK                                                 0x00000008L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR_MASK                                                0x00000010L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN_MASK                                                      0x00000020L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE_MASK                                                    0x00000040L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL_MASK                                            0x00000080L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED_MASK                                              0x00000100L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS_MASK                                                  0x00000200L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR_MASK                                                 0x00000400L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS_MASK                                                      0x00000800L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN_MASK                                              0x00001000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE_MASK                                            0x00002000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED_MASK                                      0x00004000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS_MASK                                          0x00008000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR_MASK                                         0x00010000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS_MASK                                              0x00020000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN_MASK                                                       0x00040000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE_MASK                                                     0x00080000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED_MASK                                               0x00100000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS_MASK                                                   0x00200000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR_MASK                                                  0x00400000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS_MASK                                                0x01000000L
+#define OTG0_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS_MASK                                                 0x02000000L
+//OTG0_OTG_MASTER_UPDATE_LOCK
+#define OTG0_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK__SHIFT                                            0x0
+#define OTG0_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS__SHIFT                                                0x8
+#define OTG0_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK_MASK                                              0x00000001L
+#define OTG0_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS_MASK                                                  0x00000100L
+//OTG0_OTG_GSL_CONTROL
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL0_EN__SHIFT                                                              0x0
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL1_EN__SHIFT                                                              0x1
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL2_EN__SHIFT                                                              0x2
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN__SHIFT                                                        0x3
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE__SHIFT                                                      0x4
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY__SHIFT                                                      0x8
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY__SHIFT                                                      0x10
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS__SHIFT                                                 0x1c
+#define OTG0_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN__SHIFT                                            0x1f
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL0_EN_MASK                                                                0x00000001L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL1_EN_MASK                                                                0x00000002L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL2_EN_MASK                                                                0x00000004L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN_MASK                                                          0x00000008L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE_MASK                                                        0x00000030L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY_MASK                                                        0x00000F00L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY_MASK                                                        0x001F0000L
+#define OTG0_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS_MASK                                                   0x10000000L
+#define OTG0_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN_MASK                                              0x80000000L
+//OTG0_OTG_GSL_WINDOW_X
+#define OTG0_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X__SHIFT                                                  0x0
+#define OTG0_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X__SHIFT                                                    0x10
+#define OTG0_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X_MASK                                                    0x00007FFFL
+#define OTG0_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X_MASK                                                      0x7FFF0000L
+//OTG0_OTG_GSL_WINDOW_Y
+#define OTG0_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y__SHIFT                                                  0x0
+#define OTG0_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y__SHIFT                                                    0x10
+#define OTG0_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y_MASK                                                    0x00007FFFL
+#define OTG0_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y_MASK                                                      0x7FFF0000L
+//OTG0_OTG_VUPDATE_KEEPOUT
+#define OTG0_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET__SHIFT                      0x0
+#define OTG0_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET__SHIFT                        0x10
+#define OTG0_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN__SHIFT                            0x1f
+#define OTG0_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET_MASK                        0x0000FFFFL
+#define OTG0_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET_MASK                          0x03FF0000L
+#define OTG0_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN_MASK                              0x80000000L
+//OTG0_OTG_GLOBAL_CONTROL0
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X__SHIFT                                        0x0
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X__SHIFT                                          0x10
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN__SHIFT                                             0x1f
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X_MASK                                          0x00007FFFL
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X_MASK                                            0x7FFF0000L
+#define OTG0_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN_MASK                                               0x80000000L
+//OTG0_OTG_GLOBAL_CONTROL1
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y__SHIFT                                        0x0
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y__SHIFT                                          0x10
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE__SHIFT                                       0x1f
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y_MASK                                          0x00007FFFL
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y_MASK                                            0x7FFF0000L
+#define OTG0_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE_MASK                                         0x80000000L
+//OTG0_OTG_GLOBAL_CONTROL2
+#define OTG0_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN__SHIFT                                                0xa
+#define OTG0_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL__SHIFT                                              0x10
+#define OTG0_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL__SHIFT                                           0x19
+#define OTG0_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE__SHIFT                                            0x1e
+#define OTG0_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE__SHIFT                                                    0x1f
+#define OTG0_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN_MASK                                                  0x00000400L
+#define OTG0_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL_MASK                                                0x00070000L
+#define OTG0_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL_MASK                                             0x0E000000L
+#define OTG0_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE_MASK                                              0x40000000L
+#define OTG0_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE_MASK                                                      0x80000000L
+//OTG0_OTG_GLOBAL_CONTROL3
+#define OTG0_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD__SHIFT                                          0x0
+#define OTG0_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL__SHIFT                                     0x4
+#define OTG0_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL__SHIFT                                                 0x10
+#define OTG0_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL__SHIFT                                                   0x14
+#define OTG0_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD_MASK                                            0x00000003L
+#define OTG0_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL_MASK                                       0x00000030L
+#define OTG0_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL_MASK                                                   0x00030000L
+#define OTG0_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL_MASK                                                     0x00300000L
+//OTG0_OTG_GLOBAL_CONTROL4
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X__SHIFT                                                0x0
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y__SHIFT                                                0x10
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE__SHIFT                                               0x1f
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X_MASK                                                  0x00007FFFL
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y_MASK                                                  0x7FFF0000L
+#define OTG0_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE_MASK                                                 0x80000000L
+//OTG0_OTG_TRIG_MANUAL_CONTROL
+#define OTG0_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL__SHIFT                                              0x0
+#define OTG0_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL_MASK                                                0x00000001L
+//OTG0_OTG_DRR_TIMING_INT_STATUS
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED__SHIFT                                 0x0
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT__SHIFT                             0x4
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR__SHIFT                           0x8
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK__SHIFT                         0xc
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE__SHIFT                        0xd
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED__SHIFT                                 0x10
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT__SHIFT                             0x14
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR__SHIFT                           0x18
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK__SHIFT                         0x1c
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE__SHIFT                        0x1d
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_MASK                                   0x00000001L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MASK                               0x00000010L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR_MASK                             0x00000100L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK_MASK                           0x00001000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE_MASK                          0x00002000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_MASK                                   0x00010000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MASK                               0x00100000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR_MASK                             0x01000000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK_MASK                           0x10000000L
+#define OTG0_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE_MASK                          0x20000000L
+//OTG0_OTG_DRR_V_TOTAL_REACH_RANGE
+#define OTG0_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE__SHIFT                            0x0
+#define OTG0_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE__SHIFT                            0x10
+#define OTG0_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE_MASK                              0x00007FFFL
+#define OTG0_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE_MASK                              0x7FFF0000L
+//OTG0_OTG_DRR_V_TOTAL_CHANGE
+#define OTG0_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT__SHIFT                                      0x0
+#define OTG0_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT_MASK                                        0x00007FFFL
+//OTG0_OTG_DRR_TRIGGER_WINDOW
+#define OTG0_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X__SHIFT                                    0x0
+#define OTG0_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X__SHIFT                                      0x10
+#define OTG0_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X_MASK                                      0x00007FFFL
+#define OTG0_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X_MASK                                        0x7FFF0000L
+//OTG0_OTG_DRR_CONTROL
+#define OTG0_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME__SHIFT                                                    0x0
+#define OTG0_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR__SHIFT                                             0x10
+#define OTG0_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME_MASK                                                      0x00000003L
+#define OTG0_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR_MASK                                               0x7FFF0000L
+//OTG0_OTG_DRR_CONTOL2
+#define OTG0_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR__SHIFT                                             0x0
+#define OTG0_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR_MASK                                               0xFFFFFFFFL
+//OTG0_OTG_M_CONST_DTO0
+#define OTG0_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE__SHIFT                                                   0x0
+#define OTG0_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE_MASK                                                     0xFFFFFFFFL
+//OTG0_OTG_M_CONST_DTO1
+#define OTG0_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO__SHIFT                                                  0x0
+#define OTG0_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO_MASK                                                    0xFFFFFFFFL
+//OTG0_OTG_REQUEST_CONTROL
+#define OTG0_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE__SHIFT                                     0x0
+#define OTG0_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE_MASK                                       0x00000001L
+//OTG0_OTG_PIPE_UPDATE_STATUS
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING__SHIFT                                                  0x0
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING__SHIFT                                         0x4
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING__SHIFT                                         0x8
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS__SHIFT                                        0x10
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING_MASK                                                    0x00000001L
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING_MASK                                           0x00000010L
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING_MASK                                           0x00000100L
+#define OTG0_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS_MASK                                          0x00010000L
+//OTG0_OTG_PSTATE_REGISTER
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START__SHIFT                                             0x0
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND__SHIFT                                                    0x10
+#define OTG0_OTG_PSTATE_REGISTER__OTG_UNBLANK__SHIFT                                                          0x14
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN__SHIFT                                           0x18
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START_MASK                                               0x00007FFFL
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND_MASK                                                      0x00010000L
+#define OTG0_OTG_PSTATE_REGISTER__OTG_UNBLANK_MASK                                                            0x00100000L
+#define OTG0_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN_MASK                                             0xFF000000L
+//OTG0_OTG_ENCRYPTION
+#define OTG0_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL__SHIFT                                             0x0
+#define OTG0_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL_MASK                                               0x00000003L
+//OTG0_OTG_SPARE_REGISTER
+#define OTG0_OTG_SPARE_REGISTER__OTG_SPARE_REG__SHIFT                                                         0x0
+#define OTG0_OTG_SPARE_REGISTER__OTG_SPARE_REG_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_otg1_dispdec
+//OTG1_OTG_H_TOTAL
+#define OTG1_OTG_H_TOTAL__OTG_H_TOTAL__SHIFT                                                                  0x0
+#define OTG1_OTG_H_TOTAL__OTG_H_TOTAL_MASK                                                                    0x00007FFFL
+//OTG1_OTG_H_BLANK_START_END
+#define OTG1_OTG_H_BLANK_START_END__OTG_H_BLANK_START__SHIFT                                                  0x0
+#define OTG1_OTG_H_BLANK_START_END__OTG_H_BLANK_END__SHIFT                                                    0x10
+#define OTG1_OTG_H_BLANK_START_END__OTG_H_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG1_OTG_H_BLANK_START_END__OTG_H_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG1_OTG_H_SYNC_A
+#define OTG1_OTG_H_SYNC_A__OTG_H_SYNC_A_START__SHIFT                                                          0x0
+#define OTG1_OTG_H_SYNC_A__OTG_H_SYNC_A_END__SHIFT                                                            0x10
+#define OTG1_OTG_H_SYNC_A__OTG_H_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG1_OTG_H_SYNC_A__OTG_H_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG1_OTG_H_SYNC_A_CNTL
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN__SHIFT                                                     0x10
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF__SHIFT                                                    0x11
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN_MASK                                                       0x00010000L
+#define OTG1_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF_MASK                                                      0x00020000L
+//OTG1_OTG_H_TIMING_CNTL
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE__SHIFT                                                  0x0
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL__SHIFT                                           0x8
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR__SHIFT                                             0x10
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MASK                                                    0x00000003L
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL_MASK                                             0x00000100L
+#define OTG1_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR_MASK                                               0x00030000L
+//OTG1_OTG_V_TOTAL
+#define OTG1_OTG_V_TOTAL__OTG_V_TOTAL__SHIFT                                                                  0x0
+#define OTG1_OTG_V_TOTAL__OTG_V_TOTAL_MASK                                                                    0x00007FFFL
+//OTG1_OTG_V_TOTAL_MIN
+#define OTG1_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN__SHIFT                                                          0x0
+#define OTG1_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN_MASK                                                            0x00007FFFL
+//OTG1_OTG_V_TOTAL_MAX
+#define OTG1_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX__SHIFT                                                          0x0
+#define OTG1_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX_MASK                                                            0x00007FFFL
+//OTG1_OTG_V_TOTAL_MID
+#define OTG1_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID__SHIFT                                                          0x0
+#define OTG1_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID_MASK                                                            0x00007FFFL
+//OTG1_OTG_V_TOTAL_CONTROL
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL__SHIFT                                                  0x0
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL__SHIFT                                                  0x1
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN__SHIFT                                      0x2
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN__SHIFT                                      0x3
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT__SHIFT                                              0x4
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD__SHIFT                                          0x5
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM__SHIFT                                             0x8
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK__SHIFT                                             0x10
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL_MASK                                                    0x00000001L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL_MASK                                                    0x00000002L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN_MASK                                        0x00000004L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN_MASK                                        0x00000008L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT_MASK                                                0x00000010L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD_MASK                                            0x00000020L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM_MASK                                               0x0000FF00L
+#define OTG1_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK_MASK                                               0xFFFF0000L
+//OTG1_OTG_V_COUNT_STOP_CONTROL
+#define OTG1_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP__SHIFT                                                0x0
+#define OTG1_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP_MASK                                                  0x00007FFFL
+//OTG1_OTG_V_COUNT_STOP_CONTROL2
+#define OTG1_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER__SHIFT                                         0x0
+#define OTG1_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER_MASK                                           0xFFFFFFFFL
+//OTG1_OTG_V_TOTAL_INT_STATUS
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED__SHIFT                                0x0
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT__SHIFT                            0x4
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK__SHIFT                            0x8
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK__SHIFT                            0xc
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MASK                                  0x00000001L
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT_MASK                              0x00000010L
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK_MASK                              0x00000100L
+#define OTG1_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK_MASK                              0x00001000L
+//OTG1_OTG_VSYNC_NOM_INT_STATUS
+#define OTG1_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM__SHIFT                                                   0x0
+#define OTG1_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR__SHIFT                                         0x4
+#define OTG1_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_MASK                                                     0x00000001L
+#define OTG1_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR_MASK                                           0x00000010L
+//OTG1_OTG_V_BLANK_START_END
+#define OTG1_OTG_V_BLANK_START_END__OTG_V_BLANK_START__SHIFT                                                  0x0
+#define OTG1_OTG_V_BLANK_START_END__OTG_V_BLANK_END__SHIFT                                                    0x10
+#define OTG1_OTG_V_BLANK_START_END__OTG_V_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG1_OTG_V_BLANK_START_END__OTG_V_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG1_OTG_V_SYNC_A
+#define OTG1_OTG_V_SYNC_A__OTG_V_SYNC_A_START__SHIFT                                                          0x0
+#define OTG1_OTG_V_SYNC_A__OTG_V_SYNC_A_END__SHIFT                                                            0x10
+#define OTG1_OTG_V_SYNC_A__OTG_V_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG1_OTG_V_SYNC_A__OTG_V_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG1_OTG_V_SYNC_A_CNTL
+#define OTG1_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG1_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE__SHIFT                                                        0x8
+#define OTG1_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG1_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE_MASK                                                          0x00000100L
+//OTG1_OTG_TRIGA_CNTL
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED__SHIFT                                                        0xe
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY__SHIFT                                                           0x18
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR__SHIFT                                                           0x1f
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED_MASK                                                          0x00004000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY_MASK                                                             0x1F000000L
+#define OTG1_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR_MASK                                                             0x80000000L
+//OTG1_OTG_TRIGA_MANUAL_TRIG
+#define OTG1_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG1_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG1_OTG_TRIGB_CNTL
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED__SHIFT                                                        0xe
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY__SHIFT                                                           0x18
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR__SHIFT                                                           0x1f
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED_MASK                                                          0x00004000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY_MASK                                                             0x1F000000L
+#define OTG1_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR_MASK                                                             0x80000000L
+//OTG1_OTG_TRIGB_MANUAL_TRIG
+#define OTG1_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG1_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG1_OTG_FORCE_COUNT_NOW_CNTL
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE__SHIFT                                        0x0
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK__SHIFT                                       0x4
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL__SHIFT                                    0x8
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED__SHIFT                                    0x10
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR__SHIFT                                       0x18
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE_MASK                                          0x00000003L
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK_MASK                                         0x00000010L
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL_MASK                                      0x00000100L
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED_MASK                                      0x00010000L
+#define OTG1_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR_MASK                                         0x01000000L
+//OTG1_OTG_STEREO_FORCE_NEXT_EYE
+#define OTG1_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE__SHIFT                                      0x0
+#define OTG1_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE_MASK                                        0x00000003L
+//OTG1_OTG_CONTROL
+#define OTG1_OTG_CONTROL__OTG_MASTER_EN__SHIFT                                                                0x0
+#define OTG1_OTG_CONTROL__OTG_DISABLE_POINT_CNTL__SHIFT                                                       0x8
+#define OTG1_OTG_CONTROL__OTG_START_POINT_CNTL__SHIFT                                                         0xc
+#define OTG1_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL__SHIFT                                                        0xd
+#define OTG1_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY__SHIFT                                                    0xe
+#define OTG1_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE__SHIFT                                                  0x10
+#define OTG1_OTG_CONTROL__OTG_OUT_MUX__SHIFT                                                                  0x14
+#define OTG1_OTG_CONTROL__OTG_MASTER_EN_MASK                                                                  0x00000001L
+#define OTG1_OTG_CONTROL__OTG_DISABLE_POINT_CNTL_MASK                                                         0x00000300L
+#define OTG1_OTG_CONTROL__OTG_START_POINT_CNTL_MASK                                                           0x00001000L
+#define OTG1_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL_MASK                                                          0x00002000L
+#define OTG1_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY_MASK                                                      0x00004000L
+#define OTG1_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE_MASK                                                    0x00010000L
+#define OTG1_OTG_CONTROL__OTG_OUT_MUX_MASK                                                                    0x00300000L
+//OTG1_OTG_DLPC_CONTROL
+#define OTG1_OTG_DLPC_CONTROL__OTG_RESYNC_MODE__SHIFT                                                         0x0
+#define OTG1_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION__SHIFT                                              0x10
+#define OTG1_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT__SHIFT                                               0x1f
+#define OTG1_OTG_DLPC_CONTROL__OTG_RESYNC_MODE_MASK                                                           0x00000001L
+#define OTG1_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION_MASK                                                0x7FFF0000L
+#define OTG1_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT_MASK                                                 0x80000000L
+//OTG1_OTG_PWA_FRAME_SYNC_CONTROL
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN__SHIFT                                         0x0
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE__SHIFT                                0x1
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE__SHIFT                                       0x10
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN_MASK                                           0x00000001L
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE_MASK                                  0x00000002L
+#define OTG1_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE_MASK                                         0x03FF0000L
+//OTG1_OTG_INTERLACE_CONTROL
+#define OTG1_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE__SHIFT                                               0x0
+#define OTG1_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD__SHIFT                                     0x10
+#define OTG1_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE_MASK                                                 0x00000001L
+#define OTG1_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD_MASK                                       0x00030000L
+//OTG1_OTG_INTERLACE_STATUS
+#define OTG1_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD__SHIFT                                         0x0
+#define OTG1_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD__SHIFT                                            0x1
+#define OTG1_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD_MASK                                           0x00000001L
+#define OTG1_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD_MASK                                              0x00000002L
+//OTG1_OTG_PIXEL_DATA_READBACK0
+#define OTG1_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB__SHIFT                                          0x0
+#define OTG1_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y__SHIFT                                          0x10
+#define OTG1_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB_MASK                                            0x0000FFFFL
+#define OTG1_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y_MASK                                            0xFFFF0000L
+//OTG1_OTG_PIXEL_DATA_READBACK1
+#define OTG1_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR__SHIFT                                           0x0
+#define OTG1_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR_MASK                                             0x0000FFFFL
+//OTG1_OTG_STATUS
+#define OTG1_OTG_STATUS__OTG_V_BLANK__SHIFT                                                                   0x0
+#define OTG1_OTG_STATUS__OTG_V_ACTIVE_DISP__SHIFT                                                             0x1
+#define OTG1_OTG_STATUS__OTG_V_SYNC_A__SHIFT                                                                  0x2
+#define OTG1_OTG_STATUS__OTG_V_UPDATE__SHIFT                                                                  0x3
+#define OTG1_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE__SHIFT                                                      0x5
+#define OTG1_OTG_STATUS__OTG_H_BLANK__SHIFT                                                                   0x10
+#define OTG1_OTG_STATUS__OTG_H_ACTIVE_DISP__SHIFT                                                             0x11
+#define OTG1_OTG_STATUS__OTG_H_SYNC_A__SHIFT                                                                  0x12
+#define OTG1_OTG_STATUS__OTG_V_BLANK_MASK                                                                     0x00000001L
+#define OTG1_OTG_STATUS__OTG_V_ACTIVE_DISP_MASK                                                               0x00000002L
+#define OTG1_OTG_STATUS__OTG_V_SYNC_A_MASK                                                                    0x00000004L
+#define OTG1_OTG_STATUS__OTG_V_UPDATE_MASK                                                                    0x00000008L
+#define OTG1_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE_MASK                                                        0x00000020L
+#define OTG1_OTG_STATUS__OTG_H_BLANK_MASK                                                                     0x00010000L
+#define OTG1_OTG_STATUS__OTG_H_ACTIVE_DISP_MASK                                                               0x00020000L
+#define OTG1_OTG_STATUS__OTG_H_SYNC_A_MASK                                                                    0x00040000L
+//OTG1_OTG_STATUS_POSITION
+#define OTG1_OTG_STATUS_POSITION__OTG_VERT_COUNT__SHIFT                                                       0x0
+#define OTG1_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK__SHIFT                                                 0xf
+#define OTG1_OTG_STATUS_POSITION__OTG_HORZ_COUNT__SHIFT                                                       0x10
+#define OTG1_OTG_STATUS_POSITION__OTG_VERT_COUNT_MASK                                                         0x00007FFFL
+#define OTG1_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK_MASK                                                   0x00008000L
+#define OTG1_OTG_STATUS_POSITION__OTG_HORZ_COUNT_MASK                                                         0x7FFF0000L
+//OTG1_OTG_LONG_VBLANK_STATUS
+#define OTG1_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT__SHIFT                                            0x0
+#define OTG1_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT_MASK                                              0xFFFFFFFFL
+//OTG1_OTG_NOM_VERT_POSITION
+#define OTG1_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM__SHIFT                                                 0x0
+#define OTG1_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM_MASK                                                   0x00007FFFL
+//OTG1_OTG_STATUS_FRAME_COUNT
+#define OTG1_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT__SHIFT                                                   0x0
+#define OTG1_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT_MASK                                                     0x00FFFFFFL
+//OTG1_OTG_STATUS_VF_COUNT
+#define OTG1_OTG_STATUS_VF_COUNT__OTG_VF_COUNT__SHIFT                                                         0x0
+#define OTG1_OTG_STATUS_VF_COUNT__OTG_VF_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG1_OTG_STATUS_HV_COUNT
+#define OTG1_OTG_STATUS_HV_COUNT__OTG_HV_COUNT__SHIFT                                                         0x0
+#define OTG1_OTG_STATUS_HV_COUNT__OTG_HV_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG1_OTG_COUNT_CONTROL
+#define OTG1_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN__SHIFT                                                  0x0
+#define OTG1_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT__SHIFT                                              0x1
+#define OTG1_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN_MASK                                                    0x00000001L
+#define OTG1_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT_MASK                                                0x0000001EL
+//OTG1_OTG_COUNT_RESET
+#define OTG1_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT__SHIFT                                                    0x0
+#define OTG1_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT_MASK                                                      0x00000001L
+//OTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE
+#define OTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__SHIFT                        0x0
+#define OTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_MASK                          0x00000001L
+//OTG1_OTG_VERT_SYNC_CONTROL
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED__SHIFT                                 0x0
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR__SHIFT                                    0x8
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE__SHIFT                                          0x10
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED_MASK                                   0x00000001L
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR_MASK                                      0x00000100L
+#define OTG1_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE_MASK                                            0x00030000L
+//OTG1_OTG_STEREO_STATUS
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE__SHIFT                                                 0x0
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT__SHIFT                                                 0x8
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT__SHIFT                                                 0x10
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG__SHIFT                                                    0x14
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING__SHIFT                                      0x18
+#define OTG1_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE__SHIFT                                         0x1e
+#define OTG1_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE__SHIFT                                        0x1f
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE_MASK                                                   0x00000001L
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT_MASK                                                   0x00000100L
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT_MASK                                                   0x00010000L
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG_MASK                                                      0x00100000L
+#define OTG1_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING_MASK                                        0x03000000L
+#define OTG1_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE_MASK                                           0x40000000L
+#define OTG1_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE_MASK                                          0x80000000L
+//OTG1_OTG_STEREO_CONTROL
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM__SHIFT                                       0x0
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY__SHIFT                                       0xf
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY__SHIFT                                          0x11
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP__SHIFT                                  0x12
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM__SHIFT                                                 0x13
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX__SHIFT                                        0x14
+#define OTG1_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL__SHIFT                                                     0x15
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_EN__SHIFT                                                         0x18
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM_MASK                                         0x00007FFFL
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY_MASK                                         0x00008000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY_MASK                                            0x00020000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP_MASK                                    0x00040000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM_MASK                                                   0x00080000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX_MASK                                          0x00100000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL_MASK                                                       0x00200000L
+#define OTG1_OTG_STEREO_CONTROL__OTG_STEREO_EN_MASK                                                           0x01000000L
+//OTG1_OTG_SNAPSHOT_STATUS
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED__SHIFT                                                0x0
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR__SHIFT                                                   0x1
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER__SHIFT                                          0x2
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED_MASK                                                  0x00000001L
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR_MASK                                                     0x00000002L
+#define OTG1_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER_MASK                                            0x00000004L
+//OTG1_OTG_SNAPSHOT_CONTROL
+#define OTG1_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL__SHIFT                                          0x0
+#define OTG1_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL_MASK                                            0x00000003L
+//OTG1_OTG_SNAPSHOT_POSITION
+#define OTG1_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT__SHIFT                                            0x0
+#define OTG1_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT__SHIFT                                            0x10
+#define OTG1_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT_MASK                                              0x00007FFFL
+#define OTG1_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT_MASK                                              0x7FFF0000L
+//OTG1_OTG_SNAPSHOT_FRAME
+#define OTG1_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT__SHIFT                                              0x0
+#define OTG1_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT_MASK                                                0x00FFFFFFL
+//OTG1_OTG_INTERRUPT_CONTROL
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK__SHIFT                                               0x0
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE__SHIFT                                              0x1
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK__SHIFT                                        0x8
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE__SHIFT                                       0x9
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK__SHIFT                                  0x10
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE__SHIFT                                 0x11
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK__SHIFT                                                  0x18
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK__SHIFT                                                  0x19
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE__SHIFT                                                 0x1a
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE__SHIFT                                                 0x1b
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK__SHIFT                                              0x1c
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE__SHIFT                                             0x1d
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK__SHIFT                                          0x1e
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE__SHIFT                                         0x1f
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK_MASK                                                 0x00000001L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE_MASK                                                0x00000002L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK_MASK                                          0x00000100L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE_MASK                                         0x00000200L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK_MASK                                    0x00010000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE_MASK                                   0x00020000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK_MASK                                                    0x01000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK_MASK                                                    0x02000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE_MASK                                                   0x04000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE_MASK                                                   0x08000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK_MASK                                                0x10000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE_MASK                                               0x20000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK_MASK                                            0x40000000L
+#define OTG1_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE_MASK                                           0x80000000L
+//OTG1_OTG_UPDATE_LOCK
+#define OTG1_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK__SHIFT                                                          0x0
+#define OTG1_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK_MASK                                                            0x00000001L
+//OTG1_OTG_DOUBLE_BUFFER_CONTROL
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING__SHIFT                                             0x0
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING__SHIFT                             0x4
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING__SHIFT                                   0x5
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING__SHIFT                                  0x6
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING__SHIFT                          0x7
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY__SHIFT                                           0x8
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING__SHIFT                                 0x9
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE__SHIFT                                0x18
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING_MASK                                               0x00000001L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING_MASK                               0x00000010L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING_MASK                                     0x00000020L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING_MASK                                    0x00000040L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING_MASK                            0x00000080L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY_MASK                                             0x00000100L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING_MASK                                   0x00000200L
+#define OTG1_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE_MASK                                  0x03000000L
+//OTG1_OTG_MASTER_EN
+#define OTG1_OTG_MASTER_EN__OTG_MASTER_EN__SHIFT                                                              0x0
+#define OTG1_OTG_MASTER_EN__OTG_MASTER_EN_MASK                                                                0x00000001L
+//OTG1_OTG_VERTICAL_INTERRUPT0_POSITION
+#define OTG1_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START__SHIFT                      0x0
+#define OTG1_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END__SHIFT                        0x10
+#define OTG1_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START_MASK                        0x00007FFFL
+#define OTG1_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END_MASK                          0x7FFF0000L
+//OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY__SHIFT                  0x4
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE__SHIFT                       0x8
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS__SHIFT                           0xc
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS__SHIFT                       0x10
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR__SHIFT                            0x14
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE__SHIFT                         0x18
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS__SHIFT                                         0x1c
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY_MASK                    0x00000010L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE_MASK                         0x00000100L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS_MASK                             0x00001000L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS_MASK                         0x00010000L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR_MASK                              0x00100000L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE_MASK                           0x01000000L
+#define OTG1_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS_MASK                                           0x10000000L
+//OTG1_OTG_VERTICAL_INTERRUPT1_POSITION
+#define OTG1_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START__SHIFT                      0x0
+#define OTG1_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START_MASK                        0x00007FFFL
+//OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE__SHIFT                       0x8
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS__SHIFT                           0xc
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS__SHIFT                       0x10
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR__SHIFT                            0x14
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE__SHIFT                         0x18
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE_MASK                         0x00000100L
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS_MASK                             0x00001000L
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS_MASK                         0x00010000L
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR_MASK                              0x00100000L
+#define OTG1_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE_MASK                           0x01000000L
+//OTG1_OTG_VERTICAL_INTERRUPT2_POSITION
+#define OTG1_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START__SHIFT                      0x0
+#define OTG1_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START_MASK                        0x00007FFFL
+//OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE__SHIFT                       0x8
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS__SHIFT                           0xc
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS__SHIFT                       0x10
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR__SHIFT                            0x14
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE__SHIFT                         0x18
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE_MASK                         0x00000100L
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS_MASK                             0x00001000L
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS_MASK                         0x00010000L
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR_MASK                              0x00100000L
+#define OTG1_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE_MASK                           0x01000000L
+//OTG1_OTG_CRC_CNTL
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_EN__SHIFT                                                                  0x0
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN__SHIFT                                                        0x1
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_POLY_SEL__SHIFT                                                            0x2
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY__SHIFT                                                          0x3
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CONT_EN__SHIFT                                                             0x4
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL__SHIFT                                                   0x5
+#define OTG1_OTG_CRC_CNTL__OTG_CRC1_EN__SHIFT                                                                 0x7
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE__SHIFT                                                         0x8
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CONT_MODE__SHIFT                                                           0xa
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE__SHIFT                                                      0xc
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_AES_EN__SHIFT                                                              0xe
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS__SHIFT                                         0x13
+#define OTG1_OTG_CRC_CNTL__OTG_CRC0_SELECT__SHIFT                                                             0x14
+#define OTG1_OTG_CRC_CNTL__OTG_CRC1_SELECT__SHIFT                                                             0x18
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING__SHIFT                                                   0x1c
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING__SHIFT                                                   0x1d
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING__SHIFT                                                   0x1e
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING__SHIFT                                                   0x1f
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_EN_MASK                                                                    0x00000001L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN_MASK                                                          0x00000002L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_POLY_SEL_MASK                                                              0x00000004L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY_MASK                                                            0x00000008L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CONT_EN_MASK                                                               0x00000010L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL_MASK                                                     0x00000060L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC1_EN_MASK                                                                   0x00000080L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE_MASK                                                           0x00000300L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_CONT_MODE_MASK                                                             0x00000400L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE_MASK                                                        0x00003000L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_AES_EN_MASK                                                                0x00004000L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS_MASK                                           0x00080000L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC0_SELECT_MASK                                                               0x00700000L
+#define OTG1_OTG_CRC_CNTL__OTG_CRC1_SELECT_MASK                                                               0x07000000L
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING_MASK                                                     0x10000000L
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING_MASK                                                     0x20000000L
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING_MASK                                                     0x40000000L
+#define OTG1_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING_MASK                                                     0x80000000L
+//OTG1_OTG_CRC0_WINDOWA_X_CONTROL
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWA_Y_CONTROL
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWB_X_CONTROL
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWB_Y_CONTROL
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC0_DATA_R
+#define OTG1_OTG_CRC0_DATA_R__CRC0_R_CR__SHIFT                                                                0x0
+#define OTG1_OTG_CRC0_DATA_R__CRC0_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC0_DATA_G
+#define OTG1_OTG_CRC0_DATA_G__CRC0_G_Y__SHIFT                                                                 0x0
+#define OTG1_OTG_CRC0_DATA_G__CRC0_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG1_OTG_CRC0_DATA_B
+#define OTG1_OTG_CRC0_DATA_B__CRC0_B_CB__SHIFT                                                                0x0
+#define OTG1_OTG_CRC0_DATA_B__CRC0_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC0_DATA_C
+#define OTG1_OTG_CRC0_DATA_C__CRC0_C__SHIFT                                                                   0x0
+#define OTG1_OTG_CRC0_DATA_C__CRC0_C_MASK                                                                     0xFFFFFFFFL
+//OTG1_OTG_CRC1_WINDOWA_X_CONTROL
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWA_Y_CONTROL
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWB_X_CONTROL
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWB_Y_CONTROL
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG1_OTG_CRC1_DATA_R
+#define OTG1_OTG_CRC1_DATA_R__CRC1_R_CR__SHIFT                                                                0x0
+#define OTG1_OTG_CRC1_DATA_R__CRC1_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC1_DATA_G
+#define OTG1_OTG_CRC1_DATA_G__CRC1_G_Y__SHIFT                                                                 0x0
+#define OTG1_OTG_CRC1_DATA_G__CRC1_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG1_OTG_CRC1_DATA_B
+#define OTG1_OTG_CRC1_DATA_B__CRC1_B_CB__SHIFT                                                                0x0
+#define OTG1_OTG_CRC1_DATA_B__CRC1_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC1_DATA_C
+#define OTG1_OTG_CRC1_DATA_C__CRC1_C__SHIFT                                                                   0x0
+#define OTG1_OTG_CRC1_DATA_C__CRC1_C_MASK                                                                     0xFFFFFFFFL
+//OTG1_OTG_CRC2_DATA_R
+#define OTG1_OTG_CRC2_DATA_R__CRC2_R_CR__SHIFT                                                                0x0
+#define OTG1_OTG_CRC2_DATA_R__CRC2_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC2_DATA_G
+#define OTG1_OTG_CRC2_DATA_G__CRC2_G_Y__SHIFT                                                                 0x0
+#define OTG1_OTG_CRC2_DATA_G__CRC2_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG1_OTG_CRC2_DATA_B
+#define OTG1_OTG_CRC2_DATA_B__CRC2_B_CB__SHIFT                                                                0x0
+#define OTG1_OTG_CRC2_DATA_B__CRC2_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC2_DATA_C
+#define OTG1_OTG_CRC2_DATA_C__CRC2_C__SHIFT                                                                   0x0
+#define OTG1_OTG_CRC2_DATA_C__CRC2_C_MASK                                                                     0xFFFFFFFFL
+//OTG1_OTG_CRC3_DATA_R
+#define OTG1_OTG_CRC3_DATA_R__CRC3_R_CR__SHIFT                                                                0x0
+#define OTG1_OTG_CRC3_DATA_R__CRC3_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC3_DATA_G
+#define OTG1_OTG_CRC3_DATA_G__CRC3_G_Y__SHIFT                                                                 0x0
+#define OTG1_OTG_CRC3_DATA_G__CRC3_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG1_OTG_CRC3_DATA_B
+#define OTG1_OTG_CRC3_DATA_B__CRC3_B_CB__SHIFT                                                                0x0
+#define OTG1_OTG_CRC3_DATA_B__CRC3_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG1_OTG_CRC3_DATA_C
+#define OTG1_OTG_CRC3_DATA_C__CRC3_C__SHIFT                                                                   0x0
+#define OTG1_OTG_CRC3_DATA_C__CRC3_C_MASK                                                                     0xFFFFFFFFL
+//OTG1_OTG_CRC0_DATA_AES
+#define OTG1_OTG_CRC0_DATA_AES__CRC0_AES__SHIFT                                                               0x0
+#define OTG1_OTG_CRC0_DATA_AES__CRC0_AES_MASK                                                                 0xFFFFFFFFL
+//OTG1_OTG_CRC_SIG_RED_GREEN_MASK
+#define OTG1_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK__SHIFT                                          0x0
+#define OTG1_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK__SHIFT                                        0x10
+#define OTG1_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK_MASK                                            0x0000FFFFL
+#define OTG1_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK_MASK                                          0xFFFF0000L
+//OTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK
+#define OTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK__SHIFT                                      0x0
+#define OTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK__SHIFT                                   0x10
+#define OTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK_MASK                                        0x0000FFFFL
+#define OTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK_MASK                                     0xFFFF0000L
+//OTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG1_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG1_OTG_STATIC_SCREEN_CONTROL
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK__SHIFT                                   0x0
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT__SHIFT                                  0x10
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE__SHIFT                                          0x18
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS__SHIFT                                                  0x19
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS__SHIFT                                          0x1a
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR__SHIFT                                           0x1b
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE__SHIFT                                            0x1c
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE__SHIFT                                     0x1e
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE__SHIFT                               0x1f
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK_MASK                                     0x0000FFFFL
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT_MASK                                    0x00FF0000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE_MASK                                            0x01000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS_MASK                                                    0x02000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS_MASK                                            0x04000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR_MASK                                             0x08000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE_MASK                                              0x10000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_MASK                                       0x40000000L
+#define OTG1_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE_MASK                                 0x80000000L
+//OTG1_OTG_3D_STRUCTURE_CONTROL
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN__SHIFT                                             0x0
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE__SHIFT                                  0x8
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR__SHIFT                                 0xc
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET__SHIFT                                  0x10
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING__SHIFT                          0x11
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT__SHIFT                                        0x12
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN_MASK                                               0x00000001L
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE_MASK                                    0x00000300L
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR_MASK                                   0x00001000L
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_MASK                                    0x00010000L
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING_MASK                            0x00020000L
+#define OTG1_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_MASK                                          0x000C0000L
+//OTG1_OTG_GSL_VSYNC_GAP
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT__SHIFT                                                0x0
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY__SHIFT                                                0x8
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL__SHIFT                                           0x10
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE__SHIFT                                                 0x11
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR__SHIFT                                                0x13
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED__SHIFT                                             0x14
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER__SHIFT                                        0x17
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP__SHIFT                                                      0x18
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT_MASK                                                  0x000000FFL
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY_MASK                                                  0x0000FF00L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL_MASK                                             0x00010000L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE_MASK                                                   0x00060000L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR_MASK                                                  0x00080000L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED_MASK                                               0x00100000L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER_MASK                                          0x00800000L
+#define OTG1_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASK                                                        0xFF000000L
+//OTG1_OTG_MASTER_UPDATE_MODE
+#define OTG1_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE__SHIFT                                     0x0
+#define OTG1_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE_MASK                                       0x00000003L
+//OTG1_OTG_CLOCK_CONTROL
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_EN__SHIFT                                                           0x0
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS__SHIFT                                                     0x1
+#define OTG1_OTG_CLOCK_CONTROL__OTG_SOFT_RESET__SHIFT                                                         0x4
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_ON__SHIFT                                                           0x8
+#define OTG1_OTG_CLOCK_CONTROL__OTG_BUSY__SHIFT                                                               0x10
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_EN_MASK                                                             0x00000001L
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS_MASK                                                       0x00000002L
+#define OTG1_OTG_CLOCK_CONTROL__OTG_SOFT_RESET_MASK                                                           0x00000010L
+#define OTG1_OTG_CLOCK_CONTROL__OTG_CLOCK_ON_MASK                                                             0x00000100L
+#define OTG1_OTG_CLOCK_CONTROL__OTG_BUSY_MASK                                                                 0x00010000L
+//OTG1_OTG_VSTARTUP_PARAM
+#define OTG1_OTG_VSTARTUP_PARAM__VSTARTUP_START__SHIFT                                                        0x0
+#define OTG1_OTG_VSTARTUP_PARAM__VSTARTUP_START_MASK                                                          0x000003FFL
+//OTG1_OTG_VUPDATE_PARAM
+#define OTG1_OTG_VUPDATE_PARAM__VUPDATE_OFFSET__SHIFT                                                         0x0
+#define OTG1_OTG_VUPDATE_PARAM__VUPDATE_WIDTH__SHIFT                                                          0x10
+#define OTG1_OTG_VUPDATE_PARAM__VUPDATE_OFFSET_MASK                                                           0x0000FFFFL
+#define OTG1_OTG_VUPDATE_PARAM__VUPDATE_WIDTH_MASK                                                            0x03FF0000L
+//OTG1_OTG_VREADY_PARAM
+#define OTG1_OTG_VREADY_PARAM__VREADY_OFFSET__SHIFT                                                           0x0
+#define OTG1_OTG_VREADY_PARAM__VREADY_OFFSET_MASK                                                             0x0000FFFFL
+//OTG1_OTG_GLOBAL_SYNC_STATUS
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN__SHIFT                                                   0x0
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE__SHIFT                                                 0x1
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED__SHIFT                                           0x2
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS__SHIFT                                               0x3
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR__SHIFT                                              0x4
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN__SHIFT                                                    0x5
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE__SHIFT                                                  0x6
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL__SHIFT                                          0x7
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED__SHIFT                                            0x8
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS__SHIFT                                                0x9
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR__SHIFT                                               0xa
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS__SHIFT                                                    0xb
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN__SHIFT                                            0xc
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE__SHIFT                                          0xd
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED__SHIFT                                    0xe
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS__SHIFT                                        0xf
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR__SHIFT                                       0x10
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS__SHIFT                                            0x11
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN__SHIFT                                                     0x12
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE__SHIFT                                                   0x13
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED__SHIFT                                             0x14
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS__SHIFT                                                 0x15
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR__SHIFT                                                0x16
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS__SHIFT                                              0x18
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS__SHIFT                                               0x19
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN_MASK                                                     0x00000001L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE_MASK                                                   0x00000002L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED_MASK                                             0x00000004L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS_MASK                                                 0x00000008L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR_MASK                                                0x00000010L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN_MASK                                                      0x00000020L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE_MASK                                                    0x00000040L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL_MASK                                            0x00000080L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED_MASK                                              0x00000100L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS_MASK                                                  0x00000200L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR_MASK                                                 0x00000400L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS_MASK                                                      0x00000800L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN_MASK                                              0x00001000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE_MASK                                            0x00002000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED_MASK                                      0x00004000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS_MASK                                          0x00008000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR_MASK                                         0x00010000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS_MASK                                              0x00020000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN_MASK                                                       0x00040000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE_MASK                                                     0x00080000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED_MASK                                               0x00100000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS_MASK                                                   0x00200000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR_MASK                                                  0x00400000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS_MASK                                                0x01000000L
+#define OTG1_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS_MASK                                                 0x02000000L
+//OTG1_OTG_MASTER_UPDATE_LOCK
+#define OTG1_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK__SHIFT                                            0x0
+#define OTG1_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS__SHIFT                                                0x8
+#define OTG1_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK_MASK                                              0x00000001L
+#define OTG1_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS_MASK                                                  0x00000100L
+//OTG1_OTG_GSL_CONTROL
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL0_EN__SHIFT                                                              0x0
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL1_EN__SHIFT                                                              0x1
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL2_EN__SHIFT                                                              0x2
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN__SHIFT                                                        0x3
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE__SHIFT                                                      0x4
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY__SHIFT                                                      0x8
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY__SHIFT                                                      0x10
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS__SHIFT                                                 0x1c
+#define OTG1_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN__SHIFT                                            0x1f
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL0_EN_MASK                                                                0x00000001L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL1_EN_MASK                                                                0x00000002L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL2_EN_MASK                                                                0x00000004L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN_MASK                                                          0x00000008L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE_MASK                                                        0x00000030L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY_MASK                                                        0x00000F00L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY_MASK                                                        0x001F0000L
+#define OTG1_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS_MASK                                                   0x10000000L
+#define OTG1_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN_MASK                                              0x80000000L
+//OTG1_OTG_GSL_WINDOW_X
+#define OTG1_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X__SHIFT                                                  0x0
+#define OTG1_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X__SHIFT                                                    0x10
+#define OTG1_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X_MASK                                                    0x00007FFFL
+#define OTG1_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X_MASK                                                      0x7FFF0000L
+//OTG1_OTG_GSL_WINDOW_Y
+#define OTG1_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y__SHIFT                                                  0x0
+#define OTG1_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y__SHIFT                                                    0x10
+#define OTG1_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y_MASK                                                    0x00007FFFL
+#define OTG1_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y_MASK                                                      0x7FFF0000L
+//OTG1_OTG_VUPDATE_KEEPOUT
+#define OTG1_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET__SHIFT                      0x0
+#define OTG1_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET__SHIFT                        0x10
+#define OTG1_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN__SHIFT                            0x1f
+#define OTG1_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET_MASK                        0x0000FFFFL
+#define OTG1_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET_MASK                          0x03FF0000L
+#define OTG1_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN_MASK                              0x80000000L
+//OTG1_OTG_GLOBAL_CONTROL0
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X__SHIFT                                        0x0
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X__SHIFT                                          0x10
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN__SHIFT                                             0x1f
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X_MASK                                          0x00007FFFL
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X_MASK                                            0x7FFF0000L
+#define OTG1_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN_MASK                                               0x80000000L
+//OTG1_OTG_GLOBAL_CONTROL1
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y__SHIFT                                        0x0
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y__SHIFT                                          0x10
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE__SHIFT                                       0x1f
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y_MASK                                          0x00007FFFL
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y_MASK                                            0x7FFF0000L
+#define OTG1_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE_MASK                                         0x80000000L
+//OTG1_OTG_GLOBAL_CONTROL2
+#define OTG1_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN__SHIFT                                                0xa
+#define OTG1_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL__SHIFT                                              0x10
+#define OTG1_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL__SHIFT                                           0x19
+#define OTG1_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE__SHIFT                                            0x1e
+#define OTG1_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE__SHIFT                                                    0x1f
+#define OTG1_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN_MASK                                                  0x00000400L
+#define OTG1_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL_MASK                                                0x00070000L
+#define OTG1_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL_MASK                                             0x0E000000L
+#define OTG1_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE_MASK                                              0x40000000L
+#define OTG1_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE_MASK                                                      0x80000000L
+//OTG1_OTG_GLOBAL_CONTROL3
+#define OTG1_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD__SHIFT                                          0x0
+#define OTG1_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL__SHIFT                                     0x4
+#define OTG1_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL__SHIFT                                                 0x10
+#define OTG1_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL__SHIFT                                                   0x14
+#define OTG1_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD_MASK                                            0x00000003L
+#define OTG1_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL_MASK                                       0x00000030L
+#define OTG1_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL_MASK                                                   0x00030000L
+#define OTG1_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL_MASK                                                     0x00300000L
+//OTG1_OTG_GLOBAL_CONTROL4
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X__SHIFT                                                0x0
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y__SHIFT                                                0x10
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE__SHIFT                                               0x1f
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X_MASK                                                  0x00007FFFL
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y_MASK                                                  0x7FFF0000L
+#define OTG1_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE_MASK                                                 0x80000000L
+//OTG1_OTG_TRIG_MANUAL_CONTROL
+#define OTG1_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL__SHIFT                                              0x0
+#define OTG1_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL_MASK                                                0x00000001L
+//OTG1_OTG_DRR_TIMING_INT_STATUS
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED__SHIFT                                 0x0
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT__SHIFT                             0x4
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR__SHIFT                           0x8
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK__SHIFT                         0xc
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE__SHIFT                        0xd
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED__SHIFT                                 0x10
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT__SHIFT                             0x14
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR__SHIFT                           0x18
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK__SHIFT                         0x1c
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE__SHIFT                        0x1d
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_MASK                                   0x00000001L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MASK                               0x00000010L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR_MASK                             0x00000100L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK_MASK                           0x00001000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE_MASK                          0x00002000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_MASK                                   0x00010000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MASK                               0x00100000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR_MASK                             0x01000000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK_MASK                           0x10000000L
+#define OTG1_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE_MASK                          0x20000000L
+//OTG1_OTG_DRR_V_TOTAL_REACH_RANGE
+#define OTG1_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE__SHIFT                            0x0
+#define OTG1_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE__SHIFT                            0x10
+#define OTG1_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE_MASK                              0x00007FFFL
+#define OTG1_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE_MASK                              0x7FFF0000L
+//OTG1_OTG_DRR_V_TOTAL_CHANGE
+#define OTG1_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT__SHIFT                                      0x0
+#define OTG1_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT_MASK                                        0x00007FFFL
+//OTG1_OTG_DRR_TRIGGER_WINDOW
+#define OTG1_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X__SHIFT                                    0x0
+#define OTG1_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X__SHIFT                                      0x10
+#define OTG1_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X_MASK                                      0x00007FFFL
+#define OTG1_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X_MASK                                        0x7FFF0000L
+//OTG1_OTG_DRR_CONTROL
+#define OTG1_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME__SHIFT                                                    0x0
+#define OTG1_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR__SHIFT                                             0x10
+#define OTG1_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME_MASK                                                      0x00000003L
+#define OTG1_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR_MASK                                               0x7FFF0000L
+//OTG1_OTG_DRR_CONTOL2
+#define OTG1_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR__SHIFT                                             0x0
+#define OTG1_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR_MASK                                               0xFFFFFFFFL
+//OTG1_OTG_M_CONST_DTO0
+#define OTG1_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE__SHIFT                                                   0x0
+#define OTG1_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE_MASK                                                     0xFFFFFFFFL
+//OTG1_OTG_M_CONST_DTO1
+#define OTG1_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO__SHIFT                                                  0x0
+#define OTG1_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO_MASK                                                    0xFFFFFFFFL
+//OTG1_OTG_REQUEST_CONTROL
+#define OTG1_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE__SHIFT                                     0x0
+#define OTG1_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE_MASK                                       0x00000001L
+//OTG1_OTG_PIPE_UPDATE_STATUS
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING__SHIFT                                                  0x0
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING__SHIFT                                         0x4
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING__SHIFT                                         0x8
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS__SHIFT                                        0x10
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING_MASK                                                    0x00000001L
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING_MASK                                           0x00000010L
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING_MASK                                           0x00000100L
+#define OTG1_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS_MASK                                          0x00010000L
+//OTG1_OTG_PSTATE_REGISTER
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START__SHIFT                                             0x0
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND__SHIFT                                                    0x10
+#define OTG1_OTG_PSTATE_REGISTER__OTG_UNBLANK__SHIFT                                                          0x14
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN__SHIFT                                           0x18
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START_MASK                                               0x00007FFFL
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND_MASK                                                      0x00010000L
+#define OTG1_OTG_PSTATE_REGISTER__OTG_UNBLANK_MASK                                                            0x00100000L
+#define OTG1_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN_MASK                                             0xFF000000L
+//OTG1_OTG_ENCRYPTION
+#define OTG1_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL__SHIFT                                             0x0
+#define OTG1_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL_MASK                                               0x00000003L
+//OTG1_OTG_SPARE_REGISTER
+#define OTG1_OTG_SPARE_REGISTER__OTG_SPARE_REG__SHIFT                                                         0x0
+#define OTG1_OTG_SPARE_REGISTER__OTG_SPARE_REG_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_otg2_dispdec
+//OTG2_OTG_H_TOTAL
+#define OTG2_OTG_H_TOTAL__OTG_H_TOTAL__SHIFT                                                                  0x0
+#define OTG2_OTG_H_TOTAL__OTG_H_TOTAL_MASK                                                                    0x00007FFFL
+//OTG2_OTG_H_BLANK_START_END
+#define OTG2_OTG_H_BLANK_START_END__OTG_H_BLANK_START__SHIFT                                                  0x0
+#define OTG2_OTG_H_BLANK_START_END__OTG_H_BLANK_END__SHIFT                                                    0x10
+#define OTG2_OTG_H_BLANK_START_END__OTG_H_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG2_OTG_H_BLANK_START_END__OTG_H_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG2_OTG_H_SYNC_A
+#define OTG2_OTG_H_SYNC_A__OTG_H_SYNC_A_START__SHIFT                                                          0x0
+#define OTG2_OTG_H_SYNC_A__OTG_H_SYNC_A_END__SHIFT                                                            0x10
+#define OTG2_OTG_H_SYNC_A__OTG_H_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG2_OTG_H_SYNC_A__OTG_H_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG2_OTG_H_SYNC_A_CNTL
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN__SHIFT                                                     0x10
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF__SHIFT                                                    0x11
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN_MASK                                                       0x00010000L
+#define OTG2_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF_MASK                                                      0x00020000L
+//OTG2_OTG_H_TIMING_CNTL
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE__SHIFT                                                  0x0
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL__SHIFT                                           0x8
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR__SHIFT                                             0x10
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MASK                                                    0x00000003L
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL_MASK                                             0x00000100L
+#define OTG2_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR_MASK                                               0x00030000L
+//OTG2_OTG_V_TOTAL
+#define OTG2_OTG_V_TOTAL__OTG_V_TOTAL__SHIFT                                                                  0x0
+#define OTG2_OTG_V_TOTAL__OTG_V_TOTAL_MASK                                                                    0x00007FFFL
+//OTG2_OTG_V_TOTAL_MIN
+#define OTG2_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN__SHIFT                                                          0x0
+#define OTG2_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN_MASK                                                            0x00007FFFL
+//OTG2_OTG_V_TOTAL_MAX
+#define OTG2_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX__SHIFT                                                          0x0
+#define OTG2_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX_MASK                                                            0x00007FFFL
+//OTG2_OTG_V_TOTAL_MID
+#define OTG2_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID__SHIFT                                                          0x0
+#define OTG2_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID_MASK                                                            0x00007FFFL
+//OTG2_OTG_V_TOTAL_CONTROL
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL__SHIFT                                                  0x0
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL__SHIFT                                                  0x1
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN__SHIFT                                      0x2
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN__SHIFT                                      0x3
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT__SHIFT                                              0x4
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD__SHIFT                                          0x5
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM__SHIFT                                             0x8
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK__SHIFT                                             0x10
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL_MASK                                                    0x00000001L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL_MASK                                                    0x00000002L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN_MASK                                        0x00000004L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN_MASK                                        0x00000008L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT_MASK                                                0x00000010L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD_MASK                                            0x00000020L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM_MASK                                               0x0000FF00L
+#define OTG2_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK_MASK                                               0xFFFF0000L
+//OTG2_OTG_V_COUNT_STOP_CONTROL
+#define OTG2_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP__SHIFT                                                0x0
+#define OTG2_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP_MASK                                                  0x00007FFFL
+//OTG2_OTG_V_COUNT_STOP_CONTROL2
+#define OTG2_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER__SHIFT                                         0x0
+#define OTG2_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER_MASK                                           0xFFFFFFFFL
+//OTG2_OTG_V_TOTAL_INT_STATUS
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED__SHIFT                                0x0
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT__SHIFT                            0x4
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK__SHIFT                            0x8
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK__SHIFT                            0xc
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MASK                                  0x00000001L
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT_MASK                              0x00000010L
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK_MASK                              0x00000100L
+#define OTG2_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK_MASK                              0x00001000L
+//OTG2_OTG_VSYNC_NOM_INT_STATUS
+#define OTG2_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM__SHIFT                                                   0x0
+#define OTG2_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR__SHIFT                                         0x4
+#define OTG2_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_MASK                                                     0x00000001L
+#define OTG2_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR_MASK                                           0x00000010L
+//OTG2_OTG_V_BLANK_START_END
+#define OTG2_OTG_V_BLANK_START_END__OTG_V_BLANK_START__SHIFT                                                  0x0
+#define OTG2_OTG_V_BLANK_START_END__OTG_V_BLANK_END__SHIFT                                                    0x10
+#define OTG2_OTG_V_BLANK_START_END__OTG_V_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG2_OTG_V_BLANK_START_END__OTG_V_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG2_OTG_V_SYNC_A
+#define OTG2_OTG_V_SYNC_A__OTG_V_SYNC_A_START__SHIFT                                                          0x0
+#define OTG2_OTG_V_SYNC_A__OTG_V_SYNC_A_END__SHIFT                                                            0x10
+#define OTG2_OTG_V_SYNC_A__OTG_V_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG2_OTG_V_SYNC_A__OTG_V_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG2_OTG_V_SYNC_A_CNTL
+#define OTG2_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG2_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE__SHIFT                                                        0x8
+#define OTG2_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG2_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE_MASK                                                          0x00000100L
+//OTG2_OTG_TRIGA_CNTL
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED__SHIFT                                                        0xe
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY__SHIFT                                                           0x18
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR__SHIFT                                                           0x1f
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED_MASK                                                          0x00004000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY_MASK                                                             0x1F000000L
+#define OTG2_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR_MASK                                                             0x80000000L
+//OTG2_OTG_TRIGA_MANUAL_TRIG
+#define OTG2_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG2_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG2_OTG_TRIGB_CNTL
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED__SHIFT                                                        0xe
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY__SHIFT                                                           0x18
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR__SHIFT                                                           0x1f
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED_MASK                                                          0x00004000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY_MASK                                                             0x1F000000L
+#define OTG2_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR_MASK                                                             0x80000000L
+//OTG2_OTG_TRIGB_MANUAL_TRIG
+#define OTG2_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG2_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG2_OTG_FORCE_COUNT_NOW_CNTL
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE__SHIFT                                        0x0
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK__SHIFT                                       0x4
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL__SHIFT                                    0x8
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED__SHIFT                                    0x10
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR__SHIFT                                       0x18
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE_MASK                                          0x00000003L
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK_MASK                                         0x00000010L
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL_MASK                                      0x00000100L
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED_MASK                                      0x00010000L
+#define OTG2_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR_MASK                                         0x01000000L
+//OTG2_OTG_STEREO_FORCE_NEXT_EYE
+#define OTG2_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE__SHIFT                                      0x0
+#define OTG2_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE_MASK                                        0x00000003L
+//OTG2_OTG_CONTROL
+#define OTG2_OTG_CONTROL__OTG_MASTER_EN__SHIFT                                                                0x0
+#define OTG2_OTG_CONTROL__OTG_DISABLE_POINT_CNTL__SHIFT                                                       0x8
+#define OTG2_OTG_CONTROL__OTG_START_POINT_CNTL__SHIFT                                                         0xc
+#define OTG2_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL__SHIFT                                                        0xd
+#define OTG2_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY__SHIFT                                                    0xe
+#define OTG2_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE__SHIFT                                                  0x10
+#define OTG2_OTG_CONTROL__OTG_OUT_MUX__SHIFT                                                                  0x14
+#define OTG2_OTG_CONTROL__OTG_MASTER_EN_MASK                                                                  0x00000001L
+#define OTG2_OTG_CONTROL__OTG_DISABLE_POINT_CNTL_MASK                                                         0x00000300L
+#define OTG2_OTG_CONTROL__OTG_START_POINT_CNTL_MASK                                                           0x00001000L
+#define OTG2_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL_MASK                                                          0x00002000L
+#define OTG2_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY_MASK                                                      0x00004000L
+#define OTG2_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE_MASK                                                    0x00010000L
+#define OTG2_OTG_CONTROL__OTG_OUT_MUX_MASK                                                                    0x00300000L
+//OTG2_OTG_DLPC_CONTROL
+#define OTG2_OTG_DLPC_CONTROL__OTG_RESYNC_MODE__SHIFT                                                         0x0
+#define OTG2_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION__SHIFT                                              0x10
+#define OTG2_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT__SHIFT                                               0x1f
+#define OTG2_OTG_DLPC_CONTROL__OTG_RESYNC_MODE_MASK                                                           0x00000001L
+#define OTG2_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION_MASK                                                0x7FFF0000L
+#define OTG2_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT_MASK                                                 0x80000000L
+//OTG2_OTG_PWA_FRAME_SYNC_CONTROL
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN__SHIFT                                         0x0
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE__SHIFT                                0x1
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE__SHIFT                                       0x10
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN_MASK                                           0x00000001L
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE_MASK                                  0x00000002L
+#define OTG2_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE_MASK                                         0x03FF0000L
+//OTG2_OTG_INTERLACE_CONTROL
+#define OTG2_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE__SHIFT                                               0x0
+#define OTG2_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD__SHIFT                                     0x10
+#define OTG2_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE_MASK                                                 0x00000001L
+#define OTG2_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD_MASK                                       0x00030000L
+//OTG2_OTG_INTERLACE_STATUS
+#define OTG2_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD__SHIFT                                         0x0
+#define OTG2_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD__SHIFT                                            0x1
+#define OTG2_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD_MASK                                           0x00000001L
+#define OTG2_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD_MASK                                              0x00000002L
+//OTG2_OTG_PIXEL_DATA_READBACK0
+#define OTG2_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB__SHIFT                                          0x0
+#define OTG2_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y__SHIFT                                          0x10
+#define OTG2_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB_MASK                                            0x0000FFFFL
+#define OTG2_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y_MASK                                            0xFFFF0000L
+//OTG2_OTG_PIXEL_DATA_READBACK1
+#define OTG2_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR__SHIFT                                           0x0
+#define OTG2_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR_MASK                                             0x0000FFFFL
+//OTG2_OTG_STATUS
+#define OTG2_OTG_STATUS__OTG_V_BLANK__SHIFT                                                                   0x0
+#define OTG2_OTG_STATUS__OTG_V_ACTIVE_DISP__SHIFT                                                             0x1
+#define OTG2_OTG_STATUS__OTG_V_SYNC_A__SHIFT                                                                  0x2
+#define OTG2_OTG_STATUS__OTG_V_UPDATE__SHIFT                                                                  0x3
+#define OTG2_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE__SHIFT                                                      0x5
+#define OTG2_OTG_STATUS__OTG_H_BLANK__SHIFT                                                                   0x10
+#define OTG2_OTG_STATUS__OTG_H_ACTIVE_DISP__SHIFT                                                             0x11
+#define OTG2_OTG_STATUS__OTG_H_SYNC_A__SHIFT                                                                  0x12
+#define OTG2_OTG_STATUS__OTG_V_BLANK_MASK                                                                     0x00000001L
+#define OTG2_OTG_STATUS__OTG_V_ACTIVE_DISP_MASK                                                               0x00000002L
+#define OTG2_OTG_STATUS__OTG_V_SYNC_A_MASK                                                                    0x00000004L
+#define OTG2_OTG_STATUS__OTG_V_UPDATE_MASK                                                                    0x00000008L
+#define OTG2_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE_MASK                                                        0x00000020L
+#define OTG2_OTG_STATUS__OTG_H_BLANK_MASK                                                                     0x00010000L
+#define OTG2_OTG_STATUS__OTG_H_ACTIVE_DISP_MASK                                                               0x00020000L
+#define OTG2_OTG_STATUS__OTG_H_SYNC_A_MASK                                                                    0x00040000L
+//OTG2_OTG_STATUS_POSITION
+#define OTG2_OTG_STATUS_POSITION__OTG_VERT_COUNT__SHIFT                                                       0x0
+#define OTG2_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK__SHIFT                                                 0xf
+#define OTG2_OTG_STATUS_POSITION__OTG_HORZ_COUNT__SHIFT                                                       0x10
+#define OTG2_OTG_STATUS_POSITION__OTG_VERT_COUNT_MASK                                                         0x00007FFFL
+#define OTG2_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK_MASK                                                   0x00008000L
+#define OTG2_OTG_STATUS_POSITION__OTG_HORZ_COUNT_MASK                                                         0x7FFF0000L
+//OTG2_OTG_LONG_VBLANK_STATUS
+#define OTG2_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT__SHIFT                                            0x0
+#define OTG2_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT_MASK                                              0xFFFFFFFFL
+//OTG2_OTG_NOM_VERT_POSITION
+#define OTG2_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM__SHIFT                                                 0x0
+#define OTG2_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM_MASK                                                   0x00007FFFL
+//OTG2_OTG_STATUS_FRAME_COUNT
+#define OTG2_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT__SHIFT                                                   0x0
+#define OTG2_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT_MASK                                                     0x00FFFFFFL
+//OTG2_OTG_STATUS_VF_COUNT
+#define OTG2_OTG_STATUS_VF_COUNT__OTG_VF_COUNT__SHIFT                                                         0x0
+#define OTG2_OTG_STATUS_VF_COUNT__OTG_VF_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG2_OTG_STATUS_HV_COUNT
+#define OTG2_OTG_STATUS_HV_COUNT__OTG_HV_COUNT__SHIFT                                                         0x0
+#define OTG2_OTG_STATUS_HV_COUNT__OTG_HV_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG2_OTG_COUNT_CONTROL
+#define OTG2_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN__SHIFT                                                  0x0
+#define OTG2_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT__SHIFT                                              0x1
+#define OTG2_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN_MASK                                                    0x00000001L
+#define OTG2_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT_MASK                                                0x0000001EL
+//OTG2_OTG_COUNT_RESET
+#define OTG2_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT__SHIFT                                                    0x0
+#define OTG2_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT_MASK                                                      0x00000001L
+//OTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE
+#define OTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__SHIFT                        0x0
+#define OTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_MASK                          0x00000001L
+//OTG2_OTG_VERT_SYNC_CONTROL
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED__SHIFT                                 0x0
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR__SHIFT                                    0x8
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE__SHIFT                                          0x10
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED_MASK                                   0x00000001L
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR_MASK                                      0x00000100L
+#define OTG2_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE_MASK                                            0x00030000L
+//OTG2_OTG_STEREO_STATUS
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE__SHIFT                                                 0x0
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT__SHIFT                                                 0x8
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT__SHIFT                                                 0x10
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG__SHIFT                                                    0x14
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING__SHIFT                                      0x18
+#define OTG2_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE__SHIFT                                         0x1e
+#define OTG2_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE__SHIFT                                        0x1f
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE_MASK                                                   0x00000001L
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT_MASK                                                   0x00000100L
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT_MASK                                                   0x00010000L
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG_MASK                                                      0x00100000L
+#define OTG2_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING_MASK                                        0x03000000L
+#define OTG2_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE_MASK                                           0x40000000L
+#define OTG2_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE_MASK                                          0x80000000L
+//OTG2_OTG_STEREO_CONTROL
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM__SHIFT                                       0x0
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY__SHIFT                                       0xf
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY__SHIFT                                          0x11
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP__SHIFT                                  0x12
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM__SHIFT                                                 0x13
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX__SHIFT                                        0x14
+#define OTG2_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL__SHIFT                                                     0x15
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_EN__SHIFT                                                         0x18
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM_MASK                                         0x00007FFFL
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY_MASK                                         0x00008000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY_MASK                                            0x00020000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP_MASK                                    0x00040000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM_MASK                                                   0x00080000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX_MASK                                          0x00100000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL_MASK                                                       0x00200000L
+#define OTG2_OTG_STEREO_CONTROL__OTG_STEREO_EN_MASK                                                           0x01000000L
+//OTG2_OTG_SNAPSHOT_STATUS
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED__SHIFT                                                0x0
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR__SHIFT                                                   0x1
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER__SHIFT                                          0x2
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED_MASK                                                  0x00000001L
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR_MASK                                                     0x00000002L
+#define OTG2_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER_MASK                                            0x00000004L
+//OTG2_OTG_SNAPSHOT_CONTROL
+#define OTG2_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL__SHIFT                                          0x0
+#define OTG2_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL_MASK                                            0x00000003L
+//OTG2_OTG_SNAPSHOT_POSITION
+#define OTG2_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT__SHIFT                                            0x0
+#define OTG2_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT__SHIFT                                            0x10
+#define OTG2_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT_MASK                                              0x00007FFFL
+#define OTG2_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT_MASK                                              0x7FFF0000L
+//OTG2_OTG_SNAPSHOT_FRAME
+#define OTG2_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT__SHIFT                                              0x0
+#define OTG2_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT_MASK                                                0x00FFFFFFL
+//OTG2_OTG_INTERRUPT_CONTROL
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK__SHIFT                                               0x0
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE__SHIFT                                              0x1
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK__SHIFT                                        0x8
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE__SHIFT                                       0x9
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK__SHIFT                                  0x10
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE__SHIFT                                 0x11
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK__SHIFT                                                  0x18
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK__SHIFT                                                  0x19
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE__SHIFT                                                 0x1a
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE__SHIFT                                                 0x1b
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK__SHIFT                                              0x1c
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE__SHIFT                                             0x1d
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK__SHIFT                                          0x1e
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE__SHIFT                                         0x1f
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK_MASK                                                 0x00000001L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE_MASK                                                0x00000002L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK_MASK                                          0x00000100L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE_MASK                                         0x00000200L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK_MASK                                    0x00010000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE_MASK                                   0x00020000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK_MASK                                                    0x01000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK_MASK                                                    0x02000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE_MASK                                                   0x04000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE_MASK                                                   0x08000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK_MASK                                                0x10000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE_MASK                                               0x20000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK_MASK                                            0x40000000L
+#define OTG2_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE_MASK                                           0x80000000L
+//OTG2_OTG_UPDATE_LOCK
+#define OTG2_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK__SHIFT                                                          0x0
+#define OTG2_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK_MASK                                                            0x00000001L
+//OTG2_OTG_DOUBLE_BUFFER_CONTROL
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING__SHIFT                                             0x0
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING__SHIFT                             0x4
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING__SHIFT                                   0x5
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING__SHIFT                                  0x6
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING__SHIFT                          0x7
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY__SHIFT                                           0x8
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING__SHIFT                                 0x9
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE__SHIFT                                0x18
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING_MASK                                               0x00000001L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING_MASK                               0x00000010L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING_MASK                                     0x00000020L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING_MASK                                    0x00000040L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING_MASK                            0x00000080L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY_MASK                                             0x00000100L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING_MASK                                   0x00000200L
+#define OTG2_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE_MASK                                  0x03000000L
+//OTG2_OTG_MASTER_EN
+#define OTG2_OTG_MASTER_EN__OTG_MASTER_EN__SHIFT                                                              0x0
+#define OTG2_OTG_MASTER_EN__OTG_MASTER_EN_MASK                                                                0x00000001L
+//OTG2_OTG_VERTICAL_INTERRUPT0_POSITION
+#define OTG2_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START__SHIFT                      0x0
+#define OTG2_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END__SHIFT                        0x10
+#define OTG2_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START_MASK                        0x00007FFFL
+#define OTG2_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END_MASK                          0x7FFF0000L
+//OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY__SHIFT                  0x4
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE__SHIFT                       0x8
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS__SHIFT                           0xc
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS__SHIFT                       0x10
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR__SHIFT                            0x14
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE__SHIFT                         0x18
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS__SHIFT                                         0x1c
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY_MASK                    0x00000010L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE_MASK                         0x00000100L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS_MASK                             0x00001000L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS_MASK                         0x00010000L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR_MASK                              0x00100000L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE_MASK                           0x01000000L
+#define OTG2_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS_MASK                                           0x10000000L
+//OTG2_OTG_VERTICAL_INTERRUPT1_POSITION
+#define OTG2_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START__SHIFT                      0x0
+#define OTG2_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START_MASK                        0x00007FFFL
+//OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE__SHIFT                       0x8
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS__SHIFT                           0xc
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS__SHIFT                       0x10
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR__SHIFT                            0x14
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE__SHIFT                         0x18
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE_MASK                         0x00000100L
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS_MASK                             0x00001000L
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS_MASK                         0x00010000L
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR_MASK                              0x00100000L
+#define OTG2_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE_MASK                           0x01000000L
+//OTG2_OTG_VERTICAL_INTERRUPT2_POSITION
+#define OTG2_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START__SHIFT                      0x0
+#define OTG2_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START_MASK                        0x00007FFFL
+//OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE__SHIFT                       0x8
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS__SHIFT                           0xc
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS__SHIFT                       0x10
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR__SHIFT                            0x14
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE__SHIFT                         0x18
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE_MASK                         0x00000100L
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS_MASK                             0x00001000L
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS_MASK                         0x00010000L
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR_MASK                              0x00100000L
+#define OTG2_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE_MASK                           0x01000000L
+//OTG2_OTG_CRC_CNTL
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_EN__SHIFT                                                                  0x0
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN__SHIFT                                                        0x1
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_POLY_SEL__SHIFT                                                            0x2
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY__SHIFT                                                          0x3
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CONT_EN__SHIFT                                                             0x4
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL__SHIFT                                                   0x5
+#define OTG2_OTG_CRC_CNTL__OTG_CRC1_EN__SHIFT                                                                 0x7
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE__SHIFT                                                         0x8
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CONT_MODE__SHIFT                                                           0xa
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE__SHIFT                                                      0xc
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_AES_EN__SHIFT                                                              0xe
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS__SHIFT                                         0x13
+#define OTG2_OTG_CRC_CNTL__OTG_CRC0_SELECT__SHIFT                                                             0x14
+#define OTG2_OTG_CRC_CNTL__OTG_CRC1_SELECT__SHIFT                                                             0x18
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING__SHIFT                                                   0x1c
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING__SHIFT                                                   0x1d
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING__SHIFT                                                   0x1e
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING__SHIFT                                                   0x1f
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_EN_MASK                                                                    0x00000001L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN_MASK                                                          0x00000002L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_POLY_SEL_MASK                                                              0x00000004L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY_MASK                                                            0x00000008L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CONT_EN_MASK                                                               0x00000010L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL_MASK                                                     0x00000060L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC1_EN_MASK                                                                   0x00000080L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE_MASK                                                           0x00000300L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_CONT_MODE_MASK                                                             0x00000400L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE_MASK                                                        0x00003000L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_AES_EN_MASK                                                                0x00004000L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS_MASK                                           0x00080000L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC0_SELECT_MASK                                                               0x00700000L
+#define OTG2_OTG_CRC_CNTL__OTG_CRC1_SELECT_MASK                                                               0x07000000L
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING_MASK                                                     0x10000000L
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING_MASK                                                     0x20000000L
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING_MASK                                                     0x40000000L
+#define OTG2_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING_MASK                                                     0x80000000L
+//OTG2_OTG_CRC0_WINDOWA_X_CONTROL
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWA_Y_CONTROL
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWB_X_CONTROL
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWB_Y_CONTROL
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC0_DATA_R
+#define OTG2_OTG_CRC0_DATA_R__CRC0_R_CR__SHIFT                                                                0x0
+#define OTG2_OTG_CRC0_DATA_R__CRC0_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC0_DATA_G
+#define OTG2_OTG_CRC0_DATA_G__CRC0_G_Y__SHIFT                                                                 0x0
+#define OTG2_OTG_CRC0_DATA_G__CRC0_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG2_OTG_CRC0_DATA_B
+#define OTG2_OTG_CRC0_DATA_B__CRC0_B_CB__SHIFT                                                                0x0
+#define OTG2_OTG_CRC0_DATA_B__CRC0_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC0_DATA_C
+#define OTG2_OTG_CRC0_DATA_C__CRC0_C__SHIFT                                                                   0x0
+#define OTG2_OTG_CRC0_DATA_C__CRC0_C_MASK                                                                     0xFFFFFFFFL
+//OTG2_OTG_CRC1_WINDOWA_X_CONTROL
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWA_Y_CONTROL
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWB_X_CONTROL
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWB_Y_CONTROL
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG2_OTG_CRC1_DATA_R
+#define OTG2_OTG_CRC1_DATA_R__CRC1_R_CR__SHIFT                                                                0x0
+#define OTG2_OTG_CRC1_DATA_R__CRC1_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC1_DATA_G
+#define OTG2_OTG_CRC1_DATA_G__CRC1_G_Y__SHIFT                                                                 0x0
+#define OTG2_OTG_CRC1_DATA_G__CRC1_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG2_OTG_CRC1_DATA_B
+#define OTG2_OTG_CRC1_DATA_B__CRC1_B_CB__SHIFT                                                                0x0
+#define OTG2_OTG_CRC1_DATA_B__CRC1_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC1_DATA_C
+#define OTG2_OTG_CRC1_DATA_C__CRC1_C__SHIFT                                                                   0x0
+#define OTG2_OTG_CRC1_DATA_C__CRC1_C_MASK                                                                     0xFFFFFFFFL
+//OTG2_OTG_CRC2_DATA_R
+#define OTG2_OTG_CRC2_DATA_R__CRC2_R_CR__SHIFT                                                                0x0
+#define OTG2_OTG_CRC2_DATA_R__CRC2_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC2_DATA_G
+#define OTG2_OTG_CRC2_DATA_G__CRC2_G_Y__SHIFT                                                                 0x0
+#define OTG2_OTG_CRC2_DATA_G__CRC2_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG2_OTG_CRC2_DATA_B
+#define OTG2_OTG_CRC2_DATA_B__CRC2_B_CB__SHIFT                                                                0x0
+#define OTG2_OTG_CRC2_DATA_B__CRC2_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC2_DATA_C
+#define OTG2_OTG_CRC2_DATA_C__CRC2_C__SHIFT                                                                   0x0
+#define OTG2_OTG_CRC2_DATA_C__CRC2_C_MASK                                                                     0xFFFFFFFFL
+//OTG2_OTG_CRC3_DATA_R
+#define OTG2_OTG_CRC3_DATA_R__CRC3_R_CR__SHIFT                                                                0x0
+#define OTG2_OTG_CRC3_DATA_R__CRC3_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC3_DATA_G
+#define OTG2_OTG_CRC3_DATA_G__CRC3_G_Y__SHIFT                                                                 0x0
+#define OTG2_OTG_CRC3_DATA_G__CRC3_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG2_OTG_CRC3_DATA_B
+#define OTG2_OTG_CRC3_DATA_B__CRC3_B_CB__SHIFT                                                                0x0
+#define OTG2_OTG_CRC3_DATA_B__CRC3_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG2_OTG_CRC3_DATA_C
+#define OTG2_OTG_CRC3_DATA_C__CRC3_C__SHIFT                                                                   0x0
+#define OTG2_OTG_CRC3_DATA_C__CRC3_C_MASK                                                                     0xFFFFFFFFL
+//OTG2_OTG_CRC0_DATA_AES
+#define OTG2_OTG_CRC0_DATA_AES__CRC0_AES__SHIFT                                                               0x0
+#define OTG2_OTG_CRC0_DATA_AES__CRC0_AES_MASK                                                                 0xFFFFFFFFL
+//OTG2_OTG_CRC_SIG_RED_GREEN_MASK
+#define OTG2_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK__SHIFT                                          0x0
+#define OTG2_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK__SHIFT                                        0x10
+#define OTG2_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK_MASK                                            0x0000FFFFL
+#define OTG2_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK_MASK                                          0xFFFF0000L
+//OTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK
+#define OTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK__SHIFT                                      0x0
+#define OTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK__SHIFT                                   0x10
+#define OTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK_MASK                                        0x0000FFFFL
+#define OTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK_MASK                                     0xFFFF0000L
+//OTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG2_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG2_OTG_STATIC_SCREEN_CONTROL
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK__SHIFT                                   0x0
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT__SHIFT                                  0x10
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE__SHIFT                                          0x18
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS__SHIFT                                                  0x19
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS__SHIFT                                          0x1a
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR__SHIFT                                           0x1b
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE__SHIFT                                            0x1c
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE__SHIFT                                     0x1e
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE__SHIFT                               0x1f
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK_MASK                                     0x0000FFFFL
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT_MASK                                    0x00FF0000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE_MASK                                            0x01000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS_MASK                                                    0x02000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS_MASK                                            0x04000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR_MASK                                             0x08000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE_MASK                                              0x10000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_MASK                                       0x40000000L
+#define OTG2_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE_MASK                                 0x80000000L
+//OTG2_OTG_3D_STRUCTURE_CONTROL
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN__SHIFT                                             0x0
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE__SHIFT                                  0x8
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR__SHIFT                                 0xc
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET__SHIFT                                  0x10
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING__SHIFT                          0x11
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT__SHIFT                                        0x12
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN_MASK                                               0x00000001L
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE_MASK                                    0x00000300L
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR_MASK                                   0x00001000L
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_MASK                                    0x00010000L
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING_MASK                            0x00020000L
+#define OTG2_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_MASK                                          0x000C0000L
+//OTG2_OTG_GSL_VSYNC_GAP
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT__SHIFT                                                0x0
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY__SHIFT                                                0x8
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL__SHIFT                                           0x10
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE__SHIFT                                                 0x11
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR__SHIFT                                                0x13
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED__SHIFT                                             0x14
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER__SHIFT                                        0x17
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP__SHIFT                                                      0x18
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT_MASK                                                  0x000000FFL
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY_MASK                                                  0x0000FF00L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL_MASK                                             0x00010000L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE_MASK                                                   0x00060000L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR_MASK                                                  0x00080000L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED_MASK                                               0x00100000L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER_MASK                                          0x00800000L
+#define OTG2_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASK                                                        0xFF000000L
+//OTG2_OTG_MASTER_UPDATE_MODE
+#define OTG2_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE__SHIFT                                     0x0
+#define OTG2_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE_MASK                                       0x00000003L
+//OTG2_OTG_CLOCK_CONTROL
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_EN__SHIFT                                                           0x0
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS__SHIFT                                                     0x1
+#define OTG2_OTG_CLOCK_CONTROL__OTG_SOFT_RESET__SHIFT                                                         0x4
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_ON__SHIFT                                                           0x8
+#define OTG2_OTG_CLOCK_CONTROL__OTG_BUSY__SHIFT                                                               0x10
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_EN_MASK                                                             0x00000001L
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS_MASK                                                       0x00000002L
+#define OTG2_OTG_CLOCK_CONTROL__OTG_SOFT_RESET_MASK                                                           0x00000010L
+#define OTG2_OTG_CLOCK_CONTROL__OTG_CLOCK_ON_MASK                                                             0x00000100L
+#define OTG2_OTG_CLOCK_CONTROL__OTG_BUSY_MASK                                                                 0x00010000L
+//OTG2_OTG_VSTARTUP_PARAM
+#define OTG2_OTG_VSTARTUP_PARAM__VSTARTUP_START__SHIFT                                                        0x0
+#define OTG2_OTG_VSTARTUP_PARAM__VSTARTUP_START_MASK                                                          0x000003FFL
+//OTG2_OTG_VUPDATE_PARAM
+#define OTG2_OTG_VUPDATE_PARAM__VUPDATE_OFFSET__SHIFT                                                         0x0
+#define OTG2_OTG_VUPDATE_PARAM__VUPDATE_WIDTH__SHIFT                                                          0x10
+#define OTG2_OTG_VUPDATE_PARAM__VUPDATE_OFFSET_MASK                                                           0x0000FFFFL
+#define OTG2_OTG_VUPDATE_PARAM__VUPDATE_WIDTH_MASK                                                            0x03FF0000L
+//OTG2_OTG_VREADY_PARAM
+#define OTG2_OTG_VREADY_PARAM__VREADY_OFFSET__SHIFT                                                           0x0
+#define OTG2_OTG_VREADY_PARAM__VREADY_OFFSET_MASK                                                             0x0000FFFFL
+//OTG2_OTG_GLOBAL_SYNC_STATUS
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN__SHIFT                                                   0x0
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE__SHIFT                                                 0x1
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED__SHIFT                                           0x2
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS__SHIFT                                               0x3
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR__SHIFT                                              0x4
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN__SHIFT                                                    0x5
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE__SHIFT                                                  0x6
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL__SHIFT                                          0x7
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED__SHIFT                                            0x8
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS__SHIFT                                                0x9
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR__SHIFT                                               0xa
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS__SHIFT                                                    0xb
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN__SHIFT                                            0xc
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE__SHIFT                                          0xd
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED__SHIFT                                    0xe
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS__SHIFT                                        0xf
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR__SHIFT                                       0x10
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS__SHIFT                                            0x11
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN__SHIFT                                                     0x12
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE__SHIFT                                                   0x13
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED__SHIFT                                             0x14
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS__SHIFT                                                 0x15
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR__SHIFT                                                0x16
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS__SHIFT                                              0x18
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS__SHIFT                                               0x19
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN_MASK                                                     0x00000001L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE_MASK                                                   0x00000002L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED_MASK                                             0x00000004L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS_MASK                                                 0x00000008L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR_MASK                                                0x00000010L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN_MASK                                                      0x00000020L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE_MASK                                                    0x00000040L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL_MASK                                            0x00000080L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED_MASK                                              0x00000100L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS_MASK                                                  0x00000200L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR_MASK                                                 0x00000400L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS_MASK                                                      0x00000800L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN_MASK                                              0x00001000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE_MASK                                            0x00002000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED_MASK                                      0x00004000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS_MASK                                          0x00008000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR_MASK                                         0x00010000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS_MASK                                              0x00020000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN_MASK                                                       0x00040000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE_MASK                                                     0x00080000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED_MASK                                               0x00100000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS_MASK                                                   0x00200000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR_MASK                                                  0x00400000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS_MASK                                                0x01000000L
+#define OTG2_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS_MASK                                                 0x02000000L
+//OTG2_OTG_MASTER_UPDATE_LOCK
+#define OTG2_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK__SHIFT                                            0x0
+#define OTG2_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS__SHIFT                                                0x8
+#define OTG2_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK_MASK                                              0x00000001L
+#define OTG2_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS_MASK                                                  0x00000100L
+//OTG2_OTG_GSL_CONTROL
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL0_EN__SHIFT                                                              0x0
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL1_EN__SHIFT                                                              0x1
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL2_EN__SHIFT                                                              0x2
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN__SHIFT                                                        0x3
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE__SHIFT                                                      0x4
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY__SHIFT                                                      0x8
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY__SHIFT                                                      0x10
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS__SHIFT                                                 0x1c
+#define OTG2_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN__SHIFT                                            0x1f
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL0_EN_MASK                                                                0x00000001L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL1_EN_MASK                                                                0x00000002L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL2_EN_MASK                                                                0x00000004L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN_MASK                                                          0x00000008L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE_MASK                                                        0x00000030L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY_MASK                                                        0x00000F00L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY_MASK                                                        0x001F0000L
+#define OTG2_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS_MASK                                                   0x10000000L
+#define OTG2_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN_MASK                                              0x80000000L
+//OTG2_OTG_GSL_WINDOW_X
+#define OTG2_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X__SHIFT                                                  0x0
+#define OTG2_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X__SHIFT                                                    0x10
+#define OTG2_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X_MASK                                                    0x00007FFFL
+#define OTG2_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X_MASK                                                      0x7FFF0000L
+//OTG2_OTG_GSL_WINDOW_Y
+#define OTG2_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y__SHIFT                                                  0x0
+#define OTG2_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y__SHIFT                                                    0x10
+#define OTG2_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y_MASK                                                    0x00007FFFL
+#define OTG2_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y_MASK                                                      0x7FFF0000L
+//OTG2_OTG_VUPDATE_KEEPOUT
+#define OTG2_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET__SHIFT                      0x0
+#define OTG2_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET__SHIFT                        0x10
+#define OTG2_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN__SHIFT                            0x1f
+#define OTG2_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET_MASK                        0x0000FFFFL
+#define OTG2_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET_MASK                          0x03FF0000L
+#define OTG2_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN_MASK                              0x80000000L
+//OTG2_OTG_GLOBAL_CONTROL0
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X__SHIFT                                        0x0
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X__SHIFT                                          0x10
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN__SHIFT                                             0x1f
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X_MASK                                          0x00007FFFL
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X_MASK                                            0x7FFF0000L
+#define OTG2_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN_MASK                                               0x80000000L
+//OTG2_OTG_GLOBAL_CONTROL1
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y__SHIFT                                        0x0
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y__SHIFT                                          0x10
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE__SHIFT                                       0x1f
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y_MASK                                          0x00007FFFL
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y_MASK                                            0x7FFF0000L
+#define OTG2_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE_MASK                                         0x80000000L
+//OTG2_OTG_GLOBAL_CONTROL2
+#define OTG2_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN__SHIFT                                                0xa
+#define OTG2_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL__SHIFT                                              0x10
+#define OTG2_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL__SHIFT                                           0x19
+#define OTG2_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE__SHIFT                                            0x1e
+#define OTG2_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE__SHIFT                                                    0x1f
+#define OTG2_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN_MASK                                                  0x00000400L
+#define OTG2_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL_MASK                                                0x00070000L
+#define OTG2_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL_MASK                                             0x0E000000L
+#define OTG2_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE_MASK                                              0x40000000L
+#define OTG2_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE_MASK                                                      0x80000000L
+//OTG2_OTG_GLOBAL_CONTROL3
+#define OTG2_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD__SHIFT                                          0x0
+#define OTG2_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL__SHIFT                                     0x4
+#define OTG2_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL__SHIFT                                                 0x10
+#define OTG2_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL__SHIFT                                                   0x14
+#define OTG2_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD_MASK                                            0x00000003L
+#define OTG2_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL_MASK                                       0x00000030L
+#define OTG2_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL_MASK                                                   0x00030000L
+#define OTG2_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL_MASK                                                     0x00300000L
+//OTG2_OTG_GLOBAL_CONTROL4
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X__SHIFT                                                0x0
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y__SHIFT                                                0x10
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE__SHIFT                                               0x1f
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X_MASK                                                  0x00007FFFL
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y_MASK                                                  0x7FFF0000L
+#define OTG2_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE_MASK                                                 0x80000000L
+//OTG2_OTG_TRIG_MANUAL_CONTROL
+#define OTG2_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL__SHIFT                                              0x0
+#define OTG2_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL_MASK                                                0x00000001L
+//OTG2_OTG_DRR_TIMING_INT_STATUS
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED__SHIFT                                 0x0
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT__SHIFT                             0x4
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR__SHIFT                           0x8
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK__SHIFT                         0xc
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE__SHIFT                        0xd
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED__SHIFT                                 0x10
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT__SHIFT                             0x14
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR__SHIFT                           0x18
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK__SHIFT                         0x1c
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE__SHIFT                        0x1d
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_MASK                                   0x00000001L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MASK                               0x00000010L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR_MASK                             0x00000100L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK_MASK                           0x00001000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE_MASK                          0x00002000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_MASK                                   0x00010000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MASK                               0x00100000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR_MASK                             0x01000000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK_MASK                           0x10000000L
+#define OTG2_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE_MASK                          0x20000000L
+//OTG2_OTG_DRR_V_TOTAL_REACH_RANGE
+#define OTG2_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE__SHIFT                            0x0
+#define OTG2_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE__SHIFT                            0x10
+#define OTG2_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE_MASK                              0x00007FFFL
+#define OTG2_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE_MASK                              0x7FFF0000L
+//OTG2_OTG_DRR_V_TOTAL_CHANGE
+#define OTG2_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT__SHIFT                                      0x0
+#define OTG2_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT_MASK                                        0x00007FFFL
+//OTG2_OTG_DRR_TRIGGER_WINDOW
+#define OTG2_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X__SHIFT                                    0x0
+#define OTG2_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X__SHIFT                                      0x10
+#define OTG2_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X_MASK                                      0x00007FFFL
+#define OTG2_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X_MASK                                        0x7FFF0000L
+//OTG2_OTG_DRR_CONTROL
+#define OTG2_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME__SHIFT                                                    0x0
+#define OTG2_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR__SHIFT                                             0x10
+#define OTG2_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME_MASK                                                      0x00000003L
+#define OTG2_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR_MASK                                               0x7FFF0000L
+//OTG2_OTG_DRR_CONTOL2
+#define OTG2_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR__SHIFT                                             0x0
+#define OTG2_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR_MASK                                               0xFFFFFFFFL
+//OTG2_OTG_M_CONST_DTO0
+#define OTG2_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE__SHIFT                                                   0x0
+#define OTG2_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE_MASK                                                     0xFFFFFFFFL
+//OTG2_OTG_M_CONST_DTO1
+#define OTG2_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO__SHIFT                                                  0x0
+#define OTG2_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO_MASK                                                    0xFFFFFFFFL
+//OTG2_OTG_REQUEST_CONTROL
+#define OTG2_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE__SHIFT                                     0x0
+#define OTG2_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE_MASK                                       0x00000001L
+//OTG2_OTG_PIPE_UPDATE_STATUS
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING__SHIFT                                                  0x0
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING__SHIFT                                         0x4
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING__SHIFT                                         0x8
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS__SHIFT                                        0x10
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING_MASK                                                    0x00000001L
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING_MASK                                           0x00000010L
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING_MASK                                           0x00000100L
+#define OTG2_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS_MASK                                          0x00010000L
+//OTG2_OTG_PSTATE_REGISTER
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START__SHIFT                                             0x0
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND__SHIFT                                                    0x10
+#define OTG2_OTG_PSTATE_REGISTER__OTG_UNBLANK__SHIFT                                                          0x14
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN__SHIFT                                           0x18
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START_MASK                                               0x00007FFFL
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND_MASK                                                      0x00010000L
+#define OTG2_OTG_PSTATE_REGISTER__OTG_UNBLANK_MASK                                                            0x00100000L
+#define OTG2_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN_MASK                                             0xFF000000L
+//OTG2_OTG_ENCRYPTION
+#define OTG2_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL__SHIFT                                             0x0
+#define OTG2_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL_MASK                                               0x00000003L
+//OTG2_OTG_SPARE_REGISTER
+#define OTG2_OTG_SPARE_REGISTER__OTG_SPARE_REG__SHIFT                                                         0x0
+#define OTG2_OTG_SPARE_REGISTER__OTG_SPARE_REG_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_otg3_dispdec
+//OTG3_OTG_H_TOTAL
+#define OTG3_OTG_H_TOTAL__OTG_H_TOTAL__SHIFT                                                                  0x0
+#define OTG3_OTG_H_TOTAL__OTG_H_TOTAL_MASK                                                                    0x00007FFFL
+//OTG3_OTG_H_BLANK_START_END
+#define OTG3_OTG_H_BLANK_START_END__OTG_H_BLANK_START__SHIFT                                                  0x0
+#define OTG3_OTG_H_BLANK_START_END__OTG_H_BLANK_END__SHIFT                                                    0x10
+#define OTG3_OTG_H_BLANK_START_END__OTG_H_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG3_OTG_H_BLANK_START_END__OTG_H_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG3_OTG_H_SYNC_A
+#define OTG3_OTG_H_SYNC_A__OTG_H_SYNC_A_START__SHIFT                                                          0x0
+#define OTG3_OTG_H_SYNC_A__OTG_H_SYNC_A_END__SHIFT                                                            0x10
+#define OTG3_OTG_H_SYNC_A__OTG_H_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG3_OTG_H_SYNC_A__OTG_H_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG3_OTG_H_SYNC_A_CNTL
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN__SHIFT                                                     0x10
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF__SHIFT                                                    0x11
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_COMP_SYNC_A_EN_MASK                                                       0x00010000L
+#define OTG3_OTG_H_SYNC_A_CNTL__OTG_H_SYNC_A_CUTOFF_MASK                                                      0x00020000L
+//OTG3_OTG_H_TIMING_CNTL
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE__SHIFT                                                  0x0
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL__SHIFT                                           0x8
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR__SHIFT                                             0x10
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MASK                                                    0x00000003L
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_MANUAL_MASK                                             0x00000100L
+#define OTG3_OTG_H_TIMING_CNTL__OTG_H_TIMING_DIV_MODE_CURR_MASK                                               0x00030000L
+//OTG3_OTG_V_TOTAL
+#define OTG3_OTG_V_TOTAL__OTG_V_TOTAL__SHIFT                                                                  0x0
+#define OTG3_OTG_V_TOTAL__OTG_V_TOTAL_MASK                                                                    0x00007FFFL
+//OTG3_OTG_V_TOTAL_MIN
+#define OTG3_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN__SHIFT                                                          0x0
+#define OTG3_OTG_V_TOTAL_MIN__OTG_V_TOTAL_MIN_MASK                                                            0x00007FFFL
+//OTG3_OTG_V_TOTAL_MAX
+#define OTG3_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX__SHIFT                                                          0x0
+#define OTG3_OTG_V_TOTAL_MAX__OTG_V_TOTAL_MAX_MASK                                                            0x00007FFFL
+//OTG3_OTG_V_TOTAL_MID
+#define OTG3_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID__SHIFT                                                          0x0
+#define OTG3_OTG_V_TOTAL_MID__OTG_V_TOTAL_MID_MASK                                                            0x00007FFFL
+//OTG3_OTG_V_TOTAL_CONTROL
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL__SHIFT                                                  0x0
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL__SHIFT                                                  0x1
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN__SHIFT                                      0x2
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN__SHIFT                                      0x3
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT__SHIFT                                              0x4
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD__SHIFT                                          0x5
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM__SHIFT                                             0x8
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK__SHIFT                                             0x10
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MIN_SEL_MASK                                                    0x00000001L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_V_TOTAL_MAX_SEL_MASK                                                    0x00000002L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MAX_EN_MASK                                        0x00000004L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_REPLACING_MIN_EN_MASK                                        0x00000008L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_FORCE_LOCK_ON_EVENT_MASK                                                0x00000010L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_DRR_EVENT_ACTIVE_PERIOD_MASK                                            0x00000020L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_VTOTAL_MID_FRAME_NUM_MASK                                               0x0000FF00L
+#define OTG3_OTG_V_TOTAL_CONTROL__OTG_SET_V_TOTAL_MIN_MASK_MASK                                               0xFFFF0000L
+//OTG3_OTG_V_COUNT_STOP_CONTROL
+#define OTG3_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP__SHIFT                                                0x0
+#define OTG3_OTG_V_COUNT_STOP_CONTROL__OTG_V_COUNT_STOP_MASK                                                  0x00007FFFL
+//OTG3_OTG_V_COUNT_STOP_CONTROL2
+#define OTG3_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER__SHIFT                                         0x0
+#define OTG3_OTG_V_COUNT_STOP_CONTROL2__OTG_V_COUNT_STOP_TIMER_MASK                                           0xFFFFFFFFL
+//OTG3_OTG_V_TOTAL_INT_STATUS
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED__SHIFT                                0x0
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT__SHIFT                            0x4
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK__SHIFT                            0x8
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK__SHIFT                            0xc
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MASK                                  0x00000001L
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_INT_MASK                              0x00000010L
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_ACK_MASK                              0x00000100L
+#define OTG3_OTG_V_TOTAL_INT_STATUS__OTG_SET_V_TOTAL_MIN_EVENT_OCCURRED_MSK_MASK                              0x00001000L
+//OTG3_OTG_VSYNC_NOM_INT_STATUS
+#define OTG3_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM__SHIFT                                                   0x0
+#define OTG3_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR__SHIFT                                         0x4
+#define OTG3_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_MASK                                                     0x00000001L
+#define OTG3_OTG_VSYNC_NOM_INT_STATUS__OTG_VSYNC_NOM_INT_CLEAR_MASK                                           0x00000010L
+//OTG3_OTG_V_BLANK_START_END
+#define OTG3_OTG_V_BLANK_START_END__OTG_V_BLANK_START__SHIFT                                                  0x0
+#define OTG3_OTG_V_BLANK_START_END__OTG_V_BLANK_END__SHIFT                                                    0x10
+#define OTG3_OTG_V_BLANK_START_END__OTG_V_BLANK_START_MASK                                                    0x00007FFFL
+#define OTG3_OTG_V_BLANK_START_END__OTG_V_BLANK_END_MASK                                                      0x7FFF0000L
+//OTG3_OTG_V_SYNC_A
+#define OTG3_OTG_V_SYNC_A__OTG_V_SYNC_A_START__SHIFT                                                          0x0
+#define OTG3_OTG_V_SYNC_A__OTG_V_SYNC_A_END__SHIFT                                                            0x10
+#define OTG3_OTG_V_SYNC_A__OTG_V_SYNC_A_START_MASK                                                            0x00007FFFL
+#define OTG3_OTG_V_SYNC_A__OTG_V_SYNC_A_END_MASK                                                              0x7FFF0000L
+//OTG3_OTG_V_SYNC_A_CNTL
+#define OTG3_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL__SHIFT                                                       0x0
+#define OTG3_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE__SHIFT                                                        0x8
+#define OTG3_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_A_POL_MASK                                                         0x00000001L
+#define OTG3_OTG_V_SYNC_A_CNTL__OTG_V_SYNC_MODE_MASK                                                          0x00000100L
+//OTG3_OTG_TRIGA_CNTL
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED__SHIFT                                                        0xe
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY__SHIFT                                                           0x18
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR__SHIFT                                                           0x1f
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_OCCURRED_MASK                                                          0x00004000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_DELAY_MASK                                                             0x1F000000L
+#define OTG3_OTG_TRIGA_CNTL__OTG_TRIGA_CLEAR_MASK                                                             0x80000000L
+//OTG3_OTG_TRIGA_MANUAL_TRIG
+#define OTG3_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG3_OTG_TRIGA_MANUAL_TRIG__OTG_TRIGA_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG3_OTG_TRIGB_CNTL
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT__SHIFT                                                   0x0
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT__SHIFT                                              0x5
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT__SHIFT                                                 0x8
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN__SHIFT                                                0xb
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS__SHIFT                                                    0xc
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS__SHIFT                                                 0xd
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED__SHIFT                                                        0xe
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL__SHIFT                                         0x10
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL__SHIFT                                        0x12
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT__SHIFT                                                0x14
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY__SHIFT                                                           0x18
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR__SHIFT                                                           0x1f
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_SELECT_MASK                                                     0x0000001FL
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_SOURCE_PIPE_SELECT_MASK                                                0x000000E0L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_SELECT_MASK                                                   0x00000700L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_RESYNC_BYPASS_EN_MASK                                                  0x00000800L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_INPUT_STATUS_MASK                                                      0x00001000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_POLARITY_STATUS_MASK                                                   0x00002000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_OCCURRED_MASK                                                          0x00004000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_RISING_EDGE_DETECT_CNTL_MASK                                           0x00030000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_FALLING_EDGE_DETECT_CNTL_MASK                                          0x000C0000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_FREQUENCY_SELECT_MASK                                                  0x00300000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_DELAY_MASK                                                             0x1F000000L
+#define OTG3_OTG_TRIGB_CNTL__OTG_TRIGB_CLEAR_MASK                                                             0x80000000L
+//OTG3_OTG_TRIGB_MANUAL_TRIG
+#define OTG3_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG__SHIFT                                              0x0
+#define OTG3_OTG_TRIGB_MANUAL_TRIG__OTG_TRIGB_MANUAL_TRIG_MASK                                                0x00000001L
+//OTG3_OTG_FORCE_COUNT_NOW_CNTL
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE__SHIFT                                        0x0
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK__SHIFT                                       0x4
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL__SHIFT                                    0x8
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED__SHIFT                                    0x10
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR__SHIFT                                       0x18
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_MODE_MASK                                          0x00000003L
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CHECK_MASK                                         0x00000010L
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_TRIG_SEL_MASK                                      0x00000100L
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_OCCURRED_MASK                                      0x00010000L
+#define OTG3_OTG_FORCE_COUNT_NOW_CNTL__OTG_FORCE_COUNT_NOW_CLEAR_MASK                                         0x01000000L
+//OTG3_OTG_STEREO_FORCE_NEXT_EYE
+#define OTG3_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE__SHIFT                                      0x0
+#define OTG3_OTG_STEREO_FORCE_NEXT_EYE__OTG_STEREO_FORCE_NEXT_EYE_MASK                                        0x00000003L
+//OTG3_OTG_CONTROL
+#define OTG3_OTG_CONTROL__OTG_MASTER_EN__SHIFT                                                                0x0
+#define OTG3_OTG_CONTROL__OTG_DISABLE_POINT_CNTL__SHIFT                                                       0x8
+#define OTG3_OTG_CONTROL__OTG_START_POINT_CNTL__SHIFT                                                         0xc
+#define OTG3_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL__SHIFT                                                        0xd
+#define OTG3_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY__SHIFT                                                    0xe
+#define OTG3_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE__SHIFT                                                  0x10
+#define OTG3_OTG_CONTROL__OTG_OUT_MUX__SHIFT                                                                  0x14
+#define OTG3_OTG_CONTROL__OTG_MASTER_EN_MASK                                                                  0x00000001L
+#define OTG3_OTG_CONTROL__OTG_DISABLE_POINT_CNTL_MASK                                                         0x00000300L
+#define OTG3_OTG_CONTROL__OTG_START_POINT_CNTL_MASK                                                           0x00001000L
+#define OTG3_OTG_CONTROL__OTG_FIELD_NUMBER_CNTL_MASK                                                          0x00002000L
+#define OTG3_OTG_CONTROL__OTG_FIELD_NUMBER_POLARITY_MASK                                                      0x00004000L
+#define OTG3_OTG_CONTROL__OTG_CURRENT_MASTER_EN_STATE_MASK                                                    0x00010000L
+#define OTG3_OTG_CONTROL__OTG_OUT_MUX_MASK                                                                    0x00300000L
+//OTG3_OTG_DLPC_CONTROL
+#define OTG3_OTG_DLPC_CONTROL__OTG_RESYNC_MODE__SHIFT                                                         0x0
+#define OTG3_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION__SHIFT                                              0x10
+#define OTG3_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT__SHIFT                                               0x1f
+#define OTG3_OTG_DLPC_CONTROL__OTG_RESYNC_MODE_MASK                                                           0x00000001L
+#define OTG3_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_LOCATION_MASK                                                0x7FFF0000L
+#define OTG3_OTG_DLPC_CONTROL__OTG_DLPC_SNAPSHOT_CURRENT_MASK                                                 0x80000000L
+//OTG3_OTG_PWA_FRAME_SYNC_CONTROL
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN__SHIFT                                         0x0
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE__SHIFT                                0x1
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE__SHIFT                                       0x10
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_EN_MASK                                           0x00000001L
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_VCOUNT_MODE_MASK                                  0x00000002L
+#define OTG3_OTG_PWA_FRAME_SYNC_CONTROL__OTG_PWA_FRAME_SYNC_LINE_MASK                                         0x03FF0000L
+//OTG3_OTG_INTERLACE_CONTROL
+#define OTG3_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE__SHIFT                                               0x0
+#define OTG3_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD__SHIFT                                     0x10
+#define OTG3_OTG_INTERLACE_CONTROL__OTG_INTERLACE_ENABLE_MASK                                                 0x00000001L
+#define OTG3_OTG_INTERLACE_CONTROL__OTG_INTERLACE_FORCE_NEXT_FIELD_MASK                                       0x00030000L
+//OTG3_OTG_INTERLACE_STATUS
+#define OTG3_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD__SHIFT                                         0x0
+#define OTG3_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD__SHIFT                                            0x1
+#define OTG3_OTG_INTERLACE_STATUS__OTG_INTERLACE_CURRENT_FIELD_MASK                                           0x00000001L
+#define OTG3_OTG_INTERLACE_STATUS__OTG_INTERLACE_NEXT_FIELD_MASK                                              0x00000002L
+//OTG3_OTG_PIXEL_DATA_READBACK0
+#define OTG3_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB__SHIFT                                          0x0
+#define OTG3_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y__SHIFT                                          0x10
+#define OTG3_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_BLUE_CB_MASK                                            0x0000FFFFL
+#define OTG3_OTG_PIXEL_DATA_READBACK0__OTG_PIXEL_DATA_GREEN_Y_MASK                                            0xFFFF0000L
+//OTG3_OTG_PIXEL_DATA_READBACK1
+#define OTG3_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR__SHIFT                                           0x0
+#define OTG3_OTG_PIXEL_DATA_READBACK1__OTG_PIXEL_DATA_RED_CR_MASK                                             0x0000FFFFL
+//OTG3_OTG_STATUS
+#define OTG3_OTG_STATUS__OTG_V_BLANK__SHIFT                                                                   0x0
+#define OTG3_OTG_STATUS__OTG_V_ACTIVE_DISP__SHIFT                                                             0x1
+#define OTG3_OTG_STATUS__OTG_V_SYNC_A__SHIFT                                                                  0x2
+#define OTG3_OTG_STATUS__OTG_V_UPDATE__SHIFT                                                                  0x3
+#define OTG3_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE__SHIFT                                                      0x5
+#define OTG3_OTG_STATUS__OTG_H_BLANK__SHIFT                                                                   0x10
+#define OTG3_OTG_STATUS__OTG_H_ACTIVE_DISP__SHIFT                                                             0x11
+#define OTG3_OTG_STATUS__OTG_H_SYNC_A__SHIFT                                                                  0x12
+#define OTG3_OTG_STATUS__OTG_V_BLANK_MASK                                                                     0x00000001L
+#define OTG3_OTG_STATUS__OTG_V_ACTIVE_DISP_MASK                                                               0x00000002L
+#define OTG3_OTG_STATUS__OTG_V_SYNC_A_MASK                                                                    0x00000004L
+#define OTG3_OTG_STATUS__OTG_V_UPDATE_MASK                                                                    0x00000008L
+#define OTG3_OTG_STATUS__OTG_V_BLANK_3D_STRUCTURE_MASK                                                        0x00000020L
+#define OTG3_OTG_STATUS__OTG_H_BLANK_MASK                                                                     0x00010000L
+#define OTG3_OTG_STATUS__OTG_H_ACTIVE_DISP_MASK                                                               0x00020000L
+#define OTG3_OTG_STATUS__OTG_H_SYNC_A_MASK                                                                    0x00040000L
+//OTG3_OTG_STATUS_POSITION
+#define OTG3_OTG_STATUS_POSITION__OTG_VERT_COUNT__SHIFT                                                       0x0
+#define OTG3_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK__SHIFT                                                 0xf
+#define OTG3_OTG_STATUS_POSITION__OTG_HORZ_COUNT__SHIFT                                                       0x10
+#define OTG3_OTG_STATUS_POSITION__OTG_VERT_COUNT_MASK                                                         0x00007FFFL
+#define OTG3_OTG_STATUS_POSITION__OTG_VERT_LONG_VBLANK_MASK                                                   0x00008000L
+#define OTG3_OTG_STATUS_POSITION__OTG_HORZ_COUNT_MASK                                                         0x7FFF0000L
+//OTG3_OTG_LONG_VBLANK_STATUS
+#define OTG3_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT__SHIFT                                            0x0
+#define OTG3_OTG_LONG_VBLANK_STATUS__OTG_V_COUNT_STOP_COUNT_MASK                                              0xFFFFFFFFL
+//OTG3_OTG_NOM_VERT_POSITION
+#define OTG3_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM__SHIFT                                                 0x0
+#define OTG3_OTG_NOM_VERT_POSITION__OTG_VERT_COUNT_NOM_MASK                                                   0x00007FFFL
+//OTG3_OTG_STATUS_FRAME_COUNT
+#define OTG3_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT__SHIFT                                                   0x0
+#define OTG3_OTG_STATUS_FRAME_COUNT__OTG_FRAME_COUNT_MASK                                                     0x00FFFFFFL
+//OTG3_OTG_STATUS_VF_COUNT
+#define OTG3_OTG_STATUS_VF_COUNT__OTG_VF_COUNT__SHIFT                                                         0x0
+#define OTG3_OTG_STATUS_VF_COUNT__OTG_VF_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG3_OTG_STATUS_HV_COUNT
+#define OTG3_OTG_STATUS_HV_COUNT__OTG_HV_COUNT__SHIFT                                                         0x0
+#define OTG3_OTG_STATUS_HV_COUNT__OTG_HV_COUNT_MASK                                                           0x7FFFFFFFL
+//OTG3_OTG_COUNT_CONTROL
+#define OTG3_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN__SHIFT                                                  0x0
+#define OTG3_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT__SHIFT                                              0x1
+#define OTG3_OTG_COUNT_CONTROL__OTG_HORZ_COUNT_BY2_EN_MASK                                                    0x00000001L
+#define OTG3_OTG_COUNT_CONTROL__OTG_HORZ_REPETITION_COUNT_MASK                                                0x0000001EL
+//OTG3_OTG_COUNT_RESET
+#define OTG3_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT__SHIFT                                                    0x0
+#define OTG3_OTG_COUNT_RESET__OTG_RESET_FRAME_COUNT_MASK                                                      0x00000001L
+//OTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE
+#define OTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__SHIFT                        0x0
+#define OTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE__OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_MASK                          0x00000001L
+//OTG3_OTG_VERT_SYNC_CONTROL
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED__SHIFT                                 0x0
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR__SHIFT                                    0x8
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE__SHIFT                                          0x10
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED_MASK                                   0x00000001L
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_CLEAR_MASK                                      0x00000100L
+#define OTG3_OTG_VERT_SYNC_CONTROL__OTG_AUTO_FORCE_VSYNC_MODE_MASK                                            0x00030000L
+//OTG3_OTG_STEREO_STATUS
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE__SHIFT                                                 0x0
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT__SHIFT                                                 0x8
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT__SHIFT                                                 0x10
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG__SHIFT                                                    0x14
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING__SHIFT                                      0x18
+#define OTG3_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE__SHIFT                                         0x1e
+#define OTG3_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE__SHIFT                                        0x1f
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_CURRENT_EYE_MASK                                                   0x00000001L
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_SYNC_OUTPUT_MASK                                                   0x00000100L
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_SYNC_SELECT_MASK                                                   0x00010000L
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_EYE_FLAG_MASK                                                      0x00100000L
+#define OTG3_OTG_STEREO_STATUS__OTG_STEREO_FORCE_NEXT_EYE_PENDING_MASK                                        0x03000000L
+#define OTG3_OTG_STEREO_STATUS__OTG_CURRENT_3D_STRUCTURE_STATE_MASK                                           0x40000000L
+#define OTG3_OTG_STEREO_STATUS__OTG_CURRENT_STEREOSYNC_EN_STATE_MASK                                          0x80000000L
+//OTG3_OTG_STEREO_CONTROL
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM__SHIFT                                       0x0
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY__SHIFT                                       0xf
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY__SHIFT                                          0x11
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP__SHIFT                                  0x12
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM__SHIFT                                                 0x13
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX__SHIFT                                        0x14
+#define OTG3_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL__SHIFT                                                     0x15
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_EN__SHIFT                                                         0x18
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_LINE_NUM_MASK                                         0x00007FFFL
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_SYNC_OUTPUT_POLARITY_MASK                                         0x00008000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_EYE_FLAG_POLARITY_MASK                                            0x00020000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_STEREOSYNC_OUTPUT_FOR_DP_MASK                                    0x00040000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_FIELD_NUM_MASK                                                   0x00080000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_DISABLE_V_BLANK_FOR_DP_FIX_MASK                                          0x00100000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_FIELD_NUM_SEL_MASK                                                       0x00200000L
+#define OTG3_OTG_STEREO_CONTROL__OTG_STEREO_EN_MASK                                                           0x01000000L
+//OTG3_OTG_SNAPSHOT_STATUS
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED__SHIFT                                                0x0
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR__SHIFT                                                   0x1
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER__SHIFT                                          0x2
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_OCCURRED_MASK                                                  0x00000001L
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_CLEAR_MASK                                                     0x00000002L
+#define OTG3_OTG_SNAPSHOT_STATUS__OTG_SNAPSHOT_MANUAL_TRIGGER_MASK                                            0x00000004L
+//OTG3_OTG_SNAPSHOT_CONTROL
+#define OTG3_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL__SHIFT                                          0x0
+#define OTG3_OTG_SNAPSHOT_CONTROL__OTG_AUTO_SNAPSHOT_TRIG_SEL_MASK                                            0x00000003L
+//OTG3_OTG_SNAPSHOT_POSITION
+#define OTG3_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT__SHIFT                                            0x0
+#define OTG3_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT__SHIFT                                            0x10
+#define OTG3_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_VERT_COUNT_MASK                                              0x00007FFFL
+#define OTG3_OTG_SNAPSHOT_POSITION__OTG_SNAPSHOT_HORZ_COUNT_MASK                                              0x7FFF0000L
+//OTG3_OTG_SNAPSHOT_FRAME
+#define OTG3_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT__SHIFT                                              0x0
+#define OTG3_OTG_SNAPSHOT_FRAME__OTG_SNAPSHOT_FRAME_COUNT_MASK                                                0x00FFFFFFL
+//OTG3_OTG_INTERRUPT_CONTROL
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK__SHIFT                                               0x0
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE__SHIFT                                              0x1
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK__SHIFT                                        0x8
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE__SHIFT                                       0x9
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK__SHIFT                                  0x10
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE__SHIFT                                 0x11
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK__SHIFT                                                  0x18
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK__SHIFT                                                  0x19
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE__SHIFT                                                 0x1a
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE__SHIFT                                                 0x1b
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK__SHIFT                                              0x1c
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE__SHIFT                                             0x1d
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK__SHIFT                                          0x1e
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE__SHIFT                                         0x1f
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_MSK_MASK                                                 0x00000001L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_SNAPSHOT_INT_TYPE_MASK                                                0x00000002L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_MSK_MASK                                          0x00000100L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_COUNT_NOW_INT_TYPE_MASK                                         0x00000200L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_MSK_MASK                                    0x00010000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_FORCE_VSYNC_NEXT_LINE_INT_TYPE_MASK                                   0x00020000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_MSK_MASK                                                    0x01000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_MSK_MASK                                                    0x02000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGA_INT_TYPE_MASK                                                   0x04000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_TRIGB_INT_TYPE_MASK                                                   0x08000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_MSK_MASK                                                0x10000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_VSYNC_NOM_INT_TYPE_MASK                                               0x20000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_MSK_MASK                                            0x40000000L
+#define OTG3_OTG_INTERRUPT_CONTROL__OTG_GSL_VSYNC_GAP_INT_TYPE_MASK                                           0x80000000L
+//OTG3_OTG_UPDATE_LOCK
+#define OTG3_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK__SHIFT                                                          0x0
+#define OTG3_OTG_UPDATE_LOCK__OTG_UPDATE_LOCK_MASK                                                            0x00000001L
+//OTG3_OTG_DOUBLE_BUFFER_CONTROL
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING__SHIFT                                             0x0
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING__SHIFT                             0x4
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING__SHIFT                                   0x5
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING__SHIFT                                  0x6
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING__SHIFT                          0x7
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY__SHIFT                                           0x8
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING__SHIFT                                 0x9
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE__SHIFT                                0x18
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_PENDING_MASK                                               0x00000001L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_PENDING_MASK                               0x00000010L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_TIMING_DB_UPDATE_PENDING_MASK                                     0x00000020L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_CTRL_DB_UPDATE_PENDING_MASK                                    0x00000040L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_3D_STRUCTURE_EN_DB_UPDATE_PENDING_MASK                            0x00000080L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_UPDATE_INSTANTLY_MASK                                             0x00000100L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_VSTARTUP_DB_UPDATE_PENDING_MASK                                   0x00000200L
+#define OTG3_OTG_DOUBLE_BUFFER_CONTROL__OTG_DRR_TIMING_DBUF_UPDATE_MODE_MASK                                  0x03000000L
+//OTG3_OTG_MASTER_EN
+#define OTG3_OTG_MASTER_EN__OTG_MASTER_EN__SHIFT                                                              0x0
+#define OTG3_OTG_MASTER_EN__OTG_MASTER_EN_MASK                                                                0x00000001L
+//OTG3_OTG_VERTICAL_INTERRUPT0_POSITION
+#define OTG3_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START__SHIFT                      0x0
+#define OTG3_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END__SHIFT                        0x10
+#define OTG3_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_START_MASK                        0x00007FFFL
+#define OTG3_OTG_VERTICAL_INTERRUPT0_POSITION__OTG_VERTICAL_INTERRUPT0_LINE_END_MASK                          0x7FFF0000L
+//OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY__SHIFT                  0x4
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE__SHIFT                       0x8
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS__SHIFT                           0xc
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS__SHIFT                       0x10
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR__SHIFT                            0x14
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE__SHIFT                         0x18
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS__SHIFT                                         0x1c
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_OUTPUT_POLARITY_MASK                    0x00000010L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_ENABLE_MASK                         0x00000100L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_STATUS_MASK                             0x00001000L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_STATUS_MASK                         0x00010000L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_CLEAR_MASK                              0x00100000L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VERTICAL_INTERRUPT0_INT_TYPE_MASK                           0x01000000L
+#define OTG3_OTG_VERTICAL_INTERRUPT0_CONTROL__OTG_VINTE_STATUS_MASK                                           0x10000000L
+//OTG3_OTG_VERTICAL_INTERRUPT1_POSITION
+#define OTG3_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START__SHIFT                      0x0
+#define OTG3_OTG_VERTICAL_INTERRUPT1_POSITION__OTG_VERTICAL_INTERRUPT1_LINE_START_MASK                        0x00007FFFL
+//OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE__SHIFT                       0x8
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS__SHIFT                           0xc
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS__SHIFT                       0x10
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR__SHIFT                            0x14
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE__SHIFT                         0x18
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_ENABLE_MASK                         0x00000100L
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_STATUS_MASK                             0x00001000L
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_STATUS_MASK                         0x00010000L
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_CLEAR_MASK                              0x00100000L
+#define OTG3_OTG_VERTICAL_INTERRUPT1_CONTROL__OTG_VERTICAL_INTERRUPT1_INT_TYPE_MASK                           0x01000000L
+//OTG3_OTG_VERTICAL_INTERRUPT2_POSITION
+#define OTG3_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START__SHIFT                      0x0
+#define OTG3_OTG_VERTICAL_INTERRUPT2_POSITION__OTG_VERTICAL_INTERRUPT2_LINE_START_MASK                        0x00007FFFL
+//OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE__SHIFT                       0x8
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS__SHIFT                           0xc
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS__SHIFT                       0x10
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR__SHIFT                            0x14
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE__SHIFT                         0x18
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_ENABLE_MASK                         0x00000100L
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_STATUS_MASK                             0x00001000L
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_STATUS_MASK                         0x00010000L
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_CLEAR_MASK                              0x00100000L
+#define OTG3_OTG_VERTICAL_INTERRUPT2_CONTROL__OTG_VERTICAL_INTERRUPT2_INT_TYPE_MASK                           0x01000000L
+//OTG3_OTG_CRC_CNTL
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_EN__SHIFT                                                                  0x0
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN__SHIFT                                                        0x1
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_POLY_SEL__SHIFT                                                            0x2
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY__SHIFT                                                          0x3
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CONT_EN__SHIFT                                                             0x4
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL__SHIFT                                                   0x5
+#define OTG3_OTG_CRC_CNTL__OTG_CRC1_EN__SHIFT                                                                 0x7
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE__SHIFT                                                         0x8
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CONT_MODE__SHIFT                                                           0xa
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE__SHIFT                                                      0xc
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_AES_EN__SHIFT                                                              0xe
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS__SHIFT                                         0x13
+#define OTG3_OTG_CRC_CNTL__OTG_CRC0_SELECT__SHIFT                                                             0x14
+#define OTG3_OTG_CRC_CNTL__OTG_CRC1_SELECT__SHIFT                                                             0x18
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING__SHIFT                                                   0x1c
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING__SHIFT                                                   0x1d
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING__SHIFT                                                   0x1e
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING__SHIFT                                                   0x1f
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_EN_MASK                                                                    0x00000001L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_WINDOW_DB_EN_MASK                                                          0x00000002L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_POLY_SEL_MASK                                                              0x00000004L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_BLANK_ONLY_MASK                                                            0x00000008L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CONT_EN_MASK                                                               0x00000010L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CAPTURE_START_SEL_MASK                                                     0x00000060L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC1_EN_MASK                                                                   0x00000080L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_STEREO_MODE_MASK                                                           0x00000300L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_CONT_MODE_MASK                                                             0x00000400L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_INTERLACE_MODE_MASK                                                        0x00003000L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_AES_EN_MASK                                                                0x00004000L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC_USE_NEW_AND_REPEATED_PIXELS_MASK                                           0x00080000L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC0_SELECT_MASK                                                               0x00700000L
+#define OTG3_OTG_CRC_CNTL__OTG_CRC1_SELECT_MASK                                                               0x07000000L
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC0_PENDING_MASK                                                     0x10000000L
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC1_PENDING_MASK                                                     0x20000000L
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC2_PENDING_MASK                                                     0x40000000L
+#define OTG3_OTG_CRC_CNTL__OTG_ONE_SHOT_CRC3_PENDING_MASK                                                     0x80000000L
+//OTG3_OTG_CRC0_WINDOWA_X_CONTROL
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL__OTG_CRC0_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWA_Y_CONTROL
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL__OTG_CRC0_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWB_X_CONTROL
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL__OTG_CRC0_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWB_Y_CONTROL
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL__OTG_CRC0_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC0_DATA_R
+#define OTG3_OTG_CRC0_DATA_R__CRC0_R_CR__SHIFT                                                                0x0
+#define OTG3_OTG_CRC0_DATA_R__CRC0_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC0_DATA_G
+#define OTG3_OTG_CRC0_DATA_G__CRC0_G_Y__SHIFT                                                                 0x0
+#define OTG3_OTG_CRC0_DATA_G__CRC0_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG3_OTG_CRC0_DATA_B
+#define OTG3_OTG_CRC0_DATA_B__CRC0_B_CB__SHIFT                                                                0x0
+#define OTG3_OTG_CRC0_DATA_B__CRC0_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC0_DATA_C
+#define OTG3_OTG_CRC0_DATA_C__CRC0_C__SHIFT                                                                   0x0
+#define OTG3_OTG_CRC0_DATA_C__CRC0_C_MASK                                                                     0xFFFFFFFFL
+//OTG3_OTG_CRC1_WINDOWA_X_CONTROL
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL__OTG_CRC1_WINDOWA_X_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWA_Y_CONTROL
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL__OTG_CRC1_WINDOWA_Y_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWB_X_CONTROL
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL__OTG_CRC1_WINDOWB_X_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWB_Y_CONTROL
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START__SHIFT                                      0x0
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END__SHIFT                                        0x10
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_START_MASK                                        0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL__OTG_CRC1_WINDOWB_Y_END_MASK                                          0x7FFF0000L
+//OTG3_OTG_CRC1_DATA_R
+#define OTG3_OTG_CRC1_DATA_R__CRC1_R_CR__SHIFT                                                                0x0
+#define OTG3_OTG_CRC1_DATA_R__CRC1_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC1_DATA_G
+#define OTG3_OTG_CRC1_DATA_G__CRC1_G_Y__SHIFT                                                                 0x0
+#define OTG3_OTG_CRC1_DATA_G__CRC1_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG3_OTG_CRC1_DATA_B
+#define OTG3_OTG_CRC1_DATA_B__CRC1_B_CB__SHIFT                                                                0x0
+#define OTG3_OTG_CRC1_DATA_B__CRC1_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC1_DATA_C
+#define OTG3_OTG_CRC1_DATA_C__CRC1_C__SHIFT                                                                   0x0
+#define OTG3_OTG_CRC1_DATA_C__CRC1_C_MASK                                                                     0xFFFFFFFFL
+//OTG3_OTG_CRC2_DATA_R
+#define OTG3_OTG_CRC2_DATA_R__CRC2_R_CR__SHIFT                                                                0x0
+#define OTG3_OTG_CRC2_DATA_R__CRC2_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC2_DATA_G
+#define OTG3_OTG_CRC2_DATA_G__CRC2_G_Y__SHIFT                                                                 0x0
+#define OTG3_OTG_CRC2_DATA_G__CRC2_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG3_OTG_CRC2_DATA_B
+#define OTG3_OTG_CRC2_DATA_B__CRC2_B_CB__SHIFT                                                                0x0
+#define OTG3_OTG_CRC2_DATA_B__CRC2_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC2_DATA_C
+#define OTG3_OTG_CRC2_DATA_C__CRC2_C__SHIFT                                                                   0x0
+#define OTG3_OTG_CRC2_DATA_C__CRC2_C_MASK                                                                     0xFFFFFFFFL
+//OTG3_OTG_CRC3_DATA_R
+#define OTG3_OTG_CRC3_DATA_R__CRC3_R_CR__SHIFT                                                                0x0
+#define OTG3_OTG_CRC3_DATA_R__CRC3_R_CR_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC3_DATA_G
+#define OTG3_OTG_CRC3_DATA_G__CRC3_G_Y__SHIFT                                                                 0x0
+#define OTG3_OTG_CRC3_DATA_G__CRC3_G_Y_MASK                                                                   0xFFFFFFFFL
+//OTG3_OTG_CRC3_DATA_B
+#define OTG3_OTG_CRC3_DATA_B__CRC3_B_CB__SHIFT                                                                0x0
+#define OTG3_OTG_CRC3_DATA_B__CRC3_B_CB_MASK                                                                  0xFFFFFFFFL
+//OTG3_OTG_CRC3_DATA_C
+#define OTG3_OTG_CRC3_DATA_C__CRC3_C__SHIFT                                                                   0x0
+#define OTG3_OTG_CRC3_DATA_C__CRC3_C_MASK                                                                     0xFFFFFFFFL
+//OTG3_OTG_CRC0_DATA_AES
+#define OTG3_OTG_CRC0_DATA_AES__CRC0_AES__SHIFT                                                               0x0
+#define OTG3_OTG_CRC0_DATA_AES__CRC0_AES_MASK                                                                 0xFFFFFFFFL
+//OTG3_OTG_CRC_SIG_RED_GREEN_MASK
+#define OTG3_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK__SHIFT                                          0x0
+#define OTG3_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK__SHIFT                                        0x10
+#define OTG3_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_RED_MASK_MASK                                            0x0000FFFFL
+#define OTG3_OTG_CRC_SIG_RED_GREEN_MASK__OTG_CRC_SIG_GREEN_MASK_MASK                                          0xFFFF0000L
+//OTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK
+#define OTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK__SHIFT                                      0x0
+#define OTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK__SHIFT                                   0x10
+#define OTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_BLUE_MASK_MASK                                        0x0000FFFFL
+#define OTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK__OTG_CRC_SIG_CONTROL_MASK_MASK                                     0xFFFF0000L
+//OTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWA_X_CONTROL_READBACK__OTG_CRC0_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWA_Y_CONTROL_READBACK__OTG_CRC0_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWB_X_CONTROL_READBACK__OTG_CRC0_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC0_WINDOWB_Y_CONTROL_READBACK__OTG_CRC0_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWA_X_CONTROL_READBACK__OTG_CRC1_WINDOWA_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWA_Y_CONTROL_READBACK__OTG_CRC1_WINDOWA_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWB_X_CONTROL_READBACK__OTG_CRC1_WINDOWB_X_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK__SHIFT                    0x0
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK__SHIFT                      0x10
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_START_READBACK_MASK                      0x00007FFFL
+#define OTG3_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK__OTG_CRC1_WINDOWB_Y_END_READBACK_MASK                        0x7FFF0000L
+//OTG3_OTG_STATIC_SCREEN_CONTROL
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK__SHIFT                                   0x0
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT__SHIFT                                  0x10
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE__SHIFT                                          0x18
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS__SHIFT                                                  0x19
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS__SHIFT                                          0x1a
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR__SHIFT                                           0x1b
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE__SHIFT                                            0x1c
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE__SHIFT                                     0x1e
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE__SHIFT                               0x1f
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_EVENT_MASK_MASK                                     0x0000FFFFL
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_FRAME_COUNT_MASK                                    0x00FF0000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_ENABLE_MASK                                            0x01000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_SS_STATUS_MASK                                                    0x02000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_STATUS_MASK                                            0x04000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_CLEAR_MASK                                             0x08000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_CPU_SS_INT_TYPE_MASK                                              0x10000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_MASK                                       0x40000000L
+#define OTG3_OTG_STATIC_SCREEN_CONTROL__OTG_STATIC_SCREEN_OVERRIDE_VALUE_MASK                                 0x80000000L
+//OTG3_OTG_3D_STRUCTURE_CONTROL
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN__SHIFT                                             0x0
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE__SHIFT                                  0x8
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR__SHIFT                                 0xc
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET__SHIFT                                  0x10
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING__SHIFT                          0x11
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT__SHIFT                                        0x12
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_EN_MASK                                               0x00000001L
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_V_UPDATE_MODE_MASK                                    0x00000300L
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_STEREO_SEL_OVR_MASK                                   0x00001000L
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_MASK                                    0x00010000L
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_RESET_PENDING_MASK                            0x00020000L
+#define OTG3_OTG_3D_STRUCTURE_CONTROL__OTG_3D_STRUCTURE_F_COUNT_MASK                                          0x000C0000L
+//OTG3_OTG_GSL_VSYNC_GAP
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT__SHIFT                                                0x0
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY__SHIFT                                                0x8
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL__SHIFT                                           0x10
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE__SHIFT                                                 0x11
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR__SHIFT                                                0x13
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED__SHIFT                                             0x14
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER__SHIFT                                        0x17
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP__SHIFT                                                      0x18
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_LIMIT_MASK                                                  0x000000FFL
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_DELAY_MASK                                                  0x0000FF00L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_SOURCE_SEL_MASK                                             0x00010000L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MODE_MASK                                                   0x00060000L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_CLEAR_MASK                                                  0x00080000L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_OCCURRED_MASK                                               0x00100000L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASTER_FASTER_MASK                                          0x00800000L
+#define OTG3_OTG_GSL_VSYNC_GAP__OTG_GSL_VSYNC_GAP_MASK                                                        0xFF000000L
+//OTG3_OTG_MASTER_UPDATE_MODE
+#define OTG3_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE__SHIFT                                     0x0
+#define OTG3_OTG_MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE_MASK                                       0x00000003L
+//OTG3_OTG_CLOCK_CONTROL
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_EN__SHIFT                                                           0x0
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS__SHIFT                                                     0x1
+#define OTG3_OTG_CLOCK_CONTROL__OTG_SOFT_RESET__SHIFT                                                         0x4
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_ON__SHIFT                                                           0x8
+#define OTG3_OTG_CLOCK_CONTROL__OTG_BUSY__SHIFT                                                               0x10
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_EN_MASK                                                             0x00000001L
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_GATE_DIS_MASK                                                       0x00000002L
+#define OTG3_OTG_CLOCK_CONTROL__OTG_SOFT_RESET_MASK                                                           0x00000010L
+#define OTG3_OTG_CLOCK_CONTROL__OTG_CLOCK_ON_MASK                                                             0x00000100L
+#define OTG3_OTG_CLOCK_CONTROL__OTG_BUSY_MASK                                                                 0x00010000L
+//OTG3_OTG_VSTARTUP_PARAM
+#define OTG3_OTG_VSTARTUP_PARAM__VSTARTUP_START__SHIFT                                                        0x0
+#define OTG3_OTG_VSTARTUP_PARAM__VSTARTUP_START_MASK                                                          0x000003FFL
+//OTG3_OTG_VUPDATE_PARAM
+#define OTG3_OTG_VUPDATE_PARAM__VUPDATE_OFFSET__SHIFT                                                         0x0
+#define OTG3_OTG_VUPDATE_PARAM__VUPDATE_WIDTH__SHIFT                                                          0x10
+#define OTG3_OTG_VUPDATE_PARAM__VUPDATE_OFFSET_MASK                                                           0x0000FFFFL
+#define OTG3_OTG_VUPDATE_PARAM__VUPDATE_WIDTH_MASK                                                            0x03FF0000L
+//OTG3_OTG_VREADY_PARAM
+#define OTG3_OTG_VREADY_PARAM__VREADY_OFFSET__SHIFT                                                           0x0
+#define OTG3_OTG_VREADY_PARAM__VREADY_OFFSET_MASK                                                             0x0000FFFFL
+//OTG3_OTG_GLOBAL_SYNC_STATUS
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN__SHIFT                                                   0x0
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE__SHIFT                                                 0x1
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED__SHIFT                                           0x2
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS__SHIFT                                               0x3
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR__SHIFT                                              0x4
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN__SHIFT                                                    0x5
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE__SHIFT                                                  0x6
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL__SHIFT                                          0x7
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED__SHIFT                                            0x8
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS__SHIFT                                                0x9
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR__SHIFT                                               0xa
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS__SHIFT                                                    0xb
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN__SHIFT                                            0xc
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE__SHIFT                                          0xd
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED__SHIFT                                    0xe
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS__SHIFT                                        0xf
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR__SHIFT                                       0x10
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS__SHIFT                                            0x11
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN__SHIFT                                                     0x12
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE__SHIFT                                                   0x13
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED__SHIFT                                             0x14
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS__SHIFT                                                 0x15
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR__SHIFT                                                0x16
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS__SHIFT                                              0x18
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS__SHIFT                                               0x19
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_EN_MASK                                                     0x00000001L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_TYPE_MASK                                                   0x00000002L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_OCCURRED_MASK                                             0x00000004L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_INT_STATUS_MASK                                                 0x00000008L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VSTARTUP_EVENT_CLEAR_MASK                                                0x00000010L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_EN_MASK                                                      0x00000020L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_TYPE_MASK                                                    0x00000040L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_POSITION_SEL_MASK                                            0x00000080L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_OCCURRED_MASK                                              0x00000100L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_INT_STATUS_MASK                                                  0x00000200L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_EVENT_CLEAR_MASK                                                 0x00000400L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_STATUS_MASK                                                      0x00000800L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_EN_MASK                                              0x00001000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_TYPE_MASK                                            0x00002000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_OCCURRED_MASK                                      0x00004000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_INT_STATUS_MASK                                          0x00008000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_EVENT_CLEAR_MASK                                         0x00010000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VUPDATE_NO_LOCK_STATUS_MASK                                              0x00020000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_EN_MASK                                                       0x00040000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_TYPE_MASK                                                     0x00080000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_OCCURRED_MASK                                               0x00100000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_INT_STATUS_MASK                                                   0x00200000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__VREADY_EVENT_CLEAR_MASK                                                  0x00400000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__STEREO_SELECT_STATUS_MASK                                                0x01000000L
+#define OTG3_OTG_GLOBAL_SYNC_STATUS__FIELD_NUMBER_STATUS_MASK                                                 0x02000000L
+//OTG3_OTG_MASTER_UPDATE_LOCK
+#define OTG3_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK__SHIFT                                            0x0
+#define OTG3_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS__SHIFT                                                0x8
+#define OTG3_OTG_MASTER_UPDATE_LOCK__OTG_MASTER_UPDATE_LOCK_MASK                                              0x00000001L
+#define OTG3_OTG_MASTER_UPDATE_LOCK__UPDATE_LOCK_STATUS_MASK                                                  0x00000100L
+//OTG3_OTG_GSL_CONTROL
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL0_EN__SHIFT                                                              0x0
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL1_EN__SHIFT                                                              0x1
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL2_EN__SHIFT                                                              0x2
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN__SHIFT                                                        0x3
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE__SHIFT                                                      0x4
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY__SHIFT                                                      0x8
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY__SHIFT                                                      0x10
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS__SHIFT                                                 0x1c
+#define OTG3_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN__SHIFT                                            0x1f
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL0_EN_MASK                                                                0x00000001L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL1_EN_MASK                                                                0x00000002L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL2_EN_MASK                                                                0x00000004L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_MASTER_EN_MASK                                                          0x00000008L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_MASTER_MODE_MASK                                                        0x00000030L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_CHECK_DELAY_MASK                                                        0x00000F00L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_FORCE_DELAY_MASK                                                        0x001F0000L
+#define OTG3_OTG_GSL_CONTROL__OTG_GSL_CHECK_ALL_FIELDS_MASK                                                   0x10000000L
+#define OTG3_OTG_GSL_CONTROL__OTG_MASTER_UPDATE_LOCK_GSL_EN_MASK                                              0x80000000L
+//OTG3_OTG_GSL_WINDOW_X
+#define OTG3_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X__SHIFT                                                  0x0
+#define OTG3_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X__SHIFT                                                    0x10
+#define OTG3_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_START_X_MASK                                                    0x00007FFFL
+#define OTG3_OTG_GSL_WINDOW_X__OTG_GSL_WINDOW_END_X_MASK                                                      0x7FFF0000L
+//OTG3_OTG_GSL_WINDOW_Y
+#define OTG3_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y__SHIFT                                                  0x0
+#define OTG3_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y__SHIFT                                                    0x10
+#define OTG3_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_START_Y_MASK                                                    0x00007FFFL
+#define OTG3_OTG_GSL_WINDOW_Y__OTG_GSL_WINDOW_END_Y_MASK                                                      0x7FFF0000L
+//OTG3_OTG_VUPDATE_KEEPOUT
+#define OTG3_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET__SHIFT                      0x0
+#define OTG3_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET__SHIFT                        0x10
+#define OTG3_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN__SHIFT                            0x1f
+#define OTG3_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET_MASK                        0x0000FFFFL
+#define OTG3_OTG_VUPDATE_KEEPOUT__MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET_MASK                          0x03FF0000L
+#define OTG3_OTG_VUPDATE_KEEPOUT__OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN_MASK                              0x80000000L
+//OTG3_OTG_GLOBAL_CONTROL0
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X__SHIFT                                        0x0
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X__SHIFT                                          0x10
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN__SHIFT                                             0x1f
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_START_X_MASK                                          0x00007FFFL
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_END_X_MASK                                            0x7FFF0000L
+#define OTG3_OTG_GLOBAL_CONTROL0__MASTER_UPDATE_LOCK_DB_EN_MASK                                               0x80000000L
+//OTG3_OTG_GLOBAL_CONTROL1
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y__SHIFT                                        0x0
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y__SHIFT                                          0x10
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE__SHIFT                                       0x1f
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_START_Y_MASK                                          0x00007FFFL
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_DB_END_Y_MASK                                            0x7FFF0000L
+#define OTG3_OTG_GLOBAL_CONTROL1__MASTER_UPDATE_LOCK_VCOUNT_MODE_MASK                                         0x80000000L
+//OTG3_OTG_GLOBAL_CONTROL2
+#define OTG3_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN__SHIFT                                                0xa
+#define OTG3_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL__SHIFT                                              0x10
+#define OTG3_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL__SHIFT                                           0x19
+#define OTG3_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE__SHIFT                                            0x1e
+#define OTG3_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE__SHIFT                                                    0x1f
+#define OTG3_OTG_GLOBAL_CONTROL2__GLOBAL_UPDATE_LOCK_EN_MASK                                                  0x00000400L
+#define OTG3_OTG_GLOBAL_CONTROL2__MANUAL_FLOW_CONTROL_SEL_MASK                                                0x00070000L
+#define OTG3_OTG_GLOBAL_CONTROL2__OTG_MASTER_UPDATE_LOCK_SEL_MASK                                             0x0E000000L
+#define OTG3_OTG_GLOBAL_CONTROL2__OTG_VUPDATE_BLOCK_DISABLE_MASK                                              0x40000000L
+#define OTG3_OTG_GLOBAL_CONTROL2__DCCG_VUPDATE_MODE_MASK                                                      0x80000000L
+//OTG3_OTG_GLOBAL_CONTROL3
+#define OTG3_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD__SHIFT                                          0x0
+#define OTG3_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL__SHIFT                                     0x4
+#define OTG3_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL__SHIFT                                                 0x10
+#define OTG3_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL__SHIFT                                                   0x14
+#define OTG3_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_FIELD_MASK                                            0x00000003L
+#define OTG3_OTG_GLOBAL_CONTROL3__MASTER_UPDATE_LOCK_DB_STEREO_SEL_MASK                                       0x00000030L
+#define OTG3_OTG_GLOBAL_CONTROL3__DIG_UPDATE_FIELD_SEL_MASK                                                   0x00030000L
+#define OTG3_OTG_GLOBAL_CONTROL3__DIG_UPDATE_EYE_SEL_MASK                                                     0x00300000L
+//OTG3_OTG_GLOBAL_CONTROL4
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X__SHIFT                                                0x0
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y__SHIFT                                                0x10
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE__SHIFT                                               0x1f
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_X_MASK                                                  0x00007FFFL
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_POSITION_Y_MASK                                                  0x7FFF0000L
+#define OTG3_OTG_GLOBAL_CONTROL4__DIG_UPDATE_VCOUNT_MODE_MASK                                                 0x80000000L
+//OTG3_OTG_TRIG_MANUAL_CONTROL
+#define OTG3_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL__SHIFT                                              0x0
+#define OTG3_OTG_TRIG_MANUAL_CONTROL__TRIG_MANUAL_CONTROL_MASK                                                0x00000001L
+//OTG3_OTG_DRR_TIMING_INT_STATUS
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED__SHIFT                                 0x0
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT__SHIFT                             0x4
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR__SHIFT                           0x8
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK__SHIFT                         0xc
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE__SHIFT                        0xd
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED__SHIFT                                 0x10
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT__SHIFT                             0x14
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR__SHIFT                           0x18
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK__SHIFT                         0x1c
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE__SHIFT                        0x1d
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_MASK                                   0x00000001L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MASK                               0x00000010L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_CLEAR_MASK                             0x00000100L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_MSK_MASK                           0x00001000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_TIMING_UPDATE_OCCURRED_INT_TYPE_MASK                          0x00002000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_MASK                                   0x00010000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MASK                               0x00100000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_CLEAR_MASK                             0x01000000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_MSK_MASK                           0x10000000L
+#define OTG3_OTG_DRR_TIMING_INT_STATUS__OTG_DRR_V_TOTAL_REACH_OCCURRED_INT_TYPE_MASK                          0x20000000L
+//OTG3_OTG_DRR_V_TOTAL_REACH_RANGE
+#define OTG3_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE__SHIFT                            0x0
+#define OTG3_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE__SHIFT                            0x10
+#define OTG3_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_LOWER_RANGE_MASK                              0x00007FFFL
+#define OTG3_OTG_DRR_V_TOTAL_REACH_RANGE__OTG_DRR_V_TOTAL_REACH_UPPER_RANGE_MASK                              0x7FFF0000L
+//OTG3_OTG_DRR_V_TOTAL_CHANGE
+#define OTG3_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT__SHIFT                                      0x0
+#define OTG3_OTG_DRR_V_TOTAL_CHANGE__OTG_DRR_V_TOTAL_CHANGE_LIMIT_MASK                                        0x00007FFFL
+//OTG3_OTG_DRR_TRIGGER_WINDOW
+#define OTG3_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X__SHIFT                                    0x0
+#define OTG3_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X__SHIFT                                      0x10
+#define OTG3_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_START_X_MASK                                      0x00007FFFL
+#define OTG3_OTG_DRR_TRIGGER_WINDOW__OTG_DRR_TRIGGER_WINDOW_END_X_MASK                                        0x7FFF0000L
+//OTG3_OTG_DRR_CONTROL
+#define OTG3_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME__SHIFT                                                    0x0
+#define OTG3_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR__SHIFT                                             0x10
+#define OTG3_OTG_DRR_CONTROL__OTG_DRR_AVERAGE_FRAME_MASK                                                      0x00000003L
+#define OTG3_OTG_DRR_CONTROL__OTG_V_TOTAL_LAST_USED_BY_DRR_MASK                                               0x7FFF0000L
+//OTG3_OTG_DRR_CONTOL2
+#define OTG3_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR__SHIFT                                             0x0
+#define OTG3_OTG_DRR_CONTOL2__OTG_VCOUNT2_LAST_USED_BY_DRR_MASK                                               0xFFFFFFFFL
+//OTG3_OTG_M_CONST_DTO0
+#define OTG3_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE__SHIFT                                                   0x0
+#define OTG3_OTG_M_CONST_DTO0__OTG_M_CONST_DTO_PHASE_MASK                                                     0xFFFFFFFFL
+//OTG3_OTG_M_CONST_DTO1
+#define OTG3_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO__SHIFT                                                  0x0
+#define OTG3_OTG_M_CONST_DTO1__OTG_M_CONST_DTO_MODULO_MASK                                                    0xFFFFFFFFL
+//OTG3_OTG_REQUEST_CONTROL
+#define OTG3_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE__SHIFT                                     0x0
+#define OTG3_OTG_REQUEST_CONTROL__OTG_REQUEST_MODE_FOR_H_DUPLICATE_MASK                                       0x00000001L
+//OTG3_OTG_PIPE_UPDATE_STATUS
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING__SHIFT                                                  0x0
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING__SHIFT                                         0x4
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING__SHIFT                                         0x8
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS__SHIFT                                        0x10
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_FLIP_PENDING_MASK                                                    0x00000001L
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_DC_REG_UPDATE_PENDING_MASK                                           0x00000010L
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_CURSOR_UPDATE_PENDING_MASK                                           0x00000100L
+#define OTG3_OTG_PIPE_UPDATE_STATUS__OTG_VUPDATE_KEEPOUT_STATUS_MASK                                          0x00010000L
+//OTG3_OTG_PSTATE_REGISTER
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START__SHIFT                                             0x0
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND__SHIFT                                                    0x10
+#define OTG3_OTG_PSTATE_REGISTER__OTG_UNBLANK__SHIFT                                                          0x14
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN__SHIFT                                           0x18
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_KEEPOUT_START_MASK                                               0x00007FFFL
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_EXTEND_MASK                                                      0x00010000L
+#define OTG3_OTG_PSTATE_REGISTER__OTG_UNBLANK_MASK                                                            0x00100000L
+#define OTG3_OTG_PSTATE_REGISTER__OTG_PSTATE_ALLOW_WIDTH_MIN_MASK                                             0xFF000000L
+//OTG3_OTG_ENCRYPTION
+#define OTG3_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL__SHIFT                                             0x0
+#define OTG3_OTG_ENCRYPTION__OTG_ENCRYPTION_REQUIRED_LEVEL_MASK                                               0x00000003L
+//OTG3_OTG_SPARE_REGISTER
+#define OTG3_OTG_SPARE_REGISTER__OTG_SPARE_REG__SHIFT                                                         0x0
+#define OTG3_OTG_SPARE_REGISTER__OTG_SPARE_REG_MASK                                                           0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_optc_optc_misc_dispdec
+//GSL_SOURCE_SELECT
+#define GSL_SOURCE_SELECT__GSL0_READY_SOURCE_SEL__SHIFT                                                       0x0
+#define GSL_SOURCE_SELECT__GSL1_READY_SOURCE_SEL__SHIFT                                                       0x4
+#define GSL_SOURCE_SELECT__GSL2_READY_SOURCE_SEL__SHIFT                                                       0x8
+#define GSL_SOURCE_SELECT__GSL_TIMING_SYNC_SEL__SHIFT                                                         0x10
+#define GSL_SOURCE_SELECT__GSL0_READY_SOURCE_SEL_MASK                                                         0x00000007L
+#define GSL_SOURCE_SELECT__GSL1_READY_SOURCE_SEL_MASK                                                         0x00000070L
+#define GSL_SOURCE_SELECT__GSL2_READY_SOURCE_SEL_MASK                                                         0x00000700L
+#define GSL_SOURCE_SELECT__GSL_TIMING_SYNC_SEL_MASK                                                           0x00070000L
+//OPTC_DLPC_CONTROL
+#define OPTC_DLPC_CONTROL__OPTC_DLPC_SNAPSHOT_MUX__SHIFT                                                      0x0
+#define OPTC_DLPC_CONTROL__OPTC_DLPC_SNAPSHOT_MUX_MASK                                                        0x00000007L
+//OPTC_CLOCK_CONTROL
+#define OPTC_CLOCK_CONTROL__OPTC_DISPCLK_R_GATE_DIS__SHIFT                                                    0x0
+#define OPTC_CLOCK_CONTROL__OPTC_DISPCLK_R_CLOCK_ON__SHIFT                                                    0x1
+#define OPTC_CLOCK_CONTROL__OPTC_TEST_CLK_SEL__SHIFT                                                          0x8
+#define OPTC_CLOCK_CONTROL__OPTC_FGCG_REP_DIS__SHIFT                                                          0xf
+#define OPTC_CLOCK_CONTROL__OPTC_DISPCLK_R_GATE_DIS_MASK                                                      0x00000001L
+#define OPTC_CLOCK_CONTROL__OPTC_DISPCLK_R_CLOCK_ON_MASK                                                      0x00000002L
+#define OPTC_CLOCK_CONTROL__OPTC_TEST_CLK_SEL_MASK                                                            0x00000F00L
+#define OPTC_CLOCK_CONTROL__OPTC_FGCG_REP_DIS_MASK                                                            0x00008000L
+//ODM_MEM_PWR_CTRL
+#define ODM_MEM_PWR_CTRL__ODM_MEM0_PWR_FORCE__SHIFT                                                           0x0
+#define ODM_MEM_PWR_CTRL__ODM_MEM0_PWR_DIS__SHIFT                                                             0x2
+#define ODM_MEM_PWR_CTRL__ODM_MEM1_PWR_FORCE__SHIFT                                                           0x4
+#define ODM_MEM_PWR_CTRL__ODM_MEM1_PWR_DIS__SHIFT                                                             0x6
+#define ODM_MEM_PWR_CTRL__ODM_MEM2_PWR_FORCE__SHIFT                                                           0x8
+#define ODM_MEM_PWR_CTRL__ODM_MEM2_PWR_DIS__SHIFT                                                             0xa
+#define ODM_MEM_PWR_CTRL__ODM_MEM3_PWR_FORCE__SHIFT                                                           0xc
+#define ODM_MEM_PWR_CTRL__ODM_MEM3_PWR_DIS__SHIFT                                                             0xe
+#define ODM_MEM_PWR_CTRL__ODM_MEM4_PWR_FORCE__SHIFT                                                           0x10
+#define ODM_MEM_PWR_CTRL__ODM_MEM4_PWR_DIS__SHIFT                                                             0x12
+#define ODM_MEM_PWR_CTRL__ODM_MEM5_PWR_FORCE__SHIFT                                                           0x14
+#define ODM_MEM_PWR_CTRL__ODM_MEM5_PWR_DIS__SHIFT                                                             0x16
+#define ODM_MEM_PWR_CTRL__ODM_MEM6_PWR_FORCE__SHIFT                                                           0x18
+#define ODM_MEM_PWR_CTRL__ODM_MEM6_PWR_DIS__SHIFT                                                             0x1a
+#define ODM_MEM_PWR_CTRL__ODM_MEM7_PWR_FORCE__SHIFT                                                           0x1c
+#define ODM_MEM_PWR_CTRL__ODM_MEM7_PWR_DIS__SHIFT                                                             0x1e
+#define ODM_MEM_PWR_CTRL__ODM_MEM0_PWR_FORCE_MASK                                                             0x00000003L
+#define ODM_MEM_PWR_CTRL__ODM_MEM0_PWR_DIS_MASK                                                               0x00000004L
+#define ODM_MEM_PWR_CTRL__ODM_MEM1_PWR_FORCE_MASK                                                             0x00000030L
+#define ODM_MEM_PWR_CTRL__ODM_MEM1_PWR_DIS_MASK                                                               0x00000040L
+#define ODM_MEM_PWR_CTRL__ODM_MEM2_PWR_FORCE_MASK                                                             0x00000300L
+#define ODM_MEM_PWR_CTRL__ODM_MEM2_PWR_DIS_MASK                                                               0x00000400L
+#define ODM_MEM_PWR_CTRL__ODM_MEM3_PWR_FORCE_MASK                                                             0x00003000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM3_PWR_DIS_MASK                                                               0x00004000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM4_PWR_FORCE_MASK                                                             0x00030000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM4_PWR_DIS_MASK                                                               0x00040000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM5_PWR_FORCE_MASK                                                             0x00300000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM5_PWR_DIS_MASK                                                               0x00400000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM6_PWR_FORCE_MASK                                                             0x03000000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM6_PWR_DIS_MASK                                                               0x04000000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM7_PWR_FORCE_MASK                                                             0x30000000L
+#define ODM_MEM_PWR_CTRL__ODM_MEM7_PWR_DIS_MASK                                                               0x40000000L
+//ODM_MEM_PWR_CTRL3
+#define ODM_MEM_PWR_CTRL3__ODM_MEM_UNASSIGNED_PWR_MODE__SHIFT                                                 0x0
+#define ODM_MEM_PWR_CTRL3__ODM_MEM_VBLANK_PWR_MODE__SHIFT                                                     0x2
+#define ODM_MEM_PWR_CTRL3__ODM_MEM_UNASSIGNED_PWR_MODE_MASK                                                   0x00000003L
+#define ODM_MEM_PWR_CTRL3__ODM_MEM_VBLANK_PWR_MODE_MASK                                                       0x0000000CL
+//ODM_MEM_PWR_STATUS
+#define ODM_MEM_PWR_STATUS__ODM_MEM0_PWR_STATE__SHIFT                                                         0x0
+#define ODM_MEM_PWR_STATUS__ODM_MEM1_PWR_STATE__SHIFT                                                         0x2
+#define ODM_MEM_PWR_STATUS__ODM_MEM2_PWR_STATE__SHIFT                                                         0x4
+#define ODM_MEM_PWR_STATUS__ODM_MEM3_PWR_STATE__SHIFT                                                         0x6
+#define ODM_MEM_PWR_STATUS__ODM_MEM4_PWR_STATE__SHIFT                                                         0x8
+#define ODM_MEM_PWR_STATUS__ODM_MEM5_PWR_STATE__SHIFT                                                         0xa
+#define ODM_MEM_PWR_STATUS__ODM_MEM6_PWR_STATE__SHIFT                                                         0xc
+#define ODM_MEM_PWR_STATUS__ODM_MEM7_PWR_STATE__SHIFT                                                         0xe
+#define ODM_MEM_PWR_STATUS__ODM_MEM0_PWR_STATE_MASK                                                           0x00000003L
+#define ODM_MEM_PWR_STATUS__ODM_MEM1_PWR_STATE_MASK                                                           0x0000000CL
+#define ODM_MEM_PWR_STATUS__ODM_MEM2_PWR_STATE_MASK                                                           0x00000030L
+#define ODM_MEM_PWR_STATUS__ODM_MEM3_PWR_STATE_MASK                                                           0x000000C0L
+#define ODM_MEM_PWR_STATUS__ODM_MEM4_PWR_STATE_MASK                                                           0x00000300L
+#define ODM_MEM_PWR_STATUS__ODM_MEM5_PWR_STATE_MASK                                                           0x00000C00L
+#define ODM_MEM_PWR_STATUS__ODM_MEM6_PWR_STATE_MASK                                                           0x00003000L
+#define ODM_MEM_PWR_STATUS__ODM_MEM7_PWR_STATE_MASK                                                           0x0000C000L
+//OPTC_MISC_SPARE_REGISTER
+#define OPTC_MISC_SPARE_REGISTER__OPTC_MISC_SPARE_REG__SHIFT                                                  0x0
+#define OPTC_MISC_SPARE_REGISTER__OPTC_MISC_SPARE_REG_MASK                                                    0x000000FFL
+
+
+// addressBlock: dce_dc_optc_optc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON15_PERFCOUNTER_CNTL
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON15_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON15_PERFCOUNTER_CNTL2
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON15_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON15_PERFCOUNTER_STATE
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON15_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON15_PERFMON_CNTL
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON15_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON15_PERFMON_CNTL2
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON15_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON15_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON15_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON15_PERFMON_CVALUE_LOW
+#define DC_PERFMON15_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON15_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON15_PERFMON_HI
+#define DC_PERFMON15_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON15_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON15_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON15_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON15_PERFMON_LOW
+#define DC_PERFMON15_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON15_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dout_i2c_dispdec
+//DC_I2C_CONTROL
+#define DC_I2C_CONTROL__DC_I2C_GO__SHIFT                                                                      0x0
+#define DC_I2C_CONTROL__DC_I2C_SOFT_RESET__SHIFT                                                              0x1
+#define DC_I2C_CONTROL__DC_I2C_SEND_RESET__SHIFT                                                              0x2
+#define DC_I2C_CONTROL__DC_I2C_SW_STATUS_RESET__SHIFT                                                         0x3
+#define DC_I2C_CONTROL__DC_I2C_DDC_SELECT__SHIFT                                                              0x8
+#define DC_I2C_CONTROL__DC_I2C_TRANSACTION_COUNT__SHIFT                                                       0x14
+#define DC_I2C_CONTROL__DC_I2C_DBG_REF_SEL__SHIFT                                                             0x1f
+#define DC_I2C_CONTROL__DC_I2C_GO_MASK                                                                        0x00000001L
+#define DC_I2C_CONTROL__DC_I2C_SOFT_RESET_MASK                                                                0x00000002L
+#define DC_I2C_CONTROL__DC_I2C_SEND_RESET_MASK                                                                0x00000004L
+#define DC_I2C_CONTROL__DC_I2C_SW_STATUS_RESET_MASK                                                           0x00000008L
+#define DC_I2C_CONTROL__DC_I2C_DDC_SELECT_MASK                                                                0x00000700L
+#define DC_I2C_CONTROL__DC_I2C_TRANSACTION_COUNT_MASK                                                         0x00300000L
+#define DC_I2C_CONTROL__DC_I2C_DBG_REF_SEL_MASK                                                               0x80000000L
+//DC_I2C_ARBITRATION
+#define DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY__SHIFT                                                         0x0
+#define DC_I2C_ARBITRATION__DC_I2C_REG_RW_CNTL_STATUS__SHIFT                                                  0x2
+#define DC_I2C_ARBITRATION__DC_I2C_NO_QUEUED_SW_GO__SHIFT                                                     0x4
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_HW_XFER__SHIFT                                                       0x8
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_SW_XFER__SHIFT                                                       0xc
+#define DC_I2C_ARBITRATION__DC_I2C_SW_USE_I2C_REG_REQ__SHIFT                                                  0x14
+#define DC_I2C_ARBITRATION__DC_I2C_SW_DONE_USING_I2C_REG__SHIFT                                               0x15
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_USE_I2C_REG_REQ__SHIFT                                                0x18
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_DONE_USING_I2C_REG__SHIFT                                             0x19
+#define DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY_MASK                                                           0x00000003L
+#define DC_I2C_ARBITRATION__DC_I2C_REG_RW_CNTL_STATUS_MASK                                                    0x0000000CL
+#define DC_I2C_ARBITRATION__DC_I2C_NO_QUEUED_SW_GO_MASK                                                       0x00000010L
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_HW_XFER_MASK                                                         0x00000100L
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_SW_XFER_MASK                                                         0x00001000L
+#define DC_I2C_ARBITRATION__DC_I2C_SW_USE_I2C_REG_REQ_MASK                                                    0x00100000L
+#define DC_I2C_ARBITRATION__DC_I2C_SW_DONE_USING_I2C_REG_MASK                                                 0x00200000L
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_USE_I2C_REG_REQ_MASK                                                  0x01000000L
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_DONE_USING_I2C_REG_MASK                                               0x02000000L
+//DC_I2C_INTERRUPT_CONTROL
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_INT__SHIFT                                                   0x0
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_ACK__SHIFT                                                   0x1
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_MASK__SHIFT                                                  0x2
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_INT__SHIFT                                              0x4
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_ACK__SHIFT                                              0x5
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_MASK__SHIFT                                             0x6
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_INT__SHIFT                                              0x8
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_ACK__SHIFT                                              0x9
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_MASK__SHIFT                                             0xa
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_INT__SHIFT                                              0xc
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_ACK__SHIFT                                              0xd
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_MASK__SHIFT                                             0xe
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_INT__SHIFT                                              0x10
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_ACK__SHIFT                                              0x11
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_MASK__SHIFT                                             0x12
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_INT__SHIFT                                              0x14
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_ACK__SHIFT                                              0x15
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_MASK__SHIFT                                             0x16
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_INT__SHIFT                                              0x18
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_ACK__SHIFT                                              0x19
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_MASK__SHIFT                                             0x1a
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_INT__SHIFT                                            0x1b
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_ACK__SHIFT                                            0x1c
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_MASK__SHIFT                                           0x1d
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_INT_MASK                                                     0x00000001L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_ACK_MASK                                                     0x00000002L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_MASK_MASK                                                    0x00000004L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_INT_MASK                                                0x00000010L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_ACK_MASK                                                0x00000020L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_MASK_MASK                                               0x00000040L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_INT_MASK                                                0x00000100L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_ACK_MASK                                                0x00000200L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_MASK_MASK                                               0x00000400L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_INT_MASK                                                0x00001000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_ACK_MASK                                                0x00002000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_MASK_MASK                                               0x00004000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_INT_MASK                                                0x00010000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_ACK_MASK                                                0x00020000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_MASK_MASK                                               0x00040000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_INT_MASK                                                0x00100000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_ACK_MASK                                                0x00200000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_MASK_MASK                                               0x00400000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_INT_MASK                                                0x01000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_ACK_MASK                                                0x02000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_MASK_MASK                                               0x04000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_INT_MASK                                              0x08000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_ACK_MASK                                              0x10000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_MASK_MASK                                             0x20000000L
+//DC_I2C_SW_STATUS
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STATUS__SHIFT                                                             0x0
+#define DC_I2C_SW_STATUS__DC_I2C_SW_DONE__SHIFT                                                               0x2
+#define DC_I2C_SW_STATUS__DC_I2C_SW_ABORTED__SHIFT                                                            0x4
+#define DC_I2C_SW_STATUS__DC_I2C_SW_TIMEOUT__SHIFT                                                            0x5
+#define DC_I2C_SW_STATUS__DC_I2C_SW_INTERRUPTED__SHIFT                                                        0x6
+#define DC_I2C_SW_STATUS__DC_I2C_SW_BUFFER_OVERFLOW__SHIFT                                                    0x7
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STOPPED_ON_NACK__SHIFT                                                    0x8
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK0__SHIFT                                                              0xc
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK1__SHIFT                                                              0xd
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK2__SHIFT                                                              0xe
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK3__SHIFT                                                              0xf
+#define DC_I2C_SW_STATUS__DC_I2C_SW_REQ__SHIFT                                                                0x12
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STATUS_MASK                                                               0x00000003L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_DONE_MASK                                                                 0x00000004L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_ABORTED_MASK                                                              0x00000010L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_TIMEOUT_MASK                                                              0x00000020L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_INTERRUPTED_MASK                                                          0x00000040L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_BUFFER_OVERFLOW_MASK                                                      0x00000080L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STOPPED_ON_NACK_MASK                                                      0x00000100L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK0_MASK                                                                0x00001000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK1_MASK                                                                0x00002000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK2_MASK                                                                0x00004000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK3_MASK                                                                0x00008000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_REQ_MASK                                                                  0x00040000L
+//DC_I2C_DDC1_HW_STATUS
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_STATUS__SHIFT                                                   0x0
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_DONE__SHIFT                                                     0x3
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_REQ__SHIFT                                                      0x10
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_URG__SHIFT                                                      0x11
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATUS__SHIFT                                          0x14
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_NUM_VALID_TRIES__SHIFT                                 0x18
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATE__SHIFT                                           0x1c
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_STATUS_MASK                                                     0x00000003L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_DONE_MASK                                                       0x00000008L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_REQ_MASK                                                        0x00010000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_URG_MASK                                                        0x00020000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATUS_MASK                                            0x00100000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_NUM_VALID_TRIES_MASK                                   0x0F000000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATE_MASK                                             0x70000000L
+//DC_I2C_DDC2_HW_STATUS
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_STATUS__SHIFT                                                   0x0
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_DONE__SHIFT                                                     0x3
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_REQ__SHIFT                                                      0x10
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_URG__SHIFT                                                      0x11
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATUS__SHIFT                                          0x14
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_NUM_VALID_TRIES__SHIFT                                 0x18
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATE__SHIFT                                           0x1c
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_STATUS_MASK                                                     0x00000003L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_DONE_MASK                                                       0x00000008L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_REQ_MASK                                                        0x00010000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_URG_MASK                                                        0x00020000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATUS_MASK                                            0x00100000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_NUM_VALID_TRIES_MASK                                   0x0F000000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATE_MASK                                             0x70000000L
+//DC_I2C_DDC3_HW_STATUS
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_STATUS__SHIFT                                                   0x0
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_DONE__SHIFT                                                     0x3
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_REQ__SHIFT                                                      0x10
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_URG__SHIFT                                                      0x11
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATUS__SHIFT                                          0x14
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_NUM_VALID_TRIES__SHIFT                                 0x18
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATE__SHIFT                                           0x1c
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_STATUS_MASK                                                     0x00000003L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_DONE_MASK                                                       0x00000008L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_REQ_MASK                                                        0x00010000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_URG_MASK                                                        0x00020000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATUS_MASK                                            0x00100000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_NUM_VALID_TRIES_MASK                                   0x0F000000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATE_MASK                                             0x70000000L
+//DC_I2C_DDC4_HW_STATUS
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_STATUS__SHIFT                                                   0x0
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_DONE__SHIFT                                                     0x3
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_REQ__SHIFT                                                      0x10
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_URG__SHIFT                                                      0x11
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATUS__SHIFT                                          0x14
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_NUM_VALID_TRIES__SHIFT                                 0x18
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATE__SHIFT                                           0x1c
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_STATUS_MASK                                                     0x00000003L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_DONE_MASK                                                       0x00000008L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_REQ_MASK                                                        0x00010000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_URG_MASK                                                        0x00020000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATUS_MASK                                            0x00100000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_NUM_VALID_TRIES_MASK                                   0x0F000000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATE_MASK                                             0x70000000L
+//DC_I2C_DDC5_HW_STATUS
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_STATUS__SHIFT                                                   0x0
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_DONE__SHIFT                                                     0x3
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_REQ__SHIFT                                                      0x10
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_URG__SHIFT                                                      0x11
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATUS__SHIFT                                          0x14
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_NUM_VALID_TRIES__SHIFT                                 0x18
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATE__SHIFT                                           0x1c
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_STATUS_MASK                                                     0x00000003L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_DONE_MASK                                                       0x00000008L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_REQ_MASK                                                        0x00010000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_URG_MASK                                                        0x00020000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATUS_MASK                                            0x00100000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_NUM_VALID_TRIES_MASK                                   0x0F000000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATE_MASK                                             0x70000000L
+//DC_I2C_DDC1_SPEED
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_THRESHOLD__SHIFT                                                       0x0
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_DISABLE_FILTER_DURING_STALL__SHIFT                                     0x4
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_START_STOP_TIMING_CNTL__SHIFT                                          0x8
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_PRESCALE__SHIFT                                                        0x10
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_THRESHOLD_MASK                                                         0x00000003L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_DISABLE_FILTER_DURING_STALL_MASK                                       0x00000010L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_START_STOP_TIMING_CNTL_MASK                                            0x00000300L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_PRESCALE_MASK                                                          0xFFFF0000L
+//DC_I2C_DDC1_SETUP
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_EN__SHIFT                                                   0x0
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_SEL__SHIFT                                                  0x1
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_SEND_RESET_LENGTH__SHIFT                                               0x2
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_EN__SHIFT                                                          0x3
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_ENABLE__SHIFT                                              0x4
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_MODE__SHIFT                                                0x5
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_ENABLE__SHIFT                                                          0x6
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_DRIVE_EN__SHIFT                                                    0x7
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_BYTE_DELAY__SHIFT                                                0x8
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_TRANSACTION_DELAY__SHIFT                                         0x10
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_TIME_LIMIT__SHIFT                                                      0x18
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_EN_MASK                                                     0x00000001L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_SEL_MASK                                                    0x00000002L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_SEND_RESET_LENGTH_MASK                                                 0x00000004L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_EN_MASK                                                            0x00000008L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_ENABLE_MASK                                                0x00000010L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_MODE_MASK                                                  0x00000020L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_ENABLE_MASK                                                            0x00000040L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_DRIVE_EN_MASK                                                      0x00000080L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_BYTE_DELAY_MASK                                                  0x0000FF00L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_TRANSACTION_DELAY_MASK                                           0x00FF0000L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_TIME_LIMIT_MASK                                                        0xFF000000L
+//DC_I2C_DDC2_SPEED
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_THRESHOLD__SHIFT                                                       0x0
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_DISABLE_FILTER_DURING_STALL__SHIFT                                     0x4
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_START_STOP_TIMING_CNTL__SHIFT                                          0x8
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_PRESCALE__SHIFT                                                        0x10
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_THRESHOLD_MASK                                                         0x00000003L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_DISABLE_FILTER_DURING_STALL_MASK                                       0x00000010L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_START_STOP_TIMING_CNTL_MASK                                            0x00000300L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_PRESCALE_MASK                                                          0xFFFF0000L
+//DC_I2C_DDC2_SETUP
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_EN__SHIFT                                                   0x0
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_SEL__SHIFT                                                  0x1
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_SEND_RESET_LENGTH__SHIFT                                               0x2
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_EN__SHIFT                                                          0x3
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_ENABLE__SHIFT                                              0x4
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_MODE__SHIFT                                                0x5
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_ENABLE__SHIFT                                                          0x6
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_DRIVE_EN__SHIFT                                                    0x7
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_BYTE_DELAY__SHIFT                                                0x8
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_TRANSACTION_DELAY__SHIFT                                         0x10
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_TIME_LIMIT__SHIFT                                                      0x18
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_EN_MASK                                                     0x00000001L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_SEL_MASK                                                    0x00000002L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_SEND_RESET_LENGTH_MASK                                                 0x00000004L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_EN_MASK                                                            0x00000008L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_ENABLE_MASK                                                0x00000010L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_MODE_MASK                                                  0x00000020L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_ENABLE_MASK                                                            0x00000040L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_DRIVE_EN_MASK                                                      0x00000080L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_BYTE_DELAY_MASK                                                  0x0000FF00L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_TRANSACTION_DELAY_MASK                                           0x00FF0000L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_TIME_LIMIT_MASK                                                        0xFF000000L
+//DC_I2C_DDC3_SPEED
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_THRESHOLD__SHIFT                                                       0x0
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_DISABLE_FILTER_DURING_STALL__SHIFT                                     0x4
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_START_STOP_TIMING_CNTL__SHIFT                                          0x8
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_PRESCALE__SHIFT                                                        0x10
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_THRESHOLD_MASK                                                         0x00000003L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_DISABLE_FILTER_DURING_STALL_MASK                                       0x00000010L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_START_STOP_TIMING_CNTL_MASK                                            0x00000300L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_PRESCALE_MASK                                                          0xFFFF0000L
+//DC_I2C_DDC3_SETUP
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_EN__SHIFT                                                   0x0
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_SEL__SHIFT                                                  0x1
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_SEND_RESET_LENGTH__SHIFT                                               0x2
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_EN__SHIFT                                                          0x3
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_ENABLE__SHIFT                                              0x4
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_MODE__SHIFT                                                0x5
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_ENABLE__SHIFT                                                          0x6
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_DRIVE_EN__SHIFT                                                    0x7
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_BYTE_DELAY__SHIFT                                                0x8
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_TRANSACTION_DELAY__SHIFT                                         0x10
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_TIME_LIMIT__SHIFT                                                      0x18
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_EN_MASK                                                     0x00000001L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_SEL_MASK                                                    0x00000002L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_SEND_RESET_LENGTH_MASK                                                 0x00000004L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_EN_MASK                                                            0x00000008L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_ENABLE_MASK                                                0x00000010L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_MODE_MASK                                                  0x00000020L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_ENABLE_MASK                                                            0x00000040L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_DRIVE_EN_MASK                                                      0x00000080L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_BYTE_DELAY_MASK                                                  0x0000FF00L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_TRANSACTION_DELAY_MASK                                           0x00FF0000L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_TIME_LIMIT_MASK                                                        0xFF000000L
+//DC_I2C_DDC4_SPEED
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_THRESHOLD__SHIFT                                                       0x0
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_DISABLE_FILTER_DURING_STALL__SHIFT                                     0x4
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_START_STOP_TIMING_CNTL__SHIFT                                          0x8
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_PRESCALE__SHIFT                                                        0x10
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_THRESHOLD_MASK                                                         0x00000003L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_DISABLE_FILTER_DURING_STALL_MASK                                       0x00000010L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_START_STOP_TIMING_CNTL_MASK                                            0x00000300L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_PRESCALE_MASK                                                          0xFFFF0000L
+//DC_I2C_DDC4_SETUP
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_EN__SHIFT                                                   0x0
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_SEL__SHIFT                                                  0x1
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_SEND_RESET_LENGTH__SHIFT                                               0x2
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_EN__SHIFT                                                          0x3
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_ENABLE__SHIFT                                              0x4
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_MODE__SHIFT                                                0x5
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_ENABLE__SHIFT                                                          0x6
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_DRIVE_EN__SHIFT                                                    0x7
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_BYTE_DELAY__SHIFT                                                0x8
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_TRANSACTION_DELAY__SHIFT                                         0x10
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_TIME_LIMIT__SHIFT                                                      0x18
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_EN_MASK                                                     0x00000001L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_SEL_MASK                                                    0x00000002L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_SEND_RESET_LENGTH_MASK                                                 0x00000004L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_EN_MASK                                                            0x00000008L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_ENABLE_MASK                                                0x00000010L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_MODE_MASK                                                  0x00000020L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_ENABLE_MASK                                                            0x00000040L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_DRIVE_EN_MASK                                                      0x00000080L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_BYTE_DELAY_MASK                                                  0x0000FF00L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_TRANSACTION_DELAY_MASK                                           0x00FF0000L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_TIME_LIMIT_MASK                                                        0xFF000000L
+//DC_I2C_DDC5_SPEED
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_THRESHOLD__SHIFT                                                       0x0
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_DISABLE_FILTER_DURING_STALL__SHIFT                                     0x4
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_START_STOP_TIMING_CNTL__SHIFT                                          0x8
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_PRESCALE__SHIFT                                                        0x10
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_THRESHOLD_MASK                                                         0x00000003L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_DISABLE_FILTER_DURING_STALL_MASK                                       0x00000010L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_START_STOP_TIMING_CNTL_MASK                                            0x00000300L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_PRESCALE_MASK                                                          0xFFFF0000L
+//DC_I2C_DDC5_SETUP
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_EN__SHIFT                                                   0x0
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_SEL__SHIFT                                                  0x1
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_SEND_RESET_LENGTH__SHIFT                                               0x2
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_EN__SHIFT                                                          0x3
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_ENABLE__SHIFT                                              0x4
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_MODE__SHIFT                                                0x5
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_ENABLE__SHIFT                                                          0x6
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_DRIVE_EN__SHIFT                                                    0x7
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_BYTE_DELAY__SHIFT                                                0x8
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_TRANSACTION_DELAY__SHIFT                                         0x10
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_TIME_LIMIT__SHIFT                                                      0x18
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_EN_MASK                                                     0x00000001L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_SEL_MASK                                                    0x00000002L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_SEND_RESET_LENGTH_MASK                                                 0x00000004L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_EN_MASK                                                            0x00000008L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_ENABLE_MASK                                                0x00000010L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_MODE_MASK                                                  0x00000020L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_ENABLE_MASK                                                            0x00000040L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_DRIVE_EN_MASK                                                      0x00000080L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_BYTE_DELAY_MASK                                                  0x0000FF00L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_TRANSACTION_DELAY_MASK                                           0x00FF0000L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_TIME_LIMIT_MASK                                                        0xFF000000L
+//DC_I2C_TRANSACTION0
+#define DC_I2C_TRANSACTION0__DC_I2C_RW0__SHIFT                                                                0x0
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP_ON_NACK0__SHIFT                                                      0x8
+#define DC_I2C_TRANSACTION0__DC_I2C_START0__SHIFT                                                             0xc
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP0__SHIFT                                                              0xd
+#define DC_I2C_TRANSACTION0__DC_I2C_COUNT0__SHIFT                                                             0x10
+#define DC_I2C_TRANSACTION0__DC_I2C_RW0_MASK                                                                  0x00000001L
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP_ON_NACK0_MASK                                                        0x00000100L
+#define DC_I2C_TRANSACTION0__DC_I2C_START0_MASK                                                               0x00001000L
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP0_MASK                                                                0x00002000L
+#define DC_I2C_TRANSACTION0__DC_I2C_COUNT0_MASK                                                               0x03FF0000L
+//DC_I2C_TRANSACTION1
+#define DC_I2C_TRANSACTION1__DC_I2C_RW1__SHIFT                                                                0x0
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP_ON_NACK1__SHIFT                                                      0x8
+#define DC_I2C_TRANSACTION1__DC_I2C_START1__SHIFT                                                             0xc
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP1__SHIFT                                                              0xd
+#define DC_I2C_TRANSACTION1__DC_I2C_COUNT1__SHIFT                                                             0x10
+#define DC_I2C_TRANSACTION1__DC_I2C_RW1_MASK                                                                  0x00000001L
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP_ON_NACK1_MASK                                                        0x00000100L
+#define DC_I2C_TRANSACTION1__DC_I2C_START1_MASK                                                               0x00001000L
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP1_MASK                                                                0x00002000L
+#define DC_I2C_TRANSACTION1__DC_I2C_COUNT1_MASK                                                               0x03FF0000L
+//DC_I2C_TRANSACTION2
+#define DC_I2C_TRANSACTION2__DC_I2C_RW2__SHIFT                                                                0x0
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP_ON_NACK2__SHIFT                                                      0x8
+#define DC_I2C_TRANSACTION2__DC_I2C_START2__SHIFT                                                             0xc
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP2__SHIFT                                                              0xd
+#define DC_I2C_TRANSACTION2__DC_I2C_COUNT2__SHIFT                                                             0x10
+#define DC_I2C_TRANSACTION2__DC_I2C_RW2_MASK                                                                  0x00000001L
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP_ON_NACK2_MASK                                                        0x00000100L
+#define DC_I2C_TRANSACTION2__DC_I2C_START2_MASK                                                               0x00001000L
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP2_MASK                                                                0x00002000L
+#define DC_I2C_TRANSACTION2__DC_I2C_COUNT2_MASK                                                               0x03FF0000L
+//DC_I2C_TRANSACTION3
+#define DC_I2C_TRANSACTION3__DC_I2C_RW3__SHIFT                                                                0x0
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP_ON_NACK3__SHIFT                                                      0x8
+#define DC_I2C_TRANSACTION3__DC_I2C_START3__SHIFT                                                             0xc
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP3__SHIFT                                                              0xd
+#define DC_I2C_TRANSACTION3__DC_I2C_COUNT3__SHIFT                                                             0x10
+#define DC_I2C_TRANSACTION3__DC_I2C_RW3_MASK                                                                  0x00000001L
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP_ON_NACK3_MASK                                                        0x00000100L
+#define DC_I2C_TRANSACTION3__DC_I2C_START3_MASK                                                               0x00001000L
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP3_MASK                                                                0x00002000L
+#define DC_I2C_TRANSACTION3__DC_I2C_COUNT3_MASK                                                               0x03FF0000L
+//DC_I2C_DATA
+#define DC_I2C_DATA__DC_I2C_DATA_RW__SHIFT                                                                    0x0
+#define DC_I2C_DATA__DC_I2C_DATA__SHIFT                                                                       0x8
+#define DC_I2C_DATA__DC_I2C_INDEX__SHIFT                                                                      0x10
+#define DC_I2C_DATA__DC_I2C_INDEX_WRITE__SHIFT                                                                0x1f
+#define DC_I2C_DATA__DC_I2C_DATA_RW_MASK                                                                      0x00000001L
+#define DC_I2C_DATA__DC_I2C_DATA_MASK                                                                         0x0000FF00L
+#define DC_I2C_DATA__DC_I2C_INDEX_MASK                                                                        0x03FF0000L
+#define DC_I2C_DATA__DC_I2C_INDEX_WRITE_MASK                                                                  0x80000000L
+//DC_I2C_EDID_DETECT_CTRL
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_WAIT_TIME__SHIFT                                          0x0
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_NUM_TRIES_UNTIL_VALID__SHIFT                              0x14
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_SEND_RESET__SHIFT                                         0x1c
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_WAIT_TIME_MASK                                            0x0000FFFFL
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_NUM_TRIES_UNTIL_VALID_MASK                                0x00F00000L
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_SEND_RESET_MASK                                           0x10000000L
+//DC_I2C_READ_REQUEST_INTERRUPT
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_OCCURRED__SHIFT                               0x0
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_INT__SHIFT                                    0x1
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_ACK__SHIFT                                    0x2
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_MASK__SHIFT                                   0x3
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_OCCURRED__SHIFT                               0x4
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_INT__SHIFT                                    0x5
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_ACK__SHIFT                                    0x6
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_MASK__SHIFT                                   0x7
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_OCCURRED__SHIFT                               0x8
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_INT__SHIFT                                    0x9
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_ACK__SHIFT                                    0xa
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_MASK__SHIFT                                   0xb
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_OCCURRED__SHIFT                               0xc
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_INT__SHIFT                                    0xd
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_ACK__SHIFT                                    0xe
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_MASK__SHIFT                                   0xf
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_OCCURRED__SHIFT                               0x10
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_INT__SHIFT                                    0x11
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_ACK__SHIFT                                    0x12
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_MASK__SHIFT                                   0x13
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_OCCURRED__SHIFT                               0x14
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_INT__SHIFT                                    0x15
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_ACK__SHIFT                                    0x16
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_MASK__SHIFT                                   0x17
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_OCCURRED__SHIFT                             0x18
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_INT__SHIFT                                  0x19
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_ACK__SHIFT                                  0x1a
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_MASK__SHIFT                                 0x1b
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC_READ_REQUEST_ACK_ENABLE__SHIFT                              0x1e
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC_READ_REQUEST_INT_TYPE__SHIFT                                0x1f
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_OCCURRED_MASK                                 0x00000001L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_INT_MASK                                      0x00000002L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_ACK_MASK                                      0x00000004L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC1_READ_REQUEST_MASK_MASK                                     0x00000008L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_OCCURRED_MASK                                 0x00000010L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_INT_MASK                                      0x00000020L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_ACK_MASK                                      0x00000040L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC2_READ_REQUEST_MASK_MASK                                     0x00000080L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_OCCURRED_MASK                                 0x00000100L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_INT_MASK                                      0x00000200L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_ACK_MASK                                      0x00000400L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC3_READ_REQUEST_MASK_MASK                                     0x00000800L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_OCCURRED_MASK                                 0x00001000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_INT_MASK                                      0x00002000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_ACK_MASK                                      0x00004000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC4_READ_REQUEST_MASK_MASK                                     0x00008000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_OCCURRED_MASK                                 0x00010000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_INT_MASK                                      0x00020000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_ACK_MASK                                      0x00040000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC5_READ_REQUEST_MASK_MASK                                     0x00080000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_OCCURRED_MASK                                 0x00100000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_INT_MASK                                      0x00200000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_ACK_MASK                                      0x00400000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC6_READ_REQUEST_MASK_MASK                                     0x00800000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_OCCURRED_MASK                               0x01000000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_INT_MASK                                    0x02000000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_ACK_MASK                                    0x04000000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDCVGA_READ_REQUEST_MASK_MASK                                   0x08000000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC_READ_REQUEST_ACK_ENABLE_MASK                                0x40000000L
+#define DC_I2C_READ_REQUEST_INTERRUPT__DC_I2C_DDC_READ_REQUEST_INT_TYPE_MASK                                  0x80000000L
+
+
+// addressBlock: dce_dc_dio_dio_misc_dispdec
+//DIO_SCRATCH0
+#define DIO_SCRATCH0__DIO_SCRATCH0__SHIFT                                                                     0x0
+#define DIO_SCRATCH0__DIO_SCRATCH0_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH1
+#define DIO_SCRATCH1__DIO_SCRATCH1__SHIFT                                                                     0x0
+#define DIO_SCRATCH1__DIO_SCRATCH1_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH2
+#define DIO_SCRATCH2__DIO_SCRATCH2__SHIFT                                                                     0x0
+#define DIO_SCRATCH2__DIO_SCRATCH2_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH3
+#define DIO_SCRATCH3__DIO_SCRATCH3__SHIFT                                                                     0x0
+#define DIO_SCRATCH3__DIO_SCRATCH3_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH4
+#define DIO_SCRATCH4__DIO_SCRATCH4__SHIFT                                                                     0x0
+#define DIO_SCRATCH4__DIO_SCRATCH4_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH5
+#define DIO_SCRATCH5__DIO_SCRATCH5__SHIFT                                                                     0x0
+#define DIO_SCRATCH5__DIO_SCRATCH5_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH6
+#define DIO_SCRATCH6__DIO_SCRATCH6__SHIFT                                                                     0x0
+#define DIO_SCRATCH6__DIO_SCRATCH6_MASK                                                                       0xFFFFFFFFL
+//DIO_SCRATCH7
+#define DIO_SCRATCH7__DIO_SCRATCH7__SHIFT                                                                     0x0
+#define DIO_SCRATCH7__DIO_SCRATCH7_MASK                                                                       0xFFFFFFFFL
+//DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGA_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x0
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGB_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x1
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGC_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x2
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGD_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x3
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGE_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x4
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGF_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x5
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGG_DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                      0x6
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGA_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000001L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGB_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000002L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGC_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000004L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGD_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000008L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGE_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000010L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGF_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000020L
+#define DIO_DP_ALPM_WAKEUP_INTERRUPT_STATUS__DIGG_DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                        0x00000040L
+//DIO_MEM_PWR_STATUS
+#define DIO_MEM_PWR_STATUS__I2C_MEM_PWR_STATE__SHIFT                                                          0x0
+#define DIO_MEM_PWR_STATUS__DPA_MEM_PWR_STATE__SHIFT                                                          0x3
+#define DIO_MEM_PWR_STATUS__DPB_MEM_PWR_STATE__SHIFT                                                          0x4
+#define DIO_MEM_PWR_STATUS__DPC_MEM_PWR_STATE__SHIFT                                                          0x5
+#define DIO_MEM_PWR_STATUS__DPD_MEM_PWR_STATE__SHIFT                                                          0x6
+#define DIO_MEM_PWR_STATUS__DPE_MEM_PWR_STATE__SHIFT                                                          0x7
+#define DIO_MEM_PWR_STATUS__DPF_MEM_PWR_STATE__SHIFT                                                          0x8
+#define DIO_MEM_PWR_STATUS__DPG_MEM_PWR_STATE__SHIFT                                                          0x9
+#define DIO_MEM_PWR_STATUS__I2C_MEM_PWR_STATE_MASK                                                            0x00000001L
+#define DIO_MEM_PWR_STATUS__DPA_MEM_PWR_STATE_MASK                                                            0x00000008L
+#define DIO_MEM_PWR_STATUS__DPB_MEM_PWR_STATE_MASK                                                            0x00000010L
+#define DIO_MEM_PWR_STATUS__DPC_MEM_PWR_STATE_MASK                                                            0x00000020L
+#define DIO_MEM_PWR_STATUS__DPD_MEM_PWR_STATE_MASK                                                            0x00000040L
+#define DIO_MEM_PWR_STATUS__DPE_MEM_PWR_STATE_MASK                                                            0x00000080L
+#define DIO_MEM_PWR_STATUS__DPF_MEM_PWR_STATE_MASK                                                            0x00000100L
+#define DIO_MEM_PWR_STATUS__DPG_MEM_PWR_STATE_MASK                                                            0x00000200L
+//DIO_MEM_PWR_CTRL
+#define DIO_MEM_PWR_CTRL__I2C_LIGHT_SLEEP_FORCE__SHIFT                                                        0x0
+#define DIO_MEM_PWR_CTRL__I2C_LIGHT_SLEEP_DIS__SHIFT                                                          0x1
+#define DIO_MEM_PWR_CTRL__DPA_LIGHT_SLEEP_DIS__SHIFT                                                          0x4
+#define DIO_MEM_PWR_CTRL__DPB_LIGHT_SLEEP_DIS__SHIFT                                                          0x5
+#define DIO_MEM_PWR_CTRL__DPC_LIGHT_SLEEP_DIS__SHIFT                                                          0x6
+#define DIO_MEM_PWR_CTRL__DPD_LIGHT_SLEEP_DIS__SHIFT                                                          0x7
+#define DIO_MEM_PWR_CTRL__DPE_LIGHT_SLEEP_DIS__SHIFT                                                          0x8
+#define DIO_MEM_PWR_CTRL__DPF_LIGHT_SLEEP_DIS__SHIFT                                                          0x9
+#define DIO_MEM_PWR_CTRL__DPG_LIGHT_SLEEP_DIS__SHIFT                                                          0xa
+#define DIO_MEM_PWR_CTRL__I2C_LIGHT_SLEEP_FORCE_MASK                                                          0x00000001L
+#define DIO_MEM_PWR_CTRL__I2C_LIGHT_SLEEP_DIS_MASK                                                            0x00000002L
+#define DIO_MEM_PWR_CTRL__DPA_LIGHT_SLEEP_DIS_MASK                                                            0x00000010L
+#define DIO_MEM_PWR_CTRL__DPB_LIGHT_SLEEP_DIS_MASK                                                            0x00000020L
+#define DIO_MEM_PWR_CTRL__DPC_LIGHT_SLEEP_DIS_MASK                                                            0x00000040L
+#define DIO_MEM_PWR_CTRL__DPD_LIGHT_SLEEP_DIS_MASK                                                            0x00000080L
+#define DIO_MEM_PWR_CTRL__DPE_LIGHT_SLEEP_DIS_MASK                                                            0x00000100L
+#define DIO_MEM_PWR_CTRL__DPF_LIGHT_SLEEP_DIS_MASK                                                            0x00000200L
+#define DIO_MEM_PWR_CTRL__DPG_LIGHT_SLEEP_DIS_MASK                                                            0x00000400L
+//DIO_MEM_PWR_CTRL2
+#define DIO_MEM_PWR_CTRL2__DPA_LIGHT_SLEEP_FORCE__SHIFT                                                       0x18
+#define DIO_MEM_PWR_CTRL2__DPB_LIGHT_SLEEP_FORCE__SHIFT                                                       0x19
+#define DIO_MEM_PWR_CTRL2__DPC_LIGHT_SLEEP_FORCE__SHIFT                                                       0x1a
+#define DIO_MEM_PWR_CTRL2__DPD_LIGHT_SLEEP_FORCE__SHIFT                                                       0x1b
+#define DIO_MEM_PWR_CTRL2__DPE_LIGHT_SLEEP_FORCE__SHIFT                                                       0x1c
+#define DIO_MEM_PWR_CTRL2__DPF_LIGHT_SLEEP_FORCE__SHIFT                                                       0x1d
+#define DIO_MEM_PWR_CTRL2__DPG_LIGHT_SLEEP_FORCE__SHIFT                                                       0x1e
+#define DIO_MEM_PWR_CTRL2__DPA_LIGHT_SLEEP_FORCE_MASK                                                         0x01000000L
+#define DIO_MEM_PWR_CTRL2__DPB_LIGHT_SLEEP_FORCE_MASK                                                         0x02000000L
+#define DIO_MEM_PWR_CTRL2__DPC_LIGHT_SLEEP_FORCE_MASK                                                         0x04000000L
+#define DIO_MEM_PWR_CTRL2__DPD_LIGHT_SLEEP_FORCE_MASK                                                         0x08000000L
+#define DIO_MEM_PWR_CTRL2__DPE_LIGHT_SLEEP_FORCE_MASK                                                         0x10000000L
+#define DIO_MEM_PWR_CTRL2__DPF_LIGHT_SLEEP_FORCE_MASK                                                         0x20000000L
+#define DIO_MEM_PWR_CTRL2__DPG_LIGHT_SLEEP_FORCE_MASK                                                         0x40000000L
+//DIO_CLK_CNTL
+#define DIO_CLK_CNTL__DIO_TEST_CLK_SEL__SHIFT                                                                 0x0
+#define DIO_CLK_CNTL__DISPCLK_R_GATE_DIS__SHIFT                                                               0x9
+#define DIO_CLK_CNTL__DISPCLK_G_GATE_DIS__SHIFT                                                               0xa
+#define DIO_CLK_CNTL__REFCLK_R_GATE_DIS__SHIFT                                                                0xb
+#define DIO_CLK_CNTL__REFCLK_G_GATE_DIS__SHIFT                                                                0xc
+#define DIO_CLK_CNTL__SOCCLK_G_GATE_DIS__SHIFT                                                                0xd
+#define DIO_CLK_CNTL__SYMCLK_FE_R_GATE_DIS__SHIFT                                                             0xe
+#define DIO_CLK_CNTL__SYMCLK_FE_G_GATE_DIS__SHIFT                                                             0xf
+#define DIO_CLK_CNTL__SYMCLK_R_GATE_DIS__SHIFT                                                                0x10
+#define DIO_CLK_CNTL__SYMCLK_G_GATE_DIS__SHIFT                                                                0x11
+#define DIO_CLK_CNTL__DPREFCLK_R_GATE_DIS__SHIFT                                                              0x12
+#define DIO_CLK_CNTL__DPREFCLK_G_GATE_DIS__SHIFT                                                              0x13
+#define DIO_CLK_CNTL__DIO_FGCG_REP_DIS__SHIFT                                                                 0x14
+#define DIO_CLK_CNTL__DISPCLK_G_HDCP_GATE_DIS__SHIFT                                                          0x15
+#define DIO_CLK_CNTL__SYMCLKA_G_HDCP_GATE_DIS__SHIFT                                                          0x16
+#define DIO_CLK_CNTL__SYMCLKB_G_HDCP_GATE_DIS__SHIFT                                                          0x17
+#define DIO_CLK_CNTL__SYMCLKC_G_HDCP_GATE_DIS__SHIFT                                                          0x18
+#define DIO_CLK_CNTL__SYMCLKD_G_HDCP_GATE_DIS__SHIFT                                                          0x19
+#define DIO_CLK_CNTL__SYMCLKE_G_HDCP_GATE_DIS__SHIFT                                                          0x1a
+#define DIO_CLK_CNTL__SYMCLKF_G_HDCP_GATE_DIS__SHIFT                                                          0x1b
+#define DIO_CLK_CNTL__SYMCLKG_G_HDCP_GATE_DIS__SHIFT                                                          0x1c
+#define DIO_CLK_CNTL__SOCCLK_R_GATE_DIS__SHIFT                                                                0x1d
+#define DIO_CLK_CNTL__DIO_TEST_CLK_SEL_MASK                                                                   0x0000007FL
+#define DIO_CLK_CNTL__DISPCLK_R_GATE_DIS_MASK                                                                 0x00000200L
+#define DIO_CLK_CNTL__DISPCLK_G_GATE_DIS_MASK                                                                 0x00000400L
+#define DIO_CLK_CNTL__REFCLK_R_GATE_DIS_MASK                                                                  0x00000800L
+#define DIO_CLK_CNTL__REFCLK_G_GATE_DIS_MASK                                                                  0x00001000L
+#define DIO_CLK_CNTL__SOCCLK_G_GATE_DIS_MASK                                                                  0x00002000L
+#define DIO_CLK_CNTL__SYMCLK_FE_R_GATE_DIS_MASK                                                               0x00004000L
+#define DIO_CLK_CNTL__SYMCLK_FE_G_GATE_DIS_MASK                                                               0x00008000L
+#define DIO_CLK_CNTL__SYMCLK_R_GATE_DIS_MASK                                                                  0x00010000L
+#define DIO_CLK_CNTL__SYMCLK_G_GATE_DIS_MASK                                                                  0x00020000L
+#define DIO_CLK_CNTL__DPREFCLK_R_GATE_DIS_MASK                                                                0x00040000L
+#define DIO_CLK_CNTL__DPREFCLK_G_GATE_DIS_MASK                                                                0x00080000L
+#define DIO_CLK_CNTL__DIO_FGCG_REP_DIS_MASK                                                                   0x00100000L
+#define DIO_CLK_CNTL__DISPCLK_G_HDCP_GATE_DIS_MASK                                                            0x00200000L
+#define DIO_CLK_CNTL__SYMCLKA_G_HDCP_GATE_DIS_MASK                                                            0x00400000L
+#define DIO_CLK_CNTL__SYMCLKB_G_HDCP_GATE_DIS_MASK                                                            0x00800000L
+#define DIO_CLK_CNTL__SYMCLKC_G_HDCP_GATE_DIS_MASK                                                            0x01000000L
+#define DIO_CLK_CNTL__SYMCLKD_G_HDCP_GATE_DIS_MASK                                                            0x02000000L
+#define DIO_CLK_CNTL__SYMCLKE_G_HDCP_GATE_DIS_MASK                                                            0x04000000L
+#define DIO_CLK_CNTL__SYMCLKF_G_HDCP_GATE_DIS_MASK                                                            0x08000000L
+#define DIO_CLK_CNTL__SYMCLKG_G_HDCP_GATE_DIS_MASK                                                            0x10000000L
+#define DIO_CLK_CNTL__SOCCLK_R_GATE_DIS_MASK                                                                  0x20000000L
+//DIO_POWER_MANAGEMENT_CNTL
+#define DIO_POWER_MANAGEMENT_CNTL__PM_ASSERT_RESET__SHIFT                                                     0x0
+#define DIO_POWER_MANAGEMENT_CNTL__PM_ALL_BUSY_OFF__SHIFT                                                     0x8
+#define DIO_POWER_MANAGEMENT_CNTL__PM_ASSERT_RESET_MASK                                                       0x00000001L
+#define DIO_POWER_MANAGEMENT_CNTL__PM_ALL_BUSY_OFF_MASK                                                       0x00000100L
+//DIO_HDMI_RXSTATUS_TIMER_CONTROL
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_ENABLE__SHIFT                                0x0
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_TYPE__SHIFT                                  0x4
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_STATUS__SHIFT                                0x8
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_MASK__SHIFT                                  0xc
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_INTERVAL__SHIFT                              0x10
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_ENABLE_MASK                                  0x00000001L
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_TYPE_MASK                                    0x00000010L
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_STATUS_MASK                                  0x00000100L
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_MASK_MASK                                    0x00001000L
+#define DIO_HDMI_RXSTATUS_TIMER_CONTROL__DIO_HDMI_RXSTATUS_TIMER_INTERVAL_MASK                                0x0FFF0000L
+//DIO_PSP_INTERRUPT_STATUS
+#define DIO_PSP_INTERRUPT_STATUS__DIO_PSP_INTERRUPT_STATUS__SHIFT                                             0x0
+#define DIO_PSP_INTERRUPT_STATUS__DIO_PSP_INTERRUPT_MESSAGE__SHIFT                                            0x1
+#define DIO_PSP_INTERRUPT_STATUS__DIO_PSP_INTERRUPT_STATUS_MASK                                               0x00000001L
+#define DIO_PSP_INTERRUPT_STATUS__DIO_PSP_INTERRUPT_MESSAGE_MASK                                              0xFFFFFFFEL
+//DIO_PSP_INTERRUPT_CLEAR
+#define DIO_PSP_INTERRUPT_CLEAR__DIO_PSP_INTERRUPT_CLEAR__SHIFT                                               0x0
+#define DIO_PSP_INTERRUPT_CLEAR__DIO_PSP_INTERRUPT_CLEAR_MASK                                                 0x00000001L
+//DIO_STATUS
+#define DIO_STATUS__DIO_EN__SHIFT                                                                             0x0
+#define DIO_STATUS__DIO_EN_MASK                                                                               0x00000001L
+
+
+// addressBlock: dce_dc_dio_dig_stream_mapper_dispdec
+//DIG0_STREAM_MAPPER_CONTROL
+#define DIG0_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET__SHIFT                                             0x0
+#define DIG0_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET_MASK                                               0x00000007L
+//DIG1_STREAM_MAPPER_CONTROL
+#define DIG1_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET__SHIFT                                             0x0
+#define DIG1_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET_MASK                                               0x00000007L
+//DIG2_STREAM_MAPPER_CONTROL
+#define DIG2_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET__SHIFT                                             0x0
+#define DIG2_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET_MASK                                               0x00000007L
+//DIG3_STREAM_MAPPER_CONTROL
+#define DIG3_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET__SHIFT                                             0x0
+#define DIG3_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET_MASK                                               0x00000007L
+//DIG4_STREAM_MAPPER_CONTROL
+#define DIG4_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET__SHIFT                                             0x0
+#define DIG4_STREAM_MAPPER_CONTROL__DIG_STREAM_LINK_TARGET_MASK                                               0x00000007L
+
+
+// addressBlock: dce_dc_dio_dio_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON16_PERFCOUNTER_CNTL
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON16_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON16_PERFCOUNTER_CNTL2
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON16_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON16_PERFCOUNTER_STATE
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON16_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON16_PERFMON_CNTL
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON16_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON16_PERFMON_CNTL2
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON16_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON16_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON16_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON16_PERFMON_CVALUE_LOW
+#define DC_PERFMON16_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON16_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON16_PERFMON_HI
+#define DC_PERFMON16_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON16_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON16_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON16_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON16_PERFMON_LOW
+#define DC_PERFMON16_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON16_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dig0_vpg_vpg_dispdec
+//VPG0_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG0_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG0_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG0_VPG_GENERIC_PACKET_DATA
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG0_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG0_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG0_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG0_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG0_VPG_GENERIC_STATUS
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG0_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG0_VPG_MEM_PWR
+#define VPG0_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG0_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG0_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG0_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG0_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG0_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG0_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG0_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG0_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG0_VPG_ISRC1_2_DATA
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG0_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_dio_dig0_apg_apg_dispdec
+//APG0_APG_CONTROL
+#define APG0_APG_CONTROL__APG_RESET__SHIFT                                                                    0x1
+#define APG0_APG_CONTROL__APG_RESET_DONE__SHIFT                                                               0x2
+#define APG0_APG_CONTROL__APG_RESET_MASK                                                                      0x00000002L
+#define APG0_APG_CONTROL__APG_RESET_DONE_MASK                                                                 0x00000004L
+//APG0_APG_CONTROL2
+#define APG0_APG_CONTROL2__APG_ENABLE__SHIFT                                                                  0x0
+#define APG0_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID__SHIFT                                                      0x8
+#define APG0_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE__SHIFT                                           0x18
+#define APG0_APG_CONTROL2__APG_ENABLE_MASK                                                                    0x00000001L
+#define APG0_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID_MASK                                                        0x0000FF00L
+#define APG0_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE_MASK                                             0x01000000L
+//APG0_APG_DBG_GEN_CONTROL
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG0_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG0_APG_PACKET_CONTROL
+#define APG0_APG_PACKET_CONTROL__APG_DBG_MUX_SEL__SHIFT                                                       0x0
+#define APG0_APG_PACKET_CONTROL__APG_ACP_SOURCE__SHIFT                                                        0x1
+#define APG0_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE__SHIFT                                                 0x2
+#define APG0_APG_PACKET_CONTROL__APG_DBG_MUX_SEL_MASK                                                         0x00000001L
+#define APG0_APG_PACKET_CONTROL__APG_ACP_SOURCE_MASK                                                          0x00000002L
+#define APG0_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE_MASK                                                   0x00000004L
+//APG0_APG_DBG_ACP
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE__SHIFT                                                             0x0
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                             0x8
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                             0x10
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_REQUIRED__SHIFT                                                         0x18
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE_MASK                                                               0x00000003L
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0_MASK                                               0x0000FF00L
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1_MASK                                               0x00FF0000L
+#define APG0_APG_DBG_ACP__APG_DBG_ACP_REQUIRED_MASK                                                           0x01000000L
+//APG0_APG_DBG_AUDIO_INFO
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA__SHIFT                                                 0x0
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC__SHIFT                                                 0x8
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV__SHIFT                                                0xb
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH__SHIFT                                             0xf
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL__SHIFT                                             0x10
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA_MASK                                                   0x000000FFL
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC_MASK                                                   0x00000700L
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV_MASK                                                  0x00007800L
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH_MASK                                               0x00008000L
+#define APG0_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL_MASK                                               0x00030000L
+//APG0_APG_DBG_60958_0
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_A__SHIFT                                                       0x0
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_B__SHIFT                                                       0x1
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_C__SHIFT                                                       0x2
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_D__SHIFT                                                       0x3
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_MODE__SHIFT                                                    0x6
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE__SHIFT                                           0x8
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER__SHIFT                                           0x10
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L__SHIFT                                        0x14
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY__SHIFT                                      0x18
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY__SHIFT                                          0x1c
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_A_MASK                                                         0x00000001L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_B_MASK                                                         0x00000002L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_C_MASK                                                         0x00000004L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_D_MASK                                                         0x00000038L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_MODE_MASK                                                      0x000000C0L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE_MASK                                             0x0000FF00L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER_MASK                                             0x000F0000L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L_MASK                                          0x00F00000L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY_MASK                                        0x0F000000L
+#define APG0_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY_MASK                                            0x30000000L
+//APG0_APG_DBG_60958_1
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH__SHIFT                                             0x0
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                             0x4
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_VALID__SHIFT                                                      0x10
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R__SHIFT                                        0x14
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH_MASK                                               0x0000000FL
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                               0x000000F0L
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_VALID_MASK                                                        0x00010000L
+#define APG0_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R_MASK                                          0x00F00000L
+//APG0_APG_DBG_60958_2
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2__SHIFT                                        0x0
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3__SHIFT                                        0x4
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4__SHIFT                                        0x8
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5__SHIFT                                        0xc
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6__SHIFT                                        0x10
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7__SHIFT                                        0x14
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2_MASK                                          0x0000000FL
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3_MASK                                          0x000000F0L
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4_MASK                                          0x00000F00L
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5_MASK                                          0x0000F000L
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6_MASK                                          0x000F0000L
+#define APG0_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7_MASK                                          0x00F00000L
+//APG0_APG_AUDIO_CRC_CONTROL
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN__SHIFT                                                   0x0
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT__SHIFT                                                 0x4
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL__SHIFT                                               0xd
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT__SHIFT                                                0x10
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN_MASK                                                     0x00000001L
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT_MASK                                                   0x00000010L
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL_MASK                                                 0x0000E000L
+#define APG0_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT_MASK                                                  0xFFFF0000L
+//APG0_APG_AUDIO_CRC_CONTROL2
+#define APG0_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT__SHIFT                                 0x0
+#define APG0_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT_MASK                                   0x0000FFFFL
+//APG0_APG_AUDIO_CRC_RESULT
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE__SHIFT                                                  0x0
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR__SHIFT                                            0x8
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC__SHIFT                                                       0x10
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_MASK                                                    0x00000001L
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR_MASK                                              0x00000100L
+#define APG0_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_MASK                                                         0xFFFF0000L
+//APG0_APG_DBG_RAMP_CONTROL0
+#define APG0_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT__SHIFT                                             0x0
+#define APG0_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN__SHIFT                                             0x1f
+#define APG0_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT_MASK                                               0x00FFFFFFL
+#define APG0_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN_MASK                                               0x80000000L
+//APG0_APG_DBG_RAMP_CONTROL1
+#define APG0_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT__SHIFT                                             0x0
+#define APG0_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT_MASK                                               0x00FFFFFFL
+//APG0_APG_DBG_RAMP_CONTROL2
+#define APG0_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT__SHIFT                                             0x0
+#define APG0_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT_MASK                                               0x00FFFFFFL
+//APG0_APG_DBG_RAMP_CONTROL3
+#define APG0_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT__SHIFT                                             0x0
+#define APG0_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT_MASK                                               0x00FFFFFFL
+//APG0_APG_STATUS
+#define APG0_APG_STATUS__APG_AUDIO_ENABLE__SHIFT                                                              0x4
+#define APG0_APG_STATUS__APG_HBR_ENABLE__SHIFT                                                                0x8
+#define APG0_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS__SHIFT                                                0x18
+#define APG0_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR__SHIFT                                          0x19
+#define APG0_APG_STATUS__APG_AUDIO_ENABLE_MASK                                                                0x00000010L
+#define APG0_APG_STATUS__APG_HBR_ENABLE_MASK                                                                  0x00000100L
+#define APG0_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_MASK                                                  0x01000000L
+#define APG0_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR_MASK                                            0x02000000L
+//APG0_APG_STATUS2
+#define APG0_APG_STATUS2__APG_OUTPUT_ACTIVE__SHIFT                                                            0x0
+#define APG0_APG_STATUS2__APG_OUTPUT_ACTIVE_MASK                                                              0x00000001L
+//APG0_APG_DBG_AUDIO_DTO_CNTL
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE__SHIFT                                            0x8
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI__SHIFT                                           0xc
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV__SHIFT                                             0x10
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE_MASK                                              0x00000100L
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI_MASK                                             0x00007000L
+#define APG0_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV_MASK                                               0x00070000L
+//APG0_APG_MEM_PWR
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG0_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG0_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG0_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+//APG0_APG_SPARE
+#define APG0_APG_SPARE__APG_SPARE__SHIFT                                                                      0x0
+#define APG0_APG_SPARE__APG_SPARE_MASK                                                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dig0_dme_dme_dispdec
+//DME0_DME_CONTROL
+#define DME0_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME0_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME0_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME0_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME0_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME0_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME0_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME0_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME0_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME0_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME0_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME0_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME0_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME0_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME0_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME0_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME0_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME0_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME0_DME_MEMORY_CONTROL
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME0_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig0_dispdec
+//DIG0_DIG_FE_CNTL
+#define DIG0_DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT                                                            0x0
+#define DIG0_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT                                                        0x4
+#define DIG0_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT                                                       0x8
+#define DIG0_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT__SHIFT                                                    0xc
+#define DIG0_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN__SHIFT                                                        0x14
+#define DIG0_DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK                                                              0x00000007L
+#define DIG0_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK                                                          0x00000070L
+#define DIG0_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK                                                         0x00000100L
+#define DIG0_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT_MASK                                                      0x00007000L
+#define DIG0_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN_MASK                                                          0x00100000L
+//DIG0_DIG_FE_CLK_CNTL
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_MODE__SHIFT                                                              0x0
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN__SHIFT                                                            0x4
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON__SHIFT                                                0xa
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON__SHIFT                                              0xb
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON__SHIFT                                          0xc
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON__SHIFT                                         0xd
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON__SHIFT                                             0xe
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_MODE_MASK                                                                0x00000007L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN_MASK                                                              0x00000010L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON_MASK                                                  0x00000400L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON_MASK                                                0x00000800L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON_MASK                                            0x00001000L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON_MASK                                           0x00002000L
+#define DIG0_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON_MASK                                               0x00004000L
+//DIG0_DIG_FE_EN_CNTL
+#define DIG0_DIG_FE_EN_CNTL__DIG_FE_ENABLE__SHIFT                                                             0x0
+#define DIG0_DIG_FE_EN_CNTL__DIG_FE_ENABLE_MASK                                                               0x00000001L
+//DIG0_DIG_OUTPUT_CRC_CNTL
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT                                                    0x0
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT                                              0x4
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT                                              0x8
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK                                                      0x00000001L
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK                                                0x00000010L
+#define DIG0_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK                                                0x00000300L
+//DIG0_DIG_OUTPUT_CRC_RESULT
+#define DIG0_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT                                              0x0
+#define DIG0_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK                                                0x3FFFFFFFL
+//DIG0_DIG_CLOCK_PATTERN
+#define DIG0_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT                                                      0x0
+#define DIG0_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK                                                        0x000003FFL
+//DIG0_DIG_TEST_PATTERN
+#define DIG0_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT                                                 0x0
+#define DIG0_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT                                              0x1
+#define DIG0_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT                                               0x4
+#define DIG0_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT                                                0x5
+#define DIG0_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT                                      0x6
+#define DIG0_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT                                                 0x10
+#define DIG0_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK                                                   0x00000001L
+#define DIG0_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK                                                0x00000002L
+#define DIG0_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK                                                 0x00000010L
+#define DIG0_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK                                                  0x00000020L
+#define DIG0_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK                                        0x00000040L
+#define DIG0_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK                                                   0x03FF0000L
+//DIG0_DIG_RANDOM_PATTERN_SEED
+#define DIG0_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT                                          0x0
+#define DIG0_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT                                       0x18
+#define DIG0_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK                                            0x00FFFFFFL
+#define DIG0_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK                                         0x01000000L
+//DIG0_DIG_FIFO_CTRL0
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE__SHIFT                                                           0x0
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_RESET__SHIFT                                                            0x1
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL__SHIFT                                                 0x2
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC__SHIFT                                                   0x8
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                           0x9
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE__SHIFT                                                       0x14
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_ERROR__SHIFT                                                            0x1c
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE_MASK                                                             0x00000001L
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_RESET_MASK                                                              0x00000002L
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL_MASK                                                   0x000000FCL
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC_MASK                                                     0x00000100L
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE_MASK                                             0x00000600L
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE_MASK                                                         0x00100000L
+#define DIG0_DIG_FIFO_CTRL0__DIG_FIFO_ERROR_MASK                                                              0x30000000L
+//DIG0_DIG_FIFO_CTRL1
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT                                              0x1
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL__SHIFT                                                  0x2
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY__SHIFT                                                  0x8
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT                                                0xa
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL__SHIFT                                                    0x10
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL__SHIFT                                                    0x16
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED__SHIFT                                                       0x1d
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT                                              0x1e
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT                                              0x1f
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK                                                0x00000002L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL_MASK                                                    0x000000FCL
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY_MASK                                                    0x00000300L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK                                                  0x0000FC00L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL_MASK                                                      0x001F0000L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL_MASK                                                      0x03C00000L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED_MASK                                                         0x20000000L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK                                                0x40000000L
+#define DIG0_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK                                                0x80000000L
+//DIG0_HDMI_METADATA_PACKET_CONTROL
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                                 0x0
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                         0x4
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                                 0x8
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                   0x10
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                   0x00000001L
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                           0x00000010L
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                   0x00000100L
+#define DIG0_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                     0xFFFF0000L
+//DIG0_HDMI_CONTROL
+#define DIG0_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT                                                       0x1
+#define DIG0_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT                                                     0x2
+#define DIG0_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED__SHIFT                                            0x3
+#define DIG0_HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT                                                              0x8
+#define DIG0_HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT                                                             0x9
+#define DIG0_HDMI_CONTROL__DOLBY_VISION_EN__SHIFT                                                             0xa
+#define DIG0_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED__SHIFT                                         0xb
+#define DIG0_HDMI_CONTROL__TMDS_PIXEL_ENCODING__SHIFT                                                         0xc
+#define DIG0_HDMI_CONTROL__TMDS_COLOR_FORMAT__SHIFT                                                           0xd
+#define DIG0_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM__SHIFT                                           0x10
+#define DIG0_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT                                                      0x18
+#define DIG0_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                       0x1c
+#define DIG0_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK                                                         0x00000002L
+#define DIG0_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK                                                       0x00000004L
+#define DIG0_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK                                              0x00000008L
+#define DIG0_HDMI_CONTROL__HDMI_ERROR_ACK_MASK                                                                0x00000100L
+#define DIG0_HDMI_CONTROL__HDMI_ERROR_MASK_MASK                                                               0x00000200L
+#define DIG0_HDMI_CONTROL__DOLBY_VISION_EN_MASK                                                               0x00000400L
+#define DIG0_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED_MASK                                           0x00000800L
+#define DIG0_HDMI_CONTROL__TMDS_PIXEL_ENCODING_MASK                                                           0x00001000L
+#define DIG0_HDMI_CONTROL__TMDS_COLOR_FORMAT_MASK                                                             0x00006000L
+#define DIG0_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM_MASK                                             0x003F0000L
+#define DIG0_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK                                                        0x01000000L
+#define DIG0_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK                                                         0x30000000L
+//DIG0_HDMI_STATUS
+#define DIG0_HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT                                                           0x0
+#define DIG0_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT                                                      0x10
+#define DIG0_HDMI_STATUS__HDMI_ERROR_INT__SHIFT                                                               0x1b
+#define DIG0_HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK                                                             0x00000001L
+#define DIG0_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK                                                        0x00010000L
+#define DIG0_HDMI_STATUS__HDMI_ERROR_INT_MASK                                                                 0x08000000L
+//DIG0_HDMI_AUDIO_PACKET_CONTROL
+#define DIG0_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT                                            0x4
+#define DIG0_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK                                              0x00000030L
+//DIG0_HDMI_ACR_PACKET_CONTROL
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                    0x0
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                    0x1
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                  0x4
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                  0x8
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                               0xc
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                              0x10
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                      0x00000001L
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                      0x00000002L
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                    0x00000030L
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                    0x00000100L
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                                 0x00001000L
+#define DIG0_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                                0x00070000L
+//DIG0_HDMI_VBI_PACKET_CONTROL
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT                                                   0x0
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT                                                     0x4
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT                                                     0x5
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT                                                   0x8
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT                                                   0x9
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND__SHIFT                                                    0xc
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT                                                   0x10
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE__SHIFT                                                    0x18
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK                                                     0x00000001L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK                                                       0x00000010L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK                                                       0x00000020L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK                                                     0x00000100L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK                                                     0x00000200L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND_MASK                                                      0x00001000L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK                                                     0x003F0000L
+#define DIG0_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE_MASK                                                      0x3F000000L
+//DIG0_HDMI_INFOFRAME_CONTROL0
+#define DIG0_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT                                             0x4
+#define DIG0_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE__SHIFT                                             0x8
+#define DIG0_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK                                               0x00000010L
+#define DIG0_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE_MASK                                               0x00003F00L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                          0x1
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                                0x2
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                          0x4
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                          0x5
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                                0x6
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                          0x8
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                          0x9
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                                0xa
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE__SHIFT                           0xb
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                          0xc
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                          0xd
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                                0xe
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE__SHIFT                           0xf
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                          0x11
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                                0x12
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE__SHIFT                           0x13
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                          0x14
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                          0x15
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                                0x16
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE__SHIFT                           0x17
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                          0x18
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                          0x19
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                                0x1a
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE__SHIFT                           0x1b
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                          0x1c
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                          0x1d
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                                0x1e
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE__SHIFT                           0x1f
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                            0x00000001L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                            0x00000002L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                            0x00000010L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                            0x00000020L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                            0x00000100L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                            0x00000200L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                  0x00000400L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE_MASK                             0x00000800L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                            0x00001000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                            0x00002000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                  0x00004000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE_MASK                             0x00008000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                            0x00010000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                            0x00020000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                  0x00040000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE_MASK                             0x00080000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                            0x00100000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                            0x00200000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                  0x00400000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE_MASK                             0x00800000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                            0x01000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                            0x02000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                  0x04000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE_MASK                             0x08000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                            0x10000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                            0x20000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                  0x40000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE_MASK                             0x80000000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL6
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT__SHIFT                                          0x1
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                                0x2
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND__SHIFT                                          0x4
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT__SHIFT                                          0x5
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                                0x6
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND__SHIFT                                         0x8
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT__SHIFT                                         0x9
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                               0xa
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE__SHIFT                          0xb
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND__SHIFT                                         0xc
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT__SHIFT                                         0xd
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                               0xe
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE__SHIFT                          0xf
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND__SHIFT                                         0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT__SHIFT                                         0x11
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                               0x12
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE__SHIFT                          0x13
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND__SHIFT                                         0x14
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT__SHIFT                                         0x15
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                               0x16
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE__SHIFT                          0x17
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND__SHIFT                                         0x18
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT__SHIFT                                         0x19
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                               0x1a
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE__SHIFT                          0x1b
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND_MASK                                            0x00000001L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT_MASK                                            0x00000002L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND_MASK                                            0x00000010L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT_MASK                                            0x00000020L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND_MASK                                           0x00000100L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT_MASK                                           0x00000200L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE_MASK                                 0x00000400L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE_MASK                            0x00000800L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND_MASK                                           0x00001000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT_MASK                                           0x00002000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE_MASK                                 0x00004000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE_MASK                            0x00008000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND_MASK                                           0x00010000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT_MASK                                           0x00020000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE_MASK                                 0x00040000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE_MASK                            0x00080000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND_MASK                                           0x00100000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT_MASK                                           0x00200000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE_MASK                                 0x00400000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE_MASK                            0x00800000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND_MASK                                           0x01000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT_MASK                                           0x02000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE_MASK                                 0x04000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE_MASK                            0x08000000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL5
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                                0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                        0x1
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                                0x2
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                        0x3
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                                0x4
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                        0x5
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                                0x6
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                        0x7
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                                0x8
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                        0x9
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                                0xa
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                        0xb
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                                0xc
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                        0xd
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                                0xe
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                        0xf
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                                0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                        0x11
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                                0x12
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                        0x13
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                               0x14
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                       0x15
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                               0x16
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                       0x17
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                               0x18
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                       0x19
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                               0x1a
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                       0x1b
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                               0x1c
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                       0x1d
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                  0x00000001L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                          0x00000002L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                  0x00000004L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                          0x00000008L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                  0x00000010L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                          0x00000020L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                  0x00000040L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                          0x00000080L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                  0x00000100L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                          0x00000200L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                  0x00000400L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                          0x00000800L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                  0x00001000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                          0x00002000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                  0x00004000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                          0x00008000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                  0x00010000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                          0x00020000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                  0x00040000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                          0x00080000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                                 0x00100000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                         0x00200000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                                 0x00400000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                         0x00800000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                                 0x01000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                         0x02000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                                 0x04000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                         0x08000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                                 0x10000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                         0x20000000L
+//DIG0_HDMI_GC
+#define DIG0_HDMI_GC__HDMI_GC_AVMUTE__SHIFT                                                                   0x0
+#define DIG0_HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT                                                              0x2
+#define DIG0_HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT                                                               0x4
+#define DIG0_HDMI_GC__HDMI_PACKING_PHASE__SHIFT                                                               0x8
+#define DIG0_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT                                                      0xc
+#define DIG0_HDMI_GC__HDMI_GC_AVMUTE_MASK                                                                     0x00000001L
+#define DIG0_HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK                                                                0x00000004L
+#define DIG0_HDMI_GC__HDMI_DEFAULT_PHASE_MASK                                                                 0x00000010L
+#define DIG0_HDMI_GC__HDMI_PACKING_PHASE_MASK                                                                 0x00000F00L
+#define DIG0_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK                                                        0x00001000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL1
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE_MASK                                            0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE_MASK                                            0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL2
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE_MASK                                            0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE_MASK                                            0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL3
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE_MASK                                            0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE_MASK                                            0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL4
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE_MASK                                            0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE_MASK                                            0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL7
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE__SHIFT                                          0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE__SHIFT                                          0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE_MASK                                            0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE_MASK                                            0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL8
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE__SHIFT                                         0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE__SHIFT                                         0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE_MASK                                           0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE_MASK                                           0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL9
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE__SHIFT                                         0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE__SHIFT                                         0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE_MASK                                           0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE_MASK                                           0xFFFF0000L
+//DIG0_HDMI_GENERIC_PACKET_CONTROL10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE__SHIFT                                        0x0
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING__SHIFT                                0x10
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING__SHIFT                                0x11
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING__SHIFT                                0x12
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING__SHIFT                                0x13
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING__SHIFT                                0x14
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING__SHIFT                                0x15
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING__SHIFT                                0x16
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING__SHIFT                                0x17
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING__SHIFT                                0x18
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING__SHIFT                                0x19
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING__SHIFT                               0x1a
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING__SHIFT                               0x1b
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING__SHIFT                               0x1c
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING__SHIFT                               0x1d
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING__SHIFT                               0x1e
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE_MASK                                          0x0000FFFFL
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING_MASK                                  0x00010000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING_MASK                                  0x00020000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING_MASK                                  0x00040000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING_MASK                                  0x00080000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING_MASK                                  0x00100000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING_MASK                                  0x00200000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING_MASK                                  0x00400000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING_MASK                                  0x00800000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING_MASK                                  0x01000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING_MASK                                  0x02000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING_MASK                                 0x04000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING_MASK                                 0x08000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING_MASK                                 0x10000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING_MASK                                 0x20000000L
+#define DIG0_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING_MASK                                 0x40000000L
+//DIG0_HDMI_DB_CONTROL
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                          0x0
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_TAKEN__SHIFT                                                            0x4
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR__SHIFT                                                        0x5
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_LOCK__SHIFT                                                             0x8
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                          0xc
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                       0xf
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN__SHIFT                                                         0x10
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR__SHIFT                                                     0x11
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_PENDING_MASK                                                            0x00000001L
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_TAKEN_MASK                                                              0x00000010L
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR_MASK                                                          0x00000020L
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_LOCK_MASK                                                               0x00000100L
+#define DIG0_HDMI_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                            0x00001000L
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                         0x00008000L
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_MASK                                                           0x00010000L
+#define DIG0_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR_MASK                                                       0x00020000L
+//DIG0_HDMI_ACR_32_0
+#define DIG0_HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                            0xc
+#define DIG0_HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                              0xFFFFF000L
+//DIG0_HDMI_ACR_32_1
+#define DIG0_HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                              0x0
+#define DIG0_HDMI_ACR_32_1__HDMI_ACR_N_32_MASK                                                                0x000FFFFFL
+//DIG0_HDMI_ACR_44_0
+#define DIG0_HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                            0xc
+#define DIG0_HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                              0xFFFFF000L
+//DIG0_HDMI_ACR_44_1
+#define DIG0_HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                              0x0
+#define DIG0_HDMI_ACR_44_1__HDMI_ACR_N_44_MASK                                                                0x000FFFFFL
+//DIG0_HDMI_ACR_48_0
+#define DIG0_HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                            0xc
+#define DIG0_HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                              0xFFFFF000L
+//DIG0_HDMI_ACR_48_1
+#define DIG0_HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                              0x0
+#define DIG0_HDMI_ACR_48_1__HDMI_ACR_N_48_MASK                                                                0x000FFFFFL
+//DIG0_HDMI_ACR_STATUS_0
+#define DIG0_HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                           0xc
+#define DIG0_HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                             0xFFFFF000L
+//DIG0_HDMI_ACR_STATUS_1
+#define DIG0_HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                             0x0
+#define DIG0_HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK                                                               0x000FFFFFL
+//DIG0_DIG_FE_AUDIO_CNTL
+#define DIG0_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT                               0x0
+#define DIG0_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE__SHIFT                                                       0x8
+#define DIG0_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                                 0x00000007L
+#define DIG0_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE_MASK                                                         0x00000100L
+//DIG0_DIG_BE_CLK_CNTL
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_MODE__SHIFT                                                              0x0
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN__SHIFT                                                            0x4
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG0_DIG_BE_CLK_CNTL__HDCP_SOFT_RESET__SHIFT                                                          0x6
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON__SHIFT                                                 0xb
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_HDCP_CLOCK_ON__SHIFT                                            0xc
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON__SHIFT                                            0xd
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_MODE_MASK                                                                0x00000007L
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN_MASK                                                              0x00000010L
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG0_DIG_BE_CLK_CNTL__HDCP_SOFT_RESET_MASK                                                            0x00000040L
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON_MASK                                                   0x00000800L
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_HDCP_CLOCK_ON_MASK                                              0x00001000L
+#define DIG0_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON_MASK                                              0x00002000L
+//DIG0_DIG_BE_CNTL
+#define DIG0_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT                                                         0x0
+#define DIG0_DIG_BE_CNTL__DIG_SWAP__SHIFT                                                                     0x1
+#define DIG0_DIG_BE_CNTL__DIG_RB_SWITCH_EN__SHIFT                                                             0x2
+#define DIG0_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT                                                         0x8
+#define DIG0_DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT                                                               0x1c
+#define DIG0_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE_MASK                                                           0x00000001L
+#define DIG0_DIG_BE_CNTL__DIG_SWAP_MASK                                                                       0x00000002L
+#define DIG0_DIG_BE_CNTL__DIG_RB_SWITCH_EN_MASK                                                               0x00000004L
+#define DIG0_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK                                                           0x00007F00L
+#define DIG0_DIG_BE_CNTL__DIG_HPD_SELECT_MASK                                                                 0x70000000L
+//DIG0_DIG_BE_EN_CNTL
+#define DIG0_DIG_BE_EN_CNTL__DIG_BE_ENABLE__SHIFT                                                             0x0
+#define DIG0_DIG_BE_EN_CNTL__DIG_BE_ENABLE_MASK                                                               0x00000001L
+//DIG0_TMDS_CNTL
+#define DIG0_TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT                                                                0x0
+#define DIG0_TMDS_CNTL__TMDS_SYNC_PHASE_MASK                                                                  0x00000001L
+//DIG0_TMDS_CONTROL_CHAR
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT                                              0x0
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT                                              0x1
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT                                              0x2
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT                                              0x3
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK                                                0x00000001L
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK                                                0x00000002L
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK                                                0x00000004L
+#define DIG0_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK                                                0x00000008L
+//DIG0_TMDS_CONTROL0_FEEDBACK
+#define DIG0_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT                                     0x0
+#define DIG0_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT                                      0x8
+#define DIG0_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK                                       0x00000003L
+#define DIG0_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK                                        0x00000300L
+//DIG0_TMDS_STEREOSYNC_CTL_SEL
+#define DIG0_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT                                          0x0
+#define DIG0_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK                                            0x00000003L
+//DIG0_TMDS_SYNC_CHAR_PATTERN_0_1
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT                                       0x0
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT                                       0x10
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK                                         0x000003FFL
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK                                         0x03FF0000L
+//DIG0_TMDS_SYNC_CHAR_PATTERN_2_3
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT                                       0x0
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT                                       0x10
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK                                         0x000003FFL
+#define DIG0_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK                                         0x03FF0000L
+//DIG0_TMDS_CTL_BITS
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL0__SHIFT                                                                  0x0
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL1__SHIFT                                                                  0x8
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL2__SHIFT                                                                  0x10
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL3__SHIFT                                                                  0x18
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL0_MASK                                                                    0x00000001L
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL1_MASK                                                                    0x00000100L
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL2_MASK                                                                    0x00010000L
+#define DIG0_TMDS_CTL_BITS__TMDS_CTL3_MASK                                                                    0x01000000L
+//DIG0_TMDS_DCBALANCER_CONTROL
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT                                               0x0
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT                                               0x4
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT                                          0x8
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT                                          0x10
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT                                            0x18
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK                                                 0x00000001L
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK                                                 0x00000070L
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK                                            0x00000100L
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK                                            0x000F0000L
+#define DIG0_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK                                              0x01000000L
+//DIG0_TMDS_SYNC_DCBALANCE_CHAR
+#define DIG0_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01__SHIFT                                          0x0
+#define DIG0_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11__SHIFT                                          0x10
+#define DIG0_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01_MASK                                            0x000003FFL
+#define DIG0_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11_MASK                                            0x03FF0000L
+//DIG0_TMDS_CTL0_1_GEN_CNTL
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT                                                  0x0
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT                                                0x4
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT                                               0x7
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT                                           0x8
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT                                                  0x10
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT                                                0x14
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT                                               0x17
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT                                           0x18
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT                                                0x1f
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK                                              0x10000000L
+#define DIG0_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK                                                  0x80000000L
+//DIG0_TMDS_CTL2_3_GEN_CNTL
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT                                                  0x0
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT                                                0x4
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT                                               0x7
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT                                           0x8
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT                                                  0x10
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT                                                0x14
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT                                               0x17
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT                                           0x18
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG0_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK                                              0x10000000L
+//DIG0_DIG_VERSION
+#define DIG0_DIG_VERSION__DIG_TYPE__SHIFT                                                                     0x0
+#define DIG0_DIG_VERSION__DIG_TYPE_MASK                                                                       0x00000001L
+
+
+// addressBlock: dce_dc_dio_dp0_dispdec
+//DP0_DP_LINK_CNTL
+#define DP0_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT                                                    0x4
+#define DP0_DP_LINK_CNTL__DP_LINK_STATUS__SHIFT                                                               0x8
+#define DP0_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK                                                      0x00000010L
+#define DP0_DP_LINK_CNTL__DP_LINK_STATUS_MASK                                                                 0x00000100L
+//DP0_DP_PIXEL_FORMAT
+#define DP0_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                                                       0x0
+#define DP0_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT__SHIFT                                                 0x4
+#define DP0_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                                              0x8
+#define DP0_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT__SHIFT                                                   0x10
+#define DP0_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                                         0x00000001L
+#define DP0_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT_MASK                                                   0x00000030L
+#define DP0_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                                                0x00000700L
+#define DP0_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT_MASK                                                     0x00010000L
+//DP0_DP_MSA_COLORIMETRY
+#define DP0_DP_MSA_COLORIMETRY__DP_MSA_MISC0__SHIFT                                                           0x18
+#define DP0_DP_MSA_COLORIMETRY__DP_MSA_MISC0_MASK                                                             0xFF000000L
+//DP0_DP_CONFIG
+#define DP0_DP_CONFIG__DP_UDI_LANES__SHIFT                                                                    0x0
+#define DP0_DP_CONFIG__DP_UDI_LANES_MASK                                                                      0x00000003L
+//DP0_DP_VID_STREAM_CNTL
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT                                                   0x0
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT                                                0x8
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT                                                   0x10
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT                                           0x14
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK                                                     0x00000001L
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK                                                  0x00000300L
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK                                                     0x00010000L
+#define DP0_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK                                             0x00100000L
+//DP0_DP_STEER_FIFO
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE__SHIFT                                                        0x0
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT                                                         0x1
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE__SHIFT                                                    0x2
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT                                                      0x4
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT                                                       0x5
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT                                                       0x6
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT                                                      0x7
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE_MASK                                                          0x00000001L
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK                                                           0x00000002L
+#define DP0_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE_MASK                                                      0x00000004L
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK                                                        0x00000010L
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK                                                         0x00000020L
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK                                                         0x00000040L
+#define DP0_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK                                                        0x00000080L
+//DP0_DP_MSA_MISC
+#define DP0_DP_MSA_MISC__DP_MSA_MISC1__SHIFT                                                                  0x0
+#define DP0_DP_MSA_MISC__DP_MSA_MISC2__SHIFT                                                                  0x8
+#define DP0_DP_MSA_MISC__DP_MSA_MISC3__SHIFT                                                                  0x10
+#define DP0_DP_MSA_MISC__DP_MSA_MISC4__SHIFT                                                                  0x18
+#define DP0_DP_MSA_MISC__DP_MSA_MISC1_MASK                                                                    0x000000FFL
+#define DP0_DP_MSA_MISC__DP_MSA_MISC2_MASK                                                                    0x0000FF00L
+#define DP0_DP_MSA_MISC__DP_MSA_MISC3_MASK                                                                    0x00FF0000L
+#define DP0_DP_MSA_MISC__DP_MSA_MISC4_MASK                                                                    0xFF000000L
+//DP0_DP_DPHY_INTERNAL_CTRL
+#define DP0_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN__SHIFT                                         0x0
+#define DP0_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL__SHIFT                                        0x4
+#define DP0_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN_MASK                                           0x00000001L
+#define DP0_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL_MASK                                          0x00000010L
+//DP0_DP_VID_TIMING
+#define DP0_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE__SHIFT                                               0x4
+#define DP0_DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT                                                           0x8
+#define DP0_DP_VID_TIMING__DP_VID_M_DIV__SHIFT                                                                0xc
+#define DP0_DP_VID_TIMING__DP_VID_N_INTERVAL__SHIFT                                                           0xe
+#define DP0_DP_VID_TIMING__DP_VID_N_DIV__SHIFT                                                                0x18
+#define DP0_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE_MASK                                                 0x00000010L
+#define DP0_DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK                                                             0x00000100L
+#define DP0_DP_VID_TIMING__DP_VID_M_DIV_MASK                                                                  0x00003000L
+#define DP0_DP_VID_TIMING__DP_VID_N_INTERVAL_MASK                                                             0x0000C000L
+#define DP0_DP_VID_TIMING__DP_VID_N_DIV_MASK                                                                  0xFF000000L
+//DP0_DP_VID_N
+#define DP0_DP_VID_N__DP_VID_N__SHIFT                                                                         0x0
+#define DP0_DP_VID_N__DP_VID_N_MASK                                                                           0x00FFFFFFL
+//DP0_DP_VID_M
+#define DP0_DP_VID_M__DP_VID_M__SHIFT                                                                         0x0
+#define DP0_DP_VID_M__DP_VID_M_MASK                                                                           0x00FFFFFFL
+//DP0_DP_LINK_FRAMING_CNTL
+#define DP0_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT                                                  0x0
+#define DP0_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE__SHIFT                                  0x14
+#define DP0_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT                                                      0x18
+#define DP0_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT                                           0x1c
+#define DP0_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK                                                    0x0003FFFFL
+#define DP0_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE_MASK                                    0x00100000L
+#define DP0_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK                                                        0x01000000L
+#define DP0_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK                                             0x10000000L
+//DP0_DP_HBR2_EYE_PATTERN
+#define DP0_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT                                            0x0
+#define DP0_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK                                              0x00000001L
+//DP0_DP_VID_MSA_VBID
+#define DP0_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT                                                       0x0
+#define DP0_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE__SHIFT                                            0xc
+#define DP0_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT                                                     0x18
+#define DP0_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK                                                         0x00000FFFL
+#define DP0_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE_MASK                                              0x00001000L
+#define DP0_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK                                                       0x01000000L
+//DP0_DP_VID_INTERRUPT_CNTL
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT                                           0x0
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT                                           0x1
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT                                          0x2
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK                                             0x00000001L
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK                                             0x00000002L
+#define DP0_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK                                            0x00000004L
+//DP0_DP_DPHY_CNTL
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT                                                         0x0
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT                                                         0x1
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT                                                         0x2
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT                                                         0x3
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_EN__SHIFT                                                                  0x4
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW__SHIFT                                                        0x5
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS__SHIFT                                                       0x6
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM__SHIFT                                               0x7
+#define DP0_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL__SHIFT                                                           0x8
+#define DP0_DP_DPHY_CNTL__DPHY_BYPASS__SHIFT                                                                  0x10
+#define DP0_DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT                                                             0x18
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK                                                           0x00000001L
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK                                                           0x00000002L
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK                                                           0x00000004L
+#define DP0_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK                                                           0x00000008L
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_EN_MASK                                                                    0x00000010L
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW_MASK                                                          0x00000020L
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS_MASK                                                         0x00000040L
+#define DP0_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM_MASK                                                 0x00000080L
+#define DP0_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL_MASK                                                             0x00000100L
+#define DP0_DP_DPHY_CNTL__DPHY_BYPASS_MASK                                                                    0x00010000L
+#define DP0_DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK                                                               0x01000000L
+//DP0_DP_DPHY_TRAINING_PATTERN_SEL
+#define DP0_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT                                    0x0
+#define DP0_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK                                      0x00000003L
+//DP0_DP_DPHY_SYM0
+#define DP0_DP_DPHY_SYM0__DPHY_SYM1__SHIFT                                                                    0x0
+#define DP0_DP_DPHY_SYM0__DPHY_SYM2__SHIFT                                                                    0xa
+#define DP0_DP_DPHY_SYM0__DPHY_SYM3__SHIFT                                                                    0x14
+#define DP0_DP_DPHY_SYM0__DPHY_SYM1_MASK                                                                      0x000003FFL
+#define DP0_DP_DPHY_SYM0__DPHY_SYM2_MASK                                                                      0x000FFC00L
+#define DP0_DP_DPHY_SYM0__DPHY_SYM3_MASK                                                                      0x3FF00000L
+//DP0_DP_DPHY_SYM1
+#define DP0_DP_DPHY_SYM1__DPHY_SYM4__SHIFT                                                                    0x0
+#define DP0_DP_DPHY_SYM1__DPHY_SYM5__SHIFT                                                                    0xa
+#define DP0_DP_DPHY_SYM1__DPHY_SYM6__SHIFT                                                                    0x14
+#define DP0_DP_DPHY_SYM1__DPHY_SYM4_MASK                                                                      0x000003FFL
+#define DP0_DP_DPHY_SYM1__DPHY_SYM5_MASK                                                                      0x000FFC00L
+#define DP0_DP_DPHY_SYM1__DPHY_SYM6_MASK                                                                      0x3FF00000L
+//DP0_DP_DPHY_SYM2
+#define DP0_DP_DPHY_SYM2__DPHY_SYM7__SHIFT                                                                    0x0
+#define DP0_DP_DPHY_SYM2__DPHY_SYM8__SHIFT                                                                    0xa
+#define DP0_DP_DPHY_SYM2__DPHY_SYM7_MASK                                                                      0x000003FFL
+#define DP0_DP_DPHY_SYM2__DPHY_SYM8_MASK                                                                      0x000FFC00L
+//DP0_DP_DPHY_8B10B_CNTL
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT                                                       0x8
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT                                                    0x10
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT                                                    0x18
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK                                                         0x00000100L
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK                                                      0x00010000L
+#define DP0_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK                                                      0x01000000L
+//DP0_DP_DPHY_PRBS_CNTL
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT                                                            0x0
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT                                                           0x4
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT                                                          0x8
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK                                                              0x00000001L
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK                                                             0x00000030L
+#define DP0_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK                                                            0x7FFFFF00L
+//DP0_DP_DPHY_SCRAM_CNTL
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS__SHIFT                                                     0x0
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT                                                 0x4
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT                                                0x8
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE__SHIFT                                                   0x18
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS_MASK                                                       0x00000001L
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK                                                   0x00000010L
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK                                                  0x0003FF00L
+#define DP0_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE_MASK                                                     0x01000000L
+//DP0_DP_DPHY_CRC_CONTROL0
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN__SHIFT                                                          0x0
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN__SHIFT                                                     0x4
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD__SHIFT                                                       0x8
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN_MASK                                                            0x00000001L
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN_MASK                                                       0x00000010L
+#define DP0_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD_MASK                                                         0x00000100L
+//DP0_DP_DPHY_CRC_CONTROL1
+#define DP0_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT__SHIFT                                              0x0
+#define DP0_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT__SHIFT                                               0x8
+#define DP0_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT_MASK                                                0x0000003FL
+#define DP0_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT_MASK                                                 0x00003F00L
+//DP0_DP_DPHY_CRC_RESULT0
+#define DP0_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP0_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP0_DP_DPHY_CRC_RESULT1
+#define DP0_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP0_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP0_DP_DPHY_CRC_RESULT2
+#define DP0_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP0_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP0_DP_DPHY_CRC_RESULT3
+#define DP0_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP0_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP0_DP_DPHY_CRC_STATUS
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID__SHIFT                                                  0x0
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK__SHIFT                                              0x4
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT                                                0x8
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT                                               0xc
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT                                           0x10
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_MASK                                                    0x00000001L
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK_MASK                                                0x00000010L
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK                                                  0x00000100L
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK                                                 0x00001000L
+#define DP0_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK                                             0x00010000L
+//DP0_DP_TU_CNTL
+#define DP0_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG__SHIFT                                                            0x8
+#define DP0_DP_TU_CNTL__DP_TU_OVERFLOW_ACK__SHIFT                                                             0xc
+#define DP0_DP_TU_CNTL__DP_TU_SIZE__SHIFT                                                                     0x18
+#define DP0_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG_MASK                                                              0x00000100L
+#define DP0_DP_TU_CNTL__DP_TU_OVERFLOW_ACK_MASK                                                               0x00001000L
+#define DP0_DP_TU_CNTL__DP_TU_SIZE_MASK                                                                       0x3F000000L
+//DP0_DP_PIXEL_FORMAT_DB_CNTL
+#define DP0_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE__SHIFT                                            0x0
+#define DP0_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING__SHIFT                                           0x4
+#define DP0_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE_MASK                                              0x00000001L
+#define DP0_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING_MASK                                             0x00000010L
+//DP0_DP_SEC_CNTL
+#define DP0_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT                                                          0x0
+#define DP0_DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT                                                             0x4
+#define DP0_DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT                                                             0x8
+#define DP0_DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT                                                             0xc
+#define DP0_DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT                                                             0x10
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT                                                            0x14
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT                                                            0x15
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT                                                            0x16
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT                                                            0x17
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE__SHIFT                                                            0x18
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE__SHIFT                                                            0x19
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE__SHIFT                                                            0x1a
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE__SHIFT                                                            0x1b
+#define DP0_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK                                                            0x00000001L
+#define DP0_DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK                                                               0x00000010L
+#define DP0_DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK                                                               0x00000100L
+#define DP0_DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK                                                               0x00001000L
+#define DP0_DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK                                                               0x00010000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK                                                              0x00100000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK                                                              0x00200000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK                                                              0x00400000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK                                                              0x00800000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE_MASK                                                              0x01000000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE_MASK                                                              0x02000000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE_MASK                                                              0x04000000L
+#define DP0_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE_MASK                                                              0x08000000L
+//DP0_DP_SEC_CNTL1
+#define DP0_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT                                                           0x0
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE__SHIFT                                                   0x1
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY__SHIFT                                                         0x4
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND__SHIFT                                                             0x5
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING__SHIFT                                                     0x6
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED__SHIFT                                             0x7
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE__SHIFT                                                   0x9
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE__SHIFT                                                   0xa
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE__SHIFT                                                   0xb
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE__SHIFT                                                   0xc
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE__SHIFT                                                   0xd
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE__SHIFT                                                   0xe
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE__SHIFT                                                   0xf
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK                                                             0x00000001L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE_MASK                                                     0x00000002L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY_MASK                                                           0x00000010L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_MASK                                                               0x00000020L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING_MASK                                                       0x00000040L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED_MASK                                               0x00000080L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE_MASK                                                     0x00000200L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE_MASK                                                     0x00000400L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE_MASK                                                     0x00000800L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE_MASK                                                     0x00001000L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE_MASK                                                     0x00002000L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE_MASK                                                     0x00004000L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE_MASK                                                     0x00008000L
+#define DP0_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_SEC_FRAMING1
+#define DP0_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT                                               0x0
+#define DP0_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP0_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK                                                 0x00000FFFL
+#define DP0_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP0_DP_SEC_FRAMING2
+#define DP0_DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT                                                     0x0
+#define DP0_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP0_DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK                                                       0x0000FFFFL
+#define DP0_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP0_DP_SEC_FRAMING3
+#define DP0_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT                                                    0x0
+#define DP0_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT                                                0x10
+#define DP0_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK                                                      0x00003FFFL
+#define DP0_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK                                                  0xFFFF0000L
+//DP0_DP_SEC_FRAMING4
+#define DP0_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING__SHIFT                                                      0x0
+#define DP0_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT                                                   0x14
+#define DP0_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT                                                      0x18
+#define DP0_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT                                                         0x1c
+#define DP0_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT                                                  0x1d
+#define DP0_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING_MASK                                                        0x00000001L
+#define DP0_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK                                                     0x00100000L
+#define DP0_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK                                                        0x01000000L
+#define DP0_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK                                                           0x10000000L
+#define DP0_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK                                                    0x20000000L
+//DP0_DP_SEC_AUD_N
+#define DP0_DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT                                                                 0x0
+#define DP0_DP_SEC_AUD_N__DP_SEC_AUD_N_MASK                                                                   0x00FFFFFFL
+//DP0_DP_SEC_AUD_N_READBACK
+#define DP0_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT                                               0x0
+#define DP0_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK                                                 0x00FFFFFFL
+//DP0_DP_SEC_AUD_M
+#define DP0_DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT                                                                 0x0
+#define DP0_DP_SEC_AUD_M__DP_SEC_AUD_M_MASK                                                                   0x00FFFFFFL
+//DP0_DP_SEC_AUD_M_READBACK
+#define DP0_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT                                               0x0
+#define DP0_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK                                                 0x00FFFFFFL
+//DP0_DP_SEC_TIMESTAMP
+#define DP0_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT                                                    0x0
+#define DP0_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK                                                      0x00000001L
+//DP0_DP_SEC_PACKET_CNTL
+#define DP0_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT                                                    0x4
+#define DP0_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER__SHIFT                                              0x18
+#define DP0_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK                                                      0x00000010L
+#define DP0_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER_MASK                                                0x3F000000L
+//DP0_DP_MSE_RATE_CNTL
+#define DP0_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT                                                            0x0
+#define DP0_DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT                                                            0x1a
+#define DP0_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK                                                              0x03FFFFFFL
+#define DP0_DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK                                                              0xFC000000L
+//DP0_DP_MSE_RATE_UPDATE
+#define DP0_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT                                             0x0
+#define DP0_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK                                               0x00000001L
+//DP0_DP_MSE_SAT0
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT                                                               0x0
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0__SHIFT                                                           0x4
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0__SHIFT                                                   0x5
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT                                                        0x8
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT                                                               0x10
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1__SHIFT                                                           0x14
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1__SHIFT                                                   0x15
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT                                                        0x18
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK                                                                 0x00000007L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0_MASK                                                             0x00000010L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0_MASK                                                     0x00000020L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK                                                          0x00003F00L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK                                                                 0x00070000L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1_MASK                                                             0x00100000L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1_MASK                                                     0x00200000L
+#define DP0_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK                                                          0x3F000000L
+//DP0_DP_MSE_SAT1
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT                                                               0x0
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2__SHIFT                                                           0x4
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2__SHIFT                                                   0x5
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT                                                        0x8
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT                                                               0x10
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3__SHIFT                                                           0x14
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3__SHIFT                                                   0x15
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT                                                        0x18
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK                                                                 0x00000007L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2_MASK                                                             0x00000010L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2_MASK                                                     0x00000020L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK                                                          0x00003F00L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK                                                                 0x00070000L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3_MASK                                                             0x00100000L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3_MASK                                                     0x00200000L
+#define DP0_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK                                                          0x3F000000L
+//DP0_DP_MSE_SAT2
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT                                                               0x0
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4__SHIFT                                                           0x4
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4__SHIFT                                                   0x5
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT                                                        0x8
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT                                                               0x10
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5__SHIFT                                                           0x14
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5__SHIFT                                                   0x15
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT                                                        0x18
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK                                                                 0x00000007L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4_MASK                                                             0x00000010L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4_MASK                                                     0x00000020L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK                                                          0x00003F00L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK                                                                 0x00070000L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5_MASK                                                             0x00100000L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5_MASK                                                     0x00200000L
+#define DP0_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK                                                          0x3F000000L
+//DP0_DP_MSE_SAT_UPDATE
+#define DP0_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT                                                       0x0
+#define DP0_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT                                                   0x8
+#define DP0_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK                                                         0x00000003L
+#define DP0_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK                                                     0x00000100L
+//DP0_DP_MSE_LINK_TIMING
+#define DP0_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT                                                      0x0
+#define DP0_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT                                                       0x10
+#define DP0_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK                                                        0x000003FFL
+#define DP0_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK                                                         0x00030000L
+//DP0_DP_MSE_MISC_CNTL
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT                                                        0x0
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT                                                    0x4
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT                                                      0x8
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK                                                          0x00000001L
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK                                                      0x00000010L
+#define DP0_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK                                                        0x00000100L
+//DP0_DP_DPHY_BS_SR_SWAP_CNTL
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT                                                0x0
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE__SHIFT                                              0xf
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START__SHIFT                                          0x10
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK                                                  0x000003FFL
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK                                                0x00008000L
+#define DP0_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START_MASK                                            0x00010000L
+//DP0_DP_DPHY_HBR2_PATTERN_CONTROL
+#define DP0_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL__SHIFT                                 0x0
+#define DP0_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL_MASK                                   0x00000007L
+//DP0_DP_MSE_SAT0_STATUS
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS__SHIFT                                                 0x0
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS__SHIFT                                             0x4
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS__SHIFT                                     0x5
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS__SHIFT                                          0x8
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS__SHIFT                                                 0x10
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS__SHIFT                                             0x14
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS__SHIFT                                     0x15
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS__SHIFT                                          0x18
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS_MASK                                                   0x00000007L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS_MASK                                               0x00000010L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS_MASK                                       0x00000020L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS_MASK                                            0x00003F00L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS_MASK                                                   0x00070000L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS_MASK                                               0x00100000L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS_MASK                                       0x00200000L
+#define DP0_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS_MASK                                            0x3F000000L
+//DP0_DP_MSE_SAT1_STATUS
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS__SHIFT                                                 0x0
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS__SHIFT                                             0x4
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS__SHIFT                                     0x5
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS__SHIFT                                          0x8
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS__SHIFT                                                 0x10
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS__SHIFT                                             0x14
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS__SHIFT                                     0x15
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS__SHIFT                                          0x18
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS_MASK                                                   0x00000007L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS_MASK                                               0x00000010L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS_MASK                                       0x00000020L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS_MASK                                            0x00003F00L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS_MASK                                                   0x00070000L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS_MASK                                               0x00100000L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS_MASK                                       0x00200000L
+#define DP0_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS_MASK                                            0x3F000000L
+//DP0_DP_MSE_SAT2_STATUS
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS__SHIFT                                                 0x0
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS__SHIFT                                             0x4
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS__SHIFT                                     0x5
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS__SHIFT                                          0x8
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS__SHIFT                                                 0x10
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS__SHIFT                                             0x14
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS__SHIFT                                     0x15
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS__SHIFT                                          0x18
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS_MASK                                                   0x00000007L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS_MASK                                               0x00000010L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS_MASK                                       0x00000020L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS_MASK                                            0x00003F00L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS_MASK                                                   0x00070000L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS_MASK                                               0x00100000L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS_MASK                                       0x00200000L
+#define DP0_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS_MASK                                            0x3F000000L
+//DP0_DP_HBLANK_CONTROL
+#define DP0_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                                          0x0
+#define DP0_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                                            0x0000FFFFL
+//DP0_DP_MSA_TIMING_PARAM1
+#define DP0_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL__SHIFT                                                        0x0
+#define DP0_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL__SHIFT                                                        0x10
+#define DP0_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL_MASK                                                          0x0000FFFFL
+#define DP0_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL_MASK                                                          0xFFFF0000L
+//DP0_DP_MSA_TIMING_PARAM2
+#define DP0_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART__SHIFT                                                        0x0
+#define DP0_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART__SHIFT                                                        0x10
+#define DP0_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART_MASK                                                          0x0000FFFFL
+#define DP0_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART_MASK                                                          0xFFFF0000L
+//DP0_DP_MSA_TIMING_PARAM3
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH__SHIFT                                                    0x0
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY__SHIFT                                                 0xf
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH__SHIFT                                                    0x10
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY__SHIFT                                                 0x1f
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH_MASK                                                      0x00007FFFL
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY_MASK                                                   0x00008000L
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH_MASK                                                      0x7FFF0000L
+#define DP0_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY_MASK                                                   0x80000000L
+//DP0_DP_MSA_TIMING_PARAM4
+#define DP0_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT__SHIFT                                                       0x0
+#define DP0_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH__SHIFT                                                        0x10
+#define DP0_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT_MASK                                                         0x0000FFFFL
+#define DP0_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH_MASK                                                          0xFFFF0000L
+//DP0_DP_MSO_CNTL
+#define DP0_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK__SHIFT                                                         0x0
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE__SHIFT                                                      0x4
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE__SHIFT                                                         0x8
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE__SHIFT                                                         0xc
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE__SHIFT                                                         0x10
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE__SHIFT                                                         0x14
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE__SHIFT                                                        0x18
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE__SHIFT                                                        0x1c
+#define DP0_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK_MASK                                                           0x00000003L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE_MASK                                                        0x000000F0L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE_MASK                                                           0x00000F00L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE_MASK                                                           0x0000F000L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE_MASK                                                           0x000F0000L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE_MASK                                                           0x00F00000L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE_MASK                                                          0x0F000000L
+#define DP0_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE_MASK                                                          0xF0000000L
+//DP0_DP_MSO_CNTL1
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE__SHIFT                                                       0x0
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE__SHIFT                                                       0x4
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE__SHIFT                                                       0x8
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE__SHIFT                                                       0xc
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE__SHIFT                                                       0x10
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE__SHIFT                                                       0x14
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE__SHIFT                                                       0x1c
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE_MASK                                                         0x0000000FL
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE_MASK                                                         0x000000F0L
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE_MASK                                                         0x00000F00L
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE_MASK                                                         0x0000F000L
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE_MASK                                                         0x000F0000L
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE_MASK                                                         0x00F00000L
+#define DP0_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE_MASK                                                         0xF0000000L
+//DP0_DP_STEER_FIFO_CNTL
+#define DP0_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN__SHIFT                       0x8
+#define DP0_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                        0x10
+#define DP0_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN_MASK                         0x00000100L
+#define DP0_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE_MASK                                          0x00030000L
+//DP0_DP_SEC_CNTL2
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND__SHIFT                                                             0x0
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING__SHIFT                                                     0x1
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED__SHIFT                                             0x2
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE__SHIFT                                                    0x3
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND__SHIFT                                                             0x4
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING__SHIFT                                                     0x5
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED__SHIFT                                             0x6
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE__SHIFT                                                    0x7
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND__SHIFT                                                             0x8
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING__SHIFT                                                     0x9
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED__SHIFT                                             0xa
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE__SHIFT                                                    0xb
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND__SHIFT                                                             0xc
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING__SHIFT                                                     0xd
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE__SHIFT                                                    0xf
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND__SHIFT                                                             0x10
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING__SHIFT                                                     0x11
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED__SHIFT                                             0x12
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE__SHIFT                                                    0x13
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND__SHIFT                                                             0x14
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING__SHIFT                                                     0x15
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED__SHIFT                                             0x16
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE__SHIFT                                                    0x17
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND__SHIFT                                                             0x18
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING__SHIFT                                                     0x19
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED__SHIFT                                             0x1a
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE__SHIFT                                                    0x1b
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP11_PPS__SHIFT                                                             0x1c
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_MASK                                                               0x00000001L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING_MASK                                                       0x00000002L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED_MASK                                               0x00000004L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE_MASK                                                      0x00000008L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_MASK                                                               0x00000010L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING_MASK                                                       0x00000020L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED_MASK                                               0x00000040L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE_MASK                                                      0x00000080L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_MASK                                                               0x00000100L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING_MASK                                                       0x00000200L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED_MASK                                               0x00000400L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE_MASK                                                      0x00000800L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_MASK                                                               0x00001000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING_MASK                                                       0x00002000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE_MASK                                                      0x00008000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_MASK                                                               0x00010000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING_MASK                                                       0x00020000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED_MASK                                               0x00040000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE_MASK                                                      0x00080000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_MASK                                                               0x00100000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING_MASK                                                       0x00200000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED_MASK                                               0x00400000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE_MASK                                                      0x00800000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_MASK                                                               0x01000000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING_MASK                                                       0x02000000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED_MASK                                               0x04000000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE_MASK                                                      0x08000000L
+#define DP0_DP_SEC_CNTL2__DP_SEC_GSP11_PPS_MASK                                                               0x10000000L
+//DP0_DP_SEC_CNTL3
+#define DP0_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM__SHIFT                                                         0x0
+#define DP0_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP0_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_SEC_CNTL4
+#define DP0_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM__SHIFT                                                         0x0
+#define DP0_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP0_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_SEC_CNTL5
+#define DP0_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM__SHIFT                                                         0x0
+#define DP0_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP0_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_SEC_CNTL6
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM__SHIFT                                                         0x0
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE__SHIFT                                                    0x10
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE__SHIFT                                                    0x11
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE__SHIFT                                                    0x12
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE__SHIFT                                                    0x13
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE__SHIFT                                                    0x14
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE__SHIFT                                                    0x15
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE__SHIFT                                                    0x16
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE__SHIFT                                                    0x17
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE__SHIFT                                                    0x18
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE__SHIFT                                                    0x19
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE__SHIFT                                                   0x1a
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE__SHIFT                                                   0x1b
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE_MASK                                                      0x00010000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE_MASK                                                      0x00020000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE_MASK                                                      0x00040000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE_MASK                                                      0x00080000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE_MASK                                                      0x00100000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE_MASK                                                      0x00200000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE_MASK                                                      0x00400000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE_MASK                                                      0x00800000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE_MASK                                                      0x01000000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE_MASK                                                      0x02000000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE_MASK                                                     0x04000000L
+#define DP0_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE_MASK                                                     0x08000000L
+//DP0_DP_SEC_CNTL7
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE__SHIFT                                                      0x0
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE__SHIFT                                                     0x1
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE__SHIFT                                                      0x4
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE__SHIFT                                                     0x5
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE__SHIFT                                                      0x8
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE__SHIFT                                                     0x9
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE__SHIFT                                                      0xc
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE__SHIFT                                                     0xd
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE__SHIFT                                                      0x10
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE__SHIFT                                                     0x11
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE__SHIFT                                                      0x14
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE__SHIFT                                                     0x15
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE__SHIFT                                                      0x18
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE__SHIFT                                                     0x19
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE__SHIFT                                                      0x1c
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE__SHIFT                                                     0x1d
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE_MASK                                                        0x00000001L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE_MASK                                                       0x00000002L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE_MASK                                                        0x00000010L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE_MASK                                                       0x00000020L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE_MASK                                                        0x00000100L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE_MASK                                                       0x00000200L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE_MASK                                                        0x00001000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE_MASK                                                       0x00002000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE_MASK                                                        0x00010000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE_MASK                                                       0x00020000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE_MASK                                                        0x00100000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE_MASK                                                       0x00200000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE_MASK                                                        0x01000000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE_MASK                                                       0x02000000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE_MASK                                                        0x10000000L
+#define DP0_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE_MASK                                                       0x20000000L
+//DP0_DP_DB_CNTL
+#define DP0_DP_DB_CNTL__DP_DB_PENDING__SHIFT                                                                  0x0
+#define DP0_DP_DB_CNTL__DP_DB_TAKEN__SHIFT                                                                    0x4
+#define DP0_DP_DB_CNTL__DP_DB_TAKEN_CLR__SHIFT                                                                0x5
+#define DP0_DP_DB_CNTL__DP_DB_LOCK__SHIFT                                                                     0x8
+#define DP0_DP_DB_CNTL__DP_DB_DISABLE__SHIFT                                                                  0xc
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_PENDING__SHIFT                                                          0xf
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN__SHIFT                                                            0x10
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR__SHIFT                                                        0x11
+#define DP0_DP_DB_CNTL__DP_DB_PENDING_MASK                                                                    0x00000001L
+#define DP0_DP_DB_CNTL__DP_DB_TAKEN_MASK                                                                      0x00000010L
+#define DP0_DP_DB_CNTL__DP_DB_TAKEN_CLR_MASK                                                                  0x00000020L
+#define DP0_DP_DB_CNTL__DP_DB_LOCK_MASK                                                                       0x00000100L
+#define DP0_DP_DB_CNTL__DP_DB_DISABLE_MASK                                                                    0x00001000L
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_PENDING_MASK                                                            0x00008000L
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_MASK                                                              0x00010000L
+#define DP0_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR_MASK                                                          0x00020000L
+//DP0_DP_MSA_VBID_MISC
+#define DP0_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE__SHIFT                                         0x0
+#define DP0_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                                      0x4
+#define DP0_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE__SHIFT                                                        0x8
+#define DP0_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE__SHIFT                                                        0x9
+#define DP0_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN__SHIFT                                                     0xc
+#define DP0_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN__SHIFT                                                     0xd
+#define DP0_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE__SHIFT                                                  0xf
+#define DP0_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM__SHIFT                                                        0x10
+#define DP0_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_MASK                                           0x00000003L
+#define DP0_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                                        0x00000010L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_MASK                                                          0x00000100L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_MASK                                                          0x00000200L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN_MASK                                                       0x00001000L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN_MASK                                                       0x00002000L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE_MASK                                                    0x00008000L
+#define DP0_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM_MASK                                                          0xFFFF0000L
+//DP0_DP_SEC_METADATA_TRANSMISSION
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE__SHIFT                                0x0
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE__SHIFT                        0x1
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE__SHIFT                            0x4
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE__SHIFT                                  0x10
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE_MASK                                  0x00000001L
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE_MASK                          0x00000002L
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE_MASK                              0x000000F0L
+#define DP0_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_MASK                                    0xFFFF0000L
+//DP0_DP_ALPM_CNTL
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND__SHIFT                                                         0x0
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING__SHIFT                                                      0x1
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND__SHIFT                                                       0x2
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING__SHIFT                                                    0x3
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE__SHIFT                                            0x4
+#define DP0_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO__SHIFT                                        0x5
+#define DP0_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE__SHIFT                                                  0x6
+#define DP0_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP__SHIFT                                     0x7
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM__SHIFT                                                  0x8
+#define DP0_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS__SHIFT                                   0xb
+#define DP0_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN__SHIFT                                       0xc
+#define DP0_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE__SHIFT                                                       0xf
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM__SHIFT                                             0x10
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND_MASK                                                           0x00000001L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING_MASK                                                        0x00000002L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND_MASK                                                         0x00000004L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING_MASK                                                      0x00000008L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE_MASK                                              0x00000010L
+#define DP0_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO_MASK                                          0x00000020L
+#define DP0_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE_MASK                                                    0x00000040L
+#define DP0_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_MASK                                       0x00000080L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM_MASK                                                    0x00000300L
+#define DP0_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS_MASK                                     0x00000800L
+#define DP0_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN_MASK                                         0x00001000L
+#define DP0_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE_MASK                                                         0x00008000L
+#define DP0_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM_MASK                                               0xFFFF0000L
+//DP0_DP_GSP8_CNTL
+#define DP0_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE__SHIFT                                                       0x0
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE__SHIFT                                                           0x4
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND__SHIFT                                                             0x7
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING__SHIFT                                                     0xc
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE_MASK                                                         0x0000000FL
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE_MASK                                                             0x00000010L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_MASK                                                               0x00000080L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING_MASK                                                       0x00001000L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP0_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_GSP9_CNTL
+#define DP0_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE__SHIFT                                                       0x0
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE__SHIFT                                                           0x4
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND__SHIFT                                                             0x7
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING__SHIFT                                                     0xc
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM__SHIFT                                                         0x10
+#define DP0_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE_MASK                                                         0x0000000FL
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE_MASK                                                             0x00000010L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_MASK                                                               0x00000080L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING_MASK                                                       0x00001000L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP0_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP0_DP_GSP10_CNTL
+#define DP0_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE__SHIFT                                                     0x0
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE__SHIFT                                                         0x4
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND__SHIFT                                                           0x7
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING__SHIFT                                                   0xc
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM__SHIFT                                                       0x10
+#define DP0_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE_MASK                                                       0x0000000FL
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE_MASK                                                           0x00000010L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_MASK                                                             0x00000080L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING_MASK                                                     0x00001000L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP0_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP0_DP_GSP11_CNTL
+#define DP0_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE__SHIFT                                                     0x0
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE__SHIFT                                                         0x4
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND__SHIFT                                                           0x7
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING__SHIFT                                                   0xc
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM__SHIFT                                                       0x10
+#define DP0_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE_MASK                                                       0x0000000FL
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE_MASK                                                           0x00000010L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_MASK                                                             0x00000080L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING_MASK                                                     0x00001000L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP0_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP0_DP_GSP_EN_DB_STATUS
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING__SHIFT                                             0x0
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING__SHIFT                                             0x1
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING__SHIFT                                             0x2
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING__SHIFT                                             0x3
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING__SHIFT                                             0x4
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING__SHIFT                                             0x5
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING__SHIFT                                             0x6
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING__SHIFT                                             0x7
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING__SHIFT                                             0x8
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING__SHIFT                                             0x9
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING__SHIFT                                            0xa
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING__SHIFT                                            0xb
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING_MASK                                               0x00000001L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING_MASK                                               0x00000002L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING_MASK                                               0x00000004L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING_MASK                                               0x00000008L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING_MASK                                               0x00000010L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING_MASK                                               0x00000020L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING_MASK                                               0x00000040L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING_MASK                                               0x00000080L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING_MASK                                               0x00000100L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING_MASK                                               0x00000200L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING_MASK                                              0x00000400L
+#define DP0_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING_MASK                                              0x00000800L
+//DP0_DP_AUXLESS_ALPM_CNTL1
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT__SHIFT                                              0x4
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY__SHIFT                                               0x8
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL__SHIFT                                            0x14
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE__SHIFT                                     0x1f
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT_MASK                                                0x000000F0L
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY_MASK                                                 0x0007FF00L
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL_MASK                                              0x1FF00000L
+#define DP0_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE_MASK                                       0x80000000L
+//DP0_DP_AUXLESS_ALPM_CNTL2
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME__SHIFT                                           0x0
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND__SHIFT                                                 0x7
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE__SHIFT                                            0x10
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING__SHIFT                                              0x11
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE__SHIFT                                            0x12
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING__SHIFT                                              0x13
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD__SHIFT                                          0x14
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME_MASK                                             0x0000007FL
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND_MASK                                                   0x00000080L
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE_MASK                                              0x00010000L
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING_MASK                                                0x00020000L
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE_MASK                                              0x00040000L
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING_MASK                                                0x00080000L
+#define DP0_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD_MASK                                            0x3FF00000L
+//DP0_DP_AUXLESS_ALPM_CNTL3
+#define DP0_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM__SHIFT                                             0x0
+#define DP0_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM__SHIFT                                             0x10
+#define DP0_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM_MASK                                               0x0000FFFFL
+#define DP0_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM_MASK                                               0xFFFF0000L
+//DP0_DP_AUXLESS_ALPM_CNTL4
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN__SHIFT                                                  0x1
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL__SHIFT                                   0x2
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME__SHIFT                                     0x3
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE__SHIFT                                       0x4
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS__SHIFT                                           0x5
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE__SHIFT                                               0x6
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM__SHIFT                                                   0x18
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_MASK                                                    0x00000002L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL_MASK                                     0x00000004L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME_MASK                                       0x00000008L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE_MASK                                         0x00000010L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS_MASK                                             0x00000020L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE_MASK                                                 0x00000040L
+#define DP0_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM_MASK                                                     0xFF000000L
+//DP0_DP_AUXLESS_ALPM_CNTL5
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK__SHIFT                                       0x0
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED__SHIFT                                   0x1
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                                     0x2
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR__SHIFT                                      0x3
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM__SHIFT                                  0x8
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM__SHIFT                                   0x10
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK_MASK                                         0x00000001L
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED_MASK                                     0x00000002L
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                                       0x00000004L
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR_MASK                                        0x00000008L
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM_MASK                                    0x0000FF00L
+#define DP0_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM_MASK                                     0xFFFF0000L
+//DP0_DP_STREAM_SYMBOL_COUNT_STATUS
+#define DP0_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT__SHIFT                                          0x0
+#define DP0_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT_MASK                                            0x0000FFFFL
+//DP0_DP_STREAM_SYMBOL_COUNT_CONTROL
+#define DP0_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE__SHIFT                                  0x0
+#define DP0_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET__SHIFT                                   0x4
+#define DP0_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE_MASK                                    0x00000001L
+#define DP0_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET_MASK                                     0x00000010L
+//DP0_DP_LINK_SYMBOL_COUNT_STATUS0
+#define DP0_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT__SHIFT                                             0x0
+#define DP0_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT_MASK                                               0x0000FFFFL
+//DP0_DP_LINK_SYMBOL_COUNT_STATUS1
+#define DP0_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT__SHIFT                                          0x0
+#define DP0_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT_MASK                                            0xFFFFFFFFL
+//DP0_DP_LINK_SYMBOL_COUNT_CONTROL
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE__SHIFT                                      0x0
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET__SHIFT                                       0x4
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE__SHIFT                                   0x8
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET__SHIFT                                    0xc
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE_MASK                                        0x00000001L
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET_MASK                                         0x00000010L
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE_MASK                                     0x00000100L
+#define DP0_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET_MASK                                      0x00001000L
+//DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT           0x0
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING__SHIFT  0x8
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK             0x00000001L
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+#define DP0_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING_MASK  0x00000100L
+//DP0_DP_DPHY_FAST_TRAINING
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT                                       0x0
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT                                         0x1
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT                            0x2
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING__SHIFT                              0x4
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT                                         0x8
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT                                         0x14
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK                                         0x00000001L
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK                                           0x00000002L
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK                              0x00000004L
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING_MASK                                0x00000010L
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK                                           0x000FFF00L
+#define DP0_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK                                           0xFFF00000L
+//DP0_DP_DPHY_FAST_TRAINING_STATUS
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT                                     0x0
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT                         0x4
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT                             0x8
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT                              0xc
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK                                       0x00000007L
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK                           0x00000010L
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK                               0x00000100L
+#define DP0_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK                                0x00001000L
+
+
+// addressBlock: dce_dc_dio_dig1_vpg_vpg_dispdec
+//VPG1_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG1_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG1_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG1_VPG_GENERIC_PACKET_DATA
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG1_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG1_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG1_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG1_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG1_VPG_GENERIC_STATUS
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG1_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG1_VPG_MEM_PWR
+#define VPG1_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG1_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG1_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG1_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG1_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG1_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG1_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG1_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG1_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG1_VPG_ISRC1_2_DATA
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG1_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_dio_dig1_apg_apg_dispdec
+//APG1_APG_CONTROL
+#define APG1_APG_CONTROL__APG_RESET__SHIFT                                                                    0x1
+#define APG1_APG_CONTROL__APG_RESET_DONE__SHIFT                                                               0x2
+#define APG1_APG_CONTROL__APG_RESET_MASK                                                                      0x00000002L
+#define APG1_APG_CONTROL__APG_RESET_DONE_MASK                                                                 0x00000004L
+//APG1_APG_CONTROL2
+#define APG1_APG_CONTROL2__APG_ENABLE__SHIFT                                                                  0x0
+#define APG1_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID__SHIFT                                                      0x8
+#define APG1_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE__SHIFT                                           0x18
+#define APG1_APG_CONTROL2__APG_ENABLE_MASK                                                                    0x00000001L
+#define APG1_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID_MASK                                                        0x0000FF00L
+#define APG1_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE_MASK                                             0x01000000L
+//APG1_APG_DBG_GEN_CONTROL
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG1_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG1_APG_PACKET_CONTROL
+#define APG1_APG_PACKET_CONTROL__APG_DBG_MUX_SEL__SHIFT                                                       0x0
+#define APG1_APG_PACKET_CONTROL__APG_ACP_SOURCE__SHIFT                                                        0x1
+#define APG1_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE__SHIFT                                                 0x2
+#define APG1_APG_PACKET_CONTROL__APG_DBG_MUX_SEL_MASK                                                         0x00000001L
+#define APG1_APG_PACKET_CONTROL__APG_ACP_SOURCE_MASK                                                          0x00000002L
+#define APG1_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE_MASK                                                   0x00000004L
+//APG1_APG_DBG_ACP
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE__SHIFT                                                             0x0
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                             0x8
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                             0x10
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_REQUIRED__SHIFT                                                         0x18
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE_MASK                                                               0x00000003L
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0_MASK                                               0x0000FF00L
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1_MASK                                               0x00FF0000L
+#define APG1_APG_DBG_ACP__APG_DBG_ACP_REQUIRED_MASK                                                           0x01000000L
+//APG1_APG_DBG_AUDIO_INFO
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA__SHIFT                                                 0x0
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC__SHIFT                                                 0x8
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV__SHIFT                                                0xb
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH__SHIFT                                             0xf
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL__SHIFT                                             0x10
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA_MASK                                                   0x000000FFL
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC_MASK                                                   0x00000700L
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV_MASK                                                  0x00007800L
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH_MASK                                               0x00008000L
+#define APG1_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL_MASK                                               0x00030000L
+//APG1_APG_DBG_60958_0
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_A__SHIFT                                                       0x0
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_B__SHIFT                                                       0x1
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_C__SHIFT                                                       0x2
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_D__SHIFT                                                       0x3
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_MODE__SHIFT                                                    0x6
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE__SHIFT                                           0x8
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER__SHIFT                                           0x10
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L__SHIFT                                        0x14
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY__SHIFT                                      0x18
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY__SHIFT                                          0x1c
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_A_MASK                                                         0x00000001L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_B_MASK                                                         0x00000002L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_C_MASK                                                         0x00000004L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_D_MASK                                                         0x00000038L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_MODE_MASK                                                      0x000000C0L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE_MASK                                             0x0000FF00L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER_MASK                                             0x000F0000L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L_MASK                                          0x00F00000L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY_MASK                                        0x0F000000L
+#define APG1_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY_MASK                                            0x30000000L
+//APG1_APG_DBG_60958_1
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH__SHIFT                                             0x0
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                             0x4
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_VALID__SHIFT                                                      0x10
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R__SHIFT                                        0x14
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH_MASK                                               0x0000000FL
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                               0x000000F0L
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_VALID_MASK                                                        0x00010000L
+#define APG1_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R_MASK                                          0x00F00000L
+//APG1_APG_DBG_60958_2
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2__SHIFT                                        0x0
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3__SHIFT                                        0x4
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4__SHIFT                                        0x8
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5__SHIFT                                        0xc
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6__SHIFT                                        0x10
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7__SHIFT                                        0x14
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2_MASK                                          0x0000000FL
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3_MASK                                          0x000000F0L
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4_MASK                                          0x00000F00L
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5_MASK                                          0x0000F000L
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6_MASK                                          0x000F0000L
+#define APG1_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7_MASK                                          0x00F00000L
+//APG1_APG_AUDIO_CRC_CONTROL
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN__SHIFT                                                   0x0
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT__SHIFT                                                 0x4
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL__SHIFT                                               0xd
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT__SHIFT                                                0x10
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN_MASK                                                     0x00000001L
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT_MASK                                                   0x00000010L
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL_MASK                                                 0x0000E000L
+#define APG1_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT_MASK                                                  0xFFFF0000L
+//APG1_APG_AUDIO_CRC_CONTROL2
+#define APG1_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT__SHIFT                                 0x0
+#define APG1_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT_MASK                                   0x0000FFFFL
+//APG1_APG_AUDIO_CRC_RESULT
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE__SHIFT                                                  0x0
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR__SHIFT                                            0x8
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC__SHIFT                                                       0x10
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_MASK                                                    0x00000001L
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR_MASK                                              0x00000100L
+#define APG1_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_MASK                                                         0xFFFF0000L
+//APG1_APG_DBG_RAMP_CONTROL0
+#define APG1_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT__SHIFT                                             0x0
+#define APG1_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN__SHIFT                                             0x1f
+#define APG1_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT_MASK                                               0x00FFFFFFL
+#define APG1_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN_MASK                                               0x80000000L
+//APG1_APG_DBG_RAMP_CONTROL1
+#define APG1_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT__SHIFT                                             0x0
+#define APG1_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT_MASK                                               0x00FFFFFFL
+//APG1_APG_DBG_RAMP_CONTROL2
+#define APG1_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT__SHIFT                                             0x0
+#define APG1_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT_MASK                                               0x00FFFFFFL
+//APG1_APG_DBG_RAMP_CONTROL3
+#define APG1_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT__SHIFT                                             0x0
+#define APG1_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT_MASK                                               0x00FFFFFFL
+//APG1_APG_STATUS
+#define APG1_APG_STATUS__APG_AUDIO_ENABLE__SHIFT                                                              0x4
+#define APG1_APG_STATUS__APG_HBR_ENABLE__SHIFT                                                                0x8
+#define APG1_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS__SHIFT                                                0x18
+#define APG1_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR__SHIFT                                          0x19
+#define APG1_APG_STATUS__APG_AUDIO_ENABLE_MASK                                                                0x00000010L
+#define APG1_APG_STATUS__APG_HBR_ENABLE_MASK                                                                  0x00000100L
+#define APG1_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_MASK                                                  0x01000000L
+#define APG1_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR_MASK                                            0x02000000L
+//APG1_APG_STATUS2
+#define APG1_APG_STATUS2__APG_OUTPUT_ACTIVE__SHIFT                                                            0x0
+#define APG1_APG_STATUS2__APG_OUTPUT_ACTIVE_MASK                                                              0x00000001L
+//APG1_APG_DBG_AUDIO_DTO_CNTL
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE__SHIFT                                            0x8
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI__SHIFT                                           0xc
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV__SHIFT                                             0x10
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE_MASK                                              0x00000100L
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI_MASK                                             0x00007000L
+#define APG1_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV_MASK                                               0x00070000L
+//APG1_APG_MEM_PWR
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG1_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG1_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG1_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+//APG1_APG_SPARE
+#define APG1_APG_SPARE__APG_SPARE__SHIFT                                                                      0x0
+#define APG1_APG_SPARE__APG_SPARE_MASK                                                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dig1_dme_dme_dispdec
+//DME1_DME_CONTROL
+#define DME1_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME1_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME1_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME1_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME1_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME1_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME1_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME1_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME1_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME1_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME1_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME1_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME1_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME1_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME1_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME1_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME1_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME1_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME1_DME_MEMORY_CONTROL
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME1_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig1_dispdec
+//DIG1_DIG_FE_CNTL
+#define DIG1_DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT                                                            0x0
+#define DIG1_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT                                                        0x4
+#define DIG1_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT                                                       0x8
+#define DIG1_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT__SHIFT                                                    0xc
+#define DIG1_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN__SHIFT                                                        0x14
+#define DIG1_DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK                                                              0x00000007L
+#define DIG1_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK                                                          0x00000070L
+#define DIG1_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK                                                         0x00000100L
+#define DIG1_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT_MASK                                                      0x00007000L
+#define DIG1_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN_MASK                                                          0x00100000L
+//DIG1_DIG_FE_CLK_CNTL
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_MODE__SHIFT                                                              0x0
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN__SHIFT                                                            0x4
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON__SHIFT                                                0xa
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON__SHIFT                                              0xb
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON__SHIFT                                          0xc
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON__SHIFT                                         0xd
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON__SHIFT                                             0xe
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_MODE_MASK                                                                0x00000007L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN_MASK                                                              0x00000010L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON_MASK                                                  0x00000400L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON_MASK                                                0x00000800L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON_MASK                                            0x00001000L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON_MASK                                           0x00002000L
+#define DIG1_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON_MASK                                               0x00004000L
+//DIG1_DIG_FE_EN_CNTL
+#define DIG1_DIG_FE_EN_CNTL__DIG_FE_ENABLE__SHIFT                                                             0x0
+#define DIG1_DIG_FE_EN_CNTL__DIG_FE_ENABLE_MASK                                                               0x00000001L
+//DIG1_DIG_OUTPUT_CRC_CNTL
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT                                                    0x0
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT                                              0x4
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT                                              0x8
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK                                                      0x00000001L
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK                                                0x00000010L
+#define DIG1_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK                                                0x00000300L
+//DIG1_DIG_OUTPUT_CRC_RESULT
+#define DIG1_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT                                              0x0
+#define DIG1_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK                                                0x3FFFFFFFL
+//DIG1_DIG_CLOCK_PATTERN
+#define DIG1_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT                                                      0x0
+#define DIG1_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK                                                        0x000003FFL
+//DIG1_DIG_TEST_PATTERN
+#define DIG1_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT                                                 0x0
+#define DIG1_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT                                              0x1
+#define DIG1_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT                                               0x4
+#define DIG1_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT                                                0x5
+#define DIG1_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT                                      0x6
+#define DIG1_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT                                                 0x10
+#define DIG1_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK                                                   0x00000001L
+#define DIG1_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK                                                0x00000002L
+#define DIG1_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK                                                 0x00000010L
+#define DIG1_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK                                                  0x00000020L
+#define DIG1_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK                                        0x00000040L
+#define DIG1_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK                                                   0x03FF0000L
+//DIG1_DIG_RANDOM_PATTERN_SEED
+#define DIG1_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT                                          0x0
+#define DIG1_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT                                       0x18
+#define DIG1_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK                                            0x00FFFFFFL
+#define DIG1_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK                                         0x01000000L
+//DIG1_DIG_FIFO_CTRL0
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE__SHIFT                                                           0x0
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_RESET__SHIFT                                                            0x1
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL__SHIFT                                                 0x2
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC__SHIFT                                                   0x8
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                           0x9
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE__SHIFT                                                       0x14
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_ERROR__SHIFT                                                            0x1c
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE_MASK                                                             0x00000001L
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_RESET_MASK                                                              0x00000002L
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL_MASK                                                   0x000000FCL
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC_MASK                                                     0x00000100L
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE_MASK                                             0x00000600L
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE_MASK                                                         0x00100000L
+#define DIG1_DIG_FIFO_CTRL0__DIG_FIFO_ERROR_MASK                                                              0x30000000L
+//DIG1_DIG_FIFO_CTRL1
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT                                              0x1
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL__SHIFT                                                  0x2
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY__SHIFT                                                  0x8
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT                                                0xa
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL__SHIFT                                                    0x10
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL__SHIFT                                                    0x16
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED__SHIFT                                                       0x1d
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT                                              0x1e
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT                                              0x1f
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK                                                0x00000002L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL_MASK                                                    0x000000FCL
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY_MASK                                                    0x00000300L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK                                                  0x0000FC00L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL_MASK                                                      0x001F0000L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL_MASK                                                      0x03C00000L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED_MASK                                                         0x20000000L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK                                                0x40000000L
+#define DIG1_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK                                                0x80000000L
+//DIG1_HDMI_METADATA_PACKET_CONTROL
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                                 0x0
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                         0x4
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                                 0x8
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                   0x10
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                   0x00000001L
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                           0x00000010L
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                   0x00000100L
+#define DIG1_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                     0xFFFF0000L
+//DIG1_HDMI_CONTROL
+#define DIG1_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT                                                       0x1
+#define DIG1_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT                                                     0x2
+#define DIG1_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED__SHIFT                                            0x3
+#define DIG1_HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT                                                              0x8
+#define DIG1_HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT                                                             0x9
+#define DIG1_HDMI_CONTROL__DOLBY_VISION_EN__SHIFT                                                             0xa
+#define DIG1_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED__SHIFT                                         0xb
+#define DIG1_HDMI_CONTROL__TMDS_PIXEL_ENCODING__SHIFT                                                         0xc
+#define DIG1_HDMI_CONTROL__TMDS_COLOR_FORMAT__SHIFT                                                           0xd
+#define DIG1_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM__SHIFT                                           0x10
+#define DIG1_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT                                                      0x18
+#define DIG1_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                       0x1c
+#define DIG1_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK                                                         0x00000002L
+#define DIG1_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK                                                       0x00000004L
+#define DIG1_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK                                              0x00000008L
+#define DIG1_HDMI_CONTROL__HDMI_ERROR_ACK_MASK                                                                0x00000100L
+#define DIG1_HDMI_CONTROL__HDMI_ERROR_MASK_MASK                                                               0x00000200L
+#define DIG1_HDMI_CONTROL__DOLBY_VISION_EN_MASK                                                               0x00000400L
+#define DIG1_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED_MASK                                           0x00000800L
+#define DIG1_HDMI_CONTROL__TMDS_PIXEL_ENCODING_MASK                                                           0x00001000L
+#define DIG1_HDMI_CONTROL__TMDS_COLOR_FORMAT_MASK                                                             0x00006000L
+#define DIG1_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM_MASK                                             0x003F0000L
+#define DIG1_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK                                                        0x01000000L
+#define DIG1_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK                                                         0x30000000L
+//DIG1_HDMI_STATUS
+#define DIG1_HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT                                                           0x0
+#define DIG1_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT                                                      0x10
+#define DIG1_HDMI_STATUS__HDMI_ERROR_INT__SHIFT                                                               0x1b
+#define DIG1_HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK                                                             0x00000001L
+#define DIG1_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK                                                        0x00010000L
+#define DIG1_HDMI_STATUS__HDMI_ERROR_INT_MASK                                                                 0x08000000L
+//DIG1_HDMI_AUDIO_PACKET_CONTROL
+#define DIG1_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT                                            0x4
+#define DIG1_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK                                              0x00000030L
+//DIG1_HDMI_ACR_PACKET_CONTROL
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                    0x0
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                    0x1
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                  0x4
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                  0x8
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                               0xc
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                              0x10
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                      0x00000001L
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                      0x00000002L
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                    0x00000030L
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                    0x00000100L
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                                 0x00001000L
+#define DIG1_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                                0x00070000L
+//DIG1_HDMI_VBI_PACKET_CONTROL
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT                                                   0x0
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT                                                     0x4
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT                                                     0x5
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT                                                   0x8
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT                                                   0x9
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND__SHIFT                                                    0xc
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT                                                   0x10
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE__SHIFT                                                    0x18
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK                                                     0x00000001L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK                                                       0x00000010L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK                                                       0x00000020L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK                                                     0x00000100L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK                                                     0x00000200L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND_MASK                                                      0x00001000L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK                                                     0x003F0000L
+#define DIG1_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE_MASK                                                      0x3F000000L
+//DIG1_HDMI_INFOFRAME_CONTROL0
+#define DIG1_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT                                             0x4
+#define DIG1_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE__SHIFT                                             0x8
+#define DIG1_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK                                               0x00000010L
+#define DIG1_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE_MASK                                               0x00003F00L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                          0x1
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                                0x2
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                          0x4
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                          0x5
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                                0x6
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                          0x8
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                          0x9
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                                0xa
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE__SHIFT                           0xb
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                          0xc
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                          0xd
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                                0xe
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE__SHIFT                           0xf
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                          0x11
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                                0x12
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE__SHIFT                           0x13
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                          0x14
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                          0x15
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                                0x16
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE__SHIFT                           0x17
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                          0x18
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                          0x19
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                                0x1a
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE__SHIFT                           0x1b
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                          0x1c
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                          0x1d
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                                0x1e
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE__SHIFT                           0x1f
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                            0x00000001L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                            0x00000002L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                            0x00000010L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                            0x00000020L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                            0x00000100L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                            0x00000200L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                  0x00000400L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE_MASK                             0x00000800L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                            0x00001000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                            0x00002000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                  0x00004000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE_MASK                             0x00008000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                            0x00010000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                            0x00020000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                  0x00040000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE_MASK                             0x00080000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                            0x00100000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                            0x00200000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                  0x00400000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE_MASK                             0x00800000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                            0x01000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                            0x02000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                  0x04000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE_MASK                             0x08000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                            0x10000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                            0x20000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                  0x40000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE_MASK                             0x80000000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL6
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT__SHIFT                                          0x1
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                                0x2
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND__SHIFT                                          0x4
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT__SHIFT                                          0x5
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                                0x6
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND__SHIFT                                         0x8
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT__SHIFT                                         0x9
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                               0xa
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE__SHIFT                          0xb
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND__SHIFT                                         0xc
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT__SHIFT                                         0xd
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                               0xe
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE__SHIFT                          0xf
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND__SHIFT                                         0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT__SHIFT                                         0x11
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                               0x12
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE__SHIFT                          0x13
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND__SHIFT                                         0x14
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT__SHIFT                                         0x15
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                               0x16
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE__SHIFT                          0x17
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND__SHIFT                                         0x18
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT__SHIFT                                         0x19
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                               0x1a
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE__SHIFT                          0x1b
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND_MASK                                            0x00000001L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT_MASK                                            0x00000002L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND_MASK                                            0x00000010L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT_MASK                                            0x00000020L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND_MASK                                           0x00000100L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT_MASK                                           0x00000200L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE_MASK                                 0x00000400L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE_MASK                            0x00000800L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND_MASK                                           0x00001000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT_MASK                                           0x00002000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE_MASK                                 0x00004000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE_MASK                            0x00008000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND_MASK                                           0x00010000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT_MASK                                           0x00020000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE_MASK                                 0x00040000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE_MASK                            0x00080000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND_MASK                                           0x00100000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT_MASK                                           0x00200000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE_MASK                                 0x00400000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE_MASK                            0x00800000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND_MASK                                           0x01000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT_MASK                                           0x02000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE_MASK                                 0x04000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE_MASK                            0x08000000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL5
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                                0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                        0x1
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                                0x2
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                        0x3
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                                0x4
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                        0x5
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                                0x6
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                        0x7
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                                0x8
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                        0x9
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                                0xa
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                        0xb
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                                0xc
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                        0xd
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                                0xe
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                        0xf
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                                0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                        0x11
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                                0x12
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                        0x13
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                               0x14
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                       0x15
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                               0x16
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                       0x17
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                               0x18
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                       0x19
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                               0x1a
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                       0x1b
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                               0x1c
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                       0x1d
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                  0x00000001L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                          0x00000002L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                  0x00000004L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                          0x00000008L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                  0x00000010L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                          0x00000020L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                  0x00000040L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                          0x00000080L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                  0x00000100L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                          0x00000200L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                  0x00000400L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                          0x00000800L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                  0x00001000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                          0x00002000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                  0x00004000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                          0x00008000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                  0x00010000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                          0x00020000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                  0x00040000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                          0x00080000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                                 0x00100000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                         0x00200000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                                 0x00400000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                         0x00800000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                                 0x01000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                         0x02000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                                 0x04000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                         0x08000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                                 0x10000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                         0x20000000L
+//DIG1_HDMI_GC
+#define DIG1_HDMI_GC__HDMI_GC_AVMUTE__SHIFT                                                                   0x0
+#define DIG1_HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT                                                              0x2
+#define DIG1_HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT                                                               0x4
+#define DIG1_HDMI_GC__HDMI_PACKING_PHASE__SHIFT                                                               0x8
+#define DIG1_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT                                                      0xc
+#define DIG1_HDMI_GC__HDMI_GC_AVMUTE_MASK                                                                     0x00000001L
+#define DIG1_HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK                                                                0x00000004L
+#define DIG1_HDMI_GC__HDMI_DEFAULT_PHASE_MASK                                                                 0x00000010L
+#define DIG1_HDMI_GC__HDMI_PACKING_PHASE_MASK                                                                 0x00000F00L
+#define DIG1_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK                                                        0x00001000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL1
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE_MASK                                            0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE_MASK                                            0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL2
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE_MASK                                            0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE_MASK                                            0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL3
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE_MASK                                            0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE_MASK                                            0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL4
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE_MASK                                            0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE_MASK                                            0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL7
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE__SHIFT                                          0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE__SHIFT                                          0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE_MASK                                            0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE_MASK                                            0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL8
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE__SHIFT                                         0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE__SHIFT                                         0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE_MASK                                           0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE_MASK                                           0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL9
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE__SHIFT                                         0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE__SHIFT                                         0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE_MASK                                           0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE_MASK                                           0xFFFF0000L
+//DIG1_HDMI_GENERIC_PACKET_CONTROL10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE__SHIFT                                        0x0
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING__SHIFT                                0x10
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING__SHIFT                                0x11
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING__SHIFT                                0x12
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING__SHIFT                                0x13
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING__SHIFT                                0x14
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING__SHIFT                                0x15
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING__SHIFT                                0x16
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING__SHIFT                                0x17
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING__SHIFT                                0x18
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING__SHIFT                                0x19
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING__SHIFT                               0x1a
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING__SHIFT                               0x1b
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING__SHIFT                               0x1c
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING__SHIFT                               0x1d
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING__SHIFT                               0x1e
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE_MASK                                          0x0000FFFFL
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING_MASK                                  0x00010000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING_MASK                                  0x00020000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING_MASK                                  0x00040000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING_MASK                                  0x00080000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING_MASK                                  0x00100000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING_MASK                                  0x00200000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING_MASK                                  0x00400000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING_MASK                                  0x00800000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING_MASK                                  0x01000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING_MASK                                  0x02000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING_MASK                                 0x04000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING_MASK                                 0x08000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING_MASK                                 0x10000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING_MASK                                 0x20000000L
+#define DIG1_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING_MASK                                 0x40000000L
+//DIG1_HDMI_DB_CONTROL
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                          0x0
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_TAKEN__SHIFT                                                            0x4
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR__SHIFT                                                        0x5
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_LOCK__SHIFT                                                             0x8
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                          0xc
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                       0xf
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN__SHIFT                                                         0x10
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR__SHIFT                                                     0x11
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_PENDING_MASK                                                            0x00000001L
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_TAKEN_MASK                                                              0x00000010L
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR_MASK                                                          0x00000020L
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_LOCK_MASK                                                               0x00000100L
+#define DIG1_HDMI_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                            0x00001000L
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                         0x00008000L
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_MASK                                                           0x00010000L
+#define DIG1_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR_MASK                                                       0x00020000L
+//DIG1_HDMI_ACR_32_0
+#define DIG1_HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                            0xc
+#define DIG1_HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                              0xFFFFF000L
+//DIG1_HDMI_ACR_32_1
+#define DIG1_HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                              0x0
+#define DIG1_HDMI_ACR_32_1__HDMI_ACR_N_32_MASK                                                                0x000FFFFFL
+//DIG1_HDMI_ACR_44_0
+#define DIG1_HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                            0xc
+#define DIG1_HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                              0xFFFFF000L
+//DIG1_HDMI_ACR_44_1
+#define DIG1_HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                              0x0
+#define DIG1_HDMI_ACR_44_1__HDMI_ACR_N_44_MASK                                                                0x000FFFFFL
+//DIG1_HDMI_ACR_48_0
+#define DIG1_HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                            0xc
+#define DIG1_HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                              0xFFFFF000L
+//DIG1_HDMI_ACR_48_1
+#define DIG1_HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                              0x0
+#define DIG1_HDMI_ACR_48_1__HDMI_ACR_N_48_MASK                                                                0x000FFFFFL
+//DIG1_HDMI_ACR_STATUS_0
+#define DIG1_HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                           0xc
+#define DIG1_HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                             0xFFFFF000L
+//DIG1_HDMI_ACR_STATUS_1
+#define DIG1_HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                             0x0
+#define DIG1_HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK                                                               0x000FFFFFL
+//DIG1_DIG_FE_AUDIO_CNTL
+#define DIG1_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT                               0x0
+#define DIG1_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE__SHIFT                                                       0x8
+#define DIG1_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                                 0x00000007L
+#define DIG1_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE_MASK                                                         0x00000100L
+//DIG1_DIG_BE_CLK_CNTL
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_MODE__SHIFT                                                              0x0
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN__SHIFT                                                            0x4
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON__SHIFT                                                 0xb
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON__SHIFT                                            0xd
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_MODE_MASK                                                                0x00000007L
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN_MASK                                                              0x00000010L
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON_MASK                                                   0x00000800L
+#define DIG1_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON_MASK                                              0x00002000L
+//DIG1_DIG_BE_CNTL
+#define DIG1_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT                                                         0x0
+#define DIG1_DIG_BE_CNTL__DIG_SWAP__SHIFT                                                                     0x1
+#define DIG1_DIG_BE_CNTL__DIG_RB_SWITCH_EN__SHIFT                                                             0x2
+#define DIG1_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT                                                         0x8
+#define DIG1_DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT                                                               0x1c
+#define DIG1_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE_MASK                                                           0x00000001L
+#define DIG1_DIG_BE_CNTL__DIG_SWAP_MASK                                                                       0x00000002L
+#define DIG1_DIG_BE_CNTL__DIG_RB_SWITCH_EN_MASK                                                               0x00000004L
+#define DIG1_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK                                                           0x00007F00L
+#define DIG1_DIG_BE_CNTL__DIG_HPD_SELECT_MASK                                                                 0x70000000L
+//DIG1_DIG_BE_EN_CNTL
+#define DIG1_DIG_BE_EN_CNTL__DIG_BE_ENABLE__SHIFT                                                             0x0
+#define DIG1_DIG_BE_EN_CNTL__DIG_BE_ENABLE_MASK                                                               0x00000001L
+//DIG1_TMDS_CNTL
+#define DIG1_TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT                                                                0x0
+#define DIG1_TMDS_CNTL__TMDS_SYNC_PHASE_MASK                                                                  0x00000001L
+//DIG1_TMDS_CONTROL_CHAR
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT                                              0x0
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT                                              0x1
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT                                              0x2
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT                                              0x3
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK                                                0x00000001L
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK                                                0x00000002L
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK                                                0x00000004L
+#define DIG1_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK                                                0x00000008L
+//DIG1_TMDS_CONTROL0_FEEDBACK
+#define DIG1_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT                                     0x0
+#define DIG1_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT                                      0x8
+#define DIG1_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK                                       0x00000003L
+#define DIG1_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK                                        0x00000300L
+//DIG1_TMDS_STEREOSYNC_CTL_SEL
+#define DIG1_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT                                          0x0
+#define DIG1_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK                                            0x00000003L
+//DIG1_TMDS_SYNC_CHAR_PATTERN_0_1
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT                                       0x0
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT                                       0x10
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK                                         0x000003FFL
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK                                         0x03FF0000L
+//DIG1_TMDS_SYNC_CHAR_PATTERN_2_3
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT                                       0x0
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT                                       0x10
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK                                         0x000003FFL
+#define DIG1_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK                                         0x03FF0000L
+//DIG1_TMDS_CTL_BITS
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL0__SHIFT                                                                  0x0
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL1__SHIFT                                                                  0x8
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL2__SHIFT                                                                  0x10
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL3__SHIFT                                                                  0x18
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL0_MASK                                                                    0x00000001L
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL1_MASK                                                                    0x00000100L
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL2_MASK                                                                    0x00010000L
+#define DIG1_TMDS_CTL_BITS__TMDS_CTL3_MASK                                                                    0x01000000L
+//DIG1_TMDS_DCBALANCER_CONTROL
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT                                               0x0
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT                                               0x4
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT                                          0x8
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT                                          0x10
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT                                            0x18
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK                                                 0x00000001L
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK                                                 0x00000070L
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK                                            0x00000100L
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK                                            0x000F0000L
+#define DIG1_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK                                              0x01000000L
+//DIG1_TMDS_SYNC_DCBALANCE_CHAR
+#define DIG1_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01__SHIFT                                          0x0
+#define DIG1_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11__SHIFT                                          0x10
+#define DIG1_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01_MASK                                            0x000003FFL
+#define DIG1_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11_MASK                                            0x03FF0000L
+//DIG1_TMDS_CTL0_1_GEN_CNTL
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT                                                  0x0
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT                                                0x4
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT                                               0x7
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT                                           0x8
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT                                                  0x10
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT                                                0x14
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT                                               0x17
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT                                           0x18
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT                                                0x1f
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK                                              0x10000000L
+#define DIG1_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK                                                  0x80000000L
+//DIG1_TMDS_CTL2_3_GEN_CNTL
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT                                                  0x0
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT                                                0x4
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT                                               0x7
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT                                           0x8
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT                                                  0x10
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT                                                0x14
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT                                               0x17
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT                                           0x18
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG1_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK                                              0x10000000L
+//DIG1_DIG_VERSION
+#define DIG1_DIG_VERSION__DIG_TYPE__SHIFT                                                                     0x0
+#define DIG1_DIG_VERSION__DIG_TYPE_MASK                                                                       0x00000001L
+
+
+// addressBlock: dce_dc_dio_dp1_dispdec
+//DP1_DP_LINK_CNTL
+#define DP1_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT                                                    0x4
+#define DP1_DP_LINK_CNTL__DP_LINK_STATUS__SHIFT                                                               0x8
+#define DP1_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK                                                      0x00000010L
+#define DP1_DP_LINK_CNTL__DP_LINK_STATUS_MASK                                                                 0x00000100L
+//DP1_DP_PIXEL_FORMAT
+#define DP1_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                                                       0x0
+#define DP1_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT__SHIFT                                                 0x4
+#define DP1_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                                              0x8
+#define DP1_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT__SHIFT                                                   0x10
+#define DP1_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                                         0x00000001L
+#define DP1_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT_MASK                                                   0x00000030L
+#define DP1_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                                                0x00000700L
+#define DP1_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT_MASK                                                     0x00010000L
+//DP1_DP_MSA_COLORIMETRY
+#define DP1_DP_MSA_COLORIMETRY__DP_MSA_MISC0__SHIFT                                                           0x18
+#define DP1_DP_MSA_COLORIMETRY__DP_MSA_MISC0_MASK                                                             0xFF000000L
+//DP1_DP_CONFIG
+#define DP1_DP_CONFIG__DP_UDI_LANES__SHIFT                                                                    0x0
+#define DP1_DP_CONFIG__DP_UDI_LANES_MASK                                                                      0x00000003L
+//DP1_DP_VID_STREAM_CNTL
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT                                                   0x0
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT                                                0x8
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT                                                   0x10
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT                                           0x14
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK                                                     0x00000001L
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK                                                  0x00000300L
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK                                                     0x00010000L
+#define DP1_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK                                             0x00100000L
+//DP1_DP_STEER_FIFO
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE__SHIFT                                                        0x0
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT                                                         0x1
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE__SHIFT                                                    0x2
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT                                                      0x4
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT                                                       0x5
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT                                                       0x6
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT                                                      0x7
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE_MASK                                                          0x00000001L
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK                                                           0x00000002L
+#define DP1_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE_MASK                                                      0x00000004L
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK                                                        0x00000010L
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK                                                         0x00000020L
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK                                                         0x00000040L
+#define DP1_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK                                                        0x00000080L
+//DP1_DP_MSA_MISC
+#define DP1_DP_MSA_MISC__DP_MSA_MISC1__SHIFT                                                                  0x0
+#define DP1_DP_MSA_MISC__DP_MSA_MISC2__SHIFT                                                                  0x8
+#define DP1_DP_MSA_MISC__DP_MSA_MISC3__SHIFT                                                                  0x10
+#define DP1_DP_MSA_MISC__DP_MSA_MISC4__SHIFT                                                                  0x18
+#define DP1_DP_MSA_MISC__DP_MSA_MISC1_MASK                                                                    0x000000FFL
+#define DP1_DP_MSA_MISC__DP_MSA_MISC2_MASK                                                                    0x0000FF00L
+#define DP1_DP_MSA_MISC__DP_MSA_MISC3_MASK                                                                    0x00FF0000L
+#define DP1_DP_MSA_MISC__DP_MSA_MISC4_MASK                                                                    0xFF000000L
+//DP1_DP_DPHY_INTERNAL_CTRL
+#define DP1_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN__SHIFT                                         0x0
+#define DP1_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL__SHIFT                                        0x4
+#define DP1_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN_MASK                                           0x00000001L
+#define DP1_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL_MASK                                          0x00000010L
+//DP1_DP_VID_TIMING
+#define DP1_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE__SHIFT                                               0x4
+#define DP1_DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT                                                           0x8
+#define DP1_DP_VID_TIMING__DP_VID_M_DIV__SHIFT                                                                0xc
+#define DP1_DP_VID_TIMING__DP_VID_N_INTERVAL__SHIFT                                                           0xe
+#define DP1_DP_VID_TIMING__DP_VID_N_DIV__SHIFT                                                                0x18
+#define DP1_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE_MASK                                                 0x00000010L
+#define DP1_DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK                                                             0x00000100L
+#define DP1_DP_VID_TIMING__DP_VID_M_DIV_MASK                                                                  0x00003000L
+#define DP1_DP_VID_TIMING__DP_VID_N_INTERVAL_MASK                                                             0x0000C000L
+#define DP1_DP_VID_TIMING__DP_VID_N_DIV_MASK                                                                  0xFF000000L
+//DP1_DP_VID_N
+#define DP1_DP_VID_N__DP_VID_N__SHIFT                                                                         0x0
+#define DP1_DP_VID_N__DP_VID_N_MASK                                                                           0x00FFFFFFL
+//DP1_DP_VID_M
+#define DP1_DP_VID_M__DP_VID_M__SHIFT                                                                         0x0
+#define DP1_DP_VID_M__DP_VID_M_MASK                                                                           0x00FFFFFFL
+//DP1_DP_LINK_FRAMING_CNTL
+#define DP1_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT                                                  0x0
+#define DP1_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE__SHIFT                                  0x14
+#define DP1_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT                                                      0x18
+#define DP1_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT                                           0x1c
+#define DP1_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK                                                    0x0003FFFFL
+#define DP1_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE_MASK                                    0x00100000L
+#define DP1_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK                                                        0x01000000L
+#define DP1_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK                                             0x10000000L
+//DP1_DP_HBR2_EYE_PATTERN
+#define DP1_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT                                            0x0
+#define DP1_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK                                              0x00000001L
+//DP1_DP_VID_MSA_VBID
+#define DP1_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT                                                       0x0
+#define DP1_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE__SHIFT                                            0xc
+#define DP1_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT                                                     0x18
+#define DP1_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK                                                         0x00000FFFL
+#define DP1_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE_MASK                                              0x00001000L
+#define DP1_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK                                                       0x01000000L
+//DP1_DP_VID_INTERRUPT_CNTL
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT                                           0x0
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT                                           0x1
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT                                          0x2
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK                                             0x00000001L
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK                                             0x00000002L
+#define DP1_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK                                            0x00000004L
+//DP1_DP_DPHY_CNTL
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT                                                         0x0
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT                                                         0x1
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT                                                         0x2
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT                                                         0x3
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_EN__SHIFT                                                                  0x4
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW__SHIFT                                                        0x5
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS__SHIFT                                                       0x6
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM__SHIFT                                               0x7
+#define DP1_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL__SHIFT                                                           0x8
+#define DP1_DP_DPHY_CNTL__DPHY_BYPASS__SHIFT                                                                  0x10
+#define DP1_DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT                                                             0x18
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK                                                           0x00000001L
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK                                                           0x00000002L
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK                                                           0x00000004L
+#define DP1_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK                                                           0x00000008L
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_EN_MASK                                                                    0x00000010L
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW_MASK                                                          0x00000020L
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS_MASK                                                         0x00000040L
+#define DP1_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM_MASK                                                 0x00000080L
+#define DP1_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL_MASK                                                             0x00000100L
+#define DP1_DP_DPHY_CNTL__DPHY_BYPASS_MASK                                                                    0x00010000L
+#define DP1_DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK                                                               0x01000000L
+//DP1_DP_DPHY_TRAINING_PATTERN_SEL
+#define DP1_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT                                    0x0
+#define DP1_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK                                      0x00000003L
+//DP1_DP_DPHY_SYM0
+#define DP1_DP_DPHY_SYM0__DPHY_SYM1__SHIFT                                                                    0x0
+#define DP1_DP_DPHY_SYM0__DPHY_SYM2__SHIFT                                                                    0xa
+#define DP1_DP_DPHY_SYM0__DPHY_SYM3__SHIFT                                                                    0x14
+#define DP1_DP_DPHY_SYM0__DPHY_SYM1_MASK                                                                      0x000003FFL
+#define DP1_DP_DPHY_SYM0__DPHY_SYM2_MASK                                                                      0x000FFC00L
+#define DP1_DP_DPHY_SYM0__DPHY_SYM3_MASK                                                                      0x3FF00000L
+//DP1_DP_DPHY_SYM1
+#define DP1_DP_DPHY_SYM1__DPHY_SYM4__SHIFT                                                                    0x0
+#define DP1_DP_DPHY_SYM1__DPHY_SYM5__SHIFT                                                                    0xa
+#define DP1_DP_DPHY_SYM1__DPHY_SYM6__SHIFT                                                                    0x14
+#define DP1_DP_DPHY_SYM1__DPHY_SYM4_MASK                                                                      0x000003FFL
+#define DP1_DP_DPHY_SYM1__DPHY_SYM5_MASK                                                                      0x000FFC00L
+#define DP1_DP_DPHY_SYM1__DPHY_SYM6_MASK                                                                      0x3FF00000L
+//DP1_DP_DPHY_SYM2
+#define DP1_DP_DPHY_SYM2__DPHY_SYM7__SHIFT                                                                    0x0
+#define DP1_DP_DPHY_SYM2__DPHY_SYM8__SHIFT                                                                    0xa
+#define DP1_DP_DPHY_SYM2__DPHY_SYM7_MASK                                                                      0x000003FFL
+#define DP1_DP_DPHY_SYM2__DPHY_SYM8_MASK                                                                      0x000FFC00L
+//DP1_DP_DPHY_8B10B_CNTL
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT                                                       0x8
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT                                                    0x10
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT                                                    0x18
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK                                                         0x00000100L
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK                                                      0x00010000L
+#define DP1_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK                                                      0x01000000L
+//DP1_DP_DPHY_PRBS_CNTL
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT                                                            0x0
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT                                                           0x4
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT                                                          0x8
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK                                                              0x00000001L
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK                                                             0x00000030L
+#define DP1_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK                                                            0x7FFFFF00L
+//DP1_DP_DPHY_SCRAM_CNTL
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS__SHIFT                                                     0x0
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT                                                 0x4
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT                                                0x8
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE__SHIFT                                                   0x18
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS_MASK                                                       0x00000001L
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK                                                   0x00000010L
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK                                                  0x0003FF00L
+#define DP1_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE_MASK                                                     0x01000000L
+//DP1_DP_DPHY_CRC_CONTROL0
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN__SHIFT                                                          0x0
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN__SHIFT                                                     0x4
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD__SHIFT                                                       0x8
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN_MASK                                                            0x00000001L
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN_MASK                                                       0x00000010L
+#define DP1_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD_MASK                                                         0x00000100L
+//DP1_DP_DPHY_CRC_CONTROL1
+#define DP1_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT__SHIFT                                              0x0
+#define DP1_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT__SHIFT                                               0x8
+#define DP1_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT_MASK                                                0x0000003FL
+#define DP1_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT_MASK                                                 0x00003F00L
+//DP1_DP_DPHY_CRC_RESULT0
+#define DP1_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP1_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP1_DP_DPHY_CRC_RESULT1
+#define DP1_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP1_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP1_DP_DPHY_CRC_RESULT2
+#define DP1_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP1_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP1_DP_DPHY_CRC_RESULT3
+#define DP1_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP1_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP1_DP_DPHY_CRC_STATUS
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID__SHIFT                                                  0x0
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK__SHIFT                                              0x4
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT                                                0x8
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT                                               0xc
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT                                           0x10
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_MASK                                                    0x00000001L
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK_MASK                                                0x00000010L
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK                                                  0x00000100L
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK                                                 0x00001000L
+#define DP1_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK                                             0x00010000L
+//DP1_DP_TU_CNTL
+#define DP1_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG__SHIFT                                                            0x8
+#define DP1_DP_TU_CNTL__DP_TU_OVERFLOW_ACK__SHIFT                                                             0xc
+#define DP1_DP_TU_CNTL__DP_TU_SIZE__SHIFT                                                                     0x18
+#define DP1_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG_MASK                                                              0x00000100L
+#define DP1_DP_TU_CNTL__DP_TU_OVERFLOW_ACK_MASK                                                               0x00001000L
+#define DP1_DP_TU_CNTL__DP_TU_SIZE_MASK                                                                       0x3F000000L
+//DP1_DP_PIXEL_FORMAT_DB_CNTL
+#define DP1_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE__SHIFT                                            0x0
+#define DP1_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING__SHIFT                                           0x4
+#define DP1_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE_MASK                                              0x00000001L
+#define DP1_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING_MASK                                             0x00000010L
+//DP1_DP_SEC_CNTL
+#define DP1_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT                                                          0x0
+#define DP1_DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT                                                             0x4
+#define DP1_DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT                                                             0x8
+#define DP1_DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT                                                             0xc
+#define DP1_DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT                                                             0x10
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT                                                            0x14
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT                                                            0x15
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT                                                            0x16
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT                                                            0x17
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE__SHIFT                                                            0x18
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE__SHIFT                                                            0x19
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE__SHIFT                                                            0x1a
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE__SHIFT                                                            0x1b
+#define DP1_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK                                                            0x00000001L
+#define DP1_DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK                                                               0x00000010L
+#define DP1_DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK                                                               0x00000100L
+#define DP1_DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK                                                               0x00001000L
+#define DP1_DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK                                                               0x00010000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK                                                              0x00100000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK                                                              0x00200000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK                                                              0x00400000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK                                                              0x00800000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE_MASK                                                              0x01000000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE_MASK                                                              0x02000000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE_MASK                                                              0x04000000L
+#define DP1_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE_MASK                                                              0x08000000L
+//DP1_DP_SEC_CNTL1
+#define DP1_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT                                                           0x0
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE__SHIFT                                                   0x1
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY__SHIFT                                                         0x4
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND__SHIFT                                                             0x5
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING__SHIFT                                                     0x6
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED__SHIFT                                             0x7
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE__SHIFT                                                   0x9
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE__SHIFT                                                   0xa
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE__SHIFT                                                   0xb
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE__SHIFT                                                   0xc
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE__SHIFT                                                   0xd
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE__SHIFT                                                   0xe
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE__SHIFT                                                   0xf
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK                                                             0x00000001L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE_MASK                                                     0x00000002L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY_MASK                                                           0x00000010L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_MASK                                                               0x00000020L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING_MASK                                                       0x00000040L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED_MASK                                               0x00000080L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE_MASK                                                     0x00000200L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE_MASK                                                     0x00000400L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE_MASK                                                     0x00000800L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE_MASK                                                     0x00001000L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE_MASK                                                     0x00002000L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE_MASK                                                     0x00004000L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE_MASK                                                     0x00008000L
+#define DP1_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_SEC_FRAMING1
+#define DP1_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT                                               0x0
+#define DP1_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP1_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK                                                 0x00000FFFL
+#define DP1_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP1_DP_SEC_FRAMING2
+#define DP1_DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT                                                     0x0
+#define DP1_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP1_DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK                                                       0x0000FFFFL
+#define DP1_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP1_DP_SEC_FRAMING3
+#define DP1_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT                                                    0x0
+#define DP1_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT                                                0x10
+#define DP1_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK                                                      0x00003FFFL
+#define DP1_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK                                                  0xFFFF0000L
+//DP1_DP_SEC_FRAMING4
+#define DP1_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING__SHIFT                                                      0x0
+#define DP1_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT                                                   0x14
+#define DP1_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT                                                      0x18
+#define DP1_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT                                                         0x1c
+#define DP1_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT                                                  0x1d
+#define DP1_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING_MASK                                                        0x00000001L
+#define DP1_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK                                                     0x00100000L
+#define DP1_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK                                                        0x01000000L
+#define DP1_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK                                                           0x10000000L
+#define DP1_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK                                                    0x20000000L
+//DP1_DP_SEC_AUD_N
+#define DP1_DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT                                                                 0x0
+#define DP1_DP_SEC_AUD_N__DP_SEC_AUD_N_MASK                                                                   0x00FFFFFFL
+//DP1_DP_SEC_AUD_N_READBACK
+#define DP1_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT                                               0x0
+#define DP1_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK                                                 0x00FFFFFFL
+//DP1_DP_SEC_AUD_M
+#define DP1_DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT                                                                 0x0
+#define DP1_DP_SEC_AUD_M__DP_SEC_AUD_M_MASK                                                                   0x00FFFFFFL
+//DP1_DP_SEC_AUD_M_READBACK
+#define DP1_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT                                               0x0
+#define DP1_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK                                                 0x00FFFFFFL
+//DP1_DP_SEC_TIMESTAMP
+#define DP1_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT                                                    0x0
+#define DP1_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK                                                      0x00000001L
+//DP1_DP_SEC_PACKET_CNTL
+#define DP1_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT                                                    0x4
+#define DP1_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER__SHIFT                                              0x18
+#define DP1_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK                                                      0x00000010L
+#define DP1_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER_MASK                                                0x3F000000L
+//DP1_DP_MSE_RATE_CNTL
+#define DP1_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT                                                            0x0
+#define DP1_DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT                                                            0x1a
+#define DP1_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK                                                              0x03FFFFFFL
+#define DP1_DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK                                                              0xFC000000L
+//DP1_DP_MSE_RATE_UPDATE
+#define DP1_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT                                             0x0
+#define DP1_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK                                               0x00000001L
+//DP1_DP_MSE_SAT0
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT                                                               0x0
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0__SHIFT                                                           0x4
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0__SHIFT                                                   0x5
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT                                                        0x8
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT                                                               0x10
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1__SHIFT                                                           0x14
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1__SHIFT                                                   0x15
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT                                                        0x18
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK                                                                 0x00000007L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0_MASK                                                             0x00000010L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0_MASK                                                     0x00000020L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK                                                          0x00003F00L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK                                                                 0x00070000L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1_MASK                                                             0x00100000L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1_MASK                                                     0x00200000L
+#define DP1_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK                                                          0x3F000000L
+//DP1_DP_MSE_SAT1
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT                                                               0x0
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2__SHIFT                                                           0x4
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2__SHIFT                                                   0x5
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT                                                        0x8
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT                                                               0x10
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3__SHIFT                                                           0x14
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3__SHIFT                                                   0x15
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT                                                        0x18
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK                                                                 0x00000007L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2_MASK                                                             0x00000010L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2_MASK                                                     0x00000020L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK                                                          0x00003F00L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK                                                                 0x00070000L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3_MASK                                                             0x00100000L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3_MASK                                                     0x00200000L
+#define DP1_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK                                                          0x3F000000L
+//DP1_DP_MSE_SAT2
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT                                                               0x0
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4__SHIFT                                                           0x4
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4__SHIFT                                                   0x5
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT                                                        0x8
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT                                                               0x10
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5__SHIFT                                                           0x14
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5__SHIFT                                                   0x15
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT                                                        0x18
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK                                                                 0x00000007L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4_MASK                                                             0x00000010L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4_MASK                                                     0x00000020L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK                                                          0x00003F00L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK                                                                 0x00070000L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5_MASK                                                             0x00100000L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5_MASK                                                     0x00200000L
+#define DP1_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK                                                          0x3F000000L
+//DP1_DP_MSE_SAT_UPDATE
+#define DP1_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT                                                       0x0
+#define DP1_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT                                                   0x8
+#define DP1_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK                                                         0x00000003L
+#define DP1_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK                                                     0x00000100L
+//DP1_DP_MSE_LINK_TIMING
+#define DP1_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT                                                      0x0
+#define DP1_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT                                                       0x10
+#define DP1_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK                                                        0x000003FFL
+#define DP1_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK                                                         0x00030000L
+//DP1_DP_MSE_MISC_CNTL
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT                                                        0x0
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT                                                    0x4
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT                                                      0x8
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK                                                          0x00000001L
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK                                                      0x00000010L
+#define DP1_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK                                                        0x00000100L
+//DP1_DP_DPHY_BS_SR_SWAP_CNTL
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT                                                0x0
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE__SHIFT                                              0xf
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START__SHIFT                                          0x10
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK                                                  0x000003FFL
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK                                                0x00008000L
+#define DP1_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START_MASK                                            0x00010000L
+//DP1_DP_DPHY_HBR2_PATTERN_CONTROL
+#define DP1_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL__SHIFT                                 0x0
+#define DP1_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL_MASK                                   0x00000007L
+//DP1_DP_MSE_SAT0_STATUS
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS__SHIFT                                                 0x0
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS__SHIFT                                             0x4
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS__SHIFT                                     0x5
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS__SHIFT                                          0x8
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS__SHIFT                                                 0x10
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS__SHIFT                                             0x14
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS__SHIFT                                     0x15
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS__SHIFT                                          0x18
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS_MASK                                                   0x00000007L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS_MASK                                               0x00000010L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS_MASK                                       0x00000020L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS_MASK                                            0x00003F00L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS_MASK                                                   0x00070000L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS_MASK                                               0x00100000L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS_MASK                                       0x00200000L
+#define DP1_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS_MASK                                            0x3F000000L
+//DP1_DP_MSE_SAT1_STATUS
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS__SHIFT                                                 0x0
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS__SHIFT                                             0x4
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS__SHIFT                                     0x5
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS__SHIFT                                          0x8
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS__SHIFT                                                 0x10
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS__SHIFT                                             0x14
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS__SHIFT                                     0x15
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS__SHIFT                                          0x18
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS_MASK                                                   0x00000007L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS_MASK                                               0x00000010L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS_MASK                                       0x00000020L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS_MASK                                            0x00003F00L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS_MASK                                                   0x00070000L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS_MASK                                               0x00100000L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS_MASK                                       0x00200000L
+#define DP1_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS_MASK                                            0x3F000000L
+//DP1_DP_MSE_SAT2_STATUS
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS__SHIFT                                                 0x0
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS__SHIFT                                             0x4
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS__SHIFT                                     0x5
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS__SHIFT                                          0x8
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS__SHIFT                                                 0x10
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS__SHIFT                                             0x14
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS__SHIFT                                     0x15
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS__SHIFT                                          0x18
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS_MASK                                                   0x00000007L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS_MASK                                               0x00000010L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS_MASK                                       0x00000020L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS_MASK                                            0x00003F00L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS_MASK                                                   0x00070000L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS_MASK                                               0x00100000L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS_MASK                                       0x00200000L
+#define DP1_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS_MASK                                            0x3F000000L
+//DP1_DP_HBLANK_CONTROL
+#define DP1_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                                          0x0
+#define DP1_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                                            0x0000FFFFL
+//DP1_DP_MSA_TIMING_PARAM1
+#define DP1_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL__SHIFT                                                        0x0
+#define DP1_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL__SHIFT                                                        0x10
+#define DP1_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL_MASK                                                          0x0000FFFFL
+#define DP1_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL_MASK                                                          0xFFFF0000L
+//DP1_DP_MSA_TIMING_PARAM2
+#define DP1_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART__SHIFT                                                        0x0
+#define DP1_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART__SHIFT                                                        0x10
+#define DP1_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART_MASK                                                          0x0000FFFFL
+#define DP1_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART_MASK                                                          0xFFFF0000L
+//DP1_DP_MSA_TIMING_PARAM3
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH__SHIFT                                                    0x0
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY__SHIFT                                                 0xf
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH__SHIFT                                                    0x10
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY__SHIFT                                                 0x1f
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH_MASK                                                      0x00007FFFL
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY_MASK                                                   0x00008000L
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH_MASK                                                      0x7FFF0000L
+#define DP1_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY_MASK                                                   0x80000000L
+//DP1_DP_MSA_TIMING_PARAM4
+#define DP1_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT__SHIFT                                                       0x0
+#define DP1_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH__SHIFT                                                        0x10
+#define DP1_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT_MASK                                                         0x0000FFFFL
+#define DP1_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH_MASK                                                          0xFFFF0000L
+//DP1_DP_MSO_CNTL
+#define DP1_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK__SHIFT                                                         0x0
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE__SHIFT                                                      0x4
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE__SHIFT                                                         0x8
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE__SHIFT                                                         0xc
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE__SHIFT                                                         0x10
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE__SHIFT                                                         0x14
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE__SHIFT                                                        0x18
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE__SHIFT                                                        0x1c
+#define DP1_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK_MASK                                                           0x00000003L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE_MASK                                                        0x000000F0L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE_MASK                                                           0x00000F00L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE_MASK                                                           0x0000F000L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE_MASK                                                           0x000F0000L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE_MASK                                                           0x00F00000L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE_MASK                                                          0x0F000000L
+#define DP1_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE_MASK                                                          0xF0000000L
+//DP1_DP_MSO_CNTL1
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE__SHIFT                                                       0x0
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE__SHIFT                                                       0x4
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE__SHIFT                                                       0x8
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE__SHIFT                                                       0xc
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE__SHIFT                                                       0x10
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE__SHIFT                                                       0x14
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE__SHIFT                                                       0x1c
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE_MASK                                                         0x0000000FL
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE_MASK                                                         0x000000F0L
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE_MASK                                                         0x00000F00L
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE_MASK                                                         0x0000F000L
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE_MASK                                                         0x000F0000L
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE_MASK                                                         0x00F00000L
+#define DP1_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE_MASK                                                         0xF0000000L
+//DP1_DP_STEER_FIFO_CNTL
+#define DP1_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN__SHIFT                       0x8
+#define DP1_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                        0x10
+#define DP1_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN_MASK                         0x00000100L
+#define DP1_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE_MASK                                          0x00030000L
+//DP1_DP_SEC_CNTL2
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND__SHIFT                                                             0x0
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING__SHIFT                                                     0x1
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED__SHIFT                                             0x2
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE__SHIFT                                                    0x3
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND__SHIFT                                                             0x4
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING__SHIFT                                                     0x5
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED__SHIFT                                             0x6
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE__SHIFT                                                    0x7
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND__SHIFT                                                             0x8
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING__SHIFT                                                     0x9
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED__SHIFT                                             0xa
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE__SHIFT                                                    0xb
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND__SHIFT                                                             0xc
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING__SHIFT                                                     0xd
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE__SHIFT                                                    0xf
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND__SHIFT                                                             0x10
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING__SHIFT                                                     0x11
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED__SHIFT                                             0x12
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE__SHIFT                                                    0x13
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND__SHIFT                                                             0x14
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING__SHIFT                                                     0x15
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED__SHIFT                                             0x16
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE__SHIFT                                                    0x17
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND__SHIFT                                                             0x18
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING__SHIFT                                                     0x19
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED__SHIFT                                             0x1a
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE__SHIFT                                                    0x1b
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP11_PPS__SHIFT                                                             0x1c
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_MASK                                                               0x00000001L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING_MASK                                                       0x00000002L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED_MASK                                               0x00000004L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE_MASK                                                      0x00000008L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_MASK                                                               0x00000010L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING_MASK                                                       0x00000020L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED_MASK                                               0x00000040L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE_MASK                                                      0x00000080L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_MASK                                                               0x00000100L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING_MASK                                                       0x00000200L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED_MASK                                               0x00000400L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE_MASK                                                      0x00000800L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_MASK                                                               0x00001000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING_MASK                                                       0x00002000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE_MASK                                                      0x00008000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_MASK                                                               0x00010000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING_MASK                                                       0x00020000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED_MASK                                               0x00040000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE_MASK                                                      0x00080000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_MASK                                                               0x00100000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING_MASK                                                       0x00200000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED_MASK                                               0x00400000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE_MASK                                                      0x00800000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_MASK                                                               0x01000000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING_MASK                                                       0x02000000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED_MASK                                               0x04000000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE_MASK                                                      0x08000000L
+#define DP1_DP_SEC_CNTL2__DP_SEC_GSP11_PPS_MASK                                                               0x10000000L
+//DP1_DP_SEC_CNTL3
+#define DP1_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM__SHIFT                                                         0x0
+#define DP1_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP1_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_SEC_CNTL4
+#define DP1_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM__SHIFT                                                         0x0
+#define DP1_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP1_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_SEC_CNTL5
+#define DP1_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM__SHIFT                                                         0x0
+#define DP1_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP1_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_SEC_CNTL6
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM__SHIFT                                                         0x0
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE__SHIFT                                                    0x10
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE__SHIFT                                                    0x11
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE__SHIFT                                                    0x12
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE__SHIFT                                                    0x13
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE__SHIFT                                                    0x14
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE__SHIFT                                                    0x15
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE__SHIFT                                                    0x16
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE__SHIFT                                                    0x17
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE__SHIFT                                                    0x18
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE__SHIFT                                                    0x19
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE__SHIFT                                                   0x1a
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE__SHIFT                                                   0x1b
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE_MASK                                                      0x00010000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE_MASK                                                      0x00020000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE_MASK                                                      0x00040000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE_MASK                                                      0x00080000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE_MASK                                                      0x00100000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE_MASK                                                      0x00200000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE_MASK                                                      0x00400000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE_MASK                                                      0x00800000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE_MASK                                                      0x01000000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE_MASK                                                      0x02000000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE_MASK                                                     0x04000000L
+#define DP1_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE_MASK                                                     0x08000000L
+//DP1_DP_SEC_CNTL7
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE__SHIFT                                                      0x0
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE__SHIFT                                                     0x1
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE__SHIFT                                                      0x4
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE__SHIFT                                                     0x5
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE__SHIFT                                                      0x8
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE__SHIFT                                                     0x9
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE__SHIFT                                                      0xc
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE__SHIFT                                                     0xd
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE__SHIFT                                                      0x10
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE__SHIFT                                                     0x11
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE__SHIFT                                                      0x14
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE__SHIFT                                                     0x15
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE__SHIFT                                                      0x18
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE__SHIFT                                                     0x19
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE__SHIFT                                                      0x1c
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE__SHIFT                                                     0x1d
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE_MASK                                                        0x00000001L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE_MASK                                                       0x00000002L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE_MASK                                                        0x00000010L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE_MASK                                                       0x00000020L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE_MASK                                                        0x00000100L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE_MASK                                                       0x00000200L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE_MASK                                                        0x00001000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE_MASK                                                       0x00002000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE_MASK                                                        0x00010000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE_MASK                                                       0x00020000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE_MASK                                                        0x00100000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE_MASK                                                       0x00200000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE_MASK                                                        0x01000000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE_MASK                                                       0x02000000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE_MASK                                                        0x10000000L
+#define DP1_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE_MASK                                                       0x20000000L
+//DP1_DP_DB_CNTL
+#define DP1_DP_DB_CNTL__DP_DB_PENDING__SHIFT                                                                  0x0
+#define DP1_DP_DB_CNTL__DP_DB_TAKEN__SHIFT                                                                    0x4
+#define DP1_DP_DB_CNTL__DP_DB_TAKEN_CLR__SHIFT                                                                0x5
+#define DP1_DP_DB_CNTL__DP_DB_LOCK__SHIFT                                                                     0x8
+#define DP1_DP_DB_CNTL__DP_DB_DISABLE__SHIFT                                                                  0xc
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_PENDING__SHIFT                                                          0xf
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN__SHIFT                                                            0x10
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR__SHIFT                                                        0x11
+#define DP1_DP_DB_CNTL__DP_DB_PENDING_MASK                                                                    0x00000001L
+#define DP1_DP_DB_CNTL__DP_DB_TAKEN_MASK                                                                      0x00000010L
+#define DP1_DP_DB_CNTL__DP_DB_TAKEN_CLR_MASK                                                                  0x00000020L
+#define DP1_DP_DB_CNTL__DP_DB_LOCK_MASK                                                                       0x00000100L
+#define DP1_DP_DB_CNTL__DP_DB_DISABLE_MASK                                                                    0x00001000L
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_PENDING_MASK                                                            0x00008000L
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_MASK                                                              0x00010000L
+#define DP1_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR_MASK                                                          0x00020000L
+//DP1_DP_MSA_VBID_MISC
+#define DP1_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE__SHIFT                                         0x0
+#define DP1_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                                      0x4
+#define DP1_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE__SHIFT                                                        0x8
+#define DP1_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE__SHIFT                                                        0x9
+#define DP1_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN__SHIFT                                                     0xc
+#define DP1_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN__SHIFT                                                     0xd
+#define DP1_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE__SHIFT                                                  0xf
+#define DP1_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM__SHIFT                                                        0x10
+#define DP1_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_MASK                                           0x00000003L
+#define DP1_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                                        0x00000010L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_MASK                                                          0x00000100L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_MASK                                                          0x00000200L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN_MASK                                                       0x00001000L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN_MASK                                                       0x00002000L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE_MASK                                                    0x00008000L
+#define DP1_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM_MASK                                                          0xFFFF0000L
+//DP1_DP_SEC_METADATA_TRANSMISSION
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE__SHIFT                                0x0
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE__SHIFT                        0x1
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE__SHIFT                            0x4
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE__SHIFT                                  0x10
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE_MASK                                  0x00000001L
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE_MASK                          0x00000002L
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE_MASK                              0x000000F0L
+#define DP1_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_MASK                                    0xFFFF0000L
+//DP1_DP_ALPM_CNTL
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND__SHIFT                                                         0x0
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING__SHIFT                                                      0x1
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND__SHIFT                                                       0x2
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING__SHIFT                                                    0x3
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE__SHIFT                                            0x4
+#define DP1_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO__SHIFT                                        0x5
+#define DP1_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE__SHIFT                                                  0x6
+#define DP1_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP__SHIFT                                     0x7
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM__SHIFT                                                  0x8
+#define DP1_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS__SHIFT                                   0xb
+#define DP1_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN__SHIFT                                       0xc
+#define DP1_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE__SHIFT                                                       0xf
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM__SHIFT                                             0x10
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND_MASK                                                           0x00000001L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING_MASK                                                        0x00000002L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND_MASK                                                         0x00000004L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING_MASK                                                      0x00000008L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE_MASK                                              0x00000010L
+#define DP1_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO_MASK                                          0x00000020L
+#define DP1_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE_MASK                                                    0x00000040L
+#define DP1_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_MASK                                       0x00000080L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM_MASK                                                    0x00000300L
+#define DP1_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS_MASK                                     0x00000800L
+#define DP1_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN_MASK                                         0x00001000L
+#define DP1_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE_MASK                                                         0x00008000L
+#define DP1_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM_MASK                                               0xFFFF0000L
+//DP1_DP_GSP8_CNTL
+#define DP1_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE__SHIFT                                                       0x0
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE__SHIFT                                                           0x4
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND__SHIFT                                                             0x7
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING__SHIFT                                                     0xc
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE_MASK                                                         0x0000000FL
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE_MASK                                                             0x00000010L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_MASK                                                               0x00000080L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING_MASK                                                       0x00001000L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP1_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_GSP9_CNTL
+#define DP1_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE__SHIFT                                                       0x0
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE__SHIFT                                                           0x4
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND__SHIFT                                                             0x7
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING__SHIFT                                                     0xc
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM__SHIFT                                                         0x10
+#define DP1_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE_MASK                                                         0x0000000FL
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE_MASK                                                             0x00000010L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_MASK                                                               0x00000080L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING_MASK                                                       0x00001000L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP1_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP1_DP_GSP10_CNTL
+#define DP1_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE__SHIFT                                                     0x0
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE__SHIFT                                                         0x4
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND__SHIFT                                                           0x7
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING__SHIFT                                                   0xc
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM__SHIFT                                                       0x10
+#define DP1_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE_MASK                                                       0x0000000FL
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE_MASK                                                           0x00000010L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_MASK                                                             0x00000080L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING_MASK                                                     0x00001000L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP1_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP1_DP_GSP11_CNTL
+#define DP1_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE__SHIFT                                                     0x0
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE__SHIFT                                                         0x4
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND__SHIFT                                                           0x7
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING__SHIFT                                                   0xc
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM__SHIFT                                                       0x10
+#define DP1_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE_MASK                                                       0x0000000FL
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE_MASK                                                           0x00000010L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_MASK                                                             0x00000080L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING_MASK                                                     0x00001000L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP1_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP1_DP_GSP_EN_DB_STATUS
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING__SHIFT                                             0x0
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING__SHIFT                                             0x1
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING__SHIFT                                             0x2
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING__SHIFT                                             0x3
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING__SHIFT                                             0x4
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING__SHIFT                                             0x5
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING__SHIFT                                             0x6
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING__SHIFT                                             0x7
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING__SHIFT                                             0x8
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING__SHIFT                                             0x9
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING__SHIFT                                            0xa
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING__SHIFT                                            0xb
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING_MASK                                               0x00000001L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING_MASK                                               0x00000002L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING_MASK                                               0x00000004L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING_MASK                                               0x00000008L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING_MASK                                               0x00000010L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING_MASK                                               0x00000020L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING_MASK                                               0x00000040L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING_MASK                                               0x00000080L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING_MASK                                               0x00000100L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING_MASK                                               0x00000200L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING_MASK                                              0x00000400L
+#define DP1_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING_MASK                                              0x00000800L
+//DP1_DP_AUXLESS_ALPM_CNTL1
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT__SHIFT                                              0x4
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY__SHIFT                                               0x8
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL__SHIFT                                            0x14
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE__SHIFT                                     0x1f
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT_MASK                                                0x000000F0L
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY_MASK                                                 0x0007FF00L
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL_MASK                                              0x1FF00000L
+#define DP1_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE_MASK                                       0x80000000L
+//DP1_DP_AUXLESS_ALPM_CNTL2
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME__SHIFT                                           0x0
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND__SHIFT                                                 0x7
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE__SHIFT                                            0x10
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING__SHIFT                                              0x11
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE__SHIFT                                            0x12
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING__SHIFT                                              0x13
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD__SHIFT                                          0x14
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME_MASK                                             0x0000007FL
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND_MASK                                                   0x00000080L
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE_MASK                                              0x00010000L
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING_MASK                                                0x00020000L
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE_MASK                                              0x00040000L
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING_MASK                                                0x00080000L
+#define DP1_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD_MASK                                            0x3FF00000L
+//DP1_DP_AUXLESS_ALPM_CNTL3
+#define DP1_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM__SHIFT                                             0x0
+#define DP1_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM__SHIFT                                             0x10
+#define DP1_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM_MASK                                               0x0000FFFFL
+#define DP1_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM_MASK                                               0xFFFF0000L
+//DP1_DP_AUXLESS_ALPM_CNTL4
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN__SHIFT                                                  0x1
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL__SHIFT                                   0x2
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME__SHIFT                                     0x3
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE__SHIFT                                       0x4
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS__SHIFT                                           0x5
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE__SHIFT                                               0x6
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM__SHIFT                                                   0x18
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_MASK                                                    0x00000002L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL_MASK                                     0x00000004L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME_MASK                                       0x00000008L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE_MASK                                         0x00000010L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS_MASK                                             0x00000020L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE_MASK                                                 0x00000040L
+#define DP1_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM_MASK                                                     0xFF000000L
+//DP1_DP_AUXLESS_ALPM_CNTL5
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK__SHIFT                                       0x0
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED__SHIFT                                   0x1
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                                     0x2
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR__SHIFT                                      0x3
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM__SHIFT                                  0x8
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM__SHIFT                                   0x10
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK_MASK                                         0x00000001L
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED_MASK                                     0x00000002L
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                                       0x00000004L
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR_MASK                                        0x00000008L
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM_MASK                                    0x0000FF00L
+#define DP1_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM_MASK                                     0xFFFF0000L
+//DP1_DP_STREAM_SYMBOL_COUNT_STATUS
+#define DP1_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT__SHIFT                                          0x0
+#define DP1_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT_MASK                                            0x0000FFFFL
+//DP1_DP_STREAM_SYMBOL_COUNT_CONTROL
+#define DP1_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE__SHIFT                                  0x0
+#define DP1_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET__SHIFT                                   0x4
+#define DP1_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE_MASK                                    0x00000001L
+#define DP1_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET_MASK                                     0x00000010L
+//DP1_DP_LINK_SYMBOL_COUNT_STATUS0
+#define DP1_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT__SHIFT                                             0x0
+#define DP1_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT_MASK                                               0x0000FFFFL
+//DP1_DP_LINK_SYMBOL_COUNT_STATUS1
+#define DP1_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT__SHIFT                                          0x0
+#define DP1_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT_MASK                                            0xFFFFFFFFL
+//DP1_DP_LINK_SYMBOL_COUNT_CONTROL
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE__SHIFT                                      0x0
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET__SHIFT                                       0x4
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE__SHIFT                                   0x8
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET__SHIFT                                    0xc
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE_MASK                                        0x00000001L
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET_MASK                                         0x00000010L
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE_MASK                                     0x00000100L
+#define DP1_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET_MASK                                      0x00001000L
+//DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT           0x0
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING__SHIFT  0x8
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK             0x00000001L
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+#define DP1_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING_MASK  0x00000100L
+//DP1_DP_DPHY_FAST_TRAINING
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT                                       0x0
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT                                         0x1
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT                            0x2
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING__SHIFT                              0x4
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT                                         0x8
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT                                         0x14
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK                                         0x00000001L
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK                                           0x00000002L
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK                              0x00000004L
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING_MASK                                0x00000010L
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK                                           0x000FFF00L
+#define DP1_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK                                           0xFFF00000L
+//DP1_DP_DPHY_FAST_TRAINING_STATUS
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT                                     0x0
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT                         0x4
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT                             0x8
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT                              0xc
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK                                       0x00000007L
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK                           0x00000010L
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK                               0x00000100L
+#define DP1_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK                                0x00001000L
+
+
+// addressBlock: dce_dc_dio_dig2_vpg_vpg_dispdec
+//VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG2_VPG_GENERIC_PACKET_DATA
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG2_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG2_VPG_GENERIC_STATUS
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG2_VPG_MEM_PWR
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG2_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG2_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG2_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG2_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG2_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG2_VPG_ISRC1_2_DATA
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_dio_dig2_apg_apg_dispdec
+//APG2_APG_CONTROL
+#define APG2_APG_CONTROL__APG_RESET__SHIFT                                                                    0x1
+#define APG2_APG_CONTROL__APG_RESET_DONE__SHIFT                                                               0x2
+#define APG2_APG_CONTROL__APG_RESET_MASK                                                                      0x00000002L
+#define APG2_APG_CONTROL__APG_RESET_DONE_MASK                                                                 0x00000004L
+//APG2_APG_CONTROL2
+#define APG2_APG_CONTROL2__APG_ENABLE__SHIFT                                                                  0x0
+#define APG2_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID__SHIFT                                                      0x8
+#define APG2_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE__SHIFT                                           0x18
+#define APG2_APG_CONTROL2__APG_ENABLE_MASK                                                                    0x00000001L
+#define APG2_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID_MASK                                                        0x0000FF00L
+#define APG2_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE_MASK                                             0x01000000L
+//APG2_APG_DBG_GEN_CONTROL
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG2_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG2_APG_PACKET_CONTROL
+#define APG2_APG_PACKET_CONTROL__APG_DBG_MUX_SEL__SHIFT                                                       0x0
+#define APG2_APG_PACKET_CONTROL__APG_ACP_SOURCE__SHIFT                                                        0x1
+#define APG2_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE__SHIFT                                                 0x2
+#define APG2_APG_PACKET_CONTROL__APG_DBG_MUX_SEL_MASK                                                         0x00000001L
+#define APG2_APG_PACKET_CONTROL__APG_ACP_SOURCE_MASK                                                          0x00000002L
+#define APG2_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE_MASK                                                   0x00000004L
+//APG2_APG_DBG_ACP
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE__SHIFT                                                             0x0
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                             0x8
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                             0x10
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_REQUIRED__SHIFT                                                         0x18
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE_MASK                                                               0x00000003L
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0_MASK                                               0x0000FF00L
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1_MASK                                               0x00FF0000L
+#define APG2_APG_DBG_ACP__APG_DBG_ACP_REQUIRED_MASK                                                           0x01000000L
+//APG2_APG_DBG_AUDIO_INFO
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA__SHIFT                                                 0x0
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC__SHIFT                                                 0x8
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV__SHIFT                                                0xb
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH__SHIFT                                             0xf
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL__SHIFT                                             0x10
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA_MASK                                                   0x000000FFL
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC_MASK                                                   0x00000700L
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV_MASK                                                  0x00007800L
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH_MASK                                               0x00008000L
+#define APG2_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL_MASK                                               0x00030000L
+//APG2_APG_DBG_60958_0
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_A__SHIFT                                                       0x0
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_B__SHIFT                                                       0x1
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_C__SHIFT                                                       0x2
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_D__SHIFT                                                       0x3
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_MODE__SHIFT                                                    0x6
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE__SHIFT                                           0x8
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER__SHIFT                                           0x10
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L__SHIFT                                        0x14
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY__SHIFT                                      0x18
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY__SHIFT                                          0x1c
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_A_MASK                                                         0x00000001L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_B_MASK                                                         0x00000002L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_C_MASK                                                         0x00000004L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_D_MASK                                                         0x00000038L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_MODE_MASK                                                      0x000000C0L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE_MASK                                             0x0000FF00L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER_MASK                                             0x000F0000L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L_MASK                                          0x00F00000L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY_MASK                                        0x0F000000L
+#define APG2_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY_MASK                                            0x30000000L
+//APG2_APG_DBG_60958_1
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH__SHIFT                                             0x0
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                             0x4
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_VALID__SHIFT                                                      0x10
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R__SHIFT                                        0x14
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH_MASK                                               0x0000000FL
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                               0x000000F0L
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_VALID_MASK                                                        0x00010000L
+#define APG2_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R_MASK                                          0x00F00000L
+//APG2_APG_DBG_60958_2
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2__SHIFT                                        0x0
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3__SHIFT                                        0x4
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4__SHIFT                                        0x8
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5__SHIFT                                        0xc
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6__SHIFT                                        0x10
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7__SHIFT                                        0x14
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2_MASK                                          0x0000000FL
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3_MASK                                          0x000000F0L
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4_MASK                                          0x00000F00L
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5_MASK                                          0x0000F000L
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6_MASK                                          0x000F0000L
+#define APG2_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7_MASK                                          0x00F00000L
+//APG2_APG_AUDIO_CRC_CONTROL
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN__SHIFT                                                   0x0
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT__SHIFT                                                 0x4
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL__SHIFT                                               0xd
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT__SHIFT                                                0x10
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN_MASK                                                     0x00000001L
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT_MASK                                                   0x00000010L
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL_MASK                                                 0x0000E000L
+#define APG2_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT_MASK                                                  0xFFFF0000L
+//APG2_APG_AUDIO_CRC_CONTROL2
+#define APG2_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT__SHIFT                                 0x0
+#define APG2_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT_MASK                                   0x0000FFFFL
+//APG2_APG_AUDIO_CRC_RESULT
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE__SHIFT                                                  0x0
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR__SHIFT                                            0x8
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC__SHIFT                                                       0x10
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_MASK                                                    0x00000001L
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR_MASK                                              0x00000100L
+#define APG2_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_MASK                                                         0xFFFF0000L
+//APG2_APG_DBG_RAMP_CONTROL0
+#define APG2_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT__SHIFT                                             0x0
+#define APG2_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN__SHIFT                                             0x1f
+#define APG2_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT_MASK                                               0x00FFFFFFL
+#define APG2_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN_MASK                                               0x80000000L
+//APG2_APG_DBG_RAMP_CONTROL1
+#define APG2_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT__SHIFT                                             0x0
+#define APG2_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT_MASK                                               0x00FFFFFFL
+//APG2_APG_DBG_RAMP_CONTROL2
+#define APG2_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT__SHIFT                                             0x0
+#define APG2_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT_MASK                                               0x00FFFFFFL
+//APG2_APG_DBG_RAMP_CONTROL3
+#define APG2_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT__SHIFT                                             0x0
+#define APG2_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT_MASK                                               0x00FFFFFFL
+//APG2_APG_STATUS
+#define APG2_APG_STATUS__APG_AUDIO_ENABLE__SHIFT                                                              0x4
+#define APG2_APG_STATUS__APG_HBR_ENABLE__SHIFT                                                                0x8
+#define APG2_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS__SHIFT                                                0x18
+#define APG2_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR__SHIFT                                          0x19
+#define APG2_APG_STATUS__APG_AUDIO_ENABLE_MASK                                                                0x00000010L
+#define APG2_APG_STATUS__APG_HBR_ENABLE_MASK                                                                  0x00000100L
+#define APG2_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_MASK                                                  0x01000000L
+#define APG2_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR_MASK                                            0x02000000L
+//APG2_APG_STATUS2
+#define APG2_APG_STATUS2__APG_OUTPUT_ACTIVE__SHIFT                                                            0x0
+#define APG2_APG_STATUS2__APG_OUTPUT_ACTIVE_MASK                                                              0x00000001L
+//APG2_APG_DBG_AUDIO_DTO_CNTL
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE__SHIFT                                            0x8
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI__SHIFT                                           0xc
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV__SHIFT                                             0x10
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE_MASK                                              0x00000100L
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI_MASK                                             0x00007000L
+#define APG2_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV_MASK                                               0x00070000L
+//APG2_APG_MEM_PWR
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG2_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG2_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG2_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+//APG2_APG_SPARE
+#define APG2_APG_SPARE__APG_SPARE__SHIFT                                                                      0x0
+#define APG2_APG_SPARE__APG_SPARE_MASK                                                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dig2_dme_dme_dispdec
+//DME2_DME_CONTROL
+#define DME2_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME2_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME2_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME2_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME2_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME2_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME2_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME2_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME2_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME2_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME2_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME2_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME2_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME2_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME2_DME_MEMORY_CONTROL
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig2_dispdec
+//DIG2_DIG_FE_CNTL
+#define DIG2_DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT                                                            0x0
+#define DIG2_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT                                                        0x4
+#define DIG2_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT                                                       0x8
+#define DIG2_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT__SHIFT                                                    0xc
+#define DIG2_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN__SHIFT                                                        0x14
+#define DIG2_DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK                                                              0x00000007L
+#define DIG2_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK                                                          0x00000070L
+#define DIG2_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK                                                         0x00000100L
+#define DIG2_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT_MASK                                                      0x00007000L
+#define DIG2_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN_MASK                                                          0x00100000L
+//DIG2_DIG_FE_CLK_CNTL
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_MODE__SHIFT                                                              0x0
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN__SHIFT                                                            0x4
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON__SHIFT                                                0xa
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON__SHIFT                                              0xb
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON__SHIFT                                          0xc
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON__SHIFT                                         0xd
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON__SHIFT                                             0xe
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_MODE_MASK                                                                0x00000007L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN_MASK                                                              0x00000010L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON_MASK                                                  0x00000400L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON_MASK                                                0x00000800L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON_MASK                                            0x00001000L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON_MASK                                           0x00002000L
+#define DIG2_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON_MASK                                               0x00004000L
+//DIG2_DIG_FE_EN_CNTL
+#define DIG2_DIG_FE_EN_CNTL__DIG_FE_ENABLE__SHIFT                                                             0x0
+#define DIG2_DIG_FE_EN_CNTL__DIG_FE_ENABLE_MASK                                                               0x00000001L
+//DIG2_DIG_OUTPUT_CRC_CNTL
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT                                                    0x0
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT                                              0x4
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT                                              0x8
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK                                                      0x00000001L
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK                                                0x00000010L
+#define DIG2_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK                                                0x00000300L
+//DIG2_DIG_OUTPUT_CRC_RESULT
+#define DIG2_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT                                              0x0
+#define DIG2_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK                                                0x3FFFFFFFL
+//DIG2_DIG_CLOCK_PATTERN
+#define DIG2_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT                                                      0x0
+#define DIG2_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK                                                        0x000003FFL
+//DIG2_DIG_TEST_PATTERN
+#define DIG2_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT                                                 0x0
+#define DIG2_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT                                              0x1
+#define DIG2_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT                                               0x4
+#define DIG2_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT                                                0x5
+#define DIG2_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT                                      0x6
+#define DIG2_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT                                                 0x10
+#define DIG2_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK                                                   0x00000001L
+#define DIG2_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK                                                0x00000002L
+#define DIG2_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK                                                 0x00000010L
+#define DIG2_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK                                                  0x00000020L
+#define DIG2_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK                                        0x00000040L
+#define DIG2_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK                                                   0x03FF0000L
+//DIG2_DIG_RANDOM_PATTERN_SEED
+#define DIG2_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT                                          0x0
+#define DIG2_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT                                       0x18
+#define DIG2_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK                                            0x00FFFFFFL
+#define DIG2_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK                                         0x01000000L
+//DIG2_DIG_FIFO_CTRL0
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE__SHIFT                                                           0x0
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_RESET__SHIFT                                                            0x1
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL__SHIFT                                                 0x2
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC__SHIFT                                                   0x8
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                           0x9
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE__SHIFT                                                       0x14
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_ERROR__SHIFT                                                            0x1c
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE_MASK                                                             0x00000001L
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_RESET_MASK                                                              0x00000002L
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL_MASK                                                   0x000000FCL
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC_MASK                                                     0x00000100L
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE_MASK                                             0x00000600L
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE_MASK                                                         0x00100000L
+#define DIG2_DIG_FIFO_CTRL0__DIG_FIFO_ERROR_MASK                                                              0x30000000L
+//DIG2_DIG_FIFO_CTRL1
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT                                              0x1
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL__SHIFT                                                  0x2
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY__SHIFT                                                  0x8
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT                                                0xa
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL__SHIFT                                                    0x10
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL__SHIFT                                                    0x16
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED__SHIFT                                                       0x1d
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT                                              0x1e
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT                                              0x1f
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK                                                0x00000002L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL_MASK                                                    0x000000FCL
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY_MASK                                                    0x00000300L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK                                                  0x0000FC00L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL_MASK                                                      0x001F0000L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL_MASK                                                      0x03C00000L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED_MASK                                                         0x20000000L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK                                                0x40000000L
+#define DIG2_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK                                                0x80000000L
+//DIG2_HDMI_METADATA_PACKET_CONTROL
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                                 0x0
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                         0x4
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                                 0x8
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                   0x10
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                   0x00000001L
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                           0x00000010L
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                   0x00000100L
+#define DIG2_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                     0xFFFF0000L
+//DIG2_HDMI_CONTROL
+#define DIG2_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT                                                       0x1
+#define DIG2_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT                                                     0x2
+#define DIG2_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED__SHIFT                                            0x3
+#define DIG2_HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT                                                              0x8
+#define DIG2_HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT                                                             0x9
+#define DIG2_HDMI_CONTROL__DOLBY_VISION_EN__SHIFT                                                             0xa
+#define DIG2_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED__SHIFT                                         0xb
+#define DIG2_HDMI_CONTROL__TMDS_PIXEL_ENCODING__SHIFT                                                         0xc
+#define DIG2_HDMI_CONTROL__TMDS_COLOR_FORMAT__SHIFT                                                           0xd
+#define DIG2_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM__SHIFT                                           0x10
+#define DIG2_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT                                                      0x18
+#define DIG2_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                       0x1c
+#define DIG2_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK                                                         0x00000002L
+#define DIG2_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK                                                       0x00000004L
+#define DIG2_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK                                              0x00000008L
+#define DIG2_HDMI_CONTROL__HDMI_ERROR_ACK_MASK                                                                0x00000100L
+#define DIG2_HDMI_CONTROL__HDMI_ERROR_MASK_MASK                                                               0x00000200L
+#define DIG2_HDMI_CONTROL__DOLBY_VISION_EN_MASK                                                               0x00000400L
+#define DIG2_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED_MASK                                           0x00000800L
+#define DIG2_HDMI_CONTROL__TMDS_PIXEL_ENCODING_MASK                                                           0x00001000L
+#define DIG2_HDMI_CONTROL__TMDS_COLOR_FORMAT_MASK                                                             0x00006000L
+#define DIG2_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM_MASK                                             0x003F0000L
+#define DIG2_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK                                                        0x01000000L
+#define DIG2_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK                                                         0x30000000L
+//DIG2_HDMI_STATUS
+#define DIG2_HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT                                                           0x0
+#define DIG2_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT                                                      0x10
+#define DIG2_HDMI_STATUS__HDMI_ERROR_INT__SHIFT                                                               0x1b
+#define DIG2_HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK                                                             0x00000001L
+#define DIG2_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK                                                        0x00010000L
+#define DIG2_HDMI_STATUS__HDMI_ERROR_INT_MASK                                                                 0x08000000L
+//DIG2_HDMI_AUDIO_PACKET_CONTROL
+#define DIG2_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT                                            0x4
+#define DIG2_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK                                              0x00000030L
+//DIG2_HDMI_ACR_PACKET_CONTROL
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                    0x0
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                    0x1
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                  0x4
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                  0x8
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                               0xc
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                              0x10
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                      0x00000001L
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                      0x00000002L
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                    0x00000030L
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                    0x00000100L
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                                 0x00001000L
+#define DIG2_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                                0x00070000L
+//DIG2_HDMI_VBI_PACKET_CONTROL
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT                                                   0x0
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT                                                     0x4
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT                                                     0x5
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT                                                   0x8
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT                                                   0x9
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND__SHIFT                                                    0xc
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT                                                   0x10
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE__SHIFT                                                    0x18
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK                                                     0x00000001L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK                                                       0x00000010L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK                                                       0x00000020L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK                                                     0x00000100L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK                                                     0x00000200L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND_MASK                                                      0x00001000L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK                                                     0x003F0000L
+#define DIG2_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE_MASK                                                      0x3F000000L
+//DIG2_HDMI_INFOFRAME_CONTROL0
+#define DIG2_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT                                             0x4
+#define DIG2_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE__SHIFT                                             0x8
+#define DIG2_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK                                               0x00000010L
+#define DIG2_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE_MASK                                               0x00003F00L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                          0x1
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                                0x2
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                          0x4
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                          0x5
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                                0x6
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                          0x8
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                          0x9
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                                0xa
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE__SHIFT                           0xb
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                          0xc
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                          0xd
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                                0xe
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE__SHIFT                           0xf
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                          0x11
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                                0x12
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE__SHIFT                           0x13
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                          0x14
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                          0x15
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                                0x16
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE__SHIFT                           0x17
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                          0x18
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                          0x19
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                                0x1a
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE__SHIFT                           0x1b
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                          0x1c
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                          0x1d
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                                0x1e
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE__SHIFT                           0x1f
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                            0x00000001L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                            0x00000002L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                            0x00000010L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                            0x00000020L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                            0x00000100L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                            0x00000200L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                  0x00000400L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE_MASK                             0x00000800L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                            0x00001000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                            0x00002000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                  0x00004000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE_MASK                             0x00008000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                            0x00010000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                            0x00020000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                  0x00040000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE_MASK                             0x00080000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                            0x00100000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                            0x00200000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                  0x00400000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE_MASK                             0x00800000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                            0x01000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                            0x02000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                  0x04000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE_MASK                             0x08000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                            0x10000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                            0x20000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                  0x40000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE_MASK                             0x80000000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL6
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT__SHIFT                                          0x1
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                                0x2
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND__SHIFT                                          0x4
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT__SHIFT                                          0x5
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                                0x6
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND__SHIFT                                         0x8
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT__SHIFT                                         0x9
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                               0xa
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE__SHIFT                          0xb
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND__SHIFT                                         0xc
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT__SHIFT                                         0xd
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                               0xe
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE__SHIFT                          0xf
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND__SHIFT                                         0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT__SHIFT                                         0x11
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                               0x12
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE__SHIFT                          0x13
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND__SHIFT                                         0x14
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT__SHIFT                                         0x15
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                               0x16
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE__SHIFT                          0x17
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND__SHIFT                                         0x18
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT__SHIFT                                         0x19
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                               0x1a
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE__SHIFT                          0x1b
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND_MASK                                            0x00000001L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT_MASK                                            0x00000002L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND_MASK                                            0x00000010L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT_MASK                                            0x00000020L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND_MASK                                           0x00000100L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT_MASK                                           0x00000200L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE_MASK                                 0x00000400L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE_MASK                            0x00000800L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND_MASK                                           0x00001000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT_MASK                                           0x00002000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE_MASK                                 0x00004000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE_MASK                            0x00008000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND_MASK                                           0x00010000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT_MASK                                           0x00020000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE_MASK                                 0x00040000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE_MASK                            0x00080000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND_MASK                                           0x00100000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT_MASK                                           0x00200000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE_MASK                                 0x00400000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE_MASK                            0x00800000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND_MASK                                           0x01000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT_MASK                                           0x02000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE_MASK                                 0x04000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE_MASK                            0x08000000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL5
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                                0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                        0x1
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                                0x2
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                        0x3
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                                0x4
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                        0x5
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                                0x6
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                        0x7
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                                0x8
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                        0x9
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                                0xa
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                        0xb
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                                0xc
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                        0xd
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                                0xe
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                        0xf
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                                0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                        0x11
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                                0x12
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                        0x13
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                               0x14
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                       0x15
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                               0x16
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                       0x17
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                               0x18
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                       0x19
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                               0x1a
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                       0x1b
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                               0x1c
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                       0x1d
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                  0x00000001L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                          0x00000002L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                  0x00000004L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                          0x00000008L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                  0x00000010L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                          0x00000020L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                  0x00000040L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                          0x00000080L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                  0x00000100L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                          0x00000200L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                  0x00000400L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                          0x00000800L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                  0x00001000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                          0x00002000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                  0x00004000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                          0x00008000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                  0x00010000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                          0x00020000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                  0x00040000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                          0x00080000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                                 0x00100000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                         0x00200000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                                 0x00400000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                         0x00800000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                                 0x01000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                         0x02000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                                 0x04000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                         0x08000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                                 0x10000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                         0x20000000L
+//DIG2_HDMI_GC
+#define DIG2_HDMI_GC__HDMI_GC_AVMUTE__SHIFT                                                                   0x0
+#define DIG2_HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT                                                              0x2
+#define DIG2_HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT                                                               0x4
+#define DIG2_HDMI_GC__HDMI_PACKING_PHASE__SHIFT                                                               0x8
+#define DIG2_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT                                                      0xc
+#define DIG2_HDMI_GC__HDMI_GC_AVMUTE_MASK                                                                     0x00000001L
+#define DIG2_HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK                                                                0x00000004L
+#define DIG2_HDMI_GC__HDMI_DEFAULT_PHASE_MASK                                                                 0x00000010L
+#define DIG2_HDMI_GC__HDMI_PACKING_PHASE_MASK                                                                 0x00000F00L
+#define DIG2_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK                                                        0x00001000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL1
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE_MASK                                            0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE_MASK                                            0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL2
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE_MASK                                            0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE_MASK                                            0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL3
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE_MASK                                            0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE_MASK                                            0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL4
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE_MASK                                            0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE_MASK                                            0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL7
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE__SHIFT                                          0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE__SHIFT                                          0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE_MASK                                            0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE_MASK                                            0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL8
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE__SHIFT                                         0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE__SHIFT                                         0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE_MASK                                           0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE_MASK                                           0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL9
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE__SHIFT                                         0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE__SHIFT                                         0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE_MASK                                           0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE_MASK                                           0xFFFF0000L
+//DIG2_HDMI_GENERIC_PACKET_CONTROL10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE__SHIFT                                        0x0
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING__SHIFT                                0x10
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING__SHIFT                                0x11
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING__SHIFT                                0x12
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING__SHIFT                                0x13
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING__SHIFT                                0x14
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING__SHIFT                                0x15
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING__SHIFT                                0x16
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING__SHIFT                                0x17
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING__SHIFT                                0x18
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING__SHIFT                                0x19
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING__SHIFT                               0x1a
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING__SHIFT                               0x1b
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING__SHIFT                               0x1c
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING__SHIFT                               0x1d
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING__SHIFT                               0x1e
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE_MASK                                          0x0000FFFFL
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING_MASK                                  0x00010000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING_MASK                                  0x00020000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING_MASK                                  0x00040000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING_MASK                                  0x00080000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING_MASK                                  0x00100000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING_MASK                                  0x00200000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING_MASK                                  0x00400000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING_MASK                                  0x00800000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING_MASK                                  0x01000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING_MASK                                  0x02000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING_MASK                                 0x04000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING_MASK                                 0x08000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING_MASK                                 0x10000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING_MASK                                 0x20000000L
+#define DIG2_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING_MASK                                 0x40000000L
+//DIG2_HDMI_DB_CONTROL
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                          0x0
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_TAKEN__SHIFT                                                            0x4
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR__SHIFT                                                        0x5
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_LOCK__SHIFT                                                             0x8
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                          0xc
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                       0xf
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN__SHIFT                                                         0x10
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR__SHIFT                                                     0x11
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_PENDING_MASK                                                            0x00000001L
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_TAKEN_MASK                                                              0x00000010L
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR_MASK                                                          0x00000020L
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_LOCK_MASK                                                               0x00000100L
+#define DIG2_HDMI_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                            0x00001000L
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                         0x00008000L
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_MASK                                                           0x00010000L
+#define DIG2_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR_MASK                                                       0x00020000L
+//DIG2_HDMI_ACR_32_0
+#define DIG2_HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                            0xc
+#define DIG2_HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                              0xFFFFF000L
+//DIG2_HDMI_ACR_32_1
+#define DIG2_HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                              0x0
+#define DIG2_HDMI_ACR_32_1__HDMI_ACR_N_32_MASK                                                                0x000FFFFFL
+//DIG2_HDMI_ACR_44_0
+#define DIG2_HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                            0xc
+#define DIG2_HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                              0xFFFFF000L
+//DIG2_HDMI_ACR_44_1
+#define DIG2_HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                              0x0
+#define DIG2_HDMI_ACR_44_1__HDMI_ACR_N_44_MASK                                                                0x000FFFFFL
+//DIG2_HDMI_ACR_48_0
+#define DIG2_HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                            0xc
+#define DIG2_HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                              0xFFFFF000L
+//DIG2_HDMI_ACR_48_1
+#define DIG2_HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                              0x0
+#define DIG2_HDMI_ACR_48_1__HDMI_ACR_N_48_MASK                                                                0x000FFFFFL
+//DIG2_HDMI_ACR_STATUS_0
+#define DIG2_HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                           0xc
+#define DIG2_HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                             0xFFFFF000L
+//DIG2_HDMI_ACR_STATUS_1
+#define DIG2_HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                             0x0
+#define DIG2_HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK                                                               0x000FFFFFL
+//DIG2_DIG_FE_AUDIO_CNTL
+#define DIG2_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT                               0x0
+#define DIG2_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE__SHIFT                                                       0x8
+#define DIG2_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                                 0x00000007L
+#define DIG2_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE_MASK                                                         0x00000100L
+//DIG2_DIG_BE_CLK_CNTL
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_MODE__SHIFT                                                              0x0
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN__SHIFT                                                            0x4
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON__SHIFT                                                 0xb
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON__SHIFT                                            0xd
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_MODE_MASK                                                                0x00000007L
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN_MASK                                                              0x00000010L
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON_MASK                                                   0x00000800L
+#define DIG2_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON_MASK                                              0x00002000L
+//DIG2_DIG_BE_CNTL
+#define DIG2_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT                                                         0x0
+#define DIG2_DIG_BE_CNTL__DIG_SWAP__SHIFT                                                                     0x1
+#define DIG2_DIG_BE_CNTL__DIG_RB_SWITCH_EN__SHIFT                                                             0x2
+#define DIG2_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT                                                         0x8
+#define DIG2_DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT                                                               0x1c
+#define DIG2_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE_MASK                                                           0x00000001L
+#define DIG2_DIG_BE_CNTL__DIG_SWAP_MASK                                                                       0x00000002L
+#define DIG2_DIG_BE_CNTL__DIG_RB_SWITCH_EN_MASK                                                               0x00000004L
+#define DIG2_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK                                                           0x00007F00L
+#define DIG2_DIG_BE_CNTL__DIG_HPD_SELECT_MASK                                                                 0x70000000L
+//DIG2_DIG_BE_EN_CNTL
+#define DIG2_DIG_BE_EN_CNTL__DIG_BE_ENABLE__SHIFT                                                             0x0
+#define DIG2_DIG_BE_EN_CNTL__DIG_BE_ENABLE_MASK                                                               0x00000001L
+//DIG2_TMDS_CNTL
+#define DIG2_TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT                                                                0x0
+#define DIG2_TMDS_CNTL__TMDS_SYNC_PHASE_MASK                                                                  0x00000001L
+//DIG2_TMDS_CONTROL_CHAR
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT                                              0x0
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT                                              0x1
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT                                              0x2
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT                                              0x3
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK                                                0x00000001L
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK                                                0x00000002L
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK                                                0x00000004L
+#define DIG2_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK                                                0x00000008L
+//DIG2_TMDS_CONTROL0_FEEDBACK
+#define DIG2_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT                                     0x0
+#define DIG2_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT                                      0x8
+#define DIG2_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK                                       0x00000003L
+#define DIG2_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK                                        0x00000300L
+//DIG2_TMDS_STEREOSYNC_CTL_SEL
+#define DIG2_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT                                          0x0
+#define DIG2_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK                                            0x00000003L
+//DIG2_TMDS_SYNC_CHAR_PATTERN_0_1
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT                                       0x0
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT                                       0x10
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK                                         0x000003FFL
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK                                         0x03FF0000L
+//DIG2_TMDS_SYNC_CHAR_PATTERN_2_3
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT                                       0x0
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT                                       0x10
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK                                         0x000003FFL
+#define DIG2_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK                                         0x03FF0000L
+//DIG2_TMDS_CTL_BITS
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL0__SHIFT                                                                  0x0
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL1__SHIFT                                                                  0x8
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL2__SHIFT                                                                  0x10
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL3__SHIFT                                                                  0x18
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL0_MASK                                                                    0x00000001L
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL1_MASK                                                                    0x00000100L
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL2_MASK                                                                    0x00010000L
+#define DIG2_TMDS_CTL_BITS__TMDS_CTL3_MASK                                                                    0x01000000L
+//DIG2_TMDS_DCBALANCER_CONTROL
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT                                               0x0
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT                                               0x4
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT                                          0x8
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT                                          0x10
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT                                            0x18
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK                                                 0x00000001L
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK                                                 0x00000070L
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK                                            0x00000100L
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK                                            0x000F0000L
+#define DIG2_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK                                              0x01000000L
+//DIG2_TMDS_SYNC_DCBALANCE_CHAR
+#define DIG2_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01__SHIFT                                          0x0
+#define DIG2_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11__SHIFT                                          0x10
+#define DIG2_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01_MASK                                            0x000003FFL
+#define DIG2_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11_MASK                                            0x03FF0000L
+//DIG2_TMDS_CTL0_1_GEN_CNTL
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT                                                  0x0
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT                                                0x4
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT                                               0x7
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT                                           0x8
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT                                                  0x10
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT                                                0x14
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT                                               0x17
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT                                           0x18
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT                                                0x1f
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK                                              0x10000000L
+#define DIG2_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK                                                  0x80000000L
+//DIG2_TMDS_CTL2_3_GEN_CNTL
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT                                                  0x0
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT                                                0x4
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT                                               0x7
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT                                           0x8
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT                                                  0x10
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT                                                0x14
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT                                               0x17
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT                                           0x18
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG2_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK                                              0x10000000L
+//DIG2_DIG_VERSION
+#define DIG2_DIG_VERSION__DIG_TYPE__SHIFT                                                                     0x0
+#define DIG2_DIG_VERSION__DIG_TYPE_MASK                                                                       0x00000001L
+
+
+// addressBlock: dce_dc_dio_dp2_dispdec
+//DP2_DP_LINK_CNTL
+#define DP2_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT                                                    0x4
+#define DP2_DP_LINK_CNTL__DP_LINK_STATUS__SHIFT                                                               0x8
+#define DP2_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK                                                      0x00000010L
+#define DP2_DP_LINK_CNTL__DP_LINK_STATUS_MASK                                                                 0x00000100L
+//DP2_DP_PIXEL_FORMAT
+#define DP2_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                                                       0x0
+#define DP2_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT__SHIFT                                                 0x4
+#define DP2_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                                              0x8
+#define DP2_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT__SHIFT                                                   0x10
+#define DP2_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                                         0x00000001L
+#define DP2_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT_MASK                                                   0x00000030L
+#define DP2_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                                                0x00000700L
+#define DP2_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT_MASK                                                     0x00010000L
+//DP2_DP_MSA_COLORIMETRY
+#define DP2_DP_MSA_COLORIMETRY__DP_MSA_MISC0__SHIFT                                                           0x18
+#define DP2_DP_MSA_COLORIMETRY__DP_MSA_MISC0_MASK                                                             0xFF000000L
+//DP2_DP_CONFIG
+#define DP2_DP_CONFIG__DP_UDI_LANES__SHIFT                                                                    0x0
+#define DP2_DP_CONFIG__DP_UDI_LANES_MASK                                                                      0x00000003L
+//DP2_DP_VID_STREAM_CNTL
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT                                                   0x0
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT                                                0x8
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT                                                   0x10
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT                                           0x14
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK                                                     0x00000001L
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK                                                  0x00000300L
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK                                                     0x00010000L
+#define DP2_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK                                             0x00100000L
+//DP2_DP_STEER_FIFO
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE__SHIFT                                                        0x0
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT                                                         0x1
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE__SHIFT                                                    0x2
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT                                                      0x4
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT                                                       0x5
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT                                                       0x6
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT                                                      0x7
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE_MASK                                                          0x00000001L
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK                                                           0x00000002L
+#define DP2_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE_MASK                                                      0x00000004L
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK                                                        0x00000010L
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK                                                         0x00000020L
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK                                                         0x00000040L
+#define DP2_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK                                                        0x00000080L
+//DP2_DP_MSA_MISC
+#define DP2_DP_MSA_MISC__DP_MSA_MISC1__SHIFT                                                                  0x0
+#define DP2_DP_MSA_MISC__DP_MSA_MISC2__SHIFT                                                                  0x8
+#define DP2_DP_MSA_MISC__DP_MSA_MISC3__SHIFT                                                                  0x10
+#define DP2_DP_MSA_MISC__DP_MSA_MISC4__SHIFT                                                                  0x18
+#define DP2_DP_MSA_MISC__DP_MSA_MISC1_MASK                                                                    0x000000FFL
+#define DP2_DP_MSA_MISC__DP_MSA_MISC2_MASK                                                                    0x0000FF00L
+#define DP2_DP_MSA_MISC__DP_MSA_MISC3_MASK                                                                    0x00FF0000L
+#define DP2_DP_MSA_MISC__DP_MSA_MISC4_MASK                                                                    0xFF000000L
+//DP2_DP_DPHY_INTERNAL_CTRL
+#define DP2_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN__SHIFT                                         0x0
+#define DP2_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL__SHIFT                                        0x4
+#define DP2_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN_MASK                                           0x00000001L
+#define DP2_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL_MASK                                          0x00000010L
+//DP2_DP_VID_TIMING
+#define DP2_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE__SHIFT                                               0x4
+#define DP2_DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT                                                           0x8
+#define DP2_DP_VID_TIMING__DP_VID_M_DIV__SHIFT                                                                0xc
+#define DP2_DP_VID_TIMING__DP_VID_N_INTERVAL__SHIFT                                                           0xe
+#define DP2_DP_VID_TIMING__DP_VID_N_DIV__SHIFT                                                                0x18
+#define DP2_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE_MASK                                                 0x00000010L
+#define DP2_DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK                                                             0x00000100L
+#define DP2_DP_VID_TIMING__DP_VID_M_DIV_MASK                                                                  0x00003000L
+#define DP2_DP_VID_TIMING__DP_VID_N_INTERVAL_MASK                                                             0x0000C000L
+#define DP2_DP_VID_TIMING__DP_VID_N_DIV_MASK                                                                  0xFF000000L
+//DP2_DP_VID_N
+#define DP2_DP_VID_N__DP_VID_N__SHIFT                                                                         0x0
+#define DP2_DP_VID_N__DP_VID_N_MASK                                                                           0x00FFFFFFL
+//DP2_DP_VID_M
+#define DP2_DP_VID_M__DP_VID_M__SHIFT                                                                         0x0
+#define DP2_DP_VID_M__DP_VID_M_MASK                                                                           0x00FFFFFFL
+//DP2_DP_LINK_FRAMING_CNTL
+#define DP2_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT                                                  0x0
+#define DP2_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE__SHIFT                                  0x14
+#define DP2_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT                                                      0x18
+#define DP2_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT                                           0x1c
+#define DP2_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK                                                    0x0003FFFFL
+#define DP2_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE_MASK                                    0x00100000L
+#define DP2_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK                                                        0x01000000L
+#define DP2_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK                                             0x10000000L
+//DP2_DP_HBR2_EYE_PATTERN
+#define DP2_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT                                            0x0
+#define DP2_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK                                              0x00000001L
+//DP2_DP_VID_MSA_VBID
+#define DP2_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT                                                       0x0
+#define DP2_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE__SHIFT                                            0xc
+#define DP2_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT                                                     0x18
+#define DP2_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK                                                         0x00000FFFL
+#define DP2_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE_MASK                                              0x00001000L
+#define DP2_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK                                                       0x01000000L
+//DP2_DP_VID_INTERRUPT_CNTL
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT                                           0x0
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT                                           0x1
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT                                          0x2
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK                                             0x00000001L
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK                                             0x00000002L
+#define DP2_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK                                            0x00000004L
+//DP2_DP_DPHY_CNTL
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT                                                         0x0
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT                                                         0x1
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT                                                         0x2
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT                                                         0x3
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_EN__SHIFT                                                                  0x4
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW__SHIFT                                                        0x5
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS__SHIFT                                                       0x6
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM__SHIFT                                               0x7
+#define DP2_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL__SHIFT                                                           0x8
+#define DP2_DP_DPHY_CNTL__DPHY_BYPASS__SHIFT                                                                  0x10
+#define DP2_DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT                                                             0x18
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK                                                           0x00000001L
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK                                                           0x00000002L
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK                                                           0x00000004L
+#define DP2_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK                                                           0x00000008L
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_EN_MASK                                                                    0x00000010L
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW_MASK                                                          0x00000020L
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS_MASK                                                         0x00000040L
+#define DP2_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM_MASK                                                 0x00000080L
+#define DP2_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL_MASK                                                             0x00000100L
+#define DP2_DP_DPHY_CNTL__DPHY_BYPASS_MASK                                                                    0x00010000L
+#define DP2_DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK                                                               0x01000000L
+//DP2_DP_DPHY_TRAINING_PATTERN_SEL
+#define DP2_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT                                    0x0
+#define DP2_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK                                      0x00000003L
+//DP2_DP_DPHY_SYM0
+#define DP2_DP_DPHY_SYM0__DPHY_SYM1__SHIFT                                                                    0x0
+#define DP2_DP_DPHY_SYM0__DPHY_SYM2__SHIFT                                                                    0xa
+#define DP2_DP_DPHY_SYM0__DPHY_SYM3__SHIFT                                                                    0x14
+#define DP2_DP_DPHY_SYM0__DPHY_SYM1_MASK                                                                      0x000003FFL
+#define DP2_DP_DPHY_SYM0__DPHY_SYM2_MASK                                                                      0x000FFC00L
+#define DP2_DP_DPHY_SYM0__DPHY_SYM3_MASK                                                                      0x3FF00000L
+//DP2_DP_DPHY_SYM1
+#define DP2_DP_DPHY_SYM1__DPHY_SYM4__SHIFT                                                                    0x0
+#define DP2_DP_DPHY_SYM1__DPHY_SYM5__SHIFT                                                                    0xa
+#define DP2_DP_DPHY_SYM1__DPHY_SYM6__SHIFT                                                                    0x14
+#define DP2_DP_DPHY_SYM1__DPHY_SYM4_MASK                                                                      0x000003FFL
+#define DP2_DP_DPHY_SYM1__DPHY_SYM5_MASK                                                                      0x000FFC00L
+#define DP2_DP_DPHY_SYM1__DPHY_SYM6_MASK                                                                      0x3FF00000L
+//DP2_DP_DPHY_SYM2
+#define DP2_DP_DPHY_SYM2__DPHY_SYM7__SHIFT                                                                    0x0
+#define DP2_DP_DPHY_SYM2__DPHY_SYM8__SHIFT                                                                    0xa
+#define DP2_DP_DPHY_SYM2__DPHY_SYM7_MASK                                                                      0x000003FFL
+#define DP2_DP_DPHY_SYM2__DPHY_SYM8_MASK                                                                      0x000FFC00L
+//DP2_DP_DPHY_8B10B_CNTL
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT                                                       0x8
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT                                                    0x10
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT                                                    0x18
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK                                                         0x00000100L
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK                                                      0x00010000L
+#define DP2_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK                                                      0x01000000L
+//DP2_DP_DPHY_PRBS_CNTL
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT                                                            0x0
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT                                                           0x4
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT                                                          0x8
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK                                                              0x00000001L
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK                                                             0x00000030L
+#define DP2_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK                                                            0x7FFFFF00L
+//DP2_DP_DPHY_SCRAM_CNTL
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS__SHIFT                                                     0x0
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT                                                 0x4
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT                                                0x8
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE__SHIFT                                                   0x18
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS_MASK                                                       0x00000001L
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK                                                   0x00000010L
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK                                                  0x0003FF00L
+#define DP2_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE_MASK                                                     0x01000000L
+//DP2_DP_DPHY_CRC_CONTROL0
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN__SHIFT                                                          0x0
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN__SHIFT                                                     0x4
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD__SHIFT                                                       0x8
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN_MASK                                                            0x00000001L
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN_MASK                                                       0x00000010L
+#define DP2_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD_MASK                                                         0x00000100L
+//DP2_DP_DPHY_CRC_CONTROL1
+#define DP2_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT__SHIFT                                              0x0
+#define DP2_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT__SHIFT                                               0x8
+#define DP2_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT_MASK                                                0x0000003FL
+#define DP2_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT_MASK                                                 0x00003F00L
+//DP2_DP_DPHY_CRC_RESULT0
+#define DP2_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP2_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP2_DP_DPHY_CRC_RESULT1
+#define DP2_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP2_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP2_DP_DPHY_CRC_RESULT2
+#define DP2_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP2_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP2_DP_DPHY_CRC_RESULT3
+#define DP2_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP2_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP2_DP_DPHY_CRC_STATUS
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID__SHIFT                                                  0x0
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK__SHIFT                                              0x4
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT                                                0x8
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT                                               0xc
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT                                           0x10
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_MASK                                                    0x00000001L
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK_MASK                                                0x00000010L
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK                                                  0x00000100L
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK                                                 0x00001000L
+#define DP2_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK                                             0x00010000L
+//DP2_DP_TU_CNTL
+#define DP2_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG__SHIFT                                                            0x8
+#define DP2_DP_TU_CNTL__DP_TU_OVERFLOW_ACK__SHIFT                                                             0xc
+#define DP2_DP_TU_CNTL__DP_TU_SIZE__SHIFT                                                                     0x18
+#define DP2_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG_MASK                                                              0x00000100L
+#define DP2_DP_TU_CNTL__DP_TU_OVERFLOW_ACK_MASK                                                               0x00001000L
+#define DP2_DP_TU_CNTL__DP_TU_SIZE_MASK                                                                       0x3F000000L
+//DP2_DP_PIXEL_FORMAT_DB_CNTL
+#define DP2_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE__SHIFT                                            0x0
+#define DP2_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING__SHIFT                                           0x4
+#define DP2_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE_MASK                                              0x00000001L
+#define DP2_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING_MASK                                             0x00000010L
+//DP2_DP_SEC_CNTL
+#define DP2_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT                                                          0x0
+#define DP2_DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT                                                             0x4
+#define DP2_DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT                                                             0x8
+#define DP2_DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT                                                             0xc
+#define DP2_DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT                                                             0x10
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT                                                            0x14
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT                                                            0x15
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT                                                            0x16
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT                                                            0x17
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE__SHIFT                                                            0x18
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE__SHIFT                                                            0x19
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE__SHIFT                                                            0x1a
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE__SHIFT                                                            0x1b
+#define DP2_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK                                                            0x00000001L
+#define DP2_DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK                                                               0x00000010L
+#define DP2_DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK                                                               0x00000100L
+#define DP2_DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK                                                               0x00001000L
+#define DP2_DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK                                                               0x00010000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK                                                              0x00100000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK                                                              0x00200000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK                                                              0x00400000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK                                                              0x00800000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE_MASK                                                              0x01000000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE_MASK                                                              0x02000000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE_MASK                                                              0x04000000L
+#define DP2_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE_MASK                                                              0x08000000L
+//DP2_DP_SEC_CNTL1
+#define DP2_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT                                                           0x0
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE__SHIFT                                                   0x1
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY__SHIFT                                                         0x4
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND__SHIFT                                                             0x5
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING__SHIFT                                                     0x6
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED__SHIFT                                             0x7
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE__SHIFT                                                   0x9
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE__SHIFT                                                   0xa
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE__SHIFT                                                   0xb
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE__SHIFT                                                   0xc
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE__SHIFT                                                   0xd
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE__SHIFT                                                   0xe
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE__SHIFT                                                   0xf
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK                                                             0x00000001L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE_MASK                                                     0x00000002L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY_MASK                                                           0x00000010L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_MASK                                                               0x00000020L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING_MASK                                                       0x00000040L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED_MASK                                               0x00000080L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE_MASK                                                     0x00000200L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE_MASK                                                     0x00000400L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE_MASK                                                     0x00000800L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE_MASK                                                     0x00001000L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE_MASK                                                     0x00002000L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE_MASK                                                     0x00004000L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE_MASK                                                     0x00008000L
+#define DP2_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_SEC_FRAMING1
+#define DP2_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT                                               0x0
+#define DP2_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP2_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK                                                 0x00000FFFL
+#define DP2_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP2_DP_SEC_FRAMING2
+#define DP2_DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT                                                     0x0
+#define DP2_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP2_DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK                                                       0x0000FFFFL
+#define DP2_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP2_DP_SEC_FRAMING3
+#define DP2_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT                                                    0x0
+#define DP2_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT                                                0x10
+#define DP2_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK                                                      0x00003FFFL
+#define DP2_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK                                                  0xFFFF0000L
+//DP2_DP_SEC_FRAMING4
+#define DP2_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING__SHIFT                                                      0x0
+#define DP2_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT                                                   0x14
+#define DP2_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT                                                      0x18
+#define DP2_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT                                                         0x1c
+#define DP2_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT                                                  0x1d
+#define DP2_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING_MASK                                                        0x00000001L
+#define DP2_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK                                                     0x00100000L
+#define DP2_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK                                                        0x01000000L
+#define DP2_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK                                                           0x10000000L
+#define DP2_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK                                                    0x20000000L
+//DP2_DP_SEC_AUD_N
+#define DP2_DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT                                                                 0x0
+#define DP2_DP_SEC_AUD_N__DP_SEC_AUD_N_MASK                                                                   0x00FFFFFFL
+//DP2_DP_SEC_AUD_N_READBACK
+#define DP2_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT                                               0x0
+#define DP2_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK                                                 0x00FFFFFFL
+//DP2_DP_SEC_AUD_M
+#define DP2_DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT                                                                 0x0
+#define DP2_DP_SEC_AUD_M__DP_SEC_AUD_M_MASK                                                                   0x00FFFFFFL
+//DP2_DP_SEC_AUD_M_READBACK
+#define DP2_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT                                               0x0
+#define DP2_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK                                                 0x00FFFFFFL
+//DP2_DP_SEC_TIMESTAMP
+#define DP2_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT                                                    0x0
+#define DP2_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK                                                      0x00000001L
+//DP2_DP_SEC_PACKET_CNTL
+#define DP2_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT                                                    0x4
+#define DP2_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER__SHIFT                                              0x18
+#define DP2_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK                                                      0x00000010L
+#define DP2_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER_MASK                                                0x3F000000L
+//DP2_DP_MSE_RATE_CNTL
+#define DP2_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT                                                            0x0
+#define DP2_DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT                                                            0x1a
+#define DP2_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK                                                              0x03FFFFFFL
+#define DP2_DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK                                                              0xFC000000L
+//DP2_DP_MSE_RATE_UPDATE
+#define DP2_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT                                             0x0
+#define DP2_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK                                               0x00000001L
+//DP2_DP_MSE_SAT0
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT                                                               0x0
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0__SHIFT                                                           0x4
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0__SHIFT                                                   0x5
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT                                                        0x8
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT                                                               0x10
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1__SHIFT                                                           0x14
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1__SHIFT                                                   0x15
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT                                                        0x18
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK                                                                 0x00000007L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0_MASK                                                             0x00000010L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0_MASK                                                     0x00000020L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK                                                          0x00003F00L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK                                                                 0x00070000L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1_MASK                                                             0x00100000L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1_MASK                                                     0x00200000L
+#define DP2_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK                                                          0x3F000000L
+//DP2_DP_MSE_SAT1
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT                                                               0x0
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2__SHIFT                                                           0x4
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2__SHIFT                                                   0x5
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT                                                        0x8
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT                                                               0x10
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3__SHIFT                                                           0x14
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3__SHIFT                                                   0x15
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT                                                        0x18
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK                                                                 0x00000007L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2_MASK                                                             0x00000010L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2_MASK                                                     0x00000020L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK                                                          0x00003F00L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK                                                                 0x00070000L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3_MASK                                                             0x00100000L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3_MASK                                                     0x00200000L
+#define DP2_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK                                                          0x3F000000L
+//DP2_DP_MSE_SAT2
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT                                                               0x0
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4__SHIFT                                                           0x4
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4__SHIFT                                                   0x5
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT                                                        0x8
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT                                                               0x10
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5__SHIFT                                                           0x14
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5__SHIFT                                                   0x15
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT                                                        0x18
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK                                                                 0x00000007L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4_MASK                                                             0x00000010L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4_MASK                                                     0x00000020L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK                                                          0x00003F00L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK                                                                 0x00070000L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5_MASK                                                             0x00100000L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5_MASK                                                     0x00200000L
+#define DP2_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK                                                          0x3F000000L
+//DP2_DP_MSE_SAT_UPDATE
+#define DP2_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT                                                       0x0
+#define DP2_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT                                                   0x8
+#define DP2_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK                                                         0x00000003L
+#define DP2_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK                                                     0x00000100L
+//DP2_DP_MSE_LINK_TIMING
+#define DP2_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT                                                      0x0
+#define DP2_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT                                                       0x10
+#define DP2_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK                                                        0x000003FFL
+#define DP2_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK                                                         0x00030000L
+//DP2_DP_MSE_MISC_CNTL
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT                                                        0x0
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT                                                    0x4
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT                                                      0x8
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK                                                          0x00000001L
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK                                                      0x00000010L
+#define DP2_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK                                                        0x00000100L
+//DP2_DP_DPHY_BS_SR_SWAP_CNTL
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT                                                0x0
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE__SHIFT                                              0xf
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START__SHIFT                                          0x10
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK                                                  0x000003FFL
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK                                                0x00008000L
+#define DP2_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START_MASK                                            0x00010000L
+//DP2_DP_DPHY_HBR2_PATTERN_CONTROL
+#define DP2_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL__SHIFT                                 0x0
+#define DP2_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL_MASK                                   0x00000007L
+//DP2_DP_MSE_SAT0_STATUS
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS__SHIFT                                                 0x0
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS__SHIFT                                             0x4
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS__SHIFT                                     0x5
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS__SHIFT                                          0x8
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS__SHIFT                                                 0x10
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS__SHIFT                                             0x14
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS__SHIFT                                     0x15
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS__SHIFT                                          0x18
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS_MASK                                                   0x00000007L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS_MASK                                               0x00000010L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS_MASK                                       0x00000020L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS_MASK                                            0x00003F00L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS_MASK                                                   0x00070000L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS_MASK                                               0x00100000L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS_MASK                                       0x00200000L
+#define DP2_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS_MASK                                            0x3F000000L
+//DP2_DP_MSE_SAT1_STATUS
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS__SHIFT                                                 0x0
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS__SHIFT                                             0x4
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS__SHIFT                                     0x5
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS__SHIFT                                          0x8
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS__SHIFT                                                 0x10
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS__SHIFT                                             0x14
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS__SHIFT                                     0x15
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS__SHIFT                                          0x18
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS_MASK                                                   0x00000007L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS_MASK                                               0x00000010L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS_MASK                                       0x00000020L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS_MASK                                            0x00003F00L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS_MASK                                                   0x00070000L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS_MASK                                               0x00100000L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS_MASK                                       0x00200000L
+#define DP2_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS_MASK                                            0x3F000000L
+//DP2_DP_MSE_SAT2_STATUS
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS__SHIFT                                                 0x0
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS__SHIFT                                             0x4
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS__SHIFT                                     0x5
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS__SHIFT                                          0x8
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS__SHIFT                                                 0x10
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS__SHIFT                                             0x14
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS__SHIFT                                     0x15
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS__SHIFT                                          0x18
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS_MASK                                                   0x00000007L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS_MASK                                               0x00000010L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS_MASK                                       0x00000020L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS_MASK                                            0x00003F00L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS_MASK                                                   0x00070000L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS_MASK                                               0x00100000L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS_MASK                                       0x00200000L
+#define DP2_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS_MASK                                            0x3F000000L
+//DP2_DP_HBLANK_CONTROL
+#define DP2_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                                          0x0
+#define DP2_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                                            0x0000FFFFL
+//DP2_DP_MSA_TIMING_PARAM1
+#define DP2_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL__SHIFT                                                        0x0
+#define DP2_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL__SHIFT                                                        0x10
+#define DP2_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL_MASK                                                          0x0000FFFFL
+#define DP2_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL_MASK                                                          0xFFFF0000L
+//DP2_DP_MSA_TIMING_PARAM2
+#define DP2_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART__SHIFT                                                        0x0
+#define DP2_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART__SHIFT                                                        0x10
+#define DP2_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART_MASK                                                          0x0000FFFFL
+#define DP2_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART_MASK                                                          0xFFFF0000L
+//DP2_DP_MSA_TIMING_PARAM3
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH__SHIFT                                                    0x0
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY__SHIFT                                                 0xf
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH__SHIFT                                                    0x10
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY__SHIFT                                                 0x1f
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH_MASK                                                      0x00007FFFL
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY_MASK                                                   0x00008000L
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH_MASK                                                      0x7FFF0000L
+#define DP2_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY_MASK                                                   0x80000000L
+//DP2_DP_MSA_TIMING_PARAM4
+#define DP2_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT__SHIFT                                                       0x0
+#define DP2_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH__SHIFT                                                        0x10
+#define DP2_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT_MASK                                                         0x0000FFFFL
+#define DP2_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH_MASK                                                          0xFFFF0000L
+//DP2_DP_MSO_CNTL
+#define DP2_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK__SHIFT                                                         0x0
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE__SHIFT                                                      0x4
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE__SHIFT                                                         0x8
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE__SHIFT                                                         0xc
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE__SHIFT                                                         0x10
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE__SHIFT                                                         0x14
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE__SHIFT                                                        0x18
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE__SHIFT                                                        0x1c
+#define DP2_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK_MASK                                                           0x00000003L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE_MASK                                                        0x000000F0L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE_MASK                                                           0x00000F00L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE_MASK                                                           0x0000F000L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE_MASK                                                           0x000F0000L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE_MASK                                                           0x00F00000L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE_MASK                                                          0x0F000000L
+#define DP2_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE_MASK                                                          0xF0000000L
+//DP2_DP_MSO_CNTL1
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE__SHIFT                                                       0x0
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE__SHIFT                                                       0x4
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE__SHIFT                                                       0x8
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE__SHIFT                                                       0xc
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE__SHIFT                                                       0x10
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE__SHIFT                                                       0x14
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE__SHIFT                                                       0x1c
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE_MASK                                                         0x0000000FL
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE_MASK                                                         0x000000F0L
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE_MASK                                                         0x00000F00L
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE_MASK                                                         0x0000F000L
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE_MASK                                                         0x000F0000L
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE_MASK                                                         0x00F00000L
+#define DP2_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE_MASK                                                         0xF0000000L
+//DP2_DP_STEER_FIFO_CNTL
+#define DP2_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN__SHIFT                       0x8
+#define DP2_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                        0x10
+#define DP2_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN_MASK                         0x00000100L
+#define DP2_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE_MASK                                          0x00030000L
+//DP2_DP_SEC_CNTL2
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND__SHIFT                                                             0x0
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING__SHIFT                                                     0x1
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED__SHIFT                                             0x2
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE__SHIFT                                                    0x3
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND__SHIFT                                                             0x4
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING__SHIFT                                                     0x5
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED__SHIFT                                             0x6
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE__SHIFT                                                    0x7
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND__SHIFT                                                             0x8
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING__SHIFT                                                     0x9
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED__SHIFT                                             0xa
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE__SHIFT                                                    0xb
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND__SHIFT                                                             0xc
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING__SHIFT                                                     0xd
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE__SHIFT                                                    0xf
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND__SHIFT                                                             0x10
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING__SHIFT                                                     0x11
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED__SHIFT                                             0x12
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE__SHIFT                                                    0x13
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND__SHIFT                                                             0x14
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING__SHIFT                                                     0x15
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED__SHIFT                                             0x16
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE__SHIFT                                                    0x17
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND__SHIFT                                                             0x18
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING__SHIFT                                                     0x19
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED__SHIFT                                             0x1a
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE__SHIFT                                                    0x1b
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP11_PPS__SHIFT                                                             0x1c
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_MASK                                                               0x00000001L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING_MASK                                                       0x00000002L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED_MASK                                               0x00000004L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE_MASK                                                      0x00000008L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_MASK                                                               0x00000010L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING_MASK                                                       0x00000020L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED_MASK                                               0x00000040L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE_MASK                                                      0x00000080L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_MASK                                                               0x00000100L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING_MASK                                                       0x00000200L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED_MASK                                               0x00000400L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE_MASK                                                      0x00000800L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_MASK                                                               0x00001000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING_MASK                                                       0x00002000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE_MASK                                                      0x00008000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_MASK                                                               0x00010000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING_MASK                                                       0x00020000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED_MASK                                               0x00040000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE_MASK                                                      0x00080000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_MASK                                                               0x00100000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING_MASK                                                       0x00200000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED_MASK                                               0x00400000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE_MASK                                                      0x00800000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_MASK                                                               0x01000000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING_MASK                                                       0x02000000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED_MASK                                               0x04000000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE_MASK                                                      0x08000000L
+#define DP2_DP_SEC_CNTL2__DP_SEC_GSP11_PPS_MASK                                                               0x10000000L
+//DP2_DP_SEC_CNTL3
+#define DP2_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM__SHIFT                                                         0x0
+#define DP2_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP2_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_SEC_CNTL4
+#define DP2_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM__SHIFT                                                         0x0
+#define DP2_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP2_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_SEC_CNTL5
+#define DP2_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM__SHIFT                                                         0x0
+#define DP2_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP2_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_SEC_CNTL6
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM__SHIFT                                                         0x0
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE__SHIFT                                                    0x10
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE__SHIFT                                                    0x11
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE__SHIFT                                                    0x12
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE__SHIFT                                                    0x13
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE__SHIFT                                                    0x14
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE__SHIFT                                                    0x15
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE__SHIFT                                                    0x16
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE__SHIFT                                                    0x17
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE__SHIFT                                                    0x18
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE__SHIFT                                                    0x19
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE__SHIFT                                                   0x1a
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE__SHIFT                                                   0x1b
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE_MASK                                                      0x00010000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE_MASK                                                      0x00020000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE_MASK                                                      0x00040000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE_MASK                                                      0x00080000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE_MASK                                                      0x00100000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE_MASK                                                      0x00200000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE_MASK                                                      0x00400000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE_MASK                                                      0x00800000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE_MASK                                                      0x01000000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE_MASK                                                      0x02000000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE_MASK                                                     0x04000000L
+#define DP2_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE_MASK                                                     0x08000000L
+//DP2_DP_SEC_CNTL7
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE__SHIFT                                                      0x0
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE__SHIFT                                                     0x1
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE__SHIFT                                                      0x4
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE__SHIFT                                                     0x5
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE__SHIFT                                                      0x8
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE__SHIFT                                                     0x9
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE__SHIFT                                                      0xc
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE__SHIFT                                                     0xd
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE__SHIFT                                                      0x10
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE__SHIFT                                                     0x11
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE__SHIFT                                                      0x14
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE__SHIFT                                                     0x15
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE__SHIFT                                                      0x18
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE__SHIFT                                                     0x19
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE__SHIFT                                                      0x1c
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE__SHIFT                                                     0x1d
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE_MASK                                                        0x00000001L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE_MASK                                                       0x00000002L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE_MASK                                                        0x00000010L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE_MASK                                                       0x00000020L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE_MASK                                                        0x00000100L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE_MASK                                                       0x00000200L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE_MASK                                                        0x00001000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE_MASK                                                       0x00002000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE_MASK                                                        0x00010000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE_MASK                                                       0x00020000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE_MASK                                                        0x00100000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE_MASK                                                       0x00200000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE_MASK                                                        0x01000000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE_MASK                                                       0x02000000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE_MASK                                                        0x10000000L
+#define DP2_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE_MASK                                                       0x20000000L
+//DP2_DP_DB_CNTL
+#define DP2_DP_DB_CNTL__DP_DB_PENDING__SHIFT                                                                  0x0
+#define DP2_DP_DB_CNTL__DP_DB_TAKEN__SHIFT                                                                    0x4
+#define DP2_DP_DB_CNTL__DP_DB_TAKEN_CLR__SHIFT                                                                0x5
+#define DP2_DP_DB_CNTL__DP_DB_LOCK__SHIFT                                                                     0x8
+#define DP2_DP_DB_CNTL__DP_DB_DISABLE__SHIFT                                                                  0xc
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_PENDING__SHIFT                                                          0xf
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN__SHIFT                                                            0x10
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR__SHIFT                                                        0x11
+#define DP2_DP_DB_CNTL__DP_DB_PENDING_MASK                                                                    0x00000001L
+#define DP2_DP_DB_CNTL__DP_DB_TAKEN_MASK                                                                      0x00000010L
+#define DP2_DP_DB_CNTL__DP_DB_TAKEN_CLR_MASK                                                                  0x00000020L
+#define DP2_DP_DB_CNTL__DP_DB_LOCK_MASK                                                                       0x00000100L
+#define DP2_DP_DB_CNTL__DP_DB_DISABLE_MASK                                                                    0x00001000L
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_PENDING_MASK                                                            0x00008000L
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_MASK                                                              0x00010000L
+#define DP2_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR_MASK                                                          0x00020000L
+//DP2_DP_MSA_VBID_MISC
+#define DP2_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE__SHIFT                                         0x0
+#define DP2_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                                      0x4
+#define DP2_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE__SHIFT                                                        0x8
+#define DP2_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE__SHIFT                                                        0x9
+#define DP2_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN__SHIFT                                                     0xc
+#define DP2_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN__SHIFT                                                     0xd
+#define DP2_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE__SHIFT                                                  0xf
+#define DP2_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM__SHIFT                                                        0x10
+#define DP2_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_MASK                                           0x00000003L
+#define DP2_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                                        0x00000010L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_MASK                                                          0x00000100L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_MASK                                                          0x00000200L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN_MASK                                                       0x00001000L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN_MASK                                                       0x00002000L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE_MASK                                                    0x00008000L
+#define DP2_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM_MASK                                                          0xFFFF0000L
+//DP2_DP_SEC_METADATA_TRANSMISSION
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE__SHIFT                                0x0
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE__SHIFT                        0x1
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE__SHIFT                            0x4
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE__SHIFT                                  0x10
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE_MASK                                  0x00000001L
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE_MASK                          0x00000002L
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE_MASK                              0x000000F0L
+#define DP2_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_MASK                                    0xFFFF0000L
+//DP2_DP_ALPM_CNTL
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND__SHIFT                                                         0x0
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING__SHIFT                                                      0x1
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND__SHIFT                                                       0x2
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING__SHIFT                                                    0x3
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE__SHIFT                                            0x4
+#define DP2_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO__SHIFT                                        0x5
+#define DP2_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE__SHIFT                                                  0x6
+#define DP2_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP__SHIFT                                     0x7
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM__SHIFT                                                  0x8
+#define DP2_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS__SHIFT                                   0xb
+#define DP2_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN__SHIFT                                       0xc
+#define DP2_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE__SHIFT                                                       0xf
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM__SHIFT                                             0x10
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND_MASK                                                           0x00000001L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING_MASK                                                        0x00000002L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND_MASK                                                         0x00000004L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING_MASK                                                      0x00000008L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE_MASK                                              0x00000010L
+#define DP2_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO_MASK                                          0x00000020L
+#define DP2_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE_MASK                                                    0x00000040L
+#define DP2_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_MASK                                       0x00000080L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM_MASK                                                    0x00000300L
+#define DP2_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS_MASK                                     0x00000800L
+#define DP2_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN_MASK                                         0x00001000L
+#define DP2_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE_MASK                                                         0x00008000L
+#define DP2_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM_MASK                                               0xFFFF0000L
+//DP2_DP_GSP8_CNTL
+#define DP2_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE__SHIFT                                                       0x0
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE__SHIFT                                                           0x4
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND__SHIFT                                                             0x7
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING__SHIFT                                                     0xc
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE_MASK                                                         0x0000000FL
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE_MASK                                                             0x00000010L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_MASK                                                               0x00000080L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING_MASK                                                       0x00001000L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP2_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_GSP9_CNTL
+#define DP2_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE__SHIFT                                                       0x0
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE__SHIFT                                                           0x4
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND__SHIFT                                                             0x7
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING__SHIFT                                                     0xc
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM__SHIFT                                                         0x10
+#define DP2_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE_MASK                                                         0x0000000FL
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE_MASK                                                             0x00000010L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_MASK                                                               0x00000080L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING_MASK                                                       0x00001000L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP2_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP2_DP_GSP10_CNTL
+#define DP2_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE__SHIFT                                                     0x0
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE__SHIFT                                                         0x4
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND__SHIFT                                                           0x7
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING__SHIFT                                                   0xc
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM__SHIFT                                                       0x10
+#define DP2_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE_MASK                                                       0x0000000FL
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE_MASK                                                           0x00000010L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_MASK                                                             0x00000080L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING_MASK                                                     0x00001000L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP2_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP2_DP_GSP11_CNTL
+#define DP2_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE__SHIFT                                                     0x0
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE__SHIFT                                                         0x4
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND__SHIFT                                                           0x7
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING__SHIFT                                                   0xc
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM__SHIFT                                                       0x10
+#define DP2_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE_MASK                                                       0x0000000FL
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE_MASK                                                           0x00000010L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_MASK                                                             0x00000080L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING_MASK                                                     0x00001000L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP2_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP2_DP_GSP_EN_DB_STATUS
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING__SHIFT                                             0x0
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING__SHIFT                                             0x1
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING__SHIFT                                             0x2
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING__SHIFT                                             0x3
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING__SHIFT                                             0x4
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING__SHIFT                                             0x5
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING__SHIFT                                             0x6
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING__SHIFT                                             0x7
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING__SHIFT                                             0x8
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING__SHIFT                                             0x9
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING__SHIFT                                            0xa
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING__SHIFT                                            0xb
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING_MASK                                               0x00000001L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING_MASK                                               0x00000002L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING_MASK                                               0x00000004L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING_MASK                                               0x00000008L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING_MASK                                               0x00000010L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING_MASK                                               0x00000020L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING_MASK                                               0x00000040L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING_MASK                                               0x00000080L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING_MASK                                               0x00000100L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING_MASK                                               0x00000200L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING_MASK                                              0x00000400L
+#define DP2_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING_MASK                                              0x00000800L
+//DP2_DP_AUXLESS_ALPM_CNTL1
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT__SHIFT                                              0x4
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY__SHIFT                                               0x8
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL__SHIFT                                            0x14
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE__SHIFT                                     0x1f
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT_MASK                                                0x000000F0L
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY_MASK                                                 0x0007FF00L
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL_MASK                                              0x1FF00000L
+#define DP2_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE_MASK                                       0x80000000L
+//DP2_DP_AUXLESS_ALPM_CNTL2
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME__SHIFT                                           0x0
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND__SHIFT                                                 0x7
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE__SHIFT                                            0x10
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING__SHIFT                                              0x11
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE__SHIFT                                            0x12
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING__SHIFT                                              0x13
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD__SHIFT                                          0x14
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME_MASK                                             0x0000007FL
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND_MASK                                                   0x00000080L
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE_MASK                                              0x00010000L
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING_MASK                                                0x00020000L
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE_MASK                                              0x00040000L
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING_MASK                                                0x00080000L
+#define DP2_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD_MASK                                            0x3FF00000L
+//DP2_DP_AUXLESS_ALPM_CNTL3
+#define DP2_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM__SHIFT                                             0x0
+#define DP2_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM__SHIFT                                             0x10
+#define DP2_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM_MASK                                               0x0000FFFFL
+#define DP2_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM_MASK                                               0xFFFF0000L
+//DP2_DP_AUXLESS_ALPM_CNTL4
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN__SHIFT                                                  0x1
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL__SHIFT                                   0x2
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME__SHIFT                                     0x3
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE__SHIFT                                       0x4
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS__SHIFT                                           0x5
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE__SHIFT                                               0x6
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM__SHIFT                                                   0x18
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_MASK                                                    0x00000002L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL_MASK                                     0x00000004L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME_MASK                                       0x00000008L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE_MASK                                         0x00000010L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS_MASK                                             0x00000020L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE_MASK                                                 0x00000040L
+#define DP2_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM_MASK                                                     0xFF000000L
+//DP2_DP_AUXLESS_ALPM_CNTL5
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK__SHIFT                                       0x0
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED__SHIFT                                   0x1
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                                     0x2
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR__SHIFT                                      0x3
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM__SHIFT                                  0x8
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM__SHIFT                                   0x10
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK_MASK                                         0x00000001L
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED_MASK                                     0x00000002L
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                                       0x00000004L
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR_MASK                                        0x00000008L
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM_MASK                                    0x0000FF00L
+#define DP2_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM_MASK                                     0xFFFF0000L
+//DP2_DP_STREAM_SYMBOL_COUNT_STATUS
+#define DP2_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT__SHIFT                                          0x0
+#define DP2_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT_MASK                                            0x0000FFFFL
+//DP2_DP_STREAM_SYMBOL_COUNT_CONTROL
+#define DP2_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE__SHIFT                                  0x0
+#define DP2_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET__SHIFT                                   0x4
+#define DP2_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE_MASK                                    0x00000001L
+#define DP2_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET_MASK                                     0x00000010L
+//DP2_DP_LINK_SYMBOL_COUNT_STATUS0
+#define DP2_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT__SHIFT                                             0x0
+#define DP2_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT_MASK                                               0x0000FFFFL
+//DP2_DP_LINK_SYMBOL_COUNT_STATUS1
+#define DP2_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT__SHIFT                                          0x0
+#define DP2_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT_MASK                                            0xFFFFFFFFL
+//DP2_DP_LINK_SYMBOL_COUNT_CONTROL
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE__SHIFT                                      0x0
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET__SHIFT                                       0x4
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE__SHIFT                                   0x8
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET__SHIFT                                    0xc
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE_MASK                                        0x00000001L
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET_MASK                                         0x00000010L
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE_MASK                                     0x00000100L
+#define DP2_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET_MASK                                      0x00001000L
+//DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT           0x0
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING__SHIFT  0x8
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK             0x00000001L
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+#define DP2_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING_MASK  0x00000100L
+//DP2_DP_DPHY_FAST_TRAINING
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT                                       0x0
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT                                         0x1
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT                            0x2
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING__SHIFT                              0x4
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT                                         0x8
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT                                         0x14
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK                                         0x00000001L
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK                                           0x00000002L
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK                              0x00000004L
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING_MASK                                0x00000010L
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK                                           0x000FFF00L
+#define DP2_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK                                           0xFFF00000L
+//DP2_DP_DPHY_FAST_TRAINING_STATUS
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT                                     0x0
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT                         0x4
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT                             0x8
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT                              0xc
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK                                       0x00000007L
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK                           0x00000010L
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK                               0x00000100L
+#define DP2_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK                                0x00001000L
+
+
+// addressBlock: dce_dc_dio_dig3_vpg_vpg_dispdec
+//VPG3_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG3_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG3_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG3_VPG_GENERIC_PACKET_DATA
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG3_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG3_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG3_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG3_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG3_VPG_GENERIC_STATUS
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG3_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG3_VPG_MEM_PWR
+#define VPG3_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG3_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG3_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG3_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG3_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG3_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG3_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG3_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG3_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG3_VPG_ISRC1_2_DATA
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG3_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_dio_dig3_apg_apg_dispdec
+//APG3_APG_CONTROL
+#define APG3_APG_CONTROL__APG_RESET__SHIFT                                                                    0x1
+#define APG3_APG_CONTROL__APG_RESET_DONE__SHIFT                                                               0x2
+#define APG3_APG_CONTROL__APG_RESET_MASK                                                                      0x00000002L
+#define APG3_APG_CONTROL__APG_RESET_DONE_MASK                                                                 0x00000004L
+//APG3_APG_CONTROL2
+#define APG3_APG_CONTROL2__APG_ENABLE__SHIFT                                                                  0x0
+#define APG3_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID__SHIFT                                                      0x8
+#define APG3_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE__SHIFT                                           0x18
+#define APG3_APG_CONTROL2__APG_ENABLE_MASK                                                                    0x00000001L
+#define APG3_APG_CONTROL2__APG_DP_AUDIO_STREAM_ID_MASK                                                        0x0000FF00L
+#define APG3_APG_CONTROL2__APG_DP_ASP_CHANNEL_COUNT_OVERRIDE_MASK                                             0x01000000L
+//APG3_APG_DBG_GEN_CONTROL
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG3_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG3_APG_PACKET_CONTROL
+#define APG3_APG_PACKET_CONTROL__APG_DBG_MUX_SEL__SHIFT                                                       0x0
+#define APG3_APG_PACKET_CONTROL__APG_ACP_SOURCE__SHIFT                                                        0x1
+#define APG3_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE__SHIFT                                                 0x2
+#define APG3_APG_PACKET_CONTROL__APG_DBG_MUX_SEL_MASK                                                         0x00000001L
+#define APG3_APG_PACKET_CONTROL__APG_ACP_SOURCE_MASK                                                          0x00000002L
+#define APG3_APG_PACKET_CONTROL__APG_AUDIO_INFO_SOURCE_MASK                                                   0x00000004L
+//APG3_APG_DBG_ACP
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE__SHIFT                                                             0x0
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                             0x8
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                             0x10
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_REQUIRED__SHIFT                                                         0x18
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE_MASK                                                               0x00000003L
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE0_MASK                                               0x0000FF00L
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_TYPE_DEPENDENT_BYTE1_MASK                                               0x00FF0000L
+#define APG3_APG_DBG_ACP__APG_DBG_ACP_REQUIRED_MASK                                                           0x01000000L
+//APG3_APG_DBG_AUDIO_INFO
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA__SHIFT                                                 0x0
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC__SHIFT                                                 0x8
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV__SHIFT                                                0xb
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH__SHIFT                                             0xf
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL__SHIFT                                             0x10
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CA_MASK                                                   0x000000FFL
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_CC_MASK                                                   0x00000700L
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LSV_MASK                                                  0x00007800L
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_DM_INH_MASK                                               0x00008000L
+#define APG3_APG_DBG_AUDIO_INFO__APG_DBG_AUDIO_INFO_LFEPBL_MASK                                               0x00030000L
+//APG3_APG_DBG_60958_0
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_A__SHIFT                                                       0x0
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_B__SHIFT                                                       0x1
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_C__SHIFT                                                       0x2
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_D__SHIFT                                                       0x3
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_MODE__SHIFT                                                    0x6
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE__SHIFT                                           0x8
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER__SHIFT                                           0x10
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L__SHIFT                                        0x14
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY__SHIFT                                      0x18
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY__SHIFT                                          0x1c
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_A_MASK                                                         0x00000001L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_B_MASK                                                         0x00000002L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_C_MASK                                                         0x00000004L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_D_MASK                                                         0x00000038L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_MODE_MASK                                                      0x000000C0L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CATEGORY_CODE_MASK                                             0x0000FF00L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_SOURCE_NUMBER_MASK                                             0x000F0000L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CHANNEL_NUMBER_L_MASK                                          0x00F00000L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_SAMPLING_FREQUENCY_MASK                                        0x0F000000L
+#define APG3_APG_DBG_60958_0__APG_DBG_60958_CS_CLOCK_ACCURACY_MASK                                            0x30000000L
+//APG3_APG_DBG_60958_1
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH__SHIFT                                             0x0
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                             0x4
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_VALID__SHIFT                                                      0x10
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R__SHIFT                                        0x14
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_WORD_LENGTH_MASK                                               0x0000000FL
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                               0x000000F0L
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_VALID_MASK                                                        0x00010000L
+#define APG3_APG_DBG_60958_1__APG_DBG_60958_CS_CHANNEL_NUMBER_R_MASK                                          0x00F00000L
+//APG3_APG_DBG_60958_2
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2__SHIFT                                        0x0
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3__SHIFT                                        0x4
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4__SHIFT                                        0x8
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5__SHIFT                                        0xc
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6__SHIFT                                        0x10
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7__SHIFT                                        0x14
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_2_MASK                                          0x0000000FL
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_3_MASK                                          0x000000F0L
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_4_MASK                                          0x00000F00L
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_5_MASK                                          0x0000F000L
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_6_MASK                                          0x000F0000L
+#define APG3_APG_DBG_60958_2__APG_DBG_60958_CS_CHANNEL_NUMBER_7_MASK                                          0x00F00000L
+//APG3_APG_AUDIO_CRC_CONTROL
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN__SHIFT                                                   0x0
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT__SHIFT                                                 0x4
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL__SHIFT                                               0xd
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT__SHIFT                                                0x10
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_EN_MASK                                                     0x00000001L
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CONT_MASK                                                   0x00000010L
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_CH_SEL_MASK                                                 0x0000E000L
+#define APG3_APG_AUDIO_CRC_CONTROL__APG_AUDIO_CRC_COUNT_MASK                                                  0xFFFF0000L
+//APG3_APG_AUDIO_CRC_CONTROL2
+#define APG3_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT__SHIFT                                 0x0
+#define APG3_APG_AUDIO_CRC_CONTROL2__APG_AUDIO_CRC_COUNT_FORCE_DEFAULT_MASK                                   0x0000FFFFL
+//APG3_APG_AUDIO_CRC_RESULT
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE__SHIFT                                                  0x0
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR__SHIFT                                            0x8
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC__SHIFT                                                       0x10
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_MASK                                                    0x00000001L
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_DONE_CLEAR_MASK                                              0x00000100L
+#define APG3_APG_AUDIO_CRC_RESULT__APG_AUDIO_CRC_MASK                                                         0xFFFF0000L
+//APG3_APG_DBG_RAMP_CONTROL0
+#define APG3_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT__SHIFT                                             0x0
+#define APG3_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN__SHIFT                                             0x1f
+#define APG3_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_MAX_COUNT_MASK                                               0x00FFFFFFL
+#define APG3_APG_DBG_RAMP_CONTROL0__APG_DBG_RAMP_DATA_SIGN_MASK                                               0x80000000L
+//APG3_APG_DBG_RAMP_CONTROL1
+#define APG3_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT__SHIFT                                             0x0
+#define APG3_APG_DBG_RAMP_CONTROL1__APG_DBG_RAMP_MIN_COUNT_MASK                                               0x00FFFFFFL
+//APG3_APG_DBG_RAMP_CONTROL2
+#define APG3_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT__SHIFT                                             0x0
+#define APG3_APG_DBG_RAMP_CONTROL2__APG_DBG_RAMP_INC_COUNT_MASK                                               0x00FFFFFFL
+//APG3_APG_DBG_RAMP_CONTROL3
+#define APG3_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT__SHIFT                                             0x0
+#define APG3_APG_DBG_RAMP_CONTROL3__APG_DBG_RAMP_DEC_COUNT_MASK                                               0x00FFFFFFL
+//APG3_APG_STATUS
+#define APG3_APG_STATUS__APG_AUDIO_ENABLE__SHIFT                                                              0x4
+#define APG3_APG_STATUS__APG_HBR_ENABLE__SHIFT                                                                0x8
+#define APG3_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS__SHIFT                                                0x18
+#define APG3_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR__SHIFT                                          0x19
+#define APG3_APG_STATUS__APG_AUDIO_ENABLE_MASK                                                                0x00000010L
+#define APG3_APG_STATUS__APG_HBR_ENABLE_MASK                                                                  0x00000100L
+#define APG3_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_MASK                                                  0x01000000L
+#define APG3_APG_STATUS__APG_AUDIO_FIFO_OVERFLOW_STATUS_CLEAR_MASK                                            0x02000000L
+//APG3_APG_STATUS2
+#define APG3_APG_STATUS2__APG_OUTPUT_ACTIVE__SHIFT                                                            0x0
+#define APG3_APG_STATUS2__APG_OUTPUT_ACTIVE_MASK                                                              0x00000001L
+//APG3_APG_DBG_AUDIO_DTO_CNTL
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE__SHIFT                                            0x8
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI__SHIFT                                           0xc
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV__SHIFT                                             0x10
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_BASE_MASK                                              0x00000100L
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_MULTI_MASK                                             0x00007000L
+#define APG3_APG_DBG_AUDIO_DTO_CNTL__APG_DBG_AUDIO_DTO_DIV_MASK                                               0x00070000L
+//APG3_APG_MEM_PWR
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG3_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG3_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG3_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+//APG3_APG_SPARE
+#define APG3_APG_SPARE__APG_SPARE__SHIFT                                                                      0x0
+#define APG3_APG_SPARE__APG_SPARE_MASK                                                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dio_dig3_dme_dme_dispdec
+//DME3_DME_CONTROL
+#define DME3_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME3_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME3_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME3_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME3_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME3_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME3_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME3_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME3_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME3_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME3_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME3_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME3_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME3_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME3_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME3_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME3_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME3_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME3_DME_MEMORY_CONTROL
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME3_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig3_dispdec
+//DIG3_DIG_FE_CNTL
+#define DIG3_DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT                                                            0x0
+#define DIG3_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT                                                        0x4
+#define DIG3_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT                                                       0x8
+#define DIG3_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT__SHIFT                                                    0xc
+#define DIG3_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN__SHIFT                                                        0x14
+#define DIG3_DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK                                                              0x00000007L
+#define DIG3_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK                                                          0x00000070L
+#define DIG3_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK                                                         0x00000100L
+#define DIG3_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT_MASK                                                      0x00007000L
+#define DIG3_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN_MASK                                                          0x00100000L
+//DIG3_DIG_FE_CLK_CNTL
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_MODE__SHIFT                                                              0x0
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN__SHIFT                                                            0x4
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON__SHIFT                                                0xa
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON__SHIFT                                              0xb
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON__SHIFT                                          0xc
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON__SHIFT                                         0xd
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON__SHIFT                                             0xe
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_MODE_MASK                                                                0x00000007L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN_MASK                                                              0x00000010L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON_MASK                                                  0x00000400L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON_MASK                                                0x00000800L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON_MASK                                            0x00001000L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON_MASK                                           0x00002000L
+#define DIG3_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON_MASK                                               0x00004000L
+//DIG3_DIG_FE_EN_CNTL
+#define DIG3_DIG_FE_EN_CNTL__DIG_FE_ENABLE__SHIFT                                                             0x0
+#define DIG3_DIG_FE_EN_CNTL__DIG_FE_ENABLE_MASK                                                               0x00000001L
+//DIG3_DIG_OUTPUT_CRC_CNTL
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT                                                    0x0
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT                                              0x4
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT                                              0x8
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK                                                      0x00000001L
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK                                                0x00000010L
+#define DIG3_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK                                                0x00000300L
+//DIG3_DIG_OUTPUT_CRC_RESULT
+#define DIG3_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT                                              0x0
+#define DIG3_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK                                                0x3FFFFFFFL
+//DIG3_DIG_CLOCK_PATTERN
+#define DIG3_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT                                                      0x0
+#define DIG3_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK                                                        0x000003FFL
+//DIG3_DIG_TEST_PATTERN
+#define DIG3_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT                                                 0x0
+#define DIG3_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT                                              0x1
+#define DIG3_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT                                               0x4
+#define DIG3_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT                                                0x5
+#define DIG3_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT                                      0x6
+#define DIG3_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT                                                 0x10
+#define DIG3_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK                                                   0x00000001L
+#define DIG3_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK                                                0x00000002L
+#define DIG3_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK                                                 0x00000010L
+#define DIG3_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK                                                  0x00000020L
+#define DIG3_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK                                        0x00000040L
+#define DIG3_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK                                                   0x03FF0000L
+//DIG3_DIG_RANDOM_PATTERN_SEED
+#define DIG3_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT                                          0x0
+#define DIG3_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT                                       0x18
+#define DIG3_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK                                            0x00FFFFFFL
+#define DIG3_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK                                         0x01000000L
+//DIG3_DIG_FIFO_CTRL0
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE__SHIFT                                                           0x0
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_RESET__SHIFT                                                            0x1
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL__SHIFT                                                 0x2
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC__SHIFT                                                   0x8
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                           0x9
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE__SHIFT                                                       0x14
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_ERROR__SHIFT                                                            0x1c
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE_MASK                                                             0x00000001L
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_RESET_MASK                                                              0x00000002L
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL_MASK                                                   0x000000FCL
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC_MASK                                                     0x00000100L
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE_MASK                                             0x00000600L
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE_MASK                                                         0x00100000L
+#define DIG3_DIG_FIFO_CTRL0__DIG_FIFO_ERROR_MASK                                                              0x30000000L
+//DIG3_DIG_FIFO_CTRL1
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT                                              0x1
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL__SHIFT                                                  0x2
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY__SHIFT                                                  0x8
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT                                                0xa
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL__SHIFT                                                    0x10
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL__SHIFT                                                    0x16
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED__SHIFT                                                       0x1d
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT                                              0x1e
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT                                              0x1f
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK                                                0x00000002L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL_MASK                                                    0x000000FCL
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY_MASK                                                    0x00000300L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK                                                  0x0000FC00L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL_MASK                                                      0x001F0000L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL_MASK                                                      0x03C00000L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED_MASK                                                         0x20000000L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK                                                0x40000000L
+#define DIG3_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK                                                0x80000000L
+//DIG3_HDMI_METADATA_PACKET_CONTROL
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                                 0x0
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                         0x4
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                                 0x8
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                   0x10
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                   0x00000001L
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                           0x00000010L
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                   0x00000100L
+#define DIG3_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                     0xFFFF0000L
+//DIG3_HDMI_CONTROL
+#define DIG3_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT                                                       0x1
+#define DIG3_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT                                                     0x2
+#define DIG3_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED__SHIFT                                            0x3
+#define DIG3_HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT                                                              0x8
+#define DIG3_HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT                                                             0x9
+#define DIG3_HDMI_CONTROL__DOLBY_VISION_EN__SHIFT                                                             0xa
+#define DIG3_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED__SHIFT                                         0xb
+#define DIG3_HDMI_CONTROL__TMDS_PIXEL_ENCODING__SHIFT                                                         0xc
+#define DIG3_HDMI_CONTROL__TMDS_COLOR_FORMAT__SHIFT                                                           0xd
+#define DIG3_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM__SHIFT                                           0x10
+#define DIG3_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT                                                      0x18
+#define DIG3_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                       0x1c
+#define DIG3_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK                                                         0x00000002L
+#define DIG3_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK                                                       0x00000004L
+#define DIG3_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK                                              0x00000008L
+#define DIG3_HDMI_CONTROL__HDMI_ERROR_ACK_MASK                                                                0x00000100L
+#define DIG3_HDMI_CONTROL__HDMI_ERROR_MASK_MASK                                                               0x00000200L
+#define DIG3_HDMI_CONTROL__DOLBY_VISION_EN_MASK                                                               0x00000400L
+#define DIG3_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED_MASK                                           0x00000800L
+#define DIG3_HDMI_CONTROL__TMDS_PIXEL_ENCODING_MASK                                                           0x00001000L
+#define DIG3_HDMI_CONTROL__TMDS_COLOR_FORMAT_MASK                                                             0x00006000L
+#define DIG3_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM_MASK                                             0x003F0000L
+#define DIG3_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK                                                        0x01000000L
+#define DIG3_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK                                                         0x30000000L
+//DIG3_HDMI_STATUS
+#define DIG3_HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT                                                           0x0
+#define DIG3_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT                                                      0x10
+#define DIG3_HDMI_STATUS__HDMI_ERROR_INT__SHIFT                                                               0x1b
+#define DIG3_HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK                                                             0x00000001L
+#define DIG3_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK                                                        0x00010000L
+#define DIG3_HDMI_STATUS__HDMI_ERROR_INT_MASK                                                                 0x08000000L
+//DIG3_HDMI_AUDIO_PACKET_CONTROL
+#define DIG3_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT                                            0x4
+#define DIG3_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK                                              0x00000030L
+//DIG3_HDMI_ACR_PACKET_CONTROL
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                    0x0
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                    0x1
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                  0x4
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                  0x8
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                               0xc
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                              0x10
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                      0x00000001L
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                      0x00000002L
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                    0x00000030L
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                    0x00000100L
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                                 0x00001000L
+#define DIG3_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                                0x00070000L
+//DIG3_HDMI_VBI_PACKET_CONTROL
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT                                                   0x0
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT                                                     0x4
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT                                                     0x5
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT                                                   0x8
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT                                                   0x9
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND__SHIFT                                                    0xc
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT                                                   0x10
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE__SHIFT                                                    0x18
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK                                                     0x00000001L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK                                                       0x00000010L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK                                                       0x00000020L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK                                                     0x00000100L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK                                                     0x00000200L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND_MASK                                                      0x00001000L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK                                                     0x003F0000L
+#define DIG3_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE_MASK                                                      0x3F000000L
+//DIG3_HDMI_INFOFRAME_CONTROL0
+#define DIG3_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT                                             0x4
+#define DIG3_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE__SHIFT                                             0x8
+#define DIG3_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK                                               0x00000010L
+#define DIG3_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE_MASK                                               0x00003F00L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                          0x1
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                                0x2
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                          0x4
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                          0x5
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                                0x6
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                          0x8
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                          0x9
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                                0xa
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE__SHIFT                           0xb
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                          0xc
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                          0xd
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                                0xe
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE__SHIFT                           0xf
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                          0x11
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                                0x12
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE__SHIFT                           0x13
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                          0x14
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                          0x15
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                                0x16
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE__SHIFT                           0x17
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                          0x18
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                          0x19
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                                0x1a
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE__SHIFT                           0x1b
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                          0x1c
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                          0x1d
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                                0x1e
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE__SHIFT                           0x1f
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                            0x00000001L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                            0x00000002L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                            0x00000010L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                            0x00000020L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                            0x00000100L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                            0x00000200L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                  0x00000400L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE_MASK                             0x00000800L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                            0x00001000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                            0x00002000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                  0x00004000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE_MASK                             0x00008000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                            0x00010000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                            0x00020000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                  0x00040000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE_MASK                             0x00080000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                            0x00100000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                            0x00200000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                  0x00400000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE_MASK                             0x00800000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                            0x01000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                            0x02000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                  0x04000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE_MASK                             0x08000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                            0x10000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                            0x20000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                  0x40000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE_MASK                             0x80000000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL6
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT__SHIFT                                          0x1
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                                0x2
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND__SHIFT                                          0x4
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT__SHIFT                                          0x5
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                                0x6
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND__SHIFT                                         0x8
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT__SHIFT                                         0x9
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                               0xa
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE__SHIFT                          0xb
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND__SHIFT                                         0xc
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT__SHIFT                                         0xd
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                               0xe
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE__SHIFT                          0xf
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND__SHIFT                                         0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT__SHIFT                                         0x11
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                               0x12
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE__SHIFT                          0x13
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND__SHIFT                                         0x14
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT__SHIFT                                         0x15
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                               0x16
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE__SHIFT                          0x17
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND__SHIFT                                         0x18
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT__SHIFT                                         0x19
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                               0x1a
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE__SHIFT                          0x1b
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND_MASK                                            0x00000001L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT_MASK                                            0x00000002L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND_MASK                                            0x00000010L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT_MASK                                            0x00000020L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND_MASK                                           0x00000100L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT_MASK                                           0x00000200L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE_MASK                                 0x00000400L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE_MASK                            0x00000800L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND_MASK                                           0x00001000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT_MASK                                           0x00002000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE_MASK                                 0x00004000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE_MASK                            0x00008000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND_MASK                                           0x00010000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT_MASK                                           0x00020000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE_MASK                                 0x00040000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE_MASK                            0x00080000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND_MASK                                           0x00100000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT_MASK                                           0x00200000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE_MASK                                 0x00400000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE_MASK                            0x00800000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND_MASK                                           0x01000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT_MASK                                           0x02000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE_MASK                                 0x04000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE_MASK                            0x08000000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL5
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                                0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                        0x1
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                                0x2
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                        0x3
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                                0x4
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                        0x5
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                                0x6
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                        0x7
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                                0x8
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                        0x9
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                                0xa
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                        0xb
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                                0xc
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                        0xd
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                                0xe
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                        0xf
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                                0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                        0x11
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                                0x12
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                        0x13
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                               0x14
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                       0x15
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                               0x16
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                       0x17
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                               0x18
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                       0x19
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                               0x1a
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                       0x1b
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                               0x1c
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                       0x1d
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                  0x00000001L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                          0x00000002L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                  0x00000004L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                          0x00000008L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                  0x00000010L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                          0x00000020L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                  0x00000040L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                          0x00000080L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                  0x00000100L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                          0x00000200L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                  0x00000400L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                          0x00000800L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                  0x00001000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                          0x00002000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                  0x00004000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                          0x00008000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                  0x00010000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                          0x00020000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                  0x00040000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                          0x00080000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                                 0x00100000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                         0x00200000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                                 0x00400000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                         0x00800000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                                 0x01000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                         0x02000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                                 0x04000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                         0x08000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                                 0x10000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                         0x20000000L
+//DIG3_HDMI_GC
+#define DIG3_HDMI_GC__HDMI_GC_AVMUTE__SHIFT                                                                   0x0
+#define DIG3_HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT                                                              0x2
+#define DIG3_HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT                                                               0x4
+#define DIG3_HDMI_GC__HDMI_PACKING_PHASE__SHIFT                                                               0x8
+#define DIG3_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT                                                      0xc
+#define DIG3_HDMI_GC__HDMI_GC_AVMUTE_MASK                                                                     0x00000001L
+#define DIG3_HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK                                                                0x00000004L
+#define DIG3_HDMI_GC__HDMI_DEFAULT_PHASE_MASK                                                                 0x00000010L
+#define DIG3_HDMI_GC__HDMI_PACKING_PHASE_MASK                                                                 0x00000F00L
+#define DIG3_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK                                                        0x00001000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL1
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE_MASK                                            0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE_MASK                                            0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL2
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE_MASK                                            0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE_MASK                                            0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL3
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE_MASK                                            0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE_MASK                                            0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL4
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE_MASK                                            0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE_MASK                                            0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL7
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE__SHIFT                                          0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE__SHIFT                                          0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE_MASK                                            0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE_MASK                                            0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL8
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE__SHIFT                                         0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE__SHIFT                                         0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE_MASK                                           0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE_MASK                                           0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL9
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE__SHIFT                                         0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE__SHIFT                                         0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE_MASK                                           0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE_MASK                                           0xFFFF0000L
+//DIG3_HDMI_GENERIC_PACKET_CONTROL10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE__SHIFT                                        0x0
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING__SHIFT                                0x10
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING__SHIFT                                0x11
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING__SHIFT                                0x12
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING__SHIFT                                0x13
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING__SHIFT                                0x14
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING__SHIFT                                0x15
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING__SHIFT                                0x16
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING__SHIFT                                0x17
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING__SHIFT                                0x18
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING__SHIFT                                0x19
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING__SHIFT                               0x1a
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING__SHIFT                               0x1b
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING__SHIFT                               0x1c
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING__SHIFT                               0x1d
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING__SHIFT                               0x1e
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE_MASK                                          0x0000FFFFL
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING_MASK                                  0x00010000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING_MASK                                  0x00020000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING_MASK                                  0x00040000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING_MASK                                  0x00080000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING_MASK                                  0x00100000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING_MASK                                  0x00200000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING_MASK                                  0x00400000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING_MASK                                  0x00800000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING_MASK                                  0x01000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING_MASK                                  0x02000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING_MASK                                 0x04000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING_MASK                                 0x08000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING_MASK                                 0x10000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING_MASK                                 0x20000000L
+#define DIG3_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING_MASK                                 0x40000000L
+//DIG3_HDMI_DB_CONTROL
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                          0x0
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_TAKEN__SHIFT                                                            0x4
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR__SHIFT                                                        0x5
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_LOCK__SHIFT                                                             0x8
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                          0xc
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                       0xf
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN__SHIFT                                                         0x10
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR__SHIFT                                                     0x11
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_PENDING_MASK                                                            0x00000001L
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_TAKEN_MASK                                                              0x00000010L
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR_MASK                                                          0x00000020L
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_LOCK_MASK                                                               0x00000100L
+#define DIG3_HDMI_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                            0x00001000L
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                         0x00008000L
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_MASK                                                           0x00010000L
+#define DIG3_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR_MASK                                                       0x00020000L
+//DIG3_HDMI_ACR_32_0
+#define DIG3_HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                            0xc
+#define DIG3_HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                              0xFFFFF000L
+//DIG3_HDMI_ACR_32_1
+#define DIG3_HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                              0x0
+#define DIG3_HDMI_ACR_32_1__HDMI_ACR_N_32_MASK                                                                0x000FFFFFL
+//DIG3_HDMI_ACR_44_0
+#define DIG3_HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                            0xc
+#define DIG3_HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                              0xFFFFF000L
+//DIG3_HDMI_ACR_44_1
+#define DIG3_HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                              0x0
+#define DIG3_HDMI_ACR_44_1__HDMI_ACR_N_44_MASK                                                                0x000FFFFFL
+//DIG3_HDMI_ACR_48_0
+#define DIG3_HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                            0xc
+#define DIG3_HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                              0xFFFFF000L
+//DIG3_HDMI_ACR_48_1
+#define DIG3_HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                              0x0
+#define DIG3_HDMI_ACR_48_1__HDMI_ACR_N_48_MASK                                                                0x000FFFFFL
+//DIG3_HDMI_ACR_STATUS_0
+#define DIG3_HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                           0xc
+#define DIG3_HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                             0xFFFFF000L
+//DIG3_HDMI_ACR_STATUS_1
+#define DIG3_HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                             0x0
+#define DIG3_HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK                                                               0x000FFFFFL
+//DIG3_DIG_FE_AUDIO_CNTL
+#define DIG3_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT                               0x0
+#define DIG3_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE__SHIFT                                                       0x8
+#define DIG3_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                                 0x00000007L
+#define DIG3_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE_MASK                                                         0x00000100L
+//DIG3_DIG_BE_CLK_CNTL
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_MODE__SHIFT                                                              0x0
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN__SHIFT                                                            0x4
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON__SHIFT                                                 0xb
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON__SHIFT                                            0xd
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_MODE_MASK                                                                0x00000007L
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN_MASK                                                              0x00000010L
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON_MASK                                                   0x00000800L
+#define DIG3_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON_MASK                                              0x00002000L
+//DIG3_DIG_BE_CNTL
+#define DIG3_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT                                                         0x0
+#define DIG3_DIG_BE_CNTL__DIG_SWAP__SHIFT                                                                     0x1
+#define DIG3_DIG_BE_CNTL__DIG_RB_SWITCH_EN__SHIFT                                                             0x2
+#define DIG3_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT                                                         0x8
+#define DIG3_DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT                                                               0x1c
+#define DIG3_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE_MASK                                                           0x00000001L
+#define DIG3_DIG_BE_CNTL__DIG_SWAP_MASK                                                                       0x00000002L
+#define DIG3_DIG_BE_CNTL__DIG_RB_SWITCH_EN_MASK                                                               0x00000004L
+#define DIG3_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK                                                           0x00007F00L
+#define DIG3_DIG_BE_CNTL__DIG_HPD_SELECT_MASK                                                                 0x70000000L
+//DIG3_DIG_BE_EN_CNTL
+#define DIG3_DIG_BE_EN_CNTL__DIG_BE_ENABLE__SHIFT                                                             0x0
+#define DIG3_DIG_BE_EN_CNTL__DIG_BE_ENABLE_MASK                                                               0x00000001L
+//DIG3_TMDS_CNTL
+#define DIG3_TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT                                                                0x0
+#define DIG3_TMDS_CNTL__TMDS_SYNC_PHASE_MASK                                                                  0x00000001L
+//DIG3_TMDS_CONTROL_CHAR
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT                                              0x0
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT                                              0x1
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT                                              0x2
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT                                              0x3
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK                                                0x00000001L
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK                                                0x00000002L
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK                                                0x00000004L
+#define DIG3_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK                                                0x00000008L
+//DIG3_TMDS_CONTROL0_FEEDBACK
+#define DIG3_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT                                     0x0
+#define DIG3_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT                                      0x8
+#define DIG3_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK                                       0x00000003L
+#define DIG3_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK                                        0x00000300L
+//DIG3_TMDS_STEREOSYNC_CTL_SEL
+#define DIG3_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT                                          0x0
+#define DIG3_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK                                            0x00000003L
+//DIG3_TMDS_SYNC_CHAR_PATTERN_0_1
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT                                       0x0
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT                                       0x10
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK                                         0x000003FFL
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK                                         0x03FF0000L
+//DIG3_TMDS_SYNC_CHAR_PATTERN_2_3
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT                                       0x0
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT                                       0x10
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK                                         0x000003FFL
+#define DIG3_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK                                         0x03FF0000L
+//DIG3_TMDS_CTL_BITS
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL0__SHIFT                                                                  0x0
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL1__SHIFT                                                                  0x8
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL2__SHIFT                                                                  0x10
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL3__SHIFT                                                                  0x18
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL0_MASK                                                                    0x00000001L
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL1_MASK                                                                    0x00000100L
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL2_MASK                                                                    0x00010000L
+#define DIG3_TMDS_CTL_BITS__TMDS_CTL3_MASK                                                                    0x01000000L
+//DIG3_TMDS_DCBALANCER_CONTROL
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT                                               0x0
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT                                               0x4
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT                                          0x8
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT                                          0x10
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT                                            0x18
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK                                                 0x00000001L
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK                                                 0x00000070L
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK                                            0x00000100L
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK                                            0x000F0000L
+#define DIG3_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK                                              0x01000000L
+//DIG3_TMDS_SYNC_DCBALANCE_CHAR
+#define DIG3_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01__SHIFT                                          0x0
+#define DIG3_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11__SHIFT                                          0x10
+#define DIG3_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01_MASK                                            0x000003FFL
+#define DIG3_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11_MASK                                            0x03FF0000L
+//DIG3_TMDS_CTL0_1_GEN_CNTL
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT                                                  0x0
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT                                                0x4
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT                                               0x7
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT                                           0x8
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT                                                  0x10
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT                                                0x14
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT                                               0x17
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT                                           0x18
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT                                                0x1f
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK                                              0x10000000L
+#define DIG3_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK                                                  0x80000000L
+//DIG3_TMDS_CTL2_3_GEN_CNTL
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT                                                  0x0
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT                                                0x4
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT                                               0x7
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT                                           0x8
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT                                                  0x10
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT                                                0x14
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT                                               0x17
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT                                           0x18
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG3_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK                                              0x10000000L
+//DIG3_DIG_VERSION
+#define DIG3_DIG_VERSION__DIG_TYPE__SHIFT                                                                     0x0
+#define DIG3_DIG_VERSION__DIG_TYPE_MASK                                                                       0x00000001L
+
+
+// addressBlock: dce_dc_dio_dp3_dispdec
+//DP3_DP_LINK_CNTL
+#define DP3_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT                                                    0x4
+#define DP3_DP_LINK_CNTL__DP_LINK_STATUS__SHIFT                                                               0x8
+#define DP3_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK                                                      0x00000010L
+#define DP3_DP_LINK_CNTL__DP_LINK_STATUS_MASK                                                                 0x00000100L
+//DP3_DP_PIXEL_FORMAT
+#define DP3_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                                                       0x0
+#define DP3_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT__SHIFT                                                 0x4
+#define DP3_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                                              0x8
+#define DP3_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT__SHIFT                                                   0x10
+#define DP3_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                                         0x00000001L
+#define DP3_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT_MASK                                                   0x00000030L
+#define DP3_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                                                0x00000700L
+#define DP3_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT_MASK                                                     0x00010000L
+//DP3_DP_MSA_COLORIMETRY
+#define DP3_DP_MSA_COLORIMETRY__DP_MSA_MISC0__SHIFT                                                           0x18
+#define DP3_DP_MSA_COLORIMETRY__DP_MSA_MISC0_MASK                                                             0xFF000000L
+//DP3_DP_CONFIG
+#define DP3_DP_CONFIG__DP_UDI_LANES__SHIFT                                                                    0x0
+#define DP3_DP_CONFIG__DP_UDI_LANES_MASK                                                                      0x00000003L
+//DP3_DP_VID_STREAM_CNTL
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT                                                   0x0
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT                                                0x8
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT                                                   0x10
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT                                           0x14
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK                                                     0x00000001L
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK                                                  0x00000300L
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK                                                     0x00010000L
+#define DP3_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK                                             0x00100000L
+//DP3_DP_STEER_FIFO
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE__SHIFT                                                        0x0
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT                                                         0x1
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE__SHIFT                                                    0x2
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT                                                      0x4
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT                                                       0x5
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT                                                       0x6
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT                                                      0x7
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE_MASK                                                          0x00000001L
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK                                                           0x00000002L
+#define DP3_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE_MASK                                                      0x00000004L
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK                                                        0x00000010L
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK                                                         0x00000020L
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK                                                         0x00000040L
+#define DP3_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK                                                        0x00000080L
+//DP3_DP_MSA_MISC
+#define DP3_DP_MSA_MISC__DP_MSA_MISC1__SHIFT                                                                  0x0
+#define DP3_DP_MSA_MISC__DP_MSA_MISC2__SHIFT                                                                  0x8
+#define DP3_DP_MSA_MISC__DP_MSA_MISC3__SHIFT                                                                  0x10
+#define DP3_DP_MSA_MISC__DP_MSA_MISC4__SHIFT                                                                  0x18
+#define DP3_DP_MSA_MISC__DP_MSA_MISC1_MASK                                                                    0x000000FFL
+#define DP3_DP_MSA_MISC__DP_MSA_MISC2_MASK                                                                    0x0000FF00L
+#define DP3_DP_MSA_MISC__DP_MSA_MISC3_MASK                                                                    0x00FF0000L
+#define DP3_DP_MSA_MISC__DP_MSA_MISC4_MASK                                                                    0xFF000000L
+//DP3_DP_DPHY_INTERNAL_CTRL
+#define DP3_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN__SHIFT                                         0x0
+#define DP3_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL__SHIFT                                        0x4
+#define DP3_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN_MASK                                           0x00000001L
+#define DP3_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL_MASK                                          0x00000010L
+//DP3_DP_VID_TIMING
+#define DP3_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE__SHIFT                                               0x4
+#define DP3_DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT                                                           0x8
+#define DP3_DP_VID_TIMING__DP_VID_M_DIV__SHIFT                                                                0xc
+#define DP3_DP_VID_TIMING__DP_VID_N_INTERVAL__SHIFT                                                           0xe
+#define DP3_DP_VID_TIMING__DP_VID_N_DIV__SHIFT                                                                0x18
+#define DP3_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE_MASK                                                 0x00000010L
+#define DP3_DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK                                                             0x00000100L
+#define DP3_DP_VID_TIMING__DP_VID_M_DIV_MASK                                                                  0x00003000L
+#define DP3_DP_VID_TIMING__DP_VID_N_INTERVAL_MASK                                                             0x0000C000L
+#define DP3_DP_VID_TIMING__DP_VID_N_DIV_MASK                                                                  0xFF000000L
+//DP3_DP_VID_N
+#define DP3_DP_VID_N__DP_VID_N__SHIFT                                                                         0x0
+#define DP3_DP_VID_N__DP_VID_N_MASK                                                                           0x00FFFFFFL
+//DP3_DP_VID_M
+#define DP3_DP_VID_M__DP_VID_M__SHIFT                                                                         0x0
+#define DP3_DP_VID_M__DP_VID_M_MASK                                                                           0x00FFFFFFL
+//DP3_DP_LINK_FRAMING_CNTL
+#define DP3_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT                                                  0x0
+#define DP3_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE__SHIFT                                  0x14
+#define DP3_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT                                                      0x18
+#define DP3_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT                                           0x1c
+#define DP3_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK                                                    0x0003FFFFL
+#define DP3_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE_MASK                                    0x00100000L
+#define DP3_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK                                                        0x01000000L
+#define DP3_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK                                             0x10000000L
+//DP3_DP_HBR2_EYE_PATTERN
+#define DP3_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT                                            0x0
+#define DP3_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK                                              0x00000001L
+//DP3_DP_VID_MSA_VBID
+#define DP3_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT                                                       0x0
+#define DP3_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE__SHIFT                                            0xc
+#define DP3_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT                                                     0x18
+#define DP3_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK                                                         0x00000FFFL
+#define DP3_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE_MASK                                              0x00001000L
+#define DP3_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK                                                       0x01000000L
+//DP3_DP_VID_INTERRUPT_CNTL
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT                                           0x0
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT                                           0x1
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT                                          0x2
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK                                             0x00000001L
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK                                             0x00000002L
+#define DP3_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK                                            0x00000004L
+//DP3_DP_DPHY_CNTL
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT                                                         0x0
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT                                                         0x1
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT                                                         0x2
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT                                                         0x3
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_EN__SHIFT                                                                  0x4
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW__SHIFT                                                        0x5
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS__SHIFT                                                       0x6
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM__SHIFT                                               0x7
+#define DP3_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL__SHIFT                                                           0x8
+#define DP3_DP_DPHY_CNTL__DPHY_BYPASS__SHIFT                                                                  0x10
+#define DP3_DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT                                                             0x18
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK                                                           0x00000001L
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK                                                           0x00000002L
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK                                                           0x00000004L
+#define DP3_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK                                                           0x00000008L
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_EN_MASK                                                                    0x00000010L
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW_MASK                                                          0x00000020L
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS_MASK                                                         0x00000040L
+#define DP3_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM_MASK                                                 0x00000080L
+#define DP3_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL_MASK                                                             0x00000100L
+#define DP3_DP_DPHY_CNTL__DPHY_BYPASS_MASK                                                                    0x00010000L
+#define DP3_DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK                                                               0x01000000L
+//DP3_DP_DPHY_TRAINING_PATTERN_SEL
+#define DP3_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT                                    0x0
+#define DP3_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK                                      0x00000003L
+//DP3_DP_DPHY_SYM0
+#define DP3_DP_DPHY_SYM0__DPHY_SYM1__SHIFT                                                                    0x0
+#define DP3_DP_DPHY_SYM0__DPHY_SYM2__SHIFT                                                                    0xa
+#define DP3_DP_DPHY_SYM0__DPHY_SYM3__SHIFT                                                                    0x14
+#define DP3_DP_DPHY_SYM0__DPHY_SYM1_MASK                                                                      0x000003FFL
+#define DP3_DP_DPHY_SYM0__DPHY_SYM2_MASK                                                                      0x000FFC00L
+#define DP3_DP_DPHY_SYM0__DPHY_SYM3_MASK                                                                      0x3FF00000L
+//DP3_DP_DPHY_SYM1
+#define DP3_DP_DPHY_SYM1__DPHY_SYM4__SHIFT                                                                    0x0
+#define DP3_DP_DPHY_SYM1__DPHY_SYM5__SHIFT                                                                    0xa
+#define DP3_DP_DPHY_SYM1__DPHY_SYM6__SHIFT                                                                    0x14
+#define DP3_DP_DPHY_SYM1__DPHY_SYM4_MASK                                                                      0x000003FFL
+#define DP3_DP_DPHY_SYM1__DPHY_SYM5_MASK                                                                      0x000FFC00L
+#define DP3_DP_DPHY_SYM1__DPHY_SYM6_MASK                                                                      0x3FF00000L
+//DP3_DP_DPHY_SYM2
+#define DP3_DP_DPHY_SYM2__DPHY_SYM7__SHIFT                                                                    0x0
+#define DP3_DP_DPHY_SYM2__DPHY_SYM8__SHIFT                                                                    0xa
+#define DP3_DP_DPHY_SYM2__DPHY_SYM7_MASK                                                                      0x000003FFL
+#define DP3_DP_DPHY_SYM2__DPHY_SYM8_MASK                                                                      0x000FFC00L
+//DP3_DP_DPHY_8B10B_CNTL
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT                                                       0x8
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT                                                    0x10
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT                                                    0x18
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK                                                         0x00000100L
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK                                                      0x00010000L
+#define DP3_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK                                                      0x01000000L
+//DP3_DP_DPHY_PRBS_CNTL
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT                                                            0x0
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT                                                           0x4
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT                                                          0x8
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK                                                              0x00000001L
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK                                                             0x00000030L
+#define DP3_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK                                                            0x7FFFFF00L
+//DP3_DP_DPHY_SCRAM_CNTL
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS__SHIFT                                                     0x0
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT                                                 0x4
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT                                                0x8
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE__SHIFT                                                   0x18
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS_MASK                                                       0x00000001L
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK                                                   0x00000010L
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK                                                  0x0003FF00L
+#define DP3_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE_MASK                                                     0x01000000L
+//DP3_DP_DPHY_CRC_CONTROL0
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN__SHIFT                                                          0x0
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN__SHIFT                                                     0x4
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD__SHIFT                                                       0x8
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN_MASK                                                            0x00000001L
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN_MASK                                                       0x00000010L
+#define DP3_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD_MASK                                                         0x00000100L
+//DP3_DP_DPHY_CRC_CONTROL1
+#define DP3_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT__SHIFT                                              0x0
+#define DP3_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT__SHIFT                                               0x8
+#define DP3_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT_MASK                                                0x0000003FL
+#define DP3_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT_MASK                                                 0x00003F00L
+//DP3_DP_DPHY_CRC_RESULT0
+#define DP3_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP3_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP3_DP_DPHY_CRC_RESULT1
+#define DP3_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP3_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP3_DP_DPHY_CRC_RESULT2
+#define DP3_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP3_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP3_DP_DPHY_CRC_RESULT3
+#define DP3_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP3_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP3_DP_DPHY_CRC_STATUS
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID__SHIFT                                                  0x0
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK__SHIFT                                              0x4
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT                                                0x8
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT                                               0xc
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT                                           0x10
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_MASK                                                    0x00000001L
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK_MASK                                                0x00000010L
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK                                                  0x00000100L
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK                                                 0x00001000L
+#define DP3_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK                                             0x00010000L
+//DP3_DP_TU_CNTL
+#define DP3_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG__SHIFT                                                            0x8
+#define DP3_DP_TU_CNTL__DP_TU_OVERFLOW_ACK__SHIFT                                                             0xc
+#define DP3_DP_TU_CNTL__DP_TU_SIZE__SHIFT                                                                     0x18
+#define DP3_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG_MASK                                                              0x00000100L
+#define DP3_DP_TU_CNTL__DP_TU_OVERFLOW_ACK_MASK                                                               0x00001000L
+#define DP3_DP_TU_CNTL__DP_TU_SIZE_MASK                                                                       0x3F000000L
+//DP3_DP_PIXEL_FORMAT_DB_CNTL
+#define DP3_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE__SHIFT                                            0x0
+#define DP3_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING__SHIFT                                           0x4
+#define DP3_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE_MASK                                              0x00000001L
+#define DP3_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING_MASK                                             0x00000010L
+//DP3_DP_SEC_CNTL
+#define DP3_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT                                                          0x0
+#define DP3_DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT                                                             0x4
+#define DP3_DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT                                                             0x8
+#define DP3_DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT                                                             0xc
+#define DP3_DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT                                                             0x10
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT                                                            0x14
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT                                                            0x15
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT                                                            0x16
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT                                                            0x17
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE__SHIFT                                                            0x18
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE__SHIFT                                                            0x19
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE__SHIFT                                                            0x1a
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE__SHIFT                                                            0x1b
+#define DP3_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK                                                            0x00000001L
+#define DP3_DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK                                                               0x00000010L
+#define DP3_DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK                                                               0x00000100L
+#define DP3_DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK                                                               0x00001000L
+#define DP3_DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK                                                               0x00010000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK                                                              0x00100000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK                                                              0x00200000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK                                                              0x00400000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK                                                              0x00800000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE_MASK                                                              0x01000000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE_MASK                                                              0x02000000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE_MASK                                                              0x04000000L
+#define DP3_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE_MASK                                                              0x08000000L
+//DP3_DP_SEC_CNTL1
+#define DP3_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT                                                           0x0
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE__SHIFT                                                   0x1
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY__SHIFT                                                         0x4
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND__SHIFT                                                             0x5
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING__SHIFT                                                     0x6
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED__SHIFT                                             0x7
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE__SHIFT                                                   0x9
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE__SHIFT                                                   0xa
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE__SHIFT                                                   0xb
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE__SHIFT                                                   0xc
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE__SHIFT                                                   0xd
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE__SHIFT                                                   0xe
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE__SHIFT                                                   0xf
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK                                                             0x00000001L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE_MASK                                                     0x00000002L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY_MASK                                                           0x00000010L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_MASK                                                               0x00000020L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING_MASK                                                       0x00000040L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED_MASK                                               0x00000080L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE_MASK                                                     0x00000200L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE_MASK                                                     0x00000400L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE_MASK                                                     0x00000800L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE_MASK                                                     0x00001000L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE_MASK                                                     0x00002000L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE_MASK                                                     0x00004000L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE_MASK                                                     0x00008000L
+#define DP3_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_SEC_FRAMING1
+#define DP3_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT                                               0x0
+#define DP3_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP3_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK                                                 0x00000FFFL
+#define DP3_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP3_DP_SEC_FRAMING2
+#define DP3_DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT                                                     0x0
+#define DP3_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP3_DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK                                                       0x0000FFFFL
+#define DP3_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP3_DP_SEC_FRAMING3
+#define DP3_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT                                                    0x0
+#define DP3_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT                                                0x10
+#define DP3_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK                                                      0x00003FFFL
+#define DP3_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK                                                  0xFFFF0000L
+//DP3_DP_SEC_FRAMING4
+#define DP3_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING__SHIFT                                                      0x0
+#define DP3_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT                                                   0x14
+#define DP3_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT                                                      0x18
+#define DP3_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT                                                         0x1c
+#define DP3_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT                                                  0x1d
+#define DP3_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING_MASK                                                        0x00000001L
+#define DP3_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK                                                     0x00100000L
+#define DP3_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK                                                        0x01000000L
+#define DP3_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK                                                           0x10000000L
+#define DP3_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK                                                    0x20000000L
+//DP3_DP_SEC_AUD_N
+#define DP3_DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT                                                                 0x0
+#define DP3_DP_SEC_AUD_N__DP_SEC_AUD_N_MASK                                                                   0x00FFFFFFL
+//DP3_DP_SEC_AUD_N_READBACK
+#define DP3_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT                                               0x0
+#define DP3_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK                                                 0x00FFFFFFL
+//DP3_DP_SEC_AUD_M
+#define DP3_DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT                                                                 0x0
+#define DP3_DP_SEC_AUD_M__DP_SEC_AUD_M_MASK                                                                   0x00FFFFFFL
+//DP3_DP_SEC_AUD_M_READBACK
+#define DP3_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT                                               0x0
+#define DP3_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK                                                 0x00FFFFFFL
+//DP3_DP_SEC_TIMESTAMP
+#define DP3_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT                                                    0x0
+#define DP3_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK                                                      0x00000001L
+//DP3_DP_SEC_PACKET_CNTL
+#define DP3_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT                                                    0x4
+#define DP3_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER__SHIFT                                              0x18
+#define DP3_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK                                                      0x00000010L
+#define DP3_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER_MASK                                                0x3F000000L
+//DP3_DP_MSE_RATE_CNTL
+#define DP3_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT                                                            0x0
+#define DP3_DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT                                                            0x1a
+#define DP3_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK                                                              0x03FFFFFFL
+#define DP3_DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK                                                              0xFC000000L
+//DP3_DP_MSE_RATE_UPDATE
+#define DP3_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT                                             0x0
+#define DP3_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK                                               0x00000001L
+//DP3_DP_MSE_SAT0
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT                                                               0x0
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0__SHIFT                                                           0x4
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0__SHIFT                                                   0x5
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT                                                        0x8
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT                                                               0x10
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1__SHIFT                                                           0x14
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1__SHIFT                                                   0x15
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT                                                        0x18
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK                                                                 0x00000007L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0_MASK                                                             0x00000010L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0_MASK                                                     0x00000020L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK                                                          0x00003F00L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK                                                                 0x00070000L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1_MASK                                                             0x00100000L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1_MASK                                                     0x00200000L
+#define DP3_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK                                                          0x3F000000L
+//DP3_DP_MSE_SAT1
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT                                                               0x0
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2__SHIFT                                                           0x4
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2__SHIFT                                                   0x5
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT                                                        0x8
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT                                                               0x10
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3__SHIFT                                                           0x14
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3__SHIFT                                                   0x15
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT                                                        0x18
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK                                                                 0x00000007L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2_MASK                                                             0x00000010L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2_MASK                                                     0x00000020L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK                                                          0x00003F00L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK                                                                 0x00070000L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3_MASK                                                             0x00100000L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3_MASK                                                     0x00200000L
+#define DP3_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK                                                          0x3F000000L
+//DP3_DP_MSE_SAT2
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT                                                               0x0
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4__SHIFT                                                           0x4
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4__SHIFT                                                   0x5
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT                                                        0x8
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT                                                               0x10
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5__SHIFT                                                           0x14
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5__SHIFT                                                   0x15
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT                                                        0x18
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK                                                                 0x00000007L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4_MASK                                                             0x00000010L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4_MASK                                                     0x00000020L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK                                                          0x00003F00L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK                                                                 0x00070000L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5_MASK                                                             0x00100000L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5_MASK                                                     0x00200000L
+#define DP3_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK                                                          0x3F000000L
+//DP3_DP_MSE_SAT_UPDATE
+#define DP3_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT                                                       0x0
+#define DP3_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT                                                   0x8
+#define DP3_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK                                                         0x00000003L
+#define DP3_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK                                                     0x00000100L
+//DP3_DP_MSE_LINK_TIMING
+#define DP3_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT                                                      0x0
+#define DP3_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT                                                       0x10
+#define DP3_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK                                                        0x000003FFL
+#define DP3_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK                                                         0x00030000L
+//DP3_DP_MSE_MISC_CNTL
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT                                                        0x0
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT                                                    0x4
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT                                                      0x8
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK                                                          0x00000001L
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK                                                      0x00000010L
+#define DP3_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK                                                        0x00000100L
+//DP3_DP_DPHY_BS_SR_SWAP_CNTL
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT                                                0x0
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE__SHIFT                                              0xf
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START__SHIFT                                          0x10
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK                                                  0x000003FFL
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK                                                0x00008000L
+#define DP3_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START_MASK                                            0x00010000L
+//DP3_DP_DPHY_HBR2_PATTERN_CONTROL
+#define DP3_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL__SHIFT                                 0x0
+#define DP3_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL_MASK                                   0x00000007L
+//DP3_DP_MSE_SAT0_STATUS
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS__SHIFT                                                 0x0
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS__SHIFT                                             0x4
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS__SHIFT                                     0x5
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS__SHIFT                                          0x8
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS__SHIFT                                                 0x10
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS__SHIFT                                             0x14
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS__SHIFT                                     0x15
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS__SHIFT                                          0x18
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS_MASK                                                   0x00000007L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS_MASK                                               0x00000010L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS_MASK                                       0x00000020L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS_MASK                                            0x00003F00L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS_MASK                                                   0x00070000L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS_MASK                                               0x00100000L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS_MASK                                       0x00200000L
+#define DP3_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS_MASK                                            0x3F000000L
+//DP3_DP_MSE_SAT1_STATUS
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS__SHIFT                                                 0x0
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS__SHIFT                                             0x4
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS__SHIFT                                     0x5
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS__SHIFT                                          0x8
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS__SHIFT                                                 0x10
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS__SHIFT                                             0x14
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS__SHIFT                                     0x15
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS__SHIFT                                          0x18
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS_MASK                                                   0x00000007L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS_MASK                                               0x00000010L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS_MASK                                       0x00000020L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS_MASK                                            0x00003F00L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS_MASK                                                   0x00070000L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS_MASK                                               0x00100000L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS_MASK                                       0x00200000L
+#define DP3_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS_MASK                                            0x3F000000L
+//DP3_DP_MSE_SAT2_STATUS
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS__SHIFT                                                 0x0
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS__SHIFT                                             0x4
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS__SHIFT                                     0x5
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS__SHIFT                                          0x8
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS__SHIFT                                                 0x10
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS__SHIFT                                             0x14
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS__SHIFT                                     0x15
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS__SHIFT                                          0x18
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS_MASK                                                   0x00000007L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS_MASK                                               0x00000010L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS_MASK                                       0x00000020L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS_MASK                                            0x00003F00L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS_MASK                                                   0x00070000L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS_MASK                                               0x00100000L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS_MASK                                       0x00200000L
+#define DP3_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS_MASK                                            0x3F000000L
+//DP3_DP_HBLANK_CONTROL
+#define DP3_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                                          0x0
+#define DP3_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                                            0x0000FFFFL
+//DP3_DP_MSA_TIMING_PARAM1
+#define DP3_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL__SHIFT                                                        0x0
+#define DP3_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL__SHIFT                                                        0x10
+#define DP3_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL_MASK                                                          0x0000FFFFL
+#define DP3_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL_MASK                                                          0xFFFF0000L
+//DP3_DP_MSA_TIMING_PARAM2
+#define DP3_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART__SHIFT                                                        0x0
+#define DP3_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART__SHIFT                                                        0x10
+#define DP3_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART_MASK                                                          0x0000FFFFL
+#define DP3_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART_MASK                                                          0xFFFF0000L
+//DP3_DP_MSA_TIMING_PARAM3
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH__SHIFT                                                    0x0
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY__SHIFT                                                 0xf
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH__SHIFT                                                    0x10
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY__SHIFT                                                 0x1f
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH_MASK                                                      0x00007FFFL
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY_MASK                                                   0x00008000L
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH_MASK                                                      0x7FFF0000L
+#define DP3_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY_MASK                                                   0x80000000L
+//DP3_DP_MSA_TIMING_PARAM4
+#define DP3_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT__SHIFT                                                       0x0
+#define DP3_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH__SHIFT                                                        0x10
+#define DP3_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT_MASK                                                         0x0000FFFFL
+#define DP3_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH_MASK                                                          0xFFFF0000L
+//DP3_DP_MSO_CNTL
+#define DP3_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK__SHIFT                                                         0x0
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE__SHIFT                                                      0x4
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE__SHIFT                                                         0x8
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE__SHIFT                                                         0xc
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE__SHIFT                                                         0x10
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE__SHIFT                                                         0x14
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE__SHIFT                                                        0x18
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE__SHIFT                                                        0x1c
+#define DP3_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK_MASK                                                           0x00000003L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE_MASK                                                        0x000000F0L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE_MASK                                                           0x00000F00L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE_MASK                                                           0x0000F000L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE_MASK                                                           0x000F0000L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE_MASK                                                           0x00F00000L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE_MASK                                                          0x0F000000L
+#define DP3_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE_MASK                                                          0xF0000000L
+//DP3_DP_MSO_CNTL1
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE__SHIFT                                                       0x0
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE__SHIFT                                                       0x4
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE__SHIFT                                                       0x8
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE__SHIFT                                                       0xc
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE__SHIFT                                                       0x10
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE__SHIFT                                                       0x14
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE__SHIFT                                                       0x1c
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE_MASK                                                         0x0000000FL
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE_MASK                                                         0x000000F0L
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE_MASK                                                         0x00000F00L
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE_MASK                                                         0x0000F000L
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE_MASK                                                         0x000F0000L
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE_MASK                                                         0x00F00000L
+#define DP3_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE_MASK                                                         0xF0000000L
+//DP3_DP_STEER_FIFO_CNTL
+#define DP3_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN__SHIFT                       0x8
+#define DP3_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                        0x10
+#define DP3_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN_MASK                         0x00000100L
+#define DP3_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE_MASK                                          0x00030000L
+//DP3_DP_SEC_CNTL2
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND__SHIFT                                                             0x0
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING__SHIFT                                                     0x1
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED__SHIFT                                             0x2
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE__SHIFT                                                    0x3
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND__SHIFT                                                             0x4
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING__SHIFT                                                     0x5
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED__SHIFT                                             0x6
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE__SHIFT                                                    0x7
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND__SHIFT                                                             0x8
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING__SHIFT                                                     0x9
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED__SHIFT                                             0xa
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE__SHIFT                                                    0xb
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND__SHIFT                                                             0xc
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING__SHIFT                                                     0xd
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE__SHIFT                                                    0xf
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND__SHIFT                                                             0x10
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING__SHIFT                                                     0x11
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED__SHIFT                                             0x12
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE__SHIFT                                                    0x13
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND__SHIFT                                                             0x14
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING__SHIFT                                                     0x15
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED__SHIFT                                             0x16
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE__SHIFT                                                    0x17
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND__SHIFT                                                             0x18
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING__SHIFT                                                     0x19
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED__SHIFT                                             0x1a
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE__SHIFT                                                    0x1b
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP11_PPS__SHIFT                                                             0x1c
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_MASK                                                               0x00000001L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING_MASK                                                       0x00000002L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED_MASK                                               0x00000004L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE_MASK                                                      0x00000008L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_MASK                                                               0x00000010L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING_MASK                                                       0x00000020L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED_MASK                                               0x00000040L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE_MASK                                                      0x00000080L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_MASK                                                               0x00000100L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING_MASK                                                       0x00000200L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED_MASK                                               0x00000400L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE_MASK                                                      0x00000800L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_MASK                                                               0x00001000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING_MASK                                                       0x00002000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE_MASK                                                      0x00008000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_MASK                                                               0x00010000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING_MASK                                                       0x00020000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED_MASK                                               0x00040000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE_MASK                                                      0x00080000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_MASK                                                               0x00100000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING_MASK                                                       0x00200000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED_MASK                                               0x00400000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE_MASK                                                      0x00800000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_MASK                                                               0x01000000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING_MASK                                                       0x02000000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED_MASK                                               0x04000000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE_MASK                                                      0x08000000L
+#define DP3_DP_SEC_CNTL2__DP_SEC_GSP11_PPS_MASK                                                               0x10000000L
+//DP3_DP_SEC_CNTL3
+#define DP3_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM__SHIFT                                                         0x0
+#define DP3_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP3_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_SEC_CNTL4
+#define DP3_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM__SHIFT                                                         0x0
+#define DP3_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP3_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_SEC_CNTL5
+#define DP3_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM__SHIFT                                                         0x0
+#define DP3_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP3_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_SEC_CNTL6
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM__SHIFT                                                         0x0
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE__SHIFT                                                    0x10
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE__SHIFT                                                    0x11
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE__SHIFT                                                    0x12
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE__SHIFT                                                    0x13
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE__SHIFT                                                    0x14
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE__SHIFT                                                    0x15
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE__SHIFT                                                    0x16
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE__SHIFT                                                    0x17
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE__SHIFT                                                    0x18
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE__SHIFT                                                    0x19
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE__SHIFT                                                   0x1a
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE__SHIFT                                                   0x1b
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE_MASK                                                      0x00010000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE_MASK                                                      0x00020000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE_MASK                                                      0x00040000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE_MASK                                                      0x00080000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE_MASK                                                      0x00100000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE_MASK                                                      0x00200000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE_MASK                                                      0x00400000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE_MASK                                                      0x00800000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE_MASK                                                      0x01000000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE_MASK                                                      0x02000000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE_MASK                                                     0x04000000L
+#define DP3_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE_MASK                                                     0x08000000L
+//DP3_DP_SEC_CNTL7
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE__SHIFT                                                      0x0
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE__SHIFT                                                     0x1
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE__SHIFT                                                      0x4
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE__SHIFT                                                     0x5
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE__SHIFT                                                      0x8
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE__SHIFT                                                     0x9
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE__SHIFT                                                      0xc
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE__SHIFT                                                     0xd
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE__SHIFT                                                      0x10
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE__SHIFT                                                     0x11
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE__SHIFT                                                      0x14
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE__SHIFT                                                     0x15
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE__SHIFT                                                      0x18
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE__SHIFT                                                     0x19
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE__SHIFT                                                      0x1c
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE__SHIFT                                                     0x1d
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE_MASK                                                        0x00000001L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE_MASK                                                       0x00000002L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE_MASK                                                        0x00000010L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE_MASK                                                       0x00000020L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE_MASK                                                        0x00000100L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE_MASK                                                       0x00000200L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE_MASK                                                        0x00001000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE_MASK                                                       0x00002000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE_MASK                                                        0x00010000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE_MASK                                                       0x00020000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE_MASK                                                        0x00100000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE_MASK                                                       0x00200000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE_MASK                                                        0x01000000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE_MASK                                                       0x02000000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE_MASK                                                        0x10000000L
+#define DP3_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE_MASK                                                       0x20000000L
+//DP3_DP_DB_CNTL
+#define DP3_DP_DB_CNTL__DP_DB_PENDING__SHIFT                                                                  0x0
+#define DP3_DP_DB_CNTL__DP_DB_TAKEN__SHIFT                                                                    0x4
+#define DP3_DP_DB_CNTL__DP_DB_TAKEN_CLR__SHIFT                                                                0x5
+#define DP3_DP_DB_CNTL__DP_DB_LOCK__SHIFT                                                                     0x8
+#define DP3_DP_DB_CNTL__DP_DB_DISABLE__SHIFT                                                                  0xc
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_PENDING__SHIFT                                                          0xf
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN__SHIFT                                                            0x10
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR__SHIFT                                                        0x11
+#define DP3_DP_DB_CNTL__DP_DB_PENDING_MASK                                                                    0x00000001L
+#define DP3_DP_DB_CNTL__DP_DB_TAKEN_MASK                                                                      0x00000010L
+#define DP3_DP_DB_CNTL__DP_DB_TAKEN_CLR_MASK                                                                  0x00000020L
+#define DP3_DP_DB_CNTL__DP_DB_LOCK_MASK                                                                       0x00000100L
+#define DP3_DP_DB_CNTL__DP_DB_DISABLE_MASK                                                                    0x00001000L
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_PENDING_MASK                                                            0x00008000L
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_MASK                                                              0x00010000L
+#define DP3_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR_MASK                                                          0x00020000L
+//DP3_DP_MSA_VBID_MISC
+#define DP3_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE__SHIFT                                         0x0
+#define DP3_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                                      0x4
+#define DP3_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE__SHIFT                                                        0x8
+#define DP3_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE__SHIFT                                                        0x9
+#define DP3_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN__SHIFT                                                     0xc
+#define DP3_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN__SHIFT                                                     0xd
+#define DP3_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE__SHIFT                                                  0xf
+#define DP3_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM__SHIFT                                                        0x10
+#define DP3_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_MASK                                           0x00000003L
+#define DP3_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                                        0x00000010L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_MASK                                                          0x00000100L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_MASK                                                          0x00000200L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN_MASK                                                       0x00001000L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN_MASK                                                       0x00002000L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE_MASK                                                    0x00008000L
+#define DP3_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM_MASK                                                          0xFFFF0000L
+//DP3_DP_SEC_METADATA_TRANSMISSION
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE__SHIFT                                0x0
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE__SHIFT                        0x1
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE__SHIFT                            0x4
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE__SHIFT                                  0x10
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE_MASK                                  0x00000001L
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE_MASK                          0x00000002L
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE_MASK                              0x000000F0L
+#define DP3_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_MASK                                    0xFFFF0000L
+//DP3_DP_ALPM_CNTL
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND__SHIFT                                                         0x0
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING__SHIFT                                                      0x1
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND__SHIFT                                                       0x2
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING__SHIFT                                                    0x3
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE__SHIFT                                            0x4
+#define DP3_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO__SHIFT                                        0x5
+#define DP3_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE__SHIFT                                                  0x6
+#define DP3_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP__SHIFT                                     0x7
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM__SHIFT                                                  0x8
+#define DP3_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS__SHIFT                                   0xb
+#define DP3_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN__SHIFT                                       0xc
+#define DP3_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE__SHIFT                                                       0xf
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM__SHIFT                                             0x10
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND_MASK                                                           0x00000001L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING_MASK                                                        0x00000002L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND_MASK                                                         0x00000004L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING_MASK                                                      0x00000008L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE_MASK                                              0x00000010L
+#define DP3_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO_MASK                                          0x00000020L
+#define DP3_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE_MASK                                                    0x00000040L
+#define DP3_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_MASK                                       0x00000080L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM_MASK                                                    0x00000300L
+#define DP3_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS_MASK                                     0x00000800L
+#define DP3_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN_MASK                                         0x00001000L
+#define DP3_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE_MASK                                                         0x00008000L
+#define DP3_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM_MASK                                               0xFFFF0000L
+//DP3_DP_GSP8_CNTL
+#define DP3_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE__SHIFT                                                       0x0
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE__SHIFT                                                           0x4
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND__SHIFT                                                             0x7
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING__SHIFT                                                     0xc
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE_MASK                                                         0x0000000FL
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE_MASK                                                             0x00000010L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_MASK                                                               0x00000080L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING_MASK                                                       0x00001000L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP3_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_GSP9_CNTL
+#define DP3_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE__SHIFT                                                       0x0
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE__SHIFT                                                           0x4
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND__SHIFT                                                             0x7
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING__SHIFT                                                     0xc
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM__SHIFT                                                         0x10
+#define DP3_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE_MASK                                                         0x0000000FL
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE_MASK                                                             0x00000010L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_MASK                                                               0x00000080L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING_MASK                                                       0x00001000L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP3_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP3_DP_GSP10_CNTL
+#define DP3_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE__SHIFT                                                     0x0
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE__SHIFT                                                         0x4
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND__SHIFT                                                           0x7
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING__SHIFT                                                   0xc
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM__SHIFT                                                       0x10
+#define DP3_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE_MASK                                                       0x0000000FL
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE_MASK                                                           0x00000010L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_MASK                                                             0x00000080L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING_MASK                                                     0x00001000L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP3_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP3_DP_GSP11_CNTL
+#define DP3_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE__SHIFT                                                     0x0
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE__SHIFT                                                         0x4
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND__SHIFT                                                           0x7
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING__SHIFT                                                   0xc
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM__SHIFT                                                       0x10
+#define DP3_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE_MASK                                                       0x0000000FL
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE_MASK                                                           0x00000010L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_MASK                                                             0x00000080L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING_MASK                                                     0x00001000L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP3_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP3_DP_GSP_EN_DB_STATUS
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING__SHIFT                                             0x0
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING__SHIFT                                             0x1
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING__SHIFT                                             0x2
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING__SHIFT                                             0x3
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING__SHIFT                                             0x4
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING__SHIFT                                             0x5
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING__SHIFT                                             0x6
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING__SHIFT                                             0x7
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING__SHIFT                                             0x8
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING__SHIFT                                             0x9
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING__SHIFT                                            0xa
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING__SHIFT                                            0xb
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING_MASK                                               0x00000001L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING_MASK                                               0x00000002L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING_MASK                                               0x00000004L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING_MASK                                               0x00000008L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING_MASK                                               0x00000010L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING_MASK                                               0x00000020L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING_MASK                                               0x00000040L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING_MASK                                               0x00000080L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING_MASK                                               0x00000100L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING_MASK                                               0x00000200L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING_MASK                                              0x00000400L
+#define DP3_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING_MASK                                              0x00000800L
+//DP3_DP_AUXLESS_ALPM_CNTL1
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT__SHIFT                                              0x4
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY__SHIFT                                               0x8
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL__SHIFT                                            0x14
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE__SHIFT                                     0x1f
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT_MASK                                                0x000000F0L
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY_MASK                                                 0x0007FF00L
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL_MASK                                              0x1FF00000L
+#define DP3_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE_MASK                                       0x80000000L
+//DP3_DP_AUXLESS_ALPM_CNTL2
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME__SHIFT                                           0x0
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND__SHIFT                                                 0x7
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE__SHIFT                                            0x10
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING__SHIFT                                              0x11
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE__SHIFT                                            0x12
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING__SHIFT                                              0x13
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD__SHIFT                                          0x14
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME_MASK                                             0x0000007FL
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND_MASK                                                   0x00000080L
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE_MASK                                              0x00010000L
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING_MASK                                                0x00020000L
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE_MASK                                              0x00040000L
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING_MASK                                                0x00080000L
+#define DP3_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD_MASK                                            0x3FF00000L
+//DP3_DP_AUXLESS_ALPM_CNTL3
+#define DP3_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM__SHIFT                                             0x0
+#define DP3_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM__SHIFT                                             0x10
+#define DP3_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM_MASK                                               0x0000FFFFL
+#define DP3_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM_MASK                                               0xFFFF0000L
+//DP3_DP_AUXLESS_ALPM_CNTL4
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN__SHIFT                                                  0x1
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL__SHIFT                                   0x2
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME__SHIFT                                     0x3
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE__SHIFT                                       0x4
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS__SHIFT                                           0x5
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE__SHIFT                                               0x6
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM__SHIFT                                                   0x18
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_MASK                                                    0x00000002L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL_MASK                                     0x00000004L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME_MASK                                       0x00000008L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE_MASK                                         0x00000010L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS_MASK                                             0x00000020L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE_MASK                                                 0x00000040L
+#define DP3_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM_MASK                                                     0xFF000000L
+//DP3_DP_AUXLESS_ALPM_CNTL5
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK__SHIFT                                       0x0
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED__SHIFT                                   0x1
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                                     0x2
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR__SHIFT                                      0x3
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM__SHIFT                                  0x8
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM__SHIFT                                   0x10
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK_MASK                                         0x00000001L
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED_MASK                                     0x00000002L
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                                       0x00000004L
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR_MASK                                        0x00000008L
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM_MASK                                    0x0000FF00L
+#define DP3_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM_MASK                                     0xFFFF0000L
+//DP3_DP_STREAM_SYMBOL_COUNT_STATUS
+#define DP3_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT__SHIFT                                          0x0
+#define DP3_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT_MASK                                            0x0000FFFFL
+//DP3_DP_STREAM_SYMBOL_COUNT_CONTROL
+#define DP3_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE__SHIFT                                  0x0
+#define DP3_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET__SHIFT                                   0x4
+#define DP3_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE_MASK                                    0x00000001L
+#define DP3_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET_MASK                                     0x00000010L
+//DP3_DP_LINK_SYMBOL_COUNT_STATUS0
+#define DP3_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT__SHIFT                                             0x0
+#define DP3_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT_MASK                                               0x0000FFFFL
+//DP3_DP_LINK_SYMBOL_COUNT_STATUS1
+#define DP3_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT__SHIFT                                          0x0
+#define DP3_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT_MASK                                            0xFFFFFFFFL
+//DP3_DP_LINK_SYMBOL_COUNT_CONTROL
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE__SHIFT                                      0x0
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET__SHIFT                                       0x4
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE__SHIFT                                   0x8
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET__SHIFT                                    0xc
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE_MASK                                        0x00000001L
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET_MASK                                         0x00000010L
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE_MASK                                     0x00000100L
+#define DP3_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET_MASK                                      0x00001000L
+//DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT           0x0
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING__SHIFT  0x8
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK             0x00000001L
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+#define DP3_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING_MASK  0x00000100L
+//DP3_DP_DPHY_FAST_TRAINING
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT                                       0x0
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT                                         0x1
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT                            0x2
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING__SHIFT                              0x4
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT                                         0x8
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT                                         0x14
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK                                         0x00000001L
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK                                           0x00000002L
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK                              0x00000004L
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING_MASK                                0x00000010L
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK                                           0x000FFF00L
+#define DP3_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK                                           0xFFF00000L
+//DP3_DP_DPHY_FAST_TRAINING_STATUS
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT                                     0x0
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT                         0x4
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT                             0x8
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT                              0xc
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK                                       0x00000007L
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK                           0x00000010L
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK                               0x00000100L
+#define DP3_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK                                0x00001000L
+
+
+// addressBlock: dce_dc_dio_dig4_vpg_vpg_dispdec
+//VPG4_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG4_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG4_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG4_VPG_GENERIC_PACKET_DATA
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG4_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG4_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG4_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG4_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG4_VPG_GENERIC_STATUS
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG4_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG4_VPG_MEM_PWR
+#define VPG4_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG4_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG4_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG4_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG4_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG4_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG4_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG4_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG4_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG4_VPG_ISRC1_2_DATA
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG4_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_dio_dig4_apg_apg_dispdec
+//APG4_APG_DBG_GEN_CONTROL
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG4_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG4_APG_MEM_PWR
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG4_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG4_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG4_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig4_dme_dme_dispdec
+//DME4_DME_CONTROL
+#define DME4_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME4_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME4_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME4_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME4_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME4_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME4_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME4_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME4_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME4_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME4_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME4_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME4_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME4_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME4_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME4_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME4_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME4_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME4_DME_MEMORY_CONTROL
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME4_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_dio_dig4_dispdec
+//DIG4_DIG_FE_CNTL
+#define DIG4_DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT                                                            0x0
+#define DIG4_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT                                                        0x4
+#define DIG4_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT                                                       0x8
+#define DIG4_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT__SHIFT                                                    0xc
+#define DIG4_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN__SHIFT                                                        0x14
+#define DIG4_DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK                                                              0x00000007L
+#define DIG4_DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK                                                          0x00000070L
+#define DIG4_DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK                                                         0x00000100L
+#define DIG4_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_SELECT_MASK                                                      0x00007000L
+#define DIG4_DIG_FE_CNTL__DIG_DIGITAL_BYPASS_EN_MASK                                                          0x00100000L
+//DIG4_DIG_FE_CLK_CNTL
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_MODE__SHIFT                                                              0x0
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN__SHIFT                                                            0x4
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON__SHIFT                                                0xa
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON__SHIFT                                              0xb
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON__SHIFT                                          0xc
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON__SHIFT                                         0xd
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON__SHIFT                                             0xe
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_MODE_MASK                                                                0x00000007L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_CLK_EN_MASK                                                              0x00000010L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_DISPCLK_G_CLOCK_ON_MASK                                                  0x00000400L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_CLOCK_ON_MASK                                                0x00000800L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_APG_CLOCK_ON_MASK                                            0x00001000L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SYMCLK_FE_G_TMDS_CLOCK_ON_MASK                                           0x00002000L
+#define DIG4_DIG_FE_CLK_CNTL__DIG_FE_SOCCLK_G_APG_CLOCK_ON_MASK                                               0x00004000L
+//DIG4_DIG_FE_EN_CNTL
+#define DIG4_DIG_FE_EN_CNTL__DIG_FE_ENABLE__SHIFT                                                             0x0
+#define DIG4_DIG_FE_EN_CNTL__DIG_FE_ENABLE_MASK                                                               0x00000001L
+//DIG4_DIG_OUTPUT_CRC_CNTL
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT                                                    0x0
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT                                              0x4
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT                                              0x8
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK                                                      0x00000001L
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK                                                0x00000010L
+#define DIG4_DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK                                                0x00000300L
+//DIG4_DIG_OUTPUT_CRC_RESULT
+#define DIG4_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT                                              0x0
+#define DIG4_DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK                                                0x3FFFFFFFL
+//DIG4_DIG_CLOCK_PATTERN
+#define DIG4_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT                                                      0x0
+#define DIG4_DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK                                                        0x000003FFL
+//DIG4_DIG_TEST_PATTERN
+#define DIG4_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT                                                 0x0
+#define DIG4_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT                                              0x1
+#define DIG4_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT                                               0x4
+#define DIG4_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT                                                0x5
+#define DIG4_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT                                      0x6
+#define DIG4_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT                                                 0x10
+#define DIG4_DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK                                                   0x00000001L
+#define DIG4_DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK                                                0x00000002L
+#define DIG4_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK                                                 0x00000010L
+#define DIG4_DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK                                                  0x00000020L
+#define DIG4_DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK                                        0x00000040L
+#define DIG4_DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK                                                   0x03FF0000L
+//DIG4_DIG_RANDOM_PATTERN_SEED
+#define DIG4_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT                                          0x0
+#define DIG4_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT                                       0x18
+#define DIG4_DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK                                            0x00FFFFFFL
+#define DIG4_DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK                                         0x01000000L
+//DIG4_DIG_FIFO_CTRL0
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE__SHIFT                                                           0x0
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_RESET__SHIFT                                                            0x1
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL__SHIFT                                                 0x2
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC__SHIFT                                                   0x8
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                           0x9
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE__SHIFT                                                       0x14
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_ERROR__SHIFT                                                            0x1c
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_ENABLE_MASK                                                             0x00000001L
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_RESET_MASK                                                              0x00000002L
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_READ_START_LEVEL_MASK                                                   0x000000FCL
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_READ_CLOCK_SRC_MASK                                                     0x00000100L
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_OUTPUT_PIXEL_PER_CYCLE_MASK                                             0x00000600L
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_RESET_DONE_MASK                                                         0x00100000L
+#define DIG4_DIG_FIFO_CTRL0__DIG_FIFO_ERROR_MASK                                                              0x30000000L
+//DIG4_DIG_FIFO_CTRL1
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT                                              0x1
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL__SHIFT                                                  0x2
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY__SHIFT                                                  0x8
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT                                                0xa
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL__SHIFT                                                    0x10
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL__SHIFT                                                    0x16
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED__SHIFT                                                       0x1d
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT                                              0x1e
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT                                              0x1f
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK                                                0x00000002L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_OVERWRITE_LEVEL_MASK                                                    0x000000FCL
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_LEVEL_DET_DELAY_MASK                                                    0x00000300L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK                                                  0x0000FC00L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_MAXIMUM_LEVEL_MASK                                                      0x001F0000L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_MINIMUM_LEVEL_MASK                                                      0x03C00000L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_CALIBRATED_MASK                                                         0x20000000L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK                                                0x40000000L
+#define DIG4_DIG_FIFO_CTRL1__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK                                                0x80000000L
+//DIG4_HDMI_METADATA_PACKET_CONTROL
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                                 0x0
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                         0x4
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                                 0x8
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                   0x10
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                   0x00000001L
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                           0x00000010L
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                   0x00000100L
+#define DIG4_HDMI_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                     0xFFFF0000L
+//DIG4_HDMI_CONTROL
+#define DIG4_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN__SHIFT                                                       0x1
+#define DIG4_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE__SHIFT                                                     0x2
+#define DIG4_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED__SHIFT                                            0x3
+#define DIG4_HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT                                                              0x8
+#define DIG4_HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT                                                             0x9
+#define DIG4_HDMI_CONTROL__DOLBY_VISION_EN__SHIFT                                                             0xa
+#define DIG4_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED__SHIFT                                         0xb
+#define DIG4_HDMI_CONTROL__TMDS_PIXEL_ENCODING__SHIFT                                                         0xc
+#define DIG4_HDMI_CONTROL__TMDS_COLOR_FORMAT__SHIFT                                                           0xd
+#define DIG4_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM__SHIFT                                           0x10
+#define DIG4_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT                                                      0x18
+#define DIG4_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                       0x1c
+#define DIG4_HDMI_CONTROL__HDMI_DATA_SCRAMBLE_EN_MASK                                                         0x00000002L
+#define DIG4_HDMI_CONTROL__HDMI_CLOCK_CHANNEL_RATE_MASK                                                       0x00000004L
+#define DIG4_HDMI_CONTROL__HDMI_NO_EXTRA_NULL_PACKET_FILLED_MASK                                              0x00000008L
+#define DIG4_HDMI_CONTROL__HDMI_ERROR_ACK_MASK                                                                0x00000100L
+#define DIG4_HDMI_CONTROL__HDMI_ERROR_MASK_MASK                                                               0x00000200L
+#define DIG4_HDMI_CONTROL__DOLBY_VISION_EN_MASK                                                               0x00000400L
+#define DIG4_HDMI_CONTROL__DOLBY_VISION_METADATA_PACKET_MISSED_MASK                                           0x00000800L
+#define DIG4_HDMI_CONTROL__TMDS_PIXEL_ENCODING_MASK                                                           0x00001000L
+#define DIG4_HDMI_CONTROL__TMDS_COLOR_FORMAT_MASK                                                             0x00006000L
+#define DIG4_HDMI_CONTROL__HDMI_UNSCRAMBLED_CONTROL_LINE_NUM_MASK                                             0x003F0000L
+#define DIG4_HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK                                                        0x01000000L
+#define DIG4_HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK                                                         0x30000000L
+//DIG4_HDMI_STATUS
+#define DIG4_HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT                                                           0x0
+#define DIG4_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT                                                      0x10
+#define DIG4_HDMI_STATUS__HDMI_ERROR_INT__SHIFT                                                               0x1b
+#define DIG4_HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK                                                             0x00000001L
+#define DIG4_HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK                                                        0x00010000L
+#define DIG4_HDMI_STATUS__HDMI_ERROR_INT_MASK                                                                 0x08000000L
+//DIG4_HDMI_AUDIO_PACKET_CONTROL
+#define DIG4_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT                                            0x4
+#define DIG4_HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK                                              0x00000030L
+//DIG4_HDMI_ACR_PACKET_CONTROL
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                    0x0
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                    0x1
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                  0x4
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                  0x8
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                               0xc
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                              0x10
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                      0x00000001L
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                      0x00000002L
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                    0x00000030L
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                    0x00000100L
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                                 0x00001000L
+#define DIG4_HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                                0x00070000L
+//DIG4_HDMI_VBI_PACKET_CONTROL
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT                                                   0x0
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT                                                     0x4
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT                                                     0x5
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT                                                   0x8
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT                                                   0x9
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND__SHIFT                                                    0xc
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT                                                   0x10
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE__SHIFT                                                    0x18
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK                                                     0x00000001L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK                                                       0x00000010L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK                                                       0x00000020L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK                                                     0x00000100L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK                                                     0x00000200L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_SEND_MASK                                                      0x00001000L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK                                                     0x003F0000L
+#define DIG4_HDMI_VBI_PACKET_CONTROL__HDMI_ACP_LINE_MASK                                                      0x3F000000L
+//DIG4_HDMI_INFOFRAME_CONTROL0
+#define DIG4_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT                                             0x4
+#define DIG4_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE__SHIFT                                             0x8
+#define DIG4_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK                                               0x00000010L
+#define DIG4_HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_LINE_MASK                                               0x00003F00L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                          0x1
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                                0x2
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                          0x4
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                          0x5
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                                0x6
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                          0x8
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                          0x9
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                                0xa
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE__SHIFT                           0xb
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                          0xc
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                          0xd
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                                0xe
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE__SHIFT                           0xf
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                          0x11
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                                0x12
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE__SHIFT                           0x13
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                          0x14
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                          0x15
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                                0x16
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE__SHIFT                           0x17
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                          0x18
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                          0x19
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                                0x1a
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE__SHIFT                           0x1b
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                          0x1c
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                          0x1d
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                                0x1e
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE__SHIFT                           0x1f
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                            0x00000001L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                            0x00000002L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                            0x00000010L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                            0x00000020L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                            0x00000100L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                            0x00000200L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                  0x00000400L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_UPDATE_LOCK_DISABLE_MASK                             0x00000800L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                            0x00001000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                            0x00002000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                  0x00004000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_UPDATE_LOCK_DISABLE_MASK                             0x00008000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                            0x00010000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                            0x00020000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                  0x00040000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_UPDATE_LOCK_DISABLE_MASK                             0x00080000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                            0x00100000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                            0x00200000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                  0x00400000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_UPDATE_LOCK_DISABLE_MASK                             0x00800000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                            0x01000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                            0x02000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                  0x04000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_UPDATE_LOCK_DISABLE_MASK                             0x08000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                            0x10000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                            0x20000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                  0x40000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_UPDATE_LOCK_DISABLE_MASK                             0x80000000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL6
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT__SHIFT                                          0x1
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                                0x2
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE__SHIFT                           0x3
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND__SHIFT                                          0x4
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT__SHIFT                                          0x5
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                                0x6
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE__SHIFT                           0x7
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND__SHIFT                                         0x8
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT__SHIFT                                         0x9
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                               0xa
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE__SHIFT                          0xb
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND__SHIFT                                         0xc
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT__SHIFT                                         0xd
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                               0xe
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE__SHIFT                          0xf
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND__SHIFT                                         0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT__SHIFT                                         0x11
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                               0x12
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE__SHIFT                          0x13
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND__SHIFT                                         0x14
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT__SHIFT                                         0x15
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                               0x16
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE__SHIFT                          0x17
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND__SHIFT                                         0x18
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT__SHIFT                                         0x19
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                               0x1a
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE__SHIFT                          0x1b
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_SEND_MASK                                            0x00000001L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_CONT_MASK                                            0x00000002L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_LINE_REFERENCE_MASK                                  0x00000004L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC8_UPDATE_LOCK_DISABLE_MASK                             0x00000008L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_SEND_MASK                                            0x00000010L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_CONT_MASK                                            0x00000020L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_LINE_REFERENCE_MASK                                  0x00000040L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC9_UPDATE_LOCK_DISABLE_MASK                             0x00000080L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_SEND_MASK                                           0x00000100L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_CONT_MASK                                           0x00000200L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_LINE_REFERENCE_MASK                                 0x00000400L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC10_UPDATE_LOCK_DISABLE_MASK                            0x00000800L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_SEND_MASK                                           0x00001000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_CONT_MASK                                           0x00002000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_LINE_REFERENCE_MASK                                 0x00004000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC11_UPDATE_LOCK_DISABLE_MASK                            0x00008000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_SEND_MASK                                           0x00010000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_CONT_MASK                                           0x00020000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_LINE_REFERENCE_MASK                                 0x00040000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC12_UPDATE_LOCK_DISABLE_MASK                            0x00080000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_SEND_MASK                                           0x00100000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_CONT_MASK                                           0x00200000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_LINE_REFERENCE_MASK                                 0x00400000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC13_UPDATE_LOCK_DISABLE_MASK                            0x00800000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_SEND_MASK                                           0x01000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_CONT_MASK                                           0x02000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_LINE_REFERENCE_MASK                                 0x04000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL6__HDMI_GENERIC14_UPDATE_LOCK_DISABLE_MASK                            0x08000000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL5
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                                0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                        0x1
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                                0x2
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                        0x3
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                                0x4
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                        0x5
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                                0x6
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                        0x7
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                                0x8
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                        0x9
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                                0xa
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                        0xb
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                                0xc
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                        0xd
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                                0xe
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                        0xf
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                                0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                        0x11
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                                0x12
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                        0x13
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                               0x14
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                       0x15
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                               0x16
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                       0x17
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                               0x18
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                       0x19
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                               0x1a
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                       0x1b
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                               0x1c
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                       0x1d
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                  0x00000001L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                          0x00000002L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                  0x00000004L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                          0x00000008L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                  0x00000010L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                          0x00000020L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                  0x00000040L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                          0x00000080L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                  0x00000100L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                          0x00000200L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                  0x00000400L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                          0x00000800L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                  0x00001000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                          0x00002000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                  0x00004000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                          0x00008000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                  0x00010000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                          0x00020000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                  0x00040000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                          0x00080000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                                 0x00100000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                         0x00200000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                                 0x00400000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                         0x00800000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                                 0x01000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                         0x02000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                                 0x04000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                         0x08000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                                 0x10000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL5__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                         0x20000000L
+//DIG4_HDMI_GC
+#define DIG4_HDMI_GC__HDMI_GC_AVMUTE__SHIFT                                                                   0x0
+#define DIG4_HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT                                                              0x2
+#define DIG4_HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT                                                               0x4
+#define DIG4_HDMI_GC__HDMI_PACKING_PHASE__SHIFT                                                               0x8
+#define DIG4_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT                                                      0xc
+#define DIG4_HDMI_GC__HDMI_GC_AVMUTE_MASK                                                                     0x00000001L
+#define DIG4_HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK                                                                0x00000004L
+#define DIG4_HDMI_GC__HDMI_DEFAULT_PHASE_MASK                                                                 0x00000010L
+#define DIG4_HDMI_GC__HDMI_PACKING_PHASE_MASK                                                                 0x00000F00L
+#define DIG4_HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK                                                        0x00001000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL1
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC0_LINE_MASK                                            0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC1_LINE_MASK                                            0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL2
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_LINE_MASK                                            0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_LINE_MASK                                            0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL3
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC4_LINE_MASK                                            0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL3__HDMI_GENERIC5_LINE_MASK                                            0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL4
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC6_LINE_MASK                                            0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL4__HDMI_GENERIC7_LINE_MASK                                            0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL7
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE__SHIFT                                          0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE__SHIFT                                          0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC8_LINE_MASK                                            0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL7__HDMI_GENERIC9_LINE_MASK                                            0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL8
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE__SHIFT                                         0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE__SHIFT                                         0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC10_LINE_MASK                                           0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL8__HDMI_GENERIC11_LINE_MASK                                           0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL9
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE__SHIFT                                         0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE__SHIFT                                         0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC12_LINE_MASK                                           0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL9__HDMI_GENERIC13_LINE_MASK                                           0xFFFF0000L
+//DIG4_HDMI_GENERIC_PACKET_CONTROL10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE__SHIFT                                        0x0
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING__SHIFT                                0x10
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING__SHIFT                                0x11
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING__SHIFT                                0x12
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING__SHIFT                                0x13
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING__SHIFT                                0x14
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING__SHIFT                                0x15
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING__SHIFT                                0x16
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING__SHIFT                                0x17
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING__SHIFT                                0x18
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING__SHIFT                                0x19
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING__SHIFT                               0x1a
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING__SHIFT                               0x1b
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING__SHIFT                               0x1c
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING__SHIFT                               0x1d
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING__SHIFT                               0x1e
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_LINE_MASK                                          0x0000FFFFL
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC0_EN_DB_PENDING_MASK                                  0x00010000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC1_EN_DB_PENDING_MASK                                  0x00020000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC2_EN_DB_PENDING_MASK                                  0x00040000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC3_EN_DB_PENDING_MASK                                  0x00080000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC4_EN_DB_PENDING_MASK                                  0x00100000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC5_EN_DB_PENDING_MASK                                  0x00200000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC6_EN_DB_PENDING_MASK                                  0x00400000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC7_EN_DB_PENDING_MASK                                  0x00800000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC8_EN_DB_PENDING_MASK                                  0x01000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC9_EN_DB_PENDING_MASK                                  0x02000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC10_EN_DB_PENDING_MASK                                 0x04000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC11_EN_DB_PENDING_MASK                                 0x08000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC12_EN_DB_PENDING_MASK                                 0x10000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC13_EN_DB_PENDING_MASK                                 0x20000000L
+#define DIG4_HDMI_GENERIC_PACKET_CONTROL10__HDMI_GENERIC14_EN_DB_PENDING_MASK                                 0x40000000L
+//DIG4_HDMI_DB_CONTROL
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                          0x0
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_TAKEN__SHIFT                                                            0x4
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR__SHIFT                                                        0x5
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_LOCK__SHIFT                                                             0x8
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                          0xc
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                       0xf
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN__SHIFT                                                         0x10
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR__SHIFT                                                     0x11
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_PENDING_MASK                                                            0x00000001L
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_TAKEN_MASK                                                              0x00000010L
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_TAKEN_CLR_MASK                                                          0x00000020L
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_LOCK_MASK                                                               0x00000100L
+#define DIG4_HDMI_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                            0x00001000L
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                         0x00008000L
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_MASK                                                           0x00010000L
+#define DIG4_HDMI_DB_CONTROL__VUPDATE_DB_TAKEN_CLR_MASK                                                       0x00020000L
+//DIG4_HDMI_ACR_32_0
+#define DIG4_HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                            0xc
+#define DIG4_HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                              0xFFFFF000L
+//DIG4_HDMI_ACR_32_1
+#define DIG4_HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                              0x0
+#define DIG4_HDMI_ACR_32_1__HDMI_ACR_N_32_MASK                                                                0x000FFFFFL
+//DIG4_HDMI_ACR_44_0
+#define DIG4_HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                            0xc
+#define DIG4_HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                              0xFFFFF000L
+//DIG4_HDMI_ACR_44_1
+#define DIG4_HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                              0x0
+#define DIG4_HDMI_ACR_44_1__HDMI_ACR_N_44_MASK                                                                0x000FFFFFL
+//DIG4_HDMI_ACR_48_0
+#define DIG4_HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                            0xc
+#define DIG4_HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                              0xFFFFF000L
+//DIG4_HDMI_ACR_48_1
+#define DIG4_HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                              0x0
+#define DIG4_HDMI_ACR_48_1__HDMI_ACR_N_48_MASK                                                                0x000FFFFFL
+//DIG4_HDMI_ACR_STATUS_0
+#define DIG4_HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                           0xc
+#define DIG4_HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                             0xFFFFF000L
+//DIG4_HDMI_ACR_STATUS_1
+#define DIG4_HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                             0x0
+#define DIG4_HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK                                                               0x000FFFFFL
+//DIG4_DIG_FE_AUDIO_CNTL
+#define DIG4_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT                               0x0
+#define DIG4_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE__SHIFT                                                       0x8
+#define DIG4_DIG_FE_AUDIO_CNTL__DIG_FE_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                                 0x00000007L
+#define DIG4_DIG_FE_AUDIO_CNTL__APG_CLOCK_ENABLE_MASK                                                         0x00000100L
+//DIG4_DIG_BE_CLK_CNTL
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_MODE__SHIFT                                                              0x0
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN__SHIFT                                                            0x4
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET__SHIFT                                                        0x5
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON__SHIFT                                                 0xb
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON__SHIFT                                            0xd
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_MODE_MASK                                                                0x00000007L
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_CLK_EN_MASK                                                              0x00000010L
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SOFT_RESET_MASK                                                          0x00000020L
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_CLOCK_ON_MASK                                                   0x00000800L
+#define DIG4_DIG_BE_CLK_CNTL__DIG_BE_SYMCLK_G_TMDS_CLOCK_ON_MASK                                              0x00002000L
+//DIG4_DIG_BE_CNTL
+#define DIG4_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT                                                         0x0
+#define DIG4_DIG_BE_CNTL__DIG_SWAP__SHIFT                                                                     0x1
+#define DIG4_DIG_BE_CNTL__DIG_RB_SWITCH_EN__SHIFT                                                             0x2
+#define DIG4_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT                                                         0x8
+#define DIG4_DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT                                                               0x1c
+#define DIG4_DIG_BE_CNTL__DIG_DUAL_LINK_ENABLE_MASK                                                           0x00000001L
+#define DIG4_DIG_BE_CNTL__DIG_SWAP_MASK                                                                       0x00000002L
+#define DIG4_DIG_BE_CNTL__DIG_RB_SWITCH_EN_MASK                                                               0x00000004L
+#define DIG4_DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK                                                           0x00007F00L
+#define DIG4_DIG_BE_CNTL__DIG_HPD_SELECT_MASK                                                                 0x70000000L
+//DIG4_DIG_BE_EN_CNTL
+#define DIG4_DIG_BE_EN_CNTL__DIG_BE_ENABLE__SHIFT                                                             0x0
+#define DIG4_DIG_BE_EN_CNTL__DIG_BE_ENABLE_MASK                                                               0x00000001L
+//DIG4_TMDS_CNTL
+#define DIG4_TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT                                                                0x0
+#define DIG4_TMDS_CNTL__TMDS_SYNC_PHASE_MASK                                                                  0x00000001L
+//DIG4_TMDS_CONTROL_CHAR
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT                                              0x0
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT                                              0x1
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT                                              0x2
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT                                              0x3
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK                                                0x00000001L
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK                                                0x00000002L
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK                                                0x00000004L
+#define DIG4_TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK                                                0x00000008L
+//DIG4_TMDS_CONTROL0_FEEDBACK
+#define DIG4_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT                                     0x0
+#define DIG4_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT                                      0x8
+#define DIG4_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK                                       0x00000003L
+#define DIG4_TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK                                        0x00000300L
+//DIG4_TMDS_STEREOSYNC_CTL_SEL
+#define DIG4_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT                                          0x0
+#define DIG4_TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK                                            0x00000003L
+//DIG4_TMDS_SYNC_CHAR_PATTERN_0_1
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT                                       0x0
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT                                       0x10
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK                                         0x000003FFL
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK                                         0x03FF0000L
+//DIG4_TMDS_SYNC_CHAR_PATTERN_2_3
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT                                       0x0
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT                                       0x10
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK                                         0x000003FFL
+#define DIG4_TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK                                         0x03FF0000L
+//DIG4_TMDS_CTL_BITS
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL0__SHIFT                                                                  0x0
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL1__SHIFT                                                                  0x8
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL2__SHIFT                                                                  0x10
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL3__SHIFT                                                                  0x18
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL0_MASK                                                                    0x00000001L
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL1_MASK                                                                    0x00000100L
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL2_MASK                                                                    0x00010000L
+#define DIG4_TMDS_CTL_BITS__TMDS_CTL3_MASK                                                                    0x01000000L
+//DIG4_TMDS_DCBALANCER_CONTROL
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT                                               0x0
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT                                               0x4
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT                                          0x8
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT                                          0x10
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT                                            0x18
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK                                                 0x00000001L
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK                                                 0x00000070L
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK                                            0x00000100L
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK                                            0x000F0000L
+#define DIG4_TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK                                              0x01000000L
+//DIG4_TMDS_SYNC_DCBALANCE_CHAR
+#define DIG4_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01__SHIFT                                          0x0
+#define DIG4_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11__SHIFT                                          0x10
+#define DIG4_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR01_MASK                                            0x000003FFL
+#define DIG4_TMDS_SYNC_DCBALANCE_CHAR__TMDS_SYNC_DCBAL_CHAR11_MASK                                            0x03FF0000L
+//DIG4_TMDS_CTL0_1_GEN_CNTL
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT                                                  0x0
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT                                                0x4
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT                                               0x7
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT                                           0x8
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT                                                  0x10
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT                                                0x14
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT                                               0x17
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT                                           0x18
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT                                                0x1f
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK                                              0x10000000L
+#define DIG4_TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK                                                  0x80000000L
+//DIG4_TMDS_CTL2_3_GEN_CNTL
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT                                                  0x0
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT                                                0x4
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT                                               0x7
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT                                           0x8
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT                                         0xa
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT                                              0xb
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT                                            0xc
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT                                                  0x10
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT                                                0x14
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT                                               0x17
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT                                           0x18
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT                                         0x1a
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT                                              0x1b
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT                                            0x1c
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK                                                    0x0000000FL
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK                                                  0x00000070L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK                                                 0x00000080L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK                                             0x00000300L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK                                           0x00000400L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK                                                0x00000800L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK                                              0x00001000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK                                                    0x000F0000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK                                                  0x00700000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK                                                 0x00800000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK                                             0x03000000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK                                           0x04000000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK                                                0x08000000L
+#define DIG4_TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK                                              0x10000000L
+//DIG4_DIG_VERSION
+#define DIG4_DIG_VERSION__DIG_TYPE__SHIFT                                                                     0x0
+#define DIG4_DIG_VERSION__DIG_TYPE_MASK                                                                       0x00000001L
+
+
+// addressBlock: dce_dc_dio_dp4_dispdec
+//DP4_DP_LINK_CNTL
+#define DP4_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT                                                    0x4
+#define DP4_DP_LINK_CNTL__DP_LINK_STATUS__SHIFT                                                               0x8
+#define DP4_DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK                                                      0x00000010L
+#define DP4_DP_LINK_CNTL__DP_LINK_STATUS_MASK                                                                 0x00000100L
+//DP4_DP_PIXEL_FORMAT
+#define DP4_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                                                       0x0
+#define DP4_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT__SHIFT                                                 0x4
+#define DP4_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                                              0x8
+#define DP4_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT__SHIFT                                                   0x10
+#define DP4_DP_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                                         0x00000001L
+#define DP4_DP_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_FORMAT_MASK                                                   0x00000030L
+#define DP4_DP_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                                                0x00000700L
+#define DP4_DP_PIXEL_FORMAT__COMPRESSED_PIXEL_FORMAT_MASK                                                     0x00010000L
+//DP4_DP_MSA_COLORIMETRY
+#define DP4_DP_MSA_COLORIMETRY__DP_MSA_MISC0__SHIFT                                                           0x18
+#define DP4_DP_MSA_COLORIMETRY__DP_MSA_MISC0_MASK                                                             0xFF000000L
+//DP4_DP_CONFIG
+#define DP4_DP_CONFIG__DP_UDI_LANES__SHIFT                                                                    0x0
+#define DP4_DP_CONFIG__DP_UDI_LANES_MASK                                                                      0x00000003L
+//DP4_DP_VID_STREAM_CNTL
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT                                                   0x0
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT                                                0x8
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT                                                   0x10
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT                                           0x14
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK                                                     0x00000001L
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK                                                  0x00000300L
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK                                                     0x00010000L
+#define DP4_DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK                                             0x00100000L
+//DP4_DP_STEER_FIFO
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE__SHIFT                                                        0x0
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT                                                         0x1
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE__SHIFT                                                    0x2
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT                                                      0x4
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT                                                       0x5
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT                                                       0x6
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT                                                      0x7
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_ENABLE_MASK                                                          0x00000001L
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK                                                           0x00000002L
+#define DP4_DP_STEER_FIFO__DP_STEER_FIFO_RESET_DONE_MASK                                                      0x00000004L
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK                                                        0x00000010L
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK                                                         0x00000020L
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK                                                         0x00000040L
+#define DP4_DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK                                                        0x00000080L
+//DP4_DP_MSA_MISC
+#define DP4_DP_MSA_MISC__DP_MSA_MISC1__SHIFT                                                                  0x0
+#define DP4_DP_MSA_MISC__DP_MSA_MISC2__SHIFT                                                                  0x8
+#define DP4_DP_MSA_MISC__DP_MSA_MISC3__SHIFT                                                                  0x10
+#define DP4_DP_MSA_MISC__DP_MSA_MISC4__SHIFT                                                                  0x18
+#define DP4_DP_MSA_MISC__DP_MSA_MISC1_MASK                                                                    0x000000FFL
+#define DP4_DP_MSA_MISC__DP_MSA_MISC2_MASK                                                                    0x0000FF00L
+#define DP4_DP_MSA_MISC__DP_MSA_MISC3_MASK                                                                    0x00FF0000L
+#define DP4_DP_MSA_MISC__DP_MSA_MISC4_MASK                                                                    0xFF000000L
+//DP4_DP_DPHY_INTERNAL_CTRL
+#define DP4_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN__SHIFT                                         0x0
+#define DP4_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL__SHIFT                                        0x4
+#define DP4_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_EN_MASK                                           0x00000001L
+#define DP4_DP_DPHY_INTERNAL_CTRL__DPHY_ALT_SCRAMBLER_RESET_SEL_MASK                                          0x00000010L
+//DP4_DP_VID_TIMING
+#define DP4_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE__SHIFT                                               0x4
+#define DP4_DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT                                                           0x8
+#define DP4_DP_VID_TIMING__DP_VID_M_DIV__SHIFT                                                                0xc
+#define DP4_DP_VID_TIMING__DP_VID_N_INTERVAL__SHIFT                                                           0xe
+#define DP4_DP_VID_TIMING__DP_VID_N_DIV__SHIFT                                                                0x18
+#define DP4_DP_VID_TIMING__DP_VID_M_N_DOUBLE_BUFFER_MODE_MASK                                                 0x00000010L
+#define DP4_DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK                                                             0x00000100L
+#define DP4_DP_VID_TIMING__DP_VID_M_DIV_MASK                                                                  0x00003000L
+#define DP4_DP_VID_TIMING__DP_VID_N_INTERVAL_MASK                                                             0x0000C000L
+#define DP4_DP_VID_TIMING__DP_VID_N_DIV_MASK                                                                  0xFF000000L
+//DP4_DP_VID_N
+#define DP4_DP_VID_N__DP_VID_N__SHIFT                                                                         0x0
+#define DP4_DP_VID_N__DP_VID_N_MASK                                                                           0x00FFFFFFL
+//DP4_DP_VID_M
+#define DP4_DP_VID_M__DP_VID_M__SHIFT                                                                         0x0
+#define DP4_DP_VID_M__DP_VID_M_MASK                                                                           0x00FFFFFFL
+//DP4_DP_LINK_FRAMING_CNTL
+#define DP4_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT                                                  0x0
+#define DP4_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE__SHIFT                                  0x14
+#define DP4_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT                                                      0x18
+#define DP4_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT                                           0x1c
+#define DP4_DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK                                                    0x0003FFFFL
+#define DP4_DP_LINK_FRAMING_CNTL__DP_BACK_TO_BACK_BS_AVOIDANCE_ENABLE_MASK                                    0x00100000L
+#define DP4_DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK                                                        0x01000000L
+#define DP4_DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK                                             0x10000000L
+//DP4_DP_HBR2_EYE_PATTERN
+#define DP4_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT                                            0x0
+#define DP4_DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK                                              0x00000001L
+//DP4_DP_VID_MSA_VBID
+#define DP4_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT                                                       0x0
+#define DP4_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE__SHIFT                                            0xc
+#define DP4_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT                                                     0x18
+#define DP4_DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK                                                         0x00000FFFL
+#define DP4_DP_VID_MSA_VBID__DP_VID_MSA_TRANSMISSION_ENABLE_MASK                                              0x00001000L
+#define DP4_DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK                                                       0x01000000L
+//DP4_DP_VID_INTERRUPT_CNTL
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT                                           0x0
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT                                           0x1
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT                                          0x2
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK                                             0x00000001L
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK                                             0x00000002L
+#define DP4_DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK                                            0x00000004L
+//DP4_DP_DPHY_CNTL
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT                                                         0x0
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT                                                         0x1
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT                                                         0x2
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT                                                         0x3
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_EN__SHIFT                                                                  0x4
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW__SHIFT                                                        0x5
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS__SHIFT                                                       0x6
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM__SHIFT                                               0x7
+#define DP4_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL__SHIFT                                                           0x8
+#define DP4_DP_DPHY_CNTL__DPHY_BYPASS__SHIFT                                                                  0x10
+#define DP4_DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT                                                             0x18
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK                                                           0x00000001L
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK                                                           0x00000002L
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK                                                           0x00000004L
+#define DP4_DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK                                                           0x00000008L
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_EN_MASK                                                                    0x00000010L
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_READY_SHADOW_MASK                                                          0x00000020L
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_ACTIVE_STATUS_MASK                                                         0x00000040L
+#define DP4_DP_DPHY_CNTL__DPHY_FEC_DISABLE_MODE_FOR_ALPM_MASK                                                 0x00000080L
+#define DP4_DP_DPHY_CNTL__DPHY_SCRAMBLER_SEL_MASK                                                             0x00000100L
+#define DP4_DP_DPHY_CNTL__DPHY_BYPASS_MASK                                                                    0x00010000L
+#define DP4_DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK                                                               0x01000000L
+//DP4_DP_DPHY_TRAINING_PATTERN_SEL
+#define DP4_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT                                    0x0
+#define DP4_DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK                                      0x00000003L
+//DP4_DP_DPHY_SYM0
+#define DP4_DP_DPHY_SYM0__DPHY_SYM1__SHIFT                                                                    0x0
+#define DP4_DP_DPHY_SYM0__DPHY_SYM2__SHIFT                                                                    0xa
+#define DP4_DP_DPHY_SYM0__DPHY_SYM3__SHIFT                                                                    0x14
+#define DP4_DP_DPHY_SYM0__DPHY_SYM1_MASK                                                                      0x000003FFL
+#define DP4_DP_DPHY_SYM0__DPHY_SYM2_MASK                                                                      0x000FFC00L
+#define DP4_DP_DPHY_SYM0__DPHY_SYM3_MASK                                                                      0x3FF00000L
+//DP4_DP_DPHY_SYM1
+#define DP4_DP_DPHY_SYM1__DPHY_SYM4__SHIFT                                                                    0x0
+#define DP4_DP_DPHY_SYM1__DPHY_SYM5__SHIFT                                                                    0xa
+#define DP4_DP_DPHY_SYM1__DPHY_SYM6__SHIFT                                                                    0x14
+#define DP4_DP_DPHY_SYM1__DPHY_SYM4_MASK                                                                      0x000003FFL
+#define DP4_DP_DPHY_SYM1__DPHY_SYM5_MASK                                                                      0x000FFC00L
+#define DP4_DP_DPHY_SYM1__DPHY_SYM6_MASK                                                                      0x3FF00000L
+//DP4_DP_DPHY_SYM2
+#define DP4_DP_DPHY_SYM2__DPHY_SYM7__SHIFT                                                                    0x0
+#define DP4_DP_DPHY_SYM2__DPHY_SYM8__SHIFT                                                                    0xa
+#define DP4_DP_DPHY_SYM2__DPHY_SYM7_MASK                                                                      0x000003FFL
+#define DP4_DP_DPHY_SYM2__DPHY_SYM8_MASK                                                                      0x000FFC00L
+//DP4_DP_DPHY_8B10B_CNTL
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT                                                       0x8
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT                                                    0x10
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT                                                    0x18
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK                                                         0x00000100L
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK                                                      0x00010000L
+#define DP4_DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK                                                      0x01000000L
+//DP4_DP_DPHY_PRBS_CNTL
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT                                                            0x0
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT                                                           0x4
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT                                                          0x8
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK                                                              0x00000001L
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK                                                             0x00000030L
+#define DP4_DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK                                                            0x7FFFFF00L
+//DP4_DP_DPHY_SCRAM_CNTL
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS__SHIFT                                                     0x0
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE__SHIFT                                                 0x4
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT__SHIFT                                                0x8
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE__SHIFT                                                   0x18
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_DIS_MASK                                                       0x00000001L
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_ADVANCE_MASK                                                   0x00000010L
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_BS_COUNT_MASK                                                  0x0003FF00L
+#define DP4_DP_DPHY_SCRAM_CNTL__DPHY_SCRAMBLER_KCODE_MASK                                                     0x01000000L
+//DP4_DP_DPHY_CRC_CONTROL0
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN__SHIFT                                                          0x0
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN__SHIFT                                                     0x4
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD__SHIFT                                                       0x8
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_EN_MASK                                                            0x00000001L
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_CONT_EN_MASK                                                       0x00000010L
+#define DP4_DP_DPHY_CRC_CONTROL0__DPHY_CRC_FIELD_MASK                                                         0x00000100L
+//DP4_DP_DPHY_CRC_CONTROL1
+#define DP4_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT__SHIFT                                              0x0
+#define DP4_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT__SHIFT                                               0x8
+#define DP4_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_FIRST_SLOT_MASK                                                0x0000003FL
+#define DP4_DP_DPHY_CRC_CONTROL1__DPHY_CRC_MST_LAST_SLOT_MASK                                                 0x00003F00L
+//DP4_DP_DPHY_CRC_RESULT0
+#define DP4_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP4_DP_DPHY_CRC_RESULT0__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP4_DP_DPHY_CRC_RESULT1
+#define DP4_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP4_DP_DPHY_CRC_RESULT1__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP4_DP_DPHY_CRC_RESULT2
+#define DP4_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP4_DP_DPHY_CRC_RESULT2__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP4_DP_DPHY_CRC_RESULT3
+#define DP4_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT__SHIFT                                                       0x0
+#define DP4_DP_DPHY_CRC_RESULT3__DPHY_CRC_RESULT_MASK                                                         0xFFFFFFFFL
+//DP4_DP_DPHY_CRC_STATUS
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID__SHIFT                                                  0x0
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK__SHIFT                                              0x4
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT                                                0x8
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT                                               0xc
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT                                           0x10
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_MASK                                                    0x00000001L
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_RESULT_VALID_ACK_MASK                                                0x00000010L
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK                                                  0x00000100L
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK                                                 0x00001000L
+#define DP4_DP_DPHY_CRC_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK                                             0x00010000L
+//DP4_DP_TU_CNTL
+#define DP4_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG__SHIFT                                                            0x8
+#define DP4_DP_TU_CNTL__DP_TU_OVERFLOW_ACK__SHIFT                                                             0xc
+#define DP4_DP_TU_CNTL__DP_TU_SIZE__SHIFT                                                                     0x18
+#define DP4_DP_TU_CNTL__DP_TU_OVERFLOW_FLAG_MASK                                                              0x00000100L
+#define DP4_DP_TU_CNTL__DP_TU_OVERFLOW_ACK_MASK                                                               0x00001000L
+#define DP4_DP_TU_CNTL__DP_TU_SIZE_MASK                                                                       0x3F000000L
+//DP4_DP_PIXEL_FORMAT_DB_CNTL
+#define DP4_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE__SHIFT                                            0x0
+#define DP4_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING__SHIFT                                           0x4
+#define DP4_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_ENABLE_MASK                                              0x00000001L
+#define DP4_DP_PIXEL_FORMAT_DB_CNTL__PIXEL_FORMAT_DB_PENDING_MASK                                             0x00000010L
+//DP4_DP_SEC_CNTL
+#define DP4_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT                                                          0x0
+#define DP4_DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT                                                             0x4
+#define DP4_DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT                                                             0x8
+#define DP4_DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT                                                             0xc
+#define DP4_DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT                                                             0x10
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT                                                            0x14
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT                                                            0x15
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT                                                            0x16
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT                                                            0x17
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE__SHIFT                                                            0x18
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE__SHIFT                                                            0x19
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE__SHIFT                                                            0x1a
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE__SHIFT                                                            0x1b
+#define DP4_DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK                                                            0x00000001L
+#define DP4_DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK                                                               0x00000010L
+#define DP4_DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK                                                               0x00000100L
+#define DP4_DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK                                                               0x00001000L
+#define DP4_DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK                                                               0x00010000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK                                                              0x00100000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK                                                              0x00200000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK                                                              0x00400000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK                                                              0x00800000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP4_ENABLE_MASK                                                              0x01000000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP5_ENABLE_MASK                                                              0x02000000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP6_ENABLE_MASK                                                              0x04000000L
+#define DP4_DP_SEC_CNTL__DP_SEC_GSP7_ENABLE_MASK                                                              0x08000000L
+//DP4_DP_SEC_CNTL1
+#define DP4_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT                                                           0x0
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE__SHIFT                                                   0x1
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY__SHIFT                                                         0x4
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND__SHIFT                                                             0x5
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING__SHIFT                                                     0x6
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED__SHIFT                                             0x7
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE__SHIFT                                                   0x9
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE__SHIFT                                                   0xa
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE__SHIFT                                                   0xb
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE__SHIFT                                                   0xc
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE__SHIFT                                                   0xd
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE__SHIFT                                                   0xe
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE__SHIFT                                                   0xf
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK                                                             0x00000001L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_REFERENCE_MASK                                                     0x00000002L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_PRIORITY_MASK                                                           0x00000010L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_MASK                                                               0x00000020L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_PENDING_MASK                                                       0x00000040L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_DEADLINE_MISSED_MASK                                               0x00000080L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP1_LINE_REFERENCE_MASK                                                     0x00000200L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP2_LINE_REFERENCE_MASK                                                     0x00000400L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP3_LINE_REFERENCE_MASK                                                     0x00000800L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP4_LINE_REFERENCE_MASK                                                     0x00001000L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP5_LINE_REFERENCE_MASK                                                     0x00002000L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP6_LINE_REFERENCE_MASK                                                     0x00004000L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP7_LINE_REFERENCE_MASK                                                     0x00008000L
+#define DP4_DP_SEC_CNTL1__DP_SEC_GSP0_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_SEC_FRAMING1
+#define DP4_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT                                               0x0
+#define DP4_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP4_DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK                                                 0x00000FFFL
+#define DP4_DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP4_DP_SEC_FRAMING2
+#define DP4_DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT                                                     0x0
+#define DP4_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT                                              0x10
+#define DP4_DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK                                                       0x0000FFFFL
+#define DP4_DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK                                                0xFFFF0000L
+//DP4_DP_SEC_FRAMING3
+#define DP4_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT                                                    0x0
+#define DP4_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT                                                0x10
+#define DP4_DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK                                                      0x00003FFFL
+#define DP4_DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK                                                  0xFFFF0000L
+//DP4_DP_SEC_FRAMING4
+#define DP4_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING__SHIFT                                                      0x0
+#define DP4_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT                                                   0x14
+#define DP4_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT                                                      0x18
+#define DP4_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT                                                         0x1c
+#define DP4_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT                                                  0x1d
+#define DP4_DP_SEC_FRAMING4__DP_SST_SDP_SPLITTING_MASK                                                        0x00000001L
+#define DP4_DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK                                                     0x00100000L
+#define DP4_DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK                                                        0x01000000L
+#define DP4_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK                                                           0x10000000L
+#define DP4_DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK                                                    0x20000000L
+//DP4_DP_SEC_AUD_N
+#define DP4_DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT                                                                 0x0
+#define DP4_DP_SEC_AUD_N__DP_SEC_AUD_N_MASK                                                                   0x00FFFFFFL
+//DP4_DP_SEC_AUD_N_READBACK
+#define DP4_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT                                               0x0
+#define DP4_DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK                                                 0x00FFFFFFL
+//DP4_DP_SEC_AUD_M
+#define DP4_DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT                                                                 0x0
+#define DP4_DP_SEC_AUD_M__DP_SEC_AUD_M_MASK                                                                   0x00FFFFFFL
+//DP4_DP_SEC_AUD_M_READBACK
+#define DP4_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT                                               0x0
+#define DP4_DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK                                                 0x00FFFFFFL
+//DP4_DP_SEC_TIMESTAMP
+#define DP4_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT                                                    0x0
+#define DP4_DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK                                                      0x00000001L
+//DP4_DP_SEC_PACKET_CNTL
+#define DP4_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT                                                    0x4
+#define DP4_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER__SHIFT                                              0x18
+#define DP4_DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK                                                      0x00000010L
+#define DP4_DP_SEC_PACKET_CNTL__DP_SEC_ATP_VERSION_NUMBER_MASK                                                0x3F000000L
+//DP4_DP_MSE_RATE_CNTL
+#define DP4_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT                                                            0x0
+#define DP4_DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT                                                            0x1a
+#define DP4_DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK                                                              0x03FFFFFFL
+#define DP4_DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK                                                              0xFC000000L
+//DP4_DP_MSE_RATE_UPDATE
+#define DP4_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT                                             0x0
+#define DP4_DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK                                               0x00000001L
+//DP4_DP_MSE_SAT0
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT                                                               0x0
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0__SHIFT                                                           0x4
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0__SHIFT                                                   0x5
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT                                                        0x8
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT                                                               0x10
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1__SHIFT                                                           0x14
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1__SHIFT                                                   0x15
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT                                                        0x18
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK                                                                 0x00000007L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT0_MASK                                                             0x00000010L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE0_MASK                                                     0x00000020L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK                                                          0x00003F00L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK                                                                 0x00070000L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPT1_MASK                                                             0x00100000L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_ENCRYPTION_TYPE1_MASK                                                     0x00200000L
+#define DP4_DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK                                                          0x3F000000L
+//DP4_DP_MSE_SAT1
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT                                                               0x0
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2__SHIFT                                                           0x4
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2__SHIFT                                                   0x5
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT                                                        0x8
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT                                                               0x10
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3__SHIFT                                                           0x14
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3__SHIFT                                                   0x15
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT                                                        0x18
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK                                                                 0x00000007L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT2_MASK                                                             0x00000010L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE2_MASK                                                     0x00000020L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK                                                          0x00003F00L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK                                                                 0x00070000L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPT3_MASK                                                             0x00100000L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_ENCRYPTION_TYPE3_MASK                                                     0x00200000L
+#define DP4_DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK                                                          0x3F000000L
+//DP4_DP_MSE_SAT2
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT                                                               0x0
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4__SHIFT                                                           0x4
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4__SHIFT                                                   0x5
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT                                                        0x8
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT                                                               0x10
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5__SHIFT                                                           0x14
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5__SHIFT                                                   0x15
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT                                                        0x18
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK                                                                 0x00000007L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT4_MASK                                                             0x00000010L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE4_MASK                                                     0x00000020L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK                                                          0x00003F00L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK                                                                 0x00070000L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPT5_MASK                                                             0x00100000L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_ENCRYPTION_TYPE5_MASK                                                     0x00200000L
+#define DP4_DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK                                                          0x3F000000L
+//DP4_DP_MSE_SAT_UPDATE
+#define DP4_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT                                                       0x0
+#define DP4_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT                                                   0x8
+#define DP4_DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK                                                         0x00000003L
+#define DP4_DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK                                                     0x00000100L
+//DP4_DP_MSE_LINK_TIMING
+#define DP4_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT                                                      0x0
+#define DP4_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT                                                       0x10
+#define DP4_DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK                                                        0x000003FFL
+#define DP4_DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK                                                         0x00030000L
+//DP4_DP_MSE_MISC_CNTL
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT                                                        0x0
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT                                                    0x4
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT                                                      0x8
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK                                                          0x00000001L
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK                                                      0x00000010L
+#define DP4_DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK                                                        0x00000100L
+//DP4_DP_DPHY_BS_SR_SWAP_CNTL
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT__SHIFT                                                0x0
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE__SHIFT                                              0xf
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START__SHIFT                                          0x10
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_MASK                                                  0x000003FFL
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_BS_SR_SWAP_DONE_MASK                                                0x00008000L
+#define DP4_DP_DPHY_BS_SR_SWAP_CNTL__DPHY_LOAD_BS_COUNT_START_MASK                                            0x00010000L
+//DP4_DP_DPHY_HBR2_PATTERN_CONTROL
+#define DP4_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL__SHIFT                                 0x0
+#define DP4_DP_DPHY_HBR2_PATTERN_CONTROL__DP_DPHY_HBR2_PATTERN_CONTROL_MASK                                   0x00000007L
+//DP4_DP_MSE_SAT0_STATUS
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS__SHIFT                                                 0x0
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS__SHIFT                                             0x4
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS__SHIFT                                     0x5
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS__SHIFT                                          0x8
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS__SHIFT                                                 0x10
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS__SHIFT                                             0x14
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS__SHIFT                                     0x15
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS__SHIFT                                          0x18
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC0_STATUS_MASK                                                   0x00000007L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT0_STATUS_MASK                                               0x00000010L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE0_STATUS_MASK                                       0x00000020L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT0_STATUS_MASK                                            0x00003F00L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SRC1_STATUS_MASK                                                   0x00070000L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPT1_STATUS_MASK                                               0x00100000L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE1_STATUS_MASK                                       0x00200000L
+#define DP4_DP_MSE_SAT0_STATUS__DP_MSE_SAT_SLOT_COUNT1_STATUS_MASK                                            0x3F000000L
+//DP4_DP_MSE_SAT1_STATUS
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS__SHIFT                                                 0x0
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS__SHIFT                                             0x4
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS__SHIFT                                     0x5
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS__SHIFT                                          0x8
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS__SHIFT                                                 0x10
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS__SHIFT                                             0x14
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS__SHIFT                                     0x15
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS__SHIFT                                          0x18
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC2_STATUS_MASK                                                   0x00000007L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT2_STATUS_MASK                                               0x00000010L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE2_STATUS_MASK                                       0x00000020L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT2_STATUS_MASK                                            0x00003F00L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SRC3_STATUS_MASK                                                   0x00070000L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPT3_STATUS_MASK                                               0x00100000L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE3_STATUS_MASK                                       0x00200000L
+#define DP4_DP_MSE_SAT1_STATUS__DP_MSE_SAT_SLOT_COUNT3_STATUS_MASK                                            0x3F000000L
+//DP4_DP_MSE_SAT2_STATUS
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS__SHIFT                                                 0x0
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS__SHIFT                                             0x4
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS__SHIFT                                     0x5
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS__SHIFT                                          0x8
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS__SHIFT                                                 0x10
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS__SHIFT                                             0x14
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS__SHIFT                                     0x15
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS__SHIFT                                          0x18
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC4_STATUS_MASK                                                   0x00000007L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT4_STATUS_MASK                                               0x00000010L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE4_STATUS_MASK                                       0x00000020L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT4_STATUS_MASK                                            0x00003F00L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SRC5_STATUS_MASK                                                   0x00070000L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPT5_STATUS_MASK                                               0x00100000L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_ENCRYPTION_TYPE5_STATUS_MASK                                       0x00200000L
+#define DP4_DP_MSE_SAT2_STATUS__DP_MSE_SAT_SLOT_COUNT5_STATUS_MASK                                            0x3F000000L
+//DP4_DP_HBLANK_CONTROL
+#define DP4_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                                          0x0
+#define DP4_DP_HBLANK_CONTROL__DP_HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                                            0x0000FFFFL
+//DP4_DP_MSA_TIMING_PARAM1
+#define DP4_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL__SHIFT                                                        0x0
+#define DP4_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL__SHIFT                                                        0x10
+#define DP4_DP_MSA_TIMING_PARAM1__DP_MSA_VTOTAL_MASK                                                          0x0000FFFFL
+#define DP4_DP_MSA_TIMING_PARAM1__DP_MSA_HTOTAL_MASK                                                          0xFFFF0000L
+//DP4_DP_MSA_TIMING_PARAM2
+#define DP4_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART__SHIFT                                                        0x0
+#define DP4_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART__SHIFT                                                        0x10
+#define DP4_DP_MSA_TIMING_PARAM2__DP_MSA_VSTART_MASK                                                          0x0000FFFFL
+#define DP4_DP_MSA_TIMING_PARAM2__DP_MSA_HSTART_MASK                                                          0xFFFF0000L
+//DP4_DP_MSA_TIMING_PARAM3
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH__SHIFT                                                    0x0
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY__SHIFT                                                 0xf
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH__SHIFT                                                    0x10
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY__SHIFT                                                 0x1f
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCWIDTH_MASK                                                      0x00007FFFL
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_VSYNCPOLARITY_MASK                                                   0x00008000L
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCWIDTH_MASK                                                      0x7FFF0000L
+#define DP4_DP_MSA_TIMING_PARAM3__DP_MSA_HSYNCPOLARITY_MASK                                                   0x80000000L
+//DP4_DP_MSA_TIMING_PARAM4
+#define DP4_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT__SHIFT                                                       0x0
+#define DP4_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH__SHIFT                                                        0x10
+#define DP4_DP_MSA_TIMING_PARAM4__DP_MSA_VHEIGHT_MASK                                                         0x0000FFFFL
+#define DP4_DP_MSA_TIMING_PARAM4__DP_MSA_HWIDTH_MASK                                                          0xFFFF0000L
+//DP4_DP_MSO_CNTL
+#define DP4_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK__SHIFT                                                         0x0
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE__SHIFT                                                      0x4
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE__SHIFT                                                         0x8
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE__SHIFT                                                         0xc
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE__SHIFT                                                         0x10
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE__SHIFT                                                         0x14
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE__SHIFT                                                        0x18
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE__SHIFT                                                        0x1c
+#define DP4_DP_MSO_CNTL__DP_MSO_NUM_OF_SSTLINK_MASK                                                           0x00000003L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_STREAM_ENABLE_MASK                                                        0x000000F0L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ASP_ENABLE_MASK                                                           0x00000F00L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ATP_ENABLE_MASK                                                           0x0000F000L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_AIP_ENABLE_MASK                                                           0x000F0000L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_ACM_ENABLE_MASK                                                           0x00F00000L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_GSP0_ENABLE_MASK                                                          0x0F000000L
+#define DP4_DP_MSO_CNTL__DP_MSO_SEC_GSP1_ENABLE_MASK                                                          0xF0000000L
+//DP4_DP_MSO_CNTL1
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE__SHIFT                                                       0x0
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE__SHIFT                                                       0x4
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE__SHIFT                                                       0x8
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE__SHIFT                                                       0xc
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE__SHIFT                                                       0x10
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE__SHIFT                                                       0x14
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE__SHIFT                                                       0x1c
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP2_ENABLE_MASK                                                         0x0000000FL
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP3_ENABLE_MASK                                                         0x000000F0L
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP4_ENABLE_MASK                                                         0x00000F00L
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP5_ENABLE_MASK                                                         0x0000F000L
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP6_ENABLE_MASK                                                         0x000F0000L
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_GSP7_ENABLE_MASK                                                         0x00F00000L
+#define DP4_DP_MSO_CNTL1__DP_MSO_SEC_ISRC_ENABLE_MASK                                                         0xF0000000L
+//DP4_DP_STEER_FIFO_CNTL
+#define DP4_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN__SHIFT                       0x8
+#define DP4_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE__SHIFT                                        0x10
+#define DP4_DP_STEER_FIFO_CNTL__DP_STEER_FIFO_COMPRESSED_LOW_GRANULARITY_MODE_EN_MASK                         0x00000100L
+#define DP4_DP_STEER_FIFO_CNTL__DP_STEER_OUTPUT_PIXEL_PER_CYCLE_MASK                                          0x00030000L
+//DP4_DP_SEC_CNTL2
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND__SHIFT                                                             0x0
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING__SHIFT                                                     0x1
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED__SHIFT                                             0x2
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE__SHIFT                                                    0x3
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND__SHIFT                                                             0x4
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING__SHIFT                                                     0x5
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED__SHIFT                                             0x6
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE__SHIFT                                                    0x7
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND__SHIFT                                                             0x8
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING__SHIFT                                                     0x9
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED__SHIFT                                             0xa
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE__SHIFT                                                    0xb
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND__SHIFT                                                             0xc
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING__SHIFT                                                     0xd
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE__SHIFT                                                    0xf
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND__SHIFT                                                             0x10
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING__SHIFT                                                     0x11
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED__SHIFT                                             0x12
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE__SHIFT                                                    0x13
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND__SHIFT                                                             0x14
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING__SHIFT                                                     0x15
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED__SHIFT                                             0x16
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE__SHIFT                                                    0x17
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND__SHIFT                                                             0x18
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING__SHIFT                                                     0x19
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED__SHIFT                                             0x1a
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE__SHIFT                                                    0x1b
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP11_PPS__SHIFT                                                             0x1c
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_MASK                                                               0x00000001L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_PENDING_MASK                                                       0x00000002L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_DEADLINE_MISSED_MASK                                               0x00000004L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP1_SEND_ANY_LINE_MASK                                                      0x00000008L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_MASK                                                               0x00000010L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_PENDING_MASK                                                       0x00000020L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_DEADLINE_MISSED_MASK                                               0x00000040L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP2_SEND_ANY_LINE_MASK                                                      0x00000080L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_MASK                                                               0x00000100L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_PENDING_MASK                                                       0x00000200L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_DEADLINE_MISSED_MASK                                               0x00000400L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP3_SEND_ANY_LINE_MASK                                                      0x00000800L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_MASK                                                               0x00001000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_PENDING_MASK                                                       0x00002000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP4_SEND_ANY_LINE_MASK                                                      0x00008000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_MASK                                                               0x00010000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_PENDING_MASK                                                       0x00020000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_DEADLINE_MISSED_MASK                                               0x00040000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP5_SEND_ANY_LINE_MASK                                                      0x00080000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_MASK                                                               0x00100000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_PENDING_MASK                                                       0x00200000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_DEADLINE_MISSED_MASK                                               0x00400000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP6_SEND_ANY_LINE_MASK                                                      0x00800000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_MASK                                                               0x01000000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_PENDING_MASK                                                       0x02000000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_DEADLINE_MISSED_MASK                                               0x04000000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP7_SEND_ANY_LINE_MASK                                                      0x08000000L
+#define DP4_DP_SEC_CNTL2__DP_SEC_GSP11_PPS_MASK                                                               0x10000000L
+//DP4_DP_SEC_CNTL3
+#define DP4_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM__SHIFT                                                         0x0
+#define DP4_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_SEC_CNTL3__DP_SEC_GSP1_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP4_DP_SEC_CNTL3__DP_SEC_GSP2_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_SEC_CNTL4
+#define DP4_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM__SHIFT                                                         0x0
+#define DP4_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_SEC_CNTL4__DP_SEC_GSP3_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP4_DP_SEC_CNTL4__DP_SEC_GSP4_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_SEC_CNTL5
+#define DP4_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM__SHIFT                                                         0x0
+#define DP4_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_SEC_CNTL5__DP_SEC_GSP5_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP4_DP_SEC_CNTL5__DP_SEC_GSP6_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_SEC_CNTL6
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM__SHIFT                                                         0x0
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE__SHIFT                                                    0x10
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE__SHIFT                                                    0x11
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE__SHIFT                                                    0x12
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE__SHIFT                                                    0x13
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE__SHIFT                                                    0x14
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE__SHIFT                                                    0x15
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE__SHIFT                                                    0x16
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE__SHIFT                                                    0x17
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE__SHIFT                                                    0x18
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE__SHIFT                                                    0x19
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE__SHIFT                                                   0x1a
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE__SHIFT                                                   0x1b
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP7_LINE_NUM_MASK                                                           0x0000FFFFL
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP0_EN_DB_DISABLE_MASK                                                      0x00010000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP1_EN_DB_DISABLE_MASK                                                      0x00020000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP2_EN_DB_DISABLE_MASK                                                      0x00040000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP3_EN_DB_DISABLE_MASK                                                      0x00080000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP4_EN_DB_DISABLE_MASK                                                      0x00100000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP5_EN_DB_DISABLE_MASK                                                      0x00200000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP6_EN_DB_DISABLE_MASK                                                      0x00400000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP7_EN_DB_DISABLE_MASK                                                      0x00800000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP8_EN_DB_DISABLE_MASK                                                      0x01000000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP9_EN_DB_DISABLE_MASK                                                      0x02000000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP10_EN_DB_DISABLE_MASK                                                     0x04000000L
+#define DP4_DP_SEC_CNTL6__DP_SEC_GSP11_EN_DB_DISABLE_MASK                                                     0x08000000L
+//DP4_DP_SEC_CNTL7
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE__SHIFT                                                      0x0
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE__SHIFT                                                     0x1
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE__SHIFT                                                      0x4
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE__SHIFT                                                     0x5
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE__SHIFT                                                      0x8
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE__SHIFT                                                     0x9
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE__SHIFT                                                      0xc
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE__SHIFT                                                     0xd
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE__SHIFT                                                      0x10
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE__SHIFT                                                     0x11
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE__SHIFT                                                      0x14
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE__SHIFT                                                     0x15
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE__SHIFT                                                      0x18
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE__SHIFT                                                     0x19
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE__SHIFT                                                      0x1c
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE__SHIFT                                                     0x1d
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_ACTIVE_MASK                                                        0x00000001L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP0_SEND_IN_IDLE_MASK                                                       0x00000002L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_ACTIVE_MASK                                                        0x00000010L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP1_SEND_IN_IDLE_MASK                                                       0x00000020L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_ACTIVE_MASK                                                        0x00000100L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP2_SEND_IN_IDLE_MASK                                                       0x00000200L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_ACTIVE_MASK                                                        0x00001000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP3_SEND_IN_IDLE_MASK                                                       0x00002000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_ACTIVE_MASK                                                        0x00010000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP4_SEND_IN_IDLE_MASK                                                       0x00020000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_ACTIVE_MASK                                                        0x00100000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP5_SEND_IN_IDLE_MASK                                                       0x00200000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_ACTIVE_MASK                                                        0x01000000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP6_SEND_IN_IDLE_MASK                                                       0x02000000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_ACTIVE_MASK                                                        0x10000000L
+#define DP4_DP_SEC_CNTL7__DP_SEC_GSP7_SEND_IN_IDLE_MASK                                                       0x20000000L
+//DP4_DP_DB_CNTL
+#define DP4_DP_DB_CNTL__DP_DB_PENDING__SHIFT                                                                  0x0
+#define DP4_DP_DB_CNTL__DP_DB_TAKEN__SHIFT                                                                    0x4
+#define DP4_DP_DB_CNTL__DP_DB_TAKEN_CLR__SHIFT                                                                0x5
+#define DP4_DP_DB_CNTL__DP_DB_LOCK__SHIFT                                                                     0x8
+#define DP4_DP_DB_CNTL__DP_DB_DISABLE__SHIFT                                                                  0xc
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_PENDING__SHIFT                                                          0xf
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN__SHIFT                                                            0x10
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR__SHIFT                                                        0x11
+#define DP4_DP_DB_CNTL__DP_DB_PENDING_MASK                                                                    0x00000001L
+#define DP4_DP_DB_CNTL__DP_DB_TAKEN_MASK                                                                      0x00000010L
+#define DP4_DP_DB_CNTL__DP_DB_TAKEN_CLR_MASK                                                                  0x00000020L
+#define DP4_DP_DB_CNTL__DP_DB_LOCK_MASK                                                                       0x00000100L
+#define DP4_DP_DB_CNTL__DP_DB_DISABLE_MASK                                                                    0x00001000L
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_PENDING_MASK                                                            0x00008000L
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_MASK                                                              0x00010000L
+#define DP4_DP_DB_CNTL__DP_VUPDATE_DB_TAKEN_CLR_MASK                                                          0x00020000L
+//DP4_DP_MSA_VBID_MISC
+#define DP4_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE__SHIFT                                         0x0
+#define DP4_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                                      0x4
+#define DP4_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE__SHIFT                                                        0x8
+#define DP4_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE__SHIFT                                                        0x9
+#define DP4_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN__SHIFT                                                     0xc
+#define DP4_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN__SHIFT                                                     0xd
+#define DP4_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE__SHIFT                                                  0xf
+#define DP4_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM__SHIFT                                                        0x10
+#define DP4_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_MASK                                           0x00000003L
+#define DP4_DP_MSA_VBID_MISC__DP_MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                                        0x00000010L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_MASK                                                          0x00000100L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_MASK                                                          0x00000200L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID1_OVERRIDE_EN_MASK                                                       0x00001000L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID2_OVERRIDE_EN_MASK                                                       0x00002000L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID6_LINE_REFERENCE_MASK                                                    0x00008000L
+#define DP4_DP_MSA_VBID_MISC__DP_VBID6_LINE_NUM_MASK                                                          0xFFFF0000L
+//DP4_DP_SEC_METADATA_TRANSMISSION
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE__SHIFT                                0x0
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE__SHIFT                        0x1
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE__SHIFT                            0x4
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE__SHIFT                                  0x10
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_ENABLE_MASK                                  0x00000001L
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_REFERENCE_MASK                          0x00000002L
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_MSO_METADATA_PACKET_ENABLE_MASK                              0x000000F0L
+#define DP4_DP_SEC_METADATA_TRANSMISSION__DP_SEC_METADATA_PACKET_LINE_MASK                                    0xFFFF0000L
+//DP4_DP_ALPM_CNTL
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND__SHIFT                                                         0x0
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING__SHIFT                                                      0x1
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND__SHIFT                                                       0x2
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING__SHIFT                                                    0x3
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE__SHIFT                                            0x4
+#define DP4_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO__SHIFT                                        0x5
+#define DP4_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE__SHIFT                                                  0x6
+#define DP4_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP__SHIFT                                     0x7
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM__SHIFT                                                  0x8
+#define DP4_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS__SHIFT                                   0xb
+#define DP4_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN__SHIFT                                       0xc
+#define DP4_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE__SHIFT                                                       0xf
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM__SHIFT                                             0x10
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_SEND_MASK                                                           0x00000001L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PENDING_MASK                                                        0x00000002L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_SEND_MASK                                                         0x00000004L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_STANDBY_PENDING_MASK                                                      0x00000008L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_IMMEDIATE_MASK                                              0x00000010L
+#define DP4_DP_ALPM_CNTL__DP_LINK_TRAINING_SWITCH_BETWEEN_VIDEO_MASK                                          0x00000020L
+#define DP4_DP_ALPM_CNTL__DP_ALPM_SLEEP_SEQUENCE_MODE_MASK                                                    0x00000040L
+#define DP4_DP_ALPM_CNTL__DP_STOP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_MASK                                       0x00000080L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_PATTERN_NUM_MASK                                                    0x00000300L
+#define DP4_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_STATUS_MASK                                     0x00000800L
+#define DP4_DP_ALPM_CNTL__DP_FORCE_SCRAMBLED_ZERO_AFTER_SLEEP_EN_MASK                                         0x00001000L
+#define DP4_DP_ALPM_CNTL__DP_ALPM_LINE_REFERENCE_MASK                                                         0x00008000L
+#define DP4_DP_ALPM_CNTL__DP_ML_PHY_SLEEP_STANDBY_LINE_NUM_MASK                                               0xFFFF0000L
+//DP4_DP_GSP8_CNTL
+#define DP4_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE__SHIFT                                                       0x0
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE__SHIFT                                                           0x4
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND__SHIFT                                                             0x7
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING__SHIFT                                                     0xc
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_GSP8_CNTL__DP_MSO_SEC_GSP8_ENABLE_MASK                                                         0x0000000FL
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_ENABLE_MASK                                                             0x00000010L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_MASK                                                               0x00000080L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_PENDING_MASK                                                       0x00001000L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP4_DP_GSP8_CNTL__DP_SEC_GSP8_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_GSP9_CNTL
+#define DP4_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE__SHIFT                                                       0x0
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE__SHIFT                                                           0x4
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE__SHIFT                                                   0x5
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE__SHIFT                                                     0x6
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND__SHIFT                                                             0x7
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE__SHIFT                                                    0x8
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING__SHIFT                                                     0xc
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE__SHIFT                                                      0xd
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED__SHIFT                                             0xe
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM__SHIFT                                                         0x10
+#define DP4_DP_GSP9_CNTL__DP_MSO_SEC_GSP9_ENABLE_MASK                                                         0x0000000FL
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_ENABLE_MASK                                                             0x00000010L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_REFERENCE_MASK                                                     0x00000020L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_IN_IDLE_MASK                                                       0x00000040L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_MASK                                                               0x00000080L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ANY_LINE_MASK                                                      0x00000100L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_PENDING_MASK                                                       0x00001000L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_ACTIVE_MASK                                                        0x00002000L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_SEND_DEADLINE_MISSED_MASK                                               0x00004000L
+#define DP4_DP_GSP9_CNTL__DP_SEC_GSP9_LINE_NUM_MASK                                                           0xFFFF0000L
+//DP4_DP_GSP10_CNTL
+#define DP4_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE__SHIFT                                                     0x0
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE__SHIFT                                                         0x4
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND__SHIFT                                                           0x7
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING__SHIFT                                                   0xc
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM__SHIFT                                                       0x10
+#define DP4_DP_GSP10_CNTL__DP_MSO_SEC_GSP10_ENABLE_MASK                                                       0x0000000FL
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_ENABLE_MASK                                                           0x00000010L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_MASK                                                             0x00000080L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_PENDING_MASK                                                     0x00001000L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP4_DP_GSP10_CNTL__DP_SEC_GSP10_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP4_DP_GSP11_CNTL
+#define DP4_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE__SHIFT                                                     0x0
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE__SHIFT                                                         0x4
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE__SHIFT                                                 0x5
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE__SHIFT                                                   0x6
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND__SHIFT                                                           0x7
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE__SHIFT                                                  0x8
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING__SHIFT                                                   0xc
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE__SHIFT                                                    0xd
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED__SHIFT                                           0xe
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM__SHIFT                                                       0x10
+#define DP4_DP_GSP11_CNTL__DP_MSO_SEC_GSP11_ENABLE_MASK                                                       0x0000000FL
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_ENABLE_MASK                                                           0x00000010L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_REFERENCE_MASK                                                   0x00000020L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_IN_IDLE_MASK                                                     0x00000040L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_MASK                                                             0x00000080L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ANY_LINE_MASK                                                    0x00000100L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_PENDING_MASK                                                     0x00001000L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_ACTIVE_MASK                                                      0x00002000L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_SEND_DEADLINE_MISSED_MASK                                             0x00004000L
+#define DP4_DP_GSP11_CNTL__DP_SEC_GSP11_LINE_NUM_MASK                                                         0xFFFF0000L
+//DP4_DP_GSP_EN_DB_STATUS
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING__SHIFT                                             0x0
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING__SHIFT                                             0x1
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING__SHIFT                                             0x2
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING__SHIFT                                             0x3
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING__SHIFT                                             0x4
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING__SHIFT                                             0x5
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING__SHIFT                                             0x6
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING__SHIFT                                             0x7
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING__SHIFT                                             0x8
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING__SHIFT                                             0x9
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING__SHIFT                                            0xa
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING__SHIFT                                            0xb
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP0_EN_DB_PENDING_MASK                                               0x00000001L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP1_EN_DB_PENDING_MASK                                               0x00000002L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP2_EN_DB_PENDING_MASK                                               0x00000004L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP3_EN_DB_PENDING_MASK                                               0x00000008L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP4_EN_DB_PENDING_MASK                                               0x00000010L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP5_EN_DB_PENDING_MASK                                               0x00000020L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP6_EN_DB_PENDING_MASK                                               0x00000040L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP7_EN_DB_PENDING_MASK                                               0x00000080L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP8_EN_DB_PENDING_MASK                                               0x00000100L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP9_EN_DB_PENDING_MASK                                               0x00000200L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP10_EN_DB_PENDING_MASK                                              0x00000400L
+#define DP4_DP_GSP_EN_DB_STATUS__DP_SEC_GSP11_EN_DB_PENDING_MASK                                              0x00000800L
+//DP4_DP_AUXLESS_ALPM_CNTL1
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT__SHIFT                                              0x4
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY__SHIFT                                               0x8
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL__SHIFT                                            0x14
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE__SHIFT                                     0x1f
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_REPEAT_MASK                                                0x000000F0L
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_DELAY_MASK                                                 0x0007FF00L
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_ML_PHY_SLEEP_INTERVAL_MASK                                              0x1FF00000L
+#define DP4_DP_AUXLESS_ALPM_CNTL1__DP_SET_AUXLESS_ALPM_SLEEP_STATE_MASK                                       0x80000000L
+//DP4_DP_AUXLESS_ALPM_CNTL2
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME__SHIFT                                           0x0
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND__SHIFT                                                 0x7
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE__SHIFT                                            0x10
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING__SHIFT                                              0x11
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE__SHIFT                                            0x12
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING__SHIFT                                              0x13
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD__SHIFT                                          0x14
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ML_PHY_SLEEP_HOLD_TIME_MASK                                             0x0000007FL
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_SEND_MASK                                                   0x00000080L
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_IMMEDIATE_MASK                                              0x00010000L
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_WAKEUP_PENDING_MASK                                                0x00020000L
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_IMMEDIATE_MASK                                              0x00040000L
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_FEC_EN_PENDING_MASK                                                0x00080000L
+#define DP4_DP_AUXLESS_ALPM_CNTL2__DP_ALPM_ML_PHY_LOCK_PERIOD_MASK                                            0x3FF00000L
+//DP4_DP_AUXLESS_ALPM_CNTL3
+#define DP4_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM__SHIFT                                             0x0
+#define DP4_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM__SHIFT                                             0x10
+#define DP4_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_WAKEUP_LINE_NUM_MASK                                               0x0000FFFFL
+#define DP4_DP_AUXLESS_ALPM_CNTL3__DP_ALPM_FEC_EN_LINE_NUM_MASK                                               0xFFFF0000L
+//DP4_DP_AUXLESS_ALPM_CNTL4
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN__SHIFT                                                  0x1
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL__SHIFT                                   0x2
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME__SHIFT                                     0x3
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE__SHIFT                                       0x4
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS__SHIFT                                           0x5
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE__SHIFT                                               0x6
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM__SHIFT                                                   0x18
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_MASK                                                    0x00000002L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_SLEEP_PATTERN_SEL_MASK                                     0x00000004L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FORCE_WAKEUP_NEXT_FRAME_MASK                                       0x00000008L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_DIS_IMMEDIATE_MASK                                         0x00000010L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_HW_MODE_EN_STATUS_MASK                                             0x00000020L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_CURRENT_STATE_MASK                                                 0x00000040L
+#define DP4_DP_AUXLESS_ALPM_CNTL4__DP_ALPM_FRAME_NUM_MASK                                                     0xFF000000L
+//DP4_DP_AUXLESS_ALPM_CNTL5
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK__SHIFT                                       0x0
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED__SHIFT                                   0x1
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS__SHIFT                                     0x2
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR__SHIFT                                      0x3
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM__SHIFT                                  0x8
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM__SHIFT                                   0x10
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_MASK_MASK                                         0x00000001L
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_OCCURRED_MASK                                     0x00000002L
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_STATUS_MASK                                       0x00000004L
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_CLEAR_MASK                                        0x00000008L
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_FRAME_NUM_MASK                                    0x0000FF00L
+#define DP4_DP_AUXLESS_ALPM_CNTL5__DP_ALPM_WAKEUP_INTERRUPT_LINE_NUM_MASK                                     0xFFFF0000L
+//DP4_DP_STREAM_SYMBOL_COUNT_STATUS
+#define DP4_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT__SHIFT                                          0x0
+#define DP4_DP_STREAM_SYMBOL_COUNT_STATUS__DP_STREAM_BS_COUNT_MASK                                            0x0000FFFFL
+//DP4_DP_STREAM_SYMBOL_COUNT_CONTROL
+#define DP4_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE__SHIFT                                  0x0
+#define DP4_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET__SHIFT                                   0x4
+#define DP4_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_ENABLE_MASK                                    0x00000001L
+#define DP4_DP_STREAM_SYMBOL_COUNT_CONTROL__DP_STREAM_BS_COUNT_RESET_MASK                                     0x00000010L
+//DP4_DP_LINK_SYMBOL_COUNT_STATUS0
+#define DP4_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT__SHIFT                                             0x0
+#define DP4_DP_LINK_SYMBOL_COUNT_STATUS0__DP_LINK_SR_COUNT_MASK                                               0x0000FFFFL
+//DP4_DP_LINK_SYMBOL_COUNT_STATUS1
+#define DP4_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT__SHIFT                                          0x0
+#define DP4_DP_LINK_SYMBOL_COUNT_STATUS1__DP_LINK_CYCLE_COUNT_MASK                                            0xFFFFFFFFL
+//DP4_DP_LINK_SYMBOL_COUNT_CONTROL
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE__SHIFT                                      0x0
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET__SHIFT                                       0x4
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE__SHIFT                                   0x8
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET__SHIFT                                    0xc
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_ENABLE_MASK                                        0x00000001L
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_SR_COUNT_RESET_MASK                                         0x00000010L
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_ENABLE_MASK                                     0x00000100L
+#define DP4_DP_LINK_SYMBOL_COUNT_CONTROL__DP_LINK_CYCLE_COUNT_RESET_MASK                                      0x00001000L
+//DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT           0x0
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING__SHIFT  0x8
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK             0x00000001L
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+#define DP4_DP_SYM8_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_PENDING_MASK  0x00000100L
+//DP4_DP_DPHY_FAST_TRAINING
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT                                       0x0
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT                                         0x1
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT                            0x2
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING__SHIFT                              0x4
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT                                         0x8
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT                                         0x14
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK                                         0x00000001L
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK                                           0x00000002L
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK                              0x00000004L
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_STREAM_RESET_DURING_FAST_TRAINING_MASK                                0x00000010L
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK                                           0x000FFF00L
+#define DP4_DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK                                           0xFFF00000L
+//DP4_DP_DPHY_FAST_TRAINING_STATUS
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT                                     0x0
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT                         0x4
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT                             0x8
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT                              0xc
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK                                       0x00000007L
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK                           0x00000010L
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK                               0x00000100L
+#define DP4_DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK                                0x00001000L
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_top_dispdec
+//DSC_TOP0_DSC_TOP_CONTROL
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_CLOCK_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS__SHIFT                                               0x4
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS__SHIFT                                                0x8
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS__SHIFT                                                     0xc
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN__SHIFT                                     0x10
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK                                                 0x00000010L
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS_MASK                                                       0x00001000L
+#define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN_MASK                                       0x00010000L
+//DSC_TOP0_DSC_DEBUG_CONTROL
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
+//DSC_TOP0_DSC_SPARE_DEBUG
+#define DSC_TOP0_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG__SHIFT                                                      0x0
+#define DSC_TOP0_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG_MASK                                                        0xFFFFFFFFL
+//DSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX__SHIFT                                    0x0
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE__SHIFT                               0x8
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX_MASK                                      0x000000FFL
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE_MASK                                 0x00001F00L
+//DSC_TOP0_DSC_TOP_TEST_DEBUG_DATA
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA__SHIFT                                      0x0
+#define DSC_TOP0_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
+//DSCCIF0_DSCCIF_CONFIG0
+#define DSCCIF0_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT__SHIFT                                                     0xc
+#define DSCCIF0_DSCCIF_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x10
+#define DSCCIF0_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x18
+#define DSCCIF0_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT_MASK                                                       0x00007000L
+#define DSCCIF0_DSCCIF_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0x000F0000L
+#define DSCCIF0_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x01000000L
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dscc_dispdec
+//DSCC0_DSCC_CONFIG0
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                                                   0x0
+#define DSCC0_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE__SHIFT                                                  0x4
+#define DSCC0_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN__SHIFT                                                  0x8
+#define DSCC0_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION__SHIFT                                     0x10
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                                                     0x0000000FL
+#define DSCC0_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE_MASK                                                    0x00000030L
+#define DSCC0_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN_MASK                                                    0x00000100L
+#define DSCC0_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION_MASK                                       0xFFFF0000L
+//DSCC0_DSCC_CONFIG1
+#define DSCC0_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE__SHIFT                                        0x0
+#define DSCC0_DSCC_CONFIG1__DSCC_DISABLE_ICH__SHIFT                                                           0x18
+#define DSCC0_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE_MASK                                          0x0003FFFFL
+#define DSCC0_DSCC_CONFIG1__DSCC_DISABLE_ICH_MASK                                                             0x01000000L
+//DSCC0_DSCC_CONFIG2
+#define DSCC0_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD__SHIFT                                         0x0
+#define DSCC0_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD__SHIFT                                  0x10
+#define DSCC0_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD_MASK                                           0x0000FFFFL
+#define DSCC0_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD_MASK                                    0xFFFF0000L
+//DSCC0_DSCC_STATUS
+#define DSCC0_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x0
+#define DSCC0_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x00000001L
+//DSCC0_DSCC_INTERRUPT_CONTROL0
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0__SHIFT        0x0
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1__SHIFT        0x1
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2__SHIFT        0x2
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3__SHIFT        0x3
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0_MASK          0x00000001L
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1_MASK          0x00000002L
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2_MASK          0x00000004L
+#define DSCC0_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3_MASK          0x00000008L
+//DSCC0_DSCC_INTERRUPT_CONTROL1
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0__SHIFT                    0x0
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1__SHIFT                    0x1
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2__SHIFT                    0x2
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3__SHIFT                    0x3
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0__SHIFT                   0x4
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1__SHIFT                   0x5
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2__SHIFT                   0x6
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3__SHIFT                   0x7
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN__SHIFT                   0x8
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0_MASK                      0x00000001L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1_MASK                      0x00000002L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2_MASK                      0x00000004L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3_MASK                      0x00000008L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0_MASK                     0x00000010L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1_MASK                     0x00000020L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2_MASK                     0x00000040L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3_MASK                     0x00000080L
+#define DSCC0_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN_MASK                     0x00000100L
+//DSCC0_DSCC_INTERRUPT_STATUS0
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0__SHIFT                0x0
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1__SHIFT                0x1
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2__SHIFT                0x2
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3__SHIFT                0x3
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0__SHIFT                   0x10
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1__SHIFT                   0x11
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2__SHIFT                   0x12
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3__SHIFT                   0x13
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0_MASK                  0x00000001L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1_MASK                  0x00000002L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2_MASK                  0x00000004L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3_MASK                  0x00000008L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0_MASK                     0x00010000L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1_MASK                     0x00020000L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2_MASK                     0x00040000L
+#define DSCC0_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3_MASK                     0x00080000L
+//DSCC0_DSCC_INTERRUPT_STATUS1
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0__SHIFT                            0x0
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1__SHIFT                            0x1
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2__SHIFT                            0x2
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3__SHIFT                            0x3
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0__SHIFT                           0x4
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1__SHIFT                           0x5
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2__SHIFT                           0x6
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3__SHIFT                           0x7
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED__SHIFT                           0x8
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0__SHIFT                               0x10
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1__SHIFT                               0x11
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2__SHIFT                               0x12
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3__SHIFT                               0x13
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0__SHIFT                              0x14
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1__SHIFT                              0x15
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2__SHIFT                              0x16
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3__SHIFT                              0x17
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR__SHIFT                              0x18
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0_MASK                              0x00000001L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1_MASK                              0x00000002L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2_MASK                              0x00000004L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3_MASK                              0x00000008L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0_MASK                             0x00000010L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1_MASK                             0x00000020L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2_MASK                             0x00000040L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3_MASK                             0x00000080L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_MASK                             0x00000100L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0_MASK                                 0x00010000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1_MASK                                 0x00020000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2_MASK                                 0x00040000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3_MASK                                 0x00080000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0_MASK                                0x00100000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1_MASK                                0x00200000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2_MASK                                0x00400000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3_MASK                                0x00800000L
+#define DSCC0_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR_MASK                                0x01000000L
+//DSCC0_DSCC_PPS_CONFIG0
+#define DSCC0_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR__SHIFT                                                      0x0
+#define DSCC0_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR__SHIFT                                                      0x4
+#define DSCC0_DSCC_PPS_CONFIG0__PPS_IDENTIFIER__SHIFT                                                         0x8
+#define DSCC0_DSCC_PPS_CONFIG0__LINEBUF_DEPTH__SHIFT                                                          0x18
+#define DSCC0_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x1c
+#define DSCC0_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR_MASK                                                        0x0000000FL
+#define DSCC0_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR_MASK                                                        0x000000F0L
+#define DSCC0_DSCC_PPS_CONFIG0__PPS_IDENTIFIER_MASK                                                           0x0000FF00L
+#define DSCC0_DSCC_PPS_CONFIG0__LINEBUF_DEPTH_MASK                                                            0x0F000000L
+#define DSCC0_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0xF0000000L
+//DSCC0_DSCC_PPS_CONFIG1
+#define DSCC0_DSCC_PPS_CONFIG1__BITS_PER_PIXEL__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG1__VBR_ENABLE__SHIFT                                                             0xa
+#define DSCC0_DSCC_PPS_CONFIG1__SIMPLE_422__SHIFT                                                             0xb
+#define DSCC0_DSCC_PPS_CONFIG1__CONVERT_RGB__SHIFT                                                            0xc
+#define DSCC0_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE__SHIFT                                                      0xd
+#define DSCC0_DSCC_PPS_CONFIG1__NATIVE_422__SHIFT                                                             0xe
+#define DSCC0_DSCC_PPS_CONFIG1__NATIVE_420__SHIFT                                                             0xf
+#define DSCC0_DSCC_PPS_CONFIG1__CHUNK_SIZE__SHIFT                                                             0x10
+#define DSCC0_DSCC_PPS_CONFIG1__BITS_PER_PIXEL_MASK                                                           0x000003FFL
+#define DSCC0_DSCC_PPS_CONFIG1__VBR_ENABLE_MASK                                                               0x00000400L
+#define DSCC0_DSCC_PPS_CONFIG1__SIMPLE_422_MASK                                                               0x00000800L
+#define DSCC0_DSCC_PPS_CONFIG1__CONVERT_RGB_MASK                                                              0x00001000L
+#define DSCC0_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE_MASK                                                        0x00002000L
+#define DSCC0_DSCC_PPS_CONFIG1__NATIVE_422_MASK                                                               0x00004000L
+#define DSCC0_DSCC_PPS_CONFIG1__NATIVE_420_MASK                                                               0x00008000L
+#define DSCC0_DSCC_PPS_CONFIG1__CHUNK_SIZE_MASK                                                               0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG2
+#define DSCC0_DSCC_PPS_CONFIG2__PIC_WIDTH__SHIFT                                                              0x0
+#define DSCC0_DSCC_PPS_CONFIG2__PIC_HEIGHT__SHIFT                                                             0x10
+#define DSCC0_DSCC_PPS_CONFIG2__PIC_WIDTH_MASK                                                                0x0000FFFFL
+#define DSCC0_DSCC_PPS_CONFIG2__PIC_HEIGHT_MASK                                                               0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG3
+#define DSCC0_DSCC_PPS_CONFIG3__SLICE_WIDTH__SHIFT                                                            0x0
+#define DSCC0_DSCC_PPS_CONFIG3__SLICE_HEIGHT__SHIFT                                                           0x10
+#define DSCC0_DSCC_PPS_CONFIG3__SLICE_WIDTH_MASK                                                              0x0000FFFFL
+#define DSCC0_DSCC_PPS_CONFIG3__SLICE_HEIGHT_MASK                                                             0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG4
+#define DSCC0_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY__SHIFT                                                     0x0
+#define DSCC0_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY__SHIFT                                                      0x10
+#define DSCC0_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY_MASK                                                       0x000003FFL
+#define DSCC0_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY_MASK                                                        0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG5
+#define DSCC0_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE__SHIFT                                                    0x0
+#define DSCC0_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL__SHIFT                                               0x10
+#define DSCC0_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE_MASK                                                      0x0000003FL
+#define DSCC0_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL_MASK                                                 0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG6
+#define DSCC0_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL__SHIFT                                               0x0
+#define DSCC0_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET__SHIFT                                                  0x10
+#define DSCC0_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET__SHIFT                                                 0x18
+#define DSCC0_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL_MASK                                                 0x00000FFFL
+#define DSCC0_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET_MASK                                                    0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET_MASK                                                   0x1F000000L
+//DSCC0_DSCC_PPS_CONFIG7
+#define DSCC0_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET__SHIFT                                                       0x10
+#define DSCC0_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC0_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET_MASK                                                         0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG8
+#define DSCC0_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ__SHIFT                                                 0x10
+#define DSCC0_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC0_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ_MASK                                                   0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG9
+#define DSCC0_DSCC_PPS_CONFIG9__INITIAL_OFFSET__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG9__FINAL_OFFSET__SHIFT                                                           0x10
+#define DSCC0_DSCC_PPS_CONFIG9__INITIAL_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC0_DSCC_PPS_CONFIG9__FINAL_OFFSET_MASK                                                             0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG10
+#define DSCC0_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP__SHIFT                                                       0x0
+#define DSCC0_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP__SHIFT                                                       0x8
+#define DSCC0_DSCC_PPS_CONFIG10__RC_MODEL_SIZE__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP_MASK                                                         0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP_MASK                                                         0x00001F00L
+#define DSCC0_DSCC_PPS_CONFIG10__RC_MODEL_SIZE_MASK                                                           0xFFFF0000L
+//DSCC0_DSCC_PPS_CONFIG11
+#define DSCC0_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0__SHIFT                                                  0x8
+#define DSCC0_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1__SHIFT                                                  0x10
+#define DSCC0_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO__SHIFT                                                      0x18
+#define DSCC0_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI__SHIFT                                                      0x1c
+#define DSCC0_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR_MASK                                                          0x0000000FL
+#define DSCC0_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0_MASK                                                    0x00001F00L
+#define DSCC0_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1_MASK                                                    0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO_MASK                                                        0x0F000000L
+#define DSCC0_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI_MASK                                                        0xF0000000L
+//DSCC0_DSCC_PPS_CONFIG12
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH0__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH1__SHIFT                                                        0x8
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH2__SHIFT                                                        0x10
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH3__SHIFT                                                        0x18
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH0_MASK                                                          0x000000FFL
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH1_MASK                                                          0x0000FF00L
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH2_MASK                                                          0x00FF0000L
+#define DSCC0_DSCC_PPS_CONFIG12__RC_BUF_THRESH3_MASK                                                          0xFF000000L
+//DSCC0_DSCC_PPS_CONFIG13
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH4__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH5__SHIFT                                                        0x8
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH6__SHIFT                                                        0x10
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH7__SHIFT                                                        0x18
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH4_MASK                                                          0x000000FFL
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH5_MASK                                                          0x0000FF00L
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH6_MASK                                                          0x00FF0000L
+#define DSCC0_DSCC_PPS_CONFIG13__RC_BUF_THRESH7_MASK                                                          0xFF000000L
+//DSCC0_DSCC_PPS_CONFIG14
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH8__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH9__SHIFT                                                        0x8
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH10__SHIFT                                                       0x10
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH11__SHIFT                                                       0x18
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH8_MASK                                                          0x000000FFL
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH9_MASK                                                          0x0000FF00L
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH10_MASK                                                         0x00FF0000L
+#define DSCC0_DSCC_PPS_CONFIG14__RC_BUF_THRESH11_MASK                                                         0xFF000000L
+//DSCC0_DSCC_PPS_CONFIG15
+#define DSCC0_DSCC_PPS_CONFIG15__RC_BUF_THRESH12__SHIFT                                                       0x0
+#define DSCC0_DSCC_PPS_CONFIG15__RC_BUF_THRESH13__SHIFT                                                       0x8
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_MIN_QP0__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_MAX_QP0__SHIFT                                                         0x15
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0__SHIFT                                                     0x1a
+#define DSCC0_DSCC_PPS_CONFIG15__RC_BUF_THRESH12_MASK                                                         0x000000FFL
+#define DSCC0_DSCC_PPS_CONFIG15__RC_BUF_THRESH13_MASK                                                         0x0000FF00L
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_MIN_QP0_MASK                                                           0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_MAX_QP0_MASK                                                           0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0_MASK                                                       0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG16
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MIN_QP1__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MAX_QP1__SHIFT                                                         0x5
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1__SHIFT                                                     0xa
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MIN_QP2__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MAX_QP2__SHIFT                                                         0x15
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2__SHIFT                                                     0x1a
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MIN_QP1_MASK                                                           0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MAX_QP1_MASK                                                           0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1_MASK                                                       0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MIN_QP2_MASK                                                           0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_MAX_QP2_MASK                                                           0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2_MASK                                                       0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG17
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MIN_QP3__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MAX_QP3__SHIFT                                                         0x5
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3__SHIFT                                                     0xa
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MIN_QP4__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MAX_QP4__SHIFT                                                         0x15
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4__SHIFT                                                     0x1a
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MIN_QP3_MASK                                                           0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MAX_QP3_MASK                                                           0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3_MASK                                                       0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MIN_QP4_MASK                                                           0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_MAX_QP4_MASK                                                           0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4_MASK                                                       0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG18
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MIN_QP5__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MAX_QP5__SHIFT                                                         0x5
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5__SHIFT                                                     0xa
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MIN_QP6__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MAX_QP6__SHIFT                                                         0x15
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6__SHIFT                                                     0x1a
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MIN_QP5_MASK                                                           0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MAX_QP5_MASK                                                           0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5_MASK                                                       0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MIN_QP6_MASK                                                           0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_MAX_QP6_MASK                                                           0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6_MASK                                                       0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG19
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MIN_QP7__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MAX_QP7__SHIFT                                                         0x5
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7__SHIFT                                                     0xa
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MIN_QP8__SHIFT                                                         0x10
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MAX_QP8__SHIFT                                                         0x15
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8__SHIFT                                                     0x1a
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MIN_QP7_MASK                                                           0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MAX_QP7_MASK                                                           0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7_MASK                                                       0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MIN_QP8_MASK                                                           0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_MAX_QP8_MASK                                                           0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8_MASK                                                       0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG20
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MIN_QP9__SHIFT                                                         0x0
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MAX_QP9__SHIFT                                                         0x5
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9__SHIFT                                                     0xa
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MIN_QP10__SHIFT                                                        0x10
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MAX_QP10__SHIFT                                                        0x15
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10__SHIFT                                                    0x1a
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MIN_QP9_MASK                                                           0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MAX_QP9_MASK                                                           0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9_MASK                                                       0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MIN_QP10_MASK                                                          0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_MAX_QP10_MASK                                                          0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10_MASK                                                      0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG21
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MIN_QP11__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MAX_QP11__SHIFT                                                        0x5
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11__SHIFT                                                    0xa
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MIN_QP12__SHIFT                                                        0x10
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MAX_QP12__SHIFT                                                        0x15
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12__SHIFT                                                    0x1a
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MIN_QP11_MASK                                                          0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MAX_QP11_MASK                                                          0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11_MASK                                                      0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MIN_QP12_MASK                                                          0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_MAX_QP12_MASK                                                          0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12_MASK                                                      0xFC000000L
+//DSCC0_DSCC_PPS_CONFIG22
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MIN_QP13__SHIFT                                                        0x0
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MAX_QP13__SHIFT                                                        0x5
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13__SHIFT                                                    0xa
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MIN_QP14__SHIFT                                                        0x10
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MAX_QP14__SHIFT                                                        0x15
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14__SHIFT                                                    0x1a
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MIN_QP13_MASK                                                          0x0000001FL
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MAX_QP13_MASK                                                          0x000003E0L
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13_MASK                                                      0x0000FC00L
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MIN_QP14_MASK                                                          0x001F0000L
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_MAX_QP14_MASK                                                          0x03E00000L
+#define DSCC0_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14_MASK                                                      0xFC000000L
+//DSCC0_DSCC_MEM_POWER_CONTROL0
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC0_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC0_DSCC_MEM_POWER_CONTROL1
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC0_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER
+#define DSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER__SHIFT                               0x0
+#define DSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER_MASK                                 0xFFFFFFFFL
+//DSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER
+#define DSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER__SHIFT                               0x0
+#define DSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER_MASK                                 0xFFFFFFFFL
+//DSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER
+#define DSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER
+#define DSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER
+#define DSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER
+#define DSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC0_DSCC_MAX_ABS_ERROR0
+#define DSCC0_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR__SHIFT                                              0x0
+#define DSCC0_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR__SHIFT                                             0x10
+#define DSCC0_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR_MASK                                                0x0000FFFFL
+#define DSCC0_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR_MASK                                               0xFFFF0000L
+//DSCC0_DSCC_MAX_ABS_ERROR1
+#define DSCC0_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR__SHIFT                                             0x0
+#define DSCC0_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR_MASK                                               0x0000FFFFL
+//DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__SHIFT           0x0
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_MASK             0x00007FFFL
+//DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__SHIFT           0x0
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_MASK             0x00007FFFL
+//DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__SHIFT           0x0
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_MASK             0x00007FFFL
+//DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__SHIFT           0x0
+#define DSCC0_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_MASK             0x00007FFFL
+//DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__SHIFT   0x0
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_MASK     0x0003FFFFL
+//DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__SHIFT   0x0
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_MASK     0x0003FFFFL
+//DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__SHIFT   0x0
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_MASK     0x0003FFFFL
+//DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__SHIFT   0x0
+#define DSCC0_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_MASK     0x0003FFFFL
+//DSCC0_DSCC_TEST_DEBUG_INDEX0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0__SHIFT                                           0x0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0_MASK                                             0x000000FFL
+//DSCC0_DSCC_TEST_DEBUG_INDEX1
+#define DSCC0_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1__SHIFT                                           0x0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1_MASK                                             0x000000FFL
+//DSCC0_DSCC_TEST_DEBUG_INDEX2
+#define DSCC0_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2__SHIFT                                           0x0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2_MASK                                             0x000000FFL
+//DSCC0_DSCC_TEST_DEBUG_INDEX3
+#define DSCC0_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3__SHIFT                                           0x0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3_MASK                                             0x000000FFL
+//DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
+//DSCC0_DSCC_TEST_DEBUG_DATA0
+#define DSCC0_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0__SHIFT                                             0x0
+#define DSCC0_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0_MASK                                               0xFFFFFFFFL
+//DSCC0_DSCC_TEST_DEBUG_DATA1
+#define DSCC0_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1__SHIFT                                             0x0
+#define DSCC0_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1_MASK                                               0xFFFFFFFFL
+//DSCC0_DSCC_TEST_DEBUG_DATA2
+#define DSCC0_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2__SHIFT                                             0x0
+#define DSCC0_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2_MASK                                               0xFFFFFFFFL
+//DSCC0_DSCC_TEST_DEBUG_DATA3
+#define DSCC0_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3__SHIFT                                             0x0
+#define DSCC0_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3_MASK                                               0xFFFFFFFFL
+//DSCC0_DSCC_DISPCLK_TEST_DEBUG_INDEX0
+#define DSCC0_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0__SHIFT                           0x0
+#define DSCC0_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0_MASK                             0x000000FFL
+//DSCC0_DSCC_DISPCLK_TEST_DEBUG_DATA0
+#define DSCC0_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0__SHIFT                             0x0
+#define DSCC0_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0_MASK                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON17_PERFCOUNTER_CNTL
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON17_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON17_PERFCOUNTER_CNTL2
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON17_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON17_PERFCOUNTER_STATE
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON17_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON17_PERFMON_CNTL
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON17_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON17_PERFMON_CNTL2
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON17_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON17_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON17_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON17_PERFMON_CVALUE_LOW
+#define DC_PERFMON17_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON17_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON17_PERFMON_HI
+#define DC_PERFMON17_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON17_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON17_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON17_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON17_PERFMON_LOW
+#define DC_PERFMON17_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON17_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_top_dispdec
+//DSC_TOP1_DSC_TOP_CONTROL
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_CLOCK_EN__SHIFT                                                         0x0
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS__SHIFT                                               0x4
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS__SHIFT                                                0x8
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS__SHIFT                                                     0xc
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN__SHIFT                                     0x10
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK                                                           0x00000001L
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK                                                 0x00000010L
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS_MASK                                                       0x00001000L
+#define DSC_TOP1_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN_MASK                                       0x00010000L
+//DSC_TOP1_DSC_DEBUG_CONTROL
+#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP1_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
+//DSC_TOP1_DSC_SPARE_DEBUG
+#define DSC_TOP1_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG__SHIFT                                                      0x0
+#define DSC_TOP1_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG_MASK                                                        0xFFFFFFFFL
+//DSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX__SHIFT                                    0x0
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE__SHIFT                               0x8
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX_MASK                                      0x000000FFL
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE_MASK                                 0x00001F00L
+//DSC_TOP1_DSC_TOP_TEST_DEBUG_DATA
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA__SHIFT                                      0x0
+#define DSC_TOP1_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsccif_dispdec
+//DSCCIF1_DSCCIF_CONFIG0
+#define DSCCIF1_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT__SHIFT                                                     0xc
+#define DSCCIF1_DSCCIF_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x10
+#define DSCCIF1_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x18
+#define DSCCIF1_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT_MASK                                                       0x00007000L
+#define DSCCIF1_DSCCIF_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0x000F0000L
+#define DSCCIF1_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x01000000L
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dscc_dispdec
+//DSCC1_DSCC_CONFIG0
+#define DSCC1_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                                                   0x0
+#define DSCC1_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE__SHIFT                                                  0x4
+#define DSCC1_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN__SHIFT                                                  0x8
+#define DSCC1_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION__SHIFT                                     0x10
+#define DSCC1_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                                                     0x0000000FL
+#define DSCC1_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE_MASK                                                    0x00000030L
+#define DSCC1_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN_MASK                                                    0x00000100L
+#define DSCC1_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION_MASK                                       0xFFFF0000L
+//DSCC1_DSCC_CONFIG1
+#define DSCC1_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE__SHIFT                                        0x0
+#define DSCC1_DSCC_CONFIG1__DSCC_DISABLE_ICH__SHIFT                                                           0x18
+#define DSCC1_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE_MASK                                          0x0003FFFFL
+#define DSCC1_DSCC_CONFIG1__DSCC_DISABLE_ICH_MASK                                                             0x01000000L
+//DSCC1_DSCC_CONFIG2
+#define DSCC1_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD__SHIFT                                         0x0
+#define DSCC1_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD__SHIFT                                  0x10
+#define DSCC1_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD_MASK                                           0x0000FFFFL
+#define DSCC1_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD_MASK                                    0xFFFF0000L
+//DSCC1_DSCC_STATUS
+#define DSCC1_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x0
+#define DSCC1_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x00000001L
+//DSCC1_DSCC_INTERRUPT_CONTROL0
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0__SHIFT        0x0
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1__SHIFT        0x1
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2__SHIFT        0x2
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3__SHIFT        0x3
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0_MASK          0x00000001L
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1_MASK          0x00000002L
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2_MASK          0x00000004L
+#define DSCC1_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3_MASK          0x00000008L
+//DSCC1_DSCC_INTERRUPT_CONTROL1
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0__SHIFT                    0x0
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1__SHIFT                    0x1
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2__SHIFT                    0x2
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3__SHIFT                    0x3
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0__SHIFT                   0x4
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1__SHIFT                   0x5
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2__SHIFT                   0x6
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3__SHIFT                   0x7
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN__SHIFT                   0x8
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0_MASK                      0x00000001L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1_MASK                      0x00000002L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2_MASK                      0x00000004L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3_MASK                      0x00000008L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0_MASK                     0x00000010L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1_MASK                     0x00000020L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2_MASK                     0x00000040L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3_MASK                     0x00000080L
+#define DSCC1_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN_MASK                     0x00000100L
+//DSCC1_DSCC_INTERRUPT_STATUS0
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0__SHIFT                0x0
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1__SHIFT                0x1
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2__SHIFT                0x2
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3__SHIFT                0x3
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0__SHIFT                   0x10
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1__SHIFT                   0x11
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2__SHIFT                   0x12
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3__SHIFT                   0x13
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0_MASK                  0x00000001L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1_MASK                  0x00000002L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2_MASK                  0x00000004L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3_MASK                  0x00000008L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0_MASK                     0x00010000L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1_MASK                     0x00020000L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2_MASK                     0x00040000L
+#define DSCC1_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3_MASK                     0x00080000L
+//DSCC1_DSCC_INTERRUPT_STATUS1
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0__SHIFT                            0x0
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1__SHIFT                            0x1
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2__SHIFT                            0x2
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3__SHIFT                            0x3
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0__SHIFT                           0x4
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1__SHIFT                           0x5
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2__SHIFT                           0x6
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3__SHIFT                           0x7
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED__SHIFT                           0x8
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0__SHIFT                               0x10
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1__SHIFT                               0x11
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2__SHIFT                               0x12
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3__SHIFT                               0x13
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0__SHIFT                              0x14
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1__SHIFT                              0x15
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2__SHIFT                              0x16
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3__SHIFT                              0x17
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR__SHIFT                              0x18
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0_MASK                              0x00000001L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1_MASK                              0x00000002L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2_MASK                              0x00000004L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3_MASK                              0x00000008L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0_MASK                             0x00000010L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1_MASK                             0x00000020L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2_MASK                             0x00000040L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3_MASK                             0x00000080L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_MASK                             0x00000100L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0_MASK                                 0x00010000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1_MASK                                 0x00020000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2_MASK                                 0x00040000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3_MASK                                 0x00080000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0_MASK                                0x00100000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1_MASK                                0x00200000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2_MASK                                0x00400000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3_MASK                                0x00800000L
+#define DSCC1_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR_MASK                                0x01000000L
+//DSCC1_DSCC_PPS_CONFIG0
+#define DSCC1_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR__SHIFT                                                      0x0
+#define DSCC1_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR__SHIFT                                                      0x4
+#define DSCC1_DSCC_PPS_CONFIG0__PPS_IDENTIFIER__SHIFT                                                         0x8
+#define DSCC1_DSCC_PPS_CONFIG0__LINEBUF_DEPTH__SHIFT                                                          0x18
+#define DSCC1_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x1c
+#define DSCC1_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR_MASK                                                        0x0000000FL
+#define DSCC1_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR_MASK                                                        0x000000F0L
+#define DSCC1_DSCC_PPS_CONFIG0__PPS_IDENTIFIER_MASK                                                           0x0000FF00L
+#define DSCC1_DSCC_PPS_CONFIG0__LINEBUF_DEPTH_MASK                                                            0x0F000000L
+#define DSCC1_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0xF0000000L
+//DSCC1_DSCC_PPS_CONFIG1
+#define DSCC1_DSCC_PPS_CONFIG1__BITS_PER_PIXEL__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG1__VBR_ENABLE__SHIFT                                                             0xa
+#define DSCC1_DSCC_PPS_CONFIG1__SIMPLE_422__SHIFT                                                             0xb
+#define DSCC1_DSCC_PPS_CONFIG1__CONVERT_RGB__SHIFT                                                            0xc
+#define DSCC1_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE__SHIFT                                                      0xd
+#define DSCC1_DSCC_PPS_CONFIG1__NATIVE_422__SHIFT                                                             0xe
+#define DSCC1_DSCC_PPS_CONFIG1__NATIVE_420__SHIFT                                                             0xf
+#define DSCC1_DSCC_PPS_CONFIG1__CHUNK_SIZE__SHIFT                                                             0x10
+#define DSCC1_DSCC_PPS_CONFIG1__BITS_PER_PIXEL_MASK                                                           0x000003FFL
+#define DSCC1_DSCC_PPS_CONFIG1__VBR_ENABLE_MASK                                                               0x00000400L
+#define DSCC1_DSCC_PPS_CONFIG1__SIMPLE_422_MASK                                                               0x00000800L
+#define DSCC1_DSCC_PPS_CONFIG1__CONVERT_RGB_MASK                                                              0x00001000L
+#define DSCC1_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE_MASK                                                        0x00002000L
+#define DSCC1_DSCC_PPS_CONFIG1__NATIVE_422_MASK                                                               0x00004000L
+#define DSCC1_DSCC_PPS_CONFIG1__NATIVE_420_MASK                                                               0x00008000L
+#define DSCC1_DSCC_PPS_CONFIG1__CHUNK_SIZE_MASK                                                               0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG2
+#define DSCC1_DSCC_PPS_CONFIG2__PIC_WIDTH__SHIFT                                                              0x0
+#define DSCC1_DSCC_PPS_CONFIG2__PIC_HEIGHT__SHIFT                                                             0x10
+#define DSCC1_DSCC_PPS_CONFIG2__PIC_WIDTH_MASK                                                                0x0000FFFFL
+#define DSCC1_DSCC_PPS_CONFIG2__PIC_HEIGHT_MASK                                                               0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG3
+#define DSCC1_DSCC_PPS_CONFIG3__SLICE_WIDTH__SHIFT                                                            0x0
+#define DSCC1_DSCC_PPS_CONFIG3__SLICE_HEIGHT__SHIFT                                                           0x10
+#define DSCC1_DSCC_PPS_CONFIG3__SLICE_WIDTH_MASK                                                              0x0000FFFFL
+#define DSCC1_DSCC_PPS_CONFIG3__SLICE_HEIGHT_MASK                                                             0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG4
+#define DSCC1_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY__SHIFT                                                     0x0
+#define DSCC1_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY__SHIFT                                                      0x10
+#define DSCC1_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY_MASK                                                       0x000003FFL
+#define DSCC1_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY_MASK                                                        0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG5
+#define DSCC1_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE__SHIFT                                                    0x0
+#define DSCC1_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL__SHIFT                                               0x10
+#define DSCC1_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE_MASK                                                      0x0000003FL
+#define DSCC1_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL_MASK                                                 0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG6
+#define DSCC1_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL__SHIFT                                               0x0
+#define DSCC1_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET__SHIFT                                                  0x10
+#define DSCC1_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET__SHIFT                                                 0x18
+#define DSCC1_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL_MASK                                                 0x00000FFFL
+#define DSCC1_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET_MASK                                                    0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET_MASK                                                   0x1F000000L
+//DSCC1_DSCC_PPS_CONFIG7
+#define DSCC1_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET__SHIFT                                                       0x10
+#define DSCC1_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC1_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET_MASK                                                         0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG8
+#define DSCC1_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ__SHIFT                                                 0x10
+#define DSCC1_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC1_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ_MASK                                                   0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG9
+#define DSCC1_DSCC_PPS_CONFIG9__INITIAL_OFFSET__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG9__FINAL_OFFSET__SHIFT                                                           0x10
+#define DSCC1_DSCC_PPS_CONFIG9__INITIAL_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC1_DSCC_PPS_CONFIG9__FINAL_OFFSET_MASK                                                             0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG10
+#define DSCC1_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP__SHIFT                                                       0x0
+#define DSCC1_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP__SHIFT                                                       0x8
+#define DSCC1_DSCC_PPS_CONFIG10__RC_MODEL_SIZE__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP_MASK                                                         0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP_MASK                                                         0x00001F00L
+#define DSCC1_DSCC_PPS_CONFIG10__RC_MODEL_SIZE_MASK                                                           0xFFFF0000L
+//DSCC1_DSCC_PPS_CONFIG11
+#define DSCC1_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0__SHIFT                                                  0x8
+#define DSCC1_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1__SHIFT                                                  0x10
+#define DSCC1_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO__SHIFT                                                      0x18
+#define DSCC1_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI__SHIFT                                                      0x1c
+#define DSCC1_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR_MASK                                                          0x0000000FL
+#define DSCC1_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0_MASK                                                    0x00001F00L
+#define DSCC1_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1_MASK                                                    0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO_MASK                                                        0x0F000000L
+#define DSCC1_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI_MASK                                                        0xF0000000L
+//DSCC1_DSCC_PPS_CONFIG12
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH0__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH1__SHIFT                                                        0x8
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH2__SHIFT                                                        0x10
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH3__SHIFT                                                        0x18
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH0_MASK                                                          0x000000FFL
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH1_MASK                                                          0x0000FF00L
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH2_MASK                                                          0x00FF0000L
+#define DSCC1_DSCC_PPS_CONFIG12__RC_BUF_THRESH3_MASK                                                          0xFF000000L
+//DSCC1_DSCC_PPS_CONFIG13
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH4__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH5__SHIFT                                                        0x8
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH6__SHIFT                                                        0x10
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH7__SHIFT                                                        0x18
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH4_MASK                                                          0x000000FFL
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH5_MASK                                                          0x0000FF00L
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH6_MASK                                                          0x00FF0000L
+#define DSCC1_DSCC_PPS_CONFIG13__RC_BUF_THRESH7_MASK                                                          0xFF000000L
+//DSCC1_DSCC_PPS_CONFIG14
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH8__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH9__SHIFT                                                        0x8
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH10__SHIFT                                                       0x10
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH11__SHIFT                                                       0x18
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH8_MASK                                                          0x000000FFL
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH9_MASK                                                          0x0000FF00L
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH10_MASK                                                         0x00FF0000L
+#define DSCC1_DSCC_PPS_CONFIG14__RC_BUF_THRESH11_MASK                                                         0xFF000000L
+//DSCC1_DSCC_PPS_CONFIG15
+#define DSCC1_DSCC_PPS_CONFIG15__RC_BUF_THRESH12__SHIFT                                                       0x0
+#define DSCC1_DSCC_PPS_CONFIG15__RC_BUF_THRESH13__SHIFT                                                       0x8
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_MIN_QP0__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_MAX_QP0__SHIFT                                                         0x15
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0__SHIFT                                                     0x1a
+#define DSCC1_DSCC_PPS_CONFIG15__RC_BUF_THRESH12_MASK                                                         0x000000FFL
+#define DSCC1_DSCC_PPS_CONFIG15__RC_BUF_THRESH13_MASK                                                         0x0000FF00L
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_MIN_QP0_MASK                                                           0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_MAX_QP0_MASK                                                           0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0_MASK                                                       0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG16
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MIN_QP1__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MAX_QP1__SHIFT                                                         0x5
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1__SHIFT                                                     0xa
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MIN_QP2__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MAX_QP2__SHIFT                                                         0x15
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2__SHIFT                                                     0x1a
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MIN_QP1_MASK                                                           0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MAX_QP1_MASK                                                           0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1_MASK                                                       0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MIN_QP2_MASK                                                           0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_MAX_QP2_MASK                                                           0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2_MASK                                                       0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG17
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MIN_QP3__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MAX_QP3__SHIFT                                                         0x5
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3__SHIFT                                                     0xa
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MIN_QP4__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MAX_QP4__SHIFT                                                         0x15
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4__SHIFT                                                     0x1a
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MIN_QP3_MASK                                                           0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MAX_QP3_MASK                                                           0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3_MASK                                                       0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MIN_QP4_MASK                                                           0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_MAX_QP4_MASK                                                           0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4_MASK                                                       0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG18
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MIN_QP5__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MAX_QP5__SHIFT                                                         0x5
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5__SHIFT                                                     0xa
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MIN_QP6__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MAX_QP6__SHIFT                                                         0x15
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6__SHIFT                                                     0x1a
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MIN_QP5_MASK                                                           0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MAX_QP5_MASK                                                           0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5_MASK                                                       0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MIN_QP6_MASK                                                           0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_MAX_QP6_MASK                                                           0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6_MASK                                                       0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG19
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MIN_QP7__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MAX_QP7__SHIFT                                                         0x5
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7__SHIFT                                                     0xa
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MIN_QP8__SHIFT                                                         0x10
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MAX_QP8__SHIFT                                                         0x15
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8__SHIFT                                                     0x1a
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MIN_QP7_MASK                                                           0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MAX_QP7_MASK                                                           0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7_MASK                                                       0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MIN_QP8_MASK                                                           0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_MAX_QP8_MASK                                                           0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8_MASK                                                       0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG20
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MIN_QP9__SHIFT                                                         0x0
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MAX_QP9__SHIFT                                                         0x5
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9__SHIFT                                                     0xa
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MIN_QP10__SHIFT                                                        0x10
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MAX_QP10__SHIFT                                                        0x15
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10__SHIFT                                                    0x1a
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MIN_QP9_MASK                                                           0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MAX_QP9_MASK                                                           0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9_MASK                                                       0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MIN_QP10_MASK                                                          0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_MAX_QP10_MASK                                                          0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10_MASK                                                      0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG21
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MIN_QP11__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MAX_QP11__SHIFT                                                        0x5
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11__SHIFT                                                    0xa
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MIN_QP12__SHIFT                                                        0x10
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MAX_QP12__SHIFT                                                        0x15
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12__SHIFT                                                    0x1a
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MIN_QP11_MASK                                                          0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MAX_QP11_MASK                                                          0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11_MASK                                                      0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MIN_QP12_MASK                                                          0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_MAX_QP12_MASK                                                          0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12_MASK                                                      0xFC000000L
+//DSCC1_DSCC_PPS_CONFIG22
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MIN_QP13__SHIFT                                                        0x0
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MAX_QP13__SHIFT                                                        0x5
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13__SHIFT                                                    0xa
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MIN_QP14__SHIFT                                                        0x10
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MAX_QP14__SHIFT                                                        0x15
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14__SHIFT                                                    0x1a
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MIN_QP13_MASK                                                          0x0000001FL
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MAX_QP13_MASK                                                          0x000003E0L
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13_MASK                                                      0x0000FC00L
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MIN_QP14_MASK                                                          0x001F0000L
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_MAX_QP14_MASK                                                          0x03E00000L
+#define DSCC1_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14_MASK                                                      0xFC000000L
+//DSCC1_DSCC_MEM_POWER_CONTROL0
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC1_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC1_DSCC_MEM_POWER_CONTROL1
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC1_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER
+#define DSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER__SHIFT                               0x0
+#define DSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER_MASK                                 0xFFFFFFFFL
+//DSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER
+#define DSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER__SHIFT                               0x0
+#define DSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER_MASK                                 0xFFFFFFFFL
+//DSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER
+#define DSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER
+#define DSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER
+#define DSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER
+#define DSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC1_DSCC_MAX_ABS_ERROR0
+#define DSCC1_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR__SHIFT                                              0x0
+#define DSCC1_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR__SHIFT                                             0x10
+#define DSCC1_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR_MASK                                                0x0000FFFFL
+#define DSCC1_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR_MASK                                               0xFFFF0000L
+//DSCC1_DSCC_MAX_ABS_ERROR1
+#define DSCC1_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR__SHIFT                                             0x0
+#define DSCC1_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR_MASK                                               0x0000FFFFL
+//DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__SHIFT           0x0
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_MASK             0x00007FFFL
+//DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__SHIFT           0x0
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_MASK             0x00007FFFL
+//DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__SHIFT           0x0
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_MASK             0x00007FFFL
+//DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__SHIFT           0x0
+#define DSCC1_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_MASK             0x00007FFFL
+//DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__SHIFT   0x0
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_MASK     0x0003FFFFL
+//DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__SHIFT   0x0
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_MASK     0x0003FFFFL
+//DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__SHIFT   0x0
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_MASK     0x0003FFFFL
+//DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__SHIFT   0x0
+#define DSCC1_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_MASK     0x0003FFFFL
+//DSCC1_DSCC_TEST_DEBUG_INDEX0
+#define DSCC1_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0__SHIFT                                           0x0
+#define DSCC1_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0_MASK                                             0x000000FFL
+//DSCC1_DSCC_TEST_DEBUG_INDEX1
+#define DSCC1_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1__SHIFT                                           0x0
+#define DSCC1_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1_MASK                                             0x000000FFL
+//DSCC1_DSCC_TEST_DEBUG_INDEX2
+#define DSCC1_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2__SHIFT                                           0x0
+#define DSCC1_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2_MASK                                             0x000000FFL
+//DSCC1_DSCC_TEST_DEBUG_INDEX3
+#define DSCC1_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3__SHIFT                                           0x0
+#define DSCC1_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3_MASK                                             0x000000FFL
+//DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC1_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
+//DSCC1_DSCC_TEST_DEBUG_DATA0
+#define DSCC1_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0__SHIFT                                             0x0
+#define DSCC1_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0_MASK                                               0xFFFFFFFFL
+//DSCC1_DSCC_TEST_DEBUG_DATA1
+#define DSCC1_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1__SHIFT                                             0x0
+#define DSCC1_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1_MASK                                               0xFFFFFFFFL
+//DSCC1_DSCC_TEST_DEBUG_DATA2
+#define DSCC1_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2__SHIFT                                             0x0
+#define DSCC1_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2_MASK                                               0xFFFFFFFFL
+//DSCC1_DSCC_TEST_DEBUG_DATA3
+#define DSCC1_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3__SHIFT                                             0x0
+#define DSCC1_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3_MASK                                               0xFFFFFFFFL
+//DSCC1_DSCC_DISPCLK_TEST_DEBUG_INDEX0
+#define DSCC1_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0__SHIFT                           0x0
+#define DSCC1_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0_MASK                             0x000000FFL
+//DSCC1_DSCC_DISPCLK_TEST_DEBUG_DATA0
+#define DSCC1_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0__SHIFT                             0x0
+#define DSCC1_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0_MASK                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON18_PERFCOUNTER_CNTL
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON18_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON18_PERFCOUNTER_CNTL2
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON18_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON18_PERFCOUNTER_STATE
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON18_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON18_PERFMON_CNTL
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON18_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON18_PERFMON_CNTL2
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON18_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON18_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON18_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON18_PERFMON_CVALUE_LOW
+#define DC_PERFMON18_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON18_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON18_PERFMON_HI
+#define DC_PERFMON18_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON18_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON18_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON18_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON18_PERFMON_LOW
+#define DC_PERFMON18_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON18_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_top_dispdec
+//DSC_TOP2_DSC_TOP_CONTROL
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_CLOCK_EN__SHIFT                                                         0x0
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS__SHIFT                                               0x4
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS__SHIFT                                                0x8
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS__SHIFT                                                     0xc
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN__SHIFT                                     0x10
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK                                                           0x00000001L
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK                                                 0x00000010L
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS_MASK                                                       0x00001000L
+#define DSC_TOP2_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN_MASK                                       0x00010000L
+//DSC_TOP2_DSC_DEBUG_CONTROL
+#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP2_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
+//DSC_TOP2_DSC_SPARE_DEBUG
+#define DSC_TOP2_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG__SHIFT                                                      0x0
+#define DSC_TOP2_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG_MASK                                                        0xFFFFFFFFL
+//DSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX__SHIFT                                    0x0
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE__SHIFT                               0x8
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX_MASK                                      0x000000FFL
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE_MASK                                 0x00001F00L
+//DSC_TOP2_DSC_TOP_TEST_DEBUG_DATA
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA__SHIFT                                      0x0
+#define DSC_TOP2_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsccif_dispdec
+//DSCCIF2_DSCCIF_CONFIG0
+#define DSCCIF2_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT__SHIFT                                                     0xc
+#define DSCCIF2_DSCCIF_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x10
+#define DSCCIF2_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x18
+#define DSCCIF2_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT_MASK                                                       0x00007000L
+#define DSCCIF2_DSCCIF_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0x000F0000L
+#define DSCCIF2_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x01000000L
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dscc_dispdec
+//DSCC2_DSCC_CONFIG0
+#define DSCC2_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                                                   0x0
+#define DSCC2_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE__SHIFT                                                  0x4
+#define DSCC2_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN__SHIFT                                                  0x8
+#define DSCC2_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION__SHIFT                                     0x10
+#define DSCC2_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                                                     0x0000000FL
+#define DSCC2_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE_MASK                                                    0x00000030L
+#define DSCC2_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN_MASK                                                    0x00000100L
+#define DSCC2_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION_MASK                                       0xFFFF0000L
+//DSCC2_DSCC_CONFIG1
+#define DSCC2_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE__SHIFT                                        0x0
+#define DSCC2_DSCC_CONFIG1__DSCC_DISABLE_ICH__SHIFT                                                           0x18
+#define DSCC2_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE_MASK                                          0x0003FFFFL
+#define DSCC2_DSCC_CONFIG1__DSCC_DISABLE_ICH_MASK                                                             0x01000000L
+//DSCC2_DSCC_CONFIG2
+#define DSCC2_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD__SHIFT                                         0x0
+#define DSCC2_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD__SHIFT                                  0x10
+#define DSCC2_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD_MASK                                           0x0000FFFFL
+#define DSCC2_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD_MASK                                    0xFFFF0000L
+//DSCC2_DSCC_STATUS
+#define DSCC2_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x0
+#define DSCC2_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x00000001L
+//DSCC2_DSCC_INTERRUPT_CONTROL0
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0__SHIFT        0x0
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1__SHIFT        0x1
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2__SHIFT        0x2
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3__SHIFT        0x3
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0_MASK          0x00000001L
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1_MASK          0x00000002L
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2_MASK          0x00000004L
+#define DSCC2_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3_MASK          0x00000008L
+//DSCC2_DSCC_INTERRUPT_CONTROL1
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0__SHIFT                    0x0
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1__SHIFT                    0x1
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2__SHIFT                    0x2
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3__SHIFT                    0x3
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0__SHIFT                   0x4
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1__SHIFT                   0x5
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2__SHIFT                   0x6
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3__SHIFT                   0x7
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN__SHIFT                   0x8
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0_MASK                      0x00000001L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1_MASK                      0x00000002L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2_MASK                      0x00000004L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3_MASK                      0x00000008L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0_MASK                     0x00000010L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1_MASK                     0x00000020L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2_MASK                     0x00000040L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3_MASK                     0x00000080L
+#define DSCC2_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN_MASK                     0x00000100L
+//DSCC2_DSCC_INTERRUPT_STATUS0
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0__SHIFT                0x0
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1__SHIFT                0x1
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2__SHIFT                0x2
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3__SHIFT                0x3
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0__SHIFT                   0x10
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1__SHIFT                   0x11
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2__SHIFT                   0x12
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3__SHIFT                   0x13
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0_MASK                  0x00000001L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1_MASK                  0x00000002L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2_MASK                  0x00000004L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3_MASK                  0x00000008L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0_MASK                     0x00010000L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1_MASK                     0x00020000L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2_MASK                     0x00040000L
+#define DSCC2_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3_MASK                     0x00080000L
+//DSCC2_DSCC_INTERRUPT_STATUS1
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0__SHIFT                            0x0
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1__SHIFT                            0x1
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2__SHIFT                            0x2
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3__SHIFT                            0x3
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0__SHIFT                           0x4
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1__SHIFT                           0x5
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2__SHIFT                           0x6
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3__SHIFT                           0x7
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED__SHIFT                           0x8
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0__SHIFT                               0x10
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1__SHIFT                               0x11
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2__SHIFT                               0x12
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3__SHIFT                               0x13
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0__SHIFT                              0x14
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1__SHIFT                              0x15
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2__SHIFT                              0x16
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3__SHIFT                              0x17
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR__SHIFT                              0x18
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0_MASK                              0x00000001L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1_MASK                              0x00000002L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2_MASK                              0x00000004L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3_MASK                              0x00000008L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0_MASK                             0x00000010L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1_MASK                             0x00000020L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2_MASK                             0x00000040L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3_MASK                             0x00000080L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_MASK                             0x00000100L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0_MASK                                 0x00010000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1_MASK                                 0x00020000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2_MASK                                 0x00040000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3_MASK                                 0x00080000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0_MASK                                0x00100000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1_MASK                                0x00200000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2_MASK                                0x00400000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3_MASK                                0x00800000L
+#define DSCC2_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR_MASK                                0x01000000L
+//DSCC2_DSCC_PPS_CONFIG0
+#define DSCC2_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR__SHIFT                                                      0x0
+#define DSCC2_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR__SHIFT                                                      0x4
+#define DSCC2_DSCC_PPS_CONFIG0__PPS_IDENTIFIER__SHIFT                                                         0x8
+#define DSCC2_DSCC_PPS_CONFIG0__LINEBUF_DEPTH__SHIFT                                                          0x18
+#define DSCC2_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x1c
+#define DSCC2_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR_MASK                                                        0x0000000FL
+#define DSCC2_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR_MASK                                                        0x000000F0L
+#define DSCC2_DSCC_PPS_CONFIG0__PPS_IDENTIFIER_MASK                                                           0x0000FF00L
+#define DSCC2_DSCC_PPS_CONFIG0__LINEBUF_DEPTH_MASK                                                            0x0F000000L
+#define DSCC2_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0xF0000000L
+//DSCC2_DSCC_PPS_CONFIG1
+#define DSCC2_DSCC_PPS_CONFIG1__BITS_PER_PIXEL__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG1__VBR_ENABLE__SHIFT                                                             0xa
+#define DSCC2_DSCC_PPS_CONFIG1__SIMPLE_422__SHIFT                                                             0xb
+#define DSCC2_DSCC_PPS_CONFIG1__CONVERT_RGB__SHIFT                                                            0xc
+#define DSCC2_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE__SHIFT                                                      0xd
+#define DSCC2_DSCC_PPS_CONFIG1__NATIVE_422__SHIFT                                                             0xe
+#define DSCC2_DSCC_PPS_CONFIG1__NATIVE_420__SHIFT                                                             0xf
+#define DSCC2_DSCC_PPS_CONFIG1__CHUNK_SIZE__SHIFT                                                             0x10
+#define DSCC2_DSCC_PPS_CONFIG1__BITS_PER_PIXEL_MASK                                                           0x000003FFL
+#define DSCC2_DSCC_PPS_CONFIG1__VBR_ENABLE_MASK                                                               0x00000400L
+#define DSCC2_DSCC_PPS_CONFIG1__SIMPLE_422_MASK                                                               0x00000800L
+#define DSCC2_DSCC_PPS_CONFIG1__CONVERT_RGB_MASK                                                              0x00001000L
+#define DSCC2_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE_MASK                                                        0x00002000L
+#define DSCC2_DSCC_PPS_CONFIG1__NATIVE_422_MASK                                                               0x00004000L
+#define DSCC2_DSCC_PPS_CONFIG1__NATIVE_420_MASK                                                               0x00008000L
+#define DSCC2_DSCC_PPS_CONFIG1__CHUNK_SIZE_MASK                                                               0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG2
+#define DSCC2_DSCC_PPS_CONFIG2__PIC_WIDTH__SHIFT                                                              0x0
+#define DSCC2_DSCC_PPS_CONFIG2__PIC_HEIGHT__SHIFT                                                             0x10
+#define DSCC2_DSCC_PPS_CONFIG2__PIC_WIDTH_MASK                                                                0x0000FFFFL
+#define DSCC2_DSCC_PPS_CONFIG2__PIC_HEIGHT_MASK                                                               0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG3
+#define DSCC2_DSCC_PPS_CONFIG3__SLICE_WIDTH__SHIFT                                                            0x0
+#define DSCC2_DSCC_PPS_CONFIG3__SLICE_HEIGHT__SHIFT                                                           0x10
+#define DSCC2_DSCC_PPS_CONFIG3__SLICE_WIDTH_MASK                                                              0x0000FFFFL
+#define DSCC2_DSCC_PPS_CONFIG3__SLICE_HEIGHT_MASK                                                             0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG4
+#define DSCC2_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY__SHIFT                                                     0x0
+#define DSCC2_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY__SHIFT                                                      0x10
+#define DSCC2_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY_MASK                                                       0x000003FFL
+#define DSCC2_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY_MASK                                                        0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG5
+#define DSCC2_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE__SHIFT                                                    0x0
+#define DSCC2_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL__SHIFT                                               0x10
+#define DSCC2_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE_MASK                                                      0x0000003FL
+#define DSCC2_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL_MASK                                                 0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG6
+#define DSCC2_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL__SHIFT                                               0x0
+#define DSCC2_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET__SHIFT                                                  0x10
+#define DSCC2_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET__SHIFT                                                 0x18
+#define DSCC2_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL_MASK                                                 0x00000FFFL
+#define DSCC2_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET_MASK                                                    0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET_MASK                                                   0x1F000000L
+//DSCC2_DSCC_PPS_CONFIG7
+#define DSCC2_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET__SHIFT                                                       0x10
+#define DSCC2_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC2_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET_MASK                                                         0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG8
+#define DSCC2_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ__SHIFT                                                 0x10
+#define DSCC2_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC2_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ_MASK                                                   0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG9
+#define DSCC2_DSCC_PPS_CONFIG9__INITIAL_OFFSET__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG9__FINAL_OFFSET__SHIFT                                                           0x10
+#define DSCC2_DSCC_PPS_CONFIG9__INITIAL_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC2_DSCC_PPS_CONFIG9__FINAL_OFFSET_MASK                                                             0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG10
+#define DSCC2_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP__SHIFT                                                       0x0
+#define DSCC2_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP__SHIFT                                                       0x8
+#define DSCC2_DSCC_PPS_CONFIG10__RC_MODEL_SIZE__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP_MASK                                                         0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP_MASK                                                         0x00001F00L
+#define DSCC2_DSCC_PPS_CONFIG10__RC_MODEL_SIZE_MASK                                                           0xFFFF0000L
+//DSCC2_DSCC_PPS_CONFIG11
+#define DSCC2_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0__SHIFT                                                  0x8
+#define DSCC2_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1__SHIFT                                                  0x10
+#define DSCC2_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO__SHIFT                                                      0x18
+#define DSCC2_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI__SHIFT                                                      0x1c
+#define DSCC2_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR_MASK                                                          0x0000000FL
+#define DSCC2_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0_MASK                                                    0x00001F00L
+#define DSCC2_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1_MASK                                                    0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO_MASK                                                        0x0F000000L
+#define DSCC2_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI_MASK                                                        0xF0000000L
+//DSCC2_DSCC_PPS_CONFIG12
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH0__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH1__SHIFT                                                        0x8
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH2__SHIFT                                                        0x10
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH3__SHIFT                                                        0x18
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH0_MASK                                                          0x000000FFL
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH1_MASK                                                          0x0000FF00L
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH2_MASK                                                          0x00FF0000L
+#define DSCC2_DSCC_PPS_CONFIG12__RC_BUF_THRESH3_MASK                                                          0xFF000000L
+//DSCC2_DSCC_PPS_CONFIG13
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH4__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH5__SHIFT                                                        0x8
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH6__SHIFT                                                        0x10
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH7__SHIFT                                                        0x18
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH4_MASK                                                          0x000000FFL
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH5_MASK                                                          0x0000FF00L
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH6_MASK                                                          0x00FF0000L
+#define DSCC2_DSCC_PPS_CONFIG13__RC_BUF_THRESH7_MASK                                                          0xFF000000L
+//DSCC2_DSCC_PPS_CONFIG14
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH8__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH9__SHIFT                                                        0x8
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH10__SHIFT                                                       0x10
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH11__SHIFT                                                       0x18
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH8_MASK                                                          0x000000FFL
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH9_MASK                                                          0x0000FF00L
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH10_MASK                                                         0x00FF0000L
+#define DSCC2_DSCC_PPS_CONFIG14__RC_BUF_THRESH11_MASK                                                         0xFF000000L
+//DSCC2_DSCC_PPS_CONFIG15
+#define DSCC2_DSCC_PPS_CONFIG15__RC_BUF_THRESH12__SHIFT                                                       0x0
+#define DSCC2_DSCC_PPS_CONFIG15__RC_BUF_THRESH13__SHIFT                                                       0x8
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_MIN_QP0__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_MAX_QP0__SHIFT                                                         0x15
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0__SHIFT                                                     0x1a
+#define DSCC2_DSCC_PPS_CONFIG15__RC_BUF_THRESH12_MASK                                                         0x000000FFL
+#define DSCC2_DSCC_PPS_CONFIG15__RC_BUF_THRESH13_MASK                                                         0x0000FF00L
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_MIN_QP0_MASK                                                           0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_MAX_QP0_MASK                                                           0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0_MASK                                                       0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG16
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MIN_QP1__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MAX_QP1__SHIFT                                                         0x5
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1__SHIFT                                                     0xa
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MIN_QP2__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MAX_QP2__SHIFT                                                         0x15
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2__SHIFT                                                     0x1a
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MIN_QP1_MASK                                                           0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MAX_QP1_MASK                                                           0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1_MASK                                                       0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MIN_QP2_MASK                                                           0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_MAX_QP2_MASK                                                           0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2_MASK                                                       0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG17
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MIN_QP3__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MAX_QP3__SHIFT                                                         0x5
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3__SHIFT                                                     0xa
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MIN_QP4__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MAX_QP4__SHIFT                                                         0x15
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4__SHIFT                                                     0x1a
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MIN_QP3_MASK                                                           0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MAX_QP3_MASK                                                           0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3_MASK                                                       0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MIN_QP4_MASK                                                           0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_MAX_QP4_MASK                                                           0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4_MASK                                                       0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG18
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MIN_QP5__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MAX_QP5__SHIFT                                                         0x5
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5__SHIFT                                                     0xa
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MIN_QP6__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MAX_QP6__SHIFT                                                         0x15
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6__SHIFT                                                     0x1a
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MIN_QP5_MASK                                                           0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MAX_QP5_MASK                                                           0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5_MASK                                                       0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MIN_QP6_MASK                                                           0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_MAX_QP6_MASK                                                           0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6_MASK                                                       0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG19
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MIN_QP7__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MAX_QP7__SHIFT                                                         0x5
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7__SHIFT                                                     0xa
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MIN_QP8__SHIFT                                                         0x10
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MAX_QP8__SHIFT                                                         0x15
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8__SHIFT                                                     0x1a
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MIN_QP7_MASK                                                           0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MAX_QP7_MASK                                                           0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7_MASK                                                       0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MIN_QP8_MASK                                                           0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_MAX_QP8_MASK                                                           0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8_MASK                                                       0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG20
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MIN_QP9__SHIFT                                                         0x0
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MAX_QP9__SHIFT                                                         0x5
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9__SHIFT                                                     0xa
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MIN_QP10__SHIFT                                                        0x10
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MAX_QP10__SHIFT                                                        0x15
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10__SHIFT                                                    0x1a
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MIN_QP9_MASK                                                           0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MAX_QP9_MASK                                                           0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9_MASK                                                       0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MIN_QP10_MASK                                                          0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_MAX_QP10_MASK                                                          0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10_MASK                                                      0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG21
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MIN_QP11__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MAX_QP11__SHIFT                                                        0x5
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11__SHIFT                                                    0xa
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MIN_QP12__SHIFT                                                        0x10
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MAX_QP12__SHIFT                                                        0x15
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12__SHIFT                                                    0x1a
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MIN_QP11_MASK                                                          0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MAX_QP11_MASK                                                          0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11_MASK                                                      0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MIN_QP12_MASK                                                          0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_MAX_QP12_MASK                                                          0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12_MASK                                                      0xFC000000L
+//DSCC2_DSCC_PPS_CONFIG22
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MIN_QP13__SHIFT                                                        0x0
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MAX_QP13__SHIFT                                                        0x5
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13__SHIFT                                                    0xa
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MIN_QP14__SHIFT                                                        0x10
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MAX_QP14__SHIFT                                                        0x15
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14__SHIFT                                                    0x1a
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MIN_QP13_MASK                                                          0x0000001FL
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MAX_QP13_MASK                                                          0x000003E0L
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13_MASK                                                      0x0000FC00L
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MIN_QP14_MASK                                                          0x001F0000L
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_MAX_QP14_MASK                                                          0x03E00000L
+#define DSCC2_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14_MASK                                                      0xFC000000L
+//DSCC2_DSCC_MEM_POWER_CONTROL0
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC2_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC2_DSCC_MEM_POWER_CONTROL1
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC2_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER
+#define DSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER__SHIFT                               0x0
+#define DSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER_MASK                                 0xFFFFFFFFL
+//DSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER
+#define DSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER__SHIFT                               0x0
+#define DSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER_MASK                                 0xFFFFFFFFL
+//DSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER
+#define DSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER
+#define DSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER
+#define DSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER
+#define DSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC2_DSCC_MAX_ABS_ERROR0
+#define DSCC2_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR__SHIFT                                              0x0
+#define DSCC2_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR__SHIFT                                             0x10
+#define DSCC2_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR_MASK                                                0x0000FFFFL
+#define DSCC2_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR_MASK                                               0xFFFF0000L
+//DSCC2_DSCC_MAX_ABS_ERROR1
+#define DSCC2_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR__SHIFT                                             0x0
+#define DSCC2_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR_MASK                                               0x0000FFFFL
+//DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__SHIFT           0x0
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_MASK             0x00007FFFL
+//DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__SHIFT           0x0
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_MASK             0x00007FFFL
+//DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__SHIFT           0x0
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_MASK             0x00007FFFL
+//DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__SHIFT           0x0
+#define DSCC2_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_MASK             0x00007FFFL
+//DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__SHIFT   0x0
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_MASK     0x0003FFFFL
+//DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__SHIFT   0x0
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_MASK     0x0003FFFFL
+//DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__SHIFT   0x0
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_MASK     0x0003FFFFL
+//DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__SHIFT   0x0
+#define DSCC2_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_MASK     0x0003FFFFL
+//DSCC2_DSCC_TEST_DEBUG_INDEX0
+#define DSCC2_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0__SHIFT                                           0x0
+#define DSCC2_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0_MASK                                             0x000000FFL
+//DSCC2_DSCC_TEST_DEBUG_INDEX1
+#define DSCC2_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1__SHIFT                                           0x0
+#define DSCC2_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1_MASK                                             0x000000FFL
+//DSCC2_DSCC_TEST_DEBUG_INDEX2
+#define DSCC2_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2__SHIFT                                           0x0
+#define DSCC2_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2_MASK                                             0x000000FFL
+//DSCC2_DSCC_TEST_DEBUG_INDEX3
+#define DSCC2_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3__SHIFT                                           0x0
+#define DSCC2_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3_MASK                                             0x000000FFL
+//DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC2_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
+//DSCC2_DSCC_TEST_DEBUG_DATA0
+#define DSCC2_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0__SHIFT                                             0x0
+#define DSCC2_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0_MASK                                               0xFFFFFFFFL
+//DSCC2_DSCC_TEST_DEBUG_DATA1
+#define DSCC2_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1__SHIFT                                             0x0
+#define DSCC2_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1_MASK                                               0xFFFFFFFFL
+//DSCC2_DSCC_TEST_DEBUG_DATA2
+#define DSCC2_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2__SHIFT                                             0x0
+#define DSCC2_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2_MASK                                               0xFFFFFFFFL
+//DSCC2_DSCC_TEST_DEBUG_DATA3
+#define DSCC2_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3__SHIFT                                             0x0
+#define DSCC2_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3_MASK                                               0xFFFFFFFFL
+//DSCC2_DSCC_DISPCLK_TEST_DEBUG_INDEX0
+#define DSCC2_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0__SHIFT                           0x0
+#define DSCC2_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0_MASK                             0x000000FFL
+//DSCC2_DSCC_DISPCLK_TEST_DEBUG_DATA0
+#define DSCC2_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0__SHIFT                             0x0
+#define DSCC2_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0_MASK                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON19_PERFCOUNTER_CNTL
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON19_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON19_PERFCOUNTER_CNTL2
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON19_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON19_PERFCOUNTER_STATE
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON19_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON19_PERFMON_CNTL
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON19_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON19_PERFMON_CNTL2
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON19_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON19_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON19_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON19_PERFMON_CVALUE_LOW
+#define DC_PERFMON19_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON19_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON19_PERFMON_HI
+#define DC_PERFMON19_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON19_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON19_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON19_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON19_PERFMON_LOW
+#define DC_PERFMON19_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON19_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_top_dispdec
+//DSC_TOP3_DSC_TOP_CONTROL
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_CLOCK_EN__SHIFT                                                         0x0
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS__SHIFT                                               0x4
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS__SHIFT                                                0x8
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS__SHIFT                                                     0xc
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN__SHIFT                                     0x10
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK                                                           0x00000001L
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK                                                 0x00000010L
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_FGCG_REP_DIS_MASK                                                       0x00001000L
+#define DSC_TOP3_DSC_TOP_CONTROL__DSC_DSCCLK_DYNAMIC_CLOCK_GATE_EN_MASK                                       0x00010000L
+//DSC_TOP3_DSC_DEBUG_CONTROL
+#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP3_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
+//DSC_TOP3_DSC_SPARE_DEBUG
+#define DSC_TOP3_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG__SHIFT                                                      0x0
+#define DSC_TOP3_DSC_SPARE_DEBUG__DSC_SPARE_DEBUG_MASK                                                        0xFFFFFFFFL
+//DSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX__SHIFT                                    0x0
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE__SHIFT                               0x8
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_INDEX_MASK                                      0x000000FFL
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_INDEX__DSC_TOP_TEST_DEBUG_BUS_ROTATE_MASK                                 0x00001F00L
+//DSC_TOP3_DSC_TOP_TEST_DEBUG_DATA
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA__SHIFT                                      0x0
+#define DSC_TOP3_DSC_TOP_TEST_DEBUG_DATA__DSC_TOP_TEST_DEBUG_DATA_MASK                                        0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsccif_dispdec
+//DSCCIF3_DSCCIF_CONFIG0
+#define DSCCIF3_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT__SHIFT                                                     0xc
+#define DSCCIF3_DSCCIF_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x10
+#define DSCCIF3_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x18
+#define DSCCIF3_DSCCIF_CONFIG0__INPUT_PIXEL_FORMAT_MASK                                                       0x00007000L
+#define DSCCIF3_DSCCIF_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0x000F0000L
+#define DSCCIF3_DSCCIF_CONFIG0__DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x01000000L
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dscc_dispdec
+//DSCC3_DSCC_CONFIG0
+#define DSCC3_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                                                   0x0
+#define DSCC3_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE__SHIFT                                                  0x4
+#define DSCC3_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN__SHIFT                                                  0x8
+#define DSCC3_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION__SHIFT                                     0x10
+#define DSCC3_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                                                     0x0000000FL
+#define DSCC3_DSCC_CONFIG0__NUMBER_OF_SLICES_PER_LINE_MASK                                                    0x00000030L
+#define DSCC3_DSCC_CONFIG0__ALTERNATE_ICH_ENCODING_EN_MASK                                                    0x00000100L
+#define DSCC3_DSCC_CONFIG0__NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION_MASK                                       0xFFFF0000L
+//DSCC3_DSCC_CONFIG1
+#define DSCC3_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE__SHIFT                                        0x0
+#define DSCC3_DSCC_CONFIG1__DSCC_DISABLE_ICH__SHIFT                                                           0x18
+#define DSCC3_DSCC_CONFIG1__DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE_MASK                                          0x0003FFFFL
+#define DSCC3_DSCC_CONFIG1__DSCC_DISABLE_ICH_MASK                                                             0x01000000L
+//DSCC3_DSCC_CONFIG2
+#define DSCC3_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD__SHIFT                                         0x0
+#define DSCC3_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD__SHIFT                                  0x10
+#define DSCC3_DSCC_CONFIG2__OUTPUT_BUFFER_ELASTICITY_THRESHOLD_MASK                                           0x0000FFFFL
+#define DSCC3_DSCC_CONFIG2__OUTPUT_BUFFER_TOTAL_PIXEL_COUNT_THRESHOLD_MASK                                    0xFFFF0000L
+//DSCC3_DSCC_STATUS
+#define DSCC3_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING__SHIFT                                       0x0
+#define DSCC3_DSCC_STATUS__DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING_MASK                                         0x00000001L
+//DSCC3_DSCC_INTERRUPT_CONTROL0
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0__SHIFT        0x0
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1__SHIFT        0x1
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2__SHIFT        0x2
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3__SHIFT        0x3
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN0_MASK          0x00000001L
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN1_MASK          0x00000002L
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN2_MASK          0x00000004L
+#define DSCC3_DSCC_INTERRUPT_CONTROL0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED_INT_EN3_MASK          0x00000008L
+//DSCC3_DSCC_INTERRUPT_CONTROL1
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0__SHIFT                    0x0
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1__SHIFT                    0x1
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2__SHIFT                    0x2
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3__SHIFT                    0x3
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0__SHIFT                   0x4
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1__SHIFT                   0x5
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2__SHIFT                   0x6
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3__SHIFT                   0x7
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN__SHIFT                   0x8
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN0_MASK                      0x00000001L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN1_MASK                      0x00000002L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN2_MASK                      0x00000004L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED_INT_EN3_MASK                      0x00000008L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN0_MASK                     0x00000010L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN1_MASK                     0x00000020L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN2_MASK                     0x00000040L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED_INT_EN3_MASK                     0x00000080L
+#define DSCC3_DSCC_INTERRUPT_CONTROL1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_INT_EN_MASK                     0x00000100L
+//DSCC3_DSCC_INTERRUPT_STATUS0
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0__SHIFT                0x0
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1__SHIFT                0x1
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2__SHIFT                0x2
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3__SHIFT                0x3
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0__SHIFT                   0x10
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1__SHIFT                   0x11
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2__SHIFT                   0x12
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3__SHIFT                   0x13
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED0_MASK                  0x00000001L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED1_MASK                  0x00000002L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED2_MASK                  0x00000004L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_OCCURRED3_MASK                  0x00000008L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR0_MASK                     0x00010000L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR1_MASK                     0x00020000L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR2_MASK                     0x00040000L
+#define DSCC3_DSCC_INTERRUPT_STATUS0__DSCC_RATE_CONTROL_BUFFER_MODEL_OVERFLOW_CLEAR3_MASK                     0x00080000L
+//DSCC3_DSCC_INTERRUPT_STATUS1
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0__SHIFT                            0x0
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1__SHIFT                            0x1
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2__SHIFT                            0x2
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3__SHIFT                            0x3
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0__SHIFT                           0x4
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1__SHIFT                           0x5
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2__SHIFT                           0x6
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3__SHIFT                           0x7
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED__SHIFT                           0x8
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0__SHIFT                               0x10
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1__SHIFT                               0x11
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2__SHIFT                               0x12
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3__SHIFT                               0x13
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0__SHIFT                              0x14
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1__SHIFT                              0x15
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2__SHIFT                              0x16
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3__SHIFT                              0x17
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR__SHIFT                              0x18
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED0_MASK                              0x00000001L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED1_MASK                              0x00000002L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED2_MASK                              0x00000004L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_OCCURRED3_MASK                              0x00000008L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED0_MASK                             0x00000010L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED1_MASK                             0x00000020L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED2_MASK                             0x00000040L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_OCCURRED3_MASK                             0x00000080L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_OCCURRED_MASK                             0x00000100L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR0_MASK                                 0x00010000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR1_MASK                                 0x00020000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR2_MASK                                 0x00040000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_OVERFLOW_CLEAR3_MASK                                 0x00080000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR0_MASK                                0x00100000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR1_MASK                                0x00200000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR2_MASK                                0x00400000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_OUTPUT_BUFFER_UNDERFLOW_CLEAR3_MASK                                0x00800000L
+#define DSCC3_DSCC_INTERRUPT_STATUS1__DSCC_END_OF_FRAME_NOT_REACHED_CLEAR_MASK                                0x01000000L
+//DSCC3_DSCC_PPS_CONFIG0
+#define DSCC3_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR__SHIFT                                                      0x0
+#define DSCC3_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR__SHIFT                                                      0x4
+#define DSCC3_DSCC_PPS_CONFIG0__PPS_IDENTIFIER__SHIFT                                                         0x8
+#define DSCC3_DSCC_PPS_CONFIG0__LINEBUF_DEPTH__SHIFT                                                          0x18
+#define DSCC3_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT__SHIFT                                                     0x1c
+#define DSCC3_DSCC_PPS_CONFIG0__DSC_VERSION_MINOR_MASK                                                        0x0000000FL
+#define DSCC3_DSCC_PPS_CONFIG0__DSC_VERSION_MAJOR_MASK                                                        0x000000F0L
+#define DSCC3_DSCC_PPS_CONFIG0__PPS_IDENTIFIER_MASK                                                           0x0000FF00L
+#define DSCC3_DSCC_PPS_CONFIG0__LINEBUF_DEPTH_MASK                                                            0x0F000000L
+#define DSCC3_DSCC_PPS_CONFIG0__BITS_PER_COMPONENT_MASK                                                       0xF0000000L
+//DSCC3_DSCC_PPS_CONFIG1
+#define DSCC3_DSCC_PPS_CONFIG1__BITS_PER_PIXEL__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG1__VBR_ENABLE__SHIFT                                                             0xa
+#define DSCC3_DSCC_PPS_CONFIG1__SIMPLE_422__SHIFT                                                             0xb
+#define DSCC3_DSCC_PPS_CONFIG1__CONVERT_RGB__SHIFT                                                            0xc
+#define DSCC3_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE__SHIFT                                                      0xd
+#define DSCC3_DSCC_PPS_CONFIG1__NATIVE_422__SHIFT                                                             0xe
+#define DSCC3_DSCC_PPS_CONFIG1__NATIVE_420__SHIFT                                                             0xf
+#define DSCC3_DSCC_PPS_CONFIG1__CHUNK_SIZE__SHIFT                                                             0x10
+#define DSCC3_DSCC_PPS_CONFIG1__BITS_PER_PIXEL_MASK                                                           0x000003FFL
+#define DSCC3_DSCC_PPS_CONFIG1__VBR_ENABLE_MASK                                                               0x00000400L
+#define DSCC3_DSCC_PPS_CONFIG1__SIMPLE_422_MASK                                                               0x00000800L
+#define DSCC3_DSCC_PPS_CONFIG1__CONVERT_RGB_MASK                                                              0x00001000L
+#define DSCC3_DSCC_PPS_CONFIG1__BLOCK_PRED_ENABLE_MASK                                                        0x00002000L
+#define DSCC3_DSCC_PPS_CONFIG1__NATIVE_422_MASK                                                               0x00004000L
+#define DSCC3_DSCC_PPS_CONFIG1__NATIVE_420_MASK                                                               0x00008000L
+#define DSCC3_DSCC_PPS_CONFIG1__CHUNK_SIZE_MASK                                                               0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG2
+#define DSCC3_DSCC_PPS_CONFIG2__PIC_WIDTH__SHIFT                                                              0x0
+#define DSCC3_DSCC_PPS_CONFIG2__PIC_HEIGHT__SHIFT                                                             0x10
+#define DSCC3_DSCC_PPS_CONFIG2__PIC_WIDTH_MASK                                                                0x0000FFFFL
+#define DSCC3_DSCC_PPS_CONFIG2__PIC_HEIGHT_MASK                                                               0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG3
+#define DSCC3_DSCC_PPS_CONFIG3__SLICE_WIDTH__SHIFT                                                            0x0
+#define DSCC3_DSCC_PPS_CONFIG3__SLICE_HEIGHT__SHIFT                                                           0x10
+#define DSCC3_DSCC_PPS_CONFIG3__SLICE_WIDTH_MASK                                                              0x0000FFFFL
+#define DSCC3_DSCC_PPS_CONFIG3__SLICE_HEIGHT_MASK                                                             0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG4
+#define DSCC3_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY__SHIFT                                                     0x0
+#define DSCC3_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY__SHIFT                                                      0x10
+#define DSCC3_DSCC_PPS_CONFIG4__INITIAL_XMIT_DELAY_MASK                                                       0x000003FFL
+#define DSCC3_DSCC_PPS_CONFIG4__INITIAL_DEC_DELAY_MASK                                                        0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG5
+#define DSCC3_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE__SHIFT                                                    0x0
+#define DSCC3_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL__SHIFT                                               0x10
+#define DSCC3_DSCC_PPS_CONFIG5__INITIAL_SCALE_VALUE_MASK                                                      0x0000003FL
+#define DSCC3_DSCC_PPS_CONFIG5__SCALE_INCREMENT_INTERVAL_MASK                                                 0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG6
+#define DSCC3_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL__SHIFT                                               0x0
+#define DSCC3_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET__SHIFT                                                  0x10
+#define DSCC3_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET__SHIFT                                                 0x18
+#define DSCC3_DSCC_PPS_CONFIG6__SCALE_DECREMENT_INTERVAL_MASK                                                 0x00000FFFL
+#define DSCC3_DSCC_PPS_CONFIG6__FIRST_LINE_BPG_OFFSET_MASK                                                    0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG6__SECOND_LINE_BPG_OFFSET_MASK                                                   0x1F000000L
+//DSCC3_DSCC_PPS_CONFIG7
+#define DSCC3_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET__SHIFT                                                       0x10
+#define DSCC3_DSCC_PPS_CONFIG7__NFL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC3_DSCC_PPS_CONFIG7__SLICE_BPG_OFFSET_MASK                                                         0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG8
+#define DSCC3_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ__SHIFT                                                 0x10
+#define DSCC3_DSCC_PPS_CONFIG8__NSL_BPG_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC3_DSCC_PPS_CONFIG8__SECOND_LINE_OFFSET_ADJ_MASK                                                   0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG9
+#define DSCC3_DSCC_PPS_CONFIG9__INITIAL_OFFSET__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG9__FINAL_OFFSET__SHIFT                                                           0x10
+#define DSCC3_DSCC_PPS_CONFIG9__INITIAL_OFFSET_MASK                                                           0x0000FFFFL
+#define DSCC3_DSCC_PPS_CONFIG9__FINAL_OFFSET_MASK                                                             0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG10
+#define DSCC3_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP__SHIFT                                                       0x0
+#define DSCC3_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP__SHIFT                                                       0x8
+#define DSCC3_DSCC_PPS_CONFIG10__RC_MODEL_SIZE__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG10__FLATNESS_MIN_QP_MASK                                                         0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG10__FLATNESS_MAX_QP_MASK                                                         0x00001F00L
+#define DSCC3_DSCC_PPS_CONFIG10__RC_MODEL_SIZE_MASK                                                           0xFFFF0000L
+//DSCC3_DSCC_PPS_CONFIG11
+#define DSCC3_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0__SHIFT                                                  0x8
+#define DSCC3_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1__SHIFT                                                  0x10
+#define DSCC3_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO__SHIFT                                                      0x18
+#define DSCC3_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI__SHIFT                                                      0x1c
+#define DSCC3_DSCC_PPS_CONFIG11__RC_EDGE_FACTOR_MASK                                                          0x0000000FL
+#define DSCC3_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT0_MASK                                                    0x00001F00L
+#define DSCC3_DSCC_PPS_CONFIG11__RC_QUANT_INCR_LIMIT1_MASK                                                    0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_LO_MASK                                                        0x0F000000L
+#define DSCC3_DSCC_PPS_CONFIG11__RC_TGT_OFFSET_HI_MASK                                                        0xF0000000L
+//DSCC3_DSCC_PPS_CONFIG12
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH0__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH1__SHIFT                                                        0x8
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH2__SHIFT                                                        0x10
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH3__SHIFT                                                        0x18
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH0_MASK                                                          0x000000FFL
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH1_MASK                                                          0x0000FF00L
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH2_MASK                                                          0x00FF0000L
+#define DSCC3_DSCC_PPS_CONFIG12__RC_BUF_THRESH3_MASK                                                          0xFF000000L
+//DSCC3_DSCC_PPS_CONFIG13
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH4__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH5__SHIFT                                                        0x8
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH6__SHIFT                                                        0x10
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH7__SHIFT                                                        0x18
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH4_MASK                                                          0x000000FFL
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH5_MASK                                                          0x0000FF00L
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH6_MASK                                                          0x00FF0000L
+#define DSCC3_DSCC_PPS_CONFIG13__RC_BUF_THRESH7_MASK                                                          0xFF000000L
+//DSCC3_DSCC_PPS_CONFIG14
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH8__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH9__SHIFT                                                        0x8
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH10__SHIFT                                                       0x10
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH11__SHIFT                                                       0x18
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH8_MASK                                                          0x000000FFL
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH9_MASK                                                          0x0000FF00L
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH10_MASK                                                         0x00FF0000L
+#define DSCC3_DSCC_PPS_CONFIG14__RC_BUF_THRESH11_MASK                                                         0xFF000000L
+//DSCC3_DSCC_PPS_CONFIG15
+#define DSCC3_DSCC_PPS_CONFIG15__RC_BUF_THRESH12__SHIFT                                                       0x0
+#define DSCC3_DSCC_PPS_CONFIG15__RC_BUF_THRESH13__SHIFT                                                       0x8
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_MIN_QP0__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_MAX_QP0__SHIFT                                                         0x15
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0__SHIFT                                                     0x1a
+#define DSCC3_DSCC_PPS_CONFIG15__RC_BUF_THRESH12_MASK                                                         0x000000FFL
+#define DSCC3_DSCC_PPS_CONFIG15__RC_BUF_THRESH13_MASK                                                         0x0000FF00L
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_MIN_QP0_MASK                                                           0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_MAX_QP0_MASK                                                           0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG15__RANGE_BPG_OFFSET0_MASK                                                       0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG16
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MIN_QP1__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MAX_QP1__SHIFT                                                         0x5
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1__SHIFT                                                     0xa
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MIN_QP2__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MAX_QP2__SHIFT                                                         0x15
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2__SHIFT                                                     0x1a
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MIN_QP1_MASK                                                           0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MAX_QP1_MASK                                                           0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET1_MASK                                                       0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MIN_QP2_MASK                                                           0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_MAX_QP2_MASK                                                           0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG16__RANGE_BPG_OFFSET2_MASK                                                       0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG17
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MIN_QP3__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MAX_QP3__SHIFT                                                         0x5
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3__SHIFT                                                     0xa
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MIN_QP4__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MAX_QP4__SHIFT                                                         0x15
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4__SHIFT                                                     0x1a
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MIN_QP3_MASK                                                           0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MAX_QP3_MASK                                                           0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET3_MASK                                                       0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MIN_QP4_MASK                                                           0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_MAX_QP4_MASK                                                           0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG17__RANGE_BPG_OFFSET4_MASK                                                       0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG18
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MIN_QP5__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MAX_QP5__SHIFT                                                         0x5
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5__SHIFT                                                     0xa
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MIN_QP6__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MAX_QP6__SHIFT                                                         0x15
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6__SHIFT                                                     0x1a
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MIN_QP5_MASK                                                           0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MAX_QP5_MASK                                                           0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET5_MASK                                                       0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MIN_QP6_MASK                                                           0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_MAX_QP6_MASK                                                           0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG18__RANGE_BPG_OFFSET6_MASK                                                       0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG19
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MIN_QP7__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MAX_QP7__SHIFT                                                         0x5
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7__SHIFT                                                     0xa
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MIN_QP8__SHIFT                                                         0x10
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MAX_QP8__SHIFT                                                         0x15
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8__SHIFT                                                     0x1a
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MIN_QP7_MASK                                                           0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MAX_QP7_MASK                                                           0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET7_MASK                                                       0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MIN_QP8_MASK                                                           0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_MAX_QP8_MASK                                                           0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG19__RANGE_BPG_OFFSET8_MASK                                                       0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG20
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MIN_QP9__SHIFT                                                         0x0
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MAX_QP9__SHIFT                                                         0x5
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9__SHIFT                                                     0xa
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MIN_QP10__SHIFT                                                        0x10
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MAX_QP10__SHIFT                                                        0x15
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10__SHIFT                                                    0x1a
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MIN_QP9_MASK                                                           0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MAX_QP9_MASK                                                           0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET9_MASK                                                       0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MIN_QP10_MASK                                                          0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_MAX_QP10_MASK                                                          0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG20__RANGE_BPG_OFFSET10_MASK                                                      0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG21
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MIN_QP11__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MAX_QP11__SHIFT                                                        0x5
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11__SHIFT                                                    0xa
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MIN_QP12__SHIFT                                                        0x10
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MAX_QP12__SHIFT                                                        0x15
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12__SHIFT                                                    0x1a
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MIN_QP11_MASK                                                          0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MAX_QP11_MASK                                                          0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET11_MASK                                                      0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MIN_QP12_MASK                                                          0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_MAX_QP12_MASK                                                          0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG21__RANGE_BPG_OFFSET12_MASK                                                      0xFC000000L
+//DSCC3_DSCC_PPS_CONFIG22
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MIN_QP13__SHIFT                                                        0x0
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MAX_QP13__SHIFT                                                        0x5
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13__SHIFT                                                    0xa
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MIN_QP14__SHIFT                                                        0x10
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MAX_QP14__SHIFT                                                        0x15
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14__SHIFT                                                    0x1a
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MIN_QP13_MASK                                                          0x0000001FL
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MAX_QP13_MASK                                                          0x000003E0L
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET13_MASK                                                      0x0000FC00L
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MIN_QP14_MASK                                                          0x001F0000L
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_MAX_QP14_MASK                                                          0x03E00000L
+#define DSCC3_DSCC_PPS_CONFIG22__RANGE_BPG_OFFSET14_MASK                                                      0xFC000000L
+//DSCC3_DSCC_MEM_POWER_CONTROL0
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC3_DSCC_MEM_POWER_CONTROL0__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC3_DSCC_MEM_POWER_CONTROL1
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                0x0
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE__SHIFT                                              0x4
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS__SHIFT                                                0x8
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE__SHIFT                                              0x10
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_DEFAULT_MEM_LOW_POWER_STATE_MASK                                  0x00000003L
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_FORCE_MASK                                                0x00000030L
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_DIS_MASK                                                  0x00000100L
+#define DSCC3_DSCC_MEM_POWER_CONTROL1__DSCC_MEM_PWR_STATE_MASK                                                0x00030000L
+//DSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER
+#define DSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER__SHIFT                               0x0
+#define DSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER__DSCC_R_Y_SQUARED_ERROR_LOWER_MASK                                 0xFFFFFFFFL
+//DSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER
+#define DSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER__SHIFT                               0x0
+#define DSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER__DSCC_R_Y_SQUARED_ERROR_UPPER_MASK                                 0xFFFFFFFFL
+//DSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER
+#define DSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER__DSCC_G_CB_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER
+#define DSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER__DSCC_G_CB_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER
+#define DSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER__SHIFT                             0x0
+#define DSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER__DSCC_B_CR_SQUARED_ERROR_LOWER_MASK                               0xFFFFFFFFL
+//DSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER
+#define DSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER__SHIFT                             0x0
+#define DSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER__DSCC_B_CR_SQUARED_ERROR_UPPER_MASK                               0xFFFFFFFFL
+//DSCC3_DSCC_MAX_ABS_ERROR0
+#define DSCC3_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR__SHIFT                                              0x0
+#define DSCC3_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR__SHIFT                                             0x10
+#define DSCC3_DSCC_MAX_ABS_ERROR0__DSCC_R_Y_MAX_ABS_ERROR_MASK                                                0x0000FFFFL
+#define DSCC3_DSCC_MAX_ABS_ERROR0__DSCC_G_CB_MAX_ABS_ERROR_MASK                                               0xFFFF0000L
+//DSCC3_DSCC_MAX_ABS_ERROR1
+#define DSCC3_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR__SHIFT                                             0x0
+#define DSCC3_DSCC_MAX_ABS_ERROR1__DSCC_B_CR_MAX_ABS_ERROR_MASK                                               0x0000FFFFL
+//DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__SHIFT           0x0
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL0_MASK             0x00007FFFL
+//DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__SHIFT           0x0
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL1_MASK             0x00007FFFL
+//DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__SHIFT           0x0
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL2_MASK             0x00007FFFL
+//DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__SHIFT           0x0
+#define DSCC3_DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3__DSCC_OUTPUT_BUFFER_MAX_FULLNESS_LEVEL3_MASK             0x00007FFFL
+//DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__SHIFT   0x0
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL0_MASK     0x0003FFFFL
+//DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__SHIFT   0x0
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL1_MASK     0x0003FFFFL
+//DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__SHIFT   0x0
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL2_MASK     0x0003FFFFL
+//DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__SHIFT   0x0
+#define DSCC3_DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3__DSCC_RATE_BUFFER_MODEL_MAX_FULLNESS_LEVEL3_MASK     0x0003FFFFL
+//DSCC3_DSCC_TEST_DEBUG_INDEX0
+#define DSCC3_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0__SHIFT                                           0x0
+#define DSCC3_DSCC_TEST_DEBUG_INDEX0__DSCC_TEST_DEBUG_INDEX0_MASK                                             0x000000FFL
+//DSCC3_DSCC_TEST_DEBUG_INDEX1
+#define DSCC3_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1__SHIFT                                           0x0
+#define DSCC3_DSCC_TEST_DEBUG_INDEX1__DSCC_TEST_DEBUG_INDEX1_MASK                                             0x000000FFL
+//DSCC3_DSCC_TEST_DEBUG_INDEX2
+#define DSCC3_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2__SHIFT                                           0x0
+#define DSCC3_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2_MASK                                             0x000000FFL
+//DSCC3_DSCC_TEST_DEBUG_INDEX3
+#define DSCC3_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3__SHIFT                                           0x0
+#define DSCC3_DSCC_TEST_DEBUG_INDEX3__DSCC_TEST_DEBUG_INDEX3_MASK                                             0x000000FFL
+//DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC3_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
+//DSCC3_DSCC_TEST_DEBUG_DATA0
+#define DSCC3_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0__SHIFT                                             0x0
+#define DSCC3_DSCC_TEST_DEBUG_DATA0__DSCC_TEST_DEBUG_DATA0_MASK                                               0xFFFFFFFFL
+//DSCC3_DSCC_TEST_DEBUG_DATA1
+#define DSCC3_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1__SHIFT                                             0x0
+#define DSCC3_DSCC_TEST_DEBUG_DATA1__DSCC_TEST_DEBUG_DATA1_MASK                                               0xFFFFFFFFL
+//DSCC3_DSCC_TEST_DEBUG_DATA2
+#define DSCC3_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2__SHIFT                                             0x0
+#define DSCC3_DSCC_TEST_DEBUG_DATA2__DSCC_TEST_DEBUG_DATA2_MASK                                               0xFFFFFFFFL
+//DSCC3_DSCC_TEST_DEBUG_DATA3
+#define DSCC3_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3__SHIFT                                             0x0
+#define DSCC3_DSCC_TEST_DEBUG_DATA3__DSCC_TEST_DEBUG_DATA3_MASK                                               0xFFFFFFFFL
+//DSCC3_DSCC_DISPCLK_TEST_DEBUG_INDEX0
+#define DSCC3_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0__SHIFT                           0x0
+#define DSCC3_DSCC_DISPCLK_TEST_DEBUG_INDEX0__DSCC_DISPCLK_TEST_DEBUG_INDEX0_MASK                             0x000000FFL
+//DSCC3_DSCC_DISPCLK_TEST_DEBUG_DATA0
+#define DSCC3_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0__SHIFT                             0x0
+#define DSCC3_DSCC_DISPCLK_TEST_DEBUG_DATA0__DSCC_DISPCLK_TEST_DEBUG_DATA0_MASK                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON20_PERFCOUNTER_CNTL
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON20_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON20_PERFCOUNTER_CNTL2
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON20_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON20_PERFCOUNTER_STATE
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON20_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON20_PERFMON_CNTL
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON20_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON20_PERFMON_CNTL2
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON20_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON20_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON20_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON20_PERFMON_CVALUE_LOW
+#define DC_PERFMON20_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON20_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON20_PERFMON_HI
+#define DC_PERFMON20_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON20_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON20_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON20_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON20_PERFMON_LOW
+#define DC_PERFMON20_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON20_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_wb0_dispdec_dwb_top_dispdec
+//DWB_ENABLE_CLK_CTRL
+#define DWB_ENABLE_CLK_CTRL__DWB_ENABLE__SHIFT                                                                0x0
+#define DWB_ENABLE_CLK_CTRL__DISPCLK_R_DWB_GATE_DIS__SHIFT                                                    0x4
+#define DWB_ENABLE_CLK_CTRL__DISPCLK_G_DWB_GATE_DIS__SHIFT                                                    0x8
+#define DWB_ENABLE_CLK_CTRL__DWB_TEST_CLK_SEL__SHIFT                                                          0xc
+#define DWB_ENABLE_CLK_CTRL__DWB_FGCG_REP_DIS__SHIFT                                                          0x18
+#define DWB_ENABLE_CLK_CTRL__DWB_ENABLE_MASK                                                                  0x00000001L
+#define DWB_ENABLE_CLK_CTRL__DISPCLK_R_DWB_GATE_DIS_MASK                                                      0x00000010L
+#define DWB_ENABLE_CLK_CTRL__DISPCLK_G_DWB_GATE_DIS_MASK                                                      0x00000100L
+#define DWB_ENABLE_CLK_CTRL__DWB_TEST_CLK_SEL_MASK                                                            0x00003000L
+#define DWB_ENABLE_CLK_CTRL__DWB_FGCG_REP_DIS_MASK                                                            0x01000000L
+//DWB_MEM_PWR_CTRL
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_FORCE__SHIFT                                                   0x8
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_DIS__SHIFT                                                     0xa
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_STATE__SHIFT                                                   0xc
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_FORCE__SHIFT                                                   0x10
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_DIS__SHIFT                                                     0x12
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_STATE__SHIFT                                                   0x14
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_FORCE_MASK                                                     0x00000300L
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_DIS_MASK                                                       0x00000400L
+#define DWB_MEM_PWR_CTRL__DWB_OUT_FIFO_MEM_PWR_STATE_MASK                                                     0x00003000L
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_FORCE_MASK                                                     0x00030000L
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_DIS_MASK                                                       0x00040000L
+#define DWB_MEM_PWR_CTRL__DWB_OGAM_LUT_MEM_PWR_STATE_MASK                                                     0x00300000L
+//FC_MODE_CTRL
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_EN__SHIFT                                                              0x0
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_RATE__SHIFT                                                            0x4
+#define FC_MODE_CTRL__FC_WINDOW_CROP_EN__SHIFT                                                                0x8
+#define FC_MODE_CTRL__FC_EYE_SELECTION__SHIFT                                                                 0xc
+#define FC_MODE_CTRL__FC_STEREO_EYE_POLARITY__SHIFT                                                           0x10
+#define FC_MODE_CTRL__FC_NEW_CONTENT__SHIFT                                                                   0x14
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_EN_CURRENT__SHIFT                                                      0x1f
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_EN_MASK                                                                0x00000001L
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_RATE_MASK                                                              0x00000030L
+#define FC_MODE_CTRL__FC_WINDOW_CROP_EN_MASK                                                                  0x00000100L
+#define FC_MODE_CTRL__FC_EYE_SELECTION_MASK                                                                   0x00003000L
+#define FC_MODE_CTRL__FC_STEREO_EYE_POLARITY_MASK                                                             0x00010000L
+#define FC_MODE_CTRL__FC_NEW_CONTENT_MASK                                                                     0x00100000L
+#define FC_MODE_CTRL__FC_FRAME_CAPTURE_EN_CURRENT_MASK                                                        0x80000000L
+//FC_FLOW_CTRL
+#define FC_FLOW_CTRL__FC_FIRST_PIXEL_DELAY_COUNT__SHIFT                                                       0x0
+#define FC_FLOW_CTRL__FC_FIRST_PIXEL_DELAY_COUNT_MASK                                                         0x00000FFFL
+//FC_WINDOW_START
+#define FC_WINDOW_START__FC_WINDOW_START_X__SHIFT                                                             0x0
+#define FC_WINDOW_START__FC_WINDOW_START_Y__SHIFT                                                             0x10
+#define FC_WINDOW_START__FC_WINDOW_START_X_MASK                                                               0x00001FFFL
+#define FC_WINDOW_START__FC_WINDOW_START_Y_MASK                                                               0x1FFF0000L
+//FC_WINDOW_SIZE
+#define FC_WINDOW_SIZE__FC_WINDOW_WIDTH__SHIFT                                                                0x0
+#define FC_WINDOW_SIZE__FC_WINDOW_HEIGHT__SHIFT                                                               0x10
+#define FC_WINDOW_SIZE__FC_WINDOW_WIDTH_MASK                                                                  0x00000FFFL
+#define FC_WINDOW_SIZE__FC_WINDOW_HEIGHT_MASK                                                                 0x0FFF0000L
+//FC_SOURCE_SIZE
+#define FC_SOURCE_SIZE__FC_SOURCE_WIDTH__SHIFT                                                                0x0
+#define FC_SOURCE_SIZE__FC_SOURCE_HEIGHT__SHIFT                                                               0x10
+#define FC_SOURCE_SIZE__FC_SOURCE_WIDTH_MASK                                                                  0x00007FFFL
+#define FC_SOURCE_SIZE__FC_SOURCE_HEIGHT_MASK                                                                 0x7FFF0000L
+//DWB_UPDATE_CTRL
+#define DWB_UPDATE_CTRL__DWB_UPDATE_LOCK__SHIFT                                                               0x0
+#define DWB_UPDATE_CTRL__DWB_UPDATE_PENDING__SHIFT                                                            0x4
+#define DWB_UPDATE_CTRL__DWB_UPDATE_LOCK_MASK                                                                 0x00000001L
+#define DWB_UPDATE_CTRL__DWB_UPDATE_PENDING_MASK                                                              0x00000010L
+//DWB_CRC_CTRL
+#define DWB_CRC_CTRL__DWB_CRC_EN__SHIFT                                                                       0x0
+#define DWB_CRC_CTRL__DWB_CRC_CONT_EN__SHIFT                                                                  0x4
+#define DWB_CRC_CTRL__DWB_CRC_SRC_SEL__SHIFT                                                                  0x8
+#define DWB_CRC_CTRL__DWB_CRC_EN_MASK                                                                         0x00000001L
+#define DWB_CRC_CTRL__DWB_CRC_CONT_EN_MASK                                                                    0x00000010L
+#define DWB_CRC_CTRL__DWB_CRC_SRC_SEL_MASK                                                                    0x00000300L
+//DWB_CRC_MASK_R_G
+#define DWB_CRC_MASK_R_G__DWB_CRC_RED_MASK__SHIFT                                                             0x0
+#define DWB_CRC_MASK_R_G__DWB_CRC_GREEN_MASK__SHIFT                                                           0x10
+#define DWB_CRC_MASK_R_G__DWB_CRC_RED_MASK_MASK                                                               0x0000FFFFL
+#define DWB_CRC_MASK_R_G__DWB_CRC_GREEN_MASK_MASK                                                             0xFFFF0000L
+//DWB_CRC_MASK_B_A
+#define DWB_CRC_MASK_B_A__DWB_CRC_BLUE_MASK__SHIFT                                                            0x0
+#define DWB_CRC_MASK_B_A__DWB_CRC_A_MASK__SHIFT                                                               0x10
+#define DWB_CRC_MASK_B_A__DWB_CRC_BLUE_MASK_MASK                                                              0x0000FFFFL
+#define DWB_CRC_MASK_B_A__DWB_CRC_A_MASK_MASK                                                                 0xFFFF0000L
+//DWB_CRC_VAL_R_G
+#define DWB_CRC_VAL_R_G__DWB_CRC_SIG_RED__SHIFT                                                               0x0
+#define DWB_CRC_VAL_R_G__DWB_CRC_SIG_GREEN__SHIFT                                                             0x10
+#define DWB_CRC_VAL_R_G__DWB_CRC_SIG_RED_MASK                                                                 0x0000FFFFL
+#define DWB_CRC_VAL_R_G__DWB_CRC_SIG_GREEN_MASK                                                               0xFFFF0000L
+//DWB_CRC_VAL_B_A
+#define DWB_CRC_VAL_B_A__DWB_CRC_SIG_BLUE__SHIFT                                                              0x0
+#define DWB_CRC_VAL_B_A__DWB_CRC_SIG_A__SHIFT                                                                 0x10
+#define DWB_CRC_VAL_B_A__DWB_CRC_SIG_BLUE_MASK                                                                0x0000FFFFL
+#define DWB_CRC_VAL_B_A__DWB_CRC_SIG_A_MASK                                                                   0xFFFF0000L
+//DWB_OUT_CTRL
+#define DWB_OUT_CTRL__OUT_FORMAT__SHIFT                                                                       0x0
+#define DWB_OUT_CTRL__OUT_DENORM__SHIFT                                                                       0x4
+#define DWB_OUT_CTRL__OUT_MAX__SHIFT                                                                          0x8
+#define DWB_OUT_CTRL__OUT_MIN__SHIFT                                                                          0x14
+#define DWB_OUT_CTRL__OUT_FORMAT_MASK                                                                         0x00000003L
+#define DWB_OUT_CTRL__OUT_DENORM_MASK                                                                         0x00000030L
+#define DWB_OUT_CTRL__OUT_MAX_MASK                                                                            0x0003FF00L
+#define DWB_OUT_CTRL__OUT_MIN_MASK                                                                            0x3FF00000L
+//DWB_MMHUBBUB_BACKPRESSURE_CNT_EN
+#define DWB_MMHUBBUB_BACKPRESSURE_CNT_EN__DWB_MMHUBBUB_BACKPRESSURE_CNT_EN__SHIFT                             0x0
+#define DWB_MMHUBBUB_BACKPRESSURE_CNT_EN__DWB_MMHUBBUB_BACKPRESSURE_CNT_EN_MASK                               0x00000001L
+//DWB_MMHUBBUB_BACKPRESSURE_CNT
+#define DWB_MMHUBBUB_BACKPRESSURE_CNT__DWB_MMHUBBUB_MAX_BACKPRESSURE__SHIFT                                   0x0
+#define DWB_MMHUBBUB_BACKPRESSURE_CNT__DWB_MMHUBBUB_MAX_BACKPRESSURE_MASK                                     0x0000FFFFL
+//DWB_HOST_READ_CONTROL
+#define DWB_HOST_READ_CONTROL__DWB_HOST_READ_RATE_CONTROL__SHIFT                                              0x0
+#define DWB_HOST_READ_CONTROL__DWB_HOST_READ_RATE_CONTROL_MASK                                                0x000000FFL
+//DWB_OVERFLOW_STATUS
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_FLAG__SHIFT                                                    0x0
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_ACK__SHIFT                                                     0x8
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_MASK__SHIFT                                                    0xc
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_INT_STATUS__SHIFT                                              0x10
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_INT_TYPE__SHIFT                                                0x14
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_FLAG_MASK                                                      0x00000001L
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_ACK_MASK                                                       0x00000100L
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_MASK_MASK                                                      0x00001000L
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_INT_STATUS_MASK                                                0x00010000L
+#define DWB_OVERFLOW_STATUS__DWB_DATA_OVERFLOW_INT_TYPE_MASK                                                  0x00100000L
+//DWB_OVERFLOW_COUNTER
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_TYPE__SHIFT                                                   0x0
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_OUT_X_CNT__SHIFT                                              0x4
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_OUT_Y_CNT__SHIFT                                              0x10
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_TYPE_MASK                                                     0x00000003L
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_OUT_X_CNT_MASK                                                0x0000FFF0L
+#define DWB_OVERFLOW_COUNTER__DWB_DATA_OVERFLOW_OUT_Y_CNT_MASK                                                0x0FFF0000L
+//DWB_SOFT_RESET
+#define DWB_SOFT_RESET__DWB_SOFT_RESET__SHIFT                                                                 0x0
+#define DWB_SOFT_RESET__DWB_SOFT_RESET_MASK                                                                   0x00000001L
+//DWB_DEBUG_CTRL
+#define DWB_DEBUG_CTRL__DWB_DEBUG_EN__SHIFT                                                                   0x0
+#define DWB_DEBUG_CTRL__DWB_DEBUG_SEL__SHIFT                                                                  0x6
+#define DWB_DEBUG_CTRL__DWB_DEBUG_EN_MASK                                                                     0x00000001L
+#define DWB_DEBUG_CTRL__DWB_DEBUG_SEL_MASK                                                                    0x000000C0L
+//DWB_DEBUG
+#define DWB_DEBUG__DWB_DEBUG__SHIFT                                                                           0x0
+#define DWB_DEBUG__DWB_DEBUG_MASK                                                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_wb0_dispdec_wb_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON21_PERFCOUNTER_CNTL
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON21_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON21_PERFCOUNTER_CNTL2
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON21_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON21_PERFCOUNTER_STATE
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON21_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON21_PERFMON_CNTL
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON21_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON21_PERFMON_CNTL2
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON21_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON21_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON21_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON21_PERFMON_CVALUE_LOW
+#define DC_PERFMON21_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON21_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON21_PERFMON_HI
+#define DC_PERFMON21_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON21_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON21_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON21_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON21_PERFMON_LOW
+#define DC_PERFMON21_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON21_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_wb0_dispdec_dwbcp_dispdec
+//DWB_HDR_MULT_COEF
+#define DWB_HDR_MULT_COEF__DWB_HDR_MULT_COEF__SHIFT                                                           0x0
+#define DWB_HDR_MULT_COEF__DWB_HDR_MULT_COEF_MASK                                                             0x0007FFFFL
+//DWB_GAMUT_REMAP_MODE
+#define DWB_GAMUT_REMAP_MODE__DWB_GAMUT_REMAP_MODE__SHIFT                                                     0x0
+#define DWB_GAMUT_REMAP_MODE__DWB_GAMUT_REMAP_MODE_CURRENT__SHIFT                                             0x18
+#define DWB_GAMUT_REMAP_MODE__DWB_GAMUT_REMAP_MODE_MASK                                                       0x00000003L
+#define DWB_GAMUT_REMAP_MODE__DWB_GAMUT_REMAP_MODE_CURRENT_MASK                                               0x03000000L
+//DWB_GAMUT_REMAP_COEF_FORMAT
+#define DWB_GAMUT_REMAP_COEF_FORMAT__DWB_GAMUT_REMAP_COEF_FORMAT__SHIFT                                       0x0
+#define DWB_GAMUT_REMAP_COEF_FORMAT__DWB_GAMUT_REMAP_COEF_FORMAT_MASK                                         0x00000001L
+//DWB_GAMUT_REMAPA_C11_C12
+#define DWB_GAMUT_REMAPA_C11_C12__DWB_GAMUT_REMAPA_C11__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C11_C12__DWB_GAMUT_REMAPA_C12__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C11_C12__DWB_GAMUT_REMAPA_C11_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C11_C12__DWB_GAMUT_REMAPA_C12_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPA_C13_C14
+#define DWB_GAMUT_REMAPA_C13_C14__DWB_GAMUT_REMAPA_C13__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C13_C14__DWB_GAMUT_REMAPA_C14__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C13_C14__DWB_GAMUT_REMAPA_C13_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C13_C14__DWB_GAMUT_REMAPA_C14_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPA_C21_C22
+#define DWB_GAMUT_REMAPA_C21_C22__DWB_GAMUT_REMAPA_C21__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C21_C22__DWB_GAMUT_REMAPA_C22__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C21_C22__DWB_GAMUT_REMAPA_C21_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C21_C22__DWB_GAMUT_REMAPA_C22_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPA_C23_C24
+#define DWB_GAMUT_REMAPA_C23_C24__DWB_GAMUT_REMAPA_C23__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C23_C24__DWB_GAMUT_REMAPA_C24__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C23_C24__DWB_GAMUT_REMAPA_C23_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C23_C24__DWB_GAMUT_REMAPA_C24_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPA_C31_C32
+#define DWB_GAMUT_REMAPA_C31_C32__DWB_GAMUT_REMAPA_C31__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C31_C32__DWB_GAMUT_REMAPA_C32__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C31_C32__DWB_GAMUT_REMAPA_C31_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C31_C32__DWB_GAMUT_REMAPA_C32_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPA_C33_C34
+#define DWB_GAMUT_REMAPA_C33_C34__DWB_GAMUT_REMAPA_C33__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPA_C33_C34__DWB_GAMUT_REMAPA_C34__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPA_C33_C34__DWB_GAMUT_REMAPA_C33_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPA_C33_C34__DWB_GAMUT_REMAPA_C34_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C11_C12
+#define DWB_GAMUT_REMAPB_C11_C12__DWB_GAMUT_REMAPB_C11__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C11_C12__DWB_GAMUT_REMAPB_C12__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C11_C12__DWB_GAMUT_REMAPB_C11_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C11_C12__DWB_GAMUT_REMAPB_C12_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C13_C14
+#define DWB_GAMUT_REMAPB_C13_C14__DWB_GAMUT_REMAPB_C13__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C13_C14__DWB_GAMUT_REMAPB_C14__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C13_C14__DWB_GAMUT_REMAPB_C13_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C13_C14__DWB_GAMUT_REMAPB_C14_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C21_C22
+#define DWB_GAMUT_REMAPB_C21_C22__DWB_GAMUT_REMAPB_C21__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C21_C22__DWB_GAMUT_REMAPB_C22__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C21_C22__DWB_GAMUT_REMAPB_C21_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C21_C22__DWB_GAMUT_REMAPB_C22_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C23_C24
+#define DWB_GAMUT_REMAPB_C23_C24__DWB_GAMUT_REMAPB_C23__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C23_C24__DWB_GAMUT_REMAPB_C24__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C23_C24__DWB_GAMUT_REMAPB_C23_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C23_C24__DWB_GAMUT_REMAPB_C24_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C31_C32
+#define DWB_GAMUT_REMAPB_C31_C32__DWB_GAMUT_REMAPB_C31__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C31_C32__DWB_GAMUT_REMAPB_C32__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C31_C32__DWB_GAMUT_REMAPB_C31_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C31_C32__DWB_GAMUT_REMAPB_C32_MASK                                                   0xFFFF0000L
+//DWB_GAMUT_REMAPB_C33_C34
+#define DWB_GAMUT_REMAPB_C33_C34__DWB_GAMUT_REMAPB_C33__SHIFT                                                 0x0
+#define DWB_GAMUT_REMAPB_C33_C34__DWB_GAMUT_REMAPB_C34__SHIFT                                                 0x10
+#define DWB_GAMUT_REMAPB_C33_C34__DWB_GAMUT_REMAPB_C33_MASK                                                   0x0000FFFFL
+#define DWB_GAMUT_REMAPB_C33_C34__DWB_GAMUT_REMAPB_C34_MASK                                                   0xFFFF0000L
+//DWB_OGAM_CONTROL
+#define DWB_OGAM_CONTROL__DWB_OGAM_MODE__SHIFT                                                                0x0
+#define DWB_OGAM_CONTROL__DWB_OGAM_SELECT__SHIFT                                                              0x4
+#define DWB_OGAM_CONTROL__DWB_OGAM_PWL_DISABLE__SHIFT                                                         0x8
+#define DWB_OGAM_CONTROL__DWB_OGAM_MODE_CURRENT__SHIFT                                                        0x18
+#define DWB_OGAM_CONTROL__DWB_OGAM_SELECT_CURRENT__SHIFT                                                      0x1c
+#define DWB_OGAM_CONTROL__DWB_OGAM_MODE_MASK                                                                  0x00000003L
+#define DWB_OGAM_CONTROL__DWB_OGAM_SELECT_MASK                                                                0x00000010L
+#define DWB_OGAM_CONTROL__DWB_OGAM_PWL_DISABLE_MASK                                                           0x00000100L
+#define DWB_OGAM_CONTROL__DWB_OGAM_MODE_CURRENT_MASK                                                          0x03000000L
+#define DWB_OGAM_CONTROL__DWB_OGAM_SELECT_CURRENT_MASK                                                        0x10000000L
+//DWB_OGAM_LUT_INDEX
+#define DWB_OGAM_LUT_INDEX__DWB_OGAM_LUT_INDEX__SHIFT                                                         0x0
+#define DWB_OGAM_LUT_INDEX__DWB_OGAM_LUT_INDEX_MASK                                                           0x000001FFL
+//DWB_OGAM_LUT_DATA
+#define DWB_OGAM_LUT_DATA__DWB_OGAM_LUT_DATA__SHIFT                                                           0x0
+#define DWB_OGAM_LUT_DATA__DWB_OGAM_LUT_DATA_MASK                                                             0x0003FFFFL
+//DWB_OGAM_LUT_CONTROL
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_WRITE_COLOR_MASK__SHIFT                                            0x0
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_READ_COLOR_SEL__SHIFT                                              0x4
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_READ_DBG__SHIFT                                                    0x8
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_HOST_SEL__SHIFT                                                    0xc
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_CONFIG_MODE__SHIFT                                                 0x10
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_WRITE_COLOR_MASK_MASK                                              0x00000007L
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_READ_COLOR_SEL_MASK                                                0x00000030L
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_READ_DBG_MASK                                                      0x00000100L
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_HOST_SEL_MASK                                                      0x00001000L
+#define DWB_OGAM_LUT_CONTROL__DWB_OGAM_LUT_CONFIG_MODE_MASK                                                   0x00010000L
+//DWB_OGAM_RAMA_START_CNTL_B
+#define DWB_OGAM_RAMA_START_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_B__SHIFT                                   0x0
+#define DWB_OGAM_RAMA_START_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT                           0x14
+#define DWB_OGAM_RAMA_START_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_B_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMA_START_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_B_MASK                             0x07F00000L
+//DWB_OGAM_RAMA_START_CNTL_G
+#define DWB_OGAM_RAMA_START_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_G__SHIFT                                   0x0
+#define DWB_OGAM_RAMA_START_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT                           0x14
+#define DWB_OGAM_RAMA_START_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_G_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMA_START_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_G_MASK                             0x07F00000L
+//DWB_OGAM_RAMA_START_CNTL_R
+#define DWB_OGAM_RAMA_START_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_R__SHIFT                                   0x0
+#define DWB_OGAM_RAMA_START_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT                           0x14
+#define DWB_OGAM_RAMA_START_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_R_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMA_START_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_SEGMENT_R_MASK                             0x07F00000L
+//DWB_OGAM_RAMA_START_BASE_CNTL_B
+#define DWB_OGAM_RAMA_START_BASE_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_BASE_B__SHIFT                         0x0
+#define DWB_OGAM_RAMA_START_BASE_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_BASE_B_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMA_START_SLOPE_CNTL_B
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_B__SHIFT                       0x0
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_B__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_B_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMA_START_BASE_CNTL_G
+#define DWB_OGAM_RAMA_START_BASE_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_BASE_G__SHIFT                         0x0
+#define DWB_OGAM_RAMA_START_BASE_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_BASE_G_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMA_START_SLOPE_CNTL_G
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_G__SHIFT                       0x0
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_G__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_G_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMA_START_BASE_CNTL_R
+#define DWB_OGAM_RAMA_START_BASE_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_BASE_R__SHIFT                         0x0
+#define DWB_OGAM_RAMA_START_BASE_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_BASE_R_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMA_START_SLOPE_CNTL_R
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_R__SHIFT                       0x0
+#define DWB_OGAM_RAMA_START_SLOPE_CNTL_R__DWB_OGAM_RAMA_EXP_REGION_START_SLOPE_R_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMA_END_CNTL1_B
+#define DWB_OGAM_RAMA_END_CNTL1_B__DWB_OGAM_RAMA_EXP_REGION_END_BASE_B__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_END_CNTL1_B__DWB_OGAM_RAMA_EXP_REGION_END_BASE_B_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMA_END_CNTL2_B
+#define DWB_OGAM_RAMA_END_CNTL2_B__DWB_OGAM_RAMA_EXP_REGION_END_B__SHIFT                                      0x0
+#define DWB_OGAM_RAMA_END_CNTL2_B__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                                0x10
+#define DWB_OGAM_RAMA_END_CNTL2_B__DWB_OGAM_RAMA_EXP_REGION_END_B_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMA_END_CNTL2_B__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_B_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMA_END_CNTL1_G
+#define DWB_OGAM_RAMA_END_CNTL1_G__DWB_OGAM_RAMA_EXP_REGION_END_BASE_G__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_END_CNTL1_G__DWB_OGAM_RAMA_EXP_REGION_END_BASE_G_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMA_END_CNTL2_G
+#define DWB_OGAM_RAMA_END_CNTL2_G__DWB_OGAM_RAMA_EXP_REGION_END_G__SHIFT                                      0x0
+#define DWB_OGAM_RAMA_END_CNTL2_G__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                                0x10
+#define DWB_OGAM_RAMA_END_CNTL2_G__DWB_OGAM_RAMA_EXP_REGION_END_G_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMA_END_CNTL2_G__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_G_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMA_END_CNTL1_R
+#define DWB_OGAM_RAMA_END_CNTL1_R__DWB_OGAM_RAMA_EXP_REGION_END_BASE_R__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_END_CNTL1_R__DWB_OGAM_RAMA_EXP_REGION_END_BASE_R_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMA_END_CNTL2_R
+#define DWB_OGAM_RAMA_END_CNTL2_R__DWB_OGAM_RAMA_EXP_REGION_END_R__SHIFT                                      0x0
+#define DWB_OGAM_RAMA_END_CNTL2_R__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                                0x10
+#define DWB_OGAM_RAMA_END_CNTL2_R__DWB_OGAM_RAMA_EXP_REGION_END_R_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMA_END_CNTL2_R__DWB_OGAM_RAMA_EXP_REGION_END_SLOPE_R_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMA_OFFSET_B
+#define DWB_OGAM_RAMA_OFFSET_B__DWB_OGAM_RAMA_OFFSET_B__SHIFT                                                 0x0
+#define DWB_OGAM_RAMA_OFFSET_B__DWB_OGAM_RAMA_OFFSET_B_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMA_OFFSET_G
+#define DWB_OGAM_RAMA_OFFSET_G__DWB_OGAM_RAMA_OFFSET_G__SHIFT                                                 0x0
+#define DWB_OGAM_RAMA_OFFSET_G__DWB_OGAM_RAMA_OFFSET_G_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMA_OFFSET_R
+#define DWB_OGAM_RAMA_OFFSET_R__DWB_OGAM_RAMA_OFFSET_R__SHIFT                                                 0x0
+#define DWB_OGAM_RAMA_OFFSET_R__DWB_OGAM_RAMA_OFFSET_R_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMA_REGION_0_1
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION0_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION1_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_0_1__DWB_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMA_REGION_2_3
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION2_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION3_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_2_3__DWB_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMA_REGION_4_5
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION4_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION5_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_4_5__DWB_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMA_REGION_6_7
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION6_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION7_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_6_7__DWB_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMA_REGION_8_9
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION8_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION9_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_8_9__DWB_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMA_REGION_10_11
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION10_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION11_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_10_11__DWB_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_12_13
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION12_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION13_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_12_13__DWB_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_14_15
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION14_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION15_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_14_15__DWB_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_16_17
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION16_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION17_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_16_17__DWB_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_18_19
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION18_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION19_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_18_19__DWB_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_20_21
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION20_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION21_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_20_21__DWB_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_22_23
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION22_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION23_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_22_23__DWB_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_24_25
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION24_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION25_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_24_25__DWB_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_26_27
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION26_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION27_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_26_27__DWB_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_28_29
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION28_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION29_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_28_29__DWB_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_30_31
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION30_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION31_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_30_31__DWB_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMA_REGION_32_33
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION32_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION33_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMA_REGION_32_33__DWB_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_START_CNTL_B
+#define DWB_OGAM_RAMB_START_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_B__SHIFT                                   0x0
+#define DWB_OGAM_RAMB_START_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT                           0x14
+#define DWB_OGAM_RAMB_START_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_B_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMB_START_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_B_MASK                             0x07F00000L
+//DWB_OGAM_RAMB_START_CNTL_G
+#define DWB_OGAM_RAMB_START_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_G__SHIFT                                   0x0
+#define DWB_OGAM_RAMB_START_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT                           0x14
+#define DWB_OGAM_RAMB_START_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_G_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMB_START_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_G_MASK                             0x07F00000L
+//DWB_OGAM_RAMB_START_CNTL_R
+#define DWB_OGAM_RAMB_START_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_R__SHIFT                                   0x0
+#define DWB_OGAM_RAMB_START_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT                           0x14
+#define DWB_OGAM_RAMB_START_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_R_MASK                                     0x0003FFFFL
+#define DWB_OGAM_RAMB_START_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_SEGMENT_R_MASK                             0x07F00000L
+//DWB_OGAM_RAMB_START_BASE_CNTL_B
+#define DWB_OGAM_RAMB_START_BASE_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_BASE_B__SHIFT                         0x0
+#define DWB_OGAM_RAMB_START_BASE_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_BASE_B_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMB_START_SLOPE_CNTL_B
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_B__SHIFT                       0x0
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_B__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_B_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMB_START_BASE_CNTL_G
+#define DWB_OGAM_RAMB_START_BASE_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_BASE_G__SHIFT                         0x0
+#define DWB_OGAM_RAMB_START_BASE_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_BASE_G_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMB_START_SLOPE_CNTL_G
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_G__SHIFT                       0x0
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_G__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_G_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMB_START_BASE_CNTL_R
+#define DWB_OGAM_RAMB_START_BASE_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_BASE_R__SHIFT                         0x0
+#define DWB_OGAM_RAMB_START_BASE_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_BASE_R_MASK                           0x0003FFFFL
+//DWB_OGAM_RAMB_START_SLOPE_CNTL_R
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_R__SHIFT                       0x0
+#define DWB_OGAM_RAMB_START_SLOPE_CNTL_R__DWB_OGAM_RAMB_EXP_REGION_START_SLOPE_R_MASK                         0x0003FFFFL
+//DWB_OGAM_RAMB_END_CNTL1_B
+#define DWB_OGAM_RAMB_END_CNTL1_B__DWB_OGAM_RAMB_EXP_REGION_END_BASE_B__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_END_CNTL1_B__DWB_OGAM_RAMB_EXP_REGION_END_BASE_B_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMB_END_CNTL2_B
+#define DWB_OGAM_RAMB_END_CNTL2_B__DWB_OGAM_RAMB_EXP_REGION_END_B__SHIFT                                      0x0
+#define DWB_OGAM_RAMB_END_CNTL2_B__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                                0x10
+#define DWB_OGAM_RAMB_END_CNTL2_B__DWB_OGAM_RAMB_EXP_REGION_END_B_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMB_END_CNTL2_B__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_B_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMB_END_CNTL1_G
+#define DWB_OGAM_RAMB_END_CNTL1_G__DWB_OGAM_RAMB_EXP_REGION_END_BASE_G__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_END_CNTL1_G__DWB_OGAM_RAMB_EXP_REGION_END_BASE_G_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMB_END_CNTL2_G
+#define DWB_OGAM_RAMB_END_CNTL2_G__DWB_OGAM_RAMB_EXP_REGION_END_G__SHIFT                                      0x0
+#define DWB_OGAM_RAMB_END_CNTL2_G__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                                0x10
+#define DWB_OGAM_RAMB_END_CNTL2_G__DWB_OGAM_RAMB_EXP_REGION_END_G_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMB_END_CNTL2_G__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_G_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMB_END_CNTL1_R
+#define DWB_OGAM_RAMB_END_CNTL1_R__DWB_OGAM_RAMB_EXP_REGION_END_BASE_R__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_END_CNTL1_R__DWB_OGAM_RAMB_EXP_REGION_END_BASE_R_MASK                                   0x0003FFFFL
+//DWB_OGAM_RAMB_END_CNTL2_R
+#define DWB_OGAM_RAMB_END_CNTL2_R__DWB_OGAM_RAMB_EXP_REGION_END_R__SHIFT                                      0x0
+#define DWB_OGAM_RAMB_END_CNTL2_R__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                                0x10
+#define DWB_OGAM_RAMB_END_CNTL2_R__DWB_OGAM_RAMB_EXP_REGION_END_R_MASK                                        0x0000FFFFL
+#define DWB_OGAM_RAMB_END_CNTL2_R__DWB_OGAM_RAMB_EXP_REGION_END_SLOPE_R_MASK                                  0xFFFF0000L
+//DWB_OGAM_RAMB_OFFSET_B
+#define DWB_OGAM_RAMB_OFFSET_B__DWB_OGAM_RAMB_OFFSET_B__SHIFT                                                 0x0
+#define DWB_OGAM_RAMB_OFFSET_B__DWB_OGAM_RAMB_OFFSET_B_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMB_OFFSET_G
+#define DWB_OGAM_RAMB_OFFSET_G__DWB_OGAM_RAMB_OFFSET_G__SHIFT                                                 0x0
+#define DWB_OGAM_RAMB_OFFSET_G__DWB_OGAM_RAMB_OFFSET_G_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMB_OFFSET_R
+#define DWB_OGAM_RAMB_OFFSET_R__DWB_OGAM_RAMB_OFFSET_R__SHIFT                                                 0x0
+#define DWB_OGAM_RAMB_OFFSET_R__DWB_OGAM_RAMB_OFFSET_R_MASK                                                   0x0007FFFFL
+//DWB_OGAM_RAMB_REGION_0_1
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION0_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION1_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_0_1__DWB_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMB_REGION_2_3
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION2_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION3_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_2_3__DWB_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMB_REGION_4_5
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION4_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION5_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_4_5__DWB_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMB_REGION_6_7
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION6_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION7_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_6_7__DWB_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMB_REGION_8_9
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                                 0x0
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                               0xc
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                                 0x10
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                               0x1c
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION8_LUT_OFFSET_MASK                                   0x000001FFL
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                                 0x00007000L
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION9_LUT_OFFSET_MASK                                   0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_8_9__DWB_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                                 0x70000000L
+//DWB_OGAM_RAMB_REGION_10_11
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION10_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION11_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_10_11__DWB_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_12_13
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION12_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION13_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_12_13__DWB_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_14_15
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION14_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION15_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_14_15__DWB_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_16_17
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION16_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION17_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_16_17__DWB_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_18_19
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION18_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION19_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_18_19__DWB_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_20_21
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION20_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION21_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_20_21__DWB_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_22_23
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION22_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION23_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_22_23__DWB_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_24_25
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION24_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION25_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_24_25__DWB_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_26_27
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION26_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION27_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_26_27__DWB_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_28_29
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION28_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION29_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_28_29__DWB_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_30_31
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION30_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION31_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_30_31__DWB_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                              0x70000000L
+//DWB_OGAM_RAMB_REGION_32_33
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                              0x0
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT                            0xc
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                              0x10
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT                            0x1c
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION32_LUT_OFFSET_MASK                                0x000001FFL
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                              0x00007000L
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION33_LUT_OFFSET_MASK                                0x01FF0000L
+#define DWB_OGAM_RAMB_REGION_32_33__DWB_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                              0x70000000L
+
+
+// addressBlock: dce_dc_dchvm_hvm_dispdec
+//DCHVM_CTRL0
+#define DCHVM_CTRL0__HOSTVM_INIT_REQ__SHIFT                                                                   0x0
+#define DCHVM_CTRL0__HOSTVM_INIT_REQ_MASK                                                                     0x00000001L
+//DCHVM_CTRL1
+#define DCHVM_CTRL1__DUMMY1__SHIFT                                                                            0x0
+#define DCHVM_CTRL1__DUMMY1_MASK                                                                              0xFFFFFFFFL
+//DCHVM_CLK_CTRL
+#define DCHVM_CLK_CTRL__HVM_DISPCLK_R_GATE_DIS__SHIFT                                                         0x0
+#define DCHVM_CLK_CTRL__HVM_DISPCLK_G_GATE_DIS__SHIFT                                                         0x1
+#define DCHVM_CLK_CTRL__HVM_DCFCLK_R_GATE_DIS__SHIFT                                                          0x4
+#define DCHVM_CLK_CTRL__HVM_DCFCLK_G_GATE_DIS__SHIFT                                                          0x5
+#define DCHVM_CLK_CTRL__TR_REQ_REQCLKREQ_MODE__SHIFT                                                          0x8
+#define DCHVM_CLK_CTRL__TW_RSP_COMPCLKREQ_MODE__SHIFT                                                         0xa
+#define DCHVM_CLK_CTRL__HVM_FGCG_REP_DIS__SHIFT                                                               0xc
+#define DCHVM_CLK_CTRL__TW_RSP_COMPCLKCTL_MODE__SHIFT                                                         0xf
+#define DCHVM_CLK_CTRL__HVM_DISPCLK_R_GATE_DIS_MASK                                                           0x00000001L
+#define DCHVM_CLK_CTRL__HVM_DISPCLK_G_GATE_DIS_MASK                                                           0x00000002L
+#define DCHVM_CLK_CTRL__HVM_DCFCLK_R_GATE_DIS_MASK                                                            0x00000010L
+#define DCHVM_CLK_CTRL__HVM_DCFCLK_G_GATE_DIS_MASK                                                            0x00000020L
+#define DCHVM_CLK_CTRL__TR_REQ_REQCLKREQ_MODE_MASK                                                            0x00000300L
+#define DCHVM_CLK_CTRL__TW_RSP_COMPCLKREQ_MODE_MASK                                                           0x00000C00L
+#define DCHVM_CLK_CTRL__HVM_FGCG_REP_DIS_MASK                                                                 0x00001000L
+#define DCHVM_CLK_CTRL__TW_RSP_COMPCLKCTL_MODE_MASK                                                           0x00008000L
+//DCHVM_MEM_CTRL
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_PWR_REQ_DIS__SHIFT                                                       0x0
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_FORCE_REQ__SHIFT                                                         0x2
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_POWER_STATUS__SHIFT                                                      0x4
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_PWR_REQ_DIS_MASK                                                         0x00000001L
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_FORCE_REQ_MASK                                                           0x0000000CL
+#define DCHVM_MEM_CTRL__HVM_GPUVMRET_POWER_STATUS_MASK                                                        0x00000030L
+//DCHVM_RIOMMU_CTRL0
+#define DCHVM_RIOMMU_CTRL0__HOSTVM_PREFETCH_REQ__SHIFT                                                        0x0
+#define DCHVM_RIOMMU_CTRL0__HOSTVM_POWERSTATUS__SHIFT                                                         0x1
+#define DCHVM_RIOMMU_CTRL0__HOSTVM_PREFETCH_REQ_MASK                                                          0x00000001L
+#define DCHVM_RIOMMU_CTRL0__HOSTVM_POWERSTATUS_MASK                                                           0x00000002L
+//DCHVM_RIOMMU_STAT0
+#define DCHVM_RIOMMU_STAT0__RIOMMU_ACTIVE__SHIFT                                                              0x0
+#define DCHVM_RIOMMU_STAT0__HOSTVM_PREFETCH_DONE__SHIFT                                                       0x1
+#define DCHVM_RIOMMU_STAT0__RIOMMU_ACTIVE_MASK                                                                0x00000001L
+#define DCHVM_RIOMMU_STAT0__HOSTVM_PREFETCH_DONE_MASK                                                         0x00000002L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
+//DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN__SHIFT                             0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                     0x4
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK__SHIFT                  0x9
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK__SHIFT                 0xc
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32__SHIFT                    0x10
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32__SHIFT                0x11
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN_MASK                               0x00000001L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                       0x00000010L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK_MASK                    0x00000200L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK_MASK                   0x00001000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32_MASK                      0x00010000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32_MASK                  0x00020000L
+//DP_STREAM_ENC0_DP_STREAM_ENC_INPUT_MUX_CONTROL
+#define DP_STREAM_ENC0_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL__SHIFT  0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL_MASK  0x00000007L
+//DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL
+#define DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT    0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN__SHIFT                         0x8
+#define DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK      0x00000007L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN_MASK                           0x00000100L
+//DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT             0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT              0x4
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL__SHIFT   0x8
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC__SHIFT     0x10
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT         0x14
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT  0x18
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT              0x1c
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK               0x00000001L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                0x00000010L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL_MASK     0x00003F00L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC_MASK       0x00010000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK           0x00100000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK  0x01000000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                0x30000000L
+//DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT  0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT  0x1
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT  0x2
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT    0x4
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY__SHIFT    0xa
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT      0xc
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT      0x10
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT  0x18
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT         0x1f
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK  0x00000001L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK  0x00000002L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK  0x00000004L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK      0x000003F0L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY_MASK      0x00000C00L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK        0x0000F000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK        0x001F0000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK    0x3F000000L
+#define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK           0x80000000L
+//DP_STREAM_ENC0_DP_STREAM_ENC_SPARE
+#define DP_STREAM_ENC0_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE__SHIFT                                        0x0
+#define DP_STREAM_ENC0_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_apg_apg_dispdec
+//APG5_APG_DBG_GEN_CONTROL
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG5_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG5_APG_MEM_PWR
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG5_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG5_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG5_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_dme_dme_dispdec
+//DME5_DME_CONTROL
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME5_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME5_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME5_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME5_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME5_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME5_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME5_DME_MEMORY_CONTROL
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc0_vpg_vpg_dispdec
+//VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG5_VPG_GENERIC_PACKET_DATA
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG5_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG5_VPG_GENERIC_STATUS
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG5_VPG_MEM_PWR
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG5_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG5_VPG_ISRC1_2_DATA
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc0_dispdec
+//DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE__SHIFT                                        0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET__SHIFT                                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE__SHIFT                                    0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE_MASK                                          0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_MASK                                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE_MASK                                      0x00000100L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE__SHIFT                       0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET__SHIFT                        0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE__SHIFT                   0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS__SHIFT              0xc
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE_MASK                         0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_MASK                          0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE_MASK                     0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS_MASK                0x00001000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE__SHIFT             0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING__SHIFT            0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE_MASK               0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING_MASK              0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING__SHIFT  0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING_MASK  0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                               0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING__SHIFT                       0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                      0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                 0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING_MASK                         0x00000030L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                        0x00000300L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA0__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA0__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA1__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA1__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA2__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA2__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA3__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA3__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA4__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA4__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA5__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA5__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA6
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA6__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA6__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA7__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA7__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA8__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA8__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                           0x0000FFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY__SHIFT                                          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE__SHIFT                                       0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY_MASK                                            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE_MASK                                         0x00000100L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE__SHIFT                                      0x1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE__SHIFT                                      0x2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE__SHIFT                                      0x3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE__SHIFT                                     0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY__SHIFT                                    0x5
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER__SHIFT                              0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE__SHIFT                                      0x1c
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS__SHIFT                               0x1d
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE_MASK                                        0x00000002L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE_MASK                                        0x00000004L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE_MASK                                        0x00000008L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE_MASK                                       0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY_MASK                                      0x00000020L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER_MASK                                0x00003F00L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_MASK                                        0x10000000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS_MASK                                 0x20000000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE__SHIFT                        0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0xc
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT__SHIFT   0x14
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE_MASK                          0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x000003F0L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x0003F000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT_MASK     0x03F00000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE__SHIFT                 0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE__SHIFT          0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE__SHIFT          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING__SHIFT  0xc
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER__SHIFT  0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE_MASK                   0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE_MASK            0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE_MASK            0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING_MASK    0x00001000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER_MASK  0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                   0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE__SHIFT                            0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER__SHIFT                       0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                     0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE_MASK                              0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER_MASK                         0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE__SHIFT        0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER__SHIFT          0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE_MASK          0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER_MASK            0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER__SHIFT                        0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS__SHIFT                               0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER_MASK                          0x00000030L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS_MASK                                 0x00000100L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE__SHIFT                                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE__SHIFT                               0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE_MASK                                           0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE_MASK                                 0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT1
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT2
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT3
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID__SHIFT                                           0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK__SHIFT                                       0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_MASK                                             0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK_MASK                                         0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_STATUS
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT__SHIFT                                       0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT_MASK                                         0x0000FFFFL
+//DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET__SHIFT                                0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET_MASK                                  0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER__SHIFT                                     0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE__SHIFT                                  0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST__SHIFT                                         0x14
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE__SHIFT                                            0x18
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING__SHIFT                                         0x1c
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER_MASK                                       0x0000FFFFL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE_MASK                                    0x00010000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST_MASK                                           0x00100000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE_MASK                                              0x01000000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING_MASK                                           0x10000000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER__SHIFT                                      0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE__SHIFT                                   0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST__SHIFT                                          0x14
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE__SHIFT                                             0x18
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING__SHIFT                                          0x1c
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER_MASK                                        0x0000FFFFL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE_MASK                                     0x00010000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST_MASK                                            0x00100000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE_MASK                                               0x01000000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING_MASK                                            0x10000000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET__SHIFT                                   0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET__SHIFT                                    0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET_MASK                                     0x0000FFFFL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET_MASK                                      0xFFFF0000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_READY_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE__SHIFT                                0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE_MASK                                  0x00000001L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE__SHIFT                                  0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE__SHIFT                            0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER__SHIFT                            0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE_MASK                                    0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE_MASK                              0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER_MASK                              0x0000FF00L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE__SHIFT                                    0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE__SHIFT                            0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME__SHIFT                                          0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE_MASK                                      0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE_MASK                              0x00000010L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME_MASK                                            0x0000FF00L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_START_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE__SHIFT                                     0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE_MASK                                       0x00000001L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER__SHIFT                            0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE__SHIFT                         0x10
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER__SHIFT                           0x14
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE__SHIFT                                 0x1c
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER_MASK                              0x0000FFFFL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE_MASK                           0x00010000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER_MASK                             0x0FF00000L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE_MASK                                   0x10000000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED__SHIFT                                0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR__SHIFT                                   0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED_MASK                                  0x00000001L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR_MASK                                     0x00000010L
+//DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE__SHIFT                      0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE__SHIFT                                    0x4
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS__SHIFT                                      0x8
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE__SHIFT                                    0xc
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE_MASK                        0x00000003L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE_MASK                                      0x00000030L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS_MASK                                        0x00000100L
+#define DP_SYM32_ENC0_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE_MASK                                      0x00003000L
+//DP_SYM32_ENC0_DP_SYM32_ENC_SPARE
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE__SHIFT                                           0x0
+#define DP_SYM32_ENC0_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc0_dispdec
+//DP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL
+#define DP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN__SHIFT                                   0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32__SHIFT                          0x4
+#define DP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN_MASK                                     0x00000001L
+#define DP_LINK_ENC0_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32_MASK                            0x00000010L
+//DP_LINK_ENC0_DP_LINK_ENC_SPARE
+#define DP_LINK_ENC0_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE__SHIFT                                              0x0
+#define DP_LINK_ENC0_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE_MASK                                                0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym320_dispdec
+//DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE__SHIFT                                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__DPHY_RESET__SHIFT                                               0x1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE__SHIFT                                          0x2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__MODE__SHIFT                                                     0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__NUM_LANES__SHIFT                                                0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE__SHIFT                                              0xa
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD__SHIFT                                   0xc
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP__SHIFT                                         0xd
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE_MASK                                                0x00000001L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__DPHY_RESET_MASK                                                 0x00000002L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE_MASK                                            0x00000004L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__MODE_MASK                                                       0x00000030L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__NUM_LANES_MASK                                                  0x00000300L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE_MASK                                                0x00000400L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD_MASK                                     0x00001000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP_MASK                                           0x00002000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__STATUS__SHIFT                                                    0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__RESET_STATUS__SHIFT                                              0x1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__CURRENT_MODE__SHIFT                                              0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED__SHIFT                                        0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING__SHIFT                                       0xc
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING__SHIFT                                        0x10
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS__SHIFT                                          0x12
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__CM_MODE_EN__SHIFT                                                0x14
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__STATUS_MASK                                                      0x00000001L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__RESET_STATUS_MASK                                                0x00000002L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__CURRENT_MODE_MASK                                                0x00000030L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED_MASK                                          0x00000100L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING_MASK                                         0x00001000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING_MASK                                          0x00030000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS_MASK                                            0x000C0000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_STATUS__CM_MODE_EN_MASK                                                  0x00100000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_UPDATE
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE__SHIFT                                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE_MASK                                              0x00000003L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE__SHIFT                                      0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH__SHIFT                           0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE_MASK                                        0x00000001L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH_MASK                             0x00000030L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR0__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR0__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR1__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR1__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR2__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR2__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR3__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_EDP_ASSR3__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS__SHIFT                                  0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL__SHIFT                              0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT__SHIFT                                    0xc
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME__SHIFT                                     0x10
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS_MASK                                    0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL_MASK                                0x00000FF0L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT_MASK                                      0x0000F000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME_MASK                                       0x00FF0000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD__SHIFT                                    0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD_MASK                                      0x0003FFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY__SHIFT                                         0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES__SHIFT                                    0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY_MASK                                           0x0000007FL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES_MASK                                      0x000FFF00L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0__SHIFT                                           0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1__SHIFT                                             0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1__SHIFT                                           0xc
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2__SHIFT                                             0x10
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2__SHIFT                                           0x14
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3__SHIFT                                             0x18
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3__SHIFT                                           0x1c
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0_MASK                                               0x00000007L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0_MASK                                             0x00000070L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1_MASK                                               0x00000700L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1_MASK                                             0x00007000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2_MASK                                               0x00070000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2_MASK                                             0x00700000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3_MASK                                               0x07000000L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3_MASK                                             0x70000000L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH__SHIFT                                    0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH_MASK                                      0x000000FFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM6
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM7
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM9
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM10
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM__SHIFT                                            0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM_MASK                                              0x00FFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR__SHIFT                              0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR__SHIFT                                          0x1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE__SHIFT                               0x2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR__SHIFT                                        0x3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION__SHIFT                            0x4
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL__SHIFT                               0x5
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION__SHIFT                             0x6
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW__SHIFT                                    0x7
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR__SHIFT                                        0x8
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY__SHIFT                                 0x9
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR_MASK                                0x00000001L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR_MASK                                            0x00000002L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE_MASK                                 0x00000004L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR_MASK                                          0x00000008L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION_MASK                              0x00000010L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL_MASK                                 0x00000020L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION_MASK                               0x00000040L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW_MASK                                      0x00000080L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR_MASK                                          0x00000100L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY_MASK                                   0x00000200L
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT__SHIFT                                  0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT_MASK                                    0x0000FFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT__SHIFT                                 0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT_MASK                                   0xFFFFFFFFL
+//DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE__SHIFT                           0x0
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET__SHIFT                            0x1
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE__SHIFT                          0x2
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET__SHIFT                           0x3
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE_MASK                             0x00000001L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET_MASK                              0x00000002L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE_MASK                            0x00000004L
+#define DP_DPHY_SYM320_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET_MASK                             0x00000008L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_dispdec
+//DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN__SHIFT                             0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                     0x4
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK__SHIFT                  0x9
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK__SHIFT                 0xc
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32__SHIFT                    0x10
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32__SHIFT                0x11
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN_MASK                               0x00000001L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                       0x00000010L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK_MASK                    0x00000200L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK_MASK                   0x00001000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32_MASK                      0x00010000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32_MASK                  0x00020000L
+//DP_STREAM_ENC1_DP_STREAM_ENC_INPUT_MUX_CONTROL
+#define DP_STREAM_ENC1_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL__SHIFT  0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL_MASK  0x00000007L
+//DP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL
+#define DP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT    0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN__SHIFT                         0x8
+#define DP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK      0x00000007L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN_MASK                           0x00000100L
+//DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT             0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT              0x4
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL__SHIFT   0x8
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC__SHIFT     0x10
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT         0x14
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT  0x18
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT              0x1c
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK               0x00000001L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                0x00000010L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL_MASK     0x00003F00L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC_MASK       0x00010000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK           0x00100000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK  0x01000000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                0x30000000L
+//DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT  0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT  0x1
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT  0x2
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT    0x4
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY__SHIFT    0xa
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT      0xc
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT      0x10
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT  0x18
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT         0x1f
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK  0x00000001L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK  0x00000002L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK  0x00000004L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK      0x000003F0L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY_MASK      0x00000C00L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK        0x0000F000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK        0x001F0000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK    0x3F000000L
+#define DP_STREAM_ENC1_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK           0x80000000L
+//DP_STREAM_ENC1_DP_STREAM_ENC_SPARE
+#define DP_STREAM_ENC1_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE__SHIFT                                        0x0
+#define DP_STREAM_ENC1_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_apg_apg_dispdec
+//APG6_APG_DBG_GEN_CONTROL
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG6_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG6_APG_MEM_PWR
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG6_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG6_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG6_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_dme_dme_dispdec
+//DME6_DME_CONTROL
+#define DME6_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME6_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME6_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME6_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME6_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME6_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME6_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME6_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME6_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME6_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME6_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME6_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME6_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME6_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME6_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME6_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME6_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME6_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME6_DME_MEMORY_CONTROL
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME6_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc1_vpg_vpg_dispdec
+//VPG6_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG6_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG6_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG6_VPG_GENERIC_PACKET_DATA
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG6_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG6_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG6_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG6_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG6_VPG_GENERIC_STATUS
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG6_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG6_VPG_MEM_PWR
+#define VPG6_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG6_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG6_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG6_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG6_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG6_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG6_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG6_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG6_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG6_VPG_ISRC1_2_DATA
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG6_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc1_dispdec
+//DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE__SHIFT                                        0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET__SHIFT                                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE__SHIFT                                    0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE_MASK                                          0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_MASK                                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE_MASK                                      0x00000100L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE__SHIFT                       0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET__SHIFT                        0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE__SHIFT                   0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS__SHIFT              0xc
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE_MASK                         0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_MASK                          0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE_MASK                     0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS_MASK                0x00001000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE__SHIFT             0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING__SHIFT            0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE_MASK               0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING_MASK              0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING__SHIFT  0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING_MASK  0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                               0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING__SHIFT                       0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                      0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                 0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING_MASK                         0x00000030L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                        0x00000300L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA0__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA0__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA1__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA1__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA2__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA2__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA3__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA3__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA4__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA4__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA5__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA5__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA6
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA6__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA6__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA7__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA7__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA8__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA8__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                           0x0000FFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY__SHIFT                                          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE__SHIFT                                       0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY_MASK                                            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE_MASK                                         0x00000100L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE__SHIFT                                      0x1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE__SHIFT                                      0x2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE__SHIFT                                      0x3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE__SHIFT                                     0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY__SHIFT                                    0x5
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER__SHIFT                              0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE__SHIFT                                      0x1c
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS__SHIFT                               0x1d
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE_MASK                                        0x00000002L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE_MASK                                        0x00000004L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE_MASK                                        0x00000008L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE_MASK                                       0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY_MASK                                      0x00000020L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER_MASK                                0x00003F00L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_MASK                                        0x10000000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS_MASK                                 0x20000000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE__SHIFT                        0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0xc
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT__SHIFT   0x14
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE_MASK                          0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x000003F0L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x0003F000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT_MASK     0x03F00000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE__SHIFT                 0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE__SHIFT          0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE__SHIFT          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING__SHIFT  0xc
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER__SHIFT  0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE_MASK                   0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE_MASK            0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE_MASK            0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING_MASK    0x00001000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER_MASK  0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                   0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE__SHIFT                            0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER__SHIFT                       0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                     0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE_MASK                              0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER_MASK                         0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE__SHIFT        0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER__SHIFT          0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE_MASK          0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER_MASK            0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER__SHIFT                        0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS__SHIFT                               0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER_MASK                          0x00000030L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS_MASK                                 0x00000100L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE__SHIFT                                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE__SHIFT                               0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE_MASK                                           0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE_MASK                                 0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT1
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT2
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT3
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID__SHIFT                                           0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK__SHIFT                                       0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_MASK                                             0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK_MASK                                         0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_STATUS
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT__SHIFT                                       0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT_MASK                                         0x0000FFFFL
+//DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET__SHIFT                                0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET_MASK                                  0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER__SHIFT                                     0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE__SHIFT                                  0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST__SHIFT                                         0x14
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE__SHIFT                                            0x18
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING__SHIFT                                         0x1c
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER_MASK                                       0x0000FFFFL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE_MASK                                    0x00010000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST_MASK                                           0x00100000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE_MASK                                              0x01000000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING_MASK                                           0x10000000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER__SHIFT                                      0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE__SHIFT                                   0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST__SHIFT                                          0x14
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE__SHIFT                                             0x18
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING__SHIFT                                          0x1c
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER_MASK                                        0x0000FFFFL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE_MASK                                     0x00010000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST_MASK                                            0x00100000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE_MASK                                               0x01000000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING_MASK                                            0x10000000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET__SHIFT                                   0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET__SHIFT                                    0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET_MASK                                     0x0000FFFFL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET_MASK                                      0xFFFF0000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_READY_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE__SHIFT                                0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE_MASK                                  0x00000001L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE__SHIFT                                  0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE__SHIFT                            0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER__SHIFT                            0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE_MASK                                    0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE_MASK                              0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER_MASK                              0x0000FF00L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE__SHIFT                                    0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE__SHIFT                            0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME__SHIFT                                          0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE_MASK                                      0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE_MASK                              0x00000010L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME_MASK                                            0x0000FF00L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_START_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE__SHIFT                                     0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE_MASK                                       0x00000001L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER__SHIFT                            0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE__SHIFT                         0x10
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER__SHIFT                           0x14
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE__SHIFT                                 0x1c
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER_MASK                              0x0000FFFFL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE_MASK                           0x00010000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER_MASK                             0x0FF00000L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE_MASK                                   0x10000000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED__SHIFT                                0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR__SHIFT                                   0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED_MASK                                  0x00000001L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR_MASK                                     0x00000010L
+//DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE__SHIFT                      0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE__SHIFT                                    0x4
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS__SHIFT                                      0x8
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE__SHIFT                                    0xc
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE_MASK                        0x00000003L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE_MASK                                      0x00000030L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS_MASK                                        0x00000100L
+#define DP_SYM32_ENC1_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE_MASK                                      0x00003000L
+//DP_SYM32_ENC1_DP_SYM32_ENC_SPARE
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE__SHIFT                                           0x0
+#define DP_SYM32_ENC1_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc1_dispdec
+//DP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL
+#define DP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN__SHIFT                                   0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32__SHIFT                          0x4
+#define DP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN_MASK                                     0x00000001L
+#define DP_LINK_ENC1_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32_MASK                            0x00000010L
+//DP_LINK_ENC1_DP_LINK_ENC_SPARE
+#define DP_LINK_ENC1_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE__SHIFT                                              0x0
+#define DP_LINK_ENC1_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE_MASK                                                0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym321_dispdec
+//DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE__SHIFT                                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__DPHY_RESET__SHIFT                                               0x1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE__SHIFT                                          0x2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__MODE__SHIFT                                                     0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__NUM_LANES__SHIFT                                                0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE__SHIFT                                              0xa
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD__SHIFT                                   0xc
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP__SHIFT                                         0xd
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE_MASK                                                0x00000001L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__DPHY_RESET_MASK                                                 0x00000002L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE_MASK                                            0x00000004L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__MODE_MASK                                                       0x00000030L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__NUM_LANES_MASK                                                  0x00000300L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE_MASK                                                0x00000400L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD_MASK                                     0x00001000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP_MASK                                           0x00002000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__STATUS__SHIFT                                                    0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__RESET_STATUS__SHIFT                                              0x1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__CURRENT_MODE__SHIFT                                              0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED__SHIFT                                        0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING__SHIFT                                       0xc
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING__SHIFT                                        0x10
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS__SHIFT                                          0x12
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__CM_MODE_EN__SHIFT                                                0x14
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__STATUS_MASK                                                      0x00000001L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__RESET_STATUS_MASK                                                0x00000002L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__CURRENT_MODE_MASK                                                0x00000030L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED_MASK                                          0x00000100L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING_MASK                                         0x00001000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING_MASK                                          0x00030000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS_MASK                                            0x000C0000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_STATUS__CM_MODE_EN_MASK                                                  0x00100000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_UPDATE
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE__SHIFT                                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE_MASK                                              0x00000003L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE__SHIFT                                      0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH__SHIFT                           0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE_MASK                                        0x00000001L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH_MASK                             0x00000030L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR0__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR0__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR1__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR1__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR2__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR2__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR3__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_EDP_ASSR3__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS__SHIFT                                  0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL__SHIFT                              0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT__SHIFT                                    0xc
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME__SHIFT                                     0x10
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS_MASK                                    0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL_MASK                                0x00000FF0L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT_MASK                                      0x0000F000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME_MASK                                       0x00FF0000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD__SHIFT                                    0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD_MASK                                      0x0003FFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY__SHIFT                                         0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES__SHIFT                                    0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY_MASK                                           0x0000007FL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES_MASK                                      0x000FFF00L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0__SHIFT                                           0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1__SHIFT                                             0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1__SHIFT                                           0xc
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2__SHIFT                                             0x10
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2__SHIFT                                           0x14
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3__SHIFT                                             0x18
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3__SHIFT                                           0x1c
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0_MASK                                               0x00000007L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0_MASK                                             0x00000070L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1_MASK                                               0x00000700L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1_MASK                                             0x00007000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2_MASK                                               0x00070000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2_MASK                                             0x00700000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3_MASK                                               0x07000000L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3_MASK                                             0x70000000L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_SQ_PULSE
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH__SHIFT                                    0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH_MASK                                      0x000000FFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM6
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM7
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM9
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM10
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM__SHIFT                                            0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM_MASK                                              0x00FFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR__SHIFT                              0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR__SHIFT                                          0x1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE__SHIFT                               0x2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR__SHIFT                                        0x3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION__SHIFT                            0x4
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL__SHIFT                               0x5
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION__SHIFT                             0x6
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW__SHIFT                                    0x7
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR__SHIFT                                        0x8
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY__SHIFT                                 0x9
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR_MASK                                0x00000001L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR_MASK                                            0x00000002L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE_MASK                                 0x00000004L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR_MASK                                          0x00000008L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION_MASK                              0x00000010L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL_MASK                                 0x00000020L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION_MASK                               0x00000040L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW_MASK                                      0x00000080L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR_MASK                                          0x00000100L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY_MASK                                   0x00000200L
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT__SHIFT                                  0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT_MASK                                    0x0000FFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT__SHIFT                                 0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT_MASK                                   0xFFFFFFFFL
+//DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE__SHIFT                           0x0
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET__SHIFT                            0x1
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE__SHIFT                          0x2
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET__SHIFT                           0x3
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE_MASK                             0x00000001L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET_MASK                              0x00000002L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE_MASK                            0x00000004L
+#define DP_DPHY_SYM321_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET_MASK                             0x00000008L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_dispdec
+//DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN__SHIFT                             0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                     0x4
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK__SHIFT                  0x9
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK__SHIFT                 0xc
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32__SHIFT                    0x10
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32__SHIFT                0x11
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN_MASK                               0x00000001L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                       0x00000010L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK_MASK                    0x00000200L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK_MASK                   0x00001000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32_MASK                      0x00010000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32_MASK                  0x00020000L
+//DP_STREAM_ENC2_DP_STREAM_ENC_INPUT_MUX_CONTROL
+#define DP_STREAM_ENC2_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL__SHIFT  0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL_MASK  0x00000007L
+//DP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL
+#define DP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT    0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN__SHIFT                         0x8
+#define DP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK      0x00000007L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN_MASK                           0x00000100L
+//DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT             0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT              0x4
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL__SHIFT   0x8
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC__SHIFT     0x10
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT         0x14
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT  0x18
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT              0x1c
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK               0x00000001L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                0x00000010L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL_MASK     0x00003F00L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC_MASK       0x00010000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK           0x00100000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK  0x01000000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                0x30000000L
+//DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT  0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT  0x1
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT  0x2
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT    0x4
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY__SHIFT    0xa
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT      0xc
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT      0x10
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT  0x18
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT         0x1f
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK  0x00000001L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK  0x00000002L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK  0x00000004L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK      0x000003F0L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY_MASK      0x00000C00L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK        0x0000F000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK        0x001F0000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK    0x3F000000L
+#define DP_STREAM_ENC2_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK           0x80000000L
+//DP_STREAM_ENC2_DP_STREAM_ENC_SPARE
+#define DP_STREAM_ENC2_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE__SHIFT                                        0x0
+#define DP_STREAM_ENC2_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_apg_apg_dispdec
+//APG7_APG_DBG_GEN_CONTROL
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG7_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG7_APG_MEM_PWR
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG7_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG7_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG7_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_dme_dme_dispdec
+//DME7_DME_CONTROL
+#define DME7_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME7_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME7_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME7_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME7_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME7_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME7_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME7_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME7_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME7_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME7_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME7_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME7_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME7_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME7_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME7_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME7_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME7_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME7_DME_MEMORY_CONTROL
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME7_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc2_vpg_vpg_dispdec
+//VPG7_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG7_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG7_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG7_VPG_GENERIC_PACKET_DATA
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG7_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG7_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG7_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG7_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG7_VPG_GENERIC_STATUS
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG7_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG7_VPG_MEM_PWR
+#define VPG7_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG7_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG7_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG7_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG7_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG7_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG7_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG7_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG7_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG7_VPG_ISRC1_2_DATA
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG7_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc2_dispdec
+//DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE__SHIFT                                        0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET__SHIFT                                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE__SHIFT                                    0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE_MASK                                          0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_MASK                                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE_MASK                                      0x00000100L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE__SHIFT                       0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET__SHIFT                        0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE__SHIFT                   0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS__SHIFT              0xc
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE_MASK                         0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_MASK                          0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE_MASK                     0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS_MASK                0x00001000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE__SHIFT             0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING__SHIFT            0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE_MASK               0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING_MASK              0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING__SHIFT  0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING_MASK  0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                               0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING__SHIFT                       0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                      0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                 0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING_MASK                         0x00000030L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                        0x00000300L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA0__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA0__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA1__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA1__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA2__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA2__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA3__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA3__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA4__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA4__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA5__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA5__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA6
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA6__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA6__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA7__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA7__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA8__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA8__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                           0x0000FFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY__SHIFT                                          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE__SHIFT                                       0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY_MASK                                            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE_MASK                                         0x00000100L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE__SHIFT                                      0x1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE__SHIFT                                      0x2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE__SHIFT                                      0x3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE__SHIFT                                     0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY__SHIFT                                    0x5
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER__SHIFT                              0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE__SHIFT                                      0x1c
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS__SHIFT                               0x1d
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE_MASK                                        0x00000002L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE_MASK                                        0x00000004L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE_MASK                                        0x00000008L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE_MASK                                       0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY_MASK                                      0x00000020L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER_MASK                                0x00003F00L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_MASK                                        0x10000000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS_MASK                                 0x20000000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE__SHIFT                        0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0xc
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT__SHIFT   0x14
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE_MASK                          0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x000003F0L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x0003F000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT_MASK     0x03F00000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE__SHIFT                 0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE__SHIFT          0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE__SHIFT          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING__SHIFT  0xc
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER__SHIFT  0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE_MASK                   0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE_MASK            0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE_MASK            0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING_MASK    0x00001000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER_MASK  0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                   0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE__SHIFT                            0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER__SHIFT                       0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                     0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE_MASK                              0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER_MASK                         0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE__SHIFT        0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER__SHIFT          0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE_MASK          0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER_MASK            0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER__SHIFT                        0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS__SHIFT                               0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER_MASK                          0x00000030L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS_MASK                                 0x00000100L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE__SHIFT                                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE__SHIFT                               0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE_MASK                                           0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE_MASK                                 0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT1
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT2
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT3
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID__SHIFT                                           0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK__SHIFT                                       0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_MASK                                             0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK_MASK                                         0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_STATUS
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT__SHIFT                                       0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT_MASK                                         0x0000FFFFL
+//DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET__SHIFT                                0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET_MASK                                  0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER__SHIFT                                     0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE__SHIFT                                  0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST__SHIFT                                         0x14
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE__SHIFT                                            0x18
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING__SHIFT                                         0x1c
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER_MASK                                       0x0000FFFFL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE_MASK                                    0x00010000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST_MASK                                           0x00100000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE_MASK                                              0x01000000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING_MASK                                           0x10000000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER__SHIFT                                      0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE__SHIFT                                   0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST__SHIFT                                          0x14
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE__SHIFT                                             0x18
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING__SHIFT                                          0x1c
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER_MASK                                        0x0000FFFFL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE_MASK                                     0x00010000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST_MASK                                            0x00100000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE_MASK                                               0x01000000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING_MASK                                            0x10000000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET__SHIFT                                   0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET__SHIFT                                    0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET_MASK                                     0x0000FFFFL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET_MASK                                      0xFFFF0000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_READY_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE__SHIFT                                0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE_MASK                                  0x00000001L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE__SHIFT                                  0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE__SHIFT                            0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER__SHIFT                            0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE_MASK                                    0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE_MASK                              0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER_MASK                              0x0000FF00L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE__SHIFT                                    0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE__SHIFT                            0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME__SHIFT                                          0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE_MASK                                      0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE_MASK                              0x00000010L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME_MASK                                            0x0000FF00L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_START_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE__SHIFT                                     0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE_MASK                                       0x00000001L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER__SHIFT                            0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE__SHIFT                         0x10
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER__SHIFT                           0x14
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE__SHIFT                                 0x1c
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER_MASK                              0x0000FFFFL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE_MASK                           0x00010000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER_MASK                             0x0FF00000L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE_MASK                                   0x10000000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED__SHIFT                                0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR__SHIFT                                   0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED_MASK                                  0x00000001L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR_MASK                                     0x00000010L
+//DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE__SHIFT                      0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE__SHIFT                                    0x4
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS__SHIFT                                      0x8
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE__SHIFT                                    0xc
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE_MASK                        0x00000003L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE_MASK                                      0x00000030L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS_MASK                                        0x00000100L
+#define DP_SYM32_ENC2_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE_MASK                                      0x00003000L
+//DP_SYM32_ENC2_DP_SYM32_ENC_SPARE
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE__SHIFT                                           0x0
+#define DP_SYM32_ENC2_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc2_dispdec
+//DP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL
+#define DP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN__SHIFT                                   0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32__SHIFT                          0x4
+#define DP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN_MASK                                     0x00000001L
+#define DP_LINK_ENC2_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32_MASK                            0x00000010L
+//DP_LINK_ENC2_DP_LINK_ENC_SPARE
+#define DP_LINK_ENC2_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE__SHIFT                                              0x0
+#define DP_LINK_ENC2_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE_MASK                                                0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym322_dispdec
+//DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE__SHIFT                                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__DPHY_RESET__SHIFT                                               0x1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE__SHIFT                                          0x2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__MODE__SHIFT                                                     0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__NUM_LANES__SHIFT                                                0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE__SHIFT                                              0xa
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD__SHIFT                                   0xc
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP__SHIFT                                         0xd
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE_MASK                                                0x00000001L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__DPHY_RESET_MASK                                                 0x00000002L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE_MASK                                            0x00000004L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__MODE_MASK                                                       0x00000030L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__NUM_LANES_MASK                                                  0x00000300L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE_MASK                                                0x00000400L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD_MASK                                     0x00001000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP_MASK                                           0x00002000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__STATUS__SHIFT                                                    0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__RESET_STATUS__SHIFT                                              0x1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__CURRENT_MODE__SHIFT                                              0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED__SHIFT                                        0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING__SHIFT                                       0xc
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING__SHIFT                                        0x10
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS__SHIFT                                          0x12
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__CM_MODE_EN__SHIFT                                                0x14
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__STATUS_MASK                                                      0x00000001L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__RESET_STATUS_MASK                                                0x00000002L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__CURRENT_MODE_MASK                                                0x00000030L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED_MASK                                          0x00000100L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING_MASK                                         0x00001000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING_MASK                                          0x00030000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS_MASK                                            0x000C0000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_STATUS__CM_MODE_EN_MASK                                                  0x00100000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_UPDATE
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE__SHIFT                                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE_MASK                                              0x00000003L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE__SHIFT                                      0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH__SHIFT                           0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE_MASK                                        0x00000001L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH_MASK                             0x00000030L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR0__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR0__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR1__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR1__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR2__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR2__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR3__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_EDP_ASSR3__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS__SHIFT                                  0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL__SHIFT                              0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT__SHIFT                                    0xc
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME__SHIFT                                     0x10
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS_MASK                                    0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL_MASK                                0x00000FF0L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT_MASK                                      0x0000F000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME_MASK                                       0x00FF0000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD__SHIFT                                    0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD_MASK                                      0x0003FFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY__SHIFT                                         0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES__SHIFT                                    0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY_MASK                                           0x0000007FL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES_MASK                                      0x000FFF00L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0__SHIFT                                           0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1__SHIFT                                             0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1__SHIFT                                           0xc
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2__SHIFT                                             0x10
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2__SHIFT                                           0x14
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3__SHIFT                                             0x18
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3__SHIFT                                           0x1c
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0_MASK                                               0x00000007L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0_MASK                                             0x00000070L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1_MASK                                               0x00000700L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1_MASK                                             0x00007000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2_MASK                                               0x00070000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2_MASK                                             0x00700000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3_MASK                                               0x07000000L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3_MASK                                             0x70000000L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_SQ_PULSE
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH__SHIFT                                    0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH_MASK                                      0x000000FFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM6
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM7
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM9
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM10
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM__SHIFT                                            0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM_MASK                                              0x00FFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR__SHIFT                              0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR__SHIFT                                          0x1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE__SHIFT                               0x2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR__SHIFT                                        0x3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION__SHIFT                            0x4
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL__SHIFT                               0x5
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION__SHIFT                             0x6
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW__SHIFT                                    0x7
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR__SHIFT                                        0x8
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY__SHIFT                                 0x9
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR_MASK                                0x00000001L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR_MASK                                            0x00000002L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE_MASK                                 0x00000004L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR_MASK                                          0x00000008L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION_MASK                              0x00000010L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL_MASK                                 0x00000020L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION_MASK                               0x00000040L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW_MASK                                      0x00000080L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR_MASK                                          0x00000100L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY_MASK                                   0x00000200L
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT__SHIFT                                  0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT_MASK                                    0x0000FFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT__SHIFT                                 0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT_MASK                                   0xFFFFFFFFL
+//DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE__SHIFT                           0x0
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET__SHIFT                            0x1
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE__SHIFT                          0x2
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET__SHIFT                           0x3
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE_MASK                             0x00000001L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET_MASK                              0x00000002L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE_MASK                            0x00000004L
+#define DP_DPHY_SYM322_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET_MASK                             0x00000008L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_dispdec
+//DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN__SHIFT                             0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                     0x4
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK__SHIFT                  0x9
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK__SHIFT                 0xc
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32__SHIFT                    0x10
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32__SHIFT                0x11
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN_MASK                               0x00000001L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                       0x00000010L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SOCCLK_MASK                    0x00000200L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_DPSTREAMCLK_MASK                   0x00001000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_ON_SYMCLK32_MASK                      0x00010000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_APG_CLOCK_ON_SYMCLK32_MASK                  0x00020000L
+//DP_STREAM_ENC3_DP_STREAM_ENC_INPUT_MUX_CONTROL
+#define DP_STREAM_ENC3_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL__SHIFT  0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_INPUT_MUX_CONTROL__DP_STREAM_ENC_INPUT_MUX_PIXEL_STREAM_SOURCE_SEL_MASK  0x00000007L
+//DP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL
+#define DP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT    0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN__SHIFT                         0x8
+#define DP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK      0x00000007L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_AUDIO_CONTROL__DP_STREAM_ENC_APG_CLOCK_EN_MASK                           0x00000100L
+//DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT             0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT              0x4
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL__SHIFT   0x8
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC__SHIFT     0x10
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT         0x14
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT  0x18
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT              0x1c
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK               0x00000001L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                0x00000010L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_START_LEVEL_MASK     0x00003F00L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_READ_CLOCK_SRC_MASK       0x00010000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK           0x00100000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK  0x01000000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                0x30000000L
+//DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT  0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT  0x1
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT  0x2
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT    0x4
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY__SHIFT    0xa
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT      0xc
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT      0x10
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT  0x18
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT         0x1f
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK  0x00000001L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK  0x00000002L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK  0x00000004L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK      0x000003F0L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY_MASK      0x00000C00L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK        0x0000F000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK        0x001F0000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK    0x3F000000L
+#define DP_STREAM_ENC3_DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK           0x80000000L
+//DP_STREAM_ENC3_DP_STREAM_ENC_SPARE
+#define DP_STREAM_ENC3_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE__SHIFT                                        0x0
+#define DP_STREAM_ENC3_DP_STREAM_ENC_SPARE__DP_STREAM_ENC_SPARE_MASK                                          0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_apg_apg_dispdec
+//APG8_APG_DBG_GEN_CONTROL
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG8_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG8_APG_MEM_PWR
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG8_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG8_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG8_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_dme_dme_dispdec
+//DME8_DME_CONTROL
+#define DME8_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME8_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME8_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME8_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME8_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME8_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME8_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME8_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME8_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME8_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME8_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME8_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME8_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME8_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME8_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME8_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME8_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME8_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME8_DME_MEMORY_CONTROL
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME8_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_enc3_vpg_vpg_dispdec
+//VPG8_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG8_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG8_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG8_VPG_GENERIC_PACKET_DATA
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG8_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG8_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG8_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG8_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG8_VPG_GENERIC_STATUS
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG8_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG8_VPG_MEM_PWR
+#define VPG8_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG8_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG8_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG8_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG8_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG8_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG8_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG8_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG8_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG8_VPG_ISRC1_2_DATA
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG8_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_hpo_dp_sym32_enc3_dispdec
+//DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE__SHIFT                                        0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET__SHIFT                                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE__SHIFT                                    0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_ENABLE_MASK                                          0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_MASK                                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_CONTROL__DP_SYM32_ENC_RESET_DONE_MASK                                      0x00000100L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE__SHIFT                       0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET__SHIFT                        0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE__SHIFT                   0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS__SHIFT              0xc
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_ENABLE_MASK                         0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_MASK                          0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_RESET_DONE_MASK                     0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_FIFO_CONTROL__PIXEL_TO_SYMBOL_FIFO_OVERFLOW_STATUS_MASK                0x00001000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE__SHIFT             0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING__SHIFT            0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_ENABLE_MASK               0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL__MSA_DOUBLE_BUFFER_PENDING_MASK              0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING__SHIFT  0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL__PIXEL_FORMAT_DOUBLE_BUFFER_PENDING_MASK  0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE__SHIFT                               0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING__SHIFT                       0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH__SHIFT                      0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__PIXEL_ENCODING_TYPE_MASK                                 0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_PIXEL_ENCODING_MASK                         0x00000030L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PIXEL_FORMAT__UNCOMPRESSED_COMPONENT_DEPTH_MASK                        0x00000300L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA0__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA0__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA1__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA1__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA2__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA2__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA3__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA3__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA4__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA4__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA5__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA5__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA6
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA6__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA6__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA7__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA7__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA8__MSA_DATA__SHIFT                                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA8__MSA_DATA_MASK                                                    0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_HBLANK_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH__SHIFT                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_HBLANK_CONTROL__HBLANK_MINIMUM_SYMBOL_WIDTH_MASK                           0x0000FFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL0__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL1__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL2__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL3__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL4__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL5__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL6__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL7__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL8__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT           0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                         0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                     0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE__SHIFT                                  0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE__SHIFT                                 0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                  0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING__SHIFT                         0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                      0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK             0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_SEND_MASK                           0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                       0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_ENABLE_MASK                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_PAYLOAD_SIZE_MASK                                    0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_SOF_REFERENCE_MASK                                   0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                    0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_DOUBLE_BUFFER_PENDING_MASK                           0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL9__GSP_TRANSMISSION_LINE_NUMBER_MASK                        0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL10__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL11__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL12__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL13__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE__SHIFT          0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND__SHIFT                        0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION__SHIFT                    0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE__SHIFT                         0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE__SHIFT                                 0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE__SHIFT                                0x7
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED__SHIFT         0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING__SHIFT                 0x9
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING__SHIFT                        0xa
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER__SHIFT                     0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_VIDEO_CONTINUOUS_TRANSMISSION_ENABLE_MASK           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_IDLE_CONTINUOUS_TRANSMISSION_ENABLE_MASK            0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_SEND_MASK                          0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_ONE_SHOT_POSITION_MASK                      0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_ENABLE_MASK                           0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_PAYLOAD_SIZE_MASK                                   0x00000060L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_SOF_REFERENCE_MASK                                  0x00000080L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_DEADLINE_MISSED_MASK           0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRIGGER_TRANSMISSION_PENDING_MASK                   0x00000200L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_DOUBLE_BUFFER_PENDING_MASK                          0x00000400L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_GSP_CONTROL14__GSP_TRANSMISSION_LINE_NUMBER_MASK                       0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY__SHIFT                                          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE__SHIFT                                       0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__SDP_STREAM_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__GSP0_PRIORITY_MASK                                            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_CONTROL__SDP_CRC16_ENABLE_MASK                                         0x00000100L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE__SHIFT                                      0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE__SHIFT                                      0x1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE__SHIFT                                      0x2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE__SHIFT                                      0x3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE__SHIFT                                     0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY__SHIFT                                    0x5
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER__SHIFT                              0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE__SHIFT                                      0x1c
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS__SHIFT                               0x1d
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_ENABLE_MASK                                        0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_ENABLE_MASK                                        0x00000002L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AIP_ENABLE_MASK                                        0x00000004L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ACM_ENABLE_MASK                                        0x00000008L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ISRC_ENABLE_MASK                                       0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ASP_PRIORITY_MASK                                      0x00000020L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__ATP_VERSION_NUMBER_MASK                                0x00003F00L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_MASK                                        0x10000000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL0__AUDIO_MUTE_STATUS_MASK                                 0x20000000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE__SHIFT                        0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT__SHIFT  0xc
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT__SHIFT   0x14
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_ENABLE_MASK                          0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_2_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x000003F0L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_8_CHANNEL_LAYOUT_MAX_SAMPLE_COUNT_MASK  0x0003F000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_AUDIO_CONTROL1__ASP_CONCATENATION_HBR_LAYOUT_MAX_SAMPLE_COUNT_MASK     0x03F00000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE__SHIFT                 0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE__SHIFT          0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE__SHIFT          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING__SHIFT  0xc
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER__SHIFT  0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_ENABLE_MASK                   0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_DOUBLE_BUFFER_ENABLE_MASK            0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_SOF_REFERENCE_MASK            0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_DOUBLE_BUFFER_PENDING_MASK    0x00001000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL__METADATA_PACKET_TRANSMISSION_LINE_NUMBER_MASK  0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN__SHIFT                   0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE__SHIFT                            0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER__SHIFT                       0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_MISC1_STEREOSYNC_OVERRIDE_EN_MASK                     0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_ENABLE_MASK                              0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_MSA_CONTROL__MSA_TRANSMISSION_LINE_NUMBER_MASK                         0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE__SHIFT        0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER__SHIFT          0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_SOF_REFERENCE_MASK          0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_VBID_CONTROL__VBID_6_COMPRESSEDSTREAM_FLAG_LINE_NUMBER_MASK            0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER__SHIFT                        0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS__SHIFT                               0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_DISABLE_DEFER_MASK                          0x00000030L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_STREAM_CONTROL__VID_STREAM_STATUS_MASK                                 0x00000100L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE__SHIFT  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE__SHIFT  0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_ENABLE_MASK  0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_PANEL_REPLAY_CONTROL__PANEL_REPLAY_TUNNELING_OPTIMIZATION_DOUBLE_BUFFER_ENABLE_MASK  0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE__SHIFT                                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE__SHIFT                               0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_ENABLE_MASK                                           0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_CONTROL__CRC_CONT_MODE_ENABLE_MASK                                 0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT0__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT1
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT1__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT2
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT2__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT3
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT__SHIFT                                         0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_RESULT3__CRC_RESULT_MASK                                           0xFFFFFFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID__SHIFT                                           0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK__SHIFT                                       0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_MASK                                             0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_VID_CRC_STATUS__CRC_VALID_ACK_MASK                                         0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_STATUS
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT__SHIFT                                       0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_STATUS__BS_COUNT_MASK                                         0x0000FFFFL
+//DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE__SHIFT                               0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET__SHIFT                                0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_ENABLE_MASK                                 0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SYMBOL_COUNT_CONTROL__BS_COUNT_RESET_MASK                                  0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER__SHIFT                                     0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE__SHIFT                                  0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST__SHIFT                                         0x14
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE__SHIFT                                            0x18
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING__SHIFT                                         0x1c
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_NUMBER_MASK                                       0x0000FFFFL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__LINE_REFERENCE_MASK                                    0x00010000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__REQUEST_MASK                                           0x00100000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__MODE_MASK                                              0x01000000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_SLEEP_CONTROL__PENDING_MASK                                           0x10000000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER__SHIFT                                      0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE__SHIFT                                   0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST__SHIFT                                          0x14
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE__SHIFT                                             0x18
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING__SHIFT                                          0x1c
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_NUMBER_MASK                                        0x0000FFFFL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__LINE_REFERENCE_MASK                                     0x00010000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__REQUEST_MASK                                            0x00100000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__MODE_MASK                                               0x01000000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_CONTROL__PENDING_MASK                                            0x10000000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET__SHIFT                                   0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET__SHIFT                                    0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__SLEEP_OFFSET_MASK                                     0x0000FFFFL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_REQUEST_OFFSET__WAKE_OFFSET_MASK                                      0xFFFF0000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_READY_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE__SHIFT                                0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_READY_CONTROL__SDP_PENDING_MODE_MASK                                  0x00000001L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE__SHIFT                                  0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE__SHIFT                            0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER__SHIFT                            0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__ENABLE_MASK                                    0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__DISABLE_MODE_MASK                              0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_HARDWARE_MODE_CONTROL__FRAME_NUMBER_MASK                              0x0000FF00L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE__SHIFT                                    0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE__SHIFT                            0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME__SHIFT                                          0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_SLEEP_STATE_MASK                                      0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_HARDWARE_MODE_STATE_MASK                              0x00000010L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_STATUS__CURRENT_FRAME_MASK                                            0x0000FF00L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_START_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE__SHIFT                                     0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_START_CONTROL__START_STATE_MASK                                       0x00000001L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER__SHIFT                            0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE__SHIFT                         0x10
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER__SHIFT                           0x14
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE__SHIFT                                 0x1c
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_NUMBER_MASK                              0x0000FFFFL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__LINE_REFERENCE_MASK                           0x00010000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__FRAME_NUMBER_MASK                             0x0FF00000L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_CONTROL__ENABLE_MASK                                   0x10000000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED__SHIFT                                0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR__SHIFT                                   0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__OCCURRED_MASK                                  0x00000001L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_ALPM_WAKE_INTERRUPT_STATUS__CLEAR_MASK                                     0x00000010L
+//DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE__SHIFT                      0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE__SHIFT                                    0x4
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS__SHIFT                                      0x8
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE__SHIFT                                    0xc
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_DEFAULT_LOW_POWER_STATE_MASK                        0x00000003L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_FORCE_MASK                                      0x00000030L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_DIS_MASK                                        0x00000100L
+#define DP_SYM32_ENC3_DP_SYM32_ENC_MEM_POWER_CONTROL__MEM_PWR_STATE_MASK                                      0x00003000L
+//DP_SYM32_ENC3_DP_SYM32_ENC_SPARE
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE__SHIFT                                           0x0
+#define DP_SYM32_ENC3_DP_SYM32_ENC_SPARE__DP_SYM32_ENC_SPARE_MASK                                             0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_link_enc3_dispdec
+//DP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL
+#define DP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN__SHIFT                                   0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32__SHIFT                          0x4
+#define DP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_EN_MASK                                     0x00000001L
+#define DP_LINK_ENC3_DP_LINK_ENC_CLOCK_CONTROL__DP_LINK_ENC_CLOCK_ON_SYMCLK32_MASK                            0x00000010L
+//DP_LINK_ENC3_DP_LINK_ENC_SPARE
+#define DP_LINK_ENC3_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE__SHIFT                                              0x0
+#define DP_LINK_ENC3_DP_LINK_ENC_SPARE__DP_LINK_ENC_SPARE_MASK                                                0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hpo_dp_dphy_sym323_dispdec
+//DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE__SHIFT                                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__DPHY_RESET__SHIFT                                               0x1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE__SHIFT                                          0x2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__MODE__SHIFT                                                     0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__NUM_LANES__SHIFT                                                0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE__SHIFT                                              0xa
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD__SHIFT                                   0xc
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP__SHIFT                                         0xd
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__DPHY_ENABLE_MASK                                                0x00000001L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__DPHY_RESET_MASK                                                 0x00000002L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__PRECODER_ENABLE_MASK                                            0x00000004L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__MODE_MASK                                                       0x00000030L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__NUM_LANES_MASK                                                  0x00000300L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__OUTPUT_MODE_MASK                                                0x00000400L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__SHORT_LAST_TPS2_PERIOD_MASK                                     0x00001000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_CONTROL__START_FROM_SLEEP_MASK                                           0x00002000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__STATUS__SHIFT                                                    0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__RESET_STATUS__SHIFT                                              0x1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__CURRENT_MODE__SHIFT                                              0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED__SHIFT                                        0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING__SHIFT                                       0xc
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING__SHIFT                                        0x10
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS__SHIFT                                          0x12
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__CM_MODE_EN__SHIFT                                                0x14
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__STATUS_MASK                                                      0x00000001L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__RESET_STATUS_MASK                                                0x00000002L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__CURRENT_MODE_MASK                                                0x00000030L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__ENCRYPTION_ENABLED_MASK                                          0x00000100L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__RATE_UPDATE_PENDING_MASK                                         0x00001000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__SAT_UPDATE_PENDING_MASK                                          0x00030000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__SCHEDULER_STATUS_MASK                                            0x000C0000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_STATUS__CM_MODE_EN_MASK                                                  0x00100000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_UPDATE
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE__SHIFT                                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_UPDATE__SAT_UPDATE_MASK                                              0x00000003L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL0__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL1__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL2__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y__SHIFT                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X__SHIFT                                   0x19
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_Y_MASK                                     0x01FFFFFFL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_VC_RATE_CNTL3__STREAM_VC_RATE_X_MASK                                     0xFE000000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC0__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC1__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC2__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE__SHIFT                                        0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE__SHIFT                                    0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE__SHIFT                                      0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT__SHIFT                                           0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_STREAM_SOURCE_MASK                                          0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_ENABLE_MASK                                      0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_ENCRYPTION_TYPE_MASK                                        0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC3__SAT_SLOT_COUNT_MASK                                             0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS0__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS1__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS2__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE__SHIFT                                 0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE__SHIFT                             0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE__SHIFT                               0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT__SHIFT                                    0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_STREAM_SOURCE_MASK                                   0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_ENABLE_MASK                               0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_ENCRYPTION_TYPE_MASK                                 0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SAT_VC_STATUS3__SAT_SLOT_COUNT_MASK                                      0x00007F00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE__SHIFT                                      0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH__SHIFT                           0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0__EDP_MODE_ENABLE_MASK                                        0x00000001L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_CONFIG0__EDP_LINK_SECURITY_STRENGTH_MASK                             0x00000030L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR0__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR0__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR1__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR1__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR2__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR2__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR3__SEED__SHIFT                                                   0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_EDP_ASSR3__SEED_MASK                                                     0x007FFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS__SHIFT                                  0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL__SHIFT                              0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT__SHIFT                                    0xc
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME__SHIFT                                     0x10
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_PATTERNS_MASK                                    0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__PATTERN_INTERVAL_MASK                                0x00000FF0L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__NUM_REPEAT_MASK                                      0x0000F000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_SLEEP_CONFIG0__HOLD_TIME_MASK                                       0x00FF0000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD__SHIFT                                    0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_WAKE_CONFIG0__LOCK_PERIOD_MASK                                      0x0003FFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY__SHIFT                                         0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES__SHIFT                                    0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL__SLEEP_DELAY_MASK                                           0x0000007FL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ALPM_CONTROL__MIN_SLEEP_CYCLES_MASK                                      0x000FFF00L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0__SHIFT                                           0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1__SHIFT                                             0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1__SHIFT                                           0xc
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2__SHIFT                                             0x10
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2__SHIFT                                           0x14
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3__SHIFT                                             0x18
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3__SHIFT                                           0x1c
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT0_MASK                                               0x00000007L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL0_MASK                                             0x00000070L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT1_MASK                                               0x00000700L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL1_MASK                                             0x00007000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT2_MASK                                               0x00070000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL2_MASK                                             0x00700000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_SELECT3_MASK                                               0x07000000L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CONFIG__TP_PRBS_SEL3_MASK                                             0x70000000L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED0__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED1__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED2__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED__SHIFT                                       0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_PRBS_SEED3__TP_PRBS_SEED_MASK                                         0x7FFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_SQ_PULSE
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH__SHIFT                                    0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_SQ_PULSE__TP_SQ_PULSE_WIDTH_MASK                                      0x000000FFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM0__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM1__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM2__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM3__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM4__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM5__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM6
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM6__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM7
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM7__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM8__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM9
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM__SHIFT                                             0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM9__TP_CUSTOM_MASK                                               0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM10
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM__SHIFT                                            0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_TP_CUSTOM10__TP_CUSTOM_MASK                                              0x00FFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR__SHIFT                              0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR__SHIFT                                          0x1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE__SHIFT                               0x2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR__SHIFT                                        0x3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION__SHIFT                            0x4
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL__SHIFT                               0x5
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION__SHIFT                             0x6
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW__SHIFT                                    0x7
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR__SHIFT                                        0x8
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY__SHIFT                                 0x9
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__TOTAL_SLOT_COUNT_ERROR_MASK                                0x00000001L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__RATE_ERROR_MASK                                            0x00000002L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__VC_SAME_STREAM_SOURCE_MASK                                 0x00000004L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__NO_ACT_ERROR_MASK                                          0x00000008L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__UNEXPECT_MODE_TRANSITION_MASK                              0x00000010L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__ILLEGAL_STREAM_SYMBOL_MASK                                 0x00000020L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__RATE_COUNTER_SATURATION_MASK                               0x00000040L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__COUNTER_OVERFLOW_MASK                                      0x00000080L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__CIPHER_ERROR_MASK                                          0x00000100L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_ERROR_STATUS__ALPM_WAKE_REQ_EARLY_MASK                                   0x00000200L
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT__SHIFT                                  0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS0__LLCP_COUNT_MASK                                    0x0000FFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT__SHIFT                                 0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_STATUS1__CYCLE_COUNT_MASK                                   0xFFFFFFFFL
+//DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE__SHIFT                           0x0
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET__SHIFT                            0x1
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE__SHIFT                          0x2
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET__SHIFT                           0x3
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_ENABLE_MASK                             0x00000001L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__LLCP_COUNT_RESET_MASK                              0x00000002L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_ENABLE_MASK                            0x00000004L
+#define DP_DPHY_SYM323_DP_DPHY_SYM32_SYMBOL_COUNT_CONTROL__CYCLE_COUNT_RESET_MASK                             0x00000008L
+
+
+// addressBlock: dce_dc_mpc_mpcc0_dispdec
+//MPCC0_MPCC_TOP_SEL
+#define MPCC0_MPCC_TOP_SEL__MPCC_TOP_SEL__SHIFT                                                               0x0
+#define MPCC0_MPCC_TOP_SEL__MPCC_TOP_SEL_MASK                                                                 0x0000000FL
+//MPCC0_MPCC_BOT_SEL
+#define MPCC0_MPCC_BOT_SEL__MPCC_BOT_SEL__SHIFT                                                               0x0
+#define MPCC0_MPCC_BOT_SEL__MPCC_BOT_SEL_MASK                                                                 0x0000000FL
+//MPCC0_MPCC_OPP_ID
+#define MPCC0_MPCC_OPP_ID__MPCC_OPP_ID__SHIFT                                                                 0x0
+#define MPCC0_MPCC_OPP_ID__MPCC_OPP_ID_MASK                                                                   0x0000000FL
+//MPCC0_MPCC_CONTROL
+#define MPCC0_MPCC_CONTROL__MPCC_MODE__SHIFT                                                                  0x0
+#define MPCC0_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE__SHIFT                                                       0x4
+#define MPCC0_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE__SHIFT                                                 0x6
+#define MPCC0_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY__SHIFT                                              0x7
+#define MPCC0_MPCC_CONTROL__MPCC_BG_BPC__SHIFT                                                                0x8
+#define MPCC0_MPCC_CONTROL__MPCC_BOT_GAIN_MODE__SHIFT                                                         0xb
+#define MPCC0_MPCC_CONTROL__MPCC_MODE_MASK                                                                    0x00000003L
+#define MPCC0_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE_MASK                                                         0x00000030L
+#define MPCC0_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE_MASK                                                   0x00000040L
+#define MPCC0_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY_MASK                                                0x00000080L
+#define MPCC0_MPCC_CONTROL__MPCC_BG_BPC_MASK                                                                  0x00000700L
+#define MPCC0_MPCC_CONTROL__MPCC_BOT_GAIN_MODE_MASK                                                           0x00000800L
+//MPCC0_MPCC_CONTROL2
+#define MPCC0_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA__SHIFT                                                         0x0
+#define MPCC0_MPCC_CONTROL2__MPCC_GLOBAL_GAIN__SHIFT                                                          0x10
+#define MPCC0_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA_MASK                                                           0x00000FFFL
+#define MPCC0_MPCC_CONTROL2__MPCC_GLOBAL_GAIN_MASK                                                            0x0FFF0000L
+//MPCC0_MPCC_SM_CONTROL
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_EN__SHIFT                                                              0x0
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_MODE__SHIFT                                                            0x1
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT__SHIFT                                                       0x4
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT__SHIFT                                                       0x5
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL__SHIFT                                            0x8
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL__SHIFT                                              0x10
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL__SHIFT                                               0x18
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_EN_MASK                                                                0x00000001L
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_MODE_MASK                                                              0x0000000EL
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT_MASK                                                         0x00000010L
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT_MASK                                                         0x00000020L
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL_MASK                                              0x00000300L
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL_MASK                                                0x00030000L
+#define MPCC0_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL_MASK                                                 0x01000000L
+//MPCC0_MPCC_UPDATE_LOCK_SEL
+#define MPCC0_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL__SHIFT                                               0x0
+#define MPCC0_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS__SHIFT                                          0x4
+#define MPCC0_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL_MASK                                                 0x0000000FL
+#define MPCC0_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS_MASK                                            0x00000070L
+//MPCC0_MPCC_TOP_GAIN
+#define MPCC0_MPCC_TOP_GAIN__MPCC_TOP_GAIN__SHIFT                                                             0x0
+#define MPCC0_MPCC_TOP_GAIN__MPCC_TOP_GAIN_MASK                                                               0x0007FFFFL
+//MPCC0_MPCC_BOT_GAIN_INSIDE
+#define MPCC0_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE__SHIFT                                               0x0
+#define MPCC0_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE_MASK                                                 0x0007FFFFL
+//MPCC0_MPCC_BOT_GAIN_OUTSIDE
+#define MPCC0_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE__SHIFT                                             0x0
+#define MPCC0_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE_MASK                                               0x0007FFFFL
+//MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL
+#define MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL__SHIFT                          0x0
+#define MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT__SHIFT                  0x4
+#define MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_MASK                            0x00000001L
+#define MPCC0_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT_MASK                    0x00000010L
+//MPCC0_MPCC_BG_R_CR
+#define MPCC0_MPCC_BG_R_CR__MPCC_BG_R_CR__SHIFT                                                               0x0
+#define MPCC0_MPCC_BG_R_CR__MPCC_BG_R_CR_MASK                                                                 0x00000FFFL
+//MPCC0_MPCC_BG_G_Y
+#define MPCC0_MPCC_BG_G_Y__MPCC_BG_G_Y__SHIFT                                                                 0x0
+#define MPCC0_MPCC_BG_G_Y__MPCC_BG_G_Y_MASK                                                                   0x00000FFFL
+//MPCC0_MPCC_BG_B_CB
+#define MPCC0_MPCC_BG_B_CB__MPCC_BG_B_CB__SHIFT                                                               0x0
+#define MPCC0_MPCC_BG_B_CB__MPCC_BG_B_CB_MASK                                                                 0x00000FFFL
+//MPCC0_MPCC_MEM_PWR_CTRL
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE__SHIFT                                               0x0
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS__SHIFT                                                 0x2
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE__SHIFT                                            0x4
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE__SHIFT                                               0x8
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE_MASK                                                 0x00000003L
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS_MASK                                                   0x00000004L
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE_MASK                                              0x00000030L
+#define MPCC0_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE_MASK                                                 0x00000300L
+//MPCC0_MPCC_STATUS
+#define MPCC0_MPCC_STATUS__MPCC_IDLE__SHIFT                                                                   0x0
+#define MPCC0_MPCC_STATUS__MPCC_BUSY__SHIFT                                                                   0x1
+#define MPCC0_MPCC_STATUS__MPCC_DISABLED__SHIFT                                                               0x2
+#define MPCC0_MPCC_STATUS__MPCC_IDLE_MASK                                                                     0x00000001L
+#define MPCC0_MPCC_STATUS__MPCC_BUSY_MASK                                                                     0x00000002L
+#define MPCC0_MPCC_STATUS__MPCC_DISABLED_MASK                                                                 0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc1_dispdec
+//MPCC1_MPCC_TOP_SEL
+#define MPCC1_MPCC_TOP_SEL__MPCC_TOP_SEL__SHIFT                                                               0x0
+#define MPCC1_MPCC_TOP_SEL__MPCC_TOP_SEL_MASK                                                                 0x0000000FL
+//MPCC1_MPCC_BOT_SEL
+#define MPCC1_MPCC_BOT_SEL__MPCC_BOT_SEL__SHIFT                                                               0x0
+#define MPCC1_MPCC_BOT_SEL__MPCC_BOT_SEL_MASK                                                                 0x0000000FL
+//MPCC1_MPCC_OPP_ID
+#define MPCC1_MPCC_OPP_ID__MPCC_OPP_ID__SHIFT                                                                 0x0
+#define MPCC1_MPCC_OPP_ID__MPCC_OPP_ID_MASK                                                                   0x0000000FL
+//MPCC1_MPCC_CONTROL
+#define MPCC1_MPCC_CONTROL__MPCC_MODE__SHIFT                                                                  0x0
+#define MPCC1_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE__SHIFT                                                       0x4
+#define MPCC1_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE__SHIFT                                                 0x6
+#define MPCC1_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY__SHIFT                                              0x7
+#define MPCC1_MPCC_CONTROL__MPCC_BG_BPC__SHIFT                                                                0x8
+#define MPCC1_MPCC_CONTROL__MPCC_BOT_GAIN_MODE__SHIFT                                                         0xb
+#define MPCC1_MPCC_CONTROL__MPCC_MODE_MASK                                                                    0x00000003L
+#define MPCC1_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE_MASK                                                         0x00000030L
+#define MPCC1_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE_MASK                                                   0x00000040L
+#define MPCC1_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY_MASK                                                0x00000080L
+#define MPCC1_MPCC_CONTROL__MPCC_BG_BPC_MASK                                                                  0x00000700L
+#define MPCC1_MPCC_CONTROL__MPCC_BOT_GAIN_MODE_MASK                                                           0x00000800L
+//MPCC1_MPCC_CONTROL2
+#define MPCC1_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA__SHIFT                                                         0x0
+#define MPCC1_MPCC_CONTROL2__MPCC_GLOBAL_GAIN__SHIFT                                                          0x10
+#define MPCC1_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA_MASK                                                           0x00000FFFL
+#define MPCC1_MPCC_CONTROL2__MPCC_GLOBAL_GAIN_MASK                                                            0x0FFF0000L
+//MPCC1_MPCC_SM_CONTROL
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_EN__SHIFT                                                              0x0
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_MODE__SHIFT                                                            0x1
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT__SHIFT                                                       0x4
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT__SHIFT                                                       0x5
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL__SHIFT                                            0x8
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL__SHIFT                                              0x10
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL__SHIFT                                               0x18
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_EN_MASK                                                                0x00000001L
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_MODE_MASK                                                              0x0000000EL
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT_MASK                                                         0x00000010L
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT_MASK                                                         0x00000020L
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL_MASK                                              0x00000300L
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL_MASK                                                0x00030000L
+#define MPCC1_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL_MASK                                                 0x01000000L
+//MPCC1_MPCC_UPDATE_LOCK_SEL
+#define MPCC1_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL__SHIFT                                               0x0
+#define MPCC1_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS__SHIFT                                          0x4
+#define MPCC1_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL_MASK                                                 0x0000000FL
+#define MPCC1_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS_MASK                                            0x00000070L
+//MPCC1_MPCC_TOP_GAIN
+#define MPCC1_MPCC_TOP_GAIN__MPCC_TOP_GAIN__SHIFT                                                             0x0
+#define MPCC1_MPCC_TOP_GAIN__MPCC_TOP_GAIN_MASK                                                               0x0007FFFFL
+//MPCC1_MPCC_BOT_GAIN_INSIDE
+#define MPCC1_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE__SHIFT                                               0x0
+#define MPCC1_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE_MASK                                                 0x0007FFFFL
+//MPCC1_MPCC_BOT_GAIN_OUTSIDE
+#define MPCC1_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE__SHIFT                                             0x0
+#define MPCC1_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE_MASK                                               0x0007FFFFL
+//MPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL
+#define MPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL__SHIFT                          0x0
+#define MPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT__SHIFT                  0x4
+#define MPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_MASK                            0x00000001L
+#define MPCC1_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT_MASK                    0x00000010L
+//MPCC1_MPCC_BG_R_CR
+#define MPCC1_MPCC_BG_R_CR__MPCC_BG_R_CR__SHIFT                                                               0x0
+#define MPCC1_MPCC_BG_R_CR__MPCC_BG_R_CR_MASK                                                                 0x00000FFFL
+//MPCC1_MPCC_BG_G_Y
+#define MPCC1_MPCC_BG_G_Y__MPCC_BG_G_Y__SHIFT                                                                 0x0
+#define MPCC1_MPCC_BG_G_Y__MPCC_BG_G_Y_MASK                                                                   0x00000FFFL
+//MPCC1_MPCC_BG_B_CB
+#define MPCC1_MPCC_BG_B_CB__MPCC_BG_B_CB__SHIFT                                                               0x0
+#define MPCC1_MPCC_BG_B_CB__MPCC_BG_B_CB_MASK                                                                 0x00000FFFL
+//MPCC1_MPCC_MEM_PWR_CTRL
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE__SHIFT                                               0x0
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS__SHIFT                                                 0x2
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE__SHIFT                                            0x4
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE__SHIFT                                               0x8
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE_MASK                                                 0x00000003L
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS_MASK                                                   0x00000004L
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE_MASK                                              0x00000030L
+#define MPCC1_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE_MASK                                                 0x00000300L
+//MPCC1_MPCC_STATUS
+#define MPCC1_MPCC_STATUS__MPCC_IDLE__SHIFT                                                                   0x0
+#define MPCC1_MPCC_STATUS__MPCC_BUSY__SHIFT                                                                   0x1
+#define MPCC1_MPCC_STATUS__MPCC_DISABLED__SHIFT                                                               0x2
+#define MPCC1_MPCC_STATUS__MPCC_IDLE_MASK                                                                     0x00000001L
+#define MPCC1_MPCC_STATUS__MPCC_BUSY_MASK                                                                     0x00000002L
+#define MPCC1_MPCC_STATUS__MPCC_DISABLED_MASK                                                                 0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc2_dispdec
+//MPCC2_MPCC_TOP_SEL
+#define MPCC2_MPCC_TOP_SEL__MPCC_TOP_SEL__SHIFT                                                               0x0
+#define MPCC2_MPCC_TOP_SEL__MPCC_TOP_SEL_MASK                                                                 0x0000000FL
+//MPCC2_MPCC_BOT_SEL
+#define MPCC2_MPCC_BOT_SEL__MPCC_BOT_SEL__SHIFT                                                               0x0
+#define MPCC2_MPCC_BOT_SEL__MPCC_BOT_SEL_MASK                                                                 0x0000000FL
+//MPCC2_MPCC_OPP_ID
+#define MPCC2_MPCC_OPP_ID__MPCC_OPP_ID__SHIFT                                                                 0x0
+#define MPCC2_MPCC_OPP_ID__MPCC_OPP_ID_MASK                                                                   0x0000000FL
+//MPCC2_MPCC_CONTROL
+#define MPCC2_MPCC_CONTROL__MPCC_MODE__SHIFT                                                                  0x0
+#define MPCC2_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE__SHIFT                                                       0x4
+#define MPCC2_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE__SHIFT                                                 0x6
+#define MPCC2_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY__SHIFT                                              0x7
+#define MPCC2_MPCC_CONTROL__MPCC_BG_BPC__SHIFT                                                                0x8
+#define MPCC2_MPCC_CONTROL__MPCC_BOT_GAIN_MODE__SHIFT                                                         0xb
+#define MPCC2_MPCC_CONTROL__MPCC_MODE_MASK                                                                    0x00000003L
+#define MPCC2_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE_MASK                                                         0x00000030L
+#define MPCC2_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE_MASK                                                   0x00000040L
+#define MPCC2_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY_MASK                                                0x00000080L
+#define MPCC2_MPCC_CONTROL__MPCC_BG_BPC_MASK                                                                  0x00000700L
+#define MPCC2_MPCC_CONTROL__MPCC_BOT_GAIN_MODE_MASK                                                           0x00000800L
+//MPCC2_MPCC_CONTROL2
+#define MPCC2_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA__SHIFT                                                         0x0
+#define MPCC2_MPCC_CONTROL2__MPCC_GLOBAL_GAIN__SHIFT                                                          0x10
+#define MPCC2_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA_MASK                                                           0x00000FFFL
+#define MPCC2_MPCC_CONTROL2__MPCC_GLOBAL_GAIN_MASK                                                            0x0FFF0000L
+//MPCC2_MPCC_SM_CONTROL
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_EN__SHIFT                                                              0x0
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_MODE__SHIFT                                                            0x1
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT__SHIFT                                                       0x4
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT__SHIFT                                                       0x5
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL__SHIFT                                            0x8
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL__SHIFT                                              0x10
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL__SHIFT                                               0x18
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_EN_MASK                                                                0x00000001L
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_MODE_MASK                                                              0x0000000EL
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT_MASK                                                         0x00000010L
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT_MASK                                                         0x00000020L
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL_MASK                                              0x00000300L
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL_MASK                                                0x00030000L
+#define MPCC2_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL_MASK                                                 0x01000000L
+//MPCC2_MPCC_UPDATE_LOCK_SEL
+#define MPCC2_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL__SHIFT                                               0x0
+#define MPCC2_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS__SHIFT                                          0x4
+#define MPCC2_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL_MASK                                                 0x0000000FL
+#define MPCC2_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS_MASK                                            0x00000070L
+//MPCC2_MPCC_TOP_GAIN
+#define MPCC2_MPCC_TOP_GAIN__MPCC_TOP_GAIN__SHIFT                                                             0x0
+#define MPCC2_MPCC_TOP_GAIN__MPCC_TOP_GAIN_MASK                                                               0x0007FFFFL
+//MPCC2_MPCC_BOT_GAIN_INSIDE
+#define MPCC2_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE__SHIFT                                               0x0
+#define MPCC2_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE_MASK                                                 0x0007FFFFL
+//MPCC2_MPCC_BOT_GAIN_OUTSIDE
+#define MPCC2_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE__SHIFT                                             0x0
+#define MPCC2_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE_MASK                                               0x0007FFFFL
+//MPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL
+#define MPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL__SHIFT                          0x0
+#define MPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT__SHIFT                  0x4
+#define MPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_MASK                            0x00000001L
+#define MPCC2_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT_MASK                    0x00000010L
+//MPCC2_MPCC_BG_R_CR
+#define MPCC2_MPCC_BG_R_CR__MPCC_BG_R_CR__SHIFT                                                               0x0
+#define MPCC2_MPCC_BG_R_CR__MPCC_BG_R_CR_MASK                                                                 0x00000FFFL
+//MPCC2_MPCC_BG_G_Y
+#define MPCC2_MPCC_BG_G_Y__MPCC_BG_G_Y__SHIFT                                                                 0x0
+#define MPCC2_MPCC_BG_G_Y__MPCC_BG_G_Y_MASK                                                                   0x00000FFFL
+//MPCC2_MPCC_BG_B_CB
+#define MPCC2_MPCC_BG_B_CB__MPCC_BG_B_CB__SHIFT                                                               0x0
+#define MPCC2_MPCC_BG_B_CB__MPCC_BG_B_CB_MASK                                                                 0x00000FFFL
+//MPCC2_MPCC_MEM_PWR_CTRL
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE__SHIFT                                               0x0
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS__SHIFT                                                 0x2
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE__SHIFT                                            0x4
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE__SHIFT                                               0x8
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE_MASK                                                 0x00000003L
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS_MASK                                                   0x00000004L
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE_MASK                                              0x00000030L
+#define MPCC2_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE_MASK                                                 0x00000300L
+//MPCC2_MPCC_STATUS
+#define MPCC2_MPCC_STATUS__MPCC_IDLE__SHIFT                                                                   0x0
+#define MPCC2_MPCC_STATUS__MPCC_BUSY__SHIFT                                                                   0x1
+#define MPCC2_MPCC_STATUS__MPCC_DISABLED__SHIFT                                                               0x2
+#define MPCC2_MPCC_STATUS__MPCC_IDLE_MASK                                                                     0x00000001L
+#define MPCC2_MPCC_STATUS__MPCC_BUSY_MASK                                                                     0x00000002L
+#define MPCC2_MPCC_STATUS__MPCC_DISABLED_MASK                                                                 0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc3_dispdec
+//MPCC3_MPCC_TOP_SEL
+#define MPCC3_MPCC_TOP_SEL__MPCC_TOP_SEL__SHIFT                                                               0x0
+#define MPCC3_MPCC_TOP_SEL__MPCC_TOP_SEL_MASK                                                                 0x0000000FL
+//MPCC3_MPCC_BOT_SEL
+#define MPCC3_MPCC_BOT_SEL__MPCC_BOT_SEL__SHIFT                                                               0x0
+#define MPCC3_MPCC_BOT_SEL__MPCC_BOT_SEL_MASK                                                                 0x0000000FL
+//MPCC3_MPCC_OPP_ID
+#define MPCC3_MPCC_OPP_ID__MPCC_OPP_ID__SHIFT                                                                 0x0
+#define MPCC3_MPCC_OPP_ID__MPCC_OPP_ID_MASK                                                                   0x0000000FL
+//MPCC3_MPCC_CONTROL
+#define MPCC3_MPCC_CONTROL__MPCC_MODE__SHIFT                                                                  0x0
+#define MPCC3_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE__SHIFT                                                       0x4
+#define MPCC3_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE__SHIFT                                                 0x6
+#define MPCC3_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY__SHIFT                                              0x7
+#define MPCC3_MPCC_CONTROL__MPCC_BG_BPC__SHIFT                                                                0x8
+#define MPCC3_MPCC_CONTROL__MPCC_BOT_GAIN_MODE__SHIFT                                                         0xb
+#define MPCC3_MPCC_CONTROL__MPCC_MODE_MASK                                                                    0x00000003L
+#define MPCC3_MPCC_CONTROL__MPCC_ALPHA_BLND_MODE_MASK                                                         0x00000030L
+#define MPCC3_MPCC_CONTROL__MPCC_ALPHA_MULTIPLIED_MODE_MASK                                                   0x00000040L
+#define MPCC3_MPCC_CONTROL__MPCC_BLND_ACTIVE_OVERLAP_ONLY_MASK                                                0x00000080L
+#define MPCC3_MPCC_CONTROL__MPCC_BG_BPC_MASK                                                                  0x00000700L
+#define MPCC3_MPCC_CONTROL__MPCC_BOT_GAIN_MODE_MASK                                                           0x00000800L
+//MPCC3_MPCC_CONTROL2
+#define MPCC3_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA__SHIFT                                                         0x0
+#define MPCC3_MPCC_CONTROL2__MPCC_GLOBAL_GAIN__SHIFT                                                          0x10
+#define MPCC3_MPCC_CONTROL2__MPCC_GLOBAL_ALPHA_MASK                                                           0x00000FFFL
+#define MPCC3_MPCC_CONTROL2__MPCC_GLOBAL_GAIN_MASK                                                            0x0FFF0000L
+//MPCC3_MPCC_SM_CONTROL
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_EN__SHIFT                                                              0x0
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_MODE__SHIFT                                                            0x1
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT__SHIFT                                                       0x4
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT__SHIFT                                                       0x5
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL__SHIFT                                            0x8
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL__SHIFT                                              0x10
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL__SHIFT                                               0x18
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_EN_MASK                                                                0x00000001L
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_MODE_MASK                                                              0x0000000EL
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FRAME_ALT_MASK                                                         0x00000010L
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FIELD_ALT_MASK                                                         0x00000020L
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_FRAME_POL_MASK                                              0x00000300L
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_FORCE_NEXT_TOP_POL_MASK                                                0x00030000L
+#define MPCC3_MPCC_SM_CONTROL__MPCC_SM_CURRENT_FRAME_POL_MASK                                                 0x01000000L
+//MPCC3_MPCC_UPDATE_LOCK_SEL
+#define MPCC3_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL__SHIFT                                               0x0
+#define MPCC3_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS__SHIFT                                          0x4
+#define MPCC3_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCK_SEL_MASK                                                 0x0000000FL
+#define MPCC3_MPCC_UPDATE_LOCK_SEL__MPCC_UPDATE_LOCKED_STATUS_MASK                                            0x00000070L
+//MPCC3_MPCC_TOP_GAIN
+#define MPCC3_MPCC_TOP_GAIN__MPCC_TOP_GAIN__SHIFT                                                             0x0
+#define MPCC3_MPCC_TOP_GAIN__MPCC_TOP_GAIN_MASK                                                               0x0007FFFFL
+//MPCC3_MPCC_BOT_GAIN_INSIDE
+#define MPCC3_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE__SHIFT                                               0x0
+#define MPCC3_MPCC_BOT_GAIN_INSIDE__MPCC_BOT_GAIN_INSIDE_MASK                                                 0x0007FFFFL
+//MPCC3_MPCC_BOT_GAIN_OUTSIDE
+#define MPCC3_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE__SHIFT                                             0x0
+#define MPCC3_MPCC_BOT_GAIN_OUTSIDE__MPCC_BOT_GAIN_OUTSIDE_MASK                                               0x0007FFFFL
+//MPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL
+#define MPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL__SHIFT                          0x0
+#define MPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT__SHIFT                  0x4
+#define MPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_MASK                            0x00000001L
+#define MPCC3_MPCC_MOVABLE_CM_LOCATION_CONTROL__MPCC_MOVABLE_CM_LOCATION_CNTL_CURRENT_MASK                    0x00000010L
+//MPCC3_MPCC_BG_R_CR
+#define MPCC3_MPCC_BG_R_CR__MPCC_BG_R_CR__SHIFT                                                               0x0
+#define MPCC3_MPCC_BG_R_CR__MPCC_BG_R_CR_MASK                                                                 0x00000FFFL
+//MPCC3_MPCC_BG_G_Y
+#define MPCC3_MPCC_BG_G_Y__MPCC_BG_G_Y__SHIFT                                                                 0x0
+#define MPCC3_MPCC_BG_G_Y__MPCC_BG_G_Y_MASK                                                                   0x00000FFFL
+//MPCC3_MPCC_BG_B_CB
+#define MPCC3_MPCC_BG_B_CB__MPCC_BG_B_CB__SHIFT                                                               0x0
+#define MPCC3_MPCC_BG_B_CB__MPCC_BG_B_CB_MASK                                                                 0x00000FFFL
+//MPCC3_MPCC_MEM_PWR_CTRL
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE__SHIFT                                               0x0
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS__SHIFT                                                 0x2
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE__SHIFT                                            0x4
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE__SHIFT                                               0x8
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_FORCE_MASK                                                 0x00000003L
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_DIS_MASK                                                   0x00000004L
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_LOW_PWR_MODE_MASK                                              0x00000030L
+#define MPCC3_MPCC_MEM_PWR_CTRL__MPCC_OGAM_MEM_PWR_STATE_MASK                                                 0x00000300L
+//MPCC3_MPCC_STATUS
+#define MPCC3_MPCC_STATUS__MPCC_IDLE__SHIFT                                                                   0x0
+#define MPCC3_MPCC_STATUS__MPCC_BUSY__SHIFT                                                                   0x1
+#define MPCC3_MPCC_STATUS__MPCC_DISABLED__SHIFT                                                               0x2
+#define MPCC3_MPCC_STATUS__MPCC_IDLE_MASK                                                                     0x00000001L
+#define MPCC3_MPCC_STATUS__MPCC_BUSY_MASK                                                                     0x00000002L
+#define MPCC3_MPCC_STATUS__MPCC_DISABLED_MASK                                                                 0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam0_dispdec
+//MPCC_OGAM0_MPCC_OGAM_CONTROL
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE__SHIFT                                                   0x0
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT__SHIFT                                                 0x2
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE__SHIFT                                            0x3
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT__SHIFT                                           0x7
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT__SHIFT                                         0x9
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_MASK                                                     0x00000003L
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_MASK                                                   0x00000004L
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE_MASK                                              0x00000008L
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT_MASK                                             0x00000180L
+#define MPCC_OGAM0_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT_MASK                                           0x00000200L
+//MPCC_OGAM0_MPCC_OGAM_LUT_INDEX
+#define MPCC_OGAM0_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX__SHIFT                                            0x0
+#define MPCC_OGAM0_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX_MASK                                              0x000001FFL
+//MPCC_OGAM0_MPCC_OGAM_LUT_DATA
+#define MPCC_OGAM0_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA__SHIFT                                              0x0
+#define MPCC_OGAM0_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA_MASK                                                0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK__SHIFT                               0x0
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL__SHIFT                                 0x3
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG__SHIFT                                       0x5
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL__SHIFT                                       0x6
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE__SHIFT                                    0x7
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK_MASK                                 0x00000007L
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL_MASK                                   0x00000018L
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG_MASK                                         0x00000020L
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL_MASK                                         0x00000040L
+#define MPCC_OGAM0_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE_MASK                                      0x00000080L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_B
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_G
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_R
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT__SHIFT                          0x0
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT_MASK                            0x00000001L
+//MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE__SHIFT                                        0x0
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT__SHIFT                                0x7
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_MASK                                          0x00000003L
+#define MPCC_OGAM0_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT_MASK                                  0x00000180L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B__SHIFT                                   0x0
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B__SHIFT                                   0x10
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM0_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B_MASK                                     0xFFFF0000L
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam1_dispdec
+//MPCC_OGAM1_MPCC_OGAM_CONTROL
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE__SHIFT                                                   0x0
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT__SHIFT                                                 0x2
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE__SHIFT                                            0x3
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT__SHIFT                                           0x7
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT__SHIFT                                         0x9
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_MASK                                                     0x00000003L
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_MASK                                                   0x00000004L
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE_MASK                                              0x00000008L
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT_MASK                                             0x00000180L
+#define MPCC_OGAM1_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT_MASK                                           0x00000200L
+//MPCC_OGAM1_MPCC_OGAM_LUT_INDEX
+#define MPCC_OGAM1_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX__SHIFT                                            0x0
+#define MPCC_OGAM1_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX_MASK                                              0x000001FFL
+//MPCC_OGAM1_MPCC_OGAM_LUT_DATA
+#define MPCC_OGAM1_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA__SHIFT                                              0x0
+#define MPCC_OGAM1_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA_MASK                                                0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK__SHIFT                               0x0
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL__SHIFT                                 0x3
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG__SHIFT                                       0x5
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL__SHIFT                                       0x6
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE__SHIFT                                    0x7
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK_MASK                                 0x00000007L
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL_MASK                                   0x00000018L
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG_MASK                                         0x00000020L
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL_MASK                                         0x00000040L
+#define MPCC_OGAM1_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE_MASK                                      0x00000080L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_B
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_G
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_R
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM1_MPCC_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT__SHIFT                          0x0
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT_MASK                            0x00000001L
+//MPCC_OGAM1_MPCC_GAMUT_REMAP_MODE
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE__SHIFT                                        0x0
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT__SHIFT                                0x7
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_MASK                                          0x00000003L
+#define MPCC_OGAM1_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT_MASK                                  0x00000180L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B__SHIFT                                   0x0
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B__SHIFT                                   0x10
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM1_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B_MASK                                     0xFFFF0000L
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam2_dispdec
+//MPCC_OGAM2_MPCC_OGAM_CONTROL
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE__SHIFT                                                   0x0
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT__SHIFT                                                 0x2
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE__SHIFT                                            0x3
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT__SHIFT                                           0x7
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT__SHIFT                                         0x9
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_MASK                                                     0x00000003L
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_MASK                                                   0x00000004L
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE_MASK                                              0x00000008L
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT_MASK                                             0x00000180L
+#define MPCC_OGAM2_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT_MASK                                           0x00000200L
+//MPCC_OGAM2_MPCC_OGAM_LUT_INDEX
+#define MPCC_OGAM2_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX__SHIFT                                            0x0
+#define MPCC_OGAM2_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX_MASK                                              0x000001FFL
+//MPCC_OGAM2_MPCC_OGAM_LUT_DATA
+#define MPCC_OGAM2_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA__SHIFT                                              0x0
+#define MPCC_OGAM2_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA_MASK                                                0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK__SHIFT                               0x0
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL__SHIFT                                 0x3
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG__SHIFT                                       0x5
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL__SHIFT                                       0x6
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE__SHIFT                                    0x7
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK_MASK                                 0x00000007L
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL_MASK                                   0x00000018L
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG_MASK                                         0x00000020L
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL_MASK                                         0x00000040L
+#define MPCC_OGAM2_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE_MASK                                      0x00000080L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_B
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_G
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_R
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM2_MPCC_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT__SHIFT                          0x0
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT_MASK                            0x00000001L
+//MPCC_OGAM2_MPCC_GAMUT_REMAP_MODE
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE__SHIFT                                        0x0
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT__SHIFT                                0x7
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_MASK                                          0x00000003L
+#define MPCC_OGAM2_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT_MASK                                  0x00000180L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B__SHIFT                                   0x0
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B__SHIFT                                   0x10
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM2_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B_MASK                                     0xFFFF0000L
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam3_dispdec
+//MPCC_OGAM3_MPCC_OGAM_CONTROL
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE__SHIFT                                                   0x0
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT__SHIFT                                                 0x2
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE__SHIFT                                            0x3
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT__SHIFT                                           0x7
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT__SHIFT                                         0x9
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_MASK                                                     0x00000003L
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_MASK                                                   0x00000004L
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_PWL_DISABLE_MASK                                              0x00000008L
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_MODE_CURRENT_MASK                                             0x00000180L
+#define MPCC_OGAM3_MPCC_OGAM_CONTROL__MPCC_OGAM_SELECT_CURRENT_MASK                                           0x00000200L
+//MPCC_OGAM3_MPCC_OGAM_LUT_INDEX
+#define MPCC_OGAM3_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX__SHIFT                                            0x0
+#define MPCC_OGAM3_MPCC_OGAM_LUT_INDEX__MPCC_OGAM_LUT_INDEX_MASK                                              0x000001FFL
+//MPCC_OGAM3_MPCC_OGAM_LUT_DATA
+#define MPCC_OGAM3_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA__SHIFT                                              0x0
+#define MPCC_OGAM3_MPCC_OGAM_LUT_DATA__MPCC_OGAM_LUT_DATA_MASK                                                0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK__SHIFT                               0x0
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL__SHIFT                                 0x3
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG__SHIFT                                       0x5
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL__SHIFT                                       0x6
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE__SHIFT                                    0x7
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_WRITE_COLOR_MASK_MASK                                 0x00000007L
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_COLOR_SEL_MASK                                   0x00000018L
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_READ_DBG_MASK                                         0x00000020L
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_HOST_SEL_MASK                                         0x00000040L
+#define MPCC_OGAM3_MPCC_OGAM_LUT_CONTROL__MPCC_OGAM_LUT_CONFIG_MODE_MASK                                      0x00000080L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_SLOPE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_B__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_G__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_START_BASE_CNTL_R__MPCC_OGAM_RAMA_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R__MPCC_OGAM_RAMA_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R__MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_B__MPCC_OGAM_RAMA_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_G__MPCC_OGAM_RAMA_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_OFFSET_R__MPCC_OGAM_RAMA_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1__MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3__MPCC_OGAM_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5__MPCC_OGAM_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7__MPCC_OGAM_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9__MPCC_OGAM_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11__MPCC_OGAM_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13__MPCC_OGAM_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15__MPCC_OGAM_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17__MPCC_OGAM_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19__MPCC_OGAM_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21__MPCC_OGAM_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23__MPCC_OGAM_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25__MPCC_OGAM_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27__MPCC_OGAM_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29__MPCC_OGAM_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31__MPCC_OGAM_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33__MPCC_OGAM_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_B_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_G_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_G_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R__SHIFT                      0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT              0x14
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_R_MASK                        0x0003FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_R_MASK                0x07F00000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_B_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_G_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R__SHIFT          0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_SLOPE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_SLOPE_R_MASK            0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_B__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_B_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_G__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_G_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R__SHIFT            0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_START_BASE_CNTL_R__MPCC_OGAM_RAMB_EXP_REGION_START_BASE_R_MASK              0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_B_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_G_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_G_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_G_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R__MPCC_OGAM_RAMB_EXP_REGION_END_BASE_R_MASK                      0x0003FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R__SHIFT                         0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R__SHIFT                   0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_R_MASK                           0x0000FFFFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R__MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_R_MASK                     0xFFFF0000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_B
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_B__MPCC_OGAM_RAMB_OFFSET_B_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_G
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_G__MPCC_OGAM_RAMB_OFFSET_G_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_R
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R__SHIFT                                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_OFFSET_R__MPCC_OGAM_RAMB_OFFSET_R_MASK                                      0x0007FFFFL
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1__MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3__MPCC_OGAM_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5__MPCC_OGAM_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7__MPCC_OGAM_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT                    0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT                  0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT                    0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT                  0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_LUT_OFFSET_MASK                      0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK                    0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_LUT_OFFSET_MASK                      0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9__MPCC_OGAM_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK                    0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11__MPCC_OGAM_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13__MPCC_OGAM_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15__MPCC_OGAM_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17__MPCC_OGAM_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19__MPCC_OGAM_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21__MPCC_OGAM_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23__MPCC_OGAM_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25__MPCC_OGAM_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27__MPCC_OGAM_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29__MPCC_OGAM_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31__MPCC_OGAM_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT                 0x0
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT               0xc
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT                 0x10
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT               0x1c
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_LUT_OFFSET_MASK                   0x000001FFL
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK                 0x00007000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_LUT_OFFSET_MASK                   0x01FF0000L
+#define MPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33__MPCC_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK                 0x70000000L
+//MPCC_OGAM3_MPCC_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT__SHIFT                          0x0
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_COEF_FORMAT__MPCC_GAMUT_REMAP_COEF_FORMAT_MASK                            0x00000001L
+//MPCC_OGAM3_MPCC_GAMUT_REMAP_MODE
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE__SHIFT                                        0x0
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT__SHIFT                                0x7
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_MASK                                          0x00000003L
+#define MPCC_OGAM3_MPCC_GAMUT_REMAP_MODE__MPCC_GAMUT_REMAP_MODE_CURRENT_MASK                                  0x00000180L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C11_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_A__MPCC_GAMUT_REMAP_C12_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C13_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_A__MPCC_GAMUT_REMAP_C14_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C21_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_A__MPCC_GAMUT_REMAP_C22_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C23_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_A__MPCC_GAMUT_REMAP_C24_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C31_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_A__MPCC_GAMUT_REMAP_C32_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C33_A_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_A__MPCC_GAMUT_REMAP_C34_A_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C11_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C11_C12_B__MPCC_GAMUT_REMAP_C12_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C13_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C13_C14_B__MPCC_GAMUT_REMAP_C14_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C21_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C21_C22_B__MPCC_GAMUT_REMAP_C22_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C23_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C23_C24_B__MPCC_GAMUT_REMAP_C24_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C31_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C31_C32_B__MPCC_GAMUT_REMAP_C32_B_MASK                                     0xFFFF0000L
+//MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B__SHIFT                                   0x0
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B__SHIFT                                   0x10
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C33_B_MASK                                     0x0000FFFFL
+#define MPCC_OGAM3_MPC_GAMUT_REMAP_C33_C34_B__MPCC_GAMUT_REMAP_C34_B_MASK                                     0xFFFF0000L
+
+
+// addressBlock: dce_dc_mpc_mpc_ocsc_dispdec
+//MPC_OUT0_MUX
+#define MPC_OUT0_MUX__MPC_OUT_MUX__SHIFT                                                                      0x0
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR__SHIFT                                                  0x5
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK__SHIFT                                                   0x7
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_DISABLE__SHIFT                                                     0x8
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL__SHIFT                                                             0x9
+#define MPC_OUT0_MUX__MPC_OUT_FLOW_CONTROL_MODE__SHIFT                                                        0xa
+#define MPC_OUT0_MUX__MPC_OUT_FLOW_CONTROL_COUNT__SHIFT                                                       0xb
+#define MPC_OUT0_MUX__MPC_OUT_MUX_MASK                                                                        0x0000000FL
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR_MASK                                                    0x00000020L
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK_MASK                                                     0x00000080L
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_DISABLE_MASK                                                       0x00000100L
+#define MPC_OUT0_MUX__MPC_OUT_RATE_CONTROL_MASK                                                               0x00000200L
+#define MPC_OUT0_MUX__MPC_OUT_FLOW_CONTROL_MODE_MASK                                                          0x00000400L
+#define MPC_OUT0_MUX__MPC_OUT_FLOW_CONTROL_COUNT_MASK                                                         0x007FF800L
+//MPC_OUT0_DENORM_CONTROL
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR__SHIFT                                         0x0
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR__SHIFT                                         0xc
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_MODE__SHIFT                                                   0x18
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR_MASK                                           0x00000FFFL
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR_MASK                                           0x00FFF000L
+#define MPC_OUT0_DENORM_CONTROL__MPC_OUT_DENORM_MODE_MASK                                                     0x07000000L
+//MPC_OUT0_DENORM_CLAMP_G_Y
+#define MPC_OUT0_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y__SHIFT                                        0x0
+#define MPC_OUT0_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y__SHIFT                                        0xc
+#define MPC_OUT0_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y_MASK                                          0x00000FFFL
+#define MPC_OUT0_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y_MASK                                          0x00FFF000L
+//MPC_OUT0_DENORM_CLAMP_B_CB
+#define MPC_OUT0_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB__SHIFT                                      0x0
+#define MPC_OUT0_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB__SHIFT                                      0xc
+#define MPC_OUT0_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB_MASK                                        0x00000FFFL
+#define MPC_OUT0_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB_MASK                                        0x00FFF000L
+//MPC_OUT1_MUX
+#define MPC_OUT1_MUX__MPC_OUT_MUX__SHIFT                                                                      0x0
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR__SHIFT                                                  0x5
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK__SHIFT                                                   0x7
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_DISABLE__SHIFT                                                     0x8
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL__SHIFT                                                             0x9
+#define MPC_OUT1_MUX__MPC_OUT_FLOW_CONTROL_MODE__SHIFT                                                        0xa
+#define MPC_OUT1_MUX__MPC_OUT_FLOW_CONTROL_COUNT__SHIFT                                                       0xb
+#define MPC_OUT1_MUX__MPC_OUT_MUX_MASK                                                                        0x0000000FL
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR_MASK                                                    0x00000020L
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK_MASK                                                     0x00000080L
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_DISABLE_MASK                                                       0x00000100L
+#define MPC_OUT1_MUX__MPC_OUT_RATE_CONTROL_MASK                                                               0x00000200L
+#define MPC_OUT1_MUX__MPC_OUT_FLOW_CONTROL_MODE_MASK                                                          0x00000400L
+#define MPC_OUT1_MUX__MPC_OUT_FLOW_CONTROL_COUNT_MASK                                                         0x007FF800L
+//MPC_OUT1_DENORM_CONTROL
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR__SHIFT                                         0x0
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR__SHIFT                                         0xc
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_MODE__SHIFT                                                   0x18
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR_MASK                                           0x00000FFFL
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR_MASK                                           0x00FFF000L
+#define MPC_OUT1_DENORM_CONTROL__MPC_OUT_DENORM_MODE_MASK                                                     0x07000000L
+//MPC_OUT1_DENORM_CLAMP_G_Y
+#define MPC_OUT1_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y__SHIFT                                        0x0
+#define MPC_OUT1_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y__SHIFT                                        0xc
+#define MPC_OUT1_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y_MASK                                          0x00000FFFL
+#define MPC_OUT1_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y_MASK                                          0x00FFF000L
+//MPC_OUT1_DENORM_CLAMP_B_CB
+#define MPC_OUT1_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB__SHIFT                                      0x0
+#define MPC_OUT1_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB__SHIFT                                      0xc
+#define MPC_OUT1_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB_MASK                                        0x00000FFFL
+#define MPC_OUT1_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB_MASK                                        0x00FFF000L
+//MPC_OUT2_MUX
+#define MPC_OUT2_MUX__MPC_OUT_MUX__SHIFT                                                                      0x0
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR__SHIFT                                                  0x5
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK__SHIFT                                                   0x7
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_DISABLE__SHIFT                                                     0x8
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL__SHIFT                                                             0x9
+#define MPC_OUT2_MUX__MPC_OUT_FLOW_CONTROL_MODE__SHIFT                                                        0xa
+#define MPC_OUT2_MUX__MPC_OUT_FLOW_CONTROL_COUNT__SHIFT                                                       0xb
+#define MPC_OUT2_MUX__MPC_OUT_MUX_MASK                                                                        0x0000000FL
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR_MASK                                                    0x00000020L
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK_MASK                                                     0x00000080L
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_DISABLE_MASK                                                       0x00000100L
+#define MPC_OUT2_MUX__MPC_OUT_RATE_CONTROL_MASK                                                               0x00000200L
+#define MPC_OUT2_MUX__MPC_OUT_FLOW_CONTROL_MODE_MASK                                                          0x00000400L
+#define MPC_OUT2_MUX__MPC_OUT_FLOW_CONTROL_COUNT_MASK                                                         0x007FF800L
+//MPC_OUT2_DENORM_CONTROL
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR__SHIFT                                         0x0
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR__SHIFT                                         0xc
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_MODE__SHIFT                                                   0x18
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR_MASK                                           0x00000FFFL
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR_MASK                                           0x00FFF000L
+#define MPC_OUT2_DENORM_CONTROL__MPC_OUT_DENORM_MODE_MASK                                                     0x07000000L
+//MPC_OUT2_DENORM_CLAMP_G_Y
+#define MPC_OUT2_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y__SHIFT                                        0x0
+#define MPC_OUT2_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y__SHIFT                                        0xc
+#define MPC_OUT2_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y_MASK                                          0x00000FFFL
+#define MPC_OUT2_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y_MASK                                          0x00FFF000L
+//MPC_OUT2_DENORM_CLAMP_B_CB
+#define MPC_OUT2_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB__SHIFT                                      0x0
+#define MPC_OUT2_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB__SHIFT                                      0xc
+#define MPC_OUT2_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB_MASK                                        0x00000FFFL
+#define MPC_OUT2_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB_MASK                                        0x00FFF000L
+//MPC_OUT3_MUX
+#define MPC_OUT3_MUX__MPC_OUT_MUX__SHIFT                                                                      0x0
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR__SHIFT                                                  0x5
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK__SHIFT                                                   0x7
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_DISABLE__SHIFT                                                     0x8
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL__SHIFT                                                             0x9
+#define MPC_OUT3_MUX__MPC_OUT_FLOW_CONTROL_MODE__SHIFT                                                        0xa
+#define MPC_OUT3_MUX__MPC_OUT_FLOW_CONTROL_COUNT__SHIFT                                                       0xb
+#define MPC_OUT3_MUX__MPC_OUT_MUX_MASK                                                                        0x0000000FL
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_OVFL_ERROR_MASK                                                    0x00000020L
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_ERROR_ACK_MASK                                                     0x00000080L
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_DISABLE_MASK                                                       0x00000100L
+#define MPC_OUT3_MUX__MPC_OUT_RATE_CONTROL_MASK                                                               0x00000200L
+#define MPC_OUT3_MUX__MPC_OUT_FLOW_CONTROL_MODE_MASK                                                          0x00000400L
+#define MPC_OUT3_MUX__MPC_OUT_FLOW_CONTROL_COUNT_MASK                                                         0x007FF800L
+//MPC_OUT3_DENORM_CONTROL
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR__SHIFT                                         0x0
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR__SHIFT                                         0xc
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_MODE__SHIFT                                                   0x18
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MIN_R_CR_MASK                                           0x00000FFFL
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_CLAMP_MAX_R_CR_MASK                                           0x00FFF000L
+#define MPC_OUT3_DENORM_CONTROL__MPC_OUT_DENORM_MODE_MASK                                                     0x07000000L
+//MPC_OUT3_DENORM_CLAMP_G_Y
+#define MPC_OUT3_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y__SHIFT                                        0x0
+#define MPC_OUT3_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y__SHIFT                                        0xc
+#define MPC_OUT3_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MIN_G_Y_MASK                                          0x00000FFFL
+#define MPC_OUT3_DENORM_CLAMP_G_Y__MPC_OUT_DENORM_CLAMP_MAX_G_Y_MASK                                          0x00FFF000L
+//MPC_OUT3_DENORM_CLAMP_B_CB
+#define MPC_OUT3_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB__SHIFT                                      0x0
+#define MPC_OUT3_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB__SHIFT                                      0xc
+#define MPC_OUT3_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MIN_B_CB_MASK                                        0x00000FFFL
+#define MPC_OUT3_DENORM_CLAMP_B_CB__MPC_OUT_DENORM_CLAMP_MAX_B_CB_MASK                                        0x00FFF000L
+//MPC_OUT_CSC_COEF_FORMAT
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC0_COEF_FORMAT__SHIFT                                                 0x0
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC1_COEF_FORMAT__SHIFT                                                 0x1
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC2_COEF_FORMAT__SHIFT                                                 0x2
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC3_COEF_FORMAT__SHIFT                                                 0x3
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC0_COEF_FORMAT_MASK                                                   0x00000001L
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC1_COEF_FORMAT_MASK                                                   0x00000002L
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC2_COEF_FORMAT_MASK                                                   0x00000004L
+#define MPC_OUT_CSC_COEF_FORMAT__MPC_OCSC3_COEF_FORMAT_MASK                                                   0x00000008L
+//MPC_OUT0_CSC_MODE
+#define MPC_OUT0_CSC_MODE__MPC_OCSC_MODE__SHIFT                                                               0x0
+#define MPC_OUT0_CSC_MODE__MPC_OCSC_MODE_CURRENT__SHIFT                                                       0x7
+#define MPC_OUT0_CSC_MODE__MPC_OCSC_MODE_MASK                                                                 0x00000003L
+#define MPC_OUT0_CSC_MODE__MPC_OCSC_MODE_CURRENT_MASK                                                         0x00000180L
+//MPC_OUT0_CSC_C11_C12_A
+#define MPC_OUT0_CSC_C11_C12_A__MPC_OCSC_C11_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C11_C12_A__MPC_OCSC_C12_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C11_C12_A__MPC_OCSC_C11_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C11_C12_A__MPC_OCSC_C12_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C13_C14_A
+#define MPC_OUT0_CSC_C13_C14_A__MPC_OCSC_C13_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C13_C14_A__MPC_OCSC_C14_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C13_C14_A__MPC_OCSC_C13_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C13_C14_A__MPC_OCSC_C14_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C21_C22_A
+#define MPC_OUT0_CSC_C21_C22_A__MPC_OCSC_C21_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C21_C22_A__MPC_OCSC_C22_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C21_C22_A__MPC_OCSC_C21_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C21_C22_A__MPC_OCSC_C22_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C23_C24_A
+#define MPC_OUT0_CSC_C23_C24_A__MPC_OCSC_C23_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C23_C24_A__MPC_OCSC_C24_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C23_C24_A__MPC_OCSC_C23_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C23_C24_A__MPC_OCSC_C24_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C31_C32_A
+#define MPC_OUT0_CSC_C31_C32_A__MPC_OCSC_C31_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C31_C32_A__MPC_OCSC_C32_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C31_C32_A__MPC_OCSC_C31_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C31_C32_A__MPC_OCSC_C32_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C33_C34_A
+#define MPC_OUT0_CSC_C33_C34_A__MPC_OCSC_C33_A__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C33_C34_A__MPC_OCSC_C34_A__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C33_C34_A__MPC_OCSC_C33_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C33_C34_A__MPC_OCSC_C34_A_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C11_C12_B
+#define MPC_OUT0_CSC_C11_C12_B__MPC_OCSC_C11_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C11_C12_B__MPC_OCSC_C12_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C11_C12_B__MPC_OCSC_C11_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C11_C12_B__MPC_OCSC_C12_B_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C13_C14_B
+#define MPC_OUT0_CSC_C13_C14_B__MPC_OCSC_C13_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C13_C14_B__MPC_OCSC_C14_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C13_C14_B__MPC_OCSC_C13_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C13_C14_B__MPC_OCSC_C14_B_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C21_C22_B
+#define MPC_OUT0_CSC_C21_C22_B__MPC_OCSC_C21_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C21_C22_B__MPC_OCSC_C22_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C21_C22_B__MPC_OCSC_C21_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C21_C22_B__MPC_OCSC_C22_B_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C23_C24_B
+#define MPC_OUT0_CSC_C23_C24_B__MPC_OCSC_C23_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C23_C24_B__MPC_OCSC_C24_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C23_C24_B__MPC_OCSC_C23_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C23_C24_B__MPC_OCSC_C24_B_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C31_C32_B
+#define MPC_OUT0_CSC_C31_C32_B__MPC_OCSC_C31_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C31_C32_B__MPC_OCSC_C32_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C31_C32_B__MPC_OCSC_C31_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C31_C32_B__MPC_OCSC_C32_B_MASK                                                           0xFFFF0000L
+//MPC_OUT0_CSC_C33_C34_B
+#define MPC_OUT0_CSC_C33_C34_B__MPC_OCSC_C33_B__SHIFT                                                         0x0
+#define MPC_OUT0_CSC_C33_C34_B__MPC_OCSC_C34_B__SHIFT                                                         0x10
+#define MPC_OUT0_CSC_C33_C34_B__MPC_OCSC_C33_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT0_CSC_C33_C34_B__MPC_OCSC_C34_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_MODE
+#define MPC_OUT1_CSC_MODE__MPC_OCSC_MODE__SHIFT                                                               0x0
+#define MPC_OUT1_CSC_MODE__MPC_OCSC_MODE_CURRENT__SHIFT                                                       0x7
+#define MPC_OUT1_CSC_MODE__MPC_OCSC_MODE_MASK                                                                 0x00000003L
+#define MPC_OUT1_CSC_MODE__MPC_OCSC_MODE_CURRENT_MASK                                                         0x00000180L
+//MPC_OUT1_CSC_C11_C12_A
+#define MPC_OUT1_CSC_C11_C12_A__MPC_OCSC_C11_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C11_C12_A__MPC_OCSC_C12_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C11_C12_A__MPC_OCSC_C11_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C11_C12_A__MPC_OCSC_C12_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C13_C14_A
+#define MPC_OUT1_CSC_C13_C14_A__MPC_OCSC_C13_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C13_C14_A__MPC_OCSC_C14_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C13_C14_A__MPC_OCSC_C13_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C13_C14_A__MPC_OCSC_C14_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C21_C22_A
+#define MPC_OUT1_CSC_C21_C22_A__MPC_OCSC_C21_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C21_C22_A__MPC_OCSC_C22_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C21_C22_A__MPC_OCSC_C21_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C21_C22_A__MPC_OCSC_C22_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C23_C24_A
+#define MPC_OUT1_CSC_C23_C24_A__MPC_OCSC_C23_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C23_C24_A__MPC_OCSC_C24_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C23_C24_A__MPC_OCSC_C23_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C23_C24_A__MPC_OCSC_C24_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C31_C32_A
+#define MPC_OUT1_CSC_C31_C32_A__MPC_OCSC_C31_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C31_C32_A__MPC_OCSC_C32_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C31_C32_A__MPC_OCSC_C31_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C31_C32_A__MPC_OCSC_C32_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C33_C34_A
+#define MPC_OUT1_CSC_C33_C34_A__MPC_OCSC_C33_A__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C33_C34_A__MPC_OCSC_C34_A__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C33_C34_A__MPC_OCSC_C33_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C33_C34_A__MPC_OCSC_C34_A_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C11_C12_B
+#define MPC_OUT1_CSC_C11_C12_B__MPC_OCSC_C11_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C11_C12_B__MPC_OCSC_C12_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C11_C12_B__MPC_OCSC_C11_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C11_C12_B__MPC_OCSC_C12_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C13_C14_B
+#define MPC_OUT1_CSC_C13_C14_B__MPC_OCSC_C13_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C13_C14_B__MPC_OCSC_C14_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C13_C14_B__MPC_OCSC_C13_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C13_C14_B__MPC_OCSC_C14_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C21_C22_B
+#define MPC_OUT1_CSC_C21_C22_B__MPC_OCSC_C21_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C21_C22_B__MPC_OCSC_C22_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C21_C22_B__MPC_OCSC_C21_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C21_C22_B__MPC_OCSC_C22_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C23_C24_B
+#define MPC_OUT1_CSC_C23_C24_B__MPC_OCSC_C23_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C23_C24_B__MPC_OCSC_C24_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C23_C24_B__MPC_OCSC_C23_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C23_C24_B__MPC_OCSC_C24_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C31_C32_B
+#define MPC_OUT1_CSC_C31_C32_B__MPC_OCSC_C31_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C31_C32_B__MPC_OCSC_C32_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C31_C32_B__MPC_OCSC_C31_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C31_C32_B__MPC_OCSC_C32_B_MASK                                                           0xFFFF0000L
+//MPC_OUT1_CSC_C33_C34_B
+#define MPC_OUT1_CSC_C33_C34_B__MPC_OCSC_C33_B__SHIFT                                                         0x0
+#define MPC_OUT1_CSC_C33_C34_B__MPC_OCSC_C34_B__SHIFT                                                         0x10
+#define MPC_OUT1_CSC_C33_C34_B__MPC_OCSC_C33_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT1_CSC_C33_C34_B__MPC_OCSC_C34_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_MODE
+#define MPC_OUT2_CSC_MODE__MPC_OCSC_MODE__SHIFT                                                               0x0
+#define MPC_OUT2_CSC_MODE__MPC_OCSC_MODE_CURRENT__SHIFT                                                       0x7
+#define MPC_OUT2_CSC_MODE__MPC_OCSC_MODE_MASK                                                                 0x00000003L
+#define MPC_OUT2_CSC_MODE__MPC_OCSC_MODE_CURRENT_MASK                                                         0x00000180L
+//MPC_OUT2_CSC_C11_C12_A
+#define MPC_OUT2_CSC_C11_C12_A__MPC_OCSC_C11_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C11_C12_A__MPC_OCSC_C12_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C11_C12_A__MPC_OCSC_C11_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C11_C12_A__MPC_OCSC_C12_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C13_C14_A
+#define MPC_OUT2_CSC_C13_C14_A__MPC_OCSC_C13_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C13_C14_A__MPC_OCSC_C14_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C13_C14_A__MPC_OCSC_C13_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C13_C14_A__MPC_OCSC_C14_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C21_C22_A
+#define MPC_OUT2_CSC_C21_C22_A__MPC_OCSC_C21_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C21_C22_A__MPC_OCSC_C22_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C21_C22_A__MPC_OCSC_C21_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C21_C22_A__MPC_OCSC_C22_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C23_C24_A
+#define MPC_OUT2_CSC_C23_C24_A__MPC_OCSC_C23_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C23_C24_A__MPC_OCSC_C24_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C23_C24_A__MPC_OCSC_C23_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C23_C24_A__MPC_OCSC_C24_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C31_C32_A
+#define MPC_OUT2_CSC_C31_C32_A__MPC_OCSC_C31_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C31_C32_A__MPC_OCSC_C32_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C31_C32_A__MPC_OCSC_C31_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C31_C32_A__MPC_OCSC_C32_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C33_C34_A
+#define MPC_OUT2_CSC_C33_C34_A__MPC_OCSC_C33_A__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C33_C34_A__MPC_OCSC_C34_A__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C33_C34_A__MPC_OCSC_C33_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C33_C34_A__MPC_OCSC_C34_A_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C11_C12_B
+#define MPC_OUT2_CSC_C11_C12_B__MPC_OCSC_C11_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C11_C12_B__MPC_OCSC_C12_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C11_C12_B__MPC_OCSC_C11_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C11_C12_B__MPC_OCSC_C12_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C13_C14_B
+#define MPC_OUT2_CSC_C13_C14_B__MPC_OCSC_C13_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C13_C14_B__MPC_OCSC_C14_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C13_C14_B__MPC_OCSC_C13_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C13_C14_B__MPC_OCSC_C14_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C21_C22_B
+#define MPC_OUT2_CSC_C21_C22_B__MPC_OCSC_C21_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C21_C22_B__MPC_OCSC_C22_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C21_C22_B__MPC_OCSC_C21_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C21_C22_B__MPC_OCSC_C22_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C23_C24_B
+#define MPC_OUT2_CSC_C23_C24_B__MPC_OCSC_C23_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C23_C24_B__MPC_OCSC_C24_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C23_C24_B__MPC_OCSC_C23_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C23_C24_B__MPC_OCSC_C24_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C31_C32_B
+#define MPC_OUT2_CSC_C31_C32_B__MPC_OCSC_C31_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C31_C32_B__MPC_OCSC_C32_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C31_C32_B__MPC_OCSC_C31_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C31_C32_B__MPC_OCSC_C32_B_MASK                                                           0xFFFF0000L
+//MPC_OUT2_CSC_C33_C34_B
+#define MPC_OUT2_CSC_C33_C34_B__MPC_OCSC_C33_B__SHIFT                                                         0x0
+#define MPC_OUT2_CSC_C33_C34_B__MPC_OCSC_C34_B__SHIFT                                                         0x10
+#define MPC_OUT2_CSC_C33_C34_B__MPC_OCSC_C33_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT2_CSC_C33_C34_B__MPC_OCSC_C34_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_MODE
+#define MPC_OUT3_CSC_MODE__MPC_OCSC_MODE__SHIFT                                                               0x0
+#define MPC_OUT3_CSC_MODE__MPC_OCSC_MODE_CURRENT__SHIFT                                                       0x7
+#define MPC_OUT3_CSC_MODE__MPC_OCSC_MODE_MASK                                                                 0x00000003L
+#define MPC_OUT3_CSC_MODE__MPC_OCSC_MODE_CURRENT_MASK                                                         0x00000180L
+//MPC_OUT3_CSC_C11_C12_A
+#define MPC_OUT3_CSC_C11_C12_A__MPC_OCSC_C11_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C11_C12_A__MPC_OCSC_C12_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C11_C12_A__MPC_OCSC_C11_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C11_C12_A__MPC_OCSC_C12_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C13_C14_A
+#define MPC_OUT3_CSC_C13_C14_A__MPC_OCSC_C13_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C13_C14_A__MPC_OCSC_C14_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C13_C14_A__MPC_OCSC_C13_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C13_C14_A__MPC_OCSC_C14_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C21_C22_A
+#define MPC_OUT3_CSC_C21_C22_A__MPC_OCSC_C21_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C21_C22_A__MPC_OCSC_C22_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C21_C22_A__MPC_OCSC_C21_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C21_C22_A__MPC_OCSC_C22_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C23_C24_A
+#define MPC_OUT3_CSC_C23_C24_A__MPC_OCSC_C23_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C23_C24_A__MPC_OCSC_C24_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C23_C24_A__MPC_OCSC_C23_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C23_C24_A__MPC_OCSC_C24_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C31_C32_A
+#define MPC_OUT3_CSC_C31_C32_A__MPC_OCSC_C31_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C31_C32_A__MPC_OCSC_C32_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C31_C32_A__MPC_OCSC_C31_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C31_C32_A__MPC_OCSC_C32_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C33_C34_A
+#define MPC_OUT3_CSC_C33_C34_A__MPC_OCSC_C33_A__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C33_C34_A__MPC_OCSC_C34_A__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C33_C34_A__MPC_OCSC_C33_A_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C33_C34_A__MPC_OCSC_C34_A_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C11_C12_B
+#define MPC_OUT3_CSC_C11_C12_B__MPC_OCSC_C11_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C11_C12_B__MPC_OCSC_C12_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C11_C12_B__MPC_OCSC_C11_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C11_C12_B__MPC_OCSC_C12_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C13_C14_B
+#define MPC_OUT3_CSC_C13_C14_B__MPC_OCSC_C13_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C13_C14_B__MPC_OCSC_C14_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C13_C14_B__MPC_OCSC_C13_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C13_C14_B__MPC_OCSC_C14_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C21_C22_B
+#define MPC_OUT3_CSC_C21_C22_B__MPC_OCSC_C21_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C21_C22_B__MPC_OCSC_C22_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C21_C22_B__MPC_OCSC_C21_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C21_C22_B__MPC_OCSC_C22_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C23_C24_B
+#define MPC_OUT3_CSC_C23_C24_B__MPC_OCSC_C23_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C23_C24_B__MPC_OCSC_C24_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C23_C24_B__MPC_OCSC_C23_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C23_C24_B__MPC_OCSC_C24_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C31_C32_B
+#define MPC_OUT3_CSC_C31_C32_B__MPC_OCSC_C31_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C31_C32_B__MPC_OCSC_C32_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C31_C32_B__MPC_OCSC_C31_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C31_C32_B__MPC_OCSC_C32_B_MASK                                                           0xFFFF0000L
+//MPC_OUT3_CSC_C33_C34_B
+#define MPC_OUT3_CSC_C33_C34_B__MPC_OCSC_C33_B__SHIFT                                                         0x0
+#define MPC_OUT3_CSC_C33_C34_B__MPC_OCSC_C34_B__SHIFT                                                         0x10
+#define MPC_OUT3_CSC_C33_C34_B__MPC_OCSC_C33_B_MASK                                                           0x0000FFFFL
+#define MPC_OUT3_CSC_C33_C34_B__MPC_OCSC_C34_B_MASK                                                           0xFFFF0000L
+//MPC_OCSC_TEST_DEBUG_INDEX
+#define MPC_OCSC_TEST_DEBUG_INDEX__MPC_OCSC_TEST_DEBUG_INDEX__SHIFT                                           0x0
+#define MPC_OCSC_TEST_DEBUG_INDEX__MPC_OCSC_TEST_DEBUG_WRITE_EN__SHIFT                                        0x8
+#define MPC_OCSC_TEST_DEBUG_INDEX__MPC_OCSC_TEST_DEBUG_INDEX_MASK                                             0x000000FFL
+#define MPC_OCSC_TEST_DEBUG_INDEX__MPC_OCSC_TEST_DEBUG_WRITE_EN_MASK                                          0x00000100L
+//MPC_OCSC_TEST_DEBUG_DATA
+#define MPC_OCSC_TEST_DEBUG_DATA__MPC_OCSC_TEST_DEBUG_DATA__SHIFT                                             0x0
+#define MPC_OCSC_TEST_DEBUG_DATA__MPC_OCSC_TEST_DEBUG_DATA_MASK                                               0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_mpc_mpc_rmcm0_dispdec
+//MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R__MPC_RMCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R__MPC_RMCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G__MPC_RMCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G__MPC_RMCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B__MPC_RMCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B__MPC_RMCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R__MPC_RMCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R__MPC_RMCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX__MPC_RMCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX__MPC_RMCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA__MPC_RMCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA__MPC_RMCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_MODE
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPC_RMCM0_MPC_RMCM_3DLUT_INDEX
+#define MPC_RMCM0_MPC_RMCM_3DLUT_INDEX__MPC_RMCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_INDEX__MPC_RMCM_3DLUT_INDEX_MASK                                             0x00003FFFL
+//MPC_RMCM0_MPC_RMCM_3DLUT_DATA
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT__MPC_RMCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT__MPC_RMCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR__MPC_RMCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR__MPC_RMCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__SHIFT                   0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__MPC_RMCM_GAMUT_REMAP_COEF_FORMAT_MASK                     0x00000001L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE__SHIFT                                 0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_CURRENT__SHIFT                         0x7
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_MASK                                   0x00000003L
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_CURRENT_MASK                           0x00000180L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C11_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C12_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C11_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C12_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C13_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C14_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C13_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C14_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C21_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C22_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C21_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C22_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C23_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C24_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C23_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C24_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C31_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C32_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C31_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C32_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C33_A__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C34_A__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C33_A_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C34_A_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C11_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C12_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C11_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C12_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C13_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C14_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C13_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C14_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C21_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C22_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C21_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C22_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C23_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C24_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C23_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C24_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C31_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C32_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C31_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C32_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C33_B__SHIFT                           0x0
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C34_B__SHIFT                           0x10
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C33_B_MASK                             0x0000FFFFL
+#define MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C34_B_MASK                             0xFFFF0000L
+//MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+//MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT__MPC_RMCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT__MPC_RMCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+//MPC_RMCM0_MPC_RMCM_CNTL
+#define MPC_RMCM0_MPC_RMCM_CNTL__MPC_RMCM_CNTL__SHIFT                                                         0x0
+#define MPC_RMCM0_MPC_RMCM_CNTL__MPC_RMCM_CNTL_MASK                                                           0x0000000FL
+//MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX__MPC_RMCM_TEST_DEBUG_INDEX__SHIFT                                 0x0
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX__MPC_RMCM_TEST_DEBUG_WRITE_EN__SHIFT                              0x8
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX__MPC_RMCM_TEST_DEBUG_INDEX_MASK                                   0x000000FFL
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX__MPC_RMCM_TEST_DEBUG_WRITE_EN_MASK                                0x00000100L
+//MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA__MPC_RMCM_TEST_DEBUG_DATA__SHIFT                                   0x0
+#define MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA__MPC_RMCM_TEST_DEBUG_DATA_MASK                                     0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_mpc_mpc_rmcm1_dispdec
+//MPC_RMCM1_MPC_RMCM_SHAPER_CONTROL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPC_RMCM1_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_CONTROL__MPC_RMCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_R__MPC_RMCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_R__MPC_RMCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_G
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_G__MPC_RMCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_G__MPC_RMCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_B__MPC_RMCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_OFFSET_B__MPC_RMCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_R__MPC_RMCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_R__MPC_RMCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_SCALE_G_B__MPC_RMCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_LUT_INDEX
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_INDEX__MPC_RMCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_INDEX__MPC_RMCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_LUT_DATA
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_DATA__MPC_RMCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_DATA__MPC_RMCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK__MPC_RMCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_START_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_B__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_G__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_END_CNTL_R__MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_0_1__MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_2_3__MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_4_5__MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_6_7__MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_8_9__MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_10_11__MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_12_13__MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_14_15__MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_16_17__MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_18_19__MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_20_21__MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_22_23__MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_24_25__MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_26_27__MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_28_29__MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_30_31__MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMA_REGION_32_33__MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_START_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_B__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_G__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_END_CNTL_R__MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_0_1__MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_2_3__MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_4_5__MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_6_7__MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_8_9__MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_10_11__MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_12_13__MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_14_15__MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_16_17__MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_18_19__MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_20_21__MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_22_23__MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_24_25__MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_26_27__MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_28_29__MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_30_31__MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPC_RMCM1_MPC_RMCM_SHAPER_RAMB_REGION_32_33__MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_MODE
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_MODE__MPC_RMCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPC_RMCM1_MPC_RMCM_3DLUT_INDEX
+#define MPC_RMCM1_MPC_RMCM_3DLUT_INDEX__MPC_RMCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_INDEX__MPC_RMCM_3DLUT_INDEX_MASK                                             0x00003FFFL
+//MPC_RMCM1_MPC_RMCM_3DLUT_DATA
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA__MPC_RMCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_DATA_30BIT
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA_30BIT__MPC_RMCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPC_RMCM1_MPC_RMCM_3DLUT_DATA_30BIT__MPC_RMCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_OUT_NORM_FACTOR
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_NORM_FACTOR__MPC_RMCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_NORM_FACTOR__MPC_RMCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_R__MPC_RMCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_G__MPC_RMCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_3DLUT_OUT_OFFSET_B__MPC_RMCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__SHIFT                   0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT__MPC_RMCM_GAMUT_REMAP_COEF_FORMAT_MASK                     0x00000001L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE__SHIFT                                 0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_CURRENT__SHIFT                         0x7
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_MASK                                   0x00000003L
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_MODE__MPC_RMCM_GAMUT_REMAP_MODE_CURRENT_MASK                           0x00000180L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C11_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C12_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C11_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_A__MPC_RMCM_GAMUT_REMAP_C12_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C13_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C14_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C13_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_A__MPC_RMCM_GAMUT_REMAP_C14_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C21_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C22_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C21_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_A__MPC_RMCM_GAMUT_REMAP_C22_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C23_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C24_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C23_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_A__MPC_RMCM_GAMUT_REMAP_C24_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C31_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C32_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C31_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_A__MPC_RMCM_GAMUT_REMAP_C32_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C33_A__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C34_A__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C33_A_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_A__MPC_RMCM_GAMUT_REMAP_C34_A_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C11_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C12_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C11_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C11_C12_B__MPC_RMCM_GAMUT_REMAP_C12_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C13_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C14_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C13_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C13_C14_B__MPC_RMCM_GAMUT_REMAP_C14_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C21_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C22_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C21_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C21_C22_B__MPC_RMCM_GAMUT_REMAP_C22_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C23_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C24_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C23_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C23_C24_B__MPC_RMCM_GAMUT_REMAP_C24_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C31_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C32_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C31_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C31_C32_B__MPC_RMCM_GAMUT_REMAP_C32_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C33_B__SHIFT                           0x0
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C34_B__SHIFT                           0x10
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C33_B_MASK                             0x0000FFFFL
+#define MPC_RMCM1_MPC_RMCM_GAMUT_REMAP_C33_C34_B__MPC_RMCM_GAMUT_REMAP_C34_B_MASK                             0xFFFF0000L
+//MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPC_RMCM1_MPC_RMCM_MEM_PWR_CTRL__MPC_RMCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+//MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_SELECT
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_SELECT__MPC_RMCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_SELECT__MPC_RMCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPC_RMCM1_MPC_RMCM_3DLUT_FAST_LOAD_STATUS__MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+//MPC_RMCM1_MPC_RMCM_CNTL
+#define MPC_RMCM1_MPC_RMCM_CNTL__MPC_RMCM_CNTL__SHIFT                                                         0x0
+#define MPC_RMCM1_MPC_RMCM_CNTL__MPC_RMCM_CNTL_MASK                                                           0x0000000FL
+
+
+// addressBlock: dce_dc_mpc_mpc_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON22_PERFCOUNTER_CNTL
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON22_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON22_PERFCOUNTER_CNTL2
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON22_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON22_PERFCOUNTER_STATE
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON22_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON22_PERFMON_CNTL
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON22_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON22_PERFMON_CNTL2
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON22_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON22_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON22_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON22_PERFMON_CVALUE_LOW
+#define DC_PERFMON22_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON22_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON22_PERFMON_HI
+#define DC_PERFMON22_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON22_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON22_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON22_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON22_PERFMON_LOW
+#define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm0_dispdec
+//MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_MODE
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPCC_MCM0_MPCC_MCM_3DLUT_INDEX
+#define MPCC_MCM0_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX_MASK                                             0x000007FFL
+//MPCC_MCM0_MPCC_MCM_3DLUT_DATA
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPCC_MCM0_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_OUT_NORM_FACTOR
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE__SHIFT                                          0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT__SHIFT                                        0x2
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE__SHIFT                                   0x3
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT__SHIFT                                  0x4
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT__SHIFT                                0x6
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_MASK                                            0x00000003L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_MASK                                          0x00000004L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE_MASK                                     0x00000008L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT_MASK                                    0x00000030L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT_MASK                                  0x00000040L
+//MPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX__SHIFT                                   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX_MASK                                     0x000001FFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA__SHIFT                                     0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA_MASK                                       0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK__SHIFT                      0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL__SHIFT                        0x3
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG__SHIFT                              0x5
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL__SHIFT                              0x6
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE__SHIFT                           0x7
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK_MASK                        0x00000007L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL_MASK                          0x00000018L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG_MASK                                0x00000020L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL_MASK                                0x00000040L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE_MASK                             0x00000080L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_B
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_G
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_R
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM0_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__SHIFT       0x0
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_MASK         0x00000001L
+//MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE__SHIFT                     0x0
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT__SHIFT             0x7
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_MASK                       0x00000003L
+#define MPCC_MCM0_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT_MASK               0x00000180L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B__SHIFT                0x0
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B__SHIFT                0x10
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B_MASK                  0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B_MASK                  0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__SHIFT     0x0
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_MASK       0x00000001L
+//MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE__SHIFT                   0x0
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT__SHIFT           0x7
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_MASK                     0x00000003L
+#define MPCC_MCM0_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT_MASK             0x00000180L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B__SHIFT              0x0
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B__SHIFT              0x10
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B_MASK                0x0000FFFFL
+#define MPCC_MCM0_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B_MASK                0xFFFF0000L
+//MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE__SHIFT                                  0x10
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS__SHIFT                                    0x12
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE__SHIFT                               0x14
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE__SHIFT                                  0x1c
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE_MASK                                    0x00030000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS_MASK                                      0x00040000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00300000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+#define MPCC_MCM0_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE_MASK                                    0x30000000L
+//MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPCC_MCM0_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm1_dispdec
+//MPCC_MCM1_MPCC_MCM_SHAPER_CONTROL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPCC_MCM1_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_G
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_LUT_INDEX
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_LUT_DATA
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_MODE
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPCC_MCM1_MPCC_MCM_3DLUT_INDEX
+#define MPCC_MCM1_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX_MASK                                             0x000007FFL
+//MPCC_MCM1_MPCC_MCM_3DLUT_DATA
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_DATA_30BIT
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPCC_MCM1_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_OUT_NORM_FACTOR
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE__SHIFT                                          0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT__SHIFT                                        0x2
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE__SHIFT                                   0x3
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT__SHIFT                                  0x4
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT__SHIFT                                0x6
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_MASK                                            0x00000003L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_MASK                                          0x00000004L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE_MASK                                     0x00000008L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT_MASK                                    0x00000030L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT_MASK                                  0x00000040L
+//MPCC_MCM1_MPCC_MCM_1DLUT_LUT_INDEX
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX__SHIFT                                   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX_MASK                                     0x000001FFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_LUT_DATA
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA__SHIFT                                     0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA_MASK                                       0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK__SHIFT                      0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL__SHIFT                        0x3
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG__SHIFT                              0x5
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL__SHIFT                              0x6
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE__SHIFT                           0x7
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK_MASK                        0x00000007L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL_MASK                          0x00000018L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG_MASK                                0x00000020L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL_MASK                                0x00000040L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE_MASK                             0x00000080L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_B
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_G
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_R
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM1_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__SHIFT       0x0
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_MASK         0x00000001L
+//MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE__SHIFT                     0x0
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT__SHIFT             0x7
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_MASK                       0x00000003L
+#define MPCC_MCM1_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT_MASK               0x00000180L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B__SHIFT                0x0
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B__SHIFT                0x10
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B_MASK                  0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B_MASK                  0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__SHIFT     0x0
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_MASK       0x00000001L
+//MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE__SHIFT                   0x0
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT__SHIFT           0x7
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_MASK                     0x00000003L
+#define MPCC_MCM1_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT_MASK             0x00000180L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B__SHIFT              0x0
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B__SHIFT              0x10
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B_MASK                0x0000FFFFL
+#define MPCC_MCM1_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B_MASK                0xFFFF0000L
+//MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE__SHIFT                                  0x10
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS__SHIFT                                    0x12
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE__SHIFT                               0x14
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE__SHIFT                                  0x1c
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE_MASK                                    0x00030000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS_MASK                                      0x00040000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00300000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+#define MPCC_MCM1_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE_MASK                                    0x30000000L
+//MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_SELECT
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPCC_MCM1_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm2_dispdec
+//MPCC_MCM2_MPCC_MCM_SHAPER_CONTROL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPCC_MCM2_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_G
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_LUT_INDEX
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_LUT_DATA
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_MODE
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPCC_MCM2_MPCC_MCM_3DLUT_INDEX
+#define MPCC_MCM2_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX_MASK                                             0x000007FFL
+//MPCC_MCM2_MPCC_MCM_3DLUT_DATA
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_DATA_30BIT
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPCC_MCM2_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_OUT_NORM_FACTOR
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE__SHIFT                                          0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT__SHIFT                                        0x2
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE__SHIFT                                   0x3
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT__SHIFT                                  0x4
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT__SHIFT                                0x6
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_MASK                                            0x00000003L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_MASK                                          0x00000004L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE_MASK                                     0x00000008L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT_MASK                                    0x00000030L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT_MASK                                  0x00000040L
+//MPCC_MCM2_MPCC_MCM_1DLUT_LUT_INDEX
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX__SHIFT                                   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX_MASK                                     0x000001FFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_LUT_DATA
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA__SHIFT                                     0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA_MASK                                       0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK__SHIFT                      0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL__SHIFT                        0x3
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG__SHIFT                              0x5
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL__SHIFT                              0x6
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE__SHIFT                           0x7
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK_MASK                        0x00000007L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL_MASK                          0x00000018L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG_MASK                                0x00000020L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL_MASK                                0x00000040L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE_MASK                             0x00000080L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_B
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_G
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_R
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM2_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__SHIFT       0x0
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_MASK         0x00000001L
+//MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE__SHIFT                     0x0
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT__SHIFT             0x7
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_MASK                       0x00000003L
+#define MPCC_MCM2_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT_MASK               0x00000180L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B__SHIFT                0x0
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B__SHIFT                0x10
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B_MASK                  0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B_MASK                  0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__SHIFT     0x0
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_MASK       0x00000001L
+//MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE__SHIFT                   0x0
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT__SHIFT           0x7
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_MASK                     0x00000003L
+#define MPCC_MCM2_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT_MASK             0x00000180L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B__SHIFT              0x0
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B__SHIFT              0x10
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B_MASK                0x0000FFFFL
+#define MPCC_MCM2_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B_MASK                0xFFFF0000L
+//MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE__SHIFT                                  0x10
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS__SHIFT                                    0x12
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE__SHIFT                               0x14
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE__SHIFT                                  0x1c
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE_MASK                                    0x00030000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS_MASK                                      0x00040000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00300000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+#define MPCC_MCM2_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE_MASK                                    0x30000000L
+//MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_SELECT
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPCC_MCM2_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpcc_mcm3_dispdec
+//MPCC_MCM3_MPCC_MCM_SHAPER_CONTROL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE__SHIFT                                    0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT__SHIFT                                0x2
+#define MPCC_MCM3_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_LUT_MODE_MASK                                      0x00000003L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_CONTROL__MPCC_MCM_SHAPER_MODE_CURRENT_MASK                                  0x0000000CL
+//MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_R__MPCC_MCM_SHAPER_OFFSET_R_MASK                                     0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_G
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_G__MPCC_MCM_SHAPER_OFFSET_G_MASK                                     0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_OFFSET_B__MPCC_MCM_SHAPER_OFFSET_B_MASK                                     0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R__SHIFT                                     0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_R__MPCC_MCM_SHAPER_SCALE_R_MASK                                       0x0000FFFFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B__SHIFT                                   0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_G_MASK                                     0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_SCALE_G_B__MPCC_MCM_SHAPER_SCALE_B_MASK                                     0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_LUT_INDEX
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX__SHIFT                                 0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_INDEX__MPCC_MCM_SHAPER_LUT_INDEX_MASK                                   0x000000FFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_LUT_DATA
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_DATA__MPCC_MCM_SHAPER_LUT_DATA_MASK                                     0x00FFFFFFL
+//MPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__SHIFT                 0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL__SHIFT                     0x4
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK_MASK                   0x00000007L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK__MPCC_MCM_SHAPER_LUT_WRITE_SEL_MASK                       0x00000010L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_START_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_B__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_G__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_END_CNTL_R__MPCC_MCM_SHAPER_RAMA_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_0_1__MPCC_MCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_2_3__MPCC_MCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_4_5__MPCC_MCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_6_7__MPCC_MCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_8_9__MPCC_MCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_10_11__MPCC_MCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_12_13__MPCC_MCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_14_15__MPCC_MCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_16_17__MPCC_MCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_18_19__MPCC_MCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_20_21__MPCC_MCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_22_23__MPCC_MCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_24_25__MPCC_MCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_26_27__MPCC_MCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_28_29__MPCC_MCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_30_31__MPCC_MCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMA_REGION_32_33__MPCC_MCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_B_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_G_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT   0x14
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_R_MASK             0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_START_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R_MASK     0x07F00000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_B_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_B__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_B_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_G_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_G__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_G_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R__SHIFT               0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_R_MASK                 0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_END_CNTL_R__MPCC_MCM_SHAPER_RAMB_EXP_REGION_END_BASE_R_MASK            0x3FFF0000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_0_1__MPCC_MCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_2_3__MPCC_MCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_4_5__MPCC_MCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_6_7__MPCC_MCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT         0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT       0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT         0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT       0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET_MASK           0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK         0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET_MASK           0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_8_9__MPCC_MCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK         0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_10_11__MPCC_MCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_12_13__MPCC_MCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_14_15__MPCC_MCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_16_17__MPCC_MCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_18_19__MPCC_MCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_20_21__MPCC_MCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_22_23__MPCC_MCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_24_25__MPCC_MCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_26_27__MPCC_MCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_28_29__MPCC_MCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_30_31__MPCC_MCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT      0x0
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT    0xc
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT      0x10
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT    0x1c
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET_MASK        0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK      0x00007000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET_MASK        0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_SHAPER_RAMB_REGION_32_33__MPCC_MCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK      0x70000000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_MODE
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE__SHIFT                                             0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE__SHIFT                                             0x4
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT__SHIFT                                     0x8
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_MASK                                               0x00000003L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_SIZE_MASK                                               0x00000030L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_MODE__MPCC_MCM_3DLUT_MODE_CURRENT_MASK                                       0x00000300L
+//MPCC_MCM3_MPCC_MCM_3DLUT_INDEX
+#define MPCC_MCM3_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX__SHIFT                                           0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_INDEX__MPCC_MCM_3DLUT_INDEX_MASK                                             0x000007FFL
+//MPCC_MCM3_MPCC_MCM_3DLUT_DATA
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0__SHIFT                                            0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1__SHIFT                                            0x10
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA0_MASK                                              0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA__MPCC_MCM_3DLUT_DATA1_MASK                                              0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_DATA_30BIT
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT__SHIFT                                 0x2
+#define MPCC_MCM3_MPCC_MCM_3DLUT_DATA_30BIT__MPCC_MCM_3DLUT_DATA_30BIT_MASK                                   0xFFFFFFFCL
+//MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK__SHIFT                      0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL__SHIFT                            0x4
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN__SHIFT                           0x8
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL__SHIFT                           0x10
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_WRITE_EN_MASK_MASK                        0x0000000FL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_RAM_SEL_MASK                              0x00000010L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_30BIT_EN_MASK                             0x00000100L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_READ_WRITE_CONTROL__MPCC_MCM_3DLUT_READ_SEL_MASK                             0x00030000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_OUT_NORM_FACTOR
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR__SHIFT                       0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_NORM_FACTOR__MPCC_MCM_3DLUT_OUT_NORM_FACTOR_MASK                         0x0000FFFFL
+//MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R__SHIFT                             0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R__SHIFT                              0x10
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_OFFSET_R_MASK                               0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_R__MPCC_MCM_3DLUT_OUT_SCALE_R_MASK                                0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G__SHIFT                             0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G__SHIFT                              0x10
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_OFFSET_G_MASK                               0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_G__MPCC_MCM_3DLUT_OUT_SCALE_G_MASK                                0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B__SHIFT                             0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B__SHIFT                              0x10
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_OFFSET_B_MASK                               0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_3DLUT_OUT_OFFSET_B__MPCC_MCM_3DLUT_OUT_SCALE_B_MASK                                0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE__SHIFT                                          0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT__SHIFT                                        0x2
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE__SHIFT                                   0x3
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT__SHIFT                                  0x4
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT__SHIFT                                0x6
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_MASK                                            0x00000003L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_MASK                                          0x00000004L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_PWL_DISABLE_MASK                                     0x00000008L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_MODE_CURRENT_MASK                                    0x00000030L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_CONTROL__MPCC_MCM_1DLUT_SELECT_CURRENT_MASK                                  0x00000040L
+//MPCC_MCM3_MPCC_MCM_1DLUT_LUT_INDEX
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX__SHIFT                                   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_INDEX__MPCC_MCM_1DLUT_LUT_INDEX_MASK                                     0x000001FFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_LUT_DATA
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA__SHIFT                                     0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_DATA__MPCC_MCM_1DLUT_LUT_DATA_MASK                                       0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK__SHIFT                      0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL__SHIFT                        0x3
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG__SHIFT                              0x5
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL__SHIFT                              0x6
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE__SHIFT                           0x7
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_WRITE_COLOR_MASK_MASK                        0x00000007L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_COLOR_SEL_MASK                          0x00000018L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_READ_DBG_MASK                                0x00000020L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_HOST_SEL_MASK                                0x00000040L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_LUT_CONTROL__MPCC_MCM_1DLUT_LUT_CONFIG_MODE_MASK                             0x00000080L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_B__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_G__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL1_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_END_CNTL2_R__MPCC_MCM_1DLUT_RAMA_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_B__MPCC_MCM_1DLUT_RAMA_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_G__MPCC_MCM_1DLUT_RAMA_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_OFFSET_R__MPCC_MCM_1DLUT_RAMA_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_0_1__MPCC_MCM_1DLUT_RAMA_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_2_3__MPCC_MCM_1DLUT_RAMA_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_4_5__MPCC_MCM_1DLUT_RAMA_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_6_7__MPCC_MCM_1DLUT_RAMA_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_8_9__MPCC_MCM_1DLUT_RAMA_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_10_11__MPCC_MCM_1DLUT_RAMA_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_12_13__MPCC_MCM_1DLUT_RAMA_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_14_15__MPCC_MCM_1DLUT_RAMA_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_16_17__MPCC_MCM_1DLUT_RAMA_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_18_19__MPCC_MCM_1DLUT_RAMA_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_20_21__MPCC_MCM_1DLUT_RAMA_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_22_23__MPCC_MCM_1DLUT_RAMA_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_24_25__MPCC_MCM_1DLUT_RAMA_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_26_27__MPCC_MCM_1DLUT_RAMA_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_28_29__MPCC_MCM_1DLUT_RAMA_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_30_31__MPCC_MCM_1DLUT_RAMA_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMA_REGION_32_33__MPCC_MCM_1DLUT_RAMA_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_B_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_B_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_G_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_G_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R__SHIFT             0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R__SHIFT     0x14
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_R_MASK               0x0003FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SEGMENT_R_MASK       0x07F00000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_B_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_G_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R__SHIFT  0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_SLOPE_R_MASK   0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_B_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_G_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R__SHIFT   0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_START_BASE_R_MASK     0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_B_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_B__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_B_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_G_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_G_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_G__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_G_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL1_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_BASE_R_MASK             0x0003FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R__SHIFT                0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R__SHIFT          0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_R_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_END_CNTL2_R__MPCC_MCM_1DLUT_RAMB_EXP_REGION_END_SLOPE_R_MASK            0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_B
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_B__MPCC_MCM_1DLUT_RAMB_OFFSET_B_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_G
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_G__MPCC_MCM_1DLUT_RAMB_OFFSET_G_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_R
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R__SHIFT                           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_OFFSET_R__MPCC_MCM_1DLUT_RAMB_OFFSET_R_MASK                             0x0007FFFFL
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION0_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_0_1__MPCC_MCM_1DLUT_RAMB_EXP_REGION1_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION2_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_2_3__MPCC_MCM_1DLUT_RAMB_EXP_REGION3_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION4_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_4_5__MPCC_MCM_1DLUT_RAMB_EXP_REGION5_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION6_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_6_7__MPCC_MCM_1DLUT_RAMB_EXP_REGION7_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET__SHIFT           0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS__SHIFT         0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET__SHIFT           0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS__SHIFT         0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_LUT_OFFSET_MASK             0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION8_NUM_SEGMENTS_MASK           0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_LUT_OFFSET_MASK             0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_8_9__MPCC_MCM_1DLUT_RAMB_EXP_REGION9_NUM_SEGMENTS_MASK           0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION10_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_10_11__MPCC_MCM_1DLUT_RAMB_EXP_REGION11_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION12_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_12_13__MPCC_MCM_1DLUT_RAMB_EXP_REGION13_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION14_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_14_15__MPCC_MCM_1DLUT_RAMB_EXP_REGION15_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION16_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_16_17__MPCC_MCM_1DLUT_RAMB_EXP_REGION17_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION18_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_18_19__MPCC_MCM_1DLUT_RAMB_EXP_REGION19_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION20_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_20_21__MPCC_MCM_1DLUT_RAMB_EXP_REGION21_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION22_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_22_23__MPCC_MCM_1DLUT_RAMB_EXP_REGION23_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION24_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_24_25__MPCC_MCM_1DLUT_RAMB_EXP_REGION25_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION26_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_26_27__MPCC_MCM_1DLUT_RAMB_EXP_REGION27_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION28_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_28_29__MPCC_MCM_1DLUT_RAMB_EXP_REGION29_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION30_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_30_31__MPCC_MCM_1DLUT_RAMB_EXP_REGION31_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET__SHIFT        0x0
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS__SHIFT      0xc
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET__SHIFT        0x10
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS__SHIFT      0x1c
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_LUT_OFFSET_MASK          0x000001FFL
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION32_NUM_SEGMENTS_MASK        0x00007000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_LUT_OFFSET_MASK          0x01FF0000L
+#define MPCC_MCM3_MPCC_MCM_1DLUT_RAMB_REGION_32_33__MPCC_MCM_1DLUT_RAMB_EXP_REGION33_NUM_SEGMENTS_MASK        0x70000000L
+//MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__SHIFT       0x0
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_FIRST_GAMUT_REMAP_COEF_FORMAT_MASK         0x00000001L
+//MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE__SHIFT                     0x0
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT__SHIFT             0x7
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_MASK                       0x00000003L
+#define MPCC_MCM3_MPCC_MCM_FIRST_GAMUT_REMAP_MODE__MPCC_MCM_FIRST_GAMUT_REMAP_MODE_CURRENT_MASK               0x00000180L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C11_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_A__MPCC_MCM_FIRST_GAMUT_REMAP_C12_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C13_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_A__MPCC_MCM_FIRST_GAMUT_REMAP_C14_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C21_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_A__MPCC_MCM_FIRST_GAMUT_REMAP_C22_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C23_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_A__MPCC_MCM_FIRST_GAMUT_REMAP_C24_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C31_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_A__MPCC_MCM_FIRST_GAMUT_REMAP_C32_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C33_A_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_A__MPCC_MCM_FIRST_GAMUT_REMAP_C34_A_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C11_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C11_C12_B__MPCC_MCM_FIRST_GAMUT_REMAP_C12_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C13_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C13_C14_B__MPCC_MCM_FIRST_GAMUT_REMAP_C14_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C21_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C21_C22_B__MPCC_MCM_FIRST_GAMUT_REMAP_C22_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C23_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C23_C24_B__MPCC_MCM_FIRST_GAMUT_REMAP_C24_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C31_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C31_C32_B__MPCC_MCM_FIRST_GAMUT_REMAP_C32_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B__SHIFT                0x0
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B__SHIFT                0x10
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C33_B_MASK                  0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_FIRST_GAMUT_REMAP_C33_C34_B__MPCC_MCM_FIRST_GAMUT_REMAP_C34_B_MASK                  0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__SHIFT     0x0
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT__MPCC_MCM_SECOND_GAMUT_REMAP_COEF_FORMAT_MASK       0x00000001L
+//MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE__SHIFT                   0x0
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT__SHIFT           0x7
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_MASK                     0x00000003L
+#define MPCC_MCM3_MPCC_MCM_SECOND_GAMUT_REMAP_MODE__MPCC_MCM_SECOND_GAMUT_REMAP_MODE_CURRENT_MASK             0x00000180L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C11_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_A__MPCC_MCM_SECOND_GAMUT_REMAP_C12_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C13_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_A__MPCC_MCM_SECOND_GAMUT_REMAP_C14_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C21_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_A__MPCC_MCM_SECOND_GAMUT_REMAP_C22_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C23_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_A__MPCC_MCM_SECOND_GAMUT_REMAP_C24_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C31_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_A__MPCC_MCM_SECOND_GAMUT_REMAP_C32_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C33_A_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_A__MPCC_MCM_SECOND_GAMUT_REMAP_C34_A_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C11_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C11_C12_B__MPCC_MCM_SECOND_GAMUT_REMAP_C12_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C13_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C13_C14_B__MPCC_MCM_SECOND_GAMUT_REMAP_C14_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C21_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C21_C22_B__MPCC_MCM_SECOND_GAMUT_REMAP_C22_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C23_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C23_C24_B__MPCC_MCM_SECOND_GAMUT_REMAP_C24_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C31_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C31_C32_B__MPCC_MCM_SECOND_GAMUT_REMAP_C32_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B__SHIFT              0x0
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B__SHIFT              0x10
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C33_B_MASK                0x0000FFFFL
+#define MPCC_MCM3_MPC_MCM_SECOND_GAMUT_REMAP_C33_C34_B__MPCC_MCM_SECOND_GAMUT_REMAP_C34_B_MASK                0xFFFF0000L
+//MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE__SHIFT                                 0x0
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS__SHIFT                                   0x2
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE__SHIFT                              0x4
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE__SHIFT                                  0x8
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS__SHIFT                                    0xa
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE__SHIFT                               0xc
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE__SHIFT                                  0x10
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS__SHIFT                                    0x12
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE__SHIFT                               0x14
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE__SHIFT                                 0x18
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE__SHIFT                                  0x1a
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE__SHIFT                                  0x1c
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_FORCE_MASK                                   0x00000003L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_DIS_MASK                                     0x00000004L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_LOW_PWR_MODE_MASK                                0x00000030L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_FORCE_MASK                                    0x00000300L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_DIS_MASK                                      0x00000400L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00003000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_FORCE_MASK                                    0x00030000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_DIS_MASK                                      0x00040000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_LOW_PWR_MODE_MASK                                 0x00300000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_SHAPER_MEM_PWR_STATE_MASK                                   0x03000000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_3DLUT_MEM_PWR_STATE_MASK                                    0x0C000000L
+#define MPCC_MCM3_MPCC_MCM_MEM_PWR_CTRL__MPCC_MCM_1DLUT_MEM_PWR_STATE_MASK                                    0x30000000L
+//MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_SELECT
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL__SHIFT                               0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_SELECT__MPCC_MCM_3DLUT_FL_SEL_MASK                                 0x0000000FL
+//MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE__SHIFT                              0x0
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW__SHIFT                    0x1
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW__SHIFT                    0x2
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_DONE_MASK                                0x00000001L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_SOFT_UNDERFLOW_MASK                      0x00000002L
+#define MPCC_MCM3_MPCC_MCM_3DLUT_FAST_LOAD_STATUS__MPCC_MCM_3DLUT_FL_HARD_UNDERFLOW_MASK                      0x00000004L
+
+
+// addressBlock: dce_dc_mpc_mpc_cfg_dispdec
+//MPC_CLOCK_CONTROL
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE0_GATE_DISABLE__SHIFT                                                0x0
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE1_GATE_DISABLE__SHIFT                                                0x1
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE2_GATE_DISABLE__SHIFT                                                0x2
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE3_GATE_DISABLE__SHIFT                                                0x3
+#define MPC_CLOCK_CONTROL__DISPCLK_R_GATE_DISABLE__SHIFT                                                      0x8
+#define MPC_CLOCK_CONTROL__MPC_TEST_CLK_SEL__SHIFT                                                            0xc
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE0_GATE_DISABLE_MASK                                                  0x00000001L
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE1_GATE_DISABLE_MASK                                                  0x00000002L
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE2_GATE_DISABLE_MASK                                                  0x00000004L
+#define MPC_CLOCK_CONTROL__DISPCLK_G_PIPE3_GATE_DISABLE_MASK                                                  0x00000008L
+#define MPC_CLOCK_CONTROL__DISPCLK_R_GATE_DISABLE_MASK                                                        0x00000100L
+#define MPC_CLOCK_CONTROL__MPC_TEST_CLK_SEL_MASK                                                              0x0000F000L
+//MPC_SOFT_RESET
+#define MPC_SOFT_RESET__MPCC0_SOFT_RESET__SHIFT                                                               0x0
+#define MPC_SOFT_RESET__MPCC1_SOFT_RESET__SHIFT                                                               0x1
+#define MPC_SOFT_RESET__MPCC2_SOFT_RESET__SHIFT                                                               0x2
+#define MPC_SOFT_RESET__MPCC3_SOFT_RESET__SHIFT                                                               0x3
+#define MPC_SOFT_RESET__MPC_SFR0_SOFT_RESET__SHIFT                                                            0xa
+#define MPC_SOFT_RESET__MPC_SFR1_SOFT_RESET__SHIFT                                                            0xb
+#define MPC_SOFT_RESET__MPC_SFR2_SOFT_RESET__SHIFT                                                            0xc
+#define MPC_SOFT_RESET__MPC_SFR3_SOFT_RESET__SHIFT                                                            0xd
+#define MPC_SOFT_RESET__MPC_SFT0_SOFT_RESET__SHIFT                                                            0x14
+#define MPC_SOFT_RESET__MPC_SFT1_SOFT_RESET__SHIFT                                                            0x15
+#define MPC_SOFT_RESET__MPC_SFT2_SOFT_RESET__SHIFT                                                            0x16
+#define MPC_SOFT_RESET__MPC_SFT3_SOFT_RESET__SHIFT                                                            0x17
+#define MPC_SOFT_RESET__MPC_SOFT_RESET__SHIFT                                                                 0x1f
+#define MPC_SOFT_RESET__MPCC0_SOFT_RESET_MASK                                                                 0x00000001L
+#define MPC_SOFT_RESET__MPCC1_SOFT_RESET_MASK                                                                 0x00000002L
+#define MPC_SOFT_RESET__MPCC2_SOFT_RESET_MASK                                                                 0x00000004L
+#define MPC_SOFT_RESET__MPCC3_SOFT_RESET_MASK                                                                 0x00000008L
+#define MPC_SOFT_RESET__MPC_SFR0_SOFT_RESET_MASK                                                              0x00000400L
+#define MPC_SOFT_RESET__MPC_SFR1_SOFT_RESET_MASK                                                              0x00000800L
+#define MPC_SOFT_RESET__MPC_SFR2_SOFT_RESET_MASK                                                              0x00001000L
+#define MPC_SOFT_RESET__MPC_SFR3_SOFT_RESET_MASK                                                              0x00002000L
+#define MPC_SOFT_RESET__MPC_SFT0_SOFT_RESET_MASK                                                              0x00100000L
+#define MPC_SOFT_RESET__MPC_SFT1_SOFT_RESET_MASK                                                              0x00200000L
+#define MPC_SOFT_RESET__MPC_SFT2_SOFT_RESET_MASK                                                              0x00400000L
+#define MPC_SOFT_RESET__MPC_SFT3_SOFT_RESET_MASK                                                              0x00800000L
+#define MPC_SOFT_RESET__MPC_SOFT_RESET_MASK                                                                   0x80000000L
+//MPC_CRC_CTRL
+#define MPC_CRC_CTRL__MPC_CRC_EN__SHIFT                                                                       0x0
+#define MPC_CRC_CTRL__MPC_CRC_CONT_EN__SHIFT                                                                  0x4
+#define MPC_CRC_CTRL__MPC_CRC_STEREO_MODE__SHIFT                                                              0x8
+#define MPC_CRC_CTRL__MPC_CRC_STEREO_EN__SHIFT                                                                0xa
+#define MPC_CRC_CTRL__MPC_CRC_INTERLACE_MODE__SHIFT                                                           0xc
+#define MPC_CRC_CTRL__MPC_CRC_SRC_SEL__SHIFT                                                                  0x18
+#define MPC_CRC_CTRL__MPC_CRC_ONE_SHOT_PENDING__SHIFT                                                         0x1c
+#define MPC_CRC_CTRL__MPC_CRC_UPDATE_ENABLED__SHIFT                                                           0x1e
+#define MPC_CRC_CTRL__MPC_CRC_UPDATE_LOCK__SHIFT                                                              0x1f
+#define MPC_CRC_CTRL__MPC_CRC_EN_MASK                                                                         0x00000001L
+#define MPC_CRC_CTRL__MPC_CRC_CONT_EN_MASK                                                                    0x00000010L
+#define MPC_CRC_CTRL__MPC_CRC_STEREO_MODE_MASK                                                                0x00000300L
+#define MPC_CRC_CTRL__MPC_CRC_STEREO_EN_MASK                                                                  0x00000400L
+#define MPC_CRC_CTRL__MPC_CRC_INTERLACE_MODE_MASK                                                             0x00003000L
+#define MPC_CRC_CTRL__MPC_CRC_SRC_SEL_MASK                                                                    0x03000000L
+#define MPC_CRC_CTRL__MPC_CRC_ONE_SHOT_PENDING_MASK                                                           0x10000000L
+#define MPC_CRC_CTRL__MPC_CRC_UPDATE_ENABLED_MASK                                                             0x40000000L
+#define MPC_CRC_CTRL__MPC_CRC_UPDATE_LOCK_MASK                                                                0x80000000L
+//MPC_CRC_SEL_CONTROL
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_DPP_SEL__SHIFT                                                           0x0
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_OPP_SEL__SHIFT                                                           0x4
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_DWB_SEL__SHIFT                                                           0x8
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_MASK__SHIFT                                                              0x10
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_DPP_SEL_MASK                                                             0x0000000FL
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_OPP_SEL_MASK                                                             0x000000F0L
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_DWB_SEL_MASK                                                             0x00000300L
+#define MPC_CRC_SEL_CONTROL__MPC_CRC_MASK_MASK                                                                0xFFFF0000L
+//MPC_CRC_RESULT_A
+#define MPC_CRC_RESULT_A__MPC_CRC_RESULT_A__SHIFT                                                             0x0
+#define MPC_CRC_RESULT_A__MPC_CRC_RESULT_A_MASK                                                               0xFFFFFFFFL
+//MPC_CRC_RESULT_R
+#define MPC_CRC_RESULT_R__MPC_CRC_RESULT_R__SHIFT                                                             0x0
+#define MPC_CRC_RESULT_R__MPC_CRC_RESULT_R_MASK                                                               0xFFFFFFFFL
+//MPC_CRC_RESULT_G
+#define MPC_CRC_RESULT_G__MPC_CRC_RESULT_G__SHIFT                                                             0x0
+#define MPC_CRC_RESULT_G__MPC_CRC_RESULT_G_MASK                                                               0xFFFFFFFFL
+//MPC_CRC_RESULT_B
+#define MPC_CRC_RESULT_B__MPC_CRC_RESULT_B__SHIFT                                                             0x0
+#define MPC_CRC_RESULT_B__MPC_CRC_RESULT_B_MASK                                                               0xFFFFFFFFL
+//MPC_PERFMON_EVENT_CTRL
+#define MPC_PERFMON_EVENT_CTRL__MPC_PERFMON_EVENT_EN__SHIFT                                                   0x0
+#define MPC_PERFMON_EVENT_CTRL__MPC_PERFMON_EVENT_EN_MASK                                                     0x00000001L
+//MPC_BYPASS_BG_AR
+#define MPC_BYPASS_BG_AR__MPC_BYPASS_BG_ALPHA__SHIFT                                                          0x0
+#define MPC_BYPASS_BG_AR__MPC_BYPASS_BG_R_CR__SHIFT                                                           0x10
+#define MPC_BYPASS_BG_AR__MPC_BYPASS_BG_ALPHA_MASK                                                            0x0000FFFFL
+#define MPC_BYPASS_BG_AR__MPC_BYPASS_BG_R_CR_MASK                                                             0xFFFF0000L
+//MPC_BYPASS_BG_GB
+#define MPC_BYPASS_BG_GB__MPC_BYPASS_BG_G_Y__SHIFT                                                            0x0
+#define MPC_BYPASS_BG_GB__MPC_BYPASS_BG_B_CB__SHIFT                                                           0x10
+#define MPC_BYPASS_BG_GB__MPC_BYPASS_BG_G_Y_MASK                                                              0x0000FFFFL
+#define MPC_BYPASS_BG_GB__MPC_BYPASS_BG_B_CB_MASK                                                             0xFFFF0000L
+//MPC_HOST_READ_CONTROL
+#define MPC_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL__SHIFT                                                  0x0
+#define MPC_HOST_READ_CONTROL__HOST_READ_RATE_CONTROL_MASK                                                    0x000000FFL
+//MPC_DPP_PENDING_STATUS
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_SURFACE_UPDATE_PENDING__SHIFT                                         0x0
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_CONFIG_UPDATE_PENDING__SHIFT                                          0x1
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_CURSOR_UPDATE_PENDING__SHIFT                                          0x2
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_SURFACE_UPDATE_PENDING__SHIFT                                         0x4
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_CONFIG_UPDATE_PENDING__SHIFT                                          0x5
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_CURSOR_UPDATE_PENDING__SHIFT                                          0x6
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_SURFACE_UPDATE_PENDING__SHIFT                                         0x8
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_CONFIG_UPDATE_PENDING__SHIFT                                          0x9
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_CURSOR_UPDATE_PENDING__SHIFT                                          0xa
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_SURFACE_UPDATE_PENDING__SHIFT                                         0xc
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_CONFIG_UPDATE_PENDING__SHIFT                                          0xd
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_CURSOR_UPDATE_PENDING__SHIFT                                          0xe
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_SURFACE_UPDATE_PENDING_MASK                                           0x00000001L
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_CONFIG_UPDATE_PENDING_MASK                                            0x00000002L
+#define MPC_DPP_PENDING_STATUS__IN_DPP0_CURSOR_UPDATE_PENDING_MASK                                            0x00000004L
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_SURFACE_UPDATE_PENDING_MASK                                           0x00000010L
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_CONFIG_UPDATE_PENDING_MASK                                            0x00000020L
+#define MPC_DPP_PENDING_STATUS__IN_DPP1_CURSOR_UPDATE_PENDING_MASK                                            0x00000040L
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_SURFACE_UPDATE_PENDING_MASK                                           0x00000100L
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_CONFIG_UPDATE_PENDING_MASK                                            0x00000200L
+#define MPC_DPP_PENDING_STATUS__IN_DPP2_CURSOR_UPDATE_PENDING_MASK                                            0x00000400L
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_SURFACE_UPDATE_PENDING_MASK                                           0x00001000L
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_CONFIG_UPDATE_PENDING_MASK                                            0x00002000L
+#define MPC_DPP_PENDING_STATUS__IN_DPP3_CURSOR_UPDATE_PENDING_MASK                                            0x00004000L
+//MPC_PENDING_STATUS_MISC
+#define MPC_PENDING_STATUS_MISC__OUT_OPP0_CONFIG_UPDATE_PENDING__SHIFT                                        0x0
+#define MPC_PENDING_STATUS_MISC__OUT_OPP1_CONFIG_UPDATE_PENDING__SHIFT                                        0x1
+#define MPC_PENDING_STATUS_MISC__OUT_OPP2_CONFIG_UPDATE_PENDING__SHIFT                                        0x2
+#define MPC_PENDING_STATUS_MISC__OUT_OPP3_CONFIG_UPDATE_PENDING__SHIFT                                        0x3
+#define MPC_PENDING_STATUS_MISC__MPCC0_CONFIG_UPDATE_PENDING__SHIFT                                           0x8
+#define MPC_PENDING_STATUS_MISC__MPCC1_CONFIG_UPDATE_PENDING__SHIFT                                           0x9
+#define MPC_PENDING_STATUS_MISC__MPCC2_CONFIG_UPDATE_PENDING__SHIFT                                           0xa
+#define MPC_PENDING_STATUS_MISC__MPCC3_CONFIG_UPDATE_PENDING__SHIFT                                           0xb
+#define MPC_PENDING_STATUS_MISC__IN_DWB0_CONFIG_UPDATE_PENDING__SHIFT                                         0x10
+#define MPC_PENDING_STATUS_MISC__OUT_OPP0_CONFIG_UPDATE_PENDING_MASK                                          0x00000001L
+#define MPC_PENDING_STATUS_MISC__OUT_OPP1_CONFIG_UPDATE_PENDING_MASK                                          0x00000002L
+#define MPC_PENDING_STATUS_MISC__OUT_OPP2_CONFIG_UPDATE_PENDING_MASK                                          0x00000004L
+#define MPC_PENDING_STATUS_MISC__OUT_OPP3_CONFIG_UPDATE_PENDING_MASK                                          0x00000008L
+#define MPC_PENDING_STATUS_MISC__MPCC0_CONFIG_UPDATE_PENDING_MASK                                             0x00000100L
+#define MPC_PENDING_STATUS_MISC__MPCC1_CONFIG_UPDATE_PENDING_MASK                                             0x00000200L
+#define MPC_PENDING_STATUS_MISC__MPCC2_CONFIG_UPDATE_PENDING_MASK                                             0x00000400L
+#define MPC_PENDING_STATUS_MISC__MPCC3_CONFIG_UPDATE_PENDING_MASK                                             0x00000800L
+#define MPC_PENDING_STATUS_MISC__IN_DWB0_CONFIG_UPDATE_PENDING_MASK                                           0x00010000L
+//ADR_CFG_CUR_VUPDATE_LOCK_SET0
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET0__ADR_CFG_CUR_VUPDATE_LOCK_SET__SHIFT                                    0x0
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET0__ADR_CFG_CUR_VUPDATE_LOCK_SET_MASK                                      0x00000001L
+//ADR_CFG_VUPDATE_LOCK_SET0
+#define ADR_CFG_VUPDATE_LOCK_SET0__ADR_CFG_VUPDATE_LOCK_SET__SHIFT                                            0x0
+#define ADR_CFG_VUPDATE_LOCK_SET0__ADR_CFG_VUPDATE_LOCK_SET_MASK                                              0x00000001L
+//ADR_VUPDATE_LOCK_SET0
+#define ADR_VUPDATE_LOCK_SET0__ADR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define ADR_VUPDATE_LOCK_SET0__ADR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CFG_VUPDATE_LOCK_SET0
+#define CFG_VUPDATE_LOCK_SET0__CFG_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CFG_VUPDATE_LOCK_SET0__CFG_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CUR_VUPDATE_LOCK_SET0
+#define CUR_VUPDATE_LOCK_SET0__CUR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CUR_VUPDATE_LOCK_SET0__CUR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//ADR_CFG_CUR_VUPDATE_LOCK_SET1
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET1__ADR_CFG_CUR_VUPDATE_LOCK_SET__SHIFT                                    0x0
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET1__ADR_CFG_CUR_VUPDATE_LOCK_SET_MASK                                      0x00000001L
+//ADR_CFG_VUPDATE_LOCK_SET1
+#define ADR_CFG_VUPDATE_LOCK_SET1__ADR_CFG_VUPDATE_LOCK_SET__SHIFT                                            0x0
+#define ADR_CFG_VUPDATE_LOCK_SET1__ADR_CFG_VUPDATE_LOCK_SET_MASK                                              0x00000001L
+//ADR_VUPDATE_LOCK_SET1
+#define ADR_VUPDATE_LOCK_SET1__ADR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define ADR_VUPDATE_LOCK_SET1__ADR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CFG_VUPDATE_LOCK_SET1
+#define CFG_VUPDATE_LOCK_SET1__CFG_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CFG_VUPDATE_LOCK_SET1__CFG_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CUR_VUPDATE_LOCK_SET1
+#define CUR_VUPDATE_LOCK_SET1__CUR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CUR_VUPDATE_LOCK_SET1__CUR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//ADR_CFG_CUR_VUPDATE_LOCK_SET2
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET2__ADR_CFG_CUR_VUPDATE_LOCK_SET__SHIFT                                    0x0
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET2__ADR_CFG_CUR_VUPDATE_LOCK_SET_MASK                                      0x00000001L
+//ADR_CFG_VUPDATE_LOCK_SET2
+#define ADR_CFG_VUPDATE_LOCK_SET2__ADR_CFG_VUPDATE_LOCK_SET__SHIFT                                            0x0
+#define ADR_CFG_VUPDATE_LOCK_SET2__ADR_CFG_VUPDATE_LOCK_SET_MASK                                              0x00000001L
+//ADR_VUPDATE_LOCK_SET2
+#define ADR_VUPDATE_LOCK_SET2__ADR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define ADR_VUPDATE_LOCK_SET2__ADR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CFG_VUPDATE_LOCK_SET2
+#define CFG_VUPDATE_LOCK_SET2__CFG_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CFG_VUPDATE_LOCK_SET2__CFG_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CUR_VUPDATE_LOCK_SET2
+#define CUR_VUPDATE_LOCK_SET2__CUR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CUR_VUPDATE_LOCK_SET2__CUR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//ADR_CFG_CUR_VUPDATE_LOCK_SET3
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET3__ADR_CFG_CUR_VUPDATE_LOCK_SET__SHIFT                                    0x0
+#define ADR_CFG_CUR_VUPDATE_LOCK_SET3__ADR_CFG_CUR_VUPDATE_LOCK_SET_MASK                                      0x00000001L
+//ADR_CFG_VUPDATE_LOCK_SET3
+#define ADR_CFG_VUPDATE_LOCK_SET3__ADR_CFG_VUPDATE_LOCK_SET__SHIFT                                            0x0
+#define ADR_CFG_VUPDATE_LOCK_SET3__ADR_CFG_VUPDATE_LOCK_SET_MASK                                              0x00000001L
+//ADR_VUPDATE_LOCK_SET3
+#define ADR_VUPDATE_LOCK_SET3__ADR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define ADR_VUPDATE_LOCK_SET3__ADR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CFG_VUPDATE_LOCK_SET3
+#define CFG_VUPDATE_LOCK_SET3__CFG_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CFG_VUPDATE_LOCK_SET3__CFG_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//CUR_VUPDATE_LOCK_SET3
+#define CUR_VUPDATE_LOCK_SET3__CUR_VUPDATE_LOCK_SET__SHIFT                                                    0x0
+#define CUR_VUPDATE_LOCK_SET3__CUR_VUPDATE_LOCK_SET_MASK                                                      0x00000001L
+//HUBP0_3DLUT_FL_CONFIG
+#define HUBP0_3DLUT_FL_CONFIG__HUBP0_3DLUT_FL_MODE__SHIFT                                                     0x0
+#define HUBP0_3DLUT_FL_CONFIG__HUBP0_3DLUT_FL_FORMAT__SHIFT                                                   0x4
+#define HUBP0_3DLUT_FL_CONFIG__HUBP0_3DLUT_FL_MODE_MASK                                                       0x00000003L
+#define HUBP0_3DLUT_FL_CONFIG__HUBP0_3DLUT_FL_FORMAT_MASK                                                     0x00000030L
+//HUBP0_3DLUT_FL_BIAS_SCALE
+#define HUBP0_3DLUT_FL_BIAS_SCALE__HUBP0_3DLUT_FL_BIAS__SHIFT                                                 0x0
+#define HUBP0_3DLUT_FL_BIAS_SCALE__HUBP0_3DLUT_FL_SCALE__SHIFT                                                0x10
+#define HUBP0_3DLUT_FL_BIAS_SCALE__HUBP0_3DLUT_FL_BIAS_MASK                                                   0x0000FFFFL
+#define HUBP0_3DLUT_FL_BIAS_SCALE__HUBP0_3DLUT_FL_SCALE_MASK                                                  0xFFFF0000L
+//HUBP1_3DLUT_FL_CONFIG
+#define HUBP1_3DLUT_FL_CONFIG__HUBP1_3DLUT_FL_MODE__SHIFT                                                     0x0
+#define HUBP1_3DLUT_FL_CONFIG__HUBP1_3DLUT_FL_FORMAT__SHIFT                                                   0x4
+#define HUBP1_3DLUT_FL_CONFIG__HUBP1_3DLUT_FL_MODE_MASK                                                       0x00000003L
+#define HUBP1_3DLUT_FL_CONFIG__HUBP1_3DLUT_FL_FORMAT_MASK                                                     0x00000030L
+//HUBP1_3DLUT_FL_BIAS_SCALE
+#define HUBP1_3DLUT_FL_BIAS_SCALE__HUBP1_3DLUT_FL_BIAS__SHIFT                                                 0x0
+#define HUBP1_3DLUT_FL_BIAS_SCALE__HUBP1_3DLUT_FL_SCALE__SHIFT                                                0x10
+#define HUBP1_3DLUT_FL_BIAS_SCALE__HUBP1_3DLUT_FL_BIAS_MASK                                                   0x0000FFFFL
+#define HUBP1_3DLUT_FL_BIAS_SCALE__HUBP1_3DLUT_FL_SCALE_MASK                                                  0xFFFF0000L
+//HUBP2_3DLUT_FL_CONFIG
+#define HUBP2_3DLUT_FL_CONFIG__HUBP2_3DLUT_FL_MODE__SHIFT                                                     0x0
+#define HUBP2_3DLUT_FL_CONFIG__HUBP2_3DLUT_FL_FORMAT__SHIFT                                                   0x4
+#define HUBP2_3DLUT_FL_CONFIG__HUBP2_3DLUT_FL_MODE_MASK                                                       0x00000003L
+#define HUBP2_3DLUT_FL_CONFIG__HUBP2_3DLUT_FL_FORMAT_MASK                                                     0x00000030L
+//HUBP2_3DLUT_FL_BIAS_SCALE
+#define HUBP2_3DLUT_FL_BIAS_SCALE__HUBP2_3DLUT_FL_BIAS__SHIFT                                                 0x0
+#define HUBP2_3DLUT_FL_BIAS_SCALE__HUBP2_3DLUT_FL_SCALE__SHIFT                                                0x10
+#define HUBP2_3DLUT_FL_BIAS_SCALE__HUBP2_3DLUT_FL_BIAS_MASK                                                   0x0000FFFFL
+#define HUBP2_3DLUT_FL_BIAS_SCALE__HUBP2_3DLUT_FL_SCALE_MASK                                                  0xFFFF0000L
+//HUBP3_3DLUT_FL_CONFIG
+#define HUBP3_3DLUT_FL_CONFIG__HUBP3_3DLUT_FL_MODE__SHIFT                                                     0x0
+#define HUBP3_3DLUT_FL_CONFIG__HUBP3_3DLUT_FL_FORMAT__SHIFT                                                   0x4
+#define HUBP3_3DLUT_FL_CONFIG__HUBP3_3DLUT_FL_MODE_MASK                                                       0x00000003L
+#define HUBP3_3DLUT_FL_CONFIG__HUBP3_3DLUT_FL_FORMAT_MASK                                                     0x00000030L
+//HUBP3_3DLUT_FL_BIAS_SCALE
+#define HUBP3_3DLUT_FL_BIAS_SCALE__HUBP3_3DLUT_FL_BIAS__SHIFT                                                 0x0
+#define HUBP3_3DLUT_FL_BIAS_SCALE__HUBP3_3DLUT_FL_SCALE__SHIFT                                                0x10
+#define HUBP3_3DLUT_FL_BIAS_SCALE__HUBP3_3DLUT_FL_BIAS_MASK                                                   0x0000FFFFL
+#define HUBP3_3DLUT_FL_BIAS_SCALE__HUBP3_3DLUT_FL_SCALE_MASK                                                  0xFFFF0000L
+//MPC_DWB0_MUX
+#define MPC_DWB0_MUX__MPC_DWB0_MUX__SHIFT                                                                     0x0
+#define MPC_DWB0_MUX__MPC_DWB0_MUX_STATUS__SHIFT                                                              0x4
+#define MPC_DWB0_MUX__MPC_DWB0_MUX_MASK                                                                       0x0000000FL
+#define MPC_DWB0_MUX__MPC_DWB0_MUX_STATUS_MASK                                                                0x000000F0L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_ON_SOCCLK__SHIFT                             0x9
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_ON_HDMISTREAMCLK__SHIFT                      0xd
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_ON_SOCCLK_MASK                               0x00000200L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_ON_HDMISTREAMCLK_MASK                        0x00002000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_TYPE__SHIFT             0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_UNCOMPRESSED_PIXEL_FORMAT__SHIFT       0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_COMPRESSED_PIXEL_FORMAT__SHIFT         0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_TYPE_MASK               0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_UNCOMPRESSED_PIXEL_FORMAT_MASK         0x00001000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_COMPRESSED_PIXEL_FORMAT_MASK           0x00010000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY__SHIFT                 0xa
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_LEVEL_DET_DELAY_MASK                   0x00000C00L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x6
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000003FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000040L
+//HDMI_STREAM_ENC_AUDIO_CONTROL
+#define HDMI_STREAM_ENC_AUDIO_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL__SHIFT               0x0
+#define HDMI_STREAM_ENC_AUDIO_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_EN__SHIFT                                    0x8
+#define HDMI_STREAM_ENC_AUDIO_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_AUDIO_STREAM_SOURCE_SEL_MASK                 0x00000007L
+#define HDMI_STREAM_ENC_AUDIO_CONTROL__HDMI_STREAM_ENC_APG_CLOCK_EN_MASK                                      0x00000100L
+
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_ACTIVE_AVMUTE__SHIFT                                                     0x8
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_ACTIVE_AVMUTE_MASK                                                       0x00000100L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_ENCRYPTION_CONTROL
+#define HDMI_TB_ENC_ENCRYPTION_CONTROL__HDMI_EESS_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_ENCRYPTION_CONTROL__HDMI_EESS_WHEN_AVMUTE__SHIFT                                          0x4
+#define HDMI_TB_ENC_ENCRYPTION_CONTROL__HDMI_EESS_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_ENCRYPTION_CONTROL__HDMI_EESS_WHEN_AVMUTE_MASK                                            0x00000010L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_apg_apg_dispdec
+//APG9_APG_DBG_GEN_CONTROL
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE__SHIFT                                                   0x0
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET__SHIFT                                                    0x1
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE__SHIFT                                         0x8
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE__SHIFT                                        0x18
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_GEN_ENABLE_MASK                                                     0x00000001L
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_GEN_RESET_MASK                                                      0x00000002L
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_CHANNEL_ENABLE_MASK                                           0x0000FF00L
+#define APG9_APG_DBG_GEN_CONTROL__APG_DBG_AUDIO_TEST_CH_DISABLE_MASK                                          0xFF000000L
+//APG9_APG_MEM_PWR
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_DIS__SHIFT                                                              0x0
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_FORCE__SHIFT                                                            0x4
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_STATE__SHIFT                                                            0x8
+#define APG9_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE__SHIFT                                              0xc
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_DIS_MASK                                                                0x00000001L
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_FORCE_MASK                                                              0x00000030L
+#define APG9_APG_MEM_PWR__APG_MEM_PWR_STATE_MASK                                                              0x00000300L
+#define APG9_APG_MEM_PWR__APG_MEM_DEFAULT_LOW_POWER_STATE_MASK                                                0x00003000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_vpg_vpg_dispdec
+//VPG9_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG9_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG9_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG9_VPG_GENERIC_PACKET_DATA
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG9_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG9_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG9_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG9_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG9_VPG_GENERIC_STATUS
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG9_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG9_VPG_MEM_PWR
+#define VPG9_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG9_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG9_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG9_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG9_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG9_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG9_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG9_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG9_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG9_VPG_ISRC1_2_DATA
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG9_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dme_dme_dispdec
+//DME9_DME_CONTROL
+#define DME9_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME9_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME9_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME9_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME9_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME9_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME9_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME9_DME_CONTROL__METADATA_TRANSMISSION_MISSED__SHIFT                                                 0x18
+#define DME9_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR__SHIFT                                             0x19
+#define DME9_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME9_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME9_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME9_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME9_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME9_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME9_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+#define DME9_DME_CONTROL__METADATA_TRANSMISSION_MISSED_MASK                                                   0x01000000L
+#define DME9_DME_CONTROL__METADATA_TRANSMISSION_MISSED_CLR_MASK                                               0x02000000L
+//DME9_DME_MEMORY_CONTROL
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME9_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hpo_top_dispdec
+//HPO_TOP_CLOCK_CONTROL
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS__SHIFT                                                  0x0
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS__SHIFT                                                    0x1
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS__SHIFT                                                     0x5
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS__SHIFT                                            0x8
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_G_GATE_DIS__SHIFT                                            0x9
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS__SHIFT                                              0xc
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_G_GATE_DIS__SHIFT                                              0xd
+#define HPO_TOP_CLOCK_CONTROL__HPO_DPSTREAMCLK_R_GATE_DIS__SHIFT                                              0x10
+#define HPO_TOP_CLOCK_CONTROL__HPO_DPSTREAMCLK_G_GATE_DIS__SHIFT                                              0x11
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_SE_R_GATE_DIS__SHIFT                                              0x12
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_SE_G_GATE_DIS__SHIFT                                              0x13
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_LE_R_GATE_DIS__SHIFT                                              0x14
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_LE_G_GATE_DIS__SHIFT                                              0x15
+#define HPO_TOP_CLOCK_CONTROL__HPO_FGCG_REP_DIS__SHIFT                                                        0x17
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL__SHIFT                                                        0x18
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS_MASK                                                    0x00000001L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS_MASK                                                      0x00000002L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS_MASK                                                       0x00000020L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS_MASK                                              0x00000100L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_G_GATE_DIS_MASK                                              0x00000200L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS_MASK                                                0x00001000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_G_GATE_DIS_MASK                                                0x00002000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DPSTREAMCLK_R_GATE_DIS_MASK                                                0x00010000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DPSTREAMCLK_G_GATE_DIS_MASK                                                0x00020000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_SE_R_GATE_DIS_MASK                                                0x00040000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_SE_G_GATE_DIS_MASK                                                0x00080000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_LE_R_GATE_DIS_MASK                                                0x00100000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SYMCLK32_LE_G_GATE_DIS_MASK                                                0x00200000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_FGCG_REP_DIS_MASK                                                          0x00800000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL_MASK                                                          0xFF000000L
+//HPO_TOP_HW_CONTROL
+#define HPO_TOP_HW_CONTROL__HPO_IO_EN__SHIFT                                                                  0x0
+#define HPO_TOP_HW_CONTROL__HPO_IO_EN_MASK                                                                    0x00000001L
+
+
+// addressBlock: dce_dc_hpo_dp_stream_mapper_dispdec
+//DP_STREAM_MAPPER_CONTROL0
+#define DP_STREAM_MAPPER_CONTROL0__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
+#define DP_STREAM_MAPPER_CONTROL0__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
+//DP_STREAM_MAPPER_CONTROL1
+#define DP_STREAM_MAPPER_CONTROL1__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
+#define DP_STREAM_MAPPER_CONTROL1__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
+//DP_STREAM_MAPPER_CONTROL2
+#define DP_STREAM_MAPPER_CONTROL2__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
+#define DP_STREAM_MAPPER_CONTROL2__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
+//DP_STREAM_MAPPER_CONTROL3
+#define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
+#define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
+
+
+// addressBlock: dce_dc_hpo_hpo_dcperfmon_dc_perfmon_dispdec
+//DC_PERFMON23_PERFCOUNTER_CNTL
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL__SHIFT                                           0x0
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL__SHIFT                                          0x9
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE__SHIFT                                            0xc
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL__SHIFT                                         0xf
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE__SHIFT                                          0x10
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS__SHIFT                                    0x16
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN__SHIFT                                          0x17
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN__SHIFT                                              0x18
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK__SHIFT                                            0x19
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE__SHIFT                                              0x1a
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL__SHIFT                                            0x1d
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_EVENT_SEL_MASK                                             0x000001FFL
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CVALUE_SEL_MASK                                            0x00000E00L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_INC_MODE_MASK                                              0x00007000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_HW_CNTL_SEL_MASK                                           0x00008000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_RUNEN_MODE_MASK                                            0x00010000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CNTOFF_START_DIS_MASK                                      0x00400000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_RESTART_EN_MASK                                            0x00800000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_INT_EN_MASK                                                0x01000000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_OFF_MASK_MASK                                              0x02000000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_ACTIVE_MASK                                                0x04000000L
+#define DC_PERFMON23_PERFCOUNTER_CNTL__PERFCOUNTER_CNTL_SEL_MASK                                              0xE0000000L
+//DC_PERFMON23_PERFCOUNTER_CNTL2
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE__SHIFT                                 0x0
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL__SHIFT                                       0x2
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL__SHIFT                                       0x3
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL__SHIFT                                         0x8
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL__SHIFT                                          0x1d
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_COUNTED_VALUE_TYPE_MASK                                   0x00000003L
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP1_SEL_MASK                                         0x00000004L
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_HW_STOP2_SEL_MASK                                         0x00000008L
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTOFF_SEL_MASK                                           0x00003F00L
+#define DC_PERFMON23_PERFCOUNTER_CNTL2__PERFCOUNTER_CNTL2_SEL_MASK                                            0xE0000000L
+//DC_PERFMON23_PERFCOUNTER_STATE
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE__SHIFT                                         0x0
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0__SHIFT                                         0x2
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE__SHIFT                                         0x4
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1__SHIFT                                         0x6
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE__SHIFT                                         0x8
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2__SHIFT                                         0xa
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE__SHIFT                                         0xc
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3__SHIFT                                         0xe
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE__SHIFT                                         0x10
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4__SHIFT                                         0x12
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE__SHIFT                                         0x14
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5__SHIFT                                         0x16
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE__SHIFT                                         0x18
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6__SHIFT                                         0x1a
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE__SHIFT                                         0x1c
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7__SHIFT                                         0x1e
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT0_STATE_MASK                                           0x00000003L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL0_MASK                                           0x00000004L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT1_STATE_MASK                                           0x00000030L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL1_MASK                                           0x00000040L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT2_STATE_MASK                                           0x00000300L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL2_MASK                                           0x00000400L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT3_STATE_MASK                                           0x00003000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL3_MASK                                           0x00004000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT4_STATE_MASK                                           0x00030000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL4_MASK                                           0x00040000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT5_STATE_MASK                                           0x00300000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL5_MASK                                           0x00400000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT6_STATE_MASK                                           0x03000000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL6_MASK                                           0x04000000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_CNT7_STATE_MASK                                           0x30000000L
+#define DC_PERFMON23_PERFCOUNTER_STATE__PERFCOUNTER_STATE_SEL7_MASK                                           0x40000000L
+//DC_PERFMON23_PERFMON_CNTL
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                       0x0
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_RPT_COUNT__SHIFT                                                   0x8
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR__SHIFT                                               0x1c
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN__SHIFT                                               0x1d
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS__SHIFT                                           0x1e
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK__SHIFT                                              0x1f
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_STATE_MASK                                                         0x00000003L
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_RPT_COUNT_MASK                                                     0x0FFFFF00L
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_AND_OR_MASK                                                 0x10000000L
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_EN_MASK                                                 0x20000000L
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_STATUS_MASK                                             0x40000000L
+#define DC_PERFMON23_PERFMON_CNTL__PERFMON_CNTOFF_INT_ACK_MASK                                                0x80000000L
+//DC_PERFMON23_PERFMON_CNTL2
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE__SHIFT                                            0x0
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_CLK_ENABLE__SHIFT                                                 0x1
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL__SHIFT                                       0x2
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL__SHIFT                                        0xa
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_CNTOFF_INT_TYPE_MASK                                              0x00000001L
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_CLK_ENABLE_MASK                                                   0x00000002L
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_RUN_ENABLE_START_SEL_MASK                                         0x000003FCL
+#define DC_PERFMON23_PERFMON_CNTL2__PERFMON_RUN_ENABLE_STOP_SEL_MASK                                          0x0003FC00L
+//DC_PERFMON23_PERFMON_CVALUE_INT_MISC
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS__SHIFT                                  0x0
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS__SHIFT                                  0x1
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS__SHIFT                                  0x2
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS__SHIFT                                  0x3
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS__SHIFT                                  0x4
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS__SHIFT                                  0x5
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS__SHIFT                                  0x6
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS__SHIFT                                  0x7
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK__SHIFT                                     0x8
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK__SHIFT                                     0x9
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK__SHIFT                                     0xa
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK__SHIFT                                     0xb
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK__SHIFT                                     0xc
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK__SHIFT                                     0xd
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK__SHIFT                                     0xe
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK__SHIFT                                     0xf
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI__SHIFT                                        0x10
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_STATUS_MASK                                    0x00000001L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_STATUS_MASK                                    0x00000002L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_STATUS_MASK                                    0x00000004L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_STATUS_MASK                                    0x00000008L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_STATUS_MASK                                    0x00000010L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_STATUS_MASK                                    0x00000020L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_STATUS_MASK                                    0x00000040L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_STATUS_MASK                                    0x00000080L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT0_ACK_MASK                                       0x00000100L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT1_ACK_MASK                                       0x00000200L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT2_ACK_MASK                                       0x00000400L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT3_ACK_MASK                                       0x00000800L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT4_ACK_MASK                                       0x00001000L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT5_ACK_MASK                                       0x00002000L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT6_ACK_MASK                                       0x00004000L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFCOUNTER_INT7_ACK_MASK                                       0x00008000L
+#define DC_PERFMON23_PERFMON_CVALUE_INT_MISC__PERFMON_CVALUE_HI_MASK                                          0xFFFF0000L
+//DC_PERFMON23_PERFMON_CVALUE_LOW
+#define DC_PERFMON23_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW__SHIFT                                            0x0
+#define DC_PERFMON23_PERFMON_CVALUE_LOW__PERFMON_CVALUE_LOW_MASK                                              0xFFFFFFFFL
+//DC_PERFMON23_PERFMON_HI
+#define DC_PERFMON23_PERFMON_HI__PERFMON_HI__SHIFT                                                            0x0
+#define DC_PERFMON23_PERFMON_HI__PERFMON_READ_SEL__SHIFT                                                      0x1d
+#define DC_PERFMON23_PERFMON_HI__PERFMON_HI_MASK                                                              0x0000FFFFL
+#define DC_PERFMON23_PERFMON_HI__PERFMON_READ_SEL_MASK                                                        0xE0000000L
+//DC_PERFMON23_PERFMON_LOW
+#define DC_PERFMON23_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
+#define DC_PERFMON23_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_opp_abm0_dispdec
+//ABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL
+#define ABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL__SHIFT                                  0x0
+#define ABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL_MASK                                    0x0001FFFFL
+//ABM0_BL1_PWM_USER_LEVEL
+#define ABM0_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL__SHIFT                                                    0x0
+#define ABM0_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL_MASK                                                      0x0001FFFFL
+//ABM0_BL1_PWM_TARGET_ABM_LEVEL
+#define ABM0_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL__SHIFT                                        0x0
+#define ABM0_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL_MASK                                          0x0001FFFFL
+//ABM0_BL1_PWM_CURRENT_ABM_LEVEL
+#define ABM0_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL__SHIFT                                      0x0
+#define ABM0_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL_MASK                                        0x0001FFFFL
+//ABM0_BL1_PWM_FINAL_DUTY_CYCLE
+#define ABM0_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE__SHIFT                                        0x0
+#define ABM0_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE_MASK                                          0x0001FFFFL
+//ABM0_BL1_PWM_MINIMUM_DUTY_CYCLE
+#define ABM0_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE__SHIFT                                    0x0
+#define ABM0_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE_MASK                                      0x0001FFFFL
+//ABM0_BL1_PWM_ABM_CNTL
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN__SHIFT                                                      0x0
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN__SHIFT                                            0x1
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN__SHIFT                                0x2
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN__SHIFT                                   0x3
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE__SHIFT                               0x10
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN_MASK                                                        0x00000001L
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN_MASK                                              0x00000002L
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN_MASK                                  0x00000004L
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN_MASK                                     0x00000008L
+#define ABM0_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE_MASK                                 0xFFFF0000L
+//ABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE
+#define ABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN__SHIFT                     0x0
+#define ABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT__SHIFT                  0x8
+#define ABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN_MASK                       0x00000001L
+#define ABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT_MASK                    0x0000FF00L
+//ABM0_DC_ABM1_CNTL
+#define ABM0_DC_ABM1_CNTL__ABM1_EN__SHIFT                                                                     0x0
+#define ABM0_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS__SHIFT                                                      0x4
+#define ABM0_DC_ABM1_CNTL__ABM1_EN_MASK                                                                       0x00000001L
+#define ABM0_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS_MASK                                                        0x00000010L
+//ABM0_DC_ABM1_IPCSC_COEFF_SEL
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B__SHIFT                                           0x0
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G__SHIFT                                           0xa
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R__SHIFT                                           0x14
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B_MASK                                             0x000001FFL
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G_MASK                                             0x0007FC00L
+#define ABM0_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R_MASK                                             0x1FF00000L
+//ABM0_DC_ABM1_ACE_PWL_CNTL
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX__SHIFT                                         0x0
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN__SHIFT                                      0x1c
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN__SHIFT                                   0x1d
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING__SHIFT                                    0x1e
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK__SHIFT                                                       0x1f
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX_MASK                                           0x0000003FL
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN_MASK                                        0x10000000L
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN_MASK                                     0x20000000L
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING_MASK                                      0x40000000L
+#define ABM0_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK_MASK                                                         0x80000000L
+//ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA
+#define ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA__SHIFT                                        0x0
+#define ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA__SHIFT                                       0x10
+#define ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA_MASK                                          0x00000FFFL
+#define ABM0_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA_MASK                                         0x0FFF0000L
+//ABM0_DC_ABM1_ACE_CNTL_MISC
+#define ABM0_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME__SHIFT                                       0x0
+#define ABM0_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR__SHIFT                                 0x8
+#define ABM0_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_MASK                                         0x00000001L
+#define ABM0_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR_MASK                                   0x00000100L
+//ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS__SHIFT                              0x0
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS__SHIFT                              0x1
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS__SHIFT                              0x2
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME__SHIFT                             0x8
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME__SHIFT                             0x9
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME__SHIFT                             0xa
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x10
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x18
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x1f
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS_MASK                                0x00000001L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS_MASK                                0x00000002L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS_MASK                                0x00000004L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_MASK                               0x00000100L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_MASK                               0x00000200L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_MASK                               0x00000400L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x00010000L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x01000000L
+#define ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x80000000L
+//ABM0_DC_ABM1_HG_MISC_CTRL
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL__SHIFT                                             0x0
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL__SHIFT                                                    0x8
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION__SHIFT                                              0xe
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL__SHIFT                                       0x10
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN__SHIFT                                      0x14
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL_MASK                                               0x00000003L
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL_MASK                                                      0x00000100L
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION_MASK                                                0x00004000L
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL_MASK                                         0x00030000L
+#define ABM0_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN_MASK                                        0x00100000L
+//ABM0_DC_ABM1_LS_SUM_OF_LUMA
+#define ABM0_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA__SHIFT                                               0x0
+#define ABM0_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA_MASK                                                 0xFFFFFFFFL
+//ABM0_DC_ABM1_LS_SUM_OF_LUMA_MSB
+#define ABM0_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB__SHIFT                                       0x0
+#define ABM0_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB_MASK                                         0x000003FFL
+//ABM0_DC_ABM1_LS_MIN_MAX_LUMA
+#define ABM0_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA__SHIFT                                                 0x0
+#define ABM0_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA__SHIFT                                                 0x10
+#define ABM0_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA_MASK                                                   0x00000FFFL
+#define ABM0_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA_MASK                                                   0x0FFF0000L
+//ABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA
+#define ABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA__SHIFT                               0x0
+#define ABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA__SHIFT                               0x10
+#define ABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA_MASK                                 0x00000FFFL
+#define ABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA_MASK                                 0x0FFF0000L
+//ABM0_DC_ABM1_LS_PIXEL_COUNT
+#define ABM0_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT__SHIFT                                               0x0
+#define ABM0_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT_MASK                                                 0x00FFFFFFL
+//ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES
+#define ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES__SHIFT                       0x0
+#define ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES__SHIFT                       0x10
+#define ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES_MASK                         0x00000FFFL
+#define ABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES_MASK                         0x0FFF0000L
+//ABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT
+#define ABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT
+#define ABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM0_DC_ABM1_HG_SAMPLE_RATE
+#define ABM0_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM0_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM0_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM0_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM0_DC_ABM1_LS_SAMPLE_RATE
+#define ABM0_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM0_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM0_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM0_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG
+#define ABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG__SHIFT                               0x0
+#define ABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG_MASK                                 0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG
+#define ABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG__SHIFT                             0x0
+#define ABM0_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG_MASK                               0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX__SHIFT                               0x0
+#define ABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX_MASK                                 0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX__SHIFT                             0x0
+#define ABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX_MASK                               0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX
+#define ABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX__SHIFT                           0x0
+#define ABM0_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM0_DC_ABM1_HG_RESULT_INDEX
+#define ABM0_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX__SHIFT                                             0x0
+#define ABM0_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX_MASK                                               0x0000003FL
+//ABM0_DC_ABM1_HG_RESULT_DATA
+#define ABM0_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA__SHIFT                                               0x0
+#define ABM0_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA_MASK                                                 0xFFFFFFFFL
+//ABM0_DC_ABM1_BL_MASTER_LOCK
+#define ABM0_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK__SHIFT                                               0x1f
+#define ABM0_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK_MASK                                                 0x80000000L
+
+
+// addressBlock: dce_dc_opp_abm1_dispdec
+//ABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL
+#define ABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL__SHIFT                                  0x0
+#define ABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL_MASK                                    0x0001FFFFL
+//ABM1_BL1_PWM_USER_LEVEL
+#define ABM1_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL__SHIFT                                                    0x0
+#define ABM1_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL_MASK                                                      0x0001FFFFL
+//ABM1_BL1_PWM_TARGET_ABM_LEVEL
+#define ABM1_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL__SHIFT                                        0x0
+#define ABM1_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL_MASK                                          0x0001FFFFL
+//ABM1_BL1_PWM_CURRENT_ABM_LEVEL
+#define ABM1_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL__SHIFT                                      0x0
+#define ABM1_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL_MASK                                        0x0001FFFFL
+//ABM1_BL1_PWM_FINAL_DUTY_CYCLE
+#define ABM1_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE__SHIFT                                        0x0
+#define ABM1_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE_MASK                                          0x0001FFFFL
+//ABM1_BL1_PWM_MINIMUM_DUTY_CYCLE
+#define ABM1_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE__SHIFT                                    0x0
+#define ABM1_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE_MASK                                      0x0001FFFFL
+//ABM1_BL1_PWM_ABM_CNTL
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN__SHIFT                                                      0x0
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN__SHIFT                                            0x1
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN__SHIFT                                0x2
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN__SHIFT                                   0x3
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE__SHIFT                               0x10
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN_MASK                                                        0x00000001L
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN_MASK                                              0x00000002L
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN_MASK                                  0x00000004L
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN_MASK                                     0x00000008L
+#define ABM1_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE_MASK                                 0xFFFF0000L
+//ABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE
+#define ABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN__SHIFT                     0x0
+#define ABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT__SHIFT                  0x8
+#define ABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN_MASK                       0x00000001L
+#define ABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT_MASK                    0x0000FF00L
+//ABM1_DC_ABM1_CNTL
+#define ABM1_DC_ABM1_CNTL__ABM1_EN__SHIFT                                                                     0x0
+#define ABM1_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS__SHIFT                                                      0x4
+#define ABM1_DC_ABM1_CNTL__ABM1_EN_MASK                                                                       0x00000001L
+#define ABM1_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS_MASK                                                        0x00000010L
+//ABM1_DC_ABM1_IPCSC_COEFF_SEL
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B__SHIFT                                           0x0
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G__SHIFT                                           0xa
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R__SHIFT                                           0x14
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B_MASK                                             0x000001FFL
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G_MASK                                             0x0007FC00L
+#define ABM1_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R_MASK                                             0x1FF00000L
+//ABM1_DC_ABM1_ACE_PWL_CNTL
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX__SHIFT                                         0x0
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN__SHIFT                                      0x1c
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN__SHIFT                                   0x1d
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING__SHIFT                                    0x1e
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK__SHIFT                                                       0x1f
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX_MASK                                           0x0000003FL
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN_MASK                                        0x10000000L
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN_MASK                                     0x20000000L
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING_MASK                                      0x40000000L
+#define ABM1_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK_MASK                                                         0x80000000L
+//ABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA
+#define ABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA__SHIFT                                        0x0
+#define ABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA__SHIFT                                       0x10
+#define ABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA_MASK                                          0x00000FFFL
+#define ABM1_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA_MASK                                         0x0FFF0000L
+//ABM1_DC_ABM1_ACE_CNTL_MISC
+#define ABM1_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME__SHIFT                                       0x0
+#define ABM1_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR__SHIFT                                 0x8
+#define ABM1_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_MASK                                         0x00000001L
+#define ABM1_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR_MASK                                   0x00000100L
+//ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS__SHIFT                              0x0
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS__SHIFT                              0x1
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS__SHIFT                              0x2
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME__SHIFT                             0x8
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME__SHIFT                             0x9
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME__SHIFT                             0xa
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x10
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x18
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x1f
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS_MASK                                0x00000001L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS_MASK                                0x00000002L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS_MASK                                0x00000004L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_MASK                               0x00000100L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_MASK                               0x00000200L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_MASK                               0x00000400L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x00010000L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x01000000L
+#define ABM1_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x80000000L
+//ABM1_DC_ABM1_HG_MISC_CTRL
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL__SHIFT                                             0x0
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL__SHIFT                                                    0x8
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION__SHIFT                                              0xe
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL__SHIFT                                       0x10
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN__SHIFT                                      0x14
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL_MASK                                               0x00000003L
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL_MASK                                                      0x00000100L
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION_MASK                                                0x00004000L
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL_MASK                                         0x00030000L
+#define ABM1_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN_MASK                                        0x00100000L
+//ABM1_DC_ABM1_LS_SUM_OF_LUMA
+#define ABM1_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA__SHIFT                                               0x0
+#define ABM1_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA_MASK                                                 0xFFFFFFFFL
+//ABM1_DC_ABM1_LS_SUM_OF_LUMA_MSB
+#define ABM1_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB__SHIFT                                       0x0
+#define ABM1_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB_MASK                                         0x000003FFL
+//ABM1_DC_ABM1_LS_MIN_MAX_LUMA
+#define ABM1_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA__SHIFT                                                 0x0
+#define ABM1_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA__SHIFT                                                 0x10
+#define ABM1_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA_MASK                                                   0x00000FFFL
+#define ABM1_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA_MASK                                                   0x0FFF0000L
+//ABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA
+#define ABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA__SHIFT                               0x0
+#define ABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA__SHIFT                               0x10
+#define ABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA_MASK                                 0x00000FFFL
+#define ABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA_MASK                                 0x0FFF0000L
+//ABM1_DC_ABM1_LS_PIXEL_COUNT
+#define ABM1_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT__SHIFT                                               0x0
+#define ABM1_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT_MASK                                                 0x00FFFFFFL
+//ABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES
+#define ABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES__SHIFT                       0x0
+#define ABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES__SHIFT                       0x10
+#define ABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES_MASK                         0x00000FFFL
+#define ABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES_MASK                         0x0FFF0000L
+//ABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT
+#define ABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT
+#define ABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM1_DC_ABM1_HG_SAMPLE_RATE
+#define ABM1_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM1_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM1_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM1_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM1_DC_ABM1_LS_SAMPLE_RATE
+#define ABM1_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM1_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM1_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM1_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG
+#define ABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG__SHIFT                               0x0
+#define ABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG_MASK                                 0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG
+#define ABM1_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG__SHIFT                             0x0
+#define ABM1_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG_MASK                               0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX__SHIFT                               0x0
+#define ABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX_MASK                                 0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX__SHIFT                             0x0
+#define ABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX_MASK                               0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX
+#define ABM1_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX__SHIFT                           0x0
+#define ABM1_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM1_DC_ABM1_HG_RESULT_INDEX
+#define ABM1_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX__SHIFT                                             0x0
+#define ABM1_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX_MASK                                               0x0000003FL
+//ABM1_DC_ABM1_HG_RESULT_DATA
+#define ABM1_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA__SHIFT                                               0x0
+#define ABM1_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA_MASK                                                 0xFFFFFFFFL
+//ABM1_DC_ABM1_BL_MASTER_LOCK
+#define ABM1_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK__SHIFT                                               0x1f
+#define ABM1_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK_MASK                                                 0x80000000L
+
+
+// addressBlock: dce_dc_opp_abm2_dispdec
+//ABM2_BL1_PWM_AMBIENT_LIGHT_LEVEL
+#define ABM2_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL__SHIFT                                  0x0
+#define ABM2_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL_MASK                                    0x0001FFFFL
+//ABM2_BL1_PWM_USER_LEVEL
+#define ABM2_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL__SHIFT                                                    0x0
+#define ABM2_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL_MASK                                                      0x0001FFFFL
+//ABM2_BL1_PWM_TARGET_ABM_LEVEL
+#define ABM2_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL__SHIFT                                        0x0
+#define ABM2_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL_MASK                                          0x0001FFFFL
+//ABM2_BL1_PWM_CURRENT_ABM_LEVEL
+#define ABM2_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL__SHIFT                                      0x0
+#define ABM2_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL_MASK                                        0x0001FFFFL
+//ABM2_BL1_PWM_FINAL_DUTY_CYCLE
+#define ABM2_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE__SHIFT                                        0x0
+#define ABM2_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE_MASK                                          0x0001FFFFL
+//ABM2_BL1_PWM_MINIMUM_DUTY_CYCLE
+#define ABM2_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE__SHIFT                                    0x0
+#define ABM2_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE_MASK                                      0x0001FFFFL
+//ABM2_BL1_PWM_ABM_CNTL
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN__SHIFT                                                      0x0
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN__SHIFT                                            0x1
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN__SHIFT                                0x2
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN__SHIFT                                   0x3
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE__SHIFT                               0x10
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN_MASK                                                        0x00000001L
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN_MASK                                              0x00000002L
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN_MASK                                  0x00000004L
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN_MASK                                     0x00000008L
+#define ABM2_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE_MASK                                 0xFFFF0000L
+//ABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE
+#define ABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN__SHIFT                     0x0
+#define ABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT__SHIFT                  0x8
+#define ABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN_MASK                       0x00000001L
+#define ABM2_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT_MASK                    0x0000FF00L
+//ABM2_DC_ABM1_CNTL
+#define ABM2_DC_ABM1_CNTL__ABM1_EN__SHIFT                                                                     0x0
+#define ABM2_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS__SHIFT                                                      0x4
+#define ABM2_DC_ABM1_CNTL__ABM1_EN_MASK                                                                       0x00000001L
+#define ABM2_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS_MASK                                                        0x00000010L
+//ABM2_DC_ABM1_IPCSC_COEFF_SEL
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B__SHIFT                                           0x0
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G__SHIFT                                           0xa
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R__SHIFT                                           0x14
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B_MASK                                             0x000001FFL
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G_MASK                                             0x0007FC00L
+#define ABM2_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R_MASK                                             0x1FF00000L
+//ABM2_DC_ABM1_ACE_PWL_CNTL
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX__SHIFT                                         0x0
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN__SHIFT                                      0x1c
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN__SHIFT                                   0x1d
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING__SHIFT                                    0x1e
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK__SHIFT                                                       0x1f
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX_MASK                                           0x0000003FL
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN_MASK                                        0x10000000L
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN_MASK                                     0x20000000L
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING_MASK                                      0x40000000L
+#define ABM2_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK_MASK                                                         0x80000000L
+//ABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA
+#define ABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA__SHIFT                                        0x0
+#define ABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA__SHIFT                                       0x10
+#define ABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA_MASK                                          0x00000FFFL
+#define ABM2_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA_MASK                                         0x0FFF0000L
+//ABM2_DC_ABM1_ACE_CNTL_MISC
+#define ABM2_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME__SHIFT                                       0x0
+#define ABM2_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR__SHIFT                                 0x8
+#define ABM2_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_MASK                                         0x00000001L
+#define ABM2_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR_MASK                                   0x00000100L
+//ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS__SHIFT                              0x0
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS__SHIFT                              0x1
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS__SHIFT                              0x2
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME__SHIFT                             0x8
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME__SHIFT                             0x9
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME__SHIFT                             0xa
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x10
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x18
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x1f
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS_MASK                                0x00000001L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS_MASK                                0x00000002L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS_MASK                                0x00000004L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_MASK                               0x00000100L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_MASK                               0x00000200L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_MASK                               0x00000400L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x00010000L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x01000000L
+#define ABM2_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x80000000L
+//ABM2_DC_ABM1_HG_MISC_CTRL
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL__SHIFT                                             0x0
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL__SHIFT                                                    0x8
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION__SHIFT                                              0xe
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL__SHIFT                                       0x10
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN__SHIFT                                      0x14
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL_MASK                                               0x00000003L
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL_MASK                                                      0x00000100L
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION_MASK                                                0x00004000L
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL_MASK                                         0x00030000L
+#define ABM2_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN_MASK                                        0x00100000L
+//ABM2_DC_ABM1_LS_SUM_OF_LUMA
+#define ABM2_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA__SHIFT                                               0x0
+#define ABM2_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA_MASK                                                 0xFFFFFFFFL
+//ABM2_DC_ABM1_LS_SUM_OF_LUMA_MSB
+#define ABM2_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB__SHIFT                                       0x0
+#define ABM2_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB_MASK                                         0x000003FFL
+//ABM2_DC_ABM1_LS_MIN_MAX_LUMA
+#define ABM2_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA__SHIFT                                                 0x0
+#define ABM2_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA__SHIFT                                                 0x10
+#define ABM2_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA_MASK                                                   0x00000FFFL
+#define ABM2_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA_MASK                                                   0x0FFF0000L
+//ABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA
+#define ABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA__SHIFT                               0x0
+#define ABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA__SHIFT                               0x10
+#define ABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA_MASK                                 0x00000FFFL
+#define ABM2_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA_MASK                                 0x0FFF0000L
+//ABM2_DC_ABM1_LS_PIXEL_COUNT
+#define ABM2_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT__SHIFT                                               0x0
+#define ABM2_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT_MASK                                                 0x00FFFFFFL
+//ABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES
+#define ABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES__SHIFT                       0x0
+#define ABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES__SHIFT                       0x10
+#define ABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES_MASK                         0x00000FFFL
+#define ABM2_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES_MASK                         0x0FFF0000L
+//ABM2_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT
+#define ABM2_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM2_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM2_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT
+#define ABM2_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM2_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM2_DC_ABM1_HG_SAMPLE_RATE
+#define ABM2_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM2_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM2_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM2_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM2_DC_ABM1_LS_SAMPLE_RATE
+#define ABM2_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM2_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM2_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM2_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM2_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG
+#define ABM2_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG__SHIFT                               0x0
+#define ABM2_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG_MASK                                 0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG
+#define ABM2_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG__SHIFT                             0x0
+#define ABM2_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG_MASK                               0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX__SHIFT                               0x0
+#define ABM2_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX_MASK                                 0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX__SHIFT                             0x0
+#define ABM2_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX_MASK                               0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX
+#define ABM2_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX__SHIFT                           0x0
+#define ABM2_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM2_DC_ABM1_HG_RESULT_INDEX
+#define ABM2_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX__SHIFT                                             0x0
+#define ABM2_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX_MASK                                               0x0000003FL
+//ABM2_DC_ABM1_HG_RESULT_DATA
+#define ABM2_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA__SHIFT                                               0x0
+#define ABM2_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA_MASK                                                 0xFFFFFFFFL
+//ABM2_DC_ABM1_BL_MASTER_LOCK
+#define ABM2_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK__SHIFT                                               0x1f
+#define ABM2_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK_MASK                                                 0x80000000L
+
+
+// addressBlock: dce_dc_opp_abm3_dispdec
+//ABM3_BL1_PWM_AMBIENT_LIGHT_LEVEL
+#define ABM3_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL__SHIFT                                  0x0
+#define ABM3_BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL_MASK                                    0x0001FFFFL
+//ABM3_BL1_PWM_USER_LEVEL
+#define ABM3_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL__SHIFT                                                    0x0
+#define ABM3_BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL_MASK                                                      0x0001FFFFL
+//ABM3_BL1_PWM_TARGET_ABM_LEVEL
+#define ABM3_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL__SHIFT                                        0x0
+#define ABM3_BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL_MASK                                          0x0001FFFFL
+//ABM3_BL1_PWM_CURRENT_ABM_LEVEL
+#define ABM3_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL__SHIFT                                      0x0
+#define ABM3_BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL_MASK                                        0x0001FFFFL
+//ABM3_BL1_PWM_FINAL_DUTY_CYCLE
+#define ABM3_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE__SHIFT                                        0x0
+#define ABM3_BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE_MASK                                          0x0001FFFFL
+//ABM3_BL1_PWM_MINIMUM_DUTY_CYCLE
+#define ABM3_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE__SHIFT                                    0x0
+#define ABM3_BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE_MASK                                      0x0001FFFFL
+//ABM3_BL1_PWM_ABM_CNTL
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN__SHIFT                                                      0x0
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN__SHIFT                                            0x1
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN__SHIFT                                0x2
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN__SHIFT                                   0x3
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE__SHIFT                               0x10
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN_MASK                                                        0x00000001L
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN_MASK                                              0x00000002L
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN_MASK                                  0x00000004L
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN_MASK                                     0x00000008L
+#define ABM3_BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE_MASK                                 0xFFFF0000L
+//ABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE
+#define ABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN__SHIFT                     0x0
+#define ABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT__SHIFT                  0x8
+#define ABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN_MASK                       0x00000001L
+#define ABM3_BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT_MASK                    0x0000FF00L
+//ABM3_DC_ABM1_CNTL
+#define ABM3_DC_ABM1_CNTL__ABM1_EN__SHIFT                                                                     0x0
+#define ABM3_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS__SHIFT                                                      0x4
+#define ABM3_DC_ABM1_CNTL__ABM1_EN_MASK                                                                       0x00000001L
+#define ABM3_DC_ABM1_CNTL__ABM1_PROCESSING_BYPASS_MASK                                                        0x00000010L
+//ABM3_DC_ABM1_IPCSC_COEFF_SEL
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B__SHIFT                                           0x0
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G__SHIFT                                           0xa
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R__SHIFT                                           0x14
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B_MASK                                             0x000001FFL
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G_MASK                                             0x0007FC00L
+#define ABM3_DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R_MASK                                             0x1FF00000L
+//ABM3_DC_ABM1_ACE_PWL_CNTL
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX__SHIFT                                         0x0
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN__SHIFT                                      0x1c
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN__SHIFT                                   0x1d
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING__SHIFT                                    0x1e
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK__SHIFT                                                       0x1f
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_OFFSET_SLOPE_INDEX_MASK                                           0x0000003FL
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_IGNORE_MASTER_LOCK_EN_MASK                                        0x10000000L
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_READBACK_DB_REG_VALUE_EN_MASK                                     0x20000000L
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_DBUF_REG_UPDATE_PENDING_MASK                                      0x40000000L
+#define ABM3_DC_ABM1_ACE_PWL_CNTL__ABM1_ACE_LOCK_MASK                                                         0x80000000L
+//ABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA
+#define ABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA__SHIFT                                        0x0
+#define ABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA__SHIFT                                       0x10
+#define ABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_SLOPE_DATA_MASK                                          0x00000FFFL
+#define ABM3_DC_ABM1_ACE_OFFSET_SLOPE_DATA__ABM1_ACE_OFFSET_DATA_MASK                                         0x0FFF0000L
+//ABM3_DC_ABM1_ACE_CNTL_MISC
+#define ABM3_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME__SHIFT                                       0x0
+#define ABM3_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR__SHIFT                                 0x8
+#define ABM3_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_MASK                                         0x00000001L
+#define ABM3_DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR_MASK                                   0x00000100L
+//ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS__SHIFT                              0x0
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS__SHIFT                              0x1
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS__SHIFT                              0x2
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME__SHIFT                             0x8
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME__SHIFT                             0x9
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME__SHIFT                             0xa
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x10
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x18
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR__SHIFT                       0x1f
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS_MASK                                0x00000001L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS_MASK                                0x00000002L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS_MASK                                0x00000004L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_MASK                               0x00000100L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_MASK                               0x00000200L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_MASK                               0x00000400L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x00010000L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x01000000L
+#define ABM3_DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR_MASK                         0x80000000L
+//ABM3_DC_ABM1_HG_MISC_CTRL
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL__SHIFT                                             0x0
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL__SHIFT                                                    0x8
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION__SHIFT                                              0xe
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL__SHIFT                                       0x10
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN__SHIFT                                      0x14
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL_MASK                                               0x00000003L
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL_MASK                                                      0x00000100L
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_LATCH_POSITION_MASK                                                0x00004000L
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL_MASK                                         0x00030000L
+#define ABM3_DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN_MASK                                        0x00100000L
+//ABM3_DC_ABM1_LS_SUM_OF_LUMA
+#define ABM3_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA__SHIFT                                               0x0
+#define ABM3_DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA_MASK                                                 0xFFFFFFFFL
+//ABM3_DC_ABM1_LS_SUM_OF_LUMA_MSB
+#define ABM3_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB__SHIFT                                       0x0
+#define ABM3_DC_ABM1_LS_SUM_OF_LUMA_MSB__ABM1_LS_SUM_OF_LUMA_MSB_MASK                                         0x000003FFL
+//ABM3_DC_ABM1_LS_MIN_MAX_LUMA
+#define ABM3_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA__SHIFT                                                 0x0
+#define ABM3_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA__SHIFT                                                 0x10
+#define ABM3_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA_MASK                                                   0x00000FFFL
+#define ABM3_DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA_MASK                                                   0x0FFF0000L
+//ABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA
+#define ABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA__SHIFT                               0x0
+#define ABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA__SHIFT                               0x10
+#define ABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA_MASK                                 0x00000FFFL
+#define ABM3_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA_MASK                                 0x0FFF0000L
+//ABM3_DC_ABM1_LS_PIXEL_COUNT
+#define ABM3_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT__SHIFT                                               0x0
+#define ABM3_DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT_MASK                                                 0x00FFFFFFL
+//ABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES
+#define ABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES__SHIFT                       0x0
+#define ABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES__SHIFT                       0x10
+#define ABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES_MASK                         0x00000FFFL
+#define ABM3_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES_MASK                         0x0FFF0000L
+//ABM3_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT
+#define ABM3_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM3_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM3_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT
+#define ABM3_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT__SHIFT                           0x0
+#define ABM3_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT_MASK                             0x00FFFFFFL
+//ABM3_DC_ABM1_HG_SAMPLE_RATE
+#define ABM3_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM3_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM3_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM3_DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM3_DC_ABM1_LS_SAMPLE_RATE
+#define ABM3_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN__SHIFT                                      0x0
+#define ABM3_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT__SHIFT                                   0x8
+#define ABM3_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN_MASK                                        0x00000001L
+#define ABM3_DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT_MASK                                     0x0000FF00L
+//ABM3_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG
+#define ABM3_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG__SHIFT                               0x0
+#define ABM3_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG_MASK                                 0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG
+#define ABM3_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG__SHIFT                             0x0
+#define ABM3_DC_ABM1_HG_BIN_33_64_SHIFT_FLAG__ABM1_HG_BIN_33_64_SHIFT_FLAG_MASK                               0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX__SHIFT                               0x0
+#define ABM3_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX_MASK                                 0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX__SHIFT                             0x0
+#define ABM3_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX_MASK                               0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_33_40_SHIFT_INDEX__ABM1_HG_BIN_33_40_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_41_48_SHIFT_INDEX__ABM1_HG_BIN_41_48_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_49_56_SHIFT_INDEX__ABM1_HG_BIN_49_56_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX
+#define ABM3_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX__SHIFT                           0x0
+#define ABM3_DC_ABM1_HG_BIN_57_64_SHIFT_INDEX__ABM1_HG_BIN_57_64_SHIFT_INDEX_MASK                             0xFFFFFFFFL
+//ABM3_DC_ABM1_HG_RESULT_INDEX
+#define ABM3_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX__SHIFT                                             0x0
+#define ABM3_DC_ABM1_HG_RESULT_INDEX__ABM1_HG_RESULT_INDEX_MASK                                               0x0000003FL
+//ABM3_DC_ABM1_HG_RESULT_DATA
+#define ABM3_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA__SHIFT                                               0x0
+#define ABM3_DC_ABM1_HG_RESULT_DATA__ABM1_HG_RESULT_DATA_MASK                                                 0xFFFFFFFFL
+//ABM3_DC_ABM1_BL_MASTER_LOCK
+#define ABM3_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK__SHIFT                                               0x1f
+#define ABM3_DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK_MASK                                                 0x80000000L
+
+
+// addressBlock: dce_dpia_dpia_mu0_dpiadec
+//DPIA_MU_CLOCK_CTRL
+#define DPIA_MU_CLOCK_CTRL__DPIA_REFCLK_GATE_DIS__SHIFT                                                       0x0
+#define DPIA_MU_CLOCK_CTRL__DPIA_DPIASYMCLK_GATE_DIS__SHIFT                                                   0x1
+#define DPIA_MU_CLOCK_CTRL__DPIA_CIO_CLKS_GATE_DIS__SHIFT                                                     0x2
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_REFCLK_R_GATE_DIS__SHIFT                                                  0x8
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_DPIASYMCLK_G_GATE_DIS__SHIFT                                              0x9
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_DPIASYMCLK_R_GATE_DIS__SHIFT                                              0xa
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_ML_SSCLK_G_GATE_DIS__SHIFT                                                0xb
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_AUXIN_SSCLK_G_GATE_DIS__SHIFT                                             0xc
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_AUXOUT_SSCLK_G_GATE_DIS__SHIFT                                            0xd
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_TMUCLK_G_GATE_DIS__SHIFT                                                  0xe
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_TEST_CLK_SEL__SHIFT                                                       0x18
+#define DPIA_MU_CLOCK_CTRL__DPIA_REFCLK_GATE_DIS_MASK                                                         0x00000001L
+#define DPIA_MU_CLOCK_CTRL__DPIA_DPIASYMCLK_GATE_DIS_MASK                                                     0x00000002L
+#define DPIA_MU_CLOCK_CTRL__DPIA_CIO_CLKS_GATE_DIS_MASK                                                       0x00000004L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_REFCLK_R_GATE_DIS_MASK                                                    0x00000100L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_DPIASYMCLK_G_GATE_DIS_MASK                                                0x00000200L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_DPIASYMCLK_R_GATE_DIS_MASK                                                0x00000400L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_ML_SSCLK_G_GATE_DIS_MASK                                                  0x00000800L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_AUXIN_SSCLK_G_GATE_DIS_MASK                                               0x00001000L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_AUXOUT_SSCLK_G_GATE_DIS_MASK                                              0x00002000L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_TMUCLK_G_GATE_DIS_MASK                                                    0x00004000L
+#define DPIA_MU_CLOCK_CTRL__DPIA_MU_TEST_CLK_SEL_MASK                                                         0xFF000000L
+//DPIA_MU_CLOCK_CTRL_DPIA_PORT0
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__CLK_SRC__SHIFT                                                         0x0
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__ML_CLK_EN__SHIFT                                                       0x1
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUX_CLK_EN__SHIFT                                                      0x2
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__DPIASYMCLK_ML_CLOCK_ON__SHIFT                                          0x8
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__ML_SSCLK_CLOCK_ON__SHIFT                                               0x9
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__TMUCLK_CLOCK_ON__SHIFT                                                 0xa
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__DPIASYMCLK_AUX_CLOCK_ON__SHIFT                                         0x10
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUXIN_SSCLK_CLOCK_ON__SHIFT                                            0x11
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUXOUT_SSCLK_CLOCK_ON__SHIFT                                           0x12
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__CLK_SRC_MASK                                                           0x00000001L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__ML_CLK_EN_MASK                                                         0x00000002L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUX_CLK_EN_MASK                                                        0x00000004L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__DPIASYMCLK_ML_CLOCK_ON_MASK                                            0x00000100L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__ML_SSCLK_CLOCK_ON_MASK                                                 0x00000200L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__TMUCLK_CLOCK_ON_MASK                                                   0x00000400L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__DPIASYMCLK_AUX_CLOCK_ON_MASK                                           0x00010000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUXIN_SSCLK_CLOCK_ON_MASK                                              0x00020000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT0__AUXOUT_SSCLK_CLOCK_ON_MASK                                             0x00040000L
+//DPIA_MU_RESET_CTRL_DPIA_PORT0
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CORE_SW_RST__SHIFT                                                     0x0
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CAPREG_SW_RST__SHIFT                                                   0x1
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__AUX_TPI_SW_RST__SHIFT                                                  0x2
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__ML_TPI_SW_RST__SHIFT                                                   0x3
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CAPREG_RESET_DONE__SHIFT                                               0x8
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CORE_RESET_DONE__SHIFT                                                 0x9
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CORE_SW_RST_MASK                                                       0x00000001L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CAPREG_SW_RST_MASK                                                     0x00000002L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__AUX_TPI_SW_RST_MASK                                                    0x00000004L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__ML_TPI_SW_RST_MASK                                                     0x00000008L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CAPREG_RESET_DONE_MASK                                                 0x00000100L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT0__CORE_RESET_DONE_MASK                                                   0x00000200L
+//DPIA_MU_CLOCK_CTRL_DPIA_PORT1
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__CLK_SRC__SHIFT                                                         0x0
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__ML_CLK_EN__SHIFT                                                       0x1
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUX_CLK_EN__SHIFT                                                      0x2
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__DPIASYMCLK_ML_CLOCK_ON__SHIFT                                          0x8
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__ML_SSCLK_CLOCK_ON__SHIFT                                               0x9
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__TMUCLK_CLOCK_ON__SHIFT                                                 0xa
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__DPIASYMCLK_AUX_CLOCK_ON__SHIFT                                         0x10
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUXIN_SSCLK_CLOCK_ON__SHIFT                                            0x11
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUXOUT_SSCLK_CLOCK_ON__SHIFT                                           0x12
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__CLK_SRC_MASK                                                           0x00000001L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__ML_CLK_EN_MASK                                                         0x00000002L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUX_CLK_EN_MASK                                                        0x00000004L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__DPIASYMCLK_ML_CLOCK_ON_MASK                                            0x00000100L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__ML_SSCLK_CLOCK_ON_MASK                                                 0x00000200L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__TMUCLK_CLOCK_ON_MASK                                                   0x00000400L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__DPIASYMCLK_AUX_CLOCK_ON_MASK                                           0x00010000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUXIN_SSCLK_CLOCK_ON_MASK                                              0x00020000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT1__AUXOUT_SSCLK_CLOCK_ON_MASK                                             0x00040000L
+//DPIA_MU_RESET_CTRL_DPIA_PORT1
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CORE_SW_RST__SHIFT                                                     0x0
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CAPREG_SW_RST__SHIFT                                                   0x1
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__AUX_TPI_SW_RST__SHIFT                                                  0x2
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__ML_TPI_SW_RST__SHIFT                                                   0x3
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CAPREG_RESET_DONE__SHIFT                                               0x8
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CORE_RESET_DONE__SHIFT                                                 0x9
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CORE_SW_RST_MASK                                                       0x00000001L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CAPREG_SW_RST_MASK                                                     0x00000002L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__AUX_TPI_SW_RST_MASK                                                    0x00000004L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__ML_TPI_SW_RST_MASK                                                     0x00000008L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CAPREG_RESET_DONE_MASK                                                 0x00000100L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT1__CORE_RESET_DONE_MASK                                                   0x00000200L
+//DPIA_MU_CLOCK_CTRL_DPIA_PORT2
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__CLK_SRC__SHIFT                                                         0x0
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__ML_CLK_EN__SHIFT                                                       0x1
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUX_CLK_EN__SHIFT                                                      0x2
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__DPIASYMCLK_ML_CLOCK_ON__SHIFT                                          0x8
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__ML_SSCLK_CLOCK_ON__SHIFT                                               0x9
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__TMUCLK_CLOCK_ON__SHIFT                                                 0xa
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__DPIASYMCLK_AUX_CLOCK_ON__SHIFT                                         0x10
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUXIN_SSCLK_CLOCK_ON__SHIFT                                            0x11
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUXOUT_SSCLK_CLOCK_ON__SHIFT                                           0x12
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__CLK_SRC_MASK                                                           0x00000001L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__ML_CLK_EN_MASK                                                         0x00000002L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUX_CLK_EN_MASK                                                        0x00000004L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__DPIASYMCLK_ML_CLOCK_ON_MASK                                            0x00000100L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__ML_SSCLK_CLOCK_ON_MASK                                                 0x00000200L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__TMUCLK_CLOCK_ON_MASK                                                   0x00000400L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__DPIASYMCLK_AUX_CLOCK_ON_MASK                                           0x00010000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUXIN_SSCLK_CLOCK_ON_MASK                                              0x00020000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT2__AUXOUT_SSCLK_CLOCK_ON_MASK                                             0x00040000L
+//DPIA_MU_RESET_CTRL_DPIA_PORT2
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CORE_SW_RST__SHIFT                                                     0x0
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CAPREG_SW_RST__SHIFT                                                   0x1
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__AUX_TPI_SW_RST__SHIFT                                                  0x2
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__ML_TPI_SW_RST__SHIFT                                                   0x3
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CAPREG_RESET_DONE__SHIFT                                               0x8
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CORE_RESET_DONE__SHIFT                                                 0x9
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CORE_SW_RST_MASK                                                       0x00000001L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CAPREG_SW_RST_MASK                                                     0x00000002L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__AUX_TPI_SW_RST_MASK                                                    0x00000004L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__ML_TPI_SW_RST_MASK                                                     0x00000008L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CAPREG_RESET_DONE_MASK                                                 0x00000100L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT2__CORE_RESET_DONE_MASK                                                   0x00000200L
+//DPIA_MU_CLOCK_CTRL_DPIA_PORT3
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__CLK_SRC__SHIFT                                                         0x0
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__ML_CLK_EN__SHIFT                                                       0x1
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUX_CLK_EN__SHIFT                                                      0x2
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__DPIASYMCLK_ML_CLOCK_ON__SHIFT                                          0x8
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__ML_SSCLK_CLOCK_ON__SHIFT                                               0x9
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__TMUCLK_CLOCK_ON__SHIFT                                                 0xa
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__DPIASYMCLK_AUX_CLOCK_ON__SHIFT                                         0x10
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUXIN_SSCLK_CLOCK_ON__SHIFT                                            0x11
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUXOUT_SSCLK_CLOCK_ON__SHIFT                                           0x12
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__CLK_SRC_MASK                                                           0x00000001L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__ML_CLK_EN_MASK                                                         0x00000002L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUX_CLK_EN_MASK                                                        0x00000004L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__DPIASYMCLK_ML_CLOCK_ON_MASK                                            0x00000100L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__ML_SSCLK_CLOCK_ON_MASK                                                 0x00000200L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__TMUCLK_CLOCK_ON_MASK                                                   0x00000400L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__DPIASYMCLK_AUX_CLOCK_ON_MASK                                           0x00010000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUXIN_SSCLK_CLOCK_ON_MASK                                              0x00020000L
+#define DPIA_MU_CLOCK_CTRL_DPIA_PORT3__AUXOUT_SSCLK_CLOCK_ON_MASK                                             0x00040000L
+//DPIA_MU_RESET_CTRL_DPIA_PORT3
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CORE_SW_RST__SHIFT                                                     0x0
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CAPREG_SW_RST__SHIFT                                                   0x1
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__AUX_TPI_SW_RST__SHIFT                                                  0x2
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__ML_TPI_SW_RST__SHIFT                                                   0x3
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CAPREG_RESET_DONE__SHIFT                                               0x8
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CORE_RESET_DONE__SHIFT                                                 0x9
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CORE_SW_RST_MASK                                                       0x00000001L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CAPREG_SW_RST_MASK                                                     0x00000002L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__AUX_TPI_SW_RST_MASK                                                    0x00000004L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__ML_TPI_SW_RST_MASK                                                     0x00000008L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CAPREG_RESET_DONE_MASK                                                 0x00000100L
+#define DPIA_MU_RESET_CTRL_DPIA_PORT3__CORE_RESET_DONE_MASK                                                   0x00000200L
+//DPIA_MU_TPI_STATUS_DPIA_PORT0
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__MAIN_LINK_CREDIT_COUNT__SHIFT                                          0x0
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXOUT_CREDIT_COUNT__SHIFT                                             0x8
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXIN_CREDIT_COUNT__SHIFT                                              0x10
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__MAIN_LINK_EXTRA_CREDIT_RECEIVED__SHIFT                                 0x18
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXOUT_EXTRA_CREDIT_RECEIVED__SHIFT                                    0x19
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXIN_EXTRA_PAYLOAD_RECEIVED__SHIFT                                    0x1a
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__MAIN_LINK_CREDIT_COUNT_MASK                                            0x0000007FL
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXOUT_CREDIT_COUNT_MASK                                               0x00000F00L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXIN_CREDIT_COUNT_MASK                                                0x000F0000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__MAIN_LINK_EXTRA_CREDIT_RECEIVED_MASK                                   0x01000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXOUT_EXTRA_CREDIT_RECEIVED_MASK                                      0x02000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT0__AUXIN_EXTRA_PAYLOAD_RECEIVED_MASK                                      0x04000000L
+//DPIA_MU_TPI_STATUS_DPIA_PORT1
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__MAIN_LINK_CREDIT_COUNT__SHIFT                                          0x0
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXOUT_CREDIT_COUNT__SHIFT                                             0x8
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXIN_CREDIT_COUNT__SHIFT                                              0x10
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__MAIN_LINK_EXTRA_CREDIT_RECEIVED__SHIFT                                 0x18
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXOUT_EXTRA_CREDIT_RECEIVED__SHIFT                                    0x19
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXIN_EXTRA_PAYLOAD_RECEIVED__SHIFT                                    0x1a
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__MAIN_LINK_CREDIT_COUNT_MASK                                            0x0000007FL
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXOUT_CREDIT_COUNT_MASK                                               0x00000F00L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXIN_CREDIT_COUNT_MASK                                                0x000F0000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__MAIN_LINK_EXTRA_CREDIT_RECEIVED_MASK                                   0x01000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXOUT_EXTRA_CREDIT_RECEIVED_MASK                                      0x02000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT1__AUXIN_EXTRA_PAYLOAD_RECEIVED_MASK                                      0x04000000L
+//DPIA_MU_TPI_STATUS_DPIA_PORT2
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__MAIN_LINK_CREDIT_COUNT__SHIFT                                          0x0
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXOUT_CREDIT_COUNT__SHIFT                                             0x8
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXIN_CREDIT_COUNT__SHIFT                                              0x10
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__MAIN_LINK_EXTRA_CREDIT_RECEIVED__SHIFT                                 0x18
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXOUT_EXTRA_CREDIT_RECEIVED__SHIFT                                    0x19
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXIN_EXTRA_PAYLOAD_RECEIVED__SHIFT                                    0x1a
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__MAIN_LINK_CREDIT_COUNT_MASK                                            0x0000007FL
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXOUT_CREDIT_COUNT_MASK                                               0x00000F00L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXIN_CREDIT_COUNT_MASK                                                0x000F0000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__MAIN_LINK_EXTRA_CREDIT_RECEIVED_MASK                                   0x01000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXOUT_EXTRA_CREDIT_RECEIVED_MASK                                      0x02000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT2__AUXIN_EXTRA_PAYLOAD_RECEIVED_MASK                                      0x04000000L
+//DPIA_MU_TPI_STATUS_DPIA_PORT3
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__MAIN_LINK_CREDIT_COUNT__SHIFT                                          0x0
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXOUT_CREDIT_COUNT__SHIFT                                             0x8
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXIN_CREDIT_COUNT__SHIFT                                              0x10
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__MAIN_LINK_EXTRA_CREDIT_RECEIVED__SHIFT                                 0x18
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXOUT_EXTRA_CREDIT_RECEIVED__SHIFT                                    0x19
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXIN_EXTRA_PAYLOAD_RECEIVED__SHIFT                                    0x1a
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__MAIN_LINK_CREDIT_COUNT_MASK                                            0x0000007FL
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXOUT_CREDIT_COUNT_MASK                                               0x00000F00L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXIN_CREDIT_COUNT_MASK                                                0x000F0000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__MAIN_LINK_EXTRA_CREDIT_RECEIVED_MASK                                   0x01000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXOUT_EXTRA_CREDIT_RECEIVED_MASK                                      0x02000000L
+#define DPIA_MU_TPI_STATUS_DPIA_PORT3__AUXIN_EXTRA_PAYLOAD_RECEIVED_MASK                                      0x04000000L
+//DPIA_MU_TPI_MAX_CREDIT_COUNT
+#define DPIA_MU_TPI_MAX_CREDIT_COUNT__DPIA_TPI_MAX_CREDIT_COUNT__SHIFT                                        0x0
+#define DPIA_MU_TPI_MAX_CREDIT_COUNT__DPIA_TPI_MAX_CREDIT_COUNT_MASK                                          0x0000003FL
+//DPIA_MU_INTERRUPT_STATUS
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT0_INT_STATUS__SHIFT                                                0x0
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT1_INT_STATUS__SHIFT                                                0x3
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT2_INT_STATUS__SHIFT                                                0x6
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT3_INT_STATUS__SHIFT                                                0x9
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT4_INT_STATUS__SHIFT                                                0xc
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT5_INT_STATUS__SHIFT                                                0xf
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_MU_LOCAL_INT_STATUS__SHIFT                                             0x1f
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT0_INT_STATUS_MASK                                                  0x00000007L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT1_INT_STATUS_MASK                                                  0x00000038L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT2_INT_STATUS_MASK                                                  0x000001C0L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT3_INT_STATUS_MASK                                                  0x00000E00L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT4_INT_STATUS_MASK                                                  0x00007000L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_PORT5_INT_STATUS_MASK                                                  0x00038000L
+#define DPIA_MU_INTERRUPT_STATUS__DPIA_MU_LOCAL_INT_STATUS_MASK                                               0x80000000L
+//DPIA_MU_INTERRUPT_CTRL
+#define DPIA_MU_INTERRUPT_CTRL__DPIA_DCN_INT_EN__SHIFT                                                        0x0
+#define DPIA_MU_INTERRUPT_CTRL__DPIA_PORT_INT_MASK__SHIFT                                                     0x8
+#define DPIA_MU_INTERRUPT_CTRL__RBBMIF_TIMEOUT_INT_MASK__SHIFT                                                0x18
+#define DPIA_MU_INTERRUPT_CTRL__DPIA_DCN_INT_EN_MASK                                                          0x00000001L
+#define DPIA_MU_INTERRUPT_CTRL__DPIA_PORT_INT_MASK_MASK                                                       0x0000FF00L
+#define DPIA_MU_INTERRUPT_CTRL__RBBMIF_TIMEOUT_INT_MASK_MASK                                                  0x01000000L
+//DPIA_MU_LOCAL_INTERRUPT_CTRL
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_INT_STATUS__SHIFT                                        0x0
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_ADDR__SHIFT                                              0x2
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_OP__SHIFT                                                0x14
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__DPIA_RST_DONE_INT_STATUS__SHIFT                                         0x18
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_INT_STATUS_MASK                                          0x00000001L
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_ADDR_MASK                                                0x000FFFFCL
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__RBBMIF_TIMEOUT_OP_MASK                                                  0x00100000L
+#define DPIA_MU_LOCAL_INTERRUPT_CTRL__DPIA_RST_DONE_INT_STATUS_MASK                                           0x01000000L
+//DPIA_MU_LOCAL_INTERRUPT_ACK
+#define DPIA_MU_LOCAL_INTERRUPT_ACK__RBBMIF_TIMEOUT_INT_ACK__SHIFT                                            0x0
+#define DPIA_MU_LOCAL_INTERRUPT_ACK__DPIA_RST_DONE_INT_ACK__SHIFT                                             0x1
+#define DPIA_MU_LOCAL_INTERRUPT_ACK__RBBMIF_TIMEOUT_INT_ACK_MASK                                              0x00000001L
+#define DPIA_MU_LOCAL_INTERRUPT_ACK__DPIA_RST_DONE_INT_ACK_MASK                                               0x00000002L
+//DPIA_MU_RBBMIF_TIMEOUT_CTRL
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL__RBBMIF_TIMEOUT_DELAY__SHIFT                                              0x0
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL__RBBMIF_TIMEOUT_HOLD__SHIFT                                               0x14
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL__RBBMIF_TIMEOUT_DELAY_MASK                                                0x000FFFFFL
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL__RBBMIF_TIMEOUT_HOLD_MASK                                                 0xFFF00000L
+//DPIA_MU_RBBMIF_TIMEOUT_CTRL2
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL2__RBBMIF_TIMEOUT_DIS__SHIFT                                               0x0
+#define DPIA_MU_RBBMIF_TIMEOUT_CTRL2__RBBMIF_TIMEOUT_DIS_MASK                                                 0x00000001L
+//DPIA_MU_RBBMIF_STATUS
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_FLAG__SHIFT                                              0x0
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_TYPE__SHIFT                                              0x1
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_ADDR__SHIFT                                              0x5
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_TIMEOUT_STATUS_READBACK__SHIFT                                          0x18
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_STATUS_CLEAR__SHIFT                                      0x1f
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_FLAG_MASK                                                0x00000001L
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_TYPE_MASK                                                0x00000006L
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_ADDR_MASK                                                0x007FFFE0L
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_TIMEOUT_STATUS_READBACK_MASK                                            0x01000000L
+#define DPIA_MU_RBBMIF_STATUS__RBBMIF_INVALID_ACCESS_STATUS_CLEAR_MASK                                        0x80000000L
+//DPIA_MU_MICROSECOND_REF_CTRL
+#define DPIA_MU_MICROSECOND_REF_CTRL__MICROSECOND_TIME_BASE_DIV__SHIFT                                        0x0
+#define DPIA_MU_MICROSECOND_REF_CTRL__MICROSECOND_TIME_BASE_DIV_MASK                                          0x0000007FL
+//DPIA_MU_PORT_ADP_STATUS
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT0_HIDDEN_STATUS__SHIFT                                              0x0
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT1_HIDDEN_STATUS__SHIFT                                              0x1
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT2_HIDDEN_STATUS__SHIFT                                              0x2
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT3_HIDDEN_STATUS__SHIFT                                              0x3
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT4_HIDDEN_STATUS__SHIFT                                              0x4
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT5_HIDDEN_STATUS__SHIFT                                              0x5
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT0_HIDDEN_STATUS_MASK                                                0x00000001L
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT1_HIDDEN_STATUS_MASK                                                0x00000002L
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT2_HIDDEN_STATUS_MASK                                                0x00000004L
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT3_HIDDEN_STATUS_MASK                                                0x00000008L
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT4_HIDDEN_STATUS_MASK                                                0x00000010L
+#define DPIA_MU_PORT_ADP_STATUS__DPIA_PORT5_HIDDEN_STATUS_MASK                                                0x00000020L
+//DPIA_GLUE_CTRL
+#define DPIA_GLUE_CTRL__DPIA_IO_EN__SHIFT                                                                     0x0
+#define DPIA_GLUE_CTRL__DPIA_GLUE_ML_SSCLK_DBG_BUS_SEL__SHIFT                                                 0x4
+#define DPIA_GLUE_CTRL__DPIA_GLUE_AUXOUT_SSCLK_DBG_BUS_SEL__SHIFT                                             0x8
+#define DPIA_GLUE_CTRL__DPIA_GLUE_AUXIN_SSCLK_DBG_BUS_SEL__SHIFT                                              0xa
+#define DPIA_GLUE_CTRL__DPIA_IO_EN_MASK                                                                       0x00000001L
+#define DPIA_GLUE_CTRL__DPIA_GLUE_ML_SSCLK_DBG_BUS_SEL_MASK                                                   0x00000030L
+#define DPIA_GLUE_CTRL__DPIA_GLUE_AUXOUT_SSCLK_DBG_BUS_SEL_MASK                                               0x00000300L
+#define DPIA_GLUE_CTRL__DPIA_GLUE_AUXIN_SSCLK_DBG_BUS_SEL_MASK                                                0x00000C00L
+//DPIA_PERF_COUNT_CONTROL0
+#define DPIA_PERF_COUNT_CONTROL0__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL0__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL0__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL0__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL0__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL0__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL0__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL0__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL0__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL0__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL0__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL0__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_CONTROL1
+#define DPIA_PERF_COUNT_CONTROL1__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL1__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL1__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL1__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL1__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL1__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL1__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL1__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL1__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL1__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL1__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL1__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_CONTROL2
+#define DPIA_PERF_COUNT_CONTROL2__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL2__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL2__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL2__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL2__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL2__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL2__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL2__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL2__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL2__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL2__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL2__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_CONTROL3
+#define DPIA_PERF_COUNT_CONTROL3__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL3__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL3__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL3__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL3__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL3__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL3__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL3__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL3__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL3__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL3__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL3__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_CONTROL4
+#define DPIA_PERF_COUNT_CONTROL4__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL4__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL4__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL4__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL4__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL4__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL4__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL4__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL4__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL4__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL4__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL4__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_CONTROL5
+#define DPIA_PERF_COUNT_CONTROL5__ENABLE__SHIFT                                                               0x0
+#define DPIA_PERF_COUNT_CONTROL5__MODE__SHIFT                                                                 0x1
+#define DPIA_PERF_COUNT_CONTROL5__STATUS__SHIFT                                                               0x4
+#define DPIA_PERF_COUNT_CONTROL5__PORT_SELECT__SHIFT                                                          0x5
+#define DPIA_PERF_COUNT_CONTROL5__STAT_SELECT__SHIFT                                                          0x8
+#define DPIA_PERF_COUNT_CONTROL5__COUNT_LIMIT__SHIFT                                                          0xc
+#define DPIA_PERF_COUNT_CONTROL5__ENABLE_MASK                                                                 0x00000001L
+#define DPIA_PERF_COUNT_CONTROL5__MODE_MASK                                                                   0x0000000EL
+#define DPIA_PERF_COUNT_CONTROL5__STATUS_MASK                                                                 0x00000010L
+#define DPIA_PERF_COUNT_CONTROL5__PORT_SELECT_MASK                                                            0x000000E0L
+#define DPIA_PERF_COUNT_CONTROL5__STAT_SELECT_MASK                                                            0x00000300L
+#define DPIA_PERF_COUNT_CONTROL5__COUNT_LIMIT_MASK                                                            0xFFFFF000L
+//DPIA_PERF_COUNT_INDEX
+#define DPIA_PERF_COUNT_INDEX__COUNTER_SELECT__SHIFT                                                          0x0
+#define DPIA_PERF_COUNT_INDEX__MEAS_SELECT__SHIFT                                                             0x4
+#define DPIA_PERF_COUNT_INDEX__COUNTER_SELECT_MASK                                                            0x00000007L
+#define DPIA_PERF_COUNT_INDEX__MEAS_SELECT_MASK                                                               0x00000070L
+//DPIA_PERF_COUNT_DATA_LO
+#define DPIA_PERF_COUNT_DATA_LO__VALUE__SHIFT                                                                 0x0
+#define DPIA_PERF_COUNT_DATA_LO__VALUE_MASK                                                                   0xFFFFFFFFL
+//DPIA_MU_SPARE
+#define DPIA_MU_SPARE__DPIA_MU_SPARE__SHIFT                                                                   0x0
+#define DPIA_MU_SPARE__DPIA_MU_SPARE_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azcontroller_azdec
+//GLOBAL_CAPABILITIES
+#define GLOBAL_CAPABILITIES__SIXTY_FOUR_BIT_ADDRESS_SUPPORTED__SHIFT                                          0x0
+#define GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS__SHIFT                                      0x1
+#define GLOBAL_CAPABILITIES__NUMBER_OF_BIDIRECTIONAL_STREAMS_SUPPORTED__SHIFT                                 0x3
+#define GLOBAL_CAPABILITIES__NUMBER_OF_INPUT_STREAMS_SUPPORTED__SHIFT                                         0x8
+#define GLOBAL_CAPABILITIES__NUMBER_OF_OUTPUT_STREAMS_SUPPORTED__SHIFT                                        0xc
+#define GLOBAL_CAPABILITIES__SIXTY_FOUR_BIT_ADDRESS_SUPPORTED_MASK                                            0x0001L
+#define GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS_MASK                                        0x0006L
+#define GLOBAL_CAPABILITIES__NUMBER_OF_BIDIRECTIONAL_STREAMS_SUPPORTED_MASK                                   0x00F8L
+#define GLOBAL_CAPABILITIES__NUMBER_OF_INPUT_STREAMS_SUPPORTED_MASK                                           0x0F00L
+#define GLOBAL_CAPABILITIES__NUMBER_OF_OUTPUT_STREAMS_SUPPORTED_MASK                                          0xF000L
+//MINOR_VERSION
+#define MINOR_VERSION__MINOR_VERSION__SHIFT                                                                   0x0
+#define MINOR_VERSION__MINOR_VERSION_MASK                                                                     0xFFL
+//MAJOR_VERSION
+#define MAJOR_VERSION__MAJOR_VERSION__SHIFT                                                                   0x0
+#define MAJOR_VERSION__MAJOR_VERSION_MASK                                                                     0xFFL
+//OUTPUT_PAYLOAD_CAPABILITY
+#define OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY__SHIFT                                           0x0
+#define OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY_MASK                                             0xFFFFL
+//INPUT_PAYLOAD_CAPABILITY
+#define INPUT_PAYLOAD_CAPABILITY__INPUT_PAYLOAD_CAPABILITY__SHIFT                                             0x0
+#define INPUT_PAYLOAD_CAPABILITY__INPUT_PAYLOAD_CAPABILITY_MASK                                               0xFFFFL
+//GLOBAL_CONTROL
+#define GLOBAL_CONTROL__CONTROLLER_RESET__SHIFT                                                               0x0
+#define GLOBAL_CONTROL__FLUSH_CONTROL__SHIFT                                                                  0x1
+#define GLOBAL_CONTROL__ACCEPT_UNSOLICITED_RESPONSE_ENABLE__SHIFT                                             0x8
+#define GLOBAL_CONTROL__CONTROLLER_RESET_MASK                                                                 0x00000001L
+#define GLOBAL_CONTROL__FLUSH_CONTROL_MASK                                                                    0x00000002L
+#define GLOBAL_CONTROL__ACCEPT_UNSOLICITED_RESPONSE_ENABLE_MASK                                               0x00000100L
+//WAKE_ENABLE
+#define WAKE_ENABLE__SDIN_WAKE_ENABLE_FLAG__SHIFT                                                             0x0
+#define WAKE_ENABLE__SDIN_WAKE_ENABLE_FLAG_MASK                                                               0x0001L
+//STATE_CHANGE_STATUS
+#define STATE_CHANGE_STATUS__STATE_CHANGE_STATUS__SHIFT                                                       0x0
+#define STATE_CHANGE_STATUS__STATE_CHANGE_STATUS_MASK                                                         0x0001L
+//GLOBAL_STATUS
+#define GLOBAL_STATUS__FLUSH_STATUS__SHIFT                                                                    0x1
+#define GLOBAL_STATUS__FLUSH_STATUS_MASK                                                                      0x00000002L
+//OUTPUT_STREAM_PAYLOAD_CAPABILITY
+#define OUTPUT_STREAM_PAYLOAD_CAPABILITY__OUTSTRMPAY__SHIFT                                                   0x0
+#define OUTPUT_STREAM_PAYLOAD_CAPABILITY__OUTSTRMPAY_MASK                                                     0xFFFFL
+//INPUT_STREAM_PAYLOAD_CAPABILITY
+#define INPUT_STREAM_PAYLOAD_CAPABILITY__INSTRMPAY__SHIFT                                                     0x0
+#define INPUT_STREAM_PAYLOAD_CAPABILITY__INSTRMPAY_MASK                                                       0xFFFFL
+//INTERRUPT_CONTROL
+#define INTERRUPT_CONTROL__STREAM_0_INTERRUPT_ENABLE__SHIFT                                                   0x0
+#define INTERRUPT_CONTROL__STREAM_1_INTERRUPT_ENABLE__SHIFT                                                   0x1
+#define INTERRUPT_CONTROL__STREAM_2_INTERRUPT_ENABLE__SHIFT                                                   0x2
+#define INTERRUPT_CONTROL__STREAM_3_INTERRUPT_ENABLE__SHIFT                                                   0x3
+#define INTERRUPT_CONTROL__STREAM_4_INTERRUPT_ENABLE__SHIFT                                                   0x4
+#define INTERRUPT_CONTROL__STREAM_5_INTERRUPT_ENABLE__SHIFT                                                   0x5
+#define INTERRUPT_CONTROL__STREAM_6_INTERRUPT_ENABLE__SHIFT                                                   0x6
+#define INTERRUPT_CONTROL__STREAM_7_INTERRUPT_ENABLE__SHIFT                                                   0x7
+#define INTERRUPT_CONTROL__STREAM_8_INTERRUPT_ENABLE__SHIFT                                                   0x8
+#define INTERRUPT_CONTROL__STREAM_9_INTERRUPT_ENABLE__SHIFT                                                   0x9
+#define INTERRUPT_CONTROL__STREAM_10_INTERRUPT_ENABLE__SHIFT                                                  0xa
+#define INTERRUPT_CONTROL__STREAM_11_INTERRUPT_ENABLE__SHIFT                                                  0xb
+#define INTERRUPT_CONTROL__STREAM_12_INTERRUPT_ENABLE__SHIFT                                                  0xc
+#define INTERRUPT_CONTROL__STREAM_13_INTERRUPT_ENABLE__SHIFT                                                  0xd
+#define INTERRUPT_CONTROL__STREAM_14_INTERRUPT_ENABLE__SHIFT                                                  0xe
+#define INTERRUPT_CONTROL__STREAM_15_INTERRUPT_ENABLE__SHIFT                                                  0xf
+#define INTERRUPT_CONTROL__CONTROLLER_INTERRUPT_ENABLE__SHIFT                                                 0x1e
+#define INTERRUPT_CONTROL__GLOBAL_INTERRUPT_ENABLE__SHIFT                                                     0x1f
+#define INTERRUPT_CONTROL__STREAM_0_INTERRUPT_ENABLE_MASK                                                     0x00000001L
+#define INTERRUPT_CONTROL__STREAM_1_INTERRUPT_ENABLE_MASK                                                     0x00000002L
+#define INTERRUPT_CONTROL__STREAM_2_INTERRUPT_ENABLE_MASK                                                     0x00000004L
+#define INTERRUPT_CONTROL__STREAM_3_INTERRUPT_ENABLE_MASK                                                     0x00000008L
+#define INTERRUPT_CONTROL__STREAM_4_INTERRUPT_ENABLE_MASK                                                     0x00000010L
+#define INTERRUPT_CONTROL__STREAM_5_INTERRUPT_ENABLE_MASK                                                     0x00000020L
+#define INTERRUPT_CONTROL__STREAM_6_INTERRUPT_ENABLE_MASK                                                     0x00000040L
+#define INTERRUPT_CONTROL__STREAM_7_INTERRUPT_ENABLE_MASK                                                     0x00000080L
+#define INTERRUPT_CONTROL__STREAM_8_INTERRUPT_ENABLE_MASK                                                     0x00000100L
+#define INTERRUPT_CONTROL__STREAM_9_INTERRUPT_ENABLE_MASK                                                     0x00000200L
+#define INTERRUPT_CONTROL__STREAM_10_INTERRUPT_ENABLE_MASK                                                    0x00000400L
+#define INTERRUPT_CONTROL__STREAM_11_INTERRUPT_ENABLE_MASK                                                    0x00000800L
+#define INTERRUPT_CONTROL__STREAM_12_INTERRUPT_ENABLE_MASK                                                    0x00001000L
+#define INTERRUPT_CONTROL__STREAM_13_INTERRUPT_ENABLE_MASK                                                    0x00002000L
+#define INTERRUPT_CONTROL__STREAM_14_INTERRUPT_ENABLE_MASK                                                    0x00004000L
+#define INTERRUPT_CONTROL__STREAM_15_INTERRUPT_ENABLE_MASK                                                    0x00008000L
+#define INTERRUPT_CONTROL__CONTROLLER_INTERRUPT_ENABLE_MASK                                                   0x40000000L
+#define INTERRUPT_CONTROL__GLOBAL_INTERRUPT_ENABLE_MASK                                                       0x80000000L
+//INTERRUPT_STATUS
+#define INTERRUPT_STATUS__STREAM_0_INTERRUPT_STATUS__SHIFT                                                    0x0
+#define INTERRUPT_STATUS__STREAM_1_INTERRUPT_STATUS__SHIFT                                                    0x1
+#define INTERRUPT_STATUS__STREAM_2_INTERRUPT_STATUS__SHIFT                                                    0x2
+#define INTERRUPT_STATUS__STREAM_3_INTERRUPT_STATUS__SHIFT                                                    0x3
+#define INTERRUPT_STATUS__STREAM_4_INTERRUPT_STATUS__SHIFT                                                    0x4
+#define INTERRUPT_STATUS__STREAM_5_INTERRUPT_STATUS__SHIFT                                                    0x5
+#define INTERRUPT_STATUS__STREAM_6_INTERRUPT_STATUS__SHIFT                                                    0x6
+#define INTERRUPT_STATUS__STREAM_7_INTERRUPT_STATUS__SHIFT                                                    0x7
+#define INTERRUPT_STATUS__STREAM_8_INTERRUPT_STATUS__SHIFT                                                    0x8
+#define INTERRUPT_STATUS__STREAM_9_INTERRUPT_STATUS__SHIFT                                                    0x9
+#define INTERRUPT_STATUS__STREAM_10_INTERRUPT_STATUS__SHIFT                                                   0xa
+#define INTERRUPT_STATUS__STREAM_11_INTERRUPT_STATUS__SHIFT                                                   0xb
+#define INTERRUPT_STATUS__STREAM_12_INTERRUPT_STATUS__SHIFT                                                   0xc
+#define INTERRUPT_STATUS__STREAM_13_INTERRUPT_STATUS__SHIFT                                                   0xd
+#define INTERRUPT_STATUS__STREAM_14_INTERRUPT_STATUS__SHIFT                                                   0xe
+#define INTERRUPT_STATUS__STREAM_15_INTERRUPT_STATUS__SHIFT                                                   0xf
+#define INTERRUPT_STATUS__CONTROLLER_INTERRUPT_STATUS__SHIFT                                                  0x1e
+#define INTERRUPT_STATUS__GLOBAL_INTERRUPT_STATUS__SHIFT                                                      0x1f
+#define INTERRUPT_STATUS__STREAM_0_INTERRUPT_STATUS_MASK                                                      0x00000001L
+#define INTERRUPT_STATUS__STREAM_1_INTERRUPT_STATUS_MASK                                                      0x00000002L
+#define INTERRUPT_STATUS__STREAM_2_INTERRUPT_STATUS_MASK                                                      0x00000004L
+#define INTERRUPT_STATUS__STREAM_3_INTERRUPT_STATUS_MASK                                                      0x00000008L
+#define INTERRUPT_STATUS__STREAM_4_INTERRUPT_STATUS_MASK                                                      0x00000010L
+#define INTERRUPT_STATUS__STREAM_5_INTERRUPT_STATUS_MASK                                                      0x00000020L
+#define INTERRUPT_STATUS__STREAM_6_INTERRUPT_STATUS_MASK                                                      0x00000040L
+#define INTERRUPT_STATUS__STREAM_7_INTERRUPT_STATUS_MASK                                                      0x00000080L
+#define INTERRUPT_STATUS__STREAM_8_INTERRUPT_STATUS_MASK                                                      0x00000100L
+#define INTERRUPT_STATUS__STREAM_9_INTERRUPT_STATUS_MASK                                                      0x00000200L
+#define INTERRUPT_STATUS__STREAM_10_INTERRUPT_STATUS_MASK                                                     0x00000400L
+#define INTERRUPT_STATUS__STREAM_11_INTERRUPT_STATUS_MASK                                                     0x00000800L
+#define INTERRUPT_STATUS__STREAM_12_INTERRUPT_STATUS_MASK                                                     0x00001000L
+#define INTERRUPT_STATUS__STREAM_13_INTERRUPT_STATUS_MASK                                                     0x00002000L
+#define INTERRUPT_STATUS__STREAM_14_INTERRUPT_STATUS_MASK                                                     0x00004000L
+#define INTERRUPT_STATUS__STREAM_15_INTERRUPT_STATUS_MASK                                                     0x00008000L
+#define INTERRUPT_STATUS__CONTROLLER_INTERRUPT_STATUS_MASK                                                    0x40000000L
+#define INTERRUPT_STATUS__GLOBAL_INTERRUPT_STATUS_MASK                                                        0x80000000L
+//WALL_CLOCK_COUNTER
+#define WALL_CLOCK_COUNTER__WALL_CLOCK_COUNTER__SHIFT                                                         0x0
+#define WALL_CLOCK_COUNTER__WALL_CLOCK_COUNTER_MASK                                                           0xFFFFFFFFL
+//STREAM_SYNCHRONIZATION
+#define STREAM_SYNCHRONIZATION__STREAM_0_SYNCHRONIZATION__SHIFT                                               0x0
+#define STREAM_SYNCHRONIZATION__STREAM_1_SYNCHRONIZATION__SHIFT                                               0x1
+#define STREAM_SYNCHRONIZATION__STREAM_2_SYNCHRONIZATION__SHIFT                                               0x2
+#define STREAM_SYNCHRONIZATION__STREAM_3_SYNCHRONIZATION__SHIFT                                               0x3
+#define STREAM_SYNCHRONIZATION__STREAM_4_SYNCHRONIZATION__SHIFT                                               0x4
+#define STREAM_SYNCHRONIZATION__STREAM_5_SYNCHRONIZATION__SHIFT                                               0x5
+#define STREAM_SYNCHRONIZATION__STREAM_6_SYNCHRONIZATION__SHIFT                                               0x6
+#define STREAM_SYNCHRONIZATION__STREAM_7_SYNCHRONIZATION__SHIFT                                               0x7
+#define STREAM_SYNCHRONIZATION__STREAM_8_SYNCHRONIZATION__SHIFT                                               0x8
+#define STREAM_SYNCHRONIZATION__STREAM_9_SYNCHRONIZATION__SHIFT                                               0x9
+#define STREAM_SYNCHRONIZATION__STREAM_10_SYNCHRONIZATION__SHIFT                                              0xa
+#define STREAM_SYNCHRONIZATION__STREAM_11_SYNCHRONIZATION__SHIFT                                              0xb
+#define STREAM_SYNCHRONIZATION__STREAM_12_SYNCHRONIZATION__SHIFT                                              0xc
+#define STREAM_SYNCHRONIZATION__STREAM_13_SYNCHRONIZATION__SHIFT                                              0xd
+#define STREAM_SYNCHRONIZATION__STREAM_14_SYNCHRONIZATION__SHIFT                                              0xe
+#define STREAM_SYNCHRONIZATION__STREAM_15_SYNCHRONIZATION__SHIFT                                              0xf
+#define STREAM_SYNCHRONIZATION__STREAM_0_SYNCHRONIZATION_MASK                                                 0x00000001L
+#define STREAM_SYNCHRONIZATION__STREAM_1_SYNCHRONIZATION_MASK                                                 0x00000002L
+#define STREAM_SYNCHRONIZATION__STREAM_2_SYNCHRONIZATION_MASK                                                 0x00000004L
+#define STREAM_SYNCHRONIZATION__STREAM_3_SYNCHRONIZATION_MASK                                                 0x00000008L
+#define STREAM_SYNCHRONIZATION__STREAM_4_SYNCHRONIZATION_MASK                                                 0x00000010L
+#define STREAM_SYNCHRONIZATION__STREAM_5_SYNCHRONIZATION_MASK                                                 0x00000020L
+#define STREAM_SYNCHRONIZATION__STREAM_6_SYNCHRONIZATION_MASK                                                 0x00000040L
+#define STREAM_SYNCHRONIZATION__STREAM_7_SYNCHRONIZATION_MASK                                                 0x00000080L
+#define STREAM_SYNCHRONIZATION__STREAM_8_SYNCHRONIZATION_MASK                                                 0x00000100L
+#define STREAM_SYNCHRONIZATION__STREAM_9_SYNCHRONIZATION_MASK                                                 0x00000200L
+#define STREAM_SYNCHRONIZATION__STREAM_10_SYNCHRONIZATION_MASK                                                0x00000400L
+#define STREAM_SYNCHRONIZATION__STREAM_11_SYNCHRONIZATION_MASK                                                0x00000800L
+#define STREAM_SYNCHRONIZATION__STREAM_12_SYNCHRONIZATION_MASK                                                0x00001000L
+#define STREAM_SYNCHRONIZATION__STREAM_13_SYNCHRONIZATION_MASK                                                0x00002000L
+#define STREAM_SYNCHRONIZATION__STREAM_14_SYNCHRONIZATION_MASK                                                0x00004000L
+#define STREAM_SYNCHRONIZATION__STREAM_15_SYNCHRONIZATION_MASK                                                0x00008000L
+//CORB_LOWER_BASE_ADDRESS
+#define CORB_LOWER_BASE_ADDRESS__CORB_LOWER_BASE_UNIMPLEMENTED_BITS__SHIFT                                    0x0
+#define CORB_LOWER_BASE_ADDRESS__CORB_LOWER_BASE_ADDRESS__SHIFT                                               0x7
+#define CORB_LOWER_BASE_ADDRESS__CORB_LOWER_BASE_UNIMPLEMENTED_BITS_MASK                                      0x0000007FL
+#define CORB_LOWER_BASE_ADDRESS__CORB_LOWER_BASE_ADDRESS_MASK                                                 0xFFFFFF80L
+//CORB_UPPER_BASE_ADDRESS
+#define CORB_UPPER_BASE_ADDRESS__CORB_UPPER_BASE_ADDRESS__SHIFT                                               0x0
+#define CORB_UPPER_BASE_ADDRESS__CORB_UPPER_BASE_ADDRESS_MASK                                                 0xFFFFFFFFL
+//AZCONTROLLER1_CORB_WRITE_POINTER
+#define AZCONTROLLER1_CORB_WRITE_POINTER__CORB_WRITE_POINTER__SHIFT                                           0x0
+#define AZCONTROLLER1_CORB_WRITE_POINTER__CORB_WRITE_POINTER_MASK                                             0x00FFL
+//AZCONTROLLER1_CORB_READ_POINTER
+#define AZCONTROLLER1_CORB_READ_POINTER__CORB_READ_POINTER__SHIFT                                             0x0
+#define AZCONTROLLER1_CORB_READ_POINTER__CORB_READ_POINTER_RESET__SHIFT                                       0xf
+#define AZCONTROLLER1_CORB_READ_POINTER__CORB_READ_POINTER_MASK                                               0x00FFL
+#define AZCONTROLLER1_CORB_READ_POINTER__CORB_READ_POINTER_RESET_MASK                                         0x8000L
+//AZCONTROLLER1_CORB_CONTROL
+#define AZCONTROLLER1_CORB_CONTROL__CORB_MEMORY_ERROR_INTERRUPT_ENABLE__SHIFT                                 0x0
+#define AZCONTROLLER1_CORB_CONTROL__ENABLE_CORB_DMA_ENGINE__SHIFT                                             0x1
+#define AZCONTROLLER1_CORB_CONTROL__CORB_MEMORY_ERROR_INTERRUPT_ENABLE_MASK                                   0x01L
+#define AZCONTROLLER1_CORB_CONTROL__ENABLE_CORB_DMA_ENGINE_MASK                                               0x02L
+//AZCONTROLLER1_CORB_STATUS
+#define AZCONTROLLER1_CORB_STATUS__CORB_MEMORY_ERROR_INDICATION__SHIFT                                        0x0
+#define AZCONTROLLER1_CORB_STATUS__CORB_MEMORY_ERROR_INDICATION_MASK                                          0x01L
+//AZCONTROLLER1_CORB_SIZE
+#define AZCONTROLLER1_CORB_SIZE__CORB_SIZE__SHIFT                                                             0x0
+#define AZCONTROLLER1_CORB_SIZE__CORB_SIZE_CAPABILITY__SHIFT                                                  0x4
+#define AZCONTROLLER1_CORB_SIZE__CORB_SIZE_MASK                                                               0x0003L
+#define AZCONTROLLER1_CORB_SIZE__CORB_SIZE_CAPABILITY_MASK                                                    0x00F0L
+//AZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS
+#define AZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_UNIMPLEMENTED_BITS__SHIFT                      0x0
+#define AZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_ADDRESS__SHIFT                                 0x7
+#define AZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_UNIMPLEMENTED_BITS_MASK                        0x0000007FL
+#define AZCONTROLLER1_RIRB_LOWER_BASE_ADDRESS__RIRB_LOWER_BASE_ADDRESS_MASK                                   0xFFFFFF80L
+//AZCONTROLLER1_RIRB_UPPER_BASE_ADDRESS
+#define AZCONTROLLER1_RIRB_UPPER_BASE_ADDRESS__RIRB_UPPER_BASE_ADDRESS__SHIFT                                 0x0
+#define AZCONTROLLER1_RIRB_UPPER_BASE_ADDRESS__RIRB_UPPER_BASE_ADDRESS_MASK                                   0xFFFFFFFFL
+//AZCONTROLLER1_RIRB_WRITE_POINTER
+#define AZCONTROLLER1_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER__SHIFT                                           0x0
+#define AZCONTROLLER1_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_RESET__SHIFT                                     0xf
+#define AZCONTROLLER1_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_MASK                                             0x00FFL
+#define AZCONTROLLER1_RIRB_WRITE_POINTER__RIRB_WRITE_POINTER_RESET_MASK                                       0x8000L
+//AZCONTROLLER1_RESPONSE_INTERRUPT_COUNT
+#define AZCONTROLLER1_RESPONSE_INTERRUPT_COUNT__N_RESPONSE_INTERRUPT_COUNT__SHIFT                             0x0
+#define AZCONTROLLER1_RESPONSE_INTERRUPT_COUNT__N_RESPONSE_INTERRUPT_COUNT_MASK                               0x00FFL
+//AZCONTROLLER1_RIRB_CONTROL
+#define AZCONTROLLER1_RIRB_CONTROL__RESPONSE_INTERRUPT_CONTROL__SHIFT                                         0x0
+#define AZCONTROLLER1_RIRB_CONTROL__RIRB_DMA_ENABLE__SHIFT                                                    0x1
+#define AZCONTROLLER1_RIRB_CONTROL__RESPONSE_OVERRUN_INTERRUPT_CONTROL__SHIFT                                 0x2
+#define AZCONTROLLER1_RIRB_CONTROL__RESPONSE_INTERRUPT_CONTROL_MASK                                           0x01L
+#define AZCONTROLLER1_RIRB_CONTROL__RIRB_DMA_ENABLE_MASK                                                      0x02L
+#define AZCONTROLLER1_RIRB_CONTROL__RESPONSE_OVERRUN_INTERRUPT_CONTROL_MASK                                   0x04L
+//AZCONTROLLER1_RIRB_STATUS
+#define AZCONTROLLER1_RIRB_STATUS__RESPONSE_INTERRUPT__SHIFT                                                  0x0
+#define AZCONTROLLER1_RIRB_STATUS__RESPONSE_OVERRUN_INTERRUPT_STATUS__SHIFT                                   0x2
+#define AZCONTROLLER1_RIRB_STATUS__RESPONSE_INTERRUPT_MASK                                                    0x01L
+#define AZCONTROLLER1_RIRB_STATUS__RESPONSE_OVERRUN_INTERRUPT_STATUS_MASK                                     0x04L
+//AZCONTROLLER1_RIRB_SIZE
+#define AZCONTROLLER1_RIRB_SIZE__RIRB_SIZE__SHIFT                                                             0x0
+#define AZCONTROLLER1_RIRB_SIZE__RIRB_SIZE_CAPABILITY__SHIFT                                                  0x4
+#define AZCONTROLLER1_RIRB_SIZE__RIRB_SIZE_MASK                                                               0x0003L
+#define AZCONTROLLER1_RIRB_SIZE__RIRB_SIZE_CAPABILITY_MASK                                                    0x00F0L
+//AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_VERB_AND_PAYLOAD__SHIFT     0x0
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_CODEC_ADDRESS__SHIFT        0x1c
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_VERB_AND_PAYLOAD_MASK       0x0FFFFFFFL
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE__IMMEDIATE_COMMAND_WRITE_CODEC_ADDRESS_MASK          0xF0000000L
+//AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT                 0x0
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK                   0xFFFFFFFFL
+//AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT                0x0
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK                  0x0000FFFFL
+//AZCONTROLLER1_IMMEDIATE_RESPONSE_INPUT_INTERFACE
+#define AZCONTROLLER1_IMMEDIATE_RESPONSE_INPUT_INTERFACE__IMMEDIATE_RESPONSE_READ__SHIFT                      0x0
+#define AZCONTROLLER1_IMMEDIATE_RESPONSE_INPUT_INTERFACE__IMMEDIATE_RESPONSE_READ_MASK                        0xFFFFFFFFL
+//AZCONTROLLER1_IMMEDIATE_COMMAND_STATUS
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_COMMAND_BUSY__SHIFT                                 0x0
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_RESULT_VALID__SHIFT                                 0x1
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_COMMAND_BUSY_MASK                                   0x00000001L
+#define AZCONTROLLER1_IMMEDIATE_COMMAND_STATUS__IMMEDIATE_RESULT_VALID_MASK                                   0x00000002L
+//AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_BUFFER_ENABLE__SHIFT                      0x0
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_UNIMPLEMENTED_BITS__SHIFT      0x1
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_ADDRESS__SHIFT                 0x7
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_BUFFER_ENABLE_MASK                        0x00000001L
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_UNIMPLEMENTED_BITS_MASK        0x0000007EL
+#define AZCONTROLLER1_DMA_POSITION_LOWER_BASE_ADDRESS__DMA_POSITION_LOWER_BASE_ADDRESS_MASK                   0xFFFFFF80L
+//AZCONTROLLER1_DMA_POSITION_UPPER_BASE_ADDRESS
+#define AZCONTROLLER1_DMA_POSITION_UPPER_BASE_ADDRESS__DMA_POSITION_UPPER_BASE_ADDRESS__SHIFT                 0x0
+#define AZCONTROLLER1_DMA_POSITION_UPPER_BASE_ADDRESS__DMA_POSITION_UPPER_BASE_ADDRESS_MASK                   0xFFFFFFFFL
+//AZCONTROLLER1_WALL_CLOCK_COUNTER_ALIAS
+#define AZCONTROLLER1_WALL_CLOCK_COUNTER_ALIAS__WALL_CLOCK_COUNTER_ALIAS__SHIFT                               0x0
+#define AZCONTROLLER1_WALL_CLOCK_COUNTER_ALIAS__WALL_CLOCK_COUNTER_ALIAS_MASK                                 0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azendpoint_azdec
+//AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT        0x0
+#define AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK          0xFFFFFFFFL
+//AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT       0x0
+#define AZENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK         0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azinputendpoint_azdec
+//AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA
+#define AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT    0x0
+#define AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK      0xFFFFFFFFL
+//AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX
+#define AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT   0x0
+#define AZINPUTENDPOINT1_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK     0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azroot_azdec
+//AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA
+#define AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE__SHIFT                0x0
+#define AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA__IMMEDIATE_COMMAND_WRITE_MASK                  0xFFFFFFFFL
+//AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX
+#define AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE__SHIFT               0x0
+#define AZROOT1_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX__IMMEDIATE_COMMAND_WRITE_MASK                 0x0001FFFFL
+
+
+// addressBlock: dce_dc_hda_azstream0_azdec
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream1_azdec
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream2_azdec
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream3_azdec
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream4_azdec
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream5_azdec
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream6_azdec
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+
+// addressBlock: dce_dc_hda_azstream7_azdec
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET__SHIFT                          0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN__SHIFT                            0x1
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE__SHIFT        0x2
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE__SHIFT           0x3
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE__SHIFT     0x4
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL__SHIFT                        0x10
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY__SHIFT                      0x12
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER__SHIFT                         0x14
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS__SHIFT    0x1a
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR__SHIFT                            0x1b
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR__SHIFT                      0x1c
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY__SHIFT                            0x1d
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RESET_MASK                            0x00000001L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_RUN_MASK                              0x00000002L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__INTERRUPT_ON_COMPLETION_ENABLE_MASK          0x00000004L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_INTERRUPT_ENABLE_MASK             0x00000008L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_INTERRUPT_ENABLE_MASK       0x00000010L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STRIPE_CONTROL_MASK                          0x00030000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__TRAFFIC_PRIORITY_MASK                        0x00040000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__STREAM_NUMBER_MASK                           0x00F00000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__BUFFER_COMPLETION_INTERRUPT_STATUS_MASK      0x04000000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_ERROR_MASK                              0x08000000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__DESCRIPTOR_ERROR_MASK                        0x10000000L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS__FIFO_READY_MASK                              0x20000000L
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER__SHIFT  0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER__LINK_POSITION_IN_BUFFER_MASK    0xFFFFFFFFL
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH__SHIFT                0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH__CYCLIC_BUFFER_LENGTH_MASK                  0xFFFFFFFFL
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX__SHIFT                        0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX__LAST_VALID_INDEX_MASK                          0x000000FFL
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE__SHIFT                                      0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE__FIFO_SIZE_MASK                                        0xFFFFL
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS__SHIFT                                0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE__SHIFT                                   0x4
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT                               0x8
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT                              0xb
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE__SHIFT                                  0xe
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__NUMBER_OF_CHANNELS_MASK                                  0x000FL
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__BITS_PER_SAMPLE_MASK                                     0x0070L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_DIVISOR_MASK                                 0x0700L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_MULTIPLE_MASK                                0x3800L
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT__SAMPLE_BASE_RATE_MASK                                    0x4000L
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS__SHIFT  0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS__SHIFT  0x7
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_UNIMPLEMENTED_BITS_MASK  0x0000007FL
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_LOWER_BASE_ADDRESS_MASK  0xFFFFFF80L
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS__SHIFT  0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS__BUFFER_DESCRIPTOR_LIST_UPPER_BASE_ADDRESS_MASK  0xFFFFFFFFL
+//AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS__SHIFT  0x0
+#define AZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS__LINK_POSITION_IN_BUFFER_ALIAS_MASK  0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_offset.h
new file mode 100644
index 0000000000000..5e99515fbf64a
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_offset.h
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _dpcs_4_0_0_OFFSET_HEADER
+#define _dpcs_4_0_0_OFFSET_HEADER
+
+// addressBlock: dpcssys_dcio_dcio_dispdec
+// base address: 0x0
+#define regDC_GENERICA                                                                                  0x2868
+#define regDC_GENERICA_BASE_IDX                                                                         2
+#define regDC_GENERICB                                                                                  0x2869
+#define regDC_GENERICB_BASE_IDX                                                                         2
+#define regDCIO_CLOCK_CNTL                                                                              0x286a
+#define regDCIO_CLOCK_CNTL_BASE_IDX                                                                     2
+#define regDC_REF_CLK_CNTL                                                                              0x286b
+#define regDC_REF_CLK_CNTL_BASE_IDX                                                                     2
+#define regUNIPHYA_LINK_CNTL                                                                            0x286d
+#define regUNIPHYA_LINK_CNTL_BASE_IDX                                                                   2
+#define regUNIPHYA_CHANNEL_XBAR_CNTL                                                                    0x286e
+#define regUNIPHYA_CHANNEL_XBAR_CNTL_BASE_IDX                                                           2
+#define regUNIPHYB_LINK_CNTL                                                                            0x286f
+#define regUNIPHYB_LINK_CNTL_BASE_IDX                                                                   2
+#define regUNIPHYB_CHANNEL_XBAR_CNTL                                                                    0x2870
+#define regUNIPHYB_CHANNEL_XBAR_CNTL_BASE_IDX                                                           2
+#define regUNIPHYC_LINK_CNTL                                                                            0x2871
+#define regUNIPHYC_LINK_CNTL_BASE_IDX                                                                   2
+#define regUNIPHYC_CHANNEL_XBAR_CNTL                                                                    0x2872
+#define regUNIPHYC_CHANNEL_XBAR_CNTL_BASE_IDX                                                           2
+#define regUNIPHYD_LINK_CNTL                                                                            0x2873
+#define regUNIPHYD_LINK_CNTL_BASE_IDX                                                                   2
+#define regUNIPHYD_CHANNEL_XBAR_CNTL                                                                    0x2874
+#define regUNIPHYD_CHANNEL_XBAR_CNTL_BASE_IDX                                                           2
+#define regUNIPHYE_LINK_CNTL                                                                            0x2875
+#define regUNIPHYE_LINK_CNTL_BASE_IDX                                                                   2
+#define regUNIPHYE_CHANNEL_XBAR_CNTL                                                                    0x2876
+#define regUNIPHYE_CHANNEL_XBAR_CNTL_BASE_IDX                                                           2
+#define regDCIO_WRCMD_DELAY                                                                             0x287e
+#define regDCIO_WRCMD_DELAY_BASE_IDX                                                                    2
+#define regDC_PINSTRAPS                                                                                 0x2880
+#define regDC_PINSTRAPS_BASE_IDX                                                                        2
+#define regDCIO_SPARE                                                                                   0x2882
+#define regDCIO_SPARE_BASE_IDX                                                                          2
+#define regINTERCEPT_STATE                                                                              0x2884
+#define regINTERCEPT_STATE_BASE_IDX                                                                     2
+#define regDCIO_PATTERN_GEN_PAT                                                                         0x2886
+#define regDCIO_PATTERN_GEN_PAT_BASE_IDX                                                                2
+#define regDCIO_PATTERN_GEN_EN                                                                          0x2887
+#define regDCIO_PATTERN_GEN_EN_BASE_IDX                                                                 2
+#define regDCIO_BL_PWM_FRAME_START_DISP_SEL                                                             0x288b
+#define regDCIO_BL_PWM_FRAME_START_DISP_SEL_BASE_IDX                                                    2
+#define regDCIO_GSL_GENLK_PAD_CNTL                                                                      0x288c
+#define regDCIO_GSL_GENLK_PAD_CNTL_BASE_IDX                                                             2
+#define regDCIO_GSL_SWAPLOCK_PAD_CNTL                                                                   0x288d
+#define regDCIO_GSL_SWAPLOCK_PAD_CNTL_BASE_IDX                                                          2
+#define regDPCS_DCIO_TEST_CLK_SRC                                                                       0x2890
+#define regDPCS_DCIO_TEST_CLK_SRC_BASE_IDX                                                              2
+#define regDCIO_DEBUG                                                                                   0x2897
+#define regDCIO_DEBUG_BASE_IDX                                                                          2
+#define regDCIO_TEST_DEBUG_INDEX                                                                        0x2899
+#define regDCIO_TEST_DEBUG_INDEX_BASE_IDX                                                               2
+#define regDCIO_TEST_DEBUG_DATA                                                                         0x289a
+#define regDCIO_TEST_DEBUG_DATA_BASE_IDX                                                                2
+#define regDBG_OUT_CNTL                                                                                 0x289c
+#define regDBG_OUT_CNTL_BASE_IDX                                                                        2
+#define regDCIO_DEBUG_CONFIG                                                                            0x289d
+#define regDCIO_DEBUG_CONFIG_BASE_IDX                                                                   2
+#define regDCIO_SOFT_RESET                                                                              0x289e
+#define regDCIO_SOFT_RESET_BASE_IDX                                                                     2
+
+
+// addressBlock: dpcssys_dcio_dcio_chip_dispdec
+// base address: 0x0
+#define regDC_GPIO_DDC1_MASK                                                                            0x28d0
+#define regDC_GPIO_DDC1_MASK_BASE_IDX                                                                   2
+#define regDC_GPIO_DDC1_A                                                                               0x28d1
+#define regDC_GPIO_DDC1_A_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC1_EN                                                                              0x28d2
+#define regDC_GPIO_DDC1_EN_BASE_IDX                                                                     2
+#define regDC_GPIO_DDC1_Y                                                                               0x28d3
+#define regDC_GPIO_DDC1_Y_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC2_MASK                                                                            0x28d4
+#define regDC_GPIO_DDC2_MASK_BASE_IDX                                                                   2
+#define regDC_GPIO_DDC2_A                                                                               0x28d5
+#define regDC_GPIO_DDC2_A_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC2_EN                                                                              0x28d6
+#define regDC_GPIO_DDC2_EN_BASE_IDX                                                                     2
+#define regDC_GPIO_DDC2_Y                                                                               0x28d7
+#define regDC_GPIO_DDC2_Y_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC3_MASK                                                                            0x28d8
+#define regDC_GPIO_DDC3_MASK_BASE_IDX                                                                   2
+#define regDC_GPIO_DDC3_A                                                                               0x28d9
+#define regDC_GPIO_DDC3_A_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC3_EN                                                                              0x28da
+#define regDC_GPIO_DDC3_EN_BASE_IDX                                                                     2
+#define regDC_GPIO_DDC3_Y                                                                               0x28db
+#define regDC_GPIO_DDC3_Y_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC4_MASK                                                                            0x28dc
+#define regDC_GPIO_DDC4_MASK_BASE_IDX                                                                   2
+#define regDC_GPIO_DDC4_A                                                                               0x28dd
+#define regDC_GPIO_DDC4_A_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC4_EN                                                                              0x28de
+#define regDC_GPIO_DDC4_EN_BASE_IDX                                                                     2
+#define regDC_GPIO_DDC4_Y                                                                               0x28df
+#define regDC_GPIO_DDC4_Y_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC5_MASK                                                                            0x28e0
+#define regDC_GPIO_DDC5_MASK_BASE_IDX                                                                   2
+#define regDC_GPIO_DDC5_A                                                                               0x28e1
+#define regDC_GPIO_DDC5_A_BASE_IDX                                                                      2
+#define regDC_GPIO_DDC5_EN                                                                              0x28e2
+#define regDC_GPIO_DDC5_EN_BASE_IDX                                                                     2
+#define regDC_GPIO_DDC5_Y                                                                               0x28e3
+#define regDC_GPIO_DDC5_Y_BASE_IDX                                                                      2
+#define regDC_GPIO_DDCVGA_MASK                                                                          0x28e8
+#define regDC_GPIO_DDCVGA_MASK_BASE_IDX                                                                 2
+#define regDC_GPIO_DDCVGA_A                                                                             0x28e9
+#define regDC_GPIO_DDCVGA_A_BASE_IDX                                                                    2
+#define regDC_GPIO_DDCVGA_EN                                                                            0x28ea
+#define regDC_GPIO_DDCVGA_EN_BASE_IDX                                                                   2
+#define regDC_GPIO_DDCVGA_Y                                                                             0x28eb
+#define regDC_GPIO_DDCVGA_Y_BASE_IDX                                                                    2
+#define regDC_GPIO_PWRSEQ0_EN                                                                           0x28fa
+#define regDC_GPIO_PWRSEQ0_EN_BASE_IDX                                                                  2
+#define regDC_GPIO_PAD_STRENGTH_1                                                                       0x28fc
+#define regDC_GPIO_PAD_STRENGTH_1_BASE_IDX                                                              2
+#define regPHY_AUX_CNTL                                                                                 0x28ff
+#define regPHY_AUX_CNTL_BASE_IDX                                                                        2
+#define regDC_GPIO_PWRSEQ1_EN                                                                           0x2902
+#define regDC_GPIO_PWRSEQ1_EN_BASE_IDX                                                                  2
+#define regDC_GPIO_AUX_CTRL_0                                                                           0x2916
+#define regDC_GPIO_AUX_CTRL_0_BASE_IDX                                                                  2
+#define regDC_GPIO_AUX_CTRL_1                                                                           0x2917
+#define regDC_GPIO_AUX_CTRL_1_BASE_IDX                                                                  2
+#define regDC_GPIO_AUX_CTRL_3                                                                           0x291b
+#define regDC_GPIO_AUX_CTRL_3_BASE_IDX                                                                  2
+#define regDC_GPIO_AUX_CTRL_4                                                                           0x291c
+#define regDC_GPIO_AUX_CTRL_4_BASE_IDX                                                                  2
+#define regDC_GPIO_AUX_CTRL_5                                                                           0x291d
+#define regDC_GPIO_AUX_CTRL_5_BASE_IDX                                                                  2
+#define regAUXI2C_PAD_ALL_PWR_OK                                                                        0x291e
+#define regAUXI2C_PAD_ALL_PWR_OK_BASE_IDX                                                               2
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_sh_mask.h
new file mode 100644
index 0000000000000..e124cc13bfa64
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_0_0_sh_mask.h
@@ -0,0 +1,688 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Advanced Micro Devices, Inc. */
+
+#ifndef _dpcs_4_0_0_SH_MASK_HEADER
+#define _dpcs_4_0_0_SH_MASK_HEADER
+
+// addressBlock: dpcssys_dcio_dcio_dispdec
+//DC_GENERICA
+#define DC_GENERICA__GENERICA_SEL__SHIFT                                                                      0x7
+#define DC_GENERICA__GENERICA_SEL_MASK                                                                        0x00000F80L
+//DC_GENERICB
+#define DC_GENERICB__GENERICB_SEL__SHIFT                                                                      0x8
+#define DC_GENERICB__GENERICB_SEL_MASK                                                                        0x00000F00L
+//DCIO_CLOCK_CNTL
+#define DCIO_CLOCK_CNTL__DCIO_TEST_CLK_SEL__SHIFT                                                             0x0
+#define DCIO_CLOCK_CNTL__DISPCLK_R_DCIO_GATE_DIS__SHIFT                                                       0x5
+#define DCIO_CLOCK_CNTL__DCIO_TEST_CLK_SEL_MASK                                                               0x0000001FL
+#define DCIO_CLOCK_CNTL__DISPCLK_R_DCIO_GATE_DIS_MASK                                                         0x00000020L
+//DC_REF_CLK_CNTL
+#define DC_REF_CLK_CNTL__GENLK_CLK_OUTPUT_SEL__SHIFT                                                          0x8
+#define DC_REF_CLK_CNTL__GENLK_CLK_OUTPUT_SEL_MASK                                                            0x00000300L
+//UNIPHYA_LINK_CNTL
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT                                                      0xc
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT                                                      0xd
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT                                                      0xe
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT                                                      0xf
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK                                                        0x00001000L
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK                                                        0x00002000L
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK                                                        0x00004000L
+#define UNIPHYA_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK                                                        0x00008000L
+//UNIPHYA_CHANNEL_XBAR_CNTL
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT                                         0x0
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT                                         0x8
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT                                         0x10
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT                                         0x18
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN__SHIFT                                                0x1c
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN__SHIFT                                                0x1d
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN__SHIFT                                                0x1e
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN__SHIFT                                                0x1f
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK                                           0x00000003L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK                                           0x00000300L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK                                           0x00030000L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK                                           0x03000000L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN_MASK                                                  0x10000000L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN_MASK                                                  0x20000000L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN_MASK                                                  0x40000000L
+#define UNIPHYA_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN_MASK                                                  0x80000000L
+//UNIPHYB_LINK_CNTL
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT                                                      0xc
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT                                                      0xd
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT                                                      0xe
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT                                                      0xf
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK                                                        0x00001000L
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK                                                        0x00002000L
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK                                                        0x00004000L
+#define UNIPHYB_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK                                                        0x00008000L
+//UNIPHYB_CHANNEL_XBAR_CNTL
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT                                         0x0
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT                                         0x8
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT                                         0x10
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT                                         0x18
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN__SHIFT                                                0x1c
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN__SHIFT                                                0x1d
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN__SHIFT                                                0x1e
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN__SHIFT                                                0x1f
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK                                           0x00000003L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK                                           0x00000300L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK                                           0x00030000L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK                                           0x03000000L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN_MASK                                                  0x10000000L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN_MASK                                                  0x20000000L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN_MASK                                                  0x40000000L
+#define UNIPHYB_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN_MASK                                                  0x80000000L
+//UNIPHYC_LINK_CNTL
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT                                                      0xc
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT                                                      0xd
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT                                                      0xe
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT                                                      0xf
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK                                                        0x00001000L
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK                                                        0x00002000L
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK                                                        0x00004000L
+#define UNIPHYC_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK                                                        0x00008000L
+//UNIPHYC_CHANNEL_XBAR_CNTL
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT                                         0x0
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT                                         0x8
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT                                         0x10
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT                                         0x18
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN__SHIFT                                                0x1c
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN__SHIFT                                                0x1d
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN__SHIFT                                                0x1e
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN__SHIFT                                                0x1f
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK                                           0x00000003L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK                                           0x00000300L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK                                           0x00030000L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK                                           0x03000000L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN_MASK                                                  0x10000000L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN_MASK                                                  0x20000000L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN_MASK                                                  0x40000000L
+#define UNIPHYC_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN_MASK                                                  0x80000000L
+//UNIPHYD_LINK_CNTL
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT                                                      0xc
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT                                                      0xd
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT                                                      0xe
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT                                                      0xf
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK                                                        0x00001000L
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK                                                        0x00002000L
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK                                                        0x00004000L
+#define UNIPHYD_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK                                                        0x00008000L
+//UNIPHYD_CHANNEL_XBAR_CNTL
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT                                         0x0
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT                                         0x8
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT                                         0x10
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT                                         0x18
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN__SHIFT                                                0x1c
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN__SHIFT                                                0x1d
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN__SHIFT                                                0x1e
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN__SHIFT                                                0x1f
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK                                           0x00000003L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK                                           0x00000300L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK                                           0x00030000L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK                                           0x03000000L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN_MASK                                                  0x10000000L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN_MASK                                                  0x20000000L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN_MASK                                                  0x40000000L
+#define UNIPHYD_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN_MASK                                                  0x80000000L
+//UNIPHYE_LINK_CNTL
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT                                                      0xc
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT                                                      0xd
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT                                                      0xe
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT                                                      0xf
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK                                                        0x00001000L
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK                                                        0x00002000L
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK                                                        0x00004000L
+#define UNIPHYE_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK                                                        0x00008000L
+//UNIPHYE_CHANNEL_XBAR_CNTL
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT                                         0x0
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT                                         0x8
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT                                         0x10
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT                                         0x18
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN__SHIFT                                                0x1c
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN__SHIFT                                                0x1d
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN__SHIFT                                                0x1e
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN__SHIFT                                                0x1f
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK                                           0x00000003L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK                                           0x00000300L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK                                           0x00030000L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK                                           0x03000000L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL0_EN_MASK                                                  0x10000000L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL1_EN_MASK                                                  0x20000000L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL2_EN_MASK                                                  0x40000000L
+#define UNIPHYE_CHANNEL_XBAR_CNTL__DOUT_PHY_CHANNEL3_EN_MASK                                                  0x80000000L
+//DCIO_WRCMD_DELAY
+#define DCIO_WRCMD_DELAY__UNIPHY_DELAY__SHIFT                                                                 0x18
+#define DCIO_WRCMD_DELAY__UNIPHY_DELAY_MASK                                                                   0xFF000000L
+//DC_PINSTRAPS
+#define DC_PINSTRAPS__DC_PINSTRAPS_SMS_EN_HARD__SHIFT                                                         0xd
+#define DC_PINSTRAPS__DC_PINSTRAPS_AUDIO__SHIFT                                                               0xe
+#define DC_PINSTRAPS__DC_PINSTRAPS_CCBYPASS__SHIFT                                                            0x10
+#define DC_PINSTRAPS__DC_PINSTRAPS_SMS_EN_HARD_MASK                                                           0x00002000L
+#define DC_PINSTRAPS__DC_PINSTRAPS_AUDIO_MASK                                                                 0x0000C000L
+#define DC_PINSTRAPS__DC_PINSTRAPS_CCBYPASS_MASK                                                              0x00010000L
+//DCIO_SPARE
+#define DCIO_SPARE__DCIO_SPARE__SHIFT                                                                         0x0
+#define DCIO_SPARE__DCIO_SPARE_MASK                                                                           0xFFFFFFFFL
+//INTERCEPT_STATE
+#define INTERCEPT_STATE__PWRSEQ0_INTERCEPTB_STATE__SHIFT                                                      0x0
+#define INTERCEPT_STATE__PWRSEQ1_INTERCEPTB_STATE__SHIFT                                                      0x1
+#define INTERCEPT_STATE__DLPC_INTERCEPTB_STATE__SHIFT                                                         0x2
+#define INTERCEPT_STATE__DPCS0_INTERCEPTB_STATE__SHIFT                                                        0x4
+#define INTERCEPT_STATE__DPCS1_INTERCEPTB_STATE__SHIFT                                                        0x5
+#define INTERCEPT_STATE__DPCS2_INTERCEPTB_STATE__SHIFT                                                        0x6
+#define INTERCEPT_STATE__DPCS3_INTERCEPTB_STATE__SHIFT                                                        0x7
+#define INTERCEPT_STATE__DPCS4_INTERCEPTB_STATE__SHIFT                                                        0x8
+#define INTERCEPT_STATE__DPCS5_INTERCEPTB_STATE__SHIFT                                                        0x9
+#define INTERCEPT_STATE__DPCS6_INTERCEPTB_STATE__SHIFT                                                        0xa
+#define INTERCEPT_STATE__DPCS0_DC_INTERCEPTB_STATE__SHIFT                                                     0x10
+#define INTERCEPT_STATE__DPCS1_DC_INTERCEPTB_STATE__SHIFT                                                     0x11
+#define INTERCEPT_STATE__DPCS2_DC_INTERCEPTB_STATE__SHIFT                                                     0x12
+#define INTERCEPT_STATE__DPCS3_DC_INTERCEPTB_STATE__SHIFT                                                     0x13
+#define INTERCEPT_STATE__DPCS4_DC_INTERCEPTB_STATE__SHIFT                                                     0x14
+#define INTERCEPT_STATE__DPCS5_DC_INTERCEPTB_STATE__SHIFT                                                     0x15
+#define INTERCEPT_STATE__DPCS6_DC_INTERCEPTB_STATE__SHIFT                                                     0x16
+#define INTERCEPT_STATE__PWRSEQ0_INTERCEPTB_STATE_MASK                                                        0x00000001L
+#define INTERCEPT_STATE__PWRSEQ1_INTERCEPTB_STATE_MASK                                                        0x00000002L
+#define INTERCEPT_STATE__DLPC_INTERCEPTB_STATE_MASK                                                           0x00000004L
+#define INTERCEPT_STATE__DPCS0_INTERCEPTB_STATE_MASK                                                          0x00000010L
+#define INTERCEPT_STATE__DPCS1_INTERCEPTB_STATE_MASK                                                          0x00000020L
+#define INTERCEPT_STATE__DPCS2_INTERCEPTB_STATE_MASK                                                          0x00000040L
+#define INTERCEPT_STATE__DPCS3_INTERCEPTB_STATE_MASK                                                          0x00000080L
+#define INTERCEPT_STATE__DPCS4_INTERCEPTB_STATE_MASK                                                          0x00000100L
+#define INTERCEPT_STATE__DPCS5_INTERCEPTB_STATE_MASK                                                          0x00000200L
+#define INTERCEPT_STATE__DPCS6_INTERCEPTB_STATE_MASK                                                          0x00000400L
+#define INTERCEPT_STATE__DPCS0_DC_INTERCEPTB_STATE_MASK                                                       0x00010000L
+#define INTERCEPT_STATE__DPCS1_DC_INTERCEPTB_STATE_MASK                                                       0x00020000L
+#define INTERCEPT_STATE__DPCS2_DC_INTERCEPTB_STATE_MASK                                                       0x00040000L
+#define INTERCEPT_STATE__DPCS3_DC_INTERCEPTB_STATE_MASK                                                       0x00080000L
+#define INTERCEPT_STATE__DPCS4_DC_INTERCEPTB_STATE_MASK                                                       0x00100000L
+#define INTERCEPT_STATE__DPCS5_DC_INTERCEPTB_STATE_MASK                                                       0x00200000L
+#define INTERCEPT_STATE__DPCS6_DC_INTERCEPTB_STATE_MASK                                                       0x00400000L
+//DCIO_PATTERN_GEN_PAT
+#define DCIO_PATTERN_GEN_PAT__DCIO_PATTERN_GEN_PAT__SHIFT                                                     0x0
+#define DCIO_PATTERN_GEN_PAT__DCIO_PATTERN_GEN_PAT_MASK                                                       0xFFFFFFFFL
+//DCIO_PATTERN_GEN_EN
+#define DCIO_PATTERN_GEN_EN__DCIO_PATTERN_GEN_EN__SHIFT                                                       0x0
+#define DCIO_PATTERN_GEN_EN__DCIO_PATTERN_GEN_EN_MASK                                                         0x00000001L
+//DCIO_BL_PWM_FRAME_START_DISP_SEL
+#define DCIO_BL_PWM_FRAME_START_DISP_SEL__BL_PWM0_GRP1_FRAME_START_DISP_SEL__SHIFT                            0x0
+#define DCIO_BL_PWM_FRAME_START_DISP_SEL__BL_PWM1_GRP1_FRAME_START_DISP_SEL__SHIFT                            0x4
+#define DCIO_BL_PWM_FRAME_START_DISP_SEL__BL_PWM0_GRP1_FRAME_START_DISP_SEL_MASK                              0x00000007L
+#define DCIO_BL_PWM_FRAME_START_DISP_SEL__BL_PWM1_GRP1_FRAME_START_DISP_SEL_MASK                              0x00000070L
+//DCIO_GSL_GENLK_PAD_CNTL
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_FLIP_READY_SEL__SHIFT                                     0x4
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_MASK__SHIFT                                               0x8
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_FLIP_READY_SEL__SHIFT                                   0x14
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_MASK__SHIFT                                             0x18
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_FLIP_READY_SEL_MASK                                       0x00000030L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_MASK_MASK                                                 0x00000300L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_FLIP_READY_SEL_MASK                                     0x00300000L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_MASK_MASK                                               0x03000000L
+//DCIO_GSL_SWAPLOCK_PAD_CNTL
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_FLIP_READY_SEL__SHIFT                                 0x4
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_MASK__SHIFT                                           0x8
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_FLIP_READY_SEL__SHIFT                                 0x14
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_MASK__SHIFT                                           0x18
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_FLIP_READY_SEL_MASK                                   0x00000030L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_MASK_MASK                                             0x00000300L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_FLIP_READY_SEL_MASK                                   0x00300000L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_MASK_MASK                                             0x03000000L
+//DPCS_DCIO_TEST_CLK_SRC
+#define DPCS_DCIO_TEST_CLK_SRC__DPCS_TEST_CLK_SRC_SEL__SHIFT                                                  0x0
+#define DPCS_DCIO_TEST_CLK_SRC__DPCS_TEST_CLK_SRC_SEL_MASK                                                    0x00000007L
+//DCIO_DEBUG
+#define DCIO_DEBUG__DCIO_DEBUG__SHIFT                                                                         0x0
+#define DCIO_DEBUG__DCIO_DEBUG_MASK                                                                           0xFFFFFFFFL
+//DCIO_TEST_DEBUG_INDEX
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//DCIO_TEST_DEBUG_DATA
+#define DCIO_TEST_DEBUG_DATA__DCIO_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define DCIO_TEST_DEBUG_DATA__DCIO_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+//DBG_OUT_CNTL
+#define DBG_OUT_CNTL__DBG_OUT_BLK_SEL__SHIFT                                                                  0x0
+#define DBG_OUT_CNTL__DBG_OUT_4BIT_SEL__SHIFT                                                                 0x5
+#define DBG_OUT_CNTL__DBG_OUT_12BIT_TEST_DATA__SHIFT                                                          0xc
+#define DBG_OUT_CNTL__DBG_OUT_BLK_SEL_MASK                                                                    0x00000003L
+#define DBG_OUT_CNTL__DBG_OUT_4BIT_SEL_MASK                                                                   0x000000E0L
+#define DBG_OUT_CNTL__DBG_OUT_12BIT_TEST_DATA_MASK                                                            0x00FFF000L
+//DCIO_DEBUG_CONFIG
+#define DCIO_DEBUG_CONFIG__DCIO_DBG_EN__SHIFT                                                                 0x0
+#define DCIO_DEBUG_CONFIG__DCIO_DBG_EN_MASK                                                                   0x00000001L
+//DCIO_SOFT_RESET
+#define DCIO_SOFT_RESET__UNIPHYA_SOFT_RESET__SHIFT                                                            0x0
+#define DCIO_SOFT_RESET__UNIPHYB_SOFT_RESET__SHIFT                                                            0x1
+#define DCIO_SOFT_RESET__UNIPHYC_SOFT_RESET__SHIFT                                                            0x2
+#define DCIO_SOFT_RESET__UNIPHYD_SOFT_RESET__SHIFT                                                            0x3
+#define DCIO_SOFT_RESET__UNIPHYE_SOFT_RESET__SHIFT                                                            0x4
+#define DCIO_SOFT_RESET__UNIPHYF_SOFT_RESET__SHIFT                                                            0x5
+#define DCIO_SOFT_RESET__UNIPHYG_SOFT_RESET__SHIFT                                                            0x6
+#define DCIO_SOFT_RESET__PWRSEQ0_SOFT_RESET__SHIFT                                                            0x10
+#define DCIO_SOFT_RESET__PWRSEQ1_SOFT_RESET__SHIFT                                                            0x11
+#define DCIO_SOFT_RESET__DLPC_SOFT_RESET__SHIFT                                                               0x14
+#define DCIO_SOFT_RESET__UNIPHYA_SOFT_RESET_MASK                                                              0x00000001L
+#define DCIO_SOFT_RESET__UNIPHYB_SOFT_RESET_MASK                                                              0x00000002L
+#define DCIO_SOFT_RESET__UNIPHYC_SOFT_RESET_MASK                                                              0x00000004L
+#define DCIO_SOFT_RESET__UNIPHYD_SOFT_RESET_MASK                                                              0x00000008L
+#define DCIO_SOFT_RESET__UNIPHYE_SOFT_RESET_MASK                                                              0x00000010L
+#define DCIO_SOFT_RESET__UNIPHYF_SOFT_RESET_MASK                                                              0x00000020L
+#define DCIO_SOFT_RESET__UNIPHYG_SOFT_RESET_MASK                                                              0x00000040L
+#define DCIO_SOFT_RESET__PWRSEQ0_SOFT_RESET_MASK                                                              0x00010000L
+#define DCIO_SOFT_RESET__PWRSEQ1_SOFT_RESET_MASK                                                              0x00020000L
+#define DCIO_SOFT_RESET__DLPC_SOFT_RESET_MASK                                                                 0x00100000L
+
+
+// addressBlock: dpcssys_dcio_dcio_chip_dispdec
+//DC_GPIO_DDC1_MASK
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_MASK__SHIFT                                                        0x0
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_PD_EN__SHIFT                                                       0x4
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_RECV__SHIFT                                                        0x6
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_MASK__SHIFT                                                       0x8
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_PD_EN__SHIFT                                                      0xc
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_RECV__SHIFT                                                       0xe
+#define DC_GPIO_DDC1_MASK__AUX_PAD1_MODE__SHIFT                                                               0x10
+#define DC_GPIO_DDC1_MASK__AUX1_POL__SHIFT                                                                    0x14
+#define DC_GPIO_DDC1_MASK__ALLOW_HW_DDC1_PD_EN__SHIFT                                                         0x16
+#define DC_GPIO_DDC1_MASK__DC_GPIO_AUX1_PD_HP_EN__SHIFT                                                       0x18
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_MASK_MASK                                                          0x00000001L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_PD_EN_MASK                                                         0x00000010L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_RECV_MASK                                                          0x00000040L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_MASK_MASK                                                         0x00000100L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_PD_EN_MASK                                                        0x00001000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_RECV_MASK                                                         0x00004000L
+#define DC_GPIO_DDC1_MASK__AUX_PAD1_MODE_MASK                                                                 0x00010000L
+#define DC_GPIO_DDC1_MASK__AUX1_POL_MASK                                                                      0x00100000L
+#define DC_GPIO_DDC1_MASK__ALLOW_HW_DDC1_PD_EN_MASK                                                           0x00400000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_AUX1_PD_HP_EN_MASK                                                         0x01000000L
+//DC_GPIO_DDC1_A
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A__SHIFT                                                              0x0
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A__SHIFT                                                             0x8
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK                                                                0x00000001L
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK                                                               0x00000100L
+//DC_GPIO_DDC1_EN
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1CLK_EN__SHIFT                                                            0x0
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1DATA_EN__SHIFT                                                           0x8
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1CLK_EN_MASK                                                              0x00000001L
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1DATA_EN_MASK                                                             0x00000100L
+//DC_GPIO_DDC1_Y
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1CLK_Y__SHIFT                                                              0x0
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1DATA_Y__SHIFT                                                             0x8
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1CLK_Y_MASK                                                                0x00000001L
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1DATA_Y_MASK                                                               0x00000100L
+//DC_GPIO_DDC2_MASK
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_MASK__SHIFT                                                        0x0
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_PD_EN__SHIFT                                                       0x4
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_RECV__SHIFT                                                        0x6
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_MASK__SHIFT                                                       0x8
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_PD_EN__SHIFT                                                      0xc
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_RECV__SHIFT                                                       0xe
+#define DC_GPIO_DDC2_MASK__AUX_PAD2_MODE__SHIFT                                                               0x10
+#define DC_GPIO_DDC2_MASK__AUX2_POL__SHIFT                                                                    0x14
+#define DC_GPIO_DDC2_MASK__ALLOW_HW_DDC2_PD_EN__SHIFT                                                         0x16
+#define DC_GPIO_DDC2_MASK__DC_GPIO_AUX2_PD_HP_EN__SHIFT                                                       0x18
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_MASK_MASK                                                          0x00000001L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_PD_EN_MASK                                                         0x00000010L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_RECV_MASK                                                          0x00000040L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_MASK_MASK                                                         0x00000100L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_PD_EN_MASK                                                        0x00001000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_RECV_MASK                                                         0x00004000L
+#define DC_GPIO_DDC2_MASK__AUX_PAD2_MODE_MASK                                                                 0x00010000L
+#define DC_GPIO_DDC2_MASK__AUX2_POL_MASK                                                                      0x00100000L
+#define DC_GPIO_DDC2_MASK__ALLOW_HW_DDC2_PD_EN_MASK                                                           0x00400000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_AUX2_PD_HP_EN_MASK                                                         0x01000000L
+//DC_GPIO_DDC2_A
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2CLK_A__SHIFT                                                              0x0
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2DATA_A__SHIFT                                                             0x8
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2CLK_A_MASK                                                                0x00000001L
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2DATA_A_MASK                                                               0x00000100L
+//DC_GPIO_DDC2_EN
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2CLK_EN__SHIFT                                                            0x0
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2DATA_EN__SHIFT                                                           0x8
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2CLK_EN_MASK                                                              0x00000001L
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2DATA_EN_MASK                                                             0x00000100L
+//DC_GPIO_DDC2_Y
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2CLK_Y__SHIFT                                                              0x0
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2DATA_Y__SHIFT                                                             0x8
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2CLK_Y_MASK                                                                0x00000001L
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2DATA_Y_MASK                                                               0x00000100L
+//DC_GPIO_DDC3_MASK
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_MASK__SHIFT                                                        0x0
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_PD_EN__SHIFT                                                       0x4
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_RECV__SHIFT                                                        0x6
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_MASK__SHIFT                                                       0x8
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_PD_EN__SHIFT                                                      0xc
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_RECV__SHIFT                                                       0xe
+#define DC_GPIO_DDC3_MASK__AUX_PAD3_MODE__SHIFT                                                               0x10
+#define DC_GPIO_DDC3_MASK__AUX3_POL__SHIFT                                                                    0x14
+#define DC_GPIO_DDC3_MASK__ALLOW_HW_DDC3_PD_EN__SHIFT                                                         0x16
+#define DC_GPIO_DDC3_MASK__DC_GPIO_AUX3_PD_HP_EN__SHIFT                                                       0x18
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_MASK_MASK                                                          0x00000001L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_PD_EN_MASK                                                         0x00000010L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_RECV_MASK                                                          0x00000040L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_MASK_MASK                                                         0x00000100L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_PD_EN_MASK                                                        0x00001000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_RECV_MASK                                                         0x00004000L
+#define DC_GPIO_DDC3_MASK__AUX_PAD3_MODE_MASK                                                                 0x00010000L
+#define DC_GPIO_DDC3_MASK__AUX3_POL_MASK                                                                      0x00100000L
+#define DC_GPIO_DDC3_MASK__ALLOW_HW_DDC3_PD_EN_MASK                                                           0x00400000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_AUX3_PD_HP_EN_MASK                                                         0x01000000L
+//DC_GPIO_DDC3_A
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3CLK_A__SHIFT                                                              0x0
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3DATA_A__SHIFT                                                             0x8
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3CLK_A_MASK                                                                0x00000001L
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3DATA_A_MASK                                                               0x00000100L
+//DC_GPIO_DDC3_EN
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3CLK_EN__SHIFT                                                            0x0
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3DATA_EN__SHIFT                                                           0x8
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3CLK_EN_MASK                                                              0x00000001L
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3DATA_EN_MASK                                                             0x00000100L
+//DC_GPIO_DDC3_Y
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3CLK_Y__SHIFT                                                              0x0
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3DATA_Y__SHIFT                                                             0x8
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3CLK_Y_MASK                                                                0x00000001L
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3DATA_Y_MASK                                                               0x00000100L
+//DC_GPIO_DDC4_MASK
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_MASK__SHIFT                                                        0x0
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_PD_EN__SHIFT                                                       0x4
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_RECV__SHIFT                                                        0x6
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_MASK__SHIFT                                                       0x8
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_PD_EN__SHIFT                                                      0xc
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_RECV__SHIFT                                                       0xe
+#define DC_GPIO_DDC4_MASK__AUX_PAD4_MODE__SHIFT                                                               0x10
+#define DC_GPIO_DDC4_MASK__AUX4_POL__SHIFT                                                                    0x14
+#define DC_GPIO_DDC4_MASK__ALLOW_HW_DDC4_PD_EN__SHIFT                                                         0x16
+#define DC_GPIO_DDC4_MASK__DC_GPIO_AUX4_PD_HP_EN__SHIFT                                                       0x18
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_MASK_MASK                                                          0x00000001L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_PD_EN_MASK                                                         0x00000010L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_RECV_MASK                                                          0x00000040L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_MASK_MASK                                                         0x00000100L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_PD_EN_MASK                                                        0x00001000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_RECV_MASK                                                         0x00004000L
+#define DC_GPIO_DDC4_MASK__AUX_PAD4_MODE_MASK                                                                 0x00010000L
+#define DC_GPIO_DDC4_MASK__AUX4_POL_MASK                                                                      0x00100000L
+#define DC_GPIO_DDC4_MASK__ALLOW_HW_DDC4_PD_EN_MASK                                                           0x00400000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_AUX4_PD_HP_EN_MASK                                                         0x01000000L
+//DC_GPIO_DDC4_A
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4CLK_A__SHIFT                                                              0x0
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4DATA_A__SHIFT                                                             0x8
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4CLK_A_MASK                                                                0x00000001L
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4DATA_A_MASK                                                               0x00000100L
+//DC_GPIO_DDC4_EN
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4CLK_EN__SHIFT                                                            0x0
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4DATA_EN__SHIFT                                                           0x8
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4CLK_EN_MASK                                                              0x00000001L
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4DATA_EN_MASK                                                             0x00000100L
+//DC_GPIO_DDC4_Y
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4CLK_Y__SHIFT                                                              0x0
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4DATA_Y__SHIFT                                                             0x8
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4CLK_Y_MASK                                                                0x00000001L
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4DATA_Y_MASK                                                               0x00000100L
+//DC_GPIO_DDC5_MASK
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_MASK__SHIFT                                                        0x0
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_PD_EN__SHIFT                                                       0x4
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_RECV__SHIFT                                                        0x6
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_MASK__SHIFT                                                       0x8
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_PD_EN__SHIFT                                                      0xc
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_RECV__SHIFT                                                       0xe
+#define DC_GPIO_DDC5_MASK__AUX_PAD5_MODE__SHIFT                                                               0x10
+#define DC_GPIO_DDC5_MASK__AUX5_POL__SHIFT                                                                    0x14
+#define DC_GPIO_DDC5_MASK__ALLOW_HW_DDC5_PD_EN__SHIFT                                                         0x16
+#define DC_GPIO_DDC5_MASK__DC_GPIO_AUX5_PD_HP_EN__SHIFT                                                       0x18
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_MASK_MASK                                                          0x00000001L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_PD_EN_MASK                                                         0x00000010L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_RECV_MASK                                                          0x00000040L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_MASK_MASK                                                         0x00000100L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_PD_EN_MASK                                                        0x00001000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_RECV_MASK                                                         0x00004000L
+#define DC_GPIO_DDC5_MASK__AUX_PAD5_MODE_MASK                                                                 0x00010000L
+#define DC_GPIO_DDC5_MASK__AUX5_POL_MASK                                                                      0x00100000L
+#define DC_GPIO_DDC5_MASK__ALLOW_HW_DDC5_PD_EN_MASK                                                           0x00400000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_AUX5_PD_HP_EN_MASK                                                         0x01000000L
+//DC_GPIO_DDC5_A
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A__SHIFT                                                              0x0
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A__SHIFT                                                             0x8
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK                                                                0x00000001L
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK                                                               0x00000100L
+//DC_GPIO_DDC5_EN
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5CLK_EN__SHIFT                                                            0x0
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5DATA_EN__SHIFT                                                           0x8
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5CLK_EN_MASK                                                              0x00000001L
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5DATA_EN_MASK                                                             0x00000100L
+//DC_GPIO_DDC5_Y
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5CLK_Y__SHIFT                                                              0x0
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5DATA_Y__SHIFT                                                             0x8
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5CLK_Y_MASK                                                                0x00000001L
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5DATA_Y_MASK                                                               0x00000100L
+//DC_GPIO_DDCVGA_MASK
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_MASK__SHIFT                                                    0x0
+#define DC_GPIO_DDCVGA_MASK__DDCVGA_INVERT_INPUT_POLARITY__SHIFT                                              0x4
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_RECV__SHIFT                                                    0x6
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_MASK__SHIFT                                                   0x8
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_PD_EN__SHIFT                                                  0xc
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_RECV__SHIFT                                                   0xe
+#define DC_GPIO_DDCVGA_MASK__ALLOW_HW_DDCVGA_PD_EN__SHIFT                                                     0x16
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_STR__SHIFT                                                    0x1c
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_MASK_MASK                                                      0x00000001L
+#define DC_GPIO_DDCVGA_MASK__DDCVGA_INVERT_INPUT_POLARITY_MASK                                                0x00000010L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_RECV_MASK                                                      0x00000040L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_MASK_MASK                                                     0x00000100L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_PD_EN_MASK                                                    0x00001000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_RECV_MASK                                                     0x00004000L
+#define DC_GPIO_DDCVGA_MASK__ALLOW_HW_DDCVGA_PD_EN_MASK                                                       0x00400000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_STR_MASK                                                      0xF0000000L
+//DC_GPIO_DDCVGA_A
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGACLK_A__SHIFT                                                          0x0
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGADATA_A__SHIFT                                                         0x8
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGACLK_A_MASK                                                            0x00000001L
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGADATA_A_MASK                                                           0x00000100L
+//DC_GPIO_DDCVGA_EN
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGACLK_EN__SHIFT                                                        0x0
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGADATA_EN__SHIFT                                                       0x8
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGACLK_EN_MASK                                                          0x00000001L
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGADATA_EN_MASK                                                         0x00000100L
+//DC_GPIO_DDCVGA_Y
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGACLK_Y__SHIFT                                                          0x0
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGADATA_Y__SHIFT                                                         0x8
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGACLK_Y_MASK                                                            0x00000001L
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGADATA_Y_MASK                                                           0x00000100L
+//DC_GPIO_PWRSEQ0_EN
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_OTG_VSYNC_EN__SHIFT                                               0x14
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_OTG_VSYNC_SEL__SHIFT                                              0x15
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_BLON_OTG_VSYNC_EN__SHIFT                                                  0x19
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_BLON_OTG_VSYNC_SEL__SHIFT                                                 0x1a
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_GENERICA_EN__SHIFT                                                0x1d
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_OTG_VSYNC_EN_MASK                                                 0x00100000L
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_OTG_VSYNC_SEL_MASK                                                0x00E00000L
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_BLON_OTG_VSYNC_EN_MASK                                                    0x02000000L
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_BLON_OTG_VSYNC_SEL_MASK                                                   0x1C000000L
+#define DC_GPIO_PWRSEQ0_EN__DC_GPIO_VARY_BL_GENERICA_EN_MASK                                                  0x20000000L
+//DC_GPIO_PAD_STRENGTH_1
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SN__SHIFT                                                      0x0
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SP__SHIFT                                                      0x4
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SN__SHIFT                                                       0x18
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SP__SHIFT                                                       0x1c
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SN_MASK                                                        0x0000000FL
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SP_MASK                                                        0x000000F0L
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SN_MASK                                                         0x0F000000L
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SP_MASK                                                         0xF0000000L
+//PHY_AUX_CNTL
+#define PHY_AUX_CNTL__AUX_PAD_WAKE__SHIFT                                                                     0x9
+#define PHY_AUX_CNTL__AUX1_PAD_RXSEL__SHIFT                                                                   0xa
+#define PHY_AUX_CNTL__AUX2_PAD_RXSEL__SHIFT                                                                   0xc
+#define PHY_AUX_CNTL__AUX3_PAD_RXSEL__SHIFT                                                                   0xe
+#define PHY_AUX_CNTL__AUX4_PAD_RXSEL__SHIFT                                                                   0x10
+#define PHY_AUX_CNTL__AUX5_PAD_RXSEL__SHIFT                                                                   0x12
+#define PHY_AUX_CNTL__AUX6_PAD_RXSEL__SHIFT                                                                   0x14
+#define PHY_AUX_CNTL__AUX_PAD_WAKE_MASK                                                                       0x00000200L
+#define PHY_AUX_CNTL__AUX1_PAD_RXSEL_MASK                                                                     0x00000C00L
+#define PHY_AUX_CNTL__AUX2_PAD_RXSEL_MASK                                                                     0x00003000L
+#define PHY_AUX_CNTL__AUX3_PAD_RXSEL_MASK                                                                     0x0000C000L
+#define PHY_AUX_CNTL__AUX4_PAD_RXSEL_MASK                                                                     0x00030000L
+#define PHY_AUX_CNTL__AUX5_PAD_RXSEL_MASK                                                                     0x000C0000L
+#define PHY_AUX_CNTL__AUX6_PAD_RXSEL_MASK                                                                     0x00300000L
+//DC_GPIO_PWRSEQ1_EN
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_OTG_VSYNC_EN__SHIFT                                               0x14
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_OTG_VSYNC_SEL__SHIFT                                              0x15
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_BLON_OTG_VSYNC_EN__SHIFT                                                  0x19
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_BLON_OTG_VSYNC_SEL__SHIFT                                                 0x1a
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_GENERICA_EN__SHIFT                                                0x1d
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_OTG_VSYNC_EN_MASK                                                 0x00100000L
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_OTG_VSYNC_SEL_MASK                                                0x00E00000L
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_BLON_OTG_VSYNC_EN_MASK                                                    0x02000000L
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_BLON_OTG_VSYNC_SEL_MASK                                                   0x1C000000L
+#define DC_GPIO_PWRSEQ1_EN__DC_GPIO_VARY_BL_GENERICA_EN_MASK                                                  0x20000000L
+//DC_GPIO_AUX_CTRL_0
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_FALLSLEWSEL__SHIFT                                                 0xc
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_SPIKERCEN__SHIFT                                                   0x16
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_SPIKERCSEL__SHIFT                                                  0x1e
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_FALLSLEWSEL_MASK                                                   0x00003000L
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_SPIKERCEN_MASK                                                     0x00C00000L
+#define DC_GPIO_AUX_CTRL_0__DC_GPIO_DDCVGA_SPIKERCSEL_MASK                                                    0xC0000000L
+//DC_GPIO_AUX_CTRL_1
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_CSEL_0P9__SHIFT                                                       0x4
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_CSEL_1P1__SHIFT                                                       0x5
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RSEL_0P9__SHIFT                                                       0x6
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RSEL_1P1__SHIFT                                                       0x7
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RESBIASEN__SHIFT                                                      0xb
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX1_COMPSEL__SHIFT                                                       0xd
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_SPARE__SHIFT                                                       0xe
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_BIASCRTEN__SHIFT                                                      0x10
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_SLEWN__SHIFT                                                       0x12
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_RXSEL__SHIFT                                                       0x14
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX2_COMPSEL__SHIFT                                                       0x19
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX3_COMPSEL__SHIFT                                                       0x1a
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX4_COMPSEL__SHIFT                                                       0x1b
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX5_COMPSEL__SHIFT                                                       0x1c
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX6_COMPSEL__SHIFT                                                       0x1d
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_COMPSEL__SHIFT                                                     0x1e
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_CSEL_0P9_MASK                                                         0x00000010L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_CSEL_1P1_MASK                                                         0x00000020L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RSEL_0P9_MASK                                                         0x00000040L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RSEL_1P1_MASK                                                         0x00000080L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_RESBIASEN_MASK                                                        0x00001800L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX1_COMPSEL_MASK                                                         0x00002000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_SPARE_MASK                                                         0x0000C000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_I2C_BIASCRTEN_MASK                                                        0x00030000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_SLEWN_MASK                                                         0x000C0000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_RXSEL_MASK                                                         0x00300000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX2_COMPSEL_MASK                                                         0x02000000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX3_COMPSEL_MASK                                                         0x04000000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX4_COMPSEL_MASK                                                         0x08000000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX5_COMPSEL_MASK                                                         0x10000000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_AUX6_COMPSEL_MASK                                                         0x20000000L
+#define DC_GPIO_AUX_CTRL_1__DC_GPIO_DDCVGA_COMPSEL_MASK                                                       0xC0000000L
+//DC_GPIO_AUX_CTRL_3
+#define DC_GPIO_AUX_CTRL_3__AUX1_NEN_RTERM__SHIFT                                                             0x0
+#define DC_GPIO_AUX_CTRL_3__AUX2_NEN_RTERM__SHIFT                                                             0x1
+#define DC_GPIO_AUX_CTRL_3__AUX3_NEN_RTERM__SHIFT                                                             0x2
+#define DC_GPIO_AUX_CTRL_3__AUX4_NEN_RTERM__SHIFT                                                             0x3
+#define DC_GPIO_AUX_CTRL_3__AUX5_NEN_RTERM__SHIFT                                                             0x4
+#define DC_GPIO_AUX_CTRL_3__AUX6_NEN_RTERM__SHIFT                                                             0x5
+#define DC_GPIO_AUX_CTRL_3__AUX1_DP_DN_SWAP__SHIFT                                                            0x8
+#define DC_GPIO_AUX_CTRL_3__AUX2_DP_DN_SWAP__SHIFT                                                            0x9
+#define DC_GPIO_AUX_CTRL_3__AUX3_DP_DN_SWAP__SHIFT                                                            0xa
+#define DC_GPIO_AUX_CTRL_3__AUX4_DP_DN_SWAP__SHIFT                                                            0xb
+#define DC_GPIO_AUX_CTRL_3__AUX5_DP_DN_SWAP__SHIFT                                                            0xc
+#define DC_GPIO_AUX_CTRL_3__AUX6_DP_DN_SWAP__SHIFT                                                            0xd
+#define DC_GPIO_AUX_CTRL_3__AUX1_HYS_TUNE__SHIFT                                                              0x10
+#define DC_GPIO_AUX_CTRL_3__AUX2_HYS_TUNE__SHIFT                                                              0x12
+#define DC_GPIO_AUX_CTRL_3__AUX3_HYS_TUNE__SHIFT                                                              0x14
+#define DC_GPIO_AUX_CTRL_3__AUX4_HYS_TUNE__SHIFT                                                              0x16
+#define DC_GPIO_AUX_CTRL_3__AUX5_HYS_TUNE__SHIFT                                                              0x18
+#define DC_GPIO_AUX_CTRL_3__AUX6_HYS_TUNE__SHIFT                                                              0x1a
+#define DC_GPIO_AUX_CTRL_3__AUX1_NEN_RTERM_MASK                                                               0x00000001L
+#define DC_GPIO_AUX_CTRL_3__AUX2_NEN_RTERM_MASK                                                               0x00000002L
+#define DC_GPIO_AUX_CTRL_3__AUX3_NEN_RTERM_MASK                                                               0x00000004L
+#define DC_GPIO_AUX_CTRL_3__AUX4_NEN_RTERM_MASK                                                               0x00000008L
+#define DC_GPIO_AUX_CTRL_3__AUX5_NEN_RTERM_MASK                                                               0x00000010L
+#define DC_GPIO_AUX_CTRL_3__AUX6_NEN_RTERM_MASK                                                               0x00000020L
+#define DC_GPIO_AUX_CTRL_3__AUX1_DP_DN_SWAP_MASK                                                              0x00000100L
+#define DC_GPIO_AUX_CTRL_3__AUX2_DP_DN_SWAP_MASK                                                              0x00000200L
+#define DC_GPIO_AUX_CTRL_3__AUX3_DP_DN_SWAP_MASK                                                              0x00000400L
+#define DC_GPIO_AUX_CTRL_3__AUX4_DP_DN_SWAP_MASK                                                              0x00000800L
+#define DC_GPIO_AUX_CTRL_3__AUX5_DP_DN_SWAP_MASK                                                              0x00001000L
+#define DC_GPIO_AUX_CTRL_3__AUX6_DP_DN_SWAP_MASK                                                              0x00002000L
+#define DC_GPIO_AUX_CTRL_3__AUX1_HYS_TUNE_MASK                                                                0x00030000L
+#define DC_GPIO_AUX_CTRL_3__AUX2_HYS_TUNE_MASK                                                                0x000C0000L
+#define DC_GPIO_AUX_CTRL_3__AUX3_HYS_TUNE_MASK                                                                0x00300000L
+#define DC_GPIO_AUX_CTRL_3__AUX4_HYS_TUNE_MASK                                                                0x00C00000L
+#define DC_GPIO_AUX_CTRL_3__AUX5_HYS_TUNE_MASK                                                                0x03000000L
+#define DC_GPIO_AUX_CTRL_3__AUX6_HYS_TUNE_MASK                                                                0x0C000000L
+//DC_GPIO_AUX_CTRL_4
+#define DC_GPIO_AUX_CTRL_4__AUX1_AUX_CTRL__SHIFT                                                              0x0
+#define DC_GPIO_AUX_CTRL_4__AUX2_AUX_CTRL__SHIFT                                                              0x4
+#define DC_GPIO_AUX_CTRL_4__AUX3_AUX_CTRL__SHIFT                                                              0x8
+#define DC_GPIO_AUX_CTRL_4__AUX4_AUX_CTRL__SHIFT                                                              0xc
+#define DC_GPIO_AUX_CTRL_4__AUX5_AUX_CTRL__SHIFT                                                              0x10
+#define DC_GPIO_AUX_CTRL_4__AUX6_AUX_CTRL__SHIFT                                                              0x14
+#define DC_GPIO_AUX_CTRL_4__AUX1_AUX_CTRL_MASK                                                                0x0000000FL
+#define DC_GPIO_AUX_CTRL_4__AUX2_AUX_CTRL_MASK                                                                0x000000F0L
+#define DC_GPIO_AUX_CTRL_4__AUX3_AUX_CTRL_MASK                                                                0x00000F00L
+#define DC_GPIO_AUX_CTRL_4__AUX4_AUX_CTRL_MASK                                                                0x0000F000L
+#define DC_GPIO_AUX_CTRL_4__AUX5_AUX_CTRL_MASK                                                                0x000F0000L
+#define DC_GPIO_AUX_CTRL_4__AUX6_AUX_CTRL_MASK                                                                0x00F00000L
+//DC_GPIO_AUX_CTRL_5
+#define DC_GPIO_AUX_CTRL_5__AUX1_VOD_TUNE__SHIFT                                                              0x0
+#define DC_GPIO_AUX_CTRL_5__AUX2_VOD_TUNE__SHIFT                                                              0x2
+#define DC_GPIO_AUX_CTRL_5__AUX3_VOD_TUNE__SHIFT                                                              0x4
+#define DC_GPIO_AUX_CTRL_5__AUX4_VOD_TUNE__SHIFT                                                              0x6
+#define DC_GPIO_AUX_CTRL_5__AUX5_VOD_TUNE__SHIFT                                                              0x8
+#define DC_GPIO_AUX_CTRL_5__AUX6_VOD_TUNE__SHIFT                                                              0xa
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD1_I2CMODE__SHIFT                                                           0xc
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD2_I2CMODE__SHIFT                                                           0xd
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD3_I2CMODE__SHIFT                                                           0xe
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD4_I2CMODE__SHIFT                                                           0xf
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD5_I2CMODE__SHIFT                                                           0x10
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD6_I2CMODE__SHIFT                                                           0x11
+#define DC_GPIO_AUX_CTRL_5__DDC1_I2C_VPH_1V2_EN__SHIFT                                                        0x12
+#define DC_GPIO_AUX_CTRL_5__DDC2_I2C_VPH_1V2_EN__SHIFT                                                        0x13
+#define DC_GPIO_AUX_CTRL_5__DDC3_I2C_VPH_1V2_EN__SHIFT                                                        0x14
+#define DC_GPIO_AUX_CTRL_5__DDC4_I2C_VPH_1V2_EN__SHIFT                                                        0x15
+#define DC_GPIO_AUX_CTRL_5__DDC5_I2C_VPH_1V2_EN__SHIFT                                                        0x16
+#define DC_GPIO_AUX_CTRL_5__DDC6_I2C_VPH_1V2_EN__SHIFT                                                        0x17
+#define DC_GPIO_AUX_CTRL_5__DDC1_PAD_I2C_CTRL__SHIFT                                                          0x18
+#define DC_GPIO_AUX_CTRL_5__DDC2_PAD_I2C_CTRL__SHIFT                                                          0x19
+#define DC_GPIO_AUX_CTRL_5__DDC3_PAD_I2C_CTRL__SHIFT                                                          0x1a
+#define DC_GPIO_AUX_CTRL_5__DDC4_PAD_I2C_CTRL__SHIFT                                                          0x1b
+#define DC_GPIO_AUX_CTRL_5__DDC5_PAD_I2C_CTRL__SHIFT                                                          0x1c
+#define DC_GPIO_AUX_CTRL_5__DDC6_PAD_I2C_CTRL__SHIFT                                                          0x1d
+#define DC_GPIO_AUX_CTRL_5__AUX1_VOD_TUNE_MASK                                                                0x00000003L
+#define DC_GPIO_AUX_CTRL_5__AUX2_VOD_TUNE_MASK                                                                0x0000000CL
+#define DC_GPIO_AUX_CTRL_5__AUX3_VOD_TUNE_MASK                                                                0x00000030L
+#define DC_GPIO_AUX_CTRL_5__AUX4_VOD_TUNE_MASK                                                                0x000000C0L
+#define DC_GPIO_AUX_CTRL_5__AUX5_VOD_TUNE_MASK                                                                0x00000300L
+#define DC_GPIO_AUX_CTRL_5__AUX6_VOD_TUNE_MASK                                                                0x00000C00L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD1_I2CMODE_MASK                                                             0x00001000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD2_I2CMODE_MASK                                                             0x00002000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD3_I2CMODE_MASK                                                             0x00004000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD4_I2CMODE_MASK                                                             0x00008000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD5_I2CMODE_MASK                                                             0x00010000L
+#define DC_GPIO_AUX_CTRL_5__DDC_PAD6_I2CMODE_MASK                                                             0x00020000L
+#define DC_GPIO_AUX_CTRL_5__DDC1_I2C_VPH_1V2_EN_MASK                                                          0x00040000L
+#define DC_GPIO_AUX_CTRL_5__DDC2_I2C_VPH_1V2_EN_MASK                                                          0x00080000L
+#define DC_GPIO_AUX_CTRL_5__DDC3_I2C_VPH_1V2_EN_MASK                                                          0x00100000L
+#define DC_GPIO_AUX_CTRL_5__DDC4_I2C_VPH_1V2_EN_MASK                                                          0x00200000L
+#define DC_GPIO_AUX_CTRL_5__DDC5_I2C_VPH_1V2_EN_MASK                                                          0x00400000L
+#define DC_GPIO_AUX_CTRL_5__DDC6_I2C_VPH_1V2_EN_MASK                                                          0x00800000L
+#define DC_GPIO_AUX_CTRL_5__DDC1_PAD_I2C_CTRL_MASK                                                            0x01000000L
+#define DC_GPIO_AUX_CTRL_5__DDC2_PAD_I2C_CTRL_MASK                                                            0x02000000L
+#define DC_GPIO_AUX_CTRL_5__DDC3_PAD_I2C_CTRL_MASK                                                            0x04000000L
+#define DC_GPIO_AUX_CTRL_5__DDC4_PAD_I2C_CTRL_MASK                                                            0x08000000L
+#define DC_GPIO_AUX_CTRL_5__DDC5_PAD_I2C_CTRL_MASK                                                            0x10000000L
+#define DC_GPIO_AUX_CTRL_5__DDC6_PAD_I2C_CTRL_MASK                                                            0x20000000L
+//AUXI2C_PAD_ALL_PWR_OK
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY1_ALL_PWR_OK__SHIFT                                                  0x0
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY2_ALL_PWR_OK__SHIFT                                                  0x1
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY3_ALL_PWR_OK__SHIFT                                                  0x2
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY4_ALL_PWR_OK__SHIFT                                                  0x3
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY5_ALL_PWR_OK__SHIFT                                                  0x4
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY6_ALL_PWR_OK__SHIFT                                                  0x5
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY1_ALL_PWR_OK_MASK                                                    0x00000001L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY2_ALL_PWR_OK_MASK                                                    0x00000002L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY3_ALL_PWR_OK_MASK                                                    0x00000004L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY4_ALL_PWR_OK_MASK                                                    0x00000008L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY5_ALL_PWR_OK_MASK                                                    0x00000010L
+#define AUXI2C_PAD_ALL_PWR_OK__AUXI2C_PHY6_ALL_PWR_OK_MASK                                                    0x00000020L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_offset.h
new file mode 100644
index 0000000000000..fa4e42a3ae9fc
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_offset.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _lsdma_7_1_0_OFFSET_HEADER
+#define _lsdma_7_1_0_OFFSET_HEADER
+
+#define regLSDMA_PIO_SRC_ADDR_LO                                                                        0x0080
+#define regLSDMA_PIO_SRC_ADDR_LO_BASE_IDX                                                               0
+#define regLSDMA_PIO_SRC_ADDR_HI                                                                        0x0081
+#define regLSDMA_PIO_SRC_ADDR_HI_BASE_IDX                                                               0
+#define regLSDMA_PIO_DST_ADDR_LO                                                                        0x0082
+#define regLSDMA_PIO_DST_ADDR_LO_BASE_IDX                                                               0
+#define regLSDMA_PIO_DST_ADDR_HI                                                                        0x0083
+#define regLSDMA_PIO_DST_ADDR_HI_BASE_IDX                                                               0
+#define regLSDMA_PIO_COMMAND                                                                            0x0084
+#define regLSDMA_PIO_COMMAND_BASE_IDX                                                                   0
+#define regLSDMA_PIO_CONSTFILL_DATA                                                                     0x0085
+#define regLSDMA_PIO_CONSTFILL_DATA_BASE_IDX                                                            0
+#define regLSDMA_PIO_CONTROL                                                                            0x0086
+#define regLSDMA_PIO_CONTROL_BASE_IDX                                                                   0
+
+#define regLSDMA_PIO_STATUS                                                                             0x008a
+#define regLSDMA_PIO_STATUS_BASE_IDX                                                                    0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_sh_mask.h
new file mode 100644
index 0000000000000..cf83dacf4acf5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/lsdma/lsdma_7_1_0_sh_mask.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _lsdma_7_1_0_SH_MASK_HEADER
+#define _lsdma_7_1_0_SH_MASK_HEADER
+
+
+// addressBlock: lsdma0_lsdma0dec
+//LSDMA_PIO_STATUS
+#define LSDMA_PIO_STATUS__CMD_IN_FIFO__SHIFT                                                                  0x0
+#define LSDMA_PIO_STATUS__CMD_PROCESSING__SHIFT                                                               0x3
+#define LSDMA_PIO_STATUS__ERROR_INVALID_ADDR__SHIFT                                                           0xb
+#define LSDMA_PIO_STATUS__ERROR_ZERO_COUNT__SHIFT                                                             0xc
+#define LSDMA_PIO_STATUS__ERROR_DRAM_ECC__SHIFT                                                               0xd
+#define LSDMA_PIO_STATUS__ERROR_SRAM_ECC__SHIFT                                                               0xe
+#define LSDMA_PIO_STATUS__ERROR_WRRET_NACK_GEN_ERR__SHIFT                                                     0xf
+#define LSDMA_PIO_STATUS__ERROR_RDRET_NACK_GEN_ERR__SHIFT                                                     0x10
+#define LSDMA_PIO_STATUS__ERROR_WRRET_NACK_PRT__SHIFT                                                         0x11
+#define LSDMA_PIO_STATUS__ERROR_RDRET_NACK_PRT__SHIFT                                                         0x12
+#define LSDMA_PIO_STATUS__ERROR_REQ_DROP__SHIFT                                                               0x13
+#define LSDMA_PIO_STATUS__PIO_FIFO_EMPTY__SHIFT                                                               0x1c
+#define LSDMA_PIO_STATUS__PIO_FIFO_FULL__SHIFT                                                                0x1d
+#define LSDMA_PIO_STATUS__PIO_IDLE__SHIFT                                                                     0x1f
+#define LSDMA_PIO_STATUS__CMD_IN_FIFO_MASK                                                                    0x00000007L
+#define LSDMA_PIO_STATUS__CMD_PROCESSING_MASK                                                                 0x000003F8L
+#define LSDMA_PIO_STATUS__ERROR_INVALID_ADDR_MASK                                                             0x00000800L
+#define LSDMA_PIO_STATUS__ERROR_ZERO_COUNT_MASK                                                               0x00001000L
+#define LSDMA_PIO_STATUS__ERROR_DRAM_ECC_MASK                                                                 0x00002000L
+#define LSDMA_PIO_STATUS__ERROR_SRAM_ECC_MASK                                                                 0x00004000L
+#define LSDMA_PIO_STATUS__ERROR_WRRET_NACK_GEN_ERR_MASK                                                       0x00008000L
+#define LSDMA_PIO_STATUS__ERROR_RDRET_NACK_GEN_ERR_MASK                                                       0x00010000L
+#define LSDMA_PIO_STATUS__ERROR_WRRET_NACK_PRT_MASK                                                           0x00020000L
+#define LSDMA_PIO_STATUS__ERROR_RDRET_NACK_PRT_MASK                                                           0x00040000L
+#define LSDMA_PIO_STATUS__ERROR_REQ_DROP_MASK                                                                 0x00080000L
+#define LSDMA_PIO_STATUS__PIO_FIFO_EMPTY_MASK                                                                 0x10000000L
+#define LSDMA_PIO_STATUS__PIO_FIFO_FULL_MASK                                                                  0x20000000L
+#define LSDMA_PIO_STATUS__PIO_IDLE_MASK                                                                       0x80000000L
+//LSDMA_PIO_SRC_ADDR_LO
+#define LSDMA_PIO_SRC_ADDR_LO__SRC_ADDR_LO__SHIFT                                                             0x0
+#define LSDMA_PIO_SRC_ADDR_LO__SRC_ADDR_LO_MASK                                                               0xFFFFFFFFL
+//LSDMA_PIO_SRC_ADDR_HI
+#define LSDMA_PIO_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                             0x0
+#define LSDMA_PIO_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                               0xFFFFFFFFL
+//LSDMA_PIO_DST_ADDR_LO
+#define LSDMA_PIO_DST_ADDR_LO__DST_ADDR_LO__SHIFT                                                             0x0
+#define LSDMA_PIO_DST_ADDR_LO__DST_ADDR_LO_MASK                                                               0xFFFFFFFFL
+//LSDMA_PIO_DST_ADDR_HI
+#define LSDMA_PIO_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                             0x0
+#define LSDMA_PIO_DST_ADDR_HI__DST_ADDR_HI_MASK                                                               0xFFFFFFFFL
+//LSDMA_PIO_CONTROL
+#define LSDMA_PIO_CONTROL__VMID__SHIFT                                                                        0x0
+#define LSDMA_PIO_CONTROL__DST_GPA__SHIFT                                                                     0x4
+#define LSDMA_PIO_CONTROL__DST_SYS__SHIFT                                                                     0x5
+#define LSDMA_PIO_CONTROL__DST_GCC__SHIFT                                                                     0x6
+#define LSDMA_PIO_CONTROL__DST_SNOOP__SHIFT                                                                   0x7
+#define LSDMA_PIO_CONTROL__DST_REUSE_HINT__SHIFT                                                              0x8
+#define LSDMA_PIO_CONTROL__DST_COMP_EN__SHIFT                                                                 0xa
+#define LSDMA_PIO_CONTROL__SRC_GPA__SHIFT                                                                     0x14
+#define LSDMA_PIO_CONTROL__SRC_SYS__SHIFT                                                                     0x15
+#define LSDMA_PIO_CONTROL__SRC_SNOOP__SHIFT                                                                   0x17
+#define LSDMA_PIO_CONTROL__SRC_REUSE_HINT__SHIFT                                                              0x18
+#define LSDMA_PIO_CONTROL__SRC_COMP_EN__SHIFT                                                                 0x1a
+#define LSDMA_PIO_CONTROL__VMID_MASK                                                                          0x0000000FL
+#define LSDMA_PIO_CONTROL__DST_GPA_MASK                                                                       0x00000010L
+#define LSDMA_PIO_CONTROL__DST_SYS_MASK                                                                       0x00000020L
+#define LSDMA_PIO_CONTROL__DST_GCC_MASK                                                                       0x00000040L
+#define LSDMA_PIO_CONTROL__DST_SNOOP_MASK                                                                     0x00000080L
+#define LSDMA_PIO_CONTROL__DST_REUSE_HINT_MASK                                                                0x00000300L
+#define LSDMA_PIO_CONTROL__DST_COMP_EN_MASK                                                                   0x00000400L
+#define LSDMA_PIO_CONTROL__SRC_GPA_MASK                                                                       0x00100000L
+#define LSDMA_PIO_CONTROL__SRC_SYS_MASK                                                                       0x00200000L
+#define LSDMA_PIO_CONTROL__SRC_SNOOP_MASK                                                                     0x00800000L
+#define LSDMA_PIO_CONTROL__SRC_REUSE_HINT_MASK                                                                0x03000000L
+#define LSDMA_PIO_CONTROL__SRC_COMP_EN_MASK                                                                   0x04000000L
+//LSDMA_PIO_COMMAND
+#define LSDMA_PIO_COMMAND__COUNT__SHIFT                                                                       0x0
+#define LSDMA_PIO_COMMAND__RAW_WAIT__SHIFT                                                                    0x1e
+#define LSDMA_PIO_COMMAND__CONSTANT_FILL__SHIFT                                                               0x1f
+#define LSDMA_PIO_COMMAND__COUNT_MASK                                                                         0x03FFFFFFL
+#define LSDMA_PIO_COMMAND__RAW_WAIT_MASK                                                                      0x40000000L
+#define LSDMA_PIO_COMMAND__CONSTANT_FILL_MASK                                                                 0x80000000L
+//LSDMA_PIO_CONSTFILL_DATA
+#define LSDMA_PIO_CONSTFILL_DATA__DATA__SHIFT                                                                 0x0
+#define LSDMA_PIO_CONSTFILL_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index 3d083010e734a..62f7ed0b20667 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -879,6 +879,7 @@ enum atom_connector_caps_def
 {
   ATOM_CONNECTOR_CAP_INTERNAL_DISPLAY         = 0x01,        //a cap bit to indicate that this non-embedded display connector is an internal display
   ATOM_CONNECTOR_CAP_INTERNAL_DISPLAY_BL      = 0x02,        //a cap bit to indicate that this internal display requires BL control from GPU, refers to lcd_info for BL PWM freq 
+  ATOM_CONNECTOR_CAP_DP_PLUS_PLUS_TYPE2_ONLY  = 0x10,        //a cap bit that indicates connector with DP_PLUS_PLUS_TYPE2_ONLY
 };
 
 struct atom_disp_connector_caps_record
@@ -1694,7 +1695,9 @@ struct atom_integrated_system_info_v2_3 {
   uint8_t gpu_package_id;
   struct  edp_info_table  edp1_info;
   struct  edp_info_table  edp2_info;
-  uint32_t  reserved2[8];
+  uint32_t cpuid;
+  uint32_t vram_bit_width;
+  uint32_t reserved2[6];
   struct  atom_external_display_connection_info extdispconninfo;
   uint8_t UMACarveoutVersion;
   uint8_t UMACarveoutIndexMax;
@@ -1769,6 +1772,7 @@ enum atom_dmi_t17_mem_type_def{
   Hbm2MemType,                                          ///< Assign 33 to HBM2
   Ddr5MemType,                                          ///< Assign 34 to DDR5
   LpDdr5MemType,                                        ///< Assign 35 to LPDDR5
+  LpDdr5xMemType,                                       ///< Assign 36 to LPDDR5x
 };
 
 
diff --git a/drivers/gpu/drm/amd/include/discovery.h b/drivers/gpu/drm/amd/include/discovery.h
index 710e328fad48f..76c9f951bc1c8 100644
--- a/drivers/gpu/drm/amd/include/discovery.h
+++ b/drivers/gpu/drm/amd/include/discovery.h
@@ -64,6 +64,19 @@ typedef struct binary_header
 	table_info table_list[TOTAL_TABLES];
 } binary_header;
 
+typedef struct binary_header_v2
+{
+	/* psp structure should go at the top of this structure */
+	uint32_t binary_signature; /* 0x7, 0x14, 0x21, 0x28 */
+	uint16_t version_major;     /* 0x02 */
+	uint16_t version_minor;
+	uint16_t binary_checksum;  /* Byte sum of the binary after this field */
+	uint16_t binary_size;      /* Binary Size*/
+	uint16_t num_tables;
+	uint16_t padding;
+	table_info table_list[] __counted_by(num_tables);
+} binary_header_v2;
+
 typedef struct die_info
 {
 	uint16_t die_id;
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 6683ffd6aa681..a9b73f4fd4661 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -584,6 +584,9 @@ enum amdgpu_metrics_attr_id {
 	AMDGPU_METRICS_ATTR_ID_GFX_BELOW_HOST_LIMIT_THM_ACC,
 	AMDGPU_METRICS_ATTR_ID_GFX_LOW_UTILIZATION_ACC,
 	AMDGPU_METRICS_ATTR_ID_GFX_BELOW_HOST_LIMIT_TOTAL_ACC,
+	AMDGPU_METRICS_ATTR_ID_TEMPERATURE_HBM,
+	AMDGPU_METRICS_ATTR_ID_TEMPERATURE_AID,
+	AMDGPU_METRICS_ATTR_ID_TEMPERATURE_XCD,
 	AMDGPU_METRICS_ATTR_ID_MAX,
 };
 
@@ -1832,6 +1835,8 @@ struct amdgpu_partition_metrics_v1_1 {
 enum amdgpu_xgmi_link_status {
 	AMDGPU_XGMI_LINK_INACTIVE = 0,
 	AMDGPU_XGMI_LINK_ACTIVE = 1,
+	/* Status not available */
+	AMDGPU_XGMI_LINK_NA = 2,
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index eca93a9d0b847..9e2f69663f960 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -1910,8 +1910,6 @@ static int ss_bias_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 static int pp_od_clk_voltage_attr_update(struct amdgpu_device *adev, struct amdgpu_device_attr *attr,
 					 uint32_t mask, enum amdgpu_device_attr_states *states)
 {
-	uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
-
 	*states = ATTR_STATE_SUPPORTED;
 
 	if (!amdgpu_dpm_is_overdrive_supported(adev)) {
@@ -1919,10 +1917,8 @@ static int pp_od_clk_voltage_attr_update(struct amdgpu_device *adev, struct amdg
 		return 0;
 	}
 
-	/* Enable pp_od_clk_voltage node for gc 9.4.3, 9.4.4, 9.5.0 SRIOV/BM support */
-	if (gc_ver == IP_VERSION(9, 4, 3) ||
-	    gc_ver == IP_VERSION(9, 4, 4) ||
-	    gc_ver == IP_VERSION(9, 5, 0)) {
+	/* Enable pp_od_clk_voltage node for gc 9.4.3, 9.4.4, 9.5.0, 12.1.0 SRIOV/BM support */
+	if (amdgpu_is_multi_aid(adev)) {
 		if (amdgpu_sriov_multi_vf_mode(adev))
 			*states = ATTR_STATE_UNSUPPORTED;
 		return 0;
@@ -2000,9 +1996,7 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
 		      gc_ver == IP_VERSION(11, 5, 0) ||
 		      gc_ver == IP_VERSION(11, 0, 2) ||
 		      gc_ver == IP_VERSION(11, 0, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 4) ||
-		      gc_ver == IP_VERSION(9, 5, 0)))
+		      amdgpu_is_multi_aid(adev)))
 			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_dpm_vclk1)) {
 		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
@@ -2023,9 +2017,7 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
 		      gc_ver == IP_VERSION(11, 5, 0) ||
 		      gc_ver == IP_VERSION(11, 0, 2) ||
 		      gc_ver == IP_VERSION(11, 0, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 4) ||
-		      gc_ver == IP_VERSION(9, 5, 0)))
+		      amdgpu_is_multi_aid(adev)))
 			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_dpm_dclk1)) {
 		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
@@ -2035,9 +2027,7 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
 			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_dpm_pcie)) {
 		if (gc_ver == IP_VERSION(9, 4, 2) ||
-		    gc_ver == IP_VERSION(9, 4, 3) ||
-		    gc_ver == IP_VERSION(9, 4, 4) ||
-		    gc_ver == IP_VERSION(9, 5, 0))
+		    amdgpu_is_multi_aid(adev))
 			*states = ATTR_STATE_UNSUPPORTED;
 	}
 
@@ -2651,6 +2641,7 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 		case IP_VERSION(11, 0, 3):
 		case IP_VERSION(12, 0, 0):
 		case IP_VERSION(12, 0, 1):
+		case IP_VERSION(12, 1, 0):
 			*states = ATTR_STATE_SUPPORTED;
 			break;
 		default:
@@ -3732,8 +3723,7 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 
 	/* Skip crit temp on APU */
 	if ((((adev->flags & AMD_IS_APU) && (adev->family >= AMDGPU_FAMILY_CZ)) ||
-	    (gc_ver == IP_VERSION(9, 4, 3) || gc_ver == IP_VERSION(9, 4, 4) ||
-	     gc_ver == IP_VERSION(9, 5, 0))) &&
+	     amdgpu_is_multi_aid(adev)) &&
 	    (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr ||
 	     attr == &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr))
 		return 0;
@@ -3815,18 +3805,14 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 
 	if ((adev->family == AMDGPU_FAMILY_SI ||	/* not implemented yet */
 	     adev->family == AMDGPU_FAMILY_KV ||	/* not implemented yet */
-	     (gc_ver == IP_VERSION(9, 4, 3) ||
-	      gc_ver == IP_VERSION(9, 4, 4) ||
-	      gc_ver == IP_VERSION(9, 5, 0))) &&
+	     amdgpu_is_multi_aid(adev)) &&
 	    (attr == &sensor_dev_attr_in0_input.dev_attr.attr ||
 	     attr == &sensor_dev_attr_in0_label.dev_attr.attr))
 		return 0;
 
 	/* only APUs other than gc 9,4,3 have vddnb */
 	if ((!(adev->flags & AMD_IS_APU) ||
-	     (gc_ver == IP_VERSION(9, 4, 3) ||
-	      gc_ver == IP_VERSION(9, 4, 4) ||
-	      gc_ver == IP_VERSION(9, 5, 0))) &&
+	     amdgpu_is_multi_aid(adev)) &&
 	    (attr == &sensor_dev_attr_in1_input.dev_attr.attr ||
 	     attr == &sensor_dev_attr_in1_label.dev_attr.attr))
 		return 0;
@@ -3855,9 +3841,7 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 		return 0;
 
 	/* hotspot temperature for gc 9,4,3*/
-	if (gc_ver == IP_VERSION(9, 4, 3) ||
-	    gc_ver == IP_VERSION(9, 4, 4) ||
-	    gc_ver == IP_VERSION(9, 5, 0)) {
+	if (amdgpu_is_multi_aid(adev)) {
 		if (attr == &sensor_dev_attr_temp1_input.dev_attr.attr ||
 		    attr == &sensor_dev_attr_temp1_emergency.dev_attr.attr ||
 		    attr == &sensor_dev_attr_temp1_label.dev_attr.attr)
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
index 61b1c5aa74cb5..36942467d4add 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
@@ -3454,9 +3454,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
 	if (adev->asic_type == CHIP_HAINAN) {
 		if ((adev->pdev->revision == 0x81) ||
 		    (adev->pdev->revision == 0xC3) ||
+		    (adev->pdev->device == 0x6660) ||
 		    (adev->pdev->device == 0x6664) ||
 		    (adev->pdev->device == 0x6665) ||
-		    (adev->pdev->device == 0x6667)) {
+		    (adev->pdev->device == 0x6667) ||
+		    (adev->pdev->device == 0x666F)) {
 			max_sclk = 75000;
 		}
 		if ((adev->pdev->revision == 0xC3) ||
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_smc.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_smc.c
index 281a5e377aee4..e1c509bfc3902 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_smc.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_smc.c
@@ -65,7 +65,7 @@ int amdgpu_si_copy_bytes_to_smc(struct amdgpu_device *adev,
 
 	addr = smc_start_address;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	while (byte_count >= 4) {
 		/* SMC address space is BE */
 		data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
@@ -109,7 +109,7 @@ int amdgpu_si_copy_bytes_to_smc(struct amdgpu_device *adev,
 	}
 
 done:
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return ret;
 }
@@ -252,7 +252,7 @@ int amdgpu_si_load_smc_ucode(struct amdgpu_device *adev, u32 limit)
 	if (ucode_size & 3)
 		return -EINVAL;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	WREG32(mmSMC_IND_INDEX_0, ucode_start_address);
 	WREG32_P(mmSMC_IND_ACCESS_CNTL, SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK, ~SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK);
 	while (ucode_size >= 4) {
@@ -265,7 +265,7 @@ int amdgpu_si_load_smc_ucode(struct amdgpu_device *adev, u32 limit)
 		ucode_size -= 4;
 	}
 	WREG32_P(mmSMC_IND_ACCESS_CNTL, 0, ~SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return 0;
 }
@@ -276,11 +276,11 @@ int amdgpu_si_read_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
 	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	ret = si_set_smc_sram_address(adev, smc_address, limit);
 	if (ret == 0)
 		*value = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return ret;
 }
@@ -291,11 +291,11 @@ int amdgpu_si_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
 	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	spin_lock_irqsave(&adev->reg.smc.lock, flags);
 	ret = si_set_smc_sram_address(adev, smc_address, limit);
 	if (ret == 0)
 		WREG32(mmSMC_IND_DATA_0, value);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index b05c8bbdf2f31..26b43a6276d0d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -628,6 +628,18 @@ int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type ms
 	return ret;
 }
 
+int amdgpu_smu_ras_feature_is_enabled(struct amdgpu_device *adev,
+						enum smu_feature_mask mask)
+{
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	int ret = 0;
+
+	if (smu->ppt_funcs && smu->ppt_funcs->feature_is_enabled)
+		ret = smu->ppt_funcs->feature_is_enabled(smu, mask);
+
+	return ret;
+}
+
 static int smu_sys_get_pp_table(void *handle,
 				char **table)
 {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index a6303d093c50e..609f5ab07d8a8 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -389,6 +389,7 @@ struct smu_table_context {
 	void				*metrics_table;
 	void				*clocks_table;
 	void				*watermarks_table;
+	struct mutex			metrics_lock;
 
 	void				*max_sustainable_clocks;
 	struct smu_bios_boot_up_values	boot_values;
@@ -1997,6 +1998,8 @@ const struct ras_smu_drv *smu_get_ras_smu_driver(void *handle);
 
 int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type msg,
 			    uint32_t param, uint32_t *readarg);
+int amdgpu_smu_ras_feature_is_enabled(struct amdgpu_device *adev,
+						enum smu_feature_mask mask);
 #endif
 
 void smu_feature_cap_set(struct smu_context *smu, enum smu_feature_cap_id fea_id);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index b0d6b7b0946d8..89bbda0670ef1 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -132,8 +132,6 @@ int smu_v13_0_setup_pptable(struct smu_context *smu);
 
 int smu_v13_0_get_vbios_bootup_values(struct smu_context *smu);
 
-int smu_v13_0_check_fw_version(struct smu_context *smu);
-
 int smu_v13_0_set_driver_table_location(struct smu_context *smu);
 
 int smu_v13_0_set_tool_table_location(struct smu_context *smu);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
index ab4a64f54e797..09743ccd0d15c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
@@ -142,8 +142,6 @@ int smu_v15_0_setup_pptable(struct smu_context *smu);
 
 int smu_v15_0_get_vbios_bootup_values(struct smu_context *smu);
 
-int smu_v15_0_check_fw_version(struct smu_context *smu);
-
 int smu_v15_0_set_driver_table_location(struct smu_context *smu);
 
 int smu_v15_0_set_tool_table_location(struct smu_context *smu);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
index 2b4faab376930..dc056f1e4b64f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
@@ -1988,7 +1988,7 @@ static const struct pptable_funcs aldebaran_ppt_funcs = {
 	/* pptable related */
 	.setup_pptable = aldebaran_setup_pptable,
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.write_pptable = smu_cmn_write_pptable,
 	.set_driver_table_location = smu_v13_0_set_driver_table_location,
 	.set_tool_table_location = smu_v13_0_set_tool_table_location,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index 447a9c26bb77d..a3b755c61b1f5 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -258,48 +258,6 @@ int smu_v13_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-int smu_v13_0_check_fw_version(struct smu_context *smu)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t if_version = 0xff, smu_version = 0xff;
-	uint8_t smu_program, smu_major, smu_minor, smu_debug;
-	int ret = 0;
-
-	ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version);
-	if (ret)
-		return ret;
-
-	smu_program = (smu_version >> 24) & 0xff;
-	smu_major = (smu_version >> 16) & 0xff;
-	smu_minor = (smu_version >> 8) & 0xff;
-	smu_debug = (smu_version >> 0) & 0xff;
-	adev->pm.fw_version = smu_version;
-
-	/* only for dGPU w/ SMU13*/
-	if (adev->pm.fw)
-		dev_dbg(smu->adev->dev, "smu fw reported program %d, version = 0x%08x (%d.%d.%d)\n",
-			 smu_program, smu_version, smu_major, smu_minor, smu_debug);
-
-	/*
-	 * 1. if_version mismatch is not critical as our fw is designed
-	 * to be backward compatible.
-	 * 2. New fw usually brings some optimizations. But that's visible
-	 * only on the paired driver.
-	 * Considering above, we just leave user a verbal message instead
-	 * of halt driver loading.
-	 */
-	if (smu->smc_driver_if_version != SMU_IGNORE_IF_VERSION &&
-	    if_version != smu->smc_driver_if_version) {
-		dev_info(adev->dev, "smu driver if version = 0x%08x, smu fw if version = 0x%08x, "
-			 "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n",
-			 smu->smc_driver_if_version, if_version,
-			 smu_program, smu_version, smu_major, smu_minor, smu_debug);
-		dev_info(adev->dev, "SMU driver if version not matched\n");
-	}
-
-	return ret;
-}
-
 static int smu_v13_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
 {
 	struct amdgpu_device *adev = smu->adev;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index a8d63d4d1f6e2..59720372fc17a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -3133,7 +3133,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = {
 	.fini_power = smu_v13_0_fini_power,
 	.check_fw_status = smu_v13_0_check_fw_status,
 	.setup_pptable = smu_v13_0_0_setup_pptable,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.write_pptable = smu_cmn_write_pptable,
 	.set_driver_table_location = smu_v13_0_set_driver_table_location,
 	.system_features_control = smu_v13_0_0_system_features_control,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
index 32d5e2170d80a..54a86eb77cd52 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
@@ -49,6 +49,13 @@
 #undef pr_info
 #undef pr_debug
 
+#define hbm_stack_mask_valid(umc_mask) \
+	(((umc_mask) & 0x3) == 0x3)
+
+#define for_each_hbm_stack(stack_idx, umc_mask) \
+	for ((stack_idx) = 0; (umc_mask); \
+	     (umc_mask) >>= 2, (stack_idx)++) \
+
 #define SMU_13_0_12_FEA_MAP(smu_feature, smu_13_0_12_feature)                    \
 	[smu_feature] = { 1, (smu_13_0_12_feature) }
 
@@ -262,8 +269,9 @@ static void smu_v13_0_12_init_xgmi_data(struct smu_context *smu,
 	int ret;
 
 	if (smu_table->tables[SMU_TABLE_SMU_METRICS].version >= 0x13) {
-		max_width = (uint8_t)static_metrics->MaxXgmiWidth;
-		max_speed = (uint16_t)static_metrics->MaxXgmiBitrate;
+		max_width = (uint8_t)SMUQ10_ROUND(static_metrics->MaxXgmiWidth);
+		max_speed =
+			(uint16_t)SMUQ10_ROUND(static_metrics->MaxXgmiBitrate);
 		ret = 0;
 	} else {
 		MetricsTable_t *metrics = (MetricsTable_t *)smu_table->metrics_table;
@@ -834,7 +842,7 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table,
 				  struct smu_v13_0_6_gpu_metrics *gpu_metrics)
 {
 	struct amdgpu_device *adev = smu->adev;
-	int ret = 0, xcc_id, inst, i, j;
+	int ret = 0, xcc_id, inst, i, j, idx;
 	u8 num_jpeg_rings_gpu_metrics;
 	MetricsTable_t *metrics;
 
@@ -849,6 +857,31 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table,
 	gpu_metrics->temperature_vrsoc =
 		SMUQ10_ROUND(metrics->MaxVrTemperature);
 
+	if (smu_v13_0_6_cap_supported(smu,
+				      SMU_CAP(TEMP_AID_XCD_HBM))) {
+		if (adev->umc.active_mask) {
+			u64 mask = adev->umc.active_mask;
+			int out_idx = 0;
+			int stack_idx;
+
+			if (unlikely(hweight64(mask) / 2 > SMU_13_0_6_MAX_HBM_STACKS)) {
+				dev_warn(adev->dev, "Invalid umc mask %lld\n", mask);
+			} else  {
+				for_each_hbm_stack(stack_idx, mask) {
+					if (!hbm_stack_mask_valid(mask))
+						continue;
+					gpu_metrics->temperature_hbm[out_idx++] =
+						metrics->HbmTemperature[stack_idx];
+				}
+			}
+		}
+		idx = 0;
+		for_each_inst(i, adev->aid_mask) {
+			gpu_metrics->temperature_aid[idx] = metrics->AidTemperature[i];
+			idx++;
+		}
+	}
+
 	gpu_metrics->average_gfx_activity =
 		SMUQ10_ROUND(metrics->SocketGfxBusy);
 	gpu_metrics->average_umc_activity =
@@ -964,6 +997,9 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table,
 				[i] = SMUQ10_ROUND(
 				metrics->GfxclkBelowHostLimitTotalAcc[inst]);
 		}
+		if (smu_v13_0_6_cap_supported(smu,
+					      SMU_CAP(TEMP_AID_XCD_HBM)))
+			gpu_metrics->temperature_xcd[i] = metrics->XcdTemperature[inst];
 	}
 
 	gpu_metrics->xgmi_link_width = metrics->XgmiWidth;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
index 5b1a038d6a19a..ba91bf590eed6 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
@@ -1098,7 +1098,7 @@ static int smu_v13_0_4_set_fine_grain_gfx_freq_parameters(struct smu_context *sm
 
 static const struct pptable_funcs smu_v13_0_4_ppt_funcs = {
 	.check_fw_status = smu_v13_0_check_fw_status,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.init_smc_tables = smu_v13_0_4_init_smc_tables,
 	.fini_smc_tables = smu_v13_0_4_fini_smc_tables,
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
index d534723fef913..27372a60e83d4 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
@@ -1102,7 +1102,7 @@ static int smu_v13_0_5_set_fine_grain_gfx_freq_parameters(struct smu_context *sm
 
 static const struct pptable_funcs smu_v13_0_5_ppt_funcs = {
 	.check_fw_status = smu_v13_0_check_fw_status,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.init_smc_tables = smu_v13_0_5_init_smc_tables,
 	.fini_smc_tables = smu_v13_0_5_fini_smc_tables,
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 896b51c8a9a73..bc361fee57775 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -373,6 +373,9 @@ static void smu_v13_0_12_init_caps(struct smu_context *smu)
 	} else {
 		smu_v13_0_12_tables_fini(smu);
 	}
+
+	if (fw_ver >= 0x04561000)
+		smu_v13_0_6_cap_set(smu, SMU_CAP(TEMP_AID_XCD_HBM));
 }
 
 static void smu_v13_0_6_init_caps(struct smu_context *smu)
@@ -478,7 +481,7 @@ static int smu_v13_0_6_check_fw_version(struct smu_context *smu)
 {
 	int r;
 
-	r = smu_v13_0_check_fw_version(smu);
+	r = smu_cmn_check_fw_version(smu);
 	/* Initialize caps flags once fw version is fetched */
 	if (!r)
 		smu_v13_0_x_init_caps(smu);
@@ -3160,14 +3163,25 @@ static int smu_v13_0_6_reset_vcn(struct smu_context *smu, uint32_t inst_mask)
 
 static int smu_v13_0_6_ras_send_msg(struct smu_context *smu, enum smu_message_type msg, uint32_t param, uint32_t *read_arg)
 {
+	struct amdgpu_device *adev = smu->adev;
 	int ret;
 
+	if (amdgpu_sriov_vf(adev))
+		return -EOPNOTSUPP;
+
 	switch (msg) {
 	case SMU_MSG_QueryValidMcaCount:
 	case SMU_MSG_QueryValidMcaCeCount:
 	case SMU_MSG_McaBankDumpDW:
 	case SMU_MSG_McaBankCeDumpDW:
 	case SMU_MSG_ClearMcaOnRead:
+	case SMU_MSG_GetRASTableVersion:
+	case SMU_MSG_GetBadPageCount:
+	case SMU_MSG_GetBadPageMcaAddr:
+	case SMU_MSG_SetTimestamp:
+	case SMU_MSG_GetTimestamp:
+	case SMU_MSG_GetBadPageIpid:
+	case SMU_MSG_EraseRasTable:
 		ret = smu_cmn_send_smc_msg_with_param(smu, msg, param, read_arg);
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
index ffb06564f8308..a150fc88902c4 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
@@ -78,6 +78,7 @@ enum smu_v13_0_6_caps {
 	SMU_CAP(RAS_EEPROM),
 	SMU_CAP(FAST_PPT),
 	SMU_CAP(SYSTEM_POWER_METRICS),
+	SMU_CAP(TEMP_AID_XCD_HBM),
 	SMU_CAP(ALL),
 };
 
@@ -87,6 +88,8 @@ enum smu_v13_0_6_caps {
 #define SMU_13_0_6_MAX_XCC 8
 #define SMU_13_0_6_MAX_VCN 4
 #define SMU_13_0_6_MAX_JPEG 40
+#define SMU_13_0_6_MAX_AID 4
+#define SMU_13_0_6_MAX_HBM_STACKS 8
 
 extern void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu);
 bool smu_v13_0_6_cap_supported(struct smu_context *smu, enum smu_v13_0_6_caps cap);
@@ -222,7 +225,15 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 		  SMU_13_0_6_MAX_XCC);                                         \
 	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_TOTAL_ACC), SMU_MUNIT(NONE),  \
 		  SMU_MTYPE(U64), gfx_below_host_limit_total_acc,              \
-		  SMU_13_0_6_MAX_XCC);
+		  SMU_13_0_6_MAX_XCC);					       \
+	SMU_ARRAY(SMU_MATTR(TEMPERATURE_HBM), SMU_MUNIT(TEMP_1),               \
+		  SMU_MTYPE(U16), temperature_hbm,                             \
+		  SMU_13_0_6_MAX_HBM_STACKS);                                  \
+	SMU_ARRAY(SMU_MATTR(TEMPERATURE_AID), SMU_MUNIT(TEMP_1),               \
+		  SMU_MTYPE(U16), temperature_aid, SMU_13_0_6_MAX_AID);        \
+	SMU_ARRAY(SMU_MATTR(TEMPERATURE_XCD), SMU_MUNIT(TEMP_1),               \
+		  SMU_MTYPE(U16), temperature_xcd, SMU_13_0_6_MAX_XCC);        \
+
 
 DECLARE_SMU_METRICS_CLASS(smu_v13_0_6_gpu_metrics, SMU_13_0_6_METRICS_FIELDS);
 void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 5500a0f12f0e9..88e0d99b8ad25 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -2788,7 +2788,7 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = {
 	.fini_power = smu_v13_0_fini_power,
 	.check_fw_status = smu_v13_0_7_check_fw_status,
 	.setup_pptable = smu_v13_0_7_setup_pptable,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.write_pptable = smu_cmn_write_pptable,
 	.set_driver_table_location = smu_v13_0_set_driver_table_location,
 	.system_features_control = smu_v13_0_system_features_control,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
index f43a91ac69702..7bf88ffd311b6 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
@@ -1331,7 +1331,7 @@ static int yellow_carp_set_fine_grain_gfx_freq_parameters(struct smu_context *sm
 
 static const struct pptable_funcs yellow_carp_ppt_funcs = {
 	.check_fw_status = smu_v13_0_check_fw_status,
-	.check_fw_version = smu_v13_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.init_smc_tables = yellow_carp_init_smc_tables,
 	.fini_smc_tables = yellow_carp_fini_smc_tables,
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
index 3fd84dd85e9b2..09bb3d91a0014 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
@@ -207,58 +207,6 @@ int smu_v15_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-int smu_v15_0_check_fw_version(struct smu_context *smu)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t if_version = 0xff, smu_version = 0xff;
-	uint8_t smu_program, smu_major, smu_minor, smu_debug;
-	int ret = 0;
-
-	ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version);
-	if (ret)
-		return ret;
-
-	smu_program = (smu_version >> 24) & 0xff;
-	smu_major = (smu_version >> 16) & 0xff;
-	smu_minor = (smu_version >> 8) & 0xff;
-	smu_debug = (smu_version >> 0) & 0xff;
-	if (smu->is_apu)
-		adev->pm.fw_version = smu_version;
-
-	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
-	case IP_VERSION(15, 0, 0):
-		smu->smc_driver_if_version = SMU15_DRIVER_IF_VERSION_SMU_V15_0;
-		break;
-	default:
-		dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n",
-			amdgpu_ip_version(adev, MP1_HWIP, 0));
-		smu->smc_driver_if_version = SMU15_DRIVER_IF_VERSION_INV;
-		break;
-	}
-
-	if (adev->pm.fw)
-		dev_dbg(smu->adev->dev, "smu fw reported program %d, version = 0x%08x (%d.%d.%d)\n",
-			 smu_program, smu_version, smu_major, smu_minor, smu_debug);
-
-	/*
-	 * 1. if_version mismatch is not critical as our fw is designed
-	 * to be backward compatible.
-	 * 2. New fw usually brings some optimizations. But that's visible
-	 * only on the paired driver.
-	 * Considering above, we just leave user a verbal message instead
-	 * of halt driver loading.
-	 */
-	if (if_version != smu->smc_driver_if_version) {
-		dev_info(adev->dev, "smu driver if version = 0x%08x, smu fw if version = 0x%08x, "
-			 "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n",
-			 smu->smc_driver_if_version, if_version,
-			 smu_program, smu_version, smu_major, smu_minor, smu_debug);
-		dev_info(adev->dev, "SMU driver if version not matched\n");
-	}
-
-	return ret;
-}
-
 static int smu_v15_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
 {
 	struct amdgpu_device *adev = smu->adev;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
index 49cf2b9d931e1..8d092c347076e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
@@ -1416,7 +1416,7 @@ static int smu_v15_0_common_get_dpm_table(struct smu_context *smu, struct dpm_cl
 
 static const struct pptable_funcs smu_v15_0_0_ppt_funcs = {
 	.check_fw_status = smu_v15_0_check_fw_status,
-	.check_fw_version = smu_v15_0_check_fw_version,
+	.check_fw_version = smu_cmn_check_fw_version,
 	.init_smc_tables = smu_v15_0_0_init_smc_tables,
 	.fini_smc_tables = smu_v15_0_0_fini_smc_tables,
 	.get_vbios_bootup_values = smu_v15_0_get_vbios_bootup_values,
@@ -1468,6 +1468,7 @@ void smu_v15_0_0_set_ppt_funcs(struct smu_context *smu)
 	smu->feature_map = smu_v15_0_0_feature_mask_map;
 	smu->table_map = smu_v15_0_0_table_map;
 	smu->is_apu = true;
+	smu->smc_driver_if_version = SMU15_DRIVER_IF_VERSION_SMU_V15_0;
 
 	smu_v15_0_0_init_msg_ctl(smu);
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 6fd50c2fd20e0..a644579903f40 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -405,7 +405,7 @@ static int __smu_msg_v1_ras_filter(struct smu_msg_ctl *ctl,
 }
 
 /**
- * smu_msg_proto_v1_send_msg - Complete V1 protocol with all filtering
+ * smu_msg_v1_send_msg - Complete V1 protocol with all filtering
  * @ctl: Message control block
  * @args: Message arguments
  *
@@ -880,7 +880,7 @@ static const char *smu_get_feature_name(struct smu_context *smu,
 size_t smu_cmn_get_pp_feature_mask(struct smu_context *smu,
 				   char *buf)
 {
-	int8_t sort_feature[MAX(SMU_FEATURE_COUNT, SMU_FEATURE_MAX)];
+	int16_t sort_feature[MAX(SMU_FEATURE_COUNT, SMU_FEATURE_MAX)];
 	struct smu_feature_bits feature_mask;
 	uint32_t features[2];
 	int i, feature_index;
@@ -1035,6 +1035,35 @@ int smu_cmn_get_smc_version(struct smu_context *smu,
 	return ret;
 }
 
+int smu_cmn_check_fw_version(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t if_version = 0xff, smu_version = 0xff;
+	uint8_t smu_program, smu_major, smu_minor, smu_debug;
+	int ret;
+
+	ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version);
+	if (ret)
+		return ret;
+
+	smu_program = (smu_version >> 24) & 0xff;
+	smu_major = (smu_version >> 16) & 0xff;
+	smu_minor = (smu_version >> 8) & 0xff;
+	smu_debug = (smu_version >> 0) & 0xff;
+	adev->pm.fw_version = smu_version;
+
+	dev_info_once(adev->dev, "smu driver if version = 0x%08x, smu fw if version = 0x%08x, "
+		      "smu fw program = %d, smu fw version = 0x%08x (%d.%d.%d)\n",
+		      smu->smc_driver_if_version, if_version,
+		      smu_program, smu_version, smu_major, smu_minor, smu_debug);
+
+	if (smu->smc_driver_if_version != SMU_IGNORE_IF_VERSION &&
+	    if_version != smu->smc_driver_if_version)
+		dev_info(adev->dev, "SMU driver if version not matched\n");
+
+	return 0;
+}
+
 int smu_cmn_update_table(struct smu_context *smu,
 			 enum smu_table_id table_index,
 			 int argument,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
index b7bfddc65fb21..e4d282d8bcae3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
@@ -207,6 +207,7 @@ int smu_cmn_print_pcie_levels(struct smu_context *smu,
 
 int smu_cmn_dpm_pcie_gen_idx(int gen);
 int smu_cmn_dpm_pcie_width_idx(int width);
+int smu_cmn_check_fw_version(struct smu_context *smu);
 
 /*SMU gpu metrics */
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index 9190c9cd79932..d213eea71cffe 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -535,6 +535,37 @@ int amdgpu_ras_mgr_handle_controller_interrupt(struct amdgpu_device *adev, void
 	return ret;
 }
 
+int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_info *ih_info)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	uint64_t seq_no = 0;
+	int ret = 0;
+
+	if (!amdgpu_ras_mgr_is_ready(adev))
+		return -EPERM;
+
+	if (!ih_info)
+		return 0;
+
+	if (ih_info->block == RAS_BLOCK_ID__UMC) {
+		if (ras_mgr->ras_core->poison_supported) {
+			seq_no = amdgpu_ras_mgr_gen_ras_event_seqno(adev, RAS_SEQNO_TYPE_DE);
+			RAS_DEV_INFO(adev,
+				"{%llu} RAS poison is created, no user action is needed.\n",
+				seq_no);
+		}
+
+		ret = amdgpu_ras_process_handle_umc_interrupt(adev, ih_info);
+	} else if (ras_mgr->ras_core->poison_supported) {
+		ret = amdgpu_ras_process_handle_consumption_interrupt(adev, ih_info);
+	} else {
+		RAS_DEV_WARN(adev,
+			"No RAS interrupt handler for non-UMC block with poison disabled.\n");
+	}
+
+	return ret;
+}
+
 int amdgpu_ras_mgr_handle_consumer_interrupt(struct amdgpu_device *adev, void *data)
 {
 	if (!amdgpu_ras_mgr_is_ready(adev))
@@ -571,6 +602,9 @@ bool amdgpu_ras_mgr_check_eeprom_safety_watermark(struct amdgpu_device *adev)
 	if (!amdgpu_ras_mgr_is_ready(adev))
 		return false;
 
+	if (ras_fw_eeprom_supported(ras_mgr->ras_core))
+		return ras_fw_eeprom_check_safety_watermark(ras_mgr->ras_core);
+
 	return ras_eeprom_check_safety_watermark(ras_mgr->ras_core);
 }
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
index 23c411c982319..4f44a917d48be 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
@@ -67,6 +67,7 @@ bool amdgpu_uniras_enabled(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_handle_fatal_interrupt(struct amdgpu_device *adev, void *data);
 int amdgpu_ras_mgr_handle_controller_interrupt(struct amdgpu_device *adev, void *data);
 int amdgpu_ras_mgr_handle_consumer_interrupt(struct amdgpu_device *adev, void *data);
+int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_info *ih_info);
 int amdgpu_ras_mgr_update_ras_ecc(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_reset_gpu(struct amdgpu_device *adev, uint32_t flags);
 uint64_t amdgpu_ras_mgr_gen_ras_event_seqno(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
index 79a51b1603ac1..2098f24d4940c 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
@@ -28,6 +28,16 @@
 #define RAS_MP1_MSG_QueryValidMcaCeCount  0x3A
 #define RAS_MP1_MSG_McaBankCeDumpDW       0x3B
 
+static const enum smu_message_type pmfw_eeprom_msgs[] = {
+	[RAS_SMU_GetRASTableVersion] = SMU_MSG_GetRASTableVersion,
+	[RAS_SMU_GetBadPageCount] = SMU_MSG_GetBadPageCount,
+	[RAS_SMU_SetTimestamp] = SMU_MSG_SetTimestamp,
+	[RAS_SMU_GetTimestamp] = SMU_MSG_GetTimestamp,
+	[RAS_SMU_GetBadPageIpid] = SMU_MSG_GetBadPageIpid,
+	[RAS_SMU_EraseRasTable] = SMU_MSG_EraseRasTable,
+	[RAS_SMU_GetBadPageMcaAddr] = SMU_MSG_GetBadPageMcaAddr,
+};
+
 static int mp1_v13_0_get_valid_bank_count(struct ras_core_context *ras_core,
 					  u32 msg, u32 *count)
 {
@@ -87,8 +97,44 @@ static int mp1_v13_0_dump_valid_bank(struct ras_core_context *ras_core,
 	return ret;
 }
 
+static int mp1_v13_0_eeprom_send_msg(struct ras_core_context *ras_core,
+				enum ras_fw_eeprom_cmd index, uint32_t param, uint32_t *read_arg)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	int ret = 0;
+
+	if (down_read_trylock(&adev->reset_domain->sem)) {
+		ret = amdgpu_smu_ras_send_msg(adev,
+			pmfw_eeprom_msgs[index], param, read_arg);
+		up_read(&adev->reset_domain->sem);
+	} else {
+		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
+	}
+
+	return ret;
+}
+
+static int mp1_v13_0_get_ras_enabled_mask(struct ras_core_context *ras_core,
+					     uint64_t *enabled_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	int ret = 0;
+
+	if (down_read_trylock(&adev->reset_domain->sem)) {
+		if (amdgpu_smu_ras_feature_is_enabled(adev, SMU_FEATURE_HROM_EN_BIT))
+			*enabled_mask |= RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM;
+		up_read(&adev->reset_domain->sem);
+	} else {
+		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
+	}
+
+	return ret;
+}
+
 const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func_v13_0 = {
 	.mp1_get_valid_bank_count = mp1_v13_0_get_valid_bank_count,
 	.mp1_dump_valid_bank = mp1_v13_0_dump_valid_bank,
+	.mp1_send_eeprom_msg = mp1_v13_0_eeprom_send_msg,
+	.mp1_get_ras_enabled_mask = mp1_v13_0_get_ras_enabled_mask,
 };
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
index 5782c007de71c..285d81a85a99d 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
@@ -103,6 +103,16 @@ int amdgpu_ras_process_handle_consumption_interrupt(struct amdgpu_device *adev,
 	if (!ih_info)
 		return -EINVAL;
 
+	if (amdgpu_sriov_vf(adev)) {
+		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
+			adev->virt.ops->ras_poison_handler(adev, ih_info->block);
+		else
+			dev_warn(adev->dev,
+				"No ras_poison_handler interface in SRIOV for block[%d]!\n",
+				ih_info->block);
+		return 0;
+	}
+
 	memset(&req, 0, sizeof(req));
 	req.block = ih_info->block;
 	req.data = ih_info->data;
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
index 45ed8c3b5563b..7d728e5236042 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
@@ -137,7 +137,8 @@ static int amdgpu_ras_sys_event_notifier(struct ras_core_context *ras_core,
 		break;
 	case RAS_EVENT_ID__DEVICE_RMA:
 		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_RMA, NULL, NULL);
-		ret = amdgpu_dpm_send_rma_reason(ras_core->dev);
+		if (!ras_fw_eeprom_supported(ras_core))
+			ret = amdgpu_dpm_send_rma_reason(ras_core->dev);
 		break;
 	case RAS_EVENT_ID__RESET_GPU:
 		ret = amdgpu_ras_mgr_reset_gpu(ras_core->dev, *(uint32_t *)data);
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
index de197ae4b37f8..c83357307c558 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
@@ -30,28 +30,83 @@
 #include "amdgpu_virt_ras_cmd.h"
 #include "amdgpu_ras_mgr.h"
 
+static int amdgpu_virt_ras_get_cmd_shared_mem(struct ras_core_context *ras_core,
+		uint32_t cmd, uint32_t mem_size, struct amdgpu_virt_shared_mem *shared_mem)
+{
+	struct amdgpu_device *adev = ras_core->dev;
+	struct amdsriov_ras_telemetry *ras_telemetry_cpu;
+	struct amdsriov_ras_telemetry *ras_telemetry_gpu;
+	uint64_t fw_vram_usage_start_offset = 0;
+	uint64_t ras_telemetry_offset = 0;
+
+	if (!adev->virt.fw_reserve.ras_telemetry)
+		return -EINVAL;
+
+	if (adev->mman.fw_vram_usage_va &&
+	    adev->mman.fw_vram_usage_va <= adev->virt.fw_reserve.ras_telemetry) {
+		fw_vram_usage_start_offset = adev->mman.fw_vram_usage_start_offset;
+		ras_telemetry_offset = (uintptr_t)adev->virt.fw_reserve.ras_telemetry -
+				(uintptr_t)adev->mman.fw_vram_usage_va;
+	} else if (adev->mman.drv_vram_usage_va &&
+		adev->mman.drv_vram_usage_va <= adev->virt.fw_reserve.ras_telemetry) {
+		fw_vram_usage_start_offset = adev->mman.drv_vram_usage_start_offset;
+		ras_telemetry_offset = (uintptr_t)adev->virt.fw_reserve.ras_telemetry -
+				(uintptr_t)adev->mman.drv_vram_usage_va;
+	} else {
+		return -EINVAL;
+	}
+
+	ras_telemetry_cpu =
+		(struct amdsriov_ras_telemetry *)adev->virt.fw_reserve.ras_telemetry;
+	ras_telemetry_gpu =
+		(struct amdsriov_ras_telemetry *)(uintptr_t)(fw_vram_usage_start_offset +
+				ras_telemetry_offset);
+
+	if (cmd == RAS_CMD__GET_ALL_BLOCK_ECC_STATUS) {
+		if (mem_size > AMD_SRIOV_UNIRAS_BLOCKS_BUF_SIZE)
+			return -ENOMEM;
+
+		shared_mem->cpu_addr = ras_telemetry_cpu->uniras_shared_mem.blocks_ecc_buf;
+		shared_mem->gpa =
+			(uintptr_t)ras_telemetry_gpu->uniras_shared_mem.blocks_ecc_buf -
+					adev->gmc.vram_start;
+		shared_mem->size = mem_size;
+	} else {
+		if (mem_size > AMD_SRIOV_UNIRAS_CMD_MAX_SIZE)
+			return -ENOMEM;
+
+		shared_mem->cpu_addr = ras_telemetry_cpu->uniras_shared_mem.cmd_buf;
+		shared_mem->gpa =
+			(uintptr_t)ras_telemetry_gpu->uniras_shared_mem.cmd_buf -
+					adev->gmc.vram_start;
+		shared_mem->size = mem_size;
+	}
+
+	return 0;
+}
+
 static int amdgpu_virt_ras_remote_ioctl_cmd(struct ras_core_context *ras_core,
 			struct ras_cmd_ctx *cmd, void *output_data, uint32_t output_size)
 {
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(ras_core->dev);
+	struct amdgpu_virt_ras_cmd *virt_ras = ras_mgr->virt_ras_cmd;
 	uint32_t mem_len = ALIGN(sizeof(*cmd) + output_size, AMDGPU_GPU_PAGE_SIZE);
 	struct ras_cmd_ctx *rcmd;
-	struct amdgpu_bo *rcmd_bo = NULL;
-	uint64_t mc_addr = 0;
-	void *cpu_addr = NULL;
+	struct amdgpu_virt_shared_mem shared_mem = {0};
 	int ret = 0;
 
-	ret = amdgpu_bo_create_kernel(adev, mem_len, PAGE_SIZE,
-			AMDGPU_GEM_DOMAIN_VRAM, &rcmd_bo, &mc_addr, (void **)&cpu_addr);
+	mutex_lock(&virt_ras->remote_access_lock);
+
+	ret = amdgpu_virt_ras_get_cmd_shared_mem(ras_core, cmd->cmd_id, mem_len, &shared_mem);
 	if (ret)
-		return ret;
+		goto out;
 
-	rcmd = (struct ras_cmd_ctx *)cpu_addr;
+	rcmd = (struct ras_cmd_ctx *)shared_mem.cpu_addr;
 	memset(rcmd, 0, mem_len);
 	memcpy(rcmd, cmd, sizeof(*cmd));
 
 	ret = amdgpu_virt_send_remote_ras_cmd(ras_core->dev,
-				mc_addr - adev->gmc.vram_start, mem_len);
+				shared_mem.gpa, mem_len);
 	if (!ret) {
 		if (rcmd->cmd_res) {
 			ret = rcmd->cmd_res;
@@ -65,8 +120,7 @@ static int amdgpu_virt_ras_remote_ioctl_cmd(struct ras_core_context *ras_core,
 	}
 
 out:
-	amdgpu_bo_free_kernel(&rcmd_bo, &mc_addr, &cpu_addr);
-
+	mutex_unlock(&virt_ras->remote_access_lock);
 	return ret;
 }
 
@@ -77,6 +131,9 @@ static int amdgpu_virt_ras_send_remote_cmd(struct ras_core_context *ras_core,
 	struct ras_cmd_ctx rcmd = {0};
 	int ret;
 
+	if (input_size > RAS_CMD_MAX_IN_SIZE)
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
 	rcmd.cmd_id = cmd_id;
 	rcmd.input_size = input_size;
 	memcpy(rcmd.input_buff_raw, input_data, input_size);
@@ -146,7 +203,7 @@ static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core,
 	struct ras_cmd_batch_trace_record_rsp *rsp = rsp_cache;
 	struct batch_ras_trace_info *batch;
 	int ret = 0;
-	uint8_t i;
+	uint32_t i;
 
 	if (!rsp->real_batch_num || (batch_id < rsp->start_batch_id) ||
 		(batch_id >=  (rsp->start_batch_id + rsp->real_batch_num))) {
@@ -249,14 +306,14 @@ static int __fill_get_blocks_ecc_cmd(struct amdgpu_device *adev,
 {
 	struct ras_cmd_ctx *rcmd;
 
-	if (!blks_ecc || !blks_ecc->bo || !blks_ecc->cpu_addr)
+	if (!blks_ecc || !blks_ecc->shared_mem.cpu_addr)
 		return -EINVAL;
 
-	rcmd = (struct ras_cmd_ctx *)blks_ecc->cpu_addr;
+	rcmd = (struct ras_cmd_ctx *)blks_ecc->shared_mem.cpu_addr;
 
 	rcmd->cmd_id = RAS_CMD__GET_ALL_BLOCK_ECC_STATUS;
 	rcmd->input_size = sizeof(struct ras_cmd_blocks_ecc_req);
-	rcmd->output_buf_size = blks_ecc->size - sizeof(*rcmd);
+	rcmd->output_buf_size = blks_ecc->shared_mem.size - sizeof(*rcmd);
 
 	return 0;
 }
@@ -305,15 +362,15 @@ static int amdgpu_virt_ras_get_block_ecc(struct ras_core_context *ras_core,
 
 	if (!virt_ras->blocks_ecc.auto_update_actived) {
 		ret = __set_cmd_auto_update(adev, RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
-				blks_ecc->mc_addr - adev->gmc.vram_start,
-				blks_ecc->size, true);
+				blks_ecc->shared_mem.gpa,
+				blks_ecc->shared_mem.size, true);
 		if (ret)
 			return ret;
 
 		blks_ecc->auto_update_actived = true;
 	}
 
-	blks_ecc_cmd_ctx = blks_ecc->cpu_addr;
+	blks_ecc_cmd_ctx = blks_ecc->shared_mem.cpu_addr;
 	blks_ecc_rsp = (struct ras_cmd_blocks_ecc_rsp *)blks_ecc_cmd_ctx->output_buff_raw;
 
 	output_data->ce_count = blks_ecc_rsp->blocks[input_data->block_id].ce_count;
@@ -324,6 +381,53 @@ static int amdgpu_virt_ras_get_block_ecc(struct ras_core_context *ras_core,
 	return RAS_CMD__SUCCESS;
 }
 
+int amdgpu_virt_ras_check_address_validity(struct amdgpu_device *adev,
+			uint64_t address, bool *hit)
+{
+	struct ras_cmd_address_check_req req = {0};
+	struct ras_cmd_address_check_rsp rsp = {0};
+	int ret = 0;
+
+	req.address = address;
+
+	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CHECK_ADDRESS_VALIDITY,
+		&req, sizeof(req), &rsp, sizeof(rsp));
+
+	if (ret)
+		return RAS_CMD__ERROR_GENERIC;
+
+	*hit = rsp.result ? true : false;
+
+	return RAS_CMD__SUCCESS;
+}
+
+int amdgpu_virt_ras_convert_retired_address(struct amdgpu_device *adev,
+			uint64_t address, uint64_t *pfn, uint32_t max_pfn_sz)
+{
+	struct ras_cmd_convert_retired_address_req req = {0};
+	struct ras_cmd_convert_retired_address_rsp rsp = {0};
+	int ret = 0, i;
+	int retired_page_count;
+
+	if (!pfn || !max_pfn_sz)
+		return -EINVAL;
+
+	req.address = address;
+
+	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CONVERT_RETIRED_ADDRESS,
+		&req, sizeof(req), &rsp, sizeof(rsp));
+
+	if (ret || rsp.retired_count == 0)
+		return -EINVAL;
+
+	retired_page_count = rsp.retired_count > max_pfn_sz ? max_pfn_sz : rsp.retired_count;
+
+	for (i = 0; i < retired_page_count; i++)
+		pfn[i] = rsp.retired_addr[i] >> AMDGPU_GPU_PAGE_SHIFT;
+
+	return retired_page_count;
+}
+
 static struct ras_cmd_func_map amdgpu_virt_ras_cmd_maps[] = {
 	{RAS_CMD__GET_CPER_SNAPSHOT, amdgpu_virt_ras_get_cper_snapshot},
 	{RAS_CMD__GET_CPER_RECORD, amdgpu_virt_ras_get_cper_records},
@@ -364,18 +468,24 @@ int amdgpu_virt_ras_handle_cmd(struct ras_core_context *ras_core,
 int amdgpu_virt_ras_sw_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras_cmd;
 
 	ras_mgr->virt_ras_cmd = kzalloc_obj(struct amdgpu_virt_ras_cmd);
 	if (!ras_mgr->virt_ras_cmd)
 		return -ENOMEM;
 
+	virt_ras_cmd = ras_mgr->virt_ras_cmd;
+	mutex_init(&virt_ras_cmd->remote_access_lock);
+
 	return 0;
 }
 
 int amdgpu_virt_ras_sw_fini(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras_cmd = ras_mgr->virt_ras_cmd;
 
+	mutex_destroy(&virt_ras_cmd->remote_access_lock);
 	kfree(ras_mgr->virt_ras_cmd);
 	ras_mgr->virt_ras_cmd = NULL;
 
@@ -392,11 +502,9 @@ int amdgpu_virt_ras_hw_init(struct amdgpu_device *adev)
 	amdgpu_virt_get_ras_capability(adev);
 
 	memset(blks_ecc, 0, sizeof(*blks_ecc));
-	blks_ecc->size = PAGE_SIZE;
-	if (amdgpu_bo_create_kernel(adev, blks_ecc->size,
-			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
-			&blks_ecc->bo, &blks_ecc->mc_addr,
-			(void **)&blks_ecc->cpu_addr))
+	if (amdgpu_virt_ras_get_cmd_shared_mem(ras_mgr->ras_core,
+			RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
+			AMD_SRIOV_UNIRAS_BLOCKS_BUF_SIZE, &blks_ecc->shared_mem))
 		return -ENOMEM;
 
 	return 0;
@@ -409,18 +517,15 @@ int amdgpu_virt_ras_hw_fini(struct amdgpu_device *adev)
 			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
 	struct vram_blocks_ecc *blks_ecc = &virt_ras->blocks_ecc;
 
-	if (blks_ecc->bo) {
+	if (blks_ecc->shared_mem.cpu_addr) {
 		__set_cmd_auto_update(adev,
 			RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
-			blks_ecc->mc_addr - adev->gmc.vram_start,
-			blks_ecc->size, false);
-
-		memset(blks_ecc->cpu_addr, 0, blks_ecc->size);
-		amdgpu_bo_free_kernel(&blks_ecc->bo,
-			&blks_ecc->mc_addr, &blks_ecc->cpu_addr);
+			blks_ecc->shared_mem.gpa,
+			blks_ecc->shared_mem.size, false);
 
-		memset(blks_ecc, 0, sizeof(*blks_ecc));
+		memset(blks_ecc->shared_mem.cpu_addr, 0, blks_ecc->shared_mem.size);
 	}
+	memset(blks_ecc, 0, sizeof(*blks_ecc));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
index 53b0f3f60103e..3000c354b76fd 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
@@ -30,11 +30,14 @@ struct remote_batch_trace_mgr {
 	struct ras_cmd_batch_trace_record_rsp  batch_trace;
 };
 
-struct vram_blocks_ecc {
-	struct amdgpu_bo *bo;
-	uint64_t mc_addr;
+struct amdgpu_virt_shared_mem {
+	uint64_t gpa;
 	void *cpu_addr;
 	uint32_t size;
+};
+
+struct vram_blocks_ecc {
+	struct amdgpu_virt_shared_mem shared_mem;
 	bool auto_update_actived;
 };
 
@@ -42,6 +45,7 @@ struct amdgpu_virt_ras_cmd {
 	bool remote_uniras_supported;
 	struct remote_batch_trace_mgr batch_mgr;
 	struct vram_blocks_ecc blocks_ecc;
+	struct mutex remote_access_lock;
 };
 
 int amdgpu_virt_ras_sw_init(struct amdgpu_device *adev);
@@ -54,4 +58,8 @@ int amdgpu_virt_ras_pre_reset(struct amdgpu_device *adev);
 int amdgpu_virt_ras_post_reset(struct amdgpu_device *adev);
 void amdgpu_virt_ras_set_remote_uniras(struct amdgpu_device *adev, bool en);
 bool amdgpu_virt_ras_remote_uniras_enabled(struct amdgpu_device *adev);
+int amdgpu_virt_ras_check_address_validity(struct amdgpu_device *adev,
+			uint64_t address, bool *hit);
+int amdgpu_virt_ras_convert_retired_address(struct amdgpu_device *adev,
+			uint64_t address, uint64_t *pfn, uint32_t max_pfn_sz);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/Makefile b/drivers/gpu/drm/amd/ras/rascore/Makefile
index e826a1f86424c..06b265ec1cde8 100644
--- a/drivers/gpu/drm/amd/ras/rascore/Makefile
+++ b/drivers/gpu/drm/amd/ras/rascore/Makefile
@@ -36,7 +36,8 @@ RAS_CORE_FILES = ras_core.o \
 			ras_log_ring.o \
 			ras_cper.o \
 			ras_psp.o \
-			ras_psp_v13_0.o
+			ras_psp_v13_0.o \
+			ras_eeprom_fw.o
 
 
 RAS_CORE = $(addprefix $(AMD_GPU_RAS_PATH)/rascore/,$(RAS_CORE_FILES))
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras.h b/drivers/gpu/drm/amd/ras/rascore/ras.h
index 71d807e8d2a7b..6449d7b8627d6 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras.h
@@ -36,6 +36,7 @@
 #include "ras_mp1.h"
 #include "ras_psp.h"
 #include "ras_log_ring.h"
+#include "ras_eeprom_fw.h"
 
 #define RAS_HW_ERR		"[Hardware Error]: "
 
@@ -49,6 +50,17 @@
 #define GPU_RESET_CAUSE_FATAL   (RAS_CORE_RESET_GPU | 0x0002)
 #define GPU_RESET_CAUSE_RMA     (RAS_CORE_RESET_GPU | 0x0004)
 
+enum ras_gpu_health_status {
+	RAS_GPU_HEALTH_NONE = 0,
+	RAS_GPU_HEALTH_USABLE = 1,
+	RAS_GPU_RETIRED__ECC_REACH_THRESHOLD = 2,
+	RAS_GPU_IN_BAD_STATUS = 3,
+};
+
+enum ras_core_fw_feature_flags {
+	RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM = BIT_ULL(0),
+};
+
 enum ras_block_id {
 	RAS_BLOCK_ID__UMC = 0,
 	RAS_BLOCK_ID__SDMA,
@@ -127,6 +139,16 @@ enum ras_gpu_status {
 	RAS_GPU_STATUS__IS_VF = 0x8,
 };
 
+enum ras_fw_eeprom_cmd {
+	RAS_SMU_GetRASTableVersion = 0,
+	RAS_SMU_GetBadPageCount,
+	RAS_SMU_SetTimestamp,
+	RAS_SMU_GetTimestamp,
+	RAS_SMU_GetBadPageIpid,
+	RAS_SMU_EraseRasTable,
+	RAS_SMU_GetBadPageMcaAddr,
+};
+
 struct ras_core_context;
 struct ras_bank_ecc;
 struct ras_umc;
@@ -141,6 +163,10 @@ struct ras_mp1_sys_func {
 			u32 msg, u32 *count);
 	int (*mp1_dump_valid_bank)(struct ras_core_context *ras_core,
 			u32 msg, u32 idx, u32 reg_idx, u64 *val);
+	int (*mp1_send_eeprom_msg)(struct ras_core_context *ras_core,
+			enum ras_fw_eeprom_cmd index, uint32_t param, uint32_t *read_arg);
+	int (*mp1_get_ras_enabled_mask)(struct ras_core_context *ras_core,
+			uint64_t *enabled_mask);
 };
 
 struct ras_eeprom_sys_func {
@@ -222,6 +248,7 @@ struct ras_bank_ecc {
 	uint64_t status;
 	uint64_t ipid;
 	uint64_t addr;
+	uint64_t ts;
 };
 
 struct ras_bank_ecc_node {
@@ -294,6 +321,7 @@ struct ras_core_context {
 
 	bool ras_eeprom_supported;
 	struct ras_eeprom_control ras_eeprom;
+	struct ras_fw_eeprom_control ras_fw_eeprom;
 
 	struct ras_psp ras_psp;
 	struct ras_umc ras_umc;
@@ -317,6 +345,8 @@ struct ras_core_context {
 	spinlock_t seqno_lock;
 
 	bool ras_core_enabled;
+
+	u64 ras_fw_features;
 };
 
 struct ras_core_context *ras_core_create(struct ras_core_config *init_config);
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_aca.c b/drivers/gpu/drm/amd/ras/rascore/ras_aca.c
index e433c70d29891..67a35409ff0e9 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_aca.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_aca.c
@@ -234,16 +234,27 @@ static int aca_log_bad_bank(struct ras_core_context *ras_core,
 	    bank_ecc->de_count) {
 		struct ras_bank_ecc  ras_ecc = {0};
 
-		ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
-		ras_ecc.addr = bank_ecc->bank_info.addr;
-		ras_ecc.ipid = bank_ecc->bank_info.ipid;
-		ras_ecc.status = bank_ecc->bank_info.status;
-		ras_ecc.seq_no = bank->seq_no;
-
-		if (ras_core_gpu_in_reset(ras_core))
-			ras_umc_log_bad_bank_pending(ras_core, &ras_ecc);
-		else
-			ras_umc_log_bad_bank(ras_core, &ras_ecc);
+		if (ras_fw_eeprom_supported(ras_core)) {
+			ret = ras_fw_eeprom_update_record(ras_core, &ras_ecc);
+			if (!ret) {
+				ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
+				ras_ecc.status = bank_ecc->bank_info.status;
+				ras_ecc.seq_no = bank->seq_no;
+			}
+		} else {
+			ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
+			ras_ecc.addr = bank_ecc->bank_info.addr;
+			ras_ecc.ipid = bank_ecc->bank_info.ipid;
+			ras_ecc.status = bank_ecc->bank_info.status;
+			ras_ecc.seq_no = bank->seq_no;
+		}
+
+		if (!ret) {
+			if (ras_core_gpu_in_reset(ras_core))
+				ras_umc_log_bad_bank_pending(ras_core, &ras_ecc);
+			else
+				ras_umc_log_bad_bank(ras_core, &ras_ecc);
+		}
 	}
 
 	aca_report_ecc_info(ras_core,
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
index 94e6d7420d94e..4f89810d85a15 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
@@ -146,8 +146,13 @@ static int ras_cmd_clear_bad_page_info(struct ras_core_context *ras_core,
 	if (cmd->input_size != sizeof(struct ras_cmd_dev_handle))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
-	if (ras_eeprom_reset_table(ras_core))
-		return RAS_CMD__ERROR_GENERIC;
+	if (ras_fw_eeprom_supported(ras_core)) {
+		if (ras_fw_eeprom_reset_table(ras_core))
+			return RAS_CMD__ERROR_GENERIC;
+	} else {
+		if (ras_eeprom_reset_table(ras_core))
+			return RAS_CMD__ERROR_GENERIC;
+	}
 
 	if (ras_umc_clean_badpage_data(ras_core))
 		return RAS_CMD__ERROR_GENERIC;
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
index b9833812c31fa..7ea35a0289874 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
@@ -77,6 +77,8 @@ enum ras_cmd_id {
 	RAS_CMD__GET_BATCH_TRACE_RECORD,
 	RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
 	RAS_CMD__SET_CMD_AUTO_UPDATE,
+	RAS_CMD__CHECK_ADDRESS_VALIDITY,
+	RAS_CMD__CONVERT_RETIRED_ADDRESS,
 	RAS_CMD__SUPPORTED_MAX = RAS_CMD_ID_COMMON_END,
 };
 
@@ -427,6 +429,33 @@ struct ras_cmd_auto_update_rsp {
 	uint32_t reserved[4];
 };
 
+struct ras_cmd_address_check_req {
+	struct ras_cmd_dev_handle dev;
+	uint64_t address;
+	uint32_t flags;
+	uint32_t vf_idx;
+	uint32_t reserved[4];
+};
+
+struct ras_cmd_address_check_rsp {
+	uint32_t version;
+	uint32_t result;
+	uint32_t reserved[6];
+};
+
+struct ras_cmd_convert_retired_address_req {
+	struct ras_cmd_dev_handle dev;
+	uint64_t address;
+	uint32_t reserved[6];
+};
+
+#define RAS_CMD_MAX_RETIRED_ADDR_COUNT  32
+struct ras_cmd_convert_retired_address_rsp {
+	uint32_t version;
+	uint32_t retired_count;
+	uint64_t retired_addr[RAS_CMD_MAX_RETIRED_ADDR_COUNT];
+};
+
 struct ras_cmd_blocks_ecc_req {
 	struct ras_cmd_dev_handle dev;
 };
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_core.c b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
index 3f56f26abd6da..e889baec378b0 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_core.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
@@ -62,14 +62,16 @@ int ras_core_convert_timestamp_to_time(struct ras_core_context *ras_core,
 			uint64_t timestamp, struct ras_time *tm)
 {
 	int days_in_month[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
-	uint64_t month = 0, day = 0, hour = 0, minute = 0, second = 0;
+	uint64_t month = 0, day = 0, hour = 0, minute = 0, second = 0, remainder;
 	uint32_t year = 0;
 	int seconds_per_day = 24 * 60 * 60;
 	int seconds_per_hour = 60 * 60;
 	int seconds_per_minute = 60;
 	int days, remaining_seconds;
 
-	days = div64_u64_rem(timestamp, seconds_per_day, (uint64_t *)&remaining_seconds);
+	days = div64_u64_rem(timestamp, seconds_per_day, &remainder);
+	/* remainder will always be less than seconds_per_day. */
+	remaining_seconds = remainder;
 
 	/* utc_timestamp follows the Unix epoch */
 	year = 1970;
@@ -239,7 +241,10 @@ static int ras_core_eeprom_recovery(struct ras_core_context *ras_core)
 	int count;
 	int ret;
 
-	count = ras_eeprom_get_record_count(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		count = ras_fw_eeprom_get_record_count(ras_core);
+	else
+		count = ras_eeprom_get_record_count(ras_core);
 	if (!count)
 		return 0;
 
@@ -253,7 +258,10 @@ static int ras_core_eeprom_recovery(struct ras_core_context *ras_core)
 		return ret;
 	}
 
-	ras_eeprom_sync_info(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		ras_fw_eeprom_sync_info(ras_core);
+	else
+		ras_eeprom_sync_info(ras_core);
 
 	return ret;
 }
@@ -382,7 +390,12 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 	if (ret)
 		goto init_err5;
 
-	ret = ras_eeprom_hw_init(ras_core);
+	ras_fw_init_feature_flags(ras_core);
+
+	if (ras_fw_eeprom_supported(ras_core))
+		ret = ras_fw_eeprom_hw_init(ras_core);
+	else
+		ret = ras_eeprom_hw_init(ras_core);
 	if (ret)
 		goto init_err6;
 
@@ -393,7 +406,10 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 		goto init_err6;
 	}
 
-	ret = ras_eeprom_check_storage_status(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		ret = ras_fw_eeprom_check_storage_status(ras_core);
+	else
+		ret = ras_eeprom_check_storage_status(ras_core);
 	if (ret)
 		goto init_err6;
 
@@ -406,7 +422,10 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 	return 0;
 
 init_err7:
-	ras_eeprom_hw_fini(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		ras_fw_eeprom_hw_fini(ras_core);
+	else
+		ras_eeprom_hw_fini(ras_core);
 init_err6:
 	ras_gfx_hw_fini(ras_core);
 init_err5:
@@ -427,7 +446,10 @@ int ras_core_hw_fini(struct ras_core_context *ras_core)
 	ras_core->is_initialized = false;
 
 	ras_process_fini(ras_core);
-	ras_eeprom_hw_fini(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		ras_fw_eeprom_hw_fini(ras_core);
+	else
+		ras_eeprom_hw_fini(ras_core);
 	ras_gfx_hw_fini(ras_core);
 	ras_nbio_hw_fini(ras_core);
 	ras_umc_hw_fini(ras_core);
@@ -505,8 +527,11 @@ bool ras_core_is_enabled(struct ras_core_context *ras_core)
 
 uint64_t ras_core_get_utc_second_timestamp(struct ras_core_context *ras_core)
 {
-	if (ras_core && ras_core->sys_fn &&
-		ras_core->sys_fn->get_utc_second_timestamp)
+	if (!ras_core)
+		return 0;
+
+	if (ras_core->sys_fn &&
+	    ras_core->sys_fn->get_utc_second_timestamp)
 		return ras_core->sys_fn->get_utc_second_timestamp(ras_core);
 
 	RAS_DEV_ERR(ras_core->dev, "Failed to get system timestamp!\n");
@@ -528,7 +553,9 @@ bool ras_core_ras_interrupt_detected(struct ras_core_context *ras_core)
 		ras_core->sys_fn->detect_ras_interrupt)
 		return ras_core->sys_fn->detect_ras_interrupt(ras_core);
 
-	RAS_DEV_ERR(ras_core->dev, "Failed to detect ras interrupt!\n");
+	if (ras_core && ras_core->dev)
+		RAS_DEV_ERR(ras_core->dev, "Failed to detect ras interrupt!\n");
+
 	return false;
 }
 
@@ -559,6 +586,9 @@ bool ras_core_is_ready(struct ras_core_context *ras_core)
 
 bool ras_core_check_safety_watermark(struct ras_core_context *ras_core)
 {
+	if (ras_fw_eeprom_supported(ras_core))
+		return ras_fw_eeprom_check_safety_watermark(ras_core);
+
 	return ras_eeprom_check_safety_watermark(ras_core);
 }
 
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
index cd6b057bdaf3b..65c1812a10fb1 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
@@ -829,8 +829,8 @@ static int ras_eeprom_update_header(struct ras_eeprom_control *control)
 }
 
 /**
- * ras_core_eeprom_append -- append records to the EEPROM RAS table
- * @control: pointer to control structure
+ * ras_eeprom_append -- append records to the EEPROM RAS table
+ * @ras_core: pointer to ras core context
  * @record: array of records to append
  * @num: number of records in @record array
  *
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.h b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.h
index 2abe566c18b67..f2c001ef64e1f 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.h
@@ -57,13 +57,6 @@ do { \
 	(RECORD)->retired_row_pfn = tmp; \
 } while (0)
 
-enum ras_gpu_health_status {
-	RAS_GPU_HEALTH_NONE = 0,
-	RAS_GPU_HEALTH_USABLE = 1,
-	RAS_GPU_RETIRED__ECC_REACH_THRESHOLD = 2,
-	RAS_GPU_IN_BAD_STATUS = 3,
-};
-
 enum ras_eeprom_err_type {
 	RAS_EEPROM_ERR_NA,
 	RAS_EEPROM_ERR_RECOVERABLE,
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
new file mode 100644
index 0000000000000..29001e606d1bc
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
@@ -0,0 +1,520 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "ras.h"
+
+#define RAS_SMU_MESSAGE_TIMEOUT_MS 1000 /* 1s */
+
+void ras_fw_init_feature_flags(struct ras_core_context *ras_core)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	uint64_t flags = 0ULL;
+
+	if (!sys_func || !sys_func->mp1_get_ras_enabled_mask)
+		return;
+
+	if (!sys_func->mp1_get_ras_enabled_mask(ras_core, &flags))
+		ras_core->ras_fw_features = flags;
+}
+
+bool ras_fw_eeprom_supported(struct ras_core_context *ras_core)
+{
+	return !!(ras_core->ras_fw_features & RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM);
+}
+
+int ras_fw_get_table_version(struct ras_core_context *ras_core,
+				     uint32_t *table_version)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+
+	return sys_func->mp1_send_eeprom_msg(ras_core,
+				RAS_SMU_GetRASTableVersion, 0, table_version);
+}
+
+int ras_fw_get_badpage_count(struct ras_core_context *ras_core,
+				     uint32_t *count, uint32_t timeout)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	uint64_t end, now;
+	int ret = 0;
+
+	now = (uint64_t)ktime_to_ms(ktime_get());
+	end = now + timeout;
+
+	do {
+		ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetBadPageCount, 0, count);
+		/* eeprom is not ready */
+		if (ret != -EBUSY)
+			return ret;
+
+		mdelay(10);
+		now = (uint64_t)ktime_to_ms(ktime_get());
+	} while (now < end);
+
+	RAS_DEV_ERR(ras_core->dev,
+			"smu get bad page count timeout!\n");
+	return ret;
+}
+
+int ras_fw_get_badpage_mca_addr(struct ras_core_context *ras_core,
+					uint16_t index, uint64_t *mca_addr)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	uint32_t temp_arg, temp_addr_lo, temp_addr_high;
+	int ret;
+
+	temp_arg = index | (1 << 16);
+	ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetBadPageMcaAddr, temp_arg, &temp_addr_lo);
+	if (ret)
+		return ret;
+
+	temp_arg = index | (2 << 16);
+	ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetBadPageMcaAddr, temp_arg, &temp_addr_high);
+
+	if (!ret)
+		*mca_addr = (uint64_t)temp_addr_high << 32 | temp_addr_lo;
+
+	return ret;
+}
+
+int ras_fw_set_timestamp(struct ras_core_context *ras_core,
+				 uint64_t timestamp)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+
+	return sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_SetTimestamp, (uint32_t)timestamp, 0);
+}
+
+int ras_fw_get_timestamp(struct ras_core_context *ras_core,
+				 uint16_t index, uint64_t *timestamp)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	uint32_t temp = 0;
+	int ret;
+
+	ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetTimestamp, index, &temp);
+	if (!ret)
+		*timestamp = temp;
+
+	return ret;
+}
+
+int ras_fw_get_badpage_ipid(struct ras_core_context *ras_core,
+				    uint16_t index, uint64_t *ipid)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	uint32_t temp_arg, temp_ipid_lo, temp_ipid_high;
+	int ret;
+
+	temp_arg = index | (1 << 16);
+	ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetBadPageIpid, temp_arg, &temp_ipid_lo);
+	if (ret)
+		return ret;
+
+	temp_arg = index | (2 << 16);
+	ret = sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_GetBadPageIpid, temp_arg, &temp_ipid_high);
+	if (!ret)
+		*ipid = (uint64_t)temp_ipid_high << 32 | temp_ipid_lo;
+
+	return ret;
+}
+
+int ras_fw_erase_ras_table(struct ras_core_context *ras_core,
+				   uint32_t *result)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+
+	return sys_func->mp1_send_eeprom_msg(ras_core,
+			RAS_SMU_EraseRasTable, 0, result);
+}
+
+int ras_fw_eeprom_reset_table(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	u32 erase_res = 0;
+	int res;
+
+	mutex_lock(&control->ras_tbl_mutex);
+
+	res = ras_fw_erase_ras_table(ras_core, &erase_res);
+	if (res || erase_res) {
+		RAS_DEV_WARN(ras_core->dev, "RAS EEPROM reset failed, res:%d result:%d",
+									res, erase_res);
+		if (!res)
+			res = -EIO;
+	}
+
+	control->ras_num_recs = 0;
+	control->bad_channel_bitmap = 0;
+	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
+		&control->ras_num_recs);
+	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
+		&control->bad_channel_bitmap);
+	control->update_channel_flag = false;
+
+	mutex_unlock(&control->ras_tbl_mutex);
+
+	return res;
+}
+
+bool ras_fw_eeprom_check_safety_watermark(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	bool ret = false;
+	int bad_page_count;
+
+	if (!control->record_threshold_config)
+		return false;
+
+	bad_page_count = ras_umc_get_badpage_count(ras_core);
+
+	if (bad_page_count > control->record_threshold_count)
+		RAS_DEV_WARN(ras_core->dev, "RAS records:%d exceed threshold:%d",
+			bad_page_count, control->record_threshold_count);
+
+	if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
+		(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
+		RAS_DEV_WARN(ras_core->dev,
+			"Please consult AMD Service Action Guide (SAG) for appropriate service procedures.\n");
+		ret = false;
+	} else {
+		ras_core->is_rma = true;
+		RAS_DEV_WARN(ras_core->dev,
+			"Please consider adjusting the customized threshold.\n");
+		ret = true;
+	}
+
+	return ret;
+}
+
+int ras_fw_eeprom_append(struct ras_core_context *ras_core,
+			   struct eeprom_umc_record *record, const u32 num)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	int threshold_config = control->record_threshold_config;
+	int i, bad_page_count;
+
+	mutex_lock(&control->ras_tbl_mutex);
+
+	for (i = 0; i < num; i++) {
+		/* update bad channel bitmap */
+		if ((record[i].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) &&
+			!(control->bad_channel_bitmap & (1 << record[i].mem_channel))) {
+			control->bad_channel_bitmap |= 1 << record[i].mem_channel;
+			control->update_channel_flag = true;
+		}
+	}
+	control->ras_num_recs += num;
+
+	bad_page_count = ras_umc_get_badpage_count(ras_core);
+
+	if (threshold_config != 0 &&
+		bad_page_count > control->record_threshold_count) {
+		RAS_DEV_WARN(ras_core->dev,
+			"Saved bad pages %d reaches threshold value %d\n",
+			bad_page_count, control->record_threshold_count);
+
+		if ((threshold_config != WARN_NONSTOP_OVER_THRESHOLD) &&
+			(threshold_config != NONSTOP_OVER_THRESHOLD))
+			ras_core->is_rma = true;
+
+		/* ignore the -ENOTSUPP return value */
+		ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
+	}
+
+	mutex_unlock(&control->ras_tbl_mutex);
+	return 0;
+}
+
+int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
+			 struct eeprom_umc_record *record_umc,
+			 struct ras_bank_ecc *ras_ecc,
+			 u32 rec_idx, const u32 num)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	int i, ret, end_idx;
+	u64 mca, ipid, ts;
+
+	if (!ras_core->ras_umc.ip_func ||
+	    !ras_core->ras_umc.ip_func->mca_ipid_parse)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&control->ras_tbl_mutex);
+
+	end_idx = rec_idx + num;
+	for (i = rec_idx; i < end_idx; i++) {
+		ret = ras_fw_get_badpage_mca_addr(ras_core, i, &mca);
+		if (ret)
+			goto out;
+
+		ret = ras_fw_get_badpage_ipid(ras_core, i, &ipid);
+		if (ret)
+			goto out;
+
+		ret = ras_fw_get_timestamp(ras_core, i, &ts);
+		if (ret)
+			goto out;
+
+		if (record_umc) {
+			record_umc[i - rec_idx].address = mca;
+			/* retired_page (pa) is unused now */
+			record_umc[i - rec_idx].retired_row_pfn = 0x1ULL;
+			record_umc[i - rec_idx].ts = ts;
+			record_umc[i - rec_idx].err_type = RAS_EEPROM_ERR_NON_RECOVERABLE;
+
+			ras_core->ras_umc.ip_func->mca_ipid_parse(ras_core, ipid,
+				(uint32_t *)&(record_umc[i - rec_idx].cu),
+				(uint32_t *)&(record_umc[i - rec_idx].mem_channel),
+				(uint32_t *)&(record_umc[i - rec_idx].mcumc_id), NULL);
+
+			/* update bad channel bitmap */
+			if ((record_umc[i - rec_idx].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) &&
+				!(control->bad_channel_bitmap & (1 << record_umc[i - rec_idx].mem_channel))) {
+				control->bad_channel_bitmap |= 1 << record_umc[i - rec_idx].mem_channel;
+				control->update_channel_flag = true;
+			}
+		}
+
+		if (ras_ecc) {
+			ras_ecc[i - rec_idx].addr = mca;
+			ras_ecc[i - rec_idx].ipid = ipid;
+			ras_ecc[i - rec_idx].ts = ts;
+		}
+
+	}
+
+out:
+	mutex_unlock(&control->ras_tbl_mutex);
+	return ret;
+}
+
+uint32_t ras_fw_eeprom_get_record_count(struct ras_core_context *ras_core)
+{
+	if (!ras_core)
+		return 0;
+
+	return ras_core->ras_fw_eeprom.ras_num_recs;
+}
+
+int ras_fw_eeprom_update_record(struct ras_core_context *ras_core,
+				struct ras_bank_ecc *ras_ecc)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	int ret, retry = 20;
+	u32 recs_num_new = control->ras_num_recs;
+
+	do {
+		/* 1000ms timeout is long enough, smu_get_badpage_count won't
+		 * return -EBUSY before timeout.
+		 */
+		ret = ras_fw_get_badpage_count(ras_core,
+			&recs_num_new, RAS_SMU_MESSAGE_TIMEOUT_MS);
+		if (!ret &&
+		    (recs_num_new == control->ras_num_recs)) {
+			/* record number update in PMFW needs some time,
+			 * smu_get_badpage_count may return immediately without
+			 * count update, sleep for a while and retry again.
+			 */
+			msleep(50);
+			retry--;
+		} else {
+			break;
+		}
+	} while (retry);
+
+	if (ret)
+		return ret;
+
+	if (recs_num_new > control->ras_num_recs)
+		ret = ras_fw_eeprom_read_idx(ras_core, 0,
+					ras_ecc, control->ras_num_recs, 1);
+	else
+		ret = -EINVAL;
+
+	return ret;
+}
+
+static int __check_ras_fw_table_status(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	uint64_t local_time;
+	int res;
+
+	mutex_init(&control->ras_tbl_mutex);
+
+	res = ras_fw_get_table_version(ras_core, &(control->version));
+	if (res)
+		return res;
+
+	res = ras_fw_get_badpage_count(ras_core, &(control->ras_num_recs), 100);
+	if (res)
+		return res;
+
+	local_time = (uint64_t)ktime_get_real_seconds();
+	res = ras_fw_set_timestamp(ras_core, local_time);
+	if (res)
+		return res;
+
+	control->ras_max_record_count = 4000;
+
+
+	if (control->ras_num_recs > control->ras_max_record_count) {
+		RAS_DEV_ERR(ras_core->dev,
+			"RAS header invalid, records in header: %u max allowed :%u",
+			control->ras_num_recs, control->ras_max_record_count);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int ras_fw_eeprom_hw_init(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control;
+	struct ras_eeprom_config *eeprom_cfg;
+	struct ras_mp1 *mp1;
+	const struct ras_mp1_sys_func *sys_func;
+
+	if (!ras_core)
+		return -EINVAL;
+
+	mp1 = &ras_core->ras_mp1;
+	sys_func = mp1->sys_func;
+
+	if (!sys_func || !sys_func->mp1_send_eeprom_msg)
+		return -EINVAL;
+
+	ras_core->is_rma = false;
+
+	control = &ras_core->ras_fw_eeprom;
+
+	memset(control, 0, sizeof(*control));
+
+	eeprom_cfg = &ras_core->config->eeprom_cfg;
+	control->record_threshold_config =
+		eeprom_cfg->eeprom_record_threshold_config;
+
+	control->record_threshold_count = 4000;
+	if (eeprom_cfg->eeprom_record_threshold_count <
+		control->record_threshold_count)
+		control->record_threshold_count =
+			eeprom_cfg->eeprom_record_threshold_count;
+
+	control->update_channel_flag = false;
+
+	return __check_ras_fw_table_status(ras_core);
+}
+
+int ras_fw_eeprom_hw_fini(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control;
+
+	if (!ras_core)
+		return -EINVAL;
+
+	control = &ras_core->ras_fw_eeprom;
+	mutex_destroy(&control->ras_tbl_mutex);
+
+	return 0;
+}
+
+int ras_fw_eeprom_check_storage_status(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	int bad_page_count;
+
+	bad_page_count = ras_umc_get_badpage_count(ras_core);
+
+	if ((control->record_threshold_count < bad_page_count) &&
+	    (control->record_threshold_config != 0)) {
+		RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed threshold:%d",
+				bad_page_count, control->record_threshold_count);
+		if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
+			(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
+			RAS_DEV_WARN(ras_core->dev,
+			"Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n");
+		} else {
+			ras_core->is_rma = true;
+			RAS_DEV_ERR(ras_core->dev,
+			"User defined threshold is set, runtime service will be halt when threshold is reached\n");
+		}
+		return 0;
+	}
+
+	RAS_DEV_INFO(ras_core->dev,
+			"Found existing EEPROM table with %d records\n",
+			bad_page_count);
+	/* Warn if we are at 90% of the threshold or above
+	 */
+	if (10 * bad_page_count >= 9 * control->record_threshold_count)
+		RAS_DEV_WARN(ras_core->dev,
+			"RAS records:%u exceeds 90%% of threshold:%d\n",
+			bad_page_count,
+			control->record_threshold_count);
+
+	return 0;
+}
+
+enum ras_gpu_health_status
+	ras_fw_eeprom_check_gpu_status(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+
+	if (!control->record_threshold_config)
+		return RAS_GPU_HEALTH_NONE;
+
+	if (ras_core->is_rma)
+		return RAS_GPU_RETIRED__ECC_REACH_THRESHOLD;
+
+	return RAS_GPU_HEALTH_USABLE;
+}
+
+void ras_fw_eeprom_sync_info(struct ras_core_context *ras_core)
+{
+	struct ras_fw_eeprom_control *control;
+
+	if (!ras_core)
+		return;
+
+	control = &ras_core->ras_fw_eeprom;
+	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
+		&control->ras_num_recs);
+	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
+		&control->bad_channel_bitmap);
+}
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.h b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.h
new file mode 100644
index 0000000000000..762345be075c2
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RAS_EEPROM_FW_H__
+#define __RAS_EEPROM_FW_H__
+
+struct ras_fw_eeprom_control {
+	uint32_t version;
+	/* record threshold */
+	int record_threshold_config;
+	uint32_t record_threshold_count;
+	bool update_channel_flag;
+
+	/* Number of records in the table.
+	 */
+	u32 ras_num_recs;
+
+	/* Maximum possible number of records
+	 * we could store, i.e. the maximum capacity
+	 * of the table.
+	 */
+	u32 ras_max_record_count;
+
+	/* Protect table access via this mutex.
+	 */
+	struct mutex ras_tbl_mutex;
+
+	/* Record channel info which occurred bad pages
+	 */
+	u32 bad_channel_bitmap;
+};
+
+void ras_fw_init_feature_flags(struct ras_core_context *ras_core);
+bool ras_fw_eeprom_supported(struct ras_core_context *ras_core);
+int ras_fw_get_table_version(struct ras_core_context *ras_core,
+				     uint32_t *table_version);
+int ras_fw_get_badpage_count(struct ras_core_context *ras_core,
+				     uint32_t *count, uint32_t timeout);
+int ras_fw_get_badpage_mca_addr(struct ras_core_context *ras_core,
+					uint16_t index, uint64_t *mca_addr);
+int ras_fw_set_timestamp(struct ras_core_context *ras_core,
+				 uint64_t timestamp);
+int ras_fw_get_timestamp(struct ras_core_context *ras_core,
+				 uint16_t index, uint64_t *timestamp);
+int ras_fw_get_badpage_ipid(struct ras_core_context *ras_core,
+				    uint16_t index, uint64_t *ipid);
+int ras_fw_erase_ras_table(struct ras_core_context *ras_core,
+				   uint32_t *result);
+int ras_fw_eeprom_reset_table(struct ras_core_context *ras_core);
+bool ras_fw_eeprom_check_safety_watermark(struct ras_core_context *ras_core);
+int ras_fw_eeprom_append(struct ras_core_context *ras_core,
+			   struct eeprom_umc_record *record, const u32 num);
+int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
+			 struct eeprom_umc_record *record_umc,
+			 struct ras_bank_ecc *ras_ecc,
+			 u32 rec_idx, const u32 num);
+uint32_t ras_fw_eeprom_get_record_count(struct ras_core_context *ras_core);
+int ras_fw_eeprom_update_record(struct ras_core_context *ras_core,
+				struct ras_bank_ecc *ras_ecc);
+int ras_fw_eeprom_hw_init(struct ras_core_context *ras_core);
+int ras_fw_eeprom_hw_fini(struct ras_core_context *ras_core);
+int ras_fw_eeprom_check_storage_status(struct ras_core_context *ras_core);
+enum ras_gpu_health_status
+	ras_fw_eeprom_check_gpu_status(struct ras_core_context *ras_core);
+void ras_fw_eeprom_sync_info(struct ras_core_context *ras_core);
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
index 2abe8553e479e..23118f41eb96f 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
@@ -448,17 +448,27 @@ int ras_umc_load_bad_pages(struct ras_core_context *ras_core)
 	uint32_t ras_num_recs;
 	int ret;
 
-	ras_num_recs = ras_eeprom_get_record_count(ras_core);
-	/* no bad page record, skip eeprom access */
-	if (!ras_num_recs ||
-	    ras_core->ras_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE)
-		return 0;
+	if (ras_fw_eeprom_supported(ras_core)) {
+		ras_num_recs = ras_fw_eeprom_get_record_count(ras_core);
+		/* no bad page record, skip eeprom access */
+		if (!ras_num_recs ||
+		    ras_core->ras_fw_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE)
+			return 0;
+	} else {
+		ras_num_recs = ras_eeprom_get_record_count(ras_core);
+		if (!ras_num_recs ||
+		    ras_core->ras_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE)
+			return 0;
+	}
 
 	bps = kzalloc_objs(*bps, ras_num_recs);
 	if (!bps)
 		return -ENOMEM;
 
-	ret = ras_eeprom_read(ras_core, bps, ras_num_recs);
+	if (ras_fw_eeprom_supported(ras_core))
+		ret = ras_fw_eeprom_read_idx(ras_core, bps, 0, 0, ras_num_recs);
+	else
+		ret = ras_eeprom_read(ras_core, bps, ras_num_recs);
 	if (ret) {
 		RAS_DEV_ERR(ras_core->dev, "Failed to load EEPROM table records!");
 	} else {
@@ -486,14 +496,21 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 	if (!data->bps)
 		return 0;
 
-	eeprom_record_num = ras_eeprom_get_record_count(ras_core);
+	if (ras_fw_eeprom_supported(ras_core))
+		eeprom_record_num = ras_fw_eeprom_get_record_count(ras_core);
+	else
+		eeprom_record_num = ras_eeprom_get_record_count(ras_core);
 	mutex_lock(&ras_umc->umc_lock);
 	save_count = data->count - eeprom_record_num;
 	/* only new entries are saved */
 	if (save_count > 0) {
-		if (ras_eeprom_append(ras_core,
-					   &data->bps[eeprom_record_num],
-					   save_count)) {
+		if (ras_fw_eeprom_supported(ras_core))
+			ret = ras_fw_eeprom_append(ras_core, &data->bps[eeprom_record_num],
+					save_count);
+		else
+			ret = ras_eeprom_append(ras_core, &data->bps[eeprom_record_num],
+					save_count);
+		if (ret) {
 			RAS_DEV_ERR(ras_core->dev, "Failed to save EEPROM table data!");
 			ret = -EIO;
 			goto exit;
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.h b/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
index 3aeb04977d53b..1d3026be509b8 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
@@ -108,6 +108,8 @@ struct ras_umc_ip_func {
 			struct umc_bank_addr bank_addr, uint64_t *soc_pa);
 	int (*soc_pa_to_bank)(struct ras_core_context *ras_core,
 			uint64_t soc_pa, struct umc_bank_addr *bank_addr);
+	void (*mca_ipid_parse)(struct ras_core_context *ras_core, uint64_t ipid,
+			uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid);
 };
 
 struct eeprom_store_record {
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.c
index 5d9a11c17a864..b809a2f21d731 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.c
@@ -373,6 +373,9 @@ static int umc_v12_0_bank_to_eeprom_record(struct ras_core_context *ras_core,
 		ACA_ADDR_2_ERR_ADDR(bank->addr), ACA_IPID_2_UMC_INST(bank->ipid),
 		&nps_addr, bank->nps, record);
 
+	if (ras_fw_eeprom_supported(ras_core) && bank->ts)
+		record->ts = bank->ts;
+
 	lookup_bad_pages_in_a_row(ras_core, record,
 		bank->nps, NULL, 0, bank->seq_no, true);
 
@@ -413,7 +416,7 @@ static int umc_v12_0_eeprom_record_to_nps_record(struct ras_core_context *ras_co
 	uint64_t pa = 0;
 	int ret = 0;
 
-	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record)) {
+	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record) && !ras_fw_eeprom_supported(ras_core)) {
 		record->cur_nps_retired_row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
 	} else {
 		ret = convert_eeprom_record_to_nps_addr(ras_core,
@@ -501,11 +504,25 @@ static int umc_12_0_bank_to_soc_pa(struct ras_core_context *ras_core,
 	return 0;
 }
 
+static void umc_v12_0_mca_ipid_parse(struct ras_core_context *ras_core, uint64_t ipid,
+		uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid)
+{
+	if (did)
+		*did = ACA_IPID_2_DIE_ID(ipid);
+	if (ch)
+		*ch = ACA_IPID_2_UMC_CH(ipid);
+	if (umc_inst)
+		*umc_inst = ACA_IPID_2_UMC_INST(ipid);
+	if (sid)
+		*sid = ACA_IPID_2_SOCKET_ID(ipid);
+}
+
 const struct ras_umc_ip_func ras_umc_func_v12_0 = {
 	.bank_to_eeprom_record = umc_v12_0_bank_to_eeprom_record,
 	.eeprom_record_to_nps_record = umc_v12_0_eeprom_record_to_nps_record,
 	.eeprom_record_to_nps_pages = umc_v12_0_eeprom_record_to_nps_pages,
 	.bank_to_soc_pa = umc_12_0_bank_to_soc_pa,
 	.soc_pa_to_bank = umc_12_0_soc_pa_to_bank,
+	.mca_ipid_parse = umc_v12_0_mca_ipid_parse,
 };
 
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
index 6ee909f8d5349..50e86f352838f 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
@@ -4,6 +4,8 @@
  * Author: James.Qian.Wang <james.qian.wang@arm.com>
  *
  */
+#include <linux/overflow.h>
+
 #include <drm/drm_device.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_gem.h>
@@ -93,7 +95,9 @@ komeda_fb_afbc_size_check(struct komeda_fb *kfb, struct drm_file *file,
 	kfb->afbc_size = kfb->offset_payload + n_blocks *
 			 ALIGN(bpp * AFBC_SUPERBLK_PIXELS / 8,
 			       AFBC_SUPERBLK_ALIGNMENT);
-	min_size = kfb->afbc_size + fb->offsets[0];
+	if (check_add_overflow(kfb->afbc_size, fb->offsets[0], &min_size)) {
+		goto check_failed;
+	}
 	if (min_size > obj->size) {
 		DRM_DEBUG_KMS("afbc size check failed, obj_size: 0x%zx. min_size 0x%llx.\n",
 			      obj->size, min_size);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
index ac8725e248537..37b9e92202443 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
@@ -128,6 +128,8 @@ struct komeda_component {
 	const struct komeda_component_funcs *funcs;
 };
 
+#define to_component(o)	container_of(o, struct komeda_component, obj)
+
 /**
  * struct komeda_component_output
  *
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c b/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
index ee57e306bf7b0..77b3f361091fc 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
@@ -40,7 +40,24 @@ komeda_layer_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(st);
 }
 
+static struct drm_private_state *
+komeda_layer_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_layer_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_layer_obj_funcs = {
+	.atomic_create_state	= komeda_layer_atomic_create_state,
 	.atomic_duplicate_state	= komeda_layer_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_layer_atomic_destroy_state,
 };
@@ -48,14 +65,7 @@ static const struct drm_private_state_funcs komeda_layer_obj_funcs = {
 static int komeda_layer_obj_add(struct komeda_kms_dev *kms,
 				struct komeda_layer *layer)
 {
-	struct komeda_layer_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &layer->base;
-	drm_atomic_private_obj_init(&kms->base, &layer->base.obj, &st->base.obj,
+	drm_atomic_private_obj_init(&kms->base, &layer->base.obj,
 				    &komeda_layer_obj_funcs);
 	return 0;
 }
@@ -82,7 +92,24 @@ komeda_scaler_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_scaler_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_scaler_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_scaler_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_scaler_obj_funcs = {
+	.atomic_create_state	= komeda_scaler_atomic_create_state,
 	.atomic_duplicate_state	= komeda_scaler_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_scaler_atomic_destroy_state,
 };
@@ -90,15 +117,8 @@ static const struct drm_private_state_funcs komeda_scaler_obj_funcs = {
 static int komeda_scaler_obj_add(struct komeda_kms_dev *kms,
 				 struct komeda_scaler *scaler)
 {
-	struct komeda_scaler_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &scaler->base;
 	drm_atomic_private_obj_init(&kms->base,
-				    &scaler->base.obj, &st->base.obj,
+				    &scaler->base.obj,
 				    &komeda_scaler_obj_funcs);
 	return 0;
 }
@@ -125,7 +145,24 @@ komeda_compiz_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_compiz_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_compiz_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_compiz_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_compiz_obj_funcs = {
+	.atomic_create_state	= komeda_compiz_atomic_create_state,
 	.atomic_duplicate_state	= komeda_compiz_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_compiz_atomic_destroy_state,
 };
@@ -133,14 +170,7 @@ static const struct drm_private_state_funcs komeda_compiz_obj_funcs = {
 static int komeda_compiz_obj_add(struct komeda_kms_dev *kms,
 				 struct komeda_compiz *compiz)
 {
-	struct komeda_compiz_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &compiz->base;
-	drm_atomic_private_obj_init(&kms->base, &compiz->base.obj, &st->base.obj,
+	drm_atomic_private_obj_init(&kms->base, &compiz->base.obj,
 				    &komeda_compiz_obj_funcs);
 
 	return 0;
@@ -168,7 +198,24 @@ komeda_splitter_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_splitter_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_splitter_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_splitter_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_splitter_obj_funcs = {
+	.atomic_create_state	= komeda_splitter_atomic_create_state,
 	.atomic_duplicate_state	= komeda_splitter_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_splitter_atomic_destroy_state,
 };
@@ -176,15 +223,8 @@ static const struct drm_private_state_funcs komeda_splitter_obj_funcs = {
 static int komeda_splitter_obj_add(struct komeda_kms_dev *kms,
 				   struct komeda_splitter *splitter)
 {
-	struct komeda_splitter_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &splitter->base;
 	drm_atomic_private_obj_init(&kms->base,
-				    &splitter->base.obj, &st->base.obj,
+				    &splitter->base.obj,
 				    &komeda_splitter_obj_funcs);
 
 	return 0;
@@ -211,7 +251,24 @@ static void komeda_merger_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_merger_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_merger_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_merger_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_merger_obj_funcs = {
+	.atomic_create_state	= komeda_merger_atomic_create_state,
 	.atomic_duplicate_state	= komeda_merger_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_merger_atomic_destroy_state,
 };
@@ -219,15 +276,8 @@ static const struct drm_private_state_funcs komeda_merger_obj_funcs = {
 static int komeda_merger_obj_add(struct komeda_kms_dev *kms,
 				 struct komeda_merger *merger)
 {
-	struct komeda_merger_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &merger->base;
 	drm_atomic_private_obj_init(&kms->base,
-				    &merger->base.obj, &st->base.obj,
+				    &merger->base.obj,
 				    &komeda_merger_obj_funcs);
 
 	return 0;
@@ -255,7 +305,24 @@ komeda_improc_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_improc_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_improc_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_improc_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_improc_obj_funcs = {
+	.atomic_create_state	= komeda_improc_atomic_create_state,
 	.atomic_duplicate_state	= komeda_improc_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_improc_atomic_destroy_state,
 };
@@ -263,14 +330,7 @@ static const struct drm_private_state_funcs komeda_improc_obj_funcs = {
 static int komeda_improc_obj_add(struct komeda_kms_dev *kms,
 				 struct komeda_improc *improc)
 {
-	struct komeda_improc_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &improc->base;
-	drm_atomic_private_obj_init(&kms->base, &improc->base.obj, &st->base.obj,
+	drm_atomic_private_obj_init(&kms->base, &improc->base.obj,
 				    &komeda_improc_obj_funcs);
 
 	return 0;
@@ -298,7 +358,24 @@ komeda_timing_ctrlr_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(to_ctrlr_st(priv_to_comp_st(state)));
 }
 
+static struct drm_private_state *
+komeda_timing_ctrlr_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_timing_ctrlr_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->base.obj);
+	komeda_component_state_reset(&st->base);
+	st->base.component = to_component(obj);
+
+	return &st->base.obj;
+}
+
 static const struct drm_private_state_funcs komeda_timing_ctrlr_obj_funcs = {
+	.atomic_create_state	= komeda_timing_ctrlr_atomic_create_state,
 	.atomic_duplicate_state	= komeda_timing_ctrlr_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_timing_ctrlr_atomic_destroy_state,
 };
@@ -306,14 +383,7 @@ static const struct drm_private_state_funcs komeda_timing_ctrlr_obj_funcs = {
 static int komeda_timing_ctrlr_obj_add(struct komeda_kms_dev *kms,
 				       struct komeda_timing_ctrlr *ctrlr)
 {
-	struct komeda_compiz_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->base.component = &ctrlr->base;
-	drm_atomic_private_obj_init(&kms->base, &ctrlr->base.obj, &st->base.obj,
+	drm_atomic_private_obj_init(&kms->base, &ctrlr->base.obj,
 				    &komeda_timing_ctrlr_obj_funcs);
 
 	return 0;
@@ -342,7 +412,24 @@ komeda_pipeline_atomic_destroy_state(struct drm_private_obj *obj,
 	kfree(priv_to_pipe_st(state));
 }
 
+static struct drm_private_state *
+komeda_pipeline_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct komeda_pipeline_state *st;
+
+	st = kzalloc_obj(*st);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &st->obj);
+	st->active_comps = 0;
+	st->pipe = container_of(obj, struct komeda_pipeline, obj);
+
+	return &st->obj;
+}
+
 static const struct drm_private_state_funcs komeda_pipeline_obj_funcs = {
+	.atomic_create_state	= komeda_pipeline_atomic_create_state,
 	.atomic_duplicate_state	= komeda_pipeline_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_pipeline_atomic_destroy_state,
 };
@@ -350,14 +437,7 @@ static const struct drm_private_state_funcs komeda_pipeline_obj_funcs = {
 static int komeda_pipeline_obj_add(struct komeda_kms_dev *kms,
 				   struct komeda_pipeline *pipe)
 {
-	struct komeda_pipeline_state *st;
-
-	st = kzalloc_obj(*st);
-	if (!st)
-		return -ENOMEM;
-
-	st->pipe = pipe;
-	drm_atomic_private_obj_init(&kms->base, &pipe->obj, &st->obj,
+	drm_atomic_private_obj_init(&kms->base, &pipe->obj,
 				    &komeda_pipeline_obj_funcs);
 
 	return 0;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index d1f5451ebfeaf..97482fc82ec2f 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -566,6 +566,83 @@ static const struct atmel_hlcdc_dc_desc atmel_xlcdc_dc_sam9x75 = {
 	.ops = &atmel_xlcdc_ops,
 };
 
+static const struct atmel_hlcdc_layer_desc atmel_xlcdc_sama7d65_layers[] = {
+	{
+		.name = "base",
+		.formats = &atmel_hlcdc_plane_rgb_formats,
+		.regs_offset = 0x60,
+		.id = 0,
+		.type = ATMEL_HLCDC_BASE_LAYER,
+		.cfgs_offset = 0x1c,
+		.layout = {
+			.xstride = { 2 },
+			.default_color = 3,
+			.general_config = 4,
+			.disc_pos = 5,
+			.disc_size = 6,
+		},
+		.clut_offset = 0x700,
+	},
+	{
+		.name = "overlay1",
+		.formats = &atmel_hlcdc_plane_rgb_formats,
+		.regs_offset = 0x160,
+		.id = 1,
+		.type = ATMEL_HLCDC_OVERLAY_LAYER,
+		.cfgs_offset = 0x1c,
+		.layout = {
+			.pos = 2,
+			.size = 3,
+			.xstride = { 4 },
+			.pstride = { 5 },
+			.default_color = 6,
+			.chroma_key = 7,
+			.chroma_key_mask = 8,
+			.general_config = 9,
+		},
+		.clut_offset = 0xb00,
+	},
+	{
+		.name = "high-end-overlay",
+		.formats = &atmel_hlcdc_plane_rgb_and_yuv_formats,
+		.regs_offset = 0x360,
+		.id = 2,
+		.type = ATMEL_HLCDC_OVERLAY_LAYER,
+		.cfgs_offset = 0x30,
+		.layout = {
+			.pos = 2,
+			.size = 3,
+			.memsize = 4,
+			.xstride = { 5, 7 },
+			.pstride = { 6, 8 },
+			.default_color = 9,
+			.chroma_key = 10,
+			.chroma_key_mask = 11,
+			.general_config = 12,
+			.csc = 16,
+			.scaler_config = 23,
+			.vxs_config = 30,
+			.hxs_config = 31,
+		},
+		.clut_offset = 0x1300,
+	},
+};
+
+static const struct atmel_hlcdc_dc_desc atmel_xlcdc_dc_sama7d65 = {
+	.min_width = 0,
+	.min_height = 0,
+	.max_width = 2048,
+	.max_height = 2048,
+	.max_spw = 0x3ff,
+	.max_vpw = 0x3ff,
+	.max_hpw = 0x3ff,
+	.fixed_clksrc = true,
+	.is_xlcdc = true,
+	.nlayers = ARRAY_SIZE(atmel_xlcdc_sama7d65_layers),
+	.layers = atmel_xlcdc_sama7d65_layers,
+	.ops = &atmel_xlcdc_ops,
+};
+
 static const struct of_device_id atmel_hlcdc_of_match[] = {
 	{
 		.compatible = "atmel,at91sam9n12-hlcdc",
@@ -595,6 +672,10 @@ static const struct of_device_id atmel_hlcdc_of_match[] = {
 		.compatible = "microchip,sam9x75-xlcdc",
 		.data = &atmel_xlcdc_dc_sam9x75,
 	},
+	{
+		.compatible = "microchip,sama7d65-xlcdc",
+		.data = &atmel_xlcdc_dc_sama7d65,
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, atmel_hlcdc_of_match);
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 39385deafc68e..7e4f355743842 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -191,6 +191,16 @@ config DRM_LONTIUM_LT9611UXC
 	  HDMI signals
 	  Please say Y if you have such hardware.
 
+config DRM_LONTIUM_LT8713SX
+	tristate "Lontium LT8713SX DP MST bridge"
+	depends on OF
+	select REGMAP_I2C
+	help
+	  Driver for Lontium LT8713SX DP MST bridge
+	  chip firmware upgrade, which converts Type-C/DP1.4
+	  to 3 configurable Type-C/DP1.4/HDMI2.0 outputs
+	  Please say Y if you have such hardware.
+
 config DRM_ITE_IT66121
 	tristate "ITE IT66121 HDMI bridge"
 	depends on OF
@@ -342,6 +352,16 @@ config DRM_THINE_THC63LVD1024
 	help
 	  Thine THC63LVD1024 LVDS/parallel converter driver.
 
+config DRM_THEAD_TH1520_DW_HDMI
+	tristate "T-Head TH1520 DesignWare HDMI bridge"
+	depends on OF
+	depends on COMMON_CLK
+	depends on ARCH_THEAD || COMPILE_TEST
+	select DRM_DW_HDMI
+	help
+	  Choose this to enable support for the internal HDMI bridge found
+	  on the T-Head TH1520 SoC.
+
 config DRM_TOSHIBA_TC358762
 	tristate "TC358762 DSI/DPI bridge"
 	depends on OF
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index 909c21cc3acd3..beab5b695a6e1 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_DRM_LONTIUM_LT8912B) += lontium-lt8912b.o
 obj-$(CONFIG_DRM_LONTIUM_LT9211) += lontium-lt9211.o
 obj-$(CONFIG_DRM_LONTIUM_LT9611) += lontium-lt9611.o
 obj-$(CONFIG_DRM_LONTIUM_LT9611UXC) += lontium-lt9611uxc.o
+obj-$(CONFIG_DRM_LONTIUM_LT8713SX) += lontium-lt8713sx.o
 obj-$(CONFIG_DRM_LVDS_CODEC) += lvds-codec.o
 obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v3-fw.o
 obj-$(CONFIG_DRM_MICROCHIP_LVDS_SERIALIZER) += microchip-lvds.o
@@ -29,6 +30,7 @@ obj-$(CONFIG_DRM_SII902X) += sii902x.o
 obj-$(CONFIG_DRM_SII9234) += sii9234.o
 obj-$(CONFIG_DRM_SIMPLE_BRIDGE) += simple-bridge.o
 obj-$(CONFIG_DRM_SOLOMON_SSD2825) += ssd2825.o
+obj-$(CONFIG_DRM_THEAD_TH1520_DW_HDMI) += th1520-dw-hdmi.o
 obj-$(CONFIG_DRM_THINE_THC63LVD1024) += thc63lvd1024.o
 obj-$(CONFIG_DRM_TOSHIBA_TC358762) += tc358762.o
 obj-$(CONFIG_DRM_TOSHIBA_TC358764) += tc358764.o
diff --git a/drivers/gpu/drm/bridge/analogix/Kconfig b/drivers/gpu/drm/bridge/analogix/Kconfig
index 4846b2e9be7c2..03dc7ffe824a9 100644
--- a/drivers/gpu/drm/bridge/analogix/Kconfig
+++ b/drivers/gpu/drm/bridge/analogix/Kconfig
@@ -34,6 +34,8 @@ config DRM_ANALOGIX_ANX7625
 	tristate "Analogix Anx7625 MIPI to DP interface support"
 	depends on DRM
 	depends on OF
+	depends on TYPEC
+	depends on USB_ROLE_SWITCH
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HDCP_HELPER
 	select DRM_DISPLAY_HELPER
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index efe534977d12d..b1687a4aa0473 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -1177,12 +1177,88 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp)
 	return ret;
 }
 
+static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge,
+					const struct drm_display_mode *mode)
+{
+	struct analogix_dp_device *dp = to_dp(bridge);
+	struct drm_display_info *display_info = &dp->connector.display_info;
+	struct video_info *video = &dp->video_info;
+	struct device_node *dp_node = dp->dev->of_node;
+	int vic;
+
+	/* Input video interlaces & hsync pol & vsync pol */
+	video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
+	video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC);
+	video->h_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NHSYNC);
+
+	/* Input video dynamic_range & colorimetry */
+	vic = drm_match_cea_mode(mode);
+	if ((vic == 6) || (vic == 7) || (vic == 21) || (vic == 22) ||
+	    (vic == 2) || (vic == 3) || (vic == 17) || (vic == 18)) {
+		video->dynamic_range = CEA;
+		video->ycbcr_coeff = COLOR_YCBCR601;
+	} else if (vic) {
+		video->dynamic_range = CEA;
+		video->ycbcr_coeff = COLOR_YCBCR709;
+	} else {
+		video->dynamic_range = VESA;
+		video->ycbcr_coeff = COLOR_YCBCR709;
+	}
+
+	/* Input vide bpc and color_formats */
+	switch (display_info->bpc) {
+	case 12:
+		video->color_depth = COLOR_12;
+		break;
+	case 10:
+		video->color_depth = COLOR_10;
+		break;
+	case 8:
+		video->color_depth = COLOR_8;
+		break;
+	case 6:
+		video->color_depth = COLOR_6;
+		break;
+	default:
+		video->color_depth = COLOR_8;
+		break;
+	}
+	if (display_info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
+		video->color_space = COLOR_YCBCR444;
+	else if (display_info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
+		video->color_space = COLOR_YCBCR422;
+	else
+		video->color_space = COLOR_RGB;
+
+	/*
+	 * NOTE: those property parsing code is used for providing backward
+	 * compatibility for samsung platform.
+	 * Due to we used the "of_property_read_u32" interfaces, when this
+	 * property isn't present, the "video_info" can keep the original
+	 * values and wouldn't be modified.
+	 */
+	of_property_read_u32(dp_node, "samsung,color-space",
+			     &video->color_space);
+	of_property_read_u32(dp_node, "samsung,dynamic-range",
+			     &video->dynamic_range);
+	of_property_read_u32(dp_node, "samsung,ycbcr-coeff",
+			     &video->ycbcr_coeff);
+	of_property_read_u32(dp_node, "samsung,color-depth",
+			     &video->color_depth);
+	if (of_property_read_bool(dp_node, "hsync-active-high"))
+		video->h_sync_polarity = true;
+	if (of_property_read_bool(dp_node, "vsync-active-high"))
+		video->v_sync_polarity = true;
+	if (of_property_read_bool(dp_node, "interlaced"))
+		video->interlaced = true;
+}
+
 static void analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge,
 					     struct drm_atomic_state *old_state)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
 	struct drm_crtc *crtc;
-	struct drm_crtc_state *old_crtc_state;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 	int timeout_loop = 0;
 	int ret;
 
@@ -1190,6 +1266,11 @@ static void analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge,
 	if (!crtc)
 		return;
 
+	new_crtc_state = drm_atomic_get_new_crtc_state(old_state, crtc);
+	if (!new_crtc_state)
+		return;
+	analogix_dp_bridge_mode_set(bridge, &new_crtc_state->adjusted_mode);
+
 	old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc);
 	/* Not a full enable, just disable PSR and continue */
 	if (old_crtc_state && old_crtc_state->self_refresh_active) {
@@ -1296,83 +1377,6 @@ static void analogix_dp_bridge_atomic_post_disable(struct drm_bridge *bridge,
 		DRM_ERROR("Failed to enable psr (%d)\n", ret);
 }
 
-static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge,
-				const struct drm_display_mode *orig_mode,
-				const struct drm_display_mode *mode)
-{
-	struct analogix_dp_device *dp = to_dp(bridge);
-	struct drm_display_info *display_info = &dp->connector.display_info;
-	struct video_info *video = &dp->video_info;
-	struct device_node *dp_node = dp->dev->of_node;
-	int vic;
-
-	/* Input video interlaces & hsync pol & vsync pol */
-	video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
-	video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC);
-	video->h_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NHSYNC);
-
-	/* Input video dynamic_range & colorimetry */
-	vic = drm_match_cea_mode(mode);
-	if ((vic == 6) || (vic == 7) || (vic == 21) || (vic == 22) ||
-	    (vic == 2) || (vic == 3) || (vic == 17) || (vic == 18)) {
-		video->dynamic_range = CEA;
-		video->ycbcr_coeff = COLOR_YCBCR601;
-	} else if (vic) {
-		video->dynamic_range = CEA;
-		video->ycbcr_coeff = COLOR_YCBCR709;
-	} else {
-		video->dynamic_range = VESA;
-		video->ycbcr_coeff = COLOR_YCBCR709;
-	}
-
-	/* Input vide bpc and color_formats */
-	switch (display_info->bpc) {
-	case 12:
-		video->color_depth = COLOR_12;
-		break;
-	case 10:
-		video->color_depth = COLOR_10;
-		break;
-	case 8:
-		video->color_depth = COLOR_8;
-		break;
-	case 6:
-		video->color_depth = COLOR_6;
-		break;
-	default:
-		video->color_depth = COLOR_8;
-		break;
-	}
-	if (display_info->color_formats & DRM_COLOR_FORMAT_YCBCR444)
-		video->color_space = COLOR_YCBCR444;
-	else if (display_info->color_formats & DRM_COLOR_FORMAT_YCBCR422)
-		video->color_space = COLOR_YCBCR422;
-	else
-		video->color_space = COLOR_RGB;
-
-	/*
-	 * NOTE: those property parsing code is used for providing backward
-	 * compatibility for samsung platform.
-	 * Due to we used the "of_property_read_u32" interfaces, when this
-	 * property isn't present, the "video_info" can keep the original
-	 * values and wouldn't be modified.
-	 */
-	of_property_read_u32(dp_node, "samsung,color-space",
-			     &video->color_space);
-	of_property_read_u32(dp_node, "samsung,dynamic-range",
-			     &video->dynamic_range);
-	of_property_read_u32(dp_node, "samsung,ycbcr-coeff",
-			     &video->ycbcr_coeff);
-	of_property_read_u32(dp_node, "samsung,color-depth",
-			     &video->color_depth);
-	if (of_property_read_bool(dp_node, "hsync-active-high"))
-		video->h_sync_polarity = true;
-	if (of_property_read_bool(dp_node, "vsync-active-high"))
-		video->v_sync_polarity = true;
-	if (of_property_read_bool(dp_node, "interlaced"))
-		video->interlaced = true;
-}
-
 static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
@@ -1381,7 +1385,6 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
 	.atomic_enable = analogix_dp_bridge_atomic_enable,
 	.atomic_disable = analogix_dp_bridge_atomic_disable,
 	.atomic_post_disable = analogix_dp_bridge_atomic_post_disable,
-	.mode_set = analogix_dp_bridge_mode_set,
 	.attach = analogix_dp_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index b86e93f30ed6a..91b215c6a0cfb 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -156,7 +156,7 @@ struct analogix_dp_device {
 	struct drm_device	*drm_dev;
 	struct drm_connector	connector;
 	struct drm_bridge	bridge;
-	struct drm_dp_aux       aux;
+	struct drm_dp_aux	aux;
 	struct clk		*clock;
 	unsigned int		irq;
 	void __iomem		*reg_base;
@@ -166,7 +166,7 @@ struct analogix_dp_device {
 	struct phy		*phy;
 	int			dpms_mode;
 	struct gpio_desc	*hpd_gpiod;
-	bool                    force_hpd;
+	bool			force_hpd;
 	bool			fast_train_enable;
 	bool			psr_supported;
 
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
index 4e49e4f28d552..54b02242d6db4 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
@@ -3,6 +3,7 @@
  * Copyright(c) 2020, Analogix Semiconductor. All rights reserved.
  *
  */
+#include <linux/cleanup.h>
 #include <linux/gcd.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
@@ -15,6 +16,9 @@
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/usb.h>
+#include <linux/usb/pd.h>
+#include <linux/usb/role.h>
 #include <linux/workqueue.h>
 
 #include <linux/of_graph.h>
@@ -1325,7 +1329,7 @@ static int anx7625_read_hpd_gpio_config_status(struct anx7625_data *ctx)
 static void anx7625_disable_pd_protocol(struct anx7625_data *ctx)
 {
 	struct device *dev = ctx->dev;
-	int ret, val;
+	int ret;
 
 	/* Reset main ocm */
 	ret = anx7625_reg_write(ctx, ctx->i2c.rx_p0_client, 0x88, 0x40);
@@ -1339,6 +1343,11 @@ static void anx7625_disable_pd_protocol(struct anx7625_data *ctx)
 		DRM_DEV_DEBUG_DRIVER(dev, "disable PD feature fail.\n");
 	else
 		DRM_DEV_DEBUG_DRIVER(dev, "disable PD feature succeeded.\n");
+}
+
+static void anx7625_configure_hpd(struct anx7625_data *ctx)
+{
+	int val;
 
 	/*
 	 * Make sure the HPD GPIO already be configured after OCM release before
@@ -1354,6 +1363,18 @@ static void anx7625_disable_pd_protocol(struct anx7625_data *ctx)
 	anx7625_hpd_timer_config(ctx);
 }
 
+static bool anx7625_need_pd(struct anx7625_data *ctx)
+{
+	struct fwnode_handle *fwnode;
+
+	fwnode = device_get_named_child_node(ctx->dev, "connector");
+	if (!fwnode)
+		return false;
+
+	fwnode_handle_put(fwnode);
+	return true;
+}
+
 static int anx7625_ocm_loading_check(struct anx7625_data *ctx)
 {
 	int ret;
@@ -1369,7 +1390,9 @@ static int anx7625_ocm_loading_check(struct anx7625_data *ctx)
 	if ((ret & FLASH_LOAD_STA_CHK) != FLASH_LOAD_STA_CHK)
 		return -ENODEV;
 
-	anx7625_disable_pd_protocol(ctx);
+	if (!anx7625_need_pd(ctx))
+		anx7625_disable_pd_protocol(ctx);
+	anx7625_configure_hpd(ctx);
 
 	DRM_DEV_DEBUG_DRIVER(dev, "Firmware ver %02x%02x,",
 			     anx7625_reg_read(ctx,
@@ -1472,6 +1495,161 @@ static void anx7625_start_dp_work(struct anx7625_data *ctx)
 	DRM_DEV_DEBUG_DRIVER(dev, "Secure OCM version=%02x\n", ret);
 }
 
+static u8 anx7625_checksum(u8 *buf, u8 len)
+{
+	u8 ret = 0;
+	u8 i;
+
+	for (i = 0; i < len; i++)
+		ret += buf[i];
+
+	return ret;
+}
+
+static int anx7625_read_msg_ctrl_status(struct anx7625_data *ctx)
+{
+	return anx7625_reg_read(ctx, ctx->i2c.rx_p0_client, CMD_SEND_BUF);
+}
+
+static int anx7625_wait_msg_empty(struct anx7625_data *ctx)
+{
+	int val;
+
+	return readx_poll_timeout(anx7625_read_msg_ctrl_status, ctx,
+				  val, (val < 0) || (val == 0),
+				  2000, 2000 * 150);
+}
+
+static int anx7625_send_msg(struct anx7625_data *ctx, u8 type, u8 *buf, u8 size)
+{
+	struct fw_msg *msg = &ctx->send_msg;
+	u8 crc;
+	int ret;
+
+	size = min_t(u8, size, (u8)MAX_BUF_LEN);
+	memcpy(msg->buf, buf, size);
+	msg->msg_type = type;
+
+	/* msg len equals buffer length + msg_type */
+	msg->msg_len = size + 1;
+
+	crc = anx7625_checksum((u8 *)msg, size + HEADER_LEN);
+	msg->buf[size] = 0 - crc;
+
+	ret = anx7625_wait_msg_empty(ctx);
+	if (ret)
+		return ret;
+
+	ret = anx7625_reg_block_write(ctx, ctx->i2c.rx_p0_client,
+				      CMD_SEND_BUF + 1, size + HEADER_LEN,
+				      &msg->msg_type);
+	ret |= anx7625_reg_write(ctx, ctx->i2c.rx_p0_client, CMD_SEND_BUF,
+				 msg->msg_len);
+	return ret;
+}
+
+static int anx7625_typec_dr_set(struct typec_port *port, enum typec_data_role role)
+{
+	struct anx7625_data *ctx = typec_get_drvdata(port);
+
+	if (role == ctx->typec_data_role)
+		return 0;
+
+	return anx7625_send_msg(ctx, 0x11, NULL, 0);
+}
+
+static const struct typec_operations anx7625_typec_ops = {
+	.dr_set = anx7625_typec_dr_set,
+};
+
+static void anx7625_typec_set_orientation(struct anx7625_data *ctx)
+{
+	u32 val = anx7625_reg_read(ctx, ctx->i2c.rx_p0_client, SYSTEM_STSTUS);
+
+	if (val & (CC1_RP | CC1_RD))
+		typec_set_orientation(ctx->typec_port, TYPEC_ORIENTATION_NORMAL);
+	else if (val & (CC2_RP | CC2_RD))
+		typec_set_orientation(ctx->typec_port, TYPEC_ORIENTATION_REVERSE);
+	else
+		typec_set_orientation(ctx->typec_port, TYPEC_ORIENTATION_NONE);
+}
+
+static void anx7625_typec_set_status(struct anx7625_data *ctx,
+				     unsigned int intr_status,
+				     unsigned int intr_vector)
+{
+	if (!ctx->typec_port)
+		return;
+
+	if (intr_vector & CC_STATUS)
+		anx7625_typec_set_orientation(ctx);
+	if (intr_vector & DATA_ROLE_STATUS) {
+		enum typec_data_role data_role = (intr_status & DATA_ROLE_STATUS) ?
+			TYPEC_HOST : TYPEC_DEVICE;
+		usb_role_switch_set_role(ctx->role_sw,
+					 (intr_status & DATA_ROLE_STATUS) ?
+					 USB_ROLE_HOST : USB_ROLE_DEVICE);
+		typec_set_data_role(ctx->typec_port, data_role);
+		ctx->typec_data_role = data_role;
+	}
+	if (intr_vector & VBUS_STATUS)
+		typec_set_pwr_role(ctx->typec_port,
+				   (intr_status & VBUS_STATUS) ?
+				   TYPEC_SOURCE : TYPEC_SINK);
+	if (intr_vector & VCONN_STATUS)
+		typec_set_vconn_role(ctx->typec_port,
+				     (intr_status & VCONN_STATUS) ?
+				     TYPEC_SOURCE : TYPEC_SINK);
+}
+
+static int anx7625_typec_register(struct anx7625_data *ctx)
+{
+	struct typec_capability typec_cap = { };
+	struct fwnode_handle *fwnode __free(fwnode_handle) =
+		device_get_named_child_node(ctx->dev, "connector");
+	u32 val;
+	int ret;
+
+	if (!fwnode)
+		return 0;
+
+	ret = typec_get_fw_cap(&typec_cap, fwnode);
+	if (ret < 0)
+		return ret;
+
+	typec_cap.revision = 0x0120;
+	typec_cap.pd_revision = 0x0300;
+	typec_cap.usb_capability = USB_CAPABILITY_USB2 | USB_CAPABILITY_USB3;
+	typec_cap.orientation_aware = true;
+
+	typec_cap.driver_data = ctx;
+	typec_cap.ops = &anx7625_typec_ops;
+
+	ctx->typec_port = typec_register_port(ctx->dev, &typec_cap);
+	if (IS_ERR(ctx->typec_port))
+		return PTR_ERR(ctx->typec_port);
+
+	ctx->role_sw = fwnode_usb_role_switch_get(fwnode);
+	if (IS_ERR(ctx->role_sw)) {
+		typec_unregister_port(ctx->typec_port);
+		return PTR_ERR(ctx->role_sw);
+	}
+
+	val = anx7625_reg_read(ctx, ctx->i2c.rx_p0_client, SYSTEM_STSTUS);
+
+	anx7625_typec_set_status(ctx, val,
+				 CC_STATUS | DATA_ROLE_STATUS |
+				 VBUS_STATUS | VCONN_STATUS);
+
+	return 0;
+}
+
+static void anx7625_typec_unregister(struct anx7625_data *ctx)
+{
+	usb_role_switch_put(ctx->role_sw);
+	typec_unregister_port(ctx->typec_port);
+}
+
 static int anx7625_read_hpd_status_p0(struct anx7625_data *ctx)
 {
 	return anx7625_reg_read(ctx, ctx->i2c.rx_p0_client, SYSTEM_STSTUS);
@@ -1566,7 +1744,7 @@ static void dp_hpd_change_handler(struct anx7625_data *ctx, bool on)
 	}
 }
 
-static int anx7625_hpd_change_detect(struct anx7625_data *ctx)
+static int anx7625_intr_status(struct anx7625_data *ctx)
 {
 	int intr_vector, status;
 	struct device *dev = ctx->dev;
@@ -1593,9 +1771,6 @@ static int anx7625_hpd_change_detect(struct anx7625_data *ctx)
 		return status;
 	}
 
-	if (!(intr_vector & HPD_STATUS_CHANGE))
-		return -ENOENT;
-
 	status = anx7625_reg_read(ctx, ctx->i2c.rx_p0_client,
 				  SYSTEM_STSTUS);
 	if (status < 0) {
@@ -1604,6 +1779,12 @@ static int anx7625_hpd_change_detect(struct anx7625_data *ctx)
 	}
 
 	DRM_DEV_DEBUG_DRIVER(dev, "0x7e:0x45=%x\n", status);
+
+	anx7625_typec_set_status(ctx, status, intr_vector);
+
+	if (!(intr_vector & HPD_STATUS))
+		return -ENOENT;
+
 	dp_hpd_change_handler(ctx, status & HPD_STATUS);
 
 	return 0;
@@ -1622,7 +1803,7 @@ static void anx7625_work_func(struct work_struct *work)
 		return;
 	}
 
-	event = anx7625_hpd_change_detect(ctx);
+	event = anx7625_intr_status(ctx);
 
 	mutex_unlock(&ctx->lock);
 
@@ -2741,11 +2922,24 @@ static int anx7625_i2c_probe(struct i2c_client *client)
 	}
 
 	if (!platform->pdata.low_power_mode) {
-		anx7625_disable_pd_protocol(platform);
+		if (!anx7625_need_pd(platform))
+			anx7625_disable_pd_protocol(platform);
+
+		anx7625_configure_hpd(platform);
+
 		pm_runtime_get_sync(dev);
 		_anx7625_hpd_polling(platform, 5000 * 100);
 	}
 
+	if (platform->pdata.intp_irq)
+		anx7625_reg_write(platform, platform->i2c.rx_p0_client,
+				  INTERFACE_CHANGE_INT_MASK, 0);
+
+	/* After getting runtime handle */
+	ret = anx7625_typec_register(platform);
+	if (ret)
+		goto pm_suspend;
+
 	/* Add work function */
 	if (platform->pdata.intp_irq) {
 		enable_irq(platform->pdata.intp_irq);
@@ -2759,6 +2953,10 @@ static int anx7625_i2c_probe(struct i2c_client *client)
 
 	return 0;
 
+pm_suspend:
+	if (!platform->pdata.low_power_mode)
+		pm_runtime_put_sync_suspend(&client->dev);
+
 free_wq:
 	if (platform->workqueue)
 		destroy_workqueue(platform->workqueue);
@@ -2774,6 +2972,8 @@ static void anx7625_i2c_remove(struct i2c_client *client)
 {
 	struct anx7625_data *platform = i2c_get_clientdata(client);
 
+	anx7625_typec_unregister(platform);
+
 	drm_bridge_remove(&platform->bridge);
 
 	if (platform->pdata.intp_irq)
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.h b/drivers/gpu/drm/bridge/analogix/anx7625.h
index eb5580f1ab2f8..957d234ec07c8 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.h
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.h
@@ -51,9 +51,24 @@
 #define INTR_RECEIVED_MSG BIT(5)
 
 #define SYSTEM_STSTUS 0x45
+#define INTERFACE_CHANGE_INT_MASK 0x43
 #define INTERFACE_CHANGE_INT 0x44
-#define HPD_STATUS_CHANGE 0x80
-#define HPD_STATUS 0x80
+#define VCONN_STATUS	BIT(2)
+#define VBUS_STATUS	BIT(3)
+#define CC_STATUS	BIT(4)
+#define DATA_ROLE_STATUS	BIT(5)
+#define HPD_STATUS	BIT(7)
+
+#define NEW_CC_STATUS 0x46
+#define CC1_RD                  BIT(0)
+#define CC1_RA                  BIT(1)
+#define CC1_RP			(BIT(2) | BIT(3))
+#define CC2_RD                  BIT(4)
+#define CC2_RA                  BIT(5)
+#define CC2_RP			(BIT(6) | BIT(7))
+
+#define CMD_SEND_BUF		0xC0
+#define CMD_RECV_BUF		0xE0
 
 /******** END of I2C Address 0x58 ********/
 
@@ -447,9 +462,23 @@ struct anx7625_i2c_client {
 	struct i2c_client *tcpc_client;
 };
 
+struct typec_port;
+struct usb_role_switch;
+
+#define MAX_BUF_LEN	30
+struct fw_msg {
+	u8 msg_len;
+	u8 msg_type;
+	u8 buf[MAX_BUF_LEN];
+} __packed;
+#define HEADER_LEN		2
+
 struct anx7625_data {
 	struct anx7625_platform_data pdata;
 	struct platform_device *audio_pdev;
+	struct typec_port *typec_port;
+	struct usb_role_switch *role_sw;
+	int typec_data_role;
 	int hpd_status;
 	int hpd_high_cnt;
 	int dp_en;
@@ -479,6 +508,7 @@ struct anx7625_data {
 	struct drm_connector *connector;
 	struct mipi_dsi_device *dsi;
 	struct drm_dp_aux aux;
+	struct fw_msg send_msg;
 };
 
 #endif  /* __ANX7625_H__ */
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index 9392c226ff5b1..945bb47c172ba 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -739,12 +739,8 @@ static void cdns_mhdp_fw_cb(const struct firmware *fw, void *context)
 	spin_lock(&mhdp->start_lock);
 	bridge_attached = mhdp->bridge_attached;
 	spin_unlock(&mhdp->start_lock);
-	if (bridge_attached) {
-		if (mhdp->connector.dev)
-			drm_kms_helper_hotplug_event(mhdp->bridge.dev);
-		else
-			drm_bridge_hpd_notify(&mhdp->bridge, cdns_mhdp_detect(mhdp));
-	}
+	if (bridge_attached)
+		drm_bridge_hpd_notify(&mhdp->bridge, cdns_mhdp_detect(mhdp));
 }
 
 static int cdns_mhdp_load_firmware(struct cdns_mhdp_device *mhdp)
@@ -782,7 +778,7 @@ static ssize_t cdns_mhdp_transfer(struct drm_dp_aux *aux,
 			if (!ret)
 				continue;
 
-			dev_err(mhdp->dev,
+			dev_dbg(mhdp->dev,
 				"Failed to write DPCD addr %u\n",
 				msg->address + i);
 
@@ -792,7 +788,7 @@ static ssize_t cdns_mhdp_transfer(struct drm_dp_aux *aux,
 		ret = cdns_mhdp_dpcd_read(mhdp, msg->address,
 					  msg->buffer, msg->size);
 		if (ret) {
-			dev_err(mhdp->dev,
+			dev_dbg(mhdp->dev,
 				"Failed to read DPCD addr %u\n",
 				msg->address);
 
@@ -1444,56 +1440,6 @@ static const struct drm_edid *cdns_mhdp_edid_read(struct cdns_mhdp_device *mhdp,
 	return drm_edid_read_custom(connector, cdns_mhdp_get_edid_block, mhdp);
 }
 
-static int cdns_mhdp_get_modes(struct drm_connector *connector)
-{
-	struct cdns_mhdp_device *mhdp = connector_to_mhdp(connector);
-	const struct drm_edid *drm_edid;
-	int num_modes;
-
-	if (!mhdp->plugged)
-		return 0;
-
-	drm_edid = cdns_mhdp_edid_read(mhdp, connector);
-
-	drm_edid_connector_update(connector, drm_edid);
-
-	if (!drm_edid) {
-		dev_err(mhdp->dev, "Failed to read EDID\n");
-		return 0;
-	}
-
-	num_modes = drm_edid_connector_add_modes(connector);
-	drm_edid_free(drm_edid);
-
-	/*
-	 * HACK: Warn about unsupported display formats until we deal
-	 *       with them correctly.
-	 */
-	if (connector->display_info.color_formats &&
-	    !(connector->display_info.color_formats &
-	      mhdp->display_fmt.color_format))
-		dev_warn(mhdp->dev,
-			 "%s: No supported color_format found (0x%08x)\n",
-			__func__, connector->display_info.color_formats);
-
-	if (connector->display_info.bpc &&
-	    connector->display_info.bpc < mhdp->display_fmt.bpc)
-		dev_warn(mhdp->dev, "%s: Display bpc only %d < %d\n",
-			 __func__, connector->display_info.bpc,
-			 mhdp->display_fmt.bpc);
-
-	return num_modes;
-}
-
-static int cdns_mhdp_connector_detect(struct drm_connector *conn,
-				      struct drm_modeset_acquire_ctx *ctx,
-				      bool force)
-{
-	struct cdns_mhdp_device *mhdp = connector_to_mhdp(conn);
-
-	return cdns_mhdp_detect(mhdp);
-}
-
 static u32 cdns_mhdp_get_bpp(struct cdns_mhdp_display_fmt *fmt)
 {
 	u32 bpp;
@@ -1547,114 +1493,6 @@ bool cdns_mhdp_bandwidth_ok(struct cdns_mhdp_device *mhdp,
 	return true;
 }
 
-static
-enum drm_mode_status cdns_mhdp_mode_valid(struct drm_connector *conn,
-					  const struct drm_display_mode *mode)
-{
-	struct cdns_mhdp_device *mhdp = connector_to_mhdp(conn);
-
-	mutex_lock(&mhdp->link_mutex);
-
-	if (!cdns_mhdp_bandwidth_ok(mhdp, mode, mhdp->link.num_lanes,
-				    mhdp->link.rate)) {
-		mutex_unlock(&mhdp->link_mutex);
-		return MODE_CLOCK_HIGH;
-	}
-
-	mutex_unlock(&mhdp->link_mutex);
-	return MODE_OK;
-}
-
-static int cdns_mhdp_connector_atomic_check(struct drm_connector *conn,
-					    struct drm_atomic_state *state)
-{
-	struct cdns_mhdp_device *mhdp = connector_to_mhdp(conn);
-	struct drm_connector_state *old_state, *new_state;
-	struct drm_crtc_state *crtc_state;
-	u64 old_cp, new_cp;
-
-	if (!mhdp->hdcp_supported)
-		return 0;
-
-	old_state = drm_atomic_get_old_connector_state(state, conn);
-	new_state = drm_atomic_get_new_connector_state(state, conn);
-	old_cp = old_state->content_protection;
-	new_cp = new_state->content_protection;
-
-	if (old_state->hdcp_content_type != new_state->hdcp_content_type &&
-	    new_cp != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
-		new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-		goto mode_changed;
-	}
-
-	if (!new_state->crtc) {
-		if (old_cp == DRM_MODE_CONTENT_PROTECTION_ENABLED)
-			new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-		return 0;
-	}
-
-	if (old_cp == new_cp ||
-	    (old_cp == DRM_MODE_CONTENT_PROTECTION_DESIRED &&
-	     new_cp == DRM_MODE_CONTENT_PROTECTION_ENABLED))
-		return 0;
-
-mode_changed:
-	crtc_state = drm_atomic_get_new_crtc_state(state, new_state->crtc);
-	crtc_state->mode_changed = true;
-
-	return 0;
-}
-
-static const struct drm_connector_helper_funcs cdns_mhdp_conn_helper_funcs = {
-	.detect_ctx = cdns_mhdp_connector_detect,
-	.get_modes = cdns_mhdp_get_modes,
-	.mode_valid = cdns_mhdp_mode_valid,
-	.atomic_check = cdns_mhdp_connector_atomic_check,
-};
-
-static const struct drm_connector_funcs cdns_mhdp_conn_funcs = {
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.reset = drm_atomic_helper_connector_reset,
-	.destroy = drm_connector_cleanup,
-};
-
-static int cdns_mhdp_connector_init(struct cdns_mhdp_device *mhdp)
-{
-	u32 bus_format = MEDIA_BUS_FMT_RGB121212_1X36;
-	struct drm_connector *conn = &mhdp->connector;
-	struct drm_bridge *bridge = &mhdp->bridge;
-	int ret;
-
-	conn->polled = DRM_CONNECTOR_POLL_HPD;
-
-	ret = drm_connector_init(bridge->dev, conn, &cdns_mhdp_conn_funcs,
-				 DRM_MODE_CONNECTOR_DisplayPort);
-	if (ret) {
-		dev_err(mhdp->dev, "Failed to initialize connector with drm\n");
-		return ret;
-	}
-
-	drm_connector_helper_add(conn, &cdns_mhdp_conn_helper_funcs);
-
-	ret = drm_display_info_set_bus_formats(&conn->display_info,
-					       &bus_format, 1);
-	if (ret)
-		return ret;
-
-	ret = drm_connector_attach_encoder(conn, bridge->encoder);
-	if (ret) {
-		dev_err(mhdp->dev, "Failed to attach connector to encoder\n");
-		return ret;
-	}
-
-	if (mhdp->hdcp_supported)
-		ret = drm_connector_attach_content_protection_property(conn, true);
-
-	return ret;
-}
-
 static int cdns_mhdp_attach(struct drm_bridge *bridge,
 			    struct drm_encoder *encoder,
 			    enum drm_bridge_attach_flags flags)
@@ -1671,9 +1509,11 @@ static int cdns_mhdp_attach(struct drm_bridge *bridge,
 		return ret;
 
 	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)) {
-		ret = cdns_mhdp_connector_init(mhdp);
-		if (ret)
-			goto aux_unregister;
+		ret = -EINVAL;
+		dev_err(mhdp->dev,
+			"Connector initialisation not supported in bridge_attach %d\n",
+			ret);
+		goto aux_unregister;
 	}
 
 	spin_lock(&mhdp->start_lock);
@@ -1915,17 +1755,25 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge,
 	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
 	struct cdns_mhdp_bridge_state *mhdp_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_connector *connector;
 	struct drm_connector_state *conn_state;
 	struct drm_bridge_state *new_state;
 	const struct drm_display_mode *mode;
 	u32 resp;
-	int ret;
+	int ret = 0;
 
 	dev_dbg(mhdp->dev, "bridge enable\n");
 
 	mutex_lock(&mhdp->link_mutex);
 
+	mhdp->connector = drm_atomic_get_new_connector_for_encoder(state,
+								   bridge->encoder);
+	if (WARN_ON(!mhdp->connector))
+		goto out;
+
+	conn_state = drm_atomic_get_new_connector_state(state, mhdp->connector);
+	if (WARN_ON(!conn_state))
+		goto out;
+
 	if (mhdp->plugged && !mhdp->link_up) {
 		ret = cdns_mhdp_link_up(mhdp);
 		if (ret < 0)
@@ -1945,15 +1793,6 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge,
 	cdns_mhdp_reg_write(mhdp, CDNS_DPTX_CAR,
 			    resp | CDNS_VIF_CLK_EN | CDNS_VIF_CLK_RSTN);
 
-	connector = drm_atomic_get_new_connector_for_encoder(state,
-							     bridge->encoder);
-	if (WARN_ON(!connector))
-		goto out;
-
-	conn_state = drm_atomic_get_new_connector_state(state, connector);
-	if (WARN_ON(!conn_state))
-		goto out;
-
 	if (mhdp->hdcp_supported &&
 	    mhdp->hw_state == MHDP_HW_READY &&
 	    conn_state->content_protection ==
@@ -2030,6 +1869,7 @@ static void cdns_mhdp_atomic_disable(struct drm_bridge *bridge,
 	if (mhdp->info && mhdp->info->ops && mhdp->info->ops->disable)
 		mhdp->info->ops->disable(mhdp);
 
+	mhdp->connector = NULL;
 	mutex_unlock(&mhdp->link_mutex);
 }
 
@@ -2122,6 +1962,10 @@ static int cdns_mhdp_atomic_check(struct drm_bridge *bridge,
 {
 	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
 	const struct drm_display_mode *mode = &crtc_state->adjusted_mode;
+	struct drm_connector_state *old_state, *new_state;
+	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_connector *conn = mhdp->connector;
+	u64 old_cp, new_cp;
 
 	mutex_lock(&mhdp->link_mutex);
 
@@ -2141,6 +1985,25 @@ static int cdns_mhdp_atomic_check(struct drm_bridge *bridge,
 	if (mhdp->info)
 		bridge_state->input_bus_cfg.flags = *mhdp->info->input_bus_flags;
 
+	if (conn && mhdp->hdcp_supported) {
+		old_state = drm_atomic_get_old_connector_state(state, conn);
+		new_state = drm_atomic_get_new_connector_state(state, conn);
+		old_cp = old_state->content_protection;
+		new_cp = new_state->content_protection;
+
+		if (old_state->hdcp_content_type != new_state->hdcp_content_type &&
+		    new_cp != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
+			new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+			crtc_state = drm_atomic_get_new_crtc_state(state, new_state->crtc);
+			crtc_state->mode_changed = true;
+		}
+
+		if (!new_state->crtc) {
+			if (old_cp == DRM_MODE_CONTENT_PROTECTION_ENABLED)
+				new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		}
+	}
+
 	mutex_unlock(&mhdp->link_mutex);
 	return 0;
 }
@@ -2161,6 +2024,25 @@ static const struct drm_edid *cdns_mhdp_bridge_edid_read(struct drm_bridge *brid
 	return cdns_mhdp_edid_read(mhdp, connector);
 }
 
+static enum drm_mode_status
+cdns_mhdp_bridge_mode_valid(struct drm_bridge *bridge,
+			    const struct drm_display_info *info,
+			    const struct drm_display_mode *mode)
+{
+	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
+
+	mutex_lock(&mhdp->link_mutex);
+
+	if (!cdns_mhdp_bandwidth_ok(mhdp, mode, mhdp->link.num_lanes,
+				    mhdp->link.rate)) {
+		mutex_unlock(&mhdp->link_mutex);
+		return MODE_CLOCK_HIGH;
+	}
+
+	mutex_unlock(&mhdp->link_mutex);
+	return MODE_OK;
+}
+
 static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = {
 	.atomic_enable = cdns_mhdp_atomic_enable,
 	.atomic_disable = cdns_mhdp_atomic_disable,
@@ -2175,6 +2057,7 @@ static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = {
 	.edid_read = cdns_mhdp_bridge_edid_read,
 	.hpd_enable = cdns_mhdp_bridge_hpd_enable,
 	.hpd_disable = cdns_mhdp_bridge_hpd_disable,
+	.mode_valid = cdns_mhdp_bridge_mode_valid,
 };
 
 static bool cdns_mhdp_detect_hpd(struct cdns_mhdp_device *mhdp, bool *hpd_pulse)
@@ -2296,7 +2179,7 @@ static void cdns_mhdp_modeset_retry_fn(struct work_struct *work)
 
 	mhdp = container_of(work, typeof(*mhdp), modeset_retry_work);
 
-	conn = &mhdp->connector;
+	conn = mhdp->connector;
 
 	/* Grab the locks before changing connector property */
 	mutex_lock(&conn->dev->mode_config.mutex);
@@ -2370,17 +2253,9 @@ static void cdns_mhdp_hpd_work(struct work_struct *work)
 	struct cdns_mhdp_device *mhdp = container_of(work,
 						     struct cdns_mhdp_device,
 						     hpd_work);
-	int ret;
+	cdns_mhdp_update_link_status(mhdp);
 
-	ret = cdns_mhdp_update_link_status(mhdp);
-	if (mhdp->connector.dev) {
-		if (ret < 0)
-			schedule_work(&mhdp->modeset_retry_work);
-		else
-			drm_kms_helper_hotplug_event(mhdp->bridge.dev);
-	} else {
-		drm_bridge_hpd_notify(&mhdp->bridge, cdns_mhdp_detect(mhdp));
-	}
+	drm_bridge_hpd_notify(&mhdp->bridge, cdns_mhdp_detect(mhdp));
 }
 
 static int cdns_mhdp_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h
index bad2fc0c73066..b297db53ba283 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h
@@ -375,7 +375,7 @@ struct cdns_mhdp_device {
 	 */
 	struct mutex link_mutex;
 
-	struct drm_connector connector;
+	struct drm_connector *connector;
 	struct drm_bridge bridge;
 
 	struct cdns_mhdp_link link;
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c
index 42248f179b69d..5cd0b873e16f9 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c
@@ -394,7 +394,7 @@ static int _cdns_mhdp_hdcp_disable(struct cdns_mhdp_device *mhdp)
 	int ret;
 
 	dev_dbg(mhdp->dev, "[%s:%d] HDCP is being disabled...\n",
-		mhdp->connector.name, mhdp->connector.base.id);
+		mhdp->connector->name, mhdp->connector->base.id);
 
 	ret = cdns_mhdp_hdcp_set_config(mhdp, 0, false);
 
@@ -436,6 +436,10 @@ static int cdns_mhdp_hdcp_check_link(struct cdns_mhdp_device *mhdp)
 	int ret = 0;
 
 	mutex_lock(&mhdp->hdcp.mutex);
+
+	if (!mhdp->connector)
+		goto out;
+
 	if (mhdp->hdcp.value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED)
 		goto out;
 
@@ -445,7 +449,7 @@ static int cdns_mhdp_hdcp_check_link(struct cdns_mhdp_device *mhdp)
 
 	dev_err(mhdp->dev,
 		"[%s:%d] HDCP link failed, retrying authentication\n",
-		mhdp->connector.name, mhdp->connector.base.id);
+		mhdp->connector->name, mhdp->connector->base.id);
 
 	ret = _cdns_mhdp_hdcp_disable(mhdp);
 	if (ret) {
@@ -487,13 +491,19 @@ static void cdns_mhdp_hdcp_prop_work(struct work_struct *work)
 	struct cdns_mhdp_device *mhdp = container_of(hdcp,
 						     struct cdns_mhdp_device,
 						     hdcp);
-	struct drm_device *dev = mhdp->connector.dev;
+	struct drm_device *dev = NULL;
 	struct drm_connector_state *state;
 
+	if (mhdp->connector)
+		dev = mhdp->connector->dev;
+
+	if (!dev)
+		return;
+
 	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
 	mutex_lock(&mhdp->hdcp.mutex);
 	if (mhdp->hdcp.value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
-		state = mhdp->connector.state;
+		state = mhdp->connector->state;
 		state->content_protection = mhdp->hdcp.value;
 	}
 	mutex_unlock(&mhdp->hdcp.mutex);
diff --git a/drivers/gpu/drm/bridge/fsl-ldb.c b/drivers/gpu/drm/bridge/fsl-ldb.c
index 5c3cf37200bce..7b71cde173e0c 100644
--- a/drivers/gpu/drm/bridge/fsl-ldb.c
+++ b/drivers/gpu/drm/bridge/fsl-ldb.c
@@ -92,6 +92,7 @@ struct fsl_ldb {
 	const struct fsl_ldb_devdata *devdata;
 	bool ch0_enabled;
 	bool ch1_enabled;
+	bool use_termination_resistor;
 };
 
 static bool fsl_ldb_is_dual(const struct fsl_ldb *fsl_ldb)
@@ -212,6 +213,9 @@ static void fsl_ldb_atomic_enable(struct drm_bridge *bridge,
 	/* Program LVDS_CTRL */
 	reg = LVDS_CTRL_CC_ADJ(2) | LVDS_CTRL_PRE_EMPH_EN |
 	      LVDS_CTRL_PRE_EMPH_ADJ(3) | LVDS_CTRL_VBG_EN;
+
+	if (fsl_ldb->use_termination_resistor)
+		reg |= LVDS_CTRL_HS_EN;
 	regmap_write(fsl_ldb->regmap, fsl_ldb->devdata->lvds_ctrl, reg);
 
 	/* Wait for VBG to stabilize. */
@@ -340,6 +344,9 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 	if (IS_ERR(panel))
 		return PTR_ERR(panel);
 
+	if (of_property_present(dev->of_node, "nxp,enable-termination-resistor"))
+		fsl_ldb->use_termination_resistor = true;
+
 	fsl_ldb->panel_bridge = devm_drm_panel_bridge_add(dev, panel);
 	if (IS_ERR(fsl_ldb->panel_bridge))
 		return PTR_ERR(fsl_ldb->panel_bridge);
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
index b52b7c7ce183c..63a8d8b1f76b1 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
@@ -23,7 +23,6 @@
 
 struct imx8qxp_pixel_link {
 	struct drm_bridge bridge;
-	struct drm_bridge *next_bridge;
 	struct device *dev;
 	struct imx_sc_ipc *ipc_handle;
 	u8 stream_id;
@@ -140,7 +139,7 @@ static int imx8qxp_pixel_link_bridge_attach(struct drm_bridge *bridge,
 	}
 
 	return drm_bridge_attach(encoder,
-				 pl->next_bridge, bridge,
+				 pl->bridge.next_bridge, bridge,
 				 DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 }
 
@@ -256,17 +255,13 @@ static int imx8qxp_pixel_link_disable_all_controls(struct imx8qxp_pixel_link *pl
 	return imx8qxp_pixel_link_disable_sync(pl);
 }
 
-static struct drm_bridge *
-imx8qxp_pixel_link_find_next_bridge(struct imx8qxp_pixel_link *pl)
+static int imx8qxp_pixel_link_find_next_bridge(struct imx8qxp_pixel_link *pl)
 {
 	struct device_node *np = pl->dev->of_node;
-	struct device_node *port, *remote;
-	struct drm_bridge *next_bridge[PL_MAX_NEXT_BRIDGES];
+	struct device_node *port;
 	u32 port_id;
 	bool found_port = false;
-	int reg, ep_cnt = 0;
-	/* select the first next bridge by default */
-	int bridge_sel = 0;
+	int reg;
 
 	for (port_id = 1; port_id <= PL_MAX_MST_ADDR + 1; port_id++) {
 		port = of_graph_get_port_by_id(np, port_id);
@@ -284,11 +279,12 @@ imx8qxp_pixel_link_find_next_bridge(struct imx8qxp_pixel_link *pl)
 
 	if (!found_port) {
 		DRM_DEV_ERROR(pl->dev, "no available output port\n");
-		return ERR_PTR(-ENODEV);
+		return -ENODEV;
 	}
 
 	for (reg = 0; reg < PL_MAX_NEXT_BRIDGES; reg++) {
-		remote = of_graph_get_remote_node(np, port_id, reg);
+		struct device_node *remote __free(device_node) =
+			of_graph_get_remote_node(np, port_id, reg);
 		if (!remote)
 			continue;
 
@@ -296,28 +292,26 @@ imx8qxp_pixel_link_find_next_bridge(struct imx8qxp_pixel_link *pl)
 			DRM_DEV_DEBUG(pl->dev,
 				      "port%u endpoint%u remote parent is not available\n",
 				      port_id, reg);
-			of_node_put(remote);
 			continue;
 		}
 
-		next_bridge[ep_cnt] = of_drm_find_bridge(remote);
-		if (!next_bridge[ep_cnt]) {
-			of_node_put(remote);
-			return ERR_PTR(-EPROBE_DEFER);
+		if (!pl->bridge.next_bridge) {
+			/* Select the first bridge by default... */
+			pl->bridge.next_bridge = of_drm_find_and_get_bridge(remote);
+			if (!pl->bridge.next_bridge)
+				return -EPROBE_DEFER;
+		} else if (of_property_present(remote, "fsl,companion-pxl2dpi")) {
+			/* ... but prefer the companion PXL2DPI if present */
+			drm_bridge_put(pl->bridge.next_bridge);
+			pl->bridge.next_bridge = of_drm_find_and_get_bridge(remote);
+			if (!pl->bridge.next_bridge)
+				return -EPROBE_DEFER;
 		}
-
-		/* specially select the next bridge with companion PXL2DPI */
-		if (of_property_present(remote, "fsl,companion-pxl2dpi"))
-			bridge_sel = ep_cnt;
-
-		ep_cnt++;
-
-		of_node_put(remote);
 	}
 
 	pl->mst_addr = port_id - 1;
 
-	return next_bridge[bridge_sel];
+	return 0;
 }
 
 static int imx8qxp_pixel_link_bridge_probe(struct platform_device *pdev)
@@ -373,9 +367,9 @@ static int imx8qxp_pixel_link_bridge_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	pl->next_bridge = imx8qxp_pixel_link_find_next_bridge(pl);
-	if (IS_ERR(pl->next_bridge))
-		return PTR_ERR(pl->next_bridge);
+	ret = imx8qxp_pixel_link_find_next_bridge(pl);
+	if (ret)
+		return ret;
 
 	platform_set_drvdata(pdev, pl);
 
diff --git a/drivers/gpu/drm/bridge/lontium-lt8713sx.c b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
new file mode 100644
index 0000000000000..a8ae38c847190
--- /dev/null
+++ b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
@@ -0,0 +1,598 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/crc8.h>
+#include <linux/firmware.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/sizes.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_of.h>
+
+#define FW_FILE "lt8713sx_fw.bin"
+
+#define REG_PAGE_CONTROL	0xff
+
+#define LT8713SX_PAGE_SIZE	256
+
+DECLARE_CRC8_TABLE(lt8713sx_crc_table);
+
+struct lt8713sx {
+	struct device *dev;
+	struct drm_bridge bridge;
+	struct drm_bridge *next_bridge;
+
+	struct regmap *regmap;
+	/* Protects all accesses to registers by stopping the on-chip MCU */
+	struct mutex ocm_lock;
+
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *enable_gpio;
+
+	struct i2c_client *client;
+	const struct firmware *fw;
+
+	u8 *fw_buffer;
+
+	u32 main_crc_value;
+	u32 bank_crc_value[17];
+
+	int bank_num;
+};
+
+static void lt8713sx_reset(struct lt8713sx *lt8713sx);
+
+static const struct regmap_range lt8713sx_ranges[] = {
+	{
+		.range_min = 0x0000,
+		.range_max = 0xffff
+	},
+};
+
+static const struct regmap_access_table lt8713sx_table = {
+	.yes_ranges = lt8713sx_ranges,
+	.n_yes_ranges = ARRAY_SIZE(lt8713sx_ranges),
+};
+
+static const struct regmap_range_cfg lt8713sx_range_cfg = {
+	.name = "lt8713sx",
+	.range_min = 0x0000,
+	.range_max = 0xffff,
+	.selector_reg = REG_PAGE_CONTROL,
+	.selector_mask = 0xff,
+	.selector_shift = 0,
+	.window_start = 0,
+	.window_len = 0x100,
+};
+
+static const struct regmap_config lt8713sx_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.volatile_table = &lt8713sx_table,
+	.ranges = &lt8713sx_range_cfg,
+	.num_ranges = 1,
+	.cache_type = REGCACHE_NONE,
+	.max_register = 0xffff,
+};
+
+static void lt8713sx_i2c_enable(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe0ee, 0x01);
+}
+
+static void lt8713sx_i2c_disable(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe0ee, 0x00);
+}
+
+static int lt8713sx_prepare_firmware_data(struct lt8713sx *lt8713sx)
+{
+	int ret = 0;
+	u64 sz_12k = 12 * SZ_1K;
+
+	ret = request_firmware(&lt8713sx->fw, FW_FILE, lt8713sx->dev);
+	if (ret < 0) {
+		dev_err(lt8713sx->dev, "request firmware failed\n");
+		return ret;
+	}
+
+	dev_dbg(lt8713sx->dev, "Firmware size: %zu bytes\n", lt8713sx->fw->size);
+
+	if (lt8713sx->fw->size > SZ_256K - 1) {
+		dev_err(lt8713sx->dev, "Firmware size exceeds 256KB limit\n");
+		release_firmware(lt8713sx->fw);
+		return -EINVAL;
+	}
+
+	lt8713sx->fw_buffer = kvmalloc(SZ_256K, GFP_KERNEL);
+	if (!lt8713sx->fw_buffer) {
+		release_firmware(lt8713sx->fw);
+		return -ENOMEM;
+	}
+
+	memset(lt8713sx->fw_buffer, 0xff, SZ_256K);
+
+	/* main firmware */
+	memcpy(lt8713sx->fw_buffer, lt8713sx->fw->data, SZ_64K - 1);
+
+	lt8713sx->fw_buffer[SZ_64K - 1] =
+		crc8(lt8713sx_crc_table, lt8713sx->fw_buffer, SZ_64K - 1, 0);
+	lt8713sx->main_crc_value = lt8713sx->fw_buffer[SZ_64K - 1];
+	dev_dbg(lt8713sx->dev,
+		"Main Firmware Data  Crc = 0x%02X\n", lt8713sx->main_crc_value);
+
+	/* bank firmware */
+	memcpy(lt8713sx->fw_buffer + SZ_64K,
+	       lt8713sx->fw->data + SZ_64K,
+	       lt8713sx->fw->size - SZ_64K);
+
+	lt8713sx->bank_num = (lt8713sx->fw->size - SZ_64K + sz_12k - 1) / sz_12k;
+	dev_dbg(lt8713sx->dev, "Bank Number Total is %d.\n", lt8713sx->bank_num);
+
+	for (int i = 0; i < lt8713sx->bank_num; i++) {
+		lt8713sx->bank_crc_value[i] =
+			crc8(lt8713sx_crc_table, lt8713sx->fw_buffer + SZ_64K + i * sz_12k,
+			     sz_12k, 0);
+		dev_dbg(lt8713sx->dev, "Bank number:%d; Firmware Data  Crc:0x%02X\n",
+			i, lt8713sx->bank_crc_value[i]);
+	}
+	return 0;
+}
+
+static void lt8713sx_config_parameters(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe05e, 0xc1);
+	regmap_write(lt8713sx->regmap, 0xe058, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe059, 0x50);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x10);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe058, 0x21);
+}
+
+static void lt8713sx_wren(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe103, 0xbf);
+	regmap_write(lt8713sx->regmap, 0xe103, 0xff);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x04);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x00);
+}
+
+static void lt8713sx_wrdi(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x08);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x00);
+}
+
+static void lt8713sx_fifo_reset(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe103, 0xbf);
+	regmap_write(lt8713sx->regmap, 0xe103, 0xff);
+}
+
+static void lt8713sx_disable_sram_write(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe055, 0x00);
+}
+
+static void lt8713sx_sram_to_flash(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x30);
+	regmap_write(lt8713sx->regmap, 0xe05a, 0x00);
+}
+
+static void lt8713sx_i2c_to_sram(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe055, 0x80);
+	regmap_write(lt8713sx->regmap, 0xe05e, 0xc0);
+	regmap_write(lt8713sx->regmap, 0xe058, 0x21);
+}
+
+static u8 lt8713sx_read_flash_status(struct lt8713sx *lt8713sx)
+{
+	u32 flash_status = 0;
+
+	regmap_write(lt8713sx->regmap,  0xe103, 0x3f);
+	regmap_write(lt8713sx->regmap,  0xe103, 0xff);
+
+	regmap_write(lt8713sx->regmap,  0xe05e, 0x40);
+	regmap_write(lt8713sx->regmap,  0xe056, 0x05); /* opcode=read status register */
+	regmap_write(lt8713sx->regmap,  0xe055, 0x25);
+	regmap_write(lt8713sx->regmap,  0xe055, 0x01);
+	regmap_write(lt8713sx->regmap,  0xe058, 0x21);
+
+	regmap_read(lt8713sx->regmap, 0xe05f, &flash_status);
+	dev_dbg(lt8713sx->dev, "flash_status:%x\n", flash_status);
+
+	return flash_status;
+}
+
+static void lt8713sx_block_erase(struct lt8713sx *lt8713sx)
+{
+	u32 i = 0;
+	u8 flash_status = 0;
+	u8 blocknum = 0x00;
+	u32 flashaddr = 0x00;
+
+	for (blocknum = 0; blocknum < 8; blocknum++) {
+		flashaddr = blocknum * SZ_32K;
+		regmap_write(lt8713sx->regmap,  0xe05a, 0x04);
+		regmap_write(lt8713sx->regmap,  0xe05a, 0x00);
+		regmap_write(lt8713sx->regmap,  0xe05b, flashaddr >> 16);
+		regmap_write(lt8713sx->regmap,  0xe05c, flashaddr >> 8);
+		regmap_write(lt8713sx->regmap,  0xe05d, flashaddr);
+		regmap_write(lt8713sx->regmap,  0xe05a, 0x01);
+		regmap_write(lt8713sx->regmap,  0xe05a, 0x00);
+		msleep(100);
+		i = 0;
+		while (1) {
+			flash_status = lt8713sx_read_flash_status(lt8713sx);
+			if ((flash_status & 0x01) == 0)
+				break;
+
+			if (i > 50)
+				break;
+
+			i++;
+			msleep(50);
+		}
+	}
+	dev_dbg(lt8713sx->dev, "erase flash done.\n");
+}
+
+static void lt8713sx_load_main_fw_to_sram(struct lt8713sx *lt8713sx)
+{
+	regmap_write(lt8713sx->regmap, 0xe068, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe069, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe06a, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe065, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe066, 0xff);
+	regmap_write(lt8713sx->regmap, 0xe067, 0xff);
+	regmap_write(lt8713sx->regmap, 0xe06b, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe06c, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe060, 0x01);
+	msleep(200);
+	regmap_write(lt8713sx->regmap, 0xe060, 0x00);
+}
+
+static void lt8713sx_load_bank_fw_to_sram(struct lt8713sx *lt8713sx, u64 addr)
+{
+	regmap_write(lt8713sx->regmap, 0xe068, ((addr & 0xff0000) >> 16));
+	regmap_write(lt8713sx->regmap, 0xe069, ((addr & 0x00ff00) >> 8));
+	regmap_write(lt8713sx->regmap, 0xe06a, (addr & 0x0000ff));
+	regmap_write(lt8713sx->regmap, 0xe065, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe066, 0x30);
+	regmap_write(lt8713sx->regmap, 0xe067, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe06b, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe06c, 0x00);
+	regmap_write(lt8713sx->regmap, 0xe060, 0x01);
+	msleep(50);
+	regmap_write(lt8713sx->regmap, 0xe060, 0x00);
+}
+
+static int lt8713sx_write_data(struct lt8713sx *lt8713sx, const u8 *data, u64 filesize)
+{
+	int page = 0, num = 0, i = 0, val;
+
+	page = (filesize % LT8713SX_PAGE_SIZE) ?
+			((filesize / LT8713SX_PAGE_SIZE) + 1) : (filesize / LT8713SX_PAGE_SIZE);
+
+	dev_dbg(lt8713sx->dev,
+		"Writing to Sram=%u pages, total size = %llu bytes\n", page, filesize);
+
+	for (num = 0; num < page; num++) {
+		dev_dbg(lt8713sx->dev, "page[%d]\n", num);
+		lt8713sx_i2c_to_sram(lt8713sx);
+
+		for (i = 0; i < LT8713SX_PAGE_SIZE; i++) {
+			if ((num * LT8713SX_PAGE_SIZE + i) < filesize)
+				val = *(data + (num * LT8713SX_PAGE_SIZE + i));
+			else
+				val = 0xff;
+			regmap_write(lt8713sx->regmap, 0xe059, val);
+		}
+
+		lt8713sx_wren(lt8713sx);
+		lt8713sx_sram_to_flash(lt8713sx);
+	}
+
+	lt8713sx_wrdi(lt8713sx);
+	lt8713sx_disable_sram_write(lt8713sx);
+
+	return 0;
+}
+
+static void lt8713sx_main_upgrade_result(struct lt8713sx *lt8713sx)
+{
+	u32 main_crc_result;
+
+	regmap_read(lt8713sx->regmap, 0xe023, &main_crc_result);
+
+	dev_dbg(lt8713sx->dev, "Main CRC HW: 0x%02X\n", main_crc_result);
+	dev_dbg(lt8713sx->dev, "Main CRC FW: 0x%02X\n", lt8713sx->main_crc_value);
+
+	if (main_crc_result == lt8713sx->main_crc_value)
+		dev_info(lt8713sx->dev, "Main Firmware Upgrade Success.\n");
+	else
+		dev_err(lt8713sx->dev, "Main Firmware Upgrade Failed.\n");
+}
+
+static void lt8713sx_bank_upgrade_result(struct lt8713sx *lt8713sx, u8 banknum)
+{
+	u32 bank_crc_result;
+
+	regmap_read(lt8713sx->regmap, 0xe023, &bank_crc_result);
+
+	dev_dbg(lt8713sx->dev, "Bank %d CRC Result: 0x%02X\n", banknum, bank_crc_result);
+
+	if (bank_crc_result == lt8713sx->bank_crc_value[banknum])
+		dev_info(lt8713sx->dev, "Bank %d Firmware Upgrade Success.\n", banknum);
+	else
+		dev_err(lt8713sx->dev, "Bank %d Firmware Upgrade Failed.\n", banknum);
+}
+
+static void lt8713sx_bank_result_check(struct lt8713sx *lt8713sx)
+{
+	int i;
+	u64 addr = 0x010000;
+
+	for (i = 0; i < lt8713sx->bank_num; i++) {
+		lt8713sx_load_bank_fw_to_sram(lt8713sx, addr);
+		lt8713sx_bank_upgrade_result(lt8713sx, i);
+		addr += 0x3000;
+	}
+}
+
+static int lt8713sx_firmware_upgrade(struct lt8713sx *lt8713sx)
+{
+	int ret;
+
+	lt8713sx_config_parameters(lt8713sx);
+
+	lt8713sx_block_erase(lt8713sx);
+
+	if (lt8713sx->fw->size < SZ_64K) {
+		ret = lt8713sx_write_data(lt8713sx, lt8713sx->fw_buffer, SZ_64K);
+		if (ret < 0) {
+			dev_err(lt8713sx->dev, "Failed to write firmware data: %d\n", ret);
+			return ret;
+		}
+	} else {
+		ret = lt8713sx_write_data(lt8713sx, lt8713sx->fw_buffer, lt8713sx->fw->size);
+		if (ret < 0) {
+			dev_err(lt8713sx->dev, "Failed to write firmware data: %d\n", ret);
+			return ret;
+		}
+	}
+	dev_dbg(lt8713sx->dev, "Write Data done.\n");
+
+	return 0;
+}
+
+static int lt8713sx_firmware_update(struct lt8713sx *lt8713sx)
+{
+	int ret = 0;
+
+	guard(mutex)(&lt8713sx->ocm_lock);
+	lt8713sx_i2c_enable(lt8713sx);
+
+	ret = lt8713sx_prepare_firmware_data(lt8713sx);
+	if (ret < 0) {
+		dev_err(lt8713sx->dev, "Failed to prepare firmware data: %d\n", ret);
+		goto error;
+	}
+
+	ret = lt8713sx_firmware_upgrade(lt8713sx);
+	if (ret < 0) {
+		dev_err(lt8713sx->dev, "Upgrade failure.\n");
+		goto error;
+	}
+
+	/* Validate CRC */
+	lt8713sx_load_main_fw_to_sram(lt8713sx);
+	lt8713sx_main_upgrade_result(lt8713sx);
+	lt8713sx_wrdi(lt8713sx);
+	lt8713sx_fifo_reset(lt8713sx);
+	lt8713sx_bank_result_check(lt8713sx);
+	lt8713sx_wrdi(lt8713sx);
+
+error:
+	lt8713sx_i2c_disable(lt8713sx);
+	if (!ret)
+		lt8713sx_reset(lt8713sx);
+
+	kvfree(lt8713sx->fw_buffer);
+	lt8713sx->fw_buffer = NULL;
+
+	if (lt8713sx->fw) {
+		release_firmware(lt8713sx->fw);
+		lt8713sx->fw = NULL;
+	}
+
+	return ret;
+}
+
+static void lt8713sx_reset(struct lt8713sx *lt8713sx)
+{
+	dev_dbg(lt8713sx->dev, "reset bridge.\n");
+	gpiod_set_value_cansleep(lt8713sx->reset_gpio, 1);
+	msleep(20);
+
+	gpiod_set_value_cansleep(lt8713sx->reset_gpio, 0);
+	msleep(20);
+
+	dev_dbg(lt8713sx->dev, "reset done.\n");
+}
+
+static int lt8713sx_regulator_enable(struct lt8713sx *lt8713sx)
+{
+	int ret;
+
+	ret = devm_regulator_get_enable(lt8713sx->dev, "vdd");
+	if (ret < 0)
+		return dev_err_probe(lt8713sx->dev, ret, "failed to enable vdd regulator\n");
+
+	usleep_range(1000, 10000);
+
+	ret = devm_regulator_get_enable(lt8713sx->dev, "vcc");
+	if (ret < 0)
+		return dev_err_probe(lt8713sx->dev, ret, "failed to enable vcc regulator\n");
+	return 0;
+}
+
+static int lt8713sx_bridge_attach(struct drm_bridge *bridge,
+				  struct drm_encoder *encoder,
+				  enum drm_bridge_attach_flags flags)
+{
+	struct lt8713sx *lt8713sx = container_of(bridge, struct lt8713sx, bridge);
+
+	return drm_bridge_attach(encoder,
+				lt8713sx->next_bridge,
+				bridge, flags);
+}
+
+static int lt8713sx_gpio_init(struct lt8713sx *lt8713sx)
+{
+	struct device *dev = lt8713sx->dev;
+
+	lt8713sx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(lt8713sx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(lt8713sx->reset_gpio),
+				     "failed to acquire reset gpio\n");
+
+	/* power enable gpio */
+	lt8713sx->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(lt8713sx->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(lt8713sx->enable_gpio),
+				     "failed to acquire enable gpio\n");
+	return 0;
+}
+
+static ssize_t lt8713sx_firmware_store(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t len)
+{
+	struct lt8713sx *lt8713sx = dev_get_drvdata(dev);
+	int ret;
+
+	ret = lt8713sx_firmware_update(lt8713sx);
+	if (ret < 0)
+		return ret;
+	return len;
+}
+
+static DEVICE_ATTR_WO(lt8713sx_firmware);
+
+static struct attribute *lt8713sx_attrs[] = {
+	&dev_attr_lt8713sx_firmware.attr,
+	NULL,
+};
+
+static const struct attribute_group lt8713sx_attr_group = {
+	.attrs = lt8713sx_attrs,
+};
+
+static const struct attribute_group *lt8713sx_attr_groups[] = {
+	&lt8713sx_attr_group,
+	NULL,
+};
+
+static const struct drm_bridge_funcs lt8713sx_bridge_funcs = {
+	.attach = lt8713sx_bridge_attach,
+};
+
+static int lt8713sx_probe(struct i2c_client *client)
+{
+	struct lt8713sx *lt8713sx;
+	struct device *dev = &client->dev;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
+		return dev_err_probe(dev, -ENODEV, "device doesn't support I2C\n");
+
+	lt8713sx = devm_drm_bridge_alloc(dev, struct lt8713sx, bridge, &lt8713sx_bridge_funcs);
+	if (IS_ERR(lt8713sx))
+		return PTR_ERR(lt8713sx);
+
+	lt8713sx->dev = dev;
+	lt8713sx->client = client;
+	i2c_set_clientdata(client, lt8713sx);
+
+	ret = devm_mutex_init(lt8713sx->dev, &lt8713sx->ocm_lock);
+	if (ret)
+		return ret;
+
+	lt8713sx->regmap = devm_regmap_init_i2c(client, &lt8713sx_regmap_config);
+	if (IS_ERR(lt8713sx->regmap))
+		return dev_err_probe(dev, PTR_ERR(lt8713sx->regmap), "regmap i2c init failed\n");
+
+	ret = drm_of_find_panel_or_bridge(lt8713sx->dev->of_node, 1, -1, NULL,
+					  &lt8713sx->next_bridge);
+	if (ret < 0)
+		return ret;
+
+	ret = lt8713sx_gpio_init(lt8713sx);
+	if (ret < 0)
+		return ret;
+
+	ret = lt8713sx_regulator_enable(lt8713sx);
+	if (ret)
+		return ret;
+
+	lt8713sx_reset(lt8713sx);
+
+	lt8713sx->bridge.funcs = &lt8713sx_bridge_funcs;
+	lt8713sx->bridge.of_node = dev->of_node;
+	lt8713sx->bridge.type = DRM_MODE_CONNECTOR_DisplayPort;
+	drm_bridge_add(&lt8713sx->bridge);
+
+	crc8_populate_msb(lt8713sx_crc_table, 0x31);
+
+	return 0;
+}
+
+static void lt8713sx_remove(struct i2c_client *client)
+{
+	struct lt8713sx *lt8713sx = i2c_get_clientdata(client);
+
+	drm_bridge_remove(&lt8713sx->bridge);
+}
+
+static struct i2c_device_id lt8713sx_id[] = {
+	{ "lontium,lt8713sx", 0 },
+	{ /* sentinel */ }
+};
+
+static const struct of_device_id lt8713sx_match_table[] = {
+	{ .compatible = "lontium,lt8713sx" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, lt8713sx_match_table);
+
+static struct i2c_driver lt8713sx_driver = {
+	.driver = {
+		.name = "lt8713sx",
+		.of_match_table = lt8713sx_match_table,
+		.dev_groups = lt8713sx_attr_groups,
+	},
+	.probe = lt8713sx_probe,
+	.remove = lt8713sx_remove,
+	.id_table = lt8713sx_id,
+};
+
+module_i2c_driver(lt8713sx_driver);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("lt8713sx drm bridge driver");
+MODULE_AUTHOR("Vishnu Saini <vishnu.saini@oss.qualcomm.com>");
+MODULE_FIRMWARE(FW_FILE);
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index 0628d8e737abb..4517aee83332a 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -116,12 +116,25 @@ static int lt9611_mipi_input_digital(struct lt9611 *lt9611,
 		{ 0x830a, 0x00 },
 		{ 0x824f, 0x80 },
 		{ 0x8250, 0x10 },
+		{ 0x8303, 0x00 },
 		{ 0x8302, 0x0a },
 		{ 0x8306, 0x0a },
 	};
 
-	if (lt9611->dsi1_node)
-		reg_cfg[1].def = 0x03;
+	if (lt9611->dsi1_node) {
+		if (lt9611->dsi0_node) {
+			/* Dual port (Port A + B) */
+			reg_cfg[1].def = 0x03;
+		} else {
+			/*
+			 * Single port B:
+			 * - 0x8303 bit 6: port swap (1=PortB as primary)
+			 * - 0x8250 bit 3:2: byte_clk source (01=PortB)
+			 */
+			reg_cfg[3].def = 0x14;
+			reg_cfg[4].def = 0x40;
+		}
+	}
 
 	return regmap_multi_reg_write(lt9611->regmap, reg_cfg, ARRAY_SIZE(reg_cfg));
 }
@@ -202,7 +215,9 @@ static void lt9611_pcr_setup(struct lt9611 *lt9611, const struct drm_display_mod
 	regmap_write(lt9611->regmap, 0x831d, pol);
 
 	regmap_multi_reg_write(lt9611->regmap, reg_cfg, ARRAY_SIZE(reg_cfg));
-	if (lt9611->dsi1_node) {
+
+	/* dual port: configure hact for combining two DSI inputs */
+	if (lt9611->dsi0_node && lt9611->dsi1_node) {
 		unsigned int hact = mode->hdisplay;
 
 		hact >>= 2;
@@ -759,7 +774,8 @@ static enum drm_mode_status lt9611_bridge_mode_valid(struct drm_bridge *bridge,
 	if (mode->hdisplay > 3840)
 		return MODE_BAD_HVALUE;
 
-	if (mode->hdisplay > 2000 && !lt9611->dsi1_node)
+	/* high resolution requires dual port (Port A + B) */
+	if (mode->hdisplay > 2000 && !(lt9611->dsi0_node && lt9611->dsi1_node))
 		return MODE_PANEL;
 
 	return MODE_OK;
@@ -1033,13 +1049,13 @@ static int lt9611_parse_dt(struct device *dev,
 			   struct lt9611 *lt9611)
 {
 	lt9611->dsi0_node = of_graph_get_remote_node(dev->of_node, 0, -1);
-	if (!lt9611->dsi0_node) {
-		dev_err(lt9611->dev, "failed to get remote node for primary dsi\n");
+	lt9611->dsi1_node = of_graph_get_remote_node(dev->of_node, 1, -1);
+
+	if (!lt9611->dsi0_node && !lt9611->dsi1_node) {
+		dev_err(lt9611->dev, "failed to get remote node for dsi\n");
 		return -ENODEV;
 	}
 
-	lt9611->dsi1_node = of_graph_get_remote_node(dev->of_node, 1, -1);
-
 	lt9611->ac_mode = of_property_read_bool(dev->of_node, "lt,ac-mode");
 
 	return drm_of_find_panel_or_bridge(dev->of_node, 2, -1, NULL, &lt9611->next_bridge);
@@ -1161,14 +1177,16 @@ static int lt9611_probe(struct i2c_client *client)
 
 	drm_bridge_add(&lt9611->bridge);
 
-	/* Attach primary DSI */
-	lt9611->dsi0 = lt9611_attach_dsi(lt9611, lt9611->dsi0_node);
-	if (IS_ERR(lt9611->dsi0)) {
-		ret = PTR_ERR(lt9611->dsi0);
-		goto err_remove_bridge;
+	/* Attach primary DSI (directly drives or Port A in dual-port mode) */
+	if (lt9611->dsi0_node) {
+		lt9611->dsi0 = lt9611_attach_dsi(lt9611, lt9611->dsi0_node);
+		if (IS_ERR(lt9611->dsi0)) {
+			ret = PTR_ERR(lt9611->dsi0);
+			goto err_remove_bridge;
+		}
 	}
 
-	/* Attach secondary DSI, if specified */
+	/* Attach secondary DSI (Port B in single or dual-port mode) */
 	if (lt9611->dsi1_node) {
 		lt9611->dsi1 = lt9611_attach_dsi(lt9611, lt9611->dsi1_node);
 		if (IS_ERR(lt9611->dsi1)) {
diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c
index ec632f2686442..c3eb437ef1b0b 100644
--- a/drivers/gpu/drm/bridge/samsung-dsim.c
+++ b/drivers/gpu/drm/bridge/samsung-dsim.c
@@ -1988,9 +1988,7 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
 	return 0;
 
 err_release_next_bridge:
-	drm_bridge_put(dsi->bridge.next_bridge);
-	dsi->bridge.next_bridge = NULL;
-
+	drm_bridge_clear_and_put(&dsi->bridge.next_bridge);
 	if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO))
 		samsung_dsim_unregister_te_irq(dsi);
 err_remove_bridge:
@@ -2007,8 +2005,7 @@ static int samsung_dsim_host_detach(struct mipi_dsi_host *host,
 	if (pdata->host_ops && pdata->host_ops->detach)
 		pdata->host_ops->detach(dsi, device);
 
-	drm_bridge_put(dsi->bridge.next_bridge);
-	dsi->bridge.next_bridge = NULL;
+	drm_bridge_clear_and_put(&dsi->bridge.next_bridge);
 
 	samsung_dsim_unregister_te_irq(dsi);
 
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index d3f238b1f2a94..982306eb4f0a7 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -2221,6 +2221,7 @@ static void sii8620_detach(struct drm_bridge *bridge)
 		return;
 
 	rc_unregister_device(ctx->rc_dev);
+	rc_free_device(ctx->rc_dev);
 }
 
 static int sii8620_is_packing_required(struct sii8620 *ctx,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
index fd23ca2834b0d..e7bef9150f6a8 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
@@ -352,12 +352,6 @@ enum {
 	DW_DP_YCBCR420_16BIT,
 };
 
-enum {
-	DW_DP_MP_SINGLE_PIXEL,
-	DW_DP_MP_DUAL_PIXEL,
-	DW_DP_MP_QUAD_PIXEL,
-};
-
 enum {
 	DW_DP_SDP_VERTICAL_INTERVAL = BIT(0),
 	DW_DP_SDP_HORIZONTAL_INTERVAL = BIT(1),
@@ -1984,7 +1978,7 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 		return ERR_CAST(dp);
 
 	dp->dev = dev;
-	dp->pixel_mode = DW_DP_MP_QUAD_PIXEL;
+	dp->pixel_mode = plat_data->pixel_mode;
 
 	dp->plat_data.max_link_rate = plat_data->max_link_rate;
 	bridge = &dp->bridge;
@@ -2020,13 +2014,13 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 		return ERR_CAST(dp->aux_clk);
 	}
 
-	dp->i2s_clk = devm_clk_get(dev, "i2s");
+	dp->i2s_clk = devm_clk_get_optional(dev, "i2s");
 	if (IS_ERR(dp->i2s_clk)) {
 		dev_err_probe(dev, PTR_ERR(dp->i2s_clk), "failed to get i2s clock\n");
 		return ERR_CAST(dp->i2s_clk);
 	}
 
-	dp->spdif_clk = devm_clk_get(dev, "spdif");
+	dp->spdif_clk = devm_clk_get_optional(dev, "spdif");
 	if (IS_ERR(dp->spdif_clk)) {
 		dev_err_probe(dev, PTR_ERR(dp->spdif_clk), "failed to get spdif clock\n");
 		return ERR_CAST(dp->spdif_clk);
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
index ab7fed6214e06..d649a1cf07f5c 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
@@ -11,6 +11,7 @@
 #include <linux/export.h>
 #include <linux/i2c.h>
 #include <linux/irq.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/of.h>
@@ -748,120 +749,6 @@ static struct i2c_adapter *dw_hdmi_qp_i2c_adapter(struct dw_hdmi_qp *hdmi)
 	return adap;
 }
 
-static int dw_hdmi_qp_config_avi_infoframe(struct dw_hdmi_qp *hdmi,
-					   const u8 *buffer, size_t len)
-{
-	u32 val, i, j;
-
-	if (len != HDMI_INFOFRAME_SIZE(AVI)) {
-		dev_err(hdmi->dev, "failed to configure avi infoframe\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * DW HDMI QP IP uses a different byte format from standard AVI info
-	 * frames, though generally the bits are in the correct bytes.
-	 */
-	val = buffer[1] << 8 | buffer[2] << 16;
-	dw_hdmi_qp_write(hdmi, val, PKT_AVI_CONTENTS0);
-
-	for (i = 0; i < 4; i++) {
-		for (j = 0; j < 4; j++) {
-			if (i * 4 + j >= 14)
-				break;
-			if (!j)
-				val = buffer[i * 4 + j + 3];
-			val |= buffer[i * 4 + j + 3] << (8 * j);
-		}
-
-		dw_hdmi_qp_write(hdmi, val, PKT_AVI_CONTENTS1 + i * 4);
-	}
-
-	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_AVI_FIELDRATE, PKTSCHED_PKT_CONFIG1);
-
-	dw_hdmi_qp_mod(hdmi, PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN,
-		       PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, PKTSCHED_PKT_EN);
-
-	return 0;
-}
-
-static int dw_hdmi_qp_config_drm_infoframe(struct dw_hdmi_qp *hdmi,
-					   const u8 *buffer, size_t len)
-{
-	u32 val, i;
-
-	if (len != HDMI_INFOFRAME_SIZE(DRM)) {
-		dev_err(hdmi->dev, "failed to configure drm infoframe\n");
-		return -EINVAL;
-	}
-
-	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_DRMI_TX_EN, PKTSCHED_PKT_EN);
-
-	val = buffer[1] << 8 | buffer[2] << 16;
-	dw_hdmi_qp_write(hdmi, val, PKT_DRMI_CONTENTS0);
-
-	for (i = 0; i <= buffer[2]; i++) {
-		if (i % 4 == 0)
-			val = buffer[3 + i];
-		val |= buffer[3 + i] << ((i % 4) * 8);
-
-		if ((i % 4 == 3) || i == buffer[2])
-			dw_hdmi_qp_write(hdmi, val,
-					 PKT_DRMI_CONTENTS1 + ((i / 4) * 4));
-	}
-
-	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_DRMI_FIELDRATE, PKTSCHED_PKT_CONFIG1);
-	dw_hdmi_qp_mod(hdmi, PKTSCHED_DRMI_TX_EN, PKTSCHED_DRMI_TX_EN,
-		       PKTSCHED_PKT_EN);
-
-	return 0;
-}
-
-/*
- * Static values documented in the TRM
- * Different values are only used for debug purposes
- */
-#define DW_HDMI_QP_AUDIO_INFOFRAME_HB1	0x1
-#define DW_HDMI_QP_AUDIO_INFOFRAME_HB2	0xa
-
-static int dw_hdmi_qp_config_audio_infoframe(struct dw_hdmi_qp *hdmi,
-					     const u8 *buffer, size_t len)
-{
-	/*
-	 * AUDI_CONTENTS0: { RSV, HB2, HB1, RSV }
-	 * AUDI_CONTENTS1: { PB3, PB2, PB1, PB0 }
-	 * AUDI_CONTENTS2: { PB7, PB6, PB5, PB4 }
-	 *
-	 * PB0: CheckSum
-	 * PB1: | CT3    | CT2  | CT1  | CT0  | F13  | CC2 | CC1 | CC0 |
-	 * PB2: | F27    | F26  | F25  | SF2  | SF1  | SF0 | SS1 | SS0 |
-	 * PB3: | F37    | F36  | F35  | F34  | F33  | F32 | F31 | F30 |
-	 * PB4: | CA7    | CA6  | CA5  | CA4  | CA3  | CA2 | CA1 | CA0 |
-	 * PB5: | DM_INH | LSV3 | LSV2 | LSV1 | LSV0 | F52 | F51 | F50 |
-	 * PB6~PB10: Reserved
-	 *
-	 * AUDI_CONTENTS0 default value defined by HDMI specification,
-	 * and shall only be changed for debug purposes.
-	 */
-	u32 header_bytes = (DW_HDMI_QP_AUDIO_INFOFRAME_HB1 << 8) |
-			  (DW_HDMI_QP_AUDIO_INFOFRAME_HB2 << 16);
-
-	regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS0, &header_bytes, 1);
-	regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS1, &buffer[3], 1);
-	regmap_bulk_write(hdmi->regm, PKT_AUDI_CONTENTS2, &buffer[4], 1);
-
-	/* Enable ACR, AUDI, AMD */
-	dw_hdmi_qp_mod(hdmi,
-		       PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN,
-		       PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN,
-		       PKTSCHED_PKT_EN);
-
-	/* Enable AUDS */
-	dw_hdmi_qp_mod(hdmi, PKTSCHED_AUDS_TX_EN, PKTSCHED_AUDS_TX_EN, PKTSCHED_PKT_EN);
-
-	return 0;
-}
-
 static void dw_hdmi_qp_bridge_atomic_enable(struct drm_bridge *bridge,
 					    struct drm_atomic_state *state)
 {
@@ -970,9 +857,9 @@ static int dw_hdmi_qp_bridge_clear_avi_infoframe(struct drm_bridge *bridge)
 
 static int dw_hdmi_qp_bridge_clear_hdmi_infoframe(struct drm_bridge *bridge)
 {
-	/* FIXME: add support for this InfoFrame */
+	struct dw_hdmi_qp *hdmi = bridge->driver_private;
 
-	drm_warn_once(bridge->encoder->dev, "HDMI VSI not supported\n");
+	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_VSI_TX_EN, PKTSCHED_PKT_EN);
 
 	return 0;
 }
@@ -986,6 +873,15 @@ static int dw_hdmi_qp_bridge_clear_hdr_drm_infoframe(struct drm_bridge *bridge)
 	return 0;
 }
 
+static int dw_hdmi_qp_bridge_clear_spd_infoframe(struct drm_bridge *bridge)
+{
+	struct dw_hdmi_qp *hdmi = bridge->driver_private;
+
+	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_SPDI_TX_EN, PKTSCHED_PKT_EN);
+
+	return 0;
+}
+
 static int dw_hdmi_qp_bridge_clear_audio_infoframe(struct drm_bridge *bridge)
 {
 	struct dw_hdmi_qp *hdmi = bridge->driver_private;
@@ -999,6 +895,32 @@ static int dw_hdmi_qp_bridge_clear_audio_infoframe(struct drm_bridge *bridge)
 	return 0;
 }
 
+static void dw_hdmi_qp_write_pkt(struct dw_hdmi_qp *hdmi, const u8 *buffer,
+				 size_t start, size_t len, unsigned int reg)
+{
+	u32 val = 0;
+	size_t i;
+
+	for (i = start; i < start + len; i++)
+		val |= buffer[i] << ((i % 4) * BITS_PER_BYTE);
+
+	dw_hdmi_qp_write(hdmi, val, reg);
+}
+
+static void dw_hdmi_qp_write_infoframe(struct dw_hdmi_qp *hdmi, const u8 *buffer,
+				       size_t len, unsigned int reg)
+{
+	size_t i;
+
+	/* InfoFrame packet header */
+	dw_hdmi_qp_write_pkt(hdmi, buffer, 1, 2, reg);
+
+	/* InfoFrame packet body */
+	for (i = 0; i < len - 3; i += 4)
+		dw_hdmi_qp_write_pkt(hdmi, buffer + 3, i, min(len - i - 3, 4),
+				     reg + i + 4);
+}
+
 static int dw_hdmi_qp_bridge_write_avi_infoframe(struct drm_bridge *bridge,
 						 const u8 *buffer, size_t len)
 {
@@ -1006,15 +928,27 @@ static int dw_hdmi_qp_bridge_write_avi_infoframe(struct drm_bridge *bridge,
 
 	dw_hdmi_qp_bridge_clear_avi_infoframe(bridge);
 
-	return dw_hdmi_qp_config_avi_infoframe(hdmi, buffer, len);
+	dw_hdmi_qp_write_infoframe(hdmi, buffer, len, PKT_AVI_CONTENTS0);
+
+	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_AVI_FIELDRATE, PKTSCHED_PKT_CONFIG1);
+	dw_hdmi_qp_mod(hdmi, PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN,
+		       PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, PKTSCHED_PKT_EN);
+
+	return 0;
 }
 
 static int dw_hdmi_qp_bridge_write_hdmi_infoframe(struct drm_bridge *bridge,
 						  const u8 *buffer, size_t len)
 {
+	struct dw_hdmi_qp *hdmi = bridge->driver_private;
+
 	dw_hdmi_qp_bridge_clear_hdmi_infoframe(bridge);
 
-	/* FIXME: add support for the HDMI VSI */
+	dw_hdmi_qp_write_infoframe(hdmi, buffer, len, PKT_VSI_CONTENTS0);
+
+	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_VSI_FIELDRATE, PKTSCHED_PKT_CONFIG1);
+	dw_hdmi_qp_mod(hdmi, PKTSCHED_VSI_TX_EN, PKTSCHED_VSI_TX_EN,
+		       PKTSCHED_PKT_EN);
 
 	return 0;
 }
@@ -1026,7 +960,28 @@ static int dw_hdmi_qp_bridge_write_hdr_drm_infoframe(struct drm_bridge *bridge,
 
 	dw_hdmi_qp_bridge_clear_hdr_drm_infoframe(bridge);
 
-	return dw_hdmi_qp_config_drm_infoframe(hdmi, buffer, len);
+	dw_hdmi_qp_write_infoframe(hdmi, buffer, len, PKT_DRMI_CONTENTS0);
+
+	dw_hdmi_qp_mod(hdmi, 0, PKTSCHED_DRMI_FIELDRATE, PKTSCHED_PKT_CONFIG1);
+	dw_hdmi_qp_mod(hdmi, PKTSCHED_DRMI_TX_EN, PKTSCHED_DRMI_TX_EN,
+		       PKTSCHED_PKT_EN);
+
+	return 0;
+}
+
+static int dw_hdmi_qp_bridge_write_spd_infoframe(struct drm_bridge *bridge,
+						 const u8 *buffer, size_t len)
+{
+	struct dw_hdmi_qp *hdmi = bridge->driver_private;
+
+	dw_hdmi_qp_bridge_clear_spd_infoframe(bridge);
+
+	dw_hdmi_qp_write_infoframe(hdmi, buffer, len, PKT_SPDI_CONTENTS0);
+
+	dw_hdmi_qp_mod(hdmi, PKTSCHED_SPDI_TX_EN, PKTSCHED_SPDI_TX_EN,
+		       PKTSCHED_PKT_EN);
+
+	return 0;
 }
 
 static int dw_hdmi_qp_bridge_write_audio_infoframe(struct drm_bridge *bridge,
@@ -1036,7 +991,31 @@ static int dw_hdmi_qp_bridge_write_audio_infoframe(struct drm_bridge *bridge,
 
 	dw_hdmi_qp_bridge_clear_audio_infoframe(bridge);
 
-	return dw_hdmi_qp_config_audio_infoframe(hdmi, buffer, len);
+	/*
+	 * AUDI_CONTENTS0: { RSV, HB2, HB1, RSV }
+	 * AUDI_CONTENTS1: { PB3, PB2, PB1, PB0 }
+	 * AUDI_CONTENTS2: { PB7, PB6, PB5, PB4 }
+	 *
+	 * PB0: CheckSum
+	 * PB1: | CT3    | CT2  | CT1  | CT0  | F13  | CC2 | CC1 | CC0 |
+	 * PB2: | F27    | F26  | F25  | SF2  | SF1  | SF0 | SS1 | SS0 |
+	 * PB3: | F37    | F36  | F35  | F34  | F33  | F32 | F31 | F30 |
+	 * PB4: | CA7    | CA6  | CA5  | CA4  | CA3  | CA2 | CA1 | CA0 |
+	 * PB5: | DM_INH | LSV3 | LSV2 | LSV1 | LSV0 | F52 | F51 | F50 |
+	 * PB6~PB10: Reserved
+	 */
+	dw_hdmi_qp_write_infoframe(hdmi, buffer, len, PKT_AUDI_CONTENTS0);
+
+	/* Enable ACR, AUDI, AMD */
+	dw_hdmi_qp_mod(hdmi,
+		       PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN,
+		       PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN,
+		       PKTSCHED_PKT_EN);
+
+	/* Enable AUDS */
+	dw_hdmi_qp_mod(hdmi, PKTSCHED_AUDS_TX_EN, PKTSCHED_AUDS_TX_EN, PKTSCHED_PKT_EN);
+
+	return 0;
 }
 
 #ifdef CONFIG_DRM_DW_HDMI_QP_CEC
@@ -1227,6 +1206,8 @@ static const struct drm_bridge_funcs dw_hdmi_qp_bridge_funcs = {
 	.hdmi_write_hdmi_infoframe = dw_hdmi_qp_bridge_write_hdmi_infoframe,
 	.hdmi_clear_hdr_drm_infoframe = dw_hdmi_qp_bridge_clear_hdr_drm_infoframe,
 	.hdmi_write_hdr_drm_infoframe = dw_hdmi_qp_bridge_write_hdr_drm_infoframe,
+	.hdmi_clear_spd_infoframe = dw_hdmi_qp_bridge_clear_spd_infoframe,
+	.hdmi_write_spd_infoframe = dw_hdmi_qp_bridge_write_spd_infoframe,
 	.hdmi_clear_audio_infoframe = dw_hdmi_qp_bridge_clear_audio_infoframe,
 	.hdmi_write_audio_infoframe = dw_hdmi_qp_bridge_write_audio_infoframe,
 	.hdmi_audio_startup = dw_hdmi_qp_audio_enable,
@@ -1344,7 +1325,8 @@ struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev,
 			   DRM_BRIDGE_OP_EDID |
 			   DRM_BRIDGE_OP_HDMI |
 			   DRM_BRIDGE_OP_HDMI_AUDIO |
-			   DRM_BRIDGE_OP_HDMI_HDR_DRM_INFOFRAME;
+			   DRM_BRIDGE_OP_HDMI_HDR_DRM_INFOFRAME |
+			   DRM_BRIDGE_OP_HDMI_SPD_INFOFRAME;
 	if (!hdmi->no_hpd)
 		hdmi->bridge.ops |= DRM_BRIDGE_OP_HPD;
 	hdmi->bridge.of_node = pdev->dev.of_node;
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h
index 91a15f82e32ac..c07847e8d7dd0 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h
@@ -198,6 +198,7 @@
 #define PKTSCHED_PRQUEUE2_CONFIG2			0xa94
 #define PKTSCHED_PKT_CONFIG0				0xa98
 #define PKTSCHED_PKT_CONFIG1				0xa9c
+#define PKTSCHED_VSI_FIELDRATE				BIT(14)
 #define PKTSCHED_DRMI_FIELDRATE				BIT(13)
 #define PKTSCHED_AVI_FIELDRATE				BIT(12)
 #define PKTSCHED_PKT_CONFIG2				0xaa0
@@ -205,7 +206,9 @@
 #define PKTSCHED_PKT_EN					0xaa8
 #define PKTSCHED_DRMI_TX_EN				BIT(17)
 #define PKTSCHED_AUDI_TX_EN				BIT(15)
+#define PKTSCHED_SPDI_TX_EN				BIT(14)
 #define PKTSCHED_AVI_TX_EN				BIT(13)
+#define PKTSCHED_VSI_TX_EN				BIT(12)
 #define PKTSCHED_EMP_CVTEM_TX_EN			BIT(10)
 #define PKTSCHED_AMD_TX_EN				BIT(8)
 #define PKTSCHED_GCP_TX_EN				BIT(3)
diff --git a/drivers/gpu/drm/bridge/tda998x_drv.c b/drivers/gpu/drm/bridge/tda998x_drv.c
index e636459d91857..d9b388165de15 100644
--- a/drivers/gpu/drm/bridge/tda998x_drv.c
+++ b/drivers/gpu/drm/bridge/tda998x_drv.c
@@ -4,7 +4,6 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
-#include <linux/component.h>
 #include <linux/gpio/consumer.h>
 #include <linux/hdmi.h>
 #include <linux/i2c.h>
@@ -1194,26 +1193,27 @@ static int tda998x_audio_codec_init(struct tda998x_priv *priv,
 
 /* DRM connector functions */
 
-static enum drm_connector_status
-tda998x_connector_detect(struct drm_connector *connector, bool force)
+static enum drm_connector_status tda998x_conn_detect(struct tda998x_priv *priv)
 {
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
 	u8 val = cec_read(priv, REG_CEC_RXSHPDLEV);
 
 	return (val & CEC_RXSHPDLEV_HPD) ? connector_status_connected :
 			connector_status_disconnected;
 }
 
-static void tda998x_connector_destroy(struct drm_connector *connector)
+static enum drm_connector_status
+tda998x_connector_detect(struct drm_connector *connector, bool force)
 {
-	drm_connector_cleanup(connector);
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+
+	return tda998x_conn_detect(priv);
 }
 
 static const struct drm_connector_funcs tda998x_connector_funcs = {
 	.reset = drm_atomic_helper_connector_reset,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = tda998x_connector_detect,
-	.destroy = tda998x_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
@@ -1282,11 +1282,10 @@ static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
 	return ret;
 }
 
-static int tda998x_connector_get_modes(struct drm_connector *connector)
+static const struct drm_edid *tda998x_edid_read(struct tda998x_priv *priv,
+						struct drm_connector *connector)
 {
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
 	const struct drm_edid *drm_edid;
-	int n;
 
 	/*
 	 * If we get killed while waiting for the HPD timeout, return
@@ -1304,6 +1303,16 @@ static int tda998x_connector_get_modes(struct drm_connector *connector)
 	if (priv->rev == TDA19988)
 		reg_set(priv, REG_TX4, TX4_PD_RAM);
 
+	return drm_edid;
+}
+
+static int tda998x_connector_get_modes(struct drm_connector *connector)
+{
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+	const struct drm_edid *drm_edid;
+	int n;
+
+	drm_edid = tda998x_edid_read(priv, connector);
 	drm_edid_connector_update(connector, drm_edid);
 	cec_notifier_set_phys_addr(priv->cec_notify,
 				   connector->display_info.source_physical_address);
@@ -1371,10 +1380,8 @@ static int tda998x_bridge_attach(struct drm_bridge *bridge,
 {
 	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
-	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) {
-		DRM_ERROR("Fix bridge driver to make connector optional!");
-		return -EINVAL;
-	}
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+		return 0;
 
 	return tda998x_connector_init(priv, bridge->dev);
 }
@@ -1383,7 +1390,8 @@ static void tda998x_bridge_detach(struct drm_bridge *bridge)
 {
 	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
-	drm_connector_cleanup(&priv->connector);
+	if (priv->connector.dev)
+		drm_connector_cleanup(&priv->connector);
 }
 
 static enum drm_mode_status tda998x_bridge_mode_valid(struct drm_bridge *bridge,
@@ -1683,6 +1691,59 @@ static void tda998x_bridge_mode_set(struct drm_bridge *bridge,
 	mutex_unlock(&priv->audio_mutex);
 }
 
+static const struct drm_edid *
+tda998x_bridge_edid_read(struct drm_bridge *bridge,
+			 struct drm_connector *connector)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+	const struct drm_edid *drm_edid;
+	const struct edid *edid;
+
+	drm_edid = tda998x_edid_read(priv, connector);
+	if (!drm_edid) {
+		dev_dbg(&priv->hdmi->dev, "failed to get edid\n");
+		return NULL;
+	}
+
+	/*
+	 * FIXME: This should use connector->display_info.has_audio from
+	 * a path that has read the EDID and called
+	 * drm_edid_connector_update().
+	 */
+	edid = drm_edid_raw(drm_edid);
+
+	dev_dbg(&priv->hdmi->dev, "got edid: width[%d] x height[%d]\n",
+		edid->width_cm, edid->height_cm);
+
+	priv->sink_has_audio = drm_detect_monitor_audio(edid);
+	cec_notifier_set_phys_addr_from_edid(priv->cec_notify, edid);
+
+	return drm_edid;
+}
+
+static enum drm_connector_status
+tda998x_bridge_detect(struct drm_bridge *bridge,
+		      struct drm_connector *connector)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	return tda998x_conn_detect(priv);
+}
+
+static void tda998x_bridge_hpd_enable(struct drm_bridge *bridge)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	cec_write(priv, REG_CEC_RXSHPDINTENA, CEC_RXSHPDLEV_HPD);
+}
+
+static void tda998x_bridge_hpd_disable(struct drm_bridge *bridge)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+}
+
 static const struct drm_bridge_funcs tda998x_bridge_funcs = {
 	.attach = tda998x_bridge_attach,
 	.detach = tda998x_bridge_detach,
@@ -1690,6 +1751,10 @@ static const struct drm_bridge_funcs tda998x_bridge_funcs = {
 	.disable = tda998x_bridge_disable,
 	.mode_set = tda998x_bridge_mode_set,
 	.enable = tda998x_bridge_enable,
+	.edid_read = tda998x_bridge_edid_read,
+	.detect = tda998x_bridge_detect,
+	.hpd_enable = tda998x_bridge_hpd_enable,
+	.hpd_disable = tda998x_bridge_hpd_disable,
 };
 
 /* I2C driver functions */
@@ -1749,38 +1814,20 @@ static int tda998x_get_audio_ports(struct tda998x_priv *priv,
 	return 0;
 }
 
-static void tda998x_destroy(struct device *dev)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-
-	drm_bridge_remove(&priv->bridge);
-
-	/* disable all IRQs and free the IRQ handler */
-	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
-	reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
-
-	if (priv->audio_pdev)
-		platform_device_unregister(priv->audio_pdev);
-
-	if (priv->hdmi->irq)
-		free_irq(priv->hdmi->irq, priv);
-
-	timer_delete_sync(&priv->edid_delay_timer);
-	cancel_work_sync(&priv->detect_work);
-
-	i2c_unregister_device(priv->cec);
-
-	cec_notifier_conn_unregister(priv->cec_notify);
-}
-
-static int tda998x_create(struct device *dev)
+static int
+tda998x_probe(struct i2c_client *client)
 {
-	struct i2c_client *client = to_i2c_client(dev);
 	struct device_node *np = client->dev.of_node;
+	struct device *dev = &client->dev;
 	struct i2c_board_info cec_info;
 	struct tda998x_priv *priv;
-	u32 video;
 	int rev_lo, rev_hi, ret;
+	u32 video;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_warn(&client->dev, "adapter does not support I2C\n");
+		return -EIO;
+	}
 
 	priv = devm_drm_bridge_alloc(dev, struct tda998x_priv, bridge, &tda998x_bridge_funcs);
 	if (IS_ERR(priv))
@@ -1815,13 +1862,15 @@ static int tda998x_create(struct device *dev)
 	rev_lo = reg_read(priv, REG_VERSION_LSB);
 	if (rev_lo < 0) {
 		dev_err(dev, "failed to read version: %d\n", rev_lo);
-		return rev_lo;
+		ret = rev_lo;
+		goto cancel_work;
 	}
 
 	rev_hi = reg_read(priv, REG_VERSION_MSB);
 	if (rev_hi < 0) {
 		dev_err(dev, "failed to read version: %d\n", rev_hi);
-		return rev_hi;
+		ret = rev_hi;
+		goto cancel_work;
 	}
 
 	priv->rev = rev_lo | rev_hi << 8;
@@ -1844,7 +1893,8 @@ static int tda998x_create(struct device *dev)
 		break;
 	default:
 		dev_err(dev, "found unsupported device: %04x\n", priv->rev);
-		return -ENXIO;
+		ret = -ENXIO;
+		goto cancel_work;
 	}
 
 	/* after reset, enable DDC: */
@@ -1888,17 +1938,18 @@ static int tda998x_create(struct device *dev)
 		if (ret) {
 			dev_err(dev, "failed to request IRQ#%u: %d\n",
 				client->irq, ret);
-			goto err_irq;
+			goto cancel_work;
 		}
 
 		/* enable HPD irq */
 		cec_write(priv, REG_CEC_RXSHPDINTENA, CEC_RXSHPDLEV_HPD);
+		priv->bridge.ops = DRM_BRIDGE_OP_HPD;
 	}
 
 	priv->cec_notify = cec_notifier_conn_register(dev, NULL, NULL);
 	if (!priv->cec_notify) {
 		ret = -ENOMEM;
-		goto fail;
+		goto free_irq;
 	}
 
 	priv->cec_glue.parent = dev;
@@ -1925,7 +1976,7 @@ static int tda998x_create(struct device *dev)
 	priv->cec = i2c_new_client_device(client->adapter, &cec_info);
 	if (IS_ERR(priv->cec)) {
 		ret = PTR_ERR(priv->cec);
-		goto fail;
+		goto notifier_conn_unregister;
 	}
 
 	/* enable EDID read irq: */
@@ -1942,7 +1993,7 @@ static int tda998x_create(struct device *dev)
 
 		ret = tda998x_get_audio_ports(priv, np);
 		if (ret)
-			goto fail;
+			goto unregister_dev;
 
 		if (priv->audio_port_enable[AUDIO_ROUTE_I2S] ||
 		    priv->audio_port_enable[AUDIO_ROUTE_SPDIF])
@@ -1953,96 +2004,50 @@ static int tda998x_create(struct device *dev)
 	priv->bridge.of_node = dev->of_node;
 #endif
 
+	priv->bridge.ops |= DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT;
+	priv->bridge.type = DRM_MODE_CONNECTOR_HDMIA;
 	drm_bridge_add(&priv->bridge);
 
 	return 0;
 
-fail:
-	tda998x_destroy(dev);
-err_irq:
-	return ret;
-}
-
-/* DRM encoder functions */
-
-static int tda998x_encoder_init(struct device *dev, struct drm_device *drm)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-	u32 crtcs = 0;
-	int ret;
-
-	if (dev->of_node)
-		crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
-
-	/* If no CRTCs were found, fall back to our old behaviour */
-	if (crtcs == 0) {
-		dev_warn(dev, "Falling back to first CRTC\n");
-		crtcs = 1 << 0;
+unregister_dev:
+	i2c_unregister_device(priv->cec);
+notifier_conn_unregister:
+	cec_notifier_conn_unregister(priv->cec_notify);
+free_irq:
+	if (client->irq) {
+		cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+		reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
+		free_irq(client->irq, priv);
 	}
-
-	priv->encoder.possible_crtcs = crtcs;
-
-	ret = drm_simple_encoder_init(drm, &priv->encoder,
-				      DRM_MODE_ENCODER_TMDS);
-	if (ret)
-		goto err_encoder;
-
-	ret = drm_bridge_attach(&priv->encoder, &priv->bridge, NULL, 0);
-	if (ret)
-		goto err_bridge;
-
-	return 0;
-
-err_bridge:
-	drm_encoder_cleanup(&priv->encoder);
-err_encoder:
+cancel_work:
+	timer_delete_sync(&priv->edid_delay_timer);
+	cancel_work_sync(&priv->detect_work);
 	return ret;
 }
 
-static int tda998x_bind(struct device *dev, struct device *master, void *data)
+static void tda998x_remove(struct i2c_client *client)
 {
-	struct drm_device *drm = data;
-
-	return tda998x_encoder_init(dev, drm);
-}
+	struct tda998x_priv *priv = dev_get_drvdata(&client->dev);
 
-static void tda998x_unbind(struct device *dev, struct device *master,
-			   void *data)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
+	drm_bridge_remove(&priv->bridge);
 
-	drm_encoder_cleanup(&priv->encoder);
-}
+	if (priv->audio_pdev)
+		platform_device_unregister(priv->audio_pdev);
 
-static const struct component_ops tda998x_ops = {
-	.bind = tda998x_bind,
-	.unbind = tda998x_unbind,
-};
+	i2c_unregister_device(priv->cec);
 
-static int
-tda998x_probe(struct i2c_client *client)
-{
-	int ret;
+	cec_notifier_conn_unregister(priv->cec_notify);
 
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
-		dev_warn(&client->dev, "adapter does not support I2C\n");
-		return -EIO;
+	/* disable all IRQs and free the IRQ handler */
+	if (client->irq) {
+		cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+		reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
+		free_irq(priv->hdmi->irq, priv);
 	}
 
-	ret = tda998x_create(&client->dev);
-	if (ret)
-		return ret;
-
-	ret = component_add(&client->dev, &tda998x_ops);
-	if (ret)
-		tda998x_destroy(&client->dev);
-	return ret;
-}
-
-static void tda998x_remove(struct i2c_client *client)
-{
-	component_del(&client->dev, &tda998x_ops);
-	tda998x_destroy(&client->dev);
+	timer_delete_sync(&priv->edid_delay_timer);
+	cancel_work_sync(&priv->detect_work);
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/gpu/drm/bridge/th1520-dw-hdmi.c b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c
new file mode 100644
index 0000000000000..389eead5f1c45
--- /dev/null
+++ b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on rcar_dw_hdmi.c, which is:
+ *   Copyright (C) 2016 Renesas Electronics Corporation
+ * Based on imx8mp-hdmi-tx.c, which is:
+ *   Copyright (C) 2022 Pengutronix, Lucas Stach <kernel@pengutronix.de>
+ */
+
+#include <linux/clk.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#include <drm/bridge/dw_hdmi.h>
+#include <drm/drm_modes.h>
+
+#define TH1520_HDMI_PHY_OPMODE_PLLCFG	0x06	/* Mode of operation and PLL dividers */
+#define TH1520_HDMI_PHY_CKSYMTXCTRL	0x09	/* Clock Symbol and Transmitter Control Register */
+#define TH1520_HDMI_PHY_VLEVCTRL	0x0e	/* Voltage Level Control Register */
+#define TH1520_HDMI_PHY_PLLCURRGMPCTRL	0x10	/* PLL current and Gmp (conductance) */
+#define TH1520_HDMI_PHY_PLLDIVCTRL	0x11	/* PLL dividers */
+#define TH1520_HDMI_PHY_TXTERM		0x19	/* Transmission Termination Register */
+
+struct th1520_hdmi_phy_params {
+	unsigned long mpixelclock;
+	u16 opmode_pllcfg;
+	u16 pllcurrgmpctrl;
+	u16 plldivctrl;
+	u16 cksymtxctrl;
+	u16 vlevctrl;
+	u16 txterm;
+};
+
+static const struct th1520_hdmi_phy_params th1520_hdmi_phy_params[] = {
+	{ 35500000,  0x0003, 0x0283, 0x0628, 0x8088, 0x01a0, 0x0007 },
+	{ 44900000,  0x0003, 0x0285, 0x0228, 0x8088, 0x01a0, 0x0007 },
+	{ 71000000,  0x0002, 0x1183, 0x0614, 0x8088, 0x01a0, 0x0007 },
+	{ 90000000,  0x0002, 0x1142, 0x0214, 0x8088, 0x01a0, 0x0007 },
+	{ 121750000, 0x0001, 0x20c0, 0x060a, 0x8088, 0x01a0, 0x0007 },
+	{ 165000000, 0x0001, 0x2080, 0x020a, 0x8088, 0x01a0, 0x0007 },
+	{ 198000000, 0x0000, 0x3040, 0x0605, 0x83c8, 0x0120, 0x0004 },
+	{ 297000000, 0x0000, 0x3041, 0x0205, 0x81dc, 0x0200, 0x0005 },
+	{ 371250000, 0x0640, 0x3041, 0x0205, 0x80f6, 0x0140, 0x0000 },
+	{ 495000000, 0x0640, 0x3080, 0x0005, 0x80f6, 0x0140, 0x0000 },
+	{ 594000000, 0x0640, 0x3080, 0x0005, 0x80fa, 0x01e0, 0x0004 },
+};
+
+struct th1520_hdmi {
+	struct dw_hdmi_plat_data plat_data;
+	struct dw_hdmi *dw_hdmi;
+	struct clk *pixclk;
+	struct reset_control *mainrst, *prst;
+};
+
+static enum drm_mode_status
+th1520_hdmi_mode_valid(struct dw_hdmi *hdmi, void *data,
+		       const struct drm_display_info *info,
+		       const struct drm_display_mode *mode)
+{
+	/*
+	 * The maximum supported clock frequency is 594 MHz, as shown in the PHY
+	 * parameters table.
+	 */
+	if (mode->clock > 594000)
+		return MODE_CLOCK_HIGH;
+
+	return MODE_OK;
+}
+
+static void th1520_hdmi_phy_set_params(struct dw_hdmi *hdmi,
+				const struct th1520_hdmi_phy_params *params)
+{
+	dw_hdmi_phy_i2c_write(hdmi, params->opmode_pllcfg,
+			      TH1520_HDMI_PHY_OPMODE_PLLCFG);
+	dw_hdmi_phy_i2c_write(hdmi, params->pllcurrgmpctrl,
+			      TH1520_HDMI_PHY_PLLCURRGMPCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, params->plldivctrl,
+			      TH1520_HDMI_PHY_PLLDIVCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, params->vlevctrl,
+			      TH1520_HDMI_PHY_VLEVCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, params->cksymtxctrl,
+			      TH1520_HDMI_PHY_CKSYMTXCTRL);
+	dw_hdmi_phy_i2c_write(hdmi, params->txterm,
+			      TH1520_HDMI_PHY_TXTERM);
+}
+
+static int th1520_hdmi_phy_configure(struct dw_hdmi *hdmi, void *data,
+				     unsigned long mpixelclock)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(th1520_hdmi_phy_params); i++) {
+		if (mpixelclock <= th1520_hdmi_phy_params[i].mpixelclock) {
+			th1520_hdmi_phy_set_params(hdmi,
+						   &th1520_hdmi_phy_params[i]);
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int th1520_dw_hdmi_probe(struct platform_device *pdev)
+{
+	struct th1520_hdmi *hdmi;
+	struct dw_hdmi_plat_data *plat_data;
+	struct device *dev = &pdev->dev;
+
+	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	plat_data = &hdmi->plat_data;
+
+	hdmi->pixclk = devm_clk_get_enabled(dev, "pix");
+	if (IS_ERR(hdmi->pixclk))
+		return dev_err_probe(dev, PTR_ERR(hdmi->pixclk),
+				     "Unable to get pixel clock\n");
+
+	hdmi->mainrst = devm_reset_control_get_exclusive_deasserted(dev, "main");
+	if (IS_ERR(hdmi->mainrst))
+		return dev_err_probe(dev, PTR_ERR(hdmi->mainrst),
+				     "Unable to get main reset\n");
+
+	hdmi->prst = devm_reset_control_get_exclusive_deasserted(dev, "apb");
+	if (IS_ERR(hdmi->prst))
+		return dev_err_probe(dev, PTR_ERR(hdmi->prst),
+				     "Unable to get apb reset\n");
+
+	plat_data->output_port = 1;
+	plat_data->mode_valid = th1520_hdmi_mode_valid;
+	plat_data->configure_phy = th1520_hdmi_phy_configure;
+	plat_data->priv_data = hdmi;
+
+	hdmi->dw_hdmi = dw_hdmi_probe(pdev, plat_data);
+	if (IS_ERR(hdmi))
+		return PTR_ERR(hdmi);
+
+	platform_set_drvdata(pdev, hdmi);
+
+	return 0;
+}
+
+static void th1520_dw_hdmi_remove(struct platform_device *pdev)
+{
+	struct dw_hdmi *hdmi = platform_get_drvdata(pdev);
+
+	dw_hdmi_remove(hdmi);
+}
+
+static const struct of_device_id th1520_dw_hdmi_of_table[] = {
+	{ .compatible = "thead,th1520-dw-hdmi" },
+	{ /* Sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, th1520_dw_hdmi_of_table);
+
+static struct platform_driver th1520_dw_hdmi_platform_driver = {
+	.probe		= th1520_dw_hdmi_probe,
+	.remove		= th1520_dw_hdmi_remove,
+	.driver		= {
+		.name	= "th1520-dw-hdmi",
+		.of_match_table = th1520_dw_hdmi_of_table,
+	},
+};
+
+module_platform_driver(th1520_dw_hdmi_platform_driver);
+
+MODULE_AUTHOR("Icenowy Zheng <uwu@icenowy.me>");
+MODULE_DESCRIPTION("T-Head TH1520 HDMI Encoder Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/waveshare-dsi.c b/drivers/gpu/drm/bridge/waveshare-dsi.c
index 43f4e7412d722..32d40414adb9f 100644
--- a/drivers/gpu/drm/bridge/waveshare-dsi.c
+++ b/drivers/gpu/drm/bridge/waveshare-dsi.c
@@ -66,7 +66,14 @@ static int ws_bridge_attach_dsi(struct ws_bridge *ws)
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
 			  MIPI_DSI_CLOCK_NON_CONTINUOUS;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->lanes = 2;
+	ret = drm_of_get_data_lanes_count_ep(dev->of_node, 0, 0, 1, 4);
+	if (ret < 0) {
+		dev_warn(dev, "Invalid or missing DSI lane count %d, falling back to 2 lanes\n",
+			 ret);
+		dsi->lanes = 2;	/* Old DT backward compatibility */
+	} else {
+		dsi->lanes = ret;
+	}
 
 	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0)
@@ -80,11 +87,6 @@ static int ws_bridge_bridge_attach(struct drm_bridge *bridge,
 				   enum drm_bridge_attach_flags flags)
 {
 	struct ws_bridge *ws = bridge_to_ws_bridge(bridge);
-	int ret;
-
-	ret = ws_bridge_attach_dsi(ws);
-	if (ret)
-		return ret;
 
 	return drm_bridge_attach(encoder, ws->next_bridge,
 				 &ws->bridge, flags);
@@ -179,7 +181,7 @@ static int ws_bridge_probe(struct i2c_client *i2c)
 	ws->bridge.of_node = dev->of_node;
 	devm_drm_bridge_add(dev, &ws->bridge);
 
-	return 0;
+	return ws_bridge_attach_dsi(ws);
 }
 
 static const struct of_device_id ws_bridge_of_ids[] = {
diff --git a/drivers/gpu/drm/display/drm_bridge_connector.c b/drivers/gpu/drm/display/drm_bridge_connector.c
index 258132c6b8b56..f686aa5c0ed9b 100644
--- a/drivers/gpu/drm/display/drm_bridge_connector.c
+++ b/drivers/gpu/drm/display/drm_bridge_connector.c
@@ -826,9 +826,19 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 		if (!bridge->ycbcr_420_allowed)
 			connector->ycbcr_420_allowed = false;
 
-		if (bridge->ops & DRM_BRIDGE_OP_EDID) {
+		/*
+		 * Ensure the last bridge declares OP_EDID or OP_MODES or both.
+		 */
+		if (bridge->ops & DRM_BRIDGE_OP_EDID || bridge->ops & DRM_BRIDGE_OP_MODES) {
 			drm_bridge_put(bridge_connector->bridge_edid);
-			bridge_connector->bridge_edid = drm_bridge_get(bridge);
+			bridge_connector->bridge_edid = NULL;
+			drm_bridge_put(bridge_connector->bridge_modes);
+			bridge_connector->bridge_modes = NULL;
+
+			if (bridge->ops & DRM_BRIDGE_OP_EDID)
+				bridge_connector->bridge_edid = drm_bridge_get(bridge);
+			if (bridge->ops & DRM_BRIDGE_OP_MODES)
+				bridge_connector->bridge_modes = drm_bridge_get(bridge);
 		}
 		if (bridge->ops & DRM_BRIDGE_OP_HPD) {
 			drm_bridge_put(bridge_connector->bridge_hpd);
@@ -838,10 +848,6 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 			drm_bridge_put(bridge_connector->bridge_detect);
 			bridge_connector->bridge_detect = drm_bridge_get(bridge);
 		}
-		if (bridge->ops & DRM_BRIDGE_OP_MODES) {
-			drm_bridge_put(bridge_connector->bridge_modes);
-			bridge_connector->bridge_modes = drm_bridge_get(bridge);
-		}
 		if (bridge->ops & DRM_BRIDGE_OP_HDMI) {
 			if (bridge_connector->bridge_hdmi)
 				return ERR_PTR(-EBUSY);
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 170113520a432..8757972e8e242 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -5184,6 +5184,28 @@ static void drm_dp_mst_destroy_state(struct drm_private_obj *obj,
 	kfree(mst_state);
 }
 
+static struct drm_private_state *
+drm_dp_mst_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct drm_dp_mst_topology_mgr *mgr =
+		to_dp_mst_topology_mgr(obj);
+	struct drm_dp_mst_topology_state *mst_state;
+
+	mst_state = kzalloc_obj(*mst_state);
+	if (!mst_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &mst_state->base);
+
+	mst_state->total_avail_slots = 63;
+	mst_state->start_slot = 1;
+
+	mst_state->mgr = mgr;
+	INIT_LIST_HEAD(&mst_state->payloads);
+
+	return &mst_state->base;
+}
+
 static bool drm_dp_mst_port_downstream_of_branch(struct drm_dp_mst_port *port,
 						 struct drm_dp_mst_branch *branch)
 {
@@ -5620,6 +5642,7 @@ int drm_dp_mst_atomic_check(struct drm_atomic_state *state)
 EXPORT_SYMBOL(drm_dp_mst_atomic_check);
 
 const struct drm_private_state_funcs drm_dp_mst_topology_state_funcs = {
+	.atomic_create_state = drm_dp_mst_atomic_create_state,
 	.atomic_duplicate_state = drm_dp_mst_duplicate_state,
 	.atomic_destroy_state = drm_dp_mst_destroy_state,
 };
@@ -5708,8 +5731,6 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 				 int max_dpcd_transaction_bytes, int max_payloads,
 				 int conn_base_id)
 {
-	struct drm_dp_mst_topology_state *mst_state;
-
 	mutex_init(&mgr->lock);
 	mutex_init(&mgr->qlock);
 	mutex_init(&mgr->delayed_destroy_lock);
@@ -5743,18 +5764,7 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 	mgr->max_payloads = max_payloads;
 	mgr->conn_base_id = conn_base_id;
 
-	mst_state = kzalloc_obj(*mst_state);
-	if (mst_state == NULL)
-		return -ENOMEM;
-
-	mst_state->total_avail_slots = 63;
-	mst_state->start_slot = 1;
-
-	mst_state->mgr = mgr;
-	INIT_LIST_HEAD(&mst_state->payloads);
-
 	drm_atomic_private_obj_init(dev, &mgr->base,
-				    &mst_state->base,
 				    &drm_dp_mst_topology_state_funcs);
 
 	return 0;
diff --git a/drivers/gpu/drm/display/drm_dp_tunnel.c b/drivers/gpu/drm/display/drm_dp_tunnel.c
index 7aee574169029..6519b42447285 100644
--- a/drivers/gpu/drm/display/drm_dp_tunnel.c
+++ b/drivers/gpu/drm/display/drm_dp_tunnel.c
@@ -1497,7 +1497,22 @@ static void tunnel_group_destroy_state(struct drm_private_obj *obj, struct drm_p
 	free_group_state(to_group_state(state));
 }
 
+static struct drm_private_state *tunnel_group_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct drm_dp_tunnel_group_state *group_state;
+
+	group_state = kzalloc_obj(*group_state);
+	if (!group_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &group_state->base);
+	INIT_LIST_HEAD(&group_state->tunnel_states);
+
+	return &group_state->base;
+}
+
 static const struct drm_private_state_funcs tunnel_group_funcs = {
+	.atomic_create_state = tunnel_group_atomic_create_state,
 	.atomic_duplicate_state = tunnel_group_duplicate_state,
 	.atomic_destroy_state = tunnel_group_destroy_state,
 };
@@ -1581,19 +1596,11 @@ EXPORT_SYMBOL(drm_dp_tunnel_atomic_get_new_state);
 
 static bool init_group(struct drm_dp_tunnel_mgr *mgr, struct drm_dp_tunnel_group *group)
 {
-	struct drm_dp_tunnel_group_state *group_state;
-
-	group_state = kzalloc_obj(*group_state);
-	if (!group_state)
-		return false;
-
-	INIT_LIST_HEAD(&group_state->tunnel_states);
-
 	group->mgr = mgr;
 	group->available_bw = -1;
 	INIT_LIST_HEAD(&group->tunnels);
 
-	drm_atomic_private_obj_init(mgr->dev, &group->base, &group_state->base,
+	drm_atomic_private_obj_init(mgr->dev, &group->base,
 				    &tunnel_group_funcs);
 
 	return true;
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index f2cd2e25f0098..41c57063f3b4d 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -475,6 +475,7 @@ static void drm_atomic_crtc_print_state(struct drm_printer *p,
 	drm_printf(p, "\tconnector_mask=%x\n", state->connector_mask);
 	drm_printf(p, "\tencoder_mask=%x\n", state->encoder_mask);
 	drm_printf(p, "\tmode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(&state->mode));
+	drm_printf(p, "\tbackground_color=%llx\n", state->background_color);
 
 	if (crtc->funcs->atomic_print_state)
 		crtc->funcs->atomic_print_state(p, state);
@@ -920,29 +921,35 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
  * drm_atomic_private_obj_init - initialize private object
  * @dev: DRM device this object will be attached to
  * @obj: private object
- * @state: initial private object state
  * @funcs: pointer to the struct of function pointers that identify the object
  * type
  *
  * Initialize the private object, which can be embedded into any
  * driver private object that needs its own atomic state.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
  */
-void
-drm_atomic_private_obj_init(struct drm_device *dev,
-			    struct drm_private_obj *obj,
-			    struct drm_private_state *state,
-			    const struct drm_private_state_funcs *funcs)
+int drm_atomic_private_obj_init(struct drm_device *dev,
+				struct drm_private_obj *obj,
+				const struct drm_private_state_funcs *funcs)
 {
+	struct drm_private_state *state;
 	memset(obj, 0, sizeof(*obj));
 
 	drm_modeset_lock_init(&obj->lock);
 
 	obj->dev = dev;
-	obj->state = state;
 	obj->funcs = funcs;
 	list_add_tail(&obj->head, &dev->mode_config.privobj_list);
 
-	state->obj = obj;
+	state = obj->funcs->atomic_create_state(obj);
+	if (IS_ERR(state))
+		return PTR_ERR(state);
+
+	obj->state = state;
+
+	return 0;
 }
 EXPORT_SYMBOL(drm_atomic_private_obj_init);
 
@@ -1569,6 +1576,7 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state,
 	const struct drm_crtc_state *old_crtc_state =
 		drm_atomic_get_old_crtc_state(state, crtc);
 	struct drm_plane *plane;
+	int ret;
 
 	WARN_ON(!drm_atomic_get_new_crtc_state(state, crtc));
 
@@ -1582,6 +1590,12 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state,
 
 		if (IS_ERR(plane_state))
 			return PTR_ERR(plane_state);
+
+		if (plane_state->color_pipeline) {
+			ret = drm_atomic_add_affected_colorops(state, plane);
+			if (ret)
+				return ret;
+		}
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index af02c409c2f60..26953ed6b53e8 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -2301,13 +2301,13 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	 * current layout.
 	 *
 	 * NOTE: Commit work has multiple phases, first hardware commit, then
-	 * cleanup. We want them to overlap, hence need system_unbound_wq to
+	 * cleanup. We want them to overlap, hence need system_dfl_wq to
 	 * make sure work items don't artificially stall on each another.
 	 */
 
 	drm_atomic_state_get(state);
 	if (nonblock)
-		queue_work(system_unbound_wq, &state->commit_work);
+		queue_work(system_dfl_wq, &state->commit_work);
 	else
 		commit_tail(state);
 
@@ -2340,7 +2340,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
  *
  * Asynchronous workers need to have sufficient parallelism to be able to run
  * different atomic commits on different CRTCs in parallel. The simplest way to
- * achieve this is by running them on the &system_unbound_wq work queue. Note
+ * achieve this is by running them on the &system_dfl_wq work queue. Note
  * that drivers are not required to split up atomic commits and run an
  * individual commit in parallel - userspace is supposed to do that if it cares.
  * But it might be beneficial to do that for modesets, since those necessarily
diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c
index 41f916f11a766..76746ad4a1bbb 100644
--- a/drivers/gpu/drm/drm_atomic_state_helper.c
+++ b/drivers/gpu/drm/drm_atomic_state_helper.c
@@ -75,6 +75,7 @@ __drm_atomic_helper_crtc_state_reset(struct drm_crtc_state *crtc_state,
 				     struct drm_crtc *crtc)
 {
 	crtc_state->crtc = crtc;
+	crtc_state->background_color = DRM_ARGB64_PREP(0xffff, 0, 0, 0);
 }
 EXPORT_SYMBOL(__drm_atomic_helper_crtc_state_reset);
 
@@ -713,6 +714,28 @@ void drm_atomic_helper_connector_destroy_state(struct drm_connector *connector,
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_destroy_state);
 
+/**
+ * __drm_atomic_helper_private_obj_create_state - initializes private object state
+ * @obj: private object
+ * @state: new state to initialize
+ *
+ * Initializes the newly allocated @state, usually required when
+ * initializing the drivers.
+ *
+ * @obj is assumed to be zeroed.
+ *
+ * This is useful for drivers that use private states.
+ */
+void __drm_atomic_helper_private_obj_create_state(struct drm_private_obj *obj,
+						  struct drm_private_state *state)
+{
+	if (state)
+		state->obj = obj;
+
+	obj->state = state;
+}
+EXPORT_SYMBOL(__drm_atomic_helper_private_obj_create_state);
+
 /**
  * __drm_atomic_helper_private_obj_duplicate_state - copy atomic private state
  * @obj: CRTC object
@@ -802,6 +825,7 @@ void __drm_atomic_helper_bridge_reset(struct drm_bridge *bridge,
 				      struct drm_bridge_state *state)
 {
 	memset(state, 0, sizeof(*state));
+	__drm_atomic_helper_private_obj_create_state(&bridge->base, &state->base);
 	state->bridge = bridge;
 }
 EXPORT_SYMBOL(__drm_atomic_helper_bridge_reset);
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index 87de41fb44593..5bd5bf6661df7 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -454,6 +454,8 @@ static int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 					&replaced);
 		state->color_mgmt_changed |= replaced;
 		return ret;
+	} else if (property == config->background_color_property) {
+		state->background_color = val;
 	} else if (property == config->prop_out_fence_ptr) {
 		s32 __user *fence_ptr = u64_to_user_ptr(val);
 
@@ -501,6 +503,8 @@ drm_atomic_crtc_get_property(struct drm_crtc *crtc,
 		*val = (state->ctm) ? state->ctm->base.id : 0;
 	else if (property == config->gamma_lut_property)
 		*val = (state->gamma_lut) ? state->gamma_lut->base.id : 0;
+	else if (property == config->background_color_property)
+		*val = state->background_color;
 	else if (property == config->prop_out_fence_ptr)
 		*val = 0;
 	else if (property == crtc->scaling_filter_property)
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 3b1f5f72885e6..1f3af27d2418e 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -191,10 +191,6 @@
  *		 plane does not expose the "alpha" property, then this is
  *		 assumed to be 1.0
  *
- * Note that all the property extensions described here apply either to the
- * plane or the CRTC (e.g. for the background color, which currently is not
- * exposed and assumed to be black).
- *
  * SCALING_FILTER:
  *     Indicates scaling filter to be used for plane scaler
  *
@@ -207,6 +203,25 @@
  *
  * Drivers can set up this property for a plane by calling
  * drm_plane_create_scaling_filter_property
+ *
+ * The property extensions described above all apply to the plane.  Drivers
+ * may also expose the following crtc property extension:
+ *
+ * BACKGROUND_COLOR:
+ *	Background color is set up with drm_crtc_attach_background_color_property(),
+ *	and expects a 64-bit ARGB value following DRM_FORMAT_ARGB16161616, as
+ *	generated by the DRM_ARGB64_PREP*() helpers. It controls the color of a
+ *	full-screen layer that exists below all planes. This color will be used
+ *	for pixels not covered by any plane and may also be blended with plane
+ *	contents as allowed by a plane's alpha values.
+ *	The background color defaults to black, and is assumed to be black for
+ *	drivers that do not expose this property. Although background color
+ *	isn't a plane, it is assumed that the color provided here undergoes the
+ *	CRTC degamma/CSC/gamma transformations applied after the planes blending.
+ *	Note that the color value includes an alpha channel, hence non-opaque
+ *	background color values are allowed, but since physically transparent
+ *	monitors do not (yet) exists, the final alpha value may not reach the
+ *	video sink or it may simply ignore it.
  */
 
 /**
@@ -621,3 +636,19 @@ int drm_plane_create_blend_mode_property(struct drm_plane *plane,
 	return 0;
 }
 EXPORT_SYMBOL(drm_plane_create_blend_mode_property);
+
+/**
+ * drm_crtc_attach_background_color_property - attach background color property
+ * @crtc: drm crtc
+ *
+ * Attaches the background color property to @crtc.  The property defaults to
+ * solid black and will accept 64-bit ARGB values in the format generated by
+ * DRM_ARGB64_PREP*() helpers.
+ */
+void drm_crtc_attach_background_color_property(struct drm_crtc *crtc)
+{
+	drm_object_attach_property(&crtc->base,
+				   crtc->dev->mode_config.background_color_property,
+				   DRM_ARGB64_PREP(0xffff, 0, 0, 0));
+}
+EXPORT_SYMBOL(drm_crtc_attach_background_color_property);
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index d6f11c68bb6a4..30d957675d87b 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -304,6 +304,9 @@ EXPORT_SYMBOL(drm_bridge_get);
  *
  * This function decrements the bridge's reference count and frees the
  * object if the reference count drops to zero.
+ *
+ * See also drm_bridge_clear_and_put() if you also need to set the pointer
+ * to NULL
  */
 void drm_bridge_put(struct drm_bridge *bridge)
 {
@@ -312,6 +315,37 @@ void drm_bridge_put(struct drm_bridge *bridge)
 }
 EXPORT_SYMBOL(drm_bridge_put);
 
+/**
+ * drm_bridge_clear_and_put - Given a bridge pointer, clear the pointer
+ *                            then put the bridge
+ * @bridge_pp: pointer to pointer to a struct drm_bridge; ``bridge_pp``
+ *             must be non-NULL; if ``*bridge_pp`` is NULL this function
+ *             does nothing
+ *
+ * Helper to put a DRM bridge, but only after setting its pointer to
+ * NULL. Useful when a struct drm_bridge reference must be dropped without
+ * leaving a use-after-free window where the pointed bridge might have been
+ * freed while still holding a pointer to it.
+ *
+ * For struct ``drm_bridge *some_bridge``, this code::
+ *
+ *     drm_bridge_clear_and_put(&some_bridge);
+ *
+ * is equivalent to the more complex::
+ *
+ *     struct drm_bridge *temp = some_bridge;
+ *     some_bridge = NULL;
+ *     drm_bridge_put(temp);
+ */
+void drm_bridge_clear_and_put(struct drm_bridge **bridge_pp)
+{
+	struct drm_bridge *bridge = *bridge_pp;
+
+	*bridge_pp = NULL;
+	drm_bridge_put(bridge);
+}
+EXPORT_SYMBOL(drm_bridge_clear_and_put);
+
 /**
  * drm_bridge_put_void - wrapper to drm_bridge_put() taking a void pointer
  *
@@ -460,7 +494,21 @@ drm_bridge_atomic_destroy_priv_state(struct drm_private_obj *obj,
 	bridge->funcs->atomic_destroy_state(bridge, state);
 }
 
+static struct drm_private_state *
+drm_bridge_atomic_create_priv_state(struct drm_private_obj *obj)
+{
+	struct drm_bridge *bridge = drm_priv_to_bridge(obj);
+	struct drm_bridge_state *state;
+
+	state = bridge->funcs->atomic_reset(bridge);
+	if (IS_ERR(state))
+		return ERR_CAST(state);
+
+	return &state->base;
+}
+
 static const struct drm_private_state_funcs drm_bridge_priv_state_funcs = {
+	.atomic_create_state = drm_bridge_atomic_create_priv_state,
 	.atomic_duplicate_state = drm_bridge_atomic_duplicate_priv_state,
 	.atomic_destroy_state = drm_bridge_atomic_destroy_priv_state,
 };
@@ -537,26 +585,12 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge,
 			goto err_reset_bridge;
 	}
 
-	if (drm_bridge_is_atomic(bridge)) {
-		struct drm_bridge_state *state;
-
-		state = bridge->funcs->atomic_reset(bridge);
-		if (IS_ERR(state)) {
-			ret = PTR_ERR(state);
-			goto err_detach_bridge;
-		}
-
+	if (drm_bridge_is_atomic(bridge))
 		drm_atomic_private_obj_init(bridge->dev, &bridge->base,
-					    &state->base,
 					    &drm_bridge_priv_state_funcs);
-	}
 
 	return 0;
 
-err_detach_bridge:
-	if (bridge->funcs->detach)
-		bridge->funcs->detach(bridge);
-
 err_reset_bridge:
 	bridge->dev = NULL;
 	bridge->encoder = NULL;
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
index dbf984f8e3011..841f3de5f307a 100644
--- a/drivers/gpu/drm/drm_buddy.c
+++ b/drivers/gpu/drm/drm_buddy.c
@@ -10,1247 +10,10 @@
 #include <linux/module.h>
 #include <linux/sizes.h>
 
+#include <linux/gpu_buddy.h>
 #include <drm/drm_buddy.h>
 #include <drm/drm_print.h>
 
-enum drm_buddy_free_tree {
-	DRM_BUDDY_CLEAR_TREE = 0,
-	DRM_BUDDY_DIRTY_TREE,
-	DRM_BUDDY_MAX_FREE_TREES,
-};
-
-static struct kmem_cache *slab_blocks;
-
-#define for_each_free_tree(tree) \
-	for ((tree) = 0; (tree) < DRM_BUDDY_MAX_FREE_TREES; (tree)++)
-
-static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm,
-					       struct drm_buddy_block *parent,
-					       unsigned int order,
-					       u64 offset)
-{
-	struct drm_buddy_block *block;
-
-	BUG_ON(order > DRM_BUDDY_MAX_ORDER);
-
-	block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL);
-	if (!block)
-		return NULL;
-
-	block->header = offset;
-	block->header |= order;
-	block->parent = parent;
-
-	RB_CLEAR_NODE(&block->rb);
-
-	BUG_ON(block->header & DRM_BUDDY_HEADER_UNUSED);
-	return block;
-}
-
-static void drm_block_free(struct drm_buddy *mm,
-			   struct drm_buddy_block *block)
-{
-	kmem_cache_free(slab_blocks, block);
-}
-
-static enum drm_buddy_free_tree
-get_block_tree(struct drm_buddy_block *block)
-{
-	return drm_buddy_block_is_clear(block) ?
-	       DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
-}
-
-static struct drm_buddy_block *
-rbtree_get_free_block(const struct rb_node *node)
-{
-	return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL;
-}
-
-static struct drm_buddy_block *
-rbtree_last_free_block(struct rb_root *root)
-{
-	return rbtree_get_free_block(rb_last(root));
-}
-
-static bool rbtree_is_empty(struct rb_root *root)
-{
-	return RB_EMPTY_ROOT(root);
-}
-
-static bool drm_buddy_block_offset_less(const struct drm_buddy_block *block,
-					const struct drm_buddy_block *node)
-{
-	return drm_buddy_block_offset(block) < drm_buddy_block_offset(node);
-}
-
-static bool rbtree_block_offset_less(struct rb_node *block,
-				     const struct rb_node *node)
-{
-	return drm_buddy_block_offset_less(rbtree_get_free_block(block),
-					   rbtree_get_free_block(node));
-}
-
-static void rbtree_insert(struct drm_buddy *mm,
-			  struct drm_buddy_block *block,
-			  enum drm_buddy_free_tree tree)
-{
-	rb_add(&block->rb,
-	       &mm->free_trees[tree][drm_buddy_block_order(block)],
-	       rbtree_block_offset_less);
-}
-
-static void rbtree_remove(struct drm_buddy *mm,
-			  struct drm_buddy_block *block)
-{
-	unsigned int order = drm_buddy_block_order(block);
-	enum drm_buddy_free_tree tree;
-	struct rb_root *root;
-
-	tree = get_block_tree(block);
-	root = &mm->free_trees[tree][order];
-
-	rb_erase(&block->rb, root);
-	RB_CLEAR_NODE(&block->rb);
-}
-
-static void clear_reset(struct drm_buddy_block *block)
-{
-	block->header &= ~DRM_BUDDY_HEADER_CLEAR;
-}
-
-static void mark_cleared(struct drm_buddy_block *block)
-{
-	block->header |= DRM_BUDDY_HEADER_CLEAR;
-}
-
-static void mark_allocated(struct drm_buddy *mm,
-			   struct drm_buddy_block *block)
-{
-	block->header &= ~DRM_BUDDY_HEADER_STATE;
-	block->header |= DRM_BUDDY_ALLOCATED;
-
-	rbtree_remove(mm, block);
-}
-
-static void mark_free(struct drm_buddy *mm,
-		      struct drm_buddy_block *block)
-{
-	enum drm_buddy_free_tree tree;
-
-	block->header &= ~DRM_BUDDY_HEADER_STATE;
-	block->header |= DRM_BUDDY_FREE;
-
-	tree = get_block_tree(block);
-	rbtree_insert(mm, block, tree);
-}
-
-static void mark_split(struct drm_buddy *mm,
-		       struct drm_buddy_block *block)
-{
-	block->header &= ~DRM_BUDDY_HEADER_STATE;
-	block->header |= DRM_BUDDY_SPLIT;
-
-	rbtree_remove(mm, block);
-}
-
-static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
-{
-	return s1 <= e2 && e1 >= s2;
-}
-
-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
-{
-	return s1 <= s2 && e1 >= e2;
-}
-
-static struct drm_buddy_block *
-__get_buddy(struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *parent;
-
-	parent = block->parent;
-	if (!parent)
-		return NULL;
-
-	if (parent->left == block)
-		return parent->right;
-
-	return parent->left;
-}
-
-static unsigned int __drm_buddy_free(struct drm_buddy *mm,
-				     struct drm_buddy_block *block,
-				     bool force_merge)
-{
-	struct drm_buddy_block *parent;
-	unsigned int order;
-
-	while ((parent = block->parent)) {
-		struct drm_buddy_block *buddy;
-
-		buddy = __get_buddy(block);
-
-		if (!drm_buddy_block_is_free(buddy))
-			break;
-
-		if (!force_merge) {
-			/*
-			 * Check the block and its buddy clear state and exit
-			 * the loop if they both have the dissimilar state.
-			 */
-			if (drm_buddy_block_is_clear(block) !=
-			    drm_buddy_block_is_clear(buddy))
-				break;
-
-			if (drm_buddy_block_is_clear(block))
-				mark_cleared(parent);
-		}
-
-		rbtree_remove(mm, buddy);
-		if (force_merge && drm_buddy_block_is_clear(buddy))
-			mm->clear_avail -= drm_buddy_block_size(mm, buddy);
-
-		drm_block_free(mm, block);
-		drm_block_free(mm, buddy);
-
-		block = parent;
-	}
-
-	order = drm_buddy_block_order(block);
-	mark_free(mm, block);
-
-	return order;
-}
-
-static int __force_merge(struct drm_buddy *mm,
-			 u64 start,
-			 u64 end,
-			 unsigned int min_order)
-{
-	unsigned int tree, order;
-	int i;
-
-	if (!min_order)
-		return -ENOMEM;
-
-	if (min_order > mm->max_order)
-		return -EINVAL;
-
-	for_each_free_tree(tree) {
-		for (i = min_order - 1; i >= 0; i--) {
-			struct rb_node *iter = rb_last(&mm->free_trees[tree][i]);
-
-			while (iter) {
-				struct drm_buddy_block *block, *buddy;
-				u64 block_start, block_end;
-
-				block = rbtree_get_free_block(iter);
-				iter = rb_prev(iter);
-
-				if (!block || !block->parent)
-					continue;
-
-				block_start = drm_buddy_block_offset(block);
-				block_end = block_start + drm_buddy_block_size(mm, block) - 1;
-
-				if (!contains(start, end, block_start, block_end))
-					continue;
-
-				buddy = __get_buddy(block);
-				if (!drm_buddy_block_is_free(buddy))
-					continue;
-
-				WARN_ON(drm_buddy_block_is_clear(block) ==
-					drm_buddy_block_is_clear(buddy));
-
-				/*
-				 * Advance to the next node when the current node is the buddy,
-				 * as freeing the block will also remove its buddy from the tree.
-				 */
-				if (iter == &buddy->rb)
-					iter = rb_prev(iter);
-
-				rbtree_remove(mm, block);
-				if (drm_buddy_block_is_clear(block))
-					mm->clear_avail -= drm_buddy_block_size(mm, block);
-
-				order = __drm_buddy_free(mm, block, true);
-				if (order >= min_order)
-					return 0;
-			}
-		}
-	}
-
-	return -ENOMEM;
-}
-
-/**
- * drm_buddy_init - init memory manager
- *
- * @mm: DRM buddy manager to initialize
- * @size: size in bytes to manage
- * @chunk_size: minimum page size in bytes for our allocations
- *
- * Initializes the memory manager and its resources.
- *
- * Returns:
- * 0 on success, error code on failure.
- */
-int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size)
-{
-	unsigned int i, j, root_count = 0;
-	u64 offset = 0;
-
-	if (size < chunk_size)
-		return -EINVAL;
-
-	if (chunk_size < SZ_4K)
-		return -EINVAL;
-
-	if (!is_power_of_2(chunk_size))
-		return -EINVAL;
-
-	size = round_down(size, chunk_size);
-
-	mm->size = size;
-	mm->avail = size;
-	mm->clear_avail = 0;
-	mm->chunk_size = chunk_size;
-	mm->max_order = ilog2(size) - ilog2(chunk_size);
-
-	BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER);
-
-	mm->free_trees = kmalloc_objs(*mm->free_trees, DRM_BUDDY_MAX_FREE_TREES);
-	if (!mm->free_trees)
-		return -ENOMEM;
-
-	for_each_free_tree(i) {
-		mm->free_trees[i] = kmalloc_objs(struct rb_root,
-						 mm->max_order + 1);
-		if (!mm->free_trees[i])
-			goto out_free_tree;
-
-		for (j = 0; j <= mm->max_order; ++j)
-			mm->free_trees[i][j] = RB_ROOT;
-	}
-
-	mm->n_roots = hweight64(size);
-
-	mm->roots = kmalloc_objs(struct drm_buddy_block *, mm->n_roots);
-	if (!mm->roots)
-		goto out_free_tree;
-
-	/*
-	 * Split into power-of-two blocks, in case we are given a size that is
-	 * not itself a power-of-two.
-	 */
-	do {
-		struct drm_buddy_block *root;
-		unsigned int order;
-		u64 root_size;
-
-		order = ilog2(size) - ilog2(chunk_size);
-		root_size = chunk_size << order;
-
-		root = drm_block_alloc(mm, NULL, order, offset);
-		if (!root)
-			goto out_free_roots;
-
-		mark_free(mm, root);
-
-		BUG_ON(root_count > mm->max_order);
-		BUG_ON(drm_buddy_block_size(mm, root) < chunk_size);
-
-		mm->roots[root_count] = root;
-
-		offset += root_size;
-		size -= root_size;
-		root_count++;
-	} while (size);
-
-	return 0;
-
-out_free_roots:
-	while (root_count--)
-		drm_block_free(mm, mm->roots[root_count]);
-	kfree(mm->roots);
-out_free_tree:
-	while (i--)
-		kfree(mm->free_trees[i]);
-	kfree(mm->free_trees);
-	return -ENOMEM;
-}
-EXPORT_SYMBOL(drm_buddy_init);
-
-/**
- * drm_buddy_fini - tear down the memory manager
- *
- * @mm: DRM buddy manager to free
- *
- * Cleanup memory manager resources and the freetree
- */
-void drm_buddy_fini(struct drm_buddy *mm)
-{
-	u64 root_size, size, start;
-	unsigned int order;
-	int i;
-
-	size = mm->size;
-
-	for (i = 0; i < mm->n_roots; ++i) {
-		order = ilog2(size) - ilog2(mm->chunk_size);
-		start = drm_buddy_block_offset(mm->roots[i]);
-		__force_merge(mm, start, start + size, order);
-
-		if (WARN_ON(!drm_buddy_block_is_free(mm->roots[i])))
-			kunit_fail_current_test("buddy_fini() root");
-
-		drm_block_free(mm, mm->roots[i]);
-
-		root_size = mm->chunk_size << order;
-		size -= root_size;
-	}
-
-	WARN_ON(mm->avail != mm->size);
-
-	for_each_free_tree(i)
-		kfree(mm->free_trees[i]);
-	kfree(mm->free_trees);
-	kfree(mm->roots);
-}
-EXPORT_SYMBOL(drm_buddy_fini);
-
-static int split_block(struct drm_buddy *mm,
-		       struct drm_buddy_block *block)
-{
-	unsigned int block_order = drm_buddy_block_order(block) - 1;
-	u64 offset = drm_buddy_block_offset(block);
-
-	BUG_ON(!drm_buddy_block_is_free(block));
-	BUG_ON(!drm_buddy_block_order(block));
-
-	block->left = drm_block_alloc(mm, block, block_order, offset);
-	if (!block->left)
-		return -ENOMEM;
-
-	block->right = drm_block_alloc(mm, block, block_order,
-				       offset + (mm->chunk_size << block_order));
-	if (!block->right) {
-		drm_block_free(mm, block->left);
-		return -ENOMEM;
-	}
-
-	mark_split(mm, block);
-
-	if (drm_buddy_block_is_clear(block)) {
-		mark_cleared(block->left);
-		mark_cleared(block->right);
-		clear_reset(block);
-	}
-
-	mark_free(mm, block->left);
-	mark_free(mm, block->right);
-
-	return 0;
-}
-
-/**
- * drm_get_buddy - get buddy address
- *
- * @block: DRM buddy block
- *
- * Returns the corresponding buddy block for @block, or NULL
- * if this is a root block and can't be merged further.
- * Requires some kind of locking to protect against
- * any concurrent allocate and free operations.
- */
-struct drm_buddy_block *
-drm_get_buddy(struct drm_buddy_block *block)
-{
-	return __get_buddy(block);
-}
-EXPORT_SYMBOL(drm_get_buddy);
-
-/**
- * drm_buddy_reset_clear - reset blocks clear state
- *
- * @mm: DRM buddy manager
- * @is_clear: blocks clear state
- *
- * Reset the clear state based on @is_clear value for each block
- * in the freetree.
- */
-void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear)
-{
-	enum drm_buddy_free_tree src_tree, dst_tree;
-	u64 root_size, size, start;
-	unsigned int order;
-	int i;
-
-	size = mm->size;
-	for (i = 0; i < mm->n_roots; ++i) {
-		order = ilog2(size) - ilog2(mm->chunk_size);
-		start = drm_buddy_block_offset(mm->roots[i]);
-		__force_merge(mm, start, start + size, order);
-
-		root_size = mm->chunk_size << order;
-		size -= root_size;
-	}
-
-	src_tree = is_clear ? DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE;
-	dst_tree = is_clear ? DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
-
-	for (i = 0; i <= mm->max_order; ++i) {
-		struct rb_root *root = &mm->free_trees[src_tree][i];
-		struct drm_buddy_block *block, *tmp;
-
-		rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
-			rbtree_remove(mm, block);
-			if (is_clear) {
-				mark_cleared(block);
-				mm->clear_avail += drm_buddy_block_size(mm, block);
-			} else {
-				clear_reset(block);
-				mm->clear_avail -= drm_buddy_block_size(mm, block);
-			}
-
-			rbtree_insert(mm, block, dst_tree);
-		}
-	}
-}
-EXPORT_SYMBOL(drm_buddy_reset_clear);
-
-/**
- * drm_buddy_free_block - free a block
- *
- * @mm: DRM buddy manager
- * @block: block to be freed
- */
-void drm_buddy_free_block(struct drm_buddy *mm,
-			  struct drm_buddy_block *block)
-{
-	BUG_ON(!drm_buddy_block_is_allocated(block));
-	mm->avail += drm_buddy_block_size(mm, block);
-	if (drm_buddy_block_is_clear(block))
-		mm->clear_avail += drm_buddy_block_size(mm, block);
-
-	__drm_buddy_free(mm, block, false);
-}
-EXPORT_SYMBOL(drm_buddy_free_block);
-
-static void __drm_buddy_free_list(struct drm_buddy *mm,
-				  struct list_head *objects,
-				  bool mark_clear,
-				  bool mark_dirty)
-{
-	struct drm_buddy_block *block, *on;
-
-	WARN_ON(mark_dirty && mark_clear);
-
-	list_for_each_entry_safe(block, on, objects, link) {
-		if (mark_clear)
-			mark_cleared(block);
-		else if (mark_dirty)
-			clear_reset(block);
-		drm_buddy_free_block(mm, block);
-		cond_resched();
-	}
-	INIT_LIST_HEAD(objects);
-}
-
-static void drm_buddy_free_list_internal(struct drm_buddy *mm,
-					 struct list_head *objects)
-{
-	/*
-	 * Don't touch the clear/dirty bit, since allocation is still internal
-	 * at this point. For example we might have just failed part of the
-	 * allocation.
-	 */
-	__drm_buddy_free_list(mm, objects, false, false);
-}
-
-/**
- * drm_buddy_free_list - free blocks
- *
- * @mm: DRM buddy manager
- * @objects: input list head to free blocks
- * @flags: optional flags like DRM_BUDDY_CLEARED
- */
-void drm_buddy_free_list(struct drm_buddy *mm,
-			 struct list_head *objects,
-			 unsigned int flags)
-{
-	bool mark_clear = flags & DRM_BUDDY_CLEARED;
-
-	__drm_buddy_free_list(mm, objects, mark_clear, !mark_clear);
-}
-EXPORT_SYMBOL(drm_buddy_free_list);
-
-static bool block_incompatible(struct drm_buddy_block *block, unsigned int flags)
-{
-	bool needs_clear = flags & DRM_BUDDY_CLEAR_ALLOCATION;
-
-	return needs_clear != drm_buddy_block_is_clear(block);
-}
-
-static struct drm_buddy_block *
-__alloc_range_bias(struct drm_buddy *mm,
-		   u64 start, u64 end,
-		   unsigned int order,
-		   unsigned long flags,
-		   bool fallback)
-{
-	u64 req_size = mm->chunk_size << order;
-	struct drm_buddy_block *block;
-	struct drm_buddy_block *buddy;
-	LIST_HEAD(dfs);
-	int err;
-	int i;
-
-	end = end - 1;
-
-	for (i = 0; i < mm->n_roots; ++i)
-		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
-
-	do {
-		u64 block_start;
-		u64 block_end;
-
-		block = list_first_entry_or_null(&dfs,
-						 struct drm_buddy_block,
-						 tmp_link);
-		if (!block)
-			break;
-
-		list_del(&block->tmp_link);
-
-		if (drm_buddy_block_order(block) < order)
-			continue;
-
-		block_start = drm_buddy_block_offset(block);
-		block_end = block_start + drm_buddy_block_size(mm, block) - 1;
-
-		if (!overlaps(start, end, block_start, block_end))
-			continue;
-
-		if (drm_buddy_block_is_allocated(block))
-			continue;
-
-		if (block_start < start || block_end > end) {
-			u64 adjusted_start = max(block_start, start);
-			u64 adjusted_end = min(block_end, end);
-
-			if (round_down(adjusted_end + 1, req_size) <=
-			    round_up(adjusted_start, req_size))
-				continue;
-		}
-
-		if (!fallback && block_incompatible(block, flags))
-			continue;
-
-		if (contains(start, end, block_start, block_end) &&
-		    order == drm_buddy_block_order(block)) {
-			/*
-			 * Find the free block within the range.
-			 */
-			if (drm_buddy_block_is_free(block))
-				return block;
-
-			continue;
-		}
-
-		if (!drm_buddy_block_is_split(block)) {
-			err = split_block(mm, block);
-			if (unlikely(err))
-				goto err_undo;
-		}
-
-		list_add(&block->right->tmp_link, &dfs);
-		list_add(&block->left->tmp_link, &dfs);
-	} while (1);
-
-	return ERR_PTR(-ENOSPC);
-
-err_undo:
-	/*
-	 * We really don't want to leave around a bunch of split blocks, since
-	 * bigger is better, so make sure we merge everything back before we
-	 * free the allocated blocks.
-	 */
-	buddy = __get_buddy(block);
-	if (buddy &&
-	    (drm_buddy_block_is_free(block) &&
-	     drm_buddy_block_is_free(buddy)))
-		__drm_buddy_free(mm, block, false);
-	return ERR_PTR(err);
-}
-
-static struct drm_buddy_block *
-__drm_buddy_alloc_range_bias(struct drm_buddy *mm,
-			     u64 start, u64 end,
-			     unsigned int order,
-			     unsigned long flags)
-{
-	struct drm_buddy_block *block;
-	bool fallback = false;
-
-	block = __alloc_range_bias(mm, start, end, order,
-				   flags, fallback);
-	if (IS_ERR(block))
-		return __alloc_range_bias(mm, start, end, order,
-					  flags, !fallback);
-
-	return block;
-}
-
-static struct drm_buddy_block *
-get_maxblock(struct drm_buddy *mm,
-	     unsigned int order,
-	     enum drm_buddy_free_tree tree)
-{
-	struct drm_buddy_block *max_block = NULL, *block = NULL;
-	struct rb_root *root;
-	unsigned int i;
-
-	for (i = order; i <= mm->max_order; ++i) {
-		root = &mm->free_trees[tree][i];
-		block = rbtree_last_free_block(root);
-		if (!block)
-			continue;
-
-		if (!max_block) {
-			max_block = block;
-			continue;
-		}
-
-		if (drm_buddy_block_offset(block) >
-		    drm_buddy_block_offset(max_block)) {
-			max_block = block;
-		}
-	}
-
-	return max_block;
-}
-
-static struct drm_buddy_block *
-alloc_from_freetree(struct drm_buddy *mm,
-		    unsigned int order,
-		    unsigned long flags)
-{
-	struct drm_buddy_block *block = NULL;
-	struct rb_root *root;
-	enum drm_buddy_free_tree tree;
-	unsigned int tmp;
-	int err;
-
-	tree = (flags & DRM_BUDDY_CLEAR_ALLOCATION) ?
-		DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE;
-
-	if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) {
-		block = get_maxblock(mm, order, tree);
-		if (block)
-			/* Store the obtained block order */
-			tmp = drm_buddy_block_order(block);
-	} else {
-		for (tmp = order; tmp <= mm->max_order; ++tmp) {
-			/* Get RB tree root for this order and tree */
-			root = &mm->free_trees[tree][tmp];
-			block = rbtree_last_free_block(root);
-			if (block)
-				break;
-		}
-	}
-
-	if (!block) {
-		/* Try allocating from the other tree */
-		tree = (tree == DRM_BUDDY_CLEAR_TREE) ?
-			DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE;
-
-		for (tmp = order; tmp <= mm->max_order; ++tmp) {
-			root = &mm->free_trees[tree][tmp];
-			block = rbtree_last_free_block(root);
-			if (block)
-				break;
-		}
-
-		if (!block)
-			return ERR_PTR(-ENOSPC);
-	}
-
-	BUG_ON(!drm_buddy_block_is_free(block));
-
-	while (tmp != order) {
-		err = split_block(mm, block);
-		if (unlikely(err))
-			goto err_undo;
-
-		block = block->right;
-		tmp--;
-	}
-	return block;
-
-err_undo:
-	if (tmp != order)
-		__drm_buddy_free(mm, block, false);
-	return ERR_PTR(err);
-}
-
-static int __alloc_range(struct drm_buddy *mm,
-			 struct list_head *dfs,
-			 u64 start, u64 size,
-			 struct list_head *blocks,
-			 u64 *total_allocated_on_err)
-{
-	struct drm_buddy_block *block;
-	struct drm_buddy_block *buddy;
-	u64 total_allocated = 0;
-	LIST_HEAD(allocated);
-	u64 end;
-	int err;
-
-	end = start + size - 1;
-
-	do {
-		u64 block_start;
-		u64 block_end;
-
-		block = list_first_entry_or_null(dfs,
-						 struct drm_buddy_block,
-						 tmp_link);
-		if (!block)
-			break;
-
-		list_del(&block->tmp_link);
-
-		block_start = drm_buddy_block_offset(block);
-		block_end = block_start + drm_buddy_block_size(mm, block) - 1;
-
-		if (!overlaps(start, end, block_start, block_end))
-			continue;
-
-		if (drm_buddy_block_is_allocated(block)) {
-			err = -ENOSPC;
-			goto err_free;
-		}
-
-		if (contains(start, end, block_start, block_end)) {
-			if (drm_buddy_block_is_free(block)) {
-				mark_allocated(mm, block);
-				total_allocated += drm_buddy_block_size(mm, block);
-				mm->avail -= drm_buddy_block_size(mm, block);
-				if (drm_buddy_block_is_clear(block))
-					mm->clear_avail -= drm_buddy_block_size(mm, block);
-				list_add_tail(&block->link, &allocated);
-				continue;
-			} else if (!mm->clear_avail) {
-				err = -ENOSPC;
-				goto err_free;
-			}
-		}
-
-		if (!drm_buddy_block_is_split(block)) {
-			err = split_block(mm, block);
-			if (unlikely(err))
-				goto err_undo;
-		}
-
-		list_add(&block->right->tmp_link, dfs);
-		list_add(&block->left->tmp_link, dfs);
-	} while (1);
-
-	if (total_allocated < size) {
-		err = -ENOSPC;
-		goto err_free;
-	}
-
-	list_splice_tail(&allocated, blocks);
-
-	return 0;
-
-err_undo:
-	/*
-	 * We really don't want to leave around a bunch of split blocks, since
-	 * bigger is better, so make sure we merge everything back before we
-	 * free the allocated blocks.
-	 */
-	buddy = __get_buddy(block);
-	if (buddy &&
-	    (drm_buddy_block_is_free(block) &&
-	     drm_buddy_block_is_free(buddy)))
-		__drm_buddy_free(mm, block, false);
-
-err_free:
-	if (err == -ENOSPC && total_allocated_on_err) {
-		list_splice_tail(&allocated, blocks);
-		*total_allocated_on_err = total_allocated;
-	} else {
-		drm_buddy_free_list_internal(mm, &allocated);
-	}
-
-	return err;
-}
-
-static int __drm_buddy_alloc_range(struct drm_buddy *mm,
-				   u64 start,
-				   u64 size,
-				   u64 *total_allocated_on_err,
-				   struct list_head *blocks)
-{
-	LIST_HEAD(dfs);
-	int i;
-
-	for (i = 0; i < mm->n_roots; ++i)
-		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
-
-	return __alloc_range(mm, &dfs, start, size,
-			     blocks, total_allocated_on_err);
-}
-
-static int __alloc_contig_try_harder(struct drm_buddy *mm,
-				     u64 size,
-				     u64 min_block_size,
-				     struct list_head *blocks)
-{
-	u64 rhs_offset, lhs_offset, lhs_size, filled;
-	struct drm_buddy_block *block;
-	unsigned int tree, order;
-	LIST_HEAD(blocks_lhs);
-	unsigned long pages;
-	u64 modify_size;
-	int err;
-
-	modify_size = rounddown_pow_of_two(size);
-	pages = modify_size >> ilog2(mm->chunk_size);
-	order = fls(pages) - 1;
-	if (order == 0)
-		return -ENOSPC;
-
-	for_each_free_tree(tree) {
-		struct rb_root *root;
-		struct rb_node *iter;
-
-		root = &mm->free_trees[tree][order];
-		if (rbtree_is_empty(root))
-			continue;
-
-		iter = rb_last(root);
-		while (iter) {
-			block = rbtree_get_free_block(iter);
-
-			/* Allocate blocks traversing RHS */
-			rhs_offset = drm_buddy_block_offset(block);
-			err =  __drm_buddy_alloc_range(mm, rhs_offset, size,
-						       &filled, blocks);
-			if (!err || err != -ENOSPC)
-				return err;
-
-			lhs_size = max((size - filled), min_block_size);
-			if (!IS_ALIGNED(lhs_size, min_block_size))
-				lhs_size = round_up(lhs_size, min_block_size);
-
-			/* Allocate blocks traversing LHS */
-			lhs_offset = drm_buddy_block_offset(block) - lhs_size;
-			err =  __drm_buddy_alloc_range(mm, lhs_offset, lhs_size,
-						       NULL, &blocks_lhs);
-			if (!err) {
-				list_splice(&blocks_lhs, blocks);
-				return 0;
-			} else if (err != -ENOSPC) {
-				drm_buddy_free_list_internal(mm, blocks);
-				return err;
-			}
-			/* Free blocks for the next iteration */
-			drm_buddy_free_list_internal(mm, blocks);
-
-			iter = rb_prev(iter);
-		}
-	}
-
-	return -ENOSPC;
-}
-
-/**
- * drm_buddy_block_trim - free unused pages
- *
- * @mm: DRM buddy manager
- * @start: start address to begin the trimming.
- * @new_size: original size requested
- * @blocks: Input and output list of allocated blocks.
- * MUST contain single block as input to be trimmed.
- * On success will contain the newly allocated blocks
- * making up the @new_size. Blocks always appear in
- * ascending order
- *
- * For contiguous allocation, we round up the size to the nearest
- * power of two value, drivers consume *actual* size, so remaining
- * portions are unused and can be optionally freed with this function
- *
- * Returns:
- * 0 on success, error code on failure.
- */
-int drm_buddy_block_trim(struct drm_buddy *mm,
-			 u64 *start,
-			 u64 new_size,
-			 struct list_head *blocks)
-{
-	struct drm_buddy_block *parent;
-	struct drm_buddy_block *block;
-	u64 block_start, block_end;
-	LIST_HEAD(dfs);
-	u64 new_start;
-	int err;
-
-	if (!list_is_singular(blocks))
-		return -EINVAL;
-
-	block = list_first_entry(blocks,
-				 struct drm_buddy_block,
-				 link);
-
-	block_start = drm_buddy_block_offset(block);
-	block_end = block_start + drm_buddy_block_size(mm, block);
-
-	if (WARN_ON(!drm_buddy_block_is_allocated(block)))
-		return -EINVAL;
-
-	if (new_size > drm_buddy_block_size(mm, block))
-		return -EINVAL;
-
-	if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size))
-		return -EINVAL;
-
-	if (new_size == drm_buddy_block_size(mm, block))
-		return 0;
-
-	new_start = block_start;
-	if (start) {
-		new_start = *start;
-
-		if (new_start < block_start)
-			return -EINVAL;
-
-		if (!IS_ALIGNED(new_start, mm->chunk_size))
-			return -EINVAL;
-
-		if (range_overflows(new_start, new_size, block_end))
-			return -EINVAL;
-	}
-
-	list_del(&block->link);
-	mark_free(mm, block);
-	mm->avail += drm_buddy_block_size(mm, block);
-	if (drm_buddy_block_is_clear(block))
-		mm->clear_avail += drm_buddy_block_size(mm, block);
-
-	/* Prevent recursively freeing this node */
-	parent = block->parent;
-	block->parent = NULL;
-
-	list_add(&block->tmp_link, &dfs);
-	err =  __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL);
-	if (err) {
-		mark_allocated(mm, block);
-		mm->avail -= drm_buddy_block_size(mm, block);
-		if (drm_buddy_block_is_clear(block))
-			mm->clear_avail -= drm_buddy_block_size(mm, block);
-		list_add(&block->link, blocks);
-	}
-
-	block->parent = parent;
-	return err;
-}
-EXPORT_SYMBOL(drm_buddy_block_trim);
-
-static struct drm_buddy_block *
-__drm_buddy_alloc_blocks(struct drm_buddy *mm,
-			 u64 start, u64 end,
-			 unsigned int order,
-			 unsigned long flags)
-{
-	if (flags & DRM_BUDDY_RANGE_ALLOCATION)
-		/* Allocate traversing within the range */
-		return  __drm_buddy_alloc_range_bias(mm, start, end,
-						     order, flags);
-	else
-		/* Allocate from freetree */
-		return alloc_from_freetree(mm, order, flags);
-}
-
-/**
- * drm_buddy_alloc_blocks - allocate power-of-two blocks
- *
- * @mm: DRM buddy manager to allocate from
- * @start: start of the allowed range for this block
- * @end: end of the allowed range for this block
- * @size: size of the allocation in bytes
- * @min_block_size: alignment of the allocation
- * @blocks: output list head to add allocated blocks
- * @flags: DRM_BUDDY_*_ALLOCATION flags
- *
- * alloc_range_bias() called on range limitations, which traverses
- * the tree and returns the desired block.
- *
- * alloc_from_freetree() called when *no* range restrictions
- * are enforced, which picks the block from the freetree.
- *
- * Returns:
- * 0 on success, error code on failure.
- */
-int drm_buddy_alloc_blocks(struct drm_buddy *mm,
-			   u64 start, u64 end, u64 size,
-			   u64 min_block_size,
-			   struct list_head *blocks,
-			   unsigned long flags)
-{
-	struct drm_buddy_block *block = NULL;
-	u64 original_size, original_min_size;
-	unsigned int min_order, order;
-	LIST_HEAD(allocated);
-	unsigned long pages;
-	int err;
-
-	if (size < mm->chunk_size)
-		return -EINVAL;
-
-	if (min_block_size < mm->chunk_size)
-		return -EINVAL;
-
-	if (!is_power_of_2(min_block_size))
-		return -EINVAL;
-
-	if (!IS_ALIGNED(start | end | size, mm->chunk_size))
-		return -EINVAL;
-
-	if (end > mm->size)
-		return -EINVAL;
-
-	if (range_overflows(start, size, mm->size))
-		return -EINVAL;
-
-	/* Actual range allocation */
-	if (start + size == end) {
-		if (!IS_ALIGNED(start | end, min_block_size))
-			return -EINVAL;
-
-		return __drm_buddy_alloc_range(mm, start, size, NULL, blocks);
-	}
-
-	original_size = size;
-	original_min_size = min_block_size;
-
-	/* Roundup the size to power of 2 */
-	if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) {
-		size = roundup_pow_of_two(size);
-		min_block_size = size;
-	/* Align size value to min_block_size */
-	} else if (!IS_ALIGNED(size, min_block_size)) {
-		size = round_up(size, min_block_size);
-	}
-
-	pages = size >> ilog2(mm->chunk_size);
-	order = fls(pages) - 1;
-	min_order = ilog2(min_block_size) - ilog2(mm->chunk_size);
-
-	if (order > mm->max_order || size > mm->size) {
-		if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) &&
-		    !(flags & DRM_BUDDY_RANGE_ALLOCATION))
-			return __alloc_contig_try_harder(mm, original_size,
-							 original_min_size, blocks);
-
-		return -EINVAL;
-	}
-
-	do {
-		order = min(order, (unsigned int)fls(pages) - 1);
-		BUG_ON(order > mm->max_order);
-		BUG_ON(order < min_order);
-
-		do {
-			block = __drm_buddy_alloc_blocks(mm, start,
-							 end,
-							 order,
-							 flags);
-			if (!IS_ERR(block))
-				break;
-
-			if (order-- == min_order) {
-				/* Try allocation through force merge method */
-				if (mm->clear_avail &&
-				    !__force_merge(mm, start, end, min_order)) {
-					block = __drm_buddy_alloc_blocks(mm, start,
-									 end,
-									 min_order,
-									 flags);
-					if (!IS_ERR(block)) {
-						order = min_order;
-						break;
-					}
-				}
-
-				/*
-				 * Try contiguous block allocation through
-				 * try harder method.
-				 */
-				if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION &&
-				    !(flags & DRM_BUDDY_RANGE_ALLOCATION))
-					return __alloc_contig_try_harder(mm,
-									 original_size,
-									 original_min_size,
-									 blocks);
-				err = -ENOSPC;
-				goto err_free;
-			}
-		} while (1);
-
-		mark_allocated(mm, block);
-		mm->avail -= drm_buddy_block_size(mm, block);
-		if (drm_buddy_block_is_clear(block))
-			mm->clear_avail -= drm_buddy_block_size(mm, block);
-		kmemleak_update_trace(block);
-		list_add_tail(&block->link, &allocated);
-
-		pages -= BIT(order);
-
-		if (!pages)
-			break;
-	} while (1);
-
-	/* Trim the allocated block to the required size */
-	if (!(flags & DRM_BUDDY_TRIM_DISABLE) &&
-	    original_size != size) {
-		struct list_head *trim_list;
-		LIST_HEAD(temp);
-		u64 trim_size;
-
-		trim_list = &allocated;
-		trim_size = original_size;
-
-		if (!list_is_singular(&allocated)) {
-			block = list_last_entry(&allocated, typeof(*block), link);
-			list_move(&block->link, &temp);
-			trim_list = &temp;
-			trim_size = drm_buddy_block_size(mm, block) -
-				(size - original_size);
-		}
-
-		drm_buddy_block_trim(mm,
-				     NULL,
-				     trim_size,
-				     trim_list);
-
-		if (!list_empty(&temp))
-			list_splice_tail(trim_list, &allocated);
-	}
-
-	list_splice_tail(&allocated, blocks);
-	return 0;
-
-err_free:
-	drm_buddy_free_list_internal(mm, &allocated);
-	return err;
-}
-EXPORT_SYMBOL(drm_buddy_alloc_blocks);
-
 /**
  * drm_buddy_block_print - print block information
  *
@@ -1258,12 +21,12 @@ EXPORT_SYMBOL(drm_buddy_alloc_blocks);
  * @block: DRM buddy block
  * @p: DRM printer to use
  */
-void drm_buddy_block_print(struct drm_buddy *mm,
-			   struct drm_buddy_block *block,
+void drm_buddy_block_print(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block,
 			   struct drm_printer *p)
 {
-	u64 start = drm_buddy_block_offset(block);
-	u64 size = drm_buddy_block_size(mm, block);
+	u64 start = gpu_buddy_block_offset(block);
+	u64 size = gpu_buddy_block_size(mm, block);
 
 	drm_printf(p, "%#018llx-%#018llx: %llu\n", start, start + size, size);
 }
@@ -1275,7 +38,7 @@ EXPORT_SYMBOL(drm_buddy_block_print);
  * @mm: DRM buddy manager
  * @p: DRM printer to use
  */
-void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
+void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p)
 {
 	int order;
 
@@ -1283,7 +46,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
 		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
-		struct drm_buddy_block *block, *tmp;
+		struct gpu_buddy_block *block, *tmp;
 		struct rb_root *root;
 		u64 count = 0, free;
 		unsigned int tree;
@@ -1292,7 +55,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
 			root = &mm->free_trees[tree][order];
 
 			rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
-				BUG_ON(!drm_buddy_block_is_free(block));
+				BUG_ON(!gpu_buddy_block_is_free(block));
 				count++;
 			}
 		}
@@ -1310,22 +73,5 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
 }
 EXPORT_SYMBOL(drm_buddy_print);
 
-static void drm_buddy_module_exit(void)
-{
-	kmem_cache_destroy(slab_blocks);
-}
-
-static int __init drm_buddy_module_init(void)
-{
-	slab_blocks = KMEM_CACHE(drm_buddy_block, 0);
-	if (!slab_blocks)
-		return -ENOMEM;
-
-	return 0;
-}
-
-module_init(drm_buddy_module_init);
-module_exit(drm_buddy_module_exit);
-
-MODULE_DESCRIPTION("DRM Buddy Allocator");
+MODULE_DESCRIPTION("DRM-specific GPU Buddy Allocator Print Helpers");
 MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index 6236ec46d62a9..46c465bce98c7 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -204,7 +204,7 @@ void drm_client_buffer_delete(struct drm_client_buffer *buffer)
 }
 EXPORT_SYMBOL(drm_client_buffer_delete);
 
-static struct drm_client_buffer *
+struct drm_client_buffer *
 drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height,
 			 u32 format, u32 handle, u32 pitch)
 {
@@ -265,6 +265,7 @@ drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height,
 	kfree(buffer);
 	return ERR_PTR(ret);
 }
+EXPORT_SYMBOL(drm_client_buffer_create);
 
 /**
  * drm_client_buffer_vmap_local - Map DRM client buffer into address space
diff --git a/drivers/gpu/drm/drm_colorop.c b/drivers/gpu/drm/drm_colorop.c
index 398cc81ae5880..566816e3c6f0c 100644
--- a/drivers/gpu/drm/drm_colorop.c
+++ b/drivers/gpu/drm/drm_colorop.c
@@ -93,7 +93,8 @@ static const struct drm_prop_enum_list drm_colorop_lut3d_interpolation_list[] =
 /* Init Helpers */
 
 static int drm_plane_colorop_init(struct drm_device *dev, struct drm_colorop *colorop,
-				  struct drm_plane *plane, enum drm_colorop_type type,
+				  struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				  enum drm_colorop_type type,
 				  uint32_t flags)
 {
 	struct drm_mode_config *config = &dev->mode_config;
@@ -109,6 +110,7 @@ static int drm_plane_colorop_init(struct drm_device *dev, struct drm_colorop *co
 	colorop->type = type;
 	colorop->plane = plane;
 	colorop->next = NULL;
+	colorop->funcs = funcs;
 
 	list_add_tail(&colorop->head, &config->colorop_list);
 	colorop->index = config->num_colorop++;
@@ -169,15 +171,26 @@ void drm_colorop_cleanup(struct drm_colorop *colorop)
 	list_del(&colorop->head);
 	config->num_colorop--;
 
-	if (colorop->state && colorop->state->data) {
-		drm_property_blob_put(colorop->state->data);
-		colorop->state->data = NULL;
-	}
-
-	kfree(colorop->state);
+	if (colorop->state)
+		drm_colorop_atomic_destroy_state(colorop, colorop->state);
 }
 EXPORT_SYMBOL(drm_colorop_cleanup);
 
+/**
+ * drm_colorop_destroy - destroy colorop
+ * @colorop: drm colorop
+ *
+ * Destroys @colorop by performing common DRM cleanup and freeing the
+ * colorop object. This can be used by drivers if they do not
+ * require any driver-specific teardown.
+ */
+void drm_colorop_destroy(struct drm_colorop *colorop)
+{
+	drm_colorop_cleanup(colorop);
+	kfree(colorop);
+}
+EXPORT_SYMBOL(drm_colorop_destroy);
+
 /**
  * drm_colorop_pipeline_destroy - Helper for color pipeline destruction
  *
@@ -191,8 +204,7 @@ void drm_colorop_pipeline_destroy(struct drm_device *dev)
 	struct drm_colorop *colorop, *next;
 
 	list_for_each_entry_safe(colorop, next, &config->colorop_list, head) {
-		drm_colorop_cleanup(colorop);
-		kfree(colorop);
+		colorop->funcs->destroy(colorop);
 	}
 }
 EXPORT_SYMBOL(drm_colorop_pipeline_destroy);
@@ -203,6 +215,7 @@ EXPORT_SYMBOL(drm_colorop_pipeline_destroy);
  * @dev: DRM device
  * @colorop: The drm_colorop object to initialize
  * @plane: The associated drm_plane
+ * @funcs: control functions for the new colorop
  * @supported_tfs: A bitfield of supported drm_plane_colorop_curve_1d_init enum values,
  *                 created using BIT(curve_type) and combined with the OR '|'
  *                 operator.
@@ -210,7 +223,8 @@ EXPORT_SYMBOL(drm_colorop_pipeline_destroy);
  * @return zero on success, -E value on failure
  */
 int drm_plane_colorop_curve_1d_init(struct drm_device *dev, struct drm_colorop *colorop,
-				    struct drm_plane *plane, u64 supported_tfs, uint32_t flags)
+				    struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				    u64 supported_tfs, uint32_t flags)
 {
 	struct drm_prop_enum_list enum_list[DRM_COLOROP_1D_CURVE_COUNT];
 	int i, len;
@@ -231,7 +245,7 @@ int drm_plane_colorop_curve_1d_init(struct drm_device *dev, struct drm_colorop *
 		return -EINVAL;
 	}
 
-	ret = drm_plane_colorop_init(dev, colorop, plane, DRM_COLOROP_1D_CURVE, flags);
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_1D_CURVE, flags);
 	if (ret)
 		return ret;
 
@@ -288,20 +302,23 @@ static int drm_colorop_create_data_prop(struct drm_device *dev, struct drm_color
  * @dev: DRM device
  * @colorop: The drm_colorop object to initialize
  * @plane: The associated drm_plane
+ * @funcs: control functions for new colorop
  * @lut_size: LUT size supported by driver
  * @interpolation: 1D LUT interpolation type
  * @flags: bitmask of misc, see DRM_COLOROP_FLAG_* defines.
  * @return zero on success, -E value on failure
  */
 int drm_plane_colorop_curve_1d_lut_init(struct drm_device *dev, struct drm_colorop *colorop,
-					struct drm_plane *plane, uint32_t lut_size,
+					struct drm_plane *plane,
+					const struct drm_colorop_funcs *funcs,
+					uint32_t lut_size,
 					enum drm_colorop_lut1d_interpolation_type interpolation,
 					uint32_t flags)
 {
 	struct drm_property *prop;
 	int ret;
 
-	ret = drm_plane_colorop_init(dev, colorop, plane, DRM_COLOROP_1D_LUT, flags);
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_1D_LUT, flags);
 	if (ret)
 		return ret;
 
@@ -339,11 +356,12 @@ int drm_plane_colorop_curve_1d_lut_init(struct drm_device *dev, struct drm_color
 EXPORT_SYMBOL(drm_plane_colorop_curve_1d_lut_init);
 
 int drm_plane_colorop_ctm_3x4_init(struct drm_device *dev, struct drm_colorop *colorop,
-				   struct drm_plane *plane, uint32_t flags)
+				   struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				   uint32_t flags)
 {
 	int ret;
 
-	ret = drm_plane_colorop_init(dev, colorop, plane, DRM_COLOROP_CTM_3X4, flags);
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_CTM_3X4, flags);
 	if (ret)
 		return ret;
 
@@ -363,16 +381,18 @@ EXPORT_SYMBOL(drm_plane_colorop_ctm_3x4_init);
  * @dev: DRM device
  * @colorop: The drm_colorop object to initialize
  * @plane: The associated drm_plane
+ * @funcs: control functions for the new colorop
  * @flags: bitmask of misc, see DRM_COLOROP_FLAG_* defines.
  * @return zero on success, -E value on failure
  */
 int drm_plane_colorop_mult_init(struct drm_device *dev, struct drm_colorop *colorop,
-				struct drm_plane *plane, uint32_t flags)
+				struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				uint32_t flags)
 {
 	struct drm_property *prop;
 	int ret;
 
-	ret = drm_plane_colorop_init(dev, colorop, plane, DRM_COLOROP_MULTIPLIER, flags);
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_MULTIPLIER, flags);
 	if (ret)
 		return ret;
 
@@ -391,6 +411,7 @@ EXPORT_SYMBOL(drm_plane_colorop_mult_init);
 
 int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *colorop,
 				 struct drm_plane *plane,
+				 const struct drm_colorop_funcs *funcs,
 				 uint32_t lut_size,
 				 enum drm_colorop_lut3d_interpolation_type interpolation,
 				 uint32_t flags)
@@ -398,7 +419,7 @@ int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *col
 	struct drm_property *prop;
 	int ret;
 
-	ret = drm_plane_colorop_init(dev, colorop, plane, DRM_COLOROP_3D_LUT, flags);
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_3D_LUT, flags);
 	if (ret)
 		return ret;
 
@@ -441,8 +462,6 @@ static void __drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colo
 
 	if (state->data)
 		drm_property_blob_get(state->data);
-
-	state->bypass = true;
 }
 
 struct drm_colorop_state *
@@ -460,9 +479,23 @@ drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colorop)
 	return state;
 }
 
+/**
+ * __drm_atomic_helper_colorop_destroy_state - release colorop state
+ * @state: colorop state object to release
+ *
+ * Releases all resources stored in the colorop state without actually freeing
+ * the memory of the colorop state. This is useful for drivers that subclass the
+ * colorop state.
+ */
+static void __drm_atomic_helper_colorop_destroy_state(struct drm_colorop_state *state)
+{
+	drm_property_blob_put(state->data);
+}
+
 void drm_colorop_atomic_destroy_state(struct drm_colorop *colorop,
 				      struct drm_colorop_state *state)
 {
+	__drm_atomic_helper_colorop_destroy_state(state);
 	kfree(state);
 }
 
@@ -513,7 +546,9 @@ static void __drm_colorop_reset(struct drm_colorop *colorop,
 
 void drm_colorop_reset(struct drm_colorop *colorop)
 {
-	kfree(colorop->state);
+	if (colorop->state)
+		drm_colorop_atomic_destroy_state(colorop, colorop->state);
+
 	colorop->state = kzalloc_obj(*colorop->state);
 
 	if (colorop->state)
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 2a3a1fa0594b3..e70699c59c436 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -1173,6 +1173,11 @@ static const struct drm_prop_enum_list drm_link_status_enum_list[] = {
 	{ DRM_MODE_LINK_STATUS_BAD, "Bad" },
 };
 
+static const struct drm_prop_enum_list drm_panel_type_enum_list[] = {
+	{ DRM_MODE_PANEL_TYPE_UNKNOWN, "unknown" },
+	{ DRM_MODE_PANEL_TYPE_OLED, "OLED" },
+};
+
 /**
  * drm_display_info_set_bus_formats - set the supported bus formats
  * @info: display info to store bus formats in
@@ -1501,6 +1506,9 @@ EXPORT_SYMBOL(drm_hdmi_connector_get_output_format_name);
  * 	Summarizing: Only set "DPMS" when the connector is known to be enabled,
  * 	assume that a successful SETCONFIG call also sets "DPMS" to on, and
  * 	never read back the value of "DPMS" because it can be incorrect.
+ * panel_type:
+ * 	Immutable enum property to indicate the type of connected panel.
+ * 	Possible values are "unknown" (default) and "OLED".
  * PATH:
  * 	Connector path property to identify how this sink is physically
  * 	connected. Used by DP MST. This should be set by calling
@@ -1851,6 +1859,13 @@ int drm_connector_create_standard_properties(struct drm_device *dev)
 		return -ENOMEM;
 	dev->mode_config.link_status_property = prop;
 
+	prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE, "panel_type",
+					drm_panel_type_enum_list,
+					ARRAY_SIZE(drm_panel_type_enum_list));
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.panel_type_property = prop;
+
 	prop = drm_property_create_bool(dev, DRM_MODE_PROP_IMMUTABLE, "non-desktop");
 	if (!prop)
 		return -ENOMEM;
@@ -3626,3 +3641,21 @@ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
 	return tg;
 }
 EXPORT_SYMBOL(drm_mode_create_tile_group);
+
+/**
+ * drm_connector_attach_panel_type_property - attaches panel type property
+ * @connector: connector to attach the property on.
+ *
+ * This is used to add support for panel type detection.
+ */
+void drm_connector_attach_panel_type_property(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_property *prop = dev->mode_config.panel_type_property;
+
+	if (!prop)
+		return;
+
+	drm_object_attach_property(&connector->base, prop, DRM_MODE_PANEL_TYPE_UNKNOWN);
+}
+EXPORT_SYMBOL(drm_connector_attach_panel_type_property);
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 90684f30a0483..8d6f721c2c9af 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -158,8 +158,8 @@ static const struct dma_fence_ops drm_crtc_fence_ops;
 
 static struct drm_crtc *fence_to_crtc(struct dma_fence *fence)
 {
-	BUG_ON(fence->ops != &drm_crtc_fence_ops);
-	return container_of(fence->lock, struct drm_crtc, fence_lock);
+	BUG_ON(rcu_access_pointer(fence->ops) != &drm_crtc_fence_ops);
+	return container_of(fence->extern_lock, struct drm_crtc, fence_lock);
 }
 
 static const char *drm_crtc_fence_get_driver_name(struct dma_fence *fence)
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 2915118436ce8..6b965c3d33079 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -53,6 +53,7 @@
 #include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/drm_privacy_screen_machine.h>
+#include <drm/drm_ras_genl_family.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
@@ -1223,6 +1224,7 @@ static const struct file_operations drm_stub_fops = {
 
 static void drm_core_exit(void)
 {
+	drm_ras_genl_family_unregister();
 	drm_privacy_screen_lookup_exit();
 	drm_panic_exit();
 	accel_core_exit();
@@ -1261,6 +1263,10 @@ static int __init drm_core_init(void)
 
 	drm_privacy_screen_lookup_init();
 
+	ret = drm_ras_genl_family_register();
+	if (ret < 0)
+		goto error;
+
 	drm_core_init_complete = true;
 
 	DRM_DEBUG("Initialized\n");
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 05803169bed57..c4adad77c8d6c 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -37,6 +37,7 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
@@ -343,18 +344,6 @@ EXPORT_SYMBOL(drm_fb_helper_unprepare);
 int drm_fb_helper_init(struct drm_device *dev,
 		       struct drm_fb_helper *fb_helper)
 {
-	int ret;
-
-	/*
-	 * If this is not the generic fbdev client, initialize a drm_client
-	 * without callbacks so we can use the modesets.
-	 */
-	if (!fb_helper->client.funcs) {
-		ret = drm_client_init(dev, &fb_helper->client, "drm_fb_helper", NULL);
-		if (ret)
-			return ret;
-	}
-
 	dev->fb_helper = fb_helper;
 
 	return 0;
@@ -437,9 +426,6 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 	cancel_work_sync(&fb_helper->damage_work);
 
 	drm_fb_helper_release_info(fb_helper);
-
-	if (!fb_helper->client.funcs)
-		drm_client_release(&fb_helper->client);
 }
 EXPORT_SYMBOL(drm_fb_helper_fini);
 
@@ -1786,3 +1772,32 @@ int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 	return 0;
 }
 EXPORT_SYMBOL(drm_fb_helper_hotplug_event);
+
+/**
+ * drm_fb_helper_gem_is_fb - Tests if GEM object is framebuffer
+ * @fb_helper: fb_helper instance, can be NULL
+ * @obj: The GEM object to test, can be NULL
+ *
+ * Call drm_fb_helper_gem_is_fb to test is a DRM device's fbdev emulation
+ * uses the specified GEM object for its framebuffer. The result is always
+ * false if either poiner is NULL.
+ *
+ * Returns:
+ * True if fbdev emulation uses the provided GEM object, or false otherwise.
+ */
+bool drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
+			     const struct drm_gem_object *obj)
+{
+	const struct drm_gem_object *gem = NULL;
+
+	if (!fb_helper || !obj)
+		return false;
+	if (fb_helper->buffer && fb_helper->buffer->gem)
+		gem = fb_helper->buffer->gem;
+	else if (fb_helper->fb)
+		gem = drm_gem_fb_get_obj(fb_helper->fb, 0);
+
+	return gem == obj;
+}
+EXPORT_SYMBOL_GPL(drm_fb_helper_gem_is_fb);
+
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index ec820686b3021..f52141f842a1f 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -233,6 +233,7 @@ static void drm_events_release(struct drm_file *file_priv)
 void drm_file_free(struct drm_file *file)
 {
 	struct drm_device *dev;
+	int idx;
 
 	if (!file)
 		return;
@@ -249,9 +250,11 @@ void drm_file_free(struct drm_file *file)
 
 	drm_events_release(file);
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+	if (drm_core_check_feature(dev, DRIVER_MODESET) &&
+	    drm_dev_enter(dev, &idx)) {
 		drm_fb_release(file);
 		drm_property_destroy_user_blobs(dev, file);
+		drm_dev_exit(idx);
 	}
 
 	if (drm_core_check_feature(dev, DRIVER_SYNCOBJ))
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 891c3bff5ae00..da2c988aa286a 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -38,7 +38,7 @@
 #include <linux/mman.h>
 #include <linux/module.h>
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/sched/mm.h>
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
@@ -784,7 +784,7 @@ EXPORT_SYMBOL(drm_gem_put_pages);
 static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
 			  struct drm_gem_object **objs)
 {
-	int i, ret = 0;
+	int i;
 	struct drm_gem_object *obj;
 
 	spin_lock(&filp->table_lock);
@@ -792,16 +792,23 @@ static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
 	for (i = 0; i < count; i++) {
 		/* Check if we currently have a reference on the object */
 		obj = idr_find(&filp->object_idr, handle[i]);
-		if (!obj) {
-			ret = -ENOENT;
-			break;
-		}
+		if (!obj)
+			goto err;
+
 		drm_gem_object_get(obj);
 		objs[i] = obj;
 	}
+
 	spin_unlock(&filp->table_lock);
+	return 0;
 
-	return ret;
+err:
+	spin_unlock(&filp->table_lock);
+
+	while (i--)
+		drm_gem_object_put(objs[i]);
+
+	return -ENOENT;
 }
 
 /**
@@ -829,24 +836,34 @@ int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
 	u32 *handles;
 	int ret;
 
+	*objs_out = NULL;
+
 	if (!count)
 		return 0;
 
-	objs = kvmalloc_objs(struct drm_gem_object *, count,
-			     GFP_KERNEL | __GFP_ZERO);
+	objs = kvmalloc_objs(*objs, count);
 	if (!objs)
 		return -ENOMEM;
 
-	*objs_out = objs;
-
 	handles = vmemdup_array_user(bo_handles, count, sizeof(u32));
-	if (IS_ERR(handles))
-		return PTR_ERR(handles);
+	if (IS_ERR(handles)) {
+		ret = PTR_ERR(handles);
+		goto err_free_objs;
+	}
 
 	ret = objects_lookup(filp, handles, count, objs);
+	if (ret)
+		goto err_free_handles;
+
 	kvfree(handles);
-	return ret;
+	*objs_out = objs;
+	return 0;
 
+err_free_handles:
+	kvfree(handles);
+err_free_objs:
+	kvfree(objs);
+	return ret;
 }
 EXPORT_SYMBOL(drm_gem_objects_lookup);
 
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index 7b5a49935ae40..044f0d4ebf555 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -265,6 +265,8 @@ void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
 				  shmem->pages_mark_dirty_on_put,
 				  shmem->pages_mark_accessed_on_put);
 		shmem->pages = NULL;
+		shmem->pages_mark_accessed_on_put = false;
+		shmem->pages_mark_dirty_on_put = false;
 	}
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_put_pages_locked);
@@ -397,6 +399,8 @@ int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem,
 		} else {
 			iosys_map_set_vaddr(map, shmem->vaddr);
 			refcount_set(&shmem->vmap_use_count, 1);
+			shmem->pages_mark_accessed_on_put = true;
+			shmem->pages_mark_dirty_on_put = true;
 		}
 	}
 
@@ -550,63 +554,68 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create);
 
-static bool drm_gem_shmem_try_map_pmd(struct vm_fault *vmf, unsigned long addr,
-				      struct page *page)
+static vm_fault_t drm_gem_shmem_try_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn)
 {
 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
-	unsigned long pfn = page_to_pfn(page);
 	unsigned long paddr = pfn << PAGE_SHIFT;
-	bool aligned = (addr & ~PMD_MASK) == (paddr & ~PMD_MASK);
+	bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK);
 
-	if (aligned &&
-	    pmd_none(*vmf->pmd) &&
-	    folio_test_pmd_mappable(page_folio(page))) {
+	if (aligned && pmd_none(*vmf->pmd)) {
+		/* Read-only mapping; split upon write fault */
 		pfn &= PMD_MASK >> PAGE_SHIFT;
-		if (vmf_insert_pfn_pmd(vmf, pfn, false) == VM_FAULT_NOPAGE)
-			return true;
+		return vmf_insert_pfn_pmd(vmf, pfn, false);
 	}
 #endif
 
-	return false;
+	return 0;
 }
 
-static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
+static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int order)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj = vma->vm_private_data;
+	struct drm_device *dev = obj->dev;
 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
 	loff_t num_pages = obj->size >> PAGE_SHIFT;
-	vm_fault_t ret;
+	vm_fault_t ret = VM_FAULT_SIGBUS;
 	struct page **pages = shmem->pages;
-	pgoff_t page_offset;
+	pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */
+	struct page *page;
+	struct folio *folio;
 	unsigned long pfn;
 
-	/* Offset to faulty address in the VMA. */
-	page_offset = vmf->pgoff - vma->vm_pgoff;
-
-	dma_resv_lock(shmem->base.resv, NULL);
+	dma_resv_lock(obj->resv, NULL);
 
-	if (page_offset >= num_pages ||
-	    drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||
-	    shmem->madv < 0) {
-		ret = VM_FAULT_SIGBUS;
+	if (page_offset >= num_pages || drm_WARN_ON_ONCE(dev, !shmem->pages) ||
+	    shmem->madv < 0)
 		goto out;
-	}
 
-	if (drm_gem_shmem_try_map_pmd(vmf, vmf->address, pages[page_offset])) {
-		ret = VM_FAULT_NOPAGE;
+	page = pages[page_offset];
+	if (drm_WARN_ON_ONCE(dev, !page))
 		goto out;
-	}
+	folio = page_folio(page);
 
-	pfn = page_to_pfn(pages[page_offset]);
-	ret = vmf_insert_pfn(vma, vmf->address, pfn);
+	pfn = page_to_pfn(page);
 
- out:
-	dma_resv_unlock(shmem->base.resv);
+	if (folio_test_pmd_mappable(folio))
+		ret = drm_gem_shmem_try_insert_pfn_pmd(vmf, pfn);
+	if (ret != VM_FAULT_NOPAGE)
+		ret = vmf_insert_pfn(vma, vmf->address, pfn);
+
+	if (ret == VM_FAULT_NOPAGE)
+		folio_mark_accessed(folio);
+
+out:
+	dma_resv_unlock(obj->resv);
 
 	return ret;
 }
 
+static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
+{
+	return drm_gem_shmem_any_fault(vmf, 0);
+}
+
 static void drm_gem_shmem_vm_open(struct vm_area_struct *vma)
 {
 	struct drm_gem_object *obj = vma->vm_private_data;
@@ -641,10 +650,32 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma)
 	drm_gem_vm_close(vma);
 }
 
+static vm_fault_t drm_gem_shmem_pfn_mkwrite(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	loff_t num_pages = obj->size >> PAGE_SHIFT;
+	pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */
+
+	if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages))
+		return VM_FAULT_SIGBUS;
+
+	file_update_time(vma->vm_file);
+
+	folio_mark_dirty(page_folio(shmem->pages[page_offset]));
+
+	return 0;
+}
+
 const struct vm_operations_struct drm_gem_shmem_vm_ops = {
 	.fault = drm_gem_shmem_fault,
+#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
+	.huge_fault = drm_gem_shmem_any_fault,
+#endif
 	.open = drm_gem_shmem_vm_open,
 	.close = drm_gem_shmem_vm_close,
+	.pfn_mkwrite = drm_gem_shmem_pfn_mkwrite,
 };
 EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_ops);
 
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index 9ef9e52c05473..4b928fda5b127 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -819,7 +819,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
 
 		if (!(pfns[i] & HMM_PFN_VALID)) {
 			state = DRM_GPUSVM_SCAN_UNPOPULATED;
-			goto err_free;
+			break;
 		}
 
 		page = hmm_pfn_to_page(pfns[i]);
@@ -856,9 +856,9 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
 		i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
 	}
 
-err_free:
 	drm_gpusvm_notifier_unlock(range->gpusvm);
 
+err_free:
 	kvfree(pfns);
 	return state;
 }
@@ -1488,14 +1488,17 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
 		if (is_device_private_page(page) ||
 		    is_device_coherent_page(page)) {
+			struct drm_pagemap_zdd *__zdd =
+				drm_pagemap_page_zone_device_data(page);
+
 			if (!ctx->allow_mixed &&
-			    zdd != page->zone_device_data && i > 0) {
+			    zdd != __zdd && i > 0) {
 				err = -EOPNOTSUPP;
 				goto err_unmap;
 			}
-			zdd = page->zone_device_data;
+			zdd = __zdd;
 			if (pagemap != page_pgmap(page)) {
-				if (i > 0) {
+				if (pagemap) {
 					err = -EOPNOTSUPP;
 					goto err_unmap;
 				}
@@ -1572,6 +1575,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	return 0;
 
 err_unmap:
+	svm_pages->flags.has_dma_mapping = true;
 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped);
 	drm_gpusvm_notifier_unlock(gpusvm);
 err_free:
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index d12db9b0bab81..65b7f4f22f091 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -380,6 +380,12 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
 		return -ENOMEM;
 	dev->mode_config.gamma_lut_size_property = prop;
 
+	prop = drm_property_create_range(dev, 0,
+					 "BACKGROUND_COLOR", 0, U64_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.background_color_property = prop;
+
 	prop = drm_property_create(dev,
 				   DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB,
 				   "IN_FORMATS", 0);
@@ -524,6 +530,7 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 	struct drm_property *property, *pt;
 	struct drm_property_blob *blob, *bt;
 	struct drm_plane *plane, *plt;
+	struct drm_colorop *colorop, *copt;
 
 	list_for_each_entry_safe(encoder, enct, &dev->mode_config.encoder_list,
 				 head) {
@@ -553,6 +560,11 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 		drm_property_destroy(dev, property);
 	}
 
+	list_for_each_entry_safe(colorop, copt, &dev->mode_config.colorop_list,
+				 head) {
+		colorop->funcs->destroy(colorop);
+	}
+
 	list_for_each_entry_safe(plane, plt, &dev->mode_config.plane_list,
 				 head) {
 		plane->funcs->destroy(plane);
@@ -577,10 +589,13 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 	 */
 	WARN_ON(!list_empty(&dev->mode_config.fb_list));
 	list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
-		struct drm_printer p = drm_dbg_printer(dev, DRM_UT_KMS, "[leaked fb]");
+		if (list_empty(&fb->filp_head) || drm_framebuffer_read_refcount(fb) > 1) {
+			struct drm_printer p = drm_dbg_printer(dev, DRM_UT_KMS, "[leaked fb]");
 
-		drm_printf(&p, "framebuffer[%u]:\n", fb->base.id);
-		drm_framebuffer_print_info(&p, 1, fb);
+			drm_printf(&p, "framebuffer[%u]:\n", fb->base.id);
+			drm_framebuffer_print_info(&p, 1, fb);
+		}
+		list_del_init(&fb->filp_head);
 		drm_framebuffer_free(&fb->base.refcount);
 	}
 
diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c
index 862675ac5bb21..5002049e01981 100644
--- a/drivers/gpu/drm/drm_pagemap.c
+++ b/drivers/gpu/drm/drm_pagemap.c
@@ -154,15 +154,15 @@ static void drm_pagemap_zdd_put(struct drm_pagemap_zdd *zdd)
 }
 
 /**
- * drm_pagemap_migration_unlock_put_page() - Put a migration page
- * @page: Pointer to the page to put
+ * drm_pagemap_migration_unlock_put_folio() - Put a migration folio
+ * @folio: Pointer to the folio to put
  *
- * This function unlocks and puts a page.
+ * This function unlocks and puts a folio.
  */
-static void drm_pagemap_migration_unlock_put_page(struct page *page)
+static void drm_pagemap_migration_unlock_put_folio(struct folio *folio)
 {
-	unlock_page(page);
-	put_page(page);
+	folio_unlock(folio);
+	folio_put(folio);
 }
 
 /**
@@ -177,31 +177,42 @@ static void drm_pagemap_migration_unlock_put_pages(unsigned long npages,
 {
 	unsigned long i;
 
-	for (i = 0; i < npages; ++i) {
+	for (i = 0; i < npages;) {
 		struct page *page;
+		struct folio *folio;
+		unsigned int order = 0;
 
 		if (!migrate_pfn[i])
-			continue;
+			goto next;
 
 		page = migrate_pfn_to_page(migrate_pfn[i]);
-		drm_pagemap_migration_unlock_put_page(page);
+		folio = page_folio(page);
+		order = folio_order(folio);
+
+		drm_pagemap_migration_unlock_put_folio(folio);
 		migrate_pfn[i] = 0;
+
+next:
+		i += NR_PAGES(order);
 	}
 }
 
 /**
  * drm_pagemap_get_devmem_page() - Get a reference to a device memory page
  * @page: Pointer to the page
+ * @order: Order
  * @zdd: Pointer to the GPU SVM zone device data
  *
  * This function associates the given page with the specified GPU SVM zone
  * device data and initializes it for zone device usage.
  */
 static void drm_pagemap_get_devmem_page(struct page *page,
+					unsigned int order,
 					struct drm_pagemap_zdd *zdd)
 {
-	page->zone_device_data = drm_pagemap_zdd_get(zdd);
-	zone_device_page_init(page, page_pgmap(page), 0);
+	zone_device_folio_init((struct folio *)page, zdd->dpagemap->pagemap,
+			       order);
+	folio_set_zone_device_data(page_folio(page), drm_pagemap_zdd_get(zdd));
 }
 
 /**
@@ -244,7 +255,7 @@ static int drm_pagemap_migrate_map_pages(struct device *dev,
 		order = folio_order(folio);
 
 		if (is_device_private_page(page)) {
-			struct drm_pagemap_zdd *zdd = page->zone_device_data;
+			struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
 			struct drm_pagemap *dpagemap = zdd->dpagemap;
 			struct drm_pagemap_addr addr;
 
@@ -315,7 +326,7 @@ static void drm_pagemap_migrate_unmap_pages(struct device *dev,
 			goto next;
 
 		if (is_zone_device_page(page)) {
-			struct drm_pagemap_zdd *zdd = page->zone_device_data;
+			struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
 			struct drm_pagemap *dpagemap = zdd->dpagemap;
 
 			dpagemap->ops->device_unmap(dpagemap, dev, &pagemap_addr[i]);
@@ -444,6 +455,41 @@ static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem,
 	return ret;
 }
 
+/**
+ * drm_pagemap_cpages() - Count collected pages
+ * @migrate_pfn: Array of migrate_pfn entries to account
+ * @npages: Number of entries in @migrate_pfn
+ *
+ * Compute the total number of minimum-sized pages represented by the
+ * collected entries in @migrate_pfn. The total is derived from the
+ * order encoded in each entry.
+ *
+ * Return: Total number of minimum-sized pages.
+ */
+static int drm_pagemap_cpages(unsigned long *migrate_pfn, unsigned long npages)
+{
+	unsigned long i, cpages = 0;
+
+	for (i = 0; i < npages;) {
+		struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
+		struct folio *folio;
+		unsigned int order = 0;
+
+		if (page) {
+			folio = page_folio(page);
+			order = folio_order(folio);
+			cpages += NR_PAGES(order);
+		} else if (migrate_pfn[i] & MIGRATE_PFN_COMPOUND) {
+			order = HPAGE_PMD_ORDER;
+			cpages += NR_PAGES(order);
+		}
+
+		i += NR_PAGES(order);
+	}
+
+	return cpages;
+}
+
 /**
  * drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory
  * @devmem_allocation: The device memory allocation to migrate to.
@@ -481,7 +527,7 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 		.end		= end,
 		.pgmap_owner	= pagemap->owner,
 		.flags		= MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_DEVICE_COHERENT |
-		MIGRATE_VMA_SELECT_DEVICE_PRIVATE,
+		MIGRATE_VMA_SELECT_DEVICE_PRIVATE | MIGRATE_VMA_SELECT_COMPOUND,
 	};
 	unsigned long i, npages = npages_in_range(start, end);
 	unsigned long own_pages = 0, migrated_pages = 0;
@@ -546,7 +592,8 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 		goto err_free;
 	}
 
-	if (migrate.cpages != npages) {
+	if (migrate.cpages != npages &&
+	    drm_pagemap_cpages(migrate.src, npages) != npages) {
 		/*
 		 * Some pages to migrate. But we want to migrate all or
 		 * nothing. Raced or unknown device pages.
@@ -586,20 +633,23 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 
 	own_pages = 0;
 
-	for (i = 0; i < npages; ++i) {
+	for (i = 0; i < npages;) {
+		unsigned long j;
 		struct page *page = pfn_to_page(migrate.dst[i]);
 		struct page *src_page = migrate_pfn_to_page(migrate.src[i]);
-		cur.start = i;
+		unsigned int order = 0;
 
+		cur.start = i;
 		pages[i] = NULL;
 		if (src_page && is_device_private_page(src_page)) {
-			struct drm_pagemap_zdd *src_zdd = src_page->zone_device_data;
+			struct drm_pagemap_zdd *src_zdd =
+				drm_pagemap_page_zone_device_data(src_page);
 
 			if (page_pgmap(src_page) == pagemap &&
 			    !mdetails->can_migrate_same_pagemap) {
 				migrate.dst[i] = 0;
 				own_pages++;
-				continue;
+				goto next;
 			}
 			if (mdetails->source_peer_migrates) {
 				cur.dpagemap = src_zdd->dpagemap;
@@ -615,7 +665,20 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 			pages[i] = page;
 		}
 		migrate.dst[i] = migrate_pfn(migrate.dst[i]);
-		drm_pagemap_get_devmem_page(page, zdd);
+
+		if (migrate.src[i] & MIGRATE_PFN_COMPOUND) {
+			drm_WARN_ONCE(dpagemap->drm, src_page &&
+				      folio_order(page_folio(src_page)) != HPAGE_PMD_ORDER,
+				      "Unexpected folio order\n");
+
+			order = HPAGE_PMD_ORDER;
+			migrate.dst[i] |= MIGRATE_PFN_COMPOUND;
+
+			for (j = 1; j < NR_PAGES(order) && i + j < npages; j++)
+				migrate.dst[i + j] = 0;
+		}
+
+		drm_pagemap_get_devmem_page(page, order, zdd);
 
 		/* If we switched the migrating drm_pagemap, migrate previous pages now */
 		err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,
@@ -625,7 +688,11 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 			npages = i + 1;
 			goto err_finalize;
 		}
+
+next:
+		i += NR_PAGES(order);
 	}
+
 	cur.start = npages;
 	cur.ops = NULL; /* Force migration */
 	err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,
@@ -715,8 +782,8 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
 			goto next;
 
 		if (fault_page) {
-			if (src_page->zone_device_data !=
-			    fault_page->zone_device_data)
+			if (drm_pagemap_page_zone_device_data(src_page) !=
+			    drm_pagemap_page_zone_device_data(fault_page))
 				goto next;
 		}
 
@@ -734,6 +801,8 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
 		page = folio_page(folio, 0);
 		mpfn[i] = migrate_pfn(page_to_pfn(page));
 
+		if (order)
+			mpfn[i] |= MIGRATE_PFN_COMPOUND;
 next:
 		if (page)
 			addr += page_size(page);
@@ -989,8 +1058,15 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
 	if (err)
 		goto err_finalize;
 
-	for (i = 0; i < npages; ++i)
+	for (i = 0; i < npages;) {
+		unsigned int order = 0;
+
 		pages[i] = migrate_pfn_to_page(src[i]);
+		if (pages[i])
+			order = folio_order(page_folio(pages[i]));
+
+		i += NR_PAGES(order);
+	}
 
 	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
 	if (err)
@@ -1043,7 +1119,8 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 		.vma		= vas,
 		.pgmap_owner	= page_pgmap(page)->owner,
 		.flags		= MIGRATE_VMA_SELECT_DEVICE_PRIVATE |
-		MIGRATE_VMA_SELECT_DEVICE_COHERENT,
+				  MIGRATE_VMA_SELECT_DEVICE_COHERENT |
+				  MIGRATE_VMA_SELECT_COMPOUND,
 		.fault_page	= page,
 	};
 	struct drm_pagemap_migrate_details mdetails = {};
@@ -1057,7 +1134,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 	void *buf;
 	int i, err = 0;
 
-	zdd = page->zone_device_data;
+	zdd = drm_pagemap_page_zone_device_data(page);
 	if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration))
 		return 0;
 
@@ -1109,8 +1186,15 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 	if (err)
 		goto err_finalize;
 
-	for (i = 0; i < npages; ++i)
+	for (i = 0; i < npages;) {
+		unsigned int order = 0;
+
 		pages[i] = migrate_pfn_to_page(migrate.src[i]);
+		if (pages[i])
+			order = folio_order(page_folio(pages[i]));
+
+		i += NR_PAGES(order);
+	}
 
 	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
 	if (err)
@@ -1140,7 +1224,9 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
  */
 static void drm_pagemap_folio_free(struct folio *folio)
 {
-	drm_pagemap_zdd_put(folio->page.zone_device_data);
+	struct page *page = folio_page(folio, 0);
+
+	drm_pagemap_zdd_put(drm_pagemap_page_zone_device_data(page));
 }
 
 /**
@@ -1156,7 +1242,7 @@ static void drm_pagemap_folio_free(struct folio *folio)
  */
 static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf)
 {
-	struct drm_pagemap_zdd *zdd = vmf->page->zone_device_data;
+	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(vmf->page);
 	int err;
 
 	err = __drm_pagemap_migrate_to_ram(vmf->vma,
@@ -1166,9 +1252,22 @@ static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf)
 	return err ? VM_FAULT_SIGBUS : 0;
 }
 
+static void drm_pagemap_folio_split(struct folio *orig_folio, struct folio *new_folio)
+{
+	struct drm_pagemap_zdd *zdd;
+
+	if (!new_folio)
+		return;
+
+	new_folio->pgmap = orig_folio->pgmap;
+	zdd = folio_zone_device_data(orig_folio);
+	folio_set_zone_device_data(new_folio, drm_pagemap_zdd_get(zdd));
+}
+
 static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = {
 	.folio_free = drm_pagemap_folio_free,
 	.migrate_to_ram = drm_pagemap_migrate_to_ram,
+	.folio_split = drm_pagemap_folio_split,
 };
 
 /**
@@ -1222,7 +1321,7 @@ EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init);
  */
 struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page)
 {
-	struct drm_pagemap_zdd *zdd = page->zone_device_data;
+	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
 
 	return zdd->devmem_allocation->dpagemap;
 }
diff --git a/drivers/gpu/drm/drm_pagemap_util.c b/drivers/gpu/drm/drm_pagemap_util.c
index 14ddb948a32e9..6111d90a38e24 100644
--- a/drivers/gpu/drm/drm_pagemap_util.c
+++ b/drivers/gpu/drm/drm_pagemap_util.c
@@ -65,18 +65,14 @@ static void drm_pagemap_cache_fini(void *arg)
 	drm_dbg(cache->shrinker->drm, "Destroying dpagemap cache.\n");
 	spin_lock(&cache->lock);
 	dpagemap = cache->dpagemap;
-	if (!dpagemap) {
-		spin_unlock(&cache->lock);
-		goto out;
-	}
+	cache->dpagemap = NULL;
+	if (dpagemap && !drm_pagemap_shrinker_cancel(dpagemap))
+		dpagemap = NULL;
+	spin_unlock(&cache->lock);
 
-	if (drm_pagemap_shrinker_cancel(dpagemap)) {
-		cache->dpagemap = NULL;
-		spin_unlock(&cache->lock);
+	if (dpagemap)
 		drm_pagemap_destroy(dpagemap, false);
-	}
 
-out:
 	mutex_destroy(&cache->lookup_mutex);
 	kfree(cache);
 }
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 09b12c30df69d..d4dc8cb45bcec 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -626,7 +626,7 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 		 */
 		dev->mode_config.delayed_event = true;
 		if (dev->mode_config.poll_enabled)
-			mod_delayed_work(system_wq,
+			mod_delayed_work(system_percpu_wq,
 					 &dev->mode_config.output_poll_work,
 					 0);
 	}
diff --git a/drivers/gpu/drm/drm_ras.c b/drivers/gpu/drm/drm_ras.c
new file mode 100644
index 0000000000000..b2fa5ab86d870
--- /dev/null
+++ b/drivers/gpu/drm/drm_ras.c
@@ -0,0 +1,354 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/netdevice.h>
+#include <linux/xarray.h>
+#include <net/genetlink.h>
+
+#include <drm/drm_ras.h>
+
+#include "drm_ras_nl.h"
+
+/**
+ * DOC: DRM RAS Node Management
+ *
+ * This module provides the infrastructure to manage RAS (Reliability,
+ * Availability, and Serviceability) nodes for DRM drivers. Each
+ * DRM driver may register one or more RAS nodes, which represent
+ * logical components capable of reporting error counters and other
+ * reliability metrics.
+ *
+ * The nodes are stored in a global xarray `drm_ras_xa` to allow
+ * efficient lookup by ID. Nodes can be registered or unregistered
+ * dynamically at runtime.
+ *
+ * A Generic Netlink family `drm_ras` exposes two main operations to
+ * userspace:
+ *
+ * 1. LIST_NODES: Dump all currently registered RAS nodes.
+ *    The user receives an array of node IDs, names, and types.
+ *
+ * 2. GET_ERROR_COUNTER: Get error counters of a given node.
+ *    Userspace must provide Node ID, Error ID (Optional for specific counter).
+ *    Returns all counters of a node if only Node ID is provided or specific
+ *    error counters.
+ *
+ * Node registration:
+ *
+ * - drm_ras_node_register(): Registers a new node and assigns
+ *   it a unique ID in the xarray.
+ * - drm_ras_node_unregister(): Removes a previously registered
+ *   node from the xarray.
+ *
+ * Node type:
+ *
+ * - ERROR_COUNTER:
+ *     + Currently, only error counters are supported.
+ *     + The driver must implement the query_error_counter() callback to provide
+ *       the name and the value of the error counter.
+ *     + The driver must provide a error_counter_range.last value informing the
+ *       last valid error ID.
+ *     + The driver can provide a error_counter_range.first value informing the
+ *       first valid error ID.
+ *     + The error counters in the driver doesn't need to be contiguous, but the
+ *       driver must return -ENOENT to the query_error_counter as an indication
+ *       that the ID should be skipped and not listed in the netlink API.
+ *
+ * Netlink handlers:
+ *
+ * - drm_ras_nl_list_nodes_dumpit(): Implements the LIST_NODES
+ *   operation, iterating over the xarray.
+ * - drm_ras_nl_get_error_counter_dumpit(): Implements the GET_ERROR_COUNTER dumpit
+ *   operation, fetching all counters from a specific node.
+ * - drm_ras_nl_get_error_counter_doit(): Implements the GET_ERROR_COUNTER doit
+ *   operation, fetching a counter value from a specific node.
+ */
+
+static DEFINE_XARRAY_ALLOC(drm_ras_xa);
+
+/*
+ * The netlink callback context carries dump state across multiple dumpit calls
+ */
+struct drm_ras_ctx {
+	/* Which xarray id to restart the dump from */
+	unsigned long restart;
+};
+
+/**
+ * drm_ras_nl_list_nodes_dumpit() - Dump all registered RAS nodes
+ * @skb: Netlink message buffer
+ * @cb: Callback context for multi-part dumps
+ *
+ * Iterates over all registered RAS nodes in the global xarray and appends
+ * their attributes (ID, name, type) to the given netlink message buffer.
+ * Uses @cb->ctx to track progress in case the message buffer fills up, allowing
+ * multi-part dump support. On buffer overflow, updates the context to resume
+ * from the last node on the next invocation.
+ *
+ * Return: 0 if all nodes fit in @skb, number of bytes added to @skb if
+ *          the buffer filled up (requires multi-part continuation), or
+ *          a negative error code on failure.
+ */
+int drm_ras_nl_list_nodes_dumpit(struct sk_buff *skb,
+				 struct netlink_callback *cb)
+{
+	const struct genl_info *info = genl_info_dump(cb);
+	struct drm_ras_ctx *ctx = (void *)cb->ctx;
+	struct drm_ras_node *node;
+	struct nlattr *hdr;
+	unsigned long id;
+	int ret;
+
+	xa_for_each_start(&drm_ras_xa, id, node, ctx->restart) {
+		hdr = genlmsg_iput(skb, info);
+		if (!hdr) {
+			ret = -EMSGSIZE;
+			break;
+		}
+
+		ret = nla_put_u32(skb, DRM_RAS_A_NODE_ATTRS_NODE_ID, node->id);
+		if (ret) {
+			genlmsg_cancel(skb, hdr);
+			break;
+		}
+
+		ret = nla_put_string(skb, DRM_RAS_A_NODE_ATTRS_DEVICE_NAME,
+				     node->device_name);
+		if (ret) {
+			genlmsg_cancel(skb, hdr);
+			break;
+		}
+
+		ret = nla_put_string(skb, DRM_RAS_A_NODE_ATTRS_NODE_NAME,
+				     node->node_name);
+		if (ret) {
+			genlmsg_cancel(skb, hdr);
+			break;
+		}
+
+		ret = nla_put_u32(skb, DRM_RAS_A_NODE_ATTRS_NODE_TYPE,
+				  node->type);
+		if (ret) {
+			genlmsg_cancel(skb, hdr);
+			break;
+		}
+
+		genlmsg_end(skb, hdr);
+	}
+
+	if (ret == -EMSGSIZE)
+		ctx->restart = id;
+
+	return ret;
+}
+
+static int get_node_error_counter(u32 node_id, u32 error_id,
+				  const char **name, u32 *value)
+{
+	struct drm_ras_node *node;
+
+	node = xa_load(&drm_ras_xa, node_id);
+	if (!node || !node->query_error_counter)
+		return -ENOENT;
+
+	if (error_id < node->error_counter_range.first ||
+	    error_id > node->error_counter_range.last)
+		return -EINVAL;
+
+	return node->query_error_counter(node, error_id, name, value);
+}
+
+static int msg_reply_value(struct sk_buff *msg, u32 error_id,
+			   const char *error_name, u32 value)
+{
+	int ret;
+
+	ret = nla_put_u32(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID, error_id);
+	if (ret)
+		return ret;
+
+	ret = nla_put_string(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_NAME,
+			     error_name);
+	if (ret)
+		return ret;
+
+	return nla_put_u32(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_VALUE,
+			   value);
+}
+
+static int doit_reply_value(struct genl_info *info, u32 node_id,
+			    u32 error_id)
+{
+	struct sk_buff *msg;
+	struct nlattr *hdr;
+	const char *error_name;
+	u32 value;
+	int ret;
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_iput(msg, info);
+	if (!hdr) {
+		nlmsg_free(msg);
+		return -EMSGSIZE;
+	}
+
+	ret = get_node_error_counter(node_id, error_id,
+				     &error_name, &value);
+	if (ret)
+		return ret;
+
+	ret = msg_reply_value(msg, error_id, error_name, value);
+	if (ret) {
+		genlmsg_cancel(msg, hdr);
+		nlmsg_free(msg);
+		return ret;
+	}
+
+	genlmsg_end(msg, hdr);
+
+	return genlmsg_reply(msg, info);
+}
+
+/**
+ * drm_ras_nl_get_error_counter_dumpit() - Dump all Error Counters
+ * @skb: Netlink message buffer
+ * @cb: Callback context for multi-part dumps
+ *
+ * Iterates over all error counters in a given Node and appends
+ * their attributes (ID, name, value) to the given netlink message buffer.
+ * Uses @cb->ctx to track progress in case the message buffer fills up, allowing
+ * multi-part dump support. On buffer overflow, updates the context to resume
+ * from the last node on the next invocation.
+ *
+ * Return: 0 if all errors fit in @skb, number of bytes added to @skb if
+ *          the buffer filled up (requires multi-part continuation), or
+ *          a negative error code on failure.
+ */
+int drm_ras_nl_get_error_counter_dumpit(struct sk_buff *skb,
+					struct netlink_callback *cb)
+{
+	const struct genl_info *info = genl_info_dump(cb);
+	struct drm_ras_ctx *ctx = (void *)cb->ctx;
+	struct drm_ras_node *node;
+	struct nlattr *hdr;
+	const char *error_name;
+	u32 node_id, error_id, value;
+	int ret;
+
+	if (!info->attrs || GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID))
+		return -EINVAL;
+
+	node_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID]);
+
+	node = xa_load(&drm_ras_xa, node_id);
+	if (!node)
+		return -ENOENT;
+
+	for (error_id = max(node->error_counter_range.first, ctx->restart);
+	     error_id <= node->error_counter_range.last;
+	     error_id++) {
+		ret = get_node_error_counter(node_id, error_id,
+					     &error_name, &value);
+		/*
+		 * For non-contiguous range, driver return -ENOENT as indication
+		 * to skip this ID when listing all errors.
+		 */
+		if (ret == -ENOENT)
+			continue;
+		if (ret)
+			return ret;
+
+		hdr = genlmsg_iput(skb, info);
+
+		if (!hdr) {
+			ret = -EMSGSIZE;
+			break;
+		}
+
+		ret = msg_reply_value(skb, error_id, error_name, value);
+		if (ret) {
+			genlmsg_cancel(skb, hdr);
+			break;
+		}
+
+		genlmsg_end(skb, hdr);
+	}
+
+	if (ret == -EMSGSIZE)
+		ctx->restart = error_id;
+
+	return ret;
+}
+
+/**
+ * drm_ras_nl_get_error_counter_doit() - Query an error counter of an node
+ * @skb: Netlink message buffer
+ * @info: Generic Netlink info containing attributes of the request
+ *
+ * Extracts the node ID and error ID from the netlink attributes and
+ * retrieves the current value of the corresponding error counter. Sends the
+ * result back to the requesting user via the standard Genl reply.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int drm_ras_nl_get_error_counter_doit(struct sk_buff *skb,
+				      struct genl_info *info)
+{
+	u32 node_id, error_id;
+
+	if (!info->attrs ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID))
+		return -EINVAL;
+
+	node_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID]);
+	error_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID]);
+
+	return doit_reply_value(info, node_id, error_id);
+}
+
+/**
+ * drm_ras_node_register() - Register a new RAS node
+ * @node: Node structure to register
+ *
+ * Adds the given RAS node to the global node xarray and assigns it
+ * a unique ID. Both @node->name and @node->type must be valid.
+ *
+ * Return: 0 on success, or negative errno on failure:
+ */
+int drm_ras_node_register(struct drm_ras_node *node)
+{
+	if (!node->device_name || !node->node_name)
+		return -EINVAL;
+
+	/* Currently, only Error Counter Endpoints are supported */
+	if (node->type != DRM_RAS_NODE_TYPE_ERROR_COUNTER)
+		return -EINVAL;
+
+	/* Mandatory entries for Error Counter Node */
+	if (node->type == DRM_RAS_NODE_TYPE_ERROR_COUNTER &&
+	    (!node->error_counter_range.last || !node->query_error_counter))
+		return -EINVAL;
+
+	return xa_alloc(&drm_ras_xa, &node->id, node, xa_limit_32b, GFP_KERNEL);
+}
+EXPORT_SYMBOL(drm_ras_node_register);
+
+/**
+ * drm_ras_node_unregister() - Unregister a previously registered node
+ * @node: Node structure to unregister
+ *
+ * Removes the given node from the global node xarray using its ID.
+ */
+void drm_ras_node_unregister(struct drm_ras_node *node)
+{
+	xa_erase(&drm_ras_xa, node->id);
+}
+EXPORT_SYMBOL(drm_ras_node_unregister);
diff --git a/drivers/gpu/drm/drm_ras_genl_family.c b/drivers/gpu/drm/drm_ras_genl_family.c
new file mode 100644
index 0000000000000..6f406d3d48c5e
--- /dev/null
+++ b/drivers/gpu/drm/drm_ras_genl_family.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <drm/drm_ras_genl_family.h>
+#include "drm_ras_nl.h"
+
+/* Track family registration so the drm_exit can be called at any time */
+static bool registered;
+
+/**
+ * drm_ras_genl_family_register() - Register drm-ras genl family
+ *
+ * Only to be called one at drm_drv_init()
+ */
+int drm_ras_genl_family_register(void)
+{
+	int ret;
+
+	registered = false;
+
+	ret = genl_register_family(&drm_ras_nl_family);
+	if (ret)
+		return ret;
+
+	registered = true;
+	return 0;
+}
+
+/**
+ * drm_ras_genl_family_unregister() - Unregister drm-ras genl family
+ *
+ * To be called one at drm_drv_exit() at any moment, but only once.
+ */
+void drm_ras_genl_family_unregister(void)
+{
+	if (registered) {
+		genl_unregister_family(&drm_ras_nl_family);
+		registered = false;
+	}
+}
diff --git a/drivers/gpu/drm/drm_ras_nl.c b/drivers/gpu/drm/drm_ras_nl.c
new file mode 100644
index 0000000000000..16803d0c4a44c
--- /dev/null
+++ b/drivers/gpu/drm/drm_ras_nl.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/drm_ras.yaml */
+/* YNL-GEN kernel source */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include "drm_ras_nl.h"
+
+#include <uapi/drm/drm_ras.h>
+
+/* DRM_RAS_CMD_GET_ERROR_COUNTER - do */
+static const struct nla_policy drm_ras_get_error_counter_do_nl_policy[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID + 1] = {
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID] = { .type = NLA_U32, },
+};
+
+/* DRM_RAS_CMD_GET_ERROR_COUNTER - dump */
+static const struct nla_policy drm_ras_get_error_counter_dump_nl_policy[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID + 1] = {
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
+};
+
+/* Ops table for drm_ras */
+static const struct genl_split_ops drm_ras_nl_ops[] = {
+	{
+		.cmd	= DRM_RAS_CMD_LIST_NODES,
+		.dumpit	= drm_ras_nl_list_nodes_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRM_RAS_CMD_GET_ERROR_COUNTER,
+		.doit		= drm_ras_nl_get_error_counter_doit,
+		.policy		= drm_ras_get_error_counter_do_nl_policy,
+		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRM_RAS_CMD_GET_ERROR_COUNTER,
+		.dumpit		= drm_ras_nl_get_error_counter_dumpit,
+		.policy		= drm_ras_get_error_counter_dump_nl_policy,
+		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+};
+
+struct genl_family drm_ras_nl_family __ro_after_init = {
+	.name		= DRM_RAS_FAMILY_NAME,
+	.version	= DRM_RAS_FAMILY_VERSION,
+	.netnsok	= true,
+	.parallel_ops	= true,
+	.module		= THIS_MODULE,
+	.split_ops	= drm_ras_nl_ops,
+	.n_split_ops	= ARRAY_SIZE(drm_ras_nl_ops),
+};
diff --git a/drivers/gpu/drm/drm_ras_nl.h b/drivers/gpu/drm/drm_ras_nl.h
new file mode 100644
index 0000000000000..06ccd93427739
--- /dev/null
+++ b/drivers/gpu/drm/drm_ras_nl.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/drm_ras.yaml */
+/* YNL-GEN kernel header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _LINUX_DRM_RAS_GEN_H
+#define _LINUX_DRM_RAS_GEN_H
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <uapi/drm/drm_ras.h>
+
+int drm_ras_nl_list_nodes_dumpit(struct sk_buff *skb,
+				 struct netlink_callback *cb);
+int drm_ras_nl_get_error_counter_doit(struct sk_buff *skb,
+				      struct genl_info *info);
+int drm_ras_nl_get_error_counter_dumpit(struct sk_buff *skb,
+					struct netlink_callback *cb);
+
+extern struct genl_family drm_ras_nl_family;
+
+#endif /* _LINUX_DRM_RAS_GEN_H */
diff --git a/drivers/gpu/drm/drm_self_refresh_helper.c b/drivers/gpu/drm/drm_self_refresh_helper.c
index 74907f33841ce..c08e7f590ca55 100644
--- a/drivers/gpu/drm/drm_self_refresh_helper.c
+++ b/drivers/gpu/drm/drm_self_refresh_helper.c
@@ -218,7 +218,7 @@ void drm_self_refresh_helper_alter_state(struct drm_atomic_state *state)
 			 ewma_psr_time_read(&sr_data->exit_avg_ms)) * 2;
 		mutex_unlock(&sr_data->avg_mutex);
 
-		mod_delayed_work(system_wq, &sr_data->entry_work,
+		mod_delayed_work(system_percpu_wq, &sr_data->entry_work,
 				 msecs_to_jiffies(delay));
 	}
 }
diff --git a/drivers/gpu/drm/drm_suballoc.c b/drivers/gpu/drm/drm_suballoc.c
index e44ad39e310cd..b74277bbc14bf 100644
--- a/drivers/gpu/drm/drm_suballoc.c
+++ b/drivers/gpu/drm/drm_suballoc.c
@@ -293,45 +293,66 @@ static bool drm_suballoc_next_hole(struct drm_suballoc_manager *sa_manager,
 }
 
 /**
- * drm_suballoc_new() - Make a suballocation.
+ * drm_suballoc_alloc() - Allocate uninitialized suballoc object.
+ * @gfp: gfp flags used for memory allocation.
+ *
+ * Allocate memory for an uninitialized suballoc object. Intended usage is
+ * allocate memory for suballoc object outside of a reclaim tainted context
+ * and then be initialized at a later time in a reclaim tainted context.
+ *
+ * @drm_suballoc_free() should be used to release the memory if returned
+ * suballoc object is in uninitialized state.
+ *
+ * Return: a new uninitialized suballoc object, or an ERR_PTR(-ENOMEM).
+ */
+struct drm_suballoc *drm_suballoc_alloc(gfp_t gfp)
+{
+	struct drm_suballoc *sa;
+
+	sa = kmalloc_obj(*sa, gfp);
+	if (!sa)
+		return ERR_PTR(-ENOMEM);
+
+	sa->manager = NULL;
+
+	return sa;
+}
+EXPORT_SYMBOL(drm_suballoc_alloc);
+
+/**
+ * drm_suballoc_insert() - Initialize a suballocation and insert a hole.
  * @sa_manager: pointer to the sa_manager
+ * @sa: The struct drm_suballoc.
  * @size: number of bytes we want to suballocate.
- * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL but
- *       the argument is provided for suballocations from reclaim context or
- *       where the caller wants to avoid pipelining rather than wait for
- *       reclaim.
  * @intr: Whether to perform waits interruptible. This should typically
  *        always be true, unless the caller needs to propagate a
  *        non-interruptible context from above layers.
  * @align: Alignment. Must not exceed the default manager alignment.
  *         If @align is zero, then the manager alignment is used.
  *
- * Try to make a suballocation of size @size, which will be rounded
- * up to the alignment specified in specified in drm_suballoc_manager_init().
+ * Try to make a suballocation on a pre-allocated suballoc object of size @size,
+ * which will be rounded up to the alignment specified in specified in
+ * drm_suballoc_manager_init().
  *
- * Return: a new suballocated bo, or an ERR_PTR.
+ * Return: zero on success, errno on failure.
  */
-struct drm_suballoc *
-drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
-		 gfp_t gfp, bool intr, size_t align)
+int drm_suballoc_insert(struct drm_suballoc_manager *sa_manager,
+			struct drm_suballoc *sa, size_t size,
+			bool intr, size_t align)
 {
 	struct dma_fence *fences[DRM_SUBALLOC_MAX_QUEUES];
 	unsigned int tries[DRM_SUBALLOC_MAX_QUEUES];
 	unsigned int count;
 	int i, r;
-	struct drm_suballoc *sa;
 
 	if (WARN_ON_ONCE(align > sa_manager->align))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	if (WARN_ON_ONCE(size > sa_manager->size || !size))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	if (!align)
 		align = sa_manager->align;
 
-	sa = kmalloc_obj(*sa, gfp);
-	if (!sa)
-		return ERR_PTR(-ENOMEM);
 	sa->manager = sa_manager;
 	sa->fence = NULL;
 	INIT_LIST_HEAD(&sa->olist);
@@ -348,7 +369,7 @@ drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
 			if (drm_suballoc_try_alloc(sa_manager, sa,
 						   size, align)) {
 				spin_unlock(&sa_manager->wq.lock);
-				return sa;
+				return 0;
 			}
 
 			/* see if we can skip over some allocations */
@@ -385,8 +406,48 @@ drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
 	} while (!r);
 
 	spin_unlock(&sa_manager->wq.lock);
-	kfree(sa);
-	return ERR_PTR(r);
+	sa->manager = NULL;
+	return r;
+}
+EXPORT_SYMBOL(drm_suballoc_insert);
+
+/**
+ * drm_suballoc_new() - Make a suballocation.
+ * @sa_manager: pointer to the sa_manager
+ * @size: number of bytes we want to suballocate.
+ * @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL but
+ *       the argument is provided for suballocations from reclaim context or
+ *       where the caller wants to avoid pipelining rather than wait for
+ *       reclaim.
+ * @intr: Whether to perform waits interruptible. This should typically
+ *        always be true, unless the caller needs to propagate a
+ *        non-interruptible context from above layers.
+ * @align: Alignment. Must not exceed the default manager alignment.
+ *         If @align is zero, then the manager alignment is used.
+ *
+ * Try to make a suballocation of size @size, which will be rounded
+ * up to the alignment specified in specified in drm_suballoc_manager_init().
+ *
+ * Return: a new suballocated bo, or an ERR_PTR.
+ */
+struct drm_suballoc *
+drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
+		 gfp_t gfp, bool intr, size_t align)
+{
+	struct drm_suballoc *sa;
+	int err;
+
+	sa = drm_suballoc_alloc(gfp);
+	if (IS_ERR(sa))
+		return sa;
+
+	err = drm_suballoc_insert(sa_manager, sa, size, intr, align);
+	if (err) {
+		drm_suballoc_free(sa, NULL);
+		return ERR_PTR(err);
+	}
+
+	return sa;
 }
 EXPORT_SYMBOL(drm_suballoc_new);
 
@@ -405,6 +466,11 @@ void drm_suballoc_free(struct drm_suballoc *suballoc,
 	if (!suballoc)
 		return;
 
+	if (!suballoc->manager) {
+		kfree(suballoc);
+		return;
+	}
+
 	sa_manager = suballoc->manager;
 
 	spin_lock(&sa_manager->wq.lock);
diff --git a/drivers/gpu/drm/drm_writeback.c b/drivers/gpu/drm/drm_writeback.c
index 09362cf4f22f6..4da5d60947216 100644
--- a/drivers/gpu/drm/drm_writeback.c
+++ b/drivers/gpu/drm/drm_writeback.c
@@ -81,7 +81,7 @@
  *	From userspace, this property will always read as zero.
  */
 
-#define fence_to_wb_connector(x) container_of(x->lock, \
+#define fence_to_wb_connector(x) container_of(x->extern_lock, \
 					      struct drm_writeback_connector, \
 					      fence_lock)
 
diff --git a/drivers/gpu/drm/gma500/fbdev.c b/drivers/gpu/drm/gma500/fbdev.c
index c26926babc2ab..d1e93588234f5 100644
--- a/drivers/gpu/drm/gma500/fbdev.c
+++ b/drivers/gpu/drm/gma500/fbdev.c
@@ -72,17 +72,10 @@ static int psb_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 static void psb_fbdev_fb_destroy(struct fb_info *info)
 {
 	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_framebuffer *fb = fb_helper->fb;
-	struct drm_gem_object *obj = fb->obj[0];
 
 	drm_fb_helper_fini(fb_helper);
 
-	drm_framebuffer_unregister_private(fb);
-	drm_framebuffer_cleanup(fb);
-	kfree(fb);
-
-	drm_gem_object_put(obj);
-
+	drm_client_buffer_delete(fb_helper->buffer);
 	drm_client_release(&fb_helper->client);
 }
 
@@ -105,78 +98,76 @@ static const struct drm_fb_helper_funcs psb_fbdev_fb_helper_funcs = {
 int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper,
 				 struct drm_fb_helper_surface_size *sizes)
 {
-	struct drm_device *dev = fb_helper->dev;
+	struct drm_client_dev *client = &fb_helper->client;
+	struct drm_device *dev = client->dev;
+	struct drm_file *file = client->file;
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	struct fb_info *info = fb_helper->info;
-	struct drm_framebuffer *fb;
-	struct drm_mode_fb_cmd2 mode_cmd = { };
-	int size;
-	int ret;
+	u32 fourcc, pitch;
+	u64 size;
+	const struct drm_format_info *format;
+	struct drm_client_buffer *buffer;
 	struct psb_gem_object *backing;
 	struct drm_gem_object *obj;
-	u32 bpp, depth;
+	u32 handle;
+	int ret;
 
 	/* No 24-bit packed mode */
 	if (sizes->surface_bpp == 24) {
 		sizes->surface_bpp = 32;
 		sizes->surface_depth = 24;
 	}
-	bpp = sizes->surface_bpp;
-	depth = sizes->surface_depth;
-
-	/*
-	 * If the mode does not fit in 32 bit then switch to 16 bit to get
-	 * a console on full resolution. The X mode setting server will
-	 * allocate its own 32-bit GEM framebuffer.
-	 */
-	size = ALIGN(sizes->surface_width * DIV_ROUND_UP(bpp, 8), 64) *
-		     sizes->surface_height;
-	size = ALIGN(size, PAGE_SIZE);
-
-	if (size > dev_priv->vram_stolen_size) {
-		sizes->surface_bpp = 16;
-		sizes->surface_depth = 16;
-	}
-	bpp = sizes->surface_bpp;
-	depth = sizes->surface_depth;
 
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-	mode_cmd.pitches[0] = ALIGN(mode_cmd.width * DIV_ROUND_UP(bpp, 8), 64);
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
-
-	size = mode_cmd.pitches[0] * mode_cmd.height;
-	size = ALIGN(size, PAGE_SIZE);
+try_psb_gem_create:
+	fourcc = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth);
+	format = drm_get_format_info(dev, fourcc, DRM_FORMAT_MOD_LINEAR);
+	pitch = ALIGN(drm_format_info_min_pitch(format, 0, sizes->surface_width), SZ_64);
+	size = ALIGN(pitch * sizes->surface_height, PAGE_SIZE);
 
 	/* Allocate the framebuffer in the GTT with stolen page backing */
 	backing = psb_gem_create(dev, size, "fb", true, PAGE_SIZE);
-	if (IS_ERR(backing))
-		return PTR_ERR(backing);
+	if (IS_ERR(backing)) {
+		ret = PTR_ERR(backing);
+		if (ret == -EBUSY && sizes->surface_bpp > 16) {
+			/*
+			 * If the mode does not fit in 32 bit then switch to 16 bit to
+			 * get a console on full resolution. User-space compositors will
+			 * allocate their own 32-bit framebuffers.
+			 */
+			sizes->surface_bpp = 16;
+			sizes->surface_depth = 16;
+			goto try_psb_gem_create;
+		}
+		return ret;
+	}
 	obj = &backing->base;
 
-	fb = psb_framebuffer_create(dev,
-				    drm_get_format_info(dev, mode_cmd.pixel_format,
-							mode_cmd.modifier[0]),
-				    &mode_cmd, obj);
-	if (IS_ERR(fb)) {
-		ret = PTR_ERR(fb);
+	ret = drm_gem_handle_create(file, obj, &handle);
+	if (ret)
 		goto err_drm_gem_object_put;
+
+	buffer = drm_client_buffer_create(client, sizes->surface_width, sizes->surface_height,
+					  fourcc, handle, pitch);
+	if (IS_ERR(buffer)) {
+		ret = PTR_ERR(buffer);
+		goto err_drm_gem_handle_delete;
 	}
 
 	fb_helper->funcs = &psb_fbdev_fb_helper_funcs;
-	fb_helper->fb = fb;
+	fb_helper->buffer = buffer;
+	fb_helper->fb = buffer->fb;
 
 	info->fbops = &psb_fbdev_fb_ops;
 
 	/* Accessed stolen memory directly */
 	info->screen_base = dev_priv->vram_addr + backing->offset;
-	info->screen_size = size;
+	info->screen_size = obj->size;
 
 	drm_fb_helper_fill_info(info, fb_helper, sizes);
 
 	info->fix.smem_start = dev_priv->stolen_base + backing->offset;
-	info->fix.smem_len = size;
+	info->fix.smem_len = obj->size;
 	info->fix.ywrapstep = 0;
 	info->fix.ypanstep = 0;
 	info->fix.mmio_start = pci_resource_start(pdev, 0);
@@ -186,10 +177,18 @@ int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper,
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
 
-	dev_dbg(dev->dev, "allocated %dx%d fb\n", fb->width, fb->height);
+	dev_dbg(dev->dev, "allocated %dx%d fb\n", buffer->fb->width, buffer->fb->height);
+
+	/* The handle is only needed for creating the framebuffer.*/
+	drm_gem_handle_delete(file, handle);
+
+	/* The framebuffer still holds a references on the GEM object. */
+	drm_gem_object_put(obj);
 
 	return 0;
 
+err_drm_gem_handle_delete:
+	drm_gem_handle_delete(file, handle);
 err_drm_gem_object_put:
 	drm_gem_object_put(obj);
 	return ret;
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index fe1f43f0abff9..37a9f666c0f22 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -12,111 +12,25 @@
 #include "framebuffer.h"
 #include "psb_drv.h"
 
-static const struct drm_framebuffer_funcs psb_fb_funcs = {
-	.destroy = drm_gem_fb_destroy,
-	.create_handle = drm_gem_fb_create_handle,
-};
-
-/**
- *	psb_framebuffer_init	-	initialize a framebuffer
- *	@dev: our DRM device
- *	@fb: framebuffer to set up
- *	@mode_cmd: mode description
- *	@obj: backing object
- *
- *	Configure and fill in the boilerplate for our frame buffer. Return
- *	0 on success or an error code if we fail.
- */
-static int psb_framebuffer_init(struct drm_device *dev,
-					struct drm_framebuffer *fb,
-					const struct drm_format_info *info,
-					const struct drm_mode_fb_cmd2 *mode_cmd,
-					struct drm_gem_object *obj)
+static struct drm_framebuffer *
+psb_user_framebuffer_create(struct drm_device *dev, struct drm_file *filp,
+			    const struct drm_format_info *info,
+			    const struct drm_mode_fb_cmd2 *cmd)
 {
-	int ret;
-
 	/*
 	 * Reject unknown formats, YUV formats, and formats with more than
 	 * 4 bytes per pixel.
 	 */
 	if (!info->depth || info->cpp[0] > 4)
-		return -EINVAL;
-
-	if (mode_cmd->pitches[0] & 63)
-		return -EINVAL;
-
-	drm_helper_mode_fill_fb_struct(dev, fb, info, mode_cmd);
-	fb->obj[0] = obj;
-	ret = drm_framebuffer_init(dev, fb, &psb_fb_funcs);
-	if (ret) {
-		dev_err(dev->dev, "framebuffer init failed: %d\n", ret);
-		return ret;
-	}
-	return 0;
-}
-
-/**
- *	psb_framebuffer_create	-	create a framebuffer backed by gt
- *	@dev: our DRM device
- *	@info: pixel format information
- *	@mode_cmd: the description of the requested mode
- *	@obj: the backing object
- *
- *	Create a framebuffer object backed by the gt, and fill in the
- *	boilerplate required
- *
- *	TODO: review object references
- */
-struct drm_framebuffer *psb_framebuffer_create(struct drm_device *dev,
-					       const struct drm_format_info *info,
-					       const struct drm_mode_fb_cmd2 *mode_cmd,
-					       struct drm_gem_object *obj)
-{
-	struct drm_framebuffer *fb;
-	int ret;
-
-	fb = kzalloc_obj(*fb);
-	if (!fb)
-		return ERR_PTR(-ENOMEM);
-
-	ret = psb_framebuffer_init(dev, fb, info, mode_cmd, obj);
-	if (ret) {
-		kfree(fb);
-		return ERR_PTR(ret);
-	}
-	return fb;
-}
-
-/**
- *	psb_user_framebuffer_create	-	create framebuffer
- *	@dev: our DRM device
- *	@filp: client file
- *	@cmd: mode request
- *
- *	Create a new framebuffer backed by a userspace GEM object
- */
-static struct drm_framebuffer *psb_user_framebuffer_create
-			(struct drm_device *dev, struct drm_file *filp,
-			 const struct drm_format_info *info,
-			 const struct drm_mode_fb_cmd2 *cmd)
-{
-	struct drm_gem_object *obj;
-	struct drm_framebuffer *fb;
+		return ERR_PTR(-EINVAL);
 
 	/*
-	 *	Find the GEM object and thus the gtt range object that is
-	 *	to back this space
+	 * Pitch must be aligned to 64 bytes.
 	 */
-	obj = drm_gem_object_lookup(filp, cmd->handles[0]);
-	if (obj == NULL)
-		return ERR_PTR(-ENOENT);
-
-	/* Let the core code do all the work */
-	fb = psb_framebuffer_create(dev, info, cmd, obj);
-	if (IS_ERR(fb))
-		drm_gem_object_put(obj);
+	if (cmd->pitches[0] & 63)
+		return ERR_PTR(-EINVAL);
 
-	return fb;
+	return drm_gem_fb_create(dev, filp, info, cmd);
 }
 
 static const struct drm_mode_config_funcs psb_mode_funcs = {
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 0b27112ec46fa..db197b865b90a 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -592,12 +592,6 @@ struct psb_ops {
 extern void psb_modeset_init(struct drm_device *dev);
 extern void psb_modeset_cleanup(struct drm_device *dev);
 
-/* framebuffer */
-struct drm_framebuffer *psb_framebuffer_create(struct drm_device *dev,
-					       const struct drm_format_info *info,
-					       const struct drm_mode_fb_cmd2 *mode_cmd,
-					       struct drm_gem_object *obj);
-
 /* fbdev */
 #if defined(CONFIG_DRM_FBDEV_EMULATION)
 int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper,
diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
index b355bf4d33895..495804e1d2005 100644
--- a/drivers/gpu/drm/gud/gud_pipe.c
+++ b/drivers/gpu/drm/gud/gud_pipe.c
@@ -447,7 +447,7 @@ static void gud_fb_handle_damage(struct gud_device *gdrm, struct drm_framebuffer
 	}
 
 	/* Imported buffers are assumed to be WriteCombined with uncached reads */
-	gud_flush_damage(gdrm, fb, src, !fb->obj[0]->import_attach, damage);
+	gud_flush_damage(gdrm, fb, src, !drm_gem_is_imported(fb->obj[0]), damage);
 }
 
 int gud_plane_atomic_check(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/hyperv/Kconfig b/drivers/gpu/drm/hyperv/Kconfig
index 86234f6a73f2a..e48e35fb7f8b2 100644
--- a/drivers/gpu/drm/hyperv/Kconfig
+++ b/drivers/gpu/drm/hyperv/Kconfig
@@ -8,7 +8,6 @@ config DRM_HYPERV
 	help
 	 This is a KMS driver for Hyper-V synthetic video device. Choose this
 	 option if you would like to enable drm driver for Hyper-V virtual
-	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
-	 that DRM driver is used by default.
+	 machine.
 
 	 If M is selected the module will be called hyperv_drm.
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 715433187212f..be976a90c5a63 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -76,9 +76,13 @@ i915-$(CONFIG_PERF_EVENTS) += \
 
 # core display adaptation
 i915-y += \
+	i915_bo.o \
 	i915_display_pc8.o \
+	i915_dpt.o \
+	i915_dsb_buffer.o \
 	i915_hdcp_gsc.o \
 	i915_initial_plane.o \
+	i915_overlay.o \
 	i915_panic.o
 
 # "Graphics Technology" (aka we talk to the gpu)
@@ -270,13 +274,10 @@ i915-y += \
 	display/intel_dpll.o \
 	display/intel_dpll_mgr.o \
 	display/intel_dpt.o \
-	display/intel_dpt_common.o \
 	display/intel_dram.o \
 	display/intel_drrs.o \
 	display/intel_dsb.o \
-	display/intel_dsb_buffer.o \
 	display/intel_fb.o \
-	display/intel_fb_bo.o \
 	display/intel_fb_pin.o \
 	display/intel_fbc.o \
 	display/intel_fdi.o \
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index 4cb753177fd8e..d7de329abf19d 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -10,7 +10,6 @@
 #include <drm/drm_print.h>
 
 #include "g4x_dp.h"
-#include "i915_reg.h"
 #include "intel_audio.h"
 #include "intel_backlight.h"
 #include "intel_connector.h"
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
index 8b22447e8e234..5fe5067c4237c 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
@@ -8,7 +8,6 @@
 #include <drm/drm_print.h>
 
 #include "g4x_hdmi.h"
-#include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_audio.h"
 #include "intel_connector.h"
diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c
index 008d339d5c216..cbaef3f13f005 100644
--- a/drivers/gpu/drm/i915/display/hsw_ips.c
+++ b/drivers/gpu/drm/i915/display/hsw_ips.c
@@ -6,15 +6,15 @@
 #include <linux/debugfs.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "hsw_ips.h"
-#include "i915_reg.h"
 #include "intel_color_regs.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
-#include "intel_pcode.h"
+#include "intel_parent.h"
 
 static void hsw_ips_enable(const struct intel_crtc_state *crtc_state)
 {
@@ -39,8 +39,8 @@ static void hsw_ips_enable(const struct intel_crtc_state *crtc_state)
 
 	if (display->platform.broadwell) {
 		drm_WARN_ON(display->drm,
-			    intel_pcode_write(display->drm, DISPLAY_IPS_CONTROL,
-					      val | IPS_PCODE_CONTROL));
+			    intel_parent_pcode_write(display, DISPLAY_IPS_CONTROL,
+						     val | IPS_PCODE_CONTROL));
 		/*
 		 * Quoting Art Runyan: "its not safe to expect any particular
 		 * value in IPS_CTL bit 31 after enabling IPS through the
@@ -72,7 +72,7 @@ bool hsw_ips_disable(const struct intel_crtc_state *crtc_state)
 
 	if (display->platform.broadwell) {
 		drm_WARN_ON(display->drm,
-			    intel_pcode_write(display->drm, DISPLAY_IPS_CONTROL, 0));
+			    intel_parent_pcode_write(display, DISPLAY_IPS_CONTROL, 0));
 		/*
 		 * Wait for PCODE to finish disabling IPS. The BSpec specified
 		 * 42ms timeout value leads to occasional timeouts so use 100ms
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index b1fecf1789061..9c16753a1f3ba 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -10,7 +10,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "i9xx_plane.h"
 #include "i9xx_plane_regs.h"
 #include "intel_atomic.h"
diff --git a/drivers/gpu/drm/i915/display/i9xx_wm.c b/drivers/gpu/drm/i915/display/i9xx_wm.c
index 39dfceb438ae4..9e170e79dcf67 100644
--- a/drivers/gpu/drm/i915/display/i9xx_wm.c
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.c
@@ -6,8 +6,8 @@
 #include <linux/iopoll.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
-#include "i915_reg.h"
 #include "i9xx_wm.h"
 #include "i9xx_wm_regs.h"
 #include "intel_atomic.h"
@@ -182,8 +182,8 @@ static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable)
 		intel_de_posting_read(display, DSPFW3(display));
 	} else if (display->platform.i945g || display->platform.i945gm) {
 		was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
-		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
-			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
+		val = enable ? REG_MASKED_FIELD_ENABLE(FW_BLC_SELF_EN) :
+			       REG_MASKED_FIELD_DISABLE(FW_BLC_SELF_EN);
 		intel_de_write(display, FW_BLC_SELF, val);
 		intel_de_posting_read(display, FW_BLC_SELF);
 	} else if (display->platform.i915gm) {
@@ -193,8 +193,8 @@ static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable)
 		 * FW_BLC_SELF. What's going on?
 		 */
 		was_enabled = intel_de_read(display, INSTPM) & INSTPM_SELF_EN;
-		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
-			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
+		val = enable ? REG_MASKED_FIELD_ENABLE(INSTPM_SELF_EN) :
+			       REG_MASKED_FIELD_DISABLE(INSTPM_SELF_EN);
 		intel_de_write(display, INSTPM, val);
 		intel_de_posting_read(display, INSTPM);
 	} else {
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index fc265f71d72b9..c043279796781 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -34,7 +34,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
-#include "i915_reg.h"
 #include "icl_dsi.h"
 #include "icl_dsi_regs.h"
 #include "intel_atomic.h"
@@ -1624,12 +1623,6 @@ static int gen11_dsi_dsc_compute_config(struct intel_encoder *encoder,
 	if (crtc_state->pipe_bpp < 8 * 3)
 		return -EINVAL;
 
-	/* FIXME: split only when necessary */
-	if (crtc_state->dsc.slice_count > 1)
-		crtc_state->dsc.num_streams = 2;
-	else
-		crtc_state->dsc.num_streams = 1;
-
 	/* FIXME: initialize from VBT */
 	vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST;
 
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c
index f4f1b68f75435..a7350ce8e7163 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.c
+++ b/drivers/gpu/drm/i915/display/intel_alpm.c
@@ -15,6 +15,7 @@
 #include "intel_dp_aux.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
+#include "intel_vrr.h"
 
 #define SILENCE_PERIOD_MIN_TIME	80
 #define SILENCE_PERIOD_MAX_TIME	180
@@ -242,14 +243,87 @@ bool intel_alpm_compute_params(struct intel_dp *intel_dp,
 	return true;
 }
 
+int intel_alpm_lobf_min_guardband(struct intel_crtc_state *crtc_state)
+{
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	int first_sdp_position = adjusted_mode->crtc_vtotal -
+				 adjusted_mode->crtc_vsync_start;
+	int waketime_in_lines;
+
+	/*
+	 * #FIXME: Need to check if io_wake_lines or aux_less_wake_lines
+	 * is applicable. Currently this information is not readily
+	 * available in crtc_state, so max will suffice for now.
+	 */
+	waketime_in_lines = max(crtc_state->alpm_state.io_wake_lines,
+				crtc_state->alpm_state.aux_less_wake_lines);
+
+	if (!crtc_state->has_lobf)
+		return 0;
+
+	return first_sdp_position + waketime_in_lines + crtc_state->set_context_latency;
+}
+
+static bool intel_alpm_lobf_is_window1_sufficient(struct intel_crtc_state *crtc_state)
+{
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	int vblank = adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay;
+	int window1;
+
+	/*
+	 * LOBF must be disabled if the number of lines within Window 1 is not
+	 * greater than ALPM_CTL[ALPM Entry Check]
+	 */
+	window1 = vblank - min(vblank,
+			       crtc_state->vrr.guardband +
+			       crtc_state->set_context_latency);
+
+	return window1 > crtc_state->alpm_state.check_entry_lines;
+}
+
+void intel_alpm_lobf_compute_config_late(struct intel_dp *intel_dp,
+					 struct intel_crtc_state *crtc_state)
+{
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	int waketime_in_lines, first_sdp_position;
+
+	if (!crtc_state->has_lobf)
+		return;
+
+	if (!intel_alpm_lobf_is_window1_sufficient(crtc_state)) {
+		crtc_state->has_lobf = false;
+		return;
+	}
+
+	/*
+	 * LOBF can only be enabled if the time from the start of the SCL+Guardband
+	 * window to the position of the first SDP is greater than the time it takes
+	 * to wake the main link.
+	 *
+	 * Position of first sdp : vsync_start
+	 * start of scl + guardband : vtotal - (scl + guardband)
+	 * time in lines to wake main link : waketime_in_lines
+	 *
+	 * Position of first sdp - start of (scl + guardband) > time in lines to wake main link
+	 * vsync_start - (vtotal - (scl + guardband)) > waketime_in_lines
+	 * vsync_start - vtotal + scl + guardband > waketime_in_lines
+	 * scl + guardband > waketime_in_lines + (vtotal - vsync_start)
+	 */
+	first_sdp_position = adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vsync_start;
+	if (intel_alpm_aux_less_wake_supported(intel_dp))
+		waketime_in_lines = crtc_state->alpm_state.io_wake_lines;
+	else
+		waketime_in_lines = crtc_state->alpm_state.aux_less_wake_lines;
+
+	crtc_state->has_lobf = (crtc_state->set_context_latency + crtc_state->vrr.guardband) >
+			       (first_sdp_position + waketime_in_lines);
+}
+
 void intel_alpm_lobf_compute_config(struct intel_dp *intel_dp,
 				    struct intel_crtc_state *crtc_state,
 				    struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
-	int waketime_in_lines, first_sdp_position;
-	int context_latency, guardband;
 
 	if (intel_dp->alpm.lobf_disable_debug) {
 		drm_dbg_kms(display->drm, "LOBF is disabled by debug flag\n");
@@ -271,8 +345,8 @@ void intel_alpm_lobf_compute_config(struct intel_dp *intel_dp,
 	if (crtc_state->has_psr)
 		return;
 
-	if (crtc_state->vrr.vmin != crtc_state->vrr.vmax ||
-	    crtc_state->vrr.vmin != crtc_state->vrr.flipline)
+	if (!intel_vrr_always_use_vrr_tg(display) ||
+	    !intel_vrr_is_fixed_rr(crtc_state))
 		return;
 
 	if (!(intel_alpm_aux_wake_supported(intel_dp) ||
@@ -282,17 +356,7 @@ void intel_alpm_lobf_compute_config(struct intel_dp *intel_dp,
 	if (!intel_alpm_compute_params(intel_dp, crtc_state))
 		return;
 
-	context_latency = adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay;
-	guardband = adjusted_mode->crtc_vtotal -
-		    adjusted_mode->crtc_vdisplay - context_latency;
-	first_sdp_position = adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vsync_start;
-	if (intel_alpm_aux_less_wake_supported(intel_dp))
-		waketime_in_lines = crtc_state->alpm_state.io_wake_lines;
-	else
-		waketime_in_lines = crtc_state->alpm_state.aux_less_wake_lines;
-
-	crtc_state->has_lobf = (context_latency + guardband) >
-		(first_sdp_position + waketime_in_lines);
+	crtc_state->has_lobf = true;
 }
 
 static void lnl_alpm_configure(struct intel_dp *intel_dp,
@@ -382,25 +446,14 @@ void intel_alpm_port_configure(struct intel_dp *intel_dp,
 	intel_de_write(display, PORT_ALPM_LFPS_CTL(port), lfps_ctl_val);
 }
 
-void intel_alpm_pre_plane_update(struct intel_atomic_state *state,
-				 struct intel_crtc *crtc)
+void intel_alpm_lobf_disable(const struct intel_crtc_state *new_crtc_state)
 {
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	const struct intel_crtc_state *old_crtc_state =
-		intel_atomic_get_old_crtc_state(state, crtc);
-	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	struct intel_display *display = to_intel_display(new_crtc_state);
+	enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 	struct intel_encoder *encoder;
 
-	if (DISPLAY_VER(display) < 20)
-		return;
-
-	if (crtc_state->has_lobf || crtc_state->has_lobf == old_crtc_state->has_lobf)
-		return;
-
 	for_each_intel_encoder_mask(display->drm, encoder,
-				    crtc_state->uapi.encoder_mask) {
+				    new_crtc_state->uapi.encoder_mask) {
 		struct intel_dp *intel_dp;
 
 		if (!intel_encoder_is_dp(encoder))
@@ -411,12 +464,10 @@ void intel_alpm_pre_plane_update(struct intel_atomic_state *state,
 		if (!intel_dp_is_edp(intel_dp))
 			continue;
 
-		if (old_crtc_state->has_lobf) {
-			mutex_lock(&intel_dp->alpm.lock);
-			intel_de_write(display, ALPM_CTL(display, cpu_transcoder), 0);
-			drm_dbg_kms(display->drm, "Link off between frames (LOBF) disabled\n");
-			mutex_unlock(&intel_dp->alpm.lock);
-		}
+		mutex_lock(&intel_dp->alpm.lock);
+		intel_de_write(display, ALPM_CTL(display, cpu_transcoder), 0);
+		drm_dbg_kms(display->drm, "Link off between frames (LOBF) disabled\n");
+		mutex_unlock(&intel_dp->alpm.lock);
 	}
 }
 
@@ -437,22 +488,13 @@ void intel_alpm_enable_sink(struct intel_dp *intel_dp,
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, val);
 }
 
-void intel_alpm_post_plane_update(struct intel_atomic_state *state,
-				  struct intel_crtc *crtc)
+void intel_alpm_lobf_enable(const struct intel_crtc_state *new_crtc_state)
 {
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	const struct intel_crtc_state *old_crtc_state =
-		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_display *display = to_intel_display(new_crtc_state);
 	struct intel_encoder *encoder;
 
-	if (crtc_state->has_psr || !crtc_state->has_lobf ||
-	    crtc_state->has_lobf == old_crtc_state->has_lobf)
-		return;
-
 	for_each_intel_encoder_mask(display->drm, encoder,
-				    crtc_state->uapi.encoder_mask) {
+				    new_crtc_state->uapi.encoder_mask) {
 		struct intel_dp *intel_dp;
 
 		if (!intel_encoder_is_dp(encoder))
@@ -461,8 +503,8 @@ void intel_alpm_post_plane_update(struct intel_atomic_state *state,
 		intel_dp = enc_to_intel_dp(encoder);
 
 		if (intel_dp_is_edp(intel_dp)) {
-			intel_alpm_enable_sink(intel_dp, crtc_state);
-			intel_alpm_configure(intel_dp, crtc_state);
+			intel_alpm_enable_sink(intel_dp, new_crtc_state);
+			intel_alpm_configure(intel_dp, new_crtc_state);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.h b/drivers/gpu/drm/i915/display/intel_alpm.h
index c6a4ec5b9561f..1cf70668ab1ba 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.h
+++ b/drivers/gpu/drm/i915/display/intel_alpm.h
@@ -25,12 +25,10 @@ void intel_alpm_configure(struct intel_dp *intel_dp,
 			  const struct intel_crtc_state *crtc_state);
 void intel_alpm_enable_sink(struct intel_dp *intel_dp,
 			    const struct intel_crtc_state *crtc_state);
-void intel_alpm_pre_plane_update(struct intel_atomic_state *state,
-				 struct intel_crtc *crtc);
+void intel_alpm_lobf_disable(const struct intel_crtc_state *new_crtc_state);
 void intel_alpm_port_configure(struct intel_dp *intel_dp,
 			       const struct intel_crtc_state *crtc_state);
-void intel_alpm_post_plane_update(struct intel_atomic_state *state,
-				  struct intel_crtc *crtc);
+void intel_alpm_lobf_enable(const struct intel_crtc_state *new_crtc_state);
 void intel_alpm_lobf_debugfs_add(struct intel_connector *connector);
 bool intel_alpm_aux_wake_supported(struct intel_dp *intel_dp);
 bool intel_alpm_aux_less_wake_supported(struct intel_dp *intel_dp);
@@ -38,4 +36,7 @@ bool intel_alpm_is_alpm_aux_less(struct intel_dp *intel_dp,
 				 const struct intel_crtc_state *crtc_state);
 void intel_alpm_disable(struct intel_dp *intel_dp);
 bool intel_alpm_get_error(struct intel_dp *intel_dp);
+void intel_alpm_lobf_compute_config_late(struct intel_dp *intel_dp,
+					 struct intel_crtc_state *crtc_state);
+int intel_alpm_lobf_min_guardband(struct intel_crtc_state *crtc_state);
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 5f3c175afdd27..081627e0d9171 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -37,6 +37,7 @@
 #include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
+#include "intel_display_wa.h"
 #include "intel_lpe_audio.h"
 
 /**
@@ -184,17 +185,6 @@ static const struct hdmi_aud_ncts hdmi_aud_ncts_36bpp[] = {
 	{ 192000, TMDS_445_5M, 20480, 371250 },
 };
 
-/*
- * WA_14020863754: Implement Audio Workaround
- * Corner case with Min Hblank Fix can cause audio hang
- */
-static bool needs_wa_14020863754(struct intel_display *display)
-{
-	return DISPLAY_VERx100(display) == 3000 ||
-		DISPLAY_VERx100(display) == 2000 ||
-		DISPLAY_VERx100(display) == 1401;
-}
-
 /* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */
 static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_state)
 {
@@ -440,7 +430,11 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder,
 	intel_de_rmw(display, HSW_AUD_PIN_ELD_CP_VLD,
 		     AUDIO_OUTPUT_ENABLE(cpu_transcoder), 0);
 
-	if (needs_wa_14020863754(display))
+	/*
+	 * WA_14020863754: Implement Audio Workaround
+	 * Corner case with Min Hblank Fix can cause audio hang
+	 */
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14020863754))
 		intel_de_rmw(display, AUD_CHICKENBIT_REG3, DACBE_DISABLE_MIN_HBLANK_FIX, 0);
 
 	intel_audio_sdp_split_update(old_crtc_state, false);
@@ -572,7 +566,11 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder,
 
 	intel_audio_sdp_split_update(crtc_state, true);
 
-	if (needs_wa_14020863754(display))
+	/*
+	 * WA_14020863754: Implement Audio Workaround
+	 * Corner case with Min Hblank Fix can cause audio hang
+	 */
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14020863754))
 		intel_de_rmw(display, AUD_CHICKENBIT_REG3, 0, DACBE_DISABLE_MIN_HBLANK_FIX);
 
 	/* Enable audio presence detect */
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index a68fdbd2acb90..34e95f05936e5 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -12,7 +12,6 @@
 #include <drm/drm_file.h>
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_backlight.h"
 #include "intel_backlight_regs.h"
 #include "intel_connector.h"
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index a1fa3571eca00..b6fe87c29aa7c 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -41,6 +41,7 @@
 #include "intel_display_utils.h"
 #include "intel_gmbus.h"
 #include "intel_rom.h"
+#include "intel_vdsc.h"
 
 #define _INTEL_BIOS_PRIVATE
 #include "intel_vbt_defs.h"
@@ -1545,6 +1546,10 @@ parse_edp(struct intel_display *display,
 	if (display->vbt.version >= 251)
 		panel->vbt.edp.dsc_disable =
 			panel_bool(edp->edp_dsc_disable, panel_type);
+
+	if (display->vbt.version >= 261)
+		panel->vbt.edp.pipe_joiner_enable =
+			panel_bool(edp->pipe_joiner_enable, panel_type);
 }
 
 static void
@@ -3543,12 +3548,13 @@ bool intel_bios_is_dsi_present(struct intel_display *display,
 	return false;
 }
 
-static void fill_dsc(struct intel_crtc_state *crtc_state,
+static bool fill_dsc(struct intel_crtc_state *crtc_state,
 		     struct dsc_compression_parameters_entry *dsc,
 		     int dsc_max_bpc)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
+	int slices_per_line;
 	int bpc = 8;
 
 	vdsc_cfg->dsc_version_major = dsc->version_major;
@@ -3574,26 +3580,33 @@ static void fill_dsc(struct intel_crtc_state *crtc_state,
 	 * throughput etc. into account.
 	 *
 	 * Also, per spec DSI supports 1, 2, 3 or 4 horizontal slices.
+	 *
+	 * FIXME: split only when necessary
 	 */
 	if (dsc->slices_per_line & BIT(2)) {
-		crtc_state->dsc.slice_count = 4;
+		slices_per_line = 4;
 	} else if (dsc->slices_per_line & BIT(1)) {
-		crtc_state->dsc.slice_count = 2;
+		slices_per_line = 2;
 	} else {
 		/* FIXME */
 		if (!(dsc->slices_per_line & BIT(0)))
 			drm_dbg_kms(display->drm,
 				    "VBT: Unsupported DSC slice count for DSI\n");
 
-		crtc_state->dsc.slice_count = 1;
+		slices_per_line = 1;
 	}
 
+	if (drm_WARN_ON(display->drm,
+			!intel_dsc_get_slice_config(display, 1, slices_per_line,
+						    &crtc_state->dsc.slice_config)))
+		return false;
+
 	if (crtc_state->hw.adjusted_mode.crtc_hdisplay %
-	    crtc_state->dsc.slice_count != 0)
+	    intel_dsc_line_slice_count(&crtc_state->dsc.slice_config) != 0)
 		drm_dbg_kms(display->drm,
 			    "VBT: DSC hdisplay %d not divisible by slice count %d\n",
 			    crtc_state->hw.adjusted_mode.crtc_hdisplay,
-			    crtc_state->dsc.slice_count);
+			    intel_dsc_line_slice_count(&crtc_state->dsc.slice_config));
 
 	/*
 	 * The VBT rc_buffer_block_size and rc_buffer_size definitions
@@ -3608,6 +3621,8 @@ static void fill_dsc(struct intel_crtc_state *crtc_state,
 	vdsc_cfg->block_pred_enable = dsc->block_prediction_enable;
 
 	vdsc_cfg->slice_height = dsc->slice_height;
+
+	return true;
 }
 
 /* FIXME: initially DSI specific */
@@ -3628,9 +3643,7 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 			if (!devdata->dsc)
 				return false;
 
-			fill_dsc(crtc_state, devdata->dsc, dsc_max_bpc);
-
-			return true;
+			return fill_dsc(crtc_state, devdata->dsc, dsc_max_bpc);
 		}
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_bo.c b/drivers/gpu/drm/i915/display/intel_bo.c
index 8f372b33d48bf..3b82d38a0504f 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.c
+++ b/drivers/gpu/drm/i915/display/intel_bo.c
@@ -1,87 +1,87 @@
 // SPDX-License-Identifier: MIT
-/* Copyright © 2024 Intel Corporation */
+/* Copyright © 2026 Intel Corporation */
 
-#include <drm/drm_panic.h>
+#include <drm/drm_gem.h>
+#include <drm/intel/display_parent_interface.h>
 
-#include "gem/i915_gem_mman.h"
-#include "gem/i915_gem_object.h"
-#include "gem/i915_gem_object_frontbuffer.h"
-#include "pxp/intel_pxp.h"
-#include "i915_debugfs.h"
 #include "intel_bo.h"
+#include "intel_display_core.h"
+#include "intel_display_types.h"
 
 bool intel_bo_is_tiled(struct drm_gem_object *obj)
 {
-	return i915_gem_object_is_tiled(to_intel_bo(obj));
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->is_tiled && display->parent->bo->is_tiled(obj);
 }
 
 bool intel_bo_is_userptr(struct drm_gem_object *obj)
 {
-	return i915_gem_object_is_userptr(to_intel_bo(obj));
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->is_userptr && display->parent->bo->is_userptr(obj);
 }
 
 bool intel_bo_is_shmem(struct drm_gem_object *obj)
 {
-	return i915_gem_object_is_shmem(to_intel_bo(obj));
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->is_shmem && display->parent->bo->is_shmem(obj);
 }
 
 bool intel_bo_is_protected(struct drm_gem_object *obj)
 {
-	return i915_gem_object_is_protected(to_intel_bo(obj));
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->is_protected(obj);
 }
 
 int intel_bo_key_check(struct drm_gem_object *obj)
 {
-	return intel_pxp_key_check(obj, false);
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->key_check(obj);
 }
 
 int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 {
-	return i915_gem_fb_mmap(to_intel_bo(obj), vma);
-}
+	struct intel_display *display = to_intel_display(obj->dev);
 
-int intel_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size)
-{
-	return i915_gem_object_read_from_page(to_intel_bo(obj), offset, dst, size);
+	return display->parent->bo->fb_mmap(obj, vma);
 }
 
-struct intel_frontbuffer *intel_bo_frontbuffer_get(struct drm_gem_object *_obj)
+int intel_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size)
 {
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	struct i915_frontbuffer *front;
+	struct intel_display *display = to_intel_display(obj->dev);
 
-	front = i915_gem_object_frontbuffer_get(obj);
-	if (!front)
-		return NULL;
-
-	return &front->base;
+	return display->parent->bo->read_from_page(obj, offset, dst, size);
 }
 
-void intel_bo_frontbuffer_ref(struct intel_frontbuffer *_front)
+void intel_bo_describe(struct seq_file *m, struct drm_gem_object *obj)
 {
-	struct i915_frontbuffer *front =
-		container_of(_front, typeof(*front), base);
+	struct intel_display *display = to_intel_display(obj->dev);
 
-	i915_gem_object_frontbuffer_ref(front);
+	if (display->parent->bo->describe)
+		display->parent->bo->describe(m, obj);
 }
 
-void intel_bo_frontbuffer_put(struct intel_frontbuffer *_front)
+int intel_bo_framebuffer_init(struct drm_gem_object *obj, struct drm_mode_fb_cmd2 *mode_cmd)
 {
-	struct i915_frontbuffer *front =
-		container_of(_front, typeof(*front), base);
+	struct intel_display *display = to_intel_display(obj->dev);
 
-	return i915_gem_object_frontbuffer_put(front);
+	return display->parent->bo->framebuffer_init(obj, mode_cmd);
 }
 
-void intel_bo_frontbuffer_flush_for_display(struct intel_frontbuffer *_front)
+void intel_bo_framebuffer_fini(struct drm_gem_object *obj)
 {
-	struct i915_frontbuffer *front =
-		container_of(_front, typeof(*front), base);
+	struct intel_display *display = to_intel_display(obj->dev);
 
-	i915_gem_object_flush_if_display(front->obj);
+	display->parent->bo->framebuffer_fini(obj);
 }
 
-void intel_bo_describe(struct seq_file *m, struct drm_gem_object *obj)
+struct drm_gem_object *intel_bo_framebuffer_lookup(struct intel_display *display,
+						   struct drm_file *filp,
+						   const struct drm_mode_fb_cmd2 *user_mode_cmd)
 {
-	i915_debugfs_describe_obj(m, to_intel_bo(obj));
+	return display->parent->bo->framebuffer_lookup(display->drm, filp, user_mode_cmd);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_bo.h b/drivers/gpu/drm/i915/display/intel_bo.h
index 516a3836a6bc0..aec188c706c27 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.h
+++ b/drivers/gpu/drm/i915/display/intel_bo.h
@@ -6,8 +6,11 @@
 
 #include <linux/types.h>
 
+struct drm_file;
 struct drm_gem_object;
+struct drm_mode_fb_cmd2;
 struct drm_scanout_buffer;
+struct intel_display;
 struct intel_framebuffer;
 struct seq_file;
 struct vm_area_struct;
@@ -20,11 +23,12 @@ int intel_bo_key_check(struct drm_gem_object *obj);
 int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 int intel_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size);
 
-struct intel_frontbuffer *intel_bo_frontbuffer_get(struct drm_gem_object *obj);
-void intel_bo_frontbuffer_ref(struct intel_frontbuffer *front);
-void intel_bo_frontbuffer_put(struct intel_frontbuffer *front);
-void intel_bo_frontbuffer_flush_for_display(struct intel_frontbuffer *front);
-
 void intel_bo_describe(struct seq_file *m, struct drm_gem_object *obj);
 
+void intel_bo_framebuffer_fini(struct drm_gem_object *obj);
+int intel_bo_framebuffer_init(struct drm_gem_object *obj, struct drm_mode_fb_cmd2 *mode_cmd);
+struct drm_gem_object *intel_bo_framebuffer_lookup(struct intel_display *display,
+						   struct drm_file *filp,
+						   const struct drm_mode_fb_cmd2 *user_mode_cmd);
+
 #endif /* __INTEL_BO__ */
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index fe48949b58806..07b4531a43760 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -5,8 +5,8 @@
 
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "intel_bw.h"
 #include "intel_crtc.h"
 #include "intel_display_core.h"
@@ -15,7 +15,7 @@
 #include "intel_display_utils.h"
 #include "intel_dram.h"
 #include "intel_mchbar_regs.h"
-#include "intel_pcode.h"
+#include "intel_parent.h"
 #include "intel_uncore.h"
 #include "skl_watermark.h"
 
@@ -114,9 +114,9 @@ static int icl_pcode_read_qgv_point_info(struct intel_display *display,
 	u16 dclk;
 	int ret;
 
-	ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
-			       ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
-			       &val, &val2);
+	ret = intel_parent_pcode_read(display, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
+				      ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
+				      &val, &val2);
 	if (ret)
 		return ret;
 
@@ -141,8 +141,8 @@ static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
 	int ret;
 	int i;
 
-	ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
-			       ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
+	ret = intel_parent_pcode_read(display, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
+				      ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
 	if (ret)
 		return ret;
 
@@ -189,11 +189,11 @@ static int icl_pcode_restrict_qgv_points(struct intel_display *display,
 		return 0;
 
 	/* bspec says to keep retrying for at least 1 ms */
-	ret = intel_pcode_request(display->drm, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
-				  points_mask,
-				  ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
-				  ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
-				  1);
+	ret = intel_parent_pcode_request(display, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
+					 points_mask,
+					 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
+					 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
+					 1);
 
 	if (ret < 0) {
 		drm_err(display->drm,
diff --git a/drivers/gpu/drm/i915/display/intel_casf.c b/drivers/gpu/drm/i915/display/intel_casf.c
index 95339b496f241..b167af31de5b0 100644
--- a/drivers/gpu/drm/i915/display/intel_casf.c
+++ b/drivers/gpu/drm/i915/display/intel_casf.c
@@ -3,7 +3,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_casf.h"
 #include "intel_casf_regs.h"
 #include "intel_de.h"
@@ -116,6 +115,12 @@ int intel_casf_compute_config(struct intel_crtc_state *crtc_state)
 		return 0;
 	}
 
+	/* CASF with joiner not supported in hardware */
+	if (crtc_state->joiner_pipes) {
+		drm_dbg_kms(display->drm, "CASF not supported with joiner\n");
+		return -EINVAL;
+	}
+
 	crtc_state->hw.casf_params.casf_enable = true;
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index f5946e677c93a..121a12c5b8ac2 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -27,9 +27,9 @@
 
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "hsw_ips.h"
-#include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_audio.h"
 #include "intel_cdclk.h"
@@ -42,8 +42,8 @@
 #include "intel_display_wa.h"
 #include "intel_dram.h"
 #include "intel_mchbar_regs.h"
+#include "intel_parent.h"
 #include "intel_pci_config.h"
-#include "intel_pcode.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
 #include "intel_step.h"
@@ -888,7 +888,7 @@ static void bdw_set_cdclk(struct intel_display *display,
 		     "trying to change cdclk frequency with cdclk not enabled\n"))
 		return;
 
-	ret = intel_pcode_write(display->drm, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
+	ret = intel_parent_pcode_write(display, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 	if (ret) {
 		drm_err(display->drm,
 			"failed to inform pcode about cdclk change\n");
@@ -918,8 +918,8 @@ static void bdw_set_cdclk(struct intel_display *display,
 	if (ret)
 		drm_err(display->drm, "Switching back to LCPLL failed\n");
 
-	intel_pcode_write(display->drm, HSW_PCODE_DE_WRITE_FREQ_REQ,
-			  cdclk_config->voltage_level);
+	intel_parent_pcode_write(display, HSW_PCODE_DE_WRITE_FREQ_REQ,
+				 cdclk_config->voltage_level);
 
 	intel_de_write(display, CDCLK_FREQ,
 		       DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
@@ -1175,10 +1175,10 @@ static void skl_set_cdclk(struct intel_display *display,
 	drm_WARN_ON_ONCE(display->drm,
 			 display->platform.skylake && vco == 8640000);
 
-	ret = intel_pcode_request(display->drm, SKL_PCODE_CDCLK_CONTROL,
-				  SKL_CDCLK_PREPARE_FOR_CHANGE,
-				  SKL_CDCLK_READY_FOR_CHANGE,
-				  SKL_CDCLK_READY_FOR_CHANGE, 3);
+	ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
+					 SKL_CDCLK_PREPARE_FOR_CHANGE,
+					 SKL_CDCLK_READY_FOR_CHANGE,
+					 SKL_CDCLK_READY_FOR_CHANGE, 3);
 	if (ret) {
 		drm_err(display->drm,
 			"Failed to inform PCU about cdclk change (%d)\n", ret);
@@ -1221,8 +1221,8 @@ static void skl_set_cdclk(struct intel_display *display,
 	intel_de_posting_read(display, CDCLK_CTL);
 
 	/* inform PCU of the change */
-	intel_pcode_write(display->drm, SKL_PCODE_CDCLK_CONTROL,
-			  cdclk_config->voltage_level);
+	intel_parent_pcode_write(display, SKL_PCODE_CDCLK_CONTROL,
+				 cdclk_config->voltage_level);
 
 	intel_update_cdclk(display);
 }
@@ -1870,7 +1870,7 @@ static void icl_cdclk_pll_disable(struct intel_display *display)
 	 *      after the PLL is enabled (which is already done as part of the
 	 *      normal flow of _bxt_set_cdclk()).
 	 */
-	if (intel_display_wa(display, 13012396614))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_13012396614))
 		intel_de_rmw(display, CDCLK_CTL, MDCLK_SOURCE_SEL_MASK, MDCLK_SOURCE_SEL_CD2XCLK);
 
 	intel_de_rmw(display, BXT_DE_PLL_ENABLE,
@@ -2186,7 +2186,8 @@ static u32 bxt_cdclk_ctl(struct intel_display *display,
 		 * icl_cdclk_pll_disable().  Here we are just making sure
 		 * we keep the expected value.
 		 */
-		if (intel_display_wa(display, 13012396614) && vco == 0)
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_13012396614) &&
+		    vco == 0)
 			val |= MDCLK_SOURCE_SEL_CD2XCLK;
 		else
 			val |= xe2lpd_mdclk_source_sel(display);
@@ -2247,18 +2248,18 @@ static void bxt_set_cdclk(struct intel_display *display,
 	if (DISPLAY_VER(display) >= 14 || display->platform.dg2)
 		; /* NOOP */
 	else if (DISPLAY_VER(display) >= 11)
-		ret = intel_pcode_request(display->drm, SKL_PCODE_CDCLK_CONTROL,
-					  SKL_CDCLK_PREPARE_FOR_CHANGE,
-					  SKL_CDCLK_READY_FOR_CHANGE,
-					  SKL_CDCLK_READY_FOR_CHANGE, 3);
+		ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
+						 SKL_CDCLK_PREPARE_FOR_CHANGE,
+						 SKL_CDCLK_READY_FOR_CHANGE,
+						 SKL_CDCLK_READY_FOR_CHANGE, 3);
 	else
 		/*
 		 * BSpec requires us to wait up to 150usec, but that leads to
 		 * timeouts; the 2ms used here is based on experiment.
 		 */
-		ret = intel_pcode_write_timeout(display->drm,
-						HSW_PCODE_DE_WRITE_FREQ_REQ,
-						0x80000000, 2);
+		ret = intel_parent_pcode_write_timeout(display,
+						       HSW_PCODE_DE_WRITE_FREQ_REQ,
+						       0x80000000, 2);
 
 	if (ret) {
 		drm_err(display->drm,
@@ -2287,8 +2288,8 @@ static void bxt_set_cdclk(struct intel_display *display,
 		 * Display versions 14 and beyond
 		 */;
 	else if (DISPLAY_VER(display) >= 11 && !display->platform.dg2)
-		ret = intel_pcode_write(display->drm, SKL_PCODE_CDCLK_CONTROL,
-					cdclk_config->voltage_level);
+		ret = intel_parent_pcode_write(display, SKL_PCODE_CDCLK_CONTROL,
+					       cdclk_config->voltage_level);
 	if (DISPLAY_VER(display) < 11) {
 		/*
 		 * The timeout isn't specified, the 2ms used here is based on
@@ -2296,9 +2297,9 @@ static void bxt_set_cdclk(struct intel_display *display,
 		 * FIXME: Waiting for the request completion could be delayed
 		 * until the next PCODE request based on BSpec.
 		 */
-		ret = intel_pcode_write_timeout(display->drm,
-						HSW_PCODE_DE_WRITE_FREQ_REQ,
-						cdclk_config->voltage_level, 2);
+		ret = intel_parent_pcode_write_timeout(display,
+						       HSW_PCODE_DE_WRITE_FREQ_REQ,
+						       cdclk_config->voltage_level, 2);
 	}
 	if (ret) {
 		drm_err(display->drm,
@@ -2598,11 +2599,11 @@ static void intel_pcode_notify(struct intel_display *display,
 	if (pipe_count_update_valid)
 		update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
 
-	ret = intel_pcode_request(display->drm, SKL_PCODE_CDCLK_CONTROL,
-				  SKL_CDCLK_PREPARE_FOR_CHANGE |
-				  update_mask,
-				  SKL_CDCLK_READY_FOR_CHANGE,
-				  SKL_CDCLK_READY_FOR_CHANGE, 3);
+	ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
+					 SKL_CDCLK_PREPARE_FOR_CHANGE |
+					 update_mask,
+					 SKL_CDCLK_READY_FOR_CHANGE,
+					 SKL_CDCLK_READY_FOR_CHANGE, 3);
 	if (ret)
 		drm_err(display->drm,
 			"Failed to inform PCU about display config (err %d)\n",
@@ -4006,7 +4007,7 @@ void intel_init_cdclk_hooks(struct intel_display *display)
 		display->cdclk.table = dg2_cdclk_table;
 	} else if (display->platform.alderlake_p) {
 		/* Wa_22011320316:adl-p[a0] */
-		if (display->platform.alderlake_p && IS_DISPLAY_STEP(display, STEP_A0, STEP_B0)) {
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_22011320316)) {
 			display->cdclk.table = adlp_a_step_cdclk_table;
 			display->funcs.cdclk = &tgl_cdclk_funcs;
 		} else if (display->platform.alderlake_p_raptorlake_u) {
diff --git a/drivers/gpu/drm/i915/display/intel_color_pipeline.c b/drivers/gpu/drm/i915/display/intel_color_pipeline.c
index 04af552b36488..6cf8080ee8000 100644
--- a/drivers/gpu/drm/i915/display/intel_color_pipeline.c
+++ b/drivers/gpu/drm/i915/display/intel_color_pipeline.c
@@ -2,6 +2,8 @@
 /*
  * Copyright © 2025 Intel Corporation
  */
+#include <drm/drm_print.h>
+
 #include "intel_color.h"
 #include "intel_colorop.h"
 #include "intel_color_pipeline.h"
@@ -10,72 +12,145 @@
 #include "skl_universal_plane.h"
 
 #define MAX_COLOR_PIPELINES 1
+#define MAX_COLOROP 4
 #define PLANE_DEGAMMA_SIZE 128
 #define PLANE_GAMMA_SIZE 32
 
+static const struct drm_colorop_funcs intel_colorop_funcs = {
+	.destroy = intel_colorop_destroy,
+};
+
+/*
+ * 3DLUT can be bound to all three HDR planes. However, even with the latest
+ * color pipeline UAPI, there is no good way to represent a HW block which
+ * can be shared/attached at different stages of the pipeline. So right now,
+ * we expose 3DLUT only attached with the primary plane.
+ *
+ * That way we don't confuse the userspace with opaque commit failures
+ * on trying to enable it on multiple planes which would otherwise make
+ * the pipeline totally unusable.
+ */
+static const enum intel_color_block xe3plpd_primary_plane_pipeline[] = {
+	INTEL_PLANE_CB_PRE_CSC_LUT,
+	INTEL_PLANE_CB_CSC,
+	INTEL_PLANE_CB_3DLUT,
+	INTEL_PLANE_CB_POST_CSC_LUT,
+};
+
+static const enum intel_color_block hdr_plane_pipeline[] = {
+	INTEL_PLANE_CB_PRE_CSC_LUT,
+	INTEL_PLANE_CB_CSC,
+	INTEL_PLANE_CB_POST_CSC_LUT,
+};
+
+static bool plane_has_3dlut(struct intel_display *display, enum pipe pipe,
+			    struct drm_plane *plane)
+{
+	return (DISPLAY_VER(display) >= 35 &&
+		intel_color_crtc_has_3dlut(display, pipe) &&
+		plane->type == DRM_PLANE_TYPE_PRIMARY);
+}
+
 static
-int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_enum_list *list,
-				     enum pipe pipe)
+struct intel_colorop *intel_color_pipeline_plane_add_colorop(struct drm_plane *plane,
+							     struct intel_colorop *prev,
+							     enum intel_color_block id)
 {
 	struct drm_device *dev = plane->dev;
-	struct intel_display *display = to_intel_display(dev);
-	struct drm_colorop *prev_op;
 	struct intel_colorop *colorop;
 	int ret;
 
-	colorop = intel_colorop_create(INTEL_PLANE_CB_PRE_CSC_LUT);
-
-	ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane,
-						  PLANE_DEGAMMA_SIZE,
-						  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
-						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
-
-	if (ret)
-		return ret;
-
-	list->type = colorop->base.base.id;
-
-	/* TODO: handle failures and clean up */
-	prev_op = &colorop->base;
+	colorop = intel_colorop_create(id);
+
+	if (IS_ERR(colorop))
+		return colorop;
+
+	switch (id) {
+	case INTEL_PLANE_CB_PRE_CSC_LUT:
+		ret = drm_plane_colorop_curve_1d_lut_init(dev,
+							  &colorop->base, plane,
+							  &intel_colorop_funcs,
+							  PLANE_DEGAMMA_SIZE,
+							  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
+							  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		break;
+	case INTEL_PLANE_CB_CSC:
+		ret = drm_plane_colorop_ctm_3x4_init(dev, &colorop->base, plane,
+						     &intel_colorop_funcs,
+						     DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		break;
+	case INTEL_PLANE_CB_3DLUT:
+		ret = drm_plane_colorop_3dlut_init(dev, &colorop->base, plane,
+						   &intel_colorop_funcs, 17,
+						   DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL,
+						   true);
+		break;
+	case INTEL_PLANE_CB_POST_CSC_LUT:
+		ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane,
+							  &intel_colorop_funcs,
+							  PLANE_GAMMA_SIZE,
+							  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
+							  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		break;
+	default:
+		drm_err(plane->dev, "Invalid colorop id [%d]", id);
+		ret = -EINVAL;
+	}
 
-	colorop = intel_colorop_create(INTEL_PLANE_CB_CSC);
-	ret = drm_plane_colorop_ctm_3x4_init(dev, &colorop->base, plane,
-					     DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
-		return ret;
+		goto cleanup;
 
-	drm_colorop_set_next_property(prev_op, &colorop->base);
-	prev_op = &colorop->base;
+	if (prev)
+		drm_colorop_set_next_property(&prev->base, &colorop->base);
 
-	if (DISPLAY_VER(display) >= 35 &&
-	    intel_color_crtc_has_3dlut(display, pipe) &&
-	    plane->type == DRM_PLANE_TYPE_PRIMARY) {
-		colorop = intel_colorop_create(INTEL_PLANE_CB_3DLUT);
+	return colorop;
 
-		ret = drm_plane_colorop_3dlut_init(dev, &colorop->base, plane, 17,
-						   DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL,
-						   true);
-		if (ret)
-			return ret;
+cleanup:
+	intel_colorop_destroy(&colorop->base);
+	return ERR_PTR(ret);
+}
 
-		drm_colorop_set_next_property(prev_op, &colorop->base);
+static
+int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_enum_list *list,
+				     enum pipe pipe)
+{
+	struct drm_device *dev = plane->dev;
+	struct intel_display *display = to_intel_display(dev);
+	struct intel_colorop *colorop[MAX_COLOROP];
+	struct intel_colorop *prev = NULL;
+	const enum intel_color_block *pipeline;
+	int pipeline_len;
+	int ret = 0;
+	int i;
 
-		prev_op = &colorop->base;
+	if (plane_has_3dlut(display, pipe, plane)) {
+		pipeline = xe3plpd_primary_plane_pipeline;
+		pipeline_len = ARRAY_SIZE(xe3plpd_primary_plane_pipeline);
+	} else {
+		pipeline = hdr_plane_pipeline;
+		pipeline_len = ARRAY_SIZE(hdr_plane_pipeline);
 	}
 
-	colorop = intel_colorop_create(INTEL_PLANE_CB_POST_CSC_LUT);
-	ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane,
-						  PLANE_GAMMA_SIZE,
-						  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
-						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
-	if (ret)
-		return ret;
+	for (i = 0; i < pipeline_len; i++) {
+		colorop[i] = intel_color_pipeline_plane_add_colorop(plane, prev,
+								    pipeline[i]);
+		if (IS_ERR(colorop[i])) {
+			ret = PTR_ERR(colorop[i]);
+			goto cleanup;
+		}
 
-	drm_colorop_set_next_property(prev_op, &colorop->base);
+		prev = colorop[i];
+	}
 
-	list->name = kasprintf(GFP_KERNEL, "Color Pipeline %d", list->type);
+	list->type = colorop[0]->base.base.id;
+	list->name = kasprintf(GFP_KERNEL, "Color Pipeline %d", colorop[0]->base.base.id);
 
 	return 0;
+
+cleanup:
+	while (--i >= 0)
+		intel_colorop_destroy(&colorop[i]->base);
+	return ret;
 }
 
 int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe)
diff --git a/drivers/gpu/drm/i915/display/intel_colorop.c b/drivers/gpu/drm/i915/display/intel_colorop.c
index a898b8d73dd81..2ec10cebaba56 100644
--- a/drivers/gpu/drm/i915/display/intel_colorop.c
+++ b/drivers/gpu/drm/i915/display/intel_colorop.c
@@ -35,3 +35,9 @@ struct intel_colorop *intel_colorop_create(enum intel_color_block id)
 
 	return colorop;
 }
+
+void intel_colorop_destroy(struct drm_colorop *colorop)
+{
+	drm_colorop_cleanup(colorop);
+	kfree(to_intel_colorop(colorop));
+}
diff --git a/drivers/gpu/drm/i915/display/intel_colorop.h b/drivers/gpu/drm/i915/display/intel_colorop.h
index 9276eee6e75a3..638baf67d98d6 100644
--- a/drivers/gpu/drm/i915/display/intel_colorop.h
+++ b/drivers/gpu/drm/i915/display/intel_colorop.h
@@ -13,5 +13,6 @@ struct intel_colorop;
 struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop);
 struct intel_colorop *intel_colorop_alloc(void);
 struct intel_colorop *intel_colorop_create(enum intel_color_block id);
+void intel_colorop_destroy(struct drm_colorop *colorop);
 
 #endif /* __INTEL_COLOROP_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 59d637c6a1876..6aa6a1dd6e1b0 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -33,6 +33,7 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
+#include <video/vga.h>
 
 #include "intel_connector.h"
 #include "intel_crt.h"
@@ -55,6 +56,7 @@
 #include "intel_pch_display.h"
 #include "intel_pch_refclk.h"
 #include "intel_pfit.h"
+#include "intel_vga.h"
 
 /* Here's the desired hotplug mode */
 #define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_ENABLE |			\
@@ -691,6 +693,11 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector)
 	return ret;
 }
 
+static bool intel_crt_sense_above_threshold(struct intel_display *display)
+{
+	return intel_vga_read(display, VGA_IS0_R, true) & (1 << 4);
+}
+
 static enum drm_connector_status
 intel_crt_load_detect(struct intel_crt *crt, enum pipe pipe)
 {
@@ -702,7 +709,6 @@ intel_crt_load_detect(struct intel_crt *crt, enum pipe pipe)
 	u32 vsample;
 	u32 vblank, vblank_start, vblank_end;
 	u32 dsl;
-	u8 st00;
 	enum drm_connector_status status;
 
 	drm_dbg_kms(display->drm, "starting load-detect on CRT\n");
@@ -736,8 +742,8 @@ intel_crt_load_detect(struct intel_crt *crt, enum pipe pipe)
 		 * border color for Color info.
 		 */
 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(display, pipe));
-		st00 = intel_de_read8(display, _VGA_MSR_WRITE);
-		status = ((st00 & (1 << 4)) != 0) ?
+
+		status = intel_crt_sense_above_threshold(display) ?
 			connector_status_connected :
 			connector_status_disconnected;
 
@@ -777,15 +783,13 @@ intel_crt_load_detect(struct intel_crt *crt, enum pipe pipe)
 		while ((dsl = intel_de_read(display, PIPEDSL(display, pipe))) <= vsample)
 			;
 		/*
-		 * Watch ST00 for an entire scanline
+		 * Watch sense for an entire scanline
 		 */
 		detect = 0;
 		count = 0;
 		do {
 			count++;
-			/* Read the ST00 VGA status register */
-			st00 = intel_de_read8(display, _VGA_MSR_WRITE);
-			if (st00 & (1 << 4))
+			if (intel_crt_sense_above_threshold(display))
 				detect++;
 		} while ((intel_de_read(display, PIPEDSL(display, pipe)) == dsl));
 
diff --git a/drivers/gpu/drm/i915/display/intel_crt_regs.h b/drivers/gpu/drm/i915/display/intel_crt_regs.h
index 571a67ae9afaf..9a93020b9a7e8 100644
--- a/drivers/gpu/drm/i915/display/intel_crt_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_crt_regs.h
@@ -45,6 +45,4 @@
 #define   ADPA_VSYNC_ACTIVE_HIGH		REG_BIT(4)
 #define   ADPA_HSYNC_ACTIVE_HIGH		REG_BIT(3)
 
-#define _VGA_MSR_WRITE _MMIO(0x3c2)
-
 #endif /* __INTEL_CRT_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c
index 53378d2dcbec8..b8189cd5d864a 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc.c
@@ -747,7 +747,9 @@ void intel_pipe_update_end(struct intel_atomic_state *state,
 	 * which would cause the next frame to terminate already at vmin
 	 * vblank start instead of vmax vblank start.
 	 */
-	if (!state->base.legacy_cursor_update)
+	if (!state->base.legacy_cursor_update ||
+	    (intel_psr_use_trans_push(new_crtc_state) &&
+	     !new_crtc_state->vrr.enable))
 		intel_vrr_send_push(NULL, new_crtc_state);
 
 	local_irq_enable();
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index 2c5d917fbd7e9..18d1014de3613 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -19,6 +19,7 @@
 #include "intel_display.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "intel_frontbuffer.h"
@@ -424,7 +425,7 @@ static u32 i9xx_cursor_ctl(const struct intel_plane_state *plane_state)
 		cntl |= MCURSOR_ROTATE_180;
 
 	/* Wa_22012358565:adl-p */
-	if (DISPLAY_VER(display) == 13)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_22012358565))
 		cntl |= MCURSOR_ARB_SLOTS(1);
 
 	return cntl;
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index 7288065d24619..6a471c021c0e6 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -18,6 +18,7 @@
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dp.h"
+#include "intel_dpll.h"
 #include "intel_hdmi.h"
 #include "intel_lt_phy.h"
 #include "intel_panel.h"
@@ -127,8 +128,8 @@ static void intel_cx0_phy_transaction_end(struct intel_encoder *encoder, struct
 	intel_display_power_put(display, POWER_DOMAIN_DC_OFF, wakeref);
 }
 
-void intel_clear_response_ready_flag(struct intel_encoder *encoder,
-				     int lane)
+void intel_cx0_clear_response_ready_flag(struct intel_encoder *encoder,
+					 int lane)
 {
 	struct intel_display *display = to_intel_display(encoder);
 
@@ -155,7 +156,7 @@ void intel_cx0_bus_reset(struct intel_encoder *encoder, int lane)
 		return;
 	}
 
-	intel_clear_response_ready_flag(encoder, lane);
+	intel_cx0_clear_response_ready_flag(encoder, lane);
 }
 
 int intel_cx0_wait_for_ack(struct intel_encoder *encoder,
@@ -222,6 +223,8 @@ static int __intel_cx0_read_once(struct intel_encoder *encoder,
 		return -ETIMEDOUT;
 	}
 
+	intel_cx0_clear_response_ready_flag(encoder, lane);
+
 	intel_de_write(display, XELPDP_PORT_M2P_MSGBUS_CTL(display, port, lane),
 		       XELPDP_PORT_M2P_TRANSACTION_PENDING |
 		       XELPDP_PORT_M2P_COMMAND_READ |
@@ -231,7 +234,7 @@ static int __intel_cx0_read_once(struct intel_encoder *encoder,
 	if (ack < 0)
 		return ack;
 
-	intel_clear_response_ready_flag(encoder, lane);
+	intel_cx0_clear_response_ready_flag(encoder, lane);
 
 	/*
 	 * FIXME: Workaround to let HW to settle
@@ -293,6 +296,8 @@ static int __intel_cx0_write_once(struct intel_encoder *encoder,
 		return -ETIMEDOUT;
 	}
 
+	intel_cx0_clear_response_ready_flag(encoder, lane);
+
 	intel_de_write(display, XELPDP_PORT_M2P_MSGBUS_CTL(display, port, lane),
 		       XELPDP_PORT_M2P_TRANSACTION_PENDING |
 		       (committed ? XELPDP_PORT_M2P_COMMAND_WRITE_COMMITTED :
@@ -321,7 +326,7 @@ static int __intel_cx0_write_once(struct intel_encoder *encoder,
 		return -EINVAL;
 	}
 
-	intel_clear_response_ready_flag(encoder, lane);
+	intel_cx0_clear_response_ready_flag(encoder, lane);
 
 	/*
 	 * FIXME: Workaround to let HW to settle
@@ -547,7 +552,6 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
  */
 
 static const struct intel_c10pll_state mtl_c10_dp_rbr = {
-	.clock = 162000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0xB4,
@@ -573,7 +577,6 @@ static const struct intel_c10pll_state mtl_c10_dp_rbr = {
 };
 
 static const struct intel_c10pll_state mtl_c10_edp_r216 = {
-	.clock = 216000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0x4,
@@ -599,7 +602,6 @@ static const struct intel_c10pll_state mtl_c10_edp_r216 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_edp_r243 = {
-	.clock = 243000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0x34,
@@ -625,7 +627,6 @@ static const struct intel_c10pll_state mtl_c10_edp_r243 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_dp_hbr1 = {
-	.clock = 270000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0xF4,
@@ -651,7 +652,6 @@ static const struct intel_c10pll_state mtl_c10_dp_hbr1 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_edp_r324 = {
-	.clock = 324000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0xB4,
@@ -677,7 +677,6 @@ static const struct intel_c10pll_state mtl_c10_edp_r324 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_edp_r432 = {
-	.clock = 432000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0x4,
@@ -703,7 +702,6 @@ static const struct intel_c10pll_state mtl_c10_edp_r432 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_dp_hbr2 = {
-	.clock = 540000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0xF4,
@@ -729,7 +727,6 @@ static const struct intel_c10pll_state mtl_c10_dp_hbr2 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_edp_r675 = {
-	.clock = 675000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0xB4,
@@ -755,7 +752,6 @@ static const struct intel_c10pll_state mtl_c10_edp_r675 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_dp_hbr3 = {
-	.clock = 810000,
 	.tx = 0x10,
 	.cmn = 0x21,
 	.pll[0] = 0x34,
@@ -780,30 +776,62 @@ static const struct intel_c10pll_state mtl_c10_dp_hbr3 = {
 	.pll[19] = 0x23,
 };
 
-static const struct intel_c10pll_state * const mtl_c10_dp_tables[] = {
-	&mtl_c10_dp_rbr,
-	&mtl_c10_dp_hbr1,
-	&mtl_c10_dp_hbr2,
-	&mtl_c10_dp_hbr3,
-	NULL,
+struct intel_cx0pll_params {
+	const char *name;
+	bool is_c10;
+	bool is_hdmi;
+	int clock_rate;
+	union {
+		const struct intel_c10pll_state *c10;
+		const struct intel_c20pll_state *c20;
+	};
 };
 
-static const struct intel_c10pll_state * const mtl_c10_edp_tables[] = {
-	&mtl_c10_dp_rbr,
-	&mtl_c10_edp_r216,
-	&mtl_c10_edp_r243,
-	&mtl_c10_dp_hbr1,
-	&mtl_c10_edp_r324,
-	&mtl_c10_edp_r432,
-	&mtl_c10_dp_hbr2,
-	&mtl_c10_edp_r675,
-	&mtl_c10_dp_hbr3,
-	NULL,
+#define __C10PLL_PARAMS(__is_hdmi, __clock_rate, __state) { \
+	.name = __stringify(__state), \
+	.is_c10 = true, \
+	.is_hdmi = __is_hdmi, \
+	.clock_rate = __clock_rate, \
+	.c10 = &__state, \
+}
+
+#define __C20PLL_PARAMS(__is_hdmi, __clock_rate, __state)    { \
+	.name = __stringify(__state), \
+	.is_c10 = false, \
+	.is_hdmi = __is_hdmi, \
+	.clock_rate = __clock_rate, \
+	.c20 = &__state, \
+}
+
+#define C10PLL_HDMI_PARAMS(__clock_rate, __state)       __C10PLL_PARAMS(true, __clock_rate, __state)
+#define C10PLL_DP_PARAMS(__clock_rate, __state)         __C10PLL_PARAMS(false, __clock_rate, __state)
+
+#define C20PLL_HDMI_PARAMS(__clock_rate, __state)       __C20PLL_PARAMS(true, __clock_rate, __state)
+#define C20PLL_DP_PARAMS(__clock_rate, __state)         __C20PLL_PARAMS(false, __clock_rate, __state)
+
+static const struct intel_cx0pll_params mtl_c10_dp_tables[] = {
+	C10PLL_DP_PARAMS(162000, mtl_c10_dp_rbr),
+	C10PLL_DP_PARAMS(270000, mtl_c10_dp_hbr1),
+	C10PLL_DP_PARAMS(540000, mtl_c10_dp_hbr2),
+	C10PLL_DP_PARAMS(810000, mtl_c10_dp_hbr3),
+	{}
+};
+
+static const struct intel_cx0pll_params mtl_c10_edp_tables[] = {
+	C10PLL_DP_PARAMS(162000, mtl_c10_dp_rbr),
+	C10PLL_DP_PARAMS(216000, mtl_c10_edp_r216),
+	C10PLL_DP_PARAMS(243000, mtl_c10_edp_r243),
+	C10PLL_DP_PARAMS(270000, mtl_c10_dp_hbr1),
+	C10PLL_DP_PARAMS(324000, mtl_c10_edp_r324),
+	C10PLL_DP_PARAMS(432000, mtl_c10_edp_r432),
+	C10PLL_DP_PARAMS(540000, mtl_c10_dp_hbr2),
+	C10PLL_DP_PARAMS(675000, mtl_c10_edp_r675),
+	C10PLL_DP_PARAMS(810000, mtl_c10_dp_hbr3),
+	{}
 };
 
 /* C20 basic DP 1.4 tables */
 static const struct intel_c20pll_state mtl_c20_dp_rbr = {
-	.clock = 162000,
 	.tx = {	0xbe88, /* tx cfg0 */
 		0x5800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -828,7 +856,6 @@ static const struct intel_c20pll_state mtl_c20_dp_rbr = {
 };
 
 static const struct intel_c20pll_state mtl_c20_dp_hbr1 = {
-	.clock = 270000,
 	.tx = {	0xbe88, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -853,7 +880,6 @@ static const struct intel_c20pll_state mtl_c20_dp_hbr1 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_dp_hbr2 = {
-	.clock = 540000,
 	.tx = {	0xbe88, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -878,7 +904,6 @@ static const struct intel_c20pll_state mtl_c20_dp_hbr2 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_dp_hbr3 = {
-	.clock = 810000,
 	.tx = {	0xbe88, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -904,7 +929,6 @@ static const struct intel_c20pll_state mtl_c20_dp_hbr3 = {
 
 /* C20 basic DP 2.0 tables */
 static const struct intel_c20pll_state mtl_c20_dp_uhbr10 = {
-	.clock = 1000000, /* 10 Gbps */
 	.tx = {	0xbe21, /* tx cfg0 */
 		0xe800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -928,7 +952,6 @@ static const struct intel_c20pll_state mtl_c20_dp_uhbr10 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_dp_uhbr13_5 = {
-	.clock = 1350000, /* 13.5 Gbps */
 	.tx = {	0xbea0, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -953,7 +976,6 @@ static const struct intel_c20pll_state mtl_c20_dp_uhbr13_5 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_dp_uhbr20 = {
-	.clock = 2000000, /* 20 Gbps */
 	.tx = {	0xbe20, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -976,15 +998,15 @@ static const struct intel_c20pll_state mtl_c20_dp_uhbr20 = {
 		},
 };
 
-static const struct intel_c20pll_state * const mtl_c20_dp_tables[] = {
-	&mtl_c20_dp_rbr,
-	&mtl_c20_dp_hbr1,
-	&mtl_c20_dp_hbr2,
-	&mtl_c20_dp_hbr3,
-	&mtl_c20_dp_uhbr10,
-	&mtl_c20_dp_uhbr13_5,
-	&mtl_c20_dp_uhbr20,
-	NULL,
+static const struct intel_cx0pll_params mtl_c20_dp_tables[] = {
+	C20PLL_DP_PARAMS(162000, mtl_c20_dp_rbr),
+	C20PLL_DP_PARAMS(270000, mtl_c20_dp_hbr1),
+	C20PLL_DP_PARAMS(540000, mtl_c20_dp_hbr2),
+	C20PLL_DP_PARAMS(810000, mtl_c20_dp_hbr3),
+	C20PLL_DP_PARAMS(1000000, mtl_c20_dp_uhbr10),
+	C20PLL_DP_PARAMS(1350000, mtl_c20_dp_uhbr13_5),
+	C20PLL_DP_PARAMS(2000000, mtl_c20_dp_uhbr20),
+	{}
 };
 
 /*
@@ -992,7 +1014,6 @@ static const struct intel_c20pll_state * const mtl_c20_dp_tables[] = {
  */
 
 static const struct intel_c20pll_state xe2hpd_c20_edp_r216 = {
-	.clock = 216000,
 	.tx = { 0xbe88,
 		0x4800,
 		0x0000,
@@ -1017,7 +1038,6 @@ static const struct intel_c20pll_state xe2hpd_c20_edp_r216 = {
 };
 
 static const struct intel_c20pll_state xe2hpd_c20_edp_r243 = {
-	.clock = 243000,
 	.tx = { 0xbe88,
 		0x4800,
 		0x0000,
@@ -1042,7 +1062,6 @@ static const struct intel_c20pll_state xe2hpd_c20_edp_r243 = {
 };
 
 static const struct intel_c20pll_state xe2hpd_c20_edp_r324 = {
-	.clock = 324000,
 	.tx = { 0xbe88,
 		0x4800,
 		0x0000,
@@ -1067,7 +1086,6 @@ static const struct intel_c20pll_state xe2hpd_c20_edp_r324 = {
 };
 
 static const struct intel_c20pll_state xe2hpd_c20_edp_r432 = {
-	.clock = 432000,
 	.tx = { 0xbe88,
 		0x4800,
 		0x0000,
@@ -1092,7 +1110,6 @@ static const struct intel_c20pll_state xe2hpd_c20_edp_r432 = {
 };
 
 static const struct intel_c20pll_state xe2hpd_c20_edp_r675 = {
-	.clock = 675000,
 	.tx = { 0xbe88,
 		0x4800,
 		0x0000,
@@ -1116,21 +1133,20 @@ static const struct intel_c20pll_state xe2hpd_c20_edp_r675 = {
 		  },
 };
 
-static const struct intel_c20pll_state * const xe2hpd_c20_edp_tables[] = {
-	&mtl_c20_dp_rbr,
-	&xe2hpd_c20_edp_r216,
-	&xe2hpd_c20_edp_r243,
-	&mtl_c20_dp_hbr1,
-	&xe2hpd_c20_edp_r324,
-	&xe2hpd_c20_edp_r432,
-	&mtl_c20_dp_hbr2,
-	&xe2hpd_c20_edp_r675,
-	&mtl_c20_dp_hbr3,
-	NULL,
+static const struct intel_cx0pll_params xe2hpd_c20_edp_tables[] = {
+	C20PLL_DP_PARAMS(162000, mtl_c20_dp_rbr),
+	C20PLL_DP_PARAMS(216000, xe2hpd_c20_edp_r216),
+	C20PLL_DP_PARAMS(243000, xe2hpd_c20_edp_r243),
+	C20PLL_DP_PARAMS(270000, mtl_c20_dp_hbr1),
+	C20PLL_DP_PARAMS(324000, xe2hpd_c20_edp_r324),
+	C20PLL_DP_PARAMS(432000, xe2hpd_c20_edp_r432),
+	C20PLL_DP_PARAMS(540000, mtl_c20_dp_hbr2),
+	C20PLL_DP_PARAMS(675000, xe2hpd_c20_edp_r675),
+	C20PLL_DP_PARAMS(810000, mtl_c20_dp_hbr3),
+	{}
 };
 
 static const struct intel_c20pll_state xe2hpd_c20_dp_uhbr13_5 = {
-	.clock = 1350000, /* 13.5 Gbps */
 	.tx = {	0xbea0, /* tx cfg0 */
 		0x4800, /* tx cfg1 */
 		0x0000, /* tx cfg2 */
@@ -1154,30 +1170,30 @@ static const struct intel_c20pll_state xe2hpd_c20_dp_uhbr13_5 = {
 		},
 };
 
-static const struct intel_c20pll_state * const xe2hpd_c20_dp_tables[] = {
-	&mtl_c20_dp_rbr,
-	&mtl_c20_dp_hbr1,
-	&mtl_c20_dp_hbr2,
-	&mtl_c20_dp_hbr3,
-	&mtl_c20_dp_uhbr10,
-	&xe2hpd_c20_dp_uhbr13_5,
-	NULL,
-};
-
-static const struct intel_c20pll_state * const xe3lpd_c20_dp_edp_tables[] = {
-	&mtl_c20_dp_rbr,
-	&xe2hpd_c20_edp_r216,
-	&xe2hpd_c20_edp_r243,
-	&mtl_c20_dp_hbr1,
-	&xe2hpd_c20_edp_r324,
-	&xe2hpd_c20_edp_r432,
-	&mtl_c20_dp_hbr2,
-	&xe2hpd_c20_edp_r675,
-	&mtl_c20_dp_hbr3,
-	&mtl_c20_dp_uhbr10,
-	&xe2hpd_c20_dp_uhbr13_5,
-	&mtl_c20_dp_uhbr20,
-	NULL,
+static const struct intel_cx0pll_params xe2hpd_c20_dp_tables[] = {
+	C20PLL_DP_PARAMS(162000, mtl_c20_dp_rbr),
+	C20PLL_DP_PARAMS(270000, mtl_c20_dp_hbr1),
+	C20PLL_DP_PARAMS(540000, mtl_c20_dp_hbr2),
+	C20PLL_DP_PARAMS(810000, mtl_c20_dp_hbr3),
+	C20PLL_DP_PARAMS(1000000, mtl_c20_dp_uhbr10),
+	C20PLL_DP_PARAMS(1350000, xe2hpd_c20_dp_uhbr13_5),
+	{}
+};
+
+static const struct intel_cx0pll_params xe3lpd_c20_dp_edp_tables[] = {
+	C20PLL_DP_PARAMS(162000, mtl_c20_dp_rbr),
+	C20PLL_DP_PARAMS(216000, xe2hpd_c20_edp_r216),
+	C20PLL_DP_PARAMS(243000, xe2hpd_c20_edp_r243),
+	C20PLL_DP_PARAMS(270000, mtl_c20_dp_hbr1),
+	C20PLL_DP_PARAMS(324000, xe2hpd_c20_edp_r324),
+	C20PLL_DP_PARAMS(432000, xe2hpd_c20_edp_r432),
+	C20PLL_DP_PARAMS(540000, mtl_c20_dp_hbr2),
+	C20PLL_DP_PARAMS(675000, xe2hpd_c20_edp_r675),
+	C20PLL_DP_PARAMS(810000, mtl_c20_dp_hbr3),
+	C20PLL_DP_PARAMS(1000000, mtl_c20_dp_uhbr10),
+	C20PLL_DP_PARAMS(1350000, xe2hpd_c20_dp_uhbr13_5),
+	C20PLL_DP_PARAMS(2000000, mtl_c20_dp_uhbr20),
+	{}
 };
 
 /*
@@ -1185,7 +1201,6 @@ static const struct intel_c20pll_state * const xe3lpd_c20_dp_edp_tables[] = {
  */
 
 static const struct intel_c10pll_state mtl_c10_hdmi_25_2 = {
-	.clock = 25200,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x4,
@@ -1211,7 +1226,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_25_2 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_27_0 = {
-	.clock = 27000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34,
@@ -1237,7 +1251,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_27_0 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_74_25 = {
-	.clock = 74250,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4,
@@ -1263,7 +1276,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_74_25 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_148_5 = {
-	.clock = 148500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4,
@@ -1289,7 +1301,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_148_5 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_594 = {
-	.clock = 594000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4,
@@ -1316,7 +1327,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_594 = {
 
 /* Precomputed C10 HDMI PLL tables */
 static const struct intel_c10pll_state mtl_c10_hdmi_27027 = {
-	.clock = 27027,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xC0, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1326,7 +1336,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_27027 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_28320 = {
-	.clock = 28320,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x04, .pll[1] = 0x00, .pll[2] = 0xCC, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1336,7 +1345,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_28320 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_30240 = {
-	.clock = 30240,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x04, .pll[1] = 0x00, .pll[2] = 0xDC, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1346,7 +1354,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_30240 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_31500 = {
-	.clock = 31500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x62, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1356,7 +1363,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_31500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_36000 = {
-	.clock = 36000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xC4, .pll[1] = 0x00, .pll[2] = 0x76, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1366,7 +1372,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_36000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_40000 = {
-	.clock = 40000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x86, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1376,7 +1381,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_40000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_49500 = {
-	.clock = 49500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x74, .pll[1] = 0x00, .pll[2] = 0xAE, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1386,7 +1390,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_49500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_50000 = {
-	.clock = 50000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x74, .pll[1] = 0x00, .pll[2] = 0xB0, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1396,7 +1399,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_50000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_57284 = {
-	.clock = 57284,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xCE, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1406,7 +1408,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_57284 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_58000 = {
-	.clock = 58000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xD0, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1416,7 +1417,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_58000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_65000 = {
-	.clock = 65000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x66, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1426,7 +1426,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_65000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_71000 = {
-	.clock = 71000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x72, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1436,7 +1435,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_71000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_74176 = {
-	.clock = 74176,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1446,7 +1444,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_74176 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_75000 = {
-	.clock = 75000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7C, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1456,7 +1453,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_75000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_78750 = {
-	.clock = 78750,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x84, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1466,7 +1462,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_78750 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_85500 = {
-	.clock = 85500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x92, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1476,7 +1471,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_85500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_88750 = {
-	.clock = 88750,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x74, .pll[1] = 0x00, .pll[2] = 0x98, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1486,7 +1480,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_88750 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_106500 = {
-	.clock = 106500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xBC, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1496,7 +1489,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_106500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_108000 = {
-	.clock = 108000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xC0, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1506,7 +1498,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_108000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_115500 = {
-	.clock = 115500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xD0, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1516,7 +1507,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_115500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_119000 = {
-	.clock = 119000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xD6, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1526,7 +1516,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_119000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_135000 = {
-	.clock = 135000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x6C, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1536,7 +1525,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_135000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_138500 = {
-	.clock = 138500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x70, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1546,7 +1534,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_138500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_147160 = {
-	.clock = 147160,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x78, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1556,7 +1543,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_147160 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_148352 = {
-	.clock = 148352,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1566,7 +1552,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_148352 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_154000 = {
-	.clock = 154000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x80, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1576,7 +1561,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_154000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_162000 = {
-	.clock = 162000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x88, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1586,7 +1570,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_162000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_167000 = {
-	.clock = 167000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x8C, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1596,7 +1579,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_167000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_197802 = {
-	.clock = 197802,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x74, .pll[1] = 0x00, .pll[2] = 0xAE, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1606,7 +1588,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_197802 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_198000 = {
-	.clock = 198000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x74, .pll[1] = 0x00, .pll[2] = 0xAE, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1616,7 +1597,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_198000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_209800 = {
-	.clock = 209800,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xBA, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1626,7 +1606,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_209800 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_241500 = {
-	.clock = 241500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xDA, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1636,7 +1615,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_241500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_262750 = {
-	.clock = 262750,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x68, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1646,7 +1624,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_262750 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_268500 = {
-	.clock = 268500,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x6A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1656,7 +1633,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_268500 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_296703 = {
-	.clock = 296703,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1666,7 +1642,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_296703 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_297000 = {
-	.clock = 297000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1676,7 +1651,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_297000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_319750 = {
-	.clock = 319750,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xB4, .pll[1] = 0x00, .pll[2] = 0x86, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1686,7 +1660,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_319750 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_497750 = {
-	.clock = 497750,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0x34, .pll[1] = 0x00, .pll[2] = 0xE2, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1696,7 +1669,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_497750 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_592000 = {
-	.clock = 592000,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1706,7 +1678,6 @@ static const struct intel_c10pll_state mtl_c10_hdmi_592000 = {
 };
 
 static const struct intel_c10pll_state mtl_c10_hdmi_593407 = {
-	.clock = 593407,
 	.tx = 0x10,
 	.cmn = 0x1,
 	.pll[0] = 0xF4, .pll[1] = 0x00, .pll[2] = 0x7A, .pll[3] = 0x00, .pll[4] = 0x00,
@@ -1715,82 +1686,56 @@ static const struct intel_c10pll_state mtl_c10_hdmi_593407 = {
 	.pll[15] = 0x08, .pll[16] = 0x08, .pll[17] = 0x8F, .pll[18] = 0x84, .pll[19] = 0x23,
 };
 
-static const struct intel_c10pll_state * const mtl_c10_hdmi_tables[] = {
-	&mtl_c10_hdmi_25_2, /* Consolidated Table */
-	&mtl_c10_hdmi_27_0, /* Consolidated Table */
-	&mtl_c10_hdmi_27027,
-	&mtl_c10_hdmi_28320,
-	&mtl_c10_hdmi_30240,
-	&mtl_c10_hdmi_31500,
-	&mtl_c10_hdmi_36000,
-	&mtl_c10_hdmi_40000,
-	&mtl_c10_hdmi_49500,
-	&mtl_c10_hdmi_50000,
-	&mtl_c10_hdmi_57284,
-	&mtl_c10_hdmi_58000,
-	&mtl_c10_hdmi_65000,
-	&mtl_c10_hdmi_71000,
-	&mtl_c10_hdmi_74176,
-	&mtl_c10_hdmi_74_25, /* Consolidated Table */
-	&mtl_c10_hdmi_75000,
-	&mtl_c10_hdmi_78750,
-	&mtl_c10_hdmi_85500,
-	&mtl_c10_hdmi_88750,
-	&mtl_c10_hdmi_106500,
-	&mtl_c10_hdmi_108000,
-	&mtl_c10_hdmi_115500,
-	&mtl_c10_hdmi_119000,
-	&mtl_c10_hdmi_135000,
-	&mtl_c10_hdmi_138500,
-	&mtl_c10_hdmi_147160,
-	&mtl_c10_hdmi_148352,
-	&mtl_c10_hdmi_148_5, /* Consolidated Table */
-	&mtl_c10_hdmi_154000,
-	&mtl_c10_hdmi_162000,
-	&mtl_c10_hdmi_167000,
-	&mtl_c10_hdmi_197802,
-	&mtl_c10_hdmi_198000,
-	&mtl_c10_hdmi_209800,
-	&mtl_c10_hdmi_241500,
-	&mtl_c10_hdmi_262750,
-	&mtl_c10_hdmi_268500,
-	&mtl_c10_hdmi_296703,
-	&mtl_c10_hdmi_297000,
-	&mtl_c10_hdmi_319750,
-	&mtl_c10_hdmi_497750,
-	&mtl_c10_hdmi_592000,
-	&mtl_c10_hdmi_593407,
-	&mtl_c10_hdmi_594, /* Consolidated Table */
-	NULL,
-};
-
-static const struct intel_c20pll_state mtl_c20_hdmi_25_175 = {
-	.clock = 25175,
-	.tx = {  0xbe88, /* tx cfg0 */
-		  0x9800, /* tx cfg1 */
-		  0x0000, /* tx cfg2 */
-		},
-	.cmn = { 0x0500, /* cmn cfg0*/
-		  0x0005, /* cmn cfg1 */
-		  0x0000, /* cmn cfg2 */
-		  0x0000, /* cmn cfg3 */
-		},
-	.mpllb = { 0xa0d2,	/* mpllb cfg0 */
-		   0x7d80,	/* mpllb cfg1 */
-		   0x0906,	/* mpllb cfg2 */
-		   0xbe40,	/* mpllb cfg3 */
-		   0x0000,	/* mpllb cfg4 */
-		   0x0000,	/* mpllb cfg5 */
-		   0x0200,	/* mpllb cfg6 */
-		   0x0001,	/* mpllb cfg7 */
-		   0x0000,	/* mpllb cfg8 */
-		   0x0000,	/* mpllb cfg9 */
-		   0x0001,	/* mpllb cfg10 */
-		},
+static const struct intel_cx0pll_params mtl_c10_hdmi_tables[] = {
+	C10PLL_HDMI_PARAMS(25200, mtl_c10_hdmi_25_2), /* Consolidated Table */
+	C10PLL_HDMI_PARAMS(27000, mtl_c10_hdmi_27_0), /* Consolidated Table */
+	C10PLL_HDMI_PARAMS(27027, mtl_c10_hdmi_27027),
+	C10PLL_HDMI_PARAMS(28320, mtl_c10_hdmi_28320),
+	C10PLL_HDMI_PARAMS(30240, mtl_c10_hdmi_30240),
+	C10PLL_HDMI_PARAMS(31500, mtl_c10_hdmi_31500),
+	C10PLL_HDMI_PARAMS(36000, mtl_c10_hdmi_36000),
+	C10PLL_HDMI_PARAMS(40000, mtl_c10_hdmi_40000),
+	C10PLL_HDMI_PARAMS(49500, mtl_c10_hdmi_49500),
+	C10PLL_HDMI_PARAMS(50000, mtl_c10_hdmi_50000),
+	C10PLL_HDMI_PARAMS(57284, mtl_c10_hdmi_57284),
+	C10PLL_HDMI_PARAMS(58000, mtl_c10_hdmi_58000),
+	C10PLL_HDMI_PARAMS(65000, mtl_c10_hdmi_65000),
+	C10PLL_HDMI_PARAMS(71000, mtl_c10_hdmi_71000),
+	C10PLL_HDMI_PARAMS(74176, mtl_c10_hdmi_74176),
+	C10PLL_HDMI_PARAMS(74250, mtl_c10_hdmi_74_25), /* Consolidated Table */
+	C10PLL_HDMI_PARAMS(75000, mtl_c10_hdmi_75000),
+	C10PLL_HDMI_PARAMS(78750, mtl_c10_hdmi_78750),
+	C10PLL_HDMI_PARAMS(85500, mtl_c10_hdmi_85500),
+	C10PLL_HDMI_PARAMS(88750, mtl_c10_hdmi_88750),
+	C10PLL_HDMI_PARAMS(106500, mtl_c10_hdmi_106500),
+	C10PLL_HDMI_PARAMS(108000, mtl_c10_hdmi_108000),
+	C10PLL_HDMI_PARAMS(115500, mtl_c10_hdmi_115500),
+	C10PLL_HDMI_PARAMS(119000, mtl_c10_hdmi_119000),
+	C10PLL_HDMI_PARAMS(135000, mtl_c10_hdmi_135000),
+	C10PLL_HDMI_PARAMS(138500, mtl_c10_hdmi_138500),
+	C10PLL_HDMI_PARAMS(147160, mtl_c10_hdmi_147160),
+	C10PLL_HDMI_PARAMS(148352, mtl_c10_hdmi_148352),
+	C10PLL_HDMI_PARAMS(148500, mtl_c10_hdmi_148_5), /* Consolidated Table */
+	C10PLL_HDMI_PARAMS(154000, mtl_c10_hdmi_154000),
+	C10PLL_HDMI_PARAMS(162000, mtl_c10_hdmi_162000),
+	C10PLL_HDMI_PARAMS(167000, mtl_c10_hdmi_167000),
+	C10PLL_HDMI_PARAMS(197802, mtl_c10_hdmi_197802),
+	C10PLL_HDMI_PARAMS(198000, mtl_c10_hdmi_198000),
+	C10PLL_HDMI_PARAMS(209800, mtl_c10_hdmi_209800),
+	C10PLL_HDMI_PARAMS(241500, mtl_c10_hdmi_241500),
+	C10PLL_HDMI_PARAMS(262750, mtl_c10_hdmi_262750),
+	C10PLL_HDMI_PARAMS(268500, mtl_c10_hdmi_268500),
+	C10PLL_HDMI_PARAMS(296703, mtl_c10_hdmi_296703),
+	C10PLL_HDMI_PARAMS(297000, mtl_c10_hdmi_297000),
+	C10PLL_HDMI_PARAMS(319750, mtl_c10_hdmi_319750),
+	C10PLL_HDMI_PARAMS(497750, mtl_c10_hdmi_497750),
+	C10PLL_HDMI_PARAMS(592000, mtl_c10_hdmi_592000),
+	C10PLL_HDMI_PARAMS(593407, mtl_c10_hdmi_593407),
+	C10PLL_HDMI_PARAMS(594000, mtl_c10_hdmi_594), /* Consolidated Table */
+	{}
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_27_0 = {
-	.clock = 27000,
 	.tx = {  0xbe88, /* tx cfg0 */
 		  0x9800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1815,7 +1760,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_27_0 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_74_25 = {
-	.clock = 74250,
 	.tx = {  0xbe88, /* tx cfg0 */
 		  0x9800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1840,7 +1784,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_74_25 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_148_5 = {
-	.clock = 148500,
 	.tx = {  0xbe88, /* tx cfg0 */
 		  0x9800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1865,7 +1808,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_148_5 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_594 = {
-	.clock = 594000,
 	.tx = {  0xbe88, /* tx cfg0 */
 		  0x9800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1890,7 +1832,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_594 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_300 = {
-	.clock = 3000000,
 	.tx = {  0xbe98, /* tx cfg0 */
 		  0x8800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1915,7 +1856,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_300 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_600 = {
-	.clock = 6000000,
 	.tx = {  0xbe98, /* tx cfg0 */
 		  0x8800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1940,7 +1880,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_600 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_800 = {
-	.clock = 8000000,
 	.tx = {  0xbe98, /* tx cfg0 */
 		  0x8800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1965,7 +1904,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_800 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_1000 = {
-	.clock = 10000000,
 	.tx = {  0xbe98, /* tx cfg0 */
 		  0x8800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -1990,7 +1928,6 @@ static const struct intel_c20pll_state mtl_c20_hdmi_1000 = {
 };
 
 static const struct intel_c20pll_state mtl_c20_hdmi_1200 = {
-	.clock = 12000000,
 	.tx = {  0xbe98, /* tx cfg0 */
 		  0x8800, /* tx cfg1 */
 		  0x0000, /* tx cfg2 */
@@ -2014,21 +1951,20 @@ static const struct intel_c20pll_state mtl_c20_hdmi_1200 = {
 		},
 };
 
-static const struct intel_c20pll_state * const mtl_c20_hdmi_tables[] = {
-	&mtl_c20_hdmi_25_175,
-	&mtl_c20_hdmi_27_0,
-	&mtl_c20_hdmi_74_25,
-	&mtl_c20_hdmi_148_5,
-	&mtl_c20_hdmi_594,
-	&mtl_c20_hdmi_300,
-	&mtl_c20_hdmi_600,
-	&mtl_c20_hdmi_800,
-	&mtl_c20_hdmi_1000,
-	&mtl_c20_hdmi_1200,
-	NULL,
+static const struct intel_cx0pll_params mtl_c20_hdmi_tables[] = {
+	C20PLL_HDMI_PARAMS(27000, mtl_c20_hdmi_27_0),
+	C20PLL_HDMI_PARAMS(74250, mtl_c20_hdmi_74_25),
+	C20PLL_HDMI_PARAMS(148500, mtl_c20_hdmi_148_5),
+	C20PLL_HDMI_PARAMS(594000, mtl_c20_hdmi_594),
+	C20PLL_HDMI_PARAMS(300000, mtl_c20_hdmi_300),
+	C20PLL_HDMI_PARAMS(600000, mtl_c20_hdmi_600),
+	C20PLL_HDMI_PARAMS(800000, mtl_c20_hdmi_800),
+	C20PLL_HDMI_PARAMS(1000000, mtl_c20_hdmi_1000),
+	C20PLL_HDMI_PARAMS(1200000, mtl_c20_hdmi_1200),
+	{}
 };
 
-static const struct intel_c10pll_state * const *
+static const struct intel_cx0pll_params *
 intel_c10pll_tables_get(const struct intel_crtc_state *crtc_state,
 			struct intel_encoder *encoder)
 {
@@ -2103,21 +2039,99 @@ static bool cx0pll_state_is_dp(const struct intel_cx0pll_state *pll_state)
 	return c20pll_state_is_dp(&pll_state->c20);
 }
 
+static int intel_c10pll_calc_port_clock(const struct intel_c10pll_state *pll_state)
+{
+	unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1;
+	unsigned int multiplier, tx_clk_div, hdmi_div, refclk = 38400;
+	int tmpclk = 0;
+
+	if (pll_state->pll[0] & C10_PLL0_FRACEN) {
+		frac_quot = pll_state->pll[12] << 8 | pll_state->pll[11];
+		frac_rem =  pll_state->pll[14] << 8 | pll_state->pll[13];
+		frac_den =  pll_state->pll[10] << 8 | pll_state->pll[9];
+	}
+
+	multiplier = (REG_FIELD_GET8(C10_PLL3_MULTIPLIERH_MASK, pll_state->pll[3]) << 8 |
+		      pll_state->pll[2]) / 2 + 16;
+
+	tx_clk_div = REG_FIELD_GET8(C10_PLL15_TXCLKDIV_MASK, pll_state->pll[15]);
+	hdmi_div = REG_FIELD_GET8(C10_PLL15_HDMIDIV_MASK, pll_state->pll[15]);
+
+	tmpclk = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, (multiplier << 16) + frac_quot) +
+				     DIV_ROUND_CLOSEST(refclk * frac_rem, frac_den),
+				     10 << (tx_clk_div + 16));
+	tmpclk *= (hdmi_div ? 2 : 1);
+
+	return tmpclk;
+}
+
+static bool intel_c20phy_use_mpllb(const struct intel_c20pll_state *state)
+{
+	return state->tx[0] & C20_PHY_USE_MPLLB;
+}
+
+static int intel_c20pll_calc_port_clock(const struct intel_c20pll_state *pll_state)
+{
+	unsigned int frac, frac_en, frac_quot, frac_rem, frac_den;
+	unsigned int multiplier, refclk = 38400;
+	unsigned int tx_clk_div;
+	unsigned int ref_clk_mpllb_div;
+	unsigned int fb_clk_div4_en;
+	unsigned int ref, vco;
+	unsigned int tx_rate_mult;
+	unsigned int tx_rate = REG_FIELD_GET(C20_PHY_TX_RATE, pll_state->tx[0]);
+
+	if (intel_c20phy_use_mpllb(pll_state)) {
+		tx_rate_mult = 1;
+		frac_en = REG_FIELD_GET(C20_MPLLB_FRACEN, pll_state->mpllb[6]);
+		frac_quot = pll_state->mpllb[8];
+		frac_rem =  pll_state->mpllb[9];
+		frac_den =  pll_state->mpllb[7];
+		multiplier = REG_FIELD_GET(C20_MULTIPLIER_MASK, pll_state->mpllb[0]);
+		tx_clk_div = REG_FIELD_GET(C20_MPLLB_TX_CLK_DIV_MASK, pll_state->mpllb[0]);
+		ref_clk_mpllb_div = REG_FIELD_GET(C20_REF_CLK_MPLLB_DIV_MASK, pll_state->mpllb[6]);
+		fb_clk_div4_en = 0;
+	} else {
+		tx_rate_mult = 2;
+		frac_en = REG_FIELD_GET(C20_MPLLA_FRACEN, pll_state->mplla[6]);
+		frac_quot = pll_state->mplla[8];
+		frac_rem =  pll_state->mplla[9];
+		frac_den =  pll_state->mplla[7];
+		multiplier = REG_FIELD_GET(C20_MULTIPLIER_MASK, pll_state->mplla[0]);
+		tx_clk_div = REG_FIELD_GET(C20_MPLLA_TX_CLK_DIV_MASK, pll_state->mplla[1]);
+		ref_clk_mpllb_div = REG_FIELD_GET(C20_REF_CLK_MPLLB_DIV_MASK, pll_state->mplla[6]);
+		fb_clk_div4_en = REG_FIELD_GET(C20_FB_CLK_DIV4_EN, pll_state->mplla[0]);
+	}
+
+	if (frac_en)
+		frac = frac_quot + DIV_ROUND_CLOSEST(frac_rem, frac_den);
+	else
+		frac = 0;
+
+	ref = DIV_ROUND_CLOSEST(refclk * (1 << (1 + fb_clk_div4_en)), 1 << ref_clk_mpllb_div);
+	vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(ref, (multiplier << (17 - 2)) + frac) >> 17, 10);
+
+	return vco << tx_rate_mult >> tx_clk_div >> tx_rate;
+}
+
 /*
  * TODO: Convert the following to align with intel_c20pll_find_table() and
  * intel_c20pll_calc_state_from_table().
  */
 static int intel_c10pll_calc_state_from_table(struct intel_encoder *encoder,
-					      const struct intel_c10pll_state * const *tables,
+					      const struct intel_cx0pll_params *tables,
 					      bool is_dp, int port_clock, int lane_count,
 					      struct intel_cx0pll_state *pll_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	int i;
 
-	for (i = 0; tables[i]; i++) {
-		if (port_clock == tables[i]->clock) {
-			pll_state->c10 = *tables[i];
+	for (i = 0; tables[i].name; i++) {
+		int clock = intel_c10pll_calc_port_clock(tables[i].c10);
+
+		drm_WARN_ON(display->drm, !intel_dpll_clock_matches(clock, tables[i].clock_rate));
+		if (intel_dpll_clock_matches(port_clock, clock)) {
+			pll_state->c10 = *tables[i].c10;
 			intel_cx0pll_update_ssc(encoder, pll_state, is_dp);
 			intel_c10pll_update_pll(encoder, pll_state);
 
@@ -2139,7 +2153,7 @@ static int intel_c10pll_calc_state(const struct intel_crtc_state *crtc_state,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	bool is_dp = intel_crtc_has_dp_encoder(crtc_state);
-	const struct intel_c10pll_state * const *tables;
+	const struct intel_cx0pll_params *tables;
 	int err;
 
 	tables = intel_c10pll_tables_get(crtc_state, encoder);
@@ -2166,33 +2180,6 @@ static int intel_c10pll_calc_state(const struct intel_crtc_state *crtc_state,
 	return 0;
 }
 
-static int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
-					const struct intel_c10pll_state *pll_state)
-{
-	unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1;
-	unsigned int multiplier, tx_clk_div, hdmi_div, refclk = 38400;
-	int tmpclk = 0;
-
-	if (pll_state->pll[0] & C10_PLL0_FRACEN) {
-		frac_quot = pll_state->pll[12] << 8 | pll_state->pll[11];
-		frac_rem =  pll_state->pll[14] << 8 | pll_state->pll[13];
-		frac_den =  pll_state->pll[10] << 8 | pll_state->pll[9];
-	}
-
-	multiplier = (REG_FIELD_GET8(C10_PLL3_MULTIPLIERH_MASK, pll_state->pll[3]) << 8 |
-		      pll_state->pll[2]) / 2 + 16;
-
-	tx_clk_div = REG_FIELD_GET8(C10_PLL15_TXCLKDIV_MASK, pll_state->pll[15]);
-	hdmi_div = REG_FIELD_GET8(C10_PLL15_HDMIDIV_MASK, pll_state->pll[15]);
-
-	tmpclk = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, (multiplier << 16) + frac_quot) +
-				     DIV_ROUND_CLOSEST(refclk * frac_rem, frac_den),
-				     10 << (tx_clk_div + 16));
-	tmpclk *= (hdmi_div ? 2 : 1);
-
-	return tmpclk;
-}
-
 static int readout_enabled_lane_count(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -2275,8 +2262,6 @@ static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
 
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 
-	pll_state->clock = intel_c10pll_calc_port_clock(encoder, pll_state);
-
 	cx0pll_state->ssc_enabled = readout_ssc_state(encoder, true);
 
 	if (cx0pll_state->ssc_enabled != intel_c10pll_ssc_enabled(pll_state))
@@ -2321,8 +2306,7 @@ static void intel_c10pll_dump_hw_state(struct drm_printer *p,
 	unsigned int multiplier, tx_clk_div;
 
 	fracen = hw_state->pll[0] & C10_PLL0_FRACEN;
-	drm_printf(p, "c10pll_hw_state: clock: %d, fracen: %s, ",
-		   hw_state->clock, str_yes_no(fracen));
+	drm_printf(p, "c10pll_hw_state: fracen: %s, ", str_yes_no(fracen));
 
 	if (fracen) {
 		frac_quot = hw_state->pll[12] << 8 | hw_state->pll[11];
@@ -2364,9 +2348,8 @@ static bool is_arrowlake_s_by_host_bridge(void)
 	return pdev && IS_ARROWLAKE_S_BY_HOST_BRIDGE_ID(host_bridge_pci_dev_id);
 }
 
-static u16 intel_c20_hdmi_tmds_tx_cgf_1(const struct intel_crtc_state *crtc_state)
+static u16 intel_c20_hdmi_tmds_tx_cgf_1(struct intel_display *display)
 {
-	struct intel_display *display = to_intel_display(crtc_state);
 	u16 tx_misc;
 	u16 tx_dcc_cal_dac_ctrl_range = 8;
 	u16 tx_term_ctrl = 2;
@@ -2388,7 +2371,8 @@ static u16 intel_c20_hdmi_tmds_tx_cgf_1(const struct intel_crtc_state *crtc_stat
 		C20_PHY_TX_DCC_BYPASS | C20_PHY_TX_TERM_CTL(tx_term_ctrl));
 }
 
-static int intel_c20_compute_hdmi_tmds_pll(const struct intel_crtc_state *crtc_state,
+static int intel_c20_compute_hdmi_tmds_pll(struct intel_display *display,
+					   int port_clock,
 					   struct intel_c20pll_state *pll_state)
 {
 	u64 datarate;
@@ -2402,10 +2386,10 @@ static int intel_c20_compute_hdmi_tmds_pll(const struct intel_crtc_state *crtc_s
 	u8  mpllb_ana_freq_vco;
 	u8  mpll_div_multiplier;
 
-	if (crtc_state->port_clock < 25175 || crtc_state->port_clock > 600000)
+	if (port_clock < 25175 || port_clock > 600000)
 		return -EINVAL;
 
-	datarate = ((u64)crtc_state->port_clock * 1000) * 10;
+	datarate = ((u64)port_clock * 1000) * 10;
 	mpll_tx_clk_div = ilog2(div64_u64((u64)CLOCK_9999MHZ, (u64)datarate));
 	vco_freq_shift = ilog2(div64_u64((u64)CLOCK_4999MHZ * (u64)256, (u64)datarate));
 	vco_freq = (datarate << vco_freq_shift) >> 8;
@@ -2427,9 +2411,8 @@ static int intel_c20_compute_hdmi_tmds_pll(const struct intel_crtc_state *crtc_s
 	else
 		mpllb_ana_freq_vco = MPLLB_ANA_FREQ_VCO_0;
 
-	pll_state->clock	= crtc_state->port_clock;
 	pll_state->tx[0]	= 0xbe88;
-	pll_state->tx[1]	= intel_c20_hdmi_tmds_tx_cgf_1(crtc_state);
+	pll_state->tx[1]	= intel_c20_hdmi_tmds_tx_cgf_1(display);
 	pll_state->tx[2]	= 0x0000;
 	pll_state->cmn[0]	= 0x0500;
 	pll_state->cmn[1]	= 0x0005;
@@ -2457,7 +2440,7 @@ static int intel_c20_compute_hdmi_tmds_pll(const struct intel_crtc_state *crtc_s
 	return 0;
 }
 
-static const struct intel_c20pll_state * const *
+static const struct intel_cx0pll_params *
 intel_c20_pll_tables_get(const struct intel_crtc_state *crtc_state,
 			 struct intel_encoder *encoder)
 {
@@ -2625,20 +2608,25 @@ static void intel_c20_program_vdr_params(struct intel_encoder *encoder,
 		      MB_WRITE_COMMITTED);
 }
 
-static const struct intel_c20pll_state *
+static const struct intel_cx0pll_params *
 intel_c20_pll_find_table(const struct intel_crtc_state *crtc_state,
 			 struct intel_encoder *encoder)
 {
-	const struct intel_c20pll_state * const *tables;
+	struct intel_display *display = to_intel_display(crtc_state);
+	const struct intel_cx0pll_params *tables;
 	int i;
 
 	tables = intel_c20_pll_tables_get(crtc_state, encoder);
 	if (!tables)
 		return NULL;
 
-	for (i = 0; tables[i]; i++)
-		if (crtc_state->port_clock == tables[i]->clock)
-			return tables[i];
+	for (i = 0; tables[i].name; i++) {
+		int clock = intel_c20pll_calc_port_clock(tables[i].c20);
+
+		drm_WARN_ON(display->drm, !intel_dpll_clock_matches(clock, tables[i].clock_rate));
+		if (intel_dpll_clock_matches(crtc_state->port_clock, clock))
+			return &tables[i];
+	}
 
 	return NULL;
 }
@@ -2647,13 +2635,13 @@ static int intel_c20pll_calc_state_from_table(const struct intel_crtc_state *crt
 					      struct intel_encoder *encoder,
 					      struct intel_cx0pll_state *pll_state)
 {
-	const struct intel_c20pll_state *table;
+	const struct intel_cx0pll_params *table;
 
 	table = intel_c20_pll_find_table(crtc_state, encoder);
 	if (!table)
 		return -EINVAL;
 
-	pll_state->c20 = *table;
+	pll_state->c20 = *table->c20;
 
 	intel_cx0pll_update_ssc(encoder, pll_state, intel_crtc_has_dp_encoder(crtc_state));
 
@@ -2681,7 +2669,8 @@ static int intel_c20pll_calc_state(const struct intel_crtc_state *crtc_state,
 
 	/* TODO: Update SSC state for HDMI as well */
 	if (!is_dp && err)
-		err = intel_c20_compute_hdmi_tmds_pll(crtc_state, &hw_state->cx0pll.c20);
+		err = intel_c20_compute_hdmi_tmds_pll(display, crtc_state->port_clock,
+						      &hw_state->cx0pll.c20);
 
 	if (err)
 		return err;
@@ -2705,56 +2694,6 @@ int intel_cx0pll_calc_state(const struct intel_crtc_state *crtc_state,
 	return intel_c20pll_calc_state(crtc_state, encoder, hw_state);
 }
 
-static bool intel_c20phy_use_mpllb(const struct intel_c20pll_state *state)
-{
-	return state->tx[0] & C20_PHY_USE_MPLLB;
-}
-
-static int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
-					const struct intel_c20pll_state *pll_state)
-{
-	unsigned int frac, frac_en, frac_quot, frac_rem, frac_den;
-	unsigned int multiplier, refclk = 38400;
-	unsigned int tx_clk_div;
-	unsigned int ref_clk_mpllb_div;
-	unsigned int fb_clk_div4_en;
-	unsigned int ref, vco;
-	unsigned int tx_rate_mult;
-	unsigned int tx_rate = REG_FIELD_GET(C20_PHY_TX_RATE, pll_state->tx[0]);
-
-	if (intel_c20phy_use_mpllb(pll_state)) {
-		tx_rate_mult = 1;
-		frac_en = REG_FIELD_GET(C20_MPLLB_FRACEN, pll_state->mpllb[6]);
-		frac_quot = pll_state->mpllb[8];
-		frac_rem =  pll_state->mpllb[9];
-		frac_den =  pll_state->mpllb[7];
-		multiplier = REG_FIELD_GET(C20_MULTIPLIER_MASK, pll_state->mpllb[0]);
-		tx_clk_div = REG_FIELD_GET(C20_MPLLB_TX_CLK_DIV_MASK, pll_state->mpllb[0]);
-		ref_clk_mpllb_div = REG_FIELD_GET(C20_REF_CLK_MPLLB_DIV_MASK, pll_state->mpllb[6]);
-		fb_clk_div4_en = 0;
-	} else {
-		tx_rate_mult = 2;
-		frac_en = REG_FIELD_GET(C20_MPLLA_FRACEN, pll_state->mplla[6]);
-		frac_quot = pll_state->mplla[8];
-		frac_rem =  pll_state->mplla[9];
-		frac_den =  pll_state->mplla[7];
-		multiplier = REG_FIELD_GET(C20_MULTIPLIER_MASK, pll_state->mplla[0]);
-		tx_clk_div = REG_FIELD_GET(C20_MPLLA_TX_CLK_DIV_MASK, pll_state->mplla[1]);
-		ref_clk_mpllb_div = REG_FIELD_GET(C20_REF_CLK_MPLLB_DIV_MASK, pll_state->mplla[6]);
-		fb_clk_div4_en = REG_FIELD_GET(C20_FB_CLK_DIV4_EN, pll_state->mplla[0]);
-	}
-
-	if (frac_en)
-		frac = frac_quot + DIV_ROUND_CLOSEST(frac_rem, frac_den);
-	else
-		frac = 0;
-
-	ref = DIV_ROUND_CLOSEST(refclk * (1 << (1 + fb_clk_div4_en)), 1 << ref_clk_mpllb_div);
-	vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(ref, (multiplier << (17 - 2)) + frac) >> 17, 10);
-
-	return vco << tx_rate_mult >> tx_clk_div >> tx_rate;
-}
-
 static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 					  struct intel_cx0pll_state *cx0pll_state)
 {
@@ -2823,8 +2762,6 @@ static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 		}
 	}
 
-	pll_state->clock = intel_c20pll_calc_port_clock(encoder, pll_state);
-
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 
 	cx0pll_state->ssc_enabled = readout_ssc_state(encoder, intel_c20phy_use_mpllb(pll_state));
@@ -2835,7 +2772,7 @@ static void intel_c20pll_dump_hw_state(struct drm_printer *p,
 {
 	int i;
 
-	drm_printf(p, "c20pll_hw_state: clock: %d\n", hw_state->clock);
+	drm_printf(p, "c20pll_hw_state:\n");
 	drm_printf(p,
 		   "tx[0] = 0x%.4x, tx[1] = 0x%.4x, tx[2] = 0x%.4x\n",
 		   hw_state->tx[0], hw_state->tx[1], hw_state->tx[2]);
@@ -2971,6 +2908,12 @@ static void intel_c20_pll_program(struct intel_display *display,
 		      MB_WRITE_COMMITTED);
 }
 
+static bool is_mplla_clock_rate(int clock)
+{
+	return intel_dpll_clock_matches(clock, 1000000) ||
+	       intel_dpll_clock_matches(clock, 2000000);
+}
+
 static void intel_program_port_clock_ctl(struct intel_encoder *encoder,
 					 const struct intel_cx0pll_state *pll_state,
 					 int port_clock,
@@ -2996,7 +2939,7 @@ static void intel_program_port_clock_ctl(struct intel_encoder *encoder,
 
 	/* TODO: HDMI FRL */
 	/* DP2.0 10G and 20G rates enable MPLLA*/
-	if (port_clock == 1000000 || port_clock == 2000000)
+	if (is_mplla_clock_rate(port_clock))
 		val |= pll_state->ssc_enabled ? XELPDP_SSC_ENABLE_PLLA : 0;
 	else
 		val |= pll_state->ssc_enabled ? XELPDP_SSC_ENABLE_PLLB : 0;
@@ -3223,7 +3166,6 @@ static u32 intel_cx0_get_pclk_pll_ack(u8 lane_mask)
 static void intel_cx0pll_enable(struct intel_encoder *encoder,
 				const struct intel_cx0pll_state *pll_state)
 {
-	int port_clock = pll_state->use_c10 ? pll_state->c10.clock : pll_state->c20.clock;
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
@@ -3231,6 +3173,12 @@ static void intel_cx0pll_enable(struct intel_encoder *encoder,
 	u8 maxpclk_lane = lane_reversal ? INTEL_CX0_LANE1 :
 					  INTEL_CX0_LANE0;
 	struct ref_tracker *wakeref = intel_cx0_phy_transaction_begin(encoder);
+	int port_clock;
+
+	if (pll_state->use_c10)
+		port_clock = intel_c10pll_calc_port_clock(&pll_state->c10);
+	else
+		port_clock = intel_c20pll_calc_port_clock(&pll_state->c20);
 
 	/*
 	 * Lane reversal is never used in DP-alt mode, in that case the
@@ -3730,9 +3678,9 @@ int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
 				 const struct intel_cx0pll_state *pll_state)
 {
 	if (intel_encoder_is_c10phy(encoder))
-		return intel_c10pll_calc_port_clock(encoder, &pll_state->c10);
+		return intel_c10pll_calc_port_clock(&pll_state->c10);
 
-	return intel_c20pll_calc_port_clock(encoder, &pll_state->c20);
+	return intel_c20pll_calc_port_clock(&pll_state->c20);
 }
 
 /*
@@ -3786,3 +3734,124 @@ void intel_cx0_pll_power_save_wa(struct intel_display *display)
 		intel_cx0pll_disable(encoder);
 	}
 }
+
+static void intel_c10pll_verify_clock(struct intel_display *display,
+				      int precomputed_clock,
+				      const char *pll_state_name,
+				      const struct intel_c10pll_state *pll_state,
+				      bool is_precomputed_state)
+{
+	struct drm_printer p;
+	int clock;
+
+	clock = intel_c10pll_calc_port_clock(pll_state);
+
+	if (intel_dpll_clock_matches(clock, precomputed_clock))
+		return;
+
+	drm_warn(display->drm,
+		 "PLL state %s (%s): clock difference too high: computed %d, pre-computed %d\n",
+		 pll_state_name,
+		 is_precomputed_state ? "precomputed" : "computed",
+		 clock, precomputed_clock);
+
+	if (!drm_debug_enabled(DRM_UT_KMS))
+		return;
+
+	p = drm_dbg_printer(display->drm, DRM_UT_KMS, NULL);
+
+	drm_printf(&p, "PLL state %s (%s):\n",
+		   pll_state_name,
+		   is_precomputed_state ? "precomputed" : "computed");
+	intel_c10pll_dump_hw_state(&p, pll_state);
+}
+
+static void intel_c10pll_verify_params(struct intel_display *display,
+				       const struct intel_cx0pll_params *pll_params)
+{
+	struct intel_c10pll_state pll_state;
+
+	intel_c10pll_verify_clock(display, pll_params->clock_rate, pll_params->name, pll_params->c10, true);
+
+	if (!pll_params->is_hdmi)
+		return;
+
+	intel_snps_hdmi_pll_compute_c10pll(&pll_state, pll_params->clock_rate);
+
+	intel_c10pll_verify_clock(display, pll_params->clock_rate, pll_params->name, &pll_state, false);
+}
+
+static void intel_c20pll_verify_clock(struct intel_display *display,
+				      int precomputed_clock,
+				      const char *pll_state_name,
+				      const struct intel_c20pll_state *pll_state,
+				      bool is_precomputed_state)
+{
+	struct drm_printer p;
+	int clock;
+
+	clock = intel_c20pll_calc_port_clock(pll_state);
+
+	if (intel_dpll_clock_matches(clock, precomputed_clock))
+		return;
+
+	drm_warn(display->drm,
+		 "PLL state %s (%s): clock difference too high: computed %d, pre-computed %d\n",
+		 pll_state_name,
+		 is_precomputed_state ? "precomputed" : "computed",
+		 clock, precomputed_clock);
+
+	if (!drm_debug_enabled(DRM_UT_KMS))
+		return;
+
+	p = drm_dbg_printer(display->drm, DRM_UT_KMS, NULL);
+
+	drm_printf(&p, "PLL state %s (%s):\n",
+		   pll_state_name,
+		   is_precomputed_state ? "precomputed" : "computed");
+	intel_c20pll_dump_hw_state(&p, pll_state);
+}
+
+static void intel_c20pll_verify_params(struct intel_display *display,
+				       const struct intel_cx0pll_params *pll_params)
+{
+	struct intel_c20pll_state pll_state;
+
+	intel_c20pll_verify_clock(display, pll_params->clock_rate, pll_params->name, pll_params->c20, true);
+
+	if (!pll_params->is_hdmi)
+		return;
+
+	if (intel_c20_compute_hdmi_tmds_pll(display, pll_params->clock_rate, &pll_state) != 0)
+		return;
+
+	intel_c20pll_verify_clock(display, pll_params->clock_rate, pll_params->name, &pll_state, false);
+}
+
+static void intel_cx0pll_verify_tables(struct intel_display *display,
+				       const struct intel_cx0pll_params *tables)
+{
+	int i;
+
+	for (i = 0; tables[i].name; i++) {
+		if (tables[i].is_c10)
+			intel_c10pll_verify_params(display, &tables[i]);
+		else
+			intel_c20pll_verify_params(display, &tables[i]);
+	}
+}
+
+void intel_cx0pll_verify_plls(struct intel_display *display)
+{
+	/* C10 */
+	intel_cx0pll_verify_tables(display, mtl_c10_edp_tables);
+	intel_cx0pll_verify_tables(display, mtl_c10_dp_tables);
+	intel_cx0pll_verify_tables(display, mtl_c10_hdmi_tables);
+
+	/* C20 */
+	intel_cx0pll_verify_tables(display, xe2hpd_c20_edp_tables);
+	intel_cx0pll_verify_tables(display, mtl_c20_dp_tables);
+	intel_cx0pll_verify_tables(display, xe2hpd_c20_dp_tables);
+	intel_cx0pll_verify_tables(display, xe3lpd_c20_dp_edp_tables);
+	intel_cx0pll_verify_tables(display, mtl_c20_hdmi_tables);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.h b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
index ae98ac23ea222..1d4480b8bf394 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
@@ -25,8 +25,8 @@ struct intel_dpll_hw_state;
 struct intel_encoder;
 struct intel_hdmi;
 
-void intel_clear_response_ready_flag(struct intel_encoder *encoder,
-				     int lane);
+void intel_cx0_clear_response_ready_flag(struct intel_encoder *encoder,
+					 int lane);
 bool intel_encoder_is_c10phy(struct intel_encoder *encoder);
 void intel_mtl_pll_enable(struct intel_encoder *encoder,
 			  struct intel_dpll *pll,
@@ -77,6 +77,7 @@ bool intel_mtl_tbt_pll_readout_hw_state(struct intel_display *display,
 					struct intel_dpll_hw_state *hw_state);
 int intel_mtl_tbt_calc_port_clock(struct intel_encoder *encoder);
 
+void intel_cx0pll_verify_plls(struct intel_display *display);
 void intel_cx0_pll_power_save_wa(struct intel_display *display);
 void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder,
 				 const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy_regs.h b/drivers/gpu/drm/i915/display/intel_cx0_phy_regs.h
index 658890f735153..152a4e751bdcf 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy_regs.h
@@ -78,10 +78,10 @@
 #define XELPDP_PCLK_PLL_ENABLE_TIMEOUT_US		3200
 #define XELPDP_PCLK_PLL_DISABLE_TIMEOUT_US		20
 #define XELPDP_PORT_BUF_SOC_READY_TIMEOUT_US		100
-#define XELPDP_PORT_RESET_START_TIMEOUT_US		5
+#define XELPDP_PORT_RESET_START_TIMEOUT_US		10
 #define XELPDP_PORT_POWERDOWN_UPDATE_TIMEOUT_MS		2
 #define XELPDP_PORT_RESET_END_TIMEOUT_MS		15
-#define XELPDP_REFCLK_ENABLE_TIMEOUT_US			1
+#define XELPDP_REFCLK_ENABLE_TIMEOUT_US			10
 
 #define _XELPDP_PORT_BUF_CTL1_LN0_A			0x64004
 #define _XELPDP_PORT_BUF_CTL1_LN0_B			0x64104
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index cb91d07cdaa60..7f1576bfe4b09 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -34,7 +34,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_privacy_screen_consumer.h>
 
-#include "i915_reg.h"
 #include "icl_dsi.h"
 #include "intel_alpm.h"
 #include "intel_audio.h"
@@ -53,6 +52,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dkl_phy.h"
 #include "intel_dkl_phy_regs.h"
 #include "intel_dp.h"
@@ -1402,8 +1402,7 @@ static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder,
 		int level;
 
 		/* Wa_16011342517:adl-p */
-		if (display->platform.alderlake_p &&
-		    IS_DISPLAY_STEP(display, STEP_A0, STEP_D0)) {
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_16011342517)) {
 			if ((intel_encoder_is_hdmi(encoder) &&
 			     crtc_state->port_clock == 594000) ||
 			     (intel_encoder_is_dp(encoder) &&
@@ -4247,13 +4246,15 @@ void intel_ddi_get_clock(struct intel_encoder *encoder,
 static void xe3plpd_ddi_get_config(struct intel_encoder *encoder,
 				   struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
+
 	intel_lt_phy_pll_readout_hw_state(encoder, crtc_state, &crtc_state->dpll_hw_state.ltpll);
 
 	if (crtc_state->dpll_hw_state.ltpll.tbt_mode)
 		crtc_state->port_clock = intel_mtl_tbt_calc_port_clock(encoder);
 	else
 		crtc_state->port_clock =
-			intel_lt_phy_calc_port_clock(encoder, crtc_state);
+			intel_lt_phy_calc_port_clock(display, &crtc_state->dpll_hw_state.ltpll);
 	intel_ddi_get_config(encoder, crtc_state);
 }
 
@@ -4586,8 +4587,10 @@ intel_ddi_port_sync_transcoders(const struct intel_crtc_state *ref_crtc_state,
 	/*
 	 * We don't enable port sync on BDW due to missing w/as and
 	 * due to not having adjusted the modeset sequence appropriately.
+	 * From, xe3lpd onwards we have defeatured this with reference to
+	 * Wa_16024710867
 	 */
-	if (DISPLAY_VER(display) < 9)
+	if (!IS_DISPLAY_VER(display, 9, 20))
 		return 0;
 
 	if (!intel_crtc_has_type(ref_crtc_state, INTEL_OUTPUT_DP))
diff --git a/drivers/gpu/drm/i915/display/intel_de.h b/drivers/gpu/drm/i915/display/intel_de.h
index a7ce3b875e06c..f30f3f8ebee1d 100644
--- a/drivers/gpu/drm/i915/display/intel_de.h
+++ b/drivers/gpu/drm/i915/display/intel_de.h
@@ -6,6 +6,8 @@
 #ifndef __INTEL_DE_H__
 #define __INTEL_DE_H__
 
+#include <drm/drm_print.h>
+
 #include "intel_display_core.h"
 #include "intel_dmc_wl.h"
 #include "intel_dsb.h"
@@ -34,15 +36,18 @@ intel_de_read(struct intel_display *display, i915_reg_t reg)
 static inline u8
 intel_de_read8(struct intel_display *display, i915_reg_t reg)
 {
-	u8 val;
-
-	intel_dmc_wl_get(display, reg);
+	/* this is only used on VGA registers (possible on pre-g4x) */
+	drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 5 || display->platform.g4x);
 
-	val = intel_uncore_read8(__to_uncore(display), reg);
+	return intel_uncore_read8(__to_uncore(display), reg);
+}
 
-	intel_dmc_wl_put(display, reg);
+static inline void
+intel_de_write8(struct intel_display *display, i915_reg_t reg, u8 val)
+{
+	drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 5 || display->platform.g4x);
 
-	return val;
+	intel_uncore_write8(__to_uncore(display), reg, val);
 }
 
 static inline u64
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index c4246481fc2fe..ee501009a251f 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -50,7 +50,6 @@
 #include "g4x_hdmi.h"
 #include "hsw_ips.h"
 #include "i915_config.h"
-#include "i915_reg.h"
 #include "i9xx_plane.h"
 #include "i9xx_plane_regs.h"
 #include "i9xx_wm.h"
@@ -86,7 +85,6 @@
 #include "intel_dpll.h"
 #include "intel_dpll_mgr.h"
 #include "intel_dpt.h"
-#include "intel_dpt_common.h"
 #include "intel_drrs.h"
 #include "intel_dsb.h"
 #include "intel_dsi.h"
@@ -455,7 +453,7 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 	}
 
 	/* Wa_22012358565:adl-p */
-	if (DISPLAY_VER(display) == 13)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_22012358565))
 		intel_de_rmw(display, PIPE_ARB_CTL(display, pipe),
 			     0, PIPE_ARB_USE_PROG_SLOTS);
 
@@ -709,7 +707,7 @@ static void icl_set_pipe_chicken(const struct intel_crtc_state *crtc_state)
 		tmp |= UNDERRUN_RECOVERY_DISABLE_ADLP;
 
 	/* Wa_14010547955:dg2 */
-	if (display->platform.dg2)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14010547955))
 		tmp |= DG2_RENDER_CCSTAG_4_3_EN;
 
 	intel_de_write(display, PIPE_CHICKEN(pipe), tmp);
@@ -1008,6 +1006,28 @@ static bool intel_casf_disabling(const struct intel_crtc_state *old_crtc_state,
 	return is_disabling(hw.casf_params.casf_enable, old_crtc_state, new_crtc_state);
 }
 
+static bool intel_crtc_lobf_enabling(const struct intel_crtc_state *old_crtc_state,
+				     const struct intel_crtc_state *new_crtc_state)
+{
+	if (!new_crtc_state->hw.active)
+		return false;
+
+	return is_enabling(has_lobf, old_crtc_state, new_crtc_state) ||
+	       (new_crtc_state->has_lobf &&
+		(new_crtc_state->update_lrr || new_crtc_state->update_m_n));
+}
+
+static bool intel_crtc_lobf_disabling(const struct intel_crtc_state *old_crtc_state,
+				      const struct intel_crtc_state *new_crtc_state)
+{
+	if (!old_crtc_state->hw.active)
+		return false;
+
+	return is_disabling(has_lobf, old_crtc_state, new_crtc_state) ||
+		(old_crtc_state->has_lobf &&
+		 (new_crtc_state->update_lrr || new_crtc_state->update_m_n));
+}
+
 #undef is_disabling
 #undef is_enabling
 
@@ -1050,12 +1070,13 @@ static void intel_post_plane_update(struct intel_atomic_state *state,
 	if (audio_enabling(old_crtc_state, new_crtc_state))
 		intel_encoders_audio_enable(state, crtc);
 
-	if (intel_display_wa(display, 14011503117)) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011503117)) {
 		if (old_crtc_state->pch_pfit.enabled != new_crtc_state->pch_pfit.enabled)
 			adl_scaler_ecc_unmask(new_crtc_state);
 	}
 
-	intel_alpm_post_plane_update(state, crtc);
+	if (intel_crtc_lobf_enabling(old_crtc_state, new_crtc_state))
+		intel_alpm_lobf_enable(new_crtc_state);
 
 	intel_psr_post_plane_update(state, crtc);
 }
@@ -1152,7 +1173,9 @@ static void intel_pre_plane_update(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	enum pipe pipe = crtc->pipe;
 
-	intel_alpm_pre_plane_update(state, crtc);
+	if (intel_crtc_lobf_disabling(old_crtc_state, new_crtc_state))
+		intel_alpm_lobf_disable(new_crtc_state);
+
 	intel_psr_pre_plane_update(state, crtc);
 
 	if (intel_crtc_vrr_disabling(state, crtc)) {
@@ -4324,43 +4347,58 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state,
 	return 0;
 }
 
-static int
-compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
-		      struct intel_crtc_state *crtc_state)
+static int bpc_to_bpp(int bpc)
 {
-	struct intel_display *display = to_intel_display(crtc_state);
-	struct drm_connector *connector = conn_state->connector;
-	const struct drm_display_info *info = &connector->display_info;
-	int bpp;
-
-	switch (conn_state->max_bpc) {
+	switch (bpc) {
 	case 6 ... 7:
-		bpp = 6 * 3;
-		break;
+		return 6 * 3;
 	case 8 ... 9:
-		bpp = 8 * 3;
-		break;
+		return 8 * 3;
 	case 10 ... 11:
-		bpp = 10 * 3;
-		break;
+		return 10 * 3;
 	case 12 ... 16:
-		bpp = 12 * 3;
-		break;
+		return 12 * 3;
 	default:
-		MISSING_CASE(conn_state->max_bpc);
+		MISSING_CASE(bpc);
 		return -EINVAL;
 	}
+}
 
-	if (bpp < crtc_state->pipe_bpp) {
+static int
+compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
+		      struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct drm_connector *connector = conn_state->connector;
+	const struct drm_display_info *info = &connector->display_info;
+	int edid_bpc = info->bpc ? : 8;
+	int target_pipe_bpp;
+	int max_edid_bpp;
+
+	max_edid_bpp = bpc_to_bpp(edid_bpc);
+	if (max_edid_bpp < 0)
+		return max_edid_bpp;
+
+	target_pipe_bpp = bpc_to_bpp(conn_state->max_bpc);
+	if (target_pipe_bpp < 0)
+		return target_pipe_bpp;
+
+	/*
+	 * The maximum pipe BPP is the minimum of the max platform BPP and
+	 * the max EDID BPP.
+	 */
+	crtc_state->max_pipe_bpp = min(crtc_state->pipe_bpp, max_edid_bpp);
+
+	if (target_pipe_bpp < crtc_state->pipe_bpp) {
 		drm_dbg_kms(display->drm,
-			    "[CONNECTOR:%d:%s] Limiting display bpp to %d "
+			    "[CONNECTOR:%d:%s] Limiting target display pipe bpp to %d "
 			    "(EDID bpp %d, max requested bpp %d, max platform bpp %d)\n",
 			    connector->base.id, connector->name,
-			    bpp, 3 * info->bpc,
+			    target_pipe_bpp, 3 * info->bpc,
 			    3 * conn_state->max_requested_bpc,
 			    crtc_state->pipe_bpp);
 
-		crtc_state->pipe_bpp = bpp;
+		crtc_state->pipe_bpp = target_pipe_bpp;
 	}
 
 	return 0;
@@ -5458,7 +5496,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	PIPE_CONF_CHECK_I(dsc.config.nsl_bpg_offset);
 
 	PIPE_CONF_CHECK_BOOL(dsc.compression_enable);
-	PIPE_CONF_CHECK_I(dsc.num_streams);
+	PIPE_CONF_CHECK_I(dsc.slice_config.streams_per_pipe);
 	PIPE_CONF_CHECK_I(dsc.compressed_bpp_x16);
 
 	PIPE_CONF_CHECK_BOOL(splitter.enable);
@@ -7356,9 +7394,6 @@ static void intel_atomic_dsb_finish(struct intel_atomic_state *state,
 		intel_psr_trigger_frame_change_event(new_crtc_state->dsb_commit,
 						     state, crtc);
 
-		intel_psr_wait_for_idle_dsb(new_crtc_state->dsb_commit,
-					    new_crtc_state);
-
 		if (new_crtc_state->use_dsb)
 			intel_dsb_vblank_evade(state, new_crtc_state->dsb_commit);
 
@@ -7389,9 +7424,37 @@ static void intel_atomic_dsb_finish(struct intel_atomic_state *state,
 				new_crtc_state->dsb_color);
 
 	if (new_crtc_state->use_dsb && !intel_color_uses_chained_dsb(new_crtc_state)) {
-		intel_dsb_wait_vblanks(new_crtc_state->dsb_commit, 1);
+		/*
+		 * Dsb wait vblank may or may not skip. Let's remove it for PSR
+		 * trans push case to ensure we are not waiting two vblanks
+		 */
+		if (!intel_psr_use_trans_push(new_crtc_state))
+			intel_dsb_wait_vblanks(new_crtc_state->dsb_commit, 1);
 
 		intel_vrr_send_push(new_crtc_state->dsb_commit, new_crtc_state);
+
+		/*
+		 * Wait for idle is needed for corner case where PSR HW
+		 * is transitioning into DEEP_SLEEP/SRDENT_OFF when
+		 * new Frame Change event comes in. It is ok to do it
+		 * here for both Frame Change mechanism (trans push
+		 * and register write).
+		 */
+		intel_psr_wait_for_idle_dsb(new_crtc_state->dsb_commit,
+					    new_crtc_state);
+
+		/*
+		 * In case PSR uses trans push as a "frame change" event and
+		 * VRR is not in use we need to wait vblank. Otherwise we may
+		 * miss selective updates. DSB skips all waits while PSR is
+		 * active. Check push send is skipped as well because trans push
+		 * send bit is not reset by the HW if VRR is not
+		 * enabled -> we may start configuring new selective
+		 * update while previous is not complete.
+		 */
+		if (intel_psr_use_trans_push(new_crtc_state))
+			intel_dsb_wait_vblanks(new_crtc_state->dsb_commit, 1);
+
 		intel_dsb_wait_for_delayed_vblank(state, new_crtc_state->dsb_commit);
 		intel_vrr_check_push_sent(new_crtc_state->dsb_commit,
 					  new_crtc_state);
@@ -7827,7 +7890,8 @@ static bool intel_ddi_crt_present(struct intel_display *display)
 
 bool assert_port_valid(struct intel_display *display, enum port port)
 {
-	return !drm_WARN(display->drm, !(DISPLAY_RUNTIME_INFO(display)->port_mask & BIT(port)),
+	return !drm_WARN(display->drm,
+			 !(port >= 0 && DISPLAY_RUNTIME_INFO(display)->port_mask & BIT(port)),
 			 "Platform does not support port %c\n", port_name(port));
 }
 
@@ -8000,6 +8064,25 @@ void intel_setup_outputs(struct intel_display *display)
 	drm_helper_move_panel_connectors_to_head(display->drm);
 }
 
+int intel_max_uncompressed_dotclock(struct intel_display *display)
+{
+	int max_dotclock = display->cdclk.max_dotclk_freq;
+	int limit = max_dotclock;
+
+	if (DISPLAY_VERx100(display) == 3002)
+		limit = 937500;
+	else if (DISPLAY_VER(display) >= 30)
+		limit = 1350000;
+	/*
+	 * Note: For other platforms though there are limits given
+	 * in the Bspec, however the limit is intentionally not
+	 * enforced to avoid regressions, unless real issues are
+	 * observed.
+	 */
+
+	return min(max_dotclock, limit);
+}
+
 static int max_dotclock(struct intel_display *display)
 {
 	int max_dotclock = display->cdclk.max_dotclk_freq;
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index f8e6e4e827222..552a59d19e0f6 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -135,32 +135,6 @@ enum tc_port {
 	I915_MAX_TC_PORTS
 };
 
-enum aux_ch {
-	AUX_CH_NONE = -1,
-
-	AUX_CH_A,
-	AUX_CH_B,
-	AUX_CH_C,
-	AUX_CH_D,
-	AUX_CH_E, /* ICL+ */
-	AUX_CH_F,
-	AUX_CH_G,
-	AUX_CH_H,
-	AUX_CH_I,
-
-	/* tgl+ */
-	AUX_CH_USBC1 = AUX_CH_D,
-	AUX_CH_USBC2,
-	AUX_CH_USBC3,
-	AUX_CH_USBC4,
-	AUX_CH_USBC5,
-	AUX_CH_USBC6,
-
-	/* XE_LPD repositions D/E offsets and bitfields */
-	AUX_CH_D_XELPD = AUX_CH_USBC5,
-	AUX_CH_E_XELPD,
-};
-
 enum phy {
 	PHY_NONE = -1,
 
@@ -488,6 +462,7 @@ void intel_cpu_transcoder_get_m2_n2(struct intel_crtc *crtc,
 				    struct intel_link_m_n *m_n);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
 int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config);
+int intel_max_uncompressed_dotclock(struct intel_display *display);
 enum intel_display_power_domain intel_port_to_power_domain(struct intel_digital_port *dig_port);
 enum intel_display_power_domain
 intel_aux_power_domain(struct intel_digital_port *dig_port);
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index aba13e8a90518..2614c4863c877 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -13,9 +13,9 @@
 #include <drm/drm_file.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "hsw_ips.h"
-#include "i915_reg.h"
 #include "i9xx_wm_regs.h"
 #include "intel_alpm.h"
 #include "intel_bo.h"
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index 01e0ae01fc502..be55ef8ea6178 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -10,7 +10,6 @@
 #include <drm/drm_print.h>
 #include <drm/intel/pciids.h>
 
-#include "i915_reg.h"
 #include "intel_cx0_phy_regs.h"
 #include "intel_de.h"
 #include "intel_display.h"
@@ -20,6 +19,7 @@
 #include "intel_display_reg_defs.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
+#include "intel_display_wa.h"
 #include "intel_fbc.h"
 #include "intel_step.h"
 
@@ -1500,6 +1500,7 @@ static const struct {
 	INTEL_PTL_IDS(INTEL_DISPLAY_DEVICE, &ptl_desc),
 	INTEL_WCL_IDS(INTEL_DISPLAY_DEVICE, &ptl_desc),
 	INTEL_NVLS_IDS(INTEL_DISPLAY_DEVICE, &nvl_desc),
+	INTEL_NVLP_IDS(INTEL_DISPLAY_DEVICE, &nvl_desc),
 };
 
 static const struct {
@@ -1539,9 +1540,9 @@ probe_gmdid_display(struct intel_display *display, struct intel_display_ip_ver *
 		return NULL;
 	}
 
-	gmd_id.ver = REG_FIELD_GET(GMD_ID_ARCH_MASK, val);
-	gmd_id.rel = REG_FIELD_GET(GMD_ID_RELEASE_MASK, val);
-	gmd_id.step = REG_FIELD_GET(GMD_ID_STEP, val);
+	gmd_id.ver = REG_FIELD_GET(GMD_ID_DISPLAY_ARCH_MASK, val);
+	gmd_id.rel = REG_FIELD_GET(GMD_ID_DISPLAY_RELEASE_MASK, val);
+	gmd_id.step = REG_FIELD_GET(GMD_ID_DISPLAY_STEP, val);
 
 	for (i = 0; i < ARRAY_SIZE(gmdid_display_map); i++) {
 		if (gmd_id.ver == gmdid_display_map[i].ver &&
@@ -1653,6 +1654,28 @@ static void display_platforms_or(struct intel_display_platforms *dst,
 	bitmap_or(dst->bitmap, dst->bitmap, src->bitmap, display_platforms_num_bits());
 }
 
+#define __STEP_NAME(name) [STEP_##name] = #name,
+
+static void initialize_step(struct intel_display *display, enum intel_step step)
+{
+	static const char step_names[][3] = {
+		STEP_NAME_LIST(__STEP_NAME)
+	};
+
+	DISPLAY_RUNTIME_INFO(display)->step = step;
+
+	/* Step name will remain an empty string if not applicable */
+	if (step >= 0 && step < ARRAY_SIZE(step_names))
+		strscpy(DISPLAY_RUNTIME_INFO(display)->step_name, step_names[step]);
+}
+
+#undef __STEP_NAME
+
+static const char *step_name(const struct intel_display_runtime_info *runtime)
+{
+	return strlen(runtime->step_name) ? runtime->step_name : "N/A";
+}
+
 struct intel_display *intel_display_device_probe(struct pci_dev *pdev,
 						 const struct intel_display_parent_interface *parent)
 {
@@ -1730,14 +1753,14 @@ struct intel_display *intel_display_device_probe(struct pci_dev *pdev,
 					  subdesc ? &subdesc->step_info : NULL);
 	}
 
-	DISPLAY_RUNTIME_INFO(display)->step = step;
+	initialize_step(display, step);
 
 	drm_info(display->drm, "Found %s%s%s (device ID %04x) %s display version %u.%02u stepping %s\n",
 		 desc->name, subdesc ? "/" : "", subdesc ? subdesc->name : "",
 		 pdev->device, display->platform.dgfx ? "discrete" : "integrated",
 		 DISPLAY_RUNTIME_INFO(display)->ip.ver,
 		 DISPLAY_RUNTIME_INFO(display)->ip.rel,
-		 step != STEP_NONE ? intel_step_name(step) : "N/A");
+		 step_name(DISPLAY_RUNTIME_INFO(display)));
 
 	return display;
 
@@ -1773,7 +1796,7 @@ static void __intel_display_device_info_runtime_init(struct intel_display *displ
 		display_runtime->port_mask |= BIT(PORT_F);
 
 	/* Wa_14011765242: adl-s A0,A1 */
-	if (display->platform.alderlake_s && IS_DISPLAY_STEP(display, STEP_A0, STEP_A2))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011765242))
 		for_each_pipe(display, pipe)
 			display_runtime->num_scalers[pipe] = 0;
 	else if (DISPLAY_VER(display) >= 11) {
@@ -1953,7 +1976,7 @@ void intel_display_device_info_print(const struct intel_display_device_info *inf
 		drm_printf(p, "display version: %u\n",
 			   runtime->ip.ver);
 
-	drm_printf(p, "display stepping: %s\n", intel_step_name(runtime->step));
+	drm_printf(p, "display stepping: %s\n", step_name(runtime));
 
 #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, str_yes_no(info->name))
 	DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG);
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index 13f2a629981fc..1170ac3466156 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -197,6 +197,7 @@ struct intel_display_platforms {
 #define HAS_PSR(__display)		(DISPLAY_INFO(__display)->has_psr)
 #define HAS_PSR_HW_TRACKING(__display)	(DISPLAY_INFO(__display)->has_psr_hw_tracking)
 #define HAS_PSR2_SEL_FETCH(__display)	(DISPLAY_VER(__display) >= 12)
+#define HAS_PSR_TRANS_PUSH_FRAME_CHANGE(__display)	(DISPLAY_VER(__display) >= 20)
 #define HAS_SAGV(__display)		(DISPLAY_VER(__display) >= 9 && \
 					 !(__display)->platform.broxton && !(__display)->platform.geminilake)
 #define HAS_TRANSCODER(__display, trans)	((DISPLAY_RUNTIME_INFO(__display)->cpu_transcoder_mask & \
@@ -286,6 +287,7 @@ struct intel_display_runtime_info {
 		u16 step; /* hardware */
 	} ip;
 	int step; /* symbolic */
+	char step_name[3]; /* empty string if not applicable */
 
 	u32 rawclk_freq;
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index 268b1de45b813..23bfecc983e8d 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -208,16 +208,12 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 
 	intel_bios_init(display);
 
-	ret = intel_vga_register(display);
-	if (ret)
-		goto cleanup_bios;
-
 	intel_psr_dc5_dc6_wa_init(display);
 
 	/* FIXME: completely on the wrong abstraction layer */
 	ret = intel_power_domains_init(display);
 	if (ret < 0)
-		goto cleanup_vga;
+		goto cleanup_bios;
 
 	intel_pmdemand_init_early(display);
 
@@ -229,7 +225,7 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 	display->hotplug.dp_wq = alloc_ordered_workqueue("intel-dp", 0);
 	if (!display->hotplug.dp_wq) {
 		ret = -ENOMEM;
-		goto cleanup_vga_client_pw_domain_dmc;
+		goto cleanup_pw_domain_dmc;
 	}
 
 	display->wq.modeset = alloc_ordered_workqueue("i915_modeset", 0);
@@ -245,13 +241,13 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 		goto cleanup_wq_modeset;
 	}
 
-	display->wq.cleanup = alloc_workqueue("i915_cleanup", WQ_HIGHPRI, 0);
+	display->wq.cleanup = alloc_workqueue("i915_cleanup", WQ_HIGHPRI | WQ_PERCPU, 0);
 	if (!display->wq.cleanup) {
 		ret = -ENOMEM;
 		goto cleanup_wq_flip;
 	}
 
-	display->wq.unordered = alloc_workqueue("display_unordered", 0, 0);
+	display->wq.unordered = alloc_workqueue("display_unordered", WQ_PERCPU, 0);
 	if (!display->wq.unordered) {
 		ret = -ENOMEM;
 		goto cleanup_wq_cleanup;
@@ -301,11 +297,9 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 	destroy_workqueue(display->wq.modeset);
 cleanup_wq_dp:
 	destroy_workqueue(display->hotplug.dp_wq);
-cleanup_vga_client_pw_domain_dmc:
+cleanup_pw_domain_dmc:
 	intel_dmc_fini(display);
 	intel_power_domains_driver_remove(display);
-cleanup_vga:
-	intel_vga_unregister(display);
 cleanup_bios:
 	intel_bios_driver_remove(display);
 
@@ -554,6 +548,8 @@ void intel_display_driver_register(struct intel_display *display)
 	if (!HAS_DISPLAY(display))
 		return;
 
+	intel_vga_register(display);
+
 	/* Must be done after probing outputs */
 	intel_opregion_register(display);
 	intel_acpi_video_register(display);
@@ -646,8 +642,6 @@ void intel_display_driver_remove_nogem(struct intel_display *display)
 
 	intel_power_domains_driver_remove(display);
 
-	intel_vga_unregister(display);
-
 	intel_bios_driver_remove(display);
 }
 
@@ -675,6 +669,8 @@ void intel_display_driver_unregister(struct intel_display *display)
 
 	acpi_video_unregister();
 	intel_opregion_unregister(display);
+
+	intel_vga_unregister(display);
 }
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index 6e7e4654eb796..70c1bba7c0a8f 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -5,8 +5,8 @@
 
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
-#include "i915_reg.h"
 #include "icl_dsi_regs.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
@@ -1619,7 +1619,7 @@ static void i915gm_irq_cstate_wa_enable(struct intel_display *display)
 	 */
 	if (display->irq.vblank_enabled++ == 0)
 		intel_de_write(display, SCPD0,
-			       _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
+			       REG_MASKED_FIELD_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
 }
 
 static void i915gm_irq_cstate_wa_disable(struct intel_display *display)
@@ -1628,7 +1628,7 @@ static void i915gm_irq_cstate_wa_disable(struct intel_display *display)
 
 	if (--display->irq.vblank_enabled == 0)
 		intel_de_write(display, SCPD0,
-			       _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
+			       REG_MASKED_FIELD_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
 }
 
 void i915gm_irq_cstate_wa(struct intel_display *display, bool enable)
@@ -2472,6 +2472,7 @@ void intel_display_irq_init(struct intel_display *display)
 
 struct intel_display_irq_snapshot {
 	u32 derrmr;
+	u32 err_int;
 };
 
 struct intel_display_irq_snapshot *
@@ -2486,6 +2487,9 @@ intel_display_irq_snapshot_capture(struct intel_display *display)
 	if (DISPLAY_VER(display) >= 6 && DISPLAY_VER(display) < 20 && !HAS_GMCH(display))
 		snapshot->derrmr = intel_de_read(display, DERRMR);
 
+	if (DISPLAY_VER(display) == 7)
+		snapshot->err_int = intel_de_read(display, GEN7_ERR_INT);
+
 	return snapshot;
 }
 
@@ -2496,4 +2500,5 @@ void intel_display_irq_snapshot_print(const struct intel_display_irq_snapshot *s
 		return;
 
 	drm_printf(p, "DERRMR: 0x%08x\n", snapshot->derrmr);
+	drm_printf(p, "ERR_INT: 0x%08x\n", snapshot->err_int);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_limits.h b/drivers/gpu/drm/i915/display/intel_display_limits.h
index cb3c9c665c44e..453f7b7208157 100644
--- a/drivers/gpu/drm/i915/display/intel_display_limits.h
+++ b/drivers/gpu/drm/i915/display/intel_display_limits.h
@@ -138,6 +138,32 @@ enum hpd_pin {
 	HPD_NUM_PINS
 };
 
+enum aux_ch {
+	AUX_CH_NONE = -1,
+
+	AUX_CH_A,
+	AUX_CH_B,
+	AUX_CH_C,
+	AUX_CH_D,
+	AUX_CH_E, /* ICL+ */
+	AUX_CH_F,
+	AUX_CH_G,
+	AUX_CH_H,
+	AUX_CH_I,
+
+	/* tgl+ */
+	AUX_CH_USBC1 = AUX_CH_D,
+	AUX_CH_USBC2,
+	AUX_CH_USBC3,
+	AUX_CH_USBC4,
+	AUX_CH_USBC5,
+	AUX_CH_USBC6,
+
+	/* XE_LPD repositions D/E offsets and bitfields */
+	AUX_CH_D_XELPD = AUX_CH_USBC5,
+	AUX_CH_E_XELPD,
+};
+
 enum intel_color_block {
 	INTEL_PLANE_CB_PRE_CSC_LUT,
 	INTEL_PLANE_CB_CSC,
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index d27397f438633..ec96b141c74c1 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -7,8 +7,8 @@
 #include <linux/string_helpers.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "intel_backlight_regs.h"
 #include "intel_cdclk.h"
 #include "intel_clock_gating.h"
@@ -21,12 +21,12 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_dram.h"
 #include "intel_mchbar_regs.h"
 #include "intel_parent.h"
 #include "intel_pch_refclk.h"
-#include "intel_pcode.h"
 #include "intel_pmdemand.h"
 #include "intel_pps_regs.h"
 #include "intel_snps_phy.h"
@@ -646,7 +646,7 @@ queue_async_put_domains_work(struct i915_power_domains *power_domains,
 						     power.domains);
 	drm_WARN_ON(display->drm, power_domains->async_put_wakeref);
 	power_domains->async_put_wakeref = wakeref;
-	drm_WARN_ON(display->drm, !queue_delayed_work(system_unbound_wq,
+	drm_WARN_ON(display->drm, !queue_delayed_work(system_dfl_wq,
 						      &power_domains->async_put_work,
 						      msecs_to_jiffies(delay_ms)));
 }
@@ -1260,7 +1260,7 @@ static u32 hsw_read_dcomp(struct intel_display *display)
 static void hsw_write_dcomp(struct intel_display *display, u32 val)
 {
 	if (display->platform.haswell) {
-		if (intel_pcode_write(display->drm, GEN6_PCODE_WRITE_D_COMP, val))
+		if (intel_parent_pcode_write(display, GEN6_PCODE_WRITE_D_COMP, val))
 			drm_dbg_kms(display->drm, "Failed to write to D_COMP\n");
 	} else {
 		intel_de_write(display, D_COMP_BDW, val);
@@ -1622,8 +1622,7 @@ static void tgl_bw_buddy_init(struct intel_display *display)
 	if (display->platform.dgfx && !display->platform.dg1)
 		return;
 
-	if (display->platform.alderlake_s ||
-	    (display->platform.rocketlake && IS_DISPLAY_STEP(display, STEP_A0, STEP_B0)))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_1409767108))
 		/* Wa_1409767108 */
 		table = wa_1409767108_buddy_page_masks;
 	else
@@ -1646,7 +1645,7 @@ static void tgl_bw_buddy_init(struct intel_display *display)
 				       table[config].page_mask);
 
 			/* Wa_22010178259:tgl,dg1,rkl,adl-s */
-			if (DISPLAY_VER(display) == 12)
+			if (intel_display_wa(display, INTEL_DISPLAY_WA_22010178259))
 				intel_de_rmw(display, BW_BUDDY_CTL(i),
 					     BW_BUDDY_TLB_REQ_TIMER_MASK,
 					     BW_BUDDY_TLB_REQ_TIMER(0x8));
@@ -1663,8 +1662,7 @@ static void icl_display_core_init(struct intel_display *display,
 	gen9_set_dc_state(display, DC_STATE_DISABLE);
 
 	/* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
-	if (INTEL_PCH_TYPE(display) >= PCH_TGP &&
-	    INTEL_PCH_TYPE(display) < PCH_DG1)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011294188))
 		intel_de_rmw(display, SOUTH_DSPCLK_GATE_D, 0,
 			     PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
 
@@ -1718,17 +1716,17 @@ static void icl_display_core_init(struct intel_display *display,
 		intel_dmc_load_program(display);
 
 	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p,dg2 */
-	if (IS_DISPLAY_VERx100(display, 1200, 1300))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011508470))
 		intel_de_rmw(display, GEN11_CHICKEN_DCPR_2, 0,
 			     DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
 			     DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
 
 	/* Wa_14011503030:xelpd */
-	if (DISPLAY_VER(display) == 13)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011503030))
 		intel_de_write(display, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
 
 	/* Wa_15013987218 */
-	if (DISPLAY_VER(display) == 20) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_15013987218)) {
 		intel_de_rmw(display, SOUTH_DSPCLK_GATE_D,
 			     0, PCH_GMBUSUNIT_CLOCK_GATE_DISABLE);
 		intel_de_rmw(display, SOUTH_DSPCLK_GATE_D,
@@ -2267,8 +2265,9 @@ void intel_display_power_suspend_late(struct intel_display *display, bool s2idle
 	}
 
 	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
-	if (INTEL_PCH_TYPE(display) >= PCH_CNP && INTEL_PCH_TYPE(display) < PCH_DG1)
-		intel_de_rmw(display, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14010685332))
+		intel_de_rmw(display, SOUTH_CHICKEN1,
+			     SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
 }
 
 void intel_display_power_resume_early(struct intel_display *display)
@@ -2282,7 +2281,7 @@ void intel_display_power_resume_early(struct intel_display *display)
 	}
 
 	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
-	if (INTEL_PCH_TYPE(display) >= PCH_CNP && INTEL_PCH_TYPE(display) < PCH_DG1)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14010685332))
 		intel_de_rmw(display, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
 
 	intel_power_domains_resume(display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_map.c b/drivers/gpu/drm/i915/display/intel_display_power_map.c
index c559ff000e67b..65204d68a7598 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_map.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_map.c
@@ -112,7 +112,6 @@ static const struct i915_power_well_desc hsw_power_wells_main[] = {
 				.id = HSW_DISP_PW_GLOBAL),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 	},
 };
 
@@ -146,7 +145,6 @@ static const struct i915_power_well_desc bdw_power_wells_main[] = {
 				.id = HSW_DISP_PW_GLOBAL),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
 	},
 };
@@ -390,7 +388,6 @@ static const struct i915_power_well_desc skl_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
 		.has_fuses = true,
 	}, {
@@ -469,7 +466,6 @@ static const struct i915_power_well_desc bxt_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
 		.has_fuses = true,
 	}, {
@@ -572,7 +568,6 @@ static const struct i915_power_well_desc glk_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
 		.has_fuses = true,
 	}, {
@@ -748,7 +743,6 @@ static const struct i915_power_well_desc icl_power_wells_main[] = {
 				.id = ICL_DISP_PW_3),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B),
 		.has_fuses = true,
 	}, {
@@ -914,7 +908,6 @@ static const struct i915_power_well_desc tgl_power_wells_main[] = {
 				.id = ICL_DISP_PW_3),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.irq_pipe_mask = BIT(PIPE_B),
 		.has_fuses = true,
 	}, {
@@ -1071,7 +1064,6 @@ static const struct i915_power_well_desc rkl_power_wells_main[] = {
 		),
 		.ops = &hsw_power_well_ops,
 		.irq_pipe_mask = BIT(PIPE_B),
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
@@ -1166,7 +1158,6 @@ static const struct i915_power_well_desc dg1_power_wells_main[] = {
 		),
 		.ops = &hsw_power_well_ops,
 		.irq_pipe_mask = BIT(PIPE_B),
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
@@ -1325,7 +1316,6 @@ static const struct i915_power_well_desc xelpd_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
@@ -1482,7 +1472,6 @@ static const struct i915_power_well_desc xelpdp_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
@@ -1649,7 +1638,6 @@ static const struct i915_power_well_desc xe3lpd_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
@@ -1722,7 +1710,6 @@ static const struct i915_power_well_desc wcl_power_wells_main[] = {
 				.id = SKL_DISP_PW_2),
 		),
 		.ops = &hsw_power_well_ops,
-		.has_vga = true,
 		.has_fuses = true,
 	}, {
 		.instances = &I915_PW_INSTANCES(
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index db185a8591339..f855f0f886946 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -6,8 +6,8 @@
 #include <linux/iopoll.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "intel_backlight_regs.h"
 #include "intel_combo_phy.h"
 #include "intel_combo_phy_regs.h"
@@ -18,6 +18,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
+#include "intel_display_wa.h"
 #include "intel_dkl_phy.h"
 #include "intel_dkl_phy_regs.h"
 #include "intel_dmc.h"
@@ -27,7 +28,6 @@
 #include "intel_dpll.h"
 #include "intel_hotplug.h"
 #include "intel_parent.h"
-#include "intel_pcode.h"
 #include "intel_pps.h"
 #include "intel_psr.h"
 #include "intel_tc.h"
@@ -195,6 +195,48 @@ int intel_power_well_refcount(struct i915_power_well *power_well)
 	return power_well->count;
 }
 
+static u32 dss_pipe_gating_bits(u8 irq_pipe_mask)
+{
+	u32 bits = 0;
+
+	if (irq_pipe_mask & BIT(PIPE_A))
+		bits |= DSS_PIPE_A_GATING_DISABLED;
+	if (irq_pipe_mask & BIT(PIPE_B))
+		bits |= DSS_PIPE_B_GATING_DISABLED;
+	if (irq_pipe_mask & BIT(PIPE_C))
+		bits |= DSS_PIPE_C_GATING_DISABLED;
+	if (irq_pipe_mask & BIT(PIPE_D))
+		bits |= DSS_PIPE_D_GATING_DISABLED;
+
+	return bits;
+}
+
+static void dss_pipe_gating_enable_disable(struct intel_display *display,
+					   u8 irq_pipe_mask,
+					   bool disable)
+{
+	u32 bits = dss_pipe_gating_bits(irq_pipe_mask);
+	u32 clear, set;
+
+	if (!bits)
+		return;
+
+	/*
+	 * Single intel_de_rmw() for both enable/disable:
+	 *  - disable == true, set bits (disable clock gating)
+	 *  - disable == false, clear bits (re-enable clock gating)
+	 */
+	set = disable ? bits : 0;
+	clear = disable ? 0 : bits;
+
+	intel_de_rmw(display, CLKGATE_DIS_DSSDSC, clear, set);
+
+	drm_dbg_kms(display->drm,
+		    "DSS clock gating %sd for pipe_mask=0x%x (CLKGATE_DIS_DSSDSC=0x%08x)\n",
+		    str_enable_disable(!disable), irq_pipe_mask,
+		    intel_de_read(display, CLKGATE_DIS_DSSDSC));
+}
+
 /*
  * Starting with Haswell, we have a "Power Down Well" that can be turned off
  * when not needed anymore. We have 4 registers that can request the power well
@@ -202,20 +244,25 @@ int intel_power_well_refcount(struct i915_power_well *power_well)
  * requesting it to be enabled.
  */
 static void hsw_power_well_post_enable(struct intel_display *display,
-				       u8 irq_pipe_mask, bool has_vga)
+				       u8 irq_pipe_mask)
 {
-	if (has_vga)
-		intel_vga_reset_io_mem(display);
-
-	if (irq_pipe_mask)
+	if (irq_pipe_mask) {
 		gen8_irq_power_well_post_enable(display, irq_pipe_mask);
+
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_22021048059))
+			dss_pipe_gating_enable_disable(display, irq_pipe_mask, false);
+	}
 }
 
 static void hsw_power_well_pre_disable(struct intel_display *display,
 				       u8 irq_pipe_mask)
 {
-	if (irq_pipe_mask)
+	if (irq_pipe_mask) {
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_22021048059))
+			dss_pipe_gating_enable_disable(display, irq_pipe_mask, true);
+
 		gen8_irq_power_well_pre_disable(display, irq_pipe_mask);
+	}
 }
 
 #define ICL_AUX_PW_TO_PHY(pw_idx)	\
@@ -418,8 +465,7 @@ static void hsw_power_well_enable(struct intel_display *display,
 	}
 
 	hsw_power_well_post_enable(display,
-				   power_well->desc->irq_pipe_mask,
-				   power_well->desc->has_vga);
+				   power_well->desc->irq_pipe_mask);
 }
 
 static void hsw_power_well_disable(struct intel_display *display,
@@ -522,7 +568,7 @@ static void icl_tc_cold_exit(struct intel_display *display)
 	int ret, tries = 0;
 
 	while (1) {
-		ret = intel_pcode_write(display->drm, ICL_PCODE_EXIT_TCCOLD, 0);
+		ret = intel_parent_pcode_write(display, ICL_PCODE_EXIT_TCCOLD, 0);
 		if (ret != -EAGAIN || ++tries == 3)
 			break;
 		msleep(1);
@@ -806,7 +852,7 @@ void gen9_set_dc_state(struct intel_display *display, u32 state)
 			power_domains->dc_state, val & mask);
 
 	enable_dc6 = state & DC_STATE_EN_UPTO_DC6;
-	dc6_was_enabled = val & DC_STATE_EN_UPTO_DC6;
+	dc6_was_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6;
 	if (!dc6_was_enabled && enable_dc6)
 		intel_dmc_update_dc6_allowed_count(display, true);
 
@@ -1230,7 +1276,7 @@ static void vlv_init_display_clock_gating(struct intel_display *display)
 	 * Disable trickle feed and enable pnd deadline calculation
 	 */
 	intel_de_write(display, MI_ARB_VLV,
-		       MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
+		       MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE_VLV);
 	intel_de_write(display, CBR1_VLV, 0);
 
 	drm_WARN_ON(display->drm, DISPLAY_RUNTIME_INFO(display)->rawclk_freq == 0);
@@ -1795,7 +1841,7 @@ tgl_tc_cold_request(struct intel_display *display, bool block)
 		 * Spec states that we should timeout the request after 200us
 		 * but the function below will timeout after 500us
 		 */
-		ret = intel_pcode_read(display->drm, TGL_PCODE_TCCOLD, &low_val, &high_val);
+		ret = intel_parent_pcode_read(display, TGL_PCODE_TCCOLD, &low_val, &high_val);
 		if (ret == 0) {
 			if (block &&
 			    (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.h b/drivers/gpu/drm/i915/display/intel_display_power_well.h
index ec8e508d0593d..8f5524da2d061 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.h
@@ -103,8 +103,6 @@ struct i915_power_well_desc {
 	 * the well enabled.
 	 */
 	u16 fixed_enable_delay:1;
-	/* The pw is backing the VGA functionality */
-	u16 has_vga:1;
 	u16 has_fuses:1;
 	/*
 	 * The pw is for an ICL+ TypeC PHY port in
diff --git a/drivers/gpu/drm/i915/display/intel_display_regs.h b/drivers/gpu/drm/i915/display/intel_display_regs.h
index 9e0d853f4b61a..4746e9ebd9200 100644
--- a/drivers/gpu/drm/i915/display/intel_display_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_regs.h
@@ -6,6 +6,9 @@
 
 #include "intel_display_reg_defs.h"
 
+#define GU_CNTL_PROTECTED		_MMIO(0x10100C)
+#define   DEPRESENT			REG_BIT(9)
+
 #define _GEN7_PIPEA_DE_LOAD_SL	0x70068
 #define _GEN7_PIPEB_DE_LOAD_SL	0x71068
 #define GEN7_PIPE_DE_LOAD_SL(pipe) _MMIO_PIPE(pipe, _GEN7_PIPEA_DE_LOAD_SL, _GEN7_PIPEB_DE_LOAD_SL)
@@ -79,6 +82,29 @@
 #define   DERRMR_PIPEC_VBLANK		(1 << 21)
 #define   DERRMR_PIPEC_HBLANK		(1 << 22)
 
+#define GEN7_ERR_INT	_MMIO(0x44040)
+#define   ERR_INT_POISON		(1 << 31)
+#define   ERR_INT_INVALID_GTT_PTE	(1 << 29)
+#define   ERR_INT_INVALID_PTE_DATA	(1 << 28)
+#define   ERR_INT_SPRITE_C_FAULT	(1 << 23)
+#define   ERR_INT_PRIMARY_C_FAULT	(1 << 22)
+#define   ERR_INT_CURSOR_C_FAULT	(1 << 21)
+#define   ERR_INT_SPRITE_B_FAULT	(1 << 20)
+#define   ERR_INT_PRIMARY_B_FAULT	(1 << 19)
+#define   ERR_INT_CURSOR_B_FAULT	(1 << 18)
+#define   ERR_INT_SPRITE_A_FAULT	(1 << 17)
+#define   ERR_INT_PRIMARY_A_FAULT	(1 << 16)
+#define   ERR_INT_CURSOR_A_FAULT	(1 << 15)
+#define   ERR_INT_MMIO_UNCLAIMED	(1 << 13)
+#define   ERR_INT_PIPE_CRC_DONE_C	(1 << 8)
+#define   ERR_INT_FIFO_UNDERRUN_C	(1 << 6)
+#define   ERR_INT_PIPE_CRC_DONE_B	(1 << 5)
+#define   ERR_INT_FIFO_UNDERRUN_B	(1 << 3)
+#define   ERR_INT_PIPE_CRC_DONE_A	(1 << 2)
+#define   ERR_INT_PIPE_CRC_DONE(pipe)	(1 << (2 + (pipe) * 3))
+#define   ERR_INT_FIFO_UNDERRUN_A	(1 << 0)
+#define   ERR_INT_FIFO_UNDERRUN(pipe)	(1 << ((pipe) * 3))
+
 #define VLV_IRQ_REGS		I915_IRQ_REGS(VLV_IMR, \
 					      VLV_IER, \
 					      VLV_IIR)
@@ -160,6 +186,47 @@
 
 #define   DPLL_FPA01_P1_POST_DIV_MASK_I830	0x001f0000
 
+#define DSPCLK_GATE_D			_MMIO(0x6200)
+#define VLV_DSPCLK_GATE_D		_MMIO(VLV_DISPLAY_BASE + 0x6200)
+# define DPUNIT_B_CLOCK_GATE_DISABLE		(1 << 30) /* 965 */
+# define VSUNIT_CLOCK_GATE_DISABLE		(1 << 29) /* 965 */
+# define VRHUNIT_CLOCK_GATE_DISABLE		(1 << 28) /* 965 */
+# define VRDUNIT_CLOCK_GATE_DISABLE		(1 << 27) /* 965 */
+# define AUDUNIT_CLOCK_GATE_DISABLE		(1 << 26) /* 965 */
+# define DPUNIT_A_CLOCK_GATE_DISABLE		(1 << 25) /* 965 */
+# define DPCUNIT_CLOCK_GATE_DISABLE		(1 << 24) /* 965 */
+# define PNV_GMBUSUNIT_CLOCK_GATE_DISABLE	(1 << 24) /* pnv */
+# define TVRUNIT_CLOCK_GATE_DISABLE		(1 << 23) /* 915-945 */
+# define TVCUNIT_CLOCK_GATE_DISABLE		(1 << 22) /* 915-945 */
+# define TVFUNIT_CLOCK_GATE_DISABLE		(1 << 21) /* 915-945 */
+# define TVEUNIT_CLOCK_GATE_DISABLE		(1 << 20) /* 915-945 */
+# define DVSUNIT_CLOCK_GATE_DISABLE		(1 << 19) /* 915-945 */
+# define DSSUNIT_CLOCK_GATE_DISABLE		(1 << 18) /* 915-945 */
+# define DDBUNIT_CLOCK_GATE_DISABLE		(1 << 17) /* 915-945 */
+# define DPRUNIT_CLOCK_GATE_DISABLE		(1 << 16) /* 915-945 */
+# define DPFUNIT_CLOCK_GATE_DISABLE		(1 << 15) /* 915-945 */
+# define DPBMUNIT_CLOCK_GATE_DISABLE		(1 << 14) /* 915-945 */
+# define DPLSUNIT_CLOCK_GATE_DISABLE		(1 << 13) /* 915-945 */
+# define DPLUNIT_CLOCK_GATE_DISABLE		(1 << 12) /* 915-945 */
+# define DPOUNIT_CLOCK_GATE_DISABLE		(1 << 11)
+# define DPBUNIT_CLOCK_GATE_DISABLE		(1 << 10)
+# define DCUNIT_CLOCK_GATE_DISABLE		(1 << 9)
+# define DPUNIT_CLOCK_GATE_DISABLE		(1 << 8)
+# define VRUNIT_CLOCK_GATE_DISABLE		(1 << 7) /* 915+: reserved */
+# define OVHUNIT_CLOCK_GATE_DISABLE		(1 << 6) /* 830-865 */
+# define DPIOUNIT_CLOCK_GATE_DISABLE		(1 << 6) /* 915-945 */
+# define OVFUNIT_CLOCK_GATE_DISABLE		(1 << 5)
+# define OVBUNIT_CLOCK_GATE_DISABLE		(1 << 4)
+/*
+ * This bit must be set on the 830 to prevent hangs when turning off the
+ * overlay scaler.
+ */
+# define OVRUNIT_CLOCK_GATE_DISABLE		(1 << 3)
+# define OVCUNIT_CLOCK_GATE_DISABLE		(1 << 2)
+# define OVUUNIT_CLOCK_GATE_DISABLE		(1 << 1)
+# define ZVUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 830 */
+# define OVLUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 845,865 */
+
 /* Additional CHV pll/phy registers */
 #define DPIO_PHY_STATUS			_MMIO(VLV_DISPLAY_BASE + 0x6240)
 #define   DPLL_PORTD_READY_MASK		(0xf)
@@ -281,6 +348,7 @@
 #define  FW_CSPWRDWNEN		(1 << 15)
 
 #define MI_ARB_VLV		_MMIO(VLV_DISPLAY_BASE + 0x6504)
+#define   MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE_VLV	(1 << 2)
 
 #define CZCLK_CDCLK_FREQ_RATIO	_MMIO(VLV_DISPLAY_BASE + 0x6508)
 #define   CDCLK_FREQ_SHIFT	4
@@ -311,6 +379,46 @@
 #define OGAMC1			_MMIO(0x30020)
 #define OGAMC0			_MMIO(0x30024)
 
+#define GEN8_CHICKEN_DCPR_1			_MMIO(0x46430)
+#define   _LATENCY_REPORTING_REMOVED_PIPE_D	REG_BIT(31)
+#define   SKL_SELECT_ALTERNATE_DC_EXIT		REG_BIT(30)
+#define   _LATENCY_REPORTING_REMOVED_PIPE_C	REG_BIT(25)
+#define   _LATENCY_REPORTING_REMOVED_PIPE_B	REG_BIT(24)
+#define   _LATENCY_REPORTING_REMOVED_PIPE_A	REG_BIT(23)
+#define   LATENCY_REPORTING_REMOVED(pipe)	_PICK((pipe), \
+						      _LATENCY_REPORTING_REMOVED_PIPE_A, \
+						      _LATENCY_REPORTING_REMOVED_PIPE_B, \
+						      _LATENCY_REPORTING_REMOVED_PIPE_C, \
+						      _LATENCY_REPORTING_REMOVED_PIPE_D)
+#define   ICL_DELAY_PMRSP			REG_BIT(22)
+#define   DISABLE_FLR_SRC			REG_BIT(15)
+#define   MASK_WAKEMEM				REG_BIT(13)
+#define   DDI_CLOCK_REG_ACCESS			REG_BIT(7)
+
+#define CHICKEN_PAR1_1		_MMIO(0x42080)
+#define   IGNORE_KVMR_PIPE_A		REG_BIT(23)
+#define   KBL_ARB_FILL_SPARE_22		REG_BIT(22)
+#define   DIS_RAM_BYPASS_PSR2_MAN_TRACK	REG_BIT(16)
+#define   SKL_DE_COMPRESSED_HASH_MODE	REG_BIT(15)
+#define   HSW_MASK_VBL_TO_PIPE_IN_SRD	REG_BIT(15) /* hsw/bdw */
+#define   FORCE_ARB_IDLE_PLANES		REG_BIT(14)
+#define   SKL_EDP_PSR_FIX_RDWRAP	REG_BIT(3)
+#define   IGNORE_PSR2_HW_TRACKING	REG_BIT(1)
+
+/*
+ * GEN9 clock gating regs
+ */
+#define GEN9_CLKGATE_DIS_0		_MMIO(0x46530)
+#define   DARBF_GATING_DIS		REG_BIT(27)
+#define   DMG_GATING_DIS		REG_BIT(21)
+#define   MTL_PIPEDMC_GATING_DIS(pipe)	REG_BIT(15 - (pipe))
+#define   PWM2_GATING_DIS		REG_BIT(14)
+#define   PWM1_GATING_DIS		REG_BIT(13)
+
+#define GEN9_CLKGATE_DIS_3		_MMIO(0x46538)
+#define   TGL_VRH_GATING_DIS		REG_BIT(31)
+#define   DPT_GATING_DIS		REG_BIT(22)
+
 #define GEN9_CLKGATE_DIS_4		_MMIO(0x4653C)
 #define   BXT_GMBUS_GATING_DIS		(1 << 14)
 #define   DG2_DPFC_GATING_DIS		REG_BIT(31)
@@ -1003,6 +1111,15 @@
 #define SWF3(dev_priv, i)	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x72414 + (i) * 4)
 #define SWF_ILK(i)	_MMIO(0x4F000 + (i) * 4)
 
+#define DEISR   _MMIO(0x44000)
+#define DEIMR   _MMIO(0x44004)
+#define DEIIR   _MMIO(0x44008)
+#define DEIER   _MMIO(0x4400c)
+
+#define DE_IRQ_REGS		I915_IRQ_REGS(DEIMR, \
+					      DEIER, \
+					      DEIIR)
+
 #define DIGITAL_PORT_HOTPLUG_CNTRL	_MMIO(0x44030)
 #define  DIGITAL_PORTA_HOTPLUG_ENABLE		(1 << 4)
 #define  DIGITAL_PORTA_PULSE_DURATION_2ms	(0 << 2) /* pre-HSW */
@@ -1333,6 +1450,44 @@
 						      GEN8_DE_PORT_IER, \
 						      GEN8_DE_PORT_IIR)
 
+/* interrupts */
+#define DE_MASTER_IRQ_CONTROL   (1 << 31)
+#define DE_SPRITEB_FLIP_DONE    (1 << 29)
+#define DE_SPRITEA_FLIP_DONE    (1 << 28)
+#define DE_PLANEB_FLIP_DONE     (1 << 27)
+#define DE_PLANEA_FLIP_DONE     (1 << 26)
+#define DE_PLANE_FLIP_DONE(plane) (1 << (26 + (plane)))
+#define DE_PCU_EVENT            (1 << 25)
+#define DE_GTT_FAULT            (1 << 24)
+#define DE_POISON               (1 << 23)
+#define DE_PERFORM_COUNTER      (1 << 22)
+#define DE_PCH_EVENT            (1 << 21)
+#define DE_AUX_CHANNEL_A        (1 << 20)
+#define DE_DP_A_HOTPLUG         (1 << 19)
+#define DE_GSE                  (1 << 18)
+#define DE_PIPEB_VBLANK         (1 << 15)
+#define DE_PIPEB_EVEN_FIELD     (1 << 14)
+#define DE_PIPEB_ODD_FIELD      (1 << 13)
+#define DE_PIPEB_LINE_COMPARE   (1 << 12)
+#define DE_PIPEB_VSYNC          (1 << 11)
+#define DE_PIPEB_CRC_DONE	(1 << 10)
+#define DE_PIPEB_FIFO_UNDERRUN  (1 << 8)
+#define DE_PIPEA_VBLANK         (1 << 7)
+#define DE_PIPE_VBLANK(pipe)    (1 << (7 + 8 * (pipe)))
+#define DE_PIPEA_EVEN_FIELD     (1 << 6)
+#define DE_PIPEA_ODD_FIELD      (1 << 5)
+#define DE_PIPEA_LINE_COMPARE   (1 << 4)
+#define DE_PIPEA_VSYNC          (1 << 3)
+#define DE_PIPEA_CRC_DONE	(1 << 2)
+#define DE_PIPE_CRC_DONE(pipe)	(1 << (2 + 8 * (pipe)))
+#define DE_PIPEA_FIFO_UNDERRUN  (1 << 0)
+#define DE_PIPE_FIFO_UNDERRUN(pipe)  (1 << (8 * (pipe)))
+
+/* Display Internal Timeout Register */
+#define RM_TIMEOUT		_MMIO(0x42060)
+#define RM_TIMEOUT_REG_CAPTURE	_MMIO(0x420E0)
+#define  MMIO_TIMEOUT_US(us)	((us) << 0)
+
 #define GEN8_DE_MISC_ISR _MMIO(0x44460)
 #define GEN8_DE_MISC_IMR _MMIO(0x44464)
 #define GEN8_DE_MISC_IIR _MMIO(0x44468)
@@ -1466,6 +1621,29 @@
 #define   CHICKEN_FBC_STRIDE_MASK	REG_GENMASK(12, 0)
 #define   CHICKEN_FBC_STRIDE(x)		REG_FIELD_PREP(CHICKEN_FBC_STRIDE_MASK, (x))
 
+#define _CHICKEN_PIPESL_1_A	0x420b0
+#define _CHICKEN_PIPESL_1_B	0x420b4
+#define CHICKEN_PIPESL_1(pipe)	_MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
+#define   HSW_PRI_STRETCH_MAX_MASK	REG_GENMASK(28, 27)
+#define   HSW_PRI_STRETCH_MAX_X8	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 0)
+#define   HSW_PRI_STRETCH_MAX_X4	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 1)
+#define   HSW_PRI_STRETCH_MAX_X2	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 2)
+#define   HSW_PRI_STRETCH_MAX_X1	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 3)
+#define   HSW_SPR_STRETCH_MAX_MASK	REG_GENMASK(26, 25)
+#define   HSW_SPR_STRETCH_MAX_X8	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 0)
+#define   HSW_SPR_STRETCH_MAX_X4	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 1)
+#define   HSW_SPR_STRETCH_MAX_X2	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 2)
+#define   HSW_SPR_STRETCH_MAX_X1	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 3)
+#define   HSW_FBCQ_DIS			REG_BIT(22)
+#define   HSW_UNMASK_VBL_TO_REGS_IN_SRD REG_BIT(15) /* hsw */
+#define   SKL_PSR_MASK_PLANE_FLIP	REG_BIT(11) /* skl+ */
+#define   SKL_PLANE1_STRETCH_MAX_MASK	REG_GENMASK(1, 0)
+#define   SKL_PLANE1_STRETCH_MAX_X8	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 0)
+#define   SKL_PLANE1_STRETCH_MAX_X4	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 1)
+#define   SKL_PLANE1_STRETCH_MAX_X2	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 2)
+#define   SKL_PLANE1_STRETCH_MAX_X1	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 3)
+#define   BDW_UNMASK_VBL_TO_REGS_IN_SRD	REG_BIT(0) /* bdw */
+
 #define _CHICKEN_TRANS_A	0x420c0
 #define _CHICKEN_TRANS_B	0x420c4
 #define _CHICKEN_TRANS_C	0x420c8
@@ -1552,6 +1730,11 @@
 #define   GLK_DFSM_DISPLAY_DSC_DISABLE	(1 << 7)
 #define   XE2LPD_DFSM_DBUF_OVERLAP_DISABLE	(1 << 3)
 
+#define GMD_ID_DISPLAY				_MMIO(0x510a0)
+#define   GMD_ID_DISPLAY_ARCH_MASK		REG_GENMASK(31, 22)
+#define   GMD_ID_DISPLAY_RELEASE_MASK		REG_GENMASK(21, 14)
+#define   GMD_ID_DISPLAY_STEP			REG_GENMASK(5, 0)
+
 #define XE2LPD_DE_CAP			_MMIO(0x41100)
 #define   XE2LPD_DE_CAP_3DLUT_MASK	REG_GENMASK(31, 30)
 #define   XE2LPD_DE_CAP_DSC_MASK	REG_GENMASK(29, 28)
@@ -1685,6 +1868,13 @@
 					 SDE_TC_HOTPLUG_ICP(HPD_PORT_TC2) | \
 					 SDE_TC_HOTPLUG_ICP(HPD_PORT_TC1))
 
+/* PCH */
+
+#define SDEISR  _MMIO(0xc4000)
+#define SDEIMR  _MMIO(0xc4004)
+#define SDEIIR  _MMIO(0xc4008)
+#define SDEIER  _MMIO(0xc400c)
+
 #define SDE_IRQ_REGS			I915_IRQ_REGS(SDEIMR, \
 						      SDEIER, \
 						      SDEIIR)
@@ -2021,6 +2211,28 @@
 #define  TRANS_BPC_6			REG_FIELD_PREP(TRANS_BPC_MASK, 2)
 #define  TRANS_BPC_12			REG_FIELD_PREP(TRANS_BPC_MASK, 3)
 
+/* Icelake PPS_DATA and _ECC DIP Registers.
+ * These are available for transcoders B,C and eDP.
+ * Adding the _A so as to reuse the _MMIO_TRANS2
+ * definition, with which it offsets to the right location.
+ */
+
+#define _TRANSA_CHICKEN1	 0xf0060
+#define _TRANSB_CHICKEN1	 0xf1060
+#define TRANS_CHICKEN1(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1)
+#define   TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE	REG_BIT(10)
+#define   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE	REG_BIT(4)
+
+#define _TRANSA_CHICKEN2	0xf0064
+#define _TRANSB_CHICKEN2	0xf1064
+#define TRANS_CHICKEN2(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2)
+#define   TRANS_CHICKEN2_TIMING_OVERRIDE		REG_BIT(31)
+#define   TRANS_CHICKEN2_FDI_POLARITY_REVERSED		REG_BIT(29)
+#define   TRANS_CHICKEN2_FRAME_START_DELAY_MASK		REG_GENMASK(28, 27)
+#define   TRANS_CHICKEN2_FRAME_START_DELAY(x)		REG_FIELD_PREP(TRANS_CHICKEN2_FRAME_START_DELAY_MASK, (x)) /* 0-3 */
+#define   TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER	REG_BIT(26)
+#define   TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH	REG_BIT(25)
+
 #define PCH_DP_B		_MMIO(0xe4100)
 #define PCH_DP_C		_MMIO(0xe4200)
 #define PCH_DP_D		_MMIO(0xe4300)
@@ -2211,6 +2423,13 @@
 #define   HSW_PWR_WELL_FORCE_ON			(1 << 19)
 #define HSW_PWR_WELL_CTL6			_MMIO(0x45414)
 
+/* clock gating DSS DSC disable register */
+#define CLKGATE_DIS_DSSDSC			_MMIO(0x46548)
+#define   DSS_PIPE_D_GATING_DISABLED		REG_BIT(31)
+#define   DSS_PIPE_C_GATING_DISABLED		REG_BIT(29)
+#define   DSS_PIPE_B_GATING_DISABLED		REG_BIT(27)
+#define   DSS_PIPE_A_GATING_DISABLED		REG_BIT(25)
+
 /* SKL Fuse Status */
 enum skl_power_gate {
 	SKL_PG0,
@@ -2854,6 +3073,42 @@ enum skl_power_gate {
 #define  SFUSE_STRAP_DDIC_DETECTED	(1 << 1)
 #define  SFUSE_STRAP_DDID_DETECTED	(1 << 0)
 
+#define SOUTH_CHICKEN1		_MMIO(0xc2000)
+#define  FDIA_PHASE_SYNC_SHIFT_OVR	19
+#define  FDIA_PHASE_SYNC_SHIFT_EN	18
+#define  INVERT_DDIE_HPD			REG_BIT(28)
+#define  INVERT_DDID_HPD_MTP			REG_BIT(27)
+#define  INVERT_TC4_HPD				REG_BIT(26)
+#define  INVERT_TC3_HPD				REG_BIT(25)
+#define  INVERT_TC2_HPD				REG_BIT(24)
+#define  INVERT_TC1_HPD				REG_BIT(23)
+#define  INVERT_DDID_HPD			(1 << 18)
+#define  INVERT_DDIC_HPD			(1 << 17)
+#define  INVERT_DDIB_HPD			(1 << 16)
+#define  INVERT_DDIA_HPD			(1 << 15)
+#define  FDI_PHASE_SYNC_OVR(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
+#define  FDI_PHASE_SYNC_EN(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
+#define  FDI_BC_BIFURCATION_SELECT	(1 << 12)
+#define  CHASSIS_CLK_REQ_DURATION_MASK	(0xf << 8)
+#define  CHASSIS_CLK_REQ_DURATION(x)	((x) << 8)
+#define  SBCLK_RUN_REFCLK_DIS		(1 << 7)
+#define  ICP_SECOND_PPS_IO_SELECT	REG_BIT(2)
+#define  SPT_PWM_GRANULARITY		(1 << 0)
+#define SOUTH_CHICKEN2		_MMIO(0xc2004)
+#define  FDI_MPHY_IOSFSB_RESET_STATUS	(1 << 13)
+#define  FDI_MPHY_IOSFSB_RESET_CTL	(1 << 12)
+#define  LPT_PWM_GRANULARITY		(1 << 5)
+#define  DPLS_EDP_PPS_FIX_DIS		(1 << 0)
+
+#define SOUTH_DSPCLK_GATE_D	_MMIO(0xc2020)
+#define  PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 31)
+#define  PCH_DPLUNIT_CLOCK_GATE_DISABLE (1 << 30)
+#define  PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1 << 29)
+#define  PCH_DPMGUNIT_CLOCK_GATE_DISABLE (1 << 15)
+#define  PCH_CPUNIT_CLOCK_GATE_DISABLE (1 << 14)
+#define  CNP_PWM_CGE_GATING_DISABLE (1 << 13)
+#define  PCH_LP_PARTITION_LEVEL_DISABLE  (1 << 12)
+
 /* Gen4+ Timestamp and Pipe Frame time stamp registers */
 #define GEN4_TIMESTAMP		_MMIO(0x2358)
 #define ILK_TIMESTAMP_HI	_MMIO(0x70070)
@@ -2940,6 +3195,12 @@ enum skl_power_gate {
 #define MTL_PIPE_CLKGATE_DIS2(pipe)		_MMIO_PIPE(pipe, _MTL_PIPE_CLKGATE_DIS2_A, _MTL_PIPE_CLKGATE_DIS2_B)
 #define   MTL_DPFC_GATING_DIS			REG_BIT(6)
 
+#define MTL_MEM_SS_INFO_GLOBAL			_MMIO(0x45700)
+#define   XE3P_ECC_IMPACTING_DE			REG_BIT(12)
+#define   MTL_N_OF_ENABLED_QGV_POINTS_MASK	REG_GENMASK(11, 8)
+#define   MTL_N_OF_POPULATED_CH_MASK		REG_GENMASK(7, 4)
+#define   MTL_DDR_TYPE_MASK			REG_GENMASK(3, 0)
+
 #define MTL_MEM_SS_INFO_QGV_POINT_OFFSET	0x45710
 #define MTL_MEM_SS_INFO_QGV_POINT_LOW(point)	_MMIO(MTL_MEM_SS_INFO_QGV_POINT_OFFSET + (point) * 8)
 #define   MTL_TRCD_MASK			REG_GENMASK(31, 24)
@@ -2950,6 +3211,11 @@ enum skl_power_gate {
 #define   MTL_TRAS_MASK			REG_GENMASK(16, 8)
 #define   MTL_TRDPRE_MASK		REG_GENMASK(7, 0)
 
-
+#define FW_BLC		_MMIO(0x20d8)
+#define FW_BLC2		_MMIO(0x20dc)
+#define FW_BLC_SELF	_MMIO(0x20e0) /* 915+ only */
+#define   FW_BLC_SELF_EN_MASK      REG_BIT(31)
+#define   FW_BLC_SELF_FIFO_MASK    REG_BIT(16) /* 945 only */
+#define   FW_BLC_SELF_EN           REG_BIT(15) /* 945 only */
 
 #endif /* __INTEL_DISPLAY_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_rps.c b/drivers/gpu/drm/i915/display/intel_display_rps.c
index b58281edc5636..2f82484588264 100644
--- a/drivers/gpu/drm/i915/display/intel_display_rps.c
+++ b/drivers/gpu/drm/i915/display/intel_display_rps.c
@@ -8,8 +8,8 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_vblank.h>
 
-#include "i915_reg.h"
 #include "intel_display_core.h"
+#include "intel_display_regs.h"
 #include "intel_display_irq.h"
 #include "intel_display_rps.h"
 #include "intel_display_types.h"
diff --git a/drivers/gpu/drm/i915/display/intel_display_snapshot.c b/drivers/gpu/drm/i915/display/intel_display_snapshot.c
index f650f15ad3947..7f423182aa29a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_snapshot.c
+++ b/drivers/gpu/drm/i915/display/intel_display_snapshot.c
@@ -19,7 +19,6 @@ struct intel_display_snapshot {
 	struct intel_display_device_info info;
 	struct intel_display_runtime_info runtime_info;
 	struct intel_display_params params;
-	struct intel_overlay_snapshot *overlay;
 	struct intel_dmc_snapshot *dmc;
 	struct intel_display_irq_snapshot *irq;
 };
@@ -41,7 +40,6 @@ struct intel_display_snapshot *intel_display_snapshot_capture(struct intel_displ
 	intel_display_params_copy(&snapshot->params);
 
 	snapshot->irq = intel_display_irq_snapshot_capture(display);
-	snapshot->overlay = intel_overlay_snapshot_capture(display);
 	snapshot->dmc = intel_dmc_snapshot_capture(display);
 
 	return snapshot;
@@ -61,7 +59,6 @@ void intel_display_snapshot_print(const struct intel_display_snapshot *snapshot,
 	intel_display_params_dump(&snapshot->params, display->drm->driver->name, p);
 
 	intel_display_irq_snapshot_print(snapshot->irq, p);
-	intel_overlay_snapshot_print(snapshot->overlay, p);
 	intel_dmc_snapshot_print(snapshot->dmc, p);
 }
 
@@ -73,7 +70,6 @@ void intel_display_snapshot_free(struct intel_display_snapshot *snapshot)
 	intel_display_params_free(&snapshot->params);
 
 	kfree(snapshot->irq);
-	kfree(snapshot->overlay);
 	kfree(snapshot->dmc);
 	kfree(snapshot);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 6b92f333e18bc..d3a9ace4c9d13 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -145,7 +145,7 @@ struct intel_framebuffer {
 		struct intel_fb_view remapped_view;
 	};
 
-	struct i915_address_space *dpt_vm;
+	struct intel_dpt *dpt;
 
 	unsigned int min_alignment;
 	unsigned int vtd_guard;
@@ -351,6 +351,7 @@ struct intel_vbt_panel_data {
 		bool low_vswing;
 		bool hobl;
 		bool dsc_disable;
+		bool pipe_joiner_enable;
 	} edp;
 
 	struct {
@@ -1162,6 +1163,7 @@ struct intel_crtc_state {
 	} dsi_pll;
 
 	int max_link_bpp_x16;	/* in 1/16 bpp units */
+	int max_pipe_bpp;	/* in 1 bpp units */
 	int pipe_bpp;		/* in 1 bpp units */
 	int min_hblank;
 	struct intel_link_m_n dp_m_n;
@@ -1186,6 +1188,7 @@ struct intel_crtc_state {
 	u32 dc3co_exitline;
 	u16 su_y_granularity;
 	u8 active_non_psr_pipes;
+	u8 entry_setup_frames;
 	const char *no_psr_reason;
 
 	/*
@@ -1333,10 +1336,13 @@ struct intel_crtc_state {
 		/* Only used for state computation, not read out from the HW. */
 		bool compression_enabled_on_link;
 		bool compression_enable;
-		int num_streams;
+		struct intel_dsc_slice_config {
+			int pipes_per_line;
+			int streams_per_pipe;
+			int slices_per_stream;
+		} slice_config;
 		/* Compressed Bpp in U6.4 format (first 4 bits for fractional part) */
 		u16 compressed_bpp_x16;
-		u8 slice_count;
 		struct drm_dsc_config config;
 	} dsc;
 
@@ -1791,6 +1797,7 @@ struct intel_dp {
 	int link_rate;
 	u8 lane_count;
 	u8 sink_count;
+	bool downstream_port_changed;
 	bool needs_modeset_retry;
 	bool use_max_params;
 	u8 dpcd[DP_RECEIVER_CAP_SIZE];
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index 581d943b9bdc3..081a4092cd13e 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -5,11 +5,11 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_core.h"
 #include "intel_display_regs.h"
 #include "intel_display_wa.h"
+#include "intel_step.h"
 
 static void gen11_display_wa_apply(struct intel_display *display)
 {
@@ -32,9 +32,17 @@ static void adlp_display_wa_apply(struct intel_display *display)
 	intel_de_rmw(display, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
 }
 
+static void xe3plpd_display_wa_apply(struct intel_display *display)
+{
+	/* Wa_22021451799 */
+	intel_de_rmw(display, GEN9_CLKGATE_DIS_0, 0, DMG_GATING_DIS);
+}
+
 void intel_display_wa_apply(struct intel_display *display)
 {
-	if (display->platform.alderlake_p)
+	if (DISPLAY_VER(display) == 35)
+		xe3plpd_display_wa_apply(display);
+	else if (display->platform.alderlake_p)
 		adlp_display_wa_apply(display);
 	else if (DISPLAY_VER(display) == 12)
 		xe_d_display_wa_apply(display);
@@ -62,22 +70,89 @@ static bool intel_display_needs_wa_16025573575(struct intel_display *display)
 bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa, const char *name)
 {
 	switch (wa) {
+	case INTEL_DISPLAY_WA_1409120013:
+		return IS_DISPLAY_VER(display, 11, 12);
+	case INTEL_DISPLAY_WA_1409767108:
+		return (display->platform.alderlake_s ||
+			(display->platform.rocketlake &&
+			 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0)));
 	case INTEL_DISPLAY_WA_13012396614:
-		return DISPLAY_VERx100(display) == 3000;
+		return DISPLAY_VERx100(display) == 3000 ||
+			DISPLAY_VERx100(display) == 3500;
+	case INTEL_DISPLAY_WA_14010477008:
+		return display->platform.dg1 || display->platform.rocketlake ||
+			(display->platform.tigerlake &&
+			 IS_DISPLAY_STEP(display, STEP_A0, STEP_D0));
+	case INTEL_DISPLAY_WA_14010480278:
+		return (IS_DISPLAY_VER(display, 10, 12));
+	case INTEL_DISPLAY_WA_14010547955:
+		return display->platform.dg2;
+	case INTEL_DISPLAY_WA_14010685332:
+		return INTEL_PCH_TYPE(display) >= PCH_CNP &&
+			INTEL_PCH_TYPE(display) < PCH_DG1;
+	case INTEL_DISPLAY_WA_14011294188:
+		return INTEL_PCH_TYPE(display) >= PCH_TGP &&
+			INTEL_PCH_TYPE(display) < PCH_DG1;
+	case INTEL_DISPLAY_WA_14011503030:
 	case INTEL_DISPLAY_WA_14011503117:
+	case INTEL_DISPLAY_WA_22012358565:
 		return DISPLAY_VER(display) == 13;
+	case INTEL_DISPLAY_WA_14011508470:
+		return (IS_DISPLAY_VERx100(display, 1200, 1300));
+	case INTEL_DISPLAY_WA_14011765242:
+		return display->platform.alderlake_s &&
+			IS_DISPLAY_STEP(display, STEP_A0, STEP_A2);
+	case INTEL_DISPLAY_WA_14014143976:
+		return IS_DISPLAY_STEP(display, STEP_E0, STEP_FOREVER);
+	case INTEL_DISPLAY_WA_14016740474:
+		return IS_DISPLAY_VERx100_STEP(display, 1400, STEP_A0, STEP_C0);
+	case INTEL_DISPLAY_WA_14020863754:
+		return DISPLAY_VERx100(display) == 3000 ||
+			DISPLAY_VERx100(display) == 2000 ||
+			DISPLAY_VERx100(display) == 1401;
 	case INTEL_DISPLAY_WA_14025769978:
 		return DISPLAY_VER(display) == 35;
+	case INTEL_DISPLAY_WA_15013987218:
+		return DISPLAY_VER(display) == 20;
 	case INTEL_DISPLAY_WA_15018326506:
 		return display->platform.battlemage;
+	case INTEL_DISPLAY_WA_16011303918:
+	case INTEL_DISPLAY_WA_22011320316:
+		return display->platform.alderlake_p &&
+			IS_DISPLAY_STEP(display, STEP_A0, STEP_B0);
+	case INTEL_DISPLAY_WA_16011181250:
+		return display->platform.rocketlake || display->platform.alderlake_s ||
+			display->platform.dg2;
+	case INTEL_DISPLAY_WA_16011342517:
+		return display->platform.alderlake_p &&
+			IS_DISPLAY_STEP(display, STEP_A0, STEP_D0);
+	case INTEL_DISPLAY_WA_16011863758:
+		return DISPLAY_VER(display) >= 11;
 	case INTEL_DISPLAY_WA_16023588340:
 		return intel_display_needs_wa_16023588340(display);
 	case INTEL_DISPLAY_WA_16025573575:
 		return intel_display_needs_wa_16025573575(display);
+	case INTEL_DISPLAY_WA_16025596647:
+		return DISPLAY_VER(display) == 20 &&
+			IS_DISPLAY_VERx100_STEP(display, 3000,
+						STEP_A0, STEP_B0);
+	case INTEL_DISPLAY_WA_18034343758:
+		return DISPLAY_VER(display) == 20 ||
+			(display->platform.pantherlake &&
+			 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0));
+	case INTEL_DISPLAY_WA_22010178259:
+		return DISPLAY_VER(display) == 12;
+	case INTEL_DISPLAY_WA_22010947358:
+		return display->platform.alderlake_p;
+	case INTEL_DISPLAY_WA_22012278275:
+		return display->platform.alderlake_p &&
+			IS_DISPLAY_STEP(display, STEP_A0, STEP_E0);
 	case INTEL_DISPLAY_WA_22014263786:
 		return IS_DISPLAY_VERx100(display, 1100, 1400);
+	case INTEL_DISPLAY_WA_22021048059:
+		return IS_DISPLAY_VER(display, 14, 35);
 	default:
-		drm_WARN(display->drm, 1, "Missing Wa number: %s\n", name);
+		drm_WARN(display->drm, 1, "Missing Wa: %s\n", name);
 		break;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.h b/drivers/gpu/drm/i915/display/intel_display_wa.h
index 40f989f19df17..15fec843f15e1 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.h
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.h
@@ -27,18 +27,44 @@ bool intel_display_needs_wa_16023588340(struct intel_display *display);
  * number.
  */
 enum intel_display_wa {
+	INTEL_DISPLAY_WA_1409120013,
+	INTEL_DISPLAY_WA_1409767108,
 	INTEL_DISPLAY_WA_13012396614,
+	INTEL_DISPLAY_WA_14010477008,
+	INTEL_DISPLAY_WA_14010480278,
+	INTEL_DISPLAY_WA_14010547955,
+	INTEL_DISPLAY_WA_14010685332,
+	INTEL_DISPLAY_WA_14011294188,
+	INTEL_DISPLAY_WA_14011503030,
 	INTEL_DISPLAY_WA_14011503117,
+	INTEL_DISPLAY_WA_14011508470,
+	INTEL_DISPLAY_WA_14011765242,
+	INTEL_DISPLAY_WA_14014143976,
+	INTEL_DISPLAY_WA_14016740474,
+	INTEL_DISPLAY_WA_14020863754,
 	INTEL_DISPLAY_WA_14025769978,
+	INTEL_DISPLAY_WA_15013987218,
 	INTEL_DISPLAY_WA_15018326506,
+	INTEL_DISPLAY_WA_16011181250,
+	INTEL_DISPLAY_WA_16011303918,
+	INTEL_DISPLAY_WA_16011342517,
+	INTEL_DISPLAY_WA_16011863758,
 	INTEL_DISPLAY_WA_16023588340,
 	INTEL_DISPLAY_WA_16025573575,
+	INTEL_DISPLAY_WA_16025596647,
+	INTEL_DISPLAY_WA_18034343758,
+	INTEL_DISPLAY_WA_22010178259,
+	INTEL_DISPLAY_WA_22010947358,
+	INTEL_DISPLAY_WA_22011320316,
+	INTEL_DISPLAY_WA_22012278275,
+	INTEL_DISPLAY_WA_22012358565,
 	INTEL_DISPLAY_WA_22014263786,
+	INTEL_DISPLAY_WA_22021048059,
 };
 
 bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa, const char *name);
 
 #define intel_display_wa(__display, __wa) \
-	__intel_display_wa((__display), INTEL_DISPLAY_WA_##__wa, __stringify(__wa))
+	__intel_display_wa((__display), __wa, __stringify(__wa))
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 1006b060c3f3a..1667a829e708a 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -29,7 +29,6 @@
 #include <drm/drm_file.h>
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display_power_well.h"
@@ -40,7 +39,6 @@
 #include "intel_dmc.h"
 #include "intel_dmc_regs.h"
 #include "intel_flipq.h"
-#include "intel_step.h"
 
 /**
  * DOC: DMC Firmware Support
@@ -419,15 +417,12 @@ bool intel_dmc_has_payload(struct intel_display *display)
 	return has_dmc_id_fw(display, DMC_FW_MAIN);
 }
 
-static const struct stepping_info *
-intel_get_stepping_info(struct intel_display *display,
-			struct stepping_info *si)
+static void initialize_stepping_info(struct intel_display *display, struct stepping_info *si)
 {
-	const char *step_name = intel_step_name(INTEL_DISPLAY_STEP(display));
+	const char *step_name = DISPLAY_RUNTIME_INFO(display)->step_name;
 
-	si->stepping = step_name[0];
-	si->substepping = step_name[1];
-	return si;
+	si->stepping = step_name[0] ?: '*';
+	si->substepping = step_name[1] ?: '*';
 }
 
 static void gen9_set_dc_state_debugmask(struct intel_display *display)
@@ -1275,8 +1270,7 @@ static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 	struct intel_css_header *css_header;
 	struct intel_package_header *package_header;
 	struct intel_dmc_header_base *dmc_header;
-	struct stepping_info display_info = { '*', '*'};
-	const struct stepping_info *si = intel_get_stepping_info(display, &display_info);
+	struct stepping_info si = {};
 	enum intel_dmc_id dmc_id;
 	u32 readcount = 0;
 	u32 r, offset;
@@ -1284,6 +1278,8 @@ static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 	if (!fw)
 		return -EINVAL;
 
+	initialize_stepping_info(display, &si);
+
 	/* Extract CSS Header information */
 	css_header = (struct intel_css_header *)fw->data;
 	r = parse_dmc_fw_css(dmc, css_header, fw->size);
@@ -1294,7 +1290,7 @@ static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 
 	/* Extract Package Header information */
 	package_header = (struct intel_package_header *)&fw->data[readcount];
-	r = parse_dmc_fw_package(dmc, package_header, si, fw->size - readcount);
+	r = parse_dmc_fw_package(dmc, package_header, &si, fw->size - readcount);
 	if (!r)
 		return -EINVAL;
 
@@ -1599,8 +1595,7 @@ static bool intel_dmc_get_dc6_allowed_count(struct intel_display *display, u32 *
 		return false;
 
 	mutex_lock(&power_domains->lock);
-	dc6_enabled = intel_de_read(display, DC_STATE_EN) &
-		      DC_STATE_EN_UPTO_DC6;
+	dc6_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6;
 	if (dc6_enabled)
 		intel_dmc_update_dc6_allowed_count(display, false);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 696edf40b2436..4955bd8b11d7a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -107,20 +107,6 @@
 /* Constants for DP DSC configurations */
 static const u8 valid_dsc_bpp[] = {6, 8, 10, 12, 15};
 
-/*
- * With Single pipe configuration, HW is capable of supporting maximum of:
- * 2 slices per line for ICL, BMG
- * 4 slices per line for other platforms.
- * For now consider a max of 2 slices per line, which works for all platforms.
- * With this we can have max of 4 DSC Slices per pipe.
- *
- * For higher resolutions where 12 slice support is required with
- * ultrajoiner, only then each pipe can support 3 slices.
- *
- * #TODO Split this better to use 4 slices/dsc engine where supported.
- */
-static const u8 valid_dsc_slicecount[] = {1, 2, 3, 4};
-
 /**
  * intel_dp_is_edp - is the given port attached to an eDP panel (either CPU or PCH)
  * @intel_dp: DP struct
@@ -508,11 +494,16 @@ bool intel_dp_has_joiner(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *encoder = &intel_dig_port->base;
+	struct intel_connector *connector = intel_dp->attached_connector;
 
 	/* eDP MSO is not compatible with joiner */
 	if (intel_dp->mso_link_count)
 		return false;
 
+	if (intel_dp_is_edp(intel_dp) &&
+	    !connector->panel.vbt.edp.pipe_joiner_enable)
+		return false;
+
 	return DISPLAY_VER(display) >= 12 ||
 		(DISPLAY_VER(display) == 11 &&
 		 encoder->port != PORT_A);
@@ -959,18 +950,24 @@ u32 get_max_compressed_bpp_with_joiner(struct intel_display *display,
 	return max_bpp;
 }
 
-u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
-				int mode_clock, int mode_hdisplay,
-				int num_joined_pipes)
+static int intel_dp_dsc_min_slice_count(const struct intel_connector *connector,
+					int mode_clock, int mode_hdisplay)
 {
 	struct intel_display *display = to_intel_display(connector);
-	u32 sink_slice_count_mask =
-		drm_dp_dsc_sink_slice_count_mask(connector->dp.dsc_dpcd, false);
-	u8 min_slice_count, i;
+	bool is_edp =
+		connector->base.connector_type == DRM_MODE_CONNECTOR_eDP;
+	int min_slice_count;
 	int max_slice_width;
 	int tp_rgb_yuv444;
 	int tp_yuv422_420;
 
+	/*
+	 * TODO: allow using less than the maximum number of slices
+	 * supported by the eDP sink, to allow using fewer DSC engines.
+	 */
+	if (is_edp)
+		return drm_dp_dsc_sink_max_slice_count(connector->dp.dsc_dpcd, true);
+
 	/*
 	 * TODO: Use the throughput value specific to the actual RGB/YUV
 	 * format of the output.
@@ -1011,7 +1008,7 @@ u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 	 * slice and VDSC engine, whenever we approach close enough to max CDCLK
 	 */
 	if (mode_clock >= ((display->cdclk.max_cdclk_freq * 85) / 100))
-		min_slice_count = max_t(u8, min_slice_count, 2);
+		min_slice_count = max(min_slice_count, 2);
 
 	max_slice_width = drm_dp_dsc_sink_max_slice_width(connector->dp.dsc_dpcd);
 	if (max_slice_width < DP_DSC_MIN_SLICE_WIDTH_VALUE) {
@@ -1021,39 +1018,64 @@ u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 		return 0;
 	}
 	/* Also take into account max slice width */
-	min_slice_count = max_t(u8, min_slice_count,
-				DIV_ROUND_UP(mode_hdisplay,
-					     max_slice_width));
+	min_slice_count = max(min_slice_count,
+			      DIV_ROUND_UP(mode_hdisplay, max_slice_width));
 
-	/* Find the closest match to the valid slice count values */
-	for (i = 0; i < ARRAY_SIZE(valid_dsc_slicecount); i++) {
-		u8 test_slice_count = valid_dsc_slicecount[i] * num_joined_pipes;
+	return min_slice_count;
+}
 
-		/*
-		 * 3 DSC Slices per pipe need 3 DSC engines, which is supported only
-		 * with Ultrajoiner only for some platforms.
-		 */
-		if (valid_dsc_slicecount[i] == 3 &&
-		    (!HAS_DSC_3ENGINES(display) || num_joined_pipes != 4))
+static bool
+intel_dp_dsc_get_slice_config(const struct intel_connector *connector,
+			      int mode_clock, int mode_hdisplay,
+			      int num_joined_pipes,
+			      struct intel_dsc_slice_config *config_ret)
+{
+	struct intel_display *display = to_intel_display(connector);
+	int min_slice_count =
+		intel_dp_dsc_min_slice_count(connector, mode_clock, mode_hdisplay);
+	bool is_edp =
+		connector->base.connector_type == DRM_MODE_CONNECTOR_eDP;
+	u32 sink_slice_count_mask =
+		drm_dp_dsc_sink_slice_count_mask(connector->dp.dsc_dpcd, is_edp);
+	int slices_per_pipe;
+
+	/*
+	 * Find the closest match to the valid slice count values
+	 *
+	 * Max HW DSC-per-pipe x slice-per-DSC (= slice-per-pipe) capability:
+	 * ICL:  2x2
+	 * BMG:  2x2, or for ultrajoined 4 pipes: 3x1
+	 * TGL+: 2x4 (TODO: Add support for this)
+	 *
+	 * TODO: Explore if it's worth increasing the number of slices (from 1
+	 * to 2 or 3), so that multiple VDSC engines can be used, thus
+	 * reducing the minimum CDCLK requirement, which in turn is determined
+	 * by the 1 pixel per clock VDSC engine throughput in
+	 * intel_vdsc_min_cdclk().
+	 */
+	for (slices_per_pipe = 1; slices_per_pipe <= 4; slices_per_pipe++) {
+		struct intel_dsc_slice_config config;
+		int slices_per_line;
+
+		if (!intel_dsc_get_slice_config(display,
+						num_joined_pipes, slices_per_pipe,
+						&config))
 			continue;
 
-		if (!(drm_dp_dsc_slice_count_to_mask(test_slice_count) &
+		slices_per_line = intel_dsc_line_slice_count(&config);
+
+		if (!(drm_dp_dsc_slice_count_to_mask(slices_per_line) &
 		      sink_slice_count_mask))
 			continue;
 
-		 /*
-		  * Bigjoiner needs small joiner to be enabled.
-		  * So there should be at least 2 dsc slices per pipe,
-		  * whenever bigjoiner is enabled.
-		  */
-		if (num_joined_pipes > 1 && valid_dsc_slicecount[i] < 2)
+		if (mode_hdisplay % slices_per_line)
 			continue;
 
-		if (mode_hdisplay % test_slice_count)
-			continue;
+		if (min_slice_count <= slices_per_line) {
+			*config_ret = config;
 
-		if (min_slice_count <= test_slice_count)
-			return test_slice_count;
+			return true;
+		}
 	}
 
 	/* Print slice count 1,2,4,..24 if bit#0,1,3,..23 is set in the mask. */
@@ -1064,7 +1086,21 @@ u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 		    min_slice_count,
 		    (int)BITS_PER_TYPE(sink_slice_count_mask), &sink_slice_count_mask);
 
-	return 0;
+	return false;
+}
+
+u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
+				int mode_clock, int mode_hdisplay,
+				int num_joined_pipes)
+{
+	struct intel_dsc_slice_config config;
+
+	if (!intel_dp_dsc_get_slice_config(connector,
+					   mode_clock, mode_hdisplay,
+					   num_joined_pipes, &config))
+		return 0;
+
+	return intel_dsc_line_slice_count(&config);
 }
 
 static bool source_can_output(struct intel_dp *intel_dp,
@@ -1335,44 +1371,9 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector,
 	return MODE_OK;
 }
 
-static
-bool intel_dp_needs_joiner(struct intel_dp *intel_dp,
-			   struct intel_connector *connector,
-			   int hdisplay, int clock,
-			   int num_joined_pipes)
+int intel_dp_max_hdisplay_per_pipe(struct intel_display *display)
 {
-	struct intel_display *display = to_intel_display(intel_dp);
-	int hdisplay_limit;
-
-	if (!intel_dp_has_joiner(intel_dp))
-		return false;
-
-	num_joined_pipes /= 2;
-
-	hdisplay_limit = DISPLAY_VER(display) >= 30 ? 6144 : 5120;
-
-	return clock > num_joined_pipes * display->cdclk.max_dotclk_freq ||
-	       hdisplay > num_joined_pipes * hdisplay_limit;
-}
-
-int intel_dp_num_joined_pipes(struct intel_dp *intel_dp,
-			      struct intel_connector *connector,
-			      int hdisplay, int clock)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-
-	if (connector->force_joined_pipes)
-		return connector->force_joined_pipes;
-
-	if (HAS_ULTRAJOINER(display) &&
-	    intel_dp_needs_joiner(intel_dp, connector, hdisplay, clock, 4))
-		return 4;
-
-	if ((HAS_BIGJOINER(display) || HAS_UNCOMPRESSED_JOINER(display)) &&
-	    intel_dp_needs_joiner(intel_dp, connector, hdisplay, clock, 2))
-		return 2;
-
-	return 1;
+	return DISPLAY_VER(display) >= 30 ? 6144 : 5120;
 }
 
 bool intel_dp_has_dsc(const struct intel_connector *connector)
@@ -1395,6 +1396,51 @@ bool intel_dp_has_dsc(const struct intel_connector *connector)
 	return true;
 }
 
+static
+bool intel_dp_can_join(struct intel_dp *intel_dp,
+		       int num_joined_pipes)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	if (num_joined_pipes > 1 && !intel_dp_has_joiner(intel_dp))
+		return false;
+
+	switch (num_joined_pipes) {
+	case 1:
+		return true;
+	case 2:
+		return HAS_BIGJOINER(display) ||
+		       HAS_UNCOMPRESSED_JOINER(display);
+	case 4:
+		return HAS_ULTRAJOINER(display);
+	default:
+		return false;
+	}
+}
+
+bool intel_dp_dotclk_valid(struct intel_display *display,
+			   int target_clock,
+			   int htotal,
+			   int dsc_slice_count,
+			   int num_joined_pipes)
+{
+	int max_dotclk = display->cdclk.max_dotclk_freq;
+	int effective_dotclk_limit;
+
+	effective_dotclk_limit = max_dotclk * num_joined_pipes;
+
+	if (dsc_slice_count)
+		target_clock = intel_dsc_get_pixel_rate_with_dsc_bubbles(display,
+									 target_clock,
+									 htotal,
+									 dsc_slice_count);
+	else
+		effective_dotclk_limit =
+			intel_max_uncompressed_dotclock(display) * num_joined_pipes;
+
+	return target_clock <= effective_dotclk_limit;
+}
+
 static enum drm_mode_status
 intel_dp_mode_valid(struct drm_connector *_connector,
 		    const struct drm_display_mode *mode)
@@ -1406,9 +1452,7 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	const struct drm_display_mode *fixed_mode;
 	int target_clock = mode->clock;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
-	int max_dotclk = display->cdclk.max_dotclk_freq;
 	u16 dsc_max_compressed_bpp = 0;
-	u8 dsc_slice_count = 0;
 	enum drm_mode_status status;
 	bool dsc = false;
 	int num_joined_pipes;
@@ -1424,6 +1468,9 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	if (mode->clock < 10000)
 		return MODE_CLOCK_LOW;
 
+	if (intel_dp_hdisplay_bad(display, mode->hdisplay))
+		return MODE_H_ILLEGAL;
+
 	fixed_mode = intel_panel_fixed_mode(connector, mode);
 	if (intel_dp_is_edp(intel_dp) && fixed_mode) {
 		status = intel_panel_mode_valid(connector, mode);
@@ -1433,23 +1480,9 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 		target_clock = fixed_mode->clock;
 	}
 
-	num_joined_pipes = intel_dp_num_joined_pipes(intel_dp, connector,
-						     mode->hdisplay, target_clock);
-	max_dotclk *= num_joined_pipes;
-
 	sink_format = intel_dp_sink_format(connector, mode);
 	output_format = intel_dp_output_format(connector, sink_format);
 
-	status = intel_pfit_mode_valid(display, mode, output_format, num_joined_pipes);
-	if (status != MODE_OK)
-		return status;
-
-	if (target_clock > max_dotclk)
-		return MODE_CLOCK_HIGH;
-
-	if (intel_dp_hdisplay_bad(display, mode->hdisplay))
-		return MODE_H_ILLEGAL;
-
 	max_link_clock = intel_dp_max_link_rate(intel_dp);
 	max_lanes = intel_dp_max_lane_count(intel_dp);
 
@@ -1460,52 +1493,92 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 					   target_clock, mode->hdisplay,
 					   link_bpp_x16, 0);
 
-	if (intel_dp_has_dsc(connector)) {
-		int pipe_bpp;
+	/*
+	 * We cannot determine the required pipe‑join count before knowing whether
+	 * DSC is needed, nor can we determine DSC need without knowing the pipe
+	 * count.
+	 * Because of this dependency cycle, the only correct approach is to iterate
+	 * over candidate pipe counts and evaluate each combination.
+	 */
+	status = MODE_CLOCK_HIGH;
+	for_each_joiner_candidate(connector, mode, num_joined_pipes) {
+		int dsc_slice_count = 0;
+
+		status = intel_pfit_mode_valid(display, mode, output_format, num_joined_pipes);
+		if (status != MODE_OK)
+			continue;
 
-		/*
-		 * TBD pass the connector BPC,
-		 * for now U8_MAX so that max BPC on that platform would be picked
-		 */
-		pipe_bpp = intel_dp_dsc_compute_max_bpp(connector, U8_MAX);
+		if (intel_dp_has_dsc(connector)) {
+			int pipe_bpp;
 
-		/*
-		 * Output bpp is stored in 6.4 format so right shift by 4 to get the
-		 * integer value since we support only integer values of bpp.
-		 */
-		if (intel_dp_is_edp(intel_dp)) {
-			dsc_max_compressed_bpp =
-				drm_edp_dsc_sink_output_bpp(connector->dp.dsc_dpcd) >> 4;
-			dsc_slice_count =
-				drm_dp_dsc_sink_max_slice_count(connector->dp.dsc_dpcd,
-								true);
-			dsc = dsc_max_compressed_bpp && dsc_slice_count;
-		} else if (drm_dp_sink_supports_fec(connector->dp.fec_capability)) {
-			unsigned long bw_overhead_flags = 0;
-
-			if (!drm_dp_is_uhbr_rate(max_link_clock))
-				bw_overhead_flags |= DRM_DP_BW_OVERHEAD_FEC;
-
-			dsc = intel_dp_mode_valid_with_dsc(connector,
-							   max_link_clock, max_lanes,
-							   target_clock, mode->hdisplay,
-							   num_joined_pipes,
-							   output_format, pipe_bpp,
-							   bw_overhead_flags);
+			dsc_slice_count = intel_dp_dsc_get_slice_count(connector,
+								       target_clock,
+								       mode->hdisplay,
+								       num_joined_pipes);
+
+			/*
+			 * TBD pass the connector BPC,
+			 * for now U8_MAX so that max BPC on that platform would be picked
+			 */
+			pipe_bpp = intel_dp_dsc_compute_max_bpp(connector, U8_MAX);
+
+			/*
+			 * Output bpp is stored in 6.4 format so right shift by 4 to get the
+			 * integer value since we support only integer values of bpp.
+			 */
+			if (intel_dp_is_edp(intel_dp)) {
+				dsc_max_compressed_bpp =
+					drm_edp_dsc_sink_output_bpp(connector->dp.dsc_dpcd) >> 4;
+
+				dsc = dsc_max_compressed_bpp && dsc_slice_count;
+			} else if (drm_dp_sink_supports_fec(connector->dp.fec_capability)) {
+				unsigned long bw_overhead_flags = 0;
+
+				if (!drm_dp_is_uhbr_rate(max_link_clock))
+					bw_overhead_flags |= DRM_DP_BW_OVERHEAD_FEC;
+
+				dsc = intel_dp_mode_valid_with_dsc(connector,
+								   max_link_clock, max_lanes,
+								   target_clock, mode->hdisplay,
+								   num_joined_pipes,
+								   output_format, pipe_bpp,
+								   bw_overhead_flags);
+			}
 		}
-	}
 
-	if (intel_dp_joiner_needs_dsc(display, num_joined_pipes) && !dsc)
-		return MODE_CLOCK_HIGH;
+		if (intel_dp_joiner_needs_dsc(display, num_joined_pipes) && !dsc) {
+			status = MODE_CLOCK_HIGH;
+			continue;
+		}
 
-	if (mode_rate > max_rate && !dsc)
-		return MODE_CLOCK_HIGH;
+		if (mode_rate > max_rate && !dsc) {
+			status = MODE_CLOCK_HIGH;
+			continue;
+		}
+
+		status = intel_mode_valid_max_plane_size(display, mode, num_joined_pipes);
+		if (status != MODE_OK)
+			continue;
+
+		if (!dsc)
+			dsc_slice_count = 0;
+
+		if (!intel_dp_dotclk_valid(display,
+					   target_clock,
+					   mode->htotal,
+					   dsc_slice_count,
+					   num_joined_pipes)) {
+			status = MODE_CLOCK_HIGH;
+			continue;
+		}
+
+		break;
+	}
 
-	status = intel_dp_mode_valid_downstream(connector, mode, target_clock);
 	if (status != MODE_OK)
 		return status;
 
-	return intel_mode_valid_max_plane_size(display, mode, num_joined_pipes);
+	return intel_dp_mode_valid_downstream(connector, mode, target_clock);
 }
 
 bool intel_dp_source_supports_tps3(struct intel_display *display)
@@ -1696,7 +1769,7 @@ static int intel_dp_max_bpp(struct intel_dp *intel_dp,
 	struct intel_connector *connector = intel_dp->attached_connector;
 	int bpp, bpc;
 
-	bpc = crtc_state->pipe_bpp / 3;
+	bpc = crtc_state->max_pipe_bpp / 3;
 
 	if (intel_dp->dfp.max_bpc)
 		bpc = min_t(int, bpc, intel_dp->dfp.max_bpc);
@@ -2032,12 +2105,14 @@ static int dsc_compute_link_config(struct intel_dp *intel_dp,
 			} else {
 				unsigned long bw_overhead_flags =
 					pipe_config->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0;
+				int line_slice_count =
+					intel_dsc_line_slice_count(&pipe_config->dsc.slice_config);
 
 				if (!is_bw_sufficient_for_dsc_config(intel_dp,
 								     link_rate, lane_count,
 								     adjusted_mode->crtc_clock,
 								     adjusted_mode->hdisplay,
-								     pipe_config->dsc.slice_count,
+								     line_slice_count,
 								     dsc_bpp_x16,
 								     bw_overhead_flags))
 					continue;
@@ -2344,6 +2419,17 @@ bool intel_dp_needs_8b10b_fec(const struct intel_crtc_state *crtc_state,
 	return dsc_enabled_on_crtc || intel_dsc_enabled_on_link(crtc_state);
 }
 
+void intel_dp_dsc_reset_config(struct intel_crtc_state *crtc_state)
+{
+	crtc_state->fec_enable = false;
+
+	crtc_state->dsc.compression_enable = false;
+	crtc_state->dsc.compressed_bpp_x16 = 0;
+
+	memset(&crtc_state->dsc.slice_config, 0, sizeof(crtc_state->dsc.slice_config));
+	memset(&crtc_state->dsc.config, 0, sizeof(crtc_state->dsc.config));
+}
+
 int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 				struct intel_crtc_state *pipe_config,
 				struct drm_connector_state *conn_state,
@@ -2382,47 +2468,10 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 		}
 	}
 
-	/* Calculate Slice count */
-	if (intel_dp_is_edp(intel_dp)) {
-		pipe_config->dsc.slice_count =
-			drm_dp_dsc_sink_max_slice_count(connector->dp.dsc_dpcd,
-							true);
-		if (!pipe_config->dsc.slice_count) {
-			drm_dbg_kms(display->drm,
-				    "Unsupported Slice Count %d\n",
-				    pipe_config->dsc.slice_count);
-			return -EINVAL;
-		}
-	} else {
-		u8 dsc_dp_slice_count;
-
-		dsc_dp_slice_count =
-			intel_dp_dsc_get_slice_count(connector,
-						     adjusted_mode->crtc_clock,
-						     adjusted_mode->crtc_hdisplay,
-						     num_joined_pipes);
-		if (!dsc_dp_slice_count) {
-			drm_dbg_kms(display->drm,
-				    "Compressed Slice Count not supported\n");
-			return -EINVAL;
-		}
-
-		pipe_config->dsc.slice_count = dsc_dp_slice_count;
-	}
-	/*
-	 * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate
-	 * is greater than the maximum Cdclock and if slice count is even
-	 * then we need to use 2 VDSC instances.
-	 * In case of Ultrajoiner along with 12 slices we need to use 3
-	 * VDSC instances.
-	 */
-	if (pipe_config->joiner_pipes && num_joined_pipes == 4 &&
-	    pipe_config->dsc.slice_count == 12)
-		pipe_config->dsc.num_streams = 3;
-	else if (pipe_config->joiner_pipes || pipe_config->dsc.slice_count > 1)
-		pipe_config->dsc.num_streams = 2;
-	else
-		pipe_config->dsc.num_streams = 1;
+	if (!intel_dp_dsc_get_slice_config(connector, adjusted_mode->crtc_clock,
+					   adjusted_mode->crtc_hdisplay, num_joined_pipes,
+					   &pipe_config->dsc.slice_config))
+		return -EINVAL;
 
 	ret = intel_dp_dsc_compute_params(connector, pipe_config);
 	if (ret < 0) {
@@ -2440,7 +2489,7 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 		    "Compressed Bpp = " FXP_Q4_FMT " Slice Count = %d\n",
 		    pipe_config->pipe_bpp,
 		    FXP_Q4_ARGS(pipe_config->dsc.compressed_bpp_x16),
-		    pipe_config->dsc.slice_count);
+		    intel_dsc_line_slice_count(&pipe_config->dsc.slice_config));
 
 	return 0;
 }
@@ -2476,8 +2525,8 @@ dsc_throughput_quirk_max_bpp_x16(const struct intel_connector *connector,
 	return fxp_q4_from_int(12);
 }
 
-static int compute_min_compressed_bpp_x16(struct intel_connector *connector,
-					  enum intel_output_format output_format)
+int intel_dp_compute_min_compressed_bpp_x16(struct intel_connector *connector,
+					    enum intel_output_format output_format)
 {
 	int dsc_src_min_bpp, dsc_sink_min_bpp, dsc_min_bpp;
 	int min_bpp_x16;
@@ -2543,7 +2592,8 @@ bool intel_dp_mode_valid_with_dsc(struct intel_connector *connector,
 				  int pipe_bpp, unsigned long bw_overhead_flags)
 {
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int min_bpp_x16 = compute_min_compressed_bpp_x16(connector, output_format);
+	int min_bpp_x16 = intel_dp_compute_min_compressed_bpp_x16(connector,
+								  output_format);
 	int max_bpp_x16 = compute_max_compressed_bpp_x16(connector,
 							 mode_clock, mode_hdisplay,
 							 num_joined_pipes,
@@ -2597,7 +2647,8 @@ intel_dp_compute_config_link_bpp_limits(struct intel_connector *connector,
 		limits->link.min_bpp_x16 = fxp_q4_from_int(limits->pipe.min_bpp);
 	} else {
 		limits->link.min_bpp_x16 =
-			compute_min_compressed_bpp_x16(connector, crtc_state->output_format);
+			intel_dp_compute_min_compressed_bpp_x16(connector,
+								crtc_state->output_format);
 
 		max_link_bpp_x16 =
 			compute_max_compressed_bpp_x16(connector,
@@ -2691,7 +2742,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp,
 		 * previously. This hack should be removed once we have the
 		 * proper retry logic in place.
 		 */
-		limits->pipe.max_bpp = min(crtc_state->pipe_bpp, 24);
+		limits->pipe.max_bpp = min(crtc_state->max_pipe_bpp, 24);
 	} else {
 		limits->pipe.max_bpp = intel_dp_max_bpp(intel_dp, crtc_state,
 							respect_downstream_limits);
@@ -2710,9 +2761,39 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp,
 								     crtc_state)));
 	}
 
+	if (limits->pipe.min_bpp <= 0 ||
+	    limits->pipe.min_bpp > limits->pipe.max_bpp) {
+		drm_dbg_kms(display->drm, "[CONNECTOR:%d:%s] Invalid pipe bpp range: %d-%d\n",
+			    connector->base.base.id, connector->base.name,
+			    limits->pipe.min_bpp, limits->pipe.max_bpp);
+
+		return false;
+	}
+
 	if (dsc && !intel_dp_dsc_compute_pipe_bpp_limits(connector, limits))
 		return false;
 
+	/*
+	 * crtc_state->pipe_bpp is the non-DP specific baseline (platform /
+	 * EDID) maximum pipe BPP limited by the max-BPC connector property
+	 * request. Since by now pipe.max_bpp is <= the above baseline
+	 * maximum BPP, the only remaining reason for adjusting pipe.max_bpp
+	 * is the max-BPC connector property request. Adjust pipe.max_bpp to
+	 * this request within the current valid pipe.min_bpp .. pipe.max_bpp
+	 * range.
+	 */
+	limits->pipe.max_bpp = clamp(crtc_state->pipe_bpp, limits->pipe.min_bpp,
+				     limits->pipe.max_bpp);
+	if (dsc)
+		limits->pipe.max_bpp = align_max_sink_dsc_input_bpp(connector,
+								    limits->pipe.max_bpp);
+
+	if (limits->pipe.max_bpp != crtc_state->pipe_bpp)
+		drm_dbg_kms(display->drm,
+			    "[CONNECTOR:%d:%s] Adjusting requested max pipe bpp %d -> %d\n",
+			    connector->base.base.id, connector->base.name,
+			    crtc_state->pipe_bpp, limits->pipe.max_bpp);
+
 	if (is_mst || intel_dp->use_max_params) {
 		/*
 		 * For MST we always configure max link bw - the spec doesn't
@@ -2764,13 +2845,13 @@ bool intel_dp_joiner_needs_dsc(struct intel_display *display,
 }
 
 static int
-intel_dp_compute_link_config(struct intel_encoder *encoder,
-			     struct intel_crtc_state *pipe_config,
-			     struct drm_connector_state *conn_state,
-			     bool respect_downstream_limits)
+intel_dp_compute_link_for_joined_pipes(struct intel_encoder *encoder,
+				       struct intel_crtc_state *pipe_config,
+				       struct drm_connector_state *conn_state,
+				       bool respect_downstream_limits)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
+	int num_joined_pipes = intel_crtc_num_joined_pipes(pipe_config);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
@@ -2778,18 +2859,9 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct link_config_limits limits;
 	bool dsc_needed, joiner_needs_dsc;
-	int num_joined_pipes;
 	int ret = 0;
 
-	if (pipe_config->fec_enable &&
-	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
-		return -EINVAL;
-
-	num_joined_pipes = intel_dp_num_joined_pipes(intel_dp, connector,
-						     adjusted_mode->crtc_hdisplay,
-						     adjusted_mode->crtc_clock);
-	if (num_joined_pipes > 1)
-		pipe_config->joiner_pipes = GENMASK(crtc->pipe + num_joined_pipes - 1, crtc->pipe);
+	intel_dp_dsc_reset_config(pipe_config);
 
 	joiner_needs_dsc = intel_dp_joiner_needs_dsc(display, num_joined_pipes);
 
@@ -2813,7 +2885,13 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 							     fxp_q4_from_int(pipe_config->pipe_bpp),
 							     fxp_q4_from_int(pipe_config->pipe_bpp),
 							     0, false);
-		if (ret)
+
+		if (ret ||
+		    !intel_dp_dotclk_valid(display,
+					   adjusted_mode->crtc_clock,
+					   adjusted_mode->crtc_htotal,
+					   0,
+					   num_joined_pipes))
 			dsc_needed = true;
 	}
 
@@ -2823,6 +2901,8 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	}
 
 	if (dsc_needed) {
+		int dsc_slice_count;
+
 		drm_dbg_kms(display->drm,
 			    "Try DSC (fallback=%s, joiner=%s, force=%s)\n",
 			    str_yes_no(ret), str_yes_no(joiner_needs_dsc),
@@ -2838,6 +2918,15 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 						  conn_state, &limits, 64);
 		if (ret < 0)
 			return ret;
+
+		dsc_slice_count = intel_dsc_line_slice_count(&pipe_config->dsc.slice_config);
+
+		if (!intel_dp_dotclk_valid(display,
+					   adjusted_mode->crtc_clock,
+					   adjusted_mode->crtc_htotal,
+					   dsc_slice_count,
+					   num_joined_pipes))
+			return -EINVAL;
 	}
 
 	drm_dbg_kms(display->drm,
@@ -2854,6 +2943,55 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	return 0;
 }
 
+static int
+intel_dp_compute_link_config(struct intel_encoder *encoder,
+			     struct intel_crtc_state *crtc_state,
+			     struct drm_connector_state *conn_state,
+			     bool respect_downstream_limits)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	int num_joined_pipes;
+	int ret = -EINVAL;
+
+	if (crtc_state->fec_enable &&
+	    !intel_dp_supports_fec(intel_dp, connector, crtc_state))
+		return -EINVAL;
+
+	for_each_joiner_candidate(connector, adjusted_mode, num_joined_pipes) {
+		/*
+		 * NOTE:
+		 * The crtc_state->joiner_pipes should have been set at the end
+		 * only if all the conditions are met. However that would mean
+		 * that num_joined_pipes is passed around to all helpers and
+		 * make them use it instead of using crtc_state->joiner_pipes
+		 * directly or indirectly (via intel_crtc_num_joined_pipes()).
+		 *
+		 * For now, setting crtc_state->joiner_pipes to the candidate
+		 * value to avoid the above churn and resetting it to 0, in case
+		 * no joiner candidate is found to be suitable for the given
+		 * configuration.
+		 */
+		if (num_joined_pipes > 1)
+			crtc_state->joiner_pipes = GENMASK(crtc->pipe + num_joined_pipes - 1,
+							   crtc->pipe);
+
+		ret = intel_dp_compute_link_for_joined_pipes(encoder, crtc_state, conn_state,
+							     respect_downstream_limits);
+		if (ret == 0 || ret == -EDEADLK)
+			break;
+	}
+
+	if (ret < 0)
+		crtc_state->joiner_pipes = 0;
+
+	return ret;
+}
+
 bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state,
 				  const struct drm_connector_state *conn_state)
 {
@@ -4293,20 +4431,24 @@ static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp)
 	return dprx & DP_VSC_SDP_EXT_FOR_COLORIMETRY_SUPPORTED;
 }
 
-static void intel_dp_read_dsc_dpcd(struct drm_dp_aux *aux,
-				   u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE])
+static int intel_dp_read_dsc_dpcd(struct drm_dp_aux *aux,
+				  u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE])
 {
-	if (drm_dp_dpcd_read(aux, DP_DSC_SUPPORT, dsc_dpcd,
-			     DP_DSC_RECEIVER_CAP_SIZE) < 0) {
-		drm_err(aux->drm_dev,
-			"Failed to read DPCD register 0x%x\n",
-			DP_DSC_SUPPORT);
-		return;
+	int ret;
+
+	ret = drm_dp_dpcd_read_data(aux, DP_DSC_SUPPORT, dsc_dpcd,
+				    DP_DSC_RECEIVER_CAP_SIZE);
+	if (ret) {
+		drm_dbg_kms(aux->drm_dev,
+			    "Could not read DSC DPCD register 0x%x Error: %pe\n",
+			    DP_DSC_SUPPORT, ERR_PTR(ret));
+		return ret;
 	}
 
 	drm_dbg_kms(aux->drm_dev, "DSC DPCD: %*ph\n",
 		    DP_DSC_RECEIVER_CAP_SIZE,
 		    dsc_dpcd);
+	return 0;
 }
 
 static void init_dsc_overall_throughput_limits(struct intel_connector *connector, bool is_branch)
@@ -4357,8 +4499,9 @@ void intel_dp_get_dsc_sink_cap(u8 dpcd_rev,
 	if (dpcd_rev < DP_DPCD_REV_14)
 		return;
 
-	intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux,
-			       connector->dp.dsc_dpcd);
+	if (intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux,
+				   connector->dp.dsc_dpcd) < 0)
+		return;
 
 	if (drm_dp_dpcd_readb(connector->dp.dsc_decompression_aux, DP_FEC_CAPABILITY,
 			      &connector->dp.fec_capability) < 0) {
@@ -4388,7 +4531,9 @@ static void intel_edp_get_dsc_sink_cap(u8 edp_dpcd_rev, struct intel_connector *
 	if (edp_dpcd_rev < DP_EDP_14)
 		return;
 
-	intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux, connector->dp.dsc_dpcd);
+	if (intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux,
+				   connector->dp.dsc_dpcd) < 0)
+		return;
 
 	if (connector->dp.dsc_dpcd[0] & DP_DSC_DECOMPRESSION_IS_SUPPORTED)
 		init_dsc_overall_throughput_limits(connector, false);
@@ -4791,6 +4936,24 @@ intel_dp_mst_disconnect(struct intel_dp *intel_dp)
 	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst.mgr, intel_dp->is_mst);
 }
 
+#define INTEL_DP_DEVICE_SERVICE_IRQ_MASK_SST	(DP_AUTOMATED_TEST_REQUEST | \
+						 DP_CP_IRQ | \
+						 DP_SINK_SPECIFIC_IRQ)
+
+#define INTEL_DP_DEVICE_SERVICE_IRQ_MASK_MST	(DP_CP_IRQ | \
+						 DP_DOWN_REP_MSG_RDY | \
+						 DP_UP_REQ_MSG_RDY)
+
+#define INTEL_DP_LINK_SERVICE_IRQ_MASK_SST	(RX_CAP_CHANGED | \
+						 LINK_STATUS_CHANGED | \
+						 HDMI_LINK_STATUS_CHANGED | \
+						 CONNECTED_OFF_ENTRY_REQUESTED | \
+						 DP_TUNNELING_IRQ)
+
+#define INTEL_DP_LINK_SERVICE_IRQ_MASK_MST	(RX_CAP_CHANGED | \
+						 LINK_STATUS_CHANGED | \
+						 DP_TUNNELING_IRQ)
+
 static bool
 intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *esi)
 {
@@ -4827,6 +4990,79 @@ static bool intel_dp_ack_sink_irq_esi(struct intel_dp *intel_dp, u8 esi[4])
 	return false;
 }
 
+/* Return %true if reading the ESI vector succeeded, %false otherwise. */
+static bool intel_dp_get_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
+{
+	memset(esi, 0, 4);
+
+	/*
+	 * TODO: For DP_DPCD_REV >= 0x12 read
+	 * DP_SINK_COUNT_ESI and DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0.
+	 */
+	if (drm_dp_dpcd_read_data(&intel_dp->aux, DP_SINK_COUNT, esi, 2) != 0)
+		return false;
+
+	if (intel_dp->dpcd[DP_DPCD_REV] < DP_DPCD_REV_12)
+		return true;
+
+	/* TODO: Read DP_DEVICE_SERVICE_IRQ_VECTOR_ESI1 as well */
+	if (drm_dp_dpcd_read_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &esi[3]) != 0)
+		return false;
+
+	return true;
+}
+
+/* Return %true if acking the ESI vector IRQ events succeeded, %false otherwise. */
+static bool intel_dp_ack_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
+{
+	/*
+	 * TODO: For DP_DPCD_REV >= 0x12 write
+	 * DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0
+	 */
+	if (drm_dp_dpcd_write_byte(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, esi[1]) != 0)
+		return false;
+
+	if (intel_dp->dpcd[DP_DPCD_REV] < DP_DPCD_REV_12)
+		return true;
+
+	/* TODO: Read DP_DEVICE_SERVICE_IRQ_VECTOR_ESI1 as well */
+	if (drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, esi[3]) != 0)
+		return false;
+
+	return true;
+}
+
+/*
+ * Return %true if reading the ESI vector and acking the ESI IRQ events succeeded,
+ * %false otherwise.
+ */
+static bool intel_dp_get_and_ack_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_connector *connector = intel_dp->attached_connector;
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+
+	if (!intel_dp_get_sink_irq_esi_sst(intel_dp, esi))
+		return false;
+
+	drm_dbg_kms(display->drm,
+		    "[CONNECTOR:%d:%s][ENCODER:%d:%s] DPRX ESI: %4ph\n",
+		    connector->base.base.id, connector->base.name,
+		    encoder->base.base.id, encoder->base.name,
+		    esi);
+
+	esi[1] &= INTEL_DP_DEVICE_SERVICE_IRQ_MASK_SST;
+	esi[3] &= INTEL_DP_LINK_SERVICE_IRQ_MASK_SST;
+
+	if (mem_is_zero(&esi[1], 3))
+		return true;
+
+	if (!intel_dp_ack_sink_irq_esi_sst(intel_dp, esi))
+		return false;
+
+	return true;
+}
+
 bool
 intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state,
 		       const struct drm_connector_state *conn_state)
@@ -5319,23 +5555,7 @@ intel_dp_mst_hpd_irq(struct intel_dp *intel_dp, u8 *esi, u8 *ack)
 	}
 }
 
-static bool intel_dp_mst_link_status(struct intel_dp *intel_dp)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
-	u8 link_status[DP_LINK_STATUS_SIZE] = {};
-	const size_t esi_link_status_size = DP_LINK_STATUS_SIZE - 2;
-
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_LANE0_1_STATUS_ESI, link_status,
-			     esi_link_status_size) != esi_link_status_size) {
-		drm_err(display->drm,
-			"[ENCODER:%d:%s] Failed to read link status\n",
-			encoder->base.base.id, encoder->base.name);
-		return false;
-	}
-
-	return intel_dp_link_ok(intel_dp, link_status);
-}
+static bool intel_dp_handle_link_service_irq(struct intel_dp *intel_dp, u8 irq_mask);
 
 /**
  * intel_dp_check_mst_status - service any pending MST interrupts, check link status
@@ -5355,53 +5575,51 @@ static bool
 intel_dp_check_mst_status(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *encoder = &dig_port->base;
-	bool link_ok = true;
+	bool force_retrain = intel_dp->link.force_retrain;
 	bool reprobe_needed = false;
 
 	for (;;) {
 		u8 esi[4] = {};
 		u8 ack[4] = {};
+		bool new_irqs;
 
 		if (!intel_dp_get_sink_irq_esi(intel_dp, esi)) {
 			drm_dbg_kms(display->drm,
 				    "failed to get ESI - device may have failed\n");
-			link_ok = false;
+			reprobe_needed = true;
 
 			break;
 		}
 
 		drm_dbg_kms(display->drm, "DPRX ESI: %4ph\n", esi);
 
-		if (intel_dp_mst_active_streams(intel_dp) > 0 && link_ok &&
-		    esi[3] & LINK_STATUS_CHANGED) {
-			if (!intel_dp_mst_link_status(intel_dp))
-				link_ok = false;
-			ack[3] |= LINK_STATUS_CHANGED;
-		}
+		ack[3] |= esi[3] & INTEL_DP_LINK_SERVICE_IRQ_MASK_MST;
 
 		intel_dp_mst_hpd_irq(intel_dp, esi, ack);
 
-		if (esi[3] & DP_TUNNELING_IRQ) {
-			if (drm_dp_tunnel_handle_irq(display->dp_tunnel_mgr,
-						     &intel_dp->aux))
-				reprobe_needed = true;
-			ack[3] |= DP_TUNNELING_IRQ;
-		}
+		new_irqs = !mem_is_zero(ack, sizeof(ack));
 
-		if (mem_is_zero(ack, sizeof(ack)))
-			break;
+		drm_WARN_ON(display->drm, ack[1] & ~INTEL_DP_DEVICE_SERVICE_IRQ_MASK_MST);
+		drm_WARN_ON(display->drm, ack[3] & ~INTEL_DP_LINK_SERVICE_IRQ_MASK_MST);
 
-		if (!intel_dp_ack_sink_irq_esi(intel_dp, ack))
+		if (new_irqs && !intel_dp_ack_sink_irq_esi(intel_dp, ack))
 			drm_dbg_kms(display->drm, "Failed to ack ESI\n");
 
 		if (ack[1] & (DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY))
 			drm_dp_mst_hpd_irq_send_new_request(&intel_dp->mst.mgr);
-	}
 
-	if (!link_ok || intel_dp->link.force_retrain)
-		intel_encoder_link_check_queue_work(encoder, 0);
+		if (force_retrain) {
+			/* Defer forced retraining to the regular link status check. */
+			ack[3] |= LINK_STATUS_CHANGED;
+			force_retrain = false;
+		}
+
+		if (intel_dp_handle_link_service_irq(intel_dp, ack[3]))
+			reprobe_needed = true;
+
+		if (!new_irqs)
+			break;
+	}
 
 	return !reprobe_needed;
 }
@@ -5430,6 +5648,30 @@ intel_dp_handle_hdmi_link_status_change(struct intel_dp *intel_dp)
 	}
 }
 
+static int
+intel_dp_read_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE])
+{
+	int err;
+
+	memset(link_status, 0, DP_LINK_STATUS_SIZE);
+
+	if (intel_dp_mst_active_streams(intel_dp) > 0)
+		err = drm_dp_dpcd_read_data(&intel_dp->aux, DP_LANE0_1_STATUS_ESI,
+					    link_status, DP_LINK_STATUS_SIZE - 2);
+	else
+		err = drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
+						       link_status);
+
+	if (err)
+		return err;
+
+	if (link_status[DP_LANE_ALIGN_STATUS_UPDATED - DP_LANE0_1_STATUS] &
+	    DP_DOWNSTREAM_PORT_STATUS_CHANGED)
+		WRITE_ONCE(intel_dp->downstream_port_changed, true);
+
+	return 0;
+}
+
 static bool
 intel_dp_needs_link_retrain(struct intel_dp *intel_dp)
 {
@@ -5452,8 +5694,7 @@ intel_dp_needs_link_retrain(struct intel_dp *intel_dp)
 	if (intel_dp->link.force_retrain)
 		return true;
 
-	if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
-					     link_status) < 0)
+	if (intel_dp_read_link_status(intel_dp, link_status) < 0)
 		return false;
 
 	/*
@@ -5650,55 +5891,57 @@ void intel_dp_check_link_state(struct intel_dp *intel_dp)
 	intel_encoder_link_check_queue_work(encoder, 0);
 }
 
-static void intel_dp_check_device_service_irq(struct intel_dp *intel_dp)
+static void intel_dp_handle_device_service_irq(struct intel_dp *intel_dp, u8 irq_mask)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	u8 val;
 
-	if (intel_dp->dpcd[DP_DPCD_REV] < 0x11)
-		return;
-
-	if (drm_dp_dpcd_readb(&intel_dp->aux,
-			      DP_DEVICE_SERVICE_IRQ_VECTOR, &val) != 1 || !val)
-		return;
-
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, val);
+	drm_WARN_ON(display->drm, irq_mask & ~INTEL_DP_DEVICE_SERVICE_IRQ_MASK_SST);
 
-	if (val & DP_AUTOMATED_TEST_REQUEST)
+	if (irq_mask & DP_AUTOMATED_TEST_REQUEST)
 		intel_dp_test_request(intel_dp);
 
-	if (val & DP_CP_IRQ)
+	if (irq_mask & DP_CP_IRQ)
 		intel_hdcp_handle_cp_irq(intel_dp->attached_connector);
 
-	if (val & DP_SINK_SPECIFIC_IRQ)
+	if (irq_mask & DP_SINK_SPECIFIC_IRQ)
 		drm_dbg_kms(display->drm, "Sink specific irq unhandled\n");
 }
 
-static bool intel_dp_check_link_service_irq(struct intel_dp *intel_dp)
+
+/*
+ * Return %true if a full connector reprobe is required after handling a link
+ * service IRQ event.
+ */
+static bool intel_dp_handle_link_service_irq(struct intel_dp *intel_dp, u8 irq_mask)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_connector *connector = intel_dp->attached_connector;
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	bool reprobe_needed = false;
-	u8 val;
 
-	if (intel_dp->dpcd[DP_DPCD_REV] < 0x11)
-		return false;
-
-	if (drm_dp_dpcd_readb(&intel_dp->aux,
-			      DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &val) != 1 || !val)
-		return false;
+	drm_WARN_ON(display->drm, irq_mask & ~(INTEL_DP_LINK_SERVICE_IRQ_MASK_SST |
+					       INTEL_DP_LINK_SERVICE_IRQ_MASK_MST));
 
-	if ((val & DP_TUNNELING_IRQ) &&
-	    drm_dp_tunnel_handle_irq(display->dp_tunnel_mgr,
-				     &intel_dp->aux))
+	if (irq_mask & RX_CAP_CHANGED)
 		reprobe_needed = true;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux,
-			       DP_LINK_SERVICE_IRQ_VECTOR_ESI0, val) != 1)
-		return reprobe_needed;
+	if (irq_mask & LINK_STATUS_CHANGED)
+		intel_dp_check_link_state(intel_dp);
 
-	if (val & HDMI_LINK_STATUS_CHANGED)
+	if (irq_mask & HDMI_LINK_STATUS_CHANGED)
 		intel_dp_handle_hdmi_link_status_change(intel_dp);
 
+	if (irq_mask & CONNECTED_OFF_ENTRY_REQUESTED)
+		drm_dbg_kms(display->drm,
+			    "[CONNECTOR:%d:%s][ENCODER:%d:%s] Allowing connected off request\n",
+			    connector->base.base.id, connector->base.name,
+			    encoder->base.base.id, encoder->base.name);
+
+	if ((irq_mask & DP_TUNNELING_IRQ) &&
+	    drm_dp_tunnel_handle_irq(display->dp_tunnel_mgr,
+				     &intel_dp->aux))
+		reprobe_needed = true;
+
 	return reprobe_needed;
 }
 
@@ -5718,32 +5961,42 @@ static bool intel_dp_check_link_service_irq(struct intel_dp *intel_dp)
 static bool
 intel_dp_short_pulse(struct intel_dp *intel_dp)
 {
-	u8 old_sink_count = intel_dp->sink_count;
 	bool reprobe_needed = false;
-	bool ret;
+	u8 esi[4] = {};
 
 	intel_dp_test_reset(intel_dp);
 
+	if (!intel_dp_get_and_ack_sink_irq_esi_sst(intel_dp, esi))
+		return false;
+
 	/*
-	 * Now read the DPCD to see if it's actually running
 	 * If the current value of sink count doesn't match with
-	 * the value that was stored earlier or dpcd read failed
-	 * we need to do full detection
+	 * the value that was stored earlier we need to do full
+	 * detection.
 	 */
-	ret = intel_dp_get_dpcd(intel_dp);
-
-	if ((old_sink_count != intel_dp->sink_count) || !ret) {
+	if (intel_dp_has_sink_count(intel_dp) &&
+	    DP_GET_SINK_COUNT(esi[0]) != intel_dp->sink_count)
 		/* No need to proceed if we are going to do full detect */
 		return false;
-	}
 
-	intel_dp_check_device_service_irq(intel_dp);
-	reprobe_needed = intel_dp_check_link_service_irq(intel_dp);
+	intel_dp_handle_device_service_irq(intel_dp, esi[1]);
+
+	/*
+	 * Force checking the link status for DPCD_REV < 1.2
+	 * TODO: let the link status check depend on LINK_STATUS_CHANGED
+	 * or intel_dp->link.force_retrain for DPCD_REV >= 1.2
+	 */
+	esi[3] |= LINK_STATUS_CHANGED;
+	if (intel_dp_handle_link_service_irq(intel_dp, esi[3]))
+		reprobe_needed = true;
 
 	/* Handle CEC interrupts, if any */
 	drm_dp_cec_irq(&intel_dp->aux);
 
-	intel_dp_check_link_state(intel_dp);
+	if (READ_ONCE(intel_dp->downstream_port_changed)) {
+		WRITE_ONCE(intel_dp->downstream_port_changed, false);
+		reprobe_needed = true;
+	}
 
 	intel_psr_short_pulse(intel_dp);
 
@@ -5770,6 +6023,8 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 	if (drm_WARN_ON(display->drm, intel_dp_is_edp(intel_dp)))
 		return connector_status_connected;
 
+	WRITE_ONCE(intel_dp->downstream_port_changed, false);
+
 	intel_lspcon_resume(dig_port);
 
 	if (!intel_dp_get_dpcd(intel_dp))
@@ -6196,8 +6451,6 @@ intel_dp_detect(struct drm_connector *_connector,
 	if (intel_dp_is_edp(intel_dp) || connector->detect_edid)
 		status = connector_status_connected;
 
-	intel_dp_check_device_service_irq(intel_dp);
-
 out_unset_edid:
 	if (status != connector_status_connected && !intel_dp->is_mst)
 		intel_dp_unset_edid(intel_dp);
@@ -7093,6 +7346,8 @@ int intel_dp_compute_config_late(struct intel_encoder *encoder,
 	if (ret)
 		return ret;
 
+	intel_alpm_lobf_compute_config_late(intel_dp, crtc_state);
+
 	return 0;
 }
 
@@ -7141,3 +7396,22 @@ int intel_dp_sdp_min_guardband(const struct intel_crtc_state *crtc_state,
 
 	return sdp_guardband;
 }
+
+bool intel_dp_joiner_candidate_valid(struct intel_connector *connector,
+				     int hdisplay,
+				     int num_joined_pipes)
+{
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+
+	if (!intel_dp_can_join(intel_dp, num_joined_pipes))
+		return false;
+
+	if (hdisplay > num_joined_pipes * intel_dp_max_hdisplay_per_pipe(display))
+		return false;
+
+	if (connector->force_joined_pipes && connector->force_joined_pipes != num_joined_pipes)
+		return false;
+
+	return true;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
index 25bfbfd291b0a..2849b9ecdc71a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.h
+++ b/drivers/gpu/drm/i915/display/intel_dp.h
@@ -75,6 +75,7 @@ int intel_dp_compute_config(struct intel_encoder *encoder,
 			    struct drm_connector_state *conn_state);
 bool intel_dp_needs_8b10b_fec(const struct intel_crtc_state *crtc_state,
 			      bool dsc_enabled_on_crtc);
+void intel_dp_dsc_reset_config(struct intel_crtc_state *crtc_state);
 int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 				struct intel_crtc_state *pipe_config,
 				struct drm_connector_state *conn_state,
@@ -143,6 +144,8 @@ bool intel_digital_port_connected(struct intel_encoder *encoder);
 bool intel_digital_port_connected_locked(struct intel_encoder *encoder);
 int intel_dp_dsc_compute_max_bpp(const struct intel_connector *connector,
 				 u8 dsc_max_bpc);
+int intel_dp_compute_min_compressed_bpp_x16(struct intel_connector *connector,
+					    enum intel_output_format output_format);
 bool intel_dp_mode_valid_with_dsc(struct intel_connector *connector,
 				  int link_clock, int lane_count,
 				  int mode_clock, int mode_hdisplay,
@@ -153,10 +156,6 @@ bool intel_dp_dsc_valid_compressed_bpp(struct intel_dp *intel_dp, int bpp_x16);
 u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 				int mode_clock, int mode_hdisplay,
 				int num_joined_pipes);
-int intel_dp_num_joined_pipes(struct intel_dp *intel_dp,
-			      struct intel_connector *connector,
-			      int hdisplay, int clock);
-
 static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
 {
 	return ~((1 << lane_count) - 1) & 0xf;
@@ -225,5 +224,18 @@ int intel_dp_compute_config_late(struct intel_encoder *encoder,
 				 struct drm_connector_state *conn_state);
 int intel_dp_sdp_min_guardband(const struct intel_crtc_state *crtc_state,
 			       bool assume_all_enabled);
+int intel_dp_max_hdisplay_per_pipe(struct intel_display *display);
+bool intel_dp_dotclk_valid(struct intel_display *display,
+			   int target_clock,
+			   int htotal,
+			   int dsc_slice_count,
+			   int num_joined_pipes);
+bool intel_dp_joiner_candidate_valid(struct intel_connector *connector,
+				     int hdisplay,
+				     int num_joined_pipes);
+
+#define for_each_joiner_candidate(__connector, __mode, __num_joined_pipes) \
+	for ((__num_joined_pipes) = 1; (__num_joined_pipes) <= (I915_MAX_PIPES); (__num_joined_pipes)++) \
+		for_each_if(intel_dp_joiner_candidate_valid(__connector, (__mode)->hdisplay, __num_joined_pipes))
 
 #endif /* __INTEL_DP_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index eb05ef4bd9f6c..d0c76632a946a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -368,6 +368,16 @@ static const char *dpcd_vs_pwm_str(bool aux)
 	return aux ? "DPCD" : "PWM";
 }
 
+static const char *backlight_unit_str(struct intel_panel *panel)
+{
+	if (panel->backlight.edp.vesa.info.luminance_set)
+		return "NITS";
+	else if (panel->backlight.edp.vesa.info.aux_set)
+		return "Brightness %";
+	else
+		return "PWM";
+}
+
 static void
 intel_dp_aux_write_panel_luminance_override(struct intel_connector *connector)
 {
@@ -459,7 +469,7 @@ static u32 intel_dp_aux_vesa_get_backlight(struct intel_connector *connector, en
 		return val / 1000;
 	}
 
-	return connector->panel.backlight.level;
+	return panel->backlight.level;
 }
 
 static void
@@ -486,7 +496,8 @@ intel_dp_aux_vesa_enable_backlight(const struct intel_crtc_state *crtc_state,
 	struct intel_panel *panel = &connector->panel;
 	struct intel_dp *intel_dp = enc_to_intel_dp(connector->encoder);
 
-	if (!panel->backlight.edp.vesa.info.aux_enable) {
+	if (!(panel->backlight.edp.vesa.info.aux_enable ||
+	      panel->backlight.edp.vesa.info.luminance_set)) {
 		u32 pwm_level;
 
 		if (!panel->backlight.edp.vesa.info.aux_set)
@@ -510,7 +521,8 @@ static void intel_dp_aux_vesa_disable_backlight(const struct drm_connector_state
 
 	drm_edp_backlight_disable(&intel_dp->aux, &panel->backlight.edp.vesa.info);
 
-	if (!panel->backlight.edp.vesa.info.aux_enable)
+	if (!(panel->backlight.edp.vesa.info.aux_enable ||
+	      panel->backlight.edp.vesa.info.luminance_set))
 		panel->backlight.pwm_funcs->disable(old_conn_state,
 						    intel_backlight_invert_pwm_level(connector, 0));
 }
@@ -537,11 +549,14 @@ static int intel_dp_aux_vesa_setup_backlight(struct intel_connector *connector,
 	drm_dbg_kms(display->drm,
 		    "[CONNECTOR:%d:%s] AUX VESA backlight enable is controlled through %s\n",
 		    connector->base.base.id, connector->base.name,
-		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_enable));
+		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_enable ||
+				    panel->backlight.edp.vesa.info.luminance_set));
 	drm_dbg_kms(display->drm,
-		    "[CONNECTOR:%d:%s] AUX VESA backlight level is controlled through %s\n",
+		    "[CONNECTOR:%d:%s] AUX VESA backlight level is controlled through %s using %s values\n",
 		    connector->base.base.id, connector->base.name,
-		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_set));
+		    dpcd_vs_pwm_str(panel->backlight.edp.vesa.info.aux_set ||
+				    panel->backlight.edp.vesa.info.luminance_set),
+		    backlight_unit_str(panel));
 
 	if (!panel->backlight.edp.vesa.info.aux_set ||
 	    !panel->backlight.edp.vesa.info.aux_enable) {
@@ -564,9 +579,6 @@ static int intel_dp_aux_vesa_setup_backlight(struct intel_connector *connector,
 		}
 		panel->backlight.level = intel_dp_aux_vesa_get_backlight(connector, 0);
 		panel->backlight.enabled = panel->backlight.level != 0;
-		drm_dbg_kms(display->drm,
-			    "[CONNECTOR:%d:%s] AUX VESA Nits backlight level is controlled through DPCD\n",
-			    connector->base.base.id, connector->base.name);
 	} else if (panel->backlight.edp.vesa.info.aux_set) {
 		panel->backlight.max = panel->backlight.edp.vesa.info.max;
 		panel->backlight.min = 0;
@@ -597,6 +609,34 @@ static int intel_dp_aux_vesa_setup_backlight(struct intel_connector *connector,
 	return 0;
 }
 
+static bool
+check_if_vesa_backlight_possible(struct intel_dp *intel_dp)
+{
+	int ret;
+	u8 bit_min, bit_max;
+
+	if (!(intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_AUX_SET_CAP))
+		return true;
+
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT_CAP_MIN, &bit_min);
+	if (ret < 0)
+		return false;
+
+	bit_min &= DP_EDP_PWMGEN_BIT_COUNT_MASK;
+	if (bit_min < 1)
+		return false;
+
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT_CAP_MAX, &bit_max);
+	if (ret < 0)
+		return false;
+
+	bit_max &= DP_EDP_PWMGEN_BIT_COUNT_MASK;
+	if (bit_max < bit_min)
+		return false;
+
+	return true;
+}
+
 static bool
 intel_dp_aux_supports_vesa_backlight(struct intel_connector *connector)
 {
@@ -613,12 +653,14 @@ intel_dp_aux_supports_vesa_backlight(struct intel_connector *connector)
 		return true;
 	}
 
-	if (drm_edp_backlight_supported(intel_dp->edp_dpcd)) {
+	if (drm_edp_backlight_supported(intel_dp->edp_dpcd) &&
+	    check_if_vesa_backlight_possible(intel_dp)) {
 		drm_dbg_kms(display->drm,
 			    "[CONNECTOR:%d:%s] AUX Backlight Control Supported!\n",
 			    connector->base.base.id, connector->base.name);
 		return true;
 	}
+
 	return false;
 }
 
@@ -644,9 +686,10 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct drm_device *dev = connector->base.dev;
 	struct intel_panel *panel = &connector->panel;
-	bool try_intel_interface = false, try_vesa_interface = false;
+	bool try_intel_interface = false;
 
-	/* Check the VBT and user's module parameters to figure out which
+	/*
+	 * Check the VBT and user's module parameters to figure out which
 	 * interfaces to probe
 	 */
 	switch (display->params.enable_dpcd_backlight) {
@@ -655,7 +698,6 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	case INTEL_DP_AUX_BACKLIGHT_AUTO:
 		switch (panel->vbt.backlight.type) {
 		case INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE:
-			try_vesa_interface = true;
 			break;
 		case INTEL_BACKLIGHT_DISPLAY_DDI:
 			try_intel_interface = true;
@@ -668,20 +710,12 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 		if (panel->vbt.backlight.type != INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE)
 			try_intel_interface = true;
 
-		try_vesa_interface = true;
-		break;
-	case INTEL_DP_AUX_BACKLIGHT_FORCE_VESA:
-		try_vesa_interface = true;
 		break;
 	case INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL:
 		try_intel_interface = true;
 		break;
 	}
 
-	/* For eDP 1.5 and above we are supposed to use VESA interface for brightness control */
-	if (intel_dp->edp_dpcd[0] >= DP_EDP_15)
-		try_vesa_interface = true;
-
 	/*
 	 * Since Intel has their own backlight control interface, the majority of machines out there
 	 * using DPCD backlight controls with Intel GPUs will be using this interface as opposed to
@@ -694,16 +728,19 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	 * panel with Intel's OUI - which is also required for us to be able to detect Intel's
 	 * backlight interface at all. This means that the only sensible way for us to detect both
 	 * interfaces is to probe for Intel's first, and VESA's second.
+	 *
+	 * Also there is a chance some VBTs may advertise false Intel backlight support even if the
+	 * TCON DPCD says otherwise. This means we keep VESA interface as fallback in that case.
 	 */
-	if (try_intel_interface && intel_dp_aux_supports_hdr_backlight(connector) &&
-	    intel_dp->edp_dpcd[0] <= DP_EDP_14b) {
+	if (try_intel_interface && intel_dp->edp_dpcd[0] <= DP_EDP_14b &&
+	    intel_dp_aux_supports_hdr_backlight(connector)) {
 		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Using Intel proprietary eDP backlight controls\n",
 			    connector->base.base.id, connector->base.name);
 		panel->backlight.funcs = &intel_dp_hdr_bl_funcs;
 		return 0;
 	}
 
-	if (try_vesa_interface && intel_dp_aux_supports_vesa_backlight(connector)) {
+	if (intel_dp_aux_supports_vesa_backlight(connector)) {
 		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Using VESA eDP backlight controls\n",
 			    connector->base.base.id, connector->base.name);
 		panel->backlight.funcs = &intel_dp_vesa_bl_funcs;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 70a21685a3e1b..887b6de14e467 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -43,6 +43,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dp.h"
 #include "intel_dp_hdcp.h"
 #include "intel_dp_link_training.h"
@@ -595,39 +596,22 @@ mst_stream_compute_config_limits(struct intel_dp *intel_dp,
 							    dsc);
 }
 
-static int mst_stream_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config,
-				     struct drm_connector_state *conn_state)
+static int mst_stream_compute_link_for_joined_pipes(struct intel_encoder *encoder,
+						    struct intel_crtc_state *pipe_config,
+						    struct drm_connector_state *conn_state,
+						    int num_joined_pipes)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
-	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct intel_dp *intel_dp = to_primary_dp(encoder);
-	struct intel_connector *connector =
-		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
 	struct link_config_limits limits;
 	bool dsc_needed, joiner_needs_dsc;
-	int num_joined_pipes;
 	int ret = 0;
 
-	if (pipe_config->fec_enable &&
-	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
-		return -EINVAL;
-
-	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return -EINVAL;
-
-	num_joined_pipes = intel_dp_num_joined_pipes(intel_dp, connector,
-						     adjusted_mode->crtc_hdisplay,
-						     adjusted_mode->crtc_clock);
-	if (num_joined_pipes > 1)
-		pipe_config->joiner_pipes = GENMASK(crtc->pipe + num_joined_pipes - 1, crtc->pipe);
-
-	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
-	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
-	pipe_config->has_pch_encoder = false;
+	intel_dp_dsc_reset_config(pipe_config);
 
 	joiner_needs_dsc = intel_dp_joiner_needs_dsc(display, num_joined_pipes);
 
@@ -642,7 +626,12 @@ static int mst_stream_compute_config(struct intel_encoder *encoder,
 		if (ret == -EDEADLK)
 			return ret;
 
-		if (ret)
+		if (ret ||
+		    !intel_dp_dotclk_valid(display,
+					   adjusted_mode->clock,
+					   adjusted_mode->htotal,
+					   0,
+					   num_joined_pipes))
 			dsc_needed = true;
 	}
 
@@ -653,6 +642,8 @@ static int mst_stream_compute_config(struct intel_encoder *encoder,
 
 	/* enable compression if the mode doesn't fit available BW */
 	if (dsc_needed) {
+		int dsc_slice_count;
+
 		drm_dbg_kms(display->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n",
 			    str_yes_no(ret), str_yes_no(joiner_needs_dsc),
 			    str_yes_no(intel_dp->force_dsc_en));
@@ -683,6 +674,66 @@ static int mst_stream_compute_config(struct intel_encoder *encoder,
 		ret = intel_dp_dsc_compute_config(intel_dp, pipe_config,
 						  conn_state, &limits,
 						  pipe_config->dp_m_n.tu);
+		if (ret)
+			return ret;
+
+		dsc_slice_count = intel_dp_mst_dsc_get_slice_count(connector, pipe_config);
+
+		if (!intel_dp_dotclk_valid(display,
+					   adjusted_mode->clock,
+					   adjusted_mode->htotal,
+					   dsc_slice_count,
+					   num_joined_pipes))
+			return -EINVAL;
+	}
+
+	if (ret)
+		return ret;
+
+	ret = intel_dp_compute_min_hblank(pipe_config, conn_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int mst_stream_compute_config(struct intel_encoder *encoder,
+				     struct intel_crtc_state *pipe_config,
+				     struct drm_connector_state *conn_state)
+{
+	struct intel_display *display = to_intel_display(encoder);
+	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
+	struct intel_dp *intel_dp = to_primary_dp(encoder);
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
+	const struct drm_display_mode *adjusted_mode =
+		&pipe_config->hw.adjusted_mode;
+	int num_joined_pipes;
+	int ret = -EINVAL;
+
+	if (pipe_config->fec_enable &&
+	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
+		return -EINVAL;
+
+	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
+		return -EINVAL;
+
+	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
+	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
+	pipe_config->has_pch_encoder = false;
+
+	for_each_joiner_candidate(connector, adjusted_mode, num_joined_pipes) {
+		if (num_joined_pipes > 1)
+			pipe_config->joiner_pipes = GENMASK(crtc->pipe + num_joined_pipes - 1,
+							    crtc->pipe);
+
+		ret = mst_stream_compute_link_for_joined_pipes(encoder,
+							       pipe_config,
+							       conn_state,
+							       num_joined_pipes);
+		if (ret == 0 || ret == -EDEADLK)
+			break;
 	}
 
 	if (ret)
@@ -695,10 +746,6 @@ static int mst_stream_compute_config(struct intel_encoder *encoder,
 		pipe_config->lane_lat_optim_mask =
 			bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
 
-	ret = intel_dp_compute_min_hblank(pipe_config, conn_state);
-	if (ret)
-		return ret;
-
 	intel_vrr_compute_config(pipe_config, conn_state);
 
 	intel_dp_audio_compute_config(encoder, pipe_config, conn_state);
@@ -1230,7 +1277,7 @@ static void enable_bs_jitter_was(const struct intel_crtc_state *crtc_state)
 		set |= DP_MST_FEC_BS_JITTER_WA(crtc_state->cpu_transcoder);
 
 	/* Wa_14014143976:adlp */
-	if (IS_DISPLAY_STEP(display, STEP_E0, STEP_FOREVER)) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14014143976)) {
 		if (intel_dp_is_uhbr(crtc_state))
 			set |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder);
 		else if (crtc_state->fec_enable)
@@ -1419,11 +1466,11 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 	struct intel_dp *intel_dp = connector->mst.dp;
 	struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst.mgr;
 	struct drm_dp_mst_port *port = connector->mst.port;
-	const int min_bpp = 18;
-	int max_dotclk = display->cdclk.max_dotclk_freq;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 	unsigned long bw_overhead_flags =
 		DRM_DP_BW_OVERHEAD_MST | DRM_DP_BW_OVERHEAD_SSC_REF_CLK;
+	int min_link_bpp_x16 = fxp_q4_from_int(18);
+	static bool supports_dsc;
 	int ret;
 	bool dsc = false;
 	int target_clock = mode->clock;
@@ -1448,6 +1495,13 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 		return 0;
 	}
 
+	supports_dsc = intel_dp_has_dsc(connector) &&
+		       drm_dp_sink_supports_fec(connector->dp.fec_capability);
+
+	if (supports_dsc && connector->mst.port->passthrough_aux)
+		min_link_bpp_x16 = intel_dp_compute_min_compressed_bpp_x16(connector,
+									   INTEL_OUTPUT_FORMAT_RGB);
+
 	max_link_clock = intel_dp_max_link_rate(intel_dp);
 	max_lanes = intel_dp_max_lane_count(intel_dp);
 
@@ -1455,12 +1509,19 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 					       max_link_clock, max_lanes);
 	mode_rate = intel_dp_link_required(max_link_clock, max_lanes,
 					   mode->clock, mode->hdisplay,
-					   fxp_q4_from_int(min_bpp),
+					   min_link_bpp_x16,
 					   bw_overhead_flags);
 
 	/*
 	 * TODO:
 	 * - Also check if compression would allow for the mode
+	 *   in non-passthrough mode, i.e. the last branch device
+	 *   decompressing the stream. This makes a difference only if
+	 *   the BW on the link between the last branch device and the
+	 *   sink is higher than the BW on the whole MST path from the
+	 *   source to the last branch device. Relying on the extra BW
+	 *   this provides also requires the
+	 *   DFP_Link_Available_Payload_Bandwidth_Number described below.
 	 * - Calculate the overhead using drm_dp_bw_overhead() /
 	 *   drm_dp_bw_channel_coding_efficiency(), similarly to the
 	 *   compute config code, as drm_dp_calc_pbn_mode() doesn't
@@ -1470,49 +1531,73 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 	 *   corresponding link capabilities of the sink) in case the
 	 *   stream is uncompressed for it by the last branch device.
 	 */
-	num_joined_pipes = intel_dp_num_joined_pipes(intel_dp, connector,
-						     mode->hdisplay, target_clock);
-	max_dotclk *= num_joined_pipes;
-
 	ret = drm_modeset_lock(&mgr->base.lock, ctx);
 	if (ret)
 		return ret;
 
-	if (mode_rate > max_rate || mode->clock > max_dotclk ||
-	    drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) {
+	if (mode_rate > max_rate ||
+	    drm_dp_calc_pbn_mode(mode->clock, min_link_bpp_x16) > port->full_pbn) {
 		*status = MODE_CLOCK_HIGH;
 		return 0;
 	}
 
-	if (intel_dp_has_dsc(connector) && drm_dp_sink_supports_fec(connector->dp.fec_capability)) {
-		/*
-		 * TBD pass the connector BPC,
-		 * for now U8_MAX so that max BPC on that platform would be picked
-		 */
-		int pipe_bpp = intel_dp_dsc_compute_max_bpp(connector, U8_MAX);
+	*status = MODE_CLOCK_HIGH;
+	for_each_joiner_candidate(connector, mode, num_joined_pipes) {
+		int dsc_slice_count = 0;
 
-		if (!drm_dp_is_uhbr_rate(max_link_clock))
-			bw_overhead_flags |= DRM_DP_BW_OVERHEAD_FEC;
+		if (supports_dsc) {
+			/*
+			 * TBD pass the connector BPC,
+			 * for now U8_MAX so that max BPC on that platform would be picked
+			 */
+			int pipe_bpp = intel_dp_dsc_compute_max_bpp(connector, U8_MAX);
+
+			dsc_slice_count = intel_dp_dsc_get_slice_count(connector,
+								       mode->clock,
+								       mode->hdisplay,
+								       num_joined_pipes);
+
+			if (!drm_dp_is_uhbr_rate(max_link_clock))
+				bw_overhead_flags |= DRM_DP_BW_OVERHEAD_FEC;
+
+			dsc = intel_dp_mode_valid_with_dsc(connector,
+							   max_link_clock, max_lanes,
+							   target_clock, mode->hdisplay,
+							   num_joined_pipes,
+							   INTEL_OUTPUT_FORMAT_RGB, pipe_bpp,
+							   bw_overhead_flags);
+		}
 
-		dsc = intel_dp_mode_valid_with_dsc(connector,
-						   max_link_clock, max_lanes,
-						   target_clock, mode->hdisplay,
-						   num_joined_pipes,
-						   INTEL_OUTPUT_FORMAT_RGB, pipe_bpp,
-						   bw_overhead_flags);
-	}
+		if (intel_dp_joiner_needs_dsc(display, num_joined_pipes) && !dsc) {
+			*status = MODE_CLOCK_HIGH;
+			continue;
+		}
 
-	if (intel_dp_joiner_needs_dsc(display, num_joined_pipes) && !dsc) {
-		*status = MODE_CLOCK_HIGH;
-		return 0;
-	}
+		if (mode_rate > max_rate && !dsc) {
+			*status = MODE_CLOCK_HIGH;
+			continue;
+		}
 
-	if (mode_rate > max_rate && !dsc) {
-		*status = MODE_CLOCK_HIGH;
-		return 0;
+		*status = intel_mode_valid_max_plane_size(display, mode, num_joined_pipes);
+
+		if (*status != MODE_OK)
+			continue;
+
+		if (!dsc)
+			dsc_slice_count = 0;
+
+		if (!intel_dp_dotclk_valid(display,
+					   mode->clock,
+					   mode->htotal,
+					   dsc_slice_count,
+					   num_joined_pipes)) {
+			*status = MODE_CLOCK_HIGH;
+			continue;
+		}
+
+		break;
 	}
 
-	*status = intel_mode_valid_max_plane_size(display, mode, num_joined_pipes);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
index 83865c02d477c..1fd1ac8d556d8 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
@@ -54,31 +54,20 @@ static int kbytes_to_mbits(int kbytes)
 	return DIV_ROUND_UP(kbytes * 8, 1000);
 }
 
-static int get_current_link_bw(struct intel_dp *intel_dp,
-			       bool *below_dprx_bw)
+static int get_current_link_bw(struct intel_dp *intel_dp)
 {
 	int rate = intel_dp_max_common_rate(intel_dp);
 	int lane_count = intel_dp_max_common_lane_count(intel_dp);
-	int bw;
 
-	bw = intel_dp_max_link_data_rate(intel_dp, rate, lane_count);
-	*below_dprx_bw = bw < drm_dp_max_dprx_data_rate(rate, lane_count);
-
-	return bw;
+	return intel_dp_max_link_data_rate(intel_dp, rate, lane_count);
 }
 
-static int update_tunnel_state(struct intel_dp *intel_dp)
+static int __update_tunnel_state(struct intel_dp *intel_dp, bool force_sink_update)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
-	bool old_bw_below_dprx;
-	bool new_bw_below_dprx;
-	int old_bw;
-	int new_bw;
 	int ret;
 
-	old_bw = get_current_link_bw(intel_dp, &old_bw_below_dprx);
-
 	ret = drm_dp_tunnel_update_state(intel_dp->tunnel);
 	if (ret < 0) {
 		drm_dbg_kms(display->drm,
@@ -90,18 +79,26 @@ static int update_tunnel_state(struct intel_dp *intel_dp)
 		return ret;
 	}
 
-	if (ret == 0 ||
-	    !drm_dp_tunnel_bw_alloc_is_enabled(intel_dp->tunnel))
+	if (!force_sink_update &&
+	    (ret == 0 || !drm_dp_tunnel_bw_alloc_is_enabled(intel_dp->tunnel)))
 		return 0;
 
 	intel_dp_update_sink_caps(intel_dp);
 
-	new_bw = get_current_link_bw(intel_dp, &new_bw_below_dprx);
+	return 0;
+}
+
+static bool has_tunnel_bw_changed(struct intel_dp *intel_dp, int old_bw)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	int new_bw;
+
+	new_bw = get_current_link_bw(intel_dp);
 
 	/* Suppress the notification if the mode list can't change due to bw. */
-	if (old_bw_below_dprx == new_bw_below_dprx &&
-	    !new_bw_below_dprx)
-		return 0;
+	if (old_bw == new_bw)
+		return false;
 
 	drm_dbg_kms(display->drm,
 		    "[DPTUN %s][ENCODER:%d:%s] Notify users about BW change: %d -> %d\n",
@@ -109,7 +106,29 @@ static int update_tunnel_state(struct intel_dp *intel_dp)
 		    encoder->base.base.id, encoder->base.name,
 		    kbytes_to_mbits(old_bw), kbytes_to_mbits(new_bw));
 
-	return 1;
+	return true;
+}
+
+/*
+ * Returns:
+ * - 0 in case of success - if there wasn't any change in the tunnel state
+ *   requiring a user notification
+ * - 1 in case of success - if there was a change in the tunnel state
+ *   requiring a user notification
+ * - Negative error code if updating the tunnel state failed
+ */
+static int update_tunnel_state(struct intel_dp *intel_dp)
+{
+	int old_bw;
+	int err;
+
+	old_bw = get_current_link_bw(intel_dp);
+
+	err = __update_tunnel_state(intel_dp, false);
+	if (err)
+		return err;
+
+	return has_tunnel_bw_changed(intel_dp, old_bw) ? 1 : 0;
 }
 
 /*
@@ -150,11 +169,9 @@ static int allocate_initial_tunnel_bw_for_pipes(struct intel_dp *intel_dp, u8 pi
 			    drm_dp_tunnel_name(intel_dp->tunnel),
 			    encoder->base.base.id, encoder->base.name,
 			    ERR_PTR(err));
-
-		return err;
 	}
 
-	return update_tunnel_state(intel_dp);
+	return err;
 }
 
 static int allocate_initial_tunnel_bw(struct intel_dp *intel_dp,
@@ -170,13 +187,24 @@ static int allocate_initial_tunnel_bw(struct intel_dp *intel_dp,
 	return allocate_initial_tunnel_bw_for_pipes(intel_dp, pipe_mask);
 }
 
+/*
+ * Returns:
+ * - 0 in case of success - after any tunnel detected and added to @intel_dp
+ * - 1 in case of success - after a tunnel detected and added to @intel_dp,
+ *   where the link BW via the tunnel changed in a way requiring a user
+ *   notification
+ * - Negative error code if the tunnel detection failed
+ */
 static int detect_new_tunnel(struct intel_dp *intel_dp, struct drm_modeset_acquire_ctx *ctx)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_dp_tunnel *tunnel;
+	int old_bw;
 	int ret;
 
+	old_bw = get_current_link_bw(intel_dp);
+
 	tunnel = drm_dp_tunnel_detect(display->dp_tunnel_mgr,
 				      &intel_dp->aux);
 	if (IS_ERR(tunnel))
@@ -200,10 +228,17 @@ static int detect_new_tunnel(struct intel_dp *intel_dp, struct drm_modeset_acqui
 	}
 
 	ret = allocate_initial_tunnel_bw(intel_dp, ctx);
-	if (ret < 0)
+	if (ret < 0) {
 		intel_dp_tunnel_destroy(intel_dp);
 
-	return ret;
+		return ret;
+	}
+
+	ret = __update_tunnel_state(intel_dp, true);
+	if (ret)
+		return ret;
+
+	return has_tunnel_bw_changed(intel_dp, old_bw) ? 1 : 0;
 }
 
 /**
@@ -221,9 +256,12 @@ static int detect_new_tunnel(struct intel_dp *intel_dp, struct drm_modeset_acqui
  * tunnel. If the tunnel's state change requires this - for instance the
  * tunnel's group ID has changed - the tunnel will be dropped and recreated.
  *
- * Return 0 in case of success - after any tunnel detected and added to
- * @intel_dp - 1 in case the BW on an already existing tunnel has changed in a
- * way that requires notifying user space.
+ * Returns:
+ * - 0 in case of success - after any tunnel detected and added to @intel_dp
+ * - 1 in case the link BW via the new or an already existing tunnel has changed
+ *   in a way that requires notifying user space
+ * - Negative error code, if creating a new tunnel or updating the tunnel
+ *   state failed
  */
 int intel_dp_tunnel_detect(struct intel_dp *intel_dp, struct drm_modeset_acquire_ctx *ctx)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index a4f372c9e6fca..8433e3ff0319f 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -1219,6 +1219,7 @@ static int xe3plpd_crtc_compute_clock(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder =
 		intel_get_crtc_new_encoder(state, crtc_state);
+	struct intel_display *display = to_intel_display(encoder);
 	int ret;
 
 	ret = intel_lt_phy_pll_calc_state(crtc_state, encoder);
@@ -1227,7 +1228,7 @@ static int xe3plpd_crtc_compute_clock(struct intel_atomic_state *state,
 
 	/* TODO: Do the readback via intel_compute_shared_dplls() */
 	crtc_state->port_clock =
-			intel_lt_phy_calc_port_clock(encoder, crtc_state);
+			intel_lt_phy_calc_port_clock(display, &crtc_state->dpll_hw_state.ltpll);
 
 	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
 
@@ -2333,3 +2334,8 @@ void assert_pll_disabled(struct intel_display *display, enum pipe pipe)
 {
 	assert_pll(display, pipe, false);
 }
+
+bool intel_dpll_clock_matches(int clock1, int clock2)
+{
+	return abs(clock1 - clock2) <= 1;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.h b/drivers/gpu/drm/i915/display/intel_dpll.h
index 3444a2dd3166e..8cd0d17e974e5 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll.h
@@ -48,5 +48,6 @@ void chv_crtc_clock_get(struct intel_crtc_state *crtc_state);
 
 void assert_pll_enabled(struct intel_display *display, enum pipe pipe);
 void assert_pll_disabled(struct intel_display *display, enum pipe pipe);
+bool intel_dpll_clock_matches(int clock1, int clock2);
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 9aa84a430f098..f35a9252f4e10 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -38,6 +38,7 @@
 #include "intel_dpll.h"
 #include "intel_dpll_mgr.h"
 #include "intel_hti.h"
+#include "intel_lt_phy.h"
 #include "intel_mg_phy_regs.h"
 #include "intel_pch_refclk.h"
 #include "intel_step.h"
@@ -4613,7 +4614,7 @@ void intel_dpll_init(struct intel_display *display)
 		dpll_mgr = &pch_pll_mgr;
 
 	if (!dpll_mgr)
-		return;
+		goto out_verify;
 
 	dpll_info = dpll_mgr->dpll_info;
 
@@ -4632,6 +4633,14 @@ void intel_dpll_init(struct intel_display *display)
 
 	display->dpll.mgr = dpll_mgr;
 	display->dpll.num_dpll = i;
+
+out_verify:
+	/*
+	 * TODO: Convert these to a KUnit test or dependent on a kconfig
+	 * debug option.
+	 */
+	intel_cx0pll_verify_plls(display);
+	intel_lt_phy_verify_plls(display);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
index 5b71c860515fc..4cc14ce5eebe0 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
@@ -241,14 +241,12 @@ struct intel_mpllb_state {
 };
 
 struct intel_c10pll_state {
-	u32 clock; /* in KHz */
 	u8 tx;
 	u8 cmn;
 	u8 pll[20];
 };
 
 struct intel_c20pll_state {
-	u32 clock; /* in kHz */
 	u16 tx[3];
 	u16 cmn[4];
 	union {
@@ -274,7 +272,6 @@ struct intel_cx0pll_state {
 };
 
 struct intel_lt_phy_pll_state {
-	u32 clock; /* in kHz */
 	u8 addr_msb[13];
 	u8 addr_lsb[13];
 	u8 data[13][4];
diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c
index da472371c7d7a..145dc9511116a 100644
--- a/drivers/gpu/drm/i915/display/intel_dpt.c
+++ b/drivers/gpu/drm/i915/display/intel_dpt.c
@@ -1,202 +1,51 @@
 // SPDX-License-Identifier: MIT
 /*
- * Copyright © 2021 Intel Corporation
+ * Copyright © 2023 Intel Corporation
  */
 
-#include <drm/drm_print.h>
-
-#include "gem/i915_gem_domain.h"
-#include "gem/i915_gem_internal.h"
-#include "gem/i915_gem_lmem.h"
-#include "gt/gen8_ppgtt.h"
-
-#include "i915_drv.h"
-#include "intel_display_core.h"
-#include "intel_display_rpm.h"
+#include "intel_de.h"
+#include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_dpt.h"
-#include "intel_fb.h"
-
-struct i915_dpt {
-	struct i915_address_space vm;
-
-	struct drm_i915_gem_object *obj;
-	struct i915_vma *vma;
-	void __iomem *iomem;
-};
-
-#define i915_is_dpt(vm) ((vm)->is_dpt)
-
-static inline struct i915_dpt *
-i915_vm_to_dpt(struct i915_address_space *vm)
-{
-	BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
-	drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));
-	return container_of(vm, struct i915_dpt, vm);
-}
-
-static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
-{
-	writeq(pte, addr);
-}
-
-static void dpt_insert_page(struct i915_address_space *vm,
-			    dma_addr_t addr,
-			    u64 offset,
-			    unsigned int pat_index,
-			    u32 flags)
-{
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-	gen8_pte_t __iomem *base = dpt->iomem;
-
-	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
-		     vm->pte_encode(addr, pat_index, flags));
-}
-
-static void dpt_insert_entries(struct i915_address_space *vm,
-			       struct i915_vma_resource *vma_res,
-			       unsigned int pat_index,
-			       u32 flags)
-{
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-	gen8_pte_t __iomem *base = dpt->iomem;
-	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
-	struct sgt_iter sgt_iter;
-	dma_addr_t addr;
-	int i;
-
-	/*
-	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
-	 * not to allow the user to override access to a read only page.
-	 */
-
-	i = vma_res->start / I915_GTT_PAGE_SIZE;
-	for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
-		gen8_set_pte(&base[i++], pte_encode | addr);
-}
-
-static void dpt_clear_range(struct i915_address_space *vm,
-			    u64 start, u64 length)
-{
-}
-
-static void dpt_bind_vma(struct i915_address_space *vm,
-			 struct i915_vm_pt_stash *stash,
-			 struct i915_vma_resource *vma_res,
-			 unsigned int pat_index,
-			 u32 flags)
-{
-	u32 pte_flags;
-
-	if (vma_res->bound_flags)
-		return;
-
-	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
-	pte_flags = 0;
-	if (vm->has_read_only && vma_res->bi.readonly)
-		pte_flags |= PTE_READ_ONLY;
-	if (vma_res->bi.lmem)
-		pte_flags |= PTE_LM;
+#include "intel_parent.h"
+#include "skl_universal_plane_regs.h"
 
-	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
-
-	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
-
-	/*
-	 * Without aliasing PPGTT there's no difference between
-	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
-	 * upgrade to both bound if we bind either to avoid double-binding.
-	 */
-	vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
-}
-
-static void dpt_unbind_vma(struct i915_address_space *vm,
-			   struct i915_vma_resource *vma_res)
-{
-	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
-}
-
-static void dpt_cleanup(struct i915_address_space *vm)
+void intel_dpt_configure(struct intel_crtc *crtc)
 {
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-
-	i915_gem_object_put(dpt->obj);
-}
-
-struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm,
-				       unsigned int alignment)
-{
-	struct drm_i915_private *i915 = vm->i915;
-	struct intel_display *display = i915->display;
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-	struct ref_tracker *wakeref;
-	struct i915_vma *vma;
-	void __iomem *iomem;
-	struct i915_gem_ww_ctx ww;
-	u64 pin_flags = 0;
-	int err;
-
-	if (i915_gem_object_is_stolen(dpt->obj))
-		pin_flags |= PIN_MAPPABLE;
-
-	wakeref = intel_display_rpm_get(display);
-	atomic_inc(&display->restore.pending_fb_pin);
-
-	for_i915_gem_ww(&ww, err, true) {
-		err = i915_gem_object_lock(dpt->obj, &ww);
-		if (err)
-			continue;
+	struct intel_display *display = to_intel_display(crtc);
 
-		vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0,
-						  alignment, pin_flags);
-		if (IS_ERR(vma)) {
-			err = PTR_ERR(vma);
-			continue;
-		}
+	if (DISPLAY_VER(display) == 14) {
+		enum pipe pipe = crtc->pipe;
+		enum plane_id plane_id;
 
-		iomem = i915_vma_pin_iomap(vma);
-		i915_vma_unpin(vma);
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			if (plane_id == PLANE_CURSOR)
+				continue;
 
-		if (IS_ERR(iomem)) {
-			err = PTR_ERR(iomem);
-			continue;
+			intel_de_rmw(display, PLANE_CHICKEN(pipe, plane_id),
+				     PLANE_CHICKEN_DISABLE_DPT,
+				     display->params.enable_dpt ? 0 :
+				     PLANE_CHICKEN_DISABLE_DPT);
 		}
-
-		dpt->vma = vma;
-		dpt->iomem = iomem;
-
-		i915_vma_get(vma);
+	} else if (DISPLAY_VER(display) == 13) {
+		intel_de_rmw(display, CHICKEN_MISC_2,
+			     CHICKEN_MISC_DISABLE_DPT,
+			     display->params.enable_dpt ? 0 :
+			     CHICKEN_MISC_DISABLE_DPT);
 	}
-
-	dpt->obj->mm.dirty = true;
-
-	atomic_dec(&display->restore.pending_fb_pin);
-	intel_display_rpm_put(display, wakeref);
-
-	return err ? ERR_PTR(err) : vma;
-}
-
-void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm)
-{
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-
-	i915_vma_unpin_iomap(dpt->vma);
-	i915_vma_put(dpt->vma);
 }
 
 /**
- * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
+ * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
  * @display: display device instance
  *
- * Restore the memory mapping during system resume for all framebuffers which
- * are mapped to HW via a GGTT->DPT page table. The content of these page
- * tables are not stored in the hibernation image during S4 and S3RST->S4
- * transitions, so here we reprogram the PTE entries in those tables.
+ * Suspend the memory mapping during system suspend for all framebuffers which
+ * are mapped to HW via a GGTT->DPT page table.
  *
- * This function must be called after the mappings in GGTT have been restored calling
- * i915_ggtt_resume().
+ * This function must be called before the mappings in GGTT are suspended calling
+ * i915_ggtt_suspend().
  */
-void intel_dpt_resume(struct intel_display *display)
+void intel_dpt_suspend(struct intel_display *display)
 {
 	struct drm_framebuffer *drm_fb;
 
@@ -204,26 +53,30 @@ void intel_dpt_resume(struct intel_display *display)
 		return;
 
 	mutex_lock(&display->drm->mode_config.fb_lock);
+
 	drm_for_each_fb(drm_fb, display->drm) {
 		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
 
-		if (fb->dpt_vm)
-			i915_ggtt_resume_vm(fb->dpt_vm, true);
+		if (fb->dpt)
+			intel_parent_dpt_suspend(display, fb->dpt);
 	}
+
 	mutex_unlock(&display->drm->mode_config.fb_lock);
 }
 
 /**
- * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
+ * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
  * @display: display device instance
  *
- * Suspend the memory mapping during system suspend for all framebuffers which
- * are mapped to HW via a GGTT->DPT page table.
+ * Restore the memory mapping during system resume for all framebuffers which
+ * are mapped to HW via a GGTT->DPT page table. The content of these page
+ * tables are not stored in the hibernation image during S4 and S3RST->S4
+ * transitions, so here we reprogram the PTE entries in those tables.
  *
- * This function must be called before the mappings in GGTT are suspended calling
- * i915_ggtt_suspend().
+ * This function must be called after the mappings in GGTT have been restored calling
+ * i915_ggtt_resume().
  */
-void intel_dpt_suspend(struct intel_display *display)
+void intel_dpt_resume(struct intel_display *display)
 {
 	struct drm_framebuffer *drm_fb;
 
@@ -231,96 +84,11 @@ void intel_dpt_suspend(struct intel_display *display)
 		return;
 
 	mutex_lock(&display->drm->mode_config.fb_lock);
-
 	drm_for_each_fb(drm_fb, display->drm) {
 		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
 
-		if (fb->dpt_vm)
-			i915_ggtt_suspend_vm(fb->dpt_vm, true);
+		if (fb->dpt)
+			intel_parent_dpt_resume(display, fb->dpt);
 	}
-
 	mutex_unlock(&display->drm->mode_config.fb_lock);
 }
-
-struct i915_address_space *
-intel_dpt_create(struct intel_framebuffer *fb)
-{
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
-	struct drm_i915_private *i915 = to_i915(obj->dev);
-	struct drm_i915_gem_object *dpt_obj;
-	struct i915_address_space *vm;
-	struct i915_dpt *dpt;
-	size_t size;
-	int ret;
-
-	if (intel_fb_needs_pot_stride_remap(fb))
-		size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
-	else
-		size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
-
-	size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
-
-	dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
-	if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
-		dpt_obj = i915_gem_object_create_stolen(i915, size);
-	if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
-		drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
-		dpt_obj = i915_gem_object_create_shmem(i915, size);
-	}
-	if (IS_ERR(dpt_obj))
-		return ERR_CAST(dpt_obj);
-
-	ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
-	if (!ret) {
-		ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
-		i915_gem_object_unlock(dpt_obj);
-	}
-	if (ret) {
-		i915_gem_object_put(dpt_obj);
-		return ERR_PTR(ret);
-	}
-
-	dpt = kzalloc_obj(*dpt);
-	if (!dpt) {
-		i915_gem_object_put(dpt_obj);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	vm = &dpt->vm;
-
-	vm->gt = to_gt(i915);
-	vm->i915 = i915;
-	vm->dma = i915->drm.dev;
-	vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
-	vm->is_dpt = true;
-
-	i915_address_space_init(vm, VM_CLASS_DPT);
-
-	vm->insert_page = dpt_insert_page;
-	vm->clear_range = dpt_clear_range;
-	vm->insert_entries = dpt_insert_entries;
-	vm->cleanup = dpt_cleanup;
-
-	vm->vma_ops.bind_vma    = dpt_bind_vma;
-	vm->vma_ops.unbind_vma  = dpt_unbind_vma;
-
-	vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
-
-	dpt->obj = dpt_obj;
-	dpt->obj->is_dpt = true;
-
-	return &dpt->vm;
-}
-
-void intel_dpt_destroy(struct i915_address_space *vm)
-{
-	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
-
-	dpt->obj->is_dpt = false;
-	i915_vm_put(&dpt->vm);
-}
-
-u64 intel_dpt_offset(struct i915_vma *dpt_vma)
-{
-	return i915_vma_offset(dpt_vma);
-}
diff --git a/drivers/gpu/drm/i915/display/intel_dpt.h b/drivers/gpu/drm/i915/display/intel_dpt.h
index db521401b828f..11bd495693b26 100644
--- a/drivers/gpu/drm/i915/display/intel_dpt.h
+++ b/drivers/gpu/drm/i915/display/intel_dpt.h
@@ -1,26 +1,16 @@
 /* SPDX-License-Identifier: MIT */
 /*
- * Copyright © 2021 Intel Corporation
+ * Copyright © 2023 Intel Corporation
  */
 
-#ifndef __INTEL_DPT_H__
-#define __INTEL_DPT_H__
+#ifndef __INTEL_DPT_COMMON_H__
+#define __INTEL_DPT_COMMON_H__
 
-#include <linux/types.h>
-
-struct i915_address_space;
-struct i915_vma;
+struct intel_crtc;
 struct intel_display;
-struct intel_framebuffer;
 
-void intel_dpt_destroy(struct i915_address_space *vm);
-struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm,
-				       unsigned int alignment);
-void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm);
+void intel_dpt_configure(struct intel_crtc *crtc);
 void intel_dpt_suspend(struct intel_display *display);
 void intel_dpt_resume(struct intel_display *display);
-struct i915_address_space *
-intel_dpt_create(struct intel_framebuffer *fb);
-u64 intel_dpt_offset(struct i915_vma *dpt_vma);
 
-#endif /* __INTEL_DPT_H__ */
+#endif /* __INTEL_DPT_COMMON_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dpt_common.c b/drivers/gpu/drm/i915/display/intel_dpt_common.c
deleted file mode 100644
index 5eb88d51dba1b..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_dpt_common.c
+++ /dev/null
@@ -1,35 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "intel_de.h"
-#include "intel_display_regs.h"
-#include "intel_display_types.h"
-#include "intel_dpt_common.h"
-#include "skl_universal_plane_regs.h"
-
-void intel_dpt_configure(struct intel_crtc *crtc)
-{
-	struct intel_display *display = to_intel_display(crtc);
-
-	if (DISPLAY_VER(display) == 14) {
-		enum pipe pipe = crtc->pipe;
-		enum plane_id plane_id;
-
-		for_each_plane_id_on_crtc(crtc, plane_id) {
-			if (plane_id == PLANE_CURSOR)
-				continue;
-
-			intel_de_rmw(display, PLANE_CHICKEN(pipe, plane_id),
-				     PLANE_CHICKEN_DISABLE_DPT,
-				     display->params.enable_dpt ? 0 :
-				     PLANE_CHICKEN_DISABLE_DPT);
-		}
-	} else if (DISPLAY_VER(display) == 13) {
-		intel_de_rmw(display, CHICKEN_MISC_2,
-			     CHICKEN_MISC_DISABLE_DPT,
-			     display->params.enable_dpt ? 0 :
-			     CHICKEN_MISC_DISABLE_DPT);
-	}
-}
diff --git a/drivers/gpu/drm/i915/display/intel_dpt_common.h b/drivers/gpu/drm/i915/display/intel_dpt_common.h
deleted file mode 100644
index 6d7de405126a5..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_dpt_common.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_DPT_COMMON_H__
-#define __INTEL_DPT_COMMON_H__
-
-struct intel_crtc;
-
-void intel_dpt_configure(struct intel_crtc *crtc);
-
-#endif /* __INTEL_DPT_COMMON_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dram.c b/drivers/gpu/drm/i915/display/intel_dram.c
index 170de304fe96d..bd281d4b4c05c 100644
--- a/drivers/gpu/drm/i915/display/intel_dram.c
+++ b/drivers/gpu/drm/i915/display/intel_dram.c
@@ -7,13 +7,14 @@
 
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "intel_display_core.h"
 #include "intel_display_utils.h"
+#include "intel_display_regs.h"
 #include "intel_dram.h"
 #include "intel_mchbar_regs.h"
-#include "intel_pcode.h"
+#include "intel_parent.h"
 #include "intel_uncore.h"
 #include "vlv_iosf_sb.h"
 
@@ -692,8 +693,8 @@ static int icl_pcode_read_mem_global_info(struct intel_display *display,
 	u32 val = 0;
 	int ret;
 
-	ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
-			       ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, &val, NULL);
+	ret = intel_parent_pcode_read(display, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
+				      ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, &val, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index b5d774706fec4..c8d3968f659f0 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -8,6 +8,7 @@
 
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "intel_crtc.h"
 #include "intel_de.h"
@@ -15,8 +16,8 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_dsb.h"
-#include "intel_dsb_buffer.h"
 #include "intel_dsb_regs.h"
+#include "intel_psr.h"
 #include "intel_vblank.h"
 #include "intel_vrr.h"
 #include "skl_watermark.h"
@@ -75,6 +76,57 @@ struct intel_dsb {
  * writes). There are no registers reads possible with DSB HW engine.
  */
 
+/*
+ * DSB buffer parent interface calls are here instead of intel_parent.[ch]
+ * because they're not used outside of intel_dsb.c.
+ */
+static u32 dsb_buffer_ggtt_offset(struct intel_dsb *dsb)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	return display->parent->dsb->ggtt_offset(dsb->dsb_buf);
+}
+
+static void dsb_buffer_write(struct intel_dsb *dsb, u32 idx, u32 val)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	display->parent->dsb->write(dsb->dsb_buf, idx, val);
+}
+
+static u32 dsb_buffer_read(struct intel_dsb *dsb, u32 idx)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	return display->parent->dsb->read(dsb->dsb_buf, idx);
+}
+
+static void dsb_buffer_fill(struct intel_dsb *dsb, u32 idx, u32 val, size_t size)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	display->parent->dsb->fill(dsb->dsb_buf, idx, val, size);
+}
+
+static struct intel_dsb_buffer *dsb_buffer_create(struct intel_display *display, size_t size)
+{
+	return display->parent->dsb->create(display->drm, size);
+}
+
+static void dsb_buffer_cleanup(struct intel_dsb *dsb)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	display->parent->dsb->cleanup(dsb->dsb_buf);
+}
+
+static void dsb_buffer_flush_map(struct intel_dsb *dsb)
+{
+	struct intel_display *display = to_intel_display(dsb->crtc);
+
+	display->parent->dsb->flush_map(dsb->dsb_buf);
+}
+
 /* DSB opcodes. */
 #define DSB_OPCODE_SHIFT		24
 #define DSB_OPCODE_NOOP			0x0
@@ -166,18 +218,24 @@ static int dsb_scanline_to_hw(struct intel_atomic_state *state,
  * definitely do not want to skip vblank wait. We also have concern what comes
  * to skipping vblank evasion. I.e. arming registers are latched before we have
  * managed writing them. Due to these reasons we are not setting
- * DSB_SKIP_WAITS_EN.
+ * DSB_SKIP_WAITS_EN except when using TRANS_PUSH mechanism to trigger
+ * "frame change" event.
  */
 static u32 dsb_chicken(struct intel_atomic_state *state,
 		       struct intel_crtc *crtc)
 {
+	const struct intel_crtc_state *new_crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	u32 chicken = intel_psr_use_trans_push(new_crtc_state) ?
+		DSB_SKIP_WAITS_EN : 0;
+
 	if (pre_commit_is_vrr_active(state, crtc))
-		return DSB_CTRL_WAIT_SAFE_WINDOW |
+		chicken |= DSB_CTRL_WAIT_SAFE_WINDOW |
 			DSB_CTRL_NO_WAIT_VBLANK |
 			DSB_INST_WAIT_SAFE_WINDOW |
 			DSB_INST_NO_WAIT_VBLANK;
-	else
-		return 0;
+
+	return chicken;
 }
 
 static bool assert_dsb_has_room(struct intel_dsb *dsb)
@@ -211,10 +269,10 @@ static void intel_dsb_dump(struct intel_dsb *dsb)
 	for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
 		drm_dbg_kms(display->drm,
 			    " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
-			    intel_dsb_buffer_read(dsb->dsb_buf, i),
-			    intel_dsb_buffer_read(dsb->dsb_buf, i + 1),
-			    intel_dsb_buffer_read(dsb->dsb_buf, i + 2),
-			    intel_dsb_buffer_read(dsb->dsb_buf, i + 3));
+			    dsb_buffer_read(dsb, i),
+			    dsb_buffer_read(dsb, i + 1),
+			    dsb_buffer_read(dsb, i + 2),
+			    dsb_buffer_read(dsb, i + 3));
 	drm_dbg_kms(display->drm, "}\n");
 }
 
@@ -231,12 +289,12 @@ unsigned int intel_dsb_size(struct intel_dsb *dsb)
 
 unsigned int intel_dsb_head(struct intel_dsb *dsb)
 {
-	return intel_dsb_buffer_ggtt_offset(dsb->dsb_buf);
+	return dsb_buffer_ggtt_offset(dsb);
 }
 
 static unsigned int intel_dsb_tail(struct intel_dsb *dsb)
 {
-	return intel_dsb_buffer_ggtt_offset(dsb->dsb_buf) + intel_dsb_size(dsb);
+	return dsb_buffer_ggtt_offset(dsb) + intel_dsb_size(dsb);
 }
 
 static void intel_dsb_ins_align(struct intel_dsb *dsb)
@@ -263,8 +321,8 @@ static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 	dsb->ins[0] = ldw;
 	dsb->ins[1] = udw;
 
-	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, dsb->ins[0]);
-	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, dsb->ins[1]);
+	dsb_buffer_write(dsb, dsb->free_pos++, dsb->ins[0]);
+	dsb_buffer_write(dsb, dsb->free_pos++, dsb->ins[1]);
 }
 
 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
@@ -335,13 +393,12 @@ void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
 
 	/* Update the count */
 	dsb->ins[0]++;
-	intel_dsb_buffer_write(dsb->dsb_buf, dsb->ins_start_offset + 0,
-			       dsb->ins[0]);
+	dsb_buffer_write(dsb, dsb->ins_start_offset + 0, dsb->ins[0]);
 
-	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, val);
+	dsb_buffer_write(dsb, dsb->free_pos++, val);
 	/* if number of data words is odd, then the last dword should be 0.*/
 	if (dsb->free_pos & 0x1)
-		intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos, 0);
+		dsb_buffer_write(dsb, dsb->free_pos, 0);
 }
 
 void intel_dsb_reg_write(struct intel_dsb *dsb,
@@ -521,8 +578,7 @@ static void intel_dsb_align_tail(struct intel_dsb *dsb)
 	aligned_tail = ALIGN(tail, CACHELINE_BYTES);
 
 	if (aligned_tail > tail)
-		intel_dsb_buffer_memset(dsb->dsb_buf, dsb->free_pos, 0,
-					aligned_tail - tail);
+		dsb_buffer_fill(dsb, dsb->free_pos, 0, aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
 }
@@ -541,8 +597,7 @@ static void intel_dsb_gosub_align(struct intel_dsb *dsb)
 	 * "Ensure GOSUB is not placed in cacheline QW slot 6 or 7 (numbered 0-7)"
 	 */
 	if (aligned_tail - tail <= 2 * 8)
-		intel_dsb_buffer_memset(dsb->dsb_buf, dsb->free_pos, 0,
-					aligned_tail - tail);
+		dsb_buffer_fill(dsb, dsb->free_pos, 0, aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
 }
@@ -606,14 +661,14 @@ void intel_dsb_gosub_finish(struct intel_dsb *dsb)
 	 */
 	intel_dsb_noop(dsb, 8);
 
-	intel_dsb_buffer_flush_map(dsb->dsb_buf);
+	dsb_buffer_flush_map(dsb);
 }
 
 void intel_dsb_finish(struct intel_dsb *dsb)
 {
 	intel_dsb_align_tail(dsb);
 
-	intel_dsb_buffer_flush_map(dsb->dsb_buf);
+	dsb_buffer_flush_map(dsb);
 }
 
 static u32 dsb_error_int_status(struct intel_display *display)
@@ -917,7 +972,7 @@ void intel_dsb_wait(struct intel_dsb *dsb)
 			      !is_busy,
 			      100, 1000, false);
 	if (ret) {
-		u32 offset = intel_dsb_buffer_ggtt_offset(dsb->dsb_buf);
+		u32 offset = dsb_buffer_ggtt_offset(dsb);
 
 		intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
 				  DSB_ENABLE | DSB_HALT);
@@ -983,7 +1038,7 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
 	/* ~1 qword per instruction, full cachelines */
 	size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
 
-	dsb_buf = intel_dsb_buffer_create(display->drm, size);
+	dsb_buf = dsb_buffer_create(display, size);
 	if (IS_ERR(dsb_buf))
 		goto out_put_rpm;
 
@@ -1021,7 +1076,7 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
  */
 void intel_dsb_cleanup(struct intel_dsb *dsb)
 {
-	intel_dsb_buffer_cleanup(dsb->dsb_buf);
+	dsb_buffer_cleanup(dsb);
 	kfree(dsb);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.c b/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
deleted file mode 100644
index 9b6060af250df..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
+++ /dev/null
@@ -1,105 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2023, Intel Corporation.
- */
-
-#include "gem/i915_gem_internal.h"
-#include "gem/i915_gem_lmem.h"
-#include "i915_drv.h"
-#include "i915_vma.h"
-#include "intel_dsb_buffer.h"
-
-struct intel_dsb_buffer {
-	u32 *cmd_buf;
-	struct i915_vma *vma;
-	size_t buf_size;
-};
-
-u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
-{
-	return i915_ggtt_offset(dsb_buf->vma);
-}
-
-void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
-{
-	dsb_buf->cmd_buf[idx] = val;
-}
-
-u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
-{
-	return dsb_buf->cmd_buf[idx];
-}
-
-void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
-{
-	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
-
-	memset(&dsb_buf->cmd_buf[idx], val, size);
-}
-
-struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct intel_dsb_buffer *dsb_buf;
-	struct drm_i915_gem_object *obj;
-	struct i915_vma *vma;
-	u32 *buf;
-	int ret;
-
-	dsb_buf = kzalloc_obj(*dsb_buf);
-	if (!dsb_buf)
-		return ERR_PTR(-ENOMEM);
-
-	if (HAS_LMEM(i915)) {
-		obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
-						  I915_BO_ALLOC_CONTIGUOUS);
-		if (IS_ERR(obj)) {
-			ret = PTR_ERR(obj);
-			goto err;
-		}
-	} else {
-		obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
-		if (IS_ERR(obj)) {
-			ret = PTR_ERR(obj);
-			goto err;
-		}
-
-		i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
-	}
-
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
-	if (IS_ERR(vma)) {
-		ret = PTR_ERR(vma);
-		i915_gem_object_put(obj);
-		goto err;
-	}
-
-	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
-	if (IS_ERR(buf)) {
-		ret = PTR_ERR(buf);
-		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
-		goto err;
-	}
-
-	dsb_buf->vma = vma;
-	dsb_buf->cmd_buf = buf;
-	dsb_buf->buf_size = size;
-
-	return dsb_buf;
-
-err:
-	kfree(dsb_buf);
-
-	return ERR_PTR(ret);
-}
-
-void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
-{
-	i915_vma_unpin_and_release(&dsb_buf->vma, I915_VMA_RELEASE_MAP);
-	kfree(dsb_buf);
-}
-
-void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
-{
-	i915_gem_object_flush_map(dsb_buf->vma->obj);
-}
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.h b/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
deleted file mode 100644
index f4577d1f25cd9..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef _INTEL_DSB_BUFFER_H
-#define _INTEL_DSB_BUFFER_H
-
-#include <linux/types.h>
-
-struct drm_device;
-struct intel_dsb_buffer;
-
-u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf);
-void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val);
-u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx);
-void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size);
-struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size);
-void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf);
-void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf);
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 38c33f2ca05c7..5768619f840f4 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -16,9 +16,7 @@
 #include "intel_display_core.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
-#include "intel_dpt.h"
 #include "intel_fb.h"
-#include "intel_fb_bo.h"
 #include "intel_frontbuffer.h"
 #include "intel_parent.h"
 #include "intel_plane.h"
@@ -2104,16 +2102,17 @@ int intel_plane_compute_gtt(struct intel_plane_state *plane_state)
 
 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
+	struct intel_display *display = to_intel_display(fb->dev);
 	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 
 	drm_framebuffer_cleanup(fb);
 
 	if (intel_fb_uses_dpt(fb))
-		intel_dpt_destroy(intel_fb->dpt_vm);
+		intel_parent_dpt_destroy(display, intel_fb->dpt);
 
-	intel_fb_bo_framebuffer_fini(intel_fb_bo(fb));
+	intel_bo_framebuffer_fini(intel_fb_bo(fb));
 
-	intel_frontbuffer_put(intel_fb->frontbuffer);
+	intel_parent_frontbuffer_put(display, intel_fb->frontbuffer);
 
 	kfree(intel_fb->panic);
 	kfree(intel_fb);
@@ -2221,16 +2220,16 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 		return -ENOMEM;
 
 	/*
-	 * intel_frontbuffer_get() must be done before
-	 * intel_fb_bo_framebuffer_init() to avoid set_tiling vs. addfb race.
+	 * intel_parent_frontbuffer_get() must be done before
+	 * intel_bo_framebuffer_init() to avoid set_tiling vs. addfb race.
 	 */
-	intel_fb->frontbuffer = intel_frontbuffer_get(obj);
+	intel_fb->frontbuffer = intel_parent_frontbuffer_get(display, obj);
 	if (!intel_fb->frontbuffer) {
 		ret = -ENOMEM;
 		goto err_free_panic;
 	}
 
-	ret = intel_fb_bo_framebuffer_init(obj, mode_cmd);
+	ret = intel_bo_framebuffer_init(obj, mode_cmd);
 	if (ret)
 		goto err_frontbuffer_put;
 
@@ -2304,16 +2303,21 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 		goto err_bo_framebuffer_fini;
 
 	if (intel_fb_uses_dpt(fb)) {
-		struct i915_address_space *vm;
+		struct drm_gem_object *obj = intel_fb_bo(&intel_fb->base);
+		struct intel_dpt *dpt;
+		size_t size = 0;
+
+		if (intel_fb_needs_pot_stride_remap(intel_fb))
+			size = intel_remapped_info_size(&intel_fb->remapped_view.gtt.remapped);
 
-		vm = intel_dpt_create(intel_fb);
-		if (IS_ERR(vm)) {
+		dpt = intel_parent_dpt_create(display, obj, size);
+		if (IS_ERR(dpt)) {
 			drm_dbg_kms(display->drm, "failed to create DPT\n");
-			ret = PTR_ERR(vm);
+			ret = PTR_ERR(dpt);
 			goto err_frontbuffer_put;
 		}
 
-		intel_fb->dpt_vm = vm;
+		intel_fb->dpt = dpt;
 	}
 
 	ret = drm_framebuffer_init(display->drm, fb, &intel_fb_funcs);
@@ -2326,11 +2330,11 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 
 err_free_dpt:
 	if (intel_fb_uses_dpt(fb))
-		intel_dpt_destroy(intel_fb->dpt_vm);
+		intel_parent_dpt_destroy(display, intel_fb->dpt);
 err_bo_framebuffer_fini:
-	intel_fb_bo_framebuffer_fini(obj);
+	intel_bo_framebuffer_fini(obj);
 err_frontbuffer_put:
-	intel_frontbuffer_put(intel_fb->frontbuffer);
+	intel_parent_frontbuffer_put(display, intel_fb->frontbuffer);
 err_free_panic:
 	kfree(intel_fb->panic);
 
@@ -2343,11 +2347,12 @@ intel_user_framebuffer_create(struct drm_device *dev,
 			      const struct drm_format_info *info,
 			      const struct drm_mode_fb_cmd2 *user_mode_cmd)
 {
+	struct intel_display *display = to_intel_display(dev);
 	struct drm_framebuffer *fb;
 	struct drm_gem_object *obj;
 	struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
 
-	obj = intel_fb_bo_lookup_valid_bo(dev, filp, &mode_cmd);
+	obj = intel_bo_framebuffer_lookup(display, filp, &mode_cmd);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
diff --git a/drivers/gpu/drm/i915/display/intel_fb_bo.c b/drivers/gpu/drm/i915/display/intel_fb_bo.c
deleted file mode 100644
index bfecd73d5fa03..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fb_bo.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2021 Intel Corporation
- */
-
-#include <drm/drm_framebuffer.h>
-#include <drm/drm_print.h>
-
-#include "gem/i915_gem_object.h"
-
-#include "i915_drv.h"
-#include "intel_display_core.h"
-#include "intel_display_types.h"
-#include "intel_fb.h"
-#include "intel_fb_bo.h"
-
-void intel_fb_bo_framebuffer_fini(struct drm_gem_object *obj)
-{
-	/* Nothing to do for i915 */
-}
-
-int intel_fb_bo_framebuffer_init(struct drm_gem_object *_obj,
-				 struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	struct intel_display *display = to_intel_display(obj->base.dev);
-	unsigned int tiling, stride;
-
-	i915_gem_object_lock(obj, NULL);
-	tiling = i915_gem_object_get_tiling(obj);
-	stride = i915_gem_object_get_stride(obj);
-	i915_gem_object_unlock(obj);
-
-	if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
-		/*
-		 * If there's a fence, enforce that
-		 * the fb modifier and tiling mode match.
-		 */
-		if (tiling != I915_TILING_NONE &&
-		    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
-			drm_dbg_kms(display->drm,
-				    "tiling_mode doesn't match fb modifier\n");
-			return -EINVAL;
-		}
-	} else {
-		if (tiling == I915_TILING_X) {
-			mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
-		} else if (tiling == I915_TILING_Y) {
-			drm_dbg_kms(display->drm,
-				    "No Y tiling for legacy addfb\n");
-			return -EINVAL;
-		}
-	}
-
-	/*
-	 * gen2/3 display engine uses the fence if present,
-	 * so the tiling mode must match the fb modifier exactly.
-	 */
-	if (DISPLAY_VER(display) < 4 &&
-	    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
-		drm_dbg_kms(display->drm,
-			    "tiling_mode must match fb modifier exactly on gen2/3\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * If there's a fence, enforce that
-	 * the fb pitch and fence stride match.
-	 */
-	if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
-		drm_dbg_kms(display->drm,
-			    "pitch (%d) must match tiling stride (%d)\n",
-			    mode_cmd->pitches[0], stride);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-struct drm_gem_object *
-intel_fb_bo_lookup_valid_bo(struct drm_device *drm,
-			    struct drm_file *filp,
-			    const struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct drm_i915_gem_object *obj;
-
-	obj = i915_gem_object_lookup(filp, mode_cmd->handles[0]);
-	if (!obj)
-		return ERR_PTR(-ENOENT);
-
-	/* object is backed with LMEM for discrete */
-	if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM_0)) {
-		/* object is "remote", not in local memory */
-		i915_gem_object_put(obj);
-		drm_dbg_kms(&i915->drm, "framebuffer must reside in local memory\n");
-		return ERR_PTR(-EREMOTE);
-	}
-
-	return intel_bo_to_drm_bo(obj);
-}
diff --git a/drivers/gpu/drm/i915/display/intel_fb_bo.h b/drivers/gpu/drm/i915/display/intel_fb_bo.h
deleted file mode 100644
index d775773c6c03f..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fb_bo.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2021 Intel Corporation
- */
-
-#ifndef __INTEL_FB_BO_H__
-#define __INTEL_FB_BO_H__
-
-struct drm_device;
-struct drm_file;
-struct drm_framebuffer;
-struct drm_gem_object;
-struct drm_mode_fb_cmd2;
-
-void intel_fb_bo_framebuffer_fini(struct drm_gem_object *obj);
-
-int intel_fb_bo_framebuffer_init(struct drm_gem_object *obj,
-				 struct drm_mode_fb_cmd2 *mode_cmd);
-
-struct drm_gem_object *
-intel_fb_bo_lookup_valid_bo(struct drm_device *drm,
-			    struct drm_file *filp,
-			    const struct drm_mode_fb_cmd2 *user_mode_cmd);
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c
index 7249b784fbbab..738d77a1468a5 100644
--- a/drivers/gpu/drm/i915/display/intel_fb_pin.c
+++ b/drivers/gpu/drm/i915/display/intel_fb_pin.c
@@ -17,7 +17,7 @@
 #include "intel_display_core.h"
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
-#include "intel_dpt.h"
+#include "i915_dpt.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "intel_plane.h"
@@ -27,13 +27,14 @@ intel_fb_pin_to_dpt(const struct drm_framebuffer *fb,
 		    const struct i915_gtt_view *view,
 		    unsigned int alignment,
 		    unsigned long *out_flags,
-		    struct i915_address_space *vm)
+		    struct intel_dpt *dpt)
 {
 	struct drm_device *dev = fb->dev;
 	struct intel_display *display = to_intel_display(dev);
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_gem_object *_obj = intel_fb_bo(fb);
 	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	struct i915_address_space *vm = i915_dpt_to_vm(dpt);
 	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	int ret;
@@ -284,7 +285,7 @@ int intel_plane_pin_fb(struct intel_plane_state *plane_state,
 	} else {
 		unsigned int alignment = intel_plane_fb_min_alignment(plane_state);
 
-		vma = intel_dpt_pin_to_ggtt(fb->dpt_vm, alignment / 512);
+		vma = i915_dpt_pin_to_ggtt(fb->dpt, alignment / 512);
 		if (IS_ERR(vma))
 			return PTR_ERR(vma);
 
@@ -292,9 +293,9 @@ int intel_plane_pin_fb(struct intel_plane_state *plane_state,
 
 		vma = intel_fb_pin_to_dpt(&fb->base, &plane_state->view.gtt,
 					  alignment, &plane_state->flags,
-					  fb->dpt_vm);
+					  fb->dpt);
 		if (IS_ERR(vma)) {
-			intel_dpt_unpin_from_ggtt(fb->dpt_vm);
+			i915_dpt_unpin_from_ggtt(fb->dpt);
 			plane_state->ggtt_vma = NULL;
 			return PTR_ERR(vma);
 		}
@@ -307,7 +308,7 @@ int intel_plane_pin_fb(struct intel_plane_state *plane_state,
 		 * The DPT object contains only one vma, and there is no VT-d
 		 * guard, so the VMA's offset within the DPT is always 0.
 		 */
-		drm_WARN_ON(display->drm, intel_dpt_offset(plane_state->dpt_vma));
+		drm_WARN_ON(display->drm, i915_dpt_offset(plane_state->dpt_vma));
 	}
 
 	/*
@@ -346,7 +347,7 @@ void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
 
 		vma = fetch_and_zero(&old_plane_state->ggtt_vma);
 		if (vma)
-			intel_dpt_unpin_from_ggtt(fb->dpt_vm);
+			i915_dpt_unpin_from_ggtt(fb->dpt);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index 91de383792825..ea0ce00c84743 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -45,7 +45,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
 
-#include "i915_vma.h"
 #include "i9xx_plane_regs.h"
 #include "intel_de.h"
 #include "intel_display_device.h"
@@ -184,7 +183,7 @@ static unsigned int skl_fbc_min_cfb_stride(struct intel_display *display,
 	 * Wa_16011863758: icl+
 	 * Avoid some hardware segment address miscalculation.
 	 */
-	if (DISPLAY_VER(display) >= 11)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16011863758))
 		stride += 64;
 
 	/*
@@ -950,7 +949,7 @@ static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 	}
 
 	/* Wa_1409120013:icl,jsl,tgl,dg1 */
-	if (IS_DISPLAY_VER(display, 11, 12))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_1409120013))
 		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 			     0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 	/*
@@ -958,7 +957,7 @@ static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 	 * Fixes: Screen flicker with FBC and Package C state enabled
 	 * Workaround: Forced SLB invalidation before start of new frame.
 	 */
-	if (intel_display_wa(display, 22014263786))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_22014263786))
 		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 			     0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 
@@ -980,7 +979,7 @@ static void fbc_sys_cache_update_config(struct intel_display *display, u32 reg,
 	 * Fixes: SoC hardware issue in read caching
 	 * Workaround: disable cache read setting which is enabled by default.
 	 */
-	if (!intel_display_wa(display, 14025769978))
+	if (!intel_display_wa(display, INTEL_DISPLAY_WA_14025769978))
 		/* Cache read enable is set by default */
 		reg |= FBC_SYS_CACHE_READ_ENABLE;
 
@@ -1463,7 +1462,7 @@ static void intel_fbc_update_state(struct intel_atomic_state *state,
 		    !intel_fbc_has_fences(display));
 
 	if (plane_state->flags & PLANE_HAS_FENCE)
-		fbc_state->fence_id =  i915_vma_fence_id(plane_state->ggtt_vma);
+		fbc_state->fence_id = intel_parent_vma_fence_id(display, plane_state->ggtt_vma);
 	else
 		fbc_state->fence_id = -1;
 
@@ -1490,7 +1489,7 @@ static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
 	 */
 	return DISPLAY_VER(display) >= 9 ||
 		(plane_state->flags & PLANE_HAS_FENCE &&
-		 i915_vma_fence_id(plane_state->ggtt_vma) != -1);
+		 intel_parent_vma_fence_id(display, plane_state->ggtt_vma) != -1);
 }
 
 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
@@ -1613,7 +1612,7 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 		return 0;
 	}
 
-	if (intel_display_wa(display, 16023588340)) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16023588340)) {
 		plane_state->no_fbc_reason = "Wa_16023588340";
 		return 0;
 	}
@@ -1623,7 +1622,7 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 	 * Fixes: Underrun during media decode
 	 * Workaround: Do not enable FBC
 	 */
-	if (intel_display_wa(display, 15018326506)) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_15018326506)) {
 		plane_state->no_fbc_reason = "Wa_15018326506";
 		return 0;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index 44f4fcce526e1..bdaaf3edba0cb 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -47,7 +47,6 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 
-#include "i915_vma.h"
 #include "intel_bo.h"
 #include "intel_display_core.h"
 #include "intel_display_rpm.h"
@@ -343,9 +342,7 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
 
-	drm_dbg_kms(display->drm, "allocated %dx%d fb: 0x%08x\n",
-		    fb->base.width, fb->base.height,
-		    i915_ggtt_offset(vma));
+	drm_dbg_kms(display->drm, "allocated %dx%d fb\n", fb->base.width, fb->base.height);
 	ifbdev->fb = fb;
 	ifbdev->vma = vma;
 	ifbdev->vma_flags = flags;
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c b/drivers/gpu/drm/i915/display/intel_fbdev_fb.c
index c3202ba141c58..6f84eb6355ded 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev_fb.c
@@ -17,28 +17,40 @@ u32 intel_fbdev_fb_pitch_align(u32 stride)
 	return ALIGN(stride, 64);
 }
 
+bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+
+	/* Skip stolen on MTL as Wa_22018444074 mitigation. */
+	if (IS_METEORLAKE(i915))
+		return false;
+
+	/*
+	 * If the FB is too big, just don't use it since fbdev is not very
+	 * important and we should probably use that space with FBC or other
+	 * features.
+	 */
+	return i915->dsm.usable_size >= size * 2;
+}
+
 struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size)
 {
-	struct drm_i915_private *dev_priv = to_i915(drm);
+	struct drm_i915_private *i915 = to_i915(drm);
 	struct drm_i915_gem_object *obj;
 
 	obj = ERR_PTR(-ENODEV);
-	if (HAS_LMEM(dev_priv)) {
-		obj = i915_gem_object_create_lmem(dev_priv, size,
+	if (HAS_LMEM(i915)) {
+		obj = i915_gem_object_create_lmem(i915, size,
 						  I915_BO_ALLOC_CONTIGUOUS |
 						  I915_BO_ALLOC_USER);
 	} else {
-		/*
-		 * If the FB is too big, just don't use it since fbdev is not very
-		 * important and we should probably use that space with FBC or other
-		 * features.
-		 *
-		 * Also skip stolen on MTL as Wa_22018444074 mitigation.
-		 */
-		if (!IS_METEORLAKE(dev_priv) && size * 2 < dev_priv->dsm.usable_size)
-			obj = i915_gem_object_create_stolen(dev_priv, size);
+		if (intel_fbdev_fb_prefer_stolen(drm, size))
+			obj = i915_gem_object_create_stolen(i915, size);
+		else
+			drm_info(drm, "Allocating fbdev: Stolen memory not preferred.\n");
+
 		if (IS_ERR(obj))
-			obj = i915_gem_object_create_shmem(dev_priv, size);
+			obj = i915_gem_object_create_shmem(i915, size);
 	}
 
 	if (IS_ERR(obj)) {
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h b/drivers/gpu/drm/i915/display/intel_fbdev_fb.h
index fd0b3775dc1f2..34ed2b9c2b4fc 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h
+++ b/drivers/gpu/drm/i915/display/intel_fbdev_fb.h
@@ -19,5 +19,6 @@ struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size
 void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj);
 int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info,
 			     struct drm_gem_object *obj, struct i915_vma *vma);
+bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size);
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c
index 5bb0090dd5edf..24ce8a7842c7d 100644
--- a/drivers/gpu/drm/i915/display/intel_fdi.c
+++ b/drivers/gpu/drm/i915/display/intel_fdi.c
@@ -8,7 +8,6 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_crtc.h"
 #include "intel_ddi.h"
diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
index b413b3e871d82..bf047180def9f 100644
--- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
@@ -29,7 +29,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_irq.h"
 #include "intel_display_regs.h"
diff --git a/drivers/gpu/drm/i915/display/intel_flipq.c b/drivers/gpu/drm/i915/display/intel_flipq.c
index 1e9550cb66a3a..253dc2e96d2d6 100644
--- a/drivers/gpu/drm/i915/display/intel_flipq.c
+++ b/drivers/gpu/drm/i915/display/intel_flipq.c
@@ -12,6 +12,7 @@
 #include "intel_display_core.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_dmc_regs.h"
 #include "intel_dsb.h"
@@ -447,19 +448,11 @@ void intel_flipq_add(struct intel_crtc *crtc,
 	intel_flipq_sw_dmc_wake(crtc);
 }
 
-/* Wa_18034343758 */
-static bool need_dmc_halt_wa(struct intel_display *display)
-{
-	return DISPLAY_VER(display) == 20 ||
-		(display->platform.pantherlake &&
-		 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0));
-}
-
 void intel_flipq_wait_dmc_halt(struct intel_dsb *dsb, struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
 
-	if (need_dmc_halt_wa(display))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_18034343758))
 		intel_dsb_wait_usec(dsb, 2);
 }
 
@@ -467,6 +460,6 @@ void intel_flipq_unhalt_dmc(struct intel_dsb *dsb, struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
 
-	if (need_dmc_halt_wa(display))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_18034343758))
 		intel_dsb_reg_write(dsb, PIPEDMC_CTL(crtc->pipe), 0);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index 03c4978fa5ecf..705742e117caf 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -58,13 +58,13 @@
 #include <drm/drm_gem.h>
 #include <drm/drm_print.h>
 
-#include "intel_bo.h"
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
 #include "intel_dp.h"
 #include "intel_drrs.h"
 #include "intel_fbc.h"
 #include "intel_frontbuffer.h"
+#include "intel_parent.h"
 #include "intel_psr.h"
 #include "intel_tdf.h"
 
@@ -123,9 +123,9 @@ void intel_frontbuffer_flip(struct intel_display *display,
 	frontbuffer_flush(display, frontbuffer_bits, ORIGIN_FLIP);
 }
 
-void __intel_fb_invalidate(struct intel_frontbuffer *front,
-			   enum fb_op_origin origin,
-			   unsigned int frontbuffer_bits)
+void __intel_frontbuffer_invalidate(struct intel_frontbuffer *front,
+				    enum fb_op_origin origin,
+				    unsigned int frontbuffer_bits)
 {
 	struct intel_display *display = front->display;
 
@@ -143,14 +143,14 @@ void __intel_fb_invalidate(struct intel_frontbuffer *front,
 	intel_fbc_invalidate(display, frontbuffer_bits, origin);
 }
 
-void __intel_fb_flush(struct intel_frontbuffer *front,
-		      enum fb_op_origin origin,
-		      unsigned int frontbuffer_bits)
+void __intel_frontbuffer_flush(struct intel_frontbuffer *front,
+			       enum fb_op_origin origin,
+			       unsigned int frontbuffer_bits)
 {
 	struct intel_display *display = front->display;
 
 	if (origin == ORIGIN_DIRTYFB)
-		intel_bo_frontbuffer_flush_for_display(front);
+		intel_parent_frontbuffer_flush_for_display(display, front);
 
 	if (origin == ORIGIN_CS) {
 		spin_lock(&display->fb_tracking.lock);
@@ -164,18 +164,13 @@ void __intel_fb_flush(struct intel_frontbuffer *front,
 		frontbuffer_flush(display, frontbuffer_bits, origin);
 }
 
-static void intel_frontbuffer_ref(struct intel_frontbuffer *front)
-{
-	intel_bo_frontbuffer_ref(front);
-}
-
 static void intel_frontbuffer_flush_work(struct work_struct *work)
 {
 	struct intel_frontbuffer *front =
 		container_of(work, struct intel_frontbuffer, flush_work);
 
 	intel_frontbuffer_flush(front, ORIGIN_DIRTYFB);
-	intel_frontbuffer_put(front);
+	intel_parent_frontbuffer_put(front->display, front);
 }
 
 /**
@@ -190,9 +185,9 @@ void intel_frontbuffer_queue_flush(struct intel_frontbuffer *front)
 	if (!front)
 		return;
 
-	intel_frontbuffer_ref(front);
+	intel_parent_frontbuffer_ref(front->display, front);
 	if (!schedule_work(&front->flush_work))
-		intel_frontbuffer_put(front);
+		intel_parent_frontbuffer_put(front->display, front);
 }
 
 void intel_frontbuffer_init(struct intel_frontbuffer *front, struct drm_device *drm)
@@ -207,16 +202,6 @@ void intel_frontbuffer_fini(struct intel_frontbuffer *front)
 	drm_WARN_ON(front->display->drm, atomic_read(&front->bits));
 }
 
-struct intel_frontbuffer *intel_frontbuffer_get(struct drm_gem_object *obj)
-{
-	return intel_bo_frontbuffer_get(obj);
-}
-
-void intel_frontbuffer_put(struct intel_frontbuffer *front)
-{
-	intel_bo_frontbuffer_put(front);
-}
-
 /**
  * intel_frontbuffer_track - update frontbuffer tracking
  * @old: current buffer for the frontbuffer slots
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.h b/drivers/gpu/drm/i915/display/intel_frontbuffer.h
index 22677acb4c06f..a89ce352b12b1 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.h
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.h
@@ -66,14 +66,9 @@ struct intel_frontbuffer {
 void intel_frontbuffer_flip(struct intel_display *display,
 			    unsigned frontbuffer_bits);
 
-void intel_frontbuffer_put(struct intel_frontbuffer *front);
-
-struct intel_frontbuffer *
-intel_frontbuffer_get(struct drm_gem_object *obj);
-
-void __intel_fb_invalidate(struct intel_frontbuffer *front,
-			   enum fb_op_origin origin,
-			   unsigned int frontbuffer_bits);
+void __intel_frontbuffer_invalidate(struct intel_frontbuffer *front,
+				    enum fb_op_origin origin,
+				    unsigned int frontbuffer_bits);
 
 /**
  * intel_frontbuffer_invalidate - invalidate frontbuffer object
@@ -98,13 +93,13 @@ static inline bool intel_frontbuffer_invalidate(struct intel_frontbuffer *front,
 	if (!frontbuffer_bits)
 		return false;
 
-	__intel_fb_invalidate(front, origin, frontbuffer_bits);
+	__intel_frontbuffer_invalidate(front, origin, frontbuffer_bits);
 	return true;
 }
 
-void __intel_fb_flush(struct intel_frontbuffer *front,
-		      enum fb_op_origin origin,
-		      unsigned int frontbuffer_bits);
+void __intel_frontbuffer_flush(struct intel_frontbuffer *front,
+			       enum fb_op_origin origin,
+			       unsigned int frontbuffer_bits);
 
 /**
  * intel_frontbuffer_flush - flush frontbuffer object
@@ -126,7 +121,7 @@ static inline void intel_frontbuffer_flush(struct intel_frontbuffer *front,
 	if (!frontbuffer_bits)
 		return;
 
-	__intel_fb_flush(front, origin, frontbuffer_bits);
+	__intel_frontbuffer_flush(front, origin, frontbuffer_bits);
 }
 
 void intel_frontbuffer_queue_flush(struct intel_frontbuffer *front);
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index a7bce0c6a17e1..ea5cf8f51b316 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -35,7 +35,6 @@
 #include <drm/drm_print.h>
 #include <drm/display/drm_hdcp_helper.h>
 
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
@@ -251,7 +250,7 @@ static u32 get_reserved(struct intel_gmbus *bus)
 	preserve_bits |= GPIO_DATA_PULLUP_DISABLE | GPIO_CLOCK_PULLUP_DISABLE;
 
 	/* Wa_16025573575: the masks bits need to be preserved through out */
-	if (intel_display_wa(display, 16025573575))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16025573575))
 		preserve_bits |= GPIO_CLOCK_DIR_MASK | GPIO_CLOCK_VAL_MASK |
 				 GPIO_DATA_DIR_MASK | GPIO_DATA_VAL_MASK;
 
@@ -343,7 +342,7 @@ intel_gpio_pre_xfer(struct i2c_adapter *adapter)
 	if (display->platform.pineview)
 		pnv_gmbus_clock_gating(display, false);
 
-	if (intel_display_wa(display, 16025573575))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16025573575))
 		ptl_handle_mask_bits(bus, true);
 
 	set_data(bus, 1);
@@ -364,7 +363,7 @@ intel_gpio_post_xfer(struct i2c_adapter *adapter)
 	if (display->platform.pineview)
 		pnv_gmbus_clock_gating(display, true);
 
-	if (intel_display_wa(display, 16025573575))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16025573575))
 		ptl_handle_mask_bits(bus, false);
 }
 
@@ -496,8 +495,10 @@ gmbus_xfer_read_chunk(struct intel_display *display,
 
 		val = intel_de_read_fw(display, GMBUS3(display));
 		do {
-			if (extra_byte_added && len == 1)
+			if (extra_byte_added && len == 1) {
+				len--;
 				break;
+			}
 
 			*buf++ = val & 0xff;
 			val >>= 8;
@@ -693,7 +694,7 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num,
 			goto clear_err;
 	}
 
-	/* Generate a STOP condition on the bus. Note that gmbus can't generata
+	/* Generate a STOP condition on the bus. Note that gmbus can't generate
 	 * a STOP on the very first cycle. To simplify the code we
 	 * unconditionally generate the STOP condition with an additional gmbus
 	 * cycle. */
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 555d35bae8878..892eab4b6f925 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -17,8 +17,8 @@
 #include <drm/display/drm_hdcp_helper.h>
 #include <drm/drm_print.h>
 #include <drm/intel/i915_component.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "intel_connector.h"
 #include "intel_de.h"
 #include "intel_display_jiffies.h"
@@ -33,7 +33,6 @@
 #include "intel_hdcp_regs.h"
 #include "intel_hdcp_shim.h"
 #include "intel_parent.h"
-#include "intel_pcode.h"
 #include "intel_step.h"
 
 #define USE_HDCP_GSC(__display)		(DISPLAY_VER(__display) >= 14)
@@ -76,7 +75,6 @@ static int intel_conn_to_vcpi(struct intel_atomic_state *state,
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_mst_topology_state *mst_state;
-	int vcpi = 0;
 
 	/* For HDMI this is forced to be 0x0. For DP SST also this is 0x0. */
 	if (!connector->mst.port)
@@ -87,15 +85,9 @@ static int intel_conn_to_vcpi(struct intel_atomic_state *state,
 	mst_state = to_drm_dp_mst_topology_state(mgr->base.state);
 	payload = drm_atomic_get_mst_payload_state(mst_state, connector->mst.port);
 	if (drm_WARN_ON(mgr->dev, !payload))
-		goto out;
+		return 0;
 
-	vcpi = payload->vcpi;
-	if (drm_WARN_ON(mgr->dev, vcpi < 0)) {
-		vcpi = 0;
-		goto out;
-	}
-out:
-	return vcpi;
+	return payload->vcpi;
 }
 
 /*
@@ -398,7 +390,7 @@ static int intel_hdcp_load_keys(struct intel_display *display)
 	 * Mailbox interface.
 	 */
 	if (DISPLAY_VER(display) == 9 && !display->platform.broxton) {
-		ret = intel_pcode_write(display->drm, SKL_PCODE_LOAD_HDCP_KEYS, 1);
+		ret = intel_parent_pcode_write(display, SKL_PCODE_LOAD_HDCP_KEYS, 1);
 		if (ret) {
 			drm_err(display->drm,
 				"Failed to initiate HDCP key load (%d)\n",
@@ -2239,7 +2231,7 @@ static void intel_hdcp_check_work(struct work_struct *work)
 	if (drm_connector_is_unregistered(&connector->base))
 		return;
 
-	if (!intel_hdcp2_check_link(connector))
+	if (!hdcp->force_hdcp14 && !intel_hdcp2_check_link(connector))
 		queue_delayed_work(display->wq.unordered, &hdcp->check_work,
 				   DRM_HDCP2_CHECK_PERIOD_MS);
 	else if (!intel_hdcp_check_link(connector))
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 055e68810d0d6..05e898d10a2be 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -56,6 +56,7 @@
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dp.h"
+#include "intel_dpll.h"
 #include "intel_gmbus.h"
 #include "intel_hdcp.h"
 #include "intel_hdcp_regs.h"
@@ -70,16 +71,14 @@
 
 bool intel_hdmi_is_frl(u32 clock)
 {
-	switch (clock) {
-	case 300000: /* 3 Gbps */
-	case 600000: /* 6 Gbps */
-	case 800000: /* 8 Gbps */
-	case 1000000: /* 10 Gbps */
-	case 1200000: /* 12 Gbps */
-		return true;
-	default:
-		return false;
-	}
+	u32 rates[] = { 300000, 600000, 800000, 1000000, 1200000 };
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(rates); i++)
+		if (intel_dpll_clock_matches(clock, rates[i]))
+			return true;
+
+	return false;
 }
 
 static void
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
index 82c39e4ffa371..8865cb2ac5697 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
@@ -5,7 +5,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_irq.h"
 #include "intel_display_regs.h"
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.c b/drivers/gpu/drm/i915/display/intel_lt_phy.c
index 6cdae03ee172c..eced8493e566e 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.c
@@ -5,7 +5,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_cx0_phy.h"
 #include "intel_cx0_phy_regs.h"
 #include "intel_ddi.h"
@@ -14,6 +13,7 @@
 #include "intel_display.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_dpll.h"
 #include "intel_dpll_mgr.h"
 #include "intel_hdmi.h"
 #include "intel_lt_phy.h"
@@ -60,7 +60,6 @@ struct lt_phy_params {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_rbr = {
-	.clock = 162000,
 	.config = {
 		0x83,
 		0x2d,
@@ -114,7 +113,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_rbr = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr1 = {
-	.clock = 270000,
 	.config = {
 		0x8b,
 		0x2d,
@@ -168,7 +166,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr1 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr2 = {
-	.clock = 540000,
 	.config = {
 		0x93,
 		0x2d,
@@ -222,7 +219,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr2 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr3 = {
-	.clock = 810000,
 	.config = {
 		0x9b,
 		0x2d,
@@ -276,7 +272,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_hbr3 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr10 = {
-	.clock = 1000000,
 	.config = {
 		0x43,
 		0x2d,
@@ -330,7 +325,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr10 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr13_5 = {
-	.clock = 1350000,
 	.config = {
 		0xcb,
 		0x2d,
@@ -384,7 +378,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr13_5 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr20 = {
-	.clock = 2000000,
 	.config = {
 		0x53,
 		0x2d,
@@ -437,19 +430,35 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_dp_uhbr20 = {
 	},
 };
 
-static const struct intel_lt_phy_pll_state * const xe3plpd_lt_dp_tables[] = {
-	&xe3plpd_lt_dp_rbr,
-	&xe3plpd_lt_dp_hbr1,
-	&xe3plpd_lt_dp_hbr2,
-	&xe3plpd_lt_dp_hbr3,
-	&xe3plpd_lt_dp_uhbr10,
-	&xe3plpd_lt_dp_uhbr13_5,
-	&xe3plpd_lt_dp_uhbr20,
-	NULL,
+struct intel_lt_phy_pll_params {
+	const char *name;
+	bool is_hdmi;
+	int clock_rate;
+	const struct intel_lt_phy_pll_state *state;
+};
+
+#define __LT_PHY_PLL_PARAMS(__is_hdmi, __clock_rate, __state)    { \
+	.name = __stringify(__state), \
+	.is_hdmi = __is_hdmi, \
+	.clock_rate = __clock_rate, \
+	.state = &__state, \
+}
+
+#define LT_PHY_PLL_HDMI_PARAMS(__clock_rate, __state)	__LT_PHY_PLL_PARAMS(true, __clock_rate, __state)
+#define LT_PHY_PLL_DP_PARAMS(__clock_rate, __state)	__LT_PHY_PLL_PARAMS(false, __clock_rate, __state)
+
+static const struct intel_lt_phy_pll_params xe3plpd_lt_dp_tables[] = {
+	LT_PHY_PLL_DP_PARAMS(162000, xe3plpd_lt_dp_rbr),
+	LT_PHY_PLL_DP_PARAMS(270000, xe3plpd_lt_dp_hbr1),
+	LT_PHY_PLL_DP_PARAMS(540000, xe3plpd_lt_dp_hbr2),
+	LT_PHY_PLL_DP_PARAMS(810000, xe3plpd_lt_dp_hbr3),
+	LT_PHY_PLL_DP_PARAMS(1000000, xe3plpd_lt_dp_uhbr10),
+	LT_PHY_PLL_DP_PARAMS(1350000, xe3plpd_lt_dp_uhbr13_5),
+	LT_PHY_PLL_DP_PARAMS(2000000, xe3plpd_lt_dp_uhbr20),
+	{}
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_2_16 = {
-	.clock = 216000,
 	.config = {
 		0xa3,
 		0x2d,
@@ -503,7 +512,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_2_16 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_2_43 = {
-	.clock = 243000,
 	.config = {
 		0xab,
 		0x2d,
@@ -557,7 +565,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_2_43 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_3_24 = {
-	.clock = 324000,
 	.config = {
 		0xb3,
 		0x2d,
@@ -611,7 +618,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_3_24 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_4_32 = {
-	.clock = 432000,
 	.config = {
 		0xbb,
 		0x2d,
@@ -665,7 +671,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_4_32 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_6_75 = {
-	.clock = 675000,
 	.config = {
 		0xdb,
 		0x2d,
@@ -718,21 +723,20 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_edp_6_75 = {
 	},
 };
 
-static const struct intel_lt_phy_pll_state * const xe3plpd_lt_edp_tables[] = {
-	&xe3plpd_lt_dp_rbr,
-	&xe3plpd_lt_edp_2_16,
-	&xe3plpd_lt_edp_2_43,
-	&xe3plpd_lt_dp_hbr1,
-	&xe3plpd_lt_edp_3_24,
-	&xe3plpd_lt_edp_4_32,
-	&xe3plpd_lt_dp_hbr2,
-	&xe3plpd_lt_edp_6_75,
-	&xe3plpd_lt_dp_hbr3,
-	NULL,
+static const struct intel_lt_phy_pll_params xe3plpd_lt_edp_tables[] = {
+	LT_PHY_PLL_DP_PARAMS(162000, xe3plpd_lt_dp_rbr),
+	LT_PHY_PLL_DP_PARAMS(216000, xe3plpd_lt_edp_2_16),
+	LT_PHY_PLL_DP_PARAMS(243000, xe3plpd_lt_edp_2_43),
+	LT_PHY_PLL_DP_PARAMS(270000, xe3plpd_lt_dp_hbr1),
+	LT_PHY_PLL_DP_PARAMS(324000, xe3plpd_lt_edp_3_24),
+	LT_PHY_PLL_DP_PARAMS(432000, xe3plpd_lt_edp_4_32),
+	LT_PHY_PLL_DP_PARAMS(540000, xe3plpd_lt_dp_hbr2),
+	LT_PHY_PLL_DP_PARAMS(675000, xe3plpd_lt_edp_6_75),
+	LT_PHY_PLL_DP_PARAMS(810000, xe3plpd_lt_dp_hbr3),
+	{}
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_252 = {
-	.clock = 25200,
 	.config = {
 		0x84,
 		0x2d,
@@ -785,62 +789,7 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_252 = {
 	},
 };
 
-static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_272 = {
-	.clock = 27200,
-	.config = {
-		0x84,
-		0x2d,
-		0x0,
-	},
-	.addr_msb = {
-		0x87,
-		0x87,
-		0x87,
-		0x87,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-		0x88,
-	},
-	.addr_lsb = {
-		0x10,
-		0x0c,
-		0x14,
-		0xe4,
-		0x0c,
-		0x10,
-		0x14,
-		0x18,
-		0x48,
-		0x40,
-		0x4c,
-		0x24,
-		0x44,
-	},
-	.data = {
-		{ 0x0,  0x4c, 0x2,  0x0  },
-		{ 0x0b, 0x15, 0x26, 0xa0 },
-		{ 0x60, 0x0,  0x0,  0x0  },
-		{ 0x8,  0x4,  0x96, 0x28 },
-		{ 0xfa, 0x0c, 0x84, 0x11 },
-		{ 0x80, 0x0f, 0xd9, 0x53 },
-		{ 0x86, 0x0,  0x0,  0x0  },
-		{ 0x1,  0xa0, 0x1,  0x0  },
-		{ 0x4b, 0x0,  0x0,  0x0  },
-		{ 0x28, 0x0,  0x0,  0x0  },
-		{ 0x0,  0x14, 0x2a, 0x14 },
-		{ 0x0,  0x0,  0x0,  0x0  },
-		{ 0x0,  0x0,  0x0,  0x0  },
-	},
-};
-
 static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_742p5 = {
-	.clock = 74250,
 	.config = {
 		0x84,
 		0x2d,
@@ -894,7 +843,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_742p5 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_1p485 = {
-	.clock = 148500,
 	.config = {
 		0x84,
 		0x2d,
@@ -948,7 +896,6 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_1p485 = {
 };
 
 static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_5p94 = {
-	.clock = 594000,
 	.config = {
 		0x84,
 		0x2d,
@@ -1001,13 +948,12 @@ static const struct intel_lt_phy_pll_state xe3plpd_lt_hdmi_5p94 = {
 	},
 };
 
-static const struct intel_lt_phy_pll_state * const xe3plpd_lt_hdmi_tables[] = {
-	&xe3plpd_lt_hdmi_252,
-	&xe3plpd_lt_hdmi_272,
-	&xe3plpd_lt_hdmi_742p5,
-	&xe3plpd_lt_hdmi_1p485,
-	&xe3plpd_lt_hdmi_5p94,
-	NULL,
+static const struct intel_lt_phy_pll_params xe3plpd_lt_hdmi_tables[] = {
+	LT_PHY_PLL_HDMI_PARAMS(25200, xe3plpd_lt_hdmi_252),
+	LT_PHY_PLL_HDMI_PARAMS(74250, xe3plpd_lt_hdmi_742p5),
+	LT_PHY_PLL_HDMI_PARAMS(148500, xe3plpd_lt_hdmi_1p485),
+	LT_PHY_PLL_HDMI_PARAMS(594000, xe3plpd_lt_hdmi_5p94),
+	{}
 };
 
 static u8 intel_lt_phy_get_owned_lane_mask(struct intel_encoder *encoder)
@@ -1106,7 +1052,7 @@ static int __intel_lt_phy_p2p_write_once(struct intel_encoder *encoder,
 	 * This is the time PHY takes to settle down after programming the PHY.
 	 */
 	udelay(150);
-	intel_clear_response_ready_flag(encoder, lane);
+	intel_cx0_clear_response_ready_flag(encoder, lane);
 	intel_lt_phy_clear_status_p2p(encoder, lane);
 
 	return 0;
@@ -1346,7 +1292,7 @@ static void intel_lt_phy_transaction_end(struct intel_encoder *encoder, struct r
 	intel_display_power_put(display, POWER_DOMAIN_DC_OFF, wakeref);
 }
 
-static const struct intel_lt_phy_pll_state * const *
+static const struct intel_lt_phy_pll_params *
 intel_lt_phy_pll_tables_get(struct intel_crtc_state *crtc_state,
 			    struct intel_encoder *encoder)
 {
@@ -1680,7 +1626,8 @@ intel_lt_phy_calculate_hdmi_state(struct intel_lt_phy_pll_state *lt_state,
 }
 
 static int
-intel_lt_phy_calc_hdmi_port_clock(const struct intel_crtc_state *crtc_state)
+intel_lt_phy_calc_hdmi_port_clock(struct intel_display *display,
+				  const struct intel_lt_phy_pll_state *lt_state)
 {
 #define REGVAL(i) (				\
 	(lt_state->data[i][3])		|	\
@@ -1689,9 +1636,6 @@ intel_lt_phy_calc_hdmi_port_clock(const struct intel_crtc_state *crtc_state)
 	(lt_state->data[i][0] << 24)		\
 )
 
-	struct intel_display *display = to_intel_display(crtc_state);
-	const struct intel_lt_phy_pll_state *lt_state =
-		&crtc_state->dpll_hw_state.ltpll;
 	int clk = 0;
 	u32 d8, pll_reg_5, pll_reg_3, pll_reg_57, m2div_frac, m2div_int;
 	u64 temp0, temp1;
@@ -1737,7 +1681,7 @@ intel_lt_phy_calc_hdmi_port_clock(const struct intel_crtc_state *crtc_state)
 	if (d8 == 0) {
 		drm_WARN_ON(display->drm,
 			    "Invalid port clock using lowest HDMI portclock\n");
-		return xe3plpd_lt_hdmi_252.clock;
+		return xe3plpd_lt_hdmi_tables[0].clock_rate;
 	}
 	m2div_int = (pll_reg_3  & REG_GENMASK(14, 5)) >> 5;
 	temp0 = ((u64)m2div_frac * REF_CLK_KHZ) >> 32;
@@ -1749,13 +1693,10 @@ intel_lt_phy_calc_hdmi_port_clock(const struct intel_crtc_state *crtc_state)
 }
 
 int
-intel_lt_phy_calc_port_clock(struct intel_encoder *encoder,
-			     const struct intel_crtc_state *crtc_state)
+intel_lt_phy_calc_port_clock(struct intel_display *display,
+			     const struct intel_lt_phy_pll_state *lt_state)
 {
-	struct intel_display *display = to_intel_display(encoder);
 	int clk;
-	const struct intel_lt_phy_pll_state *lt_state =
-		&crtc_state->dpll_hw_state.ltpll;
 	u8 mode, rate;
 
 	mode = REG_FIELD_GET8(LT_PHY_VDR_MODE_ENCODING_MASK,
@@ -1771,10 +1712,10 @@ intel_lt_phy_calc_port_clock(struct intel_encoder *encoder,
 				      lt_state->config[0]);
 		clk = intel_lt_phy_get_dp_clock(rate);
 	} else if (mode == MODE_HDMI_20) {
-		clk = intel_lt_phy_calc_hdmi_port_clock(crtc_state);
+		clk = intel_lt_phy_calc_hdmi_port_clock(display, lt_state);
 	} else {
 		drm_WARN_ON(display->drm, "Unsupported LT PHY Mode!\n");
-		clk = xe3plpd_lt_hdmi_252.clock;
+		clk = 25200;
 	}
 
 	return clk;
@@ -1784,16 +1725,20 @@ int
 intel_lt_phy_pll_calc_state(struct intel_crtc_state *crtc_state,
 			    struct intel_encoder *encoder)
 {
-	const struct intel_lt_phy_pll_state * const *tables;
+	struct intel_display *display = to_intel_display(crtc_state);
+	const struct intel_lt_phy_pll_params *tables;
 	int i;
 
 	tables = intel_lt_phy_pll_tables_get(crtc_state, encoder);
 	if (!tables)
 		return -EINVAL;
 
-	for (i = 0; tables[i]; i++) {
-		if (crtc_state->port_clock == tables[i]->clock) {
-			crtc_state->dpll_hw_state.ltpll = *tables[i];
+	for (i = 0; tables[i].name; i++) {
+		int clock = intel_lt_phy_calc_port_clock(display, tables[i].state);
+
+		drm_WARN_ON(display->drm, !intel_dpll_clock_matches(clock, tables[i].clock_rate));
+		if (intel_dpll_clock_matches(crtc_state->port_clock, clock)) {
+			crtc_state->dpll_hw_state.ltpll = *tables[i].state;
 			if (intel_crtc_has_dp_encoder(crtc_state)) {
 				if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
 					crtc_state->dpll_hw_state.ltpll.config[2] = 1;
@@ -2212,6 +2157,9 @@ bool
 intel_lt_phy_pll_compare_hw_state(const struct intel_lt_phy_pll_state *a,
 				  const struct intel_lt_phy_pll_state *b)
 {
+	if (a->tbt_mode || b->tbt_mode)
+		return true;
+
 	/*
 	 * With LT PHY values other than VDR0_CONFIG and VDR2_CONFIG are
 	 * unreliable. They cannot always be read back since internally
@@ -2254,8 +2202,6 @@ void intel_lt_phy_pll_readout_hw_state(struct intel_encoder *encoder,
 						  LT_PHY_VDR_X_DATAY(i, j));
 	}
 
-	pll_state->clock =
-		intel_lt_phy_calc_port_clock(encoder, crtc_state);
 	intel_lt_phy_transaction_end(encoder, wakeref);
 }
 
@@ -2319,3 +2265,66 @@ void intel_xe3plpd_pll_disable(struct intel_encoder *encoder)
 		intel_lt_phy_pll_disable(encoder);
 
 }
+
+static void intel_lt_phy_pll_verify_clock(struct intel_display *display,
+					  int precomputed_clock,
+					  const char *pll_state_name,
+					  const struct intel_lt_phy_pll_state *pll_state,
+					  bool is_precomputed_state)
+{
+	struct drm_printer p;
+	int clock;
+
+	clock = intel_lt_phy_calc_port_clock(display, pll_state);
+
+	if (intel_dpll_clock_matches(clock, precomputed_clock))
+		return;
+
+	drm_warn(display->drm,
+		 "PLL state %s (%s): clock difference too high: computed %d, pre-computed %d\n",
+		 pll_state_name,
+		 is_precomputed_state ? "precomputed" : "computed",
+		 clock, precomputed_clock);
+
+	if (!drm_debug_enabled(DRM_UT_KMS))
+		return;
+
+	p = drm_dbg_printer(display->drm, DRM_UT_KMS, NULL);
+
+	drm_printf(&p, "PLL state %s (%s):\n",
+		   pll_state_name,
+		   is_precomputed_state ? "precomputed" : "computed");
+	intel_lt_phy_dump_hw_state(display, pll_state);
+}
+
+static void intel_lt_phy_pll_verify_params(struct intel_display *display,
+					   const struct intel_lt_phy_pll_params *pll_params)
+{
+	struct intel_lt_phy_pll_state pll_state;
+
+	intel_lt_phy_pll_verify_clock(display, pll_params->clock_rate, pll_params->name, pll_params->state, true);
+
+	if (!pll_params->is_hdmi)
+		return;
+
+	if (intel_lt_phy_calculate_hdmi_state(&pll_state, pll_params->clock_rate) != 0)
+		return;
+
+	intel_lt_phy_pll_verify_clock(display, pll_params->clock_rate, pll_params->name, &pll_state, false);
+}
+
+static void intel_lt_phy_pll_verify_tables(struct intel_display *display,
+					   const struct intel_lt_phy_pll_params *tables)
+{
+	int i;
+
+	for (i = 0; tables[i].name; i++)
+		intel_lt_phy_pll_verify_params(display, &tables[i]);
+}
+
+void intel_lt_phy_verify_plls(struct intel_display *display)
+{
+	intel_lt_phy_pll_verify_tables(display, xe3plpd_lt_dp_tables);
+	intel_lt_phy_pll_verify_tables(display, xe3plpd_lt_edp_tables);
+	intel_lt_phy_pll_verify_tables(display, xe3plpd_lt_hdmi_tables);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.h b/drivers/gpu/drm/i915/display/intel_lt_phy.h
index bf41858f1bc37..db905668f86d1 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.h
@@ -21,8 +21,8 @@ void intel_lt_phy_pll_disable(struct intel_encoder *encoder);
 int
 intel_lt_phy_pll_calc_state(struct intel_crtc_state *crtc_state,
 			    struct intel_encoder *encoder);
-int intel_lt_phy_calc_port_clock(struct intel_encoder *encoder,
-				 const struct intel_crtc_state *crtc_state);
+int intel_lt_phy_calc_port_clock(struct intel_display *display,
+				 const struct intel_lt_phy_pll_state *lt_state);
 void intel_lt_phy_set_signal_levels(struct intel_encoder *encoder,
 				    const struct intel_crtc_state *crtc_state);
 void intel_lt_phy_dump_hw_state(struct intel_display *display,
@@ -41,5 +41,6 @@ intel_lt_phy_calculate_hdmi_state(struct intel_lt_phy_pll_state *lt_state,
 void intel_xe3plpd_pll_enable(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *crtc_state);
 void intel_xe3plpd_pll_disable(struct intel_encoder *encoder);
+void intel_lt_phy_verify_plls(struct intel_display *display);
 
 #endif /* __INTEL_LT_PHY_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy_regs.h b/drivers/gpu/drm/i915/display/intel_lt_phy_regs.h
index 37e46fb9abde4..ff6d7829dbb9c 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy_regs.h
@@ -6,12 +6,12 @@
 #ifndef __INTEL_LT_PHY_REGS_H__
 #define __INTEL_LT_PHY_REGS_H__
 
-#define XE3PLPD_MSGBUS_TIMEOUT_FAST_US	500
+#define XE3PLPD_MSGBUS_TIMEOUT_FAST_US		500
 #define XE3PLPD_MACCLK_TURNON_LATENCY_MS	2
-#define XE3PLPD_MACCLK_TURNOFF_LATENCY_US	1
+#define XE3PLPD_MACCLK_TURNOFF_LATENCY_US	10
 #define XE3PLPD_RATE_CALIB_DONE_LATENCY_MS	1
-#define XE3PLPD_RESET_START_LATENCY_US	10
-#define XE3PLPD_PWRDN_TO_RDY_LATENCY_US	4
+#define XE3PLPD_RESET_START_LATENCY_US		10
+#define XE3PLPD_PWRDN_TO_RDY_LATENCY_US		10
 #define XE3PLPD_RESET_END_LATENCY_MS		2
 
 /* LT Phy MAC Register */
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index d10cbf69a5f8e..4086f16a12bf4 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -11,7 +11,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
-#include "i915_reg.h"
 #include "i9xx_wm.h"
 #include "intel_atomic.h"
 #include "intel_bw.h"
@@ -26,6 +25,7 @@
 #include "intel_display_power.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
+#include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_fifo_underrun.h"
 #include "intel_modeset_setup.h"
@@ -914,7 +914,7 @@ static void intel_early_display_was(struct intel_display *display)
 	 * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl
 	 * Also known as Wa_14010480278.
 	 */
-	if (IS_DISPLAY_VER(display, 10, 12))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14010480278))
 		intel_de_rmw(display, GEN9_CLKGATE_DIS_0, 0, DARBF_GATING_DIS);
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_oprom_regs.h b/drivers/gpu/drm/i915/display/intel_oprom_regs.h
new file mode 100644
index 0000000000000..e6a6fb51b90cf
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_oprom_regs.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _INTEL_OPROM_REGS_H_
+#define _INTEL_OPROM_REGS_H_
+
+#define PRIMARY_SPI_TRIGGER			_MMIO(0x102040)
+#define PRIMARY_SPI_ADDRESS			_MMIO(0x102080)
+#define PRIMARY_SPI_REGIONID			_MMIO(0x102084)
+#define SPI_STATIC_REGIONS			_MMIO(0x102090)
+#define   OPTIONROM_SPI_REGIONID_MASK		REG_GENMASK(7, 0)
+#define OROM_OFFSET				_MMIO(0x1020c0)
+#define   OROM_OFFSET_MASK			REG_GENMASK(20, 16)
+
+#endif
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 05c7545c49e5d..12a325ceae6fe 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -27,24 +27,16 @@
  */
 
 #include <drm/drm_fourcc.h>
+#include <drm/drm_gem.h>
 #include <drm/drm_print.h>
 
-#include "gem/i915_gem_internal.h"
-#include "gem/i915_gem_object_frontbuffer.h"
-#include "gem/i915_gem_pm.h"
-
-#include "gt/intel_gpu_commands.h"
-#include "gt/intel_ring.h"
-
-#include "i915_drv.h"
-#include "i915_reg.h"
 #include "intel_color_regs.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_frontbuffer.h"
 #include "intel_overlay.h"
-#include "intel_pci_config.h"
+#include "intel_parent.h"
 #include "intel_pfit_regs.h"
 
 /* Limits for overlay size. According to intel doc, the real limits are:
@@ -121,9 +113,6 @@
 #define RGB8I_TO_COLORKEY(c) \
 	((((c) & 0xff) << 16) | (((c) & 0xff) << 8) | (((c) & 0xff) << 0))
 
-/* overlay flip addr flag */
-#define OFC_UPDATE		0x1
-
 /* polyphase filter coefficients */
 #define N_HORIZ_Y_TAPS          5
 #define N_VERT_Y_TAPS           3
@@ -189,312 +178,16 @@ struct overlay_registers {
 
 struct intel_overlay {
 	struct intel_display *display;
-	struct intel_context *context;
 	struct intel_crtc *crtc;
-	struct i915_vma *vma;
-	struct i915_vma *old_vma;
-	struct intel_frontbuffer *frontbuffer;
-	bool active;
 	bool pfit_active;
 	u32 pfit_vscale_ratio; /* shifted-point number, (1<<12) == 1.0 */
 	u32 color_key:24;
 	u32 color_key_enabled:1;
 	u32 brightness, contrast, saturation;
 	u32 old_xscale, old_yscale;
-	/* register access */
-	struct drm_i915_gem_object *reg_bo;
 	struct overlay_registers __iomem *regs;
-	u32 flip_addr;
-	/* flip handling */
-	struct i915_active last_flip;
-	void (*flip_complete)(struct intel_overlay *ovl);
 };
 
-static void i830_overlay_clock_gating(struct intel_display *display,
-				      bool enable)
-{
-	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
-	u8 val;
-
-	/* WA_OVERLAY_CLKGATE:alm */
-	if (enable)
-		intel_de_write(display, DSPCLK_GATE_D, 0);
-	else
-		intel_de_write(display, DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
-
-	/* WA_DISABLE_L2CACHE_CLOCK_GATING:alm */
-	pci_bus_read_config_byte(pdev->bus,
-				 PCI_DEVFN(0, 0), I830_CLOCK_GATE, &val);
-	if (enable)
-		val &= ~I830_L2_CACHE_CLOCK_GATE_DISABLE;
-	else
-		val |= I830_L2_CACHE_CLOCK_GATE_DISABLE;
-	pci_bus_write_config_byte(pdev->bus,
-				  PCI_DEVFN(0, 0), I830_CLOCK_GATE, val);
-}
-
-static struct i915_request *
-alloc_request(struct intel_overlay *overlay, void (*fn)(struct intel_overlay *))
-{
-	struct i915_request *rq;
-	int err;
-
-	overlay->flip_complete = fn;
-
-	rq = i915_request_create(overlay->context);
-	if (IS_ERR(rq))
-		return rq;
-
-	err = i915_active_add_request(&overlay->last_flip, rq);
-	if (err) {
-		i915_request_add(rq);
-		return ERR_PTR(err);
-	}
-
-	return rq;
-}
-
-/* overlay needs to be disable in OCMD reg */
-static int intel_overlay_on(struct intel_overlay *overlay)
-{
-	struct intel_display *display = overlay->display;
-	struct i915_request *rq;
-	u32 *cs;
-
-	drm_WARN_ON(display->drm, overlay->active);
-
-	rq = alloc_request(overlay, NULL);
-	if (IS_ERR(rq))
-		return PTR_ERR(rq);
-
-	cs = intel_ring_begin(rq, 4);
-	if (IS_ERR(cs)) {
-		i915_request_add(rq);
-		return PTR_ERR(cs);
-	}
-
-	overlay->active = true;
-
-	if (display->platform.i830)
-		i830_overlay_clock_gating(display, false);
-
-	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_ON;
-	*cs++ = overlay->flip_addr | OFC_UPDATE;
-	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
-	*cs++ = MI_NOOP;
-	intel_ring_advance(rq, cs);
-
-	i915_request_add(rq);
-
-	return i915_active_wait(&overlay->last_flip);
-}
-
-static void intel_overlay_flip_prepare(struct intel_overlay *overlay,
-				       struct i915_vma *vma)
-{
-	struct intel_display *display = overlay->display;
-	enum pipe pipe = overlay->crtc->pipe;
-	struct intel_frontbuffer *frontbuffer = NULL;
-
-	drm_WARN_ON(display->drm, overlay->old_vma);
-
-	if (vma)
-		frontbuffer = intel_frontbuffer_get(intel_bo_to_drm_bo(vma->obj));
-
-	intel_frontbuffer_track(overlay->frontbuffer, frontbuffer,
-				INTEL_FRONTBUFFER_OVERLAY(pipe));
-
-	if (overlay->frontbuffer)
-		intel_frontbuffer_put(overlay->frontbuffer);
-	overlay->frontbuffer = frontbuffer;
-
-	overlay->old_vma = overlay->vma;
-	if (vma)
-		overlay->vma = i915_vma_get(vma);
-	else
-		overlay->vma = NULL;
-}
-
-/* overlay needs to be enabled in OCMD reg */
-static int intel_overlay_continue(struct intel_overlay *overlay,
-				  struct i915_vma *vma,
-				  bool load_polyphase_filter)
-{
-	struct intel_display *display = overlay->display;
-	struct i915_request *rq;
-	u32 flip_addr = overlay->flip_addr;
-	u32 tmp, *cs;
-
-	drm_WARN_ON(display->drm, !overlay->active);
-
-	if (load_polyphase_filter)
-		flip_addr |= OFC_UPDATE;
-
-	/* check for underruns */
-	tmp = intel_de_read(display, DOVSTA);
-	if (tmp & (1 << 17))
-		drm_dbg(display->drm, "overlay underrun, DOVSTA: %x\n", tmp);
-
-	rq = alloc_request(overlay, NULL);
-	if (IS_ERR(rq))
-		return PTR_ERR(rq);
-
-	cs = intel_ring_begin(rq, 2);
-	if (IS_ERR(cs)) {
-		i915_request_add(rq);
-		return PTR_ERR(cs);
-	}
-
-	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE;
-	*cs++ = flip_addr;
-	intel_ring_advance(rq, cs);
-
-	intel_overlay_flip_prepare(overlay, vma);
-	i915_request_add(rq);
-
-	return 0;
-}
-
-static void intel_overlay_release_old_vma(struct intel_overlay *overlay)
-{
-	struct intel_display *display = overlay->display;
-	struct i915_vma *vma;
-
-	vma = fetch_and_zero(&overlay->old_vma);
-	if (drm_WARN_ON(display->drm, !vma))
-		return;
-
-	intel_frontbuffer_flip(display, INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
-
-	i915_vma_unpin(vma);
-	i915_vma_put(vma);
-}
-
-static void
-intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
-{
-	intel_overlay_release_old_vma(overlay);
-}
-
-static void intel_overlay_off_tail(struct intel_overlay *overlay)
-{
-	struct intel_display *display = overlay->display;
-
-	intel_overlay_release_old_vma(overlay);
-
-	overlay->crtc->overlay = NULL;
-	overlay->crtc = NULL;
-	overlay->active = false;
-
-	if (display->platform.i830)
-		i830_overlay_clock_gating(display, true);
-}
-
-static void intel_overlay_last_flip_retire(struct i915_active *active)
-{
-	struct intel_overlay *overlay =
-		container_of(active, typeof(*overlay), last_flip);
-
-	if (overlay->flip_complete)
-		overlay->flip_complete(overlay);
-}
-
-/* overlay needs to be disabled in OCMD reg */
-static int intel_overlay_off(struct intel_overlay *overlay)
-{
-	struct intel_display *display = overlay->display;
-	struct i915_request *rq;
-	u32 *cs, flip_addr = overlay->flip_addr;
-
-	drm_WARN_ON(display->drm, !overlay->active);
-
-	/*
-	 * According to intel docs the overlay hw may hang (when switching
-	 * off) without loading the filter coeffs. It is however unclear whether
-	 * this applies to the disabling of the overlay or to the switching off
-	 * of the hw. Do it in both cases.
-	 */
-	flip_addr |= OFC_UPDATE;
-
-	rq = alloc_request(overlay, intel_overlay_off_tail);
-	if (IS_ERR(rq))
-		return PTR_ERR(rq);
-
-	cs = intel_ring_begin(rq, 6);
-	if (IS_ERR(cs)) {
-		i915_request_add(rq);
-		return PTR_ERR(cs);
-	}
-
-	/* wait for overlay to go idle */
-	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE;
-	*cs++ = flip_addr;
-	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
-
-	/* turn overlay off */
-	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_OFF;
-	*cs++ = flip_addr;
-	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
-
-	intel_ring_advance(rq, cs);
-
-	intel_overlay_flip_prepare(overlay, NULL);
-	i915_request_add(rq);
-
-	return i915_active_wait(&overlay->last_flip);
-}
-
-/*
- * Recover from an interruption due to a signal.
- * We have to be careful not to repeat work forever an make forward progress.
- */
-static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
-{
-	return i915_active_wait(&overlay->last_flip);
-}
-
-/*
- * Wait for pending overlay flip and release old frame.
- * Needs to be called before the overlay register are changed
- * via intel_overlay_(un)map_regs.
- */
-static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
-{
-	struct intel_display *display = overlay->display;
-	struct i915_request *rq;
-	u32 *cs;
-
-	/*
-	 * Only wait if there is actually an old frame to release to
-	 * guarantee forward progress.
-	 */
-	if (!overlay->old_vma)
-		return 0;
-
-	if (!(intel_de_read(display, GEN2_ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT)) {
-		intel_overlay_release_old_vid_tail(overlay);
-		return 0;
-	}
-
-	rq = alloc_request(overlay, intel_overlay_release_old_vid_tail);
-	if (IS_ERR(rq))
-		return PTR_ERR(rq);
-
-	cs = intel_ring_begin(rq, 2);
-	if (IS_ERR(cs)) {
-		i915_request_add(rq);
-		return PTR_ERR(cs);
-	}
-
-	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
-	*cs++ = MI_NOOP;
-	intel_ring_advance(rq, cs);
-
-	i915_request_add(rq);
-
-	return i915_active_wait(&overlay->last_flip);
-}
-
 void intel_overlay_reset(struct intel_display *display)
 {
 	struct intel_overlay *overlay = display->overlay;
@@ -505,7 +198,8 @@ void intel_overlay_reset(struct intel_display *display)
 	overlay->old_xscale = 0;
 	overlay->old_yscale = 0;
 	overlay->crtc = NULL;
-	overlay->active = false;
+
+	intel_parent_overlay_reset(display);
 }
 
 static int packed_depth_bytes(u32 format)
@@ -767,34 +461,8 @@ static u32 overlay_cmd_reg(struct drm_intel_overlay_put_image *params)
 	return cmd;
 }
 
-static struct i915_vma *intel_overlay_pin_fb(struct drm_i915_gem_object *new_bo)
-{
-	struct i915_gem_ww_ctx ww;
-	struct i915_vma *vma;
-	int ret;
-
-	i915_gem_ww_ctx_init(&ww, true);
-retry:
-	ret = i915_gem_object_lock(new_bo, &ww);
-	if (!ret) {
-		vma = i915_gem_object_pin_to_display_plane(new_bo, &ww, 0, 0,
-							   NULL, PIN_MAPPABLE);
-		ret = PTR_ERR_OR_ZERO(vma);
-	}
-	if (ret == -EDEADLK) {
-		ret = i915_gem_ww_ctx_backoff(&ww);
-		if (!ret)
-			goto retry;
-	}
-	i915_gem_ww_ctx_fini(&ww);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return vma;
-}
-
 static int intel_overlay_do_put_image(struct intel_overlay *overlay,
-				      struct drm_i915_gem_object *new_bo,
+				      struct drm_gem_object *obj,
 				      struct drm_intel_overlay_put_image *params)
 {
 	struct intel_display *display = overlay->display;
@@ -804,23 +472,24 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	bool scale_changed = false;
 	struct i915_vma *vma;
 	int ret, tmp_width;
+	u32 tmp, offset;
 
 	drm_WARN_ON(display->drm,
 		    !drm_modeset_is_locked(&display->drm->mode_config.connection_mutex));
 
-	ret = intel_overlay_release_old_vid(overlay);
+	ret = intel_parent_overlay_release_old_vid(display);
 	if (ret != 0)
 		return ret;
 
 	atomic_inc(&display->restore.pending_fb_pin);
 
-	vma = intel_overlay_pin_fb(new_bo);
+	vma = intel_parent_overlay_pin_fb(display, obj, &offset);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out_pin_section;
 	}
 
-	if (!overlay->active) {
+	if (!intel_parent_overlay_is_active(display)) {
 		const struct intel_crtc_state *crtc_state =
 			overlay->crtc->config;
 		u32 oconfig = 0;
@@ -836,7 +505,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 			OCONF_PIPE_A : OCONF_PIPE_B;
 		iowrite32(oconfig, &regs->OCONFIG);
 
-		ret = intel_overlay_on(overlay);
+		ret = intel_parent_overlay_on(display, INTEL_FRONTBUFFER_OVERLAY(pipe));
 		if (ret != 0)
 			goto out_unpin;
 	}
@@ -853,7 +522,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	swidth = params->src_width;
 	swidthsw = calc_swidthsw(display, params->offset_Y, tmp_width);
 	sheight = params->src_height;
-	iowrite32(i915_ggtt_offset(vma) + params->offset_Y, &regs->OBUF_0Y);
+	iowrite32(offset + params->offset_Y, &regs->OBUF_0Y);
 	ostride = params->stride_Y;
 
 	if (params->flags & I915_OVERLAY_YUV_PLANAR) {
@@ -870,9 +539,9 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 				      params->src_width / uv_hscale);
 		swidthsw |= max(tmp_U, tmp_V) << 16;
 
-		iowrite32(i915_ggtt_offset(vma) + params->offset_U,
+		iowrite32(offset + params->offset_U,
 			  &regs->OBUF_0U);
-		iowrite32(i915_ggtt_offset(vma) + params->offset_V,
+		iowrite32(offset + params->offset_V,
 			  &regs->OBUF_0V);
 
 		ostride |= params->stride_UV << 16;
@@ -889,14 +558,19 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 
 	iowrite32(overlay_cmd_reg(params), &regs->OCMD);
 
-	ret = intel_overlay_continue(overlay, vma, scale_changed);
+	/* check for underruns */
+	tmp = intel_de_read(display, DOVSTA);
+	if (tmp & (1 << 17))
+		drm_dbg(display->drm, "overlay underrun, DOVSTA: %x\n", tmp);
+
+	ret = intel_parent_overlay_continue(display, vma, scale_changed);
 	if (ret)
 		goto out_unpin;
 
 	return 0;
 
 out_unpin:
-	i915_vma_unpin(vma);
+	intel_parent_overlay_unpin_fb(display, vma);
 out_pin_section:
 	atomic_dec(&display->restore.pending_fb_pin);
 
@@ -911,20 +585,23 @@ int intel_overlay_switch_off(struct intel_overlay *overlay)
 	drm_WARN_ON(display->drm,
 		    !drm_modeset_is_locked(&display->drm->mode_config.connection_mutex));
 
-	ret = intel_overlay_recover_from_interrupt(overlay);
+	ret = intel_parent_overlay_recover_from_interrupt(display);
 	if (ret != 0)
 		return ret;
 
-	if (!overlay->active)
+	if (!intel_parent_overlay_is_active(display))
 		return 0;
 
-	ret = intel_overlay_release_old_vid(overlay);
+	ret = intel_parent_overlay_release_old_vid(display);
 	if (ret != 0)
 		return ret;
 
 	iowrite32(0, &overlay->regs->OCMD);
 
-	return intel_overlay_off(overlay);
+	overlay->crtc->overlay = NULL;
+	overlay->crtc = NULL;
+
+	return intel_parent_overlay_off(display);
 }
 
 static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
@@ -1006,7 +683,7 @@ static int check_overlay_scaling(struct drm_intel_overlay_put_image *rec)
 
 static int check_overlay_src(struct intel_display *display,
 			     struct drm_intel_overlay_put_image *rec,
-			     struct drm_i915_gem_object *new_bo)
+			     struct drm_gem_object *obj)
 {
 	int uv_hscale = uv_hsubsampling(rec->flags);
 	int uv_vscale = uv_vsubsampling(rec->flags);
@@ -1091,7 +768,7 @@ static int check_overlay_src(struct intel_display *display,
 			return -EINVAL;
 
 		tmp = rec->stride_Y*rec->src_height;
-		if (rec->offset_Y + tmp > new_bo->base.size)
+		if (rec->offset_Y + tmp > obj->size)
 			return -EINVAL;
 		break;
 
@@ -1102,12 +779,12 @@ static int check_overlay_src(struct intel_display *display,
 			return -EINVAL;
 
 		tmp = rec->stride_Y * rec->src_height;
-		if (rec->offset_Y + tmp > new_bo->base.size)
+		if (rec->offset_Y + tmp > obj->size)
 			return -EINVAL;
 
 		tmp = rec->stride_UV * (rec->src_height / uv_vscale);
-		if (rec->offset_U + tmp > new_bo->base.size ||
-		    rec->offset_V + tmp > new_bo->base.size)
+		if (rec->offset_U + tmp > obj->size ||
+		    rec->offset_V + tmp > obj->size)
 			return -EINVAL;
 		break;
 	}
@@ -1122,8 +799,8 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 	struct drm_intel_overlay_put_image *params = data;
 	struct intel_overlay *overlay;
 	struct drm_crtc *drmmode_crtc;
+	struct drm_gem_object *obj;
 	struct intel_crtc *crtc;
-	struct drm_i915_gem_object *new_bo;
 	int ret;
 
 	overlay = display->overlay;
@@ -1145,20 +822,13 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 	crtc = to_intel_crtc(drmmode_crtc);
 
-	new_bo = i915_gem_object_lookup(file_priv, params->bo_handle);
-	if (!new_bo)
-		return -ENOENT;
+	obj = intel_parent_overlay_obj_lookup(display, file_priv, params->bo_handle);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
 
 	drm_modeset_lock_all(dev);
 
-	if (i915_gem_object_is_tiled(new_bo)) {
-		drm_dbg_kms(display->drm,
-			    "buffer used for overlay image can not be tiled\n");
-		ret = -EINVAL;
-		goto out_unlock;
-	}
-
-	ret = intel_overlay_recover_from_interrupt(overlay);
+	ret = intel_parent_overlay_recover_from_interrupt(display);
 	if (ret != 0)
 		goto out_unlock;
 
@@ -1201,7 +871,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 		goto out_unlock;
 	}
 
-	ret = check_overlay_src(display, params, new_bo);
+	ret = check_overlay_src(display, params, obj);
 	if (ret != 0)
 		goto out_unlock;
 
@@ -1210,18 +880,18 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 	if (ret != 0)
 		goto out_unlock;
 
-	ret = intel_overlay_do_put_image(overlay, new_bo, params);
+	ret = intel_overlay_do_put_image(overlay, obj, params);
 	if (ret != 0)
 		goto out_unlock;
 
 	drm_modeset_unlock_all(dev);
-	i915_gem_object_put(new_bo);
+	drm_gem_object_put(obj);
 
 	return 0;
 
 out_unlock:
 	drm_modeset_unlock_all(dev);
-	i915_gem_object_put(new_bo);
+	drm_gem_object_put(obj);
 
 	return ret;
 }
@@ -1328,7 +998,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
 			if (DISPLAY_VER(display) == 2)
 				goto out_unlock;
 
-			if (overlay->active) {
+			if (intel_parent_overlay_is_active(display)) {
 				ret = -EBUSY;
 				goto out_unlock;
 			}
@@ -1354,80 +1024,31 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
 	return ret;
 }
 
-static int get_registers(struct intel_overlay *overlay, bool use_phys)
-{
-	struct intel_display *display = overlay->display;
-	struct drm_i915_private *i915 = to_i915(display->drm);
-	struct drm_i915_gem_object *obj = ERR_PTR(-ENODEV);
-	struct i915_vma *vma;
-	int err;
-
-	if (!display->platform.meteorlake) /* Wa_22018444074 */
-		obj = i915_gem_object_create_stolen(i915, PAGE_SIZE);
-	if (IS_ERR(obj))
-		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
-	if (IS_ERR(obj))
-		return PTR_ERR(obj);
-
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
-	if (IS_ERR(vma)) {
-		err = PTR_ERR(vma);
-		goto err_put_bo;
-	}
-
-	if (use_phys)
-		overlay->flip_addr = sg_dma_address(obj->mm.pages->sgl);
-	else
-		overlay->flip_addr = i915_ggtt_offset(vma);
-	overlay->regs = i915_vma_pin_iomap(vma);
-	i915_vma_unpin(vma);
-
-	if (IS_ERR(overlay->regs)) {
-		err = PTR_ERR(overlay->regs);
-		goto err_put_bo;
-	}
-
-	overlay->reg_bo = obj;
-	return 0;
-
-err_put_bo:
-	i915_gem_object_put(obj);
-	return err;
-}
-
 void intel_overlay_setup(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	struct intel_overlay *overlay;
-	struct intel_engine_cs *engine;
-	int ret;
+	void __iomem *regs;
 
 	if (!HAS_OVERLAY(display))
 		return;
 
-	engine = to_gt(dev_priv)->engine[RCS0];
-	if (!engine || !engine->kernel_context)
-		return;
-
 	overlay = kzalloc_obj(*overlay);
 	if (!overlay)
 		return;
 
+	regs = intel_parent_overlay_setup(display,
+					  OVERLAY_NEEDS_PHYSICAL(display));
+	if (IS_ERR(regs))
+		goto out_free;
+
 	overlay->display = display;
-	overlay->context = engine->kernel_context;
+	overlay->regs = regs;
 	overlay->color_key = 0x0101fe;
 	overlay->color_key_enabled = true;
 	overlay->brightness = -19;
 	overlay->contrast = 75;
 	overlay->saturation = 146;
 
-	i915_active_init(&overlay->last_flip,
-			 NULL, intel_overlay_last_flip_retire, 0);
-
-	ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(display));
-	if (ret)
-		goto out_free;
-
 	memset_io(overlay->regs, 0, sizeof(struct overlay_registers));
 	update_polyphase_filter(overlay->regs);
 	update_reg_attrs(overlay, overlay->regs);
@@ -1447,110 +1068,11 @@ bool intel_overlay_available(struct intel_display *display)
 
 void intel_overlay_cleanup(struct intel_display *display)
 {
-	struct intel_overlay *overlay;
-
-	overlay = fetch_and_zero(&display->overlay);
-	if (!overlay)
+	if (!display->overlay)
 		return;
 
-	/*
-	 * The bo's should be free'd by the generic code already.
-	 * Furthermore modesetting teardown happens beforehand so the
-	 * hardware should be off already.
-	 */
-	drm_WARN_ON(display->drm, overlay->active);
-
-	i915_gem_object_put(overlay->reg_bo);
-	i915_active_fini(&overlay->last_flip);
+	intel_parent_overlay_cleanup(display);
 
-	kfree(overlay);
+	kfree(display->overlay);
+	display->overlay = NULL;
 }
-
-#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
-
-struct intel_overlay_snapshot {
-	struct overlay_registers regs;
-	unsigned long base;
-	u32 dovsta;
-	u32 isr;
-};
-
-struct intel_overlay_snapshot *
-intel_overlay_snapshot_capture(struct intel_display *display)
-{
-	struct intel_overlay *overlay = display->overlay;
-	struct intel_overlay_snapshot *error;
-
-	if (!overlay || !overlay->active)
-		return NULL;
-
-	error = kmalloc_obj(*error, GFP_ATOMIC);
-	if (error == NULL)
-		return NULL;
-
-	error->dovsta = intel_de_read(display, DOVSTA);
-	error->isr = intel_de_read(display, GEN2_ISR);
-	error->base = overlay->flip_addr;
-
-	memcpy_fromio(&error->regs, overlay->regs, sizeof(error->regs));
-
-	return error;
-}
-
-void
-intel_overlay_snapshot_print(const struct intel_overlay_snapshot *error,
-			     struct drm_printer *p)
-{
-	if (!error)
-		return;
-
-	drm_printf(p, "Overlay, status: 0x%08x, interrupt: 0x%08x\n",
-		   error->dovsta, error->isr);
-	drm_printf(p, "  Register file at 0x%08lx:\n", error->base);
-
-#define P(x) drm_printf(p, "    " #x ": 0x%08x\n", error->regs.x)
-	P(OBUF_0Y);
-	P(OBUF_1Y);
-	P(OBUF_0U);
-	P(OBUF_0V);
-	P(OBUF_1U);
-	P(OBUF_1V);
-	P(OSTRIDE);
-	P(YRGB_VPH);
-	P(UV_VPH);
-	P(HORZ_PH);
-	P(INIT_PHS);
-	P(DWINPOS);
-	P(DWINSZ);
-	P(SWIDTH);
-	P(SWIDTHSW);
-	P(SHEIGHT);
-	P(YRGBSCALE);
-	P(UVSCALE);
-	P(OCLRC0);
-	P(OCLRC1);
-	P(DCLRKV);
-	P(DCLRKM);
-	P(SCLRKVH);
-	P(SCLRKVL);
-	P(SCLRKEN);
-	P(OCONFIG);
-	P(OCMD);
-	P(OSTART_0Y);
-	P(OSTART_1Y);
-	P(OSTART_0U);
-	P(OSTART_0V);
-	P(OSTART_1U);
-	P(OSTART_1V);
-	P(OTILEOFF_0Y);
-	P(OTILEOFF_1Y);
-	P(OTILEOFF_0U);
-	P(OTILEOFF_0V);
-	P(OTILEOFF_1U);
-	P(OTILEOFF_1V);
-	P(FASTHSCALE);
-	P(UVSCALEV);
-#undef P
-}
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.h b/drivers/gpu/drm/i915/display/intel_overlay.h
index d259e4c74b031..a4291d6dd5284 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.h
+++ b/drivers/gpu/drm/i915/display/intel_overlay.h
@@ -13,9 +13,7 @@ struct drm_file;
 struct drm_printer;
 struct intel_display;
 struct intel_overlay;
-struct intel_overlay_snapshot;
 
-#ifdef I915
 void intel_overlay_setup(struct intel_display *display);
 bool intel_overlay_available(struct intel_display *display);
 void intel_overlay_cleanup(struct intel_display *display);
@@ -25,51 +23,5 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 void intel_overlay_reset(struct intel_display *display);
-#else
-static inline void intel_overlay_setup(struct intel_display *display)
-{
-}
-static inline bool intel_overlay_available(struct intel_display *display)
-{
-	return false;
-}
-static inline void intel_overlay_cleanup(struct intel_display *display)
-{
-}
-static inline int intel_overlay_switch_off(struct intel_overlay *overlay)
-{
-	return 0;
-}
-static inline int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
-						struct drm_file *file_priv)
-{
-	return 0;
-}
-static inline int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
-					    struct drm_file *file_priv)
-{
-	return 0;
-}
-static inline void intel_overlay_reset(struct intel_display *display)
-{
-}
-#endif
-
-#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) && defined(I915)
-struct intel_overlay_snapshot *
-intel_overlay_snapshot_capture(struct intel_display *display);
-void intel_overlay_snapshot_print(const struct intel_overlay_snapshot *error,
-				  struct drm_printer *p);
-#else
-static inline struct intel_overlay_snapshot *
-intel_overlay_snapshot_capture(struct intel_display *display)
-{
-	return NULL;
-}
-static inline void intel_overlay_snapshot_print(const struct intel_overlay_snapshot *error,
-						struct drm_printer *p)
-{
-}
-#endif
 
 #endif /* __INTEL_OVERLAY_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_parent.c b/drivers/gpu/drm/i915/display/intel_parent.c
index 72ae553f79a4a..2e3bad2b3e6b7 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.c
+++ b/drivers/gpu/drm/i915/display/intel_parent.c
@@ -23,6 +23,55 @@
 #include "intel_display_core.h"
 #include "intel_parent.h"
 
+/* dpt */
+struct intel_dpt *intel_parent_dpt_create(struct intel_display *display,
+					  struct drm_gem_object *obj, size_t size)
+{
+	if (display->parent->dpt)
+		return display->parent->dpt->create(obj, size);
+
+	return NULL;
+}
+
+void intel_parent_dpt_destroy(struct intel_display *display, struct intel_dpt *dpt)
+{
+	if (display->parent->dpt)
+		display->parent->dpt->destroy(dpt);
+}
+
+void intel_parent_dpt_suspend(struct intel_display *display, struct intel_dpt *dpt)
+{
+	if (display->parent->dpt)
+		display->parent->dpt->suspend(dpt);
+}
+
+void intel_parent_dpt_resume(struct intel_display *display, struct intel_dpt *dpt)
+{
+	if (display->parent->dpt)
+		display->parent->dpt->resume(dpt);
+}
+
+/* frontbuffer */
+struct intel_frontbuffer *intel_parent_frontbuffer_get(struct intel_display *display, struct drm_gem_object *obj)
+{
+	return display->parent->frontbuffer->get(obj);
+}
+
+void intel_parent_frontbuffer_ref(struct intel_display *display, struct intel_frontbuffer *front)
+{
+	display->parent->frontbuffer->ref(front);
+}
+
+void intel_parent_frontbuffer_put(struct intel_display *display, struct intel_frontbuffer *front)
+{
+	display->parent->frontbuffer->put(front);
+}
+
+void intel_parent_frontbuffer_flush_for_display(struct intel_display *display, struct intel_frontbuffer *front)
+{
+	display->parent->frontbuffer->flush_for_display(front);
+}
+
 /* hdcp */
 ssize_t intel_parent_hdcp_gsc_msg_send(struct intel_display *display,
 				       struct intel_hdcp_gsc_context *gsc_context,
@@ -59,6 +108,82 @@ void intel_parent_irq_synchronize(struct intel_display *display)
 	display->parent->irq->synchronize(display->drm);
 }
 
+/* overlay */
+bool intel_parent_overlay_is_active(struct intel_display *display)
+{
+	return display->parent->overlay->is_active(display->drm);
+}
+
+int intel_parent_overlay_on(struct intel_display *display,
+			    u32 frontbuffer_bits)
+{
+	return display->parent->overlay->overlay_on(display->drm,
+						    frontbuffer_bits);
+}
+
+int intel_parent_overlay_continue(struct intel_display *display,
+				  struct i915_vma *vma,
+				  bool load_polyphase_filter)
+{
+	return display->parent->overlay->overlay_continue(display->drm, vma,
+							  load_polyphase_filter);
+}
+
+int intel_parent_overlay_off(struct intel_display *display)
+{
+	return display->parent->overlay->overlay_off(display->drm);
+}
+
+int intel_parent_overlay_recover_from_interrupt(struct intel_display *display)
+{
+	return display->parent->overlay->recover_from_interrupt(display->drm);
+}
+
+int intel_parent_overlay_release_old_vid(struct intel_display *display)
+{
+	return display->parent->overlay->release_old_vid(display->drm);
+}
+
+void intel_parent_overlay_reset(struct intel_display *display)
+{
+	display->parent->overlay->reset(display->drm);
+}
+
+struct i915_vma *intel_parent_overlay_pin_fb(struct intel_display *display,
+					     struct drm_gem_object *obj,
+					     u32 *offset)
+{
+	return display->parent->overlay->pin_fb(display->drm, obj, offset);
+}
+
+void intel_parent_overlay_unpin_fb(struct intel_display *display,
+				   struct i915_vma *vma)
+{
+	return display->parent->overlay->unpin_fb(display->drm, vma);
+}
+
+struct drm_gem_object *intel_parent_overlay_obj_lookup(struct intel_display *display,
+						       struct drm_file *filp,
+						       u32 handle)
+{
+	return display->parent->overlay->obj_lookup(display->drm,
+						    filp, handle);
+}
+
+void __iomem *intel_parent_overlay_setup(struct intel_display *display,
+					 bool needs_physical)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->overlay))
+		return ERR_PTR(-ENODEV);
+
+	return display->parent->overlay->setup(display->drm, needs_physical);
+}
+
+void intel_parent_overlay_cleanup(struct intel_display *display)
+{
+	display->parent->overlay->cleanup(display->drm);
+}
+
 /* panic */
 struct intel_panic *intel_parent_panic_alloc(struct intel_display *display)
 {
@@ -92,6 +217,28 @@ void intel_parent_pc8_unblock(struct intel_display *display)
 	display->parent->pc8->unblock(display->drm);
 }
 
+/* pcode */
+int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val, u32 *val1)
+{
+	return display->parent->pcode->read(display->drm, mbox, val, val1);
+}
+
+int intel_parent_pcode_write_timeout(struct intel_display *display, u32 mbox, u32 val, int timeout_ms)
+{
+	return display->parent->pcode->write(display->drm, mbox, val, timeout_ms);
+}
+
+int intel_parent_pcode_write(struct intel_display *display, u32 mbox, u32 val)
+{
+	return intel_parent_pcode_write_timeout(display, mbox, val, 1);
+}
+
+int intel_parent_pcode_request(struct intel_display *display, u32 mbox, u32 request,
+			       u32 reply_mask, u32 reply, int timeout_base_ms)
+{
+	return display->parent->pcode->request(display->drm, mbox, request, reply_mask, reply, timeout_base_ms);
+}
+
 /* rps */
 bool intel_parent_rps_available(struct intel_display *display)
 {
@@ -191,6 +338,15 @@ void intel_parent_stolen_node_free(struct intel_display *display, const struct i
 	display->parent->stolen->node_free(node);
 }
 
+/* vma */
+int intel_parent_vma_fence_id(struct intel_display *display, const struct i915_vma *vma)
+{
+	if (!display->parent->vma)
+		return -1;
+
+	return display->parent->vma->fence_id(vma);
+}
+
 /* generic */
 void intel_parent_fence_priority_display(struct intel_display *display, struct dma_fence *fence)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_parent.h b/drivers/gpu/drm/i915/display/intel_parent.h
index 47cdc14f9aa28..2013e5ed5aa9b 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.h
+++ b/drivers/gpu/drm/i915/display/intel_parent.h
@@ -7,12 +7,30 @@
 #include <linux/types.h>
 
 struct dma_fence;
+struct drm_file;
+struct drm_gem_object;
 struct drm_scanout_buffer;
+struct i915_vma;
 struct intel_display;
+struct intel_dpt;
+struct intel_frontbuffer;
 struct intel_hdcp_gsc_context;
 struct intel_panic;
 struct intel_stolen_node;
 
+/* dpt */
+struct intel_dpt *intel_parent_dpt_create(struct intel_display *display,
+					  struct drm_gem_object *obj, size_t size);
+void intel_parent_dpt_destroy(struct intel_display *display, struct intel_dpt *dpt);
+void intel_parent_dpt_suspend(struct intel_display *display, struct intel_dpt *dpt);
+void intel_parent_dpt_resume(struct intel_display *display, struct intel_dpt *dpt);
+
+/* frontbuffer */
+struct intel_frontbuffer *intel_parent_frontbuffer_get(struct intel_display *display, struct drm_gem_object *obj);
+void intel_parent_frontbuffer_ref(struct intel_display *display, struct intel_frontbuffer *front);
+void intel_parent_frontbuffer_put(struct intel_display *display, struct intel_frontbuffer *front);
+void intel_parent_frontbuffer_flush_for_display(struct intel_display *display, struct intel_frontbuffer *front);
+
 /* hdcp */
 ssize_t intel_parent_hdcp_gsc_msg_send(struct intel_display *display,
 				       struct intel_hdcp_gsc_context *gsc_context,
@@ -27,6 +45,29 @@ void intel_parent_hdcp_gsc_context_free(struct intel_display *display,
 bool intel_parent_irq_enabled(struct intel_display *display);
 void intel_parent_irq_synchronize(struct intel_display *display);
 
+/* overlay */
+bool intel_parent_overlay_is_active(struct intel_display *display);
+int intel_parent_overlay_on(struct intel_display *display,
+			    u32 frontbuffer_bits);
+int intel_parent_overlay_continue(struct intel_display *display,
+				  struct i915_vma *vma,
+				  bool load_polyphase_filter);
+int intel_parent_overlay_off(struct intel_display *display);
+int intel_parent_overlay_recover_from_interrupt(struct intel_display *display);
+int intel_parent_overlay_release_old_vid(struct intel_display *display);
+void intel_parent_overlay_reset(struct intel_display *display);
+struct i915_vma *intel_parent_overlay_pin_fb(struct intel_display *display,
+					     struct drm_gem_object *obj,
+					     u32 *offset);
+void intel_parent_overlay_unpin_fb(struct intel_display *display,
+				   struct i915_vma *vma);
+struct drm_gem_object *intel_parent_overlay_obj_lookup(struct intel_display *display,
+						       struct drm_file *filp,
+						       u32 handle);
+void __iomem *intel_parent_overlay_setup(struct intel_display *display,
+					 bool needs_physical);
+void intel_parent_overlay_cleanup(struct intel_display *display);
+
 /* panic */
 struct intel_panic *intel_parent_panic_alloc(struct intel_display *display);
 int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic, struct drm_scanout_buffer *sb);
@@ -36,6 +77,13 @@ void intel_parent_panic_finish(struct intel_display *display, struct intel_panic
 void intel_parent_pc8_block(struct intel_display *display);
 void intel_parent_pc8_unblock(struct intel_display *display);
 
+/* pcode */
+int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val, u32 *val1);
+int intel_parent_pcode_write_timeout(struct intel_display *display, u32 mbox, u32 val, int timeout_ms);
+int intel_parent_pcode_write(struct intel_display *display, u32 mbox, u32 val);
+int intel_parent_pcode_request(struct intel_display *display, u32 mbox, u32 request,
+			       u32 reply_mask, u32 reply, int timeout_base_ms);
+
 /* rps */
 bool intel_parent_rps_available(struct intel_display *display);
 void intel_parent_rps_boost_if_not_started(struct intel_display *display, struct dma_fence *fence);
@@ -61,6 +109,9 @@ u64 intel_parent_stolen_node_size(struct intel_display *display, const struct in
 struct intel_stolen_node *intel_parent_stolen_node_alloc(struct intel_display *display);
 void intel_parent_stolen_node_free(struct intel_display *display, const struct intel_stolen_node *node);
 
+/* vma */
+int intel_parent_vma_fence_id(struct intel_display *display, const struct i915_vma *vma);
+
 /* generic */
 bool intel_parent_has_auxccs(struct intel_display *display);
 bool intel_parent_has_fenced_regions(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c
index 16619f7be5f86..69c7952a14136 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_display.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_display.c
@@ -6,7 +6,6 @@
 #include <drm/drm_print.h>
 
 #include "g4x_dp.h"
-#include "i915_reg.h"
 #include "intel_crt.h"
 #include "intel_crt_regs.h"
 #include "intel_de.h"
diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.c b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
index 9a89bb6dcf658..5f88663ef5e8c 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_refclk.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
@@ -5,7 +5,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
diff --git a/drivers/gpu/drm/i915/display/intel_pmdemand.c b/drivers/gpu/drm/i915/display/intel_pmdemand.c
index f3db557100102..244806a26da39 100644
--- a/drivers/gpu/drm/i915/display/intel_pmdemand.c
+++ b/drivers/gpu/drm/i915/display/intel_pmdemand.c
@@ -15,6 +15,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_trace.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_pmdemand.h"
 #include "intel_step.h"
 #include "skl_watermark.h"
@@ -129,9 +130,10 @@ int intel_pmdemand_init(struct intel_display *display)
 				     &pmdemand_state->base,
 				     &intel_pmdemand_funcs);
 
-	if (IS_DISPLAY_VERx100_STEP(display, 1400, STEP_A0, STEP_C0))
-		/* Wa_14016740474 */
-		intel_de_rmw(display, XELPD_CHICKEN_DCPR_3, 0, DMD_RSP_TIMEOUT_DISABLE);
+	/* Wa_14016740474 */
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14016740474))
+		intel_de_rmw(display, XELPD_CHICKEN_DCPR_3, 0,
+			     DMD_RSP_TIMEOUT_DISABLE);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index b217ec7aa7589..2d799af73bb7f 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -9,7 +9,6 @@
 #include <drm/drm_print.h>
 
 #include "g4x_dp.h"
-#include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_jiffies.h"
 #include "intel_display_power_well.h"
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index b7302a32ded46..b319e5bd62747 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -29,7 +29,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
-#include "i915_reg.h"
 #include "intel_alpm.h"
 #include "intel_atomic.h"
 #include "intel_crtc.h"
@@ -41,6 +40,7 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_dp.h"
 #include "intel_dp_aux.h"
@@ -49,6 +49,7 @@
 #include "intel_hdmi.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
+#include "intel_quirks.h"
 #include "intel_snps_phy.h"
 #include "intel_step.h"
 #include "intel_vblank.h"
@@ -609,6 +610,12 @@ static void _panel_replay_init_dpcd(struct intel_dp *intel_dp, struct intel_conn
 	if (intel_dp->mst_detect == DRM_DP_MST)
 		return;
 
+	if (intel_has_dpcd_quirk(intel_dp, QUIRK_DISABLE_PANEL_REPLAY)) {
+		drm_dbg_kms(display->drm,
+			    "Panel Replay support not currently available for this setup\n");
+		return;
+	}
+
 	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PANEL_REPLAY_CAP_SUPPORT,
 				    &connector->dp.panel_replay_caps.dpcd,
 				    sizeof(connector->dp.panel_replay_caps.dpcd));
@@ -1083,7 +1090,7 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	}
 
 	/* Wa_22012278275:adl-p */
-	if (display->platform.alderlake_p && IS_DISPLAY_STEP(display, STEP_A0, STEP_E0)) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_22012278275)) {
 		static const u8 map[] = {
 			2, /* 5 lines */
 			1, /* 6 lines */
@@ -1264,7 +1271,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 		return;
 
 	/* Wa_16011303918:adl-p */
-	if (display->platform.alderlake_p && IS_DISPLAY_STEP(display, STEP_A0, STEP_B0))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16011303918))
 		return;
 
 	/*
@@ -1546,8 +1553,7 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	}
 
 	/* Wa_16011181250 */
-	if (display->platform.rocketlake || display->platform.alderlake_s ||
-	    display->platform.dg2) {
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16011181250)) {
 		drm_dbg_kms(display->drm,
 			    "PSR2 is defeatured for this platform\n");
 		return false;
@@ -1717,7 +1723,7 @@ static bool _psr_compute_config(struct intel_dp *intel_dp,
 	entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, conn_state, adjusted_mode);
 
 	if (entry_setup_frames >= 0) {
-		intel_dp->psr.entry_setup_frames = entry_setup_frames;
+		crtc_state->entry_setup_frames = entry_setup_frames;
 	} else {
 		crtc_state->no_psr_reason = "PSR setup timing not met";
 		drm_dbg_kms(display->drm,
@@ -1815,7 +1821,7 @@ static bool intel_psr_needs_wa_18037818876(struct intel_dp *intel_dp,
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 
-	return (DISPLAY_VER(display) == 20 && intel_dp->psr.entry_setup_frames > 0 &&
+	return (DISPLAY_VER(display) == 20 && crtc_state->entry_setup_frames > 0 &&
 		!crtc_state->has_sel_update);
 }
 
@@ -1829,8 +1835,7 @@ void intel_psr_set_non_psr_pipes(struct intel_dp *intel_dp,
 	u8 active_pipes = 0;
 
 	/* Wa_16025596647 */
-	if (DISPLAY_VER(display) != 20 &&
-	    !IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0))
+	if (!intel_display_wa(display, INTEL_DISPLAY_WA_16025596647))
 		return;
 
 	/* Not needed by Panel Replay  */
@@ -2130,6 +2135,9 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 		intel_dmc_block_pkgc(display, intel_dp->psr.pipe, true);
 
 	intel_alpm_configure(intel_dp, crtc_state);
+
+	if (HAS_PSR_TRANS_PUSH_FRAME_CHANGE(display))
+		intel_vrr_psr_frame_change_enable(crtc_state);
 }
 
 static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
@@ -2189,6 +2197,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 	intel_dp->psr.pkg_c_latency_used = crtc_state->pkg_c_latency_used;
 	intel_dp->psr.io_wake_lines = crtc_state->alpm_state.io_wake_lines;
 	intel_dp->psr.fast_wake_lines = crtc_state->alpm_state.fast_wake_lines;
+	intel_dp->psr.entry_setup_frames = crtc_state->entry_setup_frames;
 
 	if (!psr_interrupt_error_check(intel_dp))
 		return;
@@ -2521,9 +2530,11 @@ void intel_psr_trigger_frame_change_event(struct intel_dsb *dsb,
 		intel_pre_commit_crtc_state(state, crtc);
 	struct intel_display *display = to_intel_display(crtc);
 
-	if (crtc_state->has_psr)
-		intel_de_write_dsb(display, dsb,
-				   CURSURFLIVE(display, crtc->pipe), 0);
+	if (!crtc_state->has_psr || intel_psr_use_trans_push(crtc_state))
+		return;
+
+	intel_de_write_dsb(display, dsb,
+			   CURSURFLIVE(display, crtc->pipe), 0);
 }
 
 /**
@@ -3109,6 +3120,8 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 			 * - Display WA #1136: skl, bxt
 			 */
 			if (intel_crtc_needs_modeset(new_crtc_state) ||
+			    new_crtc_state->update_m_n ||
+			    new_crtc_state->update_lrr ||
 			    !new_crtc_state->has_psr ||
 			    !new_crtc_state->active_planes ||
 			    new_crtc_state->has_sel_update != psr->sel_update_enabled ||
@@ -3598,7 +3611,14 @@ static void _psr_flush_handle(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 
-	if (DISPLAY_VER(display) < 20 && intel_dp->psr.psr2_sel_fetch_enabled) {
+	if (DISPLAY_VER(display) >= 20) {
+		/*
+		 * We can use PSR exit on LunarLake onwards. Also
+		 * using trans push mechanism to trigger Frame Change
+		 * event requires using PSR exit.
+		 */
+		intel_psr_exit(intel_dp);
+	} else if (intel_dp->psr.psr2_sel_fetch_enabled) {
 		/* Selective fetch prior LNL */
 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {
 			/* can we turn CFF off? */
@@ -3618,16 +3638,11 @@ static void _psr_flush_handle(struct intel_dp *intel_dp)
 		intel_psr_configure_full_frame_update(intel_dp);
 
 		intel_psr_force_update(intel_dp);
-	} else if (!intel_dp->psr.psr2_sel_fetch_enabled) {
+	} else {
 		/*
-		 * PSR1 on all platforms
-		 * PSR2 HW tracking
-		 * Panel Replay Full frame update
+		 * On older platforms using PSR exit was seen causing problems
 		 */
 		intel_psr_force_update(intel_dp);
-	} else {
-		/* Selective update LNL onwards */
-		intel_psr_exit(intel_dp);
 	}
 
 	if (!intel_dp->psr.active && !intel_dp->psr.busy_frontbuffer_bits)
@@ -4008,8 +4023,7 @@ static void psr_dc5_dc6_wa_work(struct work_struct *work)
  */
 void intel_psr_notify_dc5_dc6(struct intel_display *display)
 {
-	if (DISPLAY_VER(display) != 20 &&
-	    !IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0))
+	if (!intel_display_wa(display, INTEL_DISPLAY_WA_16025596647))
 		return;
 
 	schedule_work(&display->psr_dc5_dc6_wa_work);
@@ -4024,8 +4038,7 @@ void intel_psr_notify_dc5_dc6(struct intel_display *display)
  */
 void intel_psr_dc5_dc6_wa_init(struct intel_display *display)
 {
-	if (DISPLAY_VER(display) != 20 &&
-	    !IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0))
+	if (!intel_display_wa(display, INTEL_DISPLAY_WA_16025596647))
 		return;
 
 	INIT_WORK(&display->psr_dc5_dc6_wa_work, psr_dc5_dc6_wa_work);
@@ -4046,8 +4059,7 @@ void intel_psr_notify_pipe_change(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(state);
 	struct intel_encoder *encoder;
 
-	if (DISPLAY_VER(display) != 20 &&
-	    !IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0))
+	if (!intel_display_wa(display, INTEL_DISPLAY_WA_16025596647))
 		return;
 
 	for_each_intel_encoder_with_psr(display->drm, encoder) {
@@ -4603,3 +4615,10 @@ int intel_psr_min_guardband(struct intel_crtc_state *crtc_state)
 
 	return psr_min_guardband;
 }
+
+bool intel_psr_use_trans_push(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+
+	return HAS_PSR_TRANS_PUSH_FRAME_CHANGE(display) && crtc_state->has_psr;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index b41dc4d44ff29..394b641840b39 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -85,5 +85,6 @@ bool intel_psr_needs_alpm_aux_less(struct intel_dp *intel_dp,
 void intel_psr_compute_config_late(struct intel_dp *intel_dp,
 				   struct intel_crtc_state *crtc_state);
 int intel_psr_min_guardband(struct intel_crtc_state *crtc_state);
+bool intel_psr_use_trans_push(const struct intel_crtc_state *crtc_state);
 
 #endif /* __INTEL_PSR_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c
index 1abbdd426e587..8f1bf8f418ec7 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.c
+++ b/drivers/gpu/drm/i915/display/intel_quirks.c
@@ -86,6 +86,14 @@ static void quirk_edp_limit_rate_hbr2(struct intel_display *display)
 	drm_info(display->drm, "Applying eDP Limit rate to HBR2 quirk\n");
 }
 
+static void quirk_disable_panel_replay(struct intel_dp *intel_dp)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	intel_set_dpcd_quirk(intel_dp, QUIRK_DISABLE_PANEL_REPLAY);
+	drm_info(display->drm, "Applying disable Panel Replay quirk\n");
+}
+
 struct intel_quirk {
 	int device;
 	int subsystem_vendor;
@@ -251,7 +259,14 @@ static const struct intel_dpcd_quirk intel_dpcd_quirks[] = {
 		.sink_oui = SINK_OUI(0x38, 0xec, 0x11),
 		.hook = quirk_fw_sync_len,
 	},
-
+	/* Dell XPS 14 DA14260 */
+	{
+		.device = 0xb080,
+		.subsystem_vendor = 0x1028,
+		.subsystem_device = 0x0db9,
+		.sink_oui = SINK_OUI(0x00, 0x22, 0xb9),
+		.hook = quirk_disable_panel_replay,
+	},
 };
 
 void intel_init_quirks(struct intel_display *display)
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h
index 06da0e286c67c..77e490caed0d8 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.h
+++ b/drivers/gpu/drm/i915/display/intel_quirks.h
@@ -21,6 +21,7 @@ enum intel_quirk_id {
 	QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK,
 	QUIRK_FW_SYNC_LEN,
 	QUIRK_EDP_LIMIT_RATE_HBR2,
+	QUIRK_DISABLE_PANEL_REPLAY,
 };
 
 void intel_init_quirks(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_rom.c b/drivers/gpu/drm/i915/display/intel_rom.c
index 05b6ea764ebb2..d573059fb0d99 100644
--- a/drivers/gpu/drm/i915/display/intel_rom.c
+++ b/drivers/gpu/drm/i915/display/intel_rom.c
@@ -7,10 +7,9 @@
 
 #include <drm/drm_device.h>
 
-#include "i915_reg.h"
-
 #include "intel_rom.h"
 #include "intel_uncore.h"
+#include "intel_oprom_regs.h"
 
 struct intel_rom {
 	/* for PCI ROM */
diff --git a/drivers/gpu/drm/i915/display/intel_snps_hdmi_pll.c b/drivers/gpu/drm/i915/display/intel_snps_hdmi_pll.c
index a201edceee105..7fe6b4a182133 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_hdmi_pll.c
+++ b/drivers/gpu/drm/i915/display/intel_snps_hdmi_pll.c
@@ -332,8 +332,6 @@ void intel_snps_hdmi_pll_compute_c10pll(struct intel_c10pll_state *pll_state, u6
 			      c10_curve_1, c10_curve_2, prescaler_divider,
 			      &pll_params);
 
-	pll_state->clock = pixel_clock;
-
 	pll_state->tx = 0x10;
 	pll_state->cmn = 0x1;
 	pll_state->pll[0] = REG_FIELD_PREP(C10_PLL0_DIV5CLK_EN, pll_params.mpll_div5_en) |
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index bd12148e42f70..a21dd4e3fe4ca 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -7,7 +7,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_cx0_phy_regs.h"
 #include "intel_ddi.h"
@@ -1845,7 +1844,7 @@ bool intel_tc_port_link_reset(struct intel_digital_port *dig_port)
 	if (!intel_tc_port_link_needs_reset(dig_port))
 		return false;
 
-	queue_delayed_work(system_unbound_wq,
+	queue_delayed_work(system_dfl_wq,
 			   &to_tc_port(dig_port)->link_reset_work,
 			   msecs_to_jiffies(2000));
 
@@ -1926,7 +1925,7 @@ void intel_tc_port_unlock(struct intel_digital_port *dig_port)
 	struct intel_tc_port *tc = to_tc_port(dig_port);
 
 	if (!tc->link_refcount && tc->mode != TC_PORT_DISCONNECTED)
-		queue_delayed_work(system_unbound_wq, &tc->disconnect_phy_work,
+		queue_delayed_work(system_dfl_wq, &tc->disconnect_phy_work,
 				   msecs_to_jiffies(1000));
 
 	mutex_unlock(&tc->lock);
diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
index 57fda5824c9c1..0dc13d080e8aa 100644
--- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
@@ -1109,6 +1109,7 @@ struct bdb_edp {
 	u16 edp_dsc_disable;					/* 251+ */
 	u16 t6_delay_support;					/* 260+ */
 	u16 link_idle_time[16];					/* 260+ */
+	u16 pipe_joiner_enable;					/* 261+ */
 } __packed;
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index 2065dac1e3fdb..6c09c6d99ffe1 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -35,6 +35,58 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state)
 	return true;
 }
 
+int intel_dsc_line_slice_count(const struct intel_dsc_slice_config *config)
+{
+	return config->pipes_per_line * config->streams_per_pipe * config->slices_per_stream;
+}
+
+bool intel_dsc_get_slice_config(struct intel_display *display,
+				int pipes_per_line, int slices_per_pipe,
+				struct intel_dsc_slice_config *config)
+{
+	int streams_per_pipe;
+
+	/* TODO: Add support for 8 slices per pipe on TGL+. */
+	switch (slices_per_pipe) {
+	case 3:
+		/*
+		 * 3 DSC Slices per pipe need 3 DSC engines, which is supported only
+		 * with Ultrajoiner only for some platforms.
+		 */
+		if (!HAS_DSC_3ENGINES(display) || pipes_per_line != 4)
+			return false;
+
+		streams_per_pipe = 3;
+		break;
+	case 4:
+		/* TODO: Consider using 1 DSC engine stream x 4 slices instead. */
+	case 2:
+		/* TODO: Consider using 1 DSC engine stream x 2 slices instead. */
+		streams_per_pipe = 2;
+		break;
+	case 1:
+		 /*
+		  * Bigjoiner needs small joiner to be enabled.
+		  * So there should be at least 2 dsc slices per pipe,
+		  * whenever bigjoiner is enabled.
+		  */
+		if (pipes_per_line > 1)
+			return false;
+
+		streams_per_pipe = 1;
+		break;
+	default:
+		MISSING_CASE(slices_per_pipe);
+		return false;
+	}
+
+	config->pipes_per_line = pipes_per_line;
+	config->streams_per_pipe = streams_per_pipe;
+	config->slices_per_stream = slices_per_pipe / streams_per_pipe;
+
+	return true;
+}
+
 static bool is_pipe_dsc(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
 {
 	struct intel_display *display = to_intel_display(crtc);
@@ -278,8 +330,9 @@ int intel_dsc_compute_params(struct intel_crtc_state *pipe_config)
 	int ret;
 
 	vdsc_cfg->pic_width = pipe_config->hw.adjusted_mode.crtc_hdisplay;
-	vdsc_cfg->slice_width = DIV_ROUND_UP(vdsc_cfg->pic_width,
-					     pipe_config->dsc.slice_count);
+	vdsc_cfg->slice_width =
+		DIV_ROUND_UP(vdsc_cfg->pic_width,
+			     intel_dsc_line_slice_count(&pipe_config->dsc.slice_config));
 
 	err = intel_dsc_slice_dimensions_valid(pipe_config, vdsc_cfg);
 
@@ -416,7 +469,7 @@ intel_dsc_power_domain(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
 
 static int intel_dsc_get_vdsc_per_pipe(const struct intel_crtc_state *crtc_state)
 {
-	return crtc_state->dsc.num_streams;
+	return crtc_state->dsc.slice_config.streams_per_pipe;
 }
 
 int intel_dsc_get_num_vdsc_instances(const struct intel_crtc_state *crtc_state)
@@ -1041,12 +1094,13 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state)
 	if (!crtc_state->dsc.compression_enable)
 		goto out;
 
+	/* TODO: Read out slice_config.pipes_per_line/slices_per_stream as well */
 	if (dss_ctl1 & JOINER_ENABLE && dss_ctl2 & (VDSC2_ENABLE | SMALL_JOINER_CONFIG_3_ENGINES))
-		crtc_state->dsc.num_streams = 3;
+		crtc_state->dsc.slice_config.streams_per_pipe = 3;
 	else if (dss_ctl1 & JOINER_ENABLE && dss_ctl2 & VDSC1_ENABLE)
-		crtc_state->dsc.num_streams = 2;
+		crtc_state->dsc.slice_config.streams_per_pipe = 2;
 	else
-		crtc_state->dsc.num_streams = 1;
+		crtc_state->dsc.slice_config.streams_per_pipe = 1;
 
 	intel_dsc_get_pps_config(crtc_state);
 out:
@@ -1059,8 +1113,8 @@ static void intel_vdsc_dump_state(struct drm_printer *p, int indent,
 	drm_printf_indent(p, indent,
 			  "dsc-dss: compressed-bpp:" FXP_Q4_FMT ", slice-count: %d, num_streams: %d\n",
 			  FXP_Q4_ARGS(crtc_state->dsc.compressed_bpp_x16),
-			  crtc_state->dsc.slice_count,
-			  crtc_state->dsc.num_streams);
+			  intel_dsc_line_slice_count(&crtc_state->dsc.slice_config),
+			  crtc_state->dsc.slice_config.streams_per_pipe);
 }
 
 void intel_vdsc_state_dump(struct drm_printer *p, int indent,
@@ -1073,7 +1127,6 @@ void intel_vdsc_state_dump(struct drm_printer *p, int indent,
 	drm_dsc_dump_config(p, indent, &crtc_state->dsc.config);
 }
 
-static
 int intel_dsc_get_pixel_rate_with_dsc_bubbles(struct intel_display *display,
 					      int pixel_rate, int htotal,
 					      int dsc_horizontal_slices)
@@ -1095,7 +1148,7 @@ int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state)
 	struct intel_display *display = to_intel_display(crtc_state);
 	int num_vdsc_instances = intel_dsc_get_num_vdsc_instances(crtc_state);
 	int htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
-	int dsc_slices = crtc_state->dsc.slice_count;
+	int dsc_slices = intel_dsc_line_slice_count(&crtc_state->dsc.slice_config);
 	int pixel_rate;
 	int min_cdclk;
 
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h
index 99bb9042592a4..3372f8694054d 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.h
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.h
@@ -13,10 +13,16 @@ struct drm_printer;
 enum transcoder;
 struct intel_crtc;
 struct intel_crtc_state;
+struct intel_display;
 struct intel_dsb;
+struct intel_dsc_slice_config;
 struct intel_encoder;
 
 bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state);
+int intel_dsc_line_slice_count(const struct intel_dsc_slice_config *config);
+bool intel_dsc_get_slice_config(struct intel_display *display,
+				int num_joined_pipes, int slice_per_pipe,
+				struct intel_dsc_slice_config *config);
 void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state);
 void intel_dsc_enable(const struct intel_crtc_state *crtc_state);
 void intel_dsc_disable(const struct intel_crtc_state *crtc_state);
@@ -38,5 +44,8 @@ void intel_vdsc_state_dump(struct drm_printer *p, int indent,
 			   const struct intel_crtc_state *crtc_state);
 int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state);
 unsigned int intel_vdsc_prefill_lines(const struct intel_crtc_state *crtc_state);
+int intel_dsc_get_pixel_rate_with_dsc_bubbles(struct intel_display *display,
+					      int pixel_rate, int htotal,
+					      int dsc_horizontal_slices);
 
 #endif /* __INTEL_VDSC_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c
index c45c4bbc3f957..9832a4ade318c 100644
--- a/drivers/gpu/drm/i915/display/intel_vga.c
+++ b/drivers/gpu/drm/i915/display/intel_vga.c
@@ -18,6 +18,23 @@
 #include "intel_vga.h"
 #include "intel_vga_regs.h"
 
+static unsigned int intel_gmch_ctrl_reg(struct intel_display *display)
+{
+	return DISPLAY_VER(display) >= 6 ? SNB_GMCH_CTRL : I830_GMCH_CTRL;
+}
+
+static bool intel_vga_decode_is_enabled(struct intel_display *display)
+{
+	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
+	u16 gmch_ctrl = 0;
+
+	if (pci_bus_read_config_word(pdev->bus, PCI_DEVFN(0, 0),
+				     intel_gmch_ctrl_reg(display), &gmch_ctrl))
+		return false;
+
+	return !(gmch_ctrl & INTEL_GMCH_VGA_DISABLE);
+}
+
 static i915_reg_t intel_vga_cntrl_reg(struct intel_display *display)
 {
 	if (display->platform.valleyview || display->platform.cherryview)
@@ -41,101 +58,266 @@ static bool has_vga_pipe_sel(struct intel_display *display)
 	return DISPLAY_VER(display) < 7;
 }
 
-/* Disable the VGA plane that we never use */
-void intel_vga_disable(struct intel_display *display)
+static bool has_vga_mmio_access(struct intel_display *display)
 {
-	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
-	i915_reg_t vga_reg = intel_vga_cntrl_reg(display);
-	enum pipe pipe;
-	u32 tmp;
-	u8 sr1;
+	/* WaEnableVGAAccessThroughIOPort:ctg+ */
+	return DISPLAY_VER(display) < 5 && !display->platform.g4x;
+}
 
-	tmp = intel_de_read(display, vga_reg);
-	if (tmp & VGA_DISP_DISABLE)
-		return;
+static bool intel_pci_has_vga_io_decode(struct pci_dev *pdev)
+{
+	u16 cmd = 0;
 
-	if (display->platform.cherryview)
-		pipe = REG_FIELD_GET(VGA_PIPE_SEL_MASK_CHV, tmp);
-	else if (has_vga_pipe_sel(display))
-		pipe = REG_FIELD_GET(VGA_PIPE_SEL_MASK, tmp);
-	else
-		pipe = PIPE_A;
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	if ((cmd & PCI_COMMAND_IO) == 0)
+		return false;
 
-	drm_dbg_kms(display->drm, "Disabling VGA plane on pipe %c\n",
-		    pipe_name(pipe));
+	pdev = pdev->bus->self;
+	while (pdev) {
+		u16 ctl = 0;
 
-	/* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
-	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
-	outb(0x01, VGA_SEQ_I);
-	sr1 = inb(VGA_SEQ_D);
-	outb(sr1 | VGA_SR01_SCREEN_OFF, VGA_SEQ_D);
-	vga_put(pdev, VGA_RSRC_LEGACY_IO);
-	udelay(300);
+		pci_read_config_word(pdev, PCI_BRIDGE_CONTROL, &ctl);
+		if ((ctl & PCI_BRIDGE_CTL_VGA) == 0)
+			return false;
 
-	intel_de_write(display, vga_reg, VGA_DISP_DISABLE);
-	intel_de_posting_read(display, vga_reg);
+		pdev = pdev->bus->self;
+	}
+
+	return true;
+}
+
+static bool intel_pci_set_io_decode(struct pci_dev *pdev, bool enable)
+{
+	u16 old = 0, cmd;
+
+	pci_read_config_word(pdev, PCI_COMMAND, &old);
+	cmd = old & ~PCI_COMMAND_IO;
+	if (enable)
+		cmd |= PCI_COMMAND_IO;
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+
+	return old & PCI_COMMAND_IO;
 }
 
-void intel_vga_reset_io_mem(struct intel_display *display)
+static bool intel_pci_bridge_set_vga(struct pci_dev *pdev, bool enable)
+{
+	u16 old = 0, ctl;
+
+	pci_read_config_word(pdev->bus->self, PCI_BRIDGE_CONTROL, &old);
+	ctl = old & ~PCI_BRIDGE_CTL_VGA;
+	if (enable)
+		ctl |= PCI_BRIDGE_CTL_VGA;
+	pci_write_config_word(pdev->bus->self, PCI_BRIDGE_CONTROL, ctl);
+
+	return old & PCI_BRIDGE_CTL_VGA;
+}
+
+static int intel_vga_get(struct intel_display *display, bool mmio,
+			 bool *old_io_decode)
 {
 	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
+	int err;
+
+	if (mmio) {
+		*old_io_decode = false;
+		return 0;
+	}
 
 	/*
-	 * After we re-enable the power well, if we touch VGA register 0x3d5
-	 * we'll get unclaimed register interrupts. This stops after we write
-	 * anything to the VGA MSR register. The vgacon module uses this
-	 * register all the time, so if we unbind our driver and, as a
-	 * consequence, bind vgacon, we'll get stuck in an infinite loop at
-	 * console_unlock(). So make here we touch the VGA MSR register, making
-	 * sure vgacon can keep working normally without triggering interrupts
-	 * and error messages.
+	 * Bypass the VGA arbiter on the iGPU and just enable
+	 * IO decode by hand. This avoids clobbering the VGA
+	 * routing for an external GPU when it's the current
+	 * VGA device, and thus prevents the all 0xff/white
+	 * readout from VGA memory when taking over from vgacon.
+	 *
+	 * The iGPU has the highest VGA decode priority so it will
+	 * grab any VGA IO access when IO decode is enabled, regardless
+	 * of how any other VGA routing bits are configured.
 	 */
-	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
-	outb(inb(VGA_MIS_R), VGA_MIS_W);
-	vga_put(pdev, VGA_RSRC_LEGACY_IO);
+	if (display->platform.dgfx) {
+		err = vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
+		if (err)
+			return err;
+	}
+
+	*old_io_decode = intel_pci_set_io_decode(pdev, true);
+
+	return 0;
 }
 
-static int intel_gmch_vga_set_state(struct intel_display *display, bool enable_decode)
+static void intel_vga_put(struct intel_display *display, bool io_decode, bool mmio)
 {
 	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
-	unsigned int reg = DISPLAY_VER(display) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
-	u16 gmch_ctrl;
 
-	if (pci_bus_read_config_word(pdev->bus, PCI_DEVFN(0, 0), reg, &gmch_ctrl)) {
-		drm_err(display->drm, "failed to read control word\n");
-		return -EIO;
-	}
+	if (mmio)
+		return;
 
-	if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !enable_decode)
-		return 0;
+	/* see intel_vga_get() */
+	intel_pci_set_io_decode(pdev, io_decode);
+
+	if (display->platform.dgfx)
+		vga_put(pdev, VGA_RSRC_LEGACY_IO);
+}
 
-	if (enable_decode)
-		gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
+u8 intel_vga_read(struct intel_display *display, u16 reg, bool mmio)
+{
+	if (mmio)
+		return intel_de_read8(display, _MMIO(reg));
 	else
-		gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
+		return inb(reg);
+}
+
+static void intel_vga_write(struct intel_display *display, u16 reg, u8 val, bool mmio)
+{
+	if (mmio)
+		intel_de_write8(display, _MMIO(reg), val);
+	else
+		outb(val, reg);
+}
+
+/* Disable the VGA plane that we never use */
+void intel_vga_disable(struct intel_display *display)
+{
+	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
+	i915_reg_t vga_reg = intel_vga_cntrl_reg(display);
+	bool mmio = has_vga_mmio_access(display);
+	bool io_decode;
+	u8 msr, sr1;
+	u32 tmp;
+	int err;
 
-	if (pci_bus_write_config_word(pdev->bus, PCI_DEVFN(0, 0), reg, gmch_ctrl)) {
-		drm_err(display->drm, "failed to write control word\n");
-		return -EIO;
+	if (!intel_vga_decode_is_enabled(display)) {
+		drm_dbg_kms(display->drm, "VGA decode is disabled\n");
+
+		/*
+		 * On older hardware VGA_DISP_DISABLE defaults to 0, but
+		 * it *must* be set or else the pipe will be completely
+		 * stuck (at least on g4x).
+		 */
+		goto reset_vgacntr;
 	}
 
-	return 0;
+	tmp = intel_de_read(display, vga_reg);
+
+	if ((tmp & VGA_DISP_DISABLE) == 0) {
+		enum pipe pipe;
+
+		if (display->platform.cherryview)
+			pipe = REG_FIELD_GET(VGA_PIPE_SEL_MASK_CHV, tmp);
+		else if (has_vga_pipe_sel(display))
+			pipe = REG_FIELD_GET(VGA_PIPE_SEL_MASK, tmp);
+		else
+			pipe = PIPE_A;
+
+		drm_dbg_kms(display->drm, "Disabling VGA plane on pipe %c\n",
+			    pipe_name(pipe));
+	} else {
+		drm_dbg_kms(display->drm, "VGA plane is disabled\n");
+
+		/*
+		 * Unfortunately at least some BIOSes (eg. HSW Lenovo
+		 * ThinkCentre E73) set up the VGA registers even when
+		 * in UEFI mode with the VGA plane disabled. So we need to
+		 * always clean up the mess for iGPUs. For discrete GPUs we
+		 * don't really care about the state of the VGA registers
+		 * since all VGA accesses can be blocked via the bridge.
+		 */
+		if (display->platform.dgfx)
+			goto reset_vgacntr;
+	}
+
+	/*
+	 * This should not fail, because the vga_get() family of functions
+	 * will only report errors for dGPUs that are unreachable via the
+	 * bridge, and cannot be made reachable either. We shouldn't even
+	 * get here for this case, but if we do, we assume that the bridge
+	 * will also refuse future requests to forward VGA accesses.
+	 */
+	err = intel_vga_get(display, mmio, &io_decode);
+	if (err)
+		goto reset_vgacntr;
+
+	drm_WARN_ON(display->drm, !mmio && !intel_pci_has_vga_io_decode(pdev));
+
+	intel_vga_write(display, VGA_SEQ_I, 0x01, mmio);
+	sr1 = intel_vga_read(display, VGA_SEQ_D, mmio);
+	sr1 |= VGA_SR01_SCREEN_OFF;
+	intel_vga_write(display, VGA_SEQ_D, sr1, mmio);
+
+	msr = intel_vga_read(display, VGA_MIS_R, mmio);
+	/*
+	 * Always disable VGA memory decode for iGPU so that
+	 * intel_vga_set_decode() doesn't need to access VGA registers.
+	 * VGA_MIS_ENB_MEM_ACCESS=0 is also the reset value.
+	 */
+	msr &= ~VGA_MIS_ENB_MEM_ACCESS;
+	/*
+	 * VGA_MIS_COLOR controls both GPU level and display engine level
+	 * MDA vs. CGA decode logic. But when the register gets reset
+	 * (reset value has VGA_MIS_COLOR=0) by the power well, only the
+	 * display engine level decode logic gets notified.
+	 *
+	 * Switch to MDA mode to make sure the GPU level decode logic will
+	 * be in sync with the display engine level decode logic after the
+	 * power well has been reset. Otherwise the GPU will claim CGA
+	 * register accesses but the display engine will not, causing
+	 * RMbus NoClaim errors.
+	 */
+	msr &= ~VGA_MIS_COLOR;
+	intel_vga_write(display, VGA_MIS_W, msr, mmio);
+
+	intel_vga_put(display, io_decode, mmio);
+
+	/*
+	 * Inform the arbiter about VGA memory decode being disabled so
+	 * that it doesn't disable all memory decode for the iGPU when
+	 * targeting another GPU.
+	 */
+	if (!display->platform.dgfx)
+		vga_set_legacy_decoding(pdev, VGA_RSRC_LEGACY_IO);
+
+	udelay(300);
+
+reset_vgacntr:
+	intel_de_write(display, vga_reg, VGA_DISP_DISABLE);
+	intel_de_posting_read(display, vga_reg);
 }
 
-static unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode)
+static unsigned int intel_vga_set_decode(struct pci_dev *pdev, bool enable_decode)
 {
 	struct intel_display *display = to_intel_display(pdev);
+	unsigned int decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 
-	intel_gmch_vga_set_state(display, enable_decode);
+	drm_dbg_kms(display->drm, "%s VGA decode due to VGA arbitration\n",
+		    str_enable_disable(enable_decode));
 
-	if (enable_decode)
-		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
-		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-	else
-		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+	/*
+	 * Can't use GMCH_CTRL INTEL_GMCH_VGA_DISABLE to disable VGA
+	 * decode on ILK+ since the register is locked. Instead
+	 * intel_disable_vga() will disable VGA memory decode for the
+	 * iGPU, and here we just need to take care of the IO decode.
+	 * For discrete GPUs we rely on the bridge VGA control.
+	 *
+	 * We can't disable IO decode already in intel_vga_disable()
+	 * because at least some laptops (eg. CTG Dell Latitude E5400)
+	 * will hang during reboot/shutfown with IO decode disabled.
+	 */
+	if (display->platform.dgfx) {
+		if (!enable_decode)
+			intel_pci_bridge_set_vga(pdev, false);
+		else
+			decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
+	} else {
+		if (!enable_decode)
+			intel_pci_set_io_decode(pdev, false);
+		else
+			decodes |= VGA_RSRC_LEGACY_IO;
+	}
+
+	return decodes;
 }
 
-int intel_vga_register(struct intel_display *display)
+void intel_vga_register(struct intel_display *display)
 {
 
 	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
@@ -149,11 +331,8 @@ int intel_vga_register(struct intel_display *display)
 	 * then we do not take part in VGA arbitration and the
 	 * vga_client_register() fails with -ENODEV.
 	 */
-	ret = vga_client_register(pdev, intel_gmch_vga_set_decode);
-	if (ret && ret != -ENODEV)
-		return ret;
-
-	return 0;
+	ret = vga_client_register(pdev, intel_vga_set_decode);
+	drm_WARN_ON(display->drm, ret && ret != -ENODEV);
 }
 
 void intel_vga_unregister(struct intel_display *display)
diff --git a/drivers/gpu/drm/i915/display/intel_vga.h b/drivers/gpu/drm/i915/display/intel_vga.h
index 16d699f3b6418..72131cb536cda 100644
--- a/drivers/gpu/drm/i915/display/intel_vga.h
+++ b/drivers/gpu/drm/i915/display/intel_vga.h
@@ -6,11 +6,14 @@
 #ifndef __INTEL_VGA_H__
 #define __INTEL_VGA_H__
 
+#include <linux/types.h>
+
 struct intel_display;
 
+u8 intel_vga_read(struct intel_display *display, u16 reg, bool mmio);
 void intel_vga_reset_io_mem(struct intel_display *display);
 void intel_vga_disable(struct intel_display *display);
-int intel_vga_register(struct intel_display *display);
+void intel_vga_register(struct intel_display *display);
 void intel_vga_unregister(struct intel_display *display);
 
 #endif /* __INTEL_VGA_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index bea0057523271..8a957804cb970 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -6,6 +6,7 @@
 
 #include <drm/drm_print.h>
 
+#include "intel_alpm.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
@@ -520,6 +521,7 @@ int intel_vrr_compute_optimized_guardband(struct intel_crtc_state *crtc_state)
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
 		guardband = max(guardband, intel_psr_min_guardband(crtc_state));
 		guardband = max(guardband, intel_dp_sdp_min_guardband(crtc_state, true));
+		guardband = max(guardband, intel_alpm_lobf_min_guardband(crtc_state));
 	}
 
 	return guardband;
@@ -688,13 +690,32 @@ intel_vrr_dcb_reset(const struct intel_crtc_state *old_crtc_state,
 	intel_de_write(display, PIPEDMC_DCB_BALANCE_RESET(pipe), 0);
 }
 
+static u32 trans_vrr_push(const struct intel_crtc_state *crtc_state,
+			  bool send_push)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	u32 trans_vrr_push = 0;
+
+	if (intel_vrr_always_use_vrr_tg(display) ||
+	    crtc_state->vrr.enable)
+		trans_vrr_push |= TRANS_PUSH_EN;
+
+	if (send_push)
+		trans_vrr_push |= TRANS_PUSH_SEND;
+
+	if (HAS_PSR_TRANS_PUSH_FRAME_CHANGE(display))
+		trans_vrr_push |= LNL_TRANS_PUSH_PSR_PR_EN;
+
+	return trans_vrr_push;
+}
+
 void intel_vrr_send_push(struct intel_dsb *dsb,
 			 const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
-	if (!crtc_state->vrr.enable)
+	if (!crtc_state->vrr.enable && !intel_psr_use_trans_push(crtc_state))
 		return;
 
 	if (dsb)
@@ -702,8 +723,7 @@ void intel_vrr_send_push(struct intel_dsb *dsb,
 
 	intel_de_write_dsb(display, dsb,
 			   TRANS_PUSH(display, cpu_transcoder),
-			   TRANS_PUSH_EN | TRANS_PUSH_SEND);
-
+			   trans_vrr_push(crtc_state, true));
 	if (dsb)
 		intel_dsb_nonpost_end(dsb);
 }
@@ -888,7 +908,8 @@ static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state,
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	u32 vrr_ctl;
 
-	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder), TRANS_PUSH_EN);
+	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder),
+		       trans_vrr_push(crtc_state, false));
 
 	vrr_ctl = VRR_CTL_VRR_ENABLE | trans_vrr_ctl(crtc_state);
 
@@ -916,7 +937,8 @@ static void intel_vrr_tg_disable(const struct intel_crtc_state *old_crtc_state)
 				       VRR_STATUS_VRR_EN_LIVE, 1000))
 		drm_err(display->drm, "Timed out waiting for VRR live status to clear\n");
 
-	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder), 0);
+	intel_de_rmw(display, TRANS_PUSH(display, cpu_transcoder),
+		     TRANS_PUSH_EN, 0);
 }
 
 void intel_vrr_enable(const struct intel_crtc_state *crtc_state)
@@ -971,6 +993,15 @@ void intel_vrr_transcoder_disable(const struct intel_crtc_state *old_crtc_state)
 		intel_vrr_tg_disable(old_crtc_state);
 }
 
+void intel_vrr_psr_frame_change_enable(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+
+	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder),
+		       trans_vrr_push(crtc_state, false));
+}
+
 bool intel_vrr_is_fixed_rr(const struct intel_crtc_state *crtc_state)
 {
 	return crtc_state->vrr.flipline &&
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.h b/drivers/gpu/drm/i915/display/intel_vrr.h
index bedcc8c4bff2f..4f16ca4af91fc 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.h
+++ b/drivers/gpu/drm/i915/display/intel_vrr.h
@@ -33,6 +33,7 @@ void intel_vrr_dcb_increment_flip_count(struct intel_crtc_state *crtc_state,
 					struct intel_crtc *crtc);
 bool intel_vrr_is_push_sent(const struct intel_crtc_state *crtc_state);
 void intel_vrr_disable(const struct intel_crtc_state *old_crtc_state);
+void intel_vrr_psr_frame_change_enable(const struct intel_crtc_state *crtc_state);
 void intel_vrr_get_config(struct intel_crtc_state *crtc_state);
 int intel_vrr_vmax_vtotal(const struct intel_crtc_state *crtc_state);
 int intel_vrr_vmin_vtotal(const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_vrr_regs.h b/drivers/gpu/drm/i915/display/intel_vrr_regs.h
index 427ada0d3973b..9d4d6573a149c 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_vrr_regs.h
@@ -165,6 +165,7 @@
 #define TRANS_PUSH(display, trans)		_MMIO_TRANS2((display), (trans), _TRANS_PUSH_A)
 #define   TRANS_PUSH_EN				REG_BIT(31)
 #define   TRANS_PUSH_SEND			REG_BIT(30)
+#define   LNL_TRANS_PUSH_PSR_PR_EN		REG_BIT(16)
 
 #define _TRANS_VRR_VSYNC_A			0x60078
 #define TRANS_VRR_VSYNC(display, trans)		_MMIO_TRANS2((display), (trans), _TRANS_VRR_VSYNC_A)
diff --git a/drivers/gpu/drm/i915/display/skl_scaler.c b/drivers/gpu/drm/i915/display/skl_scaler.c
index 4c4deac7f9c84..7c5cb188ebf03 100644
--- a/drivers/gpu/drm/i915/display/skl_scaler.c
+++ b/drivers/gpu/drm/i915/display/skl_scaler.c
@@ -823,7 +823,7 @@ void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
 			crtc_state->scaler_state.scaler_id < 0))
 		return;
 
-	if (intel_display_wa(display, 14011503117))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14011503117))
 		adl_scaler_ecc_mask(crtc_state);
 
 	drm_rect_init(&src, 0, 0,
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index 746e942cafd20..11ba42c67e3ed 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -17,7 +17,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
-#include "intel_dpt.h"
+#include "intel_display_wa.h"
 #include "intel_fb.h"
 #include "intel_fbc.h"
 #include "intel_frontbuffer.h"
@@ -1217,7 +1217,7 @@ static u32 skl_plane_ctl(const struct intel_plane_state *plane_state)
 		plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
 
 	/* Wa_22012358565:adl-p */
-	if (DISPLAY_VER(display) == 13)
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_22012358565))
 		plane_ctl |= adlp_plane_ctl_arb_slots(plane_state);
 
 	return plane_ctl;
@@ -2793,8 +2793,7 @@ static bool tgl_plane_has_mc_ccs(struct intel_display *display,
 				 enum plane_id plane_id)
 {
 	/* Wa_14010477008 */
-	if (display->platform.dg1 || display->platform.rocketlake ||
-	    (display->platform.tigerlake && IS_DISPLAY_STEP(display, STEP_A0, STEP_D0)))
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_14010477008))
 		return false;
 
 	return plane_id < PLANE_6;
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index f5a6fae815d1d..b1f9546b8cda8 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -7,8 +7,8 @@
 
 #include <drm/drm_blend.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
-#include "i915_reg.h"
 #include "i9xx_wm.h"
 #include "intel_atomic.h"
 #include "intel_bw.h"
@@ -22,11 +22,12 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_display_wa.h"
 #include "intel_dram.h"
 #include "intel_fb.h"
 #include "intel_fixed.h"
 #include "intel_flipq.h"
-#include "intel_pcode.h"
+#include "intel_parent.h"
 #include "intel_plane.h"
 #include "intel_vblank.h"
 #include "intel_wm.h"
@@ -115,9 +116,8 @@ intel_sagv_block_time(struct intel_display *display)
 		u32 val = 0;
 		int ret;
 
-		ret = intel_pcode_read(display->drm,
-				       GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
-				       &val, NULL);
+		ret = intel_parent_pcode_read(display, GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
+					      &val, NULL);
 		if (ret) {
 			drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n");
 			return 0;
@@ -184,8 +184,8 @@ static void skl_sagv_enable(struct intel_display *display)
 		return;
 
 	drm_dbg_kms(display->drm, "Enabling SAGV\n");
-	ret = intel_pcode_write(display->drm, GEN9_PCODE_SAGV_CONTROL,
-				GEN9_SAGV_ENABLE);
+	ret = intel_parent_pcode_write(display, GEN9_PCODE_SAGV_CONTROL,
+				       GEN9_SAGV_ENABLE);
 
 	/* We don't need to wait for SAGV when enabling */
 
@@ -217,9 +217,9 @@ static void skl_sagv_disable(struct intel_display *display)
 
 	drm_dbg_kms(display->drm, "Disabling SAGV\n");
 	/* bspec says to keep retrying for at least 1 ms */
-	ret = intel_pcode_request(display->drm, GEN9_PCODE_SAGV_CONTROL,
-				  GEN9_SAGV_DISABLE,
-				  GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, 1);
+	ret = intel_parent_pcode_request(display, GEN9_PCODE_SAGV_CONTROL,
+					 GEN9_SAGV_DISABLE,
+					 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, 1);
 	/*
 	 * Some skl systems, pre-release machines in particular,
 	 * don't actually have SAGV.
@@ -3283,7 +3283,7 @@ static void skl_read_wm_latency(struct intel_display *display)
 
 	/* read the first set of memory latencies[0:3] */
 	val = 0; /* data0 to be programmed to 0 for first set */
-	ret = intel_pcode_read(display->drm, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
+	ret = intel_parent_pcode_read(display, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
 	if (ret) {
 		drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
 		return;
@@ -3296,7 +3296,7 @@ static void skl_read_wm_latency(struct intel_display *display)
 
 	/* read the second set of memory latencies[4:7] */
 	val = 1; /* data0 to be programmed to 1 for second set */
-	ret = intel_pcode_read(display->drm, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
+	ret = intel_parent_pcode_read(display, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
 	if (ret) {
 		drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
 		return;
@@ -3413,7 +3413,7 @@ static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc,
 	if (DISPLAY_VER(display) >= 14)
 		val |= dbuf_state->joined_mbus ?
 			MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8);
-	else if (display->platform.alderlake_p)
+	else if (intel_display_wa(display, INTEL_DISPLAY_WA_22010947358))
 		/* Wa_22010947358:adl-p */
 		val |= dbuf_state->joined_mbus ?
 			MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 60857d2afdb16..36591d7246389 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -33,7 +33,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
-#include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_backlight.h"
 #include "intel_connector.h"
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
index 30cc08583cbdb..7782ba44fabd8 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
@@ -22,7 +22,7 @@ static void __do_clflush(struct drm_i915_gem_object *obj)
 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
 	drm_clflush_sg(obj->mm.pages);
 
-	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU);
 }
 
 static void clflush_work(struct dma_fence_work *base)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index ef3b14ae2e0df..df7502391b504 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -68,7 +68,7 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
 			i915_vma_flush_writes(vma);
 		spin_unlock(&obj->vma.lock);
 
-		i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+		i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU);
 		break;
 
 	case I915_GEM_DOMAIN_WC:
@@ -647,7 +647,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	i915_gem_object_unlock(obj);
 
 	if (!err && write_domain)
-		i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
+		i915_gem_object_frontbuffer_invalidate(obj, ORIGIN_CPU);
 
 out:
 	i915_gem_object_put(obj);
@@ -759,7 +759,7 @@ int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
 	}
 
 out:
-	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_invalidate(obj, ORIGIN_CPU);
 	obj->mm.dirty = true;
 	/* return with the pages pinned */
 	return 0;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 3f6f040c359db..5172d39826548 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -372,12 +372,12 @@ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj)
 	 * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for
 	 * dma-buf, so it's safe to take the lock.
 	 */
-	if (obj->base.import_attach)
+	if (drm_gem_is_imported(&obj->base))
 		i915_gem_object_lock(obj, NULL);
 
 	__i915_gem_object_put_pages(obj);
 
-	if (obj->base.import_attach)
+	if (drm_gem_is_imported(&obj->base))
 		i915_gem_object_unlock(obj);
 
 	GEM_BUG_ON(i915_gem_object_has_pages(obj));
@@ -391,7 +391,7 @@ void __i915_gem_free_object(struct drm_i915_gem_object *obj)
 
 	bitmap_free(obj->bit_17);
 
-	if (obj->base.import_attach)
+	if (drm_gem_is_imported(&obj->base))
 		drm_prime_gem_destroy(&obj->base, NULL);
 
 	drm_gem_free_mmap_offset(&obj->base);
@@ -474,30 +474,6 @@ static void i915_gem_free_object(struct drm_gem_object *gem_obj)
 		queue_work(i915->wq, &i915->mm.free_work);
 }
 
-void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
-					 enum fb_op_origin origin)
-{
-	struct i915_frontbuffer *front;
-
-	front = i915_gem_object_frontbuffer_lookup(obj);
-	if (front) {
-		intel_frontbuffer_flush(&front->base, origin);
-		i915_gem_object_frontbuffer_put(front);
-	}
-}
-
-void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
-					      enum fb_op_origin origin)
-{
-	struct i915_frontbuffer *front;
-
-	front = i915_gem_object_frontbuffer_lookup(obj);
-	if (front) {
-		intel_frontbuffer_invalidate(&front->base, origin);
-		i915_gem_object_frontbuffer_put(front);
-	}
-}
-
 static void
 i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)
 {
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.c
index adba3fa96c05d..f885c4fb1326e 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 /* Copyright © 2025 Intel Corporation */
 
+#include <drm/intel/display_parent_interface.h>
+
 #include "i915_drv.h"
 #include "i915_gem_object_frontbuffer.h"
 
@@ -101,3 +103,70 @@ void i915_gem_object_frontbuffer_put(struct i915_frontbuffer *front)
 	kref_put_lock(&front->ref, frontbuffer_release,
 		      &i915->frontbuffer_lock);
 }
+
+void __i915_gem_object_frontbuffer_flush(struct drm_i915_gem_object *obj,
+					 enum fb_op_origin origin)
+{
+	struct i915_frontbuffer *front;
+
+	front = i915_gem_object_frontbuffer_lookup(obj);
+	if (front) {
+		intel_frontbuffer_flush(&front->base, origin);
+		i915_gem_object_frontbuffer_put(front);
+	}
+}
+
+void __i915_gem_object_frontbuffer_invalidate(struct drm_i915_gem_object *obj,
+					      enum fb_op_origin origin)
+{
+	struct i915_frontbuffer *front;
+
+	front = i915_gem_object_frontbuffer_lookup(obj);
+	if (front) {
+		intel_frontbuffer_invalidate(&front->base, origin);
+		i915_gem_object_frontbuffer_put(front);
+	}
+}
+
+static struct intel_frontbuffer *i915_frontbuffer_get(struct drm_gem_object *_obj)
+{
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	struct i915_frontbuffer *front;
+
+	front = i915_gem_object_frontbuffer_get(obj);
+	if (!front)
+		return NULL;
+
+	return &front->base;
+}
+
+static void i915_frontbuffer_ref(struct intel_frontbuffer *_front)
+{
+	struct i915_frontbuffer *front =
+		container_of(_front, typeof(*front), base);
+
+	i915_gem_object_frontbuffer_ref(front);
+}
+
+static void i915_frontbuffer_put(struct intel_frontbuffer *_front)
+{
+	struct i915_frontbuffer *front =
+		container_of(_front, typeof(*front), base);
+
+	return i915_gem_object_frontbuffer_put(front);
+}
+
+static void i915_frontbuffer_flush_for_display(struct intel_frontbuffer *_front)
+{
+	struct i915_frontbuffer *front =
+		container_of(_front, typeof(*front), base);
+
+	i915_gem_object_flush_if_display(front->obj);
+}
+
+const struct intel_display_frontbuffer_interface i915_display_frontbuffer_interface = {
+	.get = i915_frontbuffer_get,
+	.ref = i915_frontbuffer_ref,
+	.put = i915_frontbuffer_put,
+	.flush_for_display = i915_frontbuffer_flush_for_display,
+};
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
index 2133e29047c51..9c6d91f21c190 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
@@ -20,31 +20,41 @@ struct i915_frontbuffer {
 	struct kref ref;
 };
 
-void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
+void __i915_gem_object_frontbuffer_flush(struct drm_i915_gem_object *obj,
 					 enum fb_op_origin origin);
-void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
+void __i915_gem_object_frontbuffer_invalidate(struct drm_i915_gem_object *obj,
 					      enum fb_op_origin origin);
 
 static inline void
-i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
+i915_gem_object_frontbuffer_flush(struct drm_i915_gem_object *obj,
 				  enum fb_op_origin origin)
 {
 	if (unlikely(rcu_access_pointer(obj->frontbuffer)))
-		__i915_gem_object_flush_frontbuffer(obj, origin);
+		__i915_gem_object_frontbuffer_flush(obj, origin);
 }
 
 static inline void
-i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
+i915_gem_object_frontbuffer_invalidate(struct drm_i915_gem_object *obj,
 				       enum fb_op_origin origin)
 {
 	if (unlikely(rcu_access_pointer(obj->frontbuffer)))
-		__i915_gem_object_invalidate_frontbuffer(obj, origin);
+		__i915_gem_object_frontbuffer_invalidate(obj, origin);
 }
 
 struct i915_frontbuffer *i915_gem_object_frontbuffer_get(struct drm_i915_gem_object *obj);
 void i915_gem_object_frontbuffer_ref(struct i915_frontbuffer *front);
 void i915_gem_object_frontbuffer_put(struct i915_frontbuffer *front);
 
+static inline void i915_gem_object_frontbuffer_track(struct i915_frontbuffer *_old,
+						     struct i915_frontbuffer *_new,
+						     unsigned int frontbuffer_bits)
+{
+	struct intel_frontbuffer *old = _old ? &_old->base : NULL;
+	struct intel_frontbuffer *new = _new ? &_new->base : NULL;
+
+	intel_frontbuffer_track(old, new, frontbuffer_bits);
+}
+
 /**
  * i915_gem_object_frontbuffer_lookup - Look up the object's frontbuffer
  * @obj: The object whose frontbuffer to look up.
@@ -81,4 +91,6 @@ i915_gem_object_frontbuffer_lookup(const struct drm_i915_gem_object *obj)
 	return front;
 }
 
+extern const struct intel_display_frontbuffer_interface i915_display_frontbuffer_interface;
+
 #endif
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c
index ce2780ef97efa..e375afbf458ee 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c
@@ -155,7 +155,7 @@ int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
 	 * We manually control the domain here and pretend that it
 	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 	 */
-	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_invalidate(obj, ORIGIN_CPU);
 
 	if (copy_from_user(vaddr, user_data, args->size))
 		return -EFAULT;
@@ -163,7 +163,7 @@ int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
 	drm_clflush_virt_range(vaddr, args->size);
 	intel_gt_chipset_flush(to_gt(i915));
 
-	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 720a9ad39aa2a..06543ae60706e 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -3,7 +3,7 @@
  * Copyright © 2014-2016 Intel Corporation
  */
 
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/shmem_fs.h>
 #include <linux/swap.h>
 #include <linux/uio.h>
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
index 033eda38e4b5a..de70517b4ef22 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
@@ -5,7 +5,7 @@
 
 #include <linux/shmem_fs.h>
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 #include <drm/drm_print.h>
 #include <drm/ttm/ttm_placement.h>
 #include <drm/ttm/ttm_tt.h>
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
index 3a7e202ae87de..56489cc127d64 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c
@@ -408,7 +408,7 @@ static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 
 	if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
 		INIT_WORK(&copy_work->work, __memcpy_work);
-		queue_work(system_unbound_wq, &copy_work->work);
+		queue_work(system_dfl_wq, &copy_work->work);
 	} else {
 		init_irq_work(&copy_work->irq_work, __memcpy_irq_work);
 		irq_work_queue(&copy_work->irq_work);
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index 17829ad57099e..9d454d0b46f2b 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -1159,6 +1159,7 @@ static int __igt_mmap_migrate(struct intel_memory_region **placements,
 	struct drm_i915_gem_object *obj;
 	struct i915_request *rq = NULL;
 	struct vm_area_struct *area;
+	struct file *mock_file;
 	unsigned long addr;
 	LIST_HEAD(objects);
 	u64 offset;
@@ -1178,16 +1179,25 @@ static int __igt_mmap_migrate(struct intel_memory_region **placements,
 		goto out_put;
 
 	/*
-	 * This will eventually create a GEM context, due to opening dummy drm
-	 * file, which needs a tiny amount of mappable device memory for the top
-	 * level paging structures(and perhaps scratch), so make sure we
-	 * allocate early, to avoid tears.
+	 * Pretend to open("/dev/dri/card0"), which will eventually create a GEM
+	 * context along with multiple GEM objects (for paging structures and
+	 * scratch) that are placed in mappable portion of GPU memory.
+	 * Calling fput() on the file places objects' cleanup routines in delayed
+	 * worqueues, which execute after unspecified amount of time.
+	 * Keep the file open until migration and page fault checks are done to
+	 * make sure object cleanup is not executed after igt_fill_mappable()
+	 * finishes and before migration is attempted - that would leave a gap
+	 * large enough for the migration to succeed, when we'd expect it to fail.
 	 */
-	addr = igt_mmap_offset(i915, offset, obj->base.size,
-			       PROT_WRITE, MAP_SHARED);
+	mock_file = mock_drm_getfile(i915->drm.primary, O_RDWR);
+	if (IS_ERR(mock_file))
+		return PTR_ERR(mock_file);
+
+	addr = igt_mmap_offset_with_file(i915, offset, obj->base.size,
+					 PROT_WRITE, MAP_SHARED, mock_file);
 	if (IS_ERR_VALUE(addr)) {
 		err = addr;
-		goto out_put;
+		goto out_fput;
 	}
 
 	mmap_read_lock(current->mm);
@@ -1294,6 +1304,9 @@ static int __igt_mmap_migrate(struct intel_memory_region **placements,
 out_addr:
 	vm_munmap(addr, obj->base.size);
 
+out_fput:
+	fput(mock_file);
+
 out_put:
 	i915_gem_object_put(obj);
 	igt_close_objects(i915, &objects);
diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
index e8fab45759c32..438cd4724ac4c 100644
--- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
@@ -67,7 +67,7 @@ void gen6_ppgtt_enable(struct intel_gt *gt)
 	if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
 		intel_uncore_write(uncore,
 				   GFX_MODE,
-				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+				   REG_MASKED_FIELD_ENABLE(GFX_PPGTT_ENABLE));
 }
 
 /* PPGTT support for Sandybdrige/Gen6 and later */
diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
index a2b413982ce64..c10ac0ab3bfa8 100644
--- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
@@ -148,7 +148,7 @@ __dma_fence_signal__notify(struct dma_fence *fence,
 {
 	struct dma_fence_cb *cur, *tmp;
 
-	lockdep_assert_held(fence->lock);
+	dma_fence_assert_held(fence);
 
 	list_for_each_entry_safe(cur, tmp, list, node) {
 		INIT_LIST_HEAD(&cur->node);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index d37966ec7a924..c0fd349a4600c 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -1233,7 +1233,7 @@ static int intel_engine_init_tlb_invalidation(struct intel_engine_cs *engine)
 	     engine->class == VIDEO_ENHANCEMENT_CLASS ||
 	     engine->class == COMPUTE_CLASS ||
 	     engine->class == OTHER_CLASS))
-		engine->tlb_inv.request = _MASKED_BIT_ENABLE(val);
+		engine->tlb_inv.request = REG_MASKED_FIELD_ENABLE(val);
 	else
 		engine->tlb_inv.request = val;
 
@@ -1628,7 +1628,7 @@ static int __intel_engine_stop_cs(struct intel_engine_cs *engine,
 	const i915_reg_t mode = RING_MI_MODE(engine->mmio_base);
 	int err;
 
-	intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
+	intel_uncore_write_fw(uncore, mode, REG_MASKED_FIELD_ENABLE(STOP_RING));
 
 	/*
 	 * Wa_22011802037: Prior to doing a reset, ensure CS is
@@ -1636,7 +1636,7 @@ static int __intel_engine_stop_cs(struct intel_engine_cs *engine,
 	 */
 	if (intel_engine_reset_needs_wa_22011802037(engine->gt))
 		intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base),
-				      _MASKED_BIT_ENABLE(GEN12_GFX_PREFETCH_DISABLE));
+				      REG_MASKED_FIELD_ENABLE(GEN12_GFX_PREFETCH_DISABLE));
 
 	err = __intel_wait_for_register_fw(engine->uncore, mode,
 					   MODE_IDLE, MODE_IDLE,
@@ -1692,7 +1692,7 @@ void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 {
 	ENGINE_TRACE(engine, "\n");
 
-	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
+	ENGINE_WRITE_FW(engine, RING_MI_MODE, REG_MASKED_FIELD_DISABLE(STOP_RING));
 }
 
 static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine)
@@ -1967,7 +1967,8 @@ void intel_engines_reset_default_submission(struct intel_gt *gt)
 		if (engine->sanitize)
 			engine->sanitize(engine);
 
-		engine->set_default_submission(engine);
+		if (engine->set_default_submission)
+			engine->set_default_submission(engine);
 	}
 }
 
@@ -2551,7 +2552,7 @@ void xehp_enable_ccs_engines(struct intel_engine_cs *engine)
 		return;
 
 	intel_uncore_write(engine->uncore, GEN12_RCU_MODE,
-			   _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE));
+			   REG_MASKED_FIELD_ENABLE(GEN12_RCU_MODE_CCS_ENABLE));
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index fb7bff27b45a3..26196a57041eb 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -24,7 +24,7 @@ static void intel_gsc_idle_msg_enable(struct intel_engine_cs *engine)
 	if (MEDIA_VER(i915) >= 13 && engine->id == GSC0) {
 		intel_uncore_write(engine->gt->uncore,
 				   RC_PSMI_CTRL_GSCCS,
-				   _MASKED_BIT_DISABLE(IDLE_MSG_DISABLE));
+				   REG_MASKED_FIELD_DISABLE(IDLE_MSG_DISABLE));
 		/* hysteresis 0xA=5us as recommended in spec*/
 		intel_uncore_write(engine->gt->uncore,
 				   PWRCTX_MAXCNT_GSCCS,
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index cafe0b8e6bdd0..1359fc9cb88ef 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -2934,12 +2934,12 @@ static void enable_execlists(struct intel_engine_cs *engine)
 	intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
 
 	if (GRAPHICS_VER(engine->i915) >= 11)
-		mode = _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE);
+		mode = REG_MASKED_FIELD_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE);
 	else
-		mode = _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE);
+		mode = REG_MASKED_FIELD_ENABLE(GFX_RUN_LIST_ENABLE);
 	ENGINE_WRITE_FW(engine, RING_MODE_GEN7, mode);
 
-	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
+	ENGINE_WRITE_FW(engine, RING_MI_MODE, REG_MASKED_FIELD_DISABLE(STOP_RING));
 
 	ENGINE_WRITE_FW(engine,
 			RING_HWS_PGA,
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
index db995dce914a1..ac9aede823203 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
@@ -6,6 +6,7 @@
 #include <linux/highmem.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "display/intel_display.h"
 #include "i915_drv.h"
@@ -915,15 +916,15 @@ void intel_gt_init_swizzling(struct intel_gt *gt)
 	if (GRAPHICS_VER(i915) == 6)
 		intel_uncore_write(uncore,
 				   ARB_MODE,
-				   _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
+				   REG_MASKED_FIELD_ENABLE(ARB_MODE_SWIZZLE_SNB));
 	else if (GRAPHICS_VER(i915) == 7)
 		intel_uncore_write(uncore,
 				   ARB_MODE,
-				   _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
+				   REG_MASKED_FIELD_ENABLE(ARB_MODE_SWIZZLE_IVB));
 	else if (GRAPHICS_VER(i915) == 8)
 		intel_uncore_write(uncore,
 				   GAMTARBMODE,
-				   _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
+				   REG_MASKED_FIELD_ENABLE(ARB_MODE_SWIZZLE_BDW));
 	else
 		MISSING_CASE(GRAPHICS_VER(i915));
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index ac527d878820b..d76121e117e17 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -5,6 +5,7 @@
 
 #include <drm/drm_managed.h>
 #include <drm/intel/intel-gtt.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "gem/i915_gem_internal.h"
 #include "gem/i915_gem_lmem.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
index 75e802e10be24..d85c849c00815 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
@@ -5,6 +5,8 @@
 
 #include <linux/sched/clock.h>
 
+#include <drm/intel/intel_gmd_interrupt_regs.h>
+
 #include "i915_drv.h"
 #include "i915_irq.h"
 #include "i915_reg.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
index c0aff4b3cbba7..babaf16e72f27 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
@@ -8,6 +8,7 @@
 #include <linux/string_helpers.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
index 7421ed18d8d1d..3ba9b2206b798 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
@@ -61,6 +61,9 @@
 
 #define GMD_ID_GRAPHICS				_MMIO(0xd8c)
 #define GMD_ID_MEDIA				_MMIO(MTL_MEDIA_GSI_BASE + 0xd8c)
+#define   GMD_ID_ARCH_MASK			REG_GENMASK(31, 22)
+#define   GMD_ID_RELEASE_MASK			REG_GENMASK(21, 14)
+#define   GMD_ID_STEP				REG_GENMASK(5, 0)
 
 #define MCFG_MCR_SELECTOR			_MMIO(0xfd0)
 #define MTL_STEER_SEMAPHORE			_MMIO(0xfd0)
@@ -318,11 +321,9 @@
 #define _RING_FAULT_REG_VCS			0x4194
 #define _RING_FAULT_REG_BCS			0x4294
 #define _RING_FAULT_REG_VECS			0x4394
-#define RING_FAULT_REG(engine)			_MMIO(_PICK((engine)->class, \
-							    _RING_FAULT_REG_RCS, \
-							    _RING_FAULT_REG_VCS, \
-							    _RING_FAULT_REG_VECS, \
-							    _RING_FAULT_REG_BCS))
+#define RING_FAULT_REG(engine)			_MMIO(_PICK_EVEN((engine)->class, \
+								 _RING_FAULT_REG_RCS, \
+								 _RING_FAULT_REG_VCS))
 #define   RING_FAULT_VADDR_MASK			REG_GENMASK(31, 12) /* pre-bdw */
 #define   RING_FAULT_ENGINE_ID_MASK		REG_GENMASK(16, 12) /* bdw+ */
 #define   RING_FAULT_GTTSEL_MASK		REG_BIT(11) /* pre-bdw */
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
index 1154cd2b7c342..a48601395dce1 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
@@ -7,6 +7,8 @@
 #include <linux/sysfs.h>
 #include <linux/printk.h>
 
+#include <drm/intel/intel_pcode_regs.h>
+
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "i915_sysfs.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h
index 9d3a3ad567a0f..b54ee4f25af10 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.h
@@ -19,7 +19,7 @@
 #include <linux/io-mapping.h>
 #include <linux/kref.h>
 #include <linux/mm.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/scatterlist.h>
 #include <linux/workqueue.h>
 
diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c
index 1d19c073ba2ec..bcd707e3d4364 100644
--- a/drivers/gpu/drm/i915/gt/intel_llc.c
+++ b/drivers/gpu/drm/i915/gt/intel_llc.c
@@ -6,6 +6,8 @@
 #include <asm/tsc.h>
 #include <linux/cpufreq.h>
 
+#include <drm/intel/intel_pcode_regs.h>
+
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_gt.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index d36e543e98dfc..147d22907960b 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -846,16 +846,16 @@ static void init_common_regs(u32 * const regs,
 	u32 ctl;
 	int loc;
 
-	ctl = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH);
-	ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
+	ctl = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH);
+	ctl |= REG_MASKED_FIELD_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 	if (inhibit)
 		ctl |= CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT;
 	if (GRAPHICS_VER(engine->i915) < 11)
-		ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT |
-					   CTX_CTRL_RS_CTX_ENABLE);
+		ctl |= REG_MASKED_FIELD_DISABLE(CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT |
+						CTX_CTRL_RS_CTX_ENABLE);
 	/* Wa_14019159160 - Case 2.*/
 	if (ctx_needs_runalone(ce))
-		ctl |= _MASKED_BIT_ENABLE(GEN12_CTX_CTRL_RUNALONE_MODE);
+		ctl |= REG_MASKED_FIELD_ENABLE(GEN12_CTX_CTRL_RUNALONE_MODE);
 	regs[CTX_CONTEXT_CONTROL] = ctl;
 
 	regs[CTX_TIMESTAMP] = ce->stats.runtime.last;
@@ -1344,7 +1344,7 @@ gen12_invalidate_state_cache(u32 *cs)
 {
 	*cs++ = MI_LOAD_REGISTER_IMM(1);
 	*cs++ = i915_mmio_reg_offset(GEN12_CS_DEBUG_MODE2);
-	*cs++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
+	*cs++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
 	return cs;
 }
 
@@ -1736,22 +1736,19 @@ static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
 		/* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */
 		{
 			COMMON_SLICE_CHICKEN2,
-			__MASKED_FIELD(GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE,
-				       0),
+			REG_MASKED_FIELD_DISABLE(GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE),
 		},
 
 		/* BSpec: 11391 */
 		{
 			FF_SLICE_CHICKEN,
-			__MASKED_FIELD(FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX,
-				       FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX),
+			REG_MASKED_FIELD_ENABLE(FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX),
 		},
 
 		/* BSpec: 11299 */
 		{
 			_3D_CHICKEN3,
-			__MASKED_FIELD(_3D_CHICKEN_SF_PROVOKING_VERTEX_FIX,
-				       _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX),
+			REG_MASKED_FIELD_ENABLE(_3D_CHICKEN_SF_PROVOKING_VERTEX_FIX),
 		}
 	};
 
diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c
index 286d49ecc4494..e91e5cdca26ca 100644
--- a/drivers/gpu/drm/i915/gt/intel_rc6.c
+++ b/drivers/gpu/drm/i915/gt/intel_rc6.c
@@ -7,6 +7,8 @@
 #include <linux/string_helpers.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "display/vlv_clock.h"
 #include "gem/i915_gem_region.h"
@@ -376,9 +378,9 @@ static void chv_rc6_enable(struct intel_rc6 *rc6)
 
 	/* Allows RC6 residency counter to work */
 	intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
-			      _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
-						 VLV_MEDIA_RC6_COUNT_EN |
-						 VLV_RENDER_RC6_COUNT_EN));
+			      REG_MASKED_FIELD_ENABLE(VLV_COUNT_RANGE_HIGH |
+						      VLV_MEDIA_RC6_COUNT_EN |
+						      VLV_RENDER_RC6_COUNT_EN));
 
 	/* 3: Enable RC6 */
 	rc6->ctl_enable = GEN7_RC_CTL_TO_MODE;
@@ -401,11 +403,11 @@ static void vlv_rc6_enable(struct intel_rc6 *rc6)
 
 	/* Allows RC6 residency counter to work */
 	intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
-			      _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
-						 VLV_MEDIA_RC0_COUNT_EN |
-						 VLV_RENDER_RC0_COUNT_EN |
-						 VLV_MEDIA_RC6_COUNT_EN |
-						 VLV_RENDER_RC6_COUNT_EN));
+			      REG_MASKED_FIELD_ENABLE(VLV_COUNT_RANGE_HIGH |
+						      VLV_MEDIA_RC0_COUNT_EN |
+						      VLV_RENDER_RC0_COUNT_EN |
+						      VLV_MEDIA_RC6_COUNT_EN |
+						      VLV_RENDER_RC6_COUNT_EN));
 
 	rc6->ctl_enable =
 	    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
@@ -761,17 +763,17 @@ static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg)
 	 * set the high bit to be safe.
 	 */
 	intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
-			      _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
+			      REG_MASKED_FIELD_ENABLE(VLV_COUNT_RANGE_HIGH));
 	upper = intel_uncore_read_fw(uncore, reg);
 	do {
 		tmp = upper;
 
 		intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
-				      _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
+				      REG_MASKED_FIELD_DISABLE(VLV_COUNT_RANGE_HIGH));
 		lower = intel_uncore_read_fw(uncore, reg);
 
 		intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
-				      _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
+				      REG_MASKED_FIELD_ENABLE(VLV_COUNT_RANGE_HIGH));
 		upper = intel_uncore_read_fw(uncore, reg);
 	} while (upper != tmp && --loop);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 41b5036dc5385..984d0056c01c2 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -586,7 +586,7 @@ static int gen8_engine_reset_prepare(struct intel_engine_cs *engine)
 		return 0;
 	}
 
-	intel_uncore_write_fw(uncore, reg, _MASKED_BIT_ENABLE(request));
+	intel_uncore_write_fw(uncore, reg, REG_MASKED_FIELD_ENABLE(request));
 	ret = __intel_wait_for_register_fw(uncore, reg, mask, ack,
 					   700, 0, NULL);
 	if (ret)
@@ -602,7 +602,7 @@ static void gen8_engine_reset_cancel(struct intel_engine_cs *engine)
 {
 	intel_uncore_write_fw(engine->uncore,
 			      RING_RESET_CTL(engine->mmio_base),
-			      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
+			      REG_MASKED_FIELD_DISABLE(RESET_CTL_REQUEST_RESET));
 }
 
 static int gen8_reset_engines(struct intel_gt *gt,
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index 8314a4b0505ee..064e7cce412f5 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -4,6 +4,8 @@
  */
 
 #include <drm/drm_cache.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "gem/i915_gem_internal.h"
 
@@ -126,8 +128,7 @@ static void flush_cs_tlb(struct intel_engine_cs *engine)
 			 engine->name);
 
 	ENGINE_WRITE_FW(engine, RING_INSTPM,
-			_MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
-					   INSTPM_SYNC_FLUSH));
+			REG_MASKED_FIELD_ENABLE(INSTPM_TLB_INVALIDATE | INSTPM_SYNC_FLUSH));
 	if (__intel_wait_for_register_fw(engine->uncore,
 					 RING_INSTPM(engine->mmio_base),
 					 INSTPM_SYNC_FLUSH, 0,
@@ -170,7 +171,7 @@ static void set_pp_dir(struct intel_engine_cs *engine)
 	if (GRAPHICS_VER(engine->i915) >= 7) {
 		ENGINE_WRITE_FW(engine,
 				RING_MODE_GEN7,
-				_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+				REG_MASKED_FIELD_ENABLE(GFX_PPGTT_ENABLE));
 	}
 }
 
@@ -274,7 +275,7 @@ static int xcs_resume(struct intel_engine_cs *engine)
 
 	if (GRAPHICS_VER(engine->i915) > 2) {
 		ENGINE_WRITE_FW(engine,
-				RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
+				RING_MI_MODE, REG_MASKED_FIELD_DISABLE(STOP_RING));
 		ENGINE_POSTING_READ(engine, RING_MI_MODE);
 	}
 
@@ -717,7 +718,7 @@ static int load_pd_dir(struct i915_request *rq,
 
 	*cs++ = MI_LOAD_REGISTER_IMM(1);
 	*cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
-	*cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
+	*cs++ = REG_MASKED_FIELD_ENABLE(INSTPM_TLB_INVALIDATE);
 
 	intel_ring_advance(rq, cs);
 
@@ -766,8 +767,7 @@ static int mi_set_context(struct i915_request *rq,
 
 				*cs++ = i915_mmio_reg_offset(
 					   RING_PSMI_CTL(signaller->mmio_base));
-				*cs++ = _MASKED_BIT_ENABLE(
-						GEN6_PSMI_SLEEP_MSG_DISABLE);
+				*cs++ = REG_MASKED_FIELD_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE);
 			}
 		}
 	} else if (GRAPHICS_VER(i915) == 5) {
@@ -820,8 +820,7 @@ static int mi_set_context(struct i915_request *rq,
 
 				last_reg = RING_PSMI_CTL(signaller->mmio_base);
 				*cs++ = i915_mmio_reg_offset(last_reg);
-				*cs++ = _MASKED_BIT_DISABLE(
-						GEN6_PSMI_SLEEP_MSG_DISABLE);
+				*cs++ = REG_MASKED_FIELD_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE);
 			}
 
 			/* Insert a delay before the next switch! */
@@ -1053,7 +1052,7 @@ static void gen6_bsd_submit_request(struct i915_request *request)
 	 * will then assume that it is busy and bring it out of rc6.
 	 */
 	intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE),
-			      _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
+			      REG_MASKED_FIELD_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 
 	/* Clear the context id. Here be magic! */
 	intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
@@ -1074,7 +1073,7 @@ static void gen6_bsd_submit_request(struct i915_request *request)
 	 * and so let it sleep to conserve power when idle.
 	 */
 	intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE),
-			      _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
+			      REG_MASKED_FIELD_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 
 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index 90b7eee78f1f1..844f2716a386b 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -7,6 +7,7 @@
 
 #include <drm/intel/i915_drm.h>
 #include <drm/intel/display_parent_interface.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "display/intel_display_rps.h"
 #include "display/vlv_clock.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index b0ee3d0ae6812..24ea5d8d529c0 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -3,6 +3,8 @@
  * Copyright © 2014-2018 Intel Corporation
  */
 
+#include <drm/intel/intel_gmd_misc_regs.h>
+
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "i915_mmio_range.h"
@@ -298,39 +300,39 @@ wa_mcr_write_clr(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clr)
 static void
 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 {
-	wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
+	wa_add(wal, reg, 0, REG_MASKED_FIELD_ENABLE(val), val, true);
 }
 
 static void
 wa_mcr_masked_en(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
 {
-	wa_mcr_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
+	wa_mcr_add(wal, reg, 0, REG_MASKED_FIELD_ENABLE(val), val, true);
 }
 
 static void
 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 {
-	wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
+	wa_add(wal, reg, 0, REG_MASKED_FIELD_DISABLE(val), val, true);
 }
 
 static void
 wa_mcr_masked_dis(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
 {
-	wa_mcr_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
+	wa_mcr_add(wal, reg, 0, REG_MASKED_FIELD_DISABLE(val), val, true);
 }
 
 static void
 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
 		    u32 mask, u32 val)
 {
-	wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
+	wa_add(wal, reg, 0, REG_MASKED_FIELD(mask, val), mask, true);
 }
 
 static void
 wa_mcr_masked_field_set(struct i915_wa_list *wal, i915_mcr_reg_t reg,
 			u32 mask, u32 val)
 {
-	wa_mcr_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
+	wa_mcr_add(wal, reg, 0, REG_MASKED_FIELD(mask, val), mask, true);
 }
 
 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
@@ -664,7 +666,7 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 
 	/* WaEnableFloatBlendOptimization:icl */
 	wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
-		   _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
+		   REG_MASKED_FIELD_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
 		   0 /* write-only, so skip validation */,
 		   true);
 
@@ -1129,7 +1131,7 @@ hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 
 	wa_add(wal,
 	       HSW_ROW_CHICKEN3, 0,
-	       _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
+	       REG_MASKED_FIELD_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 	       0 /* XXX does this reg exist? */, true);
 
 	/* WaVSRefCountFullforceMissDisable:hsw */
@@ -2270,7 +2272,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 	    IS_DG2(i915)) {
 		/* Wa_14015150844 */
 		wa_mcr_add(wal, XEHP_HDC_CHICKEN0, 0,
-			   _MASKED_BIT_ENABLE(DIS_ATOMIC_CHAINING_TYPED_WRITES),
+			   REG_MASKED_FIELD_ENABLE(DIS_ATOMIC_CHAINING_TYPED_WRITES),
 			   0, true);
 	}
 
@@ -2661,7 +2663,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 	if (IS_GRAPHICS_VER(i915, 4, 6))
 		/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
 		wa_add(wal, RING_MI_MODE(RENDER_RING_BASE),
-		       0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
+		       0, REG_MASKED_FIELD_ENABLE(VS_TIMER_DISPATCH),
 		       /* XXX bit doesn't stick on Broadwater */
 		       IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
 
@@ -2677,7 +2679,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 		 * enabled.
 		 */
 		wa_add(wal, ECOSKPD(RENDER_RING_BASE),
-		       0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
+		       0, REG_MASKED_FIELD_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
 		       0 /* XXX bit doesn't stick on Broadwater */,
 		       true);
 }
@@ -2877,7 +2879,7 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 		 * we need to explicitly skip the readback.
 		 */
 		wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
-			   _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
+			   REG_MASKED_FIELD_ENABLE(ENABLE_PREFETCH_INTO_IC),
 			   0 /* write-only, so skip validation */,
 			   true);
 	}
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
index 52ec4421a211e..1c27644403236 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
@@ -624,7 +624,7 @@ int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action)
 	else
 		guc_err(guc, "Unknown crash notification: 0x%04X\n", action);
 
-	queue_work(system_unbound_wq, &guc->dead_guc_worker);
+	queue_work(system_dfl_wq, &guc->dead_guc_worker);
 
 	return 0;
 }
@@ -646,7 +646,7 @@ int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
 		guc_err(guc, "Received early exception notification!\n");
 
 	if (msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED | INTEL_GUC_RECV_MSG_EXCEPTION))
-		queue_work(system_unbound_wq, &guc->dead_guc_worker);
+		queue_work(system_dfl_wq, &guc->dead_guc_worker);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
index 8c4da526d461c..1c455d84bf9d8 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
@@ -31,7 +31,7 @@ static void ct_dead_ct_worker_func(struct work_struct *w);
 	do { \
 		if (!(ct)->dead_ct_reported) { \
 			(ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
-			queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
+			queue_work(system_dfl_wq, &(ct)->dead_ct_worker); \
 		} \
 	} while (0)
 #else
@@ -1238,7 +1238,7 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requ
 	list_add_tail(&request->link, &ct->requests.incoming);
 	spin_unlock_irqrestore(&ct->requests.lock, flags);
 
-	queue_work(system_unbound_wq, &ct->requests.worker);
+	queue_work(system_dfl_wq, &ct->requests.worker);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 142183d3f7fb4..788e59cdfac9b 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -3385,7 +3385,7 @@ static void guc_context_sched_disable(struct intel_context *ce)
 	} else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&
 		   delay) {
 		spin_unlock_irqrestore(&ce->guc_state.lock, flags);
-		mod_delayed_work(system_unbound_wq,
+		mod_delayed_work(system_dfl_wq,
 				 &ce->guc_state.sched_disable_delay_work,
 				 msecs_to_jiffies(delay));
 	} else {
@@ -3611,7 +3611,7 @@ static void guc_context_destroy(struct kref *kref)
 	 * take the GT PM for the first time which isn't allowed from an atomic
 	 * context.
 	 */
-	queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);
+	queue_work(system_dfl_wq, &guc->submission_state.destroyed_worker);
 }
 
 static int guc_context_alloc(struct intel_context *ce)
@@ -4414,9 +4414,9 @@ static void start_engine(struct intel_engine_cs *engine)
 {
 	ENGINE_WRITE_FW(engine,
 			RING_MODE_GEN7,
-			_MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
+			REG_MASKED_FIELD_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
 
-	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
+	ENGINE_WRITE_FW(engine, RING_MI_MODE, REG_MASKED_FIELD_DISABLE(STOP_RING));
 	ENGINE_POSTING_READ(engine, RING_MI_MODE);
 }
 
@@ -5380,7 +5380,7 @@ int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
 	 * A GT reset flushes this worker queue (G2H handler) so we must use
 	 * another worker to trigger a GT reset.
 	 */
-	queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);
+	queue_work(system_dfl_wq, &guc->submission_state.reset_fail_worker);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
index 5a9f7749acffc..7fac97fe30a63 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
@@ -1065,7 +1065,7 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 
 	/* Start the DMA */
 	intel_uncore_write_fw(uncore, DMA_CTRL,
-			      _MASKED_BIT_ENABLE(dma_flags | START_DMA));
+			      REG_MASKED_FIELD_ENABLE(dma_flags | START_DMA));
 
 	/* Wait for DMA to finish */
 	ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100, NULL);
@@ -1075,7 +1075,7 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 		       intel_uncore_read_fw(uncore, DMA_CTRL));
 
 	/* Disable the bits once DMA is over */
-	intel_uncore_write_fw(uncore, DMA_CTRL, _MASKED_BIT_DISABLE(dma_flags));
+	intel_uncore_write_fw(uncore, DMA_CTRL, REG_MASKED_FIELD_DISABLE(dma_flags));
 
 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
index 477c163d26609..63a75272924f5 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 /*
- * Copyright �� 2021 Intel Corporation
+ * Copyright © 2021 Intel Corporation
  */
 
 #include "gt/intel_gt_print.h"
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
index a40e7c32e6137..28e8a092f4e7d 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 /*
- * Copyright �� 2019 Intel Corporation
+ * Copyright © 2019 Intel Corporation
  */
 
 #include "gt/intel_gt_print.h"
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index b868a05018861..e4ed47ffd2d88 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -37,6 +37,7 @@
 #include <linux/slab.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "display/i9xx_plane_regs.h"
 #include "display/intel_display_regs.h"
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index fe4302c8cae53..00451fcb1005a 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -39,7 +39,6 @@
 #include "display/i9xx_plane_regs.h"
 #include "display/intel_crt_regs.h"
 #include "display/intel_cursor_regs.h"
-#include "display/intel_display.h"
 #include "display/intel_display_regs.h"
 #include "display/intel_dpio_phy.h"
 #include "display/intel_dpll_mgr.h"
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index 021afff1cd5d4..ca5b54466a65c 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -535,16 +535,7 @@ void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
 					i2c_edid->edid_available = true;
 			}
 		}
-	} else if ((op & 0x1) == DP_AUX_I2C_WRITE) {
-		/* TODO
-		 * We only support EDID reading from I2C_over_AUX. And
-		 * we do not expect the index mode to be used. Right now
-		 * the WRITE operation is ignored. It is good enough to
-		 * support the gfx driver to do EDID access.
-		 */
-	} else {
-		if (drm_WARN_ON(&i915->drm, (op & 0x1) != DP_AUX_I2C_READ))
-			return;
+	} else if ((op & 0x1) == DP_AUX_I2C_READ) {
 		if (drm_WARN_ON(&i915->drm, msg_length != 4))
 			return;
 		if (i2c_edid->edid_available && i2c_edid->target_selected) {
@@ -553,6 +544,13 @@ void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
 			aux_data_for_write = (val << 16);
 		} else
 			aux_data_for_write = (0xff << 16);
+	} else {
+		/* TODO
+		 * We only support EDID reading from I2C_over_AUX. And
+		 * we do not expect the index mode to be used. Right now
+		 * the WRITE operation is ignored. It is good enough to
+		 * support the gfx driver to do EDID access.
+		 */
 	}
 	/* write the return value in AUX_CH_DATA reg which includes:
 	 * ACK of I2C_WRITE
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index b0d8d3e74ae7f..a34f56630af98 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -40,16 +40,18 @@
 
 #include <drm/display/drm_dp.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "display/bxt_dpio_phy_regs.h"
 #include "display/i9xx_plane_regs.h"
 #include "display/intel_crt_regs.h"
 #include "display/intel_cursor_regs.h"
 #include "display/intel_display_regs.h"
-#include "display/intel_display_types.h"
 #include "display/intel_dmc_regs.h"
 #include "display/intel_dp_aux_regs.h"
 #include "display/intel_dpio_phy.h"
+#include "display/intel_dpll_mgr.h"
 #include "display/intel_fbc.h"
 #include "display/intel_fdi_regs.h"
 #include "display/intel_pps_regs.h"
@@ -79,6 +81,9 @@
 #define PCH_PP_OFF_DELAYS _MMIO(0xc720c)
 #define PCH_PP_DIVISOR _MMIO(0xc7210)
 
+#define pipe_name(p) ((p) + 'A')
+#define port_name(p) ((p) + 'A')
+
 unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt)
 {
 	struct drm_i915_private *i915 = gvt->gt->i915;
@@ -558,7 +563,7 @@ static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port)
 	int refclk = 100000;
 	enum dpio_phy phy = DPIO_PHY0;
 	enum dpio_channel ch = DPIO_CH0;
-	struct dpll clock = {};
+	int m1, m2, n, p1, p2, m, p, vco, dot;
 	u32 temp;
 
 	/* Port to PHY mapping is fixed, see bxt_ddi_phy_info{} */
@@ -587,30 +592,25 @@ static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port)
 		goto out;
 	}
 
-	clock.m1 = 2;
-	clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK,
-				 vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 0))) << 22;
+	m1 = 2;
+	m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 0))) << 22;
 	if (vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 3)) & PORT_PLL_M2_FRAC_ENABLE)
-		clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK,
-					  vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 2)));
-	clock.n = REG_FIELD_GET(PORT_PLL_N_MASK,
-				vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 1)));
-	clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK,
-				 vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch)));
-	clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK,
-				 vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch)));
-	clock.m = clock.m1 * clock.m2;
-	clock.p = clock.p1 * clock.p2 * 5;
-
-	if (clock.n == 0 || clock.p == 0) {
+		m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 2)));
+	n = REG_FIELD_GET(PORT_PLL_N_MASK, vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 1)));
+	p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch)));
+	p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch)));
+	m = m1 * m2;
+	p = p1 * p2 * 5;
+
+	if (n == 0 || p == 0) {
 		gvt_dbg_dpy("vgpu-%d PORT_%c PLL has invalid divider\n", vgpu->id, port_name(port));
 		goto out;
 	}
 
-	clock.vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock.m), clock.n << 22);
-	clock.dot = DIV_ROUND_CLOSEST(clock.vco, clock.p);
+	vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, m), n << 22);
+	dot = DIV_ROUND_CLOSEST(vco, p);
 
-	dp_br = clock.dot;
+	dp_br = dot;
 
 out:
 	return dp_br;
@@ -2047,10 +2047,10 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	bool enable_execlist;
 	int ret;
 
-	(*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(1);
+	(*(u32 *)p_data) &= ~REG_MASKED_FIELD_ENABLE(1);
 	if (IS_COFFEELAKE(vgpu->gvt->gt->i915) ||
 	    IS_COMETLAKE(vgpu->gvt->gt->i915))
-		(*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(2);
+		(*(u32 *)p_data) &= ~REG_MASKED_FIELD_ENABLE(2);
 	write_vreg(vgpu, offset, p_data, bytes);
 
 	if (IS_MASKED_BITS_ENABLED(data, 1)) {
@@ -2139,7 +2139,7 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
 
 	if (IS_MASKED_BITS_ENABLED(data, RESET_CTL_REQUEST_RESET))
 		data |= RESET_CTL_READY_TO_RESET;
-	else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET))
+	else if (data & REG_MASKED_FIELD_DISABLE(RESET_CTL_REQUEST_RESET))
 		data &= ~RESET_CTL_READY_TO_RESET;
 
 	vgpu_vreg(vgpu, offset) = data;
@@ -2152,7 +2152,7 @@ static int csfe_chicken1_mmio_write(struct intel_vgpu *vgpu,
 {
 	u32 data = *(u32 *)p_data;
 
-	(*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(0x18);
+	(*(u32 *)p_data) &= ~REG_MASKED_FIELD_ENABLE(0x18);
 	write_vreg(vgpu, offset, p_data, bytes);
 
 	if (IS_MASKED_BITS_ENABLED(data, 0x10) ||
@@ -2534,7 +2534,7 @@ static int init_bdw_mmio_info(struct intel_gvt *gvt)
 
 #define RING_REG(base) _MMIO((base) + 0xd0)
 	MMIO_RING_F(RING_REG, 4, F_RO, 0,
-		~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
+		~REG_MASKED_FIELD_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
 		ring_reset_ctl_write);
 #undef RING_REG
 
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c
index 91d22b1c62e20..f851132180373 100644
--- a/drivers/gpu/drm/i915/gvt/interrupt.c
+++ b/drivers/gpu/drm/i915/gvt/interrupt.c
@@ -32,6 +32,7 @@
 #include <linux/eventfd.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "display/intel_display_regs.h"
 
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
index d4e9d485d382d..a93999ba80926 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.c
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -34,6 +34,7 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "gt/intel_context.h"
 #include "gt/intel_engine_regs.h"
@@ -475,7 +476,7 @@ bool is_inhibit_context(struct intel_context *ce)
 {
 	const u32 *reg_state = ce->lrc_reg_state;
 	u32 inhibit_mask =
-		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
+		REG_MASKED_FIELD_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
 	return inhibit_mask ==
 		(reg_state[CTX_CONTEXT_CONTROL_VAL] & inhibit_mask);
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 90d8eb1761a3d..a4cf15e439901 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -91,9 +91,9 @@
 		((((bit) & 0xffff0000) == 0) && !!((val) & (((bit) << 16))))
 
 #define IS_MASKED_BITS_ENABLED(_val, _b) \
-		(((_val) & _MASKED_BIT_ENABLE(_b)) == _MASKED_BIT_ENABLE(_b))
+		(((_val) & REG_MASKED_FIELD_ENABLE(_b)) == REG_MASKED_FIELD_ENABLE(_b))
 #define IS_MASKED_BITS_DISABLED(_val, _b) \
-		((_val) & _MASKED_BIT_DISABLE(_b))
+		((_val) & REG_MASKED_FIELD_DISABLE(_b))
 
 #define FORCEWAKE_RENDER_GEN9_REG 0xa278
 #define FORCEWAKE_ACK_RENDER_GEN9_REG 0x0D84
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index 25c46d7b1ea7c..5cb7a72774a0e 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -193,7 +193,7 @@ active_retire(struct i915_active *ref)
 		return;
 
 	if (ref->flags & I915_ACTIVE_RETIRE_SLEEPS) {
-		queue_work(system_unbound_wq, &ref->work);
+		queue_work(system_dfl_wq, &ref->work);
 		return;
 	}
 
@@ -1045,9 +1045,10 @@ __i915_active_fence_set(struct i915_active_fence *active,
 	 * nesting rules for the fence->lock; the inner lock is always the
 	 * older lock.
 	 */
-	spin_lock_irqsave(fence->lock, flags);
+	dma_fence_lock_irqsave(fence, flags);
 	if (prev)
-		spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING);
+		spin_lock_nested(dma_fence_spinlock(prev),
+				 SINGLE_DEPTH_NESTING);
 
 	/*
 	 * A does the cmpxchg first, and so it sees C or NULL, as before, or
@@ -1061,17 +1062,18 @@ __i915_active_fence_set(struct i915_active_fence *active,
 	 */
 	while (cmpxchg(__active_fence_slot(active), prev, fence) != prev) {
 		if (prev) {
-			spin_unlock(prev->lock);
+			spin_unlock(dma_fence_spinlock(prev));
 			dma_fence_put(prev);
 		}
-		spin_unlock_irqrestore(fence->lock, flags);
+		dma_fence_unlock_irqrestore(fence, flags);
 
 		prev = i915_active_fence_get(active);
 		GEM_BUG_ON(prev == fence);
 
-		spin_lock_irqsave(fence->lock, flags);
+		dma_fence_lock_irqsave(fence, flags);
 		if (prev)
-			spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING);
+			spin_lock_nested(dma_fence_spinlock(prev),
+					 SINGLE_DEPTH_NESTING);
 	}
 
 	/*
@@ -1088,10 +1090,11 @@ __i915_active_fence_set(struct i915_active_fence *active,
 	 */
 	if (prev) {
 		__list_del_entry(&active->cb.node);
-		spin_unlock(prev->lock); /* serialise with prev->cb_list */
+		/* serialise with prev->cb_list */
+		spin_unlock(dma_fence_spinlock(prev));
 	}
 	list_add_tail(&active->cb.node, &fence->cb_list);
-	spin_unlock_irqrestore(fence->lock, flags);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	return prev;
 }
diff --git a/drivers/gpu/drm/i915/i915_bo.c b/drivers/gpu/drm/i915/i915_bo.c
new file mode 100644
index 0000000000000..1789f7cab05c4
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_bo.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2024 Intel Corporation */
+
+#include <drm/drm_panic.h>
+#include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "display/intel_fb.h"
+#include "gem/i915_gem_mman.h"
+#include "gem/i915_gem_object.h"
+#include "gem/i915_gem_object_frontbuffer.h"
+#include "pxp/intel_pxp.h"
+
+#include "i915_bo.h"
+#include "i915_debugfs.h"
+#include "i915_drv.h"
+
+static bool i915_bo_is_tiled(struct drm_gem_object *obj)
+{
+	return i915_gem_object_is_tiled(to_intel_bo(obj));
+}
+
+static bool i915_bo_is_userptr(struct drm_gem_object *obj)
+{
+	return i915_gem_object_is_userptr(to_intel_bo(obj));
+}
+
+static bool i915_bo_is_shmem(struct drm_gem_object *obj)
+{
+	return i915_gem_object_is_shmem(to_intel_bo(obj));
+}
+
+static bool i915_bo_is_protected(struct drm_gem_object *obj)
+{
+	return i915_gem_object_is_protected(to_intel_bo(obj));
+}
+
+static int i915_bo_key_check(struct drm_gem_object *obj)
+{
+	return intel_pxp_key_check(obj, false);
+}
+
+static int i915_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	return i915_gem_fb_mmap(to_intel_bo(obj), vma);
+}
+
+static int i915_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size)
+{
+	return i915_gem_object_read_from_page(to_intel_bo(obj), offset, dst, size);
+}
+
+static void i915_bo_describe(struct seq_file *m, struct drm_gem_object *obj)
+{
+	i915_debugfs_describe_obj(m, to_intel_bo(obj));
+}
+
+static int i915_bo_framebuffer_init(struct drm_gem_object *_obj,
+				    struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	unsigned int tiling, stride;
+
+	i915_gem_object_lock(obj, NULL);
+	tiling = i915_gem_object_get_tiling(obj);
+	stride = i915_gem_object_get_stride(obj);
+	i915_gem_object_unlock(obj);
+
+	if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
+		/*
+		 * If there's a fence, enforce that
+		 * the fb modifier and tiling mode match.
+		 */
+		if (tiling != I915_TILING_NONE &&
+		    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
+			drm_dbg_kms(&i915->drm,
+				    "tiling_mode doesn't match fb modifier\n");
+			return -EINVAL;
+		}
+	} else {
+		if (tiling == I915_TILING_X) {
+			mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
+		} else if (tiling == I915_TILING_Y) {
+			drm_dbg_kms(&i915->drm,
+				    "No Y tiling for legacy addfb\n");
+			return -EINVAL;
+		}
+	}
+
+	/*
+	 * gen2/3 display engine uses the fence if present,
+	 * so the tiling mode must match the fb modifier exactly.
+	 */
+	if (GRAPHICS_VER(i915) < 4 &&
+	    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
+		drm_dbg_kms(&i915->drm,
+			    "tiling_mode must match fb modifier exactly on gen2/3\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * If there's a fence, enforce that
+	 * the fb pitch and fence stride match.
+	 */
+	if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
+		drm_dbg_kms(&i915->drm,
+			    "pitch (%d) must match tiling stride (%d)\n",
+			    mode_cmd->pitches[0], stride);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void i915_bo_framebuffer_fini(struct drm_gem_object *obj)
+{
+	/* Nothing to do for i915 */
+}
+
+static struct drm_gem_object *
+i915_bo_framebuffer_lookup(struct drm_device *drm,
+			   struct drm_file *filp,
+			   const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct drm_i915_gem_object *obj;
+
+	obj = i915_gem_object_lookup(filp, mode_cmd->handles[0]);
+	if (!obj)
+		return ERR_PTR(-ENOENT);
+
+	/* object is backed with LMEM for discrete */
+	if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM_0)) {
+		/* object is "remote", not in local memory */
+		i915_gem_object_put(obj);
+		drm_dbg_kms(&i915->drm, "framebuffer must reside in local memory\n");
+		return ERR_PTR(-EREMOTE);
+	}
+
+	return intel_bo_to_drm_bo(obj);
+}
+
+const struct intel_display_bo_interface i915_display_bo_interface = {
+	.is_tiled = i915_bo_is_tiled,
+	.is_userptr = i915_bo_is_userptr,
+	.is_shmem = i915_bo_is_shmem,
+	.is_protected = i915_bo_is_protected,
+	.key_check = i915_bo_key_check,
+	.fb_mmap = i915_bo_fb_mmap,
+	.read_from_page = i915_bo_read_from_page,
+	.describe = i915_bo_describe,
+	.framebuffer_init = i915_bo_framebuffer_init,
+	.framebuffer_fini = i915_bo_framebuffer_fini,
+	.framebuffer_lookup = i915_bo_framebuffer_lookup,
+};
diff --git a/drivers/gpu/drm/i915/i915_bo.h b/drivers/gpu/drm/i915/i915_bo.h
new file mode 100644
index 0000000000000..57255d052dd9a
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_bo.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __I915_BO_H__
+#define __I915_BO_H__
+
+extern const struct intel_display_bo_interface i915_display_bo_interface;
+
+#endif /* __I915_BO_H__ */
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 42f6b44f00274..4778ba664ec75 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -33,6 +33,7 @@
 
 #include <drm/drm_debugfs.h>
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
 
 #include "gem/i915_gem_context.h"
 #include "gt/intel_gt.h"
diff --git a/drivers/gpu/drm/i915/i915_dpt.c b/drivers/gpu/drm/i915/i915_dpt.c
new file mode 100644
index 0000000000000..9f47bb563c858
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dpt.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+
+#include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "display/intel_display_core.h"
+#include "gem/i915_gem_domain.h"
+#include "gem/i915_gem_internal.h"
+#include "gem/i915_gem_lmem.h"
+#include "gt/gen8_ppgtt.h"
+
+#include "i915_dpt.h"
+#include "i915_drv.h"
+
+struct intel_dpt {
+	struct i915_address_space vm;
+
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	void __iomem *iomem;
+};
+
+#define i915_is_dpt(vm) ((vm)->is_dpt)
+
+static inline struct intel_dpt *
+i915_vm_to_dpt(struct i915_address_space *vm)
+{
+	BUILD_BUG_ON(offsetof(struct intel_dpt, vm));
+	drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));
+	return container_of(vm, struct intel_dpt, vm);
+}
+
+struct i915_address_space *i915_dpt_to_vm(struct intel_dpt *dpt)
+{
+	return &dpt->vm;
+}
+
+static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
+{
+	writeq(pte, addr);
+}
+
+static void dpt_insert_page(struct i915_address_space *vm,
+			    dma_addr_t addr,
+			    u64 offset,
+			    unsigned int pat_index,
+			    u32 flags)
+{
+	struct intel_dpt *dpt = i915_vm_to_dpt(vm);
+	gen8_pte_t __iomem *base = dpt->iomem;
+
+	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
+		     vm->pte_encode(addr, pat_index, flags));
+}
+
+static void dpt_insert_entries(struct i915_address_space *vm,
+			       struct i915_vma_resource *vma_res,
+			       unsigned int pat_index,
+			       u32 flags)
+{
+	struct intel_dpt *dpt = i915_vm_to_dpt(vm);
+	gen8_pte_t __iomem *base = dpt->iomem;
+	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
+	struct sgt_iter sgt_iter;
+	dma_addr_t addr;
+	int i;
+
+	/*
+	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
+	 * not to allow the user to override access to a read only page.
+	 */
+
+	i = vma_res->start / I915_GTT_PAGE_SIZE;
+	for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
+		gen8_set_pte(&base[i++], pte_encode | addr);
+}
+
+static void dpt_clear_range(struct i915_address_space *vm,
+			    u64 start, u64 length)
+{
+}
+
+static void dpt_bind_vma(struct i915_address_space *vm,
+			 struct i915_vm_pt_stash *stash,
+			 struct i915_vma_resource *vma_res,
+			 unsigned int pat_index,
+			 u32 flags)
+{
+	u32 pte_flags;
+
+	if (vma_res->bound_flags)
+		return;
+
+	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
+	pte_flags = 0;
+	if (vm->has_read_only && vma_res->bi.readonly)
+		pte_flags |= PTE_READ_ONLY;
+	if (vma_res->bi.lmem)
+		pte_flags |= PTE_LM;
+
+	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
+
+	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
+
+	/*
+	 * Without aliasing PPGTT there's no difference between
+	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
+	 * upgrade to both bound if we bind either to avoid double-binding.
+	 */
+	vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
+}
+
+static void dpt_unbind_vma(struct i915_address_space *vm,
+			   struct i915_vma_resource *vma_res)
+{
+	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
+}
+
+static void dpt_cleanup(struct i915_address_space *vm)
+{
+	struct intel_dpt *dpt = i915_vm_to_dpt(vm);
+
+	i915_gem_object_put(dpt->obj);
+}
+
+struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignment)
+{
+	struct drm_i915_private *i915 = dpt->vm.i915;
+	struct intel_display *display = i915->display;
+	struct ref_tracker *wakeref;
+	struct i915_vma *vma;
+	void __iomem *iomem;
+	struct i915_gem_ww_ctx ww;
+	u64 pin_flags = 0;
+	int err;
+
+	if (i915_gem_object_is_stolen(dpt->obj))
+		pin_flags |= PIN_MAPPABLE;
+
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+	atomic_inc(&display->restore.pending_fb_pin);
+
+	for_i915_gem_ww(&ww, err, true) {
+		err = i915_gem_object_lock(dpt->obj, &ww);
+		if (err)
+			continue;
+
+		vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0,
+						  alignment, pin_flags);
+		if (IS_ERR(vma)) {
+			err = PTR_ERR(vma);
+			continue;
+		}
+
+		iomem = i915_vma_pin_iomap(vma);
+		i915_vma_unpin(vma);
+
+		if (IS_ERR(iomem)) {
+			err = PTR_ERR(iomem);
+			continue;
+		}
+
+		dpt->vma = vma;
+		dpt->iomem = iomem;
+
+		i915_vma_get(vma);
+	}
+
+	dpt->obj->mm.dirty = true;
+
+	atomic_dec(&display->restore.pending_fb_pin);
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+
+	return err ? ERR_PTR(err) : vma;
+}
+
+void i915_dpt_unpin_from_ggtt(struct intel_dpt *dpt)
+{
+	i915_vma_unpin_iomap(dpt->vma);
+	i915_vma_put(dpt->vma);
+}
+
+static struct intel_dpt *i915_dpt_create(struct drm_gem_object *obj, size_t size)
+{
+	struct drm_i915_private *i915 = to_i915(obj->dev);
+	struct drm_i915_gem_object *dpt_obj;
+	struct i915_address_space *vm;
+	struct intel_dpt *dpt;
+	int ret;
+
+	if (!size)
+		size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
+
+	size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
+
+	dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
+	if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
+		dpt_obj = i915_gem_object_create_stolen(i915, size);
+	if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
+		drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
+		dpt_obj = i915_gem_object_create_shmem(i915, size);
+	}
+	if (IS_ERR(dpt_obj))
+		return ERR_CAST(dpt_obj);
+
+	ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
+	if (!ret) {
+		ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
+		i915_gem_object_unlock(dpt_obj);
+	}
+	if (ret) {
+		i915_gem_object_put(dpt_obj);
+		return ERR_PTR(ret);
+	}
+
+	dpt = kzalloc_obj(*dpt);
+	if (!dpt) {
+		i915_gem_object_put(dpt_obj);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	vm = &dpt->vm;
+
+	vm->gt = to_gt(i915);
+	vm->i915 = i915;
+	vm->dma = i915->drm.dev;
+	vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
+	vm->is_dpt = true;
+
+	i915_address_space_init(vm, VM_CLASS_DPT);
+
+	vm->insert_page = dpt_insert_page;
+	vm->clear_range = dpt_clear_range;
+	vm->insert_entries = dpt_insert_entries;
+	vm->cleanup = dpt_cleanup;
+
+	vm->vma_ops.bind_vma    = dpt_bind_vma;
+	vm->vma_ops.unbind_vma  = dpt_unbind_vma;
+
+	vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
+
+	dpt->obj = dpt_obj;
+	dpt->obj->is_dpt = true;
+
+	return dpt;
+}
+
+static void i915_dpt_destroy(struct intel_dpt *dpt)
+{
+	dpt->obj->is_dpt = false;
+	i915_vm_put(&dpt->vm);
+}
+
+static void i915_dpt_suspend(struct intel_dpt *dpt)
+{
+	i915_ggtt_suspend_vm(&dpt->vm, true);
+}
+
+static void i915_dpt_resume(struct intel_dpt *dpt)
+{
+	i915_ggtt_resume_vm(&dpt->vm, true);
+}
+
+u64 i915_dpt_offset(struct i915_vma *dpt_vma)
+{
+	return i915_vma_offset(dpt_vma);
+}
+
+const struct intel_display_dpt_interface i915_display_dpt_interface = {
+	.create = i915_dpt_create,
+	.destroy = i915_dpt_destroy,
+	.suspend = i915_dpt_suspend,
+	.resume = i915_dpt_resume,
+};
diff --git a/drivers/gpu/drm/i915/i915_dpt.h b/drivers/gpu/drm/i915/i915_dpt.h
new file mode 100644
index 0000000000000..08dbe444fe185
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dpt.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright 2026 Intel Corporation */
+
+#ifndef __I915_DPT_H__
+#define __I915_DPT_H__
+
+#include <linux/types.h>
+
+struct i915_address_space;
+struct i915_vma;
+struct intel_dpt;
+
+struct i915_address_space *i915_dpt_to_vm(struct intel_dpt *dpt);
+struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignment);
+void i915_dpt_unpin_from_ggtt(struct intel_dpt *dpt);
+u64 i915_dpt_offset(struct i915_vma *dpt_vma);
+
+extern const struct intel_display_dpt_interface i915_display_dpt_interface;
+
+#endif /* __I915_DPT_H__ */
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index f0105c5b49a71..385a634c3ed00 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -48,6 +48,7 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/intel/display_member.h>
 #include <drm/intel/display_parent_interface.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "display/i9xx_display_sr.h"
 #include "display/intel_bw.h"
@@ -77,6 +78,7 @@
 #include "gem/i915_gem_dmabuf.h"
 #include "gem/i915_gem_ioctls.h"
 #include "gem/i915_gem_mman.h"
+#include "gem/i915_gem_object_frontbuffer.h"
 #include "gem/i915_gem_pm.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_pm.h"
@@ -88,11 +90,14 @@
 #include "pxp/intel_pxp_debugfs.h"
 #include "pxp/intel_pxp_pm.h"
 
+#include "i915_bo.h"
 #include "i915_debugfs.h"
 #include "i915_display_pc8.h"
+#include "i915_dpt.h"
 #include "i915_driver.h"
 #include "i915_drm_client.h"
 #include "i915_drv.h"
+#include "i915_dsb_buffer.h"
 #include "i915_edram.h"
 #include "i915_file_private.h"
 #include "i915_getparam.h"
@@ -104,6 +109,7 @@
 #include "i915_ioctl.h"
 #include "i915_irq.h"
 #include "i915_memcpy.h"
+#include "i915_overlay.h"
 #include "i915_panic.h"
 #include "i915_perf.h"
 #include "i915_query.h"
@@ -147,10 +153,11 @@ static int i915_workqueues_init(struct drm_i915_private *dev_priv)
 	/*
 	 * The unordered i915 workqueue should be used for all work
 	 * scheduling that do not require running in order, which used
-	 * to be scheduled on the system_wq before moving to a driver
+	 * to be scheduled on the system_percpu_wq before moving to a driver
 	 * instance due deprecation of flush_scheduled_work().
 	 */
-	dev_priv->unordered_wq = alloc_workqueue("i915-unordered", 0, 0);
+	dev_priv->unordered_wq = alloc_workqueue("i915-unordered", WQ_PERCPU,
+						 0);
 	if (dev_priv->unordered_wq == NULL)
 		goto out_free_wq;
 
@@ -556,10 +563,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 			drm_dbg(&dev_priv->drm, "can't enable MSI");
 	}
 
-	ret = intel_gvt_init(dev_priv);
-	if (ret)
-		goto err_msi;
-
 	intel_opregion_setup(display);
 
 	ret = i915_pcode_init(dev_priv);
@@ -580,7 +583,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 
 err_opregion:
 	intel_opregion_cleanup(display);
-err_msi:
 	if (pdev->msi_enabled)
 		pci_disable_msi(pdev);
 err_mem_regions:
@@ -764,14 +766,21 @@ static bool vgpu_active(struct drm_device *drm)
 }
 
 static const struct intel_display_parent_interface parent = {
+	.bo = &i915_display_bo_interface,
+	.dpt = &i915_display_dpt_interface,
+	.dsb = &i915_display_dsb_interface,
+	.frontbuffer = &i915_display_frontbuffer_interface,
 	.hdcp = &i915_display_hdcp_interface,
 	.initial_plane = &i915_display_initial_plane_interface,
 	.irq = &i915_display_irq_interface,
+	.overlay = &i915_display_overlay_interface,
 	.panic = &i915_display_panic_interface,
 	.pc8 = &i915_display_pc8_interface,
+	.pcode = &i915_display_pcode_interface,
 	.rpm = &i915_display_rpm_interface,
 	.rps = &i915_display_rps_interface,
 	.stolen = &i915_display_stolen_interface,
+	.vma = &i915_display_vma_interface,
 
 	.fence_priority_display = fence_priority_display,
 	.has_auxccs = has_auxccs,
@@ -868,9 +877,13 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (ret < 0)
 		goto out_cleanup_mmio;
 
+	ret = intel_gvt_init(i915);
+	if (ret)
+		goto out_cleanup_hw;
+
 	ret = intel_display_driver_probe_noirq(display);
 	if (ret < 0)
-		goto out_cleanup_hw;
+		goto out_cleanup_gvt;
 
 	ret = intel_irq_install(i915);
 	if (ret)
@@ -919,6 +932,8 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	intel_irq_uninstall(i915);
 out_cleanup_modeset:
 	intel_display_driver_remove_nogem(display);
+out_cleanup_gvt:
+	intel_gvt_driver_remove(i915);
 out_cleanup_hw:
 	i915_driver_hw_remove(i915);
 	intel_memory_regions_driver_release(i915);
@@ -926,7 +941,6 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	i915_gem_drain_freed_objects(i915);
 	i915_ggtt_driver_late_release(i915);
 out_cleanup_mmio:
-	intel_gvt_driver_remove(i915);
 	i915_driver_mmio_release(i915);
 out_runtime_pm_put:
 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 44ba620325bc0..dafee3dcd1c52 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -61,6 +61,7 @@
 #include "intel_uncore.h"
 
 struct drm_i915_clock_gating_funcs;
+struct i915_overlay;
 struct intel_display;
 struct intel_pxp;
 struct vlv_s0ix_state;
@@ -248,7 +249,7 @@ struct drm_i915_private {
 	 *
 	 * This workqueue should be used for all unordered work
 	 * scheduling within i915, which used to be scheduled on the
-	 * system_wq before moving to a driver instance due
+	 * system_percpu_wq before moving to a driver instance due
 	 * deprecation of flush_scheduled_work().
 	 */
 	struct workqueue_struct *unordered_wq;
@@ -307,6 +308,8 @@ struct drm_i915_private {
 
 	struct intel_pxp *pxp;
 
+	struct i915_overlay *overlay;
+
 	struct i915_pmu pmu;
 
 	/* The TTM device structure. */
diff --git a/drivers/gpu/drm/i915/i915_dsb_buffer.c b/drivers/gpu/drm/i915/i915_dsb_buffer.c
new file mode 100644
index 0000000000000..b797e0ccb70c1
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dsb_buffer.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023, Intel Corporation.
+ */
+
+#include <drm/intel/display_parent_interface.h>
+
+#include "gem/i915_gem_internal.h"
+#include "gem/i915_gem_lmem.h"
+#include "i915_drv.h"
+#include "i915_dsb_buffer.h"
+#include "i915_vma.h"
+
+struct intel_dsb_buffer {
+	u32 *cmd_buf;
+	struct i915_vma *vma;
+	size_t buf_size;
+};
+
+static u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
+{
+	return i915_ggtt_offset(dsb_buf->vma);
+}
+
+static void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
+{
+	dsb_buf->cmd_buf[idx] = val;
+}
+
+static u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
+{
+	return dsb_buf->cmd_buf[idx];
+}
+
+static void intel_dsb_buffer_fill(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
+{
+	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
+
+	memset(&dsb_buf->cmd_buf[idx], val, size);
+}
+
+static struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_dsb_buffer *dsb_buf;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	u32 *buf;
+	int ret;
+
+	dsb_buf = kzalloc_obj(*dsb_buf);
+	if (!dsb_buf)
+		return ERR_PTR(-ENOMEM);
+
+	if (HAS_LMEM(i915)) {
+		obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
+						  I915_BO_ALLOC_CONTIGUOUS);
+		if (IS_ERR(obj)) {
+			ret = PTR_ERR(obj);
+			goto err;
+		}
+	} else {
+		obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
+		if (IS_ERR(obj)) {
+			ret = PTR_ERR(obj);
+			goto err;
+		}
+
+		i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
+	}
+
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		i915_gem_object_put(obj);
+		goto err;
+	}
+
+	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
+	if (IS_ERR(buf)) {
+		ret = PTR_ERR(buf);
+		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
+		goto err;
+	}
+
+	dsb_buf->vma = vma;
+	dsb_buf->cmd_buf = buf;
+	dsb_buf->buf_size = size;
+
+	return dsb_buf;
+
+err:
+	kfree(dsb_buf);
+
+	return ERR_PTR(ret);
+}
+
+static void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
+{
+	i915_vma_unpin_and_release(&dsb_buf->vma, I915_VMA_RELEASE_MAP);
+	kfree(dsb_buf);
+}
+
+static void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
+{
+	i915_gem_object_flush_map(dsb_buf->vma->obj);
+}
+
+const struct intel_display_dsb_interface i915_display_dsb_interface = {
+	.ggtt_offset = intel_dsb_buffer_ggtt_offset,
+	.write = intel_dsb_buffer_write,
+	.read = intel_dsb_buffer_read,
+	.fill = intel_dsb_buffer_fill,
+	.create = intel_dsb_buffer_create,
+	.cleanup = intel_dsb_buffer_cleanup,
+	.flush_map = intel_dsb_buffer_flush_map,
+};
diff --git a/drivers/gpu/drm/i915/i915_dsb_buffer.h b/drivers/gpu/drm/i915/i915_dsb_buffer.h
new file mode 100644
index 0000000000000..a01b4d8de947a
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dsb_buffer.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __I915_DSB_BUFFER_H__
+#define __I915_DSB_BUFFER_H__
+
+extern const struct intel_display_dsb_interface i915_display_dsb_interface;
+
+#endif /* __I915_DSB_BUFFER_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 160733619a4ac..7614917509142 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -579,7 +579,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 		goto out_rpm;
 	}
 
-	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_invalidate(obj, ORIGIN_CPU);
 
 	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = args->offset;
@@ -626,7 +626,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 	}
 
 	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
-	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU);
 
 	i915_gem_gtt_cleanup(obj, &node, vma);
 out_rpm:
@@ -714,7 +714,7 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 		offset = 0;
 	}
 
-	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+	i915_gem_object_frontbuffer_flush(obj, ORIGIN_CPU);
 
 	i915_gem_object_unpin_pages(obj);
 	return ret;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index a99b4e45d26c0..5353a9c66694b 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -31,7 +31,7 @@
 #include <linux/debugfs.h>
 #include <linux/highmem.h>
 #include <linux/nmi.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/scatterlist.h>
 #include <linux/string_helpers.h>
 #include <linux/utsname.h>
@@ -824,9 +824,6 @@ static void err_print_gt_global(struct drm_i915_error_state_buf *m,
 		err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
 			   gt->fault_data1, gt->fault_data0);
 
-	if (GRAPHICS_VER(m->i915) == 7)
-		err_printf(m, "ERR_INT: 0x%08x\n", gt->err_int);
-
 	if (IS_GRAPHICS_VER(m->i915, 8, 11))
 		err_printf(m, "GTT_CACHE_EN: 0x%08x\n", gt->gtt_cache);
 
@@ -1929,9 +1926,6 @@ static void gt_record_global_regs(struct intel_gt_coredump *gt)
 	if (IS_VALLEYVIEW(i915))
 		gt->forcewake = intel_uncore_read_fw(uncore, FORCEWAKE_VLV);
 
-	if (GRAPHICS_VER(i915) == 7)
-		gt->err_int = intel_uncore_read(uncore, GEN7_ERR_INT);
-
 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 		gt->fault_data0 = intel_gt_mcr_read_any((struct intel_gt *)gt->_gt,
 							XEHP_FAULT_TLB_DATA0);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 91b3df621a492..26970c5e291e9 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -149,7 +149,6 @@ struct intel_gt_coredump {
 	u32 gtier[6], ngtier;
 	u32 forcewake;
 	u32 error; /* gen6+ */
-	u32 err_int; /* gen7 */
 	u32 fault_data0; /* gen8, gen9 */
 	u32 fault_data1; /* gen8, gen9 */
 	u32 done_reg;
diff --git a/drivers/gpu/drm/i915/i915_hwmon.c b/drivers/gpu/drm/i915/i915_hwmon.c
index 16e64d752e12d..c4a799f5fe925 100644
--- a/drivers/gpu/drm/i915/i915_hwmon.c
+++ b/drivers/gpu/drm/i915/i915_hwmon.c
@@ -9,6 +9,8 @@
 #include <linux/types.h>
 #include <linux/units.h>
 
+#include <drm/intel/intel_pcode_regs.h>
+
 #include "i915_drv.h"
 #include "i915_hwmon.h"
 #include "i915_reg.h"
diff --git a/drivers/gpu/drm/i915/i915_initial_plane.c b/drivers/gpu/drm/i915/i915_initial_plane.c
index 7fb52d81f7b6d..390a9248d6314 100644
--- a/drivers/gpu/drm/i915/i915_initial_plane.c
+++ b/drivers/gpu/drm/i915/i915_initial_plane.c
@@ -9,6 +9,7 @@
 #include "display/intel_crtc.h"
 #include "display/intel_display_types.h"
 #include "display/intel_fb.h"
+#include "display/intel_fbdev_fb.h"
 #include "gem/i915_gem_lmem.h"
 #include "gem/i915_gem_region.h"
 
@@ -114,9 +115,10 @@ initial_plane_vma(struct drm_i915_private *i915,
 	 * important and we should probably use that space with FBC or other
 	 * features.
 	 */
-	if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
+	if (IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) &&
+	    IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
 	    mem == i915->mm.stolen_region &&
-	    size * 2 > i915->dsm.usable_size) {
+	    !intel_fbdev_fb_prefer_stolen(&i915->drm, size)) {
 		drm_dbg_kms(&i915->drm, "Initial FB size exceeds half of stolen, discarding\n");
 		return NULL;
 	}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 3fe978d4ea539..d4d8dd0a41749 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -34,6 +34,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "display/intel_display_irq.h"
 #include "display/intel_hotplug.h"
diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c
index c33bd3d830699..fd89e7c7d8d6f 100644
--- a/drivers/gpu/drm/i915/i915_mm.c
+++ b/drivers/gpu/drm/i915/i915_mm.c
@@ -108,7 +108,7 @@ int remap_io_mapping(struct vm_area_struct *vma,
 
 	err = apply_to_page_range(r.mm, addr, size, remap_pfn, &r);
 	if (unlikely(err)) {
-		zap_vma_ptes(vma, addr, (r.pfn - pfn) << PAGE_SHIFT);
+		zap_special_vma_range(vma, addr, (r.pfn - pfn) << PAGE_SHIFT);
 		return err;
 	}
 
@@ -156,7 +156,7 @@ int remap_io_sg(struct vm_area_struct *vma,
 
 	err = apply_to_page_range(r.mm, addr, size, remap_sg, &r);
 	if (unlikely(err)) {
-		zap_vma_ptes(vma, addr, r.pfn << PAGE_SHIFT);
+		zap_special_vma_range(vma, addr, r.pfn << PAGE_SHIFT);
 		return err;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_overlay.c b/drivers/gpu/drm/i915/i915_overlay.c
new file mode 100644
index 0000000000000..c2d712bd2b0da
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_overlay.c
@@ -0,0 +1,517 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026, Intel Corporation.
+ */
+
+#include <drm/drm_print.h>
+
+#include <drm/intel/display_parent_interface.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
+
+#include "gem/i915_gem_internal.h"
+#include "gem/i915_gem_object_frontbuffer.h"
+#include "gem/i915_gem_pm.h"
+
+#include "gt/intel_gpu_commands.h"
+#include "gt/intel_ring.h"
+
+#include "i915_drv.h"
+#include "i915_overlay.h"
+#include "i915_reg.h"
+#include "intel_pci_config.h"
+
+#include "display/intel_frontbuffer.h"
+
+/* overlay flip addr flag */
+#define OFC_UPDATE		0x1
+
+struct i915_overlay {
+	struct drm_i915_private *i915;
+	struct intel_context *context;
+	struct i915_vma *vma;
+	struct i915_vma *old_vma;
+	struct i915_frontbuffer *frontbuffer;
+	/* register access */
+	struct drm_i915_gem_object *reg_bo;
+	void __iomem *regs;
+	u32 flip_addr;
+	u32 frontbuffer_bits;
+	/* flip handling */
+	struct i915_active last_flip;
+	void (*flip_complete)(struct i915_overlay *overlay);
+};
+
+static void i830_overlay_clock_gating(struct drm_i915_private *i915,
+				      bool enable)
+{
+	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
+	u8 val;
+
+	/*
+	 * WA_OVERLAY_CLKGATE:alm
+	 *
+	 * FIXME should perhaps be done on the display side?
+	 */
+	if (enable)
+		intel_uncore_write(&i915->uncore, DSPCLK_GATE_D, 0);
+	else
+		intel_uncore_write(&i915->uncore, DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
+
+	/* WA_DISABLE_L2CACHE_CLOCK_GATING:alm */
+	pci_bus_read_config_byte(pdev->bus,
+				 PCI_DEVFN(0, 0), I830_CLOCK_GATE, &val);
+	if (enable)
+		val &= ~I830_L2_CACHE_CLOCK_GATE_DISABLE;
+	else
+		val |= I830_L2_CACHE_CLOCK_GATE_DISABLE;
+	pci_bus_write_config_byte(pdev->bus,
+				  PCI_DEVFN(0, 0), I830_CLOCK_GATE, val);
+}
+
+static struct i915_request *
+alloc_request(struct i915_overlay *overlay, void (*fn)(struct i915_overlay *))
+{
+	struct i915_request *rq;
+	int err;
+
+	overlay->flip_complete = fn;
+
+	rq = i915_request_create(overlay->context);
+	if (IS_ERR(rq))
+		return rq;
+
+	err = i915_active_add_request(&overlay->last_flip, rq);
+	if (err) {
+		i915_request_add(rq);
+		return ERR_PTR(err);
+	}
+
+	return rq;
+}
+
+static bool i915_overlay_is_active(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+
+	return overlay->frontbuffer_bits;
+}
+
+/* overlay needs to be disable in OCMD reg */
+static int i915_overlay_on(struct drm_device *drm,
+			   u32 frontbuffer_bits)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+	struct i915_request *rq;
+	u32 *cs;
+
+	drm_WARN_ON(drm, i915_overlay_is_active(drm));
+
+	rq = alloc_request(overlay, NULL);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	cs = intel_ring_begin(rq, 4);
+	if (IS_ERR(cs)) {
+		i915_request_add(rq);
+		return PTR_ERR(cs);
+	}
+
+	overlay->frontbuffer_bits = frontbuffer_bits;
+
+	if (IS_I830(i915))
+		i830_overlay_clock_gating(i915, false);
+
+	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_ON;
+	*cs++ = overlay->flip_addr | OFC_UPDATE;
+	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
+	*cs++ = MI_NOOP;
+	intel_ring_advance(rq, cs);
+
+	i915_request_add(rq);
+
+	return i915_active_wait(&overlay->last_flip);
+}
+
+static void i915_overlay_flip_prepare(struct i915_overlay *overlay,
+				      struct i915_vma *vma)
+{
+	struct drm_i915_private *i915 = overlay->i915;
+	struct i915_frontbuffer *frontbuffer = NULL;
+
+	drm_WARN_ON(&i915->drm, overlay->old_vma);
+
+	if (vma)
+		frontbuffer = i915_gem_object_frontbuffer_get(vma->obj);
+
+	i915_gem_object_frontbuffer_track(overlay->frontbuffer, frontbuffer,
+					  overlay->frontbuffer_bits);
+
+	if (overlay->frontbuffer)
+		i915_gem_object_frontbuffer_put(overlay->frontbuffer);
+	overlay->frontbuffer = frontbuffer;
+
+	overlay->old_vma = overlay->vma;
+	if (vma)
+		overlay->vma = i915_vma_get(vma);
+	else
+		overlay->vma = NULL;
+}
+
+/* overlay needs to be enabled in OCMD reg */
+static int i915_overlay_continue(struct drm_device *drm,
+				 struct i915_vma *vma,
+				 bool load_polyphase_filter)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+	struct i915_request *rq;
+	u32 flip_addr = overlay->flip_addr;
+	u32 *cs;
+
+	drm_WARN_ON(drm, !i915_overlay_is_active(drm));
+
+	if (load_polyphase_filter)
+		flip_addr |= OFC_UPDATE;
+
+	rq = alloc_request(overlay, NULL);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	cs = intel_ring_begin(rq, 2);
+	if (IS_ERR(cs)) {
+		i915_request_add(rq);
+		return PTR_ERR(cs);
+	}
+
+	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE;
+	*cs++ = flip_addr;
+	intel_ring_advance(rq, cs);
+
+	i915_overlay_flip_prepare(overlay, vma);
+	i915_request_add(rq);
+
+	return 0;
+}
+
+static void i915_overlay_release_old_vma(struct i915_overlay *overlay)
+{
+	struct drm_i915_private *i915 = overlay->i915;
+	struct intel_display *display = i915->display;
+	struct i915_vma *vma;
+
+	vma = fetch_and_zero(&overlay->old_vma);
+	if (drm_WARN_ON(&i915->drm, !vma))
+		return;
+
+	intel_frontbuffer_flip(display, overlay->frontbuffer_bits);
+
+	i915_vma_unpin(vma);
+	i915_vma_put(vma);
+}
+
+static void
+i915_overlay_release_old_vid_tail(struct i915_overlay *overlay)
+{
+	i915_overlay_release_old_vma(overlay);
+}
+
+static void i915_overlay_off_tail(struct i915_overlay *overlay)
+{
+	struct drm_i915_private *i915 = overlay->i915;
+
+	i915_overlay_release_old_vma(overlay);
+
+	overlay->frontbuffer_bits = 0;
+
+	if (IS_I830(i915))
+		i830_overlay_clock_gating(i915, true);
+}
+
+static void i915_overlay_last_flip_retire(struct i915_active *active)
+{
+	struct i915_overlay *overlay =
+		container_of(active, typeof(*overlay), last_flip);
+
+	if (overlay->flip_complete)
+		overlay->flip_complete(overlay);
+}
+
+/* overlay needs to be disabled in OCMD reg */
+static int i915_overlay_off(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+	struct i915_request *rq;
+	u32 *cs, flip_addr = overlay->flip_addr;
+
+	drm_WARN_ON(drm, !i915_overlay_is_active(drm));
+
+	/*
+	 * According to intel docs the overlay hw may hang (when switching
+	 * off) without loading the filter coeffs. It is however unclear whether
+	 * this applies to the disabling of the overlay or to the switching off
+	 * of the hw. Do it in both cases.
+	 */
+	flip_addr |= OFC_UPDATE;
+
+	rq = alloc_request(overlay, i915_overlay_off_tail);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	cs = intel_ring_begin(rq, 6);
+	if (IS_ERR(cs)) {
+		i915_request_add(rq);
+		return PTR_ERR(cs);
+	}
+
+	/* wait for overlay to go idle */
+	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE;
+	*cs++ = flip_addr;
+	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
+
+	/* turn overlay off */
+	*cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_OFF;
+	*cs++ = flip_addr;
+	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
+
+	intel_ring_advance(rq, cs);
+
+	i915_overlay_flip_prepare(overlay, NULL);
+	i915_request_add(rq);
+
+	return i915_active_wait(&overlay->last_flip);
+}
+
+/*
+ * Recover from an interruption due to a signal.
+ * We have to be careful not to repeat work forever an make forward progress.
+ */
+static int i915_overlay_recover_from_interrupt(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+
+	return i915_active_wait(&overlay->last_flip);
+}
+
+/*
+ * Wait for pending overlay flip and release old frame.
+ * Needs to be called before the overlay register are changed
+ * via intel_overlay_(un)map_regs.
+ */
+static int i915_overlay_release_old_vid(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+	struct i915_request *rq;
+	u32 *cs;
+
+	/*
+	 * Only wait if there is actually an old frame to release to
+	 * guarantee forward progress.
+	 */
+	if (!overlay->old_vma)
+		return 0;
+
+	if (!(intel_uncore_read(&i915->uncore, GEN2_ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT)) {
+		i915_overlay_release_old_vid_tail(overlay);
+		return 0;
+	}
+
+	rq = alloc_request(overlay, i915_overlay_release_old_vid_tail);
+	if (IS_ERR(rq))
+		return PTR_ERR(rq);
+
+	cs = intel_ring_begin(rq, 2);
+	if (IS_ERR(cs)) {
+		i915_request_add(rq);
+		return PTR_ERR(cs);
+	}
+
+	*cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
+	*cs++ = MI_NOOP;
+	intel_ring_advance(rq, cs);
+
+	i915_request_add(rq);
+
+	return i915_active_wait(&overlay->last_flip);
+}
+
+static void i915_overlay_reset(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+
+	if (!overlay)
+		return;
+
+	overlay->frontbuffer_bits = 0;
+}
+
+static struct i915_vma *i915_overlay_pin_fb(struct drm_device *drm,
+					    struct drm_gem_object *obj,
+					    u32 *offset)
+{
+	struct drm_i915_gem_object *new_bo = to_intel_bo(obj);
+	struct i915_gem_ww_ctx ww;
+	struct i915_vma *vma;
+	int ret;
+
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	ret = i915_gem_object_lock(new_bo, &ww);
+	if (!ret) {
+		vma = i915_gem_object_pin_to_display_plane(new_bo, &ww, 0, 0,
+							   NULL, PIN_MAPPABLE);
+		ret = PTR_ERR_OR_ZERO(vma);
+	}
+	if (ret == -EDEADLK) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
+		if (!ret)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (ret)
+		return ERR_PTR(ret);
+
+	*offset = i915_ggtt_offset(vma);
+
+	return vma;
+}
+
+static void i915_overlay_unpin_fb(struct drm_device *drm,
+				  struct i915_vma *vma)
+{
+	i915_vma_unpin(vma);
+}
+
+static struct drm_gem_object *
+i915_overlay_obj_lookup(struct drm_device *drm,
+			struct drm_file *file_priv,
+			u32 handle)
+{
+	struct drm_i915_gem_object *bo;
+
+	bo = i915_gem_object_lookup(file_priv, handle);
+	if (!bo)
+		return ERR_PTR(-ENOENT);
+
+	if (i915_gem_object_is_tiled(bo)) {
+		drm_dbg(drm, "buffer used for overlay image can not be tiled\n");
+		i915_gem_object_put(bo);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return intel_bo_to_drm_bo(bo);
+}
+
+static int get_registers(struct i915_overlay *overlay, bool use_phys)
+{
+	struct drm_i915_private *i915 = overlay->i915;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int err;
+
+	obj = i915_gem_object_create_stolen(i915, PAGE_SIZE);
+	if (IS_ERR(obj))
+		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_put_bo;
+	}
+
+	if (use_phys)
+		overlay->flip_addr = sg_dma_address(obj->mm.pages->sgl);
+	else
+		overlay->flip_addr = i915_ggtt_offset(vma);
+	overlay->regs = i915_vma_pin_iomap(vma);
+	i915_vma_unpin(vma);
+
+	if (IS_ERR(overlay->regs)) {
+		err = PTR_ERR(overlay->regs);
+		goto err_put_bo;
+	}
+
+	overlay->reg_bo = obj;
+	return 0;
+
+err_put_bo:
+	i915_gem_object_put(obj);
+	return err;
+}
+
+static void __iomem *i915_overlay_setup(struct drm_device *drm,
+					bool needs_physical)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_engine_cs *engine;
+	struct i915_overlay *overlay;
+	int ret;
+
+	engine = to_gt(i915)->engine[RCS0];
+	if (!engine || !engine->kernel_context)
+		return ERR_PTR(-ENOENT);
+
+	overlay = kzalloc_obj(*overlay);
+	if (!overlay)
+		return ERR_PTR(-ENOMEM);
+
+	overlay->i915 = i915;
+	overlay->context = engine->kernel_context;
+
+	i915_active_init(&overlay->last_flip,
+			 NULL, i915_overlay_last_flip_retire, 0);
+
+	ret = get_registers(overlay, needs_physical);
+	if (ret) {
+		kfree(overlay);
+		return ERR_PTR(ret);
+	}
+
+	i915->overlay = overlay;
+
+	return overlay->regs;
+}
+
+static void i915_overlay_cleanup(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct i915_overlay *overlay = i915->overlay;
+
+	if (!overlay)
+		return;
+
+	/*
+	 * The bo's should be free'd by the generic code already.
+	 * Furthermore modesetting teardown happens beforehand so the
+	 * hardware should be off already.
+	 */
+	drm_WARN_ON(drm, i915_overlay_is_active(drm));
+
+	i915_gem_object_put(overlay->reg_bo);
+	i915_active_fini(&overlay->last_flip);
+
+	kfree(overlay);
+	i915->overlay = NULL;
+}
+
+const struct intel_display_overlay_interface i915_display_overlay_interface = {
+	.is_active = i915_overlay_is_active,
+	.overlay_on = i915_overlay_on,
+	.overlay_continue = i915_overlay_continue,
+	.overlay_off = i915_overlay_off,
+	.recover_from_interrupt = i915_overlay_recover_from_interrupt,
+	.release_old_vid = i915_overlay_release_old_vid,
+	.reset = i915_overlay_reset,
+	.obj_lookup = i915_overlay_obj_lookup,
+	.pin_fb = i915_overlay_pin_fb,
+	.unpin_fb = i915_overlay_unpin_fb,
+	.setup = i915_overlay_setup,
+	.cleanup = i915_overlay_cleanup,
+};
diff --git a/drivers/gpu/drm/i915/i915_overlay.h b/drivers/gpu/drm/i915/i915_overlay.h
new file mode 100644
index 0000000000000..f8053eb8d1898
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_overlay.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __I915_OVERLAY_H__
+#define __I915_OVERLAY_H__
+
+extern const struct intel_display_overlay_interface i915_display_overlay_interface;
+
+#endif /* __I915_OVERLAY_H__ */
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 2820e8f0f7650..19b82427aa412 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -2635,10 +2635,9 @@ static int gen12_configure_oar_context(struct i915_perf_stream *stream,
 		{
 			RING_CONTEXT_CONTROL(ce->engine->mmio_base),
 			CTX_CONTEXT_CONTROL,
-			_MASKED_FIELD(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE,
-				      active ?
-				      GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE :
-				      0)
+			active ?
+			REG_MASKED_FIELD_ENABLE(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE) :
+			REG_MASKED_FIELD_DISABLE(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE),
 		},
 	};
 
@@ -2827,8 +2826,8 @@ gen8_enable_metric_set(struct i915_perf_stream *stream,
 	 */
 	if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {
 		intel_uncore_write(uncore, GEN8_OA_DEBUG,
-				   _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
-						      GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
+				   REG_MASKED_FIELD_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
+							   GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
 	}
 
 	/*
@@ -2847,9 +2846,10 @@ gen8_enable_metric_set(struct i915_perf_stream *stream,
 
 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
 {
-	return _MASKED_FIELD(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
-			     (stream->sample_flags & SAMPLE_OA_REPORT) ?
-			     0 : GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
+	if (stream->sample_flags & SAMPLE_OA_REPORT)
+		return REG_MASKED_FIELD_DISABLE(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
+	else
+		return REG_MASKED_FIELD_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
 }
 
 static int
@@ -2870,15 +2870,15 @@ gen12_enable_metric_set(struct i915_perf_stream *stream,
 	 */
 	if (IS_DG2(i915)) {
 		intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN,
-					     _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
+					     REG_MASKED_FIELD_ENABLE(STALL_DOP_GATING_DISABLE));
 		intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
-				   _MASKED_BIT_ENABLE(GEN12_DISABLE_DOP_GATING));
+				   REG_MASKED_FIELD_ENABLE(GEN12_DISABLE_DOP_GATING));
 	}
 
 	intel_uncore_write(uncore, __oa_regs(stream)->oa_debug,
 			   /* Disable clk ratio reports, like previous Gens. */
-			   _MASKED_BIT_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
-					      GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
+			   REG_MASKED_FIELD_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
+						   GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
 			   /*
 			    * If the user didn't require OA reports, instruct
 			    * the hardware not to emit ctx switch reports.
@@ -2949,9 +2949,9 @@ static void gen12_disable_metric_set(struct i915_perf_stream *stream)
 	 */
 	if (IS_DG2(i915)) {
 		intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN,
-					     _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
+					     REG_MASKED_FIELD_DISABLE(STALL_DOP_GATING_DISABLE));
 		intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
-				   _MASKED_BIT_DISABLE(GEN12_DISABLE_DOP_GATING));
+				   REG_MASKED_FIELD_DISABLE(GEN12_DISABLE_DOP_GATING));
 	}
 
 	/* disable the context save/restore or OAR counters */
@@ -4475,7 +4475,7 @@ static u32 mask_reg_value(u32 reg, u32 val)
 	 * programmed by userspace doesn't change this.
 	 */
 	if (REG_EQUAL(reg, HALF_SLICE_CHICKEN2))
-		val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
+		val = val & ~REG_MASKED_FIELD_ENABLE(GEN8_ST_PO_DISABLE);
 
 	/*
 	 * WAIT_FOR_RC6_EXIT has only one bit fulfilling the function
@@ -4483,7 +4483,7 @@ static u32 mask_reg_value(u32 reg, u32 val)
 	 * configs.
 	 */
 	if (REG_EQUAL(reg, WAIT_FOR_RC6_EXIT))
-		val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
+		val = val & ~REG_MASKED_FIELD_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
 
 	return val;
 }
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 5bf3b4ab2baa2..5d99b99b0c573 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -116,9 +116,6 @@
  *  #define GEN8_BAR                    _MMIO(0xb888)
  */
 
-#define GU_CNTL_PROTECTED		_MMIO(0x10100C)
-#define   DEPRESENT			REG_BIT(9)
-
 #define GU_CNTL				_MMIO(0x101010)
 #define   LMEM_INIT			REG_BIT(7)
 #define   DRIVERFLR			REG_BIT(31)
@@ -328,29 +325,6 @@
 #define GEN7_MEDIA_MAX_REQ_COUNT	_MMIO(0x4070)
 #define GEN7_GFX_MAX_REQ_COUNT		_MMIO(0x4074)
 
-#define GEN7_ERR_INT	_MMIO(0x44040)
-#define   ERR_INT_POISON		(1 << 31)
-#define   ERR_INT_INVALID_GTT_PTE	(1 << 29)
-#define   ERR_INT_INVALID_PTE_DATA	(1 << 28)
-#define   ERR_INT_SPRITE_C_FAULT	(1 << 23)
-#define   ERR_INT_PRIMARY_C_FAULT	(1 << 22)
-#define   ERR_INT_CURSOR_C_FAULT	(1 << 21)
-#define   ERR_INT_SPRITE_B_FAULT	(1 << 20)
-#define   ERR_INT_PRIMARY_B_FAULT	(1 << 19)
-#define   ERR_INT_CURSOR_B_FAULT	(1 << 18)
-#define   ERR_INT_SPRITE_A_FAULT	(1 << 17)
-#define   ERR_INT_PRIMARY_A_FAULT	(1 << 16)
-#define   ERR_INT_CURSOR_A_FAULT	(1 << 15)
-#define   ERR_INT_MMIO_UNCLAIMED	(1 << 13)
-#define   ERR_INT_PIPE_CRC_DONE_C	(1 << 8)
-#define   ERR_INT_FIFO_UNDERRUN_C	(1 << 6)
-#define   ERR_INT_PIPE_CRC_DONE_B	(1 << 5)
-#define   ERR_INT_FIFO_UNDERRUN_B	(1 << 3)
-#define   ERR_INT_PIPE_CRC_DONE_A	(1 << 2)
-#define   ERR_INT_PIPE_CRC_DONE(pipe)	(1 << (2 + (pipe) * 3))
-#define   ERR_INT_FIFO_UNDERRUN_A	(1 << 0)
-#define   ERR_INT_FIFO_UNDERRUN(pipe)	(1 << ((pipe) * 3))
-
 #define FPGA_DBG		_MMIO(0x42300)
 #define   FPGA_DBG_RM_NOCLAIM	REG_BIT(31)
 
@@ -361,9 +335,6 @@
 
 #define VLV_GU_CTL0	_MMIO(VLV_DISPLAY_BASE + 0x2030)
 #define VLV_GU_CTL1	_MMIO(VLV_DISPLAY_BASE + 0x2034)
-#define SCPD0		_MMIO(0x209c) /* 915+ only */
-#define  SCPD_FBC_IGNORE_3D			(1 << 6)
-#define  CSTATE_RENDER_CLOCK_GATE_DISABLE	(1 << 5)
 #define GEN2_IER	_MMIO(0x20a0)
 #define GEN2_IIR	_MMIO(0x20a4)
 #define GEN2_IMR	_MMIO(0x20a8)
@@ -377,13 +348,6 @@
 #define   GINT_DIS		(1 << 22)
 #define   GCFG_DIS		(1 << 8)
 #define VLV_GUNIT_CLOCK_GATE2	_MMIO(VLV_DISPLAY_BASE + 0x2064)
-#define VLV_IIR_RW	_MMIO(VLV_DISPLAY_BASE + 0x2084)
-#define VLV_IER		_MMIO(VLV_DISPLAY_BASE + 0x20a0)
-#define VLV_IIR		_MMIO(VLV_DISPLAY_BASE + 0x20a4)
-#define VLV_IMR		_MMIO(VLV_DISPLAY_BASE + 0x20a8)
-#define VLV_ISR		_MMIO(VLV_DISPLAY_BASE + 0x20ac)
-#define VLV_PCBR	_MMIO(VLV_DISPLAY_BASE + 0x2120)
-#define VLV_PCBR_ADDR_SHIFT	12
 
 #define EIR		_MMIO(0x20b0)
 #define EMR		_MMIO(0x20b4)
@@ -397,24 +361,10 @@
 
 #define GEN2_ERROR_REGS		I915_ERROR_REGS(EMR, EIR)
 
-#define INSTPM	        _MMIO(0x20c0)
-#define   INSTPM_SELF_EN (1 << 12) /* 915GM only */
-#define   INSTPM_AGPBUSY_INT_EN (1 << 11) /* gen3: when disabled, pending interrupts
-					will not assert AGPBUSY# and will only
-					be delivered when out of C3. */
-#define   INSTPM_FORCE_ORDERING				(1 << 7) /* GEN6+ */
-#define   INSTPM_TLB_INVALIDATE	(1 << 9)
-#define   INSTPM_SYNC_FLUSH	(1 << 5)
 #define MEM_MODE	_MMIO(0x20cc)
 #define   MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1 << 3) /* 830 only */
 #define   MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1 << 2) /* 830/845 only */
 #define   MEM_DISPLAY_TRICKLE_FEED_DISABLE (1 << 2) /* 85x only */
-#define FW_BLC		_MMIO(0x20d8)
-#define FW_BLC2		_MMIO(0x20dc)
-#define FW_BLC_SELF	_MMIO(0x20e0) /* 915+ only */
-#define   FW_BLC_SELF_EN_MASK      REG_BIT(31)
-#define   FW_BLC_SELF_FIFO_MASK    REG_BIT(16) /* 945 only */
-#define   FW_BLC_SELF_EN           REG_BIT(15) /* 945 only */
 #define MM_BURST_LENGTH     0x00700000
 #define MM_FIFO_WATERMARK   0x0001F000
 #define LM_BURST_LENGTH     0x00000700
@@ -524,42 +474,6 @@
 /* These are all the "old" interrupts */
 #define ILK_BSD_USER_INTERRUPT				(1 << 5)
 
-#define I915_PM_INTERRUPT				(1 << 31)
-#define I915_ISP_INTERRUPT				(1 << 22)
-#define I915_LPE_PIPE_B_INTERRUPT			(1 << 21)
-#define I915_LPE_PIPE_A_INTERRUPT			(1 << 20)
-#define I915_MIPIC_INTERRUPT				(1 << 19)
-#define I915_MIPIA_INTERRUPT				(1 << 18)
-#define I915_PIPE_CONTROL_NOTIFY_INTERRUPT		(1 << 18)
-#define I915_DISPLAY_PORT_INTERRUPT			(1 << 17)
-#define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT		(1 << 16)
-#define I915_MASTER_ERROR_INTERRUPT			(1 << 15)
-#define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT		(1 << 14)
-#define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT	(1 << 14) /* p-state */
-#define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT		(1 << 13)
-#define I915_HWB_OOM_INTERRUPT				(1 << 13)
-#define I915_LPE_PIPE_C_INTERRUPT			(1 << 12)
-#define I915_SYNC_STATUS_INTERRUPT			(1 << 12)
-#define I915_MISC_INTERRUPT				(1 << 11)
-#define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT	(1 << 11)
-#define I915_DISPLAY_PIPE_C_VBLANK_INTERRUPT		(1 << 10)
-#define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT	(1 << 10)
-#define I915_DISPLAY_PIPE_C_EVENT_INTERRUPT		(1 << 9)
-#define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT	(1 << 9)
-#define I915_DISPLAY_PIPE_C_DPBM_INTERRUPT		(1 << 8)
-#define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT	(1 << 8)
-#define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT		(1 << 7)
-#define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT		(1 << 6)
-#define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT		(1 << 5)
-#define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT		(1 << 4)
-#define I915_DISPLAY_PIPE_A_DPBM_INTERRUPT		(1 << 3)
-#define I915_DISPLAY_PIPE_B_DPBM_INTERRUPT		(1 << 2)
-#define I915_DEBUG_INTERRUPT				(1 << 2)
-#define I915_WINVALID_INTERRUPT				(1 << 1)
-#define I915_USER_INTERRUPT				(1 << 1)
-#define I915_ASLE_INTERRUPT				(1 << 0)
-#define I915_BSD_USER_INTERRUPT				(1 << 25)
-
 #define GEN6_BSD_RNCID			_MMIO(0x12198)
 
 #define GEN7_FF_THREAD_MODE		_MMIO(0x20a0)
@@ -613,47 +527,6 @@
 #define  DSTATE_GFX_CLOCK_GATING		(1 << 1)
 #define  DSTATE_DOT_CLOCK_GATING		(1 << 0)
 
-#define DSPCLK_GATE_D			_MMIO(0x6200)
-#define VLV_DSPCLK_GATE_D		_MMIO(VLV_DISPLAY_BASE + 0x6200)
-# define DPUNIT_B_CLOCK_GATE_DISABLE		(1 << 30) /* 965 */
-# define VSUNIT_CLOCK_GATE_DISABLE		(1 << 29) /* 965 */
-# define VRHUNIT_CLOCK_GATE_DISABLE		(1 << 28) /* 965 */
-# define VRDUNIT_CLOCK_GATE_DISABLE		(1 << 27) /* 965 */
-# define AUDUNIT_CLOCK_GATE_DISABLE		(1 << 26) /* 965 */
-# define DPUNIT_A_CLOCK_GATE_DISABLE		(1 << 25) /* 965 */
-# define DPCUNIT_CLOCK_GATE_DISABLE		(1 << 24) /* 965 */
-# define PNV_GMBUSUNIT_CLOCK_GATE_DISABLE	(1 << 24) /* pnv */
-# define TVRUNIT_CLOCK_GATE_DISABLE		(1 << 23) /* 915-945 */
-# define TVCUNIT_CLOCK_GATE_DISABLE		(1 << 22) /* 915-945 */
-# define TVFUNIT_CLOCK_GATE_DISABLE		(1 << 21) /* 915-945 */
-# define TVEUNIT_CLOCK_GATE_DISABLE		(1 << 20) /* 915-945 */
-# define DVSUNIT_CLOCK_GATE_DISABLE		(1 << 19) /* 915-945 */
-# define DSSUNIT_CLOCK_GATE_DISABLE		(1 << 18) /* 915-945 */
-# define DDBUNIT_CLOCK_GATE_DISABLE		(1 << 17) /* 915-945 */
-# define DPRUNIT_CLOCK_GATE_DISABLE		(1 << 16) /* 915-945 */
-# define DPFUNIT_CLOCK_GATE_DISABLE		(1 << 15) /* 915-945 */
-# define DPBMUNIT_CLOCK_GATE_DISABLE		(1 << 14) /* 915-945 */
-# define DPLSUNIT_CLOCK_GATE_DISABLE		(1 << 13) /* 915-945 */
-# define DPLUNIT_CLOCK_GATE_DISABLE		(1 << 12) /* 915-945 */
-# define DPOUNIT_CLOCK_GATE_DISABLE		(1 << 11)
-# define DPBUNIT_CLOCK_GATE_DISABLE		(1 << 10)
-# define DCUNIT_CLOCK_GATE_DISABLE		(1 << 9)
-# define DPUNIT_CLOCK_GATE_DISABLE		(1 << 8)
-# define VRUNIT_CLOCK_GATE_DISABLE		(1 << 7) /* 915+: reserved */
-# define OVHUNIT_CLOCK_GATE_DISABLE		(1 << 6) /* 830-865 */
-# define DPIOUNIT_CLOCK_GATE_DISABLE		(1 << 6) /* 915-945 */
-# define OVFUNIT_CLOCK_GATE_DISABLE		(1 << 5)
-# define OVBUNIT_CLOCK_GATE_DISABLE		(1 << 4)
-/*
- * This bit must be set on the 830 to prevent hangs when turning off the
- * overlay scaler.
- */
-# define OVRUNIT_CLOCK_GATE_DISABLE		(1 << 3)
-# define OVCUNIT_CLOCK_GATE_DISABLE		(1 << 2)
-# define OVUUNIT_CLOCK_GATE_DISABLE		(1 << 1)
-# define ZVUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 830 */
-# define OVLUNIT_CLOCK_GATE_DISABLE		(1 << 0) /* 845,865 */
-
 #define RENCLK_GATE_D1		_MMIO(0x6204)
 # define BLITTER_CLOCK_GATE_DISABLE		(1 << 13) /* 945GM only */
 # define MPEG_CLOCK_GATE_DISABLE		(1 << 12) /* 945GM only */
@@ -758,19 +631,6 @@
 #define VLV_CLK_CTL2			_MMIO(0x101104)
 #define   CLK_CTL2_CZCOUNT_30NS_SHIFT	28
 
-/*
- * GEN9 clock gating regs
- */
-#define GEN9_CLKGATE_DIS_0		_MMIO(0x46530)
-#define   DARBF_GATING_DIS		REG_BIT(27)
-#define   MTL_PIPEDMC_GATING_DIS(pipe)	REG_BIT(15 - (pipe))
-#define   PWM2_GATING_DIS		REG_BIT(14)
-#define   PWM1_GATING_DIS		REG_BIT(13)
-
-#define GEN9_CLKGATE_DIS_3		_MMIO(0x46538)
-#define   TGL_VRH_GATING_DIS		REG_BIT(31)
-#define   DPT_GATING_DIS		REG_BIT(22)
-
 #define VLV_DPFLIPSTAT				_MMIO(VLV_DISPLAY_BASE + 0x70028)
 #define   PIPEB_LINE_COMPARE_INT_EN			REG_BIT(29)
 #define   PIPEB_HLINE_INT_EN			REG_BIT(28)
@@ -799,56 +659,9 @@
 #define PCH_3DCGDIS1		_MMIO(0x46024)
 # define VFMUNIT_CLOCK_GATE_DISABLE		(1 << 11)
 
-/* Display Internal Timeout Register */
-#define RM_TIMEOUT		_MMIO(0x42060)
-#define RM_TIMEOUT_REG_CAPTURE	_MMIO(0x420E0)
-#define  MMIO_TIMEOUT_US(us)	((us) << 0)
-
-/* interrupts */
-#define DE_MASTER_IRQ_CONTROL   (1 << 31)
-#define DE_SPRITEB_FLIP_DONE    (1 << 29)
-#define DE_SPRITEA_FLIP_DONE    (1 << 28)
-#define DE_PLANEB_FLIP_DONE     (1 << 27)
-#define DE_PLANEA_FLIP_DONE     (1 << 26)
-#define DE_PLANE_FLIP_DONE(plane) (1 << (26 + (plane)))
-#define DE_PCU_EVENT            (1 << 25)
-#define DE_GTT_FAULT            (1 << 24)
-#define DE_POISON               (1 << 23)
-#define DE_PERFORM_COUNTER      (1 << 22)
-#define DE_PCH_EVENT            (1 << 21)
-#define DE_AUX_CHANNEL_A        (1 << 20)
-#define DE_DP_A_HOTPLUG         (1 << 19)
-#define DE_GSE                  (1 << 18)
-#define DE_PIPEB_VBLANK         (1 << 15)
-#define DE_PIPEB_EVEN_FIELD     (1 << 14)
-#define DE_PIPEB_ODD_FIELD      (1 << 13)
-#define DE_PIPEB_LINE_COMPARE   (1 << 12)
-#define DE_PIPEB_VSYNC          (1 << 11)
-#define DE_PIPEB_CRC_DONE	(1 << 10)
-#define DE_PIPEB_FIFO_UNDERRUN  (1 << 8)
-#define DE_PIPEA_VBLANK         (1 << 7)
-#define DE_PIPE_VBLANK(pipe)    (1 << (7 + 8 * (pipe)))
-#define DE_PIPEA_EVEN_FIELD     (1 << 6)
-#define DE_PIPEA_ODD_FIELD      (1 << 5)
-#define DE_PIPEA_LINE_COMPARE   (1 << 4)
-#define DE_PIPEA_VSYNC          (1 << 3)
-#define DE_PIPEA_CRC_DONE	(1 << 2)
-#define DE_PIPE_CRC_DONE(pipe)	(1 << (2 + 8 * (pipe)))
-#define DE_PIPEA_FIFO_UNDERRUN  (1 << 0)
-#define DE_PIPE_FIFO_UNDERRUN(pipe)  (1 << (8 * (pipe)))
-
 #define VLV_MASTER_IER			_MMIO(0x4400c) /* Gunit master IER */
 #define   MASTER_INTERRUPT_ENABLE	(1 << 31)
 
-#define DEISR   _MMIO(0x44000)
-#define DEIMR   _MMIO(0x44004)
-#define DEIIR   _MMIO(0x44008)
-#define DEIER   _MMIO(0x4400c)
-
-#define DE_IRQ_REGS		I915_IRQ_REGS(DEIMR, \
-					      DEIER, \
-					      DEIIR)
-
 #define GTISR   _MMIO(0x44010)
 #define GTIMR   _MMIO(0x44014)
 #define GTIIR   _MMIO(0x44018)
@@ -858,24 +671,6 @@
 					      GTIER, \
 					      GTIIR)
 
-#define GEN8_MASTER_IRQ			_MMIO(0x44200)
-#define  GEN8_MASTER_IRQ_CONTROL	(1 << 31)
-#define  GEN8_PCU_IRQ			(1 << 30)
-#define  GEN8_DE_PCH_IRQ		(1 << 23)
-#define  GEN8_DE_MISC_IRQ		(1 << 22)
-#define  GEN8_DE_PORT_IRQ		(1 << 20)
-#define  GEN8_DE_PIPE_C_IRQ		(1 << 18)
-#define  GEN8_DE_PIPE_B_IRQ		(1 << 17)
-#define  GEN8_DE_PIPE_A_IRQ		(1 << 16)
-#define  GEN8_DE_PIPE_IRQ(pipe)		(1 << (16 + (pipe)))
-#define  GEN8_GT_VECS_IRQ		(1 << 6)
-#define  GEN8_GT_GUC_IRQ		(1 << 5)
-#define  GEN8_GT_PM_IRQ			(1 << 4)
-#define  GEN8_GT_VCS1_IRQ		(1 << 3) /* NB: VCS2 in bspec! */
-#define  GEN8_GT_VCS0_IRQ		(1 << 2) /* NB: VCS1 in bpsec! */
-#define  GEN8_GT_BCS_IRQ		(1 << 1)
-#define  GEN8_GT_RCS_IRQ		(1 << 0)
-
 #define GEN8_GT_ISR(which) _MMIO(0x44300 + (0x10 * (which)))
 #define GEN8_GT_IMR(which) _MMIO(0x44304 + (0x10 * (which)))
 #define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which)))
@@ -901,25 +696,6 @@
 						      GEN8_PCU_IER, \
 						      GEN8_PCU_IIR)
 
-#define GEN11_GU_MISC_ISR	_MMIO(0x444f0)
-#define GEN11_GU_MISC_IMR	_MMIO(0x444f4)
-#define GEN11_GU_MISC_IIR	_MMIO(0x444f8)
-#define GEN11_GU_MISC_IER	_MMIO(0x444fc)
-#define  GEN11_GU_MISC_GSE	(1 << 27)
-
-#define GEN11_GU_MISC_IRQ_REGS		I915_IRQ_REGS(GEN11_GU_MISC_IMR, \
-						      GEN11_GU_MISC_IER, \
-						      GEN11_GU_MISC_IIR)
-
-#define GEN11_GFX_MSTR_IRQ		_MMIO(0x190010)
-#define  GEN11_MASTER_IRQ		(1 << 31)
-#define  GEN11_PCU_IRQ			(1 << 30)
-#define  GEN11_GU_MISC_IRQ		(1 << 29)
-#define  GEN11_DISPLAY_IRQ		(1 << 16)
-#define  GEN11_GT_DW_IRQ(x)		(1 << (x))
-#define  GEN11_GT_DW1_IRQ		(1 << 1)
-#define  GEN11_GT_DW0_IRQ		(1 << 0)
-
 #define DG1_MSTR_TILE_INTR		_MMIO(0x190008)
 #define   DG1_MSTR_IRQ			REG_BIT(31)
 #define   DG1_MSTR_TILE(t)		REG_BIT(t)
@@ -941,133 +717,9 @@
 #define   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE	REG_BIT(5)
 #define   CHICKEN3_DGMG_DONE_FIX_DISABLE	REG_BIT(2)
 
-#define CHICKEN_PAR1_1		_MMIO(0x42080)
-#define   IGNORE_KVMR_PIPE_A		REG_BIT(23)
-#define   KBL_ARB_FILL_SPARE_22		REG_BIT(22)
-#define   DIS_RAM_BYPASS_PSR2_MAN_TRACK	REG_BIT(16)
-#define   SKL_DE_COMPRESSED_HASH_MODE	REG_BIT(15)
-#define   HSW_MASK_VBL_TO_PIPE_IN_SRD	REG_BIT(15) /* hsw/bdw */
-#define   FORCE_ARB_IDLE_PLANES		REG_BIT(14)
-#define   SKL_EDP_PSR_FIX_RDWRAP	REG_BIT(3)
-#define   IGNORE_PSR2_HW_TRACKING	REG_BIT(1)
-
 #define CHICKEN_PAR2_1		_MMIO(0x42090)
 #define   KVM_CONFIG_CHANGE_NOTIFICATION_SELECT	REG_BIT(14)
 
-#define _CHICKEN_PIPESL_1_A	0x420b0
-#define _CHICKEN_PIPESL_1_B	0x420b4
-#define CHICKEN_PIPESL_1(pipe)	_MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
-#define   HSW_PRI_STRETCH_MAX_MASK	REG_GENMASK(28, 27)
-#define   HSW_PRI_STRETCH_MAX_X8	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 0)
-#define   HSW_PRI_STRETCH_MAX_X4	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 1)
-#define   HSW_PRI_STRETCH_MAX_X2	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 2)
-#define   HSW_PRI_STRETCH_MAX_X1	REG_FIELD_PREP(HSW_PRI_STRETCH_MAX_MASK, 3)
-#define   HSW_SPR_STRETCH_MAX_MASK	REG_GENMASK(26, 25)
-#define   HSW_SPR_STRETCH_MAX_X8	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 0)
-#define   HSW_SPR_STRETCH_MAX_X4	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 1)
-#define   HSW_SPR_STRETCH_MAX_X2	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 2)
-#define   HSW_SPR_STRETCH_MAX_X1	REG_FIELD_PREP(HSW_SPR_STRETCH_MAX_MASK, 3)
-#define   HSW_FBCQ_DIS			REG_BIT(22)
-#define   HSW_UNMASK_VBL_TO_REGS_IN_SRD REG_BIT(15) /* hsw */
-#define   SKL_PSR_MASK_PLANE_FLIP	REG_BIT(11) /* skl+ */
-#define   SKL_PLANE1_STRETCH_MAX_MASK	REG_GENMASK(1, 0)
-#define   SKL_PLANE1_STRETCH_MAX_X8	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 0)
-#define   SKL_PLANE1_STRETCH_MAX_X4	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 1)
-#define   SKL_PLANE1_STRETCH_MAX_X2	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 2)
-#define   SKL_PLANE1_STRETCH_MAX_X1	REG_FIELD_PREP(SKL_PLANE1_STRETCH_MAX_MASK, 3)
-#define   BDW_UNMASK_VBL_TO_REGS_IN_SRD	REG_BIT(0) /* bdw */
-
-#define DISP_ARB_CTL	_MMIO(0x45000)
-#define   DISP_FBC_MEMORY_WAKE		REG_BIT(31)
-#define   DISP_TILE_SURFACE_SWIZZLING	REG_BIT(13)
-#define   DISP_FBC_WM_DIS		REG_BIT(15)
-
-#define GEN8_CHICKEN_DCPR_1			_MMIO(0x46430)
-#define   _LATENCY_REPORTING_REMOVED_PIPE_D	REG_BIT(31)
-#define   SKL_SELECT_ALTERNATE_DC_EXIT		REG_BIT(30)
-#define   _LATENCY_REPORTING_REMOVED_PIPE_C	REG_BIT(25)
-#define   _LATENCY_REPORTING_REMOVED_PIPE_B	REG_BIT(24)
-#define   _LATENCY_REPORTING_REMOVED_PIPE_A	REG_BIT(23)
-#define   LATENCY_REPORTING_REMOVED(pipe)	_PICK((pipe), \
-						      _LATENCY_REPORTING_REMOVED_PIPE_A, \
-						      _LATENCY_REPORTING_REMOVED_PIPE_B, \
-						      _LATENCY_REPORTING_REMOVED_PIPE_C, \
-						      _LATENCY_REPORTING_REMOVED_PIPE_D)
-#define   ICL_DELAY_PMRSP			REG_BIT(22)
-#define   DISABLE_FLR_SRC			REG_BIT(15)
-#define   MASK_WAKEMEM				REG_BIT(13)
-#define   DDI_CLOCK_REG_ACCESS			REG_BIT(7)
-
-#define GMD_ID_DISPLAY				_MMIO(0x510a0)
-#define   GMD_ID_ARCH_MASK			REG_GENMASK(31, 22)
-#define   GMD_ID_RELEASE_MASK			REG_GENMASK(21, 14)
-#define   GMD_ID_STEP				REG_GENMASK(5, 0)
-
-/* PCH */
-
-#define SDEISR  _MMIO(0xc4000)
-#define SDEIMR  _MMIO(0xc4004)
-#define SDEIIR  _MMIO(0xc4008)
-#define SDEIER  _MMIO(0xc400c)
-
-/* Icelake PPS_DATA and _ECC DIP Registers.
- * These are available for transcoders B,C and eDP.
- * Adding the _A so as to reuse the _MMIO_TRANS2
- * definition, with which it offsets to the right location.
- */
-
-#define _TRANSA_CHICKEN1	 0xf0060
-#define _TRANSB_CHICKEN1	 0xf1060
-#define TRANS_CHICKEN1(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1)
-#define   TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE	REG_BIT(10)
-#define   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE	REG_BIT(4)
-
-#define _TRANSA_CHICKEN2	 0xf0064
-#define _TRANSB_CHICKEN2	 0xf1064
-#define TRANS_CHICKEN2(pipe)	_MMIO_PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2)
-#define   TRANS_CHICKEN2_TIMING_OVERRIDE		REG_BIT(31)
-#define   TRANS_CHICKEN2_FDI_POLARITY_REVERSED		REG_BIT(29)
-#define   TRANS_CHICKEN2_FRAME_START_DELAY_MASK		REG_GENMASK(28, 27)
-#define   TRANS_CHICKEN2_FRAME_START_DELAY(x)		REG_FIELD_PREP(TRANS_CHICKEN2_FRAME_START_DELAY_MASK, (x)) /* 0-3 */
-#define   TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER	REG_BIT(26)
-#define   TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH	REG_BIT(25)
-
-#define SOUTH_CHICKEN1		_MMIO(0xc2000)
-#define  FDIA_PHASE_SYNC_SHIFT_OVR	19
-#define  FDIA_PHASE_SYNC_SHIFT_EN	18
-#define  INVERT_DDIE_HPD			REG_BIT(28)
-#define  INVERT_DDID_HPD_MTP			REG_BIT(27)
-#define  INVERT_TC4_HPD				REG_BIT(26)
-#define  INVERT_TC3_HPD				REG_BIT(25)
-#define  INVERT_TC2_HPD				REG_BIT(24)
-#define  INVERT_TC1_HPD				REG_BIT(23)
-#define  INVERT_DDID_HPD			(1 << 18)
-#define  INVERT_DDIC_HPD			(1 << 17)
-#define  INVERT_DDIB_HPD			(1 << 16)
-#define  INVERT_DDIA_HPD			(1 << 15)
-#define  FDI_PHASE_SYNC_OVR(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
-#define  FDI_PHASE_SYNC_EN(pipe) (1 << (FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
-#define  FDI_BC_BIFURCATION_SELECT	(1 << 12)
-#define  CHASSIS_CLK_REQ_DURATION_MASK	(0xf << 8)
-#define  CHASSIS_CLK_REQ_DURATION(x)	((x) << 8)
-#define  SBCLK_RUN_REFCLK_DIS		(1 << 7)
-#define  ICP_SECOND_PPS_IO_SELECT	REG_BIT(2)
-#define  SPT_PWM_GRANULARITY		(1 << 0)
-#define SOUTH_CHICKEN2		_MMIO(0xc2004)
-#define  FDI_MPHY_IOSFSB_RESET_STATUS	(1 << 13)
-#define  FDI_MPHY_IOSFSB_RESET_CTL	(1 << 12)
-#define  LPT_PWM_GRANULARITY		(1 << 5)
-#define  DPLS_EDP_PPS_FIX_DIS		(1 << 0)
-
-#define SOUTH_DSPCLK_GATE_D	_MMIO(0xc2020)
-#define  PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1 << 31)
-#define  PCH_DPLUNIT_CLOCK_GATE_DISABLE (1 << 30)
-#define  PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1 << 29)
-#define  PCH_DPMGUNIT_CLOCK_GATE_DISABLE (1 << 15)
-#define  PCH_CPUNIT_CLOCK_GATE_DISABLE (1 << 14)
-#define  CNP_PWM_CGE_GATING_DISABLE (1 << 13)
-#define  PCH_LP_PARTITION_LEVEL_DISABLE  (1 << 12)
-
 #define  VLV_PMWGICZ				_MMIO(0x1300a4)
 
 #define  HSW_EDRAM_CAP				_MMIO(0x120010)
@@ -1076,106 +728,6 @@
 #define    EDRAM_WAYS_IDX(cap)			(((cap) >> 5) & 0x7)
 #define    EDRAM_SETS_IDX(cap)			(((cap) >> 8) & 0x3)
 
-#define GEN6_PCODE_MAILBOX			_MMIO(0x138124)
-#define   GEN6_PCODE_READY			(1 << 31)
-#define   GEN6_PCODE_MB_PARAM2			REG_GENMASK(23, 16)
-#define   GEN6_PCODE_MB_PARAM1			REG_GENMASK(15, 8)
-#define   GEN6_PCODE_MB_COMMAND			REG_GENMASK(7, 0)
-#define   GEN6_PCODE_ERROR_MASK			0xFF
-#define     GEN6_PCODE_SUCCESS			0x0
-#define     GEN6_PCODE_ILLEGAL_CMD		0x1
-#define     GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x2
-#define     GEN6_PCODE_TIMEOUT			0x3
-#define     GEN6_PCODE_UNIMPLEMENTED_CMD	0xFF
-#define     GEN7_PCODE_TIMEOUT			0x2
-#define     GEN7_PCODE_ILLEGAL_DATA		0x3
-#define     GEN11_PCODE_ILLEGAL_SUBCOMMAND	0x4
-#define     GEN11_PCODE_LOCKED			0x6
-#define     GEN11_PCODE_REJECTED		0x11
-#define     GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10
-#define   GEN6_PCODE_WRITE_RC6VIDS		0x4
-#define   GEN6_PCODE_READ_RC6VIDS		0x5
-#define     GEN6_ENCODE_RC6_VID(mv)		(((mv) - 245) / 5)
-#define     GEN6_DECODE_RC6_VID(vids)		(((vids) * 5) + 245)
-#define   BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ	0x18
-#define   GEN9_PCODE_READ_MEM_LATENCY		0x6
-#define     GEN9_MEM_LATENCY_LEVEL_3_7_MASK	REG_GENMASK(31, 24)
-#define     GEN9_MEM_LATENCY_LEVEL_2_6_MASK	REG_GENMASK(23, 16)
-#define     GEN9_MEM_LATENCY_LEVEL_1_5_MASK	REG_GENMASK(15, 8)
-#define     GEN9_MEM_LATENCY_LEVEL_0_4_MASK	REG_GENMASK(7, 0)
-#define   SKL_PCODE_LOAD_HDCP_KEYS		0x5
-#define   SKL_PCODE_CDCLK_CONTROL		0x7
-#define     SKL_CDCLK_PREPARE_FOR_CHANGE	0x3
-#define     SKL_CDCLK_READY_FOR_CHANGE		0x1
-#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE	0x8
-#define   GEN6_PCODE_READ_MIN_FREQ_TABLE	0x9
-#define   GEN6_READ_OC_PARAMS			0xc
-#define   ICL_PCODE_MEM_SUBSYSYSTEM_INFO	0xd
-#define     ICL_PCODE_MEM_SS_READ_GLOBAL_INFO	(0x0 << 8)
-#define     ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point)	(((point) << 16) | (0x1 << 8))
-#define     ADL_PCODE_MEM_SS_READ_PSF_GV_INFO	((0) | (0x2 << 8))
-#define   DISPLAY_TO_PCODE_CDCLK_MAX		0x28D
-#define   DISPLAY_TO_PCODE_VOLTAGE_MASK		REG_GENMASK(1, 0)
-#define	  DISPLAY_TO_PCODE_VOLTAGE_MAX		DISPLAY_TO_PCODE_VOLTAGE_MASK
-#define   DISPLAY_TO_PCODE_CDCLK_VALID		REG_BIT(27)
-#define   DISPLAY_TO_PCODE_PIPE_COUNT_VALID	REG_BIT(31)
-#define   DISPLAY_TO_PCODE_CDCLK_MASK		REG_GENMASK(25, 16)
-#define   DISPLAY_TO_PCODE_PIPE_COUNT_MASK	REG_GENMASK(30, 28)
-#define   DISPLAY_TO_PCODE_CDCLK(x)		REG_FIELD_PREP(DISPLAY_TO_PCODE_CDCLK_MASK, (x))
-#define   DISPLAY_TO_PCODE_PIPE_COUNT(x)	REG_FIELD_PREP(DISPLAY_TO_PCODE_PIPE_COUNT_MASK, (x))
-#define   DISPLAY_TO_PCODE_VOLTAGE(x)		REG_FIELD_PREP(DISPLAY_TO_PCODE_VOLTAGE_MASK, (x))
-#define   DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, num_pipes, voltage_level) \
-		((DISPLAY_TO_PCODE_CDCLK(cdclk)) | \
-		(DISPLAY_TO_PCODE_PIPE_COUNT(num_pipes)) | \
-		(DISPLAY_TO_PCODE_VOLTAGE(voltage_level)))
-#define   ICL_PCODE_SAGV_DE_MEM_SS_CONFIG	0xe
-#define     ICL_PCODE_REP_QGV_MASK		REG_GENMASK(1, 0)
-#define     ICL_PCODE_REP_QGV_SAFE		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 0)
-#define     ICL_PCODE_REP_QGV_POLL		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 1)
-#define     ICL_PCODE_REP_QGV_REJECTED		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 2)
-#define     ADLS_PCODE_REP_PSF_MASK		REG_GENMASK(3, 2)
-#define     ADLS_PCODE_REP_PSF_SAFE		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 0)
-#define     ADLS_PCODE_REP_PSF_POLL		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 1)
-#define     ADLS_PCODE_REP_PSF_REJECTED		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 2)
-#define     ICL_PCODE_REQ_QGV_PT_MASK		REG_GENMASK(7, 0)
-#define     ICL_PCODE_REQ_QGV_PT(x)		REG_FIELD_PREP(ICL_PCODE_REQ_QGV_PT_MASK, (x))
-#define     ADLS_PCODE_REQ_PSF_PT_MASK		REG_GENMASK(10, 8)
-#define     ADLS_PCODE_REQ_PSF_PT(x)		REG_FIELD_PREP(ADLS_PCODE_REQ_PSF_PT_MASK, (x))
-#define   GEN6_PCODE_READ_D_COMP		0x10
-#define   GEN6_PCODE_WRITE_D_COMP		0x11
-#define   ICL_PCODE_EXIT_TCCOLD			0x12
-#define   HSW_PCODE_DE_WRITE_FREQ_REQ		0x17
-#define   DISPLAY_IPS_CONTROL			0x19
-#define   TGL_PCODE_TCCOLD			0x26
-#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED	REG_BIT(0)
-#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ	0
-#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ	REG_BIT(0)
-            /* See also IPS_CTL */
-#define     IPS_PCODE_CONTROL			(1 << 30)
-#define   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL	0x1A
-#define   GEN9_PCODE_SAGV_CONTROL		0x21
-#define     GEN9_SAGV_DISABLE			0x0
-#define     GEN9_SAGV_IS_DISABLED		0x1
-#define     GEN9_SAGV_ENABLE			0x3
-#define   DG1_PCODE_STATUS			0x7E
-#define     DG1_UNCORE_GET_INIT_STATUS		0x0
-#define     DG1_UNCORE_INIT_STATUS_COMPLETE	0x1
-#define   PCODE_POWER_SETUP			0x7C
-#define     POWER_SETUP_SUBCOMMAND_READ_I1	0x4
-#define     POWER_SETUP_SUBCOMMAND_WRITE_I1	0x5
-#define	    POWER_SETUP_I1_WATTS		REG_BIT(31)
-#define	    POWER_SETUP_I1_SHIFT		6	/* 10.6 fixed point format */
-#define	    POWER_SETUP_I1_DATA_MASK		REG_GENMASK(15, 0)
-#define     POWER_SETUP_SUBCOMMAND_G8_ENABLE	0x6
-#define GEN12_PCODE_READ_SAGV_BLOCK_TIME_US	0x23
-#define   XEHP_PCODE_FREQUENCY_CONFIG		0x6e	/* pvc */
-/* XEHP_PCODE_FREQUENCY_CONFIG sub-commands (param1) */
-#define     PCODE_MBOX_FC_SC_READ_FUSED_P0	0x0
-#define     PCODE_MBOX_FC_SC_READ_FUSED_PN	0x1
-/* PCODE_MBOX_DOMAIN_* - mailbox domain IDs */
-/*   XEHP_PCODE_FREQUENCY_CONFIG param2 */
-#define     PCODE_MBOX_DOMAIN_NONE		0x0
-#define     PCODE_MBOX_DOMAIN_MEDIAFF		0x3
 #define GEN6_PCODE_DATA				_MMIO(0x138128)
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT	8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT	16
@@ -1224,20 +776,9 @@
 #define   SGGI_DIS			REG_BIT(15)
 #define   SGR_DIS			REG_BIT(13)
 
-#define PRIMARY_SPI_TRIGGER			_MMIO(0x102040)
-#define PRIMARY_SPI_ADDRESS			_MMIO(0x102080)
-#define PRIMARY_SPI_REGIONID			_MMIO(0x102084)
-#define SPI_STATIC_REGIONS			_MMIO(0x102090)
-#define   OPTIONROM_SPI_REGIONID_MASK		REG_GENMASK(7, 0)
-#define OROM_OFFSET				_MMIO(0x1020c0)
-#define   OROM_OFFSET_MASK			REG_GENMASK(20, 16)
-
-#define MTL_MEM_SS_INFO_GLOBAL			_MMIO(0x45700)
-#define   XE3P_ECC_IMPACTING_DE			REG_BIT(12)
-#define   MTL_N_OF_ENABLED_QGV_POINTS_MASK	REG_GENMASK(11, 8)
-#define   MTL_N_OF_POPULATED_CH_MASK		REG_GENMASK(7, 4)
-#define   MTL_DDR_TYPE_MASK			REG_GENMASK(3, 0)
-
 #define MTL_MEDIA_GSI_BASE		0x380000
 
+#define DSPCLK_GATE_D			_MMIO(0x6200)
+# define OVRUNIT_CLOCK_GATE_DISABLE		(1 << 3)
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h
index e81fac8ab51b5..e897d3ccbf9e5 100644
--- a/drivers/gpu/drm/i915/i915_reg_defs.h
+++ b/drivers/gpu/drm/i915/i915_reg_defs.h
@@ -6,183 +6,8 @@
 #ifndef __I915_REG_DEFS__
 #define __I915_REG_DEFS__
 
-#include <linux/bitfield.h>
-#include <linux/bits.h>
-
-/*
- * Wrappers over the generic fixed width BIT_U*() and GENMASK_U*()
- * implementations, for compatibility reasons with previous implementation.
- */
-#define REG_GENMASK(high, low)		GENMASK_U32(high, low)
-#define REG_GENMASK64(high, low)	GENMASK_U64(high, low)
-#define REG_GENMASK16(high, low)	GENMASK_U16(high, low)
-#define REG_GENMASK8(high, low)		GENMASK_U8(high, low)
-
-#define REG_BIT(n)			BIT_U32(n)
-#define REG_BIT64(n)			BIT_U64(n)
-#define REG_BIT16(n)			BIT_U16(n)
-#define REG_BIT8(n)			BIT_U8(n)
-
-/*
- * Local integer constant expression version of is_power_of_2().
- */
-#define IS_POWER_OF_2(__x)		((__x) && (((__x) & ((__x) - 1)) == 0))
-
-/**
- * REG_FIELD_PREP() - Prepare a u32 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to put in the field
- *
- * Local copy of FIELD_PREP() to generate an integer constant expression, force
- * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK().
- *
- * @return: @__val masked and shifted into the field defined by @__mask.
- */
-#define REG_FIELD_PREP(__mask, __val)						\
-	((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +	\
-	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +		\
-	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) +		\
-	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
-	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
-
-/**
- * REG_FIELD_PREP8() - Prepare a u8 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to put in the field
- *
- * Local copy of FIELD_PREP() to generate an integer constant expression, force
- * u8 and for consistency with REG_FIELD_GET8(), REG_BIT8() and REG_GENMASK8().
- *
- * @return: @__val masked and shifted into the field defined by @__mask.
- */
-#define REG_FIELD_PREP8(__mask, __val)                                          \
-	((u8)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +      \
-	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +             \
-	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U8_MAX) +          \
-	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
-	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
-
-/**
- * REG_FIELD_GET() - Extract a u32 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to extract the bitfield value from
- *
- * Local wrapper for FIELD_GET() to force u32 and for consistency with
- * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
- *
- * @return: Masked and shifted value of the field defined by @__mask in @__val.
- */
-#define REG_FIELD_GET(__mask, __val)	((u32)FIELD_GET(__mask, __val))
-
-/**
- * REG_FIELD_GET64() - Extract a u64 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to extract the bitfield value from
- *
- * Local wrapper for FIELD_GET() to force u64 and for consistency with
- * REG_GENMASK64().
- *
- * @return: Masked and shifted value of the field defined by @__mask in @__val.
- */
-#define REG_FIELD_GET64(__mask, __val)	((u64)FIELD_GET(__mask, __val))
-
-
-/**
- * REG_FIELD_PREP16() - Prepare a u16 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to put in the field
- *
- * Local copy of FIELD_PREP16() to generate an integer constant
- * expression, force u8 and for consistency with
- * REG_FIELD_GET16(), REG_BIT16() and REG_GENMASK16().
- *
- * @return: @__val masked and shifted into the field defined by @__mask.
- */
-#define REG_FIELD_PREP16(__mask, __val)                                          \
-	((u16)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +      \
-	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +             \
-	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U16_MAX) +          \
-	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
-	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
-
-#define __MASKED_FIELD(mask, value) ((mask) << 16 | (value))
-#define _MASKED_FIELD(mask, value) ({					   \
-	if (__builtin_constant_p(mask))					   \
-		BUILD_BUG_ON_MSG(((mask) & 0xffff0000), "Incorrect mask"); \
-	if (__builtin_constant_p(value))				   \
-		BUILD_BUG_ON_MSG((value) & 0xffff0000, "Incorrect value"); \
-	if (__builtin_constant_p(mask) && __builtin_constant_p(value))	   \
-		BUILD_BUG_ON_MSG((value) & ~(mask),			   \
-				 "Incorrect value for mask");		   \
-	__MASKED_FIELD(mask, value); })
-#define _MASKED_BIT_ENABLE(a)	({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); })
-#define _MASKED_BIT_DISABLE(a)	(_MASKED_FIELD((a), 0))
-
-/*
- * Given the first two numbers __a and __b of arbitrarily many evenly spaced
- * numbers, pick the 0-based __index'th value.
- *
- * Always prefer this over _PICK() if the numbers are evenly spaced.
- */
-#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a)))
-
-/*
- * Like _PICK_EVEN(), but supports 2 ranges of evenly spaced address offsets.
- * @__c_index corresponds to the index in which the second range starts to be
- * used. Using math interval notation, the first range is used for indexes [ 0,
- * @__c_index), while the second range is used for [ @__c_index, ... ). Example:
- *
- * #define _FOO_A			0xf000
- * #define _FOO_B			0xf004
- * #define _FOO_C			0xf008
- * #define _SUPER_FOO_A			0xa000
- * #define _SUPER_FOO_B			0xa100
- * #define FOO(x)			_MMIO(_PICK_EVEN_2RANGES(x, 3,		\
- *					      _FOO_A, _FOO_B,			\
- *					      _SUPER_FOO_A, _SUPER_FOO_B))
- *
- * This expands to:
- *	0: 0xf000,
- *	1: 0xf004,
- *	2: 0xf008,
- *	3: 0xa000,
- *	4: 0xa100,
- *	5: 0xa200,
- *	...
- */
-#define _PICK_EVEN_2RANGES(__index, __c_index, __a, __b, __c, __d)		\
-	(BUILD_BUG_ON_ZERO(!__is_constexpr(__c_index)) +			\
-	 ((__index) < (__c_index) ? _PICK_EVEN(__index, __a, __b) :		\
-				   _PICK_EVEN((__index) - (__c_index), __c, __d)))
-
-/*
- * Given the arbitrary numbers in varargs, pick the 0-based __index'th number.
- *
- * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced.
- */
-#define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index])
-
-/**
- * REG_FIELD_GET8() - Extract a u8 bitfield value
- * @__mask: shifted mask defining the field's length and position
- * @__val: value to extract the bitfield value from
- *
- * Local wrapper for FIELD_GET() to force u8 and for consistency with
- * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
- *
- * @return: Masked and shifted value of the field defined by @__mask in @__val.
- */
-#define REG_FIELD_GET8(__mask, __val)   ((u8)FIELD_GET(__mask, __val))
-
-/**
- * REG_FIELD_MAX() - produce the maximum value representable by a field
- * @__mask: shifted mask defining the field's length and position
- *
- * Local wrapper for FIELD_MAX() to return the maximum bit value that can
- * be held in the field specified by @_mask, cast to u32 for consistency
- * with other macros.
- */
-#define REG_FIELD_MAX(__mask)	((u32)FIELD_MAX(__mask))
+#include <drm/intel/pick.h>
+#include <drm/intel/reg_bits.h>
 
 typedef struct {
 	u32 reg;
diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c
index ad868adb30dc0..9f81e206b6469 100644
--- a/drivers/gpu/drm/i915/i915_scatterlist.c
+++ b/drivers/gpu/drm/i915/i915_scatterlist.c
@@ -7,7 +7,7 @@
 #include "i915_scatterlist.h"
 #include "i915_ttm_buddy_manager.h"
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 #include <drm/drm_mm.h>
 
 #include <linux/slab.h>
@@ -167,9 +167,9 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
 	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 	const u64 size = res->size;
 	const u32 max_segment = round_down(UINT_MAX, page_alignment);
-	struct drm_buddy *mm = bman_res->mm;
+	struct gpu_buddy *mm = bman_res->mm;
 	struct list_head *blocks = &bman_res->blocks;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	struct i915_refct_sgt *rsgt;
 	struct scatterlist *sg;
 	struct sg_table *st;
@@ -202,8 +202,8 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
 	list_for_each_entry(block, blocks, link) {
 		u64 block_size, offset;
 
-		block_size = min_t(u64, size, drm_buddy_block_size(mm, block));
-		offset = drm_buddy_block_offset(block);
+		block_size = min_t(u64, size, gpu_buddy_block_size(mm, block));
+		offset = gpu_buddy_block_offset(block);
 
 		while (block_size) {
 			u64 len;
diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.c b/drivers/gpu/drm/i915/i915_sw_fence_work.c
index d2e56b387993d..366418108f78f 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence_work.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence_work.c
@@ -38,7 +38,7 @@ fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 			if (test_bit(DMA_FENCE_WORK_IMM, &f->dma.flags))
 				fence_work(&f->work);
 			else
-				queue_work(system_unbound_wq, &f->work);
+				queue_work(system_dfl_wq, &f->work);
 		} else {
 			fence_complete(f);
 		}
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
index caceef5ab5433..10df50a54e886 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
@@ -5,6 +5,7 @@
 
 #include <linux/slab.h>
 
+#include <linux/gpu_buddy.h>
 #include <drm/drm_buddy.h>
 #include <drm/drm_print.h>
 #include <drm/ttm/ttm_placement.h>
@@ -16,7 +17,7 @@
 
 struct i915_ttm_buddy_manager {
 	struct ttm_resource_manager manager;
-	struct drm_buddy mm;
+	struct gpu_buddy mm;
 	struct list_head reserved;
 	struct mutex lock;
 	unsigned long visible_size;
@@ -38,7 +39,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 {
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 	struct i915_ttm_buddy_resource *bman_res;
-	struct drm_buddy *mm = &bman->mm;
+	struct gpu_buddy *mm = &bman->mm;
 	unsigned long n_pages, lpfn;
 	u64 min_page_size;
 	u64 size;
@@ -57,13 +58,13 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 	bman_res->mm = mm;
 
 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
-		bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
+		bman_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
 
 	if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
-		bman_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
+		bman_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
 
 	if (place->fpfn || lpfn != man->size)
-		bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
+		bman_res->flags |= GPU_BUDDY_RANGE_ALLOCATION;
 
 	GEM_BUG_ON(!bman_res->base.size);
 	size = bman_res->base.size;
@@ -89,7 +90,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 		goto err_free_res;
 	}
 
-	err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
+	err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
 				     (u64)lpfn << PAGE_SHIFT,
 				     (u64)n_pages << PAGE_SHIFT,
 				     min_page_size,
@@ -101,15 +102,15 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 	if (lpfn <= bman->visible_size) {
 		bman_res->used_visible_size = PFN_UP(bman_res->base.size);
 	} else {
-		struct drm_buddy_block *block;
+		struct gpu_buddy_block *block;
 
 		list_for_each_entry(block, &bman_res->blocks, link) {
 			unsigned long start =
-				drm_buddy_block_offset(block) >> PAGE_SHIFT;
+				gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 
 			if (start < bman->visible_size) {
 				unsigned long end = start +
-					(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+					(gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
 
 				bman_res->used_visible_size +=
 					min(end, bman->visible_size) - start;
@@ -126,7 +127,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 	return 0;
 
 err_free_blocks:
-	drm_buddy_free_list(mm, &bman_res->blocks, 0);
+	gpu_buddy_free_list(mm, &bman_res->blocks, 0);
 	mutex_unlock(&bman->lock);
 err_free_res:
 	ttm_resource_fini(man, &bman_res->base);
@@ -141,7 +142,7 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 
 	mutex_lock(&bman->lock);
-	drm_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
+	gpu_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
 	bman->visible_avail += bman_res->used_visible_size;
 	mutex_unlock(&bman->lock);
 
@@ -156,8 +157,8 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
 {
 	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
-	struct drm_buddy *mm = &bman->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &bman->mm;
+	struct gpu_buddy_block *block;
 
 	if (!place->fpfn && !place->lpfn)
 		return true;
@@ -176,9 +177,9 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
 	/* Check each drm buddy block individually */
 	list_for_each_entry(block, &bman_res->blocks, link) {
 		unsigned long fpfn =
-			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+			gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 		unsigned long lpfn = fpfn +
-			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+			(gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
 
 		if (place->fpfn < lpfn && place->lpfn > fpfn)
 			return true;
@@ -194,8 +195,8 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
 {
 	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
-	struct drm_buddy *mm = &bman->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &bman->mm;
+	struct gpu_buddy_block *block;
 
 	if (!place->fpfn && !place->lpfn)
 		return true;
@@ -209,9 +210,9 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
 	/* Check each drm buddy block individually */
 	list_for_each_entry(block, &bman_res->blocks, link) {
 		unsigned long fpfn =
-			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+			gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 		unsigned long lpfn = fpfn +
-			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+			(gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
 
 		if (fpfn < place->fpfn || lpfn > place->lpfn)
 			return false;
@@ -224,7 +225,7 @@ static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
 				     struct drm_printer *printer)
 {
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 
 	mutex_lock(&bman->lock);
 	drm_printf(printer, "default_page_size: %lluKiB\n",
@@ -293,7 +294,7 @@ int i915_ttm_buddy_man_init(struct ttm_device *bdev,
 	if (!bman)
 		return -ENOMEM;
 
-	err = drm_buddy_init(&bman->mm, size, chunk_size);
+	err = gpu_buddy_init(&bman->mm, size, chunk_size);
 	if (err)
 		goto err_free_bman;
 
@@ -333,7 +334,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
 {
 	struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
-	struct drm_buddy *mm = &bman->mm;
+	struct gpu_buddy *mm = &bman->mm;
 	int ret;
 
 	ttm_resource_manager_set_used(man, false);
@@ -345,8 +346,8 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
 	ttm_set_driver_manager(bdev, type, NULL);
 
 	mutex_lock(&bman->lock);
-	drm_buddy_free_list(mm, &bman->reserved, 0);
-	drm_buddy_fini(mm);
+	gpu_buddy_free_list(mm, &bman->reserved, 0);
+	gpu_buddy_fini(mm);
 	bman->visible_avail += bman->visible_reserved;
 	WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
 	mutex_unlock(&bman->lock);
@@ -371,15 +372,15 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
 			       u64 start, u64 size)
 {
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
-	struct drm_buddy *mm = &bman->mm;
+	struct gpu_buddy *mm = &bman->mm;
 	unsigned long fpfn = start >> PAGE_SHIFT;
 	unsigned long flags = 0;
 	int ret;
 
-	flags |= DRM_BUDDY_RANGE_ALLOCATION;
+	flags |= GPU_BUDDY_RANGE_ALLOCATION;
 
 	mutex_lock(&bman->lock);
-	ret = drm_buddy_alloc_blocks(mm, start,
+	ret = gpu_buddy_alloc_blocks(mm, start,
 				     start + size,
 				     size, mm->chunk_size,
 				     &bman->reserved,
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
index d646207128303..1cff018c1689a 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h
@@ -13,7 +13,7 @@
 
 struct ttm_device;
 struct ttm_resource_manager;
-struct drm_buddy;
+struct gpu_buddy;
 
 /**
  * struct i915_ttm_buddy_resource
@@ -33,7 +33,7 @@ struct i915_ttm_buddy_resource {
 	struct list_head blocks;
 	unsigned long flags;
 	unsigned long used_visible_size;
-	struct drm_buddy *mm;
+	struct gpu_buddy *mm;
 };
 
 /**
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index afc192d9931b8..6a3a4d4244dc3 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -27,6 +27,7 @@
 
 #include <drm/drm_gem.h>
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "display/intel_fb.h"
 #include "display/intel_frontbuffer.h"
@@ -2332,3 +2333,12 @@ int __init i915_vma_module_init(void)
 
 	return 0;
 }
+
+static int i915_vma_fence_id(const struct i915_vma *vma)
+{
+	return vma->fence ? vma->fence->id : -1;
+}
+
+const struct intel_display_vma_interface i915_display_vma_interface = {
+	.fence_id = i915_vma_fence_id,
+};
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 8054047840aa2..fa2d9b429db68 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -404,11 +404,6 @@ i915_vma_unpin_fence(struct i915_vma *vma)
 		__i915_vma_unpin_fence(vma);
 }
 
-static inline int i915_vma_fence_id(const struct i915_vma *vma)
-{
-	return vma->fence ? vma->fence->id : -1;
-}
-
 void i915_vma_parked(struct intel_gt *gt);
 
 static inline bool i915_vma_is_scanout(const struct i915_vma *vma)
@@ -481,4 +476,6 @@ int i915_vma_module_init(void);
 I915_SELFTEST_DECLARE(int i915_vma_get_pages(struct i915_vma *vma));
 I915_SELFTEST_DECLARE(void i915_vma_put_pages(struct i915_vma *vma));
 
+extern const struct intel_display_vma_interface i915_display_vma_interface;
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_vma_resource.c b/drivers/gpu/drm/i915/i915_vma_resource.c
index 53d619ef0c3d6..a8f2112ce81f7 100644
--- a/drivers/gpu/drm/i915/i915_vma_resource.c
+++ b/drivers/gpu/drm/i915/i915_vma_resource.c
@@ -202,7 +202,7 @@ i915_vma_resource_fence_notify(struct i915_sw_fence *fence,
 			i915_vma_resource_unbind_work(&vma_res->work);
 		} else {
 			INIT_WORK(&vma_res->work, i915_vma_resource_unbind_work);
-			queue_work(system_unbound_wq, &vma_res->work);
+			queue_work(system_dfl_wq, &vma_res->work);
 		}
 		break;
 	case FENCE_FREE:
diff --git a/drivers/gpu/drm/i915/i915_wait_util.h b/drivers/gpu/drm/i915/i915_wait_util.h
index 7376898e3bf83..e1ed7921ec701 100644
--- a/drivers/gpu/drm/i915/i915_wait_util.h
+++ b/drivers/gpu/drm/i915/i915_wait_util.h
@@ -25,9 +25,9 @@
 	might_sleep();							\
 	for (;;) {							\
 		const bool expired__ = ktime_after(ktime_get_raw(), end__); \
-		OP;							\
 		/* Guarantee COND check prior to timeout */		\
 		barrier();						\
+		OP;							\
 		if (COND) {						\
 			ret__ = 0;					\
 			break;						\
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
index 7336934bb9345..68a6f94f2a376 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.c
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -26,11 +26,13 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "display/i9xx_plane_regs.h"
 #include "display/intel_display.h"
 #include "display/intel_display_core.h"
-
+#include "display/intel_display_regs.h"
 #include "gt/intel_engine_regs.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_mcr.h"
@@ -452,7 +454,7 @@ static void bdw_init_clock_gating(struct drm_i915_private *i915)
 			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 
 	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
-			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 
 	/* WaDisableSDEUnitClockGating:bdw */
 	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
@@ -517,13 +519,13 @@ static void ivb_init_clock_gating(struct drm_i915_private *i915)
 
 	if (INTEL_INFO(i915)->gt == 1)
 		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
-				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+				   REG_MASKED_FIELD_ENABLE(DOP_CLOCK_GATING_DISABLE));
 	else {
 		/* must write both registers */
 		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
-				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+				   REG_MASKED_FIELD_ENABLE(DOP_CLOCK_GATING_DISABLE));
 		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
-				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+				   REG_MASKED_FIELD_ENABLE(DOP_CLOCK_GATING_DISABLE));
 	}
 
 	/*
@@ -557,7 +559,7 @@ static void vlv_init_clock_gating(struct drm_i915_private *i915)
 
 	/* WaDisableDopClockGating:vlv */
 	intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
-			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(DOP_CLOCK_GATING_DISABLE));
 
 	/* This is required by WaCatErrorRejectionIssue:vlv */
 	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
@@ -592,7 +594,7 @@ static void chv_init_clock_gating(struct drm_i915_private *i915)
 
 	/* WaDisableSemaphoreAndSyncFlipWait:chv */
 	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
-			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 
 	/* WaDisableCSUnitClockGating:chv */
 	intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
@@ -638,7 +640,7 @@ static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 	intel_uncore_write16(uncore, DEUC, 0);
 	intel_uncore_write(uncore,
 			   MI_ARB_STATE,
-			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void i965g_init_clock_gating(struct drm_i915_private *i915)
@@ -650,7 +652,7 @@ static void i965g_init_clock_gating(struct drm_i915_private *i915)
 			   I965_FBC_CLOCK_GATE_DISABLE);
 	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
-			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void gen3_init_clock_gating(struct drm_i915_private *i915)
@@ -663,21 +665,21 @@ static void gen3_init_clock_gating(struct drm_i915_private *i915)
 
 	if (IS_PINEVIEW(i915))
 		intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
-				   _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
+				   REG_MASKED_FIELD_ENABLE(ECO_GATING_CX_ONLY));
 
 	/* IIR "flip pending" means done if this bit is set */
 	intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
-			   _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
+			   REG_MASKED_FIELD_DISABLE(ECO_FLIP_DONE));
 
 	/* interrupts should cause a wake up from C3 */
-	intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
+	intel_uncore_write(&i915->uncore, INSTPM, REG_MASKED_FIELD_ENABLE(INSTPM_AGPBUSY_INT_EN));
 
 	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
-			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
+			   REG_MASKED_FIELD_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 
 	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
-			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void i85x_init_clock_gating(struct drm_i915_private *i915)
@@ -685,11 +687,11 @@ static void i85x_init_clock_gating(struct drm_i915_private *i915)
 	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 
 	/* interrupts should cause a wake up from C3 */
-	intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
-			   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
+	intel_uncore_write(&i915->uncore, MI_STATE, REG_MASKED_FIELD_ENABLE(MI_AGPBUSY_INT_EN) |
+			   REG_MASKED_FIELD_DISABLE(MI_AGPBUSY_830_MODE));
 
 	intel_uncore_write(&i915->uncore, MEM_MODE,
-			   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 
 	/*
 	 * Have FBC ignore 3D activity since we use software
@@ -699,14 +701,14 @@ static void i85x_init_clock_gating(struct drm_i915_private *i915)
 	 * until a 2D blit occurs.
 	 */
 	intel_uncore_write(&i915->uncore, SCPD0,
-			   _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
+			   REG_MASKED_FIELD_ENABLE(SCPD_FBC_IGNORE_3D));
 }
 
 static void i830_init_clock_gating(struct drm_i915_private *i915)
 {
 	intel_uncore_write(&i915->uncore, MEM_MODE,
-			   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
-			   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
+			   REG_MASKED_FIELD_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
+			   REG_MASKED_FIELD_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 }
 
 void intel_clock_gating_init(struct drm_device *drm)
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index 478d00f89a4bf..ae42818ab6e07 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -3,6 +3,11 @@
  * Copyright © 2020 Intel Corporation
  */
 
+#include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
+
+#include <drm/intel/intel_gmd_interrupt_regs.h>
+
 #include "display/bxt_dpio_phy_regs.h"
 #include "display/i9xx_plane_regs.h"
 #include "display/i9xx_wm_regs.h"
@@ -11,12 +16,12 @@
 #include "display/intel_color_regs.h"
 #include "display/intel_crt_regs.h"
 #include "display/intel_cursor_regs.h"
-#include "display/intel_display_core.h"
+#include "display/intel_display_limits.h"
 #include "display/intel_display_regs.h"
-#include "display/intel_display_types.h"
 #include "display/intel_dmc_regs.h"
 #include "display/intel_dp_aux_regs.h"
 #include "display/intel_dpio_phy.h"
+#include "display/intel_fbc.h"
 #include "display/intel_fbc_regs.h"
 #include "display/intel_fdi_regs.h"
 #include "display/intel_lvds_regs.h"
@@ -32,6 +37,7 @@
 #include "gt/intel_engine_regs.h"
 #include "gt/intel_gt_regs.h"
 
+#include "gvt/display_helpers.h"
 #include "gvt/reg.h"
 
 #include "i915_drv.h"
diff --git a/drivers/gpu/drm/i915/intel_pcode.c b/drivers/gpu/drm/i915/intel_pcode.c
index 756652b8ec975..c07d48fc1b355 100644
--- a/drivers/gpu/drm/i915/intel_pcode.c
+++ b/drivers/gpu/drm/i915/intel_pcode.c
@@ -4,6 +4,8 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+#include <drm/intel/intel_pcode_regs.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
@@ -276,26 +278,31 @@ int snb_pcode_write_p(struct intel_uncore *uncore, u32 mbcmd, u32 p1, u32 p2, u3
 	return err;
 }
 
-/* Helpers with drm device */
-int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
+static int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return snb_pcode_read(&i915->uncore, mbox, val, val1);
 }
 
-int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
+static int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return snb_pcode_write_timeout(&i915->uncore, mbox, val, timeout_ms);
 }
 
-int intel_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
-			u32 reply_mask, u32 reply, int timeout_base_ms)
+static int intel_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
+			       u32 reply_mask, u32 reply, int timeout_base_ms)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return skl_pcode_request(&i915->uncore, mbox, request, reply_mask, reply,
 				 timeout_base_ms);
 }
+
+const struct intel_display_pcode_interface i915_display_pcode_interface = {
+	.read = intel_pcode_read,
+	.write = intel_pcode_write_timeout,
+	.request = intel_pcode_request,
+};
diff --git a/drivers/gpu/drm/i915/intel_pcode.h b/drivers/gpu/drm/i915/intel_pcode.h
index c91a821a88d4a..19795ea8172e7 100644
--- a/drivers/gpu/drm/i915/intel_pcode.h
+++ b/drivers/gpu/drm/i915/intel_pcode.h
@@ -27,13 +27,6 @@ int intel_pcode_init(struct intel_uncore *uncore);
 int snb_pcode_read_p(struct intel_uncore *uncore, u32 mbcmd, u32 p1, u32 p2, u32 *val);
 int snb_pcode_write_p(struct intel_uncore *uncore, u32 mbcmd, u32 p1, u32 p2, u32 val);
 
-/* Helpers with drm device */
-int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1);
-int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms);
-#define intel_pcode_write(drm, mbox, val) \
-	intel_pcode_write_timeout((drm), (mbox), (val), 1)
-
-int intel_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
-			u32 reply_mask, u32 reply, int timeout_base_ms);
+extern const struct intel_display_pcode_interface i915_display_pcode_interface;
 
 #endif /* _INTEL_PCODE_H */
diff --git a/drivers/gpu/drm/i915/intel_step.h b/drivers/gpu/drm/i915/intel_step.h
index 22f1d69051600..2ca36eae4b5a6 100644
--- a/drivers/gpu/drm/i915/intel_step.h
+++ b/drivers/gpu/drm/i915/intel_step.h
@@ -8,6 +8,8 @@
 
 #include <linux/types.h>
 
+#include <drm/intel/step.h>
+
 struct drm_i915_private;
 
 struct intel_step_info {
@@ -19,61 +21,6 @@ struct intel_step_info {
 	u8 media_step;
 };
 
-#define STEP_ENUM_VAL(name)  STEP_##name,
-
-#define STEP_NAME_LIST(func)		\
-	func(A0)			\
-	func(A1)			\
-	func(A2)			\
-	func(A3)			\
-	func(B0)			\
-	func(B1)			\
-	func(B2)			\
-	func(B3)			\
-	func(C0)			\
-	func(C1)			\
-	func(C2)			\
-	func(C3)			\
-	func(D0)			\
-	func(D1)			\
-	func(D2)			\
-	func(D3)			\
-	func(E0)			\
-	func(E1)			\
-	func(E2)			\
-	func(E3)			\
-	func(F0)			\
-	func(F1)			\
-	func(F2)			\
-	func(F3)			\
-	func(G0)			\
-	func(G1)			\
-	func(G2)			\
-	func(G3)			\
-	func(H0)			\
-	func(H1)			\
-	func(H2)			\
-	func(H3)			\
-	func(I0)			\
-	func(I1)			\
-	func(I2)			\
-	func(I3)			\
-	func(J0)			\
-	func(J1)			\
-	func(J2)			\
-	func(J3)
-
-/*
- * Symbolic steppings that do not match the hardware. These are valid both as gt
- * and display steppings as symbolic names.
- */
-enum intel_step {
-	STEP_NONE = 0,
-	STEP_NAME_LIST(STEP_ENUM_VAL)
-	STEP_FUTURE,
-	STEP_FOREVER,
-};
-
 void intel_step_init(struct drm_i915_private *i915);
 const char *intel_step_name(enum intel_step step);
 
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index bccedd59a1145..5b698d4d7a7f0 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -132,8 +132,8 @@ intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 }
 
 #define fw_ack(d) readl((d)->reg_ack)
-#define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
-#define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
+#define fw_set(d, val) writel(REG_MASKED_FIELD_ENABLE((val)), (d)->reg_set)
+#define fw_clear(d, val) writel(REG_MASKED_FIELD_DISABLE((val)), (d)->reg_set)
 
 static inline void
 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.c b/drivers/gpu/drm/i915/pxp/intel_pxp.c
index f7ed4e18a3abc..3d7f045f662d0 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp.c
@@ -66,8 +66,8 @@ bool intel_pxp_is_active(const struct intel_pxp *pxp)
 
 static void kcr_pxp_set_status(const struct intel_pxp *pxp, bool enable)
 {
-	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
-		  _MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
+	u32 val = enable ? REG_MASKED_FIELD_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
+		  REG_MASKED_FIELD_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
 
 	intel_uncore_write(pxp->ctrl_gt->uncore, KCR_INIT(pxp->kcr_base), val);
 }
@@ -278,7 +278,7 @@ static void pxp_queue_termination(struct intel_pxp *pxp)
 	spin_lock_irq(gt->irq_lock);
 	intel_pxp_mark_termination_in_progress(pxp);
 	pxp->session_events |= PXP_TERMINATION_REQUEST;
-	queue_work(system_unbound_wq, &pxp->session_work);
+	queue_work(system_dfl_wq, &pxp->session_work);
 	spin_unlock_irq(gt->irq_lock);
 }
 
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
index d81750b9bddaa..735325e828bc9 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
@@ -48,7 +48,7 @@ void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir)
 		pxp->session_events |= PXP_TERMINATION_COMPLETE | PXP_EVENT_TYPE_IRQ;
 
 	if (pxp->session_events)
-		queue_work(system_unbound_wq, &pxp->session_work);
+		queue_work(system_dfl_wq, &pxp->session_work);
 }
 
 static inline void __pxp_set_interrupts(struct intel_gt *gt, u32 interrupts)
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c
index 52345073b409a..9fea2fabeac48 100644
--- a/drivers/gpu/drm/i915/selftests/i915_active.c
+++ b/drivers/gpu/drm/i915/selftests/i915_active.c
@@ -323,9 +323,9 @@ static void active_flush(struct i915_active *ref,
 	if (!fence)
 		return;
 
-	spin_lock_irq(fence->lock);
+	spin_lock_irq(dma_fence_spinlock(fence));
 	__list_del_entry(&active->cb.node);
-	spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */
+	spin_unlock_irq(dma_fence_spinlock(fence)); /* serialise with fence->cb_list */
 	atomic_dec(&ref->count);
 
 	GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
diff --git a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
index 6c376338bb37a..6a305322e30db 100644
--- a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
@@ -526,7 +526,7 @@ static int test_ipc(void *arg)
 	struct workqueue_struct *wq;
 	int ret = 0;
 
-	wq = alloc_workqueue("i1915-selftest", 0, 0);
+	wq = alloc_workqueue("i1915-selftest", WQ_PERCPU, 0);
 	if (wq == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/selftests/igt_mmap.c b/drivers/gpu/drm/i915/selftests/igt_mmap.c
index e920a461bd367..106d5c0dfcbc5 100644
--- a/drivers/gpu/drm/i915/selftests/igt_mmap.c
+++ b/drivers/gpu/drm/i915/selftests/igt_mmap.c
@@ -9,14 +9,14 @@
 #include "i915_drv.h"
 #include "igt_mmap.h"
 
-unsigned long igt_mmap_offset(struct drm_i915_private *i915,
-			      u64 offset,
-			      unsigned long size,
-			      unsigned long prot,
-			      unsigned long flags)
+unsigned long igt_mmap_offset_with_file(struct drm_i915_private *i915,
+					u64 offset,
+					unsigned long size,
+					unsigned long prot,
+					unsigned long flags,
+					struct file *file)
 {
 	struct drm_vma_offset_node *node;
-	struct file *file;
 	unsigned long addr;
 	int err;
 
@@ -31,22 +31,35 @@ unsigned long igt_mmap_offset(struct drm_i915_private *i915,
 		return -ENOENT;
 	}
 
-	/* Pretend to open("/dev/dri/card0") */
-	file = mock_drm_getfile(i915->drm.primary, O_RDWR);
-	if (IS_ERR(file))
-		return PTR_ERR(file);
-
 	err = drm_vma_node_allow(node, file->private_data);
 	if (err) {
-		addr = err;
-		goto out_file;
+		return err;
 	}
 
 	addr = vm_mmap(file, 0, drm_vma_node_size(node) << PAGE_SHIFT,
 		       prot, flags, drm_vma_node_offset_addr(node));
 
 	drm_vma_node_revoke(node, file->private_data);
-out_file:
+
+	return addr;
+}
+
+unsigned long igt_mmap_offset(struct drm_i915_private *i915,
+			      u64 offset,
+			      unsigned long size,
+			      unsigned long prot,
+			      unsigned long flags)
+{
+	struct file *file;
+	unsigned long addr;
+
+	/* Pretend to open("/dev/dri/card0") */
+	file = mock_drm_getfile(i915->drm.primary, O_RDWR);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	addr = igt_mmap_offset_with_file(i915, offset, size, prot, flags, file);
 	fput(file);
+
 	return addr;
 }
diff --git a/drivers/gpu/drm/i915/selftests/igt_mmap.h b/drivers/gpu/drm/i915/selftests/igt_mmap.h
index acbe34d81a6d4..7b177b44cd3c8 100644
--- a/drivers/gpu/drm/i915/selftests/igt_mmap.h
+++ b/drivers/gpu/drm/i915/selftests/igt_mmap.h
@@ -11,6 +11,7 @@
 
 struct drm_i915_private;
 struct drm_vma_offset_node;
+struct file;
 
 unsigned long igt_mmap_offset(struct drm_i915_private *i915,
 			      u64 offset,
@@ -18,4 +19,11 @@ unsigned long igt_mmap_offset(struct drm_i915_private *i915,
 			      unsigned long prot,
 			      unsigned long flags);
 
+unsigned long igt_mmap_offset_with_file(struct drm_i915_private *i915,
+					u64 offset,
+					unsigned long size,
+					unsigned long prot,
+					unsigned long flags,
+					struct file *file);
+
 #endif /* IGT_MMAP_H */
diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c
index 7b856b5090f9a..8307390943a2f 100644
--- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c
+++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c
@@ -6,7 +6,7 @@
 #include <linux/prime_numbers.h>
 #include <linux/sort.h>
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 
 #include "../i915_selftest.h"
 
@@ -371,7 +371,7 @@ static int igt_mock_splintered_region(void *arg)
 	struct drm_i915_private *i915 = mem->i915;
 	struct i915_ttm_buddy_resource *res;
 	struct drm_i915_gem_object *obj;
-	struct drm_buddy *mm;
+	struct gpu_buddy *mm;
 	unsigned int expected_order;
 	LIST_HEAD(objects);
 	u64 size;
@@ -447,8 +447,8 @@ static int igt_mock_max_segment(void *arg)
 	struct drm_i915_private *i915 = mem->i915;
 	struct i915_ttm_buddy_resource *res;
 	struct drm_i915_gem_object *obj;
-	struct drm_buddy_block *block;
-	struct drm_buddy *mm;
+	struct gpu_buddy_block *block;
+	struct gpu_buddy *mm;
 	struct list_head *blocks;
 	struct scatterlist *sg;
 	I915_RND_STATE(prng);
@@ -487,8 +487,8 @@ static int igt_mock_max_segment(void *arg)
 	mm = res->mm;
 	size = 0;
 	list_for_each_entry(block, blocks, link) {
-		if (drm_buddy_block_size(mm, block) > size)
-			size = drm_buddy_block_size(mm, block);
+		if (gpu_buddy_block_size(mm, block) > size)
+			size = gpu_buddy_block_size(mm, block);
 	}
 	if (size < max_segment) {
 		pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n",
@@ -527,14 +527,14 @@ static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj)
 	struct intel_memory_region *mr = obj->mm.region;
 	struct i915_ttm_buddy_resource *bman_res =
 		to_ttm_buddy_resource(obj->mm.res);
-	struct drm_buddy *mm = bman_res->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = bman_res->mm;
+	struct gpu_buddy_block *block;
 	u64 total;
 
 	total = 0;
 	list_for_each_entry(block, &bman_res->blocks, link) {
-		u64 start = drm_buddy_block_offset(block);
-		u64 end = start + drm_buddy_block_size(mm, block);
+		u64 start = gpu_buddy_block_offset(block);
+		u64 end = start + gpu_buddy_block_size(mm, block);
 
 		if (start < resource_size(&mr->io))
 			total += min_t(u64, end, resource_size(&mr->io)) - start;
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 210b9f8f7b611..27dc0e40a8d75 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -223,7 +223,7 @@ struct drm_i915_private *mock_gem_device(void)
 	if (!i915->wq)
 		goto err_drv;
 
-	i915->unordered_wq = alloc_workqueue("mock-unordered", 0, 0);
+	i915->unordered_wq = alloc_workqueue("mock-unordered", WQ_PERCPU, 0);
 	if (!i915->unordered_wq)
 		goto err_wq;
 
diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c
index dac4b9bac7430..3612b03cabd14 100644
--- a/drivers/gpu/drm/i915/vlv_suspend.c
+++ b/drivers/gpu/drm/i915/vlv_suspend.c
@@ -7,6 +7,7 @@
 #include <linux/kernel.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/intel_gmd_interrupt_regs.h>
 
 #include "gt/intel_gt_regs.h"
 
diff --git a/drivers/gpu/drm/imagination/pvr_ccb.c b/drivers/gpu/drm/imagination/pvr_ccb.c
index 00c236d24acc3..f89db5e3baa20 100644
--- a/drivers/gpu/drm/imagination/pvr_ccb.c
+++ b/drivers/gpu/drm/imagination/pvr_ccb.c
@@ -136,6 +136,14 @@ pvr_ccb_slot_available_locked(struct pvr_ccb *pvr_ccb, u32 *write_offset)
 static void
 process_fwccb_command(struct pvr_device *pvr_dev, struct rogue_fwif_fwccb_cmd *cmd)
 {
+	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
+
+	if ((cmd->cmd_type & ROGUE_CMD_MAGIC_DWORD_MASK) != ROGUE_CMD_MAGIC_DWORD_SHIFTED) {
+		drm_warn_once(drm_dev, "Received FWCCB command with bad magic value; ignoring (type=0x%08x)\n",
+			      cmd->cmd_type);
+		return;
+	}
+
 	switch (cmd->cmd_type) {
 	case ROGUE_FWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
 		pvr_power_reset(pvr_dev, false);
@@ -150,9 +158,17 @@ process_fwccb_command(struct pvr_device *pvr_dev, struct rogue_fwif_fwccb_cmd *c
 		pvr_free_list_process_grow_req(pvr_dev, &cmd->cmd_data.cmd_free_list_gs);
 		break;
 
+	case ROGUE_FWIF_FWCCB_CMD_UPDATE_STATS:
+		/*
+		 * We currently have no infrastructure for processing these
+		 * stats. It may be added in the future, but for now just
+		 * suppress the "unknown" warning when receiving this command.
+		 */
+		break;
+
 	default:
-		drm_info(from_pvr_device(pvr_dev), "Received unknown FWCCB command %x\n",
-			 cmd->cmd_type);
+		drm_info(drm_dev, "Received unknown FWCCB command (type=%d)\n",
+			 cmd->cmd_type & ~ROGUE_CMD_MAGIC_DWORD_MASK);
 		break;
 	}
 }
diff --git a/drivers/gpu/drm/imagination/pvr_device.c b/drivers/gpu/drm/imagination/pvr_device.c
index f58bb66a63275..dbb6f5a8ded12 100644
--- a/drivers/gpu/drm/imagination/pvr_device.c
+++ b/drivers/gpu/drm/imagination/pvr_device.c
@@ -225,29 +225,12 @@ static irqreturn_t pvr_device_irq_thread_handler(int irq, void *data)
 	}
 
 	if (pvr_dev->has_safety_events) {
-		int err;
-
-		/*
-		 * Ensure the GPU is powered on since some safety events (such
-		 * as ECC faults) can happen outside of job submissions, which
-		 * are otherwise the only time a power reference is held.
-		 */
-		err = pvr_power_get(pvr_dev);
-		if (err) {
-			drm_err_ratelimited(drm_dev,
-					    "%s: could not take power reference (%d)\n",
-					    __func__, err);
-			return ret;
-		}
-
 		while (pvr_device_safety_irq_pending(pvr_dev)) {
 			pvr_device_safety_irq_clear(pvr_dev);
 			pvr_device_handle_safety_events(pvr_dev);
 
 			ret = IRQ_HANDLED;
 		}
-
-		pvr_power_put(pvr_dev);
 	}
 
 	return ret;
diff --git a/drivers/gpu/drm/imagination/pvr_device.h b/drivers/gpu/drm/imagination/pvr_device.h
index cfda215e7428e..d51c57cf93323 100644
--- a/drivers/gpu/drm/imagination/pvr_device.h
+++ b/drivers/gpu/drm/imagination/pvr_device.h
@@ -152,15 +152,13 @@ struct pvr_device {
 	 * @power: Optional power domain devices.
 	 *
 	 * On platforms with more than one power domain for the GPU, they are
-	 * stored here in @domain_devs, along with links between them in
-	 * @domain_links. The size of @domain_devs is given by @domain_count,
-	 * while the size of @domain_links is (2 * @domain_count) - 1.
+	 * stored here in @domains, along with links between them in
+	 * @domain_links. The size of @domain_links is one less than
+	 * struct dev_pm_domain_list->num_pds in @domains.
 	 */
 	struct pvr_device_power {
-		struct device **domain_devs;
+		struct dev_pm_domain_list *domains;
 		struct device_link **domain_links;
-
-		u32 domain_count;
 	} power;
 
 	/**
diff --git a/drivers/gpu/drm/imagination/pvr_gem.c b/drivers/gpu/drm/imagination/pvr_gem.c
index 686a3fe22986d..d8660d6a8e01b 100644
--- a/drivers/gpu/drm/imagination/pvr_gem.c
+++ b/drivers/gpu/drm/imagination/pvr_gem.c
@@ -25,7 +25,10 @@
 
 static void pvr_gem_object_free(struct drm_gem_object *obj)
 {
-	drm_gem_shmem_object_free(obj);
+	struct drm_gem_shmem_object *shmem_obj = to_drm_gem_shmem_obj(obj);
+
+	shmem_obj->pages_mark_dirty_on_put = true;
+	drm_gem_shmem_free(shmem_obj);
 }
 
 static struct dma_buf *pvr_gem_export(struct drm_gem_object *obj, int flags)
@@ -363,7 +366,6 @@ pvr_gem_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags)
 	if (IS_ERR(shmem_obj))
 		return ERR_CAST(shmem_obj);
 
-	shmem_obj->pages_mark_dirty_on_put = true;
 	shmem_obj->map_wc = !(flags & PVR_BO_CPU_CACHED);
 	pvr_obj = shmem_gem_to_pvr_gem(shmem_obj);
 	pvr_obj->flags = flags;
diff --git a/drivers/gpu/drm/imagination/pvr_power.c b/drivers/gpu/drm/imagination/pvr_power.c
index 0cf7393f89c6b..88fcce433d123 100644
--- a/drivers/gpu/drm/imagination/pvr_power.c
+++ b/drivers/gpu/drm/imagination/pvr_power.c
@@ -90,11 +90,11 @@ pvr_power_request_pwr_off(struct pvr_device *pvr_dev)
 }
 
 static int
-pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset)
+pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset, bool rpm_suspend)
 {
-	if (!hard_reset) {
-		int err;
+	int err;
 
+	if (!hard_reset) {
 		cancel_delayed_work_sync(&pvr_dev->watchdog.work);
 
 		err = pvr_power_request_idle(pvr_dev);
@@ -106,29 +106,47 @@ pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset)
 			return err;
 	}
 
-	return pvr_fw_stop(pvr_dev);
+	if (rpm_suspend) {
+		/* This also waits for late processing of GPU or firmware IRQs in other cores */
+		disable_irq(pvr_dev->irq);
+	}
+
+	err = pvr_fw_stop(pvr_dev);
+	if (err && rpm_suspend)
+		enable_irq(pvr_dev->irq);
+
+	return err;
 }
 
 static int
-pvr_power_fw_enable(struct pvr_device *pvr_dev)
+pvr_power_fw_enable(struct pvr_device *pvr_dev, bool rpm_resume)
 {
 	int err;
 
+	if (rpm_resume)
+		enable_irq(pvr_dev->irq);
+
 	err = pvr_fw_start(pvr_dev);
 	if (err)
-		return err;
+		goto out;
 
 	err = pvr_wait_for_fw_boot(pvr_dev);
 	if (err) {
 		drm_err(from_pvr_device(pvr_dev), "Firmware failed to boot\n");
 		pvr_fw_stop(pvr_dev);
-		return err;
+		goto out;
 	}
 
 	queue_delayed_work(pvr_dev->sched_wq, &pvr_dev->watchdog.work,
 			   msecs_to_jiffies(WATCHDOG_TIME_MS));
 
 	return 0;
+
+out:
+	if (rpm_resume)
+		disable_irq(pvr_dev->irq);
+
+	return err;
 }
 
 bool
@@ -361,7 +379,7 @@ pvr_power_device_suspend(struct device *dev)
 		return -EIO;
 
 	if (pvr_dev->fw_dev.booted) {
-		err = pvr_power_fw_disable(pvr_dev, false);
+		err = pvr_power_fw_disable(pvr_dev, false, true);
 		if (err)
 			goto err_drm_dev_exit;
 	}
@@ -391,7 +409,7 @@ pvr_power_device_resume(struct device *dev)
 		goto err_drm_dev_exit;
 
 	if (pvr_dev->fw_dev.booted) {
-		err = pvr_power_fw_enable(pvr_dev);
+		err = pvr_power_fw_enable(pvr_dev, true);
 		if (err)
 			goto err_power_off;
 	}
@@ -510,7 +528,16 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
 	}
 
 	/* Disable IRQs for the duration of the reset. */
-	disable_irq(pvr_dev->irq);
+	if (hard_reset) {
+		disable_irq(pvr_dev->irq);
+	} else {
+		/*
+		 * Soft reset is triggered as a response to a FW command to the Host and is
+		 * processed from the threaded IRQ handler. This code cannot (nor needs to)
+		 * wait for any IRQ processing to complete.
+		 */
+		disable_irq_nosync(pvr_dev->irq);
+	}
 
 	do {
 		if (hard_reset) {
@@ -518,7 +545,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
 			queues_disabled = true;
 		}
 
-		err = pvr_power_fw_disable(pvr_dev, hard_reset);
+		err = pvr_power_fw_disable(pvr_dev, hard_reset, false);
 		if (!err) {
 			if (hard_reset) {
 				pvr_dev->fw_dev.booted = false;
@@ -541,7 +568,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
 
 			pvr_fw_irq_clear(pvr_dev);
 
-			err = pvr_power_fw_enable(pvr_dev);
+			err = pvr_power_fw_enable(pvr_dev, false);
 		}
 
 		if (err && hard_reset)
@@ -593,65 +620,61 @@ pvr_watchdog_fini(struct pvr_device *pvr_dev)
 
 int pvr_power_domains_init(struct pvr_device *pvr_dev)
 {
-	struct device *dev = from_pvr_device(pvr_dev)->dev;
+	static const char *const ROGUE_PD_NAMES[] = { "a", "b", "c", "d", "e" };
 
-	struct device_link **domain_links __free(kfree) = NULL;
-	struct device **domain_devs __free(kfree) = NULL;
+	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
+	struct device *dev = drm_dev->dev;
+
+	struct dev_pm_domain_list *domains = NULL;
+	struct device_link **domain_links = NULL;
 	int domain_count;
 	int link_count;
 
-	char dev_name[2] = "a";
 	int err;
 	int i;
 
 	domain_count = of_count_phandle_with_args(dev->of_node, "power-domains",
 						  "#power-domain-cells");
-	if (domain_count < 0)
-		return domain_count;
+	if (domain_count < 0) {
+		err = domain_count;
+		goto out;
+	}
 
-	if (domain_count <= 1)
-		return 0;
+	if (domain_count <= 1) {
+		err = 0;
+		goto out;
+	}
 
-	link_count = domain_count + (domain_count - 1);
+	if (domain_count > ARRAY_SIZE(ROGUE_PD_NAMES)) {
+		drm_err(drm_dev, "%s() only supports %zu domains on Rogue",
+			__func__, ARRAY_SIZE(ROGUE_PD_NAMES));
+		err = -EOPNOTSUPP;
+		goto out;
+	}
 
-	domain_devs = kzalloc_objs(*domain_devs, domain_count);
-	if (!domain_devs)
-		return -ENOMEM;
+	link_count = domain_count - 1;
 
 	domain_links = kzalloc_objs(*domain_links, link_count);
-	if (!domain_links)
-		return -ENOMEM;
-
-	for (i = 0; i < domain_count; i++) {
-		struct device *domain_dev;
-
-		dev_name[0] = 'a' + i;
-		domain_dev = dev_pm_domain_attach_by_name(dev, dev_name);
-		if (IS_ERR_OR_NULL(domain_dev)) {
-			err = domain_dev ? PTR_ERR(domain_dev) : -ENODEV;
-			goto err_detach;
-		}
-
-		domain_devs[i] = domain_dev;
+	if (!domain_links) {
+		err = -ENOMEM;
+		goto out;
 	}
 
-	for (i = 0; i < domain_count; i++) {
-		struct device_link *link;
-
-		link = device_link_add(dev, domain_devs[i], DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
-		if (!link) {
-			err = -ENODEV;
-			goto err_unlink;
-		}
+	const struct dev_pm_domain_attach_data pd_attach_data = {
+		.pd_names = ROGUE_PD_NAMES,
+		.num_pd_names = domain_count,
+		.pd_flags = 0,
+	};
 
-		domain_links[i] = link;
-	}
+	err = dev_pm_domain_attach_list(dev, &pd_attach_data, &domains);
+	if (err < 0)
+		goto err_free_links;
 
-	for (i = domain_count; i < link_count; i++) {
+	for (i = 0; i < link_count; i++) {
 		struct device_link *link;
 
-		link = device_link_add(domain_devs[i - domain_count + 1],
-				       domain_devs[i - domain_count],
+		link = device_link_add(domains->pd_devs[i + 1],
+				       domains->pd_devs[i],
 				       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
 		if (!link) {
 			err = -ENODEV;
@@ -661,43 +684,43 @@ int pvr_power_domains_init(struct pvr_device *pvr_dev)
 		domain_links[i] = link;
 	}
 
-	pvr_dev->power = (struct pvr_device_power){
-		.domain_devs = no_free_ptr(domain_devs),
-		.domain_links = no_free_ptr(domain_links),
-		.domain_count = domain_count,
-	};
-
-	return 0;
+	err = 0;
+	goto out;
 
 err_unlink:
 	while (--i >= 0)
 		device_link_del(domain_links[i]);
 
-	i = domain_count;
+	dev_pm_domain_detach_list(domains);
+	domains = NULL;
 
-err_detach:
-	while (--i >= 0)
-		dev_pm_domain_detach(domain_devs[i], true);
+err_free_links:
+	kfree(domain_links);
+	domain_links = NULL;
+
+out:
+	pvr_dev->power = (struct pvr_device_power){
+		.domains = domains,
+		.domain_links = domain_links,
+	};
 
 	return err;
 }
 
 void pvr_power_domains_fini(struct pvr_device *pvr_dev)
 {
-	const int domain_count = pvr_dev->power.domain_count;
+	struct pvr_device_power *pvr_power = &pvr_dev->power;
 
-	int i = domain_count + (domain_count - 1);
+	if (!pvr_power->domains)
+		goto out;
 
-	while (--i >= 0)
-		device_link_del(pvr_dev->power.domain_links[i]);
+	for (int i = (int)pvr_power->domains->num_pds - 2; i >= 0; --i)
+		device_link_del(pvr_power->domain_links[i]);
 
-	i = domain_count;
-
-	while (--i >= 0)
-		dev_pm_domain_detach(pvr_dev->power.domain_devs[i], true);
+	dev_pm_domain_detach_list(pvr_power->domains);
 
-	kfree(pvr_dev->power.domain_links);
-	kfree(pvr_dev->power.domain_devs);
+	kfree(pvr_power->domain_links);
 
-	pvr_dev->power = (struct pvr_device_power){ 0 };
+out:
+	*pvr_power = (struct pvr_device_power){ 0 };
 }
diff --git a/drivers/gpu/drm/imx/ipuv3/Kconfig b/drivers/gpu/drm/imx/ipuv3/Kconfig
index acaf250890019..b2240998df4f1 100644
--- a/drivers/gpu/drm/imx/ipuv3/Kconfig
+++ b/drivers/gpu/drm/imx/ipuv3/Kconfig
@@ -15,6 +15,7 @@ config DRM_IMX_PARALLEL_DISPLAY
 	depends on DRM_IMX
 	select DRM_BRIDGE
 	select DRM_BRIDGE_CONNECTOR
+	select DRM_DISPLAY_HELPER
 	select DRM_IMX_LEGACY_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
index 0ea70b8fd7430..c1c7be4e26c64 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
@@ -890,7 +890,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 {
 	struct ipu_plane *ipu_plane;
 	const uint64_t *modifiers = ipu_format_modifiers;
-	unsigned int zpos = (type == DRM_PLANE_TYPE_PRIMARY) ? 0 : 1;
+	unsigned int primary_zpos = 1;
 	unsigned int format_count;
 	const uint32_t *formats;
 	int ret;
@@ -915,7 +915,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 					       type, NULL);
 	if (IS_ERR(ipu_plane)) {
 		DRM_ERROR("failed to allocate and initialize %s plane\n",
-			  zpos ? "overlay" : "primary");
+			  (type == DRM_PLANE_TYPE_PRIMARY) ? "primary" : "overlay");
 		return ipu_plane;
 	}
 
@@ -923,17 +923,16 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 	ipu_plane->dma = dma;
 	ipu_plane->dp_flow = dp;
 
-	if (type == DRM_PLANE_TYPE_PRIMARY)
+	if (type == DRM_PLANE_TYPE_PRIMARY) {
 		drm_plane_helper_add(&ipu_plane->base, &ipu_primary_plane_helper_funcs);
-	else
-		drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs);
-
-	if (dp == IPU_DP_FLOW_SYNC_BG || dp == IPU_DP_FLOW_SYNC_FG)
-		ret = drm_plane_create_zpos_property(&ipu_plane->base, zpos, 0,
-						     1);
-	else
 		ret = drm_plane_create_zpos_immutable_property(&ipu_plane->base,
-							       0);
+							       primary_zpos);
+	} else {
+		drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs);
+		ret = drm_plane_create_zpos_property(&ipu_plane->base,
+						     primary_zpos + 1, 0,
+						     primary_zpos + 1);
+	}
 	if (ret)
 		return ERR_PTR(ret);
 
@@ -949,7 +948,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 	ret = ipu_plane_get_resources(dev, ipu_plane);
 	if (ret) {
 		DRM_ERROR("failed to get %s plane resources: %pe\n",
-			  zpos ? "overlay" : "primary", &ret);
+			  (type == DRM_PLANE_TYPE_PRIMARY) ? "primary" : "overlay", &ret);
 		return ERR_PTR(ret);
 	}
 
diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
index 4ce772bc3cb3a..a356f0b764cb7 100644
--- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c
+++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
@@ -110,8 +110,7 @@ imx_pd_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 		output_fmt = imxpd->bus_format ? : MEDIA_BUS_FMT_RGB888_1X24;
 
 	/* Now make sure the requested output format is supported. */
-	if ((imxpd->bus_format && imxpd->bus_format != output_fmt) ||
-	    !imx_pd_format_supported(output_fmt)) {
+	if (!imx_pd_format_supported(output_fmt)) {
 		*num_input_fmts = 0;
 		return NULL;
 	}
@@ -121,7 +120,17 @@ imx_pd_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 	if (!input_fmts)
 		return NULL;
 
-	input_fmts[0] = output_fmt;
+	/*
+	 * Prefer bus format set via legacy "interface-pix-fmt" DT property
+	 * over panel bus format. This is necessary to retain support for
+	 * DTs which configure the IPUv3 parallel output as 24bit, but
+	 * connect 18bit DPI panels to it with hardware swizzling.
+	 */
+	if (imxpd->bus_format && imxpd->bus_format != output_fmt)
+		input_fmts[0] = imxpd->bus_format;
+	else
+		input_fmts[0] = output_fmt;
+
 	return input_fmts;
 }
 
diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
index 6601589339f66..4068114adf8c2 100644
--- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
+++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
@@ -954,6 +954,20 @@ static void ingenic_drm_destroy_state(struct drm_private_obj *obj,
 	kfree(priv_state);
 }
 
+static struct drm_private_state *
+ingenic_drm_create_state(struct drm_private_obj *obj)
+{
+	struct ingenic_drm_private_state *priv_state;
+
+	priv_state = kzalloc_obj(*priv_state);
+	if (!priv_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &priv_state->base);
+
+	return &priv_state->base;
+}
+
 DEFINE_DRM_GEM_DMA_FOPS(ingenic_drm_fops);
 
 static const struct drm_driver ingenic_drm_driver_data = {
@@ -1034,6 +1048,7 @@ static struct drm_mode_config_helper_funcs ingenic_drm_mode_config_helpers = {
 };
 
 static const struct drm_private_state_funcs ingenic_drm_private_state_funcs = {
+	.atomic_create_state = ingenic_drm_create_state,
 	.atomic_duplicate_state = ingenic_drm_duplicate_state,
 	.atomic_destroy_state = ingenic_drm_destroy_state,
 };
@@ -1087,7 +1102,6 @@ static void ingenic_drm_atomic_private_obj_fini(struct drm_device *drm, void *pr
 static int ingenic_drm_bind(struct device *dev, bool has_components)
 {
 	struct platform_device *pdev = to_platform_device(dev);
-	struct ingenic_drm_private_state *private_state;
 	const struct jz_soc_info *soc_info;
 	struct ingenic_drm *priv;
 	struct clk *parent_clk;
@@ -1387,19 +1401,13 @@ static int ingenic_drm_bind(struct device *dev, bool has_components)
 		goto err_devclk_disable;
 	}
 
-	private_state = kzalloc_obj(*private_state);
-	if (!private_state) {
-		ret = -ENOMEM;
-		goto err_clk_notifier_unregister;
-	}
-
-	drm_atomic_private_obj_init(drm, &priv->private_obj, &private_state->base,
+	drm_atomic_private_obj_init(drm, &priv->private_obj,
 				    &ingenic_drm_private_state_funcs);
 
 	ret = drmm_add_action_or_reset(drm, ingenic_drm_atomic_private_obj_fini,
 				       &priv->private_obj);
 	if (ret)
-		goto err_private_state_free;
+		goto err_clk_notifier_unregister;
 
 	ret = drm_dev_register(drm, 0);
 	if (ret) {
@@ -1411,8 +1419,6 @@ static int ingenic_drm_bind(struct device *dev, bool has_components)
 
 	return 0;
 
-err_private_state_free:
-	kfree(private_state);
 err_clk_notifier_unregister:
 	clk_notifier_unregister(parent_clk, &priv->clock_nb);
 err_devclk_disable:
diff --git a/drivers/gpu/drm/ingenic/ingenic-ipu.c b/drivers/gpu/drm/ingenic/ingenic-ipu.c
index 289274f464ba7..34545b9c8c338 100644
--- a/drivers/gpu/drm/ingenic/ingenic-ipu.c
+++ b/drivers/gpu/drm/ingenic/ingenic-ipu.c
@@ -750,7 +750,22 @@ static void ingenic_ipu_destroy_state(struct drm_private_obj *obj,
 	kfree(priv_state);
 }
 
+static struct drm_private_state *
+ingenic_ipu_create_state(struct drm_private_obj *obj)
+{
+	struct ingenic_ipu_private_state *priv_state;
+
+	priv_state = kzalloc_obj(*priv_state);
+	if (!priv_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &priv_state->base);
+
+	return &priv_state->base;
+}
+
 static const struct drm_private_state_funcs ingenic_ipu_private_state_funcs = {
+	.atomic_create_state = ingenic_ipu_create_state,
 	.atomic_duplicate_state = ingenic_ipu_duplicate_state,
 	.atomic_destroy_state = ingenic_ipu_destroy_state,
 };
@@ -793,7 +808,6 @@ static const struct regmap_config ingenic_ipu_regmap_config = {
 static int ingenic_ipu_bind(struct device *dev, struct device *master, void *d)
 {
 	struct platform_device *pdev = to_platform_device(dev);
-	struct ingenic_ipu_private_state *private_state;
 	const struct soc_info *soc_info;
 	struct drm_device *drm = d;
 	struct drm_plane *plane;
@@ -887,20 +901,10 @@ static int ingenic_ipu_bind(struct device *dev, struct device *master, void *d)
 		return err;
 	}
 
-	private_state = kzalloc_obj(*private_state);
-	if (!private_state) {
-		err = -ENOMEM;
-		goto err_clk_unprepare;
-	}
-
-	drm_atomic_private_obj_init(drm, &ipu->private_obj, &private_state->base,
+	drm_atomic_private_obj_init(drm, &ipu->private_obj,
 				    &ingenic_ipu_private_state_funcs);
 
 	return 0;
-
-err_clk_unprepare:
-	clk_unprepare(ipu->clk);
-	return err;
 }
 
 static void ingenic_ipu_unbind(struct device *dev,
diff --git a/drivers/gpu/drm/lib/drm_random.c b/drivers/gpu/drm/lib/drm_random.c
deleted file mode 100644
index 0e9dba1ef4af5..0000000000000
--- a/drivers/gpu/drm/lib/drm_random.c
+++ /dev/null
@@ -1,44 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/bitops.h>
-#include <linux/export.h>
-#include <linux/kernel.h>
-#include <linux/random.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-
-#include "drm_random.h"
-
-u32 drm_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
-{
-	return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
-}
-EXPORT_SYMBOL(drm_prandom_u32_max_state);
-
-void drm_random_reorder(unsigned int *order, unsigned int count,
-			struct rnd_state *state)
-{
-	unsigned int i, j;
-
-	for (i = 0; i < count; ++i) {
-		BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));
-		j = drm_prandom_u32_max_state(count, state);
-		swap(order[i], order[j]);
-	}
-}
-EXPORT_SYMBOL(drm_random_reorder);
-
-unsigned int *drm_random_order(unsigned int count, struct rnd_state *state)
-{
-	unsigned int *order, i;
-
-	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
-	if (!order)
-		return order;
-
-	for (i = 0; i < count; i++)
-		order[i] = i;
-
-	drm_random_reorder(order, count, state);
-	return order;
-}
-EXPORT_SYMBOL(drm_random_order);
diff --git a/drivers/gpu/drm/lib/drm_random.h b/drivers/gpu/drm/lib/drm_random.h
deleted file mode 100644
index 9f827260a89db..0000000000000
--- a/drivers/gpu/drm/lib/drm_random.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __DRM_RANDOM_H__
-#define __DRM_RANDOM_H__
-
-/* This is a temporary home for a couple of utility functions that should
- * be transposed to lib/ at the earliest convenience.
- */
-
-#include <linux/prandom.h>
-
-#define DRM_RND_STATE_INITIALIZER(seed__) ({				\
-	struct rnd_state state__;					\
-	prandom_seed_state(&state__, (seed__));				\
-	state__;							\
-})
-
-#define DRM_RND_STATE(name__, seed__) \
-	struct rnd_state name__ = DRM_RND_STATE_INITIALIZER(seed__)
-
-unsigned int *drm_random_order(unsigned int count,
-			       struct rnd_state *state);
-void drm_random_reorder(unsigned int *order,
-			unsigned int count,
-			struct rnd_state *state);
-u32 drm_prandom_u32_max_state(u32 ep_ro,
-			      struct rnd_state *state);
-
-#endif /* !__DRM_RANDOM_H__ */
diff --git a/drivers/gpu/drm/loongson/lsdc_drv.c b/drivers/gpu/drm/loongson/lsdc_drv.c
index abf5bf68eec21..1ece1ea42f781 100644
--- a/drivers/gpu/drm/loongson/lsdc_drv.c
+++ b/drivers/gpu/drm/loongson/lsdc_drv.c
@@ -13,6 +13,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fbdev_ttm.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_gem_ttm_helper.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_print.h>
@@ -45,7 +46,7 @@ static const struct drm_driver lsdc_drm_driver = {
 
 	.debugfs_init = lsdc_debugfs_init,
 	.dumb_create = lsdc_dumb_create,
-	.dumb_map_offset = lsdc_dumb_map_offset,
+	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
 	.gem_prime_import_sg_table = lsdc_prime_import_sg_table,
 	DRM_FBDEV_TTM_DRIVER_OPS,
 };
diff --git a/drivers/gpu/drm/loongson/lsdc_gem.c b/drivers/gpu/drm/loongson/lsdc_gem.c
index 6372db2d30930..2fb03487c983e 100644
--- a/drivers/gpu/drm/loongson/lsdc_gem.c
+++ b/drivers/gpu/drm/loongson/lsdc_gem.c
@@ -234,21 +234,6 @@ int lsdc_dumb_create(struct drm_file *file, struct drm_device *ddev,
 	return 0;
 }
 
-int lsdc_dumb_map_offset(struct drm_file *filp, struct drm_device *ddev,
-			 u32 handle, uint64_t *offset)
-{
-	struct drm_gem_object *gobj;
-
-	gobj = drm_gem_object_lookup(filp, handle);
-	if (!gobj)
-		return -ENOENT;
-
-	*offset = drm_vma_node_offset_addr(&gobj->vma_node);
-
-	drm_gem_object_put(gobj);
-
-	return 0;
-}
 
 void lsdc_gem_init(struct drm_device *ddev)
 {
diff --git a/drivers/gpu/drm/loongson/lsdc_gem.h b/drivers/gpu/drm/loongson/lsdc_gem.h
index 92cbb10e6e13d..96729b3eebecc 100644
--- a/drivers/gpu/drm/loongson/lsdc_gem.h
+++ b/drivers/gpu/drm/loongson/lsdc_gem.h
@@ -14,10 +14,6 @@ lsdc_prime_import_sg_table(struct drm_device *ddev,
 			   struct dma_buf_attachment *attach,
 			   struct sg_table *sg);
 
-int lsdc_dumb_map_offset(struct drm_file *file,
-			 struct drm_device *dev,
-			 u32 handle,
-			 uint64_t *offset);
 
 int lsdc_dumb_create(struct drm_file *file,
 		     struct drm_device *ddev,
diff --git a/drivers/gpu/drm/mcde/mcde_dsi.c b/drivers/gpu/drm/mcde/mcde_dsi.c
index a3423459dd7ac..47d45897ed069 100644
--- a/drivers/gpu/drm/mcde/mcde_dsi.c
+++ b/drivers/gpu/drm/mcde/mcde_dsi.c
@@ -40,7 +40,6 @@ struct mcde_dsi {
 	struct mcde *mcde;
 	struct drm_bridge bridge;
 	struct drm_panel *panel;
-	struct drm_bridge *bridge_out;
 	struct mipi_dsi_host dsi_host;
 	struct mipi_dsi_device *mdsi;
 	const struct drm_display_mode *mode;
@@ -1060,7 +1059,7 @@ static int mcde_dsi_bridge_attach(struct drm_bridge *bridge,
 	}
 
 	/* Attach the DSI bridge to the output (panel etc) bridge */
-	return drm_bridge_attach(encoder, d->bridge_out, bridge, flags);
+	return drm_bridge_attach(encoder, d->bridge.next_bridge, bridge, flags);
 }
 
 static const struct drm_bridge_funcs mcde_dsi_bridge_funcs = {
@@ -1076,7 +1075,7 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 	struct mcde_dsi *d = dev_get_drvdata(dev);
 	struct device_node *child;
 	struct drm_panel *panel = NULL;
-	struct drm_bridge *bridge = NULL;
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
 
 	if (!of_get_available_child_count(dev->of_node)) {
 		dev_info(dev, "unused DSI interface\n");
@@ -1109,13 +1108,18 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 				PTR_ERR(panel));
 			panel = NULL;
 
-			bridge = of_drm_find_bridge(child);
+			bridge = of_drm_find_and_get_bridge(child);
 			if (!bridge) {
 				dev_err(dev, "failed to find bridge\n");
 				of_node_put(child);
 				return -EINVAL;
 			}
 		}
+
+		if (panel || bridge) {
+			of_node_put(child);
+			break;
+		}
 	}
 	if (panel) {
 		bridge = drm_panel_bridge_add_typed(panel,
@@ -1124,6 +1128,7 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 			dev_err(dev, "error adding panel bridge\n");
 			return PTR_ERR(bridge);
 		}
+		drm_bridge_get(bridge);
 		dev_info(dev, "connected to panel\n");
 		d->panel = panel;
 	} else if (bridge) {
@@ -1135,7 +1140,7 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 		return -ENODEV;
 	}
 
-	d->bridge_out = bridge;
+	d->bridge.next_bridge = drm_bridge_get(bridge);
 
 	/* Create a bridge for this DSI channel */
 	d->bridge.of_node = dev->of_node;
@@ -1155,7 +1160,7 @@ static void mcde_dsi_unbind(struct device *dev, struct device *master,
 	struct mcde_dsi *d = dev_get_drvdata(dev);
 
 	if (d->panel)
-		drm_panel_bridge_remove(d->bridge_out);
+		drm_panel_bridge_remove(d->bridge.next_bridge);
 	regmap_update_bits(d->prcmu, PRCM_DSI_SW_RESET,
 			   PRCM_DSI_SW_RESET_DSI0_SW_RESETN, 0);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index 61d7e65469b3a..31743699d05f6 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -367,6 +367,24 @@ static void dpu_kms_global_destroy_state(struct drm_private_obj *obj,
 	kfree(dpu_state);
 }
 
+static struct drm_private_state *
+dpu_kms_global_create_state(struct drm_private_obj *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
+	struct dpu_global_state *dpu_state;
+
+	dpu_state = kzalloc_obj(*dpu_state);
+	if (!dpu_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &dpu_state->base);
+	dpu_state->rm = &dpu_kms->rm;
+
+	return &dpu_state->base;
+}
+
 static void dpu_kms_global_print_state(struct drm_printer *p,
 				       const struct drm_private_state *state)
 {
@@ -376,28 +394,12 @@ static void dpu_kms_global_print_state(struct drm_printer *p,
 }
 
 static const struct drm_private_state_funcs dpu_kms_global_state_funcs = {
+	.atomic_create_state = dpu_kms_global_create_state,
 	.atomic_duplicate_state = dpu_kms_global_duplicate_state,
 	.atomic_destroy_state = dpu_kms_global_destroy_state,
 	.atomic_print_state = dpu_kms_global_print_state,
 };
 
-static int dpu_kms_global_obj_init(struct dpu_kms *dpu_kms)
-{
-	struct dpu_global_state *state;
-
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
-
-	drm_atomic_private_obj_init(dpu_kms->dev, &dpu_kms->global_state,
-				    &state->base,
-				    &dpu_kms_global_state_funcs);
-
-	state->rm = &dpu_kms->rm;
-
-	return 0;
-}
-
 static void dpu_kms_global_obj_fini(struct dpu_kms *dpu_kms)
 {
 	drm_atomic_private_obj_fini(&dpu_kms->global_state);
@@ -1158,9 +1160,8 @@ static int dpu_kms_hw_init(struct msm_kms *kms)
 	dev->mode_config.cursor_width = 512;
 	dev->mode_config.cursor_height = 512;
 
-	rc = dpu_kms_global_obj_init(dpu_kms);
-	if (rc)
-		return rc;
+	drm_atomic_private_obj_init(dpu_kms->dev, &dpu_kms->global_state,
+				    &dpu_kms_global_state_funcs);
 
 	atomic_set(&dpu_kms->bandwidth_ref, 0);
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index 8bb503e0f9620..2d26b07b06f54 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -114,6 +114,24 @@ static void mdp5_global_destroy_state(struct drm_private_obj *obj,
 	kfree(mdp5_state);
 }
 
+static struct drm_private_state *
+mdp5_global_create_state(struct drm_private_obj *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
+	struct mdp5_global_state *mdp5_state;
+
+	mdp5_state = kzalloc_obj(*mdp5_state);
+	if (!mdp5_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &mdp5_state->base);
+	mdp5_state->mdp5_kms = mdp5_kms;
+
+	return &mdp5_state->base;
+}
+
 static void mdp5_global_print_state(struct drm_printer *p,
 				    const struct drm_private_state *state)
 {
@@ -124,27 +142,12 @@ static void mdp5_global_print_state(struct drm_printer *p,
 }
 
 static const struct drm_private_state_funcs mdp5_global_state_funcs = {
+	.atomic_create_state = mdp5_global_create_state,
 	.atomic_duplicate_state = mdp5_global_duplicate_state,
 	.atomic_destroy_state = mdp5_global_destroy_state,
 	.atomic_print_state = mdp5_global_print_state,
 };
 
-static int mdp5_global_obj_init(struct mdp5_kms *mdp5_kms)
-{
-	struct mdp5_global_state *state;
-
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
-
-	state->mdp5_kms = mdp5_kms;
-
-	drm_atomic_private_obj_init(mdp5_kms->dev, &mdp5_kms->glob_state,
-				    &state->base,
-				    &mdp5_global_state_funcs);
-	return 0;
-}
-
 static void mdp5_enable_commit(struct msm_kms *kms)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
@@ -713,9 +716,8 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 
 	mdp5_kms->dev = dev;
 
-	ret = mdp5_global_obj_init(mdp5_kms);
-	if (ret)
-		goto fail;
+	drm_atomic_private_obj_init(mdp5_kms->dev, &mdp5_kms->glob_state,
+				    &mdp5_global_state_funcs);
 
 	/* we need to set a default rate before enabling.  Set a safe
 	 * rate first, then figure out hw revision, and then set a
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index fcb2a7517377e..47da1d9336b90 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -149,15 +149,17 @@ static int lcdif_load(struct drm_device *drm)
 
 	lcdif->clk = devm_clk_get(drm->dev, "pix");
 	if (IS_ERR(lcdif->clk))
-		return PTR_ERR(lcdif->clk);
+		return dev_err_probe(drm->dev, PTR_ERR(lcdif->clk), "Failed to get pix clock\n");
 
 	lcdif->clk_axi = devm_clk_get(drm->dev, "axi");
 	if (IS_ERR(lcdif->clk_axi))
-		return PTR_ERR(lcdif->clk_axi);
+		return dev_err_probe(drm->dev, PTR_ERR(lcdif->clk_axi),
+				     "Failed to get axi clock\n");
 
 	lcdif->clk_disp_axi = devm_clk_get(drm->dev, "disp_axi");
 	if (IS_ERR(lcdif->clk_disp_axi))
-		return PTR_ERR(lcdif->clk_disp_axi);
+		return dev_err_probe(drm->dev, PTR_ERR(lcdif->clk_disp_axi),
+				     "Failed to get disp_axi clock\n");
 
 	platform_set_drvdata(pdev, drm);
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
index a2333cfe6955c..490ce410f6cbc 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
@@ -3,9 +3,28 @@
 #define __NVKM_GR_H__
 #include <core/engine.h>
 
+struct nvkm_gr_zcull_info {
+	__u32 width_align_pixels;
+	__u32 height_align_pixels;
+	__u32 pixel_squares_by_aliquots;
+	__u32 aliquot_total;
+	__u32 zcull_region_byte_multiplier;
+	__u32 zcull_region_header_size;
+	__u32 zcull_subregion_header_size;
+	__u32 subregion_count;
+	__u32 subregion_width_align_pixels;
+	__u32 subregion_height_align_pixels;
+
+	__u32 ctxsw_size;
+	__u32 ctxsw_align;
+};
+
 struct nvkm_gr {
 	const struct nvkm_gr_func *func;
 	struct nvkm_engine engine;
+
+	struct nvkm_gr_zcull_info zcull_info;
+	bool has_zcull_info;
 };
 
 u64 nvkm_gr_units(struct nvkm_gr *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index f9201f2e73a33..7860877d909b0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -333,6 +333,35 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 	return 0;
 }
 
+int
+nouveau_abi16_ioctl_get_zcull_info(ABI16_IOCTL_ARGS)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvkm_gr *gr = nvxx_gr(drm);
+	struct drm_nouveau_get_zcull_info *out = data;
+
+	if (gr->has_zcull_info) {
+		const struct nvkm_gr_zcull_info *i = &gr->zcull_info;
+
+		out->width_align_pixels = i->width_align_pixels;
+		out->height_align_pixels = i->height_align_pixels;
+		out->pixel_squares_by_aliquots = i->pixel_squares_by_aliquots;
+		out->aliquot_total = i->aliquot_total;
+		out->zcull_region_byte_multiplier = i->zcull_region_byte_multiplier;
+		out->zcull_region_header_size = i->zcull_region_header_size;
+		out->zcull_subregion_header_size = i->zcull_subregion_header_size;
+		out->subregion_count = i->subregion_count;
+		out->subregion_width_align_pixels = i->subregion_width_align_pixels;
+		out->subregion_height_align_pixels = i->subregion_height_align_pixels;
+		out->ctxsw_size = i->ctxsw_size;
+		out->ctxsw_align = i->ctxsw_align;
+
+		return 0;
+	} else {
+		return -ENOTTY;
+	}
+}
+
 int
 nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 {
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.h b/drivers/gpu/drm/nouveau/nouveau_abi16.h
index af6b4e1cefd23..134b3ab58719b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.h
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.h
@@ -6,6 +6,7 @@
 	struct drm_device *dev, void *data, struct drm_file *file_priv
 
 int nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS);
+int nouveau_abi16_ioctl_get_zcull_info(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_channel_free(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 3c7d2e5b38508..0e8de6d4b36f7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -144,7 +144,7 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
 	nouveau_bo_del_io_reserve_lru(bo);
 	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
 
-	if (bo->base.import_attach)
+	if (drm_gem_is_imported(&bo->base))
 		drm_prime_gem_destroy(&bo->base, bo->sg);
 
 	/*
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 915f73279302d..5d8475e4895e4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -156,12 +156,13 @@ nouveau_name(struct drm_device *dev)
 static inline bool
 nouveau_cli_work_ready(struct dma_fence *fence)
 {
+	unsigned long flags;
 	bool ret = true;
 
-	spin_lock_irq(fence->lock);
+	dma_fence_lock_irqsave(fence, flags);
 	if (!dma_fence_is_signaled_locked(fence))
 		ret = false;
-	spin_unlock_irq(fence->lock);
+	dma_fence_unlock_irqrestore(fence, flags);
 
 	if (ret == true)
 		dma_fence_put(fence);
@@ -1079,6 +1080,37 @@ nouveau_pmops_resume(struct device *dev)
 	return ret;
 }
 
+static void
+nouveau_drm_shutdown(struct pci_dev *pdev)
+{
+	struct nouveau_drm *drm = pci_get_drvdata(pdev);
+	int ret;
+
+	if (!drm)
+		return;
+
+	if (drm->dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
+	    drm->dev->switch_power_state == DRM_SWITCH_POWER_DYNAMIC_OFF)
+		return;
+
+	ret = nouveau_do_suspend(drm, false);
+	if (ret)
+		NV_ERROR(drm, "shutdown suspend failed with: %d\n", ret);
+
+	pci_save_state(pdev);
+	pci_disable_device(pdev);
+	pci_set_power_state(pdev, PCI_D3hot);
+	/*
+	 *  This is just to give the pci power transition time to settle
+	 *  before an immediate kexec jump. it’s mirroring the existing
+	 *  nouveau_pmops_suspend() behavior, which already does
+	 *  udelay(200) right after pci_set_power_state(..., pci_d3hot). In
+	 *  ->shutdown() we’re allowed to sleep, so I used usleep_range()
+	 *  instead of a busy-wait udelay().
+	 */
+	usleep_range(200, 400);
+}
+
 static int
 nouveau_pmops_freeze(struct device *dev)
 {
@@ -1272,6 +1304,7 @@ nouveau_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GET_ZCULL_INFO, nouveau_abi16_ioctl_get_zcull_info, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_INIT, nouveau_svmm_init, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_BIND, nouveau_svmm_bind, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_RENDER_ALLOW),
@@ -1408,6 +1441,7 @@ nouveau_drm_pci_driver = {
 	.id_table = nouveau_drm_pci_table,
 	.probe = nouveau_drm_probe,
 	.remove = nouveau_drm_remove,
+	.shutdown = nouveau_drm_shutdown,
 	.driver.pm = &nouveau_pm_ops,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 903d326927cac..edbe9e08ba0fd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -41,7 +41,8 @@ static const struct dma_fence_ops nouveau_fence_ops_legacy;
 static inline struct nouveau_fence_chan *
 nouveau_fctx(struct nouveau_fence *fence)
 {
-	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
+	return container_of(fence->base.extern_lock, struct nouveau_fence_chan,
+			    lock);
 }
 
 static bool
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index b101e14f841e0..72848ed80df73 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -2513,7 +2513,6 @@ static const struct nvkm_device_chip
 nv170_chipset = {
 	.name = "GA100",
 	.bar      = { 0x00000001, tu102_bar_new },
-	.bios     = { 0x00000001, nvkm_bios_new },
 	.devinit  = { 0x00000001, ga100_devinit_new },
 	.fault    = { 0x00000001, tu102_fault_new },
 	.fb       = { 0x00000001, ga100_fb_new },
@@ -2530,6 +2529,7 @@ nv170_chipset = {
 	.vfn      = { 0x00000001, ga100_vfn_new },
 	.ce       = { 0x000003ff, ga100_ce_new },
 	.fifo     = { 0x00000001, ga100_fifo_new },
+	.sec2     = { 0x00000001, tu102_sec2_new },
 };
 
 static const struct nvkm_device_chip
@@ -3341,6 +3341,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 	case 0x166: device->chip = &nv166_chipset; break;
 	case 0x167: device->chip = &nv167_chipset; break;
 	case 0x168: device->chip = &nv168_chipset; break;
+	case 0x170: device->chip = &nv170_chipset; break;
 	case 0x172: device->chip = &nv172_chipset; break;
 	case 0x173: device->chip = &nv173_chipset; break;
 	case 0x174: device->chip = &nv174_chipset; break;
@@ -3360,14 +3361,6 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 	case 0x1b6: device->chip = &nv1b6_chipset; break;
 	case 0x1b7: device->chip = &nv1b7_chipset; break;
 	default:
-		if (nvkm_boolopt(device->cfgopt, "NvEnableUnsupportedChipsets", false)) {
-			switch (device->chipset) {
-			case 0x170: device->chip = &nv170_chipset; break;
-			default:
-				break;
-			}
-		}
-
 		if (!device->chip) {
 			nvdev_error(device, "unknown chipset (%08x)\n", boot0);
 			ret = -ENODEV;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
index 27a13aeccd3cb..fdd820eeef815 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
@@ -41,15 +41,11 @@ ga100_gsp_flcn = {
 static const struct nvkm_gsp_func
 ga100_gsp = {
 	.flcn = &ga100_gsp_flcn,
-	.fwsec = &tu102_gsp_fwsec,
 
 	.sig_section = ".fwsignature_ga100",
 
 	.booter.ctor = tu102_gsp_booter_ctor,
 
-	.fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor,
-	.fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor,
-
 	.dtor = r535_gsp_dtor,
 	.oneinit = tu102_gsp_oneinit,
 	.init = tu102_gsp_init,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gr.c
index 7fe488d4d5ff0..034db286d285b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gr.c
@@ -249,7 +249,7 @@ r535_gr_get_ctxbuf_info(struct r535_gr *gr, int i,
 }
 
 static int
-r535_gr_get_ctxbufs_info(struct r535_gr *gr)
+r535_gr_get_ctxbufs_and_zcull_info(struct r535_gr *gr)
 {
 	NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info;
 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
@@ -265,6 +265,9 @@ r535_gr_get_ctxbufs_info(struct r535_gr *gr)
 		r535_gr_get_ctxbuf_info(gr, i, &info->engineContextBuffersInfo[0].engine[i]);
 
 	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info);
+
+	gr->base.has_zcull_info = false;
+
 	return 0;
 }
 
@@ -312,7 +315,7 @@ r535_gr_oneinit(struct nvkm_gr *base)
 	 *
 	 * Also build the information that'll be used to create channel contexts.
 	 */
-	ret = rm->api->gr->get_ctxbufs_info(gr);
+	ret = rm->api->gr->get_ctxbufs_and_zcull_info(gr);
 	if (ret)
 		goto done;
 
@@ -352,5 +355,5 @@ r535_gr_dtor(struct nvkm_gr *base)
 
 const struct nvkm_rm_api_gr
 r535_gr = {
-	.get_ctxbufs_info = r535_gr_get_ctxbufs_info,
+	.get_ctxbufs_and_zcull_info = r535_gr_get_ctxbufs_and_zcull_info,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
index 0121d56394714..f544afa12b6bb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
@@ -796,7 +796,8 @@ r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
 	struct acpi_object_list input = { 1, &mux_arg };
 	acpi_handle iter = NULL, handle_mux = NULL;
 	acpi_status status;
-	unsigned long long value;
+	u64 value;
+	int ret;
 
 	mode->status = 0xffff;
 	part->status = 0xffff;
@@ -806,8 +807,8 @@ r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
 		if (ACPI_FAILURE(status) || !iter)
 			return;
 
-		status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
-		if (ACPI_FAILURE(status) || value != id)
+		ret = acpi_get_local_u64_address(iter, &value);
+		if (ret || value != id)
 			continue;
 
 		handle_mux = iter;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gr.c
index b6cced9b8aa1a..8dd4552aeaa5d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gr.c
@@ -164,9 +164,10 @@ r570_gr_scrubber_init(struct r535_gr *gr)
 }
 
 static int
-r570_gr_get_ctxbufs_info(struct r535_gr *gr)
+r570_gr_get_ctxbufs_and_zcull_info(struct r535_gr *gr)
 {
 	NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info;
+	NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS *zcull_info;
 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
 	struct nvkm_gsp *gsp = subdev->device->gsp;
 
@@ -179,13 +180,42 @@ r570_gr_get_ctxbufs_info(struct r535_gr *gr)
 	for (int i = 0; i < ARRAY_SIZE(info->engineContextBuffersInfo[0].engine); i++)
 		r535_gr_get_ctxbuf_info(gr, i, &info->engineContextBuffersInfo[0].engine[i]);
 
+	NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO zcull = info->engineContextBuffersInfo[0]
+		.engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ZCULL];
+	gr->base.zcull_info.ctxsw_size = zcull.size;
+	gr->base.zcull_info.ctxsw_align = zcull.alignment;
+
 	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info);
+
+	zcull_info = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+					 NV2080_CTRL_CMD_GR_GET_ZCULL_INFO,
+					 sizeof(*zcull_info));
+	if (IS_ERR(zcull_info)) {
+		nvdev_error(gr->base.engine.subdev.device, "could not fetch zcull info\n");
+		return PTR_ERR(zcull_info);
+	}
+
+	gr->base.zcull_info.width_align_pixels = zcull_info->widthAlignPixels;
+	gr->base.zcull_info.height_align_pixels = zcull_info->heightAlignPixels;
+	gr->base.zcull_info.pixel_squares_by_aliquots = zcull_info->pixelSquaresByAliquots;
+	gr->base.zcull_info.aliquot_total = zcull_info->aliquotTotal;
+	gr->base.zcull_info.zcull_region_byte_multiplier = zcull_info->zcullRegionByteMultiplier;
+	gr->base.zcull_info.zcull_region_header_size = zcull_info->zcullRegionHeaderSize;
+	gr->base.zcull_info.zcull_subregion_header_size = zcull_info->zcullSubregionHeaderSize;
+	gr->base.zcull_info.subregion_count = zcull_info->subregionCount;
+	gr->base.zcull_info.subregion_width_align_pixels = zcull_info->subregionWidthAlignPixels;
+	gr->base.zcull_info.subregion_height_align_pixels = zcull_info->subregionHeightAlignPixels;
+
+	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, zcull_info);
+
+	gr->base.has_zcull_info = true;
+
 	return 0;
 }
 
 const struct nvkm_rm_api_gr
 r570_gr = {
-	.get_ctxbufs_info = r570_gr_get_ctxbufs_info,
+	.get_ctxbufs_and_zcull_info = r570_gr_get_ctxbufs_and_zcull_info,
 	.scrubber.init = r570_gr_scrubber_init,
 	.scrubber.fini = r570_gr_scrubber_fini,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gr.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gr.h
index feed1dabd9d29..a7a15a4de9d51 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gr.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gr.h
@@ -76,4 +76,23 @@ typedef struct NV2080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS {
 } NV2080_CTRL_INTERNAL_GR_INIT_BUG4208224_WAR_PARAMS;
 
 #define NV2080_CTRL_CMD_INTERNAL_KGR_INIT_BUG4208224_WAR (0x20800a46) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_INTERNAL_KGR_INIT_BUG4208224_WAR_PARAMS_MESSAGE_ID" */
+
+#define NV2080_CTRL_CMD_GR_GET_ZCULL_INFO            (0x20801206U) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_GR_INTERFACE_ID << 8) | NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS_MESSAGE_ID" */
+
+#define NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS_SUBREGION_SUPPORTED
+#define NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS_MESSAGE_ID (0x6U)
+
+typedef struct NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS {
+    NvU32 widthAlignPixels;
+    NvU32 heightAlignPixels;
+    NvU32 pixelSquaresByAliquots;
+    NvU32 aliquotTotal;
+    NvU32 zcullRegionByteMultiplier;
+    NvU32 zcullRegionHeaderSize;
+    NvU32 zcullSubregionHeaderSize;
+    NvU32 subregionCount;
+    NvU32 subregionWidthAlignPixels;
+    NvU32 subregionHeightAlignPixels;
+} NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS;
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
index 4f0ae6cc085c3..a9af94adf9efc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
@@ -124,7 +124,7 @@ struct nvkm_rm_api {
 	} *ce, *nvdec, *nvenc, *nvjpg, *ofa;
 
 	const struct nvkm_rm_api_gr {
-		int (*get_ctxbufs_info)(struct r535_gr *);
+		int (*get_ctxbufs_and_zcull_info)(struct r535_gr *);
 		struct {
 			int (*init)(struct r535_gr *);
 			void (*fini)(struct r535_gr *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
index 19cb269e7a265..dd82c76b8b9a5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
@@ -318,8 +318,13 @@ tu102_gsp_oneinit(struct nvkm_gsp *gsp)
 	if (ret)
 		return ret;
 
-	/* Calculate FB layout. */
-	gsp->fb.wpr2.frts.size = 0x100000;
+	/*
+	 * Calculate FB layout. FRTS is a memory region created by the FWSEC-FRTS firmware.
+	 * FWSEC comes from VBIOS.  So on systems with no VBIOS (e.g. GA100), the FRTS does
+	 * not exist.  Therefore, use the existence of VBIOS to determine whether to reserve
+	 * an FRTS region.
+	 */
+	gsp->fb.wpr2.frts.size = device->bios ? 0x100000 : 0;
 	gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
 
 	gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
@@ -343,9 +348,12 @@ tu102_gsp_oneinit(struct nvkm_gsp *gsp)
 	if (ret)
 		return ret;
 
-	ret = nvkm_gsp_fwsec_frts(gsp);
-	if (WARN_ON(ret))
-		return ret;
+	/* Only boot FWSEC-FRTS if it actually exists */
+	if (gsp->fb.wpr2.frts.size) {
+		ret = nvkm_gsp_fwsec_frts(gsp);
+		if (WARN_ON(ret))
+			return ret;
+	}
 
 	/* Reset GSP into RISC-V mode. */
 	ret = gsp->func->reset(gsp);
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index bf0bad8c8cf12..ae678696fbacb 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -275,7 +275,22 @@ static void omap_global_destroy_state(struct drm_private_obj *obj,
 	kfree(omap_state);
 }
 
+static struct drm_private_state *
+omap_global_atomic_create_state(struct drm_private_obj *obj)
+{
+	struct omap_global_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &state->base);
+
+	return &state->base;
+}
+
 static const struct drm_private_state_funcs omap_global_state_funcs = {
+	.atomic_create_state = omap_global_atomic_create_state,
 	.atomic_duplicate_state = omap_global_duplicate_state,
 	.atomic_destroy_state = omap_global_destroy_state,
 };
@@ -283,13 +298,8 @@ static const struct drm_private_state_funcs omap_global_state_funcs = {
 static int omap_global_obj_init(struct drm_device *dev)
 {
 	struct omap_drm_private *priv = dev->dev_private;
-	struct omap_global_state *state;
-
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
 
-	drm_atomic_private_obj_init(dev, &priv->glob_obj, &state->base,
+	drm_atomic_private_obj_init(dev, &priv->glob_obj,
 				    &omap_global_state_funcs);
 	return 0;
 }
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 307152ad77591..a99f2e2a49fe6 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -280,6 +280,7 @@ config DRM_PANEL_ILITEK_ILI9882T
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_DSC_HELPER
 	help
 	  Say Y if you want to enable support for panels based on the
 	  Ilitek ILI9882t controller.
@@ -442,6 +443,15 @@ config DRM_PANEL_LG_SW43408
 	  pixel. It provides a MIPI DSI interface to the host and has a
 	  built-in LED backlight.
 
+config DRM_PANEL_LXD_M9189A
+	tristate "LXD M9189A MIPI-DSI LCD panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y if you want to enable support for the LXD M9189A 4-Lane
+	  1024x600 MIPI DSI panel.
+
 config DRM_PANEL_MAGNACHIP_D53E6EA8966
 	tristate "Magnachip D53E6EA8966 DSI panel"
 	depends on OF && SPI
@@ -464,6 +474,18 @@ config DRM_PANEL_MANTIX_MLAF057WE51
 	  has a resolution of 720x1440 pixels, a built in backlight and touch
 	  controller.
 
+config DRM_PANEL_MOTOROLA_MOT
+	tristate "Atrix 4G and Droid X2 540x960 DSI video mode panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select VIDEOMODE_HELPERS
+	help
+	  Say Y here if you want to enable support for the LCD panel module
+	  for Motorola Atrix 4G or Droid X2. Exact panel vendor and model are
+	  unknown. The panel has a 540x960 resolution and uses 24 bit RGB per
+	  pixel.
+
 config DRM_PANEL_NEC_NL8048HL11
 	tristate "NEC NL8048HL11 RGB panel"
 	depends on GPIOLIB && OF && SPI
@@ -553,6 +575,15 @@ config DRM_PANEL_NOVATEK_NT36672E
 	  LCD panel module. The panel has a resolution of 1080x2408 and uses 24 bit
 	  RGB per pixel.
 
+config DRM_PANEL_NOVATEK_NT37700F
+	tristate "Novatek NT37700F DSI panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Novatek NT37700F DSI
+	  panel module. The panel has a resolution of 1080x2160.
+
 config DRM_PANEL_NOVATEK_NT37801
 	tristate "Novatek NT37801/NT37810 AMOLED DSI panel"
 	depends on OF
@@ -688,6 +719,7 @@ config DRM_PANEL_RENESAS_R61307
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
+	select VIDEOMODE_HELPERS
 	help
 	  Say Y here if you want to enable support for KOE tx13d100vm0eaa
 	  IPS-LCD module with Renesas R69328 IC. The panel has a 1024x768
@@ -701,6 +733,7 @@ config DRM_PANEL_RENESAS_R69328
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
+	select VIDEOMODE_HELPERS
 	help
 	  Say Y here if you want to enable support for JDI dx12d100vm0eaa
 	  IPS-LCD module with Renesas R69328 IC. The panel has a 720x1280
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index aeffaa95666d6..3336a2c0cd864 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -44,7 +44,9 @@ obj-$(CONFIG_DRM_PANEL_LG_LB035Q02) += panel-lg-lb035q02.o
 obj-$(CONFIG_DRM_PANEL_LG_LD070WX3) += panel-lg-ld070wx3.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
 obj-$(CONFIG_DRM_PANEL_LG_SW43408) += panel-lg-sw43408.o
+obj-$(CONFIG_DRM_PANEL_LXD_M9189A) += panel-lxd-m9189a.o
 obj-$(CONFIG_DRM_PANEL_MAGNACHIP_D53E6EA8966) += panel-magnachip-d53e6ea8966.o
+obj-$(CONFIG_DRM_PANEL_MOTOROLA_MOT) += panel-motorola-mot.o
 obj-$(CONFIG_DRM_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3051D) += panel-newvision-nv3051d.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3052C) += panel-newvision-nv3052c.o
@@ -54,6 +56,7 @@ obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35950) += panel-novatek-nt35950.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36523) += panel-novatek-nt36523.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36672A) += panel-novatek-nt36672a.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36672E) += panel-novatek-nt36672e.o
+obj-$(CONFIG_DRM_PANEL_NOVATEK_NT37700F) += panel-novatek-nt37700f.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT37801) += panel-novatek-nt37801.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT39016) += panel-novatek-nt39016.o
 obj-$(CONFIG_DRM_PANEL_MANTIX_MLAF057WE51) += panel-mantix-mlaf057we51.o
diff --git a/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c b/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
index f33d4f8559294..01b4458e55ad9 100644
--- a/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
+++ b/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
@@ -221,6 +221,7 @@ static int boe_th101mb31ig002_prepare(struct drm_panel *panel)
 						      struct boe_th101mb31ig002,
 						      panel);
 	struct device *dev = &ctx->dsi->dev;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
 	int ret;
 
 	ret = regulator_enable(ctx->power);
@@ -233,9 +234,9 @@ static int boe_th101mb31ig002_prepare(struct drm_panel *panel)
 		msleep(ctx->desc->vcioo_to_lp11_delay_ms);
 
 	if (ctx->desc->lp11_before_reset) {
-		ret = mipi_dsi_dcs_nop(ctx->dsi);
-		if (ret)
-			return ret;
+		mipi_dsi_dcs_nop_multi(&dsi_ctx);
+		if (dsi_ctx.accum_err)
+			return dsi_ctx.accum_err;
 	}
 
 	if (ctx->desc->lp11_to_reset_delay_ms)
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 679f4af5246d8..497dcd48f57b9 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -1256,6 +1256,29 @@ static const struct panel_desc boe_nv140fhmn49 = {
 	},
 };
 
+static const struct drm_display_mode friendlyarm_hd702e_mode = {
+	.clock		= 67185,
+	.hdisplay	= 800,
+	.hsync_start	= 800 + 20,
+	.hsync_end	= 800 + 20 + 24,
+	.htotal		= 800 + 20 + 24 + 20,
+	.vdisplay	= 1280,
+	.vsync_start	= 1280 + 4,
+	.vsync_end	= 1280 + 4 + 8,
+	.vtotal		= 1280 + 4 + 8 + 4,
+	.flags		= DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc friendlyarm_hd702e = {
+	.modes = &friendlyarm_hd702e_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width	= 94,
+		.height	= 151,
+	},
+};
+
 static const struct drm_display_mode innolux_n116bca_ea1_mode = {
 	.clock = 76420,
 	.hdisplay = 1366,
@@ -1663,6 +1686,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "boe,nv140fhmn49",
 		.data = &boe_nv140fhmn49,
+	}, {
+		.compatible = "friendlyarm,hd702e",
+		.data = &friendlyarm_hd702e,
 	}, {
 		.compatible = "innolux,n116bca-ea1",
 		.data = &innolux_n116bca_ea1,
@@ -1788,6 +1814,13 @@ static const struct panel_delay delay_200_500_e200 = {
 	.enable = 200,
 };
 
+static const struct panel_delay delay_200_500_e200_d100 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 200,
+	.disable = 100,
+};
+
 static const struct panel_delay delay_200_500_e200_d200 = {
 	.hpd_absent = 200,
 	.unprepare = 500,
@@ -1915,6 +1948,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x89ba, &delay_200_500_e50, "B116XAT04.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8bba, &delay_200_500_e50, "B140UAN08.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xa199, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xa7b3, &delay_200_500_e50, "B140UAN04.4"),
@@ -1987,10 +2021,11 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf2, &delay_200_500_e200, "NV156FHM-N4S"),
-	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200, "NV140WUM-N64"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200_d100, "NV140WUM-N64"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cfa, &delay_200_500_e50, "NV116WHM-A4D"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0d45, &delay_200_500_e80, "NV116WHM-N4B"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0d73, &delay_200_500_e80, "NE140WUM-N6S"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0db3, &delay_200_500_e80, "NV153WUM-N42"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0ddf, &delay_200_500_e80, "NV116WHM-T01"),
 
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1130, &delay_200_500_e50, "N116BGE-EB2"),
@@ -2013,6 +2048,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1160, &delay_200_500_e80_d50, "N116BCJ-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1161, &delay_200_500_e80, "N116BCP-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1163, &delay_200_500_e80_d50, "N116BCJ-EAK"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x117a, &delay_200_500_e80_d50, "N116BCL-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x124c, &delay_200_500_e80_d50, "N122JCA-ENK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x142b, &delay_200_500_e80_d50, "N140HCA-EAC"),
@@ -2026,6 +2062,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80_d50, "N140BGA-EA4"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14e5, &delay_200_500_e80_d50, "N140HGA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1565, &delay_200_500_e80, "N156HCA-EAB"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x156b, &delay_200_500_e80_d50, "N153JCA-ELK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x162b, &delay_200_500_e80_d50, "N160JCE-ELL"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x7402, &delay_200_500_e200_d50, "N116BCA-EAK"),
 
@@ -2044,6 +2081,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x146e, &delay_80_500_e50_d50, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x147c, &delay_200_500_e50_d100, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1519, &delay_200_500_e80_d50, "MNF601BS1-3"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1529, &delay_200_500_e80_d50, "MNF307QS3-2"),
 
 	EDP_PANEL_ENTRY('E', 'T', 'C', 0x0000, &delay_50_500_e200_d200_po2e335, "LP079QX1-SP0V"),
 
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
index 8b2bfb7d36385..5f4e0d82ee673 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
@@ -592,7 +592,7 @@ static int ili9882t_unprepare(struct drm_panel *panel)
 {
 	struct ili9882t *ili = to_ili9882t(panel);
 
-	gpiod_set_value(ili->enable_gpio, 0);
+	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 	usleep_range(1000, 2000);
 	regulator_disable(ili->avee);
 	regulator_disable(ili->avdd);
@@ -608,7 +608,7 @@ static int ili9882t_prepare(struct drm_panel *panel)
 	struct ili9882t *ili = to_ili9882t(panel);
 	int ret;
 
-	gpiod_set_value(ili->enable_gpio, 0);
+	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 	usleep_range(1000, 1500);
 
 	ret = regulator_enable(ili->pp3300);
@@ -638,11 +638,11 @@ static int ili9882t_prepare(struct drm_panel *panel)
 	}
 	usleep_range(1000, 2000);
 
-	gpiod_set_value(ili->enable_gpio, 1);
+	gpiod_set_value_cansleep(ili->enable_gpio, 1);
 	usleep_range(1000, 2000);
-	gpiod_set_value(ili->enable_gpio, 0);
+	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 	msleep(50);
-	gpiod_set_value(ili->enable_gpio, 1);
+	gpiod_set_value_cansleep(ili->enable_gpio, 1);
 	usleep_range(6000, 10000);
 
 	ret = ili->desc->init(ili);
@@ -652,7 +652,7 @@ static int ili9882t_prepare(struct drm_panel *panel)
 	return 0;
 
 poweroff:
-	gpiod_set_value(ili->enable_gpio, 0);
+	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 	regulator_disable(ili->avee);
 poweroffavdd:
 	regulator_disable(ili->avdd);
@@ -793,7 +793,7 @@ static int ili9882t_add(struct ili9882t *ili)
 		return PTR_ERR(ili->enable_gpio);
 	}
 
-	gpiod_set_value(ili->enable_gpio, 0);
+	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 
 	err = of_drm_get_panel_orientation(dev->of_node, &ili->orientation);
 	if (err < 0) {
diff --git a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
index 5386a06fcd085..c33c611e03c00 100644
--- a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
+++ b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
@@ -1366,6 +1366,246 @@ static const struct jadard_panel_desc anbernic_rgds_display_desc = {
 		      MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM,
 };
 
+static int taiguan_xti05101_01a_init_cmds(struct jadard *jadard)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
+
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x74);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xd7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xd7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0xa0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x6a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x2e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x1a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x15);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x39);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x35);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x2b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x2e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x4d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x3c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x36);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x39);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x35);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x2b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x2e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x4d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x3c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x36);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x02);
+
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x52);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x4e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x4c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x53);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x51);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x4f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x4d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2c, 0x13);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0x0d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x6c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x6c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x75);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0xb4);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x6c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x6c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xbb);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x2a);
+
+	jd9365da_switch_page(&dsi_ctx, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0xb3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x61);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x58);
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+
+	mipi_dsi_msleep(&dsi_ctx, 20);
+
+	return dsi_ctx.accum_err;
+};
+
+static const struct jadard_panel_desc taiguan_xti05101_01a_desc = {
+	.mode = {
+		.clock		= (800 + 24 + 24 + 24) * (1280 + 30 + 4 + 8) * 60 / 1000,
+
+		.hdisplay	= 800,
+		.hsync_start	= 800 + 24,
+		.hsync_end	= 800 + 24 + 24,
+		.htotal		= 800 + 24 + 24 + 24,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 30,
+		.vsync_end	= 1280 + 30 + 4,
+		.vtotal		= 1280 + 30 + 4 + 8,
+
+		.width_mm	= 135,
+		.height_mm	= 216,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = taiguan_xti05101_01a_init_cmds,
+	.lp11_before_reset = true,
+	.reset_before_power_off_vcioo = true,
+	.vcioo_to_lp11_delay_ms = 5,
+	.lp11_to_reset_delay_ms = 10,
+	.backlight_off_to_display_off_delay_ms = 3,
+	.display_off_to_enter_sleep_delay_ms = 50,
+	.enter_sleep_to_reset_down_delay_ms = 100,
+};
+
 static int jadard_dsi_probe(struct mipi_dsi_device *dsi)
 {
 	struct device *dev = &dsi->dev;
@@ -1463,6 +1703,10 @@ static const struct of_device_id jadard_of_match[] = {
 		.compatible = "radxa,display-8hd-ad002",
 		.data = &radxa_display_8hd_ad002_desc
 	},
+	{
+		.compatible = "taiguanck,xti05101-01a",
+		.data = &taiguan_xti05101_01a_desc
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, jadard_of_match);
diff --git a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
index 3513e5c4dd8c6..01bd748aecec7 100644
--- a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
+++ b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
@@ -48,34 +48,16 @@ static inline struct jdi_panel *to_jdi_panel(struct drm_panel *panel)
 static int jdi_panel_init(struct jdi_panel *jdi)
 {
 	struct mipi_dsi_device *dsi = jdi->dsi;
-	struct device *dev = &jdi->dsi->dev;
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
 	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
 
-	ret = mipi_dsi_dcs_soft_reset(dsi);
-	if (ret < 0)
-		return ret;
-
-	usleep_range(10000, 20000);
-
-	ret = mipi_dsi_dcs_set_pixel_format(dsi, MIPI_DCS_PIXEL_FMT_24BIT << 4);
-	if (ret < 0) {
-		dev_err(dev, "failed to set pixel format: %d\n", ret);
-		return ret;
-	}
-
-	ret = mipi_dsi_dcs_set_column_address(dsi, 0, jdi->mode->hdisplay - 1);
-	if (ret < 0) {
-		dev_err(dev, "failed to set column address: %d\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_soft_reset_multi(&dsi_ctx);
+	mipi_dsi_usleep_range(&dsi_ctx, 10000, 20000);
 
-	ret = mipi_dsi_dcs_set_page_address(dsi, 0, jdi->mode->vdisplay - 1);
-	if (ret < 0) {
-		dev_err(dev, "failed to set page address: %d\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_set_pixel_format_multi(&dsi_ctx, MIPI_DCS_PIXEL_FMT_24BIT << 4);
+	mipi_dsi_dcs_set_column_address_multi(&dsi_ctx, 0, jdi->mode->hdisplay - 1);
+	mipi_dsi_dcs_set_page_address_multi(&dsi_ctx, 0, jdi->mode->vdisplay - 1);
 
 	/*
 	 * BIT(5) BCTRL = 1 Backlight Control Block On, Brightness registers
@@ -83,88 +65,49 @@ static int jdi_panel_init(struct jdi_panel *jdi)
 	 * BIT(3) BL = 1    Backlight Control On
 	 * BIT(2) DD = 0    Display Dimming is Off
 	 */
-	ret = mipi_dsi_dcs_write(dsi, MIPI_DCS_WRITE_CONTROL_DISPLAY,
-				 (u8[]){ 0x24 }, 1);
-	if (ret < 0) {
-		dev_err(dev, "failed to write control display: %d\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x24);
 
 	/* CABC off */
-	ret = mipi_dsi_dcs_write(dsi, MIPI_DCS_WRITE_POWER_SAVE,
-				 (u8[]){ 0x00 }, 1);
-	if (ret < 0) {
-		dev_err(dev, "failed to set cabc off: %d\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
 
-	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
-	if (ret < 0) {
-		dev_err(dev, "failed to set exit sleep mode: %d\n", ret);
-		return ret;
-	}
-
-	msleep(120);
-
-	ret = mipi_dsi_generic_write(dsi, (u8[]){0xB0, 0x00}, 2);
-	if (ret < 0) {
-		dev_err(dev, "failed to set mcap: %d\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
 
-	mdelay(10);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0x00);
+	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
 	/* Interface setting, video mode */
-	ret = mipi_dsi_generic_write(dsi, (u8[])
-				     {0xB3, 0x26, 0x08, 0x00, 0x20, 0x00}, 6);
-	if (ret < 0) {
-		dev_err(dev, "failed to set display interface setting: %d\n"
-			, ret);
-		return ret;
-	}
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb3, 0x26, 0x08, 0x00, 0x20, 0x00);
+	mipi_dsi_usleep_range(&dsi_ctx, 20000, 21000);
 
-	mdelay(20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0x03);
 
-	ret = mipi_dsi_generic_write(dsi, (u8[]){0xB0, 0x03}, 2);
-	if (ret < 0) {
-		dev_err(dev, "failed to set default values for mcap: %d\n"
-			, ret);
-		return ret;
-	}
-
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static int jdi_panel_on(struct jdi_panel *jdi)
 {
 	struct mipi_dsi_device *dsi = jdi->dsi;
-	struct device *dev = &jdi->dsi->dev;
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
 	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
 
-	ret = mipi_dsi_dcs_set_display_on(dsi);
-	if (ret < 0)
-		dev_err(dev, "failed to set display on: %d\n", ret);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
-	return ret;
+	return dsi_ctx.accum_err;
 }
 
 static void jdi_panel_off(struct jdi_panel *jdi)
 {
 	struct mipi_dsi_device *dsi = jdi->dsi;
-	struct device *dev = &jdi->dsi->dev;
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
 	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
 
-	ret = mipi_dsi_dcs_set_display_off(dsi);
-	if (ret < 0)
-		dev_err(dev, "failed to set display off: %d\n", ret);
-
-	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
-	if (ret < 0)
-		dev_err(dev, "failed to enter sleep mode: %d\n", ret);
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	/* Reset error to continue power-down even if display off failed */
+	dsi_ctx.accum_err = 0;
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
 
 	msleep(100);
 }
diff --git a/drivers/gpu/drm/panel/panel-lxd-m9189a.c b/drivers/gpu/drm/panel/panel-lxd-m9189a.c
new file mode 100644
index 0000000000000..68019e1e43a96
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-lxd-m9189a.c
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generated with linux-mdss-dsi-panel-driver-generator from vendor device tree.
+ * Copyright (c) 2024 Luca Weiss <luca.weiss@fairphone.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+#include <linux/types.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+/* Manufacturer specific DSI commands */
+#define EK79007AD3_GAMMA1		0x80
+#define EK79007AD3_GAMMA2		0x81
+#define EK79007AD3_GAMMA3		0x82
+#define EK79007AD3_GAMMA4		0x83
+#define EK79007AD3_GAMMA5		0x84
+#define EK79007AD3_GAMMA6		0x85
+#define EK79007AD3_GAMMA7		0x86
+#define EK79007AD3_PANEL_CTRL3		0xB2
+
+struct m9189_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct regulator *supply;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *standby_gpio;
+};
+
+static inline struct m9189_panel *to_m9189_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct m9189_panel, panel);
+}
+
+static void m9189_reset(struct m9189_panel *m9189)
+{
+	gpiod_set_value_cansleep(m9189->reset_gpio, 0);
+	msleep(20);
+	gpiod_set_value_cansleep(m9189->reset_gpio, 1);
+	msleep(30);
+	gpiod_set_value_cansleep(m9189->reset_gpio, 0);
+	msleep(55);
+}
+
+static int m9189_on(struct m9189_panel *m9189)
+{
+	struct mipi_dsi_multi_context ctx = { .dsi = m9189->dsi };
+
+	ctx.dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	/* Gamma 2.2 */
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA1, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA2, 0xB8);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA3, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA4, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA5, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA6, 0xD2);
+	mipi_dsi_dcs_write_seq_multi(&ctx, EK79007AD3_GAMMA7, 0x88);
+	mipi_dsi_msleep(&ctx, 50);
+
+	/* 4 Lanes */
+	mipi_dsi_generic_write_multi(&ctx, (u8[]){ EK79007AD3_PANEL_CTRL3, 0x70 }, 2);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 120);
+
+	return ctx.accum_err;
+}
+
+static int m9189_disable(struct drm_panel *panel)
+{
+	struct m9189_panel *m9189 = to_m9189_panel(panel);
+	struct mipi_dsi_multi_context ctx = { .dsi = m9189->dsi };
+
+	ctx.dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 120);
+
+	gpiod_set_value_cansleep(m9189->standby_gpio, 1);
+
+	return ctx.accum_err;
+}
+
+static int m9189_prepare(struct drm_panel *panel)
+{
+	struct m9189_panel *m9189 = to_m9189_panel(panel);
+	struct device *dev = &m9189->dsi->dev;
+	int ret;
+
+	ret = regulator_enable(m9189->supply);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	gpiod_set_value_cansleep(m9189->standby_gpio, 0);
+	msleep(20);
+	m9189_reset(m9189);
+
+	ret = m9189_on(m9189);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(m9189->reset_gpio, 1);
+		regulator_disable(m9189->supply);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int m9189_unprepare(struct drm_panel *panel)
+{
+	struct m9189_panel *m9189 = to_m9189_panel(panel);
+
+	gpiod_set_value_cansleep(m9189->standby_gpio, 1);
+	msleep(50);
+
+	gpiod_set_value_cansleep(m9189->reset_gpio, 1);
+	regulator_disable(m9189->supply);
+
+	return 0;
+}
+
+static const struct drm_display_mode m9189_mode = {
+	.clock = (1024 + 160 + 160 + 10) * (600 + 12 + 23 + 1) * 60 / 1000,
+	.hdisplay = 1024,
+	.hsync_start = 1024 + 160,
+	.hsync_end = 1024 + 160 + 160,
+	.htotal = 1024 + 160 + 160 + 10,
+	.vdisplay = 600,
+	.vsync_start = 600 + 12,
+	.vsync_end = 600 + 12 + 23,
+	.vtotal = 600 + 12 + 23 + 1,
+	.width_mm = 154,
+	.height_mm = 86,
+};
+
+static int m9189_get_modes(struct drm_panel *panel,
+				  struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &m9189_mode);
+}
+
+static const struct drm_panel_funcs m9189_panel_funcs = {
+	.prepare = m9189_prepare,
+	.unprepare = m9189_unprepare,
+	.disable = m9189_disable,
+	.get_modes = m9189_get_modes,
+};
+
+static int lxd_m9189_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct m9189_panel *m9189;
+	int ret;
+
+	m9189 = devm_kzalloc(dev, sizeof(*m9189), GFP_KERNEL);
+	if (!m9189)
+		return -ENOMEM;
+
+	m9189->supply = devm_regulator_get(dev, "power");
+	if (IS_ERR(m9189->supply))
+		return dev_err_probe(dev, PTR_ERR(m9189->supply),
+				     "Failed to get power-supply\n");
+
+	m9189->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(m9189->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(m9189->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	m9189->standby_gpio = devm_gpiod_get(dev, "standby", GPIOD_OUT_LOW);
+	if (IS_ERR(m9189->standby_gpio))
+		return dev_err_probe(dev, PTR_ERR(m9189->standby_gpio),
+				     "Failed to get standby-gpios\n");
+
+	m9189->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, m9189);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST;
+
+	drm_panel_init(&m9189->panel, dev, &m9189_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+	m9189->panel.prepare_prev_first = true;
+
+	ret = drm_panel_of_backlight(&m9189->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	drm_panel_add(&m9189->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
+		drm_panel_remove(&m9189->panel);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void lxd_m9189_remove(struct mipi_dsi_device *dsi)
+{
+	struct m9189_panel *m9189 = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&m9189->panel);
+}
+
+static const struct of_device_id lxd_m9189_of_match[] = {
+	{ .compatible = "lxd,m9189a" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, lxd_m9189_of_match);
+
+static struct mipi_dsi_driver lxd_m9189_driver = {
+	.probe = lxd_m9189_probe,
+	.remove = lxd_m9189_remove,
+	.driver = {
+		.name = "panel-lxd-m9189a",
+		.of_match_table = lxd_m9189_of_match,
+	},
+};
+module_mipi_dsi_driver(lxd_m9189_driver);
+
+MODULE_DESCRIPTION("DRM driver for LXD M9189A MIPI-DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c b/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
index 13352cb4ad774..27e188bb2d7f6 100644
--- a/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
+++ b/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
@@ -168,7 +168,7 @@ static const struct drm_display_mode default_mode_mantix = {
 	.vsync_start = 1440 + 130,
 	.vsync_end   = 1440 + 130 + 8,
 	.vtotal	     = 1440 + 130 + 8 + 106,
-	.clock	     = 85298,
+	.clock	     = 81237,
 	.flags	     = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
 	.width_mm    = 65,
 	.height_mm   = 130,
@@ -183,7 +183,7 @@ static const struct drm_display_mode default_mode_ys = {
 	.vsync_start = 1440 + 175,
 	.vsync_end   = 1440 + 175 + 8,
 	.vtotal	     = 1440 + 175 + 8 + 50,
-	.clock	     = 85298,
+	.clock	     = 80706,
 	.flags	     = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
 	.width_mm    = 65,
 	.height_mm   = 130,
@@ -295,20 +295,10 @@ static int mantix_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static void mantix_shutdown(struct mipi_dsi_device *dsi)
-{
-	struct mantix *ctx = mipi_dsi_get_drvdata(dsi);
-
-	drm_panel_unprepare(&ctx->panel);
-	drm_panel_disable(&ctx->panel);
-}
-
 static void mantix_remove(struct mipi_dsi_device *dsi)
 {
 	struct mantix *ctx = mipi_dsi_get_drvdata(dsi);
 
-	mantix_shutdown(dsi);
-
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
 }
@@ -323,7 +313,6 @@ MODULE_DEVICE_TABLE(of, mantix_of_match);
 static struct mipi_dsi_driver mantix_driver = {
 	.probe	= mantix_probe,
 	.remove = mantix_remove,
-	.shutdown = mantix_shutdown,
 	.driver = {
 		.name = DRV_NAME,
 		.of_match_table = mantix_of_match,
diff --git a/drivers/gpu/drm/panel/panel-motorola-mot.c b/drivers/gpu/drm/panel/panel-motorola-mot.c
new file mode 100644
index 0000000000000..eb1f86c3d704c
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-motorola-mot.c
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/array_size.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+static const struct regulator_bulk_data mot_panel_supplies[] = {
+	{ .supply = "vddio" }, { .supply = "vdd" },
+};
+
+struct mot_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+
+	struct gpio_desc *reset_gpio;
+
+	struct regulator_bulk_data *supplies;
+};
+
+static inline struct mot_panel *to_mot_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct mot_panel, panel);
+}
+
+static void mot_panel_reset(struct mot_panel *priv)
+{
+	gpiod_set_value_cansleep(priv->reset_gpio, 1);
+	usleep_range(50000, 51000);
+	gpiod_set_value_cansleep(priv->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+static void mot_es2(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_generic_write_seq_multi(ctx, 0x55, 0x01);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(ctx);
+	mipi_dsi_msleep(ctx, 120);
+
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf4, 0x00, 0xbb, 0x46, 0x53, 0x0c, 0x49,
+					 0x74, 0x29, 0x12, 0x15, 0x2f, 0x2f, 0x04);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf8, 0x4b, 0x04, 0x10, 0x1a, 0x2c, 0x2c,
+					 0x2c, 0x2c, 0x14, 0x12);
+
+	mipi_dsi_generic_write_seq_multi(ctx, 0xb5, 0x03, 0x7f, 0x00, 0x80, 0xc7, 0x00);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xb7, 0x66, 0xf6, 0x46, 0x9f, 0x90, 0x99,
+					 0xff, 0x80, 0x6d, 0x01);
+
+	/* Gamma R */
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf9, 0x04);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfa, 0x00, 0x2f, 0x30, 0x12, 0x0e, 0x0c,
+					 0x22, 0x27, 0x31, 0x2e, 0x07, 0x0f);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfb, 0x00, 0x2f, 0x30, 0x12, 0x0e, 0x0c,
+					 0x22, 0x27, 0x31, 0x2e, 0x07, 0x0f);
+
+	/* Gamma G */
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf9, 0x02);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfa, 0x00, 0x2f, 0x37, 0x15, 0x15, 0x11,
+					 0x1f, 0x25, 0x2d, 0x2a, 0x05, 0x0f);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfb, 0x00, 0x2f, 0x37, 0x15, 0x15, 0x11,
+					 0x1f, 0x25, 0x2d, 0x2a, 0x05, 0x0f);
+
+	/* Gamma B */
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf9, 0x01);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfa, 0x00, 0x2f, 0x3f, 0x16, 0x1f, 0x15,
+					 0x1f, 0x25, 0x2d, 0x2b, 0x06, 0x0b);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfb, 0x00, 0x2f, 0x3f, 0x16, 0x1f, 0x15,
+					 0x1f, 0x25, 0x2d, 0x2b, 0x06, 0x0b);
+
+	/* Gamma W */
+	mipi_dsi_generic_write_seq_multi(ctx, 0xf9, 0x20);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfa, 0x00, 0x2f, 0x34, 0x15, 0x1a, 0x11,
+					 0x1f, 0x23, 0x2d, 0x29, 0x02, 0x08);
+	mipi_dsi_generic_write_seq_multi(ctx, 0xfb, 0x00, 0x2f, 0x34, 0x15, 0x1a, 0x11,
+					 0x1f, 0x23, 0x2d, 0x29, 0x02, 0x08);
+
+	mipi_dsi_generic_write_seq_multi(ctx, 0x53, 0x2c);
+	mipi_dsi_generic_write_seq_multi(ctx, 0x35, 0x00);
+}
+
+static int mot_panel_prepare(struct drm_panel *panel)
+{
+	struct mot_panel *priv = to_mot_panel(panel);
+	struct mipi_dsi_multi_context ctx = { .dsi = priv->dsi };
+	struct device *dev = panel->dev;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(mot_panel_supplies), priv->supplies);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable power supplies: %d\n", ret);
+		return ret;
+	}
+
+	mot_panel_reset(priv);
+
+	mipi_dsi_generic_write_seq_multi(&ctx, 0xf0, 0x5a, 0x5a);
+	mipi_dsi_generic_write_seq_multi(&ctx, 0xf1, 0x5a, 0x5a);
+	mipi_dsi_generic_write_seq_multi(&ctx, 0xd0, 0x8e);
+
+	mot_es2(&ctx);
+
+	mipi_dsi_dcs_set_display_on_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 20);
+
+	return ctx.accum_err;
+}
+
+static int mot_panel_disable(struct drm_panel *panel)
+{
+	struct mot_panel *priv = to_mot_panel(panel);
+	struct mipi_dsi_multi_context ctx = { .dsi = priv->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&ctx);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 70);
+
+	return ctx.accum_err;
+}
+
+static int mot_panel_unprepare(struct drm_panel *panel)
+{
+	struct mot_panel *priv = to_mot_panel(panel);
+
+	usleep_range(10000, 11000);
+
+	gpiod_set_value_cansleep(priv->reset_gpio, 1);
+	usleep_range(5000, 6000);
+
+	regulator_bulk_disable(ARRAY_SIZE(mot_panel_supplies), priv->supplies);
+
+	return 0;
+}
+
+static const struct drm_display_mode mot_panel_mode = {
+	.clock = (540 + 32 + 32 + 16) * (960 + 12 + 12 + 8) * 60 / 1000,
+	.hdisplay = 540,
+	.hsync_start = 540 + 32,
+	.hsync_end = 540 + 32 + 32,
+	.htotal = 540 + 32 + 32 + 16,
+	.vdisplay = 960,
+	.vsync_start = 960 + 12,
+	.vsync_end = 960 + 12 + 12,
+	.vtotal = 960 + 12 + 12 + 8,
+	.width_mm = 51,
+	.height_mm = 91,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static int mot_panel_get_modes(struct drm_panel *panel,
+			       struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &mot_panel_mode);
+}
+
+static const struct drm_panel_funcs mot_panel_panel_funcs = {
+	.prepare = mot_panel_prepare,
+	.disable = mot_panel_disable,
+	.unprepare = mot_panel_unprepare,
+	.get_modes = mot_panel_get_modes,
+};
+
+static int mot_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct mot_panel *priv;
+	int ret;
+
+	priv = devm_drm_panel_alloc(dev, struct mot_panel, panel,
+				    &mot_panel_panel_funcs,
+				    DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(priv))
+		return PTR_ERR(priv);
+
+	ret = devm_regulator_bulk_get_const(dev, ARRAY_SIZE(mot_panel_supplies),
+					    mot_panel_supplies, &priv->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get supplies\n");
+
+	priv->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+						   GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(priv->reset_gpio),
+				     "failed to get reset gpios\n");
+
+	priv->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, priv);
+
+	dsi->lanes = 2;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_LPM;
+
+	ret = drm_panel_of_backlight(&priv->panel);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get backlight\n");
+
+	drm_panel_add(&priv->panel);
+
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0) {
+		drm_panel_remove(&priv->panel);
+		return dev_err_probe(dev, ret, "failed to attach to DSI host\n");
+	}
+
+	return 0;
+}
+
+static void mot_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct mot_panel *priv = mipi_dsi_get_drvdata(dsi);
+
+	drm_panel_remove(&priv->panel);
+}
+
+static const struct of_device_id mot_panel_of_match[] = {
+	{ .compatible = "motorola,mot-panel" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mot_panel_of_match);
+
+static struct mipi_dsi_driver mot_panel_driver = {
+	.driver = {
+		.name = "panel-motorola-mot",
+		.of_match_table = mot_panel_of_match,
+	},
+	.probe = mot_panel_probe,
+	.remove = mot_panel_remove,
+};
+module_mipi_dsi_driver(mot_panel_driver);
+
+MODULE_AUTHOR("Svyatoslav Ryhel <clamor95@gmail.com>");
+MODULE_DESCRIPTION("Motorola MOT panel driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
index 29e1f6aea4806..7e8b5e0595759 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
@@ -79,70 +79,57 @@ static inline struct nt36672a_panel *to_nt36672a_panel(struct drm_panel *panel)
 	return container_of(panel, struct nt36672a_panel, base);
 }
 
-static int nt36672a_send_cmds(struct drm_panel *panel, const struct nt36672a_panel_cmd *cmds,
-			      int num)
+static void nt36672a_send_cmds(struct mipi_dsi_multi_context *dsi_ctx,
+			       const struct nt36672a_panel_cmd *cmds, int num)
 {
-	struct nt36672a_panel *pinfo = to_nt36672a_panel(panel);
 	unsigned int i;
-	int err;
 
 	for (i = 0; i < num; i++) {
 		const struct nt36672a_panel_cmd *cmd = &cmds[i];
 
-		err = mipi_dsi_dcs_write(pinfo->link, cmd->data[0], cmd->data + 1, 1);
-
-		if (err < 0)
-			return err;
+		/* cmd->data[0] is the DCS command, cmd->data[1] is the parameter */
+		mipi_dsi_dcs_write_buffer_multi(dsi_ctx, cmd->data, sizeof(cmd->data));
 	}
-
-	return 0;
 }
 
-static int nt36672a_panel_power_off(struct drm_panel *panel)
+static void nt36672a_panel_power_off(struct drm_panel *panel)
 {
 	struct nt36672a_panel *pinfo = to_nt36672a_panel(panel);
-	int ret = 0;
+	int ret;
 
 	gpiod_set_value(pinfo->reset_gpio, 1);
 
 	ret = regulator_bulk_disable(ARRAY_SIZE(pinfo->supplies), pinfo->supplies);
 	if (ret)
 		dev_err(panel->dev, "regulator_bulk_disable failed %d\n", ret);
-
-	return ret;
 }
 
 static int nt36672a_panel_unprepare(struct drm_panel *panel)
 {
 	struct nt36672a_panel *pinfo = to_nt36672a_panel(panel);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->link };
 
 	/* send off cmds */
-	ret = nt36672a_send_cmds(panel, pinfo->desc->off_cmds,
-				 pinfo->desc->num_off_cmds);
+	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->off_cmds,
+			   pinfo->desc->num_off_cmds);
 
-	if (ret < 0)
-		dev_err(panel->dev, "failed to send DCS off cmds: %d\n", ret);
-
-	ret = mipi_dsi_dcs_set_display_off(pinfo->link);
-	if (ret < 0)
-		dev_err(panel->dev, "set_display_off cmd failed ret = %d\n", ret);
+	/* Reset error to continue with display off even if send_cmds failed */
+	dsi_ctx.accum_err = 0;
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	/* Reset error to continue power-down even if display off failed */
+	dsi_ctx.accum_err = 0;
 
 	/* 120ms delay required here as per DCS spec */
 	msleep(120);
 
-	ret = mipi_dsi_dcs_enter_sleep_mode(pinfo->link);
-	if (ret < 0)
-		dev_err(panel->dev, "enter_sleep cmd failed ret = %d\n", ret);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
 
 	/* 0x3C = 60ms delay */
 	msleep(60);
 
-	ret = nt36672a_panel_power_off(panel);
-	if (ret < 0)
-		dev_err(panel->dev, "power_off failed ret = %d\n", ret);
+	nt36672a_panel_power_off(panel);
 
-	return ret;
+	return 0;
 }
 
 static int nt36672a_panel_power_on(struct nt36672a_panel *pinfo)
@@ -170,52 +157,31 @@ static int nt36672a_panel_power_on(struct nt36672a_panel *pinfo)
 static int nt36672a_panel_prepare(struct drm_panel *panel)
 {
 	struct nt36672a_panel *pinfo = to_nt36672a_panel(panel);
-	int err;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->link };
 
-	err = nt36672a_panel_power_on(pinfo);
-	if (err < 0)
-		goto poweroff;
+	dsi_ctx.accum_err = nt36672a_panel_power_on(pinfo);
 
 	/* send first part of init cmds */
-	err = nt36672a_send_cmds(panel, pinfo->desc->on_cmds_1,
-				 pinfo->desc->num_on_cmds_1);
+	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->on_cmds_1,
+			   pinfo->desc->num_on_cmds_1);
 
-	if (err < 0) {
-		dev_err(panel->dev, "failed to send DCS Init 1st Code: %d\n", err);
-		goto poweroff;
-	}
-
-	err = mipi_dsi_dcs_exit_sleep_mode(pinfo->link);
-	if (err < 0) {
-		dev_err(panel->dev, "failed to exit sleep mode: %d\n", err);
-		goto poweroff;
-	}
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
 
 	/* 0x46 = 70 ms delay */
-	msleep(70);
+	mipi_dsi_msleep(&dsi_ctx, 70);
 
-	err = mipi_dsi_dcs_set_display_on(pinfo->link);
-	if (err < 0) {
-		dev_err(panel->dev, "failed to Set Display ON: %d\n", err);
-		goto poweroff;
-	}
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
 	/* Send rest of the init cmds */
-	err = nt36672a_send_cmds(panel, pinfo->desc->on_cmds_2,
-				 pinfo->desc->num_on_cmds_2);
+	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->on_cmds_2,
+			   pinfo->desc->num_on_cmds_2);
 
-	if (err < 0) {
-		dev_err(panel->dev, "failed to send DCS Init 2nd Code: %d\n", err);
-		goto poweroff;
-	}
+	mipi_dsi_msleep(&dsi_ctx, 120);
 
-	msleep(120);
+	if (dsi_ctx.accum_err < 0)
+		gpiod_set_value(pinfo->reset_gpio, 0);
 
-	return 0;
-
-poweroff:
-	gpiod_set_value(pinfo->reset_gpio, 0);
-	return err;
+	return dsi_ctx.accum_err;
 }
 
 static int nt36672a_panel_get_modes(struct drm_panel *panel,
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt37700f.c b/drivers/gpu/drm/panel/panel-novatek-nt37700f.c
new file mode 100644
index 0000000000000..74f46a268c0f6
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-novatek-nt37700f.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2024, The Linux Foundation. All rights reserved.
+ * Generated with linux-mdss-dsi-panel-driver-generator from vendor device tree:
+ *   Copyright (c) 2013, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/of.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct nt37700f_tianma {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct regulator *supply;
+	struct gpio_desc *reset_gpio;
+};
+
+static inline
+struct nt37700f_tianma *to_nt37700f_tianma(struct drm_panel *panel)
+{
+	return container_of(panel, struct nt37700f_tianma, panel);
+}
+
+static void nt37700f_tianma_reset(struct nt37700f_tianma *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(1000, 2000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+#define nt37700f_tianma_panel_switch_page(ctx, page) \
+	mipi_dsi_dcs_write_seq_multi((ctx), 0xf0, 0x55, 0xaa, 0x52, 0x08, (page))
+
+static int nt37700f_tianma_on(struct nt37700f_tianma *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xc0, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xca, 0x52);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb5, 0x2b, 0x1a);
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xcd, 0x04, 0x82);
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xcc, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x55);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf6, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf6, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x81);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf3, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf3, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x90, 0x01);
+
+	mipi_dsi_dcs_set_column_address_multi(&dsi_ctx, 0x0000, 1080 - 1);
+	mipi_dsi_dcs_set_page_address_multi(&dsi_ctx, 0x0000, 2160 - 1);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x20);
+	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xc0, 0x56);
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xcd, 0x00);
+	nt37700f_tianma_panel_switch_page(&dsi_ctx, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xd0, 0x11, 0x64);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb1, 0x20);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static int nt37700f_tianma_disable(struct drm_panel *panel)
+{
+	struct nt37700f_tianma *ctx = to_nt37700f_tianma(panel);
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 100);
+
+	return dsi_ctx.accum_err;
+}
+
+static int nt37700f_tianma_prepare(struct drm_panel *panel)
+{
+	struct nt37700f_tianma *ctx = to_nt37700f_tianma(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = regulator_enable(ctx->supply);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable power supply: %d\n", ret);
+		return ret;
+	}
+
+	nt37700f_tianma_reset(ctx);
+
+	ret = nt37700f_tianma_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int nt37700f_tianma_unprepare(struct drm_panel *panel)
+{
+	struct nt37700f_tianma *ctx = to_nt37700f_tianma(panel);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_disable(ctx->supply);
+
+	return 0;
+}
+
+static const struct drm_display_mode nt37700f_tianma_mode = {
+	.clock = (1080 + 32 + 32 + 98) * (2160 + 32 + 4 + 98) * 60 / 1000,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 32,
+	.hsync_end = 1080 + 32 + 32,
+	.htotal = 1080 + 32 + 32 + 98,
+	.vdisplay = 2160,
+	.vsync_start = 2160 + 32,
+	.vsync_end = 2160 + 32 + 4,
+	.vtotal = 2160 + 32 + 4 + 98,
+	.width_mm = 69,
+	.height_mm = 137,
+	.type = DRM_MODE_TYPE_DRIVER,
+};
+
+static int nt37700f_tianma_get_modes(struct drm_panel *panel,
+				     struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &nt37700f_tianma_mode);
+}
+
+static const struct drm_panel_funcs nt37700f_tianma_panel_funcs = {
+	.prepare = nt37700f_tianma_prepare,
+	.unprepare = nt37700f_tianma_unprepare,
+	.disable = nt37700f_tianma_disable,
+	.get_modes = nt37700f_tianma_get_modes,
+};
+
+static int nt37700f_tianma_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return 0;
+}
+
+static int nt37700f_tianma_bl_get_brightness(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_get_display_brightness_large(dsi, &brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return brightness;
+}
+
+static const struct backlight_ops nt37700f_tianma_bl_ops = {
+	.update_status = nt37700f_tianma_bl_update_status,
+	.get_brightness = nt37700f_tianma_bl_get_brightness,
+};
+
+static struct backlight_device *
+nt37700f_tianma_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 2047,
+		.max_brightness = 2047,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &nt37700f_tianma_bl_ops, &props);
+}
+
+static int nt37700f_tianma_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct nt37700f_tianma *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->supply = devm_regulator_get(dev, "power");
+	if (IS_ERR(ctx->supply))
+		return dev_err_probe(dev, PTR_ERR(ctx->supply),
+				     "Failed to get power-supply\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM;
+
+	drm_panel_init(&ctx->panel, dev, &nt37700f_tianma_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+	ctx->panel.prepare_prev_first = true;
+
+	ctx->panel.backlight = nt37700f_tianma_create_backlight(dsi);
+	if (IS_ERR(ctx->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+				     "Failed to create backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		drm_panel_remove(&ctx->panel);
+		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
+	}
+
+	return 0;
+}
+
+static void nt37700f_tianma_remove(struct mipi_dsi_device *dsi)
+{
+	struct nt37700f_tianma *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id nt37700f_tianma_of_match[] = {
+	{ .compatible = "novatek,nt37700f" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, nt37700f_tianma_of_match);
+
+static struct mipi_dsi_driver nt37700f_tianma_driver = {
+	.probe = nt37700f_tianma_probe,
+	.remove = nt37700f_tianma_remove,
+	.driver = {
+		.name = "panel-novatek-nt37700f",
+		.of_match_table = nt37700f_tianma_of_match,
+	},
+};
+module_mipi_dsi_driver(nt37700f_tianma_driver);
+
+MODULE_DESCRIPTION("DRM driver for nt37700f cmd mode dsi tianma panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-renesas-r61307.c b/drivers/gpu/drm/panel/panel-renesas-r61307.c
index 319415194839a..d8185cc1b5d65 100644
--- a/drivers/gpu/drm/panel/panel-renesas-r61307.c
+++ b/drivers/gpu/drm/panel/panel-renesas-r61307.c
@@ -14,6 +14,7 @@
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
 #include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
 
 #define R61307_MACP		0xb0 /* Manufacturer CMD Protect */
 #define   R61307_MACP_ON	0x03
@@ -34,8 +35,6 @@ struct renesas_r61307 {
 
 	struct gpio_desc *reset_gpio;
 
-	bool prepared;
-
 	bool dig_cont_adj;
 	bool inversion;
 	u32 gamma;
@@ -91,9 +90,6 @@ static int renesas_r61307_prepare(struct drm_panel *panel)
 	struct device *dev = &priv->dsi->dev;
 	int ret;
 
-	if (priv->prepared)
-		return 0;
-
 	ret = regulator_enable(priv->vcc_supply);
 	if (ret) {
 		dev_err(dev, "failed to enable vcc power supply\n");
@@ -112,7 +108,6 @@ static int renesas_r61307_prepare(struct drm_panel *panel)
 
 	renesas_r61307_reset(priv);
 
-	priv->prepared = true;
 	return 0;
 }
 
@@ -155,7 +150,7 @@ static int renesas_r61307_enable(struct drm_panel *panel)
 	mipi_dsi_dcs_set_display_on_multi(&ctx);
 	mipi_dsi_msleep(&ctx, 50);
 
-	return 0;
+	return ctx.accum_err;
 }
 
 static int renesas_r61307_disable(struct drm_panel *panel)
@@ -167,16 +162,13 @@ static int renesas_r61307_disable(struct drm_panel *panel)
 	mipi_dsi_msleep(&ctx, 100);
 	mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
 
-	return 0;
+	return ctx.accum_err;
 }
 
 static int renesas_r61307_unprepare(struct drm_panel *panel)
 {
 	struct renesas_r61307 *priv = to_renesas_r61307(panel);
 
-	if (!priv->prepared)
-		return 0;
-
 	usleep_range(10000, 11000);
 
 	gpiod_set_value_cansleep(priv->reset_gpio, 1);
@@ -186,7 +178,6 @@ static int renesas_r61307_unprepare(struct drm_panel *panel)
 	usleep_range(2000, 3000);
 	regulator_disable(priv->vcc_supply);
 
-	priv->prepared = false;
 	return 0;
 }
 
@@ -202,25 +193,13 @@ static const struct drm_display_mode renesas_r61307_mode = {
 	.vtotal = 1024 + 24 + 8 + 2,
 	.width_mm = 76,
 	.height_mm = 101,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
 };
 
 static int renesas_r61307_get_modes(struct drm_panel *panel,
 				    struct drm_connector *connector)
 {
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, &renesas_r61307_mode);
-	if (!mode)
-		return -ENOMEM;
-
-	drm_mode_set_name(mode);
-
-	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
-	connector->display_info.width_mm = mode->width_mm;
-	connector->display_info.height_mm = mode->height_mm;
-	drm_mode_probed_add(connector, mode);
-
-	return 1;
+	return drm_connector_helper_get_modes_fixed(connector, &renesas_r61307_mode);
 }
 
 static const struct drm_panel_funcs renesas_r61307_panel_funcs = {
@@ -259,7 +238,7 @@ static int renesas_r61307_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(priv->reset_gpio),
 				     "Failed to get reset gpios\n");
 
-	if (device_property_read_bool(dev, "renesas,inversion"))
+	if (device_property_read_bool(dev, "renesas,column-inversion"))
 		priv->inversion = true;
 
 	if (device_property_read_bool(dev, "renesas,contrast"))
@@ -282,7 +261,7 @@ static int renesas_r61307_probe(struct mipi_dsi_device *dsi)
 
 	drm_panel_add(&priv->panel);
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret) {
 		drm_panel_remove(&priv->panel);
 		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
@@ -294,11 +273,6 @@ static int renesas_r61307_probe(struct mipi_dsi_device *dsi)
 static void renesas_r61307_remove(struct mipi_dsi_device *dsi)
 {
 	struct renesas_r61307 *priv = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&priv->panel);
 }
diff --git a/drivers/gpu/drm/panel/panel-renesas-r69328.c b/drivers/gpu/drm/panel/panel-renesas-r69328.c
index 46287ab04c30e..bfe2787f8f534 100644
--- a/drivers/gpu/drm/panel/panel-renesas-r69328.c
+++ b/drivers/gpu/drm/panel/panel-renesas-r69328.c
@@ -14,6 +14,7 @@
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
 #include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
 
 #define R69328_MACP		0xb0 /* Manufacturer Access CMD Protect */
 #define   R69328_MACP_ON	0x03
@@ -32,8 +33,6 @@ struct renesas_r69328 {
 	struct regulator *vdd_supply;
 	struct regulator *vddio_supply;
 	struct gpio_desc *reset_gpio;
-
-	bool prepared;
 };
 
 static inline struct renesas_r69328 *to_renesas_r69328(struct drm_panel *panel)
@@ -55,9 +54,6 @@ static int renesas_r69328_prepare(struct drm_panel *panel)
 	struct device *dev = &priv->dsi->dev;
 	int ret;
 
-	if (priv->prepared)
-		return 0;
-
 	ret = regulator_enable(priv->vdd_supply);
 	if (ret) {
 		dev_err(dev, "failed to enable vdd power supply\n");
@@ -76,7 +72,6 @@ static int renesas_r69328_prepare(struct drm_panel *panel)
 
 	renesas_r69328_reset(priv);
 
-	priv->prepared = true;
 	return 0;
 }
 
@@ -122,7 +117,7 @@ static int renesas_r69328_enable(struct drm_panel *panel)
 	mipi_dsi_dcs_set_display_on_multi(&ctx);
 	mipi_dsi_msleep(&ctx, 50);
 
-	return 0;
+	return ctx.accum_err;
 }
 
 static int renesas_r69328_disable(struct drm_panel *panel)
@@ -134,16 +129,13 @@ static int renesas_r69328_disable(struct drm_panel *panel)
 	mipi_dsi_msleep(&ctx, 60);
 	mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
 
-	return 0;
+	return ctx.accum_err;
 }
 
 static int renesas_r69328_unprepare(struct drm_panel *panel)
 {
 	struct renesas_r69328 *priv = to_renesas_r69328(panel);
 
-	if (!priv->prepared)
-		return 0;
-
 	gpiod_set_value_cansleep(priv->reset_gpio, 1);
 
 	usleep_range(5000, 6000);
@@ -151,7 +143,6 @@ static int renesas_r69328_unprepare(struct drm_panel *panel)
 	regulator_disable(priv->vddio_supply);
 	regulator_disable(priv->vdd_supply);
 
-	priv->prepared = false;
 	return 0;
 }
 
@@ -167,25 +158,13 @@ static const struct drm_display_mode renesas_r69328_mode = {
 	.vtotal = 1280 + 6 + 3 + 1,
 	.width_mm = 59,
 	.height_mm = 105,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
 };
 
 static int renesas_r69328_get_modes(struct drm_panel *panel,
 				    struct drm_connector *connector)
 {
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, &renesas_r69328_mode);
-	if (!mode)
-		return -ENOMEM;
-
-	drm_mode_set_name(mode);
-
-	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
-	connector->display_info.width_mm = mode->width_mm;
-	connector->display_info.height_mm = mode->height_mm;
-	drm_mode_probed_add(connector, mode);
-
-	return 1;
+	return drm_connector_helper_get_modes_fixed(connector, &renesas_r69328_mode);
 }
 
 static const struct drm_panel_funcs renesas_r69328_panel_funcs = {
@@ -238,7 +217,7 @@ static int renesas_r69328_probe(struct mipi_dsi_device *dsi)
 
 	drm_panel_add(&priv->panel);
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret) {
 		drm_panel_remove(&priv->panel);
 		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
@@ -250,11 +229,6 @@ static int renesas_r69328_probe(struct mipi_dsi_device *dsi)
 static void renesas_r69328_remove(struct mipi_dsi_device *dsi)
 {
 	struct renesas_r69328 *priv = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&priv->panel);
 }
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 91ab280869bac..e1d15434ea54d 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -2096,6 +2096,35 @@ static const struct panel_desc edt_et057090dhu = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct display_timing edt_et057023udba_timing = {
+	.pixelclock = { 23200000, 24190000, 39640000 },
+	.hactive = { 640, 640, 640 },
+	.hfront_porch = { 20, 40, 200 },
+	.hback_porch = { 87, 40, 1 },
+	.hsync_len = { 1, 48, 87 },
+	.vactive = { 480, 480, 480 },
+	.vfront_porch = { 5, 13, 200 },
+	.vback_porch = { 31, 31, 29 },
+	.vsync_len = { 1, 1, 3 },
+	.flags = DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_HSYNC_LOW |
+		 DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
+		 DISPLAY_FLAGS_SYNC_POSEDGE,
+};
+
+static const struct panel_desc edt_et057023udba = {
+	.timings = &edt_et057023udba_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 115,
+		.height = 86,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct drm_display_mode edt_etm0700g0dh6_mode = {
 	.clock = 33260,
 	.hdisplay = 800,
@@ -2359,28 +2388,6 @@ static const struct panel_desc frida_frd350h54004 = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
-static const struct drm_display_mode friendlyarm_hd702e_mode = {
-	.clock		= 67185,
-	.hdisplay	= 800,
-	.hsync_start	= 800 + 20,
-	.hsync_end	= 800 + 20 + 24,
-	.htotal		= 800 + 20 + 24 + 20,
-	.vdisplay	= 1280,
-	.vsync_start	= 1280 + 4,
-	.vsync_end	= 1280 + 4 + 8,
-	.vtotal		= 1280 + 4 + 8 + 4,
-	.flags		= DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
-};
-
-static const struct panel_desc friendlyarm_hd702e = {
-	.modes = &friendlyarm_hd702e_mode,
-	.num_modes = 1,
-	.size = {
-		.width	= 94,
-		.height	= 151,
-	},
-};
-
 static const struct drm_display_mode giantplus_gpg482739qs5_mode = {
 	.clock = 9000,
 	.hdisplay = 480,
@@ -4068,6 +4075,33 @@ static const struct panel_desc powertip_ph800480t013_idf02  = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct drm_display_mode powertip_ph800480t032_zhc19_mode = {
+	.clock = 27200,
+	.hdisplay = 800,
+	.hsync_start = 800 + 52,
+	.hsync_end = 800 + 52 + 2,
+	.htotal = 800 + 52 + 2 + 44,
+	.vdisplay = 480,
+	.vsync_start = 480 + 7,
+	.vsync_end = 480 + 7 + 2,
+	.vtotal = 480 + 7 + 2 + 2,
+};
+
+static const struct panel_desc powertip_ph800480t032_zhc19 = {
+	.modes = &powertip_ph800480t032_zhc19_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 152,
+		.height = 91,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH |
+		DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE |
+		DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct drm_display_mode primeview_pm070wl4_mode = {
 	.clock = 32000,
 	.hdisplay = 800,
@@ -4998,6 +5032,33 @@ static const struct panel_desc vl050_8048nt_c01 = {
 	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE,
 };
 
+static const struct drm_display_mode waveshare_133inch_mode = {
+	.clock = 148500,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 88,
+	.hsync_end = 1920 + 88 + 44,
+	.htotal = 1920 + 88 + 44 + 148,
+	.vdisplay = 1080,
+	.vsync_start = 1080 + 4,
+	.vsync_end = 1080 + 4 + 5,
+	.vtotal = 1080 + 4 + 5 + 36,
+	.flags = DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_PHSYNC,
+};
+
+static const struct panel_desc waveshare_133inch = {
+	.modes = &waveshare_133inch_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 293,
+		.height = 163,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE,
+};
+
 static const struct drm_display_mode winstar_wf35ltiacd_mode = {
 	.clock = 6410,
 	.hdisplay = 320,
@@ -5247,6 +5308,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "edt,etm0430g0dh6",
 		.data = &edt_etm0430g0dh6,
+	}, {
+		.compatible = "edt,et057023udba",
+		.data = &edt_et057023udba,
 	}, {
 		.compatible = "edt,et057090dhu",
 		.data = &edt_et057090dhu,
@@ -5286,9 +5350,6 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "frida,frd350h54004",
 		.data = &frida_frd350h54004,
-	}, {
-		.compatible = "friendlyarm,hd702e",
-		.data = &friendlyarm_hd702e,
 	}, {
 		.compatible = "giantplus,gpg482739qs5",
 		.data = &giantplus_gpg482739qs5
@@ -5481,6 +5542,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "powertip,ph800480t013-idf02",
 		.data = &powertip_ph800480t013_idf02,
+	}, {
+		.compatible = "powertip,ph800480t032-zhc19",
+		.data = &powertip_ph800480t032_zhc19,
 	}, {
 		.compatible = "primeview,pm070wl4",
 		.data = &primeview_pm070wl4,
@@ -5598,6 +5662,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "vxt,vl050-8048nt-c01",
 		.data = &vl050_8048nt_c01,
+	}, {
+		.compatible = "waveshare,13.3inch-panel",
+		.data = &waveshare_133inch,
 	}, {
 		.compatible = "winstar,wf35ltiacd",
 		.data = &winstar_wf35ltiacd,
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
index 822633da741e4..3a7fce4288987 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -702,7 +702,7 @@ static void panfrost_gem_debugfs_bo_print(struct panfrost_gem_object *bo,
 		   resident_size,
 		   drm_vma_node_start(&bo->base.base.vma_node));
 
-	if (bo->base.base.import_attach)
+	if (drm_gem_is_imported(&bo->base.base))
 		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_IMPORTED;
 	if (bo->base.base.dma_buf)
 		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_EXPORTED;
diff --git a/drivers/gpu/drm/panthor/Makefile b/drivers/gpu/drm/panthor/Makefile
index 753a32c446df0..dd15d52a88bac 100644
--- a/drivers/gpu/drm/panthor/Makefile
+++ b/drivers/gpu/drm/panthor/Makefile
@@ -14,3 +14,5 @@ panthor-y := \
 	panthor_sched.o
 
 obj-$(CONFIG_DRM_PANTHOR) += panthor.o
+
+CFLAGS_panthor_gpu.o := -I$(src)
diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h
index f35e52b9546ac..b6696f73a5367 100644
--- a/drivers/gpu/drm/panthor/panthor_device.h
+++ b/drivers/gpu/drm/panthor/panthor_device.h
@@ -61,6 +61,17 @@ enum panthor_device_pm_state {
 	PANTHOR_DEVICE_PM_STATE_SUSPENDING,
 };
 
+enum panthor_irq_state {
+	/** @PANTHOR_IRQ_STATE_ACTIVE: IRQ is active and ready to process events. */
+	PANTHOR_IRQ_STATE_ACTIVE = 0,
+	/** @PANTHOR_IRQ_STATE_PROCESSING: IRQ is currently processing events. */
+	PANTHOR_IRQ_STATE_PROCESSING,
+	/** @PANTHOR_IRQ_STATE_SUSPENDED: IRQ is suspended. */
+	PANTHOR_IRQ_STATE_SUSPENDED,
+	/** @PANTHOR_IRQ_STATE_SUSPENDING: IRQ is being suspended. */
+	PANTHOR_IRQ_STATE_SUSPENDING,
+};
+
 /**
  * struct panthor_irq - IRQ data
  *
@@ -73,11 +84,21 @@ struct panthor_irq {
 	/** @irq: IRQ number. */
 	int irq;
 
-	/** @mask: Current mask being applied to xxx_INT_MASK. */
+	/** @mask: Values to write to xxx_INT_MASK if active. */
 	u32 mask;
 
-	/** @suspended: Set to true when the IRQ is suspended. */
-	atomic_t suspended;
+	/**
+	 * @mask_lock: protects modifications to _INT_MASK and @mask.
+	 *
+	 * In paths where _INT_MASK is updated based on a state
+	 * transition/check, it's crucial for the state update/check to be
+	 * inside the locked section, otherwise it introduces a race window
+	 * leading to potential _INT_MASK inconsistencies.
+	 */
+	spinlock_t mask_lock;
+
+	/** @state: one of &enum panthor_irq_state reflecting the current state. */
+	atomic_t state;
 };
 
 /**
@@ -409,12 +430,18 @@ static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data)
 {												\
 	struct panthor_irq *pirq = data;							\
 	struct panthor_device *ptdev = pirq->ptdev;						\
+	enum panthor_irq_state old_state;							\
 												\
-	if (atomic_read(&pirq->suspended))							\
-		return IRQ_NONE;								\
 	if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT))					\
 		return IRQ_NONE;								\
 												\
+	guard(spinlock_irqsave)(&pirq->mask_lock);						\
+	old_state = atomic_cmpxchg(&pirq->state,						\
+				   PANTHOR_IRQ_STATE_ACTIVE,					\
+				   PANTHOR_IRQ_STATE_PROCESSING);				\
+	if (old_state != PANTHOR_IRQ_STATE_ACTIVE)						\
+		return IRQ_NONE;								\
+												\
 	gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0);						\
 	return IRQ_WAKE_THREAD;									\
 }												\
@@ -426,6 +453,14 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da
 	irqreturn_t ret = IRQ_NONE;								\
 												\
 	while (true) {										\
+		/* It's safe to access pirq->mask without the lock held here. If a new		\
+		 * event gets added to the mask and the corresponding IRQ is pending,		\
+		 * we'll process it right away instead of adding an extra raw -> threaded	\
+		 * round trip. If an event is removed and the status bit is set, it will	\
+		 * be ignored, just like it would have been if the mask had been adjusted	\
+		 * right before the HW event kicks in. TLDR; it's all expected races we're	\
+		 * covered for.									\
+		 */										\
 		u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask;	\
 												\
 		if (!status)									\
@@ -435,26 +470,36 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da
 		ret = IRQ_HANDLED;								\
 	}											\
 												\
-	if (!atomic_read(&pirq->suspended))							\
-		gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
+	scoped_guard(spinlock_irqsave, &pirq->mask_lock) {					\
+		enum panthor_irq_state old_state;						\
+												\
+		old_state = atomic_cmpxchg(&pirq->state,					\
+					   PANTHOR_IRQ_STATE_PROCESSING,			\
+					   PANTHOR_IRQ_STATE_ACTIVE);				\
+		if (old_state == PANTHOR_IRQ_STATE_PROCESSING)					\
+			gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask);		\
+	}											\
 												\
 	return ret;										\
 }												\
 												\
 static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq)			\
 {												\
-	pirq->mask = 0;										\
-	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0);					\
+	scoped_guard(spinlock_irqsave, &pirq->mask_lock) {					\
+		atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING);				\
+		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0);				\
+	}											\
 	synchronize_irq(pirq->irq);								\
-	atomic_set(&pirq->suspended, true);							\
+	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED);					\
 }												\
 												\
-static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask)	\
+static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq)			\
 {												\
-	atomic_set(&pirq->suspended, false);							\
-	pirq->mask = mask;									\
-	gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask);				\
-	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask);				\
+	guard(spinlock_irqsave)(&pirq->mask_lock);						\
+												\
+	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE);					\
+	gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, pirq->mask);				\
+	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);				\
 }												\
 												\
 static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev,			\
@@ -463,13 +508,43 @@ static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev,			\
 {												\
 	pirq->ptdev = ptdev;									\
 	pirq->irq = irq;									\
-	panthor_ ## __name ## _irq_resume(pirq, mask);						\
+	pirq->mask = mask;									\
+	spin_lock_init(&pirq->mask_lock);							\
+	panthor_ ## __name ## _irq_resume(pirq);						\
 												\
 	return devm_request_threaded_irq(ptdev->base.dev, irq,					\
 					 panthor_ ## __name ## _irq_raw_handler,		\
 					 panthor_ ## __name ## _irq_threaded_handler,		\
 					 IRQF_SHARED, KBUILD_MODNAME "-" # __name,		\
 					 pirq);							\
+}												\
+												\
+static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq *pirq, u32 mask)	\
+{												\
+	guard(spinlock_irqsave)(&pirq->mask_lock);						\
+	pirq->mask |= mask;									\
+												\
+	/* The only situation where we need to write the new mask is if the IRQ is active.	\
+	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()	\
+	 * on the PROCESSING -> ACTIVE transition.						\
+	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
+	 */											\
+	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
+		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
+}												\
+												\
+static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\
+{												\
+	guard(spinlock_irqsave)(&pirq->mask_lock);						\
+	pirq->mask &= ~mask;									\
+												\
+	/* The only situation where we need to write the new mask is if the IRQ is active.	\
+	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()	\
+	 * on the PROCESSING -> ACTIVE transition.						\
+	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
+	 */											\
+	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
+		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
 }
 
 extern struct workqueue_struct *panthor_cleanup_wq;
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
index a64ec8756bed2..5a904ca64525d 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.c
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -26,6 +26,7 @@
 #include "panthor_mmu.h"
 #include "panthor_regs.h"
 #include "panthor_sched.h"
+#include "panthor_trace.h"
 
 #define CSF_FW_NAME "mali_csffw.bin"
 
@@ -1060,6 +1061,12 @@ static void panthor_fw_init_global_iface(struct panthor_device *ptdev)
 
 static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
 {
+	u32 duration;
+	u64 start = 0;
+
+	if (tracepoint_enabled(gpu_job_irq))
+		start = ktime_get_ns();
+
 	gpu_write(ptdev, JOB_INT_CLEAR, status);
 
 	if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF))
@@ -1072,6 +1079,12 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
 		return;
 
 	panthor_sched_report_fw_events(ptdev, status);
+
+	if (tracepoint_enabled(gpu_job_irq) && start) {
+		if (check_sub_overflow(ktime_get_ns(), start, &duration))
+			duration = U32_MAX;
+		trace_gpu_job_irq(ptdev->base.dev, status, duration);
+	}
 }
 PANTHOR_IRQ_HANDLER(job, JOB, panthor_job_irq_handler);
 
@@ -1080,7 +1093,8 @@ static int panthor_fw_start(struct panthor_device *ptdev)
 	bool timedout = false;
 
 	ptdev->fw->booted = false;
-	panthor_job_irq_resume(&ptdev->fw->irq, ~0);
+	panthor_job_irq_enable_events(&ptdev->fw->irq, ~0);
+	panthor_job_irq_resume(&ptdev->fw->irq);
 	gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_AUTO);
 
 	if (!wait_event_timeout(ptdev->fw->req_waitqueue,
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
index 4b4575dd6e908..6d14b0269574a 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.c
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -666,7 +666,7 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
 		   resident_size,
 		   drm_vma_node_start(&bo->base.base.vma_node));
 
-	if (bo->base.base.import_attach)
+	if (drm_gem_is_imported(&bo->base.base))
 		gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_IMPORTED;
 	if (bo->base.base.dma_buf)
 		gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_EXPORTED;
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c
index 057e167468d03..2ab444ee8c710 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.c
+++ b/drivers/gpu/drm/panthor/panthor_gpu.c
@@ -22,6 +22,9 @@
 #include "panthor_hw.h"
 #include "panthor_regs.h"
 
+#define CREATE_TRACE_POINTS
+#include "panthor_trace.h"
+
 /**
  * struct panthor_gpu - GPU block management data.
  */
@@ -48,6 +51,9 @@ struct panthor_gpu {
 	 GPU_IRQ_RESET_COMPLETED | \
 	 GPU_IRQ_CLEAN_CACHES_COMPLETED)
 
+#define GPU_POWER_INTERRUPTS_MASK	\
+	(GPU_IRQ_POWER_CHANGED | GPU_IRQ_POWER_CHANGED_ALL)
+
 static void panthor_gpu_coherency_set(struct panthor_device *ptdev)
 {
 	gpu_write(ptdev, GPU_COHERENCY_PROTOCOL,
@@ -80,6 +86,12 @@ static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
 {
 	gpu_write(ptdev, GPU_INT_CLEAR, status);
 
+	if (tracepoint_enabled(gpu_power_status) && (status & GPU_POWER_INTERRUPTS_MASK))
+		trace_gpu_power_status(ptdev->base.dev,
+				       gpu_read64(ptdev, SHADER_READY),
+				       gpu_read64(ptdev, TILER_READY),
+				       gpu_read64(ptdev, L2_READY));
+
 	if (status & GPU_IRQ_FAULT) {
 		u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS);
 		u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR);
@@ -157,6 +169,22 @@ int panthor_gpu_init(struct panthor_device *ptdev)
 	return 0;
 }
 
+int panthor_gpu_power_changed_on(struct panthor_device *ptdev)
+{
+	guard(pm_runtime_active)(ptdev->base.dev);
+
+	panthor_gpu_irq_enable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
+
+	return 0;
+}
+
+void panthor_gpu_power_changed_off(struct panthor_device *ptdev)
+{
+	guard(pm_runtime_active)(ptdev->base.dev);
+
+	panthor_gpu_irq_disable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
+}
+
 /**
  * panthor_gpu_block_power_off() - Power-off a specific block of the GPU
  * @ptdev: Device.
@@ -395,7 +423,7 @@ void panthor_gpu_suspend(struct panthor_device *ptdev)
  */
 void panthor_gpu_resume(struct panthor_device *ptdev)
 {
-	panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK);
+	panthor_gpu_irq_resume(&ptdev->gpu->irq);
 	panthor_hw_l2_power_on(ptdev);
 }
 
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.h b/drivers/gpu/drm/panthor/panthor_gpu.h
index 12e66f48ced15..12c263a399281 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.h
+++ b/drivers/gpu/drm/panthor/panthor_gpu.h
@@ -51,5 +51,7 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev);
 int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 			     u32 l2, u32 lsc, u32 other);
 int panthor_gpu_soft_reset(struct panthor_device *ptdev);
+void panthor_gpu_power_changed_off(struct panthor_device *ptdev);
+int panthor_gpu_power_changed_on(struct panthor_device *ptdev);
 
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_hw.c b/drivers/gpu/drm/panthor/panthor_hw.c
index 80c521784cd37..d135aa6724fa5 100644
--- a/drivers/gpu/drm/panthor/panthor_hw.c
+++ b/drivers/gpu/drm/panthor/panthor_hw.c
@@ -2,6 +2,8 @@
 /* Copyright 2025 ARM Limited. All rights reserved. */
 
 #include <linux/nvmem-consumer.h>
+#include <linux/platform_device.h>
+
 #include <drm/drm_print.h>
 
 #include "panthor_device.h"
@@ -30,6 +32,8 @@ static struct panthor_hw panthor_hw_arch_v10 = {
 		.soft_reset = panthor_gpu_soft_reset,
 		.l2_power_off = panthor_gpu_l2_power_off,
 		.l2_power_on = panthor_gpu_l2_power_on,
+		.power_changed_off = panthor_gpu_power_changed_off,
+		.power_changed_on = panthor_gpu_power_changed_on,
 	},
 };
 
@@ -54,6 +58,64 @@ static struct panthor_hw_entry panthor_hw_match[] = {
 	},
 };
 
+static int panthor_hw_set_power_tracing(struct device *dev, void *data)
+{
+	struct panthor_device *ptdev = dev_get_drvdata(dev);
+
+	if (!ptdev)
+		return -ENODEV;
+
+	if (!ptdev->hw)
+		return 0;
+
+	if (data) {
+		if (ptdev->hw->ops.power_changed_on)
+			return ptdev->hw->ops.power_changed_on(ptdev);
+	} else {
+		if (ptdev->hw->ops.power_changed_off)
+			ptdev->hw->ops.power_changed_off(ptdev);
+	}
+
+	return 0;
+}
+
+int panthor_hw_power_status_register(void)
+{
+	struct device_driver *drv;
+	int ret;
+
+	drv = driver_find("panthor", &platform_bus_type);
+	if (!drv)
+		return -ENODEV;
+
+	ret = driver_for_each_device(drv, NULL, (void *)true,
+				     panthor_hw_set_power_tracing);
+
+	return ret;
+}
+
+void panthor_hw_power_status_unregister(void)
+{
+	struct device_driver *drv;
+	int ret;
+
+	drv = driver_find("panthor", &platform_bus_type);
+	if (!drv)
+		return;
+
+	ret = driver_for_each_device(drv, NULL, NULL, panthor_hw_set_power_tracing);
+
+	/*
+	 * Ideally, it'd be possible to ask driver_for_each_device to hand us
+	 * another "start" to keep going after the failing device, but it
+	 * doesn't do that. Minor inconvenience in what is probably a bad day
+	 * on the computer already though.
+	 */
+	if (ret)
+		pr_warn("Couldn't mask power IRQ for at least one device: %pe\n",
+			ERR_PTR(ret));
+}
+
 static char *get_gpu_model_name(struct panthor_device *ptdev)
 {
 	const u32 gpu_id = ptdev->gpu_info.gpu_id;
diff --git a/drivers/gpu/drm/panthor/panthor_hw.h b/drivers/gpu/drm/panthor/panthor_hw.h
index 56c68c1e9c261..2c28aea828418 100644
--- a/drivers/gpu/drm/panthor/panthor_hw.h
+++ b/drivers/gpu/drm/panthor/panthor_hw.h
@@ -19,6 +19,12 @@ struct panthor_hw_ops {
 
 	/** @l2_power_on: L2 power on function pointer */
 	int (*l2_power_on)(struct panthor_device *ptdev);
+
+	/** @power_changed_on: Start listening to power change IRQs */
+	int (*power_changed_on)(struct panthor_device *ptdev);
+
+	/** @power_changed_off: Stop listening to power change IRQs */
+	void (*power_changed_off)(struct panthor_device *ptdev);
 };
 
 /**
@@ -32,6 +38,8 @@ struct panthor_hw {
 };
 
 int panthor_hw_init(struct panthor_device *ptdev);
+int panthor_hw_power_status_register(void);
+void panthor_hw_power_status_unregister(void);
 
 static inline int panthor_hw_soft_reset(struct panthor_device *ptdev)
 {
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index 41604a7aaf853..f8c41e36afa48 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -562,9 +562,21 @@ static u64 pack_region_range(struct panthor_device *ptdev, u64 *region_start, u6
 	return region_width | *region_start;
 }
 
+static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
+{
+	return BIT(as);
+}
+
+/* Forward declaration to call helpers within as_enable/disable */
+static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status);
+PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
+
 static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 				 u64 transtab, u64 transcfg, u64 memattr)
 {
+	panthor_mmu_irq_enable_events(&ptdev->mmu->irq,
+				      panthor_mmu_as_fault_mask(ptdev, as_nr));
+
 	gpu_write64(ptdev, AS_TRANSTAB(as_nr), transtab);
 	gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr);
 	gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg);
@@ -580,6 +592,9 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr,
 
 	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
 
+	panthor_mmu_irq_disable_events(&ptdev->mmu->irq,
+				       panthor_mmu_as_fault_mask(ptdev, as_nr));
+
 	/* Flush+invalidate RW caches, invalidate RO ones. */
 	ret = panthor_gpu_flush_caches(ptdev, CACHE_CLEAN | CACHE_INV,
 				       CACHE_CLEAN | CACHE_INV, CACHE_INV);
@@ -612,11 +627,6 @@ static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
 	return value & GENMASK(15, 0);
 }
 
-static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
-{
-	return BIT(as);
-}
-
 /**
  * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults
  * @vm: VM to check.
@@ -670,6 +680,7 @@ int panthor_vm_active(struct panthor_vm *vm)
 	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
 	int ret = 0, as, cookie;
 	u64 transtab, transcfg;
+	u32 fault_mask;
 
 	if (!drm_dev_enter(&ptdev->base, &cookie))
 		return -ENODEV;
@@ -743,14 +754,13 @@ int panthor_vm_active(struct panthor_vm *vm)
 	/* If the VM is re-activated, we clear the fault. */
 	vm->unhandled_fault = false;
 
-	/* Unhandled pagefault on this AS, clear the fault and re-enable interrupts
-	 * before enabling the AS.
+	/* Unhandled pagefault on this AS, clear the fault and enable the AS,
+	 * which re-enables interrupts.
 	 */
-	if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {
-		gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as));
-		ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);
-		ptdev->mmu->irq.mask |= panthor_mmu_as_fault_mask(ptdev, as);
-		gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
+	fault_mask = panthor_mmu_as_fault_mask(ptdev, as);
+	if (ptdev->mmu->as.faulty_mask & fault_mask) {
+		gpu_write(ptdev, MMU_INT_CLEAR, fault_mask);
+		ptdev->mmu->as.faulty_mask &= ~fault_mask;
 	}
 
 	/* The VM update is guarded by ::op_lock, which we take at the beginning
@@ -1696,7 +1706,6 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 	while (status) {
 		u32 as = ffs(status | (status >> 16)) - 1;
 		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
-		u32 new_int_mask;
 		u64 addr;
 		u32 fault_status;
 		u32 exception_type;
@@ -1714,8 +1723,6 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 		mutex_lock(&ptdev->mmu->as.slots_lock);
 
 		ptdev->mmu->as.faulty_mask |= mask;
-		new_int_mask =
-			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
 
 		/* terminal fault, print info about the fault */
 		drm_err(&ptdev->base,
@@ -1739,11 +1746,6 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 		 */
 		gpu_write(ptdev, MMU_INT_CLEAR, mask);
 
-		/* Ignore MMU interrupts on this AS until it's been
-		 * re-enabled.
-		 */
-		ptdev->mmu->irq.mask = new_int_mask;
-
 		if (ptdev->mmu->as.slots[as].vm)
 			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
 
@@ -1758,7 +1760,6 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 	if (has_unhandled_faults)
 		panthor_sched_report_mmu_fault(ptdev);
 }
-PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
 
 /**
  * panthor_mmu_suspend() - Suspend the MMU logic
@@ -1803,7 +1804,7 @@ void panthor_mmu_resume(struct panthor_device *ptdev)
 	ptdev->mmu->as.faulty_mask = 0;
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+	panthor_mmu_irq_resume(&ptdev->mmu->irq);
 }
 
 /**
@@ -1857,7 +1858,7 @@ void panthor_mmu_post_reset(struct panthor_device *ptdev)
 
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+	panthor_mmu_irq_resume(&ptdev->mmu->irq);
 
 	/* Restart the VM_BIND queues. */
 	mutex_lock(&ptdev->mmu->vm.lock);
diff --git a/drivers/gpu/drm/panthor/panthor_pwr.c b/drivers/gpu/drm/panthor/panthor_pwr.c
index 57cfc7ce715b8..ed3b2b4479ca3 100644
--- a/drivers/gpu/drm/panthor/panthor_pwr.c
+++ b/drivers/gpu/drm/panthor/panthor_pwr.c
@@ -545,5 +545,5 @@ void panthor_pwr_resume(struct panthor_device *ptdev)
 	if (!ptdev->pwr)
 		return;
 
-	panthor_pwr_irq_resume(&ptdev->pwr->irq, PWR_INTERRUPTS_MASK);
+	panthor_pwr_irq_resume(&ptdev->pwr->irq);
 }
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index a70f1db0764e9..2fe04d0f0e3a8 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -3916,7 +3916,7 @@ static void job_release(struct kref *ref)
 	if (job->base.s_fence)
 		drm_sched_job_cleanup(&job->base);
 
-	if (job->done_fence && job->done_fence->ops)
+	if (dma_fence_was_initialized(job->done_fence))
 		dma_fence_put(job->done_fence);
 	else
 		dma_fence_free(job->done_fence);
diff --git a/drivers/gpu/drm/panthor/panthor_trace.h b/drivers/gpu/drm/panthor/panthor_trace.h
new file mode 100644
index 0000000000000..6ffeb4fe65992
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_trace.h
@@ -0,0 +1,86 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2025 Collabora ltd. */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM panthor
+
+#if !defined(__PANTHOR_TRACE_H__) || defined(TRACE_HEADER_MULTI_READ)
+#define __PANTHOR_TRACE_H__
+
+#include <linux/tracepoint.h>
+#include <linux/types.h>
+
+#include "panthor_hw.h"
+
+/**
+ * gpu_power_status - called whenever parts of GPU hardware are turned on or off
+ * @dev: pointer to the &struct device, for printing the device name
+ * @shader_bitmap: bitmap where a high bit indicates the shader core at a given
+ *                 bit index is on, and a low bit indicates a shader core is
+ *                 either powered off or absent
+ * @tiler_bitmap: bitmap where a high bit indicates the tiler unit at a given
+ *                bit index is on, and a low bit indicates a tiler unit is
+ *                either powered off or absent
+ * @l2_bitmap: bitmap where a high bit indicates the L2 cache at a given bit
+ *             index is on, and a low bit indicates the L2 cache is either
+ *             powered off or absent
+ */
+TRACE_EVENT_FN(gpu_power_status,
+	TP_PROTO(const struct device *dev, u64 shader_bitmap, u64 tiler_bitmap,
+		 u64 l2_bitmap),
+	TP_ARGS(dev, shader_bitmap, tiler_bitmap, l2_bitmap),
+	TP_STRUCT__entry(
+		__string(dev_name, dev_name(dev))
+		__field(u64, shader_bitmap)
+		__field(u64, tiler_bitmap)
+		__field(u64, l2_bitmap)
+	),
+	TP_fast_assign(
+		__assign_str(dev_name);
+		__entry->shader_bitmap	= shader_bitmap;
+		__entry->tiler_bitmap	= tiler_bitmap;
+		__entry->l2_bitmap	= l2_bitmap;
+	),
+	TP_printk("%s: shader_bitmap=0x%llx tiler_bitmap=0x%llx l2_bitmap=0x%llx",
+		  __get_str(dev_name), __entry->shader_bitmap, __entry->tiler_bitmap,
+		  __entry->l2_bitmap
+	),
+	panthor_hw_power_status_register, panthor_hw_power_status_unregister
+);
+
+/**
+ * gpu_job_irq - called after a job interrupt from firmware completes
+ * @dev: pointer to the &struct device, for printing the device name
+ * @events: bitmask of BIT(CSG id) | BIT(31) for a global event
+ * @duration_ns: Nanoseconds between job IRQ handler entry and exit
+ *
+ * The panthor_job_irq_handler() function instrumented by this tracepoint exits
+ * once it has queued the firmware interrupts for processing, not when the
+ * firmware interrupts are fully processed. This tracepoint allows for debugging
+ * issues with delays in the workqueue's processing of events.
+ */
+TRACE_EVENT(gpu_job_irq,
+	TP_PROTO(const struct device *dev, u32 events, u32 duration_ns),
+	TP_ARGS(dev, events, duration_ns),
+	TP_STRUCT__entry(
+		__string(dev_name, dev_name(dev))
+		__field(u32, events)
+		__field(u32, duration_ns)
+	),
+	TP_fast_assign(
+		__assign_str(dev_name);
+		__entry->events		= events;
+		__entry->duration_ns	= duration_ns;
+	),
+	TP_printk("%s: events=0x%x duration_ns=%d", __get_str(dev_name),
+		  __entry->events, __entry->duration_ns)
+);
+
+#endif /* __PANTHOR_TRACE_H__ */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE panthor_trace
+
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index b1bf1e798cc60..06979d0e8a9f0 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -62,7 +62,8 @@ static long qxl_fence_wait(struct dma_fence *fence, bool intr,
 	struct qxl_device *qdev;
 	unsigned long cur, end = jiffies + timeout;
 
-	qdev = container_of(fence->lock, struct qxl_device, release_lock);
+	qdev = container_of(fence->extern_lock, struct qxl_device,
+			    release_lock);
 
 	if (!wait_event_timeout(qdev->release_event,
 				(dma_fence_is_signaled(fence) ||
@@ -146,7 +147,7 @@ qxl_release_free(struct qxl_device *qdev,
 	idr_remove(&qdev->release_idr, release->id);
 	spin_unlock(&qdev->release_idr_lock);
 
-	if (release->base.ops) {
+	if (dma_fence_was_initialized(&release->base)) {
 		WARN_ON(list_empty(&release->bos));
 		qxl_release_free_list(release);
 
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 08f8ba4fd1481..9f511ff088229 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -44,8 +44,6 @@ bool radeon_atpx_dgpu_req_power_for_displays(void);
 static inline bool radeon_atpx_dgpu_req_power_for_displays(void) { return false; }
 #endif
 
-#define ACPI_AC_CLASS           "ac_adapter"
-
 struct atif_verify_interface {
 	u16 size;		/* structure size in bytes (includes size field) */
 	u16 version;		/* version */
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 5d523d5dae884..705c012fcf9e0 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -37,6 +37,7 @@
 #include <drm/drm_client_event.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_device.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_file.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_probe_helper.h>
@@ -1574,7 +1575,6 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend,
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 		struct drm_framebuffer *fb = crtc->primary->fb;
-		struct radeon_bo *robj;
 
 		if (radeon_crtc->cursor_bo) {
 			struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo);
@@ -1588,9 +1588,10 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend,
 		if (fb == NULL || fb->obj[0] == NULL) {
 			continue;
 		}
-		robj = gem_to_radeon_bo(fb->obj[0]);
 		/* don't unpin kernel fb objects */
-		if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
+		if (!drm_fb_helper_gem_is_fb(dev->fb_helper, fb->obj[0])) {
+			struct radeon_bo *robj = gem_to_radeon_bo(fb->obj[0]);
+
 			r = radeon_bo_reserve(robj, false);
 			if (r == 0) {
 				radeon_bo_unpin(robj);
diff --git a/drivers/gpu/drm/radeon/radeon_fbdev.c b/drivers/gpu/drm/radeon/radeon_fbdev.c
index 18d61f3f7344f..3e243f5e2f443 100644
--- a/drivers/gpu/drm/radeon/radeon_fbdev.c
+++ b/drivers/gpu/drm/radeon/radeon_fbdev.c
@@ -274,20 +274,3 @@ int radeon_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper,
 	radeon_fbdev_destroy_pinned_object(gobj);
 	return ret;
 }
-
-bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
-{
-	struct drm_fb_helper *fb_helper = rdev_to_drm(rdev)->fb_helper;
-	struct drm_gem_object *gobj;
-
-	if (!fb_helper)
-		return false;
-
-	gobj = drm_gem_fb_get_obj(fb_helper->fb, 0);
-	if (!gobj)
-		return false;
-	if (gobj != &robj->tbo.base)
-		return false;
-
-	return true;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 088af85902f79..ae1ecdc2e1894 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -936,14 +936,9 @@ int radeon_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper,
 				    struct drm_fb_helper_surface_size *sizes);
 #define RADEON_FBDEV_DRIVER_OPS \
 	.fbdev_probe = radeon_fbdev_driver_fbdev_probe
-bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
 #else
 #define RADEON_FBDEV_DRIVER_OPS \
 	.fbdev_probe = NULL
-static inline bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
-{
-	return false;
-}
 #endif
 
 void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id);
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index b4aa49b1ac634..4b10715f951c3 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -2915,9 +2915,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
 	if (rdev->family == CHIP_HAINAN) {
 		if ((rdev->pdev->revision == 0x81) ||
 		    (rdev->pdev->revision == 0xC3) ||
+		    (rdev->pdev->device == 0x6660) ||
 		    (rdev->pdev->device == 0x6664) ||
 		    (rdev->pdev->device == 0x6665) ||
-		    (rdev->pdev->device == 0x6667)) {
+		    (rdev->pdev->device == 0x6667) ||
+		    (rdev->pdev->device == 0x666F)) {
 			max_sclk = 75000;
 		}
 		if ((rdev->pdev->revision == 0xC3) ||
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
index 4ef2e3c129ed7..508977b9e8926 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
@@ -84,7 +84,6 @@ struct dsi_setup_info {
 	unsigned long fout;
 	u16 m;
 	u16 n;
-	u16 vclk_divider;
 	const struct dsi_clk_config *clkset;
 };
 
@@ -335,10 +334,24 @@ rcar_mipi_dsi_post_init_phtw_v4h(struct rcar_mipi_dsi *dsi,
  * Hardware Setup
  */
 
+static unsigned int rcar_mipi_dsi_vclk_divider(struct rcar_mipi_dsi *dsi,
+					       struct dsi_setup_info *setup_info)
+{
+	switch (dsi->info->model) {
+	case RCAR_DSI_V3U:
+	default:
+		return (setup_info->clkset->vco_cntrl >> 4) & 0x3;
+
+	case RCAR_DSI_V4H:
+		return (setup_info->clkset->vco_cntrl >> 3) & 0x7;
+	}
+}
+
 static void rcar_mipi_dsi_pll_calc(struct rcar_mipi_dsi *dsi,
 				   unsigned long fin_rate,
 				   unsigned long fout_target,
-				   struct dsi_setup_info *setup_info)
+				   struct dsi_setup_info *setup_info,
+				   u16 vclk_divider)
 {
 	unsigned int best_err = -1;
 	const struct rcar_mipi_dsi_device_info *info = dsi->info;
@@ -360,7 +373,7 @@ static void rcar_mipi_dsi_pll_calc(struct rcar_mipi_dsi *dsi,
 			if (fout < info->fout_min || fout > info->fout_max)
 				continue;
 
-			fout = div64_u64(fout, setup_info->vclk_divider);
+			fout = div64_u64(fout, vclk_divider);
 
 			if (fout < setup_info->clkset->min_freq ||
 			    fout > setup_info->clkset->max_freq)
@@ -390,7 +403,9 @@ static void rcar_mipi_dsi_parameters_calc(struct rcar_mipi_dsi *dsi,
 	unsigned long fout_target;
 	unsigned long fin_rate;
 	unsigned int i;
+	unsigned int div;
 	unsigned int err;
+	u16 vclk_divider;
 
 	/*
 	 * Calculate Fout = dot clock * ColorDepth / (2 * Lane Count)
@@ -412,18 +427,20 @@ static void rcar_mipi_dsi_parameters_calc(struct rcar_mipi_dsi *dsi,
 
 	fin_rate = clk_get_rate(clk);
 
+	div = rcar_mipi_dsi_vclk_divider(dsi, setup_info);
+
 	switch (dsi->info->model) {
 	case RCAR_DSI_V3U:
 	default:
-		setup_info->vclk_divider = 1 << ((clk_cfg->vco_cntrl >> 4) & 0x3);
+		vclk_divider = BIT_U16(div);
 		break;
 
 	case RCAR_DSI_V4H:
-		setup_info->vclk_divider = 1 << (((clk_cfg->vco_cntrl >> 3) & 0x7) + 1);
+		vclk_divider = BIT_U16(div + 1);
 		break;
 	}
 
-	rcar_mipi_dsi_pll_calc(dsi, fin_rate, fout_target, setup_info);
+	rcar_mipi_dsi_pll_calc(dsi, fin_rate, fout_target, setup_info, vclk_divider);
 
 	/* Find hsfreqrange */
 	setup_info->hsfreq = setup_info->fout * 2;
@@ -439,7 +456,7 @@ static void rcar_mipi_dsi_parameters_calc(struct rcar_mipi_dsi *dsi,
 	dev_dbg(dsi->dev,
 		"Fout = %u * %lu / (%u * %u * %u) = %lu (target %lu Hz, error %d.%02u%%)\n",
 		setup_info->m, fin_rate, dsi->info->n_mul, setup_info->n,
-		setup_info->vclk_divider, setup_info->fout, fout_target,
+		vclk_divider, setup_info->fout, fout_target,
 		err / 100, err % 100);
 
 	dev_dbg(dsi->dev,
@@ -653,11 +670,11 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi,
 	switch (dsi->info->model) {
 	case RCAR_DSI_V3U:
 	default:
-		vclkset |= VCLKSET_DIV_V3U(__ffs(setup_info.vclk_divider));
+		vclkset |= VCLKSET_DIV_V3U(rcar_mipi_dsi_vclk_divider(dsi, &setup_info));
 		break;
 
 	case RCAR_DSI_V4H:
-		vclkset |= VCLKSET_DIV_V4H(__ffs(setup_info.vclk_divider) - 1);
+		vclkset |= VCLKSET_DIV_V4H(rcar_mipi_dsi_vclk_divider(dsi, &setup_info));
 		break;
 	}
 
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
index 29f2b7d24fe59..a87a301326c7a 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
@@ -808,8 +808,7 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi,
 	 * commands payload not being saved to memory.
 	 */
 	dsisetr = rzg2l_mipi_dsi_link_read(dsi, DSISETR);
-	dsisetr &= ~DSISETR_MRPSZ;
-	dsisetr |= FIELD_PREP(DSISETR_MRPSZ, RZG2L_DCS_BUF_SIZE);
+	FIELD_MODIFY(DSISETR_MRPSZ, &dsisetr, RZG2L_DCS_BUF_SIZE);
 	rzg2l_mipi_dsi_link_write(dsi, DSISETR, dsisetr);
 
 	return 0;
diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
index 25ab4e46301e8..dac3d202971ed 100644
--- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
@@ -75,7 +75,7 @@ static const struct drm_encoder_helper_funcs dw_dp_encoder_helper_funcs = {
 static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
-	struct dw_dp_plat_data plat_data;
+	const struct dw_dp_plat_data *plat_data;
 	struct drm_device *drm_dev = data;
 	struct rockchip_dw_dp *dp;
 	struct drm_encoder *encoder;
@@ -89,7 +89,10 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *
 	dp->dev = dev;
 	platform_set_drvdata(pdev, dp);
 
-	plat_data.max_link_rate = 810000;
+	plat_data = of_device_get_match_data(dev);
+	if (!plat_data)
+		return -ENODEV;
+
 	encoder = &dp->encoder.encoder;
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm_dev, dev->of_node);
 	rockchip_drm_encoder_set_crtc_endpoint_id(&dp->encoder, dev->of_node, 0, 0);
@@ -99,7 +102,7 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *
 		return ret;
 	drm_encoder_helper_add(encoder, &dw_dp_encoder_helper_funcs);
 
-	dp->base = dw_dp_bind(dev, encoder, &plat_data);
+	dp->base = dw_dp_bind(dev, encoder, plat_data);
 	if (IS_ERR(dp->base)) {
 		ret = PTR_ERR(dp->base);
 		return ret;
@@ -134,8 +137,24 @@ static void dw_dp_remove(struct platform_device *pdev)
 	component_del(dp->dev, &dw_dp_rockchip_component_ops);
 }
 
+static const struct dw_dp_plat_data rk3588_dp_plat_data = {
+	.max_link_rate = 810000,
+	.pixel_mode = DW_DP_MP_QUAD_PIXEL,
+};
+
+static const struct dw_dp_plat_data rk3576_dp_plat_data = {
+	.max_link_rate = 810000,
+	.pixel_mode = DW_DP_MP_DUAL_PIXEL,
+};
+
 static const struct of_device_id dw_dp_of_match[] = {
-	{ .compatible = "rockchip,rk3588-dp", },
+	{
+		.compatible = "rockchip,rk3588-dp",
+		.data = &rk3588_dp_plat_data,
+	}, {
+		.compatible = "rockchip,rk3576-dp",
+		.data = &rk3576_dp_plat_data,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, dw_dp_of_match);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index a195f5c819a2d..843c7ef979b21 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -1080,6 +1080,13 @@ static int vop2_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 	}
 
+	if ((cstate->background_color << 16) &&
+	    (fb->format->has_alpha || pstate->alpha != 0xffff)) {
+		drm_dbg_kms(vop2->drm,
+			    "Alpha-blending with background color is unsupported\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -1552,6 +1559,7 @@ static void vop2_post_config(struct drm_crtc *crtc)
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	u64 bgcolor = crtc->state->background_color;
 	u16 vtotal = mode->crtc_vtotal;
 	u16 hdisplay = mode->crtc_hdisplay;
 	u16 hact_st = mode->crtc_htotal - mode->crtc_hsync_start;
@@ -1597,7 +1605,15 @@ static void vop2_post_config(struct drm_crtc *crtc)
 		vop2_vp_write(vp, RK3568_VP_POST_DSP_VACT_INFO_F1, val);
 	}
 
-	vop2_vp_write(vp, RK3568_VP_DSP_BG, 0);
+	/*
+	 * Background color is programmed with 10 bits of precision.
+	 * Since performance is more important than accuracy here,
+	 * make use of the DRM_ARGB64_GET*_BPCS() helpers.
+	 */
+	val = FIELD_PREP(RK3568_VP_DSP_BG__DSP_BG_RED, DRM_ARGB64_GETR_BPCS(bgcolor, 10));
+	FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_GREEN, &val, DRM_ARGB64_GETG_BPCS(bgcolor, 10));
+	FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_BLUE, &val, DRM_ARGB64_GETB_BPCS(bgcolor, 10));
+	vop2_vp_write(vp, RK3568_VP_DSP_BG, val);
 }
 
 static int us_to_vertical_line(struct drm_display_mode *mode, int us)
@@ -1983,6 +1999,10 @@ static int vop2_crtc_state_dump(struct drm_crtc *crtc, struct seq_file *s)
 		   drm_get_bus_format_name(vcstate->bus_format));
 	seq_printf(s, "\toutput_mode[%x]", vcstate->output_mode);
 	seq_printf(s, " color_space[%d]\n", vcstate->color_space);
+	seq_printf(s, "\tbackground color (10bpc): r=0x%x g=0x%x b=0x%x\n",
+		   DRM_ARGB64_GETR_BPCS(cstate->background_color, 10),
+		   DRM_ARGB64_GETG_BPCS(cstate->background_color, 10),
+		   DRM_ARGB64_GETB_BPCS(cstate->background_color, 10));
 	seq_printf(s, "    Display mode: %dx%d%s%d\n",
 		   mode->hdisplay, mode->vdisplay, interlaced ? "i" : "p",
 		   drm_mode_vrefresh(mode));
@@ -2471,6 +2491,8 @@ static int vop2_create_crtcs(struct vop2 *vop2)
 			return dev_err_probe(drm->dev, ret,
 					     "crtc init for video_port%d failed\n", i);
 
+		drm_crtc_attach_background_color_property(&vp->crtc);
+
 		drm_crtc_helper_add(&vp->crtc, &vop2_crtc_helper_funcs);
 		if (vop2->lut_regs) {
 			const struct vop2_video_port_data *vp_data = &vop2_data->vp[vp->id];
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
index 9124191899bac..37722652844a9 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
@@ -658,6 +658,10 @@ enum dst_factor_mode {
 #define RK3588_VP_CLK_CTRL__DCLK_OUT_DIV		GENMASK(3, 2)
 #define RK3588_VP_CLK_CTRL__DCLK_CORE_DIV		GENMASK(1, 0)
 
+#define RK3568_VP_DSP_BG__DSP_BG_RED			GENMASK(29, 20)
+#define RK3568_VP_DSP_BG__DSP_BG_GREEN			GENMASK(19, 10)
+#define RK3568_VP_DSP_BG__DSP_BG_BLUE			GENMASK(9, 0)
+
 #define RK3568_VP_POST_SCL_CTRL__VSCALEDOWN		BIT(1)
 #define RK3568_VP_POST_SCL_CTRL__HSCALEDOWN		BIT(0)
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
index f3950e8476a75..02a788a4dfdda 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
@@ -2106,8 +2106,8 @@ static void rk3568_vop2_wait_for_port_mux_done(struct vop2 *vop2)
 	ret = readx_poll_timeout_atomic(rk3568_vop2_read_port_mux, vop2, port_mux_sel,
 					port_mux_sel == vop2->old_port_sel, 10, 50 * 1000);
 	if (ret)
-		DRM_DEV_ERROR(vop2->dev, "wait port_mux done timeout: 0x%x--0x%x\n",
-			      port_mux_sel, vop2->old_port_sel);
+		drm_err_ratelimited(vop2->drm, "wait port_mux done timeout: 0x%x--0x%x\n",
+				    port_mux_sel, vop2->old_port_sel);
 }
 
 static u32 rk3568_vop2_read_layer_cfg(struct vop2 *vop2)
@@ -2126,8 +2126,8 @@ static void rk3568_vop2_wait_for_layer_cfg_done(struct vop2 *vop2, u32 cfg)
 	ret = readx_poll_timeout_atomic(rk3568_vop2_read_layer_cfg, vop2, atv_layer_cfg,
 					atv_layer_cfg == cfg, 10, 50 * 1000);
 	if (ret)
-		DRM_DEV_ERROR(vop2->dev, "wait layer cfg done timeout: 0x%x--0x%x\n",
-			      atv_layer_cfg, cfg);
+		drm_err_ratelimited(vop2->drm, "wait layer cfg done timeout: 0x%x--0x%x\n",
+				    atv_layer_cfg, cfg);
 }
 
 static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 9391d6f0dc01d..096fe28aa9c9e 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -156,19 +156,19 @@ static void drm_sched_fence_set_deadline_finished(struct dma_fence *f,
 	struct dma_fence *parent;
 	unsigned long flags;
 
-	spin_lock_irqsave(&fence->lock, flags);
+	dma_fence_lock_irqsave(f, flags);
 
 	/* If we already have an earlier deadline, keep it: */
 	if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags) &&
 	    ktime_before(fence->deadline, deadline)) {
-		spin_unlock_irqrestore(&fence->lock, flags);
+		dma_fence_unlock_irqrestore(f, flags);
 		return;
 	}
 
 	fence->deadline = deadline;
 	set_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags);
 
-	spin_unlock_irqrestore(&fence->lock, flags);
+	dma_fence_unlock_irqrestore(f, flags);
 
 	/*
 	 * smp_load_aquire() to ensure that if we are racing another
@@ -195,10 +195,10 @@ static const struct dma_fence_ops drm_sched_fence_ops_finished = {
 
 struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
 {
-	if (f->ops == &drm_sched_fence_ops_scheduled)
+	if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_scheduled)
 		return container_of(f, struct drm_sched_fence, scheduled);
 
-	if (f->ops == &drm_sched_fence_ops_finished)
+	if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_finished)
 		return container_of(f, struct drm_sched_fence, finished);
 
 	return NULL;
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 2d5cb21a05b61..21dc82c75c9ef 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -1419,48 +1419,12 @@ static void drm_sched_cancel_remaining_jobs(struct drm_gpu_scheduler *sched)
  */
 void drm_sched_fini(struct drm_gpu_scheduler *sched)
 {
-	struct drm_sched_entity *s_entity;
 	int i;
 
 	drm_sched_wqueue_stop(sched);
 
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
-		struct drm_sched_rq *rq = sched->sched_rq[i];
-
-		spin_lock(&rq->lock);
-		list_for_each_entry(s_entity, &rq->entities, list) {
-			/*
-			 * Prevents reinsertion and marks job_queue as idle,
-			 * it will be removed from the rq in drm_sched_entity_fini()
-			 * eventually
-			 *
-			 * FIXME:
-			 * This lacks the proper spin_lock(&s_entity->lock) and
-			 * is, therefore, a race condition. Most notably, it
-			 * can race with drm_sched_entity_push_job(). The lock
-			 * cannot be taken here, however, because this would
-			 * lead to lock inversion -> deadlock.
-			 *
-			 * The best solution probably is to enforce the life
-			 * time rule of all entities having to be torn down
-			 * before their scheduler. Then, however, locking could
-			 * be dropped alltogether from this function.
-			 *
-			 * For now, this remains a potential race in all
-			 * drivers that keep entities alive for longer than
-			 * the scheduler.
-			 *
-			 * The READ_ONCE() is there to make the lockless read
-			 * (warning about the lockless write below) slightly
-			 * less broken...
-			 */
-			if (!READ_ONCE(s_entity->stopped))
-				dev_warn(sched->dev, "Tearing down scheduler with active entities!\n");
-			s_entity->stopped = true;
-		}
-		spin_unlock(&rq->lock);
+	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++)
 		kfree(sched->sched_rq[i]);
-	}
 
 	/* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */
 	wake_up_all(&sched->job_scheduled);
diff --git a/drivers/gpu/drm/scheduler/tests/tests_basic.c b/drivers/gpu/drm/scheduler/tests/tests_basic.c
index 82a41a456b0a8..a5a5a35a87b0d 100644
--- a/drivers/gpu/drm/scheduler/tests/tests_basic.c
+++ b/drivers/gpu/drm/scheduler/tests/tests_basic.c
@@ -421,7 +421,7 @@ static void drm_sched_change_priority(struct kunit *test)
 
 static struct kunit_case drm_sched_priority_tests[] = {
 	KUNIT_CASE(drm_sched_priorities),
-	KUNIT_CASE(drm_sched_change_priority),
+	KUNIT_CASE_SLOW(drm_sched_change_priority),
 	{}
 };
 
@@ -546,7 +546,7 @@ static void drm_sched_test_credits(struct kunit *test)
 }
 
 static struct kunit_case drm_sched_credits_tests[] = {
-	KUNIT_CASE(drm_sched_test_credits),
+	KUNIT_CASE_SLOW(drm_sched_test_credits),
 	{}
 };
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 40405a52a073a..e989f75c09b7d 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -491,6 +491,9 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine,
 	drm_for_each_plane_mask(plane, drm, crtc_state->plane_mask) {
 		struct drm_plane_state *plane_state =
 			drm_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
 		struct sun4i_layer_state *layer_state =
 			state_to_sun4i_layer_state(plane_state);
 		struct drm_framebuffer *fb = plane_state->fb;
@@ -878,7 +881,8 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 						     &sun4i_backend_regmap_config);
 	if (IS_ERR(backend->engine.regs)) {
 		dev_err(dev, "Couldn't create the backend regmap\n");
-		return PTR_ERR(backend->engine.regs);
+		ret = PTR_ERR(backend->engine.regs);
+		goto err_disable_ram_clk;
 	}
 
 	list_add_tail(&backend->engine.list, &drv->engine_list);
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index ce9c155bfad7f..02acc7cbdb979 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -321,7 +321,7 @@ static struct drm_plane **sun8i_layers_init(struct drm_device *drm,
 	unsigned int phy_index;
 	int i;
 
-	planes = devm_kcalloc(drm->dev, plane_cnt, sizeof(*planes), GFP_KERNEL);
+	planes = devm_kcalloc(drm->dev, plane_cnt + 1, sizeof(*planes), GFP_KERNEL);
 	if (!planes)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
index f08f6da55dd0c..72c92203ae633 100644
--- a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
@@ -124,25 +124,13 @@ static void sun8i_ui_layer_update_buffer(struct sun8i_layer *layer,
 {
 	struct drm_plane_state *state = plane->state;
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_dma_object *gem;
 	dma_addr_t dma_addr;
 	u32 ch_base;
-	int bpp;
 
 	ch_base = sun8i_channel_base(layer);
 
-	/* Get the physical address of the buffer in memory */
-	gem = drm_fb_dma_get_gem_obj(fb, 0);
-
-	DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->dma_addr);
-
-	/* Compute the start of the displayed memory */
-	bpp = fb->format->cpp[0];
-	dma_addr = gem->dma_addr + fb->offsets[0];
-
-	/* Fixup framebuffer address for src coordinates */
-	dma_addr += (state->src.x1 >> 16) * bpp;
-	dma_addr += (state->src.y1 >> 16) * fb->pitches[0];
+	/* Get the start of the displayed memory */
+	dma_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
 
 	/* Set the line width */
 	DRM_DEBUG_DRIVER("Layer line width: %d bytes\n", fb->pitches[0]);
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
index ca3ab59e108d4..cd8d6c2da0c72 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -197,38 +197,15 @@ static void sun8i_vi_layer_update_buffer(struct sun8i_layer *layer,
 	struct drm_plane_state *state = plane->state;
 	struct drm_framebuffer *fb = state->fb;
 	const struct drm_format_info *format = fb->format;
-	struct drm_gem_dma_object *gem;
-	u32 dx, dy, src_x, src_y;
 	dma_addr_t dma_addr;
 	u32 ch_base;
 	int i;
 
 	ch_base = sun8i_channel_base(layer);
 
-	/* Adjust x and y to be dividable by subsampling factor */
-	src_x = (state->src.x1 >> 16) & ~(format->hsub - 1);
-	src_y = (state->src.y1 >> 16) & ~(format->vsub - 1);
-
 	for (i = 0; i < format->num_planes; i++) {
-		/* Get the physical address of the buffer in memory */
-		gem = drm_fb_dma_get_gem_obj(fb, i);
-
-		DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->dma_addr);
-
-		/* Compute the start of the displayed memory */
-		dma_addr = gem->dma_addr + fb->offsets[i];
-
-		dx = src_x;
-		dy = src_y;
-
-		if (i > 0) {
-			dx /= format->hsub;
-			dy /= format->vsub;
-		}
-
-		/* Fixup framebuffer address for src coordinates */
-		dma_addr += dx * format->cpp[i];
-		dma_addr += dy * fb->pitches[i];
+		/* Get the start of the displayed memory */
+		dma_addr = drm_fb_dma_get_gem_addr(fb, state, i);
 
 		/* Set the line width */
 		DRM_DEBUG_DRIVER("Layer %d. line width: %d bytes\n",
diff --git a/drivers/gpu/drm/sun4i/sunxi_engine.h b/drivers/gpu/drm/sun4i/sunxi_engine.h
index ec0c4932f15cf..c9461de06cd04 100644
--- a/drivers/gpu/drm/sun4i/sunxi_engine.h
+++ b/drivers/gpu/drm/sun4i/sunxi_engine.h
@@ -114,7 +114,7 @@ struct sunxi_engine_ops {
 	void (*vblank_quirk)(struct sunxi_engine *engine);
 
 	/**
-	 * @mode_set
+	 * @mode_set:
 	 *
 	 * This callback is used to set mode related parameters
 	 * like interlacing, screen size, etc. once per mode set.
@@ -131,6 +131,7 @@ struct sunxi_engine_ops {
  * @node:	the of device node of the engine
  * @regs:	the regmap of the engine
  * @id:		the id of the engine (-1 if not used)
+ * @list:	engine list management
  */
 struct sunxi_engine {
 	const struct sunxi_engine_ops	*ops;
@@ -140,7 +141,6 @@ struct sunxi_engine {
 
 	int id;
 
-	/* Engine list management */
 	struct list_head		list;
 };
 
@@ -163,6 +163,9 @@ sunxi_engine_commit(struct sunxi_engine *engine,
  * sunxi_engine_layers_init() - Create planes (layers) for the engine
  * @drm:	pointer to the drm_device for which planes will be created
  * @engine:	pointer to the engine
+ *
+ * Returns: The array of struct drm_plane backing the layers, or an
+ *		error pointer on failure.
  */
 static inline struct drm_plane **
 sunxi_engine_layers_init(struct drm_device *drm, struct sunxi_engine *engine)
diff --git a/drivers/gpu/drm/sysfb/Kconfig b/drivers/gpu/drm/sysfb/Kconfig
index 9c9884c7efc6c..2559ead6cf1f8 100644
--- a/drivers/gpu/drm/sysfb/Kconfig
+++ b/drivers/gpu/drm/sysfb/Kconfig
@@ -7,6 +7,22 @@ config DRM_SYSFB_HELPER
 	tristate
 	depends on DRM
 
+config DRM_COREBOOTDRM
+	tristate "Coreboot framebuffer driver"
+	depends on DRM && MMU
+	depends on GOOGLE_FRAMEBUFFER_COREBOOT
+	select APERTURE_HELPERS
+	select DRM_CLIENT_SELECTION
+	select DRM_GEM_SHMEM_HELPER
+	select DRM_KMS_HELPER
+	select DRM_SYSFB_HELPER
+	help
+	  DRM driver for coreboot-provided framebuffers.
+
+	  This driver assumes that the display hardware has been initialized
+	  by coreboot firmware before the kernel boots. Scanout buffer, size,
+	  and display format must be provided via coreboot framebuffer device.
+
 config DRM_EFIDRM
 	tristate "EFI framebuffer driver"
 	depends on DRM && MMU && EFI && (!SYSFB_SIMPLEFB || COMPILE_TEST)
diff --git a/drivers/gpu/drm/sysfb/Makefile b/drivers/gpu/drm/sysfb/Makefile
index a156c496413d3..85c9087ab03da 100644
--- a/drivers/gpu/drm/sysfb/Makefile
+++ b/drivers/gpu/drm/sysfb/Makefile
@@ -6,6 +6,7 @@ drm_sysfb_helper-y := \
 drm_sysfb_helper-$(CONFIG_SCREEN_INFO) += drm_sysfb_screen_info.o
 obj-$(CONFIG_DRM_SYSFB_HELPER)	+= drm_sysfb_helper.o
 
+obj-$(CONFIG_DRM_COREBOOTDRM)	+= corebootdrm.o
 obj-$(CONFIG_DRM_EFIDRM)	+= efidrm.o
 obj-$(CONFIG_DRM_OFDRM)		+= ofdrm.o
 obj-$(CONFIG_DRM_SIMPLEDRM)	+= simpledrm.o
diff --git a/drivers/gpu/drm/sysfb/corebootdrm.c b/drivers/gpu/drm/sysfb/corebootdrm.c
new file mode 100644
index 0000000000000..5dc6f3c76f7bd
--- /dev/null
+++ b/drivers/gpu/drm/sysfb/corebootdrm.c
@@ -0,0 +1,434 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/aperture.h>
+#include <linux/coreboot.h>
+#include <linux/minmax.h>
+#include <linux/platform_device.h>
+
+#include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_device.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fbdev_shmem.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+
+#include "drm_sysfb_helper.h"
+
+#define DRIVER_NAME	"corebootdrm"
+#define DRIVER_DESC	"DRM driver for Coreboot framebuffers"
+#define DRIVER_MAJOR	1
+#define DRIVER_MINOR	0
+
+static const struct drm_format_info *
+corebootdrm_get_format_fb(struct drm_device *dev, const struct lb_framebuffer *fb)
+{
+	static const struct drm_sysfb_format formats[] = {
+		{ PIXEL_FORMAT_XRGB1555, DRM_FORMAT_XRGB1555, },
+		{ PIXEL_FORMAT_RGB565, DRM_FORMAT_RGB565, },
+		{ PIXEL_FORMAT_RGB888, DRM_FORMAT_RGB888, },
+		{ PIXEL_FORMAT_XRGB8888, DRM_FORMAT_XRGB8888, },
+		{ PIXEL_FORMAT_XBGR8888, DRM_FORMAT_XBGR8888, },
+		{ PIXEL_FORMAT_XRGB2101010, DRM_FORMAT_XRGB2101010, },
+	};
+	const struct pixel_format pixel = {
+		.bits_per_pixel = fb->bits_per_pixel,
+		.indexed  = false,
+		.alpha = {
+			.offset = 0,
+			.length = 0,
+		},
+		.red = {
+			.offset = fb->red_mask_pos,
+			.length = fb->red_mask_size,
+		},
+		.green = {
+			.offset = fb->green_mask_pos,
+			.length = fb->green_mask_size,
+		},
+		.blue = {
+			.offset = fb->blue_mask_pos,
+			.length = fb->blue_mask_size,
+		},
+	};
+
+	return drm_sysfb_get_format(dev, formats, ARRAY_SIZE(formats), &pixel);
+}
+
+static int corebootdrm_get_width_fb(struct drm_device *dev, const struct lb_framebuffer *fb)
+{
+	return drm_sysfb_get_validated_int0(dev, "width", fb->x_resolution, INT_MAX);
+}
+
+static int corebootdrm_get_height_fb(struct drm_device *dev, const struct lb_framebuffer *fb)
+{
+	return drm_sysfb_get_validated_int0(dev, "height", fb->y_resolution, INT_MAX);
+}
+
+static int corebootdrm_get_pitch_fb(struct drm_device *dev, const struct drm_format_info *format,
+				    unsigned int width, const struct lb_framebuffer *fb)
+{
+	u64 bytes_per_line = fb->bytes_per_line;
+
+	if (!bytes_per_line)
+		bytes_per_line = drm_format_info_min_pitch(format, 0, width);
+
+	return drm_sysfb_get_validated_int0(dev, "pitch", bytes_per_line, INT_MAX);
+}
+
+static resource_size_t corebootdrm_get_size_fb(struct drm_device *dev, unsigned int height,
+					       unsigned int pitch,
+					       const struct lb_framebuffer *fb)
+{
+	resource_size_t size;
+
+	if (check_mul_overflow(height, pitch, &size))
+		return 0;
+
+	return size;
+}
+
+static phys_addr_t corebootdrm_get_address_fb(struct drm_device *dev, resource_size_t size,
+					      const struct lb_framebuffer *fb)
+{
+	if (size > PHYS_ADDR_MAX)
+		return 0;
+	if (!fb->physical_address)
+		return 0;
+	if (fb->physical_address > (PHYS_ADDR_MAX - size))
+		return 0;
+
+	return fb->physical_address;
+}
+
+static enum drm_panel_orientation corebootdrm_get_orientation_fb(struct drm_device *dev,
+								 const struct lb_framebuffer *fb)
+{
+	if (!LB_FRAMEBUFFER_HAS_ORIENTATION(fb))
+		return DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
+
+	switch (fb->orientation) {
+	case LB_FRAMEBUFFER_ORIENTATION_NORMAL:
+		return DRM_MODE_PANEL_ORIENTATION_NORMAL;
+	case LB_FRAMEBUFFER_ORIENTATION_BOTTOM_UP:
+		return DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP;
+	case LB_FRAMEBUFFER_ORIENTATION_LEFT_UP:
+		return DRM_MODE_PANEL_ORIENTATION_LEFT_UP;
+	case LB_FRAMEBUFFER_ORIENTATION_RIGHT_UP:
+		return DRM_MODE_PANEL_ORIENTATION_RIGHT_UP;
+	}
+
+	return DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
+}
+
+/*
+ * Simple Framebuffer device
+ */
+
+struct corebootdrm_device {
+	struct drm_sysfb_device sysfb;
+
+	/* modesetting */
+	u32 formats[DRM_SYSFB_PLANE_NFORMATS(1)];
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+};
+
+/*
+ * Modesetting
+ */
+
+static const u64 corebootdrm_primary_plane_format_modifiers[] = {
+	DRM_SYSFB_PLANE_FORMAT_MODIFIERS,
+};
+
+static const struct drm_plane_helper_funcs corebootdrm_primary_plane_helper_funcs = {
+	DRM_SYSFB_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs corebootdrm_primary_plane_funcs = {
+	DRM_SYSFB_PLANE_FUNCS,
+	.destroy = drm_plane_cleanup,
+};
+
+static const struct drm_crtc_helper_funcs corebootdrm_crtc_helper_funcs = {
+	DRM_SYSFB_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs corebootdrm_crtc_funcs = {
+	DRM_SYSFB_CRTC_FUNCS,
+	.destroy = drm_crtc_cleanup,
+};
+
+static const struct drm_encoder_funcs corebootdrm_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static const struct drm_connector_helper_funcs corebootdrm_connector_helper_funcs = {
+	DRM_SYSFB_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs corebootdrm_connector_funcs = {
+	DRM_SYSFB_CONNECTOR_FUNCS,
+	.destroy = drm_connector_cleanup,
+};
+
+static const struct drm_mode_config_funcs corebootdrm_mode_config_funcs = {
+	DRM_SYSFB_MODE_CONFIG_FUNCS,
+};
+
+static int corebootdrm_mode_config_init(struct corebootdrm_device *cdev,
+					enum drm_panel_orientation orientation)
+{
+	struct drm_sysfb_device *sysfb = &cdev->sysfb;
+	struct drm_device *dev = &sysfb->dev;
+	const struct drm_format_info *format = sysfb->fb_format;
+	unsigned int width = sysfb->fb_mode.hdisplay;
+	unsigned int height = sysfb->fb_mode.vdisplay;
+	struct drm_plane *primary_plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	size_t nformats;
+	int ret;
+
+	ret = drmm_mode_config_init(dev);
+	if (ret)
+		return ret;
+
+	dev->mode_config.min_width = width;
+	dev->mode_config.max_width = max_t(unsigned int, width, DRM_SHADOW_PLANE_MAX_WIDTH);
+	dev->mode_config.min_height = height;
+	dev->mode_config.max_height = max_t(unsigned int, height, DRM_SHADOW_PLANE_MAX_HEIGHT);
+	dev->mode_config.funcs = &corebootdrm_mode_config_funcs;
+	dev->mode_config.preferred_depth = format->depth;
+
+	/* Primary plane */
+
+	nformats = drm_sysfb_build_fourcc_list(dev, &format->format, 1,
+					       cdev->formats, ARRAY_SIZE(cdev->formats));
+
+	primary_plane = &cdev->primary_plane;
+	ret = drm_universal_plane_init(dev, primary_plane, 0, &corebootdrm_primary_plane_funcs,
+				       cdev->formats, nformats,
+				       corebootdrm_primary_plane_format_modifiers,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(primary_plane, &corebootdrm_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	/* CRTC */
+
+	crtc = &cdev->crtc;
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&corebootdrm_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(crtc, &corebootdrm_crtc_helper_funcs);
+
+	/* Encoder */
+
+	encoder = &cdev->encoder;
+	ret = drm_encoder_init(dev, encoder, &corebootdrm_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	/* Connector */
+
+	connector = &cdev->connector;
+	ret = drm_connector_init(dev, connector, &corebootdrm_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
+	if (ret)
+		return ret;
+	drm_connector_helper_add(connector, &corebootdrm_connector_helper_funcs);
+	drm_connector_set_panel_orientation_with_quirk(connector, orientation,
+						       width, height);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/*
+ * DRM driver
+ */
+
+DEFINE_DRM_GEM_FOPS(corebootdrm_fops);
+
+static struct drm_driver corebootdrm_drm_driver = {
+	DRM_GEM_SHMEM_DRIVER_OPS,
+	DRM_FBDEV_SHMEM_DRIVER_OPS,
+	.name			= DRIVER_NAME,
+	.desc			= DRIVER_DESC,
+	.major			= DRIVER_MAJOR,
+	.minor			= DRIVER_MINOR,
+	.driver_features	= DRIVER_ATOMIC | DRIVER_GEM | DRIVER_MODESET,
+	.fops			= &corebootdrm_fops,
+};
+
+/*
+ * Coreboot driver
+ */
+
+static int corebootdrm_probe(struct platform_device *pdev)
+{
+	const struct lb_framebuffer *fb = dev_get_platdata(&pdev->dev);
+	struct corebootdrm_device *cdev;
+	struct drm_sysfb_device *sysfb;
+	struct drm_device *dev;
+	const struct drm_format_info *format;
+	int width, height, pitch;
+	resource_size_t size;
+	phys_addr_t address;
+	enum drm_panel_orientation orientation;
+	struct resource *res, *mem = NULL;
+	struct resource aperture;
+	void __iomem *screen_base;
+	int ret;
+
+	cdev = devm_drm_dev_alloc(&pdev->dev, &corebootdrm_drm_driver,
+				  struct corebootdrm_device, sysfb.dev);
+	if (IS_ERR(cdev))
+		return PTR_ERR(cdev);
+	platform_set_drvdata(pdev, cdev);
+
+	sysfb = &cdev->sysfb;
+	dev = &sysfb->dev;
+
+	if (!fb) {
+		drm_err(dev, "coreboot framebuffer not found\n");
+		return -EINVAL;
+	} else if (!LB_FRAMEBUFFER_HAS_LFB(fb)) {
+		drm_err(dev, "coreboot framebuffer entry too small\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Hardware settings
+	 */
+
+	format = corebootdrm_get_format_fb(dev, fb);
+	if (!format)
+		return -EINVAL;
+	width = corebootdrm_get_width_fb(dev, fb);
+	if (width < 0)
+		return width;
+	height = corebootdrm_get_height_fb(dev, fb);
+	if (height < 0)
+		return height;
+	pitch = corebootdrm_get_pitch_fb(dev, format, width, fb);
+	if (pitch < 0)
+		return pitch;
+	size = corebootdrm_get_size_fb(dev, height, pitch, fb);
+	if (!size)
+		return -EINVAL;
+	address = corebootdrm_get_address_fb(dev, size, fb);
+	if (!address)
+		return -EINVAL;
+	orientation = corebootdrm_get_orientation_fb(dev, fb);
+
+	sysfb->fb_mode = drm_sysfb_mode(width, height, 0, 0);
+	sysfb->fb_format = format;
+	sysfb->fb_pitch = pitch;
+
+	drm_dbg(dev, "display mode={" DRM_MODE_FMT "}\n", DRM_MODE_ARG(&sysfb->fb_mode));
+	drm_dbg(dev, "framebuffer format=%p4cc, size=%dx%d, pitch=%d byte\n",
+		&format->format, width, height, pitch);
+
+	/*
+	 * Memory management
+	 */
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		drm_err(dev, "memory resource not found\n");
+		return -EINVAL;
+	}
+
+	mem = devm_request_mem_region(&pdev->dev, res->start, resource_size(res),
+				      dev->driver->name);
+	if (!mem) {
+		drm_warn(dev, "could not acquire memory resource at %pr\n", res);
+		/*
+		 * We cannot make this fatal. Sometimes this comes from magic
+		 * spaces our resource handlers simply don't know about. Use
+		 * the memory resource as-is and try to map that instead.
+		 */
+		mem = res;
+	}
+
+	drm_dbg(dev, "using memory resource at %pr\n", mem);
+
+	aperture = DEFINE_RES_MEM(address, size);
+	if (!resource_contains(mem, &aperture)) {
+		drm_err(dev, "framebuffer aperture at invalid memory range %pr\n", &aperture);
+		return -EINVAL;
+	}
+
+	ret = devm_aperture_acquire_for_platform_device(pdev, address, size);
+	if (ret) {
+		drm_err(dev, "could not acquire framebuffer aperture: %d\n", ret);
+		return ret;
+	}
+
+	screen_base = devm_ioremap_wc(&pdev->dev, address, size);
+	if (!screen_base)
+		return -ENOMEM;
+
+	iosys_map_set_vaddr_iomem(&sysfb->fb_addr, screen_base);
+
+	/*
+	 * DRM mode setting and registration
+	 */
+
+	ret = corebootdrm_mode_config_init(cdev, orientation);
+	if (ret)
+		return ret;
+
+	drm_mode_config_reset(dev);
+
+	ret = drm_dev_register(dev, 0);
+	if (ret)
+		return ret;
+
+	drm_client_setup(dev, sysfb->fb_format);
+
+	return 0;
+}
+
+static void corebootdrm_remove(struct platform_device *pdev)
+{
+	struct corebootdrm_device *cdev = platform_get_drvdata(pdev);
+	struct drm_device *dev = &cdev->sysfb.dev;
+
+	drm_dev_unplug(dev);
+}
+
+static struct platform_driver corebootdrm_platform_driver = {
+	.driver = {
+		.name = "coreboot-framebuffer",
+	},
+	.probe = corebootdrm_probe,
+	.remove = corebootdrm_remove,
+};
+
+module_platform_driver(corebootdrm_platform_driver);
+
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb.c b/drivers/gpu/drm/sysfb/drm_sysfb.c
index 308f82153b158..fbfb37d0fae1a 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb.c
+++ b/drivers/gpu/drm/sysfb/drm_sysfb.c
@@ -31,5 +31,29 @@ int drm_sysfb_get_validated_int0(struct drm_device *dev, const char *name,
 }
 EXPORT_SYMBOL(drm_sysfb_get_validated_int0);
 
+const struct drm_format_info *drm_sysfb_get_format(struct drm_device *dev,
+						   const struct drm_sysfb_format *formats,
+						   size_t nformats,
+						   const struct pixel_format *pixel)
+{
+	const struct drm_format_info *format = NULL;
+	size_t i;
+
+	for (i = 0; i < nformats; ++i) {
+		const struct drm_sysfb_format *f = &formats[i];
+
+		if (pixel_format_equal(pixel, &f->pixel)) {
+			format = drm_format_info(f->fourcc);
+			break;
+		}
+	}
+
+	if (!format)
+		drm_warn(dev, "No compatible color format found\n");
+
+	return format;
+}
+EXPORT_SYMBOL(drm_sysfb_get_format);
+
 MODULE_DESCRIPTION("Helpers for DRM sysfb drivers");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
index de96bfe7562c4..b14df5b54bc9a 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
@@ -36,6 +36,10 @@ int drm_sysfb_get_validated_int(struct drm_device *dev, const char *name,
 				u64 value, u32 max);
 int drm_sysfb_get_validated_int0(struct drm_device *dev, const char *name,
 				 u64 value, u32 max);
+const struct drm_format_info *drm_sysfb_get_format(struct drm_device *dev,
+						   const struct drm_sysfb_format *formats,
+						   size_t nformats,
+						   const struct pixel_format *pixel);
 
 #if defined(CONFIG_SCREEN_INFO)
 int drm_sysfb_get_width_si(struct drm_device *dev, const struct screen_info *si);
@@ -48,10 +52,6 @@ int drm_sysfb_get_stride_si(struct drm_device *dev, const struct screen_info *si
 			    unsigned int width, unsigned int height, u64 size);
 u64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_info *si,
 				  unsigned int height, unsigned int stride, u64 size);
-const struct drm_format_info *drm_sysfb_get_format_si(struct drm_device *dev,
-						      const struct drm_sysfb_format *formats,
-						      size_t nformats,
-						      const struct screen_info *si);
 #endif
 
 /*
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c b/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c
index 885864168c54f..749290196c6af 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c
@@ -72,33 +72,3 @@ u64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_in
 	return drm_sysfb_get_validated_size0(dev, "visible size", vsize, size);
 }
 EXPORT_SYMBOL(drm_sysfb_get_visible_size_si);
-
-const struct drm_format_info *drm_sysfb_get_format_si(struct drm_device *dev,
-						      const struct drm_sysfb_format *formats,
-						      size_t nformats,
-						      const struct screen_info *si)
-{
-	const struct drm_format_info *format = NULL;
-	struct pixel_format pixel;
-	size_t i;
-	int ret;
-
-	ret = screen_info_pixel_format(si, &pixel);
-	if (ret)
-		return NULL;
-
-	for (i = 0; i < nformats; ++i) {
-		const struct drm_sysfb_format *f = &formats[i];
-
-		if (pixel_format_equal(&pixel, &f->pixel)) {
-			format = drm_format_info(f->fourcc);
-			break;
-		}
-	}
-
-	if (!format)
-		drm_warn(dev, "No compatible color format found\n");
-
-	return format;
-}
-EXPORT_SYMBOL(drm_sysfb_get_format_si);
diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c
index 50e0aeef709cb..1114359a1e626 100644
--- a/drivers/gpu/drm/sysfb/efidrm.c
+++ b/drivers/gpu/drm/sysfb/efidrm.c
@@ -44,8 +44,14 @@ static const struct drm_format_info *efidrm_get_format_si(struct drm_device *dev
 		{ PIXEL_FORMAT_XBGR8888, DRM_FORMAT_XBGR8888, },
 		{ PIXEL_FORMAT_XRGB2101010, DRM_FORMAT_XRGB2101010, },
 	};
+	struct pixel_format pixel;
+	int ret;
+
+	ret = screen_info_pixel_format(si, &pixel);
+	if (ret)
+		return NULL;
 
-	return drm_sysfb_get_format_si(dev, formats, ARRAY_SIZE(formats), si);
+	return drm_sysfb_get_format(dev, formats, ARRAY_SIZE(formats), &pixel);
 }
 
 static u64 efidrm_get_mem_flags(struct drm_device *dev, resource_size_t start,
diff --git a/drivers/gpu/drm/sysfb/vesadrm.c b/drivers/gpu/drm/sysfb/vesadrm.c
index 0680638b81317..4e00113e5c770 100644
--- a/drivers/gpu/drm/sysfb/vesadrm.c
+++ b/drivers/gpu/drm/sysfb/vesadrm.c
@@ -48,8 +48,14 @@ static const struct drm_format_info *vesadrm_get_format_si(struct drm_device *de
 		{ PIXEL_FORMAT_XBGR8888, DRM_FORMAT_XBGR8888, },
 		{ PIXEL_FORMAT_C8, DRM_FORMAT_C8, },
 	};
+	struct pixel_format pixel;
+	int ret;
+
+	ret = screen_info_pixel_format(si, &pixel);
+	if (ret)
+		return NULL;
 
-	return drm_sysfb_get_format_si(dev, formats, ARRAY_SIZE(formats), si);
+	return drm_sysfb_get_format(dev, formats, ARRAY_SIZE(formats), &pixel);
 }
 
 /*
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index 2c5aefe9621af..7f25c50621c94 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -14,6 +14,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
+#include <linux/tegra-mipi-cal.h>
 
 #include <video/mipi_display.h>
 
diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
index a6fa196c28132..10d993b8d043a 100644
--- a/drivers/gpu/drm/tegra/hub.c
+++ b/drivers/gpu/drm/tegra/hub.c
@@ -825,7 +825,22 @@ static void tegra_display_hub_destroy_state(struct drm_private_obj *obj,
 	kfree(hub_state);
 }
 
+static struct drm_private_state *
+tegra_display_hub_create_state(struct drm_private_obj *obj)
+{
+	struct tegra_display_hub_state *hub_state;
+
+	hub_state = kzalloc_obj(*hub_state);
+	if (!hub_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &hub_state->base);
+
+	return &hub_state->base;
+}
+
 static const struct drm_private_state_funcs tegra_display_hub_state_funcs = {
+	.atomic_create_state = tegra_display_hub_create_state,
 	.atomic_duplicate_state = tegra_display_hub_duplicate_state,
 	.atomic_destroy_state = tegra_display_hub_destroy_state,
 };
@@ -941,13 +956,8 @@ static int tegra_display_hub_init(struct host1x_client *client)
 	struct tegra_display_hub *hub = to_tegra_display_hub(client);
 	struct drm_device *drm = dev_get_drvdata(client->host);
 	struct tegra_drm *tegra = drm->dev_private;
-	struct tegra_display_hub_state *state;
-
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
 
-	drm_atomic_private_obj_init(drm, &hub->base, &state->base,
+	drm_atomic_private_obj_init(drm, &hub->base,
 				    &tegra_display_hub_state_funcs);
 
 	tegra->hub = hub;
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index 87d5d5f9332ac..d2e2e3d8349ae 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -7,7 +7,6 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_atomic_test.o \
 	drm_atomic_state_test.o \
 	drm_bridge_test.o \
-	drm_buddy_test.o \
 	drm_cmdline_parser_test.o \
 	drm_connector_test.o \
 	drm_damage_helper_test.o \
diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c
deleted file mode 100644
index e6f8459c6c54e..0000000000000
--- a/drivers/gpu/drm/tests/drm_buddy_test.c
+++ /dev/null
@@ -1,928 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2019 Intel Corporation
- * Copyright © 2022 Maíra Canal <mairacanal@riseup.net>
- */
-
-#include <kunit/test.h>
-
-#include <linux/prime_numbers.h>
-#include <linux/sched/signal.h>
-#include <linux/sizes.h>
-
-#include <drm/drm_buddy.h>
-
-#include "../lib/drm_random.h"
-
-static unsigned int random_seed;
-
-static inline u64 get_size(int order, u64 chunk_size)
-{
-	return (1 << order) * chunk_size;
-}
-
-static void drm_test_buddy_fragmentation_performance(struct kunit *test)
-{
-	struct drm_buddy_block *block, *tmp;
-	int num_blocks, i, ret, count = 0;
-	LIST_HEAD(allocated_blocks);
-	unsigned long elapsed_ms;
-	LIST_HEAD(reverse_list);
-	LIST_HEAD(test_blocks);
-	LIST_HEAD(clear_list);
-	LIST_HEAD(dirty_list);
-	LIST_HEAD(free_list);
-	struct drm_buddy mm;
-	u64 mm_size = SZ_4G;
-	ktime_t start, end;
-
-	/*
-	 * Allocation under severe fragmentation
-	 *
-	 * Create severe fragmentation by allocating the entire 4 GiB address space
-	 * as tiny 8 KiB blocks but forcing a 64 KiB alignment. The resulting pattern
-	 * leaves many scattered holes. Split the allocations into two groups and
-	 * return them with different flags to block coalescing, then repeatedly
-	 * allocate and free 64 KiB blocks while timing the loop. This stresses how
-	 * quickly the allocator can satisfy larger, aligned requests from a pool of
-	 * highly fragmented space.
-	 */
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
-			       "buddy_init failed\n");
-
-	num_blocks = mm_size / SZ_64K;
-
-	start = ktime_get();
-	/* Allocate with maximum fragmentation - 8K blocks with 64K alignment */
-	for (i = 0; i < num_blocks; i++)
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, SZ_8K, SZ_64K,
-								    &allocated_blocks, 0),
-					"buddy_alloc hit an error size=%u\n", SZ_8K);
-
-	list_for_each_entry_safe(block, tmp, &allocated_blocks, link) {
-		if (count % 4 == 0 || count % 4 == 3)
-			list_move_tail(&block->link, &clear_list);
-		else
-			list_move_tail(&block->link, &dirty_list);
-		count++;
-	}
-
-	/* Free with different flags to ensure no coalescing */
-	drm_buddy_free_list(&mm, &clear_list, DRM_BUDDY_CLEARED);
-	drm_buddy_free_list(&mm, &dirty_list, 0);
-
-	for (i = 0; i < num_blocks; i++)
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, SZ_64K, SZ_64K,
-								    &test_blocks, 0),
-					"buddy_alloc hit an error size=%u\n", SZ_64K);
-	drm_buddy_free_list(&mm, &test_blocks, 0);
-
-	end = ktime_get();
-	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
-
-	kunit_info(test, "Fragmented allocation took %lu ms\n", elapsed_ms);
-
-	drm_buddy_fini(&mm);
-
-	/*
-	 * Reverse free order under fragmentation
-	 *
-	 * Construct a fragmented 4 GiB space by allocating every 8 KiB block with
-	 * 64 KiB alignment, creating a dense scatter of small regions. Half of the
-	 * blocks are selectively freed to form sparse gaps, while the remaining
-	 * allocations are preserved, reordered in reverse, and released back with
-	 * the cleared flag. This models a pathological reverse-ordered free pattern
-	 * and measures how quickly the allocator can merge and reclaim space when
-	 * deallocation occurs in the opposite order of allocation, exposing the
-	 * cost difference between a linear freelist scan and an ordered tree lookup.
-	 */
-	ret = drm_buddy_init(&mm, mm_size, SZ_4K);
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
-	start = ktime_get();
-	/* Allocate maximum fragmentation */
-	for (i = 0; i < num_blocks; i++)
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, SZ_8K, SZ_64K,
-								    &allocated_blocks, 0),
-					"buddy_alloc hit an error size=%u\n", SZ_8K);
-
-	list_for_each_entry_safe(block, tmp, &allocated_blocks, link) {
-		if (count % 2 == 0)
-			list_move_tail(&block->link, &free_list);
-		count++;
-	}
-	drm_buddy_free_list(&mm, &free_list, DRM_BUDDY_CLEARED);
-
-	list_for_each_entry_safe_reverse(block, tmp, &allocated_blocks, link)
-		list_move(&block->link, &reverse_list);
-	drm_buddy_free_list(&mm, &reverse_list, DRM_BUDDY_CLEARED);
-
-	end = ktime_get();
-	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
-
-	kunit_info(test, "Reverse-ordered free took %lu ms\n", elapsed_ms);
-
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_range_bias(struct kunit *test)
-{
-	u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
-	DRM_RND_STATE(prng, random_seed);
-	unsigned int i, count, *order;
-	struct drm_buddy_block *block;
-	unsigned long flags;
-	struct drm_buddy mm;
-	LIST_HEAD(allocated);
-
-	bias_size = SZ_1M;
-	ps = roundup_pow_of_two(prandom_u32_state(&prng) % bias_size);
-	ps = max(SZ_4K, ps);
-	mm_size = (SZ_8M-1) & ~(ps-1); /* Multiple roots */
-
-	kunit_info(test, "mm_size=%u, ps=%u\n", mm_size, ps);
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
-			       "buddy_init failed\n");
-
-	count = mm_size / bias_size;
-	order = drm_random_order(count, &prng);
-	KUNIT_EXPECT_TRUE(test, order);
-
-	/*
-	 * Idea is to split the address space into uniform bias ranges, and then
-	 * in some random order allocate within each bias, using various
-	 * patterns within. This should detect if allocations leak out from a
-	 * given bias, for example.
-	 */
-
-	for (i = 0; i < count; i++) {
-		LIST_HEAD(tmp);
-		u32 size;
-
-		bias_start = order[i] * bias_size;
-		bias_end = bias_start + bias_size;
-		bias_rem = bias_size;
-
-		/* internal round_up too big */
-		KUNIT_ASSERT_TRUE_MSG(test,
-				      drm_buddy_alloc_blocks(&mm, bias_start,
-							     bias_end, bias_size + ps, bias_size,
-							     &allocated,
-							     DRM_BUDDY_RANGE_ALLOCATION),
-				      "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-				      bias_start, bias_end, bias_size, bias_size);
-
-		/* size too big */
-		KUNIT_ASSERT_TRUE_MSG(test,
-				      drm_buddy_alloc_blocks(&mm, bias_start,
-							     bias_end, bias_size + ps, ps,
-							     &allocated,
-							     DRM_BUDDY_RANGE_ALLOCATION),
-				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
-				      bias_start, bias_end, bias_size + ps, ps);
-
-		/* bias range too small for size */
-		KUNIT_ASSERT_TRUE_MSG(test,
-				      drm_buddy_alloc_blocks(&mm, bias_start + ps,
-							     bias_end, bias_size, ps,
-							     &allocated,
-							     DRM_BUDDY_RANGE_ALLOCATION),
-				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
-				      bias_start + ps, bias_end, bias_size, ps);
-
-		/* bias misaligned */
-		KUNIT_ASSERT_TRUE_MSG(test,
-				      drm_buddy_alloc_blocks(&mm, bias_start + ps,
-							     bias_end - ps,
-							     bias_size >> 1, bias_size >> 1,
-							     &allocated,
-							     DRM_BUDDY_RANGE_ALLOCATION),
-				      "buddy_alloc h didn't fail with bias(%x-%x), size=%u, ps=%u\n",
-				      bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1);
-
-		/* single big page */
-		KUNIT_ASSERT_FALSE_MSG(test,
-				       drm_buddy_alloc_blocks(&mm, bias_start,
-							      bias_end, bias_size, bias_size,
-							      &tmp,
-							      DRM_BUDDY_RANGE_ALLOCATION),
-				       "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n",
-				       bias_start, bias_end, bias_size, bias_size);
-		drm_buddy_free_list(&mm, &tmp, 0);
-
-		/* single page with internal round_up */
-		KUNIT_ASSERT_FALSE_MSG(test,
-				       drm_buddy_alloc_blocks(&mm, bias_start,
-							      bias_end, ps, bias_size,
-							      &tmp,
-							      DRM_BUDDY_RANGE_ALLOCATION),
-				       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-				       bias_start, bias_end, ps, bias_size);
-		drm_buddy_free_list(&mm, &tmp, 0);
-
-		/* random size within */
-		size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
-		if (size)
-			KUNIT_ASSERT_FALSE_MSG(test,
-					       drm_buddy_alloc_blocks(&mm, bias_start,
-								      bias_end, size, ps,
-								      &tmp,
-								      DRM_BUDDY_RANGE_ALLOCATION),
-					       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-					       bias_start, bias_end, size, ps);
-
-		bias_rem -= size;
-		/* too big for current avail */
-		KUNIT_ASSERT_TRUE_MSG(test,
-				      drm_buddy_alloc_blocks(&mm, bias_start,
-							     bias_end, bias_rem + ps, ps,
-							     &allocated,
-							     DRM_BUDDY_RANGE_ALLOCATION),
-				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
-				      bias_start, bias_end, bias_rem + ps, ps);
-
-		if (bias_rem) {
-			/* random fill of the remainder */
-			size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
-			size = max(size, ps);
-
-			KUNIT_ASSERT_FALSE_MSG(test,
-					       drm_buddy_alloc_blocks(&mm, bias_start,
-								      bias_end, size, ps,
-								      &allocated,
-								      DRM_BUDDY_RANGE_ALLOCATION),
-					       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-					       bias_start, bias_end, size, ps);
-			/*
-			 * Intentionally allow some space to be left
-			 * unallocated, and ideally not always on the bias
-			 * boundaries.
-			 */
-			drm_buddy_free_list(&mm, &tmp, 0);
-		} else {
-			list_splice_tail(&tmp, &allocated);
-		}
-	}
-
-	kfree(order);
-	drm_buddy_free_list(&mm, &allocated, 0);
-	drm_buddy_fini(&mm);
-
-	/*
-	 * Something more free-form. Idea is to pick a random starting bias
-	 * range within the address space and then start filling it up. Also
-	 * randomly grow the bias range in both directions as we go along. This
-	 * should give us bias start/end which is not always uniform like above,
-	 * and in some cases will require the allocator to jump over already
-	 * allocated nodes in the middle of the address space.
-	 */
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
-			       "buddy_init failed\n");
-
-	bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
-	bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
-	bias_end = max(bias_end, bias_start + ps);
-	bias_rem = bias_end - bias_start;
-
-	do {
-		u32 size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
-
-		KUNIT_ASSERT_FALSE_MSG(test,
-				       drm_buddy_alloc_blocks(&mm, bias_start,
-							      bias_end, size, ps,
-							      &allocated,
-							      DRM_BUDDY_RANGE_ALLOCATION),
-				       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-				       bias_start, bias_end, size, ps);
-		bias_rem -= size;
-
-		/*
-		 * Try to randomly grow the bias range in both directions, or
-		 * only one, or perhaps don't grow at all.
-		 */
-		do {
-			u32 old_bias_start = bias_start;
-			u32 old_bias_end = bias_end;
-
-			if (bias_start)
-				bias_start -= round_up(prandom_u32_state(&prng) % bias_start, ps);
-			if (bias_end != mm_size)
-				bias_end += round_up(prandom_u32_state(&prng) % (mm_size - bias_end), ps);
-
-			bias_rem += old_bias_start - bias_start;
-			bias_rem += bias_end - old_bias_end;
-		} while (!bias_rem && (bias_start || bias_end != mm_size));
-	} while (bias_rem);
-
-	KUNIT_ASSERT_EQ(test, bias_start, 0);
-	KUNIT_ASSERT_EQ(test, bias_end, mm_size);
-	KUNIT_ASSERT_TRUE_MSG(test,
-			      drm_buddy_alloc_blocks(&mm, bias_start, bias_end,
-						     ps, ps,
-						     &allocated,
-						     DRM_BUDDY_RANGE_ALLOCATION),
-			      "buddy_alloc passed with bias(%x-%x), size=%u\n",
-			      bias_start, bias_end, ps);
-
-	drm_buddy_free_list(&mm, &allocated, 0);
-	drm_buddy_fini(&mm);
-
-	/*
-	 * Allocate cleared blocks in the bias range when the DRM buddy's clear avail is
-	 * zero. This will validate the bias range allocation in scenarios like system boot
-	 * when no cleared blocks are available and exercise the fallback path too. The resulting
-	 * blocks should always be dirty.
-	 */
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
-			       "buddy_init failed\n");
-
-	bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
-	bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
-	bias_end = max(bias_end, bias_start + ps);
-	bias_rem = bias_end - bias_start;
-
-	flags = DRM_BUDDY_CLEAR_ALLOCATION | DRM_BUDDY_RANGE_ALLOCATION;
-	size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
-
-	KUNIT_ASSERT_FALSE_MSG(test,
-			       drm_buddy_alloc_blocks(&mm, bias_start,
-						      bias_end, size, ps,
-						      &allocated,
-						      flags),
-			       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
-			       bias_start, bias_end, size, ps);
-
-	list_for_each_entry(block, &allocated, link)
-		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
-
-	drm_buddy_free_list(&mm, &allocated, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_clear(struct kunit *test)
-{
-	unsigned long n_pages, total, i = 0;
-	const unsigned long ps = SZ_4K;
-	struct drm_buddy_block *block;
-	const int max_order = 12;
-	LIST_HEAD(allocated);
-	struct drm_buddy mm;
-	unsigned int order;
-	u32 mm_size, size;
-	LIST_HEAD(dirty);
-	LIST_HEAD(clean);
-
-	mm_size = SZ_4K << max_order;
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
-
-	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
-
-	/*
-	 * Idea is to allocate and free some random portion of the address space,
-	 * returning those pages as non-dirty and randomly alternate between
-	 * requesting dirty and non-dirty pages (not going over the limit
-	 * we freed as non-dirty), putting that into two separate lists.
-	 * Loop over both lists at the end checking that the dirty list
-	 * is indeed all dirty pages and vice versa. Free it all again,
-	 * keeping the dirty/clear status.
-	 */
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							    5 * ps, ps, &allocated,
-							    DRM_BUDDY_TOPDOWN_ALLOCATION),
-				"buddy_alloc hit an error size=%lu\n", 5 * ps);
-	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
-
-	n_pages = 10;
-	do {
-		unsigned long flags;
-		struct list_head *list;
-		int slot = i % 2;
-
-		if (slot == 0) {
-			list = &dirty;
-			flags = 0;
-		} else {
-			list = &clean;
-			flags = DRM_BUDDY_CLEAR_ALLOCATION;
-		}
-
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-								    ps, ps, list,
-								    flags),
-					"buddy_alloc hit an error size=%lu\n", ps);
-	} while (++i < n_pages);
-
-	list_for_each_entry(block, &clean, link)
-		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), true);
-
-	list_for_each_entry(block, &dirty, link)
-		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
-
-	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
-
-	/*
-	 * Trying to go over the clear limit for some allocation.
-	 * The allocation should never fail with reasonable page-size.
-	 */
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							    10 * ps, ps, &clean,
-							    DRM_BUDDY_CLEAR_ALLOCATION),
-				"buddy_alloc hit an error size=%lu\n", 10 * ps);
-
-	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
-	drm_buddy_free_list(&mm, &dirty, 0);
-	drm_buddy_fini(&mm);
-
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
-
-	/*
-	 * Create a new mm. Intentionally fragment the address space by creating
-	 * two alternating lists. Free both lists, one as dirty the other as clean.
-	 * Try to allocate double the previous size with matching min_page_size. The
-	 * allocation should never fail as it calls the force_merge. Also check that
-	 * the page is always dirty after force_merge. Free the page as dirty, then
-	 * repeat the whole thing, increment the order until we hit the max_order.
-	 */
-
-	i = 0;
-	n_pages = mm_size / ps;
-	do {
-		struct list_head *list;
-		int slot = i % 2;
-
-		if (slot == 0)
-			list = &dirty;
-		else
-			list = &clean;
-
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-								    ps, ps, list, 0),
-					"buddy_alloc hit an error size=%lu\n", ps);
-	} while (++i < n_pages);
-
-	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
-	drm_buddy_free_list(&mm, &dirty, 0);
-
-	order = 1;
-	do {
-		size = SZ_4K << order;
-
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-								    size, size, &allocated,
-								    DRM_BUDDY_CLEAR_ALLOCATION),
-					"buddy_alloc hit an error size=%u\n", size);
-		total = 0;
-		list_for_each_entry(block, &allocated, link) {
-			if (size != mm_size)
-				KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
-			total += drm_buddy_block_size(&mm, block);
-		}
-		KUNIT_EXPECT_EQ(test, total, size);
-
-		drm_buddy_free_list(&mm, &allocated, 0);
-	} while (++order <= max_order);
-
-	drm_buddy_fini(&mm);
-
-	/*
-	 * Create a new mm with a non power-of-two size. Allocate a random size from each
-	 * root, free as cleared and then call fini. This will ensure the multi-root
-	 * force merge during fini.
-	 */
-	mm_size = (SZ_4K << max_order) + (SZ_4K << (max_order - 2));
-
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
-	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order,
-							    4 * ps, ps, &allocated,
-							    DRM_BUDDY_RANGE_ALLOCATION),
-				"buddy_alloc hit an error size=%lu\n", 4 * ps);
-	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order,
-							    2 * ps, ps, &allocated,
-							    DRM_BUDDY_CLEAR_ALLOCATION),
-				"buddy_alloc hit an error size=%lu\n", 2 * ps);
-	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, SZ_4K << max_order, mm_size,
-							    ps, ps, &allocated,
-							    DRM_BUDDY_RANGE_ALLOCATION),
-				"buddy_alloc hit an error size=%lu\n", ps);
-	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_contiguous(struct kunit *test)
-{
-	const unsigned long ps = SZ_4K, mm_size = 16 * 3 * SZ_4K;
-	unsigned long i, n_pages, total;
-	struct drm_buddy_block *block;
-	struct drm_buddy mm;
-	LIST_HEAD(left);
-	LIST_HEAD(middle);
-	LIST_HEAD(right);
-	LIST_HEAD(allocated);
-
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
-
-	/*
-	 * Idea is to fragment the address space by alternating block
-	 * allocations between three different lists; one for left, middle and
-	 * right. We can then free a list to simulate fragmentation. In
-	 * particular we want to exercise the DRM_BUDDY_CONTIGUOUS_ALLOCATION,
-	 * including the try_harder path.
-	 */
-
-	i = 0;
-	n_pages = mm_size / ps;
-	do {
-		struct list_head *list;
-		int slot = i % 3;
-
-		if (slot == 0)
-			list = &left;
-		else if (slot == 1)
-			list = &middle;
-		else
-			list = &right;
-		KUNIT_ASSERT_FALSE_MSG(test,
-				       drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							      ps, ps, list, 0),
-				       "buddy_alloc hit an error size=%lu\n",
-				       ps);
-	} while (++i < n_pages);
-
-	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							   3 * ps, ps, &allocated,
-							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
-
-	drm_buddy_free_list(&mm, &middle, 0);
-	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							   3 * ps, ps, &allocated,
-							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
-	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							   2 * ps, ps, &allocated,
-							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc didn't error size=%lu\n", 2 * ps);
-
-	drm_buddy_free_list(&mm, &right, 0);
-	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							   3 * ps, ps, &allocated,
-							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
-	/*
-	 * At this point we should have enough contiguous space for 2 blocks,
-	 * however they are never buddies (since we freed middle and right) so
-	 * will require the try_harder logic to find them.
-	 */
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							    2 * ps, ps, &allocated,
-							    DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc hit an error size=%lu\n", 2 * ps);
-
-	drm_buddy_free_list(&mm, &left, 0);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
-							    3 * ps, ps, &allocated,
-							    DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc hit an error size=%lu\n", 3 * ps);
-
-	total = 0;
-	list_for_each_entry(block, &allocated, link)
-		total += drm_buddy_block_size(&mm, block);
-
-	KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3);
-
-	drm_buddy_free_list(&mm, &allocated, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_pathological(struct kunit *test)
-{
-	u64 mm_size, size, start = 0;
-	struct drm_buddy_block *block;
-	const int max_order = 3;
-	unsigned long flags = 0;
-	int order, top;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	LIST_HEAD(holes);
-	LIST_HEAD(tmp);
-
-	/*
-	 * Create a pot-sized mm, then allocate one of each possible
-	 * order within. This should leave the mm with exactly one
-	 * page left. Free the largest block, then whittle down again.
-	 * Eventually we will have a fully 50% fragmented mm.
-	 */
-
-	mm_size = SZ_4K << max_order;
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
-			       "buddy_init failed\n");
-
-	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
-
-	for (top = max_order; top; top--) {
-		/* Make room by freeing the largest allocated block */
-		block = list_first_entry_or_null(&blocks, typeof(*block), link);
-		if (block) {
-			list_del(&block->link);
-			drm_buddy_free_block(&mm, block);
-		}
-
-		for (order = top; order--;) {
-			size = get_size(order, mm.chunk_size);
-			KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start,
-									    mm_size, size, size,
-										&tmp, flags),
-					"buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
-					order, top);
-
-			block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-			KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-			list_move_tail(&block->link, &blocks);
-		}
-
-		/* There should be one final page for this sub-allocation */
-		size = get_size(0, mm.chunk_size);
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								    size, size, &tmp, flags),
-							   "buddy_alloc hit -ENOMEM for hole\n");
-
-		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-		list_move_tail(&block->link, &holes);
-
-		size = get_size(top, mm.chunk_size);
-		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								   size, size, &tmp, flags),
-							  "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
-							  top, max_order);
-	}
-
-	drm_buddy_free_list(&mm, &holes, 0);
-
-	/* Nothing larger than blocks of chunk_size now available */
-	for (order = 1; order <= max_order; order++) {
-		size = get_size(order, mm.chunk_size);
-		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								   size, size, &tmp, flags),
-							  "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
-							  order);
-	}
-
-	list_splice_tail(&holes, &blocks);
-	drm_buddy_free_list(&mm, &blocks, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
-{
-	u64 mm_size, size, start = 0;
-	struct drm_buddy_block *block, *bn;
-	const unsigned int max_order = 16;
-	unsigned long flags = 0;
-	struct drm_buddy mm;
-	unsigned int order;
-	LIST_HEAD(blocks);
-	LIST_HEAD(tmp);
-
-	/*
-	 * Create a pot-sized mm, then allocate one of each possible
-	 * order within. This should leave the mm with exactly one
-	 * page left.
-	 */
-
-	mm_size = SZ_4K << max_order;
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
-			       "buddy_init failed\n");
-
-	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
-
-	for (order = 0; order < max_order; order++) {
-		size = get_size(order, mm.chunk_size);
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								    size, size, &tmp, flags),
-							   "buddy_alloc hit -ENOMEM with order=%d\n",
-							   order);
-
-		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-		list_move_tail(&block->link, &blocks);
-	}
-
-	/* And now the last remaining block available */
-	size = get_size(0, mm.chunk_size);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-							    size, size, &tmp, flags),
-						   "buddy_alloc hit -ENOMEM on final alloc\n");
-
-	block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-	list_move_tail(&block->link, &blocks);
-
-	/* Should be completely full! */
-	for (order = max_order; order--;) {
-		size = get_size(order, mm.chunk_size);
-		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								   size, size, &tmp, flags),
-							  "buddy_alloc unexpectedly succeeded, it should be full!");
-	}
-
-	block = list_last_entry(&blocks, typeof(*block), link);
-	list_del(&block->link);
-	drm_buddy_free_block(&mm, block);
-
-	/* As we free in increasing size, we make available larger blocks */
-	order = 1;
-	list_for_each_entry_safe(block, bn, &blocks, link) {
-		list_del(&block->link);
-		drm_buddy_free_block(&mm, block);
-
-		size = get_size(order, mm.chunk_size);
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								    size, size, &tmp, flags),
-							   "buddy_alloc hit -ENOMEM with order=%d\n",
-							   order);
-
-		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-		list_del(&block->link);
-		drm_buddy_free_block(&mm, block);
-		order++;
-	}
-
-	/* To confirm, now the whole mm should be available */
-	size = get_size(max_order, mm.chunk_size);
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-							    size, size, &tmp, flags),
-						   "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
-						   max_order);
-
-	block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-	list_del(&block->link);
-	drm_buddy_free_block(&mm, block);
-	drm_buddy_free_list(&mm, &blocks, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_optimistic(struct kunit *test)
-{
-	u64 mm_size, size, start = 0;
-	struct drm_buddy_block *block;
-	unsigned long flags = 0;
-	const int max_order = 16;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	LIST_HEAD(tmp);
-	int order;
-
-	/*
-	 * Create a mm with one block of each order available, and
-	 * try to allocate them all.
-	 */
-
-	mm_size = SZ_4K * ((1 << (max_order + 1)) - 1);
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
-			       "buddy_init failed\n");
-
-	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
-
-	for (order = 0; order <= max_order; order++) {
-		size = get_size(order, mm.chunk_size);
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-								    size, size, &tmp, flags),
-							   "buddy_alloc hit -ENOMEM with order=%d\n",
-							   order);
-
-		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-		list_move_tail(&block->link, &blocks);
-	}
-
-	/* Should be completely full! */
-	size = get_size(0, mm.chunk_size);
-	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
-							   size, size, &tmp, flags),
-						  "buddy_alloc unexpectedly succeeded, it should be full!");
-
-	drm_buddy_free_list(&mm, &blocks, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_limit(struct kunit *test)
-{
-	u64 size = U64_MAX, start = 0;
-	struct drm_buddy_block *block;
-	unsigned long flags = 0;
-	LIST_HEAD(allocated);
-	struct drm_buddy mm;
-
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, SZ_4K));
-
-	KUNIT_EXPECT_EQ_MSG(test, mm.max_order, DRM_BUDDY_MAX_ORDER,
-			    "mm.max_order(%d) != %d\n", mm.max_order,
-						DRM_BUDDY_MAX_ORDER);
-
-	size = mm.chunk_size << mm.max_order;
-	KUNIT_EXPECT_FALSE(test, drm_buddy_alloc_blocks(&mm, start, size, size,
-							mm.chunk_size, &allocated, flags));
-
-	block = list_first_entry_or_null(&allocated, struct drm_buddy_block, link);
-	KUNIT_EXPECT_TRUE(test, block);
-
-	KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), mm.max_order,
-			    "block order(%d) != %d\n",
-						drm_buddy_block_order(block), mm.max_order);
-
-	KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_size(&mm, block),
-			    BIT_ULL(mm.max_order) * mm.chunk_size,
-						"block size(%llu) != %llu\n",
-						drm_buddy_block_size(&mm, block),
-						BIT_ULL(mm.max_order) * mm.chunk_size);
-
-	drm_buddy_free_list(&mm, &allocated, 0);
-	drm_buddy_fini(&mm);
-}
-
-static void drm_test_buddy_alloc_exceeds_max_order(struct kunit *test)
-{
-	u64 mm_size = SZ_8G + SZ_2G, size = SZ_8G + SZ_1G, min_block_size = SZ_8G;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	int err;
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
-			       "buddy_init failed\n");
-
-	/* CONTIGUOUS allocation should succeed via try_harder fallback */
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, size,
-							    SZ_4K, &blocks,
-							    DRM_BUDDY_CONTIGUOUS_ALLOCATION),
-			       "buddy_alloc hit an error size=%llu\n", size);
-	drm_buddy_free_list(&mm, &blocks, 0);
-
-	/* Non-CONTIGUOUS with large min_block_size should return -EINVAL */
-	err = drm_buddy_alloc_blocks(&mm, 0, mm_size, size, min_block_size, &blocks, 0);
-	KUNIT_EXPECT_EQ(test, err, -EINVAL);
-
-	/* Non-CONTIGUOUS + RANGE with large min_block_size should return -EINVAL */
-	err = drm_buddy_alloc_blocks(&mm, 0, mm_size, size, min_block_size, &blocks,
-				     DRM_BUDDY_RANGE_ALLOCATION);
-	KUNIT_EXPECT_EQ(test, err, -EINVAL);
-
-	/* CONTIGUOUS + RANGE should return -EINVAL (no try_harder for RANGE) */
-	err = drm_buddy_alloc_blocks(&mm, 0, mm_size, size, SZ_4K, &blocks,
-				     DRM_BUDDY_CONTIGUOUS_ALLOCATION | DRM_BUDDY_RANGE_ALLOCATION);
-	KUNIT_EXPECT_EQ(test, err, -EINVAL);
-
-	drm_buddy_fini(&mm);
-}
-
-static int drm_buddy_suite_init(struct kunit_suite *suite)
-{
-	while (!random_seed)
-		random_seed = get_random_u32();
-
-	kunit_info(suite, "Testing DRM buddy manager, with random_seed=0x%x\n",
-		   random_seed);
-
-	return 0;
-}
-
-static struct kunit_case drm_buddy_tests[] = {
-	KUNIT_CASE(drm_test_buddy_alloc_limit),
-	KUNIT_CASE(drm_test_buddy_alloc_optimistic),
-	KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
-	KUNIT_CASE(drm_test_buddy_alloc_pathological),
-	KUNIT_CASE(drm_test_buddy_alloc_contiguous),
-	KUNIT_CASE(drm_test_buddy_alloc_clear),
-	KUNIT_CASE(drm_test_buddy_alloc_range_bias),
-	KUNIT_CASE(drm_test_buddy_fragmentation_performance),
-	KUNIT_CASE(drm_test_buddy_alloc_exceeds_max_order),
-	{}
-};
-
-static struct kunit_suite drm_buddy_test_suite = {
-	.name = "drm_buddy",
-	.suite_init = drm_buddy_suite_init,
-	.test_cases = drm_buddy_tests,
-};
-
-kunit_test_suite(drm_buddy_test_suite);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_DESCRIPTION("Kunit test for drm_buddy functions");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_exec_test.c b/drivers/gpu/drm/tests/drm_exec_test.c
index 3a20c788c51f8..2fc47f3b463b7 100644
--- a/drivers/gpu/drm/tests/drm_exec_test.c
+++ b/drivers/gpu/drm/tests/drm_exec_test.c
@@ -16,8 +16,6 @@
 #include <drm/drm_gem.h>
 #include <drm/drm_kunit_helpers.h>
 
-#include "../lib/drm_random.h"
-
 struct drm_exec_priv {
 	struct device *dev;
 	struct drm_device *drm;
diff --git a/drivers/gpu/drm/tests/drm_mm_test.c b/drivers/gpu/drm/tests/drm_mm_test.c
index f4f473f878384..b12aa26e1a268 100644
--- a/drivers/gpu/drm/tests/drm_mm_test.c
+++ b/drivers/gpu/drm/tests/drm_mm_test.c
@@ -16,8 +16,6 @@
 #include <drm/drm_mm.h>
 #include <drm/drm_print.h>
 
-#include "../lib/drm_random.h"
-
 enum {
 	BEST,
 	BOTTOMUP,
diff --git a/drivers/gpu/drm/tilcdc/Kconfig b/drivers/gpu/drm/tilcdc/Kconfig
index 24f9a245ba593..89df7528756c8 100644
--- a/drivers/gpu/drm/tilcdc/Kconfig
+++ b/drivers/gpu/drm/tilcdc/Kconfig
@@ -6,6 +6,8 @@ config DRM_TILCDC
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	select DRM_BRIDGE
+	select DRM_DISPLAY_HELPER
+	select DRM_BRIDGE_CONNECTOR
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
 	select BACKLIGHT_CLASS_DEVICE
@@ -14,3 +16,19 @@ config DRM_TILCDC
 	  controller, for example AM33xx in beagle-bone, DA8xx, or
 	  OMAP-L1xx.  This driver replaces the FB_DA8XX fbdev driver.
 
+config DRM_TILCDC_PANEL_LEGACY
+	bool "Support device tree blobs using TI LCDC Panel binding"
+	default y
+	depends on DRM_TILCDC
+	depends on OF
+	depends on BACKLIGHT_CLASS_DEVICE
+	depends on PM
+	select OF_OVERLAY
+	select DRM_PANEL_SIMPLE
+	help
+	  Modifies the live device tree at early boot to convert the legacy
+	  "ti,tilcdc,panel" devicetree node to the standard panel-dpi node.
+	  This allows to maintain backward compatibility for boards which
+	  were using the deprecated tilcdc_panel driver.
+	  If you find "ti,tilcdc,panel"-string from your DTB, you probably
+	  need this. Otherwise you do not.
diff --git a/drivers/gpu/drm/tilcdc/Makefile b/drivers/gpu/drm/tilcdc/Makefile
index f5190477de721..c6b484dad711a 100644
--- a/drivers/gpu/drm/tilcdc/Makefile
+++ b/drivers/gpu/drm/tilcdc/Makefile
@@ -6,8 +6,9 @@ endif
 tilcdc-y := \
 	tilcdc_plane.o \
 	tilcdc_crtc.o \
-	tilcdc_panel.o \
-	tilcdc_external.o \
+	tilcdc_encoder.o \
 	tilcdc_drv.o
 
 obj-$(CONFIG_DRM_TILCDC)	+= tilcdc.o
+obj-$(CONFIG_DRM_TILCDC_PANEL_LEGACY)	+= tilcdc_panel_legacy.o \
+					   tilcdc_panel_legacy.dtbo.o
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 52c95131af5af..4d3b7059cd5b2 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -16,6 +16,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
@@ -30,8 +31,7 @@
 struct tilcdc_crtc {
 	struct drm_crtc base;
 
-	struct drm_plane primary;
-	const struct tilcdc_panel_info *info;
+	struct tilcdc_plane *primary;
 	struct drm_pending_vblank_event *event;
 	struct mutex enable_lock;
 	bool enabled;
@@ -47,9 +47,6 @@ struct tilcdc_crtc {
 
 	struct drm_framebuffer *next_fb;
 
-	/* Only set if an external encoder is connected */
-	bool simulate_vesa_sync;
-
 	int sync_lost_count;
 	bool frame_intact;
 	struct work_struct recover_work;
@@ -63,7 +60,7 @@ struct tilcdc_crtc {
 static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	struct drm_gem_dma_object *gem;
 	dma_addr_t start, end;
 	u64 dma_base_and_ceiling;
@@ -98,7 +95,7 @@ static void tilcdc_crtc_load_palette(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	int ret;
 
 	reinit_completion(&tilcdc_crtc->palette_loaded);
@@ -128,7 +125,7 @@ static void tilcdc_crtc_load_palette(struct drm_crtc *crtc)
 	ret = wait_for_completion_timeout(&tilcdc_crtc->palette_loaded,
 					  msecs_to_jiffies(50));
 	if (ret == 0)
-		dev_err(dev->dev, "%s: Palette loading timeout", __func__);
+		drm_err(dev, "%s: Palette loading timeout", __func__);
 
 	/* Disable LCDC DMA and DMA Palette Loaded Interrupt. */
 	tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
@@ -140,7 +137,7 @@ static void tilcdc_crtc_load_palette(struct drm_crtc *crtc)
 
 static void tilcdc_crtc_enable_irqs(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	tilcdc_clear_irqstatus(dev, 0xffffffff);
 
@@ -157,7 +154,7 @@ static void tilcdc_crtc_enable_irqs(struct drm_device *dev)
 
 static void tilcdc_crtc_disable_irqs(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	/* disable irqs that we might have enabled: */
 	if (priv->rev == 1) {
@@ -177,7 +174,7 @@ static void tilcdc_crtc_disable_irqs(struct drm_device *dev)
 static void reset(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	if (priv->rev != 2)
 		return;
@@ -202,7 +199,7 @@ static unsigned int tilcdc_pclk_diff(unsigned long rate,
 static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	unsigned long clk_rate, real_pclk_rate, pclk_rate;
 	unsigned int clkdiv;
@@ -226,7 +223,7 @@ static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 		 */
 		if (!clk_rate) {
 			/* Nothing more we can do. Just bail out. */
-			dev_err(dev->dev,
+			drm_err(dev,
 				"failed to set the pixel clock - unable to read current lcdc clock rate\n");
 			return;
 		}
@@ -243,7 +240,7 @@ static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 		real_pclk_rate = clk_rate / clkdiv;
 
 		if (tilcdc_pclk_diff(pclk_rate, real_pclk_rate) > 5) {
-			dev_warn(dev->dev,
+			drm_warn(dev,
 				 "effective pixel clock rate (%luHz) differs from the requested rate (%luHz)\n",
 				 real_pclk_rate, pclk_rate);
 		}
@@ -274,41 +271,24 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	const struct tilcdc_panel_info *info = tilcdc_crtc->info;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	uint32_t reg, hbp, hfp, hsw, vbp, vfp, vsw;
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	struct drm_framebuffer *fb = crtc->primary->state->fb;
 
-	if (WARN_ON(!info))
-		return;
-
 	if (WARN_ON(!fb))
 		return;
 
 	/* Configure the Burst Size and fifo threshold of DMA: */
 	reg = tilcdc_read(dev, LCDC_DMA_CTRL_REG) & ~0x00000770;
-	switch (info->dma_burst_sz) {
-	case 1:
-		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_1);
-		break;
-	case 2:
-		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_2);
-		break;
-	case 4:
-		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_4);
-		break;
-	case 8:
-		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_8);
-		break;
-	case 16:
-		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_16);
-		break;
-	default:
-		dev_err(dev->dev, "invalid burst size\n");
-		return;
+	/* Use 16 bit DMA burst size by default */
+	reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_16);
+
+	if (priv->fifo_th) {
+		int fifo_th_val = ilog2(priv->fifo_th) - 3;
+
+		reg |= (fifo_th_val << 8);
 	}
-	reg |= (info->fifo_th << 8);
 	tilcdc_write(dev, LCDC_DMA_CTRL_REG, reg);
 
 	/* Configure timings: */
@@ -324,8 +304,8 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 
 	/* Set AC Bias Period and Number of Transitions per Interrupt: */
 	reg = tilcdc_read(dev, LCDC_RASTER_TIMING_2_REG) & ~0x000fff00;
-	reg |= LCDC_AC_BIAS_FREQUENCY(info->ac_bias) |
-		LCDC_AC_BIAS_TRANSITIONS_PER_INT(info->ac_bias_intrpt);
+	/* Use 255 AC Bias Pin Frequency by default */
+	reg |= LCDC_AC_BIAS_FREQUENCY(255);
 
 	/*
 	 * subtract one from hfp, hbp, hsw because the hardware uses
@@ -375,8 +355,6 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 		  LCDC_V2_TFT_24BPP_MODE | LCDC_V2_TFT_24BPP_UNPACK |
 		  0x000ff000 /* Palette Loading Delay bits */);
 	reg |= LCDC_TFT_MODE; /* no monochrome/passive support */
-	if (info->tft_alt_mode)
-		reg |= LCDC_TFT_ALT_ENABLE;
 	if (priv->rev == 2) {
 		switch (fb->format->format) {
 		case DRM_FORMAT_BGR565:
@@ -391,24 +369,21 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 			reg |= LCDC_V2_TFT_24BPP_MODE;
 			break;
 		default:
-			dev_err(dev->dev, "invalid pixel format\n");
+			drm_err(dev, "invalid pixel format\n");
 			return;
 		}
 	}
-	reg |= info->fdd << 12;
+	/* Use 128 FIFO DMA Request Delay by default */
+	reg |= 128 << 12;
 	tilcdc_write(dev, LCDC_RASTER_CTRL_REG, reg);
 
-	if (info->invert_pxl_clk)
+	if (mode->flags == DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 		tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG, LCDC_INVERT_PIXEL_CLOCK);
 	else
 		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_INVERT_PIXEL_CLOCK);
 
-	if (info->sync_ctrl)
-		tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_CTRL);
-	else
-		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_CTRL);
-
-	if (info->sync_edge)
+	tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_CTRL);
+	if (mode->flags == DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE)
 		tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_EDGE);
 	else
 		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_EDGE);
@@ -423,11 +398,6 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 	else
 		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_INVERT_VSYNC);
 
-	if (info->raster_order)
-		tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ORDER);
-	else
-		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ORDER);
-
 	tilcdc_crtc_set_clk(crtc);
 
 	tilcdc_crtc_load_palette(crtc);
@@ -512,7 +482,7 @@ static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown)
 				 tilcdc_crtc->frame_done,
 				 msecs_to_jiffies(500));
 	if (ret == 0)
-		dev_err(dev->dev, "%s: timeout waiting for framedone\n",
+		drm_err(dev, "%s: timeout waiting for framedone\n",
 			__func__);
 
 	drm_crtc_vblank_off(crtc);
@@ -573,7 +543,7 @@ static void tilcdc_crtc_recover_work(struct work_struct *work)
 		container_of(work, struct tilcdc_crtc, recover_work);
 	struct drm_crtc *crtc = &tilcdc_crtc->base;
 
-	dev_info(crtc->dev->dev, "%s: Reset CRTC", __func__);
+	drm_info(crtc->dev, "%s: Reset CRTC", __func__);
 
 	drm_modeset_lock(&crtc->mutex, NULL);
 
@@ -586,16 +556,15 @@ static void tilcdc_crtc_recover_work(struct work_struct *work)
 	drm_modeset_unlock(&crtc->mutex);
 }
 
-void tilcdc_crtc_destroy(struct drm_crtc *crtc)
+static void tilcdc_crtc_destroy(struct drm_device *dev, void *data)
 {
-	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
+	struct tilcdc_drm_private *priv = (struct tilcdc_drm_private *)data;
 
-	tilcdc_crtc_shutdown(crtc);
+	tilcdc_crtc_shutdown(priv->crtc);
 
 	flush_workqueue(priv->wq);
 
-	of_node_put(crtc->port);
-	drm_crtc_cleanup(crtc);
+	of_node_put(priv->crtc->port);
 }
 
 int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
@@ -606,7 +575,7 @@ int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 
 	if (tilcdc_crtc->event) {
-		dev_err(dev->dev, "already pending page flip!\n");
+		drm_err(dev, "already pending page flip!\n");
 		return -EBUSY;
 	}
 
@@ -642,11 +611,6 @@ static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
 		const struct drm_display_mode *mode,
 		struct drm_display_mode *adjusted_mode)
 {
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-
-	if (!tilcdc_crtc->simulate_vesa_sync)
-		return true;
-
 	/*
 	 * tilcdc does not generate VESA-compliant sync but aligns
 	 * VS on the second edge of HS instead of first edge.
@@ -683,7 +647,7 @@ static int tilcdc_crtc_enable_vblank(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	unsigned long flags;
 
 	spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
@@ -706,7 +670,7 @@ static void tilcdc_crtc_disable_vblank(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	unsigned long flags;
 
 	spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
@@ -743,14 +707,13 @@ static void tilcdc_crtc_reset(struct drm_crtc *crtc)
 					 tilcdc_crtc->frame_done,
 					 msecs_to_jiffies(500));
 		if (ret == 0)
-			dev_err(dev->dev, "%s: timeout waiting for framedone\n",
+			drm_err(dev, "%s: timeout waiting for framedone\n",
 				__func__);
 	}
 	pm_runtime_put_sync(dev->dev);
 }
 
 static const struct drm_crtc_funcs tilcdc_crtc_funcs = {
-	.destroy        = tilcdc_crtc_destroy,
 	.set_config     = drm_atomic_helper_set_config,
 	.page_flip      = drm_atomic_helper_page_flip,
 	.reset		= tilcdc_crtc_reset,
@@ -764,7 +727,7 @@ static enum drm_mode_status
 tilcdc_crtc_mode_valid(struct drm_crtc *crtc,
 		       const struct drm_display_mode *mode)
 {
-	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(crtc->dev);
 	unsigned int bandwidth;
 	uint32_t hbp, hfp, hsw, vbp, vfp, vsw;
 
@@ -859,25 +822,10 @@ static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
 	.atomic_flush	= tilcdc_crtc_atomic_flush,
 };
 
-void tilcdc_crtc_set_panel_info(struct drm_crtc *crtc,
-		const struct tilcdc_panel_info *info)
-{
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-	tilcdc_crtc->info = info;
-}
-
-void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc,
-					bool simulate_vesa_sync)
-{
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-
-	tilcdc_crtc->simulate_vesa_sync = simulate_vesa_sync;
-}
-
 void tilcdc_crtc_update_clk(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 
 	drm_modeset_lock(&crtc->mutex, NULL);
@@ -901,7 +849,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	uint32_t stat, reg;
 
 	stat = tilcdc_read_irqstatus(dev);
@@ -947,7 +895,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 	}
 
 	if (stat & LCDC_FIFO_UNDERFLOW)
-		dev_err_ratelimited(dev->dev, "%s(0x%08x): FIFO underflow",
+		drm_err_ratelimited(dev, "%s(0x%08x): FIFO underflow",
 				    __func__, stat);
 
 	if (stat & LCDC_PL_LOAD_DONE) {
@@ -961,7 +909,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 	}
 
 	if (stat & LCDC_SYNC_LOST) {
-		dev_err_ratelimited(dev->dev, "%s(0x%08x): Sync lost",
+		drm_err_ratelimited(dev, "%s(0x%08x): Sync lost",
 				    __func__, stat);
 		tilcdc_crtc->frame_intact = false;
 		if (priv->rev == 1) {
@@ -975,7 +923,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 		} else {
 			if (tilcdc_crtc->sync_lost_count++ >
 			    SYNC_LOST_COUNT_LIMIT) {
-				dev_err(dev->dev,
+				drm_err(dev,
 					"%s(0x%08x): Sync lost flood detected, recovering",
 					__func__, stat);
 				queue_work(system_wq,
@@ -1009,14 +957,33 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 
 int tilcdc_crtc_create(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	struct tilcdc_crtc *tilcdc_crtc;
+	struct tilcdc_plane *primary;
 	struct drm_crtc *crtc;
 	int ret;
 
-	tilcdc_crtc = devm_kzalloc(dev->dev, sizeof(*tilcdc_crtc), GFP_KERNEL);
-	if (!tilcdc_crtc)
-		return -ENOMEM;
+	primary = tilcdc_plane_init(dev);
+	if (IS_ERR(primary)) {
+		drm_err(dev, "Failed to initialize plane: %pe\n", primary);
+		return PTR_ERR(primary);
+	}
+
+	tilcdc_crtc = drmm_crtc_alloc_with_planes(dev, struct tilcdc_crtc, base,
+						  &primary->base,
+						  NULL,
+						  &tilcdc_crtc_funcs,
+						  "tilcdc crtc");
+	if (IS_ERR(tilcdc_crtc)) {
+		drm_err(dev, "Failed to init CRTC: %pe\n", tilcdc_crtc);
+		return PTR_ERR(tilcdc_crtc);
+	}
+
+	tilcdc_crtc->primary = primary;
+	priv->crtc = &tilcdc_crtc->base;
+	ret = drmm_add_action_or_reset(dev, tilcdc_crtc_destroy, priv);
+	if (ret)
+		return ret;
 
 	init_completion(&tilcdc_crtc->palette_loaded);
 	tilcdc_crtc->palette_base = dmam_alloc_coherent(dev->dev,
@@ -1029,10 +996,6 @@ int tilcdc_crtc_create(struct drm_device *dev)
 
 	crtc = &tilcdc_crtc->base;
 
-	ret = tilcdc_plane_init(dev, &tilcdc_crtc->primary);
-	if (ret < 0)
-		goto fail;
-
 	mutex_init(&tilcdc_crtc->enable_lock);
 
 	init_waitqueue_head(&tilcdc_crtc->frame_done_wq);
@@ -1040,30 +1003,7 @@ int tilcdc_crtc_create(struct drm_device *dev)
 	spin_lock_init(&tilcdc_crtc->irq_lock);
 	INIT_WORK(&tilcdc_crtc->recover_work, tilcdc_crtc_recover_work);
 
-	ret = drm_crtc_init_with_planes(dev, crtc,
-					&tilcdc_crtc->primary,
-					NULL,
-					&tilcdc_crtc_funcs,
-					"tilcdc crtc");
-	if (ret < 0)
-		goto fail;
-
 	drm_crtc_helper_add(crtc, &tilcdc_crtc_helper_funcs);
 
-	if (priv->is_componentized) {
-		crtc->port = of_graph_get_port_by_id(dev->dev->of_node, 0);
-		if (!crtc->port) { /* This should never happen */
-			dev_err(dev->dev, "Port node not found in %pOF\n",
-				dev->dev->of_node);
-			ret = -EINVAL;
-			goto fail;
-		}
-	}
-
-	priv->crtc = crtc;
 	return 0;
-
-fail:
-	tilcdc_crtc_destroy(crtc);
-	return ret;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 3dcbec312bacb..c877b2be9c2ec 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -6,9 +6,7 @@
 
 /* LCDC DRM driver, based on da8xx-fb */
 
-#include <linux/component.h>
 #include <linux/mod_devicetable.h>
-#include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -22,16 +20,19 @@
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mm.h>
+#include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
 
 #include "tilcdc_drv.h"
-#include "tilcdc_external.h"
-#include "tilcdc_panel.h"
+#include "tilcdc_encoder.h"
 #include "tilcdc_regs.h"
 
-static LIST_HEAD(module_list);
+enum tilcdc_variant {
+	AM33XX_TILCDC,
+	DA850_TILCDC,
+};
 
 static const u32 tilcdc_rev1_formats[] = { DRM_FORMAT_RGB565 };
 
@@ -47,20 +48,6 @@ static const u32 tilcdc_legacy_formats[] = { DRM_FORMAT_RGB565,
 					     DRM_FORMAT_RGB888,
 					     DRM_FORMAT_XRGB8888 };
 
-void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
-		const struct tilcdc_module_ops *funcs)
-{
-	mod->name = name;
-	mod->funcs = funcs;
-	INIT_LIST_HEAD(&mod->list);
-	list_add(&mod->list, &module_list);
-}
-
-void tilcdc_module_cleanup(struct tilcdc_module *mod)
-{
-	list_del(&mod->list);
-}
-
 static int tilcdc_atomic_check(struct drm_device *dev,
 			       struct drm_atomic_state *state)
 {
@@ -93,13 +80,7 @@ static const struct drm_mode_config_funcs mode_config_funcs = {
 
 static void modeset_init(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	struct tilcdc_module *mod;
-
-	list_for_each_entry(mod, &module_list, list) {
-		DBG("loading module: %s", mod->name);
-		mod->funcs->modeset_init(mod, dev);
-	}
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	dev->mode_config.min_width = 0;
 	dev->mode_config.min_height = 0;
@@ -125,14 +106,14 @@ static int cpufreq_transition(struct notifier_block *nb,
 static irqreturn_t tilcdc_irq(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	return tilcdc_crtc_irq(priv->crtc);
 }
 
 static int tilcdc_irq_install(struct drm_device *dev, unsigned int irq)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	int ret;
 
 	ret = request_irq(irq, tilcdc_irq, 0, dev->driver->name, dev);
@@ -146,7 +127,7 @@ static int tilcdc_irq_install(struct drm_device *dev, unsigned int irq)
 
 static void tilcdc_irq_uninstall(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 
 	if (!priv->irq_enabled)
 		return;
@@ -159,79 +140,162 @@ static void tilcdc_irq_uninstall(struct drm_device *dev)
  * DRM operations:
  */
 
-static void tilcdc_fini(struct drm_device *dev)
+#if defined(CONFIG_DEBUG_FS)
+static const struct {
+	const char *name;
+	uint8_t  rev;
+	uint8_t  save;
+	uint32_t reg;
+} registers[] =		{
+#define REG(rev, save, reg) { #reg, rev, save, reg }
+		/* exists in revision 1: */
+		REG(1, false, LCDC_PID_REG),
+		REG(1, true,  LCDC_CTRL_REG),
+		REG(1, false, LCDC_STAT_REG),
+		REG(1, true,  LCDC_RASTER_CTRL_REG),
+		REG(1, true,  LCDC_RASTER_TIMING_0_REG),
+		REG(1, true,  LCDC_RASTER_TIMING_1_REG),
+		REG(1, true,  LCDC_RASTER_TIMING_2_REG),
+		REG(1, true,  LCDC_DMA_CTRL_REG),
+		REG(1, true,  LCDC_DMA_FB_BASE_ADDR_0_REG),
+		REG(1, true,  LCDC_DMA_FB_CEILING_ADDR_0_REG),
+		REG(1, true,  LCDC_DMA_FB_BASE_ADDR_1_REG),
+		REG(1, true,  LCDC_DMA_FB_CEILING_ADDR_1_REG),
+		/* new in revision 2: */
+		REG(2, false, LCDC_RAW_STAT_REG),
+		REG(2, false, LCDC_MASKED_STAT_REG),
+		REG(2, true, LCDC_INT_ENABLE_SET_REG),
+		REG(2, false, LCDC_INT_ENABLE_CLR_REG),
+		REG(2, false, LCDC_END_OF_INT_IND_REG),
+		REG(2, true,  LCDC_CLK_ENABLE_REG),
+#undef REG
+};
+
+static int tilcdc_regs_show(struct seq_file *m, void *arg)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
+	unsigned i;
 
-#ifdef CONFIG_CPU_FREQ
-	if (priv->freq_transition.notifier_call)
-		cpufreq_unregister_notifier(&priv->freq_transition,
-					    CPUFREQ_TRANSITION_NOTIFIER);
+	pm_runtime_get_sync(dev->dev);
+
+	seq_printf(m, "revision: %d\n", priv->rev);
+
+	for (i = 0; i < ARRAY_SIZE(registers); i++)
+		if (priv->rev >= registers[i].rev)
+			seq_printf(m, "%s:\t %08x\n", registers[i].name,
+					tilcdc_read(dev, registers[i].reg));
+
+	pm_runtime_put_sync(dev->dev);
+
+	return 0;
+}
+
+static int tilcdc_mm_show(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(m);
+	drm_mm_print(&dev->vma_offset_manager->vm_addr_space_mm, &p);
+	return 0;
+}
+
+static struct drm_info_list tilcdc_debugfs_list[] = {
+		{ "regs", tilcdc_regs_show, 0, NULL },
+		{ "mm",   tilcdc_mm_show,   0, NULL },
+};
+
+static void tilcdc_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(tilcdc_debugfs_list,
+				 ARRAY_SIZE(tilcdc_debugfs_list),
+				 minor->debugfs_root, minor);
+}
 #endif
 
-	if (priv->crtc)
-		tilcdc_crtc_shutdown(priv->crtc);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
-	drm_dev_unregister(dev);
+static const struct drm_driver tilcdc_driver = {
+	.driver_features    = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
+	DRM_GEM_DMA_DRIVER_OPS,
+	DRM_FBDEV_DMA_DRIVER_OPS,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init       = tilcdc_debugfs_init,
+#endif
+	.fops               = &fops,
+	.name               = "tilcdc",
+	.desc               = "TI LCD Controller DRM",
+	.major              = 1,
+	.minor              = 0,
+};
 
-	drm_kms_helper_poll_fini(dev);
-	drm_atomic_helper_shutdown(dev);
-	tilcdc_irq_uninstall(dev);
-	drm_mode_config_cleanup(dev);
+/*
+ * Power management:
+ */
 
-	if (priv->clk)
-		clk_put(priv->clk);
+static int tilcdc_pm_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	int ret = 0;
+
+	ret = drm_mode_config_helper_suspend(ddev);
 
-	if (priv->wq)
-		destroy_workqueue(priv->wq);
+	/* Select sleep pin state */
+	pinctrl_pm_select_sleep_state(dev);
 
-	dev->dev_private = NULL;
+	return ret;
+}
 
-	pm_runtime_disable(dev->dev);
+static int tilcdc_pm_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
 
-	drm_dev_put(dev);
+	/* Select default pin state */
+	pinctrl_pm_select_default_state(dev);
+	return  drm_mode_config_helper_resume(ddev);
 }
 
-static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
+static DEFINE_SIMPLE_DEV_PM_OPS(tilcdc_pm_ops,
+				tilcdc_pm_suspend, tilcdc_pm_resume);
+
+static int tilcdc_pdev_probe(struct platform_device *pdev)
 {
-	struct drm_device *ddev;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct device_node *node = dev->of_node;
+	struct device_node *node = pdev->dev.of_node;
 	struct tilcdc_drm_private *priv;
+	struct device *dev = &pdev->dev;
+	enum tilcdc_variant variant;
+	struct drm_device *ddev;
 	u32 bpp = 0;
 	int ret;
 
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	ddev = drm_dev_alloc(ddrv, dev);
-	if (IS_ERR(ddev))
-		return PTR_ERR(ddev);
+	priv = devm_drm_dev_alloc(dev, &tilcdc_driver,
+				  struct tilcdc_drm_private, ddev);
+	if (IS_ERR(priv))
+		return PTR_ERR(priv);
 
-	ddev->dev_private = priv;
-	platform_set_drvdata(pdev, ddev);
-	drm_mode_config_init(ddev);
+	variant = (uintptr_t)of_device_get_match_data(dev);
 
-	priv->is_componentized =
-		tilcdc_get_external_components(dev, NULL) > 0;
+	platform_set_drvdata(pdev, priv);
+	ddev = &priv->ddev;
+	ret = drmm_mode_config_init(ddev);
+	if (ret)
+		return ret;
 
 	priv->wq = alloc_ordered_workqueue("tilcdc", 0);
-	if (!priv->wq) {
-		ret = -ENOMEM;
-		goto put_drm;
-	}
+	if (!priv->wq)
+		return -ENOMEM;
 
 	priv->mmio = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->mmio)) {
-		dev_err(dev, "failed to request / ioremap\n");
+		drm_err(ddev, "failed to request / ioremap\n");
 		ret = PTR_ERR(priv->mmio);
 		goto free_wq;
 	}
 
 	priv->clk = clk_get(dev, "fck");
 	if (IS_ERR(priv->clk)) {
-		dev_err(dev, "failed to get functional clock\n");
+		drm_err(ddev, "failed to get functional clock\n");
 		ret = -ENODEV;
 		goto free_wq;
 	}
@@ -249,7 +313,7 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
 		priv->rev = 2;
 		break;
 	default:
-		dev_warn(dev, "Unknown PID Reg value 0x%08x, "
+		drm_warn(ddev, "Unknown PID Reg value 0x%08x, "
 			"defaulting to LCD revision 1\n",
 			tilcdc_read(ddev, LCDC_PID_REG));
 		priv->rev = 1;
@@ -309,9 +373,14 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
 
 	DBG("Maximum Pixel Clock Value %dKHz", priv->max_pixelclock);
 
+	if (variant == DA850_TILCDC)
+		priv->fifo_th = 16;
+	else
+		priv->fifo_th = 8;
+
 	ret = tilcdc_crtc_create(ddev);
 	if (ret < 0) {
-		dev_err(dev, "failed to create crtc\n");
+		drm_err(ddev, "failed to create crtc\n");
 		goto disable_pm;
 	}
 	modeset_init(ddev);
@@ -321,48 +390,37 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
 	ret = cpufreq_register_notifier(&priv->freq_transition,
 			CPUFREQ_TRANSITION_NOTIFIER);
 	if (ret) {
-		dev_err(dev, "failed to register cpufreq notifier\n");
+		drm_err(ddev, "failed to register cpufreq notifier\n");
 		priv->freq_transition.notifier_call = NULL;
-		goto destroy_crtc;
+		goto disable_pm;
 	}
 #endif
 
-	if (priv->is_componentized) {
-		ret = component_bind_all(dev, ddev);
-		if (ret < 0)
-			goto unregister_cpufreq_notif;
-
-		ret = tilcdc_add_component_encoder(ddev);
-		if (ret < 0)
-			goto unbind_component;
-	} else {
-		ret = tilcdc_attach_external_device(ddev);
-		if (ret)
-			goto unregister_cpufreq_notif;
-	}
+	ret = tilcdc_encoder_create(ddev);
+	if (ret)
+		goto unregister_cpufreq_notif;
 
-	if (!priv->external_connector &&
-	    ((priv->num_encoders == 0) || (priv->num_connectors == 0))) {
-		dev_err(dev, "no encoders/connectors found\n");
+	if (!priv->connector) {
+		drm_err(ddev, "no encoders/connectors found\n");
 		ret = -EPROBE_DEFER;
-		goto unbind_component;
+		goto unregister_cpufreq_notif;
 	}
 
 	ret = drm_vblank_init(ddev, 1);
 	if (ret < 0) {
-		dev_err(dev, "failed to initialize vblank\n");
-		goto unbind_component;
+		drm_err(ddev, "failed to initialize vblank\n");
+		goto unregister_cpufreq_notif;
 	}
 
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0)
-		goto unbind_component;
+		goto unregister_cpufreq_notif;
 	priv->irq = ret;
 
 	ret = tilcdc_irq_install(ddev, priv->irq);
 	if (ret < 0) {
-		dev_err(dev, "failed to install IRQ handler\n");
-		goto unbind_component;
+		drm_err(ddev, "failed to install IRQ handler\n");
+		goto unregister_cpufreq_notif;
 	}
 
 	drm_mode_config_reset(ddev);
@@ -380,226 +438,48 @@ static int tilcdc_init(const struct drm_driver *ddrv, struct device *dev)
 stop_poll:
 	drm_kms_helper_poll_fini(ddev);
 	tilcdc_irq_uninstall(ddev);
-unbind_component:
-	if (priv->is_componentized)
-		component_unbind_all(dev, ddev);
 unregister_cpufreq_notif:
 #ifdef CONFIG_CPU_FREQ
 	cpufreq_unregister_notifier(&priv->freq_transition,
 				    CPUFREQ_TRANSITION_NOTIFIER);
-destroy_crtc:
 #endif
-	tilcdc_crtc_destroy(priv->crtc);
 disable_pm:
 	pm_runtime_disable(dev);
 	clk_put(priv->clk);
 free_wq:
 	destroy_workqueue(priv->wq);
-put_drm:
-	platform_set_drvdata(pdev, NULL);
-	ddev->dev_private = NULL;
-	drm_dev_put(ddev);
-
-	return ret;
-}
-
-#if defined(CONFIG_DEBUG_FS)
-static const struct {
-	const char *name;
-	uint8_t  rev;
-	uint8_t  save;
-	uint32_t reg;
-} registers[] =		{
-#define REG(rev, save, reg) { #reg, rev, save, reg }
-		/* exists in revision 1: */
-		REG(1, false, LCDC_PID_REG),
-		REG(1, true,  LCDC_CTRL_REG),
-		REG(1, false, LCDC_STAT_REG),
-		REG(1, true,  LCDC_RASTER_CTRL_REG),
-		REG(1, true,  LCDC_RASTER_TIMING_0_REG),
-		REG(1, true,  LCDC_RASTER_TIMING_1_REG),
-		REG(1, true,  LCDC_RASTER_TIMING_2_REG),
-		REG(1, true,  LCDC_DMA_CTRL_REG),
-		REG(1, true,  LCDC_DMA_FB_BASE_ADDR_0_REG),
-		REG(1, true,  LCDC_DMA_FB_CEILING_ADDR_0_REG),
-		REG(1, true,  LCDC_DMA_FB_BASE_ADDR_1_REG),
-		REG(1, true,  LCDC_DMA_FB_CEILING_ADDR_1_REG),
-		/* new in revision 2: */
-		REG(2, false, LCDC_RAW_STAT_REG),
-		REG(2, false, LCDC_MASKED_STAT_REG),
-		REG(2, true, LCDC_INT_ENABLE_SET_REG),
-		REG(2, false, LCDC_INT_ENABLE_CLR_REG),
-		REG(2, false, LCDC_END_OF_INT_IND_REG),
-		REG(2, true,  LCDC_CLK_ENABLE_REG),
-#undef REG
-};
-
-#endif
-
-#ifdef CONFIG_DEBUG_FS
-static int tilcdc_regs_show(struct seq_file *m, void *arg)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	unsigned i;
-
-	pm_runtime_get_sync(dev->dev);
-
-	seq_printf(m, "revision: %d\n", priv->rev);
-
-	for (i = 0; i < ARRAY_SIZE(registers); i++)
-		if (priv->rev >= registers[i].rev)
-			seq_printf(m, "%s:\t %08x\n", registers[i].name,
-					tilcdc_read(dev, registers[i].reg));
-
-	pm_runtime_put_sync(dev->dev);
-
-	return 0;
-}
-
-static int tilcdc_mm_show(struct seq_file *m, void *arg)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_printer p = drm_seq_file_printer(m);
-	drm_mm_print(&dev->vma_offset_manager->vm_addr_space_mm, &p);
-	return 0;
-}
-
-static struct drm_info_list tilcdc_debugfs_list[] = {
-		{ "regs", tilcdc_regs_show, 0, NULL },
-		{ "mm",   tilcdc_mm_show,   0, NULL },
-};
-
-static void tilcdc_debugfs_init(struct drm_minor *minor)
-{
-	struct tilcdc_module *mod;
-
-	drm_debugfs_create_files(tilcdc_debugfs_list,
-				 ARRAY_SIZE(tilcdc_debugfs_list),
-				 minor->debugfs_root, minor);
-
-	list_for_each_entry(mod, &module_list, list)
-		if (mod->funcs->debugfs_init)
-			mod->funcs->debugfs_init(mod, minor);
-}
-#endif
-
-DEFINE_DRM_GEM_DMA_FOPS(fops);
-
-static const struct drm_driver tilcdc_driver = {
-	.driver_features    = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_DMA_DRIVER_OPS,
-	DRM_FBDEV_DMA_DRIVER_OPS,
-#ifdef CONFIG_DEBUG_FS
-	.debugfs_init       = tilcdc_debugfs_init,
-#endif
-	.fops               = &fops,
-	.name               = "tilcdc",
-	.desc               = "TI LCD Controller DRM",
-	.major              = 1,
-	.minor              = 0,
-};
-
-/*
- * Power management:
- */
-
-static int tilcdc_pm_suspend(struct device *dev)
-{
-	struct drm_device *ddev = dev_get_drvdata(dev);
-	int ret = 0;
-
-	ret = drm_mode_config_helper_suspend(ddev);
-
-	/* Select sleep pin state */
-	pinctrl_pm_select_sleep_state(dev);
 
 	return ret;
 }
 
-static int tilcdc_pm_resume(struct device *dev)
-{
-	struct drm_device *ddev = dev_get_drvdata(dev);
-
-	/* Select default pin state */
-	pinctrl_pm_select_default_state(dev);
-	return  drm_mode_config_helper_resume(ddev);
-}
-
-static DEFINE_SIMPLE_DEV_PM_OPS(tilcdc_pm_ops,
-				tilcdc_pm_suspend, tilcdc_pm_resume);
-
-/*
- * Platform driver:
- */
-static int tilcdc_bind(struct device *dev)
-{
-	return tilcdc_init(&tilcdc_driver, dev);
-}
-
-static void tilcdc_unbind(struct device *dev)
-{
-	struct drm_device *ddev = dev_get_drvdata(dev);
-
-	/* Check if a subcomponent has already triggered the unloading. */
-	if (!ddev->dev_private)
-		return;
-
-	tilcdc_fini(ddev);
-	dev_set_drvdata(dev, NULL);
-}
-
-static const struct component_master_ops tilcdc_comp_ops = {
-	.bind = tilcdc_bind,
-	.unbind = tilcdc_unbind,
-};
-
-static int tilcdc_pdev_probe(struct platform_device *pdev)
-{
-	struct component_match *match = NULL;
-	int ret;
-
-	/* bail out early if no DT data: */
-	if (!pdev->dev.of_node) {
-		dev_err(&pdev->dev, "device-tree data is missing\n");
-		return -ENXIO;
-	}
-
-	ret = tilcdc_get_external_components(&pdev->dev, &match);
-	if (ret < 0)
-		return ret;
-	else if (ret == 0)
-		return tilcdc_init(&tilcdc_driver, &pdev->dev);
-	else
-		return component_master_add_with_match(&pdev->dev,
-						       &tilcdc_comp_ops,
-						       match);
-}
-
 static void tilcdc_pdev_remove(struct platform_device *pdev)
 {
-	int ret;
+	struct tilcdc_drm_private *priv = platform_get_drvdata(pdev);
+	struct drm_device *ddev = &priv->ddev;
 
-	ret = tilcdc_get_external_components(&pdev->dev, NULL);
-	if (ret < 0)
-		dev_err(&pdev->dev, "tilcdc_get_external_components() failed (%pe)\n",
-			ERR_PTR(ret));
-	else if (ret == 0)
-		tilcdc_fini(platform_get_drvdata(pdev));
-	else
-		component_master_del(&pdev->dev, &tilcdc_comp_ops);
+	drm_dev_unregister(ddev);
+	drm_kms_helper_poll_fini(ddev);
+	tilcdc_irq_uninstall(ddev);
+#ifdef CONFIG_CPU_FREQ
+	cpufreq_unregister_notifier(&priv->freq_transition,
+				    CPUFREQ_TRANSITION_NOTIFIER);
+#endif
+	pm_runtime_disable(&pdev->dev);
+	clk_put(priv->clk);
+	destroy_workqueue(priv->wq);
 }
 
 static void tilcdc_pdev_shutdown(struct platform_device *pdev)
 {
-	drm_atomic_helper_shutdown(platform_get_drvdata(pdev));
+	struct tilcdc_drm_private *priv = platform_get_drvdata(pdev);
+	struct drm_device *ddev = &priv->ddev;
+
+	drm_atomic_helper_shutdown(ddev);
 }
 
 static const struct of_device_id tilcdc_of_match[] = {
-		{ .compatible = "ti,am33xx-tilcdc", },
-		{ .compatible = "ti,da850-tilcdc", },
+		{ .compatible = "ti,am33xx-tilcdc", .data = (void *)AM33XX_TILCDC},
+		{ .compatible = "ti,da850-tilcdc", .data = (void *)DA850_TILCDC},
 		{ },
 };
 MODULE_DEVICE_TABLE(of, tilcdc_of_match);
@@ -615,25 +495,7 @@ static struct platform_driver tilcdc_platform_driver = {
 	},
 };
 
-static int __init tilcdc_drm_init(void)
-{
-	if (drm_firmware_drivers_only())
-		return -ENODEV;
-
-	DBG("init");
-	tilcdc_panel_init();
-	return platform_driver_register(&tilcdc_platform_driver);
-}
-
-static void __exit tilcdc_drm_fini(void)
-{
-	DBG("fini");
-	platform_driver_unregister(&tilcdc_platform_driver);
-	tilcdc_panel_fini();
-}
-
-module_init(tilcdc_drm_init);
-module_exit(tilcdc_drm_fini);
+drm_module_platform_driver(tilcdc_platform_driver);
 
 MODULE_AUTHOR("Rob Clark <robdclark@gmail.com");
 MODULE_DESCRIPTION("TI LCD Controller DRM Driver");
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
index 58b276f82a669..17d152f9f0b69 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
@@ -48,6 +48,8 @@ struct tilcdc_drm_private {
 
 	unsigned int irq;
 
+	struct drm_device ddev;
+
 	/* don't attempt resolutions w/ higher W * H * Hz: */
 	uint32_t max_bandwidth;
 	/*
@@ -61,6 +63,8 @@ struct tilcdc_drm_private {
 	 */
 	uint32_t max_width;
 
+	u32 fifo_th;
+
 	/* Supported pixel formats */
 	const uint32_t *pixelformats;
 	uint32_t num_pixelformats;
@@ -73,101 +77,32 @@ struct tilcdc_drm_private {
 
 	struct drm_crtc *crtc;
 
-	unsigned int num_encoders;
-	struct drm_encoder *encoders[8];
-
-	unsigned int num_connectors;
-	struct drm_connector *connectors[8];
-
-	struct drm_encoder *external_encoder;
-	struct drm_connector *external_connector;
+	struct tilcdc_encoder *encoder;
+	struct drm_connector *connector;
 
-	bool is_componentized;
 	bool irq_enabled;
 };
 
-/* Sub-module for display.  Since we don't know at compile time what panels
- * or display adapter(s) might be present (for ex, off chip dvi/tfp410,
- * hdmi encoder, various lcd panels), the connector/encoder(s) are split into
- * separate drivers.  If they are probed and found to be present, they
- * register themselves with tilcdc_register_module().
- */
-struct tilcdc_module;
-
-struct tilcdc_module_ops {
-	/* create appropriate encoders/connectors: */
-	int (*modeset_init)(struct tilcdc_module *mod, struct drm_device *dev);
-#ifdef CONFIG_DEBUG_FS
-	/* create debugfs nodes (can be NULL): */
-	int (*debugfs_init)(struct tilcdc_module *mod, struct drm_minor *minor);
-#endif
-};
-
-struct tilcdc_module {
-	const char *name;
-	struct list_head list;
-	const struct tilcdc_module_ops *funcs;
-};
-
-void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
-		const struct tilcdc_module_ops *funcs);
-void tilcdc_module_cleanup(struct tilcdc_module *mod);
-
-/* Panel config that needs to be set in the crtc, but is not coming from
- * the mode timings.  The display module is expected to call
- * tilcdc_crtc_set_panel_info() to set this during modeset.
- */
-struct tilcdc_panel_info {
-
-	/* AC Bias Pin Frequency */
-	uint32_t ac_bias;
-
-	/* AC Bias Pin Transitions per Interrupt */
-	uint32_t ac_bias_intrpt;
-
-	/* DMA burst size */
-	uint32_t dma_burst_sz;
-
-	/* Bits per pixel */
-	uint32_t bpp;
-
-	/* FIFO DMA Request Delay */
-	uint32_t fdd;
-
-	/* TFT Alternative Signal Mapping (Only for active) */
-	bool tft_alt_mode;
-
-	/* Invert pixel clock */
-	bool invert_pxl_clk;
-
-	/* Horizontal and Vertical Sync Edge: 0=rising 1=falling */
-	uint32_t sync_edge;
-
-	/* Horizontal and Vertical Sync: Control: 0=ignore */
-	uint32_t sync_ctrl;
-
-	/* Raster Data Order Select: 1=Most-to-least 0=Least-to-most */
-	uint32_t raster_order;
-
-	/* DMA FIFO threshold */
-	uint32_t fifo_th;
-};
-
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
+#define ddev_to_tilcdc_priv(x) container_of(x, struct tilcdc_drm_private, ddev)
+
 int tilcdc_crtc_create(struct drm_device *dev);
 irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc);
 void tilcdc_crtc_update_clk(struct drm_crtc *crtc);
-void tilcdc_crtc_set_panel_info(struct drm_crtc *crtc,
-		const struct tilcdc_panel_info *info);
-void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc,
-					bool simulate_vesa_sync);
 void tilcdc_crtc_shutdown(struct drm_crtc *crtc);
-void tilcdc_crtc_destroy(struct drm_crtc *crtc);
 int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
 		struct drm_framebuffer *fb,
 		struct drm_pending_vblank_event *event);
 
-int tilcdc_plane_init(struct drm_device *dev, struct drm_plane *plane);
+struct tilcdc_plane {
+	struct drm_plane base;
+};
+
+struct tilcdc_encoder {
+	struct drm_encoder base;
+};
+
+struct tilcdc_plane *tilcdc_plane_init(struct drm_device *dev);
 
 #endif /* __TILCDC_DRV_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_encoder.c b/drivers/gpu/drm/tilcdc/tilcdc_encoder.c
new file mode 100644
index 0000000000000..680a2ac6ab594
--- /dev/null
+++ b/drivers/gpu/drm/tilcdc/tilcdc_encoder.c
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2015 Texas Instruments
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#include <linux/of_graph.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_bridge_connector.h>
+#include <drm/drm_of.h>
+#include <drm/drm_simple_kms_helper.h>
+
+#include "tilcdc_drv.h"
+#include "tilcdc_encoder.h"
+
+static
+int tilcdc_attach_bridge(struct drm_device *ddev, struct drm_bridge *bridge)
+{
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(ddev);
+	struct drm_connector *connector;
+	int ret;
+
+	priv->encoder->base.possible_crtcs = BIT(0);
+
+	ret = drm_bridge_attach(&priv->encoder->base, bridge, NULL,
+				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+	if (ret)
+		return ret;
+
+	connector = drm_bridge_connector_init(ddev, &priv->encoder->base);
+	if (IS_ERR(connector)) {
+		drm_err(ddev, "bridge_connector create failed\n");
+		return PTR_ERR(connector);
+	}
+
+	ret = drm_connector_attach_encoder(connector, &priv->encoder->base);
+	if (ret) {
+		drm_err(ddev, "attaching encoder to connector failed\n");
+		return ret;
+	}
+
+	priv->connector = connector;
+	return 0;
+}
+
+int tilcdc_encoder_create(struct drm_device *ddev)
+{
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(ddev);
+	struct tilcdc_encoder *encoder;
+	struct drm_bridge *bridge;
+
+	bridge = devm_drm_of_get_bridge(ddev->dev, ddev->dev->of_node, 0, 0);
+	if (PTR_ERR(bridge) == -ENODEV)
+		return 0;
+	else if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
+
+	encoder = drmm_simple_encoder_alloc(ddev, struct tilcdc_encoder,
+					    base, DRM_MODE_ENCODER_NONE);
+	if (IS_ERR(encoder)) {
+		drm_err(ddev, "drm_encoder_init() failed %pe\n", encoder);
+		return PTR_ERR(encoder);
+	}
+	priv->encoder = encoder;
+
+	return tilcdc_attach_bridge(ddev, bridge);
+}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_encoder.h b/drivers/gpu/drm/tilcdc/tilcdc_encoder.h
new file mode 100644
index 0000000000000..c8f87f59024e6
--- /dev/null
+++ b/drivers/gpu/drm/tilcdc/tilcdc_encoder.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2015 Texas Instruments
+ * Author: Jyri Sarha <jsarha@ti.com>
+ */
+
+#ifndef __TILCDC_EXTERNAL_H__
+#define __TILCDC_EXTERNAL_H__
+
+int tilcdc_encoder_create(struct drm_device *ddev);
+#endif /* __TILCDC_SLAVE_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.c b/drivers/gpu/drm/tilcdc/tilcdc_external.c
deleted file mode 100644
index 3b86d002ef62e..0000000000000
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.c
+++ /dev/null
@@ -1,179 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2015 Texas Instruments
- * Author: Jyri Sarha <jsarha@ti.com>
- */
-
-#include <linux/component.h>
-#include <linux/of_graph.h>
-
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_bridge.h>
-#include <drm/drm_of.h>
-#include <drm/drm_simple_kms_helper.h>
-
-#include "tilcdc_drv.h"
-#include "tilcdc_external.h"
-
-static const struct tilcdc_panel_info panel_info_tda998x = {
-		.ac_bias                = 255,
-		.ac_bias_intrpt         = 0,
-		.dma_burst_sz           = 16,
-		.bpp                    = 16,
-		.fdd                    = 0x80,
-		.tft_alt_mode           = 0,
-		.invert_pxl_clk		= 1,
-		.sync_edge              = 1,
-		.sync_ctrl              = 1,
-		.raster_order           = 0,
-};
-
-static const struct tilcdc_panel_info panel_info_default = {
-		.ac_bias                = 255,
-		.ac_bias_intrpt         = 0,
-		.dma_burst_sz           = 16,
-		.bpp                    = 16,
-		.fdd                    = 0x80,
-		.tft_alt_mode           = 0,
-		.sync_edge              = 0,
-		.sync_ctrl              = 1,
-		.raster_order           = 0,
-};
-
-static
-struct drm_connector *tilcdc_encoder_find_connector(struct drm_device *ddev,
-						    struct drm_encoder *encoder)
-{
-	struct drm_connector *connector;
-
-	list_for_each_entry(connector, &ddev->mode_config.connector_list, head) {
-		if (drm_connector_has_possible_encoder(connector, encoder))
-			return connector;
-	}
-
-	dev_err(ddev->dev, "No connector found for %s encoder (id %d)\n",
-		encoder->name, encoder->base.id);
-
-	return NULL;
-}
-
-int tilcdc_add_component_encoder(struct drm_device *ddev)
-{
-	struct tilcdc_drm_private *priv = ddev->dev_private;
-	struct drm_encoder *encoder = NULL, *iter;
-
-	list_for_each_entry(iter, &ddev->mode_config.encoder_list, head)
-		if (iter->possible_crtcs & (1 << priv->crtc->index)) {
-			encoder = iter;
-			break;
-		}
-
-	if (!encoder) {
-		dev_err(ddev->dev, "%s: No suitable encoder found\n", __func__);
-		return -ENODEV;
-	}
-
-	priv->external_connector =
-		tilcdc_encoder_find_connector(ddev, encoder);
-
-	if (!priv->external_connector)
-		return -ENODEV;
-
-	/* Only tda998x is supported at the moment. */
-	tilcdc_crtc_set_simulate_vesa_sync(priv->crtc, true);
-	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_tda998x);
-
-	return 0;
-}
-
-static
-int tilcdc_attach_bridge(struct drm_device *ddev, struct drm_bridge *bridge)
-{
-	struct tilcdc_drm_private *priv = ddev->dev_private;
-	int ret;
-
-	priv->external_encoder->possible_crtcs = BIT(0);
-
-	ret = drm_bridge_attach(priv->external_encoder, bridge, NULL, 0);
-	if (ret)
-		return ret;
-
-	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_default);
-
-	priv->external_connector =
-		tilcdc_encoder_find_connector(ddev, priv->external_encoder);
-	if (!priv->external_connector)
-		return -ENODEV;
-
-	return 0;
-}
-
-int tilcdc_attach_external_device(struct drm_device *ddev)
-{
-	struct tilcdc_drm_private *priv = ddev->dev_private;
-	struct drm_bridge *bridge;
-	struct drm_panel *panel;
-	int ret;
-
-	ret = drm_of_find_panel_or_bridge(ddev->dev->of_node, 0, 0,
-					  &panel, &bridge);
-	if (ret == -ENODEV)
-		return 0;
-	else if (ret)
-		return ret;
-
-	priv->external_encoder = devm_kzalloc(ddev->dev,
-					      sizeof(*priv->external_encoder),
-					      GFP_KERNEL);
-	if (!priv->external_encoder)
-		return -ENOMEM;
-
-	ret = drm_simple_encoder_init(ddev, priv->external_encoder,
-				      DRM_MODE_ENCODER_NONE);
-	if (ret) {
-		dev_err(ddev->dev, "drm_encoder_init() failed %d\n", ret);
-		return ret;
-	}
-
-	if (panel) {
-		bridge = devm_drm_panel_bridge_add_typed(ddev->dev, panel,
-							 DRM_MODE_CONNECTOR_DPI);
-		if (IS_ERR(bridge)) {
-			ret = PTR_ERR(bridge);
-			goto err_encoder_cleanup;
-		}
-	}
-
-	ret = tilcdc_attach_bridge(ddev, bridge);
-	if (ret)
-		goto err_encoder_cleanup;
-
-	return 0;
-
-err_encoder_cleanup:
-	drm_encoder_cleanup(priv->external_encoder);
-	return ret;
-}
-
-static int dev_match_of(struct device *dev, void *data)
-{
-	return dev->of_node == data;
-}
-
-int tilcdc_get_external_components(struct device *dev,
-				   struct component_match **match)
-{
-	struct device_node *node;
-
-	node = of_graph_get_remote_node(dev->of_node, 0, 0);
-
-	if (!of_device_is_compatible(node, "nxp,tda998x")) {
-		of_node_put(node);
-		return 0;
-	}
-
-	if (match)
-		drm_of_component_match_add(dev, match, dev_match_of, node);
-	of_node_put(node);
-	return 1;
-}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.h b/drivers/gpu/drm/tilcdc/tilcdc_external.h
deleted file mode 100644
index fb4476694cd89..0000000000000
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2015 Texas Instruments
- * Author: Jyri Sarha <jsarha@ti.com>
- */
-
-#ifndef __TILCDC_EXTERNAL_H__
-#define __TILCDC_EXTERNAL_H__
-
-int tilcdc_add_component_encoder(struct drm_device *dev);
-int tilcdc_get_external_components(struct device *dev,
-				   struct component_match **match);
-int tilcdc_attach_external_device(struct drm_device *ddev);
-#endif /* __TILCDC_SLAVE_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
deleted file mode 100644
index 1de3996501f7b..0000000000000
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ /dev/null
@@ -1,408 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012 Texas Instruments
- * Author: Rob Clark <robdclark@gmail.com>
- */
-
-#include <linux/backlight.h>
-#include <linux/gpio/consumer.h>
-#include <linux/platform_device.h>
-
-#include <video/display_timing.h>
-#include <video/of_display_timing.h>
-#include <video/videomode.h>
-
-#include <drm/drm_atomic_state_helper.h>
-#include <drm/drm_connector.h>
-#include <drm/drm_modeset_helper_vtables.h>
-#include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
-
-#include "tilcdc_drv.h"
-#include "tilcdc_panel.h"
-
-struct panel_module {
-	struct tilcdc_module base;
-	struct tilcdc_panel_info *info;
-	struct display_timings *timings;
-	struct backlight_device *backlight;
-	struct gpio_desc *enable_gpio;
-};
-#define to_panel_module(x) container_of(x, struct panel_module, base)
-
-
-/*
- * Encoder:
- */
-
-struct panel_encoder {
-	struct drm_encoder base;
-	struct panel_module *mod;
-};
-#define to_panel_encoder(x) container_of(x, struct panel_encoder, base)
-
-static void panel_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-	struct panel_encoder *panel_encoder = to_panel_encoder(encoder);
-	struct backlight_device *backlight = panel_encoder->mod->backlight;
-	struct gpio_desc *gpio = panel_encoder->mod->enable_gpio;
-
-	if (backlight) {
-		backlight->props.power = mode == DRM_MODE_DPMS_ON ?
-					 BACKLIGHT_POWER_ON : BACKLIGHT_POWER_OFF;
-		backlight_update_status(backlight);
-	}
-
-	if (gpio)
-		gpiod_set_value_cansleep(gpio,
-					 mode == DRM_MODE_DPMS_ON ? 1 : 0);
-}
-
-static void panel_encoder_prepare(struct drm_encoder *encoder)
-{
-	panel_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
-}
-
-static void panel_encoder_commit(struct drm_encoder *encoder)
-{
-	panel_encoder_dpms(encoder, DRM_MODE_DPMS_ON);
-}
-
-static void panel_encoder_mode_set(struct drm_encoder *encoder,
-		struct drm_display_mode *mode,
-		struct drm_display_mode *adjusted_mode)
-{
-	/* nothing needed */
-}
-
-static const struct drm_encoder_helper_funcs panel_encoder_helper_funcs = {
-		.dpms           = panel_encoder_dpms,
-		.prepare        = panel_encoder_prepare,
-		.commit         = panel_encoder_commit,
-		.mode_set       = panel_encoder_mode_set,
-};
-
-static struct drm_encoder *panel_encoder_create(struct drm_device *dev,
-		struct panel_module *mod)
-{
-	struct panel_encoder *panel_encoder;
-	struct drm_encoder *encoder;
-	int ret;
-
-	panel_encoder = devm_kzalloc(dev->dev, sizeof(*panel_encoder),
-				     GFP_KERNEL);
-	if (!panel_encoder)
-		return NULL;
-
-	panel_encoder->mod = mod;
-
-	encoder = &panel_encoder->base;
-	encoder->possible_crtcs = 1;
-
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS);
-	if (ret < 0)
-		goto fail;
-
-	drm_encoder_helper_add(encoder, &panel_encoder_helper_funcs);
-
-	return encoder;
-
-fail:
-	drm_encoder_cleanup(encoder);
-	return NULL;
-}
-
-/*
- * Connector:
- */
-
-struct panel_connector {
-	struct drm_connector base;
-
-	struct drm_encoder *encoder;  /* our connected encoder */
-	struct panel_module *mod;
-};
-#define to_panel_connector(x) container_of(x, struct panel_connector, base)
-
-
-static void panel_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
-static int panel_connector_get_modes(struct drm_connector *connector)
-{
-	struct drm_device *dev = connector->dev;
-	struct panel_connector *panel_connector = to_panel_connector(connector);
-	struct display_timings *timings = panel_connector->mod->timings;
-	int i;
-
-	for (i = 0; i < timings->num_timings; i++) {
-		struct drm_display_mode *mode;
-		struct videomode vm;
-
-		if (videomode_from_timings(timings, &vm, i))
-			break;
-
-		mode = drm_mode_create(dev);
-		if (!mode)
-			break;
-
-		drm_display_mode_from_videomode(&vm, mode);
-
-		mode->type = DRM_MODE_TYPE_DRIVER;
-
-		if (timings->native_mode == i)
-			mode->type |= DRM_MODE_TYPE_PREFERRED;
-
-		drm_mode_set_name(mode);
-		drm_mode_probed_add(connector, mode);
-	}
-
-	return i;
-}
-
-static struct drm_encoder *panel_connector_best_encoder(
-		struct drm_connector *connector)
-{
-	struct panel_connector *panel_connector = to_panel_connector(connector);
-	return panel_connector->encoder;
-}
-
-static const struct drm_connector_funcs panel_connector_funcs = {
-	.destroy            = panel_connector_destroy,
-	.fill_modes         = drm_helper_probe_single_connector_modes,
-	.reset              = drm_atomic_helper_connector_reset,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-static const struct drm_connector_helper_funcs panel_connector_helper_funcs = {
-	.get_modes          = panel_connector_get_modes,
-	.best_encoder       = panel_connector_best_encoder,
-};
-
-static struct drm_connector *panel_connector_create(struct drm_device *dev,
-		struct panel_module *mod, struct drm_encoder *encoder)
-{
-	struct panel_connector *panel_connector;
-	struct drm_connector *connector;
-	int ret;
-
-	panel_connector = devm_kzalloc(dev->dev, sizeof(*panel_connector),
-				       GFP_KERNEL);
-	if (!panel_connector)
-		return NULL;
-
-	panel_connector->encoder = encoder;
-	panel_connector->mod = mod;
-
-	connector = &panel_connector->base;
-
-	drm_connector_init(dev, connector, &panel_connector_funcs,
-			DRM_MODE_CONNECTOR_LVDS);
-	drm_connector_helper_add(connector, &panel_connector_helper_funcs);
-
-	connector->interlace_allowed = 0;
-	connector->doublescan_allowed = 0;
-
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret)
-		goto fail;
-
-	return connector;
-
-fail:
-	panel_connector_destroy(connector);
-	return NULL;
-}
-
-/*
- * Module:
- */
-
-static int panel_modeset_init(struct tilcdc_module *mod, struct drm_device *dev)
-{
-	struct panel_module *panel_mod = to_panel_module(mod);
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	struct drm_encoder *encoder;
-	struct drm_connector *connector;
-
-	encoder = panel_encoder_create(dev, panel_mod);
-	if (!encoder)
-		return -ENOMEM;
-
-	connector = panel_connector_create(dev, panel_mod, encoder);
-	if (!connector)
-		return -ENOMEM;
-
-	priv->encoders[priv->num_encoders++] = encoder;
-	priv->connectors[priv->num_connectors++] = connector;
-
-	tilcdc_crtc_set_panel_info(priv->crtc,
-				   to_panel_encoder(encoder)->mod->info);
-
-	return 0;
-}
-
-static const struct tilcdc_module_ops panel_module_ops = {
-		.modeset_init = panel_modeset_init,
-};
-
-/*
- * Device:
- */
-
-/* maybe move this somewhere common if it is needed by other outputs? */
-static struct tilcdc_panel_info *of_get_panel_info(struct device_node *np)
-{
-	struct device_node *info_np;
-	struct tilcdc_panel_info *info;
-	int ret = 0;
-
-	if (!np) {
-		pr_err("%s: no devicenode given\n", __func__);
-		return NULL;
-	}
-
-	info_np = of_get_child_by_name(np, "panel-info");
-	if (!info_np) {
-		pr_err("%s: could not find panel-info node\n", __func__);
-		return NULL;
-	}
-
-	info = kzalloc_obj(*info);
-	if (!info)
-		goto put_node;
-
-	ret |= of_property_read_u32(info_np, "ac-bias", &info->ac_bias);
-	ret |= of_property_read_u32(info_np, "ac-bias-intrpt", &info->ac_bias_intrpt);
-	ret |= of_property_read_u32(info_np, "dma-burst-sz", &info->dma_burst_sz);
-	ret |= of_property_read_u32(info_np, "bpp", &info->bpp);
-	ret |= of_property_read_u32(info_np, "fdd", &info->fdd);
-	ret |= of_property_read_u32(info_np, "sync-edge", &info->sync_edge);
-	ret |= of_property_read_u32(info_np, "sync-ctrl", &info->sync_ctrl);
-	ret |= of_property_read_u32(info_np, "raster-order", &info->raster_order);
-	ret |= of_property_read_u32(info_np, "fifo-th", &info->fifo_th);
-
-	/* optional: */
-	info->tft_alt_mode      = of_property_read_bool(info_np, "tft-alt-mode");
-	info->invert_pxl_clk    = of_property_read_bool(info_np, "invert-pxl-clk");
-
-	if (ret) {
-		pr_err("%s: error reading panel-info properties\n", __func__);
-		kfree(info);
-		info = NULL;
-	}
-
-put_node:
-	of_node_put(info_np);
-	return info;
-}
-
-static int panel_probe(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	struct backlight_device *backlight;
-	struct panel_module *panel_mod;
-	struct tilcdc_module *mod;
-	int ret;
-
-	/* bail out early if no DT data: */
-	if (!node) {
-		dev_err(&pdev->dev, "device-tree data is missing\n");
-		return -ENXIO;
-	}
-
-	panel_mod = devm_kzalloc(&pdev->dev, sizeof(*panel_mod), GFP_KERNEL);
-	if (!panel_mod)
-		return -ENOMEM;
-
-	backlight = devm_of_find_backlight(&pdev->dev);
-	if (IS_ERR(backlight))
-		return PTR_ERR(backlight);
-	panel_mod->backlight = backlight;
-
-	panel_mod->enable_gpio = devm_gpiod_get_optional(&pdev->dev, "enable",
-							 GPIOD_OUT_LOW);
-	if (IS_ERR(panel_mod->enable_gpio)) {
-		ret = PTR_ERR(panel_mod->enable_gpio);
-		dev_err(&pdev->dev, "failed to request enable GPIO\n");
-		goto fail_backlight;
-	}
-
-	if (panel_mod->enable_gpio)
-		dev_info(&pdev->dev, "found enable GPIO\n");
-
-	mod = &panel_mod->base;
-	pdev->dev.platform_data = mod;
-
-	tilcdc_module_init(mod, "panel", &panel_module_ops);
-
-	panel_mod->timings = of_get_display_timings(node);
-	if (!panel_mod->timings) {
-		dev_err(&pdev->dev, "could not get panel timings\n");
-		ret = -EINVAL;
-		goto fail_free;
-	}
-
-	panel_mod->info = of_get_panel_info(node);
-	if (!panel_mod->info) {
-		dev_err(&pdev->dev, "could not get panel info\n");
-		ret = -EINVAL;
-		goto fail_timings;
-	}
-
-	return 0;
-
-fail_timings:
-	display_timings_release(panel_mod->timings);
-
-fail_free:
-	tilcdc_module_cleanup(mod);
-
-fail_backlight:
-	if (panel_mod->backlight)
-		put_device(&panel_mod->backlight->dev);
-	return ret;
-}
-
-static void panel_remove(struct platform_device *pdev)
-{
-	struct tilcdc_module *mod = dev_get_platdata(&pdev->dev);
-	struct panel_module *panel_mod = to_panel_module(mod);
-	struct backlight_device *backlight = panel_mod->backlight;
-
-	if (backlight)
-		put_device(&backlight->dev);
-
-	display_timings_release(panel_mod->timings);
-
-	tilcdc_module_cleanup(mod);
-	kfree(panel_mod->info);
-}
-
-static const struct of_device_id panel_of_match[] = {
-		{ .compatible = "ti,tilcdc,panel", },
-		{ },
-};
-
-static struct platform_driver panel_driver = {
-	.probe = panel_probe,
-	.remove = panel_remove,
-	.driver = {
-		.name = "tilcdc-panel",
-		.of_match_table = panel_of_match,
-	},
-};
-
-int __init tilcdc_panel_init(void)
-{
-	return platform_driver_register(&panel_driver);
-}
-
-void __exit tilcdc_panel_fini(void)
-{
-	platform_driver_unregister(&panel_driver);
-}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.h b/drivers/gpu/drm/tilcdc/tilcdc_panel.h
deleted file mode 100644
index 65d735d773a48..0000000000000
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2012 Texas Instruments
- * Author: Rob Clark <robdclark@gmail.com>
- */
-
-#ifndef __TILCDC_PANEL_H__
-#define __TILCDC_PANEL_H__
-
-/* sub-module for generic lcd panel output */
-
-int tilcdc_panel_init(void);
-void tilcdc_panel_fini(void);
-
-#endif /* __TILCDC_PANEL_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c b/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c
new file mode 100644
index 0000000000000..2e7b3e87fea18
--- /dev/null
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Bootlin
+ * Author: Kory Maincent <kory.maincent@bootlin.com>
+ *
+ * To support the legacy "ti,tilcdc,panel" binding, the devicetree has to
+ * be transformed to the new panel-dpi binding with the endpoint associated.
+ */
+
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/slab.h>
+
+/* Embedded dtbo symbols created by cmd_wrap_S_dtb in scripts/Makefile.lib */
+extern char __dtbo_tilcdc_panel_legacy_begin[];
+extern char __dtbo_tilcdc_panel_legacy_end[];
+
+static int __init
+tilcdc_panel_update_prop(struct of_changeset *ocs, struct device_node *node,
+			 char *name, void *val, int length)
+{
+	struct property *prop;
+
+	prop = kzalloc(sizeof(*prop), GFP_KERNEL);
+	if (!prop)
+		return -ENOMEM;
+
+	prop->name = kstrdup(name, GFP_KERNEL);
+	prop->length = length;
+	prop->value = kmemdup(val, length, GFP_KERNEL);
+	if (!prop->name || !prop->value) {
+		kfree(prop->name);
+		kfree(prop->value);
+		kfree(prop);
+		return -ENOMEM;
+	}
+
+	return of_changeset_update_property(ocs, node, prop);
+}
+
+static int __init tilcdc_panel_copy_props(struct device_node *old_panel,
+					  struct device_node *new_panel)
+{
+	struct device_node *old_timing __free(device_node) = NULL;
+	struct device_node *new_timing __free(device_node) = NULL;
+	struct device_node *panel_info __free(device_node) = NULL;
+	struct device_node *child __free(device_node) = NULL;
+	u32 invert_pxl_clk = 0, sync_edge = 0;
+	struct of_changeset ocs;
+	struct property *prop;
+	int ret;
+
+	child = of_get_child_by_name(old_panel, "display-timings");
+	if (!child)
+		return -EINVAL;
+
+	/* The default display timing is the one specified as native-mode.
+	 * If no native-mode is specified then the first node is assumed
+	 * to be the native mode.
+	 */
+	old_timing = of_parse_phandle(child, "native-mode", 0);
+	if (!old_timing) {
+		old_timing = of_get_next_child(child, NULL);
+		if (!old_timing)
+			return -EINVAL;
+	}
+
+	panel_info = of_get_child_by_name(old_panel, "panel-info");
+	if (!panel_info)
+		return -EINVAL;
+
+	of_changeset_init(&ocs);
+
+	/* Copy all panel properties to the new panel node */
+	for_each_property_of_node(old_panel, prop) {
+		if (!strncmp(prop->name, "compatible", sizeof("compatible")))
+			continue;
+
+		ret = tilcdc_panel_update_prop(&ocs, new_panel, prop->name,
+					       prop->value, prop->length);
+		if (ret)
+			goto destroy_ocs;
+	}
+
+	new_timing = of_changeset_create_node(&ocs, new_panel, "panel-timing");
+	if (!new_timing) {
+		ret = -ENODEV;
+		goto destroy_ocs;
+	}
+
+	/* Copy all panel timing properties to the new panel node */
+	for_each_property_of_node(old_timing, prop) {
+		ret = tilcdc_panel_update_prop(&ocs, new_timing, prop->name,
+					       prop->value, prop->length);
+		if (ret)
+			goto destroy_ocs;
+	}
+
+	/* Looked only for these two parameter as all the other are always
+	 * set to default and not related to common DRM properties.
+	 */
+	of_property_read_u32(panel_info, "invert-pxl-clk", &invert_pxl_clk);
+	of_property_read_u32(panel_info, "sync-edge", &sync_edge);
+
+	if (!invert_pxl_clk) {
+		ret = tilcdc_panel_update_prop(&ocs, new_timing, "pixelclk-active",
+					       &(__be32){cpu_to_be32(1)}, sizeof(__be32));
+		if (ret)
+			goto destroy_ocs;
+	}
+
+	if (!sync_edge) {
+		ret = tilcdc_panel_update_prop(&ocs, new_timing, "syncclk-active",
+					       &(__be32){cpu_to_be32(1)}, sizeof(__be32));
+		if (ret)
+			goto destroy_ocs;
+	}
+
+	/* Remove compatible property to avoid any driver compatible match */
+	of_changeset_remove_property(&ocs, old_panel,
+				     of_find_property(old_panel, "compatible", NULL));
+
+	of_changeset_apply(&ocs);
+	return 0;
+
+destroy_ocs:
+	of_changeset_destroy(&ocs);
+	return ret;
+}
+
+static const struct of_device_id tilcdc_panel_of_match[] __initconst = {
+	{ .compatible = "ti,tilcdc,panel", },
+	{},
+};
+
+static const struct of_device_id tilcdc_of_match[] __initconst = {
+	{ .compatible = "ti,am33xx-tilcdc", },
+	{ .compatible = "ti,da850-tilcdc", },
+	{},
+};
+
+static int __init tilcdc_panel_legacy_init(void)
+{
+	struct device_node *new_panel __free(device_node) = NULL;
+	struct device_node *panel __free(device_node) = NULL;
+	struct device_node *lcdc __free(device_node) = NULL;
+	void *dtbo_start;
+	u32 dtbo_size;
+	int ovcs_id;
+	int ret;
+
+	lcdc = of_find_matching_node(NULL, tilcdc_of_match);
+	panel = of_find_matching_node(NULL, tilcdc_panel_of_match);
+
+	if (!of_device_is_available(panel) ||
+	    !of_device_is_available(lcdc))
+		return 0;
+
+	dtbo_start = __dtbo_tilcdc_panel_legacy_begin;
+	dtbo_size = __dtbo_tilcdc_panel_legacy_end -
+		    __dtbo_tilcdc_panel_legacy_begin;
+
+	ret = of_overlay_fdt_apply(dtbo_start, dtbo_size, &ovcs_id, NULL);
+	if (ret)
+		return ret;
+
+	new_panel = of_find_node_by_name(NULL, "tilcdc-panel-dpi");
+	if (!new_panel) {
+		ret = -ENODEV;
+		goto overlay_remove;
+	}
+
+	ret = tilcdc_panel_copy_props(panel, new_panel);
+	if (ret)
+		goto overlay_remove;
+
+	return 0;
+
+overlay_remove:
+	of_overlay_remove(&ovcs_id);
+	return ret;
+}
+
+subsys_initcall(tilcdc_panel_legacy_init);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.dtso b/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.dtso
new file mode 100644
index 0000000000000..ae71d10f5ec13
--- /dev/null
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel_legacy.dtso
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * DTS overlay for converting ti,tilcdc,panel binding to new binding.
+ *
+ * Copyright (C) 2025 Bootlin
+ * Author: Kory Maincent <kory.maincent@bootlin.com>
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	tilcdc-panel-dpi {
+		compatible = "panel-dpi";
+		port {
+			panel_in: endpoint@0 {
+				remote-endpoint = <&lcd_0>;
+			};
+		};
+	};
+};
+
+&lcdc {
+	port {
+		lcd_0: endpoint@0 {
+			remote-endpoint = <&panel_in>;
+		};
+	};
+};
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index aa72ca679598b..a9982a9956903 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -14,7 +14,6 @@
 static const struct drm_plane_funcs tilcdc_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_plane_cleanup,
 	.reset		= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -37,7 +36,7 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	if (new_state->crtc_x || new_state->crtc_y) {
-		dev_err(plane->dev->dev, "%s: crtc position must be zero.",
+		drm_err(plane->dev, "%s: crtc position must be zero.",
 			__func__);
 		return -EINVAL;
 	}
@@ -49,7 +48,7 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 
 	if (crtc_state->mode.hdisplay != new_state->crtc_w ||
 	    crtc_state->mode.vdisplay != new_state->crtc_h) {
-		dev_err(plane->dev->dev,
+		drm_err(plane->dev,
 			"%s: Size must match mode (%dx%d == %dx%d)", __func__,
 			crtc_state->mode.hdisplay, crtc_state->mode.vdisplay,
 			new_state->crtc_w, new_state->crtc_h);
@@ -59,13 +58,13 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 	pitch = crtc_state->mode.hdisplay *
 		new_state->fb->format->cpp[0];
 	if (new_state->fb->pitches[0] != pitch) {
-		dev_err(plane->dev->dev,
+		drm_err(plane->dev,
 			"Invalid pitch: fb and crtc widths must be the same");
 		return -EINVAL;
 	}
 
 	if (old_state->fb && new_state->fb->format != old_state->fb->format) {
-		dev_dbg(plane->dev->dev,
+		drm_dbg(plane->dev,
 			"%s(): pixel format change requires mode_change\n",
 			__func__);
 		crtc_state->mode_changed = true;
@@ -98,22 +97,20 @@ static const struct drm_plane_helper_funcs plane_helper_funcs = {
 	.atomic_update = tilcdc_plane_atomic_update,
 };
 
-int tilcdc_plane_init(struct drm_device *dev,
-		      struct drm_plane *plane)
+struct tilcdc_plane *tilcdc_plane_init(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	int ret;
-
-	ret = drm_universal_plane_init(dev, plane, 1, &tilcdc_plane_funcs,
-				       priv->pixelformats,
-				       priv->num_pixelformats,
-				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret) {
-		dev_err(dev->dev, "Failed to initialize plane: %d\n", ret);
-		return ret;
-	}
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
+	struct tilcdc_plane *plane;
 
-	drm_plane_helper_add(plane, &plane_helper_funcs);
+	plane = drmm_universal_plane_alloc(dev, struct tilcdc_plane, base,
+					   1, &tilcdc_plane_funcs,
+					   priv->pixelformats,
+					   priv->num_pixelformats,
+					   NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (IS_ERR(plane))
+		return plane;
 
-	return 0;
+	drm_plane_helper_add(&plane->base, &plane_helper_funcs);
+
+	return plane;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_regs.h b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
index f90e2dc3457cd..26ebaf1e0f70f 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_regs.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
@@ -113,13 +113,13 @@
 
 static inline void tilcdc_write(struct drm_device *dev, u32 reg, u32 data)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	iowrite32(data, priv->mmio + reg);
 }
 
 static inline void tilcdc_write64(struct drm_device *dev, u32 reg, u64 data)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	volatile void __iomem *addr = priv->mmio + reg;
 
 #if defined(iowrite64) && !defined(iowrite64_is_nonatomic)
@@ -133,7 +133,7 @@ static inline void tilcdc_write64(struct drm_device *dev, u32 reg, u64 data)
 
 static inline u32 tilcdc_read(struct drm_device *dev, u32 reg)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	return ioread32(priv->mmio + reg);
 }
 
@@ -156,7 +156,7 @@ static inline void tilcdc_clear(struct drm_device *dev, u32 reg, u32 mask)
 /* the register to read/clear irqstatus differs between v1 and v2 of the IP */
 static inline u32 tilcdc_irqstatus_reg(struct drm_device *dev)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_drm_private *priv = ddev_to_tilcdc_priv(dev);
 	return (priv->rev == 2) ? LCDC_MASKED_STAT_REG : LCDC_STAT_REG;
 }
 
diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
index 6d95447a989d2..2db221f6fc3a1 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
+++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
@@ -251,7 +251,7 @@ static void ttm_bo_validate_basic(struct kunit *test)
 				   NULL, &dummy_ttm_bo_destroy);
 	KUNIT_EXPECT_EQ(test, err, 0);
 
-	snd_place = ttm_place_kunit_init(test, snd_mem, DRM_BUDDY_TOPDOWN_ALLOCATION);
+	snd_place = ttm_place_kunit_init(test, snd_mem, GPU_BUDDY_TOPDOWN_ALLOCATION);
 	snd_placement = ttm_placement_kunit_init(test, snd_place, 1);
 
 	err = ttm_bo_validate(bo, snd_placement, &ctx_val);
@@ -263,7 +263,7 @@ static void ttm_bo_validate_basic(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, ttm_tt_is_populated(bo->ttm));
 	KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem);
 	KUNIT_EXPECT_EQ(test, bo->resource->placement,
-			DRM_BUDDY_TOPDOWN_ALLOCATION);
+			GPU_BUDDY_TOPDOWN_ALLOCATION);
 
 	ttm_bo_fini(bo);
 	ttm_mock_manager_fini(priv->ttm_dev, snd_mem);
@@ -759,6 +759,46 @@ static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test)
 	ttm_mock_manager_fini(priv->ttm_dev, snd_mem);
 }
 
+static void ttm_bo_validate_swapout(struct kunit *test)
+{
+	u32 mem_type = TTM_PL_TT;
+	struct ttm_test_devices *priv = test->priv;
+	struct ttm_operation_ctx ctx_init = { };
+	enum ttm_bo_type bo_type = ttm_bo_type_device;
+	struct ttm_resource_manager *man;
+	struct ttm_placement *placement;
+	struct ttm_buffer_object *bo;
+	struct ttm_place *place;
+	int err;
+	s64 swapped;
+
+	ttm_mock_manager_init(priv->ttm_dev, mem_type, MANAGER_SIZE);
+	man = ttm_manager_type(priv->ttm_dev, mem_type);
+	KUNIT_ASSERT_NOT_NULL(test, man);
+
+	place = ttm_place_kunit_init(test, mem_type, 0);
+	placement = ttm_placement_kunit_init(test, place, 1);
+
+	bo = kunit_kzalloc(test, sizeof(*bo), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, bo);
+
+	drm_gem_private_object_init(priv->drm, &bo->base, MANAGER_SIZE);
+	err = ttm_bo_init_reserved(priv->ttm_dev, bo, bo_type, placement,
+				   PAGE_SIZE, &ctx_init, NULL, NULL,
+				   &dummy_ttm_bo_destroy);
+	KUNIT_EXPECT_EQ(test, err, 0);
+	dma_resv_unlock(bo->base.resv);
+
+	swapped = ttm_bo_swapout(priv->ttm_dev, &ctx_init, man, GFP_KERNEL,
+				 MANAGER_SIZE / PAGE_SIZE);
+	KUNIT_EXPECT_EQ(test, swapped, MANAGER_SIZE / PAGE_SIZE);
+	KUNIT_EXPECT_EQ(test, bo->resource->mem_type, TTM_PL_SYSTEM);
+	KUNIT_EXPECT_TRUE(test, bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED);
+
+	ttm_bo_fini(bo);
+	ttm_mock_manager_fini(priv->ttm_dev, mem_type);
+}
+
 static void ttm_bo_validate_happy_evict(struct kunit *test)
 {
 	u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT,
@@ -1153,6 +1193,7 @@ static struct kunit_case ttm_bo_validate_test_cases[] = {
 	KUNIT_CASE(ttm_bo_validate_move_fence_signaled),
 	KUNIT_CASE_PARAM(ttm_bo_validate_move_fence_not_signaled,
 			 ttm_bo_validate_wait_gen_params),
+	KUNIT_CASE(ttm_bo_validate_swapout),
 	KUNIT_CASE(ttm_bo_validate_happy_evict),
 	KUNIT_CASE(ttm_bo_validate_all_pinned_evict),
 	KUNIT_CASE(ttm_bo_validate_allowed_only_evict),
diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
index c97e8b7a4e7a7..0d91bc51f1a46 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
+++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c
@@ -31,7 +31,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
 {
 	struct ttm_mock_manager *manager = to_mock_mgr(man);
 	struct ttm_mock_resource *mock_res;
-	struct drm_buddy *mm = &manager->mm;
+	struct gpu_buddy *mm = &manager->mm;
 	u64 lpfn, fpfn, alloc_size;
 	int err;
 
@@ -47,14 +47,14 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
 	INIT_LIST_HEAD(&mock_res->blocks);
 
 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
-		mock_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
+		mock_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
 
 	if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
-		mock_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
+		mock_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
 
 	alloc_size = (uint64_t)mock_res->base.size;
 	mutex_lock(&manager->lock);
-	err = drm_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size,
+	err = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size,
 				     manager->default_page_size,
 				     &mock_res->blocks,
 				     mock_res->flags);
@@ -67,7 +67,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man,
 	return 0;
 
 error_free_blocks:
-	drm_buddy_free_list(mm, &mock_res->blocks, 0);
+	gpu_buddy_free_list(mm, &mock_res->blocks, 0);
 	ttm_resource_fini(man, &mock_res->base);
 	mutex_unlock(&manager->lock);
 
@@ -79,10 +79,10 @@ static void ttm_mock_manager_free(struct ttm_resource_manager *man,
 {
 	struct ttm_mock_manager *manager = to_mock_mgr(man);
 	struct ttm_mock_resource *mock_res = to_mock_mgr_resource(res);
-	struct drm_buddy *mm = &manager->mm;
+	struct gpu_buddy *mm = &manager->mm;
 
 	mutex_lock(&manager->lock);
-	drm_buddy_free_list(mm, &mock_res->blocks, 0);
+	gpu_buddy_free_list(mm, &mock_res->blocks, 0);
 	mutex_unlock(&manager->lock);
 
 	ttm_resource_fini(man, res);
@@ -106,7 +106,7 @@ int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size)
 
 	mutex_init(&manager->lock);
 
-	err = drm_buddy_init(&manager->mm, size, PAGE_SIZE);
+	err = gpu_buddy_init(&manager->mm, size, PAGE_SIZE);
 
 	if (err) {
 		kfree(manager);
@@ -142,7 +142,7 @@ void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type)
 	ttm_resource_manager_set_used(man, false);
 
 	mutex_lock(&mock_man->lock);
-	drm_buddy_fini(&mock_man->mm);
+	gpu_buddy_fini(&mock_man->mm);
 	mutex_unlock(&mock_man->lock);
 
 	ttm_set_driver_manager(bdev, mem_type, NULL);
diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
index e4c95f86a4670..08710756fd8e0 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
+++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h
@@ -5,11 +5,11 @@
 #ifndef TTM_MOCK_MANAGER_H
 #define TTM_MOCK_MANAGER_H
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 
 struct ttm_mock_manager {
 	struct ttm_resource_manager man;
-	struct drm_buddy mm;
+	struct gpu_buddy mm;
 	u64 default_page_size;
 	/* protects allocations of mock buffer objects */
 	struct mutex lock;
diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c
index 6bd4c123d94cd..81df4cb5606b4 100644
--- a/drivers/gpu/drm/ttm/ttm_backup.c
+++ b/drivers/gpu/drm/ttm/ttm_backup.c
@@ -44,18 +44,20 @@ void ttm_backup_drop(struct file *backup, pgoff_t handle)
  * @dst: The struct page to copy into.
  * @handle: The handle returned when the page was backed up.
  * @intr: Try to perform waits interruptible or at least killable.
+ * @additional_gfp: GFP mask to add to the default GFP mask if any.
  *
  * Return: 0 on success, Negative error code on failure, notably
  * -EINTR if @intr was set to true and a signal is pending.
  */
 int ttm_backup_copy_page(struct file *backup, struct page *dst,
-			 pgoff_t handle, bool intr)
+			 pgoff_t handle, bool intr, gfp_t additional_gfp)
 {
 	struct address_space *mapping = backup->f_mapping;
 	struct folio *from_folio;
 	pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle);
 
-	from_folio = shmem_read_folio(mapping, idx);
+	from_folio = shmem_read_folio_gfp(mapping, idx, mapping_gfp_mask(mapping)
+					  | additional_gfp);
 	if (IS_ERR(from_folio))
 		return PTR_ERR(from_folio);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 0765d69423d2f..d85f0a37ac35f 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -32,6 +32,7 @@
 #define pr_fmt(fmt) "[TTM] " fmt
 
 #include <drm/drm_print.h>
+#include <drm/drm_util.h>
 #include <drm/ttm/ttm_allocation.h>
 #include <drm/ttm/ttm_bo.h>
 #include <drm/ttm/ttm_placement.h>
@@ -222,10 +223,8 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 	struct dma_fence *fence;
 
 	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
-	dma_resv_for_each_fence_unlocked(&cursor, fence) {
-		if (!fence->ops->signaled)
-			dma_fence_enable_sw_signaling(fence);
-	}
+	dma_resv_for_each_fence_unlocked(&cursor, fence)
+		dma_fence_enable_sw_signaling(fence);
 	dma_resv_iter_end(&cursor);
 }
 
@@ -1210,9 +1209,9 @@ const struct ttm_lru_walk_ops ttm_swap_ops = {
  * @man: The resource manager whose resources / buffer objects are
  * goint to be swapped out.
  * @gfp_flags: The gfp flags used for shmem page allocations.
- * @target: The desired number of bytes to swap out.
+ * @target: The desired number of pages to swap out.
  *
- * Return: The number of bytes actually swapped out, or negative error code
+ * Return: The number of pages actually swapped out, or negative error code
  * on error.
  */
 s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
@@ -1232,6 +1231,7 @@ s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
 
 	return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target);
 }
+EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_bo_swapout);
 
 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
 {
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index c0d95559197c6..aa41099c5ecfc 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -530,6 +530,8 @@ static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore,
 		p = first_page[i];
 		if (ttm_backup_page_ptr_is_handle(p)) {
 			unsigned long handle = ttm_backup_page_ptr_to_handle(p);
+			gfp_t additional_gfp = ctx->gfp_retry_mayfail ?
+				__GFP_RETRY_MAYFAIL | __GFP_NOWARN : 0;
 
 			if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible &&
 			    should_fail(&backup_fault_inject, 1)) {
@@ -543,7 +545,8 @@ static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore,
 			}
 
 			ret = ttm_backup_copy_page(backup, restore->alloced_page + i,
-						   handle, ctx->interruptible);
+						   handle, ctx->interruptible,
+						   additional_gfp);
 			if (ret)
 				break;
 
@@ -726,7 +729,7 @@ static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 		gfp_flags |= __GFP_ZERO;
 
 	if (ctx->gfp_retry_mayfail)
-		gfp_flags |= __GFP_RETRY_MAYFAIL;
+		gfp_flags |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
 
 	if (ttm_pool_uses_dma32(pool))
 		gfp_flags |= GFP_DMA32;
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 192fca24f37e4..9f36631d48b61 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -37,7 +37,7 @@
 #include <drm/drm_print.h>
 #include <drm/drm_util.h>
 
-/* Detach the cursor from the bulk move list*/
+/* Detach the cursor from the bulk move list */
 static void
 ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
 {
@@ -105,9 +105,9 @@ void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor,
  * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
  * @cursor: The struct ttm_resource_cursor to finalize.
  *
- * The function pulls the LRU list cursor off any lists it was previusly
+ * The function pulls the LRU list cursor off any lists it was previously
  * attached to. Needs to be called with the LRU lock held. The function
- * can be called multiple times after eachother.
+ * can be called multiple times after each other.
  */
 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
 {
@@ -317,10 +317,10 @@ void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
 }
 
 /**
- * ttm_resource_init - resource object constructure
- * @bo: buffer object this resources is allocated for
+ * ttm_resource_init - resource object constructor
+ * @bo: buffer object this resource is allocated for
  * @place: placement of the resource
- * @res: the resource object to inistilize
+ * @res: the resource object to initialize
  *
  * Initialize a new resource object. Counterpart of ttm_resource_fini().
  */
@@ -435,7 +435,7 @@ EXPORT_SYMBOL(ttm_resource_free);
  * @size: How many bytes the new allocation needs.
  *
  * Test if @res intersects with @place and @size. Used for testing if evictions
- * are valueable or not.
+ * are valuable or not.
  *
  * Returns true if the res placement intersects with @place and @size.
  */
@@ -513,7 +513,7 @@ void ttm_resource_set_bo(struct ttm_resource *res,
  * @bdev: ttm device this manager belongs to
  * @size: size of managed resources in arbitrary units
  *
- * Initialise core parts of a manager object.
+ * Initialize core parts of a manager object.
  */
 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 			       struct ttm_device *bdev,
@@ -536,8 +536,8 @@ EXPORT_SYMBOL(ttm_resource_manager_init);
 /*
  * ttm_resource_manager_evict_all
  *
- * @bdev - device to use
- * @man - manager to use
+ * @bdev: device to use
+ * @man: manager to use
  *
  * Evict all the objects out of a memory manager until it is empty.
  * Part of memory manager cleanup sequence.
@@ -882,7 +882,7 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 
 /**
  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
- * @iter_io: The iterator to initialize
+ * @iter_io: The iterator to finalize
  * @bdev: The TTM device
  * @mem: The ttm resource representing the iomap.
  *
@@ -921,15 +921,15 @@ DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
 /**
  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
  * resource manager.
- * @man: The TTM resource manager for which the debugfs stats file be creates
+ * @man: The TTM resource manager for which the debugfs stats file to be created
  * @parent: debugfs directory in which the file will reside
  * @name: The filename to create.
  *
- * This function setups up a debugfs file that can be used to look
+ * This function sets up a debugfs file that can be used to look
  * at debug statistics of the specified ttm_resource_manager.
  */
 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
-					 struct dentry * parent,
+					 struct dentry *parent,
 					 const char *name)
 {
 #if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
index beeffe36b6cb0..6114346415805 100644
--- a/drivers/gpu/drm/tyr/driver.rs
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -1,44 +1,56 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
-use kernel::clk::Clk;
-use kernel::clk::OptionalClk;
-use kernel::device::Bound;
-use kernel::device::Core;
-use kernel::device::Device;
-use kernel::devres::Devres;
-use kernel::drm;
-use kernel::drm::ioctl;
-use kernel::io::poll;
-use kernel::new_mutex;
-use kernel::of;
-use kernel::platform;
-use kernel::prelude::*;
-use kernel::regulator;
-use kernel::regulator::Regulator;
-use kernel::sizes::SZ_2M;
-use kernel::sync::aref::ARef;
-use kernel::sync::Arc;
-use kernel::sync::Mutex;
-use kernel::time;
-
-use crate::file::File;
-use crate::gem::TyrObject;
-use crate::gpu;
-use crate::gpu::GpuInfo;
-use crate::regs;
+use kernel::{
+    clk::{
+        Clk,
+        OptionalClk, //
+    },
+    device::{
+        Bound,
+        Core,
+        Device, //
+    },
+    devres::Devres,
+    drm,
+    drm::ioctl,
+    io::poll,
+    new_mutex,
+    of,
+    platform,
+    prelude::*,
+    regulator,
+    regulator::Regulator,
+    sizes::SZ_2M,
+    sync::{
+        aref::ARef,
+        Arc,
+        Mutex, //
+    },
+    time, //
+};
+
+use crate::{
+    file::TyrDrmFileData,
+    gem::TyrObject,
+    gpu,
+    gpu::GpuInfo,
+    regs, //
+};
 
 pub(crate) type IoMem = kernel::io::mem::IoMem<SZ_2M>;
 
+pub(crate) struct TyrDrmDriver;
+
 /// Convenience type alias for the DRM device type for this driver.
-pub(crate) type TyrDevice = drm::Device<TyrDriver>;
+pub(crate) type TyrDrmDevice = drm::Device<TyrDrmDriver>;
 
 #[pin_data(PinnedDrop)]
-pub(crate) struct TyrDriver {
-    _device: ARef<TyrDevice>,
+pub(crate) struct TyrPlatformDriverData {
+    _device: ARef<TyrDrmDevice>,
 }
 
 #[pin_data(PinnedDrop)]
-pub(crate) struct TyrData {
+pub(crate) struct TyrDrmDeviceData {
     pub(crate) pdev: ARef<platform::Device>,
 
     #[pin]
@@ -61,9 +73,9 @@ pub(crate) struct TyrData {
 // that it will be removed in a future patch.
 //
 // SAFETY: This will be removed in a future patch.
-unsafe impl Send for TyrData {}
+unsafe impl Send for TyrDrmDeviceData {}
 // SAFETY: This will be removed in a future patch.
-unsafe impl Sync for TyrData {}
+unsafe impl Sync for TyrDrmDeviceData {}
 
 fn issue_soft_reset(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result {
     regs::GPU_CMD.write(dev, iomem, regs::GPU_CMD_SOFT_RESET)?;
@@ -82,14 +94,14 @@ fn issue_soft_reset(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result {
 kernel::of_device_table!(
     OF_TABLE,
     MODULE_OF_TABLE,
-    <TyrDriver as platform::Driver>::IdInfo,
+    <TyrPlatformDriverData as platform::Driver>::IdInfo,
     [
         (of::DeviceId::new(c"rockchip,rk3588-mali"), ()),
         (of::DeviceId::new(c"arm,mali-valhall-csf"), ())
     ]
 );
 
-impl platform::Driver for TyrDriver {
+impl platform::Driver for TyrPlatformDriverData {
     type IdInfo = ();
     const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE);
 
@@ -119,7 +131,7 @@ fn probe(
 
         let platform: ARef<platform::Device> = pdev.into();
 
-        let data = try_pin_init!(TyrData {
+        let data = try_pin_init!(TyrDrmDeviceData {
                 pdev: platform.clone(),
                 clks <- new_mutex!(Clocks {
                     core: core_clk,
@@ -133,10 +145,10 @@ fn probe(
                 gpu_info,
         });
 
-        let tdev: ARef<TyrDevice> = drm::Device::new(pdev.as_ref(), data)?;
-        drm::driver::Registration::new_foreign_owned(&tdev, pdev.as_ref(), 0)?;
+        let ddev: ARef<TyrDrmDevice> = drm::Device::new(pdev.as_ref(), data)?;
+        drm::driver::Registration::new_foreign_owned(&ddev, pdev.as_ref(), 0)?;
 
-        let driver = TyrDriver { _device: tdev };
+        let driver = TyrPlatformDriverData { _device: ddev };
 
         // We need this to be dev_info!() because dev_dbg!() does not work at
         // all in Rust for now, and we need to see whether probe succeeded.
@@ -146,12 +158,12 @@ fn probe(
 }
 
 #[pinned_drop]
-impl PinnedDrop for TyrDriver {
+impl PinnedDrop for TyrPlatformDriverData {
     fn drop(self: Pin<&mut Self>) {}
 }
 
 #[pinned_drop]
-impl PinnedDrop for TyrData {
+impl PinnedDrop for TyrDrmDeviceData {
     fn drop(self: Pin<&mut Self>) {
         // TODO: the type-state pattern for Clks will fix this.
         let clks = self.clks.lock();
@@ -172,15 +184,15 @@ fn drop(self: Pin<&mut Self>) {
 };
 
 #[vtable]
-impl drm::Driver for TyrDriver {
-    type Data = TyrData;
-    type File = File;
+impl drm::Driver for TyrDrmDriver {
+    type Data = TyrDrmDeviceData;
+    type File = TyrDrmFileData;
     type Object = drm::gem::Object<TyrObject>;
 
     const INFO: drm::DriverInfo = INFO;
 
     kernel::declare_drm_ioctls! {
-        (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, File::dev_query),
+        (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, TyrDrmFileData::dev_query),
     }
 }
 
diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs
index 0ef432947b73d..31411da203c5d 100644
--- a/drivers/gpu/drm/tyr/file.rs
+++ b/drivers/gpu/drm/tyr/file.rs
@@ -1,37 +1,41 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
-use kernel::drm;
-use kernel::prelude::*;
-use kernel::uaccess::UserSlice;
-use kernel::uapi;
-
-use crate::driver::TyrDevice;
-use crate::TyrDriver;
+use kernel::{
+    drm,
+    prelude::*,
+    uaccess::UserSlice,
+    uapi, //
+};
+
+use crate::driver::{
+    TyrDrmDevice,
+    TyrDrmDriver, //
+};
 
 #[pin_data]
-pub(crate) struct File {}
+pub(crate) struct TyrDrmFileData {}
 
 /// Convenience type alias for our DRM `File` type
-pub(crate) type DrmFile = drm::file::File<File>;
+pub(crate) type TyrDrmFile = drm::file::File<TyrDrmFileData>;
 
-impl drm::file::DriverFile for File {
-    type Driver = TyrDriver;
+impl drm::file::DriverFile for TyrDrmFileData {
+    type Driver = TyrDrmDriver;
 
     fn open(_dev: &drm::Device<Self::Driver>) -> Result<Pin<KBox<Self>>> {
         KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL)
     }
 }
 
-impl File {
+impl TyrDrmFileData {
     pub(crate) fn dev_query(
-        tdev: &TyrDevice,
+        ddev: &TyrDrmDevice,
         devquery: &mut uapi::drm_panthor_dev_query,
-        _file: &DrmFile,
+        _file: &TyrDrmFile,
     ) -> Result<u32> {
         if devquery.pointer == 0 {
             match devquery.type_ {
                 uapi::drm_panthor_dev_query_type_DRM_PANTHOR_DEV_QUERY_GPU_INFO => {
-                    devquery.size = core::mem::size_of_val(&tdev.gpu_info) as u32;
+                    devquery.size = core::mem::size_of_val(&ddev.gpu_info) as u32;
                     Ok(0)
                 }
                 _ => Err(EINVAL),
@@ -45,7 +49,7 @@ pub(crate) fn dev_query(
                     )
                     .writer();
 
-                    writer.write(&tdev.gpu_info)?;
+                    writer.write(&ddev.gpu_info)?;
 
                     Ok(0)
                 }
diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs
index 1273bf89dbd5d..5cd0cd9585e8d 100644
--- a/drivers/gpu/drm/tyr/gem.rs
+++ b/drivers/gpu/drm/tyr/gem.rs
@@ -1,18 +1,23 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
-use crate::driver::TyrDevice;
-use crate::driver::TyrDriver;
-use kernel::drm::gem;
-use kernel::prelude::*;
+use kernel::{
+    drm::gem,
+    prelude::*, //
+};
+
+use crate::driver::{
+    TyrDrmDevice,
+    TyrDrmDriver, //
+};
 
 /// GEM Object inner driver data
 #[pin_data]
 pub(crate) struct TyrObject {}
 
 impl gem::DriverObject for TyrObject {
-    type Driver = TyrDriver;
+    type Driver = TyrDrmDriver;
 
-    fn new(_dev: &TyrDevice, _size: usize) -> impl PinInit<Self, Error> {
+    fn new(_dev: &TyrDrmDevice, _size: usize) -> impl PinInit<Self, Error> {
         try_pin_init!(TyrObject {})
     }
 }
diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs
index 64ca8311d4e8e..a88775160f981 100644
--- a/drivers/gpu/drm/tyr/gpu.rs
+++ b/drivers/gpu/drm/tyr/gpu.rs
@@ -1,20 +1,28 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
-use core::ops::Deref;
-use core::ops::DerefMut;
-use kernel::bits::genmask_u32;
-use kernel::device::Bound;
-use kernel::device::Device;
-use kernel::devres::Devres;
-use kernel::io::poll;
-use kernel::platform;
-use kernel::prelude::*;
-use kernel::time::Delta;
-use kernel::transmute::AsBytes;
-use kernel::uapi;
-
-use crate::driver::IoMem;
-use crate::regs;
+use core::ops::{
+    Deref,
+    DerefMut, //
+};
+use kernel::{
+    bits::genmask_u32,
+    device::{
+        Bound,
+        Device, //
+    },
+    devres::Devres,
+    io::poll,
+    platform,
+    prelude::*,
+    time::Delta,
+    transmute::AsBytes,
+    uapi, //
+};
+
+use crate::{
+    driver::IoMem,
+    regs, //
+};
 
 /// Struct containing information that can be queried by userspace. This is read from
 /// the GPU's registers.
@@ -84,13 +92,11 @@ pub(crate) fn new(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result<Self> {
     }
 
     pub(crate) fn log(&self, pdev: &platform::Device) {
-        let major = (self.gpu_id >> 16) & 0xff;
-        let minor = (self.gpu_id >> 8) & 0xff;
-        let status = self.gpu_id & 0xff;
+        let gpu_id = GpuId::from(self.gpu_id);
 
         let model_name = if let Some(model) = GPU_MODELS
             .iter()
-            .find(|&f| f.major == major && f.minor == minor)
+            .find(|&f| f.arch_major == gpu_id.arch_major && f.prod_major == gpu_id.prod_major)
         {
             model.name
         } else {
@@ -102,9 +108,9 @@ pub(crate) fn log(&self, pdev: &platform::Device) {
             "mali-{} id 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}",
             model_name,
             self.gpu_id >> 16,
-            major,
-            minor,
-            status
+            gpu_id.ver_major,
+            gpu_id.ver_minor,
+            gpu_id.ver_status
         );
 
         dev_info!(
@@ -166,14 +172,14 @@ unsafe impl AsBytes for GpuInfo {}
 
 struct GpuModels {
     name: &'static str,
-    major: u32,
-    minor: u32,
+    arch_major: u32,
+    prod_major: u32,
 }
 
 const GPU_MODELS: [GpuModels; 1] = [GpuModels {
     name: "g610",
-    major: 10,
-    minor: 7,
+    arch_major: 10,
+    prod_major: 7,
 }];
 
 #[allow(dead_code)]
diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs
index d3a541cb37c64..611870c2e6af5 100644
--- a/drivers/gpu/drm/tyr/regs.rs
+++ b/drivers/gpu/drm/tyr/regs.rs
@@ -7,12 +7,16 @@
 // does.
 #![allow(dead_code)]
 
-use kernel::bits::bit_u32;
-use kernel::device::Bound;
-use kernel::device::Device;
-use kernel::devres::Devres;
-use kernel::io::Io;
-use kernel::prelude::*;
+use kernel::{
+    bits::bit_u32,
+    device::{
+        Bound,
+        Device, //
+    },
+    devres::Devres,
+    io::Io,
+    prelude::*, //
+};
 
 use crate::driver::IoMem;
 
diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs
index 861d1db430726..9432ddd6b5b8b 100644
--- a/drivers/gpu/drm/tyr/tyr.rs
+++ b/drivers/gpu/drm/tyr/tyr.rs
@@ -5,7 +5,7 @@
 //! The name "Tyr" is inspired by Norse mythology, reflecting Arm's tradition of
 //! naming their GPUs after Nordic mythological figures and places.
 
-use crate::driver::TyrDriver;
+use crate::driver::TyrPlatformDriverData;
 
 mod driver;
 mod file;
@@ -14,7 +14,7 @@
 mod regs;
 
 kernel::module_platform_driver! {
-    type: TyrDriver,
+    type: TyrPlatformDriverData,
     name: "tyr",
     authors: ["The Tyr driver authors"],
     description: "Arm Mali Tyr DRM driver",
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index dd60acdf52c2b..4b441afcb602d 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -110,14 +110,12 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 		args->value = !!drm_gem_get_huge_mnt(dev);
 		return 0;
 	case DRM_V3D_PARAM_GLOBAL_RESET_COUNTER:
-		mutex_lock(&v3d->reset_lock);
-		args->value = v3d->reset_counter;
-		mutex_unlock(&v3d->reset_lock);
+		args->value = atomic_read(&v3d->reset_counter);
 		return 0;
 	case DRM_V3D_PARAM_CONTEXT_RESET_COUNTER:
-		mutex_lock(&v3d->reset_lock);
-		args->value = v3d_priv->reset_counter;
-		mutex_unlock(&v3d->reset_lock);
+		args->value = 0;
+		for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++)
+			args->value += atomic_read(&v3d_priv->stats[q]->reset_counter);
 		return 0;
 	default:
 		drm_dbg(dev, "Unknown parameter %d\n", args->param);
@@ -131,7 +129,7 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv;
 	struct drm_gpu_scheduler *sched;
-	int i;
+	int i, ret;
 
 	v3d_priv = kzalloc_obj(*v3d_priv);
 	if (!v3d_priv)
@@ -140,40 +138,45 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
-		sched = &v3d->queue[i].sched;
-		drm_sched_entity_init(&v3d_priv->sched_entity[i],
-				      DRM_SCHED_PRIORITY_NORMAL, &sched,
-				      1, NULL);
+		v3d_priv->stats[i] = v3d_stats_alloc();
+		if (!v3d_priv->stats[i]) {
+			ret = -ENOMEM;
+			goto err_stats;
+		}
 
-		memset(&v3d_priv->stats[i], 0, sizeof(v3d_priv->stats[i]));
-		seqcount_init(&v3d_priv->stats[i].lock);
+		sched = &v3d->queue[i].sched;
+		ret = drm_sched_entity_init(&v3d_priv->sched_entity[i],
+					    DRM_SCHED_PRIORITY_NORMAL, &sched,
+					    1, NULL);
+		if (ret)
+			goto err_sched;
 	}
 
 	v3d_perfmon_open_file(v3d_priv);
 	file->driver_priv = v3d_priv;
 
 	return 0;
+
+err_sched:
+	v3d_stats_put(v3d_priv->stats[i]);
+err_stats:
+	for (i--; i >= 0; i--) {
+		drm_sched_entity_destroy(&v3d_priv->sched_entity[i]);
+		v3d_stats_put(v3d_priv->stats[i]);
+	}
+	kfree(v3d_priv);
+	return ret;
 }
 
 static void
 v3d_postclose(struct drm_device *dev, struct drm_file *file)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv = file->driver_priv;
-	unsigned long irqflags;
 	enum v3d_queue q;
 
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		struct v3d_queue_state *queue = &v3d->queue[q];
-		struct v3d_job *job = queue->active_job;
-
 		drm_sched_entity_destroy(&v3d_priv->sched_entity[q]);
-
-		if (job && job->base.entity == &v3d_priv->sched_entity[q]) {
-			spin_lock_irqsave(&queue->queue_lock, irqflags);
-			job->file_priv = NULL;
-			spin_unlock_irqrestore(&queue->queue_lock, irqflags);
-		}
+		v3d_stats_put(v3d_priv->stats[q]);
 	}
 
 	v3d_perfmon_close_file(v3d_priv);
@@ -186,7 +189,7 @@ void v3d_get_stats(const struct v3d_stats *stats, u64 timestamp,
 	unsigned int seq;
 
 	do {
-		seq = read_seqcount_begin(&stats->lock);
+		seq = raw_read_seqcount_begin(&stats->lock);
 		*active_runtime = stats->enabled_ns;
 		if (stats->start_ns)
 			*active_runtime += timestamp - stats->start_ns;
@@ -201,7 +204,7 @@ static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
 	enum v3d_queue queue;
 
 	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
-		struct v3d_stats *stats = &file_priv->stats[queue];
+		struct v3d_stats *stats = file_priv->stats[queue];
 		u64 active_runtime, jobs_completed;
 
 		v3d_get_stats(stats, timestamp, &active_runtime, &jobs_completed);
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 99a39329bb85b..6a3cad9334398 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -38,6 +38,8 @@ static inline char *v3d_queue_to_string(enum v3d_queue queue)
 }
 
 struct v3d_stats {
+	struct kref refcount;
+
 	u64 start_ns;
 	u64 enabled_ns;
 	u64 jobs_completed;
@@ -46,8 +48,15 @@ struct v3d_stats {
 	 * This seqcount is used to protect the access to the GPU stats
 	 * variables. It must be used as, while we are reading the stats,
 	 * IRQs can happen and the stats can be updated.
+	 *
+	 * However, we use the raw seqcount helpers to interact with this lock
+	 * to avoid false positives from lockdep, which is unable to detect that
+	 * our readers are never from irq or softirq context, and that, for CPU
+	 * job queues, even the write side never is.
 	 */
 	seqcount_t lock;
+
+	atomic_t reset_counter;
 };
 
 struct v3d_queue_state {
@@ -57,13 +66,11 @@ struct v3d_queue_state {
 	u64 emit_seqno;
 
 	/* Stores the GPU stats for this queue in the global context. */
-	struct v3d_stats stats;
+	struct v3d_stats *stats;
 
 	/* Currently active job for this queue */
 	struct v3d_job *active_job;
 	spinlock_t queue_lock;
-	/* Protect dma fence for signalling job completion */
-	spinlock_t fence_lock;
 };
 
 /* Performance monitor object. The perform lifetime is controlled by userspace
@@ -196,10 +203,8 @@ struct v3d_dev {
 	 */
 	struct v3d_perfmon *global_perfmon;
 
-	/* Global reset counter. The counter must be incremented when
-	 * a GPU reset happens. It must be protected by @reset_lock.
-	 */
-	unsigned int reset_counter;
+	/* Global reset counter incremented on each GPU reset. */
+	atomic_t reset_counter;
 };
 
 static inline struct v3d_dev *
@@ -220,21 +225,12 @@ v3d_has_csd(struct v3d_dev *v3d)
 struct v3d_file_priv {
 	struct v3d_dev *v3d;
 
-	struct {
-		struct idr idr;
-		struct mutex lock;
-	} perfmon;
+	struct xarray perfmons;
 
 	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
 
 	/* Stores the GPU stats for a specific queue for this fd. */
-	struct v3d_stats stats[V3D_MAX_QUEUES];
-
-	/* Per-fd reset counter, must be incremented when a job submitted
-	 * by this fd causes a GPU reset. It must be protected by
-	 * &struct v3d_dev->reset_lock.
-	 */
-	unsigned int reset_counter;
+	struct v3d_stats *stats[V3D_MAX_QUEUES];
 };
 
 struct v3d_bo {
@@ -322,6 +318,10 @@ struct v3d_job {
 	 */
 	struct v3d_file_priv *file_priv;
 
+	/* Pointers to this job's per-fd and global queue stats. */
+	struct v3d_stats *client_stats;
+	struct v3d_stats *global_stats;
+
 	/* Callback for the freeing of the job on refcount going to 0. */
 	void (*free)(struct kref *ref);
 };
@@ -601,10 +601,23 @@ void v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,
 				   unsigned int count);
 void v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
 				     unsigned int count);
-void v3d_job_update_stats(struct v3d_job *job, enum v3d_queue q);
+struct v3d_stats *v3d_stats_alloc(void);
+void v3d_stats_release(struct kref *refcount);
+void v3d_job_update_stats(struct v3d_job *job);
 int v3d_sched_init(struct v3d_dev *v3d);
 void v3d_sched_fini(struct v3d_dev *v3d);
 
+static inline struct v3d_stats *v3d_stats_get(struct v3d_stats *stats)
+{
+	kref_get(&stats->refcount);
+	return stats;
+}
+
+static inline void v3d_stats_put(struct v3d_stats *stats)
+{
+	kref_put(&stats->refcount, v3d_stats_release);
+}
+
 /* v3d_perfmon.c */
 void v3d_perfmon_init(struct v3d_dev *v3d);
 void v3d_perfmon_get(struct v3d_perfmon *perfmon);
diff --git a/drivers/gpu/drm/v3d/v3d_fence.c b/drivers/gpu/drm/v3d/v3d_fence.c
index 3c999f60003c8..c500136d04557 100644
--- a/drivers/gpu/drm/v3d/v3d_fence.c
+++ b/drivers/gpu/drm/v3d/v3d_fence.c
@@ -15,7 +15,7 @@ struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue q)
 	fence->dev = &v3d->drm;
 	fence->queue = q;
 	fence->seqno = ++queue->emit_seqno;
-	dma_fence_init(&fence->base, &v3d_fence_ops, &queue->fence_lock,
+	dma_fence_init(&fence->base, &v3d_fence_ops, &queue->queue_lock,
 		       queue->fence_context, fence->seqno);
 
 	return &fence->base;
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 57965c0d6f6ef..75d9eccd79666 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -287,27 +287,30 @@ v3d_gem_init(struct drm_device *dev)
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
 		struct v3d_queue_state *queue = &v3d->queue[i];
 
+		queue->stats = v3d_stats_alloc();
+		if (!queue->stats) {
+			ret = -ENOMEM;
+			goto err_stats;
+		}
+
 		queue->fence_context = dma_fence_context_alloc(1);
-		memset(&queue->stats, 0, sizeof(queue->stats));
-		seqcount_init(&queue->stats.lock);
 
 		spin_lock_init(&queue->queue_lock);
-		spin_lock_init(&queue->fence_lock);
 	}
 
 	spin_lock_init(&v3d->mm_lock);
 	ret = drmm_mutex_init(dev, &v3d->bo_lock);
 	if (ret)
-		return ret;
+		goto err_stats;
 	ret = drmm_mutex_init(dev, &v3d->reset_lock);
 	if (ret)
-		return ret;
+		goto err_stats;
 	ret = drmm_mutex_init(dev, &v3d->sched_lock);
 	if (ret)
-		return ret;
+		goto err_stats;
 	ret = drmm_mutex_init(dev, &v3d->cache_clean_lock);
 	if (ret)
-		return ret;
+		goto err_stats;
 
 	/* Note: We don't allocate address 0.  Various bits of HW
 	 * treat 0 as special, such as the occlusion query counters
@@ -319,10 +322,10 @@ v3d_gem_init(struct drm_device *dev)
 			       &v3d->pt_paddr,
 			       GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
 	if (!v3d->pt) {
-		drm_mm_takedown(&v3d->mm);
 		dev_err(v3d->drm.dev,
 			"Failed to allocate page tables. Please ensure you have DMA enabled.\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto err_dma_alloc;
 	}
 
 	v3d_init_hw_state(v3d);
@@ -331,14 +334,20 @@ v3d_gem_init(struct drm_device *dev)
 	v3d_huge_mnt_init(v3d);
 
 	ret = v3d_sched_init(v3d);
-	if (ret) {
-		drm_mm_takedown(&v3d->mm);
-		dma_free_coherent(v3d->drm.dev, pt_size, (void *)v3d->pt,
-				  v3d->pt_paddr);
-		return ret;
-	}
+	if (ret)
+		goto err_sched;
 
 	return 0;
+
+err_sched:
+	dma_free_coherent(v3d->drm.dev, pt_size, (void *)v3d->pt, v3d->pt_paddr);
+err_dma_alloc:
+	drm_mm_takedown(&v3d->mm);
+err_stats:
+	for (i--; i >= 0; i--)
+		v3d_stats_put(v3d->queue[i].stats);
+
+	return ret;
 }
 
 void
@@ -352,8 +361,10 @@ v3d_gem_destroy(struct drm_device *dev)
 	/* Waiting for jobs to finish would need to be done before
 	 * unregistering V3D.
 	 */
-	for (q = 0; q < V3D_MAX_QUEUES; q++)
+	for (q = 0; q < V3D_MAX_QUEUES; q++) {
 		WARN_ON(v3d->queue[q].active_job);
+		v3d_stats_put(v3d->queue[q].stats);
+	}
 
 	drm_mm_takedown(&v3d->mm);
 
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index 1249f6e64b979..c28e74ab54428 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -92,7 +92,7 @@ v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q,
 	struct v3d_queue_state *queue = &v3d->queue[q];
 	struct v3d_fence *fence = to_v3d_fence(queue->active_job->irq_fence);
 
-	v3d_job_update_stats(queue->active_job, q);
+	v3d_job_update_stats(queue->active_job);
 	trace_irq(&v3d->drm, fence->seqno);
 
 	queue->active_job = NULL;
diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index e3d7474854eb4..8e0249580bbac 100644
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -6,9 +6,6 @@
 #include "v3d_drv.h"
 #include "v3d_regs.h"
 
-#define V3D_PERFMONID_MIN	1
-#define V3D_PERFMONID_MAX	U32_MAX
-
 static const struct v3d_perf_counter_desc v3d_v42_performance_counters[] = {
 	{"FEP", "FEP-valid-primitives-no-rendered-pixels", "[FEP] Valid primitives that result in no rendered pixels, for all rendered tiles"},
 	{"FEP", "FEP-valid-primitives-rendered-pixels", "[FEP] Valid primitives for all rendered tiles (primitives may be counted in more than one tile)"},
@@ -290,24 +287,23 @@ struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id)
 {
 	struct v3d_perfmon *perfmon;
 
-	mutex_lock(&v3d_priv->perfmon.lock);
-	perfmon = idr_find(&v3d_priv->perfmon.idr, id);
+	xa_lock(&v3d_priv->perfmons);
+	perfmon = xa_load(&v3d_priv->perfmons, id);
 	v3d_perfmon_get(perfmon);
-	mutex_unlock(&v3d_priv->perfmon.lock);
+	xa_unlock(&v3d_priv->perfmons);
 
 	return perfmon;
 }
 
 void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv)
 {
-	mutex_init(&v3d_priv->perfmon.lock);
-	idr_init_base(&v3d_priv->perfmon.idr, 1);
+	xa_init_flags(&v3d_priv->perfmons, XA_FLAGS_ALLOC1);
 }
 
-static int v3d_perfmon_idr_del(int id, void *elem, void *data)
+static void v3d_perfmon_delete(struct v3d_file_priv *v3d_priv,
+			       struct v3d_perfmon *perfmon)
 {
-	struct v3d_perfmon *perfmon = elem;
-	struct v3d_dev *v3d = (struct v3d_dev *)data;
+	struct v3d_dev *v3d = v3d_priv->v3d;
 
 	/* If the active perfmon is being destroyed, stop it first */
 	if (perfmon == v3d->active_perfmon)
@@ -317,19 +313,17 @@ static int v3d_perfmon_idr_del(int id, void *elem, void *data)
 	cmpxchg(&v3d->global_perfmon, perfmon, NULL);
 
 	v3d_perfmon_put(perfmon);
-
-	return 0;
 }
 
 void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv)
 {
-	struct v3d_dev *v3d = v3d_priv->v3d;
+	struct v3d_perfmon *perfmon;
+	unsigned long id;
+
+	xa_for_each(&v3d_priv->perfmons, id, perfmon)
+		v3d_perfmon_delete(v3d_priv, perfmon);
 
-	mutex_lock(&v3d_priv->perfmon.lock);
-	idr_for_each(&v3d_priv->perfmon.idr, v3d_perfmon_idr_del, v3d);
-	idr_destroy(&v3d_priv->perfmon.idr);
-	mutex_unlock(&v3d_priv->perfmon.lock);
-	mutex_destroy(&v3d_priv->perfmon.lock);
+	xa_destroy(&v3d_priv->perfmons);
 }
 
 int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
@@ -341,6 +335,7 @@ int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
 	struct v3d_perfmon *perfmon;
 	unsigned int i;
 	int ret;
+	u32 id;
 
 	/* Number of monitored counters cannot exceed HW limits. */
 	if (req->ncounters > DRM_V3D_MAX_PERF_COUNTERS ||
@@ -365,18 +360,15 @@ int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
 	refcount_set(&perfmon->refcnt, 1);
 	mutex_init(&perfmon->lock);
 
-	mutex_lock(&v3d_priv->perfmon.lock);
-	ret = idr_alloc(&v3d_priv->perfmon.idr, perfmon, V3D_PERFMONID_MIN,
-			V3D_PERFMONID_MAX, GFP_KERNEL);
-	mutex_unlock(&v3d_priv->perfmon.lock);
-
+	ret = xa_alloc(&v3d_priv->perfmons, &id, perfmon, xa_limit_32b,
+		       GFP_KERNEL);
 	if (ret < 0) {
 		mutex_destroy(&perfmon->lock);
 		kfree(perfmon);
 		return ret;
 	}
 
-	req->id = ret;
+	req->id = id;
 
 	return 0;
 }
@@ -386,24 +378,13 @@ int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 {
 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
 	struct drm_v3d_perfmon_destroy *req = data;
-	struct v3d_dev *v3d = v3d_priv->v3d;
 	struct v3d_perfmon *perfmon;
 
-	mutex_lock(&v3d_priv->perfmon.lock);
-	perfmon = idr_remove(&v3d_priv->perfmon.idr, req->id);
-	mutex_unlock(&v3d_priv->perfmon.lock);
-
+	perfmon = xa_erase(&v3d_priv->perfmons, req->id);
 	if (!perfmon)
 		return -EINVAL;
 
-	/* If the active perfmon is being destroyed, stop it first */
-	if (perfmon == v3d->active_perfmon)
-		v3d_perfmon_stop(v3d, perfmon, false);
-
-	/* If the global perfmon is being destroyed, set it to NULL */
-	cmpxchg(&v3d->global_perfmon, perfmon, NULL);
-
-	v3d_perfmon_put(perfmon);
+	v3d_perfmon_delete(v3d_priv, perfmon);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 6dc871fc9a623..1855ef5b3b5fe 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -66,6 +66,27 @@ to_cpu_job(struct drm_sched_job *sched_job)
 	return container_of(sched_job, struct v3d_cpu_job, base.base);
 }
 
+void v3d_stats_release(struct kref *refcount)
+{
+	struct v3d_stats *stats = container_of(refcount, typeof(*stats), refcount);
+
+	kfree(stats);
+}
+
+struct v3d_stats *v3d_stats_alloc(void)
+{
+	struct v3d_stats *stats;
+
+	stats = kzalloc_obj(*stats);
+	if (!stats)
+		return NULL;
+
+	kref_init(&stats->refcount);
+	seqcount_init(&stats->lock);
+
+	return stats;
+}
+
 static void
 v3d_sched_job_free(struct drm_sched_job *sched_job)
 {
@@ -137,90 +158,43 @@ v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
 }
 
 static void
-v3d_job_start_stats(struct v3d_job *job, enum v3d_queue queue)
+v3d_stats_start(struct v3d_stats *stats, u64 now)
 {
-	struct v3d_dev *v3d = job->v3d;
-	struct v3d_file_priv *file = job->file_priv;
-	struct v3d_stats *global_stats = &v3d->queue[queue].stats;
-	struct v3d_stats *local_stats = &file->stats[queue];
-	u64 now = local_clock();
-	unsigned long flags;
-
-	/*
-	 * We only need to disable local interrupts to appease lockdep who
-	 * otherwise would think v3d_job_start_stats vs v3d_stats_update has an
-	 * unsafe in-irq vs no-irq-off usage problem. This is a false positive
-	 * because all the locks are per queue and stats type, and all jobs are
-	 * completely one at a time serialised. More specifically:
-	 *
-	 * 1. Locks for GPU queues are updated from interrupt handlers under a
-	 *    spin lock and started here with preemption disabled.
-	 *
-	 * 2. Locks for CPU queues are updated from the worker with preemption
-	 *    disabled and equally started here with preemption disabled.
-	 *
-	 * Therefore both are consistent.
-	 *
-	 * 3. Because next job can only be queued after the previous one has
-	 *    been signaled, and locks are per queue, there is also no scope for
-	 *    the start part to race with the update part.
-	 */
-	if (IS_ENABLED(CONFIG_LOCKDEP))
-		local_irq_save(flags);
-	else
-		preempt_disable();
-
-	write_seqcount_begin(&local_stats->lock);
-	local_stats->start_ns = now;
-	write_seqcount_end(&local_stats->lock);
+	raw_write_seqcount_begin(&stats->lock);
+	stats->start_ns = now;
+	raw_write_seqcount_end(&stats->lock);
+}
 
-	write_seqcount_begin(&global_stats->lock);
-	global_stats->start_ns = now;
-	write_seqcount_end(&global_stats->lock);
+static void
+v3d_job_start_stats(struct v3d_job *job)
+{
+	u64 now = local_clock();
 
-	if (IS_ENABLED(CONFIG_LOCKDEP))
-		local_irq_restore(flags);
-	else
-		preempt_enable();
+	preempt_disable();
+	v3d_stats_start(job->client_stats, now);
+	v3d_stats_start(job->global_stats, now);
+	preempt_enable();
 }
 
 static void
 v3d_stats_update(struct v3d_stats *stats, u64 now)
 {
-	write_seqcount_begin(&stats->lock);
+	raw_write_seqcount_begin(&stats->lock);
 	stats->enabled_ns += now - stats->start_ns;
 	stats->jobs_completed++;
 	stats->start_ns = 0;
-	write_seqcount_end(&stats->lock);
+	raw_write_seqcount_end(&stats->lock);
 }
 
 void
-v3d_job_update_stats(struct v3d_job *job, enum v3d_queue q)
+v3d_job_update_stats(struct v3d_job *job)
 {
-	struct v3d_dev *v3d = job->v3d;
-	struct v3d_queue_state *queue = &v3d->queue[q];
-	struct v3d_stats *global_stats = &queue->stats;
 	u64 now = local_clock();
-	unsigned long flags;
-
-	/* See comment in v3d_job_start_stats() */
-	if (IS_ENABLED(CONFIG_LOCKDEP))
-		local_irq_save(flags);
-	else
-		preempt_disable();
 
-	/* Don't update the local stats if the file context has already closed */
-	spin_lock(&queue->queue_lock);
-	if (job->file_priv)
-		v3d_stats_update(&job->file_priv->stats[q], now);
-	spin_unlock(&queue->queue_lock);
-
-	v3d_stats_update(global_stats, now);
-
-	if (IS_ENABLED(CONFIG_LOCKDEP))
-		local_irq_restore(flags);
-	else
-		preempt_enable();
+	preempt_disable();
+	v3d_stats_update(job->client_stats, now);
+	v3d_stats_update(job->global_stats, now);
+	preempt_enable();
 }
 
 static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
@@ -263,7 +237,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
-	v3d_job_start_stats(&job->base, V3D_BIN);
+	v3d_job_start_stats(&job->base);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* Set the current and end address of the control list.
@@ -317,7 +291,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
-	v3d_job_start_stats(&job->base, V3D_RENDER);
+	v3d_job_start_stats(&job->base);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* XXX: Set the QCFG */
@@ -356,7 +330,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
-	v3d_job_start_stats(&job->base, V3D_TFU);
+	v3d_job_start_stats(&job->base);
 
 	V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);
 	V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);
@@ -406,7 +380,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
-	v3d_job_start_stats(&job->base, V3D_CSD);
+	v3d_job_start_stats(&job->base);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
@@ -694,13 +668,13 @@ v3d_cpu_job_run(struct drm_sched_job *sched_job)
 		return NULL;
 	}
 
-	v3d_job_start_stats(&job->base, V3D_CPU);
+	v3d_job_start_stats(&job->base);
 	trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);
 
 	cpu_job_function[job->job_type](job);
 
 	trace_v3d_cpu_job_end(&v3d->drm, job->job_type);
-	v3d_job_update_stats(&job->base, V3D_CPU);
+	v3d_job_update_stats(&job->base);
 
 	/* Synchronous operation, so no fence to wait on. */
 	return NULL;
@@ -712,11 +686,11 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
 
-	v3d_job_start_stats(job, V3D_CACHE_CLEAN);
+	v3d_job_start_stats(job);
 
 	v3d_clean_caches(v3d);
 
-	v3d_job_update_stats(job, V3D_CACHE_CLEAN);
+	v3d_job_update_stats(job);
 
 	/* Synchronous operation, so no fence to wait on. */
 	return NULL;
@@ -727,8 +701,6 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job,
 			  enum v3d_queue q)
 {
 	struct v3d_job *job = to_v3d_job(sched_job);
-	struct v3d_file_priv *v3d_priv = job->file_priv;
-	unsigned long irqflags;
 	enum v3d_queue i;
 
 	mutex_lock(&v3d->reset_lock);
@@ -743,11 +715,8 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job,
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
-	v3d->reset_counter++;
-	spin_lock_irqsave(&v3d->queue[q].queue_lock, irqflags);
-	if (v3d_priv)
-		v3d_priv->reset_counter++;
-	spin_unlock_irqrestore(&v3d->queue[q].queue_lock, irqflags);
+	atomic_inc(&v3d->reset_counter);
+	atomic_inc(&job->client_stats->reset_counter);
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++)
 		drm_sched_resubmit_jobs(&v3d->queue[i].sched);
diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index 18f2bf1fe89fa..8f061b6a05c6a 100644
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -103,6 +103,9 @@ v3d_job_free(struct kref *ref)
 	if (job->perfmon)
 		v3d_perfmon_put(job->perfmon);
 
+	v3d_stats_put(job->client_stats);
+	v3d_stats_put(job->global_stats);
+
 	kfree(job);
 }
 
@@ -203,6 +206,9 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 
 	kref_init(&job->refcount);
 
+	job->client_stats = v3d_stats_get(v3d_priv->stats[queue]);
+	job->global_stats = v3d_stats_get(v3d->queue[queue].stats);
+
 	return 0;
 
 fail_deps:
diff --git a/drivers/gpu/drm/v3d/v3d_sysfs.c b/drivers/gpu/drm/v3d/v3d_sysfs.c
index d610e355964ff..b45a9b3db42d6 100644
--- a/drivers/gpu/drm/v3d/v3d_sysfs.c
+++ b/drivers/gpu/drm/v3d/v3d_sysfs.c
@@ -20,7 +20,7 @@ gpu_stats_show(struct device *dev, struct device_attribute *attr, char *buf)
 	len += sysfs_emit(buf, "queue\ttimestamp\tjobs\truntime\n");
 
 	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
-		struct v3d_stats *stats = &v3d->queue[queue].stats;
+		struct v3d_stats *stats = v3d->queue[queue].stats;
 		u64 active_runtime, jobs_completed;
 
 		v3d_get_stats(stats, timestamp, &active_runtime, &jobs_completed);
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 1f93bc5a3d02e..f45ba47b4ba86 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -556,7 +556,7 @@ static void vc4_free_object(struct drm_gem_object *gem_bo)
 	mutex_lock(&vc4->bo_lock);
 	/* If the object references someone else's memory, we can't cache it.
 	 */
-	if (gem_bo->import_attach) {
+	if (drm_gem_is_imported(gem_bo)) {
 		vc4_bo_destroy(bo);
 		goto out;
 	}
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 221d8e01d5391..dbcc83b7df004 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -791,10 +791,7 @@ struct vc4_exec_info {
 struct vc4_file {
 	struct vc4_dev *dev;
 
-	struct {
-		struct idr idr;
-		struct mutex lock;
-	} perfmon;
+	struct xarray perfmons;
 
 	bool bin_bo_used;
 };
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index ad8cbd727b801..2d3df5e621c1a 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -1250,7 +1250,7 @@ int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	/* Not sure it's safe to purge imported BOs. Let's just assume it's
 	 * not until proven otherwise.
 	 */
-	if (gem_obj->import_attach) {
+	if (drm_gem_is_imported(gem_obj)) {
 		DRM_DEBUG("madvise not supported on imported BOs\n");
 		ret = -EINVAL;
 		goto out_put_gem;
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index fda214b5a466e..9898e5451a07a 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -823,7 +823,7 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 
 	vc4_hdmi->scdc_enabled = true;
 
-	queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work,
+	queue_delayed_work(system_percpu_wq, &vc4_hdmi->scrambling_work,
 			   msecs_to_jiffies(SCRAMBLING_POLLING_DELAY_MS));
 }
 
@@ -872,7 +872,7 @@ static void vc4_hdmi_scrambling_wq(struct work_struct *work)
 	drm_scdc_set_high_tmds_clock_ratio(connector, true);
 	drm_scdc_set_scrambling(connector, true);
 
-	queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work,
+	queue_delayed_work(system_percpu_wq, &vc4_hdmi->scrambling_work,
 			   msecs_to_jiffies(SCRAMBLING_POLLING_DELAY_MS));
 }
 
@@ -2449,17 +2449,23 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi)
 	int ret;
 
 	if (vc4_hdmi->variant->external_irq_controller) {
-		unsigned int hpd_con = platform_get_irq_byname(pdev, "hpd-connected");
-		unsigned int hpd_rm = platform_get_irq_byname(pdev, "hpd-removed");
+		int hpd = platform_get_irq_byname(pdev, "hpd-connected");
 
-		ret = devm_request_threaded_irq(&pdev->dev, hpd_con,
+		if (hpd < 0)
+			return hpd;
+
+		ret = devm_request_threaded_irq(&pdev->dev, hpd,
 						NULL,
 						vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
 						"vc4 hdmi hpd connected", vc4_hdmi);
 		if (ret)
 			return ret;
 
-		ret = devm_request_threaded_irq(&pdev->dev, hpd_rm,
+		hpd = platform_get_irq_byname(pdev, "hpd-removed");
+		if (hpd < 0)
+			return hpd;
+
+		ret = devm_request_threaded_irq(&pdev->dev, hpd,
 						NULL,
 						vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
 						"vc4 hdmi hpd disconnected", vc4_hdmi);
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 245485e744a50..264b5e80c24d6 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -85,7 +85,22 @@ static void vc4_ctm_destroy_state(struct drm_private_obj *obj,
 	kfree(ctm_state);
 }
 
+static struct drm_private_state *
+vc4_ctm_create_state(struct drm_private_obj *obj)
+{
+	struct vc4_ctm_state *ctm_state;
+
+	ctm_state = kzalloc_obj(*ctm_state);
+	if (!ctm_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &ctm_state->base);
+
+	return &ctm_state->base;
+}
+
 static const struct drm_private_state_funcs vc4_ctm_state_funcs = {
+	.atomic_create_state = vc4_ctm_create_state,
 	.atomic_duplicate_state = vc4_ctm_duplicate_state,
 	.atomic_destroy_state = vc4_ctm_destroy_state,
 };
@@ -99,15 +114,9 @@ static void vc4_ctm_obj_fini(struct drm_device *dev, void *unused)
 
 static int vc4_ctm_obj_init(struct vc4_dev *vc4)
 {
-	struct vc4_ctm_state *ctm_state;
-
 	drm_modeset_lock_init(&vc4->ctm_state_lock);
 
-	ctm_state = kzalloc_obj(*ctm_state);
-	if (!ctm_state)
-		return -ENOMEM;
-
-	drm_atomic_private_obj_init(&vc4->base, &vc4->ctm_manager, &ctm_state->base,
+	drm_atomic_private_obj_init(&vc4->base, &vc4->ctm_manager,
 				    &vc4_ctm_state_funcs);
 
 	return drmm_add_action_or_reset(&vc4->base, vc4_ctm_obj_fini, NULL);
@@ -718,7 +727,22 @@ static void vc4_load_tracker_destroy_state(struct drm_private_obj *obj,
 	kfree(load_state);
 }
 
+static struct drm_private_state *
+vc4_load_tracker_create_state(struct drm_private_obj *obj)
+{
+	struct vc4_load_tracker_state *load_state;
+
+	load_state = kzalloc_obj(*load_state);
+	if (!load_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &load_state->base);
+
+	return &load_state->base;
+}
+
 static const struct drm_private_state_funcs vc4_load_tracker_state_funcs = {
+	.atomic_create_state = vc4_load_tracker_create_state,
 	.atomic_duplicate_state = vc4_load_tracker_duplicate_state,
 	.atomic_destroy_state = vc4_load_tracker_destroy_state,
 };
@@ -732,14 +756,7 @@ static void vc4_load_tracker_obj_fini(struct drm_device *dev, void *unused)
 
 static int vc4_load_tracker_obj_init(struct vc4_dev *vc4)
 {
-	struct vc4_load_tracker_state *load_state;
-
-	load_state = kzalloc_obj(*load_state);
-	if (!load_state)
-		return -ENOMEM;
-
 	drm_atomic_private_obj_init(&vc4->base, &vc4->load_tracker,
-				    &load_state->base,
 				    &vc4_load_tracker_state_funcs);
 
 	return drmm_add_action_or_reset(&vc4->base, vc4_load_tracker_obj_fini, NULL);
@@ -800,7 +817,22 @@ static void vc4_hvs_channels_print_state(struct drm_printer *p,
 	}
 }
 
+static struct drm_private_state *
+vc4_hvs_channels_create_state(struct drm_private_obj *obj)
+{
+	struct vc4_hvs_state *hvs_state;
+
+	hvs_state = kzalloc_obj(*hvs_state);
+	if (!hvs_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_private_obj_create_state(obj, &hvs_state->base);
+
+	return &hvs_state->base;
+}
+
 static const struct drm_private_state_funcs vc4_hvs_state_funcs = {
+	.atomic_create_state = vc4_hvs_channels_create_state,
 	.atomic_duplicate_state = vc4_hvs_channels_duplicate_state,
 	.atomic_destroy_state = vc4_hvs_channels_destroy_state,
 	.atomic_print_state = vc4_hvs_channels_print_state,
@@ -815,14 +847,7 @@ static void vc4_hvs_channels_obj_fini(struct drm_device *dev, void *unused)
 
 static int vc4_hvs_channels_obj_init(struct vc4_dev *vc4)
 {
-	struct vc4_hvs_state *state;
-
-	state = kzalloc_obj(*state);
-	if (!state)
-		return -ENOMEM;
-
 	drm_atomic_private_obj_init(&vc4->base, &vc4->hvs_channels,
-				    &state->base,
 				    &vc4_hvs_state_funcs);
 
 	return drmm_add_action_or_reset(&vc4->base, vc4_hvs_channels_obj_fini, NULL);
diff --git a/drivers/gpu/drm/vc4/vc4_perfmon.c b/drivers/gpu/drm/vc4/vc4_perfmon.c
index eb5e1c936a011..f75dfd1567565 100644
--- a/drivers/gpu/drm/vc4/vc4_perfmon.c
+++ b/drivers/gpu/drm/vc4/vc4_perfmon.c
@@ -14,9 +14,6 @@
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-#define VC4_PERFMONID_MIN	1
-#define VC4_PERFMONID_MAX	U32_MAX
-
 void vc4_perfmon_get(struct vc4_perfmon *perfmon)
 {
 	struct vc4_dev *vc4;
@@ -95,10 +92,10 @@ struct vc4_perfmon *vc4_perfmon_find(struct vc4_file *vc4file, int id)
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return NULL;
 
-	mutex_lock(&vc4file->perfmon.lock);
-	perfmon = idr_find(&vc4file->perfmon.idr, id);
+	xa_lock(&vc4file->perfmons);
+	perfmon = xa_load(&vc4file->perfmons, id);
 	vc4_perfmon_get(perfmon);
-	mutex_unlock(&vc4file->perfmon.lock);
+	xa_unlock(&vc4file->perfmons);
 
 	return perfmon;
 }
@@ -110,37 +107,34 @@ void vc4_perfmon_open_file(struct vc4_file *vc4file)
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return;
 
-	mutex_init(&vc4file->perfmon.lock);
-	idr_init_base(&vc4file->perfmon.idr, VC4_PERFMONID_MIN);
-	vc4file->dev = vc4;
+	xa_init_flags(&vc4file->perfmons, XA_FLAGS_ALLOC1);
 }
 
-static int vc4_perfmon_idr_del(int id, void *elem, void *data)
+static void vc4_perfmon_delete(struct vc4_file *vc4file,
+			       struct vc4_perfmon *perfmon)
 {
-	struct vc4_perfmon *perfmon = elem;
-	struct vc4_dev *vc4 = (struct vc4_dev *)data;
+	struct vc4_dev *vc4 = vc4file->dev;
 
 	/* If the active perfmon is being destroyed, stop it first */
 	if (perfmon == vc4->active_perfmon)
 		vc4_perfmon_stop(vc4, perfmon, false);
 
 	vc4_perfmon_put(perfmon);
-
-	return 0;
 }
 
 void vc4_perfmon_close_file(struct vc4_file *vc4file)
 {
 	struct vc4_dev *vc4 = vc4file->dev;
+	struct vc4_perfmon *perfmon;
+	unsigned long id;
 
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return;
 
-	mutex_lock(&vc4file->perfmon.lock);
-	idr_for_each(&vc4file->perfmon.idr, vc4_perfmon_idr_del, vc4);
-	idr_destroy(&vc4file->perfmon.idr);
-	mutex_unlock(&vc4file->perfmon.lock);
-	mutex_destroy(&vc4file->perfmon.lock);
+	xa_for_each(&vc4file->perfmons, id, perfmon)
+		vc4_perfmon_delete(vc4file, perfmon);
+
+	xa_destroy(&vc4file->perfmons);
 }
 
 int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
@@ -152,6 +146,7 @@ int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
 	struct vc4_perfmon *perfmon;
 	unsigned int i;
 	int ret;
+	u32 id;
 
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return -ENODEV;
@@ -184,17 +179,15 @@ int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
 
 	refcount_set(&perfmon->refcnt, 1);
 
-	mutex_lock(&vc4file->perfmon.lock);
-	ret = idr_alloc(&vc4file->perfmon.idr, perfmon, VC4_PERFMONID_MIN,
-			VC4_PERFMONID_MAX, GFP_KERNEL);
-	mutex_unlock(&vc4file->perfmon.lock);
-
+	ret = xa_alloc(&vc4file->perfmons, &id, perfmon, xa_limit_32b,
+		       GFP_KERNEL);
 	if (ret < 0) {
 		kfree(perfmon);
 		return ret;
 	}
 
-	req->id = ret;
+	req->id = id;
+
 	return 0;
 }
 
@@ -214,14 +207,12 @@ int vc4_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 		return -ENODEV;
 	}
 
-	mutex_lock(&vc4file->perfmon.lock);
-	perfmon = idr_remove(&vc4file->perfmon.idr, req->id);
-	mutex_unlock(&vc4file->perfmon.lock);
-
+	perfmon = xa_erase(&vc4file->perfmons, req->id);
 	if (!perfmon)
 		return -EINVAL;
 
-	vc4_perfmon_put(perfmon);
+	vc4_perfmon_delete(vc4file, perfmon);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/verisilicon/Kconfig b/drivers/gpu/drm/verisilicon/Kconfig
new file mode 100644
index 0000000000000..7cce86ec8603c
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/Kconfig
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config DRM_VERISILICON_DC
+	tristate "DRM Support for Verisilicon DC-series display controllers"
+	depends on DRM && COMMON_CLK
+	depends on RISCV || COMPILE_TEST
+	select DRM_BRIDGE_CONNECTOR
+	select DRM_CLIENT_SELECTION
+	select DRM_DISPLAY_HELPER
+	select DRM_GEM_DMA_HELPER
+	select DRM_KMS_HELPER
+	select REGMAP_MMIO
+	select VIDEOMODE_HELPERS
+	help
+	  Choose this option if you have a SoC with Verisilicon DC-series
+	  display controllers. If M is selected, the module will be called
+	  verisilicon-dc.
diff --git a/drivers/gpu/drm/verisilicon/Makefile b/drivers/gpu/drm/verisilicon/Makefile
new file mode 100644
index 0000000000000..fd8d805fbcde1
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+verisilicon-dc-objs := vs_bridge.o vs_crtc.o vs_dc.o vs_drm.o vs_hwdb.o vs_plane.o vs_primary_plane.o
+
+obj-$(CONFIG_DRM_VERISILICON_DC) += verisilicon-dc.o
diff --git a/drivers/gpu/drm/verisilicon/vs_bridge.c b/drivers/gpu/drm/verisilicon/vs_bridge.c
new file mode 100644
index 0000000000000..2a0ad00a94d6d
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_bridge.c
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+#include <uapi/linux/media-bus-format.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_bridge_connector.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_of.h>
+#include <drm/drm_print.h>
+#include <drm/drm_simple_kms_helper.h>
+
+#include "vs_bridge.h"
+#include "vs_bridge_regs.h"
+#include "vs_crtc.h"
+#include "vs_dc.h"
+
+static int vs_bridge_attach(struct drm_bridge *bridge,
+			    struct drm_encoder *encoder,
+			    enum drm_bridge_attach_flags flags)
+{
+	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
+
+	return drm_bridge_attach(encoder, vbridge->next_bridge,
+				 bridge, flags);
+}
+
+struct vsdc_dp_format {
+	u32 linux_fmt;
+	bool is_yuv;
+	u32 vsdc_fmt;
+};
+
+static struct vsdc_dp_format vsdc_dp_supported_fmts[] = {
+	/* default to RGB888 */
+	{ MEDIA_BUS_FMT_FIXED, false, VSDC_DISP_DP_CONFIG_FMT_RGB888 },
+	{ MEDIA_BUS_FMT_RGB888_1X24, false, VSDC_DISP_DP_CONFIG_FMT_RGB888 },
+	{ MEDIA_BUS_FMT_RGB565_1X16, false, VSDC_DISP_DP_CONFIG_FMT_RGB565 },
+	{ MEDIA_BUS_FMT_RGB666_1X18, false, VSDC_DISP_DP_CONFIG_FMT_RGB666 },
+	{ MEDIA_BUS_FMT_RGB101010_1X30,
+	  false, VSDC_DISP_DP_CONFIG_FMT_RGB101010 },
+	{ MEDIA_BUS_FMT_UYVY8_1X16, true, VSDC_DISP_DP_CONFIG_YUV_FMT_UYVY8 },
+	{ MEDIA_BUS_FMT_UYVY10_1X20, true, VSDC_DISP_DP_CONFIG_YUV_FMT_UYVY10 },
+	{ MEDIA_BUS_FMT_YUV8_1X24, true, VSDC_DISP_DP_CONFIG_YUV_FMT_YUV8 },
+	{ MEDIA_BUS_FMT_YUV10_1X30, true, VSDC_DISP_DP_CONFIG_YUV_FMT_YUV10 },
+	{ MEDIA_BUS_FMT_UYYVYY8_0_5X24,
+	  true, VSDC_DISP_DP_CONFIG_YUV_FMT_UYYVYY8 },
+	{ MEDIA_BUS_FMT_UYYVYY10_0_5X30,
+	  true, VSDC_DISP_DP_CONFIG_YUV_FMT_UYYVYY10 },
+};
+
+static u32 *vs_bridge_atomic_get_output_bus_fmts_dpi(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts)
+{
+	u32 *output_fmts;
+
+	*num_output_fmts = 2;
+
+	output_fmts = kcalloc(*num_output_fmts, sizeof(*output_fmts),
+			      GFP_KERNEL);
+	if (!output_fmts)
+		return NULL;
+
+	/* TODO: support more DPI output formats */
+	output_fmts[0] = MEDIA_BUS_FMT_RGB888_1X24;
+	output_fmts[1] = MEDIA_BUS_FMT_FIXED;
+
+	return output_fmts;
+}
+
+static u32 *vs_bridge_atomic_get_output_bus_fmts_dp(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts)
+{
+	u32 *output_fmts;
+	unsigned int i;
+
+	*num_output_fmts = ARRAY_SIZE(vsdc_dp_supported_fmts);
+
+	output_fmts = kcalloc(*num_output_fmts, sizeof(*output_fmts),
+			      GFP_KERNEL);
+	if (!output_fmts)
+		return NULL;
+
+	for (i = 0; i < *num_output_fmts; i++)
+		output_fmts[i] = vsdc_dp_supported_fmts[i].linux_fmt;
+
+	return output_fmts;
+}
+
+static bool vs_bridge_out_dp_fmt_supported(u32 out_fmt)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vsdc_dp_supported_fmts); i++)
+		if (vsdc_dp_supported_fmts[i].linux_fmt == out_fmt)
+			return true;
+
+	return false;
+}
+
+static u32 *vs_bridge_atomic_get_input_bus_fmts_dp(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					u32 output_fmt,
+					unsigned int *num_input_fmts)
+{
+	if (!vs_bridge_out_dp_fmt_supported(output_fmt)) {
+		*num_input_fmts = 0;
+		return NULL;
+	}
+
+	return drm_atomic_helper_bridge_propagate_bus_fmt(bridge, bridge_state,
+							  crtc_state,
+							  conn_state,
+							  output_fmt,
+							  num_input_fmts);
+}
+
+static int vs_bridge_atomic_check_dp(struct drm_bridge *bridge,
+				     struct drm_bridge_state *bridge_state,
+				     struct drm_crtc_state *crtc_state,
+				     struct drm_connector_state *conn_state)
+{
+	if (!vs_bridge_out_dp_fmt_supported(bridge_state->output_bus_cfg.format))
+		return -EINVAL;
+
+	return 0;
+}
+
+static void vs_bridge_enable_common(struct vs_crtc *crtc,
+				    struct drm_bridge_state *br_state)
+{
+	struct vs_dc *dc = crtc->dc;
+	unsigned int output = crtc->id;
+
+	regmap_clear_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			  VSDC_DISP_PANEL_CONFIG_DAT_POL);
+	regmap_assign_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			   VSDC_DISP_PANEL_CONFIG_DE_POL,
+			   br_state->output_bus_cfg.flags &
+			   DRM_BUS_FLAG_DE_LOW);
+	regmap_assign_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			   VSDC_DISP_PANEL_CONFIG_CLK_POL,
+			   br_state->output_bus_cfg.flags &
+			   DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE);
+	regmap_set_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			VSDC_DISP_PANEL_CONFIG_DE_EN |
+			VSDC_DISP_PANEL_CONFIG_DAT_EN |
+			VSDC_DISP_PANEL_CONFIG_CLK_EN);
+	regmap_set_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			VSDC_DISP_PANEL_CONFIG_RUNNING);
+	regmap_clear_bits(dc->regs, VSDC_DISP_PANEL_START,
+			  VSDC_DISP_PANEL_START_MULTI_DISP_SYNC);
+	regmap_set_bits(dc->regs, VSDC_DISP_PANEL_START,
+			VSDC_DISP_PANEL_START_RUNNING(output));
+
+	regmap_set_bits(dc->regs, VSDC_DISP_PANEL_CONFIG_EX(crtc->id),
+			VSDC_DISP_PANEL_CONFIG_EX_COMMIT);
+}
+
+static void vs_bridge_atomic_enable_dpi(struct drm_bridge *bridge,
+					struct drm_atomic_state *state)
+{
+	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
+	struct drm_bridge_state *br_state =
+		drm_atomic_get_new_bridge_state(state, bridge);
+	struct vs_crtc *crtc = vbridge->crtc;
+	struct vs_dc *dc = crtc->dc;
+	unsigned int output = crtc->id;
+
+	regmap_clear_bits(dc->regs, VSDC_DISP_DP_CONFIG(output),
+			  VSDC_DISP_DP_CONFIG_DP_EN);
+	regmap_write(dc->regs, VSDC_DISP_DPI_CONFIG(output),
+		     VSDC_DISP_DPI_CONFIG_FMT_RGB888);
+
+	vs_bridge_enable_common(crtc, br_state);
+}
+
+static void vs_bridge_atomic_enable_dp(struct drm_bridge *bridge,
+					struct drm_atomic_state *state)
+{
+	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
+	struct drm_bridge_state *br_state =
+		drm_atomic_get_new_bridge_state(state, bridge);
+	struct vs_crtc *crtc = vbridge->crtc;
+	struct vs_dc *dc = crtc->dc;
+	unsigned int output = crtc->id;
+	u32 dp_fmt;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vsdc_dp_supported_fmts); i++) {
+		if (vsdc_dp_supported_fmts[i].linux_fmt ==
+		    br_state->output_bus_cfg.format)
+			break;
+	}
+	if (WARN_ON_ONCE(i == ARRAY_SIZE(vsdc_dp_supported_fmts)))
+		return;
+	dp_fmt = vsdc_dp_supported_fmts[i].vsdc_fmt;
+	dp_fmt |= VSDC_DISP_DP_CONFIG_DP_EN;
+	regmap_write(dc->regs, VSDC_DISP_DP_CONFIG(output), dp_fmt);
+	regmap_assign_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			   VSDC_DISP_PANEL_CONFIG_YUV,
+			   vsdc_dp_supported_fmts[i].is_yuv);
+
+	vs_bridge_enable_common(crtc, br_state);
+}
+
+static void vs_bridge_atomic_disable(struct drm_bridge *bridge,
+				     struct drm_atomic_state *state)
+{
+	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
+	struct vs_crtc *crtc = vbridge->crtc;
+	struct vs_dc *dc = crtc->dc;
+	unsigned int output = crtc->id;
+
+	regmap_clear_bits(dc->regs, VSDC_DISP_PANEL_START,
+			  VSDC_DISP_PANEL_START_MULTI_DISP_SYNC |
+			  VSDC_DISP_PANEL_START_RUNNING(output));
+	regmap_clear_bits(dc->regs, VSDC_DISP_PANEL_CONFIG(output),
+			  VSDC_DISP_PANEL_CONFIG_RUNNING);
+
+	regmap_set_bits(dc->regs, VSDC_DISP_PANEL_CONFIG_EX(crtc->id),
+			VSDC_DISP_PANEL_CONFIG_EX_COMMIT);
+}
+
+static const struct drm_bridge_funcs vs_dpi_bridge_funcs = {
+	.attach = vs_bridge_attach,
+	.atomic_enable = vs_bridge_atomic_enable_dpi,
+	.atomic_disable = vs_bridge_atomic_disable,
+	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
+	.atomic_get_output_bus_fmts = vs_bridge_atomic_get_output_bus_fmts_dpi,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+};
+
+static const struct drm_bridge_funcs vs_dp_bridge_funcs = {
+	.attach = vs_bridge_attach,
+	.atomic_enable = vs_bridge_atomic_enable_dp,
+	.atomic_disable = vs_bridge_atomic_disable,
+	.atomic_check = vs_bridge_atomic_check_dp,
+	.atomic_get_input_bus_fmts = vs_bridge_atomic_get_input_bus_fmts_dp,
+	.atomic_get_output_bus_fmts = vs_bridge_atomic_get_output_bus_fmts_dp,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+};
+
+static int vs_bridge_detect_output_interface(struct device_node *of_node,
+					     unsigned int output)
+{
+	int ret;
+	struct device_node *remote;
+
+	remote = of_graph_get_remote_node(of_node, output,
+					  VSDC_OUTPUT_INTERFACE_DPI);
+	if (remote) {
+		ret = VSDC_OUTPUT_INTERFACE_DPI;
+	} else {
+		remote = of_graph_get_remote_node(of_node, output,
+						  VSDC_OUTPUT_INTERFACE_DP);
+		if (remote)
+			ret = VSDC_OUTPUT_INTERFACE_DP;
+		else
+			ret = -ENODEV;
+	}
+
+	if (remote)
+		of_node_put(remote);
+
+	return ret;
+}
+
+struct vs_bridge *vs_bridge_init(struct drm_device *drm_dev,
+				 struct vs_crtc *crtc)
+{
+	unsigned int output = crtc->id;
+	struct vs_bridge *bridge;
+	struct drm_bridge *next;
+	enum vs_bridge_output_interface intf;
+	const struct drm_bridge_funcs *bridge_funcs;
+	int ret, enctype;
+
+	intf = vs_bridge_detect_output_interface(drm_dev->dev->of_node,
+						 output);
+	if (intf == -ENODEV) {
+		drm_dbg(drm_dev, "Skipping output %u\n", output);
+		return NULL;
+	}
+
+	next = devm_drm_of_get_bridge(drm_dev->dev, drm_dev->dev->of_node,
+				      output, intf);
+	if (IS_ERR(next)) {
+		ret = PTR_ERR(next);
+		if (ret != -EPROBE_DEFER)
+			drm_err(drm_dev,
+				"Cannot get downstream bridge of output %u\n",
+				output);
+		return ERR_PTR(ret);
+	}
+
+	if (intf == VSDC_OUTPUT_INTERFACE_DPI)
+		bridge_funcs = &vs_dpi_bridge_funcs;
+	else
+		bridge_funcs = &vs_dp_bridge_funcs;
+
+	bridge = devm_drm_bridge_alloc(drm_dev->dev, struct vs_bridge, base,
+				       bridge_funcs);
+	if (IS_ERR(bridge))
+		return ERR_PTR(PTR_ERR(bridge));
+
+	bridge->crtc = crtc;
+	bridge->intf = intf;
+	bridge->next_bridge = next;
+
+	if (intf == VSDC_OUTPUT_INTERFACE_DPI)
+		enctype = DRM_MODE_ENCODER_DPI;
+	else
+		enctype = DRM_MODE_ENCODER_NONE;
+
+	bridge->enc = drmm_plain_encoder_alloc(drm_dev, NULL, enctype, NULL);
+	if (IS_ERR(bridge->enc)) {
+		drm_err(drm_dev,
+			"Cannot initialize encoder for output %u\n", output);
+		ret = PTR_ERR(bridge->enc);
+		return ERR_PTR(ret);
+	}
+
+	bridge->enc->possible_crtcs = drm_crtc_mask(&crtc->base);
+
+	ret = devm_drm_bridge_add(drm_dev->dev, &bridge->base);
+	if (ret) {
+		drm_err(drm_dev,
+			"Cannot add bridge for output %u\n", output);
+		return ERR_PTR(ret);
+	}
+
+	ret = drm_bridge_attach(bridge->enc, &bridge->base, NULL,
+				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+	if (ret) {
+		drm_err(drm_dev,
+			"Cannot attach bridge for output %u\n", output);
+		return ERR_PTR(ret);
+	}
+
+	bridge->conn = drm_bridge_connector_init(drm_dev, bridge->enc);
+	if (IS_ERR(bridge->conn)) {
+		drm_err(drm_dev,
+			"Cannot create connector for output %u\n", output);
+		ret = PTR_ERR(bridge->conn);
+		return ERR_PTR(ret);
+	}
+	drm_connector_attach_encoder(bridge->conn, bridge->enc);
+
+	return bridge;
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_bridge.h b/drivers/gpu/drm/verisilicon/vs_bridge.h
new file mode 100644
index 0000000000000..70fee1749699a
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_bridge.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#ifndef _VS_BRIDGE_H_
+#define _VS_BRIDGE_H_
+
+#include <linux/types.h>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_encoder.h>
+
+struct vs_crtc;
+
+enum vs_bridge_output_interface {
+	VSDC_OUTPUT_INTERFACE_DPI = 0,
+	VSDC_OUTPUT_INTERFACE_DP = 1
+};
+
+struct vs_bridge {
+	struct drm_bridge base;
+	struct drm_encoder *enc;
+	struct drm_connector *conn;
+
+	struct vs_crtc *crtc;
+	struct drm_bridge *next_bridge;
+	enum vs_bridge_output_interface intf;
+};
+
+static inline struct vs_bridge *drm_bridge_to_vs_bridge(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct vs_bridge, base);
+}
+
+struct vs_bridge *vs_bridge_init(struct drm_device *drm_dev,
+				 struct vs_crtc *crtc);
+#endif /* _VS_BRIDGE_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_bridge_regs.h b/drivers/gpu/drm/verisilicon/vs_bridge_regs.h
new file mode 100644
index 0000000000000..9eb30e4564beb
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_bridge_regs.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_BRIDGE_REGS_H_
+#define _VS_BRIDGE_REGS_H_
+
+#include <linux/bits.h>
+
+#define VSDC_DISP_PANEL_CONFIG(n)		(0x1418 + 0x4 * (n))
+#define VSDC_DISP_PANEL_CONFIG_DE_EN		BIT(0)
+#define VSDC_DISP_PANEL_CONFIG_DE_POL		BIT(1)
+#define VSDC_DISP_PANEL_CONFIG_DAT_EN		BIT(4)
+#define VSDC_DISP_PANEL_CONFIG_DAT_POL		BIT(5)
+#define VSDC_DISP_PANEL_CONFIG_CLK_EN		BIT(8)
+#define VSDC_DISP_PANEL_CONFIG_CLK_POL		BIT(9)
+#define VSDC_DISP_PANEL_CONFIG_RUNNING		BIT(12)
+#define VSDC_DISP_PANEL_CONFIG_GAMMA		BIT(13)
+#define VSDC_DISP_PANEL_CONFIG_YUV		BIT(16)
+
+#define VSDC_DISP_DPI_CONFIG(n)			(0x14B8 + 0x4 * (n))
+#define VSDC_DISP_DPI_CONFIG_FMT_MASK		GENMASK(2, 0)
+#define VSDC_DISP_DPI_CONFIG_FMT_RGB565		(0)
+#define VSDC_DISP_DPI_CONFIG_FMT_RGB666		(3)
+#define VSDC_DISP_DPI_CONFIG_FMT_RGB888		(5)
+#define VSDC_DISP_DPI_CONFIG_FMT_RGB101010	(6)
+
+#define VSDC_DISP_PANEL_START			0x1CCC
+#define VSDC_DISP_PANEL_START_RUNNING(n)	BIT(n)
+#define VSDC_DISP_PANEL_START_MULTI_DISP_SYNC	BIT(3)
+
+#define VSDC_DISP_DP_CONFIG(n)			(0x1CD0 + 0x4 * (n))
+#define VSDC_DISP_DP_CONFIG_DP_EN		BIT(3)
+#define VSDC_DISP_DP_CONFIG_FMT_MASK		GENMASK(2, 0)
+#define VSDC_DISP_DP_CONFIG_FMT_RGB565		(0)
+#define VSDC_DISP_DP_CONFIG_FMT_RGB666		(1)
+#define VSDC_DISP_DP_CONFIG_FMT_RGB888		(2)
+#define VSDC_DISP_DP_CONFIG_FMT_RGB101010	(3)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_MASK	GENMASK(7, 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_UYVY8	(2 << 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_YUV8	(4 << 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_UYVY10	(8 << 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_YUV10	(10 << 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_UYYVYY8	(12 << 4)
+#define VSDC_DISP_DP_CONFIG_YUV_FMT_UYYVYY10	(13 << 4)
+
+#define VSDC_DISP_PANEL_CONFIG_EX(n)		(0x2518 + 0x4 * (n))
+#define VSDC_DISP_PANEL_CONFIG_EX_COMMIT	BIT(0)
+
+#endif /* _VS_BRIDGE_REGS_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_crtc.c b/drivers/gpu/drm/verisilicon/vs_crtc.c
new file mode 100644
index 0000000000000..f494017130006
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_crtc.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/clk.h>
+#include <linux/regmap.h>
+#include <linux/units.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_print.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_vblank_helper.h>
+
+#include "vs_crtc_regs.h"
+#include "vs_crtc.h"
+#include "vs_dc.h"
+#include "vs_dc_top_regs.h"
+#include "vs_drm.h"
+#include "vs_plane.h"
+
+static void vs_crtc_atomic_disable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *state)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+	unsigned int output = vcrtc->id;
+
+	drm_crtc_vblank_off(crtc);
+
+	clk_disable_unprepare(dc->pix_clk[output]);
+}
+
+static void vs_crtc_atomic_enable(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+	unsigned int output = vcrtc->id;
+
+	drm_WARN_ON(&dc->drm_dev->base,
+		    clk_prepare_enable(dc->pix_clk[output]));
+
+	drm_crtc_vblank_on(crtc);
+}
+
+static void vs_crtc_mode_set_nofb(struct drm_crtc *crtc)
+{
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+	unsigned int output = vcrtc->id;
+
+	regmap_write(dc->regs, VSDC_DISP_HSIZE(output),
+		     VSDC_DISP_HSIZE_DISP(mode->hdisplay) |
+		     VSDC_DISP_HSIZE_TOTAL(mode->htotal));
+	regmap_write(dc->regs, VSDC_DISP_VSIZE(output),
+		     VSDC_DISP_VSIZE_DISP(mode->vdisplay) |
+		     VSDC_DISP_VSIZE_TOTAL(mode->vtotal));
+	regmap_write(dc->regs, VSDC_DISP_HSYNC(output),
+		     VSDC_DISP_HSYNC_START(mode->hsync_start) |
+		     VSDC_DISP_HSYNC_END(mode->hsync_end) |
+		     VSDC_DISP_HSYNC_EN);
+	if (!(mode->flags & DRM_MODE_FLAG_PHSYNC))
+		regmap_set_bits(dc->regs, VSDC_DISP_HSYNC(output),
+				VSDC_DISP_HSYNC_POL);
+	regmap_write(dc->regs, VSDC_DISP_VSYNC(output),
+		     VSDC_DISP_VSYNC_START(mode->vsync_start) |
+		     VSDC_DISP_VSYNC_END(mode->vsync_end) |
+		     VSDC_DISP_VSYNC_EN);
+	if (!(mode->flags & DRM_MODE_FLAG_PVSYNC))
+		regmap_set_bits(dc->regs, VSDC_DISP_VSYNC(output),
+				VSDC_DISP_VSYNC_POL);
+
+	WARN_ON(clk_set_rate(dc->pix_clk[output], mode->crtc_clock * 1000));
+}
+
+static enum drm_mode_status
+vs_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+	unsigned int output = vcrtc->id;
+	long rate;
+
+	if (mode->htotal > VSDC_DISP_TIMING_VALUE_MAX)
+		return MODE_BAD_HVALUE;
+	if (mode->vtotal > VSDC_DISP_TIMING_VALUE_MAX)
+		return MODE_BAD_VVALUE;
+
+	rate = clk_round_rate(dc->pix_clk[output], mode->clock * HZ_PER_KHZ);
+	if (rate <= 0)
+		return MODE_CLOCK_RANGE;
+
+	return MODE_OK;
+}
+
+static bool vs_crtc_mode_fixup(struct drm_crtc *crtc,
+			       const struct drm_display_mode *m,
+			       struct drm_display_mode *adjusted_mode)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+	unsigned int output = vcrtc->id;
+	long clk_rate;
+
+	drm_mode_set_crtcinfo(adjusted_mode, 0);
+
+	/* Feedback the pixel clock to crtc_clock */
+	clk_rate = adjusted_mode->crtc_clock * HZ_PER_KHZ;
+	clk_rate = clk_round_rate(dc->pix_clk[output], clk_rate);
+	if (clk_rate <= 0)
+		return false;
+
+	adjusted_mode->crtc_clock = clk_rate / HZ_PER_KHZ;
+
+	return true;
+}
+
+static const struct drm_crtc_helper_funcs vs_crtc_helper_funcs = {
+	.atomic_flush	= drm_crtc_vblank_atomic_flush,
+	.atomic_enable	= vs_crtc_atomic_enable,
+	.atomic_disable	= vs_crtc_atomic_disable,
+	.mode_set_nofb	= vs_crtc_mode_set_nofb,
+	.mode_valid	= vs_crtc_mode_valid,
+	.mode_fixup	= vs_crtc_mode_fixup,
+};
+
+static int vs_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+
+	regmap_set_bits(dc->regs, VSDC_TOP_IRQ_EN, VSDC_TOP_IRQ_VSYNC(vcrtc->id));
+
+	return 0;
+}
+
+static void vs_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	struct vs_dc *dc = vcrtc->dc;
+
+	regmap_clear_bits(dc->regs, VSDC_TOP_IRQ_EN, VSDC_TOP_IRQ_VSYNC(vcrtc->id));
+}
+
+static const struct drm_crtc_funcs vs_crtc_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.reset			= drm_atomic_helper_crtc_reset,
+	.set_config		= drm_atomic_helper_set_config,
+	.enable_vblank		= vs_crtc_enable_vblank,
+	.disable_vblank		= vs_crtc_disable_vblank,
+};
+
+struct vs_crtc *vs_crtc_init(struct drm_device *drm_dev, struct vs_dc *dc,
+			     unsigned int output)
+{
+	struct vs_crtc *vcrtc;
+	struct drm_plane *primary;
+	int ret;
+
+	vcrtc = drmm_kzalloc(drm_dev, sizeof(*vcrtc), GFP_KERNEL);
+	if (!vcrtc)
+		return ERR_PTR(-ENOMEM);
+	vcrtc->dc = dc;
+	vcrtc->id = output;
+
+	/* Create our primary plane */
+	primary = vs_primary_plane_init(drm_dev, dc);
+	if (IS_ERR(primary)) {
+		drm_err(drm_dev, "Couldn't create the primary plane\n");
+		return ERR_PTR(PTR_ERR(primary));
+	}
+
+	ret = drmm_crtc_init_with_planes(drm_dev, &vcrtc->base,
+					 primary,
+					 NULL,
+					 &vs_crtc_funcs,
+					 NULL);
+	if (ret) {
+		drm_err(drm_dev, "Couldn't initialize CRTC\n");
+		return ERR_PTR(ret);
+	}
+
+	drm_crtc_helper_add(&vcrtc->base, &vs_crtc_helper_funcs);
+
+	return vcrtc;
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_crtc.h b/drivers/gpu/drm/verisilicon/vs_crtc.h
new file mode 100644
index 0000000000000..b45580bd99b33
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_crtc.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#ifndef _VS_CRTC_H_
+#define _VS_CRTC_H_
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_vblank.h>
+
+#define VSDC_DISP_TIMING_VALUE_MAX BIT_MASK(15)
+
+struct vs_dc;
+
+struct vs_crtc {
+	struct drm_crtc base;
+
+	struct vs_dc *dc;
+	unsigned int id;
+};
+
+static inline struct vs_crtc *drm_crtc_to_vs_crtc(struct drm_crtc *crtc)
+{
+	return container_of(crtc, struct vs_crtc, base);
+}
+
+struct vs_crtc *vs_crtc_init(struct drm_device *drm_dev, struct vs_dc *dc,
+			     unsigned int output);
+
+#endif /* _VS_CRTC_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_crtc_regs.h b/drivers/gpu/drm/verisilicon/vs_crtc_regs.h
new file mode 100644
index 0000000000000..c7930e817635c
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_crtc_regs.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_CRTC_REGS_H_
+#define _VS_CRTC_REGS_H_
+
+#include <linux/bits.h>
+
+#define VSDC_DISP_DITHER_CONFIG(n)		(0x1410 + 0x4 * (n))
+
+#define VSDC_DISP_DITHER_TABLE_LOW(n)		(0x1420 + 0x4 * (n))
+#define VSDC_DISP_DITHER_TABLE_LOW_DEFAULT	0x7B48F3C0
+
+#define VSDC_DISP_DITHER_TABLE_HIGH(n)		(0x1428 + 0x4 * (n))
+#define VSDC_DISP_DITHER_TABLE_HIGH_DEFAULT	0x596AD1E2
+
+#define VSDC_DISP_HSIZE(n)			(0x1430 + 0x4 * (n))
+#define VSDC_DISP_HSIZE_DISP_MASK		GENMASK(14, 0)
+#define VSDC_DISP_HSIZE_DISP(v)			((v) << 0)
+#define VSDC_DISP_HSIZE_TOTAL_MASK		GENMASK(30, 16)
+#define VSDC_DISP_HSIZE_TOTAL(v)		((v) << 16)
+
+#define VSDC_DISP_HSYNC(n)			(0x1438 + 0x4 * (n))
+#define VSDC_DISP_HSYNC_START_MASK		GENMASK(14, 0)
+#define VSDC_DISP_HSYNC_START(v)		((v) << 0)
+#define VSDC_DISP_HSYNC_END_MASK		GENMASK(29, 15)
+#define VSDC_DISP_HSYNC_END(v)			((v) << 15)
+#define VSDC_DISP_HSYNC_EN			BIT(30)
+#define VSDC_DISP_HSYNC_POL			BIT(31)
+
+#define VSDC_DISP_VSIZE(n)			(0x1440 + 0x4 * (n))
+#define VSDC_DISP_VSIZE_DISP_MASK		GENMASK(14, 0)
+#define VSDC_DISP_VSIZE_DISP(v)			((v) << 0)
+#define VSDC_DISP_VSIZE_TOTAL_MASK		GENMASK(30, 16)
+#define VSDC_DISP_VSIZE_TOTAL(v)		((v) << 16)
+
+#define VSDC_DISP_VSYNC(n)			(0x1448 + 0x4 * (n))
+#define VSDC_DISP_VSYNC_START_MASK		GENMASK(14, 0)
+#define VSDC_DISP_VSYNC_START(v)		((v) << 0)
+#define VSDC_DISP_VSYNC_END_MASK		GENMASK(29, 15)
+#define VSDC_DISP_VSYNC_END(v)			((v) << 15)
+#define VSDC_DISP_VSYNC_EN			BIT(30)
+#define VSDC_DISP_VSYNC_POL			BIT(31)
+
+#define VSDC_DISP_CURRENT_LOCATION(n)		(0x1450 + 0x4 * (n))
+
+#define VSDC_DISP_GAMMA_INDEX(n)		(0x1458 + 0x4 * (n))
+
+#define VSDC_DISP_GAMMA_DATA(n)			(0x1460 + 0x4 * (n))
+
+#define VSDC_DISP_IRQ_STA			0x147C
+
+#define VSDC_DISP_IRQ_EN			0x1480
+
+#endif /* _VS_CRTC_REGS_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_dc.c b/drivers/gpu/drm/verisilicon/vs_dc.c
new file mode 100644
index 0000000000000..5f629d2d4beac
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_dc.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/irqreturn.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+
+#include "vs_crtc.h"
+#include "vs_dc.h"
+#include "vs_dc_top_regs.h"
+#include "vs_drm.h"
+#include "vs_hwdb.h"
+
+static const struct regmap_config vs_dc_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = sizeof(u32),
+	/* VSDC_OVL_CONFIG_EX(1) */
+	.max_register = 0x2544,
+};
+
+static const struct of_device_id vs_dc_driver_dt_match[] = {
+	{ .compatible = "verisilicon,dc" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, vs_dc_driver_dt_match);
+
+static irqreturn_t vs_dc_irq_handler(int irq, void *private)
+{
+	struct vs_dc *dc = private;
+	u32 irqs;
+
+	regmap_read(dc->regs, VSDC_TOP_IRQ_ACK, &irqs);
+
+	vs_drm_handle_irq(dc, irqs);
+
+	return IRQ_HANDLED;
+}
+
+static int vs_dc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct vs_dc *dc;
+	void __iomem *regs;
+	unsigned int port_count, i;
+	/* pix%u */
+	char pixclk_name[14];
+	int irq, ret;
+
+	if (!dev->of_node) {
+		dev_err(dev, "can't find DC devices\n");
+		return -ENODEV;
+	}
+
+	port_count = of_graph_get_port_count(dev->of_node);
+	if (!port_count) {
+		dev_err(dev, "can't find DC downstream ports\n");
+		return -ENODEV;
+	}
+	if (port_count > VSDC_MAX_OUTPUTS) {
+		dev_err(dev, "too many DC downstream ports than possible\n");
+		return -EINVAL;
+	}
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(dev, "No suitable DMA available\n");
+		return ret;
+	}
+
+	dc = devm_kzalloc(dev, sizeof(*dc), GFP_KERNEL);
+	if (!dc)
+		return -ENOMEM;
+
+	dc->rsts[0].id = "core";
+	dc->rsts[1].id = "axi";
+	dc->rsts[2].id = "ahb";
+
+	ret = devm_reset_control_bulk_get_optional_shared(dev, VSDC_RESET_COUNT,
+							  dc->rsts);
+	if (ret) {
+		dev_err(dev, "can't get reset lines\n");
+		return ret;
+	}
+
+	dc->core_clk = devm_clk_get_enabled(dev, "core");
+	if (IS_ERR(dc->core_clk)) {
+		dev_err(dev, "can't get core clock\n");
+		return PTR_ERR(dc->core_clk);
+	}
+
+	dc->axi_clk = devm_clk_get_enabled(dev, "axi");
+	if (IS_ERR(dc->axi_clk)) {
+		dev_err(dev, "can't get axi clock\n");
+		return PTR_ERR(dc->axi_clk);
+	}
+
+	dc->ahb_clk = devm_clk_get_enabled(dev, "ahb");
+	if (IS_ERR(dc->ahb_clk)) {
+		dev_err(dev, "can't get ahb clock\n");
+		return PTR_ERR(dc->ahb_clk);
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "can't get irq\n");
+		return irq;
+	}
+
+	ret = reset_control_bulk_deassert(VSDC_RESET_COUNT, dc->rsts);
+	if (ret) {
+		dev_err(dev, "can't deassert reset lines\n");
+		return ret;
+	}
+
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs)) {
+		dev_err(dev, "can't map registers");
+		ret = PTR_ERR(regs);
+		goto err_rst_assert;
+	}
+
+	dc->regs = devm_regmap_init_mmio(dev, regs, &vs_dc_regmap_cfg);
+	if (IS_ERR(dc->regs)) {
+		ret = PTR_ERR(dc->regs);
+		goto err_rst_assert;
+	}
+
+	ret = vs_fill_chip_identity(dc->regs, &dc->identity);
+	if (ret)
+		goto err_rst_assert;
+
+	dev_info(dev, "Found DC%x rev %x customer %x\n", dc->identity.model,
+		 dc->identity.revision, dc->identity.customer_id);
+
+	if (port_count > dc->identity.display_count) {
+		dev_err(dev, "too many downstream ports than HW capability\n");
+		ret = -EINVAL;
+		goto err_rst_assert;
+	}
+
+	for (i = 0; i < dc->identity.display_count; i++) {
+		snprintf(pixclk_name, sizeof(pixclk_name), "pix%u", i);
+		dc->pix_clk[i] = devm_clk_get(dev, pixclk_name);
+		if (IS_ERR(dc->pix_clk[i])) {
+			dev_err(dev, "can't get pixel clk %u\n", i);
+			ret = PTR_ERR(dc->pix_clk[i]);
+			goto err_rst_assert;
+		}
+	}
+
+	ret = devm_request_irq(dev, irq, vs_dc_irq_handler, 0,
+			       dev_name(dev), dc);
+	if (ret) {
+		dev_err(dev, "can't request irq\n");
+		goto err_rst_assert;
+	}
+
+	dev_set_drvdata(dev, dc);
+
+	ret = vs_drm_initialize(dc, pdev);
+	if (ret)
+		goto err_rst_assert;
+
+	return 0;
+
+err_rst_assert:
+	reset_control_bulk_assert(VSDC_RESET_COUNT, dc->rsts);
+	return ret;
+}
+
+static void vs_dc_remove(struct platform_device *pdev)
+{
+	struct vs_dc *dc = dev_get_drvdata(&pdev->dev);
+
+	vs_drm_finalize(dc);
+
+	dev_set_drvdata(&pdev->dev, NULL);
+
+	reset_control_bulk_assert(VSDC_RESET_COUNT, dc->rsts);
+}
+
+static void vs_dc_shutdown(struct platform_device *pdev)
+{
+	struct vs_dc *dc = dev_get_drvdata(&pdev->dev);
+
+	vs_drm_shutdown_handler(dc);
+}
+
+struct platform_driver vs_dc_platform_driver = {
+	.probe = vs_dc_probe,
+	.remove = vs_dc_remove,
+	.shutdown = vs_dc_shutdown,
+	.driver = {
+		.name = "verisilicon-dc",
+		.of_match_table = vs_dc_driver_dt_match,
+	},
+};
+
+module_platform_driver(vs_dc_platform_driver);
+
+MODULE_AUTHOR("Icenowy Zheng <uwu@icenowy.me>");
+MODULE_DESCRIPTION("Verisilicon display controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/verisilicon/vs_dc.h b/drivers/gpu/drm/verisilicon/vs_dc.h
new file mode 100644
index 0000000000000..ed1016f18758e
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_dc.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_DC_H_
+#define _VS_DC_H_
+
+#include <linux/clk.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <drm/drm_device.h>
+
+#include "vs_hwdb.h"
+
+#define VSDC_MAX_OUTPUTS 2
+#define VSDC_RESET_COUNT 3
+
+struct vs_drm_dev;
+struct vs_crtc;
+
+struct vs_dc {
+	struct regmap *regs;
+	struct clk *core_clk;
+	struct clk *axi_clk;
+	struct clk *ahb_clk;
+	struct clk *pix_clk[VSDC_MAX_OUTPUTS];
+	struct reset_control_bulk_data rsts[VSDC_RESET_COUNT];
+
+	struct vs_drm_dev *drm_dev;
+	struct vs_chip_identity identity;
+};
+
+#endif /* _VS_DC_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_dc_top_regs.h b/drivers/gpu/drm/verisilicon/vs_dc_top_regs.h
new file mode 100644
index 0000000000000..50509bbbff08f
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_dc_top_regs.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_DC_TOP_H_
+#define _VS_DC_TOP_H_
+
+#include <linux/bits.h>
+
+#define VSDC_TOP_RST				0x0000
+
+#define VSDC_TOP_IRQ_ACK			0x0010
+#define VSDC_TOP_IRQ_VSYNC(n)			BIT(n)
+
+#define VSDC_TOP_IRQ_EN				0x0014
+
+#define VSDC_TOP_CHIP_MODEL			0x0020
+
+#define VSDC_TOP_CHIP_REV			0x0024
+
+#define VSDC_TOP_CHIP_CUSTOMER_ID		0x0030
+
+#endif /* _VS_DC_TOP_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_drm.c b/drivers/gpu/drm/verisilicon/vs_drm.c
new file mode 100644
index 0000000000000..fd259d53f49f1
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_drm.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/aperture.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/console.h>
+
+#include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_dumb_buffers.h>
+#include <drm/drm_fbdev_dma.h>
+#include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_vblank.h>
+
+#include "vs_bridge.h"
+#include "vs_crtc.h"
+#include "vs_dc.h"
+#include "vs_dc_top_regs.h"
+#include "vs_drm.h"
+
+#define DRIVER_NAME	"verisilicon"
+#define DRIVER_DESC	"Verisilicon DC-series display controller driver"
+#define DRIVER_MAJOR	1
+#define DRIVER_MINOR	0
+
+static int vs_gem_dumb_create(struct drm_file *file_priv,
+			      struct drm_device *drm,
+			      struct drm_mode_create_dumb *args)
+{
+	int ret;
+
+	/* The hardware wants 128B-aligned pitches for linear buffers. */
+	ret = drm_mode_size_dumb(drm, args, 128, 0);
+	if (ret)
+		return ret;
+
+	return drm_gem_dma_dumb_create_internal(file_priv, drm, args);
+}
+
+DEFINE_DRM_GEM_FOPS(vs_drm_driver_fops);
+
+static const struct drm_driver vs_drm_driver = {
+	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
+	.fops			= &vs_drm_driver_fops,
+	.name	= DRIVER_NAME,
+	.desc	= DRIVER_DESC,
+	.major	= DRIVER_MAJOR,
+	.minor	= DRIVER_MINOR,
+
+	/* GEM Operations */
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(vs_gem_dumb_create),
+	DRM_FBDEV_DMA_DRIVER_OPS,
+};
+
+static const struct drm_mode_config_funcs vs_mode_config_funcs = {
+	.fb_create		= drm_gem_fb_create,
+	.atomic_check		= drm_atomic_helper_check,
+	.atomic_commit		= drm_atomic_helper_commit,
+};
+
+static struct drm_mode_config_helper_funcs vs_mode_config_helper_funcs = {
+	.atomic_commit_tail = drm_atomic_helper_commit_tail,
+};
+
+static void vs_mode_config_init(struct drm_device *drm)
+{
+	drm->mode_config.min_width = 0;
+	drm->mode_config.min_height = 0;
+	drm->mode_config.max_width = 8192;
+	drm->mode_config.max_height = 8192;
+	drm->mode_config.funcs = &vs_mode_config_funcs;
+	drm->mode_config.helper_private = &vs_mode_config_helper_funcs;
+}
+
+int vs_drm_initialize(struct vs_dc *dc, struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct vs_drm_dev *vdrm;
+	struct drm_device *drm;
+	struct vs_crtc *crtc;
+	struct vs_bridge *bridge;
+	unsigned int i;
+	int ret;
+
+	vdrm = devm_drm_dev_alloc(dev, &vs_drm_driver, struct vs_drm_dev, base);
+	if (IS_ERR(vdrm))
+		return PTR_ERR(vdrm);
+
+	drm = &vdrm->base;
+	vdrm->dc = dc;
+	dc->drm_dev = vdrm;
+
+	ret = drmm_mode_config_init(drm);
+	if (ret)
+		return ret;
+
+	/* Remove early framebuffers (ie. simple-framebuffer) */
+	ret = aperture_remove_all_conflicting_devices(DRIVER_NAME);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < dc->identity.display_count; i++) {
+		crtc = vs_crtc_init(drm, dc, i);
+		if (IS_ERR(crtc))
+			return PTR_ERR(crtc);
+
+		bridge = vs_bridge_init(drm, crtc);
+		if (IS_ERR(bridge))
+			return PTR_ERR(bridge);
+
+		vdrm->crtcs[i] = crtc;
+	}
+
+	ret = drm_vblank_init(drm, dc->identity.display_count);
+	if (ret)
+		return ret;
+
+	vs_mode_config_init(drm);
+
+	/* Enable connectors polling */
+	drm_kms_helper_poll_init(drm);
+
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto err_fini_poll;
+
+	drm_client_setup(drm, NULL);
+
+	return 0;
+
+err_fini_poll:
+	drm_kms_helper_poll_fini(drm);
+	return ret;
+}
+
+void vs_drm_finalize(struct vs_dc *dc)
+{
+	struct vs_drm_dev *vdrm = dc->drm_dev;
+	struct drm_device *drm = &vdrm->base;
+
+	drm_dev_unregister(drm);
+	drm_kms_helper_poll_fini(drm);
+	drm_atomic_helper_shutdown(drm);
+	dc->drm_dev = NULL;
+}
+
+void vs_drm_shutdown_handler(struct vs_dc *dc)
+{
+	struct vs_drm_dev *vdrm = dc->drm_dev;
+
+	drm_atomic_helper_shutdown(&vdrm->base);
+}
+
+void vs_drm_handle_irq(struct vs_dc *dc, u32 irqs)
+{
+	unsigned int i;
+
+	for (i = 0; i < dc->identity.display_count; i++) {
+		if (irqs & VSDC_TOP_IRQ_VSYNC(i)) {
+			irqs &= ~VSDC_TOP_IRQ_VSYNC(i);
+			if (dc->drm_dev->crtcs[i])
+				drm_crtc_handle_vblank(&dc->drm_dev->crtcs[i]->base);
+		}
+	}
+
+	if (irqs)
+		drm_warn_once(&dc->drm_dev->base,
+			      "Unknown Verisilicon DC interrupt 0x%x fired!\n",
+			      irqs);
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_drm.h b/drivers/gpu/drm/verisilicon/vs_drm.h
new file mode 100644
index 0000000000000..606338206a427
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_drm.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#ifndef _VS_DRM_H_
+#define _VS_DRM_H_
+
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#include <drm/drm_device.h>
+
+struct vs_dc;
+
+struct vs_drm_dev {
+	struct drm_device base;
+
+	struct vs_dc *dc;
+	struct vs_crtc *crtcs[VSDC_MAX_OUTPUTS];
+};
+
+int vs_drm_initialize(struct vs_dc *dc, struct platform_device *pdev);
+void vs_drm_finalize(struct vs_dc *dc);
+void vs_drm_shutdown_handler(struct vs_dc *dc);
+void vs_drm_handle_irq(struct vs_dc *dc, u32 irqs);
+
+#endif /* _VS_DRM_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_hwdb.c b/drivers/gpu/drm/verisilicon/vs_hwdb.c
new file mode 100644
index 0000000000000..09336af0900ae
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_hwdb.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/errno.h>
+
+#include <drm/drm_fourcc.h>
+
+#include "vs_dc_top_regs.h"
+#include "vs_hwdb.h"
+
+static const u32 vs_formats_array_no_yuv444[] = {
+	DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_XBGR4444,
+	DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_BGRX4444,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_BGRA1010102,
+	/* TODO: non-RGB formats */
+};
+
+static const u32 vs_formats_array_with_yuv444[] = {
+	DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_XBGR4444,
+	DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_BGRX4444,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_BGRA1010102,
+	/* TODO: non-RGB formats */
+};
+
+static const struct vs_formats vs_formats_no_yuv444 = {
+	.array = vs_formats_array_no_yuv444,
+	.num = ARRAY_SIZE(vs_formats_array_no_yuv444)
+};
+
+static const struct vs_formats vs_formats_with_yuv444 = {
+	.array = vs_formats_array_with_yuv444,
+	.num = ARRAY_SIZE(vs_formats_array_with_yuv444)
+};
+
+static struct vs_chip_identity vs_chip_identities[] = {
+	{
+		.model = 0x8200,
+		.revision = 0x5720,
+		.customer_id = ~0U,
+
+		.display_count = 2,
+		.formats = &vs_formats_no_yuv444,
+	},
+	{
+		.model = 0x8200,
+		.revision = 0x5721,
+		.customer_id = 0x30B,
+
+		.display_count = 2,
+		.formats = &vs_formats_no_yuv444,
+	},
+	{
+		.model = 0x8200,
+		.revision = 0x5720,
+		.customer_id = 0x310,
+
+		.display_count = 2,
+		.formats = &vs_formats_with_yuv444,
+	},
+	{
+		.model = 0x8200,
+		.revision = 0x5720,
+		.customer_id = 0x311,
+
+		.display_count = 2,
+		.formats = &vs_formats_no_yuv444,
+	},
+};
+
+int vs_fill_chip_identity(struct regmap *regs,
+			  struct vs_chip_identity *ident)
+{
+	u32 model;
+	u32 revision;
+	u32 customer_id;
+	int i;
+
+	regmap_read(regs, VSDC_TOP_CHIP_MODEL, &model);
+	regmap_read(regs, VSDC_TOP_CHIP_REV, &revision);
+	regmap_read(regs, VSDC_TOP_CHIP_CUSTOMER_ID, &customer_id);
+
+	for (i = 0; i < ARRAY_SIZE(vs_chip_identities); i++) {
+		if (vs_chip_identities[i].model == model &&
+		    vs_chip_identities[i].revision == revision &&
+		    (vs_chip_identities[i].customer_id == customer_id ||
+		     vs_chip_identities[i].customer_id == ~0U)) {
+			memcpy(ident, &vs_chip_identities[i], sizeof(*ident));
+			ident->customer_id = customer_id;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_hwdb.h b/drivers/gpu/drm/verisilicon/vs_hwdb.h
new file mode 100644
index 0000000000000..92192e4fa0862
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_hwdb.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#ifndef _VS_HWDB_H_
+#define _VS_HWDB_H_
+
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+struct vs_formats {
+	const u32 *array;
+	unsigned int num;
+};
+
+struct vs_chip_identity {
+	u32 model;
+	u32 revision;
+	u32 customer_id;
+
+	u32 display_count;
+	const struct vs_formats *formats;
+};
+
+int vs_fill_chip_identity(struct regmap *regs,
+			  struct vs_chip_identity *ident);
+
+#endif /* _VS_HWDB_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.c b/drivers/gpu/drm/verisilicon/vs_plane.c
new file mode 100644
index 0000000000000..2f3953e588a34
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_plane.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/errno.h>
+#include <linux/printk.h>
+
+#include <drm/drm_fb_dma_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_gem_dma_helper.h>
+
+#include "vs_plane.h"
+
+void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
+{
+	switch (drm_format) {
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_BGRX4444:
+		vs_format->color = VSDC_COLOR_FORMAT_X4R4G4B4;
+		break;
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_BGRA4444:
+		vs_format->color = VSDC_COLOR_FORMAT_A4R4G4B4;
+		break;
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_BGRX5551:
+		vs_format->color = VSDC_COLOR_FORMAT_X1R5G5B5;
+		break;
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGRA5551:
+		vs_format->color = VSDC_COLOR_FORMAT_A1R5G5B5;
+		break;
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_BGR565:
+		vs_format->color = VSDC_COLOR_FORMAT_R5G6B5;
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_BGRX8888:
+		vs_format->color = VSDC_COLOR_FORMAT_X8R8G8B8;
+		break;
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_BGRA8888:
+		vs_format->color = VSDC_COLOR_FORMAT_A8R8G8B8;
+		break;
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_BGRA1010102:
+		vs_format->color = VSDC_COLOR_FORMAT_A2R10G10B10;
+		break;
+	default:
+		pr_warn("Unexpected drm format!\n");
+	}
+
+	switch (drm_format) {
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_RGBA1010102:
+		vs_format->swizzle = VSDC_SWIZZLE_RGBA;
+		break;
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ABGR2101010:
+		vs_format->swizzle = VSDC_SWIZZLE_ABGR;
+		break;
+	case DRM_FORMAT_BGRX4444:
+	case DRM_FORMAT_BGRA4444:
+	case DRM_FORMAT_BGRX5551:
+	case DRM_FORMAT_BGRA5551:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRA1010102:
+		vs_format->swizzle = VSDC_SWIZZLE_BGRA;
+		break;
+	default:
+		/* N/A for YUV formats */
+		vs_format->swizzle = VSDC_SWIZZLE_ARGB;
+	}
+
+	/* N/A for non-YUV formats */
+	vs_format->uv_swizzle = false;
+}
+
+dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
+			      const struct drm_rect *src_rect)
+{
+	struct drm_gem_dma_object *gem;
+	dma_addr_t dma_addr;
+
+	/* Get the physical address of the buffer in memory */
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
+
+	/* Compute the start of the displayed memory */
+	dma_addr = gem->dma_addr + fb->offsets[0];
+
+	/* Fixup framebuffer address for src coordinates */
+	dma_addr += drm_format_info_min_pitch(fb->format, 0,
+					      src_rect->x1 >> 16);
+	dma_addr += (src_rect->y1 >> 16) * fb->pitches[0];
+
+	return dma_addr;
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.h b/drivers/gpu/drm/verisilicon/vs_plane.h
new file mode 100644
index 0000000000000..41875ea3d66a5
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_plane.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_PLANE_H_
+#define _VS_PLANE_H_
+
+#include <linux/types.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_rect.h>
+
+#define VSDC_MAKE_PLANE_SIZE(w, h) (((w) & 0x7fff) | (((h) & 0x7fff) << 15))
+#define VSDC_MAKE_PLANE_POS(x, y) (((x) & 0x7fff) | (((y) & 0x7fff) << 15))
+
+struct vs_dc;
+
+enum vs_color_format {
+	VSDC_COLOR_FORMAT_X4R4G4B4,
+	VSDC_COLOR_FORMAT_A4R4G4B4,
+	VSDC_COLOR_FORMAT_X1R5G5B5,
+	VSDC_COLOR_FORMAT_A1R5G5B5,
+	VSDC_COLOR_FORMAT_R5G6B5,
+	VSDC_COLOR_FORMAT_X8R8G8B8,
+	VSDC_COLOR_FORMAT_A8R8G8B8,
+	VSDC_COLOR_FORMAT_YUY2,
+	VSDC_COLOR_FORMAT_UYVY,
+	VSDC_COLOR_FORMAT_INDEX8,
+	VSDC_COLOR_FORMAT_MONOCHROME,
+	VSDC_COLOR_FORMAT_YV12 = 0xf,
+	VSDC_COLOR_FORMAT_A8,
+	VSDC_COLOR_FORMAT_NV12,
+	VSDC_COLOR_FORMAT_NV16,
+	VSDC_COLOR_FORMAT_RG16,
+	VSDC_COLOR_FORMAT_R8,
+	VSDC_COLOR_FORMAT_NV12_10BIT,
+	VSDC_COLOR_FORMAT_A2R10G10B10,
+	VSDC_COLOR_FORMAT_NV16_10BIT,
+	VSDC_COLOR_FORMAT_INDEX1,
+	VSDC_COLOR_FORMAT_INDEX2,
+	VSDC_COLOR_FORMAT_INDEX4,
+	VSDC_COLOR_FORMAT_P010,
+	VSDC_COLOR_FORMAT_YUV444,
+	VSDC_COLOR_FORMAT_YUV444_10BIT
+};
+
+enum vs_swizzle {
+	VSDC_SWIZZLE_ARGB,
+	VSDC_SWIZZLE_RGBA,
+	VSDC_SWIZZLE_ABGR,
+	VSDC_SWIZZLE_BGRA,
+};
+
+struct vs_format {
+	enum vs_color_format color;
+	enum vs_swizzle swizzle;
+	bool uv_swizzle;
+};
+
+void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format);
+dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
+			      const struct drm_rect *src_rect);
+
+struct drm_plane *vs_primary_plane_init(struct drm_device *dev, struct vs_dc *dc);
+
+#endif /* _VS_PLANE_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_primary_plane.c b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
new file mode 100644
index 0000000000000..e8fcb5958615c
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ */
+
+#include <linux/regmap.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_print.h>
+
+#include "vs_crtc.h"
+#include "vs_plane.h"
+#include "vs_dc.h"
+#include "vs_primary_plane_regs.h"
+
+static int vs_primary_plane_atomic_check(struct drm_plane *plane,
+					 struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
+										 plane);
+	struct drm_crtc *crtc = new_plane_state->crtc;
+	struct drm_crtc_state *crtc_state;
+
+	if (!crtc)
+		return 0;
+
+	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+	if (WARN_ON(!crtc_state))
+		return -EINVAL;
+
+	return drm_atomic_helper_check_plane_state(new_plane_state,
+						   crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, true);
+}
+
+static void vs_primary_plane_commit(struct vs_dc *dc, unsigned int output)
+{
+	regmap_set_bits(dc->regs, VSDC_FB_CONFIG_EX(output),
+			VSDC_FB_CONFIG_EX_COMMIT);
+}
+
+static void vs_primary_plane_atomic_enable(struct drm_plane *plane,
+					   struct drm_atomic_state *atomic_state)
+{
+	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state,
+								       plane);
+	struct drm_crtc *crtc = state->crtc;
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	unsigned int output = vcrtc->id;
+	struct vs_dc *dc = vcrtc->dc;
+
+	regmap_set_bits(dc->regs, VSDC_FB_CONFIG_EX(output),
+			VSDC_FB_CONFIG_EX_FB_EN);
+	regmap_update_bits(dc->regs, VSDC_FB_CONFIG_EX(output),
+			   VSDC_FB_CONFIG_EX_DISPLAY_ID_MASK,
+			   VSDC_FB_CONFIG_EX_DISPLAY_ID(output));
+
+	vs_primary_plane_commit(dc, output);
+}
+
+static void vs_primary_plane_atomic_disable(struct drm_plane *plane,
+					    struct drm_atomic_state *atomic_state)
+{
+	struct drm_plane_state *state = drm_atomic_get_old_plane_state(atomic_state,
+								       plane);
+	struct drm_crtc *crtc = state->crtc;
+	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
+	unsigned int output = vcrtc->id;
+	struct vs_dc *dc = vcrtc->dc;
+
+	regmap_set_bits(dc->regs, VSDC_FB_CONFIG_EX(output),
+			VSDC_FB_CONFIG_EX_FB_EN);
+
+	vs_primary_plane_commit(dc, output);
+}
+
+static void vs_primary_plane_atomic_update(struct drm_plane *plane,
+					   struct drm_atomic_state *atomic_state)
+{
+	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state,
+								       plane);
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_crtc *crtc = state->crtc;
+	struct vs_dc *dc;
+	struct vs_crtc *vcrtc;
+	struct vs_format fmt;
+	unsigned int output;
+	dma_addr_t dma_addr;
+
+	if (!state->visible) {
+		vs_primary_plane_atomic_disable(plane, atomic_state);
+		return;
+	}
+
+	vcrtc = drm_crtc_to_vs_crtc(crtc);
+	output = vcrtc->id;
+	dc = vcrtc->dc;
+
+	drm_format_to_vs_format(state->fb->format->format, &fmt);
+
+	regmap_update_bits(dc->regs, VSDC_FB_CONFIG(output),
+			   VSDC_FB_CONFIG_FMT_MASK,
+			   VSDC_FB_CONFIG_FMT(fmt.color));
+	regmap_update_bits(dc->regs, VSDC_FB_CONFIG(output),
+			   VSDC_FB_CONFIG_SWIZZLE_MASK,
+			   VSDC_FB_CONFIG_SWIZZLE(fmt.swizzle));
+	regmap_assign_bits(dc->regs, VSDC_FB_CONFIG(output),
+			   VSDC_FB_CONFIG_UV_SWIZZLE_EN, fmt.uv_swizzle);
+
+	dma_addr = vs_fb_get_dma_addr(fb, &state->src);
+
+	regmap_write(dc->regs, VSDC_FB_ADDRESS(output),
+		     lower_32_bits(dma_addr));
+	regmap_write(dc->regs, VSDC_FB_STRIDE(output),
+		     fb->pitches[0]);
+
+	regmap_write(dc->regs, VSDC_FB_TOP_LEFT(output),
+		     VSDC_MAKE_PLANE_POS(state->crtc_x, state->crtc_y));
+	regmap_write(dc->regs, VSDC_FB_BOTTOM_RIGHT(output),
+		     VSDC_MAKE_PLANE_POS(state->crtc_x + state->crtc_w,
+					 state->crtc_y + state->crtc_h));
+	regmap_write(dc->regs, VSDC_FB_SIZE(output),
+		     VSDC_MAKE_PLANE_SIZE(state->crtc_w, state->crtc_h));
+
+	regmap_write(dc->regs, VSDC_FB_BLEND_CONFIG(output),
+		     VSDC_FB_BLEND_CONFIG_BLEND_DISABLE);
+
+	vs_primary_plane_commit(dc, output);
+}
+
+static const struct drm_plane_helper_funcs vs_primary_plane_helper_funcs = {
+	.atomic_check	= vs_primary_plane_atomic_check,
+	.atomic_update	= vs_primary_plane_atomic_update,
+	.atomic_enable	= vs_primary_plane_atomic_enable,
+	.atomic_disable	= vs_primary_plane_atomic_disable,
+};
+
+static const struct drm_plane_funcs vs_primary_plane_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.reset			= drm_atomic_helper_plane_reset,
+	.update_plane		= drm_atomic_helper_update_plane,
+};
+
+struct drm_plane *vs_primary_plane_init(struct drm_device *drm_dev, struct vs_dc *dc)
+{
+	struct drm_plane *plane;
+
+	plane = drmm_universal_plane_alloc(drm_dev, struct drm_plane, dev, 0,
+					   &vs_primary_plane_funcs,
+					   dc->identity.formats->array,
+					   dc->identity.formats->num,
+					   NULL,
+					   DRM_PLANE_TYPE_PRIMARY,
+					   NULL);
+
+	if (IS_ERR(plane))
+		return plane;
+
+	drm_plane_helper_add(plane, &vs_primary_plane_helper_funcs);
+
+	return plane;
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_primary_plane_regs.h b/drivers/gpu/drm/verisilicon/vs_primary_plane_regs.h
new file mode 100644
index 0000000000000..cbb125c46b390
--- /dev/null
+++ b/drivers/gpu/drm/verisilicon/vs_primary_plane_regs.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me>
+ *
+ * Based on vs_dc_hw.h, which is:
+ *   Copyright (C) 2023 VeriSilicon Holdings Co., Ltd.
+ */
+
+#ifndef _VS_PRIMARY_PLANE_REGS_H_
+#define _VS_PRIMARY_PLANE_REGS_H_
+
+#include <linux/bits.h>
+
+#define VSDC_FB_ADDRESS(n)			(0x1400 + 0x4 * (n))
+
+#define VSDC_FB_STRIDE(n)			(0x1408 + 0x4 * (n))
+
+#define VSDC_FB_CONFIG(n)			(0x1518 + 0x4 * (n))
+#define VSDC_FB_CONFIG_CLEAR_EN			BIT(8)
+#define VSDC_FB_CONFIG_ROT_MASK			GENMASK(13, 11)
+#define VSDC_FB_CONFIG_ROT(v)			((v) << 11)
+#define VSDC_FB_CONFIG_YUV_SPACE_MASK		GENMASK(16, 14)
+#define VSDC_FB_CONFIG_YUV_SPACE(v)		((v) << 14)
+#define VSDC_FB_CONFIG_TILE_MODE_MASK		GENMASK(21, 17)
+#define VSDC_FB_CONFIG_TILE_MODE(v)		((v) << 14)
+#define VSDC_FB_CONFIG_SCALE_EN			BIT(22)
+#define VSDC_FB_CONFIG_SWIZZLE_MASK		GENMASK(24, 23)
+#define VSDC_FB_CONFIG_SWIZZLE(v)		((v) << 23)
+#define VSDC_FB_CONFIG_UV_SWIZZLE_EN		BIT(25)
+#define VSDC_FB_CONFIG_FMT_MASK			GENMASK(31, 26)
+#define VSDC_FB_CONFIG_FMT(v)			((v) << 26)
+
+#define VSDC_FB_SIZE(n)				(0x1810 + 0x4 * (n))
+/* Fill with value generated with VSDC_MAKE_PLANE_SIZE(w, h) */
+
+#define VSDC_FB_CONFIG_EX(n)			(0x1CC0 + 0x4 * (n))
+#define VSDC_FB_CONFIG_EX_COMMIT		BIT(12)
+#define VSDC_FB_CONFIG_EX_FB_EN			BIT(13)
+#define VSDC_FB_CONFIG_EX_ZPOS_MASK		GENMASK(18, 16)
+#define VSDC_FB_CONFIG_EX_ZPOS(v)		((v) << 16)
+#define VSDC_FB_CONFIG_EX_DISPLAY_ID_MASK	GENMASK(19, 19)
+#define VSDC_FB_CONFIG_EX_DISPLAY_ID(v)		((v) << 19)
+
+#define VSDC_FB_TOP_LEFT(n)			(0x24D8 + 0x4 * (n))
+/* Fill with value generated with VSDC_MAKE_PLANE_POS(x, y) */
+
+#define VSDC_FB_BOTTOM_RIGHT(n)			(0x24E0 + 0x4 * (n))
+/* Fill with value generated with VSDC_MAKE_PLANE_POS(x, y) */
+
+#define VSDC_FB_BLEND_CONFIG(n)			(0x2510 + 0x4 * (n))
+#define VSDC_FB_BLEND_CONFIG_BLEND_DISABLE	BIT(1)
+
+#endif /* _VS_PRIMARY_PLANE_REGS_H_ */
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 8adcf5c15d456..70b3b836e1c99 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -286,7 +286,7 @@ static void virtgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
 
 static const struct dma_buf_attach_ops virtgpu_dma_buf_attach_ops = {
 	.allow_peer2peer = true,
-	.move_notify = virtgpu_dma_buf_move_notify
+	.invalidate_mappings = virtgpu_dma_buf_move_notify
 };
 
 struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev,
@@ -310,7 +310,7 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev,
 		}
 	}
 
-	if (!vgdev->has_resource_blob || vgdev->has_virgl_3d)
+	if (!vgdev->has_resource_blob)
 		return drm_gem_prime_import(dev, buf);
 
 	bo = kzalloc_obj(*bo);
diff --git a/drivers/gpu/drm/vkms/vkms_colorop.c b/drivers/gpu/drm/vkms/vkms_colorop.c
index bf8a0e4fc719e..071f3a8d2e7c4 100644
--- a/drivers/gpu/drm/vkms/vkms_colorop.c
+++ b/drivers/gpu/drm/vkms/vkms_colorop.c
@@ -12,6 +12,10 @@ static const u64 supported_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_INV_EOTF);
 
+static const struct drm_colorop_funcs vkms_colorop_funcs = {
+	.destroy = drm_colorop_destroy,
+};
+
 #define MAX_COLOR_PIPELINE_OPS 4
 
 static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_prop_enum_list *list)
@@ -31,7 +35,8 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, supported_tfs,
+	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &vkms_colorop_funcs,
+					      supported_tfs,
 					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
@@ -48,7 +53,8 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane, DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane, &vkms_colorop_funcs,
+					     DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -64,7 +70,8 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane, DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane, &vkms_colorop_funcs,
+					     DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -80,7 +87,8 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, supported_tfs,
+	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &vkms_colorop_funcs,
+					      supported_tfs,
 					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
diff --git a/drivers/gpu/drm/vkms/vkms_composer.c b/drivers/gpu/drm/vkms/vkms_composer.c
index cd85de4ffd03d..83d217085ad09 100644
--- a/drivers/gpu/drm/vkms/vkms_composer.c
+++ b/drivers/gpu/drm/vkms/vkms_composer.c
@@ -475,8 +475,14 @@ static void blend(struct vkms_writeback_job *wb,
 {
 	struct vkms_plane_state **plane = crtc_state->active_planes;
 	u32 n_active_planes = crtc_state->num_active_planes;
-
-	const struct pixel_argb_u16 background_color = { .a = 0xffff };
+	u64 bgcolor = crtc_state->base.background_color;
+
+	const struct pixel_argb_u16 background_color = {
+		.a = 0xffff,
+		.r = DRM_ARGB64_GETR(bgcolor),
+		.g = DRM_ARGB64_GETG(bgcolor),
+		.b = DRM_ARGB64_GETB(bgcolor),
+	};
 
 	int crtc_y_limit = crtc_state->base.mode.vdisplay;
 	int crtc_x_limit = crtc_state->base.mode.hdisplay;
diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index ba2ff353e1a9a..35ddc553a5e66 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -4,6 +4,7 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_blend.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -227,6 +228,8 @@ struct vkms_output *vkms_crtc_init(struct drm_device *dev, struct drm_plane *pri
 
 	drm_crtc_enable_color_mgmt(crtc, 0, false, VKMS_LUT_SIZE);
 
+	drm_crtc_attach_background_color_property(crtc);
+
 	spin_lock_init(&vkms_out->lock);
 	spin_lock_init(&vkms_out->composer_lock);
 
diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c
index 434c295f44ba6..95020765c4c29 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.c
+++ b/drivers/gpu/drm/vkms/vkms_drv.c
@@ -259,7 +259,6 @@ void vkms_destroy(struct vkms_config *config)
 
 	fdev = config->dev->faux_dev;
 
-	drm_colorop_pipeline_destroy(&config->dev->drm);
 	drm_dev_unregister(&config->dev->drm);
 	drm_atomic_helper_shutdown(&config->dev->drm);
 	devres_release_group(&fdev->dev, NULL);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index f2abaf1bda6a4..57465f69c6870 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -96,12 +96,17 @@ struct vmwgfx_hash_item {
 
 struct vmw_res_func;
 
+struct vmw_bo;
+struct vmw_bo;
+struct vmw_resource_dirty;
+
 /**
- * struct vmw-resource - base class for hardware resources
+ * struct vmw_resource - base class for hardware resources
  *
  * @kref: For refcounting.
  * @dev_priv: Pointer to the device private for this resource. Immutable.
  * @id: Device id. Protected by @dev_priv::resource_lock.
+ * @used_prio: Priority for this resource.
  * @guest_memory_size: Guest memory buffer size. Immutable.
  * @res_dirty: Resource contains data not yet in the guest memory buffer.
  * Protected by resource reserved.
@@ -117,18 +122,16 @@ struct vmw_res_func;
  * pin-count greater than zero. It is not on the resource LRU lists and its
  * guest memory buffer is pinned. Hence it can't be evicted.
  * @func: Method vtable for this resource. Immutable.
- * @mob_node; Node for the MOB guest memory rbtree. Protected by
+ * @mob_node: Node for the MOB guest memory rbtree. Protected by
  * @guest_memory_bo reserved.
  * @lru_head: List head for the LRU list. Protected by @dev_priv::resource_lock.
  * @binding_head: List head for the context binding list. Protected by
  * the @dev_priv::binding_mutex
+ * @dirty: resource's dirty tracker
  * @res_free: The resource destructor.
  * @hw_destroy: Callback to destroy the resource on the device, as part of
  * resource destruction.
  */
-struct vmw_bo;
-struct vmw_bo;
-struct vmw_resource_dirty;
 struct vmw_resource {
 	struct kref kref;
 	struct vmw_private *dev_priv;
@@ -196,8 +199,8 @@ struct vmw_surface_offset;
  * @quality_level: Quality level.
  * @autogen_filter: Filter for automatically generated mipmaps.
  * @array_size: Number of array elements for a 1D/2D texture. For cubemap
-                texture number of faces * array_size. This should be 0 for pre
-		SM4 device.
+ *              texture number of faces * array_size. This should be 0 for pre
+ *		SM4 device.
  * @buffer_byte_stride: Buffer byte stride.
  * @num_sizes: Size of @sizes. For GB surface this should always be 1.
  * @base_size: Surface dimension.
@@ -265,18 +268,24 @@ struct vmw_fifo_state {
 struct vmw_res_cache_entry {
 	uint32_t handle;
 	struct vmw_resource *res;
+	/* private: */
 	void *private;
+	/* public: */
 	unsigned short valid_handle;
 	unsigned short valid;
 };
 
 /**
  * enum vmw_dma_map_mode - indicate how to perform TTM page dma mappings.
+ * @vmw_dma_alloc_coherent: Use TTM coherent pages
+ * @vmw_dma_map_populate: Unmap from DMA just after unpopulate
+ * @vmw_dma_map_bind: Unmap from DMA just before unbind
  */
 enum vmw_dma_map_mode {
-	vmw_dma_alloc_coherent, /* Use TTM coherent pages */
-	vmw_dma_map_populate,   /* Unmap from DMA just after unpopulate */
-	vmw_dma_map_bind,       /* Unmap from DMA just before unbind */
+	vmw_dma_alloc_coherent,
+	vmw_dma_map_populate,
+	vmw_dma_map_bind,
+	/* private: */
 	vmw_dma_map_max
 };
 
@@ -284,8 +293,11 @@ enum vmw_dma_map_mode {
  * struct vmw_sg_table - Scatter/gather table for binding, with additional
  * device-specific information.
  *
+ * @mode: which page mapping mode to use
+ * @pages: Array of page pointers to the pages.
+ * @addrs: DMA addresses to the pages if coherent pages are used.
  * @sgt: Pointer to a struct sg_table with binding information
- * @num_regions: Number of regions with device-address contiguous pages
+ * @num_pages: Number of @pages
  */
 struct vmw_sg_table {
 	enum vmw_dma_map_mode mode;
@@ -353,6 +365,7 @@ struct vmw_ctx_validation_info;
  * than from user-space
  * @fp: If @kernel is false, points to the file of the client. Otherwise
  * NULL
+ * @filp: DRM state for this file
  * @cmd_bounce: Command bounce buffer used for command validation before
  * copying to fifo space
  * @cmd_bounce_size: Current command bounce buffer size
@@ -729,7 +742,7 @@ extern void vmw_svga_disable(struct vmw_private *dev_priv);
 bool vmwgfx_supported(struct vmw_private *vmw);
 
 
-/**
+/*
  * GMR utilities - vmwgfx_gmr.c
  */
 
@@ -739,7 +752,7 @@ extern int vmw_gmr_bind(struct vmw_private *dev_priv,
 			int gmr_id);
 extern void vmw_gmr_unbind(struct vmw_private *dev_priv, int gmr_id);
 
-/**
+/*
  * User handles
  */
 struct vmw_user_object {
@@ -759,7 +772,7 @@ void *vmw_user_object_map_size(struct vmw_user_object *uo, size_t size);
 void vmw_user_object_unmap(struct vmw_user_object *uo);
 bool vmw_user_object_is_mapped(struct vmw_user_object *uo);
 
-/**
+/*
  * Resource utilities - vmwgfx_resource.c
  */
 struct vmw_user_resource_conv;
@@ -819,7 +832,7 @@ static inline bool vmw_resource_mob_attached(const struct vmw_resource *res)
 	return !RB_EMPTY_NODE(&res->mob_node);
 }
 
-/**
+/*
  * GEM related functionality - vmwgfx_gem.c
  */
 struct vmw_bo_params;
@@ -833,7 +846,7 @@ extern int vmw_gem_object_create_ioctl(struct drm_device *dev, void *data,
 				       struct drm_file *filp);
 extern void vmw_debugfs_gem_init(struct vmw_private *vdev);
 
-/**
+/*
  * Misc Ioctl functionality - vmwgfx_ioctl.c
  */
 
@@ -846,7 +859,7 @@ extern int vmw_present_ioctl(struct drm_device *dev, void *data,
 extern int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv);
 
-/**
+/*
  * Fifo utilities - vmwgfx_fifo.c
  */
 
@@ -880,9 +893,11 @@ extern int vmw_cmd_flush(struct vmw_private *dev_priv,
 
 
 /**
- * vmw_fifo_caps - Returns the capabilities of the FIFO command
+ * vmw_fifo_caps - Get the capabilities of the FIFO command
  * queue or 0 if fifo memory isn't present.
  * @dev_priv: The device private context
+ *
+ * Returns: capabilities of the FIFO command or %0 if fifo memory not present
  */
 static inline uint32_t vmw_fifo_caps(const struct vmw_private *dev_priv)
 {
@@ -893,9 +908,11 @@ static inline uint32_t vmw_fifo_caps(const struct vmw_private *dev_priv)
 
 
 /**
- * vmw_is_cursor_bypass3_enabled - Returns TRUE iff Cursor Bypass 3
- * is enabled in the FIFO.
+ * vmw_is_cursor_bypass3_enabled - check Cursor Bypass 3 enabled setting
+ * in the FIFO.
  * @dev_priv: The device private context
+ *
+ * Returns: %true iff Cursor Bypass 3 is enabled in the FIFO
  */
 static inline bool
 vmw_is_cursor_bypass3_enabled(const struct vmw_private *dev_priv)
@@ -903,7 +920,7 @@ vmw_is_cursor_bypass3_enabled(const struct vmw_private *dev_priv)
 	return (vmw_fifo_caps(dev_priv) & SVGA_FIFO_CAP_CURSOR_BYPASS_3) != 0;
 }
 
-/**
+/*
  * TTM buffer object driver - vmwgfx_ttm_buffer.c
  */
 
@@ -927,7 +944,7 @@ extern void vmw_piter_start(struct vmw_piter *viter,
  *
  * @viter: Pointer to the iterator to advance.
  *
- * Returns false if past the list of pages, true otherwise.
+ * Returns: false if past the list of pages, true otherwise.
  */
 static inline bool vmw_piter_next(struct vmw_piter *viter)
 {
@@ -939,7 +956,7 @@ static inline bool vmw_piter_next(struct vmw_piter *viter)
  *
  * @viter: Pointer to the iterator
  *
- * Returns the DMA address of the page pointed to by @viter.
+ * Returns: the DMA address of the page pointed to by @viter.
  */
 static inline dma_addr_t vmw_piter_dma_addr(struct vmw_piter *viter)
 {
@@ -951,14 +968,14 @@ static inline dma_addr_t vmw_piter_dma_addr(struct vmw_piter *viter)
  *
  * @viter: Pointer to the iterator
  *
- * Returns the DMA address of the page pointed to by @viter.
+ * Returns: the DMA address of the page pointed to by @viter.
  */
 static inline struct page *vmw_piter_page(struct vmw_piter *viter)
 {
 	return viter->pages[viter->i];
 }
 
-/**
+/*
  * Command submission - vmwgfx_execbuf.c
  */
 
@@ -993,7 +1010,7 @@ extern int vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 					int32_t out_fence_fd);
 bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd);
 
-/**
+/*
  * IRQs and wating - vmwgfx_irq.c
  */
 
@@ -1016,7 +1033,7 @@ bool vmw_generic_waiter_add(struct vmw_private *dev_priv, u32 flag,
 bool vmw_generic_waiter_remove(struct vmw_private *dev_priv,
 			       u32 flag, int *waiter_count);
 
-/**
+/*
  * Kernel modesetting - vmwgfx_kms.c
  */
 
@@ -1048,7 +1065,7 @@ extern int vmw_resource_pin(struct vmw_resource *res, bool interruptible);
 extern void vmw_resource_unpin(struct vmw_resource *res);
 extern enum vmw_res_type vmw_res_type(const struct vmw_resource *res);
 
-/**
+/*
  * Overlay control - vmwgfx_overlay.c
  */
 
@@ -1063,20 +1080,20 @@ int vmw_overlay_unref(struct vmw_private *dev_priv, uint32_t stream_id);
 int vmw_overlay_num_overlays(struct vmw_private *dev_priv);
 int vmw_overlay_num_free_overlays(struct vmw_private *dev_priv);
 
-/**
+/*
  * GMR Id manager
  */
 
 int vmw_gmrid_man_init(struct vmw_private *dev_priv, int type);
 void vmw_gmrid_man_fini(struct vmw_private *dev_priv, int type);
 
-/**
+/*
  * System memory manager
  */
 int vmw_sys_man_init(struct vmw_private *dev_priv);
 void vmw_sys_man_fini(struct vmw_private *dev_priv);
 
-/**
+/*
  * Prime - vmwgfx_prime.c
  */
 
@@ -1292,7 +1309,7 @@ extern void vmw_cmdbuf_irqthread(struct vmw_cmdbuf_man *man);
  * @line: The current line of the blit.
  * @line_offset: Offset of the current line segment.
  * @cpp: Bytes per pixel (granularity information).
- * @memcpy: Which memcpy function to use.
+ * @do_cpy: Which memcpy function to use.
  */
 struct vmw_diff_cpy {
 	struct drm_rect rect;
@@ -1380,13 +1397,14 @@ vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf);
 
 /**
  * VMW_DEBUG_KMS - Debug output for kernel mode-setting
+ * @fmt: format string for the args
  *
  * This macro is for debugging vmwgfx mode-setting code.
  */
 #define VMW_DEBUG_KMS(fmt, ...)                                               \
 	DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__)
 
-/**
+/*
  * Inline helper functions
  */
 
@@ -1417,11 +1435,13 @@ static inline void vmw_fifo_resource_dec(struct vmw_private *dev_priv)
 
 /**
  * vmw_fifo_mem_read - Perform a MMIO read from the fifo memory
- *
+ * @vmw: The device private structure
  * @fifo_reg: The fifo register to read from
  *
  * This function is intended to be equivalent to ioread32() on
  * memremap'd memory, but without byteswapping.
+ *
+ * Returns: the value read
  */
 static inline u32 vmw_fifo_mem_read(struct vmw_private *vmw, uint32 fifo_reg)
 {
@@ -1431,8 +1451,9 @@ static inline u32 vmw_fifo_mem_read(struct vmw_private *vmw, uint32 fifo_reg)
 
 /**
  * vmw_fifo_mem_write - Perform a MMIO write to volatile memory
- *
- * @addr: The fifo register to write to
+ * @vmw: The device private structure
+ * @fifo_reg: The fifo register to write to
+ * @value: The value to write
  *
  * This function is intended to be equivalent to iowrite32 on
  * memremap'd memory, but without byteswapping.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index 3469e2c9e706c..4ef84ff9b638c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -47,7 +47,8 @@ struct vmw_event_fence_action {
 static struct vmw_fence_manager *
 fman_from_fence(struct vmw_fence_obj *fence)
 {
-	return container_of(fence->base.lock, struct vmw_fence_manager, lock);
+	return container_of(fence->base.extern_lock, struct vmw_fence_manager,
+			    lock);
 }
 
 static void vmw_fence_obj_destroy(struct dma_fence *f)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 55730e29d3ae9..e7bddf840a795 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -771,7 +771,8 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 		ret = vmw_bo_dirty_add(bo);
 		if (!ret && surface && surface->res.func->dirty_alloc) {
 			surface->res.coherent = true;
-			ret = surface->res.func->dirty_alloc(&surface->res);
+			if (surface->res.dirty == NULL)
+				ret = surface->res.func->dirty_alloc(&surface->res);
 		}
 		ttm_bo_unreserve(&bo->tbo);
 	}
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index b39cbb7562326..dab979287a96e 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -41,6 +41,7 @@ xe-y += xe_bb.o \
 	xe_device_sysfs.o \
 	xe_dma_buf.o \
 	xe_drm_client.o \
+	xe_drm_ras.o \
 	xe_eu_stall.o \
 	xe_exec.o \
 	xe_exec_queue.o \
@@ -74,6 +75,7 @@ xe-y += xe_bb.o \
 	xe_guc_log.o \
 	xe_guc_pagefault.o \
 	xe_guc_pc.o \
+	xe_guc_rc.o \
 	xe_guc_submit.o \
 	xe_guc_tlb_inval.o \
 	xe_heci_gsc.o \
@@ -209,14 +211,15 @@ $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE
 
 # Display code specific to xe
 xe-$(CONFIG_DRM_XE_DISPLAY) += \
-	display/intel_bo.o \
-	display/intel_fb_bo.o \
 	display/intel_fbdev_fb.o \
 	display/xe_display.o \
+	display/xe_display_bo.o \
+	display/xe_display_pcode.o \
 	display/xe_display_rpm.o \
 	display/xe_display_wa.o \
 	display/xe_dsb_buffer.o \
 	display/xe_fb_pin.o \
+	display/xe_frontbuffer.o \
 	display/xe_hdcp_gsc.o \
 	display/xe_initial_plane.o \
 	display/xe_panic.o \
@@ -231,6 +234,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_audio.o \
 	i915-display/intel_backlight.o \
 	i915-display/intel_bios.o \
+	i915-display/intel_bo.o \
 	i915-display/intel_bw.o \
 	i915-display/intel_casf.o \
 	i915-display/intel_cdclk.o \
@@ -273,7 +277,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_dp_test.o \
 	i915-display/intel_dpll.o \
 	i915-display/intel_dpll_mgr.o \
-	i915-display/intel_dpt_common.o \
+	i915-display/intel_dpt.o \
 	i915-display/intel_dram.o \
 	i915-display/intel_drrs.o \
 	i915-display/intel_dsb.o \
@@ -302,6 +306,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_modeset_lock.o \
 	i915-display/intel_modeset_setup.o \
 	i915-display/intel_modeset_verify.o \
+	i915-display/intel_overlay.o \
 	i915-display/intel_panel.o \
 	i915-display/intel_parent.o \
 	i915-display/intel_pch.o \
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_reg.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_reg.h
deleted file mode 100644
index 8619ec015ad43..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_reg.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/i915_reg.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_vma.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_vma.h
deleted file mode 100644
index c4b5adaaa99a3..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_vma.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef I915_VMA_H
-#define I915_VMA_H
-
-#include <uapi/drm/i915_drm.h>
-
-#include "xe_ggtt.h"
-
-#include <linux/refcount.h>
-
-/* We don't want these from i915_drm.h in case of Xe */
-#undef I915_TILING_X
-#undef I915_TILING_Y
-#define I915_TILING_X 0
-#define I915_TILING_Y 0
-
-struct xe_bo;
-
-struct i915_vma {
-	refcount_t ref;
-	struct xe_bo *bo, *dpt;
-	struct xe_ggtt_node *node;
-};
-
-#define i915_vma_fence_id(vma) -1
-
-static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
-{
-	return xe_ggtt_node_addr(vma->node);
-}
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h
deleted file mode 100644
index 4fcd3bf6b76f8..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_PCODE_H__
-#define __INTEL_PCODE_H__
-
-#include "xe_pcode.h"
-
-#endif /* __INTEL_PCODE_H__ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
index 2cf13a572ab0e..0eabe2866f5fd 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
@@ -6,9 +6,8 @@
 #ifndef __INTEL_STEP_H__
 #define __INTEL_STEP_H__
 
-#include "xe_step.h"
+#include "xe_step_types.h"
 
 #define intel_step xe_step
-#define intel_step_name xe_step_name
 
 #endif /* __INTEL_STEP_H__ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
index c05d4c4292d3c..a8cfd65119e09 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
@@ -6,6 +6,7 @@
 #ifndef __INTEL_UNCORE_H__
 #define __INTEL_UNCORE_H__
 
+#include "i915_reg_defs.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_mmio.h"
@@ -38,6 +39,14 @@ static inline u8 intel_uncore_read8(struct intel_uncore *uncore,
 	return xe_mmio_read8(__compat_uncore_to_mmio(uncore), reg);
 }
 
+static inline void intel_uncore_write8(struct intel_uncore *uncore,
+				       i915_reg_t i915_reg, u8 val)
+{
+	struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg));
+
+	xe_mmio_write8(__compat_uncore_to_mmio(uncore), reg, val);
+}
+
 static inline u16 intel_uncore_read16(struct intel_uncore *uncore,
 				      i915_reg_t i915_reg)
 {
diff --git a/drivers/gpu/drm/xe/display/intel_bo.c b/drivers/gpu/drm/xe/display/intel_bo.c
deleted file mode 100644
index 05d5e5c0a0de0..0000000000000
--- a/drivers/gpu/drm/xe/display/intel_bo.c
+++ /dev/null
@@ -1,109 +0,0 @@
-// SPDX-License-Identifier: MIT
-/* Copyright © 2024 Intel Corporation */
-
-#include <drm/drm_gem.h>
-
-#include "intel_bo.h"
-#include "intel_frontbuffer.h"
-#include "xe_bo.h"
-#include "xe_pxp.h"
-
-bool intel_bo_is_tiled(struct drm_gem_object *obj)
-{
-	/* legacy tiling is unused */
-	return false;
-}
-
-bool intel_bo_is_userptr(struct drm_gem_object *obj)
-{
-	/* xe does not have userptr bos */
-	return false;
-}
-
-bool intel_bo_is_shmem(struct drm_gem_object *obj)
-{
-	return false;
-}
-
-bool intel_bo_is_protected(struct drm_gem_object *obj)
-{
-	return xe_bo_is_protected(gem_to_xe_bo(obj));
-}
-
-int intel_bo_key_check(struct drm_gem_object *obj)
-{
-	return xe_pxp_obj_key_check(obj);
-}
-
-int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
-{
-	return drm_gem_prime_mmap(obj, vma);
-}
-
-int intel_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size)
-{
-	struct xe_bo *bo = gem_to_xe_bo(obj);
-
-	return xe_bo_read(bo, offset, dst, size);
-}
-
-struct xe_frontbuffer {
-	struct intel_frontbuffer base;
-	struct drm_gem_object *obj;
-	struct kref ref;
-};
-
-struct intel_frontbuffer *intel_bo_frontbuffer_get(struct drm_gem_object *obj)
-{
-	struct xe_frontbuffer *front;
-
-	front = kmalloc_obj(*front);
-	if (!front)
-		return NULL;
-
-	intel_frontbuffer_init(&front->base, obj->dev);
-
-	kref_init(&front->ref);
-
-	drm_gem_object_get(obj);
-	front->obj = obj;
-
-	return &front->base;
-}
-
-void intel_bo_frontbuffer_ref(struct intel_frontbuffer *_front)
-{
-	struct xe_frontbuffer *front =
-		container_of(_front, typeof(*front), base);
-
-	kref_get(&front->ref);
-}
-
-static void frontbuffer_release(struct kref *ref)
-{
-	struct xe_frontbuffer *front =
-		container_of(ref, typeof(*front), ref);
-
-	intel_frontbuffer_fini(&front->base);
-
-	drm_gem_object_put(front->obj);
-
-	kfree(front);
-}
-
-void intel_bo_frontbuffer_put(struct intel_frontbuffer *_front)
-{
-	struct xe_frontbuffer *front =
-		container_of(_front, typeof(*front), base);
-
-	kref_put(&front->ref, frontbuffer_release);
-}
-
-void intel_bo_frontbuffer_flush_for_display(struct intel_frontbuffer *front)
-{
-}
-
-void intel_bo_describe(struct seq_file *m, struct drm_gem_object *obj)
-{
-	/* FIXME */
-}
diff --git a/drivers/gpu/drm/xe/display/intel_fb_bo.c b/drivers/gpu/drm/xe/display/intel_fb_bo.c
deleted file mode 100644
index db8b1a27b4de8..0000000000000
--- a/drivers/gpu/drm/xe/display/intel_fb_bo.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2021 Intel Corporation
- */
-
-#include <drm/drm_modeset_helper.h>
-#include <drm/ttm/ttm_bo.h>
-
-#include "intel_display_types.h"
-#include "intel_fb.h"
-#include "intel_fb_bo.h"
-#include "xe_bo.h"
-
-void intel_fb_bo_framebuffer_fini(struct drm_gem_object *obj)
-{
-	struct xe_bo *bo = gem_to_xe_bo(obj);
-
-	if (bo->flags & XE_BO_FLAG_PINNED) {
-		/* Unpin our kernel fb first */
-		xe_bo_lock(bo, false);
-		xe_bo_unpin(bo);
-		xe_bo_unlock(bo);
-	}
-	xe_bo_put(bo);
-}
-
-int intel_fb_bo_framebuffer_init(struct drm_gem_object *obj,
-				 struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	struct xe_bo *bo = gem_to_xe_bo(obj);
-	struct xe_device *xe = to_xe_device(bo->ttm.base.dev);
-	int ret;
-
-	/*
-	 * Some modifiers require physical alignment of 64KiB VRAM pages;
-	 * require that the BO in those cases is created correctly.
-	 */
-	if (XE_IOCTL_DBG(xe, intel_fb_needs_64k_phys(mode_cmd->modifier[0]) &&
-			     !(bo->flags & XE_BO_FLAG_NEEDS_64K)))
-		return -EINVAL;
-
-	xe_bo_get(bo);
-
-	ret = ttm_bo_reserve(&bo->ttm, true, false, NULL);
-	if (ret)
-		goto err;
-
-	if (!(bo->flags & XE_BO_FLAG_SCANOUT)) {
-		/*
-		 * XE_BO_FLAG_SCANOUT should ideally be set at creation, or is
-		 * automatically set when creating FB. We cannot change caching
-		 * mode when the bo is VM_BINDed, so we can only set
-		 * coherency with display when unbound.
-		 */
-		if (XE_IOCTL_DBG(xe, xe_bo_is_vm_bound(bo))) {
-			ttm_bo_unreserve(&bo->ttm);
-			ret = -EINVAL;
-			goto err;
-		}
-		bo->flags |= XE_BO_FLAG_SCANOUT;
-	}
-	ttm_bo_unreserve(&bo->ttm);
-	return 0;
-
-err:
-	xe_bo_put(bo);
-	return ret;
-}
-
-struct drm_gem_object *intel_fb_bo_lookup_valid_bo(struct drm_device *drm,
-						   struct drm_file *filp,
-						   const struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct xe_bo *bo;
-	struct drm_gem_object *gem = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
-
-	if (!gem)
-		return ERR_PTR(-ENOENT);
-
-	bo = gem_to_xe_bo(gem);
-	/* Require vram placement or dma-buf import */
-	if (IS_DGFX(xe) &&
-	    !xe_bo_can_migrate(bo, XE_PL_VRAM0) &&
-	    bo->ttm.type != ttm_bo_type_sg) {
-		drm_gem_object_put(gem);
-		return ERR_PTR(-EREMOTE);
-	}
-
-	return gem;
-}
diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
index 7ad76022cb14f..87af5646c938a 100644
--- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
+++ b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
@@ -23,6 +23,29 @@ u32 intel_fbdev_fb_pitch_align(u32 stride)
 	return ALIGN(stride, XE_PAGE_SIZE);
 }
 
+bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct ttm_resource_manager *stolen;
+
+	stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+	if (!stolen)
+		return false;
+
+	if (IS_DGFX(xe))
+		return false;
+
+	if (XE_DEVICE_WA(xe, 22019338487_display))
+		return false;
+
+	/*
+	 * If the FB is too big, just don't use it since fbdev is not very
+	 * important and we should probably use that space with FBC or other
+	 * features.
+	 */
+	return stolen->size >= size * 2;
+}
+
 struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size)
 {
 	struct xe_device *xe = to_xe_device(drm);
@@ -30,7 +53,7 @@ struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size
 
 	obj = ERR_PTR(-ENODEV);
 
-	if (!IS_DGFX(xe) && !XE_DEVICE_WA(xe, 22019338487_display)) {
+	if (intel_fbdev_fb_prefer_stolen(drm, size)) {
 		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
 						size,
 						ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
@@ -40,6 +63,8 @@ struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size
 			drm_info(&xe->drm, "Allocated fbdev into stolen\n");
 		else
 			drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj));
+	} else {
+		drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n");
 	}
 
 	if (IS_ERR(obj)) {
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index f8a831b5dc7d5..49b6f98e73912 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -35,7 +35,11 @@
 #include "intel_hotplug.h"
 #include "intel_opregion.h"
 #include "skl_watermark.h"
+#include "xe_display_bo.h"
+#include "xe_display_pcode.h"
 #include "xe_display_rpm.h"
+#include "xe_dsb_buffer.h"
+#include "xe_frontbuffer.h"
 #include "xe_hdcp_gsc.h"
 #include "xe_initial_plane.h"
 #include "xe_module.h"
@@ -538,10 +542,14 @@ static const struct intel_display_irq_interface xe_display_irq_interface = {
 };
 
 static const struct intel_display_parent_interface parent = {
+	.bo = &xe_display_bo_interface,
+	.dsb = &xe_display_dsb_interface,
+	.frontbuffer = &xe_display_frontbuffer_interface,
 	.hdcp = &xe_display_hdcp_interface,
 	.initial_plane = &xe_display_initial_plane_interface,
 	.irq = &xe_display_irq_interface,
 	.panic = &xe_display_panic_interface,
+	.pcode = &xe_display_pcode_interface,
 	.rpm = &xe_display_rpm_interface,
 	.stolen = &xe_display_stolen_interface,
 };
diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c
new file mode 100644
index 0000000000000..a689f71e7b145
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_bo.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2024 Intel Corporation */
+
+#include <drm/drm_gem.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "intel_fb.h"
+#include "xe_bo.h"
+#include "xe_display_bo.h"
+#include "xe_pxp.h"
+
+static bool xe_display_bo_is_protected(struct drm_gem_object *obj)
+{
+	return xe_bo_is_protected(gem_to_xe_bo(obj));
+}
+
+static int xe_display_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size)
+{
+	struct xe_bo *bo = gem_to_xe_bo(obj);
+
+	return xe_bo_read(bo, offset, dst, size);
+}
+
+static int xe_display_bo_framebuffer_init(struct drm_gem_object *obj,
+					  struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct xe_bo *bo = gem_to_xe_bo(obj);
+	struct xe_device *xe = to_xe_device(bo->ttm.base.dev);
+	int ret;
+
+	/*
+	 * Some modifiers require physical alignment of 64KiB VRAM pages;
+	 * require that the BO in those cases is created correctly.
+	 */
+	if (XE_IOCTL_DBG(xe, intel_fb_needs_64k_phys(mode_cmd->modifier[0]) &&
+			     !(bo->flags & XE_BO_FLAG_NEEDS_64K)))
+		return -EINVAL;
+
+	xe_bo_get(bo);
+
+	ret = ttm_bo_reserve(&bo->ttm, true, false, NULL);
+	if (ret)
+		goto err;
+
+	if (!(bo->flags & XE_BO_FLAG_SCANOUT)) {
+		/*
+		 * XE_BO_FLAG_SCANOUT should ideally be set at creation, or is
+		 * automatically set when creating FB. We cannot change caching
+		 * mode when the bo is VM_BINDed, so we can only set
+		 * coherency with display when unbound.
+		 */
+		if (XE_IOCTL_DBG(xe, xe_bo_is_vm_bound(bo))) {
+			ttm_bo_unreserve(&bo->ttm);
+			ret = -EINVAL;
+			goto err;
+		}
+		bo->flags |= XE_BO_FLAG_SCANOUT;
+	}
+	ttm_bo_unreserve(&bo->ttm);
+	return 0;
+
+err:
+	xe_bo_put(bo);
+	return ret;
+}
+
+static void xe_display_bo_framebuffer_fini(struct drm_gem_object *obj)
+{
+	struct xe_bo *bo = gem_to_xe_bo(obj);
+
+	if (bo->flags & XE_BO_FLAG_PINNED) {
+		/* Unpin our kernel fb first */
+		xe_bo_lock(bo, false);
+		xe_bo_unpin(bo);
+		xe_bo_unlock(bo);
+	}
+	xe_bo_put(bo);
+}
+
+static struct drm_gem_object *
+xe_display_bo_framebuffer_lookup(struct drm_device *drm,
+				 struct drm_file *filp,
+				 const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct xe_bo *bo;
+	struct drm_gem_object *gem = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
+
+	if (!gem)
+		return ERR_PTR(-ENOENT);
+
+	bo = gem_to_xe_bo(gem);
+	/* Require vram placement or dma-buf import */
+	if (IS_DGFX(xe) &&
+	    !xe_bo_can_migrate(bo, XE_PL_VRAM0) &&
+	    bo->ttm.type != ttm_bo_type_sg) {
+		drm_gem_object_put(gem);
+		return ERR_PTR(-EREMOTE);
+	}
+
+	return gem;
+}
+
+const struct intel_display_bo_interface xe_display_bo_interface = {
+	.is_protected = xe_display_bo_is_protected,
+	.key_check = xe_pxp_obj_key_check,
+	.fb_mmap = drm_gem_prime_mmap,
+	.read_from_page = xe_display_bo_read_from_page,
+	.framebuffer_init = xe_display_bo_framebuffer_init,
+	.framebuffer_fini = xe_display_bo_framebuffer_fini,
+	.framebuffer_lookup = xe_display_bo_framebuffer_lookup,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.h b/drivers/gpu/drm/xe/display/xe_display_bo.h
new file mode 100644
index 0000000000000..6879c104b0b1f
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_bo.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __XE_DISPLAY_BO_H__
+#define __XE_DISPLAY_BO_H__
+
+extern const struct intel_display_bo_interface xe_display_bo_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_display_pcode.c b/drivers/gpu/drm/xe/display/xe_display_pcode.c
new file mode 100644
index 0000000000000..f6820ef7e666d
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_pcode.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2026 Intel Corporation */
+
+#include <drm/intel/display_parent_interface.h>
+
+#include "xe_device.h"
+#include "xe_pcode.h"
+
+static int xe_display_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+
+	return xe_pcode_read(tile, mbox, val, val1);
+}
+
+static int xe_display_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+
+	return xe_pcode_write_timeout(tile, mbox, val, timeout_ms);
+}
+
+static int xe_display_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
+				    u32 reply_mask, u32 reply, int timeout_base_ms)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+
+	return xe_pcode_request(tile, mbox, request, reply_mask, reply, timeout_base_ms);
+}
+
+const struct intel_display_pcode_interface xe_display_pcode_interface = {
+	.read = xe_display_pcode_read,
+	.write = xe_display_pcode_write_timeout,
+	.request = xe_display_pcode_request,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_display_pcode.h b/drivers/gpu/drm/xe/display/xe_display_pcode.h
new file mode 100644
index 0000000000000..58bd2fb7fb791
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_pcode.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __XE_DISPLAY_PCODE_H__
+#define __XE_DISPLAY_PCODE_H__
+
+extern const struct intel_display_pcode_interface xe_display_pcode_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_display_vma.h b/drivers/gpu/drm/xe/display/xe_display_vma.h
new file mode 100644
index 0000000000000..28267be61ae02
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_vma.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _XE_DISPLAY_VMA_H_
+#define _XE_DISPLAY_VMA_H_
+
+#include <linux/refcount.h>
+
+struct xe_bo;
+struct xe_ggtt_node;
+
+struct i915_vma {
+	refcount_t ref;
+	struct xe_bo *bo, *dpt;
+	struct xe_ggtt_node *node;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
index 8ffc13855ef7a..1c67a950c6ad5 100644
--- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
+++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
@@ -3,10 +3,12 @@
  * Copyright 2023, Intel Corporation.
  */
 
-#include "intel_dsb_buffer.h"
+#include <drm/intel/display_parent_interface.h>
+
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
+#include "xe_dsb_buffer.h"
 
 struct intel_dsb_buffer {
 	u32 *cmd_buf;
@@ -14,29 +16,29 @@ struct intel_dsb_buffer {
 	size_t buf_size;
 };
 
-u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
+static u32 xe_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
 {
 	return xe_bo_ggtt_addr(dsb_buf->bo);
 }
 
-void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
+static void xe_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
 {
 	iosys_map_wr(&dsb_buf->bo->vmap, idx * 4, u32, val);
 }
 
-u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
+static u32 xe_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
 {
 	return iosys_map_rd(&dsb_buf->bo->vmap, idx * 4, u32);
 }
 
-void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
+static void xe_dsb_buffer_fill(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
 {
 	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
 
 	iosys_map_memset(&dsb_buf->bo->vmap, idx * 4, val, size);
 }
 
-struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size)
+static struct intel_dsb_buffer *xe_dsb_buffer_create(struct drm_device *drm, size_t size)
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct intel_dsb_buffer *dsb_buf;
@@ -69,13 +71,13 @@ struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t
 	return ERR_PTR(ret);
 }
 
-void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
+static void xe_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
 {
 	xe_bo_unpin_map_no_vm(dsb_buf->bo);
 	kfree(dsb_buf);
 }
 
-void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
+static void xe_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
 {
 	struct xe_device *xe = dsb_buf->bo->tile->xe;
 
@@ -86,3 +88,13 @@ void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
 	xe_device_wmb(xe);
 	xe_device_l2_flush(xe);
 }
+
+const struct intel_display_dsb_interface xe_display_dsb_interface = {
+	.ggtt_offset = xe_dsb_buffer_ggtt_offset,
+	.write = xe_dsb_buffer_write,
+	.read = xe_dsb_buffer_read,
+	.fill = xe_dsb_buffer_fill,
+	.create = xe_dsb_buffer_create,
+	.cleanup = xe_dsb_buffer_cleanup,
+	.flush_map = xe_dsb_buffer_flush_map,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.h b/drivers/gpu/drm/xe/display/xe_dsb_buffer.h
new file mode 100644
index 0000000000000..2e47721870166
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __XE_DSB_BUFFER_H__
+#define __XE_DSB_BUFFER_H__
+
+extern const struct intel_display_dsb_interface xe_display_dsb_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index d2c4e94180fa3..dbbc61032b7fd 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -5,15 +5,14 @@
 
 #include <drm/ttm/ttm_bo.h>
 
-#include "i915_vma.h"
 #include "intel_display_core.h"
 #include "intel_display_types.h"
-#include "intel_dpt.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "intel_fbdev.h"
 #include "xe_bo.h"
 #include "xe_device.h"
+#include "xe_display_vma.h"
 #include "xe_ggtt.h"
 #include "xe_pm.h"
 #include "xe_vram_types.h"
@@ -256,7 +255,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 		size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE;
 
 	pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
-	vma->node = xe_ggtt_node_insert_transform(ggtt, bo, pte,
+	vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte,
 						  ALIGN(size, align), align,
 						  view->type == I915_GTT_VIEW_NORMAL ?
 						  NULL : write_ggtt_rotated_node,
@@ -352,8 +351,7 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma)
 
 	if (vma->dpt)
 		xe_bo_unpin_map_no_vm(vma->dpt);
-	else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) ||
-		 vma->bo->ggtt_node[tile_id] != vma->node)
+	else if (vma->bo->ggtt_node[tile_id] != vma->node)
 		xe_ggtt_node_remove(vma->node, false);
 
 	ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
@@ -410,7 +408,7 @@ static bool reuse_vma(struct intel_plane_state *new_plane_state,
 	refcount_inc(&vma->ref);
 	new_plane_state->ggtt_vma = vma;
 
-	new_plane_state->surf = i915_ggtt_offset(new_plane_state->ggtt_vma) +
+	new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) +
 		plane->surf_offset(new_plane_state);
 
 	return true;
@@ -440,7 +438,7 @@ int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
 
 	new_plane_state->ggtt_vma = vma;
 
-	new_plane_state->surf = i915_ggtt_offset(new_plane_state->ggtt_vma) +
+	new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) +
 		plane->surf_offset(new_plane_state);
 
 	return 0;
@@ -452,25 +450,6 @@ void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
 	old_plane_state->ggtt_vma = NULL;
 }
 
-/*
- * For Xe introduce dummy intel_dpt_create which just return NULL,
- * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;
- */
-struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
-{
-	return NULL;
-}
-
-void intel_dpt_destroy(struct i915_address_space *vm)
-{
-	return;
-}
-
-u64 intel_dpt_offset(struct i915_vma *dpt_vma)
-{
-	return 0;
-}
-
 void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
 {
 	*map = vma->bo->vmap;
diff --git a/drivers/gpu/drm/xe/display/xe_frontbuffer.c b/drivers/gpu/drm/xe/display/xe_frontbuffer.c
new file mode 100644
index 0000000000000..113fc017ee948
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_frontbuffer.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2026 Intel Corporation */
+
+#include <drm/drm_gem.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "intel_frontbuffer.h"
+#include "xe_frontbuffer.h"
+
+struct xe_frontbuffer {
+	struct intel_frontbuffer base;
+	struct drm_gem_object *obj;
+	struct kref ref;
+};
+
+static struct intel_frontbuffer *xe_frontbuffer_get(struct drm_gem_object *obj)
+{
+	struct xe_frontbuffer *front;
+
+	front = kmalloc_obj(*front);
+	if (!front)
+		return NULL;
+
+	intel_frontbuffer_init(&front->base, obj->dev);
+
+	kref_init(&front->ref);
+
+	drm_gem_object_get(obj);
+	front->obj = obj;
+
+	return &front->base;
+}
+
+static void xe_frontbuffer_ref(struct intel_frontbuffer *_front)
+{
+	struct xe_frontbuffer *front =
+		container_of(_front, typeof(*front), base);
+
+	kref_get(&front->ref);
+}
+
+static void frontbuffer_release(struct kref *ref)
+{
+	struct xe_frontbuffer *front =
+		container_of(ref, typeof(*front), ref);
+
+	intel_frontbuffer_fini(&front->base);
+
+	drm_gem_object_put(front->obj);
+
+	kfree(front);
+}
+
+static void xe_frontbuffer_put(struct intel_frontbuffer *_front)
+{
+	struct xe_frontbuffer *front =
+		container_of(_front, typeof(*front), base);
+
+	kref_put(&front->ref, frontbuffer_release);
+}
+
+static void xe_frontbuffer_flush_for_display(struct intel_frontbuffer *front)
+{
+}
+
+const struct intel_display_frontbuffer_interface xe_display_frontbuffer_interface = {
+	.get = xe_frontbuffer_get,
+	.ref = xe_frontbuffer_ref,
+	.put = xe_frontbuffer_put,
+	.flush_for_display = xe_frontbuffer_flush_for_display,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_frontbuffer.h b/drivers/gpu/drm/xe/display/xe_frontbuffer.h
new file mode 100644
index 0000000000000..6b4f59b42ade1
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_frontbuffer.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _XE_FRONTBUFFER_H_
+#define _XE_FRONTBUFFER_H_
+
+extern const struct intel_display_frontbuffer_interface xe_display_frontbuffer_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c
index 4cfeafcc158d5..65cc0b0c934b1 100644
--- a/drivers/gpu/drm/xe/display/xe_initial_plane.c
+++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c
@@ -3,26 +3,21 @@
  * Copyright © 2021 Intel Corporation
  */
 
-/* for ioread64 */
-#include <linux/io-64-nonatomic-lo-hi.h>
-
 #include <drm/intel/display_parent_interface.h>
 
 #include "regs/xe_gtt_defs.h"
-#include "xe_ggtt.h"
-#include "xe_mmio.h"
 
-#include "i915_vma.h"
 #include "intel_crtc.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
+#include "intel_fbdev_fb.h"
 #include "xe_bo.h"
+#include "xe_display_vma.h"
+#include "xe_ggtt.h"
+#include "xe_mmio.h"
 #include "xe_vram_types.h"
-#include "xe_wa.h"
-
-#include <generated/xe_device_wa_oob.h>
 
 /* Early xe has no irq */
 static void xe_initial_plane_vblank_wait(struct drm_crtc *_crtc)
@@ -90,17 +85,11 @@ initial_plane_bo(struct xe_device *xe,
 		phys_base = base;
 		flags |= XE_BO_FLAG_STOLEN;
 
-		if (XE_DEVICE_WA(xe, 22019338487_display))
-			return NULL;
-
-		/*
-		 * If the FB is too big, just don't use it since fbdev is not very
-		 * important and we should probably use that space with FBC or other
-		 * features.
-		 */
 		if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
-		    plane_config->size * 2 >> PAGE_SHIFT >= stolen->size)
+		    !intel_fbdev_fb_prefer_stolen(&xe->drm, plane_config->size)) {
+			drm_info(&xe->drm, "Initial FB size exceeds half of stolen, discarding\n");
 			return NULL;
+		}
 	}
 
 	size = round_up(plane_config->base + plane_config->size,
@@ -170,7 +159,7 @@ xe_initial_plane_setup(struct drm_plane_state *_plane_state,
 
 	plane_state->ggtt_vma = vma;
 
-	plane_state->surf = i915_ggtt_offset(plane_state->ggtt_vma);
+	plane_state->surf = xe_ggtt_node_addr(plane_state->ggtt_vma->node);
 
 	plane_config->vma = vma;
 
diff --git a/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
index 457881af8af97..4d854c85e5881 100644
--- a/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
+++ b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
@@ -55,6 +55,7 @@
 #define PIPELINE_SELECT				GFXPIPE_SINGLE_DW_CMD(0x1, 0x4)
 
 #define CMD_3DSTATE_DRAWING_RECTANGLE_FAST	GFXPIPE_3D_CMD(0x0, 0x0)
+#define CMD_3DSTATE_CUSTOM_SAMPLE_PATTERN	GFXPIPE_3D_CMD(0x0, 0x2)
 #define CMD_3DSTATE_CLEAR_PARAMS		GFXPIPE_3D_CMD(0x0, 0x4)
 #define CMD_3DSTATE_DEPTH_BUFFER		GFXPIPE_3D_CMD(0x0, 0x5)
 #define CMD_3DSTATE_STENCIL_BUFFER		GFXPIPE_3D_CMD(0x0, 0x6)
@@ -138,8 +139,16 @@
 #define CMD_3DSTATE_SBE_MESH			GFXPIPE_3D_CMD(0x0, 0x82)
 #define CMD_3DSTATE_CPSIZE_CONTROL_BUFFER	GFXPIPE_3D_CMD(0x0, 0x83)
 #define CMD_3DSTATE_COARSE_PIXEL		GFXPIPE_3D_CMD(0x0, 0x89)
+#define CMD_3DSTATE_MESH_SHADER_DATA_EXT	GFXPIPE_3D_CMD(0x0, 0x8A)
+#define CMD_3DSTATE_TASK_SHADER_DATA_EXT	GFXPIPE_3D_CMD(0x0, 0x8B)
+#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_CC_2	GFXPIPE_3D_CMD(0x0, 0x8D)
+#define CMD_3DSTATE_CC_STATE_POINTERS_2		GFXPIPE_3D_CMD(0x0, 0x8E)
+#define CMD_3DSTATE_SCISSOR_STATE_POINTERS_2	GFXPIPE_3D_CMD(0x0, 0x8F)
+#define CMD_3DSTATE_BLEND_STATE_POINTERS_2	GFXPIPE_3D_CMD(0x0, 0xA0)
+#define CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2	GFXPIPE_3D_CMD(0x0, 0xA1)
 
 #define CMD_3DSTATE_DRAWING_RECTANGLE		GFXPIPE_3D_CMD(0x1, 0x0)
+#define CMD_3DSTATE_URB_MEMORY			GFXPIPE_3D_CMD(0x1, 0x1)
 #define CMD_3DSTATE_CHROMA_KEY			GFXPIPE_3D_CMD(0x1, 0x4)
 #define CMD_3DSTATE_POLY_STIPPLE_OFFSET		GFXPIPE_3D_CMD(0x1, 0x6)
 #define CMD_3DSTATE_POLY_STIPPLE_PATTERN	GFXPIPE_3D_CMD(0x1, 0x7)
@@ -160,5 +169,6 @@
 #define CMD_3DSTATE_SUBSLICE_HASH_TABLE		GFXPIPE_3D_CMD(0x1, 0x1F)
 #define CMD_3DSTATE_SLICE_TABLE_STATE_POINTERS	GFXPIPE_3D_CMD(0x1, 0x20)
 #define CMD_3DSTATE_PTBR_TILE_PASS_INFO		GFXPIPE_3D_CMD(0x1, 0x22)
+#define CMD_3DSTATE_SLICE_TABLE_STATE_POINTER_2	GFXPIPE_3D_CMD(0x1, 0xA0)
 
 #endif
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 24fc64fc832e9..84b80e83ac460 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -20,6 +20,7 @@
 #define MTL_MIRROR_TARGET_WP1				XE_REG(0xc60)
 #define   MTL_CAGF_MASK					REG_GENMASK(8, 0)
 #define   MTL_CC_MASK					REG_GENMASK(12, 9)
+#define   MTL_CRST					0xf
 
 /* RPM unit config (Gen8+) */
 #define RPM_CONFIG0					XE_REG(0xd00)
@@ -58,7 +59,7 @@
 #define   MCR_SLICE(slice)			REG_FIELD_PREP(MCR_SLICE_MASK, slice)
 #define   MCR_SUBSLICE_MASK			REG_GENMASK(26, 24)
 #define   MCR_SUBSLICE(subslice)		REG_FIELD_PREP(MCR_SUBSLICE_MASK, subslice)
-#define   MTL_MCR_GROUPID			REG_GENMASK(11, 8)
+#define   MTL_MCR_GROUPID			REG_GENMASK(12, 8)
 #define   MTL_MCR_INSTANCEID			REG_GENMASK(3, 0)
 
 #define PS_INVOCATION_COUNT			XE_REG(0x2348)
@@ -100,6 +101,12 @@
 #define VE1_AUX_INV				XE_REG(0x42b8)
 #define   AUX_INV				REG_BIT(0)
 
+#define GAMSTLB_CTRL				XE_REG_MCR(0x477c)
+#define   DIS_PEND_GPA_LINK			REG_BIT(13)
+
+#define GAMSTLB_CTRL2				XE_REG_MCR(0x4788)
+#define   STLB_SINGLE_BANK_MODE			REG_BIT(11)
+
 #define XE2_LMEM_CFG				XE_REG(0x48b0)
 
 #define XE2_GAMWALK_CTRL			0x47e4
@@ -107,6 +114,9 @@
 #define XE2_GAMWALK_CTRL_3D			XE_REG_MCR(XE2_GAMWALK_CTRL)
 #define   EN_CMP_1WCOH_GW			REG_BIT(14)
 
+#define MMIOATSREQLIMIT_GAM_WALK_3D             XE_REG_MCR(0x47f8)
+#define   DIS_ATS_WRONLY_PG                     REG_BIT(18)
+
 #define XEHP_FLAT_CCS_BASE_ADDR			XE_REG_MCR(0x4910)
 #define XEHP_FLAT_CCS_PTR			REG_GENMASK(31, 8)
 
@@ -125,6 +135,7 @@
 #define   VS_HIT_MAX_VALUE_MASK			REG_GENMASK(25, 20)
 #define   DIS_MESH_PARTIAL_AUTOSTRIP		REG_BIT(16)
 #define   DIS_MESH_AUTOSTRIP			REG_BIT(15)
+#define   DIS_TE_PATCH_CTRL			REG_BIT(4)
 
 #define VFLSKPD					XE_REG_MCR(0x62a8, XE_REG_OPTION_MASKED)
 #define   DIS_PARTIAL_AUTOSTRIP			REG_BIT(9)
@@ -169,9 +180,11 @@
 #define COMMON_SLICE_CHICKEN4			XE_REG(0x7300, XE_REG_OPTION_MASKED)
 #define   SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE	REG_BIT(12)
 #define   DISABLE_TDC_LOAD_BALANCING_CALC	REG_BIT(6)
+#define   HW_FILTERING				REG_BIT(5)
 
 #define COMMON_SLICE_CHICKEN3				XE_REG(0x7304, XE_REG_OPTION_MASKED)
 #define XEHP_COMMON_SLICE_CHICKEN3			XE_REG_MCR(0x7304, XE_REG_OPTION_MASKED)
+#define   DISABLE_STATE_CACHE_PERF_FIX			REG_BIT(13)
 #define   DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN	REG_BIT(12)
 #define   XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE		REG_BIT(12)
 #define   BLEND_EMB_FIX_DISABLE_IN_RCC			REG_BIT(11)
@@ -253,6 +266,8 @@
 #define XE2_GT_COMPUTE_DSS_2			XE_REG(0x914c)
 #define XE2_GT_GEOMETRY_DSS_1			XE_REG(0x9150)
 #define XE2_GT_GEOMETRY_DSS_2			XE_REG(0x9154)
+#define XE3P_XPC_GT_GEOMETRY_DSS_3		XE_REG(0x915c)
+#define XE3P_XPC_GT_COMPUTE_DSS_3		XE_REG(0x9160)
 
 #define SERVICE_COPY_ENABLE			XE_REG(0x9170)
 #define   FUSE_SERVICE_COPY_ENABLE_MASK		REG_GENMASK(7, 0)
@@ -367,6 +382,7 @@
 #define FORCEWAKE_RENDER			XE_REG(0xa278)
 
 #define POWERGATE_DOMAIN_STATUS			XE_REG(0xa2a0)
+#define   GSC_AWAKE_STATUS			REG_BIT(8)
 #define   MEDIA_SLICE3_AWAKE_STATUS		REG_BIT(4)
 #define   MEDIA_SLICE2_AWAKE_STATUS		REG_BIT(3)
 #define   MEDIA_SLICE1_AWAKE_STATUS		REG_BIT(2)
@@ -420,6 +436,8 @@
 #define   LSN_DIM_Z_WGT(value)			REG_FIELD_PREP(LSN_DIM_Z_WGT_MASK, value)
 
 #define L3SQCREG2				XE_REG_MCR(0xb104)
+#define   L3_SQ_DISABLE_COAMA_2WAY_COH		REG_BIT(30)
+#define   L3_SQ_DISABLE_COAMA			REG_BIT(22)
 #define   COMPMEMRD256BOVRFETCHEN		REG_BIT(20)
 
 #define L3SQCREG3				XE_REG_MCR(0xb108)
@@ -436,6 +454,10 @@
 
 #define XEHPC_L3CLOS_MASK(i)			XE_REG_MCR(0xb194 + (i) * 8)
 
+#define L2COMPUTESIDECTRL			XE_REG_MCR(0xb1c0)
+#define   CECTRL				REG_GENMASK(2, 1)
+#define   CECTRL_CENODATA_ALWAYS		REG_FIELD_PREP(CECTRL, 0x0)
+
 #define XE2_GLOBAL_INVAL			XE_REG(0xb404)
 
 #define XE2LPM_L3SQCREG2			XE_REG_MCR(0xb604)
@@ -459,6 +481,8 @@
 #define   FORCE_MISS_FTLB			REG_BIT(3)
 
 #define XEHP_GAMSTLB_CTRL			XE_REG_MCR(0xcf4c)
+#define   BANK_HASH_MODE			REG_GENMASK(27, 26)
+#define   BANK_HASH_4KB_MODE			REG_FIELD_PREP(BANK_HASH_MODE, 0x3)
 #define   CONTROL_BLOCK_CLKGATE_DIS		REG_BIT(12)
 #define   EGRESS_BLOCK_CLKGATE_DIS		REG_BIT(11)
 #define   TAG_BLOCK_CLKGATE_DIS			REG_BIT(7)
@@ -550,11 +574,16 @@
 #define   UGM_FRAGMENT_THRESHOLD_TO_3		REG_BIT(58 - 32)
 #define   DIS_CHAIN_2XSIMD8			REG_BIT(55 - 32)
 #define   XE2_ALLOC_DPA_STARVE_FIX_DIS		REG_BIT(47 - 32)
+#define   SAMPLER_LD_LSC_DISABLE                REG_BIT(45 - 32)
 #define   ENABLE_SMP_LD_RENDER_SURFACE_CONTROL	REG_BIT(44 - 32)
 #define   FORCE_SLM_FENCE_SCOPE_TO_TILE		REG_BIT(42 - 32)
 #define   FORCE_UGM_FENCE_SCOPE_TO_TILE		REG_BIT(41 - 32)
 #define   MAXREQS_PER_BANK			REG_GENMASK(39 - 32, 37 - 32)
 #define   DISABLE_128B_EVICTION_COMMAND_UDW	REG_BIT(36 - 32)
+#define   LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE	REG_BIT(35 - 32)
+
+#define ROW_CHICKEN5				XE_REG_MCR(0xe7f0)
+#define   CPSS_AWARE_DIS			REG_BIT(3)
 
 #define SARB_CHICKEN1				XE_REG_MCR(0xe90c)
 #define   COMP_CKN_IN				REG_GENMASK(30, 29)
diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
index 87984713dd126..5faac8316b66c 100644
--- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
@@ -40,6 +40,9 @@
 #define   GS_BOOTROM_JUMP_PASSED		REG_FIELD_PREP(GS_BOOTROM_MASK, 0x76)
 #define   GS_MIA_IN_RESET			REG_BIT(0)
 
+#define BOOT_HASH_CHK				XE_REG(0xc010)
+#define   GUC_BOOT_UKERNEL_VALID		REG_BIT(31)
+
 #define GUC_HEADER_INFO				XE_REG(0xc014)
 
 #define GUC_WOPCM_SIZE				XE_REG(0xc050)
@@ -83,7 +86,12 @@
 #define   GUC_WOPCM_OFFSET_MASK			REG_GENMASK(31, GUC_WOPCM_OFFSET_SHIFT)
 #define   HUC_LOADING_AGENT_GUC			REG_BIT(1)
 #define   GUC_WOPCM_OFFSET_VALID		REG_BIT(0)
+
+#define GUC_SRAM_STATUS				XE_REG(0xc398)
+#define   GUC_SRAM_HANDLING_MASK		REG_GENMASK(8, 7)
+
 #define GUC_MAX_IDLE_COUNT			XE_REG(0xc3e4)
+#define   GUC_IDLE_FLOW_DISABLE			REG_BIT(31)
 #define GUC_PMTIMESTAMP_LO			XE_REG(0xc3e8)
 #define GUC_PMTIMESTAMP_HI			XE_REG(0xc3ec)
 
diff --git a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h
index c146b9ef44eb0..046e1756c6980 100644
--- a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h
@@ -6,15 +6,83 @@
 #ifndef _XE_HW_ERROR_REGS_H_
 #define _XE_HW_ERROR_REGS_H_
 
-#define HEC_UNCORR_ERR_STATUS(base)                    XE_REG((base) + 0x118)
-#define    UNCORR_FW_REPORTED_ERR                      BIT(6)
+#define HEC_UNCORR_ERR_STATUS(base)			XE_REG((base) + 0x118)
+#define   UNCORR_FW_REPORTED_ERR			REG_BIT(6)
 
-#define HEC_UNCORR_FW_ERR_DW0(base)                    XE_REG((base) + 0x124)
+#define HEC_UNCORR_FW_ERR_DW0(base)			XE_REG((base) + 0x124)
+
+#define ERR_STAT_GT_COR					0x100160
+#define   EU_GRF_COR_ERR				REG_BIT(15)
+#define   EU_IC_COR_ERR					REG_BIT(14)
+#define   SLM_COR_ERR					REG_BIT(13)
+#define   GUC_COR_ERR					REG_BIT(1)
+
+#define ERR_STAT_GT_NONFATAL				0x100164
+#define ERR_STAT_GT_FATAL				0x100168
+#define   EU_GRF_FAT_ERR				REG_BIT(15)
+#define   SLM_FAT_ERR					REG_BIT(13)
+#define   GUC_FAT_ERR					REG_BIT(6)
+#define   FPU_FAT_ERR					REG_BIT(3)
+
+#define ERR_STAT_GT_REG(x)				XE_REG(_PICK_EVEN((x), \
+									  ERR_STAT_GT_COR, \
+									  ERR_STAT_GT_NONFATAL))
+
+#define PVC_COR_ERR_MASK				(GUC_COR_ERR | SLM_COR_ERR | \
+							 EU_IC_COR_ERR | EU_GRF_COR_ERR)
+
+#define PVC_FAT_ERR_MASK				(FPU_FAT_ERR | GUC_FAT_ERR | \
+							 EU_GRF_FAT_ERR | SLM_FAT_ERR)
+
+#define DEV_ERR_STAT_NONFATAL				0x100178
+#define DEV_ERR_STAT_CORRECTABLE			0x10017c
+#define DEV_ERR_STAT_REG(x)				XE_REG(_PICK_EVEN((x), \
+									  DEV_ERR_STAT_CORRECTABLE, \
+									  DEV_ERR_STAT_NONFATAL))
+
+#define   XE_CSC_ERROR					17
+#define   XE_SOC_ERROR					16
+#define   XE_GT_ERROR					0
+
+#define ERR_STAT_GT_FATAL_VECTOR_0			0x100260
+#define ERR_STAT_GT_FATAL_VECTOR_1			0x100264
+
+#define ERR_STAT_GT_FATAL_VECTOR_REG(x)			XE_REG(_PICK_EVEN((x), \
+									  ERR_STAT_GT_FATAL_VECTOR_0, \
+									  ERR_STAT_GT_FATAL_VECTOR_1))
+
+#define ERR_STAT_GT_COR_VECTOR_0			0x1002a0
+#define ERR_STAT_GT_COR_VECTOR_1			0x1002a4
+
+#define ERR_STAT_GT_COR_VECTOR_REG(x)			XE_REG(_PICK_EVEN((x), \
+									  ERR_STAT_GT_COR_VECTOR_0, \
+									  ERR_STAT_GT_COR_VECTOR_1))
+
+#define ERR_STAT_GT_VECTOR_REG(hw_err, x)		(hw_err == HARDWARE_ERROR_CORRECTABLE ? \
+							 ERR_STAT_GT_COR_VECTOR_REG(x) : \
+							 ERR_STAT_GT_FATAL_VECTOR_REG(x))
+
+#define SOC_PVC_MASTER_BASE				0x282000
+#define SOC_PVC_SLAVE_BASE				0x283000
+
+#define SOC_GCOERRSTS					0x200
+#define SOC_GNFERRSTS					0x210
+#define SOC_GLOBAL_ERR_STAT_REG(base, x)		XE_REG(_PICK_EVEN((x), \
+									  (base) + SOC_GCOERRSTS, \
+									  (base) + SOC_GNFERRSTS))
+#define   SOC_SLAVE_IEH					REG_BIT(1)
+#define   SOC_IEH0_LOCAL_ERR_STATUS			REG_BIT(0)
+#define   SOC_IEH1_LOCAL_ERR_STATUS			REG_BIT(0)
+
+#define SOC_GSYSEVTCTL					0x264
+#define SOC_GSYSEVTCTL_REG(master, slave, x)		XE_REG(_PICK_EVEN((x), \
+									  (master) + SOC_GSYSEVTCTL, \
+									  (slave) + SOC_GSYSEVTCTL))
+
+#define SOC_LERRUNCSTS					0x280
+#define SOC_LERRCORSTS					0x294
+#define SOC_LOCAL_ERR_STAT_REG(base, hw_err)		XE_REG(hw_err == HARDWARE_ERROR_CORRECTABLE ? \
+							       (base) + SOC_LERRCORSTS : \
+							       (base) + SOC_LERRUNCSTS)
 
-#define DEV_ERR_STAT_NONFATAL			0x100178
-#define DEV_ERR_STAT_CORRECTABLE		0x10017c
-#define DEV_ERR_STAT_REG(x)			XE_REG(_PICK_EVEN((x), \
-								  DEV_ERR_STAT_CORRECTABLE, \
-								  DEV_ERR_STAT_NONFATAL))
-#define   XE_CSC_ERROR				BIT(17)
 #endif
diff --git a/drivers/gpu/drm/xe/regs/xe_reg_defs.h b/drivers/gpu/drm/xe/regs/xe_reg_defs.h
index c39aab843e357..27ac0bf1f6cda 100644
--- a/drivers/gpu/drm/xe/regs/xe_reg_defs.h
+++ b/drivers/gpu/drm/xe/regs/xe_reg_defs.h
@@ -6,12 +6,13 @@
 #ifndef _XE_REG_DEFS_H_
 #define _XE_REG_DEFS_H_
 
+#include <drm/intel/pick.h>
+#include <drm/intel/reg_bits.h>
+
 #include <linux/build_bug.h>
 #include <linux/log2.h>
 #include <linux/sizes.h>
 
-#include "compat-i915-headers/i915_reg_defs.h"
-
 /**
  * XE_REG_ADDR_MAX - The upper limit on MMIO register address
  *
diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
index 954b6b911ea0a..0be8440b39760 100644
--- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
@@ -22,8 +22,7 @@ static bool p2p_enabled(struct dma_buf_test_params *params)
 
 static bool is_dynamic(struct dma_buf_test_params *params)
 {
-	return IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY) && params->attach_ops &&
-		params->attach_ops->move_notify;
+	return params->attach_ops && params->attach_ops->invalidate_mappings;
 }
 
 static void check_residency(struct kunit *test, struct xe_bo *exported,
@@ -60,7 +59,7 @@ static void check_residency(struct kunit *test, struct xe_bo *exported,
 
 	/*
 	 * Evict exporter. Evicting the exported bo will
-	 * evict also the imported bo through the move_notify() functionality if
+	 * evict also the imported bo through the invalidate_mappings() functionality if
 	 * importer is on a different device. If they're on the same device,
 	 * the exporter and the importer should be the same bo.
 	 */
@@ -198,7 +197,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
 
 static const struct dma_buf_attach_ops nop2p_attach_ops = {
 	.allow_peer2peer = false,
-	.move_notify = xe_dma_buf_move_notify
+	.invalidate_mappings = xe_dma_buf_move_notify
 };
 
 /*
diff --git a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
index 42bfc4bcfbcfe..efa8963ec2488 100644
--- a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
+++ b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
@@ -11,14 +11,26 @@
 #include "xe_pci_test.h"
 
 #define TEST_MAX_VFS	63
+#define TEST_VRAM	0x7a800000ull	/* random size that works on 32-bit */
 
 static void pf_set_admin_mode(struct xe_device *xe, bool enable)
 {
 	/* should match logic of xe_sriov_pf_admin_only() */
-	xe->info.probe_display = !enable;
+	xe->sriov.pf.admin_only = enable;
 	KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_sriov_pf_admin_only(xe));
 }
 
+static void pf_set_usable_vram(struct xe_device *xe, u64 usable)
+{
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct kunit *test = kunit_get_current_test();
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tile);
+	xe->mem.vram->usable_size = usable;
+	tile->mem.vram->usable_size = usable;
+	KUNIT_ASSERT_EQ(test, usable, xe_vram_region_usable_size(tile->mem.vram));
+}
+
 static const void *num_vfs_gen_param(struct kunit *test, const void *prev, char *desc)
 {
 	unsigned long next = 1 + (unsigned long)prev;
@@ -34,9 +46,11 @@ static int pf_gt_config_test_init(struct kunit *test)
 {
 	struct xe_pci_fake_data fake = {
 		.sriov_mode = XE_SRIOV_MODE_PF,
-		.platform = XE_TIGERLAKE, /* any random platform with SR-IOV */
+		.platform = XE_BATTLEMAGE, /* any random DGFX platform with SR-IOV */
 		.subplatform = XE_SUBPLATFORM_NONE,
+		.graphics_verx100 = 2001,
 	};
+	struct xe_vram_region *vram;
 	struct xe_device *xe;
 	struct xe_gt *gt;
 
@@ -50,6 +64,19 @@ static int pf_gt_config_test_init(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gt);
 	test->priv = gt;
 
+	/* pretend it has some VRAM */
+	KUNIT_ASSERT_TRUE(test, IS_DGFX(xe));
+	vram = kunit_kzalloc(test, sizeof(*vram), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vram);
+	vram->usable_size = TEST_VRAM;
+	xe->mem.vram = vram;
+	xe->tiles[0].mem.vram = vram;
+
+	/* pretend we have a valid LMTT */
+	KUNIT_ASSERT_TRUE(test, xe_device_has_lmtt(xe));
+	KUNIT_ASSERT_GE(test, GRAPHICS_VERx100(xe), 1260);
+	xe->tiles[0].sriov.pf.lmtt.ops = &lmtt_ml_ops;
+
 	/* pretend it can support up to 63 VFs */
 	xe->sriov.pf.device_total_vfs = TEST_MAX_VFS;
 	xe->sriov.pf.driver_max_vfs = TEST_MAX_VFS;
@@ -189,13 +216,80 @@ static void fair_ggtt(struct kunit *test)
 		KUNIT_ASSERT_EQ(test, SZ_2G, pf_profile_fair_ggtt(gt, num_vfs));
 }
 
+static const u64 vram_sizes[] = {
+	SZ_4G - SZ_512M,
+	SZ_8G + SZ_4G - SZ_512M,
+	SZ_16G - SZ_512M,
+	SZ_32G - SZ_512M,
+	SZ_64G - SZ_512M,
+	TEST_VRAM,
+};
+
+static void u64_param_get_desc(const u64 *p, char *desc)
+{
+	string_get_size(*p, 1, STRING_UNITS_2, desc, KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(vram_size, vram_sizes, u64_param_get_desc);
+
+static void fair_vram_1vf(struct kunit *test)
+{
+	const u64 usable = *(const u64 *)test->param_value;
+	struct xe_gt *gt = test->priv;
+	struct xe_device *xe = gt_to_xe(gt);
+
+	pf_set_admin_mode(xe, false);
+	pf_set_usable_vram(xe, usable);
+
+	KUNIT_EXPECT_NE(test, 0, pf_profile_fair_lmem(gt, 1));
+	KUNIT_EXPECT_GE(test, usable, pf_profile_fair_lmem(gt, 1));
+	KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_lmem(gt, 1)));
+	KUNIT_EXPECT_GE(test, usable - pf_profile_fair_lmem(gt, 1), pf_profile_fair_lmem(gt, 1));
+}
+
+static void fair_vram_1vf_admin_only(struct kunit *test)
+{
+	const u64 usable = *(const u64 *)test->param_value;
+	struct xe_gt *gt = test->priv;
+	struct xe_device *xe = gt_to_xe(gt);
+
+	pf_set_admin_mode(xe, true);
+	pf_set_usable_vram(xe, usable);
+
+	KUNIT_EXPECT_NE(test, 0, pf_profile_fair_lmem(gt, 1));
+	KUNIT_EXPECT_GE(test, usable, pf_profile_fair_lmem(gt, 1));
+	KUNIT_EXPECT_LT(test, usable - pf_profile_fair_lmem(gt, 1), pf_profile_fair_lmem(gt, 1));
+	KUNIT_EXPECT_TRUE(test, IS_ALIGNED(pf_profile_fair_lmem(gt, 1), SZ_1G));
+}
+
+static void fair_vram(struct kunit *test)
+{
+	unsigned int num_vfs = (unsigned long)test->param_value;
+	struct xe_gt *gt = test->priv;
+	struct xe_device *xe = gt_to_xe(gt);
+	u64 alignment = pf_get_lmem_alignment(gt);
+	char size[10];
+
+	pf_set_admin_mode(xe, false);
+
+	string_get_size(pf_profile_fair_lmem(gt, num_vfs), 1, STRING_UNITS_2, size, sizeof(size));
+	kunit_info(test, "fair %s %llx\n", size, pf_profile_fair_lmem(gt, num_vfs));
+
+	KUNIT_EXPECT_TRUE(test, is_power_of_2(pf_profile_fair_lmem(gt, num_vfs)));
+	KUNIT_EXPECT_TRUE(test, IS_ALIGNED(pf_profile_fair_lmem(gt, num_vfs), alignment));
+	KUNIT_EXPECT_GE(test, TEST_VRAM, num_vfs * pf_profile_fair_lmem(gt, num_vfs));
+}
+
 static struct kunit_case pf_gt_config_test_cases[] = {
 	KUNIT_CASE(fair_contexts_1vf),
 	KUNIT_CASE(fair_doorbells_1vf),
 	KUNIT_CASE(fair_ggtt_1vf),
+	KUNIT_CASE_PARAM(fair_vram_1vf, vram_size_gen_params),
+	KUNIT_CASE_PARAM(fair_vram_1vf_admin_only, vram_size_gen_params),
 	KUNIT_CASE_PARAM(fair_contexts, num_vfs_gen_param),
 	KUNIT_CASE_PARAM(fair_doorbells, num_vfs_gen_param),
 	KUNIT_CASE_PARAM(fair_ggtt, num_vfs_gen_param),
+	KUNIT_CASE_PARAM(fair_vram, num_vfs_gen_param),
 	{}
 };
 
diff --git a/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c
index acddbedcf17cb..51e1e04001ace 100644
--- a/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c
+++ b/drivers/gpu/drm/xe/tests/xe_guc_buf_kunit.c
@@ -38,12 +38,8 @@ static struct xe_bo *replacement_xe_managed_bo_create_pin_map(struct xe_device *
 	if (flags & XE_BO_FLAG_GGTT) {
 		struct xe_ggtt *ggtt = tile->mem.ggtt;
 
-		bo->ggtt_node[tile->id] = xe_ggtt_node_init(ggtt);
+		bo->ggtt_node[tile->id] = xe_ggtt_insert_node(ggtt, xe_bo_size(bo), SZ_4K);
 		KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bo->ggtt_node[tile->id]);
-
-		KUNIT_ASSERT_EQ(test, 0,
-				xe_ggtt_node_insert(bo->ggtt_node[tile->id],
-						    xe_bo_size(bo), SZ_4K));
 	}
 
 	return bo;
diff --git a/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c b/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c
index 3b213fcae9166..ea8ca03effa90 100644
--- a/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_guc_g2g_test.c
@@ -48,6 +48,38 @@ struct g2g_test_payload  {
 	u32 seqno;
 };
 
+static int slot_index_from_gts(struct xe_gt *tx_gt, struct xe_gt *rx_gt)
+{
+	struct xe_device *xe = gt_to_xe(tx_gt);
+	int idx = 0, found = 0, id, tx_idx, rx_idx;
+	struct xe_gt *gt;
+	struct kunit *test = kunit_get_current_test();
+
+	for (id = 0; id < xe->info.tile_count * xe->info.max_gt_per_tile; id++) {
+		gt = xe_device_get_gt(xe, id);
+		if (!gt)
+			continue;
+		if (gt == tx_gt) {
+			tx_idx = idx;
+			found++;
+		}
+		if (gt == rx_gt) {
+			rx_idx = idx;
+			found++;
+		}
+
+		if (found == 2)
+			break;
+
+		idx++;
+	}
+
+	if (found != 2)
+		KUNIT_FAIL(test, "GT index not found");
+
+	return (tx_idx * xe->info.gt_count) + rx_idx;
+}
+
 static void g2g_test_send(struct kunit *test, struct xe_guc *guc,
 			  u32 far_tile, u32 far_dev,
 			  struct g2g_test_payload *payload)
@@ -163,7 +195,7 @@ int xe_guc_g2g_test_notification(struct xe_guc *guc, u32 *msg, u32 len)
 		goto done;
 	}
 
-	idx = (tx_gt->info.id * xe->info.gt_count) + rx_gt->info.id;
+	idx = slot_index_from_gts(tx_gt, rx_gt);
 
 	if (xe->g2g_test_array[idx] != payload->seqno - 1) {
 		xe_gt_err(rx_gt, "G2G: Seqno mismatch %d vs %d for %d:%d -> %d:%d!\n",
@@ -180,13 +212,17 @@ int xe_guc_g2g_test_notification(struct xe_guc *guc, u32 *msg, u32 len)
 	return ret;
 }
 
+#define G2G_WAIT_TIMEOUT_MS 100
+#define G2G_WAIT_POLL_MS 1
+
 /*
  * Send the given seqno from all GuCs to all other GuCs in tile/GT order
  */
 static void g2g_test_in_order(struct kunit *test, struct xe_device *xe, u32 seqno)
 {
 	struct xe_gt *near_gt, *far_gt;
-	int i, j;
+	int i, j, waited;
+	u32 idx;
 
 	for_each_gt(near_gt, xe, i) {
 		u32 near_tile = gt_to_tile(near_gt)->id;
@@ -205,6 +241,27 @@ static void g2g_test_in_order(struct kunit *test, struct xe_device *xe, u32 seqn
 			payload.rx_dev = far_dev;
 			payload.rx_tile = far_tile;
 			payload.seqno = seqno;
+
+			/* Calculate idx for event-based wait */
+			idx = slot_index_from_gts(near_gt, far_gt);
+			waited = 0;
+
+			/*
+			 * Wait for previous seqno to be acknowledged before sending,
+			 * to avoid queuing too many back-to-back messages and
+			 * causing a test timeout. Actual correctness of message
+			 * will be checked later in xe_guc_g2g_test_notification()
+			 */
+			while (xe->g2g_test_array[idx] != (seqno - 1)) {
+				msleep(G2G_WAIT_POLL_MS);
+				waited += G2G_WAIT_POLL_MS;
+				if (waited >= G2G_WAIT_TIMEOUT_MS) {
+					kunit_info(test, "Timeout waiting! tx gt: %d, rx gt: %d\n",
+						   near_gt->info.id, far_gt->info.id);
+					break;
+				}
+			}
+
 			g2g_test_send(test, &near_gt->uc.guc, far_tile, far_dev, &payload);
 		}
 	}
diff --git a/drivers/gpu/drm/xe/tests/xe_pci_test.c b/drivers/gpu/drm/xe/tests/xe_pci_test.c
index 4d10a7e2b5707..acf5a55551305 100644
--- a/drivers/gpu/drm/xe/tests/xe_pci_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_pci_test.c
@@ -19,6 +19,8 @@ static void check_graphics_ip(struct kunit *test)
 	const struct xe_ip *param = test->param_value;
 	const struct xe_graphics_desc *graphics = param->desc;
 	u64 mask = graphics->hw_engine_mask;
+	u8 fuse_regs = graphics->num_geometry_xecore_fuse_regs +
+		graphics->num_compute_xecore_fuse_regs;
 
 	/* RCS, CCS, and BCS engines are allowed on the graphics IP */
 	mask &= ~(XE_HW_ENGINE_RCS_MASK |
@@ -27,6 +29,12 @@ static void check_graphics_ip(struct kunit *test)
 
 	/* Any remaining engines are an error */
 	KUNIT_ASSERT_EQ(test, mask, 0);
+
+	/*
+	 * All graphics IP should have at least one geometry and/or compute
+	 * XeCore fuse register.
+	 */
+	KUNIT_ASSERT_GE(test, fuse_regs, 1);
 }
 
 static void check_media_ip(struct kunit *test)
diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
index d2255a59e58f9..e5a0f985a700b 100644
--- a/drivers/gpu/drm/xe/tests/xe_rtp_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
@@ -322,7 +322,8 @@ static void xe_rtp_process_to_sr_tests(struct kunit *test)
 		count_rtp_entries++;
 
 	xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, count_rtp_entries);
-	xe_rtp_process_to_sr(&ctx, param->entries, count_rtp_entries, reg_sr);
+	xe_rtp_process_to_sr(&ctx, param->entries, count_rtp_entries,
+			     reg_sr, false);
 
 	xa_for_each(&reg_sr->xa, idx, sre) {
 		if (idx == param->expected_reg.addr)
diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c
index 4749aa7f94660..b896b6f6615cb 100644
--- a/drivers/gpu/drm/xe/xe_bb.c
+++ b/drivers/gpu/drm/xe/xe_bb.c
@@ -59,16 +59,51 @@ struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm)
 	return ERR_PTR(err);
 }
 
-struct xe_bb *xe_bb_ccs_new(struct xe_gt *gt, u32 dwords,
-			    enum xe_sriov_vf_ccs_rw_ctxs ctx_id)
+/**
+ * xe_bb_alloc() - Allocate a new batch buffer structure
+ * @gt: the &xe_gt
+ *
+ * Allocates and initializes a new xe_bb structure with an associated
+ * uninitialized suballoc object.
+ *
+ * Returns: Batch buffer structure or an ERR_PTR(-ENOMEM).
+ */
+struct xe_bb *xe_bb_alloc(struct xe_gt *gt)
 {
 	struct xe_bb *bb = kmalloc_obj(*bb);
-	struct xe_device *xe = gt_to_xe(gt);
-	struct xe_sa_manager *bb_pool;
 	int err;
 
 	if (!bb)
 		return ERR_PTR(-ENOMEM);
+
+	bb->bo = xe_sa_bo_alloc(GFP_KERNEL);
+	if (IS_ERR(bb->bo)) {
+		err = PTR_ERR(bb->bo);
+		goto err;
+	}
+
+	return bb;
+
+err:
+	kfree(bb);
+	return ERR_PTR(err);
+}
+
+/**
+ * xe_bb_init() - Initialize a batch buffer with memory from a sub-allocator pool
+ * @bb: Batch buffer structure to initialize
+ * @bb_pool: Suballoc memory pool to allocate from
+ * @dwords: Number of dwords to be allocated
+ *
+ * Initializes the batch buffer by allocating memory from the specified
+ * suballoc pool.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_bb_init(struct xe_bb *bb, struct xe_sa_manager *bb_pool, u32 dwords)
+{
+	int err;
+
 	/*
 	 * We need to allocate space for the requested number of dwords &
 	 * one additional MI_BATCH_BUFFER_END dword. Since the whole SA
@@ -76,22 +111,14 @@ struct xe_bb *xe_bb_ccs_new(struct xe_gt *gt, u32 dwords,
 	 * is not over written when the last chunk of SA is allocated for BB.
 	 * So, this extra DW acts as a guard here.
 	 */
-
-	bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
-	bb->bo = xe_sa_bo_new(bb_pool, 4 * (dwords + 1));
-
-	if (IS_ERR(bb->bo)) {
-		err = PTR_ERR(bb->bo);
-		goto err;
-	}
+	err = xe_sa_bo_init(bb_pool, bb->bo, 4 * (dwords + 1));
+	if (err)
+		return err;
 
 	bb->cs = xe_sa_bo_cpu_addr(bb->bo);
 	bb->len = 0;
 
-	return bb;
-err:
-	kfree(bb);
-	return ERR_PTR(err);
+	return 0;
 }
 
 static struct xe_sched_job *
diff --git a/drivers/gpu/drm/xe/xe_bb.h b/drivers/gpu/drm/xe/xe_bb.h
index 2a8adc9a6dee7..231870b24c2f2 100644
--- a/drivers/gpu/drm/xe/xe_bb.h
+++ b/drivers/gpu/drm/xe/xe_bb.h
@@ -12,12 +12,12 @@ struct dma_fence;
 
 struct xe_gt;
 struct xe_exec_queue;
+struct xe_sa_manager;
 struct xe_sched_job;
-enum xe_sriov_vf_ccs_rw_ctxs;
 
 struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm);
-struct xe_bb *xe_bb_ccs_new(struct xe_gt *gt, u32 dwords,
-			    enum xe_sriov_vf_ccs_rw_ctxs ctx_id);
+struct xe_bb *xe_bb_alloc(struct xe_gt *gt);
+int xe_bb_init(struct xe_bb *bb, struct xe_sa_manager *bb_pool, u32 dwords);
 struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q,
 				      struct xe_bb *bb);
 struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q,
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 29fffc81f2401..22179b2df85c5 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -512,8 +512,8 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
 		/*
 		 * Display scanout is always non-coherent with the CPU cache.
 		 *
-		 * For Xe_LPG and beyond, PPGTT PTE lookups are also
-		 * non-coherent and require a CPU:WC mapping.
+		 * For Xe_LPG and beyond up to NVL-P (excluding), PPGTT PTE
+		 * lookups are also non-coherent and require a CPU:WC mapping.
 		 */
 		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
 		     (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
@@ -818,7 +818,7 @@ static int xe_bo_move_notify(struct xe_bo *bo,
 
 	/* Don't call move_notify() for imported dma-bufs. */
 	if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)
-		dma_buf_move_notify(ttm_bo->base.dma_buf);
+		dma_buf_invalidate_mappings(ttm_bo->base.dma_buf);
 
 	/*
 	 * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual),
@@ -3330,6 +3330,61 @@ int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
+/**
+ * xe_bo_decompress - schedule in-place decompress and install fence
+ * @bo: buffer object (caller should hold drm_exec reservations for VM+BO)
+ *
+ * Schedules an in-place resolve via the migrate layer and installs the
+ * returned dma_fence into the BO kernel reservation slot (DMA_RESV_USAGE_KERNEL).
+ * In preempt fence mode, this operation interrupts hardware execution
+ * which is expensive. Page fault mode is recommended for better performance.
+ *
+ * The resolve path only runs for VRAM-backed buffers (currently dGPU-only);
+ * iGPU/system-memory objects fail the resource check and bypass the resolve.
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int xe_bo_decompress(struct xe_bo *bo)
+{
+	struct xe_device *xe = xe_bo_device(bo);
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct dma_fence *decomp_fence = NULL;
+	struct ttm_operation_ctx op_ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false,
+		.gfp_retry_mayfail = false,
+	};
+	int err = 0;
+
+	/* Silently skip decompression for non-VRAM buffers */
+	if (!bo->ttm.resource || !mem_type_is_vram(bo->ttm.resource->mem_type))
+		return 0;
+
+	/* Notify before scheduling resolve */
+	err = xe_bo_move_notify(bo, &op_ctx);
+	if (err)
+		return err;
+
+	/* Reserve fence slot before scheduling */
+	err = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
+	if (err)
+		return err;
+
+	/* Schedule the in-place decompression */
+	decomp_fence = xe_migrate_resolve(tile->migrate,
+					  bo,
+					  bo->ttm.resource);
+
+	if (IS_ERR(decomp_fence))
+		return PTR_ERR(decomp_fence);
+
+	/* Install kernel-usage fence */
+	dma_resv_add_fence(bo->ttm.base.resv, decomp_fence, DMA_RESV_USAGE_KERNEL);
+	dma_fence_put(decomp_fence);
+
+	return 0;
+}
+
 /**
  * xe_bo_lock() - Lock the buffer object's dma_resv object
  * @bo: The struct xe_bo whose lock is to be taken
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index c914ab719f20e..2cbac16f7db75 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -312,6 +312,8 @@ int xe_bo_dumb_create(struct drm_file *file_priv,
 
 bool xe_bo_needs_ccs_pages(struct xe_bo *bo);
 
+int xe_bo_decompress(struct xe_bo *bo);
+
 static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo)
 {
 	return PAGE_ALIGN(xe_bo_size(bo));
diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c
index 7fd07d1280bb1..32102600a1481 100644
--- a/drivers/gpu/drm/xe/xe_configfs.c
+++ b/drivers/gpu/drm/xe/xe_configfs.c
@@ -15,6 +15,7 @@
 
 #include "instructions/xe_mi_commands.h"
 #include "xe_configfs.h"
+#include "xe_defaults.h"
 #include "xe_gt_types.h"
 #include "xe_hw_engine_types.h"
 #include "xe_module.h"
@@ -263,6 +264,7 @@ struct xe_config_group_device {
 		bool enable_psmi;
 		struct {
 			unsigned int max_vfs;
+			bool admin_only_pf;
 		} sriov;
 	} config;
 
@@ -280,7 +282,8 @@ static const struct xe_config_device device_defaults = {
 	.survivability_mode = false,
 	.enable_psmi = false,
 	.sriov = {
-		.max_vfs = UINT_MAX,
+		.max_vfs = XE_DEFAULT_MAX_VFS,
+		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,
 	},
 };
 
@@ -897,10 +900,40 @@ static ssize_t sriov_max_vfs_store(struct config_item *item, const char *page, s
 	return len;
 }
 
+static ssize_t sriov_admin_only_pf_show(struct config_item *item, char *page)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent);
+
+	guard(mutex)(&dev->lock);
+
+	return sprintf(page, "%s\n", str_yes_no(dev->config.sriov.admin_only_pf));
+}
+
+static ssize_t sriov_admin_only_pf_store(struct config_item *item, const char *page, size_t len)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent);
+	bool admin_only_pf;
+	int ret;
+
+	guard(mutex)(&dev->lock);
+
+	if (is_bound(dev))
+		return -EBUSY;
+
+	ret = kstrtobool(page, &admin_only_pf);
+	if (ret)
+		return ret;
+
+	dev->config.sriov.admin_only_pf = admin_only_pf;
+	return len;
+}
+
 CONFIGFS_ATTR(sriov_, max_vfs);
+CONFIGFS_ATTR(sriov_, admin_only_pf);
 
 static struct configfs_attribute *xe_config_sriov_attrs[] = {
 	&sriov_attr_max_vfs,
+	&sriov_attr_admin_only_pf,
 	NULL,
 };
 
@@ -911,6 +944,8 @@ static bool xe_config_sriov_is_visible(struct config_item *item,
 
 	if (attr == &sriov_attr_max_vfs && dev->mode != XE_SRIOV_MODE_PF)
 		return false;
+	if (attr == &sriov_attr_admin_only_pf && dev->mode != XE_SRIOV_MODE_PF)
+		return false;
 
 	return true;
 }
@@ -1064,6 +1099,7 @@ static void dump_custom_dev_config(struct pci_dev *pdev,
 	PRI_CUSTOM_ATTR("%llx", engines_allowed);
 	PRI_CUSTOM_ATTR("%d", enable_psmi);
 	PRI_CUSTOM_ATTR("%d", survivability_mode);
+	PRI_CUSTOM_ATTR("%u", sriov.admin_only_pf);
 
 #undef PRI_CUSTOM_ATTR
 }
@@ -1242,6 +1278,32 @@ u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev,
 }
 
 #ifdef CONFIG_PCI_IOV
+/**
+ * xe_configfs_admin_only_pf() - Get PF's operational mode.
+ * @pdev: the &pci_dev device
+ *
+ * Find the configfs group that belongs to the PCI device and return a flag
+ * whether the PF driver should be dedicated for VFs management only.
+ *
+ * If configfs group is not present, use driver's default value.
+ *
+ * Return: true if PF driver is dedicated for VFs administration only.
+ */
+bool xe_configfs_admin_only_pf(struct pci_dev *pdev)
+{
+	struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
+	bool admin_only_pf;
+
+	if (!dev)
+		return XE_DEFAULT_ADMIN_ONLY_PF;
+
+	scoped_guard(mutex, &dev->lock)
+		admin_only_pf = dev->config.sriov.admin_only_pf;
+
+	config_group_put(&dev->group);
+
+	return admin_only_pf;
+}
 /**
  * xe_configfs_get_max_vfs() - Get number of VFs that could be managed
  * @pdev: the &pci_dev device
diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h
index f3683bc7eb90c..07d62bf0c1524 100644
--- a/drivers/gpu/drm/xe/xe_configfs.h
+++ b/drivers/gpu/drm/xe/xe_configfs.h
@@ -8,7 +8,9 @@
 #include <linux/limits.h>
 #include <linux/types.h>
 
-#include <xe_hw_engine_types.h>
+#include "xe_defaults.h"
+#include "xe_hw_engine_types.h"
+#include "xe_module.h"
 
 struct pci_dev;
 
@@ -29,6 +31,7 @@ u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev,
 					const u32 **cs);
 #ifdef CONFIG_PCI_IOV
 unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev);
+bool xe_configfs_admin_only_pf(struct pci_dev *pdev);
 #endif
 #else
 static inline int xe_configfs_init(void) { return 0; }
@@ -45,7 +48,16 @@ static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev,
 						      enum xe_engine_class class,
 						      const u32 **cs) { return 0; }
-static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev) { return UINT_MAX; }
+#ifdef CONFIG_PCI_IOV
+static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev)
+{
+	return xe_modparam.max_vfs;
+}
+static inline bool xe_configfs_admin_only_pf(struct pci_dev *pdev)
+{
+	return XE_DEFAULT_ADMIN_ONLY_PF;
+}
+#endif
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index 844cfafe1ec79..ad2d8f179eb68 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -45,7 +45,7 @@ static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio,
 	u64 residency = 0;
 	int ret;
 
-	ret = xe_pmt_telem_read(to_pci_dev(xe->drm.dev),
+	ret = xe_pmt_telem_read(xe->drm.dev,
 				xe_mmio_read32(mmio, PUNIT_TELEMETRY_GUID),
 				&residency, offset, sizeof(residency));
 	if (ret != sizeof(residency)) {
diff --git a/drivers/gpu/drm/xe/xe_defaults.h b/drivers/gpu/drm/xe/xe_defaults.h
new file mode 100644
index 0000000000000..c8ae1d5f3d601
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_defaults.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+#ifndef _XE_DEFAULTS_H_
+#define _XE_DEFAULTS_H_
+
+#include "xe_device_types.h"
+
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define XE_DEFAULT_GUC_LOG_LEVEL		3
+#else
+#define XE_DEFAULT_GUC_LOG_LEVEL		1
+#endif
+
+#define XE_DEFAULT_PROBE_DISPLAY		IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
+#define XE_DEFAULT_VRAM_BAR_SIZE		0
+#define XE_DEFAULT_FORCE_PROBE			CONFIG_DRM_XE_FORCE_PROBE
+#define XE_DEFAULT_MAX_VFS			~0
+#define XE_DEFAULT_MAX_VFS_STR			"unlimited"
+#define XE_DEFAULT_ADMIN_ONLY_PF		false
+#define XE_DEFAULT_WEDGED_MODE			XE_WEDGED_MODE_UPON_CRITICAL_ERROR
+#define XE_DEFAULT_WEDGED_MODE_STR		"upon-critical-error"
+#define XE_DEFAULT_SVM_NOTIFIER_SIZE		512
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c
index cf41bb6d21725..558a1a9841a09 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.c
+++ b/drivers/gpu/drm/xe/xe_devcoredump.c
@@ -356,7 +356,7 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 
 	xe_engine_snapshot_capture_for_queue(q);
 
-	queue_work(system_unbound_wq, &ss->work);
+	queue_work(system_dfl_wq, &ss->work);
 
 	dma_fence_end_signalling(cookie);
 }
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 52ee24e9cd3af..e77a3a3db73db 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -26,6 +26,7 @@
 #include "xe_bo.h"
 #include "xe_bo_evict.h"
 #include "xe_debugfs.h"
+#include "xe_defaults.h"
 #include "xe_devcoredump.h"
 #include "xe_device_sysfs.h"
 #include "xe_dma_buf.h"
@@ -455,16 +456,16 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 			      xe->drm.anon_inode->i_mapping,
 			      xe->drm.vma_offset_manager, 0);
 	if (WARN_ON(err))
-		goto err;
+		return ERR_PTR(err);
 
 	xe_bo_dev_init(&xe->bo_device);
 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	err = xe_shrinker_create(xe);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	xe->info.devid = pdev->device;
 	xe->info.revid = pdev->revision;
@@ -474,7 +475,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 
 	err = xe_irq_init(xe);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	xe_validation_device_init(&xe->val);
 
@@ -484,7 +485,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 
 	err = xe_pagemap_shrinker_create(xe);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
 
@@ -503,13 +504,13 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 
 	err = xe_bo_pinned_init(xe);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
 						       WQ_MEM_RECLAIM);
 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
-	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
-	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
+	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0);
+	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU, 0);
 	if (!xe->ordered_wq || !xe->unordered_wq ||
 	    !xe->preempt_fence_wq || !xe->destroy_wq) {
 		/*
@@ -517,18 +518,14 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 		 * drmm_add_action_or_reset register above
 		 */
 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
-		err = -ENOMEM;
-		goto err;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
 	if (err)
-		goto err;
+		return ERR_PTR(err);
 
 	return xe;
-
-err:
-	return ERR_PTR(err);
 }
 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
 
@@ -743,7 +740,7 @@ int xe_device_probe_early(struct xe_device *xe)
 	assert_lmem_ready(xe);
 
 	xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ?
-			  XE_WEDGED_MODE_DEFAULT : xe_modparam.wedged_mode;
+			  XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode;
 	drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n",
 		xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode));
 
@@ -1077,10 +1074,7 @@ static void tdf_request_sync(struct xe_device *xe)
 	struct xe_gt *gt;
 	u8 id;
 
-	for_each_gt(gt, xe, id) {
-		if (xe_gt_is_media_type(gt))
-			continue;
-
+	for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) {
 		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 		if (!fw_ref.domains)
 			return;
@@ -1311,7 +1305,8 @@ void xe_device_declare_wedged(struct xe_device *xe)
 		xe->needs_flr_on_fini = true;
 		drm_err(&xe->drm,
 			"CRITICAL: Xe has declared device %s as wedged.\n"
-			"IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
+			"IOCTLs and executions are blocked.\n"
+			"For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n"
 			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
 			dev_name(xe->drm.dev));
 	}
@@ -1374,3 +1369,28 @@ const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode)
 		return "<invalid>";
 	}
 }
+
+/**
+ * xe_device_asid_to_vm() - Find VM from ASID
+ * @xe: the &xe_device
+ * @asid: Address space ID
+ *
+ * Find a VM from ASID and take a reference to VM which caller must drop.
+ * Reclaim safe.
+ *
+ * Return: VM on success, ERR_PTR on failure
+ */
+struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid)
+{
+	struct xe_vm *vm;
+
+	down_read(&xe->usm.lock);
+	vm = xa_load(&xe->usm.asid_to_vm, asid);
+	if (vm)
+		xe_vm_get(vm);
+	else
+		vm = ERR_PTR(-EINVAL);
+	up_read(&xe->usm.lock);
+
+	return vm;
+}
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index 3740143790db2..c4d2670026611 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -12,6 +12,8 @@
 #include "xe_gt_types.h"
 #include "xe_sriov.h"
 
+struct xe_vm;
+
 static inline struct xe_device *to_xe_device(const struct drm_device *dev)
 {
 	return container_of(dev, struct xe_device, drm);
@@ -60,13 +62,6 @@ static inline struct xe_tile *xe_device_get_root_tile(struct xe_device *xe)
 	return &xe->tiles[0];
 }
 
-/*
- * Highest GT/tile count for any platform.  Used only for memory allocation
- * sizing.  Any logic looping over GTs or mapping userspace GT IDs into GT
- * structures should use the per-platform xe->info.max_gt_per_tile instead.
- */
-#define XE_MAX_GT_PER_TILE 2
-
 static inline struct xe_gt *xe_device_get_gt(struct xe_device *xe, u8 gt_id)
 {
 	struct xe_tile *tile;
@@ -114,6 +109,11 @@ static inline struct xe_gt *xe_root_mmio_gt(struct xe_device *xe)
 	return xe_device_get_root_tile(xe)->primary_gt;
 }
 
+static inline struct xe_mmio *xe_root_tile_mmio(struct xe_device *xe)
+{
+	return &xe->tiles[0].mmio;
+}
+
 static inline bool xe_device_uc_enabled(struct xe_device *xe)
 {
 	return !xe->info.force_execlist;
@@ -131,6 +131,10 @@ static inline bool xe_device_uc_enabled(struct xe_device *xe)
 	for ((id__) = 0; (id__) < (xe__)->info.tile_count * (xe__)->info.max_gt_per_tile; (id__)++) \
 		for_each_if((gt__) = xe_device_get_gt((xe__), (id__)))
 
+#define for_each_gt_with_type(gt__, xe__, id__, typemask__) \
+	for_each_gt((gt__), (xe__), (id__)) \
+		for_each_if((typemask__) & BIT((gt__)->info.type))
+
 #define for_each_gt_on_tile(gt__, tile__, id__) \
 	for_each_gt((gt__), (tile__)->xe, (id__)) \
 		for_each_if((gt__)->tile == (tile__))
@@ -204,6 +208,8 @@ int xe_is_injection_active(void);
 
 bool xe_is_xe_file(const struct file *file);
 
+struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid);
+
 /*
  * Occasionally it is seen that the G2H worker starts running after a delay of more than
  * a second even after being queued and activated by the Linux workqueue subsystem. This
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index f689766adcb14..615218d775b14 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -13,11 +13,9 @@
 #include <drm/ttm/ttm_device.h>
 
 #include "xe_devcoredump_types.h"
+#include "xe_drm_ras_types.h"
 #include "xe_heci_gsc.h"
 #include "xe_late_bind_fw_types.h"
-#include "xe_lmtt_types.h"
-#include "xe_memirq_types.h"
-#include "xe_mert.h"
 #include "xe_oa_types.h"
 #include "xe_pagefault_types.h"
 #include "xe_platform_types.h"
@@ -29,14 +27,13 @@
 #include "xe_sriov_vf_ccs_types.h"
 #include "xe_step_types.h"
 #include "xe_survivability_mode_types.h"
-#include "xe_tile_sriov_vf_types.h"
+#include "xe_tile_types.h"
 #include "xe_validation.h"
 
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 #define TEST_VM_OPS_ERROR
 #endif
 
-struct dram_info;
 struct drm_pagemap_shrinker;
 struct intel_display;
 struct intel_dg_nvm_dev;
@@ -62,9 +59,6 @@ enum xe_wedged_mode {
 	XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET = 2,
 };
 
-#define XE_WEDGED_MODE_DEFAULT		XE_WEDGED_MODE_UPON_CRITICAL_ERROR
-#define XE_WEDGED_MODE_DEFAULT_STR	"upon-critical-error"
-
 #define XE_BO_INVALID_OFFSET	LONG_MAX
 
 #define GRAPHICS_VER(xe) ((xe)->info.graphics_verx100 / 100)
@@ -79,179 +73,14 @@ enum xe_wedged_mode {
 #define XE_GT1		1
 #define XE_MAX_TILES_PER_DEVICE	(XE_GT1 + 1)
 
-#define XE_MAX_ASID	(BIT(20))
-
-#define IS_PLATFORM_STEP(_xe, _platform, min_step, max_step)	\
-	((_xe)->info.platform == (_platform) &&			\
-	 (_xe)->info.step.graphics >= (min_step) &&		\
-	 (_xe)->info.step.graphics < (max_step))
-#define IS_SUBPLATFORM_STEP(_xe, _platform, sub, min_step, max_step)	\
-	((_xe)->info.platform == (_platform) &&				\
-	 (_xe)->info.subplatform == (sub) &&				\
-	 (_xe)->info.step.graphics >= (min_step) &&			\
-	 (_xe)->info.step.graphics < (max_step))
-
-#define tile_to_xe(tile__)								\
-	_Generic(tile__,								\
-		 const struct xe_tile * : (const struct xe_device *)((tile__)->xe),	\
-		 struct xe_tile * : (tile__)->xe)
-
-/**
- * struct xe_mmio - register mmio structure
- *
- * Represents an MMIO region that the CPU may use to access registers.  A
- * region may share its IO map with other regions (e.g., all GTs within a
- * tile share the same map with their parent tile, but represent different
- * subregions of the overall IO space).
- */
-struct xe_mmio {
-	/** @tile: Backpointer to tile, used for tracing */
-	struct xe_tile *tile;
-
-	/** @regs: Map used to access registers. */
-	void __iomem *regs;
-
-	/**
-	 * @sriov_vf_gt: Backpointer to GT.
-	 *
-	 * This pointer is only set for GT MMIO regions and only when running
-	 * as an SRIOV VF structure
-	 */
-	struct xe_gt *sriov_vf_gt;
-
-	/**
-	 * @regs_size: Length of the register region within the map.
-	 *
-	 * The size of the iomap set in *regs is generally larger than the
-	 * register mmio space since it includes unused regions and/or
-	 * non-register regions such as the GGTT PTEs.
-	 */
-	size_t regs_size;
-
-	/** @adj_limit: adjust MMIO address if address is below this value */
-	u32 adj_limit;
-
-	/** @adj_offset: offset to add to MMIO address when adjusting */
-	u32 adj_offset;
-};
-
-/**
- * struct xe_tile - hardware tile structure
- *
- * From a driver perspective, a "tile" is effectively a complete GPU, containing
- * an SGunit, 1-2 GTs, and (for discrete platforms) VRAM.
- *
- * Multi-tile platforms effectively bundle multiple GPUs behind a single PCI
- * device and designate one "root" tile as being responsible for external PCI
- * communication.  PCI BAR0 exposes the GGTT and MMIO register space for each
- * tile in a stacked layout, and PCI BAR2 exposes the local memory associated
- * with each tile similarly.  Device-wide interrupts can be enabled/disabled
- * at the root tile, and the MSTR_TILE_INTR register will report which tiles
- * have interrupts that need servicing.
+/*
+ * Highest GT/tile count for any platform.  Used only for memory allocation
+ * sizing.  Any logic looping over GTs or mapping userspace GT IDs into GT
+ * structures should use the per-platform xe->info.max_gt_per_tile instead.
  */
-struct xe_tile {
-	/** @xe: Backpointer to tile's PCI device */
-	struct xe_device *xe;
-
-	/** @id: ID of the tile */
-	u8 id;
-
-	/**
-	 * @primary_gt: Primary GT
-	 */
-	struct xe_gt *primary_gt;
-
-	/**
-	 * @media_gt: Media GT
-	 *
-	 * Only present on devices with media version >= 13.
-	 */
-	struct xe_gt *media_gt;
-
-	/**
-	 * @mmio: MMIO info for a tile.
-	 *
-	 * Each tile has its own 16MB space in BAR0, laid out as:
-	 * * 0-4MB: registers
-	 * * 4MB-8MB: reserved
-	 * * 8MB-16MB: global GTT
-	 */
-	struct xe_mmio mmio;
+#define XE_MAX_GT_PER_TILE 2
 
-	/** @mem: memory management info for tile */
-	struct {
-		/**
-		 * @mem.kernel_vram: kernel-dedicated VRAM info for tile.
-		 *
-		 * Although VRAM is associated with a specific tile, it can
-		 * still be accessed by all tiles' GTs.
-		 */
-		struct xe_vram_region *kernel_vram;
-
-		/**
-		 * @mem.vram: general purpose VRAM info for tile.
-		 *
-		 * Although VRAM is associated with a specific tile, it can
-		 * still be accessed by all tiles' GTs.
-		 */
-		struct xe_vram_region *vram;
-
-		/** @mem.ggtt: Global graphics translation table */
-		struct xe_ggtt *ggtt;
-
-		/**
-		 * @mem.kernel_bb_pool: Pool from which batchbuffers are allocated.
-		 *
-		 * Media GT shares a pool with its primary GT.
-		 */
-		struct xe_sa_manager *kernel_bb_pool;
-
-		/**
-		 * @mem.reclaim_pool: Pool for PRLs allocated.
-		 *
-		 * Only main GT has page reclaim list allocations.
-		 */
-		struct xe_sa_manager *reclaim_pool;
-	} mem;
-
-	/** @sriov: tile level virtualization data */
-	union {
-		struct {
-			/** @sriov.pf.lmtt: Local Memory Translation Table. */
-			struct xe_lmtt lmtt;
-		} pf;
-		struct {
-			/** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */
-			struct xe_ggtt_node *ggtt_balloon[2];
-			/** @sriov.vf.self_config: VF configuration data */
-			struct xe_tile_sriov_vf_selfconfig self_config;
-		} vf;
-	} sriov;
-
-	/** @memirq: Memory Based Interrupts. */
-	struct xe_memirq memirq;
-
-	/** @csc_hw_error_work: worker to report CSC HW errors */
-	struct work_struct csc_hw_error_work;
-
-	/** @pcode: tile's PCODE */
-	struct {
-		/** @pcode.lock: protecting tile's PCODE mailbox data */
-		struct mutex lock;
-	} pcode;
-
-	/** @migrate: Migration helper for vram blits and clearing */
-	struct xe_migrate *migrate;
-
-	/** @sysfs: sysfs' kobj used by xe_tile_sysfs */
-	struct kobject *sysfs;
-
-	/** @debugfs: debugfs directory associated with this tile */
-	struct dentry *debugfs;
-
-	/** @mert: MERT-related data */
-	struct xe_mert mert;
-};
+#define XE_MAX_ASID	(BIT(20))
 
 /**
  * struct xe_device - Top level struct of Xe device
@@ -300,6 +129,8 @@ struct xe_device {
 		u8 tile_count;
 		/** @info.max_gt_per_tile: Number of GT IDs allocated to each tile */
 		u8 max_gt_per_tile;
+		/** @info.multi_lrc_mask: bitmask of engine classes which support multi-lrc */
+		u8 multi_lrc_mask;
 		/** @info.gt_count: Total number of GTs for entire device */
 		u8 gt_count;
 		/** @info.vm_max_level: Max VM level */
@@ -313,6 +144,8 @@ struct xe_device {
 
 		/** @info.force_execlist: Forced execlist submission */
 		u8 force_execlist:1;
+		/** @info.has_access_counter: Device supports access counter */
+		u8 has_access_counter:1;
 		/** @info.has_asid: Has address space ID */
 		u8 has_asid:1;
 		/** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */
@@ -353,6 +186,8 @@ struct xe_device {
 		u8 has_pre_prod_wa:1;
 		/** @info.has_pxp: Device has PXP support */
 		u8 has_pxp:1;
+		/** @info.has_ctx_tlb_inval: Has context based TLB invalidations */
+		u8 has_ctx_tlb_inval:1;
 		/** @info.has_range_tlb_inval: Has range based TLB invalidations */
 		u8 has_range_tlb_inval:1;
 		/** @info.has_soc_remapper_sysctrl: Has SoC remapper system controller */
@@ -559,10 +394,12 @@ struct xe_device {
 		const struct xe_pat_table_entry *table;
 		/** @pat.n_entries: Number of PAT entries */
 		int n_entries;
-		/** @pat.ats_entry: PAT entry for PCIe ATS responses */
+		/** @pat.pat_ats: PAT entry for PCIe ATS responses */
 		const struct xe_pat_table_entry *pat_ats;
-		/** @pat.pta_entry: PAT entry for page table accesses */
-		const struct xe_pat_table_entry *pat_pta;
+		/** @pat.pat_primary_pta: primary GT PAT entry for page table accesses */
+		const struct xe_pat_table_entry *pat_primary_pta;
+		/** @pat.pat_media_pta: media GT PAT entry for page table accesses */
+		const struct xe_pat_table_entry *pat_media_pta;
 		u32 idx[__XE_CACHE_LEVEL_COUNT];
 	} pat;
 
@@ -665,6 +502,9 @@ struct xe_device {
 	/** @pmu: performance monitoring unit */
 	struct xe_pmu pmu;
 
+	/** @ras: RAS structure for device */
+	struct xe_drm_ras ras;
+
 	/** @i2c: I2C host controller */
 	struct xe_i2c *i2c;
 
diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules
index 55ba01bc8f38a..d129cddb6ead4 100644
--- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules
@@ -3,3 +3,4 @@
 		PLATFORM(PANTHERLAKE)
 22019338487_display	PLATFORM(LUNARLAKE)
 14022085890	SUBPLATFORM(BATTLEMAGE, G21)
+14026539277	PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0)
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 7c74a31d44860..ea370cd373e98 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -56,14 +56,10 @@ static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
 	bool allow_vram = true;
 	int ret;
 
-	if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) {
-		allow_vram = false;
-	} else {
-		list_for_each_entry(attach, &dmabuf->attachments, node) {
-			if (!attach->peer2peer) {
-				allow_vram = false;
-				break;
-			}
+	list_for_each_entry(attach, &dmabuf->attachments, node) {
+		if (!attach->peer2peer) {
+			allow_vram = false;
+			break;
 		}
 	}
 
@@ -287,7 +283,7 @@ static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
 
 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
 	.allow_peer2peer = true,
-	.move_notify = xe_dma_buf_move_notify
+	.invalidate_mappings = xe_dma_buf_move_notify
 };
 
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c
new file mode 100644
index 0000000000000..e07dc23a155ed
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_drm_ras.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/bitmap.h>
+
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+#include <drm/drm_ras.h>
+
+#include "xe_device_types.h"
+#include "xe_drm_ras.h"
+
+static const char * const error_components[] = DRM_XE_RAS_ERROR_COMPONENT_NAMES;
+static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;
+
+static int hw_query_error_counter(struct xe_drm_ras_counter *info,
+				  u32 error_id, const char **name, u32 *val)
+{
+	if (!info || !info[error_id].name)
+		return -ENOENT;
+
+	*name = info[error_id].name;
+	*val = atomic_read(&info[error_id].counter);
+
+	return 0;
+}
+
+static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_id,
+					     const char **name, u32 *val)
+{
+	struct xe_device *xe = ep->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE];
+
+	return hw_query_error_counter(info, error_id, name, val);
+}
+
+static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id,
+					   const char **name, u32 *val)
+{
+	struct xe_device *xe = ep->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
+
+	return hw_query_error_counter(info, error_id, name, val);
+}
+
+static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe)
+{
+	struct xe_drm_ras_counter *counter;
+	int i;
+
+	counter = kcalloc(DRM_XE_RAS_ERR_COMP_MAX, sizeof(*counter), GFP_KERNEL);
+	if (!counter)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; i < DRM_XE_RAS_ERR_COMP_MAX; i++) {
+		if (!error_components[i])
+			continue;
+
+		counter[i].name = error_components[i];
+		atomic_set(&counter[i].counter, 0);
+	}
+
+	return counter;
+}
+
+static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node,
+			      const enum drm_xe_ras_error_severity severity)
+{
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+	struct xe_drm_ras *ras = &xe->ras;
+	const char *device_name;
+
+	device_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d",
+				pci_domain_nr(pdev->bus), pdev->bus->number,
+				PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
+
+	if (!device_name)
+		return -ENOMEM;
+
+	node->device_name = device_name;
+	node->node_name = error_severity[severity];
+	node->type = DRM_RAS_NODE_TYPE_ERROR_COUNTER;
+	node->error_counter_range.first = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE;
+	node->error_counter_range.last = DRM_XE_RAS_ERR_COMP_MAX - 1;
+	node->priv = xe;
+
+	ras->info[severity] = allocate_and_copy_counters(xe);
+	if (IS_ERR(ras->info[severity]))
+		return PTR_ERR(ras->info[severity]);
+
+	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
+		node->query_error_counter = query_correctable_error_counter;
+	else
+		node->query_error_counter = query_uncorrectable_error_counter;
+
+	return 0;
+}
+
+static void cleanup_node_param(struct xe_drm_ras *ras, const enum drm_xe_ras_error_severity severity)
+{
+	struct drm_ras_node *node = &ras->node[severity];
+
+	kfree(ras->info[severity]);
+	ras->info[severity] = NULL;
+
+	kfree(node->device_name);
+	node->device_name = NULL;
+}
+
+static int register_nodes(struct xe_device *xe)
+{
+	struct xe_drm_ras *ras = &xe->ras;
+	int i;
+
+	for_each_error_severity(i) {
+		struct drm_ras_node *node = &ras->node[i];
+		int ret;
+
+		ret = assign_node_params(xe, node, i);
+		if (ret) {
+			cleanup_node_param(ras, i);
+			return ret;
+		}
+
+		ret = drm_ras_node_register(node);
+		if (ret) {
+			cleanup_node_param(ras, i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void xe_drm_ras_unregister_nodes(struct drm_device *device, void *arg)
+{
+	struct xe_device *xe = arg;
+	struct xe_drm_ras *ras = &xe->ras;
+	int i;
+
+	for_each_error_severity(i) {
+		struct drm_ras_node *node = &ras->node[i];
+
+		drm_ras_node_unregister(node);
+		cleanup_node_param(ras, i);
+	}
+}
+
+/**
+ * xe_drm_ras_init() - Initialize DRM RAS
+ * @xe: xe device instance
+ *
+ * Allocate and register DRM RAS nodes per device
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int xe_drm_ras_init(struct xe_device *xe)
+{
+	struct xe_drm_ras *ras = &xe->ras;
+	struct drm_ras_node *node;
+	int err;
+
+	node = drmm_kcalloc(&xe->drm, DRM_XE_RAS_ERR_SEV_MAX, sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	ras->node = node;
+
+	err = register_nodes(xe);
+	if (err) {
+		drm_err(&xe->drm, "Failed to register DRM RAS nodes (%pe)\n", ERR_PTR(err));
+		return err;
+	}
+
+	err = drmm_add_action_or_reset(&xe->drm, xe_drm_ras_unregister_nodes, xe);
+	if (err) {
+		drm_err(&xe->drm, "Failed to add action for Xe DRM RAS (%pe)\n", ERR_PTR(err));
+		return err;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h
new file mode 100644
index 0000000000000..5cc8f0124411a
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_drm_ras.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+#ifndef XE_DRM_RAS_H_
+#define XE_DRM_RAS_H_
+
+struct xe_device;
+
+#define for_each_error_severity(i)	\
+	for (i = 0; i < DRM_XE_RAS_ERR_SEV_MAX; i++)
+
+int xe_drm_ras_init(struct xe_device *xe);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_drm_ras_types.h b/drivers/gpu/drm/xe/xe_drm_ras_types.h
new file mode 100644
index 0000000000000..8d729ad6a264d
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_drm_ras_types.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_DRM_RAS_TYPES_H_
+#define _XE_DRM_RAS_TYPES_H_
+
+#include <linux/atomic.h>
+#include <drm/xe_drm.h>
+
+struct drm_ras_node;
+
+/* Error categories reported by hardware */
+enum hardware_error {
+	HARDWARE_ERROR_CORRECTABLE = 0,
+	HARDWARE_ERROR_NONFATAL,
+	HARDWARE_ERROR_FATAL,
+	HARDWARE_ERROR_MAX
+};
+
+/**
+ * struct xe_drm_ras_counter - XE RAS counter
+ *
+ * This structure contains error component and counter information
+ */
+struct xe_drm_ras_counter {
+	/** @name: error component name */
+	const char *name;
+
+	/** @counter: count of error */
+	atomic_t counter;
+};
+
+/**
+ * struct xe_drm_ras - XE DRM RAS structure
+ *
+ * This structure has details of error counters
+ */
+struct xe_drm_ras {
+	/** @node: DRM RAS node */
+	struct drm_ras_node *node;
+
+	/** @info: info array for all types of errors */
+	struct xe_drm_ras_counter *info[DRM_XE_RAS_ERR_SEV_MAX];
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c
index 39723928a0199..c34408cfd292c 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.c
+++ b/drivers/gpu/drm/xe/xe_eu_stall.c
@@ -442,9 +442,9 @@ static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 ins
 	 * On Xe2 and later GPUs, the bit has to be cleared by writing 0 to it.
 	 */
 	if (GRAPHICS_VER(xe) >= 20)
-		write_ptr_reg = _MASKED_BIT_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
+		write_ptr_reg = REG_MASKED_FIELD_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
 	else
-		write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
+		write_ptr_reg = REG_MASKED_FIELD_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
 
 	xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, write_ptr_reg, group, instance);
 }
@@ -504,7 +504,7 @@ static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream,
 	/* Read pointer can overflow into one additional bit */
 	read_ptr &= (buf_size << 1) - 1;
 	read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, (read_ptr >> 6));
-	read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
+	read_ptr_reg = REG_MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
 	xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance);
 	xecore_buf->read = read_ptr;
 	trace_xe_eu_stall_data_read(group, instance, read_ptr, write_ptr,
@@ -674,7 +674,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream)
 
 	if (XE_GT_WA(gt, 22016596838))
 		xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
-					  _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
+					  REG_MASKED_FIELD_ENABLE(DISABLE_DOP_GATING));
 
 	for_each_dss_steering(xecore, gt, group, instance) {
 		write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance);
@@ -683,7 +683,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream)
 			clear_dropped_eviction_line_bit(gt, group, instance);
 		write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg);
 		read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, write_ptr);
-		read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
+		read_ptr_reg = REG_MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
 		/* Initialize the read pointer to the write pointer */
 		xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, group, instance);
 		write_ptr <<= 6;
@@ -695,10 +695,10 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream)
 	stream->data_drop.reported_to_user = false;
 	bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS);
 
-	reg_value = _MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE,
-				  REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) |
-				  REG_FIELD_PREP(EUSTALL_SAMPLE_RATE,
-						 stream->sampling_rate_mult));
+	reg_value = REG_MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE,
+				     REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) |
+				     REG_FIELD_PREP(EUSTALL_SAMPLE_RATE,
+						    stream->sampling_rate_mult));
 	xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_CTRL, reg_value);
 	/* GGTT addresses can never be > 32 bits */
 	xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE_UPPER, 0);
@@ -830,7 +830,7 @@ static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream)
 
 	if (XE_GT_WA(gt, 22016596838))
 		xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
-					  _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
+					  REG_MASKED_FIELD_DISABLE(DISABLE_DOP_GATING));
 
 	xe_force_wake_put(gt_to_fw(gt), stream->fw_ref);
 	xe_pm_runtime_put(gt_to_xe(gt));
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 8ecdf949f9e4c..b287d0e0e60a4 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -152,8 +152,10 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
 	if (xe_exec_queue_is_multi_queue(q))
 		xe_exec_queue_group_cleanup(q);
 
-	if (q->vm)
+	if (q->vm) {
+		xe_vm_remove_exec_queue(q->vm, q);
 		xe_vm_put(q->vm);
+	}
 
 	if (q->xef)
 		xe_file_put(q->xef);
@@ -224,9 +226,12 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	q->ring_ops = gt->ring_ops[hwe->class];
 	q->ops = gt->exec_queue_ops;
 	INIT_LIST_HEAD(&q->lr.link);
+	INIT_LIST_HEAD(&q->vm_exec_queue_link);
 	INIT_LIST_HEAD(&q->multi_gt_link);
 	INIT_LIST_HEAD(&q->hw_engine_group_link);
 	INIT_LIST_HEAD(&q->pxp.link);
+	spin_lock_init(&q->multi_queue.lock);
+	spin_lock_init(&q->lrc_lookup_lock);
 	q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;
 
 	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
@@ -266,6 +271,56 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	return q;
 }
 
+static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u16 idx)
+{
+	xe_assert(gt_to_xe(q->gt), idx < q->width);
+
+	scoped_guard(spinlock, &q->lrc_lookup_lock)
+		q->lrc[idx] = lrc;
+}
+
+/**
+ * xe_exec_queue_get_lrc() - Get the LRC from exec queue.
+ * @q: The exec queue instance.
+ * @idx: Index within multi-LRC array.
+ *
+ * Retrieves LRC of given index for the exec queue under lock
+ * and takes reference.
+ *
+ * Return: Pointer to LRC on success, error on failure, NULL on
+ * lookup failure.
+ */
+struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx)
+{
+	struct xe_lrc *lrc;
+
+	xe_assert(gt_to_xe(q->gt), idx < q->width);
+
+	scoped_guard(spinlock, &q->lrc_lookup_lock) {
+		lrc = q->lrc[idx];
+		if (lrc)
+			xe_lrc_get(lrc);
+	}
+
+	return lrc;
+}
+
+/**
+ * xe_exec_queue_lrc() - Get the LRC from exec queue.
+ * @q: The exec queue instance.
+ *
+ * Retrieves the primary LRC for the exec queue. Note that this function
+ * returns only the first LRC instance, even when multiple parallel LRCs
+ * are configured. This function does not increment reference count,
+ * so the reference can be just forgotten after use.
+ *
+ * Return: Pointer to LRC on success, error on failure
+ */
+struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
+{
+	return q->lrc[0];
+}
+
 static void __xe_exec_queue_fini(struct xe_exec_queue *q)
 {
 	int i;
@@ -298,6 +353,9 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
 	if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL))
 		flags |= XE_LRC_CREATE_USER_CTX;
 
+	if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX)
+		flags |= XE_LRC_DISABLE_STATE_CACHE_PERF_FIX;
+
 	err = q->ops->init(q);
 	if (err)
 		return err;
@@ -313,18 +371,28 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
 	 * from the moment vCPU resumes execution.
 	 */
 	for (i = 0; i < q->width; ++i) {
-		struct xe_lrc *lrc;
+		struct xe_lrc *__lrc = NULL;
+		int marker;
 
-		xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
-		lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
-				    xe_lrc_ring_size(), q->msix_vec, flags);
-		if (IS_ERR(lrc)) {
-			err = PTR_ERR(lrc);
-			goto err_lrc;
-		}
+		do {
+			struct xe_lrc *lrc;
 
-		/* Pairs with READ_ONCE to xe_exec_queue_contexts_hwsp_rebase */
-		WRITE_ONCE(q->lrc[i], lrc);
+			marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
+
+			lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
+					    xe_lrc_ring_size(), q->msix_vec, flags);
+			if (IS_ERR(lrc)) {
+				err = PTR_ERR(lrc);
+				goto err_lrc;
+			}
+
+			xe_exec_queue_set_lrc(q, lrc, i);
+
+			if (__lrc)
+				xe_lrc_put(__lrc);
+			__lrc = lrc;
+
+		} while (marker != xe_vf_migration_fixups_complete_count(q->gt));
 	}
 
 	return 0;
@@ -334,6 +402,20 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
 	return err;
 }
 
+/**
+ * xe_exec_queue_create() - Create an exec queue
+ * @xe: Xe device
+ * @vm: VM for the exec queue
+ * @logical_mask: Logical mask of HW engines
+ * @width: Width of the exec queue (number of LRCs)
+ * @hwe: Hardware engine
+ * @flags: Exec queue creation flags
+ * @extensions: Extensions for exec queue creation
+ *
+ * Create an exec queue (allocate and initialize) with the specified parameters
+ *
+ * Return: Pointer to the created exec queue on success, ERR_PTR on failure
+ */
 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
 					   u32 logical_mask, u16 width,
 					   struct xe_hw_engine *hwe, u32 flags,
@@ -377,6 +459,19 @@ struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *v
 }
 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO);
 
+/**
+ * xe_exec_queue_create_class() - Create an exec queue for a specific engine class
+ * @xe: Xe device
+ * @gt: GT for the exec queue
+ * @vm: VM for the exec queue
+ * @class: Engine class
+ * @flags: Exec queue creation flags
+ * @extensions: Extensions for exec queue creation
+ *
+ * Create an exec queue for the specified engine class.
+ *
+ * Return: Pointer to the created exec queue on success, ERR_PTR on failure
+ */
 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
 						 struct xe_vm *vm,
 						 enum xe_engine_class class,
@@ -468,6 +563,14 @@ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
 }
 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
 
+/**
+ * xe_exec_queue_destroy() - Destroy an exec queue
+ * @ref: Reference count of the exec queue
+ *
+ * Called when the last reference to the exec queue is dropped.
+ * Cleans up all resources associated with the exec queue.
+ * This function should not be called directly; use xe_exec_queue_put() instead.
+ */
 void xe_exec_queue_destroy(struct kref *ref)
 {
 	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
@@ -500,6 +603,14 @@ void xe_exec_queue_destroy(struct kref *ref)
 	q->ops->destroy(q);
 }
 
+/**
+ * xe_exec_queue_fini() - Finalize an exec queue
+ * @q: The exec queue
+ *
+ * Finalizes the exec queue by updating run ticks, releasing LRC references,
+ * and freeing the queue structure. This is called after the queue has been
+ * destroyed and all references have been dropped.
+ */
 void xe_exec_queue_fini(struct xe_exec_queue *q)
 {
 	/*
@@ -514,6 +625,14 @@ void xe_exec_queue_fini(struct xe_exec_queue *q)
 	__xe_exec_queue_free(q);
 }
 
+/**
+ * xe_exec_queue_assign_name() - Assign a name to an exec queue
+ * @q: The exec queue
+ * @instance: Instance number for the engine
+ *
+ * Assigns a human-readable name to the exec queue based on its engine class
+ * and instance number (e.g., "rcs0", "vcs1", "bcs2").
+ */
 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
 {
 	switch (q->class) {
@@ -540,6 +659,15 @@ void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
 	}
 }
 
+/**
+ * xe_exec_queue_lookup() - Look up an exec queue by ID
+ * @xef: Xe file private data
+ * @id: Exec queue ID
+ *
+ * Looks up an exec queue by its ID and increments its reference count.
+ *
+ * Return: Pointer to the exec queue if found, NULL otherwise
+ */
 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
 {
 	struct xe_exec_queue *q;
@@ -553,6 +681,14 @@ struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
 	return q;
 }
 
+/**
+ * xe_exec_queue_device_get_max_priority() - Get maximum priority for an exec queues
+ * @xe: Xe device
+ *
+ * Returns the maximum priority level that can be assigned to an exec queues.
+ *
+ * Return: Maximum priority level (HIGH if CAP_SYS_NICE, NORMAL otherwise)
+ */
 enum xe_exec_queue_priority
 xe_exec_queue_device_get_max_priority(struct xe_device *xe)
 {
@@ -845,6 +981,17 @@ static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_e
 	return q->ops->set_multi_queue_priority(q, value);
 }
 
+static int exec_queue_set_state_cache_perf_fix(struct xe_device *xe, struct xe_exec_queue *q,
+					       u64 value)
+{
+	if (XE_IOCTL_DBG(xe, q->class != XE_ENGINE_CLASS_RENDER))
+		return -EOPNOTSUPP;
+
+	q->flags |= value != 0 ? EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX : 0;
+
+	return 0;
+}
+
 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
 					     struct xe_exec_queue *q,
 					     u64 value);
@@ -857,8 +1004,21 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
 							exec_queue_set_multi_queue_priority,
+	[DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX] =
+							exec_queue_set_state_cache_perf_fix,
 };
 
+/**
+ * xe_exec_queue_set_property_ioctl() - Set a property on an exec queue
+ * @dev: DRM device
+ * @data: IOCTL data
+ * @file: DRM file
+ *
+ * Allows setting properties on an existing exec queue. Currently only
+ * supports setting multi-queue priority.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
 int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
 				     struct drm_file *file)
 {
@@ -941,7 +1101,8 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe,
 			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
 			 ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
 			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
-			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY &&
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX))
 		return -EINVAL;
 
 	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
@@ -1083,6 +1244,18 @@ static bool has_sched_groups(struct xe_gt *gt)
 	return false;
 }
 
+/**
+ * xe_exec_queue_create_ioctl() - Create an exec queue via IOCTL
+ * @dev: DRM device
+ * @data: IOCTL data
+ * @file: DRM file
+ *
+ * Creates a new exec queue based on user-provided parameters. Supports
+ * creating VM bind queues, regular exec queues, multi-lrc exec queues
+ * and multi-queue groups.
+ *
+ * Return: 0 on success with exec_queue_id filled in, negative error code on failure
+ */
 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 			       struct drm_file *file)
 {
@@ -1179,6 +1352,11 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 		if (XE_IOCTL_DBG(xe, !hwe))
 			return -EINVAL;
 
+		/* multi-lrc is only supported on select engine classes */
+		if (XE_IOCTL_DBG(xe, args->width > 1 &&
+				 !(xe->info.multi_lrc_mask & BIT(hwe->class))))
+			return -EOPNOTSUPP;
+
 		vm = xe_vm_lookup(xef, args->vm_id);
 		if (XE_IOCTL_DBG(xe, !vm))
 			return -ENOENT;
@@ -1232,6 +1410,8 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 	}
 
 	q->xef = xe_file_get(xef);
+	if (eci[0].engine_class != DRM_XE_ENGINE_CLASS_VM_BIND)
+		xe_vm_add_exec_queue(vm, q);
 
 	/* user id alloc must always be last in ioctl to prevent UAF */
 	err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
@@ -1252,6 +1432,17 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 	return err;
 }
 
+/**
+ * xe_exec_queue_get_property_ioctl() - Get a property from an exec queue
+ * @dev: DRM device
+ * @data: IOCTL data
+ * @file: DRM file
+ *
+ * Retrieves property values from an existing exec queue. Currently supports
+ * getting the ban/reset status.
+ *
+ * Return: 0 on success with value filled in, negative error code on failure
+ */
 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
 				     struct drm_file *file)
 {
@@ -1282,21 +1473,6 @@ int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
 	return ret;
 }
 
-/**
- * xe_exec_queue_lrc() - Get the LRC from exec queue.
- * @q: The exec_queue.
- *
- * Retrieves the primary LRC for the exec queue. Note that this function
- * returns only the first LRC instance, even when multiple parallel LRCs
- * are configured.
- *
- * Return: Pointer to LRC on success, error on failure
- */
-struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
-{
-	return q->lrc[0];
-}
-
 /**
  * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
  * @q: The exec_queue
@@ -1404,6 +1580,16 @@ void xe_exec_queue_kill(struct xe_exec_queue *q)
 	xe_vm_remove_compute_exec_queue(q->vm, q);
 }
 
+/**
+ * xe_exec_queue_destroy_ioctl() - Destroy an exec queue via IOCTL
+ * @dev: DRM device
+ * @data: IOCTL data
+ * @file: DRM file
+ *
+ * Destroys an existing exec queue and releases its reference.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file)
 {
@@ -1656,14 +1842,14 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch)
 	for (i = 0; i < q->width; ++i) {
 		struct xe_lrc *lrc;
 
-		/* Pairs with WRITE_ONCE in __xe_exec_queue_init  */
-		lrc = READ_ONCE(q->lrc[i]);
+		lrc = xe_exec_queue_get_lrc(q, i);
 		if (!lrc)
 			continue;
 
 		xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch);
 		xe_lrc_update_hwctx_regs_with_address(lrc);
 		err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch);
+		xe_lrc_put(lrc);
 		if (err)
 			break;
 	}
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h
index c9e3a7c2d2496..a82d99bd77bcf 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue.h
@@ -160,6 +160,7 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q);
 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch);
 
 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q);
+struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx);
 
 /**
  * xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index f1142e5d3e3eb..8ce78e0b1d502 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -66,6 +66,8 @@ struct xe_exec_queue_group {
 	bool sync_pending;
 	/** @banned: Group banned */
 	bool banned;
+	/** @stopped: Group is stopped, protected by list_lock */
+	bool stopped;
 };
 
 /**
@@ -132,6 +134,8 @@ struct xe_exec_queue {
 #define EXEC_QUEUE_FLAG_LOW_LATENCY		BIT(5)
 /* for migration (kernel copy, clear, bind) jobs */
 #define EXEC_QUEUE_FLAG_MIGRATE			BIT(6)
+/* for programming COMMON_SLICE_CHICKEN3 on first submission */
+#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX	BIT(7)
 
 	/**
 	 * @flags: flags for this exec queue, should statically setup aside from ban
@@ -159,8 +163,13 @@ struct xe_exec_queue {
 		struct xe_exec_queue_group *group;
 		/** @multi_queue.link: Link into group's secondary queues list */
 		struct list_head link;
-		/** @multi_queue.priority: Queue priority within the multi-queue group */
+		/**
+		 * @multi_queue.priority: Queue priority within the multi-queue group.
+		 * It is protected by @multi_queue.lock.
+		 */
 		enum xe_multi_queue_priority priority;
+		/** @multi_queue.lock: Lock for protecting certain members */
+		spinlock_t lock;
 		/** @multi_queue.pos: Position of queue within the multi-queue group */
 		u8 pos;
 		/** @multi_queue.valid: Queue belongs to a multi queue group */
@@ -211,6 +220,9 @@ struct xe_exec_queue {
 		struct dma_fence *last_fence;
 	} tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_COUNT];
 
+	/** @vm_exec_queue_link: Link to track exec queue within a VM's list of exec queues. */
+	struct list_head vm_exec_queue_link;
+
 	/** @pxp: PXP info tracking */
 	struct {
 		/** @pxp.type: PXP session type used by this queue */
@@ -247,6 +259,11 @@ struct xe_exec_queue {
 	u64 tlb_flush_seqno;
 	/** @hw_engine_group_link: link into exec queues in the same hw engine group */
 	struct list_head hw_engine_group_link;
+	/**
+	 * @lrc_lookup_lock: Lock for protecting lrc array access. Only used when
+	 * running in parallel to queue creation is possible.
+	 */
+	spinlock_t lrc_lookup_lock;
 	/** @lrc: logical ring context for this exec queue */
 	struct xe_lrc *lrc[] __counted_by(width);
 };
@@ -301,6 +318,8 @@ struct xe_exec_queue_ops {
 	void (*resume)(struct xe_exec_queue *q);
 	/** @reset_status: check exec queue reset status */
 	bool (*reset_status)(struct xe_exec_queue *q);
+	/** @active: check exec queue is active */
+	bool (*active)(struct xe_exec_queue *q);
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c
index 371983c94a1bc..755a2bff5d7bc 100644
--- a/drivers/gpu/drm/xe/xe_execlist.c
+++ b/drivers/gpu/drm/xe/xe_execlist.c
@@ -47,7 +47,7 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 	struct xe_mmio *mmio = &gt->mmio;
 	struct xe_device *xe = gt_to_xe(gt);
 	u64 lrc_desc;
-	u32 ring_mode = _MASKED_BIT_ENABLE(GFX_DISABLE_LEGACY_MODE);
+	u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE);
 
 	lrc_desc = xe_lrc_descriptor(lrc);
 
@@ -61,7 +61,7 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 
 	if (hwe->class == XE_ENGINE_CLASS_COMPUTE)
 		xe_mmio_write32(mmio, RCU_MODE,
-				_MASKED_BIT_ENABLE(RCU_MODE_CCS_ENABLE));
+				REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE));
 
 	xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail);
 	lrc->ring.old_tail = lrc->ring.tail;
@@ -83,7 +83,7 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 	xe_mmio_read32(mmio, RING_HWS_PGA(hwe->mmio_base));
 
 	if (xe_device_has_msix(gt_to_xe(hwe->gt)))
-		ring_mode |= _MASKED_BIT_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
+		ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
 	xe_mmio_write32(mmio, RING_MODE(hwe->mmio_base), ring_mode);
 
 	xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base),
@@ -421,7 +421,7 @@ static void execlist_exec_queue_kill(struct xe_exec_queue *q)
 static void execlist_exec_queue_destroy(struct xe_exec_queue *q)
 {
 	INIT_WORK(&q->execlist->destroy_async, execlist_exec_queue_destroy_async);
-	queue_work(system_unbound_wq, &q->execlist->destroy_async);
+	queue_work(system_dfl_wq, &q->execlist->destroy_async);
 }
 
 static int execlist_exec_queue_set_priority(struct xe_exec_queue *q,
@@ -468,6 +468,12 @@ static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q)
 	return false;
 }
 
+static bool execlist_exec_queue_active(struct xe_exec_queue *q)
+{
+	/* NIY */
+	return false;
+}
+
 static const struct xe_exec_queue_ops execlist_exec_queue_ops = {
 	.init = execlist_exec_queue_init,
 	.kill = execlist_exec_queue_kill,
@@ -480,6 +486,7 @@ static const struct xe_exec_queue_ops execlist_exec_queue_ops = {
 	.suspend_wait = execlist_exec_queue_suspend_wait,
 	.resume = execlist_exec_queue_resume,
 	.reset_status = execlist_exec_queue_reset_status,
+	.active = execlist_exec_queue_active,
 };
 
 int xe_execlist_init(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_force_wake.c b/drivers/gpu/drm/xe/xe_force_wake.c
index 76e054f314ee1..197e2197bd0a3 100644
--- a/drivers/gpu/drm/xe/xe_force_wake.c
+++ b/drivers/gpu/drm/xe/xe_force_wake.c
@@ -148,12 +148,6 @@ static int domain_sleep_wait(struct xe_gt *gt,
 	return __domain_wait(gt, domain, false);
 }
 
-#define for_each_fw_domain_masked(domain__, mask__, fw__, tmp__) \
-	for (tmp__ = (mask__); tmp__; tmp__ &= ~BIT(ffs(tmp__) - 1)) \
-		for_each_if((domain__ = ((fw__)->domains + \
-					 (ffs(tmp__) - 1))) && \
-					 domain__->reg_ctl.addr)
-
 /**
  * xe_force_wake_get() : Increase the domain refcount
  * @fw: struct xe_force_wake
@@ -266,3 +260,43 @@ void xe_force_wake_put(struct xe_force_wake *fw, unsigned int fw_ref)
 	xe_gt_WARN(gt, ack_fail, "Forcewake domain%s %#x failed to acknowledge sleep request\n",
 		   str_plural(hweight_long(ack_fail)), ack_fail);
 }
+
+const char *xe_force_wake_domain_to_str(enum xe_force_wake_domain_id id)
+{
+	switch (id) {
+	case XE_FW_DOMAIN_ID_GT:
+		return "GT";
+	case XE_FW_DOMAIN_ID_RENDER:
+		return "Render";
+	case XE_FW_DOMAIN_ID_MEDIA:
+		return "Media";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX0:
+		return "VDBox0";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX1:
+		return "VDBox1";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX2:
+		return "VDBox2";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX3:
+		return "VDBox3";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX4:
+		return "VDBox4";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX5:
+		return "VDBox5";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX6:
+		return "VDBox6";
+	case XE_FW_DOMAIN_ID_MEDIA_VDBOX7:
+		return "VDBox7";
+	case XE_FW_DOMAIN_ID_MEDIA_VEBOX0:
+		return "VEBox0";
+	case XE_FW_DOMAIN_ID_MEDIA_VEBOX1:
+		return "VEBox1";
+	case XE_FW_DOMAIN_ID_MEDIA_VEBOX2:
+		return "VEBox2";
+	case XE_FW_DOMAIN_ID_MEDIA_VEBOX3:
+		return "VEBox3";
+	case XE_FW_DOMAIN_ID_GSC:
+		return "GSC";
+	default:
+		return "Unknown";
+	}
+}
diff --git a/drivers/gpu/drm/xe/xe_force_wake.h b/drivers/gpu/drm/xe/xe_force_wake.h
index 1e2198f6a0078..e2721f205d6c5 100644
--- a/drivers/gpu/drm/xe/xe_force_wake.h
+++ b/drivers/gpu/drm/xe/xe_force_wake.h
@@ -19,6 +19,17 @@ unsigned int __must_check xe_force_wake_get(struct xe_force_wake *fw,
 					    enum xe_force_wake_domains domains);
 void xe_force_wake_put(struct xe_force_wake *fw, unsigned int fw_ref);
 
+const char *xe_force_wake_domain_to_str(enum xe_force_wake_domain_id id);
+
+#define for_each_fw_domain_masked(domain__, mask__, fw__, tmp__) \
+	for (tmp__ = (mask__); tmp__; tmp__ &= ~BIT(ffs(tmp__) - 1)) \
+		for_each_if(((domain__) = ((fw__)->domains + \
+					 (ffs(tmp__) - 1))) && \
+					 (domain__)->reg_ctl.addr)
+
+#define for_each_fw_domain(domain__, fw__, tmp__) \
+	for_each_fw_domain_masked((domain__), (fw__)->initialized_domains, (fw__), (tmp__))
+
 static inline int
 xe_force_wake_ref(struct xe_force_wake *fw,
 		  enum xe_force_wake_domains domain)
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index 2bda426a6986a..21071b64b09df 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -66,12 +66,14 @@
  * give us the correct placement for free.
  */
 
+#define XE_GGTT_FLAGS_64K	BIT(0)
+#define XE_GGTT_FLAGS_ONLINE	BIT(1)
+
 /**
  * struct xe_ggtt_node - A node in GGTT.
  *
- * This struct needs to be initialized (only-once) with xe_ggtt_node_init() before any node
- * insertion, reservation, or 'ballooning'.
- * It will, then, be finalized by either xe_ggtt_node_remove() or xe_ggtt_node_deballoon().
+ * This struct is allocated with xe_ggtt_insert_node(,_transform) or xe_ggtt_insert_bo(,_at).
+ * It will be deallocated using xe_ggtt_node_remove().
  */
 struct xe_ggtt_node {
 	/** @ggtt: Back pointer to xe_ggtt where this region will be inserted at */
@@ -84,6 +86,63 @@ struct xe_ggtt_node {
 	bool invalidate_on_remove;
 };
 
+/**
+ * struct xe_ggtt_pt_ops - GGTT Page table operations
+ * Which can vary from platform to platform.
+ */
+struct xe_ggtt_pt_ops {
+	/** @pte_encode_flags: Encode PTE flags for a given BO */
+	u64 (*pte_encode_flags)(struct xe_bo *bo, u16 pat_index);
+
+	/** @ggtt_set_pte: Directly write into GGTT's PTE */
+	xe_ggtt_set_pte_fn ggtt_set_pte;
+
+	/** @ggtt_get_pte: Directly read from GGTT's PTE */
+	u64 (*ggtt_get_pte)(struct xe_ggtt *ggtt, u64 addr);
+};
+
+/**
+ * struct xe_ggtt - Main GGTT struct
+ *
+ * In general, each tile can contains its own Global Graphics Translation Table
+ * (GGTT) instance.
+ */
+struct xe_ggtt {
+	/** @tile: Back pointer to tile where this GGTT belongs */
+	struct xe_tile *tile;
+        /** @start: Start offset of GGTT */
+	u64 start;
+	/** @size: Total usable size of this GGTT */
+	u64 size;
+
+#define XE_GGTT_FLAGS_64K BIT(0)
+	/**
+	 * @flags: Flags for this GGTT
+	 * Acceptable flags:
+	 * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
+	 * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock
+	 *   after init
+	 */
+	unsigned int flags;
+	/** @scratch: Internal object allocation used as a scratch page */
+	struct xe_bo *scratch;
+	/** @lock: Mutex lock to protect GGTT data */
+	struct mutex lock;
+	/**
+	 *  @gsm: The iomem pointer to the actual location of the translation
+	 * table located in the GSM for easy PTE manipulation
+	 */
+	u64 __iomem *gsm;
+	/** @pt_ops: Page Table operations per platform */
+	const struct xe_ggtt_pt_ops *pt_ops;
+	/** @mm: The memory manager used to manage individual GGTT allocations */
+	struct drm_mm mm;
+	/** @access_count: counts GGTT writes */
+	unsigned int access_count;
+	/** @wq: Dedicated unordered work queue to process node removals */
+	struct workqueue_struct *wq;
+};
+
 static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
 {
 	u64 pte = XE_PAGE_PRESENT;
@@ -193,7 +252,7 @@ static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
 static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr)
 {
 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
-	xe_tile_assert(ggtt->tile, addr < ggtt->size);
+	xe_tile_assert(ggtt->tile, addr < ggtt->start + ggtt->size);
 
 	return readq(&ggtt->gsm[addr >> XE_PTE_SHIFT]);
 }
@@ -299,7 +358,7 @@ static void __xe_ggtt_init_early(struct xe_ggtt *ggtt, u64 start, u64 size)
 {
 	ggtt->start = start;
 	ggtt->size = size;
-	drm_mm_init(&ggtt->mm, start, size);
+	drm_mm_init(&ggtt->mm, 0, size);
 }
 
 int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 start, u32 size)
@@ -313,6 +372,8 @@ static void dev_fini_ggtt(void *arg)
 {
 	struct xe_ggtt *ggtt = arg;
 
+	scoped_guard(mutex, &ggtt->lock)
+		ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE;
 	drain_workqueue(ggtt->wq);
 }
 
@@ -347,9 +408,15 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 		ggtt_start = wopcm;
 		ggtt_size = (gsm_size / 8) * (u64)XE_PAGE_SIZE - ggtt_start;
 	} else {
-		/* GGTT is expected to be 4GiB */
-		ggtt_start = wopcm;
-		ggtt_size = SZ_4G - ggtt_start;
+		ggtt_start = xe_tile_sriov_vf_ggtt_base(ggtt->tile);
+		ggtt_size = xe_tile_sriov_vf_ggtt(ggtt->tile);
+
+		if (ggtt_start < wopcm ||
+		    ggtt_start + ggtt_size > GUC_GGTT_TOP) {
+			xe_tile_err(ggtt->tile, "Invalid GGTT configuration: %#llx-%#llx\n",
+				    ggtt_start, ggtt_start + ggtt_size - 1);
+			return -ERANGE;
+		}
 	}
 
 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
@@ -367,7 +434,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 	else
 		ggtt->pt_ops = &xelp_pt_ops;
 
-	ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM, 0);
+	ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM | WQ_PERCPU, 0);
 	if (!ggtt->wq)
 		return -ENOMEM;
 
@@ -377,17 +444,8 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 	if (err)
 		return err;
 
-	err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
-	if (err)
-		return err;
-
-	if (IS_SRIOV_VF(xe)) {
-		err = xe_tile_sriov_vf_prepare_ggtt(ggtt->tile);
-		if (err)
-			return err;
-	}
-
-	return 0;
+	ggtt->flags |= XE_GGTT_FLAGS_ONLINE;
+	return devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
 }
 ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
 
@@ -401,24 +459,26 @@ static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
 	/* Display may have allocated inside ggtt, so be careful with clearing here */
 	mutex_lock(&ggtt->lock);
 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
-		xe_ggtt_clear(ggtt, start, end - start);
+		xe_ggtt_clear(ggtt, ggtt->start + start, end - start);
 
 	xe_ggtt_invalidate(ggtt);
 	mutex_unlock(&ggtt->lock);
 }
 
+static void ggtt_node_fini(struct xe_ggtt_node *node)
+{
+	kfree(node);
+}
+
 static void ggtt_node_remove(struct xe_ggtt_node *node)
 {
 	struct xe_ggtt *ggtt = node->ggtt;
-	struct xe_device *xe = tile_to_xe(ggtt->tile);
 	bool bound;
-	int idx;
-
-	bound = drm_dev_enter(&xe->drm, &idx);
 
 	mutex_lock(&ggtt->lock);
+	bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE;
 	if (bound)
-		xe_ggtt_clear(ggtt, node->base.start, node->base.size);
+		xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node));
 	drm_mm_remove_node(&node->base);
 	node->base.size = 0;
 	mutex_unlock(&ggtt->lock);
@@ -429,10 +489,8 @@ static void ggtt_node_remove(struct xe_ggtt_node *node)
 	if (node->invalidate_on_remove)
 		xe_ggtt_invalidate(ggtt);
 
-	drm_dev_exit(idx);
-
 free_node:
-	xe_ggtt_node_fini(node);
+	ggtt_node_fini(node);
 }
 
 static void ggtt_node_remove_work_func(struct work_struct *work)
@@ -538,169 +596,38 @@ static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
 }
 
-static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
-			      const struct drm_mm_node *node, const char *description)
-{
-	char buf[10];
-
-	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
-		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
-		xe_tile_dbg(ggtt->tile, "GGTT %#llx-%#llx (%s) %s\n",
-			    node->start, node->start + node->size, buf, description);
-	}
-}
-
-/**
- * xe_ggtt_node_insert_balloon_locked - prevent allocation of specified GGTT addresses
- * @node: the &xe_ggtt_node to hold reserved GGTT node
- * @start: the starting GGTT address of the reserved region
- * @end: then end GGTT address of the reserved region
- *
- * To be used in cases where ggtt->lock is already taken.
- * Use xe_ggtt_node_remove_balloon_locked() to release a reserved GGTT node.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int xe_ggtt_node_insert_balloon_locked(struct xe_ggtt_node *node, u64 start, u64 end)
-{
-	struct xe_ggtt *ggtt = node->ggtt;
-	int err;
-
-	xe_tile_assert(ggtt->tile, start < end);
-	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
-	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
-	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(&node->base));
-	lockdep_assert_held(&ggtt->lock);
-
-	node->base.color = 0;
-	node->base.start = start;
-	node->base.size = end - start;
-
-	err = drm_mm_reserve_node(&ggtt->mm, &node->base);
-
-	if (xe_tile_WARN(ggtt->tile, err, "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
-			 node->base.start, node->base.start + node->base.size, ERR_PTR(err)))
-		return err;
-
-	xe_ggtt_dump_node(ggtt, &node->base, "balloon");
-	return 0;
-}
-
-/**
- * xe_ggtt_node_remove_balloon_locked - release a reserved GGTT region
- * @node: the &xe_ggtt_node with reserved GGTT region
- *
- * To be used in cases where ggtt->lock is already taken.
- * See xe_ggtt_node_insert_balloon_locked() for details.
- */
-void xe_ggtt_node_remove_balloon_locked(struct xe_ggtt_node *node)
-{
-	if (!xe_ggtt_node_allocated(node))
-		return;
-
-	lockdep_assert_held(&node->ggtt->lock);
-
-	xe_ggtt_dump_node(node->ggtt, &node->base, "remove-balloon");
-
-	drm_mm_remove_node(&node->base);
-}
-
-static void xe_ggtt_assert_fit(struct xe_ggtt *ggtt, u64 start, u64 size)
-{
-	struct xe_tile *tile = ggtt->tile;
-
-	xe_tile_assert(tile, start >= ggtt->start);
-	xe_tile_assert(tile, start + size <= ggtt->start + ggtt->size);
-}
-
 /**
- * xe_ggtt_shift_nodes_locked - Shift GGTT nodes to adjust for a change in usable address range.
+ * xe_ggtt_shift_nodes() - Shift GGTT nodes to adjust for a change in usable address range.
  * @ggtt: the &xe_ggtt struct instance
- * @shift: change to the location of area provisioned for current VF
+ * @new_start: new location of area provisioned for current VF
  *
- * This function moves all nodes from the GGTT VM, to a temp list. These nodes are expected
- * to represent allocations in range formerly assigned to current VF, before the range changed.
- * When the GGTT VM is completely clear of any nodes, they are re-added with shifted offsets.
+ * Ensure that all struct &xe_ggtt_node are moved to the @new_start base address
+ * by changing the base offset of the GGTT.
  *
- * The function has no ability of failing - because it shifts existing nodes, without
- * any additional processing. If the nodes were successfully existing at the old address,
- * they will do the same at the new one. A fail inside this function would indicate that
- * the list of nodes was either already damaged, or that the shift brings the address range
- * outside of valid bounds. Both cases justify an assert rather than error code.
+ * This function may be called multiple times during recovery, but if
+ * @new_start is unchanged from the current base, it's a noop.
+ *
+ * @new_start should be a value between xe_wopcm_size() and #GUC_GGTT_TOP.
  */
-void xe_ggtt_shift_nodes_locked(struct xe_ggtt *ggtt, s64 shift)
+void xe_ggtt_shift_nodes(struct xe_ggtt *ggtt, u64 new_start)
 {
-	struct xe_tile *tile __maybe_unused = ggtt->tile;
-	struct drm_mm_node *node, *tmpn;
-	LIST_HEAD(temp_list_head);
-
-	lockdep_assert_held(&ggtt->lock);
+	guard(mutex)(&ggtt->lock);
 
-	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG))
-		drm_mm_for_each_node_safe(node, tmpn, &ggtt->mm)
-			xe_ggtt_assert_fit(ggtt, node->start + shift, node->size);
+	xe_tile_assert(ggtt->tile, new_start >= xe_wopcm_size(tile_to_xe(ggtt->tile)));
+	xe_tile_assert(ggtt->tile, new_start + ggtt->size <= GUC_GGTT_TOP);
 
-	drm_mm_for_each_node_safe(node, tmpn, &ggtt->mm) {
-		drm_mm_remove_node(node);
-		list_add(&node->node_list, &temp_list_head);
-	}
-
-	list_for_each_entry_safe(node, tmpn, &temp_list_head, node_list) {
-		list_del(&node->node_list);
-		node->start += shift;
-		drm_mm_reserve_node(&ggtt->mm, node);
-		xe_tile_assert(tile, drm_mm_node_allocated(node));
-	}
+	/* pairs with READ_ONCE in xe_ggtt_node_addr() */
+	WRITE_ONCE(ggtt->start, new_start);
 }
 
-static int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node,
+static int xe_ggtt_insert_node_locked(struct xe_ggtt_node *node,
 				      u32 size, u32 align, u32 mm_flags)
 {
 	return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
 					  mm_flags);
 }
 
-/**
- * xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT
- * @node: the &xe_ggtt_node to be inserted
- * @size: size of the node
- * @align: alignment constrain of the node
- *
- * It cannot be called without first having called xe_ggtt_init() once.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align)
-{
-	int ret;
-
-	if (!node || !node->ggtt)
-		return -ENOENT;
-
-	mutex_lock(&node->ggtt->lock);
-	ret = xe_ggtt_node_insert_locked(node, size, align,
-					 DRM_MM_INSERT_HIGH);
-	mutex_unlock(&node->ggtt->lock);
-
-	return ret;
-}
-
-/**
- * xe_ggtt_node_init - Initialize %xe_ggtt_node struct
- * @ggtt: the &xe_ggtt where the new node will later be inserted/reserved.
- *
- * This function will allocate the struct %xe_ggtt_node and return its pointer.
- * This struct will then be freed after the node removal upon xe_ggtt_node_remove()
- * or xe_ggtt_node_remove_balloon_locked().
- *
- * Having %xe_ggtt_node struct allocated doesn't mean that the node is already
- * allocated in GGTT. Only xe_ggtt_node_insert(), allocation through
- * xe_ggtt_node_insert_transform(), or xe_ggtt_node_insert_balloon_locked() will ensure the node is inserted or reserved
- * in GGTT.
- *
- * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
- **/
-struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
+static struct xe_ggtt_node *ggtt_node_init(struct xe_ggtt *ggtt)
 {
 	struct xe_ggtt_node *node = kzalloc_obj(*node, GFP_NOFS);
 
@@ -714,30 +641,31 @@ struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
 }
 
 /**
- * xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct
- * @node: the &xe_ggtt_node to be freed
- *
- * If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
- * or xe_ggtt_node_insert_balloon_locked(); and this @node is not going to be reused, then,
- * this function needs to be called to free the %xe_ggtt_node struct
- **/
-void xe_ggtt_node_fini(struct xe_ggtt_node *node)
-{
-	kfree(node);
-}
-
-/**
- * xe_ggtt_node_allocated - Check if node is allocated in GGTT
- * @node: the &xe_ggtt_node to be inspected
+ * xe_ggtt_insert_node - Insert a &xe_ggtt_node into the GGTT
+ * @ggtt: the &xe_ggtt into which the node should be inserted.
+ * @size: size of the node
+ * @align: alignment constrain of the node
  *
- * Return: True if allocated, False otherwise.
+ * Return: &xe_ggtt_node on success or a ERR_PTR on failure.
  */
-bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node)
+struct xe_ggtt_node *xe_ggtt_insert_node(struct xe_ggtt *ggtt, u32 size, u32 align)
 {
-	if (!node || !node->ggtt)
-		return false;
+	struct xe_ggtt_node *node;
+	int ret;
+
+	node = ggtt_node_init(ggtt);
+	if (IS_ERR(node))
+		return node;
+
+	guard(mutex)(&ggtt->lock);
+	ret = xe_ggtt_insert_node_locked(node, size, align,
+					 DRM_MM_INSERT_HIGH);
+	if (ret) {
+		ggtt_node_fini(node);
+		return ERR_PTR(ret);
+	}
 
-	return drm_mm_node_allocated(&node->base);
+	return node;
 }
 
 /**
@@ -770,7 +698,7 @@ static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node,
 	if (XE_WARN_ON(!node))
 		return;
 
-	start = node->base.start;
+	start = xe_ggtt_node_addr(node);
 	end = start + xe_bo_size(bo);
 
 	if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
@@ -811,7 +739,7 @@ void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo)
 }
 
 /**
- * xe_ggtt_node_insert_transform - Insert a newly allocated &xe_ggtt_node into the GGTT
+ * xe_ggtt_insert_node_transform - Insert a newly allocated &xe_ggtt_node into the GGTT
  * @ggtt: the &xe_ggtt where the node will inserted/reserved.
  * @bo: The bo to be transformed
  * @pte_flags: The extra GGTT flags to add to mapping.
@@ -825,7 +753,7 @@ void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo)
  *
  * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
  */
-struct xe_ggtt_node *xe_ggtt_node_insert_transform(struct xe_ggtt *ggtt,
+struct xe_ggtt_node *xe_ggtt_insert_node_transform(struct xe_ggtt *ggtt,
 						   struct xe_bo *bo, u64 pte_flags,
 						   u64 size, u32 align,
 						   xe_ggtt_transform_cb transform, void *arg)
@@ -833,7 +761,7 @@ struct xe_ggtt_node *xe_ggtt_node_insert_transform(struct xe_ggtt *ggtt,
 	struct xe_ggtt_node *node;
 	int ret;
 
-	node = xe_ggtt_node_init(ggtt);
+	node = ggtt_node_init(ggtt);
 	if (IS_ERR(node))
 		return ERR_CAST(node);
 
@@ -842,7 +770,7 @@ struct xe_ggtt_node *xe_ggtt_node_insert_transform(struct xe_ggtt *ggtt,
 		goto err;
 	}
 
-	ret = xe_ggtt_node_insert_locked(node, size, align, 0);
+	ret = xe_ggtt_insert_node_locked(node, size, align, 0);
 	if (ret)
 		goto err_unlock;
 
@@ -857,7 +785,7 @@ struct xe_ggtt_node *xe_ggtt_node_insert_transform(struct xe_ggtt *ggtt,
 err_unlock:
 	mutex_unlock(&ggtt->lock);
 err:
-	xe_ggtt_node_fini(node);
+	ggtt_node_fini(node);
 	return ERR_PTR(ret);
 }
 
@@ -883,7 +811,7 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 
 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
 
-	bo->ggtt_node[tile_id] = xe_ggtt_node_init(ggtt);
+	bo->ggtt_node[tile_id] = ggtt_node_init(ggtt);
 	if (IS_ERR(bo->ggtt_node[tile_id])) {
 		err = PTR_ERR(bo->ggtt_node[tile_id]);
 		bo->ggtt_node[tile_id] = NULL;
@@ -891,10 +819,30 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 	}
 
 	mutex_lock(&ggtt->lock);
+	/*
+	 * When inheriting the initial framebuffer, the framebuffer is
+	 * physically located at VRAM address 0, and usually at GGTT address 0 too.
+	 *
+	 * The display code will ask for a GGTT allocation between end of BO and
+	 * remainder of GGTT, unaware that the start is reserved by WOPCM.
+	 */
+	if (start >= ggtt->start)
+		start -= ggtt->start;
+	else
+		start = 0;
+
+	/* Should never happen, but since we handle start, fail graciously for end */
+	if (end >= ggtt->start)
+		end -= ggtt->start;
+	else
+		end = 0;
+
+	xe_tile_assert(ggtt->tile, end >= start + xe_bo_size(bo));
+
 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node[tile_id]->base,
 					  xe_bo_size(bo), alignment, 0, start, end, 0);
 	if (err) {
-		xe_ggtt_node_fini(bo->ggtt_node[tile_id]);
+		ggtt_node_fini(bo->ggtt_node[tile_id]);
 		bo->ggtt_node[tile_id] = NULL;
 	} else {
 		u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
@@ -1002,18 +950,16 @@ static u64 xe_encode_vfid_pte(u16 vfid)
 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
 }
 
-static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
+static void xe_ggtt_assign_locked(const struct xe_ggtt_node *node, u16 vfid)
 {
-	u64 start = node->start;
-	u64 size = node->size;
+	struct xe_ggtt *ggtt = node->ggtt;
+	u64 start = xe_ggtt_node_addr(node);
+	u64 size = xe_ggtt_node_size(node);
 	u64 end = start + size - 1;
 	u64 pte = xe_encode_vfid_pte(vfid);
 
 	lockdep_assert_held(&ggtt->lock);
 
-	if (!drm_mm_node_allocated(node))
-		return;
-
 	while (start < end) {
 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
 		start += XE_PAGE_SIZE;
@@ -1033,9 +979,8 @@ static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node
  */
 void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
 {
-	mutex_lock(&node->ggtt->lock);
-	xe_ggtt_assign_locked(node->ggtt, &node->base, vfid);
-	mutex_unlock(&node->ggtt->lock);
+	guard(mutex)(&node->ggtt->lock);
+	xe_ggtt_assign_locked(node, vfid);
 }
 
 /**
@@ -1057,14 +1002,14 @@ int xe_ggtt_node_save(struct xe_ggtt_node *node, void *dst, size_t size, u16 vfi
 	if (!node)
 		return -ENOENT;
 
-	guard(mutex)(&node->ggtt->lock);
+	ggtt = node->ggtt;
+	guard(mutex)(&ggtt->lock);
 
 	if (xe_ggtt_node_pt_size(node) != size)
 		return -EINVAL;
 
-	ggtt = node->ggtt;
-	start = node->base.start;
-	end = start + node->base.size - 1;
+	start = xe_ggtt_node_addr(node);
+	end = start + xe_ggtt_node_size(node) - 1;
 
 	while (start < end) {
 		pte = ggtt->pt_ops->ggtt_get_pte(ggtt, start);
@@ -1097,14 +1042,14 @@ int xe_ggtt_node_load(struct xe_ggtt_node *node, const void *src, size_t size, u
 	if (!node)
 		return -ENOENT;
 
-	guard(mutex)(&node->ggtt->lock);
+	ggtt = node->ggtt;
+	guard(mutex)(&ggtt->lock);
 
 	if (xe_ggtt_node_pt_size(node) != size)
 		return -EINVAL;
 
-	ggtt = node->ggtt;
-	start = node->base.start;
-	end = start + node->base.size - 1;
+	start = xe_ggtt_node_addr(node);
+	end = start + xe_ggtt_node_size(node) - 1;
 
 	while (start < end) {
 		vfid_pte = u64_replace_bits(*buf++, vfid, GGTT_PTE_VFID);
@@ -1211,7 +1156,8 @@ u64 xe_ggtt_read_pte(struct xe_ggtt *ggtt, u64 offset)
  */
 u64 xe_ggtt_node_addr(const struct xe_ggtt_node *node)
 {
-	return node->base.start;
+	/* pairs with WRITE_ONCE in xe_ggtt_shift_nodes() */
+	return node->base.start + READ_ONCE(node->ggtt->start);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_ggtt.h b/drivers/gpu/drm/xe/xe_ggtt.h
index 70d5e07ac4b66..c864cc975a695 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.h
+++ b/drivers/gpu/drm/xe/xe_ggtt.h
@@ -9,6 +9,7 @@
 #include "xe_ggtt_types.h"
 
 struct drm_printer;
+struct xe_bo;
 struct xe_tile;
 struct drm_exec;
 
@@ -17,23 +18,18 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt);
 int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 reserved, u32 size);
 int xe_ggtt_init(struct xe_ggtt *ggtt);
 
-struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt);
-void xe_ggtt_node_fini(struct xe_ggtt_node *node);
-int xe_ggtt_node_insert_balloon_locked(struct xe_ggtt_node *node,
-				       u64 start, u64 size);
-void xe_ggtt_node_remove_balloon_locked(struct xe_ggtt_node *node);
-void xe_ggtt_shift_nodes_locked(struct xe_ggtt *ggtt, s64 shift);
+void xe_ggtt_shift_nodes(struct xe_ggtt *ggtt, u64 new_base);
 u64 xe_ggtt_start(struct xe_ggtt *ggtt);
 u64 xe_ggtt_size(struct xe_ggtt *ggtt);
 
-int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align);
 struct xe_ggtt_node *
-xe_ggtt_node_insert_transform(struct xe_ggtt *ggtt,
+xe_ggtt_insert_node(struct xe_ggtt *ggtt, u32 size, u32 align);
+struct xe_ggtt_node *
+xe_ggtt_insert_node_transform(struct xe_ggtt *ggtt,
 			      struct xe_bo *bo, u64 pte,
 			      u64 size, u32 align,
 			      xe_ggtt_transform_cb transform, void *arg);
 void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate);
-bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node);
 size_t xe_ggtt_node_pt_size(const struct xe_ggtt_node *node);
 void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo);
 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo, struct drm_exec *exec);
diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h
index d82b71a198bc2..cf754e4d502ad 100644
--- a/drivers/gpu/drm/xe/xe_ggtt_types.h
+++ b/drivers/gpu/drm/xe/xe_ggtt_types.h
@@ -6,72 +6,16 @@
 #ifndef _XE_GGTT_TYPES_H_
 #define _XE_GGTT_TYPES_H_
 
+#include <linux/types.h>
 #include <drm/drm_mm.h>
 
-#include "xe_pt_types.h"
-
-struct xe_bo;
+struct xe_ggtt;
 struct xe_ggtt_node;
-struct xe_gt;
-
-/**
- * struct xe_ggtt - Main GGTT struct
- *
- * In general, each tile can contains its own Global Graphics Translation Table
- * (GGTT) instance.
- */
-struct xe_ggtt {
-	/** @tile: Back pointer to tile where this GGTT belongs */
-	struct xe_tile *tile;
-	/** @start: Start offset of GGTT */
-	u64 start;
-	/** @size: Total usable size of this GGTT */
-	u64 size;
-
-#define XE_GGTT_FLAGS_64K BIT(0)
-	/**
-	 * @flags: Flags for this GGTT
-	 * Acceptable flags:
-	 * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
-	 */
-	unsigned int flags;
-	/** @scratch: Internal object allocation used as a scratch page */
-	struct xe_bo *scratch;
-	/** @lock: Mutex lock to protect GGTT data */
-	struct mutex lock;
-	/**
-	 *  @gsm: The iomem pointer to the actual location of the translation
-	 * table located in the GSM for easy PTE manipulation
-	 */
-	u64 __iomem *gsm;
-	/** @pt_ops: Page Table operations per platform */
-	const struct xe_ggtt_pt_ops *pt_ops;
-	/** @mm: The memory manager used to manage individual GGTT allocations */
-	struct drm_mm mm;
-	/** @access_count: counts GGTT writes */
-	unsigned int access_count;
-	/** @wq: Dedicated unordered work queue to process node removals */
-	struct workqueue_struct *wq;
-};
 
 typedef void (*xe_ggtt_set_pte_fn)(struct xe_ggtt *ggtt, u64 addr, u64 pte);
 typedef void (*xe_ggtt_transform_cb)(struct xe_ggtt *ggtt,
 				     struct xe_ggtt_node *node,
 				     u64 pte_flags,
 				     xe_ggtt_set_pte_fn set_pte, void *arg);
-/**
- * struct xe_ggtt_pt_ops - GGTT Page table operations
- * Which can vary from platform to platform.
- */
-struct xe_ggtt_pt_ops {
-	/** @pte_encode_flags: Encode PTE flags for a given BO */
-	u64 (*pte_encode_flags)(struct xe_bo *bo, u16 pat_index);
-
-	/** @ggtt_set_pte: Directly write into GGTT's PTE */
-	xe_ggtt_set_pte_fn ggtt_set_pte;
-
-	/** @ggtt_get_pte: Directly read from GGTT's PTE */
-	u64 (*ggtt_get_pte)(struct xe_ggtt *ggtt, u64 addr);
-};
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index df6d04704823d..bae895fa066a0 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -10,6 +10,7 @@
 #include <drm/drm_managed.h>
 #include <uapi/drm/xe_drm.h>
 
+#include <generated/xe_device_wa_oob.h>
 #include <generated/xe_wa_oob.h>
 
 #include "instructions/xe_alu_commands.h"
@@ -33,10 +34,12 @@
 #include "xe_gt_printk.h"
 #include "xe_gt_sriov_pf.h"
 #include "xe_gt_sriov_vf.h"
+#include "xe_gt_stats.h"
 #include "xe_gt_sysfs.h"
 #include "xe_gt_topology.h"
 #include "xe_guc_exec_queue_types.h"
 #include "xe_guc_pc.h"
+#include "xe_guc_rc.h"
 #include "xe_guc_submit.h"
 #include "xe_hw_fence.h"
 #include "xe_hw_engine_class_sysfs.h"
@@ -141,15 +144,14 @@ static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
 static void xe_gt_enable_comp_1wcoh(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 reg;
 
 	if (IS_SRIOV_VF(xe))
 		return;
 
 	if (GRAPHICS_VER(xe) >= 30 && xe->info.has_flat_ccs) {
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+		if (!fw_ref.domains)
 			return;
 
 		reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
@@ -163,8 +165,6 @@ static void xe_gt_enable_comp_1wcoh(struct xe_gt *gt)
 			reg |= EN_CMP_1WCOH_GW;
 			xe_gt_mcr_multicast_write(gt, XE2_GAMWALK_CTRL_3D, reg);
 		}
-
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 }
 
@@ -452,6 +452,35 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt)
 	return err;
 }
 
+static void wa_14026539277(struct xe_gt *gt)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	u32 val;
+
+	/*
+	 * FIXME: We currently can't use FUNC(xe_rtp_match_not_sriov_vf) in the
+	 * rules for Wa_14026539277 due to xe_wa_process_device_oob() being
+	 * called before xe_sriov_probe_early(); and we can't move the call to
+	 * the former to happen after the latter because MMIO read functions
+	 * already depend on a device OOB workaround.  This needs to be fixed by
+	 * allowing workaround checks to happen at different stages of driver
+	 * initialization.
+	 */
+	if (IS_SRIOV_VF(xe))
+		return;
+
+	if (!XE_DEVICE_WA(xe, 14026539277))
+		return;
+
+	if (!xe_gt_is_main_type(gt))
+		return;
+
+	val = xe_gt_mcr_unicast_read_any(gt, L2COMPUTESIDECTRL);
+	val &= ~CECTRL;
+	val |= CECTRL_CENODATA_ALWAYS;
+	xe_gt_mcr_multicast_write(gt, L2COMPUTESIDECTRL, val);
+}
+
 int xe_gt_init_early(struct xe_gt *gt)
 {
 	int err;
@@ -500,6 +529,10 @@ int xe_gt_init_early(struct xe_gt *gt)
 	if (err)
 		return err;
 
+	err = xe_gt_stats_init(gt);
+	if (err)
+		return err;
+
 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 	if (!fw_ref.domains)
 		return -ETIMEDOUT;
@@ -573,6 +606,15 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 	 */
 	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
 
+	/*
+	 * Wa_14026539277 can't be implemented as a regular GT workaround (i.e.
+	 * as an entry in gt_was[]) for two reasons: it is actually a device
+	 * workaround that happens to involve programming a GT register; and it
+	 * needs to be applied early to avoid getting the hardware in a bad
+	 * state before we have a chance to do the necessary programming.
+	 */
+	wa_14026539277(gt);
+
 	return 0;
 }
 
@@ -894,7 +936,7 @@ static void gt_reset_worker(struct work_struct *w)
 	if (IS_SRIOV_PF(gt_to_xe(gt)))
 		xe_gt_sriov_pf_stop_prepare(gt);
 
-	xe_uc_gucrc_disable(&gt->uc);
+	xe_guc_rc_disable(&gt->uc.guc);
 	xe_uc_stop_prepare(&gt->uc);
 	xe_pagefault_reset(gt_to_xe(gt), gt);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
index fe944687728cd..baee1f4a6b013 100644
--- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
+++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
@@ -12,7 +12,9 @@
 #include "xe_gt_printk.h"
 #include "xe_gt_sysfs.h"
 #include "xe_mmio.h"
+#include "xe_pm.h"
 #include "xe_sriov.h"
+#include "xe_sriov_pf.h"
 
 static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines)
 {
@@ -88,6 +90,11 @@ void xe_gt_apply_ccs_mode(struct xe_gt *gt)
 	__xe_gt_apply_ccs_mode(gt, gt->ccs_mode);
 }
 
+static bool gt_ccs_mode_default(struct xe_gt *gt)
+{
+	return gt->ccs_mode == 1;
+}
+
 static ssize_t
 num_cslices_show(struct device *kdev,
 		 struct device_attribute *attr, char *buf)
@@ -117,12 +124,6 @@ ccs_mode_store(struct device *kdev, struct device_attribute *attr,
 	u32 num_engines, num_slices;
 	int ret;
 
-	if (IS_SRIOV(xe)) {
-		xe_gt_dbg(gt, "Can't change compute mode when running as %s\n",
-			  xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
-		return -EOPNOTSUPP;
-	}
-
 	ret = kstrtou32(buff, 0, &num_engines);
 	if (ret)
 		return ret;
@@ -139,21 +140,36 @@ ccs_mode_store(struct device *kdev, struct device_attribute *attr,
 	}
 
 	/* CCS mode can only be updated when there are no drm clients */
-	mutex_lock(&xe->drm.filelist_mutex);
+	guard(mutex)(&xe->drm.filelist_mutex);
 	if (!list_empty(&xe->drm.filelist)) {
-		mutex_unlock(&xe->drm.filelist_mutex);
 		xe_gt_dbg(gt, "Rejecting compute mode change as there are active drm clients\n");
 		return -EBUSY;
 	}
 
-	if (gt->ccs_mode != num_engines) {
-		xe_gt_info(gt, "Setting compute mode to %d\n", num_engines);
-		gt->ccs_mode = num_engines;
-		xe_gt_record_user_engines(gt);
-		xe_gt_reset(gt);
+	if (gt->ccs_mode == num_engines)
+		return count;
+
+	/*
+	 * Changing default CCS mode is only allowed when there
+	 * are no VFs. Try to lockdown PF to find out.
+	 */
+	if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe)) {
+		ret = xe_sriov_pf_lockdown(xe);
+		if (ret) {
+			xe_gt_dbg(gt, "Can't change CCS Mode: VFs are enabled\n");
+			return ret;
+		}
 	}
 
-	mutex_unlock(&xe->drm.filelist_mutex);
+	xe_gt_info(gt, "Setting compute mode to %d\n", num_engines);
+	gt->ccs_mode = num_engines;
+	xe_gt_record_user_engines(gt);
+	guard(xe_pm_runtime)(xe);
+	xe_gt_reset(gt);
+
+	/* We may end PF lockdown once CCS mode is default again */
+	if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe))
+		xe_sriov_pf_end_lockdown(xe);
 
 	return count;
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c
index 4363bc9c36067..f45306308cd66 100644
--- a/drivers/gpu/drm/xe/xe_gt_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c
@@ -155,6 +155,30 @@ static int register_save_restore(struct xe_gt *gt, struct drm_printer *p)
 	return 0;
 }
 
+/*
+ * Check the registers referenced on a save-restore list and report any
+ * save-restore entries that did not get applied.
+ */
+static int register_save_restore_check(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_hw_engine *hwe;
+	enum xe_hw_engine_id id;
+
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		drm_printf(p, "ERROR: Could not acquire forcewake\n");
+		return -ETIMEDOUT;
+	}
+
+	xe_reg_sr_readback_check(&gt->reg_sr, gt, p);
+	for_each_hw_engine(hwe, gt, id)
+		xe_reg_sr_readback_check(&hwe->reg_sr, gt, p);
+	for_each_hw_engine(hwe, gt, id)
+		xe_reg_sr_lrc_check(&hwe->reg_lrc, gt, hwe, p);
+
+	return 0;
+}
+
 static int rcs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
 	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_RENDER);
@@ -209,6 +233,8 @@ static const struct drm_info_list vf_safe_debugfs_list[] = {
 	{ "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc },
 	{ "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig },
 	{ "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config },
+	{ "register-save-restore-check",
+		.show = xe_gt_debugfs_show_with_rpm, .data = register_save_restore_check },
 };
 
 /* everything else should be added here */
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c
index 7a569e1730a48..4a2d9edb6a4c0 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.c
+++ b/drivers/gpu/drm/xe/xe_gt_idle.c
@@ -168,6 +168,24 @@ void xe_gt_idle_disable_pg(struct xe_gt *gt)
 	xe_mmio_write32(&gt->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
 }
 
+static void force_wake_domains_show(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_force_wake_domain *domain;
+	struct xe_force_wake *fw = gt_to_fw(gt);
+	unsigned int tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fw->lock, flags);
+	for_each_fw_domain(domain, fw, tmp) {
+		drm_printf(p, "%s.ref_count=%u, %s.fwake=0x%x\n",
+			   xe_force_wake_domain_to_str(domain->id),
+			   READ_ONCE(domain->ref),
+			   xe_force_wake_domain_to_str(domain->id),
+			   xe_mmio_read32(&gt->mmio, domain->reg_ctl));
+	}
+	spin_unlock_irqrestore(&fw->lock, flags);
+}
+
 /**
  * xe_gt_idle_pg_print - Xe powergating info
  * @gt: GT object
@@ -254,6 +272,13 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
 		drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
 			   str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
 
+	if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) {
+		drm_printf(p, "GSC Power Gate Status: %s\n",
+			   str_up_down(pg_status & GSC_AWAKE_STATUS));
+	}
+
+	force_wake_domains_show(gt, p);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c
index 7c1fe9ac120d7..7c6f039c880df 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.c
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.c
@@ -201,7 +201,7 @@ static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {
 	{ 0x009680, 0x0096FF },         /* DSS */
 	{ 0x00D800, 0x00D87F },         /* SLICE */
 	{ 0x00DC00, 0x00DCFF },         /* SLICE */
-	{ 0x00DE80, 0x00E8FF },         /* DSS (0xE000-0xE0FF reserved) */
+	{ 0x00DE00, 0x00E8FF },         /* DSS (0xE000-0xE0FF reserved) */
 	{ 0x00E980, 0x00E9FF },         /* SLICE */
 	{ 0x013000, 0x0133FF },         /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */
 	{},
@@ -280,6 +280,19 @@ static const struct xe_mmio_range xe3p_xpc_instance0_steering_table[] = {
 	{},
 };
 
+static const struct xe_mmio_range xe3p_lpg_instance0_steering_table[] = {
+	{ 0x004000, 0x004AFF },         /* GAM, rsvd, GAMWKR */
+	{ 0x008700, 0x00887F },         /* NODE */
+	{ 0x00B000, 0x00B3FF },         /* NODE, L3BANK */
+	{ 0x00B500, 0x00B6FF },		/* PSMI */
+	{ 0x00C800, 0x00CFFF },         /* GAM */
+	{ 0x00D880, 0x00D8FF },         /* NODE */
+	{ 0x00DD00, 0x00DD7F },         /* MEMPIPE */
+	{ 0x00F000, 0x00FFFF },         /* GAM, GAMWKR */
+	{ 0x013400, 0x0135FF },         /* MEMPIPE */
+	{},
+};
+
 static void init_steering_l3bank(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
@@ -505,9 +518,6 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 
 	spin_lock_init(&gt->mcr_lock);
 
-	if (IS_SRIOV_VF(xe))
-		return;
-
 	if (gt->info.type == XE_GT_TYPE_MEDIA) {
 		drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
 
@@ -522,17 +532,14 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 		}
 	} else {
 		if (GRAPHICS_VERx100(xe) == 3511) {
-			/*
-			 * TODO: there are some ranges in bspec with missing
-			 * termination: [0x00B000, 0x00B0FF] and
-			 * [0x00D880, 0x00D8FF] (NODE); [0x00B100, 0x00B3FF]
-			 * (L3BANK). Update them here once bspec is updated.
-			 */
 			gt->steering[DSS].ranges = xe3p_xpc_xecore_steering_table;
 			gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table;
 			gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table;
 			gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
 			gt->steering[NODE].ranges = xe3p_xpc_node_steering_table;
+		} else if (GRAPHICS_VERx100(xe) >= 3510) {
+			gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
+			gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table;
 		} else if (GRAPHICS_VER(xe) >= 20) {
 			gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
 			gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
@@ -568,9 +575,6 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
  */
 void xe_gt_mcr_init(struct xe_gt *gt)
 {
-	if (IS_SRIOV_VF(gt_to_xe(gt)))
-		return;
-
 	/* Select non-terminated steering target for each type */
 	for (int i = 0; i < NUM_STEERING_TYPES; i++) {
 		gt->steering[i].initialized = true;
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
index 23601ce79348a..2f376b5fb0882 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
@@ -279,7 +279,7 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config,
 {
 	struct xe_ggtt_node *node = config->ggtt_region;
 
-	if (!xe_ggtt_node_allocated(node))
+	if (!node)
 		return 0;
 
 	return encode_ggtt(cfg, xe_ggtt_node_addr(node), xe_ggtt_node_size(node), details);
@@ -482,23 +482,9 @@ static int pf_distribute_config_ggtt(struct xe_tile *tile, unsigned int vfid, u6
 	return err ?: err2;
 }
 
-static void pf_release_ggtt(struct xe_tile *tile, struct xe_ggtt_node *node)
-{
-	if (xe_ggtt_node_allocated(node)) {
-		/*
-		 * explicit GGTT PTE assignment to the PF using xe_ggtt_assign()
-		 * is redundant, as PTE will be implicitly re-assigned to PF by
-		 * the xe_ggtt_clear() called by below xe_ggtt_remove_node().
-		 */
-		xe_ggtt_node_remove(node, false);
-	} else {
-		xe_ggtt_node_fini(node);
-	}
-}
-
 static void pf_release_vf_config_ggtt(struct xe_gt *gt, struct xe_gt_sriov_config *config)
 {
-	pf_release_ggtt(gt_to_tile(gt), config->ggtt_region);
+	xe_ggtt_node_remove(config->ggtt_region, false);
 	config->ggtt_region = NULL;
 }
 
@@ -517,7 +503,7 @@ static int pf_provision_vf_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size)
 
 	size = round_up(size, alignment);
 
-	if (xe_ggtt_node_allocated(config->ggtt_region)) {
+	if (config->ggtt_region) {
 		err = pf_distribute_config_ggtt(tile, vfid, 0, 0);
 		if (unlikely(err))
 			return err;
@@ -528,19 +514,15 @@ static int pf_provision_vf_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size)
 		if (unlikely(err))
 			return err;
 	}
-	xe_gt_assert(gt, !xe_ggtt_node_allocated(config->ggtt_region));
+	xe_gt_assert(gt, !config->ggtt_region);
 
 	if (!size)
 		return 0;
 
-	node = xe_ggtt_node_init(ggtt);
+	node = xe_ggtt_insert_node(ggtt, size, alignment);
 	if (IS_ERR(node))
 		return PTR_ERR(node);
 
-	err = xe_ggtt_node_insert(node, size, alignment);
-	if (unlikely(err))
-		goto err;
-
 	xe_ggtt_assign(node, vfid);
 	xe_gt_sriov_dbg_verbose(gt, "VF%u assigned GGTT %llx-%llx\n",
 				vfid, xe_ggtt_node_addr(node), xe_ggtt_node_addr(node) + size - 1);
@@ -552,7 +534,7 @@ static int pf_provision_vf_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size)
 	config->ggtt_region = node;
 	return 0;
 err:
-	pf_release_ggtt(tile, node);
+	xe_ggtt_node_remove(node, false);
 	return err;
 }
 
@@ -562,7 +544,7 @@ static u64 pf_get_vf_config_ggtt(struct xe_gt *gt, unsigned int vfid)
 	struct xe_ggtt_node *node = config->ggtt_region;
 
 	xe_gt_assert(gt, xe_gt_is_main_type(gt));
-	return xe_ggtt_node_allocated(node) ? xe_ggtt_node_size(node) : 0;
+	return node ? xe_ggtt_node_size(node) : 0;
 }
 
 /**
@@ -1469,8 +1451,8 @@ int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid,
 
 static u64 pf_get_lmem_alignment(struct xe_gt *gt)
 {
-	/* this might be platform dependent */
-	return SZ_2M;
+	return xe_device_has_lmtt(gt_to_xe(gt)) ?
+		xe_lmtt_page_size(&gt_to_tile(gt)->sriov.pf.lmtt) : XE_PAGE_SIZE;
 }
 
 static u64 pf_get_min_spare_lmem(struct xe_gt *gt)
@@ -1597,10 +1579,7 @@ static int pf_update_vf_lmtt(struct xe_device *xe, unsigned int vfid)
 			goto fail;
 
 		offset = 0;
-		for_each_gt(gt, xe, gtid) {
-			if (xe_gt_is_media_type(gt))
-				continue;
-
+		for_each_gt_with_type(gt, xe, gtid, BIT(XE_GT_TYPE_MAIN)) {
 			config = pf_pick_vf_config(gt, vfid);
 			bo = config->lmem_obj;
 			if (!bo)
@@ -1645,13 +1624,15 @@ static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_bo *bo;
+	u64 alignment;
 	int err;
 
 	xe_gt_assert(gt, vfid);
 	xe_gt_assert(gt, IS_DGFX(xe));
 	xe_gt_assert(gt, xe_gt_is_main_type(gt));
 
-	size = round_up(size, pf_get_lmem_alignment(gt));
+	alignment = pf_get_lmem_alignment(gt);
+	size = round_up(size, alignment);
 
 	if (config->lmem_obj) {
 		err = pf_distribute_config_lmem(gt, vfid, 0);
@@ -1667,12 +1648,12 @@ static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
 	if (!size)
 		return 0;
 
-	xe_gt_assert(gt, pf_get_lmem_alignment(gt) == SZ_2M);
+	xe_gt_assert(gt, alignment == XE_PAGE_SIZE || alignment == SZ_2M);
 	bo = xe_bo_create_pin_range_novm(xe, tile,
 					 ALIGN(size, PAGE_SIZE), 0, ~0ull,
 					 ttm_bo_type_kernel,
-					 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
-					 XE_BO_FLAG_NEEDS_2M |
+					 XE_BO_FLAG_VRAM(tile->mem.vram) |
+					 (alignment == SZ_2M ? XE_BO_FLAG_NEEDS_2M : 0) |
 					 XE_BO_FLAG_PINNED |
 					 XE_BO_FLAG_PINNED_LATE_RESTORE |
 					 XE_BO_FLAG_FORCE_USER_VRAM);
@@ -1754,7 +1735,7 @@ int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size
 }
 
 /**
- * xe_gt_sriov_pf_config_bulk_set_lmem - Provision many VFs with LMEM.
+ * xe_gt_sriov_pf_config_bulk_set_lmem_locked() - Provision many VFs with LMEM.
  * @gt: the &xe_gt (can't be media)
  * @vfid: starting VF identifier (can't be 0)
  * @num_vfs: number of VFs to provision
@@ -1764,31 +1745,94 @@ int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid,
-					unsigned int num_vfs, u64 size)
+int xe_gt_sriov_pf_config_bulk_set_lmem_locked(struct xe_gt *gt, unsigned int vfid,
+					       unsigned int num_vfs, u64 size)
 {
 	unsigned int n;
 	int err = 0;
 
-	xe_gt_assert(gt, vfid);
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+	xe_gt_assert(gt, xe_device_has_lmtt(gt_to_xe(gt)));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
 	xe_gt_assert(gt, xe_gt_is_main_type(gt));
+	xe_gt_assert(gt, vfid);
 
 	if (!num_vfs)
 		return 0;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
 	for (n = vfid; n < vfid + num_vfs; n++) {
 		err = pf_provision_vf_lmem(gt, n, size);
 		if (err)
 			break;
 	}
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size,
-					   xe_gt_sriov_pf_config_get_lmem,
+					   pf_get_vf_config_lmem,
 					   "LMEM", n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_lmem() - Provision many VFs with LMEM.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @size: requested LMEM size
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u64 size)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_bulk_set_lmem_locked(gt, vfid, num_vfs, size);
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_lmem_locked() - Get VF's LMEM quota.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier (can't be 0 == PFID)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: VF's LMEM quota.
+ */
+u64 xe_gt_sriov_pf_config_get_lmem_locked(struct xe_gt *gt, unsigned int vfid)
+{
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	xe_gt_assert(gt, vfid);
+
+	return pf_get_vf_config_lmem(gt, vfid);
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_lmem_locked() - Provision VF with LMEM.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: the VF identifier (can't be 0 == PFID)
+ * @size: requested LMEM size
+ *
+ * This function can only be called on PF.
+ */
+int xe_gt_sriov_pf_config_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	int err;
+
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+	xe_gt_assert(gt, xe_device_has_lmtt(gt_to_xe(gt)));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	xe_gt_assert(gt, xe_gt_is_main_type(gt));
+	xe_gt_assert(gt, vfid);
+
+	err = pf_provision_vf_lmem(gt, vfid, size);
+
+	return pf_config_set_u64_done(gt, vfid, size,
+				      pf_get_vf_config_lmem(gt, vfid),
+				      "LMEM", err);
+}
+
 static struct xe_bo *pf_get_vf_config_lmem_obj(struct xe_gt *gt, unsigned int vfid)
 {
 	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
@@ -1856,6 +1900,81 @@ static u64 pf_estimate_fair_lmem(struct xe_gt *gt, unsigned int num_vfs)
 	return fair;
 }
 
+static u64 pf_profile_fair_lmem(struct xe_gt *gt, unsigned int num_vfs)
+{
+	struct xe_tile *tile = gt_to_tile(gt);
+	bool admin_only_pf = xe_sriov_pf_admin_only(tile->xe);
+	u64 usable = xe_vram_region_usable_size(tile->mem.vram);
+	u64 spare = pf_get_min_spare_lmem(gt);
+	u64 available = usable > spare ? usable - spare : 0;
+	u64 shareable = ALIGN_DOWN(available, SZ_1G);
+	u64 alignment = pf_get_lmem_alignment(gt);
+	u64 fair;
+
+	if (admin_only_pf)
+		fair = div_u64(shareable, num_vfs);
+	else
+		fair = div_u64(shareable, 1 + num_vfs);
+
+	if (!admin_only_pf && fair)
+		fair = rounddown_pow_of_two(fair);
+
+	return ALIGN_DOWN(fair, alignment);
+}
+
+static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf,
+					unsigned int num_vfs, bool provisioned)
+{
+	unsigned int allvfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); /* PF plus VFs */
+	unsigned long *bitmap __free(bitmap) = bitmap_zalloc(allvfs, GFP_KERNEL);
+	unsigned int weight;
+	unsigned int n;
+
+	if (!bitmap)
+		return;
+
+	for (n = first_vf; n < first_vf + num_vfs; n++) {
+		if (!!pf_get_vf_config_lmem(gt, VFID(n)) == provisioned)
+			bitmap_set(bitmap, n, 1);
+	}
+
+	weight = bitmap_weight(bitmap, allvfs);
+	if (!weight)
+		return;
+
+	xe_gt_sriov_info(gt, "VF%s%*pbl %s provisioned with VRAM\n",
+			 weight > 1 ? "s " : "", allvfs, bitmap,
+			 provisioned ? "already" : "not");
+}
+
+static void pf_show_all_provisioned_lmem(struct xe_gt *gt)
+{
+	__pf_show_provisioning_lmem(gt, VFID(1), xe_gt_sriov_pf_get_totalvfs(gt), true);
+}
+
+static void pf_show_unprovisioned_lmem(struct xe_gt *gt, unsigned int first_vf,
+				       unsigned int num_vfs)
+{
+	__pf_show_provisioning_lmem(gt, first_vf, num_vfs, false);
+}
+
+static bool pf_needs_provision_lmem(struct xe_gt *gt, unsigned int first_vf,
+				    unsigned int num_vfs)
+{
+	unsigned int vfid;
+
+	for (vfid = first_vf; vfid < first_vf + num_vfs; vfid++) {
+		if (pf_get_vf_config_lmem(gt, vfid)) {
+			pf_show_all_provisioned_lmem(gt);
+			pf_show_unprovisioned_lmem(gt, first_vf, num_vfs);
+			return false;
+		}
+	}
+
+	pf_show_all_provisioned_lmem(gt);
+	return true;
+}
+
 /**
  * xe_gt_sriov_pf_config_set_fair_lmem - Provision many VFs with fair LMEM.
  * @gt: the &xe_gt (can't be media)
@@ -1869,6 +1988,7 @@ static u64 pf_estimate_fair_lmem(struct xe_gt *gt, unsigned int num_vfs)
 int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid,
 					unsigned int num_vfs)
 {
+	u64 profile;
 	u64 fair;
 
 	xe_gt_assert(gt, vfid);
@@ -1878,14 +1998,22 @@ int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid,
 	if (!xe_device_has_lmtt(gt_to_xe(gt)))
 		return 0;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	fair = pf_estimate_fair_lmem(gt, num_vfs);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
+	if (!pf_needs_provision_lmem(gt, vfid, num_vfs))
+		return 0;
+
+	fair = pf_estimate_fair_lmem(gt, num_vfs);
 	if (!fair)
 		return -ENOSPC;
 
-	return xe_gt_sriov_pf_config_bulk_set_lmem(gt, vfid, num_vfs, fair);
+	profile = pf_profile_fair_lmem(gt, num_vfs);
+	fair = min(fair, profile);
+	if (fair < profile)
+		xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %llu vs %llu)\n",
+				 "VRAM", fair, profile);
+
+	return xe_gt_sriov_pf_config_bulk_set_lmem_locked(gt, vfid, num_vfs, fair);
 }
 
 /**
@@ -2576,7 +2704,7 @@ int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool forc
 
 static void pf_sanitize_ggtt(struct xe_ggtt_node *ggtt_region, unsigned int vfid)
 {
-	if (xe_ggtt_node_allocated(ggtt_region))
+	if (ggtt_region)
 		xe_ggtt_assign(ggtt_region, vfid);
 }
 
@@ -3035,7 +3163,7 @@ int xe_gt_sriov_pf_config_print_ggtt(struct xe_gt *gt, struct drm_printer *p)
 
 	for (n = 1; n <= total_vfs; n++) {
 		config = &gt->sriov.pf.vfs[n].config;
-		if (!xe_ggtt_node_allocated(config->ggtt_region))
+		if (!config->ggtt_region)
 			continue;
 
 		string_get_size(xe_ggtt_node_size(config->ggtt_region), 1, STRING_UNITS_2,
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
index 3c6c8b6655af7..4a004ecd61406 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
@@ -36,6 +36,10 @@ int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size
 int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
 					u64 size);
+u64 xe_gt_sriov_pf_config_get_lmem_locked(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_lmem_locked(struct xe_gt *gt, unsigned int vfid, u64 size);
+int xe_gt_sriov_pf_config_bulk_set_lmem_locked(struct xe_gt *gt, unsigned int vfid,
+					       unsigned int num_vfs, u64 size);
 struct xe_bo *xe_gt_sriov_pf_config_get_lmem_obj(struct xe_gt *gt, unsigned int vfid);
 
 u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
index bf48b05797de7..5cb705c7ee7a5 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
@@ -1259,7 +1259,7 @@ int xe_gt_sriov_pf_control_process_restore_data(struct xe_gt *gt, unsigned int v
 }
 
 /**
- * xe_gt_sriov_pf_control_trigger restore_vf() - Start an SR-IOV VF migration data restore sequence.
+ * xe_gt_sriov_pf_control_trigger_restore_vf() - Start an SR-IOV VF migration data restore sequence.
  * @gt: the &xe_gt
  * @vfid: the VF identifier
  *
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c
index b5e0a5b7723ea..6586df2fcb915 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_service.c
@@ -111,6 +111,8 @@ static const struct xe_reg ver_35_runtime_regs[] = {
 	XE2_GT_COMPUTE_DSS_2,		/* _MMIO(0x914c) */
 	XE2_GT_GEOMETRY_DSS_1,		/* _MMIO(0x9150) */
 	XE2_GT_GEOMETRY_DSS_2,		/* _MMIO(0x9154) */
+	XE3P_XPC_GT_GEOMETRY_DSS_3,	/* _MMIO(0x915c) */
+	XE3P_XPC_GT_COMPUTE_DSS_3,	/* _MMIO(0x9160) */
 	SERVICE_COPY_ENABLE,		/* _MMIO(0x9170) */
 };
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
index 30e8c2cf5f09a..8989c8e1be95e 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
@@ -488,16 +488,12 @@ u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt)
 static int vf_get_ggtt_info(struct xe_gt *gt)
 {
 	struct xe_tile *tile = gt_to_tile(gt);
-	struct xe_ggtt *ggtt = tile->mem.ggtt;
 	struct xe_guc *guc = &gt->uc.guc;
 	u64 start, size, ggtt_size;
-	s64 shift;
 	int err;
 
 	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
 
-	guard(mutex)(&ggtt->lock);
-
 	err = guc_action_query_single_klv64(guc, GUC_KLV_VF_CFG_GGTT_START_KEY, &start);
 	if (unlikely(err))
 		return err;
@@ -509,8 +505,21 @@ static int vf_get_ggtt_info(struct xe_gt *gt)
 	if (!size)
 		return -ENODATA;
 
+	xe_tile_sriov_vf_ggtt_base_store(tile, start);
 	ggtt_size = xe_tile_sriov_vf_ggtt(tile);
-	if (ggtt_size && ggtt_size != size) {
+	if (!ggtt_size) {
+		/*
+		 * This function is called once during xe_guc_init_noalloc(),
+		 * at which point ggtt_size = 0 and we have to initialize everything,
+		 * and GGTT is not yet initialized.
+		 *
+		 * Return early as there's nothing to fixup.
+		 */
+		xe_tile_sriov_vf_ggtt_store(tile, size);
+		return 0;
+	}
+
+	if (ggtt_size != size) {
 		xe_gt_sriov_err(gt, "Unexpected GGTT reassignment: %lluK != %lluK\n",
 				size / SZ_1K, ggtt_size / SZ_1K);
 		return -EREMCHG;
@@ -519,21 +528,13 @@ static int vf_get_ggtt_info(struct xe_gt *gt)
 	xe_gt_sriov_dbg_verbose(gt, "GGTT %#llx-%#llx = %lluK\n",
 				start, start + size - 1, size / SZ_1K);
 
-	shift = start - (s64)xe_tile_sriov_vf_ggtt_base(tile);
-	xe_tile_sriov_vf_ggtt_base_store(tile, start);
-	xe_tile_sriov_vf_ggtt_store(tile, size);
-
-	if (shift && shift != start) {
-		xe_gt_sriov_info(gt, "Shifting GGTT base by %lld to 0x%016llx\n",
-				 shift, start);
-		xe_tile_sriov_vf_fixup_ggtt_nodes_locked(gt_to_tile(gt), shift);
-	}
-
-	if (xe_sriov_vf_migration_supported(gt_to_xe(gt))) {
-		WRITE_ONCE(gt->sriov.vf.migration.ggtt_need_fixes, false);
-		smp_wmb();	/* Ensure above write visible before wake */
-		wake_up_all(&gt->sriov.vf.migration.wq);
-	}
+	/*
+	 * This function can be called repeatedly from post migration fixups,
+	 * at which point we inform the GGTT of the new base address.
+	 * xe_ggtt_shift_nodes() may be called multiple times for each migration,
+	 * but will be a noop if the base is unchanged.
+	 */
+	xe_ggtt_shift_nodes(tile->mem.ggtt, start);
 
 	return 0;
 }
@@ -839,6 +840,13 @@ static void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt)
 		xe_default_lrc_update_memirq_regs_with_address(hwe);
 }
 
+static void vf_post_migration_mark_fixups_done(struct xe_gt *gt)
+{
+	WRITE_ONCE(gt->sriov.vf.migration.ggtt_need_fixes, false);
+	smp_wmb();	/* Ensure above write visible before wake */
+	wake_up_all(&gt->sriov.vf.migration.wq);
+}
+
 static void vf_start_migration_recovery(struct xe_gt *gt)
 {
 	bool started;
@@ -1269,6 +1277,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt)
 	if (err)
 		return err;
 
+	atomic_inc(&gt->sriov.vf.migration.fixups_complete_count);
+
 	return 0;
 }
 
@@ -1373,6 +1383,7 @@ static void vf_post_migration_recovery(struct xe_gt *gt)
 	if (err)
 		goto fail;
 
+	vf_post_migration_mark_fixups_done(gt);
 	vf_post_migration_rearm(gt);
 
 	err = vf_post_migration_resfix_done(gt, marker);
@@ -1507,19 +1518,49 @@ static bool vf_valid_ggtt(struct xe_gt *gt)
 }
 
 /**
- * xe_gt_sriov_vf_wait_valid_ggtt() - VF wait for valid GGTT addresses
- * @gt: the &xe_gt
+ * xe_vf_migration_fixups_complete_count() - Get count of VF fixups completions.
+ * @gt: the &xe_gt instance which contains affected Global GTT
+ *
+ * Return: number of times VF fixups were completed since driver
+ * probe, or 0 if migration is not available, or -1 if fixups are
+ * pending or being applied right now.
+ */
+int xe_vf_migration_fixups_complete_count(struct xe_gt *gt)
+{
+	if (!IS_SRIOV_VF(gt_to_xe(gt)) ||
+	    !xe_sriov_vf_migration_supported(gt_to_xe(gt)))
+		return 0;
+
+	/* should never match fixups_complete_count value */
+	if (!vf_valid_ggtt(gt))
+		return -1;
+
+	return atomic_read(&gt->sriov.vf.migration.fixups_complete_count);
+}
+
+/**
+ * xe_gt_sriov_vf_wait_valid_ggtt() - wait for valid GGTT nodes and address refs
+ * @gt: the &xe_gt instance which contains affected Global GTT
+ *
+ * Return: number of times VF fixups were completed since driver
+ * probe, or 0 if migration is not available.
  */
-void xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt)
+int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt)
 {
 	int ret;
 
+	/*
+	 * this condition needs to be identical to one in
+	 * xe_vf_migration_fixups_complete_count()
+	 */
 	if (!IS_SRIOV_VF(gt_to_xe(gt)) ||
 	    !xe_sriov_vf_migration_supported(gt_to_xe(gt)))
-		return;
+		return 0;
 
 	ret = wait_event_interruptible_timeout(gt->sriov.vf.migration.wq,
 					       vf_valid_ggtt(gt),
 					       HZ * 5);
 	xe_gt_WARN_ON(gt, !ret);
+
+	return atomic_read(&gt->sriov.vf.migration.fixups_complete_count);
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
index 7d97189c2d3d9..a6f7127521a5d 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
@@ -39,6 +39,7 @@ void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p);
 void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p);
 void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p);
 
-void xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt);
+int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt);
+int xe_vf_migration_fixups_complete_count(struct xe_gt *gt);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
index 4ef881b9b6623..80562ffadb169 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
@@ -54,6 +54,8 @@ struct xe_gt_sriov_vf_migration {
 	wait_queue_head_t wq;
 	/** @scratch: Scratch memory for VF recovery */
 	void *scratch;
+	/** @fixups_complete_count: Counts completed fixups stages */
+	atomic_t fixups_complete_count;
 	/** @debug: Debug hooks for delaying migration */
 	struct {
 		/**
@@ -73,7 +75,7 @@ struct xe_gt_sriov_vf_migration {
 	bool recovery_queued;
 	/** @recovery_inprogress: VF post migration recovery in progress */
 	bool recovery_inprogress;
-	/** @ggtt_need_fixes: VF GGTT needs fixes */
+	/** @ggtt_need_fixes: VF GGTT and references to it need fixes */
 	bool ggtt_need_fixes;
 };
 
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c
index 37506434d7a36..81cec441b4490 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.c
+++ b/drivers/gpu/drm/xe/xe_gt_stats.c
@@ -3,12 +3,37 @@
  * Copyright © 2024 Intel Corporation
  */
 
-#include <linux/atomic.h>
-
+#include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 
+#include "xe_device.h"
 #include "xe_gt_stats.h"
-#include "xe_gt_types.h"
+
+static void xe_gt_stats_fini(struct drm_device *drm, void *arg)
+{
+	struct xe_gt *gt = arg;
+
+	free_percpu(gt->stats);
+}
+
+/**
+ * xe_gt_stats_init() - Initialize GT statistics
+ * @gt: GT structure
+ *
+ * Allocate per-CPU GT statistics. Using per-CPU stats allows increments
+ * to occur without cross-CPU atomics.
+ *
+ * Return: 0 on success, -ENOMEM on failure.
+ */
+int xe_gt_stats_init(struct xe_gt *gt)
+{
+	gt->stats = alloc_percpu(struct xe_gt_stats);
+	if (!gt->stats)
+		return -ENOMEM;
+
+	return drmm_add_action_or_reset(&gt_to_xe(gt)->drm, xe_gt_stats_fini,
+					gt);
+}
 
 /**
  * xe_gt_stats_incr - Increments the specified stats counter
@@ -23,7 +48,7 @@ void xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, int incr)
 	if (id >= __XE_GT_STATS_NUM_IDS)
 		return;
 
-	atomic64_add(incr, &gt->stats.counters[id]);
+	this_cpu_add(gt->stats->counters[id], incr);
 }
 
 #define DEF_STAT_STR(ID, name) [XE_GT_STATS_ID_##ID] = name
@@ -35,6 +60,7 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = {
 	DEF_STAT_STR(SVM_TLB_INVAL_US, "svm_tlb_inval_us"),
 	DEF_STAT_STR(VMA_PAGEFAULT_COUNT, "vma_pagefault_count"),
 	DEF_STAT_STR(VMA_PAGEFAULT_KB, "vma_pagefault_kb"),
+	DEF_STAT_STR(INVALID_PREFETCH_PAGEFAULT_COUNT, "invalid_prefetch_pagefault_count"),
 	DEF_STAT_STR(SVM_4K_PAGEFAULT_COUNT, "svm_4K_pagefault_count"),
 	DEF_STAT_STR(SVM_64K_PAGEFAULT_COUNT, "svm_64K_pagefault_count"),
 	DEF_STAT_STR(SVM_2M_PAGEFAULT_COUNT, "svm_2M_pagefault_count"),
@@ -94,23 +120,37 @@ int xe_gt_stats_print_info(struct xe_gt *gt, struct drm_printer *p)
 {
 	enum xe_gt_stats_id id;
 
-	for (id = 0; id < __XE_GT_STATS_NUM_IDS; ++id)
-		drm_printf(p, "%s: %lld\n", stat_description[id],
-			   atomic64_read(&gt->stats.counters[id]));
+	for (id = 0; id < __XE_GT_STATS_NUM_IDS; ++id) {
+		u64 total = 0;
+		int cpu;
+
+		for_each_possible_cpu(cpu) {
+			struct xe_gt_stats *s = per_cpu_ptr(gt->stats, cpu);
+
+			total += s->counters[id];
+		}
+
+		drm_printf(p, "%s: %lld\n", stat_description[id], total);
+	}
 
 	return 0;
 }
 
 /**
- * xe_gt_stats_clear - Clear the GT stats
+ * xe_gt_stats_clear() - Clear the GT stats
  * @gt: GT structure
  *
- * This clear (zeros) all the available GT stats.
+ * Clear (zero) all available GT stats. Note that if the stats are being
+ * updated while this function is running, the results may be unpredictable.
+ * Intended to be called on an idle GPU.
  */
 void xe_gt_stats_clear(struct xe_gt *gt)
 {
-	int id;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct xe_gt_stats *s = per_cpu_ptr(gt->stats, cpu);
 
-	for (id = 0; id < ARRAY_SIZE(gt->stats.counters); ++id)
-		atomic64_set(&gt->stats.counters[id], 0);
+		memset(s, 0, sizeof(*s));
+	}
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.h b/drivers/gpu/drm/xe/xe_gt_stats.h
index 59a7bf60e242c..3d0defab9b30e 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats.h
@@ -14,10 +14,16 @@ struct xe_gt;
 struct drm_printer;
 
 #ifdef CONFIG_DEBUG_FS
+int xe_gt_stats_init(struct xe_gt *gt);
 int xe_gt_stats_print_info(struct xe_gt *gt, struct drm_printer *p);
 void xe_gt_stats_clear(struct xe_gt *gt);
 void xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, int incr);
 #else
+static inline int xe_gt_stats_init(struct xe_gt *gt)
+{
+	return 0;
+}
+
 static inline void
 xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id,
 		 int incr)
diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h
index b8accdbc54eb3..b6081c3124743 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h
@@ -6,6 +6,8 @@
 #ifndef _XE_GT_STATS_TYPES_H_
 #define _XE_GT_STATS_TYPES_H_
 
+#include <linux/types.h>
+
 enum xe_gt_stats_id {
 	XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_TLB_INVAL,
@@ -13,6 +15,7 @@ enum xe_gt_stats_id {
 	XE_GT_STATS_ID_SVM_TLB_INVAL_US,
 	XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
+	XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT,
@@ -58,4 +61,21 @@ enum xe_gt_stats_id {
 	__XE_GT_STATS_NUM_IDS,
 };
 
+/**
+ * struct xe_gt_stats - Per-CPU GT statistics counters
+ * @counters: Array of 64-bit counters indexed by &enum xe_gt_stats_id
+ *
+ * This structure is used for high-frequency, per-CPU statistics collection
+ * in the Xe driver. By using a per-CPU allocation and ensuring the structure
+ * is cache-line aligned, we avoid the performance-heavy atomics and cache
+ * coherency traffic.
+ *
+ * Updates to these counters should be performed using the this_cpu_add()
+ * macro to ensure they are atomic with respect to local interrupts and
+ * preemption-safe without the overhead of explicit locking.
+ */
+struct xe_gt_stats {
+	u64 counters[__XE_GT_STATS_NUM_IDS];
+} ____cacheline_aligned;
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c
index bd5260221d8db..bfe87e6828793 100644
--- a/drivers/gpu/drm/xe/xe_gt_topology.c
+++ b/drivers/gpu/drm/xe/xe_gt_topology.c
@@ -205,24 +205,6 @@ load_l3_bank_mask(struct xe_gt *gt, xe_l3_bank_mask_t l3_bank_mask)
 	}
 }
 
-static void
-get_num_dss_regs(struct xe_device *xe, int *geometry_regs, int *compute_regs)
-{
-	if (GRAPHICS_VER(xe) > 20) {
-		*geometry_regs = 3;
-		*compute_regs = 3;
-	} else if (GRAPHICS_VERx100(xe) == 1260) {
-		*geometry_regs = 0;
-		*compute_regs = 2;
-	} else if (GRAPHICS_VERx100(xe) >= 1250) {
-		*geometry_regs = 1;
-		*compute_regs = 1;
-	} else {
-		*geometry_regs = 1;
-		*compute_regs = 0;
-	}
-}
-
 void
 xe_gt_topology_init(struct xe_gt *gt)
 {
@@ -230,29 +212,27 @@ xe_gt_topology_init(struct xe_gt *gt)
 		XELP_GT_GEOMETRY_DSS_ENABLE,
 		XE2_GT_GEOMETRY_DSS_1,
 		XE2_GT_GEOMETRY_DSS_2,
+		XE3P_XPC_GT_GEOMETRY_DSS_3,
 	};
 	static const struct xe_reg compute_regs[] = {
 		XEHP_GT_COMPUTE_DSS_ENABLE,
 		XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,
 		XE2_GT_COMPUTE_DSS_2,
+		XE3P_XPC_GT_COMPUTE_DSS_3,
 	};
-	int num_geometry_regs, num_compute_regs;
-	struct xe_device *xe = gt_to_xe(gt);
 	struct drm_printer p;
 
-	get_num_dss_regs(xe, &num_geometry_regs, &num_compute_regs);
-
 	/*
 	 * Register counts returned shouldn't exceed the number of registers
 	 * passed as parameters below.
 	 */
-	xe_gt_assert(gt, num_geometry_regs <= ARRAY_SIZE(geometry_regs));
-	xe_gt_assert(gt, num_compute_regs <= ARRAY_SIZE(compute_regs));
+	xe_gt_assert(gt, gt->info.num_geometry_xecore_fuse_regs <= ARRAY_SIZE(geometry_regs));
+	xe_gt_assert(gt, gt->info.num_compute_xecore_fuse_regs <= ARRAY_SIZE(compute_regs));
 
 	load_dss_mask(gt, gt->fuse_topo.g_dss_mask,
-		      num_geometry_regs, geometry_regs);
+		      gt->info.num_geometry_xecore_fuse_regs, geometry_regs);
 	load_dss_mask(gt, gt->fuse_topo.c_dss_mask,
-		      num_compute_regs, compute_regs);
+		      gt->info.num_compute_xecore_fuse_regs, compute_regs);
 
 	load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss, &gt->fuse_topo.eu_type);
 	load_l3_bank_mask(gt, gt->fuse_topo.l3_bank_mask);
@@ -330,15 +310,14 @@ xe_l3_bank_mask_ffs(const xe_l3_bank_mask_t mask)
  */
 bool xe_gt_topology_has_dss_in_quadrant(struct xe_gt *gt, int quad)
 {
-	struct xe_device *xe = gt_to_xe(gt);
 	xe_dss_mask_t all_dss;
-	int g_dss_regs, c_dss_regs, dss_per_quad, quad_first;
+	int dss_per_quad, quad_first;
 
 	bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
 		  XE_MAX_DSS_FUSE_BITS);
 
-	get_num_dss_regs(xe, &g_dss_regs, &c_dss_regs);
-	dss_per_quad = 32 * max(g_dss_regs, c_dss_regs) / 4;
+	dss_per_quad = 32 * max(gt->info.num_geometry_xecore_fuse_regs,
+				gt->info.num_compute_xecore_fuse_regs) / 4;
 
 	quad_first = xe_dss_mask_group_ffs(all_dss, dss_per_quad, quad);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index 5318d92fd473f..8b55cf25a75fa 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -35,7 +35,7 @@ enum xe_gt_eu_type {
 	XE_GT_EU_TYPE_SIMD16,
 };
 
-#define XE_MAX_DSS_FUSE_REGS		3
+#define XE_MAX_DSS_FUSE_REGS		4
 #define XE_MAX_DSS_FUSE_BITS		(32 * XE_MAX_DSS_FUSE_REGS)
 #define XE_MAX_EU_FUSE_REGS		1
 #define XE_MAX_EU_FUSE_BITS		(32 * XE_MAX_EU_FUSE_REGS)
@@ -45,11 +45,6 @@ typedef unsigned long xe_dss_mask_t[BITS_TO_LONGS(XE_MAX_DSS_FUSE_BITS)];
 typedef unsigned long xe_eu_mask_t[BITS_TO_LONGS(XE_MAX_EU_FUSE_BITS)];
 typedef unsigned long xe_l3_bank_mask_t[BITS_TO_LONGS(XE_MAX_L3_BANK_MASK_BITS)];
 
-struct xe_mmio_range {
-	u32 start;
-	u32 end;
-};
-
 /*
  * The hardware has multiple kinds of multicast register ranges that need
  * special register steering (and future platforms are expected to add
@@ -149,14 +144,21 @@ struct xe_gt {
 		u8 id;
 		/** @info.has_indirect_ring_state: GT has indirect ring state support */
 		u8 has_indirect_ring_state:1;
+		/**
+		 * @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse
+		 * registers the geometry XeCore mask spans.
+		 */
+		u8 num_geometry_xecore_fuse_regs;
+		/**
+		 * @info.num_compute_xecore_fuse_regs: Number of 32b-bit fuse
+		 * registers the compute XeCore mask spans.
+		 */
+		u8 num_compute_xecore_fuse_regs;
 	} info;
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	/** @stats: GT stats */
-	struct {
-		/** @stats.counters: counters for various GT stats */
-		atomic64_t counters[__XE_GT_STATS_NUM_IDS];
-	} stats;
+	struct xe_gt_stats __percpu *stats;
 #endif
 
 	/**
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index 6df7c3f260e5b..576f3d500390c 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -35,11 +35,13 @@
 #include "xe_guc_klv_helpers.h"
 #include "xe_guc_log.h"
 #include "xe_guc_pc.h"
+#include "xe_guc_rc.h"
 #include "xe_guc_relay.h"
 #include "xe_guc_submit.h"
 #include "xe_memirq.h"
 #include "xe_mmio.h"
 #include "xe_platform_types.h"
+#include "xe_sleep.h"
 #include "xe_sriov.h"
 #include "xe_sriov_pf_migration.h"
 #include "xe_uc.h"
@@ -211,9 +213,6 @@ static u32 guc_ctl_wa_flags(struct xe_guc *guc)
 	    !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
 		flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
 
-	if (XE_GT_WA(gt, 1509372804))
-		flags |= GUC_WA_RENDER_RST_RC6_EXIT;
-
 	if (XE_GT_WA(gt, 14018913170))
 		flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
 
@@ -668,6 +667,13 @@ static void guc_fini_hw(void *arg)
 	guc_g2g_fini(guc);
 }
 
+static void vf_guc_fini_hw(void *arg)
+{
+	struct xe_guc *guc = arg;
+
+	xe_gt_sriov_vf_reset(guc_to_gt(guc));
+}
+
 /**
  * xe_guc_comm_init_early - early initialization of GuC communication
  * @guc: the &xe_guc to initialize
@@ -772,6 +778,10 @@ int xe_guc_init(struct xe_guc *guc)
 		xe->info.has_page_reclaim_hw_assist = false;
 
 	if (IS_SRIOV_VF(xe)) {
+		ret = devm_add_action_or_reset(xe->drm.dev, vf_guc_fini_hw, guc);
+		if (ret)
+			goto out;
+
 		ret = xe_guc_ct_init(&guc->ct);
 		if (ret)
 			goto out;
@@ -869,6 +879,10 @@ int xe_guc_init_post_hwconfig(struct xe_guc *guc)
 	if (ret)
 		return ret;
 
+	ret = xe_guc_rc_init(guc);
+	if (ret)
+		return ret;
+
 	ret = xe_guc_engine_activity_init(guc);
 	if (ret)
 		return ret;
@@ -900,6 +914,41 @@ int xe_guc_post_load_init(struct xe_guc *guc)
 	return xe_guc_submit_enable(guc);
 }
 
+/*
+ * Wa_14025883347: Prevent GuC firmware DMA failures during GuC-only reset by ensuring
+ * SRAM save/restore operations are complete before reset.
+ */
+static void guc_prevent_fw_dma_failure_on_reset(struct xe_guc *guc)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	u32 boot_hash_chk, guc_status, sram_status;
+	int ret;
+
+	guc_status = xe_mmio_read32(&gt->mmio, GUC_STATUS);
+	if (guc_status & GS_MIA_IN_RESET)
+		return;
+
+	boot_hash_chk = xe_mmio_read32(&gt->mmio, BOOT_HASH_CHK);
+	if (!(boot_hash_chk & GUC_BOOT_UKERNEL_VALID))
+		return;
+
+	/* Disable idle flow during reset (GuC reset re-enables it automatically) */
+	xe_mmio_rmw32(&gt->mmio, GUC_MAX_IDLE_COUNT, 0, GUC_IDLE_FLOW_DISABLE);
+
+	ret = xe_mmio_wait32(&gt->mmio, GUC_STATUS, GS_UKERNEL_MASK,
+			     FIELD_PREP(GS_UKERNEL_MASK, XE_GUC_LOAD_STATUS_READY),
+			     100000, &guc_status, false);
+	if (ret)
+		xe_gt_warn(gt, "GuC not ready after disabling idle flow (GUC_STATUS: 0x%x)\n",
+			   guc_status);
+
+	ret = xe_mmio_wait32(&gt->mmio, GUC_SRAM_STATUS, GUC_SRAM_HANDLING_MASK,
+			     0, 5000, &sram_status, false);
+	if (ret)
+		xe_gt_warn(gt, "SRAM handling not complete (GUC_SRAM_STATUS: 0x%x)\n",
+			   sram_status);
+}
+
 int xe_guc_reset(struct xe_guc *guc)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
@@ -912,6 +961,9 @@ int xe_guc_reset(struct xe_guc *guc)
 	if (IS_SRIOV_VF(gt_to_xe(gt)))
 		return xe_gt_sriov_vf_bootstrap(gt);
 
+	if (XE_GT_WA(gt, 14025883347))
+		guc_prevent_fw_dma_failure_on_reset(guc);
+
 	xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
 
 	ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
@@ -1124,14 +1176,14 @@ static int guc_wait_ucode(struct xe_guc *guc)
 	struct xe_guc_pc *guc_pc = &gt->uc.guc.pc;
 	u32 before_freq, act_freq, cur_freq;
 	u32 status = 0, tries = 0;
+	int load_result, ret;
 	ktime_t before;
 	u64 delta_ms;
-	int ret;
 
 	before_freq = xe_guc_pc_get_act_freq(guc_pc);
 	before = ktime_get();
 
-	ret = poll_timeout_us(ret = guc_load_done(gt, &status, &tries), ret,
+	ret = poll_timeout_us(load_result = guc_load_done(gt, &status, &tries), load_result,
 			      10 * USEC_PER_MSEC,
 			      GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false);
 
@@ -1139,7 +1191,7 @@ static int guc_wait_ucode(struct xe_guc *guc)
 	act_freq = xe_guc_pc_get_act_freq(guc_pc);
 	cur_freq = xe_guc_pc_get_cur_freq_fw(guc_pc);
 
-	if (ret) {
+	if (ret || load_result <= 0) {
 		xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n",
 			  status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
 			  xe_guc_pc_get_cur_freq_fw(guc_pc));
@@ -1347,15 +1399,37 @@ int xe_guc_enable_communication(struct xe_guc *guc)
 	return 0;
 }
 
-int xe_guc_suspend(struct xe_guc *guc)
+/**
+ * xe_guc_softreset() - Soft reset GuC
+ * @guc: The GuC object
+ *
+ * Send soft reset command to GuC through mmio send.
+ *
+ * Return: 0 if success, otherwise error code
+ */
+int xe_guc_softreset(struct xe_guc *guc)
 {
-	struct xe_gt *gt = guc_to_gt(guc);
 	u32 action[] = {
 		XE_GUC_ACTION_CLIENT_SOFT_RESET,
 	};
 	int ret;
 
+	if (!xe_uc_fw_is_running(&guc->fw))
+		return 0;
+
 	ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int xe_guc_suspend(struct xe_guc *guc)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	int ret;
+
+	ret = xe_guc_softreset(guc);
 	if (ret) {
 		xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
 		return ret;
@@ -1388,17 +1462,21 @@ int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
 	return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
 }
 
+#define MAX_RETRIES_ON_FLR	2
+#define MIN_SLEEP_MS_ON_FLR	256
+
 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 			  u32 len, u32 *response_buf)
 {
 	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
 	struct xe_mmio *mmio = &gt->mmio;
-	u32 header, reply;
 	struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
 		MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
 	const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
-	bool lost = false;
+	unsigned int sleep_period_ms = 1;
+	unsigned int lost = 0;
+	u32 header;
 	int ret;
 	int i;
 
@@ -1430,21 +1508,25 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 
 	ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN,
 			     FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
-			     50000, &reply, false);
+			     50000, &header, false);
 	if (ret) {
 		/* scratch registers might be cleared during FLR, try once more */
-		if (!reply && !lost) {
+		if (!header) {
+			if (++lost > MAX_RETRIES_ON_FLR) {
+				xe_gt_err(gt, "GuC mmio request %#x: lost, too many retries %u\n",
+					  request[0], lost);
+				return -ENOLINK;
+			}
 			xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]);
-			lost = true;
+			xe_sleep_relaxed_ms(MIN_SLEEP_MS_ON_FLR);
 			goto retry;
 		}
 timeout:
 		xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
-			  request[0], reply);
+			  request[0], header);
 		return ret;
 	}
 
-	header = xe_mmio_read32(mmio, reply_reg);
 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
 	    GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
 		/*
@@ -1480,6 +1562,8 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 
 		xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
 			  request[0], reason);
+
+		xe_sleep_exponential_ms(&sleep_period_ms, 256);
 		goto retry;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h
index 66e7edc70ed9f..02514914f4048 100644
--- a/drivers/gpu/drm/xe/xe_guc.h
+++ b/drivers/gpu/drm/xe/xe_guc.h
@@ -44,6 +44,7 @@ int xe_guc_opt_in_features_enable(struct xe_guc *guc);
 void xe_guc_runtime_suspend(struct xe_guc *guc);
 void xe_guc_runtime_resume(struct xe_guc *guc);
 int xe_guc_suspend(struct xe_guc *guc);
+int xe_guc_softreset(struct xe_guc *guc);
 void xe_guc_notify(struct xe_guc *guc);
 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index f4cbc030f4c81..81b5f01b1f65c 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -819,6 +819,7 @@ static void guc_um_init_params(struct xe_guc_ads *ads)
 {
 	u32 um_queue_offset = guc_ads_um_queues_offset(ads);
 	struct xe_guc *guc = ads_to_guc(ads);
+	struct xe_device *xe = ads_to_xe(ads);
 	u64 base_dpa;
 	u32 base_ggtt;
 	bool with_dpa;
@@ -830,6 +831,16 @@ static void guc_um_init_params(struct xe_guc_ads *ads)
 	base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset;
 
 	for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) {
+		/*
+		 * Some platforms support USM but not access counters.
+		 * Skip ACCESS_COUNTER queue initialization for such
+		 * platforms, leaving queue_params[2] zero-initialized
+		 * to signal unavailability to the GuC.
+		 */
+		if (i == GUC_UM_HW_QUEUE_ACCESS_COUNTER &&
+		    !xe->info.has_access_counter)
+			continue;
+
 		ads_blob_write(ads, um_init_params.queue_params[i].base_dpa,
 			       with_dpa ? (base_dpa + (i * GUC_UM_QUEUE_SIZE)) : 0);
 		ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address,
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index d04589140b776..a11cff7a20be7 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -31,7 +31,9 @@
 #include "xe_guc_submit.h"
 #include "xe_guc_tlb_inval.h"
 #include "xe_map.h"
+#include "xe_page_reclaim.h"
 #include "xe_pm.h"
+#include "xe_sleep.h"
 #include "xe_sriov_vf.h"
 #include "xe_trace_guc.h"
 
@@ -254,6 +256,7 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
 
 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
 #define CTB_H2G_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
+#define CTB_G2H_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
 #define CTB_H2G_BUFFER_DWORDS	(CTB_H2G_BUFFER_SIZE / sizeof(u32))
 #define CTB_G2H_BUFFER_SIZE	(SZ_128K)
@@ -274,14 +277,18 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
  */
 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
 {
-	BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
+	BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4K));
 	return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
 }
 
-static size_t guc_ct_size(void)
+static size_t guc_h2g_size(void)
 {
-	return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE +
-		CTB_G2H_BUFFER_SIZE;
+	return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE;
+}
+
+static size_t guc_g2h_size(void)
+{
+	return CTB_G2H_BUFFER_OFFSET + CTB_G2H_BUFFER_SIZE;
 }
 
 static void guc_ct_fini(struct drm_device *drm, void *arg)
@@ -310,7 +317,8 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct)
 	struct xe_gt *gt = ct_to_gt(ct);
 	int err;
 
-	xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
+	xe_gt_assert(gt, !(guc_h2g_size() % PAGE_SIZE));
+	xe_gt_assert(gt, !(guc_g2h_size() % PAGE_SIZE));
 
 	err = drmm_mutex_init(&xe->drm, &ct->lock);
 	if (err)
@@ -345,6 +353,7 @@ static void guc_action_disable_ct(void *arg)
 {
 	struct xe_guc_ct *ct = arg;
 
+	xe_guc_ct_stop(ct);
 	guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED);
 }
 
@@ -355,7 +364,7 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_bo *bo;
 
-	bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
+	bo = xe_managed_bo_create_pin_map(xe, tile, guc_h2g_size(),
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
@@ -363,7 +372,17 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
-	ct->bo = bo;
+	ct->ctbs.h2g.bo = bo;
+
+	bo = xe_managed_bo_create_pin_map(xe, tile, guc_g2h_size(),
+					  XE_BO_FLAG_SYSTEM |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE |
+					  XE_BO_FLAG_PINNED_NORESTORE);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	ct->ctbs.g2h.bo = bo;
 
 	return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct);
 }
@@ -388,7 +407,7 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct)
 	xe_assert(xe, !xe_guc_ct_enabled(ct));
 
 	if (IS_DGFX(xe)) {
-		ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->bo);
+		ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->ctbs.h2g.bo);
 		if (ret)
 			return ret;
 	}
@@ -438,8 +457,7 @@ static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
 	g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
 	xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
 
-	g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET +
-					    CTB_H2G_BUFFER_SIZE);
+	g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_G2H_BUFFER_OFFSET);
 }
 
 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
@@ -448,8 +466,8 @@ static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
 	u32 desc_addr, ctb_addr, size;
 	int err;
 
-	desc_addr = xe_bo_ggtt_addr(ct->bo);
-	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET;
+	desc_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo);
+	ctb_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo) + CTB_H2G_BUFFER_OFFSET;
 	size = ct->ctbs.h2g.info.size * sizeof(u32);
 
 	err = xe_guc_self_cfg64(guc,
@@ -475,9 +493,8 @@ static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
 	u32 desc_addr, ctb_addr, size;
 	int err;
 
-	desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
-	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET +
-		CTB_H2G_BUFFER_SIZE;
+	desc_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_DESC_SIZE;
+	ctb_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_G2H_BUFFER_OFFSET;
 	size = ct->ctbs.g2h.info.size * sizeof(u32);
 
 	err = xe_guc_self_cfg64(guc,
@@ -604,9 +621,12 @@ static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register)
 	xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
 
 	if (needs_register) {
-		xe_map_memset(xe, &ct->bo->vmap, 0, 0, xe_bo_size(ct->bo));
-		guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
-		guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
+		xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, 0, 0,
+			      xe_bo_size(ct->ctbs.h2g.bo));
+		xe_map_memset(xe, &ct->ctbs.g2h.bo->vmap, 0, 0,
+			      xe_bo_size(ct->ctbs.g2h.bo));
+		guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->ctbs.h2g.bo->vmap);
+		guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->ctbs.g2h.bo->vmap);
 
 		err = guc_ct_ctb_h2g_register(ct);
 		if (err)
@@ -623,7 +643,7 @@ static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register)
 		ct->ctbs.h2g.info.broken = false;
 		ct->ctbs.g2h.info.broken = false;
 		/* Skip everything in H2G buffer */
-		xe_map_memset(xe, &ct->bo->vmap, CTB_H2G_BUFFER_OFFSET, 0,
+		xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, CTB_H2G_BUFFER_OFFSET, 0,
 			      CTB_H2G_BUFFER_SIZE);
 	}
 
@@ -643,7 +663,7 @@ static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register)
 	spin_lock_irq(&ct->dead.lock);
 	if (ct->dead.reason) {
 		ct->dead.reason |= (1 << CT_DEAD_STATE_REARM);
-		queue_work(system_unbound_wq, &ct->dead.worker);
+		queue_work(system_dfl_wq, &ct->dead.worker);
 	}
 	spin_unlock_irq(&ct->dead.lock);
 #endif
@@ -921,22 +941,22 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	u32 full_len;
 	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
 							 tail * sizeof(u32));
-	u32 desc_status;
 
 	full_len = len + GUC_CTB_HDR_LEN;
 
 	lockdep_assert_held(&ct->lock);
 	xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
 
-	desc_status = desc_read(xe, h2g, status);
-	if (desc_status) {
-		xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
-		goto corrupted;
-	}
-
 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
 		u32 desc_tail = desc_read(xe, h2g, tail);
 		u32 desc_head = desc_read(xe, h2g, head);
+		u32 desc_status;
+
+		desc_status = desc_read(xe, h2g, status);
+		if (desc_status) {
+			xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
+			goto corrupted;
+		}
 
 		if (tail != desc_tail) {
 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
@@ -1005,8 +1025,15 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	/* Update descriptor */
 	desc_write(xe, h2g, tail, h2g->info.tail);
 
-	trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
-			     desc_read(xe, h2g, head), h2g->info.tail);
+	/*
+	 * desc_read() performs an VRAM read which serializes the CPU and drains
+	 * posted writes on dGPU platforms. Tracepoints evaluate arguments even
+	 * when disabled, so guard the event to avoid adding µs-scale latency to
+	 * the fast H2G submission path when tracing is not active.
+	 */
+	if (trace_xe_guc_ctb_h2g_enabled())
+		trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
+				     desc_read(xe, h2g, head), h2g->info.tail);
 
 	return 0;
 
@@ -1101,7 +1128,8 @@ static int dequeue_one_g2h(struct xe_guc_ct *ct);
  */
 static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len,
 				       u32 g2h_len, struct g2h_fence *g2h_fence,
-				       unsigned int *sleep_period_ms)
+				       unsigned int *sleep_period_ms,
+				       unsigned int *sleep_total_ms)
 {
 	struct xe_device *xe = ct_to_xe(ct);
 
@@ -1115,17 +1143,15 @@ static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len,
 	if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) {
 		struct guc_ctb *h2g = &ct->ctbs.h2g;
 
-		if (*sleep_period_ms == 1024)
+		if (*sleep_total_ms > 1000)
 			return false;
 
 		trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
 						 h2g->info.size,
 						 h2g->info.space,
 						 len + GUC_CTB_HDR_LEN);
-		msleep(*sleep_period_ms);
-		*sleep_period_ms <<= 1;
+		*sleep_total_ms += xe_sleep_exponential_ms(sleep_period_ms, 64);
 	} else {
-		struct xe_device *xe = ct_to_xe(ct);
 		struct guc_ctb *g2h = &ct->ctbs.g2h;
 		int ret;
 
@@ -1147,7 +1173,7 @@ static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len,
 		ret = dequeue_one_g2h(ct);
 		if (ret < 0) {
 			if (ret != -ECANCELED)
-				xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)",
+				xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)\n",
 					  ERR_PTR(ret));
 			return false;
 		}
@@ -1161,6 +1187,7 @@ static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
 {
 	struct xe_gt *gt = ct_to_gt(ct);
 	unsigned int sleep_period_ms = 1;
+	unsigned int sleep_total_ms = 0;
 	int ret;
 
 	xe_gt_assert(gt, !g2h_len || !g2h_fence);
@@ -1173,7 +1200,7 @@ static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
 
 	if (unlikely(ret == -EBUSY)) {
 		if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence,
-						&sleep_period_ms))
+						&sleep_period_ms, &sleep_total_ms))
 			goto broken;
 		goto try_again;
 	}
@@ -1322,7 +1349,7 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	 */
 	mutex_lock(&ct->lock);
 	if (!ret) {
-		xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",
+		xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s\n",
 			  g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));
 		xa_erase(&ct->fence_lookup, g2h_fence.seqno);
 		mutex_unlock(&ct->lock);
@@ -1604,17 +1631,11 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
-	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
-		/*
-		 * Page reclamation is an extension of TLB invalidation. Both
-		 * operations share the same seqno and fence. When either
-		 * action completes, we need to signal the corresponding
-		 * fence. Since the handling logic (lookup fence by seqno,
-		 * fence signalling) is identical, we use the same handler
-		 * for both G2H events.
-		 */
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len);
+		break;
 	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
 		ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
 		break;
@@ -1822,17 +1843,15 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
-	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
-		/*
-		 * Seqno and fence handling of page reclamation and TLB
-		 * invalidation is identical, so we can use the same handler
-		 * for both actions.
-		 */
 		__g2h_release_space(ct, len);
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		__g2h_release_space(ct, len);
+		ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len);
+		break;
 	default:
-		xe_gt_warn(gt, "NOT_POSSIBLE");
+		xe_gt_warn(gt, "NOT_POSSIBLE\n");
 	}
 
 	if (ret) {
@@ -1935,7 +1954,7 @@ static void receive_g2h(struct xe_guc_ct *ct)
 		mutex_unlock(&ct->lock);
 
 		if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
-			xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret);
+			xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d\n", ret);
 			CT_DEAD(ct, NULL, G2H_RECV);
 			kick_reset(ct);
 		}
@@ -1961,8 +1980,9 @@ static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bo
 	if (!snapshot)
 		return NULL;
 
-	if (ct->bo && want_ctb) {
-		snapshot->ctb_size = xe_bo_size(ct->bo);
+	if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && want_ctb) {
+		snapshot->ctb_size = xe_bo_size(ct->ctbs.h2g.bo) +
+			xe_bo_size(ct->ctbs.g2h.bo);
 		snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL);
 	}
 
@@ -2010,8 +2030,13 @@ static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct,
 		guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h);
 	}
 
-	if (ct->bo && snapshot->ctb)
-		xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size);
+	if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && snapshot->ctb) {
+		xe_map_memcpy_from(xe, snapshot->ctb, &ct->ctbs.h2g.bo->vmap, 0,
+				   xe_bo_size(ct->ctbs.h2g.bo));
+		xe_map_memcpy_from(xe, snapshot->ctb + xe_bo_size(ct->ctbs.h2g.bo),
+				   &ct->ctbs.g2h.bo->vmap, 0,
+				   xe_bo_size(ct->ctbs.g2h.bo));
+	}
 
 	return snapshot;
 }
@@ -2165,7 +2190,7 @@ static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reaso
 
 	spin_unlock_irqrestore(&ct->dead.lock, flags);
 
-	queue_work(system_unbound_wq, &(ct)->dead.worker);
+	queue_work(system_dfl_wq, &(ct)->dead.worker);
 }
 
 static void ct_dead_print(struct xe_dead_ct *dead)
diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h
index 09d7ff1ef42a0..5da1ce5dc3724 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h
@@ -39,6 +39,8 @@ struct guc_ctb_info {
  * struct guc_ctb - GuC command transport buffer (CTB)
  */
 struct guc_ctb {
+	/** @bo: Xe BO for CTB */
+	struct xe_bo *bo;
 	/** @desc: dma buffer map for CTB descriptor */
 	struct iosys_map desc;
 	/** @cmds: dma buffer map for CTB commands */
@@ -72,7 +74,7 @@ struct xe_guc_ct_snapshot {
 	/** @ctb_size: size of the snapshot of the CTB */
 	size_t ctb_size;
 	/** @ctb: snapshot of the entire CTB */
-	u32 *ctb;
+	void *ctb;
 };
 
 /**
@@ -126,8 +128,6 @@ struct xe_fast_req_fence {
  * for the H2G and G2H requests sent and received through the buffers.
  */
 struct xe_guc_ct {
-	/** @bo: Xe BO for CT */
-	struct xe_bo *bo;
 	/** @lock: protects everything in CT layer */
 	struct mutex lock;
 	/** @fast_lock: protects G2H channel and credits */
diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h
index a33ea288b907f..bb8f71d386114 100644
--- a/drivers/gpu/drm/xe/xe_guc_fwif.h
+++ b/drivers/gpu/drm/xe/xe_guc_fwif.h
@@ -261,7 +261,8 @@ struct xe_guc_pagefault_desc {
 #define PFD_ACCESS_TYPE		GENMASK(1, 0)
 #define PFD_FAULT_TYPE		GENMASK(3, 2)
 #define PFD_VFID		GENMASK(9, 4)
-#define PFD_RSVD_1		GENMASK(11, 10)
+#define PFD_RSVD_1		BIT(10)
+#define PFD_PREFETCH		BIT(11) /* Only valid on Xe3+, reserved on prior platforms */
 #define PFD_VIRTUAL_ADDR_LO	GENMASK(31, 12)
 #define PFD_VIRTUAL_ADDR_LO_SHIFT 12
 
@@ -281,7 +282,7 @@ struct xe_guc_pagefault_reply {
 
 	u32 dw1;
 #define PFR_VFID		GENMASK(5, 0)
-#define PFR_RSVD_1		BIT(6)
+#define PFR_PREFETCH		BIT(6)  /* Only valid on Xe3+, reserved on prior platforms */
 #define PFR_ENG_INSTANCE	GENMASK(12, 7)
 #define PFR_ENG_CLASS		GENMASK(15, 13)
 #define PFR_PDATA		GENMASK(31, 16)
diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h
index 1b05bb60c1c7f..4649a260755ed 100644
--- a/drivers/gpu/drm/xe/xe_guc_log.h
+++ b/drivers/gpu/drm/xe/xe_guc_log.h
@@ -13,9 +13,13 @@ struct drm_printer;
 struct xe_device;
 
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_8M
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_16M
 #define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_1M
 #define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_2M
+#elif IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_8M
+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_1M
+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_1M
 #else
 #define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_64K
 #define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_16K
diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.c b/drivers/gpu/drm/xe/xe_guc_pagefault.c
index 719a18187a31d..607e32392f46f 100644
--- a/drivers/gpu/drm/xe/xe_guc_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_guc_pagefault.c
@@ -8,15 +8,18 @@
 #include "xe_guc_ct.h"
 #include "xe_guc_pagefault.h"
 #include "xe_pagefault.h"
+#include "xe_pagefault_types.h"
 
 static void guc_ack_fault(struct xe_pagefault *pf, int err)
 {
 	u32 vfid = FIELD_GET(PFD_VFID, pf->producer.msg[2]);
+	u32 prefetch = FIELD_GET(PFD_PREFETCH, pf->producer.msg[2]);
 	u32 engine_instance = FIELD_GET(PFD_ENG_INSTANCE, pf->producer.msg[0]);
 	u32 engine_class = FIELD_GET(PFD_ENG_CLASS, pf->producer.msg[0]);
 	u32 pdata = FIELD_GET(PFD_PDATA_LO, pf->producer.msg[0]) |
 		(FIELD_GET(PFD_PDATA_HI, pf->producer.msg[1]) <<
 		 PFD_PDATA_HI_SHIFT);
+	u32 asid = FIELD_GET(PFD_ASID, pf->producer.msg[1]);
 	u32 action[] = {
 		XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
 
@@ -24,9 +27,10 @@ static void guc_ack_fault(struct xe_pagefault *pf, int err)
 		FIELD_PREP(PFR_SUCCESS, !!err) |
 		FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
 		FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
-		FIELD_PREP(PFR_ASID, pf->consumer.asid),
+		FIELD_PREP(PFR_ASID, asid),
 
 		FIELD_PREP(PFR_VFID, vfid) |
+		FIELD_PREP(PFR_PREFETCH, err ? prefetch : 0) |
 		FIELD_PREP(PFR_ENG_INSTANCE, engine_instance) |
 		FIELD_PREP(PFR_ENG_CLASS, engine_class) |
 		FIELD_PREP(PFR_PDATA, pdata),
@@ -75,12 +79,16 @@ int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
 		(FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) <<
 		 PFD_VIRTUAL_ADDR_LO_SHIFT);
 	pf.consumer.asid = FIELD_GET(PFD_ASID, msg[1]);
-	pf.consumer.access_type = FIELD_GET(PFD_ACCESS_TYPE, msg[2]);
-	pf.consumer.fault_type = FIELD_GET(PFD_FAULT_TYPE, msg[2]);
+	pf.consumer.access_type = FIELD_GET(PFD_ACCESS_TYPE, msg[2]) |
+		(FIELD_GET(PFD_PREFETCH, msg[2]) ? XE_PAGEFAULT_ACCESS_PREFETCH : 0);
 	if (FIELD_GET(XE2_PFD_TRVA_FAULT, msg[0]))
-		pf.consumer.fault_level = XE_PAGEFAULT_LEVEL_NACK;
+		pf.consumer.fault_type_level = XE_PAGEFAULT_TYPE_LEVEL_NACK;
 	else
-		pf.consumer.fault_level = FIELD_GET(PFD_FAULT_LEVEL, msg[0]);
+		pf.consumer.fault_type_level =
+			FIELD_PREP(XE_PAGEFAULT_LEVEL_MASK,
+				   FIELD_GET(PFD_FAULT_LEVEL, msg[0])) |
+			FIELD_PREP(XE_PAGEFAULT_TYPE_MASK,
+				   FIELD_GET(PFD_FAULT_TYPE, msg[2]));
 	pf.consumer.engine_class = FIELD_GET(PFD_ENG_CLASS, msg[0]);
 	pf.consumer.engine_instance = FIELD_GET(PFD_ENG_INSTANCE, msg[0]);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index 5e5495a39a3c9..bb8c4e7934926 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -92,6 +92,17 @@
  * Render-C states is also a GuC PC feature that is now enabled in Xe for
  * all platforms.
  *
+ * Implementation details:
+ * -----------------------
+ * The implementation for GuC Power Management features is split as follows:
+ *
+ * xe_guc_rc:  Logic for handling GuC RC
+ * xe_gt_idle: Host side logic for RC6 and Coarse Power gating (CPG)
+ * xe_guc_pc:  Logic for all other SLPC related features
+ *
+ * There is some cross interaction between these where host C6 will need to be
+ * enabled when we plan to skip GuC RC. Also, the GuC RC mode is currently
+ * overridden through 0x3003 which is an SLPC H2G call.
  */
 
 static struct xe_guc *pc_to_guc(struct xe_guc_pc *pc)
@@ -253,20 +264,35 @@ static int pc_action_unset_param(struct xe_guc_pc *pc, u8 id)
 	return ret;
 }
 
-static int pc_action_setup_gucrc(struct xe_guc_pc *pc, u32 mode)
+/**
+ * xe_guc_pc_action_set_param() - Set value of SLPC param
+ * @pc: Xe_GuC_PC instance
+ * @id: Param id
+ * @value: Value to set
+ *
+ * This function can be used to set any SLPC param.
+ *
+ * Return: 0 on Success
+ */
+int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value)
 {
-	struct xe_guc_ct *ct = pc_to_ct(pc);
-	u32 action[] = {
-		GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC,
-		mode,
-	};
-	int ret;
+	xe_device_assert_mem_access(pc_to_xe(pc));
+	return pc_action_set_param(pc, id, value);
+}
 
-	ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
-	if (ret && !(xe_device_wedged(pc_to_xe(pc)) && ret == -ECANCELED))
-		xe_gt_err(pc_to_gt(pc), "GuC RC enable mode=%u failed: %pe\n",
-			  mode, ERR_PTR(ret));
-	return ret;
+/**
+ * xe_guc_pc_action_unset_param() - Revert to default value
+ * @pc: Xe_GuC_PC instance
+ * @id: Param id
+ *
+ * This function can be used revert any SLPC param to its default value.
+ *
+ * Return: 0 on Success
+ */
+int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id)
+{
+	xe_device_assert_mem_access(pc_to_xe(pc));
+	return pc_action_unset_param(pc, id);
 }
 
 static u32 decode_freq(u32 raw)
@@ -730,6 +756,14 @@ enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc)
 	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
 		reg = xe_mmio_read32(&gt->mmio, MTL_MIRROR_TARGET_WP1);
 		gt_c_state = REG_FIELD_GET(MTL_CC_MASK, reg);
+
+		/*
+		 * There are higher level sleep states that will cause this
+		 * field to read out as its reset state, and those are only
+		 * possible after the GT is already in C6.
+		 */
+		if (gt_c_state == MTL_CRST)
+			gt_c_state = GT_C6;
 	} else {
 		reg = xe_mmio_read32(&gt->mmio, GT_CORE_STATUS);
 		gt_c_state = REG_FIELD_GET(RCN_MASK, reg);
@@ -1050,55 +1084,6 @@ int xe_guc_pc_restore_stashed_freq(struct xe_guc_pc *pc)
 	return ret;
 }
 
-/**
- * xe_guc_pc_gucrc_disable - Disable GuC RC
- * @pc: Xe_GuC_PC instance
- *
- * Disables GuC RC by taking control of RC6 back from GuC.
- *
- * Return: 0 on success, negative error code on error.
- */
-int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc)
-{
-	struct xe_device *xe = pc_to_xe(pc);
-	struct xe_gt *gt = pc_to_gt(pc);
-	int ret = 0;
-
-	if (xe->info.skip_guc_pc)
-		return 0;
-
-	ret = pc_action_setup_gucrc(pc, GUCRC_HOST_CONTROL);
-	if (ret)
-		return ret;
-
-	return xe_gt_idle_disable_c6(gt);
-}
-
-/**
- * xe_guc_pc_override_gucrc_mode - override GUCRC mode
- * @pc: Xe_GuC_PC instance
- * @mode: new value of the mode.
- *
- * Return: 0 on success, negative error code on error
- */
-int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
-{
-	guard(xe_pm_runtime)(pc_to_xe(pc));
-	return pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
-}
-
-/**
- * xe_guc_pc_unset_gucrc_mode - unset GUCRC mode override
- * @pc: Xe_GuC_PC instance
- *
- * Return: 0 on success, negative error code on error
- */
-int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
-{
-	guard(xe_pm_runtime)(pc_to_xe(pc));
-	return pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
-}
-
 static void pc_init_pcode_freq(struct xe_guc_pc *pc)
 {
 	u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER);
@@ -1247,9 +1232,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 		return -ETIMEDOUT;
 
 	if (xe->info.skip_guc_pc) {
-		if (xe->info.platform != XE_PVC)
-			xe_gt_idle_enable_c6(gt);
-
 		/* Request max possible since dynamic freq mgmt is not enabled */
 		pc_set_cur_freq(pc, UINT_MAX);
 		return 0;
@@ -1291,15 +1273,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 	if (ret)
 		return ret;
 
-	if (xe->info.platform == XE_PVC) {
-		xe_guc_pc_gucrc_disable(pc);
-		return 0;
-	}
-
-	ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
-	if (ret)
-		return ret;
-
 	/* Enable SLPC Optimized Strategy for compute */
 	ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);
 
@@ -1319,10 +1292,8 @@ int xe_guc_pc_stop(struct xe_guc_pc *pc)
 {
 	struct xe_device *xe = pc_to_xe(pc);
 
-	if (xe->info.skip_guc_pc) {
-		xe_gt_idle_disable_c6(pc_to_gt(pc));
+	if (xe->info.skip_guc_pc)
 		return 0;
-	}
 
 	mutex_lock(&pc->freq_lock);
 	pc->freq_ready = false;
@@ -1343,8 +1314,7 @@ static void xe_guc_pc_fini_hw(void *arg)
 	if (xe_device_wedged(xe))
 		return;
 
-	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
-	xe_guc_pc_gucrc_disable(pc);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT);
 	XE_WARN_ON(xe_guc_pc_stop(pc));
 
 	/* Bind requested freq to mert_freq_cap before unload */
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h
index 0e31396f103cd..0678a4e787b3f 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.h
+++ b/drivers/gpu/drm/xe/xe_guc_pc.h
@@ -9,16 +9,14 @@
 #include <linux/types.h>
 
 struct xe_guc_pc;
-enum slpc_gucrc_mode;
 struct drm_printer;
 
 int xe_guc_pc_init(struct xe_guc_pc *pc);
 int xe_guc_pc_start(struct xe_guc_pc *pc);
 int xe_guc_pc_stop(struct xe_guc_pc *pc);
-int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc);
-int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode);
-int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc);
 void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p);
+int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value);
+int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id);
 
 u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc);
 int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq);
diff --git a/drivers/gpu/drm/xe/xe_guc_rc.c b/drivers/gpu/drm/xe/xe_guc_rc.c
new file mode 100644
index 0000000000000..99fa127b261ff
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_rc.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <drm/drm_print.h>
+
+#include "abi/guc_actions_slpc_abi.h"
+#include "xe_device.h"
+#include "xe_force_wake.h"
+#include "xe_gt.h"
+#include "xe_gt_idle.h"
+#include "xe_gt_printk.h"
+#include "xe_guc.h"
+#include "xe_guc_ct.h"
+#include "xe_guc_pc.h"
+#include "xe_guc_rc.h"
+#include "xe_pm.h"
+
+/**
+ * DOC: GuC RC (Render C-states)
+ *
+ * GuC handles the GT transition to deeper C-states in conjunction with Pcode.
+ * GuC RC can be enabled independently of the frequency component in SLPC,
+ * which is also controlled by GuC.
+ *
+ * This file will contain all H2G related logic for handling Render C-states.
+ * There are some calls to xe_gt_idle, where we enable host C6 when GuC RC is
+ * skipped. GuC RC is mostly independent of xe_guc_pc with the exception of
+ * functions that override the mode for which we have to rely on the SLPC H2G
+ * calls.
+ */
+
+static int guc_action_setup_gucrc(struct xe_guc *guc, u32 control)
+{
+	u32 action[] = {
+		GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC,
+		control,
+	};
+	int ret;
+
+	ret = xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
+	if (ret && !(xe_device_wedged(guc_to_xe(guc)) && ret == -ECANCELED))
+		xe_gt_err(guc_to_gt(guc),
+			  "GuC RC setup %s(%u) failed (%pe)\n",
+			   control == GUCRC_HOST_CONTROL ? "HOST_CONTROL" :
+			   control == GUCRC_FIRMWARE_CONTROL ? "FIRMWARE_CONTROL" :
+			   "UNKNOWN", control, ERR_PTR(ret));
+	return ret;
+}
+
+/**
+ * xe_guc_rc_disable() - Disable GuC RC
+ * @guc: Xe GuC instance
+ *
+ * Disables GuC RC by taking control of RC6 back from GuC.
+ */
+void xe_guc_rc_disable(struct xe_guc *guc)
+{
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
+
+	if (!xe->info.skip_guc_pc && xe->info.platform != XE_PVC)
+		if (guc_action_setup_gucrc(guc, GUCRC_HOST_CONTROL))
+			return;
+
+	xe_gt_WARN_ON(gt, xe_gt_idle_disable_c6(gt));
+}
+
+static void xe_guc_rc_fini_hw(void *arg)
+{
+	struct xe_guc *guc = arg;
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
+
+	if (xe_device_wedged(xe))
+		return;
+
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	xe_guc_rc_disable(guc);
+}
+
+/**
+ * xe_guc_rc_init() - Init GuC RC
+ * @guc: Xe GuC instance
+ *
+ * Add callback action for GuC RC
+ *
+ * Return: 0 on success, negative error code on error.
+ */
+int xe_guc_rc_init(struct xe_guc *guc)
+{
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
+
+	xe_gt_assert(gt, xe_device_uc_enabled(xe));
+
+	return devm_add_action_or_reset(xe->drm.dev, xe_guc_rc_fini_hw, guc);
+}
+
+/**
+ * xe_guc_rc_enable() - Enable GuC RC feature if applicable
+ * @guc: Xe GuC instance
+ *
+ * Enables GuC RC feature.
+ *
+ * Return: 0 on success, negative error code on error.
+ */
+int xe_guc_rc_enable(struct xe_guc *guc)
+{
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
+
+	xe_gt_assert(gt, xe_device_uc_enabled(xe));
+
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
+		return -ETIMEDOUT;
+
+	if (xe->info.platform == XE_PVC) {
+		xe_guc_rc_disable(guc);
+		return 0;
+	}
+
+	if (xe->info.skip_guc_pc) {
+		xe_gt_idle_enable_c6(gt);
+		return 0;
+	}
+
+	return guc_action_setup_gucrc(guc, GUCRC_FIRMWARE_CONTROL);
+}
diff --git a/drivers/gpu/drm/xe/xe_guc_rc.h b/drivers/gpu/drm/xe/xe_guc_rc.h
new file mode 100644
index 0000000000000..b083fc364dd43
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_rc.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_GUC_RC_H_
+#define _XE_GUC_RC_H_
+
+struct xe_guc;
+enum slpc_gucrc_mode;
+
+int xe_guc_rc_init(struct xe_guc *guc);
+int xe_guc_rc_enable(struct xe_guc *guc);
+void xe_guc_rc_disable(struct xe_guc *guc);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index 799ef9f480036..a145234f662b5 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -8,9 +8,7 @@
 #include <linux/bitfield.h>
 #include <linux/bitmap.h>
 #include <linux/circ_buf.h>
-#include <linux/delay.h>
 #include <linux/dma-fence-array.h>
-#include <linux/math64.h>
 
 #include <drm/drm_managed.h>
 
@@ -42,12 +40,15 @@
 #include "xe_pm.h"
 #include "xe_ring_ops_types.h"
 #include "xe_sched_job.h"
+#include "xe_sleep.h"
 #include "xe_trace.h"
 #include "xe_uc_fw.h"
 #include "xe_vm.h"
 
 #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN		6
 
+static int guc_submit_reset_prepare(struct xe_guc *guc);
+
 static struct xe_guc *
 exec_queue_to_guc(struct xe_exec_queue *q)
 {
@@ -239,7 +240,7 @@ static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
 		 EXEC_QUEUE_STATE_BANNED));
 }
 
-static void guc_submit_fini(struct drm_device *drm, void *arg)
+static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc *guc = arg;
 	struct xe_device *xe = guc_to_xe(guc);
@@ -257,6 +258,19 @@ static void guc_submit_fini(struct drm_device *drm, void *arg)
 	xa_destroy(&guc->submission_state.exec_queue_lookup);
 }
 
+static void guc_submit_fini(void *arg)
+{
+	struct xe_guc *guc = arg;
+
+	/* Forcefully kill any remaining exec queues */
+	xe_guc_ct_stop(&guc->ct);
+	guc_submit_reset_prepare(guc);
+	xe_guc_softreset(guc);
+	xe_guc_submit_stop(guc);
+	xe_uc_fw_sanitize(&guc->fw);
+	xe_guc_submit_pause_abort(guc);
+}
+
 static void guc_submit_wedged_fini(void *arg)
 {
 	struct xe_guc *guc = arg;
@@ -326,7 +340,11 @@ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids)
 
 	guc->submission_state.initialized = true;
 
-	return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
+	err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc);
+	if (err)
+		return err;
+
+	return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc);
 }
 
 /*
@@ -556,6 +574,72 @@ static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
 	xe_sched_tdr_queue_imm(&q->guc->sched);
 }
 
+static void xe_guc_exec_queue_group_stop(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_exec_queue *eq, *next;
+	LIST_HEAD(tmp);
+
+	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
+		     xe_exec_queue_is_multi_queue(q));
+
+	mutex_lock(&group->list_lock);
+
+	/*
+	 * Stop all future queues being from executing while group is stopped.
+	 */
+	group->stopped = true;
+
+	list_for_each_entry_safe(eq, next, &group->list, multi_queue.link)
+		/*
+		 * Refcount prevents an attempted removal from &group->list,
+		 * temporary list allows safe iteration after dropping
+		 * &group->list_lock.
+		 */
+		if (xe_exec_queue_get_unless_zero(eq))
+			list_move_tail(&eq->multi_queue.link, &tmp);
+
+	mutex_unlock(&group->list_lock);
+
+	/* We cannot stop under list lock without getting inversions */
+	xe_sched_submission_stop(&primary->guc->sched);
+	list_for_each_entry(eq, &tmp, multi_queue.link)
+		xe_sched_submission_stop(&eq->guc->sched);
+
+	mutex_lock(&group->list_lock);
+	list_for_each_entry_safe(eq, next, &tmp, multi_queue.link) {
+		/*
+		 * Corner where we got banned while stopping and not on
+		 * &group->list
+		 */
+		if (READ_ONCE(group->banned))
+			xe_guc_exec_queue_trigger_cleanup(eq);
+
+		list_move_tail(&eq->multi_queue.link, &group->list);
+		xe_exec_queue_put(eq);
+	}
+	mutex_unlock(&group->list_lock);
+}
+
+static void xe_guc_exec_queue_group_start(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_exec_queue *eq;
+
+	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
+		     xe_exec_queue_is_multi_queue(q));
+
+	xe_sched_submission_start(&primary->guc->sched);
+
+	mutex_lock(&group->list_lock);
+	group->stopped = false;
+	list_for_each_entry(eq, &group->list, multi_queue.link)
+		xe_sched_submission_start(&eq->guc->sched);
+	mutex_unlock(&group->list_lock);
+}
+
 static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
 {
 	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
@@ -738,6 +822,7 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 {
 	struct xe_exec_queue_group *group = q->multi_queue.group;
 	struct xe_device *xe = guc_to_xe(guc);
+	enum xe_multi_queue_priority priority;
 	long ret;
 
 	/*
@@ -761,7 +846,10 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 		return;
 	}
 
-	xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority);
+	scoped_guard(spinlock, &q->multi_queue.lock)
+		priority = q->multi_queue.priority;
+
+	xe_lrc_set_multi_queue_priority(q->lrc[0], priority);
 	xe_guc_exec_queue_group_cgp_update(xe, q);
 
 	WRITE_ONCE(group->sync_pending, true);
@@ -962,24 +1050,6 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q)
 	return (WQ_SIZE - q->guc->wqi_tail);
 }
 
-static inline void relaxed_ms_sleep(unsigned int delay_ms)
-{
-	unsigned long min_us, max_us;
-
-	if (!delay_ms)
-		return;
-
-	if (delay_ms > 20) {
-		msleep(delay_ms);
-		return;
-	}
-
-	min_us = mul_u32_u32(delay_ms, 1000);
-	max_us = min_us + 500;
-
-	usleep_range(min_us, max_us);
-}
-
 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
 {
 	struct xe_guc *guc = exec_queue_to_guc(q);
@@ -998,10 +1068,7 @@ static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
 				return -ENODEV;
 			}
 
-			msleep(sleep_period_ms);
-			sleep_total_ms += sleep_period_ms;
-			if (sleep_period_ms < 64)
-				sleep_period_ms <<= 1;
+			sleep_total_ms += xe_sleep_exponential_ms(&sleep_period_ms, 64);
 			goto try_again;
 		}
 	}
@@ -1252,6 +1319,7 @@ static void disable_scheduling_deregister(struct xe_guc *guc,
  */
 void xe_guc_submit_wedge(struct xe_guc *guc)
 {
+	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
 	struct xe_exec_queue *q;
 	unsigned long index;
@@ -1266,20 +1334,29 @@ void xe_guc_submit_wedge(struct xe_guc *guc)
 	if (!guc->submission_state.initialized)
 		return;
 
-	err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
-				       guc_submit_wedged_fini, guc);
-	if (err) {
-		xe_gt_err(gt, "Failed to register clean-up in wedged.mode=%s; "
-			  "Although device is wedged.\n",
-			  xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET));
-		return;
-	}
+	if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
+		err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
+					       guc_submit_wedged_fini, guc);
+		if (err) {
+			xe_gt_err(gt, "Failed to register clean-up on wedged.mode=%s; "
+				  "Although device is wedged.\n",
+				  xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET));
+			return;
+		}
 
-	mutex_lock(&guc->submission_state.lock);
-	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
-		if (xe_exec_queue_get_unless_zero(q))
-			set_exec_queue_wedged(q);
-	mutex_unlock(&guc->submission_state.lock);
+		mutex_lock(&guc->submission_state.lock);
+		xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
+			if (xe_exec_queue_get_unless_zero(q))
+				set_exec_queue_wedged(q);
+		mutex_unlock(&guc->submission_state.lock);
+	} else {
+		/* Forcefully kill any remaining exec queues, signal fences */
+		guc_submit_reset_prepare(guc);
+		xe_guc_submit_stop(guc);
+		xe_guc_softreset(guc);
+		xe_uc_fw_sanitize(&guc->fw);
+		xe_guc_submit_pause_abort(guc);
+	}
 }
 
 static bool guc_submit_hint_wedged(struct xe_guc *guc)
@@ -1414,7 +1491,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 {
 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
 	struct drm_sched_job *tmp_job;
-	struct xe_exec_queue *q = job->q;
+	struct xe_exec_queue *q = job->q, *primary;
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	const char *process_name = "no process";
@@ -1425,6 +1502,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 
 	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
 
+	primary = xe_exec_queue_multi_queue_primary(q);
+
 	/*
 	 * TDR has fired before free job worker. Common if exec queue
 	 * immediately closed after last fence signaled. Add back to pending
@@ -1436,7 +1515,10 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 		return DRM_GPU_SCHED_STAT_NO_HANG;
 
 	/* Kill the run_job entry point */
-	xe_sched_submission_stop(sched);
+	if (xe_exec_queue_is_multi_queue(q))
+		xe_guc_exec_queue_group_stop(q);
+	else
+		xe_sched_submission_stop(sched);
 
 	/* Must check all state after stopping scheduler */
 	skip_timeout_check = exec_queue_reset(q) ||
@@ -1451,14 +1533,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	if (xe_exec_queue_is_lr(q))
 		xe_gt_assert(guc_to_gt(guc), skip_timeout_check);
 
-	/*
-	 * FIXME: In multi-queue scenario, the TDR must ensure that the whole
-	 * multi-queue group is off the HW before signaling the fences to avoid
-	 * possible memory corruptions. This means disabling scheduling on the
-	 * primary queue before or during the secondary queue's TDR. Need to
-	 * implement this in least obtrusive way.
-	 */
-
 	/*
 	 * If devcoredump not captured and GuC capture for the job is not ready
 	 * do manual capture first and decide later if we need to use it
@@ -1485,10 +1559,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	set_exec_queue_banned(q);
 
 	/* Kick job / queue off hardware */
-	if (!wedged && (exec_queue_enabled(q) || exec_queue_pending_disable(q))) {
+	if (!wedged && (exec_queue_enabled(primary) ||
+			exec_queue_pending_disable(primary))) {
 		int ret;
 
-		if (exec_queue_reset(q))
+		if (exec_queue_reset(primary))
 			err = -EIO;
 
 		if (xe_uc_fw_is_running(&guc->fw)) {
@@ -1497,8 +1572,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 			 * modifying state
 			 */
 			ret = wait_event_timeout(guc->ct.wq,
-						 (!exec_queue_pending_enable(q) &&
-						  !exec_queue_pending_disable(q)) ||
+						 (!exec_queue_pending_enable(primary) &&
+						  !exec_queue_pending_disable(primary)) ||
 						 xe_guc_read_stopped(guc) ||
 						 vf_recovery(guc), HZ * 5);
 			if (vf_recovery(guc))
@@ -1506,7 +1581,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 			if (!ret || xe_guc_read_stopped(guc))
 				goto trigger_reset;
 
-			disable_scheduling(q, skip_timeout_check);
+			disable_scheduling(primary, skip_timeout_check);
 		}
 
 		/*
@@ -1520,7 +1595,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 		smp_rmb();
 		ret = wait_event_timeout(guc->ct.wq,
 					 !xe_uc_fw_is_running(&guc->fw) ||
-					 !exec_queue_pending_disable(q) ||
+					 !exec_queue_pending_disable(primary) ||
 					 xe_guc_read_stopped(guc) ||
 					 vf_recovery(guc), HZ * 5);
 		if (vf_recovery(guc))
@@ -1530,11 +1605,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 			if (!ret)
 				xe_gt_warn(guc_to_gt(guc),
 					   "Schedule disable failed to respond, guc_id=%d",
-					   q->guc->id);
-			xe_devcoredump(q, job,
+					   primary->guc->id);
+			xe_devcoredump(primary, job,
 				       "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d",
-				       q->guc->id, ret, xe_guc_read_stopped(guc));
-			xe_gt_reset_async(q->gt);
+				       primary->guc->id, ret, xe_guc_read_stopped(guc));
+			xe_gt_reset_async(primary->gt);
 			xe_sched_tdr_queue_imm(sched);
 			goto rearm;
 		}
@@ -1580,12 +1655,13 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL)
 		xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED);
 
-	xe_sched_submission_start(sched);
-
-	if (xe_exec_queue_is_multi_queue(q))
+	if (xe_exec_queue_is_multi_queue(q)) {
+		xe_guc_exec_queue_group_start(q);
 		xe_guc_exec_queue_group_trigger_cleanup(q);
-	else
+	} else {
+		xe_sched_submission_start(sched);
 		xe_guc_exec_queue_trigger_cleanup(q);
+	}
 
 	/*
 	 * We want the job added back to the pending list so it gets freed; this
@@ -1599,7 +1675,10 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	 * but there is not currently an easy way to do in DRM scheduler. With
 	 * some thought, do this in a follow up.
 	 */
-	xe_sched_submission_start(sched);
+	if (xe_exec_queue_is_multi_queue(q))
+		xe_guc_exec_queue_group_start(q);
+	else
+		xe_sched_submission_start(sched);
 handle_vf_resume:
 	return DRM_GPU_SCHED_STAT_NO_HANG;
 }
@@ -1762,7 +1841,7 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
 				since_resume_ms;
 
 			if (wait_ms > 0 && q->guc->resume_time)
-				relaxed_ms_sleep(wait_ms);
+				xe_sleep_relaxed_ms(wait_ms);
 
 			set_exec_queue_suspended(q);
 			disable_scheduling(q, false);
@@ -1965,6 +2044,8 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 
 		INIT_LIST_HEAD(&q->multi_queue.link);
 		mutex_lock(&group->list_lock);
+		if (group->stopped)
+			WRITE_ONCE(q->guc->sched.base.pause_submit, true);
 		list_add_tail(&q->multi_queue.link, &group->list);
 		mutex_unlock(&group->list_lock);
 	}
@@ -2111,15 +2192,22 @@ static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
 
 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));
 
-	if (q->multi_queue.priority == priority ||
-	    exec_queue_killed_or_banned_or_wedged(q))
+	if (exec_queue_killed_or_banned_or_wedged(q))
 		return 0;
 
 	msg = kmalloc_obj(*msg);
 	if (!msg)
 		return -ENOMEM;
 
-	q->multi_queue.priority = priority;
+	scoped_guard(spinlock, &q->multi_queue.lock) {
+		if (q->multi_queue.priority == priority) {
+			kfree(msg);
+			return 0;
+		}
+
+		q->multi_queue.priority = priority;
+	}
+
 	guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);
 
 	return 0;
@@ -2206,6 +2294,14 @@ static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
 	return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);
 }
 
+static bool guc_exec_queue_active(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+
+	return exec_queue_enabled(primary) &&
+		!exec_queue_pending_disable(primary);
+}
+
 /*
  * All of these functions are an abstraction layer which other parts of Xe can
  * use to trap into the GuC backend. All of these functions, aside from init,
@@ -2225,11 +2321,13 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = {
 	.suspend_wait = guc_exec_queue_suspend_wait,
 	.resume = guc_exec_queue_resume,
 	.reset_status = guc_exec_queue_reset_status,
+	.active = guc_exec_queue_active,
 };
 
 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
+	bool do_destroy = false;
 
 	/* Stop scheduling + flush any DRM scheduler operations */
 	xe_sched_submission_stop(sched);
@@ -2237,7 +2335,7 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
 	/* Clean up lost G2H + reset engine state */
 	if (exec_queue_registered(q)) {
 		if (exec_queue_destroyed(q))
-			__guc_exec_queue_destroy(guc, q);
+			do_destroy = true;
 	}
 	if (q->guc->suspend_pending) {
 		set_exec_queue_suspended(q);
@@ -2273,18 +2371,15 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
 			xe_guc_exec_queue_trigger_cleanup(q);
 		}
 	}
+
+	if (do_destroy)
+		__guc_exec_queue_destroy(guc, q);
 }
 
-int xe_guc_submit_reset_prepare(struct xe_guc *guc)
+static int guc_submit_reset_prepare(struct xe_guc *guc)
 {
 	int ret;
 
-	if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
-		return 0;
-
-	if (!guc->submission_state.initialized)
-		return 0;
-
 	/*
 	 * Using an atomic here rather than submission_state.lock as this
 	 * function can be called while holding the CT lock (engine reset
@@ -2299,6 +2394,17 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc)
 	return ret;
 }
 
+int xe_guc_submit_reset_prepare(struct xe_guc *guc)
+{
+	if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
+		return 0;
+
+	if (!guc->submission_state.initialized)
+		return 0;
+
+	return guc_submit_reset_prepare(guc);
+}
+
 void xe_guc_submit_reset_wait(struct xe_guc *guc)
 {
 	wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) ||
@@ -2695,8 +2801,7 @@ void xe_guc_submit_pause_abort(struct xe_guc *guc)
 			continue;
 
 		xe_sched_submission_start(sched);
-		if (exec_queue_killed_or_banned_or_wedged(q))
-			xe_guc_exec_queue_trigger_cleanup(q);
+		guc_exec_queue_kill(q);
 	}
 	mutex_unlock(&guc->submission_state.lock);
 }
diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
index 774467befbb9f..ced58f46f8466 100644
--- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
@@ -6,15 +6,19 @@
 #include "abi/guc_actions_abi.h"
 
 #include "xe_device.h"
+#include "xe_exec_queue.h"
+#include "xe_exec_queue_types.h"
 #include "xe_gt_stats.h"
 #include "xe_gt_types.h"
 #include "xe_guc.h"
 #include "xe_guc_ct.h"
+#include "xe_guc_exec_queue_types.h"
 #include "xe_guc_tlb_inval.h"
 #include "xe_force_wake.h"
 #include "xe_mmio.h"
 #include "xe_sa.h"
 #include "xe_tlb_inval.h"
+#include "xe_vm.h"
 
 #include "regs/xe_guc_regs.h"
 
@@ -111,6 +115,38 @@ static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
 			      G2H_LEN_DW_PAGE_RECLAMATION, 1);
 }
 
+static u64 normalize_invalidation_range(struct xe_gt *gt, u64 *start, u64 *end)
+{
+	u64 orig_start = *start;
+	u64 length = *end - *start;
+	u64 align;
+
+	if (length < SZ_4K)
+		length = SZ_4K;
+
+	align = roundup_pow_of_two(length);
+	*start = ALIGN_DOWN(*start, align);
+	*end = ALIGN(*end, align);
+	length = align;
+	while (*start + length < *end) {
+		length <<= 1;
+		*start = ALIGN_DOWN(orig_start, length);
+	}
+
+	if (length >= SZ_2M) {
+		length = max_t(u64, SZ_16M, length);
+		*start = ALIGN_DOWN(orig_start, length);
+	}
+
+	xe_gt_assert(gt, length >= SZ_4K);
+	xe_gt_assert(gt, is_power_of_2(length));
+	xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
+					    ilog2(SZ_2M) + 1)));
+	xe_gt_assert(gt, IS_ALIGNED(*start, length));
+
+	return length;
+}
+
 /*
  * Ensure that roundup_pow_of_two(length) doesn't overflow.
  * Note that roundup_pow_of_two() operates on unsigned long,
@@ -118,19 +154,21 @@ static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
  */
 #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
 
-static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
-				u64 start, u64 end, u32 asid,
+static int send_tlb_inval_ppgtt(struct xe_guc *guc, u32 seqno, u64 start,
+				u64 end, u32 id, u32 type,
 				struct drm_suballoc *prl_sa)
 {
 #define MAX_TLB_INVALIDATION_LEN	7
-	struct xe_guc *guc = tlb_inval->private;
 	struct xe_gt *gt = guc_to_gt(guc);
+	struct xe_device *xe = guc_to_xe(guc);
 	u32 action[MAX_TLB_INVALIDATION_LEN];
 	u64 length = end - start;
 	int len = 0, err;
 
-	if (guc_to_xe(guc)->info.force_execlist)
-		return -ECANCELED;
+	xe_gt_assert(gt, (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE &&
+			  !xe->info.has_ctx_tlb_inval) ||
+		     (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX &&
+		      xe->info.has_ctx_tlb_inval));
 
 	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
 	action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID;
@@ -138,55 +176,150 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
 	    length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
 	} else {
-		u64 orig_start = start;
-		u64 align;
+		u64 normalize_len = normalize_invalidation_range(gt, &start,
+								 &end);
+		bool need_flush = !prl_sa &&
+			seqno != TLB_INVALIDATION_SEQNO_INVALID;
+
+		/* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
+		action[len++] = MAKE_INVAL_OP_FLUSH(type, need_flush);
+		action[len++] = id;
+		action[len++] = lower_32_bits(start);
+		action[len++] = upper_32_bits(start);
+		action[len++] = ilog2(normalize_len) - ilog2(SZ_4K);
+	}
 
-		if (length < SZ_4K)
-			length = SZ_4K;
+	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
+#undef MAX_TLB_INVALIDATION_LEN
 
-		/*
-		 * We need to invalidate a higher granularity if start address
-		 * is not aligned to length. When start is not aligned with
-		 * length we need to find the length large enough to create an
-		 * address mask covering the required range.
-		 */
-		align = roundup_pow_of_two(length);
-		start = ALIGN_DOWN(start, align);
-		end = ALIGN(end, align);
-		length = align;
-		while (start + length < end) {
-			length <<= 1;
-			start = ALIGN_DOWN(orig_start, length);
+	err = send_tlb_inval(guc, action, len);
+	if (!err && prl_sa) {
+		xe_gt_assert(gt, seqno != TLB_INVALIDATION_SEQNO_INVALID);
+		err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
+	}
+	return err;
+}
+
+static int send_tlb_inval_asid_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
+				     u64 start, u64 end, u32 asid,
+				     struct drm_suballoc *prl_sa)
+{
+	struct xe_guc *guc = tlb_inval->private;
+
+	lockdep_assert_held(&tlb_inval->seqno_lock);
+
+	if (guc_to_xe(guc)->info.force_execlist)
+		return -ECANCELED;
+
+	return send_tlb_inval_ppgtt(guc, seqno, start, end, asid,
+				    XE_GUC_TLB_INVAL_PAGE_SELECTIVE, prl_sa);
+}
+
+static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
+				    u64 start, u64 end, u32 asid,
+				    struct drm_suballoc *prl_sa)
+{
+	struct xe_guc *guc = tlb_inval->private;
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_exec_queue *q, *next, *last_q = NULL;
+	struct xe_vm *vm;
+	LIST_HEAD(tlb_inval_list);
+	int err = 0, id = guc_to_gt(guc)->info.id;
+
+	lockdep_assert_held(&tlb_inval->seqno_lock);
+
+	if (xe->info.force_execlist)
+		return -ECANCELED;
+
+	vm = xe_device_asid_to_vm(xe, asid);
+	if (IS_ERR(vm))
+		return PTR_ERR(vm);
+
+	down_read(&vm->exec_queues.lock);
+
+	/*
+	 * XXX: Randomly picking a threshold for now. This will need to be
+	 * tuned based on expected UMD queue counts and performance profiling.
+	 */
+#define EXEC_QUEUE_COUNT_FULL_THRESHOLD	8
+	if (vm->exec_queues.count[id] >= EXEC_QUEUE_COUNT_FULL_THRESHOLD) {
+		u32 action[] = {
+			XE_GUC_ACTION_TLB_INVALIDATION,
+			seqno,
+			MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
+		};
+
+		err = send_tlb_inval(guc, action, ARRAY_SIZE(action));
+		goto err_unlock;
+	}
+#undef EXEC_QUEUE_COUNT_FULL_THRESHOLD
+
+	/*
+	 * Move exec queues to a temporary list to issue invalidations. The exec
+	 * queue must active and a reference must be taken to prevent concurrent
+	 * deregistrations.
+	 *
+	 * List modification is safe because we hold 'vm->exec_queues.lock' for
+	 * reading, which prevents external modifications. Using a per-GT list
+	 * is also safe since 'tlb_inval->seqno_lock' ensures no other GT users
+	 * can enter this code path.
+	 */
+	list_for_each_entry_safe(q, next, &vm->exec_queues.list[id],
+				 vm_exec_queue_link) {
+		if (q->ops->active(q) && xe_exec_queue_get_unless_zero(q)) {
+			last_q = q;
+			list_move_tail(&q->vm_exec_queue_link, &tlb_inval_list);
 		}
+	}
 
+	if (!last_q) {
 		/*
-		 * Minimum invalidation size for a 2MB page that the hardware
-		 * expects is 16MB
+		 * We can't break fence ordering for TLB invalidation jobs, if
+		 * TLB invalidations are inflight issue a dummy invalidation to
+		 * maintain ordering. Nor can we move safely the seqno_recv when
+		 * returning -ECANCELED if TLB invalidations are in flight. Use
+		 * GGTT invalidation as dummy invalidation given ASID
+		 * invalidations are unsupported here.
 		 */
-		if (length >= SZ_2M) {
-			length = max_t(u64, SZ_16M, length);
-			start = ALIGN_DOWN(orig_start, length);
+		if (xe_tlb_inval_idle(tlb_inval))
+			err = -ECANCELED;
+		else
+			err = send_tlb_inval_ggtt(tlb_inval, seqno);
+		goto err_unlock;
+	}
+
+	list_for_each_entry_safe(q, next, &tlb_inval_list, vm_exec_queue_link) {
+		struct drm_suballoc *__prl_sa = NULL;
+		int __seqno = TLB_INVALIDATION_SEQNO_INVALID;
+		u32 type = XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX;
+
+		xe_assert(xe, q->vm == vm);
+
+		if (err)
+			goto unref;
+
+		if (last_q == q) {
+			__prl_sa = prl_sa;
+			__seqno = seqno;
 		}
 
-		xe_gt_assert(gt, length >= SZ_4K);
-		xe_gt_assert(gt, is_power_of_2(length));
-		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
-						    ilog2(SZ_2M) + 1)));
-		xe_gt_assert(gt, IS_ALIGNED(start, length));
+		err = send_tlb_inval_ppgtt(guc, __seqno, start, end,
+					   q->guc->id, type, __prl_sa);
 
-		/* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
-		action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, !prl_sa);
-		action[len++] = asid;
-		action[len++] = lower_32_bits(start);
-		action[len++] = upper_32_bits(start);
-		action[len++] = ilog2(length) - ilog2(SZ_4K);
+unref:
+		/*
+		 * Must always return exec queue to original list / drop
+		 * reference
+		 */
+		list_move_tail(&q->vm_exec_queue_link,
+			       &vm->exec_queues.list[id]);
+		xe_exec_queue_put(q);
 	}
 
-	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
+err_unlock:
+	up_read(&vm->exec_queues.lock);
+	xe_vm_put(vm);
 
-	err = send_tlb_inval(guc, action, len);
-	if (!err && prl_sa)
-		err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
 	return err;
 }
 
@@ -217,10 +350,19 @@ static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval)
 	return hw_tlb_timeout + 2 * delay;
 }
 
-static const struct xe_tlb_inval_ops guc_tlb_inval_ops = {
+static const struct xe_tlb_inval_ops guc_tlb_inval_asid_ops = {
 	.all = send_tlb_inval_all,
 	.ggtt = send_tlb_inval_ggtt,
-	.ppgtt = send_tlb_inval_ppgtt,
+	.ppgtt = send_tlb_inval_asid_ppgtt,
+	.initialized = tlb_inval_initialized,
+	.flush = tlb_inval_flush,
+	.timeout_delay = tlb_inval_timeout_delay,
+};
+
+static const struct xe_tlb_inval_ops guc_tlb_inval_ctx_ops = {
+	.ggtt = send_tlb_inval_ggtt,
+	.all = send_tlb_inval_all,
+	.ppgtt = send_tlb_inval_ctx_ppgtt,
 	.initialized = tlb_inval_initialized,
 	.flush = tlb_inval_flush,
 	.timeout_delay = tlb_inval_timeout_delay,
@@ -237,8 +379,14 @@ static const struct xe_tlb_inval_ops guc_tlb_inval_ops = {
 void xe_guc_tlb_inval_init_early(struct xe_guc *guc,
 				 struct xe_tlb_inval *tlb_inval)
 {
+	struct xe_device *xe = guc_to_xe(guc);
+
 	tlb_inval->private = guc;
-	tlb_inval->ops = &guc_tlb_inval_ops;
+
+	if (xe->info.has_ctx_tlb_inval)
+		tlb_inval->ops = &guc_tlb_inval_ctx_ops;
+	else
+		tlb_inval->ops = &guc_tlb_inval_asid_ops;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index 688d645e0e735..337baf0a6e878 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -327,21 +327,21 @@ void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe)
 {
 	u32 ccs_mask =
 		xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE);
-	u32 ring_mode = _MASKED_BIT_ENABLE(GFX_DISABLE_LEGACY_MODE);
+	u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE);
 
 	if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask)
 		xe_mmio_write32(&hwe->gt->mmio, RCU_MODE,
-				_MASKED_BIT_ENABLE(RCU_MODE_CCS_ENABLE));
+				REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE));
 
 	xe_hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0);
 	xe_hw_engine_mmio_write32(hwe, RING_HWS_PGA(0),
 				  xe_bo_ggtt_addr(hwe->hwsp));
 
 	if (xe_device_has_msix(gt_to_xe(hwe->gt)))
-		ring_mode |= _MASKED_BIT_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
+		ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
 	xe_hw_engine_mmio_write32(hwe, RING_MODE(0), ring_mode);
 	xe_hw_engine_mmio_write32(hwe, RING_MI_MODE(0),
-				  _MASKED_BIT_DISABLE(STOP_RING));
+				  REG_MASKED_FIELD_DISABLE(STOP_RING));
 	xe_hw_engine_mmio_read32(hwe, RING_MI_MODE(0));
 }
 
@@ -408,7 +408,8 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
 		},
 	};
 
-	xe_rtp_process_to_sr(&ctx, lrc_setup, ARRAY_SIZE(lrc_setup), &hwe->reg_lrc);
+	xe_rtp_process_to_sr(&ctx, lrc_setup, ARRAY_SIZE(lrc_setup),
+			     &hwe->reg_lrc, true);
 }
 
 static void
@@ -472,7 +473,8 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe)
 		},
 	};
 
-	xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries), &hwe->reg_sr);
+	xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries),
+			     &hwe->reg_sr, false);
 }
 
 static const struct engine_info *find_engine_info(enum xe_engine_class class, int instance)
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c
index 2ef33dfbe3a2b..4c2b113364d33 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_group.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c
@@ -51,7 +51,8 @@ hw_engine_group_alloc(struct xe_device *xe)
 	if (!group)
 		return ERR_PTR(-ENOMEM);
 
-	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", 0, 0);
+	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU,
+					   0);
 	if (!group->resume_wq)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c
index 8c65291f36fc0..2a31b430570e3 100644
--- a/drivers/gpu/drm/xe/xe_hw_error.c
+++ b/drivers/gpu/drm/xe/xe_hw_error.c
@@ -3,6 +3,7 @@
  * Copyright © 2025 Intel Corporation
  */
 
+#include <linux/bitmap.h>
 #include <linux/fault-inject.h>
 
 #include "regs/xe_gsc_regs.h"
@@ -10,20 +11,23 @@
 #include "regs/xe_irq_regs.h"
 
 #include "xe_device.h"
+#include "xe_drm_ras.h"
 #include "xe_hw_error.h"
 #include "xe_mmio.h"
 #include "xe_survivability_mode.h"
 
-#define  HEC_UNCORR_FW_ERR_BITS 4
+#define GT_HW_ERROR_MAX_ERR_BITS		16
+#define HEC_UNCORR_FW_ERR_BITS			4
+#define XE_RAS_REG_SIZE				32
+#define XE_SOC_NUM_IEH				2
+
+#define PVC_ERROR_MASK_SET(hw_err, err_bit)	((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \
+						 (PVC_COR_ERR_MASK & REG_BIT(err_bit)) : \
+						 (PVC_FAT_ERR_MASK & REG_BIT(err_bit)))
+
 extern struct fault_attr inject_csc_hw_error;
 
-/* Error categories reported by hardware */
-enum hardware_error {
-	HARDWARE_ERROR_CORRECTABLE = 0,
-	HARDWARE_ERROR_NONFATAL = 1,
-	HARDWARE_ERROR_FATAL = 2,
-	HARDWARE_ERROR_MAX,
-};
+static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;
 
 static const char * const hec_uncorrected_fw_errors[] = {
 	"Fatal",
@@ -32,20 +36,130 @@ static const char * const hec_uncorrected_fw_errors[] = {
 	"Data Corruption"
 };
 
-static const char *hw_error_to_str(const enum hardware_error hw_err)
+static const unsigned long xe_hw_error_map[] = {
+	[XE_GT_ERROR]	= DRM_XE_RAS_ERR_COMP_CORE_COMPUTE,
+	[XE_SOC_ERROR]	= DRM_XE_RAS_ERR_COMP_SOC_INTERNAL,
+};
+
+enum gt_vector_regs {
+	ERR_STAT_GT_VECTOR0 = 0,
+	ERR_STAT_GT_VECTOR1,
+	ERR_STAT_GT_VECTOR2,
+	ERR_STAT_GT_VECTOR3,
+	ERR_STAT_GT_VECTOR4,
+	ERR_STAT_GT_VECTOR5,
+	ERR_STAT_GT_VECTOR6,
+	ERR_STAT_GT_VECTOR7,
+	ERR_STAT_GT_VECTOR_MAX
+};
+
+#define PVC_GT_VECTOR_LEN(hw_err)	((hw_err == HARDWARE_ERROR_CORRECTABLE) ? \
+					 ERR_STAT_GT_VECTOR4 : ERR_STAT_GT_VECTOR_MAX)
+
+static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_error hw_err)
 {
-	switch (hw_err) {
-	case HARDWARE_ERROR_CORRECTABLE:
-		return "CORRECTABLE";
-	case HARDWARE_ERROR_NONFATAL:
-		return "NONFATAL";
-	case HARDWARE_ERROR_FATAL:
-		return "FATAL";
-	default:
-		return "UNKNOWN";
-	}
+	if (hw_err == HARDWARE_ERROR_CORRECTABLE)
+		return DRM_XE_RAS_ERR_SEV_CORRECTABLE;
+
+	/* Uncorrectable errors comprise of both fatal and non-fatal errors */
+	return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE;
 }
 
+static const char * const pvc_master_global_err_reg[] = {
+	[0 ... 1]	= "Undefined",
+	[2]		= "HBM SS0: Channel0",
+	[3]		= "HBM SS0: Channel1",
+	[4]		= "HBM SS0: Channel2",
+	[5]		= "HBM SS0: Channel3",
+	[6]		= "HBM SS0: Channel4",
+	[7]		= "HBM SS0: Channel5",
+	[8]		= "HBM SS0: Channel6",
+	[9]		= "HBM SS0: Channel7",
+	[10]		= "HBM SS1: Channel0",
+	[11]		= "HBM SS1: Channel1",
+	[12]		= "HBM SS1: Channel2",
+	[13]		= "HBM SS1: Channel3",
+	[14]		= "HBM SS1: Channel4",
+	[15]		= "HBM SS1: Channel5",
+	[16]		= "HBM SS1: Channel6",
+	[17]		= "HBM SS1: Channel7",
+	[18 ... 31]	= "Undefined",
+};
+static_assert(ARRAY_SIZE(pvc_master_global_err_reg) == XE_RAS_REG_SIZE);
+
+static const char * const pvc_slave_global_err_reg[] = {
+	[0]		= "Undefined",
+	[1]		= "HBM SS2: Channel0",
+	[2]		= "HBM SS2: Channel1",
+	[3]		= "HBM SS2: Channel2",
+	[4]		= "HBM SS2: Channel3",
+	[5]		= "HBM SS2: Channel4",
+	[6]		= "HBM SS2: Channel5",
+	[7]		= "HBM SS2: Channel6",
+	[8]		= "HBM SS2: Channel7",
+	[9]		= "HBM SS3: Channel0",
+	[10]		= "HBM SS3: Channel1",
+	[11]		= "HBM SS3: Channel2",
+	[12]		= "HBM SS3: Channel3",
+	[13]		= "HBM SS3: Channel4",
+	[14]		= "HBM SS3: Channel5",
+	[15]		= "HBM SS3: Channel6",
+	[16]		= "HBM SS3: Channel7",
+	[17]		= "Undefined",
+	[18]		= "ANR MDFI",
+	[19 ... 31]	= "Undefined",
+};
+static_assert(ARRAY_SIZE(pvc_slave_global_err_reg) == XE_RAS_REG_SIZE);
+
+static const char * const pvc_slave_local_fatal_err_reg[] = {
+	[0]		= "Local IEH: Malformed PCIe AER",
+	[1]		= "Local IEH: Malformed PCIe ERR",
+	[2]		= "Local IEH: UR conditions in IEH",
+	[3]		= "Local IEH: From SERR Sources",
+	[4 ... 19]	= "Undefined",
+	[20]		= "Malformed MCA error packet (HBM/Punit)",
+	[21 ... 31]	= "Undefined",
+};
+static_assert(ARRAY_SIZE(pvc_slave_local_fatal_err_reg) == XE_RAS_REG_SIZE);
+
+static const char * const pvc_master_local_fatal_err_reg[] = {
+	[0]		= "Local IEH: Malformed IOSF PCIe AER",
+	[1]		= "Local IEH: Malformed IOSF PCIe ERR",
+	[2]		= "Local IEH: UR RESPONSE",
+	[3]		= "Local IEH: From SERR SPI controller",
+	[4]		= "Base Die MDFI T2T",
+	[5]		= "Undefined",
+	[6]		= "Base Die MDFI T2C",
+	[7]		= "Undefined",
+	[8]		= "Invalid CSC PSF Command Parity",
+	[9]		= "Invalid CSC PSF Unexpected Completion",
+	[10]		= "Invalid CSC PSF Unsupported Request",
+	[11]		= "Invalid PCIe PSF Command Parity",
+	[12]		= "PCIe PSF Unexpected Completion",
+	[13]		= "PCIe PSF Unsupported Request",
+	[14 ... 19]	= "Undefined",
+	[20]		= "Malformed MCA error packet (HBM/Punit)",
+	[21 ... 31]	= "Undefined",
+};
+static_assert(ARRAY_SIZE(pvc_master_local_fatal_err_reg) == XE_RAS_REG_SIZE);
+
+static const char * const pvc_master_local_nonfatal_err_reg[] = {
+	[0 ... 3]	= "Undefined",
+	[4]		= "Base Die MDFI T2T",
+	[5]		= "Undefined",
+	[6]		= "Base Die MDFI T2C",
+	[7]		= "Undefined",
+	[8]		= "Invalid CSC PSF Command Parity",
+	[9]		= "Invalid CSC PSF Unexpected Completion",
+	[10]		= "Invalid PCIe PSF Command Parity",
+	[11 ... 31]	= "Undefined",
+};
+static_assert(ARRAY_SIZE(pvc_master_local_nonfatal_err_reg) == XE_RAS_REG_SIZE);
+
+#define PVC_MASTER_LOCAL_REG_INFO(hw_err)	((hw_err == HARDWARE_ERROR_FATAL) ? \
+						 pvc_master_local_fatal_err_reg : \
+						 pvc_master_local_nonfatal_err_reg)
+
 static bool fault_inject_csc_hw_error(void)
 {
 	return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(&inject_csc_hw_error, 1);
@@ -64,7 +178,8 @@ static void csc_hw_error_work(struct work_struct *work)
 
 static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err)
 {
-	const char *hw_err_str = hw_error_to_str(hw_err);
+	const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
+	const char *severity_str = error_severity[severity];
 	struct xe_device *xe = tile_to_xe(tile);
 	struct xe_mmio *mmio = &tile->mmio;
 	u32 base, err_bit, err_src;
@@ -77,8 +192,8 @@ static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error
 	lockdep_assert_held(&xe->irq.lock);
 	err_src = xe_mmio_read32(mmio, HEC_UNCORR_ERR_STATUS(base));
 	if (!err_src) {
-		drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported HEC_ERR_STATUS_%s blank\n",
-				    tile->id, hw_err_str);
+		drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s HEC_ERR_STATUS register blank\n",
+				    tile->id, severity_str);
 		return;
 	}
 
@@ -86,8 +201,8 @@ static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error
 		fw_err = xe_mmio_read32(mmio, HEC_UNCORR_FW_ERR_DW0(base));
 		for_each_set_bit(err_bit, &fw_err, HEC_UNCORR_FW_ERR_BITS) {
 			drm_err_ratelimited(&xe->drm, HW_ERR
-					    "%s: HEC Uncorrected FW %s error reported, bit[%d] is set\n",
-					     hw_err_str, hec_uncorrected_fw_errors[err_bit],
+					    "HEC FW %s %s reported, bit[%d] is set\n",
+					     hec_uncorrected_fw_errors[err_bit], severity_str,
 					     err_bit);
 
 			schedule_work(&tile->csc_hw_error_work);
@@ -97,29 +212,285 @@ static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error
 	xe_mmio_write32(mmio, HEC_UNCORR_ERR_STATUS(base), err_src);
 }
 
+static void log_hw_error(struct xe_tile *tile, const char *name,
+			 const enum drm_xe_ras_error_severity severity)
+{
+	const char *severity_str = error_severity[severity];
+	struct xe_device *xe = tile_to_xe(tile);
+
+	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
+		drm_warn(&xe->drm, "%s %s detected\n", name, severity_str);
+	else
+		drm_err_ratelimited(&xe->drm, "%s %s detected\n", name, severity_str);
+}
+
+static void log_gt_err(struct xe_tile *tile, const char *name, int i, u32 err,
+		       const enum drm_xe_ras_error_severity severity)
+{
+	const char *severity_str = error_severity[severity];
+	struct xe_device *xe = tile_to_xe(tile);
+
+	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
+		drm_warn(&xe->drm, "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n",
+			 name, severity_str, i, err);
+	else
+		drm_err_ratelimited(&xe->drm, "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n",
+				    name, severity_str, i, err);
+}
+
+static void log_soc_error(struct xe_tile *tile, const char * const *reg_info,
+			  const enum drm_xe_ras_error_severity severity, u32 err_bit, u32 index)
+{
+	const char *severity_str = error_severity[severity];
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+	const char *name;
+
+	name = reg_info[err_bit];
+
+	if (strcmp(name, "Undefined")) {
+		if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
+			drm_warn(&xe->drm, "%s SOC %s detected", name, severity_str);
+		else
+			drm_err_ratelimited(&xe->drm, "%s SOC %s detected", name, severity_str);
+		atomic_inc(&info[index].counter);
+	}
+}
+
+static void gt_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err,
+				u32 error_id)
+{
+	const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+	struct xe_mmio *mmio = &tile->mmio;
+	unsigned long err_stat = 0;
+	int i;
+
+	if (xe->info.platform != XE_PVC)
+		return;
+
+	if (hw_err == HARDWARE_ERROR_NONFATAL) {
+		atomic_inc(&info[error_id].counter);
+		log_hw_error(tile, info[error_id].name, severity);
+		return;
+	}
+
+	for (i = 0; i < PVC_GT_VECTOR_LEN(hw_err); i++) {
+		u32 vector, val;
+
+		vector = xe_mmio_read32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i));
+		if (!vector)
+			continue;
+
+		switch (i) {
+		case ERR_STAT_GT_VECTOR0:
+		case ERR_STAT_GT_VECTOR1: {
+			u32 errbit;
+
+			val = hweight32(vector);
+			atomic_add(val, &info[error_id].counter);
+			log_gt_err(tile, "Subslice", i, vector, severity);
+
+			/*
+			 * Error status register is only populated once per error.
+			 * Read the register and clear once.
+			 */
+			if (err_stat)
+				break;
+
+			err_stat = xe_mmio_read32(mmio, ERR_STAT_GT_REG(hw_err));
+			for_each_set_bit(errbit, &err_stat, GT_HW_ERROR_MAX_ERR_BITS) {
+				if (PVC_ERROR_MASK_SET(hw_err, errbit))
+					atomic_inc(&info[error_id].counter);
+			}
+			if (err_stat)
+				xe_mmio_write32(mmio, ERR_STAT_GT_REG(hw_err), err_stat);
+			break;
+		}
+		case ERR_STAT_GT_VECTOR2:
+		case ERR_STAT_GT_VECTOR3:
+			val = hweight32(vector);
+			atomic_add(val, &info[error_id].counter);
+			log_gt_err(tile, "L3 BANK", i, vector, severity);
+			break;
+		case ERR_STAT_GT_VECTOR6:
+			val = hweight32(vector);
+			atomic_add(val, &info[error_id].counter);
+			log_gt_err(tile, "TLB", i, vector, severity);
+			break;
+		case ERR_STAT_GT_VECTOR7:
+			val = hweight32(vector);
+			atomic_add(val, &info[error_id].counter);
+			log_gt_err(tile, "L3 Fabric", i, vector, severity);
+			break;
+		default:
+			log_gt_err(tile, "Undefined", i, vector, severity);
+		}
+
+		xe_mmio_write32(mmio, ERR_STAT_GT_VECTOR_REG(hw_err, i), vector);
+	}
+}
+
+static void soc_slave_ieh_handler(struct xe_tile *tile, const enum hardware_error hw_err, u32 error_id)
+{
+	const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
+	unsigned long slave_global_errstat, slave_local_errstat;
+	struct xe_mmio *mmio = &tile->mmio;
+	u32 regbit, slave;
+
+	slave = SOC_PVC_SLAVE_BASE;
+	slave_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err));
+
+	if (slave_global_errstat & SOC_IEH1_LOCAL_ERR_STATUS) {
+		slave_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err));
+
+		if (hw_err == HARDWARE_ERROR_FATAL) {
+			for_each_set_bit(regbit, &slave_local_errstat, XE_RAS_REG_SIZE)
+				log_soc_error(tile, pvc_slave_local_fatal_err_reg, severity,
+					      regbit, error_id);
+		}
+
+		xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err),
+				slave_local_errstat);
+	}
+
+	for_each_set_bit(regbit, &slave_global_errstat, XE_RAS_REG_SIZE)
+		log_soc_error(tile, pvc_slave_global_err_reg, severity, regbit, error_id);
+
+	xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), slave_global_errstat);
+}
+
+static void soc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err,
+				 u32 error_id)
+{
+	const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_mmio *mmio = &tile->mmio;
+	unsigned long master_global_errstat, master_local_errstat;
+	u32 master, slave, regbit;
+	int i;
+
+	if (xe->info.platform != XE_PVC)
+		return;
+
+	master = SOC_PVC_MASTER_BASE;
+	slave = SOC_PVC_SLAVE_BASE;
+
+	/* Mask error type in GSYSEVTCTL so that no new errors of the type will be reported */
+	for (i = 0; i < XE_SOC_NUM_IEH; i++)
+		xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i), ~REG_BIT(hw_err));
+
+	if (hw_err == HARDWARE_ERROR_CORRECTABLE) {
+		xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0));
+		xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), REG_GENMASK(31, 0));
+		xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0));
+		xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(slave, hw_err), REG_GENMASK(31, 0));
+		goto unmask_gsysevtctl;
+	}
+
+	/*
+	 * Read the master global IEH error register, if BIT(1) is set then process
+	 * the slave IEH first. If BIT(0) in global error register is set then process
+	 * the corresponding local error registers.
+	 */
+	master_global_errstat = xe_mmio_read32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err));
+	if (master_global_errstat & SOC_SLAVE_IEH)
+		soc_slave_ieh_handler(tile, hw_err, error_id);
+
+	if (master_global_errstat & SOC_IEH0_LOCAL_ERR_STATUS) {
+		master_local_errstat = xe_mmio_read32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err));
+
+		for_each_set_bit(regbit, &master_local_errstat, XE_RAS_REG_SIZE)
+			log_soc_error(tile, PVC_MASTER_LOCAL_REG_INFO(hw_err), severity, regbit, error_id);
+
+		xe_mmio_write32(mmio, SOC_LOCAL_ERR_STAT_REG(master, hw_err), master_local_errstat);
+	}
+
+	for_each_set_bit(regbit, &master_global_errstat, XE_RAS_REG_SIZE)
+		log_soc_error(tile, pvc_master_global_err_reg, severity, regbit, error_id);
+
+	xe_mmio_write32(mmio, SOC_GLOBAL_ERR_STAT_REG(master, hw_err), master_global_errstat);
+
+unmask_gsysevtctl:
+	for (i = 0; i < XE_SOC_NUM_IEH; i++)
+		xe_mmio_write32(mmio, SOC_GSYSEVTCTL_REG(master, slave, i),
+				(HARDWARE_ERROR_MAX << 1) + 1);
+}
+
 static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_error hw_err)
 {
-	const char *hw_err_str = hw_error_to_str(hw_err);
+	const enum drm_xe_ras_error_severity severity = hw_err_to_severity(hw_err);
+	const char *severity_str = error_severity[severity];
 	struct xe_device *xe = tile_to_xe(tile);
-	unsigned long flags;
-	u32 err_src;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+	unsigned long flags, err_src;
+	u32 err_bit;
 
-	if (xe->info.platform != XE_BATTLEMAGE)
+	if (!IS_DGFX(xe))
 		return;
 
 	spin_lock_irqsave(&xe->irq.lock, flags);
 	err_src = xe_mmio_read32(&tile->mmio, DEV_ERR_STAT_REG(hw_err));
 	if (!err_src) {
-		drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported DEV_ERR_STAT_%s blank!\n",
-				    tile->id, hw_err_str);
+		drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported %s DEV_ERR_STAT register blank!\n",
+				    tile->id, severity_str);
 		goto unlock;
 	}
 
-	if (err_src & XE_CSC_ERROR)
+	/*
+	 * On encountering CSC firmware errors, the graphics device becomes unrecoverable
+	 * so return immediately on error. The only way to recover from these errors is
+	 * firmware flash. The device will enter Runtime Survivability mode when such
+	 * errors are detected.
+	 */
+	if (err_src & REG_BIT(XE_CSC_ERROR)) {
 		csc_hw_error_handler(tile, hw_err);
+		goto clear_reg;
+	}
 
-	xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src);
+	if (!info)
+		goto clear_reg;
+
+	for_each_set_bit(err_bit, &err_src, XE_RAS_REG_SIZE) {
+		const char *name;
+		u32 error_id;
+
+		/* Check error bit is within bounds */
+		if (err_bit >= ARRAY_SIZE(xe_hw_error_map))
+			break;
+
+		error_id = xe_hw_error_map[err_bit];
 
+		/* Check error component is within max */
+		if (!error_id || error_id >= DRM_XE_RAS_ERR_COMP_MAX)
+			continue;
+
+		name = info[error_id].name;
+		if (!name)
+			continue;
+
+		if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) {
+			drm_warn(&xe->drm, HW_ERR
+				 "TILE%d reported %s %s, bit[%d] is set\n",
+				 tile->id, name, severity_str, err_bit);
+		} else {
+			drm_err_ratelimited(&xe->drm, HW_ERR
+					    "TILE%d reported %s %s, bit[%d] is set\n",
+					    tile->id, name, severity_str, err_bit);
+		}
+
+		if (err_bit == XE_GT_ERROR)
+			gt_hw_error_handler(tile, hw_err, error_id);
+		if (err_bit == XE_SOC_ERROR)
+			soc_hw_error_handler(tile, hw_err, error_id);
+	}
+
+clear_reg:
+	xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src);
 unlock:
 	spin_unlock_irqrestore(&xe->irq.lock, flags);
 }
@@ -141,9 +512,18 @@ void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl)
 	if (fault_inject_csc_hw_error())
 		schedule_work(&tile->csc_hw_error_work);
 
-	for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++)
+	for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) {
 		if (master_ctl & ERROR_IRQ(hw_err))
 			hw_error_source_handler(tile, hw_err);
+	}
+}
+
+static int hw_error_info_init(struct xe_device *xe)
+{
+	if (xe->info.platform != XE_PVC)
+		return 0;
+
+	return xe_drm_ras_init(xe);
 }
 
 /*
@@ -172,11 +552,16 @@ static void process_hw_errors(struct xe_device *xe)
 void xe_hw_error_init(struct xe_device *xe)
 {
 	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	int ret;
 
 	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
 		return;
 
 	INIT_WORK(&tile->csc_hw_error_work, csc_hw_error_work);
 
+	ret = hw_error_info_init(xe);
+	if (ret)
+		drm_err(&xe->drm, "Failed to initialize XE DRM RAS (%pe)\n", ERR_PTR(ret));
+
 	process_hw_errors(xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c
index ae8ed15b64c55..14720623ad001 100644
--- a/drivers/gpu/drm/xe/xe_hw_fence.c
+++ b/drivers/gpu/drm/xe/xe_hw_fence.c
@@ -124,7 +124,8 @@ static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);
 
 static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
 {
-	return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
+	return container_of(fence->dma.extern_lock, struct xe_hw_fence_irq,
+			    lock);
 }
 
 static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c
index 0fd4d4f1014af..92e423a339f15 100644
--- a/drivers/gpu/drm/xe/xe_hwmon.c
+++ b/drivers/gpu/drm/xe/xe_hwmon.c
@@ -506,7 +506,7 @@ xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy)
 	if (hwmon->xe->info.platform == XE_BATTLEMAGE) {
 		u64 pmt_val;
 
-		ret = xe_pmt_telem_read(to_pci_dev(hwmon->xe->drm.dev),
+		ret = xe_pmt_telem_read(hwmon->xe->drm.dev,
 					xe_mmio_read32(mmio, PUNIT_TELEMETRY_GUID),
 					&pmt_val, BMG_ENERGY_STATUS_PMT_OFFSET,	sizeof(pmt_val));
 		if (ret != sizeof(pmt_val)) {
diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c
index 1e1fb72e49bf2..706783863d07d 100644
--- a/drivers/gpu/drm/xe/xe_i2c.c
+++ b/drivers/gpu/drm/xe/xe_i2c.c
@@ -27,7 +27,7 @@
 #include "regs/xe_i2c_regs.h"
 #include "regs/xe_irq_regs.h"
 
-#include "xe_device_types.h"
+#include "xe_device.h"
 #include "xe_i2c.h"
 #include "xe_mmio.h"
 #include "xe_sriov.h"
@@ -176,11 +176,18 @@ static bool xe_i2c_irq_present(struct xe_device *xe)
  */
 void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl)
 {
-	if (!xe_i2c_irq_present(xe))
+	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
+
+	if (!(master_ctl & I2C_IRQ) || !xe_i2c_irq_present(xe))
 		return;
 
-	if (master_ctl & I2C_IRQ)
-		generic_handle_irq_safe(xe->i2c->adapter_irq);
+	/* Forward interrupt to I2C adapter */
+	generic_handle_irq_safe(xe->i2c->adapter_irq);
+
+	/* Deassert after I2C adapter clears the interrupt */
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE);
+	/* Reassert to allow subsequent interrupt generation */
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0);
 }
 
 void xe_i2c_irq_reset(struct xe_device *xe)
@@ -190,6 +197,7 @@ void xe_i2c_irq_reset(struct xe_device *xe)
 	if (!xe_i2c_irq_present(xe))
 		return;
 
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE);
 	xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, ACPI_INTR_EN, 0);
 }
 
@@ -201,6 +209,7 @@ void xe_i2c_irq_postinstall(struct xe_device *xe)
 		return;
 
 	xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, 0, ACPI_INTR_EN);
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0);
 }
 
 static int xe_i2c_irq_map(struct irq_domain *h, unsigned int virq,
diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c
index 8163c3a8fc87c..0c726eda93906 100644
--- a/drivers/gpu/drm/xe/xe_lmtt.c
+++ b/drivers/gpu/drm/xe/xe_lmtt.c
@@ -57,6 +57,23 @@ static u64 lmtt_page_size(struct xe_lmtt *lmtt)
 	return BIT_ULL(lmtt->ops->lmtt_pte_shift(0));
 }
 
+/**
+ * xe_lmtt_page_size() - Get LMTT page size.
+ * @lmtt: the &xe_lmtt
+ *
+ * This function shall be called only by PF.
+ *
+ * Return: LMTT page size.
+ */
+u64 xe_lmtt_page_size(struct xe_lmtt *lmtt)
+{
+	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
+	lmtt_assert(lmtt, xe_device_has_lmtt(lmtt_to_xe(lmtt)));
+	lmtt_assert(lmtt, lmtt->ops);
+
+	return lmtt_page_size(lmtt);
+}
+
 static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level)
 {
 	unsigned int num_entries = level ? lmtt->ops->lmtt_pte_num(level) : 0;
diff --git a/drivers/gpu/drm/xe/xe_lmtt.h b/drivers/gpu/drm/xe/xe_lmtt.h
index 75a234fbf3670..8fa387b38c520 100644
--- a/drivers/gpu/drm/xe/xe_lmtt.h
+++ b/drivers/gpu/drm/xe/xe_lmtt.h
@@ -20,6 +20,7 @@ int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range);
 int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset);
 void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid);
 u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size);
+u64 xe_lmtt_page_size(struct xe_lmtt *lmtt);
 #else
 static inline int xe_lmtt_init(struct xe_lmtt *lmtt) { return 0; }
 static inline void xe_lmtt_init_hw(struct xe_lmtt *lmtt) { }
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index b0f037bc227ff..c5cfd8f75a943 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -14,6 +14,7 @@
 #include "instructions/xe_gfxpipe_commands.h"
 #include "instructions/xe_gfx_state_commands.h"
 #include "regs/xe_engine_regs.h"
+#include "regs/xe_gt_regs.h"
 #include "regs/xe_lrc_layout.h"
 #include "xe_bb.h"
 #include "xe_bo.h"
@@ -113,13 +114,17 @@ size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class)
 	/* Engine context image */
 	switch (class) {
 	case XE_ENGINE_CLASS_RENDER:
-		if (GRAPHICS_VER(xe) >= 20)
+		if (GRAPHICS_VERx100(xe) >= 3510)
+			size += 7 * SZ_4K;
+		else if (GRAPHICS_VER(xe) >= 20)
 			size += 3 * SZ_4K;
 		else
 			size += 13 * SZ_4K;
 		break;
 	case XE_ENGINE_CLASS_COMPUTE:
-		if (GRAPHICS_VER(xe) >= 20)
+		if (GRAPHICS_VERx100(xe) >= 3510)
+			size += 5 * SZ_4K;
+		else if (GRAPHICS_VER(xe) >= 20)
 			size += 2 * SZ_4K;
 		else
 			size += 13 * SZ_4K;
@@ -637,12 +642,12 @@ static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class)
 
 static void set_context_control(u32 *regs, struct xe_hw_engine *hwe)
 {
-	regs[CTX_CONTEXT_CONTROL] = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
-						       CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
+	regs[CTX_CONTEXT_CONTROL] = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
+							    CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
 	if (xe_gt_has_indirect_ring_state(hwe->gt))
 		regs[CTX_CONTEXT_CONTROL] |=
-			_MASKED_BIT_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE);
+			REG_MASKED_FIELD_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE);
 }
 
 static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe)
@@ -711,12 +716,13 @@ u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc)
 #define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset
 #define __xe_lrc_regs_offset xe_lrc_regs_offset
 
-#define LRC_SEQNO_PPHWSP_OFFSET 512
-#define LRC_START_SEQNO_PPHWSP_OFFSET (LRC_SEQNO_PPHWSP_OFFSET + 8)
-#define LRC_CTX_JOB_TIMESTAMP_OFFSET (LRC_START_SEQNO_PPHWSP_OFFSET + 8)
+#define LRC_CTX_JOB_TIMESTAMP_OFFSET 512
 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024
 #define LRC_PARALLEL_PPHWSP_OFFSET 2048
 
+#define LRC_SEQNO_OFFSET 0
+#define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8)
+
 u32 xe_lrc_regs_offset(struct xe_lrc *lrc)
 {
 	return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE;
@@ -743,14 +749,12 @@ size_t xe_lrc_skip_size(struct xe_device *xe)
 
 static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc)
 {
-	/* The seqno is stored in the driver-defined portion of PPHWSP */
-	return xe_lrc_pphwsp_offset(lrc) + LRC_SEQNO_PPHWSP_OFFSET;
+	return LRC_SEQNO_OFFSET;
 }
 
 static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc)
 {
-	/* The start seqno is stored in the driver-defined portion of PPHWSP */
-	return xe_lrc_pphwsp_offset(lrc) + LRC_START_SEQNO_PPHWSP_OFFSET;
+	return LRC_START_SEQNO_OFFSET;
 }
 
 static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc)
@@ -801,10 +805,11 @@ static inline u32 __xe_lrc_wa_bb_offset(struct xe_lrc *lrc)
 	return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE;
 }
 
-#define DECL_MAP_ADDR_HELPERS(elem) \
+#define DECL_MAP_ADDR_HELPERS(elem, bo_expr) \
 static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \
 { \
-	struct iosys_map map = lrc->bo->vmap; \
+	struct xe_bo *bo = (bo_expr); \
+	struct iosys_map map = bo->vmap; \
 \
 	xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map));  \
 	iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \
@@ -812,20 +817,22 @@ static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \
 } \
 static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \
 { \
-	return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \
+	struct xe_bo *bo = (bo_expr); \
+\
+	return xe_bo_ggtt_addr(bo) + __xe_lrc_##elem##_offset(lrc); \
 } \
 
-DECL_MAP_ADDR_HELPERS(ring)
-DECL_MAP_ADDR_HELPERS(pphwsp)
-DECL_MAP_ADDR_HELPERS(seqno)
-DECL_MAP_ADDR_HELPERS(regs)
-DECL_MAP_ADDR_HELPERS(start_seqno)
-DECL_MAP_ADDR_HELPERS(ctx_job_timestamp)
-DECL_MAP_ADDR_HELPERS(ctx_timestamp)
-DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw)
-DECL_MAP_ADDR_HELPERS(parallel)
-DECL_MAP_ADDR_HELPERS(indirect_ring)
-DECL_MAP_ADDR_HELPERS(engine_id)
+DECL_MAP_ADDR_HELPERS(ring, lrc->bo)
+DECL_MAP_ADDR_HELPERS(pphwsp, lrc->bo)
+DECL_MAP_ADDR_HELPERS(seqno, lrc->seqno_bo)
+DECL_MAP_ADDR_HELPERS(regs, lrc->bo)
+DECL_MAP_ADDR_HELPERS(start_seqno, lrc->seqno_bo)
+DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo)
+DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo)
+DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo)
+DECL_MAP_ADDR_HELPERS(parallel, lrc->bo)
+DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo)
+DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo)
 
 #undef DECL_MAP_ADDR_HELPERS
 
@@ -1032,6 +1039,7 @@ static void xe_lrc_finish(struct xe_lrc *lrc)
 {
 	xe_hw_fence_ctx_finish(&lrc->fence_ctx);
 	xe_bo_unpin_map_no_vm(lrc->bo);
+	xe_bo_unpin_map_no_vm(lrc->seqno_bo);
 }
 
 /*
@@ -1204,7 +1212,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc,
 
 	*cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
 	*cmd++ = CS_DEBUG_MODE2(0).addr;
-	*cmd++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
+	*cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
 
 	return cmd - batch;
 }
@@ -1431,53 +1439,17 @@ void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_pri
 	lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
 }
 
-static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
-		       struct xe_vm *vm, void *replay_state, u32 ring_size,
-		       u16 msix_vec,
-		       u32 init_flags)
+static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm,
+			   void *replay_state, u16 msix_vec, u32 init_flags)
 {
 	struct xe_gt *gt = hwe->gt;
-	const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class);
-	u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE;
 	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_device *xe = gt_to_xe(gt);
 	struct iosys_map map;
 	u32 arb_enable;
-	u32 bo_flags;
+	u32 state_cache_perf_fix[3];
 	int err;
 
-	kref_init(&lrc->refcount);
-	lrc->gt = gt;
-	lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
-	lrc->size = lrc_size;
-	lrc->flags = 0;
-	lrc->ring.size = ring_size;
-	lrc->ring.tail = 0;
-
-	if (gt_engine_needs_indirect_ctx(gt, hwe->class)) {
-		lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX;
-		bo_size += LRC_INDIRECT_CTX_BO_SIZE;
-	}
-
-	if (xe_gt_has_indirect_ring_state(gt))
-		lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE;
-
-	bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT |
-		   XE_BO_FLAG_GGTT_INVALIDATE;
-
-	if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */
-		bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
-
-	lrc->bo = xe_bo_create_pin_map_novm(xe, tile,
-					    bo_size,
-					    ttm_bo_type_kernel,
-					    bo_flags, false);
-	if (IS_ERR(lrc->bo))
-		return PTR_ERR(lrc->bo);
-
-	xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt,
-			     hwe->fence_irq, hwe->name);
-
 	/*
 	 * Init Per-Process of HW status Page, LRC / context state to known
 	 * values. If there's already a primed default_lrc, just copy it, otherwise
@@ -1489,7 +1461,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 		xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE);	/* PPHWSP */
 		xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
 				 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE,
-				 lrc_size - LRC_PPHWSP_SIZE);
+				 lrc->size - LRC_PPHWSP_SIZE);
 		if (replay_state)
 			xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
 					 replay_state, lrc->replay_size);
@@ -1497,21 +1469,16 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 		void *init_data = empty_lrc_data(hwe);
 
 		if (!init_data) {
-			err = -ENOMEM;
-			goto err_lrc_finish;
+			return -ENOMEM;
 		}
 
-		xe_map_memcpy_to(xe, &map, 0, init_data, lrc_size);
+		xe_map_memcpy_to(xe, &map, 0, init_data, lrc->size);
 		kfree(init_data);
 	}
 
-	if (vm) {
+	if (vm)
 		xe_lrc_set_ppgtt(lrc, vm);
 
-		if (vm->xef)
-			xe_drm_client_add_bo(vm->xef->client, lrc->bo);
-	}
-
 	if (xe_device_has_msix(xe)) {
 		xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR,
 				     xe_memirq_status_ptr(&tile->memirq, hwe));
@@ -1527,14 +1494,20 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START,
 					      __xe_lrc_ring_ggtt_addr(lrc));
 		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0);
-		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, 0);
+
+		/* Match head and tail pointers */
+		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, lrc->ring.tail);
 		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail);
+
 		xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL,
 					      RING_CTL_SIZE(lrc->ring.size) | RING_VALID);
 	} else {
 		xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc));
-		xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, 0);
+
+		/* Match head and tail pointers */
+		xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, lrc->ring.tail);
 		xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail);
+
 		xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL,
 				     RING_CTL_SIZE(lrc->ring.size) | RING_VALID);
 	}
@@ -1542,12 +1515,12 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 	if (init_flags & XE_LRC_CREATE_RUNALONE)
 		xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL,
 				     xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) |
-				     _MASKED_BIT_ENABLE(CTX_CTRL_RUN_ALONE));
+				     REG_MASKED_FIELD_ENABLE(CTX_CTRL_RUN_ALONE));
 
 	if (init_flags & XE_LRC_CREATE_PXP)
 		xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL,
 				     xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) |
-				     _MASKED_BIT_ENABLE(CTX_CTRL_PXP_ENABLE));
+				     REG_MASKED_FIELD_ENABLE(CTX_CTRL_PXP_ENABLE));
 
 	lrc->ctx_timestamp = 0;
 	xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0);
@@ -1575,6 +1548,13 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 	arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 	xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable));
 
+	if (init_flags & XE_LRC_DISABLE_STATE_CACHE_PERF_FIX) {
+		state_cache_perf_fix[0] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
+		state_cache_perf_fix[1] = COMMON_SLICE_CHICKEN3.addr;
+		state_cache_perf_fix[2] = REG_MASKED_FIELD_ENABLE(DISABLE_STATE_CACHE_PERF_FIX);
+		xe_lrc_write_ring(lrc, state_cache_perf_fix, sizeof(state_cache_perf_fix));
+	}
+
 	map = __xe_lrc_seqno_map(lrc);
 	xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1);
 
@@ -1583,12 +1563,76 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 
 	err = setup_wa_bb(lrc, hwe);
 	if (err)
-		goto err_lrc_finish;
+		return err;
 
 	err = setup_indirect_ctx(lrc, hwe);
+
+	return err;
+}
+
+static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm,
+		       void *replay_state, u32 ring_size, u16 msix_vec, u32 init_flags)
+{
+	struct xe_gt *gt = hwe->gt;
+	const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class);
+	u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE;
+	struct xe_tile *tile = gt_to_tile(gt);
+	struct xe_device *xe = gt_to_xe(gt);
+	struct xe_bo *bo;
+	u32 bo_flags;
+	int err;
+
+	kref_init(&lrc->refcount);
+	lrc->gt = gt;
+	lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
+	lrc->size = lrc_size;
+	lrc->flags = 0;
+	lrc->ring.size = ring_size;
+	lrc->ring.tail = 0;
+
+	if (gt_engine_needs_indirect_ctx(gt, hwe->class)) {
+		lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX;
+		bo_size += LRC_INDIRECT_CTX_BO_SIZE;
+	}
+
+	if (xe_gt_has_indirect_ring_state(gt))
+		lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE;
+
+	bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT |
+		   XE_BO_FLAG_GGTT_INVALIDATE;
+
+	if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */
+		bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
+
+	bo = xe_bo_create_pin_map_novm(xe, tile, bo_size,
+				       ttm_bo_type_kernel,
+				       bo_flags, false);
+	if (IS_ERR(lrc->bo))
+		return PTR_ERR(lrc->bo);
+
+	lrc->bo = bo;
+
+	bo = xe_bo_create_pin_map_novm(xe, tile, PAGE_SIZE,
+				       ttm_bo_type_kernel,
+				       XE_BO_FLAG_GGTT |
+				       XE_BO_FLAG_GGTT_INVALIDATE |
+				       XE_BO_FLAG_SYSTEM, false);
+	if (IS_ERR(bo)) {
+		err = PTR_ERR(bo);
+		goto err_lrc_finish;
+	}
+	lrc->seqno_bo = bo;
+
+	xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt,
+			     hwe->fence_irq, hwe->name);
+
+	err = xe_lrc_ctx_init(lrc, hwe, vm, replay_state, msix_vec, init_flags);
 	if (err)
 		goto err_lrc_finish;
 
+	if (vm && vm->xef)
+		xe_drm_client_add_bo(vm->xef->client, lrc->bo);
+
 	return 0;
 
 err_lrc_finish:
@@ -1858,6 +1902,7 @@ static int instr_dw(u32 cmd_header)
 
 static int dump_mi_command(struct drm_printer *p,
 			   struct xe_gt *gt,
+			   u32 *start,
 			   u32 *dw,
 			   int remaining_dw)
 {
@@ -1873,15 +1918,18 @@ static int dump_mi_command(struct drm_printer *p,
 		while (num_noop < remaining_dw &&
 		       (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP)
 			num_noop++;
-		drm_printf(p, "[%#010x] MI_NOOP (%d dwords)\n", inst_header, num_noop);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_NOOP (%d dwords)\n",
+			   dw - num_noop - start, inst_header, num_noop);
 		return num_noop;
 
 	case MI_TOPOLOGY_FILTER:
-		drm_printf(p, "[%#010x] MI_TOPOLOGY_FILTER\n", inst_header);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_TOPOLOGY_FILTER\n",
+			   dw - start, inst_header);
 		return 1;
 
 	case MI_BATCH_BUFFER_END:
-		drm_printf(p, "[%#010x] MI_BATCH_BUFFER_END\n", inst_header);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_BATCH_BUFFER_END\n",
+			   dw - start, inst_header);
 		/* Return 'remaining_dw' to consume the rest of the LRC */
 		return remaining_dw;
 	}
@@ -1895,39 +1943,43 @@ static int dump_mi_command(struct drm_printer *p,
 
 	switch (inst_header & MI_OPCODE) {
 	case MI_LOAD_REGISTER_IMM:
-		drm_printf(p, "[%#010x] MI_LOAD_REGISTER_IMM: %d regs\n",
-			   inst_header, (numdw - 1) / 2);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n",
+			   dw - start, inst_header, (numdw - 1) / 2);
 		for (int i = 1; i < numdw; i += 2)
-			drm_printf(p, " - %#6x = %#010x\n", dw[i], dw[i + 1]);
+			drm_printf(p, "LRC[%#5lx]  =  - %#6x = %#010x\n",
+				   &dw[i] - start, dw[i], dw[i + 1]);
 		return numdw;
 
 	case MI_LOAD_REGISTER_MEM & MI_OPCODE:
-		drm_printf(p, "[%#010x] MI_LOAD_REGISTER_MEM: %s%s\n",
-			   inst_header,
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n",
+			   dw - start, inst_header,
 			   dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "",
 			   dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : "");
 		if (numdw == 4)
-			drm_printf(p, " - %#6x = %#010llx\n",
+			drm_printf(p, "LRC[%#5lx]  =  - %#6x = %#010llx\n",
+				   dw - start,
 				   dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2])));
 		else
-			drm_printf(p, " - %*ph (%s)\n",
-				   (int)sizeof(u32) * (numdw - 1), dw + 1,
-				   numdw < 4 ? "truncated" : "malformed");
+			drm_printf(p, "LRC[%#5lx]  =  - %*ph (%s)\n",
+				   dw - start, (int)sizeof(u32) * (numdw - 1),
+				   dw + 1, numdw < 4 ? "truncated" : "malformed");
 		return numdw;
 
 	case MI_FORCE_WAKEUP:
-		drm_printf(p, "[%#010x] MI_FORCE_WAKEUP\n", inst_header);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] MI_FORCE_WAKEUP\n",
+			   dw - start, inst_header);
 		return numdw;
 
 	default:
-		drm_printf(p, "[%#010x] unknown MI opcode %#x, likely %d dwords\n",
-			   inst_header, opcode, numdw);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] unknown MI opcode %#x, likely %d dwords\n",
+			   dw - start, inst_header, opcode, numdw);
 		return numdw;
 	}
 }
 
 static int dump_gfxpipe_command(struct drm_printer *p,
 				struct xe_gt *gt,
+				u32 *start,
 				u32 *dw,
 				int remaining_dw)
 {
@@ -1946,11 +1998,13 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	switch (*dw & GFXPIPE_MATCH_MASK) {
 #define MATCH(cmd) \
 	case cmd: \
-		drm_printf(p, "[%#010x] " #cmd " (%d dwords)\n", *dw, numdw); \
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] " #cmd " (%d dwords)\n", \
+			   dw - start, *dw, numdw); \
 		return numdw
 #define MATCH3D(cmd) \
 	case CMD_##cmd: \
-		drm_printf(p, "[%#010x] " #cmd " (%d dwords)\n", *dw, numdw); \
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] " #cmd " (%d dwords)\n", \
+			   dw - start, *dw, numdw); \
 		return numdw
 
 	MATCH(STATE_BASE_ADDRESS);
@@ -1966,6 +2020,7 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	MATCH(PIPELINE_SELECT);
 
 	MATCH3D(3DSTATE_DRAWING_RECTANGLE_FAST);
+	MATCH3D(3DSTATE_CUSTOM_SAMPLE_PATTERN);
 	MATCH3D(3DSTATE_CLEAR_PARAMS);
 	MATCH3D(3DSTATE_DEPTH_BUFFER);
 	MATCH3D(3DSTATE_STENCIL_BUFFER);
@@ -2049,8 +2104,16 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	MATCH3D(3DSTATE_SBE_MESH);
 	MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER);
 	MATCH3D(3DSTATE_COARSE_PIXEL);
+	MATCH3D(3DSTATE_MESH_SHADER_DATA_EXT);
+	MATCH3D(3DSTATE_TASK_SHADER_DATA_EXT);
+	MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC_2);
+	MATCH3D(3DSTATE_CC_STATE_POINTERS_2);
+	MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS_2);
+	MATCH3D(3DSTATE_BLEND_STATE_POINTERS_2);
+	MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2);
 
 	MATCH3D(3DSTATE_DRAWING_RECTANGLE);
+	MATCH3D(3DSTATE_URB_MEMORY);
 	MATCH3D(3DSTATE_CHROMA_KEY);
 	MATCH3D(3DSTATE_POLY_STIPPLE_OFFSET);
 	MATCH3D(3DSTATE_POLY_STIPPLE_PATTERN);
@@ -2070,16 +2133,18 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	MATCH3D(3DSTATE_SUBSLICE_HASH_TABLE);
 	MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTERS);
 	MATCH3D(3DSTATE_PTBR_TILE_PASS_INFO);
+	MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2);
 
 	default:
-		drm_printf(p, "[%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n",
-			   *dw, pipeline, opcode, subopcode, numdw);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n",
+			   dw - start, *dw, pipeline, opcode, subopcode, numdw);
 		return numdw;
 	}
 }
 
 static int dump_gfx_state_command(struct drm_printer *p,
 				  struct xe_gt *gt,
+				  u32 *start,
 				  u32 *dw,
 				  int remaining_dw)
 {
@@ -2097,8 +2162,8 @@ static int dump_gfx_state_command(struct drm_printer *p,
 	MATCH(STATE_WRITE_INLINE);
 
 	default:
-		drm_printf(p, "[%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
-			   *dw, opcode, numdw);
+		drm_printf(p, "LRC[%#5lx]  =  [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
+			   dw - start, *dw, opcode, numdw);
 		return numdw;
 	}
 }
@@ -2107,7 +2172,7 @@ void xe_lrc_dump_default(struct drm_printer *p,
 			 struct xe_gt *gt,
 			 enum xe_engine_class hwe_class)
 {
-	u32 *dw;
+	u32 *dw, *start;
 	int remaining_dw, num_dw;
 
 	if (!gt->default_lrc[hwe_class]) {
@@ -2120,18 +2185,20 @@ void xe_lrc_dump_default(struct drm_printer *p,
 	 * hardware status page.
 	 */
 	dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE;
+	start = dw;
 	remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4;
 
 	while (remaining_dw > 0) {
 		if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) {
-			num_dw = dump_mi_command(p, gt, dw, remaining_dw);
+			num_dw = dump_mi_command(p, gt, start, dw, remaining_dw);
 		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) {
-			num_dw = dump_gfxpipe_command(p, gt, dw, remaining_dw);
+			num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw);
 		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) {
-			num_dw = dump_gfx_state_command(p, gt, dw, remaining_dw);
+			num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw);
 		} else {
 			num_dw = min(instr_dw(*dw), remaining_dw);
-			drm_printf(p, "[%#10x] Unknown instruction of type %#x, likely %d dwords\n",
+			drm_printf(p, "LRC[%#5lx]  =  [%#10x] Unknown instruction of type %#x, likely %d dwords\n",
+				   dw - start,
 				   *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw),
 				   num_dw);
 		}
@@ -2141,6 +2208,102 @@ void xe_lrc_dump_default(struct drm_printer *p,
 	}
 }
 
+/*
+ * Lookup the value of a register within the offset/value pairs of an
+ * MI_LOAD_REGISTER_IMM instruction.
+ *
+ * Return -ENOENT if the register is not present in the MI_LRI instruction.
+ */
+static int lookup_reg_in_mi_lri(u32 offset, u32 *value,
+				const u32 *dword_pair, int num_regs)
+{
+	for (int i = 0; i < num_regs; i++) {
+		if (dword_pair[2 * i] == offset) {
+			*value = dword_pair[2 * i + 1];
+			return 0;
+		}
+	}
+
+	return -ENOENT;
+}
+
+/*
+ * Lookup the value of a register in a specific engine type's default LRC.
+ *
+ * Return -EINVAL if the default LRC doesn't exist, or ENOENT if the register
+ * cannot be found in the default LRC.
+ */
+int xe_lrc_lookup_default_reg_value(struct xe_gt *gt,
+				    enum xe_engine_class hwe_class,
+				    u32 offset,
+				    u32 *value)
+{
+	u32 *dw;
+	int remaining_dw, ret;
+
+	if (!gt->default_lrc[hwe_class])
+		return -EINVAL;
+
+	/*
+	 * Skip the beginning of the LRC since it contains the per-process
+	 * hardware status page.
+	 */
+	dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE;
+	remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4;
+
+	while (remaining_dw > 0) {
+		u32 num_dw = instr_dw(*dw);
+
+		if (num_dw > remaining_dw)
+			num_dw = remaining_dw;
+
+		switch (*dw & XE_INSTR_CMD_TYPE) {
+		case XE_INSTR_MI:
+			switch (*dw & MI_OPCODE) {
+			case MI_BATCH_BUFFER_END:
+				/* End of LRC; register not found */
+				return -ENOENT;
+
+			case MI_NOOP:
+			case MI_TOPOLOGY_FILTER:
+				/*
+				 * MI_NOOP and MI_TOPOLOGY_FILTER don't have
+				 * a length field and are always 1-dword
+				 * instructions.
+				 */
+				remaining_dw--;
+				dw++;
+				break;
+
+			case MI_LOAD_REGISTER_IMM:
+				ret = lookup_reg_in_mi_lri(offset, value,
+							   dw + 1, (num_dw - 1) / 2);
+				if (ret == 0)
+					return 0;
+
+				fallthrough;
+
+			default:
+				/*
+				 * Jump to next instruction based on length
+				 * field.
+				 */
+				remaining_dw -= num_dw;
+				dw += num_dw;
+				break;
+			}
+			break;
+
+		default:
+			/* Jump to next instruction based on length field. */
+			remaining_dw -= num_dw;
+			dw += num_dw;
+		}
+	}
+
+	return -ENOENT;
+}
+
 struct instr_state {
 	u32 instr;
 	u16 num_dw;
@@ -2413,14 +2576,14 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts)
  * @lrc: Pointer to the lrc.
  *
  * Return latest ctx timestamp. With support for active contexts, the
- * calculation may bb slightly racy, so follow a read-again logic to ensure that
+ * calculation may be slightly racy, so follow a read-again logic to ensure that
  * the context is still active before returning the right timestamp.
  *
  * Returns: New ctx timestamp value
  */
 u64 xe_lrc_timestamp(struct xe_lrc *lrc)
 {
-	u64 lrc_ts, reg_ts, new_ts;
+	u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp;
 	u32 engine_id;
 
 	lrc_ts = xe_lrc_ctx_timestamp(lrc);
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index c1c615447c857..e7c975f9e2d97 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -49,6 +49,7 @@ struct xe_lrc_snapshot {
 #define XE_LRC_CREATE_RUNALONE		BIT(0)
 #define XE_LRC_CREATE_PXP		BIT(1)
 #define XE_LRC_CREATE_USER_CTX		BIT(2)
+#define XE_LRC_DISABLE_STATE_CACHE_PERF_FIX	BIT(3)
 
 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
 			     void *replay_state, u32 ring_size, u16 msix_vec, u32 flags);
@@ -134,6 +135,10 @@ size_t xe_lrc_skip_size(struct xe_device *xe);
 void xe_lrc_dump_default(struct drm_printer *p,
 			 struct xe_gt *gt,
 			 enum xe_engine_class);
+int xe_lrc_lookup_default_reg_value(struct xe_gt *gt,
+				    enum xe_engine_class hwe_class,
+				    u32 offset,
+				    u32 *value);
 
 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs);
 
diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h
index a4373d280c397..5a718f759ed69 100644
--- a/drivers/gpu/drm/xe/xe_lrc_types.h
+++ b/drivers/gpu/drm/xe/xe_lrc_types.h
@@ -22,6 +22,12 @@ struct xe_lrc {
 	 */
 	struct xe_bo *bo;
 
+	/**
+	 * @seqno_bo: Buffer object (memory) for seqno numbers. Always in system
+	 * memory as this a CPU read, GPU write path object.
+	 */
+	struct xe_bo *seqno_bo;
+
 	/** @size: size of the lrc and optional indirect ring state */
 	u32 size;
 
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index e58b9b4336543..fc918b4fba54b 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -25,6 +25,7 @@
 #include "xe_exec_queue.h"
 #include "xe_ggtt.h"
 #include "xe_gt.h"
+#include "xe_gt_printk.h"
 #include "xe_hw_engine.h"
 #include "xe_lrc.h"
 #include "xe_map.h"
@@ -183,19 +184,11 @@ static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm,
 	xe_assert(xe, pos == vram_limit);
 }
 
-static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
-				 struct xe_vm *vm, struct drm_exec *exec)
+static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m,
+				  struct xe_vm *vm, struct drm_exec *exec)
 {
-	struct xe_device *xe = tile_to_xe(tile);
-	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
-	u8 id = tile->id;
-	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
-#define VRAM_IDENTITY_MAP_COUNT	2
-	u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
-#undef VRAM_IDENTITY_MAP_COUNT
-	u32 map_ofs, level, i;
 	struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo;
-	u64 entry, pt29_ofs;
+	u32 num_entries = NUM_PT_SLOTS;
 
 	/* Can't bump NUM_PT_SLOTS too high */
 	BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
@@ -215,6 +208,24 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
+	m->pt_bo = bo;
+	return 0;
+}
+
+static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
+				  struct xe_vm *vm, u32 *ofs)
+{
+	struct xe_device *xe = tile_to_xe(tile);
+	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
+	u8 id = tile->id;
+	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
+#define VRAM_IDENTITY_MAP_COUNT	2
+	u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
+#undef VRAM_IDENTITY_MAP_COUNT
+	u32 map_ofs, level, i;
+	struct xe_bo *bo = m->pt_bo, *batch = tile->mem.kernel_bb_pool->bo;
+	u64 entry, pt29_ofs;
+
 	/* PT30 & PT31 reserved for 2M identity map */
 	pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE;
 	entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs);
@@ -337,6 +348,12 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 		}
 	}
 
+	if (ofs)
+		*ofs = map_ofs;
+}
+
+static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
+{
 	/*
 	 * Example layout created above, with root level = 3:
 	 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
@@ -362,9 +379,6 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 	drm_suballoc_manager_init(&m->vm_update_sa,
 				  (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
 				  NUM_VMUSA_UNIT_PER_PAGE, 0);
-
-	m->pt_bo = bo;
-	return 0;
 }
 
 /*
@@ -415,12 +429,22 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m
 	struct xe_device *xe = tile_to_xe(tile);
 	struct xe_validation_ctx ctx;
 	struct drm_exec exec;
+	u32 map_ofs;
 	int err = 0;
 
 	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) {
 		err = xe_vm_drm_exec_lock(vm, &exec);
+		if (err)
+			return err;
+
 		drm_exec_retry_on_contention(&exec);
-		err = xe_migrate_prepare_vm(tile, m, vm, &exec);
+
+		err = xe_migrate_pt_bo_alloc(tile, m, vm, &exec);
+		if (err)
+			return err;
+
+		xe_migrate_prepare_vm(tile, m, vm, &map_ofs);
+		xe_migrate_suballoc_manager_init(m, map_ofs);
 		drm_exec_retry_on_contention(&exec);
 		xe_validation_retry_on_oom(&ctx, &err);
 	}
@@ -836,31 +860,13 @@ static u32 xe_migrate_ccs_copy(struct xe_migrate *m,
 	return flush_flags;
 }
 
-/**
- * xe_migrate_copy() - Copy content of TTM resources.
- * @m: The migration context.
- * @src_bo: The buffer object @src is currently bound to.
- * @dst_bo: If copying between resources created for the same bo, set this to
- * the same value as @src_bo. If copying between buffer objects, set it to
- * the buffer object @dst is currently bound to.
- * @src: The source TTM resource.
- * @dst: The dst TTM resource.
- * @copy_only_ccs: If true copy only CCS metadata
- *
- * Copies the contents of @src to @dst: On flat CCS devices,
- * the CCS metadata is copied as well if needed, or if not present,
- * the CCS metadata of @dst is cleared for security reasons.
- *
- * Return: Pointer to a dma_fence representing the last copy batch, or
- * an error pointer on failure. If there is a failure, any copy operation
- * started by the function call has been synced.
- */
-struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
-				  struct xe_bo *src_bo,
-				  struct xe_bo *dst_bo,
-				  struct ttm_resource *src,
-				  struct ttm_resource *dst,
-				  bool copy_only_ccs)
+static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
+					   struct xe_bo *src_bo,
+					   struct xe_bo *dst_bo,
+					   struct ttm_resource *src,
+					   struct ttm_resource *dst,
+					   bool copy_only_ccs,
+					   bool is_vram_resolve)
 {
 	struct xe_gt *gt = m->tile->primary_gt;
 	struct xe_device *xe = gt_to_xe(gt);
@@ -881,8 +887,15 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
 	bool copy_ccs = xe_device_has_flat_ccs(xe) &&
 		xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
 	bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);
-	bool use_comp_pat = type_device && xe_device_has_flat_ccs(xe) &&
-		GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram;
+
+	/*
+	 * For decompression operation, always use the compression PAT index.
+	 * Otherwise, only use the compression PAT index for device memory
+	 * when copying from VRAM to system memory.
+	 */
+	bool use_comp_pat = is_vram_resolve || (type_device &&
+			    xe_device_has_flat_ccs(xe) &&
+			    GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram);
 
 	/* Copying CCS between two different BOs is not supported yet. */
 	if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
@@ -1041,6 +1054,53 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
 	return fence;
 }
 
+/**
+ * xe_migrate_copy() - Copy content of TTM resources.
+ * @m: The migration context.
+ * @src_bo: The buffer object @src is currently bound to.
+ * @dst_bo: If copying between resources created for the same bo, set this to
+ * the same value as @src_bo. If copying between buffer objects, set it to
+ * the buffer object @dst is currently bound to.
+ * @src: The source TTM resource.
+ * @dst: The dst TTM resource.
+ * @copy_only_ccs: If true copy only CCS metadata
+ *
+ * Copies the contents of @src to @dst: On flat CCS devices,
+ * the CCS metadata is copied as well if needed, or if not present,
+ * the CCS metadata of @dst is cleared for security reasons.
+ *
+ * Return: Pointer to a dma_fence representing the last copy batch, or
+ * an error pointer on failure. If there is a failure, any copy operation
+ * started by the function call has been synced.
+ */
+struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
+				  struct xe_bo *src_bo,
+				  struct xe_bo *dst_bo,
+				  struct ttm_resource *src,
+				  struct ttm_resource *dst,
+				  bool copy_only_ccs)
+{
+	return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, copy_only_ccs, false);
+}
+
+/**
+ * xe_migrate_resolve() - Resolve and decompress a buffer object if required.
+ * @m: The migrate context
+ * @bo: The buffer object to resolve
+ * @res: The reservation object
+ *
+ * Wrapper around __xe_migrate_copy() with is_vram_resolve set to true
+ * to trigger decompression if needed.
+ *
+ * Return: A dma_fence that signals on completion, or an ERR_PTR on failure.
+ */
+struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
+				     struct xe_bo *bo,
+				     struct ttm_resource *res)
+{
+	return __xe_migrate_copy(m, bo, bo, res, res, false, true);
+}
+
 /**
  * xe_migrate_lrc() - Get the LRC from migrate context.
  * @migrate: Migrate context.
@@ -1148,65 +1208,73 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 		size -= src_L0;
 	}
 
+	bb = xe_bb_alloc(gt);
+	if (IS_ERR(bb))
+		return PTR_ERR(bb);
+
 	bb_pool = ctx->mem.ccs_bb_pool;
-	guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
-	xe_sa_bo_swap_shadow(bb_pool);
+	scoped_guard(mutex, xe_sa_bo_swap_guard(bb_pool)) {
+		xe_sa_bo_swap_shadow(bb_pool);
 
-	bb = xe_bb_ccs_new(gt, batch_size, read_write);
-	if (IS_ERR(bb)) {
-		drm_err(&xe->drm, "BB allocation failed.\n");
-		err = PTR_ERR(bb);
-		return err;
-	}
+		err = xe_bb_init(bb, bb_pool, batch_size);
+		if (err) {
+			xe_gt_err(gt, "BB allocation failed.\n");
+			xe_bb_free(bb, NULL);
+			return err;
+		}
 
-	batch_size_allocated = batch_size;
-	size = xe_bo_size(src_bo);
-	batch_size = 0;
+		batch_size_allocated = batch_size;
+		size = xe_bo_size(src_bo);
+		batch_size = 0;
 
-	/*
-	 * Emit PTE and copy commands here.
-	 * The CCS copy command can only support limited size. If the size to be
-	 * copied is more than the limit, divide copy into chunks. So, calculate
-	 * sizes here again before copy command is emitted.
-	 */
-	while (size) {
-		batch_size += 10; /* Flush + ggtt addr + 2 NOP */
-		u32 flush_flags = 0;
-		u64 ccs_ofs, ccs_size;
-		u32 ccs_pt;
+		/*
+		 * Emit PTE and copy commands here.
+		 * The CCS copy command can only support limited size. If the size to be
+		 * copied is more than the limit, divide copy into chunks. So, calculate
+		 * sizes here again before copy command is emitted.
+		 */
 
-		u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
+		while (size) {
+			batch_size += 10; /* Flush + ggtt addr + 2 NOP */
+			u32 flush_flags = 0;
+			u64 ccs_ofs, ccs_size;
+			u32 ccs_pt;
 
-		src_L0 = xe_migrate_res_sizes(m, &src_it);
+			u32 avail_pts = max_mem_transfer_per_pass(xe) /
+					LEVEL0_PAGE_TABLE_ENCODE_SIZE;
 
-		batch_size += pte_update_size(m, false, src, &src_it, &src_L0,
-					      &src_L0_ofs, &src_L0_pt, 0, 0,
-					      avail_pts);
+			src_L0 = xe_migrate_res_sizes(m, &src_it);
 
-		ccs_size = xe_device_ccs_bytes(xe, src_L0);
-		batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs,
-					      &ccs_pt, 0, avail_pts, avail_pts);
-		xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
-		batch_size += EMIT_COPY_CCS_DW;
+			batch_size += pte_update_size(m, false, src, &src_it, &src_L0,
+						      &src_L0_ofs, &src_L0_pt, 0, 0,
+						      avail_pts);
 
-		emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src);
+			ccs_size = xe_device_ccs_bytes(xe, src_L0);
+			batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs,
+						      &ccs_pt, 0, avail_pts, avail_pts);
+			xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
+			batch_size += EMIT_COPY_CCS_DW;
 
-		emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
+			emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src);
 
-		bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
-		flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt,
-						  src_L0_ofs, dst_is_pltt,
-						  src_L0, ccs_ofs, true);
-		bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
+			emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
 
-		size -= src_L0;
-	}
+			bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
+			flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt,
+							  src_L0_ofs, dst_is_pltt,
+							  src_L0, ccs_ofs, true);
+			bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
 
-	xe_assert(xe, (batch_size_allocated == bb->len));
-	src_bo->bb_ccs[read_write] = bb;
+			size -= src_L0;
+		}
+
+		xe_assert(xe, (batch_size_allocated == bb->len));
+		src_bo->bb_ccs[read_write] = bb;
+
+		xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
+		xe_sa_bo_sync_shadow(bb->bo);
+	}
 
-	xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
-	xe_sa_bo_sync_shadow(bb->bo);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 1522afb37dcfa..169279d9d8c24 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -132,6 +132,10 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
 				  struct ttm_resource *dst,
 				  bool copy_only_ccs);
 
+struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
+				     struct xe_bo *bo,
+				     struct ttm_resource *res);
+
 int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 			   struct xe_bo *src_bo,
 			   enum xe_sriov_vf_ccs_rw_ctxs read_write);
diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
index a1a05c68dc7d6..78adb303b6630 100644
--- a/drivers/gpu/drm/xe/xe_mmio.c
+++ b/drivers/gpu/drm/xe/xe_mmio.c
@@ -154,6 +154,15 @@ u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg)
 	return val;
 }
 
+void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val)
+{
+	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
+
+	trace_xe_reg_rw(mmio, true, addr, val, sizeof(val));
+
+	writeb(val, mmio->regs + addr);
+}
+
 u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg)
 {
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h
index 15362789ab994..befe021f2215a 100644
--- a/drivers/gpu/drm/xe/xe_mmio.h
+++ b/drivers/gpu/drm/xe/xe_mmio.h
@@ -6,7 +6,7 @@
 #ifndef _XE_MMIO_H_
 #define _XE_MMIO_H_
 
-#include "xe_gt_types.h"
+#include "xe_mmio_types.h"
 
 struct xe_device;
 struct xe_reg;
@@ -17,6 +17,7 @@ int xe_mmio_probe_tiles(struct xe_device *xe);
 void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr, u32 size);
 
 u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg);
+void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val);
 u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg);
 void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val);
 u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg);
@@ -37,11 +38,6 @@ static inline u32 xe_mmio_adjusted_addr(const struct xe_mmio *mmio, u32 addr)
 	return addr;
 }
 
-static inline struct xe_mmio *xe_root_tile_mmio(struct xe_device *xe)
-{
-	return &xe->tiles[0].mmio;
-}
-
 #ifdef CONFIG_PCI_IOV
 void xe_mmio_init_vf_view(struct xe_mmio *mmio, const struct xe_mmio *base, unsigned int vfid);
 #endif
diff --git a/drivers/gpu/drm/xe/xe_mmio_types.h b/drivers/gpu/drm/xe/xe_mmio_types.h
new file mode 100644
index 0000000000000..99e8f269eaf2e
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mmio_types.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022-2026 Intel Corporation
+ */
+
+#ifndef _XE_MMIO_TYPES_H_
+#define _XE_MMIO_TYPES_H_
+
+#include <linux/types.h>
+
+struct xe_gt;
+struct xe_tile;
+
+/**
+ * struct xe_mmio - register mmio structure
+ *
+ * Represents an MMIO region that the CPU may use to access registers.  A
+ * region may share its IO map with other regions (e.g., all GTs within a
+ * tile share the same map with their parent tile, but represent different
+ * subregions of the overall IO space).
+ */
+struct xe_mmio {
+	/** @tile: Backpointer to tile, used for tracing */
+	struct xe_tile *tile;
+
+	/** @regs: Map used to access registers. */
+	void __iomem *regs;
+
+	/**
+	 * @sriov_vf_gt: Backpointer to GT.
+	 *
+	 * This pointer is only set for GT MMIO regions and only when running
+	 * as an SRIOV VF structure
+	 */
+	struct xe_gt *sriov_vf_gt;
+
+	/**
+	 * @regs_size: Length of the register region within the map.
+	 *
+	 * The size of the iomap set in *regs is generally larger than the
+	 * register mmio space since it includes unused regions and/or
+	 * non-register regions such as the GGTT PTEs.
+	 */
+	size_t regs_size;
+
+	/** @adj_limit: adjust MMIO address if address is below this value */
+	u32 adj_limit;
+
+	/** @adj_offset: offset to add to MMIO address when adjusting */
+	u32 adj_offset;
+};
+
+/**
+ * struct xe_mmio_range - register range structure
+ *
+ * @start: first register offset in the range.
+ * @end: last register offset in the range.
+ */
+struct xe_mmio_range {
+	u32 start;
+	u32 end;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c
index 54822497c21eb..1d19df860bea1 100644
--- a/drivers/gpu/drm/xe/xe_mocs.c
+++ b/drivers/gpu/drm/xe/xe_mocs.c
@@ -600,6 +600,7 @@ static unsigned int get_mocs_settings(struct xe_device *xe,
 		info->wb_index = 4;
 		info->unused_entries_index = 4;
 		break;
+	case XE_NOVALAKE_P:
 	case XE_NOVALAKE_S:
 	case XE_PANTHERLAKE:
 	case XE_LUNARLAKE:
diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c
index a0048f64ed12d..4cb5781829122 100644
--- a/drivers/gpu/drm/xe/xe_module.c
+++ b/drivers/gpu/drm/xe/xe_module.c
@@ -10,6 +10,7 @@
 
 #include <drm/drm_module.h>
 
+#include "xe_defaults.h"
 #include "xe_device_types.h"
 #include "xe_drv.h"
 #include "xe_configfs.h"
@@ -19,51 +20,38 @@
 #include "xe_observation.h"
 #include "xe_sched_job.h"
 
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-#define DEFAULT_GUC_LOG_LEVEL		3
-#else
-#define DEFAULT_GUC_LOG_LEVEL		1
-#endif
-
-#define DEFAULT_PROBE_DISPLAY		true
-#define DEFAULT_VRAM_BAR_SIZE		0
-#define DEFAULT_FORCE_PROBE		CONFIG_DRM_XE_FORCE_PROBE
-#define DEFAULT_MAX_VFS			~0
-#define DEFAULT_MAX_VFS_STR		"unlimited"
-#define DEFAULT_WEDGED_MODE		XE_WEDGED_MODE_DEFAULT
-#define DEFAULT_WEDGED_MODE_STR		XE_WEDGED_MODE_DEFAULT_STR
-#define DEFAULT_SVM_NOTIFIER_SIZE	512
-
 struct xe_modparam xe_modparam = {
-	.probe_display =	DEFAULT_PROBE_DISPLAY,
-	.guc_log_level =	DEFAULT_GUC_LOG_LEVEL,
-	.force_probe =		DEFAULT_FORCE_PROBE,
+	.probe_display =	XE_DEFAULT_PROBE_DISPLAY,
+	.guc_log_level =	XE_DEFAULT_GUC_LOG_LEVEL,
+	.force_probe =		XE_DEFAULT_FORCE_PROBE,
 #ifdef CONFIG_PCI_IOV
-	.max_vfs =		DEFAULT_MAX_VFS,
+	.max_vfs =		XE_DEFAULT_MAX_VFS,
 #endif
-	.wedged_mode =		DEFAULT_WEDGED_MODE,
-	.svm_notifier_size =	DEFAULT_SVM_NOTIFIER_SIZE,
+	.wedged_mode =		XE_DEFAULT_WEDGED_MODE,
+	.svm_notifier_size =	XE_DEFAULT_SVM_NOTIFIER_SIZE,
 	/* the rest are 0 by default */
 };
 
 module_param_named(svm_notifier_size, xe_modparam.svm_notifier_size, uint, 0600);
 MODULE_PARM_DESC(svm_notifier_size, "Set the svm notifier size in MiB, must be power of 2 "
-		 "[default=" __stringify(DEFAULT_SVM_NOTIFIER_SIZE) "]");
+		 "[default=" __stringify(XE_DEFAULT_SVM_NOTIFIER_SIZE) "]");
 
 module_param_named_unsafe(force_execlist, xe_modparam.force_execlist, bool, 0444);
 MODULE_PARM_DESC(force_execlist, "Force Execlist submission");
 
+#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 module_param_named(probe_display, xe_modparam.probe_display, bool, 0444);
 MODULE_PARM_DESC(probe_display, "Probe display HW, otherwise it's left untouched "
-		 "[default=" __stringify(DEFAULT_PROBE_DISPLAY) "])");
+		 "[default=" __stringify(XE_DEFAULT_PROBE_DISPLAY) "])");
+#endif
 
 module_param_named(vram_bar_size, xe_modparam.force_vram_bar_size, int, 0600);
 MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size in MiB (<0=disable-resize, 0=max-needed-size, >0=force-size "
-		 "[default=" __stringify(DEFAULT_VRAM_BAR_SIZE) "])");
+		 "[default=" __stringify(XE_DEFAULT_VRAM_BAR_SIZE) "])");
 
 module_param_named(guc_log_level, xe_modparam.guc_log_level, int, 0600);
 MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1=normal, 2..5=verbose-levels "
-		 "[default=" __stringify(DEFAULT_GUC_LOG_LEVEL) "])");
+		 "[default=" __stringify(XE_DEFAULT_GUC_LOG_LEVEL) "])");
 
 module_param_named_unsafe(guc_firmware_path, xe_modparam.guc_firmware_path, charp, 0400);
 MODULE_PARM_DESC(guc_firmware_path,
@@ -80,20 +68,20 @@ MODULE_PARM_DESC(gsc_firmware_path,
 module_param_named_unsafe(force_probe, xe_modparam.force_probe, charp, 0400);
 MODULE_PARM_DESC(force_probe,
 		 "Force probe options for specified devices. See CONFIG_DRM_XE_FORCE_PROBE for details "
-		 "[default=" DEFAULT_FORCE_PROBE "])");
+		 "[default=" XE_DEFAULT_FORCE_PROBE "])");
 
 #ifdef CONFIG_PCI_IOV
 module_param_named(max_vfs, xe_modparam.max_vfs, uint, 0400);
 MODULE_PARM_DESC(max_vfs,
 		 "Limit number of Virtual Functions (VFs) that could be managed. "
 		 "(0=no VFs; N=allow up to N VFs "
-		 "[default=" DEFAULT_MAX_VFS_STR "])");
+		 "[default=" XE_DEFAULT_MAX_VFS_STR "])");
 #endif
 
 module_param_named_unsafe(wedged_mode, xe_modparam.wedged_mode, uint, 0600);
 MODULE_PARM_DESC(wedged_mode,
 		 "Module's default policy for the wedged mode (0=never, 1=upon-critical-error, 2=upon-any-hang-no-reset "
-		 "[default=" DEFAULT_WEDGED_MODE_STR "])");
+		 "[default=" XE_DEFAULT_WEDGED_MODE_STR "])");
 
 static int xe_check_nomodeset(void)
 {
diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c
index 9c4ccd3b39d4d..33487e91f366c 100644
--- a/drivers/gpu/drm/xe/xe_nvm.c
+++ b/drivers/gpu/drm/xe/xe_nvm.c
@@ -6,7 +6,7 @@
 #include <linux/intel_dg_nvm_aux.h>
 #include <linux/pci.h>
 
-#include "xe_device_types.h"
+#include "xe_device.h"
 #include "xe_mmio.h"
 #include "xe_nvm.h"
 #include "xe_pcode_api.h"
@@ -133,12 +133,10 @@ int xe_nvm_init(struct xe_device *xe)
 	if (WARN_ON(xe->nvm))
 		return -EFAULT;
 
-	xe->nvm = kzalloc_obj(*nvm);
-	if (!xe->nvm)
+	nvm = kzalloc_obj(*nvm);
+	if (!nvm)
 		return -ENOMEM;
 
-	nvm = xe->nvm;
-
 	nvm->writable_override = xe_nvm_writable_override(xe);
 	nvm->non_posted_erase = xe_nvm_non_posted_erase(xe);
 	nvm->bar.parent = &pdev->resource[0];
@@ -165,7 +163,6 @@ int xe_nvm_init(struct xe_device *xe)
 	if (ret) {
 		drm_err(&xe->drm, "xe-nvm aux init failed %d\n", ret);
 		kfree(nvm);
-		xe->nvm = NULL;
 		return ret;
 	}
 
@@ -173,8 +170,9 @@ int xe_nvm_init(struct xe_device *xe)
 	if (ret) {
 		drm_err(&xe->drm, "xe-nvm aux add failed %d\n", ret);
 		auxiliary_device_uninit(aux_dev);
-		xe->nvm = NULL;
 		return ret;
 	}
+
+	xe->nvm = nvm;
 	return devm_add_action_or_reset(xe->drm.dev, xe_nvm_fini, xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index 4dd3f29933cf1..6337e671c97ae 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -29,7 +29,7 @@
 #include "xe_gt.h"
 #include "xe_gt_mcr.h"
 #include "xe_gt_printk.h"
-#include "xe_guc_pc.h"
+#include "xe_guc_rc.h"
 #include "xe_macros.h"
 #include "xe_mmio.h"
 #include "xe_oa.h"
@@ -543,8 +543,7 @@ static ssize_t xe_oa_read(struct file *file, char __user *buf,
 	size_t offset = 0;
 	int ret;
 
-	/* Can't read from disabled streams */
-	if (!stream->enabled || !stream->sample)
+	if (!stream->sample)
 		return -EINVAL;
 
 	if (!(file->f_flags & O_NONBLOCK)) {
@@ -758,8 +757,9 @@ static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)
 		},
 		{
 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
-			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
-				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0)
+			enable ?
+			REG_MASKED_FIELD_ENABLE(CTX_CTRL_OAC_CONTEXT_ENABLE) :
+			REG_MASKED_FIELD_DISABLE(CTX_CTRL_OAC_CONTEXT_ENABLE)
 		},
 	};
 
@@ -782,9 +782,9 @@ static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)
 		},
 		{
 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
-			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
-				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) |
-			_MASKED_FIELD(CTX_CTRL_RUN_ALONE, enable ? CTX_CTRL_RUN_ALONE : 0),
+			enable ?
+			REG_MASKED_FIELD_ENABLE(CTX_CTRL_OAC_CONTEXT_ENABLE | CTX_CTRL_RUN_ALONE) :
+			REG_MASKED_FIELD_DISABLE(CTX_CTRL_OAC_CONTEXT_ENABLE | CTX_CTRL_RUN_ALONE),
 		},
 	};
 
@@ -812,9 +812,10 @@ static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)
 
 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)
 {
-	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG,
-			     enable && stream && stream->sample ?
-			     0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG);
+	if (enable && stream && stream->sample)
+		return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_DISABLE_MMIO_TRG);
+	else
+		return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_DISABLE_MMIO_TRG);
 }
 
 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
@@ -825,9 +826,9 @@ static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
 	/* Enable thread stall DOP gating and EU DOP gating. */
 	if (XE_GT_WA(stream->gt, 1508761755)) {
 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
-					  _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
+					  REG_MASKED_FIELD_DISABLE(STALL_DOP_GATING_DISABLE));
 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
-					  _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
+					  REG_MASKED_FIELD_DISABLE(DISABLE_DOP_GATING));
 	}
 
 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
@@ -873,10 +874,6 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
 	xe_force_wake_put(gt_to_fw(gt), stream->fw_ref);
 	xe_pm_runtime_put(stream->oa->xe);
 
-	/* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */
-	if (stream->override_gucrc)
-		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
-
 	xe_oa_free_configs(stream);
 	xe_file_put(stream->xef);
 }
@@ -969,7 +966,7 @@ static void xe_oa_config_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 	struct xe_oa_fence *ofence = container_of(cb, typeof(*ofence), cb);
 
 	INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn);
-	queue_delayed_work(system_unbound_wq, &ofence->work,
+	queue_delayed_work(system_dfl_wq, &ofence->work,
 			   usecs_to_jiffies(NOA_PROGRAM_ADDITIONAL_DELAY_US));
 	dma_fence_put(fence);
 }
@@ -1059,16 +1056,18 @@ static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config
 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)
 {
 	/* If user didn't require OA reports, ask HW not to emit ctx switch reports */
-	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
-			     stream->sample ?
-			     0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
+	if (stream->sample)
+		return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
+	else
+		return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
 }
 
 static u32 oag_buf_size_select(const struct xe_oa_stream *stream)
 {
-	return _MASKED_FIELD(OAG_OA_DEBUG_BUF_SIZE_SELECT,
-			     xe_bo_size(stream->oa_buffer.bo) > SZ_16M ?
-			     OAG_OA_DEBUG_BUF_SIZE_SELECT : 0);
+	if (xe_bo_size(stream->oa_buffer.bo) > SZ_16M)
+		return REG_MASKED_FIELD_ENABLE(OAG_OA_DEBUG_BUF_SIZE_SELECT);
+	else
+		return REG_MASKED_FIELD_DISABLE(OAG_OA_DEBUG_BUF_SIZE_SELECT);
 }
 
 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
@@ -1083,9 +1082,9 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
 	 */
 	if (XE_GT_WA(stream->gt, 1508761755)) {
 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
-					  _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
+					  REG_MASKED_FIELD_ENABLE(STALL_DOP_GATING_DISABLE));
 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
-					  _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
+					  REG_MASKED_FIELD_ENABLE(DISABLE_DOP_GATING));
 	}
 
 	/* Disable clk ratio reports */
@@ -1100,7 +1099,7 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
 			OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL;
 
 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
-			_MASKED_BIT_ENABLE(oa_debug) |
+			REG_MASKED_FIELD_ENABLE(oa_debug) |
 			oag_report_ctx_switches(stream) |
 			oag_buf_size_select(stream) |
 			oag_configure_mmio_trigger(stream, true));
@@ -1460,6 +1459,10 @@ static void xe_oa_stream_disable(struct xe_oa_stream *stream)
 
 	if (stream->sample)
 		hrtimer_cancel(&stream->poll_check_timer);
+
+	/* Update stream->oa_buffer.tail to allow any final reports to be read */
+	if (xe_oa_buffer_check_unlocked(stream))
+		wake_up(&stream->poll_wq);
 }
 
 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)
@@ -1760,19 +1763,6 @@ static int xe_oa_stream_init(struct xe_oa_stream *stream,
 		goto exit;
 	}
 
-	/*
-	 * GuC reset of engines causes OA to lose configuration
-	 * state. Prevent this by overriding GUCRC mode.
-	 */
-	if (XE_GT_WA(stream->gt, 1509372804)) {
-		ret = xe_guc_pc_override_gucrc_mode(&gt->uc.guc.pc,
-						    SLPC_GUCRC_MODE_GUCRC_NO_RC6);
-		if (ret)
-			goto err_free_configs;
-
-		stream->override_gucrc = true;
-	}
-
 	/* Take runtime pm ref and forcewake to disable RC6 */
 	xe_pm_runtime_get(stream->oa->xe);
 	stream->fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
@@ -1823,9 +1813,6 @@ static int xe_oa_stream_init(struct xe_oa_stream *stream,
 err_fw_put:
 	xe_force_wake_put(gt_to_fw(gt), stream->fw_ref);
 	xe_pm_runtime_put(stream->oa->xe);
-	if (stream->override_gucrc)
-		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
-err_free_configs:
 	xe_oa_free_configs(stream);
 exit:
 	xe_file_put(stream->xef);
diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h
index 08cc8d7c22158..b03ffd5134834 100644
--- a/drivers/gpu/drm/xe/xe_oa_types.h
+++ b/drivers/gpu/drm/xe/xe_oa_types.h
@@ -239,9 +239,6 @@ struct xe_oa_stream {
 	/** @poll_period_ns: hrtimer period for checking OA buffer for available data */
 	u64 poll_period_ns;
 
-	/** @override_gucrc: GuC RC has been overridden for the OA stream */
-	bool override_gucrc;
-
 	/** @oa_status: temporary storage for oa_status register value */
 	u32 oa_status;
 
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c
index e13c71a89da2c..da1ed99cd3f8d 100644
--- a/drivers/gpu/drm/xe/xe_page_reclaim.c
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.c
@@ -11,6 +11,7 @@
 #include "xe_page_reclaim.h"
 
 #include "xe_gt_stats.h"
+#include "xe_guc_tlb_inval.h"
 #include "xe_macros.h"
 #include "xe_pat.h"
 #include "xe_sa.h"
@@ -26,12 +27,18 @@
  * flushes.
  * - pat_index is transient display (1)
  *
+ * For cases of NULL VMA, there should be no corresponding PRL entry
+ * so skip over.
+ *
  * Return: true when page reclamation is unnecessary, false otherwise.
  */
 bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma)
 {
 	u8 l3_policy;
 
+	if (xe_vma_is_null(vma))
+		return true;
+
 	l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
 
 	/*
@@ -130,3 +137,22 @@ int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl)
 
 	return page ? 0 : -ENOMEM;
 }
+
+/**
+ * xe_guc_page_reclaim_done_handler() - Page reclaim done handler
+ * @guc: guc
+ * @msg: message indicating page reclamation done
+ * @len: length of message
+ *
+ * Page reclamation is an extension of TLB invalidation. Both
+ * operations share the same seqno and fence. When either
+ * action completes, we need to signal the corresponding
+ * fence. Since the handling logic is currently identical, this
+ * function delegates to the TLB invalidation handler.
+ *
+ * Return: 0 on success, -EPROTO for malformed messages.
+ */
+int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
+{
+	return xe_guc_tlb_inval_done_handler(guc, msg, len);
+}
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h
index 3dd103e37bebb..0412611f3af77 100644
--- a/drivers/gpu/drm/xe/xe_page_reclaim.h
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.h
@@ -20,6 +20,7 @@ struct xe_tlb_inval;
 struct xe_tlb_inval_fence;
 struct xe_tile;
 struct xe_gt;
+struct xe_guc;
 struct xe_vma;
 
 struct xe_guc_page_reclaim_entry {
@@ -122,4 +123,6 @@ static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry
 		put_page(virt_to_page(entries));
 }
 
+int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
+
 #endif	/* _XE_PAGE_RECLAIM_H_ */
diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c
index 6bee53d6ffc37..ea4857acf28d8 100644
--- a/drivers/gpu/drm/xe/xe_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_pagefault.c
@@ -136,7 +136,7 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
 static bool
 xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type)
 {
-	return access_type == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
+	return (access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK) == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
 }
 
 static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)
@@ -164,7 +164,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
 	bool atomic;
 
 	/* Producer flagged this fault to be nacked */
-	if (pf->consumer.fault_level == XE_PAGEFAULT_LEVEL_NACK)
+	if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK)
 		return -EFAULT;
 
 	vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid);
@@ -225,17 +225,20 @@ static void xe_pagefault_print(struct xe_pagefault *pf)
 {
 	xe_gt_info(pf->gt, "\n\tASID: %d\n"
 		   "\tFaulted Address: 0x%08x%08x\n"
-		   "\tFaultType: %d\n"
-		   "\tAccessType: %d\n"
-		   "\tFaultLevel: %d\n"
+		   "\tFaultType: %lu\n"
+		   "\tAccessType: %lu\n"
+		   "\tFaultLevel: %lu\n"
 		   "\tEngineClass: %d %s\n"
 		   "\tEngineInstance: %d\n",
 		   pf->consumer.asid,
 		   upper_32_bits(pf->consumer.page_addr),
 		   lower_32_bits(pf->consumer.page_addr),
-		   pf->consumer.fault_type,
-		   pf->consumer.access_type,
-		   pf->consumer.fault_level,
+		   FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
+			     pf->consumer.fault_type_level),
+		   FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK,
+			     pf->consumer.access_type),
+		   FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
+			     pf->consumer.fault_type_level),
 		   pf->consumer.engine_class,
 		   xe_hw_engine_class_to_str(pf->consumer.engine_class),
 		   pf->consumer.engine_instance);
@@ -259,9 +262,15 @@ static void xe_pagefault_queue_work(struct work_struct *w)
 
 		err = xe_pagefault_service(&pf);
 		if (err) {
-			xe_pagefault_print(&pf);
-			xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",
-				   ERR_PTR(err));
+			if (!(pf.consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {
+				xe_pagefault_print(&pf);
+				xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",
+					   ERR_PTR(err));
+			} else {
+				xe_gt_stats_incr(pf.gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
+				xe_gt_dbg(pf.gt, "Prefetch Fault response: Unsuccessful %pe\n",
+					  ERR_PTR(err));
+			}
 		}
 
 		pf.producer.ops->ack_fault(&pf, err);
diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h
index d3b516407d600..b3289219b1be5 100644
--- a/drivers/gpu/drm/xe/xe_pagefault_types.h
+++ b/drivers/gpu/drm/xe/xe_pagefault_types.h
@@ -68,24 +68,26 @@ struct xe_pagefault {
 		/** @consumer.asid: address space ID */
 		u32 asid;
 		/**
-		 * @consumer.access_type: access type, u8 rather than enum to
-		 * keep size compact
+		 * @consumer.access_type: access type and prefetch flag packed
+		 * into a u8.
 		 */
 		u8 access_type;
+#define XE_PAGEFAULT_ACCESS_TYPE_MASK	GENMASK(1, 0)
+#define XE_PAGEFAULT_ACCESS_PREFETCH	BIT(7)
 		/**
-		 * @consumer.fault_type: fault type, u8 rather than enum to
-		 * keep size compact
+		 * @consumer.fault_type_level: fault type and level, u8 rather
+		 * than enum to keep size compact
 		 */
-		u8 fault_type;
-#define XE_PAGEFAULT_LEVEL_NACK		0xff	/* Producer indicates nack fault */
-		/** @consumer.fault_level: fault level */
-		u8 fault_level;
+		u8 fault_type_level;
+#define XE_PAGEFAULT_TYPE_LEVEL_NACK		0xff	/* Producer indicates nack fault */
+#define XE_PAGEFAULT_LEVEL_MASK			GENMASK(3, 0)
+#define XE_PAGEFAULT_TYPE_MASK			GENMASK(7, 4)
 		/** @consumer.engine_class: engine class */
 		u8 engine_class;
 		/** @consumer.engine_instance: engine instance */
 		u8 engine_instance;
 		/** consumer.reserved: reserved bits for future expansion */
-		u8 reserved[7];
+		u64 reserved;
 	} consumer;
 	/**
 	 * @producer: State for the producer (i.e., HW/FW interface). Populated
diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c
index 14d0dce5190a0..34c9031e1e74b 100644
--- a/drivers/gpu/drm/xe/xe_pat.c
+++ b/drivers/gpu/drm/xe/xe_pat.c
@@ -88,6 +88,7 @@ struct xe_pat_ops {
 	void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 			      int n_entries);
 	int (*dump)(struct xe_gt *gt, struct drm_printer *p);
+	void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd);
 };
 
 static const struct xe_pat_table_entry xelp_pat_table[] = {
@@ -123,7 +124,8 @@ static const struct xe_pat_table_entry xelpg_pat_table[] = {
  *   - no_promote:  0=promotable, 1=no promote
  *   - comp_en:     0=disable, 1=enable
  *   - l3clos:      L3 class of service (0-3)
- *   - l3_policy:   0=WB, 1=XD ("WB - Transient Display"), 3=UC
+ *   - l3_policy:   0=WB, 1=XD ("WB - Transient Display"),
+ *                  2=XA ("WB - Transient App" for Xe3p), 3=UC
  *   - l4_policy:   0=WB, 1=WT, 3=UC
  *   - coh_mode:    0=no snoop, 2=1-way coherent, 3=2-way coherent
  *
@@ -252,6 +254,44 @@ static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = {
 	[31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ),
 };
 
+static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3);
+static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2);
+
+static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = {
+	[ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ),
+	[ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ),
+	[ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ),
+	[ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ),
+	[ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ),
+	[ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ),
+	[ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ),
+	[ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ),
+	[ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ),
+	[ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ),
+	[10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ),
+	[11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ),
+	[12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ),
+	[13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ),
+	[14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ),
+	[15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ),
+	[16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ),
+	/* 17 is reserved; leave set to all 0's */
+	[18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ),
+	[19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ),
+	[20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ),
+	[21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ),
+	[22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ),
+	[23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ),
+	[24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ),
+	[25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ),
+	[26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ),
+	[27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ),
+	[28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ),
+	[29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ),
+	[30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ),
+	[31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ),
+};
+
 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
 {
 	WARN_ON(pat_index >= xe->pat.n_entries);
@@ -271,10 +311,25 @@ u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index)
 	return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value);
 }
 
+static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+
+	if (xe_gt_is_main_type(gt))
+		return xe->pat.pat_primary_pta;
+
+	if (xe_gt_is_media_type(gt))
+		return xe->pat.pat_media_pta;
+
+	xe_assert(xe, false);
+	return NULL;
+}
+
 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 			int n_entries)
 {
 	struct xe_device *xe = gt_to_xe(gt);
+	const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt);
 
 	for (int i = 0; i < n_entries; i++) {
 		struct xe_reg reg = XE_REG(_PAT_INDEX(i));
@@ -284,14 +339,16 @@ static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[
 
 	if (xe->pat.pat_ats)
 		xe_mmio_write32(&gt->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value);
-	if (xe->pat.pat_pta)
-		xe_mmio_write32(&gt->mmio, XE_REG(_PAT_PTA), xe->pat.pat_pta->value);
+
+	if (pta_entry)
+		xe_mmio_write32(&gt->mmio, XE_REG(_PAT_PTA), pta_entry->value);
 }
 
 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 			    int n_entries)
 {
 	struct xe_device *xe = gt_to_xe(gt);
+	const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt);
 
 	for (int i = 0; i < n_entries; i++) {
 		struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i));
@@ -301,8 +358,9 @@ static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry ta
 
 	if (xe->pat.pat_ats)
 		xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value);
-	if (xe->pat.pat_pta)
-		xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe->pat.pat_pta->value);
+
+	if (pta_entry)
+		xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value);
 }
 
 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p)
@@ -458,7 +516,7 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
 
 		xe_pat_index_label(label, sizeof(label), i);
-		xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+		xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid);
 	}
 
 	/*
@@ -471,7 +529,7 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
 
 	drm_printf(p, "Page Table Access:\n");
-	xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
+	xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false);
 
 	return 0;
 }
@@ -480,44 +538,14 @@ static const struct xe_pat_ops xe2_pat_ops = {
 	.program_graphics = program_pat_mcr,
 	.program_media = program_pat,
 	.dump = xe2_dump,
+	.entry_dump = xe2_pat_entry_dump,
 };
 
-static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p)
-{
-	struct xe_device *xe = gt_to_xe(gt);
-	u32 pat;
-	int i;
-	char label[PAT_LABEL_LEN];
-
-	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref.domains)
-		return -ETIMEDOUT;
-
-	drm_printf(p, "PAT table: (* = reserved entry)\n");
-
-	for (i = 0; i < xe->pat.n_entries; i++) {
-		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-
-		xe_pat_index_label(label, sizeof(label), i);
-		xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
-	}
-
-	/*
-	 * Also print PTA_MODE, which describes how the hardware accesses
-	 * PPGTT entries.
-	 */
-	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
-
-	drm_printf(p, "Page Table Access:\n");
-	xe3p_xpc_pat_entry_dump(p, "PTA_MODE", pat, false);
-
-	return 0;
-}
-
 static const struct xe_pat_ops xe3p_xpc_pat_ops = {
 	.program_graphics = program_pat_mcr,
 	.program_media = program_pat,
-	.dump = xe3p_xpc_dump,
+	.dump = xe2_dump,
+	.entry_dump = xe3p_xpc_pat_entry_dump,
 };
 
 void xe_pat_init_early(struct xe_device *xe)
@@ -527,11 +555,26 @@ void xe_pat_init_early(struct xe_device *xe)
 		xe->pat.ops = &xe3p_xpc_pat_ops;
 		xe->pat.table = xe3p_xpc_pat_table;
 		xe->pat.pat_ats = &xe3p_xpc_pat_ats;
-		xe->pat.pat_pta = &xe3p_xpc_pat_pta;
+		xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta;
+		xe->pat.pat_media_pta = &xe3p_xpc_pat_pta;
 		xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table);
 		xe->pat.idx[XE_CACHE_NONE] = 3;
 		xe->pat.idx[XE_CACHE_WT] = 3;	/* N/A (no display); use UC */
 		xe->pat.idx[XE_CACHE_WB] = 2;
+	} else if (GRAPHICS_VER(xe) == 35) {
+		xe->pat.ops = &xe2_pat_ops;
+		xe->pat.table = xe3p_lpg_pat_table;
+		xe->pat.pat_ats = &xe2_pat_ats;
+		if (!IS_DGFX(xe)) {
+			xe->pat.pat_primary_pta = &xe3p_primary_pat_pta;
+			xe->pat.pat_media_pta = &xe3p_media_pat_pta;
+		}
+		xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table);
+		xe->pat.idx[XE_CACHE_NONE] = 3;
+		xe->pat.idx[XE_CACHE_WT] = 15;
+		xe->pat.idx[XE_CACHE_WB] = 2;
+		xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12;
+		xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16;
 	} else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) {
 		xe->pat.ops = &xe2_pat_ops;
 		if (GRAPHICS_VER(xe) == 30) {
@@ -541,8 +584,10 @@ void xe_pat_init_early(struct xe_device *xe)
 			xe->pat.table = xe2_pat_table;
 		}
 		xe->pat.pat_ats = &xe2_pat_ats;
-		if (IS_DGFX(xe))
-			xe->pat.pat_pta = &xe2_pat_pta;
+		if (IS_DGFX(xe)) {
+			xe->pat.pat_primary_pta = &xe2_pat_pta;
+			xe->pat.pat_media_pta = &xe2_pat_pta;
+		}
 
 		/* Wa_16023588340. XXX: Should use XE_WA */
 		if (GRAPHICS_VERx100(xe) == 2001)
@@ -600,20 +645,17 @@ void xe_pat_init_early(struct xe_device *xe)
 			GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);
 	}
 
-	/* VFs can't program nor dump PAT settings */
-	if (IS_SRIOV_VF(xe))
-		xe->pat.ops = NULL;
-
-	xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump);
-	xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics);
-	xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);
+	xe_assert(xe, xe->pat.ops->dump);
+	xe_assert(xe, xe->pat.ops->program_graphics);
+	xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);
+	xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump);
 }
 
 void xe_pat_init(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
-	if (!xe->pat.ops)
+	if (IS_SRIOV_VF(xe))
 		return;
 
 	if (xe_gt_is_media_type(gt))
@@ -633,7 +675,7 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
-	if (!xe->pat.ops)
+	if (IS_SRIOV_VF(xe))
 		return -EOPNOTSUPP;
 
 	return xe->pat.ops->dump(gt, p);
@@ -649,6 +691,7 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
 int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
+	const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt);
 	char label[PAT_LABEL_LEN];
 
 	if (!xe->pat.table || !xe->pat.n_entries)
@@ -658,12 +701,9 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p)
 	for (u32 i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe->pat.table[i].value;
 
-		if (GRAPHICS_VERx100(xe) == 3511) {
-			xe_pat_index_label(label, sizeof(label), i);
-			xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
-		} else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) {
+		if (GRAPHICS_VER(xe) >= 20) {
 			xe_pat_index_label(label, sizeof(label), i);
-			xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+			xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid);
 		} else if (xe->info.platform == XE_METEORLAKE) {
 			xelpg_pat_entry_dump(p, i, pat);
 		} else if (xe->info.platform == XE_PVC) {
@@ -675,18 +715,18 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p)
 		}
 	}
 
-	if (xe->pat.pat_pta) {
-		u32 pat = xe->pat.pat_pta->value;
+	if (pta_entry) {
+		u32 pat = pta_entry->value;
 
 		drm_printf(p, "Page Table Access:\n");
-		xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
+		xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false);
 	}
 
 	if (xe->pat.pat_ats) {
 		u32 pat = xe->pat.pat_ats->value;
 
 		drm_printf(p, "PCIe ATS/PASID:\n");
-		xe2_pat_entry_dump(p, "PAT_ATS ", pat, false);
+		xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false);
 	}
 
 	drm_printf(p, "Cache Level:\n");
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 5c0b3224f20d9..01673d2b24649 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -52,6 +52,7 @@ __diag_ignore_all("-Woverride-init", "Allow field overrides in table");
 
 static const struct xe_graphics_desc graphics_xelp = {
 	.hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0),
+	.num_geometry_xecore_fuse_regs = 1,
 };
 
 #define XE_HP_FEATURES \
@@ -62,6 +63,8 @@ static const struct xe_graphics_desc graphics_xehpg = {
 		BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) |
 		BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) |
 		BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3),
+	.num_geometry_xecore_fuse_regs = 1,
+	.num_compute_xecore_fuse_regs = 1,
 
 	XE_HP_FEATURES,
 };
@@ -78,20 +81,25 @@ static const struct xe_graphics_desc graphics_xehpc = {
 
 	XE_HP_FEATURES,
 
+	.has_access_counter = 1,
 	.has_asid = 1,
 	.has_atomic_enable_pte_bit = 1,
 	.has_usm = 1,
+	.num_compute_xecore_fuse_regs = 2,
 };
 
 static const struct xe_graphics_desc graphics_xelpg = {
 	.hw_engine_mask =
 		BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) |
 		BIT(XE_HW_ENGINE_CCS0),
+	.num_geometry_xecore_fuse_regs = 1,
+	.num_compute_xecore_fuse_regs = 1,
 
 	XE_HP_FEATURES,
 };
 
 #define XE2_GFX_FEATURES \
+	.has_access_counter = 1, \
 	.has_asid = 1, \
 	.has_atomic_enable_pte_bit = 1, \
 	.has_range_tlb_inval = 1, \
@@ -104,14 +112,28 @@ static const struct xe_graphics_desc graphics_xelpg = {
 
 static const struct xe_graphics_desc graphics_xe2 = {
 	XE2_GFX_FEATURES,
+	.num_geometry_xecore_fuse_regs = 3,
+	.num_compute_xecore_fuse_regs = 3,
+};
+
+static const struct xe_graphics_desc graphics_xe3p_lpg = {
+	XE2_GFX_FEATURES,
+	.multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE),
+	.num_geometry_xecore_fuse_regs = 3,
+	.num_compute_xecore_fuse_regs = 3,
 };
 
 static const struct xe_graphics_desc graphics_xe3p_xpc = {
 	XE2_GFX_FEATURES,
+	.has_access_counter = 0,
 	.has_indirect_ring_state = 1,
 	.hw_engine_mask =
 		GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) |
 		GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0),
+	.multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) |
+					 BIT(XE_ENGINE_CLASS_COMPUTE),
+	.num_geometry_xecore_fuse_regs = 4,
+	.num_compute_xecore_fuse_regs = 4,
 };
 
 static const struct xe_media_desc media_xem = {
@@ -146,6 +168,7 @@ static const struct xe_ip graphics_ips[] = {
 	{ 3003, "Xe3_LPG", &graphics_xe2 },
 	{ 3004, "Xe3_LPG", &graphics_xe2 },
 	{ 3005, "Xe3_LPG", &graphics_xe2 },
+	{ 3510, "Xe3p_LPG", &graphics_xe3p_lpg },
 	{ 3511, "Xe3p_XPC", &graphics_xe3p_xpc },
 };
 
@@ -164,6 +187,10 @@ static const struct xe_ip media_ips[] = {
 	{ 3503, "Xe3p_HPM", &media_xelpmp },
 };
 
+#define MULTI_LRC_MASK \
+	.multi_lrc_mask = BIT(XE_ENGINE_CLASS_VIDEO_DECODE) | \
+			  BIT(XE_ENGINE_CLASS_VIDEO_ENHANCE)
+
 static const struct xe_device_desc tgl_desc = {
 	.pre_gmdid_graphics_ip = &graphics_ip_xelp,
 	.pre_gmdid_media_ip = &media_ip_xem,
@@ -174,6 +201,7 @@ static const struct xe_device_desc tgl_desc = {
 	.has_llc = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.vm_max_level = 3,
@@ -188,6 +216,7 @@ static const struct xe_device_desc rkl_desc = {
 	.has_display = true,
 	.has_llc = true,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.vm_max_level = 3,
@@ -205,6 +234,7 @@ static const struct xe_device_desc adl_s_desc = {
 	.has_llc = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.subplatforms = (const struct xe_subplatform_desc[]) {
 		{ XE_SUBPLATFORM_ALDERLAKE_S_RPLS, "RPLS", adls_rpls_ids },
@@ -226,6 +256,7 @@ static const struct xe_device_desc adl_p_desc = {
 	.has_llc = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.subplatforms = (const struct xe_subplatform_desc[]) {
 		{ XE_SUBPLATFORM_ALDERLAKE_P_RPLU, "RPLU", adlp_rplu_ids },
@@ -245,6 +276,7 @@ static const struct xe_device_desc adl_n_desc = {
 	.has_llc = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.vm_max_level = 3,
@@ -263,6 +295,7 @@ static const struct xe_device_desc dg1_desc = {
 	.has_gsc_nvm = 1,
 	.has_heci_gscfi = 1,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.vm_max_level = 3,
@@ -293,6 +326,7 @@ static const struct xe_device_desc ats_m_desc = {
 	.pre_gmdid_media_ip = &media_ip_xehpm,
 	.dma_mask_size = 46,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 
 	DG2_FEATURES,
@@ -305,6 +339,7 @@ static const struct xe_device_desc dg2_desc = {
 	.pre_gmdid_media_ip = &media_ip_xehpm,
 	.dma_mask_size = 46,
 	.max_gt_per_tile = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 
 	DG2_FEATURES,
@@ -323,6 +358,7 @@ static const __maybe_unused struct xe_device_desc pvc_desc = {
 	.has_heci_gscfi = 1,
 	.max_gt_per_tile = 1,
 	.max_remote_tiles = 1,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 57,
 	.vm_max_level = 4,
@@ -338,6 +374,7 @@ static const struct xe_device_desc mtl_desc = {
 	.has_display = true,
 	.has_pxp = true,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.va_bits = 48,
 	.vm_max_level = 3,
 };
@@ -349,6 +386,7 @@ static const struct xe_device_desc lnl_desc = {
 	.has_flat_ccs = 1,
 	.has_pxp = true,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.needs_scratch = true,
 	.va_bits = 48,
 	.vm_max_level = 4,
@@ -373,6 +411,7 @@ static const struct xe_device_desc bmg_desc = {
 	.has_soc_remapper_telem = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.needs_scratch = true,
 	.subplatforms = (const struct xe_subplatform_desc[]) {
 		{ XE_SUBPLATFORM_BATTLEMAGE_G21, "G21", bmg_g21_ids },
@@ -391,6 +430,7 @@ static const struct xe_device_desc ptl_desc = {
 	.has_pre_prod_wa = 1,
 	.has_pxp = true,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.needs_scratch = true,
 	.needs_shared_vf_gt_wq = true,
 	.va_bits = 48,
@@ -404,6 +444,7 @@ static const struct xe_device_desc nvls_desc = {
 	.has_flat_ccs = 1,
 	.has_pre_prod_wa = 1,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.vm_max_level = 4,
@@ -425,11 +466,27 @@ static const struct xe_device_desc cri_desc = {
 	.has_soc_remapper_telem = true,
 	.has_sriov = true,
 	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
 	.require_force_probe = true,
 	.va_bits = 57,
 	.vm_max_level = 4,
 };
 
+static const struct xe_device_desc nvlp_desc = {
+	PLATFORM(NOVALAKE_P),
+	.dma_mask_size = 46,
+	.has_cached_pt = true,
+	.has_display = true,
+	.has_flat_ccs = 1,
+	.has_page_reclaim_hw_assist = true,
+	.has_pre_prod_wa = true,
+	.max_gt_per_tile = 2,
+	MULTI_LRC_MASK,
+	.require_force_probe = true,
+	.va_bits = 48,
+	.vm_max_level = 4,
+};
+
 #undef PLATFORM
 __diag_pop();
 
@@ -459,6 +516,7 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_WCL_IDS(INTEL_VGA_DEVICE, &ptl_desc),
 	INTEL_NVLS_IDS(INTEL_VGA_DEVICE, &nvls_desc),
 	INTEL_CRI_IDS(INTEL_PCI_DEVICE, &cri_desc),
+	INTEL_NVLP_IDS(INTEL_VGA_DEVICE, &nvlp_desc),
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
@@ -710,6 +768,7 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.skip_pcode = desc->skip_pcode;
 	xe->info.needs_scratch = desc->needs_scratch;
 	xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq;
+	xe->info.multi_lrc_mask = desc->multi_lrc_mask;
 
 	xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) &&
 				 xe_modparam.probe_display &&
@@ -720,6 +779,8 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.max_gt_per_tile = desc->max_gt_per_tile;
 	xe->info.tile_count = 1 + desc->max_remote_tiles;
 
+	xe_step_platform_get(xe);
+
 	err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0);
 	if (err)
 		return err;
@@ -786,6 +847,8 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile,
 	gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state;
 	gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask;
 	gt->info.engine_mask = graphics_desc->hw_engine_mask;
+	gt->info.num_geometry_xecore_fuse_regs = graphics_desc->num_geometry_xecore_fuse_regs;
+	gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs;
 
 	/*
 	 * Before media version 13, the media IP was part of the primary GT
@@ -852,7 +915,7 @@ static int xe_info_init(struct xe_device *xe,
 	if (desc->pre_gmdid_graphics_ip) {
 		graphics_ip = desc->pre_gmdid_graphics_ip;
 		media_ip = desc->pre_gmdid_media_ip;
-		xe->info.step = xe_step_pre_gmdid_get(xe);
+		xe_step_pre_gmdid_get(xe);
 	} else {
 		xe_assert(xe, !desc->pre_gmdid_media_ip);
 		ret = handle_gmdid(xe, &graphics_ip, &media_ip,
@@ -860,9 +923,7 @@ static int xe_info_init(struct xe_device *xe,
 		if (ret)
 			return ret;
 
-		xe->info.step = xe_step_gmdid_get(xe,
-						  graphics_gmdid_revid,
-						  media_gmdid_revid);
+		xe_step_gmdid_get(xe, graphics_gmdid_revid, media_gmdid_revid);
 	}
 
 	/*
@@ -886,12 +947,14 @@ static int xe_info_init(struct xe_device *xe,
 		media_desc = NULL;
 	}
 
+	xe->info.has_access_counter = graphics_desc->has_access_counter;
 	xe->info.has_asid = graphics_desc->has_asid;
 	xe->info.has_atomic_enable_pte_bit = graphics_desc->has_atomic_enable_pte_bit;
 	if (xe->info.platform != XE_PVC)
 		xe->info.has_device_atomics_on_smem = 1;
 
 	xe->info.has_range_tlb_inval = graphics_desc->has_range_tlb_inval;
+	xe->info.has_ctx_tlb_inval = graphics_desc->has_ctx_tlb_inval;
 	xe->info.has_usm = graphics_desc->has_usm;
 	xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp;
 	xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20;
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index c5fe9b1836d22..8eee4fb1c57cf 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -30,6 +30,7 @@ struct xe_device_desc {
 	u8 dma_mask_size;
 	u8 max_remote_tiles:2;
 	u8 max_gt_per_tile:2;
+	u8 multi_lrc_mask;
 	u8 va_bits;
 	u8 vm_max_level;
 	u8 vram_flags;
@@ -66,11 +67,15 @@ struct xe_device_desc {
 struct xe_graphics_desc {
 	u64 hw_engine_mask;	/* hardware engines provided by graphics IP */
 	u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */
+	u8 num_geometry_xecore_fuse_regs;
+	u8 num_compute_xecore_fuse_regs;
 
+	u8 has_access_counter:1;
 	u8 has_asid:1;
 	u8 has_atomic_enable_pte_bit:1;
 	u8 has_indirect_ring_state:1;
 	u8 has_range_tlb_inval:1;
+	u8 has_ctx_tlb_inval:1;
 	u8 has_usm:1;
 	u8 has_64bit_timestamp:1;
 };
diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c
index 0d33c14ea0cf6..dc66d0c7ee062 100644
--- a/drivers/gpu/drm/xe/xe_pcode.c
+++ b/drivers/gpu/drm/xe/xe_pcode.c
@@ -348,33 +348,3 @@ int xe_pcode_probe_early(struct xe_device *xe)
 	return xe_pcode_ready(xe, false);
 }
 ALLOW_ERROR_INJECTION(xe_pcode_probe_early, ERRNO); /* See xe_pci_probe */
-
-/* Helpers with drm device. These should only be called by the display side */
-#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
-
-int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct xe_tile *tile = xe_device_get_root_tile(xe);
-
-	return xe_pcode_read(tile, mbox, val, val1);
-}
-
-int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct xe_tile *tile = xe_device_get_root_tile(xe);
-
-	return xe_pcode_write_timeout(tile, mbox, val, timeout_ms);
-}
-
-int intel_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
-			u32 reply_mask, u32 reply, int timeout_base_ms)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct xe_tile *tile = xe_device_get_root_tile(xe);
-
-	return xe_pcode_request(tile, mbox, request, reply_mask, reply, timeout_base_ms);
-}
-
-#endif
diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h
index a5584c1c75f93..490e4f2696078 100644
--- a/drivers/gpu/drm/xe/xe_pcode.h
+++ b/drivers/gpu/drm/xe/xe_pcode.h
@@ -34,12 +34,4 @@ int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request,
 	| FIELD_PREP(PCODE_MB_PARAM1, param1)\
 	| FIELD_PREP(PCODE_MB_PARAM2, param2))
 
-/* Helpers with drm device */
-int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1);
-int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms);
-#define intel_pcode_write(drm, mbox, val) \
-	intel_pcode_write_timeout((drm), (mbox), (val), 1)
-int intel_pcode_request(struct drm_device *drm, u32 mbox, u32 request,
-			u32 reply_mask, u32 reply, int timeout_base_ms);
-
 #endif
diff --git a/drivers/gpu/drm/xe/xe_platform_types.h b/drivers/gpu/drm/xe/xe_platform_types.h
index f516dbddfd882..6cff385227ea1 100644
--- a/drivers/gpu/drm/xe/xe_platform_types.h
+++ b/drivers/gpu/drm/xe/xe_platform_types.h
@@ -26,6 +26,7 @@ enum xe_platform {
 	XE_PANTHERLAKE,
 	XE_NOVALAKE_S,
 	XE_CRESCENTISLAND,
+	XE_NOVALAKE_P,
 };
 
 enum xe_subplatform {
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 13b355fadd581..2d9ce2c4cb4fe 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -1655,14 +1655,35 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
 	XE_WARN_ON(!level);
 	/* Check for leaf node */
 	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
-	    (!xe_child->base.children || !xe_child->base.children[first])) {
+	    xe_child->level <= MAX_HUGEPTE_LEVEL) {
 		struct iosys_map *leaf_map = &xe_child->bo->vmap;
 		pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk);
 
 		for (pgoff_t i = 0; i < count; i++) {
-			u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
+			u64 pte;
 			int ret;
 
+			/*
+			 * If not a leaf pt, skip unless non-leaf pt is interleaved between
+			 * leaf ptes which causes the page walk to skip over the child leaves
+			 */
+			if (xe_child->base.children && xe_child->base.children[first + i]) {
+				u64 pt_size = 1ULL << walk->shifts[xe_child->level];
+				bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) ||
+					       (i == count - 1 && !IS_ALIGNED(next, pt_size));
+
+				if (!edge_pt) {
+					xe_page_reclaim_list_abort(xe_walk->tile->primary_gt,
+								   xe_walk->prl,
+								   "PT is skipped by walk at level=%u offset=%lu",
+								   xe_child->level, first + i);
+					break;
+				}
+				continue;
+			}
+
+			pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
+
 			/*
 			 * In rare scenarios, pte may not be written yet due to racy conditions.
 			 * In such cases, invalidate the PRL and fallback to full PPC invalidation.
@@ -1674,9 +1695,8 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
 			}
 
 			/* Ensure it is a defined page */
-			xe_tile_assert(xe_walk->tile,
-				       xe_child->level == 0 ||
-				       (pte & (XE_PTE_PS64 | XE_PDE_PS_2M | XE_PDPE_PS_1G)));
+			xe_tile_assert(xe_walk->tile, xe_child->level == 0 ||
+				       (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G)));
 
 			/* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */
 			if (pte & XE_PTE_PS64)
@@ -1701,11 +1721,11 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
 	killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
 
 	/*
-	 * Verify PRL is active and if entry is not a leaf pte (base.children conditions),
-	 * there is a potential need to invalidate the PRL if any PTE (num_live) are dropped.
+	 * Verify if any PTE are potentially dropped at non-leaf levels, either from being
+	 * killed or the page walk covers the region.
 	 */
-	if (xe_walk->prl && level > 1 && xe_child->num_live &&
-	    xe_child->base.children && xe_child->base.children[first]) {
+	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
+	    xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) {
 		bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base);
 
 		/*
diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c
index d61446bf9c19b..e2978e48f6607 100644
--- a/drivers/gpu/drm/xe/xe_pxp.c
+++ b/drivers/gpu/drm/xe/xe_pxp.c
@@ -312,8 +312,8 @@ void xe_pxp_irq_handler(struct xe_device *xe, u16 iir)
 
 static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable)
 {
-	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
-		  _MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
+	u32 val = enable ? REG_MASKED_FIELD_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
+		  REG_MASKED_FIELD_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
 
 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
 	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index f96f2844c5bab..4852fdcb4b959 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -142,9 +142,6 @@ query_engine_cycles(struct xe_device *xe,
 		return -EINVAL;
 
 	eci = &resp.eci;
-	if (eci->gt_id >= xe->info.max_gt_per_tile)
-		return -EINVAL;
-
 	gt = xe_device_get_gt(xe, eci->gt_id);
 	if (!gt)
 		return -EINVAL;
@@ -343,6 +340,8 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query)
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT;
 	config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY;
+	config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
+		DRM_XE_QUERY_CONFIG_FLAG_HAS_DISABLE_STATE_CACHE_PERF_FIX;
 	config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
 		xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K;
 	config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits;
diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c
index a07be161cfa24..2df0277efb2f5 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr.c
+++ b/drivers/gpu/drm/xe/xe_reg_sr.c
@@ -13,6 +13,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 
+#include "xe_assert.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
@@ -20,6 +21,7 @@
 #include "xe_gt_printk.h"
 #include "xe_gt_types.h"
 #include "xe_hw_engine_types.h"
+#include "xe_lrc.h"
 #include "xe_mmio.h"
 #include "xe_rtp_types.h"
 
@@ -171,8 +173,11 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
 	if (xa_empty(&sr->xa))
 		return;
 
-	if (IS_SRIOV_VF(gt_to_xe(gt)))
-		return;
+	/*
+	 * We don't process non-LRC reg_sr lists in VF, so they should have
+	 * been empty in the check above.
+	 */
+	xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
 
 	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
 
@@ -206,3 +211,66 @@ void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
 			   str_yes_no(entry->reg.masked),
 			   str_yes_no(entry->reg.mcr));
 }
+
+static u32 readback_reg(struct xe_gt *gt, struct xe_reg reg)
+{
+	struct xe_reg_mcr mcr_reg = to_xe_reg_mcr(reg);
+
+	if (reg.mcr)
+		return xe_gt_mcr_unicast_read_any(gt, mcr_reg);
+	else
+		return xe_mmio_read32(&gt->mmio, reg);
+}
+
+/**
+ * xe_reg_sr_readback_check() - Readback registers referenced in save/restore
+ *     entries and check whether the programming is in place.
+ * @sr: Save/restore entries
+ * @gt: GT to read register from
+ * @p: DRM printer to report discrepancies on
+ */
+void xe_reg_sr_readback_check(struct xe_reg_sr *sr,
+			      struct xe_gt *gt,
+			      struct drm_printer *p)
+{
+	struct xe_reg_sr_entry *entry;
+	unsigned long offset;
+
+	xa_for_each(&sr->xa, offset, entry) {
+		u32 val = readback_reg(gt, entry->reg);
+		u32 mask = entry->clr_bits | entry->set_bits;
+
+		if ((val & mask) != entry->set_bits)
+			drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n",
+				   offset, mask, entry->set_bits, val & mask);
+	}
+}
+
+/**
+ * xe_reg_sr_lrc_check() - Check LRC for registers referenced in save/restore
+ *     entries and check whether the programming is in place.
+ * @sr: Save/restore entries
+ * @gt: GT to read register from
+ * @hwe: Hardware engine type to check LRC for
+ * @p: DRM printer to report discrepancies on
+ */
+void xe_reg_sr_lrc_check(struct xe_reg_sr *sr,
+			 struct xe_gt *gt,
+			 struct xe_hw_engine *hwe,
+			 struct drm_printer *p)
+{
+	struct xe_reg_sr_entry *entry;
+	unsigned long offset;
+
+	xa_for_each(&sr->xa, offset, entry) {
+		u32 val;
+		int ret = xe_lrc_lookup_default_reg_value(gt, hwe->class, offset, &val);
+		u32 mask = entry->clr_bits | entry->set_bits;
+
+		if (ret == -ENOENT)
+			drm_printf(p, "%#8lx :: not found in LRC for %s\n", offset, hwe->name);
+		else if ((val & mask) != entry->set_bits)
+			drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n",
+				   offset, mask, entry->set_bits, val & mask);
+	}
+}
diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h
index 51fbba423e27f..1ec6e8ecf2784 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr.h
+++ b/drivers/gpu/drm/xe/xe_reg_sr.h
@@ -19,6 +19,13 @@ struct drm_printer;
 
 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe);
 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p);
+void xe_reg_sr_readback_check(struct xe_reg_sr *sr,
+			      struct xe_gt *gt,
+			      struct drm_printer *p);
+void xe_reg_sr_lrc_check(struct xe_reg_sr *sr,
+			 struct xe_gt *gt,
+			 struct xe_hw_engine *hwe,
+			 struct drm_printer *p);
 
 int xe_reg_sr_add(struct xe_reg_sr *sr, const struct xe_reg_sr_entry *e,
 		  struct xe_gt *gt);
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c
index 1d36c09681aaa..80577e4b7437c 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.c
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c
@@ -75,7 +75,15 @@ static const struct xe_rtp_entry_sr register_whitelist[] = {
 	  XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))
 	},
 	{ XE_RTP_NAME("14024997852"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(WHITELIST(FF_MODE,
+				   RING_FORCE_TO_NONPRIV_ACCESS_RW),
+			 WHITELIST(VFLSKPD,
+				   RING_FORCE_TO_NONPRIV_ACCESS_RW))
+	},
+	{ XE_RTP_NAME("14024997852"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
+		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(WHITELIST(FF_MODE,
 				   RING_FORCE_TO_NONPRIV_ACCESS_RW),
 			 WHITELIST(VFLSKPD,
@@ -181,7 +189,7 @@ void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe)
 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
 
 	xe_rtp_process_to_sr(&ctx, register_whitelist, ARRAY_SIZE(register_whitelist),
-			     &hwe->reg_whitelist);
+			     &hwe->reg_whitelist, false);
 	whitelist_apply_to_hwe(hwe);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h
index 4e00008b70817..5f4ab08c06869 100644
--- a/drivers/gpu/drm/xe/xe_res_cursor.h
+++ b/drivers/gpu/drm/xe/xe_res_cursor.h
@@ -58,7 +58,7 @@ struct xe_res_cursor {
 	/** @dma_addr: Current element in a struct drm_pagemap_addr array */
 	const struct drm_pagemap_addr *dma_addr;
 	/** @mm: Buddy allocator for VRAM cursor */
-	struct drm_buddy *mm;
+	struct gpu_buddy *mm;
 	/**
 	 * @dma_start: DMA start address for the current segment.
 	 * This may be different to @dma_addr.addr since elements in
@@ -69,7 +69,7 @@ struct xe_res_cursor {
 	u64 dma_seg_size;
 };
 
-static struct drm_buddy *xe_res_get_buddy(struct ttm_resource *res)
+static struct gpu_buddy *xe_res_get_buddy(struct ttm_resource *res)
 {
 	struct ttm_resource_manager *mgr;
 
@@ -104,30 +104,30 @@ static inline void xe_res_first(struct ttm_resource *res,
 	case XE_PL_STOLEN:
 	case XE_PL_VRAM0:
 	case XE_PL_VRAM1: {
-		struct drm_buddy_block *block;
+		struct gpu_buddy_block *block;
 		struct list_head *head, *next;
-		struct drm_buddy *mm = xe_res_get_buddy(res);
+		struct gpu_buddy *mm = xe_res_get_buddy(res);
 
 		head = &to_xe_ttm_vram_mgr_resource(res)->blocks;
 
 		block = list_first_entry_or_null(head,
-						 struct drm_buddy_block,
+						 struct gpu_buddy_block,
 						 link);
 		if (!block)
 			goto fallback;
 
-		while (start >= drm_buddy_block_size(mm, block)) {
-			start -= drm_buddy_block_size(mm, block);
+		while (start >= gpu_buddy_block_size(mm, block)) {
+			start -= gpu_buddy_block_size(mm, block);
 
 			next = block->link.next;
 			if (next != head)
-				block = list_entry(next, struct drm_buddy_block,
+				block = list_entry(next, struct gpu_buddy_block,
 						   link);
 		}
 
 		cur->mm = mm;
-		cur->start = drm_buddy_block_offset(block) + start;
-		cur->size = min(drm_buddy_block_size(mm, block) - start,
+		cur->start = gpu_buddy_block_offset(block) + start;
+		cur->size = min(gpu_buddy_block_size(mm, block) - start,
 				size);
 		cur->remaining = size;
 		cur->node = block;
@@ -259,7 +259,7 @@ static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr,
  */
 static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	struct list_head *next;
 	u64 start;
 
@@ -295,18 +295,18 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
 		block = cur->node;
 
 		next = block->link.next;
-		block = list_entry(next, struct drm_buddy_block, link);
+		block = list_entry(next, struct gpu_buddy_block, link);
 
 
-		while (start >= drm_buddy_block_size(cur->mm, block)) {
-			start -= drm_buddy_block_size(cur->mm, block);
+		while (start >= gpu_buddy_block_size(cur->mm, block)) {
+			start -= gpu_buddy_block_size(cur->mm, block);
 
 			next = block->link.next;
-			block = list_entry(next, struct drm_buddy_block, link);
+			block = list_entry(next, struct gpu_buddy_block, link);
 		}
 
-		cur->start = drm_buddy_block_offset(block) + start;
-		cur->size = min(drm_buddy_block_size(cur->mm, block) - start,
+		cur->start = gpu_buddy_block_offset(block) + start;
+		cur->size = min(gpu_buddy_block_size(cur->mm, block) - start,
 				cur->remaining);
 		cur->node = block;
 		break;
diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c
index b7c26e2fb411e..991f218f1cc30 100644
--- a/drivers/gpu/drm/xe/xe_rtp.c
+++ b/drivers/gpu/drm/xe/xe_rtp.c
@@ -55,6 +55,13 @@ static bool rule_matches(const struct xe_device *xe,
 			match = xe->info.platform == r->platform &&
 				xe->info.subplatform == r->subplatform;
 			break;
+		case XE_RTP_MATCH_PLATFORM_STEP:
+			if (drm_WARN_ON(&xe->drm, xe->info.step.platform == STEP_NONE))
+				return false;
+
+			match = xe->info.step.platform >= r->step_start &&
+				xe->info.step.platform < r->step_end;
+			break;
 		case XE_RTP_MATCH_GRAPHICS_VERSION:
 			if (drm_WARN_ON(&xe->drm, !gt))
 				return false;
@@ -270,6 +277,8 @@ static void rtp_mark_active(struct xe_device *xe,
  * @sr: Save-restore struct where matching rules execute the action. This can be
  *      viewed as the "coalesced view" of multiple the tables. The bits for each
  *      register set are expected not to collide with previously added entries
+ * @process_in_vf: Whether this RTP table should get processed for SR-IOV VF
+ *      devices.  Should generally only be 'true' for LRC tables.
  *
  * Walk the table pointed by @entries (with an empty sentinel) and add all
  * entries with matching rules to @sr. If @hwe is not NULL, its mmio_base is
@@ -278,7 +287,8 @@ static void rtp_mark_active(struct xe_device *xe,
 void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx,
 			  const struct xe_rtp_entry_sr *entries,
 			  size_t n_entries,
-			  struct xe_reg_sr *sr)
+			  struct xe_reg_sr *sr,
+			  bool process_in_vf)
 {
 	const struct xe_rtp_entry_sr *entry;
 	struct xe_hw_engine *hwe = NULL;
@@ -287,6 +297,9 @@ void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx,
 
 	rtp_get_context(ctx, &hwe, &gt, &xe);
 
+	if (!process_in_vf && IS_SRIOV_VF(xe))
+		return;
+
 	xe_assert(xe, entries);
 
 	for (entry = entries; entry - entries < n_entries; entry++) {
diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h
index ba5f940c0a961..7d6daa7eb1e4c 100644
--- a/drivers/gpu/drm/xe/xe_rtp.h
+++ b/drivers/gpu/drm/xe/xe_rtp.h
@@ -35,6 +35,10 @@ struct xe_reg_sr;
 	{ .match_type = XE_RTP_MATCH_SUBPLATFORM,				\
 	  .platform = plat__, .subplatform = sub__ }
 
+#define _XE_RTP_RULE_PLATFORM_STEP(start__, end__)				\
+	{ .match_type = XE_RTP_MATCH_PLATFORM_STEP,				\
+	  .step_start = start__, .step_end = end__ }
+
 #define _XE_RTP_RULE_GRAPHICS_STEP(start__, end__)				\
 	{ .match_type = XE_RTP_MATCH_GRAPHICS_STEP,				\
 	  .step_start = start__, .step_end = end__ }
@@ -66,6 +70,22 @@ struct xe_reg_sr;
 #define XE_RTP_RULE_SUBPLATFORM(plat_, sub_)					\
 	_XE_RTP_RULE_SUBPLATFORM(XE_##plat_, XE_SUBPLATFORM_##plat_##_##sub_)
 
+/**
+ * XE_RTP_RULE_PLATFORM_STEP - Create rule matching platform-level stepping
+ * @start_: First stepping matching the rule
+ * @end_: First stepping that does not match the rule
+ *
+ * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive
+ * on the left, exclusive on the right.
+ *
+ * You need to make sure that proper support for reading platform-level stepping
+ * information is present for the target platform before using this rule.
+ *
+ * Refer to XE_RTP_RULES() for expected usage.
+ */
+#define XE_RTP_RULE_PLATFORM_STEP(start_, end_)					\
+	_XE_RTP_RULE_PLATFORM_STEP(STEP_##start_, STEP_##end_)
+
 /**
  * XE_RTP_RULE_GRAPHICS_STEP - Create rule matching graphics stepping
  * @start_: First stepping matching the rule
@@ -431,7 +451,8 @@ void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx,
 
 void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx,
 			  const struct xe_rtp_entry_sr *entries,
-			  size_t n_entries, struct xe_reg_sr *sr);
+			  size_t n_entries, struct xe_reg_sr *sr,
+			  bool process_in_vf);
 
 void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 		    const struct xe_rtp_entry *entries);
diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h
index 6ba7f226c2274..166251615be13 100644
--- a/drivers/gpu/drm/xe/xe_rtp_types.h
+++ b/drivers/gpu/drm/xe/xe_rtp_types.h
@@ -41,6 +41,7 @@ struct xe_rtp_action {
 enum {
 	XE_RTP_MATCH_PLATFORM,
 	XE_RTP_MATCH_SUBPLATFORM,
+	XE_RTP_MATCH_PLATFORM_STEP,
 	XE_RTP_MATCH_GRAPHICS_VERSION,
 	XE_RTP_MATCH_GRAPHICS_VERSION_RANGE,
 	XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT,
diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c
index b738102575d40..f32045f40b7a1 100644
--- a/drivers/gpu/drm/xe/xe_sa.c
+++ b/drivers/gpu/drm/xe/xe_sa.c
@@ -89,6 +89,12 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size,
 		if (ret)
 			return ERR_PTR(ret);
 
+		if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
+			fs_reclaim_acquire(GFP_KERNEL);
+			might_lock(&sa_manager->swap_guard);
+			fs_reclaim_release(GFP_KERNEL);
+		}
+
 		shadow = xe_managed_bo_create_pin_map(xe, tile, size,
 						      XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 						      XE_BO_FLAG_GGTT |
@@ -175,6 +181,36 @@ struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size,
 	return drm_suballoc_new(&sa_manager->base, size, gfp, true, 0);
 }
 
+/**
+ * xe_sa_bo_alloc() - Allocate uninitialized suballoc object.
+ * @gfp: gfp flags used for memory allocation.
+ *
+ * Allocate memory for an uninitialized suballoc object. Intended usage is
+ * allocate memory for suballoc object outside of a reclaim tainted context
+ * and then be initialized at a later time in a reclaim tainted context.
+ *
+ * Return: a new uninitialized suballoc object, or an ERR_PTR(-ENOMEM).
+ */
+struct drm_suballoc *xe_sa_bo_alloc(gfp_t gfp)
+{
+	return drm_suballoc_alloc(gfp);
+}
+
+/**
+ * xe_sa_bo_init() - Initialize a suballocation.
+ * @sa_manager: pointer to the sa_manager
+ * @sa: The struct drm_suballoc.
+ * @size: number of bytes we want to suballocate.
+ *
+ * Try to make a suballocation on a pre-allocated suballoc object of size @size.
+ *
+ * Return: zero on success, errno on failure.
+ */
+int xe_sa_bo_init(struct xe_sa_manager *sa_manager, struct drm_suballoc *sa, size_t size)
+{
+	return drm_suballoc_insert(&sa_manager->base, sa, size, true, 0);
+}
+
 /**
  * xe_sa_bo_flush_write() - Copy the data from the sub-allocation to the GPU memory.
  * @sa_bo: the &drm_suballoc to flush
diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h
index 05e9a4e00e789..50218b0d14041 100644
--- a/drivers/gpu/drm/xe/xe_sa.h
+++ b/drivers/gpu/drm/xe/xe_sa.h
@@ -38,6 +38,8 @@ static inline struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager
 	return __xe_sa_bo_new(sa_manager, size, GFP_KERNEL);
 }
 
+struct drm_suballoc *xe_sa_bo_alloc(gfp_t gfp);
+int xe_sa_bo_init(struct xe_sa_manager *sa_manager, struct drm_suballoc *sa, size_t size);
 void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo);
 void xe_sa_bo_sync_read(struct drm_suballoc *sa_bo);
 void xe_sa_bo_free(struct drm_suballoc *sa_bo, struct dma_fence *fence);
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c
index 3927666fe5566..ae5b38b2a884a 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.c
+++ b/drivers/gpu/drm/xe/xe_sched_job.c
@@ -190,11 +190,11 @@ static bool xe_fence_set_error(struct dma_fence *fence, int error)
 	unsigned long irq_flags;
 	bool signaled;
 
-	spin_lock_irqsave(fence->lock, irq_flags);
+	dma_fence_lock_irqsave(fence, irq_flags);
 	signaled = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
 	if (!signaled)
 		dma_fence_set_error(fence, error);
-	spin_unlock_irqrestore(fence->lock, irq_flags);
+	dma_fence_unlock_irqrestore(fence, irq_flags);
 
 	return signaled;
 }
diff --git a/drivers/gpu/drm/xe/xe_sleep.h b/drivers/gpu/drm/xe/xe_sleep.h
new file mode 100644
index 0000000000000..2bc3f4c0ee0b3
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sleep.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SLEEP_H_
+#define _XE_SLEEP_H_
+
+#include <linux/delay.h>
+#include <linux/math64.h>
+
+/**
+ * xe_sleep_relaxed_ms() - Sleep for an approximate time.
+ * @delay_ms: time in msec to sleep
+ *
+ * For smaller timeouts, sleep with 0.5ms accuracy.
+ */
+static inline void xe_sleep_relaxed_ms(unsigned int delay_ms)
+{
+	unsigned long min_us, max_us;
+
+	if (!delay_ms)
+		return;
+
+	if (delay_ms > 20) {
+		msleep(delay_ms);
+		return;
+	}
+
+	min_us = mul_u32_u32(delay_ms, 1000);
+	max_us = min_us + 500;
+
+	usleep_range(min_us, max_us);
+}
+
+/**
+ * xe_sleep_exponential_ms() - Sleep for a exponentially increased time.
+ * @sleep_period_ms: current time in msec to sleep
+ * @max_sleep_ms: maximum time in msec to sleep
+ *
+ * Sleep for the @sleep_period_ms and exponentially increase this time for the
+ * next loop, unless reaching the @max_sleep_ms limit.
+ *
+ * Return: approximate time in msec the task was delayed.
+ */
+static inline unsigned int xe_sleep_exponential_ms(unsigned int *sleep_period_ms,
+						   unsigned int max_sleep_ms)
+{
+	unsigned int delay_ms = *sleep_period_ms;
+	unsigned int next_delay_ms = 2 * delay_ms;
+
+	xe_sleep_relaxed_ms(delay_ms);
+	*sleep_period_ms = min(next_delay_ms, max_sleep_ms);
+	return delay_ms;
+}
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_soc_remapper.c b/drivers/gpu/drm/xe/xe_soc_remapper.c
index 1c391d7191966..c031336a6d753 100644
--- a/drivers/gpu/drm/xe/xe_soc_remapper.c
+++ b/drivers/gpu/drm/xe/xe_soc_remapper.c
@@ -4,6 +4,7 @@
  */
 
 #include "regs/xe_soc_remapper_regs.h"
+#include "xe_device.h"
 #include "xe_mmio.h"
 #include "xe_soc_remapper.h"
 
diff --git a/drivers/gpu/drm/xe/xe_sriov.c b/drivers/gpu/drm/xe/xe_sriov.c
index ea411944609b2..f3835867fce5d 100644
--- a/drivers/gpu/drm/xe/xe_sriov.c
+++ b/drivers/gpu/drm/xe/xe_sriov.c
@@ -120,7 +120,7 @@ int xe_sriov_init(struct xe_device *xe)
 		xe_sriov_vf_init_early(xe);
 
 	xe_assert(xe, !xe->sriov.wq);
-	xe->sriov.wq = alloc_workqueue("xe-sriov-wq", 0, 0);
+	xe->sriov.wq = alloc_workqueue("xe-sriov-wq", WQ_PERCPU, 0);
 	if (!xe->sriov.wq)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/xe/xe_sriov.h b/drivers/gpu/drm/xe/xe_sriov.h
index 6db45df556158..72e55543c30e4 100644
--- a/drivers/gpu/drm/xe/xe_sriov.h
+++ b/drivers/gpu/drm/xe/xe_sriov.h
@@ -28,7 +28,8 @@ static inline enum xe_sriov_mode xe_device_sriov_mode(const struct xe_device *xe
 
 static inline bool xe_device_is_sriov_pf(const struct xe_device *xe)
 {
-	return xe_device_sriov_mode(xe) == XE_SRIOV_MODE_PF;
+	return IS_ENABLED(CONFIG_PCI_IOV) &&
+		xe_device_sriov_mode(xe) == XE_SRIOV_MODE_PF;
 }
 
 static inline bool xe_device_is_sriov_vf(const struct xe_device *xe)
@@ -36,11 +37,7 @@ static inline bool xe_device_is_sriov_vf(const struct xe_device *xe)
 	return xe_device_sriov_mode(xe) == XE_SRIOV_MODE_VF;
 }
 
-#ifdef CONFIG_PCI_IOV
 #define IS_SRIOV_PF(xe) xe_device_is_sriov_pf(xe)
-#else
-#define IS_SRIOV_PF(xe) (typecheck(struct xe_device *, (xe)) && false)
-#endif
 #define IS_SRIOV_VF(xe) xe_device_is_sriov_vf(xe)
 
 #define IS_SRIOV(xe) (IS_SRIOV_PF(xe) || IS_SRIOV_VF(xe))
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c
index 6ce3c58e003c7..47a6e0fd66e0f 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf.c
@@ -20,11 +20,14 @@
 #include "xe_sriov_pf_sysfs.h"
 #include "xe_sriov_printk.h"
 
+static bool wanted_admin_only(struct xe_device *xe)
+{
+	return xe_configfs_admin_only_pf(to_pci_dev(xe->drm.dev));
+}
+
 static unsigned int wanted_max_vfs(struct xe_device *xe)
 {
-	if (IS_ENABLED(CONFIG_CONFIGFS_FS))
-		return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev));
-	return xe_modparam.max_vfs;
+	return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev));
 }
 
 static int pf_reduce_totalvfs(struct xe_device *xe, int limit)
@@ -76,6 +79,7 @@ bool xe_sriov_pf_readiness(struct xe_device *xe)
 
 	pf_reduce_totalvfs(xe, newlimit);
 
+	xe->sriov.pf.admin_only = wanted_admin_only(xe);
 	xe->sriov.pf.device_total_vfs = totalvfs;
 	xe->sriov.pf.driver_max_vfs = newlimit;
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
index 9054fdc345974..0fcc6cec4afc4 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
@@ -56,7 +56,8 @@ static inline unsigned int xe_sriov_pf_num_vfs(const struct xe_device *xe)
  */
 static inline bool xe_sriov_pf_admin_only(const struct xe_device *xe)
 {
-	return !xe->info.probe_display;
+	xe_assert(xe, IS_SRIOV_PF(xe));
+	return xe->sriov.pf.admin_only;
 }
 
 static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
index 01470c42e8a72..abe3677d33edf 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
@@ -7,6 +7,7 @@
 #include "xe_device.h"
 #include "xe_gt_sriov_pf_config.h"
 #include "xe_gt_sriov_pf_policy.h"
+#include "xe_lmtt.h"
 #include "xe_sriov.h"
 #include "xe_sriov_pf_helpers.h"
 #include "xe_sriov_pf_provision.h"
@@ -32,17 +33,6 @@ static bool pf_auto_provisioning_mode(struct xe_device *xe)
 	return xe->sriov.pf.provision.mode == XE_SRIOV_PROVISIONING_MODE_AUTO;
 }
 
-static bool pf_needs_provisioning(struct xe_gt *gt, unsigned int num_vfs)
-{
-	unsigned int n;
-
-	for (n = 1; n <= num_vfs; n++)
-		if (!xe_gt_sriov_pf_config_is_empty(gt, n))
-			return false;
-
-	return true;
-}
-
 static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs)
 {
 	struct xe_gt *gt;
@@ -51,8 +41,6 @@ static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs)
 	int err;
 
 	for_each_gt(gt, xe, id) {
-		if (!pf_needs_provisioning(gt, num_vfs))
-			return -EUCLEAN;
 		err = xe_gt_sriov_pf_config_set_fair(gt, VFID(1), num_vfs);
 		result = result ?: err;
 	}
@@ -436,3 +424,108 @@ int xe_sriov_pf_provision_query_vf_priority(struct xe_device *xe, unsigned int v
 
 	return !count ? -ENODATA : 0;
 }
+
+static u64 vram_per_tile(struct xe_tile *tile, u64 total)
+{
+	struct xe_device *xe = tile->xe;
+	unsigned int tcount = xe->info.tile_count;
+	u64 alignment = xe_lmtt_page_size(&tile->sriov.pf.lmtt);
+
+	total = round_up(total, tcount * alignment);
+	return div_u64(total, tcount);
+}
+
+/**
+ * xe_sriov_pf_provision_bulk_apply_vram() - Change VRAM provisioning for all VFs.
+ * @xe: the PF &xe_device
+ * @size: the VRAM size in [bytes] to set
+ *
+ * Change all VFs VRAM (LMEM) provisioning on all tiles.
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_sriov_pf_provision_bulk_apply_vram(struct xe_device *xe, u64 size)
+{
+	unsigned int num_vfs = xe_sriov_pf_get_totalvfs(xe);
+	struct xe_tile *tile;
+	unsigned int id;
+	int result = 0;
+	int err;
+
+	xe_assert(xe, xe_device_has_lmtt(xe));
+
+	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+
+	for_each_tile(tile, xe, id) {
+		err = xe_gt_sriov_pf_config_bulk_set_lmem_locked(tile->primary_gt,
+								 VFID(1), num_vfs,
+								 vram_per_tile(tile, size));
+		result = result ?: err;
+	}
+
+	return result;
+}
+
+/**
+ * xe_sriov_pf_provision_apply_vf_vram() - Change single VF VRAM allocation.
+ * @xe: the PF &xe_device
+ * @vfid: the VF identifier (can't be 0 == PFID)
+ * @size: VRAM size to set
+ *
+ * Change VF's VRAM provisioning on all tiles/GTs.
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_sriov_pf_provision_apply_vf_vram(struct xe_device *xe, unsigned int vfid, u64 size)
+{
+	struct xe_tile *tile;
+	unsigned int id;
+	int result = 0;
+	int err;
+
+	xe_assert(xe, vfid);
+	xe_assert(xe, xe_device_has_lmtt(xe));
+
+	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+
+	for_each_tile(tile, xe, id) {
+		err = xe_gt_sriov_pf_config_set_lmem_locked(tile->primary_gt, vfid,
+							    vram_per_tile(tile, size));
+		result = result ?: err;
+	}
+
+	return result;
+}
+
+/**
+ * xe_sriov_pf_provision_query_vf_vram() - Query VF's VRAM allocation.
+ * @xe: the PF &xe_device
+ * @vfid: the VF identifier (can't be 0 == PFID)
+ * @size: placeholder for the returned VRAM size
+ *
+ * Query VF's VRAM provisioning from all tiles/GTs.
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, u64 *size)
+{
+	struct xe_tile *tile;
+	unsigned int id;
+	u64 total = 0;
+
+	xe_assert(xe, vfid);
+
+	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+
+	for_each_tile(tile, xe, id)
+		total += xe_gt_sriov_pf_config_get_lmem_locked(tile->primary_gt, vfid);
+
+	*size = total;
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
index bccf23d51396b..f26f49539697b 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
@@ -24,6 +24,10 @@ int xe_sriov_pf_provision_bulk_apply_priority(struct xe_device *xe, u32 prio);
 int xe_sriov_pf_provision_apply_vf_priority(struct xe_device *xe, unsigned int vfid, u32 prio);
 int xe_sriov_pf_provision_query_vf_priority(struct xe_device *xe, unsigned int vfid, u32 *prio);
 
+int xe_sriov_pf_provision_bulk_apply_vram(struct xe_device *xe, u64 size);
+int xe_sriov_pf_provision_apply_vf_vram(struct xe_device *xe, unsigned int vfid, u64 size);
+int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, u64 *size);
+
 int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs);
 int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs);
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
index 0b88cdade6f19..ffc3447fc8d71 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
@@ -9,6 +9,7 @@
 #include <drm/drm_managed.h>
 
 #include "xe_assert.h"
+#include "xe_device.h"
 #include "xe_pci_sriov.h"
 #include "xe_pm.h"
 #include "xe_sriov.h"
@@ -44,7 +45,8 @@ static int emit_choice(char *buf, int choice, const char * const *array, size_t
  *     ├── .bulk_profile
  *     │   ├── exec_quantum_ms
  *     │   ├── preempt_timeout_us
- *     │   └── sched_priority
+ *     │   ├── sched_priority
+ *     │   └── vram_quota
  *     ├── pf/
  *     │   ├── ...
  *     │   ├── device -> ../../../BDF
@@ -59,7 +61,8 @@ static int emit_choice(char *buf, int choice, const char * const *array, size_t
  *     │   └── profile
  *     │       ├── exec_quantum_ms
  *     │       ├── preempt_timeout_us
- *     │       └── sched_priority
+ *     │       ├── sched_priority
+ *     │       └── vram_quota
  *     ├── vf2/
  *     :
  *     └── vfN/
@@ -132,6 +135,7 @@ static XE_SRIOV_DEV_ATTR_WO(NAME)
 
 DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(exec_quantum_ms, eq, u32);
 DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(preempt_timeout_us, pt, u32);
+DEFINE_SIMPLE_BULK_PROVISIONING_SRIOV_DEV_ATTR_WO(vram_quota, vram, u64);
 
 static const char * const sched_priority_names[] = {
 	[GUC_SCHED_PRIORITY_LOW] = "low",
@@ -181,12 +185,26 @@ static struct attribute *bulk_profile_dev_attrs[] = {
 	&xe_sriov_dev_attr_exec_quantum_ms.attr,
 	&xe_sriov_dev_attr_preempt_timeout_us.attr,
 	&xe_sriov_dev_attr_sched_priority.attr,
+	&xe_sriov_dev_attr_vram_quota.attr,
 	NULL
 };
 
+static umode_t profile_dev_attr_is_visible(struct kobject *kobj,
+					   struct attribute *attr, int index)
+{
+	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
+
+	if (attr == &xe_sriov_dev_attr_vram_quota.attr &&
+	    !xe_device_has_lmtt(vkobj->xe))
+		return 0;
+
+	return attr->mode;
+}
+
 static const struct attribute_group bulk_profile_dev_attr_group = {
 	.name = ".bulk_profile",
 	.attrs = bulk_profile_dev_attrs,
+	.is_visible = profile_dev_attr_is_visible,
 };
 
 static const struct attribute_group *xe_sriov_dev_attr_groups[] = {
@@ -228,6 +246,7 @@ static XE_SRIOV_VF_ATTR(NAME)
 
 DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(exec_quantum_ms, eq, u32, "%u\n");
 DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(preempt_timeout_us, pt, u32, "%u\n");
+DEFINE_SIMPLE_PROVISIONING_SRIOV_VF_ATTR(vram_quota, vram, u64, "%llu\n");
 
 static ssize_t xe_sriov_vf_attr_sched_priority_show(struct xe_device *xe, unsigned int vfid,
 						    char *buf)
@@ -274,6 +293,7 @@ static struct attribute *profile_vf_attrs[] = {
 	&xe_sriov_vf_attr_exec_quantum_ms.attr,
 	&xe_sriov_vf_attr_preempt_timeout_us.attr,
 	&xe_sriov_vf_attr_sched_priority.attr,
+	&xe_sriov_vf_attr_vram_quota.attr,
 	NULL
 };
 
@@ -286,6 +306,13 @@ static umode_t profile_vf_attr_is_visible(struct kobject *kobj,
 	    !sched_priority_change_allowed(vkobj->vfid))
 		return attr->mode & 0444;
 
+	if (attr == &xe_sriov_vf_attr_vram_quota.attr) {
+		if (!IS_DGFX(vkobj->xe) || vkobj->vfid == PFID)
+			return 0;
+		if (!xe_device_has_lmtt(vkobj->xe))
+			return attr->mode & 0444;
+	}
+
 	return attr->mode;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_types.h
index b0253e1ae5da4..080cf10512f4b 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_types.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_types.h
@@ -36,6 +36,9 @@ struct xe_sriov_metadata {
  * @XE_SRIOV_MODE_PF mode.
  */
 struct xe_device_pf {
+	/** @admin_only: PF functionality focused on VFs management only. */
+	bool admin_only;
+
 	/** @device_total_vfs: Maximum number of VFs supported by the device. */
 	u16 device_total_vfs;
 
diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c
index 2860986f82f78..d0f888c31831f 100644
--- a/drivers/gpu/drm/xe/xe_step.c
+++ b/drivers/gpu/drm/xe/xe_step.c
@@ -108,6 +108,28 @@ static const int pvc_basedie_subids[] = {
 
 __diag_pop();
 
+/**
+ * xe_step_platform_get - Determine platform-level stepping from PCI revid
+ * @xe: Xe device
+ *
+ * Convert the PCI revid into a platform-level stepping value and store that
+ * in the device info.
+ */
+void xe_step_platform_get(struct xe_device *xe)
+{
+	/*
+	 * Not all platforms map PCI revid directly into our symbolic stepping
+	 * enumeration. Some platforms will have a single PCI revid used for a
+	 * range platform level steppings and some might even require specific
+	 * mappings. So prefer to err on the side of caution and include only
+	 * the platforms from which we need the stepping info for workaround
+	 * checks.
+	 */
+
+	if (xe->info.platform == XE_NOVALAKE_P)
+		xe->info.step.platform = STEP_A0 + xe->info.revid;
+}
+
 /**
  * xe_step_pre_gmdid_get - Determine IP steppings from PCI revid
  * @xe: Xe device
@@ -115,15 +137,17 @@ __diag_pop();
  * Convert the PCI revid into proper IP steppings.  This should only be
  * used on platforms that do not have GMD_ID support.
  */
-struct xe_step_info xe_step_pre_gmdid_get(struct xe_device *xe)
+void xe_step_pre_gmdid_get(struct xe_device *xe)
 {
 	const struct xe_step_info *revids = NULL;
-	struct xe_step_info step = {};
 	u16 revid = xe->info.revid;
 	int size = 0;
 	const int *basedie_info = NULL;
 	int basedie_size = 0;
 	int baseid = 0;
+	u8 graphics = STEP_NONE;
+	u8 media = STEP_NONE;
+	u8 basedie = STEP_NONE;
 
 	if (xe->info.platform == XE_PVC) {
 		baseid = FIELD_GET(GENMASK(5, 3), xe->info.revid);
@@ -166,10 +190,12 @@ struct xe_step_info xe_step_pre_gmdid_get(struct xe_device *xe)
 
 	/* Not using the stepping scheme for the platform yet. */
 	if (!revids)
-		return step;
+		goto done;
 
 	if (revid < size && revids[revid].graphics != STEP_NONE) {
-		step = revids[revid];
+		graphics = revids[revid].graphics;
+		media = revids[revid].media;
+		basedie = revids[revid].basedie;
 	} else {
 		drm_warn(&xe->drm, "Unknown revid 0x%02x\n", revid);
 
@@ -187,25 +213,30 @@ struct xe_step_info xe_step_pre_gmdid_get(struct xe_device *xe)
 		if (revid < size) {
 			drm_dbg(&xe->drm, "Using steppings for revid 0x%02x\n",
 				revid);
-			step = revids[revid];
+			graphics = revids[revid].graphics;
+			media = revids[revid].media;
+			basedie = revids[revid].basedie;
 		} else {
 			drm_dbg(&xe->drm, "Using future steppings\n");
-			step.graphics = STEP_FUTURE;
+			graphics = STEP_FUTURE;
 		}
 	}
 
-	drm_WARN_ON(&xe->drm, step.graphics == STEP_NONE);
+	drm_WARN_ON(&xe->drm, graphics == STEP_NONE);
 
 	if (basedie_info && basedie_size) {
 		if (baseid < basedie_size && basedie_info[baseid] != STEP_NONE) {
-			step.basedie = basedie_info[baseid];
+			basedie = basedie_info[baseid];
 		} else {
 			drm_warn(&xe->drm, "Unknown baseid 0x%02x\n", baseid);
-			step.basedie = STEP_FUTURE;
+			basedie = STEP_FUTURE;
 		}
 	}
 
-	return step;
+done:
+	xe->info.step.graphics = graphics;
+	xe->info.step.media = media;
+	xe->info.step.basedie = basedie;
 }
 
 /**
@@ -220,28 +251,27 @@ struct xe_step_info xe_step_pre_gmdid_get(struct xe_device *xe)
  * all platforms:  major steppings (A0, B0, etc.) are 4 apart, with minor
  * steppings (A1, A2, etc.) taking the values in between.
  */
-struct xe_step_info xe_step_gmdid_get(struct xe_device *xe,
-				      u32 graphics_gmdid_revid,
-				      u32 media_gmdid_revid)
+void xe_step_gmdid_get(struct xe_device *xe,
+		       u32 graphics_gmdid_revid,
+		       u32 media_gmdid_revid)
 {
-	struct xe_step_info step = {
-		.graphics = STEP_A0 + graphics_gmdid_revid,
-		.media = STEP_A0 + media_gmdid_revid,
-	};
+	u8 graphics = STEP_A0 + graphics_gmdid_revid;
+	u8 media = STEP_A0 + media_gmdid_revid;
 
-	if (step.graphics >= STEP_FUTURE) {
-		step.graphics = STEP_FUTURE;
+	if (graphics >= STEP_FUTURE) {
+		graphics = STEP_FUTURE;
 		drm_dbg(&xe->drm, "Graphics GMD_ID revid value %d treated as future stepping\n",
 			graphics_gmdid_revid);
 	}
 
-	if (step.media >= STEP_FUTURE) {
-		step.media = STEP_FUTURE;
+	if (media >= STEP_FUTURE) {
+		media = STEP_FUTURE;
 		drm_dbg(&xe->drm, "Media GMD_ID revid value %d treated as future stepping\n",
 			media_gmdid_revid);
 	}
 
-	return step;
+	xe->info.step.graphics = graphics;
+	xe->info.step.media = media;
 }
 
 #define STEP_NAME_CASE(name)	\
diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h
index 686cb59200c28..41f1c95c46e5c 100644
--- a/drivers/gpu/drm/xe/xe_step.h
+++ b/drivers/gpu/drm/xe/xe_step.h
@@ -12,10 +12,12 @@
 
 struct xe_device;
 
-struct xe_step_info xe_step_pre_gmdid_get(struct xe_device *xe);
-struct xe_step_info xe_step_gmdid_get(struct xe_device *xe,
-				      u32 graphics_gmdid_revid,
-				      u32 media_gmdid_revid);
+void xe_step_platform_get(struct xe_device *xe);
+
+void xe_step_pre_gmdid_get(struct xe_device *xe);
+void xe_step_gmdid_get(struct xe_device *xe,
+		       u32 graphics_gmdid_revid,
+		       u32 media_gmdid_revid);
 static inline u32 xe_step_to_gmdid(enum xe_step step) { return step - STEP_A0; }
 
 const char *xe_step_name(enum xe_step step);
diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h
index d978cc2512f25..43ca738507396 100644
--- a/drivers/gpu/drm/xe/xe_step_types.h
+++ b/drivers/gpu/drm/xe/xe_step_types.h
@@ -9,6 +9,7 @@
 #include <linux/types.h>
 
 struct xe_step_info {
+	u8 platform;
 	u8 graphics;
 	u8 media;
 	u8 basedie;
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index ca67d0bdbfac5..a91c84487a674 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -19,6 +19,7 @@
 #include "xe_pt.h"
 #include "xe_svm.h"
 #include "xe_tile.h"
+#include "xe_tlb_inval.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vm.h"
 #include "xe_vm_types.h"
@@ -225,6 +226,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
 			      const struct mmu_notifier_range *mmu_range)
 {
 	struct xe_vm *vm = gpusvm_to_vm(gpusvm);
+	struct xe_tlb_inval_batch batch;
 	struct xe_device *xe = vm->xe;
 	struct drm_gpusvm_range *r, *first;
 	struct xe_tile *tile;
@@ -276,8 +278,10 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
 
 	xe_device_wmb(xe);
 
-	err = xe_vm_range_tilemask_tlb_inval(vm, adj_start, adj_end, tile_mask);
-	WARN_ON_ONCE(err);
+	err = xe_tlb_inval_range_tilemask_submit(xe, vm->usm.asid, adj_start, adj_end,
+						 tile_mask, &batch);
+	if (!WARN_ON_ONCE(err))
+		xe_tlb_inval_batch_wait(&batch);
 
 range_notifier_event_end:
 	r = first;
@@ -747,7 +751,7 @@ static u64 block_offset_to_pfn(struct drm_pagemap *dpagemap, u64 offset)
 	return PHYS_PFN(offset + xpagemap->hpa_base);
 }
 
-static struct drm_buddy *vram_to_buddy(struct xe_vram_region *vram)
+static struct gpu_buddy *vram_to_buddy(struct xe_vram_region *vram)
 {
 	return &vram->ttm.mm;
 }
@@ -758,17 +762,17 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati
 	struct xe_bo *bo = to_xe_bo(devmem_allocation);
 	struct ttm_resource *res = bo->ttm.resource;
 	struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	int j = 0;
 
 	list_for_each_entry(block, blocks, link) {
 		struct xe_vram_region *vr = block->private;
-		struct drm_buddy *buddy = vram_to_buddy(vr);
+		struct gpu_buddy *buddy = vram_to_buddy(vr);
 		u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap,
-						    drm_buddy_block_offset(block));
+						    gpu_buddy_block_offset(block));
 		int i;
 
-		for (i = 0; i < drm_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i)
+		for (i = 0; i < gpu_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i)
 			pfn[j++] = block_pfn + i;
 	}
 
@@ -1033,7 +1037,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 	struct dma_fence *pre_migrate_fence = NULL;
 	struct xe_device *xe = vr->xe;
 	struct device *dev = xe->drm.dev;
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	struct xe_validation_ctx vctx;
 	struct list_head *blocks;
 	struct drm_exec exec;
diff --git a/drivers/gpu/drm/xe/xe_tile.h b/drivers/gpu/drm/xe/xe_tile.h
index 734132eddda5c..a2a2c0e936b8e 100644
--- a/drivers/gpu/drm/xe/xe_tile.h
+++ b/drivers/gpu/drm/xe/xe_tile.h
@@ -6,10 +6,10 @@
 #ifndef _XE_TILE_H_
 #define _XE_TILE_H_
 
-#include "xe_device_types.h"
+#include "xe_tile_types.h"
 
+struct xe_device;
 struct xe_pagemap;
-struct xe_tile;
 
 int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id);
 int xe_tile_init_noalloc(struct xe_tile *tile);
diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_vf.c b/drivers/gpu/drm/xe/xe_tile_sriov_vf.c
index c9bac2cfdd044..24293521e0904 100644
--- a/drivers/gpu/drm/xe/xe_tile_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_tile_sriov_vf.c
@@ -14,173 +14,12 @@
 #include "xe_tile_sriov_vf.h"
 #include "xe_wopcm.h"
 
-static int vf_init_ggtt_balloons(struct xe_tile *tile)
-{
-	struct xe_ggtt *ggtt = tile->mem.ggtt;
-
-	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
-
-	tile->sriov.vf.ggtt_balloon[0] = xe_ggtt_node_init(ggtt);
-	if (IS_ERR(tile->sriov.vf.ggtt_balloon[0]))
-		return PTR_ERR(tile->sriov.vf.ggtt_balloon[0]);
-
-	tile->sriov.vf.ggtt_balloon[1] = xe_ggtt_node_init(ggtt);
-	if (IS_ERR(tile->sriov.vf.ggtt_balloon[1])) {
-		xe_ggtt_node_fini(tile->sriov.vf.ggtt_balloon[0]);
-		return PTR_ERR(tile->sriov.vf.ggtt_balloon[1]);
-	}
-
-	return 0;
-}
-
-/**
- * xe_tile_sriov_vf_balloon_ggtt_locked - Insert balloon nodes to limit used GGTT address range.
- * @tile: the &xe_tile struct instance
- *
- * Return: 0 on success or a negative error code on failure.
- */
-static int xe_tile_sriov_vf_balloon_ggtt_locked(struct xe_tile *tile)
-{
-	u64 ggtt_base = tile->sriov.vf.self_config.ggtt_base;
-	u64 ggtt_size = tile->sriov.vf.self_config.ggtt_size;
-	struct xe_device *xe = tile_to_xe(tile);
-	u64 wopcm = xe_wopcm_size(xe);
-	u64 start, end;
-	int err;
-
-	xe_tile_assert(tile, IS_SRIOV_VF(xe));
-	xe_tile_assert(tile, ggtt_size);
-	lockdep_assert_held(&tile->mem.ggtt->lock);
-
-	/*
-	 * VF can only use part of the GGTT as allocated by the PF:
-	 *
-	 *      WOPCM                                  GUC_GGTT_TOP
-	 *      |<------------ Total GGTT size ------------------>|
-	 *
-	 *           VF GGTT base -->|<- size ->|
-	 *
-	 *      +--------------------+----------+-----------------+
-	 *      |////////////////////|   block  |\\\\\\\\\\\\\\\\\|
-	 *      +--------------------+----------+-----------------+
-	 *
-	 *      |<--- balloon[0] --->|<-- VF -->|<-- balloon[1] ->|
-	 */
-
-	if (ggtt_base < wopcm || ggtt_base > GUC_GGTT_TOP ||
-	    ggtt_size > GUC_GGTT_TOP - ggtt_base) {
-		xe_sriov_err(xe, "tile%u: Invalid GGTT configuration: %#llx-%#llx\n",
-			     tile->id, ggtt_base, ggtt_base + ggtt_size - 1);
-		return -ERANGE;
-	}
-
-	start = wopcm;
-	end = ggtt_base;
-	if (end != start) {
-		err = xe_ggtt_node_insert_balloon_locked(tile->sriov.vf.ggtt_balloon[0],
-							 start, end);
-		if (err)
-			return err;
-	}
-
-	start = ggtt_base + ggtt_size;
-	end = GUC_GGTT_TOP;
-	if (end != start) {
-		err = xe_ggtt_node_insert_balloon_locked(tile->sriov.vf.ggtt_balloon[1],
-							 start, end);
-		if (err) {
-			xe_ggtt_node_remove_balloon_locked(tile->sriov.vf.ggtt_balloon[0]);
-			return err;
-		}
-	}
-
-	return 0;
-}
-
-static int vf_balloon_ggtt(struct xe_tile *tile)
-{
-	struct xe_ggtt *ggtt = tile->mem.ggtt;
-	int err;
-
-	mutex_lock(&ggtt->lock);
-	err = xe_tile_sriov_vf_balloon_ggtt_locked(tile);
-	mutex_unlock(&ggtt->lock);
-
-	return err;
-}
-
-/**
- * xe_tile_sriov_vf_deballoon_ggtt_locked - Remove balloon nodes.
- * @tile: the &xe_tile struct instance
- */
-void xe_tile_sriov_vf_deballoon_ggtt_locked(struct xe_tile *tile)
-{
-	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
-
-	xe_ggtt_node_remove_balloon_locked(tile->sriov.vf.ggtt_balloon[1]);
-	xe_ggtt_node_remove_balloon_locked(tile->sriov.vf.ggtt_balloon[0]);
-}
-
-static void vf_deballoon_ggtt(struct xe_tile *tile)
-{
-	mutex_lock(&tile->mem.ggtt->lock);
-	xe_tile_sriov_vf_deballoon_ggtt_locked(tile);
-	mutex_unlock(&tile->mem.ggtt->lock);
-}
-
-static void vf_fini_ggtt_balloons(struct xe_tile *tile)
-{
-	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
-
-	xe_ggtt_node_fini(tile->sriov.vf.ggtt_balloon[1]);
-	xe_ggtt_node_fini(tile->sriov.vf.ggtt_balloon[0]);
-}
-
-static void cleanup_ggtt(struct drm_device *drm, void *arg)
-{
-	struct xe_tile *tile = arg;
-
-	vf_deballoon_ggtt(tile);
-	vf_fini_ggtt_balloons(tile);
-}
-
-/**
- * xe_tile_sriov_vf_prepare_ggtt - Prepare a VF's GGTT configuration.
- * @tile: the &xe_tile
- *
- * This function is for VF use only.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int xe_tile_sriov_vf_prepare_ggtt(struct xe_tile *tile)
-{
-	struct xe_device *xe = tile_to_xe(tile);
-	int err;
-
-	err = vf_init_ggtt_balloons(tile);
-	if (err)
-		return err;
-
-	err = vf_balloon_ggtt(tile);
-	if (err) {
-		vf_fini_ggtt_balloons(tile);
-		return err;
-	}
-
-	return drmm_add_action_or_reset(&xe->drm, cleanup_ggtt, tile);
-}
-
 /**
  * DOC: GGTT nodes shifting during VF post-migration recovery
  *
  * The first fixup applied to the VF KMD structures as part of post-migration
  * recovery is shifting nodes within &xe_ggtt instance. The nodes are moved
  * from range previously assigned to this VF, into newly provisioned area.
- * The changes include balloons, which are resized accordingly.
- *
- * The balloon nodes are there to eliminate unavailable ranges from use: one
- * reserves the GGTT area below the range for current VF, and another one
- * reserves area above.
  *
  * Below is a GGTT layout of example VF, with a certain address range assigned to
  * said VF, and inaccessible areas above and below:
@@ -198,10 +37,6 @@ int xe_tile_sriov_vf_prepare_ggtt(struct xe_tile *tile)
  *
  *  |<------- inaccessible for VF ------->|<VF owned>|<-- inaccessible for VF ->|
  *
- * GGTT nodes used for tracking allocations:
- *
- *      |<---------- balloon ------------>|<- nodes->|<----- balloon ------>|
- *
  * After the migration, GGTT area assigned to the VF might have shifted, either
  * to lower or to higher address. But we expect the total size and extra areas to
  * be identical, as migration can only happen between matching platforms.
@@ -219,37 +54,12 @@ int xe_tile_sriov_vf_prepare_ggtt(struct xe_tile *tile)
  * So the VF has a new slice of GGTT assigned, and during migration process, the
  * memory content was copied to that new area. But the &xe_ggtt nodes are still
  * tracking allocations using the old addresses. The nodes within VF owned area
- * have to be shifted, and balloon nodes need to be resized to properly mask out
- * areas not owned by the VF.
- *
- * Fixed &xe_ggtt nodes used for tracking allocations:
- *
- *     |<------ balloon ------>|<- nodes->|<----------- balloon ----------->|
+ * have to be shifted, and the start offset for GGTT adjusted.
  *
- * Due to use of GPU profiles, we do not expect the old and new GGTT ares to
+ * Due to use of GPU profiles, we do not expect the old and new GGTT areas to
  * overlap; but our node shifting will fix addresses properly regardless.
  */
 
-/**
- * xe_tile_sriov_vf_fixup_ggtt_nodes_locked - Shift GGTT allocations to match assigned range.
- * @tile: the &xe_tile struct instance
- * @shift: the shift value
- *
- * Since Global GTT is not virtualized, each VF has an assigned range
- * within the global space. This range might have changed during migration,
- * which requires all memory addresses pointing to GGTT to be shifted.
- */
-void xe_tile_sriov_vf_fixup_ggtt_nodes_locked(struct xe_tile *tile, s64 shift)
-{
-	struct xe_ggtt *ggtt = tile->mem.ggtt;
-
-	lockdep_assert_held(&ggtt->lock);
-
-	xe_tile_sriov_vf_deballoon_ggtt_locked(tile);
-	xe_ggtt_shift_nodes_locked(ggtt, shift);
-	xe_tile_sriov_vf_balloon_ggtt_locked(tile);
-}
-
 /**
  * xe_tile_sriov_vf_lmem - VF LMEM configuration.
  * @tile: the &xe_tile
@@ -330,7 +140,7 @@ u64 xe_tile_sriov_vf_ggtt_base(struct xe_tile *tile)
 
 	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
 
-	return config->ggtt_base;
+	return READ_ONCE(config->ggtt_base);
 }
 
 /**
@@ -346,5 +156,5 @@ void xe_tile_sriov_vf_ggtt_base_store(struct xe_tile *tile, u64 ggtt_base)
 
 	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
 
-	config->ggtt_base = ggtt_base;
+	WRITE_ONCE(config->ggtt_base, ggtt_base);
 }
diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_vf.h b/drivers/gpu/drm/xe/xe_tile_sriov_vf.h
index 749f41504883c..f2bbc4fc57347 100644
--- a/drivers/gpu/drm/xe/xe_tile_sriov_vf.h
+++ b/drivers/gpu/drm/xe/xe_tile_sriov_vf.h
@@ -10,9 +10,6 @@
 
 struct xe_tile;
 
-int xe_tile_sriov_vf_prepare_ggtt(struct xe_tile *tile);
-void xe_tile_sriov_vf_deballoon_ggtt_locked(struct xe_tile *tile);
-void xe_tile_sriov_vf_fixup_ggtt_nodes_locked(struct xe_tile *tile, s64 shift);
 u64 xe_tile_sriov_vf_ggtt(struct xe_tile *tile);
 void xe_tile_sriov_vf_ggtt_store(struct xe_tile *tile, u64 ggtt_size);
 u64 xe_tile_sriov_vf_ggtt_base(struct xe_tile *tile);
diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs.c b/drivers/gpu/drm/xe/xe_tile_sysfs.c
index 0bfd28422dc21..510aa0ac44287 100644
--- a/drivers/gpu/drm/xe/xe_tile_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_tile_sysfs.c
@@ -7,8 +7,8 @@
 #include <linux/sysfs.h>
 #include <drm/drm_managed.h>
 
+#include "xe_device_types.h"
 #include "xe_pm.h"
-#include "xe_tile.h"
 #include "xe_tile_sysfs.h"
 #include "xe_vram_freq.h"
 
diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h
new file mode 100644
index 0000000000000..33932fd547d71
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_tile_types.h
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022-2026 Intel Corporation
+ */
+
+#ifndef _XE_TILE_TYPES_H_
+#define _XE_TILE_TYPES_H_
+
+#include <linux/mutex_types.h>
+#include <linux/workqueue_types.h>
+
+#include "xe_lmtt_types.h"
+#include "xe_memirq_types.h"
+#include "xe_mert.h"
+#include "xe_mmio_types.h"
+#include "xe_tile_sriov_vf_types.h"
+
+#define tile_to_xe(tile__)								\
+	_Generic(tile__,								\
+		 const struct xe_tile * : (const struct xe_device *)((tile__)->xe),	\
+		 struct xe_tile * : (tile__)->xe)
+
+/**
+ * struct xe_tile - hardware tile structure
+ *
+ * From a driver perspective, a "tile" is effectively a complete GPU, containing
+ * an SGunit, 1-2 GTs, and (for discrete platforms) VRAM.
+ *
+ * Multi-tile platforms effectively bundle multiple GPUs behind a single PCI
+ * device and designate one "root" tile as being responsible for external PCI
+ * communication.  PCI BAR0 exposes the GGTT and MMIO register space for each
+ * tile in a stacked layout, and PCI BAR2 exposes the local memory associated
+ * with each tile similarly.  Device-wide interrupts can be enabled/disabled
+ * at the root tile, and the MSTR_TILE_INTR register will report which tiles
+ * have interrupts that need servicing.
+ */
+struct xe_tile {
+	/** @xe: Backpointer to tile's PCI device */
+	struct xe_device *xe;
+
+	/** @id: ID of the tile */
+	u8 id;
+
+	/**
+	 * @primary_gt: Primary GT
+	 */
+	struct xe_gt *primary_gt;
+
+	/**
+	 * @media_gt: Media GT
+	 *
+	 * Only present on devices with media version >= 13.
+	 */
+	struct xe_gt *media_gt;
+
+	/**
+	 * @mmio: MMIO info for a tile.
+	 *
+	 * Each tile has its own 16MB space in BAR0, laid out as:
+	 * * 0-4MB: registers
+	 * * 4MB-8MB: reserved
+	 * * 8MB-16MB: global GTT
+	 */
+	struct xe_mmio mmio;
+
+	/** @mem: memory management info for tile */
+	struct {
+		/**
+		 * @mem.kernel_vram: kernel-dedicated VRAM info for tile.
+		 *
+		 * Although VRAM is associated with a specific tile, it can
+		 * still be accessed by all tiles' GTs.
+		 */
+		struct xe_vram_region *kernel_vram;
+
+		/**
+		 * @mem.vram: general purpose VRAM info for tile.
+		 *
+		 * Although VRAM is associated with a specific tile, it can
+		 * still be accessed by all tiles' GTs.
+		 */
+		struct xe_vram_region *vram;
+
+		/** @mem.ggtt: Global graphics translation table */
+		struct xe_ggtt *ggtt;
+
+		/**
+		 * @mem.kernel_bb_pool: Pool from which batchbuffers are allocated.
+		 *
+		 * Media GT shares a pool with its primary GT.
+		 */
+		struct xe_sa_manager *kernel_bb_pool;
+
+		/**
+		 * @mem.reclaim_pool: Pool for PRLs allocated.
+		 *
+		 * Only main GT has page reclaim list allocations.
+		 */
+		struct xe_sa_manager *reclaim_pool;
+	} mem;
+
+	/** @sriov: tile level virtualization data */
+	union {
+		struct {
+			/** @sriov.pf.lmtt: Local Memory Translation Table. */
+			struct xe_lmtt lmtt;
+		} pf;
+		struct {
+			/** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */
+			struct xe_ggtt_node *ggtt_balloon[2];
+			/** @sriov.vf.self_config: VF configuration data */
+			struct xe_tile_sriov_vf_selfconfig self_config;
+		} vf;
+	} sriov;
+
+	/** @memirq: Memory Based Interrupts. */
+	struct xe_memirq memirq;
+
+	/** @csc_hw_error_work: worker to report CSC HW errors */
+	struct work_struct csc_hw_error_work;
+
+	/** @pcode: tile's PCODE */
+	struct {
+		/** @pcode.lock: protecting tile's PCODE mailbox data */
+		struct mutex lock;
+	} pcode;
+
+	/** @migrate: Migration helper for vram blits and clearing */
+	struct xe_migrate *migrate;
+
+	/** @sysfs: sysfs' kobj used by xe_tile_sysfs */
+	struct kobject *sysfs;
+
+	/** @debugfs: debugfs directory associated with this tile */
+	struct dentry *debugfs;
+
+	/** @mert: MERT-related data */
+	struct xe_mert mert;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
index f6d522dbdf86a..10dcd4abb00f0 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
@@ -41,11 +41,14 @@ static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
 static void
 xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
 {
+	struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
 	bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);
 
 	lockdep_assert_held(&fence->tlb_inval->pending_lock);
 
 	list_del(&fence->link);
+	if (list_empty(&tlb_inval->pending_fences))
+		cancel_delayed_work(&tlb_inval->fence_tdr);
 	trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence);
 	xe_tlb_inval_fence_fini(fence);
 	dma_fence_signal(&fence->base);
@@ -111,6 +114,16 @@ static void tlb_inval_fini(struct drm_device *drm, void *arg)
 	xe_tlb_inval_reset(tlb_inval);
 }
 
+static void primelockdep(struct xe_tlb_inval *tlb_inval)
+{
+	if (!IS_ENABLED(CONFIG_LOCKDEP))
+		return;
+
+	fs_reclaim_acquire(GFP_KERNEL);
+	might_lock(&tlb_inval->seqno_lock);
+	fs_reclaim_release(GFP_KERNEL);
+}
+
 /**
  * xe_gt_tlb_inval_init_early() - Initialize TLB invalidation state
  * @gt: GT structure
@@ -137,6 +150,8 @@ int xe_gt_tlb_inval_init_early(struct xe_gt *gt)
 	if (err)
 		return err;
 
+	primelockdep(tlb_inval);
+
 	tlb_inval->job_wq = drmm_alloc_ordered_workqueue(&xe->drm,
 							 "gt-tbl-inval-job-wq",
 							 WQ_MEM_RECLAIM);
@@ -453,3 +468,105 @@ void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
 		dma_fence_get(&fence->base);
 	fence->tlb_inval = tlb_inval;
 }
+
+/**
+ * xe_tlb_inval_idle() - Initialize TLB invalidation is idle
+ * @tlb_inval: TLB invalidation client
+ *
+ * Check the TLB invalidation seqno to determine if it is idle (i.e., no TLB
+ * invalidations are in flight). Expected to be called in the backend after the
+ * fence has been added to the pending list, and takes this into account.
+ *
+ * Return: True if TLB invalidation client is idle, False otherwise
+ */
+bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval)
+{
+	lockdep_assert_held(&tlb_inval->seqno_lock);
+
+	guard(spinlock_irq)(&tlb_inval->pending_lock);
+	return list_is_singular(&tlb_inval->pending_fences);
+}
+
+/**
+ * xe_tlb_inval_batch_wait() - Wait for all fences in a TLB invalidation batch
+ * @batch: Batch of TLB invalidation fences to wait on
+ *
+ * Waits for every fence in @batch to signal, then resets @batch so it can be
+ * reused for a subsequent invalidation.
+ */
+void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch)
+{
+	struct xe_tlb_inval_fence *fence = &batch->fence[0];
+	unsigned int i;
+
+	for (i = 0; i < batch->num_fences; ++i)
+		xe_tlb_inval_fence_wait(fence++);
+
+	batch->num_fences = 0;
+}
+
+/**
+ * xe_tlb_inval_range_tilemask_submit() - Submit TLB invalidations for an
+ * address range on a tile mask
+ * @xe: The xe device
+ * @asid: Address space ID
+ * @start: start address
+ * @end: end address
+ * @tile_mask: mask for which gt's issue tlb invalidation
+ * @batch: Batch of tlb invalidate fences
+ *
+ * Issue a range based TLB invalidation for gt's in tilemask
+ * If the function returns an error, there is no need to call
+ * xe_tlb_inval_batch_wait() on @batch.
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
+int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid,
+				       u64 start, u64 end, u8 tile_mask,
+				       struct xe_tlb_inval_batch *batch)
+{
+	struct xe_tlb_inval_fence *fence = &batch->fence[0];
+	struct xe_tile *tile;
+	u32 fence_id = 0;
+	u8 id;
+	int err;
+
+	batch->num_fences = 0;
+	if (!tile_mask)
+		return 0;
+
+	for_each_tile(tile, xe, id) {
+		if (!(tile_mask & BIT(id)))
+			continue;
+
+		xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
+					&fence[fence_id], true);
+
+		err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
+					 &fence[fence_id], start, end,
+					 asid, NULL);
+		if (err)
+			goto wait;
+		++fence_id;
+
+		if (!tile->media_gt)
+			continue;
+
+		xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
+					&fence[fence_id], true);
+
+		err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
+					 &fence[fence_id], start, end,
+					 asid, NULL);
+		if (err)
+			goto wait;
+		++fence_id;
+	}
+
+wait:
+	batch->num_fences = fence_id;
+	if (err)
+		xe_tlb_inval_batch_wait(batch);
+
+	return err;
+}
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h
index 858d0690f995f..a76b7823a5f2e 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.h
@@ -43,4 +43,12 @@ xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence)
 
 void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno);
 
+bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval);
+
+int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid,
+				       u64 start, u64 end, u8 tile_mask,
+				       struct xe_tlb_inval_batch *batch);
+
+void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch);
+
 #endif	/* _XE_TLB_INVAL_ */
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
index 3b089f90f0021..3d1797d186fd7 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
@@ -9,6 +9,8 @@
 #include <linux/workqueue.h>
 #include <linux/dma-fence.h>
 
+#include "xe_device_types.h"
+
 struct drm_suballoc;
 struct xe_tlb_inval;
 
@@ -132,4 +134,16 @@ struct xe_tlb_inval_fence {
 	ktime_t inval_time;
 };
 
+/**
+ * struct xe_tlb_inval_batch - Batch of TLB invalidation fences
+ *
+ * Holds one fence per GT covered by a TLB invalidation request.
+ */
+struct xe_tlb_inval_batch {
+	/** @fence: per-GT TLB invalidation fences */
+	struct xe_tlb_inval_fence fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
+	/** @num_fences: number of valid entries in @fence */
+	unsigned int num_fences;
+};
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 344769d939e05..5fd0d5506a7ea 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -6,6 +6,7 @@
 
 #include <drm/drm_managed.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_buddy.h>
 
 #include <drm/ttm/ttm_placement.h>
 #include <drm/ttm/ttm_range_manager.h>
@@ -16,16 +17,16 @@
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vram_types.h"
 
-static inline struct drm_buddy_block *
+static inline struct gpu_buddy_block *
 xe_ttm_vram_mgr_first_block(struct list_head *list)
 {
-	return list_first_entry_or_null(list, struct drm_buddy_block, link);
+	return list_first_entry_or_null(list, struct gpu_buddy_block, link);
 }
 
-static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
+static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm,
 						    struct list_head *head)
 {
-	struct drm_buddy_block *block;
+	struct gpu_buddy_block *block;
 	u64 start, size;
 
 	block = xe_ttm_vram_mgr_first_block(head);
@@ -33,12 +34,12 @@ static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
 		return false;
 
 	while (head != block->link.next) {
-		start = drm_buddy_block_offset(block);
-		size = drm_buddy_block_size(mm, block);
+		start = gpu_buddy_block_offset(block);
+		size = gpu_buddy_block_size(mm, block);
 
-		block = list_entry(block->link.next, struct drm_buddy_block,
+		block = list_entry(block->link.next, struct gpu_buddy_block,
 				   link);
-		if (start + size != drm_buddy_block_offset(block))
+		if (start + size != gpu_buddy_block_offset(block))
 			return false;
 	}
 
@@ -52,7 +53,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 {
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
 	struct xe_ttm_vram_mgr_resource *vres;
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 	u64 size, min_page_size;
 	unsigned long lpfn;
 	int err;
@@ -79,10 +80,13 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 	INIT_LIST_HEAD(&vres->blocks);
 
 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
-		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
+		vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
+
+	if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
+		vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
 
 	if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)
-		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
+		vres->flags |= GPU_BUDDY_RANGE_ALLOCATION;
 
 	if (WARN_ON(!vres->base.size)) {
 		err = -EINVAL;
@@ -110,35 +114,22 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 		goto error_unlock;
 	}
 
-	if (place->fpfn + (size >> PAGE_SHIFT) != lpfn &&
-	    place->flags & TTM_PL_FLAG_CONTIGUOUS) {
-		size = roundup_pow_of_two(size);
-		min_page_size = size;
-
-		lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn);
-	}
-
-	err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
+	err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
 				     (u64)lpfn << PAGE_SHIFT, size,
 				     min_page_size, &vres->blocks, vres->flags);
 	if (err)
 		goto error_unlock;
 
-	if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
-		if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks))
-			size = vres->base.size;
-	}
-
 	if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
 		vres->used_visible_size = size;
 	} else {
-		struct drm_buddy_block *block;
+		struct gpu_buddy_block *block;
 
 		list_for_each_entry(block, &vres->blocks, link) {
-			u64 start = drm_buddy_block_offset(block);
+			u64 start = gpu_buddy_block_offset(block);
 
 			if (start < mgr->visible_size) {
-				u64 end = start + drm_buddy_block_size(mm, block);
+				u64 end = start + gpu_buddy_block_size(mm, block);
 
 				vres->used_visible_size +=
 					min(end, mgr->visible_size) - start;
@@ -158,11 +149,11 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 	 * the object.
 	 */
 	if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) {
-		struct drm_buddy_block *block = list_first_entry(&vres->blocks,
+		struct gpu_buddy_block *block = list_first_entry(&vres->blocks,
 								 typeof(*block),
 								 link);
 
-		vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT;
+		vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 	} else {
 		vres->base.start = XE_BO_INVALID_OFFSET;
 	}
@@ -184,10 +175,10 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
 	struct xe_ttm_vram_mgr_resource *vres =
 		to_xe_ttm_vram_mgr_resource(res);
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 
 	mutex_lock(&mgr->lock);
-	drm_buddy_free_list(mm, &vres->blocks, 0);
+	gpu_buddy_free_list(mm, &vres->blocks, 0);
 	mgr->visible_avail += vres->used_visible_size;
 	mutex_unlock(&mgr->lock);
 
@@ -200,7 +191,7 @@ static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
 				  struct drm_printer *printer)
 {
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
-	struct drm_buddy *mm = &mgr->mm;
+	struct gpu_buddy *mm = &mgr->mm;
 
 	mutex_lock(&mgr->lock);
 	drm_printf(printer, "default_page_size: %lluKiB\n",
@@ -223,8 +214,8 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
 	struct xe_ttm_vram_mgr_resource *vres =
 		to_xe_ttm_vram_mgr_resource(res);
-	struct drm_buddy *mm = &mgr->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &mgr->mm;
+	struct gpu_buddy_block *block;
 
 	if (!place->fpfn && !place->lpfn)
 		return true;
@@ -234,9 +225,9 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
 
 	list_for_each_entry(block, &vres->blocks, link) {
 		unsigned long fpfn =
-			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+			gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 		unsigned long lpfn = fpfn +
-			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+			(gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
 
 		if (place->fpfn < lpfn && place->lpfn > fpfn)
 			return true;
@@ -253,8 +244,8 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
 	struct xe_ttm_vram_mgr_resource *vres =
 		to_xe_ttm_vram_mgr_resource(res);
-	struct drm_buddy *mm = &mgr->mm;
-	struct drm_buddy_block *block;
+	struct gpu_buddy *mm = &mgr->mm;
+	struct gpu_buddy_block *block;
 
 	if (!place->fpfn && !place->lpfn)
 		return true;
@@ -264,9 +255,9 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
 
 	list_for_each_entry(block, &vres->blocks, link) {
 		unsigned long fpfn =
-			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+			gpu_buddy_block_offset(block) >> PAGE_SHIFT;
 		unsigned long lpfn = fpfn +
-			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+			(gpu_buddy_block_size(mm, block) >> PAGE_SHIFT);
 
 		if (fpfn < place->fpfn || lpfn > place->lpfn)
 			return false;
@@ -296,7 +287,7 @@ static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
 
 	WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);
 
-	drm_buddy_fini(&mgr->mm);
+	gpu_buddy_fini(&mgr->mm);
 
 	ttm_resource_manager_cleanup(&mgr->manager);
 
@@ -327,7 +318,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 	mgr->visible_avail = io_size;
 
 	ttm_resource_manager_init(man, &xe->ttm, size);
-	err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
+	err = gpu_buddy_init(&mgr->mm, man->size, default_page_size);
 	if (err)
 		return err;
 
@@ -375,7 +366,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 	if (!*sgt)
 		return -ENOMEM;
 
-	/* Determine the number of DRM_BUDDY blocks to export */
+	/* Determine the number of GPU_BUDDY blocks to export */
 	xe_res_first(res, offset, length, &cursor);
 	while (cursor.remaining) {
 		num_entries++;
@@ -392,10 +383,10 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 		sg->length = 0;
 
 	/*
-	 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
-	 * @note: Use iterator api to get first the DRM_BUDDY block
+	 * Walk down GPU_BUDDY blocks to populate scatterlist nodes
+	 * @note: Use iterator api to get first the GPU_BUDDY block
 	 * and the number of bytes from it. Access the following
-	 * DRM_BUDDY block(s) if more buffer needs to exported
+	 * GPU_BUDDY block(s) if more buffer needs to exported
 	 */
 	xe_res_first(res, offset, length, &cursor);
 	for_each_sgtable_sg((*sgt), sg, i) {
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
index a71e14818ec2a..9106da056b490 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
@@ -6,7 +6,7 @@
 #ifndef _XE_TTM_VRAM_MGR_TYPES_H_
 #define _XE_TTM_VRAM_MGR_TYPES_H_
 
-#include <drm/drm_buddy.h>
+#include <linux/gpu_buddy.h>
 #include <drm/ttm/ttm_device.h>
 
 /**
@@ -18,7 +18,7 @@ struct xe_ttm_vram_mgr {
 	/** @manager: Base TTM resource manager */
 	struct ttm_resource_manager manager;
 	/** @mm: DRM buddy allocator which manages the VRAM */
-	struct drm_buddy mm;
+	struct gpu_buddy mm;
 	/** @visible_size: Proped size of the CPU visible portion */
 	u64 visible_size;
 	/** @visible_avail: CPU visible portion still unallocated */
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index 5766fa7742d31..f8de6a4bf1897 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -10,10 +10,12 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 
+#include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
 #include "xe_gt_types.h"
 #include "xe_platform_types.h"
 #include "xe_rtp.h"
+#include "xe_sriov.h"
 
 #undef XE_REG_MCR
 #define XE_REG_MCR(...)     XE_REG(__VA_ARGS__, .mcr = 1)
@@ -31,12 +33,12 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 	/* Xe2 */
 
 	{ XE_RTP_NAME("Tuning: L3 cache"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499)),
 	  XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
 	{ XE_RTP_NAME("Tuning: L3 cache - media"),
-	  XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(2000, 3499)),
 	  XE_RTP_ACTIONS(FIELD_SET(XE2LPM_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
@@ -52,7 +54,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 			 SET(XE2LPM_CCCHKNREG1, L3CMPCTRL))
 	},
 	{ XE_RTP_NAME("Tuning: Enable compressible partial write overfetch in L3"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499)),
 	  XE_RTP_ACTIONS(SET(L3SQCREG3, COMPPWOVERFETCHEN))
 	},
 	{ XE_RTP_NAME("Tuning: Enable compressible partial write overfetch in L3 - media"),
@@ -89,6 +91,15 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 	  XE_RTP_RULES(MEDIA_VERSION(2000)),
 	  XE_RTP_ACTIONS(SET(XE2LPM_SCRATCH3_LBCF, RWFLUSHALLEN))
 	},
+
+	/* Xe3p */
+
+	{ XE_RTP_NAME("Tuning: Set STLB Bank Hash Mode to 4KB"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED),
+		       IS_INTEGRATED),
+	  XE_RTP_ACTIONS(FIELD_SET(XEHP_GAMSTLB_CTRL, BANK_HASH_MODE,
+				   BANK_HASH_4KB_MODE))
+	},
 };
 
 static const struct xe_rtp_entry_sr engine_tunings[] = {
@@ -107,9 +118,20 @@ static const struct xe_rtp_entry_sr engine_tunings[] = {
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(RT_CTRL, DIS_NULL_QUERY))
 	},
+	{ XE_RTP_NAME("Tuning: disable HW reporting of ctx switch to GHWSP"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
+			     GHWSP_CSB_REPORT_DIS,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
 };
 
 static const struct xe_rtp_entry_sr lrc_tunings[] = {
+	{ XE_RTP_NAME("Tuning: Windower HW Filtering"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, HW_FILTERING))
+	},
+
 	/* DG2 */
 
 	{ XE_RTP_NAME("Tuning: L3 cache"),
@@ -184,7 +206,8 @@ void xe_tuning_process_gt(struct xe_gt *gt)
 	xe_rtp_process_ctx_enable_active_tracking(&ctx,
 						  gt->tuning_active.gt,
 						  ARRAY_SIZE(gt_tunings));
-	xe_rtp_process_to_sr(&ctx, gt_tunings, ARRAY_SIZE(gt_tunings), &gt->reg_sr);
+	xe_rtp_process_to_sr(&ctx, gt_tunings, ARRAY_SIZE(gt_tunings),
+			     &gt->reg_sr, false);
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_gt);
 
@@ -196,7 +219,7 @@ void xe_tuning_process_engine(struct xe_hw_engine *hwe)
 						  hwe->gt->tuning_active.engine,
 						  ARRAY_SIZE(engine_tunings));
 	xe_rtp_process_to_sr(&ctx, engine_tunings, ARRAY_SIZE(engine_tunings),
-			     &hwe->reg_sr);
+			     &hwe->reg_sr, false);
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_engine);
 
@@ -215,7 +238,8 @@ void xe_tuning_process_lrc(struct xe_hw_engine *hwe)
 	xe_rtp_process_ctx_enable_active_tracking(&ctx,
 						  hwe->gt->tuning_active.lrc,
 						  ARRAY_SIZE(lrc_tunings));
-	xe_rtp_process_to_sr(&ctx, lrc_tunings, ARRAY_SIZE(lrc_tunings), &hwe->reg_lrc);
+	xe_rtp_process_to_sr(&ctx, lrc_tunings, ARRAY_SIZE(lrc_tunings),
+			     &hwe->reg_lrc, true);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
index 3f63c2a7e86de..75091bde0d50c 100644
--- a/drivers/gpu/drm/xe/xe_uc.c
+++ b/drivers/gpu/drm/xe/xe_uc.c
@@ -13,6 +13,7 @@
 #include "xe_gt_sriov_vf.h"
 #include "xe_guc.h"
 #include "xe_guc_pc.h"
+#include "xe_guc_rc.h"
 #include "xe_guc_engine_activity.h"
 #include "xe_huc.h"
 #include "xe_sriov.h"
@@ -156,23 +157,19 @@ static int vf_uc_load_hw(struct xe_uc *uc)
 
 	err = xe_gt_sriov_vf_connect(uc_to_gt(uc));
 	if (err)
-		goto err_out;
+		return err;
 
 	uc->guc.submission_state.enabled = true;
 
 	err = xe_guc_opt_in_features_enable(&uc->guc);
 	if (err)
-		goto err_out;
+		return err;
 
 	err = xe_gt_record_default_lrcs(uc_to_gt(uc));
 	if (err)
-		goto err_out;
+		return err;
 
 	return 0;
-
-err_out:
-	xe_guc_sanitize(&uc->guc);
-	return err;
 }
 
 /*
@@ -204,15 +201,19 @@ int xe_uc_load_hw(struct xe_uc *uc)
 
 	ret = xe_gt_record_default_lrcs(uc_to_gt(uc));
 	if (ret)
-		goto err_out;
+		return ret;
 
 	ret = xe_guc_post_load_init(&uc->guc);
 	if (ret)
-		goto err_out;
+		return ret;
 
 	ret = xe_guc_pc_start(&uc->guc.pc);
 	if (ret)
-		goto err_out;
+		return ret;
+
+	ret = xe_guc_rc_enable(&uc->guc);
+	if (ret)
+		return ret;
 
 	xe_guc_engine_activity_enable_stats(&uc->guc);
 
@@ -227,10 +228,6 @@ int xe_uc_load_hw(struct xe_uc *uc)
 	xe_gsc_load_start(&uc->gsc);
 
 	return 0;
-
-err_out:
-	xe_guc_sanitize(&uc->guc);
-	return ret;
 }
 
 int xe_uc_reset_prepare(struct xe_uc *uc)
@@ -242,11 +239,6 @@ int xe_uc_reset_prepare(struct xe_uc *uc)
 	return xe_guc_reset_prepare(&uc->guc);
 }
 
-void xe_uc_gucrc_disable(struct xe_uc *uc)
-{
-	XE_WARN_ON(xe_guc_pc_gucrc_disable(&uc->guc.pc));
-}
-
 void xe_uc_stop_prepare(struct xe_uc *uc)
 {
 	xe_gsc_stop_prepare(&uc->gsc);
diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h
index 5398da1a8097c..255a54a8f876c 100644
--- a/drivers/gpu/drm/xe/xe_uc.h
+++ b/drivers/gpu/drm/xe/xe_uc.h
@@ -12,7 +12,6 @@ int xe_uc_init_noalloc(struct xe_uc *uc);
 int xe_uc_init(struct xe_uc *uc);
 int xe_uc_init_post_hwconfig(struct xe_uc *uc);
 int xe_uc_load_hw(struct xe_uc *uc);
-void xe_uc_gucrc_disable(struct xe_uc *uc);
 int xe_uc_reset_prepare(struct xe_uc *uc);
 void xe_uc_runtime_resume(struct xe_uc *uc);
 void xe_uc_runtime_suspend(struct xe_uc *uc);
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index d35bc4989144a..9cebb24902457 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -881,7 +881,7 @@ static int uc_fw_xfer(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags)
 
 	/* Start the DMA */
 	xe_mmio_write32(mmio, DMA_CTRL,
-			_MASKED_BIT_ENABLE(dma_flags | START_DMA));
+			REG_MASKED_FIELD_ENABLE(dma_flags | START_DMA));
 
 	/* Wait for DMA to finish */
 	ret = xe_mmio_wait32(mmio, DMA_CTRL, START_DMA, 0, 100000, &dma_ctrl,
@@ -891,7 +891,7 @@ static int uc_fw_xfer(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags)
 			xe_uc_fw_type_repr(uc_fw->type), dma_ctrl);
 
 	/* Disable the bits once DMA is over */
-	xe_mmio_write32(mmio, DMA_CTRL, _MASKED_BIT_DISABLE(dma_flags));
+	xe_mmio_write32(mmio, DMA_CTRL, REG_MASKED_FIELD_DISABLE(dma_flags));
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c
index e120323c43bcd..6761005c0b900 100644
--- a/drivers/gpu/drm/xe/xe_userptr.c
+++ b/drivers/gpu/drm/xe/xe_userptr.c
@@ -8,8 +8,17 @@
 
 #include <linux/mm.h>
 
+#include "xe_tlb_inval.h"
 #include "xe_trace_bo.h"
 
+static void xe_userptr_assert_in_notifier(struct xe_vm *vm)
+{
+	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
+		       (lockdep_is_held(&vm->lock) &&
+			lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
+			dma_resv_held(xe_vm_resv(vm))));
+}
+
 /**
  * xe_vma_userptr_check_repin() - Advisory check for repin needed
  * @uvma: The userptr vma
@@ -73,18 +82,83 @@ int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
 				    &ctx);
 }
 
-static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma)
+static struct mmu_interval_notifier_finish *
+xe_vma_userptr_do_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma, bool is_deferred)
 {
 	struct xe_userptr *userptr = &uvma->userptr;
 	struct xe_vma *vma = &uvma->vma;
-	struct dma_resv_iter cursor;
-	struct dma_fence *fence;
 	struct drm_gpusvm_ctx ctx = {
 		.in_notifier = true,
 		.read_only = xe_vma_read_only(vma),
 	};
 	long err;
 
+	xe_userptr_assert_in_notifier(vm);
+	if (is_deferred)
+		xe_assert(vm->xe, userptr->finish_inuse && !userptr->tlb_inval_submitted);
+
+	err = dma_resv_wait_timeout(xe_vm_resv(vm),
+				    DMA_RESV_USAGE_BOOKKEEP,
+				    false, MAX_SCHEDULE_TIMEOUT);
+	XE_WARN_ON(err <= 0);
+
+	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
+		if (!userptr->finish_inuse) {
+			/*
+			 * Defer the TLB wait to an extra pass so the caller
+			 * can pipeline TLB flushes across GPUs before waiting
+			 * on any of them.
+			 */
+			xe_assert(vm->xe, !userptr->tlb_inval_submitted);
+			userptr->finish_inuse = true;
+			userptr->tlb_inval_submitted = true;
+			err = xe_vm_invalidate_vma_submit(vma, &userptr->inval_batch);
+			XE_WARN_ON(err);
+			return &userptr->finish;
+		}
+		err = xe_vm_invalidate_vma(vma);
+		XE_WARN_ON(err);
+	}
+
+	if (is_deferred)
+		userptr->finish_inuse = false;
+	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
+			       xe_vma_size(vma) >> PAGE_SHIFT, &ctx);
+	return NULL;
+}
+
+static void
+xe_vma_userptr_complete_tlb_inval(struct xe_vm *vm, struct xe_userptr_vma *uvma)
+{
+	struct xe_userptr *userptr = &uvma->userptr;
+	struct xe_vma *vma = &uvma->vma;
+	struct drm_gpusvm_ctx ctx = {
+		.in_notifier = true,
+		.read_only = xe_vma_read_only(vma),
+	};
+
+	xe_userptr_assert_in_notifier(vm);
+	xe_assert(vm->xe, userptr->finish_inuse);
+	xe_assert(vm->xe, userptr->tlb_inval_submitted);
+
+	xe_tlb_inval_batch_wait(&userptr->inval_batch);
+	userptr->tlb_inval_submitted = false;
+	userptr->finish_inuse = false;
+	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
+			       xe_vma_size(vma) >> PAGE_SHIFT, &ctx);
+}
+
+static struct mmu_interval_notifier_finish *
+xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma)
+{
+	struct xe_userptr *userptr = &uvma->userptr;
+	struct xe_vma *vma = &uvma->vma;
+	struct dma_resv_iter cursor;
+	struct dma_fence *fence;
+	bool signaled = true;
+
+	xe_userptr_assert_in_notifier(vm);
+
 	/*
 	 * Tell exec and rebind worker they need to repin and rebind this
 	 * userptr.
@@ -105,27 +179,31 @@ static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uv
 	 */
 	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
 			    DMA_RESV_USAGE_BOOKKEEP);
-	dma_resv_for_each_fence_unlocked(&cursor, fence)
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
 		dma_fence_enable_sw_signaling(fence);
+		if (signaled && !dma_fence_is_signaled(fence))
+			signaled = false;
+	}
 	dma_resv_iter_end(&cursor);
 
-	err = dma_resv_wait_timeout(xe_vm_resv(vm),
-				    DMA_RESV_USAGE_BOOKKEEP,
-				    false, MAX_SCHEDULE_TIMEOUT);
-	XE_WARN_ON(err <= 0);
+	/*
+	 * Only one caller at a time can use the multi-pass state.
+	 * If it's already in use, or all fences are already signaled,
+	 * proceed directly to invalidation without deferring.
+	 */
+	if (signaled || userptr->finish_inuse)
+		return xe_vma_userptr_do_inval(vm, uvma, false);
 
-	if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
-		err = xe_vm_invalidate_vma(vma);
-		XE_WARN_ON(err);
-	}
+	/* Defer: the notifier core will call invalidate_finish once done. */
+	userptr->finish_inuse = true;
 
-	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
-			       xe_vma_size(vma) >> PAGE_SHIFT, &ctx);
+	return &userptr->finish;
 }
 
-static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
-				   const struct mmu_notifier_range *range,
-				   unsigned long cur_seq)
+static bool xe_vma_userptr_invalidate_start(struct mmu_interval_notifier *mni,
+					    const struct mmu_notifier_range *range,
+					    unsigned long cur_seq,
+					    struct mmu_interval_notifier_finish **p_finish)
 {
 	struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);
 	struct xe_vma *vma = &uvma->vma;
@@ -138,21 +216,48 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 		return false;
 
 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
-	       "NOTIFIER: addr=0x%016llx, range=0x%016llx",
+	       "NOTIFIER PASS1: addr=0x%016llx, range=0x%016llx",
 		xe_vma_start(vma), xe_vma_size(vma));
 
 	down_write(&vm->svm.gpusvm.notifier_lock);
 	mmu_interval_set_seq(mni, cur_seq);
 
-	__vma_userptr_invalidate(vm, uvma);
+	*p_finish = xe_vma_userptr_invalidate_pass1(vm, uvma);
+
 	up_write(&vm->svm.gpusvm.notifier_lock);
-	trace_xe_vma_userptr_invalidate_complete(vma);
+	if (!*p_finish)
+		trace_xe_vma_userptr_invalidate_complete(vma);
 
 	return true;
 }
 
+static void xe_vma_userptr_invalidate_finish(struct mmu_interval_notifier_finish *finish)
+{
+	struct xe_userptr_vma *uvma = container_of(finish, typeof(*uvma), userptr.finish);
+	struct xe_vma *vma = &uvma->vma;
+	struct xe_vm *vm = xe_vma_vm(vma);
+
+	vm_dbg(&xe_vma_vm(vma)->xe->drm,
+	       "NOTIFIER PASS2: addr=0x%016llx, range=0x%016llx",
+		xe_vma_start(vma), xe_vma_size(vma));
+
+	down_write(&vm->svm.gpusvm.notifier_lock);
+	/*
+	 * If a TLB invalidation was previously submitted (deferred from the
+	 * synchronous pass1 fallback), wait for it and unmap pages.
+	 * Otherwise, fences have now completed: invalidate the TLB and unmap.
+	 */
+	if (uvma->userptr.tlb_inval_submitted)
+		xe_vma_userptr_complete_tlb_inval(vm, uvma);
+	else
+		xe_vma_userptr_do_inval(vm, uvma, true);
+	up_write(&vm->svm.gpusvm.notifier_lock);
+	trace_xe_vma_userptr_invalidate_complete(vma);
+}
+
 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
-	.invalidate = vma_userptr_invalidate,
+	.invalidate_start = xe_vma_userptr_invalidate_start,
+	.invalidate_finish = xe_vma_userptr_invalidate_finish,
 };
 
 #if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
@@ -164,6 +269,7 @@ static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
  */
 void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
 {
+	static struct mmu_interval_notifier_finish *finish;
 	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
 
 	/* Protect against concurrent userptr pinning */
@@ -179,7 +285,12 @@ void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
 	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
 				     uvma->userptr.pages.notifier_seq))
 		uvma->userptr.pages.notifier_seq -= 2;
-	__vma_userptr_invalidate(vm, uvma);
+
+	finish = xe_vma_userptr_invalidate_pass1(vm, uvma);
+	if (finish)
+		finish = xe_vma_userptr_do_inval(vm, uvma, true);
+	if (finish)
+		xe_vma_userptr_complete_tlb_inval(vm, uvma);
 }
 #endif
 
diff --git a/drivers/gpu/drm/xe/xe_userptr.h b/drivers/gpu/drm/xe/xe_userptr.h
index ef801234991e6..2a3cd1b5efbb0 100644
--- a/drivers/gpu/drm/xe/xe_userptr.h
+++ b/drivers/gpu/drm/xe/xe_userptr.h
@@ -14,6 +14,8 @@
 
 #include <drm/drm_gpusvm.h>
 
+#include "xe_tlb_inval_types.h"
+
 struct xe_vm;
 struct xe_vma;
 struct xe_userptr_vma;
@@ -56,7 +58,34 @@ struct xe_userptr {
 	 * @notifier: MMU notifier for user pointer (invalidation call back)
 	 */
 	struct mmu_interval_notifier notifier;
-
+	/**
+	 * @finish: MMU notifier finish structure for two-pass invalidation.
+	 * Embedded here to avoid allocation in the notifier callback.
+	 * Protected by struct xe_vm::svm.gpusvm.notifier_lock in write mode
+	 * alternatively by the same lock in read mode *and* the vm resv held.
+	 */
+	struct mmu_interval_notifier_finish finish;
+	/**
+	 * @inval_batch: TLB invalidation batch for deferred completion.
+	 * Stores an in-flight TLB invalidation submitted during a two-pass
+	 * notifier so the wait can be deferred to a subsequent pass, allowing
+	 * multiple GPUs to be signalled before any of them are waited on.
+	 * Protected using the same locking as @finish.
+	 */
+	struct xe_tlb_inval_batch inval_batch;
+	/**
+	 * @finish_inuse: Whether @finish is currently in use by an in-progress
+	 * two-pass invalidation.
+	 * Protected using the same locking as @finish.
+	 */
+	bool finish_inuse;
+	/**
+	 * @tlb_inval_submitted: Whether a TLB invalidation has been submitted
+	 * via @inval_batch and is pending completion.  When set, the next pass
+	 * must call xe_tlb_inval_batch_wait() before reusing @inval_batch.
+	 * Protected using the same locking as @finish.
+	 */
+	bool tlb_inval_submitted;
 	/**
 	 * @initial_bind: user pointer has been bound at least once.
 	 * write: vm->svm.gpusvm.notifier_lock in read mode and vm->resv held.
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index a82e3a4fb389d..5572e12c2a7eb 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1112,7 +1112,7 @@ static void vma_destroy_cb(struct dma_fence *fence,
 	struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb);
 
 	INIT_WORK(&vma->destroy_work, vma_destroy_work_func);
-	queue_work(system_unbound_wq, &vma->destroy_work);
+	queue_work(system_dfl_wq, &vma->destroy_work);
 }
 
 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
@@ -1474,6 +1474,20 @@ static void xe_vm_pt_destroy(struct xe_vm *vm)
 	}
 }
 
+static void xe_vm_init_prove_locking(struct xe_device *xe, struct xe_vm *vm)
+{
+	if (!IS_ENABLED(CONFIG_PROVE_LOCKING))
+		return;
+
+	fs_reclaim_acquire(GFP_KERNEL);
+	might_lock(&vm->exec_queues.lock);
+	fs_reclaim_release(GFP_KERNEL);
+
+	down_read(&vm->exec_queues.lock);
+	might_lock(&xe_root_mmio_gt(xe)->uc.guc.ct.lock);
+	up_read(&vm->exec_queues.lock);
+}
+
 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 {
 	struct drm_gem_object *vm_resv_obj;
@@ -1529,11 +1543,16 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
 
 	INIT_LIST_HEAD(&vm->preempt.exec_queues);
+	for (id = 0; id < XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE; ++id)
+		INIT_LIST_HEAD(&vm->exec_queues.list[id]);
 	if (flags & XE_VM_FLAG_FAULT_MODE)
 		vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms;
 	else
 		vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms;
 
+	init_rwsem(&vm->exec_queues.lock);
+	xe_vm_init_prove_locking(xe, vm);
+
 	for_each_tile(tile, xe, id)
 		xe_range_fence_tree_init(&vm->rftree[id]);
 
@@ -1638,6 +1657,9 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 			if (!vm->pt_root[id])
 				continue;
 
+			if (!xef) /* Not from userspace */
+				create_flags |= EXEC_QUEUE_FLAG_KERNEL;
+
 			q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0);
 			if (IS_ERR(q)) {
 				err = PTR_ERR(q);
@@ -1653,7 +1675,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 		down_write(&xe->usm.lock);
 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm,
 				      XA_LIMIT(1, XE_MAX_ASID - 1),
-				      &xe->usm.next_asid, GFP_KERNEL);
+				      &xe->usm.next_asid, GFP_NOWAIT);
 		up_write(&xe->usm.lock);
 		if (err < 0)
 			goto err_close;
@@ -1875,7 +1897,7 @@ static void xe_vm_free(struct drm_gpuvm *gpuvm)
 	struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
 
 	/* To destroy the VM we need to be able to sleep */
-	queue_work(system_unbound_wq, &vm->destroy_work);
+	queue_work(system_dfl_wq, &vm->destroy_work);
 }
 
 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
@@ -1919,7 +1941,8 @@ find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)
 
 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \
 				    DRM_XE_VM_CREATE_FLAG_LR_MODE | \
-				    DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
+				    DRM_XE_VM_CREATE_FLAG_FAULT_MODE | \
+				    DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)
 
 int xe_vm_create_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file)
@@ -1958,12 +1981,18 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data,
 			 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE))
 		return -EINVAL;
 
+	if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) &&
+			 args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT))
+		return -EINVAL;
+
 	if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE)
 		flags |= XE_VM_FLAG_SCRATCH_PAGE;
 	if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE)
 		flags |= XE_VM_FLAG_LR_MODE;
 	if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
 		flags |= XE_VM_FLAG_FAULT_MODE;
+	if (args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)
+		flags |= XE_VM_FLAG_NO_VM_OVERCOMMIT;
 
 	vm = xe_vm_create(xe, flags, xef);
 	if (IS_ERR(vm))
@@ -2333,6 +2362,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 				op->map.vma_flags |= XE_VMA_DUMPABLE;
 			if (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET)
 				op->map.vma_flags |= XE_VMA_MADV_AUTORESET;
+			op->map.request_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS;
 			op->map.pat_index = pat_index;
 			op->map.invalidate_on_bind =
 				__xe_vm_needs_clear_scratch_pages(vm, flags);
@@ -2873,7 +2903,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
 }
 
 static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma,
-				 bool res_evict, bool validate)
+				 bool res_evict, bool validate, bool request_decompress)
 {
 	struct xe_bo *bo = xe_vma_bo(vma);
 	struct xe_vm *vm = xe_vma_vm(vma);
@@ -2884,8 +2914,14 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma,
 			err = drm_exec_lock_obj(exec, &bo->ttm.base);
 		if (!err && validate)
 			err = xe_bo_validate(bo, vm,
-					     !xe_vm_in_preempt_fence_mode(vm) &&
+					     xe_vm_allow_vm_eviction(vm) &&
 					     res_evict, exec);
+
+		if (err)
+			return err;
+
+		if (request_decompress)
+			err = xe_bo_decompress(bo);
 	}
 
 	return err;
@@ -2980,7 +3016,8 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 			err = vma_lock_and_validate(exec, op->map.vma,
 						    res_evict,
 						    !xe_vm_in_fault_mode(vm) ||
-						    op->map.immediate);
+						    op->map.immediate,
+						    op->map.request_decompress);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 		err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va));
@@ -2989,13 +3026,13 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 
 		err = vma_lock_and_validate(exec,
 					    gpuva_to_vma(op->base.remap.unmap->va),
-					    res_evict, false);
+					    res_evict, false, false);
 		if (!err && op->remap.prev)
 			err = vma_lock_and_validate(exec, op->remap.prev,
-						    res_evict, true);
+						    res_evict, true, false);
 		if (!err && op->remap.next)
 			err = vma_lock_and_validate(exec, op->remap.next,
-						    res_evict, true);
+						    res_evict, true, false);
 		break;
 	case DRM_GPUVA_OP_UNMAP:
 		err = check_ufence(gpuva_to_vma(op->base.unmap.va));
@@ -3004,7 +3041,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 
 		err = vma_lock_and_validate(exec,
 					    gpuva_to_vma(op->base.unmap.va),
-					    res_evict, false);
+					    res_evict, false, false);
 		break;
 	case DRM_GPUVA_OP_PREFETCH:
 	{
@@ -3019,7 +3056,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 
 		err = vma_lock_and_validate(exec,
 					    gpuva_to_vma(op->base.prefetch.va),
-					    res_evict, false);
+					    res_evict, false, false);
 		if (!err && !xe_vma_has_no_bo(vma))
 			err = xe_bo_migrate(xe_vma_bo(vma),
 					    region_to_mem_type[region],
@@ -3341,7 +3378,8 @@ ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO);
 	 DRM_XE_VM_BIND_FLAG_DUMPABLE | \
 	 DRM_XE_VM_BIND_FLAG_CHECK_PXP | \
 	 DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR | \
-	 DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET)
+	 DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET | \
+	 DRM_XE_VM_BIND_FLAG_DECOMPRESS)
 
 #ifdef TEST_VM_OPS_ERROR
 #define SUPPORTED_FLAGS	(SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR)
@@ -3401,6 +3439,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
 		bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
 		bool is_cpu_addr_mirror = flags &
 			DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR;
+		bool is_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS;
 		u16 pat_index = (*bind_ops)[i].pat_index;
 		u16 coh_mode;
 		bool comp_en;
@@ -3437,7 +3476,9 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
 		    XE_IOCTL_DBG(xe, obj_offset && (is_null ||
 						    is_cpu_addr_mirror)) ||
 		    XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP &&
-				 (is_null || is_cpu_addr_mirror)) ||
+				 (is_decompress || is_null || is_cpu_addr_mirror)) ||
+		    XE_IOCTL_DBG(xe, is_decompress &&
+				 xe_pat_index_get_comp_en(xe, pat_index)) ||
 		    XE_IOCTL_DBG(xe, !obj &&
 				 op == DRM_XE_VM_BIND_OP_MAP &&
 				 !is_null && !is_cpu_addr_mirror) ||
@@ -3479,6 +3520,13 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
 			err = -EINVAL;
 			goto free_bind_ops;
 		}
+
+		if (is_decompress && (XE_IOCTL_DBG(xe, !xe_device_has_flat_ccs(xe)) ||
+				      XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20) ||
+				      XE_IOCTL_DBG(xe, !IS_DGFX(xe)))) {
+			err = -EOPNOTSUPP;
+			goto free_bind_ops;
+		}
 	}
 
 	return 0;
@@ -3938,76 +3986,20 @@ void xe_vm_unlock(struct xe_vm *vm)
 }
 
 /**
- * xe_vm_range_tilemask_tlb_inval - Issue a TLB invalidation on this tilemask for an
- * address range
- * @vm: The VM
- * @start: start address
- * @end: end address
- * @tile_mask: mask for which gt's issue tlb invalidation
- *
- * Issue a range based TLB invalidation for gt's in tilemask
- *
- * Returns 0 for success, negative error code otherwise.
- */
-int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
-				   u64 end, u8 tile_mask)
-{
-	struct xe_tlb_inval_fence
-		fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
-	struct xe_tile *tile;
-	u32 fence_id = 0;
-	u8 id;
-	int err;
-
-	if (!tile_mask)
-		return 0;
-
-	for_each_tile(tile, vm->xe, id) {
-		if (!(tile_mask & BIT(id)))
-			continue;
-
-		xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
-					&fence[fence_id], true);
-
-		err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
-					 &fence[fence_id], start, end,
-					 vm->usm.asid, NULL);
-		if (err)
-			goto wait;
-		++fence_id;
-
-		if (!tile->media_gt)
-			continue;
-
-		xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
-					&fence[fence_id], true);
-
-		err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
-					 &fence[fence_id], start, end,
-					 vm->usm.asid, NULL);
-		if (err)
-			goto wait;
-		++fence_id;
-	}
-
-wait:
-	for (id = 0; id < fence_id; ++id)
-		xe_tlb_inval_fence_wait(&fence[id]);
-
-	return err;
-}
-
-/**
- * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
+ * xe_vm_invalidate_vma_submit - Submit a job to invalidate GPU mappings for
+ * VMA.
  * @vma: VMA to invalidate
+ * @batch: TLB invalidation batch to populate; caller must later call
+ *         xe_tlb_inval_batch_wait() on it to wait for completion
  *
  * Walks a list of page tables leaves which it memset the entries owned by this
- * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
- * complete.
+ * VMA to zero, invalidates the TLBs, but doesn't block waiting for TLB flush
+ * to complete, but instead populates @batch which can be waited on using
+ * xe_tlb_inval_batch_wait().
  *
  * Returns 0 for success, negative error code otherwise.
  */
-int xe_vm_invalidate_vma(struct xe_vma *vma)
+int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch)
 {
 	struct xe_device *xe = xe_vma_vm(vma)->xe;
 	struct xe_vm *vm = xe_vma_vm(vma);
@@ -4051,12 +4043,35 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 
 	xe_device_wmb(xe);
 
-	ret = xe_vm_range_tilemask_tlb_inval(xe_vma_vm(vma), xe_vma_start(vma),
-					     xe_vma_end(vma), tile_mask);
+	ret = xe_tlb_inval_range_tilemask_submit(xe, xe_vma_vm(vma)->usm.asid,
+						 xe_vma_start(vma), xe_vma_end(vma),
+						 tile_mask, batch);
 
 	/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
 	WRITE_ONCE(vma->tile_invalidated, vma->tile_mask);
+	return ret;
+}
 
+/**
+ * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
+ * @vma: VMA to invalidate
+ *
+ * Walks a list of page tables leaves which it memset the entries owned by this
+ * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
+ * complete.
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
+int xe_vm_invalidate_vma(struct xe_vma *vma)
+{
+	struct xe_tlb_inval_batch batch;
+	int ret;
+
+	ret = xe_vm_invalidate_vma_submit(vma, &batch);
+	if (ret)
+		return ret;
+
+	xe_tlb_inval_batch_wait(&batch);
 	return ret;
 }
 
@@ -4571,3 +4586,52 @@ int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t r
 	return xe_vm_alloc_vma(vm, &map_req, false);
 }
 
+/**
+ * xe_vm_add_exec_queue() - Add exec queue to VM
+ * @vm: The VM.
+ * @q: The exec_queue
+ *
+ * Add exec queue to VM, skipped if the device does not have context based TLB
+ * invalidations.
+ */
+void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
+{
+	struct xe_device *xe = vm->xe;
+
+	/* User VMs and queues only */
+	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL));
+	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
+	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
+	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE));
+	xe_assert(xe, vm->xef);
+	xe_assert(xe, vm == q->vm);
+
+	if (!xe->info.has_ctx_tlb_inval)
+		return;
+
+	down_write(&vm->exec_queues.lock);
+	list_add(&q->vm_exec_queue_link, &vm->exec_queues.list[q->gt->info.id]);
+	++vm->exec_queues.count[q->gt->info.id];
+	up_write(&vm->exec_queues.lock);
+}
+
+/**
+ * xe_vm_remove_exec_queue() - Remove exec queue from VM
+ * @vm: The VM.
+ * @q: The exec_queue
+ *
+ * Remove exec queue from VM, skipped if the device does not have context based
+ * TLB invalidations.
+ */
+void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
+{
+	if (!vm->xe->info.has_ctx_tlb_inval)
+		return;
+
+	down_write(&vm->exec_queues.lock);
+	if (!list_empty(&q->vm_exec_queue_link)) {
+		list_del(&q->vm_exec_queue_link);
+		--vm->exec_queues.count[q->gt->info.id];
+	}
+	up_write(&vm->exec_queues.lock);
+}
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 6cc98df472916..0bc7ed23eeae0 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -220,6 +220,13 @@ static inline bool xe_vm_in_preempt_fence_mode(struct xe_vm *vm)
 	return xe_vm_in_lr_mode(vm) && !xe_vm_in_fault_mode(vm);
 }
 
+static inline bool xe_vm_allow_vm_eviction(struct xe_vm *vm)
+{
+	return !xe_vm_in_lr_mode(vm) ||
+		(xe_vm_in_fault_mode(vm) &&
+		 !(vm->flags & XE_VM_FLAG_NO_VM_OVERCOMMIT));
+}
+
 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q);
 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q);
 
@@ -233,11 +240,10 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm,
 struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
 				     struct xe_svm_range *range);
 
-int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
-				   u64 end, u8 tile_mask);
-
 int xe_vm_invalidate_vma(struct xe_vma *vma);
 
+int xe_vm_invalidate_vma_submit(struct xe_vma *vma, struct xe_tlb_inval_batch *batch);
+
 int xe_vm_validate_protected(struct xe_vm *vm);
 
 static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm)
@@ -287,6 +293,9 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
 
 void xe_vm_kill(struct xe_vm *vm, bool unlocked);
 
+void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q);
+void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q);
+
 /**
  * xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.
  * @vm: The vm
diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c
index bc39a9a9790c3..869db304d96de 100644
--- a/drivers/gpu/drm/xe/xe_vm_madvise.c
+++ b/drivers/gpu/drm/xe/xe_vm_madvise.c
@@ -12,6 +12,7 @@
 #include "xe_pat.h"
 #include "xe_pt.h"
 #include "xe_svm.h"
+#include "xe_tlb_inval.h"
 
 struct xe_vmas_in_madvise_range {
 	u64 addr;
@@ -235,13 +236,20 @@ static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end)
 static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end)
 {
 	u8 tile_mask = xe_zap_ptes_in_madvise_range(vm, start, end);
+	struct xe_tlb_inval_batch batch;
+	int err;
 
 	if (!tile_mask)
 		return 0;
 
 	xe_device_wmb(vm->xe);
 
-	return xe_vm_range_tilemask_tlb_inval(vm, start, end, tile_mask);
+	err = xe_tlb_inval_range_tilemask_submit(vm->xe, vm->usm.asid, start, end,
+						 tile_mask, &batch);
+	if (!err)
+		xe_tlb_inval_batch_wait(&batch);
+
+	return err;
 }
 
 static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args)
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 437f64202f3b7..69e80c94138a2 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -18,6 +18,7 @@
 #include "xe_device_types.h"
 #include "xe_pt_types.h"
 #include "xe_range_fence.h"
+#include "xe_tlb_inval_types.h"
 #include "xe_userptr.h"
 
 struct drm_pagemap;
@@ -232,6 +233,7 @@ struct xe_vm {
 #define XE_VM_FLAG_TILE_ID(flags)	FIELD_GET(GENMASK(7, 6), flags)
 #define XE_VM_FLAG_SET_TILE_ID(tile)	FIELD_PREP(GENMASK(7, 6), (tile)->id)
 #define XE_VM_FLAG_GSC			BIT(8)
+#define XE_VM_FLAG_NO_VM_OVERCOMMIT     BIT(9)
 	unsigned long flags;
 
 	/**
@@ -298,6 +300,22 @@ struct xe_vm {
 		struct list_head pm_activate_link;
 	} preempt;
 
+	/** @exec_queues: Manages list of exec queues attached to this VM, protected by lock. */
+	struct {
+		/**
+		 * @exec_queues.list: list of exec queues attached to this VM,
+		 * per GT
+		 */
+		struct list_head list[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
+		/**
+		 * @exec_queues.count: count of exec queues attached to this VM,
+		 * per GT
+		 */
+		int count[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
+		/** @exec_queues.lock: lock to protect exec_queues list */
+		struct rw_semaphore lock;
+	} exec_queues;
+
 	/** @um: unified memory state */
 	struct {
 		/** @asid: address space ID, unique to each VM */
@@ -359,6 +377,8 @@ struct xe_vma_op_map {
 	bool immediate;
 	/** @read_only: Read only */
 	bool invalidate_on_bind;
+	/** @request_decompress: schedule decompression for GPU map */
+	bool request_decompress;
 	/** @pat_index: The pat index to use for this operation. */
 	u16 pat_index;
 };
diff --git a/drivers/gpu/drm/xe/xe_vram_freq.c b/drivers/gpu/drm/xe/xe_vram_freq.c
index 6f8281e0b96a3..8717367ccd4c7 100644
--- a/drivers/gpu/drm/xe/xe_vram_freq.c
+++ b/drivers/gpu/drm/xe/xe_vram_freq.c
@@ -5,9 +5,9 @@
 #include <linux/sysfs.h>
 #include <drm/drm_managed.h>
 
+#include "xe_device_types.h"
 #include "xe_pcode.h"
 #include "xe_pcode_api.h"
-#include "xe_tile.h"
 #include "xe_tile_sysfs.h"
 #include "xe_vram_freq.h"
 
diff --git a/drivers/gpu/drm/xe/xe_vsec.c b/drivers/gpu/drm/xe/xe_vsec.c
index 4ebb4dbe1c9b9..a9baf0bfe572c 100644
--- a/drivers/gpu/drm/xe/xe_vsec.c
+++ b/drivers/gpu/drm/xe/xe_vsec.c
@@ -140,10 +140,10 @@ static int xe_guid_decode(u32 guid, int *index, u32 *offset)
 	return 0;
 }
 
-int xe_pmt_telem_read(struct pci_dev *pdev, u32 guid, u64 *data, loff_t user_offset,
+int xe_pmt_telem_read(struct device *dev, u32 guid, u64 *data, loff_t user_offset,
 		      u32 count)
 {
-	struct xe_device *xe = pdev_to_xe_device(pdev);
+	struct xe_device *xe = kdev_to_xe_device(dev);
 	void __iomem *telem_addr = xe->mmio.regs + BMG_TELEMETRY_OFFSET;
 	u32 mem_region;
 	u32 offset;
@@ -198,7 +198,6 @@ void xe_vsec_init(struct xe_device *xe)
 {
 	struct intel_vsec_platform_info *info;
 	struct device *dev = xe->drm.dev;
-	struct pci_dev *pdev = to_pci_dev(dev);
 	enum xe_vsec platform;
 
 	platform = get_platform_info(xe);
@@ -221,6 +220,6 @@ void xe_vsec_init(struct xe_device *xe)
 	 * Register a VSEC. Cleanup is handled using device managed
 	 * resources.
 	 */
-	intel_vsec_register(pdev, info);
+	intel_vsec_register(dev, info);
 }
 MODULE_IMPORT_NS("INTEL_VSEC");
diff --git a/drivers/gpu/drm/xe/xe_vsec.h b/drivers/gpu/drm/xe/xe_vsec.h
index dabfb4e02d707..a25b4e6e681b5 100644
--- a/drivers/gpu/drm/xe/xe_vsec.h
+++ b/drivers/gpu/drm/xe/xe_vsec.h
@@ -6,10 +6,10 @@
 
 #include <linux/types.h>
 
-struct pci_dev;
+struct device;
 struct xe_device;
 
 void xe_vsec_init(struct xe_device *xe);
-int xe_pmt_telem_read(struct pci_dev *pdev, u32 guid, u64 *data, loff_t user_offset, u32 count);
+int xe_pmt_telem_read(struct device *dev, u32 guid, u64 *data, loff_t user_offset, u32 count);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index 462c2fa712e01..a02ee1282d35b 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -111,6 +111,17 @@
  *    difference of how they are maintained in the code. In xe it uses the
  *    xe_rtp infrastructure so the workarounds can be kept in tables, following
  *    a more declarative approach rather than procedural.
+ *
+ * .. note::
+ *    When a workaround applies to every single known IP version in a range,
+ *    the preferred handling is to use a single range-based RTP entry rather
+ *    than individual entries for each version, even if some of the intermediate
+ *    version numbers are currently unused.  If a new intermediate IP version
+ *    appears in the future and is enabled in the driver, any existing
+ *    range-based entries that contain the new version number will need to be
+ *    analyzed to determine whether their workarounds should apply to the new
+ *    version, or whether any existing range based entries needs to be split
+ *    into two entries that do not include the new intermediate version.
  */
 
 #undef XE_REG_MCR
@@ -120,6 +131,8 @@ __diag_push();
 __diag_ignore_all("-Woverride-init", "Allow field overrides in table");
 
 static const struct xe_rtp_entry_sr gt_was[] = {
+	/* Workarounds applying over a range of IPs */
+
 	{ XE_RTP_NAME("14011060649"),
 	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1200, 1255),
 		       ENGINE_CLASS(VIDEO_DECODE),
@@ -135,6 +148,27 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260)),
 	  XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE))
 	},
+	{ XE_RTP_NAME("16021867713"),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, 3002),
+		       ENGINE_CLASS(VIDEO_DECODE)),
+	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
+	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
+	},
+	{ XE_RTP_NAME("14019449301"),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1301, 2000), ENGINE_CLASS(VIDEO_DECODE)),
+	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
+	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
+	},
+	{ XE_RTP_NAME("16028005424"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), OR,
+		       MEDIA_VERSION_RANGE(1301, 3500)),
+	  XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
+	},
+	{ XE_RTP_NAME("14026578760"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, 3511), OR,
+		       MEDIA_VERSION(3503)),
+	  XE_RTP_ACTIONS(SET(GAMSTLB_CTRL, DIS_PEND_GPA_LINK))
+	},
 
 	/* DG1 */
 
@@ -191,10 +225,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 
 	/* Xe_LPG */
 
-	{ XE_RTP_NAME("14015795083"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271), GRAPHICS_STEP(A0, B0)),
-	  XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE))
-	},
 	{ XE_RTP_NAME("14018575942"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),
 	  XE_RTP_ACTIONS(SET(COMP_MOD_CTRL, FORCE_MISS_FTLB))
@@ -206,12 +236,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 
 	/* Xe_LPM+ */
 
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(1300),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
 	{ XE_RTP_NAME("22016670082"),
 	  XE_RTP_RULES(MEDIA_VERSION(1300)),
 	  XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR))
@@ -225,17 +249,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(2000),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
-	{ XE_RTP_NAME("14019449301"),
-	  XE_RTP_RULES(MEDIA_VERSION(2000), ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
 
 	/* Xe2_HPG */
 
@@ -250,20 +263,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 				   LSN_DIM_Z_WGT(1)))
 	},
 
-	/* Xe2_HPM */
-
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
-	{ XE_RTP_NAME("14019449301"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
-
 	/* Xe3_LPG */
 
 	{ XE_RTP_NAME("14021871409"),
@@ -273,43 +272,44 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 
 	/* Xe3_LPM */
 
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(3000),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
 	{ XE_RTP_NAME("16021865536"),
-	  XE_RTP_RULES(MEDIA_VERSION(3000),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(3000, 3002),
 		       ENGINE_CLASS(VIDEO_DECODE)),
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
-	{ XE_RTP_NAME("16021865536"),
-	  XE_RTP_RULES(MEDIA_VERSION(3002),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(3002),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
 	{ XE_RTP_NAME("14021486841"),
 	  XE_RTP_RULES(MEDIA_VERSION(3000), MEDIA_STEP(A0, B0),
 		       ENGINE_CLASS(VIDEO_DECODE)),
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
+
+	/* Xe3P_LPG */
+
+	{ XE_RTP_NAME("14025160223"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
+	  XE_RTP_ACTIONS(SET(MMIOATSREQLIMIT_GAM_WALK_3D,
+			     DIS_ATS_WRONLY_PG))
+	},
+	{ XE_RTP_NAME("14026144927, 16029437861"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
+	  XE_RTP_ACTIONS(SET(L3SQCREG2, L3_SQ_DISABLE_COAMA_2WAY_COH |
+			     L3_SQ_DISABLE_COAMA))
+	},
+	{ XE_RTP_NAME("14025635424"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
+	  XE_RTP_ACTIONS(SET(GAMSTLB_CTRL2, STLB_SINGLE_BANK_MODE))
+	},
 	{ XE_RTP_NAME("16028005424"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
 	  XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
 	},
 };
 
 static const struct xe_rtp_entry_sr engine_was[] = {
+	/* Workarounds applying over a range of IPs */
+
 	{ XE_RTP_NAME("22010931296, 18011464164, 14010919138"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(1200), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(FF_THREAD_MODE(RENDER_RING_BASE),
@@ -345,6 +345,54 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN1(RENDER_RING_BASE),
 			     FFSC_PERCTX_PREEMPT_CTRL))
 	},
+	{ XE_RTP_NAME("18032247524"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))
+	},
+	{ XE_RTP_NAME("16018712365"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
+	},
+	{ XE_RTP_NAME("14020338487"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
+	},
+	{ XE_RTP_NAME("14018471104"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
+	},
+	/*
+	 * Although this workaround isn't required for the RCS, disabling these
+	 * reports has no impact for our driver or the GuC, so we go ahead and
+	 * apply this to all engines for simplicity.
+	 */
+	{ XE_RTP_NAME("16021639441"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), OR,
+		       MEDIA_VERSION_RANGE(1301, 2000)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
+			     GHWSP_CSB_REPORT_DIS |
+			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	{ XE_RTP_NAME("14021402888"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
+	},
+	{ XE_RTP_NAME("13012615864"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS))
+	},
+	{ XE_RTP_NAME("18041344222"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3000),
+		       FUNC(xe_rtp_match_first_render_or_compute),
+		       FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
+	  XE_RTP_ACTIONS(SET(TDL_CHICKEN, EUSTALL_PERF_SAMPLING_DISABLE))
+	},
 
 	/* TGL */
 
@@ -460,11 +508,6 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 		       ENGINE_CLASS(COMPUTE)),
 	  XE_RTP_ACTIONS(SET(RING_HWSTAM(RENDER_RING_BASE), ~0))
 	},
-	{ XE_RTP_NAME("14014999345"),
-	  XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COMPUTE),
-		       GRAPHICS_STEP(B0, C0)),
-	  XE_RTP_ACTIONS(SET(CACHE_MODE_SS, DISABLE_ECC))
-	},
 
 	/* Xe_LPG */
 
@@ -487,149 +530,36 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 
 	/* Xe2_LPG */
 
-	{ XE_RTP_NAME("18032247524"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))
-	},
-	{ XE_RTP_NAME("16018712365"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
-	},
-	{ XE_RTP_NAME("14020338487"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
-	},
 	{ XE_RTP_NAME("18034896535, 16021540221"), /* 16021540221: GRAPHICS_STEP(A0, B0) */
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))
 	},
-	{ XE_RTP_NAME("14018471104"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
-	},
-	/*
-	 * These two workarounds are the same, just applying to different
-	 * engines.  Although Wa_18032095049 (for the RCS) isn't required on
-	 * all steppings, disabling these reports has no impact for our
-	 * driver or the GuC, so we go ahead and treat it the same as
-	 * Wa_16021639441 which does apply to all steppings.
-	 */
-	{ XE_RTP_NAME("18032095049, 16021639441"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004)),
-	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
-			     GHWSP_CSB_REPORT_DIS |
-			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
-			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-	},
 	{ XE_RTP_NAME("16018610683"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, SLM_WMTP_RESTORE))
 	},
-	{ XE_RTP_NAME("14021402888"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
-	},
-	{ XE_RTP_NAME("13012615864"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS))
-	},
 
 	/* Xe2_HPG */
 
-	{ XE_RTP_NAME("16018712365"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
-	},
 	{ XE_RTP_NAME("16018737384"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2999),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS))
 	},
-	{ XE_RTP_NAME("14020338487"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
-	},
-	{ XE_RTP_NAME("18032247524"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))
-	},
-	{ XE_RTP_NAME("14018471104"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
-	},
-	/*
-	 * Although this workaround isn't required for the RCS, disabling these
-	 * reports has no impact for our driver or the GuC, so we go ahead and
-	 * apply this to all engines for simplicity.
-	 */
-	{ XE_RTP_NAME("16021639441"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002)),
-	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
-			     GHWSP_CSB_REPORT_DIS |
-			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
-			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-	},
 	{ XE_RTP_NAME("14019811474"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2001),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, WR_REQ_CHAINING_DIS))
 	},
-	{ XE_RTP_NAME("14021402888"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
-	},
 	{ XE_RTP_NAME("14021821874, 14022954250"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, STK_ID_RESTRICT))
 	},
-	{ XE_RTP_NAME("13012615864"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS))
-	},
-	{ XE_RTP_NAME("18041344222"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002),
-		       FUNC(xe_rtp_match_first_render_or_compute),
-		       FUNC(xe_rtp_match_not_sriov_vf),
-		       FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
-	  XE_RTP_ACTIONS(SET(TDL_CHICKEN, EUSTALL_PERF_SAMPLING_DISABLE))
-	},
-
-	/* Xe2_LPM */
-
-	{ XE_RTP_NAME("16021639441"),
-	  XE_RTP_RULES(MEDIA_VERSION(2000)),
-	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
-			     GHWSP_CSB_REPORT_DIS |
-			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
-			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-	},
-
-	/* Xe2_HPM */
-
-	{ XE_RTP_NAME("16021639441"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301)),
-	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
-			     GHWSP_CSB_REPORT_DIS |
-			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
-			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-	},
 
 	/* Xe3_LPG */
 
-	{ XE_RTP_NAME("14021402888"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
-	},
 	{ XE_RTP_NAME("18034896535"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
@@ -642,34 +572,33 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 				   SMP_FORCE_128B_OVERFETCH))
 	},
 	{ XE_RTP_NAME("14023061436"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001),
-		       FUNC(xe_rtp_match_first_render_or_compute), OR,
-		       GRAPHICS_VERSION_RANGE(3003, 3005),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(TDL_CHICKEN, QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE))
 	},
-	{ XE_RTP_NAME("13012615864"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001), OR,
-		       GRAPHICS_VERSION_RANGE(3003, 3005),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS))
-	},
 	{ XE_RTP_NAME("16023105232"),
 	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1301, 3000), OR,
 		       GRAPHICS_VERSION_RANGE(2001, 3001)),
 	  XE_RTP_ACTIONS(SET(RING_PSMI_CTL(0), RC_SEMA_IDLE_MSG_DISABLE,
 			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
 	},
-	{ XE_RTP_NAME("14021402888"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3003, 3005), FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
+
+	/* Xe3p_LPG*/
+
+	{ XE_RTP_NAME("22021149932"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, SAMPLER_LD_LSC_DISABLE))
 	},
-	{ XE_RTP_NAME("18041344222"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(3000),
-		       FUNC(xe_rtp_match_first_render_or_compute),
-		       FUNC(xe_rtp_match_not_sriov_vf),
-		       FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
-	  XE_RTP_ACTIONS(SET(TDL_CHICKEN, EUSTALL_PERF_SAMPLING_DISABLE))
+	{ XE_RTP_NAME("14025676848"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE))
+	},
+	{ XE_RTP_NAME("16028951944"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN5, CPSS_AWARE_DIS))
 	},
 };
 
@@ -707,6 +636,30 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION(1200)),
 	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC))
 	},
+	{ XE_RTP_NAME("14019877138"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
+	},
+	{ XE_RTP_NAME("14019386621"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
+	},
+	{ XE_RTP_NAME("14019988906"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
+	},
+	{ XE_RTP_NAME("18033852989"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
+	},
+	{ XE_RTP_NAME("15016589081"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))
+	},
+	{ XE_RTP_NAME("14026781792"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3510), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(FF_MODE, DIS_TE_PATCH_CTRL))
+	},
 
 	/* DG1 */
 
@@ -743,10 +696,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	  XE_RTP_RULES(PLATFORM(DG2)),
 	  XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE))
 	},
-	{ XE_RTP_NAME("14019877138"),
-	  XE_RTP_RULES(PLATFORM(DG2)),
-	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
-	},
 
 	/* PVC */
 
@@ -764,29 +713,9 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),
 	  XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE))
 	},
-	{ XE_RTP_NAME("14019877138"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
-	},
 
 	/* Xe2_LPG */
 
-	{ XE_RTP_NAME("14019386621"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
-	},
-	{ XE_RTP_NAME("14019877138"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
-	},
-	{ XE_RTP_NAME("14019988906"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
-	},
-	{ XE_RTP_NAME("18033852989"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
-	},
 	{ XE_RTP_NAME("14021567978"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED),
 		       ENGINE_CLASS(RENDER)),
@@ -806,20 +735,9 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 			     DIS_PARTIAL_AUTOSTRIP |
 			     DIS_AUTOSTRIP))
 	},
-	{ XE_RTP_NAME("15016589081"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))
-	},
 
 	/* Xe2_HPG */
-	{ XE_RTP_NAME("15010599737"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN))
-	},
-	{ XE_RTP_NAME("14019386621"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
-	},
+
 	{ XE_RTP_NAME("14020756599"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
@@ -841,18 +759,10 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 			     DIS_PARTIAL_AUTOSTRIP |
 			     DIS_AUTOSTRIP))
 	},
-	{ XE_RTP_NAME("15016589081"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))
-	},
 	{ XE_RTP_NAME("22021007897"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
 	},
-	{ XE_RTP_NAME("18033852989"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
-	},
 
 	/* Xe3_LPG */
 	{ XE_RTP_NAME("14021490052"),
@@ -944,7 +854,8 @@ void xe_wa_process_gt(struct xe_gt *gt)
 
 	xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.gt,
 						  ARRAY_SIZE(gt_was));
-	xe_rtp_process_to_sr(&ctx, gt_was, ARRAY_SIZE(gt_was), &gt->reg_sr);
+	xe_rtp_process_to_sr(&ctx, gt_was, ARRAY_SIZE(gt_was),
+			     &gt->reg_sr, false);
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_wa_process_gt);
 
@@ -962,7 +873,8 @@ void xe_wa_process_engine(struct xe_hw_engine *hwe)
 
 	xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.engine,
 						  ARRAY_SIZE(engine_was));
-	xe_rtp_process_to_sr(&ctx, engine_was, ARRAY_SIZE(engine_was), &hwe->reg_sr);
+	xe_rtp_process_to_sr(&ctx, engine_was, ARRAY_SIZE(engine_was),
+			     &hwe->reg_sr, false);
 }
 
 /**
@@ -979,7 +891,8 @@ void xe_wa_process_lrc(struct xe_hw_engine *hwe)
 
 	xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.lrc,
 						  ARRAY_SIZE(lrc_was));
-	xe_rtp_process_to_sr(&ctx, lrc_was, ARRAY_SIZE(lrc_was), &hwe->reg_lrc);
+	xe_rtp_process_to_sr(&ctx, lrc_was, ARRAY_SIZE(lrc_was),
+			     &hwe->reg_lrc, true);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index 5cd7fa6d2a5c0..80b54b195f20c 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -2,14 +2,12 @@
 16010904313	GRAPHICS_VERSION_RANGE(1200, 1210)
 18022495364	GRAPHICS_VERSION_RANGE(1200, 1210)
 22012773006	GRAPHICS_VERSION_RANGE(1200, 1250)
-14014475959	GRAPHICS_VERSION_RANGE(1270, 1271), GRAPHICS_STEP(A0, B0)
-		PLATFORM(DG2)
+14014475959	PLATFORM(DG2)
 22011391025	PLATFORM(DG2)
 22012727170	SUBPLATFORM(DG2, G11)
 22012727685	SUBPLATFORM(DG2, G11)
 22016596838	PLATFORM(PVC)
 18020744125	PLATFORM(PVC)
-1509372804	PLATFORM(PVC), GRAPHICS_STEP(A0, C0)
 1409600907	GRAPHICS_VERSION_RANGE(1200, 1250)
 22014953428	SUBPLATFORM(DG2, G10)
 		SUBPLATFORM(DG2, G12)
@@ -26,18 +24,9 @@
 		MEDIA_VERSION(2000)
 16022287689	GRAPHICS_VERSION(2001)
 		GRAPHICS_VERSION(2004)
-13011645652	GRAPHICS_VERSION(2004)
-		GRAPHICS_VERSION_RANGE(3000, 3001)
-		GRAPHICS_VERSION(3003)
-		GRAPHICS_VERSION_RANGE(3004, 3005)
-14022293748	GRAPHICS_VERSION_RANGE(2001, 2002)
-		GRAPHICS_VERSION(2004)
-		GRAPHICS_VERSION_RANGE(3000, 3005)
-22019794406	GRAPHICS_VERSION_RANGE(2001, 2002)
-		GRAPHICS_VERSION(2004)
-		GRAPHICS_VERSION_RANGE(3000, 3001)
-		GRAPHICS_VERSION(3003)
-		GRAPHICS_VERSION_RANGE(3004, 3005)
+13011645652	GRAPHICS_VERSION_RANGE(2004, 3005)
+14022293748	GRAPHICS_VERSION_RANGE(2001, 3005)
+22019794406	GRAPHICS_VERSION_RANGE(2001, 3005)
 22019338487	MEDIA_VERSION(2000)
 		GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_not_sriov_vf)
 		MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_not_sriov_vf)
@@ -54,22 +43,18 @@
 18013179988	GRAPHICS_VERSION(1255)
 		GRAPHICS_VERSION_RANGE(1270, 1274)
 1508761755	GRAPHICS_VERSION(1255)
-		GRAPHICS_VERSION(1260), GRAPHICS_STEP(A0, B0)
-16023105232	GRAPHICS_VERSION_RANGE(2001, 3001)
-		MEDIA_VERSION_RANGE(1301, 3000)
-		MEDIA_VERSION(3002)
-		GRAPHICS_VERSION_RANGE(3003, 3005)
-16026508708	GRAPHICS_VERSION_RANGE(1200, 3001)
-		MEDIA_VERSION_RANGE(1300, 3000)
-		MEDIA_VERSION(3002)
-		GRAPHICS_VERSION_RANGE(3003, 3005)
+16023105232	GRAPHICS_VERSION_RANGE(2001, 3005)
+		MEDIA_VERSION_RANGE(1301, 3002)
+16026508708	GRAPHICS_VERSION_RANGE(1200, 3005)
+		MEDIA_VERSION_RANGE(1300, 3002)
 14020001231	GRAPHICS_VERSION_RANGE(2001,2004), FUNC(xe_rtp_match_psmi_enabled)
-		MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled)
-		MEDIA_VERSION(3000), FUNC(xe_rtp_match_psmi_enabled)
-		MEDIA_VERSION(3002), FUNC(xe_rtp_match_psmi_enabled)
+		MEDIA_VERSION_RANGE(2000, 3002), FUNC(xe_rtp_match_psmi_enabled)
 16023683509	MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled)
 		MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled)
 
 15015404425_disable	PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER)
 16026007364    MEDIA_VERSION(3000)
 14020316580    MEDIA_VERSION(1301)
+
+14025883347	MEDIA_VERSION_RANGE(1301, 3503)
+		GRAPHICS_VERSION_RANGE(2004, 3005)
diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c
index dde4f4967ca37..900daf1d1b1bb 100644
--- a/drivers/gpu/drm/xe/xe_wopcm.c
+++ b/drivers/gpu/drm/xe/xe_wopcm.c
@@ -55,8 +55,6 @@
 #define MTL_WOPCM_SIZE			SZ_4M
 #define WOPCM_SIZE			SZ_2M
 
-#define MAX_WOPCM_SIZE			SZ_8M
-
 /* 16KB WOPCM (RSVD WOPCM) is reserved from HuC firmware top. */
 #define WOPCM_RESERVED_SIZE		SZ_16K
 
@@ -186,6 +184,14 @@ u32 xe_wopcm_size(struct xe_device *xe)
 		WOPCM_SIZE;
 }
 
+static u32 max_wopcm_size(struct xe_device *xe)
+{
+	if (xe->info.platform == XE_NOVALAKE_P)
+		return SZ_16M;
+	else
+		return SZ_8M;
+}
+
 /**
  * xe_wopcm_init() - Initialize the WOPCM structure.
  * @wopcm: pointer to xe_wopcm.
@@ -227,8 +233,11 @@ int xe_wopcm_init(struct xe_wopcm *wopcm)
 		 * When the GuC wopcm base and size are preprogrammed by
 		 * BIOS/IFWI, check against the max allowed wopcm size to
 		 * validate if the programmed values align to the wopcm layout.
+		 *
+		 * FIXME: This is giving the maximum overall WOPCM size and not
+		 * the size relative to each GT.
 		 */
-		wopcm->size = MAX_WOPCM_SIZE;
+		wopcm->size = max_wopcm_size(xe);
 
 		goto check;
 	}
diff --git a/drivers/gpu/host1x/Makefile b/drivers/gpu/host1x/Makefile
index ee5286ffe08d5..fead483af0b4f 100644
--- a/drivers/gpu/host1x/Makefile
+++ b/drivers/gpu/host1x/Makefile
@@ -9,6 +9,7 @@ host1x-y = \
 	job.o \
 	debug.o \
 	mipi.o \
+	tegra114-mipi.o \
 	fence.o \
 	hw/host1x01.o \
 	hw/host1x02.o \
diff --git a/drivers/gpu/host1x/mipi.c b/drivers/gpu/host1x/mipi.c
index fea9f491df66c..01513b775d899 100644
--- a/drivers/gpu/host1x/mipi.c
+++ b/drivers/gpu/host1x/mipi.c
@@ -1,215 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2013 NVIDIA Corporation
- *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that copyright
- * notice and this permission notice appear in supporting documentation, and
- * that the name of the copyright holders not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  The copyright holders make no representations
- * about the suitability of this software for any purpose.  It is provided "as
- * is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
- * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
- * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
- * OF THIS SOFTWARE.
+ * Copyright (C) 2025 Svyatoslav Ryhel <clamor95@gmail.com>
  */
 
 #include <linux/clk.h>
-#include <linux/host1x.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/tegra-mipi-cal.h>
 
-#include "dev.h"
-
-#define MIPI_CAL_CTRL			0x00
-#define MIPI_CAL_CTRL_NOISE_FILTER(x)	(((x) & 0xf) << 26)
-#define MIPI_CAL_CTRL_PRESCALE(x)	(((x) & 0x3) << 24)
-#define MIPI_CAL_CTRL_CLKEN_OVR		(1 << 4)
-#define MIPI_CAL_CTRL_START		(1 << 0)
-
-#define MIPI_CAL_AUTOCAL_CTRL		0x01
-
-#define MIPI_CAL_STATUS			0x02
-#define MIPI_CAL_STATUS_DONE		(1 << 16)
-#define MIPI_CAL_STATUS_ACTIVE		(1 <<  0)
-
-#define MIPI_CAL_CONFIG_CSIA		0x05
-#define MIPI_CAL_CONFIG_CSIB		0x06
-#define MIPI_CAL_CONFIG_CSIC		0x07
-#define MIPI_CAL_CONFIG_CSID		0x08
-#define MIPI_CAL_CONFIG_CSIE		0x09
-#define MIPI_CAL_CONFIG_CSIF		0x0a
-#define MIPI_CAL_CONFIG_DSIA		0x0e
-#define MIPI_CAL_CONFIG_DSIB		0x0f
-#define MIPI_CAL_CONFIG_DSIC		0x10
-#define MIPI_CAL_CONFIG_DSID		0x11
-
-#define MIPI_CAL_CONFIG_DSIA_CLK	0x19
-#define MIPI_CAL_CONFIG_DSIB_CLK	0x1a
-#define MIPI_CAL_CONFIG_CSIAB_CLK	0x1b
-#define MIPI_CAL_CONFIG_DSIC_CLK	0x1c
-#define MIPI_CAL_CONFIG_CSICD_CLK	0x1c
-#define MIPI_CAL_CONFIG_DSID_CLK	0x1d
-#define MIPI_CAL_CONFIG_CSIE_CLK	0x1d
-
-/* for data and clock lanes */
-#define MIPI_CAL_CONFIG_SELECT		(1 << 21)
-
-/* for data lanes */
-#define MIPI_CAL_CONFIG_HSPDOS(x)	(((x) & 0x1f) << 16)
-#define MIPI_CAL_CONFIG_HSPUOS(x)	(((x) & 0x1f) <<  8)
-#define MIPI_CAL_CONFIG_TERMOS(x)	(((x) & 0x1f) <<  0)
-
-/* for clock lanes */
-#define MIPI_CAL_CONFIG_HSCLKPDOSD(x)	(((x) & 0x1f) <<  8)
-#define MIPI_CAL_CONFIG_HSCLKPUOSD(x)	(((x) & 0x1f) <<  0)
-
-#define MIPI_CAL_BIAS_PAD_CFG0		0x16
-#define MIPI_CAL_BIAS_PAD_PDVCLAMP	(1 << 1)
-#define MIPI_CAL_BIAS_PAD_E_VCLAMP_REF	(1 << 0)
-
-#define MIPI_CAL_BIAS_PAD_CFG1		0x17
-#define MIPI_CAL_BIAS_PAD_DRV_DN_REF(x) (((x) & 0x7) << 16)
-#define MIPI_CAL_BIAS_PAD_DRV_UP_REF(x) (((x) & 0x7) << 8)
-
-#define MIPI_CAL_BIAS_PAD_CFG2		0x18
-#define MIPI_CAL_BIAS_PAD_VCLAMP(x)	(((x) & 0x7) << 16)
-#define MIPI_CAL_BIAS_PAD_VAUXP(x)	(((x) & 0x7) << 4)
-#define MIPI_CAL_BIAS_PAD_PDVREG	(1 << 1)
-
-struct tegra_mipi_pad {
-	unsigned long data;
-	unsigned long clk;
-};
-
-struct tegra_mipi_soc {
-	bool has_clk_lane;
-	const struct tegra_mipi_pad *pads;
-	unsigned int num_pads;
-
-	bool clock_enable_override;
-	bool needs_vclamp_ref;
-
-	/* bias pad configuration settings */
-	u8 pad_drive_down_ref;
-	u8 pad_drive_up_ref;
+/* only need to support one provider */
+static struct {
+	struct device_node *np;
+	const struct tegra_mipi_ops *ops;
+} provider;
 
-	u8 pad_vclamp_level;
-	u8 pad_vauxp_level;
-
-	/* calibration settings for data lanes */
-	u8 hspdos;
-	u8 hspuos;
-	u8 termos;
-
-	/* calibration settings for clock lanes */
-	u8 hsclkpdos;
-	u8 hsclkpuos;
-};
-
-struct tegra_mipi {
-	const struct tegra_mipi_soc *soc;
-	struct device *dev;
-	void __iomem *regs;
-	struct mutex lock;
-	struct clk *clk;
-
-	unsigned long usage_count;
-};
-
-struct tegra_mipi_device {
-	struct platform_device *pdev;
-	struct tegra_mipi *mipi;
-	struct device *device;
-	unsigned long pads;
-};
-
-static inline u32 tegra_mipi_readl(struct tegra_mipi *mipi,
-				   unsigned long offset)
+/**
+ * tegra_mipi_enable - Enable the Tegra MIPI calibration device.
+ * @device: Handle to the Tegra MIPI calibration device.
+ *
+ * This calls the enable sequence for the Tegra MIPI calibration device.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int tegra_mipi_enable(struct tegra_mipi_device *device)
 {
-	return readl(mipi->regs + (offset << 2));
-}
+	if (device->ops->enable)
+		return device->ops->enable(device);
 
-static inline void tegra_mipi_writel(struct tegra_mipi *mipi, u32 value,
-				     unsigned long offset)
-{
-	writel(value, mipi->regs + (offset << 2));
+	return 0;
 }
+EXPORT_SYMBOL(tegra_mipi_enable);
 
-static int tegra_mipi_power_up(struct tegra_mipi *mipi)
+/**
+ * tegra_mipi_disable - Disable the Tegra MIPI calibration device.
+ * @device: Handle to the Tegra MIPI calibration device.
+ *
+ * This calls the disable sequence for the Tegra MIPI calibration device.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int tegra_mipi_disable(struct tegra_mipi_device *device)
 {
-	u32 value;
-	int err;
-
-	err = clk_enable(mipi->clk);
-	if (err < 0)
-		return err;
-
-	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG0);
-	value &= ~MIPI_CAL_BIAS_PAD_PDVCLAMP;
-
-	if (mipi->soc->needs_vclamp_ref)
-		value |= MIPI_CAL_BIAS_PAD_E_VCLAMP_REF;
-
-	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG0);
-
-	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG2);
-	value &= ~MIPI_CAL_BIAS_PAD_PDVREG;
-	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
-
-	clk_disable(mipi->clk);
+	if (device->ops->disable)
+		return device->ops->disable(device);
 
 	return 0;
 }
+EXPORT_SYMBOL(tegra_mipi_disable);
 
-static int tegra_mipi_power_down(struct tegra_mipi *mipi)
+/**
+ * tegra_mipi_start_calibration - Start the Tegra MIPI calibration sequence.
+ * @device: Handle to the Tegra MIPI calibration device.
+ *
+ * This initiates the calibration of CSI/DSI interfaces via the Tegra MIPI
+ * calibration device.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int tegra_mipi_start_calibration(struct tegra_mipi_device *device)
 {
-	u32 value;
-	int err;
-
-	err = clk_enable(mipi->clk);
-	if (err < 0)
-		return err;
-
-	/*
-	 * The MIPI_CAL_BIAS_PAD_PDVREG controls a voltage regulator that
-	 * supplies the DSI pads. This must be kept enabled until none of the
-	 * DSI lanes are used anymore.
-	 */
-	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG2);
-	value |= MIPI_CAL_BIAS_PAD_PDVREG;
-	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
-
-	/*
-	 * MIPI_CAL_BIAS_PAD_PDVCLAMP and MIPI_CAL_BIAS_PAD_E_VCLAMP_REF
-	 * control a regulator that supplies current to the pre-driver logic.
-	 * Powering down this regulator causes DSI to fail, so it must remain
-	 * powered on until none of the DSI lanes are used anymore.
-	 */
-	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG0);
+	if (device->ops->start_calibration)
+		return device->ops->start_calibration(device);
 
-	if (mipi->soc->needs_vclamp_ref)
-		value &= ~MIPI_CAL_BIAS_PAD_E_VCLAMP_REF;
+	return 0;
+}
+EXPORT_SYMBOL(tegra_mipi_start_calibration);
 
-	value |= MIPI_CAL_BIAS_PAD_PDVCLAMP;
-	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG0);
+/**
+ * tegra_mipi_finish_calibration - Finish the Tegra MIPI calibration sequence.
+ * @device: Handle to the Tegra MIPI calibration device.
+ *
+ * This completes the calibration of CSI/DSI interfaces via the Tegra MIPI
+ * calibration device.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int tegra_mipi_finish_calibration(struct tegra_mipi_device *device)
+{
+	if (device->ops->finish_calibration)
+		return device->ops->finish_calibration(device);
 
 	return 0;
 }
+EXPORT_SYMBOL(tegra_mipi_finish_calibration);
 
+/**
+ * tegra_mipi_request - Request a Tegra MIPI calibration device.
+ * @device: Handle of the device requesting the MIPI calibration function.
+ * @np: Device node pointer of the device requesting the MIPI calibration
+ *	function.
+ *
+ * This function requests a reference to a Tegra MIPI calibration device.
+ *
+ * Returns a pointer to the Tegra MIPI calibration device on success,
+ * or an ERR_PTR-encoded error code on failure.
+ */
 struct tegra_mipi_device *tegra_mipi_request(struct device *device,
 					     struct device_node *np)
 {
-	struct tegra_mipi_device *dev;
+	struct tegra_mipi_device *mipidev;
 	struct of_phandle_args args;
 	int err;
 
@@ -219,321 +114,80 @@ struct tegra_mipi_device *tegra_mipi_request(struct device *device,
 	if (err < 0)
 		return ERR_PTR(err);
 
-	dev = kzalloc_obj(*dev);
-	if (!dev) {
+	if (provider.np != args.np)
+		return ERR_PTR(-ENODEV);
+
+	mipidev = kzalloc_obj(*mipidev);
+	if (!mipidev) {
 		err = -ENOMEM;
 		goto out;
 	}
 
-	dev->pdev = of_find_device_by_node(args.np);
-	if (!dev->pdev) {
+	mipidev->pdev = of_find_device_by_node(args.np);
+	if (!mipidev->pdev) {
 		err = -ENODEV;
 		goto free;
 	}
 
-	dev->mipi = platform_get_drvdata(dev->pdev);
-	if (!dev->mipi) {
-		err = -EPROBE_DEFER;
-		goto put;
-	}
-
 	of_node_put(args.np);
 
-	dev->pads = args.args[0];
-	dev->device = device;
+	mipidev->ops = provider.ops;
+	mipidev->pads = args.args[0];
 
-	return dev;
+	return mipidev;
 
-put:
-	platform_device_put(dev->pdev);
 free:
-	kfree(dev);
+	kfree(mipidev);
 out:
 	of_node_put(args.np);
 	return ERR_PTR(err);
 }
 EXPORT_SYMBOL(tegra_mipi_request);
 
-void tegra_mipi_free(struct tegra_mipi_device *device)
+/**
+ * tegra_mipi_free - Free a Tegra MIPI calibration device.
+ * @mipidev: Handle to the Tegra MIPI calibration device.
+ *
+ * This function releases a reference to a Tegra MIPI calibration device
+ * previously requested by tegra_mipi_request().
+ */
+void tegra_mipi_free(struct tegra_mipi_device *mipidev)
 {
-	platform_device_put(device->pdev);
-	kfree(device);
+	platform_device_put(mipidev->pdev);
+	kfree(mipidev);
 }
 EXPORT_SYMBOL(tegra_mipi_free);
 
-int tegra_mipi_enable(struct tegra_mipi_device *dev)
+static void tegra_mipi_remove_provider(void *data)
 {
-	int err = 0;
-
-	mutex_lock(&dev->mipi->lock);
-
-	if (dev->mipi->usage_count++ == 0)
-		err = tegra_mipi_power_up(dev->mipi);
-
-	mutex_unlock(&dev->mipi->lock);
-
-	return err;
-
-}
-EXPORT_SYMBOL(tegra_mipi_enable);
-
-int tegra_mipi_disable(struct tegra_mipi_device *dev)
-{
-	int err = 0;
-
-	mutex_lock(&dev->mipi->lock);
-
-	if (--dev->mipi->usage_count == 0)
-		err = tegra_mipi_power_down(dev->mipi);
-
-	mutex_unlock(&dev->mipi->lock);
-
-	return err;
-
-}
-EXPORT_SYMBOL(tegra_mipi_disable);
-
-int tegra_mipi_finish_calibration(struct tegra_mipi_device *device)
-{
-	struct tegra_mipi *mipi = device->mipi;
-	void __iomem *status_reg = mipi->regs + (MIPI_CAL_STATUS << 2);
-	u32 value;
-	int err;
-
-	err = readl_relaxed_poll_timeout(status_reg, value,
-					 !(value & MIPI_CAL_STATUS_ACTIVE) &&
-					 (value & MIPI_CAL_STATUS_DONE), 50,
-					 250000);
-	mutex_unlock(&device->mipi->lock);
-	clk_disable(device->mipi->clk);
-
-	return err;
+	provider.np = NULL;
+	provider.ops = NULL;
 }
-EXPORT_SYMBOL(tegra_mipi_finish_calibration);
-
-int tegra_mipi_start_calibration(struct tegra_mipi_device *device)
-{
-	const struct tegra_mipi_soc *soc = device->mipi->soc;
-	unsigned int i;
-	u32 value;
-	int err;
-
-	err = clk_enable(device->mipi->clk);
-	if (err < 0)
-		return err;
-
-	mutex_lock(&device->mipi->lock);
-
-	value = MIPI_CAL_BIAS_PAD_DRV_DN_REF(soc->pad_drive_down_ref) |
-		MIPI_CAL_BIAS_PAD_DRV_UP_REF(soc->pad_drive_up_ref);
-	tegra_mipi_writel(device->mipi, value, MIPI_CAL_BIAS_PAD_CFG1);
-
-	value = tegra_mipi_readl(device->mipi, MIPI_CAL_BIAS_PAD_CFG2);
-	value &= ~MIPI_CAL_BIAS_PAD_VCLAMP(0x7);
-	value &= ~MIPI_CAL_BIAS_PAD_VAUXP(0x7);
-	value |= MIPI_CAL_BIAS_PAD_VCLAMP(soc->pad_vclamp_level);
-	value |= MIPI_CAL_BIAS_PAD_VAUXP(soc->pad_vauxp_level);
-	tegra_mipi_writel(device->mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
-
-	for (i = 0; i < soc->num_pads; i++) {
-		u32 clk = 0, data = 0;
-
-		if (device->pads & BIT(i)) {
-			data = MIPI_CAL_CONFIG_SELECT |
-			       MIPI_CAL_CONFIG_HSPDOS(soc->hspdos) |
-			       MIPI_CAL_CONFIG_HSPUOS(soc->hspuos) |
-			       MIPI_CAL_CONFIG_TERMOS(soc->termos);
-			clk = MIPI_CAL_CONFIG_SELECT |
-			      MIPI_CAL_CONFIG_HSCLKPDOSD(soc->hsclkpdos) |
-			      MIPI_CAL_CONFIG_HSCLKPUOSD(soc->hsclkpuos);
-		}
 
-		tegra_mipi_writel(device->mipi, data, soc->pads[i].data);
-
-		if (soc->has_clk_lane && soc->pads[i].clk != 0)
-			tegra_mipi_writel(device->mipi, clk, soc->pads[i].clk);
-	}
-
-	value = tegra_mipi_readl(device->mipi, MIPI_CAL_CTRL);
-	value &= ~MIPI_CAL_CTRL_NOISE_FILTER(0xf);
-	value &= ~MIPI_CAL_CTRL_PRESCALE(0x3);
-	value |= MIPI_CAL_CTRL_NOISE_FILTER(0xa);
-	value |= MIPI_CAL_CTRL_PRESCALE(0x2);
-
-	if (!soc->clock_enable_override)
-		value &= ~MIPI_CAL_CTRL_CLKEN_OVR;
-	else
-		value |= MIPI_CAL_CTRL_CLKEN_OVR;
-
-	tegra_mipi_writel(device->mipi, value, MIPI_CAL_CTRL);
-
-	/* clear any pending status bits */
-	value = tegra_mipi_readl(device->mipi, MIPI_CAL_STATUS);
-	tegra_mipi_writel(device->mipi, value, MIPI_CAL_STATUS);
-
-	value = tegra_mipi_readl(device->mipi, MIPI_CAL_CTRL);
-	value |= MIPI_CAL_CTRL_START;
-	tegra_mipi_writel(device->mipi, value, MIPI_CAL_CTRL);
-
-	/*
-	 * Wait for min 72uS to let calibration logic finish calibration
-	 * sequence codes before waiting for pads idle state to apply the
-	 * results.
-	 */
-	usleep_range(75, 80);
-
-	return 0;
-}
-EXPORT_SYMBOL(tegra_mipi_start_calibration);
-
-static const struct tegra_mipi_pad tegra114_mipi_pads[] = {
-	{ .data = MIPI_CAL_CONFIG_CSIA },
-	{ .data = MIPI_CAL_CONFIG_CSIB },
-	{ .data = MIPI_CAL_CONFIG_CSIC },
-	{ .data = MIPI_CAL_CONFIG_CSID },
-	{ .data = MIPI_CAL_CONFIG_CSIE },
-	{ .data = MIPI_CAL_CONFIG_DSIA },
-	{ .data = MIPI_CAL_CONFIG_DSIB },
-	{ .data = MIPI_CAL_CONFIG_DSIC },
-	{ .data = MIPI_CAL_CONFIG_DSID },
-};
-
-static const struct tegra_mipi_soc tegra114_mipi_soc = {
-	.has_clk_lane = false,
-	.pads = tegra114_mipi_pads,
-	.num_pads = ARRAY_SIZE(tegra114_mipi_pads),
-	.clock_enable_override = true,
-	.needs_vclamp_ref = true,
-	.pad_drive_down_ref = 0x2,
-	.pad_drive_up_ref = 0x0,
-	.pad_vclamp_level = 0x0,
-	.pad_vauxp_level = 0x0,
-	.hspdos = 0x0,
-	.hspuos = 0x4,
-	.termos = 0x5,
-	.hsclkpdos = 0x0,
-	.hsclkpuos = 0x4,
-};
-
-static const struct tegra_mipi_pad tegra124_mipi_pads[] = {
-	{ .data = MIPI_CAL_CONFIG_CSIA, .clk = MIPI_CAL_CONFIG_CSIAB_CLK },
-	{ .data = MIPI_CAL_CONFIG_CSIB, .clk = MIPI_CAL_CONFIG_CSIAB_CLK },
-	{ .data = MIPI_CAL_CONFIG_CSIC, .clk = MIPI_CAL_CONFIG_CSICD_CLK },
-	{ .data = MIPI_CAL_CONFIG_CSID, .clk = MIPI_CAL_CONFIG_CSICD_CLK },
-	{ .data = MIPI_CAL_CONFIG_CSIE, .clk = MIPI_CAL_CONFIG_CSIE_CLK  },
-	{ .data = MIPI_CAL_CONFIG_DSIA, .clk = MIPI_CAL_CONFIG_DSIA_CLK  },
-	{ .data = MIPI_CAL_CONFIG_DSIB, .clk = MIPI_CAL_CONFIG_DSIB_CLK  },
-};
-
-static const struct tegra_mipi_soc tegra124_mipi_soc = {
-	.has_clk_lane = true,
-	.pads = tegra124_mipi_pads,
-	.num_pads = ARRAY_SIZE(tegra124_mipi_pads),
-	.clock_enable_override = true,
-	.needs_vclamp_ref = true,
-	.pad_drive_down_ref = 0x2,
-	.pad_drive_up_ref = 0x0,
-	.pad_vclamp_level = 0x0,
-	.pad_vauxp_level = 0x0,
-	.hspdos = 0x0,
-	.hspuos = 0x0,
-	.termos = 0x0,
-	.hsclkpdos = 0x1,
-	.hsclkpuos = 0x2,
-};
-
-static const struct tegra_mipi_soc tegra132_mipi_soc = {
-	.has_clk_lane = true,
-	.pads = tegra124_mipi_pads,
-	.num_pads = ARRAY_SIZE(tegra124_mipi_pads),
-	.clock_enable_override = false,
-	.needs_vclamp_ref = false,
-	.pad_drive_down_ref = 0x0,
-	.pad_drive_up_ref = 0x3,
-	.pad_vclamp_level = 0x0,
-	.pad_vauxp_level = 0x0,
-	.hspdos = 0x0,
-	.hspuos = 0x0,
-	.termos = 0x0,
-	.hsclkpdos = 0x3,
-	.hsclkpuos = 0x2,
-};
-
-static const struct tegra_mipi_pad tegra210_mipi_pads[] = {
-	{ .data = MIPI_CAL_CONFIG_CSIA, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_CSIB, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_CSIC, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_CSID, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_CSIE, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_CSIF, .clk = 0 },
-	{ .data = MIPI_CAL_CONFIG_DSIA, .clk = MIPI_CAL_CONFIG_DSIA_CLK },
-	{ .data = MIPI_CAL_CONFIG_DSIB, .clk = MIPI_CAL_CONFIG_DSIB_CLK },
-	{ .data = MIPI_CAL_CONFIG_DSIC, .clk = MIPI_CAL_CONFIG_DSIC_CLK },
-	{ .data = MIPI_CAL_CONFIG_DSID, .clk = MIPI_CAL_CONFIG_DSID_CLK },
-};
-
-static const struct tegra_mipi_soc tegra210_mipi_soc = {
-	.has_clk_lane = true,
-	.pads = tegra210_mipi_pads,
-	.num_pads = ARRAY_SIZE(tegra210_mipi_pads),
-	.clock_enable_override = true,
-	.needs_vclamp_ref = false,
-	.pad_drive_down_ref = 0x0,
-	.pad_drive_up_ref = 0x3,
-	.pad_vclamp_level = 0x1,
-	.pad_vauxp_level = 0x1,
-	.hspdos = 0x0,
-	.hspuos = 0x2,
-	.termos = 0x0,
-	.hsclkpdos = 0x0,
-	.hsclkpuos = 0x2,
-};
-
-static const struct of_device_id tegra_mipi_of_match[] = {
-	{ .compatible = "nvidia,tegra114-mipi", .data = &tegra114_mipi_soc },
-	{ .compatible = "nvidia,tegra124-mipi", .data = &tegra124_mipi_soc },
-	{ .compatible = "nvidia,tegra132-mipi", .data = &tegra132_mipi_soc },
-	{ .compatible = "nvidia,tegra210-mipi", .data = &tegra210_mipi_soc },
-	{ },
-};
-
-static int tegra_mipi_probe(struct platform_device *pdev)
+/**
+ * devm_tegra_mipi_add_provider - Managed registration of a Tegra MIPI
+ *				  calibration function provider.
+ * @device: Handle to the device providing the MIPI calibration function.
+ * @np: Device node pointer of the device providing the MIPI calibration
+ *	function.
+ * @ops: Operations supported by the MIPI calibration device.
+ *
+ * This registers a device that provides MIPI calibration functions.
+ * For Tegra20 and Tegra30, this is the CSI block, while Tegra114 and
+ * newer SoC generations have a dedicated hardware block for these
+ * functions.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int devm_tegra_mipi_add_provider(struct device *device, struct device_node *np,
+				 const struct tegra_mipi_ops *ops)
 {
-	const struct of_device_id *match;
-	struct tegra_mipi *mipi;
-
-	match = of_match_node(tegra_mipi_of_match, pdev->dev.of_node);
-	if (!match)
-		return -ENODEV;
+	if (provider.np)
+		return -EBUSY;
 
-	mipi = devm_kzalloc(&pdev->dev, sizeof(*mipi), GFP_KERNEL);
-	if (!mipi)
-		return -ENOMEM;
+	provider.np = np;
+	provider.ops = ops;
 
-	mipi->soc = match->data;
-	mipi->dev = &pdev->dev;
-
-	mipi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
-	if (IS_ERR(mipi->regs))
-		return PTR_ERR(mipi->regs);
-
-	mutex_init(&mipi->lock);
-
-	mipi->clk = devm_clk_get_prepared(&pdev->dev, NULL);
-	if (IS_ERR(mipi->clk)) {
-		dev_err(&pdev->dev, "failed to get clock\n");
-		return PTR_ERR(mipi->clk);
-	}
-
-	platform_set_drvdata(pdev, mipi);
-
-	return 0;
+	return devm_add_action_or_reset(device, tegra_mipi_remove_provider, NULL);
 }
-
-struct platform_driver tegra_mipi_driver = {
-	.driver = {
-		.name = "tegra-mipi",
-		.of_match_table = tegra_mipi_of_match,
-	},
-	.probe = tegra_mipi_probe,
-};
+EXPORT_SYMBOL(devm_tegra_mipi_add_provider);
diff --git a/drivers/gpu/host1x/tegra114-mipi.c b/drivers/gpu/host1x/tegra114-mipi.c
new file mode 100644
index 0000000000000..c084a09784d14
--- /dev/null
+++ b/drivers/gpu/host1x/tegra114-mipi.c
@@ -0,0 +1,483 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/clk.h>
+#include <linux/host1x.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/tegra-mipi-cal.h>
+
+#include "dev.h"
+
+#define MIPI_CAL_CTRL			0x00
+#define MIPI_CAL_CTRL_NOISE_FILTER(x)	(((x) & 0xf) << 26)
+#define MIPI_CAL_CTRL_PRESCALE(x)	(((x) & 0x3) << 24)
+#define MIPI_CAL_CTRL_CLKEN_OVR		BIT(4)
+#define MIPI_CAL_CTRL_START		BIT(0)
+
+#define MIPI_CAL_AUTOCAL_CTRL		0x01
+
+#define MIPI_CAL_STATUS			0x02
+#define MIPI_CAL_STATUS_DONE		BIT(16)
+#define MIPI_CAL_STATUS_ACTIVE		BIT(0)
+
+#define MIPI_CAL_CONFIG_CSIA		0x05
+#define MIPI_CAL_CONFIG_CSIB		0x06
+#define MIPI_CAL_CONFIG_CSIC		0x07
+#define MIPI_CAL_CONFIG_CSID		0x08
+#define MIPI_CAL_CONFIG_CSIE		0x09
+#define MIPI_CAL_CONFIG_CSIF		0x0a
+#define MIPI_CAL_CONFIG_DSIA		0x0e
+#define MIPI_CAL_CONFIG_DSIB		0x0f
+#define MIPI_CAL_CONFIG_DSIC		0x10
+#define MIPI_CAL_CONFIG_DSID		0x11
+
+#define MIPI_CAL_CONFIG_DSIA_CLK	0x19
+#define MIPI_CAL_CONFIG_DSIB_CLK	0x1a
+#define MIPI_CAL_CONFIG_CSIAB_CLK	0x1b
+#define MIPI_CAL_CONFIG_DSIC_CLK	0x1c
+#define MIPI_CAL_CONFIG_CSICD_CLK	0x1c
+#define MIPI_CAL_CONFIG_DSID_CLK	0x1d
+#define MIPI_CAL_CONFIG_CSIE_CLK	0x1d
+
+/* for data and clock lanes */
+#define MIPI_CAL_CONFIG_SELECT		BIT(21)
+
+/* for data lanes */
+#define MIPI_CAL_CONFIG_HSPDOS(x)	(((x) & 0x1f) << 16)
+#define MIPI_CAL_CONFIG_HSPUOS(x)	(((x) & 0x1f) <<  8)
+#define MIPI_CAL_CONFIG_TERMOS(x)	(((x) & 0x1f) <<  0)
+
+/* for clock lanes */
+#define MIPI_CAL_CONFIG_HSCLKPDOSD(x)	(((x) & 0x1f) <<  8)
+#define MIPI_CAL_CONFIG_HSCLKPUOSD(x)	(((x) & 0x1f) <<  0)
+
+#define MIPI_CAL_BIAS_PAD_CFG0		0x16
+#define MIPI_CAL_BIAS_PAD_PDVCLAMP	BIT(1)
+#define MIPI_CAL_BIAS_PAD_E_VCLAMP_REF	BIT(0)
+
+#define MIPI_CAL_BIAS_PAD_CFG1		0x17
+#define MIPI_CAL_BIAS_PAD_DRV_DN_REF(x) (((x) & 0x7) << 16)
+#define MIPI_CAL_BIAS_PAD_DRV_UP_REF(x) (((x) & 0x7) << 8)
+
+#define MIPI_CAL_BIAS_PAD_CFG2		0x18
+#define MIPI_CAL_BIAS_PAD_VCLAMP(x)	(((x) & 0x7) << 16)
+#define MIPI_CAL_BIAS_PAD_VAUXP(x)	(((x) & 0x7) << 4)
+#define MIPI_CAL_BIAS_PAD_PDVREG	BIT(1)
+
+struct tegra_mipi_pad {
+	unsigned long data;
+	unsigned long clk;
+};
+
+struct tegra_mipi_soc {
+	bool has_clk_lane;
+	const struct tegra_mipi_pad *pads;
+	unsigned int num_pads;
+
+	bool clock_enable_override;
+	bool needs_vclamp_ref;
+
+	/* bias pad configuration settings */
+	u8 pad_drive_down_ref;
+	u8 pad_drive_up_ref;
+
+	u8 pad_vclamp_level;
+	u8 pad_vauxp_level;
+
+	/* calibration settings for data lanes */
+	u8 hspdos;
+	u8 hspuos;
+	u8 termos;
+
+	/* calibration settings for clock lanes */
+	u8 hsclkpdos;
+	u8 hsclkpuos;
+};
+
+struct tegra_mipi {
+	const struct tegra_mipi_soc *soc;
+	struct device *dev;
+	void __iomem *regs;
+	struct mutex lock; /* for register access */
+	struct clk *clk;
+
+	unsigned long usage_count;
+};
+
+static inline u32 tegra_mipi_readl(struct tegra_mipi *mipi,
+				   unsigned long offset)
+{
+	return readl(mipi->regs + (offset << 2));
+}
+
+static inline void tegra_mipi_writel(struct tegra_mipi *mipi, u32 value,
+				     unsigned long offset)
+{
+	writel(value, mipi->regs + (offset << 2));
+}
+
+static int tegra114_mipi_power_up(struct tegra_mipi *mipi)
+{
+	u32 value;
+	int err;
+
+	err = clk_enable(mipi->clk);
+	if (err < 0)
+		return err;
+
+	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG0);
+	value &= ~MIPI_CAL_BIAS_PAD_PDVCLAMP;
+
+	if (mipi->soc->needs_vclamp_ref)
+		value |= MIPI_CAL_BIAS_PAD_E_VCLAMP_REF;
+
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG0);
+
+	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG2);
+	value &= ~MIPI_CAL_BIAS_PAD_PDVREG;
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
+
+	clk_disable(mipi->clk);
+
+	return 0;
+}
+
+static int tegra114_mipi_power_down(struct tegra_mipi *mipi)
+{
+	u32 value;
+	int err;
+
+	err = clk_enable(mipi->clk);
+	if (err < 0)
+		return err;
+
+	/*
+	 * The MIPI_CAL_BIAS_PAD_PDVREG controls a voltage regulator that
+	 * supplies the DSI pads. This must be kept enabled until none of the
+	 * DSI lanes are used anymore.
+	 */
+	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG2);
+	value |= MIPI_CAL_BIAS_PAD_PDVREG;
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
+
+	/*
+	 * MIPI_CAL_BIAS_PAD_PDVCLAMP and MIPI_CAL_BIAS_PAD_E_VCLAMP_REF
+	 * control a regulator that supplies current to the pre-driver logic.
+	 * Powering down this regulator causes DSI to fail, so it must remain
+	 * powered on until none of the DSI lanes are used anymore.
+	 */
+	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG0);
+
+	if (mipi->soc->needs_vclamp_ref)
+		value &= ~MIPI_CAL_BIAS_PAD_E_VCLAMP_REF;
+
+	value |= MIPI_CAL_BIAS_PAD_PDVCLAMP;
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG0);
+
+	return 0;
+}
+
+static int tegra114_mipi_enable(struct tegra_mipi_device *mipidev)
+{
+	struct tegra_mipi *mipi = platform_get_drvdata(mipidev->pdev);
+	int err = 0;
+
+	mutex_lock(&mipi->lock);
+
+	if (mipi->usage_count++ == 0)
+		err = tegra114_mipi_power_up(mipi);
+
+	mutex_unlock(&mipi->lock);
+
+	return err;
+}
+
+static int tegra114_mipi_disable(struct tegra_mipi_device *mipidev)
+{
+	struct tegra_mipi *mipi = platform_get_drvdata(mipidev->pdev);
+	int err = 0;
+
+	mutex_lock(&mipi->lock);
+
+	if (--mipi->usage_count == 0)
+		err = tegra114_mipi_power_down(mipi);
+
+	mutex_unlock(&mipi->lock);
+
+	return err;
+}
+
+static int tegra114_mipi_finish_calibration(struct tegra_mipi_device *mipidev)
+{
+	struct tegra_mipi *mipi = platform_get_drvdata(mipidev->pdev);
+	void __iomem *status_reg = mipi->regs + (MIPI_CAL_STATUS << 2);
+	u32 value;
+	int err;
+
+	err = readl_relaxed_poll_timeout(status_reg, value,
+					 !(value & MIPI_CAL_STATUS_ACTIVE) &&
+					 (value & MIPI_CAL_STATUS_DONE), 50,
+					 250000);
+	mutex_unlock(&mipi->lock);
+	clk_disable(mipi->clk);
+
+	return err;
+}
+
+static int tegra114_mipi_start_calibration(struct tegra_mipi_device *mipidev)
+{
+	struct tegra_mipi *mipi = platform_get_drvdata(mipidev->pdev);
+	const struct tegra_mipi_soc *soc = mipi->soc;
+	unsigned int i;
+	u32 value;
+	int err;
+
+	err = clk_enable(mipi->clk);
+	if (err < 0)
+		return err;
+
+	mutex_lock(&mipi->lock);
+
+	value = MIPI_CAL_BIAS_PAD_DRV_DN_REF(soc->pad_drive_down_ref) |
+		MIPI_CAL_BIAS_PAD_DRV_UP_REF(soc->pad_drive_up_ref);
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG1);
+
+	value = tegra_mipi_readl(mipi, MIPI_CAL_BIAS_PAD_CFG2);
+	value &= ~MIPI_CAL_BIAS_PAD_VCLAMP(0x7);
+	value &= ~MIPI_CAL_BIAS_PAD_VAUXP(0x7);
+	value |= MIPI_CAL_BIAS_PAD_VCLAMP(soc->pad_vclamp_level);
+	value |= MIPI_CAL_BIAS_PAD_VAUXP(soc->pad_vauxp_level);
+	tegra_mipi_writel(mipi, value, MIPI_CAL_BIAS_PAD_CFG2);
+
+	for (i = 0; i < soc->num_pads; i++) {
+		u32 clk = 0, data = 0;
+
+		if (mipidev->pads & BIT(i)) {
+			data = MIPI_CAL_CONFIG_SELECT |
+			       MIPI_CAL_CONFIG_HSPDOS(soc->hspdos) |
+			       MIPI_CAL_CONFIG_HSPUOS(soc->hspuos) |
+			       MIPI_CAL_CONFIG_TERMOS(soc->termos);
+			clk = MIPI_CAL_CONFIG_SELECT |
+			      MIPI_CAL_CONFIG_HSCLKPDOSD(soc->hsclkpdos) |
+			      MIPI_CAL_CONFIG_HSCLKPUOSD(soc->hsclkpuos);
+		}
+
+		tegra_mipi_writel(mipi, data, soc->pads[i].data);
+
+		if (soc->has_clk_lane && soc->pads[i].clk != 0)
+			tegra_mipi_writel(mipi, clk, soc->pads[i].clk);
+	}
+
+	value = tegra_mipi_readl(mipi, MIPI_CAL_CTRL);
+	value &= ~MIPI_CAL_CTRL_NOISE_FILTER(0xf);
+	value &= ~MIPI_CAL_CTRL_PRESCALE(0x3);
+	value |= MIPI_CAL_CTRL_NOISE_FILTER(0xa);
+	value |= MIPI_CAL_CTRL_PRESCALE(0x2);
+
+	if (!soc->clock_enable_override)
+		value &= ~MIPI_CAL_CTRL_CLKEN_OVR;
+	else
+		value |= MIPI_CAL_CTRL_CLKEN_OVR;
+
+	tegra_mipi_writel(mipi, value, MIPI_CAL_CTRL);
+
+	/* clear any pending status bits */
+	value = tegra_mipi_readl(mipi, MIPI_CAL_STATUS);
+	tegra_mipi_writel(mipi, value, MIPI_CAL_STATUS);
+
+	value = tegra_mipi_readl(mipi, MIPI_CAL_CTRL);
+	value |= MIPI_CAL_CTRL_START;
+	tegra_mipi_writel(mipi, value, MIPI_CAL_CTRL);
+
+	/*
+	 * Wait for min 72uS to let calibration logic finish calibration
+	 * sequence codes before waiting for pads idle state to apply the
+	 * results.
+	 */
+	usleep_range(75, 80);
+
+	return 0;
+}
+
+static const struct tegra_mipi_ops tegra114_mipi_ops = {
+	.enable = tegra114_mipi_enable,
+	.disable = tegra114_mipi_disable,
+	.start_calibration = tegra114_mipi_start_calibration,
+	.finish_calibration = tegra114_mipi_finish_calibration,
+};
+
+static const struct tegra_mipi_pad tegra114_mipi_pads[] = {
+	{ .data = MIPI_CAL_CONFIG_CSIA },
+	{ .data = MIPI_CAL_CONFIG_CSIB },
+	{ .data = MIPI_CAL_CONFIG_CSIC },
+	{ .data = MIPI_CAL_CONFIG_CSID },
+	{ .data = MIPI_CAL_CONFIG_CSIE },
+	{ .data = MIPI_CAL_CONFIG_DSIA },
+	{ .data = MIPI_CAL_CONFIG_DSIB },
+	{ .data = MIPI_CAL_CONFIG_DSIC },
+	{ .data = MIPI_CAL_CONFIG_DSID },
+};
+
+static const struct tegra_mipi_soc tegra114_mipi_soc = {
+	.has_clk_lane = false,
+	.pads = tegra114_mipi_pads,
+	.num_pads = ARRAY_SIZE(tegra114_mipi_pads),
+	.clock_enable_override = true,
+	.needs_vclamp_ref = true,
+	.pad_drive_down_ref = 0x2,
+	.pad_drive_up_ref = 0x0,
+	.pad_vclamp_level = 0x0,
+	.pad_vauxp_level = 0x0,
+	.hspdos = 0x0,
+	.hspuos = 0x4,
+	.termos = 0x5,
+	.hsclkpdos = 0x0,
+	.hsclkpuos = 0x4,
+};
+
+static const struct tegra_mipi_pad tegra124_mipi_pads[] = {
+	{ .data = MIPI_CAL_CONFIG_CSIA, .clk = MIPI_CAL_CONFIG_CSIAB_CLK },
+	{ .data = MIPI_CAL_CONFIG_CSIB, .clk = MIPI_CAL_CONFIG_CSIAB_CLK },
+	{ .data = MIPI_CAL_CONFIG_CSIC, .clk = MIPI_CAL_CONFIG_CSICD_CLK },
+	{ .data = MIPI_CAL_CONFIG_CSID, .clk = MIPI_CAL_CONFIG_CSICD_CLK },
+	{ .data = MIPI_CAL_CONFIG_CSIE, .clk = MIPI_CAL_CONFIG_CSIE_CLK  },
+	{ .data = MIPI_CAL_CONFIG_DSIA, .clk = MIPI_CAL_CONFIG_DSIA_CLK  },
+	{ .data = MIPI_CAL_CONFIG_DSIB, .clk = MIPI_CAL_CONFIG_DSIB_CLK  },
+};
+
+static const struct tegra_mipi_soc tegra124_mipi_soc = {
+	.has_clk_lane = true,
+	.pads = tegra124_mipi_pads,
+	.num_pads = ARRAY_SIZE(tegra124_mipi_pads),
+	.clock_enable_override = true,
+	.needs_vclamp_ref = true,
+	.pad_drive_down_ref = 0x2,
+	.pad_drive_up_ref = 0x0,
+	.pad_vclamp_level = 0x0,
+	.pad_vauxp_level = 0x0,
+	.hspdos = 0x0,
+	.hspuos = 0x0,
+	.termos = 0x0,
+	.hsclkpdos = 0x1,
+	.hsclkpuos = 0x2,
+};
+
+static const struct tegra_mipi_soc tegra132_mipi_soc = {
+	.has_clk_lane = true,
+	.pads = tegra124_mipi_pads,
+	.num_pads = ARRAY_SIZE(tegra124_mipi_pads),
+	.clock_enable_override = false,
+	.needs_vclamp_ref = false,
+	.pad_drive_down_ref = 0x0,
+	.pad_drive_up_ref = 0x3,
+	.pad_vclamp_level = 0x0,
+	.pad_vauxp_level = 0x0,
+	.hspdos = 0x0,
+	.hspuos = 0x0,
+	.termos = 0x0,
+	.hsclkpdos = 0x3,
+	.hsclkpuos = 0x2,
+};
+
+static const struct tegra_mipi_pad tegra210_mipi_pads[] = {
+	{ .data = MIPI_CAL_CONFIG_CSIA, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_CSIB, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_CSIC, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_CSID, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_CSIE, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_CSIF, .clk = 0 },
+	{ .data = MIPI_CAL_CONFIG_DSIA, .clk = MIPI_CAL_CONFIG_DSIA_CLK },
+	{ .data = MIPI_CAL_CONFIG_DSIB, .clk = MIPI_CAL_CONFIG_DSIB_CLK },
+	{ .data = MIPI_CAL_CONFIG_DSIC, .clk = MIPI_CAL_CONFIG_DSIC_CLK },
+	{ .data = MIPI_CAL_CONFIG_DSID, .clk = MIPI_CAL_CONFIG_DSID_CLK },
+};
+
+static const struct tegra_mipi_soc tegra210_mipi_soc = {
+	.has_clk_lane = true,
+	.pads = tegra210_mipi_pads,
+	.num_pads = ARRAY_SIZE(tegra210_mipi_pads),
+	.clock_enable_override = true,
+	.needs_vclamp_ref = false,
+	.pad_drive_down_ref = 0x0,
+	.pad_drive_up_ref = 0x3,
+	.pad_vclamp_level = 0x1,
+	.pad_vauxp_level = 0x1,
+	.hspdos = 0x0,
+	.hspuos = 0x2,
+	.termos = 0x0,
+	.hsclkpdos = 0x0,
+	.hsclkpuos = 0x2,
+};
+
+static const struct of_device_id tegra_mipi_of_match[] = {
+	{ .compatible = "nvidia,tegra114-mipi", .data = &tegra114_mipi_soc },
+	{ .compatible = "nvidia,tegra124-mipi", .data = &tegra124_mipi_soc },
+	{ .compatible = "nvidia,tegra132-mipi", .data = &tegra132_mipi_soc },
+	{ .compatible = "nvidia,tegra210-mipi", .data = &tegra210_mipi_soc },
+	{ },
+};
+
+static int tegra_mipi_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	struct tegra_mipi *mipi;
+
+	match = of_match_node(tegra_mipi_of_match, pdev->dev.of_node);
+	if (!match)
+		return -ENODEV;
+
+	mipi = devm_kzalloc(&pdev->dev, sizeof(*mipi), GFP_KERNEL);
+	if (!mipi)
+		return -ENOMEM;
+
+	mipi->soc = match->data;
+	mipi->dev = &pdev->dev;
+
+	mipi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
+	if (IS_ERR(mipi->regs))
+		return PTR_ERR(mipi->regs);
+
+	mutex_init(&mipi->lock);
+
+	mipi->clk = devm_clk_get_prepared(&pdev->dev, NULL);
+	if (IS_ERR(mipi->clk)) {
+		dev_err(&pdev->dev, "failed to get clock\n");
+		return PTR_ERR(mipi->clk);
+	}
+
+	platform_set_drvdata(pdev, mipi);
+
+	return devm_tegra_mipi_add_provider(&pdev->dev, pdev->dev.of_node,
+					    &tegra114_mipi_ops);
+}
+
+struct platform_driver tegra_mipi_driver = {
+	.driver = {
+		.name = "tegra-mipi",
+		.of_match_table = tegra_mipi_of_match,
+	},
+	.probe = tegra_mipi_probe,
+};
diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig
index 527920f9c4d39..a4f2380654e2d 100644
--- a/drivers/gpu/nova-core/Kconfig
+++ b/drivers/gpu/nova-core/Kconfig
@@ -3,8 +3,8 @@ config NOVA_CORE
 	depends on 64BIT
 	depends on PCI
 	depends on RUST
-	select RUST_FW_LOADER_ABSTRACTIONS
 	select AUXILIARY_BUS
+	select RUST_FW_LOADER_ABSTRACTIONS
 	default n
 	help
 	  Choose this if you want to build the Nova Core driver for Nvidia
diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 5a4cc047bcfc9..84b0e1703150e 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -14,11 +14,20 @@
     },
     prelude::*,
     sizes::SZ_16M,
-    sync::Arc, //
+    sync::{
+        atomic::{
+            Atomic,
+            Relaxed, //
+        },
+        Arc,
+    },
 };
 
 use crate::gpu::Gpu;
 
+/// Counter for generating unique auxiliary device IDs.
+static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0);
+
 #[pin_data]
 pub(crate) struct NovaCore {
     #[pin]
@@ -70,7 +79,7 @@ impl pci::Driver for NovaCore {
 
     fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<Self, Error> {
         pin_init::pin_init_scope(move || {
-            dev_dbg!(pdev.as_ref(), "Probe Nova Core GPU driver.\n");
+            dev_dbg!(pdev, "Probe Nova Core GPU driver.\n");
 
             pdev.enable_device_mem()?;
             pdev.set_master();
@@ -90,7 +99,9 @@ fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<Self, E
                 _reg <- auxiliary::Registration::new(
                     pdev.as_ref(),
                     c"nova-drm",
-                    0, // TODO[XARR]: Once it lands, use XArray; for now we don't use the ID.
+                    // TODO[XARR]: Use XArray or perhaps IDA for proper ID allocation/recycling. For
+                    // now, use a simple atomic counter that never recycles IDs.
+                    AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed),
                     crate::MODULE_NAME
                 ),
             }))
diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 37bfee1d09492..7097a206ec3ca 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -2,12 +2,13 @@
 
 //! Falcon microprocessor base support
 
-use core::ops::Deref;
-
 use hal::FalconHal;
 
 use kernel::{
-    device,
+    device::{
+        self,
+        Device, //
+    },
     dma::{
         DmaAddress,
         DmaMask, //
@@ -15,9 +16,7 @@
     io::poll::read_poll_timeout,
     prelude::*,
     sync::aref::ARef,
-    time::{
-        Delta, //
-    },
+    time::Delta,
 };
 
 use crate::{
@@ -26,6 +25,7 @@
     falcon::hal::LoadMethod,
     gpu::Chipset,
     num::{
+        self,
         FromSafeCast,
         IntoSafeCast, //
     },
@@ -37,6 +37,9 @@
 mod hal;
 pub(crate) mod sec2;
 
+/// Alignment (in bytes) of falcon memory blocks.
+pub(crate) const MEM_BLOCK_ALIGNMENT: usize = 256;
+
 // TODO[FPRI]: Replace with `ToPrimitive`.
 macro_rules! impl_from_enum_to_u8 {
     ($enum_type:ty) => {
@@ -327,9 +330,10 @@ pub(crate) trait FalconEngine:
     const ID: Self;
 }
 
-/// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM).
+/// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM)
+/// using DMA.
 #[derive(Debug, Clone)]
-pub(crate) struct FalconLoadTarget {
+pub(crate) struct FalconDmaLoadTarget {
     /// Offset from the start of the source object to copy from.
     pub(crate) src_start: u32,
     /// Offset from the start of the destination memory to copy into.
@@ -349,32 +353,155 @@ pub(crate) struct FalconBromParams {
     pub(crate) ucode_id: u8,
 }
 
-/// Trait for providing load parameters of falcon firmwares.
-pub(crate) trait FalconLoadParams {
+/// Trait implemented by falcon firmwares that can be loaded using DMA.
+pub(crate) trait FalconDmaLoadable {
+    /// Returns the firmware data as a slice of bytes.
+    fn as_slice(&self) -> &[u8];
+
     /// Returns the load parameters for Secure `IMEM`.
-    fn imem_sec_load_params(&self) -> FalconLoadTarget;
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget;
 
     /// Returns the load parameters for Non-Secure `IMEM`,
     /// used only on Turing and GA100.
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget>;
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget>;
 
     /// Returns the load parameters for `DMEM`.
-    fn dmem_load_params(&self) -> FalconLoadTarget;
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget;
 
-    /// Returns the parameters to write into the BROM registers.
-    fn brom_params(&self) -> FalconBromParams;
+    /// Returns an adapter that provides the required parameter to load this firmware using PIO.
+    ///
+    /// This can only fail if some `u32` fields cannot be converted to `u16`, or if the indices in
+    /// the headers are invalid.
+    fn try_as_pio_loadable(&self) -> Result<FalconDmaFirmwarePioAdapter<'_, Self>> {
+        let new_pio_imem = |params: FalconDmaLoadTarget, secure| {
+            let start = usize::from_safe_cast(params.src_start);
+            let end = start + usize::from_safe_cast(params.len);
+            let data = self.as_slice().get(start..end).ok_or(EINVAL)?;
+
+            let dst_start = u16::try_from(params.dst_start).map_err(|_| EINVAL)?;
+
+            Ok::<_, Error>(FalconPioImemLoadTarget {
+                data,
+                dst_start,
+                secure,
+                start_tag: dst_start >> 8,
+            })
+        };
 
-    /// Returns the start address of the firmware.
-    fn boot_addr(&self) -> u32;
+        let imem_sec = new_pio_imem(self.imem_sec_load_params(), true)?;
+
+        let imem_ns = if let Some(params) = self.imem_ns_load_params() {
+            Some(new_pio_imem(params, false)?)
+        } else {
+            None
+        };
+
+        let dmem = {
+            let params = self.dmem_load_params();
+            let start = usize::from_safe_cast(params.src_start);
+            let end = start + usize::from_safe_cast(params.len);
+            let data = self.as_slice().get(start..end).ok_or(EINVAL)?;
+
+            let dst_start = u16::try_from(params.dst_start).map_err(|_| EINVAL)?;
+
+            FalconPioDmemLoadTarget { data, dst_start }
+        };
+
+        Ok(FalconDmaFirmwarePioAdapter {
+            fw: self,
+            imem_sec,
+            imem_ns,
+            dmem,
+        })
+    }
+}
+
+/// Represents a portion of the firmware to be loaded into IMEM using PIO.
+#[derive(Clone)]
+pub(crate) struct FalconPioImemLoadTarget<'a> {
+    pub(crate) data: &'a [u8],
+    pub(crate) dst_start: u16,
+    pub(crate) secure: bool,
+    pub(crate) start_tag: u16,
+}
+
+/// Represents a portion of the firmware to be loaded into DMEM using PIO.
+#[derive(Clone)]
+pub(crate) struct FalconPioDmemLoadTarget<'a> {
+    pub(crate) data: &'a [u8],
+    pub(crate) dst_start: u16,
+}
+
+/// Trait for providing PIO load parameters of falcon firmwares.
+pub(crate) trait FalconPioLoadable {
+    /// Returns the load parameters for Secure `IMEM`, if any.
+    fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>>;
+
+    /// Returns the load parameters for Non-Secure `IMEM`, if any.
+    fn imem_ns_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>>;
+
+    /// Returns the load parameters for `DMEM`.
+    fn dmem_load_params(&self) -> FalconPioDmemLoadTarget<'_>;
+}
+
+/// Adapter type that makes any DMA-loadable firmware also loadable via PIO.
+///
+/// Created using [`FalconDmaLoadable::try_as_pio_loadable`].
+pub(crate) struct FalconDmaFirmwarePioAdapter<'a, T: FalconDmaLoadable + ?Sized> {
+    /// Reference to the DMA firmware.
+    fw: &'a T,
+    /// Validated secure IMEM parameters.
+    imem_sec: FalconPioImemLoadTarget<'a>,
+    /// Validated non-secure IMEM parameters.
+    imem_ns: Option<FalconPioImemLoadTarget<'a>>,
+    /// Validated DMEM parameters.
+    dmem: FalconPioDmemLoadTarget<'a>,
+}
+
+impl<'a, T> FalconPioLoadable for FalconDmaFirmwarePioAdapter<'a, T>
+where
+    T: FalconDmaLoadable + ?Sized,
+{
+    fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
+        Some(self.imem_sec.clone())
+    }
+
+    fn imem_ns_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
+        self.imem_ns.clone()
+    }
+
+    fn dmem_load_params(&self) -> FalconPioDmemLoadTarget<'_> {
+        self.dmem.clone()
+    }
+}
+
+impl<'a, T> FalconFirmware for FalconDmaFirmwarePioAdapter<'a, T>
+where
+    T: FalconDmaLoadable + FalconFirmware + ?Sized,
+{
+    type Target = <T as FalconFirmware>::Target;
+
+    fn brom_params(&self) -> FalconBromParams {
+        self.fw.brom_params()
+    }
+
+    fn boot_addr(&self) -> u32 {
+        self.fw.boot_addr()
+    }
 }
 
 /// Trait for a falcon firmware.
 ///
-/// A falcon firmware can be loaded on a given engine, and is presented in the form of a DMA
-/// object.
-pub(crate) trait FalconFirmware: FalconLoadParams + Deref<Target = DmaObject> {
+/// A falcon firmware can be loaded on a given engine.
+pub(crate) trait FalconFirmware {
     /// Engine on which this firmware is to be loaded.
     type Target: FalconEngine;
+
+    /// Returns the parameters to write into the BROM registers.
+    fn brom_params(&self) -> FalconBromParams;
+
+    /// Returns the start address of the firmware.
+    fn boot_addr(&self) -> u32;
 }
 
 /// Contains the base parameters common to all Falcon instances.
@@ -411,18 +538,110 @@ pub(crate) fn reset(&self, bar: &Bar0) -> Result {
         Ok(())
     }
 
+    /// Falcons supports up to four ports, but we only ever use one, so just hard-code it.
+    const PIO_PORT: usize = 0;
+
+    /// Write a slice to Falcon IMEM memory using programmed I/O (PIO).
+    ///
+    /// Returns `EINVAL` if `img.len()` is not a multiple of 4.
+    fn pio_wr_imem_slice(&self, bar: &Bar0, load_offsets: FalconPioImemLoadTarget<'_>) -> Result {
+        // Rejecting misaligned images here allows us to avoid checking
+        // inside the loops.
+        if load_offsets.data.len() % 4 != 0 {
+            return Err(EINVAL);
+        }
+
+        regs::NV_PFALCON_FALCON_IMEMC::default()
+            .set_secure(load_offsets.secure)
+            .set_aincw(true)
+            .set_offs(load_offsets.dst_start)
+            .write(bar, &E::ID, Self::PIO_PORT);
+
+        for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() {
+            let n = u16::try_from(n)?;
+            let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?;
+            regs::NV_PFALCON_FALCON_IMEMT::default().set_tag(tag).write(
+                bar,
+                &E::ID,
+                Self::PIO_PORT,
+            );
+            for word in block.chunks_exact(4) {
+                let w = [word[0], word[1], word[2], word[3]];
+                regs::NV_PFALCON_FALCON_IMEMD::default()
+                    .set_data(u32::from_le_bytes(w))
+                    .write(bar, &E::ID, Self::PIO_PORT);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Write a slice to Falcon DMEM memory using programmed I/O (PIO).
+    ///
+    /// Returns `EINVAL` if `img.len()` is not a multiple of 4.
+    fn pio_wr_dmem_slice(&self, bar: &Bar0, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result {
+        // Rejecting misaligned images here allows us to avoid checking
+        // inside the loops.
+        if load_offsets.data.len() % 4 != 0 {
+            return Err(EINVAL);
+        }
+
+        regs::NV_PFALCON_FALCON_DMEMC::default()
+            .set_aincw(true)
+            .set_offs(load_offsets.dst_start)
+            .write(bar, &E::ID, Self::PIO_PORT);
+
+        for word in load_offsets.data.chunks_exact(4) {
+            let w = [word[0], word[1], word[2], word[3]];
+            regs::NV_PFALCON_FALCON_DMEMD::default()
+                .set_data(u32::from_le_bytes(w))
+                .write(bar, &E::ID, Self::PIO_PORT);
+        }
+
+        Ok(())
+    }
+
+    /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.
+    pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
+        &self,
+        bar: &Bar0,
+        fw: &F,
+    ) -> Result {
+        regs::NV_PFALCON_FBIF_CTL::read(bar, &E::ID)
+            .set_allow_phys_no_ctx(true)
+            .write(bar, &E::ID);
+
+        regs::NV_PFALCON_FALCON_DMACTL::default().write(bar, &E::ID);
+
+        if let Some(imem_ns) = fw.imem_ns_load_params() {
+            self.pio_wr_imem_slice(bar, imem_ns)?;
+        }
+        if let Some(imem_sec) = fw.imem_sec_load_params() {
+            self.pio_wr_imem_slice(bar, imem_sec)?;
+        }
+        self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?;
+
+        self.hal.program_brom(self, bar, &fw.brom_params())?;
+
+        regs::NV_PFALCON_FALCON_BOOTVEC::default()
+            .set_value(fw.boot_addr())
+            .write(bar, &E::ID);
+
+        Ok(())
+    }
+
     /// Perform a DMA write according to `load_offsets` from `dma_handle` into the falcon's
     /// `target_mem`.
     ///
     /// `sec` is set if the loaded firmware is expected to run in secure mode.
-    fn dma_wr<F: FalconFirmware<Target = E>>(
+    fn dma_wr(
         &self,
         bar: &Bar0,
-        fw: &F,
+        dma_obj: &DmaObject,
         target_mem: FalconMem,
-        load_offsets: FalconLoadTarget,
+        load_offsets: FalconDmaLoadTarget,
     ) -> Result {
-        const DMA_LEN: u32 = 256;
+        const DMA_LEN: u32 = num::usize_into_u32::<{ MEM_BLOCK_ALIGNMENT }>();
 
         // For IMEM, we want to use the start offset as a virtual address tag for each page, since
         // code addresses in the firmware (and the boot vector) are virtual.
@@ -430,11 +649,11 @@ fn dma_wr<F: FalconFirmware<Target = E>>(
         // For DMEM we can fold the start offset into the DMA handle.
         let (src_start, dma_start) = match target_mem {
             FalconMem::ImemSecure | FalconMem::ImemNonSecure => {
-                (load_offsets.src_start, fw.dma_handle())
+                (load_offsets.src_start, dma_obj.dma_handle())
             }
             FalconMem::Dmem => (
                 0,
-                fw.dma_handle_with_offset(load_offsets.src_start.into_safe_cast())?,
+                dma_obj.dma_handle_with_offset(load_offsets.src_start.into_safe_cast())?,
             ),
         };
         if dma_start % DmaAddress::from(DMA_LEN) > 0 {
@@ -466,7 +685,7 @@ fn dma_wr<F: FalconFirmware<Target = E>>(
                 dev_err!(self.dev, "DMA transfer length overflow\n");
                 return Err(EOVERFLOW);
             }
-            Some(upper_bound) if usize::from_safe_cast(upper_bound) > fw.size() => {
+            Some(upper_bound) if usize::from_safe_cast(upper_bound) > dma_obj.size() => {
                 dev_err!(self.dev, "DMA transfer goes beyond range of DMA object\n");
                 return Err(EINVAL);
             }
@@ -515,13 +734,14 @@ fn dma_wr<F: FalconFirmware<Target = E>>(
     }
 
     /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.
-    fn dma_load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result {
-        // The Non-Secure section only exists on firmware used by Turing and GA100, and
-        // those platforms do not use DMA.
-        if fw.imem_ns_load_params().is_some() {
-            debug_assert!(false);
-            return Err(EINVAL);
-        }
+    fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
+        &self,
+        dev: &Device<device::Bound>,
+        bar: &Bar0,
+        fw: &F,
+    ) -> Result {
+        // Create DMA object with firmware content as the source of the DMA engine.
+        let dma_obj = DmaObject::from_data(dev, fw.as_slice())?;
 
         self.dma_reset(bar);
         regs::NV_PFALCON_FBIF_TRANSCFG::update(bar, &E::ID, 0, |v| {
@@ -529,8 +749,13 @@ fn dma_load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result
                 .set_mem_type(FalconFbifMemType::Physical)
         });
 
-        self.dma_wr(bar, fw, FalconMem::ImemSecure, fw.imem_sec_load_params())?;
-        self.dma_wr(bar, fw, FalconMem::Dmem, fw.dmem_load_params())?;
+        self.dma_wr(
+            bar,
+            &dma_obj,
+            FalconMem::ImemSecure,
+            fw.imem_sec_load_params(),
+        )?;
+        self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?;
 
         self.hal.program_brom(self, bar, &fw.brom_params())?;
 
@@ -640,11 +865,17 @@ pub(crate) fn is_riscv_active(&self, bar: &Bar0) -> bool {
         self.hal.is_riscv_active(bar)
     }
 
-    // Load a firmware image into Falcon memory
-    pub(crate) fn load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result {
+    /// Load a firmware image into Falcon memory, using the preferred method for the current
+    /// chipset.
+    pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
+        &self,
+        dev: &Device<device::Bound>,
+        bar: &Bar0,
+        fw: &F,
+    ) -> Result {
         match self.hal.load_method() {
-            LoadMethod::Dma => self.dma_load(bar, fw),
-            LoadMethod::Pio => Err(ENOTSUPP),
+            LoadMethod::Dma => self.dma_load(dev, bar, fw),
+            LoadMethod::Pio => self.pio_load(bar, &fw.try_as_pio_loadable()?),
         }
     }
 
diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs
index 89babd5f93257..a7e5ea8d0272c 100644
--- a/drivers/gpu/nova-core/falcon/hal.rs
+++ b/drivers/gpu/nova-core/falcon/hal.rs
@@ -58,7 +58,11 @@ fn signature_reg_fuse_version(
     /// Reset the falcon engine.
     fn reset_eng(&self, bar: &Bar0) -> Result;
 
-    /// returns the method needed to load data into Falcon memory
+    /// Returns the method used to load data into the falcon's memory.
+    ///
+    /// The only chipsets supporting PIO are those < GA102, and PIO is the preferred method for
+    /// these. For anything above, the PIO registers appear to be masked to the CPU, so DMA is the
+    /// only usable method.
     fn load_method(&self) -> LoadMethod;
 }
 
diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs
index c62abcaed547b..6536d0035cb13 100644
--- a/drivers/gpu/nova-core/fb.rs
+++ b/drivers/gpu/nova-core/fb.rs
@@ -1,9 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use core::ops::Range;
+use core::ops::{
+    Deref,
+    Range, //
+};
 
 use kernel::{
     device,
+    fmt,
     prelude::*,
     ptr::{
         Alignable,
@@ -94,26 +98,77 @@ pub(crate) fn unregister(&self, bar: &Bar0) {
     }
 }
 
+pub(crate) struct FbRange(Range<u64>);
+
+impl FbRange {
+    pub(crate) fn len(&self) -> u64 {
+        self.0.end - self.0.start
+    }
+}
+
+impl From<Range<u64>> for FbRange {
+    fn from(range: Range<u64>) -> Self {
+        Self(range)
+    }
+}
+
+impl Deref for FbRange {
+    type Target = Range<u64>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl fmt::Debug for FbRange {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Use alternate format ({:#?}) to include size, compact format ({:?}) for just the range.
+        if f.alternate() {
+            let size = self.len();
+
+            if size < usize_as_u64(SZ_1M) {
+                let size_kib = size / usize_as_u64(SZ_1K);
+                f.write_fmt(fmt!(
+                    "{:#x}..{:#x} ({} KiB)",
+                    self.0.start,
+                    self.0.end,
+                    size_kib
+                ))
+            } else {
+                let size_mib = size / usize_as_u64(SZ_1M);
+                f.write_fmt(fmt!(
+                    "{:#x}..{:#x} ({} MiB)",
+                    self.0.start,
+                    self.0.end,
+                    size_mib
+                ))
+            }
+        } else {
+            f.write_fmt(fmt!("{:#x}..{:#x}", self.0.start, self.0.end))
+        }
+    }
+}
+
 /// Layout of the GPU framebuffer memory.
 ///
 /// Contains ranges of GPU memory reserved for a given purpose during the GSP boot process.
 #[derive(Debug)]
 pub(crate) struct FbLayout {
     /// Range of the framebuffer. Starts at `0`.
-    pub(crate) fb: Range<u64>,
+    pub(crate) fb: FbRange,
     /// VGA workspace, small area of reserved memory at the end of the framebuffer.
-    pub(crate) vga_workspace: Range<u64>,
+    pub(crate) vga_workspace: FbRange,
     /// FRTS range.
-    pub(crate) frts: Range<u64>,
+    pub(crate) frts: FbRange,
     /// Memory area containing the GSP bootloader image.
-    pub(crate) boot: Range<u64>,
+    pub(crate) boot: FbRange,
     /// Memory area containing the GSP firmware image.
-    pub(crate) elf: Range<u64>,
+    pub(crate) elf: FbRange,
     /// WPR2 heap.
-    pub(crate) wpr2_heap: Range<u64>,
+    pub(crate) wpr2_heap: FbRange,
     /// WPR2 region range, starting with an instance of `GspFwWprMeta`.
-    pub(crate) wpr2: Range<u64>,
-    pub(crate) heap: Range<u64>,
+    pub(crate) wpr2: FbRange,
+    pub(crate) heap: FbRange,
     pub(crate) vf_partition_count: u8,
 }
 
@@ -125,7 +180,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
         let fb = {
             let fb_size = hal.vidmem_size(bar);
 
-            0..fb_size
+            FbRange(0..fb_size)
         };
 
         let vga_workspace = {
@@ -152,7 +207,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
                 }
             };
 
-            vga_base..fb.end
+            FbRange(vga_base..fb.end)
         };
 
         let frts = {
@@ -160,7 +215,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
             const FRTS_SIZE: u64 = usize_as_u64(SZ_1M);
             let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE;
 
-            frts_base..frts_base + FRTS_SIZE
+            FbRange(frts_base..frts_base + FRTS_SIZE)
         };
 
         let boot = {
@@ -168,7 +223,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
             let bootloader_size = u64::from_safe_cast(gsp_fw.bootloader.ucode.size());
             let bootloader_base = (frts.start - bootloader_size).align_down(BOOTLOADER_DOWN_ALIGN);
 
-            bootloader_base..bootloader_base + bootloader_size
+            FbRange(bootloader_base..bootloader_base + bootloader_size)
         };
 
         let elf = {
@@ -176,7 +231,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
             let elf_size = u64::from_safe_cast(gsp_fw.size);
             let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN);
 
-            elf_addr..elf_addr + elf_size
+            FbRange(elf_addr..elf_addr + elf_size)
         };
 
         let wpr2_heap = {
@@ -185,7 +240,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
                 gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end);
             let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN);
 
-            wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)
+            FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN))
         };
 
         let wpr2 = {
@@ -193,13 +248,13 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
             let wpr2_addr = (wpr2_heap.start - u64::from_safe_cast(size_of::<gsp::GspFwWprMeta>()))
                 .align_down(WPR2_DOWN_ALIGN);
 
-            wpr2_addr..frts.end
+            FbRange(wpr2_addr..frts.end)
         };
 
         let heap = {
             const HEAP_SIZE: u64 = usize_as_u64(SZ_1M);
 
-            wpr2.start - HEAP_SIZE..wpr2.start
+            FbRange(wpr2.start - HEAP_SIZE..wpr2.start)
         };
 
         Ok(Self {
diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs
index 68779540aa284..2bb20081befd0 100644
--- a/drivers/gpu/nova-core/firmware.rs
+++ b/drivers/gpu/nova-core/firmware.rs
@@ -15,10 +15,9 @@
 };
 
 use crate::{
-    dma::DmaObject,
     falcon::{
-        FalconFirmware,
-        FalconLoadTarget, //
+        FalconDmaLoadTarget,
+        FalconFirmware, //
     },
     gpu,
     num::{
@@ -64,7 +63,8 @@ pub(crate) struct FalconUCodeDescV2 {
     pub(crate) interface_offset: u32,
     /// Base address at which to load the code segment into 'IMEM'.
     pub(crate) imem_phys_base: u32,
-    /// Size in bytes of the code to copy into 'IMEM'.
+    /// Size in bytes of the code to copy into 'IMEM' (includes both secure and non-secure
+    /// segments).
     pub(crate) imem_load_size: u32,
     /// Virtual 'IMEM' address (i.e. 'tag') at which the code should start.
     pub(crate) imem_virt_base: u32,
@@ -171,9 +171,9 @@ fn size(&self) -> usize {
         ((hdr & HDR_SIZE_MASK) >> HDR_SIZE_SHIFT).into_safe_cast()
     }
 
-    fn imem_sec_load_params(&self) -> FalconLoadTarget;
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget>;
-    fn dmem_load_params(&self) -> FalconLoadTarget;
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget;
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget>;
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget;
 }
 
 impl FalconUCodeDescriptor for FalconUCodeDescV2 {
@@ -205,24 +205,31 @@ fn signature_versions(&self) -> u16 {
         0
     }
 
-    fn imem_sec_load_params(&self) -> FalconLoadTarget {
-        FalconLoadTarget {
-            src_start: 0,
-            dst_start: self.imem_sec_base,
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget {
+        // `imem_sec_base` is the *virtual* start address of the secure IMEM segment, so subtract
+        // `imem_virt_base` to get its physical offset.
+        let imem_sec_start = self.imem_sec_base.saturating_sub(self.imem_virt_base);
+
+        FalconDmaLoadTarget {
+            src_start: imem_sec_start,
+            dst_start: self.imem_phys_base.saturating_add(imem_sec_start),
             len: self.imem_sec_size,
         }
     }
 
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> {
-        Some(FalconLoadTarget {
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget> {
+        Some(FalconDmaLoadTarget {
+            // Non-secure code always starts at offset 0.
             src_start: 0,
             dst_start: self.imem_phys_base,
-            len: self.imem_load_size.checked_sub(self.imem_sec_size)?,
+            // `imem_load_size` includes the size of the secure segment, so subtract it to
+            // get the correct amount of data to copy.
+            len: self.imem_load_size.saturating_sub(self.imem_sec_size),
         })
     }
 
-    fn dmem_load_params(&self) -> FalconLoadTarget {
-        FalconLoadTarget {
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget {
+        FalconDmaLoadTarget {
             src_start: self.dmem_offset,
             dst_start: self.dmem_phys_base,
             len: self.dmem_load_size,
@@ -259,21 +266,23 @@ fn signature_versions(&self) -> u16 {
         self.signature_versions
     }
 
-    fn imem_sec_load_params(&self) -> FalconLoadTarget {
-        FalconLoadTarget {
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget {
+        FalconDmaLoadTarget {
+            // IMEM segment always starts at offset 0.
             src_start: 0,
             dst_start: self.imem_phys_base,
             len: self.imem_load_size,
         }
     }
 
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> {
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget> {
         // Not used on V3 platforms
         None
     }
 
-    fn dmem_load_params(&self) -> FalconLoadTarget {
-        FalconLoadTarget {
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget {
+        FalconDmaLoadTarget {
+            // DMEM segment starts right after the IMEM one.
             src_start: self.imem_load_size,
             dst_start: self.dmem_phys_base,
             len: self.dmem_load_size,
@@ -292,7 +301,7 @@ impl SignedState for Unsigned {}
 struct Signed;
 impl SignedState for Signed {}
 
-/// A [`DmaObject`] containing a specific microcode ready to be loaded into a falcon.
+/// Microcode to be loaded into a specific falcon.
 ///
 /// This is module-local and meant for sub-modules to use internally.
 ///
@@ -300,34 +309,35 @@ impl SignedState for Signed {}
 /// before it can be loaded (with an exception for development hardware). The
 /// [`Self::patch_signature`] and [`Self::no_patch_signature`] methods are used to transition the
 /// firmware to its [`Signed`] state.
-struct FirmwareDmaObject<F: FalconFirmware, S: SignedState>(DmaObject, PhantomData<(F, S)>);
+// TODO: Consider replacing this with a coherent memory object once `CoherentAllocation` supports
+// temporary CPU-exclusive access to the object without unsafe methods.
+struct FirmwareObject<F: FalconFirmware, S: SignedState>(KVVec<u8>, PhantomData<(F, S)>);
 
 /// Trait for signatures to be patched directly into a given firmware.
 ///
 /// This is module-local and meant for sub-modules to use internally.
 trait FirmwareSignature<F: FalconFirmware>: AsRef<[u8]> {}
 
-impl<F: FalconFirmware> FirmwareDmaObject<F, Unsigned> {
-    /// Patches the firmware at offset `sig_base_img` with `signature`.
+impl<F: FalconFirmware> FirmwareObject<F, Unsigned> {
+    /// Patches the firmware at offset `signature_start` with `signature`.
     fn patch_signature<S: FirmwareSignature<F>>(
         mut self,
         signature: &S,
-        sig_base_img: usize,
-    ) -> Result<FirmwareDmaObject<F, Signed>> {
+        signature_start: usize,
+    ) -> Result<FirmwareObject<F, Signed>> {
         let signature_bytes = signature.as_ref();
-        if sig_base_img + signature_bytes.len() > self.0.size() {
-            return Err(EINVAL);
-        }
-
-        // SAFETY: We are the only user of this object, so there cannot be any race.
-        let dst = unsafe { self.0.start_ptr_mut().add(sig_base_img) };
+        let signature_end = signature_start
+            .checked_add(signature_bytes.len())
+            .ok_or(EOVERFLOW)?;
+        let dst = self
+            .0
+            .get_mut(signature_start..signature_end)
+            .ok_or(EINVAL)?;
 
-        // SAFETY: `signature` and `dst` are valid, properly aligned, and do not overlap.
-        unsafe {
-            core::ptr::copy_nonoverlapping(signature_bytes.as_ptr(), dst, signature_bytes.len())
-        };
+        // PANIC: `dst` and `signature_bytes` have the same length.
+        dst.copy_from_slice(signature_bytes);
 
-        Ok(FirmwareDmaObject(self.0, PhantomData))
+        Ok(FirmwareObject(self.0, PhantomData))
     }
 
     /// Mark the firmware as signed without patching it.
@@ -335,8 +345,8 @@ fn patch_signature<S: FirmwareSignature<F>>(
     /// This method is used to explicitly confirm that we do not need to sign the firmware, while
     /// allowing us to continue as if it was. This is typically only needed for development
     /// hardware.
-    fn no_patch_signature(self) -> FirmwareDmaObject<F, Signed> {
-        FirmwareDmaObject(self.0, PhantomData)
+    fn no_patch_signature(self) -> FirmwareObject<F, Signed> {
+        FirmwareObject(self.0, PhantomData)
     }
 }
 
@@ -394,8 +404,9 @@ fn new(fw: &'a firmware::Firmware) -> Result<Self> {
     fn data(&self) -> Option<&[u8]> {
         let fw_start = usize::from_safe_cast(self.hdr.data_offset);
         let fw_size = usize::from_safe_cast(self.hdr.data_size);
+        let fw_end = fw_start.checked_add(fw_size)?;
 
-        self.fw.get(fw_start..fw_start + fw_size)
+        self.fw.get(fw_start..fw_end)
     }
 }
 
@@ -416,21 +427,30 @@ const fn make_entry_file(self, chipset: &str, fw: &str) -> Self {
         )
     }
 
-    const fn make_entry_chipset(self, chipset: &str) -> Self {
-        self.make_entry_file(chipset, "booter_load")
-            .make_entry_file(chipset, "booter_unload")
-            .make_entry_file(chipset, "bootloader")
-            .make_entry_file(chipset, "gsp")
+    const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self {
+        let name = chipset.name();
+
+        let this = self
+            .make_entry_file(name, "booter_load")
+            .make_entry_file(name, "booter_unload")
+            .make_entry_file(name, "bootloader")
+            .make_entry_file(name, "gsp");
+
+        if chipset.needs_fwsec_bootloader() {
+            this.make_entry_file(name, "gen_bootloader")
+        } else {
+            this
+        }
     }
 
     pub(crate) const fn create(
-        module_name: &'static kernel::str::CStr,
+        module_name: &'static core::ffi::CStr,
     ) -> firmware::ModInfoBuilder<N> {
         let mut this = Self(firmware::ModInfoBuilder::new(module_name));
         let mut i = 0;
 
         while i < gpu::Chipset::ALL.len() {
-            this = this.make_entry_chipset(gpu::Chipset::ALL[i].name());
+            this = this.make_entry_chipset(gpu::Chipset::ALL[i]);
             i += 1;
         }
 
diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs
index 86556cee8e67b..de2a4536b5320 100644
--- a/drivers/gpu/nova-core/firmware/booter.rs
+++ b/drivers/gpu/nova-core/firmware/booter.rs
@@ -4,10 +4,7 @@
 //! running on [`Sec2`], that is used on Turing/Ampere to load the GSP firmware into the GSP falcon
 //! (and optionally unload it through a separate firmware image).
 
-use core::{
-    marker::PhantomData,
-    ops::Deref, //
-};
+use core::marker::PhantomData;
 
 use kernel::{
     device,
@@ -16,19 +13,18 @@
 };
 
 use crate::{
-    dma::DmaObject,
     driver::Bar0,
     falcon::{
         sec2::Sec2,
         Falcon,
         FalconBromParams,
-        FalconFirmware,
-        FalconLoadParams,
-        FalconLoadTarget, //
+        FalconDmaLoadTarget,
+        FalconDmaLoadable,
+        FalconFirmware, //
     },
     firmware::{
         BinFirmware,
-        FirmwareDmaObject,
+        FirmwareObject,
         FirmwareSignature,
         Signed,
         Unsigned, //
@@ -43,8 +39,9 @@
 /// Local convenience function to return a copy of `S` by reinterpreting the bytes starting at
 /// `offset` in `slice`.
 fn frombytes_at<S: FromBytes + Sized>(slice: &[u8], offset: usize) -> Result<S> {
+    let end = offset.checked_add(size_of::<S>()).ok_or(EINVAL)?;
     slice
-        .get(offset..offset + size_of::<S>())
+        .get(offset..end)
         .and_then(S::from_bytes_copy)
         .ok_or(EINVAL)
 }
@@ -119,14 +116,21 @@ fn signatures_iter(&'a self) -> Result<impl Iterator<Item = BooterSignature<'a>>
             Some(sig_size) => {
                 let patch_sig =
                     frombytes_at::<u32>(self.fw, self.hdr.patch_sig_offset.into_safe_cast())?;
-                let signatures_start = usize::from_safe_cast(self.hdr.sig_prod_offset + patch_sig);
+
+                let signatures_start = self
+                    .hdr
+                    .sig_prod_offset
+                    .checked_add(patch_sig)
+                    .map(usize::from_safe_cast)
+                    .ok_or(EINVAL)?;
+
+                let signatures_end = signatures_start
+                    .checked_add(usize::from_safe_cast(self.hdr.sig_prod_size))
+                    .ok_or(EINVAL)?;
 
                 self.fw
                     // Get signatures range.
-                    .get(
-                        signatures_start
-                            ..signatures_start + usize::from_safe_cast(self.hdr.sig_prod_size),
-                    )
+                    .get(signatures_start..signatures_end)
                     .ok_or(EINVAL)?
                     .chunks_exact(sig_size.into_safe_cast())
             }
@@ -252,21 +256,24 @@ impl<'a> FirmwareSignature<BooterFirmware> for BooterSignature<'a> {}
 /// The `Booter` loader firmware, responsible for loading the GSP.
 pub(crate) struct BooterFirmware {
     // Load parameters for Secure `IMEM` falcon memory.
-    imem_sec_load_target: FalconLoadTarget,
+    imem_sec_load_target: FalconDmaLoadTarget,
     // Load parameters for Non-Secure `IMEM` falcon memory,
     // used only on Turing and GA100
-    imem_ns_load_target: Option<FalconLoadTarget>,
+    imem_ns_load_target: Option<FalconDmaLoadTarget>,
     // Load parameters for `DMEM` falcon memory.
-    dmem_load_target: FalconLoadTarget,
+    dmem_load_target: FalconDmaLoadTarget,
     // BROM falcon parameters.
     brom_params: FalconBromParams,
     // Device-mapped firmware image.
-    ucode: FirmwareDmaObject<Self, Signed>,
+    ucode: FirmwareObject<Self, Signed>,
 }
 
-impl FirmwareDmaObject<BooterFirmware, Unsigned> {
-    fn new_booter(dev: &device::Device<device::Bound>, data: &[u8]) -> Result<Self> {
-        DmaObject::from_data(dev, data).map(|ucode| Self(ucode, PhantomData))
+impl FirmwareObject<BooterFirmware, Unsigned> {
+    fn new_booter(data: &[u8]) -> Result<Self> {
+        let mut ucode = KVVec::new();
+        ucode.extend_from_slice(data, GFP_KERNEL)?;
+
+        Ok(Self(ucode, PhantomData))
     }
 }
 
@@ -320,7 +327,7 @@ pub(crate) fn new(
         let ucode = bin_fw
             .data()
             .ok_or(EINVAL)
-            .and_then(|data| FirmwareDmaObject::<Self, _>::new_booter(dev, data))?;
+            .and_then(FirmwareObject::<Self, _>::new_booter)?;
 
         let ucode_signed = {
             let mut signatures = hs_fw.signatures_iter()?.peekable();
@@ -363,7 +370,7 @@ pub(crate) fn new(
         let (imem_sec_dst_start, imem_ns_load_target) = if chipset <= Chipset::GA100 {
             (
                 app0.offset,
-                Some(FalconLoadTarget {
+                Some(FalconDmaLoadTarget {
                     src_start: 0,
                     dst_start: load_hdr.os_code_offset,
                     len: load_hdr.os_code_size,
@@ -374,13 +381,13 @@ pub(crate) fn new(
         };
 
         Ok(Self {
-            imem_sec_load_target: FalconLoadTarget {
+            imem_sec_load_target: FalconDmaLoadTarget {
                 src_start: app0.offset,
                 dst_start: imem_sec_dst_start,
                 len: app0.len,
             },
             imem_ns_load_target,
-            dmem_load_target: FalconLoadTarget {
+            dmem_load_target: FalconDmaLoadTarget {
                 src_start: load_hdr.os_data_offset,
                 dst_start: 0,
                 len: load_hdr.os_data_size,
@@ -391,18 +398,26 @@ pub(crate) fn new(
     }
 }
 
-impl FalconLoadParams for BooterFirmware {
-    fn imem_sec_load_params(&self) -> FalconLoadTarget {
+impl FalconDmaLoadable for BooterFirmware {
+    fn as_slice(&self) -> &[u8] {
+        self.ucode.0.as_slice()
+    }
+
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget {
         self.imem_sec_load_target.clone()
     }
 
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> {
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget> {
         self.imem_ns_load_target.clone()
     }
 
-    fn dmem_load_params(&self) -> FalconLoadTarget {
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget {
         self.dmem_load_target.clone()
     }
+}
+
+impl FalconFirmware for BooterFirmware {
+    type Target = Sec2;
 
     fn brom_params(&self) -> FalconBromParams {
         self.brom_params.clone()
@@ -416,15 +431,3 @@ fn boot_addr(&self) -> u32 {
         }
     }
 }
-
-impl Deref for BooterFirmware {
-    type Target = DmaObject;
-
-    fn deref(&self) -> &Self::Target {
-        &self.ucode.0
-    }
-}
-
-impl FalconFirmware for BooterFirmware {
-    type Target = Sec2;
-}
diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs
index bfb7b06b13d15..8810cb49db672 100644
--- a/drivers/gpu/nova-core/firmware/fwsec.rs
+++ b/drivers/gpu/nova-core/firmware/fwsec.rs
@@ -10,10 +10,9 @@
 //! - The command to be run, as this firmware can perform several tasks ;
 //! - The ucode signature, so the GSP falcon can run FWSEC in HS mode.
 
-use core::{
-    marker::PhantomData,
-    ops::Deref, //
-};
+pub(crate) mod bootloader;
+
+use core::marker::PhantomData;
 
 use kernel::{
     device::{
@@ -28,27 +27,23 @@
 };
 
 use crate::{
-    dma::DmaObject,
     driver::Bar0,
     falcon::{
         gsp::Gsp,
         Falcon,
         FalconBromParams,
-        FalconFirmware,
-        FalconLoadParams,
-        FalconLoadTarget, //
+        FalconDmaLoadTarget,
+        FalconDmaLoadable,
+        FalconFirmware, //
     },
     firmware::{
         FalconUCodeDesc,
-        FirmwareDmaObject,
+        FirmwareObject,
         FirmwareSignature,
         Signed,
         Unsigned, //
     },
-    num::{
-        FromSafeCast,
-        IntoSafeCast, //
-    },
+    num::FromSafeCast,
     vbios::Vbios,
 };
 
@@ -177,63 +172,36 @@ fn as_ref(&self) -> &[u8] {
 
 impl FirmwareSignature<FwsecFirmware> for Bcrt30Rsa3kSignature {}
 
-/// Reinterpret the area starting from `offset` in `fw` as an instance of `T` (which must implement
-/// [`FromBytes`]) and return a reference to it.
-///
-/// # Safety
-///
-/// * Callers must ensure that the device does not read/write to/from memory while the returned
-///   reference is live.
-/// * Callers must ensure that this call does not race with a write to the same region while
-///   the returned reference is live.
-unsafe fn transmute<T: Sized + FromBytes>(fw: &DmaObject, offset: usize) -> Result<&T> {
-    // SAFETY: The safety requirements of the function guarantee the device won't read
-    // or write to memory while the reference is alive and that this call won't race
-    // with writes to the same memory region.
-    T::from_bytes(unsafe { fw.as_slice(offset, size_of::<T>())? }).ok_or(EINVAL)
-}
-
-/// Reinterpret the area starting from `offset` in `fw` as a mutable instance of `T` (which must
-/// implement [`FromBytes`]) and return a reference to it.
-///
-/// # Safety
-///
-/// * Callers must ensure that the device does not read/write to/from memory while the returned
-///   slice is live.
-/// * Callers must ensure that this call does not race with a read or write to the same region
-///   while the returned slice is live.
-unsafe fn transmute_mut<T: Sized + FromBytes + AsBytes>(
-    fw: &mut DmaObject,
-    offset: usize,
-) -> Result<&mut T> {
-    // SAFETY: The safety requirements of the function guarantee the device won't read
-    // or write to memory while the reference is alive and that this call won't race
-    // with writes or reads to the same memory region.
-    T::from_bytes_mut(unsafe { fw.as_slice_mut(offset, size_of::<T>())? }).ok_or(EINVAL)
-}
-
 /// The FWSEC microcode, extracted from the BIOS and to be run on the GSP falcon.
 ///
 /// It is responsible for e.g. carving out the WPR2 region as the first step of the GSP bootflow.
 pub(crate) struct FwsecFirmware {
     /// Descriptor of the firmware.
     desc: FalconUCodeDesc,
-    /// GPU-accessible DMA object containing the firmware.
-    ucode: FirmwareDmaObject<Self, Signed>,
+    /// Object containing the firmware binary.
+    ucode: FirmwareObject<Self, Signed>,
 }
 
-impl FalconLoadParams for FwsecFirmware {
-    fn imem_sec_load_params(&self) -> FalconLoadTarget {
+impl FalconDmaLoadable for FwsecFirmware {
+    fn as_slice(&self) -> &[u8] {
+        self.ucode.0.as_slice()
+    }
+
+    fn imem_sec_load_params(&self) -> FalconDmaLoadTarget {
         self.desc.imem_sec_load_params()
     }
 
-    fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> {
+    fn imem_ns_load_params(&self) -> Option<FalconDmaLoadTarget> {
         self.desc.imem_ns_load_params()
     }
 
-    fn dmem_load_params(&self) -> FalconLoadTarget {
+    fn dmem_load_params(&self) -> FalconDmaLoadTarget {
         self.desc.dmem_load_params()
     }
+}
+
+impl FalconFirmware for FwsecFirmware {
+    type Target = Gsp;
 
     fn brom_params(&self) -> FalconBromParams {
         FalconBromParams {
@@ -248,27 +216,23 @@ fn boot_addr(&self) -> u32 {
     }
 }
 
-impl Deref for FwsecFirmware {
-    type Target = DmaObject;
-
-    fn deref(&self) -> &Self::Target {
-        &self.ucode.0
-    }
-}
-
-impl FalconFirmware for FwsecFirmware {
-    type Target = Gsp;
-}
-
-impl FirmwareDmaObject<FwsecFirmware, Unsigned> {
-    fn new_fwsec(dev: &Device<device::Bound>, bios: &Vbios, cmd: FwsecCommand) -> Result<Self> {
+impl FirmwareObject<FwsecFirmware, Unsigned> {
+    fn new_fwsec(bios: &Vbios, cmd: FwsecCommand) -> Result<Self> {
         let desc = bios.fwsec_image().header()?;
-        let ucode = bios.fwsec_image().ucode(&desc)?;
-        let mut dma_object = DmaObject::from_data(dev, ucode)?;
+        let mut ucode = KVVec::new();
+        ucode.extend_from_slice(bios.fwsec_image().ucode(&desc)?, GFP_KERNEL)?;
 
-        let hdr_offset = usize::from_safe_cast(desc.imem_load_size() + desc.interface_offset());
-        // SAFETY: we have exclusive access to `dma_object`.
-        let hdr: &FalconAppifHdrV1 = unsafe { transmute(&dma_object, hdr_offset) }?;
+        let hdr_offset = desc
+            .imem_load_size()
+            .checked_add(desc.interface_offset())
+            .map(usize::from_safe_cast)
+            .ok_or(EINVAL)?;
+
+        let hdr = ucode
+            .get(hdr_offset..)
+            .and_then(FalconAppifHdrV1::from_bytes_prefix)
+            .ok_or(EINVAL)?
+            .0;
 
         if hdr.version != 1 {
             return Err(EINVAL);
@@ -276,26 +240,34 @@ fn new_fwsec(dev: &Device<device::Bound>, bios: &Vbios, cmd: FwsecCommand) -> Re
 
         // Find the DMEM mapper section in the firmware.
         for i in 0..usize::from(hdr.entry_count) {
-            // SAFETY: we have exclusive access to `dma_object`.
-            let app: &FalconAppifV1 = unsafe {
-                transmute(
-                    &dma_object,
-                    hdr_offset + usize::from(hdr.header_size) + i * usize::from(hdr.entry_size),
-                )
-            }?;
+            // CALC: hdr_offset + header_size + i * entry_size.
+            let entry_offset = hdr_offset
+                .checked_add(usize::from(hdr.header_size))
+                .and_then(|o| o.checked_add(i.checked_mul(usize::from(hdr.entry_size))?))
+                .ok_or(EINVAL)?;
+
+            let app = ucode
+                .get(entry_offset..)
+                .and_then(FalconAppifV1::from_bytes_prefix)
+                .ok_or(EINVAL)?
+                .0;
 
             if app.id != NVFW_FALCON_APPIF_ID_DMEMMAPPER {
                 continue;
             }
             let dmem_base = app.dmem_base;
 
-            // SAFETY: we have exclusive access to `dma_object`.
-            let dmem_mapper: &mut FalconAppifDmemmapperV3 = unsafe {
-                transmute_mut(
-                    &mut dma_object,
-                    (desc.imem_load_size() + dmem_base).into_safe_cast(),
-                )
-            }?;
+            let dmem_mapper_offset = desc
+                .imem_load_size()
+                .checked_add(dmem_base)
+                .map(usize::from_safe_cast)
+                .ok_or(EINVAL)?;
+
+            let dmem_mapper = ucode
+                .get_mut(dmem_mapper_offset..)
+                .and_then(FalconAppifDmemmapperV3::from_bytes_mut_prefix)
+                .ok_or(EINVAL)?
+                .0;
 
             dmem_mapper.init_cmd = match cmd {
                 FwsecCommand::Frts { .. } => NVFW_FALCON_APPIF_DMEMMAPPER_CMD_FRTS,
@@ -303,13 +275,17 @@ fn new_fwsec(dev: &Device<device::Bound>, bios: &Vbios, cmd: FwsecCommand) -> Re
             };
             let cmd_in_buffer_offset = dmem_mapper.cmd_in_buffer_offset;
 
-            // SAFETY: we have exclusive access to `dma_object`.
-            let frts_cmd: &mut FrtsCmd = unsafe {
-                transmute_mut(
-                    &mut dma_object,
-                    (desc.imem_load_size() + cmd_in_buffer_offset).into_safe_cast(),
-                )
-            }?;
+            let frts_cmd_offset = desc
+                .imem_load_size()
+                .checked_add(cmd_in_buffer_offset)
+                .map(usize::from_safe_cast)
+                .ok_or(EINVAL)?;
+
+            let frts_cmd = ucode
+                .get_mut(frts_cmd_offset..)
+                .and_then(FrtsCmd::from_bytes_mut_prefix)
+                .ok_or(EINVAL)?
+                .0;
 
             frts_cmd.read_vbios = ReadVbios {
                 ver: 1,
@@ -333,7 +309,7 @@ fn new_fwsec(dev: &Device<device::Bound>, bios: &Vbios, cmd: FwsecCommand) -> Re
             }
 
             // Return early as we found and patched the DMEMMAPPER region.
-            return Ok(Self(dma_object, PhantomData));
+            return Ok(Self(ucode, PhantomData));
         }
 
         Err(ENOTSUPP)
@@ -350,13 +326,16 @@ pub(crate) fn new(
         bios: &Vbios,
         cmd: FwsecCommand,
     ) -> Result<Self> {
-        let ucode_dma = FirmwareDmaObject::<Self, _>::new_fwsec(dev, bios, cmd)?;
+        let ucode_dma = FirmwareObject::<Self, _>::new_fwsec(bios, cmd)?;
 
         // Patch signature if needed.
         let desc = bios.fwsec_image().header()?;
         let ucode_signed = if desc.signature_count() != 0 {
-            let sig_base_img =
-                usize::from_safe_cast(desc.imem_load_size() + desc.pkc_data_offset());
+            let sig_base_img = desc
+                .imem_load_size()
+                .checked_add(desc.pkc_data_offset())
+                .map(usize::from_safe_cast)
+                .ok_or(EINVAL)?;
             let desc_sig_versions = u32::from(desc.signature_versions());
             let reg_fuse_version =
                 falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?;
@@ -408,6 +387,10 @@ pub(crate) fn new(
     }
 
     /// Loads the FWSEC firmware into `falcon` and execute it.
+    ///
+    /// This must only be called on chipsets that do not need the FWSEC bootloader (i.e., where
+    /// [`Chipset::needs_fwsec_bootloader()`](crate::gpu::Chipset::needs_fwsec_bootloader) returns
+    /// `false`). On chipsets that do, use [`bootloader::FwsecFirmwareWithBl`] instead.
     pub(crate) fn run(
         &self,
         dev: &Device<device::Bound>,
@@ -419,7 +402,7 @@ pub(crate) fn run(
             .reset(bar)
             .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?;
         falcon
-            .load(bar, self)
+            .load(dev, bar, self)
             .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
         let (mbox0, _) = falcon
             .boot(bar, Some(0), None)
diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
new file mode 100644
index 0000000000000..342dba59b2f9b
--- /dev/null
+++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Bootloader support for the FWSEC firmware.
+//!
+//! On Turing, the FWSEC firmware is not loaded directly, but is instead loaded through a small
+//! bootloader program that performs the required DMA operations. This bootloader itself needs to
+//! be loaded using PIO.
+
+use kernel::{
+    alloc::KVec,
+    device::{
+        self,
+        Device, //
+    },
+    prelude::*,
+    ptr::{
+        Alignable,
+        Alignment, //
+    },
+    sizes,
+    transmute::{
+        AsBytes,
+        FromBytes, //
+    },
+};
+
+use crate::{
+    dma::DmaObject,
+    driver::Bar0,
+    falcon::{
+        self,
+        gsp::Gsp,
+        Falcon,
+        FalconBromParams,
+        FalconDmaLoadable,
+        FalconEngine,
+        FalconFbifMemType,
+        FalconFbifTarget,
+        FalconFirmware,
+        FalconPioDmemLoadTarget,
+        FalconPioImemLoadTarget,
+        FalconPioLoadable, //
+    },
+    firmware::{
+        fwsec::FwsecFirmware,
+        request_firmware,
+        BinHdr,
+        FIRMWARE_VERSION, //
+    },
+    gpu::Chipset,
+    num::FromSafeCast,
+    regs,
+};
+
+/// Descriptor used by RM to figure out the requirements of the boot loader.
+///
+/// Most of its fields appear to be legacy and carry incorrect values, so they are left unused.
+#[repr(C)]
+#[derive(Debug, Clone)]
+struct BootloaderDesc {
+    /// Starting tag of bootloader.
+    start_tag: u32,
+    /// DMEM load offset - unused here as we always load at offset `0`.
+    _dmem_load_off: u32,
+    /// Offset of code section in the image. Unused as there is only one section in the bootloader
+    /// binary.
+    _code_off: u32,
+    /// Size of code section in the image.
+    code_size: u32,
+    /// Offset of data section in the image. Unused as we build the data section ourselves.
+    _data_off: u32,
+    /// Size of data section in the image. Unused as we build the data section ourselves.
+    _data_size: u32,
+}
+// SAFETY: any byte sequence is valid for this struct.
+unsafe impl FromBytes for BootloaderDesc {}
+
+/// Structure used by the boot-loader to load the rest of the code.
+///
+/// This has to be filled by the GPU driver and copied into DMEM at offset
+/// [`BootloaderDesc.dmem_load_off`].
+#[repr(C, packed)]
+#[derive(Debug, Clone)]
+struct BootloaderDmemDescV2 {
+    /// Reserved, should always be first element.
+    reserved: [u32; 4],
+    /// 16B signature for secure code, 0s if no secure code.
+    signature: [u32; 4],
+    /// DMA context used by the bootloader while loading code/data.
+    ctx_dma: u32,
+    /// 256B-aligned physical FB address where code is located.
+    code_dma_base: u64,
+    /// Offset from `code_dma_base` where the non-secure code is located.
+    ///
+    /// Also used as destination IMEM offset of non-secure code as the DMA firmware object is
+    /// expected to be a mirror image of its loaded state.
+    ///
+    /// Must be multiple of 256.
+    non_sec_code_off: u32,
+    /// Size of the non-secure code part.
+    non_sec_code_size: u32,
+    /// Offset from `code_dma_base` where the secure code is located (must be multiple of 256).
+    ///
+    /// Also used as destination IMEM offset of secure code as the DMA firmware object is expected
+    /// to be a mirror image of its loaded state.
+    ///
+    /// Must be multiple of 256.
+    sec_code_off: u32,
+    /// Size of the secure code part.
+    sec_code_size: u32,
+    /// Code entry point invoked by the bootloader after code is loaded.
+    code_entry_point: u32,
+    /// 256B-aligned physical FB address where data is located.
+    data_dma_base: u64,
+    /// Size of data block (should be multiple of 256B).
+    data_size: u32,
+    /// Number of arguments to be passed to the target firmware being loaded.
+    argc: u32,
+    /// Arguments to be passed to the target firmware being loaded.
+    argv: u32,
+}
+// SAFETY: This struct doesn't contain uninitialized bytes and doesn't have interior mutability.
+unsafe impl AsBytes for BootloaderDmemDescV2 {}
+
+/// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load
+/// operation.
+pub(crate) struct FwsecFirmwareWithBl {
+    /// DMA object the bootloader will copy the firmware from.
+    _firmware_dma: DmaObject,
+    /// Code of the bootloader to be loaded into non-secure IMEM.
+    ucode: KVec<u8>,
+    /// Descriptor to be loaded into DMEM for the bootloader to read.
+    dmem_desc: BootloaderDmemDescV2,
+    /// Range-validated start offset of the firmware code in IMEM.
+    imem_dst_start: u16,
+    /// BROM parameters of the loaded firmware.
+    brom_params: FalconBromParams,
+    /// Range-validated `desc.start_tag`.
+    start_tag: u16,
+}
+
+impl FwsecFirmwareWithBl {
+    /// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be
+    /// loaded using it.
+    pub(crate) fn new(
+        firmware: FwsecFirmware,
+        dev: &Device<device::Bound>,
+        chipset: Chipset,
+    ) -> Result<Self> {
+        let fw = request_firmware(dev, chipset, "gen_bootloader", FIRMWARE_VERSION)?;
+        let hdr = fw
+            .data()
+            .get(0..size_of::<BinHdr>())
+            .and_then(BinHdr::from_bytes_copy)
+            .ok_or(EINVAL)?;
+
+        let desc = {
+            let desc_offset = usize::from_safe_cast(hdr.header_offset);
+
+            fw.data()
+                .get(desc_offset..)
+                .and_then(BootloaderDesc::from_bytes_copy_prefix)
+                .ok_or(EINVAL)?
+                .0
+        };
+
+        let ucode = {
+            let ucode_start = usize::from_safe_cast(hdr.data_offset);
+            let code_size = usize::from_safe_cast(desc.code_size);
+            // Align to falcon block size (256 bytes).
+            let aligned_code_size = code_size
+                .align_up(Alignment::new::<{ falcon::MEM_BLOCK_ALIGNMENT }>())
+                .ok_or(EINVAL)?;
+
+            let mut ucode = KVec::with_capacity(aligned_code_size, GFP_KERNEL)?;
+            ucode.extend_from_slice(
+                fw.data()
+                    .get(ucode_start..ucode_start + code_size)
+                    .ok_or(EINVAL)?,
+                GFP_KERNEL,
+            )?;
+            ucode.resize(aligned_code_size, 0, GFP_KERNEL)?;
+
+            ucode
+        };
+
+        // `BootloaderDmemDescV2` expects the source to be a mirror image of the destination and
+        // uses the same offset parameter for both.
+        //
+        // Thus, the start of the source object needs to be padded with the difference between the
+        // destination and source offsets.
+        //
+        // In practice, this is expected to always be zero but is required for code correctness.
+        let (align_padding, firmware_dma) = {
+            let align_padding = {
+                let imem_sec = firmware.imem_sec_load_params();
+
+                imem_sec
+                    .dst_start
+                    .checked_sub(imem_sec.src_start)
+                    .map(usize::from_safe_cast)
+                    .ok_or(EOVERFLOW)?
+            };
+
+            let mut firmware_obj = KVVec::new();
+            firmware_obj.extend_with(align_padding, 0u8, GFP_KERNEL)?;
+            firmware_obj.extend_from_slice(firmware.ucode.0.as_slice(), GFP_KERNEL)?;
+
+            (
+                align_padding,
+                DmaObject::from_data(dev, firmware_obj.as_slice())?,
+            )
+        };
+
+        let dmem_desc = {
+            // Bootloader payload is in non-coherent system memory.
+            const FALCON_DMAIDX_PHYS_SYS_NCOH: u32 = 4;
+
+            let imem_sec = firmware.imem_sec_load_params();
+            let imem_ns = firmware.imem_ns_load_params().ok_or(EINVAL)?;
+            let dmem = firmware.dmem_load_params();
+
+            // The bootloader does not have a data destination offset field and copies the data at
+            // the start of DMEM, so it can only be used if the destination offset of the firmware
+            // is 0.
+            if dmem.dst_start != 0 {
+                return Err(EINVAL);
+            }
+
+            BootloaderDmemDescV2 {
+                reserved: [0; 4],
+                signature: [0; 4],
+                ctx_dma: FALCON_DMAIDX_PHYS_SYS_NCOH,
+                code_dma_base: firmware_dma.dma_handle(),
+                // `dst_start` is also valid as the source offset since the firmware DMA object is
+                // a mirror image of the target IMEM layout.
+                non_sec_code_off: imem_ns.dst_start,
+                non_sec_code_size: imem_ns.len,
+                // `dst_start` is also valid as the source offset since the firmware DMA object is
+                // a mirror image of the target IMEM layout.
+                sec_code_off: imem_sec.dst_start,
+                sec_code_size: imem_sec.len,
+                code_entry_point: 0,
+                // Start of data section is the added padding + the DMEM `src_start` field.
+                data_dma_base: firmware_dma
+                    .dma_handle()
+                    .checked_add(u64::from_safe_cast(align_padding))
+                    .and_then(|offset| offset.checked_add(dmem.src_start.into()))
+                    .ok_or(EOVERFLOW)?,
+                data_size: dmem.len,
+                argc: 0,
+                argv: 0,
+            }
+        };
+
+        // The bootloader's code must be loaded in the area right below the first 64K of IMEM.
+        const BOOTLOADER_LOAD_CEILING: usize = sizes::SZ_64K;
+        let imem_dst_start = BOOTLOADER_LOAD_CEILING
+            .checked_sub(ucode.len())
+            .ok_or(EOVERFLOW)?;
+
+        Ok(Self {
+            _firmware_dma: firmware_dma,
+            ucode,
+            dmem_desc,
+            brom_params: firmware.brom_params(),
+            imem_dst_start: u16::try_from(imem_dst_start)?,
+            start_tag: u16::try_from(desc.start_tag)?,
+        })
+    }
+
+    /// Loads the bootloader into `falcon` and execute it.
+    ///
+    /// The bootloader will load the FWSEC firmware and then execute it. This function returns
+    /// after FWSEC has reached completion.
+    pub(crate) fn run(
+        &self,
+        dev: &Device<device::Bound>,
+        falcon: &Falcon<Gsp>,
+        bar: &Bar0,
+    ) -> Result<()> {
+        // Reset falcon, load the firmware, and run it.
+        falcon
+            .reset(bar)
+            .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?;
+        falcon
+            .pio_load(bar, self)
+            .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
+
+        // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory.
+        regs::NV_PFALCON_FBIF_TRANSCFG::try_update(
+            bar,
+            &Gsp::ID,
+            usize::from_safe_cast(self.dmem_desc.ctx_dma),
+            |v| {
+                v.set_target(FalconFbifTarget::CoherentSysmem)
+                    .set_mem_type(FalconFbifMemType::Physical)
+            },
+        )?;
+
+        let (mbox0, _) = falcon
+            .boot(bar, Some(0), None)
+            .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?;
+        if mbox0 != 0 {
+            dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0);
+            Err(EIO)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl FalconFirmware for FwsecFirmwareWithBl {
+    type Target = Gsp;
+
+    fn brom_params(&self) -> FalconBromParams {
+        self.brom_params.clone()
+    }
+
+    fn boot_addr(&self) -> u32 {
+        // On V2 platforms, the boot address is extracted from the generic bootloader, because the
+        // gbl is what actually copies FWSEC into memory, so that is what needs to be booted.
+        u32::from(self.start_tag) << 8
+    }
+}
+
+impl FalconPioLoadable for FwsecFirmwareWithBl {
+    fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
+        None
+    }
+
+    fn imem_ns_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
+        Some(FalconPioImemLoadTarget {
+            data: self.ucode.as_ref(),
+            dst_start: self.imem_dst_start,
+            secure: false,
+            start_tag: self.start_tag,
+        })
+    }
+
+    fn dmem_load_params(&self) -> FalconPioDmemLoadTarget<'_> {
+        FalconPioDmemLoadTarget {
+            data: self.dmem_desc.as_bytes(),
+            dst_start: 0,
+        }
+    }
+}
diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs
index 4bdd89bd07578..14aad2f0ee8af 100644
--- a/drivers/gpu/nova-core/firmware/riscv.rs
+++ b/drivers/gpu/nova-core/firmware/riscv.rs
@@ -45,10 +45,11 @@ impl RmRiscvUCodeDesc {
     /// Fails if the header pointed at by `bin_fw` is not within the bounds of the firmware image.
     fn new(bin_fw: &BinFirmware<'_>) -> Result<Self> {
         let offset = usize::from_safe_cast(bin_fw.hdr.header_offset);
+        let end = offset.checked_add(size_of::<Self>()).ok_or(EINVAL)?;
 
         bin_fw
             .fw
-            .get(offset..offset + size_of::<Self>())
+            .get(offset..end)
             .and_then(Self::from_bytes_copy)
             .ok_or(EINVAL)
     }
@@ -78,8 +79,9 @@ pub(crate) fn new(dev: &device::Device<device::Bound>, fw: &Firmware) -> Result<
         let ucode = {
             let start = usize::from_safe_cast(bin_fw.hdr.data_offset);
             let len = usize::from_safe_cast(bin_fw.hdr.data_size);
+            let end = start.checked_add(len).ok_or(EINVAL)?;
 
-            DmaObject::from_data(dev, fw.data().get(start..start + len).ok_or(EINVAL)?)?
+            DmaObject::from_data(dev, fw.data().get(start..end).ok_or(EINVAL)?)?
         };
 
         Ok(Self {
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index 9b042ef1a3086..8579d632e717d 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -92,7 +92,7 @@ fn try_from(value: u32) -> Result<Self, Self::Error> {
 });
 
 impl Chipset {
-    pub(crate) fn arch(&self) -> Architecture {
+    pub(crate) const fn arch(self) -> Architecture {
         match self {
             Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => {
                 Architecture::Turing
@@ -105,6 +105,13 @@ pub(crate) fn arch(&self) -> Architecture {
             }
         }
     }
+
+    /// Returns `true` if this chipset requires the PIO-loaded bootloader in order to boot FWSEC.
+    ///
+    /// This includes all chipsets < GA102.
+    pub(crate) const fn needs_fwsec_bootloader(self) -> bool {
+        matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100)
+    }
 }
 
 // TODO
@@ -262,13 +269,13 @@ pub(crate) fn new<'a>(
     ) -> impl PinInit<Self, Error> + 'a {
         try_pin_init!(Self {
             spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| {
-                dev_info!(pdev.as_ref(),"NVIDIA ({})\n", spec);
+                dev_info!(pdev,"NVIDIA ({})\n", spec);
             })?,
 
             // We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
             _: {
                 gfw::wait_gfw_boot_completion(bar)
-                    .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete\n"))?;
+                    .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?;
             },
 
             sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?,
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs
index c69adaa92bbe7..72f173726f87e 100644
--- a/drivers/gpu/nova-core/gsp.rs
+++ b/drivers/gpu/nova-core/gsp.rs
@@ -114,6 +114,7 @@ pub(crate) struct Gsp {
     /// RM log buffer.
     logrm: LogBuffer,
     /// Command queue.
+    #[pin]
     pub(crate) cmdq: Cmdq,
     /// RM arguments.
     rmargs: CoherentAllocation<GspArgumentsPadded>,
@@ -134,7 +135,7 @@ pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error
                 loginit: LogBuffer::new(dev)?,
                 logintr: LogBuffer::new(dev)?,
                 logrm: LogBuffer::new(dev)?,
-                cmdq: Cmdq::new(dev)?,
+                cmdq <- Cmdq::new(dev),
                 rmargs: CoherentAllocation::<GspArgumentsPadded>::alloc_coherent(
                     dev,
                     1,
@@ -151,7 +152,7 @@ pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error
                         libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
                     );
                     dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0));
-                    dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq));
+                    dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(&cmdq));
                     dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs));
                 },
             }))
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index 94833f7996e8a..5e73bd769dcc9 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -24,6 +24,7 @@
             BooterKind, //
         },
         fwsec::{
+            bootloader::FwsecFirmwareWithBl,
             FwsecCommand,
             FwsecFirmware, //
         },
@@ -48,6 +49,7 @@ impl super::Gsp {
     /// created the WPR2 region.
     fn run_fwsec_frts(
         dev: &device::Device<device::Bound>,
+        chipset: Chipset,
         falcon: &Falcon<Gsp>,
         bar: &Bar0,
         bios: &Vbios,
@@ -63,6 +65,7 @@ fn run_fwsec_frts(
             return Err(EBUSY);
         }
 
+        // FWSEC-FRTS will create the WPR2 region.
         let fwsec_frts = FwsecFirmware::new(
             dev,
             falcon,
@@ -70,12 +73,18 @@ fn run_fwsec_frts(
             bios,
             FwsecCommand::Frts {
                 frts_addr: fb_layout.frts.start,
-                frts_size: fb_layout.frts.end - fb_layout.frts.start,
+                frts_size: fb_layout.frts.len(),
             },
         )?;
 
-        // Run FWSEC-FRTS to create the WPR2 region.
-        fwsec_frts.run(dev, falcon, bar)?;
+        if chipset.needs_fwsec_bootloader() {
+            let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?;
+            // Load and run the bootloader, which will load FWSEC-FRTS and run it.
+            fwsec_frts_bl.run(dev, falcon, bar)?;
+        } else {
+            // Load and run FWSEC-FRTS directly.
+            fwsec_frts.run(dev, falcon, bar)?;
+        }
 
         // SCRATCH_E contains the error code for FWSEC-FRTS.
         let frts_status = regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR::read(bar).frts_err_code();
@@ -128,7 +137,7 @@ fn run_fwsec_frts(
     ///
     /// Upon return, the GSP is up and running, and its runtime object given as return value.
     pub(crate) fn boot(
-        mut self: Pin<&mut Self>,
+        self: Pin<&mut Self>,
         pdev: &pci::Device<device::Bound>,
         bar: &Bar0,
         chipset: Chipset,
@@ -144,7 +153,7 @@ pub(crate) fn boot(
         let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?;
         dev_dbg!(dev, "{:#x?}\n", fb_layout);
 
-        Self::run_fwsec_frts(dev, gsp_falcon, bar, &bios, &fb_layout)?;
+        Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?;
 
         let booter_loader = BooterFirmware::new(
             dev,
@@ -160,8 +169,9 @@ pub(crate) fn boot(
         dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout));
 
         self.cmdq
-            .send_command(bar, commands::SetSystemInfo::new(pdev))?;
-        self.cmdq.send_command(bar, commands::SetRegistry::new())?;
+            .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev))?;
+        self.cmdq
+            .send_command_no_wait(bar, commands::SetRegistry::new())?;
 
         gsp_falcon.reset(bar)?;
         let libos_handle = self.libos.dma_handle();
@@ -170,39 +180,25 @@ pub(crate) fn boot(
             Some(libos_handle as u32),
             Some((libos_handle >> 32) as u32),
         )?;
-        dev_dbg!(
-            pdev.as_ref(),
-            "GSP MBOX0: {:#x}, MBOX1: {:#x}\n",
-            mbox0,
-            mbox1
-        );
+        dev_dbg!(pdev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1);
 
         dev_dbg!(
-            pdev.as_ref(),
+            pdev,
             "Using SEC2 to load and run the booter_load firmware...\n"
         );
 
         sec2_falcon.reset(bar)?;
-        sec2_falcon.load(bar, &booter_loader)?;
+        sec2_falcon.load(dev, bar, &booter_loader)?;
         let wpr_handle = wpr_meta.dma_handle();
         let (mbox0, mbox1) = sec2_falcon.boot(
             bar,
             Some(wpr_handle as u32),
             Some((wpr_handle >> 32) as u32),
         )?;
-        dev_dbg!(
-            pdev.as_ref(),
-            "SEC2 MBOX0: {:#x}, MBOX1{:#x}\n",
-            mbox0,
-            mbox1
-        );
+        dev_dbg!(pdev, "SEC2 MBOX0: {:#x}, MBOX1{:#x}\n", mbox0, mbox1);
 
         if mbox0 != 0 {
-            dev_err!(
-                pdev.as_ref(),
-                "Booter-load failed with error {:#x}\n",
-                mbox0
-            );
+            dev_err!(pdev, "Booter-load failed with error {:#x}\n", mbox0);
             return Err(ENODEV);
         }
 
@@ -216,11 +212,7 @@ pub(crate) fn boot(
             Delta::from_secs(5),
         )?;
 
-        dev_dbg!(
-            pdev.as_ref(),
-            "RISC-V active? {}\n",
-            gsp_falcon.is_riscv_active(bar),
-        );
+        dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),);
 
         // Create and run the GSP sequencer.
         let seq_params = GspSequencerParams {
@@ -231,16 +223,16 @@ pub(crate) fn boot(
             dev: pdev.as_ref().into(),
             bar,
         };
-        GspSequencer::run(&mut self.cmdq, seq_params)?;
+        GspSequencer::run(&self.cmdq, seq_params)?;
 
         // Wait until GSP is fully initialized.
-        commands::wait_gsp_init_done(&mut self.cmdq)?;
+        commands::wait_gsp_init_done(&self.cmdq)?;
 
         // Obtain and display basic GPU information.
-        let info = commands::get_gsp_info(&mut self.cmdq, bar)?;
+        let info = commands::get_gsp_info(&self.cmdq, bar)?;
         match info.gpu_name() {
-            Ok(name) => dev_info!(pdev.as_ref(), "GPU name: {}\n", name),
-            Err(e) => dev_warn!(pdev.as_ref(), "GPU name unavailable: {:?}\n", e),
+            Ok(name) => dev_info!(pdev, "GPU name: {}\n", name),
+            Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e),
         }
 
         Ok(())
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 03a4f35998498..d36a62ba1c600 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -1,9 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use core::{
-    cmp,
-    mem, //
-};
+mod continuation;
+
+use core::mem;
 
 use kernel::{
     device,
@@ -13,8 +12,12 @@
     },
     dma_write,
     io::poll::read_poll_timeout,
+    new_mutex,
     prelude::*,
-    sync::aref::ARef,
+    sync::{
+        aref::ARef,
+        Mutex, //
+    },
     time::Delta,
     transmute::{
         AsBytes,
@@ -22,6 +25,11 @@
     },
 };
 
+use continuation::{
+    ContinuationRecord,
+    SplitState, //
+};
+
 use crate::{
     driver::Bar0,
     gsp::{
@@ -29,7 +37,8 @@
             GspMsgElement,
             MsgFunction,
             MsgqRxHeader,
-            MsgqTxHeader, //
+            MsgqTxHeader,
+            GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, //
         },
         PteArray,
         GSP_PAGE_SHIFT,
@@ -40,10 +49,14 @@
     sbuffer::SBufferIter, //
 };
 
+/// Marker type representing the absence of a reply for a command. Commands using this as their
+/// reply type are sent using [`Cmdq::send_command_no_wait`].
+pub(crate) struct NoReply;
+
 /// Trait implemented by types representing a command to send to the GSP.
 ///
-/// The main purpose of this trait is to provide [`Cmdq::send_command`] with the information it
-/// needs to send a given command.
+/// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send
+/// a given command.
 ///
 /// [`CommandToGsp::init`] in particular is responsible for initializing the command directly
 /// into the space reserved for it in the command queue buffer.
@@ -58,6 +71,10 @@ pub(crate) trait CommandToGsp {
     /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer.
     type Command: FromBytes + AsBytes;
 
+    /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't
+    /// have a reply.
+    type Reply;
+
     /// Error type returned by [`CommandToGsp::init`].
     type InitError;
 
@@ -90,6 +107,12 @@ fn init_variable_payload(
     ) -> Result {
         Ok(())
     }
+
+    /// Total size of the command (including its variable-length payload) without the
+    /// [`GspMsgElement`] header.
+    fn size(&self) -> usize {
+        size_of::<Self::Command>() + self.variable_payload_len()
+    }
 }
 
 /// Trait representing messages received from the GSP.
@@ -159,12 +182,14 @@ pub(super) struct GspMem {
     /// Self-mapping page table entries.
     ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
     /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
-    /// write and read pointers that the CPU updates.
+    /// write and read pointers that the CPU updates. This means that the read pointer here is an
+    /// index into the GSP queue.
     ///
     /// This member is read-only for the GSP.
     pub(super) cpuq: Msgq,
     /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
-    /// write and read pointers that the GSP updates.
+    /// write and read pointers that the GSP updates. This means that the read pointer here is an
+    /// index into the CPU queue.
     ///
     /// This member is read-only for the driver.
     pub(super) gspq: Msgq,
@@ -234,27 +259,46 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
         // - We will only access the driver-owned part of the shared memory.
         // - Per the safety statement of the function, no concurrent access will be performed.
         let gsp_mem = &mut unsafe { self.0.as_slice_mut(0, 1) }.unwrap()[0];
-        // PANIC: per the invariant of `cpu_write_ptr`, `tx` is `<= MSGQ_NUM_PAGES`.
+        // PANIC: per the invariant of `cpu_write_ptr`, `tx` is `< MSGQ_NUM_PAGES`.
         let (before_tx, after_tx) = gsp_mem.cpuq.msgq.data.split_at_mut(tx);
 
-        if rx <= tx {
-            // The area from `tx` up to the end of the ring, and from the beginning of the ring up
-            // to `rx`, minus one unit, belongs to the driver.
-            if rx == 0 {
-                let last = after_tx.len() - 1;
-                (&mut after_tx[..last], &mut before_tx[0..0])
-            } else {
-                (after_tx, &mut before_tx[..rx])
-            }
+        // The area starting at `tx` and ending at `rx - 2` modulo MSGQ_NUM_PAGES, inclusive,
+        // belongs to the driver for writing.
+
+        if rx == 0 {
+            // Since `rx` is zero, leave an empty slot at end of the buffer.
+            let last = after_tx.len() - 1;
+            (&mut after_tx[..last], &mut [])
+        } else if rx <= tx {
+            // The area is discontiguous and we leave an empty slot before `rx`.
+            // PANIC:
+            // - The index `rx - 1` is non-negative because `rx != 0` in this branch.
+            // - The index does not exceed `before_tx.len()` (which equals `tx`) because
+            //   `rx <= tx` in this branch.
+            (after_tx, &mut before_tx[..(rx - 1)])
         } else {
-            // The area from `tx` to `rx`, minus one unit, belongs to the driver.
-            //
-            // PANIC: per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, `rx` and `tx` are
-            // `<= MSGQ_NUM_PAGES`, and the test above ensured that `rx > tx`.
-            (after_tx.split_at_mut(rx - tx).0, &mut before_tx[0..0])
+            // The area is contiguous and we leave an empty slot before `rx`.
+            // PANIC:
+            // - The index `rx - tx - 1` is non-negative because `rx > tx` in this branch.
+            // - The index does not exceed `after_tx.len()` (which is `MSGQ_NUM_PAGES - tx`)
+            //   because `rx < MSGQ_NUM_PAGES` by the `gsp_read_ptr` invariant.
+            (&mut after_tx[..(rx - tx - 1)], &mut [])
         }
     }
 
+    /// Returns the size of the region of the CPU message queue that the driver is currently allowed
+    /// to write to, in bytes.
+    fn driver_write_area_size(&self) -> usize {
+        let tx = self.cpu_write_ptr();
+        let rx = self.gsp_read_ptr();
+
+        // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and
+        // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES -
+        // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`.
+        let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES;
+        num::u32_as_usize(slots) * GSP_PAGE_SIZE
+    }
+
     /// Returns the region of the GSP message queue that the driver is currently allowed to read
     /// from.
     ///
@@ -269,26 +313,43 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
         // - We will only access the driver-owned part of the shared memory.
         // - Per the safety statement of the function, no concurrent access will be performed.
         let gsp_mem = &unsafe { self.0.as_slice(0, 1) }.unwrap()[0];
-        // PANIC: per the invariant of `cpu_read_ptr`, `xx` is `<= MSGQ_NUM_PAGES`.
-        let (before_rx, after_rx) = gsp_mem.gspq.msgq.data.split_at(rx);
+        let data = &gsp_mem.gspq.msgq.data;
 
-        match tx.cmp(&rx) {
-            cmp::Ordering::Equal => (&after_rx[0..0], &after_rx[0..0]),
-            cmp::Ordering::Greater => (&after_rx[..tx], &before_rx[0..0]),
-            cmp::Ordering::Less => (after_rx, &before_rx[..tx]),
+        // The area starting at `rx` and ending at `tx - 1` modulo MSGQ_NUM_PAGES, inclusive,
+        // belongs to the driver for reading.
+        // PANIC:
+        // - per the invariant of `cpu_read_ptr`, `rx < MSGQ_NUM_PAGES`
+        // - per the invariant of `gsp_write_ptr`, `tx < MSGQ_NUM_PAGES`
+        if rx <= tx {
+            // The area is contiguous.
+            (&data[rx..tx], &[])
+        } else {
+            // The area is discontiguous.
+            (&data[rx..], &data[..tx])
         }
     }
 
     /// Allocates a region on the command queue that is large enough to send a command of `size`
-    /// bytes.
+    /// bytes, waiting for space to become available based on the provided timeout.
     ///
     /// This returns a [`GspCommand`] ready to be written to by the caller.
     ///
     /// # Errors
     ///
-    /// - `EAGAIN` if the driver area is too small to hold the requested command.
+    /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`].
+    /// - `ETIMEDOUT` if space does not become available within the timeout.
     /// - `EIO` if the command header is not properly aligned.
-    fn allocate_command(&mut self, size: usize) -> Result<GspCommand<'_>> {
+    fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> {
+        if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX {
+            return Err(EMSGSIZE);
+        }
+        read_poll_timeout(
+            || Ok(self.driver_write_area_size()),
+            |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size,
+            Delta::from_micros(1),
+            timeout,
+        )?;
+
         // Get the current writable area as an array of bytes.
         let (slice_1, slice_2) = {
             let (slice_1, slice_2) = self.driver_write_area();
@@ -297,13 +358,6 @@ fn allocate_command(&mut self, size: usize) -> Result<GspCommand<'_>> {
             (slice_1.as_flattened_mut(), slice_2.as_flattened_mut())
         };
 
-        // If the GSP is still processing previous messages the shared region
-        // may be full in which case we will have to retry once the GSP has
-        // processed the existing commands.
-        if size_of::<GspMsgElement>() + size > slice_1.len() + slice_2.len() {
-            return Err(EAGAIN);
-        }
-
         // Extract area for the `GspMsgElement`.
         let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?;
 
@@ -327,7 +381,7 @@ fn allocate_command(&mut self, size: usize) -> Result<GspCommand<'_>> {
     //
     // # Invariants
     //
-    // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
+    // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_write_ptr(&self) -> u32 {
         super::fw::gsp_mem::gsp_write_ptr(&self.0)
     }
@@ -336,7 +390,7 @@ fn gsp_write_ptr(&self) -> u32 {
     //
     // # Invariants
     //
-    // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
+    // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_read_ptr(&self) -> u32 {
         super::fw::gsp_mem::gsp_read_ptr(&self.0)
     }
@@ -345,7 +399,7 @@ fn gsp_read_ptr(&self) -> u32 {
     //
     // # Invariants
     //
-    // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
+    // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn cpu_read_ptr(&self) -> u32 {
         super::fw::gsp_mem::cpu_read_ptr(&self.0)
     }
@@ -359,7 +413,7 @@ fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
     //
     // # Invariants
     //
-    // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
+    // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn cpu_write_ptr(&self) -> u32 {
         super::fw::gsp_mem::cpu_write_ptr(&self.0)
     }
@@ -396,13 +450,11 @@ struct GspMessage<'a> {
 ///
 /// Provides the ability to send commands and receive messages from the GSP using a shared memory
 /// area.
+#[pin_data]
 pub(crate) struct Cmdq {
-    /// Device this command queue belongs to.
-    dev: ARef<device::Device>,
-    /// Current command sequence number.
-    seq: u32,
-    /// Memory area shared with the GSP for communicating commands and messages.
-    gsp_mem: DmaGspMem,
+    /// Inner mutex-protected state.
+    #[pin]
+    inner: Mutex<CmdqInner>,
 }
 
 impl Cmdq {
@@ -422,14 +474,17 @@ impl Cmdq {
     /// Number of page table entries for the GSP shared region.
     pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT;
 
-    /// Creates a new command queue for `dev`.
-    pub(crate) fn new(dev: &device::Device<device::Bound>) -> Result<Cmdq> {
-        let gsp_mem = DmaGspMem::new(dev)?;
+    /// Default timeout for receiving a message from the GSP.
+    pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5);
 
-        Ok(Cmdq {
-            dev: dev.into(),
-            seq: 0,
-            gsp_mem,
+    /// Creates a new command queue for `dev`.
+    pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
+        try_pin_init!(Self {
+            inner <- new_mutex!(CmdqInner {
+                dev: dev.into(),
+                gsp_mem: DmaGspMem::new(dev)?,
+                seq: 0,
+            }),
         })
     }
 
@@ -453,29 +508,117 @@ fn notify_gsp(bar: &Bar0) {
             .write(bar);
     }
 
-    /// Sends `command` to the GSP.
+    /// Sends `command` to the GSP and waits for the reply.
+    ///
+    /// Messages with non-matching function codes are silently consumed until the expected reply
+    /// arrives.
+    ///
+    /// The queue is locked for the entire send+receive cycle to ensure that no other command can
+    /// be interleaved.
     ///
     /// # Errors
     ///
-    /// - `EAGAIN` if there was not enough space in the command queue to send the command.
+    /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is
+    ///   not received within the timeout.
     /// - `EIO` if the variable payload requested by the command has not been entirely
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
-    /// Error codes returned by the command initializers are propagated as-is.
-    pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
+    /// Error codes returned by the command and reply initializers are propagated as-is.
+    pub(crate) fn send_command<M>(&self, bar: &Bar0, command: M) -> Result<M::Reply>
     where
         M: CommandToGsp,
+        M::Reply: MessageFromGsp,
+        Error: From<M::InitError>,
+        Error: From<<M::Reply as MessageFromGsp>::InitError>,
+    {
+        let mut inner = self.inner.lock();
+        inner.send_command(bar, command)?;
+
+        loop {
+            match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) {
+                Ok(reply) => break Ok(reply),
+                Err(ERANGE) => continue,
+                Err(e) => break Err(e),
+            }
+        }
+    }
+
+    /// Sends `command` to the GSP without waiting for a reply.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if space does not become available within the timeout.
+    /// - `EIO` if the variable payload requested by the command has not been entirely
+    ///   written to by its [`CommandToGsp::init_variable_payload`] method.
+    ///
+    /// Error codes returned by the command initializers are propagated as-is.
+    pub(crate) fn send_command_no_wait<M>(&self, bar: &Bar0, command: M) -> Result
+    where
+        M: CommandToGsp<Reply = NoReply>,
+        Error: From<M::InitError>,
+    {
+        self.inner.lock().send_command(bar, command)
+    }
+
+    /// Receive a message from the GSP.
+    ///
+    /// See [`CmdqInner::receive_msg`] for details.
+    pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M>
+    where
         // This allows all error types, including `Infallible`, to be used for `M::InitError`.
         Error: From<M::InitError>,
     {
-        let command_size = size_of::<M::Command>() + command.variable_payload_len();
-        let dst = self.gsp_mem.allocate_command(command_size)?;
+        self.inner.lock().receive_msg(timeout)
+    }
 
-        // Extract area for the command itself.
+    /// Returns the DMA handle of the command queue's shared memory region.
+    pub(crate) fn dma_handle(&self) -> DmaAddress {
+        self.inner.lock().gsp_mem.0.dma_handle()
+    }
+}
+
+/// Inner mutex protected state of [`Cmdq`].
+struct CmdqInner {
+    /// Device this command queue belongs to.
+    dev: ARef<device::Device>,
+    /// Current command sequence number.
+    seq: u32,
+    /// Memory area shared with the GSP for communicating commands and messages.
+    gsp_mem: DmaGspMem,
+}
+
+impl CmdqInner {
+    /// Timeout for waiting for space on the command queue.
+    const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1);
+
+    /// Sends `command` to the GSP, without splitting it.
+    ///
+    /// # Errors
+    ///
+    /// - `EMSGSIZE` if the command exceeds the maximum queue element size.
+    /// - `ETIMEDOUT` if space does not become available within the timeout.
+    /// - `EIO` if the variable payload requested by the command has not been entirely
+    ///   written to by its [`CommandToGsp::init_variable_payload`] method.
+    ///
+    /// Error codes returned by the command initializers are propagated as-is.
+    fn send_single_command<M>(&mut self, bar: &Bar0, command: M) -> Result
+    where
+        M: CommandToGsp,
+        // This allows all error types, including `Infallible`, to be used for `M::InitError`.
+        Error: From<M::InitError>,
+    {
+        let size_in_bytes = command.size();
+        let dst = self
+            .gsp_mem
+            .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?;
+
+        // Extract area for the command itself. The GSP message header and the command header
+        // together are guaranteed to fit entirely into a single page, so it's ok to only look
+        // at `dst.contents.0` here.
         let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?;
 
         // Fill the header and command in-place.
-        let msg_element = GspMsgElement::init(self.seq, command_size, M::FUNCTION);
+        let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION);
         // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer
         // fails.
         unsafe {
@@ -483,16 +626,14 @@ pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
             command.init().__init(core::ptr::from_mut(cmd))?;
         }
 
-        // Fill the variable-length payload.
-        if command_size > size_of::<M::Command>() {
-            let mut sbuffer =
-                SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]);
-            command.init_variable_payload(&mut sbuffer)?;
+        // Fill the variable-length payload, which may be empty.
+        let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]);
+        command.init_variable_payload(&mut sbuffer)?;
 
-            if !sbuffer.is_empty() {
-                return Err(EIO);
-            }
+        if !sbuffer.is_empty() {
+            return Err(EIO);
         }
+        drop(sbuffer);
 
         // Compute checksum now that the whole message is ready.
         dst.header
@@ -504,7 +645,7 @@ pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
 
         dev_dbg!(
             &self.dev,
-            "GSP RPC: send: seq# {}, function={}, length=0x{:x}\n",
+            "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n",
             self.seq,
             M::FUNCTION,
             dst.header.length(),
@@ -519,6 +660,37 @@ pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
         Ok(())
     }
 
+    /// Sends `command` to the GSP.
+    ///
+    /// The command may be split into multiple messages if it is large.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if space does not become available within the timeout.
+    /// - `EIO` if the variable payload requested by the command has not been entirely
+    ///   written to by its [`CommandToGsp::init_variable_payload`] method.
+    ///
+    /// Error codes returned by the command initializers are propagated as-is.
+    fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
+    where
+        M: CommandToGsp,
+        Error: From<M::InitError>,
+    {
+        match SplitState::new(command)? {
+            SplitState::Single(command) => self.send_single_command(bar, command),
+            SplitState::Split(command, mut continuations) => {
+                self.send_single_command(bar, command)?;
+
+                while let Some(continuation) = continuations.next() {
+                    // Turbofish needed because the compiler cannot infer M here.
+                    self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?;
+                }
+
+                Ok(())
+            }
+        }
+    }
+
     /// Wait for a message to become available on the message queue.
     ///
     /// This works purely at the transport layer and does not interpret or validate the message
@@ -554,7 +726,7 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
         let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?;
 
         dev_dbg!(
-            self.dev,
+            &self.dev,
             "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n",
             header.sequence(),
             header.function(),
@@ -589,7 +761,7 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
         ])) != 0
         {
             dev_err!(
-                self.dev,
+                &self.dev,
                 "GSP RPC: receive: Call {} - bad checksum\n",
                 header.sequence()
             );
@@ -604,23 +776,21 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
 
     /// Receive a message from the GSP.
     ///
-    /// `init` is a closure tasked with processing the message. It receives a reference to the
-    /// message in the message queue, and a [`SBufferIter`] pointing to its variable-length
-    /// payload, if any.
+    /// The expected message type is specified using the `M` generic parameter. If the pending
+    /// message has a different function code, `ERANGE` is returned and the message is consumed.
     ///
-    /// The expected message is specified using the `M` generic parameter. If the pending message
-    /// is different, `EAGAIN` is returned and the unexpected message is dropped.
-    ///
-    /// This design is by no means final, but it is simple and will let us go through GSP
-    /// initialization.
+    /// The read pointer is always advanced past the message, regardless of whether it matched.
     ///
     /// # Errors
     ///
     /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available.
     /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the
     ///   message queue.
-    /// - `EINVAL` if the function of the message was unrecognized.
-    pub(crate) fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M>
+    /// - `EINVAL` if the function code of the message was not recognized.
+    /// - `ERANGE` if the message had a recognized but non-matching function code.
+    ///
+    /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is.
+    fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M>
     where
         // This allows all error types, including `Infallible`, to be used for `M::InitError`.
         Error: From<M::InitError>,
@@ -634,7 +804,17 @@ pub(crate) fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Resul
             let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?;
             let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]);
 
-            M::read(cmd, &mut sbuffer).map_err(|e| e.into())
+            M::read(cmd, &mut sbuffer)
+                .map_err(|e| e.into())
+                .inspect(|_| {
+                    if !sbuffer.is_empty() {
+                        dev_warn!(
+                            &self.dev,
+                            "GSP message {:?} has unprocessed data\n",
+                            function
+                        );
+                    }
+                })
         } else {
             Err(ERANGE)
         };
@@ -646,9 +826,4 @@ pub(crate) fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Resul
 
         result
     }
-
-    /// Returns the DMA handle of the command queue's shared memory region.
-    pub(crate) fn dma_handle(&self) -> DmaAddress {
-        self.gsp_mem.0.dma_handle()
-    }
 }
diff --git a/drivers/gpu/nova-core/gsp/cmdq/continuation.rs b/drivers/gpu/nova-core/gsp/cmdq/continuation.rs
new file mode 100644
index 0000000000000..05e904f180978
--- /dev/null
+++ b/drivers/gpu/nova-core/gsp/cmdq/continuation.rs
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Support for splitting large GSP commands across continuation records.
+
+use core::convert::Infallible;
+
+use kernel::prelude::*;
+
+use super::{
+    CommandToGsp,
+    NoReply, //
+};
+
+use crate::{
+    gsp::fw::{
+        GspMsgElement,
+        MsgFunction,
+        GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, //
+    },
+    sbuffer::SBufferIter,
+};
+
+/// Maximum command size that fits in a single queue element.
+const MAX_CMD_SIZE: usize = GSP_MSG_QUEUE_ELEMENT_SIZE_MAX - size_of::<GspMsgElement>();
+
+/// Acts as an iterator over the continuation records for a split command.
+pub(super) struct ContinuationRecords {
+    payload: KVVec<u8>,
+    offset: usize,
+}
+
+impl ContinuationRecords {
+    /// Creates a new iterator over continuation records for the given payload.
+    fn new(payload: KVVec<u8>) -> Self {
+        Self { payload, offset: 0 }
+    }
+
+    /// Returns the next continuation record, or [`None`] if there are no more.
+    pub(super) fn next(&mut self) -> Option<ContinuationRecord<'_>> {
+        let remaining = self.payload.len() - self.offset;
+
+        if remaining > 0 {
+            let chunk_size = remaining.min(MAX_CMD_SIZE);
+            let record =
+                ContinuationRecord::new(&self.payload[self.offset..(self.offset + chunk_size)]);
+            self.offset += chunk_size;
+            Some(record)
+        } else {
+            None
+        }
+    }
+}
+
+/// The [`ContinuationRecord`] command.
+pub(super) struct ContinuationRecord<'a> {
+    data: &'a [u8],
+}
+
+impl<'a> ContinuationRecord<'a> {
+    /// Creates a new [`ContinuationRecord`] command with the given data.
+    fn new(data: &'a [u8]) -> Self {
+        Self { data }
+    }
+}
+
+impl<'a> CommandToGsp for ContinuationRecord<'a> {
+    const FUNCTION: MsgFunction = MsgFunction::ContinuationRecord;
+    type Command = ();
+    type Reply = NoReply;
+    type InitError = Infallible;
+
+    fn init(&self) -> impl Init<Self::Command, Self::InitError> {
+        <()>::init_zeroed()
+    }
+
+    fn variable_payload_len(&self) -> usize {
+        self.data.len()
+    }
+
+    fn init_variable_payload(
+        &self,
+        dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
+    ) -> Result {
+        dst.write_all(self.data)
+    }
+}
+
+/// Whether a command needs to be split across continuation records or not.
+pub(super) enum SplitState<C: CommandToGsp> {
+    /// A command that fits in a single queue element.
+    Single(C),
+    /// A command split across continuation records.
+    Split(SplitCommand<C>, ContinuationRecords),
+}
+
+impl<C: CommandToGsp> SplitState<C> {
+    /// Maximum variable payload size that fits in the first command alongside the command header.
+    const MAX_FIRST_PAYLOAD: usize = MAX_CMD_SIZE - size_of::<C::Command>();
+
+    /// Creates a new [`SplitState`] for the given command.
+    ///
+    /// If the command is too large, it will be split into a main command and some number of
+    /// continuation records.
+    pub(super) fn new(command: C) -> Result<Self> {
+        let payload_len = command.variable_payload_len();
+
+        if command.size() > MAX_CMD_SIZE {
+            let mut command_payload =
+                KVVec::<u8>::from_elem(0u8, payload_len.min(Self::MAX_FIRST_PAYLOAD), GFP_KERNEL)?;
+            let mut continuation_payload =
+                KVVec::<u8>::from_elem(0u8, payload_len - command_payload.len(), GFP_KERNEL)?;
+            let mut sbuffer = SBufferIter::new_writer([
+                command_payload.as_mut_slice(),
+                continuation_payload.as_mut_slice(),
+            ]);
+
+            command.init_variable_payload(&mut sbuffer)?;
+            if !sbuffer.is_empty() {
+                return Err(EIO);
+            }
+            drop(sbuffer);
+
+            Ok(Self::Split(
+                SplitCommand::new(command, command_payload),
+                ContinuationRecords::new(continuation_payload),
+            ))
+        } else {
+            Ok(Self::Single(command))
+        }
+    }
+}
+
+/// A command that has been truncated to maximum accepted length of the command queue.
+///
+/// The remainder of its payload is expected to be sent using [`ContinuationRecords`].
+pub(super) struct SplitCommand<C: CommandToGsp> {
+    command: C,
+    payload: KVVec<u8>,
+}
+
+impl<C: CommandToGsp> SplitCommand<C> {
+    /// Creates a new [`SplitCommand`] wrapping `command` with the given truncated payload.
+    fn new(command: C, payload: KVVec<u8>) -> Self {
+        Self { command, payload }
+    }
+}
+
+impl<C: CommandToGsp> CommandToGsp for SplitCommand<C> {
+    const FUNCTION: MsgFunction = C::FUNCTION;
+    type Command = C::Command;
+    type Reply = C::Reply;
+    type InitError = C::InitError;
+
+    fn init(&self) -> impl Init<Self::Command, Self::InitError> {
+        self.command.init()
+    }
+
+    fn variable_payload_len(&self) -> usize {
+        self.payload.len()
+    }
+
+    fn init_variable_payload(
+        &self,
+        dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
+    ) -> Result {
+        dst.write_all(&self.payload)
+    }
+}
+
+#[kunit_tests(nova_core_gsp_continuation)]
+mod tests {
+    use super::*;
+
+    use kernel::transmute::{
+        AsBytes,
+        FromBytes, //
+    };
+
+    /// Non-zero-sized command header for testing.
+    #[repr(C)]
+    #[derive(Clone, Copy, Zeroable)]
+    struct TestHeader([u8; 64]);
+
+    // SAFETY: `TestHeader` is a plain array of bytes for which all bit patterns are valid.
+    unsafe impl FromBytes for TestHeader {}
+
+    // SAFETY: `TestHeader` is a plain array of bytes for which all bit patterns are valid.
+    unsafe impl AsBytes for TestHeader {}
+
+    struct TestPayload {
+        data: KVVec<u8>,
+    }
+
+    impl TestPayload {
+        fn generate_pattern(len: usize) -> Result<KVVec<u8>> {
+            let mut data = KVVec::with_capacity(len, GFP_KERNEL)?;
+            for i in 0..len {
+                // Mix in higher bits so the pattern does not repeat every 256 bytes.
+                data.push((i ^ (i >> 8)) as u8, GFP_KERNEL)?;
+            }
+            Ok(data)
+        }
+
+        fn new(len: usize) -> Result<Self> {
+            Ok(Self {
+                data: Self::generate_pattern(len)?,
+            })
+        }
+    }
+
+    impl CommandToGsp for TestPayload {
+        const FUNCTION: MsgFunction = MsgFunction::Nop;
+        type Command = TestHeader;
+        type Reply = NoReply;
+        type InitError = Infallible;
+
+        fn init(&self) -> impl Init<Self::Command, Self::InitError> {
+            TestHeader::init_zeroed()
+        }
+
+        fn variable_payload_len(&self) -> usize {
+            self.data.len()
+        }
+
+        fn init_variable_payload(
+            &self,
+            dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
+        ) -> Result {
+            dst.write_all(self.data.as_slice())
+        }
+    }
+
+    /// Maximum variable payload size that fits in the first command alongside the header.
+    const MAX_FIRST_PAYLOAD: usize = SplitState::<TestPayload>::MAX_FIRST_PAYLOAD;
+
+    fn read_payload(cmd: impl CommandToGsp) -> Result<KVVec<u8>> {
+        let len = cmd.variable_payload_len();
+        let mut buf = KVVec::from_elem(0u8, len, GFP_KERNEL)?;
+        let mut sbuf = SBufferIter::new_writer([buf.as_mut_slice(), &mut []]);
+        cmd.init_variable_payload(&mut sbuf)?;
+        drop(sbuf);
+        Ok(buf)
+    }
+
+    struct SplitTest {
+        payload_size: usize,
+        num_continuations: usize,
+    }
+
+    fn check_split(t: SplitTest) -> Result {
+        let payload = TestPayload::new(t.payload_size)?;
+        let mut num_continuations = 0;
+
+        let buf = match SplitState::new(payload)? {
+            SplitState::Single(cmd) => read_payload(cmd)?,
+            SplitState::Split(cmd, mut continuations) => {
+                let mut buf = read_payload(cmd)?;
+                assert!(size_of::<TestHeader>() + buf.len() <= MAX_CMD_SIZE);
+
+                while let Some(cont) = continuations.next() {
+                    let payload = read_payload(cont)?;
+                    assert!(payload.len() <= MAX_CMD_SIZE);
+                    buf.extend_from_slice(&payload, GFP_KERNEL)?;
+                    num_continuations += 1;
+                }
+
+                buf
+            }
+        };
+
+        assert_eq!(num_continuations, t.num_continuations);
+        assert_eq!(
+            buf.as_slice(),
+            TestPayload::generate_pattern(t.payload_size)?.as_slice()
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn split_command() -> Result {
+        check_split(SplitTest {
+            payload_size: 0,
+            num_continuations: 0,
+        })?;
+        check_split(SplitTest {
+            payload_size: MAX_FIRST_PAYLOAD,
+            num_continuations: 0,
+        })?;
+        check_split(SplitTest {
+            payload_size: MAX_FIRST_PAYLOAD + 1,
+            num_continuations: 1,
+        })?;
+        check_split(SplitTest {
+            payload_size: MAX_FIRST_PAYLOAD + MAX_CMD_SIZE,
+            num_continuations: 1,
+        })?;
+        check_split(SplitTest {
+            payload_size: MAX_FIRST_PAYLOAD + MAX_CMD_SIZE + 1,
+            num_continuations: 2,
+        })?;
+        check_split(SplitTest {
+            payload_size: MAX_FIRST_PAYLOAD + MAX_CMD_SIZE * 3 + MAX_CMD_SIZE / 2,
+            num_continuations: 4,
+        })?;
+        Ok(())
+    }
+}
diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index 8f270eca33bec..c89c7b57a751e 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -11,7 +11,6 @@
     device,
     pci,
     prelude::*,
-    time::Delta,
     transmute::{
         AsBytes,
         FromBytes, //
@@ -24,7 +23,8 @@
         cmdq::{
             Cmdq,
             CommandToGsp,
-            MessageFromGsp, //
+            MessageFromGsp,
+            NoReply, //
         },
         fw::{
             commands::*,
@@ -49,6 +49,7 @@ pub(crate) fn new(pdev: &'a pci::Device<device::Bound>) -> Self {
 impl<'a> CommandToGsp for SetSystemInfo<'a> {
     const FUNCTION: MsgFunction = MsgFunction::GspSetSystemInfo;
     type Command = GspSetSystemInfo;
+    type Reply = NoReply;
     type InitError = Error;
 
     fn init(&self) -> impl Init<Self::Command, Self::InitError> {
@@ -100,6 +101,7 @@ pub(crate) fn new() -> Self {
 impl CommandToGsp for SetRegistry {
     const FUNCTION: MsgFunction = MsgFunction::SetRegistry;
     type Command = PackedRegistryTable;
+    type Reply = NoReply;
     type InitError = Infallible;
 
     fn init(&self) -> impl Init<Self::Command, Self::InitError> {
@@ -163,9 +165,9 @@ fn read(
 }
 
 /// Waits for GSP initialization to complete.
-pub(crate) fn wait_gsp_init_done(cmdq: &mut Cmdq) -> Result {
+pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result {
     loop {
-        match cmdq.receive_msg::<GspInitDone>(Delta::from_secs(10)) {
+        match cmdq.receive_msg::<GspInitDone>(Cmdq::RECEIVE_TIMEOUT) {
             Ok(_) => break Ok(()),
             Err(ERANGE) => continue,
             Err(e) => break Err(e),
@@ -179,6 +181,7 @@ pub(crate) fn wait_gsp_init_done(cmdq: &mut Cmdq) -> Result {
 impl CommandToGsp for GetGspStaticInfo {
     const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo;
     type Command = GspStaticConfigInfo;
+    type Reply = GetGspStaticInfoReply;
     type InitError = Infallible;
 
     fn init(&self) -> impl Init<Self::Command, Self::InitError> {
@@ -231,14 +234,6 @@ pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> {
 }
 
 /// Send the [`GetGspInfo`] command and awaits for its reply.
-pub(crate) fn get_gsp_info(cmdq: &mut Cmdq, bar: &Bar0) -> Result<GetGspStaticInfoReply> {
-    cmdq.send_command(bar, GetGspStaticInfo)?;
-
-    loop {
-        match cmdq.receive_msg::<GetGspStaticInfoReply>(Delta::from_secs(5)) {
-            Ok(info) => return Ok(info),
-            Err(ERANGE) => continue,
-            Err(e) => return Err(e),
-        }
-    }
+pub(crate) fn get_gsp_info(cmdq: &Cmdq, bar: &Bar0) -> Result<GetGspStaticInfoReply> {
+    cmdq.send_command(bar, GetGspStaticInfo)
 }
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 040b30ec3089b..a061131b54125 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -10,7 +10,6 @@
 
 use kernel::{
     dma::CoherentAllocation,
-    fmt,
     prelude::*,
     ptr::{
         Alignable,
@@ -109,6 +108,10 @@ pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, cou
     }
 }
 
+/// Maximum size of a single GSP message queue element in bytes.
+pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize =
+    num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX);
+
 /// Empty type to group methods related to heap parameters for running the GSP firmware.
 enum GspFwHeapParams {}
 
@@ -261,111 +264,81 @@ pub(crate) fn new(gsp_firmware: &GspFirmware, fb_layout: &FbLayout) -> Self {
 #[repr(u32)]
 pub(crate) enum MsgFunction {
     // Common function codes
-    Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP,
-    SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO,
-    AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT,
+    AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA,
+    AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA,
     AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE,
     AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY,
-    AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA,
-    AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA,
-    MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY,
-    BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA,
     AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT,
+    AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT,
+    BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA,
+    ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD,
     Free = bindings::NV_VGPU_MSG_FUNCTION_FREE,
-    Log = bindings::NV_VGPU_MSG_FUNCTION_LOG,
     GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO,
-    SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY,
-    GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO,
+    GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO,
     GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU,
     GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL,
-    GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO,
+    GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO,
+    Log = bindings::NV_VGPU_MSG_FUNCTION_LOG,
+    MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY,
+    Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP,
+    SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO,
+    SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY,
 
     // Event codes
     GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE,
+    GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE,
+    GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD,
     GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
-    PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT,
-    RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED,
     MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
     OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG,
-    GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD,
-    GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE,
+    PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT,
+    RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED,
     UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT,
 }
 
-impl fmt::Display for MsgFunction {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            // Common function codes
-            MsgFunction::Nop => write!(f, "NOP"),
-            MsgFunction::SetGuestSystemInfo => write!(f, "SET_GUEST_SYSTEM_INFO"),
-            MsgFunction::AllocRoot => write!(f, "ALLOC_ROOT"),
-            MsgFunction::AllocDevice => write!(f, "ALLOC_DEVICE"),
-            MsgFunction::AllocMemory => write!(f, "ALLOC_MEMORY"),
-            MsgFunction::AllocCtxDma => write!(f, "ALLOC_CTX_DMA"),
-            MsgFunction::AllocChannelDma => write!(f, "ALLOC_CHANNEL_DMA"),
-            MsgFunction::MapMemory => write!(f, "MAP_MEMORY"),
-            MsgFunction::BindCtxDma => write!(f, "BIND_CTX_DMA"),
-            MsgFunction::AllocObject => write!(f, "ALLOC_OBJECT"),
-            MsgFunction::Free => write!(f, "FREE"),
-            MsgFunction::Log => write!(f, "LOG"),
-            MsgFunction::GetGspStaticInfo => write!(f, "GET_GSP_STATIC_INFO"),
-            MsgFunction::SetRegistry => write!(f, "SET_REGISTRY"),
-            MsgFunction::GspSetSystemInfo => write!(f, "GSP_SET_SYSTEM_INFO"),
-            MsgFunction::GspInitPostObjGpu => write!(f, "GSP_INIT_POST_OBJGPU"),
-            MsgFunction::GspRmControl => write!(f, "GSP_RM_CONTROL"),
-            MsgFunction::GetStaticInfo => write!(f, "GET_STATIC_INFO"),
-
-            // Event codes
-            MsgFunction::GspInitDone => write!(f, "INIT_DONE"),
-            MsgFunction::GspRunCpuSequencer => write!(f, "RUN_CPU_SEQUENCER"),
-            MsgFunction::PostEvent => write!(f, "POST_EVENT"),
-            MsgFunction::RcTriggered => write!(f, "RC_TRIGGERED"),
-            MsgFunction::MmuFaultQueued => write!(f, "MMU_FAULT_QUEUED"),
-            MsgFunction::OsErrorLog => write!(f, "OS_ERROR_LOG"),
-            MsgFunction::GspPostNoCat => write!(f, "NOCAT"),
-            MsgFunction::GspLockdownNotice => write!(f, "LOCKDOWN_NOTICE"),
-            MsgFunction::UcodeLibOsPrint => write!(f, "LIBOS_PRINT"),
-        }
-    }
-}
-
 impl TryFrom<u32> for MsgFunction {
     type Error = kernel::error::Error;
 
     fn try_from(value: u32) -> Result<MsgFunction> {
         match value {
-            bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop),
-            bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => {
-                Ok(MsgFunction::SetGuestSystemInfo)
-            }
-            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot),
+            // Common function codes
+            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma),
+            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma),
             bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice),
             bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory),
-            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma),
-            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma),
-            bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory),
-            bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma),
             bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject),
+            bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot),
+            bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma),
+            bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => {
+                Ok(MsgFunction::ContinuationRecord)
+            }
             bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free),
-            bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log),
             bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo),
-            bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry),
-            bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo),
+            bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo),
             bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => {
                 Ok(MsgFunction::GspInitPostObjGpu)
             }
             bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl),
-            bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo),
+            bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo),
+            bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log),
+            bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory),
+            bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop),
+            bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => {
+                Ok(MsgFunction::SetGuestSystemInfo)
+            }
+            bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry),
+
+            // Event codes
             bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone),
+            bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice),
+            bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat),
             bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => {
                 Ok(MsgFunction::GspRunCpuSequencer)
             }
-            bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent),
-            bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered),
             bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued),
             bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog),
-            bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat),
-            bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice),
+            bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent),
+            bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered),
             bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint),
             _ => Err(EINVAL),
         }
@@ -399,22 +372,6 @@ pub(crate) enum SeqBufOpcode {
     RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE,
 }
 
-impl fmt::Display for SeqBufOpcode {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            SeqBufOpcode::CoreReset => write!(f, "CORE_RESET"),
-            SeqBufOpcode::CoreResume => write!(f, "CORE_RESUME"),
-            SeqBufOpcode::CoreStart => write!(f, "CORE_START"),
-            SeqBufOpcode::CoreWaitForHalt => write!(f, "CORE_WAIT_FOR_HALT"),
-            SeqBufOpcode::DelayUs => write!(f, "DELAY_US"),
-            SeqBufOpcode::RegModify => write!(f, "REG_MODIFY"),
-            SeqBufOpcode::RegPoll => write!(f, "REG_POLL"),
-            SeqBufOpcode::RegStore => write!(f, "REG_STORE"),
-            SeqBufOpcode::RegWrite => write!(f, "REG_WRITE"),
-        }
-    }
-}
-
 impl TryFrom<u32> for SeqBufOpcode {
     type Error = kernel::error::Error;
 
@@ -453,7 +410,7 @@ fn from(value: SeqBufOpcode) -> Self {
 
 /// Wrapper for GSP sequencer register write payload.
 #[repr(transparent)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE);
 
 impl RegWritePayload {
@@ -476,7 +433,7 @@ unsafe impl AsBytes for RegWritePayload {}
 
 /// Wrapper for GSP sequencer register modify payload.
 #[repr(transparent)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY);
 
 impl RegModifyPayload {
@@ -504,7 +461,7 @@ unsafe impl AsBytes for RegModifyPayload {}
 
 /// Wrapper for GSP sequencer register poll payload.
 #[repr(transparent)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL);
 
 impl RegPollPayload {
@@ -537,7 +494,7 @@ unsafe impl AsBytes for RegPollPayload {}
 
 /// Wrapper for GSP sequencer delay payload.
 #[repr(transparent)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US);
 
 impl DelayUsPayload {
@@ -555,7 +512,7 @@ unsafe impl AsBytes for DelayUsPayload {}
 
 /// Wrapper for GSP sequencer register store payload.
 #[repr(transparent)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE);
 
 impl RegStorePayload {
@@ -595,13 +552,7 @@ pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> {
             return Err(EINVAL);
         }
         // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`.
-        let payload_bytes = unsafe {
-            core::slice::from_raw_parts(
-                core::ptr::addr_of!(self.0.payload.regWrite).cast::<u8>(),
-                core::mem::size_of::<RegWritePayload>(),
-            )
-        };
-        Ok(*RegWritePayload::from_bytes(payload_bytes).ok_or(EINVAL)?)
+        Ok(RegWritePayload(unsafe { self.0.payload.regWrite }))
     }
 
     /// Returns the register modify payload by value.
@@ -612,13 +563,7 @@ pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> {
             return Err(EINVAL);
         }
         // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`.
-        let payload_bytes = unsafe {
-            core::slice::from_raw_parts(
-                core::ptr::addr_of!(self.0.payload.regModify).cast::<u8>(),
-                core::mem::size_of::<RegModifyPayload>(),
-            )
-        };
-        Ok(*RegModifyPayload::from_bytes(payload_bytes).ok_or(EINVAL)?)
+        Ok(RegModifyPayload(unsafe { self.0.payload.regModify }))
     }
 
     /// Returns the register poll payload by value.
@@ -629,13 +574,7 @@ pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> {
             return Err(EINVAL);
         }
         // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`.
-        let payload_bytes = unsafe {
-            core::slice::from_raw_parts(
-                core::ptr::addr_of!(self.0.payload.regPoll).cast::<u8>(),
-                core::mem::size_of::<RegPollPayload>(),
-            )
-        };
-        Ok(*RegPollPayload::from_bytes(payload_bytes).ok_or(EINVAL)?)
+        Ok(RegPollPayload(unsafe { self.0.payload.regPoll }))
     }
 
     /// Returns the delay payload by value.
@@ -646,13 +585,7 @@ pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> {
             return Err(EINVAL);
         }
         // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`.
-        let payload_bytes = unsafe {
-            core::slice::from_raw_parts(
-                core::ptr::addr_of!(self.0.payload.delayUs).cast::<u8>(),
-                core::mem::size_of::<DelayUsPayload>(),
-            )
-        };
-        Ok(*DelayUsPayload::from_bytes(payload_bytes).ok_or(EINVAL)?)
+        Ok(DelayUsPayload(unsafe { self.0.payload.delayUs }))
     }
 
     /// Returns the register store payload by value.
@@ -663,13 +596,7 @@ pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> {
             return Err(EINVAL);
         }
         // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`.
-        let payload_bytes = unsafe {
-            core::slice::from_raw_parts(
-                core::ptr::addr_of!(self.0.payload.regStore).cast::<u8>(),
-                core::mem::size_of::<RegStorePayload>(),
-            )
-        };
-        Ok(*RegStorePayload::from_bytes(payload_bytes).ok_or(EINVAL)?)
+        Ok(RegStorePayload(unsafe { self.0.payload.regStore }))
     }
 }
 
diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs
index 21be441996936..db46276430be1 100644
--- a/drivers/gpu/nova-core/gsp/fw/commands.rs
+++ b/drivers/gpu/nova-core/gsp/fw/commands.rs
@@ -1,8 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use kernel::prelude::*;
-use kernel::transmute::{AsBytes, FromBytes};
-use kernel::{device, pci};
+use kernel::{
+    device,
+    pci,
+    prelude::*,
+    transmute::{
+        AsBytes,
+        FromBytes, //
+    }, //
+};
 
 use crate::gsp::GSP_PAGE_SIZE;
 
@@ -107,6 +113,7 @@ unsafe impl FromBytes for PackedRegistryTable {}
 
 /// Payload of the `GetGspStaticInfo` command and message.
 #[repr(transparent)]
+#[derive(Zeroable)]
 pub(crate) struct GspStaticConfigInfo(bindings::GspStaticConfigInfo_t);
 
 impl GspStaticConfigInfo {
@@ -122,7 +129,3 @@ unsafe impl AsBytes for GspStaticConfigInfo {}
 // SAFETY: This struct only contains integer types for which all bit patterns
 // are valid.
 unsafe impl FromBytes for GspStaticConfigInfo {}
-
-// SAFETY: This struct only contains integer types and fixed-size arrays for which
-// all bit patterns are valid.
-unsafe impl Zeroable for GspStaticConfigInfo {}
diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
index 6d25fe0bffa97..334e8be5fde8e 100644
--- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
+++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
@@ -43,6 +43,7 @@ fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
 pub const GSP_FW_WPR_META_REVISION: u32 = 1;
 pub const GSP_FW_WPR_META_MAGIC: i64 = -2577556379034558285;
 pub const REGISTRY_TABLE_ENTRY_TYPE_DWORD: u32 = 1;
+pub const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: u32 = 65536;
 pub type __u8 = ffi::c_uchar;
 pub type __u16 = ffi::c_ushort;
 pub type __u32 = ffi::c_uint;
diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs
index e415a2aa3203e..474e4c8021dbd 100644
--- a/drivers/gpu/nova-core/gsp/sequencer.rs
+++ b/drivers/gpu/nova-core/gsp/sequencer.rs
@@ -67,6 +67,7 @@ fn read(
 /// GSP Sequencer Command types with payload data.
 /// Commands have an opcode and an opcode-dependent struct.
 #[allow(clippy::enum_variant_names)]
+#[derive(Debug)]
 pub(crate) enum GspSeqCmd {
     RegWrite(fw::RegWritePayload),
     RegModify(fw::RegModifyPayload),
@@ -144,12 +145,7 @@ pub(crate) struct GspSequencer<'a> {
     dev: ARef<device::Device>,
 }
 
-/// Trait for running sequencer commands.
-pub(crate) trait GspSeqCmdRunner {
-    fn run(&self, sequencer: &GspSequencer<'_>) -> Result;
-}
-
-impl GspSeqCmdRunner for fw::RegWritePayload {
+impl fw::RegWritePayload {
     fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
         let addr = usize::from_safe_cast(self.addr());
 
@@ -157,7 +153,7 @@ fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
     }
 }
 
-impl GspSeqCmdRunner for fw::RegModifyPayload {
+impl fw::RegModifyPayload {
     fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
         let addr = usize::from_safe_cast(self.addr());
 
@@ -169,7 +165,7 @@ fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
     }
 }
 
-impl GspSeqCmdRunner for fw::RegPollPayload {
+impl fw::RegPollPayload {
     fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
         let addr = usize::from_safe_cast(self.addr());
 
@@ -194,14 +190,14 @@ fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
     }
 }
 
-impl GspSeqCmdRunner for fw::DelayUsPayload {
+impl fw::DelayUsPayload {
     fn run(&self, _sequencer: &GspSequencer<'_>) -> Result {
         fsleep(Delta::from_micros(i64::from(self.val())));
         Ok(())
     }
 }
 
-impl GspSeqCmdRunner for fw::RegStorePayload {
+impl fw::RegStorePayload {
     fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
         let addr = usize::from_safe_cast(self.addr());
 
@@ -209,7 +205,7 @@ fn run(&self, sequencer: &GspSequencer<'_>) -> Result {
     }
 }
 
-impl GspSeqCmdRunner for GspSeqCmd {
+impl GspSeqCmd {
     fn run(&self, seq: &GspSequencer<'_>) -> Result {
         match self {
             GspSeqCmd::RegWrite(cmd) => cmd.run(seq),
@@ -360,9 +356,9 @@ pub(crate) struct GspSequencerParams<'a> {
 }
 
 impl<'a> GspSequencer<'a> {
-    pub(crate) fn run(cmdq: &mut Cmdq, params: GspSequencerParams<'a>) -> Result {
+    pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result {
         let seq_info = loop {
-            match cmdq.receive_msg::<GspSequence>(Delta::from_secs(10)) {
+            match cmdq.receive_msg::<GspSequence>(Cmdq::RECEIVE_TIMEOUT) {
                 Ok(seq_info) => break seq_info,
                 Err(ERANGE) => continue,
                 Err(e) => return Err(e),
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index c1121e7c64c57..b5caf1044697a 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -18,7 +18,7 @@
 mod sbuffer;
 mod vbios;
 
-pub(crate) const MODULE_NAME: &kernel::str::CStr = <LocalModule as kernel::ModuleMetadata>::NAME;
+pub(crate) const MODULE_NAME: &core::ffi::CStr = <LocalModule as kernel::ModuleMetadata>::NAME;
 
 kernel::module_pci_driver! {
     type: driver::NovaCore,
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index ea0d32f5396cf..53f412f0ca327 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -364,6 +364,36 @@ pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self {
     1:1     startcpu as bool;
 });
 
+// IMEM access control register. Up to 4 ports are available for IMEM access.
+register!(NV_PFALCON_FALCON_IMEMC @ PFalconBase[0x00000180[4; 16]] {
+    15:0      offs as u16, "IMEM block and word offset";
+    24:24     aincw as bool, "Auto-increment on write";
+    28:28     secure as bool, "Access secure IMEM";
+});
+
+// IMEM data register. Reading/writing this register accesses IMEM at the address
+// specified by the corresponding IMEMC register.
+register!(NV_PFALCON_FALCON_IMEMD @ PFalconBase[0x00000184[4; 16]] {
+    31:0      data as u32;
+});
+
+// IMEM tag register. Used to set the tag for the current IMEM block.
+register!(NV_PFALCON_FALCON_IMEMT @ PFalconBase[0x00000188[4; 16]] {
+    15:0      tag as u16;
+});
+
+// DMEM access control register. Up to 8 ports are available for DMEM access.
+register!(NV_PFALCON_FALCON_DMEMC @ PFalconBase[0x000001c0[8; 8]] {
+    15:0      offs as u16, "DMEM block and word offset";
+    24:24     aincw as bool, "Auto-increment on write";
+});
+
+// DMEM data register. Reading/writing this register accesses DMEM at the address
+// specified by the corresponding DMEMC register.
+register!(NV_PFALCON_FALCON_DMEMD @ PFalconBase[0x000001c4[8; 8]] {
+    31:0      data as u32;
+});
+
 // Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the falcon
 // instance.
 register!(NV_PFALCON_FALCON_ENGINE @ PFalconBase[0x000003c0] {
diff --git a/drivers/gpu/tests/Makefile b/drivers/gpu/tests/Makefile
new file mode 100644
index 0000000000000..4183e6e2de451
--- /dev/null
+++ b/drivers/gpu/tests/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+gpu_buddy_tests-y = gpu_buddy_test.o gpu_random.o
+obj-$(CONFIG_GPU_BUDDY_KUNIT_TEST) += gpu_buddy_tests.o
diff --git a/drivers/gpu/tests/gpu_buddy_test.c b/drivers/gpu/tests/gpu_buddy_test.c
new file mode 100644
index 0000000000000..7df5c2ae83bb8
--- /dev/null
+++ b/drivers/gpu/tests/gpu_buddy_test.c
@@ -0,0 +1,1421 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2019 Intel Corporation
+ * Copyright © 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <linux/prime_numbers.h>
+#include <linux/sched/signal.h>
+#include <linux/sizes.h>
+
+#include <linux/gpu_buddy.h>
+
+#include "gpu_random.h"
+
+static unsigned int random_seed;
+
+static inline u64 get_size(int order, u64 chunk_size)
+{
+	return (1 << order) * chunk_size;
+}
+
+static void gpu_test_buddy_subtree_offset_alignment_stress(struct kunit *test)
+{
+	struct gpu_buddy_block *block;
+	struct rb_node *node = NULL;
+	const u64 mm_size = SZ_2M;
+	const u64 alignments[] = {
+		SZ_1M,
+		SZ_512K,
+		SZ_256K,
+		SZ_128K,
+		SZ_64K,
+		SZ_32K,
+		SZ_16K,
+		SZ_8K,
+	};
+	struct list_head allocated[ARRAY_SIZE(alignments)];
+	unsigned int i, max_subtree_align = 0;
+	int ret, tree, order;
+	struct gpu_buddy mm;
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	for (i = 0; i < ARRAY_SIZE(allocated); i++)
+		INIT_LIST_HEAD(&allocated[i]);
+
+	/*
+	 * Exercise subtree_max_alignment tracking by allocating blocks with descending
+	 * alignment constraints and freeing them in reverse order. This verifies that
+	 * free-tree augmentation correctly propagates the maximum offset alignment
+	 * present in each subtree at every stage.
+	 */
+
+	for (i = 0; i < ARRAY_SIZE(alignments); i++) {
+		struct gpu_buddy_block *root = NULL;
+		unsigned int expected;
+		u64 align;
+
+		align = alignments[i];
+		expected = ilog2(align) - 1;
+
+		for (;;) {
+			ret = gpu_buddy_alloc_blocks(&mm,
+						     0, mm_size,
+						     SZ_4K, align,
+						     &allocated[i],
+						     0);
+			if (ret)
+				break;
+
+			block = list_last_entry(&allocated[i],
+						struct gpu_buddy_block,
+						link);
+			KUNIT_EXPECT_TRUE(test, IS_ALIGNED(gpu_buddy_block_offset(block), align));
+		}
+
+		for (order = mm.max_order; order >= 0 && !root; order--) {
+			for (tree = 0; tree < 2; tree++) {
+				node = mm.free_trees[tree][order].rb_node;
+				if (node) {
+					root = container_of(node,
+							    struct gpu_buddy_block,
+							    rb);
+					break;
+				}
+			}
+		}
+
+		KUNIT_ASSERT_NOT_NULL(test, root);
+		KUNIT_EXPECT_EQ(test, root->subtree_max_alignment, expected);
+	}
+
+	for (i = ARRAY_SIZE(alignments); i-- > 0; ) {
+		gpu_buddy_free_list(&mm, &allocated[i], 0);
+
+		for (order = 0; order <= mm.max_order; order++) {
+			for (tree = 0; tree < 2; tree++) {
+				node = mm.free_trees[tree][order].rb_node;
+				if (!node)
+					continue;
+
+				block = container_of(node, struct gpu_buddy_block, rb);
+				max_subtree_align = max(max_subtree_align,
+							block->subtree_max_alignment);
+			}
+		}
+
+		KUNIT_EXPECT_GE(test, max_subtree_align, ilog2(alignments[i]));
+	}
+
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_offset_aligned_allocation(struct kunit *test)
+{
+	struct gpu_buddy_block *block, *tmp;
+	int num_blocks, i, count = 0;
+	LIST_HEAD(allocated);
+	struct gpu_buddy mm;
+	u64 mm_size = SZ_4M;
+	LIST_HEAD(freed);
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	num_blocks = mm_size / SZ_256K;
+	/*
+	 * Allocate multiple sizes under a fixed offset alignment.
+	 * Ensures alignment handling is independent of allocation size and
+	 * exercises subtree max-alignment pruning for small requests.
+	 */
+	for (i = 0; i < num_blocks; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_8K, SZ_256K,
+								    &allocated, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_8K);
+
+	list_for_each_entry(block, &allocated, link) {
+		/* Ensure the allocated block uses the expected 8 KB size */
+		KUNIT_EXPECT_EQ(test, gpu_buddy_block_size(&mm, block), SZ_8K);
+		/* Ensure the block starts at a 256 KB-aligned offset for proper alignment */
+		KUNIT_EXPECT_TRUE(test, IS_ALIGNED(gpu_buddy_block_offset(block), SZ_256K));
+	}
+	gpu_buddy_free_list(&mm, &allocated, 0);
+
+	for (i = 0; i < num_blocks; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_16K, SZ_256K,
+								    &allocated, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_16K);
+
+	list_for_each_entry(block, &allocated, link) {
+		/* Ensure the allocated block uses the expected 16 KB size */
+		KUNIT_EXPECT_EQ(test, gpu_buddy_block_size(&mm, block), SZ_16K);
+		/* Ensure the block starts at a 256 KB-aligned offset for proper alignment */
+		KUNIT_EXPECT_TRUE(test, IS_ALIGNED(gpu_buddy_block_offset(block), SZ_256K));
+	}
+
+	/*
+	 * Free alternating aligned blocks to introduce fragmentation.
+	 * Ensures offset-aligned allocations remain valid after frees and
+	 * verifies subtree max-alignment metadata is correctly maintained.
+	 */
+	list_for_each_entry_safe(block, tmp, &allocated, link) {
+		if (count % 2 == 0)
+			list_move_tail(&block->link, &freed);
+		count++;
+	}
+	gpu_buddy_free_list(&mm, &freed, 0);
+
+	for (i = 0; i < num_blocks / 2; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_16K, SZ_256K,
+								    &allocated, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_16K);
+
+	/*
+	 * Allocate with offset alignment after all slots are used; must fail.
+	 * Confirms that no aligned offsets remain.
+	 */
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_16K, SZ_256K,
+							   &allocated, 0),
+			       "buddy_alloc hit an error size=%u\n", SZ_16K);
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_fragmentation_performance(struct kunit *test)
+{
+	struct gpu_buddy_block *block, *tmp;
+	int num_blocks, i, ret, count = 0;
+	LIST_HEAD(allocated_blocks);
+	unsigned long elapsed_ms;
+	LIST_HEAD(reverse_list);
+	LIST_HEAD(test_blocks);
+	LIST_HEAD(clear_list);
+	LIST_HEAD(dirty_list);
+	LIST_HEAD(free_list);
+	struct gpu_buddy mm;
+	u64 mm_size = SZ_4G;
+	ktime_t start, end;
+
+	/*
+	 * Allocation under severe fragmentation
+	 *
+	 * Create severe fragmentation by allocating the entire 4 GiB address space
+	 * as tiny 8 KiB blocks but forcing a 64 KiB alignment. The resulting pattern
+	 * leaves many scattered holes. Split the allocations into two groups and
+	 * return them with different flags to block coalescing, then repeatedly
+	 * allocate and free 64 KiB blocks while timing the loop. This stresses how
+	 * quickly the allocator can satisfy larger, aligned requests from a pool of
+	 * highly fragmented space.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	num_blocks = mm_size / SZ_64K;
+
+	start = ktime_get();
+	/* Allocate with maximum fragmentation - 8K blocks with 64K alignment */
+	for (i = 0; i < num_blocks; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_8K, SZ_64K,
+								    &allocated_blocks, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_8K);
+
+	list_for_each_entry_safe(block, tmp, &allocated_blocks, link) {
+		if (count % 4 == 0 || count % 4 == 3)
+			list_move_tail(&block->link, &clear_list);
+		else
+			list_move_tail(&block->link, &dirty_list);
+		count++;
+	}
+
+	/* Free with different flags to ensure no coalescing */
+	gpu_buddy_free_list(&mm, &clear_list, GPU_BUDDY_CLEARED);
+	gpu_buddy_free_list(&mm, &dirty_list, 0);
+
+	for (i = 0; i < num_blocks; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_64K, SZ_64K,
+								    &test_blocks, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_64K);
+	gpu_buddy_free_list(&mm, &test_blocks, 0);
+
+	end = ktime_get();
+	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
+
+	kunit_info(test, "Fragmented allocation took %lu ms\n", elapsed_ms);
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Reverse free order under fragmentation
+	 *
+	 * Construct a fragmented 4 GiB space by allocating every 8 KiB block with
+	 * 64 KiB alignment, creating a dense scatter of small regions. Half of the
+	 * blocks are selectively freed to form sparse gaps, while the remaining
+	 * allocations are preserved, reordered in reverse, and released back with
+	 * the cleared flag. This models a pathological reverse-ordered free pattern
+	 * and measures how quickly the allocator can merge and reclaim space when
+	 * deallocation occurs in the opposite order of allocation, exposing the
+	 * cost difference between a linear freelist scan and an ordered tree lookup.
+	 */
+	ret = gpu_buddy_init(&mm, mm_size, SZ_4K);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	start = ktime_get();
+	/* Allocate maximum fragmentation */
+	for (i = 0; i < num_blocks; i++)
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, SZ_8K, SZ_64K,
+								    &allocated_blocks, 0),
+					"buddy_alloc hit an error size=%u\n", SZ_8K);
+
+	list_for_each_entry_safe(block, tmp, &allocated_blocks, link) {
+		if (count % 2 == 0)
+			list_move_tail(&block->link, &free_list);
+		count++;
+	}
+	gpu_buddy_free_list(&mm, &free_list, GPU_BUDDY_CLEARED);
+
+	list_for_each_entry_safe_reverse(block, tmp, &allocated_blocks, link)
+		list_move(&block->link, &reverse_list);
+	gpu_buddy_free_list(&mm, &reverse_list, GPU_BUDDY_CLEARED);
+
+	end = ktime_get();
+	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
+
+	kunit_info(test, "Reverse-ordered free took %lu ms\n", elapsed_ms);
+
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_range_bias(struct kunit *test)
+{
+	u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
+	GPU_RND_STATE(prng, random_seed);
+	unsigned int i, count, *order;
+	struct gpu_buddy_block *block;
+	unsigned long flags;
+	struct gpu_buddy mm;
+	LIST_HEAD(allocated);
+
+	bias_size = SZ_1M;
+	ps = roundup_pow_of_two(prandom_u32_state(&prng) % bias_size);
+	ps = max(SZ_4K, ps);
+	mm_size = (SZ_8M-1) & ~(ps-1); /* Multiple roots */
+
+	kunit_info(test, "mm_size=%u, ps=%u\n", mm_size, ps);
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
+			       "buddy_init failed\n");
+
+	count = mm_size / bias_size;
+	order = gpu_random_order(count, &prng);
+	KUNIT_EXPECT_TRUE(test, order);
+
+	/*
+	 * Idea is to split the address space into uniform bias ranges, and then
+	 * in some random order allocate within each bias, using various
+	 * patterns within. This should detect if allocations leak out from a
+	 * given bias, for example.
+	 */
+
+	for (i = 0; i < count; i++) {
+		LIST_HEAD(tmp);
+		u32 size;
+
+		bias_start = order[i] * bias_size;
+		bias_end = bias_start + bias_size;
+		bias_rem = bias_size;
+
+		/* internal round_up too big */
+		KUNIT_ASSERT_TRUE_MSG(test,
+				      gpu_buddy_alloc_blocks(&mm, bias_start,
+							     bias_end, bias_size + ps, bias_size,
+							     &allocated,
+							     GPU_BUDDY_RANGE_ALLOCATION),
+				      "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+				      bias_start, bias_end, bias_size, bias_size);
+
+		/* size too big */
+		KUNIT_ASSERT_TRUE_MSG(test,
+				      gpu_buddy_alloc_blocks(&mm, bias_start,
+							     bias_end, bias_size + ps, ps,
+							     &allocated,
+							     GPU_BUDDY_RANGE_ALLOCATION),
+				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+				      bias_start, bias_end, bias_size + ps, ps);
+
+		/* bias range too small for size */
+		KUNIT_ASSERT_TRUE_MSG(test,
+				      gpu_buddy_alloc_blocks(&mm, bias_start + ps,
+							     bias_end, bias_size, ps,
+							     &allocated,
+							     GPU_BUDDY_RANGE_ALLOCATION),
+				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+				      bias_start + ps, bias_end, bias_size, ps);
+
+		/* bias misaligned */
+		KUNIT_ASSERT_TRUE_MSG(test,
+				      gpu_buddy_alloc_blocks(&mm, bias_start + ps,
+							     bias_end - ps,
+							     bias_size >> 1, bias_size >> 1,
+							     &allocated,
+							     GPU_BUDDY_RANGE_ALLOCATION),
+				      "buddy_alloc h didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+				      bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1);
+
+		/* single big page */
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, bias_start,
+							      bias_end, bias_size, bias_size,
+							      &tmp,
+							      GPU_BUDDY_RANGE_ALLOCATION),
+				       "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n",
+				       bias_start, bias_end, bias_size, bias_size);
+		gpu_buddy_free_list(&mm, &tmp, 0);
+
+		/* single page with internal round_up */
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, bias_start,
+							      bias_end, ps, bias_size,
+							      &tmp,
+							      GPU_BUDDY_RANGE_ALLOCATION),
+				       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+				       bias_start, bias_end, ps, bias_size);
+		gpu_buddy_free_list(&mm, &tmp, 0);
+
+		/* random size within */
+		size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+		if (size)
+			KUNIT_ASSERT_FALSE_MSG(test,
+					       gpu_buddy_alloc_blocks(&mm, bias_start,
+								      bias_end, size, ps,
+								      &tmp,
+								      GPU_BUDDY_RANGE_ALLOCATION),
+					       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+					       bias_start, bias_end, size, ps);
+
+		bias_rem -= size;
+		/* too big for current avail */
+		KUNIT_ASSERT_TRUE_MSG(test,
+				      gpu_buddy_alloc_blocks(&mm, bias_start,
+							     bias_end, bias_rem + ps, ps,
+							     &allocated,
+							     GPU_BUDDY_RANGE_ALLOCATION),
+				      "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n",
+				      bias_start, bias_end, bias_rem + ps, ps);
+
+		if (bias_rem) {
+			/* random fill of the remainder */
+			size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+			size = max(size, ps);
+
+			KUNIT_ASSERT_FALSE_MSG(test,
+					       gpu_buddy_alloc_blocks(&mm, bias_start,
+								      bias_end, size, ps,
+								      &allocated,
+								      GPU_BUDDY_RANGE_ALLOCATION),
+					       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+					       bias_start, bias_end, size, ps);
+			/*
+			 * Intentionally allow some space to be left
+			 * unallocated, and ideally not always on the bias
+			 * boundaries.
+			 */
+			gpu_buddy_free_list(&mm, &tmp, 0);
+		} else {
+			list_splice_tail(&tmp, &allocated);
+		}
+	}
+
+	kfree(order);
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Something more free-form. Idea is to pick a random starting bias
+	 * range within the address space and then start filling it up. Also
+	 * randomly grow the bias range in both directions as we go along. This
+	 * should give us bias start/end which is not always uniform like above,
+	 * and in some cases will require the allocator to jump over already
+	 * allocated nodes in the middle of the address space.
+	 */
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
+			       "buddy_init failed\n");
+
+	bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
+	bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
+	bias_end = max(bias_end, bias_start + ps);
+	bias_rem = bias_end - bias_start;
+
+	do {
+		u32 size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, bias_start,
+							      bias_end, size, ps,
+							      &allocated,
+							      GPU_BUDDY_RANGE_ALLOCATION),
+				       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+				       bias_start, bias_end, size, ps);
+		bias_rem -= size;
+
+		/*
+		 * Try to randomly grow the bias range in both directions, or
+		 * only one, or perhaps don't grow at all.
+		 */
+		do {
+			u32 old_bias_start = bias_start;
+			u32 old_bias_end = bias_end;
+
+			if (bias_start)
+				bias_start -= round_up(prandom_u32_state(&prng) % bias_start, ps);
+			if (bias_end != mm_size)
+				bias_end += round_up(prandom_u32_state(&prng) % (mm_size - bias_end), ps);
+
+			bias_rem += old_bias_start - bias_start;
+			bias_rem += bias_end - old_bias_end;
+		} while (!bias_rem && (bias_start || bias_end != mm_size));
+	} while (bias_rem);
+
+	KUNIT_ASSERT_EQ(test, bias_start, 0);
+	KUNIT_ASSERT_EQ(test, bias_end, mm_size);
+	KUNIT_ASSERT_TRUE_MSG(test,
+			      gpu_buddy_alloc_blocks(&mm, bias_start, bias_end,
+						     ps, ps,
+						     &allocated,
+						     GPU_BUDDY_RANGE_ALLOCATION),
+			      "buddy_alloc passed with bias(%x-%x), size=%u\n",
+			      bias_start, bias_end, ps);
+
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Allocate cleared blocks in the bias range when the GPU buddy's clear avail is
+	 * zero. This will validate the bias range allocation in scenarios like system boot
+	 * when no cleared blocks are available and exercise the fallback path too. The resulting
+	 * blocks should always be dirty.
+	 */
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
+			       "buddy_init failed\n");
+
+	bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
+	bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
+	bias_end = max(bias_end, bias_start + ps);
+	bias_rem = bias_end - bias_start;
+
+	flags = GPU_BUDDY_CLEAR_ALLOCATION | GPU_BUDDY_RANGE_ALLOCATION;
+	size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+
+	KUNIT_ASSERT_FALSE_MSG(test,
+			       gpu_buddy_alloc_blocks(&mm, bias_start,
+						      bias_end, size, ps,
+						      &allocated,
+						      flags),
+			       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+			       bias_start, bias_end, size, ps);
+
+	list_for_each_entry(block, &allocated, link)
+		KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
+
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_range(struct kunit *test)
+{
+	GPU_RND_STATE(prng, random_seed);
+	struct gpu_buddy_block *block;
+	struct gpu_buddy mm;
+	u32 mm_size, total;
+	LIST_HEAD(blocks);
+	LIST_HEAD(tmp);
+	u32 ps = SZ_4K;
+	int ret;
+
+	mm_size = SZ_16M;
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps),
+			       "buddy_init failed\n");
+
+	/*
+	 * Basic exact-range allocation.
+	 * Allocate the entire mm as one exact range (start + size == end).
+	 * This is the simplest case exercising __gpu_buddy_alloc_range.
+	 */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, mm_size, mm_size, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ_MSG(test, ret, 0,
+			    "exact-range alloc of full mm failed\n");
+
+	total = 0;
+	list_for_each_entry(block, &blocks, link) {
+		u64 offset = gpu_buddy_block_offset(block);
+		u64 bsize = gpu_buddy_block_size(&mm, block);
+
+		KUNIT_EXPECT_TRUE_MSG(test, offset + bsize <= (u64)mm_size,
+				      "block [%llx, %llx) outside mm\n", offset, offset + bsize);
+		total += (u32)bsize;
+	}
+	KUNIT_EXPECT_EQ(test, total, mm_size);
+	KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+
+	/* Full mm should be exhausted */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, ps, ps, ps, &tmp, 0);
+	KUNIT_EXPECT_NE_MSG(test, ret, 0, "alloc should fail when mm is full\n");
+
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	KUNIT_EXPECT_EQ(test, mm.avail, (u64)mm_size);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Exact-range allocation of sub-ranges.
+	 * Split the mm into four equal quarters and allocate each as an exact
+	 * range. Validates splitting and non-overlapping exact allocations.
+	 */
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	{
+		u32 quarter = mm_size / 4;
+		int i;
+
+		for (i = 0; i < 4; i++) {
+			u32 start = i * quarter;
+			u32 end = start + quarter;
+
+			ret = gpu_buddy_alloc_blocks(&mm, start, end, quarter, ps, &blocks, 0);
+			KUNIT_ASSERT_EQ_MSG(test, ret, 0,
+					    "exact-range alloc quarter %d [%x, %x) failed\n",
+					    i, start, end);
+		}
+		KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+		gpu_buddy_free_list(&mm, &blocks, 0);
+	}
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Minimum chunk-size exact range at various offsets.
+	 * Allocate single-page exact ranges at the start, middle and end.
+	 */
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	ret = gpu_buddy_alloc_blocks(&mm, 0, ps, ps, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = gpu_buddy_alloc_blocks(&mm, mm_size / 2, mm_size / 2 + ps, ps, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = gpu_buddy_alloc_blocks(&mm, mm_size - ps, mm_size, ps, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	total = 0;
+	list_for_each_entry(block, &blocks, link)
+		total += (u32)gpu_buddy_block_size(&mm, block);
+	KUNIT_EXPECT_EQ(test, total, 3 * ps);
+
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Non power-of-two mm size (multiple roots).
+	 * Exact-range allocations that span root boundaries must still work.
+	 */
+	mm_size = SZ_4M + SZ_2M + SZ_1M; /* 7 MiB, three roots */
+
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+	KUNIT_EXPECT_GT(test, mm.n_roots, 1U);
+
+	/* Allocate first 4M root exactly */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, SZ_4M, SZ_4M, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Allocate second root (4M-6M) exactly */
+	ret = gpu_buddy_alloc_blocks(&mm, SZ_4M, SZ_4M + SZ_2M, SZ_2M, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Allocate third root (6M-7M) exactly */
+	ret = gpu_buddy_alloc_blocks(&mm, SZ_4M + SZ_2M, mm_size, SZ_1M, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+	gpu_buddy_free_list(&mm, &blocks, 0);
+
+	/* Cross-root exact-range: the entire non-pot mm */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, mm_size, mm_size, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Randomized exact-range allocations.
+	 * Divide the mm into N random-sized, contiguous, page-aligned slices
+	 * and allocate each as an exact range in random order.
+	 */
+	mm_size = SZ_16M;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	{
+#define N_RAND_RANGES 16
+		u32 ranges[N_RAND_RANGES + 1]; /* boundaries */
+		u32 order_arr[N_RAND_RANGES];
+		u32 remaining = mm_size;
+		int i;
+
+		ranges[0] = 0;
+		for (i = 0; i < N_RAND_RANGES - 1; i++) {
+			u32 max_chunk = remaining - (N_RAND_RANGES - 1 - i) * ps;
+			u32 sz = max(round_up(prandom_u32_state(&prng) % max_chunk, ps), ps);
+
+			ranges[i + 1] = ranges[i] + sz;
+			remaining -= sz;
+		}
+		ranges[N_RAND_RANGES] = mm_size;
+
+		/* Create a random order */
+		for (i = 0; i < N_RAND_RANGES; i++)
+			order_arr[i] = i;
+		for (i = N_RAND_RANGES - 1; i > 0; i--) {
+			u32 j = prandom_u32_state(&prng) % (i + 1);
+			u32 tmp_val = order_arr[i];
+
+			order_arr[i] = order_arr[j];
+			order_arr[j] = tmp_val;
+		}
+
+		for (i = 0; i < N_RAND_RANGES; i++) {
+			u32 idx = order_arr[i];
+			u32 start = ranges[idx];
+			u32 end = ranges[idx + 1];
+			u32 sz = end - start;
+
+			ret = gpu_buddy_alloc_blocks(&mm, start, end, sz, ps, &blocks, 0);
+			KUNIT_ASSERT_EQ_MSG(test, ret, 0,
+					    "random exact-range [%x, %x) sz=%x failed\n",
+					    start, end, sz);
+		}
+
+		KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+		gpu_buddy_free_list(&mm, &blocks, 0);
+#undef N_RAND_RANGES
+	}
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Negative case - partially allocated range.
+	 * Allocate the first half, then try to exact-range allocate the full
+	 * mm. This must fail because the first half is already occupied.
+	 */
+	mm_size = SZ_16M;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	ret = gpu_buddy_alloc_blocks(&mm, 0, mm_size / 2, mm_size / 2, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = gpu_buddy_alloc_blocks(&mm, 0, mm_size, mm_size, ps, &tmp, 0);
+	KUNIT_EXPECT_NE_MSG(test, ret, 0,
+			    "exact-range alloc should fail when range is partially used\n");
+
+	/* Also try the already-occupied sub-range directly */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, mm_size / 2, mm_size / 2, ps, &tmp, 0);
+	KUNIT_EXPECT_NE_MSG(test, ret, 0,
+			    "double alloc of same exact range should fail\n");
+
+	/* The free second half should still be allocatable */
+	ret = gpu_buddy_alloc_blocks(&mm, mm_size / 2, mm_size, mm_size / 2, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, mm.avail, 0ULL);
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Negative case - checkerboard partial allocation.
+	 * Allocate every other page-sized chunk in a small mm, then try to
+	 * exact-range allocate a range covering two pages (one allocated, one
+	 * free). This must fail.
+	 */
+	mm_size = SZ_64K;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	{
+		u32 off;
+
+		for (off = 0; off < mm_size; off += 2 * ps) {
+			ret = gpu_buddy_alloc_blocks(&mm, off, off + ps, ps, ps, &blocks, 0);
+			KUNIT_ASSERT_EQ(test, ret, 0);
+		}
+
+		/* Try exact range over a pair [allocated, free] */
+		ret = gpu_buddy_alloc_blocks(&mm, 0, 2 * ps, 2 * ps, ps, &tmp, 0);
+		KUNIT_EXPECT_NE_MSG(test, ret, 0,
+				    "exact-range over partially allocated pair should fail\n");
+
+		/* The free pages individually should still work */
+		ret = gpu_buddy_alloc_blocks(&mm, ps, 2 * ps, ps, ps, &blocks, 0);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+
+		gpu_buddy_free_list(&mm, &blocks, 0);
+	}
+
+	gpu_buddy_fini(&mm);
+
+	/* Negative case - misaligned start/end/size */
+	mm_size = SZ_16M;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	/* start not aligned to chunk_size */
+	ret = gpu_buddy_alloc_blocks(&mm, ps / 2, ps / 2 + ps, ps, ps, &tmp, 0);
+	KUNIT_EXPECT_NE(test, ret, 0);
+
+	/* size not aligned */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, ps + 1, ps + 1, ps, &tmp, 0);
+	KUNIT_EXPECT_NE(test, ret, 0);
+
+	/* end exceeds mm size */
+	ret = gpu_buddy_alloc_blocks(&mm, mm_size, mm_size + ps, ps, ps, &tmp, 0);
+	KUNIT_EXPECT_NE(test, ret, 0);
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Free and re-allocate the same exact range.
+	 * This exercises merge-on-free followed by exact-range re-split.
+	 */
+	mm_size = SZ_16M;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	{
+		int i;
+
+		for (i = 0; i < 5; i++) {
+			ret = gpu_buddy_alloc_blocks(&mm, SZ_4M, SZ_4M + SZ_2M,
+						     SZ_2M, ps, &blocks, 0);
+			KUNIT_ASSERT_EQ_MSG(test, ret, 0,
+					    "re-alloc iteration %d failed\n", i);
+
+			total = 0;
+			list_for_each_entry(block, &blocks, link) {
+				u64 offset = gpu_buddy_block_offset(block);
+				u64 bsize = gpu_buddy_block_size(&mm, block);
+
+				KUNIT_EXPECT_GE(test, offset, (u64)SZ_4M);
+				KUNIT_EXPECT_LE(test, offset + bsize, (u64)(SZ_4M + SZ_2M));
+				total += (u32)bsize;
+			}
+			KUNIT_EXPECT_EQ(test, total, SZ_2M);
+
+			gpu_buddy_free_list(&mm, &blocks, 0);
+		}
+
+		KUNIT_EXPECT_EQ(test, mm.avail, (u64)mm_size);
+	}
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Various power-of-two exact ranges within a large mm.
+	 * Allocate non-overlapping power-of-two exact ranges at their natural
+	 * alignment, validating that the allocator handles different orders.
+	 */
+	mm_size = SZ_16M;
+	KUNIT_ASSERT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	/* Allocate 4K at offset 0 */
+	ret = gpu_buddy_alloc_blocks(&mm, 0, SZ_4K, SZ_4K, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Allocate 64K at offset 64K */
+	ret = gpu_buddy_alloc_blocks(&mm, SZ_64K, SZ_64K + SZ_64K, SZ_64K, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Allocate 1M at offset 1M */
+	ret = gpu_buddy_alloc_blocks(&mm, SZ_1M, SZ_1M + SZ_1M, SZ_1M, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Allocate 4M at offset 4M */
+	ret = gpu_buddy_alloc_blocks(&mm, SZ_4M, SZ_4M + SZ_4M, SZ_4M, ps, &blocks, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	total = 0;
+	list_for_each_entry(block, &blocks, link)
+		total += (u32)gpu_buddy_block_size(&mm, block);
+	KUNIT_EXPECT_EQ(test, total, SZ_4K + SZ_64K + SZ_1M + SZ_4M);
+
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_clear(struct kunit *test)
+{
+	unsigned long n_pages, total, i = 0;
+	const unsigned long ps = SZ_4K;
+	struct gpu_buddy_block *block;
+	const int max_order = 12;
+	LIST_HEAD(allocated);
+	struct gpu_buddy mm;
+	unsigned int order;
+	u32 mm_size, size;
+	LIST_HEAD(dirty);
+	LIST_HEAD(clean);
+
+	mm_size = SZ_4K << max_order;
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	/*
+	 * Idea is to allocate and free some random portion of the address space,
+	 * returning those pages as non-dirty and randomly alternate between
+	 * requesting dirty and non-dirty pages (not going over the limit
+	 * we freed as non-dirty), putting that into two separate lists.
+	 * Loop over both lists at the end checking that the dirty list
+	 * is indeed all dirty pages and vice versa. Free it all again,
+	 * keeping the dirty/clear status.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							    5 * ps, ps, &allocated,
+							    GPU_BUDDY_TOPDOWN_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 5 * ps);
+	gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
+
+	n_pages = 10;
+	do {
+		unsigned long flags;
+		struct list_head *list;
+		int slot = i % 2;
+
+		if (slot == 0) {
+			list = &dirty;
+			flags = 0;
+		} else {
+			list = &clean;
+			flags = GPU_BUDDY_CLEAR_ALLOCATION;
+		}
+
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+								    ps, ps, list,
+								    flags),
+					"buddy_alloc hit an error size=%lu\n", ps);
+	} while (++i < n_pages);
+
+	list_for_each_entry(block, &clean, link)
+		KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), true);
+
+	list_for_each_entry(block, &dirty, link)
+		KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
+
+	gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
+
+	/*
+	 * Trying to go over the clear limit for some allocation.
+	 * The allocation should never fail with reasonable page-size.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							    10 * ps, ps, &clean,
+							    GPU_BUDDY_CLEAR_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 10 * ps);
+
+	gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
+	gpu_buddy_free_list(&mm, &dirty, 0);
+	gpu_buddy_fini(&mm);
+
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	/*
+	 * Create a new mm. Intentionally fragment the address space by creating
+	 * two alternating lists. Free both lists, one as dirty the other as clean.
+	 * Try to allocate double the previous size with matching min_page_size. The
+	 * allocation should never fail as it calls the force_merge. Also check that
+	 * the page is always dirty after force_merge. Free the page as dirty, then
+	 * repeat the whole thing, increment the order until we hit the max_order.
+	 */
+
+	i = 0;
+	n_pages = mm_size / ps;
+	do {
+		struct list_head *list;
+		int slot = i % 2;
+
+		if (slot == 0)
+			list = &dirty;
+		else
+			list = &clean;
+
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+								    ps, ps, list, 0),
+					"buddy_alloc hit an error size=%lu\n", ps);
+	} while (++i < n_pages);
+
+	gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
+	gpu_buddy_free_list(&mm, &dirty, 0);
+
+	order = 1;
+	do {
+		size = SZ_4K << order;
+
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+								    size, size, &allocated,
+								    GPU_BUDDY_CLEAR_ALLOCATION),
+					"buddy_alloc hit an error size=%u\n", size);
+		total = 0;
+		list_for_each_entry(block, &allocated, link) {
+			if (size != mm_size)
+				KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false);
+			total += gpu_buddy_block_size(&mm, block);
+		}
+		KUNIT_EXPECT_EQ(test, total, size);
+
+		gpu_buddy_free_list(&mm, &allocated, 0);
+	} while (++order <= max_order);
+
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Create a new mm with a non power-of-two size. Allocate a random size from each
+	 * root, free as cleared and then call fini. This will ensure the multi-root
+	 * force merge during fini.
+	 */
+	mm_size = (SZ_4K << max_order) + (SZ_4K << (max_order - 2));
+
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order,
+							    4 * ps, ps, &allocated,
+							    GPU_BUDDY_RANGE_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 4 * ps);
+	gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order,
+							    2 * ps, ps, &allocated,
+							    GPU_BUDDY_CLEAR_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 2 * ps);
+	gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, SZ_4K << max_order, mm_size,
+							    ps, ps, &allocated,
+							    GPU_BUDDY_RANGE_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", ps);
+	gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_contiguous(struct kunit *test)
+{
+	const unsigned long ps = SZ_4K, mm_size = 16 * 3 * SZ_4K;
+	unsigned long i, n_pages, total;
+	struct gpu_buddy_block *block;
+	struct gpu_buddy mm;
+	LIST_HEAD(left);
+	LIST_HEAD(middle);
+	LIST_HEAD(right);
+	LIST_HEAD(allocated);
+
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps));
+
+	/*
+	 * Idea is to fragment the address space by alternating block
+	 * allocations between three different lists; one for left, middle and
+	 * right. We can then free a list to simulate fragmentation. In
+	 * particular we want to exercise the GPU_BUDDY_CONTIGUOUS_ALLOCATION,
+	 * including the try_harder path.
+	 */
+
+	i = 0;
+	n_pages = mm_size / ps;
+	do {
+		struct list_head *list;
+		int slot = i % 3;
+
+		if (slot == 0)
+			list = &left;
+		else if (slot == 1)
+			list = &middle;
+		else
+			list = &right;
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							      ps, ps, list, 0),
+				       "buddy_alloc hit an error size=%lu\n",
+				       ps);
+	} while (++i < n_pages);
+
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							   3 * ps, ps, &allocated,
+							   GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
+
+	gpu_buddy_free_list(&mm, &middle, 0);
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							   3 * ps, ps, &allocated,
+							   GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							   2 * ps, ps, &allocated,
+							   GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc didn't error size=%lu\n", 2 * ps);
+
+	gpu_buddy_free_list(&mm, &right, 0);
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							   3 * ps, ps, &allocated,
+							   GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
+	/*
+	 * At this point we should have enough contiguous space for 2 blocks,
+	 * however they are never buddies (since we freed middle and right) so
+	 * will require the try_harder logic to find them.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							    2 * ps, ps, &allocated,
+							    GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc hit an error size=%lu\n", 2 * ps);
+
+	gpu_buddy_free_list(&mm, &left, 0);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size,
+							    3 * ps, ps, &allocated,
+							    GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc hit an error size=%lu\n", 3 * ps);
+
+	total = 0;
+	list_for_each_entry(block, &allocated, link)
+		total += gpu_buddy_block_size(&mm, block);
+
+	KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3);
+
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_pathological(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct gpu_buddy_block *block;
+	const int max_order = 3;
+	unsigned long flags = 0;
+	int order, top;
+	struct gpu_buddy mm;
+	LIST_HEAD(blocks);
+	LIST_HEAD(holes);
+	LIST_HEAD(tmp);
+
+	/*
+	 * Create a pot-sized mm, then allocate one of each possible
+	 * order within. This should leave the mm with exactly one
+	 * page left. Free the largest block, then whittle down again.
+	 * Eventually we will have a fully 50% fragmented mm.
+	 */
+
+	mm_size = SZ_4K << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (top = max_order; top; top--) {
+		/* Make room by freeing the largest allocated block */
+		block = list_first_entry_or_null(&blocks, typeof(*block), link);
+		if (block) {
+			list_del(&block->link);
+			gpu_buddy_free_block(&mm, block);
+		}
+
+		for (order = top; order--;) {
+			size = get_size(order, mm.chunk_size);
+			KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start,
+									    mm_size, size, size,
+										&tmp, flags),
+					"buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
+					order, top);
+
+			block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+			KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+			list_move_tail(&block->link, &blocks);
+		}
+
+		/* There should be one final page for this sub-allocation */
+		size = get_size(0, mm.chunk_size);
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM for hole\n");
+
+		block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &holes);
+
+		size = get_size(top, mm.chunk_size);
+		KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
+							  top, max_order);
+	}
+
+	gpu_buddy_free_list(&mm, &holes, 0);
+
+	/* Nothing larger than blocks of chunk_size now available */
+	for (order = 1; order <= max_order; order++) {
+		size = get_size(order, mm.chunk_size);
+		KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
+							  order);
+	}
+
+	list_splice_tail(&holes, &blocks);
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_pessimistic(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct gpu_buddy_block *block, *bn;
+	const unsigned int max_order = 16;
+	unsigned long flags = 0;
+	struct gpu_buddy mm;
+	unsigned int order;
+	LIST_HEAD(blocks);
+	LIST_HEAD(tmp);
+
+	/*
+	 * Create a pot-sized mm, then allocate one of each possible
+	 * order within. This should leave the mm with exactly one
+	 * page left.
+	 */
+
+	mm_size = SZ_4K << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (order = 0; order < max_order; order++) {
+		size = get_size(order, mm.chunk_size);
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &blocks);
+	}
+
+	/* And now the last remaining block available */
+	size = get_size(0, mm.chunk_size);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+							    size, size, &tmp, flags),
+						   "buddy_alloc hit -ENOMEM on final alloc\n");
+
+	block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+	list_move_tail(&block->link, &blocks);
+
+	/* Should be completely full! */
+	for (order = max_order; order--;) {
+		size = get_size(order, mm.chunk_size);
+		KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded, it should be full!");
+	}
+
+	block = list_last_entry(&blocks, typeof(*block), link);
+	list_del(&block->link);
+	gpu_buddy_free_block(&mm, block);
+
+	/* As we free in increasing size, we make available larger blocks */
+	order = 1;
+	list_for_each_entry_safe(block, bn, &blocks, link) {
+		list_del(&block->link);
+		gpu_buddy_free_block(&mm, block);
+
+		size = get_size(order, mm.chunk_size);
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_del(&block->link);
+		gpu_buddy_free_block(&mm, block);
+		order++;
+	}
+
+	/* To confirm, now the whole mm should be available */
+	size = get_size(max_order, mm.chunk_size);
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+							    size, size, &tmp, flags),
+						   "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
+						   max_order);
+
+	block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+	list_del(&block->link);
+	gpu_buddy_free_block(&mm, block);
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_optimistic(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct gpu_buddy_block *block;
+	unsigned long flags = 0;
+	const int max_order = 16;
+	struct gpu_buddy mm;
+	LIST_HEAD(blocks);
+	LIST_HEAD(tmp);
+	int order;
+
+	/*
+	 * Create a mm with one block of each order available, and
+	 * try to allocate them all.
+	 */
+
+	mm_size = SZ_4K * ((1 << (max_order + 1)) - 1);
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (order = 0; order <= max_order; order++) {
+		size = get_size(order, mm.chunk_size);
+		KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &blocks);
+	}
+
+	/* Should be completely full! */
+	size = get_size(0, mm.chunk_size);
+	KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size,
+							   size, size, &tmp, flags),
+						  "buddy_alloc unexpectedly succeeded, it should be full!");
+
+	gpu_buddy_free_list(&mm, &blocks, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_limit(struct kunit *test)
+{
+	u64 size = U64_MAX, start = 0;
+	struct gpu_buddy_block *block;
+	unsigned long flags = 0;
+	LIST_HEAD(allocated);
+	struct gpu_buddy mm;
+
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, size, SZ_4K));
+
+	KUNIT_EXPECT_EQ_MSG(test, mm.max_order, GPU_BUDDY_MAX_ORDER,
+			    "mm.max_order(%d) != %d\n", mm.max_order,
+						GPU_BUDDY_MAX_ORDER);
+
+	size = mm.chunk_size << mm.max_order;
+	KUNIT_EXPECT_FALSE(test, gpu_buddy_alloc_blocks(&mm, start, size, size,
+							mm.chunk_size, &allocated, flags));
+
+	block = list_first_entry_or_null(&allocated, struct gpu_buddy_block, link);
+	KUNIT_EXPECT_TRUE(test, block);
+
+	KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_order(block), mm.max_order,
+			    "block order(%d) != %d\n",
+						gpu_buddy_block_order(block), mm.max_order);
+
+	KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_size(&mm, block),
+			    BIT_ULL(mm.max_order) * mm.chunk_size,
+						"block size(%llu) != %llu\n",
+						gpu_buddy_block_size(&mm, block),
+						BIT_ULL(mm.max_order) * mm.chunk_size);
+
+	gpu_buddy_free_list(&mm, &allocated, 0);
+	gpu_buddy_fini(&mm);
+}
+
+static void gpu_test_buddy_alloc_exceeds_max_order(struct kunit *test)
+{
+	u64 mm_size = SZ_8G + SZ_2G, size = SZ_8G + SZ_1G, min_block_size = SZ_8G;
+	struct gpu_buddy mm;
+	LIST_HEAD(blocks);
+	int err;
+
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
+			       "buddy_init failed\n");
+
+	/* CONTIGUOUS allocation should succeed via try_harder fallback */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, size,
+							    SZ_4K, &blocks,
+							    GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "buddy_alloc hit an error size=%llu\n", size);
+	gpu_buddy_free_list(&mm, &blocks, 0);
+
+	/* Non-CONTIGUOUS with large min_block_size should return -EINVAL */
+	err = gpu_buddy_alloc_blocks(&mm, 0, mm_size, size, min_block_size, &blocks, 0);
+	KUNIT_EXPECT_EQ(test, err, -EINVAL);
+
+	/* Non-CONTIGUOUS + RANGE with large min_block_size should return -EINVAL */
+	err = gpu_buddy_alloc_blocks(&mm, 0, mm_size, size, min_block_size, &blocks,
+				     GPU_BUDDY_RANGE_ALLOCATION);
+	KUNIT_EXPECT_EQ(test, err, -EINVAL);
+
+	/* CONTIGUOUS + RANGE should return -EINVAL (no try_harder for RANGE) */
+	err = gpu_buddy_alloc_blocks(&mm, 0, mm_size, size, SZ_4K, &blocks,
+				     GPU_BUDDY_CONTIGUOUS_ALLOCATION | GPU_BUDDY_RANGE_ALLOCATION);
+	KUNIT_EXPECT_EQ(test, err, -EINVAL);
+
+	gpu_buddy_fini(&mm);
+}
+
+static int gpu_buddy_suite_init(struct kunit_suite *suite)
+{
+	while (!random_seed)
+		random_seed = get_random_u32();
+
+	kunit_info(suite, "Testing GPU buddy manager, with random_seed=0x%x\n",
+		   random_seed);
+
+	return 0;
+}
+
+static struct kunit_case gpu_buddy_tests[] = {
+	KUNIT_CASE(gpu_test_buddy_alloc_limit),
+	KUNIT_CASE(gpu_test_buddy_alloc_optimistic),
+	KUNIT_CASE(gpu_test_buddy_alloc_pessimistic),
+	KUNIT_CASE(gpu_test_buddy_alloc_pathological),
+	KUNIT_CASE(gpu_test_buddy_alloc_contiguous),
+	KUNIT_CASE(gpu_test_buddy_alloc_clear),
+	KUNIT_CASE(gpu_test_buddy_alloc_range),
+	KUNIT_CASE(gpu_test_buddy_alloc_range_bias),
+	KUNIT_CASE_SLOW(gpu_test_buddy_fragmentation_performance),
+	KUNIT_CASE(gpu_test_buddy_alloc_exceeds_max_order),
+	KUNIT_CASE(gpu_test_buddy_offset_aligned_allocation),
+	KUNIT_CASE(gpu_test_buddy_subtree_offset_alignment_stress),
+	{}
+};
+
+static struct kunit_suite gpu_buddy_test_suite = {
+	.name = "gpu_buddy",
+	.suite_init = gpu_buddy_suite_init,
+	.test_cases = gpu_buddy_tests,
+};
+
+kunit_test_suite(gpu_buddy_test_suite);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("Kunit test for gpu_buddy functions");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/tests/gpu_random.c b/drivers/gpu/tests/gpu_random.c
new file mode 100644
index 0000000000000..6356372f7e52e
--- /dev/null
+++ b/drivers/gpu/tests/gpu_random.c
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bitops.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "gpu_random.h"
+
+u32 gpu_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
+{
+	return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
+}
+EXPORT_SYMBOL(gpu_prandom_u32_max_state);
+
+void gpu_random_reorder(unsigned int *order, unsigned int count,
+			struct rnd_state *state)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < count; ++i) {
+		BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));
+		j = gpu_prandom_u32_max_state(count, state);
+		swap(order[i], order[j]);
+	}
+}
+EXPORT_SYMBOL(gpu_random_reorder);
+
+unsigned int *gpu_random_order(unsigned int count, struct rnd_state *state)
+{
+	unsigned int *order, i;
+
+	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
+	if (!order)
+		return order;
+
+	for (i = 0; i < count; i++)
+		order[i] = i;
+
+	gpu_random_reorder(order, count, state);
+	return order;
+}
+EXPORT_SYMBOL(gpu_random_order);
diff --git a/drivers/gpu/tests/gpu_random.h b/drivers/gpu/tests/gpu_random.h
new file mode 100644
index 0000000000000..b68cf34482644
--- /dev/null
+++ b/drivers/gpu/tests/gpu_random.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __GPU_RANDOM_H__
+#define __GPU_RANDOM_H__
+
+/* This is a temporary home for a couple of utility functions that should
+ * be transposed to lib/ at the earliest convenience.
+ */
+
+#include <linux/prandom.h>
+
+#define GPU_RND_STATE_INITIALIZER(seed__) ({				\
+	struct rnd_state state__;					\
+	prandom_seed_state(&state__, (seed__));				\
+	state__;							\
+})
+
+#define GPU_RND_STATE(name__, seed__) \
+	struct rnd_state name__ = GPU_RND_STATE_INITIALIZER(seed__)
+
+unsigned int *gpu_random_order(unsigned int count,
+			       struct rnd_state *state);
+void gpu_random_reorder(unsigned int *order,
+			unsigned int count,
+			struct rnd_state *state);
+u32 gpu_prandom_u32_max_state(u32 ep_ro,
+			      struct rnd_state *state);
+
+#endif /* !__GPU_RANDOM_H__ */
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index c1d9f7c6a5f23..10c12d8e65579 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -623,6 +623,30 @@ config HID_LENOVO
 	- ThinkPad Compact Bluetooth Keyboard with TrackPoint (supports Fn keys)
 	- ThinkPad Compact USB Keyboard with TrackPoint (supports Fn keys)
 
+config HID_LENOVO_GO
+	tristate "HID Driver for Lenovo Legion Go Series Controllers"
+	depends on USB_HID
+	select LEDS_CLASS
+	select LEDS_CLASS_MULTICOLOR
+	help
+	Support for Lenovo Legion Go devices with detachable controllers.
+
+	Say Y here to include configuration interface support for the Lenovo Legion Go
+	and Legion Go 2 Handheld Console Controllers. Say M here to compile this
+	driver as a module. The module will be called hid-lenovo-go.
+
+config HID_LENOVO_GO_S
+	tristate "HID Driver for Lenovo Legion Go S Controller"
+	depends on USB_HID
+	select LEDS_CLASS
+	select LEDS_CLASS_MULTICOLOR
+	help
+	Support for Lenovo Legion Go S Handheld Console Controller.
+
+	Say Y here to include configuration interface support for the Lenovo Legion Go
+	S. Say M here to compile this driver as a module. The module will be called
+	hid-lenovo-go-s.
+
 config HID_LETSKETCH
 	tristate "Letsketch WP9620N tablets"
 	depends on USB_HID
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index e01838239ae64..07dfdb6a49c59 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -76,6 +76,8 @@ obj-$(CONFIG_HID_KYE)		+= hid-kye.o
 obj-$(CONFIG_HID_KYSONA)	+= hid-kysona.o
 obj-$(CONFIG_HID_LCPOWER)	+= hid-lcpower.o
 obj-$(CONFIG_HID_LENOVO)	+= hid-lenovo.o
+obj-$(CONFIG_HID_LENOVO_GO)	+= hid-lenovo-go.o
+obj-$(CONFIG_HID_LENOVO_GO_S)	+= hid-lenovo-go-s.o
 obj-$(CONFIG_HID_LETSKETCH)	+= hid-letsketch.o
 obj-$(CONFIG_HID_LOGITECH)	+= hid-logitech.o
 obj-$(CONFIG_HID_LOGITECH)	+= hid-lg-g15.o
diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c
index f3d15994ca1ed..50c7b45c59e3f 100644
--- a/drivers/hid/bpf/hid_bpf_dispatch.c
+++ b/drivers/hid/bpf/hid_bpf_dispatch.c
@@ -444,6 +444,8 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz,
 					      (u64)(long)ctx,
 					      true); /* prevent infinite recursions */
 
+	if (ret > size)
+		ret = size;
 	if (ret > 0)
 		memcpy(buf, dma_data, ret);
 
diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c
index fc5897a6bb532..8b8b05c8a441f 100644
--- a/drivers/hid/hid-apple.c
+++ b/drivers/hid/hid-apple.c
@@ -623,17 +623,19 @@ static int apple_fetch_battery(struct hid_device *hdev)
 	struct apple_sc *asc = hid_get_drvdata(hdev);
 	struct hid_report_enum *report_enum;
 	struct hid_report *report;
+	struct hid_battery *bat;
 
-	if (!(asc->quirks & APPLE_RDESC_BATTERY) || !hdev->battery)
+	bat = hid_get_battery(hdev);
+	if (!(asc->quirks & APPLE_RDESC_BATTERY) || !bat)
 		return -1;
 
-	report_enum = &hdev->report_enum[hdev->battery_report_type];
-	report = report_enum->report_id_hash[hdev->battery_report_id];
+	report_enum = &hdev->report_enum[bat->report_type];
+	report = report_enum->report_id_hash[bat->report_id];
 
 	if (!report || report->maxfield < 1)
 		return -1;
 
-	if (hdev->battery_capacity == hdev->battery_max)
+	if (bat->capacity == bat->max)
 		return -1;
 
 	hid_hw_request(hdev, report, HID_REQ_GET_REPORT);
diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c
index a1db3b3d06676..0fdc0968b9ef2 100644
--- a/drivers/hid/hid-appletb-kbd.c
+++ b/drivers/hid/hid-appletb-kbd.c
@@ -476,7 +476,7 @@ static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg)
 	return 0;
 }
 
-static int appletb_kbd_reset_resume(struct hid_device *hdev)
+static int appletb_kbd_resume(struct hid_device *hdev)
 {
 	struct appletb_kbd *kbd = hid_get_drvdata(hdev);
 
@@ -500,7 +500,8 @@ static struct hid_driver appletb_kbd_hid_driver = {
 	.event = appletb_kbd_hid_event,
 	.input_configured = appletb_kbd_input_configured,
 	.suspend = pm_ptr(appletb_kbd_suspend),
-	.reset_resume = pm_ptr(appletb_kbd_reset_resume),
+	.resume = pm_ptr(appletb_kbd_resume),
+	.reset_resume = pm_ptr(appletb_kbd_resume),
 	.driver.dev_groups = appletb_kbd_groups,
 };
 module_hid_driver(appletb_kbd_hid_driver);
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index 687b785e2d0c1..d34d74df3dc0b 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -20,10 +20,8 @@
  *  Copyright (c) 2016 Frederik Wenigwieser <frederik.wenigwieser@gmail.com>
  */
 
-/*
- */
-
 #include <linux/acpi.h>
+#include <linux/cleanup.h>
 #include <linux/dmi.h>
 #include <linux/hid.h>
 #include <linux/module.h>
@@ -101,7 +99,6 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
 #define QUIRK_ROG_CLAYMORE_II_KEYBOARD	BIT(12)
 #define QUIRK_ROG_ALLY_XPAD		BIT(13)
 #define QUIRK_HID_FN_LOCK		BIT(14)
-#define QUIRK_ROG_NKEY_ID1ID2_INIT	BIT(15)
 
 #define I2C_KEYBOARD_QUIRKS			(QUIRK_FIX_NOTEBOOK_REPORT | \
 						 QUIRK_NO_INIT_REPORTS | \
@@ -208,6 +205,12 @@ static const struct asus_touchpad_info medion_e1239t_tp = {
 	.report_size = 32 /* 2 byte header + 5 * 5 + 5 byte footer */,
 };
 
+static const u8 asus_report_id_init[] = {
+	FEATURE_KBD_REPORT_ID,
+	FEATURE_KBD_LED_REPORT_ID1,
+	FEATURE_KBD_LED_REPORT_ID2
+};
+
 static void asus_report_contact_down(struct asus_drvdata *drvdat,
 		int toolType, u8 *data)
 {
@@ -354,7 +357,7 @@ static int asus_event(struct hid_device *hdev, struct hid_field *field,
 		      struct hid_usage *usage, __s32 value)
 {
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
-	
+
 	if ((usage->hid & HID_USAGE_PAGE) == HID_UP_ASUSVENDOR &&
 	    (usage->hid & HID_USAGE) != 0x00 &&
 	    (usage->hid & HID_USAGE) != 0xff && !usage->type) {
@@ -443,21 +446,18 @@ static int asus_raw_event(struct hid_device *hdev,
 		/*
 		 * G713 and G733 send these codes on some keypresses, depending on
 		 * the key pressed it can trigger a shutdown event if not caught.
-		*/
-		if (data[0] == 0x02 && data[1] == 0x30) {
+		 */
+		if (data[0] == 0x02 && data[1] == 0x30)
 			return -1;
-		}
 	}
 
 	if (drvdata->quirks & QUIRK_ROG_CLAYMORE_II_KEYBOARD) {
 		/*
 		 * CLAYMORE II keyboard sends this packet when it goes to sleep
 		 * this causes the whole system to go into suspend.
-		*/
-
-		if(size == 2 && data[0] == 0x02 && data[1] == 0x00) {
+		 */
+		if (size == 2 && data[0] == 0x02 && data[1] == 0x00)
 			return -1;
-		}
 	}
 
 	return 0;
@@ -465,23 +465,16 @@ static int asus_raw_event(struct hid_device *hdev,
 
 static int asus_kbd_set_report(struct hid_device *hdev, const u8 *buf, size_t buf_size)
 {
-	unsigned char *dmabuf;
-	int ret;
-
-	dmabuf = kmemdup(buf, buf_size, GFP_KERNEL);
+	u8 *dmabuf __free(kfree) = kmemdup(buf, buf_size, GFP_KERNEL);
 	if (!dmabuf)
 		return -ENOMEM;
 
 	/*
 	 * The report ID should be set from the incoming buffer due to LED and key
 	 * interfaces having different pages
-	*/
-	ret = hid_hw_raw_request(hdev, buf[0], dmabuf,
-				 buf_size, HID_FEATURE_REPORT,
-				 HID_REQ_SET_REPORT);
-	kfree(dmabuf);
-
-	return ret;
+	 */
+	return hid_hw_raw_request(hdev, buf[0], dmabuf, buf_size, HID_FEATURE_REPORT,
+				  HID_REQ_SET_REPORT);
 }
 
 static int asus_kbd_init(struct hid_device *hdev, u8 report_id)
@@ -722,6 +715,21 @@ static void validate_mcu_fw_version(struct hid_device *hdev, int idProduct)
 	}
 }
 
+static bool asus_has_report_id(struct hid_device *hdev, u16 report_id)
+{
+	struct hid_report *report;
+	int t;
+
+	for (t = HID_INPUT_REPORT; t <= HID_FEATURE_REPORT; t++) {
+		list_for_each_entry(report, &hdev->report_enum[t].report_list, list) {
+			if (report->id == report_id)
+				return true;
+		}
+	}
+
+	return false;
+}
+
 static int asus_kbd_register_leds(struct hid_device *hdev)
 {
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
@@ -730,10 +738,6 @@ static int asus_kbd_register_leds(struct hid_device *hdev)
 	unsigned char kbd_func;
 	int ret;
 
-	ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID);
-	if (ret < 0)
-		return ret;
-
 	/* Get keyboard functions */
 	ret = asus_kbd_get_functions(hdev, &kbd_func, FEATURE_KBD_REPORT_ID);
 	if (ret < 0)
@@ -743,11 +747,6 @@ static int asus_kbd_register_leds(struct hid_device *hdev)
 	if (!(kbd_func & SUPPORT_KBD_BACKLIGHT))
 		return -ENODEV;
 
-	if (drvdata->quirks & QUIRK_ROG_NKEY_ID1ID2_INIT) {
-		asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID1);
-		asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID2);
-	}
-
 	if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) {
 		ret = asus_kbd_disable_oobe(hdev);
 		if (ret < 0)
@@ -966,6 +965,12 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		}
 	}
 
+	if (drvdata->quirks & QUIRK_HID_FN_LOCK) {
+		drvdata->fn_lock = true;
+		INIT_WORK(&drvdata->fn_lock_sync_work, asus_sync_fn_lock);
+		asus_kbd_set_fn_lock(hdev, true);
+	}
+
 	if (drvdata->tp) {
 		int ret;
 
@@ -1163,7 +1168,8 @@ static int asus_start_multitouch(struct hid_device *hdev)
 	return 0;
 }
 
-static int __maybe_unused asus_resume(struct hid_device *hdev) {
+static int __maybe_unused asus_resume(struct hid_device *hdev)
+{
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
 	int ret = 0;
 
@@ -1294,8 +1300,19 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		return ret;
 	}
 
+	for (int r = 0; r < ARRAY_SIZE(asus_report_id_init); r++) {
+		if (asus_has_report_id(hdev, asus_report_id_init[r])) {
+			ret = asus_kbd_init(hdev, asus_report_id_init[r]);
+			if (ret < 0)
+				hid_warn(hdev, "Failed to initialize 0x%x: %d.\n",
+					 asus_report_id_init[r], ret);
+		}
+	}
+
+	/* Laptops keyboard backlight is always at 0x5a */
 	if (is_vendor && (drvdata->quirks & QUIRK_USE_KBD_BACKLIGHT) &&
-	    asus_kbd_register_leds(hdev))
+	    (asus_has_report_id(hdev, FEATURE_KBD_REPORT_ID)) &&
+		(asus_kbd_register_leds(hdev)))
 		hid_warn(hdev, "Failed to initialize backlight.\n");
 
 	/*
@@ -1311,22 +1328,17 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	 * were freed during registration due to no usages being mapped,
 	 * leaving drvdata->input pointing to freed memory.
 	 */
-	if (!drvdata->input || !(hdev->claimed & HID_CLAIMED_INPUT)) {
-		hid_err(hdev, "Asus input not registered\n");
-		ret = -ENOMEM;
-		goto err_stop_hw;
-	}
-
-	if (drvdata->tp) {
-		drvdata->input->name = "Asus TouchPad";
-	} else {
-		drvdata->input->name = "Asus Keyboard";
-	}
+	if (drvdata->input && (hdev->claimed & HID_CLAIMED_INPUT)) {
+		if (drvdata->tp)
+			drvdata->input->name = "Asus TouchPad";
+		else
+			drvdata->input->name = "Asus Keyboard";
 
-	if (drvdata->tp) {
-		ret = asus_start_multitouch(hdev);
-		if (ret)
-			goto err_stop_hw;
+		if (drvdata->tp) {
+			ret = asus_start_multitouch(hdev);
+			if (ret)
+				goto err_stop_hw;
+		}
 	}
 
 	return 0;
@@ -1484,10 +1496,10 @@ static const struct hid_device_id asus_devices[] = {
 	  QUIRK_USE_KBD_BACKLIGHT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD),
-	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_NKEY_ID1ID2_INIT },
+	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2),
-	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_HID_FN_LOCK | QUIRK_ROG_NKEY_ID1ID2_INIT },
+	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_HID_FN_LOCK },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR),
 	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
@@ -1497,6 +1509,9 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X),
 	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
+	    USB_DEVICE_ID_ASUSTEK_XGM_2022),
+	},
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_XGM_2023),
 	},
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 840a601138688..e7658c771f5dd 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -2057,9 +2057,10 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
 		rsize = max_buffer_size;
 
 	if (csize < rsize) {
-		dbg_hid("report %d is too short, (%d < %d)\n", report->id,
-				csize, rsize);
-		memset(cdata + csize, 0, rsize - csize);
+		hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %d)\n",
+				     report->id, rsize, csize);
+		ret = -EINVAL;
+		goto out;
 	}
 
 	if ((hid->claimed & HID_CLAIMED_HIDDEV) && hid->hiddev_report_event)
@@ -2887,6 +2888,11 @@ static int hid_uevent(const struct device *dev, struct kobj_uevent_env *env)
 	if (add_uevent_var(env, "MODALIAS=hid:b%04Xg%04Xv%08Xp%08X",
 			   hdev->bus, hdev->group, hdev->vendor, hdev->product))
 		return -ENOMEM;
+	if (hdev->firmware_version) {
+		if (add_uevent_var(env, "HID_FIRMWARE_VERSION=0x%04llX",
+				   hdev->firmware_version))
+			return -ENOMEM;
+	}
 
 	return 0;
 }
@@ -2990,6 +2996,10 @@ struct hid_device *hid_allocate_device(void)
 	mutex_init(&hdev->ll_open_lock);
 	kref_init(&hdev->ref);
 
+#ifdef CONFIG_HID_BATTERY_STRENGTH
+	INIT_LIST_HEAD(&hdev->batteries);
+#endif
+
 	ret = hid_bpf_device_init(hdev);
 	if (ret)
 		goto out_err;
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 4ab7640b119ac..933b7943bdb50 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -229,6 +229,7 @@
 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X		0x1b4c
 #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD	0x196b
 #define USB_DEVICE_ID_ASUSTEK_FX503VD_KEYBOARD	0x1869
+#define USB_DEVICE_ID_ASUSTEK_XGM_2022	0x1970
 #define USB_DEVICE_ID_ASUSTEK_XGM_2023	0x1a9a
 
 #define USB_VENDOR_ID_ATEN		0x0557
@@ -454,8 +455,6 @@
 #define USB_DEVICE_ID_TOSHIBA_CLICK_L9W	0x0401
 #define USB_DEVICE_ID_HP_X2		0x074d
 #define USB_DEVICE_ID_HP_X2_10_COVER	0x0755
-#define USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN	0x2544
-#define USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN	0x2706
 #define I2C_DEVICE_ID_CHROMEBOOK_TROGDOR_POMPOM	0x2F81
 
 #define USB_VENDOR_ID_ELECOM		0x056e
@@ -740,6 +739,10 @@
 #define USB_DEVICE_ID_ITE8595		0x8595
 #define USB_DEVICE_ID_ITE_MEDION_E1239T	0xce50
 
+#define USB_VENDOR_ID_QHE		0x1a86
+#define USB_DEVICE_ID_LENOVO_LEGION_GO_S_XINPUT 0xe310
+#define USB_DEVICE_ID_LENOVO_LEGION_GO_S_DINPUT 0xe311
+
 #define USB_VENDOR_ID_JABRA		0x0b0e
 #define USB_DEVICE_ID_JABRA_SPEAK_410	0x0412
 #define USB_DEVICE_ID_JABRA_SPEAK_510	0x0420
@@ -859,7 +862,10 @@
 #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_602E	0x602e
 #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6093	0x6093
 #define USB_DEVICE_ID_LENOVO_LEGION_GO_DUAL_DINPUT	0x6184
+#define USB_DEVICE_ID_LENOVO_LEGION_GO2_XINPUT		0x61eb
+#define USB_DEVICE_ID_LENOVO_LEGION_GO2_DINPUT		0x61ec
 #define USB_DEVICE_ID_LENOVO_LEGION_GO2_DUAL_DINPUT	0x61ed
+#define USB_DEVICE_ID_LENOVO_LEGION_GO2_FPS		0x61ee
 
 #define USB_VENDOR_ID_LETSKETCH		0x6161
 #define USB_DEVICE_ID_WP9620N		0x4d15
diff --git a/drivers/hid/hid-input-test.c b/drivers/hid/hid-input-test.c
index 6f5c71660d823..c92008dafddfd 100644
--- a/drivers/hid/hid-input-test.c
+++ b/drivers/hid/hid-input-test.c
@@ -9,54 +9,59 @@
 
 static void hid_test_input_update_battery_charge_status(struct kunit *test)
 {
-	struct hid_device *dev;
+	struct hid_battery *bat;
 	bool handled;
 
-	dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	bat = kunit_kzalloc(test, sizeof(*bat), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bat);
 
-	handled = hidinput_update_battery_charge_status(dev, HID_DG_HEIGHT, 0);
+	handled = hidinput_update_battery_charge_status(bat, HID_DG_HEIGHT, 0);
 	KUNIT_EXPECT_FALSE(test, handled);
-	KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_UNKNOWN);
+	KUNIT_EXPECT_EQ(test, bat->charge_status, POWER_SUPPLY_STATUS_UNKNOWN);
 
-	handled = hidinput_update_battery_charge_status(dev, HID_BAT_CHARGING, 0);
+	handled = hidinput_update_battery_charge_status(bat, HID_BAT_CHARGING, 0);
 	KUNIT_EXPECT_TRUE(test, handled);
-	KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_DISCHARGING);
+	KUNIT_EXPECT_EQ(test, bat->charge_status, POWER_SUPPLY_STATUS_DISCHARGING);
 
-	handled = hidinput_update_battery_charge_status(dev, HID_BAT_CHARGING, 1);
+	handled = hidinput_update_battery_charge_status(bat, HID_BAT_CHARGING, 1);
 	KUNIT_EXPECT_TRUE(test, handled);
-	KUNIT_EXPECT_EQ(test, dev->battery_charge_status, POWER_SUPPLY_STATUS_CHARGING);
+	KUNIT_EXPECT_EQ(test, bat->charge_status, POWER_SUPPLY_STATUS_CHARGING);
 }
 
 static void hid_test_input_get_battery_property(struct kunit *test)
 {
 	struct power_supply *psy;
+	struct hid_battery *bat;
 	struct hid_device *dev;
 	union power_supply_propval val;
 	int ret;
 
 	dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
-	dev->battery_avoid_query = true;
+
+	bat = kunit_kzalloc(test, sizeof(*bat), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bat);
+	bat->dev = dev;
+	bat->avoid_query = true;
 
 	psy = kunit_kzalloc(test, sizeof(*psy), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, psy);
-	psy->drv_data = dev;
+	psy->drv_data = bat;
 
-	dev->battery_status = HID_BATTERY_UNKNOWN;
-	dev->battery_charge_status = POWER_SUPPLY_STATUS_CHARGING;
+	bat->status = HID_BATTERY_UNKNOWN;
+	bat->charge_status = POWER_SUPPLY_STATUS_CHARGING;
 	ret = hidinput_get_battery_property(psy, POWER_SUPPLY_PROP_STATUS, &val);
 	KUNIT_EXPECT_EQ(test, ret, 0);
 	KUNIT_EXPECT_EQ(test, val.intval, POWER_SUPPLY_STATUS_UNKNOWN);
 
-	dev->battery_status = HID_BATTERY_REPORTED;
-	dev->battery_charge_status = POWER_SUPPLY_STATUS_CHARGING;
+	bat->status = HID_BATTERY_REPORTED;
+	bat->charge_status = POWER_SUPPLY_STATUS_CHARGING;
 	ret = hidinput_get_battery_property(psy, POWER_SUPPLY_PROP_STATUS, &val);
 	KUNIT_EXPECT_EQ(test, ret, 0);
 	KUNIT_EXPECT_EQ(test, val.intval, POWER_SUPPLY_STATUS_CHARGING);
 
-	dev->battery_status = HID_BATTERY_REPORTED;
-	dev->battery_charge_status = POWER_SUPPLY_STATUS_DISCHARGING;
+	bat->status = HID_BATTERY_REPORTED;
+	bat->charge_status = POWER_SUPPLY_STATUS_DISCHARGING;
 	ret = hidinput_get_battery_property(psy, POWER_SUPPLY_PROP_STATUS, &val);
 	KUNIT_EXPECT_EQ(test, ret, 0);
 	KUNIT_EXPECT_EQ(test, val.intval, POWER_SUPPLY_STATUS_DISCHARGING);
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index d5308adb28948..8fff185fe0e6b 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -354,6 +354,7 @@ static enum power_supply_property hidinput_battery_props[] = {
 #define HID_BATTERY_QUIRK_FEATURE	(1 << 1) /* ask for feature report */
 #define HID_BATTERY_QUIRK_IGNORE	(1 << 2) /* completely ignore the battery */
 #define HID_BATTERY_QUIRK_AVOID_QUERY	(1 << 3) /* do not query the battery */
+#define HID_BATTERY_QUIRK_DYNAMIC	(1 << 4) /* report present only after life signs */
 
 static const struct hid_device_id hid_battery_quirks[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
@@ -386,10 +387,6 @@ static const struct hid_device_id hid_battery_quirks[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH,
 		USB_DEVICE_ID_LOGITECH_DINOVO_EDGE_KBD),
 	  HID_BATTERY_QUIRK_IGNORE },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN),
-	  HID_BATTERY_QUIRK_IGNORE },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN),
-	  HID_BATTERY_QUIRK_IGNORE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_L),
 	  HID_BATTERY_QUIRK_AVOID_QUERY },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_MW),
@@ -402,8 +399,8 @@ static const struct hid_device_id hid_battery_quirks[] = {
 	 * Elan HID touchscreens seem to all report a non present battery,
 	 * set HID_BATTERY_QUIRK_IGNORE for all Elan I2C and USB HID devices.
 	 */
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_BATTERY_QUIRK_IGNORE },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_BATTERY_QUIRK_IGNORE },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_BATTERY_QUIRK_DYNAMIC },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_BATTERY_QUIRK_DYNAMIC },
 	{}
 };
 
@@ -419,60 +416,59 @@ static unsigned find_battery_quirk(struct hid_device *hdev)
 	return quirks;
 }
 
-static int hidinput_scale_battery_capacity(struct hid_device *dev,
+static int hidinput_scale_battery_capacity(struct hid_battery *bat,
 					   int value)
 {
-	if (dev->battery_min < dev->battery_max &&
-	    value >= dev->battery_min && value <= dev->battery_max)
-		value = ((value - dev->battery_min) * 100) /
-			(dev->battery_max - dev->battery_min);
+	if (bat->min < bat->max &&
+	    value >= bat->min && value <= bat->max)
+		value = ((value - bat->min) * 100) /
+			(bat->max - bat->min);
 
 	return value;
 }
 
-static int hidinput_query_battery_capacity(struct hid_device *dev)
+static int hidinput_query_battery_capacity(struct hid_battery *bat)
 {
-	u8 *buf;
 	int ret;
 
-	buf = kmalloc(4, GFP_KERNEL);
+	u8 *buf __free(kfree) = kmalloc(4, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	ret = hid_hw_raw_request(dev, dev->battery_report_id, buf, 4,
-				 dev->battery_report_type, HID_REQ_GET_REPORT);
-	if (ret < 2) {
-		kfree(buf);
+	ret = hid_hw_raw_request(bat->dev, bat->report_id, buf, 4,
+				 bat->report_type, HID_REQ_GET_REPORT);
+	if (ret < 2)
 		return -ENODATA;
-	}
 
-	ret = hidinput_scale_battery_capacity(dev, buf[1]);
-	kfree(buf);
-	return ret;
+	return hidinput_scale_battery_capacity(bat, buf[1]);
 }
 
 static int hidinput_get_battery_property(struct power_supply *psy,
 					 enum power_supply_property prop,
 					 union power_supply_propval *val)
 {
-	struct hid_device *dev = power_supply_get_drvdata(psy);
+	struct hid_battery *bat = power_supply_get_drvdata(psy);
+	struct hid_device *dev = bat->dev;
 	int value;
 	int ret = 0;
 
 	switch (prop) {
-	case POWER_SUPPLY_PROP_PRESENT:
 	case POWER_SUPPLY_PROP_ONLINE:
 		val->intval = 1;
 		break;
 
+	case POWER_SUPPLY_PROP_PRESENT:
+		val->intval = bat->present;
+		break;
+
 	case POWER_SUPPLY_PROP_CAPACITY:
-		if (dev->battery_status != HID_BATTERY_REPORTED &&
-		    !dev->battery_avoid_query) {
-			value = hidinput_query_battery_capacity(dev);
+		if (bat->status != HID_BATTERY_REPORTED &&
+		    !bat->avoid_query) {
+			value = hidinput_query_battery_capacity(bat);
 			if (value < 0)
 				return value;
 		} else  {
-			value = dev->battery_capacity;
+			value = bat->capacity;
 		}
 
 		val->intval = value;
@@ -483,20 +479,20 @@ static int hidinput_get_battery_property(struct power_supply *psy,
 		break;
 
 	case POWER_SUPPLY_PROP_STATUS:
-		if (dev->battery_status != HID_BATTERY_REPORTED &&
-		    !dev->battery_avoid_query) {
-			value = hidinput_query_battery_capacity(dev);
+		if (bat->status != HID_BATTERY_REPORTED &&
+		    !bat->avoid_query) {
+			value = hidinput_query_battery_capacity(bat);
 			if (value < 0)
 				return value;
 
-			dev->battery_capacity = value;
-			dev->battery_status = HID_BATTERY_QUERIED;
+			bat->capacity = value;
+			bat->status = HID_BATTERY_QUERIED;
 		}
 
-		if (dev->battery_status == HID_BATTERY_UNKNOWN)
+		if (bat->status == HID_BATTERY_UNKNOWN)
 			val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
 		else
-			val->intval = dev->battery_charge_status;
+			val->intval = bat->charge_status;
 		break;
 
 	case POWER_SUPPLY_PROP_SCOPE:
@@ -511,36 +507,59 @@ static int hidinput_get_battery_property(struct power_supply *psy,
 	return ret;
 }
 
+static struct hid_battery *hidinput_find_battery(struct hid_device *dev,
+						 int report_id)
+{
+	struct hid_battery *bat;
+
+	list_for_each_entry(bat, &dev->batteries, list) {
+		if (bat->report_id == report_id)
+			return bat;
+	}
+	return NULL;
+}
+
 static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 				  struct hid_field *field, bool is_percentage)
 {
+	struct hid_battery *bat;
 	struct power_supply_desc *psy_desc;
-	struct power_supply_config psy_cfg = { .drv_data = dev, };
+	struct power_supply_config psy_cfg = { 0 };
 	unsigned quirks;
 	s32 min, max;
 	int error;
 
-	if (dev->battery)
-		return 0;	/* already initialized? */
+	/* Check if battery for this report ID already exists */
+	if (hidinput_find_battery(dev, field->report->id))
+		return 0;
 
 	quirks = find_battery_quirk(dev);
 
-	hid_dbg(dev, "device %x:%x:%x %d quirks %d\n",
-		dev->bus, dev->vendor, dev->product, dev->version, quirks);
+	hid_dbg(dev, "device %x:%x:%x %d quirks %d report_id %d\n",
+		dev->bus, dev->vendor, dev->product, dev->version, quirks,
+		field->report->id);
 
 	if (quirks & HID_BATTERY_QUIRK_IGNORE)
 		return 0;
 
-	psy_desc = kzalloc_obj(*psy_desc);
-	if (!psy_desc)
+	bat = devm_kzalloc(&dev->dev, sizeof(*bat), GFP_KERNEL);
+	if (!bat)
 		return -ENOMEM;
 
-	psy_desc->name = kasprintf(GFP_KERNEL, "hid-%s-battery",
-				   strlen(dev->uniq) ?
-					dev->uniq : dev_name(&dev->dev));
+	psy_desc = devm_kzalloc(&dev->dev, sizeof(*psy_desc), GFP_KERNEL);
+	if (!psy_desc) {
+		error = -ENOMEM;
+		goto err_free_bat;
+	}
+
+	psy_desc->name = devm_kasprintf(&dev->dev, GFP_KERNEL,
+					"hid-%s-battery-%d",
+					strlen(dev->uniq) ?
+						dev->uniq : dev_name(&dev->dev),
+					field->report->id);
 	if (!psy_desc->name) {
 		error = -ENOMEM;
-		goto err_free_mem;
+		goto err_free_desc;
 	}
 
 	psy_desc->type = POWER_SUPPLY_TYPE_BATTERY;
@@ -560,98 +579,95 @@ static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 	if (quirks & HID_BATTERY_QUIRK_FEATURE)
 		report_type = HID_FEATURE_REPORT;
 
-	dev->battery_min = min;
-	dev->battery_max = max;
-	dev->battery_report_type = report_type;
-	dev->battery_report_id = field->report->id;
-	dev->battery_charge_status = POWER_SUPPLY_STATUS_DISCHARGING;
+	bat->dev = dev;
+	bat->min = min;
+	bat->max = max;
+	bat->report_type = report_type;
+	bat->report_id = field->report->id;
+	bat->charge_status = POWER_SUPPLY_STATUS_DISCHARGING;
+	bat->status = HID_BATTERY_UNKNOWN;
 
 	/*
 	 * Stylus is normally not connected to the device and thus we
 	 * can't query the device and get meaningful battery strength.
 	 * We have to wait for the device to report it on its own.
 	 */
-	dev->battery_avoid_query = report_type == HID_INPUT_REPORT &&
-				   field->physical == HID_DG_STYLUS;
+	bat->avoid_query = report_type == HID_INPUT_REPORT &&
+			   field->physical == HID_DG_STYLUS;
 
 	if (quirks & HID_BATTERY_QUIRK_AVOID_QUERY)
-		dev->battery_avoid_query = true;
+		bat->avoid_query = true;
+
+	bat->present = (quirks & HID_BATTERY_QUIRK_DYNAMIC) ? false : true;
 
-	dev->battery = power_supply_register(&dev->dev, psy_desc, &psy_cfg);
-	if (IS_ERR(dev->battery)) {
-		error = PTR_ERR(dev->battery);
+	psy_cfg.drv_data = bat;
+	bat->ps = devm_power_supply_register(&dev->dev, psy_desc, &psy_cfg);
+	if (IS_ERR(bat->ps)) {
+		error = PTR_ERR(bat->ps);
 		hid_warn(dev, "can't register power supply: %d\n", error);
 		goto err_free_name;
 	}
 
-	power_supply_powers(dev->battery, &dev->dev);
+	power_supply_powers(bat->ps, &dev->dev);
+	list_add_tail(&bat->list, &dev->batteries);
 	return 0;
 
 err_free_name:
-	kfree(psy_desc->name);
-err_free_mem:
-	kfree(psy_desc);
-	dev->battery = NULL;
+	devm_kfree(&dev->dev, psy_desc->name);
+err_free_desc:
+	devm_kfree(&dev->dev, psy_desc);
+err_free_bat:
+	devm_kfree(&dev->dev, bat);
 	return error;
 }
 
-static void hidinput_cleanup_battery(struct hid_device *dev)
-{
-	const struct power_supply_desc *psy_desc;
-
-	if (!dev->battery)
-		return;
-
-	psy_desc = dev->battery->desc;
-	power_supply_unregister(dev->battery);
-	kfree(psy_desc->name);
-	kfree(psy_desc);
-	dev->battery = NULL;
-}
-
-static bool hidinput_update_battery_charge_status(struct hid_device *dev,
+static bool hidinput_update_battery_charge_status(struct hid_battery *bat,
 						  unsigned int usage, int value)
 {
 	switch (usage) {
 	case HID_BAT_CHARGING:
-		dev->battery_charge_status = value ?
-					     POWER_SUPPLY_STATUS_CHARGING :
-					     POWER_SUPPLY_STATUS_DISCHARGING;
+		bat->charge_status = value ?
+				     POWER_SUPPLY_STATUS_CHARGING :
+				     POWER_SUPPLY_STATUS_DISCHARGING;
 		return true;
 	}
 
 	return false;
 }
 
-static void hidinput_update_battery(struct hid_device *dev, unsigned int usage,
-				    int value)
+static void hidinput_update_battery(struct hid_device *dev, int report_id,
+				    unsigned int usage, int value)
 {
+	struct hid_battery *bat;
 	int capacity;
 
-	if (!dev->battery)
+	bat = hidinput_find_battery(dev, report_id);
+	if (!bat)
 		return;
 
-	if (hidinput_update_battery_charge_status(dev, usage, value)) {
-		power_supply_changed(dev->battery);
+	if (hidinput_update_battery_charge_status(bat, usage, value)) {
+		bat->present = true;
+		power_supply_changed(bat->ps);
 		return;
 	}
 
 	if ((usage & HID_USAGE_PAGE) == HID_UP_DIGITIZER && value == 0)
 		return;
 
-	if (value < dev->battery_min || value > dev->battery_max)
+	if (value < bat->min || value > bat->max)
 		return;
 
-	capacity = hidinput_scale_battery_capacity(dev, value);
+	capacity = hidinput_scale_battery_capacity(bat, value);
 
-	if (dev->battery_status != HID_BATTERY_REPORTED ||
-	    capacity != dev->battery_capacity ||
-	    ktime_after(ktime_get_coarse(), dev->battery_ratelimit_time)) {
-		dev->battery_capacity = capacity;
-		dev->battery_status = HID_BATTERY_REPORTED;
-		dev->battery_ratelimit_time =
+	if (bat->status != HID_BATTERY_REPORTED ||
+	    capacity != bat->capacity ||
+	    ktime_after(ktime_get_coarse(), bat->ratelimit_time)) {
+		bat->present = true;
+		bat->capacity = capacity;
+		bat->status = HID_BATTERY_REPORTED;
+		bat->ratelimit_time =
 			ktime_add_ms(ktime_get_coarse(), 30 * 1000);
-		power_supply_changed(dev->battery);
+		power_supply_changed(bat->ps);
 	}
 }
 #else  /* !CONFIG_HID_BATTERY_STRENGTH */
@@ -661,12 +677,8 @@ static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 	return 0;
 }
 
-static void hidinput_cleanup_battery(struct hid_device *dev)
-{
-}
-
-static void hidinput_update_battery(struct hid_device *dev, unsigned int usage,
-				    int value)
+static void hidinput_update_battery(struct hid_device *dev, int report_id,
+				    unsigned int usage, int value)
 {
 }
 #endif	/* CONFIG_HID_BATTERY_STRENGTH */
@@ -1550,7 +1562,7 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct
 		return;
 
 	if (usage->type == EV_PWR) {
-		hidinput_update_battery(hid, usage->hid, value);
+		hidinput_update_battery(hid, report->id, usage->hid, value);
 		return;
 	}
 
@@ -1832,7 +1844,6 @@ static void hidinput_led_worker(struct work_struct *work)
 	struct hid_report *report;
 	int ret;
 	u32 len;
-	__u8 *buf;
 
 	field = hidinput_get_led_field(hid);
 	if (!field)
@@ -1859,7 +1870,7 @@ static void hidinput_led_worker(struct work_struct *work)
 
 	/* fall back to generic raw-output-report */
 	len = hid_report_len(report);
-	buf = hid_alloc_report_buf(report, GFP_KERNEL);
+	u8 *buf __free(kfree) = hid_alloc_report_buf(report, GFP_KERNEL);
 	if (!buf)
 		return;
 
@@ -1869,7 +1880,6 @@ static void hidinput_led_worker(struct work_struct *work)
 	if (ret == -ENOSYS)
 		hid_hw_raw_request(hid, report->id, buf, len, HID_OUTPUT_REPORT,
 				HID_REQ_SET_REPORT);
-	kfree(buf);
 }
 
 static int hidinput_input_event(struct input_dev *dev, unsigned int type,
@@ -2396,8 +2406,6 @@ void hidinput_disconnect(struct hid_device *hid)
 {
 	struct hid_input *hidinput, *next;
 
-	hidinput_cleanup_battery(hid);
-
 	list_for_each_entry_safe(hidinput, next, &hid->inputs, list) {
 		list_del(&hidinput->list);
 		if (hidinput->registered)
diff --git a/drivers/hid/hid-lenovo-go-s.c b/drivers/hid/hid-lenovo-go-s.c
new file mode 100644
index 0000000000000..01c7bdd4fbe04
--- /dev/null
+++ b/drivers/hid/hid-lenovo-go-s.c
@@ -0,0 +1,1504 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *  HID driver for Lenovo Legion Go S devices.
+ *
+ *  Copyright (c) 2026 Derek J. Clark <derekjohn.clark@gmail.com>
+ *  Copyright (c) 2026 Valve Corporation
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/array_size.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/device.h>
+#include <linux/hid.h>
+#include <linux/jiffies.h>
+#include <linux/kstrtox.h>
+#include <linux/led-class-multicolor.h>
+#include <linux/mutex.h>
+#include <linux/printk.h>
+#include <linux/string.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+#include <linux/usb.h>
+#include <linux/workqueue.h>
+#include <linux/workqueue_types.h>
+
+#include "hid-ids.h"
+
+#define GO_S_CFG_INTF_IN	0x84
+#define GO_S_PACKET_SIZE	64
+
+static struct hid_gos_cfg {
+	struct delayed_work gos_cfg_setup;
+	struct completion send_cmd_complete;
+	struct led_classdev *led_cdev;
+	struct hid_device *hdev;
+	struct mutex cfg_mutex; /*ensure single synchronous output report*/
+	u8 gp_auto_sleep_time;
+	u8 gp_dpad_mode;
+	u8 gp_mode;
+	u8 gp_poll_rate;
+	u8 imu_bypass_en;
+	u8 imu_manufacturer;
+	u8 imu_sensor_en;
+	u8 mcu_id[12];
+	u8 mouse_step;
+	u8 os_mode;
+	u8 rgb_effect;
+	u8 rgb_en;
+	u8 rgb_mode;
+	u8 rgb_profile;
+	u8 rgb_speed;
+	u8 tp_en;
+	u8 tp_linux_mode;
+	u8 tp_windows_mode;
+	u8 tp_version;
+	u8 tp_manufacturer;
+} drvdata;
+
+struct gos_cfg_attr {
+	u8 index;
+};
+
+struct command_report {
+	u8 cmd;
+	u8 sub_cmd;
+	u8 data[63];
+} __packed;
+
+struct version_report {
+	u8 cmd;
+	u32 version;
+	u8 reserved[59];
+} __packed;
+
+enum mcu_command_index {
+	GET_VERSION = 0x01,
+	GET_MCU_ID,
+	GET_GAMEPAD_CFG,
+	SET_GAMEPAD_CFG,
+	GET_TP_PARAM,
+	SET_TP_PARAM,
+	GET_RGB_CFG = 0x0f,
+	SET_RGB_CFG,
+	GET_PL_TEST = 0xdf,
+};
+
+enum feature_enabled_index {
+	FEATURE_DISABLED,
+	FEATURE_ENABLED,
+};
+
+static const char *const feature_enabled_text[] = {
+	[FEATURE_DISABLED] = "false",
+	[FEATURE_ENABLED] = "true",
+};
+
+enum feature_status_index {
+	FEATURE_NONE = 0x00,
+	FEATURE_GAMEPAD_MODE = 0x01,
+	FEATURE_AUTO_SLEEP_TIME = 0x04,
+	FEATURE_IMU_BYPASS,
+	FEATURE_RGB_ENABLE,
+	FEATURE_IMU_ENABLE,
+	FEATURE_TOUCHPAD_ENABLE,
+	FEATURE_OS_MODE = 0x0A,
+	FEATURE_POLL_RATE = 0x10,
+	FEATURE_DPAD_MODE,
+	FEATURE_MOUSE_WHEEL_STEP,
+};
+
+enum gamepad_mode_index {
+	XINPUT,
+	DINPUT,
+};
+
+static const char *const gamepad_mode_text[] = {
+	[XINPUT] = "xinput",
+	[DINPUT] = "dinput",
+};
+
+enum os_type_index {
+	WINDOWS,
+	LINUX,
+};
+
+static const char *const os_type_text[] = {
+	[WINDOWS] = "windows",
+	[LINUX] = "linux",
+};
+
+enum poll_rate_index {
+	HZ125,
+	HZ250,
+	HZ500,
+	HZ1000,
+};
+
+static const char *const poll_rate_text[] = {
+	[HZ125] = "125",
+	[HZ250] = "250",
+	[HZ500] = "500",
+	[HZ1000] = "1000",
+};
+
+enum dpad_mode_index {
+	DIR8,
+	DIR4,
+};
+
+static const char *const dpad_mode_text[] = {
+	[DIR8] = "8-way",
+	[DIR4] = "4-way",
+};
+
+enum touchpad_mode_index {
+	TP_REL,
+	TP_ABS,
+};
+
+static const char *const touchpad_mode_text[] = {
+	[TP_REL] = "relative",
+	[TP_ABS] = "absolute",
+};
+
+enum touchpad_config_index {
+	CFG_WINDOWS_MODE = 0x03,
+	CFG_LINUX_MODE,
+
+};
+
+enum rgb_mode_index {
+	RGB_MODE_DYNAMIC,
+	RGB_MODE_CUSTOM,
+};
+
+static const char *const rgb_mode_text[] = {
+	[RGB_MODE_DYNAMIC] = "dynamic",
+	[RGB_MODE_CUSTOM] = "custom",
+};
+
+enum rgb_effect_index {
+	RGB_EFFECT_MONO,
+	RGB_EFFECT_BREATHE,
+	RGB_EFFECT_CHROMA,
+	RGB_EFFECT_RAINBOW,
+};
+
+static const char *const rgb_effect_text[] = {
+	[RGB_EFFECT_MONO] = "monocolor",
+	[RGB_EFFECT_BREATHE] = "breathe",
+	[RGB_EFFECT_CHROMA] = "chroma",
+	[RGB_EFFECT_RAINBOW] = "rainbow",
+};
+
+enum rgb_config_index {
+	LIGHT_MODE_SEL = 0x01,
+	LIGHT_PROFILE_SEL,
+	USR_LIGHT_PROFILE_1,
+	USR_LIGHT_PROFILE_2,
+	USR_LIGHT_PROFILE_3,
+};
+
+enum test_command_index {
+	TEST_TP_MFR = 0x02,
+	TEST_IMU_MFR,
+	TEST_TP_VER,
+};
+
+enum tp_mfr_index {
+	TP_NONE,
+	TP_BETTERLIFE,
+	TP_SIPO,
+};
+
+static const char *const touchpad_manufacturer_text[] = {
+	[TP_NONE] = "none",
+	[TP_BETTERLIFE] = "BetterLife",
+	[TP_SIPO] = "SIPO",
+};
+
+enum imu_mfr_index {
+	IMU_NONE,
+	IMU_BOSCH,
+	IMU_ST,
+};
+
+static const char *const imu_manufacturer_text[] = {
+	[IMU_NONE] = "none",
+	[IMU_BOSCH] = "Bosch",
+	[IMU_ST] = "ST",
+};
+
+static int hid_gos_version_event(u8 *data)
+{
+	struct version_report *ver_rep = (struct version_report *)data;
+
+	drvdata.hdev->firmware_version = get_unaligned_le32(&ver_rep->version);
+	return 0;
+}
+
+static int hid_gos_mcu_id_event(struct command_report *cmd_rep)
+{
+	drvdata.mcu_id[0] = cmd_rep->sub_cmd;
+	memcpy(&drvdata.mcu_id[1], cmd_rep->data, 11);
+
+	return 0;
+}
+
+static int hid_gos_gamepad_cfg_event(struct command_report *cmd_rep)
+{
+	int ret = 0;
+
+	switch (cmd_rep->sub_cmd) {
+	case FEATURE_GAMEPAD_MODE:
+		drvdata.gp_mode = cmd_rep->data[0];
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		drvdata.gp_auto_sleep_time = cmd_rep->data[0];
+		break;
+	case FEATURE_IMU_BYPASS:
+		drvdata.imu_bypass_en = cmd_rep->data[0];
+		break;
+	case FEATURE_RGB_ENABLE:
+		drvdata.rgb_en = cmd_rep->data[0];
+		break;
+	case FEATURE_IMU_ENABLE:
+		drvdata.imu_sensor_en = cmd_rep->data[0];
+		break;
+	case FEATURE_TOUCHPAD_ENABLE:
+		drvdata.tp_en = cmd_rep->data[0];
+		break;
+	case FEATURE_OS_MODE:
+		drvdata.os_mode = cmd_rep->data[0];
+		break;
+	case FEATURE_POLL_RATE:
+		drvdata.gp_poll_rate = cmd_rep->data[0];
+		break;
+	case FEATURE_DPAD_MODE:
+		drvdata.gp_dpad_mode = cmd_rep->data[0];
+		break;
+	case FEATURE_MOUSE_WHEEL_STEP:
+		drvdata.mouse_step = cmd_rep->data[0];
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int hid_gos_touchpad_event(struct command_report *cmd_rep)
+{
+	int ret = 0;
+
+	switch (cmd_rep->sub_cmd) {
+	case CFG_LINUX_MODE:
+		drvdata.tp_linux_mode = cmd_rep->data[0];
+		break;
+	case CFG_WINDOWS_MODE:
+		drvdata.tp_windows_mode = cmd_rep->data[0];
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int hid_gos_pl_test_event(struct command_report *cmd_rep)
+{
+	int ret = 0;
+
+	switch (cmd_rep->sub_cmd) {
+	case TEST_TP_MFR:
+		drvdata.tp_manufacturer = cmd_rep->data[0];
+		ret = 0;
+		break;
+	case TEST_IMU_MFR:
+		drvdata.imu_manufacturer = cmd_rep->data[0];
+		ret = 0;
+		break;
+	case TEST_TP_VER:
+		drvdata.tp_version = cmd_rep->data[0];
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int hid_gos_light_event(struct command_report *cmd_rep)
+{
+	struct led_classdev_mc *mc_cdev;
+	int ret = 0;
+
+	switch (cmd_rep->sub_cmd) {
+	case LIGHT_MODE_SEL:
+		drvdata.rgb_mode = cmd_rep->data[0];
+		ret = 0;
+		break;
+	case LIGHT_PROFILE_SEL:
+		drvdata.rgb_profile = cmd_rep->data[0];
+		ret = 0;
+		break;
+	case USR_LIGHT_PROFILE_1:
+	case USR_LIGHT_PROFILE_2:
+	case USR_LIGHT_PROFILE_3:
+		mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+		drvdata.rgb_effect = cmd_rep->data[0];
+		mc_cdev->subled_info[0].intensity = cmd_rep->data[1];
+		mc_cdev->subled_info[1].intensity = cmd_rep->data[2];
+		mc_cdev->subled_info[2].intensity = cmd_rep->data[3];
+		drvdata.led_cdev->brightness = cmd_rep->data[4];
+		drvdata.rgb_speed = cmd_rep->data[5];
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int hid_gos_set_event_return(struct command_report *cmd_rep)
+{
+	if (cmd_rep->data[0] != 0)
+		return -EIO;
+
+	return 0;
+}
+
+static int get_endpoint_address(struct hid_device *hdev)
+{
+	struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+	struct usb_host_endpoint *ep;
+
+	if (intf) {
+		ep = intf->cur_altsetting->endpoint;
+		if (ep)
+			return ep->desc.bEndpointAddress;
+	}
+
+	return -ENODEV;
+}
+
+static int hid_gos_raw_event(struct hid_device *hdev, struct hid_report *report,
+			     u8 *data, int size)
+{
+	struct command_report *cmd_rep;
+	int ep, ret;
+
+	ep = get_endpoint_address(hdev);
+	if (ep != GO_S_CFG_INTF_IN)
+		return 0;
+
+	if (size != GO_S_PACKET_SIZE)
+		return -EINVAL;
+
+	cmd_rep = (struct command_report *)data;
+
+	switch (cmd_rep->cmd) {
+	case GET_VERSION:
+		ret = hid_gos_version_event(data);
+		break;
+	case GET_MCU_ID:
+		ret = hid_gos_mcu_id_event(cmd_rep);
+		break;
+	case GET_GAMEPAD_CFG:
+		ret = hid_gos_gamepad_cfg_event(cmd_rep);
+		break;
+	case GET_TP_PARAM:
+		ret = hid_gos_touchpad_event(cmd_rep);
+		break;
+	case GET_PL_TEST:
+		ret = hid_gos_pl_test_event(cmd_rep);
+		break;
+	case GET_RGB_CFG:
+		ret = hid_gos_light_event(cmd_rep);
+		break;
+	case SET_GAMEPAD_CFG:
+	case SET_RGB_CFG:
+	case SET_TP_PARAM:
+		ret = hid_gos_set_event_return(cmd_rep);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	dev_dbg(&hdev->dev, "Rx data as raw input report: [%*ph]\n",
+		GO_S_PACKET_SIZE, data);
+
+	complete(&drvdata.send_cmd_complete);
+	return ret;
+}
+
+static int mcu_property_out(struct hid_device *hdev, u8 command, u8 index,
+			    u8 *data, size_t len)
+{
+	unsigned char *dmabuf __free(kfree) = NULL;
+	u8 header[] = { command, index };
+	size_t header_size = ARRAY_SIZE(header);
+	int timeout, ret;
+
+	if (header_size + len > GO_S_PACKET_SIZE)
+		return -EINVAL;
+
+	guard(mutex)(&drvdata.cfg_mutex);
+	/* We can't use a devm_alloc reusable buffer without side effects during suspend */
+	dmabuf = kzalloc(GO_S_PACKET_SIZE, GFP_KERNEL);
+	if (!dmabuf)
+		return -ENOMEM;
+
+	memcpy(dmabuf, header, header_size);
+	memcpy(dmabuf + header_size, data, len);
+
+	dev_dbg(&hdev->dev, "Send data as raw output report: [%*ph]\n",
+		GO_S_PACKET_SIZE, dmabuf);
+
+	ret = hid_hw_output_report(hdev, dmabuf, GO_S_PACKET_SIZE);
+	if (ret < 0)
+		return ret;
+
+	ret = ret == GO_S_PACKET_SIZE ? 0 : -EINVAL;
+	if (ret)
+		return ret;
+
+	/* PL_TEST commands can take longer because they go out to another device */
+	timeout = (command == GET_PL_TEST) ? 200 : 5;
+	ret = wait_for_completion_interruptible_timeout(&drvdata.send_cmd_complete,
+							msecs_to_jiffies(timeout));
+
+	if (ret == 0) /* timeout occurred */
+		ret = -EBUSY;
+
+	reinit_completion(&drvdata.send_cmd_complete);
+	return 0;
+}
+
+static ssize_t gamepad_property_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t count,
+				      enum feature_status_index index)
+{
+	size_t size = 1;
+	u8 val = 0;
+	int ret;
+
+	switch (index) {
+	case FEATURE_GAMEPAD_MODE:
+		ret = sysfs_match_string(gamepad_mode_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		ret = kstrtou8(buf, 10, &val);
+		if (ret)
+			return ret;
+		break;
+	case FEATURE_IMU_ENABLE:
+		ret = sysfs_match_string(feature_enabled_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_IMU_BYPASS:
+		ret = sysfs_match_string(feature_enabled_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_RGB_ENABLE:
+		ret = sysfs_match_string(feature_enabled_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_TOUCHPAD_ENABLE:
+		ret = sysfs_match_string(feature_enabled_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_OS_MODE:
+		ret = sysfs_match_string(os_type_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_POLL_RATE:
+		ret = sysfs_match_string(poll_rate_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_DPAD_MODE:
+		ret = sysfs_match_string(dpad_mode_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case FEATURE_MOUSE_WHEEL_STEP:
+		ret = kstrtou8(buf, 10, &val);
+		if (ret)
+			return ret;
+		if (val < 1 || val > 127)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!val)
+		size = 0;
+
+	ret = mcu_property_out(drvdata.hdev, SET_GAMEPAD_CFG, index, &val,
+			       size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t gamepad_property_show(struct device *dev,
+				     struct device_attribute *attr, char *buf,
+				     enum feature_status_index index)
+{
+	ssize_t count = 0;
+	u8 i;
+
+	count = mcu_property_out(drvdata.hdev, GET_GAMEPAD_CFG, index, NULL, 0);
+	if (count < 0)
+		return count;
+
+	switch (index) {
+	case FEATURE_GAMEPAD_MODE:
+		i = drvdata.gp_mode;
+		if (i >= ARRAY_SIZE(gamepad_mode_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", gamepad_mode_text[i]);
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		count = sysfs_emit(buf, "%u\n", drvdata.gp_auto_sleep_time);
+		break;
+	case FEATURE_IMU_ENABLE:
+		i = drvdata.imu_sensor_en;
+		if (i >= ARRAY_SIZE(feature_enabled_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", feature_enabled_text[i]);
+		break;
+	case FEATURE_IMU_BYPASS:
+		i = drvdata.imu_bypass_en;
+		if (i >= ARRAY_SIZE(feature_enabled_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", feature_enabled_text[i]);
+		break;
+	case FEATURE_RGB_ENABLE:
+		i = drvdata.rgb_en;
+		if (i >= ARRAY_SIZE(feature_enabled_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", feature_enabled_text[i]);
+		break;
+	case FEATURE_TOUCHPAD_ENABLE:
+		i = drvdata.tp_en;
+		if (i >= ARRAY_SIZE(feature_enabled_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", feature_enabled_text[i]);
+		break;
+	case FEATURE_OS_MODE:
+		i = drvdata.os_mode;
+		if (i >= ARRAY_SIZE(os_type_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", os_type_text[i]);
+		break;
+	case FEATURE_POLL_RATE:
+		i = drvdata.gp_poll_rate;
+		if (i >= ARRAY_SIZE(poll_rate_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", poll_rate_text[i]);
+		break;
+	case FEATURE_DPAD_MODE:
+		i = drvdata.gp_dpad_mode;
+		if (i >= ARRAY_SIZE(dpad_mode_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", dpad_mode_text[i]);
+		break;
+	case FEATURE_MOUSE_WHEEL_STEP:
+		i = drvdata.mouse_step;
+		if (i < 1 || i > 127)
+			return -EINVAL;
+		count = sysfs_emit(buf, "%u\n", i);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static ssize_t gamepad_property_options(struct device *dev,
+					struct device_attribute *attr,
+					char *buf,
+					enum feature_status_index index)
+{
+	size_t count = 0;
+	unsigned int i;
+
+	switch (index) {
+	case FEATURE_GAMEPAD_MODE:
+		for (i = 0; i < ARRAY_SIZE(gamepad_mode_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       gamepad_mode_text[i]);
+		}
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		return sysfs_emit(buf, "0-255\n");
+	case FEATURE_IMU_ENABLE:
+		for (i = 0; i < ARRAY_SIZE(feature_enabled_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       feature_enabled_text[i]);
+		}
+		break;
+	case FEATURE_IMU_BYPASS:
+	case FEATURE_RGB_ENABLE:
+	case FEATURE_TOUCHPAD_ENABLE:
+		for (i = 0; i < ARRAY_SIZE(feature_enabled_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       feature_enabled_text[i]);
+		}
+		break;
+	case FEATURE_OS_MODE:
+		for (i = 0; i < ARRAY_SIZE(os_type_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       os_type_text[i]);
+		}
+		break;
+	case FEATURE_POLL_RATE:
+		for (i = 0; i < ARRAY_SIZE(poll_rate_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       poll_rate_text[i]);
+		}
+		break;
+	case FEATURE_DPAD_MODE:
+		for (i = 0; i < ARRAY_SIZE(dpad_mode_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       dpad_mode_text[i]);
+		}
+		break;
+	case FEATURE_MOUSE_WHEEL_STEP:
+		return sysfs_emit(buf, "1-127\n");
+	default:
+		return count;
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t touchpad_property_store(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t count,
+				       enum touchpad_config_index index)
+{
+	size_t size = 1;
+	u8 val = 0;
+	int ret;
+
+	switch (index) {
+	case CFG_WINDOWS_MODE:
+		ret = sysfs_match_string(touchpad_mode_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	case CFG_LINUX_MODE:
+		ret = sysfs_match_string(touchpad_mode_text, buf);
+		if (ret < 0)
+			return ret;
+		val = ret;
+		break;
+	default:
+		return -EINVAL;
+	}
+	if (!val)
+		size = 0;
+
+	ret = mcu_property_out(drvdata.hdev, SET_TP_PARAM, index, &val, size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t touchpad_property_show(struct device *dev,
+				      struct device_attribute *attr, char *buf,
+				      enum touchpad_config_index index)
+{
+	int ret = 0;
+	u8 i;
+
+	ret = mcu_property_out(drvdata.hdev, GET_TP_PARAM, index, NULL, 0);
+	if (ret < 0)
+		return ret;
+
+	switch (index) {
+	case CFG_WINDOWS_MODE:
+		i = drvdata.tp_windows_mode;
+		break;
+	case CFG_LINUX_MODE:
+		i = drvdata.tp_linux_mode;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (i >= ARRAY_SIZE(touchpad_mode_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", touchpad_mode_text[i]);
+}
+
+static ssize_t touchpad_property_options(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf,
+					 enum touchpad_config_index index)
+{
+	size_t count = 0;
+	unsigned int i;
+
+	switch (index) {
+	case CFG_WINDOWS_MODE:
+	case CFG_LINUX_MODE:
+		for (i = 0; i < ARRAY_SIZE(touchpad_mode_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       touchpad_mode_text[i]);
+		}
+		break;
+	default:
+		return count;
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t test_property_show(struct device *dev,
+				  struct device_attribute *attr, char *buf,
+				  enum test_command_index index)
+{
+	size_t count = 0;
+	u8 i;
+
+	switch (index) {
+	case TEST_TP_MFR:
+		i = drvdata.tp_manufacturer;
+		if (i >= ARRAY_SIZE(touchpad_manufacturer_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", touchpad_manufacturer_text[i]);
+		break;
+	case TEST_IMU_MFR:
+		i = drvdata.imu_manufacturer;
+		if (i >= ARRAY_SIZE(imu_manufacturer_text))
+			return -EINVAL;
+		count = sysfs_emit(buf, "%s\n", imu_manufacturer_text[i]);
+		break;
+	case TEST_TP_VER:
+		count = sysfs_emit(buf, "%u\n", drvdata.tp_version);
+		break;
+	default:
+		count = -EINVAL;
+		break;
+	}
+
+	return count;
+}
+
+static ssize_t mcu_id_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	return sysfs_emit(buf, "%*phN\n", 12, &drvdata.mcu_id);
+}
+
+static int rgb_cfg_call(struct hid_device *hdev, enum mcu_command_index cmd,
+			enum rgb_config_index index, u8 *val, size_t size)
+{
+	if (cmd != SET_RGB_CFG && cmd != GET_RGB_CFG)
+		return -EINVAL;
+
+	if (index < LIGHT_MODE_SEL || index > USR_LIGHT_PROFILE_3)
+		return -EINVAL;
+
+	return mcu_property_out(hdev, cmd, index, val, size);
+}
+
+static int rgb_attr_show(void)
+{
+	enum rgb_config_index index;
+
+	index = drvdata.rgb_profile + 2;
+
+	return rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, index, NULL, 0);
+};
+
+static ssize_t rgb_effect_store(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	u8 effect;
+	int ret;
+
+	ret = sysfs_match_string(rgb_effect_text, buf);
+	if (ret < 0)
+		return ret;
+
+	effect = ret;
+	index = drvdata.rgb_profile + 2;
+	u8 rgb_profile[6] = { effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      drvdata.led_cdev->brightness,
+			      drvdata.rgb_speed };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_effect = effect;
+	return count;
+};
+
+static ssize_t rgb_effect_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	int ret;
+
+	ret = rgb_attr_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_effect >= ARRAY_SIZE(rgb_effect_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", rgb_effect_text[drvdata.rgb_effect]);
+}
+
+static ssize_t rgb_effect_index_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(rgb_effect_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", rgb_effect_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t rgb_speed_store(struct device *dev,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	int val = 0;
+	int ret;
+
+	ret = kstrtoint(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val < 0 || val > 100)
+		return -EINVAL;
+
+	index = drvdata.rgb_profile + 2;
+	u8 rgb_profile[6] = { drvdata.rgb_effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      drvdata.led_cdev->brightness,
+			      val };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_speed = val;
+
+	return count;
+};
+
+static ssize_t rgb_speed_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	int ret;
+
+	ret = rgb_attr_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_speed > 100)
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%hhu\n", drvdata.rgb_speed);
+}
+
+static ssize_t rgb_speed_range_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "0-100\n");
+}
+
+static ssize_t rgb_mode_store(struct device *dev, struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = sysfs_match_string(rgb_mode_text, buf);
+	if (ret <= 0)
+		return ret;
+
+	val = ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, LIGHT_MODE_SEL, &val,
+			   1);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_mode = val;
+
+	return count;
+};
+
+static ssize_t rgb_mode_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	int ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, LIGHT_MODE_SEL, NULL, 0);
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_mode >= ARRAY_SIZE(rgb_mode_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", rgb_mode_text[drvdata.rgb_mode]);
+};
+
+static ssize_t rgb_mode_index_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(rgb_mode_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", rgb_mode_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t rgb_profile_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	size_t size = 1;
+	int ret;
+	u8 val;
+
+	ret = kstrtou8(buf, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val < 1 || val > 3)
+		return -EINVAL;
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, LIGHT_PROFILE_SEL, &val, size);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_profile = val;
+
+	return count;
+};
+
+static ssize_t rgb_profile_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	int ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, LIGHT_PROFILE_SEL, NULL, 0);
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_profile < 1 || drvdata.rgb_profile > 3)
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%hhu\n", drvdata.rgb_profile);
+};
+
+static ssize_t rgb_profile_range_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "1-3\n");
+}
+
+static void hid_gos_brightness_set(struct led_classdev *led_cdev,
+				   enum led_brightness brightness)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	int ret;
+
+	if (brightness > led_cdev->max_brightness) {
+		dev_err(led_cdev->dev, "Invalid argument\n");
+		return;
+	}
+
+	index = drvdata.rgb_profile + 2;
+	u8 rgb_profile[6] = { drvdata.rgb_effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      brightness,
+			      drvdata.rgb_speed };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	switch (ret) {
+	case 0:
+		led_cdev->brightness = brightness;
+		break;
+	case -ENODEV: /* during switch to IAP -ENODEV is expected */
+	case -ENOSYS: /* during rmmod -ENOSYS is expected */
+		dev_dbg(led_cdev->dev, "Failed to write RGB profile: %i\n",
+			ret);
+		break;
+	default:
+		dev_err(led_cdev->dev, "Failed to write RGB profile: %i\n",
+			ret);
+	}
+}
+
+#define LEGOS_DEVICE_ATTR_RW(_name, _attrname, _rtype, _group)                 \
+	static ssize_t _name##_store(struct device *dev,                       \
+				     struct device_attribute *attr,            \
+				     const char *buf, size_t count)            \
+	{                                                                      \
+		return _group##_property_store(dev, attr, buf, count,          \
+					       _name.index);                   \
+	}                                                                      \
+	static ssize_t _name##_show(struct device *dev,                        \
+				    struct device_attribute *attr, char *buf)  \
+	{                                                                      \
+		return _group##_property_show(dev, attr, buf, _name.index);    \
+	}                                                                      \
+	static ssize_t _name##_##_rtype##_show(                                \
+		struct device *dev, struct device_attribute *attr, char *buf)  \
+	{                                                                      \
+		return _group##_property_options(dev, attr, buf, _name.index); \
+	}                                                                      \
+	static DEVICE_ATTR_RW_NAMED(_name, _attrname)
+
+#define LEGOS_DEVICE_ATTR_RO(_name, _attrname, _group)                        \
+	static ssize_t _name##_show(struct device *dev,                       \
+				    struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return _group##_property_show(dev, attr, buf, _name.index);   \
+	}                                                                     \
+	static DEVICE_ATTR_RO_NAMED(_name, _attrname)
+
+/* Gamepad */
+static struct gos_cfg_attr auto_sleep_time = { FEATURE_AUTO_SLEEP_TIME };
+LEGOS_DEVICE_ATTR_RW(auto_sleep_time, "auto_sleep_time", range, gamepad);
+static DEVICE_ATTR_RO(auto_sleep_time_range);
+
+static struct gos_cfg_attr dpad_mode = { FEATURE_DPAD_MODE };
+LEGOS_DEVICE_ATTR_RW(dpad_mode, "dpad_mode", index, gamepad);
+static DEVICE_ATTR_RO(dpad_mode_index);
+
+static struct gos_cfg_attr gamepad_mode = { FEATURE_GAMEPAD_MODE };
+LEGOS_DEVICE_ATTR_RW(gamepad_mode, "mode", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(gamepad_mode_index, "mode_index");
+
+static struct gos_cfg_attr gamepad_poll_rate = { FEATURE_POLL_RATE };
+LEGOS_DEVICE_ATTR_RW(gamepad_poll_rate, "poll_rate", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(gamepad_poll_rate_index, "poll_rate_index");
+
+static struct attribute *legos_gamepad_attrs[] = {
+	&dev_attr_auto_sleep_time.attr,
+	&dev_attr_auto_sleep_time_range.attr,
+	&dev_attr_dpad_mode.attr,
+	&dev_attr_dpad_mode_index.attr,
+	&dev_attr_gamepad_mode.attr,
+	&dev_attr_gamepad_mode_index.attr,
+	&dev_attr_gamepad_poll_rate.attr,
+	&dev_attr_gamepad_poll_rate_index.attr,
+	NULL,
+};
+
+static const struct attribute_group gamepad_attr_group = {
+	.name = "gamepad",
+	.attrs = legos_gamepad_attrs,
+};
+
+/* IMU */
+static struct gos_cfg_attr imu_bypass_enabled = { FEATURE_IMU_BYPASS };
+LEGOS_DEVICE_ATTR_RW(imu_bypass_enabled, "bypass_enabled", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(imu_bypass_enabled_index, "bypass_enabled_index");
+
+static struct gos_cfg_attr imu_manufacturer = { TEST_IMU_MFR };
+LEGOS_DEVICE_ATTR_RO(imu_manufacturer, "manufacturer", test);
+
+static struct gos_cfg_attr imu_sensor_enabled = { FEATURE_IMU_ENABLE };
+LEGOS_DEVICE_ATTR_RW(imu_sensor_enabled, "sensor_enabled", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(imu_sensor_enabled_index, "sensor_enabled_index");
+
+static struct attribute *legos_imu_attrs[] = {
+	&dev_attr_imu_bypass_enabled.attr,
+	&dev_attr_imu_bypass_enabled_index.attr,
+	&dev_attr_imu_manufacturer.attr,
+	&dev_attr_imu_sensor_enabled.attr,
+	&dev_attr_imu_sensor_enabled_index.attr,
+	NULL,
+};
+
+static const struct attribute_group imu_attr_group = {
+	.name = "imu",
+	.attrs = legos_imu_attrs,
+};
+
+/* MCU */
+static DEVICE_ATTR_RO(mcu_id);
+
+static struct gos_cfg_attr os_mode = { FEATURE_OS_MODE };
+LEGOS_DEVICE_ATTR_RW(os_mode, "os_mode", index, gamepad);
+static DEVICE_ATTR_RO(os_mode_index);
+
+static struct attribute *legos_mcu_attrs[] = {
+	&dev_attr_mcu_id.attr,
+	&dev_attr_os_mode.attr,
+	&dev_attr_os_mode_index.attr,
+	NULL,
+};
+
+static const struct attribute_group mcu_attr_group = {
+	.attrs = legos_mcu_attrs,
+};
+
+/* Mouse */
+static struct gos_cfg_attr mouse_wheel_step = { FEATURE_MOUSE_WHEEL_STEP };
+LEGOS_DEVICE_ATTR_RW(mouse_wheel_step, "step", range, gamepad);
+static DEVICE_ATTR_RO_NAMED(mouse_wheel_step_range, "step_range");
+
+static struct attribute *legos_mouse_attrs[] = {
+	&dev_attr_mouse_wheel_step.attr,
+	&dev_attr_mouse_wheel_step_range.attr,
+	NULL,
+};
+
+static const struct attribute_group mouse_attr_group = {
+	.name = "mouse",
+	.attrs = legos_mouse_attrs,
+};
+
+/* Touchpad */
+static struct gos_cfg_attr touchpad_enabled = { FEATURE_TOUCHPAD_ENABLE };
+LEGOS_DEVICE_ATTR_RW(touchpad_enabled, "enabled", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(touchpad_enabled_index, "enabled_index");
+
+static struct gos_cfg_attr touchpad_linux_mode = { CFG_LINUX_MODE };
+LEGOS_DEVICE_ATTR_RW(touchpad_linux_mode, "linux_mode", index, touchpad);
+static DEVICE_ATTR_RO_NAMED(touchpad_linux_mode_index, "linux_mode_index");
+
+static struct gos_cfg_attr touchpad_manufacturer = { TEST_TP_MFR };
+LEGOS_DEVICE_ATTR_RO(touchpad_manufacturer, "manufacturer", test);
+
+static struct gos_cfg_attr touchpad_version = { TEST_TP_VER };
+LEGOS_DEVICE_ATTR_RO(touchpad_version, "version", test);
+
+static struct gos_cfg_attr touchpad_windows_mode = { CFG_WINDOWS_MODE };
+LEGOS_DEVICE_ATTR_RW(touchpad_windows_mode, "windows_mode", index, touchpad);
+static DEVICE_ATTR_RO_NAMED(touchpad_windows_mode_index, "windows_mode_index");
+
+static struct attribute *legos_touchpad_attrs[] = {
+	&dev_attr_touchpad_enabled.attr,
+	&dev_attr_touchpad_enabled_index.attr,
+	&dev_attr_touchpad_linux_mode.attr,
+	&dev_attr_touchpad_linux_mode_index.attr,
+	&dev_attr_touchpad_manufacturer.attr,
+	&dev_attr_touchpad_version.attr,
+	&dev_attr_touchpad_windows_mode.attr,
+	&dev_attr_touchpad_windows_mode_index.attr,
+	NULL,
+};
+
+static const struct attribute_group touchpad_attr_group = {
+	.name = "touchpad",
+	.attrs = legos_touchpad_attrs,
+};
+
+static const struct attribute_group *top_level_attr_groups[] = {
+	&gamepad_attr_group,
+	&imu_attr_group,
+	&mcu_attr_group,
+	&mouse_attr_group,
+	&touchpad_attr_group,
+	NULL,
+};
+
+/* RGB */
+static struct gos_cfg_attr rgb_enabled = { FEATURE_RGB_ENABLE };
+LEGOS_DEVICE_ATTR_RW(rgb_enabled, "enabled", index, gamepad);
+static DEVICE_ATTR_RO_NAMED(rgb_enabled_index, "enabled_index");
+
+static DEVICE_ATTR_RW_NAMED(rgb_effect, "effect");
+static DEVICE_ATTR_RO_NAMED(rgb_effect_index, "effect_index");
+static DEVICE_ATTR_RW_NAMED(rgb_mode, "mode");
+static DEVICE_ATTR_RO_NAMED(rgb_mode_index, "mode_index");
+static DEVICE_ATTR_RW_NAMED(rgb_profile, "profile");
+static DEVICE_ATTR_RO_NAMED(rgb_profile_range, "profile_range");
+static DEVICE_ATTR_RW_NAMED(rgb_speed, "speed");
+static DEVICE_ATTR_RO_NAMED(rgb_speed_range, "speed_range");
+
+static struct attribute *gos_rgb_attrs[] = {
+	&dev_attr_rgb_enabled.attr,
+	&dev_attr_rgb_enabled_index.attr,
+	&dev_attr_rgb_effect.attr,
+	&dev_attr_rgb_effect_index.attr,
+	&dev_attr_rgb_mode.attr,
+	&dev_attr_rgb_mode_index.attr,
+	&dev_attr_rgb_profile.attr,
+	&dev_attr_rgb_profile_range.attr,
+	&dev_attr_rgb_speed.attr,
+	&dev_attr_rgb_speed_range.attr,
+	NULL,
+};
+
+static struct attribute_group rgb_attr_group = {
+	.attrs = gos_rgb_attrs,
+};
+
+static struct mc_subled gos_rgb_subled_info[] = {
+	{
+		.color_index = LED_COLOR_ID_RED,
+		.brightness = 0x50,
+		.intensity = 0x24,
+		.channel = 0x1,
+	},
+	{
+		.color_index = LED_COLOR_ID_GREEN,
+		.brightness = 0x50,
+		.intensity = 0x22,
+		.channel = 0x2,
+	},
+	{
+		.color_index = LED_COLOR_ID_BLUE,
+		.brightness = 0x50,
+		.intensity = 0x99,
+		.channel = 0x3,
+	},
+};
+
+static struct led_classdev_mc gos_cdev_rgb = {
+	.led_cdev = {
+		.name = "go_s:rgb:joystick_rings",
+		.brightness = 0x50,
+		.max_brightness = 0x64,
+		.brightness_set = hid_gos_brightness_set,
+	},
+	.num_colors = ARRAY_SIZE(gos_rgb_subled_info),
+	.subled_info = gos_rgb_subled_info,
+};
+
+static void cfg_setup(struct work_struct *work)
+{
+	int ret;
+
+	/* MCU */
+	ret = mcu_property_out(drvdata.hdev, GET_MCU_ID, FEATURE_NONE, NULL, 0);
+	if (ret) {
+		dev_err(&drvdata.hdev->dev, "Failed to retrieve MCU ID: %i\n",
+			ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, GET_VERSION, FEATURE_NONE, NULL, 0);
+	if (ret) {
+		dev_err(&drvdata.hdev->dev, "Failed to retrieve MCU Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, GET_PL_TEST, TEST_TP_MFR, NULL, 0);
+	if (ret) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve Touchpad Manufacturer: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, GET_PL_TEST, TEST_TP_VER, NULL, 0);
+	if (ret) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve Touchpad Firmware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, GET_PL_TEST, TEST_IMU_MFR, NULL, 0);
+	if (ret) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve IMU Manufacturer: %i\n", ret);
+		return;
+	}
+}
+
+static int hid_gos_cfg_probe(struct hid_device *hdev,
+			     const struct hid_device_id *_id)
+{
+	int ret;
+
+	hid_set_drvdata(hdev, &drvdata);
+	drvdata.hdev = hdev;
+	mutex_init(&drvdata.cfg_mutex);
+
+	ret = sysfs_create_groups(&hdev->dev.kobj, top_level_attr_groups);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret,
+			      "Failed to create gamepad configuration attributes\n");
+		return ret;
+	}
+
+	ret = devm_led_classdev_multicolor_register(&hdev->dev, &gos_cdev_rgb);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret, "Failed to create RGB device\n");
+		return ret;
+	}
+
+	ret = devm_device_add_group(gos_cdev_rgb.led_cdev.dev, &rgb_attr_group);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret,
+			      "Failed to create RGB configuration attributes\n");
+		return ret;
+	}
+
+	drvdata.led_cdev = &gos_cdev_rgb.led_cdev;
+
+	init_completion(&drvdata.send_cmd_complete);
+
+	/* Executing calls prior to returning from probe will lock the MCU. Schedule
+	 * initial data call after probe has completed and MCU can accept calls.
+	 */
+	INIT_DELAYED_WORK(&drvdata.gos_cfg_setup, &cfg_setup);
+	ret = schedule_delayed_work(&drvdata.gos_cfg_setup, msecs_to_jiffies(2));
+	if (!ret) {
+		dev_err(&hdev->dev, "Failed to schedule startup delayed work\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static void hid_gos_cfg_remove(struct hid_device *hdev)
+{
+	guard(mutex)(&drvdata.cfg_mutex);
+	cancel_delayed_work_sync(&drvdata.gos_cfg_setup);
+	sysfs_remove_groups(&hdev->dev.kobj, top_level_attr_groups);
+	hid_hw_close(hdev);
+	hid_hw_stop(hdev);
+	hid_set_drvdata(hdev, NULL);
+}
+
+static int hid_gos_probe(struct hid_device *hdev,
+			 const struct hid_device_id *id)
+{
+	int ret, ep;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "Parse failed\n");
+		return ret;
+	}
+
+	ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW);
+	if (ret) {
+		hid_err(hdev, "Failed to start HID device\n");
+		return ret;
+	}
+
+	ret = hid_hw_open(hdev);
+	if (ret) {
+		hid_err(hdev, "Failed to open HID device\n");
+		hid_hw_stop(hdev);
+		return ret;
+	}
+
+	ep = get_endpoint_address(hdev);
+	if (ep != GO_S_CFG_INTF_IN) {
+		dev_dbg(&hdev->dev, "Started interface %x as generic HID device.\n", ep);
+		return 0;
+	}
+
+	ret = hid_gos_cfg_probe(hdev, id);
+	if (ret)
+		dev_err_probe(&hdev->dev, ret, "Failed to start configuration interface");
+
+	dev_dbg(&hdev->dev, "Started interface %x as Go S configuration interface\n", ep);
+	return ret;
+}
+
+static void hid_gos_remove(struct hid_device *hdev)
+{
+	int ep = get_endpoint_address(hdev);
+
+	switch (ep) {
+	case GO_S_CFG_INTF_IN:
+		hid_gos_cfg_remove(hdev);
+		break;
+	default:
+		hid_hw_close(hdev);
+		hid_hw_stop(hdev);
+
+		break;
+	}
+}
+
+static const struct hid_device_id hid_gos_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_QHE,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO_S_XINPUT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_QHE,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO_S_DINPUT) },
+	{}
+};
+
+MODULE_DEVICE_TABLE(hid, hid_gos_devices);
+static struct hid_driver hid_lenovo_go_s = {
+	.name = "hid-lenovo-go-s",
+	.id_table = hid_gos_devices,
+	.probe = hid_gos_probe,
+	.remove = hid_gos_remove,
+	.raw_event = hid_gos_raw_event,
+};
+module_hid_driver(hid_lenovo_go_s);
+
+MODULE_AUTHOR("Derek J. Clark");
+MODULE_DESCRIPTION("HID Driver for Lenovo Legion Go S Series gamepad.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-lenovo-go.c b/drivers/hid/hid-lenovo-go.c
new file mode 100644
index 0000000000000..d4d26c7833563
--- /dev/null
+++ b/drivers/hid/hid-lenovo-go.c
@@ -0,0 +1,2500 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *  HID driver for Lenovo Legion Go series gamepads.
+ *
+ *  Copyright (c) 2026 Derek J. Clark <derekjohn.clark@gmail.com>
+ *  Copyright (c) 2026 Valve Corporation
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/array_size.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/device.h>
+#include <linux/device/devres.h>
+#include <linux/hid.h>
+#include <linux/jiffies.h>
+#include <linux/kstrtox.h>
+#include <linux/led-class-multicolor.h>
+#include <linux/mutex.h>
+#include <linux/printk.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+#include <linux/usb.h>
+#include <linux/workqueue.h>
+#include <linux/workqueue_types.h>
+
+#include "hid-ids.h"
+
+#define GO_GP_INTF_IN		0x83
+#define GO_OUTPUT_REPORT_ID	0x05
+#define GO_GP_RESET_SUCCESS	0x01
+#define GO_PACKET_SIZE		64
+
+static struct hid_go_cfg {
+	struct delayed_work go_cfg_setup;
+	struct completion send_cmd_complete;
+	struct led_classdev *led_cdev;
+	struct hid_device *hdev;
+	struct mutex cfg_mutex; /*ensure single synchronous output report*/
+	u8 fps_mode;
+	u8 gp_left_auto_sleep_time;
+	u8 gp_left_gyro_cal_status;
+	u8 gp_left_joy_cal_status;
+	u8 gp_left_notify_en;
+	u8 gp_left_rumble_mode;
+	u8 gp_left_trigg_cal_status;
+	u32 gp_left_version_firmware;
+	u8 gp_left_version_gen;
+	u32 gp_left_version_hardware;
+	u32 gp_left_version_product;
+	u32 gp_left_version_protocol;
+	u8 gp_mode;
+	u8 gp_right_auto_sleep_time;
+	u8 gp_right_gyro_cal_status;
+	u8 gp_right_joy_cal_status;
+	u8 gp_right_notify_en;
+	u8 gp_right_rumble_mode;
+	u8 gp_right_trigg_cal_status;
+	u32 gp_right_version_firmware;
+	u8 gp_right_version_gen;
+	u32 gp_right_version_hardware;
+	u32 gp_right_version_product;
+	u32 gp_right_version_protocol;
+	u8 gp_rumble_intensity;
+	u8 imu_left_bypass_en;
+	u8 imu_left_sensor_en;
+	u8 imu_right_bypass_en;
+	u8 imu_right_sensor_en;
+	u32 mcu_version_firmware;
+	u8 mcu_version_gen;
+	u32 mcu_version_hardware;
+	u32 mcu_version_product;
+	u32 mcu_version_protocol;
+	u32 mouse_dpi;
+	u8 os_mode;
+	u8 rgb_effect;
+	u8 rgb_en;
+	u8 rgb_mode;
+	u8 rgb_profile;
+	u8 rgb_speed;
+	u8 tp_en;
+	u8 tp_vibration_en;
+	u8 tp_vibration_intensity;
+	u32 tx_dongle_version_firmware;
+	u8 tx_dongle_version_gen;
+	u32 tx_dongle_version_hardware;
+	u32 tx_dongle_version_product;
+	u32 tx_dongle_version_protocol;
+} drvdata;
+
+struct go_cfg_attr {
+	u8 index;
+};
+
+struct command_report {
+	u8 report_id;
+	u8 id;
+	u8 cmd;
+	u8 sub_cmd;
+	u8 device_type;
+	u8 data[59];
+} __packed;
+
+enum command_id {
+	MCU_CONFIG_DATA = 0x00,
+	OS_MODE_DATA = 0x06,
+	GAMEPAD_DATA = 0x3c,
+};
+
+enum mcu_command_index {
+	GET_VERSION_DATA = 0x02,
+	GET_FEATURE_STATUS,
+	SET_FEATURE_STATUS,
+	GET_MOTOR_CFG,
+	SET_MOTOR_CFG,
+	GET_DPI_CFG,
+	SET_DPI_CFG,
+	SET_TRIGGER_CFG = 0x0a,
+	SET_JOYSTICK_CFG = 0x0c,
+	SET_GYRO_CFG = 0x0e,
+	GET_RGB_CFG,
+	SET_RGB_CFG,
+	GET_DEVICE_STATUS = 0xa0,
+
+};
+
+enum dev_type {
+	UNSPECIFIED,
+	USB_MCU,
+	TX_DONGLE,
+	LEFT_CONTROLLER,
+	RIGHT_CONTROLLER,
+};
+
+enum enabled_status_index {
+	FEATURE_UNKNOWN,
+	FEATURE_ENABLED,
+	FEATURE_DISABLED,
+};
+
+static const char *const enabled_status_text[] = {
+	[FEATURE_UNKNOWN] = "unknown",
+	[FEATURE_ENABLED] = "true",
+	[FEATURE_DISABLED] = "false",
+};
+
+enum version_data_index {
+	PRODUCT_VERSION = 0x02,
+	PROTOCOL_VERSION,
+	FIRMWARE_VERSION,
+	HARDWARE_VERSION,
+	HARDWARE_GENERATION,
+};
+
+enum feature_status_index {
+	FEATURE_RESET_GAMEPAD = 0x02,
+	FEATURE_IMU_BYPASS,
+	FEATURE_IMU_ENABLE = 0x05,
+	FEATURE_TOUCHPAD_ENABLE = 0x07,
+	FEATURE_LIGHT_ENABLE,
+	FEATURE_AUTO_SLEEP_TIME,
+	FEATURE_FPS_SWITCH_STATUS = 0x0b,
+	FEATURE_GAMEPAD_MODE = 0x0e,
+};
+
+#define FEATURE_OS_MODE 0x69
+
+enum fps_switch_status_index {
+	FPS_STATUS_UNKNOWN,
+	GAMEPAD,
+	FPS,
+};
+
+static const char *const fps_switch_text[] = {
+	[FPS_STATUS_UNKNOWN] = "unknown",
+	[GAMEPAD] = "gamepad",
+	[FPS] = "fps",
+};
+
+enum gamepad_mode_index {
+	GAMEPAD_MODE_UNKNOWN,
+	XINPUT,
+	DINPUT,
+};
+
+static const char *const gamepad_mode_text[] = {
+	[GAMEPAD_MODE_UNKNOWN] = "unknown",
+	[XINPUT] = "xinput",
+	[DINPUT] = "dinput",
+};
+
+enum motor_cfg_index {
+	MOTOR_CFG_ALL = 0x01,
+	MOTOR_INTENSITY,
+	VIBRATION_NOTIFY_ENABLE,
+	RUMBLE_MODE,
+	TP_VIBRATION_ENABLE,
+	TP_VIBRATION_INTENSITY,
+};
+
+enum intensity_index {
+	INTENSITY_UNKNOWN,
+	INTENSITY_OFF,
+	INTENSITY_LOW,
+	INTENSITY_MEDIUM,
+	INTENSITY_HIGH,
+};
+
+static const char *const intensity_text[] = {
+	[INTENSITY_UNKNOWN] = "unknown",
+	[INTENSITY_OFF] = "off",
+	[INTENSITY_LOW] = "low",
+	[INTENSITY_MEDIUM] = "medium",
+	[INTENSITY_HIGH] = "high",
+};
+
+enum rumble_mode_index {
+	RUMBLE_MODE_UNKNOWN,
+	RUMBLE_MODE_FPS,
+	RUMBLE_MODE_RACE,
+	RUMBLE_MODE_AVERAGE,
+	RUMBLE_MODE_SPG,
+	RUMBLE_MODE_RPG,
+};
+
+static const char *const rumble_mode_text[] = {
+	[RUMBLE_MODE_UNKNOWN] = "unknown",
+	[RUMBLE_MODE_FPS] = "fps",
+	[RUMBLE_MODE_RACE] = "racing",
+	[RUMBLE_MODE_AVERAGE] = "standard",
+	[RUMBLE_MODE_SPG] = "spg",
+	[RUMBLE_MODE_RPG] = "rpg",
+};
+
+#define FPS_MODE_DPI		0x02
+#define TRIGGER_CALIBRATE	0x04
+#define JOYSTICK_CALIBRATE	0x04
+#define GYRO_CALIBRATE		0x06
+
+enum cal_device_type {
+	CALDEV_GYROSCOPE = 0x01,
+	CALDEV_JOYSTICK,
+	CALDEV_TRIGGER,
+	CALDEV_JOY_TRIGGER,
+};
+
+enum cal_enable {
+	CAL_UNKNOWN,
+	CAL_START,
+	CAL_STOP,
+};
+
+static const char *const cal_enabled_text[] = {
+	[CAL_UNKNOWN] = "unknown",
+	[CAL_START] = "start",
+	[CAL_STOP] = "stop",
+};
+
+enum cal_status_index {
+	CAL_STAT_UNKNOWN,
+	CAL_STAT_SUCCESS,
+	CAL_STAT_FAILURE,
+};
+
+static const char *const cal_status_text[] = {
+	[CAL_STAT_UNKNOWN] = "unknown",
+	[CAL_STAT_SUCCESS] = "success",
+	[CAL_STAT_FAILURE] = "failure",
+};
+
+enum rgb_config_index {
+	LIGHT_CFG_ALL = 0x01,
+	LIGHT_MODE_SEL,
+	LIGHT_PROFILE_SEL,
+	USR_LIGHT_PROFILE_1,
+	USR_LIGHT_PROFILE_2,
+	USR_LIGHT_PROFILE_3,
+};
+
+enum rgb_mode_index {
+	RGB_MODE_UNKNOWN,
+	RGB_MODE_DYNAMIC,
+	RGB_MODE_CUSTOM,
+};
+
+static const char *const rgb_mode_text[] = {
+	[RGB_MODE_UNKNOWN] = "unknown",
+	[RGB_MODE_DYNAMIC] = "dynamic",
+	[RGB_MODE_CUSTOM] = "custom",
+};
+
+enum rgb_effect_index {
+	RGB_EFFECT_MONO,
+	RGB_EFFECT_BREATHE,
+	RGB_EFFECT_CHROMA,
+	RGB_EFFECT_RAINBOW,
+};
+
+static const char *const rgb_effect_text[] = {
+	[RGB_EFFECT_MONO] = "monocolor",
+	[RGB_EFFECT_BREATHE] = "breathe",
+	[RGB_EFFECT_CHROMA] = "chroma",
+	[RGB_EFFECT_RAINBOW] = "rainbow",
+};
+
+enum device_status_index {
+	GET_CAL_STATUS = 0x02,
+	GET_UPGRADE_STATUS,
+	GET_MACRO_REC_STATUS,
+	GET_HOTKEY_TRIGG_STATUS,
+};
+
+enum os_mode_cfg_index {
+	SET_OS_MODE = 0x09,
+	GET_OS_MODE,
+};
+
+enum os_mode_type_index {
+	OS_UNKNOWN,
+	WINDOWS,
+	LINUX,
+};
+
+static const char *const os_mode_text[] = {
+	[OS_UNKNOWN] = "unknown",
+	[WINDOWS] = "windows",
+	[LINUX] = "linux",
+};
+
+static int hid_go_version_event(struct command_report *cmd_rep)
+{
+	switch (cmd_rep->sub_cmd) {
+	case PRODUCT_VERSION:
+		switch (cmd_rep->device_type) {
+		case USB_MCU:
+			drvdata.mcu_version_product =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case TX_DONGLE:
+			drvdata.tx_dongle_version_product =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_version_product =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_version_product =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case PROTOCOL_VERSION:
+		switch (cmd_rep->device_type) {
+		case USB_MCU:
+			drvdata.mcu_version_protocol =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case TX_DONGLE:
+			drvdata.tx_dongle_version_protocol =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_version_protocol =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_version_protocol =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case FIRMWARE_VERSION:
+		switch (cmd_rep->device_type) {
+		case USB_MCU:
+			drvdata.mcu_version_firmware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case TX_DONGLE:
+			drvdata.tx_dongle_version_firmware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_version_firmware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_version_firmware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case HARDWARE_VERSION:
+		switch (cmd_rep->device_type) {
+		case USB_MCU:
+			drvdata.mcu_version_hardware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case TX_DONGLE:
+			drvdata.tx_dongle_version_hardware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_version_hardware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_version_hardware =
+				get_unaligned_be32(cmd_rep->data);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case HARDWARE_GENERATION:
+		switch (cmd_rep->device_type) {
+		case USB_MCU:
+			drvdata.mcu_version_gen = cmd_rep->data[0];
+			return 0;
+		case TX_DONGLE:
+			drvdata.tx_dongle_version_gen = cmd_rep->data[0];
+			return 0;
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_version_gen = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_version_gen = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hid_go_feature_status_event(struct command_report *cmd_rep)
+{
+	switch (cmd_rep->sub_cmd) {
+	case FEATURE_RESET_GAMEPAD:
+		return 0;
+	case FEATURE_IMU_ENABLE:
+		switch (cmd_rep->device_type) {
+		case LEFT_CONTROLLER:
+			drvdata.imu_left_sensor_en = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.imu_right_sensor_en = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case FEATURE_IMU_BYPASS:
+		switch (cmd_rep->device_type) {
+		case LEFT_CONTROLLER:
+			drvdata.imu_left_bypass_en = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.imu_right_bypass_en = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case FEATURE_LIGHT_ENABLE:
+		drvdata.rgb_en = cmd_rep->data[0];
+		return 0;
+	case FEATURE_AUTO_SLEEP_TIME:
+		switch (cmd_rep->device_type) {
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_auto_sleep_time = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_auto_sleep_time = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case FEATURE_TOUCHPAD_ENABLE:
+		drvdata.tp_en = cmd_rep->data[0];
+		return 0;
+	case FEATURE_GAMEPAD_MODE:
+		drvdata.gp_mode = cmd_rep->data[0];
+		return 0;
+	case FEATURE_FPS_SWITCH_STATUS:
+		drvdata.fps_mode = cmd_rep->data[0];
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hid_go_motor_event(struct command_report *cmd_rep)
+{
+	switch (cmd_rep->sub_cmd) {
+	case MOTOR_CFG_ALL:
+		return -EINVAL;
+	case MOTOR_INTENSITY:
+		drvdata.gp_rumble_intensity = cmd_rep->data[0];
+		return 0;
+	case VIBRATION_NOTIFY_ENABLE:
+		switch (cmd_rep->device_type) {
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_notify_en = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_notify_en = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case RUMBLE_MODE:
+		switch (cmd_rep->device_type) {
+		case LEFT_CONTROLLER:
+			drvdata.gp_left_rumble_mode = cmd_rep->data[0];
+			return 0;
+		case RIGHT_CONTROLLER:
+			drvdata.gp_right_rumble_mode = cmd_rep->data[0];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	case TP_VIBRATION_ENABLE:
+		drvdata.tp_vibration_en = cmd_rep->data[0];
+		return 0;
+	case TP_VIBRATION_INTENSITY:
+		drvdata.tp_vibration_intensity = cmd_rep->data[0];
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int hid_go_fps_dpi_event(struct command_report *cmd_rep)
+{
+	if (cmd_rep->sub_cmd != FPS_MODE_DPI)
+		return -EINVAL;
+
+	drvdata.mouse_dpi = get_unaligned_le32(cmd_rep->data);
+
+	return 0;
+}
+
+static int hid_go_light_event(struct command_report *cmd_rep)
+{
+	struct led_classdev_mc *mc_cdev;
+
+	switch (cmd_rep->sub_cmd) {
+	case LIGHT_MODE_SEL:
+		drvdata.rgb_mode = cmd_rep->data[0];
+		return 0;
+	case LIGHT_PROFILE_SEL:
+		drvdata.rgb_profile = cmd_rep->data[0];
+		return 0;
+	case USR_LIGHT_PROFILE_1:
+	case USR_LIGHT_PROFILE_2:
+	case USR_LIGHT_PROFILE_3:
+		mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+		drvdata.rgb_effect = cmd_rep->data[0];
+		mc_cdev->subled_info[0].intensity = cmd_rep->data[1];
+		mc_cdev->subled_info[1].intensity = cmd_rep->data[2];
+		mc_cdev->subled_info[2].intensity = cmd_rep->data[3];
+		drvdata.led_cdev->brightness = cmd_rep->data[4];
+		drvdata.rgb_speed = 100 - cmd_rep->data[5];
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hid_go_device_status_event(struct command_report *cmd_rep)
+{
+	switch (cmd_rep->device_type) {
+	case LEFT_CONTROLLER:
+		switch (cmd_rep->data[0]) {
+		case CALDEV_GYROSCOPE:
+			drvdata.gp_left_gyro_cal_status = cmd_rep->data[1];
+			return 0;
+		case CALDEV_JOYSTICK:
+			drvdata.gp_left_joy_cal_status = cmd_rep->data[1];
+			return 0;
+		case CALDEV_TRIGGER:
+			drvdata.gp_left_trigg_cal_status = cmd_rep->data[1];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case RIGHT_CONTROLLER:
+		switch (cmd_rep->data[0]) {
+		case CALDEV_GYROSCOPE:
+			drvdata.gp_right_gyro_cal_status = cmd_rep->data[1];
+			return 0;
+		case CALDEV_JOYSTICK:
+			drvdata.gp_right_joy_cal_status = cmd_rep->data[1];
+			return 0;
+		case CALDEV_TRIGGER:
+			drvdata.gp_right_trigg_cal_status = cmd_rep->data[1];
+			return 0;
+		default:
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hid_go_os_mode_cfg_event(struct command_report *cmd_rep)
+{
+	switch (cmd_rep->sub_cmd) {
+	case SET_OS_MODE:
+		if (cmd_rep->data[0] != 1)
+			return -EIO;
+		return 0;
+	case GET_OS_MODE:
+		drvdata.os_mode = cmd_rep->data[0];
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hid_go_set_event_return(struct command_report *cmd_rep)
+{
+	if (cmd_rep->data[0] != 0)
+		return -EIO;
+
+	return 0;
+}
+
+static int get_endpoint_address(struct hid_device *hdev)
+{
+	struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+	struct usb_host_endpoint *ep;
+
+	if (!intf)
+		return -ENODEV;
+
+	ep = intf->cur_altsetting->endpoint;
+	if (!ep)
+		return -ENODEV;
+
+	return ep->desc.bEndpointAddress;
+}
+
+static int hid_go_raw_event(struct hid_device *hdev, struct hid_report *report,
+			    u8 *data, int size)
+{
+	struct command_report *cmd_rep;
+	int ep, ret;
+
+	if (size != GO_PACKET_SIZE)
+		goto passthrough;
+
+	ep = get_endpoint_address(hdev);
+	if (ep != GO_GP_INTF_IN)
+		goto passthrough;
+
+	cmd_rep = (struct command_report *)data;
+
+	switch (cmd_rep->id) {
+	case MCU_CONFIG_DATA:
+		switch (cmd_rep->cmd) {
+		case GET_VERSION_DATA:
+			ret = hid_go_version_event(cmd_rep);
+			break;
+		case GET_FEATURE_STATUS:
+			ret = hid_go_feature_status_event(cmd_rep);
+			break;
+		case GET_MOTOR_CFG:
+			ret = hid_go_motor_event(cmd_rep);
+			break;
+		case GET_DPI_CFG:
+			ret = hid_go_fps_dpi_event(cmd_rep);
+			break;
+		case GET_RGB_CFG:
+			ret = hid_go_light_event(cmd_rep);
+			break;
+		case GET_DEVICE_STATUS:
+			ret = hid_go_device_status_event(cmd_rep);
+			break;
+		case SET_FEATURE_STATUS:
+		case SET_MOTOR_CFG:
+		case SET_DPI_CFG:
+		case SET_RGB_CFG:
+		case SET_TRIGGER_CFG:
+		case SET_JOYSTICK_CFG:
+		case SET_GYRO_CFG:
+			ret = hid_go_set_event_return(cmd_rep);
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+		break;
+	case OS_MODE_DATA:
+		ret = hid_go_os_mode_cfg_event(cmd_rep);
+		break;
+	default:
+		goto passthrough;
+	}
+	dev_dbg(&hdev->dev, "Rx data as raw input report: [%*ph]\n",
+		GO_PACKET_SIZE, data);
+
+	complete(&drvdata.send_cmd_complete);
+	return ret;
+
+passthrough:
+	/* Forward other HID reports so they generate events */
+	hid_input_report(hdev, HID_INPUT_REPORT, data, size, 1);
+	return 0;
+}
+
+static int mcu_property_out(struct hid_device *hdev, u8 id, u8 command,
+			    u8 index, enum dev_type device, u8 *data, size_t len)
+{
+	unsigned char *dmabuf __free(kfree) = NULL;
+	u8 header[] = { GO_OUTPUT_REPORT_ID, id, command, index, device };
+	size_t header_size = ARRAY_SIZE(header);
+	int timeout = 50;
+	int ret;
+
+	if (header_size + len > GO_PACKET_SIZE)
+		return -EINVAL;
+
+	guard(mutex)(&drvdata.cfg_mutex);
+	/* We can't use a devm_alloc reusable buffer without side effects during suspend */
+	dmabuf = kzalloc(GO_PACKET_SIZE, GFP_KERNEL);
+	if (!dmabuf)
+		return -ENOMEM;
+
+	memcpy(dmabuf, header, header_size);
+	memcpy(dmabuf + header_size, data, len);
+
+	dev_dbg(&hdev->dev, "Send data as raw output report: [%*ph]\n",
+		GO_PACKET_SIZE, dmabuf);
+
+	ret = hid_hw_output_report(hdev, dmabuf, GO_PACKET_SIZE);
+	if (ret < 0)
+		return ret;
+
+	ret = ret == GO_PACKET_SIZE ? 0 : -EINVAL;
+	if (ret)
+		return ret;
+
+	ret = wait_for_completion_interruptible_timeout(&drvdata.send_cmd_complete,
+							msecs_to_jiffies(timeout));
+
+	if (ret == 0) /* timeout occurred */
+		ret = -EBUSY;
+
+	reinit_completion(&drvdata.send_cmd_complete);
+	return 0;
+}
+
+static ssize_t version_show(struct device *dev, struct device_attribute *attr,
+			    char *buf, enum version_data_index index,
+			    enum dev_type device_type)
+{
+	ssize_t count = 0;
+	int ret;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       index, device_type, NULL, 0);
+	if (ret)
+		return ret;
+
+	switch (index) {
+	case PRODUCT_VERSION:
+		switch (device_type) {
+		case USB_MCU:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.mcu_version_product);
+			break;
+		case TX_DONGLE:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.tx_dongle_version_product);
+			break;
+		case LEFT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_left_version_product);
+			break;
+		case RIGHT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_right_version_product);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case PROTOCOL_VERSION:
+		switch (device_type) {
+		case USB_MCU:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.mcu_version_protocol);
+			break;
+		case TX_DONGLE:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.tx_dongle_version_protocol);
+			break;
+		case LEFT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_left_version_protocol);
+			break;
+		case RIGHT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_right_version_protocol);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case FIRMWARE_VERSION:
+		switch (device_type) {
+		case USB_MCU:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.mcu_version_firmware);
+			break;
+		case TX_DONGLE:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.tx_dongle_version_firmware);
+			break;
+		case LEFT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_left_version_firmware);
+			break;
+		case RIGHT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_right_version_firmware);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case HARDWARE_VERSION:
+		switch (device_type) {
+		case USB_MCU:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.mcu_version_hardware);
+			break;
+		case TX_DONGLE:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.tx_dongle_version_hardware);
+			break;
+		case LEFT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_left_version_hardware);
+			break;
+		case RIGHT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_right_version_hardware);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case HARDWARE_GENERATION:
+		switch (device_type) {
+		case USB_MCU:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.mcu_version_gen);
+			break;
+		case TX_DONGLE:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.tx_dongle_version_gen);
+			break;
+		case LEFT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_left_version_gen);
+			break;
+		case RIGHT_CONTROLLER:
+			count = sysfs_emit(buf, "%x\n",
+					   drvdata.gp_right_version_gen);
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	}
+
+	return count;
+}
+
+static ssize_t feature_status_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count,
+				    enum feature_status_index index,
+				    enum dev_type device_type)
+{
+	size_t size = 1;
+	u8 val = 0;
+	int ret;
+
+	switch (index) {
+	case FEATURE_IMU_ENABLE:
+	case FEATURE_IMU_BYPASS:
+	case FEATURE_LIGHT_ENABLE:
+	case FEATURE_TOUCHPAD_ENABLE:
+		ret = sysfs_match_string(enabled_status_text, buf);
+		val = ret;
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		ret = kstrtou8(buf, 10, &val);
+		break;
+	case FEATURE_RESET_GAMEPAD:
+		ret = kstrtou8(buf, 10, &val);
+		if (val != GO_GP_RESET_SUCCESS)
+			return -EINVAL;
+		break;
+	case FEATURE_FPS_SWITCH_STATUS:
+		ret = sysfs_match_string(fps_switch_text, buf);
+		val = ret;
+		break;
+	case FEATURE_GAMEPAD_MODE:
+		ret = sysfs_match_string(gamepad_mode_text, buf);
+		val = ret;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (ret < 0)
+		return ret;
+
+	if (!val)
+		size = 0;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA,
+			       SET_FEATURE_STATUS, index, device_type, &val,
+			       size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t feature_status_show(struct device *dev,
+				   struct device_attribute *attr, char *buf,
+				   enum feature_status_index index,
+				   enum dev_type device_type)
+{
+	ssize_t count = 0;
+	int ret;
+	u8 i;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA,
+			       GET_FEATURE_STATUS, index, device_type, NULL, 0);
+	if (ret)
+		return ret;
+
+	switch (index) {
+	case FEATURE_IMU_ENABLE:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			i = drvdata.imu_left_sensor_en;
+			break;
+		case RIGHT_CONTROLLER:
+			i = drvdata.imu_right_sensor_en;
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case FEATURE_IMU_BYPASS:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			i = drvdata.imu_left_bypass_en;
+			break;
+		case RIGHT_CONTROLLER:
+			i = drvdata.imu_right_bypass_en;
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case FEATURE_LIGHT_ENABLE:
+		i = drvdata.rgb_en;
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case FEATURE_TOUCHPAD_ENABLE:
+		i = drvdata.tp_en;
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			i = drvdata.gp_left_auto_sleep_time;
+			break;
+		case RIGHT_CONTROLLER:
+			i = drvdata.gp_right_auto_sleep_time;
+			break;
+		default:
+			return -EINVAL;
+		}
+		count = sysfs_emit(buf, "%u\n", i);
+		break;
+	case FEATURE_FPS_SWITCH_STATUS:
+		i = drvdata.fps_mode;
+		if (i >= ARRAY_SIZE(fps_switch_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", fps_switch_text[i]);
+		break;
+	case FEATURE_GAMEPAD_MODE:
+		i = drvdata.gp_mode;
+		if (i >= ARRAY_SIZE(gamepad_mode_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", gamepad_mode_text[i]);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static ssize_t feature_status_options(struct device *dev,
+				      struct device_attribute *attr, char *buf,
+				      enum feature_status_index index)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	switch (index) {
+	case FEATURE_IMU_ENABLE:
+	case FEATURE_IMU_BYPASS:
+	case FEATURE_LIGHT_ENABLE:
+	case FEATURE_TOUCHPAD_ENABLE:
+		for (i = 1; i < ARRAY_SIZE(enabled_status_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       enabled_status_text[i]);
+		}
+		break;
+	case FEATURE_AUTO_SLEEP_TIME:
+		return sysfs_emit(buf, "0-255\n");
+	case FEATURE_FPS_SWITCH_STATUS:
+		for (i = 1; i < ARRAY_SIZE(fps_switch_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       fps_switch_text[i]);
+		}
+		break;
+	case FEATURE_GAMEPAD_MODE:
+		for (i = 1; i < ARRAY_SIZE(gamepad_mode_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       gamepad_mode_text[i]);
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t motor_config_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count,
+				  enum motor_cfg_index index,
+				  enum dev_type device_type)
+{
+	size_t size = 1;
+	u8 val = 0;
+	int ret;
+
+	switch (index) {
+	case MOTOR_CFG_ALL:
+		return -EINVAL;
+	case MOTOR_INTENSITY:
+		ret = sysfs_match_string(intensity_text, buf);
+		val = ret;
+		break;
+	case VIBRATION_NOTIFY_ENABLE:
+		ret = sysfs_match_string(enabled_status_text, buf);
+		val = ret;
+		break;
+	case RUMBLE_MODE:
+		ret = sysfs_match_string(rumble_mode_text, buf);
+		val = ret;
+		break;
+	case TP_VIBRATION_ENABLE:
+		ret = sysfs_match_string(enabled_status_text, buf);
+		val = ret;
+		break;
+	case TP_VIBRATION_INTENSITY:
+		ret = sysfs_match_string(intensity_text, buf);
+		val = ret;
+		break;
+	}
+
+	if (ret < 0)
+		return ret;
+
+	if (!val)
+		size = 0;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, SET_MOTOR_CFG,
+			       index, device_type, &val, size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t motor_config_show(struct device *dev,
+				 struct device_attribute *attr, char *buf,
+				 enum motor_cfg_index index,
+				 enum dev_type device_type)
+{
+	ssize_t count = 0;
+	int ret;
+	u8 i;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_MOTOR_CFG,
+			       index, device_type, NULL, 0);
+	if (ret)
+		return ret;
+
+	switch (index) {
+	case MOTOR_CFG_ALL:
+		return -EINVAL;
+	case MOTOR_INTENSITY:
+		i = drvdata.gp_rumble_intensity;
+		if (i >= ARRAY_SIZE(intensity_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", intensity_text[i]);
+		break;
+	case VIBRATION_NOTIFY_ENABLE:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			i = drvdata.gp_left_notify_en;
+			break;
+		case RIGHT_CONTROLLER:
+			i = drvdata.gp_right_notify_en;
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case RUMBLE_MODE:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			i = drvdata.gp_left_rumble_mode;
+			break;
+		case RIGHT_CONTROLLER:
+			i = drvdata.gp_right_rumble_mode;
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (i >= ARRAY_SIZE(rumble_mode_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", rumble_mode_text[i]);
+		break;
+	case TP_VIBRATION_ENABLE:
+		i = drvdata.tp_vibration_en;
+		if (i >= ARRAY_SIZE(enabled_status_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", enabled_status_text[i]);
+		break;
+	case TP_VIBRATION_INTENSITY:
+		i = drvdata.tp_vibration_intensity;
+		if (i >= ARRAY_SIZE(intensity_text))
+			return -EINVAL;
+
+		count = sysfs_emit(buf, "%s\n", intensity_text[i]);
+		break;
+	}
+
+	return count;
+}
+
+static ssize_t motor_config_options(struct device *dev,
+				    struct device_attribute *attr, char *buf,
+				    enum motor_cfg_index index)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	switch (index) {
+	case MOTOR_CFG_ALL:
+		break;
+	case RUMBLE_MODE:
+		for (i = 1; i < ARRAY_SIZE(rumble_mode_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       rumble_mode_text[i]);
+		}
+		break;
+	case MOTOR_INTENSITY:
+	case TP_VIBRATION_INTENSITY:
+		for (i = 1; i < ARRAY_SIZE(intensity_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       intensity_text[i]);
+		}
+		break;
+	case VIBRATION_NOTIFY_ENABLE:
+	case TP_VIBRATION_ENABLE:
+		for (i = 1; i < ARRAY_SIZE(enabled_status_text); i++) {
+			count += sysfs_emit_at(buf, count, "%s ",
+					       enabled_status_text[i]);
+		}
+		break;
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t fps_mode_dpi_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+
+{
+	size_t size = 4;
+	u32 value;
+	u8 val[4];
+	int ret;
+
+	ret = kstrtou32(buf, 10, &value);
+	if (ret)
+		return ret;
+
+	if (value != 500 && value != 800 && value != 1200 && value != 1800)
+		return -EINVAL;
+
+	put_unaligned_le32(value, val);
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, SET_DPI_CFG,
+			       FPS_MODE_DPI, UNSPECIFIED, val, size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t fps_mode_dpi_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	int ret;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_DPI_CFG,
+			       FPS_MODE_DPI, UNSPECIFIED, NULL, 0);
+	if (ret < 0)
+		return ret;
+
+	return sysfs_emit(buf, "%u\n", drvdata.mouse_dpi);
+}
+
+static ssize_t fps_mode_dpi_index_show(struct device *dev,
+				       struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "500 800 1200 1800\n");
+}
+
+static ssize_t device_status_show(struct device *dev,
+				  struct device_attribute *attr, char *buf,
+				  enum device_status_index index,
+				  enum dev_type device_type,
+				  enum cal_device_type cal_type)
+{
+	u8 i;
+
+	switch (index) {
+	case GET_CAL_STATUS:
+		switch (device_type) {
+		case LEFT_CONTROLLER:
+			switch (cal_type) {
+			case CALDEV_GYROSCOPE:
+				i = drvdata.gp_left_gyro_cal_status;
+				break;
+			case CALDEV_JOYSTICK:
+				i = drvdata.gp_left_joy_cal_status;
+				break;
+			case CALDEV_TRIGGER:
+				i = drvdata.gp_left_trigg_cal_status;
+				break;
+			default:
+				return -EINVAL;
+			}
+			break;
+		case RIGHT_CONTROLLER:
+			switch (cal_type) {
+			case CALDEV_GYROSCOPE:
+				i = drvdata.gp_right_gyro_cal_status;
+				break;
+			case CALDEV_JOYSTICK:
+				i = drvdata.gp_right_joy_cal_status;
+				break;
+			case CALDEV_TRIGGER:
+				i = drvdata.gp_right_trigg_cal_status;
+				break;
+			default:
+				return -EINVAL;
+			}
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (i >= ARRAY_SIZE(cal_status_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", cal_status_text[i]);
+}
+
+static ssize_t calibrate_config_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, u8 cmd, u8 sub_cmd,
+				      size_t count, enum dev_type device_type)
+{
+	size_t size = 1;
+	u8 val = 0;
+	int ret;
+
+	ret = sysfs_match_string(cal_enabled_text, buf);
+	if (ret < 0)
+		return ret;
+
+	val = ret;
+	if (!val)
+		size = 0;
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, cmd, sub_cmd,
+			       device_type, &val, size);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t calibrate_config_options(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(cal_enabled_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", cal_enabled_text[i]);
+
+	buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t os_mode_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	size_t size = 1;
+	int ret;
+	u8 val;
+
+	ret = sysfs_match_string(os_mode_text, buf);
+	if (ret <= 0)
+		return ret;
+
+	val = ret;
+	ret = mcu_property_out(drvdata.hdev, OS_MODE_DATA, FEATURE_OS_MODE,
+			       SET_OS_MODE, USB_MCU, &val, size);
+	if (ret < 0)
+		return ret;
+
+	drvdata.os_mode = val;
+
+	return count;
+}
+
+static ssize_t os_mode_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	ssize_t count = 0;
+	int ret;
+	u8 i;
+
+	ret = mcu_property_out(drvdata.hdev, OS_MODE_DATA, FEATURE_OS_MODE,
+			       GET_OS_MODE, USB_MCU, NULL, 0);
+	if (ret)
+		return ret;
+
+	i = drvdata.os_mode;
+	if (i >= ARRAY_SIZE(os_mode_text))
+		return -EINVAL;
+
+	count = sysfs_emit(buf, "%s\n", os_mode_text[i]);
+
+	return count;
+}
+
+static ssize_t os_mode_index_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(os_mode_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", os_mode_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static int rgb_cfg_call(struct hid_device *hdev, enum mcu_command_index cmd,
+			enum rgb_config_index index, u8 *val, size_t size)
+{
+	if (cmd != SET_RGB_CFG && cmd != GET_RGB_CFG)
+		return -EINVAL;
+
+	if (index < LIGHT_CFG_ALL || index > USR_LIGHT_PROFILE_3)
+		return -EINVAL;
+
+	return mcu_property_out(hdev, MCU_CONFIG_DATA, cmd, index, UNSPECIFIED,
+				val, size);
+}
+
+static int rgb_attr_show(void)
+{
+	enum rgb_config_index index;
+
+	index = drvdata.rgb_profile + 3;
+
+	return rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, index, NULL, 0);
+}
+
+static ssize_t rgb_effect_store(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	u8 effect;
+	int ret;
+
+	ret = sysfs_match_string(rgb_effect_text, buf);
+	if (ret < 0)
+		return ret;
+
+	effect = ret;
+	index = drvdata.rgb_profile + 3;
+	u8 rgb_profile[6] = { effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      drvdata.led_cdev->brightness,
+			      drvdata.rgb_speed };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_effect = effect;
+	return count;
+}
+
+static ssize_t rgb_effect_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	int ret;
+
+	ret = rgb_attr_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_effect >= ARRAY_SIZE(rgb_effect_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", rgb_effect_text[drvdata.rgb_effect]);
+}
+
+static ssize_t rgb_effect_index_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(rgb_effect_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", rgb_effect_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t rgb_speed_store(struct device *dev,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	int val = 0;
+	int ret;
+
+	ret = kstrtoint(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val < 0 || val > 100)
+		return -EINVAL;
+
+	/* This is a delay setting, invert logic for consistency with other drivers */
+	val = 100 - val;
+
+	index = drvdata.rgb_profile + 3;
+	u8 rgb_profile[6] = { drvdata.rgb_effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      drvdata.led_cdev->brightness,
+			      val };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_speed = val;
+
+	return count;
+}
+
+static ssize_t rgb_speed_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	int ret, val;
+
+	ret = rgb_attr_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_speed > 100)
+		return -EINVAL;
+
+	val = drvdata.rgb_speed;
+
+	return sysfs_emit(buf, "%hhu\n", val);
+}
+
+static ssize_t rgb_speed_range_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "0-100\n");
+}
+
+static ssize_t rgb_mode_store(struct device *dev, struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = sysfs_match_string(rgb_mode_text, buf);
+	if (ret <= 0)
+		return ret;
+
+	val = ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, LIGHT_MODE_SEL, &val, 1);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_mode = val;
+
+	return count;
+}
+
+static ssize_t rgb_mode_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	int ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, LIGHT_MODE_SEL, NULL, 0);
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_mode >= ARRAY_SIZE(rgb_mode_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", rgb_mode_text[drvdata.rgb_mode]);
+}
+
+static ssize_t rgb_mode_index_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(rgb_mode_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", rgb_mode_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+
+static ssize_t rgb_profile_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	size_t size = 1;
+	int ret;
+	u8 val;
+
+	ret = kstrtou8(buf, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val < 1 || val > 3)
+		return -EINVAL;
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, LIGHT_PROFILE_SEL, &val, size);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_profile = val;
+
+	return count;
+}
+
+static ssize_t rgb_profile_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	int ret;
+
+	ret = rgb_cfg_call(drvdata.hdev, GET_RGB_CFG, LIGHT_PROFILE_SEL, NULL, 0);
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_profile < 1 || drvdata.rgb_profile > 3)
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%hhu\n", drvdata.rgb_profile);
+}
+
+static ssize_t rgb_profile_range_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "1-3\n");
+}
+
+static void hid_go_brightness_set(struct led_classdev *led_cdev,
+				  enum led_brightness brightness)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(drvdata.led_cdev);
+	enum rgb_config_index index;
+	int ret;
+
+	if (brightness > led_cdev->max_brightness) {
+		dev_err(led_cdev->dev, "Invalid argument\n");
+		return;
+	}
+
+	index = drvdata.rgb_profile + 3;
+	u8 rgb_profile[6] = { drvdata.rgb_effect,
+			      mc_cdev->subled_info[0].intensity,
+			      mc_cdev->subled_info[1].intensity,
+			      mc_cdev->subled_info[2].intensity,
+			      brightness,
+			      drvdata.rgb_speed };
+
+	ret = rgb_cfg_call(drvdata.hdev, SET_RGB_CFG, index, rgb_profile, 6);
+	switch (ret) {
+	case 0:
+		led_cdev->brightness = brightness;
+		break;
+	case -ENODEV: /* during switch to IAP -ENODEV is expected */
+	case -ENOSYS: /* during rmmod -ENOSYS is expected */
+		dev_dbg(led_cdev->dev, "Failed to write RGB profile: %i\n", ret);
+		break;
+	default:
+		dev_err(led_cdev->dev, "Failed to write RGB profile: %i\n", ret);
+	}
+}
+
+#define LEGO_DEVICE_ATTR_RW(_name, _attrname, _dtype, _rtype, _group)         \
+	static ssize_t _name##_store(struct device *dev,                      \
+				     struct device_attribute *attr,           \
+				     const char *buf, size_t count)           \
+	{                                                                     \
+		return _group##_store(dev, attr, buf, count, _name.index,     \
+				      _dtype);                                \
+	}                                                                     \
+	static ssize_t _name##_show(struct device *dev,                       \
+				    struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return _group##_show(dev, attr, buf, _name.index, _dtype);    \
+	}                                                                     \
+	static ssize_t _name##_##_rtype##_show(                               \
+		struct device *dev, struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return _group##_options(dev, attr, buf, _name.index);         \
+	}                                                                     \
+	static DEVICE_ATTR_RW_NAMED(_name, _attrname)
+
+#define LEGO_DEVICE_ATTR_WO(_name, _attrname, _dtype, _group)             \
+	static ssize_t _name##_store(struct device *dev,                  \
+				     struct device_attribute *attr,       \
+				     const char *buf, size_t count)       \
+	{                                                                 \
+		return _group##_store(dev, attr, buf, count, _name.index, \
+				      _dtype);                            \
+	}                                                                 \
+	static DEVICE_ATTR_WO_NAMED(_name, _attrname)
+
+#define LEGO_DEVICE_ATTR_RO(_name, _attrname, _dtype, _group)                 \
+	static ssize_t _name##_show(struct device *dev,                       \
+				    struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return _group##_show(dev, attr, buf, _name.index, _dtype);    \
+	}                                                                     \
+	static DEVICE_ATTR_RO_NAMED(_name, _attrname)
+
+#define LEGO_CAL_DEVICE_ATTR(_name, _attrname, _scmd, _dtype, _rtype)         \
+	static ssize_t _name##_store(struct device *dev,                      \
+				     struct device_attribute *attr,           \
+				     const char *buf, size_t count)           \
+	{                                                                     \
+		return calibrate_config_store(dev, attr, buf, _name.index,    \
+					      _scmd, count, _dtype);          \
+	}                                                                     \
+	static ssize_t _name##_##_rtype##_show(                               \
+		struct device *dev, struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return calibrate_config_options(dev, attr, buf);              \
+	}                                                                     \
+	static DEVICE_ATTR_WO_NAMED(_name, _attrname)
+
+#define LEGO_DEVICE_STATUS_ATTR(_name, _attrname, _scmd, _dtype)              \
+	static ssize_t _name##_show(struct device *dev,                       \
+				    struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return device_status_show(dev, attr, buf, _name.index, _scmd, \
+					  _dtype);                            \
+	}                                                                     \
+	static DEVICE_ATTR_RO_NAMED(_name, _attrname)
+
+/* Gamepad - MCU */
+static struct go_cfg_attr version_product_mcu = { PRODUCT_VERSION };
+LEGO_DEVICE_ATTR_RO(version_product_mcu, "product_version", USB_MCU, version);
+
+static struct go_cfg_attr version_protocol_mcu = { PROTOCOL_VERSION };
+LEGO_DEVICE_ATTR_RO(version_protocol_mcu, "protocol_version", USB_MCU, version);
+
+static struct go_cfg_attr version_firmware_mcu = { FIRMWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_firmware_mcu, "firmware_version", USB_MCU, version);
+
+static struct go_cfg_attr version_hardware_mcu = { HARDWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_hardware_mcu, "hardware_version", USB_MCU, version);
+
+static struct go_cfg_attr version_gen_mcu = { HARDWARE_GENERATION };
+LEGO_DEVICE_ATTR_RO(version_gen_mcu, "hardware_generation", USB_MCU, version);
+
+static struct go_cfg_attr fps_switch_status = { FEATURE_FPS_SWITCH_STATUS };
+LEGO_DEVICE_ATTR_RO(fps_switch_status, "fps_switch_status", UNSPECIFIED,
+		    feature_status);
+
+static struct go_cfg_attr gamepad_mode = { FEATURE_GAMEPAD_MODE };
+LEGO_DEVICE_ATTR_RW(gamepad_mode, "mode", UNSPECIFIED, index, feature_status);
+static DEVICE_ATTR_RO_NAMED(gamepad_mode_index, "mode_index");
+
+static struct go_cfg_attr reset_mcu = { FEATURE_RESET_GAMEPAD };
+LEGO_DEVICE_ATTR_WO(reset_mcu, "reset_mcu", USB_MCU, feature_status);
+
+static struct go_cfg_attr gamepad_rumble_intensity = { MOTOR_INTENSITY };
+LEGO_DEVICE_ATTR_RW(gamepad_rumble_intensity, "rumble_intensity", UNSPECIFIED,
+		    index, motor_config);
+static DEVICE_ATTR_RO_NAMED(gamepad_rumble_intensity_index,
+			    "rumble_intensity_index");
+
+static DEVICE_ATTR_RW(fps_mode_dpi);
+static DEVICE_ATTR_RO(fps_mode_dpi_index);
+
+static DEVICE_ATTR_RW(os_mode);
+static DEVICE_ATTR_RO(os_mode_index);
+
+static struct attribute *mcu_attrs[] = {
+	&dev_attr_fps_mode_dpi.attr,
+	&dev_attr_fps_mode_dpi_index.attr,
+	&dev_attr_fps_switch_status.attr,
+	&dev_attr_gamepad_mode.attr,
+	&dev_attr_gamepad_mode_index.attr,
+	&dev_attr_gamepad_rumble_intensity.attr,
+	&dev_attr_gamepad_rumble_intensity_index.attr,
+	&dev_attr_os_mode.attr,
+	&dev_attr_os_mode_index.attr,
+	&dev_attr_reset_mcu.attr,
+	&dev_attr_version_firmware_mcu.attr,
+	&dev_attr_version_gen_mcu.attr,
+	&dev_attr_version_hardware_mcu.attr,
+	&dev_attr_version_product_mcu.attr,
+	&dev_attr_version_protocol_mcu.attr,
+	NULL,
+};
+
+static const struct attribute_group mcu_attr_group = {
+	.attrs = mcu_attrs,
+};
+
+/* Gamepad - TX Dongle */
+static struct go_cfg_attr version_product_tx_dongle = { PRODUCT_VERSION };
+LEGO_DEVICE_ATTR_RO(version_product_tx_dongle, "product_version", TX_DONGLE, version);
+
+static struct go_cfg_attr version_protocol_tx_dongle = { PROTOCOL_VERSION };
+LEGO_DEVICE_ATTR_RO(version_protocol_tx_dongle, "protocol_version", TX_DONGLE, version);
+
+static struct go_cfg_attr version_firmware_tx_dongle = { FIRMWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_firmware_tx_dongle, "firmware_version", TX_DONGLE, version);
+
+static struct go_cfg_attr version_hardware_tx_dongle = { HARDWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_hardware_tx_dongle, "hardware_version", TX_DONGLE, version);
+
+static struct go_cfg_attr version_gen_tx_dongle = { HARDWARE_GENERATION };
+LEGO_DEVICE_ATTR_RO(version_gen_tx_dongle, "hardware_generation", TX_DONGLE, version);
+
+static struct go_cfg_attr reset_tx_dongle = { FEATURE_RESET_GAMEPAD };
+LEGO_DEVICE_ATTR_RO(reset_tx_dongle, "reset", TX_DONGLE, feature_status);
+
+static struct attribute *tx_dongle_attrs[] = {
+	&dev_attr_reset_tx_dongle.attr,
+	&dev_attr_version_hardware_tx_dongle.attr,
+	&dev_attr_version_firmware_tx_dongle.attr,
+	&dev_attr_version_gen_tx_dongle.attr,
+	&dev_attr_version_product_tx_dongle.attr,
+	&dev_attr_version_protocol_tx_dongle.attr,
+	NULL,
+};
+
+static const struct attribute_group tx_dongle_attr_group = {
+	.name = "tx_dongle",
+	.attrs = tx_dongle_attrs,
+};
+
+/* Gamepad - Left */
+static struct go_cfg_attr version_product_left = { PRODUCT_VERSION };
+LEGO_DEVICE_ATTR_RO(version_product_left, "product_version", LEFT_CONTROLLER, version);
+
+static struct go_cfg_attr version_protocol_left = { PROTOCOL_VERSION };
+LEGO_DEVICE_ATTR_RO(version_protocol_left, "protocol_version", LEFT_CONTROLLER, version);
+
+static struct go_cfg_attr version_firmware_left = { FIRMWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_firmware_left, "firmware_version", LEFT_CONTROLLER, version);
+
+static struct go_cfg_attr version_hardware_left = { HARDWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_hardware_left, "hardware_version", LEFT_CONTROLLER, version);
+
+static struct go_cfg_attr version_gen_left = { HARDWARE_GENERATION };
+LEGO_DEVICE_ATTR_RO(version_gen_left, "hardware_generation", LEFT_CONTROLLER, version);
+
+static struct go_cfg_attr auto_sleep_time_left = { FEATURE_AUTO_SLEEP_TIME };
+LEGO_DEVICE_ATTR_RW(auto_sleep_time_left, "auto_sleep_time", LEFT_CONTROLLER,
+		    range, feature_status);
+static DEVICE_ATTR_RO_NAMED(auto_sleep_time_left_range,
+			    "auto_sleep_time_range");
+
+static struct go_cfg_attr imu_bypass_left = { FEATURE_IMU_BYPASS };
+LEGO_DEVICE_ATTR_RW(imu_bypass_left, "imu_bypass_enabled", LEFT_CONTROLLER,
+		    index, feature_status);
+static DEVICE_ATTR_RO_NAMED(imu_bypass_left_index, "imu_bypass_enabled_index");
+
+static struct go_cfg_attr imu_enabled_left = { FEATURE_IMU_ENABLE };
+LEGO_DEVICE_ATTR_RW(imu_enabled_left, "imu_enabled", LEFT_CONTROLLER, index,
+		    feature_status);
+static DEVICE_ATTR_RO_NAMED(imu_enabled_left_index, "imu_enabled_index");
+
+static struct go_cfg_attr reset_left = { FEATURE_RESET_GAMEPAD };
+LEGO_DEVICE_ATTR_WO(reset_left, "reset", LEFT_CONTROLLER, feature_status);
+
+static struct go_cfg_attr rumble_mode_left = { RUMBLE_MODE };
+LEGO_DEVICE_ATTR_RW(rumble_mode_left, "rumble_mode", LEFT_CONTROLLER, index,
+		    motor_config);
+static DEVICE_ATTR_RO_NAMED(rumble_mode_left_index, "rumble_mode_index");
+
+static struct go_cfg_attr rumble_notification_left = { VIBRATION_NOTIFY_ENABLE };
+LEGO_DEVICE_ATTR_RW(rumble_notification_left, "rumble_notification",
+		    LEFT_CONTROLLER, index, motor_config);
+static DEVICE_ATTR_RO_NAMED(rumble_notification_left_index,
+			    "rumble_notification_index");
+
+static struct go_cfg_attr cal_trigg_left = { TRIGGER_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_trigg_left, "calibrate_trigger", SET_TRIGGER_CFG,
+		     LEFT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_trigg_left_index, "calibrate_trigger_index");
+
+static struct go_cfg_attr cal_joy_left = { JOYSTICK_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_joy_left, "calibrate_joystick", SET_JOYSTICK_CFG,
+		     LEFT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_joy_left_index, "calibrate_joystick_index");
+
+static struct go_cfg_attr cal_gyro_left = { GYRO_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_gyro_left, "calibrate_gyro", SET_GYRO_CFG,
+		     LEFT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_gyro_left_index, "calibrate_gyro_index");
+
+static struct go_cfg_attr cal_trigg_left_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_trigg_left_status, "calibrate_trigger_status",
+			LEFT_CONTROLLER, CALDEV_TRIGGER);
+
+static struct go_cfg_attr cal_joy_left_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_joy_left_status, "calibrate_joystick_status",
+			LEFT_CONTROLLER, CALDEV_JOYSTICK);
+
+static struct go_cfg_attr cal_gyro_left_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_gyro_left_status, "calibrate_gyro_status",
+			LEFT_CONTROLLER, CALDEV_GYROSCOPE);
+
+static struct attribute *left_gamepad_attrs[] = {
+	&dev_attr_auto_sleep_time_left.attr,
+	&dev_attr_auto_sleep_time_left_range.attr,
+	&dev_attr_cal_gyro_left.attr,
+	&dev_attr_cal_gyro_left_index.attr,
+	&dev_attr_cal_gyro_left_status.attr,
+	&dev_attr_cal_joy_left.attr,
+	&dev_attr_cal_joy_left_index.attr,
+	&dev_attr_cal_joy_left_status.attr,
+	&dev_attr_cal_trigg_left.attr,
+	&dev_attr_cal_trigg_left_index.attr,
+	&dev_attr_cal_trigg_left_status.attr,
+	&dev_attr_imu_bypass_left.attr,
+	&dev_attr_imu_bypass_left_index.attr,
+	&dev_attr_imu_enabled_left.attr,
+	&dev_attr_imu_enabled_left_index.attr,
+	&dev_attr_reset_left.attr,
+	&dev_attr_rumble_mode_left.attr,
+	&dev_attr_rumble_mode_left_index.attr,
+	&dev_attr_rumble_notification_left.attr,
+	&dev_attr_rumble_notification_left_index.attr,
+	&dev_attr_version_hardware_left.attr,
+	&dev_attr_version_firmware_left.attr,
+	&dev_attr_version_gen_left.attr,
+	&dev_attr_version_product_left.attr,
+	&dev_attr_version_protocol_left.attr,
+	NULL,
+};
+
+static const struct attribute_group left_gamepad_attr_group = {
+	.name = "left_handle",
+	.attrs = left_gamepad_attrs,
+};
+
+/* Gamepad - Right */
+static struct go_cfg_attr version_product_right = { PRODUCT_VERSION };
+LEGO_DEVICE_ATTR_RO(version_product_right, "product_version", RIGHT_CONTROLLER, version);
+
+static struct go_cfg_attr version_protocol_right = { PROTOCOL_VERSION };
+LEGO_DEVICE_ATTR_RO(version_protocol_right, "protocol_version", RIGHT_CONTROLLER, version);
+
+static struct go_cfg_attr version_firmware_right = { FIRMWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_firmware_right, "firmware_version", RIGHT_CONTROLLER, version);
+
+static struct go_cfg_attr version_hardware_right = { HARDWARE_VERSION };
+LEGO_DEVICE_ATTR_RO(version_hardware_right, "hardware_version", RIGHT_CONTROLLER, version);
+
+static struct go_cfg_attr version_gen_right = { HARDWARE_GENERATION };
+LEGO_DEVICE_ATTR_RO(version_gen_right, "hardware_generation", RIGHT_CONTROLLER, version);
+
+static struct go_cfg_attr auto_sleep_time_right = { FEATURE_AUTO_SLEEP_TIME };
+LEGO_DEVICE_ATTR_RW(auto_sleep_time_right, "auto_sleep_time", RIGHT_CONTROLLER,
+		    range, feature_status);
+static DEVICE_ATTR_RO_NAMED(auto_sleep_time_right_range,
+			    "auto_sleep_time_range");
+
+static struct go_cfg_attr imu_bypass_right = { FEATURE_IMU_BYPASS };
+LEGO_DEVICE_ATTR_RW(imu_bypass_right, "imu_bypass_enabled", RIGHT_CONTROLLER,
+		    index, feature_status);
+static DEVICE_ATTR_RO_NAMED(imu_bypass_right_index, "imu_bypass_enabled_index");
+
+static struct go_cfg_attr imu_enabled_right = { FEATURE_IMU_BYPASS };
+LEGO_DEVICE_ATTR_RW(imu_enabled_right, "imu_enabled", RIGHT_CONTROLLER, index,
+		    feature_status);
+static DEVICE_ATTR_RO_NAMED(imu_enabled_right_index, "imu_enabled_index");
+
+static struct go_cfg_attr reset_right = { FEATURE_RESET_GAMEPAD };
+LEGO_DEVICE_ATTR_WO(reset_right, "reset", LEFT_CONTROLLER, feature_status);
+
+static struct go_cfg_attr rumble_mode_right = { RUMBLE_MODE };
+LEGO_DEVICE_ATTR_RW(rumble_mode_right, "rumble_mode", RIGHT_CONTROLLER, index,
+		    motor_config);
+static DEVICE_ATTR_RO_NAMED(rumble_mode_right_index, "rumble_mode_index");
+
+static struct go_cfg_attr rumble_notification_right = { VIBRATION_NOTIFY_ENABLE };
+LEGO_DEVICE_ATTR_RW(rumble_notification_right, "rumble_notification",
+		    RIGHT_CONTROLLER, index, motor_config);
+static DEVICE_ATTR_RO_NAMED(rumble_notification_right_index,
+			    "rumble_notification_index");
+
+static struct go_cfg_attr cal_trigg_right = { TRIGGER_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_trigg_right, "calibrate_trigger", SET_TRIGGER_CFG,
+		     RIGHT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_trigg_right_index, "calibrate_trigger_index");
+
+static struct go_cfg_attr cal_joy_right = { JOYSTICK_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_joy_right, "calibrate_joystick", SET_JOYSTICK_CFG,
+		     RIGHT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_joy_right_index, "calibrate_joystick_index");
+
+static struct go_cfg_attr cal_gyro_right = { GYRO_CALIBRATE };
+LEGO_CAL_DEVICE_ATTR(cal_gyro_right, "calibrate_gyro", SET_GYRO_CFG,
+		     RIGHT_CONTROLLER, index);
+static DEVICE_ATTR_RO_NAMED(cal_gyro_right_index, "calibrate_gyro_index");
+
+static struct go_cfg_attr cal_trigg_right_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_trigg_right_status, "calibrate_trigger_status",
+			RIGHT_CONTROLLER, CALDEV_TRIGGER);
+
+static struct go_cfg_attr cal_joy_right_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_joy_right_status, "calibrate_joystick_status",
+			RIGHT_CONTROLLER, CALDEV_JOYSTICK);
+
+static struct go_cfg_attr cal_gyro_right_status = { GET_CAL_STATUS };
+LEGO_DEVICE_STATUS_ATTR(cal_gyro_right_status, "calibrate_gyro_status",
+			RIGHT_CONTROLLER, CALDEV_GYROSCOPE);
+
+static struct attribute *right_gamepad_attrs[] = {
+	&dev_attr_auto_sleep_time_right.attr,
+	&dev_attr_auto_sleep_time_right_range.attr,
+	&dev_attr_cal_gyro_right.attr,
+	&dev_attr_cal_gyro_right_index.attr,
+	&dev_attr_cal_gyro_right_status.attr,
+	&dev_attr_cal_joy_right.attr,
+	&dev_attr_cal_joy_right_index.attr,
+	&dev_attr_cal_joy_right_status.attr,
+	&dev_attr_cal_trigg_right.attr,
+	&dev_attr_cal_trigg_right_index.attr,
+	&dev_attr_cal_trigg_right_status.attr,
+	&dev_attr_imu_bypass_right.attr,
+	&dev_attr_imu_bypass_right_index.attr,
+	&dev_attr_imu_enabled_right.attr,
+	&dev_attr_imu_enabled_right_index.attr,
+	&dev_attr_reset_right.attr,
+	&dev_attr_rumble_mode_right.attr,
+	&dev_attr_rumble_mode_right_index.attr,
+	&dev_attr_rumble_notification_right.attr,
+	&dev_attr_rumble_notification_right_index.attr,
+	&dev_attr_version_hardware_right.attr,
+	&dev_attr_version_firmware_right.attr,
+	&dev_attr_version_gen_right.attr,
+	&dev_attr_version_product_right.attr,
+	&dev_attr_version_protocol_right.attr,
+	NULL,
+};
+
+static const struct attribute_group right_gamepad_attr_group = {
+	.name = "right_handle",
+	.attrs = right_gamepad_attrs,
+};
+
+/* Touchpad */
+static struct go_cfg_attr touchpad_enabled = { FEATURE_TOUCHPAD_ENABLE };
+LEGO_DEVICE_ATTR_RW(touchpad_enabled, "enabled", UNSPECIFIED, index,
+		    feature_status);
+static DEVICE_ATTR_RO_NAMED(touchpad_enabled_index, "enabled_index");
+
+static struct go_cfg_attr touchpad_vibration_enabled = { TP_VIBRATION_ENABLE };
+LEGO_DEVICE_ATTR_RW(touchpad_vibration_enabled, "vibration_enabled", UNSPECIFIED,
+		    index, motor_config);
+static DEVICE_ATTR_RO_NAMED(touchpad_vibration_enabled_index,
+			    "vibration_enabled_index");
+
+static struct go_cfg_attr touchpad_vibration_intensity = { TP_VIBRATION_INTENSITY };
+LEGO_DEVICE_ATTR_RW(touchpad_vibration_intensity, "vibration_intensity",
+		    UNSPECIFIED, index, motor_config);
+static DEVICE_ATTR_RO_NAMED(touchpad_vibration_intensity_index,
+			    "vibration_intensity_index");
+
+static struct attribute *touchpad_attrs[] = {
+	&dev_attr_touchpad_enabled.attr,
+	&dev_attr_touchpad_enabled_index.attr,
+	&dev_attr_touchpad_vibration_enabled.attr,
+	&dev_attr_touchpad_vibration_enabled_index.attr,
+	&dev_attr_touchpad_vibration_intensity.attr,
+	&dev_attr_touchpad_vibration_intensity_index.attr,
+	NULL,
+};
+
+static const struct attribute_group touchpad_attr_group = {
+	.name = "touchpad",
+	.attrs = touchpad_attrs,
+};
+
+static const struct attribute_group *top_level_attr_groups[] = {
+	&mcu_attr_group,	  &tx_dongle_attr_group,
+	&left_gamepad_attr_group, &right_gamepad_attr_group,
+	&touchpad_attr_group,	  NULL,
+};
+
+/* RGB */
+static struct go_cfg_attr rgb_enabled = { FEATURE_LIGHT_ENABLE };
+
+LEGO_DEVICE_ATTR_RW(rgb_enabled, "enabled", UNSPECIFIED, index, feature_status);
+static DEVICE_ATTR_RO_NAMED(rgb_effect_index, "effect_index");
+static DEVICE_ATTR_RO_NAMED(rgb_enabled_index, "enabled_index");
+static DEVICE_ATTR_RO_NAMED(rgb_mode_index, "mode_index");
+static DEVICE_ATTR_RO_NAMED(rgb_profile_range, "profile_range");
+static DEVICE_ATTR_RO_NAMED(rgb_speed_range, "speed_range");
+static DEVICE_ATTR_RW_NAMED(rgb_effect, "effect");
+static DEVICE_ATTR_RW_NAMED(rgb_mode, "mode");
+static DEVICE_ATTR_RW_NAMED(rgb_profile, "profile");
+static DEVICE_ATTR_RW_NAMED(rgb_speed, "speed");
+
+static struct attribute *go_rgb_attrs[] = {
+	&dev_attr_rgb_effect.attr,
+	&dev_attr_rgb_effect_index.attr,
+	&dev_attr_rgb_enabled.attr,
+	&dev_attr_rgb_enabled_index.attr,
+	&dev_attr_rgb_mode.attr,
+	&dev_attr_rgb_mode_index.attr,
+	&dev_attr_rgb_profile.attr,
+	&dev_attr_rgb_profile_range.attr,
+	&dev_attr_rgb_speed.attr,
+	&dev_attr_rgb_speed_range.attr,
+	NULL,
+};
+
+static struct attribute_group rgb_attr_group = {
+	.attrs = go_rgb_attrs,
+};
+
+static struct mc_subled go_rgb_subled_info[] = {
+	{
+		.color_index = LED_COLOR_ID_RED,
+		.brightness = 0x50,
+		.intensity = 0x24,
+		.channel = 0x1,
+	},
+	{
+		.color_index = LED_COLOR_ID_GREEN,
+		.brightness = 0x50,
+		.intensity = 0x22,
+		.channel = 0x2,
+	},
+	{
+		.color_index = LED_COLOR_ID_BLUE,
+		.brightness = 0x50,
+		.intensity = 0x99,
+		.channel = 0x3,
+	},
+};
+
+static struct led_classdev_mc go_cdev_rgb = {
+	.led_cdev = {
+		.name = "go:rgb:joystick_rings",
+		.color = LED_COLOR_ID_RGB,
+		.brightness = 0x50,
+		.max_brightness = 0x64,
+		.brightness_set = hid_go_brightness_set,
+	},
+	.num_colors = ARRAY_SIZE(go_rgb_subled_info),
+	.subled_info = go_rgb_subled_info,
+};
+
+static void cfg_setup(struct work_struct *work)
+{
+	int ret;
+
+	/* MCU Version Attrs */
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PRODUCT_VERSION, USB_MCU, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve USB_MCU Product Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PROTOCOL_VERSION, USB_MCU, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve USB_MCU Protocol Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       FIRMWARE_VERSION, USB_MCU, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve USB_MCU Firmware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_VERSION, USB_MCU, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve USB_MCU Hardware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_GENERATION, USB_MCU, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve USB_MCU Hardware Generation: %i\n", ret);
+		return;
+	}
+
+	/* TX Dongle Version Attrs */
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PRODUCT_VERSION, TX_DONGLE, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve TX_DONGLE Product Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PROTOCOL_VERSION, TX_DONGLE, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve TX_DONGLE Protocol Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       FIRMWARE_VERSION, TX_DONGLE, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve TX_DONGLE Firmware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_VERSION, TX_DONGLE, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve TX_DONGLE Hardware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_GENERATION, TX_DONGLE, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve TX_DONGLE Hardware Generation: %i\n", ret);
+		return;
+	}
+
+	/* Left Handle Version Attrs */
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PRODUCT_VERSION, LEFT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve LEFT_CONTROLLER Product Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PROTOCOL_VERSION, LEFT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve LEFT_CONTROLLER Protocol Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       FIRMWARE_VERSION, LEFT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve LEFT_CONTROLLER Firmware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_VERSION, LEFT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve LEFT_CONTROLLER Hardware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_GENERATION, LEFT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve LEFT_CONTROLLER Hardware Generation: %i\n", ret);
+		return;
+	}
+
+	/* Right Handle Version Attrs */
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PRODUCT_VERSION, RIGHT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve RIGHT_CONTROLLER Product Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       PROTOCOL_VERSION, RIGHT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve RIGHT_CONTROLLER Protocol Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       FIRMWARE_VERSION, RIGHT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve RIGHT_CONTROLLER Firmware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_VERSION, RIGHT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve RIGHT_CONTROLLER Hardware Version: %i\n", ret);
+		return;
+	}
+
+	ret = mcu_property_out(drvdata.hdev, MCU_CONFIG_DATA, GET_VERSION_DATA,
+			       HARDWARE_GENERATION, RIGHT_CONTROLLER, NULL, 0);
+	if (ret < 0) {
+		dev_err(&drvdata.hdev->dev,
+			"Failed to retrieve RIGHT_CONTROLLER Hardware Generation: %i\n", ret);
+		return;
+	}
+}
+
+static int hid_go_cfg_probe(struct hid_device *hdev,
+			    const struct hid_device_id *_id)
+{
+	unsigned char *buf;
+	int ret;
+
+	buf = devm_kzalloc(&hdev->dev, GO_PACKET_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	hid_set_drvdata(hdev, &drvdata);
+	drvdata.hdev = hdev;
+	mutex_init(&drvdata.cfg_mutex);
+
+	ret = sysfs_create_groups(&hdev->dev.kobj, top_level_attr_groups);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret,
+			      "Failed to create gamepad configuration attributes\n");
+		return ret;
+	}
+
+	ret = devm_led_classdev_multicolor_register(&hdev->dev, &go_cdev_rgb);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret, "Failed to create RGB device\n");
+		return ret;
+	}
+
+	ret = devm_device_add_group(go_cdev_rgb.led_cdev.dev, &rgb_attr_group);
+	if (ret) {
+		dev_err_probe(&hdev->dev, ret,
+			      "Failed to create RGB configuration attributes\n");
+		return ret;
+	}
+
+	drvdata.led_cdev = &go_cdev_rgb.led_cdev;
+
+	init_completion(&drvdata.send_cmd_complete);
+
+	/* Executing calls prior to returning from probe will lock the MCU. Schedule
+	 * initial data call after probe has completed and MCU can accept calls.
+	 */
+	INIT_DELAYED_WORK(&drvdata.go_cfg_setup, &cfg_setup);
+	ret = schedule_delayed_work(&drvdata.go_cfg_setup, msecs_to_jiffies(2));
+	if (!ret) {
+		dev_err(&hdev->dev,
+			"Failed to schedule startup delayed work\n");
+		return -ENODEV;
+	}
+	return 0;
+}
+
+static void hid_go_cfg_remove(struct hid_device *hdev)
+{
+	guard(mutex)(&drvdata.cfg_mutex);
+	sysfs_remove_groups(&hdev->dev.kobj, top_level_attr_groups);
+	hid_hw_close(hdev);
+	hid_hw_stop(hdev);
+	hid_set_drvdata(hdev, NULL);
+}
+
+static int hid_go_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	int ret, ep;
+
+	hdev->quirks |= HID_QUIRK_INPUT_PER_APP | HID_QUIRK_MULTI_INPUT;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "Parse failed\n");
+		return ret;
+	}
+
+	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+	if (ret) {
+		hid_err(hdev, "Failed to start HID device\n");
+		return ret;
+	}
+
+	ret = hid_hw_open(hdev);
+	if (ret) {
+		hid_err(hdev, "Failed to open HID device\n");
+		hid_hw_stop(hdev);
+		return ret;
+	}
+
+	ep = get_endpoint_address(hdev);
+	if (ep != GO_GP_INTF_IN) {
+		dev_dbg(&hdev->dev, "Started interface %x as generic HID device\n", ep);
+		return 0;
+	}
+
+	ret = hid_go_cfg_probe(hdev, id);
+	if (ret)
+		dev_err_probe(&hdev->dev, ret, "Failed to start configuration interface\n");
+
+	dev_dbg(&hdev->dev, "Started Legion Go HID Device: %x\n", ep);
+
+	return ret;
+}
+
+static void hid_go_remove(struct hid_device *hdev)
+{
+	int ep = get_endpoint_address(hdev);
+
+	if (ep <= 0)
+		return;
+
+	switch (ep) {
+	case GO_GP_INTF_IN:
+		hid_go_cfg_remove(hdev);
+		break;
+	default:
+		hid_hw_close(hdev);
+		hid_hw_stop(hdev);
+		break;
+	}
+}
+
+static const struct hid_device_id hid_go_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO2_XINPUT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO2_DINPUT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO2_DUAL_DINPUT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO,
+			 USB_DEVICE_ID_LENOVO_LEGION_GO2_FPS) },
+	{}
+};
+MODULE_DEVICE_TABLE(hid, hid_go_devices);
+
+static struct hid_driver hid_lenovo_go = {
+	.name = "hid-lenovo-go",
+	.id_table = hid_go_devices,
+	.probe = hid_go_probe,
+	.remove = hid_go_remove,
+	.raw_event = hid_go_raw_event,
+};
+module_hid_driver(hid_lenovo_go);
+
+MODULE_AUTHOR("Derek J. Clark");
+MODULE_DESCRIPTION("HID Driver for Lenovo Legion Go Series Gamepads.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index d40932809ce1a..d1dea7297712d 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -4487,10 +4487,12 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		if (!ret)
 			ret = hidpp_ff_init(hidpp, &data);
 
-		if (ret)
+		if (ret) {
 			hid_warn(hidpp->hid_dev,
 		     "Unable to initialize force feedback support, errno %d\n",
 				 ret);
+			ret = 0;
+		}
 	}
 
 	/*
@@ -4668,6 +4670,8 @@ static const struct hid_device_id hidpp_devices[] = {
 	  HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb038) },
 	{ /* Slim Solar+ K980 Keyboard over Bluetooth */
 	  HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb391) },
+	{ /* MX Master 4 mouse over Bluetooth */
+	  HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb042) },
 	{}
 };
 
diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c
index 9eadf3252d0dc..e70bd3dc07ab7 100644
--- a/drivers/hid/hid-magicmouse.c
+++ b/drivers/hid/hid-magicmouse.c
@@ -817,19 +817,21 @@ static int magicmouse_fetch_battery(struct hid_device *hdev)
 #ifdef CONFIG_HID_BATTERY_STRENGTH
 	struct hid_report_enum *report_enum;
 	struct hid_report *report;
+	struct hid_battery *bat;
 
-	if (!hdev->battery ||
+	bat = hid_get_battery(hdev);
+	if (!bat ||
 	    (!is_usb_magicmouse2(hdev->vendor, hdev->product) &&
 	     !is_usb_magictrackpad2(hdev->vendor, hdev->product)))
 		return -1;
 
-	report_enum = &hdev->report_enum[hdev->battery_report_type];
-	report = report_enum->report_id_hash[hdev->battery_report_id];
+	report_enum = &hdev->report_enum[bat->report_type];
+	report = report_enum->report_id_hash[bat->report_id];
 
 	if (!report || report->maxfield < 1)
 		return -1;
 
-	if (hdev->battery_capacity == hdev->battery_max)
+	if (bat->capacity == bat->max)
 		return -1;
 
 	hid_hw_request(hdev, report, HID_REQ_GET_REPORT);
diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c
index ef3b5c77c38e3..be80970ab48e2 100644
--- a/drivers/hid/hid-mcp2221.c
+++ b/drivers/hid/hid-mcp2221.c
@@ -19,8 +19,15 @@
 #include <linux/gpio/driver.h>
 #include <linux/iio/iio.h>
 #include <linux/minmax.h>
+#include <linux/moduleparam.h>
 #include "hid-ids.h"
 
+static bool gpio_mode_enforce;
+
+module_param(gpio_mode_enforce, bool, 0644);
+MODULE_PARM_DESC(gpio_mode_enforce,
+	 "Enforce GPIO mode for GP0 thru GP3 (default: false, will be used for IIO)");
+
 /* Commands codes in a raw output report */
 enum {
 	MCP2221_I2C_WR_DATA = 0x90,
@@ -536,10 +543,10 @@ static int mcp_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 			if (ret)
 				goto exit;
 
-			mcp->rxbuf_idx = 0;
-			mcp->rxbuf = data->block;
-			mcp->txbuf[0] = MCP2221_I2C_GET_DATA;
-			ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
+			ret = mcp_i2c_smbus_read(mcp, NULL,
+						MCP2221_I2C_RD_RPT_START,
+						addr, data->block[0] + 1,
+						data->block);
 			if (ret)
 				goto exit;
 		} else {
@@ -555,14 +562,14 @@ static int mcp_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 	case I2C_SMBUS_I2C_BLOCK_DATA:
 		if (read_write == I2C_SMBUS_READ) {
 			ret = mcp_smbus_write(mcp, addr, command, NULL,
-						0, MCP2221_I2C_WR_NO_STOP, 1);
+						0, MCP2221_I2C_WR_NO_STOP, 0);
 			if (ret)
 				goto exit;
 
-			mcp->rxbuf_idx = 0;
-			mcp->rxbuf = data->block;
-			mcp->txbuf[0] = MCP2221_I2C_GET_DATA;
-			ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
+			ret = mcp_i2c_smbus_read(mcp, NULL,
+						MCP2221_I2C_RD_RPT_START,
+						addr, data->block[0],
+						&data->block[1]);
 			if (ret)
 				goto exit;
 		} else {
@@ -650,7 +657,7 @@ static int mcp2221_check_gpio_pinfunc(struct mcp2221 *mcp)
 	int needgpiofix = 0;
 	int ret;
 
-	if (IS_ENABLED(CONFIG_IIO))
+	if (IS_ENABLED(CONFIG_IIO) && !gpio_mode_enforce)
 		return 0;
 
 	ret = mcp_gpio_read_sram(mcp);
@@ -1045,7 +1052,8 @@ static void mcp2221_remove(struct hid_device *hdev)
 #if IS_REACHABLE(CONFIG_IIO)
 	struct mcp2221 *mcp = hid_get_drvdata(hdev);
 
-	cancel_delayed_work_sync(&mcp->init_work);
+	if (!gpio_mode_enforce)
+		cancel_delayed_work_sync(&mcp->init_work);
 #endif
 }
 
@@ -1319,8 +1327,10 @@ static int mcp2221_probe(struct hid_device *hdev,
 #endif
 
 #if IS_REACHABLE(CONFIG_IIO)
-	INIT_DELAYED_WORK(&mcp->init_work, mcp_init_work);
-	schedule_delayed_work(&mcp->init_work, msecs_to_jiffies(100));
+	if (!gpio_mode_enforce) {
+		INIT_DELAYED_WORK(&mcp->init_work, mcp_init_work);
+		schedule_delayed_work(&mcp->init_work, msecs_to_jiffies(100));
+	}
 #endif
 
 	return 0;
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index b8a748bbf0fd8..e82a3c4e5b44e 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -526,12 +526,19 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report)
 		dev_warn(&hdev->dev, "failed to fetch feature %d\n",
 			 report->id);
 	} else {
+		/* The report ID in the request and the response should match */
+		if (report->id != buf[0]) {
+			hid_err(hdev, "Returned feature report did not match the request\n");
+			goto free;
+		}
+
 		ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf,
 					   size, 0);
 		if (ret)
 			dev_warn(&hdev->dev, "failed to report feature\n");
 	}
 
+free:
 	kfree(buf);
 }
 
diff --git a/drivers/hid/hid-picolcd_cir.c b/drivers/hid/hid-picolcd_cir.c
index d6faa0e00f95a..6d4c636e1c9f7 100644
--- a/drivers/hid/hid-picolcd_cir.c
+++ b/drivers/hid/hid-picolcd_cir.c
@@ -134,5 +134,6 @@ void picolcd_exit_cir(struct picolcd_data *data)
 
 	data->rc_dev = NULL;
 	rc_unregister_device(rdev);
+	rc_free_device(rdev);
 }
 
diff --git a/drivers/hid/hid-pl.c b/drivers/hid/hid-pl.c
index c6c2961dd5741..8bba29ef6c7a8 100644
--- a/drivers/hid/hid-pl.c
+++ b/drivers/hid/hid-pl.c
@@ -24,10 +24,6 @@
  */
 
 
-/* #define DEBUG */
-
-#define debug(format, arg...) pr_debug("hid-plff: " format "\n" , ## arg)
-
 #include <linux/input.h>
 #include <linux/slab.h>
 #include <linux/module.h>
@@ -53,14 +49,14 @@ static int hid_plff_play(struct input_dev *dev, void *data,
 
 	left = effect->u.rumble.strong_magnitude;
 	right = effect->u.rumble.weak_magnitude;
-	debug("called with 0x%04x 0x%04x", left, right);
+	hid_dbg(dev, "called with 0x%04x 0x%04x", left, right);
 
 	left = left * plff->maxval / 0xffff;
 	right = right * plff->maxval / 0xffff;
 
 	*plff->strong = left;
 	*plff->weak = right;
-	debug("running with 0x%02x 0x%02x", left, right);
+	hid_dbg(dev, "running with 0x%02x 0x%02x", left, right);
 	hid_hw_request(hid, plff->report, HID_REQ_SET_REPORT);
 
 	return 0;
@@ -119,7 +115,7 @@ static int plff_init(struct hid_device *hid)
 			report->field[0]->value[1] = 0x00;
 			strong = &report->field[0]->value[2];
 			weak = &report->field[0]->value[3];
-			debug("detected single-field device");
+			hid_dbg(hid, "detected single-field device");
 		} else if (report->field[0]->maxusage == 1 &&
 			   report->field[0]->usage[0].hid ==
 				(HID_UP_LED | 0x43) &&
@@ -134,7 +130,7 @@ static int plff_init(struct hid_device *hid)
 			weak = &report->field[3]->value[0];
 			if (hid->vendor == USB_VENDOR_ID_JESS2)
 				maxval = 0xff;
-			debug("detected 4-field device");
+			hid_dbg(hid, "detected 4-field device");
 		} else {
 			hid_err(hid, "not enough fields or values\n");
 			return -ENODEV;
diff --git a/drivers/hid/i2c-hid/i2c-hid-of-elan.c b/drivers/hid/i2c-hid/i2c-hid-of-elan.c
index b81fcc6ff49ee..919e32c47e12c 100644
--- a/drivers/hid/i2c-hid/i2c-hid-of-elan.c
+++ b/drivers/hid/i2c-hid/i2c-hid-of-elan.c
@@ -195,12 +195,20 @@ static const struct elan_i2c_hid_chip_data ilitek_ili2901_chip_data = {
 	.main_supply_name = "vcc33",
 };
 
+static const struct elan_i2c_hid_chip_data parade_tc3408_chip_data = {
+	.post_power_delay_ms = 10,
+	.post_gpio_reset_on_delay_ms = 300,
+	.hid_descriptor_address = 0x0001,
+	.main_supply_name = "vcc33",
+};
+
 static const struct of_device_id elan_i2c_hid_of_match[] = {
 	{ .compatible = "elan,ekth6915", .data = &elan_ekth6915_chip_data },
 	{ .compatible = "elan,ekth6a12nay", .data = &elan_ekth6a12nay_chip_data },
 	{ .compatible = "focaltech,ft8112", .data = &focaltech_ft8112_chip_data },
 	{ .compatible = "ilitek,ili9882t", .data = &ilitek_ili9882t_chip_data },
 	{ .compatible = "ilitek,ili2901", .data = &ilitek_ili2901_chip_data },
+	{ .compatible = "parade,tc3408", .data = &parade_tc3408_chip_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, elan_i2c_hid_of_match);
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
index f178017352ba7..46d3e9a019995 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
@@ -26,6 +26,11 @@ static struct quicki2c_ddata ptl_ddata = {
 	.max_interrupt_delay = MAX_RX_INTERRUPT_DELAY,
 };
 
+static struct quicki2c_ddata nvl_ddata = {
+	.max_detect_size = MAX_RX_DETECT_SIZE_NVL,
+	.max_interrupt_delay = MAX_RX_INTERRUPT_DELAY,
+};
+
 /* THC QuickI2C ACPI method to get device properties */
 /* HIDI2C device method */
 static guid_t i2c_hid_guid =
@@ -1032,6 +1037,8 @@ static const struct pci_device_id quicki2c_pci_tbl[] = {
 	{ PCI_DEVICE_DATA(INTEL, THC_PTL_U_DEVICE_ID_I2C_PORT2, &ptl_ddata) },
 	{ PCI_DEVICE_DATA(INTEL, THC_WCL_DEVICE_ID_I2C_PORT1, &ptl_ddata) },
 	{ PCI_DEVICE_DATA(INTEL, THC_WCL_DEVICE_ID_I2C_PORT2, &ptl_ddata) },
+	{ PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_I2C_PORT1, &nvl_ddata) },
+	{ PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_I2C_PORT2, &nvl_ddata) },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, quicki2c_pci_tbl);
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
index 33a1e3db1cb2a..61dbdece59a13 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
@@ -15,6 +15,8 @@
 #define PCI_DEVICE_ID_INTEL_THC_PTL_U_DEVICE_ID_I2C_PORT2	0xE44A
 #define PCI_DEVICE_ID_INTEL_THC_WCL_DEVICE_ID_I2C_PORT1 	0x4D48
 #define PCI_DEVICE_ID_INTEL_THC_WCL_DEVICE_ID_I2C_PORT2 	0x4D4A
+#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_I2C_PORT1	0xD348
+#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_I2C_PORT2	0xD34A
 
 /* Packet size value, the unit is 16 bytes */
 #define MAX_PACKET_SIZE_VALUE_LNL			256
@@ -40,6 +42,8 @@
 
 /* PTL Max packet size detection capability is 255 Bytes */
 #define MAX_RX_DETECT_SIZE_PTL			255
+/* NVL Max packet size detection capability is 64K Bytes */
+#define MAX_RX_DETECT_SIZE_NVL			65535
 /* Max interrupt delay capability is 2.56ms */
 #define MAX_RX_INTERRUPT_DELAY			256
 
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
index f9fcb398673b2..8075992e8732e 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
@@ -127,6 +127,7 @@ int quicki2c_hid_probe(struct quicki2c_device *qcdev)
 	hid->product = le16_to_cpu(qcdev->dev_desc.product_id);
 	snprintf(hid->name, sizeof(hid->name), "%s %04X:%04X", "quicki2c-hid",
 		 hid->vendor, hid->product);
+	strscpy(hid->phys, dev_name(qcdev->dev), sizeof(hid->phys));
 
 	ret = hid_add_device(hid);
 	if (ret) {
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
index ad6bd59963b28..b6a69995692cb 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
@@ -37,6 +37,10 @@ struct quickspi_driver_data arl = {
 	.max_packet_size_value = MAX_PACKET_SIZE_VALUE_MTL,
 };
 
+struct quickspi_driver_data nvl = {
+	.max_packet_size_value = MAX_PACKET_SIZE_VALUE_LNL,
+};
+
 /* THC QuickSPI ACPI method to get device properties */
 /* HIDSPI Method: {6e2ac436-0fcf-41af-a265-b32a220dcfab} */
 static guid_t hidspi_guid =
@@ -982,6 +986,8 @@ static const struct pci_device_id quickspi_pci_tbl[] = {
 	{PCI_DEVICE_DATA(INTEL, THC_WCL_DEVICE_ID_SPI_PORT2, &ptl), },
 	{PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT1, &arl), },
 	{PCI_DEVICE_DATA(INTEL, THC_ARL_DEVICE_ID_SPI_PORT2, &arl), },
+	{PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT1, &nvl), },
+	{PCI_DEVICE_DATA(INTEL, THC_NVL_H_DEVICE_ID_SPI_PORT2, &nvl), },
 	{}
 };
 MODULE_DEVICE_TABLE(pci, quickspi_pci_tbl);
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
index c30e1a42eb098..bf5e18f5a5f42 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
@@ -23,6 +23,8 @@
 #define PCI_DEVICE_ID_INTEL_THC_WCL_DEVICE_ID_SPI_PORT2 	0x4D4B
 #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT1 	0x7749
 #define PCI_DEVICE_ID_INTEL_THC_ARL_DEVICE_ID_SPI_PORT2 	0x774B
+#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT1	0xD349
+#define PCI_DEVICE_ID_INTEL_THC_NVL_H_DEVICE_ID_SPI_PORT2	0xD34B
 
 /* HIDSPI special ACPI parameters DSM methods */
 #define ACPI_QUICKSPI_REVISION_NUM			2
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
index 82c72bfa2795e..91d5807b4a830 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
@@ -118,6 +118,7 @@ int quickspi_hid_probe(struct quickspi_device *qsdev)
 	hid->product = le16_to_cpu(qsdev->dev_desc.product_id);
 	snprintf(hid->name, sizeof(hid->name), "%s %04X:%04X", "quickspi-hid",
 		 hid->vendor, hid->product);
+	strscpy(hid->phys, dev_name(qsdev->dev), sizeof(hid->phys));
 
 	ret = hid_add_device(hid);
 	if (ret) {
diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c
index fbf3dbc92e669..aee8a44433059 100644
--- a/drivers/hid/usbhid/hid-pidff.c
+++ b/drivers/hid/usbhid/hid-pidff.c
@@ -13,6 +13,7 @@
 #include <linux/input.h>
 #include <linux/minmax.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/usb.h>
 
 #define	PID_EFFECTS_MAX		64
@@ -81,7 +82,7 @@ static const u8 pidff_set_envelope[] = { 0x22, 0x5b, 0x5c, 0x5d, 0x5e };
 #define PID_NEG_COEFFICIENT	4
 #define PID_POS_SATURATION	5
 #define PID_NEG_SATURATION	6
-#define PID_DEAD_BAND		7
+#define PID_DEADBAND		7
 static const u8 pidff_set_condition[] = {
 	0x22, 0x23, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65
 };
@@ -618,14 +619,24 @@ static void pidff_set_condition_report(struct pidff_device *pidff,
 				 effect->u.condition[i].center);
 		pidff_set_signed(&pidff->set_condition[PID_POS_COEFFICIENT],
 				 effect->u.condition[i].right_coeff);
-		pidff_set_signed(&pidff->set_condition[PID_NEG_COEFFICIENT],
-				 effect->u.condition[i].left_coeff);
 		pidff_set(&pidff->set_condition[PID_POS_SATURATION],
 			  effect->u.condition[i].right_saturation);
-		pidff_set(&pidff->set_condition[PID_NEG_SATURATION],
-			  effect->u.condition[i].left_saturation);
-		pidff_set(&pidff->set_condition[PID_DEAD_BAND],
-			  effect->u.condition[i].deadband);
+
+		/* Omit Negative Coefficient if missing */
+		if (!(pidff->quirks & HID_PIDFF_QUIRK_MISSING_NEG_COEFFICIENT))
+			pidff_set_signed(&pidff->set_condition[PID_NEG_COEFFICIENT],
+					effect->u.condition[i].left_coeff);
+
+		/* Omit Negative Saturation if missing */
+		if (!(pidff->quirks & HID_PIDFF_QUIRK_MISSING_NEG_SATURATION))
+			pidff_set_signed(&pidff->set_condition[PID_NEG_SATURATION],
+					effect->u.condition[i].left_saturation);
+
+		/* Omit Deadband field if missing */
+		if (!(pidff->quirks & HID_PIDFF_QUIRK_MISSING_DEADBAND))
+			pidff_set(&pidff->set_condition[PID_DEADBAND],
+				effect->u.condition[i].deadband);
+
 		hid_hw_request(pidff->hid, pidff->reports[PID_SET_CONDITION],
 			       HID_REQ_SET_REPORT);
 	}
@@ -1053,6 +1064,11 @@ static int pidff_find_field_with_usage(int *usage_index,
 	return -1;
 }
 
+#define PIDFF_MISSING_FIELD(name, quirks) \
+	({ pr_debug("%s field not found, but that's OK\n", __stringify(name)); \
+	   pr_debug("Setting MISSING_%s quirk\n", __stringify(name)); \
+	   *quirks |= HID_PIDFF_QUIRK_MISSING_ ## name; })
+
 /*
  * Find fields from a report and fill a pidff_usage
  */
@@ -1060,9 +1076,6 @@ static int pidff_find_fields(struct pidff_usage *usage, const u8 *table,
 			     struct hid_report *report, int count, int strict,
 			     u32 *quirks)
 {
-	const u8 block_offset = pidff_set_condition[PID_PARAM_BLOCK_OFFSET];
-	const u8 delay = pidff_set_effect[PID_START_DELAY];
-
 	if (!report) {
 		pr_debug("%s, null report\n", __func__);
 		return -1;
@@ -1080,17 +1093,23 @@ static int pidff_find_fields(struct pidff_usage *usage, const u8 *table,
 			continue;
 		}
 
-		if (table[i] == delay) {
-			pr_debug("Delay field not found, but that's OK\n");
-			pr_debug("Setting MISSING_DELAY quirk\n");
-			*quirks |= HID_PIDFF_QUIRK_MISSING_DELAY;
+		/* Field quirks auto-detection */
+		if (table[i] == pidff_set_effect[PID_START_DELAY])
+			PIDFF_MISSING_FIELD(DELAY, quirks);
+
+		else if (table[i] == pidff_set_condition[PID_PARAM_BLOCK_OFFSET])
+			PIDFF_MISSING_FIELD(PBO, quirks);
+
+		else if (table[i] == pidff_set_condition[PID_NEG_COEFFICIENT])
+			PIDFF_MISSING_FIELD(NEG_COEFFICIENT, quirks);
+
+		else if (table[i] == pidff_set_condition[PID_NEG_SATURATION])
+			PIDFF_MISSING_FIELD(NEG_SATURATION, quirks);
 
-		} else if (table[i] == block_offset) {
-			pr_debug("PBO field not found, but that's OK\n");
-			pr_debug("Setting MISSING_PBO quirk\n");
-			*quirks |= HID_PIDFF_QUIRK_MISSING_PBO;
+		else if (table[i] == pidff_set_condition[PID_DEADBAND])
+			PIDFF_MISSING_FIELD(DEADBAND, quirks);
 
-		} else if (strict) {
+		else if (strict) {
 			pr_debug("failed to locate %d\n", i);
 			return -1;
 		}
diff --git a/drivers/hid/usbhid/hid-pidff.h b/drivers/hid/usbhid/hid-pidff.h
index f321f675e1318..c413aa732842f 100644
--- a/drivers/hid/usbhid/hid-pidff.h
+++ b/drivers/hid/usbhid/hid-pidff.h
@@ -21,6 +21,15 @@
 /* Force all periodic effects to be uploaded as SINE */
 #define HID_PIDFF_QUIRK_PERIODIC_SINE_ONLY	BIT(4)
 
+/* Allow devices with missing negative coefficient in the set condition usage */
+#define HID_PIDFF_QUIRK_MISSING_NEG_COEFFICIENT	BIT(5)
+
+/* Allow devices with missing negative saturation in the set condition usage */
+#define HID_PIDFF_QUIRK_MISSING_NEG_SATURATION	BIT(6)
+
+/* Allow devices with missing deadband in the set condition usage */
+#define HID_PIDFF_QUIRK_MISSING_DEADBAND	BIT(7)
+
 #ifdef CONFIG_HID_PID
 int hid_pidff_init(struct hid_device *hid);
 int hid_pidff_init_with_quirks(struct hid_device *hid, u32 initial_quirks);
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 9b2c710f8da18..da1f0ea85625d 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -1208,10 +1208,20 @@ static int wacom_intuos_bt_irq(struct wacom_wac *wacom, size_t len)
 
 	switch (data[0]) {
 	case 0x04:
+		if (len < 32) {
+			dev_warn(wacom->pen_input->dev.parent,
+				 "Report 0x04 too short: %zu bytes\n", len);
+			break;
+		}
 		wacom_intuos_bt_process_data(wacom, data + i);
 		i += 10;
 		fallthrough;
 	case 0x03:
+		if (i == 1 && len < 22) {
+			dev_warn(wacom->pen_input->dev.parent,
+				 "Report 0x03 too short: %zu bytes\n", len);
+			break;
+		}
 		wacom_intuos_bt_process_data(wacom, data + i);
 		i += 10;
 		wacom_intuos_bt_process_data(wacom, data + i);
diff --git a/drivers/hsi/hsi_core.c b/drivers/hsi/hsi_core.c
index 7cb2dcb30fdb0..754949f5ebd62 100644
--- a/drivers/hsi/hsi_core.c
+++ b/drivers/hsi/hsi_core.c
@@ -342,7 +342,6 @@ static void hsi_controller_release(struct device *dev)
 {
 	struct hsi_controller *hsi = to_hsi_controller(dev);
 
-	kfree(hsi->port);
 	kfree(hsi);
 }
 
@@ -446,7 +445,7 @@ void hsi_put_controller(struct hsi_controller *hsi)
 		return;
 
 	for (i = 0; i < hsi->num_ports; i++)
-		if (hsi->port && hsi->port[i])
+		if (hsi->port[i])
 			put_device(&hsi->port[i]->device);
 	put_device(&hsi->device);
 }
@@ -462,39 +461,33 @@ EXPORT_SYMBOL_GPL(hsi_put_controller);
 struct hsi_controller *hsi_alloc_controller(unsigned int n_ports, gfp_t flags)
 {
 	struct hsi_controller	*hsi;
-	struct hsi_port		**port;
 	unsigned int		i;
 
 	if (!n_ports)
 		return NULL;
 
-	hsi = kzalloc_obj(*hsi, flags);
+	hsi = kzalloc_flex(*hsi, port, n_ports, flags);
 	if (!hsi)
 		return NULL;
-	port = kzalloc_objs(*port, n_ports, flags);
-	if (!port) {
-		kfree(hsi);
-		return NULL;
-	}
+
 	hsi->num_ports = n_ports;
-	hsi->port = port;
 	hsi->device.release = hsi_controller_release;
 	device_initialize(&hsi->device);
 
 	for (i = 0; i < n_ports; i++) {
-		port[i] = kzalloc_obj(**port, flags);
-		if (port[i] == NULL)
+		hsi->port[i] = kzalloc_obj(**hsi->port, flags);
+		if (hsi->port[i] == NULL)
 			goto out;
-		port[i]->num = i;
-		port[i]->async = hsi_dummy_msg;
-		port[i]->setup = hsi_dummy_cl;
-		port[i]->flush = hsi_dummy_cl;
-		port[i]->start_tx = hsi_dummy_cl;
-		port[i]->stop_tx = hsi_dummy_cl;
-		port[i]->release = hsi_dummy_cl;
-		mutex_init(&port[i]->lock);
-		BLOCKING_INIT_NOTIFIER_HEAD(&port[i]->n_head);
-		dev_set_name(&port[i]->device, "port%d", i);
+		hsi->port[i]->num = i;
+		hsi->port[i]->async = hsi_dummy_msg;
+		hsi->port[i]->setup = hsi_dummy_cl;
+		hsi->port[i]->flush = hsi_dummy_cl;
+		hsi->port[i]->start_tx = hsi_dummy_cl;
+		hsi->port[i]->stop_tx = hsi_dummy_cl;
+		hsi->port[i]->release = hsi_dummy_cl;
+		mutex_init(&hsi->port[i]->lock);
+		BLOCKING_INIT_NOTIFIER_HEAD(&hsi->port[i]->n_head);
+		dev_set_name(&hsi->port[i]->device, "port%d", i);
 		hsi->port[i]->device.release = hsi_port_release;
 		device_initialize(&hsi->port[i]->device);
 	}
diff --git a/drivers/hte/Kconfig b/drivers/hte/Kconfig
index 641af722b555d..f57bad67deef0 100644
--- a/drivers/hte/Kconfig
+++ b/drivers/hte/Kconfig
@@ -16,13 +16,13 @@ if HTE
 
 config HTE_TEGRA194
 	tristate "NVIDIA Tegra194 HTE Support"
-	depends on (ARCH_TEGRA_194_SOC || COMPILE_TEST)
+	depends on (ARCH_TEGRA || COMPILE_TEST)
 	depends on GPIOLIB
 	help
 	  Enable this option for integrated hardware timestamping engine also
 	  known as generic timestamping engine (GTE) support on NVIDIA Tegra194
-	  systems-on-chip. The driver supports 352 LIC IRQs and 39 AON GPIOs
-	  lines for timestamping in realtime.
+	  and later systems-on-chip. The driver supports 352 LIC IRQs and 39
+	  AON GPIOs lines for timestamping in realtime.
 
 config HTE_TEGRA194_TEST
         tristate "NVIDIA Tegra194 HTE Test"
diff --git a/drivers/hte/hte.c b/drivers/hte/hte.c
index fd0f29038a5f0..a423503482132 100644
--- a/drivers/hte/hte.c
+++ b/drivers/hte/hte.c
@@ -826,7 +826,7 @@ int hte_push_ts_ns(const struct hte_chip *chip, u32 xlated_id,
 
 	ret = ei->cb(data, ei->cl_data);
 	if (ret == HTE_RUN_SECOND_CB && ei->tcb) {
-		queue_work(system_unbound_wq, &ei->cb_work);
+		queue_work(system_dfl_wq, &ei->cb_work);
 		set_bit(HTE_TS_QUEUE_WK, &ei->flags);
 	}
 
diff --git a/drivers/hv/Makefile b/drivers/hv/Makefile
index 2593711c36283..888a748cc7cb9 100644
--- a/drivers/hv/Makefile
+++ b/drivers/hv/Makefile
@@ -16,6 +16,7 @@ hv_utils-y := hv_util.o hv_kvp.o hv_snapshot.o hv_utils_transport.o
 mshv_root-y := mshv_root_main.o mshv_synic.o mshv_eventfd.o mshv_irq.o \
 	       mshv_root_hv_call.o mshv_portid_table.o mshv_regions.o
 mshv_root-$(CONFIG_DEBUG_FS) += mshv_debugfs.o
+mshv_root-$(CONFIG_TRACEPOINTS) += mshv_trace.o
 mshv_vtl-y := mshv_vtl_main.o
 
 # Code that must be built-in
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index 7c77ada12b2e9..84eb0a6a0b546 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -384,8 +384,18 @@ static void free_channel(struct vmbus_channel *channel)
 
 void vmbus_channel_map_relid(struct vmbus_channel *channel)
 {
-	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
+	u32 new_relid = channel->offermsg.child_relid;
+
+	if (WARN_ON(new_relid >= MAX_CHANNEL_RELIDS))
 		return;
+
+	/*
+	 * This function is always called in the tasklet for the connect CPU.
+	 * So updating the relid hiwater mark does not need to be atomic.
+	 */
+	if (new_relid > READ_ONCE(vmbus_connection.relid_hiwater))
+		WRITE_ONCE(vmbus_connection.relid_hiwater, new_relid);
+
 	/*
 	 * The mapping of the channel's relid is visible from the CPUs that
 	 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
@@ -411,9 +421,7 @@ void vmbus_channel_map_relid(struct vmbus_channel *channel)
 	 *      of the VMBus driver and vmbus_chan_sched() can not run before
 	 *      vmbus_bus_resume() has completed execution (cf. resume_noirq).
 	 */
-	virt_store_mb(
-		vmbus_connection.channels[channel->offermsg.child_relid],
-		channel);
+	virt_store_mb(vmbus_connection.channels[new_relid], channel);
 }
 
 void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index a848400a59a2d..9a55f5c433073 100644
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -1663,7 +1663,7 @@ static void enable_page_reporting(void)
 	 * We let the page_reporting_order parameter decide the order
 	 * in the page_reporting code
 	 */
-	dm_device.pr_dev_info.order = 0;
+	dm_device.pr_dev_info.order = PAGE_REPORTING_ORDER_UNSPECIFIED;
 	ret = page_reporting_register(&dm_device.pr_dev_info);
 	if (ret < 0) {
 		dm_device.pr_dev_info.report = NULL;
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 7bd8f8486e858..05a36854389af 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -276,8 +276,9 @@ struct vmbus_connection {
 	struct list_head chn_list;
 	struct mutex channel_mutex;
 
-	/* Array of channels */
+	/* Array of channel pointers, indexed by relid */
 	struct vmbus_channel **channels;
+	u32 relid_hiwater;
 
 	/*
 	 * An offer message is handled first on the work_queue, and then
@@ -545,8 +546,8 @@ static inline int hv_debug_add_dev_dir(struct hv_device *dev)
 
 /* Create and remove sysfs entry for memory mapped ring buffers for a channel */
 int hv_create_ring_sysfs(struct vmbus_channel *channel,
-			 int (*hv_mmap_ring_buffer)(struct vmbus_channel *channel,
-						    struct vm_area_struct *vma));
+			 int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel,
+							    struct vm_area_desc *desc));
 int hv_remove_ring_sysfs(struct vmbus_channel *channel);
 
 #endif /* _HYPERV_VMBUS_H */
diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c
index d8471546e6a0c..90959f639dc32 100644
--- a/drivers/hv/mshv_eventfd.c
+++ b/drivers/hv/mshv_eventfd.c
@@ -733,6 +733,14 @@ static int mshv_assign_ioeventfd(struct mshv_partition *pt,
 	ret = mshv_register_doorbell(pt->pt_id, ioeventfd_mmio_write,
 				     (void *)pt, p->iovntfd_addr,
 				     p->iovntfd_datamatch, doorbell_flags);
+
+	trace_mshv_assign_ioeventfd(pt->pt_id, p->iovntfd_addr,
+				    p->iovntfd_length,
+				    p->iovntfd_datamatch,
+				    p->iovntfd_wildcard,
+				    p->iovntfd_eventfd,
+				    ret);
+
 	if (ret < 0)
 		goto unlock_fail;
 
@@ -780,6 +788,12 @@ static int mshv_deassign_ioeventfd(struct mshv_partition *pt,
 		    p->iovntfd_datamatch != args->datamatch)
 			continue;
 
+		trace_mshv_deassign_ioeventfd(pt->pt_id, p->iovntfd_addr,
+					      p->iovntfd_length,
+					      p->iovntfd_datamatch,
+					      p->iovntfd_wildcard,
+					      p->iovntfd_eventfd);
+
 		hlist_del_rcu(&p->iovntfd_hnode);
 		synchronize_rcu();
 		ioeventfd_release(p, pt->pt_id);
diff --git a/drivers/hv/mshv_irq.c b/drivers/hv/mshv_irq.c
index 02c56fc3a498c..b3142c84dcbc2 100644
--- a/drivers/hv/mshv_irq.c
+++ b/drivers/hv/mshv_irq.c
@@ -71,6 +71,10 @@ int mshv_update_routing_table(struct mshv_partition *partition,
 	mutex_unlock(&partition->pt_irq_lock);
 
 	synchronize_srcu_expedited(&partition->pt_irq_srcu);
+
+	trace_mshv_update_routing_table(partition->pt_id,
+					old, new, numents);
+
 	new = old;
 
 out:
diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c
index c28aac0726deb..fdffd4f002f6f 100644
--- a/drivers/hv/mshv_regions.c
+++ b/drivers/hv/mshv_regions.c
@@ -314,15 +314,17 @@ int mshv_region_pin(struct mshv_mem_region *region)
 		ret = pin_user_pages_fast(userspace_addr, nr_pages,
 					  FOLL_WRITE | FOLL_LONGTERM,
 					  pages);
-		if (ret < 0)
+		if (ret != nr_pages)
 			goto release_pages;
 	}
 
 	return 0;
 
 release_pages:
+	if (ret > 0)
+		done_count += ret;
 	mshv_region_invalidate_pages(region, 0, done_count);
-	return ret;
+	return ret < 0 ? ret : -ENOMEM;
 }
 
 static int mshv_region_chunk_unmap(struct mshv_mem_region *region,
diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h
index 04c2a1910a8af..1f086dcb7aa1a 100644
--- a/drivers/hv/mshv_root.h
+++ b/drivers/hv/mshv_root.h
@@ -17,6 +17,7 @@
 #include <linux/build_bug.h>
 #include <linux/mmu_notifier.h>
 #include <uapi/linux/mshv.h>
+#include "mshv_trace.h"
 
 /*
  * Hypervisor must be between these version numbers (inclusive)
@@ -190,7 +191,6 @@ struct hv_synic_pages {
 };
 
 struct mshv_root {
-	struct hv_synic_pages __percpu *synic_pages;
 	spinlock_t pt_ht_lock;
 	DECLARE_HASHTABLE(pt_htable, MSHV_PARTITIONS_HASH_BITS);
 	struct hv_partition_property_vmm_capabilities vmm_caps;
@@ -249,8 +249,8 @@ int mshv_register_doorbell(u64 partition_id, doorbell_cb_t doorbell_cb,
 void mshv_unregister_doorbell(u64 partition_id, int doorbell_portid);
 
 void mshv_isr(void);
-int mshv_synic_init(unsigned int cpu);
-int mshv_synic_cleanup(unsigned int cpu);
+int mshv_synic_init(struct device *dev);
+void mshv_synic_exit(void);
 
 static inline bool mshv_partition_encrypted(struct mshv_partition *partition)
 {
diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c
index 7f91096f95a8e..cb55d4d4be2e5 100644
--- a/drivers/hv/mshv_root_hv_call.c
+++ b/drivers/hv/mshv_root_hv_call.c
@@ -45,8 +45,7 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
 	struct hv_output_withdraw_memory *output_page;
 	struct page *page;
 	u16 completed;
-	unsigned long remaining = count;
-	u64 status;
+	u64 status, withdrawn = 0;
 	int i;
 	unsigned long flags;
 
@@ -55,7 +54,7 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
 		return -ENOMEM;
 	output_page = page_address(page);
 
-	while (remaining) {
+	while (withdrawn < count) {
 		local_irq_save(flags);
 
 		input_page = *this_cpu_ptr(hyperv_pcpu_input_arg);
@@ -63,7 +62,7 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
 		memset(input_page, 0, sizeof(*input_page));
 		input_page->partition_id = partition_id;
 		status = hv_do_rep_hypercall(HVCALL_WITHDRAW_MEMORY,
-					     min(remaining, HV_WITHDRAW_BATCH_SIZE),
+					     min(count - withdrawn, HV_WITHDRAW_BATCH_SIZE),
 					     0, input_page, output_page);
 
 		local_irq_restore(flags);
@@ -79,10 +78,12 @@ int hv_call_withdraw_memory(u64 count, int node, u64 partition_id)
 			break;
 		}
 
-		remaining -= completed;
+		withdrawn += completed;
 	}
 	free_page((unsigned long)output_page);
 
+	trace_mshv_hvcall_withdraw_memory(partition_id, withdrawn, status);
+
 	return hv_result_to_errno(status);
 }
 
@@ -126,6 +127,8 @@ int hv_call_create_partition(u64 flags,
 		ret = hv_deposit_memory(hv_current_partition_id, status);
 	} while (!ret);
 
+	trace_mshv_hvcall_create_partition(flags, ret ? ret : *partition_id);
+
 	return ret;
 }
 
@@ -153,6 +156,8 @@ int hv_call_initialize_partition(u64 partition_id)
 		ret = hv_deposit_memory(partition_id, status);
 	} while (!ret);
 
+	trace_mshv_hvcall_initialize_partition(partition_id, status);
+
 	return ret;
 }
 
@@ -165,6 +170,8 @@ int hv_call_finalize_partition(u64 partition_id)
 	status = hv_do_fast_hypercall8(HVCALL_FINALIZE_PARTITION,
 				       *(u64 *)&input);
 
+	trace_mshv_hvcall_finalize_partition(partition_id, status);
+
 	return hv_result_to_errno(status);
 }
 
@@ -176,6 +183,8 @@ int hv_call_delete_partition(u64 partition_id)
 	input.partition_id = partition_id;
 	status = hv_do_fast_hypercall8(HVCALL_DELETE_PARTITION, *(u64 *)&input);
 
+	trace_mshv_hvcall_delete_partition(partition_id, status);
+
 	return hv_result_to_errno(status);
 }
 
@@ -573,6 +582,9 @@ static int hv_call_map_vp_state_page(u64 partition_id, u32 vp_index, u32 type,
 		ret = hv_deposit_memory(partition_id, status);
 	} while (!ret);
 
+	trace_mshv_hvcall_map_vp_state_page(partition_id, vp_index,
+					    type, status);
+
 	return ret;
 }
 
diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c
index 82ff823ef0ca9..c29be7197a379 100644
--- a/drivers/hv/mshv_root_main.c
+++ b/drivers/hv/mshv_root_main.c
@@ -120,7 +120,6 @@ static u16 mshv_passthru_hvcalls[] = {
 	HVCALL_SET_VP_REGISTERS,
 	HVCALL_TRANSLATE_VIRTUAL_ADDRESS,
 	HVCALL_CLEAR_VIRTUAL_INTERRUPT,
-	HVCALL_SCRUB_PARTITION,
 	HVCALL_REGISTER_INTERCEPT_RESULT,
 	HVCALL_ASSERT_VIRTUAL_INTERRUPT,
 	HVCALL_GET_GPA_PAGES_ACCESS_STATES,
@@ -430,6 +429,17 @@ mshv_vp_dispatch(struct mshv_vp *vp, u32 flags,
 	status = hv_do_hypercall(HVCALL_DISPATCH_VP, input, output);
 	vp->run.flags.root_sched_dispatched = 0;
 
+	trace_mshv_hvcall_dispatch_vp(vp->vp_partition->pt_id,
+				      vp->vp_index, flags,
+				      output->dispatch_state,
+				      output->dispatch_event,
+#if defined(CONFIG_X86_64)
+				      vp->vp_register_page->interrupt_vectors.as_uint64,
+#else
+				      0,
+#endif
+				      status);
+
 	*res = *output;
 	preempt_enable();
 
@@ -452,6 +462,9 @@ mshv_vp_clear_explicit_suspend(struct mshv_vp *vp)
 	ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
 				    1, &explicit_suspend);
 
+	trace_mshv_vp_clear_explicit_suspend(vp->vp_partition->pt_id,
+					     vp->vp_index, ret);
+
 	if (ret)
 		vp_err(vp, "Failed to unsuspend\n");
 
@@ -494,6 +507,12 @@ mshv_vp_wait_for_hv_kick(struct mshv_vp *vp)
 	if (ret)
 		return -EINTR;
 
+	trace_mshv_vp_wait_for_hv_kick(vp->vp_partition->pt_id,
+				       vp->vp_index,
+				       vp->run.kicked_by_hv,
+				       mshv_vp_dispatch_thread_blocked(vp),
+				       mshv_vp_interrupt_pending(vp));
+
 	vp->run.flags.root_sched_blocked = 0;
 	vp->run.kicked_by_hv = 0;
 
@@ -522,6 +541,12 @@ static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp)
 
 		if (__xfer_to_guest_mode_work_pending()) {
 			ret = xfer_to_guest_mode_handle_work();
+
+			trace_mshv_xfer_to_guest_mode_work(vp->vp_partition->pt_id,
+							   vp->vp_index,
+							   read_thread_flags(),
+							   ret);
+
 			if (ret)
 				break;
 		}
@@ -673,6 +698,8 @@ static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg)
 {
 	long rc;
 
+	trace_mshv_run_vp_entry(vp->vp_partition->pt_id, vp->vp_index);
+
 	do {
 		if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
 			rc = mshv_run_vp_with_root_scheduler(vp);
@@ -680,6 +707,10 @@ static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg)
 			rc = mshv_run_vp_with_hyp_scheduler(vp);
 	} while (rc == 0 && mshv_vp_handle_intercept(vp));
 
+	trace_mshv_run_vp_exit(vp->vp_partition->pt_id, vp->vp_index,
+			       vp->vp_intercept_msg_page->header.message_type,
+			       rc);
+
 	if (rc)
 		return rc;
 
@@ -941,6 +972,8 @@ mshv_vp_release(struct inode *inode, struct file *filp)
 {
 	struct mshv_vp *vp = filp->private_data;
 
+	trace_mshv_vp_release(vp->vp_partition->pt_id, vp->vp_index);
+
 	/* Rest of VP cleanup happens in destroy_partition() */
 	mshv_partition_put(vp->vp_partition);
 	return 0;
@@ -1113,7 +1146,7 @@ mshv_partition_ioctl_create_vp(struct mshv_partition *partition,
 	partition->pt_vp_count++;
 	partition->pt_vp_array[args.vp_index] = vp;
 
-	return ret;
+	goto out;
 
 remove_debugfs_vp:
 	mshv_debugfs_vp_remove(vp);
@@ -1139,6 +1172,8 @@ mshv_partition_ioctl_create_vp(struct mshv_partition *partition,
 			       intercept_msg_page, input_vtl_zero);
 destroy_vp:
 	hv_call_delete_vp(partition->pt_id, args.vp_index);
+out:
+	trace_mshv_create_vp(partition->pt_id, args.vp_index, ret);
 	return ret;
 }
 
@@ -1289,7 +1324,7 @@ static int mshv_prepare_pinned_region(struct mshv_mem_region *region)
  */
 static long
 mshv_map_user_memory(struct mshv_partition *partition,
-		     struct mshv_user_mem_region mem)
+		     struct mshv_user_mem_region *mem)
 {
 	struct mshv_mem_region *region;
 	struct vm_area_struct *vma;
@@ -1297,12 +1332,12 @@ mshv_map_user_memory(struct mshv_partition *partition,
 	ulong mmio_pfn;
 	long ret;
 
-	if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP) ||
-	    !access_ok((const void __user *)mem.userspace_addr, mem.size))
+	if (mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP) ||
+	    !access_ok((const void __user *)mem->userspace_addr, mem->size))
 		return -EINVAL;
 
 	mmap_read_lock(current->mm);
-	vma = vma_lookup(current->mm, mem.userspace_addr);
+	vma = vma_lookup(current->mm, mem->userspace_addr);
 	is_mmio = vma ? !!(vma->vm_flags & (VM_IO | VM_PFNMAP)) : 0;
 	mmio_pfn = is_mmio ? vma->vm_pgoff : 0;
 	mmap_read_unlock(current->mm);
@@ -1310,7 +1345,7 @@ mshv_map_user_memory(struct mshv_partition *partition,
 	if (!vma)
 		return -EINVAL;
 
-	ret = mshv_partition_create_region(partition, &mem, &region,
+	ret = mshv_partition_create_region(partition, mem, &region,
 					   is_mmio);
 	if (ret)
 		return ret;
@@ -1338,6 +1373,10 @@ mshv_map_user_memory(struct mshv_partition *partition,
 		break;
 	}
 
+	trace_mshv_map_user_memory(partition->pt_id, region->start_uaddr,
+				   region->start_gfn, region->nr_pages,
+				   region->hv_map_flags, ret);
+
 	if (ret)
 		goto errout;
 
@@ -1348,32 +1387,32 @@ mshv_map_user_memory(struct mshv_partition *partition,
 	return 0;
 
 errout:
-	vfree(region);
+	mshv_region_put(region);
 	return ret;
 }
 
 /* Called for unmapping both the guest ram and the mmio space */
 static long
 mshv_unmap_user_memory(struct mshv_partition *partition,
-		       struct mshv_user_mem_region mem)
+		       struct mshv_user_mem_region *mem)
 {
 	struct mshv_mem_region *region;
 
-	if (!(mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP)))
+	if (!(mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP)))
 		return -EINVAL;
 
 	spin_lock(&partition->pt_mem_regions_lock);
 
-	region = mshv_partition_region_by_gfn(partition, mem.guest_pfn);
+	region = mshv_partition_region_by_gfn(partition, mem->guest_pfn);
 	if (!region) {
 		spin_unlock(&partition->pt_mem_regions_lock);
 		return -ENOENT;
 	}
 
 	/* Paranoia check */
-	if (region->start_uaddr != mem.userspace_addr ||
-	    region->start_gfn != mem.guest_pfn ||
-	    region->nr_pages != HVPFN_DOWN(mem.size)) {
+	if (region->start_uaddr != mem->userspace_addr ||
+	    region->start_gfn != mem->guest_pfn ||
+	    region->nr_pages != HVPFN_DOWN(mem->size)) {
 		spin_unlock(&partition->pt_mem_regions_lock);
 		return -EINVAL;
 	}
@@ -1404,9 +1443,9 @@ mshv_partition_ioctl_set_memory(struct mshv_partition *partition,
 		return -EINVAL;
 
 	if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP))
-		return mshv_unmap_user_memory(partition, mem);
+		return mshv_unmap_user_memory(partition, &mem);
 
-	return mshv_map_user_memory(partition, mem);
+	return mshv_map_user_memory(partition, &mem);
 }
 
 static long
@@ -1633,6 +1672,9 @@ disable_vp_dispatch(struct mshv_vp *vp)
 	if (ret)
 		vp_err(vp, "failed to suspend\n");
 
+	trace_mshv_disable_vp_dispatch(vp->vp_partition->pt_id,
+				       vp->vp_index, ret);
+
 	return ret;
 }
 
@@ -1681,6 +1723,8 @@ drain_vp_signals(struct mshv_vp *vp)
 		vp->run.kicked_by_hv = 0;
 		vp_signal_count = atomic64_read(&vp->run.vp_signaled_count);
 	}
+
+	trace_mshv_drain_vp_signals(vp->vp_partition->pt_id, vp->vp_index);
 }
 
 static void drain_all_vps(const struct mshv_partition *partition)
@@ -1734,6 +1778,8 @@ static void destroy_partition(struct mshv_partition *partition)
 		return;
 	}
 
+	trace_mshv_destroy_partition(partition->pt_id);
+
 	if (partition->pt_initialized) {
 		/*
 		 * We only need to drain signals for root scheduler. This should be
@@ -1840,6 +1886,8 @@ mshv_partition_release(struct inode *inode, struct file *filp)
 {
 	struct mshv_partition *partition = filp->private_data;
 
+	trace_mshv_partition_release(partition->pt_id);
+
 	mshv_eventfd_release(partition);
 
 	cleanup_srcu_struct(&partition->pt_irq_srcu);
@@ -1969,6 +2017,7 @@ mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev)
 	struct hv_partition_creation_properties creation_properties;
 	union hv_partition_isolation_properties isolation_properties;
 	struct mshv_partition *partition;
+	u64 pt_id = -1;
 	long ret;
 
 	ret = mshv_ioctl_process_pt_flags(user_arg, &creation_flags,
@@ -2008,22 +2057,29 @@ mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev)
 	ret = hv_call_create_partition(creation_flags,
 				       creation_properties,
 				       isolation_properties,
-				       &partition->pt_id);
+				       &pt_id);
 	if (ret)
 		goto cleanup_irq_srcu;
 
+	partition->pt_id = pt_id;
+
 	ret = add_partition(partition);
 	if (ret)
 		goto delete_partition;
 
 	ret = mshv_init_async_handler(partition);
-	if (!ret) {
-		ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition",
-							   &mshv_partition_fops,
-							   partition, O_RDWR));
-		if (ret >= 0)
-			return ret;
-	}
+	if (ret)
+		goto remove_partition;
+
+	ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition",
+						   &mshv_partition_fops,
+						   partition, O_RDWR));
+	if (ret < 0)
+		goto remove_partition;
+
+	goto out;
+
+remove_partition:
 	remove_partition(partition);
 delete_partition:
 	hv_call_delete_partition(partition->pt_id);
@@ -2031,7 +2087,8 @@ mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev)
 	cleanup_srcu_struct(&partition->pt_irq_srcu);
 free_partition:
 	kfree(partition);
-
+out:
+	trace_mshv_create_partition(pt_id, ret);
 	return ret;
 }
 
@@ -2064,7 +2121,6 @@ mshv_dev_release(struct inode *inode, struct file *filp)
 	return 0;
 }
 
-static int mshv_cpuhp_online;
 static int mshv_root_sched_online;
 
 static const char *scheduler_type_to_string(enum hv_scheduler_type type)
@@ -2249,27 +2305,6 @@ root_scheduler_deinit(void)
 	free_percpu(root_scheduler_output);
 }
 
-static int mshv_reboot_notify(struct notifier_block *nb,
-			      unsigned long code, void *unused)
-{
-	cpuhp_remove_state(mshv_cpuhp_online);
-	return 0;
-}
-
-struct notifier_block mshv_reboot_nb = {
-	.notifier_call = mshv_reboot_notify,
-};
-
-static void mshv_root_partition_exit(void)
-{
-	unregister_reboot_notifier(&mshv_reboot_nb);
-}
-
-static int __init mshv_root_partition_init(struct device *dev)
-{
-	return register_reboot_notifier(&mshv_reboot_nb);
-}
-
 static int __init mshv_init_vmm_caps(struct device *dev)
 {
 	int ret;
@@ -2314,39 +2349,21 @@ static int __init mshv_parent_partition_init(void)
 			MSHV_HV_MAX_VERSION);
 	}
 
-	mshv_root.synic_pages = alloc_percpu(struct hv_synic_pages);
-	if (!mshv_root.synic_pages) {
-		dev_err(dev, "Failed to allocate percpu synic page\n");
-		ret = -ENOMEM;
+	ret = mshv_synic_init(dev);
+	if (ret)
 		goto device_deregister;
-	}
-
-	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_synic",
-				mshv_synic_init,
-				mshv_synic_cleanup);
-	if (ret < 0) {
-		dev_err(dev, "Failed to setup cpu hotplug state: %i\n", ret);
-		goto free_synic_pages;
-	}
-
-	mshv_cpuhp_online = ret;
 
 	ret = mshv_init_vmm_caps(dev);
 	if (ret)
-		goto remove_cpu_state;
+		goto synic_cleanup;
 
 	ret = mshv_retrieve_scheduler_type(dev);
 	if (ret)
-		goto remove_cpu_state;
-
-	if (hv_root_partition())
-		ret = mshv_root_partition_init(dev);
-	if (ret)
-		goto remove_cpu_state;
+		goto synic_cleanup;
 
 	ret = root_scheduler_init(dev);
 	if (ret)
-		goto exit_partition;
+		goto synic_cleanup;
 
 	ret = mshv_debugfs_init();
 	if (ret)
@@ -2367,13 +2384,8 @@ static int __init mshv_parent_partition_init(void)
 	mshv_debugfs_exit();
 deinit_root_scheduler:
 	root_scheduler_deinit();
-exit_partition:
-	if (hv_root_partition())
-		mshv_root_partition_exit();
-remove_cpu_state:
-	cpuhp_remove_state(mshv_cpuhp_online);
-free_synic_pages:
-	free_percpu(mshv_root.synic_pages);
+synic_cleanup:
+	mshv_synic_exit();
 device_deregister:
 	misc_deregister(&mshv_dev);
 	return ret;
@@ -2387,10 +2399,7 @@ static void __exit mshv_parent_partition_exit(void)
 	misc_deregister(&mshv_dev);
 	mshv_irqfd_wq_cleanup();
 	root_scheduler_deinit();
-	if (hv_root_partition())
-		mshv_root_partition_exit();
-	cpuhp_remove_state(mshv_cpuhp_online);
-	free_percpu(mshv_root.synic_pages);
+	mshv_synic_exit();
 }
 
 module_init(mshv_parent_partition_init);
diff --git a/drivers/hv/mshv_synic.c b/drivers/hv/mshv_synic.c
index 216065e21d280..43f1bcbbf2d34 100644
--- a/drivers/hv/mshv_synic.c
+++ b/drivers/hv/mshv_synic.c
@@ -10,13 +10,22 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/random.h>
+#include <linux/cpuhotplug.h>
+#include <linux/reboot.h>
 #include <asm/mshyperv.h>
+#include <linux/acpi.h>
 
 #include "mshv_eventfd.h"
 #include "mshv.h"
 
+static int synic_cpuhp_online;
+static struct hv_synic_pages __percpu *synic_pages;
+static int mshv_sint_vector = -1; /* hwirq for the SynIC SINTs */
+static int mshv_sint_irq = -1; /* Linux IRQ for mshv_sint_vector */
+
 static u32 synic_event_ring_get_queued_port(u32 sint_index)
 {
 	struct hv_synic_event_ring_page **event_ring_page;
@@ -26,7 +35,7 @@ static u32 synic_event_ring_get_queued_port(u32 sint_index)
 	u32 message;
 	u8 tail;
 
-	spages = this_cpu_ptr(mshv_root.synic_pages);
+	spages = this_cpu_ptr(synic_pages);
 	event_ring_page = &spages->synic_event_ring_page;
 	synic_eventring_tail = (u8 **)this_cpu_ptr(hv_synic_eventring_tail);
 
@@ -393,7 +402,7 @@ mshv_intercept_isr(struct hv_message *msg)
 
 void mshv_isr(void)
 {
-	struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages);
+	struct hv_synic_pages *spages = this_cpu_ptr(synic_pages);
 	struct hv_message_page **msg_page = &spages->hyp_synic_message_page;
 	struct hv_message *msg;
 	bool handled;
@@ -437,25 +446,21 @@ void mshv_isr(void)
 		if (msg->header.message_flags.msg_pending)
 			hv_set_non_nested_msr(HV_MSR_EOM, 0);
 
-#ifdef HYPERVISOR_CALLBACK_VECTOR
-		add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR);
-#endif
+		add_interrupt_randomness(mshv_sint_vector);
 	} else {
 		pr_warn_once("%s: unknown message type 0x%x\n", __func__,
 			     msg->header.message_type);
 	}
 }
 
-int mshv_synic_init(unsigned int cpu)
+static int mshv_synic_cpu_init(unsigned int cpu)
 {
 	union hv_synic_simp simp;
 	union hv_synic_siefp siefp;
 	union hv_synic_sirbp sirbp;
-#ifdef HYPERVISOR_CALLBACK_VECTOR
 	union hv_synic_sint sint;
-#endif
 	union hv_synic_scontrol sctrl;
-	struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages);
+	struct hv_synic_pages *spages = this_cpu_ptr(synic_pages);
 	struct hv_message_page **msg_page = &spages->hyp_synic_message_page;
 	struct hv_synic_event_flags_page **event_flags_page =
 			&spages->synic_event_flags_page;
@@ -496,10 +501,12 @@ int mshv_synic_init(unsigned int cpu)
 
 	hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
 
-#ifdef HYPERVISOR_CALLBACK_VECTOR
+	if (mshv_sint_irq != -1)
+		enable_percpu_irq(mshv_sint_irq, 0);
+
 	/* Enable intercepts */
 	sint.as_uint64 = 0;
-	sint.vector = HYPERVISOR_CALLBACK_VECTOR;
+	sint.vector = mshv_sint_vector;
 	sint.masked = false;
 	sint.auto_eoi = hv_recommend_using_aeoi();
 	hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX,
@@ -507,13 +514,12 @@ int mshv_synic_init(unsigned int cpu)
 
 	/* Doorbell SINT */
 	sint.as_uint64 = 0;
-	sint.vector = HYPERVISOR_CALLBACK_VECTOR;
+	sint.vector = mshv_sint_vector;
 	sint.masked = false;
 	sint.as_intercept = 1;
 	sint.auto_eoi = hv_recommend_using_aeoi();
 	hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX,
 			      sint.as_uint64);
-#endif
 
 	/* Enable global synic bit */
 	sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL);
@@ -542,14 +548,14 @@ int mshv_synic_init(unsigned int cpu)
 	return -EFAULT;
 }
 
-int mshv_synic_cleanup(unsigned int cpu)
+static int mshv_synic_cpu_exit(unsigned int cpu)
 {
 	union hv_synic_sint sint;
 	union hv_synic_simp simp;
 	union hv_synic_siefp siefp;
 	union hv_synic_sirbp sirbp;
 	union hv_synic_scontrol sctrl;
-	struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages);
+	struct hv_synic_pages *spages = this_cpu_ptr(synic_pages);
 	struct hv_message_page **msg_page = &spages->hyp_synic_message_page;
 	struct hv_synic_event_flags_page **event_flags_page =
 		&spages->synic_event_flags_page;
@@ -568,6 +574,9 @@ int mshv_synic_cleanup(unsigned int cpu)
 	hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX,
 			      sint.as_uint64);
 
+	if (mshv_sint_irq != -1)
+		disable_percpu_irq(mshv_sint_irq);
+
 	/* Disable Synic's event ring page */
 	sirbp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIRBP);
 	sirbp.sirbp_enabled = false;
@@ -663,3 +672,152 @@ mshv_unregister_doorbell(u64 partition_id, int doorbell_portid)
 
 	mshv_portid_free(doorbell_portid);
 }
+
+static int mshv_synic_reboot_notify(struct notifier_block *nb,
+			      unsigned long code, void *unused)
+{
+	if (!hv_root_partition())
+		return 0;
+
+	cpuhp_remove_state(synic_cpuhp_online);
+	return 0;
+}
+
+static struct notifier_block mshv_synic_reboot_nb = {
+	.notifier_call = mshv_synic_reboot_notify,
+};
+
+#ifndef HYPERVISOR_CALLBACK_VECTOR
+static DEFINE_PER_CPU(long, mshv_evt);
+
+static irqreturn_t mshv_percpu_isr(int irq, void *dev_id)
+{
+	mshv_isr();
+	return IRQ_HANDLED;
+}
+
+#ifdef CONFIG_ACPI
+static int __init mshv_acpi_setup_sint_irq(void)
+{
+	return acpi_register_gsi(NULL, mshv_sint_vector, ACPI_EDGE_SENSITIVE,
+					ACPI_ACTIVE_HIGH);
+}
+
+static void mshv_acpi_cleanup_sint_irq(void)
+{
+	acpi_unregister_gsi(mshv_sint_vector);
+}
+#else
+static int __init mshv_acpi_setup_sint_irq(void)
+{
+	return -ENODEV;
+}
+
+static void mshv_acpi_cleanup_sint_irq(void)
+{
+}
+#endif
+
+static int __init mshv_sint_vector_setup(void)
+{
+	int ret;
+	struct hv_register_assoc reg = {
+		.name = HV_ARM64_REGISTER_SINT_RESERVED_INTERRUPT_ID,
+	};
+	union hv_input_vtl input_vtl = { 0 };
+
+	if (acpi_disabled)
+		return -ENODEV;
+
+	ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
+				1, input_vtl, &reg);
+	if (ret || !reg.value.reg64)
+		return -ENODEV;
+
+	mshv_sint_vector = reg.value.reg64;
+	ret = mshv_acpi_setup_sint_irq();
+	if (ret < 0) {
+		pr_err("Failed to setup IRQ for MSHV SINT vector %d: %d\n",
+			mshv_sint_vector, ret);
+		goto out_fail;
+	}
+
+	mshv_sint_irq = ret;
+
+	ret = request_percpu_irq(mshv_sint_irq, mshv_percpu_isr, "MSHV",
+		&mshv_evt);
+	if (ret)
+		goto out_unregister;
+
+	return 0;
+
+out_unregister:
+	mshv_acpi_cleanup_sint_irq();
+out_fail:
+	return ret;
+}
+
+static void mshv_sint_vector_cleanup(void)
+{
+	free_percpu_irq(mshv_sint_irq, &mshv_evt);
+	mshv_acpi_cleanup_sint_irq();
+}
+#else /* !HYPERVISOR_CALLBACK_VECTOR */
+static int __init mshv_sint_vector_setup(void)
+{
+	mshv_sint_vector = HYPERVISOR_CALLBACK_VECTOR;
+	return 0;
+}
+
+static void mshv_sint_vector_cleanup(void)
+{
+}
+#endif /* HYPERVISOR_CALLBACK_VECTOR */
+
+int __init mshv_synic_init(struct device *dev)
+{
+	int ret = 0;
+
+	ret = mshv_sint_vector_setup();
+	if (ret)
+		return ret;
+
+	synic_pages = alloc_percpu(struct hv_synic_pages);
+	if (!synic_pages) {
+		dev_err(dev, "Failed to allocate percpu synic page\n");
+		ret = -ENOMEM;
+		goto sint_vector_cleanup;
+	}
+
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_synic",
+				mshv_synic_cpu_init,
+				mshv_synic_cpu_exit);
+	if (ret < 0) {
+		dev_err(dev, "Failed to setup cpu hotplug state: %i\n", ret);
+		goto free_synic_pages;
+	}
+
+	synic_cpuhp_online = ret;
+
+	ret = register_reboot_notifier(&mshv_synic_reboot_nb);
+	if (ret)
+		goto remove_cpuhp_state;
+
+	return 0;
+
+remove_cpuhp_state:
+	cpuhp_remove_state(synic_cpuhp_online);
+free_synic_pages:
+	free_percpu(synic_pages);
+sint_vector_cleanup:
+	mshv_sint_vector_cleanup();
+	return ret;
+}
+
+void mshv_synic_exit(void)
+{
+	unregister_reboot_notifier(&mshv_synic_reboot_nb);
+	cpuhp_remove_state(synic_cpuhp_online);
+	free_percpu(synic_pages);
+	mshv_sint_vector_cleanup();
+}
diff --git a/drivers/hv/mshv_trace.c b/drivers/hv/mshv_trace.c
new file mode 100644
index 0000000000000..0936b2f95edd9
--- /dev/null
+++ b/drivers/hv/mshv_trace.c
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026, Microsoft Corporation.
+ *
+ * Tracepoint definitions for mshv driver.
+ */
+
+#define CREATE_TRACE_POINTS
+#include "mshv_trace.h"
diff --git a/drivers/hv/mshv_trace.h b/drivers/hv/mshv_trace.h
new file mode 100644
index 0000000000000..ba3b3f575983f
--- /dev/null
+++ b/drivers/hv/mshv_trace.h
@@ -0,0 +1,515 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026, Microsoft Corporation.
+ *
+ * Tracepoint declarations for mshv driver.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mshv
+
+#if !defined(__MSHV_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _MSHV_TRACE_H_
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/hv
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE mshv_trace
+
+TRACE_EVENT(mshv_create_partition,
+	    TP_PROTO(u64 partition_id, int vm_fd),
+	    TP_ARGS(partition_id, vm_fd),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(int, vm_fd)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vm_fd = vm_fd;
+	    ),
+	    TP_printk("partition_id=%llu vm_fd=%d",
+		    __entry->partition_id,
+		    __entry->vm_fd
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_create_partition,
+	    TP_PROTO(u64 flags, s64 partition_id),
+	    TP_ARGS(flags, partition_id),
+	    TP_STRUCT__entry(
+		    __field(u64, flags)
+		    __field(s64, partition_id)
+	    ),
+	    TP_fast_assign(
+		    __entry->flags = flags;
+		    __entry->partition_id = partition_id;
+	    ),
+	    TP_printk("flags=%#llx partition_id=%lld",
+		    __entry->flags,
+		    __entry->partition_id
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_initialize_partition,
+	    TP_PROTO(u64 partition_id, u64 status),
+	    TP_ARGS(partition_id, status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu status=%#llx",
+		    __entry->partition_id,
+		    __entry->status
+	    )
+);
+
+TRACE_EVENT(mshv_partition_release,
+	    TP_PROTO(u64 partition_id),
+	    TP_ARGS(partition_id),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+	    ),
+	    TP_printk("partition_id=%llu",
+		    __entry->partition_id
+	    )
+);
+
+TRACE_EVENT(mshv_destroy_partition,
+	    TP_PROTO(u64 partition_id),
+	    TP_ARGS(partition_id),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+	    ),
+	    TP_printk("partition_id=%llu",
+		    __entry->partition_id
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_finalize_partition,
+	    TP_PROTO(u64 partition_id, u64 status),
+	    TP_ARGS(partition_id, status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu status=%#llx ",
+		    __entry->partition_id,
+		    __entry->status
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_withdraw_memory,
+	    TP_PROTO(u64 partition_id, u64 withdrawn, u64 status),
+	    TP_ARGS(partition_id, withdrawn, status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, withdrawn)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->withdrawn = withdrawn;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu withdrawn=%llu status=%#llx",
+		    __entry->partition_id,
+		    __entry->withdrawn,
+		    __entry->status
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_delete_partition,
+	    TP_PROTO(u64 partition_id, u64 status),
+	    TP_ARGS(partition_id, status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu status=%#llx",
+		    __entry->partition_id,
+		    __entry->status
+	    )
+);
+
+TRACE_EVENT(mshv_create_vp,
+	    TP_PROTO(u64 partition_id, u32 vp_index, long vp_fd),
+	    TP_ARGS(partition_id, vp_index, vp_fd),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(long, vp_fd)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->vp_fd = vp_fd;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u vp_fd=%ld",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->vp_fd
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_map_vp_state_page,
+	    TP_PROTO(u64 partition_id, u32 vp_index, u32 page_type, u64 status),
+	    TP_ARGS(partition_id, vp_index, page_type, status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(u32, page_type)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->page_type = page_type;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u page_type=%u status=%#llx",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->page_type,
+		    __entry->status
+	    )
+);
+
+TRACE_EVENT(mshv_drain_vp_signals,
+	    TP_PROTO(u64 partition_id, u32 vp_index),
+	    TP_ARGS(partition_id, vp_index),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u",
+		    __entry->partition_id,
+		    __entry->vp_index
+	    )
+);
+
+TRACE_EVENT(mshv_disable_vp_dispatch,
+	    TP_PROTO(u64 partition_id, u32 vp_index, int ret),
+	    TP_ARGS(partition_id, vp_index, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(int, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u ret=%d",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->ret
+	    )
+);
+
+TRACE_EVENT(mshv_vp_release,
+	    TP_PROTO(u64 partition_id, u32 vp_index),
+	    TP_ARGS(partition_id, vp_index),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u",
+		    __entry->partition_id,
+		    __entry->vp_index
+	    )
+);
+
+TRACE_EVENT(mshv_run_vp_entry,
+	    TP_PROTO(u64 partition_id, u32 vp_index),
+	    TP_ARGS(partition_id, vp_index),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u",
+		    __entry->partition_id,
+		    __entry->vp_index
+	    )
+);
+
+TRACE_EVENT(mshv_run_vp_exit,
+	    TP_PROTO(u64 partition_id, u32 vp_index, u64 hv_message_type, long ret),
+	    TP_ARGS(partition_id, vp_index, hv_message_type, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(u64, hv_message_type)
+		    __field(long, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->hv_message_type = hv_message_type;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u hv_message_type=%#llx ret=%ld",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->hv_message_type,
+		    __entry->ret
+	    )
+);
+
+TRACE_EVENT(mshv_vp_clear_explicit_suspend,
+	    TP_PROTO(u64 partition_id, u32 vp_index, int ret),
+	    TP_ARGS(partition_id, vp_index, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(int, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u ret=%d",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->ret
+	    )
+);
+
+TRACE_EVENT(mshv_xfer_to_guest_mode_work,
+	    TP_PROTO(u64 partition_id, u32 vp_index, unsigned long thread_info_flag, long ret),
+	    TP_ARGS(partition_id, vp_index, thread_info_flag, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(unsigned long, thread_info_flag)
+		    __field(long, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->thread_info_flag = thread_info_flag;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u thread_info_flag=%#lx ret=%ld",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->thread_info_flag,
+		    __entry->ret
+	    )
+);
+
+TRACE_EVENT(mshv_hvcall_dispatch_vp,
+	    TP_PROTO(u64 partition_id, u32 vp_index, u32 flags,
+		     u32 dispatch_state, u32 dispatch_event, u64 irq_vectors, u64 status),
+	    TP_ARGS(partition_id, vp_index, flags, dispatch_state, dispatch_event, irq_vectors,
+		    status),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(u32, flags)
+		    __field(u32, dispatch_state)
+		    __field(u32, dispatch_event)
+		    __field(u64, irq_vectors)
+		    __field(u64, status)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->flags = flags;
+		    __entry->dispatch_state = dispatch_state;
+		    __entry->dispatch_event = dispatch_event;
+		    __entry->irq_vectors = irq_vectors;
+		    __entry->status = status;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u flags=%#x dispatch_state=%#x dispatch_event=%#x irq_vectors=%#016llx status=%#llx",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->flags,
+		    __entry->dispatch_state,
+		    __entry->dispatch_event,
+		    __entry->irq_vectors,
+		    __entry->status
+	     )
+);
+
+TRACE_EVENT(mshv_update_routing_table,
+	    TP_PROTO(u64 partition_id, void *old, void *new, u32 numents),
+	    TP_ARGS(partition_id, old, new, numents),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(struct mshv_girq_routing_table *, old)
+		    __field(struct mshv_girq_routing_table *, new)
+		    __field(u32, numents)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->old = old;
+		    __entry->new = new;
+		    __entry->numents = numents;
+	    ),
+	    TP_printk("partition_id=%llu old=%p new=%p numents=%u",
+		    __entry->partition_id,
+		    __entry->old,
+		    __entry->new,
+		    __entry->numents
+	    )
+);
+
+TRACE_EVENT(mshv_map_user_memory,
+	    TP_PROTO(u64 partition_id, u64 start_uaddr, u64 start_gfn, u64 nr_pages, u32 map_flags,
+		     long ret),
+	    TP_ARGS(partition_id, start_uaddr, start_gfn, nr_pages, map_flags, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, start_uaddr)
+		    __field(u64, start_gfn)
+		    __field(u64, nr_pages)
+		    __field(u32, map_flags)
+		    __field(long, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->start_uaddr = start_uaddr;
+		    __entry->start_gfn = start_gfn;
+		    __entry->nr_pages = nr_pages;
+		    __entry->map_flags = map_flags;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu start_uaddr=%#llx start_gfn=%#llx nr_pages=%llu map_flags=%#x ret=%ld",
+		    __entry->partition_id,
+		    __entry->start_uaddr,
+		    __entry->start_gfn,
+		    __entry->nr_pages,
+		    __entry->map_flags,
+		    __entry->ret
+	     )
+);
+
+TRACE_EVENT(mshv_assign_ioeventfd,
+	    TP_PROTO(u64 partition_id, u64 addr, u64 length, u64 datamatch, bool wildcard,
+		     void *eventfd, int ret),
+	    TP_ARGS(partition_id, addr, length, datamatch, wildcard, eventfd, ret),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, addr)
+		    __field(u64, length)
+		    __field(u64, datamatch)
+		    __field(bool, wildcard)
+		    __field(struct eventfd_ctx *, eventfd)
+		    __field(int, ret)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->addr = addr;
+		    __entry->length = length;
+		    __entry->datamatch = datamatch;
+		    __entry->wildcard = wildcard;
+		    __entry->eventfd = eventfd;
+		    __entry->ret = ret;
+	    ),
+	    TP_printk("partition_id=%llu addr=%#016llx length=%#llx datamatch=%#llx wildcard=%d eventfd=%p ret=%d",
+		    __entry->partition_id,
+		    __entry->addr,
+		    __entry->length,
+		    __entry->datamatch,
+		    __entry->wildcard,
+		    __entry->eventfd,
+		    __entry->ret
+	     )
+);
+
+TRACE_EVENT(mshv_deassign_ioeventfd,
+	    TP_PROTO(u64 partition_id, u64 addr, u64 length, u64 datamatch, bool wildcard,
+		     void *eventfd),
+	    TP_ARGS(partition_id, addr, length, datamatch, wildcard, eventfd),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u64, addr)
+		    __field(u64, length)
+		    __field(u64, datamatch)
+		    __field(bool, wildcard)
+		    __field(struct eventfd_ctx *, eventfd)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->addr = addr;
+		    __entry->length = length;
+		    __entry->datamatch = datamatch;
+		    __entry->wildcard = wildcard;
+		    __entry->eventfd = eventfd;
+	    ),
+	    TP_printk("partition_id=%llu addr=%#016llx length=%#llx datamatch=%#llx wildcard=%d eventfd=%p",
+		    __entry->partition_id,
+		    __entry->addr,
+		    __entry->length,
+		    __entry->datamatch,
+		    __entry->wildcard,
+		    __entry->eventfd
+	     )
+);
+
+TRACE_EVENT(mshv_vp_wait_for_hv_kick,
+	    TP_PROTO(u64 partition_id, u32 vp_index, bool kicked_by_hv, bool blocked,
+		     bool irq_pending),
+	    TP_ARGS(partition_id, vp_index, kicked_by_hv, blocked, irq_pending),
+	    TP_STRUCT__entry(
+		    __field(u64, partition_id)
+		    __field(u32, vp_index)
+		    __field(bool, kicked_by_hv)
+		    __field(bool, blocked)
+		    __field(bool, irq_pending)
+	    ),
+	    TP_fast_assign(
+		    __entry->partition_id = partition_id;
+		    __entry->vp_index = vp_index;
+		    __entry->kicked_by_hv = kicked_by_hv;
+		    __entry->blocked = blocked;
+		    __entry->irq_pending = irq_pending;
+	    ),
+	    TP_printk("partition_id=%llu vp_index=%u kicked_by_hv=%d blocked=%d irq_pending=%d",
+		    __entry->partition_id,
+		    __entry->vp_index,
+		    __entry->kicked_by_hv,
+		    __entry->blocked,
+		    __entry->irq_pending
+	    )
+);
+
+#endif /* _MSHV_TRACE_H_ */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index bc4fc1951ae1c..1d71c28fadbab 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -1258,17 +1258,12 @@ static void vmbus_chan_sched(void *event_page_addr)
 		return;
 	event = (union hv_synic_event_flags *)event_page_addr + VMBUS_MESSAGE_SINT;
 
-	maxbits = HV_EVENT_FLAGS_COUNT;
+	maxbits = READ_ONCE(vmbus_connection.relid_hiwater) + 1;
 	recv_int_page = event->flags;
 
 	if (unlikely(!recv_int_page))
 		return;
 
-	/*
-	 * Suggested-by: Michael Kelley <mhklinux@outlook.com>
-	 * One possible optimization would be to keep track of the largest relID that's in use,
-	 * and only scan up to that relID.
-	 */
 	for_each_set_bit(relid, recv_int_page, maxbits) {
 		void (*callback_fn)(void *context);
 		struct vmbus_channel *channel;
@@ -1951,12 +1946,19 @@ static int hv_mmap_ring_buffer_wrapper(struct file *filp, struct kobject *kobj,
 				       struct vm_area_struct *vma)
 {
 	struct vmbus_channel *channel = container_of(kobj, struct vmbus_channel, kobj);
+	struct vm_area_desc desc;
+	int err;
 
 	/*
-	 * hv_(create|remove)_ring_sysfs implementation ensures that mmap_ring_buffer
-	 * is not NULL.
+	 * hv_(create|remove)_ring_sysfs implementation ensures that
+	 * mmap_prepare_ring_buffer is not NULL.
 	 */
-	return channel->mmap_ring_buffer(channel, vma);
+	compat_set_desc_from_vma(&desc, filp, vma);
+	err = channel->mmap_prepare_ring_buffer(channel, &desc);
+	if (err)
+		return err;
+
+	return __compat_vma_mmap(&desc, vma);
 }
 
 static struct bin_attribute chan_attr_ring_buffer = {
@@ -2048,13 +2050,13 @@ static const struct kobj_type vmbus_chan_ktype = {
 /**
  * hv_create_ring_sysfs() - create "ring" sysfs entry corresponding to ring buffers for a channel.
  * @channel: Pointer to vmbus_channel structure
- * @hv_mmap_ring_buffer: function pointer for initializing the function to be called on mmap of
+ * @hv_mmap_prepare_ring_buffer: function pointer for initializing the function to be called on mmap
  *                       channel's "ring" sysfs node, which is for the ring buffer of that channel.
  *                       Function pointer is of below type:
- *                       int (*hv_mmap_ring_buffer)(struct vmbus_channel *channel,
- *                                                  struct vm_area_struct *vma))
- *                       This has a pointer to the channel and a pointer to vm_area_struct,
- *                       used for mmap, as arguments.
+ *                       int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel,
+ *                                                          struct vm_area_desc *desc))
+ *                       This has a pointer to the channel and a pointer to vm_area_desc,
+ *                       used for mmap_prepare, as arguments.
  *
  * Sysfs node for ring buffer of a channel is created along with other fields, however its
  * visibility is disabled by default. Sysfs creation needs to be controlled when the use-case
@@ -2071,12 +2073,12 @@ static const struct kobj_type vmbus_chan_ktype = {
  * Returns 0 on success or error code on failure.
  */
 int hv_create_ring_sysfs(struct vmbus_channel *channel,
-			 int (*hv_mmap_ring_buffer)(struct vmbus_channel *channel,
-						    struct vm_area_struct *vma))
+			 int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel,
+							    struct vm_area_desc *desc))
 {
 	struct kobject *kobj = &channel->kobj;
 
-	channel->mmap_ring_buffer = hv_mmap_ring_buffer;
+	channel->mmap_prepare_ring_buffer = hv_mmap_prepare_ring_buffer;
 	channel->ring_sysfs_visible = true;
 
 	return sysfs_update_group(kobj, &vmbus_chan_group);
@@ -2098,7 +2100,7 @@ int hv_remove_ring_sysfs(struct vmbus_channel *channel)
 
 	channel->ring_sysfs_visible = false;
 	ret = sysfs_update_group(kobj, &vmbus_chan_group);
-	channel->mmap_ring_buffer = NULL;
+	channel->mmap_prepare_ring_buffer = NULL;
 	return ret;
 }
 EXPORT_SYMBOL_GPL(hv_remove_ring_sysfs);
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 328867242cb37..8af80e17d25e1 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -457,32 +457,6 @@ config SENSORS_ATXP1
 	  This driver can also be built as a module. If so, the module
 	  will be called atxp1.
 
-config SENSORS_BT1_PVT
-	tristate "Baikal-T1 Process, Voltage, Temperature sensor driver"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
-	select POLYNOMIAL
-	help
-	  If you say yes here you get support for Baikal-T1 PVT sensor
-	  embedded into the SoC.
-
-	  This driver can also be built as a module. If so, the module will be
-	  called bt1-pvt.
-
-config SENSORS_BT1_PVT_ALARMS
-	bool "Enable Baikal-T1 PVT sensor alarms"
-	depends on SENSORS_BT1_PVT
-	help
-	  Baikal-T1 PVT IP-block provides threshold registers for each
-	  supported sensor. But the corresponding interrupts might be
-	  generated by the thresholds comparator only in synchronization with
-	  a data conversion. Additionally there is only one sensor data can
-	  be converted at a time. All of these makes the interface impossible
-	  to be used for the hwmon alarms implementation without periodic
-	  switch between the PVT sensors. By default the data conversion is
-	  performed on demand from the user-space. If this config is enabled
-	  the data conversion will be periodically performed and the data will be
-	  saved in the internal driver cache.
-
 config SENSORS_CGBC
 	tristate "Congatec Board Controller Sensors"
 	depends on MFD_CGBC
@@ -801,7 +775,6 @@ config SENSORS_G762
 
 config SENSORS_GPIO_FAN
 	tristate "GPIO fan"
-	depends on OF_GPIO
 	depends on GPIOLIB || COMPILE_TEST
 	depends on THERMAL || THERMAL=n
 	help
@@ -990,6 +963,23 @@ config SENSORS_LAN966X
 	  This driver can also be built as a module. If so, the module
 	  will be called lan966x-hwmon.
 
+config SENSORS_LATTEPANDA_SIGMA_EC
+	tristate "LattePanda Sigma EC hardware monitoring"
+	depends on X86
+	depends on DMI
+	depends on HAS_IOPORT
+	help
+	  If you say yes here you get support for the hardware monitoring
+	  features of the Embedded Controller on LattePanda Sigma
+	  single-board computers, including CPU fan speed (RPM) and
+	  board and CPU temperatures.
+
+	  The driver reads the EC directly via ACPI EC I/O ports and
+	  uses DMI matching to ensure it only loads on supported hardware.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called lattepanda-sigma-ec.
+
 config SENSORS_LENOVO_EC
         tristate "Sensor reader for Lenovo ThinkStations"
         depends on X86
@@ -2273,7 +2263,7 @@ config SENSORS_INA2XX
 	select REGMAP_I2C
 	help
 	  If you say yes here you get support for INA219, INA220, INA226,
-	  INA230, INA231, INA260, and SY24655 power monitor chips.
+	  INA230, INA231, INA234, INA260, and SY24655 power monitor chips.
 
 	  The INA2xx driver is configured for the default configuration of
 	  the part as described in the datasheet.
@@ -2362,8 +2352,8 @@ config SENSORS_TMP102
 	depends on I2C
 	select REGMAP_I2C
 	help
-	  If you say yes here you get support for Texas Instruments TMP102
-	  sensor chips.
+	  If you say yes here you get support for Texas Instruments TMP102,
+	  TMP110 and TMP113 sensor chips.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called tmp102.
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 5833c807c688a..556e86d277b19 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -58,7 +58,6 @@ obj-$(CONFIG_SENSORS_ASPEED_G6) += aspeed-g6-pwm-tach.o
 obj-$(CONFIG_SENSORS_ASUS_ROG_RYUJIN)	+= asus_rog_ryujin.o
 obj-$(CONFIG_SENSORS_ATXP1)	+= atxp1.o
 obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o
-obj-$(CONFIG_SENSORS_BT1_PVT)	+= bt1-pvt.o
 obj-$(CONFIG_SENSORS_CGBC)	+= cgbc-hwmon.o
 obj-$(CONFIG_SENSORS_CHIPCAP2) += chipcap2.o
 obj-$(CONFIG_SENSORS_CORETEMP)	+= coretemp.o
@@ -114,6 +113,7 @@ obj-$(CONFIG_SENSORS_K10TEMP)	+= k10temp.o
 obj-$(CONFIG_SENSORS_KBATT)	+= kbatt.o
 obj-$(CONFIG_SENSORS_KFAN)	+= kfan.o
 obj-$(CONFIG_SENSORS_LAN966X)	+= lan966x-hwmon.o
+obj-$(CONFIG_SENSORS_LATTEPANDA_SIGMA_EC) += lattepanda-sigma-ec.o
 obj-$(CONFIG_SENSORS_LENOVO_EC)	+= lenovo-ec-sensors.o
 obj-$(CONFIG_SENSORS_LINEAGE)	+= lineage-pem.o
 obj-$(CONFIG_SENSORS_LOCHNAGAR)	+= lochnagar-hwmon.o
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index 1e3fab5f79468..be7f702dcde9c 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -18,6 +18,7 @@
 #include <linux/time.h>
 #include <linux/err.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 
 #define ACPI_POWER_METER_NAME		"power_meter"
 #define ACPI_POWER_METER_DEVICE_NAME	"Power Meter"
@@ -814,16 +815,12 @@ static int read_capabilities(struct acpi_power_meter_resource *resource)
 }
 
 /* Handle ACPI event notifications */
-static void acpi_power_meter_notify(struct acpi_device *device, u32 event)
+static void acpi_power_meter_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct acpi_power_meter_resource *resource;
+	struct device *dev = data;
+	struct acpi_power_meter_resource *resource = dev_get_drvdata(dev);
 	int res;
 
-	if (!device || !acpi_driver_data(device))
-		return;
-
-	resource = acpi_driver_data(device);
-
 	guard(mutex)(&acpi_notify_lock);
 
 	switch (event) {
@@ -837,43 +834,43 @@ static void acpi_power_meter_notify(struct acpi_device *device, u32 event)
 		remove_domain_devices(resource);
 		res = read_capabilities(resource);
 		if (res)
-			dev_err_once(&device->dev, "read capabilities failed.\n");
+			dev_err_once(dev, "read capabilities failed.\n");
 		res = read_domain_devices(resource);
 		if (res && res != -ENODEV)
-			dev_err_once(&device->dev, "read domain devices failed.\n");
+			dev_err_once(dev, "read domain devices failed.\n");
 
 		mutex_unlock(&resource->lock);
 
 		resource->hwmon_dev =
-			hwmon_device_register_with_info(&device->dev,
+			hwmon_device_register_with_info(dev,
 							ACPI_POWER_METER_NAME,
 							resource,
 							&power_meter_chip_info,
 							power_extra_groups);
 		if (IS_ERR(resource->hwmon_dev))
-			dev_err_once(&device->dev, "register hwmon device failed.\n");
+			dev_err_once(dev, "register hwmon device failed.\n");
 
 		break;
 	case METER_NOTIFY_TRIP:
-		sysfs_notify(&device->dev.kobj, NULL, POWER_AVERAGE_NAME);
+		sysfs_notify(&dev->kobj, NULL, POWER_AVERAGE_NAME);
 		break;
 	case METER_NOTIFY_CAP:
 		mutex_lock(&resource->lock);
 		res = update_cap(resource);
 		if (res)
-			dev_err_once(&device->dev, "update cap failed when capping value is changed.\n");
+			dev_err_once(dev, "update cap failed when capping value is changed.\n");
 		mutex_unlock(&resource->lock);
-		sysfs_notify(&device->dev.kobj, NULL, POWER_CAP_NAME);
+		sysfs_notify(&dev->kobj, NULL, POWER_CAP_NAME);
 		break;
 	case METER_NOTIFY_INTERVAL:
-		sysfs_notify(&device->dev.kobj, NULL, POWER_AVG_INTERVAL_NAME);
+		sysfs_notify(&dev->kobj, NULL, POWER_AVG_INTERVAL_NAME);
 		break;
 	case METER_NOTIFY_CAPPING:
 		mutex_lock(&resource->lock);
 		resource->power_alarm = true;
 		mutex_unlock(&resource->lock);
-		sysfs_notify(&device->dev.kobj, NULL, POWER_ALARM_NAME);
-		dev_info(&device->dev, "Capping in progress.\n");
+		sysfs_notify(&dev->kobj, NULL, POWER_ALARM_NAME);
+		dev_info(dev, "Capping in progress.\n");
 		break;
 	default:
 		WARN(1, "Unexpected event %d\n", event);
@@ -881,16 +878,15 @@ static void acpi_power_meter_notify(struct acpi_device *device, u32 event)
 	}
 
 	acpi_bus_generate_netlink_event(ACPI_POWER_METER_CLASS,
-					dev_name(&device->dev), event, 0);
+					dev_name(&resource->acpi_dev->dev),
+					event, 0);
 }
 
-static int acpi_power_meter_add(struct acpi_device *device)
+static int acpi_power_meter_probe(struct platform_device *pdev)
 {
-	int res;
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct acpi_power_meter_resource *resource;
-
-	if (!device)
-		return -EINVAL;
+	int res;
 
 	resource = kzalloc_obj(*resource);
 	if (!resource)
@@ -901,7 +897,8 @@ static int acpi_power_meter_add(struct acpi_device *device)
 	mutex_init(&resource->lock);
 	strscpy(acpi_device_name(device), ACPI_POWER_METER_DEVICE_NAME);
 	strscpy(acpi_device_class(device), ACPI_POWER_METER_CLASS);
-	device->driver_data = resource;
+
+	platform_set_drvdata(pdev, resource);
 
 #if IS_REACHABLE(CONFIG_ACPI_IPMI)
 	/*
@@ -914,7 +911,7 @@ static int acpi_power_meter_add(struct acpi_device *device)
 		struct acpi_device *ipi_device = acpi_dev_get_first_match_dev("IPI0001", NULL, -1);
 
 		if (ipi_device && acpi_wait_for_acpi_ipmi())
-			dev_warn(&device->dev, "Waiting for ACPI IPMI timeout");
+			dev_warn(&pdev->dev, "Waiting for ACPI IPMI timeout");
 		acpi_dev_put(ipi_device);
 	}
 #endif
@@ -932,7 +929,7 @@ static int acpi_power_meter_add(struct acpi_device *device)
 		goto exit_free_capability;
 
 	resource->hwmon_dev =
-		hwmon_device_register_with_info(&device->dev,
+		hwmon_device_register_with_info(&pdev->dev,
 						ACPI_POWER_METER_NAME, resource,
 						&power_meter_chip_info,
 						power_extra_groups);
@@ -941,9 +938,16 @@ static int acpi_power_meter_add(struct acpi_device *device)
 		goto exit_remove;
 	}
 
+	res = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+					      acpi_power_meter_notify, &pdev->dev);
+	if (res)
+		goto exit_hwmon;
+
 	res = 0;
 	goto exit;
 
+exit_hwmon:
+	hwmon_device_unregister(resource->hwmon_dev);
 exit_remove:
 	remove_domain_devices(resource);
 exit_free_capability:
@@ -954,14 +958,13 @@ static int acpi_power_meter_add(struct acpi_device *device)
 	return res;
 }
 
-static void acpi_power_meter_remove(struct acpi_device *device)
+static void acpi_power_meter_remove(struct platform_device *pdev)
 {
-	struct acpi_power_meter_resource *resource;
+	struct acpi_power_meter_resource *resource = platform_get_drvdata(pdev);
 
-	if (!device || !acpi_driver_data(device))
-		return;
+	acpi_dev_remove_notify_handler(resource->acpi_dev, ACPI_DEVICE_NOTIFY,
+				       acpi_power_meter_notify);
 
-	resource = acpi_driver_data(device);
 	if (!IS_ERR(resource->hwmon_dev))
 		hwmon_device_unregister(resource->hwmon_dev);
 
@@ -973,14 +976,7 @@ static void acpi_power_meter_remove(struct acpi_device *device)
 
 static int acpi_power_meter_resume(struct device *dev)
 {
-	struct acpi_power_meter_resource *resource;
-
-	if (!dev)
-		return -EINVAL;
-
-	resource = acpi_driver_data(to_acpi_device(dev));
-	if (!resource)
-		return -EINVAL;
+	struct acpi_power_meter_resource *resource = dev_get_drvdata(dev);
 
 	free_capabilities(resource);
 	read_capabilities(resource);
@@ -991,16 +987,14 @@ static int acpi_power_meter_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(acpi_power_meter_pm, NULL,
 				acpi_power_meter_resume);
 
-static struct acpi_driver acpi_power_meter_driver = {
-	.name = "power_meter",
-	.class = ACPI_POWER_METER_CLASS,
-	.ids = power_meter_ids,
-	.ops = {
-		.add = acpi_power_meter_add,
-		.remove = acpi_power_meter_remove,
-		.notify = acpi_power_meter_notify,
-		},
-	.drv.pm = pm_sleep_ptr(&acpi_power_meter_pm),
+static struct platform_driver acpi_power_meter_driver = {
+	.probe = acpi_power_meter_probe,
+	.remove = acpi_power_meter_remove,
+	.driver = {
+		.name = "acpi-power-meter",
+		.acpi_match_table = power_meter_ids,
+		.pm = &acpi_power_meter_pm,
+	},
 };
 
 /* Module init/exit routines */
@@ -1029,7 +1023,7 @@ static int __init acpi_power_meter_init(void)
 
 	dmi_check_system(pm_dmi_table);
 
-	result = acpi_bus_register_driver(&acpi_power_meter_driver);
+	result = platform_driver_register(&acpi_power_meter_driver);
 	if (result < 0)
 		return result;
 
@@ -1038,7 +1032,7 @@ static int __init acpi_power_meter_init(void)
 
 static void __exit acpi_power_meter_exit(void)
 {
-	acpi_bus_unregister_driver(&acpi_power_meter_driver);
+	platform_driver_unregister(&acpi_power_meter_driver);
 }
 
 MODULE_AUTHOR("Darrick J. Wong <darrick.wong@oracle.com>");
diff --git a/drivers/hwmon/ads7828.c b/drivers/hwmon/ads7828.c
index 436637264056c..7f43565ca2841 100644
--- a/drivers/hwmon/ads7828.c
+++ b/drivers/hwmon/ads7828.c
@@ -62,8 +62,8 @@ static ssize_t ads7828_in_show(struct device *dev,
 	if (err < 0)
 		return err;
 
-	return sprintf(buf, "%d\n",
-		       DIV_ROUND_CLOSEST(regval * data->lsb_resol, 1000));
+	return sysfs_emit(buf, "%d\n",
+			  DIV_ROUND_CLOSEST(regval * data->lsb_resol, 1000));
 }
 
 static SENSOR_DEVICE_ATTR_RO(in0_input, ads7828_in, 0);
diff --git a/drivers/hwmon/ads7871.c b/drivers/hwmon/ads7871.c
index 5434c37969d75..9bfdf9e6bcd77 100644
--- a/drivers/hwmon/ads7871.c
+++ b/drivers/hwmon/ads7871.c
@@ -104,10 +104,14 @@ static ssize_t voltage_show(struct device *dev, struct device_attribute *da,
 	 */
 	/*MUX_M3_BM forces single ended*/
 	/*This is also where the gain of the PGA would be set*/
-	ads7871_write_reg8(spi, REG_GAIN_MUX,
-		(MUX_CNV_BM | MUX_M3_BM | channel));
+	ret = ads7871_write_reg8(spi, REG_GAIN_MUX,
+				 (MUX_CNV_BM | MUX_M3_BM | channel));
+	if (ret < 0)
+		return ret;
 
 	ret = ads7871_read_reg8(spi, REG_GAIN_MUX);
+	if (ret < 0)
+		return ret;
 	mux_cnv = ((ret & MUX_CNV_BM) >> MUX_CNV_BV);
 	/*
 	 * on 400MHz arm9 platform the conversion
@@ -116,18 +120,22 @@ static ssize_t voltage_show(struct device *dev, struct device_attribute *da,
 	while ((i < 2) && mux_cnv) {
 		i++;
 		ret = ads7871_read_reg8(spi, REG_GAIN_MUX);
+		if (ret < 0)
+			return ret;
 		mux_cnv = ((ret & MUX_CNV_BM) >> MUX_CNV_BV);
 		msleep_interruptible(1);
 	}
 
 	if (mux_cnv == 0) {
 		val = ads7871_read_reg16(spi, REG_LS_BYTE);
+		if (val < 0)
+			return val;
 		/*result in volts*10000 = (val/8192)*2.5*10000*/
 		val = ((val >> 2) * 25000) / 8192;
-		return sprintf(buf, "%d\n", val);
-	} else {
-		return -1;
+		return sysfs_emit(buf, "%d\n", val);
 	}
+
+	return -ETIMEDOUT;
 }
 
 static SENSOR_DEVICE_ATTR_RO(in0_input, voltage, 0);
diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c
index 4ce019d2cc80e..66955395d0583 100644
--- a/drivers/hwmon/aht10.c
+++ b/drivers/hwmon/aht10.c
@@ -62,6 +62,15 @@ static const struct i2c_device_id aht10_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, aht10_id);
 
+static const struct of_device_id aht10_of_match[] = {
+	{ .compatible = "aosong,aht10", .data = (void *)aht10 },
+	{ .compatible = "aosong,aht20", .data = (void *)aht20 },
+	{ .compatible = "aosong,dht20", .data = (void *)dht20 },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, aht10_of_match);
+
 /**
  *   struct aht10_data - All the data required to operate an AHT10/AHT20 chip
  *   @client: the i2c client associated with the AHT10/AHT20
@@ -377,6 +386,7 @@ static int aht10_probe(struct i2c_client *client)
 static struct i2c_driver aht10_driver = {
 	.driver = {
 		.name = "aht10",
+		.of_match_table = aht10_of_match,
 	},
 	.probe      = aht10_probe,
 	.id_table   = aht10_id,
diff --git a/drivers/hwmon/aspeed-g6-pwm-tach.c b/drivers/hwmon/aspeed-g6-pwm-tach.c
index 44e1ecba205d7..4f6e6d440dd40 100644
--- a/drivers/hwmon/aspeed-g6-pwm-tach.c
+++ b/drivers/hwmon/aspeed-g6-pwm-tach.c
@@ -517,13 +517,6 @@ static int aspeed_pwm_tach_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static void aspeed_pwm_tach_remove(struct platform_device *pdev)
-{
-	struct aspeed_pwm_tach_data *priv = platform_get_drvdata(pdev);
-
-	reset_control_assert(priv->reset);
-}
-
 static const struct of_device_id aspeed_pwm_tach_match[] = {
 	{
 		.compatible = "aspeed,ast2600-pwm-tach",
@@ -537,7 +530,6 @@ MODULE_DEVICE_TABLE(of, aspeed_pwm_tach_match);
 
 static struct platform_driver aspeed_pwm_tach_driver = {
 	.probe = aspeed_pwm_tach_probe,
-	.remove = aspeed_pwm_tach_remove,
 	.driver	= {
 		.name = "aspeed-g6-pwm-tach",
 		.of_match_table = aspeed_pwm_tach_match,
diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
index 86f444498650e..070bb368f2b70 100644
--- a/drivers/hwmon/asus-ec-sensors.c
+++ b/drivers/hwmon/asus-ec-sensors.c
@@ -451,6 +451,15 @@ static const struct ec_board_info board_info_crosshair_viii_impact = {
 	.family = family_amd_500_series,
 };
 
+static const struct ec_board_info board_info_crosshair_x670e_extreme = {
+	.sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE |
+		SENSOR_TEMP_MB | SENSOR_TEMP_VRM |
+		SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_WATER_IN |
+		SENSOR_TEMP_WATER_OUT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0,
+	.family = family_amd_600_series,
+};
+
 static const struct ec_board_info board_info_crosshair_x670e_gene = {
 	.sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE |
 		SENSOR_TEMP_T_SENSOR |
@@ -621,6 +630,14 @@ static const struct ec_board_info board_info_strix_b850_i_gaming_wifi = {
 	.family = family_amd_800_series,
 };
 
+static const struct ec_board_info board_info_strix_x470_f_gaming = {
+	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
+		SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT |
+		SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX,
+	.family = family_amd_400_series,
+};
+
 static const struct ec_board_info board_info_strix_x470_i_gaming = {
 	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
 		SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM |
@@ -745,6 +762,12 @@ static const struct ec_board_info board_info_strix_z790_e_gaming_wifi_ii = {
 	.family = family_intel_700_series,
 };
 
+static const struct ec_board_info board_info_strix_z790_h_gaming_wifi = {
+	.sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PC00_LPCB_SIO1_MUT0,
+	.family = family_intel_700_series,
+};
+
 static const struct ec_board_info board_info_strix_z790_i_gaming_wifi = {
 	.sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_T_SENSOR_2 |
 		SENSOR_TEMP_VRM,
@@ -820,6 +843,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_crosshair_viii_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII IMPACT",
 					&board_info_crosshair_viii_impact),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR X670E EXTREME",
+					&board_info_crosshair_x670e_extreme),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR X670E GENE",
 					&board_info_crosshair_x670e_gene),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR X670E HERO",
@@ -840,6 +865,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_b650e_i_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B850-I GAMING WIFI",
 					&board_info_strix_b850_i_gaming_wifi),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X470-F GAMING",
+					&board_info_strix_x470_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X470-I GAMING",
 					&board_info_strix_x470_i_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-E GAMING",
@@ -872,6 +899,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_z690_e_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z790-E GAMING WIFI II",
 					&board_info_strix_z790_e_gaming_wifi_ii),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z790-H GAMING WIFI",
+					&board_info_strix_z790_h_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z790-I GAMING WIFI",
 					&board_info_strix_z790_i_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG ZENITH II EXTREME",
diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c
index c80350e499e97..5688ff5f7c28d 100644
--- a/drivers/hwmon/asus_atk0110.c
+++ b/drivers/hwmon/asus_atk0110.c
@@ -17,6 +17,7 @@
 #include <linux/jiffies.h>
 #include <linux/err.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 #include <linux/string_choices.h>
 
 #define ATK_HID "ATK0110"
@@ -107,7 +108,7 @@ enum atk_pack_member {
 struct atk_data {
 	struct device *hwmon_dev;
 	acpi_handle atk_handle;
-	struct acpi_device *acpi_dev;
+	struct device *dev;
 
 	bool old_interface;
 
@@ -187,18 +188,17 @@ struct atk_acpi_input_buf {
 	u32 param2;
 };
 
-static int atk_add(struct acpi_device *device);
-static void atk_remove(struct acpi_device *device);
+static int atk_probe(struct platform_device *pdev);
+static void atk_remove(struct platform_device *pdev);
 static void atk_print_sensor(struct atk_data *data, union acpi_object *obj);
 static int atk_read_value(struct atk_sensor_data *sensor, u64 *value);
 
-static struct acpi_driver atk_driver = {
-	.name	= ATK_HID,
-	.class	= "hwmon",
-	.ids	= atk_ids,
-	.ops	= {
-		.add	= atk_add,
-		.remove	= atk_remove,
+static struct platform_driver atk_driver = {
+	.probe = atk_probe,
+	.remove = atk_remove,
+	.driver = {
+		.name = ATK_HID,
+		.acpi_match_table = atk_ids,
 	},
 };
 
@@ -327,7 +327,7 @@ static union acpi_object *atk_get_pack_member(struct atk_data *data,
  */
 static int validate_hwmon_pack(struct atk_data *data, union acpi_object *obj)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *tmp;
 	bool old_if = data->old_interface;
 	int const expected_size = old_if ? _HWMON_OLD_PACK_SIZE :
@@ -422,7 +422,7 @@ static char const *atk_sensor_type(union acpi_object *flags)
 static void atk_print_sensor(struct atk_data *data, union acpi_object *obj)
 {
 #ifdef DEBUG
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *flags;
 	union acpi_object *name;
 	union acpi_object *limit1;
@@ -449,7 +449,7 @@ static void atk_print_sensor(struct atk_data *data, union acpi_object *obj)
 static int atk_read_value_old(struct atk_sensor_data *sensor, u64 *value)
 {
 	struct atk_data *data = sensor->data;
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct acpi_object_list params;
 	union acpi_object id;
 	acpi_status status;
@@ -487,7 +487,7 @@ static int atk_read_value_old(struct atk_sensor_data *sensor, u64 *value)
 
 static union acpi_object *atk_ggrp(struct atk_data *data, u16 mux)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct acpi_buffer buf;
 	acpi_status ret;
 	struct acpi_object_list params;
@@ -523,7 +523,7 @@ static union acpi_object *atk_ggrp(struct atk_data *data, u16 mux)
 
 static union acpi_object *atk_gitm(struct atk_data *data, u64 id)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct atk_acpi_input_buf buf;
 	union acpi_object tmp;
 	struct acpi_object_list params;
@@ -565,7 +565,7 @@ static union acpi_object *atk_gitm(struct atk_data *data, u64 id)
 static union acpi_object *atk_sitm(struct atk_data *data,
 		struct atk_acpi_input_buf *buf)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct acpi_object_list params;
 	union acpi_object tmp;
 	struct acpi_buffer ret;
@@ -602,7 +602,7 @@ static union acpi_object *atk_sitm(struct atk_data *data,
 static int atk_read_value_new(struct atk_sensor_data *sensor, u64 *value)
 {
 	struct atk_data *data = sensor->data;
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *obj;
 	struct atk_acpi_ret_buffer *buf;
 	int err = 0;
@@ -819,7 +819,7 @@ static void atk_debugfs_cleanup(struct atk_data *data)
 
 static int atk_add_sensor(struct atk_data *data, union acpi_object *obj)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *flags;
 	union acpi_object *name;
 	union acpi_object *limit1;
@@ -937,7 +937,7 @@ static int atk_add_sensor(struct atk_data *data, union acpi_object *obj)
 
 static int atk_enumerate_old_hwmon(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct acpi_buffer buf;
 	union acpi_object *pack;
 	acpi_status status;
@@ -1012,7 +1012,7 @@ static int atk_enumerate_old_hwmon(struct atk_data *data)
 
 static int atk_ec_present(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *pack;
 	union acpi_object *ec;
 	int ret;
@@ -1058,7 +1058,7 @@ static int atk_ec_present(struct atk_data *data)
 
 static int atk_ec_enabled(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *obj;
 	struct atk_acpi_ret_buffer *buf;
 	int err;
@@ -1084,7 +1084,7 @@ static int atk_ec_enabled(struct atk_data *data)
 
 static int atk_ec_ctl(struct atk_data *data, int enable)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *obj;
 	struct atk_acpi_input_buf sitm;
 	struct atk_acpi_ret_buffer *ec_ret;
@@ -1113,7 +1113,7 @@ static int atk_ec_ctl(struct atk_data *data, int enable)
 
 static int atk_enumerate_new_hwmon(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	union acpi_object *pack;
 	int err;
 	int i;
@@ -1155,7 +1155,7 @@ static int atk_enumerate_new_hwmon(struct atk_data *data)
 
 static int atk_init_attribute_groups(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	struct atk_sensor_data *s;
 	struct attribute **attrs;
 	int i = 0;
@@ -1181,7 +1181,7 @@ static int atk_init_attribute_groups(struct atk_data *data)
 
 static int atk_register_hwmon(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 
 	dev_dbg(dev, "registering hwmon device\n");
 	data->hwmon_dev = hwmon_device_register_with_groups(dev, "atk0110",
@@ -1193,7 +1193,7 @@ static int atk_register_hwmon(struct atk_data *data)
 
 static int atk_probe_if(struct atk_data *data)
 {
-	struct device *dev = &data->acpi_dev->dev;
+	struct device *dev = data->dev;
 	acpi_handle ret;
 	acpi_status status;
 	int err = 0;
@@ -1266,7 +1266,7 @@ static int atk_probe_if(struct atk_data *data)
 	return err;
 }
 
-static int atk_add(struct acpi_device *device)
+static int atk_probe(struct platform_device *pdev)
 {
 	acpi_status ret;
 	int err;
@@ -1274,14 +1274,14 @@ static int atk_add(struct acpi_device *device)
 	union acpi_object *obj;
 	struct atk_data *data;
 
-	dev_dbg(&device->dev, "adding...\n");
+	dev_dbg(&pdev->dev, "adding...\n");
 
-	data = devm_kzalloc(&device->dev, sizeof(*data), GFP_KERNEL);
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	data->acpi_dev = device;
-	data->atk_handle = device->handle;
+	data->dev = &pdev->dev;
+	data->atk_handle = ACPI_HANDLE(&pdev->dev);
 	INIT_LIST_HEAD(&data->sensor_list);
 	data->disable_ec = false;
 
@@ -1289,13 +1289,13 @@ static int atk_add(struct acpi_device *device)
 	ret = acpi_evaluate_object_typed(data->atk_handle, BOARD_ID, NULL,
 			&buf, ACPI_TYPE_PACKAGE);
 	if (ret != AE_OK) {
-		dev_dbg(&device->dev, "atk: method MBIF not found\n");
+		dev_dbg(&pdev->dev, "atk: method MBIF not found\n");
 	} else {
 		obj = buf.pointer;
 		if (obj->package.count >= 2) {
 			union acpi_object *id = &obj->package.elements[1];
 			if (id->type == ACPI_TYPE_STRING)
-				dev_dbg(&device->dev, "board ID = %s\n",
+				dev_dbg(&pdev->dev, "board ID = %s\n",
 					id->string.pointer);
 		}
 		ACPI_FREE(buf.pointer);
@@ -1303,21 +1303,21 @@ static int atk_add(struct acpi_device *device)
 
 	err = atk_probe_if(data);
 	if (err) {
-		dev_err(&device->dev, "No usable hwmon interface detected\n");
+		dev_err(&pdev->dev, "No usable hwmon interface detected\n");
 		goto out;
 	}
 
 	if (data->old_interface) {
-		dev_dbg(&device->dev, "Using old hwmon interface\n");
+		dev_dbg(&pdev->dev, "Using old hwmon interface\n");
 		err = atk_enumerate_old_hwmon(data);
 	} else {
-		dev_dbg(&device->dev, "Using new hwmon interface\n");
+		dev_dbg(&pdev->dev, "Using new hwmon interface\n");
 		err = atk_enumerate_new_hwmon(data);
 	}
 	if (err < 0)
 		goto out;
 	if (err == 0) {
-		dev_info(&device->dev,
+		dev_info(&pdev->dev,
 			 "No usable sensor detected, bailing out\n");
 		err = -ENODEV;
 		goto out;
@@ -1332,7 +1332,8 @@ static int atk_add(struct acpi_device *device)
 
 	atk_debugfs_init(data);
 
-	device->driver_data = data;
+	platform_set_drvdata(pdev, data);
+
 	return 0;
 out:
 	if (data->disable_ec)
@@ -1340,12 +1341,11 @@ static int atk_add(struct acpi_device *device)
 	return err;
 }
 
-static void atk_remove(struct acpi_device *device)
+static void atk_remove(struct platform_device *pdev)
 {
-	struct atk_data *data = device->driver_data;
-	dev_dbg(&device->dev, "removing...\n");
+	struct atk_data *data = platform_get_drvdata(pdev);
 
-	device->driver_data = NULL;
+	dev_dbg(&pdev->dev, "removing...\n");
 
 	atk_debugfs_cleanup(data);
 
@@ -1353,7 +1353,7 @@ static void atk_remove(struct acpi_device *device)
 
 	if (data->disable_ec) {
 		if (atk_ec_ctl(data, 0))
-			dev_err(&device->dev, "Failed to disable EC\n");
+			dev_err(&pdev->dev, "Failed to disable EC\n");
 	}
 }
 
@@ -1370,16 +1370,16 @@ static int __init atk0110_init(void)
 	if (dmi_check_system(atk_force_new_if))
 		new_if = true;
 
-	ret = acpi_bus_register_driver(&atk_driver);
+	ret = platform_driver_register(&atk_driver);
 	if (ret)
-		pr_info("acpi_bus_register_driver failed: %d\n", ret);
+		pr_info("platform_driver_register failed: %d\n", ret);
 
 	return ret;
 }
 
 static void __exit atk0110_exit(void)
 {
-	acpi_bus_unregister_driver(&atk_driver);
+	platform_driver_unregister(&atk_driver);
 }
 
 module_init(atk0110_init);
diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c
index b7bb325c3ad96..01590dfa55e60 100644
--- a/drivers/hwmon/axi-fan-control.c
+++ b/drivers/hwmon/axi-fan-control.c
@@ -507,7 +507,7 @@ static int axi_fan_control_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(&pdev->dev, ctl->irq, NULL,
 					axi_fan_control_irq_handler,
 					IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
-					pdev->driver_override, ctl);
+					NULL, ctl);
 	if (ret)
 		return dev_err_probe(&pdev->dev, ret,
 				     "failed to request an irq\n");
diff --git a/drivers/hwmon/bt1-pvt.c b/drivers/hwmon/bt1-pvt.c
deleted file mode 100644
index b77ebac2e0cea..0000000000000
--- a/drivers/hwmon/bt1-pvt.c
+++ /dev/null
@@ -1,1171 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru>
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 Process, Voltage, Temperature sensor driver
- */
-
-#include <linux/bitfield.h>
-#include <linux/bitops.h>
-#include <linux/clk.h>
-#include <linux/completion.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/hwmon-sysfs.h>
-#include <linux/hwmon.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/ktime.h>
-#include <linux/limits.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/polynomial.h>
-#include <linux/seqlock.h>
-#include <linux/sysfs.h>
-#include <linux/types.h>
-
-#include "bt1-pvt.h"
-
-/*
- * For the sake of the code simplification we created the sensors info table
- * with the sensor names, activation modes, threshold registers base address
- * and the thresholds bit fields.
- */
-static const struct pvt_sensor_info pvt_info[] = {
-	PVT_SENSOR_INFO(0, "CPU Core Temperature", hwmon_temp, TEMP, TTHRES),
-	PVT_SENSOR_INFO(0, "CPU Core Voltage", hwmon_in, VOLT, VTHRES),
-	PVT_SENSOR_INFO(1, "CPU Core Low-Vt", hwmon_in, LVT, LTHRES),
-	PVT_SENSOR_INFO(2, "CPU Core High-Vt", hwmon_in, HVT, HTHRES),
-	PVT_SENSOR_INFO(3, "CPU Core Standard-Vt", hwmon_in, SVT, STHRES),
-};
-
-/*
- * The original translation formulae of the temperature (in degrees of Celsius)
- * to PVT data and vice-versa are following:
- * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) +
- *     1.7204e2,
- * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) +
- *     3.1020e-1*(N^1) - 4.838e1,
- * where T = [-48.380, 147.438]C and N = [0, 1023].
- * They must be accordingly altered to be suitable for the integer arithmetics.
- * The technique is called 'factor redistribution', which just makes sure the
- * multiplications and divisions are made so to have a result of the operations
- * within the integer numbers limit. In addition we need to translate the
- * formulae to accept millidegrees of Celsius. Here what they look like after
- * the alterations:
- * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T +
- *     17204e2) / 1e4,
- * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D -
- *     48380,
- * where T = [-48380, 147438] mC and N = [0, 1023].
- */
-static const struct polynomial __maybe_unused poly_temp_to_N = {
-	.total_divider = 10000,
-	.terms = {
-		{4, 18322, 10000, 10000},
-		{3, 2343, 10000, 10},
-		{2, 87018, 10000, 10},
-		{1, 39269, 1000, 1},
-		{0, 1720400, 1, 1}
-	}
-};
-
-static const struct polynomial poly_N_to_temp = {
-	.total_divider = 1,
-	.terms = {
-		{4, -16743, 1000, 1},
-		{3, 81542, 1000, 1},
-		{2, -182010, 1000, 1},
-		{1, 310200, 1000, 1},
-		{0, -48380, 1, 1}
-	}
-};
-
-/*
- * Similar alterations are performed for the voltage conversion equations.
- * The original formulae are:
- * N = 1.8658e3*V - 1.1572e3,
- * V = (N + 1.1572e3) / 1.8658e3,
- * where V = [0.620, 1.168] V and N = [0, 1023].
- * After the optimization they looks as follows:
- * N = (18658e-3*V - 11572) / 10,
- * V = N * 10^5 / 18658 + 11572 * 10^4 / 18658.
- */
-static const struct polynomial __maybe_unused poly_volt_to_N = {
-	.total_divider = 10,
-	.terms = {
-		{1, 18658, 1000, 1},
-		{0, -11572, 1, 1}
-	}
-};
-
-static const struct polynomial poly_N_to_volt = {
-	.total_divider = 10,
-	.terms = {
-		{1, 100000, 18658, 1},
-		{0, 115720000, 1, 18658}
-	}
-};
-
-static inline u32 pvt_update(void __iomem *reg, u32 mask, u32 data)
-{
-	u32 old;
-
-	old = readl_relaxed(reg);
-	writel((old & ~mask) | (data & mask), reg);
-
-	return old & mask;
-}
-
-/*
- * Baikal-T1 PVT mode can be updated only when the controller is disabled.
- * So first we disable it, then set the new mode together with the controller
- * getting back enabled. The same concerns the temperature trim and
- * measurements timeout. If it is necessary the interface mutex is supposed
- * to be locked at the time the operations are performed.
- */
-static inline void pvt_set_mode(struct pvt_hwmon *pvt, u32 mode)
-{
-	u32 old;
-
-	mode = FIELD_PREP(PVT_CTRL_MODE_MASK, mode);
-
-	old = pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_MODE_MASK | PVT_CTRL_EN,
-		   mode | old);
-}
-
-static inline u32 pvt_calc_trim(long temp)
-{
-	temp = clamp_val(temp, 0, PVT_TRIM_TEMP);
-
-	return DIV_ROUND_UP(temp, PVT_TRIM_STEP);
-}
-
-static inline void pvt_set_trim(struct pvt_hwmon *pvt, u32 trim)
-{
-	u32 old;
-
-	trim = FIELD_PREP(PVT_CTRL_TRIM_MASK, trim);
-
-	old = pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_TRIM_MASK | PVT_CTRL_EN,
-		   trim | old);
-}
-
-static inline void pvt_set_tout(struct pvt_hwmon *pvt, u32 tout)
-{
-	u32 old;
-
-	old = pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	writel(tout, pvt->regs + PVT_TTIMEOUT);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, old);
-}
-
-/*
- * This driver can optionally provide the hwmon alarms for each sensor the PVT
- * controller supports. The alarms functionality is made compile-time
- * configurable due to the hardware interface implementation peculiarity
- * described further in this comment. So in case if alarms are unnecessary in
- * your system design it's recommended to have them disabled to prevent the PVT
- * IRQs being periodically raised to get the data cache/alarms status up to
- * date.
- *
- * Baikal-T1 PVT embedded controller is based on the Analog Bits PVT sensor,
- * but is equipped with a dedicated control wrapper. It exposes the PVT
- * sub-block registers space via the APB3 bus. In addition the wrapper provides
- * a common interrupt vector of the sensors conversion completion events and
- * threshold value alarms. Alas the wrapper interface hasn't been fully thought
- * through. There is only one sensor can be activated at a time, for which the
- * thresholds comparator is enabled right after the data conversion is
- * completed. Due to this if alarms need to be implemented for all available
- * sensors we can't just set the thresholds and enable the interrupts. We need
- * to enable the sensors one after another and let the controller to detect
- * the alarms by itself at each conversion. This also makes pointless to handle
- * the alarms interrupts, since in occasion they happen synchronously with
- * data conversion completion. The best driver design would be to have the
- * completion interrupts enabled only and keep the converted value in the
- * driver data cache. This solution is implemented if hwmon alarms are enabled
- * in this driver. In case if the alarms are disabled, the conversion is
- * performed on demand at the time a sensors input file is read.
- */
-
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-
-#define pvt_hard_isr NULL
-
-static irqreturn_t pvt_soft_isr(int irq, void *data)
-{
-	const struct pvt_sensor_info *info;
-	struct pvt_hwmon *pvt = data;
-	struct pvt_cache *cache;
-	u32 val, thres_sts, old;
-
-	/*
-	 * DVALID bit will be cleared by reading the data. We need to save the
-	 * status before the next conversion happens. Threshold events will be
-	 * handled a bit later.
-	 */
-	thres_sts = readl(pvt->regs + PVT_RAW_INTR_STAT);
-
-	/*
-	 * Then lets recharge the PVT interface with the next sampling mode.
-	 * Lock the interface mutex to serialize trim, timeouts and alarm
-	 * thresholds settings.
-	 */
-	cache = &pvt->cache[pvt->sensor];
-	info = &pvt_info[pvt->sensor];
-	pvt->sensor = (pvt->sensor == PVT_SENSOR_LAST) ?
-		      PVT_SENSOR_FIRST : (pvt->sensor + 1);
-
-	/*
-	 * For some reason we have to mask the interrupt before changing the
-	 * mode, otherwise sometimes the temperature mode doesn't get
-	 * activated even though the actual mode in the ctrl register
-	 * corresponds to one. Then we read the data. By doing so we also
-	 * recharge the data conversion. After this the mode corresponding
-	 * to the next sensor in the row is set. Finally we enable the
-	 * interrupts back.
-	 */
-	mutex_lock(&pvt->iface_mtx);
-
-	old = pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID,
-			 PVT_INTR_DVALID);
-
-	val = readl(pvt->regs + PVT_DATA);
-
-	pvt_set_mode(pvt, pvt_info[pvt->sensor].mode);
-
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID, old);
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	/*
-	 * We can now update the data cache with data just retrieved from the
-	 * sensor. Lock write-seqlock to make sure the reader has a coherent
-	 * data.
-	 */
-	write_seqlock(&cache->data_seqlock);
-
-	cache->data = FIELD_GET(PVT_DATA_DATA_MASK, val);
-
-	write_sequnlock(&cache->data_seqlock);
-
-	/*
-	 * While PVT core is doing the next mode data conversion, we'll check
-	 * whether the alarms were triggered for the current sensor. Note that
-	 * according to the documentation only one threshold IRQ status can be
-	 * set at a time, that's why if-else statement is utilized.
-	 */
-	if ((thres_sts & info->thres_sts_lo) ^ cache->thres_sts_lo) {
-		WRITE_ONCE(cache->thres_sts_lo, thres_sts & info->thres_sts_lo);
-		hwmon_notify_event(pvt->hwmon, info->type, info->attr_min_alarm,
-				   info->channel);
-	} else if ((thres_sts & info->thres_sts_hi) ^ cache->thres_sts_hi) {
-		WRITE_ONCE(cache->thres_sts_hi, thres_sts & info->thres_sts_hi);
-		hwmon_notify_event(pvt->hwmon, info->type, info->attr_max_alarm,
-				   info->channel);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static inline umode_t pvt_limit_is_visible(enum pvt_sensor_type type)
-{
-	return 0644;
-}
-
-static inline umode_t pvt_alarm_is_visible(enum pvt_sensor_type type)
-{
-	return 0444;
-}
-
-static int pvt_read_data(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			 long *val)
-{
-	struct pvt_cache *cache = &pvt->cache[type];
-	unsigned int seq;
-	u32 data;
-
-	do {
-		seq = read_seqbegin(&cache->data_seqlock);
-		data = cache->data;
-	} while (read_seqretry(&cache->data_seqlock, seq));
-
-	if (type == PVT_TEMP)
-		*val = polynomial_calc(&poly_N_to_temp, data);
-	else
-		*val = polynomial_calc(&poly_N_to_volt, data);
-
-	return 0;
-}
-
-static int pvt_read_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			  bool is_low, long *val)
-{
-	u32 data;
-
-	/* No need in serialization, since it is just read from MMIO. */
-	data = readl(pvt->regs + pvt_info[type].thres_base);
-
-	if (is_low)
-		data = FIELD_GET(PVT_THRES_LO_MASK, data);
-	else
-		data = FIELD_GET(PVT_THRES_HI_MASK, data);
-
-	if (type == PVT_TEMP)
-		*val = polynomial_calc(&poly_N_to_temp, data);
-	else
-		*val = polynomial_calc(&poly_N_to_volt, data);
-
-	return 0;
-}
-
-static int pvt_write_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			   bool is_low, long val)
-{
-	u32 data, limit, mask;
-	int ret;
-
-	if (type == PVT_TEMP) {
-		val = clamp(val, PVT_TEMP_MIN, PVT_TEMP_MAX);
-		data = polynomial_calc(&poly_temp_to_N, val);
-	} else {
-		val = clamp(val, PVT_VOLT_MIN, PVT_VOLT_MAX);
-		data = polynomial_calc(&poly_volt_to_N, val);
-	}
-
-	/* Serialize limit update, since a part of the register is changed. */
-	ret = mutex_lock_interruptible(&pvt->iface_mtx);
-	if (ret)
-		return ret;
-
-	/* Make sure the upper and lower ranges don't intersect. */
-	limit = readl(pvt->regs + pvt_info[type].thres_base);
-	if (is_low) {
-		limit = FIELD_GET(PVT_THRES_HI_MASK, limit);
-		data = clamp_val(data, PVT_DATA_MIN, limit);
-		data = FIELD_PREP(PVT_THRES_LO_MASK, data);
-		mask = PVT_THRES_LO_MASK;
-	} else {
-		limit = FIELD_GET(PVT_THRES_LO_MASK, limit);
-		data = clamp_val(data, limit, PVT_DATA_MAX);
-		data = FIELD_PREP(PVT_THRES_HI_MASK, data);
-		mask = PVT_THRES_HI_MASK;
-	}
-
-	pvt_update(pvt->regs + pvt_info[type].thres_base, mask, data);
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	return 0;
-}
-
-static int pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			  bool is_low, long *val)
-{
-	if (is_low)
-		*val = !!READ_ONCE(pvt->cache[type].thres_sts_lo);
-	else
-		*val = !!READ_ONCE(pvt->cache[type].thres_sts_hi);
-
-	return 0;
-}
-
-static const struct hwmon_channel_info * const pvt_channel_info[] = {
-	HWMON_CHANNEL_INFO(chip,
-			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
-	HWMON_CHANNEL_INFO(temp,
-			   HWMON_T_INPUT | HWMON_T_TYPE | HWMON_T_LABEL |
-			   HWMON_T_MIN | HWMON_T_MIN_ALARM |
-			   HWMON_T_MAX | HWMON_T_MAX_ALARM |
-			   HWMON_T_OFFSET),
-	HWMON_CHANNEL_INFO(in,
-			   HWMON_I_INPUT | HWMON_I_LABEL |
-			   HWMON_I_MIN | HWMON_I_MIN_ALARM |
-			   HWMON_I_MAX | HWMON_I_MAX_ALARM,
-			   HWMON_I_INPUT | HWMON_I_LABEL |
-			   HWMON_I_MIN | HWMON_I_MIN_ALARM |
-			   HWMON_I_MAX | HWMON_I_MAX_ALARM,
-			   HWMON_I_INPUT | HWMON_I_LABEL |
-			   HWMON_I_MIN | HWMON_I_MIN_ALARM |
-			   HWMON_I_MAX | HWMON_I_MAX_ALARM,
-			   HWMON_I_INPUT | HWMON_I_LABEL |
-			   HWMON_I_MIN | HWMON_I_MIN_ALARM |
-			   HWMON_I_MAX | HWMON_I_MAX_ALARM),
-	NULL
-};
-
-#else /* !CONFIG_SENSORS_BT1_PVT_ALARMS */
-
-static irqreturn_t pvt_hard_isr(int irq, void *data)
-{
-	struct pvt_hwmon *pvt = data;
-	struct pvt_cache *cache;
-	u32 val;
-
-	/*
-	 * Mask the DVALID interrupt so after exiting from the handler a
-	 * repeated conversion wouldn't happen.
-	 */
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID,
-		   PVT_INTR_DVALID);
-
-	/*
-	 * Nothing special for alarm-less driver. Just read the data, update
-	 * the cache and notify a waiter of this event.
-	 */
-	val = readl(pvt->regs + PVT_DATA);
-	if (!(val & PVT_DATA_VALID)) {
-		dev_err(pvt->dev, "Got IRQ when data isn't valid\n");
-		return IRQ_HANDLED;
-	}
-
-	cache = &pvt->cache[pvt->sensor];
-
-	WRITE_ONCE(cache->data, FIELD_GET(PVT_DATA_DATA_MASK, val));
-
-	complete(&cache->conversion);
-
-	return IRQ_HANDLED;
-}
-
-#define pvt_soft_isr NULL
-
-static inline umode_t pvt_limit_is_visible(enum pvt_sensor_type type)
-{
-	return 0;
-}
-
-static inline umode_t pvt_alarm_is_visible(enum pvt_sensor_type type)
-{
-	return 0;
-}
-
-static int pvt_read_data(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			 long *val)
-{
-	struct pvt_cache *cache = &pvt->cache[type];
-	unsigned long timeout;
-	u32 data;
-	int ret;
-
-	/*
-	 * Lock PVT conversion interface until data cache is updated. The
-	 * data read procedure is following: set the requested PVT sensor
-	 * mode, enable IRQ and conversion, wait until conversion is finished,
-	 * then disable conversion and IRQ, and read the cached data.
-	 */
-	ret = mutex_lock_interruptible(&pvt->iface_mtx);
-	if (ret)
-		return ret;
-
-	pvt->sensor = type;
-	pvt_set_mode(pvt, pvt_info[type].mode);
-
-	/*
-	 * Unmask the DVALID interrupt and enable the sensors conversions.
-	 * Do the reverse procedure when conversion is done.
-	 */
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID, 0);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, PVT_CTRL_EN);
-
-	/*
-	 * Wait with timeout since in case if the sensor is suddenly powered
-	 * down the request won't be completed and the caller will hang up on
-	 * this procedure until the power is back up again. Multiply the
-	 * timeout by the factor of two to prevent a false timeout.
-	 */
-	timeout = 2 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
-	ret = wait_for_completion_timeout(&cache->conversion, timeout);
-
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID,
-		   PVT_INTR_DVALID);
-
-	data = READ_ONCE(cache->data);
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	if (!ret)
-		return -ETIMEDOUT;
-
-	if (type == PVT_TEMP)
-		*val = polynomial_calc(&poly_N_to_temp, data);
-	else
-		*val = polynomial_calc(&poly_N_to_volt, data);
-
-	return 0;
-}
-
-static int pvt_read_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			  bool is_low, long *val)
-{
-	return -EOPNOTSUPP;
-}
-
-static int pvt_write_limit(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			   bool is_low, long val)
-{
-	return -EOPNOTSUPP;
-}
-
-static int pvt_read_alarm(struct pvt_hwmon *pvt, enum pvt_sensor_type type,
-			  bool is_low, long *val)
-{
-	return -EOPNOTSUPP;
-}
-
-static const struct hwmon_channel_info * const pvt_channel_info[] = {
-	HWMON_CHANNEL_INFO(chip,
-			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
-	HWMON_CHANNEL_INFO(temp,
-			   HWMON_T_INPUT | HWMON_T_TYPE | HWMON_T_LABEL |
-			   HWMON_T_OFFSET),
-	HWMON_CHANNEL_INFO(in,
-			   HWMON_I_INPUT | HWMON_I_LABEL,
-			   HWMON_I_INPUT | HWMON_I_LABEL,
-			   HWMON_I_INPUT | HWMON_I_LABEL,
-			   HWMON_I_INPUT | HWMON_I_LABEL),
-	NULL
-};
-
-#endif /* !CONFIG_SENSORS_BT1_PVT_ALARMS */
-
-static inline bool pvt_hwmon_channel_is_valid(enum hwmon_sensor_types type,
-					      int ch)
-{
-	switch (type) {
-	case hwmon_temp:
-		if (ch < 0 || ch >= PVT_TEMP_CHS)
-			return false;
-		break;
-	case hwmon_in:
-		if (ch < 0 || ch >= PVT_VOLT_CHS)
-			return false;
-		break;
-	default:
-		break;
-	}
-
-	/* The rest of the types are independent from the channel number. */
-	return true;
-}
-
-static umode_t pvt_hwmon_is_visible(const void *data,
-				    enum hwmon_sensor_types type,
-				    u32 attr, int ch)
-{
-	if (!pvt_hwmon_channel_is_valid(type, ch))
-		return 0;
-
-	switch (type) {
-	case hwmon_chip:
-		switch (attr) {
-		case hwmon_chip_update_interval:
-			return 0644;
-		}
-		break;
-	case hwmon_temp:
-		switch (attr) {
-		case hwmon_temp_input:
-		case hwmon_temp_type:
-		case hwmon_temp_label:
-			return 0444;
-		case hwmon_temp_min:
-		case hwmon_temp_max:
-			return pvt_limit_is_visible(ch);
-		case hwmon_temp_min_alarm:
-		case hwmon_temp_max_alarm:
-			return pvt_alarm_is_visible(ch);
-		case hwmon_temp_offset:
-			return 0644;
-		}
-		break;
-	case hwmon_in:
-		switch (attr) {
-		case hwmon_in_input:
-		case hwmon_in_label:
-			return 0444;
-		case hwmon_in_min:
-		case hwmon_in_max:
-			return pvt_limit_is_visible(PVT_VOLT + ch);
-		case hwmon_in_min_alarm:
-		case hwmon_in_max_alarm:
-			return pvt_alarm_is_visible(PVT_VOLT + ch);
-		}
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static int pvt_read_trim(struct pvt_hwmon *pvt, long *val)
-{
-	u32 data;
-
-	data = readl(pvt->regs + PVT_CTRL);
-	*val = FIELD_GET(PVT_CTRL_TRIM_MASK, data) * PVT_TRIM_STEP;
-
-	return 0;
-}
-
-static int pvt_write_trim(struct pvt_hwmon *pvt, long val)
-{
-	u32 trim;
-	int ret;
-
-	/*
-	 * Serialize trim update, since a part of the register is changed and
-	 * the controller is supposed to be disabled during this operation.
-	 */
-	ret = mutex_lock_interruptible(&pvt->iface_mtx);
-	if (ret)
-		return ret;
-
-	trim = pvt_calc_trim(val);
-	pvt_set_trim(pvt, trim);
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	return 0;
-}
-
-static int pvt_read_timeout(struct pvt_hwmon *pvt, long *val)
-{
-	int ret;
-
-	ret = mutex_lock_interruptible(&pvt->iface_mtx);
-	if (ret)
-		return ret;
-
-	/* Return the result in msec as hwmon sysfs interface requires. */
-	*val = ktime_to_ms(pvt->timeout);
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	return 0;
-}
-
-static int pvt_write_timeout(struct pvt_hwmon *pvt, long val)
-{
-	unsigned long rate;
-	ktime_t kt, cache;
-	u32 data;
-	int ret;
-
-	rate = clk_get_rate(pvt->clks[PVT_CLOCK_REF].clk);
-	if (!rate)
-		return -ENODEV;
-
-	/*
-	 * If alarms are enabled, the requested timeout must be divided
-	 * between all available sensors to have the requested delay
-	 * applicable to each individual sensor.
-	 */
-	cache = kt = ms_to_ktime(val);
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-	kt = ktime_divns(kt, PVT_SENSORS_NUM);
-#endif
-
-	/*
-	 * Subtract a constant lag, which always persists due to the limited
-	 * PVT sampling rate. Make sure the timeout is not negative.
-	 */
-	kt = ktime_sub_ns(kt, PVT_TOUT_MIN);
-	if (ktime_to_ns(kt) < 0)
-		kt = ktime_set(0, 0);
-
-	/*
-	 * Finally recalculate the timeout in terms of the reference clock
-	 * period.
-	 */
-	data = ktime_divns(kt * rate, NSEC_PER_SEC);
-
-	/*
-	 * Update the measurements delay, but lock the interface first, since
-	 * we have to disable PVT in order to have the new delay actually
-	 * updated.
-	 */
-	ret = mutex_lock_interruptible(&pvt->iface_mtx);
-	if (ret)
-		return ret;
-
-	pvt_set_tout(pvt, data);
-	pvt->timeout = cache;
-
-	mutex_unlock(&pvt->iface_mtx);
-
-	return 0;
-}
-
-static int pvt_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
-			  u32 attr, int ch, long *val)
-{
-	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
-
-	if (!pvt_hwmon_channel_is_valid(type, ch))
-		return -EINVAL;
-
-	switch (type) {
-	case hwmon_chip:
-		switch (attr) {
-		case hwmon_chip_update_interval:
-			return pvt_read_timeout(pvt, val);
-		}
-		break;
-	case hwmon_temp:
-		switch (attr) {
-		case hwmon_temp_input:
-			return pvt_read_data(pvt, ch, val);
-		case hwmon_temp_type:
-			*val = 1;
-			return 0;
-		case hwmon_temp_min:
-			return pvt_read_limit(pvt, ch, true, val);
-		case hwmon_temp_max:
-			return pvt_read_limit(pvt, ch, false, val);
-		case hwmon_temp_min_alarm:
-			return pvt_read_alarm(pvt, ch, true, val);
-		case hwmon_temp_max_alarm:
-			return pvt_read_alarm(pvt, ch, false, val);
-		case hwmon_temp_offset:
-			return pvt_read_trim(pvt, val);
-		}
-		break;
-	case hwmon_in:
-		switch (attr) {
-		case hwmon_in_input:
-			return pvt_read_data(pvt, PVT_VOLT + ch, val);
-		case hwmon_in_min:
-			return pvt_read_limit(pvt, PVT_VOLT + ch, true, val);
-		case hwmon_in_max:
-			return pvt_read_limit(pvt, PVT_VOLT + ch, false, val);
-		case hwmon_in_min_alarm:
-			return pvt_read_alarm(pvt, PVT_VOLT + ch, true, val);
-		case hwmon_in_max_alarm:
-			return pvt_read_alarm(pvt, PVT_VOLT + ch, false, val);
-		}
-		break;
-	default:
-		break;
-	}
-
-	return -EOPNOTSUPP;
-}
-
-static int pvt_hwmon_read_string(struct device *dev,
-				 enum hwmon_sensor_types type,
-				 u32 attr, int ch, const char **str)
-{
-	if (!pvt_hwmon_channel_is_valid(type, ch))
-		return -EINVAL;
-
-	switch (type) {
-	case hwmon_temp:
-		switch (attr) {
-		case hwmon_temp_label:
-			*str = pvt_info[ch].label;
-			return 0;
-		}
-		break;
-	case hwmon_in:
-		switch (attr) {
-		case hwmon_in_label:
-			*str = pvt_info[PVT_VOLT + ch].label;
-			return 0;
-		}
-		break;
-	default:
-		break;
-	}
-
-	return -EOPNOTSUPP;
-}
-
-static int pvt_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
-			   u32 attr, int ch, long val)
-{
-	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
-
-	if (!pvt_hwmon_channel_is_valid(type, ch))
-		return -EINVAL;
-
-	switch (type) {
-	case hwmon_chip:
-		switch (attr) {
-		case hwmon_chip_update_interval:
-			return pvt_write_timeout(pvt, val);
-		}
-		break;
-	case hwmon_temp:
-		switch (attr) {
-		case hwmon_temp_min:
-			return pvt_write_limit(pvt, ch, true, val);
-		case hwmon_temp_max:
-			return pvt_write_limit(pvt, ch, false, val);
-		case hwmon_temp_offset:
-			return pvt_write_trim(pvt, val);
-		}
-		break;
-	case hwmon_in:
-		switch (attr) {
-		case hwmon_in_min:
-			return pvt_write_limit(pvt, PVT_VOLT + ch, true, val);
-		case hwmon_in_max:
-			return pvt_write_limit(pvt, PVT_VOLT + ch, false, val);
-		}
-		break;
-	default:
-		break;
-	}
-
-	return -EOPNOTSUPP;
-}
-
-static const struct hwmon_ops pvt_hwmon_ops = {
-	.is_visible = pvt_hwmon_is_visible,
-	.read = pvt_hwmon_read,
-	.read_string = pvt_hwmon_read_string,
-	.write = pvt_hwmon_write
-};
-
-static const struct hwmon_chip_info pvt_hwmon_info = {
-	.ops = &pvt_hwmon_ops,
-	.info = pvt_channel_info
-};
-
-static void pvt_clear_data(void *data)
-{
-	struct pvt_hwmon *pvt = data;
-#if !defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-	int idx;
-
-	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
-		complete_all(&pvt->cache[idx].conversion);
-#endif
-
-	mutex_destroy(&pvt->iface_mtx);
-}
-
-static struct pvt_hwmon *pvt_create_data(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct pvt_hwmon *pvt;
-	int ret, idx;
-
-	pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
-	if (!pvt)
-		return ERR_PTR(-ENOMEM);
-
-	ret = devm_add_action(dev, pvt_clear_data, pvt);
-	if (ret) {
-		dev_err(dev, "Can't add PVT data clear action\n");
-		return ERR_PTR(ret);
-	}
-
-	pvt->dev = dev;
-	pvt->sensor = PVT_SENSOR_FIRST;
-	mutex_init(&pvt->iface_mtx);
-
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
-		seqlock_init(&pvt->cache[idx].data_seqlock);
-#else
-	for (idx = 0; idx < PVT_SENSORS_NUM; ++idx)
-		init_completion(&pvt->cache[idx].conversion);
-#endif
-
-	return pvt;
-}
-
-static int pvt_request_regs(struct pvt_hwmon *pvt)
-{
-	struct platform_device *pdev = to_platform_device(pvt->dev);
-
-	pvt->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(pvt->regs))
-		return PTR_ERR(pvt->regs);
-
-	return 0;
-}
-
-static void pvt_disable_clks(void *data)
-{
-	struct pvt_hwmon *pvt = data;
-
-	clk_bulk_disable_unprepare(PVT_CLOCK_NUM, pvt->clks);
-}
-
-static int pvt_request_clks(struct pvt_hwmon *pvt)
-{
-	int ret;
-
-	pvt->clks[PVT_CLOCK_APB].id = "pclk";
-	pvt->clks[PVT_CLOCK_REF].id = "ref";
-
-	ret = devm_clk_bulk_get(pvt->dev, PVT_CLOCK_NUM, pvt->clks);
-	if (ret) {
-		dev_err(pvt->dev, "Couldn't get PVT clocks descriptors\n");
-		return ret;
-	}
-
-	ret = clk_bulk_prepare_enable(PVT_CLOCK_NUM, pvt->clks);
-	if (ret) {
-		dev_err(pvt->dev, "Couldn't enable the PVT clocks\n");
-		return ret;
-	}
-
-	ret = devm_add_action_or_reset(pvt->dev, pvt_disable_clks, pvt);
-	if (ret) {
-		dev_err(pvt->dev, "Can't add PVT clocks disable action\n");
-		return ret;
-	}
-
-	return 0;
-}
-
-static int pvt_check_pwr(struct pvt_hwmon *pvt)
-{
-	unsigned long tout;
-	int ret = 0;
-	u32 data;
-
-	/*
-	 * Test out the sensor conversion functionality. If it is not done on
-	 * time then the domain must have been unpowered and we won't be able
-	 * to use the device later in this driver.
-	 * Note If the power source is lost during the normal driver work the
-	 * data read procedure will either return -ETIMEDOUT (for the
-	 * alarm-less driver configuration) or just stop the repeated
-	 * conversion. In the later case alas we won't be able to detect the
-	 * problem.
-	 */
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_ALL, PVT_INTR_ALL);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, PVT_CTRL_EN);
-	pvt_set_tout(pvt, 0);
-	readl(pvt->regs + PVT_DATA);
-
-	tout = PVT_TOUT_MIN / NSEC_PER_USEC;
-	usleep_range(tout, 2 * tout);
-
-	data = readl(pvt->regs + PVT_DATA);
-	if (!(data & PVT_DATA_VALID)) {
-		ret = -ENODEV;
-		dev_err(pvt->dev, "Sensor is powered down\n");
-	}
-
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-
-	return ret;
-}
-
-static int pvt_init_iface(struct pvt_hwmon *pvt)
-{
-	unsigned long rate;
-	u32 trim, temp;
-
-	rate = clk_get_rate(pvt->clks[PVT_CLOCK_REF].clk);
-	if (!rate) {
-		dev_err(pvt->dev, "Invalid reference clock rate\n");
-		return -ENODEV;
-	}
-
-	/*
-	 * Make sure all interrupts and controller are disabled so not to
-	 * accidentally have ISR executed before the driver data is fully
-	 * initialized. Clear the IRQ status as well.
-	 */
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_ALL, PVT_INTR_ALL);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	readl(pvt->regs + PVT_CLR_INTR);
-	readl(pvt->regs + PVT_DATA);
-
-	/* Setup default sensor mode, timeout and temperature trim. */
-	pvt_set_mode(pvt, pvt_info[pvt->sensor].mode);
-	pvt_set_tout(pvt, PVT_TOUT_DEF);
-
-	/*
-	 * Preserve the current ref-clock based delay (Ttotal) between the
-	 * sensors data samples in the driver data so not to recalculate it
-	 * each time on the data requests and timeout reads. It consists of the
-	 * delay introduced by the internal ref-clock timer (N / Fclk) and the
-	 * constant timeout caused by each conversion latency (Tmin):
-	 *   Ttotal = N / Fclk + Tmin
-	 * If alarms are enabled the sensors are polled one after another and
-	 * in order to get the next measurement of a particular sensor the
-	 * caller will have to wait for at most until all the others are
-	 * polled. In that case the formulae will look a bit different:
-	 *   Ttotal = 5 * (N / Fclk + Tmin)
-	 */
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-	pvt->timeout = ktime_set(PVT_SENSORS_NUM * PVT_TOUT_DEF, 0);
-	pvt->timeout = ktime_divns(pvt->timeout, rate);
-	pvt->timeout = ktime_add_ns(pvt->timeout, PVT_SENSORS_NUM * PVT_TOUT_MIN);
-#else
-	pvt->timeout = ktime_set(PVT_TOUT_DEF, 0);
-	pvt->timeout = ktime_divns(pvt->timeout, rate);
-	pvt->timeout = ktime_add_ns(pvt->timeout, PVT_TOUT_MIN);
-#endif
-
-	trim = PVT_TRIM_DEF;
-	if (!of_property_read_u32(pvt->dev->of_node,
-	     "baikal,pvt-temp-offset-millicelsius", &temp))
-		trim = pvt_calc_trim(temp);
-
-	pvt_set_trim(pvt, trim);
-
-	return 0;
-}
-
-static int pvt_request_irq(struct pvt_hwmon *pvt)
-{
-	struct platform_device *pdev = to_platform_device(pvt->dev);
-	int ret;
-
-	pvt->irq = platform_get_irq(pdev, 0);
-	if (pvt->irq < 0)
-		return pvt->irq;
-
-	ret = devm_request_threaded_irq(pvt->dev, pvt->irq,
-					pvt_hard_isr, pvt_soft_isr,
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-					IRQF_SHARED | IRQF_TRIGGER_HIGH |
-					IRQF_ONESHOT,
-#else
-					IRQF_SHARED | IRQF_TRIGGER_HIGH,
-#endif
-					"pvt", pvt);
-	if (ret) {
-		dev_err(pvt->dev, "Couldn't request PVT IRQ\n");
-		return ret;
-	}
-
-	return 0;
-}
-
-static int pvt_create_hwmon(struct pvt_hwmon *pvt)
-{
-	pvt->hwmon = devm_hwmon_device_register_with_info(pvt->dev, "pvt", pvt,
-		&pvt_hwmon_info, NULL);
-	if (IS_ERR(pvt->hwmon)) {
-		dev_err(pvt->dev, "Couldn't create hwmon device\n");
-		return PTR_ERR(pvt->hwmon);
-	}
-
-	return 0;
-}
-
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-
-static void pvt_disable_iface(void *data)
-{
-	struct pvt_hwmon *pvt = data;
-
-	mutex_lock(&pvt->iface_mtx);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID,
-		   PVT_INTR_DVALID);
-	mutex_unlock(&pvt->iface_mtx);
-}
-
-static int pvt_enable_iface(struct pvt_hwmon *pvt)
-{
-	int ret;
-
-	ret = devm_add_action(pvt->dev, pvt_disable_iface, pvt);
-	if (ret) {
-		dev_err(pvt->dev, "Can't add PVT disable interface action\n");
-		return ret;
-	}
-
-	/*
-	 * Enable sensors data conversion and IRQ. We need to lock the
-	 * interface mutex since hwmon has just been created and the
-	 * corresponding sysfs files are accessible from user-space,
-	 * which theoretically may cause races.
-	 */
-	mutex_lock(&pvt->iface_mtx);
-	pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID, 0);
-	pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, PVT_CTRL_EN);
-	mutex_unlock(&pvt->iface_mtx);
-
-	return 0;
-}
-
-#else /* !CONFIG_SENSORS_BT1_PVT_ALARMS */
-
-static int pvt_enable_iface(struct pvt_hwmon *pvt)
-{
-	return 0;
-}
-
-#endif /* !CONFIG_SENSORS_BT1_PVT_ALARMS */
-
-static int pvt_probe(struct platform_device *pdev)
-{
-	struct pvt_hwmon *pvt;
-	int ret;
-
-	pvt = pvt_create_data(pdev);
-	if (IS_ERR(pvt))
-		return PTR_ERR(pvt);
-
-	ret = pvt_request_regs(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_request_clks(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_check_pwr(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_init_iface(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_request_irq(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_create_hwmon(pvt);
-	if (ret)
-		return ret;
-
-	ret = pvt_enable_iface(pvt);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static const struct of_device_id pvt_of_match[] = {
-	{ .compatible = "baikal,bt1-pvt" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, pvt_of_match);
-
-static struct platform_driver pvt_driver = {
-	.probe = pvt_probe,
-	.driver = {
-		.name = "bt1-pvt",
-		.of_match_table = pvt_of_match
-	}
-};
-module_platform_driver(pvt_driver);
-
-MODULE_AUTHOR("Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru>");
-MODULE_DESCRIPTION("Baikal-T1 PVT driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/hwmon/bt1-pvt.h b/drivers/hwmon/bt1-pvt.h
deleted file mode 100644
index 93b8dd5e7c944..0000000000000
--- a/drivers/hwmon/bt1-pvt.h
+++ /dev/null
@@ -1,247 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Baikal-T1 Process, Voltage, Temperature sensor driver
- */
-#ifndef __HWMON_BT1_PVT_H__
-#define __HWMON_BT1_PVT_H__
-
-#include <linux/completion.h>
-#include <linux/hwmon.h>
-#include <linux/kernel.h>
-#include <linux/ktime.h>
-#include <linux/mutex.h>
-#include <linux/seqlock.h>
-
-/* Baikal-T1 PVT registers and their bitfields */
-#define PVT_CTRL			0x00
-#define PVT_CTRL_EN			BIT(0)
-#define PVT_CTRL_MODE_FLD		1
-#define PVT_CTRL_MODE_MASK		GENMASK(3, PVT_CTRL_MODE_FLD)
-#define PVT_CTRL_MODE_TEMP		0x0
-#define PVT_CTRL_MODE_VOLT		0x1
-#define PVT_CTRL_MODE_LVT		0x2
-#define PVT_CTRL_MODE_HVT		0x4
-#define PVT_CTRL_MODE_SVT		0x6
-#define PVT_CTRL_TRIM_FLD		4
-#define PVT_CTRL_TRIM_MASK		GENMASK(8, PVT_CTRL_TRIM_FLD)
-#define PVT_DATA			0x04
-#define PVT_DATA_VALID			BIT(10)
-#define PVT_DATA_DATA_FLD		0
-#define PVT_DATA_DATA_MASK		GENMASK(9, PVT_DATA_DATA_FLD)
-#define PVT_TTHRES			0x08
-#define PVT_VTHRES			0x0C
-#define PVT_LTHRES			0x10
-#define PVT_HTHRES			0x14
-#define PVT_STHRES			0x18
-#define PVT_THRES_LO_FLD		0
-#define PVT_THRES_LO_MASK		GENMASK(9, PVT_THRES_LO_FLD)
-#define PVT_THRES_HI_FLD		10
-#define PVT_THRES_HI_MASK		GENMASK(19, PVT_THRES_HI_FLD)
-#define PVT_TTIMEOUT			0x1C
-#define PVT_INTR_STAT			0x20
-#define PVT_INTR_MASK			0x24
-#define PVT_RAW_INTR_STAT		0x28
-#define PVT_INTR_DVALID			BIT(0)
-#define PVT_INTR_TTHRES_LO		BIT(1)
-#define PVT_INTR_TTHRES_HI		BIT(2)
-#define PVT_INTR_VTHRES_LO		BIT(3)
-#define PVT_INTR_VTHRES_HI		BIT(4)
-#define PVT_INTR_LTHRES_LO		BIT(5)
-#define PVT_INTR_LTHRES_HI		BIT(6)
-#define PVT_INTR_HTHRES_LO		BIT(7)
-#define PVT_INTR_HTHRES_HI		BIT(8)
-#define PVT_INTR_STHRES_LO		BIT(9)
-#define PVT_INTR_STHRES_HI		BIT(10)
-#define PVT_INTR_ALL			GENMASK(10, 0)
-#define PVT_CLR_INTR			0x2C
-
-/*
- * PVT sensors-related limits and default values
- * @PVT_TEMP_MIN: Minimal temperature in millidegrees of Celsius.
- * @PVT_TEMP_MAX: Maximal temperature in millidegrees of Celsius.
- * @PVT_TEMP_CHS: Number of temperature hwmon channels.
- * @PVT_VOLT_MIN: Minimal voltage in mV.
- * @PVT_VOLT_MAX: Maximal voltage in mV.
- * @PVT_VOLT_CHS: Number of voltage hwmon channels.
- * @PVT_DATA_MIN: Minimal PVT raw data value.
- * @PVT_DATA_MAX: Maximal PVT raw data value.
- * @PVT_TRIM_MIN: Minimal temperature sensor trim value.
- * @PVT_TRIM_MAX: Maximal temperature sensor trim value.
- * @PVT_TRIM_DEF: Default temperature sensor trim value (set a proper value
- *		  when one is determined for Baikal-T1 SoC).
- * @PVT_TRIM_TEMP: Maximum temperature encoded by the trim factor.
- * @PVT_TRIM_STEP: Temperature stride corresponding to the trim value.
- * @PVT_TOUT_MIN: Minimal timeout between samples in nanoseconds.
- * @PVT_TOUT_DEF: Default data measurements timeout. In case if alarms are
- *		  activated the PVT IRQ is enabled to be raised after each
- *		  conversion in order to have the thresholds checked and the
- *		  converted value cached. Too frequent conversions may cause
- *		  the system CPU overload. Lets set the 50ms delay between
- *		  them by default to prevent this.
- */
-#define PVT_TEMP_MIN		-48380L
-#define PVT_TEMP_MAX		147438L
-#define PVT_TEMP_CHS		1
-#define PVT_VOLT_MIN		620L
-#define PVT_VOLT_MAX		1168L
-#define PVT_VOLT_CHS		4
-#define PVT_DATA_MIN		0
-#define PVT_DATA_MAX		(PVT_DATA_DATA_MASK >> PVT_DATA_DATA_FLD)
-#define PVT_TRIM_MIN		0
-#define PVT_TRIM_MAX		(PVT_CTRL_TRIM_MASK >> PVT_CTRL_TRIM_FLD)
-#define PVT_TRIM_TEMP		7130
-#define PVT_TRIM_STEP		(PVT_TRIM_TEMP / PVT_TRIM_MAX)
-#define PVT_TRIM_DEF		0
-#define PVT_TOUT_MIN		(NSEC_PER_SEC / 3000)
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-# define PVT_TOUT_DEF		60000
-#else
-# define PVT_TOUT_DEF		0
-#endif
-
-/*
- * enum pvt_sensor_type - Baikal-T1 PVT sensor types (correspond to each PVT
- *			  sampling mode)
- * @PVT_SENSOR*: helpers to traverse the sensors in loops.
- * @PVT_TEMP: PVT Temperature sensor.
- * @PVT_VOLT: PVT Voltage sensor.
- * @PVT_LVT: PVT Low-Voltage threshold sensor.
- * @PVT_HVT: PVT High-Voltage threshold sensor.
- * @PVT_SVT: PVT Standard-Voltage threshold sensor.
- */
-enum pvt_sensor_type {
-	PVT_SENSOR_FIRST,
-	PVT_TEMP = PVT_SENSOR_FIRST,
-	PVT_VOLT,
-	PVT_LVT,
-	PVT_HVT,
-	PVT_SVT,
-	PVT_SENSOR_LAST = PVT_SVT,
-	PVT_SENSORS_NUM
-};
-
-/*
- * enum pvt_clock_type - Baikal-T1 PVT clocks.
- * @PVT_CLOCK_APB: APB clock.
- * @PVT_CLOCK_REF: PVT reference clock.
- */
-enum pvt_clock_type {
-	PVT_CLOCK_APB,
-	PVT_CLOCK_REF,
-	PVT_CLOCK_NUM
-};
-
-/*
- * struct pvt_sensor_info - Baikal-T1 PVT sensor informational structure
- * @channel: Sensor channel ID.
- * @label: hwmon sensor label.
- * @mode: PVT mode corresponding to the channel.
- * @thres_base: upper and lower threshold values of the sensor.
- * @thres_sts_lo: low threshold status bitfield.
- * @thres_sts_hi: high threshold status bitfield.
- * @type: Sensor type.
- * @attr_min_alarm: Min alarm attribute ID.
- * @attr_min_alarm: Max alarm attribute ID.
- */
-struct pvt_sensor_info {
-	int channel;
-	const char *label;
-	u32 mode;
-	unsigned long thres_base;
-	u32 thres_sts_lo;
-	u32 thres_sts_hi;
-	enum hwmon_sensor_types type;
-	u32 attr_min_alarm;
-	u32 attr_max_alarm;
-};
-
-#define PVT_SENSOR_INFO(_ch, _label, _type, _mode, _thres)	\
-	{							\
-		.channel = _ch,					\
-		.label = _label,				\
-		.mode = PVT_CTRL_MODE_ ##_mode,			\
-		.thres_base = PVT_ ##_thres,			\
-		.thres_sts_lo = PVT_INTR_ ##_thres## _LO,	\
-		.thres_sts_hi = PVT_INTR_ ##_thres## _HI,	\
-		.type = _type,					\
-		.attr_min_alarm = _type## _min,			\
-		.attr_max_alarm = _type## _max,			\
-	}
-
-/*
- * struct pvt_cache - PVT sensors data cache
- * @data: data cache in raw format.
- * @thres_sts_lo: low threshold status saved on the previous data conversion.
- * @thres_sts_hi: high threshold status saved on the previous data conversion.
- * @data_seqlock: cached data seq-lock.
- * @conversion: data conversion completion.
- */
-struct pvt_cache {
-	u32 data;
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
-	seqlock_t data_seqlock;
-	u32 thres_sts_lo;
-	u32 thres_sts_hi;
-#else
-	struct completion conversion;
-#endif
-};
-
-/*
- * struct pvt_hwmon - Baikal-T1 PVT private data
- * @dev: device structure of the PVT platform device.
- * @hwmon: hwmon device structure.
- * @regs: pointer to the Baikal-T1 PVT registers region.
- * @irq: PVT events IRQ number.
- * @clks: Array of the PVT clocks descriptor (APB/ref clocks).
- * @ref_clk: Pointer to the reference clocks descriptor.
- * @iface_mtx: Generic interface mutex (used to lock the alarm registers
- *	       when the alarms enabled, or the data conversion interface
- *	       if alarms are disabled).
- * @sensor: current PVT sensor the data conversion is being performed for.
- * @cache: data cache descriptor.
- * @timeout: conversion timeout cache.
- */
-struct pvt_hwmon {
-	struct device *dev;
-	struct device *hwmon;
-
-	void __iomem *regs;
-	int irq;
-
-	struct clk_bulk_data clks[PVT_CLOCK_NUM];
-
-	struct mutex iface_mtx;
-	enum pvt_sensor_type sensor;
-	struct pvt_cache cache[PVT_SENSORS_NUM];
-	ktime_t timeout;
-};
-
-/*
- * struct pvt_poly_term - a term descriptor of the PVT data translation
- *			  polynomial
- * @deg: degree of the term.
- * @coef: multiplication factor of the term.
- * @divider: distributed divider per each degree.
- * @divider_leftover: divider leftover, which couldn't be redistributed.
- */
-struct pvt_poly_term {
-	unsigned int deg;
-	long coef;
-	long divider;
-	long divider_leftover;
-};
-
-/*
- * struct pvt_poly - PVT data translation polynomial descriptor
- * @total_divider: total data divider.
- * @terms: polynomial terms up to a free one.
- */
-struct pvt_poly {
-	long total_divider;
-	struct pvt_poly_term terms[];
-};
-
-#endif /* __HWMON_BT1_PVT_H__ */
diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c
index ccce948a4306e..964a8cb278f16 100644
--- a/drivers/hwmon/emc1403.c
+++ b/drivers/hwmon/emc1403.c
@@ -40,7 +40,7 @@ static ssize_t power_state_show(struct device *dev, struct device_attribute *att
 	retval = regmap_read(data->regmap, 0x03, &val);
 	if (retval < 0)
 		return retval;
-	return sprintf(buf, "%d\n", !!(val & BIT(6)));
+	return sysfs_emit(buf, "%d\n", !!(val & BIT(6)));
 }
 
 static ssize_t power_state_store(struct device *dev, struct device_attribute *attr,
diff --git a/drivers/hwmon/gpd-fan.c b/drivers/hwmon/gpd-fan.c
index 1729729b135f8..80de5f20781eb 100644
--- a/drivers/hwmon/gpd-fan.c
+++ b/drivers/hwmon/gpd-fan.c
@@ -209,6 +209,14 @@ static const struct dmi_system_id dmi_table[] = {
 		},
 		.driver_data = &gpd_duo_drvdata,
 	},
+	{
+		// GPD Win 5 with AMD AI MAX 395
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "GPD"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "G1618-05"),
+		},
+		.driver_data = &gpd_duo_drvdata,
+	},
 	{
 		// GPD Pocket 4
 		.matches = {
diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c
index 69ac0468dee4f..613ffb622b7c4 100644
--- a/drivers/hwmon/ina2xx.c
+++ b/drivers/hwmon/ina2xx.c
@@ -1,22 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Driver for Texas Instruments INA219, INA226 power monitor chips
- *
- * INA219:
- * Zero Drift Bi-Directional Current/Power Monitor with I2C Interface
- * Datasheet: https://www.ti.com/product/ina219
- *
- * INA220:
- * Bi-Directional Current/Power Monitor with I2C Interface
- * Datasheet: https://www.ti.com/product/ina220
- *
- * INA226:
- * Bi-Directional Current/Power Monitor with I2C Interface
- * Datasheet: https://www.ti.com/product/ina226
- *
- * INA230:
- * Bi-directional Current/Power Monitor with I2C Interface
- * Datasheet: https://www.ti.com/product/ina230
+ * Driver for Texas Instruments INA219, INA226 and register-layout compatible
+ * current/power monitor chips with I2C Interface
  *
  * Copyright (C) 2012 Lothar Felten <lothar.felten@gmail.com>
  * Thanks to Jan Volkering
@@ -49,7 +34,6 @@
 /* INA226 register definitions */
 #define INA226_MASK_ENABLE		0x06
 #define INA226_ALERT_LIMIT		0x07
-#define INA226_DIE_ID			0xFF
 
 /* SY24655 register definitions */
 #define SY24655_EIN				0x0A
@@ -135,18 +119,27 @@ static const struct regmap_config ina2xx_regmap_config = {
 	.writeable_reg = ina2xx_writeable_reg,
 };
 
-enum ina2xx_ids { ina219, ina226, ina260, sy24655 };
+enum ina2xx_ids {
+	ina219,
+	ina226,
+	ina234,
+	ina260,
+	sy24655
+};
 
 struct ina2xx_config {
 	u16 config_default;
 	bool has_alerts;	/* chip supports alerts and limits */
 	bool has_ishunt;	/* chip has internal shunt resistor */
-	bool has_power_average;	/* chip has internal shunt resistor */
+	bool has_power_average;	/* chip supports average power */
+	bool has_update_interval;
 	int calibration_value;
 	int shunt_div;
+	int shunt_voltage_shift;
 	int bus_voltage_shift;
 	int bus_voltage_lsb;	/* uV */
 	int power_lsb_factor;
+	int current_shift;
 };
 
 struct ina2xx_data {
@@ -165,44 +158,70 @@ static const struct ina2xx_config ina2xx_config[] = {
 		.config_default = INA219_CONFIG_DEFAULT,
 		.calibration_value = 4096,
 		.shunt_div = 100,
+		.shunt_voltage_shift = 0,
 		.bus_voltage_shift = 3,
 		.bus_voltage_lsb = 4000,
 		.power_lsb_factor = 20,
 		.has_alerts = false,
 		.has_ishunt = false,
 		.has_power_average = false,
+		.current_shift = 0,
+		.has_update_interval = false,
 	},
 	[ina226] = {
 		.config_default = INA226_CONFIG_DEFAULT,
 		.calibration_value = 2048,
 		.shunt_div = 400,
+		.shunt_voltage_shift = 0,
 		.bus_voltage_shift = 0,
 		.bus_voltage_lsb = 1250,
 		.power_lsb_factor = 25,
 		.has_alerts = true,
 		.has_ishunt = false,
 		.has_power_average = false,
+		.current_shift = 0,
+		.has_update_interval = true,
+	},
+	[ina234] = {
+		.config_default = INA226_CONFIG_DEFAULT,
+		.calibration_value = 2048,
+		.shunt_div = 25, /* 2.5 µV/LSB raw ADC reading from INA2XX_SHUNT_VOLTAGE */
+		.shunt_voltage_shift = 4,
+		.bus_voltage_shift = 4,
+		.bus_voltage_lsb = 25600,
+		.power_lsb_factor = 32,
+		.has_alerts = true,
+		.has_ishunt = false,
+		.has_power_average = false,
+		.current_shift = 4,
+		.has_update_interval = true,
 	},
 	[ina260] = {
 		.config_default = INA260_CONFIG_DEFAULT,
 		.shunt_div = 400,
+		.shunt_voltage_shift = 0,
 		.bus_voltage_shift = 0,
 		.bus_voltage_lsb = 1250,
 		.power_lsb_factor = 8,
 		.has_alerts = true,
 		.has_ishunt = true,
 		.has_power_average = false,
+		.current_shift = 0,
+		.has_update_interval = true,
 	},
 	[sy24655] = {
 		.config_default = SY24655_CONFIG_DEFAULT,
 		.calibration_value = 4096,
 		.shunt_div = 400,
+		.shunt_voltage_shift = 0,
 		.bus_voltage_shift = 0,
 		.bus_voltage_lsb = 1250,
 		.power_lsb_factor = 25,
 		.has_alerts = true,
 		.has_ishunt = false,
 		.has_power_average = true,
+		.current_shift = 0,
+		.has_update_interval = false,
 	},
 };
 
@@ -255,7 +274,8 @@ static int ina2xx_get_value(struct ina2xx_data *data, u8 reg,
 	switch (reg) {
 	case INA2XX_SHUNT_VOLTAGE:
 		/* signed register */
-		val = DIV_ROUND_CLOSEST((s16)regval, data->config->shunt_div);
+		val = (s16)regval >> data->config->shunt_voltage_shift;
+		val = DIV_ROUND_CLOSEST(val, data->config->shunt_div);
 		break;
 	case INA2XX_BUS_VOLTAGE:
 		val = (regval >> data->config->bus_voltage_shift) *
@@ -267,7 +287,8 @@ static int ina2xx_get_value(struct ina2xx_data *data, u8 reg,
 		break;
 	case INA2XX_CURRENT:
 		/* signed register, result in mA */
-		val = (s16)regval * data->current_lsb_uA;
+		val = ((s16)regval >> data->config->current_shift) *
+		  data->current_lsb_uA;
 		val = DIV_ROUND_CLOSEST(val, 1000);
 		break;
 	case INA2XX_CALIBRATION:
@@ -361,6 +382,7 @@ static u16 ina226_alert_to_reg(struct ina2xx_data *data, int reg, long val)
 	case INA2XX_SHUNT_VOLTAGE:
 		val = clamp_val(val, 0, SHRT_MAX * data->config->shunt_div);
 		val *= data->config->shunt_div;
+		val <<= data->config->shunt_voltage_shift;
 		return clamp_val(val, 0, SHRT_MAX);
 	case INA2XX_BUS_VOLTAGE:
 		val = clamp_val(val, 0, 200000);
@@ -375,6 +397,7 @@ static u16 ina226_alert_to_reg(struct ina2xx_data *data, int reg, long val)
 		val = clamp_val(val, INT_MIN / 1000, INT_MAX / 1000);
 		/* signed register, result in mA */
 		val = DIV_ROUND_CLOSEST(val * 1000, data->current_lsb_uA);
+		val <<= data->config->current_shift;
 		return clamp_val(val, SHRT_MIN, SHRT_MAX);
 	default:
 		/* programmer goofed */
@@ -706,7 +729,7 @@ static umode_t ina2xx_is_visible(const void *_data, enum hwmon_sensor_types type
 	const struct ina2xx_data *data = _data;
 	bool has_alerts = data->config->has_alerts;
 	bool has_power_average = data->config->has_power_average;
-	enum ina2xx_ids chip = data->chip;
+	bool has_update_interval = data->config->has_update_interval;
 
 	switch (type) {
 	case hwmon_in:
@@ -768,7 +791,7 @@ static umode_t ina2xx_is_visible(const void *_data, enum hwmon_sensor_types type
 	case hwmon_chip:
 		switch (attr) {
 		case hwmon_chip_update_interval:
-			if (chip == ina226 || chip == ina260)
+			if (has_update_interval)
 				return 0644;
 			break;
 		default:
@@ -982,6 +1005,7 @@ static const struct i2c_device_id ina2xx_id[] = {
 	{ "ina226", ina226 },
 	{ "ina230", ina226 },
 	{ "ina231", ina226 },
+	{ "ina234", ina234 },
 	{ "ina260", ina260 },
 	{ "sy24655", sy24655 },
 	{ }
@@ -1013,6 +1037,10 @@ static const struct of_device_id __maybe_unused ina2xx_of_match[] = {
 		.compatible = "ti,ina231",
 		.data = (void *)ina226
 	},
+	{
+		.compatible = "ti,ina234",
+		.data = (void *)ina234
+	},
 	{
 		.compatible = "ti,ina260",
 		.data = (void *)ina260
diff --git a/drivers/hwmon/lattepanda-sigma-ec.c b/drivers/hwmon/lattepanda-sigma-ec.c
new file mode 100644
index 0000000000000..f060974222019
--- /dev/null
+++ b/drivers/hwmon/lattepanda-sigma-ec.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Hardware monitoring driver for LattePanda Sigma EC.
+ *
+ * The LattePanda Sigma is an x86 SBC made by DFRobot with an ITE IT8613E
+ * Embedded Controller that manages a CPU fan and thermal sensors.
+ *
+ * The BIOS declares the ACPI Embedded Controller (PNP0C09) with _STA
+ * returning 0 and provides only stub ECRD/ECWT methods that return Zero
+ * for all registers. Since the kernel's ACPI EC subsystem never initializes,
+ * ec_read() is not available and direct port I/O to the standard ACPI EC
+ * ports (0x62/0x66) is used instead.
+ *
+ * Because ACPI never initializes the EC, there is no concurrent firmware
+ * access to these ports, and no ACPI Global Lock or namespace mutex is
+ * required. The hwmon with_info API serializes all sysfs callbacks,
+ * so no additional driver-level locking is needed.
+ *
+ * The EC register map was discovered by dumping all 256 registers,
+ * identifying those that change in real-time, and validating by physically
+ * stopping the fan and observing the RPM register drop to zero. The map
+ * has been verified on BIOS version 5.27; other versions may differ.
+ *
+ * Copyright (c) 2026 Mariano Abad <weimaraner@gmail.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/dmi.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#define DRIVER_NAME	"lattepanda_sigma_ec"
+
+/* EC I/O ports (standard ACPI EC interface) */
+#define EC_DATA_PORT	0x62
+#define EC_CMD_PORT	0x66	/* also status port */
+
+/* EC commands */
+#define EC_CMD_READ	0x80
+
+/* EC status register bits */
+#define EC_STATUS_OBF	0x01	/* Output Buffer Full */
+#define EC_STATUS_IBF	0x02	/* Input Buffer Full */
+
+/* EC register offsets for LattePanda Sigma (BIOS 5.27) */
+#define EC_REG_FAN_RPM_HI	0x2E
+#define EC_REG_FAN_RPM_LO	0x2F
+#define EC_REG_TEMP_BOARD	0x60
+#define EC_REG_TEMP_CPU		0x70
+#define EC_REG_FAN_DUTY		0x93
+
+/*
+ * EC polling uses udelay() because the EC typically responds within a
+ * few microseconds. The kernel's own ACPI EC driver (drivers/acpi/ec.c)
+ * likewise uses udelay() for busy-polling with a per-poll delay of 550us.
+ *
+ * usleep_range() was tested but caused EC protocol failures: the EC
+ * clears its status flags within microseconds, and sleeping for 50-100us
+ * between polls allowed the flags to transition past the expected state.
+ *
+ * The worst-case total busy-wait of 25ms covers EC recovery after errors.
+ * In practice the EC responds within 10us so the loop exits immediately.
+ */
+#define EC_TIMEOUT_US		25000
+#define EC_POLL_US		1
+
+static bool force;
+module_param(force, bool, 0444);
+MODULE_PARM_DESC(force,
+		 "Force loading on untested BIOS versions (default: false)");
+
+static struct platform_device *lps_ec_pdev;
+
+static int ec_wait_ibf_clear(void)
+{
+	int i;
+
+	for (i = 0; i < EC_TIMEOUT_US; i++) {
+		if (!(inb(EC_CMD_PORT) & EC_STATUS_IBF))
+			return 0;
+		udelay(EC_POLL_US);
+	}
+	return -ETIMEDOUT;
+}
+
+static int ec_wait_obf_set(void)
+{
+	int i;
+
+	for (i = 0; i < EC_TIMEOUT_US; i++) {
+		if (inb(EC_CMD_PORT) & EC_STATUS_OBF)
+			return 0;
+		udelay(EC_POLL_US);
+	}
+	return -ETIMEDOUT;
+}
+
+static int ec_read_reg(u8 reg, u8 *val)
+{
+	int ret;
+
+	ret = ec_wait_ibf_clear();
+	if (ret)
+		return ret;
+
+	outb(EC_CMD_READ, EC_CMD_PORT);
+
+	ret = ec_wait_ibf_clear();
+	if (ret)
+		return ret;
+
+	outb(reg, EC_DATA_PORT);
+
+	ret = ec_wait_obf_set();
+	if (ret)
+		return ret;
+
+	*val = inb(EC_DATA_PORT);
+	return 0;
+}
+
+/*
+ * Read a 16-bit big-endian value from two consecutive EC registers.
+ *
+ * The EC may update the register pair between reading the high and low
+ * bytes, which could produce a corrupted value if the high byte rolls
+ * over (e.g., 0x0100 -> 0x00FF read as 0x01FF). Guard against this by
+ * re-reading the high byte after reading the low byte. If the high byte
+ * changed, re-read the low byte to get a consistent pair.
+ * See also lm90_read16() which uses the same approach.
+ */
+static int ec_read_reg16(u8 reg_hi, u8 reg_lo, u16 *val)
+{
+	int ret;
+	u8 oldh, newh, lo;
+
+	ret = ec_read_reg(reg_hi, &oldh);
+	if (ret)
+		return ret;
+
+	ret = ec_read_reg(reg_lo, &lo);
+	if (ret)
+		return ret;
+
+	ret = ec_read_reg(reg_hi, &newh);
+	if (ret)
+		return ret;
+
+	if (oldh != newh) {
+		ret = ec_read_reg(reg_lo, &lo);
+		if (ret)
+			return ret;
+	}
+
+	*val = ((u16)newh << 8) | lo;
+	return 0;
+}
+
+static int
+lps_ec_read_string(struct device *dev,
+		   enum hwmon_sensor_types type,
+		   u32 attr, int channel,
+		   const char **str)
+{
+	switch (type) {
+	case hwmon_fan:
+		*str = "CPU Fan";
+		return 0;
+	case hwmon_temp:
+		*str = channel == 0 ? "Board Temp" : "CPU Temp";
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t
+lps_ec_is_visible(const void *drvdata,
+		  enum hwmon_sensor_types type,
+		  u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_fan:
+		if (attr == hwmon_fan_input || attr == hwmon_fan_label)
+			return 0444;
+		break;
+	case hwmon_temp:
+		if (attr == hwmon_temp_input || attr == hwmon_temp_label)
+			return 0444;
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int
+lps_ec_read(struct device *dev,
+	    enum hwmon_sensor_types type,
+	    u32 attr, int channel, long *val)
+{
+	u16 rpm;
+	u8 v;
+	int ret;
+
+	switch (type) {
+	case hwmon_fan:
+		if (attr != hwmon_fan_input)
+			return -EOPNOTSUPP;
+		ret = ec_read_reg16(EC_REG_FAN_RPM_HI,
+				    EC_REG_FAN_RPM_LO, &rpm);
+		if (ret)
+			return ret;
+		*val = rpm;
+		return 0;
+
+	case hwmon_temp:
+		if (attr != hwmon_temp_input)
+			return -EOPNOTSUPP;
+		ret = ec_read_reg(channel == 0 ? EC_REG_TEMP_BOARD
+					       : EC_REG_TEMP_CPU,
+				  &v);
+		if (ret)
+			return ret;
+		/* EC reports unsigned 8-bit temperature in degrees Celsius */
+		*val = (unsigned long)v * 1000;
+		return 0;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const struct hwmon_channel_info * const lps_ec_info[] = {
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_LABEL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL),
+	NULL
+};
+
+static const struct hwmon_ops lps_ec_ops = {
+	.is_visible = lps_ec_is_visible,
+	.read = lps_ec_read,
+	.read_string = lps_ec_read_string,
+};
+
+static const struct hwmon_chip_info lps_ec_chip_info = {
+	.ops = &lps_ec_ops,
+	.info = lps_ec_info,
+};
+
+static int lps_ec_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device *hwmon;
+	u8 test;
+	int ret;
+
+	if (!devm_request_region(dev, EC_DATA_PORT, 1, DRIVER_NAME))
+		return dev_err_probe(dev, -EBUSY,
+				     "Failed to request EC data port 0x%x\n",
+				     EC_DATA_PORT);
+
+	if (!devm_request_region(dev, EC_CMD_PORT, 1, DRIVER_NAME))
+		return dev_err_probe(dev, -EBUSY,
+				     "Failed to request EC cmd port 0x%x\n",
+				     EC_CMD_PORT);
+
+	/* Sanity check: verify EC is responsive */
+	ret = ec_read_reg(EC_REG_FAN_DUTY, &test);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "EC not responding on ports 0x%x/0x%x\n",
+				     EC_DATA_PORT, EC_CMD_PORT);
+
+	hwmon = devm_hwmon_device_register_with_info(dev, DRIVER_NAME, NULL,
+						     &lps_ec_chip_info, NULL);
+	if (IS_ERR(hwmon))
+		return dev_err_probe(dev, PTR_ERR(hwmon),
+				     "Failed to register hwmon device\n");
+
+	dev_info(dev, "EC hwmon registered (fan duty: %u%%)\n", test);
+	return 0;
+}
+
+/* DMI table with strict BIOS version match (override with force=1) */
+static const struct dmi_system_id lps_ec_dmi_table[] = {
+	{
+		.ident = "LattePanda Sigma",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LattePanda"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "LattePanda Sigma"),
+			DMI_MATCH(DMI_BIOS_VERSION, "5.27"),
+		},
+	},
+	{ }	/* terminator */
+};
+MODULE_DEVICE_TABLE(dmi, lps_ec_dmi_table);
+
+/* Loose table (vendor + product only) for use with force=1 */
+static const struct dmi_system_id lps_ec_dmi_table_force[] = {
+	{
+		.ident = "LattePanda Sigma",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LattePanda"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "LattePanda Sigma"),
+		},
+	},
+	{ }	/* terminator */
+};
+
+static struct platform_driver lps_ec_driver = {
+	.probe	= lps_ec_probe,
+	.driver	= {
+		.name = DRIVER_NAME,
+	},
+};
+
+static int __init lps_ec_init(void)
+{
+	int ret;
+
+	if (!dmi_check_system(lps_ec_dmi_table)) {
+		if (!force || !dmi_check_system(lps_ec_dmi_table_force))
+			return -ENODEV;
+		pr_warn("%s: BIOS version not verified, loading due to force=1\n",
+			DRIVER_NAME);
+	}
+
+	ret = platform_driver_register(&lps_ec_driver);
+	if (ret)
+		return ret;
+
+	lps_ec_pdev = platform_device_register_simple(DRIVER_NAME, -1,
+						      NULL, 0);
+	if (IS_ERR(lps_ec_pdev)) {
+		platform_driver_unregister(&lps_ec_driver);
+		return PTR_ERR(lps_ec_pdev);
+	}
+
+	return 0;
+}
+
+static void __exit lps_ec_exit(void)
+{
+	platform_device_unregister(lps_ec_pdev);
+	platform_driver_unregister(&lps_ec_driver);
+}
+
+module_init(lps_ec_init);
+module_exit(lps_ec_exit);
+
+MODULE_AUTHOR("Mariano Abad <weimaraner@gmail.com>");
+MODULE_DESCRIPTION("Hardware monitoring driver for LattePanda Sigma EC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c
index db6534e679911..b9084424160d6 100644
--- a/drivers/hwmon/ltc4282.c
+++ b/drivers/hwmon/ltc4282.c
@@ -1328,15 +1328,16 @@ static int ltc4282_setup(struct ltc4282_state *st, struct device *dev)
 	if (ret)
 		return ret;
 
+	/* default to 1 milli-ohm so we can probe without FW properties */
+	st->rsense = 1 * (NANO / MILLI);
 	ret = device_property_read_u32(dev, "adi,rsense-nano-ohms",
 				       &st->rsense);
-	if (ret)
-		return dev_err_probe(dev, ret,
-				     "Failed to read adi,rsense-nano-ohms\n");
-	if (st->rsense < CENTI)
-		return dev_err_probe(dev, -EINVAL,
-				     "adi,rsense-nano-ohms too small (< %lu)\n",
-				     CENTI);
+	if (!ret) {
+		if (st->rsense < CENTI)
+			return dev_err_probe(dev, -EINVAL,
+					     "adi,rsense-nano-ohms too small (< %lu)\n",
+					     CENTI);
+	}
 
 	/*
 	 * The resolution for rsense is tenths of micro (eg: 62.5 uOhm) which
diff --git a/drivers/hwmon/max31722.c b/drivers/hwmon/max31722.c
index 9a31ef3883969..6c5c86c75c36e 100644
--- a/drivers/hwmon/max31722.c
+++ b/drivers/hwmon/max31722.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/spi/spi.h>
+#include <linux/sysfs.h>
 
 #define MAX31722_REG_CFG				0x00
 #define MAX31722_REG_TEMP_LSB				0x01
@@ -56,7 +57,7 @@ static ssize_t max31722_temp_show(struct device *dev,
 	if (ret < 0)
 		return ret;
 	/* Keep 12 bits and multiply by the scale of 62.5 millidegrees/bit. */
-	return sprintf(buf, "%d\n", (s16)le16_to_cpu(ret) * 125 / 32);
+	return sysfs_emit(buf, "%d\n", (s16)le16_to_cpu(ret) * 125 / 32);
 }
 
 static SENSOR_DEVICE_ATTR_RO(temp1_input, max31722_temp, 0);
diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c
index 9a3c515efe2ed..1e0550e276268 100644
--- a/drivers/hwmon/max6639.c
+++ b/drivers/hwmon/max6639.c
@@ -540,7 +540,7 @@ static int max6639_probe_child_from_dt(struct i2c_client *client,
 
 	err = of_property_read_u32(child, "pulses-per-revolution", &val);
 	if (!err) {
-		if (val < 1 || val > 5) {
+		if (val < 1 || val > 4) {
 			dev_err(dev, "invalid pulses-per-revolution %d of %pOFn\n", val, child);
 			return -EINVAL;
 		}
diff --git a/drivers/hwmon/max6650.c b/drivers/hwmon/max6650.c
index 9649c6611d5f7..56b8157885bbf 100644
--- a/drivers/hwmon/max6650.c
+++ b/drivers/hwmon/max6650.c
@@ -27,6 +27,7 @@
 #include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
 #include <linux/of.h>
+#include <linux/sysfs.h>
 #include <linux/thermal.h>
 
 /*
@@ -312,7 +313,7 @@ static ssize_t alarm_show(struct device *dev,
 		mutex_unlock(&data->update_lock);
 	}
 
-	return sprintf(buf, "%d\n", alarm);
+	return sysfs_emit(buf, "%d\n", alarm);
 }
 
 static SENSOR_DEVICE_ATTR_RO(gpio1_alarm, alarm, MAX6650_ALRM_GPIO1);
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index fc1273abe3578..a4513fc6bc265 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -711,6 +711,15 @@ config SENSORS_XDPE152
 	  This driver can also be built as a module. If so, the module will
 	  be called xdpe152c4.
 
+config SENSORS_XDPE1A2G7B
+	tristate "Infineon XDPE1A2G7B"
+	help
+	  If you say yes here you get hardware monitoring support for Infineon
+	  XDPE1A2G5B and XDPE1A2G7B.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called xdpe1a2g7b.
+
 config SENSORS_XDPE122
 	tristate "Infineon XDPE122 family"
 	help
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index d6c86924f8877..d592d8c77bec3 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -70,6 +70,7 @@ obj-$(CONFIG_SENSORS_UCD9200)	+= ucd9200.o
 obj-$(CONFIG_SENSORS_XDP710)	+= xdp710.o
 obj-$(CONFIG_SENSORS_XDPE122)	+= xdpe12284.o
 obj-$(CONFIG_SENSORS_XDPE152)	+= xdpe152c4.o
+obj-$(CONFIG_SENSORS_XDPE1A2G7B)	+= xdpe1a2g7b.o
 obj-$(CONFIG_SENSORS_ZL6100)	+= zl6100.o
 obj-$(CONFIG_SENSORS_PIM4328)	+= pim4328.o
 obj-$(CONFIG_SENSORS_CRPS)	+= crps.o
diff --git a/drivers/hwmon/pmbus/bel-pfe.c b/drivers/hwmon/pmbus/bel-pfe.c
index ddf9d9a2958c4..6499556f735b4 100644
--- a/drivers/hwmon/pmbus/bel-pfe.c
+++ b/drivers/hwmon/pmbus/bel-pfe.c
@@ -88,13 +88,10 @@ static struct pmbus_driver_info pfe_driver_info[] = {
 	},
 };
 
-static const struct i2c_device_id pfe_device_id[];
-
 static int pfe_pmbus_probe(struct i2c_client *client)
 {
-	int model;
+	int model = (uintptr_t)i2c_get_match_data(client);
 
-	model = (int)i2c_match_id(pfe_device_id, client)->driver_data;
 	client->dev.platform_data = &pfe_plat_data;
 
 	/*
diff --git a/drivers/hwmon/pmbus/fsp-3y.c b/drivers/hwmon/pmbus/fsp-3y.c
index a4dc09e2ef75a..cad4d23300036 100644
--- a/drivers/hwmon/pmbus/fsp-3y.c
+++ b/drivers/hwmon/pmbus/fsp-3y.c
@@ -222,12 +222,6 @@ static int fsp3y_detect(struct i2c_client *client)
 	return -ENODEV;
 }
 
-static const struct i2c_device_id fsp3y_id[] = {
-	{"ym2151e", ym2151e},
-	{"yh5151e", yh5151e},
-	{ }
-};
-
 static int fsp3y_probe(struct i2c_client *client)
 {
 	struct fsp3y_data *data;
@@ -242,7 +236,7 @@ static int fsp3y_probe(struct i2c_client *client)
 	if (data->chip < 0)
 		return data->chip;
 
-	id = i2c_match_id(fsp3y_id, client);
+	id = i2c_client_get_device_id(client);
 	if (data->chip != id->driver_data)
 		dev_warn(&client->dev, "Device mismatch: Configured %s (%d), detected %d\n",
 			 id->name, (int)id->driver_data, data->chip);
@@ -276,6 +270,11 @@ static int fsp3y_probe(struct i2c_client *client)
 	return pmbus_do_probe(client, &data->info);
 }
 
+static const struct i2c_device_id fsp3y_id[] = {
+	{"ym2151e", ym2151e},
+	{"yh5151e", yh5151e},
+	{ }
+};
 MODULE_DEVICE_TABLE(i2c, fsp3y_id);
 
 static struct i2c_driver fsp3y_driver = {
diff --git a/drivers/hwmon/pmbus/hac300s.c b/drivers/hwmon/pmbus/hac300s.c
index 0a1d52cae91ed..a073db1cfe2e4 100644
--- a/drivers/hwmon/pmbus/hac300s.c
+++ b/drivers/hwmon/pmbus/hac300s.c
@@ -58,6 +58,8 @@ static int hac300s_read_word_data(struct i2c_client *client, int page,
 	case PMBUS_MFR_VOUT_MIN:
 	case PMBUS_READ_VOUT:
 		rv = pmbus_read_word_data(client, page, phase, reg);
+		if (rv < 0)
+			return rv;
 		return FIELD_GET(LINEAR11_MANTISSA_MASK, rv);
 	default:
 		return -ENODATA;
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
index d05ef7a968a96..6c7256d997f46 100644
--- a/drivers/hwmon/pmbus/ibm-cffps.c
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -58,7 +58,7 @@ enum {
 	CFFPS_DEBUGFS_NUM_ENTRIES
 };
 
-enum versions { cffps1, cffps2, cffps_unknown };
+enum versions { cffps_unknown, cffps1, cffps2 };
 
 struct ibm_cffps {
 	enum versions version;
@@ -482,19 +482,9 @@ MODULE_DEVICE_TABLE(i2c, ibm_cffps_id);
 static int ibm_cffps_probe(struct i2c_client *client)
 {
 	int i, rc;
-	enum versions vs = cffps_unknown;
+	enum versions vs = (uintptr_t)i2c_get_match_data(client);
 	struct dentry *debugfs;
 	struct ibm_cffps *psu;
-	const void *md = of_device_get_match_data(&client->dev);
-	const struct i2c_device_id *id;
-
-	if (md) {
-		vs = (uintptr_t)md;
-	} else {
-		id = i2c_match_id(ibm_cffps_id, client);
-		if (id)
-			vs = (enum versions)id->driver_data;
-	}
 
 	if (vs == cffps_unknown) {
 		u16 ccin_revision = 0;
@@ -534,7 +524,7 @@ static int ibm_cffps_probe(struct i2c_client *client)
 		}
 
 		/* Set the client name to include the version number. */
-		snprintf(client->name, I2C_NAME_SIZE, "cffps%d", vs + 1);
+		snprintf(client->name, I2C_NAME_SIZE, "cffps%d", vs);
 	}
 
 	client->dev.platform_data = &ibm_cffps_pdata;
diff --git a/drivers/hwmon/pmbus/ina233.c b/drivers/hwmon/pmbus/ina233.c
index dde1e16783943..2d8b5a5347edc 100644
--- a/drivers/hwmon/pmbus/ina233.c
+++ b/drivers/hwmon/pmbus/ina233.c
@@ -67,6 +67,8 @@ static int ina233_read_word_data(struct i2c_client *client, int page,
 	switch (reg) {
 	case PMBUS_VIRT_READ_VMON:
 		ret = pmbus_read_word_data(client, 0, 0xff, MFR_READ_VSHUNT);
+		if (ret < 0)
+			return ret;
 
 		/* Adjust returned value to match VIN coefficients */
 		/* VIN: 1.25 mV VSHUNT: 2.5 uV LSB */
diff --git a/drivers/hwmon/pmbus/isl68137.c b/drivers/hwmon/pmbus/isl68137.c
index 97b61836f53a4..c6be84b1217d8 100644
--- a/drivers/hwmon/pmbus/isl68137.c
+++ b/drivers/hwmon/pmbus/isl68137.c
@@ -23,52 +23,6 @@
 #define RAA_DMPVR2_READ_VMON	0xc8
 #define MAX_CHANNELS            4
 
-enum chips {
-	isl68137,
-	isl68220,
-	isl68221,
-	isl68222,
-	isl68223,
-	isl68224,
-	isl68225,
-	isl68226,
-	isl68227,
-	isl68229,
-	isl68233,
-	isl68239,
-	isl69222,
-	isl69223,
-	isl69224,
-	isl69225,
-	isl69227,
-	isl69228,
-	isl69234,
-	isl69236,
-	isl69239,
-	isl69242,
-	isl69243,
-	isl69247,
-	isl69248,
-	isl69254,
-	isl69255,
-	isl69256,
-	isl69259,
-	isl69260,
-	isl69268,
-	isl69269,
-	isl69298,
-	raa228000,
-	raa228004,
-	raa228006,
-	raa228228,
-	raa228244,
-	raa228246,
-	raa229001,
-	raa229004,
-	raa229141,
-	raa229621,
-};
-
 enum variants {
 	raa_dmpvr1_2rail,
 	raa_dmpvr2_1rail,
@@ -90,16 +44,17 @@ struct isl68137_data {
 
 #define to_isl68137_data(x)	container_of(x, struct isl68137_data, info)
 
-static const struct i2c_device_id raa_dmpvr_id[];
-
 static ssize_t isl68137_avs_enable_show_page(struct i2c_client *client,
 					     int page,
 					     char *buf)
 {
 	int val = pmbus_read_byte_data(client, page, PMBUS_OPERATION);
 
-	return sprintf(buf, "%d\n",
-		       (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS ? 1 : 0);
+	if (val < 0)
+		return val;
+
+	return sysfs_emit(buf, "%d\n",
+			   (val & ISL68137_VOUT_AVS) == ISL68137_VOUT_AVS);
 }
 
 static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client,
@@ -375,7 +330,7 @@ static int isl68137_probe(struct i2c_client *client)
 	memcpy(&data->info, &raa_dmpvr_info, sizeof(data->info));
 	info = &data->info;
 
-	switch (i2c_match_id(raa_dmpvr_id, client)->driver_data) {
+	switch ((uintptr_t)i2c_get_match_data(client)) {
 	case raa_dmpvr1_2rail:
 		info->pages = 2;
 		info->R[PSC_VOLTAGE_IN] = 3;
diff --git a/drivers/hwmon/pmbus/ltc2978.c b/drivers/hwmon/pmbus/ltc2978.c
index 8f5be520a15db..d69a5e675e80e 100644
--- a/drivers/hwmon/pmbus/ltc2978.c
+++ b/drivers/hwmon/pmbus/ltc2978.c
@@ -733,7 +733,7 @@ static int ltc2978_probe(struct i2c_client *client)
 		return chip_id;
 
 	data->id = chip_id;
-	id = i2c_match_id(ltc2978_id, client);
+	id = i2c_client_get_device_id(client);
 	if (data->id != id->driver_data)
 		dev_warn(&client->dev,
 			 "Device mismatch: Configured %s (%d), detected %d\n",
diff --git a/drivers/hwmon/pmbus/max16601.c b/drivers/hwmon/pmbus/max16601.c
index d696e506aafba..36dc13424d929 100644
--- a/drivers/hwmon/pmbus/max16601.c
+++ b/drivers/hwmon/pmbus/max16601.c
@@ -318,7 +318,7 @@ static int max16601_probe(struct i2c_client *client)
 	if (chip_id < 0)
 		return chip_id;
 
-	id = i2c_match_id(max16601_id, client);
+	id = i2c_client_get_device_id(client);
 	if (chip_id != id->driver_data)
 		dev_warn(&client->dev,
 			 "Device mismatch: Configured %s (%d), detected %d\n",
diff --git a/drivers/hwmon/pmbus/max20730.c b/drivers/hwmon/pmbus/max20730.c
index 95869d198ecf0..fe03164788dfa 100644
--- a/drivers/hwmon/pmbus/max20730.c
+++ b/drivers/hwmon/pmbus/max20730.c
@@ -715,10 +715,7 @@ static int max20730_probe(struct i2c_client *client)
 		return -ENODEV;
 	}
 
-	if (client->dev.of_node)
-		chip_id = (uintptr_t)of_device_get_match_data(dev);
-	else
-		chip_id = i2c_match_id(max20730_id, client)->driver_data;
+	chip_id = (uintptr_t)i2c_get_match_data(client);
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
diff --git a/drivers/hwmon/pmbus/max31785.c b/drivers/hwmon/pmbus/max31785.c
index 1f94d38a16371..50073fe0c5e88 100644
--- a/drivers/hwmon/pmbus/max31785.c
+++ b/drivers/hwmon/pmbus/max31785.c
@@ -55,7 +55,7 @@ static inline void max31785_wait(const struct max31785_data *data)
 
 static int max31785_i2c_write_byte_data(struct i2c_client *client,
 					struct max31785_data *driver_data,
-					int command, u16 data)
+					int command, u8 data)
 {
 	int rc;
 
diff --git a/drivers/hwmon/pmbus/max34440.c b/drivers/hwmon/pmbus/max34440.c
index 8ea4e68d4e9de..cc96bb22f8f5a 100644
--- a/drivers/hwmon/pmbus/max34440.c
+++ b/drivers/hwmon/pmbus/max34440.c
@@ -71,8 +71,6 @@ struct max34440_data {
 
 #define to_max34440_data(x)  container_of(x, struct max34440_data, info)
 
-static const struct i2c_device_id max34440_id[];
-
 static int max34440_read_word_data(struct i2c_client *client, int page,
 				   int phase, int reg)
 {
@@ -628,7 +626,7 @@ static int max34440_probe(struct i2c_client *client)
 			    GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
-	data->id = i2c_match_id(max34440_id, client)->driver_data;
+	data->id = (uintptr_t)i2c_get_match_data(client);
 	data->info = max34440_info[data->id];
 	data->iout_oc_fault_limit = MAX34440_IOUT_OC_FAULT_LIMIT;
 	data->iout_oc_warn_limit = MAX34440_IOUT_OC_WARN_LIMIT;
diff --git a/drivers/hwmon/pmbus/mp2869.c b/drivers/hwmon/pmbus/mp2869.c
index cc69a1e91dfe8..4647892e51121 100644
--- a/drivers/hwmon/pmbus/mp2869.c
+++ b/drivers/hwmon/pmbus/mp2869.c
@@ -165,7 +165,7 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg)
 {
 	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
 	struct mp2869_data *data = to_mp2869_data(info);
-	int ret;
+	int ret, mfr;
 
 	switch (reg) {
 	case PMBUS_VOUT_MODE:
@@ -188,11 +188,14 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg)
 		if (ret < 0)
 			return ret;
 
+		mfr = pmbus_read_byte_data(client, page,
+					   PMBUS_STATUS_MFR_SPECIFIC);
+		if (mfr < 0)
+			return mfr;
+
 		ret = (ret & ~GENMASK(2, 2)) |
 			FIELD_PREP(GENMASK(2, 2),
-				   FIELD_GET(GENMASK(1, 1),
-					     pmbus_read_byte_data(client, page,
-								  PMBUS_STATUS_MFR_SPECIFIC)));
+				   FIELD_GET(GENMASK(1, 1), mfr));
 		break;
 	case PMBUS_STATUS_TEMPERATURE:
 		/*
@@ -207,15 +210,16 @@ static int mp2869_read_byte_data(struct i2c_client *client, int page, int reg)
 		if (ret < 0)
 			return ret;
 
+		mfr = pmbus_read_byte_data(client, page,
+					   PMBUS_STATUS_MFR_SPECIFIC);
+		if (mfr < 0)
+			return mfr;
+
 		ret = (ret & ~GENMASK(7, 6)) |
 			FIELD_PREP(GENMASK(6, 6),
-				   FIELD_GET(GENMASK(1, 1),
-					     pmbus_read_byte_data(client, page,
-								  PMBUS_STATUS_MFR_SPECIFIC))) |
+				   FIELD_GET(GENMASK(1, 1), mfr)) |
 			 FIELD_PREP(GENMASK(7, 7),
-				    FIELD_GET(GENMASK(1, 1),
-					      pmbus_read_byte_data(client, page,
-								   PMBUS_STATUS_MFR_SPECIFIC)));
+				    FIELD_GET(GENMASK(1, 1), mfr));
 		break;
 	default:
 		ret = -ENODATA;
@@ -230,7 +234,7 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase,
 {
 	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
 	struct mp2869_data *data = to_mp2869_data(info);
-	int ret;
+	int ret, mfr;
 
 	switch (reg) {
 	case PMBUS_STATUS_WORD:
@@ -246,11 +250,14 @@ static int mp2869_read_word_data(struct i2c_client *client, int page, int phase,
 		if (ret < 0)
 			return ret;
 
+		mfr = pmbus_read_byte_data(client, page,
+					   PMBUS_STATUS_MFR_SPECIFIC);
+		if (mfr < 0)
+			return mfr;
+
 		ret = (ret & ~GENMASK(2, 2)) |
 			 FIELD_PREP(GENMASK(2, 2),
-				    FIELD_GET(GENMASK(1, 1),
-					      pmbus_read_byte_data(client, page,
-								   PMBUS_STATUS_MFR_SPECIFIC)));
+				    FIELD_GET(GENMASK(1, 1), mfr));
 		break;
 	case PMBUS_READ_VIN:
 		/*
diff --git a/drivers/hwmon/pmbus/mp2975.c b/drivers/hwmon/pmbus/mp2975.c
index c31982d851962..d0bc47b12cb07 100644
--- a/drivers/hwmon/pmbus/mp2975.c
+++ b/drivers/hwmon/pmbus/mp2975.c
@@ -313,6 +313,8 @@ static int mp2973_read_word_data(struct i2c_client *client, int page,
 	case PMBUS_STATUS_WORD:
 		/* MP2973 & MP2971 return PGOOD instead of PB_STATUS_POWER_GOOD_N. */
 		ret = pmbus_read_word_data(client, page, phase, reg);
+		if (ret < 0)
+			return ret;
 		ret ^= PB_STATUS_POWER_GOOD_N;
 		break;
 	case PMBUS_OT_FAULT_LIMIT:
diff --git a/drivers/hwmon/pmbus/pmbus.c b/drivers/hwmon/pmbus/pmbus.c
index 920cd5408141a..d1844c7a51eef 100644
--- a/drivers/hwmon/pmbus/pmbus.c
+++ b/drivers/hwmon/pmbus/pmbus.c
@@ -20,8 +20,6 @@ struct pmbus_device_info {
 	u32 flags;
 };
 
-static const struct i2c_device_id pmbus_id[];
-
 /*
  * Find sensor groups and status registers on each page.
  */
@@ -174,7 +172,7 @@ static int pmbus_probe(struct i2c_client *client)
 	if (!info)
 		return -ENOMEM;
 
-	device_info = (struct pmbus_device_info *)i2c_match_id(pmbus_id, client)->driver_data;
+	device_info = (struct pmbus_device_info *)i2c_get_match_data(client);
 	if (device_info->flags) {
 		pdata = devm_kzalloc(dev, sizeof(struct pmbus_platform_data),
 				     GFP_KERNEL);
diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
index d2e9bfb5320fc..3ddcb742d289e 100644
--- a/drivers/hwmon/pmbus/pmbus.h
+++ b/drivers/hwmon/pmbus/pmbus.h
@@ -416,7 +416,7 @@ enum pmbus_sensor_classes {
 #define PMBUS_PAGE_VIRTUAL	BIT(31)	/* Page is virtual */
 
 enum pmbus_data_format { linear = 0, ieee754, direct, vid };
-enum vrm_version { vr11 = 0, vr12, vr13, imvp9, amd625mv };
+enum vrm_version { vr11 = 0, vr12, vr13, imvp9, amd625mv, nvidia195mv };
 
 /* PMBus revision identifiers */
 #define PMBUS_REV_10 0x00	/* PMBus revision 1.0 */
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index be6d05def1152..4d7634ee61484 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -885,6 +885,10 @@ static s64 pmbus_reg2data_vid(struct pmbus_data *data,
 		if (val >= 0x0 && val <= 0xd8)
 			rv = DIV_ROUND_CLOSEST(155000 - val * 625, 100);
 		break;
+	case nvidia195mv:
+		if (val >= 0x01)
+			rv = 195 + (val - 1) * 5;  /* VID step is 5mv */
+		break;
 	}
 	return rv;
 }
diff --git a/drivers/hwmon/pmbus/q54sj108a2.c b/drivers/hwmon/pmbus/q54sj108a2.c
index d5d60a9af8c58..a368cfa9d45ac 100644
--- a/drivers/hwmon/pmbus/q54sj108a2.c
+++ b/drivers/hwmon/pmbus/q54sj108a2.c
@@ -271,6 +271,8 @@ static const struct file_operations q54sj108a2_fops = {
 
 static const struct i2c_device_id q54sj108a2_id[] = {
 	{ "q54sj108a2", q54sj108a2 },
+	{ "q54sn120a1", q54sj108a2 },
+	{ "q54sw120a7", q54sj108a2 },
 	{ },
 };
 
@@ -280,6 +282,7 @@ static int q54sj108a2_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	u8 buf[I2C_SMBUS_BLOCK_MAX + 1];
+	const struct i2c_device_id *mid;
 	enum chips chip_id;
 	int ret, i;
 	struct dentry *debugfs;
@@ -292,10 +295,7 @@ static int q54sj108a2_probe(struct i2c_client *client)
 				     I2C_FUNC_SMBUS_BLOCK_DATA))
 		return -ENODEV;
 
-	if (client->dev.of_node)
-		chip_id = (enum chips)(unsigned long)of_device_get_match_data(dev);
-	else
-		chip_id = i2c_match_id(q54sj108a2_id, client)->driver_data;
+	chip_id = (enum chips)(uintptr_t)i2c_get_match_data(client);
 
 	ret = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf);
 	if (ret < 0) {
@@ -316,8 +316,12 @@ static int q54sj108a2_probe(struct i2c_client *client)
 		dev_err(dev, "Failed to read Manufacturer Model\n");
 		return ret;
 	}
-	if (ret != 14 || strncmp(buf, "Q54SJ108A2", 10)) {
-		buf[ret] = '\0';
+	buf[ret] = '\0';
+	for (mid = q54sj108a2_id; mid->name[0]; mid++) {
+		if (!strncasecmp(mid->name, buf, strlen(mid->name)))
+			break;
+	}
+	if (!mid->name[0]) {
 		dev_err(dev, "Unsupported Manufacturer Model '%s'\n", buf);
 		return -ENODEV;
 	}
@@ -327,7 +331,10 @@ static int q54sj108a2_probe(struct i2c_client *client)
 		dev_err(dev, "Failed to read Manufacturer Revision\n");
 		return ret;
 	}
-	if (ret != 4 || buf[0] != 'S') {
+	/*
+	 * accept manufacturer revision with optional NUL byte
+	 */
+	if (!(ret == 4 || ret == 5) || buf[0] != 'S') {
 		buf[ret] = '\0';
 		dev_err(dev, "Unsupported Manufacturer Revision '%s'\n", buf);
 		return -ENODEV;
@@ -404,6 +411,8 @@ static int q54sj108a2_probe(struct i2c_client *client)
 
 static const struct of_device_id q54sj108a2_of_match[] = {
 	{ .compatible = "delta,q54sj108a2", .data = (void *)q54sj108a2 },
+	{ .compatible = "delta,q54sn120a1", .data = (void *)q54sj108a2 },
+	{ .compatible = "delta,q54sw120a7", .data = (void *)q54sj108a2 },
 	{ },
 };
 
diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c
index ca2bfa25eb04c..df2726659a4ed 100644
--- a/drivers/hwmon/pmbus/tps53679.c
+++ b/drivers/hwmon/pmbus/tps53679.c
@@ -253,10 +253,7 @@ static int tps53679_probe(struct i2c_client *client)
 	struct pmbus_driver_info *info;
 	enum chips chip_id;
 
-	if (dev->of_node)
-		chip_id = (uintptr_t)of_device_get_match_data(dev);
-	else
-		chip_id = i2c_match_id(tps53679_id, client)->driver_data;
+	chip_id = (uintptr_t)i2c_get_match_data(client);
 
 	info = devm_kmemdup(dev, &tps53679_info, sizeof(*info), GFP_KERNEL);
 	if (!info)
diff --git a/drivers/hwmon/pmbus/xdpe1a2g7b.c b/drivers/hwmon/pmbus/xdpe1a2g7b.c
new file mode 100644
index 0000000000000..1755e3522ede5
--- /dev/null
+++ b/drivers/hwmon/pmbus/xdpe1a2g7b.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hardware monitoring driver for Infineon Multi-phase Digital XDPE1A2G5B
+ * and XDPE1A2G7B Controllers
+ *
+ * Copyright (c) 2026 Infineon Technologies. All rights reserved.
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "pmbus.h"
+
+#define XDPE1A2G7B_PAGE_NUM 2
+#define XDPE1A2G7B_NVIDIA_195MV 0x1E /* NVIDIA mode 1.95mV, VID step is 5mV */
+
+static int xdpe1a2g7b_identify(struct i2c_client *client,
+			       struct pmbus_driver_info *info)
+{
+	u8 vout_params;
+	int vout_mode;
+
+	/*
+	 * XDPE1A2G5B and XDPE1A2G7B support both Linear and NVIDIA PWM VID data
+	 * formats via VOUT_MODE. Note that the device pages/loops are not fully
+	 * independent: configuration is shared, so programming each page/loop
+	 * separately is not supported.
+	 */
+	vout_mode = pmbus_read_byte_data(client, 0, PMBUS_VOUT_MODE);
+	if (vout_mode < 0)
+		return vout_mode;
+
+	switch (vout_mode >> 5) {
+	case 0:
+		info->format[PSC_VOLTAGE_OUT] = linear;
+		return 0;
+	case 1:
+		info->format[PSC_VOLTAGE_OUT] = vid;
+		vout_params = vout_mode & GENMASK(4, 0);
+		/* Check for VID Code Type */
+		switch (vout_params) {
+		case XDPE1A2G7B_NVIDIA_195MV:
+			/* VID vrm_version for PAGE0 and PAGE1 */
+			info->vrm_version[0] = nvidia195mv;
+			info->vrm_version[1] = nvidia195mv;
+			break;
+		default:
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static struct pmbus_driver_info xdpe1a2g7b_info = {
+	.pages = XDPE1A2G7B_PAGE_NUM,
+	.identify = xdpe1a2g7b_identify,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.format[PSC_CURRENT_IN] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_POWER] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_TEMP | PMBUS_HAVE_TEMP2 | PMBUS_HAVE_STATUS_TEMP |
+		   PMBUS_HAVE_POUT | PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT,
+	.func[1] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_PIN | PMBUS_HAVE_POUT | PMBUS_HAVE_STATUS_INPUT,
+};
+
+static int xdpe1a2g7b_probe(struct i2c_client *client)
+{
+	struct pmbus_driver_info *info;
+
+	info = devm_kmemdup(&client->dev, &xdpe1a2g7b_info, sizeof(*info),
+			    GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	return pmbus_do_probe(client, info);
+}
+
+static const struct i2c_device_id xdpe1a2g7b_id[] = {
+	{ "xdpe1a2g5b" },
+	{ "xdpe1a2g7b" },
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, xdpe1a2g7b_id);
+
+static const struct of_device_id __maybe_unused xdpe1a2g7b_of_match[] = {
+	{ .compatible = "infineon,xdpe1a2g5b" },
+	{ .compatible = "infineon,xdpe1a2g7b" },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, xdpe1a2g7b_of_match);
+
+static struct i2c_driver xdpe1a2g7b_driver = {
+	.driver = {
+		.name = "xdpe1a2g7b",
+		.of_match_table = of_match_ptr(xdpe1a2g7b_of_match),
+	},
+	.probe = xdpe1a2g7b_probe,
+	.id_table = xdpe1a2g7b_id,
+};
+
+module_i2c_driver(xdpe1a2g7b_driver);
+
+MODULE_AUTHOR("Ashish Yadav <ashish.yadav@infineon.com>");
+MODULE_DESCRIPTION("PMBus driver for Infineon XDPE1A2G5B/7B");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/tc74.c b/drivers/hwmon/tc74.c
index 9984373a25fb7..7fb7b50ad1adb 100644
--- a/drivers/hwmon/tc74.c
+++ b/drivers/hwmon/tc74.c
@@ -92,7 +92,7 @@ static ssize_t temp_input_show(struct device *dev,
 	if (ret)
 		return ret;
 
-	return sprintf(buf, "%d\n", data->temp_input * 1000);
+	return sysfs_emit(buf, "%d\n", data->temp_input * 1000);
 }
 static SENSOR_DEVICE_ATTR_RO(temp1_input, temp_input, 0);
 
diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c
index dfd035852b120..ce71dcddfca25 100644
--- a/drivers/hwtracing/coresight/coresight-catu.c
+++ b/drivers/hwtracing/coresight/coresight-catu.c
@@ -30,8 +30,6 @@
 #define catu_dbg(x, ...) do {} while (0)
 #endif
 
-DEFINE_CORESIGHT_DEVLIST(catu_devs, "catu");
-
 struct catu_etr_buf {
 	struct tmc_sg_table *catu_table;
 	dma_addr_t sladdr;
@@ -530,7 +528,7 @@ static int __catu_probe(struct device *dev, struct resource *res)
 	if (ret)
 		return ret;
 
-	catu_desc.name = coresight_alloc_device_name(&catu_devs, dev);
+	catu_desc.name = coresight_alloc_device_name("catu", dev);
 	if (!catu_desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c
index 80e26396ad0a0..46f247f73cf64 100644
--- a/drivers/hwtracing/coresight/coresight-core.c
+++ b/drivers/hwtracing/coresight/coresight-core.c
@@ -53,6 +53,9 @@ struct coresight_node {
 const u32 coresight_barrier_pkt[4] = {0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff};
 EXPORT_SYMBOL_GPL(coresight_barrier_pkt);
 
+/* List maintains the device index */
+static LIST_HEAD(coresight_dev_idx_list);
+
 static const struct cti_assoc_op *cti_assoc_ops;
 
 void coresight_set_cti_ops(const struct cti_assoc_op *cti_op)
@@ -1149,7 +1152,6 @@ static int coresight_clear_filter_source(struct device *dev, void *data)
 	return 0;
 }
 
-/* coresight_remove_conns - Remove other device's references to this device */
 static void coresight_remove_conns(struct coresight_device *csdev)
 {
 	int i, j;
@@ -1159,10 +1161,6 @@ static void coresight_remove_conns(struct coresight_device *csdev)
 		bus_for_each_dev(&coresight_bustype, NULL, csdev,
 				 coresight_clear_filter_source);
 
-	/*
-	 * Remove the input connection references from the destination device
-	 * for each output connection.
-	 */
 	for (i = 0; i < csdev->pdata->nr_outconns; i++) {
 		conn = csdev->pdata->out_conns[i];
 		if (conn->filter_src_fwnode) {
@@ -1173,6 +1171,13 @@ static void coresight_remove_conns(struct coresight_device *csdev)
 		if (!conn->dest_dev)
 			continue;
 
+		/* Remove sysfs links for the output connection */
+		coresight_remove_links(csdev, conn);
+
+		/*
+		 * Remove the input connection references from the destination
+		 * device for each output connection.
+		 */
 		for (j = 0; j < conn->dest_dev->pdata->nr_inconns; ++j)
 			if (conn->dest_dev->pdata->in_conns[j] == conn) {
 				conn->dest_dev->pdata->in_conns[j] = NULL;
@@ -1195,6 +1200,8 @@ static void coresight_remove_conns(struct coresight_device *csdev)
 		coresight_remove_links(conn->src_dev, conn);
 		conn->dest_dev = NULL;
 	}
+
+	coresight_remove_conns_sysfs_group(csdev);
 }
 
 /**
@@ -1295,17 +1302,13 @@ void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset)
  * coresight_release_platform_data: Release references to the devices connected
  * to the output port of this device.
  */
-void coresight_release_platform_data(struct coresight_device *csdev,
-				     struct device *dev,
+void coresight_release_platform_data(struct device *dev,
 				     struct coresight_platform_data *pdata)
 {
 	int i;
 	struct coresight_connection **conns = pdata->out_conns;
 
 	for (i = 0; i < pdata->nr_outconns; i++) {
-		/* If we have made the links, remove them now */
-		if (csdev && conns[i]->dest_dev)
-			coresight_remove_links(csdev, conns[i]);
 		/*
 		 * Drop the refcount and clear the handle as this device
 		 * is going away
@@ -1317,8 +1320,6 @@ void coresight_release_platform_data(struct coresight_device *csdev,
 	devm_kfree(dev, pdata->out_conns);
 	devm_kfree(dev, pdata->in_conns);
 	devm_kfree(dev, pdata);
-	if (csdev)
-		coresight_remove_conns_sysfs_group(csdev);
 }
 
 struct coresight_device *coresight_register(struct coresight_desc *desc)
@@ -1346,12 +1347,6 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 	csdev->dev.parent = desc->dev;
 	csdev->dev.release = coresight_device_release;
 	csdev->dev.bus = &coresight_bustype;
-	/*
-	 * Hold the reference to our parent device. This will be
-	 * dropped only in coresight_device_release().
-	 */
-	csdev->dev.fwnode = fwnode_handle_get(dev_fwnode(desc->dev));
-	dev_set_name(&csdev->dev, "%s", desc->name);
 
 	if (csdev->type == CORESIGHT_DEV_TYPE_SINK ||
 	    csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) {
@@ -1363,6 +1358,14 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 			goto err_out;
 		}
 	}
+
+	/*
+	 * Hold the reference to our parent device. This will be
+	 * dropped only in coresight_device_release().
+	 */
+	csdev->dev.fwnode = fwnode_handle_get(dev_fwnode(desc->dev));
+	dev_set_name(&csdev->dev, "%s", desc->name);
+
 	/*
 	 * Make sure the device registration and the connection fixup
 	 * are synchronised, so that we don't see uninitialised devices
@@ -1380,37 +1383,30 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 		goto out_unlock;
 	}
 
-	if ((csdev->type == CORESIGHT_DEV_TYPE_SINK ||
-	     csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) &&
-	    sink_ops(csdev)->alloc_buffer) {
-		ret = etm_perf_add_symlink_sink(csdev);
-
-		if (ret) {
-			device_unregister(&csdev->dev);
-			/*
-			 * As with the above, all resources are free'd
-			 * explicitly via coresight_device_release() triggered
-			 * from put_device(), which is in turn called from
-			 * function device_unregister().
-			 */
-			goto out_unlock;
-		}
-	}
 	/* Device is now registered */
 	registered = true;
 
+	ret = etm_perf_add_symlink_sink(csdev);
+	if (ret && ret != -EOPNOTSUPP)
+		goto out_unlock;
+
 	ret = coresight_create_conns_sysfs_group(csdev);
-	if (!ret)
-		ret = coresight_fixup_orphan_conns(csdev);
+	if (ret)
+		goto out_unlock;
+
+	ret = coresight_fixup_orphan_conns(csdev);
+	if (ret)
+		goto out_unlock;
+
+	mutex_unlock(&coresight_mutex);
+
+	if (cti_assoc_ops && cti_assoc_ops->add)
+		cti_assoc_ops->add(csdev);
+
+	return csdev;
 
 out_unlock:
 	mutex_unlock(&coresight_mutex);
-	/* Success */
-	if (!ret) {
-		if (cti_assoc_ops && cti_assoc_ops->add)
-			cti_assoc_ops->add(csdev);
-		return csdev;
-	}
 
 	/* Unregister the device if needed */
 	if (registered) {
@@ -1419,41 +1415,76 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 	}
 
 err_out:
-	/* Cleanup the connection information */
-	coresight_release_platform_data(NULL, desc->dev, desc->pdata);
+	coresight_release_platform_data(desc->dev, desc->pdata);
 	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(coresight_register);
 
 void coresight_unregister(struct coresight_device *csdev)
 {
-	etm_perf_del_symlink_sink(csdev);
 	/* Remove references of that device in the topology */
 	if (cti_assoc_ops && cti_assoc_ops->remove)
 		cti_assoc_ops->remove(csdev);
+
+	mutex_lock(&coresight_mutex);
+	etm_perf_del_symlink_sink(csdev);
 	coresight_remove_conns(csdev);
 	coresight_clear_default_sink(csdev);
-	coresight_release_platform_data(csdev, csdev->dev.parent, csdev->pdata);
+	coresight_release_platform_data(csdev->dev.parent, csdev->pdata);
 	device_unregister(&csdev->dev);
+	mutex_unlock(&coresight_mutex);
 }
 EXPORT_SYMBOL_GPL(coresight_unregister);
 
+static struct coresight_dev_list *
+coresight_allocate_device_list(const char *prefix)
+{
+	struct coresight_dev_list *list;
+
+	/* Check if have already allocated */
+	list_for_each_entry(list, &coresight_dev_idx_list, node) {
+		if (!strcmp(list->pfx, prefix))
+			return list;
+	}
 
-/*
- * coresight_search_device_idx - Search the fwnode handle of a device
- * in the given dev_idx list. Must be called with the coresight_mutex held.
- *
- * Returns the index of the entry, when found. Otherwise, -ENOENT.
- */
-static int coresight_search_device_idx(struct coresight_dev_list *dict,
-				       struct fwnode_handle *fwnode)
+	list = kzalloc(sizeof(*list), GFP_KERNEL);
+	if (!list)
+		return NULL;
+
+	list->pfx = kstrdup(prefix, GFP_KERNEL);
+	if (!list->pfx) {
+		kfree(list);
+		return NULL;
+	}
+
+	list_add(&list->node, &coresight_dev_idx_list);
+	return list;
+}
+
+static int coresight_allocate_device_idx(struct coresight_dev_list *list,
+					 struct device *dev)
 {
-	int i;
+	struct fwnode_handle **fwnode_list;
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
+	int idx;
+
+	for (idx = 0; idx < list->nr_idx; idx++)
+		if (list->fwnode_list[idx] == fwnode)
+			return idx;
 
-	for (i = 0; i < dict->nr_idx; i++)
-		if (dict->fwnode_list[i] == fwnode)
-			return i;
-	return -ENOENT;
+	/* Make space for the new entry */
+	idx = list->nr_idx;
+	fwnode_list = krealloc_array(list->fwnode_list,
+				     idx + 1, sizeof(*list->fwnode_list),
+				     GFP_KERNEL);
+	if (!fwnode_list)
+		return -ENOMEM;
+
+	fwnode_list[idx] = fwnode;
+	list->fwnode_list = fwnode_list;
+	list->nr_idx = idx + 1;
+
+	return idx;
 }
 
 static bool coresight_compare_type(enum coresight_dev_type type_a,
@@ -1527,45 +1558,63 @@ bool coresight_loses_context_with_cpu(struct device *dev)
 EXPORT_SYMBOL_GPL(coresight_loses_context_with_cpu);
 
 /*
- * coresight_alloc_device_name - Get an index for a given device in the
- * device index list specific to a driver. An index is allocated for a
- * device and is tracked with the fwnode_handle to prevent allocating
+ * coresight_alloc_device_name - Get an index for a given device in the list
+ * specific to a driver (presented by the prefix string). An index is allocated
+ * for a device and is tracked with the fwnode_handle to prevent allocating
  * duplicate indices for the same device (e.g, if we defer probing of
  * a device due to dependencies), in case the index is requested again.
  */
-char *coresight_alloc_device_name(struct coresight_dev_list *dict,
-				  struct device *dev)
+char *coresight_alloc_device_name(const char *prefix, struct device *dev)
 {
-	int idx;
+	struct coresight_dev_list *list;
 	char *name = NULL;
-	struct fwnode_handle **list;
+	int idx;
 
 	mutex_lock(&coresight_mutex);
 
-	idx = coresight_search_device_idx(dict, dev_fwnode(dev));
-	if (idx < 0) {
-		/* Make space for the new entry */
-		idx = dict->nr_idx;
-		list = krealloc_array(dict->fwnode_list,
-				      idx + 1, sizeof(*dict->fwnode_list),
-				      GFP_KERNEL);
-		if (ZERO_OR_NULL_PTR(list)) {
-			idx = -ENOMEM;
-			goto done;
-		}
+	list = coresight_allocate_device_list(prefix);
+	if (!list)
+		goto done;
 
-		list[idx] = dev_fwnode(dev);
-		dict->fwnode_list = list;
-		dict->nr_idx = idx + 1;
-	}
+	idx = coresight_allocate_device_idx(list, dev);
+
+	/*
+	 * If index allocation fails, the device list is not released here;
+	 * it is instead freed later by coresight_release_device_list() when
+	 * the coresight_core module is unloaded.
+	 */
+	if (idx < 0)
+		goto done;
 
-	name = devm_kasprintf(dev, GFP_KERNEL, "%s%d", dict->pfx, idx);
+	name = devm_kasprintf(dev, GFP_KERNEL, "%s%d", list->pfx, idx);
 done:
 	mutex_unlock(&coresight_mutex);
 	return name;
 }
 EXPORT_SYMBOL_GPL(coresight_alloc_device_name);
 
+static void coresight_release_device_list(void)
+{
+	struct coresight_dev_list *list, *next;
+	int i;
+
+	/*
+	 * Here is no need to take coresight_mutex; this is during core module
+	 * unloading, no race condition with other modules.
+	 */
+
+	list_for_each_entry_safe(list, next, &coresight_dev_idx_list, node) {
+		for (i = 0; i < list->nr_idx; i++)
+			list->fwnode_list[i] = NULL;
+		list->nr_idx = 0;
+		list_del(&list->node);
+
+		kfree(list->pfx);
+		kfree(list->fwnode_list);
+		kfree(list);
+	}
+}
+
 const struct bus_type coresight_bustype = {
 	.name	= "coresight",
 };
@@ -1639,6 +1688,7 @@ static void __exit coresight_exit(void)
 					     &coresight_notifier);
 	etm_perf_exit();
 	bus_unregister(&coresight_bustype);
+	coresight_release_device_list();
 }
 
 module_init(coresight_init);
diff --git a/drivers/hwtracing/coresight/coresight-ctcu-core.c b/drivers/hwtracing/coresight/coresight-ctcu-core.c
index abed15eb72b4a..9043cad42f01e 100644
--- a/drivers/hwtracing/coresight/coresight-ctcu-core.c
+++ b/drivers/hwtracing/coresight/coresight-ctcu-core.c
@@ -19,8 +19,6 @@
 #include "coresight-ctcu.h"
 #include "coresight-priv.h"
 
-DEFINE_CORESIGHT_DEVLIST(ctcu_devs, "ctcu");
-
 #define ctcu_writel(drvdata, val, offset)	__raw_writel((val), drvdata->base + offset)
 #define ctcu_readl(drvdata, offset)		__raw_readl(drvdata->base + offset)
 
@@ -187,7 +185,7 @@ static int ctcu_probe(struct platform_device *pdev)
 	void __iomem *base;
 	int i, ret;
 
-	desc.name = coresight_alloc_device_name(&ctcu_devs, dev);
+	desc.name = coresight_alloc_device_name("ctcu", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
@@ -228,6 +226,7 @@ static int ctcu_probe(struct platform_device *pdev)
 	desc.dev = dev;
 	desc.ops = &ctcu_ops;
 	desc.access = CSDEV_ACCESS_IOMEM(base);
+	raw_spin_lock_init(&drvdata->spin_lock);
 
 	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c
index bfbc365bb2ef2..2f4c9362709a9 100644
--- a/drivers/hwtracing/coresight/coresight-cti-core.c
+++ b/drivers/hwtracing/coresight/coresight-cti-core.c
@@ -42,21 +42,6 @@ static DEFINE_MUTEX(ect_mutex);
 #define csdev_to_cti_drvdata(csdev)	\
 	dev_get_drvdata(csdev->dev.parent)
 
-/* power management handling */
-static int nr_cti_cpu;
-
-/* quick lookup list for CPU bound CTIs when power handling */
-static struct cti_drvdata *cti_cpu_drvdata[NR_CPUS];
-
-/*
- * CTI naming. CTI bound to cores will have the name cti_cpu<N> where
- * N is the CPU ID. System CTIs will have the name cti_sys<I> where I
- * is an index allocated by order of discovery.
- *
- * CTI device name list - for CTI not bound to cores.
- */
-DEFINE_CORESIGHT_DEVLIST(cti_sys_devs, "cti_sys");
-
 /* write set of regs to hardware - call with spinlock claimed */
 void cti_write_all_hw_regs(struct cti_drvdata *drvdata)
 {
@@ -77,7 +62,8 @@ void cti_write_all_hw_regs(struct cti_drvdata *drvdata)
 
 	/* other regs */
 	writel_relaxed(config->ctigate, drvdata->base + CTIGATE);
-	writel_relaxed(config->asicctl, drvdata->base + ASICCTL);
+	if (config->asicctl_impl)
+		writel_relaxed(config->asicctl, drvdata->base + ASICCTL);
 	writel_relaxed(config->ctiappset, drvdata->base + CTIAPPSET);
 
 	/* re-enable CTI */
@@ -90,60 +76,24 @@ void cti_write_all_hw_regs(struct cti_drvdata *drvdata)
 static int cti_enable_hw(struct cti_drvdata *drvdata)
 {
 	struct cti_config *config = &drvdata->config;
-	unsigned long flags;
-	int rc = 0;
+	int rc;
 
-	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
-	/* no need to do anything if enabled or unpowered*/
-	if (config->hw_enabled || !config->hw_powered)
+	/* no need to do anything if enabled */
+	if (cti_is_active(config))
 		goto cti_state_unchanged;
 
 	/* claim the device */
 	rc = coresight_claim_device(drvdata->csdev);
 	if (rc)
-		goto cti_err_not_enabled;
+		return rc;
 
 	cti_write_all_hw_regs(drvdata);
 
-	config->hw_enabled = true;
-	drvdata->config.enable_req_count++;
-	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
-	return rc;
-
 cti_state_unchanged:
 	drvdata->config.enable_req_count++;
-
-	/* cannot enable due to error */
-cti_err_not_enabled:
-	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
-	return rc;
-}
-
-/* re-enable CTI on CPU when using CPU hotplug */
-static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata)
-{
-	struct cti_config *config = &drvdata->config;
-
-	raw_spin_lock(&drvdata->spinlock);
-	config->hw_powered = true;
-
-	/* no need to do anything if no enable request */
-	if (!drvdata->config.enable_req_count)
-		goto cti_hp_not_enabled;
-
-	/* try to claim the device */
-	if (coresight_claim_device(drvdata->csdev))
-		goto cti_hp_not_enabled;
-
-	cti_write_all_hw_regs(drvdata);
-	config->hw_enabled = true;
-	raw_spin_unlock(&drvdata->spinlock);
-	return;
-
-	/* did not re-enable due to no claim / no request */
-cti_hp_not_enabled:
-	raw_spin_unlock(&drvdata->spinlock);
+	return 0;
 }
 
 /* disable hardware */
@@ -151,39 +101,36 @@ static int cti_disable_hw(struct cti_drvdata *drvdata)
 {
 	struct cti_config *config = &drvdata->config;
 	struct coresight_device *csdev = drvdata->csdev;
-	int ret = 0;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
 	/* don't allow negative refcounts, return an error */
-	if (!drvdata->config.enable_req_count) {
-		ret = -EINVAL;
-		goto cti_not_disabled;
-	}
+	if (!cti_is_active(config))
+		return -EINVAL;
 
 	/* check refcount - disable on 0 */
 	if (--drvdata->config.enable_req_count > 0)
-		goto cti_not_disabled;
-
-	/* no need to do anything if disabled or cpu unpowered */
-	if (!config->hw_enabled || !config->hw_powered)
-		goto cti_not_disabled;
+		return 0;
 
 	CS_UNLOCK(drvdata->base);
 
 	/* disable CTI */
 	writel_relaxed(0, drvdata->base + CTICONTROL);
-	config->hw_enabled = false;
 
 	coresight_disclaim_device_unlocked(csdev);
 	CS_LOCK(drvdata->base);
-	raw_spin_unlock(&drvdata->spinlock);
-	return ret;
+	return 0;
+}
 
-	/* not disabled this call */
-cti_not_disabled:
-	raw_spin_unlock(&drvdata->spinlock);
-	return ret;
+u32 cti_read_single_reg(struct cti_drvdata *drvdata, int offset)
+{
+	int val;
+
+	CS_UNLOCK(drvdata->base);
+	val = readl_relaxed(drvdata->base + offset);
+	CS_LOCK(drvdata->base);
+
+	return val;
 }
 
 void cti_write_single_reg(struct cti_drvdata *drvdata, int offset, u32 value)
@@ -198,11 +145,11 @@ void cti_write_intack(struct device *dev, u32 ackval)
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct cti_config *config = &drvdata->config;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	/* write if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, CTIINTACK, ackval);
-	raw_spin_unlock(&drvdata->spinlock);
 }
 
 /*
@@ -240,6 +187,8 @@ static void cti_set_default_config(struct device *dev,
 	config->trig_filter_enable = true;
 	config->ctigate = GENMASK(config->nr_ctm_channels - 1, 0);
 	config->enable_req_count = 0;
+
+	config->asicctl_impl = !!FIELD_GET(GENMASK(4, 0), devid);
 }
 
 /*
@@ -369,7 +318,7 @@ int cti_channel_trig_op(struct device *dev, enum cti_chan_op op,
 	reg_offset = (direction == CTI_TRIG_IN ? CTIINEN(trigger_idx) :
 		      CTIOUTEN(trigger_idx));
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
 	/* read - modify write - the trigger / channel enable value */
 	reg_value = direction == CTI_TRIG_IN ? config->ctiinen[trigger_idx] :
@@ -386,9 +335,9 @@ int cti_channel_trig_op(struct device *dev, enum cti_chan_op op,
 		config->ctiouten[trigger_idx] = reg_value;
 
 	/* write through if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, reg_offset, reg_value);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return 0;
 }
 
@@ -406,7 +355,8 @@ int cti_channel_gate_op(struct device *dev, enum cti_chan_gate_op op,
 
 	chan_bitmask = BIT(channel_idx);
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	reg_value = config->ctigate;
 	switch (op) {
 	case CTI_GATE_CHAN_ENABLE:
@@ -423,10 +373,10 @@ int cti_channel_gate_op(struct device *dev, enum cti_chan_gate_op op,
 	}
 	if (err == 0) {
 		config->ctigate = reg_value;
-		if (cti_active(config))
+		if (cti_is_active(config))
 			cti_write_single_reg(drvdata, CTIGATE, reg_value);
 	}
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return err;
 }
 
@@ -445,7 +395,8 @@ int cti_channel_setop(struct device *dev, enum cti_chan_set_op op,
 
 	chan_bitmask = BIT(channel_idx);
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	reg_value = config->ctiappset;
 	switch (op) {
 	case CTI_CHAN_SET:
@@ -471,9 +422,8 @@ int cti_channel_setop(struct device *dev, enum cti_chan_set_op op,
 		break;
 	}
 
-	if ((err == 0) && cti_active(config))
+	if ((err == 0) && cti_is_active(config))
 		cti_write_single_reg(drvdata, reg_offset, reg_value);
-	raw_spin_unlock(&drvdata->spinlock);
 
 	return err;
 }
@@ -658,146 +608,6 @@ static void cti_remove_conn_xrefs(struct cti_drvdata *drvdata)
 	}
 }
 
-/** cti PM callbacks **/
-static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
-			     void *v)
-{
-	struct cti_drvdata *drvdata;
-	struct coresight_device *csdev;
-	unsigned int cpu = smp_processor_id();
-	int notify_res = NOTIFY_OK;
-
-	if (!cti_cpu_drvdata[cpu])
-		return NOTIFY_OK;
-
-	drvdata = cti_cpu_drvdata[cpu];
-	csdev = drvdata->csdev;
-
-	if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu))
-		return NOTIFY_BAD;
-
-	raw_spin_lock(&drvdata->spinlock);
-
-	switch (cmd) {
-	case CPU_PM_ENTER:
-		/* CTI regs all static - we have a copy & nothing to save */
-		drvdata->config.hw_powered = false;
-		if (drvdata->config.hw_enabled)
-			coresight_disclaim_device(csdev);
-		break;
-
-	case CPU_PM_ENTER_FAILED:
-		drvdata->config.hw_powered = true;
-		if (drvdata->config.hw_enabled) {
-			if (coresight_claim_device(csdev))
-				drvdata->config.hw_enabled = false;
-		}
-		break;
-
-	case CPU_PM_EXIT:
-		/* write hardware registers to re-enable. */
-		drvdata->config.hw_powered = true;
-		drvdata->config.hw_enabled = false;
-
-		/* check enable reference count to enable HW */
-		if (drvdata->config.enable_req_count) {
-			/* check we can claim the device as we re-power */
-			if (coresight_claim_device(csdev))
-				goto cti_notify_exit;
-
-			drvdata->config.hw_enabled = true;
-			cti_write_all_hw_regs(drvdata);
-		}
-		break;
-
-	default:
-		notify_res = NOTIFY_DONE;
-		break;
-	}
-
-cti_notify_exit:
-	raw_spin_unlock(&drvdata->spinlock);
-	return notify_res;
-}
-
-static struct notifier_block cti_cpu_pm_nb = {
-	.notifier_call = cti_cpu_pm_notify,
-};
-
-/* CPU HP handlers */
-static int cti_starting_cpu(unsigned int cpu)
-{
-	struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
-
-	if (!drvdata)
-		return 0;
-
-	cti_cpuhp_enable_hw(drvdata);
-	return 0;
-}
-
-static int cti_dying_cpu(unsigned int cpu)
-{
-	struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
-
-	if (!drvdata)
-		return 0;
-
-	raw_spin_lock(&drvdata->spinlock);
-	drvdata->config.hw_powered = false;
-	if (drvdata->config.hw_enabled)
-		coresight_disclaim_device(drvdata->csdev);
-	raw_spin_unlock(&drvdata->spinlock);
-	return 0;
-}
-
-static int cti_pm_setup(struct cti_drvdata *drvdata)
-{
-	int ret;
-
-	if (drvdata->ctidev.cpu == -1)
-		return 0;
-
-	if (nr_cti_cpu)
-		goto done;
-
-	cpus_read_lock();
-	ret = cpuhp_setup_state_nocalls_cpuslocked(
-			CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
-			"arm/coresight_cti:starting",
-			cti_starting_cpu, cti_dying_cpu);
-	if (ret) {
-		cpus_read_unlock();
-		return ret;
-	}
-
-	ret = cpu_pm_register_notifier(&cti_cpu_pm_nb);
-	cpus_read_unlock();
-	if (ret) {
-		cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_CTI_STARTING);
-		return ret;
-	}
-
-done:
-	nr_cti_cpu++;
-	cti_cpu_drvdata[drvdata->ctidev.cpu] = drvdata;
-
-	return 0;
-}
-
-/* release PM registrations */
-static void cti_pm_release(struct cti_drvdata *drvdata)
-{
-	if (drvdata->ctidev.cpu == -1)
-		return;
-
-	cti_cpu_drvdata[drvdata->ctidev.cpu] = NULL;
-	if (--nr_cti_cpu == 0) {
-		cpu_pm_unregister_notifier(&cti_cpu_pm_nb);
-		cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_CTI_STARTING);
-	}
-}
-
 /** cti ect operations **/
 int cti_enable(struct coresight_device *csdev, enum cs_mode mode,
 	       struct coresight_path *path)
@@ -823,17 +633,13 @@ static const struct coresight_ops cti_ops = {
 	.helper_ops = &cti_ops_ect,
 };
 
-/*
- * Free up CTI specific resources
- * called by dev->release, need to call down to underlying csdev release.
- */
-static void cti_device_release(struct device *dev)
+static void cti_remove(struct amba_device *adev)
 {
-	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	struct cti_drvdata *drvdata = dev_get_drvdata(&adev->dev);
 	struct cti_drvdata *ect_item, *ect_tmp;
 
 	mutex_lock(&ect_mutex);
-	cti_pm_release(drvdata);
+	cti_remove_conn_xrefs(drvdata);
 
 	/* remove from the list */
 	list_for_each_entry_safe(ect_item, ect_tmp, &ect_net, node) {
@@ -844,17 +650,6 @@ static void cti_device_release(struct device *dev)
 	}
 	mutex_unlock(&ect_mutex);
 
-	if (drvdata->csdev_release)
-		drvdata->csdev_release(dev);
-}
-static void cti_remove(struct amba_device *adev)
-{
-	struct cti_drvdata *drvdata = dev_get_drvdata(&adev->dev);
-
-	mutex_lock(&ect_mutex);
-	cti_remove_conn_xrefs(drvdata);
-	mutex_unlock(&ect_mutex);
-
 	coresight_unregister(drvdata->csdev);
 }
 
@@ -900,29 +695,27 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 		return  PTR_ERR(pdata);
 	}
 
-	/* default to powered - could change on PM notifications */
-	drvdata->config.hw_powered = true;
-
-	/* set up device name - will depend if cpu bound or otherwise */
+	/*
+	 * Set up device name - will depend if cpu bound or otherwise.
+	 *
+	 * CTI bound to cores will have the name cti_cpu<N> where N is th
+	 * eCPU ID. System CTIs will have the name cti_sys<I> where I is an
+	 * index allocated by order of discovery.
+	 */
 	if (drvdata->ctidev.cpu >= 0)
 		cti_desc.name = devm_kasprintf(dev, GFP_KERNEL, "cti_cpu%d",
 					       drvdata->ctidev.cpu);
 	else
-		cti_desc.name = coresight_alloc_device_name(&cti_sys_devs, dev);
+		cti_desc.name = coresight_alloc_device_name("cti_sys", dev);
 	if (!cti_desc.name)
 		return -ENOMEM;
 
-	/* setup CPU power management handling for CPU bound CTI devices. */
-	ret = cti_pm_setup(drvdata);
-	if (ret)
-		return ret;
-
 	/* create dynamic attributes for connections */
 	ret = cti_create_cons_sysfs(dev, drvdata);
 	if (ret) {
 		dev_err(dev, "%s: create dynamic sysfs entries failed\n",
 			cti_desc.name);
-		goto pm_release;
+		return ret;
 	}
 
 	/* set up coresight component description */
@@ -935,10 +728,8 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 
 	coresight_clear_self_claim_tag(&cti_desc.access);
 	drvdata->csdev = coresight_register(&cti_desc);
-	if (IS_ERR(drvdata->csdev)) {
-		ret = PTR_ERR(drvdata->csdev);
-		goto pm_release;
-	}
+	if (IS_ERR(drvdata->csdev))
+		return PTR_ERR(drvdata->csdev);
 
 	/* add to list of CTI devices */
 	mutex_lock(&ect_mutex);
@@ -947,18 +738,10 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 	cti_update_conn_xrefs(drvdata);
 	mutex_unlock(&ect_mutex);
 
-	/* set up release chain */
-	drvdata->csdev_release = drvdata->csdev->dev.release;
-	drvdata->csdev->dev.release = cti_device_release;
-
 	/* all done - dec pm refcount */
 	pm_runtime_put(&adev->dev);
 	dev_info(&drvdata->csdev->dev, "CTI initialized\n");
 	return 0;
-
-pm_release:
-	cti_pm_release(drvdata);
-	return ret;
 }
 
 static struct amba_cs_uci_id uci_id_cti[] = {
diff --git a/drivers/hwtracing/coresight/coresight-cti-sysfs.c b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
index 572b80ee96fbf..4c0a60840efb9 100644
--- a/drivers/hwtracing/coresight/coresight-cti-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
@@ -81,19 +81,12 @@ static ssize_t enable_show(struct device *dev,
 			   char *buf)
 {
 	int enable_req;
-	bool enabled, powered;
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 
-	raw_spin_lock(&drvdata->spinlock);
-	enable_req = drvdata->config.enable_req_count;
-	powered = drvdata->config.hw_powered;
-	enabled = drvdata->config.hw_enabled;
-	raw_spin_unlock(&drvdata->spinlock);
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock)
+		enable_req = cti_is_active(&drvdata->config);
 
-	if (powered)
-		return sprintf(buf, "%d\n", enabled);
-	else
-		return sprintf(buf, "%d\n", !!enable_req);
+	return sprintf(buf, "%d\n", !!enable_req);
 }
 
 static ssize_t enable_store(struct device *dev,
@@ -131,12 +124,7 @@ static ssize_t powered_show(struct device *dev,
 			    struct device_attribute *attr,
 			    char *buf)
 {
-	bool powered;
-	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
-
-	raw_spin_lock(&drvdata->spinlock);
-	powered = drvdata->config.hw_powered;
-	raw_spin_unlock(&drvdata->spinlock);
+	bool powered = pm_runtime_active(dev->parent);
 
 	return sprintf(buf, "%d\n", powered);
 }
@@ -181,10 +169,10 @@ static ssize_t coresight_cti_reg_show(struct device *dev,
 	u32 val = 0;
 
 	pm_runtime_get_sync(dev->parent);
-	raw_spin_lock(&drvdata->spinlock);
-	if (drvdata->config.hw_powered)
-		val = readl_relaxed(drvdata->base + cti_attr->off);
-	raw_spin_unlock(&drvdata->spinlock);
+
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock)
+		val = cti_read_single_reg(drvdata, cti_attr->off);
+
 	pm_runtime_put_sync(dev->parent);
 	return sysfs_emit(buf, "0x%x\n", val);
 }
@@ -202,10 +190,10 @@ static __maybe_unused ssize_t coresight_cti_reg_store(struct device *dev,
 		return -EINVAL;
 
 	pm_runtime_get_sync(dev->parent);
-	raw_spin_lock(&drvdata->spinlock);
-	if (drvdata->config.hw_powered)
+
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock)
 		cti_write_single_reg(drvdata, cti_attr->off, val);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	pm_runtime_put_sync(dev->parent);
 	return size;
 }
@@ -264,17 +252,19 @@ static ssize_t cti_reg32_show(struct device *dev, char *buf,
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct cti_config *config = &drvdata->config;
 
-	raw_spin_lock(&drvdata->spinlock);
-	if ((reg_offset >= 0) && cti_active(config)) {
-		CS_UNLOCK(drvdata->base);
-		val = readl_relaxed(drvdata->base + reg_offset);
-		if (pcached_val)
-			*pcached_val = val;
-		CS_LOCK(drvdata->base);
-	} else if (pcached_val) {
-		val = *pcached_val;
+	if (reg_offset < 0)
+		return -EINVAL;
+
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock) {
+		if (cti_is_active(config)) {
+			val = cti_read_single_reg(drvdata, reg_offset);
+			if (pcached_val)
+				*pcached_val = val;
+		} else if (pcached_val) {
+			val = *pcached_val;
+		}
 	}
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return sprintf(buf, "%#x\n", val);
 }
 
@@ -293,15 +283,19 @@ static ssize_t cti_reg32_store(struct device *dev, const char *buf,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
-	/* local store */
-	if (pcached_val)
-		*pcached_val = (u32)val;
+	if (reg_offset < 0)
+		return -EINVAL;
+
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock) {
+		/* local store */
+		if (pcached_val)
+			*pcached_val = (u32)val;
+
+		/* write through if offset and enabled */
+		if (cti_is_active(config))
+			cti_write_single_reg(drvdata, reg_offset, val);
+	}
 
-	/* write through if offset and enabled */
-	if ((reg_offset >= 0) && cti_active(config))
-		cti_write_single_reg(drvdata, reg_offset, val);
-	raw_spin_unlock(&drvdata->spinlock);
 	return size;
 }
 
@@ -349,9 +343,9 @@ static ssize_t inout_sel_store(struct device *dev,
 	if (val > (CTIINOUTEN_MAX - 1))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	drvdata->config.ctiinout_sel = val;
-	raw_spin_unlock(&drvdata->spinlock);
 	return size;
 }
 static DEVICE_ATTR_RW(inout_sel);
@@ -364,10 +358,11 @@ static ssize_t inen_show(struct device *dev,
 	int index;
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 
-	raw_spin_lock(&drvdata->spinlock);
-	index = drvdata->config.ctiinout_sel;
-	val = drvdata->config.ctiinen[index];
-	raw_spin_unlock(&drvdata->spinlock);
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock) {
+		index = drvdata->config.ctiinout_sel;
+		val = drvdata->config.ctiinen[index];
+	}
+
 	return sprintf(buf, "%#lx\n", val);
 }
 
@@ -383,14 +378,15 @@ static ssize_t inen_store(struct device *dev,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	index = config->ctiinout_sel;
 	config->ctiinen[index] = val;
 
 	/* write through if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, CTIINEN(index), val);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return size;
 }
 static DEVICE_ATTR_RW(inen);
@@ -403,10 +399,11 @@ static ssize_t outen_show(struct device *dev,
 	int index;
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 
-	raw_spin_lock(&drvdata->spinlock);
-	index = drvdata->config.ctiinout_sel;
-	val = drvdata->config.ctiouten[index];
-	raw_spin_unlock(&drvdata->spinlock);
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock) {
+		index = drvdata->config.ctiinout_sel;
+		val = drvdata->config.ctiouten[index];
+	}
+
 	return sprintf(buf, "%#lx\n", val);
 }
 
@@ -422,14 +419,15 @@ static ssize_t outen_store(struct device *dev,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	index = config->ctiinout_sel;
 	config->ctiouten[index] = val;
 
 	/* write through if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, CTIOUTEN(index), val);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return size;
 }
 static DEVICE_ATTR_RW(outen);
@@ -463,15 +461,15 @@ static ssize_t appclear_store(struct device *dev,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
 	/* a 1'b1 in appclr clears down the same bit in appset*/
 	config->ctiappset &= ~val;
 
 	/* write through if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, CTIAPPCLEAR, val);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return size;
 }
 static DEVICE_ATTR_WO(appclear);
@@ -487,12 +485,12 @@ static ssize_t apppulse_store(struct device *dev,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
 	/* write through if enabled */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_single_reg(drvdata, CTIAPPPULSE, val);
-	raw_spin_unlock(&drvdata->spinlock);
+
 	return size;
 }
 static DEVICE_ATTR_WO(apppulse);
@@ -530,6 +528,18 @@ static struct attribute *coresight_cti_regs_attrs[] = {
 	NULL,
 };
 
+static umode_t coresight_cti_regs_is_visible(struct kobject *kobj,
+					     struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	if (attr == &dev_attr_asicctl.attr && !drvdata->config.asicctl_impl)
+		return 0;
+
+	return attr->mode;
+}
+
 /* CTI channel x-trigger programming */
 static int
 cti_trig_op_parse(struct device *dev, enum cti_chan_op op,
@@ -681,9 +691,9 @@ static ssize_t trig_filter_enable_show(struct device *dev,
 	u32 val;
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 
-	raw_spin_lock(&drvdata->spinlock);
-	val = drvdata->config.trig_filter_enable;
-	raw_spin_unlock(&drvdata->spinlock);
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock)
+		val = drvdata->config.trig_filter_enable;
+
 	return sprintf(buf, "%d\n", val);
 }
 
@@ -697,9 +707,9 @@ static ssize_t trig_filter_enable_store(struct device *dev,
 	if (kstrtoul(buf, 0, &val))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	drvdata->config.trig_filter_enable = !!val;
-	raw_spin_unlock(&drvdata->spinlock);
 	return size;
 }
 static DEVICE_ATTR_RW(trig_filter_enable);
@@ -728,7 +738,7 @@ static ssize_t chan_xtrigs_reset_store(struct device *dev,
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	struct cti_config *config = &drvdata->config;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
 
 	/* clear the CTI trigger / channel programming registers */
 	for (i = 0; i < config->nr_trig_max; i++) {
@@ -744,10 +754,9 @@ static ssize_t chan_xtrigs_reset_store(struct device *dev,
 	config->xtrig_rchan_sel = 0;
 
 	/* if enabled then write through */
-	if (cti_active(config))
+	if (cti_is_active(config))
 		cti_write_all_hw_regs(drvdata);
 
-	raw_spin_unlock(&drvdata->spinlock);
 	return size;
 }
 static DEVICE_ATTR_WO(chan_xtrigs_reset);
@@ -768,9 +777,9 @@ static ssize_t chan_xtrigs_sel_store(struct device *dev,
 	if (val > (drvdata->config.nr_ctm_channels - 1))
 		return -EINVAL;
 
-	raw_spin_lock(&drvdata->spinlock);
+	guard(raw_spinlock_irqsave)(&drvdata->spinlock);
+
 	drvdata->config.xtrig_rchan_sel = val;
-	raw_spin_unlock(&drvdata->spinlock);
 	return size;
 }
 
@@ -781,9 +790,8 @@ static ssize_t chan_xtrigs_sel_show(struct device *dev,
 	unsigned long val;
 	struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent);
 
-	raw_spin_lock(&drvdata->spinlock);
-	val = drvdata->config.xtrig_rchan_sel;
-	raw_spin_unlock(&drvdata->spinlock);
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock)
+		val = drvdata->config.xtrig_rchan_sel;
 
 	return sprintf(buf, "%ld\n", val);
 }
@@ -838,12 +846,12 @@ static ssize_t print_chan_list(struct device *dev,
 	unsigned long inuse_bits = 0, chan_mask;
 
 	/* scan regs to get bitmap of channels in use. */
-	raw_spin_lock(&drvdata->spinlock);
-	for (i = 0; i < config->nr_trig_max; i++) {
-		inuse_bits |= config->ctiinen[i];
-		inuse_bits |= config->ctiouten[i];
+	scoped_guard(raw_spinlock_irqsave, &drvdata->spinlock) {
+		for (i = 0; i < config->nr_trig_max; i++) {
+			inuse_bits |= config->ctiinen[i];
+			inuse_bits |= config->ctiouten[i];
+		}
 	}
-	raw_spin_unlock(&drvdata->spinlock);
 
 	/* inverse bits if printing free channels */
 	if (!inuse)
@@ -1169,6 +1177,7 @@ static const struct attribute_group coresight_cti_mgmt_group = {
 
 static const struct attribute_group coresight_cti_regs_group = {
 	.attrs = coresight_cti_regs_attrs,
+	.is_visible = coresight_cti_regs_is_visible,
 	.name = "regs",
 };
 
diff --git a/drivers/hwtracing/coresight/coresight-cti.h b/drivers/hwtracing/coresight/coresight-cti.h
index 4f89091ee93f5..c5f9e79fabc60 100644
--- a/drivers/hwtracing/coresight/coresight-cti.h
+++ b/drivers/hwtracing/coresight/coresight-cti.h
@@ -119,9 +119,8 @@ struct cti_device {
  * @nr_trig_max: Max number of trigger signals implemented on device.
  *		 (max of trig_in or trig_out) - from ID register.
  * @nr_ctm_channels: number of available CTM channels - from ID register.
+ * @asicctl_impl: true if asicctl is implemented.
  * @enable_req_count: CTI is enabled alongside >=1 associated devices.
- * @hw_enabled: true if hw is currently enabled.
- * @hw_powered: true if associated cpu powered on, or no cpu.
  * @trig_in_use: bitfield of in triggers registered as in use.
  * @trig_out_use: bitfield of out triggers registered as in use.
  * @trig_out_filter: bitfield of out triggers that are blocked if filter
@@ -140,11 +139,10 @@ struct cti_config {
 	/* hardware description */
 	int nr_ctm_channels;
 	int nr_trig_max;
+	bool asicctl_impl;
 
 	/* cti enable control */
 	int enable_req_count;
-	bool hw_enabled;
-	bool hw_powered;
 
 	/* registered triggers and filtering */
 	u32 trig_in_use;
@@ -170,7 +168,6 @@ struct cti_config {
  * @spinlock:	Control data access to one at a time.
  * @config:	Configuration data for this CTI device.
  * @node:	List entry of this device in the list of CTI devices.
- * @csdev_release: release function for underlying coresight_device.
  */
 struct cti_drvdata {
 	void __iomem *base;
@@ -179,7 +176,6 @@ struct cti_drvdata {
 	raw_spinlock_t spinlock;
 	struct cti_config config;
 	struct list_head node;
-	void (*csdev_release)(struct device *dev);
 };
 
 /*
@@ -222,6 +218,7 @@ int cti_disable(struct coresight_device *csdev, struct coresight_path *path);
 void cti_write_all_hw_regs(struct cti_drvdata *drvdata);
 void cti_write_intack(struct device *dev, u32 ackval);
 void cti_write_single_reg(struct cti_drvdata *drvdata, int offset, u32 value);
+u32 cti_read_single_reg(struct cti_drvdata *drvdata, int offset);
 int cti_channel_trig_op(struct device *dev, enum cti_chan_op op,
 			enum cti_trig_dir direction, u32 channel_idx,
 			u32 trigger_idx);
@@ -234,10 +231,10 @@ struct coresight_platform_data *
 coresight_cti_get_platform_data(struct device *dev);
 const char *cti_plat_get_node_name(struct fwnode_handle *fwnode);
 
-/* cti powered and enabled */
-static inline bool cti_active(struct cti_config *cfg)
+/* Check if a cti device is enabled */
+static inline bool cti_is_active(struct cti_config *cfg)
 {
-	return cfg->hw_powered && cfg->hw_enabled;
+	return !!cfg->enable_req_count;
 }
 
 #endif  /* _CORESIGHT_CORESIGHT_CTI_H */
diff --git a/drivers/hwtracing/coresight/coresight-dummy.c b/drivers/hwtracing/coresight/coresight-dummy.c
index 14322c99e29d4..c176a2f57300f 100644
--- a/drivers/hwtracing/coresight/coresight-dummy.c
+++ b/drivers/hwtracing/coresight/coresight-dummy.c
@@ -19,9 +19,6 @@ struct dummy_drvdata {
 	u8				traceid;
 };
 
-DEFINE_CORESIGHT_DEVLIST(source_devs, "dummy_source");
-DEFINE_CORESIGHT_DEVLIST(sink_devs, "dummy_sink");
-
 static int dummy_source_enable(struct coresight_device *csdev,
 			       struct perf_event *event, enum cs_mode mode,
 			       __maybe_unused struct coresight_path *path)
@@ -126,7 +123,7 @@ static int dummy_probe(struct platform_device *pdev)
 
 	if (of_device_is_compatible(node, "arm,coresight-dummy-source")) {
 
-		desc.name = coresight_alloc_device_name(&source_devs, dev);
+		desc.name = coresight_alloc_device_name("dummy_source", dev);
 		if (!desc.name)
 			return -ENOMEM;
 
@@ -155,7 +152,7 @@ static int dummy_probe(struct platform_device *pdev)
 		drvdata->traceid = (u8)trace_id;
 
 	} else if (of_device_is_compatible(node, "arm,coresight-dummy-sink")) {
-		desc.name = coresight_alloc_device_name(&sink_devs, dev);
+		desc.name = coresight_alloc_device_name("dummy_sink", dev);
 		if (!desc.name)
 			return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index 6657602d8f2e1..b952a1d47f12f 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -63,8 +63,6 @@
 #define ETB_FFSR_BIT		1
 #define ETB_FRAME_SIZE_WORDS	4
 
-DEFINE_CORESIGHT_DEVLIST(etb_devs, "etb");
-
 /**
  * struct etb_drvdata - specifics associated to an ETB component
  * @base:	memory mapped base address for this component.
@@ -722,7 +720,7 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id)
 	struct resource *res = &adev->res;
 	struct coresight_desc desc = { 0 };
 
-	desc.name = coresight_alloc_device_name(&etb_devs, dev);
+	desc.name = coresight_alloc_device_name("etb", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 72017dcc3b7f1..f85dedf89a3f9 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -902,7 +902,10 @@ int etm_perf_add_symlink_sink(struct coresight_device *csdev)
 
 	if (csdev->type != CORESIGHT_DEV_TYPE_SINK &&
 	    csdev->type != CORESIGHT_DEV_TYPE_LINKSINK)
-		return -EINVAL;
+		return -EOPNOTSUPP;
+
+	if (!sink_ops(csdev)->alloc_buffer)
+		return -EOPNOTSUPP;
 
 	if (csdev->ea != NULL)
 		return -EINVAL;
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index 3b248e54471a3..3f56ceccd8c9f 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -30,8 +30,6 @@
 #define FUNNEL_HOLDTIME		(0x7 << FUNNEL_HOLDTIME_SHFT)
 #define FUNNEL_ENSx_MASK	0xff
 
-DEFINE_CORESIGHT_DEVLIST(funnel_devs, "funnel");
-
 /**
  * struct funnel_drvdata - specifics associated to a funnel component
  * @base:	memory mapped base address for this component.
@@ -223,7 +221,7 @@ static int funnel_probe(struct device *dev, struct resource *res)
 	    of_device_is_compatible(dev->of_node, "arm,coresight-funnel"))
 		dev_warn_once(dev, "Uses OBSOLETE CoreSight funnel binding\n");
 
-	desc.name = coresight_alloc_device_name(&funnel_devs, dev);
+	desc.name = coresight_alloc_device_name("funnel", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c
index 0db64c5f49959..0ca3bd7624543 100644
--- a/drivers/hwtracing/coresight/coresight-platform.c
+++ b/drivers/hwtracing/coresight/coresight-platform.c
@@ -849,7 +849,7 @@ coresight_get_platform_data(struct device *dev)
 error:
 	if (!IS_ERR_OR_NULL(pdata))
 		/* Cleanup the connection information */
-		coresight_release_platform_data(NULL, dev, pdata);
+		coresight_release_platform_data(dev, pdata);
 	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(coresight_get_platform_data);
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index fd896ac07942e..1ea882dffd703 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -239,8 +239,7 @@ static inline void *coresight_get_uci_data_from_amba(const struct amba_id *table
 	return NULL;
 }
 
-void coresight_release_platform_data(struct coresight_device *csdev,
-				     struct device *dev,
+void coresight_release_platform_data(struct device *dev,
 				     struct coresight_platform_data *pdata);
 struct coresight_device *
 coresight_find_csdev_by_fwnode(struct fwnode_handle *r_fwnode);
diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c
index e6472658235dc..07fc04f53b88f 100644
--- a/drivers/hwtracing/coresight/coresight-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-replicator.c
@@ -24,8 +24,6 @@
 #define REPLICATOR_IDFILTER0		0x000
 #define REPLICATOR_IDFILTER1		0x004
 
-DEFINE_CORESIGHT_DEVLIST(replicator_devs, "replicator");
-
 /**
  * struct replicator_drvdata - specifics associated to a replicator component
  * @base:	memory mapped base address for this component. Also indicates
@@ -230,7 +228,7 @@ static int replicator_probe(struct device *dev, struct resource *res)
 		dev_warn_once(dev,
 			      "Uses OBSOLETE CoreSight replicator binding\n");
 
-	desc.name = coresight_alloc_device_name(&replicator_devs, dev);
+	desc.name = coresight_alloc_device_name("replicator", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c
index e68529bf89c98..aca6cec7885a5 100644
--- a/drivers/hwtracing/coresight/coresight-stm.c
+++ b/drivers/hwtracing/coresight/coresight-stm.c
@@ -110,8 +110,6 @@ struct channel_space {
 	unsigned long		*guaranteed;
 };
 
-DEFINE_CORESIGHT_DEVLIST(stm_devs, "stm");
-
 /**
  * struct stm_drvdata - specifics associated to an STM component
  * @base:		memory mapped base address for this component.
@@ -834,7 +832,7 @@ static int __stm_probe(struct device *dev, struct resource *res)
 	struct resource ch_res;
 	struct coresight_desc desc = { 0 };
 
-	desc.name = coresight_alloc_device_name(&stm_devs, dev);
+	desc.name = coresight_alloc_device_name("stm", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c
index 36599c431be62..c89fe996af231 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-core.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-core.c
@@ -32,10 +32,6 @@
 #include "coresight-priv.h"
 #include "coresight-tmc.h"
 
-DEFINE_CORESIGHT_DEVLIST(etb_devs, "tmc_etb");
-DEFINE_CORESIGHT_DEVLIST(etf_devs, "tmc_etf");
-DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr");
-
 int tmc_wait_for_tmcready(struct tmc_drvdata *drvdata)
 {
 	struct coresight_device *csdev = drvdata->csdev;
@@ -401,7 +397,6 @@ static ssize_t tmc_crashdata_read(struct file *file, char __user *data,
 
 static int tmc_crashdata_release(struct inode *inode, struct file *file)
 {
-	int ret = 0;
 	unsigned long flags;
 	struct tmc_resrv_buf *rbuf;
 	struct tmc_drvdata *drvdata = container_of(file->private_data,
@@ -414,7 +409,7 @@ static int tmc_crashdata_release(struct inode *inode, struct file *file)
 	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
 	dev_dbg(&drvdata->csdev->dev, "%s: released\n", __func__);
-	return ret;
+	return 0;
 }
 
 static const struct file_operations tmc_crashdata_fops = {
@@ -777,7 +772,7 @@ static int __tmc_probe(struct device *dev, struct resource *res)
 	struct coresight_platform_data *pdata = NULL;
 	struct tmc_drvdata *drvdata;
 	struct coresight_desc desc = { 0 };
-	struct coresight_dev_list *dev_list = NULL;
+	const char *dev_list = NULL;
 
 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
 	if (!drvdata)
@@ -827,7 +822,7 @@ static int __tmc_probe(struct device *dev, struct resource *res)
 		desc.type = CORESIGHT_DEV_TYPE_SINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.ops = &tmc_etb_cs_ops;
-		dev_list = &etb_devs;
+		dev_list = "tmc_etb";
 		break;
 	case TMC_CONFIG_TYPE_ETR:
 		desc.groups = coresight_etr_groups;
@@ -839,7 +834,7 @@ static int __tmc_probe(struct device *dev, struct resource *res)
 			goto out;
 		idr_init(&drvdata->idr);
 		mutex_init(&drvdata->idr_mutex);
-		dev_list = &etr_devs;
+		dev_list = "tmc_etr";
 		break;
 	case TMC_CONFIG_TYPE_ETF:
 		desc.groups = coresight_etf_groups;
@@ -847,7 +842,7 @@ static int __tmc_probe(struct device *dev, struct resource *res)
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
 		desc.ops = &tmc_etf_cs_ops;
-		dev_list = &etf_devs;
+		dev_list = "tmc_etf";
 		break;
 	default:
 		pr_err("%s: Unsupported TMC config\n", desc.name);
diff --git a/drivers/hwtracing/coresight/coresight-tnoc.c b/drivers/hwtracing/coresight/coresight-tnoc.c
index 1128612e70a73..96a25877b8240 100644
--- a/drivers/hwtracing/coresight/coresight-tnoc.c
+++ b/drivers/hwtracing/coresight/coresight-tnoc.c
@@ -47,8 +47,6 @@ struct trace_noc_drvdata {
 	int			atid;
 };
 
-DEFINE_CORESIGHT_DEVLIST(trace_noc_devs, "traceNoc");
-
 static void trace_noc_enable_hw(struct trace_noc_drvdata *drvdata)
 {
 	u32 val;
@@ -191,7 +189,7 @@ static int _tnoc_probe(struct device *dev, struct resource *res)
 	struct coresight_desc desc = { 0 };
 	int ret;
 
-	desc.name = coresight_alloc_device_name(&trace_noc_devs, dev);
+	desc.name = coresight_alloc_device_name("traceNoc", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c
index 7055f8f134278..89c8f71f0aff0 100644
--- a/drivers/hwtracing/coresight/coresight-tpda.c
+++ b/drivers/hwtracing/coresight/coresight-tpda.c
@@ -20,8 +20,6 @@
 #include "coresight-trace-id.h"
 #include "coresight-tpdm.h"
 
-DEFINE_CORESIGHT_DEVLIST(tpda_devs, "tpda");
-
 static void tpda_clear_element_size(struct coresight_device *csdev)
 {
 	struct tpda_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
@@ -585,7 +583,7 @@ static int tpda_probe(struct amba_device *adev, const struct amba_id *id)
 	if (ret)
 		return ret;
 
-	desc.name = coresight_alloc_device_name(&tpda_devs, dev);
+	desc.name = coresight_alloc_device_name("tpda", dev);
 	if (!desc.name)
 		return -ENOMEM;
 	desc.type = CORESIGHT_DEV_TYPE_LINK;
diff --git a/drivers/hwtracing/coresight/coresight-tpdm.c b/drivers/hwtracing/coresight/coresight-tpdm.c
index 06e0a905a67d7..da77bdaad0a45 100644
--- a/drivers/hwtracing/coresight/coresight-tpdm.c
+++ b/drivers/hwtracing/coresight/coresight-tpdm.c
@@ -19,8 +19,6 @@
 #include "coresight-priv.h"
 #include "coresight-tpdm.h"
 
-DEFINE_CORESIGHT_DEVLIST(tpdm_devs, "tpdm");
-
 static bool tpdm_has_dsb_dataset(struct tpdm_drvdata *drvdata)
 {
 	return (drvdata->datasets & TPDM_PIDR0_DS_DSB);
@@ -1416,7 +1414,7 @@ static int tpdm_probe(struct device *dev, struct resource *res)
 	}
 
 	/* Set up coresight component description */
-	desc.name = coresight_alloc_device_name(&tpdm_devs, dev);
+	desc.name = coresight_alloc_device_name("tpdm", dev);
 	if (!desc.name)
 		return -ENOMEM;
 	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c
index aaa44bc521c33..b8560b140e0fa 100644
--- a/drivers/hwtracing/coresight/coresight-tpiu.c
+++ b/drivers/hwtracing/coresight/coresight-tpiu.c
@@ -49,8 +49,6 @@
 #define FFCR_FON_MAN		BIT(6)
 #define FFCR_STOP_FI		BIT(12)
 
-DEFINE_CORESIGHT_DEVLIST(tpiu_devs, "tpiu");
-
 /*
  * @base:	memory mapped base address for this component.
  * @atclk:	optional clock for the core parts of the TPIU.
@@ -134,7 +132,7 @@ static int __tpiu_probe(struct device *dev, struct resource *res)
 	struct coresight_desc desc = { 0 };
 	int ret;
 
-	desc.name = coresight_alloc_device_name(&tpiu_devs, dev);
+	desc.name = coresight_alloc_device_name("tpiu", dev);
 	if (!desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c
index 8f7922a5e534f..5776f63468fa0 100644
--- a/drivers/hwtracing/coresight/ultrasoc-smb.c
+++ b/drivers/hwtracing/coresight/ultrasoc-smb.c
@@ -17,8 +17,6 @@
 #include "coresight-priv.h"
 #include "ultrasoc-smb.h"
 
-DEFINE_CORESIGHT_DEVLIST(sink_devs, "ultra_smb");
-
 #define ULTRASOC_SMB_DSM_UUID	"82ae1283-7f6a-4cbe-aa06-53e8fb24db18"
 
 static bool smb_buffer_not_empty(struct smb_drv_data *drvdata)
@@ -478,7 +476,7 @@ static int smb_register_sink(struct platform_device *pdev,
 	desc.pdata = pdata;
 	desc.dev = &pdev->dev;
 	desc.groups = smb_sink_groups;
-	desc.name = coresight_alloc_device_name(&sink_devs, &pdev->dev);
+	desc.name = coresight_alloc_device_name("ultra_smb", &pdev->dev);
 	if (!desc.name) {
 		dev_err(&pdev->dev, "Failed to alloc coresight device name");
 		return -ENOMEM;
diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c
index 37584e786bb52..f48c6a8a0654e 100644
--- a/drivers/hwtracing/stm/core.c
+++ b/drivers/hwtracing/stm/core.c
@@ -666,6 +666,16 @@ static ssize_t stm_char_write(struct file *file, const char __user *buf,
 	return count;
 }
 
+static int stm_mmap_mapped(unsigned long start, unsigned long end, pgoff_t pgoff,
+			   const struct file *file, void **vm_private_data)
+{
+	struct stm_file *stmf = file->private_data;
+	struct stm_device *stm = stmf->stm;
+
+	pm_runtime_get_sync(&stm->dev);
+	return 0;
+}
+
 static void stm_mmap_open(struct vm_area_struct *vma)
 {
 	struct stm_file *stmf = vma->vm_file->private_data;
@@ -684,12 +694,14 @@ static void stm_mmap_close(struct vm_area_struct *vma)
 }
 
 static const struct vm_operations_struct stm_mmap_vmops = {
+	.mapped = stm_mmap_mapped,
 	.open	= stm_mmap_open,
 	.close	= stm_mmap_close,
 };
 
-static int stm_char_mmap(struct file *file, struct vm_area_struct *vma)
+static int stm_char_mmap_prepare(struct vm_area_desc *desc)
 {
+	struct file *file = desc->file;
 	struct stm_file *stmf = file->private_data;
 	struct stm_device *stm = stmf->stm;
 	unsigned long size, phys;
@@ -697,10 +709,10 @@ static int stm_char_mmap(struct file *file, struct vm_area_struct *vma)
 	if (!stm->data->mmio_addr)
 		return -EOPNOTSUPP;
 
-	if (vma->vm_pgoff)
+	if (desc->pgoff)
 		return -EINVAL;
 
-	size = vma->vm_end - vma->vm_start;
+	size = vma_desc_size(desc);
 
 	if (stmf->output.nr_chans * stm->data->sw_mmiosz != size)
 		return -EINVAL;
@@ -712,13 +724,12 @@ static int stm_char_mmap(struct file *file, struct vm_area_struct *vma)
 	if (!phys)
 		return -EINVAL;
 
-	pm_runtime_get_sync(&stm->dev);
-
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-	vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_ops = &stm_mmap_vmops;
-	vm_iomap_memory(vma, phys, size);
+	desc->page_prot = pgprot_noncached(desc->page_prot);
+	vma_desc_set_flags(desc, VMA_IO_BIT, VMA_DONTEXPAND_BIT,
+			   VMA_DONTDUMP_BIT);
+	desc->vm_ops = &stm_mmap_vmops;
 
+	mmap_action_simple_ioremap(desc, phys, size);
 	return 0;
 }
 
@@ -836,7 +847,7 @@ static const struct file_operations stm_fops = {
 	.open		= stm_char_open,
 	.release	= stm_char_release,
 	.write		= stm_char_write,
-	.mmap		= stm_char_mmap,
+	.mmap_prepare	= stm_char_mmap_prepare,
 	.unlocked_ioctl	= stm_char_ioctl,
 	.compat_ioctl	= compat_ptr_ioctl,
 };
diff --git a/drivers/i2c/busses/i2c-cp2615.c b/drivers/i2c/busses/i2c-cp2615.c
index e2d7cd2390fc4..8212875700e15 100644
--- a/drivers/i2c/busses/i2c-cp2615.c
+++ b/drivers/i2c/busses/i2c-cp2615.c
@@ -298,6 +298,9 @@ cp2615_i2c_probe(struct usb_interface *usbif, const struct usb_device_id *id)
 	if (!adap)
 		return -ENOMEM;
 
+	if (!usbdev->serial)
+		return -EINVAL;
+
 	strscpy(adap->name, usbdev->serial, sizeof(adap->name));
 	adap->owner = THIS_MODULE;
 	adap->dev.parent = &usbif->dev;
diff --git a/drivers/i2c/busses/i2c-diolan-u2c.c b/drivers/i2c/busses/i2c-diolan-u2c.c
index 077b093ba8343..d502f6e4732fc 100644
--- a/drivers/i2c/busses/i2c-diolan-u2c.c
+++ b/drivers/i2c/busses/i2c-diolan-u2c.c
@@ -427,12 +427,6 @@ static const struct usb_device_id diolan_u2c_table[] = {
 
 MODULE_DEVICE_TABLE(usb, diolan_u2c_table);
 
-static void diolan_u2c_free(struct i2c_diolan_u2c *dev)
-{
-	usb_put_dev(dev->usb_dev);
-	kfree(dev);
-}
-
 static int diolan_u2c_probe(struct usb_interface *interface,
 			    const struct usb_device_id *id)
 {
@@ -453,7 +447,7 @@ static int diolan_u2c_probe(struct usb_interface *interface,
 	dev->ep_out = hostif->endpoint[0].desc.bEndpointAddress;
 	dev->ep_in = hostif->endpoint[1].desc.bEndpointAddress;
 
-	dev->usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	dev->usb_dev = interface_to_usbdev(interface);
 	dev->interface = interface;
 
 	/* save our data pointer in this interface device */
@@ -488,7 +482,7 @@ static int diolan_u2c_probe(struct usb_interface *interface,
 
 error_free:
 	usb_set_intfdata(interface, NULL);
-	diolan_u2c_free(dev);
+	kfree(dev);
 error:
 	return ret;
 }
@@ -499,7 +493,7 @@ static void diolan_u2c_disconnect(struct usb_interface *interface)
 
 	i2c_del_adapter(&dev->adapter);
 	usb_set_intfdata(interface, NULL);
-	diolan_u2c_free(dev);
+	kfree(dev);
 
 	dev_dbg(&interface->dev, "disconnected\n");
 }
diff --git a/drivers/i2c/busses/i2c-fsi.c b/drivers/i2c/busses/i2c-fsi.c
index 82c87e04ac6fe..b2dc5ae1d0e4b 100644
--- a/drivers/i2c/busses/i2c-fsi.c
+++ b/drivers/i2c/busses/i2c-fsi.c
@@ -729,6 +729,7 @@ static int fsi_i2c_probe(struct fsi_device *fsi_dev)
 		rc = i2c_add_adapter(&port->adapter);
 		if (rc < 0) {
 			dev_err(dev, "Failed to register adapter: %d\n", rc);
+			of_node_put(np);
 			kfree(port);
 			continue;
 		}
diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c
index 8b7e15240fb0b..f667a873b81e0 100644
--- a/drivers/i2c/busses/i2c-npcm7xx.c
+++ b/drivers/i2c/busses/i2c-npcm7xx.c
@@ -1384,7 +1384,7 @@ static irqreturn_t npcm_i2c_int_slave_handler(struct npcm_i2c *bus)
 		 */
 		bus->operation = I2C_NO_OPER;
 		bus->own_slave_addr = 0xFF;
-		i2c_slave_event(bus->slave, I2C_SLAVE_STOP, 0);
+		i2c_slave_event(bus->slave, I2C_SLAVE_STOP, NULL);
 		iowrite8(NPCM_I2CST_SLVSTP, bus->reg + NPCM_I2CST);
 		if (bus->fifo_use) {
 			npcm_i2c_clear_fifo_int(bus);
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index d9f590f0c3843..f02d294db42a6 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -1453,27 +1453,16 @@ omap_i2c_probe(struct platform_device *pdev)
 				       (1000 * omap->speed / 8);
 	}
 
-	if (of_property_present(node, "mux-states")) {
-		struct mux_state *mux_state;
-
-		mux_state = devm_mux_state_get(&pdev->dev, NULL);
-		if (IS_ERR(mux_state)) {
-			r = PTR_ERR(mux_state);
-			dev_dbg(&pdev->dev, "failed to get I2C mux: %d\n", r);
-			goto err_put_pm;
-		}
-		omap->mux_state = mux_state;
-		r = mux_state_select(omap->mux_state);
-		if (r) {
-			dev_err(&pdev->dev, "failed to select I2C mux: %d\n", r);
-			goto err_put_pm;
-		}
+	omap->mux_state = devm_mux_state_get_optional_selected(&pdev->dev, NULL);
+	if (IS_ERR(omap->mux_state)) {
+		r = PTR_ERR(omap->mux_state);
+		goto err_put_pm;
 	}
 
 	/* reset ASAP, clearing any IRQs */
 	r = omap_i2c_init(omap);
 	if (r)
-		goto err_mux_state_deselect;
+		goto err_put_pm;
 
 	if (omap->rev < OMAP_I2C_OMAP1_REV_2)
 		r = devm_request_irq(&pdev->dev, omap->irq, omap_i2c_omap1_isr,
@@ -1515,9 +1504,6 @@ omap_i2c_probe(struct platform_device *pdev)
 
 err_unuse_clocks:
 	omap_i2c_write_reg(omap, OMAP_I2C_CON_REG, 0);
-err_mux_state_deselect:
-	if (omap->mux_state)
-		mux_state_deselect(omap->mux_state);
 err_put_pm:
 	pm_runtime_put_sync(omap->dev);
 err_disable_pm:
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 09af3b3625f11..f55840b2eb9ab 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -268,6 +268,7 @@ struct pxa_i2c {
 	struct pinctrl		*pinctrl;
 	struct pinctrl_state	*pinctrl_default;
 	struct pinctrl_state	*pinctrl_recovery;
+	bool			reset_before_xfer;
 };
 
 #define _IBMR(i2c)	((i2c)->reg_ibmr)
@@ -1144,6 +1145,11 @@ static int i2c_pxa_xfer(struct i2c_adapter *adap,
 {
 	struct pxa_i2c *i2c = adap->algo_data;
 
+	if (i2c->reset_before_xfer) {
+		i2c_pxa_reset(i2c);
+		i2c->reset_before_xfer = false;
+	}
+
 	return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_xfer);
 }
 
@@ -1521,7 +1527,16 @@ static int i2c_pxa_probe(struct platform_device *dev)
 		}
 	}
 
-	i2c_pxa_reset(i2c);
+	/*
+	 * Skip reset on Armada 3700 when recovery is used to avoid
+	 * controller hang due to the pinctrl state changes done by
+	 * the generic recovery initialization code. The reset will
+	 * be performed later, prior to the first transfer.
+	 */
+	if (i2c_type == REGS_A3700 && i2c->adap.bus_recovery_info)
+		i2c->reset_before_xfer = true;
+	else
+		i2c_pxa_reset(i2c);
 
 	ret = i2c_add_numbered_adapter(&i2c->adap);
 	if (ret < 0)
diff --git a/drivers/i2c/busses/i2c-robotfuzz-osif.c b/drivers/i2c/busses/i2c-robotfuzz-osif.c
index e0a76fb5bc31f..412fa8e37f69e 100644
--- a/drivers/i2c/busses/i2c-robotfuzz-osif.c
+++ b/drivers/i2c/busses/i2c-robotfuzz-osif.c
@@ -141,7 +141,7 @@ static int osif_probe(struct usb_interface *interface,
 	if (!priv)
 		return -ENOMEM;
 
-	priv->usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	priv->usb_dev = interface_to_usbdev(interface);
 	priv->interface = interface;
 
 	usb_set_intfdata(interface, priv);
@@ -163,7 +163,6 @@ static int osif_probe(struct usb_interface *interface,
 			    NULL, 0);
 	if (ret) {
 		dev_err(&interface->dev, "failure sending bit rate");
-		usb_put_dev(priv->usb_dev);
 		return ret;
 	}
 
@@ -184,7 +183,6 @@ static void osif_disconnect(struct usb_interface *interface)
 
 	i2c_del_adapter(&(priv->adapter));
 	usb_set_intfdata(interface, NULL);
-	usb_put_dev(priv->usb_dev);
 }
 
 static struct usb_driver osif_driver = {
diff --git a/drivers/i2c/busses/i2c-tiny-usb.c b/drivers/i2c/busses/i2c-tiny-usb.c
index 9ef495f88ef20..88d66593d9fc1 100644
--- a/drivers/i2c/busses/i2c-tiny-usb.c
+++ b/drivers/i2c/busses/i2c-tiny-usb.c
@@ -213,12 +213,6 @@ static int usb_write(struct i2c_adapter *adapter, int cmd,
 	return ret;
 }
 
-static void i2c_tiny_usb_free(struct i2c_tiny_usb *dev)
-{
-	usb_put_dev(dev->usb_dev);
-	kfree(dev);
-}
-
 static int i2c_tiny_usb_probe(struct usb_interface *interface,
 			      const struct usb_device_id *id)
 {
@@ -237,7 +231,7 @@ static int i2c_tiny_usb_probe(struct usb_interface *interface,
 	if (!dev)
 		goto error;
 
-	dev->usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	dev->usb_dev = interface_to_usbdev(interface);
 	dev->interface = interface;
 
 	/* save our data pointer in this interface device */
@@ -277,8 +271,7 @@ static int i2c_tiny_usb_probe(struct usb_interface *interface,
 	return 0;
 
  error:
-	if (dev)
-		i2c_tiny_usb_free(dev);
+	kfree(dev);
 
 	return retval;
 }
@@ -289,7 +282,7 @@ static void i2c_tiny_usb_disconnect(struct usb_interface *interface)
 
 	i2c_del_adapter(&dev->adapter);
 	usb_set_intfdata(interface, NULL);
-	i2c_tiny_usb_free(dev);
+	kfree(dev);
 
 	dev_dbg(&interface->dev, "disconnected\n");
 }
diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c
index 9e6be49bebb2c..c32847bc4d0d0 100644
--- a/drivers/i3c/master.c
+++ b/drivers/i3c/master.c
@@ -758,6 +758,32 @@ static ssize_t dev_nack_retry_count_store(struct device *dev,
 
 static DEVICE_ATTR_RW(dev_nack_retry_count);
 
+static ssize_t do_daa_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct i3c_master_controller *master = dev_to_i3cmaster(dev);
+	bool val;
+	int ret;
+
+	if (kstrtobool(buf, &val))
+		return -EINVAL;
+
+	if (!val)
+		return -EINVAL;
+
+	if (!master->init_done)
+		return -EAGAIN;
+
+	ret = i3c_master_do_daa(master);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DEVICE_ATTR_WO(do_daa);
+
 static struct attribute *i3c_masterdev_attrs[] = {
 	&dev_attr_mode.attr,
 	&dev_attr_current_master.attr,
@@ -769,6 +795,7 @@ static struct attribute *i3c_masterdev_attrs[] = {
 	&dev_attr_dynamic_address.attr,
 	&dev_attr_hdrcap.attr,
 	&dev_attr_hotjoin.attr,
+	&dev_attr_do_daa.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(i3c_masterdev);
@@ -2793,10 +2820,10 @@ struct i3c_generic_ibi_slot {
 struct i3c_generic_ibi_pool {
 	spinlock_t lock;
 	unsigned int num_slots;
-	struct i3c_generic_ibi_slot *slots;
 	void *payload_buf;
 	struct list_head free_slots;
 	struct list_head pending;
+	struct i3c_generic_ibi_slot slots[] __counted_by(num_slots);
 };
 
 /**
@@ -2824,7 +2851,6 @@ void i3c_generic_ibi_free_pool(struct i3c_generic_ibi_pool *pool)
 	WARN_ON(nslots != pool->num_slots);
 
 	kfree(pool->payload_buf);
-	kfree(pool->slots);
 	kfree(pool);
 }
 EXPORT_SYMBOL_GPL(i3c_generic_ibi_free_pool);
@@ -2847,20 +2873,16 @@ i3c_generic_ibi_alloc_pool(struct i3c_dev_desc *dev,
 	unsigned int i;
 	int ret;
 
-	pool = kzalloc_obj(*pool);
+	pool = kzalloc_flex(*pool, slots, req->num_slots);
 	if (!pool)
 		return ERR_PTR(-ENOMEM);
 
+	pool->num_slots = req->num_slots;
+
 	spin_lock_init(&pool->lock);
 	INIT_LIST_HEAD(&pool->free_slots);
 	INIT_LIST_HEAD(&pool->pending);
 
-	pool->slots = kzalloc_objs(*slot, req->num_slots);
-	if (!pool->slots) {
-		ret = -ENOMEM;
-		goto err_free_pool;
-	}
-
 	if (req->max_payload_len) {
 		pool->payload_buf = kcalloc(req->num_slots,
 					    req->max_payload_len, GFP_KERNEL);
@@ -2879,7 +2901,6 @@ i3c_generic_ibi_alloc_pool(struct i3c_dev_desc *dev,
 					  (i * req->max_payload_len);
 
 		list_add_tail(&slot->node, &pool->free_slots);
-		pool->num_slots++;
 	}
 
 	return pool;
diff --git a/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
index 30302e4d08e2a..22a5ba4ad7460 100644
--- a/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
+++ b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
@@ -337,6 +337,9 @@ static const struct pci_device_id mipi_i3c_hci_pci_devices[] = {
 	/* Nova Lake-S */
 	{ PCI_VDEVICE(INTEL, 0x6e2c), (kernel_ulong_t)&intel_mi_1_info},
 	{ PCI_VDEVICE(INTEL, 0x6e2d), (kernel_ulong_t)&intel_mi_2_info},
+	/* Nova Lake-H */
+	{ PCI_VDEVICE(INTEL, 0xd37c), (kernel_ulong_t)&intel_mi_1_info},
+	{ PCI_VDEVICE(INTEL, 0xd36f), (kernel_ulong_t)&intel_mi_2_info},
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, mipi_i3c_hci_pci_devices);
diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c
index b84b324e41116..8fefe2927f262 100644
--- a/drivers/i3c/master/svc-i3c-master.c
+++ b/drivers/i3c/master/svc-i3c-master.c
@@ -344,7 +344,7 @@ static void svc_i3c_master_reset_fifo_trigger(struct svc_i3c_master *master)
 {
 	u32 reg;
 
-	/* Set RX and TX tigger levels, flush FIFOs */
+	/* Set RX and TX trigger levels, flush FIFOs */
 	reg = SVC_I3C_MDATACTRL_FLUSHTB |
 	      SVC_I3C_MDATACTRL_FLUSHRB |
 	      SVC_I3C_MDATACTRL_UNLOCK_TRIG |
@@ -572,7 +572,7 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master)
 	 * 3. IBI isr writes an AutoIBI request.
 	 * 4. The controller will not start AutoIBI process because SDA is not low.
 	 * 5. IBIWON polling times out.
-	 * 6. Controller reamins in AutoIBI state and doesn't accept EmitStop request.
+	 * 6. Controller remains in AutoIBI state and doesn't accept EmitStop request.
 	 */
 	writel(SVC_I3C_MCTRL_REQUEST_START_ADDR |
 	       SVC_I3C_MCTRL_TYPE_I3C |
@@ -774,7 +774,7 @@ static int svc_i3c_master_bus_init(struct i3c_master_controller *m)
 
 	/*
 	 * Using I3C Open-Drain mode, target is 4.17MHz/240ns with a
-	 * duty-cycle tuned so that high levels are filetered out by
+	 * duty-cycle tuned so that high levels are filtered out by
 	 * the 50ns filter (target being 40ns).
 	 */
 	odhpp = 1;
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index f49c939d636f4..f49354e377777 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -983,6 +983,43 @@ static struct cpuidle_state mtl_l_cstates[] __initdata = {
 		.enter = NULL }
 };
 
+static struct cpuidle_state ptl_cstates[] __initdata = {
+	{
+		.name = "C1",
+		.desc = "MWAIT 0x00",
+		.flags = MWAIT2flg(0x00),
+		.exit_latency = 1,
+		.target_residency = 1,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C1E",
+		.desc = "MWAIT 0x01",
+		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+		.exit_latency = 10,
+		.target_residency = 10,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C6S",
+		.desc = "MWAIT 0x21",
+		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 300,
+		.target_residency = 300,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C10",
+		.desc = "MWAIT 0x60",
+		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 370,
+		.target_residency = 2500,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.enter = NULL }
+};
+
 static struct cpuidle_state gmt_cstates[] __initdata = {
 	{
 		.name = "C1",
@@ -1561,6 +1598,10 @@ static const struct idle_cpu idle_cpu_mtl_l __initconst = {
 	.state_table = mtl_l_cstates,
 };
 
+static const struct idle_cpu idle_cpu_ptl __initconst = {
+	.state_table = ptl_cstates,
+};
+
 static const struct idle_cpu idle_cpu_gmt __initconst = {
 	.state_table = gmt_cstates,
 };
@@ -1669,6 +1710,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = {
 	X86_MATCH_VFM(INTEL_ALDERLAKE,		&idle_cpu_adl),
 	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&idle_cpu_adl_l),
 	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&idle_cpu_mtl_l),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&idle_cpu_ptl),
 	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&idle_cpu_gmt),
 	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&idle_cpu_spr),
 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&idle_cpu_spr),
diff --git a/drivers/iio/accel/adxl313_core.c b/drivers/iio/accel/adxl313_core.c
index 9f5d4d2cb325b..084037c89ad30 100644
--- a/drivers/iio/accel/adxl313_core.c
+++ b/drivers/iio/accel/adxl313_core.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/overflow.h>
@@ -356,19 +357,15 @@ static int adxl313_read_axis(struct adxl313_data *data,
 {
 	int ret;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	ret = regmap_bulk_read(data->regmap,
 			       ADXL313_REG_DATA_AXIS(chan->address),
 			       &data->transf_buf, sizeof(data->transf_buf));
 	if (ret)
-		goto unlock_ret;
-
-	ret = le16_to_cpu(data->transf_buf);
+		return ret;
 
-unlock_ret:
-	mutex_unlock(&data->lock);
-	return ret;
+	return le16_to_cpu(data->transf_buf);
 }
 
 static int adxl313_read_freq_avail(struct iio_dev *indio_dev,
diff --git a/drivers/iio/accel/adxl345_core.c b/drivers/iio/accel/adxl345_core.c
index 78e3f799ecc1e..6c9080d88c60b 100644
--- a/drivers/iio/accel/adxl345_core.c
+++ b/drivers/iio/accel/adxl345_core.c
@@ -213,6 +213,7 @@ static const struct iio_event_spec adxl345_events[] = {
 		.dir = IIO_EV_DIR_RISING,
 		.mask_shared_by_type =
 			BIT(IIO_EV_INFO_ENABLE) |
+			BIT(IIO_EV_INFO_SCALE) |
 			BIT(IIO_EV_INFO_VALUE),
 	},
 	{
@@ -221,6 +222,7 @@ static const struct iio_event_spec adxl345_events[] = {
 		.dir = IIO_EV_DIR_RISING,
 		.mask_shared_by_type =
 			BIT(IIO_EV_INFO_ENABLE) |
+			BIT(IIO_EV_INFO_SCALE) |
 			BIT(IIO_EV_INFO_VALUE),
 	},
 	{
@@ -228,14 +230,19 @@ static const struct iio_event_spec adxl345_events[] = {
 		.type = IIO_EV_TYPE_GESTURE,
 		.dir = IIO_EV_DIR_SINGLETAP,
 		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
-		.mask_shared_by_type = BIT(IIO_EV_INFO_VALUE) |
+		.mask_shared_by_type =
+			BIT(IIO_EV_INFO_SCALE) |
+			BIT(IIO_EV_INFO_VALUE) |
 			BIT(IIO_EV_INFO_TIMEOUT),
 	},
 	{
 		/* double tap */
 		.type = IIO_EV_TYPE_GESTURE,
 		.dir = IIO_EV_DIR_DOUBLETAP,
-		.mask_shared_by_type = BIT(IIO_EV_INFO_ENABLE) |
+		.mask_shared_by_type =
+			BIT(IIO_EV_INFO_ENABLE) |
+			BIT(IIO_EV_INFO_SCALE) |
+			BIT(IIO_EV_INFO_VALUE) |
 			BIT(IIO_EV_INFO_RESET_TIMEOUT) |
 			BIT(IIO_EV_INFO_TAP2_MIN_DELAY),
 	},
@@ -274,6 +281,7 @@ static const struct iio_event_spec adxl345_fake_chan_events[] = {
 		.dir = IIO_EV_DIR_FALLING,
 		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
 		.mask_shared_by_type =
+			BIT(IIO_EV_INFO_SCALE) |
 			BIT(IIO_EV_INFO_VALUE) |
 			BIT(IIO_EV_INFO_PERIOD),
 	},
@@ -283,6 +291,7 @@ static const struct iio_event_spec adxl345_fake_chan_events[] = {
 		.dir = IIO_EV_DIR_FALLING,
 		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
 		.mask_shared_by_type =
+			BIT(IIO_EV_INFO_SCALE) |
 			BIT(IIO_EV_INFO_VALUE) |
 			BIT(IIO_EV_INFO_PERIOD),
 	},
@@ -1341,6 +1350,16 @@ static int adxl345_read_event_value(struct iio_dev *indio_dev,
 	unsigned int tap_threshold;
 	int ret;
 
+	/*
+	 * The event threshold LSB is fixed at 62.5 mg/LSB
+	 * 0.0625 * 9.80665 = 0.612915625 m/s^2
+	 */
+	if (info == IIO_EV_INFO_SCALE) {
+		*val = 0;
+		*val2 = 612915;
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
+
 	switch (type) {
 	case IIO_EV_TYPE_MAG:
 		return adxl345_read_mag_value(st, dir, info,
@@ -1355,12 +1374,6 @@ static int adxl345_read_event_value(struct iio_dev *indio_dev,
 	case IIO_EV_TYPE_GESTURE:
 		switch (info) {
 		case IIO_EV_INFO_VALUE:
-			/*
-			 * The scale factor would be 62.5mg/LSB (i.e. 0xFF = 16g) but
-			 * not applied here. In context of this general purpose sensor,
-			 * what imports is rather signal intensity than the absolute
-			 * measured g value.
-			 */
 			ret = regmap_read(st->regmap, ADXL345_REG_THRESH_TAP,
 					  &tap_threshold);
 			if (ret)
@@ -1401,6 +1414,9 @@ static int adxl345_write_event_value(struct iio_dev *indio_dev,
 	if (ret)
 		return ret;
 
+	if (info == IIO_EV_INFO_SCALE)
+		return -EINVAL;
+
 	switch (type) {
 	case IIO_EV_TYPE_MAG:
 		ret = adxl345_write_mag_value(st, dir, info,
diff --git a/drivers/iio/accel/adxl372.c b/drivers/iio/accel/adxl372.c
index 28a8793a53b6f..6763f8ed9f7f1 100644
--- a/drivers/iio/accel/adxl372.c
+++ b/drivers/iio/accel/adxl372.c
@@ -7,6 +7,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/module.h>
@@ -336,18 +337,14 @@ static ssize_t adxl372_write_threshold_value(struct iio_dev *indio_dev, unsigned
 	struct adxl372_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&st->threshold_m);
+	guard(mutex)(&st->threshold_m);
+
 	ret = regmap_write(st->regmap, addr, ADXL372_THRESH_VAL_H_SEL(threshold));
 	if (ret < 0)
-		goto unlock;
-
-	ret = regmap_update_bits(st->regmap, addr + 1, GENMASK(7, 5),
-				 ADXL372_THRESH_VAL_L_SEL(threshold) << 5);
-
-unlock:
-	mutex_unlock(&st->threshold_m);
+		return ret;
 
-	return ret;
+	return regmap_update_bits(st->regmap, addr + 1, GENMASK(7, 5),
+				  ADXL372_THRESH_VAL_L_SEL(threshold) << 5);
 }
 
 static int adxl372_read_axis(struct adxl372_state *st, u8 addr)
diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c
index 8fab2fdbe147e..a51d1d61c4127 100644
--- a/drivers/iio/accel/adxl380.c
+++ b/drivers/iio/accel/adxl380.c
@@ -877,7 +877,7 @@ static int adxl380_set_fifo_samples(struct adxl380_state *st)
 	ret = regmap_update_bits(st->regmap, ADXL380_FIFO_CONFIG_0_REG,
 				 ADXL380_FIFO_SAMPLES_8_MSK,
 				 FIELD_PREP(ADXL380_FIFO_SAMPLES_8_MSK,
-					    (fifo_samples & BIT(8))));
+					    !!(fifo_samples & BIT(8))));
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
index 4a827be439a23..573831199bbaa 100644
--- a/drivers/iio/accel/sca3000.c
+++ b/drivers/iio/accel/sca3000.c
@@ -7,6 +7,7 @@
  * See industrialio/accels/sca3000.h for comments.
  */
 
+#include <linux/cleanup.h>
 #include <linux/interrupt.h>
 #include <linux/fs.h>
 #include <linux/device.h>
@@ -171,6 +172,7 @@ struct sca3000_state {
 
 /**
  * struct sca3000_chip_info - model dependent parameters
+ * @name:			name of the chip
  * @scale:			scale * 10^-6
  * @temp_output:		some devices have temperature sensors.
  * @measurement_mode_freq:	normal mode sampling frequency
@@ -193,6 +195,7 @@ struct sca3000_state {
  * sca3000 variant.
  **/
 struct sca3000_chip_info {
+	const char		*name;
 	unsigned int		scale;
 	bool			temp_output;
 	int			measurement_mode_freq;
@@ -207,69 +210,59 @@ struct sca3000_chip_info {
 	int			mot_det_mult_y[7];
 };
 
-enum sca3000_variant {
-	d01,
-	e02,
-	e04,
-	e05,
+static const struct sca3000_chip_info sca3000_chip_info_d01 = {
+	.name = "sca3000_d01",
+	.scale = 7357,
+	.temp_output = true,
+	.measurement_mode_freq = 250,
+	.measurement_mode_3db_freq = 45,
+	.option_mode_1 = SCA3000_OP_MODE_BYPASS,
+	.option_mode_1_freq = 250,
+	.option_mode_1_3db_freq = 70,
+	.mot_det_mult_xz = { 50, 100, 200, 350, 650, 1300 },
+	.mot_det_mult_y = { 50, 100, 150, 250, 450, 850, 1750 },
 };
 
-/*
- * Note where option modes are not defined, the chip simply does not
- * support any.
- * Other chips in the sca3000 series use i2c and are not included here.
- *
- * Some of these devices are only listed in the family data sheet and
- * do not actually appear to be available.
- */
-static const struct sca3000_chip_info sca3000_spi_chip_info_tbl[] = {
-	[d01] = {
-		.scale = 7357,
-		.temp_output = true,
-		.measurement_mode_freq = 250,
-		.measurement_mode_3db_freq = 45,
-		.option_mode_1 = SCA3000_OP_MODE_BYPASS,
-		.option_mode_1_freq = 250,
-		.option_mode_1_3db_freq = 70,
-		.mot_det_mult_xz = {50, 100, 200, 350, 650, 1300},
-		.mot_det_mult_y = {50, 100, 150, 250, 450, 850, 1750},
-	},
-	[e02] = {
-		.scale = 9810,
-		.measurement_mode_freq = 125,
-		.measurement_mode_3db_freq = 40,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 63,
-		.option_mode_1_3db_freq = 11,
-		.mot_det_mult_xz = {100, 150, 300, 550, 1050, 2050},
-		.mot_det_mult_y = {50, 100, 200, 350, 700, 1350, 2700},
-	},
-	[e04] = {
-		.scale = 19620,
-		.measurement_mode_freq = 100,
-		.measurement_mode_3db_freq = 38,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 50,
-		.option_mode_1_3db_freq = 9,
-		.option_mode_2 = SCA3000_OP_MODE_WIDE,
-		.option_mode_2_freq = 400,
-		.option_mode_2_3db_freq = 70,
-		.mot_det_mult_xz = {200, 300, 600, 1100, 2100, 4100},
-		.mot_det_mult_y = {100, 200, 400, 7000, 1400, 2700, 54000},
-	},
-	[e05] = {
-		.scale = 61313,
-		.measurement_mode_freq = 200,
-		.measurement_mode_3db_freq = 60,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 50,
-		.option_mode_1_3db_freq = 9,
-		.option_mode_2 = SCA3000_OP_MODE_WIDE,
-		.option_mode_2_freq = 400,
-		.option_mode_2_3db_freq = 75,
-		.mot_det_mult_xz = {600, 900, 1700, 3200, 6100, 11900},
-		.mot_det_mult_y = {300, 600, 1200, 2000, 4100, 7800, 15600},
-	},
+static const struct sca3000_chip_info sca3000_chip_info_e02 = {
+	.name = "sca3000_e02",
+	.scale = 9810,
+	.measurement_mode_freq = 125,
+	.measurement_mode_3db_freq = 40,
+	.option_mode_1 = SCA3000_OP_MODE_NARROW,
+	.option_mode_1_freq = 63,
+	.option_mode_1_3db_freq = 11,
+	.mot_det_mult_xz = { 100, 150, 300, 550, 1050, 2050 },
+	.mot_det_mult_y = { 50, 100, 200, 350, 700, 1350, 2700 },
+};
+
+static const struct sca3000_chip_info sca3000_chip_info_e04 = {
+	.name = "sca3000_e04",
+	.scale = 19620,
+	.measurement_mode_freq = 100,
+	.measurement_mode_3db_freq = 38,
+	.option_mode_1 = SCA3000_OP_MODE_NARROW,
+	.option_mode_1_freq = 50,
+	.option_mode_1_3db_freq = 9,
+	.option_mode_2 = SCA3000_OP_MODE_WIDE,
+	.option_mode_2_freq = 400,
+	.option_mode_2_3db_freq = 70,
+	.mot_det_mult_xz = { 200, 300, 600, 1100, 2100, 4100 },
+	.mot_det_mult_y = { 100, 200, 400, 7000, 1400, 2700, 54000 },
+};
+
+static const struct sca3000_chip_info sca3000_chip_info_e05 = {
+	.name = "sca3000_e05",
+	.scale = 61313,
+	.measurement_mode_freq = 200,
+	.measurement_mode_3db_freq = 60,
+	.option_mode_1 = SCA3000_OP_MODE_NARROW,
+	.option_mode_1_freq = 50,
+	.option_mode_1_3db_freq = 9,
+	.option_mode_2 = SCA3000_OP_MODE_WIDE,
+	.option_mode_2_freq = 400,
+	.option_mode_2_3db_freq = 75,
+	.mot_det_mult_xz = { 600, 900, 1700, 3200, 6100, 11900 },
+	.mot_det_mult_y = { 300, 600, 1200, 2000, 4100, 7800, 15600 },
 };
 
 static int sca3000_write_reg(struct sca3000_state *st, u8 address, u8 val)
@@ -1435,24 +1428,42 @@ static const struct iio_info sca3000_info = {
 	.write_event_config = &sca3000_write_event_config,
 };
 
-static int sca3000_probe(struct spi_device *spi)
+static void sca3000_stop_all_interrupts(void *data)
 {
+	struct iio_dev *indio_dev = data;
+	struct sca3000_state *st = iio_priv(indio_dev);
 	int ret;
+
+	guard(mutex)(&st->lock);
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	if (ret)
+		return;
+
+	sca3000_write_reg(st, SCA3000_REG_INT_MASK_ADDR,
+			  (st->rx[0] &
+			   ~(SCA3000_REG_INT_MASK_RING_THREE_QUARTER |
+			     SCA3000_REG_INT_MASK_RING_HALF |
+			     SCA3000_REG_INT_MASK_ALL_INTS)));
+}
+
+static int sca3000_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
 	struct sca3000_state *st;
 	struct iio_dev *indio_dev;
+	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
 	st->us = spi;
 	mutex_init(&st->lock);
-	st->info = &sca3000_spi_chip_info_tbl[spi_get_device_id(spi)
-					      ->driver_data];
+	st->info = spi_get_device_match_data(spi);
 
-	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->name = st->info->name;
 	indio_dev->info = &sca3000_info;
 	if (st->info->temp_output) {
 		indio_dev->channels = sca3000_channels_with_temp;
@@ -1464,78 +1475,39 @@ static int sca3000_probe(struct spi_device *spi)
 	}
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = devm_iio_kfifo_buffer_setup(&spi->dev, indio_dev,
-					  &sca3000_ring_setup_ops);
+	ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, &sca3000_ring_setup_ops);
 	if (ret)
 		return ret;
 
 	if (spi->irq) {
-		ret = request_threaded_irq(spi->irq,
-					   NULL,
-					   &sca3000_event_handler,
-					   IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-					   "sca3000",
-					   indio_dev);
+		ret = devm_request_threaded_irq(dev, spi->irq, NULL,
+						&sca3000_event_handler,
+						IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+						"sca3000",
+						indio_dev);
 		if (ret)
 			return ret;
 	}
 	ret = sca3000_clean_setup(st);
 	if (ret)
-		goto error_free_irq;
+		return ret;
 
 	ret = sca3000_print_rev(indio_dev);
 	if (ret)
-		goto error_free_irq;
-
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_free_irq;
-
-	return 0;
-
-error_free_irq:
-	if (spi->irq)
-		free_irq(spi->irq, indio_dev);
-
-	return ret;
-}
-
-static int sca3000_stop_all_interrupts(struct sca3000_state *st)
-{
-	int ret;
+		return ret;
 
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	ret = devm_add_action_or_reset(dev, sca3000_stop_all_interrupts, indio_dev);
 	if (ret)
-		goto error_ret;
-	ret = sca3000_write_reg(st, SCA3000_REG_INT_MASK_ADDR,
-				(st->rx[0] &
-				 ~(SCA3000_REG_INT_MASK_RING_THREE_QUARTER |
-				   SCA3000_REG_INT_MASK_RING_HALF |
-				   SCA3000_REG_INT_MASK_ALL_INTS)));
-error_ret:
-	mutex_unlock(&st->lock);
-	return ret;
-}
-
-static void sca3000_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct sca3000_state *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
+		return ret;
 
-	/* Must ensure no interrupts can be generated after this! */
-	sca3000_stop_all_interrupts(st);
-	if (spi->irq)
-		free_irq(spi->irq, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id sca3000_id[] = {
-	{"sca3000_d01", d01},
-	{"sca3000_e02", e02},
-	{"sca3000_e04", e04},
-	{"sca3000_e05", e05},
+	{ "sca3000_d01", (kernel_ulong_t)&sca3000_chip_info_d01 },
+	{ "sca3000_e02", (kernel_ulong_t)&sca3000_chip_info_e02 },
+	{ "sca3000_e04", (kernel_ulong_t)&sca3000_chip_info_e04 },
+	{ "sca3000_e05", (kernel_ulong_t)&sca3000_chip_info_e05 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, sca3000_id);
@@ -1545,7 +1517,6 @@ static struct spi_driver sca3000_driver = {
 		.name = "sca3000",
 	},
 	.probe = sca3000_probe,
-	.remove = sca3000_remove,
 	.id_table = sca3000_id,
 };
 module_spi_driver(sca3000_driver);
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 60038ae8dfc4d..a9dedbb8eb461 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -60,9 +60,14 @@ config AD4030
 	tristate "Analog Devices AD4030 ADC Driver"
 	depends on SPI
 	depends on GPIOLIB
+	depends on PWM
 	select REGMAP
 	select IIO_BUFFER
+	select IIO_BUFFER_DMA
+	select IIO_BUFFER_DMAENGINE
 	select IIO_TRIGGERED_BUFFER
+	select SPI_OFFLOAD
+	select SPI_OFFLOAD_TRIGGER_PWM
 	help
 	  Say yes here to build support for Analog Devices AD4030 and AD4630 high speed
 	  SPI analog to digital converters (ADC).
@@ -413,8 +418,10 @@ config AD7768_1
 	select REGMAP_SPI
 	select RATIONAL
 	select IIO_BUFFER
+	select IIO_BUFFER_DMAENGINE
 	select IIO_TRIGGER
 	select IIO_TRIGGERED_BUFFER
+	select SPI_OFFLOAD
 	help
 	  Say yes here to build support for Analog Devices AD7768-1 SPI
 	  simultaneously sampling sigma-delta analog to digital converter (ADC).
@@ -1366,6 +1373,32 @@ config QCOM_SPMI_ADC5
 	  To compile this driver as a module, choose M here: the module will
 	  be called qcom-spmi-adc5.
 
+config QCOM_SPMI_ADC5_GEN3
+	tristate "Qualcomm Technologies Inc. SPMI PMIC5 GEN3 ADC"
+	depends on SPMI && THERMAL
+	select REGMAP_SPMI
+	select QCOM_VADC_COMMON
+	select AUXILIARY_BUS
+	help
+	  IIO Voltage PMIC5 Gen3 ADC driver for Qualcomm Technologies Inc.
+
+	  The driver supports reading multiple channels. The ADC is a 16-bit
+	  sigma-delta ADC. The hardware supports calibrated results for
+	  conversion requests and clients include reading phone power supply
+	  voltage, on board system thermistors connected to the PMIC ADC,
+	  PMIC die temperature, charger temperature, battery current, USB
+	  voltage input and voltage signals connected to supported PMIC GPIO
+	  pins. The hardware supports internal pull-up for thermistors and can
+	  choose between a 30k, 100k or 400k ohm pull up using the ADC channels.
+
+	  In addition, the same driver supports ADC thermal monitoring devices
+	  too. They appear as thermal zones with multiple trip points. A thermal
+	  client sets threshold temperature for both warm and cool trips and
+	  gets updated when a threshold is reached.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called qcom-spmi-adc5-gen3.
+
 config RCAR_GYRO_ADC
 	tristate "Renesas R-Car GyroADC driver"
 	depends on ARCH_RCAR_GEN2 || COMPILE_TEST
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index c76550415ff17..097357d146baf 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -116,6 +116,7 @@ obj-$(CONFIG_PAC1934) += pac1934.o
 obj-$(CONFIG_PALMAS_GPADC) += palmas_gpadc.o
 obj-$(CONFIG_QCOM_PM8XXX_XOADC) += qcom-pm8xxx-xoadc.o
 obj-$(CONFIG_QCOM_SPMI_ADC5) += qcom-spmi-adc5.o
+obj-$(CONFIG_QCOM_SPMI_ADC5_GEN3) += qcom-spmi-adc5-gen3.o
 obj-$(CONFIG_QCOM_SPMI_IADC) += qcom-spmi-iadc.o
 obj-$(CONFIG_QCOM_SPMI_RRADC) += qcom-spmi-rradc.o
 obj-$(CONFIG_QCOM_SPMI_VADC) += qcom-spmi-vadc.o
diff --git a/drivers/iio/adc/ad4030.c b/drivers/iio/adc/ad4030.c
index 68446db9bef1b..9c5f19321e3b8 100644
--- a/drivers/iio/adc/ad4030.c
+++ b/drivers/iio/adc/ad4030.c
@@ -14,15 +14,26 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
+#include <linux/dmaengine.h>
+#include <linux/limits.h>
+#include <linux/log2.h>
+#include <linux/math64.h>
+#include <linux/minmax.h>
+#include <linux/pwm.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/spi/offload/consumer.h>
 #include <linux/spi/spi.h>
 #include <linux/unaligned.h>
 #include <linux/units.h>
+#include <linux/types.h>
+
+#include <linux/iio/buffer-dmaengine.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #define AD4030_REG_INTERFACE_CONFIG_A			0x00
 #define     AD4030_REG_INTERFACE_CONFIG_A_SW_RESET	(BIT(0) | BIT(7))
@@ -37,6 +48,8 @@
 #define     AD4030_REG_CHIP_GRADE_AD4630_24_GRADE	0x00
 #define     AD4030_REG_CHIP_GRADE_AD4632_16_GRADE	0x05
 #define     AD4030_REG_CHIP_GRADE_AD4632_24_GRADE	0x02
+#define     AD4030_REG_CHIP_GRADE_ADAQ4216_GRADE	0x1E
+#define     AD4030_REG_CHIP_GRADE_ADAQ4224_GRADE	0x1C
 #define     AD4030_REG_CHIP_GRADE_MASK_CHIP_GRADE	GENMASK(7, 3)
 #define AD4030_REG_SCRATCH_PAD			0x0A
 #define AD4030_REG_SPI_REVISION			0x0B
@@ -111,6 +124,12 @@
 #define AD4632_TCYC_NS			2000
 #define AD4632_TCYC_ADJUSTED_NS		(AD4632_TCYC_NS - AD4030_TCNVL_NS)
 #define AD4030_TRESET_COM_DELAY_MS	750
+/* Datasheet says 9.8ns, so use the closest integer value */
+#define AD4030_TQUIET_CNV_DELAY_NS	10
+
+/* HARDWARE_GAIN */
+#define ADAQ4616_PGA_PINS		2
+#define ADAQ4616_PGA_GAIN_MAX_NANO	(NANO * 2 / 3)
 
 enum ad4030_out_mode {
 	AD4030_OUT_DATA_MD_DIFF,
@@ -132,15 +151,35 @@ enum {
 	AD4030_SCAN_TYPE_AVG,
 };
 
+/*
+ * Gains computed as fractions of 1000 so they can be expressed by integers.
+ */
+static const int adaq4216_hw_gains_vpv[] = {
+	1 * MILLI / 3,		/* 0.333 */
+	5 * MILLI / 9,		/* 0.555 */
+	20 * MILLI / 9,		/* 0.2222 */
+	20 * MILLI / 3,		/* 0.6666 */
+};
+
+static const int adaq4216_hw_gains_frac[][2] = {
+	{ 1, 3 },  /* 1/3 V/V gain */
+	{ 5, 9 },  /* 5/9 V/V gain */
+	{ 20, 9 }, /* 20/9 V/V gain */
+	{ 20, 3 }, /* 20/3 V/V gain */
+};
+
 struct ad4030_chip_info {
 	const char *name;
 	const unsigned long *available_masks;
 	const struct iio_chan_spec channels[AD4030_MAX_IIO_CHANNEL_NB];
+	const struct iio_chan_spec offload_channels[AD4030_MAX_IIO_CHANNEL_NB];
 	u8 grade;
 	u8 precision_bits;
+	bool has_pga;
 	/* Number of hardware channels */
 	int num_voltage_inputs;
 	unsigned int tcyc_ns;
+	unsigned int max_sample_rate_hz;
 };
 
 struct ad4030_state {
@@ -153,6 +192,18 @@ struct ad4030_state {
 	int offset_avail[3];
 	unsigned int avg_log2;
 	enum ad4030_out_mode mode;
+	/* Offload sampling */
+	struct spi_transfer offload_xfer;
+	struct spi_message offload_msg;
+	struct spi_offload *offload;
+	struct spi_offload_trigger *offload_trigger;
+	struct spi_offload_trigger_config offload_trigger_config;
+	struct pwm_device *cnv_trigger;
+	size_t scale_avail_size;
+	struct pwm_waveform cnv_wf;
+	unsigned int scale_avail[ARRAY_SIZE(adaq4216_hw_gains_vpv)][2];
+	struct gpio_descs *pga_gpios;
+	unsigned int pga_index;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the transfer buffers
@@ -209,8 +260,9 @@ struct ad4030_state {
  * - voltage0-voltage1
  * - voltage2-voltage3
  */
-#define AD4030_CHAN_DIFF(_idx, _scan_type) {				\
+#define __AD4030_CHAN_DIFF(_idx, _scan_type, _offload, _pga) {		\
 	.info_mask_shared_by_all =					\
+		(_offload ? BIT(IIO_CHAN_INFO_SAMP_FREQ) : 0) |		\
 		BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),			\
 	.info_mask_shared_by_all_available =				\
 		BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),			\
@@ -219,6 +271,7 @@ struct ad4030_state {
 		BIT(IIO_CHAN_INFO_CALIBBIAS) |				\
 		BIT(IIO_CHAN_INFO_RAW),					\
 	.info_mask_separate_available = BIT(IIO_CHAN_INFO_CALIBBIAS) |	\
+		(_pga ? BIT(IIO_CHAN_INFO_SCALE) : 0) |			\
 		BIT(IIO_CHAN_INFO_CALIBSCALE),				\
 	.type = IIO_VOLTAGE,						\
 	.indexed = 1,							\
@@ -232,10 +285,33 @@ struct ad4030_state {
 	.num_ext_scan_type = ARRAY_SIZE(_scan_type),			\
 }
 
+#define AD4030_CHAN_DIFF(_idx, _scan_type)				\
+	__AD4030_CHAN_DIFF(_idx, _scan_type, 0, 0)
+
+#define AD4030_OFFLOAD_CHAN_DIFF(_idx, _scan_type)			\
+	__AD4030_CHAN_DIFF(_idx, _scan_type, 1, 0)
+
+#define ADAQ4216_CHAN_DIFF(_idx, _scan_type)				\
+	__AD4030_CHAN_DIFF(_idx, _scan_type, 0, 1)
+
+#define ADAQ4216_OFFLOAD_CHAN_DIFF(_idx, _scan_type)			\
+	__AD4030_CHAN_DIFF(_idx, _scan_type, 1, 1)
+
+/*
+ * AD4030 can average over 2^N samples, where N = 1, 2, 3, ..., 16.
+ * We use N = 0 to mean no sample averaging.
+ */
 static const int ad4030_average_modes[] = {
-	1, 2, 4, 8, 16, 32, 64, 128,
-	256, 512, 1024, 2048, 4096, 8192, 16384, 32768,
-	65536,
+	BIT(0),					/* No sampling average */
+	BIT(1), BIT(2), BIT(3), BIT(4),
+	BIT(5), BIT(6), BIT(7), BIT(8),
+	BIT(9), BIT(10), BIT(11), BIT(12),
+	BIT(13), BIT(14), BIT(15), BIT(16),
+};
+
+static const struct spi_offload_config ad4030_offload_config = {
+	.capability_flags = SPI_OFFLOAD_CAP_TRIGGER |
+			    SPI_OFFLOAD_CAP_RX_STREAM_DMA,
 };
 
 static int ad4030_enter_config_mode(struct ad4030_state *st)
@@ -377,6 +453,65 @@ static const struct regmap_config ad4030_regmap_config = {
 	.max_register = AD4030_REG_DIG_ERR,
 };
 
+static void ad4030_fill_scale_avail(struct ad4030_state *st)
+{
+	unsigned int mag_bits, int_part, fract_part;
+	u64 range;
+
+	/*
+	 * The maximum precision of differential channels is retrieved from the
+	 * chip properties. The output code of differential channels is in two's
+	 * complement format (i.e. signed), so the MSB is the sign bit and only
+	 * (precision_bits - 1) bits express voltage magnitude.
+	 */
+	mag_bits = st->chip->precision_bits - 1;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(adaq4216_hw_gains_frac); i++) {
+		range = mult_frac(st->vref_uv, adaq4216_hw_gains_frac[i][1],
+				  adaq4216_hw_gains_frac[i][0]);
+		/*
+		 * If range were in mV, we would multiply it by NANO below.
+		 * Though, range is in µV so multiply it by MICRO only so the
+		 * result after right shift and division scales output codes to
+		 * millivolts.
+		 */
+		int_part = div_u64_rem((range * MICRO) >> mag_bits, NANO, &fract_part);
+		st->scale_avail[i][0] = int_part;
+		st->scale_avail[i][1] = fract_part;
+	}
+}
+
+static int ad4030_set_pga_gain(struct ad4030_state *st)
+{
+	DECLARE_BITMAP(bitmap, ADAQ4616_PGA_PINS) = { };
+
+	bitmap_write(bitmap, st->pga_index, 0, ADAQ4616_PGA_PINS);
+
+	return gpiod_multi_set_value_cansleep(st->pga_gpios, bitmap);
+}
+
+static int ad4030_set_pga(struct iio_dev *indio_dev, int gain_int, int gain_fract)
+{
+	struct ad4030_state *st = iio_priv(indio_dev);
+	unsigned int mag_bits = st->chip->precision_bits - 1;
+	unsigned int tmp;
+	u64 gain_nano;
+
+	if (!st->pga_gpios)
+		return -EINVAL;
+
+	gain_nano = gain_int * NANO + gain_fract;
+	if (!in_range(gain_nano, 1, ADAQ4616_PGA_GAIN_MAX_NANO))
+		return -EINVAL;
+
+	tmp = DIV_ROUND_CLOSEST_ULL(gain_nano << mag_bits, NANO);
+	gain_nano = DIV_ROUND_CLOSEST(st->vref_uv, tmp);
+	st->pga_index = find_closest(gain_nano, adaq4216_hw_gains_vpv,
+				     ARRAY_SIZE(adaq4216_hw_gains_vpv));
+
+	return ad4030_set_pga_gain(st);
+}
+
 static int ad4030_get_chan_scale(struct iio_dev *indio_dev,
 				 struct iio_chan_spec const *chan,
 				 int *val,
@@ -389,6 +524,13 @@ static int ad4030_get_chan_scale(struct iio_dev *indio_dev,
 	if (IS_ERR(scan_type))
 		return PTR_ERR(scan_type);
 
+	/* The LSB of the 8-bit common-mode data is always vref/256. */
+	if (st->chip->has_pga && scan_type->realbits != 8) {
+		*val = st->scale_avail[st->pga_index][0];
+		*val2 = st->scale_avail[st->pga_index][1];
+		return IIO_VAL_INT_PLUS_NANO;
+	}
+
 	if (chan->differential)
 		*val = (st->vref_uv * 2) / MILLI;
 	else
@@ -451,6 +593,102 @@ static int ad4030_get_chan_calibbias(struct iio_dev *indio_dev,
 	}
 }
 
+static void ad4030_get_sampling_freq(struct ad4030_state *st, int *freq)
+{
+	struct spi_offload_trigger_config *config = &st->offload_trigger_config;
+
+	/*
+	 * Conversion data is fetched from the device when the offload transfer
+	 * is triggered. Thus, provide the SPI offload trigger frequency as the
+	 * sampling frequency.
+	 */
+	*freq = config->periodic.frequency_hz;
+}
+
+static int ad4030_update_conversion_rate(struct ad4030_state *st,
+					 unsigned int freq_hz, unsigned int avg_log2)
+{
+	struct spi_offload_trigger_config *config = &st->offload_trigger_config;
+	unsigned int offload_period_ns, cnv_rate_hz;
+	struct pwm_waveform cnv_wf = { };
+	u64 target = AD4030_TCNVH_NS;
+	u64 offload_offset_ns;
+	int ret;
+
+	/*
+	 * When averaging/oversampling over N samples, we fire the offload
+	 * trigger once at every N pulses of the CNV signal. Conversely, the CNV
+	 * signal needs to be N times faster than the offload trigger. Take that
+	 * into account to correctly re-evaluate both the PWM waveform connected
+	 * to CNV and the SPI offload trigger.
+	 */
+	cnv_rate_hz = freq_hz << avg_log2;
+
+	cnv_wf.period_length_ns = DIV_ROUND_CLOSEST(NSEC_PER_SEC, cnv_rate_hz);
+	/*
+	 * The datasheet lists a minimum time of 9.8 ns, but no maximum. If the
+	 * rounded PWM's value is less than 10, increase the target value by 10
+	 * and attempt to round the waveform again, until the value is at least
+	 * 10 ns. Use a separate variable to represent the target in case the
+	 * rounding is severe enough to keep putting the first few results under
+	 * the minimum 10ns condition checked by the while loop.
+	 */
+	do {
+		cnv_wf.duty_length_ns = target;
+		ret = pwm_round_waveform_might_sleep(st->cnv_trigger, &cnv_wf);
+		if (ret)
+			return ret;
+		target += AD4030_TCNVH_NS;
+	} while (cnv_wf.duty_length_ns < AD4030_TCNVH_NS);
+
+	/*
+	 * The CNV waveform period (period_length_ns) might get rounded down by
+	 * pwm_round_waveform_might_sleep(). Check the resultant PWM period
+	 * is not smaller than the minimum data conversion cycle time.
+	 */
+	if (!in_range(cnv_wf.period_length_ns, AD4030_TCYC_NS, INT_MAX))
+		return -EINVAL;
+
+	offload_period_ns = DIV_ROUND_CLOSEST(NSEC_PER_SEC, freq_hz);
+
+	config->periodic.frequency_hz = DIV_ROUND_UP(HZ_PER_GHZ, offload_period_ns);
+
+	/*
+	 * The hardware does the capture on zone 2 (when SPI trigger PWM
+	 * is used). This means that the SPI trigger signal should happen at
+	 * tsync + tquiet_con_delay being tsync the conversion signal period
+	 * and tquiet_con_delay 9.8ns. Hence set the PWM phase accordingly.
+	 *
+	 * The PWM waveform API only supports nanosecond resolution right now,
+	 * so round this setting to the closest available value.
+	 */
+	offload_offset_ns = AD4030_TQUIET_CNV_DELAY_NS;
+	do {
+		config->periodic.offset_ns = offload_offset_ns;
+		ret = spi_offload_trigger_validate(st->offload_trigger, config);
+		if (ret)
+			return ret;
+		offload_offset_ns += AD4030_TQUIET_CNV_DELAY_NS;
+	} while (config->periodic.offset_ns < AD4030_TQUIET_CNV_DELAY_NS);
+
+	st->cnv_wf = cnv_wf;
+
+	return 0;
+}
+
+static int ad4030_set_sampling_freq(struct iio_dev *indio_dev, int freq_hz)
+{
+	struct ad4030_state *st = iio_priv(indio_dev);
+
+	if (freq_hz == 0)
+		return -EINVAL;
+
+	if (!in_range(freq_hz, 0, st->chip->max_sample_rate_hz))
+		return -ERANGE;
+
+	return ad4030_update_conversion_rate(st, freq_hz, st->avg_log2);
+}
+
 static int ad4030_set_chan_calibscale(struct iio_dev *indio_dev,
 				      struct iio_chan_spec const *chan,
 				      int gain_int,
@@ -510,11 +748,30 @@ static int ad4030_set_avg_frame_len(struct iio_dev *dev, int avg_val)
 	struct ad4030_state *st = iio_priv(dev);
 	unsigned int avg_log2 = ilog2(avg_val);
 	unsigned int last_avg_idx = ARRAY_SIZE(ad4030_average_modes) - 1;
+	int freq_hz;
 	int ret;
 
 	if (avg_val < 0 || avg_val > ad4030_average_modes[last_avg_idx])
 		return -EINVAL;
 
+	if (st->offload_trigger) {
+		/*
+		 * The sample averaging and sampling frequency configurations
+		 * are mutually dependent on each other. That's because the
+		 * effective data sample rate is fCNV / 2^N, where N is the
+		 * number of samples being averaged.
+		 *
+		 * When SPI offload is supported and we have control over the
+		 * sample rate, the conversion start signal (CNV) and the SPI
+		 * offload trigger frequencies must be re-evaluated so data is
+		 * fetched only after 'avg_val' conversions.
+		 */
+		ad4030_get_sampling_freq(st, &freq_hz);
+		ret = ad4030_update_conversion_rate(st, freq_hz, avg_log2);
+		if (ret)
+			return ret;
+	}
+
 	ret = regmap_write(st->regmap, AD4030_REG_AVG,
 			   AD4030_REG_AVG_MASK_AVG_SYNC |
 			   FIELD_PREP(AD4030_REG_AVG_MASK_AVG_VAL, avg_log2));
@@ -623,7 +880,7 @@ static int ad4030_conversion(struct iio_dev *indio_dev)
 	/* Add one byte if we are using a differential + common byte mode */
 	bytes_to_read += (st->mode == AD4030_OUT_DATA_MD_24_DIFF_8_COM ||
 			st->mode == AD4030_OUT_DATA_MD_16_DIFF_8_COM) ? 1 : 0;
-	/* Mulitiply by the number of hardware channels */
+	/* Multiply by the number of hardware channels */
 	bytes_to_read *= st->chip->num_voltage_inputs;
 
 	for (i = 0; i < cnv_nb; i++) {
@@ -742,6 +999,15 @@ static int ad4030_read_avail(struct iio_dev *indio_dev,
 		*length = ARRAY_SIZE(ad4030_average_modes);
 		return IIO_AVAIL_LIST;
 
+	case IIO_CHAN_INFO_SCALE:
+		if (st->scale_avail_size == 1)
+			*vals = (int *)st->scale_avail[st->pga_index];
+		else
+			*vals = (int *)st->scale_avail;
+		*length = st->scale_avail_size * 2; /* print int and nano part */
+		*type = IIO_VAL_INT_PLUS_NANO;
+		return IIO_AVAIL_LIST;
+
 	default:
 		return -EINVAL;
 	}
@@ -767,6 +1033,10 @@ static int ad4030_read_raw_dispatch(struct iio_dev *indio_dev,
 		*val = BIT(st->avg_log2);
 		return IIO_VAL_INT;
 
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ad4030_get_sampling_freq(st, val);
+		return IIO_VAL_INT;
+
 	default:
 		return -EINVAL;
 	}
@@ -807,6 +1077,12 @@ static int ad4030_write_raw_dispatch(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
 		return ad4030_set_avg_frame_len(indio_dev, val);
 
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return ad4030_set_sampling_freq(indio_dev, val);
+
+	case IIO_CHAN_INFO_SCALE:
+		return ad4030_set_pga(indio_dev, val, val2);
+
 	default:
 		return -EINVAL;
 	}
@@ -828,6 +1104,17 @@ static int ad4030_write_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
+static int ad4030_write_raw_get_fmt(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan, long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
+}
+
 static int ad4030_reg_access(struct iio_dev *indio_dev, unsigned int reg,
 			     unsigned int writeval, unsigned int *readval)
 {
@@ -874,6 +1161,7 @@ static const struct iio_info ad4030_iio_info = {
 	.read_avail = ad4030_read_avail,
 	.read_raw = ad4030_read_raw,
 	.write_raw = ad4030_write_raw,
+	.write_raw_get_fmt = &ad4030_write_raw_get_fmt,
 	.debugfs_reg_access = ad4030_reg_access,
 	.read_label = ad4030_read_label,
 	.get_current_scan_type = ad4030_get_current_scan_type,
@@ -896,6 +1184,86 @@ static const struct iio_buffer_setup_ops ad4030_buffer_setup_ops = {
 	.validate_scan_mask = ad4030_validate_scan_mask,
 };
 
+static void ad4030_prepare_offload_msg(struct iio_dev *indio_dev)
+{
+	struct ad4030_state *st = iio_priv(indio_dev);
+	u8 offload_bpw;
+
+	if (st->mode == AD4030_OUT_DATA_MD_30_AVERAGED_DIFF)
+		offload_bpw = 32;
+	else
+		offload_bpw = st->chip->precision_bits;
+
+	st->offload_xfer.bits_per_word = offload_bpw;
+	st->offload_xfer.len = spi_bpw_to_bytes(offload_bpw);
+	st->offload_xfer.offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
+	spi_message_init_with_transfers(&st->offload_msg, &st->offload_xfer, 1);
+}
+
+static int ad4030_offload_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct ad4030_state *st = iio_priv(indio_dev);
+	unsigned int reg_modes;
+	int ret;
+
+	/*
+	 * When data from 2 analog input channels is output through a single
+	 * bus line (interleaved mode (LANE_MD == 0b11)) and gets pushed through
+	 * DMA, extra hardware is required to do the de-interleaving. While we
+	 * don't support such hardware configurations, disallow interleaved mode
+	 * when using SPI offload.
+	 */
+	ret = regmap_read(st->regmap, AD4030_REG_MODES, &reg_modes);
+	if (ret)
+		return ret;
+
+	if (st->chip->num_voltage_inputs > 1 &&
+	    FIELD_GET(AD4030_REG_MODES_MASK_LANE_MODE, reg_modes) == AD4030_LANE_MD_INTERLEAVED)
+		return -EINVAL;
+
+	ad4030_prepare_offload_msg(indio_dev);
+	st->offload_msg.offload = st->offload;
+	ret = spi_optimize_message(st->spi, &st->offload_msg);
+	if (ret)
+		return ret;
+
+	ret = pwm_set_waveform_might_sleep(st->cnv_trigger, &st->cnv_wf, false);
+	if (ret)
+		goto out_unoptimize;
+
+	ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
+					 &st->offload_trigger_config);
+	if (ret)
+		goto out_pwm_disable;
+
+	return 0;
+
+out_pwm_disable:
+	pwm_disable(st->cnv_trigger);
+out_unoptimize:
+	spi_unoptimize_message(&st->offload_msg);
+
+	return ret;
+}
+
+static int ad4030_offload_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct ad4030_state *st = iio_priv(indio_dev);
+
+	spi_offload_trigger_disable(st->offload, st->offload_trigger);
+
+	pwm_disable(st->cnv_trigger);
+
+	spi_unoptimize_message(&st->offload_msg);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops ad4030_offload_buffer_setup_ops = {
+	.postenable = &ad4030_offload_buffer_postenable,
+	.predisable = &ad4030_offload_buffer_predisable,
+};
+
 static int ad4030_regulators_get(struct ad4030_state *st)
 {
 	struct device *dev = &st->spi->dev;
@@ -965,6 +1333,24 @@ static int ad4030_detect_chip_info(const struct ad4030_state *st)
 	return 0;
 }
 
+static int ad4030_pwm_get(struct ad4030_state *st)
+{
+	struct device *dev = &st->spi->dev;
+
+	st->cnv_trigger = devm_pwm_get(dev, NULL);
+	if (IS_ERR(st->cnv_trigger))
+		return dev_err_probe(dev, PTR_ERR(st->cnv_trigger),
+				     "Failed to get CNV PWM\n");
+
+	/*
+	 * Preemptively disable the PWM, since we only want to enable it with
+	 * the buffer.
+	 */
+	pwm_disable(st->cnv_trigger);
+
+	return 0;
+}
+
 static int ad4030_config(struct ad4030_state *st)
 {
 	int ret;
@@ -992,6 +1378,51 @@ static int ad4030_config(struct ad4030_state *st)
 	return 0;
 }
 
+static int ad4030_spi_offload_setup(struct iio_dev *indio_dev,
+				    struct ad4030_state *st)
+{
+	struct device *dev = &st->spi->dev;
+	struct dma_chan *rx_dma;
+
+	indio_dev->setup_ops = &ad4030_offload_buffer_setup_ops;
+
+	st->offload_trigger = devm_spi_offload_trigger_get(dev, st->offload,
+							   SPI_OFFLOAD_TRIGGER_PERIODIC);
+	if (IS_ERR(st->offload_trigger))
+		return dev_err_probe(dev, PTR_ERR(st->offload_trigger),
+				     "failed to get offload trigger\n");
+
+	st->offload_trigger_config.type = SPI_OFFLOAD_TRIGGER_PERIODIC;
+
+	rx_dma = devm_spi_offload_rx_stream_request_dma_chan(dev, st->offload);
+	if (IS_ERR(rx_dma))
+		return dev_err_probe(dev, PTR_ERR(rx_dma),
+				     "failed to get offload RX DMA\n");
+
+	return devm_iio_dmaengine_buffer_setup_with_handle(dev, indio_dev, rx_dma,
+							   IIO_BUFFER_DIRECTION_IN);
+}
+
+static int ad4030_setup_pga(struct device *dev, struct iio_dev *indio_dev,
+			    struct ad4030_state *st)
+{
+	/* Setup GPIOs for PGA control */
+	st->pga_gpios = devm_gpiod_get_array(dev, "pga", GPIOD_OUT_LOW);
+	if (IS_ERR(st->pga_gpios))
+		return dev_err_probe(dev, PTR_ERR(st->pga_gpios),
+				     "Failed to get PGA gpios.\n");
+
+	if (st->pga_gpios->ndescs != ADAQ4616_PGA_PINS)
+		return dev_err_probe(dev, -EINVAL,
+				     "Expected %d GPIOs for PGA control.\n",
+				     ADAQ4616_PGA_PINS);
+
+	st->scale_avail_size = ARRAY_SIZE(adaq4216_hw_gains_vpv);
+	st->pga_index = 0;
+
+	return 0;
+}
+
 static int ad4030_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
@@ -1034,6 +1465,14 @@ static int ad4030_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
+	if (st->chip->has_pga) {
+		ret = ad4030_setup_pga(dev, indio_dev, st);
+		if (ret)
+			return ret;
+
+		ad4030_fill_scale_avail(st);
+	}
+
 	ret = ad4030_config(st);
 	if (ret)
 		return ret;
@@ -1043,24 +1482,58 @@ static int ad4030_probe(struct spi_device *spi)
 		return dev_err_probe(dev, PTR_ERR(st->cnv_gpio),
 				     "Failed to get cnv gpio\n");
 
-	/*
-	 * One hardware channel is split in two software channels when using
-	 * common byte mode. Add one more channel for the timestamp.
-	 */
-	indio_dev->num_channels = 2 * st->chip->num_voltage_inputs + 1;
 	indio_dev->name = st->chip->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = &ad4030_iio_info;
-	indio_dev->channels = st->chip->channels;
 	indio_dev->available_scan_masks = st->chip->available_masks;
 
-	ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
-					      iio_pollfunc_store_time,
-					      ad4030_trigger_handler,
-					      &ad4030_buffer_setup_ops);
-	if (ret)
-		return dev_err_probe(dev, ret,
-				     "Failed to setup triggered buffer\n");
+	st->offload = devm_spi_offload_get(dev, spi, &ad4030_offload_config);
+	ret = PTR_ERR_OR_ZERO(st->offload);
+	/* Fall back to low speed usage when no SPI offload is available. */
+	if (ret == -ENODEV) {
+		/*
+		 * One hardware channel is split in two software channels when
+		 * using common byte mode. Add one more channel for the timestamp.
+		 */
+		indio_dev->num_channels = 2 * st->chip->num_voltage_inputs + 1;
+		indio_dev->channels = st->chip->channels;
+
+		ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
+						      iio_pollfunc_store_time,
+						      ad4030_trigger_handler,
+						      &ad4030_buffer_setup_ops);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to setup triggered buffer\n");
+	} else if (ret) {
+		return dev_err_probe(dev, ret, "failed to get offload\n");
+	} else {
+		/*
+		 * Offloaded SPI transfers can't support software timestamp so
+		 * no additional timestamp channel is added.
+		 */
+		indio_dev->num_channels = st->chip->num_voltage_inputs;
+		indio_dev->channels = st->chip->offload_channels;
+		ret = ad4030_spi_offload_setup(indio_dev, st);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to setup SPI offload\n");
+
+		ret = ad4030_pwm_get(st);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to get PWM\n");
+
+		/*
+		 * Start with a slower sampling rate so there is some room for
+		 * adjusting the sample averaging and the sampling frequency
+		 * without hitting the maximum conversion rate.
+		 */
+		ret = ad4030_update_conversion_rate(st, st->chip->max_sample_rate_hz >> 4,
+						    st->avg_log2);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to set offload samp freq\n");
+	}
 
 	return devm_iio_device_register(dev, indio_dev);
 }
@@ -1098,11 +1571,28 @@ static const struct iio_scan_type ad4030_24_scan_types[] = {
 	},
 };
 
-static const struct iio_scan_type ad4030_16_scan_types[] = {
+static const struct iio_scan_type ad4030_24_offload_scan_types[] = {
 	[AD4030_SCAN_TYPE_NORMAL] = {
 		.sign = 's',
+		.realbits = 24,
 		.storagebits = 32,
+		.shift = 0,
+		.endianness = IIO_CPU,
+	},
+	[AD4030_SCAN_TYPE_AVG] = {
+		.sign = 's',
+		.realbits = 30,
+		.storagebits = 32,
+		.shift = 2,
+		.endianness = IIO_CPU,
+	},
+};
+
+static const struct iio_scan_type ad4030_16_scan_types[] = {
+	[AD4030_SCAN_TYPE_NORMAL] = {
+		.sign = 's',
 		.realbits = 16,
+		.storagebits = 32,
 		.shift = 16,
 		.endianness = IIO_BE,
 	},
@@ -1115,6 +1605,23 @@ static const struct iio_scan_type ad4030_16_scan_types[] = {
 	}
 };
 
+static const struct iio_scan_type ad4030_16_offload_scan_types[] = {
+	[AD4030_SCAN_TYPE_NORMAL] = {
+		.sign = 's',
+		.realbits = 16,
+		.storagebits = 32,
+		.shift = 0,
+		.endianness = IIO_CPU,
+	},
+	[AD4030_SCAN_TYPE_AVG] = {
+		.sign = 's',
+		.realbits = 30,
+		.storagebits = 32,
+		.shift = 2,
+		.endianness = IIO_CPU,
+	},
+};
+
 static const struct ad4030_chip_info ad4030_24_chip_info = {
 	.name = "ad4030-24",
 	.available_masks = ad4030_channel_masks,
@@ -1123,10 +1630,14 @@ static const struct ad4030_chip_info ad4030_24_chip_info = {
 		AD4030_CHAN_CMO(1, 0),
 		IIO_CHAN_SOFT_TIMESTAMP(2),
 	},
+	.offload_channels = {
+		AD4030_OFFLOAD_CHAN_DIFF(0, ad4030_24_offload_scan_types),
+	},
 	.grade = AD4030_REG_CHIP_GRADE_AD4030_24_GRADE,
 	.precision_bits = 24,
 	.num_voltage_inputs = 1,
 	.tcyc_ns = AD4030_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 2 * HZ_PER_MHZ,
 };
 
 static const struct ad4030_chip_info ad4630_16_chip_info = {
@@ -1139,10 +1650,15 @@ static const struct ad4030_chip_info ad4630_16_chip_info = {
 		AD4030_CHAN_CMO(3, 1),
 		IIO_CHAN_SOFT_TIMESTAMP(4),
 	},
+	.offload_channels = {
+		AD4030_OFFLOAD_CHAN_DIFF(0, ad4030_16_offload_scan_types),
+		AD4030_OFFLOAD_CHAN_DIFF(1, ad4030_16_offload_scan_types),
+	},
 	.grade = AD4030_REG_CHIP_GRADE_AD4630_16_GRADE,
 	.precision_bits = 16,
 	.num_voltage_inputs = 2,
 	.tcyc_ns = AD4030_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 2 * HZ_PER_MHZ,
 };
 
 static const struct ad4030_chip_info ad4630_24_chip_info = {
@@ -1155,10 +1671,15 @@ static const struct ad4030_chip_info ad4630_24_chip_info = {
 		AD4030_CHAN_CMO(3, 1),
 		IIO_CHAN_SOFT_TIMESTAMP(4),
 	},
+	.offload_channels = {
+		AD4030_OFFLOAD_CHAN_DIFF(0, ad4030_24_offload_scan_types),
+		AD4030_OFFLOAD_CHAN_DIFF(1, ad4030_24_offload_scan_types),
+	},
 	.grade = AD4030_REG_CHIP_GRADE_AD4630_24_GRADE,
 	.precision_bits = 24,
 	.num_voltage_inputs = 2,
 	.tcyc_ns = AD4030_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 2 * HZ_PER_MHZ,
 };
 
 static const struct ad4030_chip_info ad4632_16_chip_info = {
@@ -1171,10 +1692,15 @@ static const struct ad4030_chip_info ad4632_16_chip_info = {
 		AD4030_CHAN_CMO(3, 1),
 		IIO_CHAN_SOFT_TIMESTAMP(4),
 	},
+	.offload_channels = {
+		AD4030_OFFLOAD_CHAN_DIFF(0, ad4030_16_offload_scan_types),
+		AD4030_OFFLOAD_CHAN_DIFF(1, ad4030_16_offload_scan_types),
+	},
 	.grade = AD4030_REG_CHIP_GRADE_AD4632_16_GRADE,
 	.precision_bits = 16,
 	.num_voltage_inputs = 2,
 	.tcyc_ns = AD4632_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 500 * HZ_PER_KHZ,
 };
 
 static const struct ad4030_chip_info ad4632_24_chip_info = {
@@ -1187,10 +1713,53 @@ static const struct ad4030_chip_info ad4632_24_chip_info = {
 		AD4030_CHAN_CMO(3, 1),
 		IIO_CHAN_SOFT_TIMESTAMP(4),
 	},
+	.offload_channels = {
+		AD4030_OFFLOAD_CHAN_DIFF(0, ad4030_24_offload_scan_types),
+		AD4030_OFFLOAD_CHAN_DIFF(1, ad4030_24_offload_scan_types),
+	},
 	.grade = AD4030_REG_CHIP_GRADE_AD4632_24_GRADE,
 	.precision_bits = 24,
 	.num_voltage_inputs = 2,
 	.tcyc_ns = AD4632_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 500 * HZ_PER_KHZ,
+};
+
+static const struct ad4030_chip_info adaq4216_chip_info = {
+	.name = "adaq4216",
+	.available_masks = ad4030_channel_masks,
+	.channels = {
+		ADAQ4216_CHAN_DIFF(0, ad4030_16_scan_types),
+		AD4030_CHAN_CMO(1, 0),
+		IIO_CHAN_SOFT_TIMESTAMP(2),
+	},
+	.offload_channels = {
+		ADAQ4216_OFFLOAD_CHAN_DIFF(0, ad4030_16_offload_scan_types),
+	},
+	.grade = AD4030_REG_CHIP_GRADE_ADAQ4216_GRADE,
+	.precision_bits = 16,
+	.has_pga = true,
+	.num_voltage_inputs = 1,
+	.tcyc_ns = AD4030_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 2 * HZ_PER_MHZ,
+};
+
+static const struct ad4030_chip_info adaq4224_chip_info = {
+	.name = "adaq4224",
+	.available_masks = ad4030_channel_masks,
+	.channels = {
+		ADAQ4216_CHAN_DIFF(0, ad4030_24_scan_types),
+		AD4030_CHAN_CMO(1, 0),
+		IIO_CHAN_SOFT_TIMESTAMP(2),
+	},
+	.offload_channels = {
+		ADAQ4216_OFFLOAD_CHAN_DIFF(0, ad4030_24_offload_scan_types),
+	},
+	.grade = AD4030_REG_CHIP_GRADE_ADAQ4224_GRADE,
+	.precision_bits = 24,
+	.has_pga = true,
+	.num_voltage_inputs = 1,
+	.tcyc_ns = AD4030_TCYC_ADJUSTED_NS,
+	.max_sample_rate_hz = 2 * HZ_PER_MHZ,
 };
 
 static const struct spi_device_id ad4030_id_table[] = {
@@ -1199,6 +1768,8 @@ static const struct spi_device_id ad4030_id_table[] = {
 	{ "ad4630-24", (kernel_ulong_t)&ad4630_24_chip_info },
 	{ "ad4632-16", (kernel_ulong_t)&ad4632_16_chip_info },
 	{ "ad4632-24", (kernel_ulong_t)&ad4632_24_chip_info },
+	{ "adaq4216", (kernel_ulong_t)&adaq4216_chip_info },
+	{ "adaq4224", (kernel_ulong_t)&adaq4224_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4030_id_table);
@@ -1209,6 +1780,8 @@ static const struct of_device_id ad4030_of_match[] = {
 	{ .compatible = "adi,ad4630-24", .data = &ad4630_24_chip_info },
 	{ .compatible = "adi,ad4632-16", .data = &ad4632_16_chip_info },
 	{ .compatible = "adi,ad4632-24", .data = &ad4632_24_chip_info },
+	{ .compatible = "adi,adaq4216", .data = &adaq4216_chip_info },
+	{ .compatible = "adi,adaq4224", .data = &adaq4224_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad4030_of_match);
@@ -1226,3 +1799,4 @@ module_spi_driver(ad4030_driver);
 MODULE_AUTHOR("Esteban Blanc <eblanc@baylibre.com>");
 MODULE_DESCRIPTION("Analog Devices AD4630 ADC family driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_DMAENGINE_BUFFER");
diff --git a/drivers/iio/adc/ad4062.c b/drivers/iio/adc/ad4062.c
index dd4ad32aa6f53..222499a45b9b4 100644
--- a/drivers/iio/adc/ad4062.c
+++ b/drivers/iio/adc/ad4062.c
@@ -719,10 +719,8 @@ static int ad4062_request_irq(struct iio_dev *indio_dev)
 	}
 	st->gpo_irq[1] = true;
 
-	return devm_request_threaded_irq(dev, ret,
-					 ad4062_irq_handler_drdy,
-					 NULL, IRQF_ONESHOT, indio_dev->name,
-					 indio_dev);
+	return devm_request_irq(dev, ret, ad4062_irq_handler_drdy,
+				IRQF_NO_THREAD, indio_dev->name, indio_dev);
 }
 
 static const struct iio_trigger_ops ad4062_trigger_ops = {
@@ -955,7 +953,7 @@ static int ad4062_write_raw_dispatch(struct ad4062_state *st, int val, int val2,
 	default:
 		return -EINVAL;
 	}
-};
+}
 
 static int ad4062_write_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan, int val,
@@ -1199,11 +1197,14 @@ static int ad4062_write_event_value(struct iio_dev *indio_dev,
  * The AD4062 in burst averaging mode increases realbits from 16-bits to
  * 20-bits, increasing the storagebits from 16-bits to 32-bits.
  */
-static inline size_t ad4062_sizeof_storagebits(struct ad4062_state *st)
+static inline int ad4062_sizeof_storagebits(struct ad4062_state *st)
 {
 	const struct iio_scan_type *scan_type =
 		iio_get_current_scan_type(st->indio_dev, st->chip->channels);
 
+	if (IS_ERR(scan_type))
+		return PTR_ERR(scan_type);
+
 	return BITS_TO_BYTES(scan_type->storagebits);
 }
 
@@ -1233,7 +1234,12 @@ static int pm_ad4062_triggered_buffer_postenable(struct ad4062_state *st)
 	if (ret)
 		return ret;
 
-	st->conv_sizeof = ad4062_sizeof_storagebits(st);
+	ret = ad4062_sizeof_storagebits(st);
+	if (ret < 0)
+		return ret;
+
+	st->conv_sizeof = ret;
+
 	st->conv_addr = ad4062_get_conv_addr(st, st->conv_sizeof);
 	/* CONV_READ requires read to trigger first sample. */
 	struct i3c_xfer xfer_sample[2] = {
diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c
index 7cf3b6ed7940a..204ad198342ba 100644
--- a/drivers/iio/adc/ad4080.c
+++ b/drivers/iio/adc/ad4080.c
@@ -127,10 +127,13 @@
 #define AD4080_SPI_READ						BIT(7)
 #define AD4080_CHIP_ID						0x0050
 #define AD4081_CHIP_ID						0x0051
+#define AD4082_CHIP_ID						0x0052
 #define AD4083_CHIP_ID						0x0053
 #define AD4084_CHIP_ID						0x0054
+#define AD4085_CHIP_ID						0x0055
 #define AD4086_CHIP_ID						0x0056
 #define AD4087_CHIP_ID						0x0057
+#define AD4088_CHIP_ID						0x0058
 
 #define AD4080_LVDS_CNV_CLK_CNT_MAX				7
 
@@ -185,7 +188,6 @@ struct ad4080_state {
 	 */
 	struct mutex			lock;
 	unsigned int			num_lanes;
-	unsigned int			dec_rate;
 	unsigned long			clk_rate;
 	enum ad4080_filter_type		filter_type;
 	bool				lvds_cnv_en;
@@ -442,14 +444,20 @@ static const struct iio_chan_spec ad4080_channel = AD4080_CHANNEL_DEFINE(20, 32)
 
 static const struct iio_chan_spec ad4081_channel = AD4080_CHANNEL_DEFINE(20, 32);
 
+static const struct iio_chan_spec ad4082_channel = AD4080_CHANNEL_DEFINE(20, 32);
+
 static const struct iio_chan_spec ad4083_channel = AD4080_CHANNEL_DEFINE(16, 16);
 
 static const struct iio_chan_spec ad4084_channel = AD4080_CHANNEL_DEFINE(16, 16);
 
+static const struct iio_chan_spec ad4085_channel = AD4080_CHANNEL_DEFINE(16, 16);
+
 static const struct iio_chan_spec ad4086_channel = AD4080_CHANNEL_DEFINE(14, 16);
 
 static const struct iio_chan_spec ad4087_channel = AD4080_CHANNEL_DEFINE(14, 16);
 
+static const struct iio_chan_spec ad4088_channel = AD4080_CHANNEL_DEFINE(14, 16);
+
 static const struct ad4080_chip_info ad4080_chip_info = {
 	.name = "ad4080",
 	.product_id = AD4080_CHIP_ID,
@@ -470,6 +478,16 @@ static const struct ad4080_chip_info ad4081_chip_info = {
 	.lvds_cnv_clk_cnt_max = 2,
 };
 
+static const struct ad4080_chip_info ad4082_chip_info = {
+	.name = "ad4082",
+	.product_id = AD4082_CHIP_ID,
+	.scale_table = ad4080_scale_table,
+	.num_scales = ARRAY_SIZE(ad4080_scale_table),
+	.num_channels = 1,
+	.channels = &ad4082_channel,
+	.lvds_cnv_clk_cnt_max = 8,
+};
+
 static const struct ad4080_chip_info ad4083_chip_info = {
 	.name = "ad4083",
 	.product_id = AD4083_CHIP_ID,
@@ -490,6 +508,16 @@ static const struct ad4080_chip_info ad4084_chip_info = {
 	.lvds_cnv_clk_cnt_max = 2,
 };
 
+static const struct ad4080_chip_info ad4085_chip_info = {
+	.name = "ad4085",
+	.product_id = AD4085_CHIP_ID,
+	.scale_table = ad4080_scale_table,
+	.num_scales = ARRAY_SIZE(ad4080_scale_table),
+	.num_channels = 1,
+	.channels = &ad4085_channel,
+	.lvds_cnv_clk_cnt_max = 8,
+};
+
 static const struct ad4080_chip_info ad4086_chip_info = {
 	.name = "ad4086",
 	.product_id = AD4086_CHIP_ID,
@@ -510,6 +538,16 @@ static const struct ad4080_chip_info ad4087_chip_info = {
 	.lvds_cnv_clk_cnt_max = 1,
 };
 
+static const struct ad4080_chip_info ad4088_chip_info = {
+	.name = "ad4088",
+	.product_id = AD4088_CHIP_ID,
+	.scale_table = ad4080_scale_table,
+	.num_scales = ARRAY_SIZE(ad4080_scale_table),
+	.num_channels = 1,
+	.channels = &ad4088_channel,
+	.lvds_cnv_clk_cnt_max = 8,
+};
+
 static int ad4080_setup(struct iio_dev *indio_dev)
 {
 	struct ad4080_state *st = iio_priv(indio_dev);
@@ -666,10 +704,13 @@ static int ad4080_probe(struct spi_device *spi)
 static const struct spi_device_id ad4080_id[] = {
 	{ "ad4080", (kernel_ulong_t)&ad4080_chip_info },
 	{ "ad4081", (kernel_ulong_t)&ad4081_chip_info },
+	{ "ad4082", (kernel_ulong_t)&ad4082_chip_info },
 	{ "ad4083", (kernel_ulong_t)&ad4083_chip_info },
 	{ "ad4084", (kernel_ulong_t)&ad4084_chip_info },
+	{ "ad4085", (kernel_ulong_t)&ad4085_chip_info },
 	{ "ad4086", (kernel_ulong_t)&ad4086_chip_info },
 	{ "ad4087", (kernel_ulong_t)&ad4087_chip_info },
+	{ "ad4088", (kernel_ulong_t)&ad4088_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4080_id);
@@ -677,10 +718,13 @@ MODULE_DEVICE_TABLE(spi, ad4080_id);
 static const struct of_device_id ad4080_of_match[] = {
 	{ .compatible = "adi,ad4080", &ad4080_chip_info },
 	{ .compatible = "adi,ad4081", &ad4081_chip_info },
+	{ .compatible = "adi,ad4082", &ad4082_chip_info },
 	{ .compatible = "adi,ad4083", &ad4083_chip_info },
 	{ .compatible = "adi,ad4084", &ad4084_chip_info },
+	{ .compatible = "adi,ad4085", &ad4085_chip_info },
 	{ .compatible = "adi,ad4086", &ad4086_chip_info },
 	{ .compatible = "adi,ad4087", &ad4087_chip_info },
+	{ .compatible = "adi,ad4088", &ad4088_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad4080_of_match);
diff --git a/drivers/iio/adc/ad4170-4.c b/drivers/iio/adc/ad4170-4.c
index 82205bfae531e..77af0e6b2c596 100644
--- a/drivers/iio/adc/ad4170-4.c
+++ b/drivers/iio/adc/ad4170-4.c
@@ -275,9 +275,9 @@ static const unsigned int ad4170_reg_size[] = {
 };
 
 enum ad4170_ref_buf {
-	AD4170_REF_BUF_PRE,	/* Pre-charge referrence buffer */
-	AD4170_REF_BUF_FULL,	/* Full referrence buffering */
-	AD4170_REF_BUF_BYPASS,	/* Bypass referrence buffering */
+	AD4170_REF_BUF_PRE,	/* Pre-charge reference buffer */
+	AD4170_REF_BUF_FULL,	/* Full reference buffering */
+	AD4170_REF_BUF_BYPASS,	/* Bypass reference buffering */
 };
 
 /* maps adi,positive/negative-reference-buffer property values to enum */
diff --git a/drivers/iio/adc/ad7173.c b/drivers/iio/adc/ad7173.c
index d36612352b443..f76a9e08f39e4 100644
--- a/drivers/iio/adc/ad7173.c
+++ b/drivers/iio/adc/ad7173.c
@@ -1763,7 +1763,8 @@ static int ad7173_validate_openwire_ain_inputs(struct ad7173_state *st,
 static unsigned int ad7173_calc_openwire_thrsh_raw(struct ad7173_state *st,
 						   struct iio_chan_spec *chan,
 						   struct ad7173_channel *chan_st_priv,
-						   unsigned int thrsh_mv) {
+						   unsigned int thrsh_mv)
+{
 	unsigned int thrsh_raw;
 
 	thrsh_raw =
diff --git a/drivers/iio/adc/ad7192.c b/drivers/iio/adc/ad7192.c
index 530e1d307860e..8b1664f6b102c 100644
--- a/drivers/iio/adc/ad7192.c
+++ b/drivers/iio/adc/ad7192.c
@@ -1402,9 +1402,6 @@ static int ad7192_probe(struct spi_device *spi)
 	st->int_vref_mv = ret == -ENODEV ? avdd_mv : ret / MILLI;
 
 	st->chip_info = spi_get_device_match_data(spi);
-	if (!st->chip_info)
-		return -ENODEV;
-
 	indio_dev->name = st->chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = st->chip_info->info;
diff --git a/drivers/iio/adc/ad7266.c b/drivers/iio/adc/ad7266.c
index 3364ac6c4631f..0ef36c249ab80 100644
--- a/drivers/iio/adc/ad7266.c
+++ b/drivers/iio/adc/ad7266.c
@@ -409,10 +409,8 @@ static int ad7266_probe(struct spi_device *spi)
 				st->gpios[i] = devm_gpiod_get(&spi->dev,
 						      ad7266_gpio_labels[i],
 						      GPIOD_OUT_LOW);
-				if (IS_ERR(st->gpios[i])) {
-					ret = PTR_ERR(st->gpios[i]);
-					return ret;
-				}
+				if (IS_ERR(st->gpios[i]))
+					return PTR_ERR(st->gpios[i]);
 			}
 		}
 	} else {
diff --git a/drivers/iio/adc/ad7380.c b/drivers/iio/adc/ad7380.c
index bfd908deefc0f..9f77990a03f93 100644
--- a/drivers/iio/adc/ad7380.c
+++ b/drivers/iio/adc/ad7380.c
@@ -77,8 +77,7 @@
 #define AD7380_CONFIG1_REFSEL		BIT(1)
 #define AD7380_CONFIG1_PMODE		BIT(0)
 
-#define AD7380_CONFIG2_SDO2		GENMASK(9, 8)
-#define AD7380_CONFIG2_SDO		BIT(8)
+#define AD7380_CONFIG2_SDO		GENMASK(9, 8)
 #define AD7380_CONFIG2_RESET		GENMASK(7, 0)
 
 #define AD7380_CONFIG2_RESET_SOFT	0x3C
@@ -92,11 +91,6 @@
 #define T_CONVERT_X_NS 500		/* xth conversion start time (oversampling) */
 #define T_POWERUP_US 5000		/* Power up */
 
-/*
- * AD738x support several SDO lines to increase throughput, but driver currently
- * supports only 1 SDO line (standard SPI transaction)
- */
-#define AD7380_NUM_SDO_LINES		1
 #define AD7380_DEFAULT_GAIN_MILLI	1000
 
 /*
@@ -888,6 +882,8 @@ struct ad7380_state {
 	bool resolution_boost_enabled;
 	unsigned int ch;
 	bool seq;
+	/* How many SDO lines are wired up. */
+	u8 num_sdo_lines;
 	unsigned int vref_mv;
 	unsigned int vcm_mv[MAX_NUM_CHANNELS];
 	unsigned int gain_milli[MAX_NUM_CHANNELS];
@@ -1084,7 +1080,7 @@ static int ad7380_set_ch(struct ad7380_state *st, unsigned int ch)
 	if (oversampling_ratio > 1)
 		xfer.delay.value = T_CONVERT_0_NS +
 			T_CONVERT_X_NS * (oversampling_ratio - 1) *
-			st->chip_info->num_simult_channels / AD7380_NUM_SDO_LINES;
+			st->chip_info->num_simult_channels / st->num_sdo_lines;
 
 	return spi_sync_transfer(st->spi, &xfer, 1);
 }
@@ -1113,7 +1109,7 @@ static int ad7380_update_xfers(struct ad7380_state *st,
 	if (oversampling_ratio > 1)
 		t_convert = T_CONVERT_0_NS + T_CONVERT_X_NS *
 			(oversampling_ratio - 1) *
-			st->chip_info->num_simult_channels / AD7380_NUM_SDO_LINES;
+			st->chip_info->num_simult_channels / st->num_sdo_lines;
 
 	if (st->seq) {
 		xfer[0].delay.value = xfer[1].delay.value = t_convert;
@@ -1198,6 +1194,8 @@ static int ad7380_init_offload_msg(struct ad7380_state *st,
 	xfer->bits_per_word = scan_type->realbits;
 	xfer->offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
 	xfer->len = AD7380_SPI_BYTES(scan_type) * st->chip_info->num_simult_channels;
+	if (st->num_sdo_lines > 1)
+		xfer->multi_lane_mode = SPI_MULTI_LANE_MODE_STRIPE;
 
 	spi_message_init_with_transfers(&st->offload_msg, xfer, 1);
 	st->offload_msg.offload = st->offload;
@@ -1793,6 +1791,7 @@ static const struct iio_info ad7380_info = {
 
 static int ad7380_init(struct ad7380_state *st, bool external_ref_en)
 {
+	u32 sdo;
 	int ret;
 
 	/* perform hard reset */
@@ -1815,11 +1814,24 @@ static int ad7380_init(struct ad7380_state *st, bool external_ref_en)
 	st->ch = 0;
 	st->seq = false;
 
-	/* SPI 1-wire mode */
+	/* SDO field has an irregular mapping. */
+	switch (st->num_sdo_lines) {
+	case 1:
+		sdo = 1;
+		break;
+	case 2:
+		sdo = 0;
+		break;
+	case 4:
+		sdo = 2;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	return regmap_update_bits(st->regmap, AD7380_REG_ADDR_CONFIG2,
 				  AD7380_CONFIG2_SDO,
-				  FIELD_PREP(AD7380_CONFIG2_SDO,
-					     AD7380_NUM_SDO_LINES));
+				  FIELD_PREP(AD7380_CONFIG2_SDO, sdo));
 }
 
 static int ad7380_probe_spi_offload(struct iio_dev *indio_dev,
@@ -1842,7 +1854,7 @@ static int ad7380_probe_spi_offload(struct iio_dev *indio_dev,
 				     "failed to get offload trigger\n");
 
 	sample_rate = st->chip_info->max_conversion_rate_hz *
-		      AD7380_NUM_SDO_LINES / st->chip_info->num_simult_channels;
+		      st->num_sdo_lines / st->chip_info->num_simult_channels;
 
 	st->sample_freq_range[0] = 1; /* min */
 	st->sample_freq_range[1] = 1; /* step */
@@ -1850,7 +1862,7 @@ static int ad7380_probe_spi_offload(struct iio_dev *indio_dev,
 
 	/*
 	 * Starting with a quite low frequency, to allow oversampling x32,
-	 * user is then reponsible to adjust the frequency for the specific case.
+	 * user is then responsible to adjust the frequency for the specific case.
 	 */
 	ret = ad7380_set_sample_freq(st, sample_rate / 32);
 	if (ret)
@@ -1887,6 +1899,13 @@ static int ad7380_probe(struct spi_device *spi)
 	if (!st->chip_info)
 		return dev_err_probe(dev, -EINVAL, "missing match data\n");
 
+	st->num_sdo_lines = spi->num_rx_lanes;
+
+	if (st->num_sdo_lines < 1 || st->num_sdo_lines > st->chip_info->num_simult_channels)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid number of SDO lines (%d)\n",
+				     st->num_sdo_lines);
+
 	ret = devm_regulator_bulk_get_enable(dev, st->chip_info->num_supplies,
 					     st->chip_info->supplies);
 
@@ -2010,6 +2029,8 @@ static int ad7380_probe(struct spi_device *spi)
 	st->normal_xfer[0].cs_change_delay.value = st->chip_info->timing_specs->t_csh_ns;
 	st->normal_xfer[0].cs_change_delay.unit = SPI_DELAY_UNIT_NSECS;
 	st->normal_xfer[1].rx_buf = st->scan_data;
+	if (st->num_sdo_lines > 1)
+		st->normal_xfer[1].multi_lane_mode = SPI_MULTI_LANE_MODE_STRIPE;
 
 	spi_message_init_with_transfers(&st->normal_msg, st->normal_xfer,
 					ARRAY_SIZE(st->normal_xfer));
@@ -2031,6 +2052,10 @@ static int ad7380_probe(struct spi_device *spi)
 	st->seq_xfer[2].cs_change = 1;
 	st->seq_xfer[2].cs_change_delay.value = st->chip_info->timing_specs->t_csh_ns;
 	st->seq_xfer[2].cs_change_delay.unit = SPI_DELAY_UNIT_NSECS;
+	if (st->num_sdo_lines > 1) {
+		st->seq_xfer[2].multi_lane_mode = SPI_MULTI_LANE_MODE_STRIPE;
+		st->seq_xfer[3].multi_lane_mode = SPI_MULTI_LANE_MODE_STRIPE;
+	}
 
 	spi_message_init_with_transfers(&st->seq_msg, st->seq_xfer,
 					ARRAY_SIZE(st->seq_xfer));
diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c
index e16dede687d3f..73fb734d06b23 100644
--- a/drivers/iio/adc/ad7768-1.c
+++ b/drivers/iio/adc/ad7768-1.c
@@ -25,12 +25,15 @@
 #include <linux/regulator/consumer.h>
 #include <linux/regulator/driver.h>
 #include <linux/sysfs.h>
+#include <linux/spi/offload/consumer.h>
+#include <linux/spi/offload/provider.h>
 #include <linux/spi/spi.h>
 #include <linux/unaligned.h>
 #include <linux/units.h>
 #include <linux/util_macros.h>
 
 #include <linux/iio/buffer.h>
+#include <linux/iio/buffer-dmaengine.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/trigger.h>
@@ -161,6 +164,8 @@ enum ad7768_filter_regval {
 enum ad7768_scan_type {
 	AD7768_SCAN_TYPE_NORMAL,
 	AD7768_SCAN_TYPE_HIGH_SPEED,
+	AD7768_SCAN_TYPE_OFFLOAD_NORMAL,
+	AD7768_SCAN_TYPE_OFFLOAD_HIGH_SPEED,
 };
 
 enum {
@@ -266,6 +271,18 @@ static const struct iio_scan_type ad7768_scan_type[] = {
 		.storagebits = 16,
 		.endianness = IIO_BE,
 	},
+	[AD7768_SCAN_TYPE_OFFLOAD_NORMAL] = {
+		.sign = 's',
+		.realbits = 24,
+		.storagebits = 32,
+		.endianness = IIO_CPU,
+	},
+	[AD7768_SCAN_TYPE_OFFLOAD_HIGH_SPEED] = {
+		.sign = 's',
+		.realbits = 16,
+		.storagebits = 32,
+		.endianness = IIO_CPU,
+	},
 };
 
 struct ad7768_chip_info {
@@ -283,6 +300,8 @@ struct ad7768_chip_info {
 
 struct ad7768_state {
 	struct spi_device *spi;
+	struct spi_offload *offload;
+	struct spi_offload_trigger *offload_trigger;
 	struct regmap *regmap;
 	struct regmap *regmap24;
 	int vref_uv;
@@ -290,7 +309,6 @@ struct ad7768_state {
 	unsigned int vcm_output_sel;
 	struct clk *mclk;
 	unsigned int mclk_freq;
-	unsigned int mclk_div;
 	unsigned int oversampling_ratio;
 	enum ad7768_filter_type filter_type;
 	unsigned int samp_freq;
@@ -306,6 +324,8 @@ struct ad7768_state {
 	struct gpio_desc *gpio_reset;
 	const char *labels[AD7768_MAX_CHANNELS];
 	struct gpio_chip gpiochip;
+	struct spi_transfer offload_xfer;
+	struct spi_message offload_msg;
 	const struct ad7768_chip_info *chip;
 	bool en_spi_sync;
 	struct mutex pga_lock; /* protect device internal state (PGA) */
@@ -464,13 +484,11 @@ static int ad7768_scan_direct(struct iio_dev *indio_dev)
 	int readval, ret;
 
 	reinit_completion(&st->completion);
-
-	ret = ad7768_set_mode(st, AD7768_ONE_SHOT);
-	if (ret < 0)
-		return ret;
+	enable_irq(st->spi->irq);
 
 	ret = wait_for_completion_timeout(&st->completion,
 					  msecs_to_jiffies(1000));
+	disable_irq(st->spi->irq);
 	if (!ret)
 		return -ETIMEDOUT;
 
@@ -487,14 +505,6 @@ static int ad7768_scan_direct(struct iio_dev *indio_dev)
 	if (st->oversampling_ratio == 8)
 		readval >>= 8;
 
-	/*
-	 * Any SPI configuration of the AD7768-1 can only be
-	 * performed in continuous conversion mode.
-	 */
-	ret = ad7768_set_mode(st, AD7768_CONTINUOUS);
-	if (ret < 0)
-		return ret;
-
 	return readval;
 }
 
@@ -1138,6 +1148,10 @@ static int ad7768_get_current_scan_type(const struct iio_dev *indio_dev,
 {
 	struct ad7768_state *st = iio_priv(indio_dev);
 
+	if (st->offload)
+		return st->oversampling_ratio == 8 ?
+		       AD7768_SCAN_TYPE_OFFLOAD_HIGH_SPEED : AD7768_SCAN_TYPE_OFFLOAD_NORMAL;
+
 	return st->oversampling_ratio == 8 ?
 	       AD7768_SCAN_TYPE_HIGH_SPEED : AD7768_SCAN_TYPE_NORMAL;
 }
@@ -1252,6 +1266,10 @@ static int ad7768_setup(struct iio_dev *indio_dev)
 			return ret;
 	}
 
+	ret = ad7768_set_mode(st, AD7768_CONTINUOUS);
+	if (ret)
+		return ret;
+
 	/* For backwards compatibility, try the adi,sync-in-gpios property */
 	st->gpio_sync_in = devm_gpiod_get_optional(&st->spi->dev, "adi,sync-in",
 						   GPIOD_OUT_LOW);
@@ -1356,8 +1374,94 @@ static const struct iio_buffer_setup_ops ad7768_buffer_ops = {
 	.predisable = &ad7768_buffer_predisable,
 };
 
+static int ad7768_offload_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	struct spi_offload_trigger_config config = {
+		.type = SPI_OFFLOAD_TRIGGER_DATA_READY,
+	};
+	const struct iio_scan_type *scan_type;
+	unsigned int unused;
+	int ret;
+
+	scan_type = iio_get_current_scan_type(indio_dev, &indio_dev->channels[0]);
+	if (IS_ERR(scan_type))
+		return PTR_ERR(scan_type);
+
+	st->offload_xfer.len = spi_bpw_to_bytes(scan_type->realbits);
+	st->offload_xfer.bits_per_word = scan_type->realbits;
+	st->offload_xfer.offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
+
+	spi_message_init_with_transfers(&st->offload_msg, &st->offload_xfer, 1);
+	st->offload_msg.offload = st->offload;
+
+	ret = spi_optimize_message(st->spi, &st->offload_msg);
+	if (ret) {
+		dev_err(&st->spi->dev, "failed to prepare offload, err: %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Write a 1 to the LSB of the INTERFACE_FORMAT register to enter
+	 * continuous read mode. Subsequent data reads do not require an
+	 * initial 8-bit write to query the ADC_DATA register.
+	 */
+	ret =  regmap_write(st->regmap, AD7768_REG_INTERFACE_FORMAT, 0x01);
+	if (ret)
+		goto err_unoptimize_message;
+
+	ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
+					 &config);
+	if (ret)
+		goto err_exit_continuous_read_mode;
+
+	return 0;
+
+err_exit_continuous_read_mode:
+	regmap_read(st->regmap24, AD7768_REG24_ADC_DATA, &unused);
+
+err_unoptimize_message:
+	spi_unoptimize_message(&st->offload_msg);
+
+	return ret;
+}
+
+static int ad7768_offload_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	unsigned int unused;
+
+	spi_offload_trigger_disable(st->offload, st->offload_trigger);
+	spi_unoptimize_message(&st->offload_msg);
+
+	/*
+	 * To exit continuous read mode, perform a single read of the ADC_DATA
+	 * reg (0x2C), which allows further configuration of the device.
+	 */
+	return regmap_read(st->regmap24, AD7768_REG24_ADC_DATA, &unused);
+}
+
+static const struct iio_buffer_setup_ops ad7768_offload_buffer_ops = {
+	.postenable = ad7768_offload_buffer_postenable,
+	.predisable = ad7768_offload_buffer_predisable,
+};
+
+static int ad7768_set_trigger_state(struct iio_trigger *trig, bool enable)
+{
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	if (enable)
+		enable_irq(st->spi->irq);
+	else
+		disable_irq(st->spi->irq);
+
+	return 0;
+}
+
 static const struct iio_trigger_ops ad7768_trigger_ops = {
 	.validate_device = iio_trigger_validate_own_device,
+	.set_trigger_state = ad7768_set_trigger_state,
 };
 
 static int ad7768_set_channel_label(struct iio_dev *indio_dev,
@@ -1590,6 +1694,36 @@ static int ad7768_parse_aaf_gain(struct device *dev, struct ad7768_state *st)
 	return 0;
 }
 
+static bool ad7768_offload_trigger_match(struct spi_offload_trigger *trigger,
+					 enum spi_offload_trigger_type type,
+					 u64 *args, u32 nargs)
+{
+	if (type != SPI_OFFLOAD_TRIGGER_DATA_READY)
+		return false;
+
+	/* Up to 2 args are allowed, but only 1 is used */
+	if (nargs == 0 || nargs > 2 || args[0] != AD7768_TRIGGER_SOURCE_DRDY)
+		return false;
+
+	return true;
+}
+
+static int ad7768_offload_trigger_request(struct spi_offload_trigger *trigger,
+					  enum spi_offload_trigger_type type,
+					  u64 *args, u32 nargs)
+{
+	/* Should already be validated by match, but just in case */
+	if (nargs == 0 || nargs > 2)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct spi_offload_trigger_ops ad7768_offload_trigger_ops = {
+	.match = ad7768_offload_trigger_match,
+	.request = ad7768_offload_trigger_request,
+};
+
 static const struct ad7768_chip_info ad7768_chip_info = {
 	.name = "ad7768-1",
 	.channel_spec = ad7768_channels,
@@ -1627,10 +1761,51 @@ static const struct ad7768_chip_info adaq7769_chip_info = {
 	.has_variable_aaf = true,
 };
 
+static const struct spi_offload_config ad7768_spi_offload_config = {
+	.capability_flags = SPI_OFFLOAD_CAP_TRIGGER | SPI_OFFLOAD_CAP_RX_STREAM_DMA,
+};
+
+static int ad7768_spi_offload_probe(struct iio_dev *indio_dev,
+				    struct ad7768_state *st)
+{
+	struct device *dev = &st->spi->dev;
+	struct spi_offload_trigger_info trigger_info = {
+		.fwnode = dev_fwnode(dev),
+		.ops = &ad7768_offload_trigger_ops,
+		.priv = st,
+	};
+	struct dma_chan *rx_dma;
+	int ret;
+
+	ret = devm_spi_offload_trigger_register(dev, &trigger_info);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register offload trigger\n");
+
+	st->offload_trigger = devm_spi_offload_trigger_get(dev, st->offload,
+							   SPI_OFFLOAD_TRIGGER_DATA_READY);
+	if (IS_ERR(st->offload_trigger))
+		return dev_err_probe(dev, PTR_ERR(st->offload_trigger),
+				     "failed to get offload trigger\n");
+
+	rx_dma = devm_spi_offload_rx_stream_request_dma_chan(dev, st->offload);
+	if (IS_ERR(rx_dma))
+		return dev_err_probe(dev, PTR_ERR(rx_dma), "failed to get offload RX DMA\n");
+
+	ret = devm_iio_dmaengine_buffer_setup_with_handle(dev, indio_dev, rx_dma,
+							  IIO_BUFFER_DIRECTION_IN);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to setup offload RX DMA\n");
+
+	indio_dev->setup_ops = &ad7768_offload_buffer_ops;
+
+	return 0;
+}
+
 static int ad7768_probe(struct spi_device *spi)
 {
 	struct ad7768_state *st;
 	struct iio_dev *indio_dev;
+	struct device *dev = &spi->dev;
 	int ret;
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
@@ -1721,14 +1896,25 @@ static int ad7768_probe(struct spi_device *spi)
 		return ret;
 
 	ret = devm_request_irq(&spi->dev, spi->irq, &ad7768_interrupt,
-			       IRQF_TRIGGER_RISING | IRQF_NO_THREAD,
+			       IRQF_TRIGGER_RISING | IRQF_NO_THREAD | IRQF_NO_AUTOEN,
 			       indio_dev->name, indio_dev);
 	if (ret)
 		return ret;
 
-	ret = ad7768_triggered_buffer_alloc(indio_dev);
-	if (ret)
-		return ret;
+	st->offload = devm_spi_offload_get(dev, spi, &ad7768_spi_offload_config);
+	ret = PTR_ERR_OR_ZERO(st->offload);
+	if (ret == -ENODEV) {
+		/* If not using SPI offload, fall back to low speed usage */
+		ret = ad7768_triggered_buffer_alloc(indio_dev);
+		if (ret)
+			return ret;
+	} else if (ret) {
+		return dev_err_probe(dev, ret, "failed to get SPI offload\n");
+	} else {
+		ret = ad7768_spi_offload_probe(indio_dev, st);
+		if (ret)
+			return ret;
+	}
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
@@ -1764,3 +1950,4 @@ module_spi_driver(ad7768_driver);
 MODULE_AUTHOR("Stefan Popa <stefan.popa@analog.com>");
 MODULE_DESCRIPTION("Analog Devices AD7768-1 ADC driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_DMAENGINE_BUFFER");
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index ccf18ce48e349..8ff7b70d6632e 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -152,7 +152,6 @@ struct ad7793_chip_info {
 
 struct ad7793_state {
 	const struct ad7793_chip_info	*chip_info;
-	u16				int_vref_mv;
 	u16				mode;
 	u16				conf;
 	u32				scale_avail[8][2];
@@ -805,7 +804,7 @@ static int ad7793_probe(struct spi_device *spi)
 
 		vref_mv = ret / 1000;
 	} else {
-		vref_mv = 1170; /* Build-in ref */
+		vref_mv = 1170; /* Built-in ref */
 	}
 
 	st->chip_info =
diff --git a/drivers/iio/adc/ad7887.c b/drivers/iio/adc/ad7887.c
index 87ff95643794c..068171d54596a 100644
--- a/drivers/iio/adc/ad7887.c
+++ b/drivers/iio/adc/ad7887.c
@@ -104,7 +104,7 @@ static int ad7887_ring_postdisable(struct iio_dev *indio_dev)
 {
 	struct ad7887_state *st = iio_priv(indio_dev);
 
-	/* dummy read: restore default CH0 settin */
+	/* dummy read: restore default CH0 settings */
 	return spi_sync(st->spi, &st->msg[AD7887_CH0]);
 }
 
diff --git a/drivers/iio/adc/ad7923.c b/drivers/iio/adc/ad7923.c
index 0369151c7db1e..acc87d486aa42 100644
--- a/drivers/iio/adc/ad7923.c
+++ b/drivers/iio/adc/ad7923.c
@@ -30,7 +30,7 @@
 #define AD7923_PM_MODE_AS	(1)		/* auto shutdown */
 #define AD7923_PM_MODE_FS	(2)		/* full shutdown */
 #define AD7923_PM_MODE_OPS	(3)		/* normal operation */
-#define AD7923_SEQUENCE_OFF	(0)		/* no sequence fonction */
+#define AD7923_SEQUENCE_OFF	(0)		/* no sequence function */
 #define AD7923_SEQUENCE_PROTECT	(2)		/* no interrupt write cycle */
 #define AD7923_SEQUENCE_ON	(3)		/* continuous sequence */
 
@@ -39,7 +39,7 @@
 #define AD7923_CHANNEL_WRITE(channel)	((channel) << 6) /* write channel */
 #define AD7923_SEQUENCE_WRITE(sequence)	((((sequence) & 1) << 3) \
 					+ (((sequence) & 2) << 9))
-						/* write sequence fonction */
+						/* write sequence function */
 /* left shift for CR : bit 11 transmit in first */
 #define AD7923_SHIFT_REGISTER	4
 
diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c
index 0228885455807..0bf67437508f7 100644
--- a/drivers/iio/adc/ad9467.c
+++ b/drivers/iio/adc/ad9467.c
@@ -176,7 +176,6 @@ struct ad9467_state {
 	struct clk *clk;
 	/* used for debugfs */
 	struct ad9467_chan_test_mode *chan_test;
-	unsigned int output_mode;
 	unsigned int (*scales)[2];
 	/*
 	 * Times 2 because we may also invert the signal polarity and run the
@@ -925,7 +924,11 @@ static int __ad9467_update_clock(struct ad9467_state *st, long r_clk)
 		return ret;
 
 	guard(mutex)(&st->lock);
-	return ad9467_calibrate(st);
+
+	if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION))
+		return ad9467_calibrate(st);
+
+	return 0;
 }
 
 static int ad9467_write_raw(struct iio_dev *indio_dev,
@@ -1131,12 +1134,15 @@ static ssize_t ad9467_chan_test_mode_read(struct file *file,
 		len = scnprintf(buf, sizeof(buf), "Running \"%s\" Test:\n\t",
 				ad9467_test_modes[chan->mode]);
 
-		ret = iio_backend_debugfs_print_chan_status(st->back, chan->idx,
-							    buf + len,
-							    sizeof(buf) - len);
-		if (ret < 0)
-			return ret;
-		len += ret;
+		if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION)) {
+			ret = iio_backend_debugfs_print_chan_status(st->back,
+								    chan->idx,
+								    buf + len,
+								    sizeof(buf) - len);
+			if (ret < 0)
+				return ret;
+			len += ret;
+		}
 	} else if (chan->mode == AN877_ADC_TESTMODE_OFF) {
 		len = scnprintf(buf, sizeof(buf), "No test Running...\n");
 	} else {
@@ -1175,11 +1181,13 @@ static ssize_t ad9467_chan_test_mode_write(struct file *file,
 	if (mode == AN877_ADC_TESTMODE_OFF) {
 		unsigned int out_mode;
 
-		if (chan->mode == AN877_ADC_TESTMODE_PN9_SEQ ||
-		    chan->mode == AN877_ADC_TESTMODE_PN23_SEQ) {
-			ret = ad9467_backend_testmode_off(st, chan->idx);
-			if (ret)
-				return ret;
+		if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION)) {
+			if (chan->mode == AN877_ADC_TESTMODE_PN9_SEQ ||
+			    chan->mode == AN877_ADC_TESTMODE_PN23_SEQ) {
+				ret = ad9467_backend_testmode_off(st, chan->idx);
+				if (ret)
+					return ret;
+			}
 		}
 
 		ret = ad9467_testmode_set(st, chan->idx, mode);
@@ -1205,16 +1213,18 @@ static ssize_t ad9467_chan_test_mode_write(struct file *file,
 			return ret;
 
 		/*  some patterns have a backend matching monitoring block */
-		if (mode == AN877_ADC_TESTMODE_PN9_SEQ) {
-			ret = ad9467_backend_testmode_on(st, chan->idx,
+		if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION)) {
+			if (mode == AN877_ADC_TESTMODE_PN9_SEQ) {
+				ret = ad9467_backend_testmode_on(st, chan->idx,
 							 IIO_BACKEND_ADI_PRBS_9A);
-			if (ret)
-				return ret;
-		} else if (mode == AN877_ADC_TESTMODE_PN23_SEQ) {
-			ret = ad9467_backend_testmode_on(st, chan->idx,
+				if (ret)
+					return ret;
+			} else if (mode == AN877_ADC_TESTMODE_PN23_SEQ) {
+				ret = ad9467_backend_testmode_on(st, chan->idx,
 							 IIO_BACKEND_ADI_PRBS_23A);
-			if (ret)
-				return ret;
+				if (ret)
+					return ret;
+			}
 		}
 	}
 
@@ -1280,8 +1290,9 @@ static void ad9467_debugfs_init(struct iio_dev *indio_dev)
 	if (!st->chan_test)
 		return;
 
-	debugfs_create_file("calibration_table_dump", 0400, d, st,
-			    &ad9467_calib_table_fops);
+	if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION))
+		debugfs_create_file("calibration_table_dump", 0400, d, st,
+				    &ad9467_calib_table_fops);
 
 	for (chan = 0; chan < st->info->num_channels; chan++) {
 		snprintf(attr_name, sizeof(attr_name), "in_voltage%u_test_mode",
@@ -1300,12 +1311,13 @@ static void ad9467_debugfs_init(struct iio_dev *indio_dev)
 
 static int ad9467_probe(struct spi_device *spi)
 {
+	struct device *dev = &spi->dev;
 	struct iio_dev *indio_dev;
 	struct ad9467_state *st;
 	unsigned int id;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -1320,16 +1332,15 @@ static int ad9467_probe(struct spi_device *spi)
 	if (AD9467_CAN_INVERT(st))
 		st->calib_map_size *= 2;
 
-	st->clk = devm_clk_get_enabled(&spi->dev, "adc-clk");
+	st->clk = devm_clk_get_enabled(dev, "adc-clk");
 	if (IS_ERR(st->clk))
 		return PTR_ERR(st->clk);
 
-	st->pwrdown_gpio = devm_gpiod_get_optional(&spi->dev, "powerdown",
-						   GPIOD_OUT_LOW);
+	st->pwrdown_gpio = devm_gpiod_get_optional(dev, "powerdown", GPIOD_OUT_LOW);
 	if (IS_ERR(st->pwrdown_gpio))
 		return PTR_ERR(st->pwrdown_gpio);
 
-	ret = ad9467_reset(&spi->dev);
+	ret = ad9467_reset(dev);
 	if (ret)
 		return ret;
 
@@ -1339,7 +1350,7 @@ static int ad9467_probe(struct spi_device *spi)
 
 	id = ad9467_spi_read(st, AN877_ADC_REG_CHIP_ID);
 	if (id != st->info->id) {
-		dev_err(&spi->dev, "Mismatch CHIP_ID, got 0x%X, expected 0x%X\n",
+		dev_err(dev, "Mismatch CHIP_ID, got 0x%X, expected 0x%X\n",
 			id, st->info->id);
 		return -ENODEV;
 	}
@@ -1356,19 +1367,25 @@ static int ad9467_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = devm_iio_backend_request_buffer(&spi->dev, st->back, indio_dev);
-	if (ret)
-		return ret;
+	if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_BUFFER)) {
+		ret = devm_iio_backend_request_buffer(dev, st->back, indio_dev);
+		if (ret)
+			return ret;
+	}
 
-	ret = devm_iio_backend_enable(&spi->dev, st->back);
-	if (ret)
-		return ret;
+	if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_ENABLE)) {
+		ret = devm_iio_backend_enable(dev, st->back);
+		if (ret)
+			return ret;
+	}
 
-	ret = ad9467_calibrate(st);
-	if (ret)
-		return ret;
+	if (iio_backend_has_caps(st->back, IIO_BACKEND_CAP_CALIBRATION)) {
+		ret = ad9467_calibrate(st);
+		if (ret)
+			return ret;
+	}
 
-	ret = devm_iio_device_register(&spi->dev, indio_dev);
+	ret = devm_iio_device_register(dev, indio_dev);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c
index db085dc5e5266..c0a9babfa519a 100644
--- a/drivers/iio/adc/ade9000.c
+++ b/drivers/iio/adc/ade9000.c
@@ -218,9 +218,6 @@
 #define ADE9000_ST1_ERROR1_BIT		BIT(29)
 #define ADE9000_ST1_ERROR2_BIT		BIT(30)
 #define ADE9000_ST1_ERROR3_BIT		BIT(31)
-#define ADE9000_ST_ERROR \
-	(ADE9000_ST1_ERROR0 | ADE9000_ST1_ERROR1 | \
-	 ADE9000_ST1_ERROR2 | ADE9000_ST1_ERROR3)
 #define ADE9000_ST1_CROSSING_FIRST	6
 #define ADE9000_ST1_CROSSING_DEPTH	25
 
@@ -283,7 +280,6 @@ enum ade9000_wfb_cfg {
 #define ADE9000_PHASE_C_POS_BIT		BIT(6)
 
 #define ADE9000_MAX_PHASE_NR		3
-#define AD9000_CHANNELS_PER_PHASE	10
 
 /*
  * Calculate register address for multi-phase device.
@@ -787,7 +783,7 @@ static int ade9000_iio_push_streaming(struct iio_dev *indio_dev)
 				   ADE9000_MIDDLE_PAGE_BIT);
 		if (ret) {
 			dev_err_ratelimited(dev, "IRQ0 WFB write fail");
-			return IRQ_HANDLED;
+			return ret;
 		}
 
 		ade9000_configure_scan(indio_dev, ADE9000_REG_WF_BUFF);
@@ -1123,7 +1119,7 @@ static int ade9000_write_raw(struct iio_dev *indio_dev,
 			tmp &= ~ADE9000_PHASE_C_POS_BIT;
 
 			switch (tmp) {
-			case ADE9000_REG_AWATTOS:
+			case ADE9000_REG_AWATT:
 				return regmap_write(st->regmap,
 						    ADE9000_ADDR_ADJUST(ADE9000_REG_AWATTOS,
 									chan->channel), val);
@@ -1549,7 +1545,7 @@ static int ade9000_buffer_postdisable(struct iio_dev *indio_dev)
 
 	ret = regmap_clear_bits(st->regmap, ADE9000_REG_MASK0, interrupts);
 	if (ret) {
-		dev_err(dev, "Post-disable update maks0 fail\n");
+		dev_err(dev, "Post-disable update mask0 fail\n");
 		return ret;
 	}
 
@@ -1589,10 +1585,9 @@ static int ade9000_reset(struct ade9000_state *st)
 	/* Only wait for completion if IRQ1 is available to signal reset done */
 	if (fwnode_irq_get_byname(dev_fwnode(dev), "irq1") >= 0) {
 		if (!wait_for_completion_timeout(&st->reset_completion,
-						 msecs_to_jiffies(1000))) {
-			dev_err(dev, "Reset timeout after 1s\n");
-			return -ETIMEDOUT;
-		}
+						 msecs_to_jiffies(1000)))
+			return dev_err_probe(dev, -ETIMEDOUT,
+					     "Reset timeout after 1s\n");
 	}
 	/* If no IRQ available, reset is already complete after the 50ms delay above */
 
@@ -1706,19 +1701,19 @@ static int ade9000_probe(struct spi_device *spi)
 
 	init_completion(&st->reset_completion);
 
-	ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev);
+	ret = devm_mutex_init(dev, &st->lock);
 	if (ret)
 		return ret;
 
-	ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev);
+	ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev);
 	if (ret)
 		return ret;
 
-	ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev);
+	ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev);
 	if (ret)
 		return ret;
 
-	ret = devm_mutex_init(dev, &st->lock);
+	ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c
index 5f445e0de9eac..ced0a2321ecfd 100644
--- a/drivers/iio/adc/adi-axi-adc.c
+++ b/drivers/iio/adc/adi-axi-adc.c
@@ -621,6 +621,8 @@ static const struct iio_backend_ops adi_axi_adc_ops = {
 static const struct iio_backend_info adi_axi_adc_generic = {
 	.name = "axi-adc",
 	.ops = &adi_axi_adc_ops,
+	.caps = IIO_BACKEND_CAP_CALIBRATION | IIO_BACKEND_CAP_BUFFER |
+		IIO_BACKEND_CAP_ENABLE,
 };
 
 static const struct iio_backend_ops adi_ad485x_ops = {
@@ -645,6 +647,8 @@ static const struct iio_backend_ops adi_ad485x_ops = {
 static const struct iio_backend_info axi_ad485x = {
 	.name = "axi-ad485x",
 	.ops = &adi_ad485x_ops,
+	.caps = IIO_BACKEND_CAP_CALIBRATION | IIO_BACKEND_CAP_BUFFER |
+		IIO_BACKEND_CAP_ENABLE,
 };
 
 static const struct iio_backend_ops adi_ad408x_ops = {
@@ -665,6 +669,7 @@ static const struct iio_backend_ops adi_ad408x_ops = {
 static const struct iio_backend_info axi_ad408x = {
 	.name = "axi-ad408x",
 	.ops = &adi_ad408x_ops,
+	.caps = IIO_BACKEND_CAP_BUFFER | IIO_BACKEND_CAP_ENABLE,
 };
 
 static int adi_axi_adc_probe(struct platform_device *pdev)
diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c
index 4be44c524b4da..83a9885b9ae4d 100644
--- a/drivers/iio/adc/aspeed_adc.c
+++ b/drivers/iio/adc/aspeed_adc.c
@@ -415,6 +415,7 @@ static int aspeed_adc_vref_config(struct iio_dev *indio_dev)
 	}
 	adc_engine_control_reg_val =
 		readl(data->base + ASPEED_REG_ENGINE_CONTROL);
+	adc_engine_control_reg_val &= ~ASPEED_ADC_REF_VOLTAGE;
 
 	ret = devm_regulator_get_enable_read_voltage(data->dev, "vref");
 	if (ret < 0 && ret != -ENODEV)
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index aa4ba3f5a5060..255970b2e7479 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -2259,7 +2259,7 @@ static int at91_adc_temp_sensor_init(struct at91_adc_state *st,
 		return 0;
 
 	/* Get the calibration data from NVMEM. */
-	temp_calib = devm_nvmem_cell_get(dev, "temperature_calib");
+	temp_calib = nvmem_cell_get(dev, "temperature_calib");
 	if (IS_ERR(temp_calib)) {
 		ret = PTR_ERR(temp_calib);
 		if (ret != -ENOENT)
@@ -2268,6 +2268,7 @@ static int at91_adc_temp_sensor_init(struct at91_adc_state *st,
 	}
 
 	buf = nvmem_cell_read(temp_calib, &len);
+	nvmem_cell_put(temp_calib);
 	if (IS_ERR(buf)) {
 		dev_err(dev, "Failed to read calibration data!\n");
 		return PTR_ERR(buf);
@@ -2507,7 +2508,7 @@ static int at91_adc_suspend(struct device *dev)
 		at91_adc_buffer_postdisable(indio_dev);
 
 	/*
-	 * Do a sofware reset of the ADC before we go to suspend.
+	 * Do a software reset of the ADC before we go to suspend.
 	 * this will ensure that all pins are free from being muxed by the ADC
 	 * and can be used by for other devices.
 	 * Otherwise, ADC will hog them and we can't go to suspend mode.
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index 920dd9ffd27a5..6e1930f7c65d9 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -171,7 +171,7 @@ struct at91_adc_trigger {
 };
 
 /**
- * struct at91_adc_reg_desc - Various informations relative to registers
+ * struct at91_adc_reg_desc - Various information relative to registers
  * @channel_base:	Base offset for the channel data registers
  * @drdy_mask:		Mask of the DRDY field in the relevant registers
  *			(Interruptions registers mostly)
@@ -231,7 +231,7 @@ struct at91_adc_state {
 	struct iio_trigger	**trig;
 	bool			use_external;
 	u32			vref_mv;
-	u32			res;		/* resolution used for convertions */
+	u32			res;		/* resolution used for conversions */
 	wait_queue_head_t	wq_data_avail;
 	const struct at91_adc_caps	*caps;
 
@@ -304,7 +304,7 @@ static void handle_adc_eoc_trigger(int irq, struct iio_dev *idev)
 	}
 }
 
-static int at91_ts_sample(struct iio_dev *idev)
+static void at91_ts_sample(struct iio_dev *idev)
 {
 	struct at91_adc_state *st = iio_priv(idev);
 	unsigned int xscale, yscale, reg, z1, z2;
@@ -323,7 +323,7 @@ static int at91_ts_sample(struct iio_dev *idev)
 	xscale = (reg >> 16) & xyz_mask;
 	if (xscale == 0) {
 		dev_err(&idev->dev, "Error: xscale == 0!\n");
-		return -1;
+		return;
 	}
 	x /= xscale;
 
@@ -334,7 +334,7 @@ static int at91_ts_sample(struct iio_dev *idev)
 	yscale = (reg >> 16) & xyz_mask;
 	if (yscale == 0) {
 		dev_err(&idev->dev, "Error: yscale == 0!\n");
-		return -1;
+		return;
 	}
 	y /= yscale;
 
@@ -363,8 +363,6 @@ static int at91_ts_sample(struct iio_dev *idev)
 	} else {
 		dev_dbg(&idev->dev, "pressure too low: not reporting\n");
 	}
-
-	return 0;
 }
 
 static irqreturn_t at91_adc_rl_interrupt(int irq, void *private)
diff --git a/drivers/iio/adc/cpcap-adc.c b/drivers/iio/adc/cpcap-adc.c
index d9ee2ea116a78..f6f72efcc6eda 100644
--- a/drivers/iio/adc/cpcap-adc.c
+++ b/drivers/iio/adc/cpcap-adc.c
@@ -934,6 +934,17 @@ static const struct cpcap_adc_ato mapphone_adc = {
 	.atox_ps_factor_out = 0,
 };
 
+static const struct cpcap_adc_ato mot_adc = {
+	.ato_in = 0x0300,
+	.atox_in = 0,
+	.adc_ps_factor_in = 0x0200,
+	.atox_ps_factor_in = 0,
+	.ato_out = 0x0780,
+	.atox_out = 0,
+	.adc_ps_factor_out = 0x0600,
+	.atox_ps_factor_out = 0,
+};
+
 static const struct of_device_id cpcap_adc_id_table[] = {
 	{
 		.compatible = "motorola,cpcap-adc",
@@ -942,6 +953,10 @@ static const struct of_device_id cpcap_adc_id_table[] = {
 		.compatible = "motorola,mapphone-cpcap-adc",
 		.data = &mapphone_adc,
 	},
+	{
+		.compatible = "motorola,mot-cpcap-adc",
+		.data = &mot_adc,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, cpcap_adc_id_table);
diff --git a/drivers/iio/adc/fsl-imx25-gcq.c b/drivers/iio/adc/fsl-imx25-gcq.c
index f8c220f6a7b47..e6268f7ac400c 100644
--- a/drivers/iio/adc/fsl-imx25-gcq.c
+++ b/drivers/iio/adc/fsl-imx25-gcq.c
@@ -47,7 +47,7 @@ struct mx25_gcq_priv {
 	 * of register writes, then a wait for a completion callback,
 	 * and finally a register read, during which userspace could issue
 	 * another read request. This lock protects a read access from
-	 * ocurring before another one has finished.
+	 * occurring before another one has finished.
 	 */
 	struct mutex lock;
 };
diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c
index 9dd547e62b6c1..4d0b79cfeb278 100644
--- a/drivers/iio/adc/max1363.c
+++ b/drivers/iio/adc/max1363.c
@@ -121,7 +121,7 @@ enum max1363_modes {
 };
 
 /**
- * struct max1363_chip_info - chip specifc information
+ * struct max1363_chip_info - chip specific information
  * @info:		iio core function callbacks structure
  * @channels:		channel specification
  * @num_channels:       number of channels
@@ -149,7 +149,6 @@ struct max1363_chip_info {
  * @configbyte:		cache of current device config byte
  * @chip_info:		chip model specific constants, available modes, etc.
  * @current_mode:	the scan mode of this chip
- * @requestedmask:	a valid requested set of channels
  * @lock:		lock to ensure state is consistent
  * @monitor_on:		whether monitor mode is enabled
  * @monitor_speed:	parameter corresponding to device monitor speed setting
@@ -169,7 +168,6 @@ struct max1363_state {
 	u8				configbyte;
 	const struct max1363_chip_info	*chip_info;
 	const struct max1363_mode	*current_mode;
-	u32				requestedmask;
 	struct mutex			lock;
 
 	/* Using monitor modes and buffer at the same time is
@@ -637,48 +635,51 @@ static const enum max1363_modes max11644_mode_list[] = {
 static const struct iio_chan_spec max11646_channels[] = MAX1363_2X_CHANS(10);
 static const struct iio_chan_spec max11644_channels[] = MAX1363_2X_CHANS(12);
 
-enum { max1361,
-       max1362,
-       max1363,
-       max1364,
-       max1036,
-       max1037,
-       max1038,
-       max1039,
-       max1136,
-       max1137,
-       max1138,
-       max1139,
-       max1236,
-       max1237,
-       max1238,
-       max1239,
-       max11600,
-       max11601,
-       max11602,
-       max11603,
-       max11604,
-       max11605,
-       max11606,
-       max11607,
-       max11608,
-       max11609,
-       max11610,
-       max11611,
-       max11612,
-       max11613,
-       max11614,
-       max11615,
-       max11616,
-       max11617,
-       max11644,
-       max11645,
-       max11646,
-       max11647
+enum {
+	max1361,
+	max1362,
+	max1363,
+	max1364,
+	max1036,
+	max1037,
+	max1038,
+	max1039,
+	max1136,
+	max1137,
+	max1138,
+	max1139,
+	max1236,
+	max1237,
+	max1238,
+	max1239,
+	max11600,
+	max11601,
+	max11602,
+	max11603,
+	max11604,
+	max11605,
+	max11606,
+	max11607,
+	max11608,
+	max11609,
+	max11610,
+	max11611,
+	max11612,
+	max11613,
+	max11614,
+	max11615,
+	max11616,
+	max11617,
+	max11644,
+	max11645,
+	max11646,
+	max11647,
 };
 
-static const int max1363_monitor_speeds[] = { 133000, 665000, 33300, 16600,
-					      8300, 4200, 2000, 1000 };
+static const int max1363_monitor_speeds[] = {
+	133000, 665000, 33300, 16600,
+	8300, 4200, 2000, 1000,
+};
 
 static ssize_t max1363_monitor_show_freq(struct device *dev,
 					struct device_attribute *attr,
diff --git a/drivers/iio/adc/mcp3564.c b/drivers/iio/adc/mcp3564.c
index fcdf13f49c48f..36675563829ef 100644
--- a/drivers/iio/adc/mcp3564.c
+++ b/drivers/iio/adc/mcp3564.c
@@ -349,7 +349,7 @@ struct mcp3564_chip_info {
  * struct mcp3564_state - working data for a ADC device
  * @chip_info:		chip specific data
  * @spi:		SPI device structure
- * @vref_mv:		voltage reference value in miliVolts
+ * @vref_mv:		voltage reference value in millivolts
  * @lock:		synchronize access to driver's state members
  * @dev_addr:		hardware device address
  * @oversampling:	the index inside oversampling list of the ADC
diff --git a/drivers/iio/adc/men_z188_adc.c b/drivers/iio/adc/men_z188_adc.c
index 90919d282e7b9..5bd334ec56554 100644
--- a/drivers/iio/adc/men_z188_adc.c
+++ b/drivers/iio/adc/men_z188_adc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * MEN 16z188 Analog to Digial Converter
+ * MEN 16z188 Analog to Digital Converter
  *
  * Copyright (C) 2014 MEN Mikroelektronik GmbH (www.men.de)
  * Author: Johannes Thumshirn <johannes.thumshirn@men.de>
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 47cd350498a0d..ed91edf0e391a 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -792,7 +792,7 @@ static int meson_sar_adc_temp_sensor_init(struct iio_dev *indio_dev)
 	size_t read_len;
 	int ret;
 
-	temperature_calib = devm_nvmem_cell_get(dev, "temperature_calib");
+	temperature_calib = nvmem_cell_get(dev, "temperature_calib");
 	if (IS_ERR(temperature_calib)) {
 		ret = PTR_ERR(temperature_calib);
 
@@ -806,13 +806,9 @@ static int meson_sar_adc_temp_sensor_init(struct iio_dev *indio_dev)
 		return dev_err_probe(dev, ret, "failed to get temperature_calib cell\n");
 	}
 
-	priv->tsc_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "amlogic,hhi-sysctrl");
-	if (IS_ERR(priv->tsc_regmap))
-		return dev_err_probe(dev, PTR_ERR(priv->tsc_regmap),
-				     "failed to get amlogic,hhi-sysctrl regmap\n");
-
 	read_len = MESON_SAR_ADC_EFUSE_BYTES;
 	buf = nvmem_cell_read(temperature_calib, &read_len);
+	nvmem_cell_put(temperature_calib);
 	if (IS_ERR(buf))
 		return dev_err_probe(dev, PTR_ERR(buf), "failed to read temperature_calib cell\n");
 	if (read_len != MESON_SAR_ADC_EFUSE_BYTES) {
@@ -820,6 +816,11 @@ static int meson_sar_adc_temp_sensor_init(struct iio_dev *indio_dev)
 		return dev_err_probe(dev, -EINVAL, "invalid read size of temperature_calib cell\n");
 	}
 
+	priv->tsc_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "amlogic,hhi-sysctrl");
+	if (IS_ERR(priv->tsc_regmap))
+		return dev_err_probe(dev, PTR_ERR(priv->tsc_regmap),
+				     "failed to get amlogic,hhi-sysctrl regmap\n");
+
 	trimming_bits = priv->param->temperature_trimming_bits;
 	trimming_mask = BIT(trimming_bits) - 1;
 
diff --git a/drivers/iio/adc/nau7802.c b/drivers/iio/adc/nau7802.c
index 458544cb8ee48..970afb27b8396 100644
--- a/drivers/iio/adc/nau7802.c
+++ b/drivers/iio/adc/nau7802.c
@@ -55,7 +55,6 @@ struct nau7802_state {
 	struct mutex		data_lock;
 	u32			vref_mv;
 	u32			conversion_count;
-	u32			min_conversions;
 	u8			sample_rate;
 	u32			scale_avail[8];
 	struct completion	value_ok;
@@ -257,7 +256,7 @@ static int nau7802_read_poll(struct iio_dev *indio_dev,
 	/*
 	 * Because there is actually only one ADC for both channels, we have to
 	 * wait for enough conversions to happen before getting a significant
-	 * value when changing channels and the values are far appart.
+	 * value when changing channels and the values are far apart.
 	 */
 	do {
 		ret = i2c_smbus_read_byte_data(st->client, NAU7802_REG_PUCTRL);
diff --git a/drivers/iio/adc/npcm_adc.c b/drivers/iio/adc/npcm_adc.c
index c8283873cdee6..ddabb9600d462 100644
--- a/drivers/iio/adc/npcm_adc.c
+++ b/drivers/iio/adc/npcm_adc.c
@@ -38,7 +38,7 @@ struct npcm_adc {
 	 * read access from userspace. Reading a raw value requires a sequence
 	 * of register writes, then a wait for a event and finally a register
 	 * read, during which userspace could issue another read request.
-	 * This lock protects a read access from ocurring before another one
+	 * This lock protects a read access from occurring before another one
 	 * has finished.
 	 */
 	struct mutex lock;
diff --git a/drivers/iio/adc/nxp-sar-adc.c b/drivers/iio/adc/nxp-sar-adc.c
index 9efa883c277d2..a6e4888a84644 100644
--- a/drivers/iio/adc/nxp-sar-adc.c
+++ b/drivers/iio/adc/nxp-sar-adc.c
@@ -247,7 +247,8 @@ static inline void nxp_sar_adc_calibration_start(void __iomem *base)
 
 static inline int nxp_sar_adc_calibration_wait(void __iomem *base)
 {
-	u32 msr, ret;
+	u32 msr;
+	int ret;
 
 	ret = readl_poll_timeout(NXP_SAR_ADC_MSR(base), msr,
 				 !FIELD_GET(NXP_SAR_ADC_MSR_CALBUSY, msr),
diff --git a/drivers/iio/adc/pac1921.c b/drivers/iio/adc/pac1921.c
index a0227b57f2388..bce7185953ecb 100644
--- a/drivers/iio/adc/pac1921.c
+++ b/drivers/iio/adc/pac1921.c
@@ -856,7 +856,7 @@ static ssize_t pac1921_format_scale_avail(const int (*const scales_tbl)[2],
 /*
  * Read available scales for a specific channel
  *
- * NOTE: using extended info insted of iio.read_avail() because access to
+ * NOTE: using extended info instead of iio.read_avail() because access to
  * current scales must be locked as they depend on shunt resistor which may
  * change runtime. Caller of iio.read_avail() would access the table unlocked
  * instead.
diff --git a/drivers/iio/adc/pac1934.c b/drivers/iio/adc/pac1934.c
index 712b5e9caba6d..23055405a6e0c 100644
--- a/drivers/iio/adc/pac1934.c
+++ b/drivers/iio/adc/pac1934.c
@@ -1351,7 +1351,7 @@ static int pac1934_prep_iio_channels(struct pac1934_chip_info *info, struct iio_
 
 	dyn_ch_struct = devm_kzalloc(dev, channel_size, GFP_KERNEL);
 	if (!dyn_ch_struct)
-		return -EINVAL;
+		return -ENOMEM;
 
 	tmp_data = dyn_ch_struct;
 
diff --git a/drivers/iio/adc/palmas_gpadc.c b/drivers/iio/adc/palmas_gpadc.c
index 3f433064618e4..f777986a6aba3 100644
--- a/drivers/iio/adc/palmas_gpadc.c
+++ b/drivers/iio/adc/palmas_gpadc.c
@@ -105,7 +105,7 @@ struct palmas_gpadc_thresholds {
  *		of register writes, then a wait for a completion callback,
  *		and finally a register read, during which userspace could issue
  *		another read request. This lock protects a read access from
- *		ocurring before another one has finished.
+ *		occurring before another one has finished.
  *
  * This is the palmas_gpadc structure to store run-time information
  * and pointers for this driver instance.
@@ -521,16 +521,16 @@ static int palmas_gpadc_get_low_threshold_raw(struct palmas_gpadc *adc,
 
 	val = (val * 1000) / adc->adc_info[adc_chan].gain;
 
-        if (adc->adc_info[adc_chan].is_uncalibrated) {
+	if (adc->adc_info[adc_chan].is_uncalibrated) {
 		/* 2% worse */
 		min_gain_error -= 20;
 		min_offset_error = -36;
-        } else {
+	} else {
 		val = (val * adc->adc_info[adc_chan].gain_error -
 		       adc->adc_info[adc_chan].offset) /
 			1000;
 		min_offset_error = -2;
-        }
+	}
 
 	return palmas_gpadc_threshold_with_tolerance(val,
 						     min_INL,
diff --git a/drivers/iio/adc/qcom-spmi-adc5-gen3.c b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
new file mode 100644
index 0000000000000..f8168a14b9073
--- /dev/null
+++ b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
@@ -0,0 +1,860 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/container_of.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/device/devres.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/iio/adc/qcom-adc5-gen3-common.h>
+#include <linux/iio/iio.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+
+#define ADC5_GEN3_VADC_SDAM			0x0
+
+struct adc5_chip;
+
+/**
+ * struct adc5_channel_prop - ADC channel structure
+ * @common_props: structure with ADC channel properties (common to TM usage).
+ * @adc_tm: indicates TM type if the channel is used for TM measurements.
+ * @chip: pointer to top-level ADC device structure.
+ */
+struct adc5_channel_prop {
+	struct adc5_channel_common_prop common_props;
+	int adc_tm;
+	struct adc5_chip *chip;
+};
+
+/**
+ * struct adc5_chip - ADC private structure.
+ * @dev: SPMI ADC5 Gen3 device.
+ * @dev_data: Top-level ADC device data.
+ * @nchannels: number of ADC channels.
+ * @chan_props: array of ADC channel properties.
+ * @iio_chans: array of IIO channels specification.
+ * @complete: ADC result notification after interrupt is received.
+ * @lock: ADC lock for access to the peripheral, to prevent concurrent
+ *	requests from multiple clients.
+ * @data: software configuration data.
+ * @n_tm_channels: number of ADC channels used for TM measurements.
+ * @handler: TM callback to be called for threshold violation interrupt
+ *	on first SDAM.
+ * @tm_aux: pointer to auxiliary TM device.
+ */
+struct adc5_chip {
+	struct device *dev;
+	struct adc5_device_data dev_data;
+	unsigned int nchannels;
+	struct adc5_channel_prop *chan_props;
+	struct iio_chan_spec *iio_chans;
+	struct completion complete;
+	struct mutex lock;
+	const struct adc5_data *data;
+	unsigned int n_tm_channels;
+	void (*handler)(struct auxiliary_device *tm_aux);
+	struct auxiliary_device *tm_aux;
+};
+
+int adc5_gen3_read(struct adc5_device_data *adc, unsigned int sdam_index,
+		   u16 offset, u8 *data, int len)
+{
+	return regmap_bulk_read(adc->regmap,
+				adc->base[sdam_index].base_addr + offset,
+				data, len);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_read, "QCOM_SPMI_ADC5_GEN3");
+
+int adc5_gen3_write(struct adc5_device_data *adc, unsigned int sdam_index,
+		    u16 offset, u8 *data, int len)
+{
+	return regmap_bulk_write(adc->regmap,
+				 adc->base[sdam_index].base_addr + offset,
+				 data, len);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_write, "QCOM_SPMI_ADC5_GEN3");
+
+static int adc5_gen3_read_voltage_data(struct adc5_chip *adc, u16 *data)
+{
+	u8 rslt[2];
+	int ret;
+
+	ret = adc5_gen3_read(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+			     ADC5_GEN3_CH_DATA0(0), rslt, sizeof(rslt));
+	if (ret)
+		return ret;
+
+	*data = get_unaligned_le16(rslt);
+
+	if (*data == ADC5_USR_DATA_CHECK) {
+		dev_err(adc->dev, "Invalid data:%#x\n", *data);
+		return -EINVAL;
+	}
+
+	dev_dbg(adc->dev, "voltage raw code:%#x\n", *data);
+
+	return 0;
+}
+
+void adc5_gen3_update_dig_param(struct adc5_channel_common_prop *prop, u8 *data)
+{
+	/* Update calibration select and decimation ratio select */
+	*data &= ~(ADC5_GEN3_DIG_PARAM_CAL_SEL_MASK | ADC5_GEN3_DIG_PARAM_DEC_RATIO_SEL_MASK);
+	*data |= FIELD_PREP(ADC5_GEN3_DIG_PARAM_CAL_SEL_MASK, prop->cal_method);
+	*data |= FIELD_PREP(ADC5_GEN3_DIG_PARAM_DEC_RATIO_SEL_MASK, prop->decimation);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_update_dig_param, "QCOM_SPMI_ADC5_GEN3");
+
+#define ADC5_GEN3_READ_CONFIG_REGS 7
+
+static int adc5_gen3_configure(struct adc5_chip *adc,
+			       struct adc5_channel_common_prop *prop)
+{
+	u8 buf[ADC5_GEN3_READ_CONFIG_REGS];
+	u8 conv_req = 0;
+	int ret;
+
+	ret = adc5_gen3_read(&adc->dev_data, ADC5_GEN3_VADC_SDAM, ADC5_GEN3_SID,
+			     buf, sizeof(buf));
+	if (ret)
+		return ret;
+
+	/* Write SID */
+	buf[0] = FIELD_PREP(ADC5_GEN3_SID_MASK, prop->sid);
+
+	/*
+	 * Use channel 0 by default for immediate conversion and to indicate
+	 * there is an actual conversion request
+	 */
+	buf[1] = ADC5_GEN3_CHAN_CONV_REQ | 0;
+
+	buf[2] = ADC5_GEN3_TIME_IMMEDIATE;
+
+	/* Digital param selection */
+	adc5_gen3_update_dig_param(prop, &buf[3]);
+
+	/* Update fast average sample value */
+	buf[4] = FIELD_PREP(ADC5_GEN3_FAST_AVG_CTL_SAMPLES_MASK,
+			    prop->avg_samples) | ADC5_GEN3_FAST_AVG_CTL_EN;
+
+	/* Select ADC channel */
+	buf[5] = prop->channel;
+
+	/* Select HW settle delay for channel */
+	buf[6] = FIELD_PREP(ADC5_GEN3_HW_SETTLE_DELAY_MASK,
+			    prop->hw_settle_time_us);
+
+	reinit_completion(&adc->complete);
+
+	ret = adc5_gen3_write(&adc->dev_data, ADC5_GEN3_VADC_SDAM, ADC5_GEN3_SID,
+			      buf, sizeof(buf));
+	if (ret)
+		return ret;
+
+	conv_req = ADC5_GEN3_CONV_REQ_REQ;
+	return adc5_gen3_write(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+			       ADC5_GEN3_CONV_REQ, &conv_req, sizeof(conv_req));
+}
+
+/*
+ * Worst case delay from PBS in readying handshake bit  can be up to 15ms, when
+ * PBS is busy running other simultaneous transactions, while in the best case,
+ * it is already ready at this point. Assigning polling delay and retry count
+ * accordingly.
+ */
+
+#define ADC5_GEN3_HS_DELAY_US			100
+#define ADC5_GEN3_HS_RETRY_COUNT		150
+
+int adc5_gen3_poll_wait_hs(struct adc5_device_data *adc,
+			   unsigned int sdam_index)
+{
+	u8 conv_req = ADC5_GEN3_CONV_REQ_REQ;
+	int ret, count;
+	u8 status = 0;
+
+	for (count = 0; count < ADC5_GEN3_HS_RETRY_COUNT; count++) {
+		ret = adc5_gen3_read(adc, sdam_index, ADC5_GEN3_HS, &status, sizeof(status));
+		if (ret)
+			return ret;
+
+		if (status == ADC5_GEN3_HS_READY) {
+			ret = adc5_gen3_read(adc, sdam_index, ADC5_GEN3_CONV_REQ,
+					     &conv_req, sizeof(conv_req));
+			if (ret)
+				return ret;
+
+			if (!conv_req)
+				return 0;
+		}
+
+		fsleep(ADC5_GEN3_HS_DELAY_US);
+	}
+
+	pr_err("Setting HS ready bit timed out, sdam_index:%d, status:%#x\n",
+	       sdam_index, status);
+	return -ETIMEDOUT;
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_poll_wait_hs, "QCOM_SPMI_ADC5_GEN3");
+
+int adc5_gen3_status_clear(struct adc5_device_data *adc,
+			   int sdam_index, u16 offset, u8 *val, int len)
+{
+	u8 value;
+	int ret;
+
+	ret = adc5_gen3_write(adc, sdam_index, offset, val, len);
+	if (ret)
+		return ret;
+
+	/* To indicate conversion request is only to clear a status */
+	value = 0;
+	ret = adc5_gen3_write(adc, sdam_index, ADC5_GEN3_PERPH_CH, &value,
+			      sizeof(value));
+	if (ret)
+		return ret;
+
+	value = ADC5_GEN3_CONV_REQ_REQ;
+	return adc5_gen3_write(adc, sdam_index, ADC5_GEN3_CONV_REQ, &value,
+			      sizeof(value));
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_status_clear, "QCOM_SPMI_ADC5_GEN3");
+
+/*
+ * Worst case delay from PBS for conversion time can be up to 500ms, when PBS
+ * has timed out twice, once for the initial attempt and once for a retry of
+ * the same transaction.
+ */
+
+#define ADC5_GEN3_CONV_TIMEOUT_MS	501
+
+static int adc5_gen3_do_conversion(struct adc5_chip *adc,
+				   struct adc5_channel_common_prop *prop,
+				   u16 *data_volt)
+{
+	unsigned long rc;
+	int ret;
+	u8 val;
+
+	guard(mutex)(&adc->lock);
+	ret = adc5_gen3_poll_wait_hs(&adc->dev_data, ADC5_GEN3_VADC_SDAM);
+	if (ret)
+		return ret;
+
+	ret = adc5_gen3_configure(adc, prop);
+	if (ret) {
+		dev_err(adc->dev, "ADC configure failed with %d\n", ret);
+		return ret;
+	}
+
+	/* No support for polling mode at present */
+	rc = wait_for_completion_timeout(&adc->complete,
+					 msecs_to_jiffies(ADC5_GEN3_CONV_TIMEOUT_MS));
+	if (!rc) {
+		dev_err(adc->dev, "Reading ADC channel %s timed out\n",
+			prop->label);
+		return -ETIMEDOUT;
+	}
+
+	ret = adc5_gen3_read_voltage_data(adc, data_volt);
+	if (ret)
+		return ret;
+
+	val = BIT(0);
+	return adc5_gen3_status_clear(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+				      ADC5_GEN3_EOC_CLR, &val, 1);
+}
+
+static irqreturn_t adc5_gen3_isr(int irq, void *dev_id)
+{
+	struct adc5_chip *adc = dev_id;
+	struct device *dev = adc->dev;
+	struct auxiliary_device *adev;
+	u8 status, eoc_status, val;
+	u8 tm_status[2];
+	int ret;
+
+	ret = adc5_gen3_read(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+			     ADC5_GEN3_STATUS1, &status, sizeof(status));
+	if (ret) {
+		dev_err(dev, "adc read status1 failed with %d\n", ret);
+		return IRQ_HANDLED;
+	}
+
+	ret = adc5_gen3_read(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+			     ADC5_GEN3_EOC_STS, &eoc_status, sizeof(eoc_status));
+	if (ret) {
+		dev_err(dev, "adc read eoc status failed with %d\n", ret);
+		return IRQ_HANDLED;
+	}
+
+	if (status & ADC5_GEN3_STATUS1_CONV_FAULT) {
+		dev_err_ratelimited(dev,
+				    "Unexpected conversion fault, status:%#x, eoc_status:%#x\n",
+				    status, eoc_status);
+		val = ADC5_GEN3_CONV_ERR_CLR_REQ;
+		adc5_gen3_status_clear(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+				       ADC5_GEN3_CONV_ERR_CLR, &val, 1);
+		return IRQ_HANDLED;
+	}
+
+	/* CHAN0 is the preconfigured channel for immediate conversion */
+	if (eoc_status & ADC5_GEN3_EOC_CHAN_0)
+		complete(&adc->complete);
+
+	ret = adc5_gen3_read(&adc->dev_data, ADC5_GEN3_VADC_SDAM,
+			     ADC5_GEN3_TM_HIGH_STS, tm_status, sizeof(tm_status));
+	if (ret) {
+		dev_err(dev, "adc read TM status failed with %d\n", ret);
+		return IRQ_HANDLED;
+	}
+
+	dev_dbg(dev, "Interrupt status:%#x, EOC status:%#x, high:%#x, low:%#x\n",
+		status, eoc_status, tm_status[0], tm_status[1]);
+
+	if (tm_status[0] || tm_status[1]) {
+		adev = adc->tm_aux;
+		if (!adev || !adev->dev.driver) {
+			dev_err(dev, "adc_tm auxiliary device not initialized\n");
+			return IRQ_HANDLED;
+		}
+
+		adc->handler(adev);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int adc5_gen3_fwnode_xlate(struct iio_dev *indio_dev,
+				  const struct fwnode_reference_args *iiospec)
+{
+	struct adc5_chip *adc = iio_priv(indio_dev);
+	int i, v_channel;
+
+	for (i = 0; i < adc->nchannels; i++) {
+		v_channel = ADC5_GEN3_V_CHAN(adc->chan_props[i].common_props);
+		if (v_channel == iiospec->args[0])
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+static int adc5_gen3_read_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan, int *val,
+			      int *val2, long mask)
+{
+	struct adc5_chip *adc = iio_priv(indio_dev);
+	struct adc5_channel_common_prop *prop;
+	u16 adc_code_volt;
+	int ret;
+
+	prop = &adc->chan_props[chan->address].common_props;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_PROCESSED:
+		ret = adc5_gen3_do_conversion(adc, prop, &adc_code_volt);
+		if (ret)
+			return ret;
+
+		ret = qcom_adc5_hw_scale(prop->scale_fn_type, prop->prescale,
+					 adc->data, adc_code_volt, val);
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adc5_gen3_read_label(struct iio_dev *indio_dev,
+				const struct iio_chan_spec *chan, char *label)
+{
+	struct adc5_chip *adc = iio_priv(indio_dev);
+	struct adc5_channel_prop *prop;
+
+	prop = &adc->chan_props[chan->address];
+	return sprintf(label, "%s\n", prop->common_props.label);
+}
+
+static const struct iio_info adc5_gen3_info = {
+	.read_raw = adc5_gen3_read_raw,
+	.read_label = adc5_gen3_read_label,
+	.fwnode_xlate = adc5_gen3_fwnode_xlate,
+};
+
+struct adc5_channels {
+	unsigned int prescale_index;
+	enum iio_chan_type type;
+	long info_mask;
+	enum vadc_scale_fn_type scale_fn_type;
+};
+
+/* In these definitions, _pre refers to an index into adc5_prescale_ratios. */
+#define ADC5_CHAN(_type, _mask, _pre, _scale)	\
+	{						\
+		.prescale_index = _pre,			\
+		.type = _type,				\
+		.info_mask = _mask,			\
+		.scale_fn_type = _scale,		\
+	},						\
+
+#define ADC5_CHAN_TEMP(_pre, _scale)		\
+	ADC5_CHAN(IIO_TEMP, BIT(IIO_CHAN_INFO_PROCESSED), _pre, _scale)	\
+
+#define ADC5_CHAN_VOLT(_pre, _scale)		\
+	ADC5_CHAN(IIO_VOLTAGE, BIT(IIO_CHAN_INFO_PROCESSED), _pre, _scale)	\
+
+#define ADC5_CHAN_CUR(_pre, _scale)		\
+	ADC5_CHAN(IIO_CURRENT, BIT(IIO_CHAN_INFO_PROCESSED), _pre, _scale)	\
+
+static const struct adc5_channels adc5_gen3_chans_pmic[ADC5_MAX_CHANNEL] = {
+	[ADC5_GEN3_REF_GND]		= ADC5_CHAN_VOLT(0, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_1P25VREF]		= ADC5_CHAN_VOLT(0, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_VPH_PWR]		= ADC5_CHAN_VOLT(1, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_VBAT_SNS_QBG]	= ADC5_CHAN_VOLT(1, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_USB_SNS_V_16]	= ADC5_CHAN_TEMP(8, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_VIN_DIV16_MUX]	= ADC5_CHAN_TEMP(8, SCALE_HW_CALIB_DEFAULT)
+	[ADC5_GEN3_DIE_TEMP]		= ADC5_CHAN_TEMP(0,
+						SCALE_HW_CALIB_PMIC_THERM_PM7)
+	[ADC5_GEN3_AMUX1_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX2_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX3_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX4_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX5_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX6_THM_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX1_GPIO_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX2_GPIO_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX3_GPIO_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+	[ADC5_GEN3_AMUX4_GPIO_100K_PU]	= ADC5_CHAN_TEMP(0,
+					SCALE_HW_CALIB_THERM_100K_PU_PM7)
+};
+
+static int adc5_gen3_get_fw_channel_data(struct adc5_chip *adc,
+					 struct adc5_channel_prop *prop,
+					 struct fwnode_handle *fwnode)
+{
+	const char *name = fwnode_get_name(fwnode);
+	const struct adc5_data *data = adc->data;
+	struct device *dev = adc->dev;
+	const char *channel_name;
+	u32 chan, value, sid;
+	u32 varr[2];
+	int ret;
+
+	ret = fwnode_property_read_u32(fwnode, "reg", &chan);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "invalid channel number %s\n",
+				     name);
+
+	/*
+	 * Value read from "reg" is virtual channel number
+	 * virtual channel number = sid << 8 | channel number
+	 */
+	sid = FIELD_GET(ADC5_GEN3_VIRTUAL_SID_MASK, chan);
+	chan = FIELD_GET(ADC5_GEN3_CHANNEL_MASK, chan);
+
+	if (chan > ADC5_MAX_CHANNEL)
+		return dev_err_probe(dev, -EINVAL,
+				     "%s invalid channel number %d\n",
+				     name, chan);
+
+	prop->common_props.channel = chan;
+	prop->common_props.sid = sid;
+
+	if (!adc->data->adc_chans[chan].info_mask)
+		return dev_err_probe(dev, -EINVAL, "Channel %#x not supported\n", chan);
+
+	channel_name = name;
+	fwnode_property_read_string(fwnode, "label", &channel_name);
+	prop->common_props.label = channel_name;
+
+	value = data->decimation[ADC5_DECIMATION_DEFAULT];
+	fwnode_property_read_u32(fwnode, "qcom,decimation", &value);
+	ret = qcom_adc5_decimation_from_dt(value, data->decimation);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "%#x invalid decimation %d\n",
+				     chan, value);
+	prop->common_props.decimation = ret;
+
+	prop->common_props.prescale = adc->data->adc_chans[chan].prescale_index;
+	ret = fwnode_property_read_u32_array(fwnode, "qcom,pre-scaling", varr, 2);
+	if (!ret) {
+		ret = qcom_adc5_prescaling_from_dt(varr[0], varr[1]);
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "%#x invalid pre-scaling <%d %d>\n",
+					     chan, varr[0], varr[1]);
+		prop->common_props.prescale = ret;
+	}
+
+	value = data->hw_settle_1[VADC_DEF_HW_SETTLE_TIME];
+	fwnode_property_read_u32(fwnode, "qcom,hw-settle-time", &value);
+	ret = qcom_adc5_hw_settle_time_from_dt(value, data->hw_settle_1);
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "%#x invalid hw-settle-time %d us\n",
+				     chan, value);
+	prop->common_props.hw_settle_time_us = ret;
+
+	value = BIT(VADC_DEF_AVG_SAMPLES);
+	fwnode_property_read_u32(fwnode, "qcom,avg-samples", &value);
+	ret = qcom_adc5_avg_samples_from_dt(value);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "%#x invalid avg-samples %d\n",
+				     chan, value);
+	prop->common_props.avg_samples = ret;
+
+	if (fwnode_property_read_bool(fwnode, "qcom,ratiometric"))
+		prop->common_props.cal_method = ADC5_RATIOMETRIC_CAL;
+	else
+		prop->common_props.cal_method = ADC5_ABSOLUTE_CAL;
+
+	prop->adc_tm = fwnode_property_read_bool(fwnode, "qcom,adc-tm");
+	if (prop->adc_tm) {
+		adc->n_tm_channels++;
+		if (adc->n_tm_channels > (adc->dev_data.num_sdams * 8 - 1))
+			return dev_err_probe(dev, -EINVAL,
+					     "Number of TM nodes %u greater than channels supported:%u\n",
+					     adc->n_tm_channels,
+					     adc->dev_data.num_sdams * 8 - 1);
+	}
+
+	return 0;
+}
+
+static const struct adc5_data adc5_gen3_data_pmic = {
+	.full_scale_code_volt = 0x70e4,
+	.adc_chans = adc5_gen3_chans_pmic,
+	.info = &adc5_gen3_info,
+	.decimation = (unsigned int [ADC5_DECIMATION_SAMPLES_MAX])
+			   { 85, 340, 1360 },
+	.hw_settle_1 = (unsigned int [VADC_HW_SETTLE_SAMPLES_MAX])
+			   { 15, 100, 200, 300,
+			     400, 500, 600, 700,
+			     1000, 2000, 4000, 8000,
+			     16000, 32000, 64000, 128000 },
+};
+
+static const struct of_device_id adc5_match_table[] = {
+	{
+		.compatible = "qcom,spmi-adc5-gen3",
+		.data = &adc5_gen3_data_pmic,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adc5_match_table);
+
+static int adc5_get_fw_data(struct adc5_chip *adc)
+{
+	const struct adc5_channels *adc_chan;
+	struct adc5_channel_prop *chan_props;
+	struct iio_chan_spec *iio_chan;
+	struct device *dev = adc->dev;
+	unsigned int index = 0;
+	int ret;
+
+	adc->nchannels = device_get_child_node_count(dev);
+	if (!adc->nchannels)
+		return dev_err_probe(dev, -EINVAL, "No ADC channels found\n");
+
+	adc->iio_chans = devm_kcalloc(dev, adc->nchannels,
+				      sizeof(*adc->iio_chans), GFP_KERNEL);
+	if (!adc->iio_chans)
+		return -ENOMEM;
+
+	adc->chan_props = devm_kcalloc(dev, adc->nchannels,
+				       sizeof(*adc->chan_props), GFP_KERNEL);
+	if (!adc->chan_props)
+		return -ENOMEM;
+
+	chan_props = adc->chan_props;
+	adc->n_tm_channels = 0;
+	iio_chan = adc->iio_chans;
+	adc->data = device_get_match_data(dev);
+
+	device_for_each_child_node_scoped(dev, child) {
+		ret = adc5_gen3_get_fw_channel_data(adc, chan_props, child);
+		if (ret)
+			return ret;
+
+		chan_props->chip = adc;
+		adc_chan = &adc->data->adc_chans[chan_props->common_props.channel];
+		chan_props->common_props.scale_fn_type = adc_chan->scale_fn_type;
+
+		iio_chan->channel = ADC5_GEN3_V_CHAN(chan_props->common_props);
+		iio_chan->info_mask_separate = adc_chan->info_mask;
+		iio_chan->type = adc_chan->type;
+		iio_chan->address = index;
+		iio_chan->indexed = 1;
+		iio_chan++;
+		chan_props++;
+		index++;
+	}
+
+	return 0;
+}
+
+static void adc5_gen3_uninit_aux(void *data)
+{
+	auxiliary_device_uninit(data);
+}
+
+static void adc5_gen3_delete_aux(void *data)
+{
+	auxiliary_device_delete(data);
+}
+
+static void adc5_gen3_aux_device_release(struct device *dev) {}
+
+static int adc5_gen3_add_aux_tm_device(struct adc5_chip *adc)
+{
+	struct tm5_aux_dev_wrapper *aux_device;
+	int i, ret, i_tm = 0;
+
+	aux_device = devm_kzalloc(adc->dev, sizeof(*aux_device), GFP_KERNEL);
+	if (!aux_device)
+		return -ENOMEM;
+
+	aux_device->aux_dev.name = "adc5_tm_gen3";
+	aux_device->aux_dev.dev.parent = adc->dev;
+	aux_device->aux_dev.dev.release = adc5_gen3_aux_device_release;
+
+	aux_device->tm_props = devm_kcalloc(adc->dev, adc->n_tm_channels,
+					    sizeof(*aux_device->tm_props),
+					    GFP_KERNEL);
+	if (!aux_device->tm_props)
+		return -ENOMEM;
+
+	aux_device->dev_data = &adc->dev_data;
+
+	for (i = 0; i < adc->nchannels; i++) {
+		if (!adc->chan_props[i].adc_tm)
+			continue;
+		aux_device->tm_props[i_tm] = adc->chan_props[i].common_props;
+		i_tm++;
+	}
+
+	device_set_of_node_from_dev(&aux_device->aux_dev.dev, adc->dev);
+
+	aux_device->n_tm_channels = adc->n_tm_channels;
+
+	ret = auxiliary_device_init(&aux_device->aux_dev);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(adc->dev, adc5_gen3_uninit_aux,
+				       &aux_device->aux_dev);
+	if (ret)
+		return ret;
+
+	ret = auxiliary_device_add(&aux_device->aux_dev);
+	if (ret)
+		return ret;
+	ret = devm_add_action_or_reset(adc->dev, adc5_gen3_delete_aux,
+				       &aux_device->aux_dev);
+	if (ret)
+		return ret;
+
+	adc->tm_aux = &aux_device->aux_dev;
+
+	return 0;
+}
+
+void adc5_gen3_mutex_lock(struct device *dev)
+	__acquires(&adc->lock)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev->parent);
+	struct adc5_chip *adc = iio_priv(indio_dev);
+
+	mutex_lock(&adc->lock);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_mutex_lock, "QCOM_SPMI_ADC5_GEN3");
+
+void adc5_gen3_mutex_unlock(struct device *dev)
+	__releases(&adc->lock)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev->parent);
+	struct adc5_chip *adc = iio_priv(indio_dev);
+
+	mutex_unlock(&adc->lock);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_mutex_unlock, "QCOM_SPMI_ADC5_GEN3");
+
+int adc5_gen3_get_scaled_reading(struct device *dev,
+				 struct adc5_channel_common_prop *common_props,
+				 int *val)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev->parent);
+	struct adc5_chip *adc = iio_priv(indio_dev);
+	u16 adc_code_volt;
+	int ret;
+
+	ret = adc5_gen3_do_conversion(adc, common_props, &adc_code_volt);
+	if (ret)
+		return ret;
+
+	return qcom_adc5_hw_scale(common_props->scale_fn_type,
+				  common_props->prescale,
+				  adc->data, adc_code_volt, val);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_get_scaled_reading, "QCOM_SPMI_ADC5_GEN3");
+
+int adc5_gen3_therm_code_to_temp(struct device *dev,
+				 struct adc5_channel_common_prop *common_props,
+				 u16 code, int *val)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev->parent);
+	struct adc5_chip *adc = iio_priv(indio_dev);
+
+	return qcom_adc5_hw_scale(common_props->scale_fn_type,
+				  common_props->prescale,
+				  adc->data, code, val);
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_therm_code_to_temp, "QCOM_SPMI_ADC5_GEN3");
+
+void adc5_gen3_register_tm_event_notifier(struct device *dev,
+					  void (*handler)(struct auxiliary_device *))
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev->parent);
+	struct adc5_chip *adc = iio_priv(indio_dev);
+
+	adc->handler = handler;
+}
+EXPORT_SYMBOL_NS_GPL(adc5_gen3_register_tm_event_notifier, "QCOM_SPMI_ADC5_GEN3");
+
+static int adc5_gen3_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *indio_dev;
+	struct adc5_chip *adc;
+	struct regmap *regmap;
+	int ret, i;
+	u32 *reg;
+
+	regmap = dev_get_regmap(dev->parent, NULL);
+	if (!regmap)
+		return -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*adc));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	adc = iio_priv(indio_dev);
+	adc->dev_data.regmap = regmap;
+	adc->dev = dev;
+
+	ret = device_property_count_u32(dev, "reg");
+	if (ret < 0)
+		return ret;
+
+	adc->dev_data.num_sdams = ret;
+
+	reg = devm_kcalloc(dev, adc->dev_data.num_sdams, sizeof(u32),
+			   GFP_KERNEL);
+	if (!reg)
+		return -ENOMEM;
+
+	ret = device_property_read_u32_array(dev, "reg", reg,
+					     adc->dev_data.num_sdams);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to read reg property\n");
+
+	adc->dev_data.base = devm_kcalloc(dev, adc->dev_data.num_sdams,
+					  sizeof(*adc->dev_data.base),
+					  GFP_KERNEL);
+	if (!adc->dev_data.base)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, indio_dev);
+	init_completion(&adc->complete);
+	ret = devm_mutex_init(dev, &adc->lock);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < adc->dev_data.num_sdams; i++) {
+		adc->dev_data.base[i].base_addr = reg[i];
+
+		ret = platform_get_irq(pdev, i);
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "Getting IRQ %d failed\n", i);
+
+		adc->dev_data.base[i].irq = ret;
+
+		adc->dev_data.base[i].irq_name = devm_kasprintf(dev, GFP_KERNEL,
+								"sdam%d", i);
+		if (!adc->dev_data.base[i].irq_name)
+			return -ENOMEM;
+	}
+
+	ret = devm_request_irq(dev, adc->dev_data.base[ADC5_GEN3_VADC_SDAM].irq,
+			       adc5_gen3_isr, 0,
+			       adc->dev_data.base[ADC5_GEN3_VADC_SDAM].irq_name,
+			       adc);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to request SDAM%d irq\n",
+				     ADC5_GEN3_VADC_SDAM);
+
+	ret = adc5_get_fw_data(adc);
+	if (ret)
+		return ret;
+
+	if (adc->n_tm_channels > 0) {
+		ret = adc5_gen3_add_aux_tm_device(adc);
+		if (ret)
+			dev_err_probe(dev, ret,
+				      "Failed to add auxiliary TM device\n");
+	}
+
+	indio_dev->name = "spmi-adc5-gen3";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->info = &adc5_gen3_info;
+	indio_dev->channels = adc->iio_chans;
+	indio_dev->num_channels = adc->nchannels;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static struct platform_driver adc5_gen3_driver = {
+	.driver = {
+		.name = "qcom-spmi-adc5-gen3",
+		.of_match_table = adc5_match_table,
+	},
+	.probe = adc5_gen3_probe,
+};
+module_platform_driver(adc5_gen3_driver);
+
+MODULE_DESCRIPTION("Qualcomm Technologies Inc. PMIC5 Gen3 ADC driver");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("QCOM_SPMI_ADC5_GEN3");
diff --git a/drivers/iio/adc/rohm-bd79124.c b/drivers/iio/adc/rohm-bd79124.c
index fc0452749b792..40d00bd0cc9dd 100644
--- a/drivers/iio/adc/rohm-bd79124.c
+++ b/drivers/iio/adc/rohm-bd79124.c
@@ -75,7 +75,7 @@
 
 /*
  * The high limit, low limit and last measurement result are each stored in
- * 2 consequtive registers. 4 bits are in the high bits of the first register
+ * 2 consecutive registers. 4 bits are in the high bits of the first register
  * and 8 bits in the next register.
  *
  * These macros return the address of the first reg for the given channel.
@@ -962,7 +962,7 @@ static int bd79124_hw_init(struct bd79124_data *data)
 	if (ret)
 		return ret;
 
-	/* Enable writing the measured values to the regsters */
+	/* Enable writing the measured values to the registers */
 	ret = regmap_set_bits(data->map, BD79124_REG_GEN_CFG,
 			      BD79124_MSK_STATS_EN);
 	if (ret)
diff --git a/drivers/iio/adc/sd_adc_modulator.c b/drivers/iio/adc/sd_adc_modulator.c
index 9f7a75168aac3..218117c45ec88 100644
--- a/drivers/iio/adc/sd_adc_modulator.c
+++ b/drivers/iio/adc/sd_adc_modulator.c
@@ -77,6 +77,7 @@ static const struct iio_backend_ops sd_backend_ops = {
 static const struct iio_backend_info sd_backend_info = {
 	.name = "sd-modulator",
 	.ops = &sd_backend_ops,
+	.caps = IIO_BACKEND_CAP_ENABLE,
 };
 
 static int iio_sd_mod_register(struct platform_device *pdev)
diff --git a/drivers/iio/adc/spear_adc.c b/drivers/iio/adc/spear_adc.c
index 50b0a607baeb7..91995489bb1c6 100644
--- a/drivers/iio/adc/spear_adc.c
+++ b/drivers/iio/adc/spear_adc.c
@@ -82,7 +82,7 @@ struct spear_adc_state {
 	 * of register writes, then a wait for a completion callback,
 	 * and finally a register read, during which userspace could issue
 	 * another read request. This lock protects a read access from
-	 * ocurring before another one has finished.
+	 * occurring before another one has finished.
 	 */
 	struct mutex lock;
 	u32 current_clk;
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
index e39a4c0db25ea..a42d82d61cb85 100644
--- a/drivers/iio/adc/stm32-adc-core.c
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -227,7 +227,7 @@ static int stm32h7_adc_clk_sel(struct platform_device *pdev,
 	if (priv->aclk) {
 		/*
 		 * Asynchronous clock modes (e.g. ckmode == 0)
-		 * From spec: PLL output musn't exceed max rate
+		 * From spec: PLL output mustn't exceed max rate
 		 */
 		rate = clk_get_rate(priv->aclk);
 		if (!rate) {
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index 2d7f88459c7ce..46106200bb862 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -1662,7 +1662,7 @@ static irqreturn_t stm32_adc_threaded_isr(int irq, void *data)
 		/*
 		 * Clear ovr bit to avoid subsequent calls to IRQ handler.
 		 * This requires to stop ADC first. OVR bit state in ISR,
-		 * is propaged to CSR register by hardware.
+		 * is propagated to CSR register by hardware.
 		 */
 		adc->cfg->stop_conv(indio_dev);
 		stm32_adc_irq_clear(indio_dev, regs->isr_ovr.mask);
diff --git a/drivers/iio/adc/sun20i-gpadc-iio.c b/drivers/iio/adc/sun20i-gpadc-iio.c
index e4dfe76e63626..861c14da75add 100644
--- a/drivers/iio/adc/sun20i-gpadc-iio.c
+++ b/drivers/iio/adc/sun20i-gpadc-iio.c
@@ -55,7 +55,7 @@ struct sun20i_gpadc_iio {
 	 * of register writes, then a wait for a completion callback,
 	 * and finally a register read, during which userspace could issue
 	 * another read request. This lock protects a read access from
-	 * ocurring before another one has finished.
+	 * occurring before another one has finished.
 	 */
 	struct mutex		lock;
 };
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index f2a93c63ca14e..c7ffe47449e21 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -231,7 +231,6 @@ static const struct iio_event_spec ads1015_events[] = {
 }
 
 struct ads1015_channel_data {
-	bool enabled;
 	unsigned int pga;
 	unsigned int data_rate;
 };
diff --git a/drivers/iio/adc/ti-ads1018.c b/drivers/iio/adc/ti-ads1018.c
index 6246b3cab71fd..0780abd0d0db4 100644
--- a/drivers/iio/adc/ti-ads1018.c
+++ b/drivers/iio/adc/ti-ads1018.c
@@ -249,7 +249,7 @@ static int ads1018_single_shot(struct ads1018 *ads1018,
 			       struct iio_chan_spec const *chan, u16 *cnv)
 {
 	u8 max_drate_mode = ads1018->chip_info->num_data_rate_mode_to_hz - 1;
-	u8 drate = ads1018->chip_info->data_rate_mode_to_hz[max_drate_mode];
+	u32 drate = ads1018->chip_info->data_rate_mode_to_hz[max_drate_mode];
 	u8 pga_mode = ads1018->chan_data[chan->scan_index].pga_mode;
 	struct spi_transfer xfer[2] = {
 		{
diff --git a/drivers/iio/adc/ti-ads1119.c b/drivers/iio/adc/ti-ads1119.c
index c9cedc59cdcdb..2320be0efb508 100644
--- a/drivers/iio/adc/ti-ads1119.c
+++ b/drivers/iio/adc/ti-ads1119.c
@@ -274,12 +274,15 @@ static int ads1119_single_conversion(struct ads1119_state *st,
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret)
-		goto pdown;
+		return ret;
 
 	ret = ads1119_configure_channel(st, mux, gain, datarate);
 	if (ret)
 		goto pdown;
 
+	if (st->client->irq)
+		reinit_completion(&st->completion);
+
 	ret = i2c_smbus_write_byte(st->client, ADS1119_CMD_START_SYNC);
 	if (ret)
 		goto pdown;
@@ -735,13 +738,10 @@ static int ads1119_probe(struct i2c_client *client)
 		return dev_err_probe(dev, ret, "Failed to setup IIO buffer\n");
 
 	if (client->irq > 0) {
-		ret = devm_request_threaded_irq(dev, client->irq,
-						ads1119_irq_handler,
-						NULL, IRQF_ONESHOT,
-						"ads1119", indio_dev);
+		ret = devm_request_irq(dev, client->irq, ads1119_irq_handler,
+				       IRQF_NO_THREAD, "ads1119", indio_dev);
 		if (ret)
-			return dev_err_probe(dev, ret,
-					     "Failed to allocate irq\n");
+			return ret;
 
 		st->trig = devm_iio_trigger_alloc(dev, "%s-dev%d",
 						  indio_dev->name,
diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c
index bbe1ce5777895..a1ce6daabf266 100644
--- a/drivers/iio/adc/ti-ads7950.c
+++ b/drivers/iio/adc/ti-ads7950.c
@@ -118,21 +118,6 @@ struct ti_ads7950_chip_info {
 	unsigned int num_channels;
 };
 
-enum ti_ads7950_id {
-	TI_ADS7950,
-	TI_ADS7951,
-	TI_ADS7952,
-	TI_ADS7953,
-	TI_ADS7954,
-	TI_ADS7955,
-	TI_ADS7956,
-	TI_ADS7957,
-	TI_ADS7958,
-	TI_ADS7959,
-	TI_ADS7960,
-	TI_ADS7961,
-};
-
 #define TI_ADS7950_V_CHAN(index, bits)				\
 {								\
 	.type = IIO_VOLTAGE,					\
@@ -225,55 +210,64 @@ static DECLARE_TI_ADS7950_8_CHANNELS(ti_ads7959, 8);
 static DECLARE_TI_ADS7950_12_CHANNELS(ti_ads7960, 8);
 static DECLARE_TI_ADS7950_16_CHANNELS(ti_ads7961, 8);
 
-static const struct ti_ads7950_chip_info ti_ads7950_chip_info[] = {
-	[TI_ADS7950] = {
-		.channels	= ti_ads7950_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7950_channels),
-	},
-	[TI_ADS7951] = {
-		.channels	= ti_ads7951_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7951_channels),
-	},
-	[TI_ADS7952] = {
-		.channels	= ti_ads7952_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7952_channels),
-	},
-	[TI_ADS7953] = {
-		.channels	= ti_ads7953_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7953_channels),
-	},
-	[TI_ADS7954] = {
-		.channels	= ti_ads7954_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7954_channels),
-	},
-	[TI_ADS7955] = {
-		.channels	= ti_ads7955_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7955_channels),
-	},
-	[TI_ADS7956] = {
-		.channels	= ti_ads7956_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7956_channels),
-	},
-	[TI_ADS7957] = {
-		.channels	= ti_ads7957_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7957_channels),
-	},
-	[TI_ADS7958] = {
-		.channels	= ti_ads7958_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7958_channels),
-	},
-	[TI_ADS7959] = {
-		.channels	= ti_ads7959_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7959_channels),
-	},
-	[TI_ADS7960] = {
-		.channels	= ti_ads7960_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7960_channels),
-	},
-	[TI_ADS7961] = {
-		.channels	= ti_ads7961_channels,
-		.num_channels	= ARRAY_SIZE(ti_ads7961_channels),
-	},
+static const struct ti_ads7950_chip_info ti_ads7950_chip_info = {
+	.channels	= ti_ads7950_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7950_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7951_chip_info = {
+	.channels	= ti_ads7951_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7951_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7952_chip_info = {
+	.channels	= ti_ads7952_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7952_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7953_chip_info = {
+	.channels	= ti_ads7953_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7953_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7954_chip_info = {
+	.channels	= ti_ads7954_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7954_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7955_chip_info = {
+	.channels	= ti_ads7955_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7955_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7956_chip_info = {
+	.channels	= ti_ads7956_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7956_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7957_chip_info = {
+	.channels	= ti_ads7957_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7957_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7958_chip_info = {
+	.channels	= ti_ads7958_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7958_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7959_chip_info = {
+	.channels	= ti_ads7959_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7959_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7960_chip_info = {
+	.channels	= ti_ads7960_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7960_channels),
+};
+
+static const struct ti_ads7950_chip_info ti_ads7961_chip_info = {
+	.channels	= ti_ads7961_channels,
+	.num_channels	= ARRAY_SIZE(ti_ads7961_channels),
 };
 
 /*
@@ -427,13 +421,15 @@ static int ti_ads7950_set(struct gpio_chip *chip, unsigned int offset,
 static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct ti_ads7950_state *st = gpiochip_get_data(chip);
+	bool state;
 	int ret;
 
 	mutex_lock(&st->slock);
 
 	/* If set as output, return the output */
 	if (st->gpio_cmd_settings_bitmask & BIT(offset)) {
-		ret = st->cmd_settings_bitmask & BIT(offset);
+		state = st->cmd_settings_bitmask & BIT(offset);
+		ret = 0;
 		goto out;
 	}
 
@@ -444,7 +440,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 	if (ret)
 		goto out;
 
-	ret = ((st->single_rx >> 12) & BIT(offset)) ? 1 : 0;
+	state = (st->single_rx >> 12) & BIT(offset);
 
 	/* Revert back to original settings */
 	st->cmd_settings_bitmask &= ~TI_ADS7950_CR_GPIO_DATA;
@@ -456,7 +452,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 out:
 	mutex_unlock(&st->slock);
 
-	return ret;
+	return ret ?: state;
 }
 
 static int ti_ads7950_get_direction(struct gpio_chip *chip,
@@ -561,7 +557,7 @@ static int ti_ads7950_probe(struct spi_device *spi)
 
 	st->spi = spi;
 
-	info = &ti_ads7950_chip_info[spi_get_device_id(spi)->driver_data];
+	info = spi_get_device_match_data(spi);
 
 	indio_dev->name = spi_get_device_id(spi)->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
@@ -683,35 +679,35 @@ static void ti_ads7950_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id ti_ads7950_id[] = {
-	{ "ads7950", TI_ADS7950 },
-	{ "ads7951", TI_ADS7951 },
-	{ "ads7952", TI_ADS7952 },
-	{ "ads7953", TI_ADS7953 },
-	{ "ads7954", TI_ADS7954 },
-	{ "ads7955", TI_ADS7955 },
-	{ "ads7956", TI_ADS7956 },
-	{ "ads7957", TI_ADS7957 },
-	{ "ads7958", TI_ADS7958 },
-	{ "ads7959", TI_ADS7959 },
-	{ "ads7960", TI_ADS7960 },
-	{ "ads7961", TI_ADS7961 },
+	{ "ads7950", (kernel_ulong_t)&ti_ads7950_chip_info },
+	{ "ads7951", (kernel_ulong_t)&ti_ads7951_chip_info },
+	{ "ads7952", (kernel_ulong_t)&ti_ads7952_chip_info },
+	{ "ads7953", (kernel_ulong_t)&ti_ads7953_chip_info },
+	{ "ads7954", (kernel_ulong_t)&ti_ads7954_chip_info },
+	{ "ads7955", (kernel_ulong_t)&ti_ads7955_chip_info },
+	{ "ads7956", (kernel_ulong_t)&ti_ads7956_chip_info },
+	{ "ads7957", (kernel_ulong_t)&ti_ads7957_chip_info },
+	{ "ads7958", (kernel_ulong_t)&ti_ads7958_chip_info },
+	{ "ads7959", (kernel_ulong_t)&ti_ads7959_chip_info },
+	{ "ads7960", (kernel_ulong_t)&ti_ads7960_chip_info },
+	{ "ads7961", (kernel_ulong_t)&ti_ads7961_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ti_ads7950_id);
 
 static const struct of_device_id ads7950_of_table[] = {
-	{ .compatible = "ti,ads7950", .data = &ti_ads7950_chip_info[TI_ADS7950] },
-	{ .compatible = "ti,ads7951", .data = &ti_ads7950_chip_info[TI_ADS7951] },
-	{ .compatible = "ti,ads7952", .data = &ti_ads7950_chip_info[TI_ADS7952] },
-	{ .compatible = "ti,ads7953", .data = &ti_ads7950_chip_info[TI_ADS7953] },
-	{ .compatible = "ti,ads7954", .data = &ti_ads7950_chip_info[TI_ADS7954] },
-	{ .compatible = "ti,ads7955", .data = &ti_ads7950_chip_info[TI_ADS7955] },
-	{ .compatible = "ti,ads7956", .data = &ti_ads7950_chip_info[TI_ADS7956] },
-	{ .compatible = "ti,ads7957", .data = &ti_ads7950_chip_info[TI_ADS7957] },
-	{ .compatible = "ti,ads7958", .data = &ti_ads7950_chip_info[TI_ADS7958] },
-	{ .compatible = "ti,ads7959", .data = &ti_ads7950_chip_info[TI_ADS7959] },
-	{ .compatible = "ti,ads7960", .data = &ti_ads7950_chip_info[TI_ADS7960] },
-	{ .compatible = "ti,ads7961", .data = &ti_ads7950_chip_info[TI_ADS7961] },
+	{ .compatible = "ti,ads7950", .data = &ti_ads7950_chip_info },
+	{ .compatible = "ti,ads7951", .data = &ti_ads7951_chip_info },
+	{ .compatible = "ti,ads7952", .data = &ti_ads7952_chip_info },
+	{ .compatible = "ti,ads7953", .data = &ti_ads7953_chip_info },
+	{ .compatible = "ti,ads7954", .data = &ti_ads7954_chip_info },
+	{ .compatible = "ti,ads7955", .data = &ti_ads7955_chip_info },
+	{ .compatible = "ti,ads7956", .data = &ti_ads7956_chip_info },
+	{ .compatible = "ti,ads7957", .data = &ti_ads7957_chip_info },
+	{ .compatible = "ti,ads7958", .data = &ti_ads7958_chip_info },
+	{ .compatible = "ti,ads7959", .data = &ti_ads7959_chip_info },
+	{ .compatible = "ti,ads7960", .data = &ti_ads7960_chip_info },
+	{ .compatible = "ti,ads7961", .data = &ti_ads7961_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ads7950_of_table);
diff --git a/drivers/iio/adc/twl4030-madc.c b/drivers/iio/adc/twl4030-madc.c
index fe3b31ec976ef..f0274cd749731 100644
--- a/drivers/iio/adc/twl4030-madc.c
+++ b/drivers/iio/adc/twl4030-madc.c
@@ -252,7 +252,7 @@ static const struct s16_fract twl4030_divider_ratios[16] = {
 	{5, 11},	/* CHANNEL 15 */
 };
 
-/* Conversion table from -3 to 55 degrees Celcius */
+/* Conversion table from -3 to 55 degrees Celsius */
 static int twl4030_therm_tbl[] = {
 	30800,	29500,	28300,	27100,
 	26000,	24900,	23900,	22900,	22000,	21100,	20300,	19400,	18700,
diff --git a/drivers/iio/adc/twl6030-gpadc.c b/drivers/iio/adc/twl6030-gpadc.c
index 3ac774ebf6784..7810d6b2b6683 100644
--- a/drivers/iio/adc/twl6030-gpadc.c
+++ b/drivers/iio/adc/twl6030-gpadc.c
@@ -416,7 +416,7 @@ static u8 twl6032_channel_to_reg(int channel)
 {
 	/*
 	 * for any prior chosen channel, when the conversion is ready
-	 * the result is avalable in GPCH0_LSB, GPCH0_MSB.
+	 * the result is available in GPCH0_LSB, GPCH0_MSB.
 	 */
 
 	return TWL6032_GPADC_GPCH0_LSB;
diff --git a/drivers/iio/addac/ad74413r.c b/drivers/iio/addac/ad74413r.c
index a20b4d48c5f71..fe930ce5ee30f 100644
--- a/drivers/iio/addac/ad74413r.c
+++ b/drivers/iio/addac/ad74413r.c
@@ -839,12 +839,9 @@ static int _ad74413r_get_single_adc_result(struct ad74413r_state *st,
 	if (ret)
 		return ret;
 
-	ret = wait_for_completion_timeout(&st->adc_data_completion,
-					  msecs_to_jiffies(1000));
-	if (!ret) {
-		ret = -ETIMEDOUT;
-		return ret;
-	}
+	if (!wait_for_completion_timeout(&st->adc_data_completion,
+					 msecs_to_jiffies(1000)))
+		return -ETIMEDOUT;
 
 	ret = regmap_read(st->regmap, AD74413R_REG_ADC_RESULT_X(channel),
 			  &uval);
diff --git a/drivers/iio/amplifiers/Kconfig b/drivers/iio/amplifiers/Kconfig
index a8a604863eedf..39d280d4d4377 100644
--- a/drivers/iio/amplifiers/Kconfig
+++ b/drivers/iio/amplifiers/Kconfig
@@ -18,7 +18,13 @@ config AD8366
 	    AD8366 Dual-Digital Variable Gain Amplifier (VGA)
 	    ADA4961 BiCMOS RF Digital Gain Amplifier (DGA)
 	    ADL5240 Digitally controlled variable gain amplifier (VGA)
+	    ADRF5720: 0.5 dB LSB, 6-Bit, Silicon Digital Attenuator
+	    ADRF5730: 0.5 dB LSB, 6-Bit, Silicon Digital Attenuator
+	    ADRF5731: 2 dB LSB, 4-Bit, Silicon Digital Attenuator
+	    HMC271A: 1dB LSB 5-Bit Digital Attenuator SMT
 	    HMC792A 0.25 dB LSB GaAs MMIC 6-Bit Digital Attenuator
+	    HMC1018A: 1.0 dB LSB GaAs MMIC 5-BIT Digital Attenuator
+	    HMC1019A: 0.5 dB LSB GaAs MMIC 5-BIT Digital Attenuator
 	    HMC1119 0.25 dB LSB, 7-Bit, Silicon Digital Attenuator
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/iio/amplifiers/ad8366.c b/drivers/iio/amplifiers/ad8366.c
index d06ac786501c4..334ca91c0f597 100644
--- a/drivers/iio/amplifiers/ad8366.c
+++ b/drivers/iio/amplifiers/ad8366.c
@@ -5,46 +5,50 @@
  *   AD8366 Dual-Digital Variable Gain Amplifier (VGA)
  *   ADA4961 BiCMOS RF Digital Gain Amplifier (DGA)
  *   ADL5240 Digitally controlled variable gain amplifier (VGA)
+ *   ADRF5720: 0.5 dB LSB, 6-Bit, Silicon Digital Attenuator, 9 kHz to 40 GHz
+ *   ADRF5730: 0.5 dB LSB, 6-Bit, Silicon Digital Attenuator, 100 MHz to 40 GHz
+ *   ADRF5731: 2 dB LSB, 4-Bit, Silicon Digital Attenuator, 100 MHz to 40 GHz
+ *   HMC271A: 1dB LSB 5-Bit Digital Attenuator SMT, 0.7 - 3.7 GHz
  *   HMC792A 0.25 dB LSB GaAs MMIC 6-Bit Digital Attenuator
+ *   HMC1018A: 1.0 dB LSB GaAs MMIC 5-BIT DIGITAL ATTENUATOR, 0.1 - 30 GHz
+ *   HMC1019A: 0.5 dB LSB GaAs MMIC 5-BIT DIGITAL ATTENUATOR, 0.1 - 30 GHz
  *   HMC1119 0.25 dB LSB, 7-Bit, Silicon Digital Attenuator
  *
- * Copyright 2012-2019 Analog Devices Inc.
+ * Copyright 2012-2026 Analog Devices Inc.
  */
 
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/spi/spi.h>
-#include <linux/regulator/consumer.h>
-#include <linux/gpio/consumer.h>
+#include <linux/bitrev.h>
+#include <linux/bits.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/math.h>
+#include <linux/minmax.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/bitrev.h>
+#include <linux/mutex.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
 
 #include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-
-enum ad8366_type {
-	ID_AD8366,
-	ID_ADA4961,
-	ID_ADL5240,
-	ID_HMC792,
-	ID_HMC1119,
-};
 
 struct ad8366_info {
+	const char *name;
 	int gain_min;
 	int gain_max;
+	int gain_step;
+	size_t num_channels;
+	size_t (*pack_code)(const unsigned char *code, size_t num_channels,
+			    unsigned char *data);
 };
 
 struct ad8366_state {
 	struct spi_device	*spi;
-	struct regulator	*reg;
 	struct mutex            lock; /* protect sensor state */
-	struct gpio_desc	*reset_gpio;
 	unsigned char		ch[2];
-	enum ad8366_type	type;
 	const struct ad8366_info *info;
 	/*
 	 * DMA (thus cache coherency maintenance) may require the
@@ -53,60 +57,132 @@ struct ad8366_state {
 	unsigned char		data[2] __aligned(IIO_DMA_MINALIGN);
 };
 
-static const struct ad8366_info ad8366_infos[] = {
-	[ID_AD8366] = {
-		.gain_min = 4500,
-		.gain_max = 20500,
-	},
-	[ID_ADA4961] = {
-		.gain_min = -6000,
-		.gain_max = 15000,
-	},
-	[ID_ADL5240] = {
-		.gain_min = -11500,
-		.gain_max = 20000,
-	},
-	[ID_HMC792] = {
-		.gain_min = -15750,
-		.gain_max = 0,
-	},
-	[ID_HMC1119] = {
-		.gain_min = -31750,
-		.gain_max = 0,
-	},
-};
+static size_t ad8366_pack_code(const unsigned char *code, size_t num_channels,
+			       unsigned char *data)
+{
+	u8 ch_a = bitrev8(code[0]) >> 2;
+	u8 ch_b = bitrev8(code[1]) >> 2;
+
+	put_unaligned_be16((ch_b << 6) | ch_a, &data[0]);
+	return sizeof(__be16);
+}
 
-static int ad8366_write(struct iio_dev *indio_dev,
-			unsigned char ch_a, unsigned char ch_b)
+static size_t adrf5731_pack_code(const unsigned char *code, size_t num_channels,
+				 unsigned char *data)
 {
-	struct ad8366_state *st = iio_priv(indio_dev);
-	int ret;
+	data[0] = code[0] << 2;
+	return 1;
+}
+
+static size_t hmc271_pack_code(const unsigned char *code, size_t num_channels,
+			       unsigned char *data)
+{
+	data[0] = bitrev8(code[0]) >> 3;
+	return 1;
+}
 
-	switch (st->type) {
-	case ID_AD8366:
-		ch_a = bitrev8(ch_a & 0x3F);
-		ch_b = bitrev8(ch_b & 0x3F);
+static const struct ad8366_info ad8366_chip_info = {
+	.name = "ad8366",
+	.gain_min = 4500,
+	.gain_max = 20500,
+	.gain_step = 253,
+	.num_channels = 2,
+	.pack_code = ad8366_pack_code,
+};
 
-		st->data[0] = ch_b >> 4;
-		st->data[1] = (ch_b << 4) | (ch_a >> 2);
-		break;
-	case ID_ADA4961:
-		st->data[0] = ch_a & 0x1F;
-		break;
-	case ID_ADL5240:
-		st->data[0] = (ch_a & 0x3F);
-		break;
-	case ID_HMC792:
-	case ID_HMC1119:
-		st->data[0] = ch_a;
-		break;
-	}
+static const struct ad8366_info ada4961_chip_info = {
+	.name = "ada4961",
+	.gain_min = -6000,
+	.gain_max = 15000,
+	.gain_step = -1000,
+	.num_channels = 1,
+};
 
-	ret = spi_write(st->spi, st->data, indio_dev->num_channels);
-	if (ret < 0)
-		dev_err(&indio_dev->dev, "write failed (%d)", ret);
+static const struct ad8366_info adl5240_chip_info = {
+	.name = "adl5240",
+	.gain_min = -11500,
+	.gain_max = 20000,
+	.gain_step = 500,
+	.num_channels = 1,
+};
 
-	return ret;
+static const struct ad8366_info adrf5720_chip_info = {
+	.name = "adrf5720",
+	.gain_min = -31500,
+	.gain_max = 0,
+	.gain_step = -500,
+	.num_channels = 1,
+};
+
+static const struct ad8366_info adrf5730_chip_info = {
+	.name = "adrf5730",
+	.gain_min = -31500,
+	.gain_max = 0,
+	.gain_step = -500,
+	.num_channels = 1,
+};
+
+static const struct ad8366_info adrf5731_chip_info = {
+	.name = "adrf5731",
+	.gain_min = -30000,
+	.gain_max = 0,
+	.gain_step = -2000,
+	.num_channels = 1,
+	.pack_code = adrf5731_pack_code,
+};
+
+static const struct ad8366_info hmc271_chip_info = {
+	.name = "hmc271a",
+	.gain_min = -31000,
+	.gain_max = 0,
+	.gain_step = 1000,
+	.num_channels = 1,
+	.pack_code = hmc271_pack_code,
+};
+
+static const struct ad8366_info hmc792_chip_info = {
+	.name = "hmc792a",
+	.gain_min = -15750,
+	.gain_max = 0,
+	.gain_step = 250,
+	.num_channels = 1,
+};
+
+static const struct ad8366_info hmc1018_chip_info = {
+	.name = "hmc1018a",
+	.gain_min = -31000,
+	.gain_max = 0,
+	.gain_step = 1000,
+	.num_channels = 1,
+};
+
+static const struct ad8366_info hmc1019_chip_info = {
+	.name = "hmc1019a",
+	.gain_min = -15500,
+	.gain_max = 0,
+	.gain_step = 500,
+	.num_channels = 1,
+};
+
+static const struct ad8366_info hmc1119_chip_info = {
+	.name = "hmc1119",
+	.gain_min = -31750,
+	.gain_max = 0,
+	.gain_step = -250,
+	.num_channels = 1,
+};
+
+static int ad8366_write_code(struct ad8366_state *st)
+{
+	const struct ad8366_info *inf = st->info;
+	size_t len = 1;
+
+	if (inf->pack_code)
+		len = inf->pack_code(st->ch, inf->num_channels, st->data);
+	else
+		st->data[0] = st->ch[0];
+
+	return spi_write(st->spi, st->data, len);
 }
 
 static int ad8366_read_raw(struct iio_dev *indio_dev,
@@ -116,6 +192,7 @@ static int ad8366_read_raw(struct iio_dev *indio_dev,
 			   long m)
 {
 	struct ad8366_state *st = iio_priv(indio_dev);
+	const struct ad8366_info *inf = st->info;
 	int ret;
 	int code, gain = 0;
 
@@ -123,25 +200,8 @@ static int ad8366_read_raw(struct iio_dev *indio_dev,
 	switch (m) {
 	case IIO_CHAN_INFO_HARDWAREGAIN:
 		code = st->ch[chan->channel];
-
-		switch (st->type) {
-		case ID_AD8366:
-			gain = code * 253 + 4500;
-			break;
-		case ID_ADA4961:
-			gain = 15000 - code * 1000;
-			break;
-		case ID_ADL5240:
-			gain = 20000 - 31500 + code * 500;
-			break;
-		case ID_HMC792:
-			gain = -1 * code * 500;
-			break;
-		case ID_HMC1119:
-			gain = -1 * code * 250;
-			break;
-		}
-
+		gain = inf->gain_step > 0 ? inf->gain_min : inf->gain_max;
+		gain += inf->gain_step * code;
 		/* Values in dB */
 		*val = gain / 1000;
 		*val2 = (gain % 1000) * 1000;
@@ -176,29 +236,14 @@ static int ad8366_write_raw(struct iio_dev *indio_dev,
 	if (gain > inf->gain_max || gain < inf->gain_min)
 		return -EINVAL;
 
-	switch (st->type) {
-	case ID_AD8366:
-		code = (gain - 4500) / 253;
-		break;
-	case ID_ADA4961:
-		code = (15000 - gain) / 1000;
-		break;
-	case ID_ADL5240:
-		code = ((gain - 500 - 20000) / 500) & 0x3F;
-		break;
-	case ID_HMC792:
-		code = (abs(gain) / 500) & 0x3F;
-		break;
-	case ID_HMC1119:
-		code = (abs(gain) / 250) & 0x7F;
-		break;
-	}
+	gain -= inf->gain_step > 0 ? inf->gain_min : inf->gain_max;
+	code = DIV_ROUND_CLOSEST(gain, inf->gain_step);
 
 	mutex_lock(&st->lock);
 	switch (mask) {
 	case IIO_CHAN_INFO_HARDWAREGAIN:
 		st->ch[chan->channel] = code;
-		ret = ad8366_write(indio_dev, st->ch[0], st->ch[1]);
+		ret = ad8366_write_code(st);
 		break;
 	default:
 		ret = -EINVAL;
@@ -239,107 +284,93 @@ static const struct iio_chan_spec ad8366_channels[] = {
 	AD8366_CHAN(1),
 };
 
-static const struct iio_chan_spec ada4961_channels[] = {
-	AD8366_CHAN(0),
-};
-
 static int ad8366_probe(struct spi_device *spi)
 {
+	struct device *dev = &spi->dev;
+	struct gpio_desc *enable_gpio;
+	struct reset_control *rstc;
 	struct iio_dev *indio_dev;
 	struct ad8366_state *st;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (indio_dev == NULL)
 		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
 
-	st->reg = devm_regulator_get(&spi->dev, "vcc");
-	if (!IS_ERR(st->reg)) {
-		ret = regulator_enable(st->reg);
-		if (ret)
-			return ret;
-	}
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_regulator_get_enable(dev, "vcc");
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get regulator\n");
 
-	spi_set_drvdata(spi, indio_dev);
-	mutex_init(&st->lock);
 	st->spi = spi;
-	st->type = spi_get_device_id(spi)->driver_data;
+	st->info = spi_get_device_match_data(spi);
 
-	switch (st->type) {
-	case ID_AD8366:
-		indio_dev->channels = ad8366_channels;
-		indio_dev->num_channels = ARRAY_SIZE(ad8366_channels);
-		break;
-	case ID_ADA4961:
-	case ID_ADL5240:
-	case ID_HMC792:
-	case ID_HMC1119:
-		st->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset", GPIOD_OUT_HIGH);
-		if (IS_ERR(st->reset_gpio)) {
-			ret = PTR_ERR(st->reset_gpio);
-			goto error_disable_reg;
-		}
-		indio_dev->channels = ada4961_channels;
-		indio_dev->num_channels = ARRAY_SIZE(ada4961_channels);
-		break;
-	default:
-		dev_err(&spi->dev, "Invalid device ID\n");
-		ret = -EINVAL;
-		goto error_disable_reg;
-	}
+	enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(enable_gpio),
+				     "Failed to get enable GPIO\n");
+
+	rstc = devm_reset_control_get_optional_exclusive_deasserted(dev, NULL);
+	if (IS_ERR(rstc))
+		return dev_err_probe(dev, PTR_ERR(rstc),
+				     "Failed to get reset controller\n");
 
-	st->info = &ad8366_infos[st->type];
-	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->name = st->info->name;
 	indio_dev->info = &ad8366_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad8366_channels;
+	indio_dev->num_channels = st->info->num_channels;
 
-	ret = ad8366_write(indio_dev, 0, 0);
+	ret = ad8366_write_code(st);
 	if (ret < 0)
-		goto error_disable_reg;
-
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_disable_reg;
-
-	return 0;
+		return dev_err_probe(dev, ret, "failed to write initial gain\n");
 
-error_disable_reg:
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
-
-	return ret;
-}
-
-static void ad8366_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct ad8366_state *st = iio_priv(indio_dev);
-	struct regulator *reg = st->reg;
-
-	iio_device_unregister(indio_dev);
-
-	if (!IS_ERR(reg))
-		regulator_disable(reg);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad8366_id[] = {
-	{"ad8366",  ID_AD8366},
-	{"ada4961", ID_ADA4961},
-	{"adl5240", ID_ADL5240},
-	{"hmc792a", ID_HMC792},
-	{"hmc1119", ID_HMC1119},
+	{ "ad8366", (kernel_ulong_t)&ad8366_chip_info },
+	{ "ada4961", (kernel_ulong_t)&ada4961_chip_info },
+	{ "adl5240", (kernel_ulong_t)&adl5240_chip_info },
+	{ "adrf5720", (kernel_ulong_t)&adrf5720_chip_info },
+	{ "adrf5730", (kernel_ulong_t)&adrf5730_chip_info },
+	{ "adrf5731", (kernel_ulong_t)&adrf5731_chip_info },
+	{ "hmc271a", (kernel_ulong_t)&hmc271_chip_info },
+	{ "hmc792a", (kernel_ulong_t)&hmc792_chip_info },
+	{ "hmc1018a", (kernel_ulong_t)&hmc1018_chip_info },
+	{ "hmc1019a", (kernel_ulong_t)&hmc1019_chip_info },
+	{ "hmc1119", (kernel_ulong_t)&hmc1119_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad8366_id);
 
+static const struct of_device_id ad8366_of_match[] = {
+	{ .compatible = "adi,ad8366", .data = &ad8366_chip_info },
+	{ .compatible = "adi,ada4961", .data = &ada4961_chip_info },
+	{ .compatible = "adi,adl5240", .data = &adl5240_chip_info },
+	{ .compatible = "adi,adrf5720", .data = &adrf5720_chip_info },
+	{ .compatible = "adi,adrf5730", .data = &adrf5730_chip_info },
+	{ .compatible = "adi,adrf5731", .data = &adrf5731_chip_info },
+	{ .compatible = "adi,hmc271a", .data = &hmc271_chip_info },
+	{ .compatible = "adi,hmc792a", .data = &hmc792_chip_info },
+	{ .compatible = "adi,hmc1018a", .data = &hmc1018_chip_info },
+	{ .compatible = "adi,hmc1019a", .data = &hmc1019_chip_info },
+	{ .compatible = "adi,hmc1119", .data = &hmc1119_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad8366_of_match);
+
 static struct spi_driver ad8366_driver = {
 	.driver = {
-		.name	= KBUILD_MODNAME,
+		.name		= KBUILD_MODNAME,
+		.of_match_table	= ad8366_of_match,
 	},
 	.probe		= ad8366_probe,
-	.remove		= ad8366_remove,
 	.id_table	= ad8366_id,
 };
 
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
index cb771ef8eeb33..24d7df603760f 100644
--- a/drivers/iio/buffer/industrialio-hw-consumer.c
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -28,7 +28,6 @@ struct hw_consumer_buffer {
 	struct list_head head;
 	struct iio_dev *indio_dev;
 	struct iio_buffer buffer;
-	long scan_mask[];
 };
 
 static struct hw_consumer_buffer *iio_buffer_to_hw_consumer_buffer(
@@ -52,7 +51,6 @@ static const struct iio_buffer_access_funcs iio_hw_buf_access = {
 static struct hw_consumer_buffer *iio_hw_consumer_get_buffer(
 	struct iio_hw_consumer *hwc, struct iio_dev *indio_dev)
 {
-	unsigned int mask_longs = BITS_TO_LONGS(iio_get_masklength(indio_dev));
 	struct hw_consumer_buffer *buf;
 
 	list_for_each_entry(buf, &hwc->buffers, head) {
@@ -60,13 +58,18 @@ static struct hw_consumer_buffer *iio_hw_consumer_get_buffer(
 			return buf;
 	}
 
-	buf = kzalloc_flex(*buf, scan_mask, mask_longs);
+	buf = kzalloc_obj(*buf);
 	if (!buf)
 		return NULL;
 
 	buf->buffer.access = &iio_hw_buf_access;
 	buf->indio_dev = indio_dev;
-	buf->buffer.scan_mask = buf->scan_mask;
+	buf->buffer.scan_mask = bitmap_zalloc(iio_get_masklength(indio_dev),
+					      GFP_KERNEL);
+	if (!buf->buffer.scan_mask) {
+		kfree(buf);
+		return NULL;
+	}
 
 	iio_buffer_init(&buf->buffer);
 	list_add_tail(&buf->head, &hwc->buffers);
diff --git a/drivers/iio/buffer/kfifo_buf.c b/drivers/iio/buffer/kfifo_buf.c
index a6ff9085b8a28..b67ea64682265 100644
--- a/drivers/iio/buffer/kfifo_buf.c
+++ b/drivers/iio/buffer/kfifo_buf.c
@@ -224,35 +224,9 @@ void iio_kfifo_free(struct iio_buffer *r)
 }
 EXPORT_SYMBOL(iio_kfifo_free);
 
-static void devm_iio_kfifo_release(struct device *dev, void *res)
+static void devm_iio_kfifo_release(void *buffer)
 {
-	iio_kfifo_free(*(struct iio_buffer **)res);
-}
-
-/**
- * devm_iio_kfifo_allocate - Resource-managed iio_kfifo_allocate()
- * @dev:		Device to allocate kfifo buffer for
- *
- * RETURNS:
- * Pointer to allocated iio_buffer on success, NULL on failure.
- */
-static struct iio_buffer *devm_iio_kfifo_allocate(struct device *dev)
-{
-	struct iio_buffer **ptr, *r;
-
-	ptr = devres_alloc(devm_iio_kfifo_release, sizeof(*ptr), GFP_KERNEL);
-	if (!ptr)
-		return NULL;
-
-	r = iio_kfifo_allocate();
-	if (r) {
-		*ptr = r;
-		devres_add(dev, ptr);
-	} else {
-		devres_free(ptr);
-	}
-
-	return r;
+	iio_kfifo_free(buffer);
 }
 
 /**
@@ -262,10 +236,12 @@ static struct iio_buffer *devm_iio_kfifo_allocate(struct device *dev)
  * @setup_ops: The setup_ops required to configure the HW part of the buffer (optional)
  * @buffer_attrs: Extra sysfs buffer attributes for this IIO buffer
  *
- * This function allocates a kfifo buffer via devm_iio_kfifo_allocate() and
+ * This function allocates a kfifo buffer via iio_kfifo_allocate() and
  * attaches it to the IIO device via iio_device_attach_buffer().
  * This is meant to be a bit of a short-hand/helper function as there are a few
  * drivers that seem to do this.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
 int devm_iio_kfifo_buffer_setup_ext(struct device *dev,
 				    struct iio_dev *indio_dev,
@@ -273,11 +249,16 @@ int devm_iio_kfifo_buffer_setup_ext(struct device *dev,
 				    const struct iio_dev_attr **buffer_attrs)
 {
 	struct iio_buffer *buffer;
+	int ret;
 
-	buffer = devm_iio_kfifo_allocate(dev);
+	buffer = iio_kfifo_allocate();
 	if (!buffer)
 		return -ENOMEM;
 
+	ret = devm_add_action_or_reset(dev, devm_iio_kfifo_release, buffer);
+	if (ret)
+		return ret;
+
 	indio_dev->modes |= INDIO_BUFFER_SOFTWARE;
 	indio_dev->setup_ops = setup_ops;
 
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index 5540e2d28f4ac..417c4ab8c1b27 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -14,6 +14,7 @@
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/kfifo_buf.h>
 #include "hid-sensor-trigger.h"
 
 static ssize_t _hid_sensor_set_report_latency(struct device *dev,
@@ -202,12 +203,21 @@ static void hid_sensor_set_power_work(struct work_struct *work)
 		_hid_sensor_power_state(attrb, true);
 }
 
-static int hid_sensor_data_rdy_trigger_set_state(struct iio_trigger *trig,
-						bool state)
+static int buffer_postenable(struct iio_dev *indio_dev)
 {
-	return hid_sensor_power_state(iio_trigger_get_drvdata(trig), state);
+	return hid_sensor_power_state(iio_device_get_drvdata(indio_dev), 1);
 }
 
+static int buffer_predisable(struct iio_dev *indio_dev)
+{
+	return hid_sensor_power_state(iio_device_get_drvdata(indio_dev), 0);
+}
+
+static const struct iio_buffer_setup_ops hid_sensor_buffer_ops = {
+	.postenable = buffer_postenable,
+	.predisable = buffer_predisable,
+};
+
 void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
 			       struct hid_sensor_common *attrb)
 {
@@ -219,14 +229,9 @@ void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
 	cancel_work_sync(&attrb->work);
 	iio_trigger_unregister(attrb->trigger);
 	iio_trigger_free(attrb->trigger);
-	iio_triggered_buffer_cleanup(indio_dev);
 }
 EXPORT_SYMBOL_NS(hid_sensor_remove_trigger, "IIO_HID");
 
-static const struct iio_trigger_ops hid_sensor_trigger_ops = {
-	.set_trigger_state = &hid_sensor_data_rdy_trigger_set_state,
-};
-
 int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 				struct hid_sensor_common *attrb)
 {
@@ -239,25 +244,34 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 	else
 		fifo_attrs = NULL;
 
-	ret = iio_triggered_buffer_setup_ext(indio_dev,
-					     &iio_pollfunc_store_time, NULL,
-					     IIO_BUFFER_DIRECTION_IN,
-					     NULL, fifo_attrs);
+	indio_dev->modes = INDIO_DIRECT_MODE | INDIO_HARDWARE_TRIGGERED;
+
+	ret = devm_iio_kfifo_buffer_setup_ext(&indio_dev->dev, indio_dev,
+					      &hid_sensor_buffer_ops,
+					      fifo_attrs);
 	if (ret) {
-		dev_err(&indio_dev->dev, "Triggered Buffer Setup Failed\n");
+		dev_err(&indio_dev->dev, "Kfifo Buffer Setup Failed\n");
 		return ret;
 	}
 
+	/*
+	 * The current user space in distro "iio-sensor-proxy" is not working in
+	 * trigerless mode and it expects
+	 * /sys/bus/iio/devices/iio:device0/trigger/current_trigger.
+	 * The change replacing iio_triggered_buffer_setup_ext() with
+	 * devm_iio_kfifo_buffer_setup_ext() will not create attribute without
+	 * registering a trigger with INDIO_HARDWARE_TRIGGERED.
+	 * So the below code fragment is still required.
+	 */
+
 	trig = iio_trigger_alloc(indio_dev->dev.parent,
 				 "%s-dev%d", name, iio_device_id(indio_dev));
 	if (trig == NULL) {
 		dev_err(&indio_dev->dev, "Trigger Allocate Failed\n");
-		ret = -ENOMEM;
-		goto error_triggered_buffer_cleanup;
+		return -ENOMEM;
 	}
 
 	iio_trigger_set_drvdata(trig, attrb);
-	trig->ops = &hid_sensor_trigger_ops;
 	ret = iio_trigger_register(trig);
 
 	if (ret) {
@@ -284,8 +298,6 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 	iio_trigger_unregister(trig);
 error_free_trig:
 	iio_trigger_free(trig);
-error_triggered_buffer_cleanup:
-	iio_triggered_buffer_cleanup(indio_dev);
 	return ret;
 }
 EXPORT_SYMBOL_NS(hid_sensor_setup_trigger, "IIO_HID");
diff --git a/drivers/iio/dac/ad5360.c b/drivers/iio/dac/ad5360.c
index bd32fa57b1d78..20316fd568e62 100644
--- a/drivers/iio/dac/ad5360.c
+++ b/drivers/iio/dac/ad5360.c
@@ -206,14 +206,10 @@ static int ad5360_write_unlocked(struct iio_dev *indio_dev,
 static int ad5360_write(struct iio_dev *indio_dev, unsigned int cmd,
 	unsigned int addr, unsigned int val, unsigned int shift)
 {
-	int ret;
 	struct ad5360_state *st = iio_priv(indio_dev);
 
-	mutex_lock(&st->lock);
-	ret = ad5360_write_unlocked(indio_dev, cmd, addr, val, shift);
-	mutex_unlock(&st->lock);
-
-	return ret;
+	guard(mutex)(&st->lock);
+	return ad5360_write_unlocked(indio_dev, cmd, addr, val, shift);
 }
 
 static int ad5360_read(struct iio_dev *indio_dev, unsigned int type,
@@ -232,7 +228,7 @@ static int ad5360_read(struct iio_dev *indio_dev, unsigned int type,
 		},
 	};
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
 
 	st->data[0].d32 = cpu_to_be32(AD5360_CMD(AD5360_CMD_SPECIAL_FUNCTION) |
 		AD5360_ADDR(AD5360_REG_SF_READBACK) |
@@ -240,12 +236,10 @@ static int ad5360_read(struct iio_dev *indio_dev, unsigned int type,
 		AD5360_READBACK_ADDR(addr));
 
 	ret = spi_sync_transfer(st->spi, t, ARRAY_SIZE(t));
-	if (ret >= 0)
-		ret = be32_to_cpu(st->data[1].d32) & 0xffff;
-
-	mutex_unlock(&st->lock);
+	if (ret < 0)
+		return ret;
 
-	return ret;
+	return be32_to_cpu(st->data[1].d32) & 0xffff;
 }
 
 static ssize_t ad5360_read_dac_powerdown(struct device *dev,
@@ -262,19 +256,14 @@ static int ad5360_update_ctrl(struct iio_dev *indio_dev, unsigned int set,
 	unsigned int clr)
 {
 	struct ad5360_state *st = iio_priv(indio_dev);
-	int ret;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
 
 	st->ctrl |= set;
 	st->ctrl &= ~clr;
 
-	ret = ad5360_write_unlocked(indio_dev, AD5360_CMD_SPECIAL_FUNCTION,
-			AD5360_REG_SF_CTRL, st->ctrl, 0);
-
-	mutex_unlock(&st->lock);
-
-	return ret;
+	return ad5360_write_unlocked(indio_dev, AD5360_CMD_SPECIAL_FUNCTION,
+				     AD5360_REG_SF_CTRL, st->ctrl, 0);
 }
 
 static ssize_t ad5360_write_dac_powerdown(struct device *dev,
diff --git a/drivers/iio/dac/ad7293.c b/drivers/iio/dac/ad7293.c
index c3797e40cdd9f..df6f126abf05c 100644
--- a/drivers/iio/dac/ad7293.c
+++ b/drivers/iio/dac/ad7293.c
@@ -776,27 +776,27 @@ static int ad7293_reset(struct ad7293_state *st)
 
 static int ad7293_properties_parse(struct ad7293_state *st)
 {
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	int ret;
 
-	ret = devm_regulator_get_enable(&spi->dev, "avdd");
+	ret = devm_regulator_get_enable(dev, "avdd");
 	if (ret)
-		return dev_err_probe(&spi->dev, ret, "failed to enable AVDD\n");
+		return dev_err_probe(dev, ret, "failed to enable AVDD\n");
 
-	ret = devm_regulator_get_enable(&spi->dev, "vdrive");
+	ret = devm_regulator_get_enable(dev, "vdrive");
 	if (ret)
-		return dev_err_probe(&spi->dev, ret, "failed to enable VDRIVE\n");
+		return dev_err_probe(dev, ret, "failed to enable VDRIVE\n");
 
-	ret = devm_regulator_get_enable_optional(&spi->dev, "vrefin");
+	ret = devm_regulator_get_enable_optional(dev, "vrefin");
 	if (ret < 0 && ret != -ENODEV)
-		return dev_err_probe(&spi->dev, ret, "failed to enable VREFIN\n");
+		return dev_err_probe(dev, ret, "failed to enable VREFIN\n");
 
 	st->vrefin_en = ret != -ENODEV;
 
-	st->gpio_reset = devm_gpiod_get_optional(&st->spi->dev, "reset",
+	st->gpio_reset = devm_gpiod_get_optional(dev, "reset",
 						 GPIOD_OUT_HIGH);
 	if (IS_ERR(st->gpio_reset))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->gpio_reset),
+		return dev_err_probe(dev, PTR_ERR(st->gpio_reset),
 				     "failed to get the reset GPIO\n");
 
 	return 0;
@@ -806,7 +806,7 @@ static int ad7293_init(struct ad7293_state *st)
 {
 	int ret;
 	u16 chip_id;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 
 	ret = ad7293_properties_parse(st);
 	if (ret)
@@ -821,10 +821,8 @@ static int ad7293_init(struct ad7293_state *st)
 	if (ret)
 		return ret;
 
-	if (chip_id != AD7293_CHIP_ID) {
-		dev_err(&spi->dev, "Invalid Chip ID.\n");
-		return -EINVAL;
-	}
+	if (chip_id != AD7293_CHIP_ID)
+		return dev_err_probe(dev, -EINVAL, "Invalid Chip ID.\n");
 
 	if (!st->vrefin_en)
 		return __ad7293_spi_update_bits(st, AD7293_REG_GENERAL,
@@ -845,9 +843,10 @@ static int ad7293_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
 	struct ad7293_state *st;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -867,7 +866,7 @@ static int ad7293_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad7293_id[] = {
diff --git a/drivers/iio/dac/adi-axi-dac.c b/drivers/iio/dac/adi-axi-dac.c
index 9cc895bbe51a1..451fad34e7eeb 100644
--- a/drivers/iio/dac/adi-axi-dac.c
+++ b/drivers/iio/dac/adi-axi-dac.c
@@ -114,7 +114,6 @@ struct axi_dac_state {
 	const struct axi_dac_info *info;
 	u64 dac_clk;
 	u32 reg_config;
-	bool int_tone;
 	int dac_clk_rate;
 };
 
@@ -869,11 +868,13 @@ static const struct iio_backend_ops axi_ad3552r_ops = {
 static const struct iio_backend_info axi_dac_generic = {
 	.name = "axi-dac",
 	.ops = &axi_dac_generic_ops,
+	.caps = IIO_BACKEND_CAP_BUFFER | IIO_BACKEND_CAP_ENABLE,
 };
 
 static const struct iio_backend_info axi_ad3552r = {
 	.name = "axi-ad3552r",
 	.ops = &axi_ad3552r_ops,
+	.caps = IIO_BACKEND_CAP_BUFFER | IIO_BACKEND_CAP_ENABLE,
 };
 
 static const struct regmap_config axi_dac_regmap_config = {
diff --git a/drivers/iio/dac/max5522.c b/drivers/iio/dac/max5522.c
index 1b8fe6b8d26ef..b52a9cc1da798 100644
--- a/drivers/iio/dac/max5522.c
+++ b/drivers/iio/dac/max5522.c
@@ -14,6 +14,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 #include <linux/spi/spi.h>
+#include <linux/units.h>
 
 #include <linux/iio/iio.h>
 
@@ -34,7 +35,7 @@ struct max5522_state {
 	struct regmap *regmap;
 	const struct max5522_chip_info *chip_info;
 	unsigned short dac_cache[2];
-	struct regulator *vrefin_reg;
+	int vref_mV;
 };
 
 #define MAX5522_CHANNEL(chan) {	\
@@ -79,17 +80,13 @@ static int max5522_read_raw(struct iio_dev *indio_dev,
 			    int *val, int *val2, long info)
 {
 	struct max5522_state *state = iio_priv(indio_dev);
-	int ret;
 
 	switch (info) {
 	case IIO_CHAN_INFO_RAW:
 		*val = state->dac_cache[chan->channel];
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
-		ret = regulator_get_voltage(state->vrefin_reg);
-		if (ret < 0)
-			return -EINVAL;
-		*val = ret / 1000;
+		*val = state->vref_mV;
 		*val2 = 10;
 		return IIO_VAL_FRACTIONAL_LOG2;
 	default:
@@ -147,16 +144,11 @@ static int max5522_spi_probe(struct spi_device *spi)
 	if (!state->chip_info)
 		return -EINVAL;
 
-	state->vrefin_reg = devm_regulator_get(&spi->dev, "vrefin");
-	if (IS_ERR(state->vrefin_reg))
-		return dev_err_probe(&spi->dev, PTR_ERR(state->vrefin_reg),
-				     "Vrefin regulator not specified\n");
-
-	ret = regulator_enable(state->vrefin_reg);
-	if (ret) {
+	ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vrefin");
+	if (ret < 0)
 		return dev_err_probe(&spi->dev, ret,
-				     "Failed to enable vref regulators\n");
-	}
+				     "Failed to get vrefin regulator\n");
+	state->vref_mV = ret / (MICRO / MILLI);
 
 	state->regmap = devm_regmap_init_spi(spi, &max5522_regmap_config);
 
diff --git a/drivers/iio/dac/mcp47feb02.c b/drivers/iio/dac/mcp47feb02.c
index b218f0c3a0bd7..faccb804a5ed5 100644
--- a/drivers/iio/dac/mcp47feb02.c
+++ b/drivers/iio/dac/mcp47feb02.c
@@ -65,7 +65,7 @@
 #define MCP47FEB02_MAX_SCALES_CH			3
 #define MCP47FEB02_DAC_WIPER_UNLOCKED			0
 #define MCP47FEB02_NORMAL_OPERATION			0
-#define MCP47FEB02_INTERNAL_BAND_GAP_mV			2440
+#define MCP47FEB02_INTERNAL_BAND_GAP_uV			2440000
 #define NV_DAC_ADDR_OFFSET				0x10
 
 enum mcp47feb02_vref_mode {
@@ -697,44 +697,40 @@ static const struct iio_chan_spec mcp47febxx_ch_template = {
 };
 
 static void mcp47feb02_init_scale(struct mcp47feb02_data *data, enum mcp47feb02_scale scale,
-				  int vref_mV, int scale_avail[])
+				  int vref_uV, int scale_avail[])
 {
 	u32 value_micro, value_int;
 	u64 tmp;
 
-	/* vref_mV should not be negative */
-	tmp = (u64)vref_mV * MICRO >> data->chip_features->resolution;
+	/* vref_uV should not be negative */
+	tmp = (u64)vref_uV * MILLI >> data->chip_features->resolution;
 	value_int = div_u64_rem(tmp, MICRO, &value_micro);
 	scale_avail[scale * 2] = value_int;
 	scale_avail[scale * 2 + 1] = value_micro;
 }
 
-static int mcp47feb02_init_scales_avail(struct mcp47feb02_data *data, int vdd_mV,
-					int vref_mV, int vref1_mV)
+static int mcp47feb02_init_scales_avail(struct mcp47feb02_data *data, int vdd_uV,
+					int vref_uV, int vref1_uV)
 {
-	struct device *dev = regmap_get_device(data->regmap);
 	int tmp_vref;
 
-	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_mV, data->scale);
+	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale);
 
 	if (data->use_vref)
-		tmp_vref = vref_mV;
+		tmp_vref = vref_uV;
 	else
-		tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_mV;
+		tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
 
 	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1, tmp_vref, data->scale);
 	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X2, tmp_vref * 2, data->scale);
 
 	if (data->phys_channels >= 4) {
-		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_mV, data->scale_1);
-
-		if (data->use_vref1 && vref1_mV <= 0)
-			return dev_err_probe(dev, vref1_mV, "Invalid voltage for Vref1\n");
+		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale_1);
 
 		if (data->use_vref1)
-			tmp_vref = vref1_mV;
+			tmp_vref = vref1_uV;
 		else
-			tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_mV;
+			tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
 
 		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1,
 				      tmp_vref, data->scale_1);
@@ -955,8 +951,6 @@ static int mcp47feb02_parse_fw(struct iio_dev *indio_dev,
 	u32 num_channels;
 	u8 chan_idx = 0;
 
-	guard(mutex)(&data->lock);
-
 	num_channels = device_get_child_node_count(dev);
 	if (num_channels > chip_features->phys_channels)
 		return dev_err_probe(dev, -EINVAL, "More channels than the chip supports\n");
@@ -1080,8 +1074,8 @@ static int mcp47feb02_init_ctrl_regs(struct mcp47feb02_data *data)
 	return 0;
 }
 
-static int mcp47feb02_init_ch_scales(struct mcp47feb02_data *data, int vdd_mV,
-				     int vref_mV, int vref1_mV)
+static int mcp47feb02_init_ch_scales(struct mcp47feb02_data *data, int vdd_uV,
+				     int vref_uV, int vref1_uV)
 {
 	unsigned int i;
 
@@ -1089,7 +1083,7 @@ static int mcp47feb02_init_ch_scales(struct mcp47feb02_data *data, int vdd_mV,
 		struct device *dev = regmap_get_device(data->regmap);
 		int ret;
 
-		ret = mcp47feb02_init_scales_avail(data, vdd_mV, vref_mV, vref1_mV);
+		ret = mcp47feb02_init_scales_avail(data, vdd_uV, vref_uV, vref1_uV);
 		if (ret)
 			return dev_err_probe(dev, ret, "failed to init scales for ch %u\n", i);
 	}
@@ -1103,10 +1097,7 @@ static int mcp47feb02_probe(struct i2c_client *client)
 	struct device *dev = &client->dev;
 	struct mcp47feb02_data *data;
 	struct iio_dev *indio_dev;
-	int vref1_mV = 0;
-	int vref_mV = 0;
-	int vdd_mV;
-	int ret;
+	int vref1_uV, vref_uV, vdd_uV, ret;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
 	if (!indio_dev)
@@ -1143,13 +1134,14 @@ static int mcp47feb02_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
-	vdd_mV = ret / MILLI;
+	vdd_uV = ret;
 
 	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
 	if (ret > 0) {
-		vref_mV = ret / MILLI;
+		vref_uV = ret;
 		data->use_vref = true;
 	} else {
+		vref_uV = 0;
 		dev_dbg(dev, "using internal band gap as voltage reference.\n");
 		dev_dbg(dev, "Vref is unavailable.\n");
 	}
@@ -1157,9 +1149,10 @@ static int mcp47feb02_probe(struct i2c_client *client)
 	if (chip_features->have_ext_vref1) {
 		ret = devm_regulator_get_enable_read_voltage(dev, "vref1");
 		if (ret > 0) {
-			vref1_mV = ret / MILLI;
+			vref1_uV = ret;
 			data->use_vref1 = true;
 		} else {
+			vref1_uV = 0;
 			dev_dbg(dev, "using internal band gap as voltage reference 1.\n");
 			dev_dbg(dev, "Vref1 is unavailable.\n");
 		}
@@ -1169,7 +1162,7 @@ static int mcp47feb02_probe(struct i2c_client *client)
 	if (ret)
 		return dev_err_probe(dev, ret, "Error initialising vref register\n");
 
-	ret = mcp47feb02_init_ch_scales(data, vdd_mV, vref_mV, vref1_mV);
+	ret = mcp47feb02_init_ch_scales(data, vdd_uV, vref_uV, vref1_uV);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/dac/ti-dac5571.c b/drivers/iio/dac/ti-dac5571.c
index bdc3f94aef98d..455d61fc3f135 100644
--- a/drivers/iio/dac/ti-dac5571.c
+++ b/drivers/iio/dac/ti-dac5571.c
@@ -45,7 +45,6 @@ static const struct dac5571_spec dac5571_spec[] = {
 
 struct dac5571_data {
 	struct i2c_client *client;
-	int id;
 	struct mutex lock;
 	struct regulator *vref;
 	u16 val[4];
diff --git a/drivers/iio/filter/admv8818.c b/drivers/iio/filter/admv8818.c
index 19f823446cdad..a4984b8672485 100644
--- a/drivers/iio/filter/admv8818.c
+++ b/drivers/iio/filter/admv8818.c
@@ -657,56 +657,49 @@ static void admv8818_clk_disable(void *data)
 static int admv8818_init(struct admv8818_state *st)
 {
 	int ret;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	unsigned int chip_id;
 
 	ret = regmap_write(st->regmap, ADMV8818_REG_SPI_CONFIG_A,
 			   ADMV8818_SOFTRESET_N_MSK | ADMV8818_SOFTRESET_MSK);
-	if (ret) {
-		dev_err(&spi->dev, "ADMV8818 Soft Reset failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "ADMV8818 Soft Reset failed.\n");
 
 	ret = regmap_write(st->regmap, ADMV8818_REG_SPI_CONFIG_A,
 			   ADMV8818_SDOACTIVE_N_MSK | ADMV8818_SDOACTIVE_MSK);
-	if (ret) {
-		dev_err(&spi->dev, "ADMV8818 SDO Enable failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "ADMV8818 SDO Enable failed.\n");
 
 	ret = regmap_read(st->regmap, ADMV8818_REG_CHIPTYPE, &chip_id);
-	if (ret) {
-		dev_err(&spi->dev, "ADMV8818 Chip ID read failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADMV8818 Chip ID read failed.\n");
 
-	if (chip_id != 0x1) {
-		dev_err(&spi->dev, "ADMV8818 Invalid Chip ID.\n");
-		return -EINVAL;
-	}
+	if (chip_id != 0x1)
+		return dev_err_probe(dev, -EINVAL,
+				     "ADMV8818 Invalid Chip ID.\n");
 
 	ret = regmap_update_bits(st->regmap, ADMV8818_REG_SPI_CONFIG_B,
 				 ADMV8818_SINGLE_INSTRUCTION_MSK,
 				 FIELD_PREP(ADMV8818_SINGLE_INSTRUCTION_MSK, 1));
-	if (ret) {
-		dev_err(&spi->dev, "ADMV8818 Single Instruction failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADMV8818 Single Instruction failed.\n");
 
 	if (st->clkin)
 		return admv8818_rfin_band_select(st);
-	else
-		return 0;
+
+	return 0;
 }
 
 static int admv8818_clk_setup(struct admv8818_state *st)
 {
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	int ret;
 
-	st->clkin = devm_clk_get_optional(&spi->dev, "rf_in");
+	st->clkin = devm_clk_get_optional(dev, "rf_in");
 	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+		return dev_err_probe(dev, PTR_ERR(st->clkin),
 				     "failed to get the input clock\n");
 	else if (!st->clkin)
 		return 0;
@@ -715,7 +708,7 @@ static int admv8818_clk_setup(struct admv8818_state *st)
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(&spi->dev, admv8818_clk_disable, st);
+	ret = devm_add_action_or_reset(dev, admv8818_clk_disable, st);
 	if (ret)
 		return ret;
 
@@ -724,16 +717,16 @@ static int admv8818_clk_setup(struct admv8818_state *st)
 	if (ret < 0)
 		return ret;
 
-	return devm_add_action_or_reset(&spi->dev, admv8818_clk_notifier_unreg, st);
+	return devm_add_action_or_reset(dev, admv8818_clk_notifier_unreg, st);
 }
 
 static int admv8818_read_properties(struct admv8818_state *st)
 {
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	u32 mhz;
 	int ret;
 
-	ret = device_property_read_u32(&spi->dev, "adi,lpf-margin-mhz", &mhz);
+	ret = device_property_read_u32(dev, "adi,lpf-margin-mhz", &mhz);
 	if (ret == 0)
 		st->lpf_margin_hz = (u64)mhz * HZ_PER_MHZ;
 	else if (ret == -EINVAL)
@@ -742,7 +735,7 @@ static int admv8818_read_properties(struct admv8818_state *st)
 		return ret;
 
 
-	ret = device_property_read_u32(&spi->dev, "adi,hpf-margin-mhz", &mhz);
+	ret = device_property_read_u32(dev, "adi,hpf-margin-mhz", &mhz);
 	if (ret == 0)
 		st->hpf_margin_hz = (u64)mhz * HZ_PER_MHZ;
 	else if (ret == -EINVAL)
@@ -758,9 +751,10 @@ static int admv8818_probe(struct spi_device *spi)
 	struct iio_dev *indio_dev;
 	struct regmap *regmap;
 	struct admv8818_state *st;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -792,7 +786,7 @@ static int admv8818_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id admv8818_id[] = {
diff --git a/drivers/iio/frequency/ad9523.c b/drivers/iio/frequency/ad9523.c
index 63c485e9e44cd..ad32eb66edca0 100644
--- a/drivers/iio/frequency/ad9523.c
+++ b/drivers/iio/frequency/ad9523.c
@@ -167,9 +167,9 @@
 
 /* AD9523_CHANNEL_CLOCK_DIST */
 #define AD9523_CLK_DIST_DIV_PHASE(x)		(((x) & 0x3F) << 18)
-#define AD9523_CLK_DIST_DIV_PHASE_REV(x)	((ret >> 18) & 0x3F)
+#define AD9523_CLK_DIST_DIV_PHASE_REV(x)	(((x) >> 18) & 0x3F)
 #define AD9523_CLK_DIST_DIV(x)			((((x) - 1) & 0x3FF) << 8)
-#define AD9523_CLK_DIST_DIV_REV(x)		(((ret >> 8) & 0x3FF) + 1)
+#define AD9523_CLK_DIST_DIV_REV(x)		((((x) >> 8) & 0x3FF) + 1)
 #define AD9523_CLK_DIST_INV_DIV_OUTPUT_EN	(1 << 7)
 #define AD9523_CLK_DIST_IGNORE_SYNC_EN		(1 << 6)
 #define AD9523_CLK_DIST_PWR_DOWN_EN		(1 << 5)
@@ -558,55 +558,35 @@ static ssize_t ad9523_show(struct device *dev,
 	return ret;
 }
 
-static IIO_DEVICE_ATTR(pll1_locked, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_PLL1_LD);
-
-static IIO_DEVICE_ATTR(pll2_locked, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_PLL2_LD);
-
-static IIO_DEVICE_ATTR(pll1_reference_clk_a_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_REFA);
-
-static IIO_DEVICE_ATTR(pll1_reference_clk_b_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_REFB);
-
-static IIO_DEVICE_ATTR(pll1_reference_clk_test_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_REF_TEST);
-
-static IIO_DEVICE_ATTR(vcxo_clk_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_VCXO);
-
-static IIO_DEVICE_ATTR(pll2_feedback_clk_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_PLL2_FB_CLK);
-
-static IIO_DEVICE_ATTR(pll2_reference_clk_present, S_IRUGO,
-			ad9523_show,
-			NULL,
-			AD9523_STAT_PLL2_REF_CLK);
-
-static IIO_DEVICE_ATTR(sync_dividers, S_IWUSR,
-			NULL,
-			ad9523_store,
-			AD9523_SYNC);
-
-static IIO_DEVICE_ATTR(store_eeprom, S_IWUSR,
-			NULL,
-			ad9523_store,
-			AD9523_EEPROM);
+static IIO_DEVICE_ATTR(pll1_locked, 0444, ad9523_show, NULL,
+		       AD9523_STAT_PLL1_LD);
+
+static IIO_DEVICE_ATTR(pll2_locked, 0444, ad9523_show, NULL,
+		       AD9523_STAT_PLL2_LD);
+
+static IIO_DEVICE_ATTR(pll1_reference_clk_a_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_REFA);
+
+static IIO_DEVICE_ATTR(pll1_reference_clk_b_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_REFB);
+
+static IIO_DEVICE_ATTR(pll1_reference_clk_test_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_REF_TEST);
+
+static IIO_DEVICE_ATTR(vcxo_clk_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_VCXO);
+
+static IIO_DEVICE_ATTR(pll2_feedback_clk_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_PLL2_FB_CLK);
+
+static IIO_DEVICE_ATTR(pll2_reference_clk_present, 0444, ad9523_show, NULL,
+		       AD9523_STAT_PLL2_REF_CLK);
+
+static IIO_DEVICE_ATTR(sync_dividers, 0200, NULL, ad9523_store,
+		       AD9523_SYNC);
+
+static IIO_DEVICE_ATTR(store_eeprom, 0200, NULL, ad9523_store,
+		       AD9523_EEPROM);
 
 static struct attribute *ad9523_attributes[] = {
 	&iio_dev_attr_sync_dividers.dev_attr.attr,
@@ -797,8 +777,7 @@ static int ad9523_setup(struct iio_dev *indio_dev)
 		return ret;
 
 	ret = ad9523_write(indio_dev, AD9523_PLL1_CHARGE_PUMP_CTRL,
-		AD9523_PLL1_CHARGE_PUMP_CURRENT_nA(pdata->
-			pll1_charge_pump_current_nA) |
+		AD9523_PLL1_CHARGE_PUMP_CURRENT_nA(pdata->pll1_charge_pump_current_nA) |
 		AD9523_PLL1_CHARGE_PUMP_MODE_NORMAL |
 		AD9523_PLL1_BACKLASH_PW_MIN);
 	if (ret < 0)
@@ -842,8 +821,7 @@ static int ad9523_setup(struct iio_dev *indio_dev)
 	 */
 
 	ret = ad9523_write(indio_dev, AD9523_PLL2_CHARGE_PUMP,
-		AD9523_PLL2_CHARGE_PUMP_CURRENT_nA(pdata->
-			pll2_charge_pump_current_nA));
+		AD9523_PLL2_CHARGE_PUMP_CURRENT_nA(pdata->pll2_charge_pump_current_nA));
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/iio/frequency/adf4350.c b/drivers/iio/frequency/adf4350.c
index ed1741165f555..3883b63dcc3ce 100644
--- a/drivers/iio/frequency/adf4350.c
+++ b/drivers/iio/frequency/adf4350.c
@@ -607,7 +607,7 @@ static int adf4350_probe(struct spi_device *spi)
 	if (dev_fwnode(&spi->dev)) {
 		pdata = adf4350_parse_dt(&spi->dev);
 		if (pdata == NULL)
-			return -EINVAL;
+			return -ENOMEM;
 	} else {
 		pdata = dev_get_platdata(&spi->dev);
 	}
diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c
index 8e2da218d48a4..ff6077e29c739 100644
--- a/drivers/iio/frequency/adf4377.c
+++ b/drivers/iio/frequency/adf4377.c
@@ -706,23 +706,20 @@ static void adf4377_gpio_init(struct adf4377_state *st)
 
 static int adf4377_init(struct adf4377_state *st)
 {
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	int ret;
 
 	adf4377_gpio_init(st);
 
 	ret = adf4377_soft_reset(st);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to soft reset.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to soft reset.\n");
 
 	ret = regmap_multi_reg_write(st->regmap, adf4377_reg_defaults,
 				     ARRAY_SIZE(adf4377_reg_defaults));
-	if (ret) {
-		dev_err(&spi->dev, "Failed to set default registers.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to set default registers.\n");
 
 	ret = regmap_update_bits(st->regmap, 0x00,
 				 ADF4377_0000_SDO_ACTIVE_MSK | ADF4377_0000_SDO_ACTIVE_R_MSK,
@@ -730,10 +727,9 @@ static int adf4377_init(struct adf4377_state *st)
 					    ADF4377_0000_SDO_ACTIVE_SPI_4W) |
 				 FIELD_PREP(ADF4377_0000_SDO_ACTIVE_R_MSK,
 					    ADF4377_0000_SDO_ACTIVE_SPI_4W));
-	if (ret) {
-		dev_err(&spi->dev, "Failed to set 4-Wire Operation.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to set 4-Wire Operation.\n");
 
 	st->clkin_freq = clk_get_rate(st->clkin);
 
@@ -747,10 +743,9 @@ static int adf4377_init(struct adf4377_state *st)
 			   FIELD_PREP(ADF4377_001A_PD_PFDCP_MSK, 0) |
 			   FIELD_PREP(ADF4377_001A_PD_CLKOUT1_MSK, 0) |
 			   FIELD_PREP(ADF4377_001A_PD_CLKOUT2_MSK, 0));
-	if (ret) {
-		dev_err(&spi->dev, "Failed to set power down registers.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to set power down registers.\n");
 
 	/* Set Mux Output */
 	ret = regmap_update_bits(st->regmap, 0x1D,
@@ -882,35 +877,35 @@ static const struct iio_chan_spec adf4377_channels[] = {
 
 static int adf4377_properties_parse(struct adf4377_state *st)
 {
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 	int ret;
 
-	st->clkin = devm_clk_get_enabled(&spi->dev, "ref_in");
+	st->clkin = devm_clk_get_enabled(dev, "ref_in");
 	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+		return dev_err_probe(dev, PTR_ERR(st->clkin),
 				     "failed to get the reference input clock\n");
 
-	st->gpio_ce = devm_gpiod_get_optional(&st->spi->dev, "chip-enable",
+	st->gpio_ce = devm_gpiod_get_optional(dev, "chip-enable",
 					      GPIOD_OUT_LOW);
 	if (IS_ERR(st->gpio_ce))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->gpio_ce),
+		return dev_err_probe(dev, PTR_ERR(st->gpio_ce),
 				     "failed to get the CE GPIO\n");
 
-	st->gpio_enclk1 = devm_gpiod_get_optional(&st->spi->dev, "clk1-enable",
+	st->gpio_enclk1 = devm_gpiod_get_optional(dev, "clk1-enable",
 						  GPIOD_OUT_LOW);
 	if (IS_ERR(st->gpio_enclk1))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->gpio_enclk1),
+		return dev_err_probe(dev, PTR_ERR(st->gpio_enclk1),
 				     "failed to get the CE GPIO\n");
 
 	if (st->chip_info->has_gpio_enclk2) {
-		st->gpio_enclk2 = devm_gpiod_get_optional(&st->spi->dev, "clk2-enable",
+		st->gpio_enclk2 = devm_gpiod_get_optional(dev, "clk2-enable",
 							  GPIOD_OUT_LOW);
 		if (IS_ERR(st->gpio_enclk2))
-			return dev_err_probe(&spi->dev, PTR_ERR(st->gpio_enclk2),
+			return dev_err_probe(dev, PTR_ERR(st->gpio_enclk2),
 					"failed to get the CE GPIO\n");
 	}
 
-	ret = device_property_match_property_string(&spi->dev, "adi,muxout-select",
+	ret = device_property_match_property_string(dev, "adi,muxout-select",
 						    adf4377_muxout_modes,
 						    ARRAY_SIZE(adf4377_muxout_modes));
 	if (ret >= 0)
@@ -1055,9 +1050,10 @@ static int adf4377_probe(struct spi_device *spi)
 	struct iio_dev *indio_dev;
 	struct regmap *regmap;
 	struct adf4377_state *st;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -1080,7 +1076,7 @@ static int adf4377_probe(struct spi_device *spi)
 		return ret;
 
 	st->nb.notifier_call = adf4377_freq_change;
-	ret = devm_clk_notifier_register(&spi->dev, st->clkin, &st->nb);
+	ret = devm_clk_notifier_register(dev, st->clkin, &st->nb);
 	if (ret)
 		return ret;
 
@@ -1097,7 +1093,7 @@ static int adf4377_probe(struct spi_device *spi)
 		indio_dev->num_channels = ARRAY_SIZE(adf4377_channels);
 	}
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id adf4377_id[] = {
diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c
index d8e8d541990f8..b852378b3f68d 100644
--- a/drivers/iio/frequency/admv1013.c
+++ b/drivers/iio/frequency/admv1013.c
@@ -85,9 +85,9 @@ enum {
 };
 
 enum {
-	ADMV1013_SE_MODE_POS = 6,
-	ADMV1013_SE_MODE_NEG = 9,
-	ADMV1013_SE_MODE_DIFF = 12
+	ADMV1013_SE_MODE_POS,
+	ADMV1013_SE_MODE_NEG,
+	ADMV1013_SE_MODE_DIFF,
 };
 
 struct admv1013_state {
@@ -441,7 +441,7 @@ static int admv1013_init(struct admv1013_state *st, int vcm_uv)
 {
 	int ret;
 	unsigned int data;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 
 	/* Perform a software reset */
 	ret = __admv1013_spi_update_bits(st, ADMV1013_REG_SPI_CONTROL,
@@ -461,19 +461,30 @@ static int admv1013_init(struct admv1013_state *st, int vcm_uv)
 		return ret;
 
 	data = FIELD_GET(ADMV1013_CHIP_ID_MSK, data);
-	if (data != ADMV1013_CHIP_ID) {
-		dev_err(&spi->dev, "Invalid Chip ID.\n");
-		return -EINVAL;
-	}
+	if (data != ADMV1013_CHIP_ID)
+		return dev_err_probe(dev, -EINVAL, "Invalid Chip ID.\n");
 
 	ret = __admv1013_spi_write(st, ADMV1013_REG_VVA_TEMP_COMP, 0xE700);
 	if (ret)
 		return ret;
 
-	data = FIELD_PREP(ADMV1013_QUAD_SE_MODE_MSK, st->quad_se_mode);
+	switch (st->quad_se_mode) {
+	case ADMV1013_SE_MODE_POS:
+		data = 6;
+		break;
+	case ADMV1013_SE_MODE_NEG:
+		data = 9;
+		break;
+	case ADMV1013_SE_MODE_DIFF:
+		data = 12;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	ret = __admv1013_spi_update_bits(st, ADMV1013_REG_QUAD,
-					 ADMV1013_QUAD_SE_MODE_MSK, data);
+					 ADMV1013_QUAD_SE_MODE_MSK,
+					 FIELD_PREP(ADMV1013_QUAD_SE_MODE_MSK, data));
 	if (ret)
 		return ret;
 
@@ -514,43 +525,39 @@ static void admv1013_powerdown(void *data)
 	admv1013_spi_update_bits(data, ADMV1013_REG_ENABLE, enable_reg_msk, enable_reg);
 }
 
+static const char * const admv1013_input_modes[] = {
+	[ADMV1013_IQ_MODE] = "iq",
+	[ADMV1013_IF_MODE] = "if",
+};
+
+static const char * const admv1013_quad_se_modes[] = {
+	[ADMV1013_SE_MODE_POS] = "se-pos",
+	[ADMV1013_SE_MODE_NEG] = "se-neg",
+	[ADMV1013_SE_MODE_DIFF] = "diff",
+};
+
 static int admv1013_properties_parse(struct admv1013_state *st)
 {
 	int ret;
-	const char *str;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 
-	st->det_en = device_property_read_bool(&spi->dev, "adi,detector-enable");
+	st->det_en = device_property_read_bool(dev, "adi,detector-enable");
 
-	ret = device_property_read_string(&spi->dev, "adi,input-mode", &str);
-	if (ret)
-		st->input_mode = ADMV1013_IQ_MODE;
+	ret = device_property_match_property_string(dev, "adi,input-mode",
+						    admv1013_input_modes,
+						    ARRAY_SIZE(admv1013_input_modes));
+	st->input_mode = ret >= 0 ? ret : ADMV1013_IQ_MODE;
 
-	if (!strcmp(str, "iq"))
-		st->input_mode = ADMV1013_IQ_MODE;
-	else if (!strcmp(str, "if"))
-		st->input_mode = ADMV1013_IF_MODE;
-	else
-		return -EINVAL;
+	ret = device_property_match_property_string(dev, "adi,quad-se-mode",
+						    admv1013_quad_se_modes,
+						    ARRAY_SIZE(admv1013_quad_se_modes));
+	st->quad_se_mode = ret >= 0 ? ret : ADMV1013_SE_MODE_DIFF;
 
-	ret = device_property_read_string(&spi->dev, "adi,quad-se-mode", &str);
-	if (ret)
-		st->quad_se_mode = ADMV1013_SE_MODE_DIFF;
-
-	if (!strcmp(str, "diff"))
-		st->quad_se_mode = ADMV1013_SE_MODE_DIFF;
-	else if (!strcmp(str, "se-pos"))
-		st->quad_se_mode = ADMV1013_SE_MODE_POS;
-	else if (!strcmp(str, "se-neg"))
-		st->quad_se_mode = ADMV1013_SE_MODE_NEG;
-	else
-		return -EINVAL;
-
-	ret = devm_regulator_bulk_get_enable(&st->spi->dev,
+	ret = devm_regulator_bulk_get_enable(dev,
 					     ARRAY_SIZE(admv1013_vcc_regs),
 					     admv1013_vcc_regs);
 	if (ret) {
-		dev_err_probe(&spi->dev, ret,
+		dev_err_probe(dev, ret,
 			      "Failed to request VCC regulators\n");
 		return ret;
 	}
@@ -562,9 +569,10 @@ static int admv1013_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
 	struct admv1013_state *st;
+	struct device *dev = &spi->dev;
 	int ret, vcm_uv;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -581,36 +589,34 @@ static int admv1013_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vcm");
+	ret = devm_regulator_get_enable_read_voltage(dev, "vcm");
 	if (ret < 0)
-		return dev_err_probe(&spi->dev, ret,
+		return dev_err_probe(dev, ret,
 				     "failed to get the common-mode voltage\n");
 
 	vcm_uv = ret;
 
-	st->clkin = devm_clk_get_enabled(&spi->dev, "lo_in");
+	st->clkin = devm_clk_get_enabled(dev, "lo_in");
 	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+		return dev_err_probe(dev, PTR_ERR(st->clkin),
 				     "failed to get the LO input clock\n");
 
 	st->nb.notifier_call = admv1013_freq_change;
-	ret = devm_clk_notifier_register(&spi->dev, st->clkin, &st->nb);
+	ret = devm_clk_notifier_register(dev, st->clkin, &st->nb);
 	if (ret)
 		return ret;
 
 	mutex_init(&st->lock);
 
 	ret = admv1013_init(st, vcm_uv);
-	if (ret) {
-		dev_err(&spi->dev, "admv1013 init failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "admv1013 init failed\n");
 
-	ret = devm_add_action_or_reset(&spi->dev, admv1013_powerdown, st);
+	ret = devm_add_action_or_reset(dev, admv1013_powerdown, st);
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id admv1013_id[] = {
diff --git a/drivers/iio/frequency/admv1014.c b/drivers/iio/frequency/admv1014.c
index 7a8f92ec80a2e..25e8cd8135ad6 100644
--- a/drivers/iio/frequency/admv1014.c
+++ b/drivers/iio/frequency/admv1014.c
@@ -610,15 +610,14 @@ static int admv1014_init(struct admv1014_state *st)
 {
 	unsigned int chip_id, enable_reg, enable_reg_msk;
 	struct spi_device *spi = st->spi;
+	struct device *dev = &spi->dev;
 	int ret;
 
 	ret = regulator_bulk_enable(ADMV1014_NUM_REGULATORS, st->regulators);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to enable regulators");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable regulators");
 
-	ret = devm_add_action_or_reset(&spi->dev, admv1014_reg_disable, st->regulators);
+	ret = devm_add_action_or_reset(dev, admv1014_reg_disable, st->regulators);
 	if (ret)
 		return ret;
 
@@ -626,16 +625,16 @@ static int admv1014_init(struct admv1014_state *st)
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(&spi->dev, admv1014_clk_disable, st->clkin);
+	ret = devm_add_action_or_reset(dev, admv1014_clk_disable, st->clkin);
 	if (ret)
 		return ret;
 
 	st->nb.notifier_call = admv1014_freq_change;
-	ret = devm_clk_notifier_register(&spi->dev, st->clkin, &st->nb);
+	ret = devm_clk_notifier_register(dev, st->clkin, &st->nb);
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(&spi->dev, admv1014_powerdown, st);
+	ret = devm_add_action_or_reset(dev, admv1014_powerdown, st);
 	if (ret)
 		return ret;
 
@@ -643,55 +642,47 @@ static int admv1014_init(struct admv1014_state *st)
 	ret = __admv1014_spi_update_bits(st, ADMV1014_REG_SPI_CONTROL,
 					 ADMV1014_SPI_SOFT_RESET_MSK,
 					 FIELD_PREP(ADMV1014_SPI_SOFT_RESET_MSK, 1));
-	if (ret) {
-		dev_err(&spi->dev, "ADMV1014 SPI software reset failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADMV1014 SPI software reset failed.\n");
 
 	ret = __admv1014_spi_update_bits(st, ADMV1014_REG_SPI_CONTROL,
 					 ADMV1014_SPI_SOFT_RESET_MSK,
 					 FIELD_PREP(ADMV1014_SPI_SOFT_RESET_MSK, 0));
-	if (ret) {
-		dev_err(&spi->dev, "ADMV1014 SPI software reset disable failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADMV1014 SPI software reset disable failed.\n");
 
 	ret = __admv1014_spi_write(st, ADMV1014_REG_VVA_TEMP_COMP, 0x727C);
-	if (ret) {
-		dev_err(&spi->dev, "Writing default Temperature Compensation value failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Writing default Temperature Compensation value failed.\n");
 
 	ret = __admv1014_spi_read(st, ADMV1014_REG_SPI_CONTROL, &chip_id);
 	if (ret)
 		return ret;
 
 	chip_id = FIELD_GET(ADMV1014_CHIP_ID_MSK, chip_id);
-	if (chip_id != ADMV1014_CHIP_ID) {
-		dev_err(&spi->dev, "Invalid Chip ID.\n");
-		return -EINVAL;
-	}
+	if (chip_id != ADMV1014_CHIP_ID)
+		return dev_err_probe(dev, -EINVAL, "Invalid Chip ID.\n");
 
 	ret = __admv1014_spi_update_bits(st, ADMV1014_REG_QUAD,
 					 ADMV1014_QUAD_SE_MODE_MSK,
 					 FIELD_PREP(ADMV1014_QUAD_SE_MODE_MSK,
 						    st->quad_se_mode));
-	if (ret) {
-		dev_err(&spi->dev, "Writing Quad SE Mode failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Writing Quad SE Mode failed.\n");
 
 	ret = admv1014_update_quad_filters(st);
-	if (ret) {
-		dev_err(&spi->dev, "Update Quad Filters failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Update Quad Filters failed.\n");
 
 	ret = admv1014_update_vcm_settings(st);
-	if (ret) {
-		dev_err(&spi->dev, "Update VCM Settings failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Update VCM Settings failed.\n");
 
 	enable_reg_msk = ADMV1014_P1DB_COMPENSATION_MSK |
 			 ADMV1014_IF_AMP_PD_MSK |
@@ -712,13 +703,14 @@ static int admv1014_properties_parse(struct admv1014_state *st)
 {
 	unsigned int i;
 	struct spi_device *spi = st->spi;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	st->det_en = device_property_read_bool(&spi->dev, "adi,detector-enable");
+	st->det_en = device_property_read_bool(dev, "adi,detector-enable");
 
-	st->p1db_comp = device_property_read_bool(&spi->dev, "adi,p1db-compensation-enable");
+	st->p1db_comp = device_property_read_bool(dev, "adi,p1db-compensation-enable");
 
-	ret = device_property_match_property_string(&spi->dev, "adi,input-mode",
+	ret = device_property_match_property_string(dev, "adi,input-mode",
 						    input_mode_names,
 						    ARRAY_SIZE(input_mode_names));
 	if (ret >= 0)
@@ -726,7 +718,7 @@ static int admv1014_properties_parse(struct admv1014_state *st)
 	else
 		st->input_mode = ADMV1014_IQ_MODE;
 
-	ret = device_property_match_property_string(&spi->dev, "adi,quad-se-mode",
+	ret = device_property_match_property_string(dev, "adi,quad-se-mode",
 						    quad_se_mode_names,
 						    ARRAY_SIZE(quad_se_mode_names));
 	if (ret >= 0)
@@ -737,16 +729,14 @@ static int admv1014_properties_parse(struct admv1014_state *st)
 	for (i = 0; i < ADMV1014_NUM_REGULATORS; ++i)
 		st->regulators[i].supply = admv1014_reg_name[i];
 
-	ret = devm_regulator_bulk_get(&st->spi->dev, ADMV1014_NUM_REGULATORS,
+	ret = devm_regulator_bulk_get(dev, ADMV1014_NUM_REGULATORS,
 				      st->regulators);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to request regulators");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to request regulators");
 
-	st->clkin = devm_clk_get(&spi->dev, "lo_in");
+	st->clkin = devm_clk_get(dev, "lo_in");
 	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+		return dev_err_probe(dev, PTR_ERR(st->clkin),
 				     "failed to get the LO input clock\n");
 
 	return 0;
@@ -756,9 +746,10 @@ static int admv1014_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
 	struct admv1014_state *st;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -787,7 +778,7 @@ static int admv1014_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id admv1014_id[] = {
diff --git a/drivers/iio/frequency/admv4420.c b/drivers/iio/frequency/admv4420.c
index 3ae462b4f5c9f..8748d97476397 100644
--- a/drivers/iio/frequency/admv4420.c
+++ b/drivers/iio/frequency/admv4420.c
@@ -243,7 +243,7 @@ static int admv4420_calc_parameters(struct admv4420_state *st)
 		st->n_counter.n_counter = 1;
 	}
 	if (!sol_found)
-		return -1;
+		return -EINVAL;
 
 	st->n_counter.int_val = div_u64_rem(st->n_counter.n_counter, 10, &st->n_counter.frac_val);
 	st->n_counter.mod_val = 10;
diff --git a/drivers/iio/frequency/adrf6780.c b/drivers/iio/frequency/adrf6780.c
index a7a21f929970d..257fd31a0b0e6 100644
--- a/drivers/iio/frequency/adrf6780.c
+++ b/drivers/iio/frequency/adrf6780.c
@@ -346,23 +346,21 @@ static const struct iio_chan_spec adrf6780_channels[] = {
 static int adrf6780_reset(struct adrf6780_state *st)
 {
 	int ret;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 
 	ret = __adrf6780_spi_update_bits(st, ADRF6780_REG_CONTROL,
 					 ADRF6780_SOFT_RESET_MSK,
 					 FIELD_PREP(ADRF6780_SOFT_RESET_MSK, 1));
-	if (ret) {
-		dev_err(&spi->dev, "ADRF6780 SPI software reset failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADRF6780 SPI software reset failed.\n");
 
 	ret = __adrf6780_spi_update_bits(st, ADRF6780_REG_CONTROL,
 					 ADRF6780_SOFT_RESET_MSK,
 					 FIELD_PREP(ADRF6780_SOFT_RESET_MSK, 0));
-	if (ret) {
-		dev_err(&spi->dev, "ADRF6780 SPI software reset disable failed.\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "ADRF6780 SPI software reset disable failed.\n");
 
 	return 0;
 }
@@ -371,7 +369,7 @@ static int adrf6780_init(struct adrf6780_state *st)
 {
 	int ret;
 	unsigned int chip_id, enable_reg, enable_reg_msk;
-	struct spi_device *spi = st->spi;
+	struct device *dev = &st->spi->dev;
 
 	/* Perform a software reset */
 	ret = adrf6780_reset(st);
@@ -383,10 +381,9 @@ static int adrf6780_init(struct adrf6780_state *st)
 		return ret;
 
 	chip_id = FIELD_GET(ADRF6780_CHIP_ID_MSK, chip_id);
-	if (chip_id != ADRF6780_CHIP_ID) {
-		dev_err(&spi->dev, "ADRF6780 Invalid Chip ID.\n");
-		return -EINVAL;
-	}
+	if (chip_id != ADRF6780_CHIP_ID)
+		return dev_err_probe(dev, -EINVAL,
+				     "ADRF6780 Invalid Chip ID.\n");
 
 	enable_reg_msk = ADRF6780_VGA_BUFFER_EN_MSK |
 			ADRF6780_DETECTOR_EN_MSK |
@@ -426,18 +423,18 @@ static int adrf6780_init(struct adrf6780_state *st)
 
 static void adrf6780_properties_parse(struct adrf6780_state *st)
 {
-	struct spi_device *spi = st->spi;
-
-	st->vga_buff_en = device_property_read_bool(&spi->dev, "adi,vga-buff-en");
-	st->lo_buff_en = device_property_read_bool(&spi->dev, "adi,lo-buff-en");
-	st->if_mode_en = device_property_read_bool(&spi->dev, "adi,if-mode-en");
-	st->iq_mode_en = device_property_read_bool(&spi->dev, "adi,iq-mode-en");
-	st->lo_x2_en = device_property_read_bool(&spi->dev, "adi,lo-x2-en");
-	st->lo_ppf_en = device_property_read_bool(&spi->dev, "adi,lo-ppf-en");
-	st->lo_en = device_property_read_bool(&spi->dev, "adi,lo-en");
-	st->uc_bias_en = device_property_read_bool(&spi->dev, "adi,uc-bias-en");
-	st->lo_sideband = device_property_read_bool(&spi->dev, "adi,lo-sideband");
-	st->vdet_out_en = device_property_read_bool(&spi->dev, "adi,vdet-out-en");
+	struct device *dev = &st->spi->dev;
+
+	st->vga_buff_en = device_property_read_bool(dev, "adi,vga-buff-en");
+	st->lo_buff_en = device_property_read_bool(dev, "adi,lo-buff-en");
+	st->if_mode_en = device_property_read_bool(dev, "adi,if-mode-en");
+	st->iq_mode_en = device_property_read_bool(dev, "adi,iq-mode-en");
+	st->lo_x2_en = device_property_read_bool(dev, "adi,lo-x2-en");
+	st->lo_ppf_en = device_property_read_bool(dev, "adi,lo-ppf-en");
+	st->lo_en = device_property_read_bool(dev, "adi,lo-en");
+	st->uc_bias_en = device_property_read_bool(dev, "adi,uc-bias-en");
+	st->lo_sideband = device_property_read_bool(dev, "adi,lo-sideband");
+	st->vdet_out_en = device_property_read_bool(dev, "adi,vdet-out-en");
 }
 
 static void adrf6780_powerdown(void *data)
@@ -450,9 +447,10 @@ static int adrf6780_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
 	struct adrf6780_state *st;
+	struct device *dev = &spi->dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -467,9 +465,9 @@ static int adrf6780_probe(struct spi_device *spi)
 
 	adrf6780_properties_parse(st);
 
-	st->clkin = devm_clk_get_enabled(&spi->dev, "lo_in");
+	st->clkin = devm_clk_get_enabled(dev, "lo_in");
 	if (IS_ERR(st->clkin))
-		return dev_err_probe(&spi->dev, PTR_ERR(st->clkin),
+		return dev_err_probe(dev, PTR_ERR(st->clkin),
 				     "failed to get the LO input clock\n");
 
 	mutex_init(&st->lock);
@@ -478,11 +476,11 @@ static int adrf6780_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(&spi->dev, adrf6780_powerdown, st);
+	ret = devm_add_action_or_reset(dev, adrf6780_powerdown, st);
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id adrf6780_id[] = {
diff --git a/drivers/iio/gyro/hid-sensor-gyro-3d.c b/drivers/iio/gyro/hid-sensor-gyro-3d.c
index c43990c518f77..c340cc899a7c9 100644
--- a/drivers/iio/gyro/hid-sensor-gyro-3d.c
+++ b/drivers/iio/gyro/hid-sensor-gyro-3d.c
@@ -42,7 +42,7 @@ static const u32 gyro_3d_addresses[GYRO_3D_CHANNEL_MAX] = {
 	HID_USAGE_SENSOR_ANGL_VELOCITY_Z_AXIS
 };
 
-static const u32 gryo_3d_sensitivity_addresses[] = {
+static const u32 gyro_3d_sensitivity_addresses[] = {
 	HID_USAGE_SENSOR_DATA_ANGL_VELOCITY,
 };
 
@@ -297,8 +297,8 @@ static int hid_gyro_3d_probe(struct platform_device *pdev)
 	ret = hid_sensor_parse_common_attributes(hsdev,
 						HID_USAGE_SENSOR_GYRO_3D,
 						&gyro_state->common_attributes,
-						gryo_3d_sensitivity_addresses,
-						ARRAY_SIZE(gryo_3d_sensitivity_addresses));
+						gyro_3d_sensitivity_addresses,
+						ARRAY_SIZE(gyro_3d_sensitivity_addresses));
 	if (ret) {
 		dev_err(&pdev->dev, "failed to setup common attributes\n");
 		return ret;
diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c
index 317e7b217ec6b..d84e04e4b4314 100644
--- a/drivers/iio/gyro/mpu3050-core.c
+++ b/drivers/iio/gyro/mpu3050-core.c
@@ -1129,11 +1129,16 @@ static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq)
 
 	ret = iio_trigger_register(mpu3050->trig);
 	if (ret)
-		return ret;
+		goto err_iio_trigger;
 
 	indio_dev->trig = iio_trigger_get(mpu3050->trig);
 
 	return 0;
+
+err_iio_trigger:
+	free_irq(mpu3050->irq, mpu3050->trig);
+
+	return ret;
 }
 
 int mpu3050_common_probe(struct device *dev,
@@ -1221,12 +1226,6 @@ int mpu3050_common_probe(struct device *dev,
 		goto err_power_down;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(dev, "device register failed\n");
-		goto err_cleanup_buffer;
-	}
-
 	dev_set_drvdata(dev, indio_dev);
 
 	/* Check if we have an assigned IRQ to use as trigger */
@@ -1249,9 +1248,20 @@ int mpu3050_common_probe(struct device *dev,
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_put(dev);
 
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(dev, "device register failed\n");
+		goto err_iio_device_register;
+	}
+
 	return 0;
 
-err_cleanup_buffer:
+err_iio_device_register:
+	pm_runtime_get_sync(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_disable(dev);
+	if (irq)
+		free_irq(mpu3050->irq, mpu3050->trig);
 	iio_triggered_buffer_cleanup(indio_dev);
 err_power_down:
 	mpu3050_power_down(mpu3050);
@@ -1264,13 +1274,13 @@ void mpu3050_common_remove(struct device *dev)
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
 
+	iio_device_unregister(indio_dev);
 	pm_runtime_get_sync(dev);
 	pm_runtime_put_noidle(dev);
 	pm_runtime_disable(dev);
-	iio_triggered_buffer_cleanup(indio_dev);
 	if (mpu3050->irq)
-		free_irq(mpu3050->irq, mpu3050);
-	iio_device_unregister(indio_dev);
+		free_irq(mpu3050->irq, mpu3050->trig);
+	iio_triggered_buffer_cleanup(indio_dev);
 	mpu3050_power_down(mpu3050);
 }
 
diff --git a/drivers/iio/humidity/hdc2010.c b/drivers/iio/humidity/hdc2010.c
index 894a8b4ab193c..1a0f182513815 100644
--- a/drivers/iio/humidity/hdc2010.c
+++ b/drivers/iio/humidity/hdc2010.c
@@ -44,7 +44,6 @@ struct hdc2010_data {
 	struct i2c_client *client;
 	struct mutex lock;
 	u8 measurement_config;
-	u8 interrupt_config;
 	u8 drdy_config;
 };
 
diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c
index 28f0dbd0226cb..1f2af506f4bdd 100644
--- a/drivers/iio/imu/adis16550.c
+++ b/drivers/iio/imu/adis16550.c
@@ -643,12 +643,12 @@ static int adis16550_read_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
 		switch (chan->type) {
 		case IIO_ANGL_VEL:
-			ret = adis16550_get_accl_filter_freq(st, val);
+			ret = adis16550_get_gyro_filter_freq(st, val);
 			if (ret)
 				return ret;
 			return IIO_VAL_INT;
 		case IIO_ACCEL:
-			ret = adis16550_get_gyro_filter_freq(st, val);
+			ret = adis16550_get_accl_filter_freq(st, val);
 			if (ret)
 				return ret;
 			return IIO_VAL_INT;
@@ -681,9 +681,9 @@ static int adis16550_write_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
 		switch (chan->type) {
 		case IIO_ANGL_VEL:
-			return adis16550_set_accl_filter_freq(st, val);
-		case IIO_ACCEL:
 			return adis16550_set_gyro_filter_freq(st, val);
+		case IIO_ACCEL:
+			return adis16550_set_accl_filter_freq(st, val);
 		default:
 			return -EINVAL;
 		}
diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c
index 5f47708b4c5dc..4abb83b75e2e8 100644
--- a/drivers/iio/imu/bmi160/bmi160_core.c
+++ b/drivers/iio/imu/bmi160/bmi160_core.c
@@ -573,12 +573,16 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin,
 		int_out_ctrl_shift = BMI160_INT1_OUT_CTRL_SHIFT;
 		int_latch_mask = BMI160_INT1_LATCH_MASK;
 		int_map_mask = BMI160_INT1_MAP_DRDY_EN;
+		pin_name = "INT1";
 		break;
 	case BMI160_PIN_INT2:
 		int_out_ctrl_shift = BMI160_INT2_OUT_CTRL_SHIFT;
 		int_latch_mask = BMI160_INT2_LATCH_MASK;
 		int_map_mask = BMI160_INT2_MAP_DRDY_EN;
+		pin_name = "INT2";
 		break;
+	default:
+		return -EINVAL;
 	}
 	int_out_ctrl_mask = BMI160_INT_OUT_CTRL_MASK << int_out_ctrl_shift;
 
@@ -612,17 +616,8 @@ static int bmi160_config_pin(struct regmap *regmap, enum bmi160_int_pin pin,
 	ret = bmi160_write_conf_reg(regmap, BMI160_REG_INT_MAP,
 				    int_map_mask, int_map_mask,
 				    write_usleep);
-	if (ret) {
-		switch (pin) {
-		case BMI160_PIN_INT1:
-			pin_name = "INT1";
-			break;
-		case BMI160_PIN_INT2:
-			pin_name = "INT2";
-			break;
-		}
+	if (ret)
 		dev_err(dev, "Failed to configure %s IRQ pin", pin_name);
-	}
 
 	return ret;
 }
diff --git a/drivers/iio/imu/bmi323/bmi323_core.c b/drivers/iio/imu/bmi323/bmi323_core.c
index 6bcb9a4365813..f3d4994233998 100644
--- a/drivers/iio/imu/bmi323/bmi323_core.c
+++ b/drivers/iio/imu/bmi323/bmi323_core.c
@@ -156,7 +156,6 @@ struct bmi323_data {
 	struct iio_mount_matrix orientation;
 	enum bmi323_irq_pin irq_pin;
 	struct iio_trigger *trig;
-	bool drdy_trigger_enabled;
 	enum bmi323_state state;
 	s64 fifo_tstamp, old_fifo_tstamp;
 	u32 odrns[BMI323_SENSORS_CNT];
diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c
index 303bc308f80a8..c96fec2ebb3e7 100644
--- a/drivers/iio/imu/bno055/bno055.c
+++ b/drivers/iio/imu/bno055/bno055.c
@@ -64,7 +64,7 @@
 #define BNO055_GRAVITY_DATA_X_LSB_REG	0x2E
 #define BNO055_GRAVITY_DATA_Y_LSB_REG	0x30
 #define BNO055_GRAVITY_DATA_Z_LSB_REG	0x32
-#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2)
+#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2 + 1)
 #define BNO055_TEMP_REG			0x34
 #define BNO055_CALIB_STAT_REG		0x35
 #define BNO055_CALIB_STAT_MAGN_SHIFT 0
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
index 0ab6eddf0543f..532d5fdffaf8d 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
@@ -1211,7 +1211,7 @@ int inv_icm42600_accel_parse_fifo(struct iio_dev *indio_dev)
 	ssize_t i, size;
 	unsigned int no;
 	const void *accel, *gyro, *timestamp;
-	const int8_t *temp;
+	const s8 *temp;
 	unsigned int odr;
 	int64_t ts_val;
 	/* buffer is copied to userspace, zeroing it to avoid any data leak */
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 55d877745575c..5b28a3ffcc3d0 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -225,6 +225,10 @@ static int st_lsm6dsx_set_fifo_odr(struct st_lsm6dsx_sensor *sensor,
 	const struct st_lsm6dsx_reg *batch_reg;
 	u8 data;
 
+	/* Only internal sensors have a FIFO ODR configuration register. */
+	if (sensor->id >= ARRAY_SIZE(hw->settings->batch))
+		return 0;
+
 	batch_reg = &hw->settings->batch[sensor->id];
 	if (batch_reg->addr) {
 		int val;
@@ -858,12 +862,21 @@ int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw)
 	int i, ret;
 
 	for (i = 0; i < ST_LSM6DSX_ID_MAX; i++) {
+		const struct iio_dev_attr **attrs;
+
 		if (!hw->iio_devs[i])
 			continue;
 
+		/*
+		 * For the accelerometer, allow setting FIFO sampling frequency
+		 * values different from the sensor sampling frequency, which
+		 * may be needed to keep FIFO data rate low while sampling
+		 * acceleration data at high rates for accurate event detection.
+		 */
+		attrs = i == ST_LSM6DSX_ID_ACC ? st_lsm6dsx_buffer_attrs : NULL;
 		ret = devm_iio_kfifo_buffer_setup_ext(hw->dev, hw->iio_devs[i],
 						      &st_lsm6dsx_buffer_ops,
-						      st_lsm6dsx_buffer_attrs);
+						      attrs);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c
index 447b694d6d5f7..1afd00763da9e 100644
--- a/drivers/iio/industrialio-backend.c
+++ b/drivers/iio/industrialio-backend.c
@@ -56,6 +56,7 @@ struct iio_backend {
 	void *priv;
 	const char *name;
 	unsigned int cached_reg_addr;
+	u32 caps;
 	/*
 	 * This index is relative to the frontend. Meaning that for
 	 * frontends with multiple backends, this will be the index of this
@@ -774,6 +775,20 @@ int iio_backend_extend_chan_spec(struct iio_backend *back,
 }
 EXPORT_SYMBOL_NS_GPL(iio_backend_extend_chan_spec, "IIO_BACKEND");
 
+/**
+ * iio_backend_has_caps - Check if backend has specific capabilities
+ * @back: Backend device
+ * @caps: Capabilities to check
+ *
+ * RETURNS:
+ * True if backend has all the requested capabilities, false otherwise.
+ */
+bool iio_backend_has_caps(struct iio_backend *back, u32 caps)
+{
+	return (back->caps & caps) == caps;
+}
+EXPORT_SYMBOL_NS_GPL(iio_backend_has_caps, "IIO_BACKEND");
+
 static void iio_backend_release(void *arg)
 {
 	struct iio_backend *back = arg;
@@ -1114,6 +1129,7 @@ int devm_iio_backend_register(struct device *dev,
 
 	back->ops = info->ops;
 	back->name = info->name;
+	back->caps = info->caps;
 	back->owner = dev->driver->owner;
 	back->dev = dev;
 	back->priv = priv;
diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c
index 4149efcd55390..a0d6fcf2a9c9b 100644
--- a/drivers/iio/industrialio-event.c
+++ b/drivers/iio/industrialio-event.c
@@ -256,6 +256,7 @@ static const char * const iio_ev_info_text[] = {
 	[IIO_EV_INFO_TAP2_MIN_DELAY] = "tap2_min_delay",
 	[IIO_EV_INFO_RUNNING_PERIOD] = "runningperiod",
 	[IIO_EV_INFO_RUNNING_COUNT] = "runningcount",
+	[IIO_EV_INFO_SCALE] = "scale",
 };
 
 static enum iio_event_direction iio_ev_attr_dir(struct iio_dev_attr *attr)
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index 7f34fe7bad07d..17781c12bc858 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -561,10 +561,6 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent,
 	if (!trig)
 		return NULL;
 
-	trig->dev.parent = parent;
-	trig->dev.type = &iio_trig_type;
-	trig->dev.bus = &iio_bus_type;
-	device_initialize(&trig->dev);
 	INIT_WORK(&trig->reenable_work, iio_reenable_work_fn);
 
 	mutex_init(&trig->pool_lock);
@@ -592,6 +588,11 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent,
 				  IRQ_NOREQUEST | IRQ_NOAUTOEN, IRQ_NOPROBE);
 	}
 
+	trig->dev.parent = parent;
+	trig->dev.type = &iio_trig_type;
+	trig->dev.bus = &iio_bus_type;
+	device_initialize(&trig->dev);
+
 	return trig;
 
 free_descs:
@@ -634,9 +635,9 @@ void iio_trigger_free(struct iio_trigger *trig)
 }
 EXPORT_SYMBOL(iio_trigger_free);
 
-static void devm_iio_trigger_release(struct device *dev, void *res)
+static void devm_iio_trigger_release(void *trig)
 {
-	iio_trigger_free(*(struct iio_trigger **)res);
+	iio_trigger_free(trig);
 }
 
 /**
@@ -658,24 +659,20 @@ struct iio_trigger *__devm_iio_trigger_alloc(struct device *parent,
 					     struct module *this_mod,
 					     const char *fmt, ...)
 {
-	struct iio_trigger **ptr, *trig;
+	struct iio_trigger *trig;
 	va_list vargs;
-
-	ptr = devres_alloc(devm_iio_trigger_release, sizeof(*ptr),
-			   GFP_KERNEL);
-	if (!ptr)
-		return NULL;
+	int ret;
 
 	/* use raw alloc_dr for kmalloc caller tracing */
 	va_start(vargs, fmt);
 	trig = viio_trigger_alloc(parent, this_mod, fmt, vargs);
 	va_end(vargs);
-	if (trig) {
-		*ptr = trig;
-		devres_add(parent, ptr);
-	} else {
-		devres_free(ptr);
-	}
+	if (!trig)
+		return NULL;
+
+	ret = devm_add_action_or_reset(parent, devm_iio_trigger_release, trig);
+	if (ret)
+		return NULL;
 
 	return trig;
 }
diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c
index d5d1a8b9c0354..ab229318dce94 100644
--- a/drivers/iio/light/acpi-als.c
+++ b/drivers/iio/light/acpi-als.c
@@ -18,6 +18,7 @@
 #include <linux/err.h>
 #include <linux/irq.h>
 #include <linux/mutex.h>
+#include <linux/platform_device.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
@@ -90,9 +91,9 @@ static int acpi_als_read_value(struct acpi_als *als, char *prop, s32 *val)
 	return 0;
 }
 
-static void acpi_als_notify(struct acpi_device *device, u32 event)
+static void acpi_als_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct iio_dev *indio_dev = acpi_driver_data(device);
+	struct iio_dev *indio_dev = data;
 	struct acpi_als *als = iio_priv(indio_dev);
 
 	if (iio_buffer_enabled(indio_dev) && iio_trigger_using_own(indio_dev)) {
@@ -102,7 +103,7 @@ static void acpi_als_notify(struct acpi_device *device, u32 event)
 			break;
 		default:
 			/* Unhandled event */
-			dev_dbg(&device->dev,
+			dev_dbg(&als->device->dev,
 				"Unhandled ACPI ALS event (%08x)!\n",
 				event);
 		}
@@ -175,9 +176,10 @@ static irqreturn_t acpi_als_trigger_handler(int irq, void *p)
 	return IRQ_HANDLED;
 }
 
-static int acpi_als_add(struct acpi_device *device)
+static int acpi_als_probe(struct platform_device *pdev)
 {
-	struct device *dev = &device->dev;
+	struct device *dev = &pdev->dev;
+	struct acpi_device *device = ACPI_COMPANION(dev);
 	struct iio_dev *indio_dev;
 	struct acpi_als *als;
 	int ret;
@@ -188,7 +190,6 @@ static int acpi_als_add(struct acpi_device *device)
 
 	als = iio_priv(indio_dev);
 
-	device->driver_data = indio_dev;
 	als->device = device;
 	mutex_init(&als->lock);
 
@@ -218,7 +219,18 @@ static int acpi_als_add(struct acpi_device *device)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(dev, indio_dev);
+	ret = devm_iio_device_register(dev, indio_dev);
+	if (ret)
+		return ret;
+
+	return acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+					       acpi_als_notify, indio_dev);
+}
+
+static void acpi_als_remove(struct platform_device *pdev)
+{
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, acpi_als_notify);
 }
 
 static const struct acpi_device_id acpi_als_device_ids[] = {
@@ -228,17 +240,15 @@ static const struct acpi_device_id acpi_als_device_ids[] = {
 
 MODULE_DEVICE_TABLE(acpi, acpi_als_device_ids);
 
-static struct acpi_driver acpi_als_driver = {
-	.name	= "acpi_als",
-	.class	= ACPI_ALS_CLASS,
-	.ids	= acpi_als_device_ids,
-	.ops = {
-		.add	= acpi_als_add,
-		.notify	= acpi_als_notify,
+static struct platform_driver acpi_als_driver = {
+	.probe = acpi_als_probe,
+	.remove = acpi_als_remove,
+	.driver = {
+		.name = "acpi_als",
+		.acpi_match_table = acpi_als_device_ids,
 	},
 };
-
-module_acpi_driver(acpi_als_driver);
+module_platform_driver(acpi_als_driver);
 
 MODULE_AUTHOR("Zhang Rui <rui.zhang@intel.com>");
 MODULE_AUTHOR("Martin Liska <marxin.liska@gmail.com>");
diff --git a/drivers/iio/light/apds9306.c b/drivers/iio/light/apds9306.c
index 7e68cca0edfa5..5ca4c87524fe2 100644
--- a/drivers/iio/light/apds9306.c
+++ b/drivers/iio/light/apds9306.c
@@ -168,7 +168,6 @@ struct apds9306_regfields {
  *         respectively.
  * @regmap: Regmap structure pointer
  * @rf: Regmap register fields structure
- * @nlux_per_count: Nano lux per ADC count for a particular model
  * @read_data_available: Flag set by IRQ handler for ADC data available
  */
 struct apds9306_data {
@@ -180,7 +179,6 @@ struct apds9306_data {
 	struct regmap *regmap;
 	struct apds9306_regfields rf;
 
-	int nlux_per_count;
 	int read_data_available;
 };
 
@@ -1176,7 +1174,7 @@ static int apds9306_init_iio_gts(struct apds9306_data *data)
 
 static void apds9306_powerdown(void *ptr)
 {
-	struct apds9306_data *data = (struct apds9306_data *)ptr;
+	struct apds9306_data *data = ptr;
 	struct apds9306_regfields *rf = &data->rf;
 	int ret;
 
diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c
index c7df4b258e2c2..7e388319ee2e3 100644
--- a/drivers/iio/light/gp2ap020a00f.c
+++ b/drivers/iio/light/gp2ap020a00f.c
@@ -31,12 +31,14 @@
  * the other one.
  */
 
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irq_work.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
@@ -44,6 +46,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 #include <linux/unaligned.h>
+
 #include <linux/iio/buffer.h>
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
@@ -52,8 +55,6 @@
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 
-#define GP2A_I2C_NAME "gp2ap020a00f"
-
 /* Registers */
 #define GP2AP020A00F_OP_REG	0x00 /* Basic operations */
 #define GP2AP020A00F_ALS_REG	0x01 /* ALS related settings */
@@ -174,10 +175,8 @@
 #define GP2AP020A00F_CHAN_TIMESTAMP		3
 
 #define GP2AP020A00F_DATA_READY_TIMEOUT		msecs_to_jiffies(1000)
-#define GP2AP020A00F_DATA_REG(chan)		(GP2AP020A00F_D0_L_REG + \
-							(chan) * 2)
-#define GP2AP020A00F_THRESH_REG(th_val_id)	(GP2AP020A00F_TL_L_REG + \
-							(th_val_id) * 2)
+#define GP2AP020A00F_DATA_REG(chan)		(GP2AP020A00F_D0_L_REG + (chan) * 2)
+#define GP2AP020A00F_THRESH_REG(th_val_id)	(GP2AP020A00F_TL_L_REG + (th_val_id) * 2)
 #define GP2AP020A00F_THRESH_VAL_ID(reg_addr)	((reg_addr - 4) / 2)
 
 #define GP2AP020A00F_SUBTRACT_MODE	0
@@ -194,7 +193,7 @@ enum gp2ap020a00f_opmode {
 	GP2AP020A00F_OPMODE_ALS_AND_PS,
 	GP2AP020A00F_OPMODE_PROX_DETECT,
 	GP2AP020A00F_OPMODE_SHUTDOWN,
-	GP2AP020A00F_NUM_OPMODES,
+	GP2AP020A00F_NUM_OPMODES
 };
 
 enum gp2ap020a00f_cmd {
@@ -246,7 +245,6 @@ struct gp2ap020a00f_data {
 	struct iio_trigger *trig;
 	struct regmap *regmap;
 	unsigned int thresh_val[4];
-	u8 debug_reg_addr;
 	struct irq_work work;
 	wait_queue_head_t data_ready_queue;
 };
@@ -389,20 +387,17 @@ static int gp2ap020a00f_set_operation_mode(struct gp2ap020a00f_data *data,
 		}
 
 		err = regmap_update_bits(data->regmap, GP2AP020A00F_ALS_REG,
-			GP2AP020A00F_PRST_MASK, opmode_regs_settings[op]
-								.als_reg);
+			GP2AP020A00F_PRST_MASK, opmode_regs_settings[op].als_reg);
 		if (err < 0)
 			return err;
 
 		err = regmap_update_bits(data->regmap, GP2AP020A00F_PS_REG,
-			GP2AP020A00F_INTTYPE_MASK, opmode_regs_settings[op]
-								.ps_reg);
+			GP2AP020A00F_INTTYPE_MASK, opmode_regs_settings[op].ps_reg);
 		if (err < 0)
 			return err;
 
 		err = regmap_update_bits(data->regmap, GP2AP020A00F_LED_REG,
-			GP2AP020A00F_PIN_MASK, opmode_regs_settings[op]
-								.led_reg);
+			GP2AP020A00F_PIN_MASK, opmode_regs_settings[op].led_reg);
 		if (err < 0)
 			return err;
 	}
@@ -453,15 +448,13 @@ static int gp2ap020a00f_write_event_threshold(struct gp2ap020a00f_data *data,
 		 */
 		thresh_reg_val = data->thresh_val[th_val_id] / 16;
 	else
-		thresh_reg_val = data->thresh_val[th_val_id] > 16000 ?
-					16000 :
-					data->thresh_val[th_val_id];
+		thresh_reg_val = min(data->thresh_val[th_val_id], 16000U);
 
 	thresh_buf = cpu_to_le16(thresh_reg_val);
 
 	return regmap_bulk_write(data->regmap,
 				 GP2AP020A00F_THRESH_REG(th_val_id),
-				 (u8 *)&thresh_buf, 2);
+				 &thresh_buf, sizeof(thresh_buf));
 }
 
 static int gp2ap020a00f_alter_opmode(struct gp2ap020a00f_data *data,
@@ -493,27 +486,24 @@ static int gp2ap020a00f_alter_opmode(struct gp2ap020a00f_data *data,
 static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 					enum gp2ap020a00f_cmd cmd)
 {
-	int err = 0;
+	int err;
 
 	switch (cmd) {
 	case GP2AP020A00F_CMD_READ_RAW_CLEAR:
 		if (data->cur_opmode != GP2AP020A00F_OPMODE_SHUTDOWN)
 			return -EBUSY;
-		err = gp2ap020a00f_set_operation_mode(data,
+		return gp2ap020a00f_set_operation_mode(data,
 					GP2AP020A00F_OPMODE_READ_RAW_CLEAR);
-		break;
 	case GP2AP020A00F_CMD_READ_RAW_IR:
 		if (data->cur_opmode != GP2AP020A00F_OPMODE_SHUTDOWN)
 			return -EBUSY;
-		err = gp2ap020a00f_set_operation_mode(data,
+		return gp2ap020a00f_set_operation_mode(data,
 					GP2AP020A00F_OPMODE_READ_RAW_IR);
-		break;
 	case GP2AP020A00F_CMD_READ_RAW_PROXIMITY:
 		if (data->cur_opmode != GP2AP020A00F_OPMODE_SHUTDOWN)
 			return -EBUSY;
-		err = gp2ap020a00f_set_operation_mode(data,
+		return gp2ap020a00f_set_operation_mode(data,
 					GP2AP020A00F_OPMODE_READ_RAW_PROXIMITY);
-		break;
 	case GP2AP020A00F_CMD_TRIGGER_CLEAR_EN:
 		if (data->cur_opmode == GP2AP020A00F_OPMODE_PROX_DETECT)
 			return -EBUSY;
@@ -521,16 +511,17 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 			err = gp2ap020a00f_alter_opmode(data,
 						GP2AP020A00F_OPMODE_ALS,
 						GP2AP020A00F_ADD_MODE);
+		else
+			err = 0;
 		set_bit(GP2AP020A00F_FLAG_ALS_CLEAR_TRIGGER, &data->flags);
-		break;
+		return err;
 	case GP2AP020A00F_CMD_TRIGGER_CLEAR_DIS:
 		clear_bit(GP2AP020A00F_FLAG_ALS_CLEAR_TRIGGER, &data->flags);
 		if (gp2ap020a00f_als_enabled(data))
 			break;
-		err = gp2ap020a00f_alter_opmode(data,
+		return gp2ap020a00f_alter_opmode(data,
 						GP2AP020A00F_OPMODE_ALS,
 						GP2AP020A00F_SUBTRACT_MODE);
-		break;
 	case GP2AP020A00F_CMD_TRIGGER_IR_EN:
 		if (data->cur_opmode == GP2AP020A00F_OPMODE_PROX_DETECT)
 			return -EBUSY;
@@ -538,16 +529,17 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 			err = gp2ap020a00f_alter_opmode(data,
 						GP2AP020A00F_OPMODE_ALS,
 						GP2AP020A00F_ADD_MODE);
+		else
+			err = 0;
 		set_bit(GP2AP020A00F_FLAG_ALS_IR_TRIGGER, &data->flags);
-		break;
+		return err;
 	case GP2AP020A00F_CMD_TRIGGER_IR_DIS:
 		clear_bit(GP2AP020A00F_FLAG_ALS_IR_TRIGGER, &data->flags);
 		if (gp2ap020a00f_als_enabled(data))
 			break;
-		err = gp2ap020a00f_alter_opmode(data,
+		return gp2ap020a00f_alter_opmode(data,
 						GP2AP020A00F_OPMODE_ALS,
 						GP2AP020A00F_SUBTRACT_MODE);
-		break;
 	case GP2AP020A00F_CMD_TRIGGER_PROX_EN:
 		if (data->cur_opmode == GP2AP020A00F_OPMODE_PROX_DETECT)
 			return -EBUSY;
@@ -555,13 +547,12 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 						GP2AP020A00F_OPMODE_PS,
 						GP2AP020A00F_ADD_MODE);
 		set_bit(GP2AP020A00F_FLAG_PROX_TRIGGER, &data->flags);
-		break;
+		return err;
 	case GP2AP020A00F_CMD_TRIGGER_PROX_DIS:
 		clear_bit(GP2AP020A00F_FLAG_PROX_TRIGGER, &data->flags);
-		err = gp2ap020a00f_alter_opmode(data,
+		return gp2ap020a00f_alter_opmode(data,
 						GP2AP020A00F_OPMODE_PS,
 						GP2AP020A00F_SUBTRACT_MODE);
-		break;
 	case GP2AP020A00F_CMD_ALS_HIGH_EV_EN:
 		if (test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV, &data->flags))
 			return 0;
@@ -575,9 +566,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 				return err;
 		}
 		set_bit(GP2AP020A00F_FLAG_ALS_RISING_EV, &data->flags);
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_TH, true);
-		break;
 	case GP2AP020A00F_CMD_ALS_HIGH_EV_DIS:
 		if (!test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV, &data->flags))
 			return 0;
@@ -589,9 +579,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 			if (err < 0)
 				return err;
 		}
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_TH, false);
-		break;
 	case GP2AP020A00F_CMD_ALS_LOW_EV_EN:
 		if (test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &data->flags))
 			return 0;
@@ -605,9 +594,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 				return err;
 		}
 		set_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &data->flags);
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_TL, true);
-		break;
 	case GP2AP020A00F_CMD_ALS_LOW_EV_DIS:
 		if (!test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &data->flags))
 			return 0;
@@ -619,9 +607,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 			if (err < 0)
 				return err;
 		}
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_TL, false);
-		break;
 	case GP2AP020A00F_CMD_PROX_HIGH_EV_EN:
 		if (test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV, &data->flags))
 			return 0;
@@ -635,9 +622,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 				return err;
 		}
 		set_bit(GP2AP020A00F_FLAG_PROX_RISING_EV, &data->flags);
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_PH, true);
-		break;
 	case GP2AP020A00F_CMD_PROX_HIGH_EV_DIS:
 		if (!test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV, &data->flags))
 			return 0;
@@ -646,9 +632,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 					GP2AP020A00F_OPMODE_SHUTDOWN);
 		if (err < 0)
 			return err;
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_PH, false);
-		break;
 	case GP2AP020A00F_CMD_PROX_LOW_EV_EN:
 		if (test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV, &data->flags))
 			return 0;
@@ -662,9 +647,8 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 				return err;
 		}
 		set_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV, &data->flags);
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_PL, true);
-		break;
 	case GP2AP020A00F_CMD_PROX_LOW_EV_DIS:
 		if (!test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV, &data->flags))
 			return 0;
@@ -673,12 +657,11 @@ static int gp2ap020a00f_exec_cmd(struct gp2ap020a00f_data *data,
 					GP2AP020A00F_OPMODE_SHUTDOWN);
 		if (err < 0)
 			return err;
-		err =  gp2ap020a00f_write_event_threshold(data,
+		return gp2ap020a00f_write_event_threshold(data,
 					GP2AP020A00F_THRESH_PL, false);
-		break;
 	}
 
-	return err;
+	return 0;
 }
 
 static int wait_conversion_complete_irq(struct gp2ap020a00f_data *data)
@@ -697,18 +680,18 @@ static int wait_conversion_complete_irq(struct gp2ap020a00f_data *data)
 static int gp2ap020a00f_read_output(struct gp2ap020a00f_data *data,
 					unsigned int output_reg, int *val)
 {
-	u8 reg_buf[2];
+	__le16 reg_buf;
 	int err;
 
 	err = wait_conversion_complete_irq(data);
 	if (err < 0)
 		dev_dbg(&data->client->dev, "data ready timeout\n");
 
-	err = regmap_bulk_read(data->regmap, output_reg, reg_buf, 2);
+	err = regmap_bulk_read(data->regmap, output_reg, &reg_buf, sizeof(reg_buf));
 	if (err < 0)
 		return err;
 
-	*val = le16_to_cpup((__le16 *)reg_buf);
+	*val = le16_to_cpu(reg_buf);
 
 	return err;
 }
@@ -866,13 +849,13 @@ static irqreturn_t gp2ap020a00f_thresh_event_handler(int irq, void *data)
 {
 	struct iio_dev *indio_dev = data;
 	struct gp2ap020a00f_data *priv = iio_priv(indio_dev);
-	u8 op_reg_flags, d0_reg_buf[2];
 	unsigned int output_val, op_reg_val;
+	__le16 d0_reg_buf;
+	u8 op_reg_flags;
 	int thresh_val_id, ret;
 
 	/* Read interrupt flags */
-	ret = regmap_read(priv->regmap, GP2AP020A00F_OP_REG,
-							&op_reg_val);
+	ret = regmap_read(priv->regmap, GP2AP020A00F_OP_REG, &op_reg_val);
 	if (ret < 0)
 		goto done;
 
@@ -884,8 +867,7 @@ static irqreturn_t gp2ap020a00f_thresh_event_handler(int irq, void *data)
 
 	/* Clear interrupt flags (if not in INTTYPE_PULSE mode) */
 	if (priv->cur_opmode != GP2AP020A00F_OPMODE_PROX_DETECT) {
-		ret = regmap_write(priv->regmap, GP2AP020A00F_OP_REG,
-								op_reg_val);
+		ret = regmap_write(priv->regmap, GP2AP020A00F_OP_REG, op_reg_val);
 		if (ret < 0)
 			goto done;
 	}
@@ -895,11 +877,11 @@ static irqreturn_t gp2ap020a00f_thresh_event_handler(int irq, void *data)
 		 * transition is required.
 		 */
 		ret = regmap_bulk_read(priv->regmap, GP2AP020A00F_D0_L_REG,
-							d0_reg_buf, 2);
+				       &d0_reg_buf, sizeof(d0_reg_buf));
 		if (ret < 0)
 			goto done;
 
-		output_val = le16_to_cpup((__le16 *)d0_reg_buf);
+		output_val = le16_to_cpu(d0_reg_buf);
 
 		if (gp2ap020a00f_adjust_lux_mode(priv, output_val))
 			goto done;
@@ -966,17 +948,15 @@ static irqreturn_t gp2ap020a00f_trigger_handler(int irq, void *data)
 	int i, out_val, ret;
 
 	iio_for_each_active_channel(indio_dev, i) {
-		ret = regmap_bulk_read(priv->regmap,
-				GP2AP020A00F_DATA_REG(i),
-				&priv->buffer[d_size], 2);
+		ret = regmap_bulk_read(priv->regmap, GP2AP020A00F_DATA_REG(i),
+				       &priv->buffer[d_size], 2);
 		if (ret < 0)
 			goto done;
 
 		if (i == GP2AP020A00F_SCAN_MODE_LIGHT_CLEAR ||
 		    i == GP2AP020A00F_SCAN_MODE_LIGHT_IR) {
-			out_val = le16_to_cpup((__le16 *)&priv->buffer[d_size]);
+			out_val = get_unaligned_le16(&priv->buffer[d_size]);
 			gp2ap020a00f_output_to_lux(priv, &out_val);
-
 			put_unaligned_le32(out_val, &priv->buffer[d_size]);
 			d_size += 4;
 		} else {
@@ -984,16 +964,15 @@ static irqreturn_t gp2ap020a00f_trigger_handler(int irq, void *data)
 		}
 	}
 
-	iio_push_to_buffers_with_timestamp(indio_dev, priv->buffer,
-		pf->timestamp);
+	iio_push_to_buffers_with_timestamp(indio_dev, priv->buffer, pf->timestamp);
 done:
 	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
 }
 
-static u8 gp2ap020a00f_get_thresh_reg(const struct iio_chan_spec *chan,
-					     enum iio_event_direction event_dir)
+static int gp2ap020a00f_get_thresh_reg(const struct iio_chan_spec *chan,
+				       enum iio_event_direction event_dir)
 {
 	switch (chan->type) {
 	case IIO_PROXIMITY:
@@ -1007,10 +986,8 @@ static u8 gp2ap020a00f_get_thresh_reg(const struct iio_chan_spec *chan,
 		else
 			return GP2AP020A00F_TL_L_REG;
 	default:
-		break;
+		return -EINVAL;
 	}
-
-	return -EINVAL;
 }
 
 static int gp2ap020a00f_write_event_val(struct iio_dev *indio_dev,
@@ -1023,53 +1000,41 @@ static int gp2ap020a00f_write_event_val(struct iio_dev *indio_dev,
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
 	bool event_en = false;
 	u8 thresh_val_id;
-	u8 thresh_reg_l;
-	int err = 0;
+	int thresh_reg_l;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	thresh_reg_l = gp2ap020a00f_get_thresh_reg(chan, dir);
-	thresh_val_id = GP2AP020A00F_THRESH_VAL_ID(thresh_reg_l);
+	if (thresh_reg_l < 0)
+		return thresh_reg_l;
 
-	if (thresh_val_id > GP2AP020A00F_THRESH_PH) {
-		err = -EINVAL;
-		goto error_unlock;
-	}
+	thresh_val_id = GP2AP020A00F_THRESH_VAL_ID(thresh_reg_l);
+	if (thresh_val_id > GP2AP020A00F_THRESH_PH)
+		return -EINVAL;
 
 	switch (thresh_reg_l) {
 	case GP2AP020A00F_TH_L_REG:
-		event_en = test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV,
-							&data->flags);
+		event_en = test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV, &data->flags);
 		break;
 	case GP2AP020A00F_TL_L_REG:
-		event_en = test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV,
-							&data->flags);
+		event_en = test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &data->flags);
 		break;
 	case GP2AP020A00F_PH_L_REG:
-		if (val == 0) {
-			err = -EINVAL;
-			goto error_unlock;
-		}
-		event_en = test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV,
-							&data->flags);
+		if (val == 0)
+			return -EINVAL;
+
+		event_en = test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV, &data->flags);
 		break;
 	case GP2AP020A00F_PL_L_REG:
-		if (val == 0) {
-			err = -EINVAL;
-			goto error_unlock;
-		}
-		event_en = test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV,
-							&data->flags);
+		if (val == 0)
+			return -EINVAL;
+
+		event_en = test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV, &data->flags);
 		break;
 	}
 
 	data->thresh_val[thresh_val_id] = val;
-	err =  gp2ap020a00f_write_event_threshold(data, thresh_val_id,
-							event_en);
-error_unlock:
-	mutex_unlock(&data->lock);
-
-	return err;
+	return gp2ap020a00f_write_event_threshold(data, thresh_val_id, event_en);
 }
 
 static int gp2ap020a00f_read_event_val(struct iio_dev *indio_dev,
@@ -1080,24 +1045,19 @@ static int gp2ap020a00f_read_event_val(struct iio_dev *indio_dev,
 				       int *val, int *val2)
 {
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
-	u8 thresh_reg_l;
-	int err = IIO_VAL_INT;
+	int thresh_reg_l;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	thresh_reg_l = gp2ap020a00f_get_thresh_reg(chan, dir);
-
-	if (thresh_reg_l > GP2AP020A00F_PH_L_REG) {
-		err = -EINVAL;
-		goto error_unlock;
-	}
+	if (thresh_reg_l < 0)
+		return thresh_reg_l;
+	if (thresh_reg_l > GP2AP020A00F_PH_L_REG)
+		return -EINVAL;
 
 	*val = data->thresh_val[GP2AP020A00F_THRESH_VAL_ID(thresh_reg_l)];
 
-error_unlock:
-	mutex_unlock(&data->lock);
-
-	return err;
+	return IIO_VAL_INT;
 }
 
 static int gp2ap020a00f_write_prox_event_config(struct iio_dev *indio_dev,
@@ -1163,32 +1123,25 @@ static int gp2ap020a00f_write_event_config(struct iio_dev *indio_dev,
 {
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
 	enum gp2ap020a00f_cmd cmd;
-	int err;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	switch (chan->type) {
 	case IIO_PROXIMITY:
-		err = gp2ap020a00f_write_prox_event_config(indio_dev, state);
-		break;
+		return gp2ap020a00f_write_prox_event_config(indio_dev, state);
 	case IIO_LIGHT:
 		if (dir == IIO_EV_DIR_RISING) {
 			cmd = state ? GP2AP020A00F_CMD_ALS_HIGH_EV_EN :
 				      GP2AP020A00F_CMD_ALS_HIGH_EV_DIS;
-			err = gp2ap020a00f_exec_cmd(data, cmd);
+			return gp2ap020a00f_exec_cmd(data, cmd);
 		} else {
 			cmd = state ? GP2AP020A00F_CMD_ALS_LOW_EV_EN :
 				      GP2AP020A00F_CMD_ALS_LOW_EV_DIS;
-			err = gp2ap020a00f_exec_cmd(data, cmd);
+			return gp2ap020a00f_exec_cmd(data, cmd);
 		}
-		break;
 	default:
-		err = -EINVAL;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&data->lock);
-
-	return err;
 }
 
 static int gp2ap020a00f_read_event_config(struct iio_dev *indio_dev,
@@ -1197,40 +1150,29 @@ static int gp2ap020a00f_read_event_config(struct iio_dev *indio_dev,
 					   enum iio_event_direction dir)
 {
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
-	int event_en = 0;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	switch (chan->type) {
 	case IIO_PROXIMITY:
 		if (dir == IIO_EV_DIR_RISING)
-			event_en = test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV,
-								&data->flags);
+			return test_bit(GP2AP020A00F_FLAG_PROX_RISING_EV, &data->flags);
 		else
-			event_en = test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV,
-								&data->flags);
-		break;
+			return test_bit(GP2AP020A00F_FLAG_PROX_FALLING_EV, &data->flags);
 	case IIO_LIGHT:
 		if (dir == IIO_EV_DIR_RISING)
-			event_en = test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV,
-								&data->flags);
+			return test_bit(GP2AP020A00F_FLAG_ALS_RISING_EV, &data->flags);
 		else
-			event_en = test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV,
-								&data->flags);
-		break;
+			return test_bit(GP2AP020A00F_FLAG_ALS_FALLING_EV, &data->flags);
 	default:
-		event_en = -EINVAL;
-		break;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&data->lock);
-
-	return event_en;
 }
 
 static int gp2ap020a00f_read_channel(struct gp2ap020a00f_data *data,
 				struct iio_chan_spec const *chan, int *val)
 {
+	struct device *dev = &data->client->dev;
 	enum gp2ap020a00f_cmd cmd;
 	int err;
 
@@ -1250,27 +1192,23 @@ static int gp2ap020a00f_read_channel(struct gp2ap020a00f_data *data,
 
 	err = gp2ap020a00f_exec_cmd(data, cmd);
 	if (err < 0) {
-		dev_err(&data->client->dev,
-			"gp2ap020a00f_exec_cmd failed\n");
-		goto error_ret;
+		dev_err(dev, "gp2ap020a00f_exec_cmd failed\n");
+		return err;
 	}
 
 	err = gp2ap020a00f_read_output(data, chan->address, val);
 	if (err < 0)
-		dev_err(&data->client->dev,
-			"gp2ap020a00f_read_output failed\n");
+		dev_err(dev, "gp2ap020a00f_read_output failed\n");
 
 	err = gp2ap020a00f_set_operation_mode(data,
 					GP2AP020A00F_OPMODE_SHUTDOWN);
 	if (err < 0)
-		dev_err(&data->client->dev,
-			"Failed to shut down the device.\n");
+		dev_err(dev, "Failed to shut down the device.\n");
 
 	if (cmd == GP2AP020A00F_CMD_READ_RAW_CLEAR ||
 	    cmd == GP2AP020A00F_CMD_READ_RAW_IR)
 		gp2ap020a00f_output_to_lux(data, val);
 
-error_ret:
 	return err;
 }
 
@@ -1383,9 +1321,9 @@ static const struct iio_info gp2ap020a00f_info = {
 static int gp2ap020a00f_buffer_postenable(struct iio_dev *indio_dev)
 {
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
-	int i, err = 0;
+	int i, err;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	/*
 	 * Enable triggers according to the scan_mask. Enabling either
@@ -1409,28 +1347,27 @@ static int gp2ap020a00f_buffer_postenable(struct iio_dev *indio_dev)
 			err = gp2ap020a00f_exec_cmd(data,
 					GP2AP020A00F_CMD_TRIGGER_PROX_EN);
 			break;
+		default:
+			err = -EINVAL;
+			break;
 		}
+		if (err)
+			return err;
 	}
 
-	if (err < 0)
-		goto error_unlock;
-
 	data->buffer = kmalloc(indio_dev->scan_bytes, GFP_KERNEL);
 	if (!data->buffer)
-		err = -ENOMEM;
-
-error_unlock:
-	mutex_unlock(&data->lock);
+		return -ENOMEM;
 
-	return err;
+	return 0;
 }
 
 static int gp2ap020a00f_buffer_predisable(struct iio_dev *indio_dev)
 {
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
-	int i, err = 0;
+	int i, err;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	iio_for_each_active_channel(indio_dev, i) {
 		switch (i) {
@@ -1446,15 +1383,16 @@ static int gp2ap020a00f_buffer_predisable(struct iio_dev *indio_dev)
 			err = gp2ap020a00f_exec_cmd(data,
 					GP2AP020A00F_CMD_TRIGGER_PROX_DIS);
 			break;
+		default:
+			err = -EINVAL;
+			break;
 		}
+		if (err)
+			return err;
 	}
 
-	if (err == 0)
-		kfree(data->buffer);
-
-	mutex_unlock(&data->lock);
-
-	return err;
+	kfree(data->buffer);
+	return 0;
 }
 
 static const struct iio_buffer_setup_ops gp2ap020a00f_buffer_setup_ops = {
@@ -1465,18 +1403,19 @@ static const struct iio_buffer_setup_ops gp2ap020a00f_buffer_setup_ops = {
 static int gp2ap020a00f_probe(struct i2c_client *client)
 {
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
+	struct device *dev = &client->dev;
 	struct gp2ap020a00f_data *data;
 	struct iio_dev *indio_dev;
 	struct regmap *regmap;
 	int err;
 
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
 	if (!indio_dev)
 		return -ENOMEM;
 
 	data = iio_priv(indio_dev);
 
-	data->vled_reg = devm_regulator_get(&client->dev, "vled");
+	data->vled_reg = devm_regulator_get(dev, "vled");
 	if (IS_ERR(data->vled_reg))
 		return PTR_ERR(data->vled_reg);
 
@@ -1486,7 +1425,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 
 	regmap = devm_regmap_init_i2c(client, &gp2ap020a00f_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Regmap initialization failed.\n");
+		dev_err(dev, "Regmap initialization failed.\n");
 		err = PTR_ERR(regmap);
 		goto error_regulator_disable;
 	}
@@ -1497,7 +1436,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 			ARRAY_SIZE(gp2ap020a00f_reg_init_tab));
 
 	if (err < 0) {
-		dev_err(&client->dev, "Device initialization failed.\n");
+		dev_err(dev, "Device initialization failed.\n");
 		goto error_regulator_disable;
 	}
 
@@ -1522,11 +1461,10 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 		goto error_regulator_disable;
 
 	/* Allocate trigger */
-	data->trig = devm_iio_trigger_alloc(&client->dev, "%s-trigger",
-							indio_dev->name);
+	data->trig = devm_iio_trigger_alloc(dev, "%s-trigger", indio_dev->name);
 	if (data->trig == NULL) {
 		err = -ENOMEM;
-		dev_err(&indio_dev->dev, "Failed to allocate iio trigger.\n");
+		dev_err(dev, "Failed to allocate iio trigger.\n");
 		goto error_uninit_buffer;
 	}
 
@@ -1538,7 +1476,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 				   "gp2ap020a00f_als_event",
 				   indio_dev);
 	if (err < 0) {
-		dev_err(&client->dev, "Irq request failed.\n");
+		dev_err(dev, "Irq request failed.\n");
 		goto error_uninit_buffer;
 	}
 
@@ -1546,7 +1484,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 
 	err = iio_trigger_register(data->trig);
 	if (err < 0) {
-		dev_err(&client->dev, "Failed to register iio trigger.\n");
+		dev_err(dev, "Failed to register iio trigger.\n");
 		goto error_free_irq;
 	}
 
@@ -1572,12 +1510,12 @@ static void gp2ap020a00f_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
+	struct device *dev = &client->dev;
 	int err;
 
-	err = gp2ap020a00f_set_operation_mode(data,
-					GP2AP020A00F_OPMODE_SHUTDOWN);
+	err = gp2ap020a00f_set_operation_mode(data, GP2AP020A00F_OPMODE_SHUTDOWN);
 	if (err < 0)
-		dev_err(&indio_dev->dev, "Failed to power off the device.\n");
+		dev_err(dev, "Failed to power off the device.\n");
 
 	iio_device_unregister(indio_dev);
 	iio_trigger_unregister(data->trig);
@@ -1587,10 +1525,9 @@ static void gp2ap020a00f_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id gp2ap020a00f_id[] = {
-	{ GP2A_I2C_NAME },
+	{ "gp2ap020a00f" },
 	{ }
 };
-
 MODULE_DEVICE_TABLE(i2c, gp2ap020a00f_id);
 
 static const struct of_device_id gp2ap020a00f_of_match[] = {
@@ -1601,14 +1538,13 @@ MODULE_DEVICE_TABLE(of, gp2ap020a00f_of_match);
 
 static struct i2c_driver gp2ap020a00f_driver = {
 	.driver = {
-		.name	= GP2A_I2C_NAME,
+		.name	= "gp2ap020a00f",
 		.of_match_table = gp2ap020a00f_of_match,
 	},
 	.probe		= gp2ap020a00f_probe,
 	.remove		= gp2ap020a00f_remove,
 	.id_table	= gp2ap020a00f_id,
 };
-
 module_i2c_driver(gp2ap020a00f_driver);
 
 MODULE_AUTHOR("Jacek Anaszewski <j.anaszewski@samsung.com>");
diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
index 022e0693983bd..4d99ae336f610 100644
--- a/drivers/iio/light/ltr501.c
+++ b/drivers/iio/light/ltr501.c
@@ -754,7 +754,7 @@ static int ltr501_get_gain_index(const struct ltr501_gain *gain, int size,
 		if (val == gain[i].scale && val2 == gain[i].uscale)
 			return i;
 
-	return -1;
+	return -EINVAL;
 }
 
 static int __ltr501_write_raw(struct iio_dev *indio_dev,
@@ -773,7 +773,7 @@ static int __ltr501_write_raw(struct iio_dev *indio_dev,
 						  info->als_gain_tbl_size,
 						  val, val2);
 			if (i < 0)
-				return -EINVAL;
+				return i;
 
 			data->als_contr &= ~info->als_gain_mask;
 			data->als_contr |= i << info->als_gain_shift;
@@ -785,7 +785,7 @@ static int __ltr501_write_raw(struct iio_dev *indio_dev,
 						  info->ps_gain_tbl_size,
 						  val, val2);
 			if (i < 0)
-				return -EINVAL;
+				return i;
 
 			data->ps_contr &= ~LTR501_CONTR_PS_GAIN_MASK;
 			data->ps_contr |= i << LTR501_CONTR_PS_GAIN_SHIFT;
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index a36c238136792..5e03c3d8874b3 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -185,6 +185,7 @@ static const int vcnl4040_ps_oversampling_ratio[] = {1, 2, 4, 8};
 #define VCNL4000_SLEEP_DELAY_MS	2000 /* before we enter pm_runtime_suspend */
 
 enum vcnl4000_device_ids {
+	CM36672P,
 	VCNL4000,
 	VCNL4010,
 	VCNL4040,
@@ -235,6 +236,8 @@ struct vcnl4000_chip_spec {
 };
 
 static const struct i2c_device_id vcnl4000_id[] = {
+	{ "cm36672p", CM36672P },
+	{ "cm36686", VCNL4040 },
 	{ "vcnl4000", VCNL4000 },
 	{ "vcnl4010", VCNL4010 },
 	{ "vcnl4020", VCNL4010 },
@@ -1842,6 +1845,22 @@ static const struct iio_chan_spec vcnl4040_channels[] = {
 	}
 };
 
+static const struct iio_chan_spec cm36672p_channels[] = {
+	{
+		.type = IIO_PROXIMITY,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_INT_TIME) |
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) |
+			BIT(IIO_CHAN_INFO_CALIBBIAS),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_INT_TIME) |
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) |
+			BIT(IIO_CHAN_INFO_CALIBBIAS),
+		.ext_info = vcnl4000_ext_info,
+		.event_spec = vcnl4040_event_spec,
+		.num_event_specs = ARRAY_SIZE(vcnl4040_event_spec),
+	},
+};
+
 static const struct iio_info vcnl4000_info = {
 	.read_raw = vcnl4000_read_raw,
 };
@@ -1867,6 +1886,19 @@ static const struct iio_info vcnl4040_info = {
 };
 
 static const struct vcnl4000_chip_spec vcnl4000_chip_spec_cfg[] = {
+	[CM36672P] = {
+		.prod = "CM36672P",
+		.init = vcnl4200_init,
+		.measure_proximity = vcnl4200_measure_proximity,
+		.set_power_state = vcnl4200_set_power_state,
+		.channels = cm36672p_channels,
+		.num_channels = ARRAY_SIZE(cm36672p_channels),
+		.info = &vcnl4040_info,
+		.irq_thread = vcnl4040_irq_thread,
+		.int_reg = VCNL4040_INT_FLAGS,
+		.ps_it_times = &vcnl4040_ps_it_times,
+		.num_ps_it_times = ARRAY_SIZE(vcnl4040_ps_it_times),
+	},
 	[VCNL4000] = {
 		.prod = "VCNL4000",
 		.init = vcnl4000_init,
@@ -2033,6 +2065,15 @@ static int vcnl4000_probe(struct i2c_client *client)
 }
 
 static const struct of_device_id vcnl_4000_of_match[] = {
+	{
+		.compatible = "capella,cm36672p",
+		.data = (void *)CM36672P,
+	},
+	/* Capella CM36686 is fully compatible with Vishay VCNL4040 */
+	{
+		.compatible = "capella,cm36686",
+		.data = (void *)VCNL4040,
+	},
 	{
 		.compatible = "vishay,vcnl4000",
 		.data = (void *)VCNL4000,
diff --git a/drivers/iio/light/zopt2201.c b/drivers/iio/light/zopt2201.c
index 1dba1b949cc32..0990e4d266ebe 100644
--- a/drivers/iio/light/zopt2201.c
+++ b/drivers/iio/light/zopt2201.c
@@ -10,17 +10,17 @@
  * TODO: interrupt support, ALS/UVB raw mode
  */
 
-#include <linux/module.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/err.h>
 #include <linux/i2c.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/err.h>
-#include <linux/delay.h>
+#include <linux/unaligned.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
-#include <linux/unaligned.h>
-
 #define ZOPT2201_DRV_NAME "zopt2201"
 
 /* Registers */
@@ -186,10 +186,10 @@ static int zopt2201_read(struct zopt2201_data *data, u8 reg)
 	u8 buf[3];
 	int ret;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 	ret = zopt2201_enable_mode(data, reg == ZOPT2201_UVB_DATA);
 	if (ret < 0)
-		goto fail;
+		return ret;
 
 	while (tries--) {
 		unsigned long t = zopt2201_resolution[data->res].us;
@@ -200,30 +200,25 @@ static int zopt2201_read(struct zopt2201_data *data, u8 reg)
 			msleep(t / 1000);
 		ret = i2c_smbus_read_byte_data(client, ZOPT2201_MAIN_STATUS);
 		if (ret < 0)
-			goto fail;
+			return ret;
 		if (ret & ZOPT2201_MAIN_STATUS_DRDY)
 			break;
 	}
 
 	if (tries < 0) {
 		ret = -ETIMEDOUT;
-		goto fail;
+		return ret;
 	}
 
 	ret = i2c_smbus_read_i2c_block_data(client, reg, sizeof(buf), buf);
 	if (ret < 0)
-		goto fail;
+		return ret;
 
 	ret = i2c_smbus_write_byte_data(client, ZOPT2201_MAIN_CTRL, 0x00);
 	if (ret < 0)
-		goto fail;
-	mutex_unlock(&data->lock);
+		return ret;
 
 	return get_unaligned_le24(&buf[0]);
-
-fail:
-	mutex_unlock(&data->lock);
-	return ret;
 }
 
 static const struct iio_chan_spec zopt2201_channels[] = {
@@ -317,17 +312,15 @@ static int zopt2201_set_resolution(struct zopt2201_data *data, u8 res)
 static int zopt2201_write_resolution(struct zopt2201_data *data,
 				     int val, int val2)
 {
-	int i, ret;
+	int i;
 
 	if (val != 0)
 		return -EINVAL;
 
 	for (i = 0; i < ARRAY_SIZE(zopt2201_resolution); i++)
 		if (val2 == zopt2201_resolution[i].us) {
-			mutex_lock(&data->lock);
-			ret = zopt2201_set_resolution(data, i);
-			mutex_unlock(&data->lock);
-			return ret;
+			guard(mutex)(&data->lock);
+			return zopt2201_set_resolution(data, i);
 		}
 
 	return -EINVAL;
@@ -351,16 +344,12 @@ static int zopt2201_write_scale_by_idx(struct zopt2201_data *data, int idx,
 {
 	int ret;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 	ret = zopt2201_set_resolution(data, zopt2201_scale_array[idx].res);
 	if (ret < 0)
-		goto unlock;
-
-	ret = zopt2201_set_gain(data, zopt2201_scale_array[idx].gain);
+		return ret;
 
-unlock:
-	mutex_unlock(&data->lock);
-	return ret;
+	return zopt2201_set_gain(data, zopt2201_scale_array[idx].gain);
 }
 
 static int zopt2201_write_scale_als(struct zopt2201_data *data,
diff --git a/drivers/iio/magnetometer/bmc150_magn.c b/drivers/iio/magnetometer/bmc150_magn.c
index a022e1805dff7..bf25519880089 100644
--- a/drivers/iio/magnetometer/bmc150_magn.c
+++ b/drivers/iio/magnetometer/bmc150_magn.c
@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/pm.h>
@@ -148,14 +149,16 @@ struct bmc150_magn_data {
 static const struct {
 	int freq;
 	u8 reg_val;
-} bmc150_magn_samp_freq_table[] = { {2, 0x01},
-				    {6, 0x02},
-				    {8, 0x03},
-				    {10, 0x00},
-				    {15, 0x04},
-				    {20, 0x05},
-				    {25, 0x06},
-				    {30, 0x07} };
+} bmc150_magn_samp_freq_table[] = {
+	{ 2, 0x01 },
+	{ 6, 0x02 },
+	{ 8, 0x03 },
+	{ 10, 0x00 },
+	{ 15, 0x04 },
+	{ 20, 0x05 },
+	{ 25, 0x06 },
+	{ 30, 0x07 },
+};
 
 enum bmc150_magn_presets {
 	LOW_POWER_PRESET,
@@ -169,10 +172,10 @@ static const struct bmc150_magn_preset {
 	u8 rep_z;
 	u8 odr;
 } bmc150_magn_presets_table[] = {
-	[LOW_POWER_PRESET] = {3, 3, 10},
-	[REGULAR_PRESET] =  {9, 15, 10},
-	[ENHANCED_REGULAR_PRESET] =  {15, 27, 10},
-	[HIGH_ACCURACY_PRESET] =  {47, 83, 20},
+	[LOW_POWER_PRESET] = { 3, 3, 10 },
+	[REGULAR_PRESET] = { 9, 15, 10 },
+	[ENHANCED_REGULAR_PRESET] = { 15, 27, 10 },
+	[HIGH_ACCURACY_PRESET] = { 47, 83, 20 },
 };
 
 #define BMC150_MAGN_DEFAULT_PRESET REGULAR_PRESET
@@ -255,6 +258,13 @@ static int bmc150_magn_set_power_mode(struct bmc150_magn_data *data,
 	return -EINVAL;
 }
 
+static int bmc150_magn_set_power_mode_locked(struct bmc150_magn_data *data,
+					     enum bmc150_magn_power_modes mode)
+{
+	guard(mutex)(&data->mutex);
+	return bmc150_magn_set_power_mode(data, mode, true);
+}
+
 static int bmc150_magn_set_power_state(struct bmc150_magn_data *data, bool on)
 {
 	int ret = 0;
@@ -453,33 +463,29 @@ static int bmc150_magn_read_raw(struct iio_dev *indio_dev,
 	s32 values[AXIS_XYZ_MAX];
 
 	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_RAW: {
 		if (iio_buffer_enabled(indio_dev))
 			return -EBUSY;
-		mutex_lock(&data->mutex);
+
+		guard(mutex)(&data->mutex);
 
 		ret = bmc150_magn_set_power_state(data, true);
-		if (ret < 0) {
-			mutex_unlock(&data->mutex);
+		if (ret < 0)
 			return ret;
-		}
 
 		ret = bmc150_magn_read_xyz(data, values);
 		if (ret < 0) {
 			bmc150_magn_set_power_state(data, false);
-			mutex_unlock(&data->mutex);
 			return ret;
 		}
 		*val = values[chan->scan_index];
 
 		ret = bmc150_magn_set_power_state(data, false);
-		if (ret < 0) {
-			mutex_unlock(&data->mutex);
+		if (ret < 0)
 			return ret;
-		}
 
-		mutex_unlock(&data->mutex);
 		return IIO_VAL_INT;
+	}
 	case IIO_CHAN_INFO_SCALE:
 		/*
 		 * The API/driver performs an off-chip temperature
@@ -527,48 +533,39 @@ static int bmc150_magn_write_raw(struct iio_dev *indio_dev,
 	int ret;
 
 	switch (mask) {
-	case IIO_CHAN_INFO_SAMP_FREQ:
+	case IIO_CHAN_INFO_SAMP_FREQ: {
 		if (val > data->max_odr)
 			return -EINVAL;
-		mutex_lock(&data->mutex);
-		ret = bmc150_magn_set_odr(data, val);
-		mutex_unlock(&data->mutex);
-		return ret;
+		guard(mutex)(&data->mutex);
+		return bmc150_magn_set_odr(data, val);
+	}
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
 		switch (chan->channel2) {
 		case IIO_MOD_X:
-		case IIO_MOD_Y:
+		case IIO_MOD_Y: {
 			if (val < 1 || val > 511)
 				return -EINVAL;
-			mutex_lock(&data->mutex);
+			guard(mutex)(&data->mutex);
 			ret = bmc150_magn_set_max_odr(data, val, 0, 0);
-			if (ret < 0) {
-				mutex_unlock(&data->mutex);
+			if (ret < 0)
 				return ret;
-			}
-			ret = regmap_update_bits(data->regmap,
+			return regmap_update_bits(data->regmap,
 						 BMC150_MAGN_REG_REP_XY,
 						 BMC150_MAGN_REG_REP_DATAMASK,
-						 BMC150_MAGN_REPXY_TO_REGVAL
-						 (val));
-			mutex_unlock(&data->mutex);
-			return ret;
-		case IIO_MOD_Z:
+						 BMC150_MAGN_REPXY_TO_REGVAL(val));
+		}
+		case IIO_MOD_Z: {
 			if (val < 1 || val > 256)
 				return -EINVAL;
-			mutex_lock(&data->mutex);
+			guard(mutex)(&data->mutex);
 			ret = bmc150_magn_set_max_odr(data, 0, val, 0);
-			if (ret < 0) {
-				mutex_unlock(&data->mutex);
+			if (ret < 0)
 				return ret;
-			}
-			ret = regmap_update_bits(data->regmap,
+			return regmap_update_bits(data->regmap,
 						 BMC150_MAGN_REG_REP_Z,
 						 BMC150_MAGN_REG_REP_DATAMASK,
-						 BMC150_MAGN_REPZ_TO_REGVAL
-						 (val));
-			mutex_unlock(&data->mutex);
-			return ret;
+						 BMC150_MAGN_REPZ_TO_REGVAL(val));
+		}
 		default:
 			return -EINVAL;
 		}
@@ -655,8 +652,9 @@ static const struct iio_info bmc150_magn_info = {
 };
 
 static const unsigned long bmc150_magn_scan_masks[] = {
-					BIT(AXIS_X) | BIT(AXIS_Y) | BIT(AXIS_Z),
-					0};
+	BIT(AXIS_X) | BIT(AXIS_Y) | BIT(AXIS_Z),
+	0
+};
 
 static irqreturn_t bmc150_magn_trigger_handler(int irq, void *p)
 {
@@ -695,7 +693,7 @@ static int bmc150_magn_init(struct bmc150_magn_data *data)
 	 * 3ms power-on time according to datasheet, let's better
 	 * be safe than sorry and set this delay to 5ms.
 	 */
-	msleep(5);
+	fsleep(5 * USEC_PER_MSEC);
 
 	ret = bmc150_magn_set_power_mode(data, BMC150_MAGN_POWER_MODE_SUSPEND,
 					 false);
@@ -782,9 +780,8 @@ static void bmc150_magn_trig_reen(struct iio_trigger *trig)
 	if (!data->dready_trigger_on)
 		return;
 
-	mutex_lock(&data->mutex);
+	guard(mutex)(&data->mutex);
 	ret = bmc150_magn_reset_intr(data);
-	mutex_unlock(&data->mutex);
 	if (ret)
 		dev_err(data->dev, "Failed to reset interrupt\n");
 }
@@ -794,32 +791,28 @@ static int bmc150_magn_data_rdy_trigger_set_state(struct iio_trigger *trig,
 {
 	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
 	struct bmc150_magn_data *data = iio_priv(indio_dev);
-	int ret = 0;
+	int ret;
+
+	guard(mutex)(&data->mutex);
 
-	mutex_lock(&data->mutex);
 	if (state == data->dready_trigger_on)
-		goto err_unlock;
+		return 0;
 
 	ret = regmap_update_bits(data->regmap, BMC150_MAGN_REG_INT_DRDY,
 				 BMC150_MAGN_MASK_DRDY_EN,
 				 state << BMC150_MAGN_SHIFT_DRDY_EN);
 	if (ret < 0)
-		goto err_unlock;
+		return ret;
 
 	data->dready_trigger_on = state;
 
 	if (state) {
 		ret = bmc150_magn_reset_intr(data);
 		if (ret < 0)
-			goto err_unlock;
+			return ret;
 	}
-	mutex_unlock(&data->mutex);
 
 	return 0;
-
-err_unlock:
-	mutex_unlock(&data->mutex);
-	return ret;
 }
 
 static const struct iio_trigger_ops bmc150_magn_trigger_ops = {
@@ -977,9 +970,7 @@ void bmc150_magn_remove(struct device *dev)
 	if (data->dready_trig)
 		iio_trigger_unregister(data->dready_trig);
 
-	mutex_lock(&data->mutex);
-	bmc150_magn_set_power_mode(data, BMC150_MAGN_POWER_MODE_SUSPEND, true);
-	mutex_unlock(&data->mutex);
+	bmc150_magn_set_power_mode_locked(data, BMC150_MAGN_POWER_MODE_SUSPEND);
 
 	regulator_bulk_disable(ARRAY_SIZE(data->regulators), data->regulators);
 }
@@ -992,10 +983,8 @@ static int bmc150_magn_runtime_suspend(struct device *dev)
 	struct bmc150_magn_data *data = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&data->mutex);
-	ret = bmc150_magn_set_power_mode(data, BMC150_MAGN_POWER_MODE_SLEEP,
-					 true);
-	mutex_unlock(&data->mutex);
+	ret = bmc150_magn_set_power_mode_locked(data,
+						BMC150_MAGN_POWER_MODE_SLEEP);
 	if (ret < 0) {
 		dev_err(dev, "powering off device failed\n");
 		return ret;
@@ -1021,28 +1010,18 @@ static int bmc150_magn_suspend(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct bmc150_magn_data *data = iio_priv(indio_dev);
-	int ret;
 
-	mutex_lock(&data->mutex);
-	ret = bmc150_magn_set_power_mode(data, BMC150_MAGN_POWER_MODE_SLEEP,
-					 true);
-	mutex_unlock(&data->mutex);
-
-	return ret;
+	return bmc150_magn_set_power_mode_locked(data,
+						 BMC150_MAGN_POWER_MODE_SLEEP);
 }
 
 static int bmc150_magn_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct bmc150_magn_data *data = iio_priv(indio_dev);
-	int ret;
 
-	mutex_lock(&data->mutex);
-	ret = bmc150_magn_set_power_mode(data, BMC150_MAGN_POWER_MODE_NORMAL,
-					 true);
-	mutex_unlock(&data->mutex);
-
-	return ret;
+	return bmc150_magn_set_power_mode_locked(data,
+						 BMC150_MAGN_POWER_MODE_NORMAL);
 }
 #endif
 
diff --git a/drivers/iio/magnetometer/si7210.c b/drivers/iio/magnetometer/si7210.c
index 27e3feba7a0f0..2a36abd1c99d7 100644
--- a/drivers/iio/magnetometer/si7210.c
+++ b/drivers/iio/magnetometer/si7210.c
@@ -128,8 +128,8 @@ static const struct regmap_config si7210_regmap_conf = {
 struct si7210_data {
 	struct regmap *regmap;
 	struct i2c_client *client;
-	struct regulator *vdd;
 	struct mutex fetch_lock; /* lock for a single measurement fetch */
+	unsigned int vdd_uV;
 	s8 temp_offset;
 	s8 temp_gain;
 	s8 scale_20_a[A_REGS_COUNT];
@@ -221,12 +221,8 @@ static int si7210_read_raw(struct iio_dev *indio_dev,
 		temp *= (1 + (data->temp_gain / 2048));
 		temp += (int)(MICRO / 16) * data->temp_offset;
 
-		ret = regulator_get_voltage(data->vdd);
-		if (ret < 0)
-			return ret;
-
 		/* temp -= 0.222 * VDD */
-		temp -= 222 * div_s64(ret, MILLI);
+		temp -= 222 * (data->vdd_uV / MILLI);
 
 		*val = div_s64(temp, MILLI);
 
@@ -396,14 +392,11 @@ static int si7210_probe(struct i2c_client *client)
 		return dev_err_probe(&client->dev, PTR_ERR(data->regmap),
 				     "failed to register regmap\n");
 
-	data->vdd = devm_regulator_get(&client->dev, "vdd");
-	if (IS_ERR(data->vdd))
-		return dev_err_probe(&client->dev, PTR_ERR(data->vdd),
-				     "failed to get VDD regulator\n");
-
-	ret = regulator_enable(data->vdd);
-	if (ret)
-		return ret;
+	ret = devm_regulator_get_enable_read_voltage(&client->dev, "vdd");
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret,
+				     "Failed to get vdd regulator\n");
+	data->vdd_uV = ret;
 
 	indio_dev->name = dev_name(&client->dev);
 	indio_dev->modes = INDIO_DIRECT_MODE;
diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c
index e759f91a710a2..5a5e6e4fbe34b 100644
--- a/drivers/iio/orientation/hid-sensor-rotation.c
+++ b/drivers/iio/orientation/hid-sensor-rotation.c
@@ -19,8 +19,13 @@ struct dev_rot_state {
 	struct hid_sensor_common common_attributes;
 	struct hid_sensor_hub_attribute_info quaternion;
 	struct {
-		s32 sampled_vals[4];
-		aligned_s64 timestamp;
+		IIO_DECLARE_QUATERNION(s32, sampled_vals);
+		/*
+		 * ABI regression avoidance: There are two copies of the same
+		 * timestamp in case of userspace depending on broken alignment
+		 * from older kernels.
+		 */
+		aligned_s64 timestamp[2];
 	} scan;
 	int scale_pre_decml;
 	int scale_post_decml;
@@ -154,8 +159,19 @@ static int dev_rot_proc_event(struct hid_sensor_hub_device *hsdev,
 		if (!rot_state->timestamp)
 			rot_state->timestamp = iio_get_time_ns(indio_dev);
 
-		iio_push_to_buffers_with_timestamp(indio_dev, &rot_state->scan,
-						   rot_state->timestamp);
+		/*
+		 * ABI regression avoidance: IIO previously had an incorrect
+		 * implementation of iio_push_to_buffers_with_timestamp() that
+		 * put the timestamp in the last 8 bytes of the buffer, which
+		 * was incorrect according to the IIO ABI. To avoid breaking
+		 * userspace that may be depending on this broken behavior, we
+		 * put the timestamp in both the correct place [0] and the old
+		 * incorrect place [1].
+		 */
+		rot_state->scan.timestamp[0] = rot_state->timestamp;
+		rot_state->scan.timestamp[1] = rot_state->timestamp;
+
+		iio_push_to_buffers(indio_dev, &rot_state->scan);
 
 		rot_state->timestamp = 0;
 	}
diff --git a/drivers/iio/pressure/abp2030pa.c b/drivers/iio/pressure/abp2030pa.c
index 4ca056a73cefc..b44f1bf4c633f 100644
--- a/drivers/iio/pressure/abp2030pa.c
+++ b/drivers/iio/pressure/abp2030pa.c
@@ -520,7 +520,7 @@ int abp2_common_probe(struct device *dev, const struct abp2_ops *ops, int irq)
 	data->p_offset = div_s64(odelta * data->pmin, pdelta) - data->outmin;
 
 	if (data->irq > 0) {
-		ret = devm_request_irq(dev, irq, abp2_eoc_handler, IRQF_ONESHOT,
+		ret = devm_request_irq(dev, irq, abp2_eoc_handler, 0,
 				       dev_name(dev), data);
 		if (ret)
 			return ret;
diff --git a/drivers/iio/pressure/hsc030pa_i2c.c b/drivers/iio/pressure/hsc030pa_i2c.c
index a34ef4653f342..3500bda03d750 100644
--- a/drivers/iio/pressure/hsc030pa_i2c.c
+++ b/drivers/iio/pressure/hsc030pa_i2c.c
@@ -34,8 +34,13 @@ static int hsc_i2c_recv(struct hsc_data *data)
 	msg.buf = data->buffer;
 
 	ret = i2c_transfer(client->adapter, &msg, 1);
+	if (ret < 0)
+		return ret;
 
-	return (ret == 2) ? 0 : ret;
+	if (ret != 1)
+		return -EIO;
+
+	return 0;
 }
 
 static int hsc_i2c_probe(struct i2c_client *client)
diff --git a/drivers/iio/proximity/hx9023s.c b/drivers/iio/proximity/hx9023s.c
index 17e00ee2b6f84..9efaa5b6b5bdc 100644
--- a/drivers/iio/proximity/hx9023s.c
+++ b/drivers/iio/proximity/hx9023s.c
@@ -1088,6 +1088,7 @@ static int hx9023s_probe(struct i2c_client *client)
 	struct device *dev = &client->dev;
 	struct iio_dev *indio_dev;
 	struct hx9023s_data *data;
+	const char *fw_name;
 	int ret;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
@@ -1125,7 +1126,9 @@ static int hx9023s_probe(struct i2c_client *client)
 	if (ret)
 		return dev_err_probe(dev, ret, "channel config failed\n");
 
-	ret = request_firmware_nowait(THIS_MODULE, true, "hx9023s.bin", dev,
+	fw_name = "hx9023s.bin";
+	device_property_read_string(dev, "firmware-name", &fw_name);
+	ret = request_firmware_nowait(THIS_MODULE, true, fw_name, dev,
 				      GFP_KERNEL, data, hx9023s_cfg_update);
 	if (ret)
 		return dev_err_probe(dev, ret, "reg config failed\n");
diff --git a/drivers/iio/proximity/rfd77402.c b/drivers/iio/proximity/rfd77402.c
index 6afdbfca3e5a1..81b8daf17a543 100644
--- a/drivers/iio/proximity/rfd77402.c
+++ b/drivers/iio/proximity/rfd77402.c
@@ -173,10 +173,8 @@ static int rfd77402_wait_for_result(struct rfd77402_data *data)
 	struct i2c_client *client = data->client;
 	int val, ret;
 
-	if (data->irq_en) {
-		reinit_completion(&data->completion);
+	if (data->irq_en)
 		return rfd77402_wait_for_irq(data);
-	}
 
 	/*
 	 * As per RFD77402 datasheet section '3.1.1 Single Measure', the
@@ -204,6 +202,9 @@ static int rfd77402_measure(struct rfd77402_data *data)
 	if (ret < 0)
 		return ret;
 
+	if (data->irq_en)
+		reinit_completion(&data->completion);
+
 	ret = i2c_smbus_write_byte_data(client, RFD77402_CMD_R,
 					RFD77402_CMD_SINGLE |
 					RFD77402_CMD_VALID);
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index 78ac2ff5befd4..aa85ec57f2a71 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -112,6 +112,4 @@ source "drivers/infiniband/ulp/iser/Kconfig"
 source "drivers/infiniband/ulp/isert/Kconfig"
 source "drivers/infiniband/ulp/rtrs/Kconfig"
 
-source "drivers/infiniband/ulp/opa_vnic/Kconfig"
-
 endif # INFINIBAND
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index a2a7a9d2e0d3e..dce798d8cfe67 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -12,10 +12,12 @@ ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
 				multicast.o mad.o smi.o agent.o mad_rmpp.o \
 				nldev.o restrack.o counters.o ib_core_uverbs.o \
-				trace.o lag.o
+				trace.o lag.o iter.o frmr_pools.o
 
 ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
 ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
+ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o
+ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
 
 ib_cm-y :=			cm.o cm_trace.o
 
@@ -43,5 +45,3 @@ ib_uverbs-y :=			uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
 				uverbs_std_types_wq.o \
 				uverbs_std_types_qp.o \
 				ucaps.o
-ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o
-ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index 81316228f6146..3d7b6cddd131c 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -220,6 +220,9 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
 	struct ib_cq *cq;
 	int ret = -ENOMEM;
 
+	if (WARN_ON_ONCE(!nr_cqe))
+		return ERR_PTR(-EINVAL);
+
 	cq = rdma_zalloc_drv_obj(dev, ib_cq);
 	if (!cq)
 		return ERR_PTR(ret);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 558b73940d668..236061a33bf67 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -2706,6 +2706,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 
 	dev_ops->uverbs_no_driver_id_binding |=
 		ops->uverbs_no_driver_id_binding;
+	dev_ops->uverbs_robust_udata |= ops->uverbs_robust_udata;
 
 	SET_DEVICE_OP(dev_ops, add_gid);
 	SET_DEVICE_OP(dev_ops, add_sub_dev);
@@ -2732,7 +2733,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, create_ah);
 	SET_DEVICE_OP(dev_ops, create_counters);
 	SET_DEVICE_OP(dev_ops, create_cq);
-	SET_DEVICE_OP(dev_ops, create_cq_umem);
+	SET_DEVICE_OP(dev_ops, create_user_cq);
 	SET_DEVICE_OP(dev_ops, create_flow);
 	SET_DEVICE_OP(dev_ops, create_qp);
 	SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
@@ -2781,7 +2782,6 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, get_netdev);
 	SET_DEVICE_OP(dev_ops, get_numa_node);
 	SET_DEVICE_OP(dev_ops, get_port_immutable);
-	SET_DEVICE_OP(dev_ops, get_vector_affinity);
 	SET_DEVICE_OP(dev_ops, get_vf_config);
 	SET_DEVICE_OP(dev_ops, get_vf_guid);
 	SET_DEVICE_OP(dev_ops, get_vf_stats);
diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c
new file mode 100644
index 0000000000000..5e992ff3d7cff
--- /dev/null
+++ b/drivers/infiniband/core/frmr_pools.c
@@ -0,0 +1,547 @@
+// SPDX-License-Identifier: GPL-2.0  OR Linux-OpenIB
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#include <linux/slab.h>
+#include <linux/rbtree.h>
+#include <linux/sort.h>
+#include <linux/spinlock.h>
+#include <rdma/ib_verbs.h>
+#include <linux/timer.h>
+
+#include "frmr_pools.h"
+
+#define FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS 60
+
+static int push_handle_to_queue_locked(struct frmr_queue *queue, u32 handle)
+{
+	u32 tmp = queue->ci % NUM_HANDLES_PER_PAGE;
+	struct frmr_handles_page *page;
+
+	if (queue->ci >= queue->num_pages * NUM_HANDLES_PER_PAGE) {
+		page = kzalloc_obj(*page, GFP_ATOMIC);
+		if (!page)
+			return -ENOMEM;
+		queue->num_pages++;
+		list_add_tail(&page->list, &queue->pages_list);
+	} else {
+		page = list_last_entry(&queue->pages_list,
+				       struct frmr_handles_page, list);
+	}
+
+	page->handles[tmp] = handle;
+	queue->ci++;
+	return 0;
+}
+
+static u32 pop_handle_from_queue_locked(struct frmr_queue *queue)
+{
+	u32 tmp = (queue->ci - 1) % NUM_HANDLES_PER_PAGE;
+	struct frmr_handles_page *page;
+	u32 handle;
+
+	page = list_last_entry(&queue->pages_list, struct frmr_handles_page,
+			       list);
+	handle = page->handles[tmp];
+	queue->ci--;
+
+	if (!tmp) {
+		list_del(&page->list);
+		queue->num_pages--;
+		kfree(page);
+	}
+
+	return handle;
+}
+
+static bool pop_frmr_handles_page(struct ib_frmr_pool *pool,
+				  struct frmr_queue *queue,
+				  struct frmr_handles_page **page, u32 *count)
+{
+	spin_lock(&pool->lock);
+	if (list_empty(&queue->pages_list)) {
+		spin_unlock(&pool->lock);
+		return false;
+	}
+
+	*page = list_first_entry(&queue->pages_list, struct frmr_handles_page,
+				 list);
+	list_del(&(*page)->list);
+	queue->num_pages--;
+
+	/* If this is the last page, count may be less than
+	 * NUM_HANDLES_PER_PAGE.
+	 */
+	if (queue->ci >= NUM_HANDLES_PER_PAGE)
+		*count = NUM_HANDLES_PER_PAGE;
+	else
+		*count = queue->ci;
+
+	queue->ci -= *count;
+	spin_unlock(&pool->lock);
+	return true;
+}
+
+static void destroy_all_handles_in_queue(struct ib_device *device,
+					 struct ib_frmr_pool *pool,
+					 struct frmr_queue *queue)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct frmr_handles_page *page;
+	u32 count;
+
+	while (pop_frmr_handles_page(pool, queue, &page, &count)) {
+		pools->pool_ops->destroy_frmrs(device, page->handles, count);
+		kfree(page);
+	}
+}
+
+static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	u32 total, to_destroy, destroyed = 0;
+	bool has_work = false;
+	u32 *handles;
+	u32 handle;
+
+	spin_lock(&pool->lock);
+	total = pool->queue.ci + pool->inactive_queue.ci + pool->in_use;
+	if (total <= pool->pinned_handles) {
+		spin_unlock(&pool->lock);
+		return false;
+	}
+
+	to_destroy = total - pool->pinned_handles;
+
+	handles = kcalloc(to_destroy, sizeof(*handles), GFP_ATOMIC);
+	if (!handles) {
+		spin_unlock(&pool->lock);
+		return true;
+	}
+
+	/* Destroy all excess handles in the inactive queue */
+	while (pool->inactive_queue.ci && destroyed < to_destroy) {
+		handles[destroyed++] = pop_handle_from_queue_locked(
+			&pool->inactive_queue);
+	}
+
+	/* Move all handles from regular queue to inactive queue */
+	while (pool->queue.ci) {
+		handle = pop_handle_from_queue_locked(&pool->queue);
+		push_handle_to_queue_locked(&pool->inactive_queue, handle);
+		has_work = true;
+	}
+
+	spin_unlock(&pool->lock);
+
+	if (destroyed)
+		pools->pool_ops->destroy_frmrs(device, handles, destroyed);
+	kfree(handles);
+	return has_work;
+}
+
+static void pool_aging_work(struct work_struct *work)
+{
+	struct ib_frmr_pool *pool = container_of(
+		to_delayed_work(work), struct ib_frmr_pool, aging_work);
+	struct ib_frmr_pools *pools = pool->device->frmr_pools;
+	bool has_work = false;
+
+	if (pool->pinned_handles) {
+		has_work = age_pinned_pool(pool->device, pool);
+		goto out;
+	}
+
+	destroy_all_handles_in_queue(pool->device, pool, &pool->inactive_queue);
+
+	/* Move all pages from regular queue to inactive queue */
+	spin_lock(&pool->lock);
+	if (pool->queue.ci > 0) {
+		list_splice_tail_init(&pool->queue.pages_list,
+				      &pool->inactive_queue.pages_list);
+		pool->inactive_queue.num_pages = pool->queue.num_pages;
+		pool->inactive_queue.ci = pool->queue.ci;
+
+		pool->queue.num_pages = 0;
+		pool->queue.ci = 0;
+		has_work = true;
+	}
+	spin_unlock(&pool->lock);
+
+out:
+	/* Reschedule if there are handles to age in next aging period */
+	if (has_work)
+		queue_delayed_work(
+			pools->aging_wq, &pool->aging_work,
+			secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
+}
+
+static void destroy_frmr_pool(struct ib_device *device,
+			      struct ib_frmr_pool *pool)
+{
+	cancel_delayed_work_sync(&pool->aging_work);
+	destroy_all_handles_in_queue(device, pool, &pool->queue);
+	destroy_all_handles_in_queue(device, pool, &pool->inactive_queue);
+
+	kfree(pool);
+}
+
+/*
+ * Initialize the FRMR pools for a device.
+ *
+ * @device: The device to initialize the FRMR pools for.
+ * @pool_ops: The pool operations to use.
+ *
+ * Returns 0 on success, negative error code on failure.
+ */
+int ib_frmr_pools_init(struct ib_device *device,
+		       const struct ib_frmr_pool_ops *pool_ops)
+{
+	struct ib_frmr_pools *pools;
+
+	pools = kzalloc_obj(*pools);
+	if (!pools)
+		return -ENOMEM;
+
+	pools->rb_root = RB_ROOT;
+	rwlock_init(&pools->rb_lock);
+	pools->pool_ops = pool_ops;
+	pools->aging_wq = create_singlethread_workqueue("frmr_aging_wq");
+	if (!pools->aging_wq) {
+		kfree(pools);
+		return -ENOMEM;
+	}
+
+	pools->aging_period_sec = FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS;
+
+	device->frmr_pools = pools;
+	return 0;
+}
+EXPORT_SYMBOL(ib_frmr_pools_init);
+
+/*
+ * Clean up the FRMR pools for a device.
+ *
+ * @device: The device to clean up the FRMR pools for.
+ *
+ * Call cleanup only after all FRMR handles have been pushed back to the pool
+ * and no other FRMR operations are allowed to run in parallel.
+ * Ensuring this allows us to save synchronization overhead in pop and push
+ * operations.
+ */
+void ib_frmr_pools_cleanup(struct ib_device *device)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct ib_frmr_pool *pool, *next;
+
+	if (!pools)
+		return;
+
+	rbtree_postorder_for_each_entry_safe(pool, next, &pools->rb_root, node)
+		destroy_frmr_pool(device, pool);
+
+	destroy_workqueue(pools->aging_wq);
+	kfree(pools);
+	device->frmr_pools = NULL;
+}
+EXPORT_SYMBOL(ib_frmr_pools_cleanup);
+
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct ib_frmr_pool *pool;
+	struct rb_node *node;
+
+	if (!pools)
+		return -EINVAL;
+
+	if (period_sec == 0)
+		return -EINVAL;
+
+	WRITE_ONCE(pools->aging_period_sec, period_sec);
+
+	read_lock(&pools->rb_lock);
+	for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) {
+		pool = rb_entry(node, struct ib_frmr_pool, node);
+		mod_delayed_work(pools->aging_wq, &pool->aging_work,
+				 secs_to_jiffies(period_sec));
+	}
+	read_unlock(&pools->rb_lock);
+
+	return 0;
+}
+
+static inline int compare_keys(struct ib_frmr_key *key1,
+			       struct ib_frmr_key *key2)
+{
+	int res;
+
+	res = cmp_int(key1->ats, key2->ats);
+	if (res)
+		return res;
+
+	res = cmp_int(key1->access_flags, key2->access_flags);
+	if (res)
+		return res;
+
+	res = cmp_int(key1->vendor_key, key2->vendor_key);
+	if (res)
+		return res;
+
+	res = cmp_int(key1->kernel_vendor_key, key2->kernel_vendor_key);
+	if (res)
+		return res;
+
+	/*
+	 * allow using handles that support more DMA blocks, up to twice the
+	 * requested number
+	 */
+	res = cmp_int(key1->num_dma_blocks, key2->num_dma_blocks);
+	if (res > 0) {
+		if (key1->num_dma_blocks - key2->num_dma_blocks <
+		    key2->num_dma_blocks)
+			return 0;
+	}
+
+	return res;
+}
+
+static int frmr_pool_cmp_find(const void *key, const struct rb_node *node)
+{
+	struct ib_frmr_pool *pool = rb_entry(node, struct ib_frmr_pool, node);
+
+	return compare_keys(&pool->key, (struct ib_frmr_key *)key);
+}
+
+static int frmr_pool_cmp_add(struct rb_node *new, const struct rb_node *node)
+{
+	struct ib_frmr_pool *new_pool =
+		rb_entry(new, struct ib_frmr_pool, node);
+	struct ib_frmr_pool *pool = rb_entry(node, struct ib_frmr_pool, node);
+
+	return compare_keys(&pool->key, &new_pool->key);
+}
+
+static struct ib_frmr_pool *ib_frmr_pool_find(struct ib_frmr_pools *pools,
+					      struct ib_frmr_key *key)
+{
+	struct ib_frmr_pool *pool;
+	struct rb_node *node;
+
+	/* find operation is done under read lock for performance reasons.
+	 * The case of threads failing to find the same pool and creating it
+	 * is handled by the create_frmr_pool function.
+	 */
+	read_lock(&pools->rb_lock);
+	node = rb_find(key, &pools->rb_root, frmr_pool_cmp_find);
+	pool = rb_entry_safe(node, struct ib_frmr_pool, node);
+	read_unlock(&pools->rb_lock);
+
+	return pool;
+}
+
+static struct ib_frmr_pool *create_frmr_pool(struct ib_device *device,
+					     struct ib_frmr_key *key)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct ib_frmr_pool *pool;
+	struct rb_node *existing;
+
+	pool = kzalloc_obj(*pool);
+	if (!pool)
+		return ERR_PTR(-ENOMEM);
+
+	memcpy(&pool->key, key, sizeof(*key));
+	INIT_LIST_HEAD(&pool->queue.pages_list);
+	INIT_LIST_HEAD(&pool->inactive_queue.pages_list);
+	spin_lock_init(&pool->lock);
+	INIT_DELAYED_WORK(&pool->aging_work, pool_aging_work);
+	pool->device = device;
+
+	write_lock(&pools->rb_lock);
+	existing = rb_find_add(&pool->node, &pools->rb_root, frmr_pool_cmp_add);
+	write_unlock(&pools->rb_lock);
+
+	/* If a different thread has already created the pool, return it.
+	 * The insert operation is done under the write lock so we are sure
+	 * that the pool is not inserted twice.
+	 */
+	if (existing) {
+		kfree(pool);
+		return rb_entry(existing, struct ib_frmr_pool, node);
+	}
+
+	return pool;
+}
+
+int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
+			     u32 pinned_handles)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct ib_frmr_key driver_key = {};
+	struct ib_frmr_pool *pool;
+	u32 needed_handles;
+	u32 current_total;
+	int i, ret = 0;
+	u32 *handles;
+
+	if (!pools)
+		return -EINVAL;
+
+	ret = ib_check_mr_access(device, key->access_flags);
+	if (ret)
+		return ret;
+
+	if (pools->pool_ops->build_key) {
+		ret = pools->pool_ops->build_key(device, key, &driver_key);
+		if (ret)
+			return ret;
+	} else {
+		memcpy(&driver_key, key, sizeof(*key));
+	}
+
+	pool = ib_frmr_pool_find(pools, &driver_key);
+	if (!pool) {
+		pool = create_frmr_pool(device, &driver_key);
+		if (IS_ERR(pool))
+			return PTR_ERR(pool);
+	}
+
+	spin_lock(&pool->lock);
+	current_total = pool->in_use + pool->queue.ci + pool->inactive_queue.ci;
+
+	if (current_total < pinned_handles)
+		needed_handles = pinned_handles - current_total;
+	else
+		needed_handles = 0;
+
+	pool->pinned_handles = pinned_handles;
+	spin_unlock(&pool->lock);
+
+	if (!needed_handles)
+		goto schedule_aging;
+
+	handles = kcalloc(needed_handles, sizeof(*handles), GFP_KERNEL);
+	if (!handles)
+		return -ENOMEM;
+
+	ret = pools->pool_ops->create_frmrs(device, key, handles,
+					    needed_handles);
+	if (ret) {
+		kfree(handles);
+		return ret;
+	}
+
+	spin_lock(&pool->lock);
+	for (i = 0; i < needed_handles; i++) {
+		ret = push_handle_to_queue_locked(&pool->queue,
+						  handles[i]);
+		if (ret)
+			goto end;
+	}
+
+end:
+	spin_unlock(&pool->lock);
+	kfree(handles);
+
+schedule_aging:
+	/* Ensure aging is scheduled to adjust to new pinned handles count */
+	mod_delayed_work(pools->aging_wq, &pool->aging_work, 0);
+
+	return ret;
+}
+
+static int get_frmr_from_pool(struct ib_device *device,
+			      struct ib_frmr_pool *pool, struct ib_mr *mr)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	u32 handle;
+	int err;
+
+	spin_lock(&pool->lock);
+	if (pool->queue.ci == 0) {
+		if (pool->inactive_queue.ci > 0) {
+			handle = pop_handle_from_queue_locked(
+				&pool->inactive_queue);
+		} else {
+			spin_unlock(&pool->lock);
+			err = pools->pool_ops->create_frmrs(device, &pool->key,
+							    &handle, 1);
+			if (err)
+				return err;
+			spin_lock(&pool->lock);
+		}
+	} else {
+		handle = pop_handle_from_queue_locked(&pool->queue);
+	}
+
+	pool->in_use++;
+	if (pool->in_use > pool->max_in_use)
+		pool->max_in_use = pool->in_use;
+
+	spin_unlock(&pool->lock);
+
+	mr->frmr.pool = pool;
+	mr->frmr.handle = handle;
+
+	return 0;
+}
+
+/*
+ * Pop an FRMR handle from the pool.
+ *
+ * @device: The device to pop the FRMR handle from.
+ * @mr: The MR to pop the FRMR handle from.
+ *
+ * Returns 0 on success, negative error code on failure.
+ */
+int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr)
+{
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	struct ib_frmr_pool *pool;
+
+	WARN_ON_ONCE(!device->frmr_pools);
+	pool = ib_frmr_pool_find(pools, &mr->frmr.key);
+	if (!pool) {
+		pool = create_frmr_pool(device, &mr->frmr.key);
+		if (IS_ERR(pool))
+			return PTR_ERR(pool);
+	}
+
+	return get_frmr_from_pool(device, pool, mr);
+}
+EXPORT_SYMBOL(ib_frmr_pool_pop);
+
+/*
+ * Push an FRMR handle back to the pool.
+ *
+ * @device: The device to push the FRMR handle to.
+ * @mr: The MR containing the FRMR handle to push back to the pool.
+ *
+ * Returns 0 on success, negative error code on failure.
+ */
+int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
+{
+	struct ib_frmr_pool *pool = mr->frmr.pool;
+	struct ib_frmr_pools *pools = device->frmr_pools;
+	bool schedule_aging = false;
+	int ret;
+
+	spin_lock(&pool->lock);
+	/* Schedule aging every time an empty pool becomes non-empty */
+	if (pool->queue.ci == 0)
+		schedule_aging = true;
+	ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle);
+	if (ret == 0)
+		pool->in_use--;
+
+	spin_unlock(&pool->lock);
+
+	if (ret == 0 && schedule_aging)
+		queue_delayed_work(pools->aging_wq, &pool->aging_work,
+			secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_frmr_pool_push);
diff --git a/drivers/infiniband/core/frmr_pools.h b/drivers/infiniband/core/frmr_pools.h
new file mode 100644
index 0000000000000..67e1402169aef
--- /dev/null
+++ b/drivers/infiniband/core/frmr_pools.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+ *
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#ifndef RDMA_CORE_FRMR_POOLS_H
+#define RDMA_CORE_FRMR_POOLS_H
+
+#include <rdma/frmr_pools.h>
+#include <linux/rbtree_types.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include <asm/page.h>
+#include <linux/workqueue.h>
+
+#define NUM_HANDLES_PER_PAGE \
+	((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))
+
+struct frmr_handles_page {
+	struct list_head list;
+	u32 handles[NUM_HANDLES_PER_PAGE];
+};
+
+/* FRMR queue holds a list of frmr_handles_page.
+ * num_pages: number of pages in the queue.
+ * ci: current index in the handles array across all pages.
+ */
+struct frmr_queue {
+	struct list_head pages_list;
+	u32 num_pages;
+	unsigned long ci;
+};
+
+struct ib_frmr_pool {
+	struct rb_node node;
+	struct ib_frmr_key key; /* Pool key */
+
+	/* Protect access to the queue */
+	spinlock_t lock;
+	struct frmr_queue queue;
+	struct frmr_queue inactive_queue;
+
+	struct delayed_work aging_work;
+	struct ib_device *device;
+
+	u32 max_in_use;
+	u32 in_use;
+	u32 pinned_handles;
+};
+
+struct ib_frmr_pools {
+	struct rb_root rb_root;
+	rwlock_t rb_lock;
+	const struct ib_frmr_pool_ops *pool_ops;
+
+	struct workqueue_struct *aging_wq;
+	u32 aging_period_sec;
+};
+
+int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
+			     u32 pinned_handles);
+int ib_frmr_pools_set_aging_period(struct ib_device *device, u32 period_sec);
+#endif /* RDMA_CORE_FRMR_POOLS_H */
diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c
index d3836a62a0049..1f7a5c119cc9c 100644
--- a/drivers/infiniband/core/ib_core_uverbs.c
+++ b/drivers/infiniband/core/ib_core_uverbs.c
@@ -246,7 +246,7 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry)
 		dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL);
 		list_del(&uverbs_dmabuf->dmabufs_elm);
 		uverbs_dmabuf->revoked = true;
-		dma_buf_move_notify(uverbs_dmabuf->dmabuf);
+		dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf);
 		dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv,
 				      DMA_RESV_USAGE_BOOKKEEP, false,
 				      MAX_SCHEDULE_TIMEOUT);
@@ -389,3 +389,30 @@ int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
 						 U32_MAX);
 }
 EXPORT_SYMBOL(rdma_user_mmap_entry_insert);
+
+/**
+ * rdma_udata_to_dev - Get a ib_device from a udata
+ * @udata: The system calls ib_udata struct
+ *
+ * The struct ib_device that is handling the uverbs call. Must not be called if
+ * udata is NULL. The result can be NULL.
+ */
+struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
+{
+	struct uverbs_attr_bundle *bundle =
+		rdma_udata_to_uverbs_attr_bundle(udata);
+
+	lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
+
+	if (bundle->context)
+		return bundle->context->device;
+
+	/*
+	 * If the context hasn't been created yet use the ufile's dev, but it
+	 * might be NULL if we are racing with disassociate.
+	 */
+	return srcu_dereference(bundle->ufile->device->ib_dev,
+				&bundle->ufile->device->disassociate_srcu);
+}
+EXPORT_SYMBOL(rdma_udata_to_dev);
+
diff --git a/drivers/infiniband/core/iter.c b/drivers/infiniband/core/iter.c
new file mode 100644
index 0000000000000..8e543d100657e
--- /dev/null
+++ b/drivers/infiniband/core/iter.c
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. */
+
+#include <linux/export.h>
+#include <rdma/iter.h>
+
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist, unsigned int nents,
+			     unsigned long pgsz)
+{
+	memset(biter, 0, sizeof(struct ib_block_iter));
+	biter->__sg = sglist;
+	biter->__sg_nents = nents;
+
+	/* Driver provides best block size to use */
+	biter->__pg_bit = __fls(pgsz);
+}
+EXPORT_SYMBOL(__rdma_block_iter_start);
+
+bool __rdma_block_iter_next(struct ib_block_iter *biter)
+{
+	unsigned int block_offset;
+	unsigned int delta;
+
+	if (!biter->__sg_nents || !biter->__sg)
+		return false;
+
+	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
+	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
+	delta = BIT_ULL(biter->__pg_bit) - block_offset;
+
+	while (biter->__sg_nents && biter->__sg &&
+	       sg_dma_len(biter->__sg) - biter->__sg_advance <= delta) {
+		delta -= sg_dma_len(biter->__sg) - biter->__sg_advance;
+		biter->__sg_advance = 0;
+		biter->__sg = sg_next(biter->__sg);
+		biter->__sg_nents--;
+	}
+	biter->__sg_advance += delta;
+
+	return true;
+}
+EXPORT_SYMBOL(__rdma_block_iter_next);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 2220a2dfab240..cb18699633e83 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -37,11 +37,13 @@
 #include <net/netlink.h>
 #include <rdma/rdma_cm.h>
 #include <rdma/rdma_netlink.h>
+#include <rdma/frmr_pools.h>
 
 #include "core_priv.h"
 #include "cma_priv.h"
 #include "restrack.h"
 #include "uverbs.h"
+#include "frmr_pools.h"
 
 /*
  * This determines whether a non-privileged user is allowed to specify a
@@ -172,6 +174,19 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE]	= { .type = NLA_U8 },
 	[RDMA_NLDEV_ATTR_EVENT_TYPE]		= { .type = NLA_U8 },
 	[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED] = { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_FRMR_POOLS]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY]	= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS] = { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY] = { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS] = { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES] = { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE]	= { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE]	= { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES] = { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY] = { .type = NLA_U64 },
 };
 
 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -2637,6 +2652,270 @@ static int nldev_deldev(struct sk_buff *skb, struct nlmsghdr *nlh,
 	return ib_del_sub_device_and_put(device);
 }
 
+static int fill_frmr_pool_key(struct sk_buff *msg, struct ib_frmr_key *key)
+{
+	struct nlattr *key_attr;
+
+	key_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY);
+	if (!key_attr)
+		return -EMSGSIZE;
+
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS, key->ats))
+		goto err;
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS,
+			key->access_flags))
+		goto err;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY,
+			      key->vendor_key, RDMA_NLDEV_ATTR_PAD))
+		goto err;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS,
+			      key->num_dma_blocks, RDMA_NLDEV_ATTR_PAD))
+		goto err;
+
+	if (key->kernel_vendor_key &&
+	    nla_put_u64_64bit(msg,
+			      RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY,
+			      key->kernel_vendor_key, RDMA_NLDEV_ATTR_PAD))
+		goto err;
+
+	nla_nest_end(msg, key_attr);
+	return 0;
+
+err:
+	return -EMSGSIZE;
+}
+
+static int fill_frmr_pool_entry(struct sk_buff *msg, struct ib_frmr_pool *pool)
+{
+	if (fill_frmr_pool_key(msg, &pool->key))
+		return -EMSGSIZE;
+
+	spin_lock(&pool->lock);
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES,
+			pool->queue.ci + pool->inactive_queue.ci))
+		goto err_unlock;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE,
+			      pool->max_in_use, RDMA_NLDEV_ATTR_PAD))
+		goto err_unlock;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE,
+			      pool->in_use, RDMA_NLDEV_ATTR_PAD))
+		goto err_unlock;
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES,
+			pool->pinned_handles))
+		goto err_unlock;
+	spin_unlock(&pool->lock);
+
+	return 0;
+
+err_unlock:
+	spin_unlock(&pool->lock);
+	return -EMSGSIZE;
+}
+
+static int nldev_frmr_pools_parse_key(struct nlattr *tb[],
+				      struct ib_frmr_key *key,
+				      struct netlink_ext_ack *extack)
+{
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS])
+		key->ats = nla_get_u8(tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS]);
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS])
+		key->access_flags = nla_get_u32(
+			tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS]);
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY])
+		key->vendor_key = nla_get_u64(
+			tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY]);
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS])
+		key->num_dma_blocks = nla_get_u64(
+			tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS]);
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY])
+		return -EINVAL;
+
+	return 0;
+}
+
+static int nldev_frmr_pools_set_pinned(struct ib_device *device,
+				       struct nlattr *tb[],
+				       struct netlink_ext_ack *extack)
+{
+	struct nlattr *key_tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_frmr_key key = { 0 };
+	u32 pinned_handles = 0;
+	int err = 0;
+
+	pinned_handles =
+		nla_get_u32(tb[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES]);
+
+	if (!tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY])
+		return -EINVAL;
+
+	err = nla_parse_nested(key_tb, RDMA_NLDEV_ATTR_MAX - 1,
+			       tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY], nldev_policy,
+			       extack);
+	if (err)
+		return err;
+
+	err = nldev_frmr_pools_parse_key(key_tb, &key, extack);
+	if (err)
+		return err;
+
+	err = ib_frmr_pools_set_pinned(device, &key, pinned_handles);
+
+	return err;
+}
+
+static int nldev_frmr_pools_get_dumpit(struct sk_buff *skb,
+				       struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_frmr_pools *pools;
+	int err, ret = 0, idx = 0;
+	struct ib_frmr_pool *pool;
+	struct nlattr *table_attr;
+	struct nlattr *entry_attr;
+	bool show_details = false;
+	struct ib_device *device;
+	int start = cb->args[0];
+	struct rb_node *node;
+	struct nlmsghdr *nlh;
+	bool filled = false;
+
+	err = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			    nldev_policy, NL_VALIDATE_LIBERAL, NULL);
+	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	device = ib_device_get_by_index(
+		sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]));
+	if (!device)
+		return -EINVAL;
+
+	if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS])
+		show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]);
+
+	pools = device->frmr_pools;
+	if (!pools) {
+		ib_device_put(device);
+		return 0;
+	}
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					 RDMA_NLDEV_CMD_FRMR_POOLS_GET),
+			0, NLM_F_MULTI);
+
+	if (!nlh || fill_nldev_handle(skb, device)) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	table_attr = nla_nest_start_noflag(skb, RDMA_NLDEV_ATTR_FRMR_POOLS);
+	if (!table_attr) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	read_lock(&pools->rb_lock);
+	for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) {
+		pool = rb_entry(node, struct ib_frmr_pool, node);
+		if (pool->key.kernel_vendor_key && !show_details)
+			continue;
+
+		if (idx < start) {
+			idx++;
+			continue;
+		}
+
+		filled = true;
+
+		entry_attr = nla_nest_start_noflag(
+			skb, RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY);
+		if (!entry_attr) {
+			ret = -EMSGSIZE;
+			goto end_msg;
+		}
+
+		if (fill_frmr_pool_entry(skb, pool)) {
+			nla_nest_cancel(skb, entry_attr);
+			ret = -EMSGSIZE;
+			goto end_msg;
+		}
+
+		nla_nest_end(skb, entry_attr);
+		idx++;
+	}
+end_msg:
+	read_unlock(&pools->rb_lock);
+
+	nla_nest_end(skb, table_attr);
+	nlmsg_end(skb, nlh);
+	cb->args[0] = idx;
+
+	/*
+	 * No more entries to fill, cancel the message and
+	 * return 0 to mark end of dumpit.
+	 */
+	if (!filled)
+		goto err;
+
+	ib_device_put(device);
+	return skb->len;
+
+err:
+	nlmsg_cancel(skb, nlh);
+	ib_device_put(device);
+	return ret;
+}
+
+static int nldev_frmr_pools_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+				     struct netlink_ext_ack *extack)
+{
+	struct ib_device *device;
+	struct nlattr **tb;
+	u32 aging_period;
+	int err;
+
+	tb = kzalloc_objs(*tb, RDMA_NLDEV_ATTR_MAX, GFP_KERNEL);
+	if (!tb)
+		return -ENOMEM;
+
+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
+			  extack);
+	if (err)
+		goto free_tb;
+
+	if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {
+		err = -EINVAL;
+		goto free_tb;
+	}
+
+	device = ib_device_get_by_index(
+		sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]));
+	if (!device) {
+		err = -EINVAL;
+		goto free_tb;
+	}
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]) {
+		aging_period = nla_get_u32(
+			tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]);
+		err = ib_frmr_pools_set_aging_period(device, aging_period);
+		goto done;
+	}
+
+	if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES])
+		err = nldev_frmr_pools_set_pinned(device, tb, extack);
+
+done:
+	ib_device_put(device);
+free_tb:
+	kfree(tb);
+	return err;
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 	[RDMA_NLDEV_CMD_GET] = {
 		.doit = nldev_get_doit,
@@ -2743,6 +3022,13 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 		.doit = nldev_deldev,
 		.flags = RDMA_NL_ADMIN_PERM,
 	},
+	[RDMA_NLDEV_CMD_FRMR_POOLS_GET] = {
+		.dump = nldev_frmr_pools_get_dumpit,
+	},
+	[RDMA_NLDEV_CMD_FRMR_POOLS_SET] = {
+		.doit = nldev_frmr_pools_set_doit,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 };
 
 static int fill_mon_netdev_rename(struct sk_buff *msg,
diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h
index 55f1e3558856f..269b393799abb 100644
--- a/drivers/infiniband/core/rdma_core.h
+++ b/drivers/infiniband/core/rdma_core.h
@@ -151,6 +151,9 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
 			      unsigned int num_attrs);
 void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile);
 
+typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs);
+uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata);
+
 extern const struct uapi_definition uverbs_def_obj_async_fd[];
 extern const struct uapi_definition uverbs_def_obj_counters[];
 extern const struct uapi_definition uverbs_def_obj_cq[];
diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index fc45c384833fa..4fafe393a48c7 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -608,14 +608,29 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num,
 	if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) {
 		ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt,
 				sg_offset, remote_addr, rkey, dir);
-	} else if (sg_cnt > 1) {
+		/*
+		 * If MR init succeeded or failed for a reason other
+		 * than pool exhaustion, that result is final.
+		 *
+		 * Pool exhaustion (-EAGAIN) from the max_sgl_rd
+		 * optimization is recoverable: fall back to
+		 * direct SGE posting. iWARP and force_mr require
+		 * MRs unconditionally, so -EAGAIN is terminal.
+		 */
+		if (ret != -EAGAIN ||
+		    rdma_protocol_iwarp(qp->device, port_num) ||
+		    unlikely(rdma_rw_force_mr))
+			goto out;
+	}
+
+	if (sg_cnt > 1)
 		ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset,
 				remote_addr, rkey, dir);
-	} else {
+	else
 		ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset,
 				remote_addr, rkey, dir);
-	}
 
+out:
 	if (ret < 0)
 		goto out_unmap_sg;
 	return ret;
@@ -686,14 +701,16 @@ int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		return ret;
 
 	/*
-	 * IOVA mapping not available. Check if MR registration provides
-	 * better performance than multiple SGE entries.
+	 * IOVA not available; fall back to the map_wrs path, which maps
+	 * each bvec as a direct SGE. This is always correct: the MR path
+	 * is a throughput optimization, not a correctness requirement.
+	 * (iWARP, which does require MRs, is handled by the check above.)
+	 *
+	 * The rdma_rw_io_needs_mr() gate is not used here because nr_bvec
+	 * is a raw page count that overstates DMA entry demand -- the bvec
+	 * caller has no post-DMA-coalescing segment count, and feeding the
+	 * inflated count into the MR path exhausts the pool on RDMA READs.
 	 */
-	if (rdma_rw_io_needs_mr(dev, port_num, dir, nr_bvec))
-		return rdma_rw_init_mr_wrs_bvec(ctx, qp, port_num, bvecs,
-						nr_bvec, &iter, remote_addr,
-						rkey, dir);
-
 	return rdma_rw_init_map_wrs_bvec(ctx, qp, bvecs, nr_bvec, &iter,
 			remote_addr, rkey, dir);
 }
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index cff4fcca2c345..1b6c28f090e38 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -55,7 +55,8 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 
 	if (dirty)
 		ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
-					   DMA_BIDIRECTIONAL, 0);
+					   DMA_BIDIRECTIONAL,
+					   DMA_ATTR_REQUIRE_COHERENT);
 
 	for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) {
 		unpin_user_page_range_dirty_lock(sg_page(sg),
@@ -169,7 +170,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
 	unsigned long lock_limit;
 	unsigned long new_pinned;
 	unsigned long cur_base;
-	unsigned long dma_attr = 0;
+	unsigned long dma_attr = DMA_ATTR_REQUIRE_COHERENT;
 	struct mm_struct *mm;
 	unsigned long npages;
 	int pinned, ret;
@@ -283,7 +284,7 @@ EXPORT_SYMBOL(ib_umem_get);
  */
 void ib_umem_release(struct ib_umem *umem)
 {
-	if (!umem)
+	if (IS_ERR_OR_NULL(umem))
 		return;
 	if (umem->is_dmabuf)
 		return ib_umem_dmabuf_release(to_ib_umem_dmabuf(umem));
diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c
index d30f24b90bca9..ad023c2d84d85 100644
--- a/drivers/infiniband/core/umem_dmabuf.c
+++ b/drivers/infiniband/core/umem_dmabuf.c
@@ -181,32 +181,48 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
 }
 EXPORT_SYMBOL(ib_umem_dmabuf_get);
 
-static void
-ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)
+static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
+	.allow_peer2peer = true,
+};
+
+static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach)
 {
 	struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
 
-	ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev,
-			       "Invalidate callback should not be called when memory is pinned\n");
+	dma_resv_assert_held(attach->dmabuf->resv);
+
+	if (umem_dmabuf->revoked)
+		return;
+
+	if (umem_dmabuf->pinned_revoke)
+		umem_dmabuf->pinned_revoke(umem_dmabuf->private);
+
+	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
+	if (umem_dmabuf->pinned) {
+		dma_buf_unpin(umem_dmabuf->attach);
+		umem_dmabuf->pinned = 0;
+	}
+	umem_dmabuf->revoked = 1;
 }
 
-static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
+static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = {
 	.allow_peer2peer = true,
-	.move_notify = ib_umem_dmabuf_unsupported_move_notify,
+	.invalidate_mappings = ib_umem_dmabuf_revoke_locked,
 };
 
-struct ib_umem_dmabuf *
-ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
-					  struct device *dma_device,
-					  unsigned long offset, size_t size,
-					  int fd, int access)
+static struct ib_umem_dmabuf *
+ib_umem_dmabuf_get_pinned_and_lock(struct ib_device *device,
+				   struct device *dma_device,
+				   unsigned long offset,
+				   size_t size, int fd, int access,
+				   const struct dma_buf_attach_ops *ops)
 {
 	struct ib_umem_dmabuf *umem_dmabuf;
 	int err;
 
-	umem_dmabuf = ib_umem_dmabuf_get_with_dma_device(device, dma_device, offset,
-							 size, fd, access,
-							 &ib_umem_dmabuf_attach_pinned_ops);
+	umem_dmabuf =
+		ib_umem_dmabuf_get_with_dma_device(device, dma_device, offset,
+						   size, fd, access, ops);
 	if (IS_ERR(umem_dmabuf))
 		return umem_dmabuf;
 
@@ -219,7 +235,6 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
 	err = ib_umem_dmabuf_map_pages(umem_dmabuf);
 	if (err)
 		goto err_release;
-	dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
 
 	return umem_dmabuf;
 
@@ -228,8 +243,77 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
 	ib_umem_release(&umem_dmabuf->umem);
 	return ERR_PTR(err);
 }
+
+struct ib_umem_dmabuf *
+ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
+					  struct device *dma_device,
+					  unsigned long offset, size_t size,
+					  int fd, int access)
+{
+	struct ib_umem_dmabuf *umem_dmabuf =
+		ib_umem_dmabuf_get_pinned_and_lock(device, dma_device, offset,
+						   size, fd, access,
+						   &ib_umem_dmabuf_attach_pinned_ops);
+	if (IS_ERR(umem_dmabuf))
+		return umem_dmabuf;
+
+	dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
+	return umem_dmabuf;
+}
 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_with_dma_device);
 
+/**
+ * ib_umem_dmabuf_get_pinned_revocable_and_lock - Map & pin a revocable dmabuf
+ * @device: IB device.
+ * @offset: Start offset.
+ * @size: Length.
+ * @fd: dmabuf fd.
+ * @access: Access flags.
+ *
+ * Obtains a umem from a dmabuf for drivers/devices that can support revocation.
+ *
+ * Returns with dma_resv_lock held upon success. The driver must set the revoke
+ * callback prior to unlock by calling ib_umem_dmabuf_set_revoke_locked().
+ *
+ * When a revocation occurs, the revoke callback will be called. The driver must
+ * ensure that the region is no longer accessed when the callback returns. Any
+ * subsequent access attempts should also probably cause an AE for MRs.
+ *
+ * If the umem is used for an MR, the driver must ensure that the key remains in
+ * use such that it cannot be obtained by a new region until this region is
+ * fully deregistered (i.e., ibv_dereg_mr). If a driver needs to serialize with
+ * revoke calls, it can use dma_resv_lock.
+ *
+ * If successful, then the revoke callback may be called at any time and will
+ * also be called automatically upon ib_umem_release (serialized). The revoke
+ * callback will be called one time at most.
+ *
+ * Return: A pointer to ib_umem_dmabuf on success, or an ERR_PTR on failure.
+ */
+struct ib_umem_dmabuf *
+ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device,
+					     unsigned long offset, size_t size,
+					     int fd, int access)
+{
+	const struct dma_buf_attach_ops *ops =
+		&ib_umem_dmabuf_attach_pinned_revocable_ops;
+
+	return ib_umem_dmabuf_get_pinned_and_lock(device, device->dma_device,
+						  offset, size, fd, access,
+						  ops);
+}
+EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_revocable_and_lock);
+
+void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf,
+				      void (*revoke)(void *priv), void *priv)
+{
+	dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
+
+	umem_dmabuf->pinned_revoke = revoke;
+	umem_dmabuf->private = priv;
+}
+EXPORT_SYMBOL(ib_umem_dmabuf_set_revoke_locked);
+
 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
 						 unsigned long offset,
 						 size_t size, int fd,
@@ -240,20 +324,28 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
 }
 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);
 
+void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf)
+{
+	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
+
+	dma_resv_lock(dmabuf->resv, NULL);
+}
+EXPORT_SYMBOL(ib_umem_dmabuf_revoke_lock);
+
+void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf)
+{
+	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
+
+	dma_resv_unlock(dmabuf->resv);
+}
+EXPORT_SYMBOL(ib_umem_dmabuf_revoke_unlock);
+
 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf)
 {
 	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
 
 	dma_resv_lock(dmabuf->resv, NULL);
-	if (umem_dmabuf->revoked)
-		goto end;
-	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
-	if (umem_dmabuf->pinned) {
-		dma_buf_unpin(umem_dmabuf->attach);
-		umem_dmabuf->pinned = 0;
-	}
-	umem_dmabuf->revoked = 1;
-end:
+	ib_umem_dmabuf_revoke_locked(umem_dmabuf->attach);
 	dma_resv_unlock(dmabuf->resv);
 }
 EXPORT_SYMBOL(ib_umem_dmabuf_revoke);
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 758ed4ae5f7a8..f31650ef7bc34 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -83,28 +83,16 @@ static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
 	return 0;
 }
 
-/*
- * Copy a request from userspace. If the provided 'req' is larger than the
- * user buffer then the user buffer is zero extended into the 'req'. If 'req'
- * is smaller than the user buffer then the uncopied bytes in the user buffer
- * must be zero.
- */
 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
 			  size_t req_len)
 {
-	if (copy_from_user(req, attrs->ucore.inbuf,
-			   min(attrs->ucore.inlen, req_len)))
-		return -EFAULT;
+	int ret;
 
-	if (attrs->ucore.inlen < req_len) {
-		memset(req + attrs->ucore.inlen, 0,
-		       req_len - attrs->ucore.inlen);
-	} else if (attrs->ucore.inlen > req_len) {
-		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
-					  attrs->ucore.inlen - req_len))
-			return -EOPNOTSUPP;
-	}
-	return 0;
+	ret = copy_struct_from_user(req, req_len, attrs->ucore.inbuf,
+				    attrs->ucore.inlen);
+	if (ret == -E2BIG)
+		ret = -EOPNOTSUPP;
+	return ret;
 }
 
 /*
@@ -1032,6 +1020,9 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
 	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
 		return -EINVAL;
 
+	if (!cmd->cqe)
+		return -EINVAL;
+
 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
 						 &ib_dev);
 	if (IS_ERR(obj))
@@ -1068,7 +1059,10 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
 	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
 	rdma_restrack_set_name(&cq->res, NULL);
 
-	ret = ib_dev->ops.create_cq(cq, &attr, attrs);
+	if (ib_dev->ops.create_user_cq)
+		ret = ib_dev->ops.create_user_cq(cq, &attr, attrs);
+	else
+		ret = ib_dev->ops.create_cq(cq, &attr, attrs);
 	if (ret)
 		goto err_free;
 	rdma_restrack_add(&cq->res);
@@ -1085,6 +1079,7 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
 	return uverbs_response(attrs, &resp, sizeof(resp));
 
 err_free:
+	ib_umem_release(cq->umem);
 	rdma_restrack_put(&cq->res);
 	kfree(cq);
 err_file:
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index f37bb447c2306..b61af625e679b 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -70,6 +70,19 @@ struct bundle_priv {
 	u64 internal_buffer[32];
 };
 
+uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
+{
+	struct uverbs_attr_bundle *bundle =
+		rdma_udata_to_uverbs_attr_bundle(udata);
+	struct bundle_priv *pbundle =
+		container_of(&bundle->hdr, struct bundle_priv, bundle);
+
+	lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
+
+	return srcu_dereference(pbundle->method_elm->handler,
+				&bundle->ufile->device->disassociate_srcu);
+}
+
 /*
  * Each method has an absolute minimum amount of memory it needs to allocate,
  * precompute that amount and determine if the onstack memory can be used or
@@ -847,3 +860,77 @@ void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
 		  pbundle->uobj_hw_obj_valid);
 }
 EXPORT_SYMBOL(uverbs_finalize_uobj_create);
+
+int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
+			       size_t kernel_size, size_t minimum_size)
+{
+	int err;
+
+	if (udata->inlen < minimum_size) {
+		ibdev_dbg(
+			rdma_udata_to_dev(udata),
+			"System call driver input udata too small (%zu < %zu) for ioctl %ps called by %pSR\n",
+			udata->inlen, minimum_size,
+			uverbs_get_handler_fn(udata),
+			__builtin_return_address(0));
+		return -EINVAL;
+	}
+
+	err = copy_struct_from_user(req, kernel_size, udata->inbuf,
+				    udata->inlen);
+	if (err) {
+		if (err == -E2BIG) {
+			ibdev_dbg(
+				rdma_udata_to_dev(udata),
+				"System call driver input udata not zero from %zu -> %zu for ioctl %ps called by %pSR\n",
+				minimum_size, udata->inlen,
+				uverbs_get_handler_fn(udata),
+				__builtin_return_address(0));
+			return -EOPNOTSUPP;
+		}
+		ibdev_dbg(
+			rdma_udata_to_dev(udata),
+			"System call driver input udata EFAULT for ioctl %ps called by %pSR\n",
+			uverbs_get_handler_fn(udata),
+			__builtin_return_address(0));
+		return err;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_in);
+
+int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
+				    u64 valid_cm)
+{
+	ibdev_dbg(
+		rdma_udata_to_dev(udata),
+		"System call driver input udata has unsupported comp_mask %llx & ~%llx = %llx for ioctl %ps called by %pSR\n",
+		req_cm, valid_cm, req_cm & ~valid_cm,
+		uverbs_get_handler_fn(udata), __builtin_return_address(0));
+	return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_cm_fail);
+
+int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
+{
+	size_t copy_len;
+
+	/* 0 length copy_len is a NOP for copy_to_user() and doesn't fail. */
+	copy_len = min(len, udata->outlen);
+	if (copy_to_user(udata->outbuf, src, copy_len))
+		goto err_fault;
+	if (copy_len < udata->outlen) {
+		if (clear_user(udata->outbuf + copy_len,
+			       udata->outlen - copy_len))
+			goto err_fault;
+	}
+	return 0;
+err_fault:
+	ibdev_dbg(
+		rdma_udata_to_dev(udata),
+		"System call driver out udata has EFAULT (%zu into %zu) for ioctl %ps called by %pSR\n",
+		len, udata->outlen, uverbs_get_handler_fn(udata),
+		__builtin_return_address(0));
+	return -EFAULT;
+}
+EXPORT_SYMBOL(_ib_respond_udata);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 7b68967a6301e..f5837da47299c 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -756,7 +756,7 @@ static void rdma_umap_open(struct vm_area_struct *vma)
 	 * point, so zap it.
 	 */
 	vma->vm_private_data = NULL;
-	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
+	zap_special_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start);
 }
 
 static void rdma_umap_close(struct vm_area_struct *vma)
@@ -782,7 +782,7 @@ static void rdma_umap_close(struct vm_area_struct *vma)
 }
 
 /*
- * Once the zap_vma_ptes has been called touches to the VMA will come here and
+ * Once the zap_special_vma_range has been called touches to the VMA will come here and
  * we return a dummy writable zero page for all the pfns.
  */
 static vm_fault_t rdma_umap_fault(struct vm_fault *vmf)
@@ -878,7 +878,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 				continue;
 			list_del_init(&priv->list);
 
-			zap_vma_ptes(vma, vma->vm_start,
+			zap_special_vma_range(vma, vma->vm_start,
 				     vma->vm_end - vma->vm_start);
 
 			if (priv->entry) {
diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index fab5d914029dd..d2c8f71f934cc 100644
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -78,17 +78,21 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	int buffer_fd;
 	int ret;
 
-	if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_cq_umem) || !ib_dev->ops.destroy_cq)
+	if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_user_cq) ||
+	    !ib_dev->ops.destroy_cq)
 		return -EOPNOTSUPP;
 
 	ret = uverbs_copy_from(&attr.comp_vector, attrs,
 			       UVERBS_ATTR_CREATE_CQ_COMP_VECTOR);
-	if (!ret)
-		ret = uverbs_copy_from(&attr.cqe, attrs,
-				       UVERBS_ATTR_CREATE_CQ_CQE);
-	if (!ret)
-		ret = uverbs_copy_from(&user_handle, attrs,
-				       UVERBS_ATTR_CREATE_CQ_USER_HANDLE);
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_from(&attr.cqe, attrs, UVERBS_ATTR_CREATE_CQ_CQE);
+	if (ret || !attr.cqe)
+		return ret ? : -EINVAL;
+
+	ret = uverbs_copy_from(&user_handle, attrs,
+			       UVERBS_ATTR_CREATE_CQ_USER_HANDLE);
 	if (ret)
 		return ret;
 
@@ -130,7 +134,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 
 		if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) ||
 		    uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
-		    !ib_dev->ops.create_cq_umem) {
+		    !ib_dev->ops.create_user_cq) {
 			ret = -EINVAL;
 			goto err_event_file;
 		}
@@ -155,7 +159,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 			goto err_event_file;
 
 		if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) ||
-		    !ib_dev->ops.create_cq_umem) {
+		    !ib_dev->ops.create_user_cq) {
 			ret = -EINVAL;
 			goto err_event_file;
 		}
@@ -186,16 +190,26 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	cq->comp_handler  = ib_uverbs_comp_handler;
 	cq->event_handler = ib_uverbs_cq_event_handler;
 	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
+	/*
+	 * If UMEM is not provided here, legacy drivers will set it during
+	 * CQ creation based on their internal udata.
+	 */
+	cq->umem = umem;
 	atomic_set(&cq->usecnt, 0);
 
 	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
 	rdma_restrack_set_name(&cq->res, NULL);
 
-	ret = umem ? ib_dev->ops.create_cq_umem(cq, &attr, umem, attrs) :
-		ib_dev->ops.create_cq(cq, &attr, attrs);
+	if (ib_dev->ops.create_user_cq)
+		ret = ib_dev->ops.create_user_cq(cq, &attr, attrs);
+	else
+		ret = ib_dev->ops.create_cq(cq, &attr, attrs);
 	if (ret)
 		goto err_free;
 
+	/* Check that driver didn't overrun existing umem */
+	WARN_ON(umem && cq->umem != umem);
+
 	obj->uevent.uobject.object = cq;
 	obj->uevent.uobject.user_handle = user_handle;
 	rdma_restrack_add(&cq->res);
@@ -206,7 +220,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	return ret;
 
 err_free:
-	ib_umem_release(umem);
+	ib_umem_release(cq->umem);
 	rdma_restrack_put(&cq->res);
 	kfree(cq);
 err_event_file:
diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c
index a28f9f21bed89..12ca15739cd2c 100644
--- a/drivers/infiniband/core/uverbs_std_types_device.c
+++ b/drivers/infiniband/core/uverbs_std_types_device.c
@@ -247,13 +247,21 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(
 {
 	u32 num_comp = attrs->ufile->device->num_comp_vectors;
 	u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;
+	struct ib_device *ib_dev;
 	int ret;
 
+	ib_dev = srcu_dereference(attrs->ufile->device->ib_dev,
+				  &attrs->ufile->device->disassociate_srcu);
+	if (!ib_dev)
+		return -EIO;
+
 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
 			     &num_comp, sizeof(num_comp));
 	if (IS_UVERBS_COPY_ERR(ret))
 		return ret;
 
+	if (ib_dev->ops.uverbs_robust_udata)
+		core_support |= IB_UVERBS_CORE_SUPPORT_ROBUST_UDATA;
 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,
 			     &core_support, sizeof(core_support));
 	if (IS_UVERBS_COPY_ERR(ret))
diff --git a/drivers/infiniband/core/uverbs_std_types_dmabuf.c b/drivers/infiniband/core/uverbs_std_types_dmabuf.c
index 4a7f8b6f9dc8f..2411ebee69e2d 100644
--- a/drivers/infiniband/core/uverbs_std_types_dmabuf.c
+++ b/drivers/infiniband/core/uverbs_std_types_dmabuf.c
@@ -169,7 +169,7 @@ static void uverbs_dmabuf_fd_destroy_uobj(struct ib_uobject *uobj,
 	if (!uverbs_dmabuf->revoked) {
 		uverbs_dmabuf->revoked = true;
 		list_del(&uverbs_dmabuf->dmabufs_elm);
-		dma_buf_move_notify(uverbs_dmabuf->dmabuf);
+		dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf);
 		dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv,
 				      DMA_RESV_USAGE_BOOKKEEP, false,
 				      MAX_SCHEDULE_TIMEOUT);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 575b4a4b200bb..721cd43212380 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -49,6 +49,7 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_cache.h>
 #include <rdma/ib_addr.h>
+#include <rdma/ib_umem.h>
 #include <rdma/rw.h>
 #include <rdma/lag.h>
 
@@ -2202,8 +2203,10 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
 	if (!cq)
 		return ERR_PTR(-ENOMEM);
 
+	if (WARN_ON_ONCE(!cq_attr->cqe))
+		return ERR_PTR(-EINVAL);
+
 	cq->device = device;
-	cq->uobject = NULL;
 	cq->comp_handler = comp_handler;
 	cq->event_handler = event_handler;
 	cq->cq_context = cq_context;
@@ -2218,6 +2221,11 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
 		kfree(cq);
 		return ERR_PTR(ret);
 	}
+	/*
+	 * We are in kernel verbs flow and drivers are not allowed
+	 * to set umem pointer, it needs to stay NULL.
+	 */
+	WARN_ON_ONCE(cq->umem);
 
 	rdma_restrack_add(&cq->res);
 	return cq;
@@ -2249,6 +2257,7 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 	if (ret)
 		return ret;
 
+	ib_umem_release(cq->umem);
 	rdma_restrack_del(&cq->res);
 	kfree(cq);
 	return ret;
@@ -3154,44 +3163,6 @@ int rdma_init_netdev(struct ib_device *device, u32 port_num,
 }
 EXPORT_SYMBOL(rdma_init_netdev);
 
-void __rdma_block_iter_start(struct ib_block_iter *biter,
-			     struct scatterlist *sglist, unsigned int nents,
-			     unsigned long pgsz)
-{
-	memset(biter, 0, sizeof(struct ib_block_iter));
-	biter->__sg = sglist;
-	biter->__sg_nents = nents;
-
-	/* Driver provides best block size to use */
-	biter->__pg_bit = __fls(pgsz);
-}
-EXPORT_SYMBOL(__rdma_block_iter_start);
-
-bool __rdma_block_iter_next(struct ib_block_iter *biter)
-{
-	unsigned int block_offset;
-	unsigned int delta;
-
-	if (!biter->__sg_nents || !biter->__sg)
-		return false;
-
-	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
-	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
-	delta = BIT_ULL(biter->__pg_bit) - block_offset;
-
-	while (biter->__sg_nents && biter->__sg &&
-	       sg_dma_len(biter->__sg) - biter->__sg_advance <= delta) {
-		delta -= sg_dma_len(biter->__sg) - biter->__sg_advance;
-		biter->__sg_advance = 0;
-		biter->__sg = sg_next(biter->__sg);
-		biter->__sg_nents--;
-	}
-	biter->__sg_advance += delta;
-
-	return true;
-}
-EXPORT_SYMBOL(__rdma_block_iter_next);
-
 /**
  * rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct
  *   for the drivers.
diff --git a/drivers/infiniband/hw/bng_re/bng_dev.c b/drivers/infiniband/hw/bng_re/bng_dev.c
index d34b5f88cd40b..71a7ca2196ad8 100644
--- a/drivers/infiniband/hw/bng_re/bng_dev.c
+++ b/drivers/infiniband/hw/bng_re/bng_dev.c
@@ -210,7 +210,7 @@ static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev)
 	return rc;
 }
 
-static void bng_re_query_hwrm_version(struct bng_re_dev *rdev)
+static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
 {
 	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
 	struct hwrm_ver_get_output ver_get_resp = {};
@@ -230,7 +230,7 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev)
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
 			  rc);
-		return;
+		return rc;
 	}
 
 	cctx = rdev->chip_ctx;
@@ -244,6 +244,8 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev)
 
 	if (!cctx->hwrm_cmd_max_timeout)
 		cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT;
+
+	return 0;
 }
 
 static void bng_re_dev_uninit(struct bng_re_dev *rdev)
@@ -306,13 +308,15 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
 		goto msix_ctx_fail;
 	}
 
-	bng_re_query_hwrm_version(rdev);
+	rc = bng_re_query_hwrm_version(rdev);
+	if (rc)
+		goto destroy_chip_ctx;
 
 	rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw);
 	if (rc) {
 		ibdev_err(&rdev->ibdev,
 			  "Failed to allocate RCFW Channel: %#x\n", rc);
-		goto alloc_fw_chl_fail;
+		goto destroy_chip_ctx;
 	}
 
 	/* Allocate nq record memory */
@@ -391,7 +395,7 @@ static int bng_re_dev_init(struct bng_re_dev *rdev)
 	kfree(rdev->nqr);
 nq_alloc_fail:
 	bng_re_free_rcfw_channel(&rdev->rcfw);
-alloc_fw_chl_fail:
+destroy_chip_ctx:
 	bng_re_destroy_chip_ctx(rdev);
 msix_ctx_fail:
 	bnge_unregister_dev(rdev->aux_dev);
diff --git a/drivers/infiniband/hw/bnxt_re/Makefile b/drivers/infiniband/hw/bnxt_re/Makefile
index f63417d2ccc64..1533c079c9da1 100644
--- a/drivers/infiniband/hw/bnxt_re/Makefile
+++ b/drivers/infiniband/hw/bnxt_re/Makefile
@@ -5,4 +5,4 @@ obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re.o
 bnxt_re-y := main.o ib_verbs.o \
 	     qplib_res.o qplib_rcfw.o	\
 	     qplib_sp.o qplib_fp.o  hw_counters.o	\
-	     debugfs.o
+	     debugfs.o uapi.o
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 5c5ecfacf5061..182128ee4f242 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -187,9 +187,12 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 	struct bnxt_re_query_device_ex_resp resp = {};
-	size_t outlen = (udata) ? udata->outlen : 0;
 	int rc = 0;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	memset(ib_attr, 0, sizeof(*ib_attr));
 	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 	       min(sizeof(dev_attr->fw_ver),
@@ -254,8 +257,7 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	ib_attr->max_pkeys = 1;
 	ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 
-	if ((offsetofend(typeof(resp), packet_pacing_caps) <= outlen) &&
-	    _is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2)) {
+	if (_is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2)) {
 		resp.packet_pacing_caps.qp_rate_limit_min =
 			dev_attr->rate_limit_min;
 		resp.packet_pacing_caps.qp_rate_limit_max =
@@ -263,11 +265,7 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 		resp.packet_pacing_caps.supported_qpts =
 			1 << IB_QPT_RC;
 	}
-	if (outlen)
-		rc = ib_copy_to_udata(udata, &resp,
-				      min(sizeof(resp), outlen));
-
-	return rc;
+	return ib_respond_udata(udata, resp);
 }
 
 int bnxt_re_modify_device(struct ib_device *ibdev,
@@ -644,7 +642,7 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 	return rc;
 }
 
-static struct bnxt_re_user_mmap_entry*
+struct bnxt_re_user_mmap_entry*
 bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
 			  enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
 {
@@ -692,6 +690,11 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 {
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	struct bnxt_re_dev *rdev = pd->rdev;
+	int ret;
+
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ret;
 
 	if (udata) {
 		rdma_user_mmap_entry_remove(pd->pd_db_mmap);
@@ -706,7 +709,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 					   &pd->qplib_pd))
 			atomic_dec(&rdev->stats.res.pd_count);
 	}
-	return 0;
+	return ib_respond_empty_udata(udata);
 }
 
 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
@@ -720,6 +723,10 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	u32 active_pds;
 	int rc = 0;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	pd->rdev = rdev;
 	if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
 		ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
@@ -756,7 +763,7 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 
 		pd->pd_db_mmap = &entry->rdma_entry;
 
-		rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
+		rc = ib_respond_udata(udata, resp);
 		if (rc) {
 			rdma_user_mmap_entry_remove(pd->pd_db_mmap);
 			rc = -EFAULT;
@@ -834,6 +841,10 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 	u8 nw_type;
 	int rc;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 		ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 		return -EINVAL;
@@ -887,7 +898,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 	if (active_ahs > rdev->stats.res.ah_watermark)
 		rdev->stats.res.ah_watermark = active_ahs;
 
-	return 0;
+	return ib_respond_empty_udata(udata);
 }
 
 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
@@ -984,6 +995,12 @@ static void bnxt_re_del_unique_gid(struct bnxt_re_dev *rdev)
 		dev_err(rdev_to_dev(rdev), "Failed to delete unique GID, rc: %d\n", rc);
 }
 
+static void bnxt_re_qp_free_umem(struct bnxt_re_qp *qp)
+{
+	ib_umem_release(qp->rumem);
+	ib_umem_release(qp->sumem);
+}
+
 /* Queue Pairs */
 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 {
@@ -995,6 +1012,10 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 	unsigned int flags;
 	int rc;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	bnxt_re_debug_rem_qpinfo(rdev, qp);
 
 	bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
@@ -1026,8 +1047,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)
 		bnxt_re_del_unique_gid(rdev);
 
-	ib_umem_release(qp->rumem);
-	ib_umem_release(qp->sumem);
+	bnxt_re_qp_free_umem(qp);
 
 	/* Flush all the entries of notification queue associated with
 	 * given qp.
@@ -1038,7 +1058,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 	if (scq_nq != rcq_nq)
 		bnxt_re_synchronize_nq(rcq_nq);
 
-	return 0;
+	return ib_respond_empty_udata(udata);
 }
 
 static u8 __from_ib_qp_type(enum ib_qp_type type)
@@ -1171,6 +1191,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 	}
 
 	qplib_qp->dpi = &cntx->dpi;
+	qplib_qp->is_user = true;
 	return 0;
 rqfail:
 	ib_umem_release(qp->sumem);
@@ -1228,6 +1249,114 @@ static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 	return NULL;
 }
 
+static int bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_res *res = &qp->rdev->qplib_res;
+	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct bnxt_qplib_hwq *irrq, *orrq;
+	int rc, req_size;
+
+	orrq = &qplib_qp->orrq;
+	orrq->max_elements =
+		ORD_LIMIT_TO_ORRQ_SLOTS(qplib_qp->max_rd_atomic);
+	req_size = orrq->max_elements *
+		BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+	req_size &= ~(PAGE_SIZE - 1);
+	sginfo.pgsize = req_size;
+	sginfo.pgshft = PAGE_SHIFT;
+
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.depth = orrq->max_elements;
+	hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
+	hwq_attr.aux_stride = 0;
+	hwq_attr.aux_depth = 0;
+	hwq_attr.type = HWQ_TYPE_CTX;
+	rc = bnxt_qplib_alloc_init_hwq(orrq, &hwq_attr);
+	if (rc)
+		return rc;
+
+	irrq = &qplib_qp->irrq;
+	irrq->max_elements =
+		IRD_LIMIT_TO_IRRQ_SLOTS(qplib_qp->max_dest_rd_atomic);
+	req_size = irrq->max_elements *
+		BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+	req_size &= ~(PAGE_SIZE - 1);
+	sginfo.pgsize = req_size;
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.depth =  irrq->max_elements;
+	hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
+	rc = bnxt_qplib_alloc_init_hwq(irrq, &hwq_attr);
+	if (rc)
+		goto free_orrq_hwq;
+	return 0;
+free_orrq_hwq:
+	bnxt_qplib_free_hwq(res, orrq);
+	return rc;
+}
+
+static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_res *res = &qp->rdev->qplib_res;
+	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_q *sq = &qplib_qp->sq;
+	struct bnxt_qplib_q *rq = &qplib_qp->rq;
+	u8 wqe_mode = qplib_qp->wqe_mode;
+	u8 pg_sz_lvl;
+	int rc;
+
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &sq->sg_info;
+	hwq_attr.stride = bnxt_qplib_get_stride();
+	hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true);
+	hwq_attr.aux_stride = qplib_qp->psn_sz;
+	hwq_attr.aux_depth = (qplib_qp->psn_sz) ?
+		bnxt_qplib_set_sq_size(sq, wqe_mode) : 0;
+	if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz)
+		hwq_attr.aux_depth = qplib_qp->msn_tbl_sz;
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
+	if (rc)
+		return rc;
+
+	pg_sz_lvl = bnxt_qplib_base_pg_size(&sq->hwq) << CMDQ_CREATE_QP_SQ_PG_SIZE_SFT;
+	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<
+		      CMDQ_CREATE_QP_SQ_LVL_SFT);
+	sq->hwq.pg_sz_lvl = pg_sz_lvl;
+
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &rq->sg_info;
+	hwq_attr.stride = bnxt_qplib_get_stride();
+	hwq_attr.depth = bnxt_qplib_get_depth(rq, qplib_qp->wqe_mode, false);
+	hwq_attr.aux_stride = 0;
+	hwq_attr.aux_depth = 0;
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
+	if (rc)
+		goto free_sq_hwq;
+	pg_sz_lvl = bnxt_qplib_base_pg_size(&rq->hwq) <<
+		CMDQ_CREATE_QP_RQ_PG_SIZE_SFT;
+	pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
+		      CMDQ_CREATE_QP_RQ_LVL_SFT);
+	rq->hwq.pg_sz_lvl = pg_sz_lvl;
+
+	if (qplib_qp->psn_sz) {
+		rc = bnxt_re_qp_alloc_init_xrrq(qp);
+		if (rc)
+			goto free_rq_hwq;
+	}
+
+	return 0;
+free_rq_hwq:
+	bnxt_qplib_free_hwq(res, &rq->hwq);
+free_sq_hwq:
+	bnxt_qplib_free_hwq(res, &sq->hwq);
+	return rc;
+}
+
 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 				(struct bnxt_re_pd *pd,
 				 struct bnxt_qplib_res *qp1_res,
@@ -1249,6 +1378,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	qp->qplib_qp.pd = &pd->qplib_pd;
 	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
 	qp->qplib_qp.type = IB_QPT_UD;
+	qp->qplib_qp.cctx = rdev->chip_ctx;
 
 	qp->qplib_qp.max_inline_data = 0;
 	qp->qplib_qp.sig_type = true;
@@ -1281,10 +1411,14 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
 	qp->qplib_qp.dpi = &rdev->dpi_privileged;
 
-	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
+	rc = bnxt_re_setup_qp_hwqs(qp);
 	if (rc)
 		goto fail;
 
+	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
+	if (rc)
+		goto free_hwq;
+
 	spin_lock_init(&qp->sq_lock);
 	INIT_LIST_HEAD(&qp->list);
 	mutex_lock(&rdev->qp_lock);
@@ -1292,6 +1426,9 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	atomic_inc(&rdev->stats.res.qp_count);
 	mutex_unlock(&rdev->qp_lock);
 	return qp;
+
+free_hwq:
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 fail:
 	kfree(qp);
 	return NULL;
@@ -1462,6 +1599,39 @@ static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
 	return qptype;
 }
 
+static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
+	struct bnxt_qplib_q *sq = &qplib_qp->sq;
+	struct bnxt_re_dev *rdev = qp->rdev;
+	u8 wqe_mode = qplib_qp->wqe_mode;
+
+	if (rdev->dev_attr)
+		qplib_qp->is_host_msn_tbl =
+			_is_host_msn_table(rdev->dev_attr->dev_cap_flags2);
+
+	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+		qplib_qp->psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
+			sizeof(struct sq_psn_search_ext) :
+			sizeof(struct sq_psn_search);
+		if (qplib_qp->is_host_msn_tbl) {
+			qplib_qp->psn_sz = sizeof(struct sq_msn_search);
+			qplib_qp->msn = 0;
+		}
+	}
+
+	/* Update msn tbl size */
+	if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) {
+		if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+			qplib_qp->msn_tbl_sz =
+				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode));
+		else
+			qplib_qp->msn_tbl_sz =
+				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2;
+		qplib_qp->msn = 0;
+	}
+}
+
 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 				struct ib_qp_init_attr *init_attr,
 				struct bnxt_re_ucontext *uctx,
@@ -1484,12 +1654,12 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 	qplqp->max_inline_data = init_attr->cap.max_inline_data;
 	qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
 	qptype = bnxt_re_init_qp_type(rdev, init_attr);
-	if (qptype < 0) {
-		rc = qptype;
-		goto out;
-	}
+	if (qptype < 0)
+		return qptype;
 	qplqp->type = (u8)qptype;
 	qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
+	qplqp->dev_cap_flags = dev_attr->dev_cap_flags;
+	qplqp->cctx = rdev->chip_ctx;
 	if (init_attr->qp_type == IB_QPT_RC) {
 		qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
 		qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
@@ -1519,20 +1689,32 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 	/* Setup RQ/SRQ */
 	rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
 	if (rc)
-		goto out;
+		return rc;
 	if (init_attr->qp_type == IB_QPT_GSI)
 		bnxt_re_adjust_gsi_rq_attr(qp);
 
 	/* Setup SQ */
 	rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
 	if (rc)
-		goto out;
+		return rc;
 	if (init_attr->qp_type == IB_QPT_GSI)
 		bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
 
-	if (uctx) /* This will update DPI and qp_handle */
+	if (uctx) { /* This will update DPI and qp_handle */
 		rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);
-out:
+		if (rc)
+			return rc;
+	}
+
+	bnxt_re_qp_calculate_msn_psn_size(qp);
+
+	rc = bnxt_re_setup_qp_hwqs(qp);
+	if (rc)
+		goto free_umem;
+
+	return 0;
+free_umem:
+	bnxt_re_qp_free_umem(qp);
 	return rc;
 }
 
@@ -1589,6 +1771,7 @@ static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 
 	rdev = qp->rdev;
 	qplqp = &qp->qplib_qp;
+	qplqp->cctx = rdev->chip_ctx;
 
 	qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
 	qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
@@ -1671,9 +1854,11 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
 	qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 
 	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
-	if (udata)
-		if (ib_copy_from_udata(&ureq, udata,  min(udata->inlen, sizeof(ureq))))
-			return -EFAULT;
+	if (udata) {
+		rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0);
+		if (rc)
+			return rc;
+	}
 
 	rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
 	if (!rc) {
@@ -1692,23 +1877,22 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
 		if (rc == -ENODEV)
 			goto qp_destroy;
 		if (rc)
-			goto fail;
+			goto free_hwq;
 	} else {
 		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
 		if (rc) {
 			ibdev_err(&rdev->ibdev, "Failed to create HW QP");
-			goto free_umem;
+			goto free_hwq;
 		}
+
 		if (udata) {
 			struct bnxt_re_qp_resp resp;
 
 			resp.qpid = qp->qplib_qp.id;
 			resp.rsvd = 0;
-			rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
-			if (rc) {
-				ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
+			rc = ib_respond_udata(udata, resp);
+			if (rc)
 				goto qp_destroy;
-			}
 		}
 	}
 
@@ -1749,9 +1933,9 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
 	return 0;
 qp_destroy:
 	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
-free_umem:
-	ib_umem_release(qp->rumem);
-	ib_umem_release(qp->sumem);
+free_hwq:
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
+	bnxt_re_qp_free_umem(qp);
 fail:
 	return rc;
 }
@@ -1841,6 +2025,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
 					       ib_srq);
 	struct bnxt_re_dev *rdev = srq->rdev;
 	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
+	int ret;
+
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ret;
 
 	if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
 		free_page((unsigned long)srq->uctx_srq_page);
@@ -1849,7 +2038,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
 	bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
 	ib_umem_release(srq->umem);
 	atomic_dec(&rdev->stats.res.srq_count);
-	return 0;
+	return ib_respond_empty_udata(udata);
 }
 
 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
@@ -1863,9 +2052,11 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
 	int bytes = 0;
 	struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
 		udata, struct bnxt_re_ucontext, ib_uctx);
+	int rc;
 
-	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
-		return -EFAULT;
+	rc = ib_copy_validate_udata_in(udata, ureq, srq_handle);
+	if (rc)
+		return rc;
 
 	bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
 	bytes = PAGE_ALIGN(bytes);
@@ -1960,9 +2151,8 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 			}
 			resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
 		}
-		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		rc = ib_respond_udata(udata, resp);
 		if (rc) {
-			ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
 			bnxt_qplib_destroy_srq(&rdev->qplib_res,
 					       &srq->qplib_srq);
 			goto fail;
@@ -1988,6 +2178,11 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
 	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
 					       ib_srq);
 	struct bnxt_re_dev *rdev = srq->rdev;
+	int ret;
+
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ret;
 
 	switch (srq_attr_mask) {
 	case IB_SRQ_MAX_WR:
@@ -2004,7 +2199,7 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
 		/* On success, update the shadow */
 		srq->srq_limit = srq_attr->srq_limit;
 		/* No need to Build and send response back to udata */
-		return 0;
+		return ib_respond_empty_udata(udata);
 	default:
 		ibdev_err(&rdev->ibdev,
 			  "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
@@ -2105,6 +2300,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 	unsigned int flags;
 	u8 nw_type;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	if (qp_attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
 		return -EOPNOTSUPP;
 
@@ -2345,9 +2544,12 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
 		return rc;
 	}
-	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
+	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
 		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
-	return rc;
+		if (rc)
+			return rc;
+	}
+	return ib_respond_empty_udata(udata);
 }
 
 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
@@ -3122,12 +3324,17 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 	struct bnxt_qplib_nq *nq;
 	struct bnxt_re_dev *rdev;
 	struct bnxt_re_cq *cq;
+	int ret;
 
 	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
 	rdev = cq->rdev;
 	nq = cq->qplib_cq.nq;
 	cctx = rdev->chip_ctx;
 
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ret;
+
 	if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
 		free_page((unsigned long)cq->uctx_cq_page);
 		hash_del(&cq->hash_entry);
@@ -3135,15 +3342,34 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 	bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
 
 	bnxt_re_put_nq(rdev, nq);
-	ib_umem_release(cq->umem);
 
 	atomic_dec(&rdev->stats.res.cq_count);
 	kfree(cq->cql);
+	return ib_respond_empty_udata(udata);
+}
+
+static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev,
+				struct ib_umem *umem,
+				struct bnxt_qplib_sg_info *sginfo)
+{
+	unsigned long page_size;
+
+	if (!umem)
+		return -EINVAL;
+
+	page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0);
+	if (!page_size || page_size != SZ_4K)
+		return -EINVAL;
+
+	sginfo->umem = umem;
+	sginfo->npages = ib_umem_num_dma_blocks(umem, page_size);
+	sginfo->pgsize = page_size;
+	sginfo->pgshft = __builtin_ctz(page_size);
 	return 0;
 }
 
-int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		      struct uverbs_attr_bundle *attrs)
+int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs)
 {
 	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
@@ -3152,6 +3378,8 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
 	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_re_cq_resp resp = {};
+	struct bnxt_re_cq_req req;
 	int cqe = attr->cqe;
 	int rc, entries;
 	u32 active_cqs;
@@ -3173,34 +3401,29 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	if (entries > dev_attr->max_cq_wqes + 1)
 		entries = dev_attr->max_cq_wqes + 1;
 
-	cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
-	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
-	if (udata) {
-		struct bnxt_re_cq_req req;
-		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
-			rc = -EFAULT;
-			goto fail;
-		}
+	rc = ib_copy_validate_udata_in_cm(udata, req, cq_handle,
+					  BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE);
+	if (rc)
+		return rc;
 
-		cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
-				       entries * sizeof(struct cq_base),
-				       IB_ACCESS_LOCAL_WRITE);
-		if (IS_ERR(cq->umem)) {
-			rc = PTR_ERR(cq->umem);
-			goto fail;
-		}
-		cq->qplib_cq.sg_info.umem = cq->umem;
-		cq->qplib_cq.dpi = &uctx->dpi;
-	} else {
-		cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
-		cq->cql = kzalloc_objs(struct bnxt_qplib_cqe, cq->max_cql);
-		if (!cq->cql) {
-			rc = -ENOMEM;
+	if (req.comp_mask & BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE)
+		entries = cqe;
+
+	if (!ibcq->umem) {
+		ibcq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
+					 entries * sizeof(struct cq_base),
+					 IB_ACCESS_LOCAL_WRITE);
+		if (IS_ERR(ibcq->umem)) {
+			rc = PTR_ERR(ibcq->umem);
 			goto fail;
 		}
-
-		cq->qplib_cq.dpi = &rdev->dpi_privileged;
 	}
+
+	rc = bnxt_re_setup_sginfo(rdev, ibcq->umem, &cq->qplib_cq.sg_info);
+	if (rc)
+		goto fail;
+
+	cq->qplib_cq.dpi = &uctx->dpi;
 	cq->qplib_cq.max_wqe = entries;
 	cq->qplib_cq.coalescing = &rdev->cq_coalescing;
 	cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
@@ -3214,43 +3437,100 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 	cq->ib_cq.cqe = entries;
 	cq->cq_period = cq->qplib_cq.period;
-
 	active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
 	if (active_cqs > rdev->stats.res.cq_watermark)
 		rdev->stats.res.cq_watermark = active_cqs;
 	spin_lock_init(&cq->cq_lock);
 
-	if (udata) {
-		struct bnxt_re_cq_resp resp = {};
-
-		if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
-			hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
-			/* Allocate a page */
-			cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
-			if (!cq->uctx_cq_page) {
-				rc = -ENOMEM;
-				goto c2fail;
-			}
-			resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
-		}
-		resp.cqid = cq->qplib_cq.id;
-		resp.tail = cq->qplib_cq.hwq.cons;
-		resp.phase = cq->qplib_cq.period;
-		resp.rsvd = 0;
-		rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
-		if (rc) {
-			ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
-			bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
-			goto free_mem;
+	if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
+		hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
+		/* Allocate a page */
+		cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
+		if (!cq->uctx_cq_page) {
+			rc = -ENOMEM;
+			goto fail;
 		}
+		resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
+	}
+	resp.cqid = cq->qplib_cq.id;
+	resp.tail = cq->qplib_cq.hwq.cons;
+	resp.phase = cq->qplib_cq.period;
+	resp.rsvd = 0;
+	rc = ib_respond_udata(udata, resp);
+	if (rc) {
+		bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
+		goto free_mem;
 	}
 
 	return 0;
 
 free_mem:
 	free_page((unsigned long)cq->uctx_cq_page);
-c2fail:
-	ib_umem_release(cq->umem);
+fail:
+	return rc;
+}
+
+int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+		      struct uverbs_attr_bundle *attrs)
+{
+	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
+	struct ib_udata *udata = &attrs->driver_udata;
+	struct bnxt_re_ucontext *uctx =
+		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
+	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+	int cqe = attr->cqe;
+	int rc, entries;
+	u32 active_cqs;
+
+	if (udata)
+		return bnxt_re_create_user_cq(ibcq, attr, attrs);
+
+	if (attr->flags)
+		return -EOPNOTSUPP;
+
+	/* Validate CQ fields */
+	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+		ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
+		return -EINVAL;
+	}
+
+	cq->rdev = rdev;
+	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
+
+	entries = bnxt_re_init_depth(cqe + 1, uctx);
+	if (entries > dev_attr->max_cq_wqes + 1)
+		entries = dev_attr->max_cq_wqes + 1;
+
+	cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
+	cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
+			  GFP_KERNEL);
+	if (!cq->cql)
+		return -ENOMEM;
+
+	cq->qplib_cq.sg_info.pgsize = SZ_4K;
+	cq->qplib_cq.sg_info.pgshft = __builtin_ctz(SZ_4K);
+	cq->qplib_cq.dpi = &rdev->dpi_privileged;
+	cq->qplib_cq.max_wqe = entries;
+	cq->qplib_cq.coalescing = &rdev->cq_coalescing;
+	cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
+	cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
+
+	rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
+	if (rc) {
+		ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
+		goto fail;
+	}
+
+	cq->ib_cq.cqe = entries;
+	cq->cq_period = cq->qplib_cq.period;
+	active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
+	if (active_cqs > rdev->stats.res.cq_watermark)
+		rdev->stats.res.cq_watermark = active_cqs;
+	spin_lock_init(&cq->cq_lock);
+
+	return 0;
+
 fail:
 	kfree(cq->cql);
 	return rc;
@@ -3264,8 +3544,8 @@ static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
 
 	cq->qplib_cq.max_wqe = cq->resize_cqe;
 	if (cq->resize_umem) {
-		ib_umem_release(cq->umem);
-		cq->umem = cq->resize_umem;
+		ib_umem_release(cq->ib_cq.umem);
+		cq->ib_cq.umem = cq->resize_umem;
 		cq->resize_umem = NULL;
 		cq->resize_cqe = 0;
 	}
@@ -3309,10 +3589,9 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 		entries = dev_attr->max_cq_wqes + 1;
 
 	/* uverbs consumer */
-	if (ib_copy_from_udata(&req, udata, sizeof(req))) {
-		rc = -EFAULT;
+	rc = ib_copy_validate_udata_in(udata, req, cq_va);
+	if (rc)
 		goto fail;
-	}
 
 	cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
 				      entries * sizeof(struct cq_base),
@@ -3343,7 +3622,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 	cq->ib_cq.cqe = cq->resize_cqe;
 	atomic_inc(&rdev->stats.res.resize_count);
 
-	return 0;
+	return ib_respond_empty_udata(udata);
 
 fail:
 	if (cq->resize_umem) {
@@ -3865,7 +4144,7 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
 	/* User CQ; the only processing we do is to
 	 * complete any pending CQ resize operation.
 	 */
-	if (cq->umem) {
+	if (cq->ib_cq.umem) {
 		if (cq->resize_umem)
 			bnxt_re_resize_cq_complete(cq);
 		return 0;
@@ -4075,6 +4354,10 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 	struct bnxt_re_dev *rdev = mr->rdev;
 	int rc;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return rc;
+
 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
@@ -4092,7 +4375,9 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 
 	kfree(mr);
 	atomic_dec(&rdev->stats.res.mr_count);
-	return rc;
+	if (rc)
+		return rc;
+	return ib_respond_empty_udata(udata);
 }
 
 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
@@ -4183,6 +4468,10 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
 	u32 active_mws;
 	int rc;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return ERR_PTR(rc);
+
 	mw = kzalloc_obj(*mw);
 	if (!mw)
 		return ERR_PTR(-ENOMEM);
@@ -4310,6 +4599,11 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 	struct bnxt_re_dev *rdev = pd->rdev;
 	struct ib_umem *umem;
 	struct ib_mr *ib_mr;
+	int ret;
+
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ERR_PTR(ret);
 
 	if (dmah)
 		return ERR_PTR(-EOPNOTSUPP);
@@ -4414,8 +4708,11 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 	if (_is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2))
 		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_QP_RATE_LIMIT_ENABLED;
 
-	if (udata->inlen >= sizeof(ureq)) {
-		rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
+	if (udata->inlen) {
+		rc = ib_copy_validate_udata_in_cm(
+			udata, ureq, comp_mask,
+			BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT |
+				BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT);
 		if (rc)
 			goto cfail;
 		if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
@@ -4430,12 +4727,9 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 		}
 	}
 
-	rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
-	if (rc) {
-		ibdev_err(ibdev, "Failed to copy user context");
-		rc = -EFAULT;
+	rc = ib_respond_udata(udata, resp);
+	if (rc)
 		goto cfail;
-	}
 
 	return 0;
 cfail:
@@ -4494,6 +4788,10 @@ struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,
 	struct bnxt_re_flow *flow;
 	int rc;
 
+	rc = ib_is_udata_in_empty(udata);
+	if (rc)
+		return ERR_PTR(rc);
+
 	if (attr->type != IB_FLOW_ATTR_SNIFFER ||
 	    !rdev->rcfw.roce_mirror)
 		return ERR_PTR(-EOPNOTSUPP);
@@ -4554,32 +4852,6 @@ int bnxt_re_destroy_flow(struct ib_flow *flow_id)
 	return rc;
 }
 
-static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)
-{
-	struct bnxt_re_cq *cq = NULL, *tmp_cq;
-
-	hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) {
-		if (tmp_cq->qplib_cq.id == cq_id) {
-			cq = tmp_cq;
-			break;
-		}
-	}
-	return cq;
-}
-
-static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
-{
-	struct bnxt_re_srq *srq = NULL, *tmp_srq;
-
-	hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
-		if (tmp_srq->qplib_srq.id == srq_id) {
-			srq = tmp_srq;
-			break;
-		}
-	}
-	return srq;
-}
-
 /* Helper function to mmap the virtual memory from user app */
 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
 {
@@ -4682,280 +4954,3 @@ int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags,
 	ret |= IB_MAD_RESULT_REPLY;
 	return ret;
 }
-
-static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
-{
-	struct bnxt_re_ucontext *uctx;
-
-	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
-	bnxt_re_pacing_alert(uctx->rdev);
-	return 0;
-}
-
-static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
-{
-	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
-	enum bnxt_re_alloc_page_type alloc_type;
-	struct bnxt_re_user_mmap_entry *entry;
-	enum bnxt_re_mmap_flag mmap_flag;
-	struct bnxt_qplib_chip_ctx *cctx;
-	struct bnxt_re_ucontext *uctx;
-	struct bnxt_re_dev *rdev;
-	u64 mmap_offset;
-	u32 length;
-	u32 dpi;
-	u64 addr;
-	int err;
-
-	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
-	if (IS_ERR(uctx))
-		return PTR_ERR(uctx);
-
-	err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
-	if (err)
-		return err;
-
-	rdev = uctx->rdev;
-	cctx = rdev->chip_ctx;
-
-	switch (alloc_type) {
-	case BNXT_RE_ALLOC_WC_PAGE:
-		if (cctx->modes.db_push)  {
-			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
-						 uctx, BNXT_QPLIB_DPI_TYPE_WC))
-				return -ENOMEM;
-			length = PAGE_SIZE;
-			dpi = uctx->wcdpi.dpi;
-			addr = (u64)uctx->wcdpi.umdbr;
-			mmap_flag = BNXT_RE_MMAP_WC_DB;
-		} else {
-			return -EINVAL;
-		}
-
-		break;
-	case BNXT_RE_ALLOC_DBR_BAR_PAGE:
-		length = PAGE_SIZE;
-		addr = (u64)rdev->pacing.dbr_bar_addr;
-		mmap_flag = BNXT_RE_MMAP_DBR_BAR;
-		break;
-
-	case BNXT_RE_ALLOC_DBR_PAGE:
-		length = PAGE_SIZE;
-		addr = (u64)rdev->pacing.dbr_page;
-		mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
-	if (!entry)
-		return -ENOMEM;
-
-	uobj->object = entry;
-	uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
-	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
-			     &mmap_offset, sizeof(mmap_offset));
-	if (err)
-		return err;
-
-	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
-			     &length, sizeof(length));
-	if (err)
-		return err;
-
-	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,
-			     &dpi, sizeof(dpi));
-	if (err)
-		return err;
-
-	return 0;
-}
-
-static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
-				  enum rdma_remove_reason why,
-			    struct uverbs_attr_bundle *attrs)
-{
-	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
-	struct bnxt_re_ucontext *uctx = entry->uctx;
-
-	switch (entry->mmap_flag) {
-	case BNXT_RE_MMAP_WC_DB:
-		if (uctx && uctx->wcdpi.dbr) {
-			struct bnxt_re_dev *rdev = uctx->rdev;
-
-			bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
-			uctx->wcdpi.dbr = NULL;
-		}
-		break;
-	case BNXT_RE_MMAP_DBR_BAR:
-	case BNXT_RE_MMAP_DBR_PAGE:
-		break;
-	default:
-		goto exit;
-	}
-	rdma_user_mmap_entry_remove(&entry->rdma_entry);
-exit:
-	return 0;
-}
-
-DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
-			    UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
-					    BNXT_RE_OBJECT_ALLOC_PAGE,
-					    UVERBS_ACCESS_NEW,
-					    UA_MANDATORY),
-			    UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
-						 enum bnxt_re_alloc_page_type,
-						 UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
-						UVERBS_ATTR_TYPE(u64),
-						UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
-						UVERBS_ATTR_TYPE(u32),
-						UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
-						UVERBS_ATTR_TYPE(u32),
-						UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
-				    UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
-						    BNXT_RE_OBJECT_ALLOC_PAGE,
-						    UVERBS_ACCESS_DESTROY,
-						    UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
-			    UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
-			    &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
-			    &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
-
-DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
-
-DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
-			      &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
-
-/* Toggle MEM */
-static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)
-{
-	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
-	enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;
-	enum bnxt_re_get_toggle_mem_type res_type;
-	struct bnxt_re_user_mmap_entry *entry;
-	struct bnxt_re_ucontext *uctx;
-	struct ib_ucontext *ib_uctx;
-	struct bnxt_re_dev *rdev;
-	struct bnxt_re_srq *srq;
-	u32 length = PAGE_SIZE;
-	struct bnxt_re_cq *cq;
-	u64 mem_offset;
-	u32 offset = 0;
-	u64 addr = 0;
-	u32 res_id;
-	int err;
-
-	ib_uctx = ib_uverbs_get_ucontext(attrs);
-	if (IS_ERR(ib_uctx))
-		return PTR_ERR(ib_uctx);
-
-	err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);
-	if (err)
-		return err;
-
-	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
-	rdev = uctx->rdev;
-	err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);
-	if (err)
-		return err;
-
-	switch (res_type) {
-	case BNXT_RE_CQ_TOGGLE_MEM:
-		cq = bnxt_re_search_for_cq(rdev, res_id);
-		if (!cq)
-			return -EINVAL;
-
-		addr = (u64)cq->uctx_cq_page;
-		break;
-	case BNXT_RE_SRQ_TOGGLE_MEM:
-		srq = bnxt_re_search_for_srq(rdev, res_id);
-		if (!srq)
-			return -EINVAL;
-
-		addr = (u64)srq->uctx_srq_page;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset);
-	if (!entry)
-		return -ENOMEM;
-
-	uobj->object = entry;
-	uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
-	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
-			     &mem_offset, sizeof(mem_offset));
-	if (err)
-		return err;
-
-	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
-			     &length, sizeof(length));
-	if (err)
-		return err;
-
-	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
-			     &offset, sizeof(offset));
-	if (err)
-		return err;
-
-	return 0;
-}
-
-static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,
-				      enum rdma_remove_reason why,
-				      struct uverbs_attr_bundle *attrs)
-{
-	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
-
-	rdma_user_mmap_entry_remove(&entry->rdma_entry);
-	return 0;
-}
-
-DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,
-			    UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,
-					    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
-					    UVERBS_ACCESS_NEW,
-					    UA_MANDATORY),
-			    UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,
-						 enum bnxt_re_get_toggle_mem_type,
-						 UA_MANDATORY),
-			    UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,
-					       UVERBS_ATTR_TYPE(u32),
-					       UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
-						UVERBS_ATTR_TYPE(u64),
-						UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
-						UVERBS_ATTR_TYPE(u32),
-						UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
-						UVERBS_ATTR_TYPE(u32),
-						UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
-				    UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,
-						    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
-						    UVERBS_ACCESS_DESTROY,
-						    UA_MANDATORY));
-
-DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,
-			    UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),
-			    &UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),
-			    &UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));
-
-const struct uapi_definition bnxt_re_uapi_defs[] = {
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
-	{}
-};
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 76ba9ab04d5ce..3d02c16f54b61 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -108,7 +108,6 @@ struct bnxt_re_cq {
 	struct bnxt_qplib_cqe	*cql;
 #define MAX_CQL_PER_POLL	1024
 	u32			max_cql;
-	struct ib_umem		*umem;
 	struct ib_umem		*resize_umem;
 	int			resize_cqe;
 	void			*uctx_cq_page;
@@ -164,6 +163,13 @@ struct bnxt_re_user_mmap_entry {
 	u8 mmap_flag;
 };
 
+struct bnxt_re_dbr_obj {
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_dpi dpi;
+	struct bnxt_re_user_mmap_entry *entry;
+	atomic_t usecnt; /* QPs using this dbr */
+};
+
 struct bnxt_re_flow {
 	struct ib_flow		ib_flow;
 	struct bnxt_re_dev	*rdev;
@@ -247,6 +253,8 @@ int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
 		      const struct ib_recv_wr **bad_recv_wr);
 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs);
+int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs);
 int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
@@ -293,4 +301,7 @@ static inline u32 __to_ib_port_num(u16 port_id)
 
 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp);
 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);
+struct bnxt_re_user_mmap_entry*
+bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
+			  enum bnxt_re_mmap_flag mmap_flag, u64 *offset);
 #endif /* __BNXT_RE_IB_VERBS_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index b576f05e3b26b..13ad63b9b1de7 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -1326,6 +1326,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
 	.owner = THIS_MODULE,
 	.driver_id = RDMA_DRIVER_BNXT_RE,
 	.uverbs_abi_ver = BNXT_RE_ABI_VERSION,
+	.uverbs_robust_udata = true,
 
 	.add_gid = bnxt_re_add_gid,
 	.alloc_hw_port_stats = bnxt_re_ib_alloc_hw_port_stats,
@@ -1334,6 +1335,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
 	.alloc_ucontext = bnxt_re_alloc_ucontext,
 	.create_ah = bnxt_re_create_ah,
 	.create_cq = bnxt_re_create_cq,
+	.create_user_cq = bnxt_re_create_user_cq,
 	.create_qp = bnxt_re_create_qp,
 	.create_srq = bnxt_re_create_srq,
 	.create_user_ah = bnxt_re_create_ah,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 2d7932b3c492f..c15b6a361ac60 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -792,8 +792,6 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 	return 0;
 }
 
-/* QP */
-
 static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
 {
 	int indx;
@@ -812,9 +810,71 @@ static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
 	return 0;
 }
 
+static int bnxt_re_setup_qp_swqs(struct bnxt_qplib_qp *qplqp)
+{
+	struct bnxt_qplib_q *sq = &qplqp->sq;
+	struct bnxt_qplib_q *rq = &qplqp->rq;
+	int rc;
+
+	if (qplqp->is_user)
+		return 0;
+
+	rc = bnxt_qplib_alloc_init_swq(sq);
+	if (rc)
+		return rc;
+
+	if (!qplqp->srq) {
+		rc = bnxt_qplib_alloc_init_swq(rq);
+		if (rc)
+			goto free_sq_swq;
+	}
+
+	return 0;
+free_sq_swq:
+	kfree(sq->swq);
+	return rc;
+}
+
+static void bnxt_qp_init_dbinfo(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_q *rq = &qp->rq;
+
+	sq->dbinfo.hwq = &sq->hwq;
+	sq->dbinfo.xid = qp->id;
+	sq->dbinfo.db = qp->dpi->dbr;
+	sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
+	sq->dbinfo.flags = 0;
+	if (rq->max_wqe) {
+		rq->dbinfo.hwq = &rq->hwq;
+		rq->dbinfo.xid = qp->id;
+		rq->dbinfo.db = qp->dpi->dbr;
+		rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
+		rq->dbinfo.flags = 0;
+	}
+}
+
+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
+{
+	struct bnxt_qplib_hwq *sq_hwq;
+	struct bnxt_qplib_q *sq;
+	u64 fpsne, psn_pg;
+	u16 indx_pad = 0;
+
+	sq = &qp->sq;
+	sq_hwq = &sq->hwq;
+	/* First psn entry */
+	fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg);
+	if (!IS_ALIGNED(fpsne, PAGE_SIZE))
+		indx_pad = (fpsne & ~PAGE_MASK) / size;
+	sq_hwq->pad_pgofft = indx_pad;
+	sq_hwq->pad_pg = (u64 *)psn_pg;
+	sq_hwq->pad_stride = size;
+}
+
+/* QP */
 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
-	struct bnxt_qplib_hwq_attr hwq_attr = {};
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct creq_create_qp1_resp resp = {};
 	struct bnxt_qplib_cmdqmsg msg = {};
@@ -823,7 +883,6 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	struct cmdq_create_qp1 req = {};
 	struct bnxt_qplib_pbl *pbl;
 	u32 qp_flags = 0;
-	u8 pg_sz_lvl;
 	u32 tbl_indx;
 	int rc;
 
@@ -837,26 +896,12 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	req.qp_handle = cpu_to_le64(qp->qp_handle);
 
 	/* SQ */
-	hwq_attr.res = res;
-	hwq_attr.sginfo = &sq->sg_info;
-	hwq_attr.stride = sizeof(struct sq_sge);
-	hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, false);
-	hwq_attr.type = HWQ_TYPE_QUEUE;
-	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
-	if (rc)
-		return rc;
+	sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qp->wqe_mode, true);
+	req.sq_size = cpu_to_le32(sq->max_sw_wqe);
+	req.sq_pg_size_sq_lvl = sq->hwq.pg_sz_lvl;
 
-	rc = bnxt_qplib_alloc_init_swq(sq);
-	if (rc)
-		goto fail_sq;
-
-	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
 	pbl = &sq->hwq.pbl[PBL_LVL_0];
 	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
-		     CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
-	pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
-	req.sq_pg_size_sq_lvl = pg_sz_lvl;
 	req.sq_fwo_sq_sge =
 		cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
 			     CMDQ_CREATE_QP1_SQ_SGE_SFT);
@@ -865,24 +910,10 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	/* RQ */
 	if (rq->max_wqe) {
 		rq->dbinfo.flags = 0;
-		hwq_attr.res = res;
-		hwq_attr.sginfo = &rq->sg_info;
-		hwq_attr.stride = sizeof(struct sq_sge);
-		hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false);
-		hwq_attr.type = HWQ_TYPE_QUEUE;
-		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
-		if (rc)
-			goto sq_swq;
-		rc = bnxt_qplib_alloc_init_swq(rq);
-		if (rc)
-			goto fail_rq;
 		req.rq_size = cpu_to_le32(rq->max_wqe);
 		pbl = &rq->hwq.pbl[PBL_LVL_0];
 		req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-		pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
-			     CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
-		pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
-		req.rq_pg_size_rq_lvl = pg_sz_lvl;
+		req.rq_pg_size_rq_lvl = rq->hwq.pg_sz_lvl;
 		req.rq_fwo_rq_sge =
 			cpu_to_le16((rq->max_sge &
 				     CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
@@ -893,7 +924,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 	if (rc) {
 		rc = -ENOMEM;
-		goto rq_rwq;
+		return rc;
 	}
 	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
 	req.qp_flags = cpu_to_le32(qp_flags);
@@ -906,73 +937,39 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	qp->id = le32_to_cpu(resp.xid);
 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
-	qp->cctx = res->cctx;
-	sq->dbinfo.hwq = &sq->hwq;
-	sq->dbinfo.xid = qp->id;
-	sq->dbinfo.db = qp->dpi->dbr;
-	sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
-	if (rq->max_wqe) {
-		rq->dbinfo.hwq = &rq->hwq;
-		rq->dbinfo.xid = qp->id;
-		rq->dbinfo.db = qp->dpi->dbr;
-		rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
-	}
+
+	rc = bnxt_re_setup_qp_swqs(qp);
+	if (rc)
+		goto destroy_qp;
+	bnxt_qp_init_dbinfo(res, qp);
+
 	tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
 	rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
 	rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp;
 
 	return 0;
 
+destroy_qp:
+	bnxt_qplib_destroy_qp(res, qp);
 fail:
 	bnxt_qplib_free_qp_hdr_buf(res, qp);
-rq_rwq:
-	kfree(rq->swq);
-fail_rq:
-	bnxt_qplib_free_hwq(res, &rq->hwq);
-sq_swq:
-	kfree(sq->swq);
-fail_sq:
-	bnxt_qplib_free_hwq(res, &sq->hwq);
 	return rc;
 }
 
-static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
-{
-	struct bnxt_qplib_hwq *hwq;
-	struct bnxt_qplib_q *sq;
-	u64 fpsne, psn_pg;
-	u16 indx_pad = 0;
-
-	sq = &qp->sq;
-	hwq = &sq->hwq;
-	/* First psn entry */
-	fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
-	if (!IS_ALIGNED(fpsne, PAGE_SIZE))
-		indx_pad = (fpsne & ~PAGE_MASK) / size;
-	hwq->pad_pgofft = indx_pad;
-	hwq->pad_pg = (u64 *)psn_pg;
-	hwq->pad_stride = size;
-}
-
 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
-	struct bnxt_qplib_hwq_attr hwq_attr = {};
-	struct bnxt_qplib_sg_info sginfo = {};
 	struct creq_create_qp_resp resp = {};
 	struct bnxt_qplib_cmdqmsg msg = {};
 	struct bnxt_qplib_q *sq = &qp->sq;
 	struct bnxt_qplib_q *rq = &qp->rq;
 	struct cmdq_create_qp req = {};
-	int rc, req_size, psn_sz = 0;
-	struct bnxt_qplib_hwq *xrrq;
 	struct bnxt_qplib_pbl *pbl;
 	u32 qp_flags = 0;
-	u8 pg_sz_lvl;
 	u32 tbl_indx;
 	u16 nsge;
+	int rc;
 
-	qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_flags2);
 	sq->dbinfo.flags = 0;
 	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
 				 CMDQ_BASE_OPCODE_CREATE_QP,
@@ -984,56 +981,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	req.qp_handle = cpu_to_le64(qp->qp_handle);
 
 	/* SQ */
-	if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
-		psn_sz = bnxt_qplib_is_chip_gen_p5_p7(res->cctx) ?
-			 sizeof(struct sq_psn_search_ext) :
-			 sizeof(struct sq_psn_search);
-
-		if (qp->is_host_msn_tbl) {
-			psn_sz = sizeof(struct sq_msn_search);
-			qp->msn = 0;
-		}
-	}
-
-	hwq_attr.res = res;
-	hwq_attr.sginfo = &sq->sg_info;
-	hwq_attr.stride = sizeof(struct sq_sge);
-	hwq_attr.depth = bnxt_qplib_get_depth(sq, qp->wqe_mode, true);
-	hwq_attr.aux_stride = psn_sz;
-	hwq_attr.aux_depth = psn_sz ? bnxt_qplib_set_sq_size(sq, qp->wqe_mode)
-				    : 0;
-	/* Update msn tbl size */
-	if (qp->is_host_msn_tbl && psn_sz) {
-		if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
-			hwq_attr.aux_depth =
-				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
-		else
-			hwq_attr.aux_depth =
-				roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)) / 2;
-		qp->msn_tbl_sz = hwq_attr.aux_depth;
-		qp->msn = 0;
-	}
-
-	hwq_attr.type = HWQ_TYPE_QUEUE;
-	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
-	if (rc)
-		return rc;
-
-	if (!sq->hwq.is_user) {
-		rc = bnxt_qplib_alloc_init_swq(sq);
-		if (rc)
-			goto fail_sq;
-
-		if (psn_sz)
-			bnxt_qplib_init_psn_ptr(qp, psn_sz);
-	}
-	req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
+	req.sq_size = cpu_to_le32(sq->max_sw_wqe);
 	pbl = &sq->hwq.pbl[PBL_LVL_0];
 	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
-		     CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
-	pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
-	req.sq_pg_size_sq_lvl = pg_sz_lvl;
+	req.sq_pg_size_sq_lvl = sq->hwq.pg_sz_lvl;
 	req.sq_fwo_sq_sge =
 		cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
 			     CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
@@ -1042,29 +993,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	/* RQ */
 	if (!qp->srq) {
 		rq->dbinfo.flags = 0;
-		hwq_attr.res = res;
-		hwq_attr.sginfo = &rq->sg_info;
-		hwq_attr.stride = sizeof(struct sq_sge);
-		hwq_attr.depth = bnxt_qplib_get_depth(rq, qp->wqe_mode, false);
-		hwq_attr.aux_stride = 0;
-		hwq_attr.aux_depth = 0;
-		hwq_attr.type = HWQ_TYPE_QUEUE;
-		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
-		if (rc)
-			goto sq_swq;
-		if (!rq->hwq.is_user) {
-			rc = bnxt_qplib_alloc_init_swq(rq);
-			if (rc)
-				goto fail_rq;
-		}
-
 		req.rq_size = cpu_to_le32(rq->max_wqe);
 		pbl = &rq->hwq.pbl[PBL_LVL_0];
 		req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-		pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
-			     CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
-		pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
-		req.rq_pg_size_rq_lvl = pg_sz_lvl;
+		req.rq_pg_size_rq_lvl = rq->hwq.pg_sz_lvl;
 		nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
 			6 : rq->max_sge;
 		req.rq_fwo_rq_sge =
@@ -1090,44 +1022,9 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	req.qp_flags = cpu_to_le32(qp_flags);
 
 	/* ORRQ and IRRQ */
-	if (psn_sz) {
-		xrrq = &qp->orrq;
-		xrrq->max_elements =
-			ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
-		req_size = xrrq->max_elements *
-			   BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
-		req_size &= ~(PAGE_SIZE - 1);
-		sginfo.pgsize = req_size;
-		sginfo.pgshft = PAGE_SHIFT;
-
-		hwq_attr.res = res;
-		hwq_attr.sginfo = &sginfo;
-		hwq_attr.depth = xrrq->max_elements;
-		hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
-		hwq_attr.aux_stride = 0;
-		hwq_attr.aux_depth = 0;
-		hwq_attr.type = HWQ_TYPE_CTX;
-		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
-		if (rc)
-			goto rq_swq;
-		pbl = &xrrq->pbl[PBL_LVL_0];
-		req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
-
-		xrrq = &qp->irrq;
-		xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
-						qp->max_dest_rd_atomic);
-		req_size = xrrq->max_elements *
-			   BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
-		req_size &= ~(PAGE_SIZE - 1);
-		sginfo.pgsize = req_size;
-		hwq_attr.depth =  xrrq->max_elements;
-		hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
-		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
-		if (rc)
-			goto fail_orrq;
-
-		pbl = &xrrq->pbl[PBL_LVL_0];
-		req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
+	if (qp->psn_sz) {
+		req.orrq_addr = cpu_to_le64(bnxt_qplib_get_base_addr(&qp->orrq));
+		req.irrq_addr = cpu_to_le64(bnxt_qplib_get_base_addr(&qp->irrq));
 	}
 	req.pd_id = cpu_to_le32(qp->pd->id);
 
@@ -1135,23 +1032,23 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 				sizeof(resp), 0);
 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
 	if (rc)
-		goto fail;
+		return rc;
 
 	qp->id = le32_to_cpu(resp.xid);
+
+	if (!qp->is_user) {
+		rc = bnxt_re_setup_qp_swqs(qp);
+		if (rc)
+			goto destroy_qp;
+	}
+	bnxt_qp_init_dbinfo(res, qp);
+	if (qp->psn_sz)
+		bnxt_qplib_init_psn_ptr(qp, qp->psn_sz);
+
 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 	INIT_LIST_HEAD(&qp->sq_flush);
 	INIT_LIST_HEAD(&qp->rq_flush);
 	qp->cctx = res->cctx;
-	sq->dbinfo.hwq = &sq->hwq;
-	sq->dbinfo.xid = qp->id;
-	sq->dbinfo.db = qp->dpi->dbr;
-	sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
-	if (rq->max_wqe) {
-		rq->dbinfo.hwq = &rq->hwq;
-		rq->dbinfo.xid = qp->id;
-		rq->dbinfo.db = qp->dpi->dbr;
-		rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
-	}
 	spin_lock_bh(&rcfw->tbl_lock);
 	tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
 	rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
@@ -1159,18 +1056,8 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	spin_unlock_bh(&rcfw->tbl_lock);
 
 	return 0;
-fail:
-	bnxt_qplib_free_hwq(res, &qp->irrq);
-fail_orrq:
-	bnxt_qplib_free_hwq(res, &qp->orrq);
-rq_swq:
-	kfree(rq->swq);
-fail_rq:
-	bnxt_qplib_free_hwq(res, &rq->hwq);
-sq_swq:
-	kfree(sq->swq);
-fail_sq:
-	bnxt_qplib_free_hwq(res, &sq->hwq);
+destroy_qp:
+	bnxt_qplib_destroy_qp(res, qp);
 	return rc;
 }
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 30c3f99be07b4..5f671cc591001 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -279,6 +279,7 @@ struct bnxt_qplib_qp {
 	u8				wqe_mode;
 	u8				state;
 	u8				cur_qp_state;
+	u8				is_user;
 	u64				modify_flags;
 	u32				ext_modify_flags;
 	u32				max_inline_data;
@@ -344,9 +345,11 @@ struct bnxt_qplib_qp {
 	struct list_head		rq_flush;
 	u32				msn;
 	u32				msn_tbl_sz;
+	u32				psn_sz;
 	bool				is_host_msn_tbl;
 	u8				tos_dscp;
 	u32				ugid_index;
+	u16				dev_cap_flags;
 	u32				rate_limit;
 	u8				shaper_allocation_status;
 };
@@ -617,6 +620,11 @@ static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
 	que->swq_start = que->swq[idx].next_idx;
 }
 
+static inline u32 bnxt_qplib_get_stride(void)
+{
+	return sizeof(struct sq_sge);
+}
+
 static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que, u8 wqe_mode, bool is_sq)
 {
 	u32 slots;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 341bae3d8a1d9..95e0489c53c37 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -46,7 +46,7 @@
 #include <linux/if_vlan.h>
 #include <linux/vmalloc.h>
 #include <rdma/ib_verbs.h>
-#include <rdma/ib_umem.h>
+#include <rdma/iter.h>
 
 #include "roce_hsi.h"
 #include "qplib_res.h"
@@ -683,6 +683,49 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 }
 
 /* DPIs */
+int bnxt_qplib_alloc_uc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi)
+{
+	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+	struct bnxt_qplib_reg_desc *reg;
+	u32 bit_num;
+	int rc = 0;
+
+	reg = &dpit->wcreg;
+	mutex_lock(&res->dpi_tbl_lock);
+	bit_num = find_first_bit(dpit->tbl, dpit->max);
+	if (bit_num >= dpit->max) {
+		rc = -ENOMEM;
+		goto unlock;
+	}
+	/* Found unused DPI */
+	clear_bit(bit_num, dpit->tbl);
+	dpi->bit = bit_num;
+	dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
+	dpi->umdbr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
+unlock:
+	mutex_unlock(&res->dpi_tbl_lock);
+	return rc;
+}
+
+int bnxt_qplib_free_uc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi)
+{
+	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+	int rc = 0;
+
+	mutex_lock(&res->dpi_tbl_lock);
+	if (dpi->bit >= dpit->max) {
+		rc = -EINVAL;
+		goto unlock;
+	}
+
+	if (test_and_set_bit(dpi->bit, dpit->tbl))
+		rc = -EINVAL;
+	memset(dpi, 0, sizeof(*dpi));
+unlock:
+	mutex_unlock(&res->dpi_tbl_lock);
+	return rc;
+}
+
 int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
 			 struct bnxt_qplib_dpi *dpi,
 			 void *app, u8 type)
@@ -790,7 +833,7 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
 	if (dev_attr->max_dpi)
 		dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
 
-	dpit->app_tbl = kcalloc(dpit->max,  sizeof(void *), GFP_KERNEL);
+	dpit->app_tbl = kzalloc_objs(void *, dpit->max);
 	if (!dpit->app_tbl)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index 9a5dcf97b6f44..ffe31c952d50f 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -198,6 +198,7 @@ struct bnxt_qplib_hwq {
 	u32				cons;		/* raw */
 	u8				cp_bit;
 	u8				is_user;
+	u8				pg_sz_lvl;
 	u64				*pad_pg;
 	u32				pad_stride;
 	u32				pad_pgofft;
@@ -358,6 +359,11 @@ static inline u8 bnxt_qplib_get_ring_type(struct bnxt_qplib_chip_ctx *cctx)
 	       RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL;
 }
 
+static inline u64 bnxt_qplib_get_base_addr(struct bnxt_qplib_hwq *hwq)
+{
+	return hwq->pbl[PBL_LVL_0].pg_map_arr[0];
+}
+
 static inline u8 bnxt_qplib_base_pg_size(struct bnxt_qplib_hwq *hwq)
 {
 	u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
@@ -430,6 +436,10 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
 			 void *app, u8 type);
 int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
 			   struct bnxt_qplib_dpi *dpi);
+int bnxt_qplib_alloc_uc_dpi(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_dpi *dpi);
+int bnxt_qplib_free_uc_dpi(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_dpi *dpi);
 void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res);
 int bnxt_qplib_init_res(struct bnxt_qplib_res *res);
 void bnxt_qplib_free_res(struct bnxt_qplib_res *res);
diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c
new file mode 100644
index 0000000000000..3eaee7101615b
--- /dev/null
+++ b/drivers/infiniband/hw/bnxt_re/uapi.c
@@ -0,0 +1,469 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright (c) 2025, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Description: uapi interpreter
+ */
+
+#include <rdma/ib_addr.h>
+#include <rdma/uverbs_types.h>
+#include <rdma/uverbs_std_types.h>
+#include <rdma/ib_user_ioctl_cmds.h>
+#define UVERBS_MODULE_NAME bnxt_re
+#include <rdma/uverbs_named_ioctl.h>
+#include <rdma/bnxt_re-abi.h>
+
+#include "roce_hsi.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "qplib_rcfw.h"
+#include "bnxt_re.h"
+#include "ib_verbs.h"
+
+static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)
+{
+	struct bnxt_re_cq *cq = NULL, *tmp_cq;
+
+	hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) {
+		if (tmp_cq->qplib_cq.id == cq_id) {
+			cq = tmp_cq;
+			break;
+		}
+	}
+	return cq;
+}
+
+static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
+{
+	struct bnxt_re_srq *srq = NULL, *tmp_srq;
+
+	hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
+		if (tmp_srq->qplib_srq.id == srq_id) {
+			srq = tmp_srq;
+			break;
+		}
+	}
+	return srq;
+}
+
+static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
+{
+	struct bnxt_re_ucontext *uctx;
+	struct ib_ucontext *ib_uctx;
+
+	ib_uctx = ib_uverbs_get_ucontext(attrs);
+	if (IS_ERR(ib_uctx))
+		return PTR_ERR(ib_uctx);
+
+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
+	if (IS_ERR(uctx))
+		return PTR_ERR(uctx);
+
+	bnxt_re_pacing_alert(uctx->rdev);
+	return 0;
+}
+
+static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
+	enum bnxt_re_alloc_page_type alloc_type;
+	struct bnxt_re_user_mmap_entry *entry;
+	enum bnxt_re_mmap_flag mmap_flag;
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_re_ucontext *uctx;
+	struct ib_ucontext *ib_uctx;
+	struct bnxt_re_dev *rdev;
+	u64 mmap_offset;
+	u32 length;
+	u32 dpi;
+	u64 addr;
+	int err;
+
+	ib_uctx = ib_uverbs_get_ucontext(attrs);
+	if (IS_ERR(ib_uctx))
+		return PTR_ERR(ib_uctx);
+
+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
+	if (IS_ERR(uctx))
+		return PTR_ERR(uctx);
+
+	err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
+	if (err)
+		return err;
+
+	rdev = uctx->rdev;
+	cctx = rdev->chip_ctx;
+
+	switch (alloc_type) {
+	case BNXT_RE_ALLOC_WC_PAGE:
+		if (cctx->modes.db_push)  {
+			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
+						 uctx, BNXT_QPLIB_DPI_TYPE_WC))
+				return -ENOMEM;
+			length = PAGE_SIZE;
+			dpi = uctx->wcdpi.dpi;
+			addr = (u64)uctx->wcdpi.umdbr;
+			mmap_flag = BNXT_RE_MMAP_WC_DB;
+		} else {
+			return -EINVAL;
+		}
+
+		break;
+	case BNXT_RE_ALLOC_DBR_BAR_PAGE:
+		length = PAGE_SIZE;
+		addr = (u64)rdev->pacing.dbr_bar_addr;
+		mmap_flag = BNXT_RE_MMAP_DBR_BAR;
+		break;
+
+	case BNXT_RE_ALLOC_DBR_PAGE:
+		length = PAGE_SIZE;
+		addr = (u64)rdev->pacing.dbr_page;
+		mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
+		break;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
+	if (!entry)
+		return -ENOMEM;
+
+	uobj->object = entry;
+	uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
+			     &mmap_offset, sizeof(mmap_offset));
+	if (err)
+		return err;
+
+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
+			     &length, sizeof(length));
+	if (err)
+		return err;
+
+	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,
+			     &dpi, sizeof(dpi));
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
+				  enum rdma_remove_reason why,
+			    struct uverbs_attr_bundle *attrs)
+{
+	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
+	struct bnxt_re_ucontext *uctx = entry->uctx;
+
+	switch (entry->mmap_flag) {
+	case BNXT_RE_MMAP_WC_DB:
+		if (uctx && uctx->wcdpi.dbr) {
+			struct bnxt_re_dev *rdev = uctx->rdev;
+
+			bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
+			uctx->wcdpi.dbr = NULL;
+		}
+		break;
+	case BNXT_RE_MMAP_DBR_BAR:
+	case BNXT_RE_MMAP_DBR_PAGE:
+		break;
+	default:
+		goto exit;
+	}
+	rdma_user_mmap_entry_remove(&entry->rdma_entry);
+exit:
+	return 0;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
+			    UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
+					    BNXT_RE_OBJECT_ALLOC_PAGE,
+					    UVERBS_ACCESS_NEW,
+					    UA_MANDATORY),
+			    UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
+						 enum bnxt_re_alloc_page_type,
+						 UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
+						UVERBS_ATTR_TYPE(u64),
+						UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
+						UVERBS_ATTR_TYPE(u32),
+						UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
+						UVERBS_ATTR_TYPE(u32),
+						UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
+				    UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
+						    BNXT_RE_OBJECT_ALLOC_PAGE,
+						    UVERBS_ACCESS_DESTROY,
+						    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
+			    UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
+
+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
+
+DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
+			      &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
+
+/* Toggle MEM */
+static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
+	enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;
+	enum bnxt_re_get_toggle_mem_type res_type;
+	struct bnxt_re_user_mmap_entry *entry;
+	struct bnxt_re_ucontext *uctx;
+	struct ib_ucontext *ib_uctx;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_srq *srq;
+	u32 length = PAGE_SIZE;
+	struct bnxt_re_cq *cq;
+	u64 mem_offset;
+	u32 offset = 0;
+	u64 addr = 0;
+	u32 res_id;
+	int err;
+
+	ib_uctx = ib_uverbs_get_ucontext(attrs);
+	if (IS_ERR(ib_uctx))
+		return PTR_ERR(ib_uctx);
+
+	err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);
+	if (err)
+		return err;
+
+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
+	rdev = uctx->rdev;
+	err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);
+	if (err)
+		return err;
+
+	switch (res_type) {
+	case BNXT_RE_CQ_TOGGLE_MEM:
+		cq = bnxt_re_search_for_cq(rdev, res_id);
+		if (!cq)
+			return -EINVAL;
+
+		addr = (u64)cq->uctx_cq_page;
+		break;
+	case BNXT_RE_SRQ_TOGGLE_MEM:
+		srq = bnxt_re_search_for_srq(rdev, res_id);
+		if (!srq)
+			return -EINVAL;
+
+		addr = (u64)srq->uctx_srq_page;
+		break;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset);
+	if (!entry)
+		return -ENOMEM;
+
+	uobj->object = entry;
+	uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
+			     &mem_offset, sizeof(mem_offset));
+	if (err)
+		return err;
+
+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
+			     &length, sizeof(length));
+	if (err)
+		return err;
+
+	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
+			     &offset, sizeof(offset));
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,
+				      enum rdma_remove_reason why,
+				      struct uverbs_attr_bundle *attrs)
+{
+	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
+
+	rdma_user_mmap_entry_remove(&entry->rdma_entry);
+	return 0;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,
+			    UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,
+					    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
+					    UVERBS_ACCESS_NEW,
+					    UA_MANDATORY),
+			    UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,
+						 enum bnxt_re_get_toggle_mem_type,
+						 UA_MANDATORY),
+			    UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,
+					       UVERBS_ATTR_TYPE(u32),
+					       UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
+						UVERBS_ATTR_TYPE(u64),
+						UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
+						UVERBS_ATTR_TYPE(u32),
+						UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
+						UVERBS_ATTR_TYPE(u32),
+						UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
+				    UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,
+						    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
+						    UVERBS_ACCESS_DESTROY,
+						    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,
+			    UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));
+
+static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *attrs)
+{
+	struct bnxt_re_db_region dbr = {};
+	struct bnxt_re_ucontext *uctx;
+	struct bnxt_re_dbr_obj *obj;
+	struct ib_ucontext *ib_uctx;
+	struct bnxt_qplib_dpi *dpi;
+	struct bnxt_re_dev *rdev;
+	struct ib_uobject *uobj;
+	u64 mmap_offset;
+	int ret;
+
+	ib_uctx = ib_uverbs_get_ucontext(attrs);
+	if (IS_ERR(ib_uctx))
+		return PTR_ERR(ib_uctx);
+
+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
+	rdev = uctx->rdev;
+	uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_DBR_HANDLE);
+
+	obj = kzalloc_obj(*obj);
+	if (!obj)
+		return -ENOMEM;
+
+	dpi = &obj->dpi;
+	ret = bnxt_qplib_alloc_uc_dpi(&rdev->qplib_res, dpi);
+	if (ret)
+		goto free_mem;
+
+	obj->entry = bnxt_re_mmap_entry_insert(uctx, dpi->umdbr,
+					       BNXT_RE_MMAP_UC_DB,
+					       &mmap_offset);
+	if (!obj->entry) {
+		ret = -ENOMEM;
+		goto free_dpi;
+	}
+
+	obj->rdev = rdev;
+	uobj->object = obj;
+	uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE);
+
+	dbr.umdbr = dpi->umdbr;
+	dbr.dpi = dpi->dpi;
+	ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_ALLOC_DBR_ATTR,
+					    &dbr, sizeof(dbr));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, BNXT_RE_ALLOC_DBR_OFFSET,
+			     &mmap_offset, sizeof(mmap_offset));
+	if (ret)
+		return ret;
+	return 0;
+free_dpi:
+	bnxt_qplib_free_uc_dpi(&rdev->qplib_res, dpi);
+free_mem:
+	kfree(obj);
+	return ret;
+}
+
+static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject,
+			       enum rdma_remove_reason why,
+			       struct uverbs_attr_bundle *attrs)
+{
+	struct bnxt_re_dbr_obj *obj = uobject->object;
+	struct bnxt_re_dev *rdev = obj->rdev;
+
+	rdma_user_mmap_entry_remove(&obj->entry->rdma_entry);
+	bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi);
+	return 0;
+}
+
+static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_DEFAULT_DBR)(struct uverbs_attr_bundle *attrs)
+{
+	struct bnxt_re_db_region dpi = {};
+	struct bnxt_re_ucontext *uctx;
+	struct ib_ucontext *ib_uctx;
+	int ret;
+
+	ib_uctx = ib_uverbs_get_ucontext(attrs);
+	if (IS_ERR(ib_uctx))
+		return PTR_ERR(ib_uctx);
+
+	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
+	dpi.umdbr = uctx->dpi.umdbr;
+	dpi.dpi = uctx->dpi.dpi;
+
+	ret = uverbs_copy_to_struct_or_zero(attrs, BNXT_RE_DEFAULT_DBR_ATTR,
+					    &dpi, sizeof(dpi));
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_DBR_ALLOC,
+			    UVERBS_ATTR_IDR(BNXT_RE_ALLOC_DBR_HANDLE,
+					    BNXT_RE_OBJECT_DBR,
+					    UVERBS_ACCESS_NEW,
+					    UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_ATTR,
+						UVERBS_ATTR_STRUCT(struct bnxt_re_db_region,
+								   umdbr),
+								   UA_MANDATORY),
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_DBR_OFFSET,
+						UVERBS_ATTR_TYPE(u64),
+						UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DBR_FREE,
+				    UVERBS_ATTR_IDR(BNXT_RE_FREE_DBR_HANDLE,
+						    BNXT_RE_OBJECT_DBR,
+						    UVERBS_ACCESS_DESTROY,
+						    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_DBR,
+			    UVERBS_TYPE_ALLOC_IDR(bnxt_re_dbr_cleanup),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_DBR_ALLOC),
+			    &UVERBS_METHOD(BNXT_RE_METHOD_DBR_FREE));
+
+DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR,
+			    UVERBS_ATTR_PTR_OUT(BNXT_RE_DEFAULT_DBR_ATTR,
+						UVERBS_ATTR_STRUCT(struct bnxt_re_db_region,
+								   umdbr),
+						UA_MANDATORY));
+
+DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR,
+			      &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR));
+
+const struct uapi_definition bnxt_re_uapi_defs[] = {
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR),
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR),
+	{}
+};
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index b8d49abde0994..9fde78b746900 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -32,9 +32,9 @@
 
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <rdma/ib_umem.h>
 #include <linux/atomic.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/iter.h>
 
 #include "iw_cxgb4.h"
 
diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index 96f9c3bc98b2e..00b19f2ba3da3 100644
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -161,10 +161,8 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 		  struct ib_udata *udata);
 int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
-int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		  struct uverbs_attr_bundle *attrs);
-int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		       struct ib_umem *umem, struct uverbs_attr_bundle *attrs);
+int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+		       struct uverbs_attr_bundle *attrs);
 struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
 			 u64 virt_addr, int access_flags,
 			 struct ib_dmah *dmah,
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index 57178dad5eb7b..ad34ea5da6b07 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_ADMIN_CMDS_H_
@@ -38,10 +38,11 @@ enum efa_admin_aq_feature_id {
 	EFA_ADMIN_DEVICE_ATTR                       = 1,
 	EFA_ADMIN_AENQ_CONFIG                       = 2,
 	EFA_ADMIN_NETWORK_ATTR                      = 3,
-	EFA_ADMIN_QUEUE_ATTR                        = 4,
+	EFA_ADMIN_QUEUE_ATTR_1                      = 4,
 	EFA_ADMIN_HW_HINTS                          = 5,
 	EFA_ADMIN_HOST_INFO                         = 6,
 	EFA_ADMIN_EVENT_QUEUE_ATTR                  = 7,
+	EFA_ADMIN_QUEUE_ATTR_2                      = 9,
 };
 
 /* QP transport type */
@@ -744,14 +745,19 @@ struct efa_admin_feature_device_attr_desc {
 	u32 reserved1;
 };
 
-struct efa_admin_feature_queue_attr_desc {
+struct efa_admin_feature_queue_attr_desc_1 {
 	/* The maximum number of queue pairs supported */
 	u32 max_qp;
 
 	/* Maximum number of WQEs per Send Queue */
 	u32 max_sq_depth;
 
-	/* Maximum size of data that can be sent inline in a Send WQE */
+	/*
+	 * Maximum size of data that can be sent inline in a Send WQE
+	 * (deprecated by
+	 * efa_admin_feature_queue_attr_desc_2::inline_buf_size_ex on
+	 * supporting devices)
+	 */
 	u32 inline_buf_size;
 
 	/* Maximum number of buffer descriptors per Recv Queue */
@@ -805,6 +811,11 @@ struct efa_admin_feature_queue_attr_desc {
 	u16 max_tx_batch;
 };
 
+struct efa_admin_feature_queue_attr_desc_2 {
+	/* Maximum size of data that can be sent inline in a Send WQE */
+	u16 inline_buf_size_ex;
+};
+
 struct efa_admin_event_queue_attr_desc {
 	/* The maximum number of event queues supported */
 	u32 max_eq;
@@ -872,7 +883,9 @@ struct efa_admin_get_feature_resp {
 
 		struct efa_admin_feature_network_attr_desc network_attr;
 
-		struct efa_admin_feature_queue_attr_desc queue_attr;
+		struct efa_admin_feature_queue_attr_desc_1 queue_attr_1;
+
+		struct efa_admin_feature_queue_attr_desc_2 queue_attr_2;
 
 		struct efa_admin_event_queue_attr_desc event_queue_attr;
 
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index 229b0ad3b0cbb..e97b5f0d70038 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <linux/log2.h>
@@ -310,23 +310,19 @@ static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com
 	return &aq->comp_ctx[ctx_id];
 }
 
-static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
-						       struct efa_admin_aq_entry *cmd,
-						       size_t cmd_size_in_bytes,
-						       struct efa_admin_acq_entry *comp,
-						       size_t comp_size_in_bytes)
+static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
+				       struct efa_comp_ctx *comp_ctx,
+				       struct efa_admin_aq_entry *cmd,
+				       size_t cmd_size_in_bytes,
+				       struct efa_admin_acq_entry *comp,
+				       size_t comp_size_in_bytes)
 {
 	struct efa_admin_aq_entry *aqe;
-	struct efa_comp_ctx *comp_ctx;
 	u16 queue_size_mask;
 	u16 cmd_id;
 	u16 ctx_id;
 	u16 pi;
 
-	comp_ctx = efa_com_alloc_comp_ctx(aq);
-	if (!comp_ctx)
-		return ERR_PTR(-EINVAL);
-
 	queue_size_mask = aq->depth - 1;
 	pi = aq->sq.pc & queue_size_mask;
 	ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx);
@@ -360,8 +356,6 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu
 
 	/* barrier not needed in case of writel */
 	writel(aq->sq.pc, aq->sq.db_addr);
-
-	return comp_ctx;
 }
 
 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)
@@ -394,28 +388,25 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)
 	return 0;
 }
 
-static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
-						     struct efa_admin_aq_entry *cmd,
-						     size_t cmd_size_in_bytes,
-						     struct efa_admin_acq_entry *comp,
-						     size_t comp_size_in_bytes)
+static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
+				    struct efa_comp_ctx *comp_ctx,
+				    struct efa_admin_aq_entry *cmd,
+				    size_t cmd_size_in_bytes,
+				    struct efa_admin_acq_entry *comp,
+				    size_t comp_size_in_bytes)
 {
-	struct efa_comp_ctx *comp_ctx;
-
 	spin_lock(&aq->sq.lock);
 	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {
 		ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n");
 		spin_unlock(&aq->sq.lock);
-		return ERR_PTR(-ENODEV);
+		return -ENODEV;
 	}
 
-	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,
-					      comp_size_in_bytes);
+	__efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size_in_bytes, comp,
+				   comp_size_in_bytes);
 	spin_unlock(&aq->sq.lock);
-	if (IS_ERR(comp_ctx))
-		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 
-	return comp_ctx;
+	return 0;
 }
 
 static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
@@ -512,7 +503,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c
 {
 	unsigned long timeout;
 	unsigned long flags;
-	int err;
 
 	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);
 
@@ -532,24 +522,20 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c
 			atomic64_inc(&aq->stats.no_completion);
 
 			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
-			err = -ETIME;
-			goto out;
+			return -ETIME;
 		}
 
 		msleep(aq->poll_interval);
 	}
 
-	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
-out:
-	efa_com_dealloc_comp_ctx(aq, comp_ctx);
-	return err;
+	return efa_com_comp_status_to_errno(
+		comp_ctx->user_cqe->acq_common_descriptor.status);
 }
 
 static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx,
 							struct efa_com_admin_queue *aq)
 {
 	unsigned long flags;
-	int err;
 
 	wait_for_completion_timeout(&comp_ctx->wait_event,
 				    usecs_to_jiffies(aq->completion_timeout));
@@ -585,14 +571,11 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com
 				aq->cq.cc);
 
 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
-		err = -ETIME;
-		goto out;
+		return -ETIME;
 	}
 
-	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
-out:
-	efa_com_dealloc_comp_ctx(aq, comp_ctx);
-	return err;
+	return efa_com_comp_status_to_errno(
+		comp_ctx->user_cqe->acq_common_descriptor.status);
 }
 
 /*
@@ -642,30 +625,39 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
 	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",
 		  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 		  cmd->aq_common_descriptor.opcode);
-	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);
-	if (IS_ERR(comp_ctx)) {
+
+	comp_ctx = efa_com_alloc_comp_ctx(aq);
+	if (!comp_ctx) {
+		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+		up(&aq->avail_cmds);
+		return -EINVAL;
+	}
+
+	err = efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size, comp, comp_size);
+	if (err) {
 		ibdev_err_ratelimited(
 			aq->efa_dev,
-			"Failed to submit command %s (opcode %u) err %pe\n",
+			"Failed to submit command %s (opcode %u) err %d\n",
 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
-			cmd->aq_common_descriptor.opcode, comp_ctx);
+			cmd->aq_common_descriptor.opcode, err);
 
+		efa_com_dealloc_comp_ctx(aq, comp_ctx);
 		up(&aq->avail_cmds);
 		atomic64_inc(&aq->stats.cmd_err);
-		return PTR_ERR(comp_ctx);
+		return err;
 	}
 
 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
 	if (err) {
 		ibdev_err_ratelimited(
 			aq->efa_dev,
-			"Failed to process command %s (opcode %u) comp_status %d err %d\n",
+			"Failed to process command %s (opcode %u) err %d\n",
 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
-			cmd->aq_common_descriptor.opcode,
-			comp_ctx->user_cqe->acq_common_descriptor.status, err);
+			cmd->aq_common_descriptor.opcode, err);
 		atomic64_inc(&aq->stats.cmd_err);
 	}
 
+	efa_com_dealloc_comp_ctx(aq, comp_ctx);
 	up(&aq->avail_cmds);
 
 	return err;
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 9ead02800ac7c..63c7f07806a82 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include "efa_com.h"
@@ -479,31 +479,46 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,
 
 	edev->supported_features = resp.u.device_attr.supported_features;
 	err = efa_com_get_feature(edev, &resp,
-				  EFA_ADMIN_QUEUE_ATTR);
+				  EFA_ADMIN_QUEUE_ATTR_1);
 	if (err) {
 		ibdev_err_ratelimited(edev->efa_dev,
-				      "Failed to get queue attributes %d\n",
+				      "Failed to get queue attributes1 %d\n",
 				      err);
 		return err;
 	}
 
-	result->max_qp = resp.u.queue_attr.max_qp;
-	result->max_sq_depth = resp.u.queue_attr.max_sq_depth;
-	result->max_rq_depth = resp.u.queue_attr.max_rq_depth;
-	result->max_cq = resp.u.queue_attr.max_cq;
-	result->max_cq_depth = resp.u.queue_attr.max_cq_depth;
-	result->inline_buf_size = resp.u.queue_attr.inline_buf_size;
-	result->max_sq_sge = resp.u.queue_attr.max_wr_send_sges;
-	result->max_rq_sge = resp.u.queue_attr.max_wr_recv_sges;
-	result->max_mr = resp.u.queue_attr.max_mr;
-	result->max_mr_pages = resp.u.queue_attr.max_mr_pages;
-	result->max_pd = resp.u.queue_attr.max_pd;
-	result->max_ah = resp.u.queue_attr.max_ah;
-	result->max_llq_size = resp.u.queue_attr.max_llq_size;
-	result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;
-	result->max_wr_rdma_sge = resp.u.queue_attr.max_wr_rdma_sges;
-	result->max_tx_batch = resp.u.queue_attr.max_tx_batch;
-	result->min_sq_depth = resp.u.queue_attr.min_sq_depth;
+	result->max_qp = resp.u.queue_attr_1.max_qp;
+	result->max_sq_depth = resp.u.queue_attr_1.max_sq_depth;
+	result->max_rq_depth = resp.u.queue_attr_1.max_rq_depth;
+	result->max_cq = resp.u.queue_attr_1.max_cq;
+	result->max_cq_depth = resp.u.queue_attr_1.max_cq_depth;
+	result->inline_buf_size = resp.u.queue_attr_1.inline_buf_size;
+	result->max_sq_sge = resp.u.queue_attr_1.max_wr_send_sges;
+	result->max_rq_sge = resp.u.queue_attr_1.max_wr_recv_sges;
+	result->max_mr = resp.u.queue_attr_1.max_mr;
+	result->max_mr_pages = resp.u.queue_attr_1.max_mr_pages;
+	result->max_pd = resp.u.queue_attr_1.max_pd;
+	result->max_ah = resp.u.queue_attr_1.max_ah;
+	result->max_llq_size = resp.u.queue_attr_1.max_llq_size;
+	result->sub_cqs_per_cq = resp.u.queue_attr_1.sub_cqs_per_cq;
+	result->max_wr_rdma_sge = resp.u.queue_attr_1.max_wr_rdma_sges;
+	result->max_tx_batch = resp.u.queue_attr_1.max_tx_batch;
+	result->min_sq_depth = resp.u.queue_attr_1.min_sq_depth;
+
+	if (efa_com_check_supported_feature_id(edev, EFA_ADMIN_QUEUE_ATTR_2)) {
+		err = efa_com_get_feature(edev, &resp,
+					  EFA_ADMIN_QUEUE_ATTR_2);
+		if (err) {
+			ibdev_err_ratelimited(
+				edev->efa_dev,
+				"Failed to get queue attributes2 %d\n", err);
+			return err;
+		}
+
+		result->inline_buf_size_ex = resp.u.queue_attr_2.inline_buf_size_ex;
+	} else {
+		result->inline_buf_size_ex = result->inline_buf_size;
+	}
 
 	err = efa_com_get_feature(edev, &resp, EFA_ADMIN_NETWORK_ATTR);
 	if (err) {
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index 3ac2686abba1f..ef15b3c38429c 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_CMD_H_
@@ -127,6 +127,7 @@ struct efa_com_get_device_attr_result {
 	u32 max_cq;
 	u32 max_cq_depth; /* cqes */
 	u32 inline_buf_size;
+	u32 inline_buf_size_ex;
 	u32 max_mr;
 	u32 max_pd;
 	u32 max_ah;
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index c1397086dc47c..03c237c8c81ee 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -371,8 +371,7 @@ static const struct ib_device_ops efa_dev_ops = {
 	.alloc_hw_device_stats = efa_alloc_hw_device_stats,
 	.alloc_pd = efa_alloc_pd,
 	.alloc_ucontext = efa_alloc_ucontext,
-	.create_cq = efa_create_cq,
-	.create_cq_umem = efa_create_cq_umem,
+	.create_user_cq = efa_create_user_cq,
 	.create_qp = efa_create_qp,
 	.create_user_ah = efa_create_ah,
 	.dealloc_pd = efa_dealloc_pd,
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index b0aa7c0238ede..fc498663cd372 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /*
- * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <linux/dma-buf.h>
@@ -9,9 +9,9 @@
 #include <linux/log2.h>
 
 #include <rdma/ib_addr.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/ib_verbs.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 #define UVERBS_MODULE_NAME efa_ib
 #include <rdma/uverbs_named_ioctl.h>
@@ -579,7 +579,7 @@ static int qp_mmap_entries_setup(struct efa_qp *qp,
 
 	resp->llq_desc_offset &= ~PAGE_MASK;
 
-	if (qp->rq_size) {
+	if (qp->rq_cpu_addr) {
 		address = dev->db_bar_addr + resp->rq_db_offset;
 
 		qp->rq_db_mmap_entry =
@@ -641,11 +641,11 @@ static int efa_qp_validate_cap(struct efa_dev *dev,
 			  init_attr->cap.max_recv_sge, dev->dev_attr.max_rq_sge);
 		return -EINVAL;
 	}
-	if (init_attr->cap.max_inline_data > dev->dev_attr.inline_buf_size) {
+	if (init_attr->cap.max_inline_data > dev->dev_attr.inline_buf_size_ex) {
 		ibdev_dbg(&dev->ibdev,
 			  "qp: requested inline data[%u] exceeds the max[%u]\n",
 			  init_attr->cap.max_inline_data,
-			  dev->dev_attr.inline_buf_size);
+			  dev->dev_attr.inline_buf_size_ex);
 		return -EINVAL;
 	}
 
@@ -828,7 +828,7 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 err_destroy_qp:
 	efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);
 err_free_mapped:
-	if (qp->rq_size)
+	if (qp->rq_cpu_addr)
 		efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr,
 				qp->rq_size, DMA_TO_DEVICE);
 err_out:
@@ -1083,15 +1083,14 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 		  cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
 
 	efa_destroy_cq_idx(dev, cq->cq_idx);
-	efa_cq_user_mmap_entries_remove(cq);
+	if (cq->cpu_addr)
+		efa_cq_user_mmap_entries_remove(cq);
 	if (cq->eq) {
 		xa_erase(&dev->cqs_xa, cq->cq_idx);
 		synchronize_irq(cq->eq->irq.irqn);
 	}
 
-	if (cq->umem)
-		ib_umem_release(cq->umem);
-	else
+	if (cq->cpu_addr)
 		efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE);
 	return 0;
 }
@@ -1131,8 +1130,8 @@ static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
 	return 0;
 }
 
-int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		       struct ib_umem *umem, struct uverbs_attr_bundle *attrs)
+int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+		       struct uverbs_attr_bundle *attrs)
 {
 	struct ib_udata *udata = &attrs->driver_udata;
 	struct efa_ucontext *ucontext = rdma_udata_to_drv_context(
@@ -1153,9 +1152,9 @@ int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	if (attr->flags)
 		return -EOPNOTSUPP;
 
-	if (entries < 1 || entries > dev->dev_attr.max_cq_depth) {
+	if (entries > dev->dev_attr.max_cq_depth) {
 		ibdev_dbg(ibdev,
-			  "cq: requested entries[%u] non-positive or greater than max[%u]\n",
+			  "cq: requested entries[%u] greater than max[%u]\n",
 			  entries, dev->dev_attr.max_cq_depth);
 		err = -EINVAL;
 		goto err_out;
@@ -1212,22 +1211,20 @@ int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	cq->ucontext = ucontext;
 	cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);
 
-	if (umem) {
-		if (umem->length < cq->size) {
+	if (ibcq->umem) {
+		if (ibcq->umem->length < cq->size) {
 			ibdev_dbg(&dev->ibdev, "External memory too small\n");
 			err = -EINVAL;
 			goto err_out;
 		}
 
-		if (!ib_umem_is_contiguous(umem)) {
+		if (!ib_umem_is_contiguous(ibcq->umem)) {
 			ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n");
 			err = -EINVAL;
 			goto err_out;
 		}
 
-		cq->cpu_addr = NULL;
-		cq->dma_addr = ib_umem_start_dma_addr(umem);
-		cq->umem = umem;
+		cq->dma_addr = ib_umem_start_dma_addr(ibcq->umem);
 	} else {
 		cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
 						 DMA_FROM_DEVICE);
@@ -1259,7 +1256,7 @@ int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	cq->ibcq.cqe = result.actual_depth;
 	WARN_ON_ONCE(entries != result.actual_depth);
 
-	if (!umem)
+	if (cq->cpu_addr)
 		err = cq_mmap_entries_setup(dev, cq, &resp, result.db_valid);
 
 	if (err) {
@@ -1296,11 +1293,12 @@ int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	if (cq->eq)
 		xa_erase(&dev->cqs_xa, cq->cq_idx);
 err_remove_mmap:
-	efa_cq_user_mmap_entries_remove(cq);
+	if (cq->cpu_addr)
+		efa_cq_user_mmap_entries_remove(cq);
 err_destroy_cq:
 	efa_destroy_cq_idx(dev, cq->cq_idx);
 err_free_mapped:
-	if (!umem)
+	if (cq->cpu_addr)
 		efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
 				DMA_FROM_DEVICE);
 err_out:
@@ -1308,12 +1306,6 @@ int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	return err;
 }
 
-int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		  struct uverbs_attr_bundle *attrs)
-{
-	return efa_create_cq_umem(ibcq, attr, NULL, attrs);
-}
-
 static int umem_to_page_list(struct efa_dev *dev,
 			     struct ib_umem *umem,
 			     u64 *page_list,
@@ -1988,6 +1980,7 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
 	resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_CREATE_AH;
 	resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq;
 	resp.inline_buf_size = dev->dev_attr.inline_buf_size;
+	resp.inline_buf_size_ex = dev->dev_attr.inline_buf_size_ex;
 	resp.max_llq_size = dev->dev_attr.max_llq_size;
 	resp.max_tx_batch = dev->dev_attr.max_tx_batch;
 	resp.min_sq_wr = dev->dev_attr.min_sq_depth;
diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h
index 2a023b99f9926..ceabbdf2556f8 100644
--- a/drivers/infiniband/hw/erdma/erdma.h
+++ b/drivers/infiniband/hw/erdma/erdma.h
@@ -127,7 +127,6 @@ struct erdma_devattr {
 	unsigned char peer_addr[ETH_ALEN];
 	unsigned long cap_flags;
 
-	int numa_node;
 	enum erdma_cc_alg cc;
 	u32 irq_num;
 
diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c
index 6486234a23600..d5b9d19882b2a 100644
--- a/drivers/infiniband/hw/erdma/erdma_eq.c
+++ b/drivers/infiniband/hw/erdma/erdma_eq.c
@@ -197,7 +197,8 @@ static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)
 	tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task,
 		     (unsigned long)&dev->ceqs[ceqn]);
 
-	cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node),
+	cpumask_set_cpu(cpumask_local_spread(ceqn + 1,
+					     dev_to_node(&dev->pdev->dev)),
 			&eqc->irq.affinity_hint_mask);
 
 	err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0,
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index f35b30235018b..7e87a815e853b 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -261,7 +261,6 @@ static int erdma_probe_dev(struct pci_dev *pdev)
 
 	pci_set_drvdata(pdev, dev);
 	dev->pdev = pdev;
-	dev->attrs.numa_node = dev_to_node(&pdev->dev);
 
 	bars = pci_select_bars(pdev, IORESOURCE_MEM);
 	err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 9f74aadc30470..04136a0281aa4 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -12,7 +12,7 @@
 #include <linux/vmalloc.h>
 #include <net/addrconf.h>
 #include <rdma/erdma-abi.h>
-#include <rdma/ib_umem.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include "erdma.h"
diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile
index 5d977f3636847..b5551bd4703b8 100644
--- a/drivers/infiniband/hw/hfi1/Makefile
+++ b/drivers/infiniband/hw/hfi1/Makefile
@@ -49,9 +49,7 @@ hfi1-y := \
 	user_pages.o \
 	user_sdma.o \
 	verbs.o \
-	verbs_txreq.o \
-	vnic_main.o \
-	vnic_sdma.o
+	verbs_txreq.o
 
 ifdef CONFIG_DEBUG_FS
 hfi1-y += debugfs.o
diff --git a/drivers/infiniband/hw/hfi1/aspm.c b/drivers/infiniband/hw/hfi1/aspm.c
index 79990d09522bc..61455d4ac6c2a 100644
--- a/drivers/infiniband/hw/hfi1/aspm.c
+++ b/drivers/infiniband/hw/hfi1/aspm.c
@@ -179,7 +179,7 @@ static  void aspm_ctx_timer_function(struct timer_list *t)
 }
 
 /*
- * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
+ * Disable interrupt processing for verbs contexts when PSM contexts
  * are open.
  */
 void aspm_disable_all(struct hfi1_devdata *dd)
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 6a9db4d17c5ac..44c524e453968 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -85,12 +85,12 @@ struct flag_table {
 /*
  * RSM instance allocation
  *   0 - User Fecn Handling
- *   1 - Vnic
+ *   1 - Deprecated
  *   2 - AIP
  *   3 - Verbs
  */
 #define RSM_INS_FECN              0
-#define RSM_INS_VNIC              1
+#define RSM_INS_DEPRECATED        1
 #define RSM_INS_AIP               2
 #define RSM_INS_VERBS             3
 
@@ -152,15 +152,6 @@ struct flag_table {
 #define DETH_AIP_SQPN_SELECT_OFFSET \
 	DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)
 
-/* RSM fields for Vnic */
-/* L2_TYPE: QW 0, OFFSET 61 - for match */
-#define L2_TYPE_QW             0ull
-#define L2_TYPE_BIT_OFFSET     61ull
-#define L2_TYPE_OFFSET(off)    ((L2_TYPE_QW << QW_SHIFT) | (off))
-#define L2_TYPE_MATCH_OFFSET   L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
-#define L2_TYPE_MASK           3ull
-#define L2_16B_VALUE           2ull
-
 /* L4_TYPE QW 1, OFFSET 0 - for match */
 #define L4_TYPE_QW              1ull
 #define L4_TYPE_BIT_OFFSET      0ull
@@ -6844,9 +6835,9 @@ static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
 	for (i = 0; i < dd->num_rcv_contexts; i++) {
 		rcd = hfi1_rcd_get_by_index(dd, i);
 
-		/* Ensure all non-user contexts(including vnic) are enabled */
+		/* Ensure all non-user contexts are enabled */
 		if (!rcd ||
-		    (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) {
+		    (i >= dd->first_dyn_alloc_ctxt)) {
 			hfi1_rcd_put(rcd);
 			continue;
 		}
@@ -8467,7 +8458,7 @@ int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget)
 	return work_done;
 }
 
-/* Receive packet napi handler for netdevs VNIC and AIP  */
+/* Receive packet napi handler for netdevs AIP  */
 irqreturn_t receive_context_interrupt_napi(int irq, void *data)
 {
 	struct hfi1_ctxtdata *rcd = data;
@@ -14506,7 +14497,7 @@ static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd)
 	int ctxt_count = hfi1_netdev_ctxt_count(dd);
 
 	/* We already have contexts mapped in RMT */
-	if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) {
+	if (has_rsm_rule(dd, RSM_INS_AIP)) {
 		dd_dev_info(dd, "Contexts are already mapped in RMT\n");
 		return true;
 	}
@@ -14587,37 +14578,6 @@ void hfi1_init_aip_rsm(struct hfi1_devdata *dd)
 	}
 }
 
-/* Initialize RSM for VNIC */
-void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
-{
-	int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
-	struct rsm_rule_data rrd = {
-		/* Add rule for vnic */
-		.offset = rmt_start,
-		.pkt_type = 4,
-		/* Match 16B packets */
-		.field1_off = L2_TYPE_MATCH_OFFSET,
-		.mask1 = L2_TYPE_MASK,
-		.value1 = L2_16B_VALUE,
-		/* Match ETH L4 packets */
-		.field2_off = L4_TYPE_MATCH_OFFSET,
-		.mask2 = L4_16B_TYPE_MASK,
-		.value2 = L4_16B_ETH_VALUE,
-		/* Calc context from veswid and entropy */
-		.index1_off = L4_16B_HDR_VESWID_OFFSET,
-		.index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
-		.index2_off = L2_16B_ENTROPY_OFFSET,
-		.index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
-	};
-
-	hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd);
-}
-
-void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
-{
-	clear_rsm_rule(dd, RSM_INS_VNIC);
-}
-
 void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd)
 {
 	/* only actually clear the rule if it's the last user asking to do so */
@@ -15195,7 +15155,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 		 (dd->revision >> CCE_REVISION_SW_SHIFT)
 		    & CCE_REVISION_SW_MASK);
 
-	/* alloc VNIC/AIP rx data */
+	/* alloc AIP rx data */
 	ret = hfi1_alloc_rx(dd);
 	if (ret)
 		goto bail_cleanup;
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index 6992f6d402559..56e03d486acef 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -1392,8 +1392,6 @@ int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt,
 		       u16 pkey);
 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt);
 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality);
-void hfi1_init_vnic_rsm(struct hfi1_devdata *dd);
-void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd);
 
 irqreturn_t general_interrupt(int irq, void *data);
 irqreturn_t sdma_interrupt(int irq, void *data);
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index 06487e20f7237..c7259cc390138 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -20,7 +20,6 @@
 #include "qp.h"
 #include "sdma.h"
 #include "debugfs.h"
-#include "vnic.h"
 #include "fault.h"
 
 #include "ipoib.h"
@@ -909,11 +908,11 @@ static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 	u16 i;
 
 	/*
-	 * For dynamically allocated kernel contexts (like vnic) switch
+	 * For dynamically allocated kernel contexts switch
 	 * interrupt handler only for that context. Otherwise, switch
 	 * interrupt handler for all statically allocated kernel contexts.
 	 */
-	if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) {
+	if (rcd->ctxt >= dd->first_dyn_alloc_ctxt) {
 		hfi1_rcd_get(rcd);
 		hfi1_set_fast(rcd);
 		hfi1_rcd_put(rcd);
@@ -922,7 +921,7 @@ static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 
 	for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
 		rcd = hfi1_rcd_get_by_index(dd, i);
-		if (rcd && (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic))
+		if (rcd && (i < dd->first_dyn_alloc_ctxt))
 			hfi1_set_fast(rcd);
 		hfi1_rcd_put(rcd);
 	}
@@ -938,7 +937,7 @@ void set_all_slowpath(struct hfi1_devdata *dd)
 		rcd = hfi1_rcd_get_by_index(dd, i);
 		if (!rcd)
 			continue;
-		if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)
+		if (i < dd->first_dyn_alloc_ctxt)
 			rcd->do_interrupt = rcd->slow_handler;
 
 		hfi1_rcd_put(rcd);
@@ -1400,7 +1399,7 @@ int hfi1_reset_device(int unit)
 		goto bail;
 	}
 
-	/* If there are any user/vnic contexts, we cannot reset */
+	/* If there are any user contexts, we cannot reset */
 	mutex_lock(&hfi1_mutex);
 	if (dd->rcd)
 		if (hfi1_stats.sps_ctxts) {
@@ -1899,7 +1898,7 @@ const rhf_rcv_function_ptr netdev_rhf_rcv_functions[] = {
 	[RHF_RCV_TYPE_EAGER] = process_receive_invalid,
 	[RHF_RCV_TYPE_IB] = hfi1_ipoib_ib_rcv,
 	[RHF_RCV_TYPE_ERROR] = process_receive_error,
-	[RHF_RCV_TYPE_BYPASS] = hfi1_vnic_bypass_rcv,
+	[RHF_RCV_TYPE_BYPASS] = process_receive_invalid,
 	[RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
 	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
 	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index cb630551cf1a3..5a0310f758dc2 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -212,10 +212,6 @@ struct hfi1_ctxtdata {
 	u8 rhf_offset;
 	/* dynamic receive available interrupt timeout */
 	u8 rcvavail_timeout;
-	/* Indicates that this is vnic context */
-	bool is_vnic;
-	/* vnic queue index this context is mapped to */
-	u8 vnic_q_idx;
 	/* Is ASPM interrupt supported for this context */
 	bool aspm_intr_supported;
 	/* ASPM state (enabled/disabled) for this context */
@@ -402,7 +398,6 @@ struct hfi1_packet {
 #define OPA_16B_L4_FM		0x08
 #define OPA_16B_L4_IB_LOCAL	0x09
 #define OPA_16B_L4_IB_GLOBAL	0x0A
-#define OPA_16B_L4_ETHR		OPA_VNIC_L4_ETHR
 
 /*
  * OPA 16B Management
@@ -997,14 +992,6 @@ struct hfi1_asic_data {
 #define NUM_MAP_ENTRIES	 256
 #define NUM_MAP_REGS      32
 
-/* Virtual NIC information */
-struct hfi1_vnic_data {
-	struct kmem_cache *txreq_cache;
-	u8 num_vports;
-};
-
-struct hfi1_vnic_vport_info;
-
 /* device data struct now contains only "general per-device" info.
  * fields related to a physical IB port are in a hfi1_pportdata struct.
  */
@@ -1298,9 +1285,6 @@ struct hfi1_devdata {
 	send_routine process_dma_send;
 	void (*pio_inline_send)(struct hfi1_devdata *dd, struct pio_buf *pbuf,
 				u64 pbc, const void *from, size_t count);
-	int (*process_vnic_dma_send)(struct hfi1_devdata *dd, u8 q_idx,
-				     struct hfi1_vnic_vport_info *vinfo,
-				     struct sk_buff *skb, u64 pbc, u8 plen);
 	/* hfi1_pportdata, points to array of (physical) port-specific
 	 * data structs, indexed by pidx (0..n-1)
 	 */
@@ -1314,7 +1298,6 @@ struct hfi1_devdata {
 	u16 flags;
 	/* Number of physical ports available */
 	u8 num_pports;
-	/* Lowest context number which can be used by user processes or VNIC */
 	u8 first_dyn_alloc_ctxt;
 	/* adding a new field here would make it part of this cacheline */
 
@@ -1353,11 +1336,8 @@ struct hfi1_devdata {
 	bool aspm_enabled;	/* ASPM state: enabled/disabled */
 	struct rhashtable *sdma_rht;
 
-	/* vnic data */
-	struct hfi1_vnic_data vnic;
 	/* Lock to protect IRQ SRC register access */
 	spinlock_t irq_src_lock;
-	int vnic_num_vports;
 	struct hfi1_netdev_rx *netdev_rx;
 	struct hfi1_affinity_node *affinity_entry;
 
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 07333dd376526..8b5a5b32b0fa2 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -26,7 +26,6 @@
 #include "verbs.h"
 #include "aspm.h"
 #include "affinity.h"
-#include "vnic.h"
 #include "exp_rcv.h"
 #include "netdev.h"
 
@@ -349,7 +348,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 		 * We do this here because we have to take into account all
 		 * the RcvArray entries that previous context would have
 		 * taken and we have to account for any extra groups assigned
-		 * to the static (kernel) or dynamic (vnic/user) contexts.
+		 * to the static (kernel) or dynamic (user) contexts.
 		 */
 		if (ctxt < dd->first_dyn_alloc_ctxt) {
 			if (ctxt < kctxt_ngroups) {
@@ -851,7 +850,6 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
 	dd->process_pio_send = hfi1_verbs_send_pio;
 	dd->process_dma_send = hfi1_verbs_send_dma;
 	dd->pio_inline_send = pio_copy;
-	dd->process_vnic_dma_send = hfi1_vnic_send_dma;
 
 	if (is_ax(dd)) {
 		atomic_set(&dd->drop_packet, DROP_PACKET_ON);
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 03467e6c19a0d..585f1d99b91bc 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -12,7 +12,6 @@
 #include "mad.h"
 #include "trace.h"
 #include "qp.h"
-#include "vnic.h"
 
 /* the reset value from the FM is supposed to be 0xffff, handle both */
 #define OPA_LINK_WIDTH_RESET_OLD 0x0fff
diff --git a/drivers/infiniband/hw/hfi1/msix.c b/drivers/infiniband/hw/hfi1/msix.c
index 3ac50ca4afccd..c06f4741c89e2 100644
--- a/drivers/infiniband/hw/hfi1/msix.c
+++ b/drivers/infiniband/hw/hfi1/msix.c
@@ -24,7 +24,6 @@ int msix_initialize(struct hfi1_devdata *dd)
 	 *	one for the general, "slow path" interrupt
 	 *	one per used SDMA engine
 	 *	one per kernel receive context
-	 *	one for each VNIC context
 	 *      ...any new IRQs should be added here.
 	 */
 	total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts;
@@ -127,8 +126,7 @@ static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,
 				       irq_handler_t thread,
 				       const char *name)
 {
-	int nr = msix_request_irq(rcd->dd, rcd, handler, thread,
-				  rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,
+	int nr = msix_request_irq(rcd->dd, rcd, handler, thread, IRQ_RCVCTXT,
 				  name);
 	if (nr < 0)
 		return nr;
diff --git a/drivers/infiniband/hw/hfi1/netdev.h b/drivers/infiniband/hw/hfi1/netdev.h
index 07c8f77c91817..c6440bd07d2e7 100644
--- a/drivers/infiniband/hw/hfi1/netdev.h
+++ b/drivers/infiniband/hw/hfi1/netdev.h
@@ -14,7 +14,7 @@
 
 /**
  * struct hfi1_netdev_rxq - Receive Queue for HFI
- * Both IPoIB and VNIC netdevices will be working on the rx abstraction.
+ * IPoIB netdevices will be working on the rx abstraction.
  * @napi: napi object
  * @rx: ptr to netdev_rx
  * @rcd:  ptr to receive context data
@@ -25,10 +25,6 @@ struct hfi1_netdev_rxq {
 	struct hfi1_ctxtdata *rcd;
 };
 
-/*
- * Number of netdev contexts used. Ensure it is less than or equal to
- * max queues supported by VNIC (HFI1_VNIC_MAX_QUEUE).
- */
 #define HFI1_MAX_NETDEV_CTXTS   8
 
 /* Number of NETDEV RSM entries */
@@ -42,7 +38,7 @@ struct hfi1_netdev_rxq {
  * @num_rx_q:	number of receive queues
  * @rmt_index:	first free index in RMT Array
  * @msix_start: first free MSI-X interrupt vector.
- * @dev_tbl:	netdev table for unique identifier VNIC and IPoIb VLANs.
+ * @dev_tbl:	netdev table for unique identifier IPoIb VLANs.
  * @enabled:	atomic counter of netdevs enabling receive queues.
  *		When 0 NAPI will be disabled.
  * @netdevs:	atomic counter of netdevs using dummy netdev.
diff --git a/drivers/infiniband/hw/hfi1/netdev_rx.c b/drivers/infiniband/hw/hfi1/netdev_rx.c
index 8608044203bb9..ca2ae52b21e38 100644
--- a/drivers/infiniband/hw/hfi1/netdev_rx.c
+++ b/drivers/infiniband/hw/hfi1/netdev_rx.c
@@ -78,7 +78,6 @@ static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,
 	uctxt->fast_handler = handle_receive_interrupt_napi_fp;
 	uctxt->slow_handler = handle_receive_interrupt_napi_sp;
 	hfi1_set_seq_cnt(uctxt, 1);
-	uctxt->is_vnic = true;
 
 	hfi1_stats.sps_ctxts++;
 
@@ -427,7 +426,7 @@ void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
 
 /**
  * hfi1_netdev_add_data - Registers data with unique identifier
- * to be requested later this is needed for VNIC and IPoIB VLANs
+ * to be requested later this is needed for IPoIB VLANs
  * implementations.
  * This call is protected by mutex idr_lock.
  *
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
index 5b01070ed66f8..8b50a2ad792cb 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
@@ -257,7 +257,7 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	if (tinfo->length == 0)
 		return -EINVAL;
 
-	tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);
+	tidbuf = kzalloc_flex(*tidbuf, psets, uctxt->expected_count);
 	if (!tidbuf)
 		return -ENOMEM;
 
@@ -265,12 +265,6 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	tidbuf->vaddr = tinfo->vaddr;
 	tidbuf->length = tinfo->length;
 	tidbuf->npages = num_user_pages(tidbuf->vaddr, tidbuf->length);
-	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),
-				GFP_KERNEL);
-	if (!tidbuf->psets) {
-		ret = -ENOMEM;
-		goto fail_release_mem;
-	}
 
 	if (fd->use_mn) {
 		ret = mmu_interval_notifier_insert(
@@ -306,7 +300,7 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	}
 
 	ngroups = pageset_count / dd->rcv_entries.group_size;
-	tidlist = kcalloc(pageset_count, sizeof(*tidlist), GFP_KERNEL);
+	tidlist = kzalloc_objs(*tidlist, pageset_count);
 	if (!tidlist) {
 		ret = -ENOMEM;
 		goto fail_unreserve;
@@ -448,7 +442,6 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	if (fd->use_mn)
 		mmu_interval_notifier_remove(&tidbuf->notifier);
 	kfree(tidbuf->pages);
-	kfree(tidbuf->psets);
 	kfree(tidbuf);
 	kfree(tidlist);
 	return 0;
@@ -471,7 +464,6 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 		unpin_rcv_pages(fd, tidbuf, NULL, 0, pinned, false);
 fail_release_mem:
 	kfree(tidbuf->pages);
-	kfree(tidbuf->psets);
 	kfree(tidbuf);
 	kfree(tidlist);
 	return ret;
@@ -527,7 +519,7 @@ int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd,
 	 * for a long time.
 	 * Copy the data to a local buffer so we can release the lock.
 	 */
-	array = kcalloc(uctxt->expected_count, sizeof(*array), GFP_KERNEL);
+	array = kzalloc_objs(*array, uctxt->expected_count);
 	if (!array)
 		return -EFAULT;
 
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.h b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
index 055726f7c139f..b4a309a051f91 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.h
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
@@ -22,8 +22,8 @@ struct tid_user_buf {
 	unsigned long length;
 	unsigned int npages;
 	struct page **pages;
-	struct tid_pageset *psets;
 	unsigned int n_psets;
+	struct tid_pageset psets[];
 };
 
 struct tid_rb_node {
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 3cbbfccdd8cdd..e569b647d611a 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -21,7 +21,6 @@
 #include "qp.h"
 #include "verbs_txreq.h"
 #include "debugfs.h"
-#include "vnic.h"
 #include "fault.h"
 #include "affinity.h"
 #include "ipoib.h"
@@ -1729,7 +1728,6 @@ static const struct ib_device_ops hfi1_dev_ops = {
 
 	.alloc_hw_device_stats = hfi1_alloc_hw_device_stats,
 	.alloc_hw_port_stats = hfi_alloc_hw_port_stats,
-	.alloc_rdma_netdev = hfi1_vnic_alloc_rn,
 	.device_group = &ib_hfi1_attr_group,
 	.get_dev_fw_str = hfi1_get_dev_fw_str,
 	.get_hw_stats = get_hw_stats,
diff --git a/drivers/infiniband/hw/hfi1/vnic.h b/drivers/infiniband/hw/hfi1/vnic.h
deleted file mode 100644
index bbafeb5fc0ecb..0000000000000
--- a/drivers/infiniband/hw/hfi1/vnic.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
-/*
- * Copyright(c) 2017 - 2020 Intel Corporation.
- */
-
-#ifndef _HFI1_VNIC_H
-#define _HFI1_VNIC_H
-#include <rdma/opa_vnic.h>
-#include "hfi.h"
-#include "sdma.h"
-
-#define HFI1_VNIC_MAX_TXQ     16
-#define HFI1_VNIC_MAX_PAD     12
-
-/* L4 header definitions */
-#define HFI1_VNIC_L4_HDR_OFFSET  OPA_VNIC_L2_HDR_LEN
-
-#define HFI1_VNIC_GET_L4_HDR(data)   \
-	(*((u16 *)((u8 *)(data) + HFI1_VNIC_L4_HDR_OFFSET)))
-
-#define HFI1_VNIC_GET_VESWID(data)   \
-	(HFI1_VNIC_GET_L4_HDR(data) & 0xFFF)
-
-/* Service class */
-#define HFI1_VNIC_SC_OFFSET_LOW 6
-#define HFI1_VNIC_SC_OFFSET_HI  7
-#define HFI1_VNIC_SC_SHIFT      4
-
-#define HFI1_VNIC_MAX_QUEUE 16
-#define HFI1_NUM_VNIC_CTXT 8
-
-/**
- * struct hfi1_vnic_sdma - VNIC per Tx ring SDMA information
- * @dd - device data pointer
- * @sde - sdma engine
- * @vinfo - vnic info pointer
- * @wait - iowait structure
- * @stx - sdma tx request
- * @state - vnic Tx ring SDMA state
- * @q_idx - vnic Tx queue index
- */
-struct hfi1_vnic_sdma {
-	struct hfi1_devdata *dd;
-	struct sdma_engine  *sde;
-	struct hfi1_vnic_vport_info *vinfo;
-	struct iowait wait;
-	struct sdma_txreq stx;
-	unsigned int state;
-	u8 q_idx;
-	bool pkts_sent;
-};
-
-/**
- * struct hfi1_vnic_rx_queue - HFI1 VNIC receive queue
- * @idx: queue index
- * @vinfo: pointer to vport information
- * @netdev: network device
- * @napi: netdev napi structure
- * @skbq: queue of received socket buffers
- */
-struct hfi1_vnic_rx_queue {
-	u8                           idx;
-	struct hfi1_vnic_vport_info *vinfo;
-	struct net_device           *netdev;
-	struct napi_struct           napi;
-};
-
-/**
- * struct hfi1_vnic_vport_info - HFI1 VNIC virtual port information
- * @dd: device data pointer
- * @netdev: net device pointer
- * @flags: state flags
- * @lock: vport lock
- * @num_tx_q: number of transmit queues
- * @num_rx_q: number of receive queues
- * @vesw_id: virtual switch id
- * @rxq: Array of receive queues
- * @stats: per queue stats
- * @sdma: VNIC SDMA structure per TXQ
- */
-struct hfi1_vnic_vport_info {
-	struct hfi1_devdata *dd;
-	struct net_device   *netdev;
-	unsigned long        flags;
-
-	/* Lock used around state updates */
-	struct mutex         lock;
-
-	u8  num_tx_q;
-	u8  num_rx_q;
-	u16 vesw_id;
-	struct hfi1_vnic_rx_queue rxq[HFI1_NUM_VNIC_CTXT];
-
-	struct opa_vnic_stats  stats[HFI1_VNIC_MAX_QUEUE];
-	struct hfi1_vnic_sdma  sdma[HFI1_VNIC_MAX_TXQ];
-};
-
-#define v_dbg(format, arg...) \
-	netdev_dbg(vinfo->netdev, format, ## arg)
-#define v_err(format, arg...) \
-	netdev_err(vinfo->netdev, format, ## arg)
-#define v_info(format, arg...) \
-	netdev_info(vinfo->netdev, format, ## arg)
-
-/* vnic hfi1 internal functions */
-void hfi1_vnic_setup(struct hfi1_devdata *dd);
-int hfi1_vnic_txreq_init(struct hfi1_devdata *dd);
-void hfi1_vnic_txreq_deinit(struct hfi1_devdata *dd);
-
-void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet);
-void hfi1_vnic_sdma_init(struct hfi1_vnic_vport_info *vinfo);
-bool hfi1_vnic_sdma_write_avail(struct hfi1_vnic_vport_info *vinfo,
-				u8 q_idx);
-
-/* vnic rdma netdev operations */
-struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
-				      u32 port_num,
-				      enum rdma_netdev_t type,
-				      const char *name,
-				      unsigned char name_assign_type,
-				      void (*setup)(struct net_device *));
-int hfi1_vnic_send_dma(struct hfi1_devdata *dd, u8 q_idx,
-		       struct hfi1_vnic_vport_info *vinfo,
-		       struct sk_buff *skb, u64 pbc, u8 plen);
-
-#endif /* _HFI1_VNIC_H */
diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c
deleted file mode 100644
index 16a4c297a8977..0000000000000
--- a/drivers/infiniband/hw/hfi1/vnic_main.c
+++ /dev/null
@@ -1,615 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
-/*
- * Copyright(c) 2017 - 2020 Intel Corporation.
- */
-
-/*
- * This file contains HFI1 support for VNIC functionality
- */
-
-#include <linux/io.h>
-#include <linux/if_vlan.h>
-
-#include "vnic.h"
-#include "netdev.h"
-
-#define HFI_TX_TIMEOUT_MS 1000
-
-#define HFI1_VNIC_RCV_Q_SIZE   1024
-
-#define HFI1_VNIC_UP 0
-
-static DEFINE_SPINLOCK(vport_cntr_lock);
-
-#define SUM_GRP_COUNTERS(stats, qstats, x_grp) do {            \
-		u64 *src64, *dst64;                            \
-		for (src64 = &qstats->x_grp.unicast,           \
-			dst64 = &stats->x_grp.unicast;         \
-			dst64 <= &stats->x_grp.s_1519_max;) {  \
-			*dst64++ += *src64++;                  \
-		}                                              \
-	} while (0)
-
-#define VNIC_MASK (0xFF)
-#define VNIC_ID(val) ((1ull << 24) | ((val) & VNIC_MASK))
-
-/* hfi1_vnic_update_stats - update statistics */
-static void hfi1_vnic_update_stats(struct hfi1_vnic_vport_info *vinfo,
-				   struct opa_vnic_stats *stats)
-{
-	struct net_device *netdev = vinfo->netdev;
-	u8 i;
-
-	/* add tx counters on different queues */
-	for (i = 0; i < vinfo->num_tx_q; i++) {
-		struct opa_vnic_stats *qstats = &vinfo->stats[i];
-		struct rtnl_link_stats64 *qnstats = &vinfo->stats[i].netstats;
-
-		stats->netstats.tx_fifo_errors += qnstats->tx_fifo_errors;
-		stats->netstats.tx_carrier_errors += qnstats->tx_carrier_errors;
-		stats->tx_drop_state += qstats->tx_drop_state;
-		stats->tx_dlid_zero += qstats->tx_dlid_zero;
-
-		SUM_GRP_COUNTERS(stats, qstats, tx_grp);
-		stats->netstats.tx_packets += qnstats->tx_packets;
-		stats->netstats.tx_bytes += qnstats->tx_bytes;
-	}
-
-	/* add rx counters on different queues */
-	for (i = 0; i < vinfo->num_rx_q; i++) {
-		struct opa_vnic_stats *qstats = &vinfo->stats[i];
-		struct rtnl_link_stats64 *qnstats = &vinfo->stats[i].netstats;
-
-		stats->netstats.rx_fifo_errors += qnstats->rx_fifo_errors;
-		stats->netstats.rx_nohandler += qnstats->rx_nohandler;
-		stats->rx_drop_state += qstats->rx_drop_state;
-		stats->rx_oversize += qstats->rx_oversize;
-		stats->rx_runt += qstats->rx_runt;
-
-		SUM_GRP_COUNTERS(stats, qstats, rx_grp);
-		stats->netstats.rx_packets += qnstats->rx_packets;
-		stats->netstats.rx_bytes += qnstats->rx_bytes;
-	}
-
-	stats->netstats.tx_errors = stats->netstats.tx_fifo_errors +
-				    stats->netstats.tx_carrier_errors +
-				    stats->tx_drop_state + stats->tx_dlid_zero;
-	stats->netstats.tx_dropped = stats->netstats.tx_errors;
-
-	stats->netstats.rx_errors = stats->netstats.rx_fifo_errors +
-				    stats->netstats.rx_nohandler +
-				    stats->rx_drop_state + stats->rx_oversize +
-				    stats->rx_runt;
-	stats->netstats.rx_dropped = stats->netstats.rx_errors;
-
-	netdev->stats.tx_packets = stats->netstats.tx_packets;
-	netdev->stats.tx_bytes = stats->netstats.tx_bytes;
-	netdev->stats.tx_fifo_errors = stats->netstats.tx_fifo_errors;
-	netdev->stats.tx_carrier_errors = stats->netstats.tx_carrier_errors;
-	netdev->stats.tx_errors = stats->netstats.tx_errors;
-	netdev->stats.tx_dropped = stats->netstats.tx_dropped;
-
-	netdev->stats.rx_packets = stats->netstats.rx_packets;
-	netdev->stats.rx_bytes = stats->netstats.rx_bytes;
-	netdev->stats.rx_fifo_errors = stats->netstats.rx_fifo_errors;
-	netdev->stats.multicast = stats->rx_grp.mcastbcast;
-	netdev->stats.rx_length_errors = stats->rx_oversize + stats->rx_runt;
-	netdev->stats.rx_errors = stats->netstats.rx_errors;
-	netdev->stats.rx_dropped = stats->netstats.rx_dropped;
-}
-
-/* update_len_counters - update pkt's len histogram counters */
-static inline void update_len_counters(struct opa_vnic_grp_stats *grp,
-				       int len)
-{
-	/* account for 4 byte FCS */
-	if (len >= 1515)
-		grp->s_1519_max++;
-	else if (len >= 1020)
-		grp->s_1024_1518++;
-	else if (len >= 508)
-		grp->s_512_1023++;
-	else if (len >= 252)
-		grp->s_256_511++;
-	else if (len >= 124)
-		grp->s_128_255++;
-	else if (len >= 61)
-		grp->s_65_127++;
-	else
-		grp->s_64++;
-}
-
-/* hfi1_vnic_update_tx_counters - update transmit counters */
-static void hfi1_vnic_update_tx_counters(struct hfi1_vnic_vport_info *vinfo,
-					 u8 q_idx, struct sk_buff *skb, int err)
-{
-	struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb);
-	struct opa_vnic_stats *stats = &vinfo->stats[q_idx];
-	struct opa_vnic_grp_stats *tx_grp = &stats->tx_grp;
-	u16 vlan_tci;
-
-	stats->netstats.tx_packets++;
-	stats->netstats.tx_bytes += skb->len + ETH_FCS_LEN;
-
-	update_len_counters(tx_grp, skb->len);
-
-	/* rest of the counts are for good packets only */
-	if (unlikely(err))
-		return;
-
-	if (is_multicast_ether_addr(mac_hdr->h_dest))
-		tx_grp->mcastbcast++;
-	else
-		tx_grp->unicast++;
-
-	if (!__vlan_get_tag(skb, &vlan_tci))
-		tx_grp->vlan++;
-	else
-		tx_grp->untagged++;
-}
-
-/* hfi1_vnic_update_rx_counters - update receive counters */
-static void hfi1_vnic_update_rx_counters(struct hfi1_vnic_vport_info *vinfo,
-					 u8 q_idx, struct sk_buff *skb, int err)
-{
-	struct ethhdr *mac_hdr = (struct ethhdr *)skb->data;
-	struct opa_vnic_stats *stats = &vinfo->stats[q_idx];
-	struct opa_vnic_grp_stats *rx_grp = &stats->rx_grp;
-	u16 vlan_tci;
-
-	stats->netstats.rx_packets++;
-	stats->netstats.rx_bytes += skb->len + ETH_FCS_LEN;
-
-	update_len_counters(rx_grp, skb->len);
-
-	/* rest of the counts are for good packets only */
-	if (unlikely(err))
-		return;
-
-	if (is_multicast_ether_addr(mac_hdr->h_dest))
-		rx_grp->mcastbcast++;
-	else
-		rx_grp->unicast++;
-
-	if (!__vlan_get_tag(skb, &vlan_tci))
-		rx_grp->vlan++;
-	else
-		rx_grp->untagged++;
-}
-
-/* This function is overloaded for opa_vnic specific implementation */
-static void hfi1_vnic_get_stats64(struct net_device *netdev,
-				  struct rtnl_link_stats64 *stats)
-{
-	struct opa_vnic_stats *vstats = (struct opa_vnic_stats *)stats;
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-
-	hfi1_vnic_update_stats(vinfo, vstats);
-}
-
-static u64 create_bypass_pbc(u32 vl, u32 dw_len)
-{
-	u64 pbc;
-
-	pbc = ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
-		| PBC_INSERT_BYPASS_ICRC | PBC_CREDIT_RETURN
-		| PBC_PACKET_BYPASS
-		| ((vl & PBC_VL_MASK) << PBC_VL_SHIFT)
-		| (dw_len & PBC_LENGTH_DWS_MASK) << PBC_LENGTH_DWS_SHIFT;
-
-	return pbc;
-}
-
-/* hfi1_vnic_maybe_stop_tx - stop tx queue if required */
-static void hfi1_vnic_maybe_stop_tx(struct hfi1_vnic_vport_info *vinfo,
-				    u8 q_idx)
-{
-	netif_stop_subqueue(vinfo->netdev, q_idx);
-	if (!hfi1_vnic_sdma_write_avail(vinfo, q_idx))
-		return;
-
-	netif_start_subqueue(vinfo->netdev, q_idx);
-}
-
-static netdev_tx_t hfi1_netdev_start_xmit(struct sk_buff *skb,
-					  struct net_device *netdev)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-	u8 pad_len, q_idx = skb->queue_mapping;
-	struct hfi1_devdata *dd = vinfo->dd;
-	struct opa_vnic_skb_mdata *mdata;
-	u32 pkt_len, total_len;
-	int err = -EINVAL;
-	u64 pbc;
-
-	v_dbg("xmit: queue %d skb len %d\n", q_idx, skb->len);
-	if (unlikely(!netif_oper_up(netdev))) {
-		vinfo->stats[q_idx].tx_drop_state++;
-		goto tx_finish;
-	}
-
-	/* take out meta data */
-	mdata = (struct opa_vnic_skb_mdata *)skb->data;
-	skb_pull(skb, sizeof(*mdata));
-	if (unlikely(mdata->flags & OPA_VNIC_SKB_MDATA_ENCAP_ERR)) {
-		vinfo->stats[q_idx].tx_dlid_zero++;
-		goto tx_finish;
-	}
-
-	/* add tail padding (for 8 bytes size alignment) and icrc */
-	pad_len = -(skb->len + OPA_VNIC_ICRC_TAIL_LEN) & 0x7;
-	pad_len += OPA_VNIC_ICRC_TAIL_LEN;
-
-	/*
-	 * pkt_len is how much data we have to write, includes header and data.
-	 * total_len is length of the packet in Dwords plus the PBC should not
-	 * include the CRC.
-	 */
-	pkt_len = (skb->len + pad_len) >> 2;
-	total_len = pkt_len + 2; /* PBC + packet */
-
-	pbc = create_bypass_pbc(mdata->vl, total_len);
-
-	skb_get(skb);
-	v_dbg("pbc 0x%016llX len %d pad_len %d\n", pbc, skb->len, pad_len);
-	err = dd->process_vnic_dma_send(dd, q_idx, vinfo, skb, pbc, pad_len);
-	if (unlikely(err)) {
-		if (err == -ENOMEM)
-			vinfo->stats[q_idx].netstats.tx_fifo_errors++;
-		else if (err != -EBUSY)
-			vinfo->stats[q_idx].netstats.tx_carrier_errors++;
-	}
-	/* remove the header before updating tx counters */
-	skb_pull(skb, OPA_VNIC_HDR_LEN);
-
-	if (unlikely(err == -EBUSY)) {
-		hfi1_vnic_maybe_stop_tx(vinfo, q_idx);
-		dev_kfree_skb_any(skb);
-		return NETDEV_TX_BUSY;
-	}
-
-tx_finish:
-	/* update tx counters */
-	hfi1_vnic_update_tx_counters(vinfo, q_idx, skb, err);
-	dev_kfree_skb_any(skb);
-	return NETDEV_TX_OK;
-}
-
-static u16 hfi1_vnic_select_queue(struct net_device *netdev,
-				  struct sk_buff *skb,
-				  struct net_device *sb_dev)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-	struct opa_vnic_skb_mdata *mdata;
-	struct sdma_engine *sde;
-
-	mdata = (struct opa_vnic_skb_mdata *)skb->data;
-	sde = sdma_select_engine_vl(vinfo->dd, mdata->entropy, mdata->vl);
-	return sde->this_idx;
-}
-
-/* hfi1_vnic_decap_skb - strip OPA header from the skb (ethernet) packet */
-static inline int hfi1_vnic_decap_skb(struct hfi1_vnic_rx_queue *rxq,
-				      struct sk_buff *skb)
-{
-	struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
-	int max_len = vinfo->netdev->mtu + VLAN_ETH_HLEN;
-	int rc = -EFAULT;
-
-	skb_pull(skb, OPA_VNIC_HDR_LEN);
-
-	/* Validate Packet length */
-	if (unlikely(skb->len > max_len))
-		vinfo->stats[rxq->idx].rx_oversize++;
-	else if (unlikely(skb->len < ETH_ZLEN))
-		vinfo->stats[rxq->idx].rx_runt++;
-	else
-		rc = 0;
-	return rc;
-}
-
-static struct hfi1_vnic_vport_info *get_vnic_port(struct hfi1_devdata *dd,
-						  int vesw_id)
-{
-	int vnic_id = VNIC_ID(vesw_id);
-
-	return hfi1_netdev_get_data(dd, vnic_id);
-}
-
-static struct hfi1_vnic_vport_info *get_first_vnic_port(struct hfi1_devdata *dd)
-{
-	struct hfi1_vnic_vport_info *vinfo;
-	int next_id = VNIC_ID(0);
-
-	vinfo = hfi1_netdev_get_first_data(dd, &next_id);
-
-	if (next_id > VNIC_ID(VNIC_MASK))
-		return NULL;
-
-	return vinfo;
-}
-
-void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
-{
-	struct hfi1_devdata *dd = packet->rcd->dd;
-	struct hfi1_vnic_vport_info *vinfo = NULL;
-	struct hfi1_vnic_rx_queue *rxq;
-	struct sk_buff *skb;
-	int l4_type, vesw_id = -1, rc;
-	u8 q_idx;
-	unsigned char *pad_info;
-
-	l4_type = hfi1_16B_get_l4(packet->ebuf);
-	if (likely(l4_type == OPA_16B_L4_ETHR)) {
-		vesw_id = HFI1_VNIC_GET_VESWID(packet->ebuf);
-		vinfo = get_vnic_port(dd, vesw_id);
-
-		/*
-		 * In case of invalid vesw id, count the error on
-		 * the first available vport.
-		 */
-		if (unlikely(!vinfo)) {
-			struct hfi1_vnic_vport_info *vinfo_tmp;
-
-			vinfo_tmp = get_first_vnic_port(dd);
-			if (vinfo_tmp) {
-				spin_lock(&vport_cntr_lock);
-				vinfo_tmp->stats[0].netstats.rx_nohandler++;
-				spin_unlock(&vport_cntr_lock);
-			}
-		}
-	}
-
-	if (unlikely(!vinfo)) {
-		dd_dev_warn(dd, "vnic rcv err: l4 %d vesw id %d ctx %d\n",
-			    l4_type, vesw_id, packet->rcd->ctxt);
-		return;
-	}
-
-	q_idx = packet->rcd->vnic_q_idx;
-	rxq = &vinfo->rxq[q_idx];
-	if (unlikely(!netif_oper_up(vinfo->netdev))) {
-		vinfo->stats[q_idx].rx_drop_state++;
-		return;
-	}
-
-	skb = netdev_alloc_skb(vinfo->netdev, packet->tlen);
-	if (unlikely(!skb)) {
-		vinfo->stats[q_idx].netstats.rx_fifo_errors++;
-		return;
-	}
-
-	memcpy(skb->data, packet->ebuf, packet->tlen);
-	skb_put(skb, packet->tlen);
-
-	pad_info = skb->data + skb->len - 1;
-	skb_trim(skb, (skb->len - OPA_VNIC_ICRC_TAIL_LEN -
-		       ((*pad_info) & 0x7)));
-
-	rc = hfi1_vnic_decap_skb(rxq, skb);
-
-	/* update rx counters */
-	hfi1_vnic_update_rx_counters(vinfo, rxq->idx, skb, rc);
-	if (unlikely(rc)) {
-		dev_kfree_skb_any(skb);
-		return;
-	}
-
-	skb_checksum_none_assert(skb);
-	skb->protocol = eth_type_trans(skb, rxq->netdev);
-
-	napi_gro_receive(&rxq->napi, skb);
-}
-
-static int hfi1_vnic_up(struct hfi1_vnic_vport_info *vinfo)
-{
-	struct hfi1_devdata *dd = vinfo->dd;
-	struct net_device *netdev = vinfo->netdev;
-	int rc;
-
-	/* ensure virtual eth switch id is valid */
-	if (!vinfo->vesw_id)
-		return -EINVAL;
-
-	rc = hfi1_netdev_add_data(dd, VNIC_ID(vinfo->vesw_id), vinfo);
-	if (rc < 0)
-		return rc;
-
-	rc = hfi1_netdev_rx_init(dd);
-	if (rc)
-		goto err_remove;
-
-	netif_carrier_on(netdev);
-	netif_tx_start_all_queues(netdev);
-	set_bit(HFI1_VNIC_UP, &vinfo->flags);
-
-	return 0;
-
-err_remove:
-	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo->vesw_id));
-	return rc;
-}
-
-static void hfi1_vnic_down(struct hfi1_vnic_vport_info *vinfo)
-{
-	struct hfi1_devdata *dd = vinfo->dd;
-
-	clear_bit(HFI1_VNIC_UP, &vinfo->flags);
-	netif_carrier_off(vinfo->netdev);
-	netif_tx_disable(vinfo->netdev);
-	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo->vesw_id));
-
-	hfi1_netdev_rx_destroy(dd);
-}
-
-static int hfi1_netdev_open(struct net_device *netdev)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-	int rc;
-
-	mutex_lock(&vinfo->lock);
-	rc = hfi1_vnic_up(vinfo);
-	mutex_unlock(&vinfo->lock);
-	return rc;
-}
-
-static int hfi1_netdev_close(struct net_device *netdev)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-
-	mutex_lock(&vinfo->lock);
-	if (test_bit(HFI1_VNIC_UP, &vinfo->flags))
-		hfi1_vnic_down(vinfo);
-	mutex_unlock(&vinfo->lock);
-	return 0;
-}
-
-static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)
-{
-	struct hfi1_devdata *dd = vinfo->dd;
-	int rc = 0;
-
-	mutex_lock(&hfi1_mutex);
-	if (!dd->vnic_num_vports) {
-		rc = hfi1_vnic_txreq_init(dd);
-		if (rc)
-			goto txreq_fail;
-	}
-
-	rc = hfi1_netdev_rx_init(dd);
-	if (rc) {
-		dd_dev_err(dd, "Unable to initialize netdev contexts\n");
-		goto alloc_fail;
-	}
-
-	hfi1_init_vnic_rsm(dd);
-
-	dd->vnic_num_vports++;
-	hfi1_vnic_sdma_init(vinfo);
-
-alloc_fail:
-	if (!dd->vnic_num_vports)
-		hfi1_vnic_txreq_deinit(dd);
-txreq_fail:
-	mutex_unlock(&hfi1_mutex);
-	return rc;
-}
-
-static void hfi1_vnic_deinit(struct hfi1_vnic_vport_info *vinfo)
-{
-	struct hfi1_devdata *dd = vinfo->dd;
-
-	mutex_lock(&hfi1_mutex);
-	if (--dd->vnic_num_vports == 0) {
-		hfi1_deinit_vnic_rsm(dd);
-		hfi1_vnic_txreq_deinit(dd);
-	}
-	mutex_unlock(&hfi1_mutex);
-	hfi1_netdev_rx_destroy(dd);
-}
-
-static void hfi1_vnic_set_vesw_id(struct net_device *netdev, int id)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-	bool reopen = false;
-
-	/*
-	 * If vesw_id is being changed, and if the vnic port is up,
-	 * reset the vnic port to ensure new vesw_id gets picked up
-	 */
-	if (id != vinfo->vesw_id) {
-		mutex_lock(&vinfo->lock);
-		if (test_bit(HFI1_VNIC_UP, &vinfo->flags)) {
-			hfi1_vnic_down(vinfo);
-			reopen = true;
-		}
-
-		vinfo->vesw_id = id;
-		if (reopen)
-			hfi1_vnic_up(vinfo);
-
-		mutex_unlock(&vinfo->lock);
-	}
-}
-
-/* netdev ops */
-static const struct net_device_ops hfi1_netdev_ops = {
-	.ndo_open = hfi1_netdev_open,
-	.ndo_stop = hfi1_netdev_close,
-	.ndo_start_xmit = hfi1_netdev_start_xmit,
-	.ndo_select_queue = hfi1_vnic_select_queue,
-	.ndo_get_stats64 = hfi1_vnic_get_stats64,
-};
-
-static void hfi1_vnic_free_rn(struct net_device *netdev)
-{
-	struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
-
-	hfi1_vnic_deinit(vinfo);
-	mutex_destroy(&vinfo->lock);
-	free_netdev(netdev);
-}
-
-struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
-				      u32 port_num,
-				      enum rdma_netdev_t type,
-				      const char *name,
-				      unsigned char name_assign_type,
-				      void (*setup)(struct net_device *))
-{
-	struct hfi1_devdata *dd = dd_from_ibdev(device);
-	struct hfi1_vnic_vport_info *vinfo;
-	struct net_device *netdev;
-	struct rdma_netdev *rn;
-	int i, size, rc;
-
-	if (!dd->num_netdev_contexts)
-		return ERR_PTR(-ENOMEM);
-
-	if (!port_num || (port_num > dd->num_pports))
-		return ERR_PTR(-EINVAL);
-
-	if (type != RDMA_NETDEV_OPA_VNIC)
-		return ERR_PTR(-EOPNOTSUPP);
-
-	size = sizeof(struct opa_vnic_rdma_netdev) + sizeof(*vinfo);
-	netdev = alloc_netdev_mqs(size, name, name_assign_type, setup,
-				  chip_sdma_engines(dd),
-				  dd->num_netdev_contexts);
-	if (!netdev)
-		return ERR_PTR(-ENOMEM);
-
-	rn = netdev_priv(netdev);
-	vinfo = opa_vnic_dev_priv(netdev);
-	vinfo->dd = dd;
-	vinfo->num_tx_q = chip_sdma_engines(dd);
-	vinfo->num_rx_q = dd->num_netdev_contexts;
-	vinfo->netdev = netdev;
-	rn->free_rdma_netdev = hfi1_vnic_free_rn;
-	rn->set_id = hfi1_vnic_set_vesw_id;
-
-	netdev->features = NETIF_F_HIGHDMA | NETIF_F_SG;
-	netdev->hw_features = netdev->features;
-	netdev->vlan_features = netdev->features;
-	netdev->watchdog_timeo = msecs_to_jiffies(HFI_TX_TIMEOUT_MS);
-	netdev->netdev_ops = &hfi1_netdev_ops;
-	mutex_init(&vinfo->lock);
-
-	for (i = 0; i < vinfo->num_rx_q; i++) {
-		struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
-
-		rxq->idx = i;
-		rxq->vinfo = vinfo;
-		rxq->netdev = netdev;
-	}
-
-	rc = hfi1_vnic_init(vinfo);
-	if (rc)
-		goto init_fail;
-
-	return netdev;
-init_fail:
-	mutex_destroy(&vinfo->lock);
-	free_netdev(netdev);
-	return ERR_PTR(rc);
-}
diff --git a/drivers/infiniband/hw/hfi1/vnic_sdma.c b/drivers/infiniband/hw/hfi1/vnic_sdma.c
deleted file mode 100644
index 6caf01ba0bca0..0000000000000
--- a/drivers/infiniband/hw/hfi1/vnic_sdma.c
+++ /dev/null
@@ -1,282 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
-/*
- * Copyright(c) 2017 - 2018 Intel Corporation.
- */
-
-/*
- * This file contains HFI1 support for VNIC SDMA functionality
- */
-
-#include "sdma.h"
-#include "vnic.h"
-
-#define HFI1_VNIC_SDMA_Q_ACTIVE   BIT(0)
-#define HFI1_VNIC_SDMA_Q_DEFERRED BIT(1)
-
-#define HFI1_VNIC_TXREQ_NAME_LEN   32
-#define HFI1_VNIC_SDMA_DESC_WTRMRK 64
-
-/*
- * struct vnic_txreq - VNIC transmit descriptor
- * @txreq: sdma transmit request
- * @sdma: vnic sdma pointer
- * @skb: skb to send
- * @pad: pad buffer
- * @plen: pad length
- * @pbc_val: pbc value
- */
-struct vnic_txreq {
-	struct sdma_txreq       txreq;
-	struct hfi1_vnic_sdma   *sdma;
-
-	struct sk_buff         *skb;
-	unsigned char           pad[HFI1_VNIC_MAX_PAD];
-	u16                     plen;
-	__le64                  pbc_val;
-};
-
-static void vnic_sdma_complete(struct sdma_txreq *txreq,
-			       int status)
-{
-	struct vnic_txreq *tx = container_of(txreq, struct vnic_txreq, txreq);
-	struct hfi1_vnic_sdma *vnic_sdma = tx->sdma;
-
-	sdma_txclean(vnic_sdma->dd, txreq);
-	dev_kfree_skb_any(tx->skb);
-	kmem_cache_free(vnic_sdma->dd->vnic.txreq_cache, tx);
-}
-
-static noinline int build_vnic_ulp_payload(struct sdma_engine *sde,
-					   struct vnic_txreq *tx)
-{
-	int i, ret = 0;
-
-	ret = sdma_txadd_kvaddr(
-		sde->dd,
-		&tx->txreq,
-		tx->skb->data,
-		skb_headlen(tx->skb));
-	if (unlikely(ret))
-		goto bail_txadd;
-
-	for (i = 0; i < skb_shinfo(tx->skb)->nr_frags; i++) {
-		skb_frag_t *frag = &skb_shinfo(tx->skb)->frags[i];
-
-		/* combine physically continuous fragments later? */
-		ret = sdma_txadd_page(sde->dd,
-				      &tx->txreq,
-				      skb_frag_page(frag),
-				      skb_frag_off(frag),
-				      skb_frag_size(frag),
-				      NULL, NULL, NULL);
-		if (unlikely(ret))
-			goto bail_txadd;
-	}
-
-	if (tx->plen)
-		ret = sdma_txadd_kvaddr(sde->dd, &tx->txreq,
-					tx->pad + HFI1_VNIC_MAX_PAD - tx->plen,
-					tx->plen);
-
-bail_txadd:
-	return ret;
-}
-
-static int build_vnic_tx_desc(struct sdma_engine *sde,
-			      struct vnic_txreq *tx,
-			      u64 pbc)
-{
-	int ret = 0;
-	u16 hdrbytes = 2 << 2;  /* PBC */
-
-	ret = sdma_txinit_ahg(
-		&tx->txreq,
-		0,
-		hdrbytes + tx->skb->len + tx->plen,
-		0,
-		0,
-		NULL,
-		0,
-		vnic_sdma_complete);
-	if (unlikely(ret))
-		goto bail_txadd;
-
-	/* add pbc */
-	tx->pbc_val = cpu_to_le64(pbc);
-	ret = sdma_txadd_kvaddr(
-		sde->dd,
-		&tx->txreq,
-		&tx->pbc_val,
-		hdrbytes);
-	if (unlikely(ret))
-		goto bail_txadd;
-
-	/* add the ulp payload */
-	ret = build_vnic_ulp_payload(sde, tx);
-bail_txadd:
-	return ret;
-}
-
-/* setup the last plen bypes of pad */
-static inline void hfi1_vnic_update_pad(unsigned char *pad, u8 plen)
-{
-	pad[HFI1_VNIC_MAX_PAD - 1] = plen - OPA_VNIC_ICRC_TAIL_LEN;
-}
-
-int hfi1_vnic_send_dma(struct hfi1_devdata *dd, u8 q_idx,
-		       struct hfi1_vnic_vport_info *vinfo,
-		       struct sk_buff *skb, u64 pbc, u8 plen)
-{
-	struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[q_idx];
-	struct sdma_engine *sde = vnic_sdma->sde;
-	struct vnic_txreq *tx;
-	int ret = -ECOMM;
-
-	if (unlikely(READ_ONCE(vnic_sdma->state) != HFI1_VNIC_SDMA_Q_ACTIVE))
-		goto tx_err;
-
-	if (unlikely(!sde || !sdma_running(sde)))
-		goto tx_err;
-
-	tx = kmem_cache_alloc(dd->vnic.txreq_cache, GFP_ATOMIC);
-	if (unlikely(!tx)) {
-		ret = -ENOMEM;
-		goto tx_err;
-	}
-
-	tx->sdma = vnic_sdma;
-	tx->skb = skb;
-	hfi1_vnic_update_pad(tx->pad, plen);
-	tx->plen = plen;
-	ret = build_vnic_tx_desc(sde, tx, pbc);
-	if (unlikely(ret))
-		goto free_desc;
-
-	ret = sdma_send_txreq(sde, iowait_get_ib_work(&vnic_sdma->wait),
-			      &tx->txreq, vnic_sdma->pkts_sent);
-	/* When -ECOMM, sdma callback will be called with ABORT status */
-	if (unlikely(ret && unlikely(ret != -ECOMM)))
-		goto free_desc;
-
-	if (!ret) {
-		vnic_sdma->pkts_sent = true;
-		iowait_starve_clear(vnic_sdma->pkts_sent, &vnic_sdma->wait);
-	}
-	return ret;
-
-free_desc:
-	sdma_txclean(dd, &tx->txreq);
-	kmem_cache_free(dd->vnic.txreq_cache, tx);
-tx_err:
-	if (ret != -EBUSY)
-		dev_kfree_skb_any(skb);
-	else
-		vnic_sdma->pkts_sent = false;
-	return ret;
-}
-
-/*
- * hfi1_vnic_sdma_sleep - vnic sdma sleep function
- *
- * This function gets called from sdma_send_txreq() when there are not enough
- * sdma descriptors available to send the packet. It adds Tx queue's wait
- * structure to sdma engine's dmawait list to be woken up when descriptors
- * become available.
- */
-static int hfi1_vnic_sdma_sleep(struct sdma_engine *sde,
-				struct iowait_work *wait,
-				struct sdma_txreq *txreq,
-				uint seq,
-				bool pkts_sent)
-{
-	struct hfi1_vnic_sdma *vnic_sdma =
-		container_of(wait->iow, struct hfi1_vnic_sdma, wait);
-
-	write_seqlock(&sde->waitlock);
-	if (sdma_progress(sde, seq, txreq)) {
-		write_sequnlock(&sde->waitlock);
-		return -EAGAIN;
-	}
-
-	vnic_sdma->state = HFI1_VNIC_SDMA_Q_DEFERRED;
-	if (list_empty(&vnic_sdma->wait.list)) {
-		iowait_get_priority(wait->iow);
-		iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
-	}
-	write_sequnlock(&sde->waitlock);
-	return -EBUSY;
-}
-
-/*
- * hfi1_vnic_sdma_wakeup - vnic sdma wakeup function
- *
- * This function gets called when SDMA descriptors becomes available and Tx
- * queue's wait structure was previously added to sdma engine's dmawait list.
- * It notifies the upper driver about Tx queue wakeup.
- */
-static void hfi1_vnic_sdma_wakeup(struct iowait *wait, int reason)
-{
-	struct hfi1_vnic_sdma *vnic_sdma =
-		container_of(wait, struct hfi1_vnic_sdma, wait);
-	struct hfi1_vnic_vport_info *vinfo = vnic_sdma->vinfo;
-
-	vnic_sdma->state = HFI1_VNIC_SDMA_Q_ACTIVE;
-	if (__netif_subqueue_stopped(vinfo->netdev, vnic_sdma->q_idx))
-		netif_wake_subqueue(vinfo->netdev, vnic_sdma->q_idx);
-};
-
-inline bool hfi1_vnic_sdma_write_avail(struct hfi1_vnic_vport_info *vinfo,
-				       u8 q_idx)
-{
-	struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[q_idx];
-
-	return (READ_ONCE(vnic_sdma->state) == HFI1_VNIC_SDMA_Q_ACTIVE);
-}
-
-void hfi1_vnic_sdma_init(struct hfi1_vnic_vport_info *vinfo)
-{
-	int i;
-
-	for (i = 0; i < vinfo->num_tx_q; i++) {
-		struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[i];
-
-		iowait_init(&vnic_sdma->wait, 0, NULL, NULL,
-			    hfi1_vnic_sdma_sleep,
-			    hfi1_vnic_sdma_wakeup, NULL, NULL);
-		vnic_sdma->sde = &vinfo->dd->per_sdma[i];
-		vnic_sdma->dd = vinfo->dd;
-		vnic_sdma->vinfo = vinfo;
-		vnic_sdma->q_idx = i;
-		vnic_sdma->state = HFI1_VNIC_SDMA_Q_ACTIVE;
-
-		/* Add a free descriptor watermark for wakeups */
-		if (vnic_sdma->sde->descq_cnt > HFI1_VNIC_SDMA_DESC_WTRMRK) {
-			struct iowait_work *work;
-
-			INIT_LIST_HEAD(&vnic_sdma->stx.list);
-			vnic_sdma->stx.num_desc = HFI1_VNIC_SDMA_DESC_WTRMRK;
-			work = iowait_get_ib_work(&vnic_sdma->wait);
-			list_add_tail(&vnic_sdma->stx.list, &work->tx_head);
-		}
-	}
-}
-
-int hfi1_vnic_txreq_init(struct hfi1_devdata *dd)
-{
-	char buf[HFI1_VNIC_TXREQ_NAME_LEN];
-
-	snprintf(buf, sizeof(buf), "hfi1_%u_vnic_txreq_cache", dd->unit);
-	dd->vnic.txreq_cache = kmem_cache_create(buf,
-						 sizeof(struct vnic_txreq),
-						 0, SLAB_HWCACHE_ALIGN,
-						 NULL);
-	if (!dd->vnic.txreq_cache)
-		return -ENOMEM;
-	return 0;
-}
-
-void hfi1_vnic_txreq_deinit(struct hfi1_devdata *dd)
-{
-	kmem_cache_destroy(dd->vnic.txreq_cache);
-	dd->vnic.txreq_cache = NULL;
-}
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
index 8e802f118bc92..142c86f462fa9 100644
--- a/drivers/infiniband/hw/hns/hns_roce_alloc.c
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -32,7 +32,7 @@
  */
 
 #include <linux/vmalloc.h>
-#include <rdma/ib_umem.h>
+#include <rdma/iter.h>
 #include "hns_roce_device.h"
 
 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 4eaaedcc76521..e7c9e30ad2d8b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -771,9 +771,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 			unsigned long num_bt_l1;
 
 			num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
-			table->bt_l1 = kcalloc(num_bt_l1,
-					       sizeof(*table->bt_l1),
-					       GFP_KERNEL);
+			table->bt_l1 = kzalloc_objs(*table->bt_l1, num_bt_l1);
 			if (!table->bt_l1)
 				goto err_kcalloc_bt_l1;
 
@@ -786,8 +784,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 
 		if (check_whether_bt_num_2(type, hop_num) ||
 			check_whether_bt_num_3(type, hop_num)) {
-			table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
-					       GFP_KERNEL);
+			table->bt_l0 = kzalloc_objs(*table->bt_l0, num_bt_l0);
 			if (!table->bt_l0)
 				goto err_kcalloc_bt_l0;
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 5f7ea6c166446..6a2dff4bd2d0f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -1023,21 +1023,16 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
 			     struct hns_roce_qp *hr_qp)
 {
-	struct ib_device *ibdev = &hr_dev->ib_dev;
-	u64 *sq_wrid = NULL;
-	u64 *rq_wrid = NULL;
+	u64 *sq_wrid, *rq_wrid = NULL;
 	int ret;
 
-	sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
-	if (!sq_wrid) {
-		ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
+	sq_wrid = kzalloc_objs(*sq_wrid, hr_qp->sq.wqe_cnt);
+	if (!sq_wrid)
 		return -ENOMEM;
-	}
 
 	if (hr_qp->rq.wqe_cnt) {
-		rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
+		rq_wrid = kzalloc_objs(*rq_wrid, hr_qp->rq.wqe_cnt);
 		if (!rq_wrid) {
-			ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
 			ret = -ENOMEM;
 			goto err_sq;
 		}
diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c
index 4842931f5316e..a5671da3db648 100644
--- a/drivers/infiniband/hw/ionic/ionic_controlpath.c
+++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c
@@ -508,6 +508,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev,
 {
 	const struct ib_global_route *grh;
 	enum rdma_network_type net;
+	u8 smac[ETH_ALEN];
 	u16 vlan;
 	int rc;
 
@@ -518,7 +519,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev,
 
 	grh = rdma_ah_read_grh(attr);
 
-	rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, &hdr->eth.smac_h[0]);
+	rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac);
 	if (rc)
 		return rc;
 
@@ -536,6 +537,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev,
 	if (rc)
 		return rc;
 
+	ether_addr_copy(hdr->eth.smac_h, smac);
 	ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac);
 
 	if (net == RDMA_NETWORK_IPV4) {
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h
index 82fda1e3cdb65..63828240d6590 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.h
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h
@@ -4,9 +4,9 @@
 #ifndef _IONIC_IBDEV_H_
 #define _IONIC_IBDEV_H_
 
-#include <rdma/ib_umem.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_pack.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include <rdma/ionic-abi.h>
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index f4ae530f56db0..6e0466ce83d18 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -1033,7 +1033,7 @@ static int irdma_create_cqp(struct irdma_pci_f *rf)
 	if (!cqp->cqp_requests)
 		return -ENOMEM;
 
-	cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
+	cqp->scratch_array = kzalloc_objs(*cqp->scratch_array, sqsize);
 	if (!cqp->scratch_array) {
 		status = -ENOMEM;
 		goto err_scratch;
@@ -1942,7 +1942,7 @@ int irdma_rt_init_hw(struct irdma_device *iwdev,
 	if (status)
 		return status;
 
-	stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
+	stats_info.pestat = kzalloc_obj(*stats_info.pestat);
 	if (!stats_info.pestat) {
 		irdma_cleanup_cm_core(&iwdev->cm_core);
 		return -ENOMEM;
diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
index d320d1a228b33..3d49bd57bae7c 100644
--- a/drivers/infiniband/hw/irdma/main.h
+++ b/drivers/infiniband/hw/irdma/main.h
@@ -37,8 +37,8 @@
 #include <rdma/rdma_cm.h>
 #include <rdma/iw_cm.h>
 #include <rdma/ib_user_verbs.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_cache.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 #include "osdep.h"
 #include "defs.h"
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 7251cd7a21471..d076371f121c9 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -3590,6 +3590,36 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 	return ERR_PTR(err);
 }
 
+static int irdma_hwdereg_mr(struct ib_mr *ib_mr);
+
+static void irdma_umem_dmabuf_revoke(void *priv)
+{
+	/* priv is guaranteed to be valid any time this callback is invoked
+	 * because we do not set the callback until after successful iwmr
+	 * allocation and initialization.
+	 */
+	struct irdma_mr *iwmr = priv;
+	int err;
+
+	/* Invalidate the key in hardware. This does not actually release the
+	 * key for potential reuse - that only occurs when the region is fully
+	 * deregistered.
+	 *
+	 * The irdma_hwdereg_mr call is a no-op if the region is not currently
+	 * registered with hardware.
+	 */
+	err = irdma_hwdereg_mr(&iwmr->ibmr);
+	if (err) {
+		struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
+
+		ibdev_err(&iwdev->ibdev, "dmabuf mr revoke failed %d", err);
+		if (!iwdev->rf->reset) {
+			iwdev->rf->reset = true;
+			iwdev->rf->gen_ops.request_reset(iwdev->rf);
+		}
+	}
+}
+
 static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
 					      u64 len, u64 virt,
 					      int fd, int access,
@@ -3607,7 +3637,9 @@ static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
 		return ERR_PTR(-EINVAL);
 
-	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd, access);
+	umem_dmabuf =
+		ib_umem_dmabuf_get_pinned_revocable_and_lock(pd->device, start,
+							     len, fd, access);
 	if (IS_ERR(umem_dmabuf)) {
 		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%pe]\n",
 			  umem_dmabuf);
@@ -3624,12 +3656,20 @@ static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
 	if (err)
 		goto err_iwmr;
 
+	ib_umem_dmabuf_set_revoke_locked(umem_dmabuf, irdma_umem_dmabuf_revoke,
+					 iwmr);
+	ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
 	return &iwmr->ibmr;
 
 err_iwmr:
 	irdma_free_iwmr(iwmr);
 
 err_release:
+	ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
+
+	/* Will result in a call to revoke, but driver callback is not set and
+	 * is therefore skipped.
+	 */
 	ib_umem_release(&umem_dmabuf->umem);
 
 	return ERR_PTR(err);
@@ -3723,6 +3763,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
 
 err:
 	ib_umem_release(region);
+	iwmr->region = NULL;
 	return err;
 }
 
@@ -3749,6 +3790,8 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
 	struct irdma_device *iwdev = to_iwdev(ib_mr->device);
 	struct irdma_mr *iwmr = to_iwmr(ib_mr);
 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+	bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf;
+	struct ib_umem_dmabuf *umem_dmabuf;
 	int ret;
 
 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
@@ -3757,9 +3800,26 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
 	if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
 		return ERR_PTR(-EOPNOTSUPP);
 
+	if (dmabuf_revocable) {
+		umem_dmabuf = to_ib_umem_dmabuf(iwmr->region);
+
+		ib_umem_dmabuf_revoke_lock(umem_dmabuf);
+
+		/* If the dmabuf has been revoked, it means that the region has
+		 * been invalidated in HW. We must not allow it to become valid
+		 * again unless the user is requesting a change in translation
+		 * which will end up dropping the umem dmabuf and allocating an
+		 * entirely new umem anyway.
+		 */
+		if (umem_dmabuf->revoked && !(flags & IB_MR_REREG_TRANS)) {
+			ret = -EINVAL;
+			goto err_unlock;
+		}
+	}
+
 	ret = irdma_hwdereg_mr(ib_mr);
 	if (ret)
-		return ERR_PTR(ret);
+		goto err_unlock;
 
 	if (flags & IB_MR_REREG_ACCESS)
 		iwmr->access = new_access;
@@ -3775,18 +3835,28 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
 					&iwpbl->pble_alloc);
 			iwpbl->pbl_allocated = false;
 		}
+
+		if (dmabuf_revocable) {
+			/* Must unlock before release to prevent deadlock */
+			ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
+			dmabuf_revocable = false;
+		}
+
 		if (iwmr->region) {
 			ib_umem_release(iwmr->region);
 			iwmr->region = NULL;
 		}
 
 		ret = irdma_rereg_mr_trans(iwmr, start, len, virt);
-	} else
+	} else {
 		ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
-	if (ret)
-		return ERR_PTR(ret);
+	}
 
-	return NULL;
+err_unlock:
+	if (dmabuf_revocable)
+		ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
+
+	return ret ? ERR_PTR(ret) : NULL;
 }
 
 /**
@@ -3909,6 +3979,7 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 	struct irdma_mr *iwmr = to_iwmr(ib_mr);
 	struct irdma_device *iwdev = to_iwdev(ib_mr->device);
 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+	bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf;
 	int ret;
 
 	if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
@@ -3923,17 +3994,28 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 		goto done;
 	}
 
-	ret = irdma_hwdereg_mr(ib_mr);
-	if (ret)
-		return ret;
+	if (!dmabuf_revocable) {
+		ret = irdma_hwdereg_mr(ib_mr);
+		if (ret)
+			return ret;
 
-	irdma_free_stag(iwdev, iwmr->stag);
+		irdma_free_stag(iwdev, iwmr->stag);
+	}
 done:
+	if (iwmr->region)
+		/* For dmabuf MRs, ib_umem_release will trigger a synchronous
+		 * call to the revoke callback which will perform the actual HW
+		 * invalidation via irdma_hwdereg_mr. We rely on this for its
+		 * implicit serialization w.r.t. concurrent revocations. This
+		 * must be done before freeing the PBLEs.
+		 */
+		ib_umem_release(iwmr->region);
+
 	if (iwpbl->pbl_allocated)
 		irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
 
-	if (iwmr->region)
-		ib_umem_release(iwmr->region);
+	if (dmabuf_revocable)
+		irdma_free_stag(iwdev, iwmr->stag);
 
 	kfree(iwmr);
 
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index e447acfd2071a..a7c8c0fd70192 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -8,7 +8,7 @@
 
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_mad.h>
-#include <rdma/ib_umem.h>
+#include <rdma/iter.h>
 #include <rdma/mana-abi.h>
 #include <rdma/uverbs_ioctl.h>
 #include <linux/dmapool.h>
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index d98892283065d..8535fd561691d 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -135,160 +135,164 @@ static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *
 	mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf);
 }
 
-static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev,
-			       struct mlx4_ib_cq_buf *buf,
-			       struct ib_umem **umem, u64 buf_addr, int cqe)
+#define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
+int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
+			   const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs)
 {
-	int err;
+	struct ib_udata *udata = &attrs->driver_udata;
+	struct ib_device *ibdev = ibcq->device;
+	int entries = attr->cqe;
+	int vector = attr->comp_vector;
+	struct mlx4_ib_dev *dev = to_mdev(ibdev);
+	struct mlx4_ib_cq *cq = to_mcq(ibcq);
+	struct mlx4_ib_create_cq ucmd;
 	int cqe_size = dev->dev->caps.cqe_size;
+	void *buf_addr;
 	int shift;
 	int n;
+	int err;
+	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx4_ib_ucontext, ibucontext);
+
+	if (attr->cqe > dev->dev->caps.max_cqes)
+		return -EINVAL;
+
+	if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
+		return -EINVAL;
 
-	*umem = ib_umem_get(&dev->ib_dev, buf_addr, cqe * cqe_size,
-			    IB_ACCESS_LOCAL_WRITE);
-	if (IS_ERR(*umem))
-		return PTR_ERR(*umem);
+	entries      = roundup_pow_of_two(entries + 1);
+	cq->ibcq.cqe = entries - 1;
+	mutex_init(&cq->resize_mutex);
+	spin_lock_init(&cq->lock);
+	INIT_LIST_HEAD(&cq->send_qp_list);
+	INIT_LIST_HEAD(&cq->recv_qp_list);
 
-	shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n);
+	if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+		err = -EFAULT;
+		goto err_cq;
+	}
+
+	buf_addr = (void *)(unsigned long)ucmd.buf_addr;
+
+	if (!ibcq->umem)
+		ibcq->umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
+					 entries * cqe_size,
+					 IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(ibcq->umem)) {
+		err = PTR_ERR(ibcq->umem);
+		goto err_cq;
+	}
+
+	shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->ibcq.umem, 0, &n);
 	if (shift < 0) {
 		err = shift;
-		goto err_buf;
+		goto err_cq;
 	}
 
-	err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt);
+	err = mlx4_mtt_init(dev->dev, n, shift, &cq->buf.mtt);
 	if (err)
-		goto err_buf;
+		goto err_cq;
 
-	err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);
+	err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->ibcq.umem);
 	if (err)
 		goto err_mtt;
 
+	err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db);
+	if (err)
+		goto err_mtt;
+
+	if (dev->eq_table)
+		vector = dev->eq_table[vector % ibdev->num_comp_vectors];
+
+	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, &context->uar,
+			    cq->db.dma, &cq->mcq, vector, 0,
+			    attr->flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION,
+			    buf_addr, true);
+	if (err)
+		goto err_dbmap;
+
+	cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp;
+	cq->mcq.event = mlx4_ib_cq_event;
+	cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
+
+	if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
+		err = -EFAULT;
+		goto err_cq_free;
+	}
+
 	return 0;
 
-err_mtt:
-	mlx4_mtt_cleanup(dev->dev, &buf->mtt);
+err_cq_free:
+	mlx4_cq_free(dev->dev, &cq->mcq);
 
-err_buf:
-	ib_umem_release(*umem);
+err_dbmap:
+	mlx4_ib_db_unmap_user(context, &cq->db);
 
+err_mtt:
+	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
+	/* UMEM is released by ib_core */
+
+err_cq:
 	return err;
 }
 
-#define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
 int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs)
 {
-	struct ib_udata *udata = &attrs->driver_udata;
 	struct ib_device *ibdev = ibcq->device;
 	int entries = attr->cqe;
 	int vector = attr->comp_vector;
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
 	struct mlx4_ib_cq *cq = to_mcq(ibcq);
-	struct mlx4_uar *uar;
 	void *buf_addr;
 	int err;
-	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
-		udata, struct mlx4_ib_ucontext, ibucontext);
 
-	if (entries < 1 || entries > dev->dev->caps.max_cqes)
-		return -EINVAL;
-
-	if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
+	if (attr->cqe > dev->dev->caps.max_cqes)
 		return -EINVAL;
 
 	entries      = roundup_pow_of_two(entries + 1);
 	cq->ibcq.cqe = entries - 1;
 	mutex_init(&cq->resize_mutex);
 	spin_lock_init(&cq->lock);
-	cq->resize_buf = NULL;
-	cq->resize_umem = NULL;
-	cq->create_flags = attr->flags;
 	INIT_LIST_HEAD(&cq->send_qp_list);
 	INIT_LIST_HEAD(&cq->recv_qp_list);
 
-	if (udata) {
-		struct mlx4_ib_create_cq ucmd;
-
-		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
-			err = -EFAULT;
-			goto err_cq;
-		}
-
-		buf_addr = (void *)(unsigned long)ucmd.buf_addr;
-		err = mlx4_ib_get_cq_umem(dev, &cq->buf, &cq->umem,
-					  ucmd.buf_addr, entries);
-		if (err)
-			goto err_cq;
-
-		err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db);
-		if (err)
-			goto err_mtt;
-
-		uar = &context->uar;
-		cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
-	} else {
-		err = mlx4_db_alloc(dev->dev, &cq->db, 1);
-		if (err)
-			goto err_cq;
-
-		cq->mcq.set_ci_db  = cq->db.db;
-		cq->mcq.arm_db     = cq->db.db + 1;
-		*cq->mcq.set_ci_db = 0;
-		*cq->mcq.arm_db    = 0;
+	err = mlx4_db_alloc(dev->dev, &cq->db, 1);
+	if (err)
+		return err;
 
-		err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
-		if (err)
-			goto err_db;
+	cq->mcq.set_ci_db  = cq->db.db;
+	cq->mcq.arm_db     = cq->db.db + 1;
+	*cq->mcq.set_ci_db = 0;
+	*cq->mcq.arm_db    = 0;
 
-		buf_addr = &cq->buf.buf;
+	err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
+	if (err)
+		goto err_db;
 
-		uar = &dev->priv_uar;
-		cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
-	}
+	buf_addr = &cq->buf.buf;
 
 	if (dev->eq_table)
 		vector = dev->eq_table[vector % ibdev->num_comp_vectors];
 
-	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, cq->db.dma,
-			    &cq->mcq, vector, 0,
-			    !!(cq->create_flags &
-			       IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION),
-			    buf_addr, !!udata);
+	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, &dev->priv_uar,
+			    cq->db.dma, &cq->mcq, vector, 0, 0,
+			    buf_addr, false);
 	if (err)
-		goto err_dbmap;
+		goto err_buf;
 
-	if (udata)
-		cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp;
-	else
-		cq->mcq.comp = mlx4_ib_cq_comp;
+	cq->mcq.comp = mlx4_ib_cq_comp;
 	cq->mcq.event = mlx4_ib_cq_event;
-
-	if (udata)
-		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
-			err = -EFAULT;
-			goto err_cq_free;
-		}
+	cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
 
 	return 0;
 
-err_cq_free:
-	mlx4_cq_free(dev->dev, &cq->mcq);
-
-err_dbmap:
-	if (udata)
-		mlx4_ib_db_unmap_user(context, &cq->db);
-
-err_mtt:
-	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
-
-	ib_umem_release(cq->umem);
-	if (!udata)
-		mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
+err_buf:
+	mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
 
 err_db:
-	if (!udata)
-		mlx4_db_free(dev->dev, &cq->db);
-err_cq:
+	mlx4_db_free(dev->dev, &cq->db);
 	return err;
 }
 
@@ -320,6 +324,9 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
 				   int entries, struct ib_udata *udata)
 {
 	struct mlx4_ib_resize_cq ucmd;
+	int cqe_size = dev->dev->caps.cqe_size;
+	int shift;
+	int n;
 	int err;
 
 	if (cq->resize_umem)
@@ -332,17 +339,43 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
 	if (!cq->resize_buf)
 		return -ENOMEM;
 
-	err = mlx4_ib_get_cq_umem(dev, &cq->resize_buf->buf, &cq->resize_umem,
-				  ucmd.buf_addr, entries);
-	if (err) {
-		kfree(cq->resize_buf);
-		cq->resize_buf = NULL;
-		return err;
+	cq->resize_umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
+				      entries * cqe_size,
+				      IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(cq->resize_umem)) {
+		err = PTR_ERR(cq->resize_umem);
+		goto err_buf;
 	}
 
+	shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->resize_umem, 0, &n);
+	if (shift < 0) {
+		err = shift;
+		goto err_umem;
+	}
+
+	err = mlx4_mtt_init(dev->dev, n, shift, &cq->resize_buf->buf.mtt);
+	if (err)
+		goto err_umem;
+
+	err = mlx4_ib_umem_write_mtt(dev, &cq->resize_buf->buf.mtt,
+				     cq->resize_umem);
+	if (err)
+		goto err_mtt;
+
 	cq->resize_buf->cqe = entries - 1;
 
 	return 0;
+
+err_mtt:
+	mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt);
+
+err_umem:
+	ib_umem_release(cq->resize_umem);
+
+err_buf:
+	kfree(cq->resize_buf);
+	cq->resize_buf = NULL;
+	return err;
 }
 
 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
@@ -433,8 +466,8 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 	if (ibcq->uobject) {
 		cq->buf      = cq->resize_buf->buf;
 		cq->ibcq.cqe = cq->resize_buf->cqe;
-		ib_umem_release(cq->umem);
-		cq->umem     = cq->resize_umem;
+		ib_umem_release(cq->ibcq.umem);
+		cq->ibcq.umem     = cq->resize_umem;
 
 		kfree(cq->resize_buf);
 		cq->resize_buf = NULL;
@@ -494,11 +527,11 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 				struct mlx4_ib_ucontext,
 				ibucontext),
 			&mcq->db);
+		/* UMEM is released by ib_core */
 	} else {
 		mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
 		mlx4_db_free(dev->dev, &mcq->db);
 	}
-	ib_umem_release(mcq->umem);
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 63360c5a73c77..73e17b4339eb6 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -2161,7 +2161,7 @@ static int __mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev,
 	if (!*pdescs)
 		return -ENOMEM;
 
-	*offset = kcalloc(num_counters, sizeof(**offset), GFP_KERNEL);
+	*offset = kzalloc_objs(**offset, num_counters);
 	if (!*offset)
 		goto err;
 
@@ -2525,6 +2525,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = {
 	.attach_mcast = mlx4_ib_mcg_attach,
 	.create_ah = mlx4_ib_create_ah,
 	.create_cq = mlx4_ib_create_cq,
+	.create_user_cq = mlx4_ib_create_user_cq,
 	.create_qp = mlx4_ib_create_qp,
 	.create_srq = mlx4_ib_create_srq,
 	.dealloc_pd = mlx4_ib_dealloc_pd,
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 5df5b955114ee..6a7ed5225c7db 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -121,9 +121,7 @@ struct mlx4_ib_cq {
 	struct mlx4_db		db;
 	spinlock_t		lock;
 	struct mutex		resize_mutex;
-	struct ib_umem	       *umem;
 	struct ib_umem	       *resize_umem;
-	int			create_flags;
 	/* List of qps that it serves.*/
 	struct list_head		send_qp_list;
 	struct list_head		recv_qp_list;
@@ -772,6 +770,9 @@ int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
 int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs);
+int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
+			   const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs);
 int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 77a72d2b0dd23..650b4a9121ff6 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -33,6 +33,7 @@
 
 #include <linux/slab.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/iter.h>
 
 #include "mlx4_ib.h"
 
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index d0cd3f805bcb9..43a7b5ca49dcc 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -749,16 +749,15 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 
 	*cqe_size = ucmd.cqe_size;
 
-	cq->buf.umem =
-		ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
-			    entries * ucmd.cqe_size, IB_ACCESS_LOCAL_WRITE);
-	if (IS_ERR(cq->buf.umem)) {
-		err = PTR_ERR(cq->buf.umem);
-		return err;
-	}
+	if (!cq->ibcq.umem)
+		cq->ibcq.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
+					    entries * ucmd.cqe_size,
+					    IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(cq->ibcq.umem))
+		return PTR_ERR(cq->ibcq.umem);
 
 	page_size = mlx5_umem_find_best_cq_quantized_pgoff(
-		cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
+		cq->ibcq.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
 		page_offset, 64, &page_offset_quantized);
 	if (!page_size) {
 		err = -EINVAL;
@@ -769,12 +768,12 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	if (err)
 		goto err_umem;
 
-	ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size);
+	ncont = ib_umem_num_dma_blocks(cq->ibcq.umem, page_size);
 	mlx5_ib_dbg(
 		dev,
 		"addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n",
 		ucmd.buf_addr, entries * ucmd.cqe_size,
-		ib_umem_num_pages(cq->buf.umem), page_size, ncont);
+		ib_umem_num_pages(cq->ibcq.umem), page_size, ncont);
 
 	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
 		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
@@ -785,7 +784,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	}
 
 	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
-	mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0);
+	mlx5_ib_populate_pas(cq->ibcq.umem, page_size, pas, 0);
 
 	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
 	MLX5_SET(cqc, cqc, log_page_size,
@@ -858,7 +857,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	mlx5_ib_db_unmap_user(context, &cq->db);
 
 err_umem:
-	ib_umem_release(cq->buf.umem);
+	/* UMEM is released by ib_core */
 	return err;
 }
 
@@ -868,7 +867,6 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)
 		udata, struct mlx5_ib_ucontext, ibucontext);
 
 	mlx5_ib_db_unmap_user(context, &cq->db);
-	ib_umem_release(cq->buf.umem);
 }
 
 static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf)
@@ -949,8 +947,9 @@ static void notify_soft_wc_handler(struct work_struct *work)
 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
 
-int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-		      struct uverbs_attr_bundle *attrs)
+int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
+			   const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_udata *udata = &attrs->driver_udata;
 	struct ib_device *ibdev = ibcq->device;
@@ -967,8 +966,7 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	int eqn;
 	int err;
 
-	if (entries < 0 ||
-	    (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
+	if (attr->cqe > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
 		return -EINVAL;
 
 	if (check_cq_create_flags(attr->flags))
@@ -981,26 +979,15 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	cq->ibcq.cqe = entries - 1;
 	mutex_init(&cq->resize_mutex);
 	spin_lock_init(&cq->lock);
-	cq->resize_buf = NULL;
-	cq->resize_umem = NULL;
-	cq->create_flags = attr->flags;
+	if (attr->flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)
+		cq->private_flags |= MLX5_IB_CQ_PR_TIMESTAMP_COMPLETION;
 	INIT_LIST_HEAD(&cq->list_send_qp);
 	INIT_LIST_HEAD(&cq->list_recv_qp);
 
-	if (udata) {
-		err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
-				     &index, &inlen, attrs);
-		if (err)
-			return err;
-	} else {
-		cqe_size = cache_line_size() == 128 ? 128 : 64;
-		err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
-				       &index, &inlen);
-		if (err)
-			return err;
-
-		INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
-	}
+	err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size, &index,
+			     &inlen, attrs);
+	if (err)
+		return err;
 
 	err = mlx5_comp_eqn_get(dev->mdev, vector, &eqn);
 	if (err)
@@ -1017,15 +1004,11 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	MLX5_SET(cqc, cqc, uar_page, index);
 	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
 	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
-	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)
+	if (attr->flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)
 		MLX5_SET(cqc, cqc, oi, 1);
 
-	if (udata) {
-		cq->mcq.comp = mlx5_add_cq_to_tasklet;
-		cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;
-	} else {
-		cq->mcq.comp  = mlx5_ib_cq_comp;
-	}
+	cq->mcq.comp = mlx5_add_cq_to_tasklet;
+	cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;
 
 	err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out, sizeof(out));
 	if (err)
@@ -1036,12 +1019,10 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 	INIT_LIST_HEAD(&cq->wc_list);
 
-	if (udata)
-		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
-			err = -EFAULT;
-			goto err_cmd;
-		}
-
+	if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
+		err = -EFAULT;
+		goto err_cmd;
+	}
 
 	kvfree(cqb);
 	return 0;
@@ -1051,10 +1032,82 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 err_cqb:
 	kvfree(cqb);
-	if (udata)
-		destroy_cq_user(cq, udata);
-	else
-		destroy_cq_kernel(dev, cq);
+	destroy_cq_user(cq, udata);
+	return err;
+}
+
+
+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+		      struct uverbs_attr_bundle *attrs)
+{
+	struct ib_device *ibdev = ibcq->device;
+	int entries = attr->cqe;
+	int vector = attr->comp_vector;
+	struct mlx5_ib_dev *dev = to_mdev(ibdev);
+	struct mlx5_ib_cq *cq = to_mcq(ibcq);
+	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+	int index;
+	int inlen;
+	u32 *cqb = NULL;
+	void *cqc;
+	int cqe_size;
+	int eqn;
+	int err;
+
+	if (attr->cqe > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
+		return -EINVAL;
+
+	entries = roundup_pow_of_two(entries + 1);
+	if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
+		return -EINVAL;
+
+	cq->ibcq.cqe = entries - 1;
+	mutex_init(&cq->resize_mutex);
+	spin_lock_init(&cq->lock);
+	INIT_LIST_HEAD(&cq->list_send_qp);
+	INIT_LIST_HEAD(&cq->list_recv_qp);
+
+	cqe_size = cache_line_size() == 128 ? 128 : 64;
+	err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index,
+			       &inlen);
+	if (err)
+		return err;
+
+	INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
+
+	err = mlx5_comp_eqn_get(dev->mdev, vector, &eqn);
+	if (err)
+		goto err_cqb;
+
+	cq->cqe_size = cqe_size;
+
+	cqc = MLX5_ADDR_OF(create_cq_in, cqb, cq_context);
+	MLX5_SET(cqc, cqc, cqe_sz,
+		 cqe_sz_to_mlx_sz(cqe_size,
+				  cq->private_flags &
+				  MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD));
+	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
+	MLX5_SET(cqc, cqc, uar_page, index);
+	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
+	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
+
+	cq->mcq.comp = mlx5_ib_cq_comp;
+
+	err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out,
+				  sizeof(out));
+	if (err)
+		goto err_cqb;
+
+	mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
+	cq->mcq.event = mlx5_ib_cq_event;
+
+	INIT_LIST_HEAD(&cq->wc_list);
+	kvfree(cqb);
+	return 0;
+
+err_cqb:
+	kvfree(cqb);
+	destroy_cq_kernel(dev, cq);
 	return err;
 }
 
@@ -1389,8 +1442,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 
 	if (udata) {
 		cq->ibcq.cqe = entries - 1;
-		ib_umem_release(cq->buf.umem);
-		cq->buf.umem = cq->resize_umem;
+		ib_umem_release(cq->ibcq.umem);
+		cq->ibcq.umem = cq->resize_umem;
 		cq->resize_umem = NULL;
 	} else {
 		struct mlx5_ib_cq_buf tbuf;
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 0066b2738ac89..645ebcc0832d7 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -1557,7 +1557,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
 	if (IS_ERR(cmd_out))
 		return PTR_ERR(cmd_out);
 
-	obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
+	obj = kzalloc_obj(*obj);
 	if (!obj)
 		return -ENOMEM;
 
@@ -2158,7 +2158,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
 		if (err)
 			goto err;
 
-		event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
+		event_sub = kzalloc_obj(*event_sub);
 		if (!event_sub) {
 			err = -ENOMEM;
 			goto err;
@@ -2398,7 +2398,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
 	if (err)
 		return err;
 
-	obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
+	obj = kzalloc_obj(*obj);
 	if (!obj)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx5/dm.c b/drivers/infiniband/hw/mlx5/dm.c
index 9972f38ba88a9..7a6fe4fea3e2c 100644
--- a/drivers/infiniband/hw/mlx5/dm.c
+++ b/drivers/infiniband/hw/mlx5/dm.c
@@ -228,7 +228,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)(
 	if (!err || err != -ENOENT)
 		goto err_unlock;
 
-	op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL);
+	op_entry = kzalloc_obj(*op_entry);
 	if (!op_entry)
 		goto err_unlock;
 
diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c
index cbccb0b9ac10d..b155baee0017c 100644
--- a/drivers/infiniband/hw/mlx5/fs.c
+++ b/drivers/infiniband/hw/mlx5/fs.c
@@ -2917,7 +2917,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
 	struct mlx5_ib_flow_matcher *obj;
 	int err;
 
-	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
+	obj = kzalloc_obj(*obj);
 	if (!obj)
 		return -ENOMEM;
 
@@ -3017,7 +3017,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
 	if (err)
 		return err;
 
-	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+	obj = kzalloc_obj(*obj);
 	if (!obj)
 		return -ENOMEM;
 
@@ -3259,7 +3259,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
 	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
 		return -EOPNOTSUPP;
 
-	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
+	maction = kzalloc_obj(*maction);
 	if (!maction)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c
index 621834d752052..1709b628702ee 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.c
+++ b/drivers/infiniband/hw/mlx5/ib_rep.c
@@ -3,17 +3,20 @@
  * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
  */
 
+#include <linux/mlx5/lag.h>
 #include <linux/mlx5/vport.h>
 #include "ib_rep.h"
 #include "srq.h"
 
 static int
 mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev,
+		      struct mlx5_core_dev *rep_dev,
 		      struct mlx5_eswitch_rep *rep,
 		      int vport_index)
 {
 	struct mlx5_ib_dev *ibdev;
 	struct net_device *ndev;
+	int ret;
 
 	ibdev = mlx5_eswitch_uplink_get_proto_dev(dev->priv.eswitch, REP_IB);
 	if (!ibdev)
@@ -23,7 +26,17 @@ mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev,
 	rep->rep_data[REP_IB].priv = ibdev;
 	ndev = mlx5_ib_get_rep_netdev(rep->esw, rep->vport);
 
-	return ib_device_set_netdev(&ibdev->ib_dev, ndev, vport_index + 1);
+	ret = ib_device_set_netdev(&ibdev->ib_dev, ndev, vport_index + 1);
+	if (ret)
+		return ret;
+
+	/* Only Vports that are not native to the LAG master eswitch need to add
+	 * demux rule.
+	 */
+	if (mlx5_eswitch_get_total_vports(dev) > vport_index)
+		return 0;
+
+	return mlx5_lag_demux_rule_add(rep_dev, rep->vport, vport_index);
 }
 
 static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev);
@@ -130,11 +143,12 @@ mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 
 				if (mlx5_lag_is_master(peer_dev))
 					lag_master = peer_dev;
-				else if (!mlx5_lag_is_mpesw(dev))
+				else if (!mlx5_lag_is_mpesw(peer_dev))
 				/* Only 1 ib port is the representor for all uplinks */
 					peer_n_ports--;
 
-				if (mlx5_get_dev_index(peer_dev) < mlx5_get_dev_index(dev))
+				if (mlx5_lag_get_dev_seq(peer_dev) <
+				    mlx5_lag_get_dev_seq(dev))
 					vport_index += peer_n_ports;
 			}
 		}
@@ -143,7 +157,7 @@ mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 	if (rep->vport == MLX5_VPORT_UPLINK && !new_uplink)
 		profile = &raw_eth_profile;
 	else
-		return mlx5_ib_set_vport_rep(lag_master, rep, vport_index);
+		return mlx5_ib_set_vport_rep(lag_master, dev, rep, vport_index);
 
 	if (mlx5_lag_is_shared_fdb(dev)) {
 		ret = mlx5_ib_take_transport(lag_master);
@@ -231,6 +245,8 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 		vport_index = i;
 	}
 
+	mlx5_lag_demux_rule_del(mdev, vport_index);
+
 	port = &dev->port[vport_index];
 
 	ib_device_set_netdev(&dev->ib_dev, NULL, vport_index + 1);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 635002e684a55..cd0af0fa95a2e 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -26,6 +26,7 @@
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/eswitch.h>
 #include <linux/mlx5/driver.h>
+#include <linux/mlx5/lag.h>
 #include <linux/list.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_umem_odp.h>
@@ -2244,16 +2245,14 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 
 	mutex_init(&bfregi->lock);
 	bfregi->lib_uar_4k = lib_uar_4k;
-	bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
-				GFP_KERNEL);
+	bfregi->count = kzalloc_objs(*bfregi->count, bfregi->total_num_bfregs);
 	if (!bfregi->count) {
 		err = -ENOMEM;
 		goto out_ucap;
 	}
 
-	bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
-				    sizeof(*bfregi->sys_pages),
-				    GFP_KERNEL);
+	bfregi->sys_pages =
+		kzalloc_objs(*bfregi->sys_pages, bfregi->num_sys_pages);
 	if (!bfregi->sys_pages) {
 		err = -ENOMEM;
 		goto out_count;
@@ -2518,12 +2517,22 @@ mlx5_ib_pgoff_to_mmap_entry(struct ib_ucontext *ucontext, off_t pg_off)
 	return rdma_user_mmap_entry_get_pgoff(ucontext, entry_pgoff);
 }
 
+static void mlx5_ib_free_var_mmap_entry(struct mlx5_user_mmap_entry *mentry,
+					struct mlx5_var_region *var_region)
+{
+	mutex_lock(&var_region->bitmap_lock);
+	clear_bit(mentry->page_idx, var_region->bitmap);
+	mutex_unlock(&var_region->bitmap_lock);
+	kfree(mentry);
+}
+
 static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
 {
 	struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
 	struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
 	struct mlx5_var_table *var_table = &dev->var_table;
 	struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
+	struct mlx5_var_region *var_region;
 
 	switch (mentry->mmap_flag) {
 	case MLX5_IB_MMAP_TYPE_MEMIC:
@@ -2531,10 +2540,12 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
 		mlx5_ib_dm_mmap_free(dev, mentry);
 		break;
 	case MLX5_IB_MMAP_TYPE_VAR:
-		mutex_lock(&var_table->bitmap_lock);
-		clear_bit(mentry->page_idx, var_table->bitmap);
-		mutex_unlock(&var_table->bitmap_lock);
-		kfree(mentry);
+		var_region = &var_table->var_region;
+		mlx5_ib_free_var_mmap_entry(mentry, var_region);
+		break;
+	case MLX5_IB_MMAP_TYPE_TLP_VAR:
+		var_region = &var_table->tlp_var_region;
+		mlx5_ib_free_var_mmap_entry(mentry, var_region);
 		break;
 	case MLX5_IB_MMAP_TYPE_UAR_WC:
 	case MLX5_IB_MMAP_TYPE_UAR_NC:
@@ -2685,6 +2696,7 @@ static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
 	mentry = to_mmmap(entry);
 	pfn = (mentry->address >> PAGE_SHIFT);
 	if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
+	    mentry->mmap_flag == MLX5_IB_MMAP_TYPE_TLP_VAR ||
 	    mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
 		prot = pgprot_noncached(vma->vm_page_prot);
 	else
@@ -3678,12 +3690,12 @@ static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)
 
 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
 {
+	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_core_dev *mdev = dev->mdev;
-	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
-								 MLX5_FLOW_NAMESPACE_LAG);
-	struct mlx5_flow_table *ft;
+	struct mlx5_flow_namespace *ns;
 	int err;
 
+	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_LAG);
 	if (!ns || !mlx5_lag_is_active(mdev))
 		return 0;
 
@@ -3691,14 +3703,15 @@ static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
 	if (err)
 		return err;
 
-	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
-	if (IS_ERR(ft)) {
-		err = PTR_ERR(ft);
+	ft_attr.level = 0;
+	ft_attr.prio = 0;
+	ft_attr.max_fte = dev->num_ports;
+
+	err = mlx5_lag_demux_init(mdev, &ft_attr);
+	if (err)
 		goto err_destroy_vport_lag;
-	}
 
 	mlx5e_lag_event_register(dev);
-	dev->flow_db->lag_demux_ft = ft;
 	dev->lag_ports = mlx5_lag_get_num_ports(mdev);
 	dev->lag_active = true;
 	return 0;
@@ -3716,8 +3729,7 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
 		dev->lag_active = false;
 
 		mlx5e_lag_event_unregister(dev);
-		mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
-		dev->flow_db->lag_demux_ft = NULL;
+		mlx5_lag_demux_cleanup(mdev);
 
 		mlx5_cmd_destroy_vport_lag(mdev);
 	}
@@ -4140,46 +4152,54 @@ static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
 }
 
 static struct mlx5_user_mmap_entry *
-alloc_var_entry(struct mlx5_ib_ucontext *c)
+alloc_var_entry(struct mlx5_ib_ucontext *c, u32 flags)
 {
 	struct mlx5_user_mmap_entry *entry;
+	struct mlx5_var_region *var_region;
 	struct mlx5_var_table *var_table;
 	u32 page_idx;
 	int err;
 
 	var_table = &to_mdev(c->ibucontext.device)->var_table;
+	if (flags & MLX5_IB_UAPI_VAR_ALLOC_FLAG_TLP)
+		var_region = &var_table->tlp_var_region;
+	else
+		var_region = &var_table->var_region;
+
 	entry = kzalloc_obj(*entry);
 	if (!entry)
 		return ERR_PTR(-ENOMEM);
 
-	mutex_lock(&var_table->bitmap_lock);
-	page_idx = find_first_zero_bit(var_table->bitmap,
-				       var_table->num_var_hw_entries);
-	if (page_idx >= var_table->num_var_hw_entries) {
+	mutex_lock(&var_region->bitmap_lock);
+	page_idx = find_first_zero_bit(var_region->bitmap,
+				       var_region->num_var_hw_entries);
+	if (page_idx >= var_region->num_var_hw_entries) {
 		err = -ENOSPC;
-		mutex_unlock(&var_table->bitmap_lock);
+		mutex_unlock(&var_region->bitmap_lock);
 		goto end;
 	}
 
-	set_bit(page_idx, var_table->bitmap);
-	mutex_unlock(&var_table->bitmap_lock);
+	set_bit(page_idx, var_region->bitmap);
+	mutex_unlock(&var_region->bitmap_lock);
 
-	entry->address = var_table->hw_start_addr +
-				(page_idx * var_table->stride_size);
+	entry->address = var_region->hw_start_addr +
+				(page_idx * var_region->stride_size);
 	entry->page_idx = page_idx;
-	entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
+	entry->mmap_flag = flags & MLX5_IB_UAPI_VAR_ALLOC_FLAG_TLP ?
+				   MLX5_IB_MMAP_TYPE_TLP_VAR :
+				   MLX5_IB_MMAP_TYPE_VAR;
 
 	err = mlx5_rdma_user_mmap_entry_insert(c, entry,
-					       var_table->stride_size);
+					       var_region->stride_size);
 	if (err)
 		goto err_insert;
 
 	return entry;
 
 err_insert:
-	mutex_lock(&var_table->bitmap_lock);
-	clear_bit(page_idx, var_table->bitmap);
-	mutex_unlock(&var_table->bitmap_lock);
+	mutex_lock(&var_region->bitmap_lock);
+	clear_bit(page_idx, var_region->bitmap);
+	mutex_unlock(&var_region->bitmap_lock);
 end:
 	kfree(entry);
 	return ERR_PTR(err);
@@ -4190,9 +4210,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
-	struct mlx5_ib_ucontext *c;
 	struct mlx5_user_mmap_entry *entry;
+	struct mlx5_ib_ucontext *c;
 	u64 mmap_offset;
+	u32 flags = 0;
 	u32 length;
 	int err;
 
@@ -4200,7 +4221,24 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
 	if (IS_ERR(c))
 		return PTR_ERR(c);
 
-	entry = alloc_var_entry(c);
+	err = uverbs_get_flags32(&flags, attrs,
+				 MLX5_IB_ATTR_VAR_OBJ_ALLOC_FLAGS,
+				 MLX5_IB_UAPI_VAR_ALLOC_FLAG_TLP);
+	if (err)
+		return err;
+
+	if (flags & MLX5_IB_UAPI_VAR_ALLOC_FLAG_TLP) {
+		if (!MLX5_CAP_GEN(to_mdev(c->ibucontext.device)->mdev,
+				  tlp_device_emulation_manager))
+			return -EOPNOTSUPP;
+	} else {
+		if (!(MLX5_CAP_GEN_64(to_mdev(c->ibucontext.device)->mdev,
+				      general_obj_types) &
+		      MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q))
+			return -EOPNOTSUPP;
+	}
+
+	entry = alloc_var_entry(c, flags);
 	if (IS_ERR(entry))
 		return PTR_ERR(entry);
 
@@ -4230,6 +4268,9 @@ DECLARE_UVERBS_NAMED_METHOD(
 			MLX5_IB_OBJECT_VAR,
 			UVERBS_ACCESS_NEW,
 			UA_MANDATORY),
+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_VAR_OBJ_ALLOC_FLAGS,
+			     enum mlx5_ib_uapi_var_alloc_flags,
+			     UA_OPTIONAL),
 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
 			   UVERBS_ATTR_TYPE(u32),
 			   UA_MANDATORY),
@@ -4257,7 +4298,8 @@ static bool var_is_supported(struct ib_device *device)
 	struct mlx5_ib_dev *dev = to_mdev(device);
 
 	return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
-			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
+			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) ||
+		MLX5_CAP_GEN(dev->mdev, tlp_device_emulation_manager);
 }
 
 static struct mlx5_user_mmap_entry *
@@ -4418,6 +4460,7 @@ static const struct uapi_definition mlx5_ib_defs[] = {
 
 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
 {
+	mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
 	mlx5_ib_data_direct_cleanup(dev);
 	mlx5_ib_cleanup_multiport_master(dev);
 	WARN_ON(!xa_empty(&dev->odp_mkeys));
@@ -4487,6 +4530,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (err && err != -EOPNOTSUPP)
 		goto err_dd;
 
+	mlx5_cmd_init_async_ctx(mdev, &dev->async_ctx);
+
 	return 0;
 err_dd:
 	mlx5_ib_data_direct_cleanup(dev);
@@ -4517,6 +4562,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
 	.check_mr_status = mlx5_ib_check_mr_status,
 	.create_ah = mlx5_ib_create_ah,
 	.create_cq = mlx5_ib_create_cq,
+	.create_user_cq = mlx5_ib_create_user_cq,
 	.create_qp = mlx5_ib_create_qp,
 	.create_srq = mlx5_ib_create_srq,
 	.create_user_ah = mlx5_ib_create_ah,
@@ -4606,10 +4652,10 @@ static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
 };
 
-static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
+static int mlx5_ib_init_var_region(struct mlx5_ib_dev *dev)
 {
+	struct mlx5_var_region *var_region = &dev->var_table.var_region;
 	struct mlx5_core_dev *mdev = dev->mdev;
-	struct mlx5_var_table *var_table = &dev->var_table;
 	u8 log_doorbell_bar_size;
 	u8 log_doorbell_stride;
 	u64 bar_size;
@@ -4618,17 +4664,39 @@ static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
 					log_doorbell_bar_size);
 	log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
 					log_doorbell_stride);
-	var_table->hw_start_addr = dev->mdev->bar_addr +
+	var_region->hw_start_addr = dev->mdev->bar_addr +
 				MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
 					doorbell_bar_offset);
 	bar_size = (1ULL << log_doorbell_bar_size) * 4096;
-	var_table->stride_size = 1ULL << log_doorbell_stride;
-	var_table->num_var_hw_entries = div_u64(bar_size,
-						var_table->stride_size);
-	mutex_init(&var_table->bitmap_lock);
-	var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
-					  GFP_KERNEL);
-	return (var_table->bitmap) ? 0 : -ENOMEM;
+	var_region->stride_size = 1ULL << log_doorbell_stride;
+	var_region->num_var_hw_entries = div_u64(bar_size,
+						 var_region->stride_size);
+	mutex_init(&var_region->bitmap_lock);
+	var_region->bitmap = bitmap_zalloc(var_region->num_var_hw_entries,
+					   GFP_KERNEL);
+	return (var_region->bitmap) ? 0 : -ENOMEM;
+}
+
+static int mlx5_ib_init_tlp_var_region(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_var_region *var_region = &dev->var_table.tlp_var_region;
+	struct mlx5_core_dev *mdev = dev->mdev;
+	u8 log_tlp_var_stride;
+
+	log_tlp_var_stride =
+		MLX5_CAP_DEV_TLP_EMULATION(mdev, log_tlp_rsp_gw_page_stride);
+	var_region->hw_start_addr =
+		dev->mdev->bar_addr +
+		MLX5_CAP64_DEV_TLP_EMULATION(mdev, tlp_rsp_gw_pages_bar_offset);
+
+	var_region->stride_size = (1ULL << log_tlp_var_stride) * 4096;
+	var_region->num_var_hw_entries =
+		MLX5_CAP_DEV_TLP_EMULATION(mdev, tlp_rsp_gw_num_pages);
+
+	mutex_init(&var_region->bitmap_lock);
+	var_region->bitmap = bitmap_zalloc(var_region->num_var_hw_entries,
+					   GFP_KERNEL);
+	return (var_region->bitmap) ? 0 : -ENOMEM;
 }
 
 static void mlx5_ib_cleanup_ucaps(struct mlx5_ib_dev *dev)
@@ -4666,13 +4734,19 @@ static int mlx5_ib_init_ucaps(struct mlx5_ib_dev *dev)
 	return ret;
 }
 
+static void mlx5_ib_cleanup_var_table(struct mlx5_ib_dev *dev)
+{
+	bitmap_free(dev->var_table.var_region.bitmap);
+	bitmap_free(dev->var_table.tlp_var_region.bitmap);
+}
+
 static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
 {
 	if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
 	    MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL)
 		mlx5_ib_cleanup_ucaps(dev);
 
-	bitmap_free(dev->var_table.bitmap);
+	mlx5_ib_cleanup_var_table(dev);
 }
 
 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
@@ -4720,7 +4794,7 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
 
 	if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
-		err = mlx5_ib_init_var_table(dev);
+		err = mlx5_ib_init_var_region(dev);
 		if (err)
 			return err;
 	}
@@ -4732,12 +4806,20 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
 			goto err_ucaps;
 	}
 
+	if (MLX5_CAP_GEN(dev->mdev, tlp_device_emulation_manager)) {
+		err = mlx5_ib_init_tlp_var_region(dev);
+		if (err)
+			goto err_tlp_var;
+	}
+
 	dev->ib_dev.use_cq_dim = true;
 
 	return 0;
 
+err_tlp_var:
+	mlx5_ib_cleanup_ucaps(dev);
 err_ucaps:
-	bitmap_free(dev->var_table.bitmap);
+	bitmap_free(dev->var_table.var_region.bitmap);
 	return err;
 }
 
@@ -4873,7 +4955,7 @@ static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
 
 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
 {
-	mlx5_mkey_cache_cleanup(dev);
+	mlx5r_frmr_pools_cleanup(&dev->ib_dev);
 	mlx5r_umr_resource_cleanup(dev);
 	mlx5r_umr_cleanup(dev);
 }
@@ -4891,9 +4973,10 @@ static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
 	if (ret)
 		return ret;
 
-	ret = mlx5_mkey_cache_init(dev);
+	ret = mlx5r_frmr_pools_init(&dev->ib_dev);
 	if (ret)
-		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
+		mlx5_ib_warn(dev, "frmr pools init failed %d\n", ret);
+
 	return ret;
 }
 
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index af321f6ef7f54..75d5b5672b5cf 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -31,6 +31,7 @@
  */
 
 #include <rdma/ib_umem_odp.h>
+#include <rdma/iter.h>
 #include "mlx5_ib.h"
 
 /*
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 4f4114d951300..f9a9a200ccf18 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -162,6 +162,7 @@ enum mlx5_ib_mmap_type {
 	MLX5_IB_MMAP_TYPE_UAR_WC = 3,
 	MLX5_IB_MMAP_TYPE_UAR_NC = 4,
 	MLX5_IB_MMAP_TYPE_MEMIC_OP = 5,
+	MLX5_IB_MMAP_TYPE_TLP_VAR = 6,
 };
 
 struct mlx5_bfreg_info {
@@ -306,7 +307,6 @@ struct mlx5_ib_flow_db {
 	struct mlx5_ib_flow_prio	rdma_rx[MLX5_IB_NUM_FLOW_FT];
 	struct mlx5_ib_flow_prio	rdma_tx[MLX5_IB_NUM_FLOW_FT];
 	struct mlx5_ib_flow_prio	opfcs[MLX5_IB_OPCOUNTER_MAX];
-	struct mlx5_flow_table		*lag_demux_ft;
 	struct mlx5_ib_flow_prio        *rdma_transport_rx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];
 	struct mlx5_ib_flow_prio        *rdma_transport_tx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];
 	/* Protect flow steering bypass flow tables
@@ -561,6 +561,7 @@ struct mlx5_ib_cq_buf {
 enum mlx5_ib_cq_pr_flags {
 	MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD	= 1 << 0,
 	MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS = 1 << 1,
+	MLX5_IB_CQ_PR_TIMESTAMP_COMPLETION = 1 << 2,
 };
 
 struct mlx5_ib_cq {
@@ -581,7 +582,6 @@ struct mlx5_ib_cq {
 	int			cqe_size;
 	struct list_head	list_send_qp;
 	struct list_head	list_recv_qp;
-	u32			create_flags;
 	struct list_head	wc_list;
 	enum ib_cq_notify_flags notify_flags;
 	struct work_struct	notify_work;
@@ -641,25 +641,12 @@ enum mlx5_mkey_type {
 /* Used for non-existent ph value */
 #define MLX5_IB_NO_PH 0xff
 
-struct mlx5r_cache_rb_key {
-	u8 ats:1;
-	u8 ph;
-	u16 st_index;
-	unsigned int access_mode;
-	unsigned int access_flags;
-	unsigned int ndescs;
-};
-
 struct mlx5_ib_mkey {
 	u32 key;
 	enum mlx5_mkey_type type;
 	unsigned int ndescs;
 	struct wait_queue_head wait;
 	refcount_t usecount;
-	/* Cacheable user Mkey must hold either a rb_key or a cache_ent. */
-	struct mlx5r_cache_rb_key rb_key;
-	struct mlx5_cache_ent *cache_ent;
-	u8 cacheable : 1;
 };
 
 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
@@ -784,68 +771,6 @@ struct umr_common {
 	struct mutex init_lock;
 };
 
-#define NUM_MKEYS_PER_PAGE \
-	((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))
-
-struct mlx5_mkeys_page {
-	u32 mkeys[NUM_MKEYS_PER_PAGE];
-	struct list_head list;
-};
-static_assert(sizeof(struct mlx5_mkeys_page) == PAGE_SIZE);
-
-struct mlx5_mkeys_queue {
-	struct list_head pages_list;
-	u32 num_pages;
-	unsigned long ci;
-	spinlock_t lock; /* sync list ops */
-};
-
-struct mlx5_cache_ent {
-	struct mlx5_mkeys_queue	mkeys_queue;
-	u32			pending;
-
-	char                    name[4];
-
-	struct rb_node		node;
-	struct mlx5r_cache_rb_key rb_key;
-
-	u8 is_tmp:1;
-	u8 disabled:1;
-	u8 fill_to_high_water:1;
-	u8 tmp_cleanup_scheduled:1;
-
-	/*
-	 * - limit is the low water mark for stored mkeys, 2* limit is the
-	 *   upper water mark.
-	 */
-	u32 in_use;
-	u32 limit;
-
-	/* Statistics */
-	u32                     miss;
-
-	struct mlx5_ib_dev     *dev;
-	struct delayed_work	dwork;
-};
-
-struct mlx5r_async_create_mkey {
-	union {
-		u32 in[MLX5_ST_SZ_BYTES(create_mkey_in)];
-		u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
-	};
-	struct mlx5_async_work cb_work;
-	struct mlx5_cache_ent *ent;
-	u32 mkey;
-};
-
-struct mlx5_mkey_cache {
-	struct workqueue_struct *wq;
-	struct rb_root		rb_root;
-	struct mutex		rb_lock;
-	struct dentry		*fs_root;
-	unsigned long		last_add;
-};
-
 struct mlx5_ib_port_resources {
 	struct mlx5_ib_gsi_qp *gsi;
 	struct work_struct pkey_change_work;
@@ -1132,7 +1057,7 @@ struct mlx5_devx_event_table {
 	struct xarray event_xa;
 };
 
-struct mlx5_var_table {
+struct mlx5_var_region {
 	/* serialize updating the bitmap */
 	struct mutex bitmap_lock;
 	unsigned long *bitmap;
@@ -1141,6 +1066,11 @@ struct mlx5_var_table {
 	u64 num_var_hw_entries;
 };
 
+struct mlx5_var_table {
+	struct mlx5_var_region var_region;
+	struct mlx5_var_region tlp_var_region;
+};
+
 struct mlx5_port_caps {
 	bool has_smi;
 	u8 ext_port_cap;
@@ -1182,8 +1112,6 @@ struct mlx5_ib_dev {
 	struct mlx5_ib_resources	devr;
 
 	atomic_t			mkey_var;
-	struct mlx5_mkey_cache		cache;
-	struct timer_list		delay_timer;
 	/* Prevents soft lock on massive reg MRs */
 	struct mutex			slow_path_mutex;
 	struct ib_odp_caps	odp_caps;
@@ -1371,6 +1299,9 @@ int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
 			 size_t buflen, size_t *bc);
 int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs);
+int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
+			   const struct ib_cq_init_attr *attr,
+			   struct uverbs_attr_bundle *attrs);
 int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mlx5_ib_pre_destroy_cq(struct ib_cq *cq);
@@ -1442,13 +1373,8 @@ int mlx5_ib_query_port_speed(struct ib_device *ibdev, u32 port_num,
 void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas,
 			  u64 access_flags);
 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
-int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev);
-void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);
-struct mlx5_cache_ent *
-mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
-			      struct mlx5r_cache_rb_key rb_key,
-			      bool persistent_entry);
-
+int mlx5r_frmr_pools_init(struct ib_device *device);
+void mlx5r_frmr_pools_cleanup(struct ib_device *device);
 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
 				       int access_flags, int access_mode,
 				       int ndescs);
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 665323b90b64f..95fa873dae935 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -31,7 +31,6 @@
  * SOFTWARE.
  */
 
-
 #include <linux/kref.h>
 #include <linux/random.h>
 #include <linux/debugfs.h>
@@ -39,6 +38,7 @@
 #include <linux/delay.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-resv.h>
+#include <rdma/frmr_pools.h>
 #include <rdma/ib_umem_odp.h>
 #include "dm.h"
 #include "mlx5_ib.h"
@@ -46,15 +46,14 @@
 #include "data_direct.h"
 #include "dmah.h"
 
-enum {
-	MAX_PENDING_REG_MR = 8,
-};
 
-#define MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS 4
-#define MLX5_UMR_ALIGN 2048
+static int mkey_max_umr_order(struct mlx5_ib_dev *dev)
+{
+	if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
+		return MLX5_MAX_UMR_EXTENDED_SHIFT;
+	return MLX5_MAX_UMR_SHIFT;
+}
 
-static void
-create_mkey_callback(int status, struct mlx5_async_work *context);
 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
 				     u64 iova, int access_flags,
 				     unsigned long page_size, bool populate,
@@ -111,23 +110,6 @@ static int mlx5_ib_create_mkey(struct mlx5_ib_dev *dev,
 	return ret;
 }
 
-static int mlx5_ib_create_mkey_cb(struct mlx5r_async_create_mkey *async_create)
-{
-	struct mlx5_ib_dev *dev = async_create->ent->dev;
-	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
-	size_t outlen = MLX5_ST_SZ_BYTES(create_mkey_out);
-
-	MLX5_SET(create_mkey_in, async_create->in, opcode,
-		 MLX5_CMD_OP_CREATE_MKEY);
-	assign_mkey_variant(dev, &async_create->mkey, async_create->in);
-	return mlx5_cmd_exec_cb(&dev->async_ctx, async_create->in, inlen,
-				async_create->out, outlen, create_mkey_callback,
-				&async_create->cb_work);
-}
-
-static int mkey_cache_max_order(struct mlx5_ib_dev *dev);
-static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
-
 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 {
 	WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
@@ -135,94 +117,6 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 	return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);
 }
 
-static void create_mkey_warn(struct mlx5_ib_dev *dev, int status, void *out)
-{
-	if (status == -ENXIO) /* core driver is not available */
-		return;
-
-	mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
-	if (status != -EREMOTEIO) /* driver specific failure */
-		return;
-
-	/* Failed in FW, print cmd out failure details */
-	mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);
-}
-
-static int push_mkey_locked(struct mlx5_cache_ent *ent, u32 mkey)
-{
-	unsigned long tmp = ent->mkeys_queue.ci % NUM_MKEYS_PER_PAGE;
-	struct mlx5_mkeys_page *page;
-
-	lockdep_assert_held(&ent->mkeys_queue.lock);
-	if (ent->mkeys_queue.ci >=
-	    ent->mkeys_queue.num_pages * NUM_MKEYS_PER_PAGE) {
-		page = kzalloc_obj(*page, GFP_ATOMIC);
-		if (!page)
-			return -ENOMEM;
-		ent->mkeys_queue.num_pages++;
-		list_add_tail(&page->list, &ent->mkeys_queue.pages_list);
-	} else {
-		page = list_last_entry(&ent->mkeys_queue.pages_list,
-				       struct mlx5_mkeys_page, list);
-	}
-
-	page->mkeys[tmp] = mkey;
-	ent->mkeys_queue.ci++;
-	return 0;
-}
-
-static int pop_mkey_locked(struct mlx5_cache_ent *ent)
-{
-	unsigned long tmp = (ent->mkeys_queue.ci - 1) % NUM_MKEYS_PER_PAGE;
-	struct mlx5_mkeys_page *last_page;
-	u32 mkey;
-
-	lockdep_assert_held(&ent->mkeys_queue.lock);
-	last_page = list_last_entry(&ent->mkeys_queue.pages_list,
-				    struct mlx5_mkeys_page, list);
-	mkey = last_page->mkeys[tmp];
-	last_page->mkeys[tmp] = 0;
-	ent->mkeys_queue.ci--;
-	if (ent->mkeys_queue.num_pages > 1 && !tmp) {
-		list_del(&last_page->list);
-		ent->mkeys_queue.num_pages--;
-		kfree(last_page);
-	}
-	return mkey;
-}
-
-static void create_mkey_callback(int status, struct mlx5_async_work *context)
-{
-	struct mlx5r_async_create_mkey *mkey_out =
-		container_of(context, struct mlx5r_async_create_mkey, cb_work);
-	struct mlx5_cache_ent *ent = mkey_out->ent;
-	struct mlx5_ib_dev *dev = ent->dev;
-	unsigned long flags;
-
-	if (status) {
-		create_mkey_warn(dev, status, mkey_out->out);
-		kfree(mkey_out);
-		spin_lock_irqsave(&ent->mkeys_queue.lock, flags);
-		ent->pending--;
-		WRITE_ONCE(dev->fill_delay, 1);
-		spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags);
-		mod_timer(&dev->delay_timer, jiffies + HZ);
-		return;
-	}
-
-	mkey_out->mkey |= mlx5_idx_to_mkey(
-		MLX5_GET(create_mkey_out, mkey_out->out, mkey_index));
-	WRITE_ONCE(dev->cache.last_add, jiffies);
-
-	spin_lock_irqsave(&ent->mkeys_queue.lock, flags);
-	push_mkey_locked(ent, mkey_out->mkey);
-	ent->pending--;
-	/* If we are doing fill_to_high_water then keep going. */
-	queue_adjust_cache_locked(ent);
-	spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags);
-	kfree(mkey_out);
-}
-
 static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)
 {
 	int ret = 0;
@@ -242,537 +136,6 @@ static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)
 	return ret;
 }
 
-static void set_cache_mkc(struct mlx5_cache_ent *ent, void *mkc)
-{
-	set_mkc_access_pd_addr_fields(mkc, ent->rb_key.access_flags, 0,
-				      ent->dev->umrc.pd);
-	MLX5_SET(mkc, mkc, free, 1);
-	MLX5_SET(mkc, mkc, umr_en, 1);
-	MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3);
-	MLX5_SET(mkc, mkc, access_mode_4_2,
-		(ent->rb_key.access_mode >> 2) & 0x7);
-	MLX5_SET(mkc, mkc, ma_translation_mode, !!ent->rb_key.ats);
-
-	MLX5_SET(mkc, mkc, translations_octword_size,
-		 get_mkc_octo_size(ent->rb_key.access_mode,
-				   ent->rb_key.ndescs));
-	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
-
-	if (ent->rb_key.ph != MLX5_IB_NO_PH) {
-		MLX5_SET(mkc, mkc, pcie_tph_en, 1);
-		MLX5_SET(mkc, mkc, pcie_tph_ph, ent->rb_key.ph);
-		if (ent->rb_key.st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
-			MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
-				 ent->rb_key.st_index);
-	}
-}
-
-/* Asynchronously schedule new MRs to be populated in the cache. */
-static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
-{
-	struct mlx5r_async_create_mkey *async_create;
-	void *mkc;
-	int err = 0;
-	int i;
-
-	for (i = 0; i < num; i++) {
-		async_create = kzalloc_obj(struct mlx5r_async_create_mkey);
-		if (!async_create)
-			return -ENOMEM;
-		mkc = MLX5_ADDR_OF(create_mkey_in, async_create->in,
-				   memory_key_mkey_entry);
-		set_cache_mkc(ent, mkc);
-		async_create->ent = ent;
-
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		if (ent->pending >= MAX_PENDING_REG_MR) {
-			err = -EAGAIN;
-			goto free_async_create;
-		}
-		ent->pending++;
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-
-		err = mlx5_ib_create_mkey_cb(async_create);
-		if (err) {
-			mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
-			goto err_create_mkey;
-		}
-	}
-
-	return 0;
-
-err_create_mkey:
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	ent->pending--;
-free_async_create:
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-	kfree(async_create);
-	return err;
-}
-
-/* Synchronously create a MR in the cache */
-static int create_cache_mkey(struct mlx5_cache_ent *ent, u32 *mkey)
-{
-	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
-	void *mkc;
-	u32 *in;
-	int err;
-
-	in = kzalloc(inlen, GFP_KERNEL);
-	if (!in)
-		return -ENOMEM;
-	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
-	set_cache_mkc(ent, mkc);
-
-	err = mlx5_core_create_mkey(ent->dev->mdev, mkey, in, inlen);
-	if (err)
-		goto free_in;
-
-	WRITE_ONCE(ent->dev->cache.last_add, jiffies);
-free_in:
-	kfree(in);
-	return err;
-}
-
-static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
-{
-	u32 mkey;
-
-	lockdep_assert_held(&ent->mkeys_queue.lock);
-	if (!ent->mkeys_queue.ci)
-		return;
-	mkey = pop_mkey_locked(ent);
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-	mlx5_core_destroy_mkey(ent->dev->mdev, mkey);
-	spin_lock_irq(&ent->mkeys_queue.lock);
-}
-
-static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
-				bool limit_fill)
-	__acquires(&ent->mkeys_queue.lock) __releases(&ent->mkeys_queue.lock)
-{
-	int err;
-
-	lockdep_assert_held(&ent->mkeys_queue.lock);
-
-	while (true) {
-		if (limit_fill)
-			target = ent->limit * 2;
-		if (target == ent->pending + ent->mkeys_queue.ci)
-			return 0;
-		if (target > ent->pending + ent->mkeys_queue.ci) {
-			u32 todo = target - (ent->pending + ent->mkeys_queue.ci);
-
-			spin_unlock_irq(&ent->mkeys_queue.lock);
-			err = add_keys(ent, todo);
-			if (err == -EAGAIN)
-				usleep_range(3000, 5000);
-			spin_lock_irq(&ent->mkeys_queue.lock);
-			if (err) {
-				if (err != -EAGAIN)
-					return err;
-			} else
-				return 0;
-		} else {
-			remove_cache_mr_locked(ent);
-		}
-	}
-}
-
-static ssize_t size_write(struct file *filp, const char __user *buf,
-			  size_t count, loff_t *pos)
-{
-	struct mlx5_cache_ent *ent = filp->private_data;
-	u32 target;
-	int err;
-
-	err = kstrtou32_from_user(buf, count, 0, &target);
-	if (err)
-		return err;
-
-	/*
-	 * Target is the new value of total_mrs the user requests, however we
-	 * cannot free MRs that are in use. Compute the target value for stored
-	 * mkeys.
-	 */
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	if (target < ent->in_use) {
-		err = -EINVAL;
-		goto err_unlock;
-	}
-	target = target - ent->in_use;
-	if (target < ent->limit || target > ent->limit*2) {
-		err = -EINVAL;
-		goto err_unlock;
-	}
-	err = resize_available_mrs(ent, target, false);
-	if (err)
-		goto err_unlock;
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-
-	return count;
-
-err_unlock:
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-	return err;
-}
-
-static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
-			 loff_t *pos)
-{
-	struct mlx5_cache_ent *ent = filp->private_data;
-	char lbuf[20];
-	int err;
-
-	err = snprintf(lbuf, sizeof(lbuf), "%ld\n",
-		       ent->mkeys_queue.ci + ent->in_use);
-	if (err < 0)
-		return err;
-
-	return simple_read_from_buffer(buf, count, pos, lbuf, err);
-}
-
-static const struct file_operations size_fops = {
-	.owner	= THIS_MODULE,
-	.open	= simple_open,
-	.write	= size_write,
-	.read	= size_read,
-};
-
-static ssize_t limit_write(struct file *filp, const char __user *buf,
-			   size_t count, loff_t *pos)
-{
-	struct mlx5_cache_ent *ent = filp->private_data;
-	u32 var;
-	int err;
-
-	err = kstrtou32_from_user(buf, count, 0, &var);
-	if (err)
-		return err;
-
-	/*
-	 * Upon set we immediately fill the cache to high water mark implied by
-	 * the limit.
-	 */
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	ent->limit = var;
-	err = resize_available_mrs(ent, 0, true);
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-	if (err)
-		return err;
-	return count;
-}
-
-static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
-			  loff_t *pos)
-{
-	struct mlx5_cache_ent *ent = filp->private_data;
-	char lbuf[20];
-	int err;
-
-	err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
-	if (err < 0)
-		return err;
-
-	return simple_read_from_buffer(buf, count, pos, lbuf, err);
-}
-
-static const struct file_operations limit_fops = {
-	.owner	= THIS_MODULE,
-	.open	= simple_open,
-	.write	= limit_write,
-	.read	= limit_read,
-};
-
-static bool someone_adding(struct mlx5_mkey_cache *cache)
-{
-	struct mlx5_cache_ent *ent;
-	struct rb_node *node;
-	bool ret;
-
-	mutex_lock(&cache->rb_lock);
-	for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {
-		ent = rb_entry(node, struct mlx5_cache_ent, node);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		ret = ent->mkeys_queue.ci < ent->limit;
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		if (ret) {
-			mutex_unlock(&cache->rb_lock);
-			return true;
-		}
-	}
-	mutex_unlock(&cache->rb_lock);
-	return false;
-}
-
-/*
- * Check if the bucket is outside the high/low water mark and schedule an async
- * update. The cache refill has hysteresis, once the low water mark is hit it is
- * refilled up to the high mark.
- */
-static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
-{
-	lockdep_assert_held(&ent->mkeys_queue.lock);
-
-	if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp)
-		return;
-	if (ent->mkeys_queue.ci < ent->limit) {
-		ent->fill_to_high_water = true;
-		mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
-	} else if (ent->fill_to_high_water &&
-		   ent->mkeys_queue.ci + ent->pending < 2 * ent->limit) {
-		/*
-		 * Once we start populating due to hitting a low water mark
-		 * continue until we pass the high water mark.
-		 */
-		mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
-	} else if (ent->mkeys_queue.ci == 2 * ent->limit) {
-		ent->fill_to_high_water = false;
-	} else if (ent->mkeys_queue.ci > 2 * ent->limit) {
-		/* Queue deletion of excess entries */
-		ent->fill_to_high_water = false;
-		if (ent->pending)
-			queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
-					   secs_to_jiffies(1));
-		else
-			mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
-	}
-}
-
-static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent)
-{
-	u32 mkey;
-
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	while (ent->mkeys_queue.ci) {
-		mkey = pop_mkey_locked(ent);
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		mlx5_core_destroy_mkey(dev->mdev, mkey);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-	}
-	ent->tmp_cleanup_scheduled = false;
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-}
-
-static void __cache_work_func(struct mlx5_cache_ent *ent)
-{
-	struct mlx5_ib_dev *dev = ent->dev;
-	struct mlx5_mkey_cache *cache = &dev->cache;
-	int err;
-
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	if (ent->disabled)
-		goto out;
-
-	if (ent->fill_to_high_water &&
-	    ent->mkeys_queue.ci + ent->pending < 2 * ent->limit &&
-	    !READ_ONCE(dev->fill_delay)) {
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		err = add_keys(ent, 1);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		if (ent->disabled)
-			goto out;
-		if (err) {
-			/*
-			 * EAGAIN only happens if there are pending MRs, so we
-			 * will be rescheduled when storing them. The only
-			 * failure path here is ENOMEM.
-			 */
-			if (err != -EAGAIN) {
-				mlx5_ib_warn(
-					dev,
-					"add keys command failed, err %d\n",
-					err);
-				queue_delayed_work(cache->wq, &ent->dwork,
-						   secs_to_jiffies(1));
-			}
-		}
-	} else if (ent->mkeys_queue.ci > 2 * ent->limit) {
-		bool need_delay;
-
-		/*
-		 * The remove_cache_mr() logic is performed as garbage
-		 * collection task. Such task is intended to be run when no
-		 * other active processes are running.
-		 *
-		 * The need_resched() will return TRUE if there are user tasks
-		 * to be activated in near future.
-		 *
-		 * In such case, we don't execute remove_cache_mr() and postpone
-		 * the garbage collection work to try to run in next cycle, in
-		 * order to free CPU resources to other tasks.
-		 */
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		need_delay = need_resched() || someone_adding(cache) ||
-			     !time_after(jiffies,
-					 READ_ONCE(cache->last_add) + 300 * HZ);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		if (ent->disabled)
-			goto out;
-		if (need_delay) {
-			queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
-			goto out;
-		}
-		remove_cache_mr_locked(ent);
-		queue_adjust_cache_locked(ent);
-	}
-out:
-	spin_unlock_irq(&ent->mkeys_queue.lock);
-}
-
-static void delayed_cache_work_func(struct work_struct *work)
-{
-	struct mlx5_cache_ent *ent;
-
-	ent = container_of(work, struct mlx5_cache_ent, dwork.work);
-	/* temp entries are never filled, only cleaned */
-	if (ent->is_tmp)
-		clean_keys(ent->dev, ent);
-	else
-		__cache_work_func(ent);
-}
-
-static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1,
-			     struct mlx5r_cache_rb_key key2)
-{
-	int res;
-
-	res = key1.ats - key2.ats;
-	if (res)
-		return res;
-
-	res = key1.access_mode - key2.access_mode;
-	if (res)
-		return res;
-
-	res = key1.access_flags - key2.access_flags;
-	if (res)
-		return res;
-
-	res = key1.st_index - key2.st_index;
-	if (res)
-		return res;
-
-	res = key1.ph - key2.ph;
-	if (res)
-		return res;
-
-	/*
-	 * keep ndescs the last in the compare table since the find function
-	 * searches for an exact match on all properties and only closest
-	 * match in size.
-	 */
-	return key1.ndescs - key2.ndescs;
-}
-
-static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,
-				 struct mlx5_cache_ent *ent)
-{
-	struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL;
-	struct mlx5_cache_ent *cur;
-	int cmp;
-
-	/* Figure out where to put new node */
-	while (*new) {
-		cur = rb_entry(*new, struct mlx5_cache_ent, node);
-		parent = *new;
-		cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key);
-		if (cmp > 0)
-			new = &((*new)->rb_left);
-		if (cmp < 0)
-			new = &((*new)->rb_right);
-		if (cmp == 0)
-			return -EEXIST;
-	}
-
-	/* Add new node and rebalance tree. */
-	rb_link_node(&ent->node, parent, new);
-	rb_insert_color(&ent->node, &cache->rb_root);
-
-	return 0;
-}
-
-static struct mlx5_cache_ent *
-mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev,
-			   struct mlx5r_cache_rb_key rb_key)
-{
-	struct rb_node *node = dev->cache.rb_root.rb_node;
-	struct mlx5_cache_ent *cur, *smallest = NULL;
-	u64 ndescs_limit;
-	int cmp;
-
-	/*
-	 * Find the smallest ent with order >= requested_order.
-	 */
-	while (node) {
-		cur = rb_entry(node, struct mlx5_cache_ent, node);
-		cmp = cache_ent_key_cmp(cur->rb_key, rb_key);
-		if (cmp > 0) {
-			smallest = cur;
-			node = node->rb_left;
-		}
-		if (cmp < 0)
-			node = node->rb_right;
-		if (cmp == 0)
-			return cur;
-	}
-
-	/*
-	 * Limit the usage of mkeys larger than twice the required size while
-	 * also allowing the usage of smallest cache entry for small MRs.
-	 */
-	ndescs_limit = max_t(u64, rb_key.ndescs * 2,
-			     MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS);
-
-	return (smallest &&
-		smallest->rb_key.access_mode == rb_key.access_mode &&
-		smallest->rb_key.access_flags == rb_key.access_flags &&
-		smallest->rb_key.ats == rb_key.ats &&
-		smallest->rb_key.st_index == rb_key.st_index &&
-		smallest->rb_key.ph == rb_key.ph &&
-		smallest->rb_key.ndescs <= ndescs_limit) ?
-		       smallest :
-		       NULL;
-}
-
-static struct mlx5_ib_mr *_mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
-					       struct mlx5_cache_ent *ent)
-{
-	struct mlx5_ib_mr *mr;
-	int err;
-
-	mr = kzalloc_obj(*mr);
-	if (!mr)
-		return ERR_PTR(-ENOMEM);
-
-	spin_lock_irq(&ent->mkeys_queue.lock);
-	ent->in_use++;
-
-	if (!ent->mkeys_queue.ci) {
-		queue_adjust_cache_locked(ent);
-		ent->miss++;
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		err = create_cache_mkey(ent, &mr->mmkey.key);
-		if (err) {
-			spin_lock_irq(&ent->mkeys_queue.lock);
-			ent->in_use--;
-			spin_unlock_irq(&ent->mkeys_queue.lock);
-			kfree(mr);
-			return ERR_PTR(err);
-		}
-	} else {
-		mr->mmkey.key = pop_mkey_locked(ent);
-		queue_adjust_cache_locked(ent);
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-	}
-	mr->mmkey.cache_ent = ent;
-	mr->mmkey.type = MLX5_MKEY_MR;
-	mr->mmkey.rb_key = ent->rb_key;
-	mr->mmkey.cacheable = true;
-	init_waitqueue_head(&mr->mmkey.wait);
-	return mr;
-}
-
 static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
 					 int access_flags)
 {
@@ -797,256 +160,201 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
 	return ret;
 }
 
-struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
-				       int access_flags, int access_mode,
-				       int ndescs)
-{
-	struct mlx5r_cache_rb_key rb_key = {
-		.ndescs = ndescs,
-		.access_mode = access_mode,
-		.access_flags = get_unchangeable_access_flags(dev, access_flags),
-		.ph = MLX5_IB_NO_PH,
-	};
-	struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key);
+#define MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK 1ULL
+#define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \
+	MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK
 
-	if (!ent)
-		return ERR_PTR(-EOPNOTSUPP);
+#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16
+#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000
+#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF
 
-	return _mlx5_mr_cache_alloc(dev, ent);
-}
-
-static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
-{
-	if (!mlx5_debugfs_root || dev->is_rep)
-		return;
-
-	debugfs_remove_recursive(dev->cache.fs_root);
-	dev->cache.fs_root = NULL;
-}
-
-static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev,
-					    struct mlx5_cache_ent *ent)
+static struct mlx5_ib_mr *
+_mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem,
+		      int access_flags, int access_mode,
+		      unsigned long page_size, u16 st_index, u8 ph)
 {
-	int order = order_base_2(ent->rb_key.ndescs);
-	struct dentry *dir;
-
-	if (!mlx5_debugfs_root || dev->is_rep)
-		return;
-
-	if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
-		order = MLX5_IMR_KSM_CACHE_ENTRY + 2;
-
-	sprintf(ent->name, "%d", order);
-	dir = debugfs_create_dir(ent->name, dev->cache.fs_root);
-	debugfs_create_file("size", 0600, dir, ent, &size_fops);
-	debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
-	debugfs_create_ulong("cur", 0400, dir, &ent->mkeys_queue.ci);
-	debugfs_create_u32("miss", 0600, dir, &ent->miss);
-}
+	struct mlx5_ib_mr *mr;
+	int err;
 
-static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
-{
-	struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev);
-	struct mlx5_mkey_cache *cache = &dev->cache;
+	mr = kzalloc_obj(*mr);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
 
-	if (!mlx5_debugfs_root || dev->is_rep)
-		return;
+	mr->ibmr.frmr.key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);
+	mr->ibmr.frmr.key.access_flags =
+		get_unchangeable_access_flags(dev, access_flags);
+	mr->ibmr.frmr.key.num_dma_blocks =
+		ib_umem_num_dma_blocks(umem, page_size);
+	mr->ibmr.frmr.key.vendor_key =
+		access_mode == MLX5_MKC_ACCESS_MODE_KSM ?
+			MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK :
+			0;
+
+	/* Normalize ph: swap 0 and MLX5_IB_NO_PH */
+	if (ph == MLX5_IB_NO_PH || ph == 0)
+		ph ^= MLX5_IB_NO_PH;
+
+	mr->ibmr.frmr.key.kernel_vendor_key =
+		st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT);
+	err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr);
+	if (err) {
+		kfree(mr);
+		return ERR_PTR(err);
+	}
+	mr->mmkey.key = mr->ibmr.frmr.handle;
+	init_waitqueue_head(&mr->mmkey.wait);
 
-	cache->fs_root = debugfs_create_dir("mr_cache", dbg_root);
+	return mr;
 }
 
-static void delay_time_func(struct timer_list *t)
+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
+				       int access_flags, int access_mode,
+				       int ndescs)
 {
-	struct mlx5_ib_dev *dev = timer_container_of(dev, t, delay_timer);
-
-	WRITE_ONCE(dev->fill_delay, 0);
-}
+	struct ib_frmr_key key = {
+		.access_flags =
+			get_unchangeable_access_flags(dev, access_flags),
+		.vendor_key = access_mode == MLX5_MKC_ACCESS_MODE_MTT ?
+				      0 :
+				      MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK,
+		.num_dma_blocks = ndescs,
+		.kernel_vendor_key = 0, /* no PH and no ST index */
+	};
+	struct mlx5_ib_mr *mr;
+	int ret;
 
-static int mlx5r_mkeys_init(struct mlx5_cache_ent *ent)
-{
-	struct mlx5_mkeys_page *page;
+	mr = kzalloc_obj(*mr);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
 
-	page = kzalloc_obj(*page);
-	if (!page)
-		return -ENOMEM;
-	INIT_LIST_HEAD(&ent->mkeys_queue.pages_list);
-	spin_lock_init(&ent->mkeys_queue.lock);
-	list_add_tail(&page->list, &ent->mkeys_queue.pages_list);
-	ent->mkeys_queue.num_pages++;
-	return 0;
-}
+	init_waitqueue_head(&mr->mmkey.wait);
 
-static void mlx5r_mkeys_uninit(struct mlx5_cache_ent *ent)
-{
-	struct mlx5_mkeys_page *page;
+	mr->ibmr.frmr.key = key;
+	ret = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr);
+	if (ret) {
+		kfree(mr);
+		return ERR_PTR(ret);
+	}
+	mr->mmkey.key = mr->ibmr.frmr.handle;
+	mr->mmkey.type = MLX5_MKEY_MR;
 
-	WARN_ON(ent->mkeys_queue.ci || ent->mkeys_queue.num_pages > 1);
-	page = list_last_entry(&ent->mkeys_queue.pages_list,
-			       struct mlx5_mkeys_page, list);
-	list_del(&page->list);
-	kfree(page);
+	return mr;
 }
 
-struct mlx5_cache_ent *
-mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev,
-			      struct mlx5r_cache_rb_key rb_key,
-			      bool persistent_entry)
+static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key,
+			      u32 *handles, unsigned int count)
 {
-	struct mlx5_cache_ent *ent;
-	int order;
-	int ret;
+	int access_mode =
+		key->vendor_key & MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK ?
+			MLX5_MKC_ACCESS_MODE_KSM :
+			MLX5_MKC_ACCESS_MODE_MTT;
 
-	ent = kzalloc_obj(*ent);
-	if (!ent)
-		return ERR_PTR(-ENOMEM);
+	struct mlx5_ib_dev *dev = to_mdev(device);
+	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+	u16 st_index;
+	void *mkc;
+	u32 *in;
+	int err, i;
+	u8 ph;
 
-	ret = mlx5r_mkeys_init(ent);
-	if (ret)
-		goto mkeys_err;
-	ent->rb_key = rb_key;
-	ent->dev = dev;
-	ent->is_tmp = !persistent_entry;
+	in = kzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 
-	INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
+	set_mkc_access_pd_addr_fields(mkc, key->access_flags, 0, dev->umrc.pd);
+	MLX5_SET(mkc, mkc, free, 1);
+	MLX5_SET(mkc, mkc, umr_en, 1);
+	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
+	MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
+	MLX5_SET(mkc, mkc, ma_translation_mode, !!key->ats);
+	MLX5_SET(mkc, mkc, translations_octword_size,
+		 get_mkc_octo_size(access_mode, key->num_dma_blocks));
+	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 
-	ret = mlx5_cache_ent_insert(&dev->cache, ent);
-	if (ret)
-		goto ent_insert_err;
-
-	if (persistent_entry) {
-		if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
-			order = MLX5_IMR_KSM_CACHE_ENTRY;
-		else
-			order = order_base_2(rb_key.ndescs) - 2;
-
-		if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
-		    !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
-		    mlx5r_umr_can_load_pas(dev, 0))
-			ent->limit = dev->mdev->profile.mr_cache[order].limit;
-		else
-			ent->limit = 0;
-
-		mlx5_mkey_cache_debugfs_add_ent(dev, ent);
+	st_index = key->kernel_vendor_key &
+		   MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK;
+	ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK;
+	if (ph) {
+		/* Normalize ph: swap MLX5_IB_NO_PH for 0 */
+		if (ph == MLX5_IB_NO_PH)
+			ph = 0;
+		MLX5_SET(mkc, mkc, pcie_tph_en, 1);
+		MLX5_SET(mkc, mkc, pcie_tph_ph, ph);
+		if (st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
+			MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
+				 st_index);
 	}
 
-	return ent;
-ent_insert_err:
-	mlx5r_mkeys_uninit(ent);
-mkeys_err:
-	kfree(ent);
-	return ERR_PTR(ret);
-}
-
-static void mlx5r_destroy_cache_entries(struct mlx5_ib_dev *dev)
-{
-	struct rb_root *root = &dev->cache.rb_root;
-	struct mlx5_cache_ent *ent;
-	struct rb_node *node;
-
-	mutex_lock(&dev->cache.rb_lock);
-	node = rb_first(root);
-	while (node) {
-		ent = rb_entry(node, struct mlx5_cache_ent, node);
-		node = rb_next(node);
-		clean_keys(dev, ent);
-		rb_erase(&ent->node, root);
-		mlx5r_mkeys_uninit(ent);
-		kfree(ent);
+	for (i = 0; i < count; i++) {
+		assign_mkey_variant(dev, handles + i, in);
+		err = mlx5_core_create_mkey(dev->mdev, handles + i, in, inlen);
+		if (err)
+			goto free_in;
 	}
-	mutex_unlock(&dev->cache.rb_lock);
+free_in:
+	kfree(in);
+	if (err)
+		for (; i > 0; i--)
+			mlx5_core_destroy_mkey(dev->mdev, handles[i]);
+	return err;
 }
 
-int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
+static void mlx5r_destroy_mkeys(struct ib_device *device, u32 *handles,
+				unsigned int count)
 {
-	struct mlx5_mkey_cache *cache = &dev->cache;
-	struct rb_root *root = &dev->cache.rb_root;
-	struct mlx5r_cache_rb_key rb_key = {
-		.access_mode = MLX5_MKC_ACCESS_MODE_MTT,
-		.ph = MLX5_IB_NO_PH,
-	};
-	struct mlx5_cache_ent *ent;
-	struct rb_node *node;
-	int ret;
-	int i;
+	struct mlx5_ib_dev *dev = to_mdev(device);
+	int i, err;
 
-	mutex_init(&dev->slow_path_mutex);
-	mutex_init(&dev->cache.rb_lock);
-	dev->cache.rb_root = RB_ROOT;
-	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
-	if (!cache->wq) {
-		mlx5_ib_warn(dev, "failed to create work queue\n");
-		return -ENOMEM;
+	for (i = 0; i < count; i++) {
+		err = mlx5_core_destroy_mkey(dev->mdev, handles[i]);
+		if (err)
+			pr_warn_ratelimited(
+				"mlx5_ib: failed to destroy mkey %d: %d",
+				handles[i], err);
 	}
+}
 
-	mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
-	timer_setup(&dev->delay_timer, delay_time_func, 0);
-	mlx5_mkey_cache_debugfs_init(dev);
-	mutex_lock(&cache->rb_lock);
-	for (i = 0; i <= mkey_cache_max_order(dev); i++) {
-		rb_key.ndescs = MLX5_MR_CACHE_PERSISTENT_ENTRY_MIN_DESCS << i;
-		ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
-		if (IS_ERR(ent)) {
-			ret = PTR_ERR(ent);
-			goto err;
-		}
-	}
+static int mlx5r_build_frmr_key(struct ib_device *device,
+				const struct ib_frmr_key *in,
+				struct ib_frmr_key *out)
+{
+	struct mlx5_ib_dev *dev = to_mdev(device);
 
-	ret = mlx5_odp_init_mkey_cache(dev);
-	if (ret)
-		goto err;
+	/* check HW capabilities of users requested frmr key */
+	if ((in->ats && !MLX5_CAP_GEN(dev->mdev, ats)) ||
+	    ilog2(in->num_dma_blocks) > mkey_max_umr_order(dev))
+		return -EOPNOTSUPP;
 
-	mutex_unlock(&cache->rb_lock);
-	for (node = rb_first(root); node; node = rb_next(node)) {
-		ent = rb_entry(node, struct mlx5_cache_ent, node);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		queue_adjust_cache_locked(ent);
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-	}
+	if (in->vendor_key & ~MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED)
+		return -EOPNOTSUPP;
 
-	return 0;
+	out->ats = in->ats;
+	out->access_flags =
+		get_unchangeable_access_flags(dev, in->access_flags);
+	out->vendor_key = in->vendor_key;
+	out->num_dma_blocks = in->num_dma_blocks;
 
-err:
-	mutex_unlock(&cache->rb_lock);
-	mlx5_mkey_cache_debugfs_cleanup(dev);
-	mlx5r_destroy_cache_entries(dev);
-	destroy_workqueue(cache->wq);
-	mlx5_ib_warn(dev, "failed to create mkey cache entry\n");
-	return ret;
+	return 0;
 }
 
-void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)
-{
-	struct rb_root *root = &dev->cache.rb_root;
-	struct mlx5_cache_ent *ent;
-	struct rb_node *node;
-
-	if (!dev->cache.wq)
-		return;
-
-	mutex_lock(&dev->cache.rb_lock);
-	for (node = rb_first(root); node; node = rb_next(node)) {
-		ent = rb_entry(node, struct mlx5_cache_ent, node);
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		ent->disabled = true;
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-		cancel_delayed_work(&ent->dwork);
-	}
-	mutex_unlock(&dev->cache.rb_lock);
-
-	/*
-	 * After all entries are disabled and will not reschedule on WQ,
-	 * flush it and all async commands.
-	 */
-	flush_workqueue(dev->cache.wq);
+static struct ib_frmr_pool_ops mlx5r_frmr_pool_ops = {
+	.create_frmrs = mlx5r_create_mkeys,
+	.destroy_frmrs = mlx5r_destroy_mkeys,
+	.build_key = mlx5r_build_frmr_key,
+};
 
-	mlx5_mkey_cache_debugfs_cleanup(dev);
-	mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
+int mlx5r_frmr_pools_init(struct ib_device *device)
+{
+	struct mlx5_ib_dev *dev = to_mdev(device);
 
-	/* At this point all entries are disabled and have no concurrent work. */
-	mlx5r_destroy_cache_entries(dev);
+	mutex_init(&dev->slow_path_mutex);
+	return ib_frmr_pools_init(device, &mlx5r_frmr_pool_ops);
+}
 
-	destroy_workqueue(dev->cache.wq);
-	timer_delete_sync(&dev->delay_timer);
+void mlx5r_frmr_pools_cleanup(struct ib_device *device)
+{
+	ib_frmr_pools_cleanup(device);
 }
 
 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
@@ -1108,13 +416,6 @@ static int get_octo_len(u64 addr, u64 len, int page_shift)
 	return (npages + 1) / 2;
 }
 
-static int mkey_cache_max_order(struct mlx5_ib_dev *dev)
-{
-	if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
-		return MKEY_CACHE_LAST_STD_ENTRY;
-	return MLX5_MAX_UMR_SHIFT;
-}
-
 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
 			  u64 length, int access_flags, u64 iova)
 {
@@ -1143,8 +444,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
 					     u16 st_index, u8 ph)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5r_cache_rb_key rb_key = {};
-	struct mlx5_cache_ent *ent;
 	struct mlx5_ib_mr *mr;
 	unsigned long page_size;
 
@@ -1156,33 +455,12 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
 	if (WARN_ON(!page_size))
 		return ERR_PTR(-EINVAL);
 
-	rb_key.access_mode = access_mode;
-	rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size);
-	rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);
-	rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags);
-	rb_key.st_index = st_index;
-	rb_key.ph = ph;
-	ent = mkey_cache_ent_from_rb_key(dev, rb_key);
-	/*
-	 * If the MR can't come from the cache then synchronously create an uncached
-	 * one.
-	 */
-	if (!ent) {
-		mutex_lock(&dev->slow_path_mutex);
-		mr = reg_create(pd, umem, iova, access_flags, page_size, false, access_mode,
-				st_index, ph);
-		mutex_unlock(&dev->slow_path_mutex);
-		if (IS_ERR(mr))
-			return mr;
-		mr->mmkey.rb_key = rb_key;
-		mr->mmkey.cacheable = true;
-		return mr;
-	}
-
-	mr = _mlx5_mr_cache_alloc(dev, ent);
+	mr = _mlx5_frmr_pool_alloc(dev, umem, access_flags, access_mode,
+				   page_size, st_index, ph);
 	if (IS_ERR(mr))
 		return mr;
 
+	mr->mmkey.type = MLX5_MKEY_MR;
 	mr->ibmr.pd = pd;
 	mr->umem = umem;
 	mr->page_shift = order_base_2(page_size);
@@ -1619,7 +897,7 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)
 
 static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {
 	.allow_peer2peer = 1,
-	.move_notify = mlx5_ib_dmabuf_invalidate_cb,
+	.invalidate_mappings = mlx5_ib_dmabuf_invalidate_cb,
 };
 
 static struct ib_mr *
@@ -1812,18 +1090,24 @@ static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr,
 				  unsigned long *page_size)
 {
 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
+	u8 access_mode;
 
-	/* We only track the allocated sizes of MRs from the cache */
-	if (!mr->mmkey.cache_ent)
+	/* We only track the allocated sizes of MRs from the frmr pools */
+	if (!mr->ibmr.frmr.pool)
 		return false;
 	if (!mlx5r_umr_can_load_pas(dev, new_umem->length))
 		return false;
 
-	*page_size = mlx5_umem_mkc_find_best_pgsz(
-		dev, new_umem, iova, mr->mmkey.cache_ent->rb_key.access_mode);
+	access_mode = mr->ibmr.frmr.key.vendor_key &
+				      MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK ?
+			      MLX5_MKC_ACCESS_MODE_KSM :
+			      MLX5_MKC_ACCESS_MODE_MTT;
+
+	*page_size =
+		mlx5_umem_mkc_find_best_pgsz(dev, new_umem, iova, access_mode);
 	if (WARN_ON(!*page_size))
 		return false;
-	return (mr->mmkey.cache_ent->rb_key.ndescs) >=
+	return (mr->ibmr.frmr.key.num_dma_blocks) >=
 	       ib_umem_num_dma_blocks(new_umem, *page_size);
 }
 
@@ -1884,7 +1168,8 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 	int err;
 
 	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || mr->data_direct ||
-	    mr->mmkey.rb_key.ph != MLX5_IB_NO_PH)
+	    (mr->ibmr.frmr.key.kernel_vendor_key &
+	     MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK) != 0)
 		return ERR_PTR(-EOPNOTSUPP);
 
 	mlx5_ib_dbg(
@@ -2025,47 +1310,6 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
 	}
 }
 
-static int cache_ent_find_and_store(struct mlx5_ib_dev *dev,
-				    struct mlx5_ib_mr *mr)
-{
-	struct mlx5_mkey_cache *cache = &dev->cache;
-	struct mlx5_cache_ent *ent;
-	int ret;
-
-	if (mr->mmkey.cache_ent) {
-		spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
-		goto end;
-	}
-
-	mutex_lock(&cache->rb_lock);
-	ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key);
-	if (ent) {
-		if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) {
-			if (ent->disabled) {
-				mutex_unlock(&cache->rb_lock);
-				return -EOPNOTSUPP;
-			}
-			mr->mmkey.cache_ent = ent;
-			spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
-			mutex_unlock(&cache->rb_lock);
-			goto end;
-		}
-	}
-
-	ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false);
-	mutex_unlock(&cache->rb_lock);
-	if (IS_ERR(ent))
-		return PTR_ERR(ent);
-
-	mr->mmkey.cache_ent = ent;
-	spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
-
-end:
-	ret = push_mkey_locked(mr->mmkey.cache_ent, mr->mmkey.key);
-	spin_unlock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);
-	return ret;
-}
-
 static int mlx5_ib_revoke_data_direct_mr(struct mlx5_ib_mr *mr)
 {
 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
@@ -2131,33 +1375,12 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)
 	bool is_odp_dma_buf = is_dmabuf_mr(mr) &&
 			      !to_ib_umem_dmabuf(mr->umem)->pinned;
 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
-	struct mlx5_cache_ent *ent = mr->mmkey.cache_ent;
 	bool is_odp = is_odp_mr(mr);
-	bool from_cache = !!ent;
 	int ret;
 
-	if (mr->mmkey.cacheable && !mlx5_umr_revoke_mr_with_lock(mr) &&
-	    !cache_ent_find_and_store(dev, mr)) {
-		ent = mr->mmkey.cache_ent;
-		/* upon storing to a clean temp entry - schedule its cleanup */
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		if (from_cache)
-			ent->in_use--;
-		if (ent->is_tmp && !ent->tmp_cleanup_scheduled) {
-			mod_delayed_work(ent->dev->cache.wq, &ent->dwork,
-					 secs_to_jiffies(30));
-			ent->tmp_cleanup_scheduled = true;
-		}
-		spin_unlock_irq(&ent->mkeys_queue.lock);
+	if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) &&
+	    !ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr))
 		return 0;
-	}
-
-	if (ent) {
-		spin_lock_irq(&ent->mkeys_queue.lock);
-		ent->in_use--;
-		mr->mmkey.cache_ent = NULL;
-		spin_unlock_irq(&ent->mkeys_queue.lock);
-	}
 
 	if (is_odp)
 		mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex);
@@ -2241,7 +1464,7 @@ static int __mlx5_ib_dereg_mr(struct ib_mr *ibmr)
 			mlx5_ib_free_odp_mr(mr);
 	}
 
-	if (!mr->mmkey.cache_ent)
+	if (!mr->ibmr.frmr.pool)
 		mlx5_free_priv_descs(mr);
 
 	kfree(mr);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 4c1dea4fd526a..1119ce163ea78 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -1875,25 +1875,6 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
 	return err;
 }
 
-int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
-{
-	struct mlx5r_cache_rb_key rb_key = {
-		.access_mode = MLX5_MKC_ACCESS_MODE_KSM,
-		.ndescs = mlx5_imr_ksm_entries,
-		.ph = MLX5_IB_NO_PH,
-	};
-	struct mlx5_cache_ent *ent;
-
-	if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
-		return 0;
-
-	ent = mlx5r_cache_create_ent_locked(dev, rb_key, true);
-	if (IS_ERR(ent))
-		return PTR_ERR(ent);
-
-	return 0;
-}
-
 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
 	.advise_mr = mlx5_ib_advise_mr,
 };
diff --git a/drivers/infiniband/hw/mlx5/qos.c b/drivers/infiniband/hw/mlx5/qos.c
index dce92554142a2..ab7f5db18c933 100644
--- a/drivers/infiniband/hw/mlx5/qos.c
+++ b/drivers/infiniband/hw/mlx5/qos.c
@@ -45,7 +45,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_PP_OBJ_ALLOC)(
 		return -EINVAL;
 
 	dev = to_mdev(c->ibucontext.device);
-	pp_entry = kzalloc(sizeof(*pp_entry), GFP_KERNEL);
+	pp_entry = kzalloc_obj(*pp_entry);
 	if (!pp_entry)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 47de8cc9937e9..59f9ddb35d462 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1273,7 +1273,7 @@ static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 		}
 		return MLX5_TIMESTAMP_FORMAT_REAL_TIME;
 	}
-	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
+	if (cq->private_flags & MLX5_IB_CQ_PR_TIMESTAMP_COMPLETION) {
 		if (!fr_sup) {
 			mlx5_ib_dbg(dev,
 				    "Free running TS format is not supported\n");
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 4e562e0dd9e12..29488fba21a03 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -2,6 +2,7 @@
 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
 
 #include <rdma/ib_umem_odp.h>
+#include <rdma/iter.h>
 #include "mlx5_ib.h"
 #include "umr.h"
 #include "wr.h"
diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h
index e9361f0140e7b..7eeaf6a94c974 100644
--- a/drivers/infiniband/hw/mlx5/umr.h
+++ b/drivers/infiniband/hw/mlx5/umr.h
@@ -9,6 +9,7 @@
 
 #define MLX5_MAX_UMR_SHIFT 16
 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
+#define MLX5_MAX_UMR_EXTENDED_SHIFT 43
 
 #define MLX5_IB_UMR_OCTOWORD	       16
 #define MLX5_IB_UMR_XLT_ALIGNMENT      64
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 5c182ae4fc2f6..6a0795332616d 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -35,8 +35,8 @@
  */
 
 #include <rdma/ib_smi.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include <linux/sched.h>
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h b/drivers/infiniband/hw/ocrdma/ocrdma.h
index 5584b781e2e8a..da2deae6857b2 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma.h
@@ -190,8 +190,8 @@ struct ocrdma_mr {
 	struct ib_mr ibmr;
 	struct ib_umem *umem;
 	struct ocrdma_hw_mr hwmr;
-	u64 *pages;
 	u32 npages;
+	u64 pages[];
 };
 
 struct ocrdma_stats {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index e89be2fbd5eb2..eb922b9b00753 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -45,9 +45,9 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/iw_cm.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include "ocrdma.h"
@@ -794,7 +794,7 @@ static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 	void *va;
 	dma_addr_t pa;
 
-	mr->pbl_table = kzalloc_objs(struct ocrdma_pbl, mr->num_pbls);
+	mr->pbl_table = kzalloc_objs(*mr->pbl_table, mr->num_pbls);
 
 	if (!mr->pbl_table)
 		return -ENOMEM;
@@ -910,7 +910,6 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 
 	(void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
 
-	kfree(mr->pages);
 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 
 	/* it could be user registered memory. */
@@ -1253,12 +1252,11 @@ static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
 
 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
 {
-	qp->wqe_wr_id_tbl =
-	    kzalloc_objs(*(qp->wqe_wr_id_tbl), qp->sq.max_cnt);
+	qp->wqe_wr_id_tbl = kzalloc_objs(*qp->wqe_wr_id_tbl, qp->sq.max_cnt);
 	if (qp->wqe_wr_id_tbl == NULL)
 		return -ENOMEM;
-	qp->rqe_wr_id_tbl =
-	    kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
+
+	qp->rqe_wr_id_tbl = kzalloc_objs(*qp->rqe_wr_id_tbl, qp->rq.max_cnt);
 	if (qp->rqe_wr_id_tbl == NULL)
 		return -ENOMEM;
 
@@ -1788,8 +1786,8 @@ int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
 		return status;
 
 	if (!udata) {
-		srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
-					     GFP_KERNEL);
+		srq->rqe_wr_id_tbl =
+			kzalloc_objs(*srq->rqe_wr_id_tbl, srq->rq.max_cnt);
 		if (!srq->rqe_wr_id_tbl) {
 			status = -ENOMEM;
 			goto arm_err;
@@ -2909,19 +2907,13 @@ struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	if (max_num_sg > dev->attr.max_pages_per_frmr)
 		return ERR_PTR(-EINVAL);
 
-	mr = kzalloc_obj(*mr);
+	mr = kzalloc_flex(*mr, pages, max_num_sg);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
-	if (!mr->pages) {
-		status = -ENOMEM;
-		goto pl_err;
-	}
-
 	status = ocrdma_get_pbl_info(dev, mr, max_num_sg);
 	if (status)
-		goto pbl_err;
+		goto pl_err;
 	mr->hwmr.fr_mr = 1;
 	mr->hwmr.remote_rd = 0;
 	mr->hwmr.remote_wr = 0;
@@ -2930,7 +2922,7 @@ struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	mr->hwmr.mw_bind = 0;
 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
 	if (status)
-		goto pbl_err;
+		goto pl_err;
 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
 	if (status)
 		goto mbx_err;
@@ -2941,8 +2933,6 @@ struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	return &mr->ibmr;
 mbx_err:
 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
-pbl_err:
-	kfree(mr->pages);
 pl_err:
 	kfree(mr);
 	return ERR_PTR(-ENOMEM);
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 33b4a0e6d3a8f..2fa9e07710d31 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -39,9 +39,9 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/iw_cm.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
+#include <rdma/iter.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include <linux/qed/common_hsi.h>
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 763ddc6f25d1a..23e547d4b3a71 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -53,8 +53,8 @@
 #include <linux/pci.h>
 #include <linux/semaphore.h>
 #include <linux/workqueue.h>
-#include <rdma/ib_umem.h>
 #include <rdma/ib_verbs.h>
+#include <rdma/iter.h>
 #include <rdma/vmw_pvrdma-abi.h>
 
 #include "pvrdma_ring.h"
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
index 0864ad25b9d2c..64ce5cf5fb96f 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
@@ -65,8 +65,7 @@ int pvrdma_page_dir_init(struct pvrdma_dev *dev, struct pvrdma_page_dir *pdir,
 		goto err;
 
 	pdir->ntables = PVRDMA_PAGE_DIR_TABLE(npages - 1) + 1;
-	pdir->tables = kcalloc(pdir->ntables, sizeof(*pdir->tables),
-			       GFP_KERNEL);
+	pdir->tables = kzalloc_objs(*pdir->tables, pdir->ntables);
 	if (!pdir->tables)
 		goto err;
 
diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index 46e3b3e0643ab..473f464f33fa2 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -9,6 +9,11 @@
 #include <rdma/uverbs_ioctl.h>
 #include "mmap.h"
 
+/* number of reserved mmaps for the driver */
+#define MMAP_RESERVED 256
+/* start point for dynamic offsets */
+#define MMAP_OFFSET_START (MMAP_RESERVED * PAGE_SIZE)
+
 /**
  * rvt_mmap_init - init link list and lock for mem map
  * @rdi: rvt dev struct
@@ -17,7 +22,7 @@ void rvt_mmap_init(struct rvt_dev_info *rdi)
 {
 	INIT_LIST_HEAD(&rdi->pending_mmaps);
 	spin_lock_init(&rdi->pending_lock);
-	rdi->mmap_offset = PAGE_SIZE;
+	rdi->mmap_offset = MMAP_OFFSET_START;
 	spin_lock_init(&rdi->mmap_offset_lock);
 }
 
@@ -73,6 +78,13 @@ int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 	struct rvt_mmap_info *ip, *pp;
 	int ret = -EINVAL;
 
+	/* call driver if in reserved range */
+	if (offset < MMAP_OFFSET_START) {
+		if (rdi->driver_f.mmap)
+			return rdi->driver_f.mmap(context, vma);
+		return -EINVAL;
+	}
+
 	/*
 	 * Search the device's list of objects waiting for a mmap call.
 	 * Normally, this list is very short since a call to create a
@@ -129,9 +141,9 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, u32 size,
 
 	spin_lock_irq(&rdi->mmap_offset_lock);
 	if (rdi->mmap_offset == 0)
-		rdi->mmap_offset = ALIGN(PAGE_SIZE, SHMLBA);
+		rdi->mmap_offset = MMAP_OFFSET_START;
 	ip->offset = rdi->mmap_offset;
-	rdi->mmap_offset += ALIGN(size, SHMLBA);
+	rdi->mmap_offset += PAGE_SIZE;
 	spin_unlock_irq(&rdi->mmap_offset_lock);
 
 	INIT_LIST_HEAD(&ip->pending_mmaps);
@@ -159,9 +171,9 @@ void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
 
 	spin_lock_irq(&rdi->mmap_offset_lock);
 	if (rdi->mmap_offset == 0)
-		rdi->mmap_offset = PAGE_SIZE;
+		rdi->mmap_offset = MMAP_OFFSET_START;
 	ip->offset = rdi->mmap_offset;
-	rdi->mmap_offset += size;
+	rdi->mmap_offset += PAGE_SIZE;
 	spin_unlock_irq(&rdi->mmap_offset_lock);
 
 	ip->size = size;
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c1199ea5d41fd..b519d9d0e4291 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -2707,7 +2707,7 @@ int rvt_qp_iter_next(struct rvt_qp_iter *iter)
 				struct rvt_ibport *rvp;
 				int pidx;
 
-				pidx = n % rdi->ibdev.phys_port_cnt;
+				pidx = n / 2; /* QP0 and QP1 */
 				rvp = rdi->ports[pidx];
 				qp = rcu_dereference(rvp->qp[n & 1]);
 			} else {
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 0c28b412d81a4..033d8932aff16 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -244,6 +244,10 @@ static int rvt_query_gid(struct ib_device *ibdev, u32 port_num,
  */
 static int rvt_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 {
+	struct rvt_dev_info *rdi = ib_to_rvt(uctx->device);
+
+	if (rdi->driver_f.alloc_ucontext)
+		return rdi->driver_f.alloc_ucontext(uctx, udata);
 	return 0;
 }
 
@@ -253,6 +257,10 @@ static int rvt_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
  */
 static void rvt_dealloc_ucontext(struct ib_ucontext *context)
 {
+	struct rvt_dev_info *rdi = ib_to_rvt(context->device);
+
+	if (rdi->driver_f.dealloc_ucontext)
+		rdi->driver_f.dealloc_ucontext(context);
 	return;
 }
 
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 711f73e0bbb1c..9faf8c09aa8e4 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -37,6 +37,7 @@ static char *resp_state_name[] = {
 	[RESPST_ERR_MISSING_OPCODE_LAST_D1E]	= "ERR_MISSING_OPCODE_LAST_D1E",
 	[RESPST_ERR_TOO_MANY_RDMA_ATM_REQ]	= "ERR_TOO_MANY_RDMA_ATM_REQ",
 	[RESPST_ERR_RNR]			= "ERR_RNR",
+	[RESPST_ERR_RKEY_VIOLATION_EVENT]	= "ERR_RKEY_VIOLATION_EVENT",
 	[RESPST_ERR_RKEY_VIOLATION]		= "ERR_RKEY_VIOLATION",
 	[RESPST_ERR_INVALIDATE_RKEY]		= "ERR_INVALIDATE_RKEY_VIOLATION",
 	[RESPST_ERR_LENGTH]			= "ERR_LENGTH",
@@ -423,6 +424,19 @@ static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
 	qp->resp.resid = sizeof(u64);
 }
 
+/* Transition to an rkey violation state. C9-222.1 requires an async event
+ * at the responder, but only if the error cannot be attached to an RX WQE.
+ * WRITE_WITH_IMM is the only op that might have that more precise RX WQE
+ * to pin the error on.
+ */
+static enum resp_states get_rkey_violation_state(struct rxe_pkt_info *pkt)
+{
+	if (pkt->mask & RXE_IMMDT_MASK)
+		return RESPST_ERR_RKEY_VIOLATION;
+
+	return RESPST_ERR_RKEY_VIOLATION_EVENT;
+}
+
 /* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL
  * if an invalid rkey is received or the rdma length is zero. For middle
  * or last packets use the stored value of mr.
@@ -486,14 +500,14 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 		mw = rxe_lookup_mw(qp, access, rkey);
 		if (!mw) {
 			rxe_dbg_qp(qp, "no MW matches rkey %#x\n", rkey);
-			state = RESPST_ERR_RKEY_VIOLATION;
+			state = get_rkey_violation_state(pkt);
 			goto err;
 		}
 
 		mr = mw->mr;
 		if (!mr) {
 			rxe_dbg_qp(qp, "MW doesn't have an MR\n");
-			state = RESPST_ERR_RKEY_VIOLATION;
+			state = get_rkey_violation_state(pkt);
 			goto err;
 		}
 
@@ -507,7 +521,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 		mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);
 		if (!mr) {
 			rxe_dbg_qp(qp, "no MR matches rkey %#x\n", rkey);
-			state = RESPST_ERR_RKEY_VIOLATION;
+			state = get_rkey_violation_state(pkt);
 			goto err;
 		}
 	}
@@ -521,7 +535,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 	}
 
 	if (mr_check_range(mr, va + qp->resp.offset, resid)) {
-		state = RESPST_ERR_RKEY_VIOLATION;
+		state = get_rkey_violation_state(pkt);
 		goto err;
 	}
 
@@ -586,7 +600,7 @@ static enum resp_states write_data_in(struct rxe_qp *qp,
 	err = rxe_mr_copy(qp->resp.mr, qp->resp.va + qp->resp.offset,
 			  payload_addr(pkt), data_len, RXE_TO_MR_OBJ);
 	if (err) {
-		rc = RESPST_ERR_RKEY_VIOLATION;
+		rc = get_rkey_violation_state(pkt);
 		goto out;
 	}
 
@@ -667,7 +681,7 @@ static enum resp_states process_flush(struct rxe_qp *qp,
 
 	if (res->flush.type & IB_FLUSH_PERSISTENT) {
 		if (rxe_flush_pmem_iova(mr, start, length))
-			return RESPST_ERR_RKEY_VIOLATION;
+			return get_rkey_violation_state(pkt);
 		/* Make data persistent. */
 		wmb();
 	} else if (res->flush.type & IB_FLUSH_GLOBAL) {
@@ -1383,6 +1397,20 @@ static enum resp_states duplicate_request(struct rxe_qp *qp,
 	return rc;
 }
 
+static void do_qp_event(struct rxe_qp *qp, enum ib_event_type etype)
+{
+	struct ib_event event;
+	struct ib_qp *ibqp = &qp->ibqp;
+
+	event.event = etype;
+	event.device = ibqp->device;
+	event.element.qp = ibqp;
+	if (ibqp->event_handler) {
+		rxe_dbg_qp(qp, "reporting QP event %d\n", etype);
+		ibqp->event_handler(&event, ibqp->qp_context);
+	}
+}
+
 /* Process a class A or C. Both are treated the same in this implementation. */
 static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome,
 			      enum ib_wc_status status)
@@ -1476,14 +1504,9 @@ static void flush_recv_queue(struct rxe_qp *qp, bool notify)
 	int err;
 
 	if (qp->srq) {
-		if (notify && qp->ibqp.event_handler) {
-			struct ib_event ev;
+		if (notify && qp->ibqp.event_handler)
+			do_qp_event(qp, IB_EVENT_QP_LAST_WQE_REACHED);
 
-			ev.device = qp->ibqp.device;
-			ev.element.qp = &qp->ibqp;
-			ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
-			qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
-		}
 		return;
 	}
 
@@ -1613,6 +1636,13 @@ int rxe_receiver(struct rxe_qp *qp)
 			state = RESPST_CLEANUP;
 			break;
 
+		case RESPST_ERR_RKEY_VIOLATION_EVENT:
+			if (qp_type(qp) == IB_QPT_RC)
+				do_qp_event(qp, IB_EVENT_QP_ACCESS_ERR);
+
+			state = RESPST_ERR_RKEY_VIOLATION;
+			break;
+
 		case RESPST_ERR_RKEY_VIOLATION:
 			if (qp_type(qp) == IB_QPT_RC) {
 				/* Class C */
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index fb149f37e91d9..d92f80d16f784 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -154,6 +154,7 @@ enum resp_states {
 	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
 	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
 	RESPST_ERR_RNR,
+	RESPST_ERR_RKEY_VIOLATION_EVENT,
 	RESPST_ERR_RKEY_VIOLATION,
 	RESPST_ERR_INVALIDATE_RKEY,
 	RESPST_ERR_LENGTH,
diff --git a/drivers/infiniband/ulp/Makefile b/drivers/infiniband/ulp/Makefile
index 4d0004b58377d..51b0d41699b8c 100644
--- a/drivers/infiniband/ulp/Makefile
+++ b/drivers/infiniband/ulp/Makefile
@@ -4,5 +4,4 @@ obj-$(CONFIG_INFINIBAND_SRP)		+= srp/
 obj-$(CONFIG_INFINIBAND_SRPT)		+= srpt/
 obj-$(CONFIG_INFINIBAND_ISER)		+= iser/
 obj-$(CONFIG_INFINIBAND_ISERT)		+= isert/
-obj-$(CONFIG_INFINIBAND_OPA_VNIC)	+= opa_vnic/
 obj-$(CONFIG_INFINIBAND_RTRS)		+= rtrs/
diff --git a/drivers/infiniband/ulp/opa_vnic/Kconfig b/drivers/infiniband/ulp/opa_vnic/Kconfig
deleted file mode 100644
index 4d43d055fa8e5..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/Kconfig
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config INFINIBAND_OPA_VNIC
-	tristate "Cornelis OPX VNIC support"
-	depends on X86_64 && INFINIBAND
-	help
-	This is Omni-Path Express (OPX) Virtual Network Interface Controller (VNIC)
-	driver for Ethernet over Omni-Path feature. It implements the HW
-	independent VNIC functionality. It interfaces with Linux stack for
-	data path and IB MAD for the control path.
diff --git a/drivers/infiniband/ulp/opa_vnic/Makefile b/drivers/infiniband/ulp/opa_vnic/Makefile
deleted file mode 100644
index 196183817cdc5..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Makefile - Cornelis Omni-Path Express Virtual Network Controller driver
-# Copyright(c) 2017, Intel Corporation.
-# Copyright(c) 2021, Cornelis Networks.
-#
-obj-$(CONFIG_INFINIBAND_OPA_VNIC) += opa_vnic.o
-
-opa_vnic-y := opa_vnic_netdev.o opa_vnic_encap.o opa_vnic_ethtool.o \
-              opa_vnic_vema.o opa_vnic_vema_iface.o
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
deleted file mode 100644
index 53dcf06fbee02..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.c
+++ /dev/null
@@ -1,513 +0,0 @@
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPA VNIC encapsulation/decapsulation function.
- */
-
-#include <linux/if_ether.h>
-#include <linux/if_vlan.h>
-
-#include "opa_vnic_internal.h"
-
-/* OPA 16B Header fields */
-#define OPA_16B_LID_MASK        0xFFFFFull
-#define OPA_16B_SLID_HIGH_SHFT  8
-#define OPA_16B_SLID_MASK       0xF00ull
-#define OPA_16B_DLID_MASK       0xF000ull
-#define OPA_16B_DLID_HIGH_SHFT  12
-#define OPA_16B_LEN_SHFT        20
-#define OPA_16B_SC_SHFT         20
-#define OPA_16B_RC_SHFT         25
-#define OPA_16B_PKEY_SHFT       16
-
-#define OPA_VNIC_L4_HDR_SHFT    16
-
-/* L2+L4 hdr len is 20 bytes (5 quad words) */
-#define OPA_VNIC_HDR_QW_LEN   5
-
-static inline void opa_vnic_make_header(u8 *hdr, u32 slid, u32 dlid, u16 len,
-					u16 pkey, u16 entropy, u8 sc, u8 rc,
-					u8 l4_type, u16 l4_hdr)
-{
-	/* h[1]: LT=1, 16B L2=10 */
-	u32 h[OPA_VNIC_HDR_QW_LEN] = {0, 0xc0000000, 0, 0, 0};
-
-	h[2] = l4_type;
-	h[3] = entropy;
-	h[4] = l4_hdr << OPA_VNIC_L4_HDR_SHFT;
-
-	/* Extract and set 4 upper bits and 20 lower bits of the lids */
-	h[0] |= (slid & OPA_16B_LID_MASK);
-	h[2] |= ((slid >> (20 - OPA_16B_SLID_HIGH_SHFT)) & OPA_16B_SLID_MASK);
-
-	h[1] |= (dlid & OPA_16B_LID_MASK);
-	h[2] |= ((dlid >> (20 - OPA_16B_DLID_HIGH_SHFT)) & OPA_16B_DLID_MASK);
-
-	h[0] |= (len << OPA_16B_LEN_SHFT);
-	h[1] |= (rc << OPA_16B_RC_SHFT);
-	h[1] |= (sc << OPA_16B_SC_SHFT);
-	h[2] |= ((u32)pkey << OPA_16B_PKEY_SHFT);
-
-	memcpy(hdr, h, OPA_VNIC_HDR_LEN);
-}
-
-/*
- * Using a simple hash table for mac table implementation with the last octet
- * of mac address as a key.
- */
-static void opa_vnic_free_mac_tbl(struct hlist_head *mactbl)
-{
-	struct opa_vnic_mac_tbl_node *node;
-	struct hlist_node *tmp;
-	int bkt;
-
-	if (!mactbl)
-		return;
-
-	vnic_hash_for_each_safe(mactbl, bkt, tmp, node, hlist) {
-		hash_del(&node->hlist);
-		kfree(node);
-	}
-	kfree(mactbl);
-}
-
-static struct hlist_head *opa_vnic_alloc_mac_tbl(void)
-{
-	u32 size = sizeof(struct hlist_head) * OPA_VNIC_MAC_TBL_SIZE;
-	struct hlist_head *mactbl;
-
-	mactbl = kzalloc(size, GFP_KERNEL);
-	if (!mactbl)
-		return ERR_PTR(-ENOMEM);
-
-	vnic_hash_init(mactbl);
-	return mactbl;
-}
-
-/* opa_vnic_release_mac_tbl - empty and free the mac table */
-void opa_vnic_release_mac_tbl(struct opa_vnic_adapter *adapter)
-{
-	struct hlist_head *mactbl;
-
-	mutex_lock(&adapter->mactbl_lock);
-	mactbl = rcu_access_pointer(adapter->mactbl);
-	rcu_assign_pointer(adapter->mactbl, NULL);
-	synchronize_rcu();
-	opa_vnic_free_mac_tbl(mactbl);
-	adapter->info.vport.mac_tbl_digest = 0;
-	mutex_unlock(&adapter->mactbl_lock);
-}
-
-/*
- * opa_vnic_query_mac_tbl - query the mac table for a section
- *
- * This function implements query of specific function of the mac table.
- * The function also expects the requested range to be valid.
- */
-void opa_vnic_query_mac_tbl(struct opa_vnic_adapter *adapter,
-			    struct opa_veswport_mactable *tbl)
-{
-	struct opa_vnic_mac_tbl_node *node;
-	struct hlist_head *mactbl;
-	int bkt;
-	u16 loffset, lnum_entries;
-
-	rcu_read_lock();
-	mactbl = rcu_dereference(adapter->mactbl);
-	if (!mactbl)
-		goto get_mac_done;
-
-	loffset = be16_to_cpu(tbl->offset);
-	lnum_entries = be16_to_cpu(tbl->num_entries);
-
-	vnic_hash_for_each(mactbl, bkt, node, hlist) {
-		struct __opa_vnic_mactable_entry *nentry = &node->entry;
-		struct opa_veswport_mactable_entry *entry;
-
-		if ((node->index < loffset) ||
-		    (node->index >= (loffset + lnum_entries)))
-			continue;
-
-		/* populate entry in the tbl corresponding to the index */
-		entry = &tbl->tbl_entries[node->index - loffset];
-		memcpy(entry->mac_addr, nentry->mac_addr,
-		       ARRAY_SIZE(entry->mac_addr));
-		memcpy(entry->mac_addr_mask, nentry->mac_addr_mask,
-		       ARRAY_SIZE(entry->mac_addr_mask));
-		entry->dlid_sd = cpu_to_be32(nentry->dlid_sd);
-	}
-	tbl->mac_tbl_digest = cpu_to_be32(adapter->info.vport.mac_tbl_digest);
-get_mac_done:
-	rcu_read_unlock();
-}
-
-/*
- * opa_vnic_update_mac_tbl - update mac table section
- *
- * This function updates the specified section of the mac table.
- * The procedure includes following steps.
- *  - Allocate a new mac (hash) table.
- *  - Add the specified entries to the new table.
- *    (except the ones that are requested to be deleted).
- *  - Add all the other entries from the old mac table.
- *  - If there is a failure, free the new table and return.
- *  - Switch to the new table.
- *  - Free the old table and return.
- *
- * The function also expects the requested range to be valid.
- */
-int opa_vnic_update_mac_tbl(struct opa_vnic_adapter *adapter,
-			    struct opa_veswport_mactable *tbl)
-{
-	struct opa_vnic_mac_tbl_node *node, *new_node;
-	struct hlist_head *new_mactbl, *old_mactbl;
-	int i, bkt, rc = 0;
-	u8 key;
-	u16 loffset, lnum_entries;
-
-	mutex_lock(&adapter->mactbl_lock);
-	/* allocate new mac table */
-	new_mactbl = opa_vnic_alloc_mac_tbl();
-	if (IS_ERR(new_mactbl)) {
-		mutex_unlock(&adapter->mactbl_lock);
-		return PTR_ERR(new_mactbl);
-	}
-
-	loffset = be16_to_cpu(tbl->offset);
-	lnum_entries = be16_to_cpu(tbl->num_entries);
-
-	/* add updated entries to the new mac table */
-	for (i = 0; i < lnum_entries; i++) {
-		struct __opa_vnic_mactable_entry *nentry;
-		struct opa_veswport_mactable_entry *entry =
-							&tbl->tbl_entries[i];
-		u8 *mac_addr = entry->mac_addr;
-		u8 empty_mac[ETH_ALEN] = { 0 };
-
-		v_dbg("new mac entry %4d: %02x:%02x:%02x:%02x:%02x:%02x %x\n",
-		      loffset + i, mac_addr[0], mac_addr[1], mac_addr[2],
-		      mac_addr[3], mac_addr[4], mac_addr[5],
-		      entry->dlid_sd);
-
-		/* if the entry is being removed, do not add it */
-		if (!memcmp(mac_addr, empty_mac, ARRAY_SIZE(empty_mac)))
-			continue;
-
-		node = kzalloc_obj(*node);
-		if (!node) {
-			rc = -ENOMEM;
-			goto updt_done;
-		}
-
-		node->index = loffset + i;
-		nentry = &node->entry;
-		memcpy(nentry->mac_addr, entry->mac_addr,
-		       ARRAY_SIZE(nentry->mac_addr));
-		memcpy(nentry->mac_addr_mask, entry->mac_addr_mask,
-		       ARRAY_SIZE(nentry->mac_addr_mask));
-		nentry->dlid_sd = be32_to_cpu(entry->dlid_sd);
-		key = node->entry.mac_addr[OPA_VNIC_MAC_HASH_IDX];
-		vnic_hash_add(new_mactbl, &node->hlist, key);
-	}
-
-	/* add other entries from current mac table to new mac table */
-	old_mactbl = rcu_access_pointer(adapter->mactbl);
-	if (!old_mactbl)
-		goto switch_tbl;
-
-	vnic_hash_for_each(old_mactbl, bkt, node, hlist) {
-		if ((node->index >= loffset) &&
-		    (node->index < (loffset + lnum_entries)))
-			continue;
-
-		new_node = kzalloc_obj(*new_node);
-		if (!new_node) {
-			rc = -ENOMEM;
-			goto updt_done;
-		}
-
-		new_node->index = node->index;
-		memcpy(&new_node->entry, &node->entry, sizeof(node->entry));
-		key = new_node->entry.mac_addr[OPA_VNIC_MAC_HASH_IDX];
-		vnic_hash_add(new_mactbl, &new_node->hlist, key);
-	}
-
-switch_tbl:
-	/* switch to new table */
-	rcu_assign_pointer(adapter->mactbl, new_mactbl);
-	synchronize_rcu();
-
-	adapter->info.vport.mac_tbl_digest = be32_to_cpu(tbl->mac_tbl_digest);
-updt_done:
-	/* upon failure, free the new table; otherwise, free the old table */
-	if (rc)
-		opa_vnic_free_mac_tbl(new_mactbl);
-	else
-		opa_vnic_free_mac_tbl(old_mactbl);
-
-	mutex_unlock(&adapter->mactbl_lock);
-	return rc;
-}
-
-/* opa_vnic_chk_mac_tbl - check mac table for dlid */
-static uint32_t opa_vnic_chk_mac_tbl(struct opa_vnic_adapter *adapter,
-				     struct ethhdr *mac_hdr)
-{
-	struct opa_vnic_mac_tbl_node *node;
-	struct hlist_head *mactbl;
-	u32 dlid = 0;
-	u8 key;
-
-	rcu_read_lock();
-	mactbl = rcu_dereference(adapter->mactbl);
-	if (unlikely(!mactbl))
-		goto chk_done;
-
-	key = mac_hdr->h_dest[OPA_VNIC_MAC_HASH_IDX];
-	vnic_hash_for_each_possible(mactbl, node, hlist, key) {
-		struct __opa_vnic_mactable_entry *entry = &node->entry;
-
-		/* if related to source mac, skip */
-		if (unlikely(OPA_VNIC_DLID_SD_IS_SRC_MAC(entry->dlid_sd)))
-			continue;
-
-		if (!memcmp(node->entry.mac_addr, mac_hdr->h_dest,
-			    ARRAY_SIZE(node->entry.mac_addr))) {
-			/* mac address found */
-			dlid = OPA_VNIC_DLID_SD_GET_DLID(node->entry.dlid_sd);
-			break;
-		}
-	}
-
-chk_done:
-	rcu_read_unlock();
-	return dlid;
-}
-
-/* opa_vnic_get_dlid - find and return the DLID */
-static uint32_t opa_vnic_get_dlid(struct opa_vnic_adapter *adapter,
-				  struct sk_buff *skb, u8 def_port)
-{
-	struct __opa_veswport_info *info = &adapter->info;
-	struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb);
-	u32 dlid;
-
-	dlid = opa_vnic_chk_mac_tbl(adapter, mac_hdr);
-	if (dlid)
-		return dlid;
-
-	if (is_multicast_ether_addr(mac_hdr->h_dest)) {
-		dlid = info->vesw.u_mcast_dlid;
-	} else {
-		if (is_local_ether_addr(mac_hdr->h_dest)) {
-			dlid = ((uint32_t)mac_hdr->h_dest[5] << 16) |
-				((uint32_t)mac_hdr->h_dest[4] << 8)  |
-				mac_hdr->h_dest[3];
-			if (unlikely(!dlid))
-				v_warn("Null dlid in MAC address\n");
-		} else if (def_port != OPA_VNIC_INVALID_PORT) {
-			if (def_port < OPA_VESW_MAX_NUM_DEF_PORT)
-				dlid = info->vesw.u_ucast_dlid[def_port];
-		}
-	}
-
-	return dlid;
-}
-
-/* opa_vnic_get_sc - return the service class */
-static u8 opa_vnic_get_sc(struct __opa_veswport_info *info,
-			  struct sk_buff *skb)
-{
-	struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb);
-	u16 vlan_tci;
-	u8 sc;
-
-	if (!__vlan_get_tag(skb, &vlan_tci)) {
-		u8 pcp = OPA_VNIC_VLAN_PCP(vlan_tci);
-
-		if (is_multicast_ether_addr(mac_hdr->h_dest))
-			sc = info->vport.pcp_to_sc_mc[pcp];
-		else
-			sc = info->vport.pcp_to_sc_uc[pcp];
-	} else {
-		if (is_multicast_ether_addr(mac_hdr->h_dest))
-			sc = info->vport.non_vlan_sc_mc;
-		else
-			sc = info->vport.non_vlan_sc_uc;
-	}
-
-	return sc;
-}
-
-u8 opa_vnic_get_vl(struct opa_vnic_adapter *adapter, struct sk_buff *skb)
-{
-	struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb);
-	struct __opa_veswport_info *info = &adapter->info;
-	u8 vl;
-
-	if (skb_vlan_tag_present(skb)) {
-		u8 pcp = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT;
-
-		if (is_multicast_ether_addr(mac_hdr->h_dest))
-			vl = info->vport.pcp_to_vl_mc[pcp];
-		else
-			vl = info->vport.pcp_to_vl_uc[pcp];
-	} else {
-		if (is_multicast_ether_addr(mac_hdr->h_dest))
-			vl = info->vport.non_vlan_vl_mc;
-		else
-			vl = info->vport.non_vlan_vl_uc;
-	}
-
-	return vl;
-}
-
-/* opa_vnic_get_rc - return the routing control */
-static u8 opa_vnic_get_rc(struct __opa_veswport_info *info,
-			  struct sk_buff *skb)
-{
-	u8 proto, rout_ctrl;
-
-	switch (vlan_get_protocol(skb)) {
-	case htons(ETH_P_IPV6):
-		proto = ipv6_hdr(skb)->nexthdr;
-		if (proto == IPPROTO_TCP)
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc,
-							  IPV6_TCP);
-		else if (proto == IPPROTO_UDP)
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc,
-							  IPV6_UDP);
-		else
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc, IPV6);
-		break;
-	case htons(ETH_P_IP):
-		proto = ip_hdr(skb)->protocol;
-		if (proto == IPPROTO_TCP)
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc,
-							  IPV4_TCP);
-		else if (proto == IPPROTO_UDP)
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc,
-							  IPV4_UDP);
-		else
-			rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc, IPV4);
-		break;
-	default:
-		rout_ctrl = OPA_VNIC_ENCAP_RC_EXT(info->vesw.rc, DEFAULT);
-	}
-
-	return rout_ctrl;
-}
-
-/* opa_vnic_calc_entropy - calculate the packet entropy */
-u8 opa_vnic_calc_entropy(struct sk_buff *skb)
-{
-	u32 hash = skb_get_hash(skb);
-
-	/* store XOR of all bytes in lower 8 bits */
-	hash ^= hash >> 8;
-	hash ^= hash >> 16;
-
-	/* return lower 8 bits as entropy */
-	return (u8)(hash & 0xFF);
-}
-
-/* opa_vnic_get_def_port - get default port based on entropy */
-static inline u8 opa_vnic_get_def_port(struct opa_vnic_adapter *adapter,
-				       u8 entropy)
-{
-	u8 flow_id;
-
-	/* Add the upper and lower 4-bits of entropy to get the flow id */
-	flow_id = ((entropy & 0xf) + (entropy >> 4));
-	return adapter->flow_tbl[flow_id & (OPA_VNIC_FLOW_TBL_SIZE - 1)];
-}
-
-/* Calculate packet length including OPA header, crc and padding */
-static inline int opa_vnic_wire_length(struct sk_buff *skb)
-{
-	u32 pad_len;
-
-	/* padding for 8 bytes size alignment */
-	pad_len = -(skb->len + OPA_VNIC_ICRC_TAIL_LEN) & 0x7;
-	pad_len += OPA_VNIC_ICRC_TAIL_LEN;
-
-	return (skb->len + pad_len) >> 3;
-}
-
-/* opa_vnic_encap_skb - encapsulate skb packet with OPA header and meta data */
-void opa_vnic_encap_skb(struct opa_vnic_adapter *adapter, struct sk_buff *skb)
-{
-	struct __opa_veswport_info *info = &adapter->info;
-	struct opa_vnic_skb_mdata *mdata;
-	u8 def_port, sc, rc, entropy, *hdr;
-	u16 len, l4_hdr;
-	u32 dlid;
-
-	hdr = skb_push(skb, OPA_VNIC_HDR_LEN);
-
-	entropy = opa_vnic_calc_entropy(skb);
-	def_port = opa_vnic_get_def_port(adapter, entropy);
-	len = opa_vnic_wire_length(skb);
-	dlid = opa_vnic_get_dlid(adapter, skb, def_port);
-	sc = opa_vnic_get_sc(info, skb);
-	rc = opa_vnic_get_rc(info, skb);
-	l4_hdr = info->vesw.vesw_id;
-
-	mdata = skb_push(skb, sizeof(*mdata));
-	mdata->vl = opa_vnic_get_vl(adapter, skb);
-	mdata->entropy = entropy;
-	mdata->flags = 0;
-	if (unlikely(!dlid)) {
-		mdata->flags = OPA_VNIC_SKB_MDATA_ENCAP_ERR;
-		return;
-	}
-
-	opa_vnic_make_header(hdr, info->vport.encap_slid, dlid, len,
-			     info->vesw.pkey, entropy, sc, rc,
-			     OPA_VNIC_L4_ETHR, l4_hdr);
-}
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
deleted file mode 100644
index 012fc27c5c93b..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h
+++ /dev/null
@@ -1,524 +0,0 @@
-#ifndef _OPA_VNIC_ENCAP_H
-#define _OPA_VNIC_ENCAP_H
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains all OPA VNIC declaration required for encapsulation
- * and decapsulation of Ethernet packets
- */
-
-#include <linux/types.h>
-#include <rdma/ib_mad.h>
-
-/* EMA class version */
-#define OPA_EMA_CLASS_VERSION               0x80
-
-/*
- * Define the Intel vendor management class for OPA
- * ETHERNET MANAGEMENT
- */
-#define OPA_MGMT_CLASS_INTEL_EMA            0x34
-
-/* EM attribute IDs */
-#define OPA_EM_ATTR_CLASS_PORT_INFO                 0x0001
-#define OPA_EM_ATTR_VESWPORT_INFO                   0x0011
-#define OPA_EM_ATTR_VESWPORT_MAC_ENTRIES            0x0012
-#define OPA_EM_ATTR_IFACE_UCAST_MACS                0x0013
-#define OPA_EM_ATTR_IFACE_MCAST_MACS                0x0014
-#define OPA_EM_ATTR_DELETE_VESW                     0x0015
-#define OPA_EM_ATTR_VESWPORT_SUMMARY_COUNTERS       0x0020
-#define OPA_EM_ATTR_VESWPORT_ERROR_COUNTERS         0x0022
-
-/* VNIC configured and operational state values */
-#define OPA_VNIC_STATE_DROP_ALL        0x1
-#define OPA_VNIC_STATE_FORWARDING      0x3
-
-#define OPA_VESW_MAX_NUM_DEF_PORT   16
-#define OPA_VNIC_MAX_NUM_PCP        8
-
-#define OPA_VNIC_EMA_DATA    (OPA_MGMT_MAD_SIZE - IB_MGMT_VENDOR_HDR)
-
-/* Defines for vendor specific notice(trap) attributes */
-#define OPA_INTEL_EMA_NOTICE_TYPE_INFO 0x04
-
-/* INTEL OUI */
-#define INTEL_OUI_1 0x00
-#define INTEL_OUI_2 0x06
-#define INTEL_OUI_3 0x6a
-
-/* Trap opcodes sent from VNIC */
-#define OPA_VESWPORT_TRAP_IFACE_UCAST_MAC_CHANGE 0x1
-#define OPA_VESWPORT_TRAP_IFACE_MCAST_MAC_CHANGE 0x2
-#define OPA_VESWPORT_TRAP_ETH_LINK_STATUS_CHANGE 0x3
-
-#define OPA_VNIC_DLID_SD_IS_SRC_MAC(dlid_sd)  (!!((dlid_sd) & 0x20))
-#define OPA_VNIC_DLID_SD_GET_DLID(dlid_sd)    ((dlid_sd) >> 8)
-
-/* VNIC Ethernet link status */
-#define OPA_VNIC_ETH_LINK_UP     1
-#define OPA_VNIC_ETH_LINK_DOWN   2
-
-/* routing control */
-#define OPA_VNIC_ENCAP_RC_DEFAULT   0
-#define OPA_VNIC_ENCAP_RC_IPV4      4
-#define OPA_VNIC_ENCAP_RC_IPV4_UDP  8
-#define OPA_VNIC_ENCAP_RC_IPV4_TCP  12
-#define OPA_VNIC_ENCAP_RC_IPV6      16
-#define OPA_VNIC_ENCAP_RC_IPV6_TCP  20
-#define OPA_VNIC_ENCAP_RC_IPV6_UDP  24
-
-#define OPA_VNIC_ENCAP_RC_EXT(w, b) (((w) >> OPA_VNIC_ENCAP_RC_ ## b) & 0x7)
-
-/**
- * struct opa_vesw_info - OPA vnic switch information
- * @fabric_id: 10-bit fabric id
- * @vesw_id: 12-bit virtual ethernet switch id
- * @rsvd0: reserved bytes
- * @def_port_mask: bitmask of default ports
- * @rsvd1: reserved bytes
- * @pkey: partition key
- * @rsvd2: reserved bytes
- * @u_mcast_dlid: unknown multicast dlid
- * @u_ucast_dlid: array of unknown unicast dlids
- * @rsvd3: reserved bytes
- * @rc: routing control
- * @eth_mtu: Ethernet MTU
- * @rsvd4: reserved bytes
- */
-struct opa_vesw_info {
-	__be16  fabric_id;
-	__be16  vesw_id;
-
-	u8      rsvd0[6];
-	__be16  def_port_mask;
-
-	u8      rsvd1[2];
-	__be16  pkey;
-
-	u8      rsvd2[4];
-	__be32  u_mcast_dlid;
-	__be32  u_ucast_dlid[OPA_VESW_MAX_NUM_DEF_PORT];
-
-	__be32  rc;
-
-	u8      rsvd3[56];
-	__be16  eth_mtu;
-	u8      rsvd4[2];
-} __packed;
-
-/**
- * struct opa_per_veswport_info - OPA vnic per port information
- * @port_num: port number
- * @eth_link_status: current ethernet link state
- * @rsvd0: reserved bytes
- * @base_mac_addr: base mac address
- * @config_state: configured port state
- * @oper_state: operational port state
- * @max_mac_tbl_ent: max number of mac table entries
- * @max_smac_ent: max smac entries in mac table
- * @mac_tbl_digest: mac table digest
- * @rsvd1: reserved bytes
- * @encap_slid: base slid for the port
- * @pcp_to_sc_uc: sc by pcp index for unicast ethernet packets
- * @pcp_to_vl_uc: vl by pcp index for unicast ethernet packets
- * @pcp_to_sc_mc: sc by pcp index for multicast ethernet packets
- * @pcp_to_vl_mc: vl by pcp index for multicast ethernet packets
- * @non_vlan_sc_uc: sc for non-vlan unicast ethernet packets
- * @non_vlan_vl_uc: vl for non-vlan unicast ethernet packets
- * @non_vlan_sc_mc: sc for non-vlan multicast ethernet packets
- * @non_vlan_vl_mc: vl for non-vlan multicast ethernet packets
- * @rsvd2: reserved bytes
- * @uc_macs_gen_count: generation count for unicast macs list
- * @mc_macs_gen_count: generation count for multicast macs list
- * @rsvd3: reserved bytes
- */
-struct opa_per_veswport_info {
-	__be32  port_num;
-
-	u8      eth_link_status;
-	u8      rsvd0[3];
-
-	u8      base_mac_addr[ETH_ALEN];
-	u8      config_state;
-	u8      oper_state;
-
-	__be16  max_mac_tbl_ent;
-	__be16  max_smac_ent;
-	__be32  mac_tbl_digest;
-	u8      rsvd1[4];
-
-	__be32  encap_slid;
-
-	u8      pcp_to_sc_uc[OPA_VNIC_MAX_NUM_PCP];
-	u8      pcp_to_vl_uc[OPA_VNIC_MAX_NUM_PCP];
-	u8      pcp_to_sc_mc[OPA_VNIC_MAX_NUM_PCP];
-	u8      pcp_to_vl_mc[OPA_VNIC_MAX_NUM_PCP];
-
-	u8      non_vlan_sc_uc;
-	u8      non_vlan_vl_uc;
-	u8      non_vlan_sc_mc;
-	u8      non_vlan_vl_mc;
-
-	u8      rsvd2[48];
-
-	__be16  uc_macs_gen_count;
-	__be16  mc_macs_gen_count;
-
-	u8      rsvd3[8];
-} __packed;
-
-/**
- * struct opa_veswport_info - OPA vnic port information
- * @vesw: OPA vnic switch information
- * @vport: OPA vnic per port information
- *
- * On host, each of the virtual ethernet ports belongs
- * to a different virtual ethernet switches.
- */
-struct opa_veswport_info {
-	struct opa_vesw_info          vesw;
-	struct opa_per_veswport_info  vport;
-};
-
-/**
- * struct opa_veswport_mactable_entry - single entry in the forwarding table
- * @mac_addr: MAC address
- * @mac_addr_mask: MAC address bit mask
- * @dlid_sd: Matching DLID and side data
- *
- * On the host each virtual ethernet port will have
- * a forwarding table. These tables are used to
- * map a MAC to a LID and other data. For more
- * details see struct opa_veswport_mactable_entries.
- * This is the structure of a single mactable entry
- */
-struct opa_veswport_mactable_entry {
-	u8      mac_addr[ETH_ALEN];
-	u8      mac_addr_mask[ETH_ALEN];
-	__be32  dlid_sd;
-} __packed;
-
-/**
- * struct opa_veswport_mactable - Forwarding table array
- * @offset: mac table starting offset
- * @num_entries: Number of entries to get or set
- * @mac_tbl_digest: mac table digest
- * @tbl_entries: Array of table entries
- *
- * The EM sends down this structure in a MAD indicating
- * the starting offset in the forwarding table that this
- * entry is to be loaded into and the number of entries
- * that that this MAD instance contains
- * The mac_tbl_digest has been added to this MAD structure. It will be set by
- * the EM and it will be used by the EM to check if there are any
- * discrepancies with this value and the value
- * maintained by the EM in the case of VNIC port being deleted or unloaded
- * A new instantiation of a VNIC will always have a value of zero.
- * This value is stored as part of the vnic adapter structure and will be
- * accessed by the GET and SET routines for both the mactable entries and the
- * veswport info.
- */
-struct opa_veswport_mactable {
-	__be16                              offset;
-	__be16                              num_entries;
-	__be32                              mac_tbl_digest;
-	struct opa_veswport_mactable_entry  tbl_entries[];
-} __packed;
-
-/**
- * struct opa_veswport_summary_counters - summary counters
- * @vp_instance: vport instance on the OPA port
- * @vesw_id: virtual ethernet switch id
- * @veswport_num: virtual ethernet switch port number
- * @tx_errors: transmit errors
- * @rx_errors: receive errors
- * @tx_packets: transmit packets
- * @rx_packets: receive packets
- * @tx_bytes: transmit bytes
- * @rx_bytes: receive bytes
- * @tx_unicast: unicast packets transmitted
- * @tx_mcastbcast: multicast/broadcast packets transmitted
- * @tx_untagged: non-vlan packets transmitted
- * @tx_vlan: vlan packets transmitted
- * @tx_64_size: transmit packet length is 64 bytes
- * @tx_65_127: transmit packet length is >=65 and < 127 bytes
- * @tx_128_255: transmit packet length is >=128 and < 255 bytes
- * @tx_256_511: transmit packet length is >=256 and < 511 bytes
- * @tx_512_1023: transmit packet length is >=512 and < 1023 bytes
- * @tx_1024_1518: transmit packet length is >=1024 and < 1518 bytes
- * @tx_1519_max: transmit packet length >= 1519 bytes
- * @rx_unicast: unicast packets received
- * @rx_mcastbcast: multicast/broadcast packets received
- * @rx_untagged: non-vlan packets received
- * @rx_vlan: vlan packets received
- * @rx_64_size: received packet length is 64 bytes
- * @rx_65_127: received packet length is >=65 and < 127 bytes
- * @rx_128_255: received packet length is >=128 and < 255 bytes
- * @rx_256_511: received packet length is >=256 and < 511 bytes
- * @rx_512_1023: received packet length is >=512 and < 1023 bytes
- * @rx_1024_1518: received packet length is >=1024 and < 1518 bytes
- * @rx_1519_max: received packet length >= 1519 bytes
- * @reserved: reserved bytes
- *
- * All the above are counters of corresponding conditions.
- */
-struct opa_veswport_summary_counters {
-	__be16  vp_instance;
-	__be16  vesw_id;
-	__be32  veswport_num;
-
-	__be64  tx_errors;
-	__be64  rx_errors;
-	__be64  tx_packets;
-	__be64  rx_packets;
-	__be64  tx_bytes;
-	__be64  rx_bytes;
-
-	__be64  tx_unicast;
-	__be64  tx_mcastbcast;
-
-	__be64  tx_untagged;
-	__be64  tx_vlan;
-
-	__be64  tx_64_size;
-	__be64  tx_65_127;
-	__be64  tx_128_255;
-	__be64  tx_256_511;
-	__be64  tx_512_1023;
-	__be64  tx_1024_1518;
-	__be64  tx_1519_max;
-
-	__be64  rx_unicast;
-	__be64  rx_mcastbcast;
-
-	__be64  rx_untagged;
-	__be64  rx_vlan;
-
-	__be64  rx_64_size;
-	__be64  rx_65_127;
-	__be64  rx_128_255;
-	__be64  rx_256_511;
-	__be64  rx_512_1023;
-	__be64  rx_1024_1518;
-	__be64  rx_1519_max;
-
-	__be64  reserved[16];
-} __packed;
-
-/**
- * struct opa_veswport_error_counters - error counters
- * @vp_instance: vport instance on the OPA port
- * @vesw_id: virtual ethernet switch id
- * @veswport_num: virtual ethernet switch port number
- * @tx_errors: transmit errors
- * @rx_errors: receive errors
- * @rsvd0: reserved bytes
- * @tx_smac_filt: smac filter errors
- * @rsvd1: reserved bytes
- * @rsvd2: reserved bytes
- * @rsvd3: reserved bytes
- * @tx_dlid_zero: transmit packets with invalid dlid
- * @rsvd4: reserved bytes
- * @tx_logic: other transmit errors
- * @rsvd5: reserved bytes
- * @tx_drop_state: packet tansmission in non-forward port state
- * @rx_bad_veswid: received packet with invalid vesw id
- * @rsvd6: reserved bytes
- * @rx_runt: received ethernet packet with length < 64 bytes
- * @rx_oversize: received ethernet packet with length > MTU size
- * @rsvd7: reserved bytes
- * @rx_eth_down: received packets when interface is down
- * @rx_drop_state: received packets in non-forwarding port state
- * @rx_logic: other receive errors
- * @rsvd8: reserved bytes
- * @rsvd9: reserved bytes
- *
- * All the above are counters of corresponding error conditions.
- */
-struct opa_veswport_error_counters {
-	__be16  vp_instance;
-	__be16  vesw_id;
-	__be32  veswport_num;
-
-	__be64  tx_errors;
-	__be64  rx_errors;
-
-	__be64  rsvd0;
-	__be64  tx_smac_filt;
-	__be64  rsvd1;
-	__be64  rsvd2;
-	__be64  rsvd3;
-	__be64  tx_dlid_zero;
-	__be64  rsvd4;
-	__be64  tx_logic;
-	__be64  rsvd5;
-	__be64  tx_drop_state;
-
-	__be64  rx_bad_veswid;
-	__be64  rsvd6;
-	__be64  rx_runt;
-	__be64  rx_oversize;
-	__be64  rsvd7;
-	__be64  rx_eth_down;
-	__be64  rx_drop_state;
-	__be64  rx_logic;
-	__be64  rsvd8;
-
-	__be64  rsvd9[16];
-} __packed;
-
-/**
- * struct opa_veswport_trap - Trap message sent to EM by VNIC
- * @fabric_id: 10 bit fabric id
- * @veswid: 12 bit virtual ethernet switch id
- * @veswportnum: logical port number on the Virtual switch
- * @opaportnum: physical port num (redundant on host)
- * @veswportindex: switch port index on opa port 0 based
- * @opcode: operation
- * @reserved: 32 bit for alignment
- *
- * The VNIC will send trap messages to the Ethernet manager to
- * inform it about changes to the VNIC config, behaviour etc.
- * This is the format of the trap payload.
- */
-struct opa_veswport_trap {
-	__be16  fabric_id;
-	__be16  veswid;
-	__be32  veswportnum;
-	__be16  opaportnum;
-	u8      veswportindex;
-	u8      opcode;
-	__be32  reserved;
-} __packed;
-
-/**
- * struct opa_vnic_iface_mac_entry - single entry in the mac list
- * @mac_addr: MAC address
- */
-struct opa_vnic_iface_mac_entry {
-	u8 mac_addr[ETH_ALEN];
-};
-
-/**
- * struct opa_veswport_iface_macs - Msg to set globally administered MAC
- * @start_idx: position of first entry (0 based)
- * @num_macs_in_msg: number of MACs in this message
- * @tot_macs_in_lst: The total number of MACs the agent has
- * @gen_count: gen_count to indicate change
- * @entry: The mac list entry
- *
- * Same attribute IDS and attribute modifiers as in locally administered
- * addresses used to set globally administered addresses
- */
-struct opa_veswport_iface_macs {
-	__be16 start_idx;
-	__be16 num_macs_in_msg;
-	__be16 tot_macs_in_lst;
-	__be16 gen_count;
-	struct opa_vnic_iface_mac_entry entry[];
-} __packed;
-
-/**
- * struct opa_vnic_vema_mad - Generic VEMA MAD
- * @mad_hdr: Generic MAD header
- * @rmpp_hdr: RMPP header for vendor specific MADs
- * @reserved: reserved bytes
- * @oui: Unique org identifier
- * @data: MAD data
- */
-struct opa_vnic_vema_mad {
-	struct ib_mad_hdr  mad_hdr;
-	struct ib_rmpp_hdr rmpp_hdr;
-	u8                 reserved;
-	u8                 oui[3];
-	u8                 data[OPA_VNIC_EMA_DATA];
-};
-
-/**
- * struct opa_vnic_notice_attr - Generic Notice MAD
- * @gen_type: Generic/Specific bit and type of notice
- * @oui_1: Vendor ID byte 1
- * @oui_2: Vendor ID byte 2
- * @oui_3: Vendor ID byte 3
- * @trap_num: Trap number
- * @toggle_count: Notice toggle bit and count value
- * @issuer_lid: Trap issuer's lid
- * @reserved: reserved bytes
- * @issuer_gid: Issuer GID (only if Report method)
- * @raw_data: Trap message body
- */
-struct opa_vnic_notice_attr {
-	u8     gen_type;
-	u8     oui_1;
-	u8     oui_2;
-	u8     oui_3;
-	__be16 trap_num;
-	__be16 toggle_count;
-	__be32 issuer_lid;
-	__be32 reserved;
-	u8     issuer_gid[16];
-	u8     raw_data[64];
-} __packed;
-
-/**
- * struct opa_vnic_vema_mad_trap - Generic VEMA MAD Trap
- * @mad_hdr: Generic MAD header
- * @rmpp_hdr: RMPP header for vendor specific MADs
- * @reserved: reserved bytes
- * @oui: Unique org identifier
- * @notice: Notice structure
- */
-struct opa_vnic_vema_mad_trap {
-	struct ib_mad_hdr            mad_hdr;
-	struct ib_rmpp_hdr           rmpp_hdr;
-	u8                           reserved;
-	u8                           oui[3];
-	struct opa_vnic_notice_attr  notice;
-};
-
-#endif /* _OPA_VNIC_ENCAP_H */
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c
deleted file mode 100644
index 316959940d2fc..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPA VNIC ethtool functions
- */
-
-#include <linux/ethtool.h>
-
-#include "opa_vnic_internal.h"
-
-enum {NETDEV_STATS, VNIC_STATS};
-
-struct vnic_stats {
-	char stat_string[ETH_GSTRING_LEN];
-	struct {
-		int sizeof_stat;
-		int stat_offset;
-	};
-};
-
-#define VNIC_STAT(m)            { sizeof_field(struct opa_vnic_stats, m),   \
-				  offsetof(struct opa_vnic_stats, m) }
-
-static struct vnic_stats vnic_gstrings_stats[] = {
-	/* NETDEV stats */
-	{"rx_packets", VNIC_STAT(netstats.rx_packets)},
-	{"tx_packets", VNIC_STAT(netstats.tx_packets)},
-	{"rx_bytes", VNIC_STAT(netstats.rx_bytes)},
-	{"tx_bytes", VNIC_STAT(netstats.tx_bytes)},
-	{"rx_errors", VNIC_STAT(netstats.rx_errors)},
-	{"tx_errors", VNIC_STAT(netstats.tx_errors)},
-	{"rx_dropped", VNIC_STAT(netstats.rx_dropped)},
-	{"tx_dropped", VNIC_STAT(netstats.tx_dropped)},
-
-	/* SUMMARY counters */
-	{"tx_unicast", VNIC_STAT(tx_grp.unicast)},
-	{"tx_mcastbcast", VNIC_STAT(tx_grp.mcastbcast)},
-	{"tx_untagged", VNIC_STAT(tx_grp.untagged)},
-	{"tx_vlan", VNIC_STAT(tx_grp.vlan)},
-
-	{"tx_64_size", VNIC_STAT(tx_grp.s_64)},
-	{"tx_65_127", VNIC_STAT(tx_grp.s_65_127)},
-	{"tx_128_255", VNIC_STAT(tx_grp.s_128_255)},
-	{"tx_256_511", VNIC_STAT(tx_grp.s_256_511)},
-	{"tx_512_1023", VNIC_STAT(tx_grp.s_512_1023)},
-	{"tx_1024_1518", VNIC_STAT(tx_grp.s_1024_1518)},
-	{"tx_1519_max", VNIC_STAT(tx_grp.s_1519_max)},
-
-	{"rx_unicast", VNIC_STAT(rx_grp.unicast)},
-	{"rx_mcastbcast", VNIC_STAT(rx_grp.mcastbcast)},
-	{"rx_untagged", VNIC_STAT(rx_grp.untagged)},
-	{"rx_vlan", VNIC_STAT(rx_grp.vlan)},
-
-	{"rx_64_size", VNIC_STAT(rx_grp.s_64)},
-	{"rx_65_127", VNIC_STAT(rx_grp.s_65_127)},
-	{"rx_128_255", VNIC_STAT(rx_grp.s_128_255)},
-	{"rx_256_511", VNIC_STAT(rx_grp.s_256_511)},
-	{"rx_512_1023", VNIC_STAT(rx_grp.s_512_1023)},
-	{"rx_1024_1518", VNIC_STAT(rx_grp.s_1024_1518)},
-	{"rx_1519_max", VNIC_STAT(rx_grp.s_1519_max)},
-
-	/* ERROR counters */
-	{"rx_fifo_errors", VNIC_STAT(netstats.rx_fifo_errors)},
-	{"rx_length_errors", VNIC_STAT(netstats.rx_length_errors)},
-
-	{"tx_fifo_errors", VNIC_STAT(netstats.tx_fifo_errors)},
-	{"tx_carrier_errors", VNIC_STAT(netstats.tx_carrier_errors)},
-
-	{"tx_dlid_zero", VNIC_STAT(tx_dlid_zero)},
-	{"tx_drop_state", VNIC_STAT(tx_drop_state)},
-	{"rx_drop_state", VNIC_STAT(rx_drop_state)},
-	{"rx_oversize", VNIC_STAT(rx_oversize)},
-	{"rx_runt", VNIC_STAT(rx_runt)},
-};
-
-#define VNIC_STATS_LEN  ARRAY_SIZE(vnic_gstrings_stats)
-
-/* vnic_get_drvinfo - get driver info */
-static void vnic_get_drvinfo(struct net_device *netdev,
-			     struct ethtool_drvinfo *drvinfo)
-{
-	strscpy(drvinfo->driver, opa_vnic_driver_name, sizeof(drvinfo->driver));
-	strscpy(drvinfo->bus_info, dev_name(netdev->dev.parent),
-		sizeof(drvinfo->bus_info));
-}
-
-/* vnic_get_sset_count - get string set count */
-static int vnic_get_sset_count(struct net_device *netdev, int sset)
-{
-	return (sset == ETH_SS_STATS) ? VNIC_STATS_LEN : -EOPNOTSUPP;
-}
-
-/* vnic_get_ethtool_stats - get statistics */
-static void vnic_get_ethtool_stats(struct net_device *netdev,
-				   struct ethtool_stats *stats, u64 *data)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	struct opa_vnic_stats vstats;
-	int i;
-
-	memset(&vstats, 0, sizeof(vstats));
-	spin_lock(&adapter->stats_lock);
-	adapter->rn_ops->ndo_get_stats64(netdev, &vstats.netstats);
-	spin_unlock(&adapter->stats_lock);
-	for (i = 0; i < VNIC_STATS_LEN; i++) {
-		char *p = (char *)&vstats + vnic_gstrings_stats[i].stat_offset;
-
-		data[i] = (vnic_gstrings_stats[i].sizeof_stat ==
-			   sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
-	}
-}
-
-/* vnic_get_strings - get strings */
-static void vnic_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
-{
-	int i;
-
-	if (stringset != ETH_SS_STATS)
-		return;
-
-	for (i = 0; i < VNIC_STATS_LEN; i++)
-		ethtool_puts(&data, vnic_gstrings_stats[i].stat_string);
-}
-
-/* ethtool ops */
-static const struct ethtool_ops opa_vnic_ethtool_ops = {
-	.get_drvinfo = vnic_get_drvinfo,
-	.get_link = ethtool_op_get_link,
-	.get_strings = vnic_get_strings,
-	.get_sset_count = vnic_get_sset_count,
-	.get_ethtool_stats = vnic_get_ethtool_stats,
-};
-
-/* opa_vnic_set_ethtool_ops - set ethtool ops */
-void opa_vnic_set_ethtool_ops(struct net_device *netdev)
-{
-	netdev->ethtool_ops = &opa_vnic_ethtool_ops;
-}
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h b/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
deleted file mode 100644
index dd942dd642bd3..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_internal.h
+++ /dev/null
@@ -1,329 +0,0 @@
-#ifndef _OPA_VNIC_INTERNAL_H
-#define _OPA_VNIC_INTERNAL_H
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPA VNIC driver internal declarations
- */
-
-#include <linux/bitops.h>
-#include <linux/etherdevice.h>
-#include <linux/hashtable.h>
-#include <linux/sizes.h>
-#include <rdma/opa_vnic.h>
-
-#include "opa_vnic_encap.h"
-
-#define OPA_VNIC_VLAN_PCP(vlan_tci)  \
-			(((vlan_tci) & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT)
-
-/* Flow to default port redirection table size */
-#define OPA_VNIC_FLOW_TBL_SIZE    32
-
-/* Invalid port number */
-#define OPA_VNIC_INVALID_PORT     0xff
-
-struct opa_vnic_adapter;
-
-/*
- * struct __opa_vesw_info - OPA vnic virtual switch info
- *
- * Same as opa_vesw_info without bitwise attribute.
- */
-struct __opa_vesw_info {
-	u16  fabric_id;
-	u16  vesw_id;
-
-	u8   rsvd0[6];
-	u16  def_port_mask;
-
-	u8   rsvd1[2];
-	u16  pkey;
-
-	u8   rsvd2[4];
-	u32  u_mcast_dlid;
-	u32  u_ucast_dlid[OPA_VESW_MAX_NUM_DEF_PORT];
-
-	u32  rc;
-
-	u8   rsvd3[56];
-	u16  eth_mtu;
-	u8   rsvd4[2];
-} __packed;
-
-/*
- * struct __opa_per_veswport_info - OPA vnic per port info
- *
- * Same as opa_per_veswport_info without bitwise attribute.
- */
-struct __opa_per_veswport_info {
-	u32  port_num;
-
-	u8   eth_link_status;
-	u8   rsvd0[3];
-
-	u8   base_mac_addr[ETH_ALEN];
-	u8   config_state;
-	u8   oper_state;
-
-	u16  max_mac_tbl_ent;
-	u16  max_smac_ent;
-	u32  mac_tbl_digest;
-	u8   rsvd1[4];
-
-	u32  encap_slid;
-
-	u8   pcp_to_sc_uc[OPA_VNIC_MAX_NUM_PCP];
-	u8   pcp_to_vl_uc[OPA_VNIC_MAX_NUM_PCP];
-	u8   pcp_to_sc_mc[OPA_VNIC_MAX_NUM_PCP];
-	u8   pcp_to_vl_mc[OPA_VNIC_MAX_NUM_PCP];
-
-	u8   non_vlan_sc_uc;
-	u8   non_vlan_vl_uc;
-	u8   non_vlan_sc_mc;
-	u8   non_vlan_vl_mc;
-
-	u8   rsvd2[48];
-
-	u16  uc_macs_gen_count;
-	u16  mc_macs_gen_count;
-
-	u8   rsvd3[8];
-} __packed;
-
-/*
- * struct __opa_veswport_info - OPA vnic port info
- *
- * Same as opa_veswport_info without bitwise attribute.
- */
-struct __opa_veswport_info {
-	struct __opa_vesw_info            vesw;
-	struct __opa_per_veswport_info    vport;
-};
-
-/*
- * struct __opa_veswport_trap - OPA vnic trap info
- *
- * Same as opa_veswport_trap without bitwise attribute.
- */
-struct __opa_veswport_trap {
-	u16	fabric_id;
-	u16	veswid;
-	u32	veswportnum;
-	u16	opaportnum;
-	u8	veswportindex;
-	u8	opcode;
-	u32	reserved;
-} __packed;
-
-/**
- * struct opa_vnic_ctrl_port - OPA virtual NIC control port
- * @ibdev: pointer to ib device
- * @ops: opa vnic control operations
- * @num_ports: number of opa ports
- */
-struct opa_vnic_ctrl_port {
-	struct ib_device           *ibdev;
-	struct opa_vnic_ctrl_ops   *ops;
-	u8                          num_ports;
-};
-
-/**
- * struct opa_vnic_adapter - OPA VNIC netdev private data structure
- * @netdev: pointer to associated netdev
- * @ibdev: ib device
- * @cport: pointer to opa vnic control port
- * @rn_ops: rdma netdev's net_device_ops
- * @port_num: OPA port number
- * @vport_num: vesw port number
- * @lock: adapter lock
- * @info: virtual ethernet switch port information
- * @vema_mac_addr: mac address configured by vema
- * @umac_hash: unicast maclist hash
- * @mmac_hash: multicast maclist hash
- * @mactbl: hash table of MAC entries
- * @mactbl_lock: mac table lock
- * @stats_lock: statistics lock
- * @flow_tbl: flow to default port redirection table
- * @trap_timeout: trap timeout
- * @trap_count: no. of traps allowed within timeout period
- */
-struct opa_vnic_adapter {
-	struct net_device             *netdev;
-	struct ib_device              *ibdev;
-	struct opa_vnic_ctrl_port     *cport;
-	const struct net_device_ops   *rn_ops;
-
-	u8 port_num;
-	u8 vport_num;
-
-	/* Lock used around concurrent updates to netdev */
-	struct mutex lock;
-
-	struct __opa_veswport_info  info;
-	u8                          vema_mac_addr[ETH_ALEN];
-	u32                         umac_hash;
-	u32                         mmac_hash;
-	struct hlist_head  __rcu   *mactbl;
-
-	/* Lock used to protect updates to mac table */
-	struct mutex mactbl_lock;
-
-	/* Lock used to protect access to vnic counters */
-	spinlock_t stats_lock;
-
-	u8 flow_tbl[OPA_VNIC_FLOW_TBL_SIZE];
-
-	unsigned long trap_timeout;
-	u8            trap_count;
-};
-
-/* Same as opa_veswport_mactable_entry, but without bitwise attribute */
-struct __opa_vnic_mactable_entry {
-	u8  mac_addr[ETH_ALEN];
-	u8  mac_addr_mask[ETH_ALEN];
-	u32 dlid_sd;
-} __packed;
-
-/**
- * struct opa_vnic_mac_tbl_node - OPA VNIC mac table node
- * @hlist: hash list handle
- * @index: index of entry in the mac table
- * @entry: entry in the table
- */
-struct opa_vnic_mac_tbl_node {
-	struct hlist_node                    hlist;
-	u16                                  index;
-	struct __opa_vnic_mactable_entry     entry;
-};
-
-#define v_dbg(format, arg...) \
-	netdev_dbg(adapter->netdev, format, ## arg)
-#define v_err(format, arg...) \
-	netdev_err(adapter->netdev, format, ## arg)
-#define v_info(format, arg...) \
-	netdev_info(adapter->netdev, format, ## arg)
-#define v_warn(format, arg...) \
-	netdev_warn(adapter->netdev, format, ## arg)
-
-#define c_err(format, arg...) \
-	dev_err(&cport->ibdev->dev, format, ## arg)
-#define c_info(format, arg...) \
-	dev_info(&cport->ibdev->dev, format, ## arg)
-#define c_dbg(format, arg...) \
-	dev_dbg(&cport->ibdev->dev, format, ## arg)
-
-/* The maximum allowed entries in the mac table */
-#define OPA_VNIC_MAC_TBL_MAX_ENTRIES  2048
-/* Limit of smac entries in mac table */
-#define OPA_VNIC_MAX_SMAC_LIMIT       256
-
-/* The last octet of the MAC address is used as the key to the hash table */
-#define OPA_VNIC_MAC_HASH_IDX         5
-
-/* The VNIC MAC hash table is of size 2^8 */
-#define OPA_VNIC_MAC_TBL_HASH_BITS    8
-#define OPA_VNIC_MAC_TBL_SIZE  BIT(OPA_VNIC_MAC_TBL_HASH_BITS)
-
-/* VNIC HASH MACROS */
-#define vnic_hash_init(hashtable) __hash_init(hashtable, OPA_VNIC_MAC_TBL_SIZE)
-
-#define vnic_hash_add(hashtable, node, key)                                   \
-	hlist_add_head(node,                                                  \
-		&hashtable[hash_min(key, ilog2(OPA_VNIC_MAC_TBL_SIZE))])
-
-#define vnic_hash_for_each_safe(name, bkt, tmp, obj, member)                  \
-	for ((bkt) = 0, obj = NULL;                                           \
-		    !obj && (bkt) < OPA_VNIC_MAC_TBL_SIZE; (bkt)++)           \
-		hlist_for_each_entry_safe(obj, tmp, &name[bkt], member)
-
-#define vnic_hash_for_each_possible(name, obj, member, key)                   \
-	hlist_for_each_entry(obj,                                             \
-		&name[hash_min(key, ilog2(OPA_VNIC_MAC_TBL_SIZE))], member)
-
-#define vnic_hash_for_each(name, bkt, obj, member)                            \
-	for ((bkt) = 0, obj = NULL;                                           \
-		    !obj && (bkt) < OPA_VNIC_MAC_TBL_SIZE; (bkt)++)           \
-		hlist_for_each_entry(obj, &name[bkt], member)
-
-extern char opa_vnic_driver_name[];
-
-struct opa_vnic_adapter *opa_vnic_add_netdev(struct ib_device *ibdev,
-					     u8 port_num, u8 vport_num);
-void opa_vnic_rem_netdev(struct opa_vnic_adapter *adapter);
-void opa_vnic_encap_skb(struct opa_vnic_adapter *adapter, struct sk_buff *skb);
-u8 opa_vnic_get_vl(struct opa_vnic_adapter *adapter, struct sk_buff *skb);
-u8 opa_vnic_calc_entropy(struct sk_buff *skb);
-void opa_vnic_process_vema_config(struct opa_vnic_adapter *adapter);
-void opa_vnic_release_mac_tbl(struct opa_vnic_adapter *adapter);
-void opa_vnic_query_mac_tbl(struct opa_vnic_adapter *adapter,
-			    struct opa_veswport_mactable *tbl);
-int opa_vnic_update_mac_tbl(struct opa_vnic_adapter *adapter,
-			    struct opa_veswport_mactable *tbl);
-void opa_vnic_query_ucast_macs(struct opa_vnic_adapter *adapter,
-			       struct opa_veswport_iface_macs *macs);
-void opa_vnic_query_mcast_macs(struct opa_vnic_adapter *adapter,
-			       struct opa_veswport_iface_macs *macs);
-void opa_vnic_get_summary_counters(struct opa_vnic_adapter *adapter,
-				   struct opa_veswport_summary_counters *cntrs);
-void opa_vnic_get_error_counters(struct opa_vnic_adapter *adapter,
-				 struct opa_veswport_error_counters *cntrs);
-void opa_vnic_get_vesw_info(struct opa_vnic_adapter *adapter,
-			    struct opa_vesw_info *info);
-void opa_vnic_set_vesw_info(struct opa_vnic_adapter *adapter,
-			    struct opa_vesw_info *info);
-void opa_vnic_get_per_veswport_info(struct opa_vnic_adapter *adapter,
-				    struct opa_per_veswport_info *info);
-void opa_vnic_set_per_veswport_info(struct opa_vnic_adapter *adapter,
-				    struct opa_per_veswport_info *info);
-void opa_vnic_vema_report_event(struct opa_vnic_adapter *adapter, u8 event);
-void opa_vnic_set_ethtool_ops(struct net_device *netdev);
-void opa_vnic_vema_send_trap(struct opa_vnic_adapter *adapter,
-			     struct __opa_veswport_trap *data, u32 lid);
-
-#endif /* _OPA_VNIC_INTERNAL_H */
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
deleted file mode 100644
index 1c3e7251f0f46..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_netdev.c
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPA Virtual Network Interface Controller (VNIC) driver
- * netdev functionality.
- */
-
-#include <linux/if_vlan.h>
-#include <linux/crc32.h>
-
-#include "opa_vnic_internal.h"
-
-#define OPA_TX_TIMEOUT_MS 1000
-
-#define OPA_VNIC_SKB_HEADROOM  \
-			ALIGN((OPA_VNIC_HDR_LEN + OPA_VNIC_SKB_MDATA_LEN), 8)
-
-/* This function is overloaded for opa_vnic specific implementation */
-static void opa_vnic_get_stats64(struct net_device *netdev,
-				 struct rtnl_link_stats64 *stats)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	struct opa_vnic_stats vstats;
-
-	memset(&vstats, 0, sizeof(vstats));
-	spin_lock(&adapter->stats_lock);
-	adapter->rn_ops->ndo_get_stats64(netdev, &vstats.netstats);
-	spin_unlock(&adapter->stats_lock);
-	memcpy(stats, &vstats.netstats, sizeof(*stats));
-}
-
-/* opa_netdev_start_xmit - transmit function */
-static netdev_tx_t opa_netdev_start_xmit(struct sk_buff *skb,
-					 struct net_device *netdev)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-
-	v_dbg("xmit: queue %d skb len %d\n", skb->queue_mapping, skb->len);
-	/* pad to ensure mininum ethernet packet length */
-	if (unlikely(skb->len < ETH_ZLEN)) {
-		if (skb_padto(skb, ETH_ZLEN))
-			return NETDEV_TX_OK;
-
-		skb_put(skb, ETH_ZLEN - skb->len);
-	}
-
-	opa_vnic_encap_skb(adapter, skb);
-	return adapter->rn_ops->ndo_start_xmit(skb, netdev);
-}
-
-static u16 opa_vnic_select_queue(struct net_device *netdev, struct sk_buff *skb,
-				 struct net_device *sb_dev)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	struct opa_vnic_skb_mdata *mdata;
-	int rc;
-
-	/* pass entropy and vl as metadata in skb */
-	mdata = skb_push(skb, sizeof(*mdata));
-	mdata->entropy = opa_vnic_calc_entropy(skb);
-	mdata->vl = opa_vnic_get_vl(adapter, skb);
-	rc = adapter->rn_ops->ndo_select_queue(netdev, skb, sb_dev);
-	skb_pull(skb, sizeof(*mdata));
-	return rc;
-}
-
-static void opa_vnic_update_state(struct opa_vnic_adapter *adapter, bool up)
-{
-	struct __opa_veswport_info *info = &adapter->info;
-
-	mutex_lock(&adapter->lock);
-	/* Operational state can only be DROP_ALL or FORWARDING */
-	if ((info->vport.config_state == OPA_VNIC_STATE_FORWARDING) && up) {
-		info->vport.oper_state = OPA_VNIC_STATE_FORWARDING;
-		info->vport.eth_link_status = OPA_VNIC_ETH_LINK_UP;
-	} else {
-		info->vport.oper_state = OPA_VNIC_STATE_DROP_ALL;
-		info->vport.eth_link_status = OPA_VNIC_ETH_LINK_DOWN;
-	}
-
-	if (info->vport.config_state == OPA_VNIC_STATE_FORWARDING)
-		netif_dormant_off(adapter->netdev);
-	else
-		netif_dormant_on(adapter->netdev);
-	mutex_unlock(&adapter->lock);
-}
-
-/* opa_vnic_process_vema_config - process vema configuration updates */
-void opa_vnic_process_vema_config(struct opa_vnic_adapter *adapter)
-{
-	struct __opa_veswport_info *info = &adapter->info;
-	struct rdma_netdev *rn = netdev_priv(adapter->netdev);
-	u8 port_num[OPA_VESW_MAX_NUM_DEF_PORT] = { 0 };
-	struct net_device *netdev = adapter->netdev;
-	u8 i, port_count = 0;
-	u16 port_mask;
-
-	/* If the base_mac_addr is changed, update the interface mac address */
-	if (memcmp(info->vport.base_mac_addr, adapter->vema_mac_addr,
-		   ARRAY_SIZE(info->vport.base_mac_addr))) {
-		struct sockaddr saddr;
-
-		memcpy(saddr.sa_data, info->vport.base_mac_addr,
-		       ARRAY_SIZE(info->vport.base_mac_addr));
-		mutex_lock(&adapter->lock);
-		eth_commit_mac_addr_change(netdev, &saddr);
-		memcpy(adapter->vema_mac_addr,
-		       info->vport.base_mac_addr, ETH_ALEN);
-		mutex_unlock(&adapter->lock);
-	}
-
-	rn->set_id(netdev, info->vesw.vesw_id);
-
-	/* Handle MTU limit change */
-	rtnl_lock();
-	netdev->max_mtu = max_t(unsigned int, info->vesw.eth_mtu,
-				netdev->min_mtu);
-	if (netdev->mtu > netdev->max_mtu)
-		dev_set_mtu(netdev, netdev->max_mtu);
-	rtnl_unlock();
-
-	/* Update flow to default port redirection table */
-	port_mask = info->vesw.def_port_mask;
-	for (i = 0; i < OPA_VESW_MAX_NUM_DEF_PORT; i++) {
-		if (port_mask & 1)
-			port_num[port_count++] = i;
-		port_mask >>= 1;
-	}
-
-	/*
-	 * Build the flow table. Flow table is required when destination LID
-	 * is not available. Up to OPA_VNIC_FLOW_TBL_SIZE flows supported.
-	 * Each flow need a default port number to get its dlid from the
-	 * u_ucast_dlid array.
-	 */
-	for (i = 0; i < OPA_VNIC_FLOW_TBL_SIZE; i++)
-		adapter->flow_tbl[i] = port_count ? port_num[i % port_count] :
-						    OPA_VNIC_INVALID_PORT;
-
-	/* update state */
-	opa_vnic_update_state(adapter, !!(netdev->flags & IFF_UP));
-}
-
-/*
- * Set the power on default values in adapter's vema interface structure.
- */
-static inline void opa_vnic_set_pod_values(struct opa_vnic_adapter *adapter)
-{
-	adapter->info.vport.max_mac_tbl_ent = OPA_VNIC_MAC_TBL_MAX_ENTRIES;
-	adapter->info.vport.max_smac_ent = OPA_VNIC_MAX_SMAC_LIMIT;
-	adapter->info.vport.config_state = OPA_VNIC_STATE_DROP_ALL;
-	adapter->info.vport.eth_link_status = OPA_VNIC_ETH_LINK_DOWN;
-	adapter->info.vesw.eth_mtu = ETH_DATA_LEN;
-}
-
-/* opa_vnic_set_mac_addr - change mac address */
-static int opa_vnic_set_mac_addr(struct net_device *netdev, void *addr)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	struct sockaddr *sa = addr;
-	int rc;
-
-	if (!memcmp(netdev->dev_addr, sa->sa_data, ETH_ALEN))
-		return 0;
-
-	mutex_lock(&adapter->lock);
-	rc = eth_mac_addr(netdev, addr);
-	mutex_unlock(&adapter->lock);
-	if (rc)
-		return rc;
-
-	adapter->info.vport.uc_macs_gen_count++;
-	opa_vnic_vema_report_event(adapter,
-				   OPA_VESWPORT_TRAP_IFACE_UCAST_MAC_CHANGE);
-	return 0;
-}
-
-/*
- * opa_vnic_mac_send_event - post event on possible mac list exchange
- *  Send trap when digest from uc/mc mac list differs from previous run.
- *  Digest is evaluated similar to how cksum does.
- */
-static void opa_vnic_mac_send_event(struct net_device *netdev, u8 event)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	struct netdev_hw_addr *ha;
-	struct netdev_hw_addr_list *hw_list;
-	u32 *ref_crc;
-	u32 l, crc = 0;
-
-	switch (event) {
-	case OPA_VESWPORT_TRAP_IFACE_UCAST_MAC_CHANGE:
-		hw_list = &netdev->uc;
-		adapter->info.vport.uc_macs_gen_count++;
-		ref_crc = &adapter->umac_hash;
-		break;
-	case OPA_VESWPORT_TRAP_IFACE_MCAST_MAC_CHANGE:
-		hw_list = &netdev->mc;
-		adapter->info.vport.mc_macs_gen_count++;
-		ref_crc = &adapter->mmac_hash;
-		break;
-	default:
-		return;
-	}
-	netdev_hw_addr_list_for_each(ha, hw_list) {
-		crc = crc32_le(crc, ha->addr, ETH_ALEN);
-	}
-	l = netdev_hw_addr_list_count(hw_list) * ETH_ALEN;
-	crc = ~crc32_le(crc, (void *)&l, sizeof(l));
-
-	if (crc != *ref_crc) {
-		*ref_crc = crc;
-		opa_vnic_vema_report_event(adapter, event);
-	}
-}
-
-/* opa_vnic_set_rx_mode - handle uc/mc mac list change */
-static void opa_vnic_set_rx_mode(struct net_device *netdev)
-{
-	opa_vnic_mac_send_event(netdev,
-				OPA_VESWPORT_TRAP_IFACE_UCAST_MAC_CHANGE);
-
-	opa_vnic_mac_send_event(netdev,
-				OPA_VESWPORT_TRAP_IFACE_MCAST_MAC_CHANGE);
-}
-
-/* opa_netdev_open - activate network interface */
-static int opa_netdev_open(struct net_device *netdev)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	int rc;
-
-	rc = adapter->rn_ops->ndo_open(adapter->netdev);
-	if (rc) {
-		v_dbg("open failed %d\n", rc);
-		return rc;
-	}
-
-	/* Update status and send trap */
-	opa_vnic_update_state(adapter, true);
-	opa_vnic_vema_report_event(adapter,
-				   OPA_VESWPORT_TRAP_ETH_LINK_STATUS_CHANGE);
-	return 0;
-}
-
-/* opa_netdev_close - disable network interface */
-static int opa_netdev_close(struct net_device *netdev)
-{
-	struct opa_vnic_adapter *adapter = opa_vnic_priv(netdev);
-	int rc;
-
-	rc = adapter->rn_ops->ndo_stop(adapter->netdev);
-	if (rc) {
-		v_dbg("close failed %d\n", rc);
-		return rc;
-	}
-
-	/* Update status and send trap */
-	opa_vnic_update_state(adapter, false);
-	opa_vnic_vema_report_event(adapter,
-				   OPA_VESWPORT_TRAP_ETH_LINK_STATUS_CHANGE);
-	return 0;
-}
-
-/* netdev ops */
-static const struct net_device_ops opa_netdev_ops = {
-	.ndo_open = opa_netdev_open,
-	.ndo_stop = opa_netdev_close,
-	.ndo_start_xmit = opa_netdev_start_xmit,
-	.ndo_get_stats64 = opa_vnic_get_stats64,
-	.ndo_set_rx_mode = opa_vnic_set_rx_mode,
-	.ndo_select_queue = opa_vnic_select_queue,
-	.ndo_set_mac_address = opa_vnic_set_mac_addr,
-};
-
-/* opa_vnic_add_netdev - create vnic netdev interface */
-struct opa_vnic_adapter *opa_vnic_add_netdev(struct ib_device *ibdev,
-					     u8 port_num, u8 vport_num)
-{
-	struct opa_vnic_adapter *adapter;
-	struct net_device *netdev;
-	struct rdma_netdev *rn;
-	int rc;
-
-	netdev = ibdev->ops.alloc_rdma_netdev(ibdev, port_num,
-					      RDMA_NETDEV_OPA_VNIC,
-					      "veth%d", NET_NAME_UNKNOWN,
-					      ether_setup);
-	if (!netdev)
-		return ERR_PTR(-ENOMEM);
-	else if (IS_ERR(netdev))
-		return ERR_CAST(netdev);
-
-	rn = netdev_priv(netdev);
-	adapter = kzalloc_obj(*adapter);
-	if (!adapter) {
-		rc = -ENOMEM;
-		goto adapter_err;
-	}
-
-	rn->clnt_priv = adapter;
-	rn->hca = ibdev;
-	rn->port_num = port_num;
-	adapter->netdev = netdev;
-	adapter->ibdev = ibdev;
-	adapter->port_num = port_num;
-	adapter->vport_num = vport_num;
-	adapter->rn_ops = netdev->netdev_ops;
-
-	netdev->netdev_ops = &opa_netdev_ops;
-	netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
-	netdev->hard_header_len += OPA_VNIC_SKB_HEADROOM;
-	mutex_init(&adapter->lock);
-	mutex_init(&adapter->mactbl_lock);
-	spin_lock_init(&adapter->stats_lock);
-
-	SET_NETDEV_DEV(netdev, ibdev->dev.parent);
-
-	opa_vnic_set_ethtool_ops(netdev);
-
-	opa_vnic_set_pod_values(adapter);
-
-	rc = register_netdev(netdev);
-	if (rc)
-		goto netdev_err;
-
-	netif_carrier_off(netdev);
-	netif_dormant_on(netdev);
-	v_info("initialized\n");
-
-	return adapter;
-netdev_err:
-	mutex_destroy(&adapter->lock);
-	mutex_destroy(&adapter->mactbl_lock);
-	kfree(adapter);
-adapter_err:
-	rn->free_rdma_netdev(netdev);
-
-	return ERR_PTR(rc);
-}
-
-/* opa_vnic_rem_netdev - remove vnic netdev interface */
-void opa_vnic_rem_netdev(struct opa_vnic_adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-	struct rdma_netdev *rn = netdev_priv(netdev);
-
-	v_info("removing\n");
-	unregister_netdev(netdev);
-	opa_vnic_release_mac_tbl(adapter);
-	mutex_destroy(&adapter->lock);
-	mutex_destroy(&adapter->mactbl_lock);
-	kfree(adapter);
-	rn->free_rdma_netdev(netdev);
-}
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
deleted file mode 100644
index 21c6cea8b1db7..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+++ /dev/null
@@ -1,1056 +0,0 @@
-/*
- * Copyright(c) 2017 Intel Corporation.
- * Copyright(c) 2021 Cornelis Networks.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPX Virtual Network Interface Controller (VNIC)
- * Ethernet Management Agent (EMA) driver
- */
-
-#include <linux/module.h>
-#include <linux/xarray.h>
-#include <rdma/ib_addr.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/opa_smi.h>
-#include <rdma/opa_port_info.h>
-
-#include "opa_vnic_internal.h"
-
-char opa_vnic_driver_name[] = "opa_vnic";
-
-/*
- * The trap service level is kept in bits 3 to 7 in the trap_sl_rsvd
- * field in the class port info MAD.
- */
-#define GET_TRAP_SL_FROM_CLASS_PORT_INFO(x)  (((x) >> 3) & 0x1f)
-
-/* Cap trap bursts to a reasonable limit good for normal cases */
-#define OPA_VNIC_TRAP_BURST_LIMIT 4
-
-/*
- * VNIC trap limit timeout.
- * Inverse of cap2_mask response time out (1.0737 secs) = 0.9
- * secs approx IB spec 13.4.6.2.1 PortInfoSubnetTimeout and
- * 13.4.9 Traps.
- */
-#define OPA_VNIC_TRAP_TIMEOUT  ((4096 * (1UL << 18)) / 1000)
-
-#define OPA_VNIC_UNSUP_ATTR  \
-		cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB)
-
-#define OPA_VNIC_INVAL_ATTR  \
-		cpu_to_be16(IB_MGMT_MAD_STATUS_INVALID_ATTRIB_VALUE)
-
-#define OPA_VNIC_CLASS_CAP_TRAP   0x1
-
-/* Maximum number of VNIC ports supported */
-#define OPA_VNIC_MAX_NUM_VPORT    255
-
-/**
- * struct opa_vnic_vema_port -- VNIC VEMA port details
- * @cport: pointer to port
- * @mad_agent: pointer to mad agent for port
- * @class_port_info: Class port info information.
- * @tid: Transaction id
- * @port_num: OPA port number
- * @vports: vnic ports
- * @event_handler: ib event handler
- * @lock: adapter interface lock
- */
-struct opa_vnic_vema_port {
-	struct opa_vnic_ctrl_port      *cport;
-	struct ib_mad_agent            *mad_agent;
-	struct opa_class_port_info      class_port_info;
-	u64                             tid;
-	u8                              port_num;
-	struct xarray                   vports;
-	struct ib_event_handler         event_handler;
-
-	/* Lock to query/update network adapter */
-	struct mutex                    lock;
-};
-
-static int opa_vnic_vema_add_one(struct ib_device *device);
-static void opa_vnic_vema_rem_one(struct ib_device *device,
-				  void *client_data);
-
-static struct ib_client opa_vnic_client = {
-	.name   = opa_vnic_driver_name,
-	.add    = opa_vnic_vema_add_one,
-	.remove = opa_vnic_vema_rem_one,
-};
-
-/**
- * vema_get_vport_num -- Get the vnic from the mad
- * @recvd_mad:  Received mad
- *
- * Return: returns value of the vnic port number
- */
-static inline u8 vema_get_vport_num(struct opa_vnic_vema_mad *recvd_mad)
-{
-	return be32_to_cpu(recvd_mad->mad_hdr.attr_mod) & 0xff;
-}
-
-/**
- * vema_get_vport_adapter -- Get vnic port adapter from recvd mad
- * @recvd_mad: received mad
- * @port: ptr to port struct on which MAD was recvd
- *
- * Return: vnic adapter
- */
-static inline struct opa_vnic_adapter *
-vema_get_vport_adapter(struct opa_vnic_vema_mad *recvd_mad,
-		       struct opa_vnic_vema_port *port)
-{
-	u8 vport_num = vema_get_vport_num(recvd_mad);
-
-	return xa_load(&port->vports, vport_num);
-}
-
-/**
- * vema_mac_tbl_req_ok -- Check if mac request has correct values
- * @mac_tbl: mac table
- *
- * This function checks for the validity of the offset and number of
- * entries required.
- *
- * Return: true if offset and num_entries are valid
- */
-static inline bool vema_mac_tbl_req_ok(struct opa_veswport_mactable *mac_tbl)
-{
-	u16 offset, num_entries;
-	u16 req_entries = ((OPA_VNIC_EMA_DATA - sizeof(*mac_tbl)) /
-			   sizeof(mac_tbl->tbl_entries[0]));
-
-	offset = be16_to_cpu(mac_tbl->offset);
-	num_entries = be16_to_cpu(mac_tbl->num_entries);
-
-	return ((num_entries <= req_entries) &&
-		(offset + num_entries <= OPA_VNIC_MAC_TBL_MAX_ENTRIES));
-}
-
-/*
- * Return the power on default values in the port info structure
- * in big endian format as required by MAD.
- */
-static inline void vema_get_pod_values(struct opa_veswport_info *port_info)
-{
-	memset(port_info, 0, sizeof(*port_info));
-	port_info->vport.max_mac_tbl_ent =
-		cpu_to_be16(OPA_VNIC_MAC_TBL_MAX_ENTRIES);
-	port_info->vport.max_smac_ent =
-		cpu_to_be16(OPA_VNIC_MAX_SMAC_LIMIT);
-	port_info->vport.oper_state = OPA_VNIC_STATE_DROP_ALL;
-	port_info->vport.config_state = OPA_VNIC_STATE_DROP_ALL;
-	port_info->vesw.eth_mtu = cpu_to_be16(ETH_DATA_LEN);
-}
-
-/**
- * vema_add_vport -- Add a new vnic port
- * @port: ptr to opa_vnic_vema_port struct
- * @vport_num: vnic port number (to be added)
- *
- * Return a pointer to the vnic adapter structure
- */
-static struct opa_vnic_adapter *vema_add_vport(struct opa_vnic_vema_port *port,
-					       u8 vport_num)
-{
-	struct opa_vnic_ctrl_port *cport = port->cport;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = opa_vnic_add_netdev(cport->ibdev, port->port_num, vport_num);
-	if (!IS_ERR(adapter)) {
-		int rc;
-
-		adapter->cport = cport;
-		rc = xa_insert(&port->vports, vport_num, adapter, GFP_KERNEL);
-		if (rc < 0) {
-			opa_vnic_rem_netdev(adapter);
-			adapter = ERR_PTR(rc);
-		}
-	}
-
-	return adapter;
-}
-
-/**
- * vema_get_class_port_info -- Get class info for port
- * @port:  Port on whic MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function copies the latest class port info value set for the
- * port and stores it for generating traps
- */
-static void vema_get_class_port_info(struct opa_vnic_vema_port *port,
-				     struct opa_vnic_vema_mad *recvd_mad,
-				     struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_class_port_info *port_info;
-
-	port_info = (struct opa_class_port_info *)rsp_mad->data;
-	memcpy(port_info, &port->class_port_info, sizeof(*port_info));
-	port_info->base_version = OPA_MGMT_BASE_VERSION;
-	port_info->class_version = OPA_EMA_CLASS_VERSION;
-
-	/*
-	 * Set capability mask bit indicating agent generates traps,
-	 * and set the maximum number of VNIC ports supported.
-	 */
-	port_info->cap_mask = cpu_to_be16((OPA_VNIC_CLASS_CAP_TRAP |
-					   (OPA_VNIC_MAX_NUM_VPORT << 8)));
-
-	/*
-	 * Since a get routine is always sent by the EM first we
-	 * set the expected response time to
-	 * 4.096 usec * 2^18 == 1.0737 sec here.
-	 */
-	port_info->cap_mask2_resp_time = cpu_to_be32(18);
-}
-
-/**
- * vema_set_class_port_info -- Get class info for port
- * @port:  Port on whic MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function updates the port class info for the specific vnic
- * and sets up the response mad data
- */
-static void vema_set_class_port_info(struct opa_vnic_vema_port *port,
-				     struct opa_vnic_vema_mad *recvd_mad,
-				     struct opa_vnic_vema_mad *rsp_mad)
-{
-	memcpy(&port->class_port_info, recvd_mad->data,
-	       sizeof(port->class_port_info));
-
-	vema_get_class_port_info(port, recvd_mad, rsp_mad);
-}
-
-/**
- * vema_get_veswport_info -- Get veswport info
- * @port:      source port on which MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- */
-static void vema_get_veswport_info(struct opa_vnic_vema_port *port,
-				   struct opa_vnic_vema_mad *recvd_mad,
-				   struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_info *port_info =
-				  (struct opa_veswport_info *)rsp_mad->data;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (adapter) {
-		memset(port_info, 0, sizeof(*port_info));
-		opa_vnic_get_vesw_info(adapter, &port_info->vesw);
-		opa_vnic_get_per_veswport_info(adapter,
-					       &port_info->vport);
-	} else {
-		vema_get_pod_values(port_info);
-	}
-}
-
-/**
- * vema_set_veswport_info -- Set veswport info
- * @port:      source port on which MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function gets the port class infor for vnic
- */
-static void vema_set_veswport_info(struct opa_vnic_vema_port *port,
-				   struct opa_vnic_vema_mad *recvd_mad,
-				   struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_vnic_ctrl_port *cport = port->cport;
-	struct opa_veswport_info *port_info;
-	struct opa_vnic_adapter *adapter;
-	u8 vport_num;
-
-	vport_num = vema_get_vport_num(recvd_mad);
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (!adapter) {
-		adapter = vema_add_vport(port, vport_num);
-		if (IS_ERR(adapter)) {
-			c_err("failed to add vport %d: %ld\n",
-			      vport_num, PTR_ERR(adapter));
-			goto err_exit;
-		}
-	}
-
-	port_info = (struct opa_veswport_info *)recvd_mad->data;
-	opa_vnic_set_vesw_info(adapter, &port_info->vesw);
-	opa_vnic_set_per_veswport_info(adapter, &port_info->vport);
-
-	/* Process the new config settings */
-	opa_vnic_process_vema_config(adapter);
-
-	vema_get_veswport_info(port, recvd_mad, rsp_mad);
-	return;
-
-err_exit:
-	rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-}
-
-/**
- * vema_get_mac_entries -- Get MAC entries in VNIC MAC table
- * @port:      source port on which MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function gets the MAC entries that are programmed into
- * the VNIC MAC forwarding table. It checks for the validity of
- * the index into the MAC table and the number of entries that
- * are to be retrieved.
- */
-static void vema_get_mac_entries(struct opa_vnic_vema_port *port,
-				 struct opa_vnic_vema_mad *recvd_mad,
-				 struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_mactable *mac_tbl_in, *mac_tbl_out;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (!adapter) {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-		return;
-	}
-
-	mac_tbl_in = (struct opa_veswport_mactable *)recvd_mad->data;
-	mac_tbl_out = (struct opa_veswport_mactable *)rsp_mad->data;
-
-	if (vema_mac_tbl_req_ok(mac_tbl_in)) {
-		mac_tbl_out->offset = mac_tbl_in->offset;
-		mac_tbl_out->num_entries = mac_tbl_in->num_entries;
-		opa_vnic_query_mac_tbl(adapter, mac_tbl_out);
-	} else {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-	}
-}
-
-/**
- * vema_set_mac_entries -- Set MAC entries in VNIC MAC table
- * @port:      source port on which MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function sets the MAC entries in the VNIC forwarding table
- * It checks for the validity of the index and the number of forwarding
- * table entries to be programmed.
- */
-static void vema_set_mac_entries(struct opa_vnic_vema_port *port,
-				 struct opa_vnic_vema_mad *recvd_mad,
-				 struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_mactable *mac_tbl;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (!adapter) {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-		return;
-	}
-
-	mac_tbl = (struct opa_veswport_mactable *)recvd_mad->data;
-	if (vema_mac_tbl_req_ok(mac_tbl)) {
-		if (opa_vnic_update_mac_tbl(adapter, mac_tbl))
-			rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
-	} else {
-		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
-	}
-	vema_get_mac_entries(port, recvd_mad, rsp_mad);
-}
-
-/**
- * vema_set_delete_vesw -- Reset VESW info to POD values
- * @port:      source port on which MAD was received
- * @recvd_mad: pointer to the received mad
- * @rsp_mad:   pointer to respose mad
- *
- * This function clears all the fields of veswport info for the requested vesw
- * and sets them back to the power-on default values. It does not delete the
- * vesw.
- */
-static void vema_set_delete_vesw(struct opa_vnic_vema_port *port,
-				 struct opa_vnic_vema_mad *recvd_mad,
-				 struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_info *port_info =
-				  (struct opa_veswport_info *)rsp_mad->data;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (!adapter) {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-		return;
-	}
-
-	vema_get_pod_values(port_info);
-	opa_vnic_set_vesw_info(adapter, &port_info->vesw);
-	opa_vnic_set_per_veswport_info(adapter, &port_info->vport);
-
-	/* Process the new config settings */
-	opa_vnic_process_vema_config(adapter);
-
-	opa_vnic_release_mac_tbl(adapter);
-
-	vema_get_veswport_info(port, recvd_mad, rsp_mad);
-}
-
-/**
- * vema_get_mac_list -- Get the unicast/multicast macs.
- * @port:      source port on which MAD was received
- * @recvd_mad: Received mad contains fields to set vnic parameters
- * @rsp_mad:   Response mad to be built
- * @attr_id:   Attribute ID indicating multicast or unicast mac list
- */
-static void vema_get_mac_list(struct opa_vnic_vema_port *port,
-			      struct opa_vnic_vema_mad *recvd_mad,
-			      struct opa_vnic_vema_mad *rsp_mad,
-			      u16 attr_id)
-{
-	struct opa_veswport_iface_macs *macs_in, *macs_out;
-	int max_entries = (OPA_VNIC_EMA_DATA - sizeof(*macs_out)) / ETH_ALEN;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (!adapter) {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-		return;
-	}
-
-	macs_in = (struct opa_veswport_iface_macs *)recvd_mad->data;
-	macs_out = (struct opa_veswport_iface_macs *)rsp_mad->data;
-
-	macs_out->start_idx = macs_in->start_idx;
-	if (macs_in->num_macs_in_msg)
-		macs_out->num_macs_in_msg = macs_in->num_macs_in_msg;
-	else
-		macs_out->num_macs_in_msg = cpu_to_be16(max_entries);
-
-	if (attr_id == OPA_EM_ATTR_IFACE_MCAST_MACS)
-		opa_vnic_query_mcast_macs(adapter, macs_out);
-	else
-		opa_vnic_query_ucast_macs(adapter, macs_out);
-}
-
-/**
- * vema_get_summary_counters -- Gets summary counters.
- * @port:      source port on which MAD was received
- * @recvd_mad: Received mad contains fields to set vnic parameters
- * @rsp_mad:   Response mad to be built
- */
-static void vema_get_summary_counters(struct opa_vnic_vema_port *port,
-				      struct opa_vnic_vema_mad *recvd_mad,
-				      struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_summary_counters *cntrs;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (adapter) {
-		cntrs = (struct opa_veswport_summary_counters *)rsp_mad->data;
-		opa_vnic_get_summary_counters(adapter, cntrs);
-	} else {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-	}
-}
-
-/**
- * vema_get_error_counters -- Gets summary counters.
- * @port:      source port on which MAD was received
- * @recvd_mad: Received mad contains fields to set vnic parameters
- * @rsp_mad:   Response mad to be built
- */
-static void vema_get_error_counters(struct opa_vnic_vema_port *port,
-				    struct opa_vnic_vema_mad *recvd_mad,
-				    struct opa_vnic_vema_mad *rsp_mad)
-{
-	struct opa_veswport_error_counters *cntrs;
-	struct opa_vnic_adapter *adapter;
-
-	adapter = vema_get_vport_adapter(recvd_mad, port);
-	if (adapter) {
-		cntrs = (struct opa_veswport_error_counters *)rsp_mad->data;
-		opa_vnic_get_error_counters(adapter, cntrs);
-	} else {
-		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
-	}
-}
-
-/**
- * vema_get -- Process received get MAD
- * @port:      source port on which MAD was received
- * @recvd_mad: Received mad
- * @rsp_mad:   Response mad to be built
- */
-static void vema_get(struct opa_vnic_vema_port *port,
-		     struct opa_vnic_vema_mad *recvd_mad,
-		     struct opa_vnic_vema_mad *rsp_mad)
-{
-	u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id);
-
-	switch (attr_id) {
-	case OPA_EM_ATTR_CLASS_PORT_INFO:
-		vema_get_class_port_info(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_VESWPORT_INFO:
-		vema_get_veswport_info(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES:
-		vema_get_mac_entries(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_IFACE_UCAST_MACS:
-	case OPA_EM_ATTR_IFACE_MCAST_MACS:
-		vema_get_mac_list(port, recvd_mad, rsp_mad, attr_id);
-		break;
-	case OPA_EM_ATTR_VESWPORT_SUMMARY_COUNTERS:
-		vema_get_summary_counters(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_VESWPORT_ERROR_COUNTERS:
-		vema_get_error_counters(port, recvd_mad, rsp_mad);
-		break;
-	default:
-		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
-		break;
-	}
-}
-
-/**
- * vema_set -- Process received set MAD
- * @port:      source port on which MAD was received
- * @recvd_mad: Received mad contains fields to set vnic parameters
- * @rsp_mad:   Response mad to be built
- */
-static void vema_set(struct opa_vnic_vema_port *port,
-		     struct opa_vnic_vema_mad *recvd_mad,
-		     struct opa_vnic_vema_mad *rsp_mad)
-{
-	u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id);
-
-	switch (attr_id) {
-	case OPA_EM_ATTR_CLASS_PORT_INFO:
-		vema_set_class_port_info(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_VESWPORT_INFO:
-		vema_set_veswport_info(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES:
-		vema_set_mac_entries(port, recvd_mad, rsp_mad);
-		break;
-	case OPA_EM_ATTR_DELETE_VESW:
-		vema_set_delete_vesw(port, recvd_mad, rsp_mad);
-		break;
-	default:
-		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
-		break;
-	}
-}
-
-/**
- * vema_send -- Send handler for VEMA MAD agent
- * @mad_agent: pointer to the mad agent
- * @mad_wc:    pointer to mad send work completion information
- *
- * Free all the data structures associated with the sent MAD
- */
-static void vema_send(struct ib_mad_agent *mad_agent,
-		      struct ib_mad_send_wc *mad_wc)
-{
-	rdma_destroy_ah(mad_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE);
-	ib_free_send_mad(mad_wc->send_buf);
-}
-
-/**
- * vema_recv -- Recv handler for VEMA MAD agent
- * @mad_agent: pointer to the mad agent
- * @send_buf: Send buffer if found, else NULL
- * @mad_wc:    pointer to mad send work completion information
- *
- * Handle only set and get methods and respond to other methods
- * as unsupported. Allocate response buffer and address handle
- * for the response MAD.
- */
-static void vema_recv(struct ib_mad_agent *mad_agent,
-		      struct ib_mad_send_buf *send_buf,
-		      struct ib_mad_recv_wc *mad_wc)
-{
-	struct opa_vnic_vema_port *port;
-	struct ib_ah              *ah;
-	struct ib_mad_send_buf    *rsp;
-	struct opa_vnic_vema_mad  *vema_mad;
-
-	if (!mad_wc || !mad_wc->recv_buf.mad)
-		return;
-
-	port = mad_agent->context;
-	ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
-				  mad_wc->recv_buf.grh, mad_agent->port_num);
-	if (IS_ERR(ah))
-		goto free_recv_mad;
-
-	rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
-				 mad_wc->wc->pkey_index, 0,
-				 IB_MGMT_VENDOR_HDR, OPA_VNIC_EMA_DATA,
-				 GFP_KERNEL, OPA_MGMT_BASE_VERSION);
-	if (IS_ERR(rsp))
-		goto err_rsp;
-
-	rsp->ah = ah;
-	vema_mad = rsp->mad;
-	memcpy(vema_mad, mad_wc->recv_buf.mad, IB_MGMT_VENDOR_HDR);
-	vema_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
-	vema_mad->mad_hdr.status = 0;
-
-	/* Lock ensures network adapter is not removed */
-	mutex_lock(&port->lock);
-
-	switch (mad_wc->recv_buf.mad->mad_hdr.method) {
-	case IB_MGMT_METHOD_GET:
-		vema_get(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad,
-			 vema_mad);
-		break;
-	case IB_MGMT_METHOD_SET:
-		vema_set(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad,
-			 vema_mad);
-		break;
-	default:
-		vema_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
-		break;
-	}
-	mutex_unlock(&port->lock);
-
-	if (!ib_post_send_mad(rsp, NULL)) {
-		/*
-		 * with post send successful ah and send mad
-		 * will be destroyed in send handler
-		 */
-		goto free_recv_mad;
-	}
-
-	ib_free_send_mad(rsp);
-
-err_rsp:
-	rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
-free_recv_mad:
-	ib_free_recv_mad(mad_wc);
-}
-
-/**
- * vema_get_port -- Gets the opa_vnic_vema_port
- * @cport: pointer to control dev
- * @port_num: Port number
- *
- * This function loops through the ports and returns
- * the opa_vnic_vema port structure that is associated
- * with the OPA port number
- *
- * Return: ptr to requested opa_vnic_vema_port strucure
- *         if success, NULL if not
- */
-static struct opa_vnic_vema_port *
-vema_get_port(struct opa_vnic_ctrl_port *cport, u8 port_num)
-{
-	struct opa_vnic_vema_port *port = (void *)cport + sizeof(*cport);
-
-	if (port_num > cport->num_ports)
-		return NULL;
-
-	return port + (port_num - 1);
-}
-
-/**
- * opa_vnic_vema_send_trap -- This function sends a trap to the EM
- * @adapter: pointer to vnic adapter
- * @data: pointer to trap data filled by calling function
- * @lid:  issuers lid (encap_slid from vesw_port_info)
- *
- * This function is called from the VNIC driver to send a trap if there
- * is somethng the EM should be notified about. These events currently
- * are
- * 1) UNICAST INTERFACE MACADDRESS changes
- * 2) MULTICAST INTERFACE MACADDRESS changes
- * 3) ETHERNET LINK STATUS changes
- * While allocating the send mad the remote site qpn used is 1
- * as this is the well known QP.
- *
- */
-void opa_vnic_vema_send_trap(struct opa_vnic_adapter *adapter,
-			     struct __opa_veswport_trap *data, u32 lid)
-{
-	struct opa_vnic_ctrl_port *cport = adapter->cport;
-	struct ib_mad_send_buf *send_buf;
-	struct opa_vnic_vema_port *port;
-	struct ib_device *ibp;
-	struct opa_vnic_vema_mad_trap *trap_mad;
-	struct opa_class_port_info *class;
-	struct rdma_ah_attr ah_attr;
-	struct ib_ah *ah;
-	struct opa_veswport_trap *trap;
-	u32 trap_lid;
-	u16 pkey_idx;
-
-	if (!cport)
-		goto err_exit;
-	ibp = cport->ibdev;
-	port = vema_get_port(cport, data->opaportnum);
-	if (!port || !port->mad_agent)
-		goto err_exit;
-
-	if (time_before(jiffies, adapter->trap_timeout)) {
-		if (adapter->trap_count == OPA_VNIC_TRAP_BURST_LIMIT) {
-			v_warn("Trap rate exceeded\n");
-			goto err_exit;
-		} else {
-			adapter->trap_count++;
-		}
-	} else {
-		adapter->trap_count = 0;
-	}
-
-	class = &port->class_port_info;
-	/* Set up address handle */
-	memset(&ah_attr, 0, sizeof(ah_attr));
-	ah_attr.type = rdma_ah_find_type(ibp, port->port_num);
-	rdma_ah_set_sl(&ah_attr,
-		       GET_TRAP_SL_FROM_CLASS_PORT_INFO(class->trap_sl_rsvd));
-	rdma_ah_set_port_num(&ah_attr, port->port_num);
-	trap_lid = be32_to_cpu(class->trap_lid);
-	/*
-	 * check for trap lid validity, must not be zero
-	 * The trap sink could change after we fashion the MAD but since traps
-	 * are not guaranteed we won't use a lock as anyway the change will take
-	 * place even with locking.
-	 */
-	if (!trap_lid) {
-		c_err("%s: Invalid dlid\n", __func__);
-		goto err_exit;
-	}
-
-	rdma_ah_set_dlid(&ah_attr, trap_lid);
-	ah = rdma_create_ah(port->mad_agent->qp->pd, &ah_attr, 0);
-	if (IS_ERR(ah)) {
-		c_err("%s:Couldn't create new AH = %p\n", __func__, ah);
-		c_err("%s:dlid = %d, sl = %d, port = %d\n", __func__,
-		      rdma_ah_get_dlid(&ah_attr), rdma_ah_get_sl(&ah_attr),
-		      rdma_ah_get_port_num(&ah_attr));
-		goto err_exit;
-	}
-
-	if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_FULL,
-			 &pkey_idx) < 0) {
-		c_err("%s:full key not found, defaulting to partial\n",
-		      __func__);
-		if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_PARTIAL,
-				 &pkey_idx) < 0)
-			pkey_idx = 1;
-	}
-
-	send_buf = ib_create_send_mad(port->mad_agent, 1, pkey_idx, 0,
-				      IB_MGMT_VENDOR_HDR, IB_MGMT_MAD_DATA,
-				      GFP_ATOMIC, OPA_MGMT_BASE_VERSION);
-	if (IS_ERR(send_buf)) {
-		c_err("%s:Couldn't allocate send buf\n", __func__);
-		goto err_sndbuf;
-	}
-
-	send_buf->ah = ah;
-
-	/* Set up common MAD hdr */
-	trap_mad = send_buf->mad;
-	trap_mad->mad_hdr.base_version = OPA_MGMT_BASE_VERSION;
-	trap_mad->mad_hdr.mgmt_class = OPA_MGMT_CLASS_INTEL_EMA;
-	trap_mad->mad_hdr.class_version = OPA_EMA_CLASS_VERSION;
-	trap_mad->mad_hdr.method = IB_MGMT_METHOD_TRAP;
-	port->tid++;
-	trap_mad->mad_hdr.tid = cpu_to_be64(port->tid);
-	trap_mad->mad_hdr.attr_id = IB_SMP_ATTR_NOTICE;
-
-	/* Set up vendor OUI */
-	trap_mad->oui[0] = INTEL_OUI_1;
-	trap_mad->oui[1] = INTEL_OUI_2;
-	trap_mad->oui[2] = INTEL_OUI_3;
-
-	/* Setup notice attribute portion */
-	trap_mad->notice.gen_type = OPA_INTEL_EMA_NOTICE_TYPE_INFO << 1;
-	trap_mad->notice.oui_1 = INTEL_OUI_1;
-	trap_mad->notice.oui_2 = INTEL_OUI_2;
-	trap_mad->notice.oui_3 = INTEL_OUI_3;
-	trap_mad->notice.issuer_lid = cpu_to_be32(lid);
-
-	/* copy the actual trap data */
-	trap = (struct opa_veswport_trap *)trap_mad->notice.raw_data;
-	trap->fabric_id = cpu_to_be16(data->fabric_id);
-	trap->veswid = cpu_to_be16(data->veswid);
-	trap->veswportnum = cpu_to_be32(data->veswportnum);
-	trap->opaportnum = cpu_to_be16(data->opaportnum);
-	trap->veswportindex = data->veswportindex;
-	trap->opcode = data->opcode;
-
-	/* If successful send set up rate limit timeout else bail */
-	if (ib_post_send_mad(send_buf, NULL)) {
-		ib_free_send_mad(send_buf);
-	} else {
-		if (adapter->trap_count)
-			return;
-		adapter->trap_timeout = jiffies +
-					usecs_to_jiffies(OPA_VNIC_TRAP_TIMEOUT);
-		return;
-	}
-
-err_sndbuf:
-	rdma_destroy_ah(ah, 0);
-err_exit:
-	v_err("Aborting trap\n");
-}
-
-static void opa_vnic_event(struct ib_event_handler *handler,
-			   struct ib_event *record)
-{
-	struct opa_vnic_vema_port *port =
-		container_of(handler, struct opa_vnic_vema_port, event_handler);
-	struct opa_vnic_ctrl_port *cport = port->cport;
-	struct opa_vnic_adapter *adapter;
-	unsigned long index;
-
-	if (record->element.port_num != port->port_num)
-		return;
-
-	c_dbg("OPA_VNIC received event %d on device %s port %d\n",
-	      record->event, dev_name(&record->device->dev),
-	      record->element.port_num);
-
-	if (record->event != IB_EVENT_PORT_ERR &&
-	    record->event != IB_EVENT_PORT_ACTIVE)
-		return;
-
-	xa_for_each(&port->vports, index, adapter) {
-		if (record->event == IB_EVENT_PORT_ACTIVE)
-			netif_carrier_on(adapter->netdev);
-		else
-			netif_carrier_off(adapter->netdev);
-	}
-}
-
-/**
- * vema_unregister -- Unregisters agent
- * @cport: pointer to control port
- *
- * This deletes the registration by VEMA for MADs
- */
-static void vema_unregister(struct opa_vnic_ctrl_port *cport)
-{
-	struct opa_vnic_adapter *adapter;
-	unsigned long index;
-	int i;
-
-	for (i = 1; i <= cport->num_ports; i++) {
-		struct opa_vnic_vema_port *port = vema_get_port(cport, i);
-
-		if (!port->mad_agent)
-			continue;
-
-		/* Lock ensures no MAD is being processed */
-		mutex_lock(&port->lock);
-		xa_for_each(&port->vports, index, adapter)
-			opa_vnic_rem_netdev(adapter);
-		mutex_unlock(&port->lock);
-
-		ib_unregister_mad_agent(port->mad_agent);
-		port->mad_agent = NULL;
-		mutex_destroy(&port->lock);
-		xa_destroy(&port->vports);
-		ib_unregister_event_handler(&port->event_handler);
-	}
-}
-
-/**
- * vema_register -- Registers agent
- * @cport: pointer to control port
- *
- * This function registers the handlers for the VEMA MADs
- *
- * Return: returns 0 on success. non zero otherwise
- */
-static int vema_register(struct opa_vnic_ctrl_port *cport)
-{
-	struct ib_mad_reg_req reg_req = {
-		.mgmt_class = OPA_MGMT_CLASS_INTEL_EMA,
-		.mgmt_class_version = OPA_MGMT_BASE_VERSION,
-		.oui = { INTEL_OUI_1, INTEL_OUI_2, INTEL_OUI_3 }
-	};
-	int i;
-
-	set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
-	set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
-
-	/* register ib event handler and mad agent for each port on dev */
-	for (i = 1; i <= cport->num_ports; i++) {
-		struct opa_vnic_vema_port *port = vema_get_port(cport, i);
-		int ret;
-
-		port->cport = cport;
-		port->port_num = i;
-
-		INIT_IB_EVENT_HANDLER(&port->event_handler,
-				      cport->ibdev, opa_vnic_event);
-		ib_register_event_handler(&port->event_handler);
-
-		xa_init(&port->vports);
-		mutex_init(&port->lock);
-		port->mad_agent = ib_register_mad_agent(cport->ibdev, i,
-							IB_QPT_GSI, &reg_req,
-							IB_MGMT_RMPP_VERSION,
-							vema_send, vema_recv,
-							port, 0);
-		if (IS_ERR(port->mad_agent)) {
-			ret = PTR_ERR(port->mad_agent);
-			port->mad_agent = NULL;
-			mutex_destroy(&port->lock);
-			vema_unregister(cport);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * opa_vnic_ctrl_config_dev -- This function sends a trap to the EM
- * by way of ib_modify_port to indicate support for ethernet on the
- * fabric.
- * @cport: pointer to control port
- * @en: enable or disable ethernet on fabric support
- */
-static void opa_vnic_ctrl_config_dev(struct opa_vnic_ctrl_port *cport, bool en)
-{
-	struct ib_port_modify pm = { 0 };
-	int i;
-
-	if (en)
-		pm.set_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported;
-	else
-		pm.clr_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported;
-
-	for (i = 1; i <= cport->num_ports; i++)
-		ib_modify_port(cport->ibdev, i, IB_PORT_OPA_MASK_CHG, &pm);
-}
-
-/**
- * opa_vnic_vema_add_one -- Handle new ib device
- * @device: ib device pointer
- *
- * Allocate the vnic control port and initialize it.
- */
-static int opa_vnic_vema_add_one(struct ib_device *device)
-{
-	struct opa_vnic_ctrl_port *cport;
-	int rc, size = sizeof(*cport);
-
-	if (!rdma_cap_opa_vnic(device))
-		return -EOPNOTSUPP;
-
-	size += device->phys_port_cnt * sizeof(struct opa_vnic_vema_port);
-	cport = kzalloc(size, GFP_KERNEL);
-	if (!cport)
-		return -ENOMEM;
-
-	cport->num_ports = device->phys_port_cnt;
-	cport->ibdev = device;
-
-	/* Initialize opa vnic management agent (vema) */
-	rc = vema_register(cport);
-	if (!rc)
-		c_info("VNIC client initialized\n");
-
-	ib_set_client_data(device, &opa_vnic_client, cport);
-	opa_vnic_ctrl_config_dev(cport, true);
-	return 0;
-}
-
-/**
- * opa_vnic_vema_rem_one -- Handle ib device removal
- * @device: ib device pointer
- * @client_data: ib client data
- *
- * Uninitialize and free the vnic control port.
- */
-static void opa_vnic_vema_rem_one(struct ib_device *device,
-				  void *client_data)
-{
-	struct opa_vnic_ctrl_port *cport = client_data;
-
-	c_info("removing VNIC client\n");
-	opa_vnic_ctrl_config_dev(cport, false);
-	vema_unregister(cport);
-	kfree(cport);
-}
-
-static int __init opa_vnic_init(void)
-{
-	int rc;
-
-	rc = ib_register_client(&opa_vnic_client);
-	if (rc)
-		pr_err("VNIC driver register failed %d\n", rc);
-
-	return rc;
-}
-module_init(opa_vnic_init);
-
-static void opa_vnic_deinit(void)
-{
-	ib_unregister_client(&opa_vnic_client);
-}
-module_exit(opa_vnic_deinit);
-
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_AUTHOR("Cornelis Networks");
-MODULE_DESCRIPTION("Cornelis OPX Virtual Network driver");
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c
deleted file mode 100644
index 292c037aa239d..0000000000000
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema_iface.c
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Copyright(c) 2017 Intel Corporation.
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  - Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  - Neither the name of Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * This file contains OPA VNIC EMA Interface functions.
- */
-
-#include "opa_vnic_internal.h"
-
-/**
- * opa_vnic_vema_report_event - sent trap to report the specified event
- * @adapter: vnic port adapter
- * @event: event to be reported
- *
- * This function calls vema api to sent a trap for the given event.
- */
-void opa_vnic_vema_report_event(struct opa_vnic_adapter *adapter, u8 event)
-{
-	struct __opa_veswport_info *info = &adapter->info;
-	struct __opa_veswport_trap trap_data;
-
-	trap_data.fabric_id = info->vesw.fabric_id;
-	trap_data.veswid = info->vesw.vesw_id;
-	trap_data.veswportnum = info->vport.port_num;
-	trap_data.opaportnum = adapter->port_num;
-	trap_data.veswportindex = adapter->vport_num;
-	trap_data.opcode = event;
-
-	opa_vnic_vema_send_trap(adapter, &trap_data, info->vport.encap_slid);
-}
-
-/**
- * opa_vnic_get_summary_counters - get summary counters
- * @adapter: vnic port adapter
- * @cntrs: pointer to destination summary counters structure
- *
- * This function populates the summary counters that is maintained by the
- * given adapter to destination address provided.
- */
-void opa_vnic_get_summary_counters(struct opa_vnic_adapter *adapter,
-				   struct opa_veswport_summary_counters *cntrs)
-{
-	struct opa_vnic_stats vstats;
-	__be64 *dst;
-	u64 *src;
-
-	memset(&vstats, 0, sizeof(vstats));
-	spin_lock(&adapter->stats_lock);
-	adapter->rn_ops->ndo_get_stats64(adapter->netdev, &vstats.netstats);
-	spin_unlock(&adapter->stats_lock);
-
-	cntrs->vp_instance = cpu_to_be16(adapter->vport_num);
-	cntrs->vesw_id = cpu_to_be16(adapter->info.vesw.vesw_id);
-	cntrs->veswport_num = cpu_to_be32(adapter->port_num);
-
-	cntrs->tx_errors = cpu_to_be64(vstats.netstats.tx_errors);
-	cntrs->rx_errors = cpu_to_be64(vstats.netstats.rx_errors);
-	cntrs->tx_packets = cpu_to_be64(vstats.netstats.tx_packets);
-	cntrs->rx_packets = cpu_to_be64(vstats.netstats.rx_packets);
-	cntrs->tx_bytes = cpu_to_be64(vstats.netstats.tx_bytes);
-	cntrs->rx_bytes = cpu_to_be64(vstats.netstats.rx_bytes);
-
-	/*
-	 * This loop depends on layout of
-	 * opa_veswport_summary_counters opa_vnic_stats structures.
-	 */
-	for (dst = &cntrs->tx_unicast, src = &vstats.tx_grp.unicast;
-	     dst < &cntrs->reserved[0]; dst++, src++) {
-		*dst = cpu_to_be64(*src);
-	}
-}
-
-/**
- * opa_vnic_get_error_counters - get error counters
- * @adapter: vnic port adapter
- * @cntrs: pointer to destination error counters structure
- *
- * This function populates the error counters that is maintained by the
- * given adapter to destination address provided.
- */
-void opa_vnic_get_error_counters(struct opa_vnic_adapter *adapter,
-				 struct opa_veswport_error_counters *cntrs)
-{
-	struct opa_vnic_stats vstats;
-
-	memset(&vstats, 0, sizeof(vstats));
-	spin_lock(&adapter->stats_lock);
-	adapter->rn_ops->ndo_get_stats64(adapter->netdev, &vstats.netstats);
-	spin_unlock(&adapter->stats_lock);
-
-	cntrs->vp_instance = cpu_to_be16(adapter->vport_num);
-	cntrs->vesw_id = cpu_to_be16(adapter->info.vesw.vesw_id);
-	cntrs->veswport_num = cpu_to_be32(adapter->port_num);
-
-	cntrs->tx_errors = cpu_to_be64(vstats.netstats.tx_errors);
-	cntrs->rx_errors = cpu_to_be64(vstats.netstats.rx_errors);
-	cntrs->tx_dlid_zero = cpu_to_be64(vstats.tx_dlid_zero);
-	cntrs->tx_drop_state = cpu_to_be64(vstats.tx_drop_state);
-	cntrs->tx_logic = cpu_to_be64(vstats.netstats.tx_fifo_errors +
-				      vstats.netstats.tx_carrier_errors);
-
-	cntrs->rx_bad_veswid = cpu_to_be64(vstats.netstats.rx_nohandler);
-	cntrs->rx_runt = cpu_to_be64(vstats.rx_runt);
-	cntrs->rx_oversize = cpu_to_be64(vstats.rx_oversize);
-	cntrs->rx_drop_state = cpu_to_be64(vstats.rx_drop_state);
-	cntrs->rx_logic = cpu_to_be64(vstats.netstats.rx_fifo_errors);
-}
-
-/**
- * opa_vnic_get_vesw_info -- Get the vesw information
- * @adapter: vnic port adapter
- * @info: pointer to destination vesw info structure
- *
- * This function copies the vesw info that is maintained by the
- * given adapter to destination address provided.
- */
-void opa_vnic_get_vesw_info(struct opa_vnic_adapter *adapter,
-			    struct opa_vesw_info *info)
-{
-	struct __opa_vesw_info *src = &adapter->info.vesw;
-	int i;
-
-	info->fabric_id = cpu_to_be16(src->fabric_id);
-	info->vesw_id = cpu_to_be16(src->vesw_id);
-	memcpy(info->rsvd0, src->rsvd0, ARRAY_SIZE(src->rsvd0));
-	info->def_port_mask = cpu_to_be16(src->def_port_mask);
-	memcpy(info->rsvd1, src->rsvd1, ARRAY_SIZE(src->rsvd1));
-	info->pkey = cpu_to_be16(src->pkey);
-
-	memcpy(info->rsvd2, src->rsvd2, ARRAY_SIZE(src->rsvd2));
-	info->u_mcast_dlid = cpu_to_be32(src->u_mcast_dlid);
-	for (i = 0; i < OPA_VESW_MAX_NUM_DEF_PORT; i++)
-		info->u_ucast_dlid[i] = cpu_to_be32(src->u_ucast_dlid[i]);
-
-	info->rc = cpu_to_be32(src->rc);
-
-	memcpy(info->rsvd3, src->rsvd3, ARRAY_SIZE(src->rsvd3));
-	info->eth_mtu = cpu_to_be16(src->eth_mtu);
-	memcpy(info->rsvd4, src->rsvd4, ARRAY_SIZE(src->rsvd4));
-}
-
-/**
- * opa_vnic_set_vesw_info -- Set the vesw information
- * @adapter: vnic port adapter
- * @info: pointer to vesw info structure
- *
- * This function updates the vesw info that is maintained by the
- * given adapter with vesw info provided. Reserved fields are stored
- * and returned back to EM as is.
- */
-void opa_vnic_set_vesw_info(struct opa_vnic_adapter *adapter,
-			    struct opa_vesw_info *info)
-{
-	struct __opa_vesw_info *dst = &adapter->info.vesw;
-	int i;
-
-	dst->fabric_id = be16_to_cpu(info->fabric_id);
-	dst->vesw_id = be16_to_cpu(info->vesw_id);
-	memcpy(dst->rsvd0, info->rsvd0, ARRAY_SIZE(info->rsvd0));
-	dst->def_port_mask = be16_to_cpu(info->def_port_mask);
-	memcpy(dst->rsvd1, info->rsvd1, ARRAY_SIZE(info->rsvd1));
-	dst->pkey = be16_to_cpu(info->pkey);
-
-	memcpy(dst->rsvd2, info->rsvd2, ARRAY_SIZE(info->rsvd2));
-	dst->u_mcast_dlid = be32_to_cpu(info->u_mcast_dlid);
-	for (i = 0; i < OPA_VESW_MAX_NUM_DEF_PORT; i++)
-		dst->u_ucast_dlid[i] = be32_to_cpu(info->u_ucast_dlid[i]);
-
-	dst->rc = be32_to_cpu(info->rc);
-
-	memcpy(dst->rsvd3, info->rsvd3, ARRAY_SIZE(info->rsvd3));
-	dst->eth_mtu = be16_to_cpu(info->eth_mtu);
-	memcpy(dst->rsvd4, info->rsvd4, ARRAY_SIZE(info->rsvd4));
-}
-
-/**
- * opa_vnic_get_per_veswport_info -- Get the vesw per port information
- * @adapter: vnic port adapter
- * @info: pointer to destination vport info structure
- *
- * This function copies the vesw per port info that is maintained by the
- * given adapter to destination address provided.
- * Note that the read only fields are not copied.
- */
-void opa_vnic_get_per_veswport_info(struct opa_vnic_adapter *adapter,
-				    struct opa_per_veswport_info *info)
-{
-	struct __opa_per_veswport_info *src = &adapter->info.vport;
-
-	info->port_num = cpu_to_be32(src->port_num);
-	info->eth_link_status = src->eth_link_status;
-	memcpy(info->rsvd0, src->rsvd0, ARRAY_SIZE(src->rsvd0));
-
-	memcpy(info->base_mac_addr, src->base_mac_addr,
-	       ARRAY_SIZE(info->base_mac_addr));
-	info->config_state = src->config_state;
-	info->oper_state = src->oper_state;
-	info->max_mac_tbl_ent = cpu_to_be16(src->max_mac_tbl_ent);
-	info->max_smac_ent = cpu_to_be16(src->max_smac_ent);
-	info->mac_tbl_digest = cpu_to_be32(src->mac_tbl_digest);
-	memcpy(info->rsvd1, src->rsvd1, ARRAY_SIZE(src->rsvd1));
-
-	info->encap_slid = cpu_to_be32(src->encap_slid);
-	memcpy(info->pcp_to_sc_uc, src->pcp_to_sc_uc,
-	       ARRAY_SIZE(info->pcp_to_sc_uc));
-	memcpy(info->pcp_to_vl_uc, src->pcp_to_vl_uc,
-	       ARRAY_SIZE(info->pcp_to_vl_uc));
-	memcpy(info->pcp_to_sc_mc, src->pcp_to_sc_mc,
-	       ARRAY_SIZE(info->pcp_to_sc_mc));
-	memcpy(info->pcp_to_vl_mc, src->pcp_to_vl_mc,
-	       ARRAY_SIZE(info->pcp_to_vl_mc));
-	info->non_vlan_sc_uc = src->non_vlan_sc_uc;
-	info->non_vlan_vl_uc = src->non_vlan_vl_uc;
-	info->non_vlan_sc_mc = src->non_vlan_sc_mc;
-	info->non_vlan_vl_mc = src->non_vlan_vl_mc;
-	memcpy(info->rsvd2, src->rsvd2, ARRAY_SIZE(src->rsvd2));
-
-	info->uc_macs_gen_count = cpu_to_be16(src->uc_macs_gen_count);
-	info->mc_macs_gen_count = cpu_to_be16(src->mc_macs_gen_count);
-	memcpy(info->rsvd3, src->rsvd3, ARRAY_SIZE(src->rsvd3));
-}
-
-/**
- * opa_vnic_set_per_veswport_info -- Set vesw per port information
- * @adapter: vnic port adapter
- * @info: pointer to vport info structure
- *
- * This function updates the vesw per port info that is maintained by the
- * given adapter with vesw per port info provided. Reserved fields are
- * stored and returned back to EM as is.
- */
-void opa_vnic_set_per_veswport_info(struct opa_vnic_adapter *adapter,
-				    struct opa_per_veswport_info *info)
-{
-	struct __opa_per_veswport_info *dst = &adapter->info.vport;
-
-	dst->port_num = be32_to_cpu(info->port_num);
-	memcpy(dst->rsvd0, info->rsvd0, ARRAY_SIZE(info->rsvd0));
-
-	memcpy(dst->base_mac_addr, info->base_mac_addr,
-	       ARRAY_SIZE(dst->base_mac_addr));
-	dst->config_state = info->config_state;
-	memcpy(dst->rsvd1, info->rsvd1, ARRAY_SIZE(info->rsvd1));
-
-	dst->encap_slid = be32_to_cpu(info->encap_slid);
-	memcpy(dst->pcp_to_sc_uc, info->pcp_to_sc_uc,
-	       ARRAY_SIZE(dst->pcp_to_sc_uc));
-	memcpy(dst->pcp_to_vl_uc, info->pcp_to_vl_uc,
-	       ARRAY_SIZE(dst->pcp_to_vl_uc));
-	memcpy(dst->pcp_to_sc_mc, info->pcp_to_sc_mc,
-	       ARRAY_SIZE(dst->pcp_to_sc_mc));
-	memcpy(dst->pcp_to_vl_mc, info->pcp_to_vl_mc,
-	       ARRAY_SIZE(dst->pcp_to_vl_mc));
-	dst->non_vlan_sc_uc = info->non_vlan_sc_uc;
-	dst->non_vlan_vl_uc = info->non_vlan_vl_uc;
-	dst->non_vlan_sc_mc = info->non_vlan_sc_mc;
-	dst->non_vlan_vl_mc = info->non_vlan_vl_mc;
-	memcpy(dst->rsvd2, info->rsvd2, ARRAY_SIZE(info->rsvd2));
-	memcpy(dst->rsvd3, info->rsvd3, ARRAY_SIZE(info->rsvd3));
-}
-
-/**
- * opa_vnic_query_mcast_macs - query multicast mac list
- * @adapter: vnic port adapter
- * @macs: pointer mac list
- *
- * This function populates the provided mac list with the configured
- * multicast addresses in the adapter.
- */
-void opa_vnic_query_mcast_macs(struct opa_vnic_adapter *adapter,
-			       struct opa_veswport_iface_macs *macs)
-{
-	u16 start_idx, num_macs, idx = 0, count = 0;
-	struct netdev_hw_addr *ha;
-
-	start_idx = be16_to_cpu(macs->start_idx);
-	num_macs = be16_to_cpu(macs->num_macs_in_msg);
-	netdev_for_each_mc_addr(ha, adapter->netdev) {
-		struct opa_vnic_iface_mac_entry *entry = &macs->entry[count];
-
-		if (start_idx > idx++)
-			continue;
-		else if (num_macs == count)
-			break;
-		memcpy(entry, ha->addr, sizeof(*entry));
-		count++;
-	}
-
-	macs->tot_macs_in_lst = cpu_to_be16(netdev_mc_count(adapter->netdev));
-	macs->num_macs_in_msg = cpu_to_be16(count);
-	macs->gen_count = cpu_to_be16(adapter->info.vport.mc_macs_gen_count);
-}
-
-/**
- * opa_vnic_query_ucast_macs - query unicast mac list
- * @adapter: vnic port adapter
- * @macs: pointer mac list
- *
- * This function populates the provided mac list with the configured
- * unicast addresses in the adapter.
- */
-void opa_vnic_query_ucast_macs(struct opa_vnic_adapter *adapter,
-			       struct opa_veswport_iface_macs *macs)
-{
-	u16 start_idx, tot_macs, num_macs, idx = 0, count = 0, em_macs = 0;
-	struct netdev_hw_addr *ha;
-
-	start_idx = be16_to_cpu(macs->start_idx);
-	num_macs = be16_to_cpu(macs->num_macs_in_msg);
-	/* loop through dev_addrs list first */
-	for_each_dev_addr(adapter->netdev, ha) {
-		struct opa_vnic_iface_mac_entry *entry = &macs->entry[count];
-
-		/* Do not include EM specified MAC address */
-		if (!memcmp(adapter->info.vport.base_mac_addr, ha->addr,
-			    ARRAY_SIZE(adapter->info.vport.base_mac_addr))) {
-			em_macs++;
-			continue;
-		}
-
-		if (start_idx > idx++)
-			continue;
-		else if (num_macs == count)
-			break;
-		memcpy(entry, ha->addr, sizeof(*entry));
-		count++;
-	}
-
-	/* loop through uc list */
-	netdev_for_each_uc_addr(ha, adapter->netdev) {
-		struct opa_vnic_iface_mac_entry *entry = &macs->entry[count];
-
-		if (start_idx > idx++)
-			continue;
-		else if (num_macs == count)
-			break;
-		memcpy(entry, ha->addr, sizeof(*entry));
-		count++;
-	}
-
-	tot_macs = netdev_hw_addr_list_count(&adapter->netdev->dev_addrs) +
-		   netdev_uc_count(adapter->netdev) - em_macs;
-	macs->tot_macs_in_lst = cpu_to_be16(tot_macs);
-	macs->num_macs_in_msg = cpu_to_be16(count);
-	macs->gen_count = cpu_to_be16(adapter->info.vport.uc_macs_gen_count);
-}
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index 3362362f9e2e5..e351552733df2 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -3219,7 +3219,7 @@ static int __init rtrs_client_init(void)
 		pr_err("Failed to create rtrs-client dev class\n");
 		return ret;
 	}
-	rtrs_wq = alloc_workqueue("rtrs_client_wq", 0, 0);
+	rtrs_wq = alloc_workqueue("rtrs_client_wq", WQ_PERCPU, 0);
 	if (!rtrs_wq) {
 		class_unregister(&rtrs_clt_dev_class);
 		return -ENOMEM;
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index 0140bfaed7216..6482ad859bd10 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -2385,7 +2385,7 @@ static int __init rtrs_server_init(void)
 	if (err)
 		goto out_err;
 
-	rtrs_wq = alloc_workqueue("rtrs_server_wq", 0, 0);
+	rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_PERCPU, 0);
 	if (!rtrs_wq) {
 		err = -ENOMEM;
 		goto out_dev_class;
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index e00b87acf481a..9aec5d80117f1 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -3925,6 +3925,7 @@ static const struct target_core_fabric_ops srpt_template = {
 	.tfc_wwn_attrs			= srpt_wwn_attrs,
 	.tfc_tpg_attrib_attrs		= srpt_tpg_attrib_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/input/input.c b/drivers/input/input.c
index 7b7490a1fbc6d..39d9d2b1e3ca6 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -800,14 +800,30 @@ static int input_default_getkeycode(struct input_dev *dev,
 	return 0;
 }
 
-static int input_default_setkeycode(struct input_dev *dev,
-				    const struct input_keymap_entry *ke,
-				    unsigned int *old_keycode)
+/**
+ * input_default_setkeycode - default setkeycode method
+ * @dev: input device which keymap is being updated.
+ * @ke: new keymap entry.
+ * @old_keycode: pointer to the location where old keycode should be stored.
+ *
+ * This function is the default implementation of &input_dev.setkeycode()
+ * method. It is typically used when a driver does not provide its own
+ * implementation, but it is also exported so drivers can extend it.
+ *
+ * The function must be called with &input_dev.event_lock held.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+int input_default_setkeycode(struct input_dev *dev,
+			     const struct input_keymap_entry *ke,
+			     unsigned int *old_keycode)
 {
 	unsigned int index;
 	int error;
 	int i;
 
+	lockdep_assert_held(&dev->event_lock);
+
 	if (!dev->keycodesize)
 		return -EINVAL;
 
@@ -861,6 +877,7 @@ static int input_default_setkeycode(struct input_dev *dev,
 	__set_bit(ke->keycode, dev->keybit);
 	return 0;
 }
+EXPORT_SYMBOL(input_default_setkeycode);
 
 /**
  * input_get_keycode - retrieve keycode currently mapped to a given scancode
diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c
index 840cdf5878dcd..c8ad55f26ea83 100644
--- a/drivers/input/keyboard/atkbd.c
+++ b/drivers/input/keyboard/atkbd.c
@@ -3,10 +3,7 @@
  * AT and PS/2 keyboard driver
  *
  * Copyright (c) 1999-2002 Vojtech Pavlik
- */
-
-
-/*
+ *
  * This driver can handle standard AT keyboards and PS/2 keyboards in
  * Translated and Raw Set 2 and Set 3, as well as AT keyboards on dumb
  * input-only controllers and AT keyboards connected over a one way RS232
@@ -65,8 +62,8 @@ static bool atkbd_terminal;
 module_param_named(terminal, atkbd_terminal, bool, 0);
 MODULE_PARM_DESC(terminal, "Enable break codes on an IBM Terminal keyboard connected via AT/PS2");
 
-#define SCANCODE(keymap)	((keymap >> 16) & 0xFFFF)
-#define KEYCODE(keymap)		(keymap & 0xFFFF)
+#define SCANCODE(keymap)	(((keymap) >> 16) & 0xFFFF)
+#define KEYCODE(keymap)		((keymap) & 0xFFFF)
 
 /*
  * Scancode to keycode tables. These are just the default setting, and
@@ -76,7 +73,6 @@ MODULE_PARM_DESC(terminal, "Enable break codes on an IBM Terminal keyboard conne
 #define ATKBD_KEYMAP_SIZE	512
 
 static const unsigned short atkbd_set2_keycode[ATKBD_KEYMAP_SIZE] = {
-
 #ifdef CONFIG_KEYBOARD_ATKBD_HP_KEYCODES
 
 /* XXX: need a more general approach */
@@ -107,7 +103,6 @@ static const unsigned short atkbd_set2_keycode[ATKBD_KEYMAP_SIZE] = {
 };
 
 static const unsigned short atkbd_set3_keycode[ATKBD_KEYMAP_SIZE] = {
-
 	  0,  0,  0,  0,  0,  0,  0, 59,  1,138,128,129,130, 15, 41, 60,
 	131, 29, 42, 86, 58, 16,  2, 61,133, 56, 44, 31, 30, 17,  3, 62,
 	134, 46, 45, 32, 18,  5,  4, 63,135, 57, 47, 33, 20, 19,  6, 64,
@@ -122,15 +117,15 @@ static const unsigned short atkbd_set3_keycode[ATKBD_KEYMAP_SIZE] = {
 	148,149,147,140
 };
 
-static const unsigned short atkbd_unxlate_table[128] = {
-          0,118, 22, 30, 38, 37, 46, 54, 61, 62, 70, 69, 78, 85,102, 13,
-         21, 29, 36, 45, 44, 53, 60, 67, 68, 77, 84, 91, 90, 20, 28, 27,
-         35, 43, 52, 51, 59, 66, 75, 76, 82, 14, 18, 93, 26, 34, 33, 42,
-         50, 49, 58, 65, 73, 74, 89,124, 17, 41, 88,  5,  6,  4, 12,  3,
-         11,  2, 10,  1,  9,119,126,108,117,125,123,107,115,116,121,105,
-        114,122,112,113,127, 96, 97,120,  7, 15, 23, 31, 39, 47, 55, 63,
-         71, 79, 86, 94,  8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 87,111,
-         19, 25, 57, 81, 83, 92, 95, 98, 99,100,101,103,104,106,109,110
+static const u8 atkbd_unxlate_table[128] = {
+	  0,118, 22, 30, 38, 37, 46, 54, 61, 62, 70, 69, 78, 85,102, 13,
+	 21, 29, 36, 45, 44, 53, 60, 67, 68, 77, 84, 91, 90, 20, 28, 27,
+	 35, 43, 52, 51, 59, 66, 75, 76, 82, 14, 18, 93, 26, 34, 33, 42,
+	 50, 49, 58, 65, 73, 74, 89,124, 17, 41, 88,  5,  6,  4, 12,  3,
+	 11,  2, 10,  1,  9,119,126,108,117,125,123,107,115,116,121,105,
+	114,122,112,113,127, 96, 97,120,  7, 15, 23, 31, 39, 47, 55, 63,
+	 71, 79, 86, 94,  8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 87,111,
+	 19, 25, 57, 81, 83, 92, 95, 98, 99,100,101,103,104,106,109,110
 };
 
 #define ATKBD_CMD_SETLEDS	0x10ed
@@ -184,7 +179,7 @@ static const unsigned short atkbd_unxlate_table[128] = {
 
 static const struct {
 	unsigned short keycode;
-	unsigned char set2;
+	u8 set2;
 } atkbd_scroll_keys[] = {
 	{ ATKBD_SCR_1,     0xc5 },
 	{ ATKBD_SCR_2,     0x9d },
@@ -200,7 +195,6 @@ static const struct {
  */
 
 struct atkbd {
-
 	struct ps2dev ps2dev;
 	struct input_dev *dev;
 
@@ -211,7 +205,7 @@ struct atkbd {
 	unsigned short id;
 	unsigned short keycode[ATKBD_KEYMAP_SIZE];
 	DECLARE_BITMAP(force_release_mask, ATKBD_KEYMAP_SIZE);
-	unsigned char set;
+	u8 set;
 	bool translated;
 	bool extra;
 	bool write;
@@ -221,7 +215,7 @@ struct atkbd {
 	bool enabled;
 
 	/* Accessed only from interrupt */
-	unsigned char emul;
+	u8 emul;
 	bool resend;
 	bool release;
 	unsigned long xl_bit;
@@ -253,9 +247,9 @@ static unsigned int (*atkbd_platform_scancode_fixup)(struct atkbd *, unsigned in
 static bool atkbd_skip_deactivate;
 
 static ssize_t atkbd_attr_show_helper(struct device *dev, char *buf,
-				ssize_t (*handler)(struct atkbd *, char *));
+				      ssize_t (*handler)(struct atkbd *, char *));
 static ssize_t atkbd_attr_set_helper(struct device *dev, const char *buf, size_t count,
-				ssize_t (*handler)(struct atkbd *, const char *, size_t));
+				     ssize_t (*handler)(struct atkbd *, const char *, size_t));
 #define ATKBD_DEFINE_ATTR(_name)						\
 static ssize_t atkbd_show_##_name(struct atkbd *, char *);			\
 static ssize_t atkbd_set_##_name(struct atkbd *, const char *, size_t);		\
@@ -270,7 +264,7 @@ static ssize_t atkbd_do_set_##_name(struct device *d,				\
 	return atkbd_attr_set_helper(d, b, s, atkbd_set_##_name);		\
 }										\
 static struct device_attribute atkbd_attr_##_name =				\
-	__ATTR(_name, S_IWUSR | S_IRUGO, atkbd_do_show_##_name, atkbd_do_set_##_name);
+	__ATTR(_name, S_IWUSR | S_IRUGO, atkbd_do_show_##_name, atkbd_do_set_##_name)
 
 ATKBD_DEFINE_ATTR(extra);
 ATKBD_DEFINE_ATTR(force_release);
@@ -287,7 +281,7 @@ static ssize_t atkbd_do_show_##_name(struct device *d,				\
 	return atkbd_attr_show_helper(d, b, atkbd_show_##_name);		\
 }										\
 static struct device_attribute atkbd_attr_##_name =				\
-	__ATTR(_name, S_IRUGO, atkbd_do_show_##_name, NULL);
+	__ATTR(_name, S_IRUGO, atkbd_do_show_##_name, NULL)
 
 ATKBD_DEFINE_RO_ATTR(err_count);
 ATKBD_DEFINE_RO_ATTR(function_row_physmap);
@@ -317,7 +311,7 @@ static struct atkbd *atkbd_from_serio(struct serio *serio)
 }
 
 static umode_t atkbd_attr_is_visible(struct kobject *kobj,
-				struct attribute *attr, int i)
+				     struct attribute *attr, int i)
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct serio *serio = to_serio_port(dev);
@@ -337,7 +331,7 @@ static const struct attribute_group atkbd_attribute_group = {
 
 __ATTRIBUTE_GROUPS(atkbd_attribute);
 
-static const unsigned int xl_table[] = {
+static const u8 xl_table[] = {
 	ATKBD_RET_BAT, ATKBD_RET_ERR, ATKBD_RET_ACK,
 	ATKBD_RET_NAK, ATKBD_RET_HANJA, ATKBD_RET_HANGEUL,
 };
@@ -346,7 +340,7 @@ static const unsigned int xl_table[] = {
  * Checks if we should mangle the scancode to extract 'release' bit
  * in translated mode.
  */
-static bool atkbd_need_xlate(unsigned long xl_bit, unsigned char code)
+static bool atkbd_need_xlate(unsigned long xl_bit, u8 code)
 {
 	int i;
 
@@ -365,7 +359,7 @@ static bool atkbd_need_xlate(unsigned long xl_bit, unsigned char code)
  * between make/break pair of scancodes for select keys and PS/2
  * protocol responses.
  */
-static void atkbd_calculate_xl_bit(struct atkbd *atkbd, unsigned char code)
+static void atkbd_calculate_xl_bit(struct atkbd *atkbd, u8 code)
 {
 	int i;
 
@@ -389,7 +383,7 @@ static unsigned int atkbd_compat_scancode(struct atkbd *atkbd, unsigned int code
 	if (atkbd->set == 3) {
 		if (atkbd->emul == 1)
 			code |= 0x100;
-        } else {
+	} else {
 		code = (code & 0x7f) | ((code & 0x80) << 1);
 		if (atkbd->emul == 1)
 			code |= 0x80;
@@ -431,7 +425,7 @@ static enum ps2_disposition atkbd_pre_receive_byte(struct ps2dev *ps2dev,
 
 	dev_dbg(&serio->dev, "Received %02x flags %02x\n", data, flags);
 
-#if !defined(__i386__) && !defined (__x86_64__)
+#if !defined(__i386__) && !defined(__x86_64__)
 	if (atkbd_handle_frame_error(ps2dev, data, flags))
 		return PS2_IGNORE;
 #endif
@@ -460,7 +454,6 @@ static void atkbd_receive_byte(struct ps2dev *ps2dev, u8 data)
 		code = atkbd_platform_scancode_fixup(atkbd, code);
 
 	if (atkbd->translated) {
-
 		if (atkbd->emul || atkbd_need_xlate(atkbd->xl_bit, code)) {
 			atkbd->release = code >> 7;
 			code &= 0x7f;
@@ -486,11 +479,9 @@ static void atkbd_receive_byte(struct ps2dev *ps2dev, u8 data)
 		return;
 	case ATKBD_RET_ACK:
 	case ATKBD_RET_NAK:
-		if (printk_ratelimit())
-			dev_warn(&serio->dev,
-				 "Spurious %s on %s. "
-				 "Some program might be trying to access hardware directly.\n",
-				 data == ATKBD_RET_ACK ? "ACK" : "NAK", serio->phys);
+		dev_warn_ratelimited(&serio->dev,
+				     "Spurious %s on %s. Some program might be trying to access hardware directly.\n",
+				     data == ATKBD_RET_ACK ? "ACK" : "NAK", serio->phys);
 		return;
 	case ATKBD_RET_ERR:
 		atkbd->err_count++;
@@ -582,14 +573,14 @@ static void atkbd_receive_byte(struct ps2dev *ps2dev, u8 data)
 
 static int atkbd_set_repeat_rate(struct atkbd *atkbd)
 {
-	const short period[32] =
-		{ 33,  37,  42,  46,  50,  54,  58,  63,  67,  75,  83,  92, 100, 109, 116, 125,
-		 133, 149, 167, 182, 200, 217, 232, 250, 270, 303, 333, 370, 400, 435, 470, 500 };
-	const short delay[4] =
-		{ 250, 500, 750, 1000 };
+	const short period[32] = {
+		  33,  37,  42,  46,  50,  54,  58,  63,  67,  75,  83,  92, 100, 109, 116, 125,
+		 133, 149, 167, 182, 200, 217, 232, 250, 270, 303, 333, 370, 400, 435, 470, 500
+	};
+	const short delay[4] = { 250, 500, 750, 1000 };
 
 	struct input_dev *dev = atkbd->dev;
-	unsigned char param;
+	u8 param;
 	int i = 0, j = 0;
 
 	while (i < ARRAY_SIZE(period) - 1 && period[i] < dev->rep[REP_PERIOD])
@@ -607,7 +598,7 @@ static int atkbd_set_repeat_rate(struct atkbd *atkbd)
 static int atkbd_set_leds(struct atkbd *atkbd)
 {
 	struct input_dev *dev = atkbd->dev;
-	unsigned char param[2];
+	u8 param[2];
 
 	param[0] = (test_bit(LED_SCROLLL, dev->led) ? 1 : 0)
 		 | (test_bit(LED_NUML,    dev->led) ? 2 : 0)
@@ -648,8 +639,7 @@ static void atkbd_event_work(struct work_struct *work)
 		 * it may not be ready yet. In this case we need to keep
 		 * rescheduling till reconnect completes.
 		 */
-		schedule_delayed_work(&atkbd->event_work,
-					msecs_to_jiffies(100));
+		schedule_delayed_work(&atkbd->event_work, msecs_to_jiffies(100));
 	} else {
 		if (test_and_clear_bit(ATKBD_LED_EVENT_BIT, &atkbd->event_mask))
 			atkbd_set_leds(atkbd);
@@ -683,7 +673,7 @@ static void atkbd_schedule_event_work(struct atkbd *atkbd, int event_bit)
  */
 
 static int atkbd_event(struct input_dev *dev,
-			unsigned int type, unsigned int code, int value)
+		       unsigned int type, unsigned int code, int value)
 {
 	struct atkbd *atkbd = input_get_drvdata(dev);
 
@@ -691,7 +681,6 @@ static int atkbd_event(struct input_dev *dev,
 		return -1;
 
 	switch (type) {
-
 	case EV_LED:
 		atkbd_schedule_event_work(atkbd, ATKBD_LED_EVENT_BIT);
 		return 0;
@@ -808,7 +797,7 @@ static inline bool atkbd_skip_getid(struct atkbd *atkbd) { return false; }
 static int atkbd_probe(struct atkbd *atkbd)
 {
 	struct ps2dev *ps2dev = &atkbd->ps2dev;
-	unsigned char param[2];
+	u8 param[2];
 
 /*
  * Some systems, where the bit-twiddling when testing the io-lines of the
@@ -836,7 +825,6 @@ static int atkbd_probe(struct atkbd *atkbd)
 
 	param[0] = param[1] = 0xa5;	/* initialize with invalid values */
 	if (ps2_command(ps2dev, param, ATKBD_CMD_GETID)) {
-
 /*
  * If the get ID command failed, we check if we can at least set
  * the LEDs on the keyboard. This should work on every keyboard out there.
@@ -856,8 +844,7 @@ static int atkbd_probe(struct atkbd *atkbd)
 
 	if (atkbd->id == 0xaca1 && atkbd->translated) {
 		dev_err(&ps2dev->serio->dev,
-			"NCD terminal keyboards are only supported on non-translating controllers. "
-			"Use i8042.direct=1 to disable translation.\n");
+			"NCD terminal keyboards are only supported on non-translating controllers. Use i8042.direct=1 to disable translation.\n");
 		return -1;
 	}
 
@@ -881,7 +868,7 @@ static int atkbd_probe(struct atkbd *atkbd)
 static int atkbd_select_set(struct atkbd *atkbd, int target_set, int allow_extra)
 {
 	struct ps2dev *ps2dev = &atkbd->ps2dev;
-	unsigned char param[2];
+	u8 param[2];
 
 	atkbd->extra = false;
 /*
@@ -941,8 +928,8 @@ static int atkbd_select_set(struct atkbd *atkbd, int target_set, int allow_extra
 
 static int atkbd_reset_state(struct atkbd *atkbd)
 {
-        struct ps2dev *ps2dev = &atkbd->ps2dev;
-	unsigned char param[1];
+	struct ps2dev *ps2dev = &atkbd->ps2dev;
+	u8 param[1];
 
 /*
  * Set the LEDs to a predefined state (all off).
@@ -967,7 +954,6 @@ static int atkbd_reset_state(struct atkbd *atkbd)
  * atkbd_cleanup() restores the keyboard state so that BIOS is happy after a
  * reboot.
  */
-
 static void atkbd_cleanup(struct serio *serio)
 {
 	struct atkbd *atkbd = atkbd_from_serio(serio);
@@ -976,11 +962,9 @@ static void atkbd_cleanup(struct serio *serio)
 	ps2_command(&atkbd->ps2dev, NULL, ATKBD_CMD_RESET_DEF);
 }
 
-
 /*
  * atkbd_disconnect() closes and frees.
  */
-
 static void atkbd_disconnect(struct serio *serio)
 {
 	struct atkbd *atkbd = atkbd_from_serio(serio);
@@ -1005,8 +989,7 @@ static void atkbd_disconnect(struct serio *serio)
 /*
  * generate release events for the keycodes given in data
  */
-static void atkbd_apply_forced_release_keylist(struct atkbd* atkbd,
-						const void *data)
+static void atkbd_apply_forced_release_keylist(struct atkbd *atkbd, const void *data)
 {
 	const unsigned int *keys = data;
 	unsigned int i;
@@ -1088,7 +1071,6 @@ static int atkbd_get_keymap_from_fwnode(struct atkbd *atkbd)
 {
 	struct device *dev = &atkbd->ps2dev.serio->dev;
 	int i, n;
-	u32 *ptr;
 	u16 scancode, keycode;
 
 	/* Parse "linux,keymap" property */
@@ -1096,13 +1078,12 @@ static int atkbd_get_keymap_from_fwnode(struct atkbd *atkbd)
 	if (n <= 0 || n > ATKBD_KEYMAP_SIZE)
 		return -ENXIO;
 
-	ptr = kcalloc(n, sizeof(u32), GFP_KERNEL);
+	u32 *ptr __free(kfree) = kcalloc(n, sizeof(*ptr), GFP_KERNEL);
 	if (!ptr)
 		return -ENOMEM;
 
 	if (device_property_read_u32_array(dev, "linux,keymap", ptr, n)) {
 		dev_err(dev, "problem parsing FW keymap property\n");
-		kfree(ptr);
 		return -EINVAL;
 	}
 
@@ -1110,10 +1091,14 @@ static int atkbd_get_keymap_from_fwnode(struct atkbd *atkbd)
 	for (i = 0; i < n; i++) {
 		scancode = SCANCODE(ptr[i]);
 		keycode = KEYCODE(ptr[i]);
+		if (scancode >= ATKBD_KEYMAP_SIZE) {
+			dev_warn(dev, "invalid scancode %#x in FW keymap entry %d\n",
+				 scancode, i);
+			return -EINVAL;
+		}
 		atkbd->keycode[scancode] = keycode;
 	}
 
-	kfree(ptr);
 	return 0;
 }
 
@@ -1235,7 +1220,7 @@ static void atkbd_set_device_attrs(struct atkbd *atkbd)
 	}
 
 	input_dev->keycode = atkbd->keycode;
-	input_dev->keycodesize = sizeof(unsigned short);
+	input_dev->keycodesize = sizeof(atkbd->keycode[0]);
 	input_dev->keycodemax = ARRAY_SIZE(atkbd_set2_keycode);
 
 	for (i = 0; i < ATKBD_KEYMAP_SIZE; i++) {
@@ -1289,7 +1274,6 @@ static int atkbd_connect(struct serio *serio, struct serio_driver *drv)
 	mutex_init(&atkbd->mutex);
 
 	switch (serio->id.type) {
-
 	case SERIO_8042_XL:
 		atkbd->translated = true;
 		fallthrough;
@@ -1314,7 +1298,6 @@ static int atkbd_connect(struct serio *serio, struct serio_driver *drv)
 		goto fail2;
 
 	if (atkbd->write) {
-
 		if (atkbd_probe(atkbd)) {
 			err = -ENODEV;
 			goto fail3;
@@ -1354,7 +1337,6 @@ static int atkbd_connect(struct serio *serio, struct serio_driver *drv)
  * atkbd_reconnect() tries to restore keyboard into a sane state and is
  * most likely called on resume.
  */
-
 static int atkbd_reconnect(struct serio *serio)
 {
 	struct atkbd *atkbd = atkbd_from_serio(serio);
@@ -1389,7 +1371,6 @@ static int atkbd_reconnect(struct serio *serio)
 		atkbd_set_leds(atkbd);
 		if (!atkbd->softrepeat)
 			atkbd_set_repeat_rate(atkbd);
-
 	}
 
 	/*
@@ -1445,7 +1426,7 @@ static struct serio_driver atkbd_drv = {
 };
 
 static ssize_t atkbd_attr_show_helper(struct device *dev, char *buf,
-				ssize_t (*handler)(struct atkbd *, char *))
+				      ssize_t (*handler)(struct atkbd *, char *))
 {
 	struct serio *serio = to_serio_port(dev);
 	struct atkbd *atkbd = atkbd_from_serio(serio);
@@ -1454,7 +1435,7 @@ static ssize_t atkbd_attr_show_helper(struct device *dev, char *buf,
 }
 
 static ssize_t atkbd_attr_set_helper(struct device *dev, const char *buf, size_t count,
-				ssize_t (*handler)(struct atkbd *, const char *, size_t))
+				     ssize_t (*handler)(struct atkbd *, const char *, size_t))
 {
 	struct serio *serio = to_serio_port(dev);
 	struct atkbd *atkbd = atkbd_from_serio(serio);
@@ -1482,7 +1463,7 @@ static ssize_t atkbd_set_extra(struct atkbd *atkbd, const char *buf, size_t coun
 	unsigned int value;
 	int err;
 	bool old_extra;
-	unsigned char old_set;
+	u8 old_set;
 
 	if (!atkbd->write)
 		return -EIO;
@@ -1527,8 +1508,8 @@ static ssize_t atkbd_set_extra(struct atkbd *atkbd, const char *buf, size_t coun
 			return err;
 		}
 		input_unregister_device(old_dev);
-
 	}
+
 	return count;
 }
 
@@ -1544,7 +1525,7 @@ static ssize_t atkbd_show_force_release(struct atkbd *atkbd, char *buf)
 }
 
 static ssize_t atkbd_set_force_release(struct atkbd *atkbd,
-					const char *buf, size_t count)
+				       const char *buf, size_t count)
 {
 	/* 64 bytes on stack should be acceptable */
 	DECLARE_BITMAP(new_mask, ATKBD_KEYMAP_SIZE);
@@ -1558,7 +1539,6 @@ static ssize_t atkbd_set_force_release(struct atkbd *atkbd,
 	return count;
 }
 
-
 static ssize_t atkbd_show_scroll(struct atkbd *atkbd, char *buf)
 {
 	return sprintf(buf, "%d\n", atkbd->scroll ? 1 : 0);
@@ -1617,7 +1597,7 @@ static ssize_t atkbd_set_set(struct atkbd *atkbd, const char *buf, size_t count)
 	struct input_dev *old_dev, *new_dev;
 	unsigned int value;
 	int err;
-	unsigned char old_set;
+	u8 old_set;
 	bool old_extra;
 
 	if (!atkbd->write)
@@ -1715,7 +1695,6 @@ static ssize_t atkbd_set_softrepeat(struct atkbd *atkbd, const char *buf, size_t
 	return count;
 }
 
-
 static ssize_t atkbd_show_softraw(struct atkbd *atkbd, char *buf)
 {
 	return sprintf(buf, "%d\n", atkbd->softraw ? 1 : 0);
diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c
index 2822c592880b0..177e5d4a3382c 100644
--- a/drivers/input/keyboard/cros_ec_keyb.c
+++ b/drivers/input/keyboard/cros_ec_keyb.c
@@ -29,6 +29,12 @@
 
 #include <linux/unaligned.h>
 
+/*
+ * Maximum size of the normal key matrix, this is limited by the host command
+ * key_matrix field defined in ec_response_get_next_data_v3
+ */
+#define CROS_EC_KEYBOARD_COLS_MAX 18
+
 /**
  * struct cros_ec_keyb - Structure representing EC keyboard device
  *
@@ -44,14 +50,17 @@
  * @bs_idev: The input device for non-matrix buttons and switches (or NULL).
  * @notifier: interrupt event notifier for transport devices
  * @vdata: vivaldi function row data
+ * @has_fn_map: whether the driver uses an fn function-map layer
+ * @fn_active: tracks whether the function key is currently pressed
+ * @fn_combo_active: tracks whether another key was pressed while fn is active
  */
 struct cros_ec_keyb {
 	unsigned int rows;
 	unsigned int cols;
 	int row_shift;
 	bool ghost_filter;
-	uint8_t *valid_keys;
-	uint8_t *old_kb_state;
+	u8 valid_keys[CROS_EC_KEYBOARD_COLS_MAX];
+	u8 old_kb_state[CROS_EC_KEYBOARD_COLS_MAX];
 
 	struct device *dev;
 	struct cros_ec_device *ec;
@@ -61,6 +70,10 @@ struct cros_ec_keyb {
 	struct notifier_block notifier;
 
 	struct vivaldi_data vdata;
+
+	bool has_fn_map;
+	bool fn_active;
+	bool fn_combo_active;
 };
 
 /**
@@ -132,11 +145,11 @@ static const struct cros_ec_bs_map cros_ec_keyb_bs[] = {
  * Returns true when there is at least one combination of pressed keys that
  * results in ghosting.
  */
-static bool cros_ec_keyb_has_ghosting(struct cros_ec_keyb *ckdev, uint8_t *buf)
+static bool cros_ec_keyb_has_ghosting(struct cros_ec_keyb *ckdev, u8 *buf)
 {
 	int col1, col2, buf1, buf2;
 	struct device *dev = ckdev->dev;
-	uint8_t *valid_keys = ckdev->valid_keys;
+	u8 *valid_keys = ckdev->valid_keys;
 
 	/*
 	 * Ghosting happens if for any pressed key X there are other keys
@@ -166,20 +179,108 @@ static bool cros_ec_keyb_has_ghosting(struct cros_ec_keyb *ckdev, uint8_t *buf)
 	return false;
 }
 
+static void cros_ec_emit_fn_key(struct input_dev *input, unsigned int pos)
+{
+	input_event(input, EV_MSC, MSC_SCAN, pos);
+	input_report_key(input, KEY_FN, true);
+	input_sync(input);
+
+	input_event(input, EV_MSC, MSC_SCAN, pos);
+	input_report_key(input, KEY_FN, false);
+}
+
+static void cros_ec_keyb_process_key_plain(struct cros_ec_keyb *ckdev,
+					   int row, int col, bool state)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = idev->keycode;
+	int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+
+	input_event(idev, EV_MSC, MSC_SCAN, pos);
+	input_report_key(idev, keycodes[pos], state);
+}
+
+static void cros_ec_keyb_process_key_fn_map(struct cros_ec_keyb *ckdev,
+					    int row, int col, bool state)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = idev->keycode;
+	unsigned int pos, fn_pos;
+	unsigned int code, fn_code;
+
+	pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+	code = keycodes[pos];
+
+	if (code == KEY_FN) {
+		ckdev->fn_active = state;
+		if (state) {
+			ckdev->fn_combo_active = false;
+		} else if (!ckdev->fn_combo_active) {
+			/*
+			 * Send both Fn press and release events if nothing
+			 * else has been pressed together with Fn.
+			 */
+			cros_ec_emit_fn_key(idev, pos);
+		}
+		return;
+	}
+
+	fn_pos = MATRIX_SCAN_CODE(row + ckdev->rows, col, ckdev->row_shift);
+	fn_code = keycodes[fn_pos];
+
+	if (state) {
+		if (ckdev->fn_active) {
+			ckdev->fn_combo_active = true;
+			if (!fn_code)
+				return; /* Discard if no Fn mapping exists */
+
+			pos = fn_pos;
+			code = fn_code;
+		}
+	} else {
+		/*
+		 * If the Fn-remapped code is currently pressed, release it.
+		 * Otherwise, release the standard code (if it was pressed).
+		 */
+		if (fn_code && test_bit(fn_code, idev->key)) {
+			pos = fn_pos;
+			code = fn_code;
+		} else if (!test_bit(code, idev->key)) {
+			return; /* Discard, key press code was not sent */
+		}
+	}
+
+	input_event(idev, EV_MSC, MSC_SCAN, pos);
+	input_report_key(idev, code, state);
+}
+
+static void cros_ec_keyb_process_col(struct cros_ec_keyb *ckdev, int col,
+				     u8 col_state, u8 changed)
+{
+	for (int row = 0; row < ckdev->rows; row++) {
+		if (changed & BIT(row)) {
+			u8 key_state = col_state & BIT(row);
+
+			dev_dbg(ckdev->dev, "changed: [r%d c%d]: byte %02x\n",
+				row, col, key_state);
+
+			if (ckdev->has_fn_map)
+				cros_ec_keyb_process_key_fn_map(ckdev, row, col,
+								key_state);
+			else
+				cros_ec_keyb_process_key_plain(ckdev, row, col,
+							       key_state);
+		}
+	}
+}
 
 /*
  * Compares the new keyboard state to the old one and produces key
- * press/release events accordingly.  The keyboard state is 13 bytes (one byte
- * per column)
+ * press/release events accordingly.  The keyboard state is one byte
+ * per column.
  */
-static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev,
-			 uint8_t *kb_state, int len)
+static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev, u8 *kb_state, int len)
 {
-	struct input_dev *idev = ckdev->idev;
-	int col, row;
-	int new_state;
-	int old_state;
-
 	if (ckdev->ghost_filter && cros_ec_keyb_has_ghosting(ckdev, kb_state)) {
 		/*
 		 * Simple-minded solution: ignore this state. The obvious
@@ -190,25 +291,15 @@ static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev,
 		return;
 	}
 
-	for (col = 0; col < ckdev->cols; col++) {
-		for (row = 0; row < ckdev->rows; row++) {
-			int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
-			const unsigned short *keycodes = idev->keycode;
-
-			new_state = kb_state[col] & (1 << row);
-			old_state = ckdev->old_kb_state[col] & (1 << row);
-			if (new_state != old_state) {
-				dev_dbg(ckdev->dev,
-					"changed: [r%d c%d]: byte %02x\n",
-					row, col, new_state);
-
-				input_event(idev, EV_MSC, MSC_SCAN, pos);
-				input_report_key(idev, keycodes[pos],
-						 new_state);
-			}
-		}
-		ckdev->old_kb_state[col] = kb_state[col];
+	for (int col = 0; col < ckdev->cols; col++) {
+		u8 changed = kb_state[col] ^ ckdev->old_kb_state[col];
+
+		if (changed)
+			cros_ec_keyb_process_col(ckdev, col, kb_state[col],
+						 changed);
 	}
+
+	memcpy(ckdev->old_kb_state, kb_state, sizeof(ckdev->old_kb_state));
 	input_sync(ckdev->idev);
 }
 
@@ -246,8 +337,10 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 {
 	struct cros_ec_keyb *ckdev = container_of(nb, struct cros_ec_keyb,
 						  notifier);
-	u32 val;
+	struct ec_response_get_next_event_v3 *event_data;
+	unsigned int event_size;
 	unsigned int ev_type;
+	u32 val;
 
 	/*
 	 * If not wake enabled, discard key state changes during
@@ -257,32 +350,32 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 	if (queued_during_suspend && !device_may_wakeup(ckdev->dev))
 		return NOTIFY_OK;
 
-	switch (ckdev->ec->event_data.event_type) {
+	event_data = &ckdev->ec->event_data;
+	event_size = ckdev->ec->event_size;
+
+	switch (event_data->event_type) {
 	case EC_MKBP_EVENT_KEY_MATRIX:
 		pm_wakeup_event(ckdev->dev, 0);
 
 		if (!ckdev->idev) {
-			dev_warn_once(ckdev->dev,
-				      "Unexpected key matrix event\n");
+			dev_warn_once(ckdev->dev, "Unexpected key matrix event\n");
 			return NOTIFY_OK;
 		}
 
-		if (ckdev->ec->event_size != ckdev->cols) {
+		if (event_size != ckdev->cols) {
 			dev_err(ckdev->dev,
 				"Discarded key matrix event, unexpected length: %d != %d\n",
 				ckdev->ec->event_size, ckdev->cols);
 			return NOTIFY_OK;
 		}
 
-		cros_ec_keyb_process(ckdev,
-				     ckdev->ec->event_data.data.key_matrix,
-				     ckdev->ec->event_size);
+		cros_ec_keyb_process(ckdev, event_data->data.key_matrix, event_size);
 		break;
 
 	case EC_MKBP_EVENT_SYSRQ:
 		pm_wakeup_event(ckdev->dev, 0);
 
-		val = get_unaligned_le32(&ckdev->ec->event_data.data.sysrq);
+		val = get_unaligned_le32(&event_data->data.sysrq);
 		dev_dbg(ckdev->dev, "sysrq code from EC: %#x\n", val);
 		handle_sysrq(val);
 		break;
@@ -291,13 +384,11 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 	case EC_MKBP_EVENT_SWITCH:
 		pm_wakeup_event(ckdev->dev, 0);
 
-		if (ckdev->ec->event_data.event_type == EC_MKBP_EVENT_BUTTON) {
-			val = get_unaligned_le32(
-					&ckdev->ec->event_data.data.buttons);
+		if (event_data->event_type == EC_MKBP_EVENT_BUTTON) {
+			val = get_unaligned_le32(&event_data->data.buttons);
 			ev_type = EV_KEY;
 		} else {
-			val = get_unaligned_le32(
-					&ckdev->ec->event_data.data.switches);
+			val = get_unaligned_le32(&event_data->data.switches);
 			ev_type = EV_SW;
 		}
 		cros_ec_keyb_report_bs(ckdev, ev_type, val);
@@ -326,8 +417,8 @@ static void cros_ec_keyb_compute_valid_keys(struct cros_ec_keyb *ckdev)
 	for (col = 0; col < ckdev->cols; col++) {
 		for (row = 0; row < ckdev->rows; row++) {
 			code = keymap[MATRIX_SCAN_CODE(row, col, row_shift)];
-			if (code && (code != KEY_BATTERY))
-				ckdev->valid_keys[col] |= 1 << row;
+			if (code != KEY_RESERVED && code != KEY_BATTERY)
+				ckdev->valid_keys[col] |= BIT(row);
 		}
 		dev_dbg(ckdev->dev, "valid_keys[%02d] = 0x%02x\n",
 			col, ckdev->valid_keys[col]);
@@ -583,6 +674,62 @@ static void cros_ec_keyb_parse_vivaldi_physmap(struct cros_ec_keyb *ckdev)
 	ckdev->vdata.num_function_row_keys = n_physmap;
 }
 
+/* Returns true if there is a KEY_FN code defined in the normal keymap */
+static bool cros_ec_keyb_has_fn_key(struct cros_ec_keyb *ckdev)
+{
+	const unsigned short *keycodes = ckdev->idev->keycode;
+	int i;
+
+	for (i = 0; i < MATRIX_SCAN_CODE(ckdev->rows, 0, ckdev->row_shift); i++) {
+		if (keycodes[i] == KEY_FN)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Returns true if there is a KEY_FN defined and at least one key in the fn
+ * layer keymap
+ */
+static bool cros_ec_keyb_has_fn_map(struct cros_ec_keyb *ckdev)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = ckdev->idev->keycode;
+	int i;
+
+	if (!cros_ec_keyb_has_fn_key(ckdev))
+		return false;
+
+	for (i = MATRIX_SCAN_CODE(ckdev->rows, 0, ckdev->row_shift);
+	     i < idev->keycodemax; i++) {
+		if (keycodes[i] != KEY_RESERVED)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Custom handler for the set keycode ioctl, calls the default handler and
+ * recomputes has_fn_map.
+ */
+static int cros_ec_keyb_setkeycode(struct input_dev *idev,
+				   const struct input_keymap_entry *ke,
+				   unsigned int *old_keycode)
+{
+	struct cros_ec_keyb *ckdev = input_get_drvdata(idev);
+	int ret;
+
+	ret = input_default_setkeycode(idev, ke, old_keycode);
+	if (ret)
+		return ret;
+
+	ckdev->has_fn_map = cros_ec_keyb_has_fn_map(ckdev);
+
+	return 0;
+}
+
 /**
  * cros_ec_keyb_register_matrix - Register matrix keys
  *
@@ -604,13 +751,11 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	if (err)
 		return err;
 
-	ckdev->valid_keys = devm_kzalloc(dev, ckdev->cols, GFP_KERNEL);
-	if (!ckdev->valid_keys)
-		return -ENOMEM;
-
-	ckdev->old_kb_state = devm_kzalloc(dev, ckdev->cols, GFP_KERNEL);
-	if (!ckdev->old_kb_state)
-		return -ENOMEM;
+	if (ckdev->cols > CROS_EC_KEYBOARD_COLS_MAX) {
+		dev_err(dev, "keypad,num-columns too large: %d (max: %d)\n",
+			ckdev->cols, CROS_EC_KEYBOARD_COLS_MAX);
+		return -EINVAL;
+	}
 
 	/*
 	 * We call the keyboard matrix 'input0'. Allocate phys before input
@@ -632,11 +777,11 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	idev->id.version = 1;
 	idev->id.product = 0;
 	idev->dev.parent = dev;
+	idev->setkeycode = cros_ec_keyb_setkeycode;
 
-	ckdev->ghost_filter = device_property_read_bool(dev,
-					"google,needs-ghost-filter");
+	ckdev->ghost_filter = device_property_read_bool(dev, "google,needs-ghost-filter");
 
-	err = matrix_keypad_build_keymap(NULL, NULL, ckdev->rows, ckdev->cols,
+	err = matrix_keypad_build_keymap(NULL, NULL, ckdev->rows * 2, ckdev->cols,
 					 NULL, idev);
 	if (err) {
 		dev_err(dev, "cannot build key matrix\n");
@@ -651,6 +796,8 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	cros_ec_keyb_compute_valid_keys(ckdev);
 	cros_ec_keyb_parse_vivaldi_physmap(ckdev);
 
+	ckdev->has_fn_map = cros_ec_keyb_has_fn_map(ckdev);
+
 	err = input_register_device(ckdev->idev);
 	if (err) {
 		dev_err(dev, "cannot register input device\n");
diff --git a/drivers/input/keyboard/mpr121_touchkey.c b/drivers/input/keyboard/mpr121_touchkey.c
index bd1a944ded46f..47edc161ec772 100644
--- a/drivers/input/keyboard/mpr121_touchkey.c
+++ b/drivers/input/keyboard/mpr121_touchkey.c
@@ -295,8 +295,6 @@ static int mpr_touchkey_probe(struct i2c_client *client)
 		return error;
 
 	i2c_set_clientdata(client, mpr121);
-	device_init_wakeup(dev,
-			device_property_read_bool(dev, "wakeup-source"));
 
 	return 0;
 }
@@ -305,9 +303,6 @@ static int mpr_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
 
-	if (device_may_wakeup(&client->dev))
-		enable_irq_wake(client->irq);
-
 	i2c_smbus_write_byte_data(client, ELECTRODE_CONF_ADDR, 0x00);
 
 	return 0;
@@ -318,9 +313,6 @@ static int mpr_resume(struct device *dev)
 	struct i2c_client *client = to_i2c_client(dev);
 	struct mpr121_touchkey *mpr121 = i2c_get_clientdata(client);
 
-	if (device_may_wakeup(&client->dev))
-		disable_irq_wake(client->irq);
-
 	i2c_smbus_write_byte_data(client, ELECTRODE_CONF_ADDR,
 				  mpr121->keycount);
 
diff --git a/drivers/input/misc/adxl34x-i2c.c b/drivers/input/misc/adxl34x-i2c.c
index c05d898898e84..5ea0ce42a5074 100644
--- a/drivers/input/misc/adxl34x-i2c.c
+++ b/drivers/input/misc/adxl34x-i2c.c
@@ -77,11 +77,8 @@ static const struct adxl34x_bus_ops adxl34x_i2c_bops = {
 static int adxl34x_i2c_probe(struct i2c_client *client)
 {
 	struct adxl34x *ac;
-	int error;
 
-	error = i2c_check_functionality(client->adapter,
-			I2C_FUNC_SMBUS_BYTE_DATA);
-	if (!error) {
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)) {
 		dev_err(&client->dev, "SMBUS Byte Data not Supported\n");
 		return -EIO;
 	}
diff --git a/drivers/input/misc/atlas_btns.c b/drivers/input/misc/atlas_btns.c
index 5b9be2957746d..47b31725e8507 100644
--- a/drivers/input/misc/atlas_btns.c
+++ b/drivers/input/misc/atlas_btns.c
@@ -14,6 +14,7 @@
 #include <linux/input.h>
 #include <linux/types.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 #include <linux/uaccess.h>
 
 #define ACPI_ATLAS_NAME		"Atlas ACPI"
@@ -57,8 +58,9 @@ static acpi_status acpi_atlas_button_handler(u32 function,
 	return status;
 }
 
-static int atlas_acpi_button_add(struct acpi_device *device)
+static int atlas_acpi_button_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	acpi_status status;
 	int i;
 	int err;
@@ -106,8 +108,9 @@ static int atlas_acpi_button_add(struct acpi_device *device)
 	return err;
 }
 
-static void atlas_acpi_button_remove(struct acpi_device *device)
+static void atlas_acpi_button_remove(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	acpi_status status;
 
 	status = acpi_remove_address_space_handler(device->handle,
@@ -124,16 +127,15 @@ static const struct acpi_device_id atlas_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, atlas_device_ids);
 
-static struct acpi_driver atlas_acpi_driver = {
-	.name	= ACPI_ATLAS_NAME,
-	.class	= ACPI_ATLAS_CLASS,
-	.ids	= atlas_device_ids,
-	.ops	= {
-		.add	= atlas_acpi_button_add,
-		.remove	= atlas_acpi_button_remove,
+static struct platform_driver atlas_acpi_driver = {
+	.probe = atlas_acpi_button_probe,
+	.remove = atlas_acpi_button_remove,
+	.driver = {
+		.name = ACPI_ATLAS_NAME,
+		.acpi_match_table = atlas_device_ids,
 	},
 };
-module_acpi_driver(atlas_acpi_driver);
+module_platform_driver(atlas_acpi_driver);
 
 MODULE_AUTHOR("Jaya Kumar");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/drv260x.c b/drivers/input/misc/drv260x.c
index 96cd6a078c8ab..e2089d6ac8500 100644
--- a/drivers/input/misc/drv260x.c
+++ b/drivers/input/misc/drv260x.c
@@ -7,14 +7,16 @@
  * Copyright:   (C) 2014 Texas Instruments, Inc.
  */
 
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/device/devres.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
 #include <linux/regulator/consumer.h>
+#include <linux/slab.h>
 
 #include <dt-bindings/input/ti-drv260x.h>
 
@@ -165,6 +167,12 @@
 #define DRV260X_AUTOCAL_TIME_500MS		(2 << 4)
 #define DRV260X_AUTOCAL_TIME_1000MS		(3 << 4)
 
+/*
+ * Timeout for waiting for the GO status bit, in seconds. Should be reasonably
+ * large to wait for a auto-calibration cycle completion.
+ */
+#define DRV260X_GO_TIMEOUT_S 5
+
 /**
  * struct drv260x_data -
  * @input_dev: Pointer to the input device
@@ -308,6 +316,7 @@ static int drv260x_init(struct drv260x_data *haptics)
 {
 	int error;
 	unsigned int cal_buf;
+	unsigned long timeout;
 
 	error = regmap_write(haptics->regmap,
 			     DRV260X_RATED_VOLT, haptics->rated_voltage);
@@ -397,6 +406,7 @@ static int drv260x_init(struct drv260x_data *haptics)
 		return error;
 	}
 
+	timeout = jiffies + DRV260X_GO_TIMEOUT_S * HZ;
 	do {
 		usleep_range(15000, 15500);
 		error = regmap_read(haptics->regmap, DRV260X_GO, &cal_buf);
@@ -406,6 +416,11 @@ static int drv260x_init(struct drv260x_data *haptics)
 				error);
 			return error;
 		}
+		if (time_after(jiffies, timeout)) {
+			dev_err(&haptics->client->dev,
+				"Calibration timeout. The device cannot be used.\n");
+			return -ETIMEDOUT;
+		}
 	} while (cal_buf == DRV260X_GO_BIT);
 
 	return 0;
@@ -419,6 +434,13 @@ static const struct regmap_config drv260x_regmap_config = {
 	.cache_type = REGCACHE_NONE,
 };
 
+static void drv260x_power_off(void *data)
+{
+	struct drv260x_data *haptics = data;
+
+	regulator_disable(haptics->regulator);
+}
+
 static int drv260x_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
@@ -484,6 +506,16 @@ static int drv260x_probe(struct i2c_client *client)
 		return error;
 	}
 
+	error = regulator_enable(haptics->regulator);
+	if (error) {
+		dev_err(dev, "Failed to enable regulator: %d\n", error);
+		return error;
+	}
+
+	error = devm_add_action_or_reset(dev, drv260x_power_off, haptics);
+	if (error)
+		return error;
+
 	haptics->enable_gpio = devm_gpiod_get_optional(dev, "enable",
 						       GPIOD_OUT_HIGH);
 	if (IS_ERR(haptics->enable_gpio))
@@ -598,11 +630,22 @@ static int drv260x_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(drv260x_pm_ops, drv260x_suspend, drv260x_resume);
 
 static const struct i2c_device_id drv260x_id[] = {
+	{ "drv2604" },
+	{ "drv2604l" },
+	{ "drv2605" },
 	{ "drv2605l" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, drv260x_id);
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id drv260x_acpi_match[] = {
+	{ "DRV2604" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, drv260x_acpi_match);
+#endif
+
 static const struct of_device_id drv260x_of_match[] = {
 	{ .compatible = "ti,drv2604", },
 	{ .compatible = "ti,drv2604l", },
@@ -616,6 +659,7 @@ static struct i2c_driver drv260x_driver = {
 	.probe		= drv260x_probe,
 	.driver		= {
 		.name	= "drv260x-haptics",
+		.acpi_match_table = ACPI_PTR(drv260x_acpi_match),
 		.of_match_table = drv260x_of_match,
 		.pm	= pm_sleep_ptr(&drv260x_pm_ops),
 	},
diff --git a/drivers/input/mouse/Kconfig b/drivers/input/mouse/Kconfig
index 833b643f06164..30d119d4634bd 100644
--- a/drivers/input/mouse/Kconfig
+++ b/drivers/input/mouse/Kconfig
@@ -164,16 +164,6 @@ config MOUSE_PS2_TOUCHKIT
 
 	  If unsure, say N.
 
-config MOUSE_PS2_OLPC
-	bool "OLPC PS/2 mouse protocol extension"
-	depends on MOUSE_PS2 && OLPC
-	help
-	  Say Y here if you have an OLPC XO-1 laptop (with built-in
-	  PS/2 touchpad/tablet device).  The manufacturer calls the
-	  touchpad an HGPK.
-
-	  If unsure, say N.
-
 config MOUSE_PS2_FOCALTECH
 	bool "FocalTech PS/2 mouse protocol extension" if EXPERT
 	default y
diff --git a/drivers/input/mouse/Makefile b/drivers/input/mouse/Makefile
index a1336d5bee6f3..137ed0c32d90a 100644
--- a/drivers/input/mouse/Makefile
+++ b/drivers/input/mouse/Makefile
@@ -29,7 +29,6 @@ psmouse-objs := psmouse-base.o synaptics.o focaltech.o
 psmouse-$(CONFIG_MOUSE_PS2_ALPS)	+= alps.o
 psmouse-$(CONFIG_MOUSE_PS2_BYD)		+= byd.o
 psmouse-$(CONFIG_MOUSE_PS2_ELANTECH)	+= elantech.o
-psmouse-$(CONFIG_MOUSE_PS2_OLPC)	+= hgpk.o
 psmouse-$(CONFIG_MOUSE_PS2_LOGIPS2PP)	+= logips2pp.o
 psmouse-$(CONFIG_MOUSE_PS2_LIFEBOOK)	+= lifebook.o
 psmouse-$(CONFIG_MOUSE_PS2_SENTELIC)	+= sentelic.o
diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c
index f3d3b6b4e02d7..0ee1a30b9aaf3 100644
--- a/drivers/input/mouse/alps.c
+++ b/drivers/input/mouse/alps.c
@@ -12,6 +12,7 @@
  * tpconfig utility (by C. Scott Ananian and Bruce Kall).
  */
 
+#include "linux/workqueue.h"
 #include <linux/slab.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
@@ -1452,7 +1453,7 @@ static int alps_do_register_bare_ps2_mouse(struct alps_data *priv)
 static void alps_register_bare_ps2_mouse(struct work_struct *work)
 {
 	struct alps_data *priv = container_of(work, struct alps_data,
-					      dev3_register_work.work);
+					      dev3_register_work);
 	int error;
 
 	guard(mutex)(&alps_mutex);
@@ -1485,8 +1486,7 @@ static void alps_report_bare_ps2_packet(struct psmouse *psmouse,
 	} else if (unlikely(IS_ERR_OR_NULL(priv->dev3))) {
 		/* Register dev3 mouse if we received PS/2 packet first time */
 		if (!IS_ERR(priv->dev3))
-			psmouse_queue_work(psmouse, &priv->dev3_register_work,
-					   0);
+			schedule_work(&priv->dev3_register_work);
 		return;
 	} else {
 		dev = priv->dev3;
@@ -2975,7 +2975,7 @@ static void alps_disconnect(struct psmouse *psmouse)
 
 	psmouse_reset(psmouse);
 	timer_shutdown_sync(&priv->timer);
-	disable_delayed_work_sync(&priv->dev3_register_work);
+	disable_work_sync(&priv->dev3_register_work);
 	if (priv->dev2)
 		input_unregister_device(priv->dev2);
 	if (!IS_ERR_OR_NULL(priv->dev3))
@@ -3147,8 +3147,7 @@ int alps_init(struct psmouse *psmouse)
 
 	priv->psmouse = psmouse;
 
-	INIT_DELAYED_WORK(&priv->dev3_register_work,
-			  alps_register_bare_ps2_mouse);
+	INIT_WORK(&priv->dev3_register_work, alps_register_bare_ps2_mouse);
 
 	psmouse->protocol_handler = alps_process_byte;
 	psmouse->poll = alps_poll;
diff --git a/drivers/input/mouse/alps.h b/drivers/input/mouse/alps.h
index 0a1048cf23f6a..17bbf6cdba55f 100644
--- a/drivers/input/mouse/alps.h
+++ b/drivers/input/mouse/alps.h
@@ -257,7 +257,7 @@ struct alps_fields {
  * @dev3: Generic PS/2 mouse (can be NULL, delayed registering).
  * @phys2: Physical path for the trackstick device.
  * @phys3: Physical path for the generic PS/2 mouse.
- * @dev3_register_work: Delayed work for registering PS/2 mouse.
+ * @dev3_register_work: A work instance for registering PS/2 mouse.
  * @nibble_commands: Command mapping used for touchpad register accesses.
  * @addr_command: Command used to tell the touchpad that a register address
  *   follows.
@@ -289,7 +289,7 @@ struct alps_data {
 	struct input_dev *dev3;
 	char phys2[32];
 	char phys3[32];
-	struct delayed_work dev3_register_work;
+	struct work_struct dev3_register_work;
 
 	/* these are autodetected when the device is identified */
 	const struct alps_nibble_commands *nibble_commands;
diff --git a/drivers/input/mouse/bcm5974.c b/drivers/input/mouse/bcm5974.c
index 8e3cea9db3659..e772bac3b64c0 100644
--- a/drivers/input/mouse/bcm5974.c
+++ b/drivers/input/mouse/bcm5974.c
@@ -286,6 +286,8 @@ struct bcm5974 {
 	const struct tp_finger *index[MAX_FINGERS];	/* finger index data */
 	struct input_mt_pos pos[MAX_FINGERS];		/* position array */
 	int slots[MAX_FINGERS];				/* slot assignments */
+	struct work_struct mode_reset_work;
+	unsigned long last_mode_reset;
 };
 
 /* trackpad finger block data, le16-aligned */
@@ -696,6 +698,32 @@ static int bcm5974_wellspring_mode(struct bcm5974 *dev, bool on)
 	return retval;
 }
 
+/*
+ * Mode switches sent before the control response are ignored.
+ * Fixing this state requires switching to normal mode and waiting
+ * about 1ms before switching back to wellspring mode.
+ */
+static void bcm5974_mode_reset_work(struct work_struct *work)
+{
+	struct bcm5974 *dev = container_of(work, struct bcm5974, mode_reset_work);
+	int error;
+
+	guard(mutex)(&dev->pm_mutex);
+	dev->last_mode_reset = jiffies;
+
+	error = bcm5974_wellspring_mode(dev, false);
+	if (error) {
+		dev_err(&dev->intf->dev, "reset to normal mode failed\n");
+		return;
+	}
+
+	fsleep(1000);
+
+	error = bcm5974_wellspring_mode(dev, true);
+	if (error)
+		dev_err(&dev->intf->dev, "mode switch after reset failed\n");
+}
+
 static void bcm5974_irq_button(struct urb *urb)
 {
 	struct bcm5974 *dev = urb->context;
@@ -752,10 +780,20 @@ static void bcm5974_irq_trackpad(struct urb *urb)
 	if (dev->tp_urb->actual_length == 2)
 		goto exit;
 
-	if (report_tp_state(dev, dev->tp_urb->actual_length))
+	if (report_tp_state(dev, dev->tp_urb->actual_length)) {
 		dprintk(1, "bcm5974: bad trackpad package, length: %d\n",
 			dev->tp_urb->actual_length);
 
+		/*
+		 * Receiving a HID packet means we aren't in wellspring mode.
+		 * If we haven't tried a reset in the last second, try now.
+		 */
+		if (dev->tp_urb->actual_length == 8 &&
+		    time_after(jiffies, dev->last_mode_reset + msecs_to_jiffies(1000))) {
+			schedule_work(&dev->mode_reset_work);
+		}
+	}
+
 exit:
 	error = usb_submit_urb(dev->tp_urb, GFP_ATOMIC);
 	if (error)
@@ -906,6 +944,7 @@ static int bcm5974_probe(struct usb_interface *iface,
 	dev->intf = iface;
 	dev->input = input_dev;
 	dev->cfg = *cfg;
+	INIT_WORK(&dev->mode_reset_work, bcm5974_mode_reset_work);
 	mutex_init(&dev->pm_mutex);
 
 	/* setup urbs */
@@ -998,6 +1037,7 @@ static void bcm5974_disconnect(struct usb_interface *iface)
 {
 	struct bcm5974 *dev = usb_get_intfdata(iface);
 
+	disable_work_sync(&dev->mode_reset_work);
 	usb_set_intfdata(iface, NULL);
 
 	input_unregister_device(dev->input);
diff --git a/drivers/input/mouse/hgpk.c b/drivers/input/mouse/hgpk.c
deleted file mode 100644
index 3c4d16ed88a91..0000000000000
--- a/drivers/input/mouse/hgpk.c
+++ /dev/null
@@ -1,1063 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * OLPC HGPK (XO-1) touchpad PS/2 mouse driver
- *
- * Copyright (c) 2006-2008 One Laptop Per Child
- * Authors:
- *   Zephaniah E. Hull
- *   Andres Salomon <dilinger@debian.org>
- *
- * This driver is partly based on the ALPS driver, which is:
- *
- * Copyright (c) 2003 Neil Brown <neilb@cse.unsw.edu.au>
- * Copyright (c) 2003-2005 Peter Osterlund <petero2@telia.com>
- * Copyright (c) 2004 Dmitry Torokhov <dtor@mail.ru>
- * Copyright (c) 2005 Vojtech Pavlik <vojtech@suse.cz>
- */
-
-/*
- * The spec from ALPS is available from
- * <http://wiki.laptop.org/go/Touch_Pad/Tablet>.  It refers to this
- * device as HGPK (Hybrid GS, PT, and Keymatrix).
- *
- * The earliest versions of the device had simultaneous reporting; that
- * was removed.  After that, the device used the Advanced Mode GS/PT streaming
- * stuff.  That turned out to be too buggy to support, so we've finally
- * switched to Mouse Mode (which utilizes only the center 1/3 of the touchpad).
- */
-
-#define DEBUG
-#include <linux/slab.h>
-#include <linux/input.h>
-#include <linux/module.h>
-#include <linux/serio.h>
-#include <linux/libps2.h>
-#include <linux/delay.h>
-#include <asm/olpc.h>
-
-#include "psmouse.h"
-#include "hgpk.h"
-
-#define ILLEGAL_XY 999999
-
-static bool tpdebug;
-module_param(tpdebug, bool, 0644);
-MODULE_PARM_DESC(tpdebug, "enable debugging, dumping packets to KERN_DEBUG.");
-
-static int recalib_delta = 100;
-module_param(recalib_delta, int, 0644);
-MODULE_PARM_DESC(recalib_delta,
-	"packets containing a delta this large will be discarded, and a "
-	"recalibration may be scheduled.");
-
-static int jumpy_delay = 20;
-module_param(jumpy_delay, int, 0644);
-MODULE_PARM_DESC(jumpy_delay,
-	"delay (ms) before recal after jumpiness detected");
-
-static int spew_delay = 1;
-module_param(spew_delay, int, 0644);
-MODULE_PARM_DESC(spew_delay,
-	"delay (ms) before recal after packet spew detected");
-
-static int recal_guard_time;
-module_param(recal_guard_time, int, 0644);
-MODULE_PARM_DESC(recal_guard_time,
-	"interval (ms) during which recal will be restarted if packet received");
-
-static int post_interrupt_delay = 40;
-module_param(post_interrupt_delay, int, 0644);
-MODULE_PARM_DESC(post_interrupt_delay,
-	"delay (ms) before recal after recal interrupt detected");
-
-static bool autorecal = true;
-module_param(autorecal, bool, 0644);
-MODULE_PARM_DESC(autorecal, "enable recalibration in the driver");
-
-static char hgpk_mode_name[16];
-module_param_string(hgpk_mode, hgpk_mode_name, sizeof(hgpk_mode_name), 0644);
-MODULE_PARM_DESC(hgpk_mode,
-	"default hgpk mode: mouse, glidesensor or pentablet");
-
-static int hgpk_default_mode = HGPK_MODE_MOUSE;
-
-static const char * const hgpk_mode_names[] = {
-	[HGPK_MODE_MOUSE] = "Mouse",
-	[HGPK_MODE_GLIDESENSOR] = "GlideSensor",
-	[HGPK_MODE_PENTABLET] = "PenTablet",
-};
-
-static int hgpk_mode_from_name(const char *buf, int len)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(hgpk_mode_names); i++) {
-		const char *name = hgpk_mode_names[i];
-		if (strlen(name) == len && !strncasecmp(name, buf, len))
-			return i;
-	}
-
-	return HGPK_MODE_INVALID;
-}
-
-/*
- * see if new value is within 20% of half of old value
- */
-static int approx_half(int curr, int prev)
-{
-	int belowhalf, abovehalf;
-
-	if (curr < 5 || prev < 5)
-		return 0;
-
-	belowhalf = (prev * 8) / 20;
-	abovehalf = (prev * 12) / 20;
-
-	return belowhalf < curr && curr <= abovehalf;
-}
-
-/*
- * Throw out oddly large delta packets, and any that immediately follow whose
- * values are each approximately half of the previous.  It seems that the ALPS
- * firmware emits errant packets, and they get averaged out slowly.
- */
-static int hgpk_discard_decay_hack(struct psmouse *psmouse, int x, int y)
-{
-	struct hgpk_data *priv = psmouse->private;
-	int avx, avy;
-	bool do_recal = false;
-
-	avx = abs(x);
-	avy = abs(y);
-
-	/* discard if too big, or half that but > 4 times the prev delta */
-	if (avx > recalib_delta ||
-		(avx > recalib_delta / 2 && ((avx / 4) > priv->xlast))) {
-		psmouse_warn(psmouse, "detected %dpx jump in x\n", x);
-		priv->xbigj = avx;
-	} else if (approx_half(avx, priv->xbigj)) {
-		psmouse_warn(psmouse, "detected secondary %dpx jump in x\n", x);
-		priv->xbigj = avx;
-		priv->xsaw_secondary++;
-	} else {
-		if (priv->xbigj && priv->xsaw_secondary > 1)
-			do_recal = true;
-		priv->xbigj = 0;
-		priv->xsaw_secondary = 0;
-	}
-
-	if (avy > recalib_delta ||
-		(avy > recalib_delta / 2 && ((avy / 4) > priv->ylast))) {
-		psmouse_warn(psmouse, "detected %dpx jump in y\n", y);
-		priv->ybigj = avy;
-	} else if (approx_half(avy, priv->ybigj)) {
-		psmouse_warn(psmouse, "detected secondary %dpx jump in y\n", y);
-		priv->ybigj = avy;
-		priv->ysaw_secondary++;
-	} else {
-		if (priv->ybigj && priv->ysaw_secondary > 1)
-			do_recal = true;
-		priv->ybigj = 0;
-		priv->ysaw_secondary = 0;
-	}
-
-	priv->xlast = avx;
-	priv->ylast = avy;
-
-	if (do_recal && jumpy_delay) {
-		psmouse_warn(psmouse, "scheduling recalibration\n");
-		psmouse_queue_work(psmouse, &priv->recalib_wq,
-				msecs_to_jiffies(jumpy_delay));
-	}
-
-	return priv->xbigj || priv->ybigj;
-}
-
-static void hgpk_reset_spew_detection(struct hgpk_data *priv)
-{
-	priv->spew_count = 0;
-	priv->dupe_count = 0;
-	priv->x_tally = 0;
-	priv->y_tally = 0;
-	priv->spew_flag = NO_SPEW;
-}
-
-static void hgpk_reset_hack_state(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	priv->abs_x = priv->abs_y = -1;
-	priv->xlast = priv->ylast = ILLEGAL_XY;
-	priv->xbigj = priv->ybigj = 0;
-	priv->xsaw_secondary = priv->ysaw_secondary = 0;
-	hgpk_reset_spew_detection(priv);
-}
-
-/*
- * We have no idea why this particular hardware bug occurs.  The touchpad
- * will randomly start spewing packets without anything touching the
- * pad.  This wouldn't necessarily be bad, but it's indicative of a
- * severely miscalibrated pad; attempting to use the touchpad while it's
- * spewing means the cursor will jump all over the place, and act "drunk".
- *
- * The packets that are spewed tend to all have deltas between -2 and 2, and
- * the cursor will move around without really going very far.  It will
- * tend to end up in the same location; if we tally up the changes over
- * 100 packets, we end up w/ a final delta of close to 0.  This happens
- * pretty regularly when the touchpad is spewing, and is pretty hard to
- * manually trigger (at least for *my* fingers).  So, it makes a perfect
- * scheme for detecting spews.
- */
-static void hgpk_spewing_hack(struct psmouse *psmouse,
-			      int l, int r, int x, int y)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	/* ignore button press packets; many in a row could trigger
-	 * a false-positive! */
-	if (l || r)
-		return;
-
-	/* don't track spew if the workaround feature has been turned off */
-	if (!spew_delay)
-		return;
-
-	if (abs(x) > 3 || abs(y) > 3) {
-		/* no spew, or spew ended */
-		hgpk_reset_spew_detection(priv);
-		return;
-	}
-
-	/* Keep a tally of the overall delta to the cursor position caused by
-	 * the spew */
-	priv->x_tally += x;
-	priv->y_tally += y;
-
-	switch (priv->spew_flag) {
-	case NO_SPEW:
-		/* we're not spewing, but this packet might be the start */
-		priv->spew_flag = MAYBE_SPEWING;
-
-		fallthrough;
-
-	case MAYBE_SPEWING:
-		priv->spew_count++;
-
-		if (priv->spew_count < SPEW_WATCH_COUNT)
-			break;
-
-		/* excessive spew detected, request recalibration */
-		priv->spew_flag = SPEW_DETECTED;
-
-		fallthrough;
-
-	case SPEW_DETECTED:
-		/* only recalibrate when the overall delta to the cursor
-		 * is really small. if the spew is causing significant cursor
-		 * movement, it is probably a case of the user moving the
-		 * cursor very slowly across the screen. */
-		if (abs(priv->x_tally) < 3 && abs(priv->y_tally) < 3) {
-			psmouse_warn(psmouse, "packet spew detected (%d,%d)\n",
-				     priv->x_tally, priv->y_tally);
-			priv->spew_flag = RECALIBRATING;
-			psmouse_queue_work(psmouse, &priv->recalib_wq,
-					   msecs_to_jiffies(spew_delay));
-		}
-
-		break;
-	case RECALIBRATING:
-		/* we already detected a spew and requested a recalibration,
-		 * just wait for the queue to kick into action. */
-		break;
-	}
-}
-
-/*
- * HGPK Mouse Mode format (standard mouse format, sans middle button)
- *
- * byte 0:	y-over	x-over	y-neg	x-neg	1	0	swr	swl
- * byte 1:	x7	x6	x5	x4	x3	x2	x1	x0
- * byte 2:	y7	y6	y5	y4	y3	y2	y1	y0
- *
- * swr/swl are the left/right buttons.
- * x-neg/y-neg are the x and y delta negative bits
- * x-over/y-over are the x and y overflow bits
- *
- * ---
- *
- * HGPK Advanced Mode - single-mode format
- *
- * byte 0(PT):  1    1    0    0    1    1     1     1
- * byte 0(GS):  1    1    1    1    1    1     1     1
- * byte 1:      0   x6   x5   x4   x3   x2    x1    x0
- * byte 2(PT):  0    0   x9   x8   x7    ? pt-dsw    0
- * byte 2(GS):  0  x10   x9   x8   x7    ? gs-dsw pt-dsw
- * byte 3:      0   y9   y8   y7    1    0   swr   swl
- * byte 4:      0   y6   y5   y4   y3   y2    y1    y0
- * byte 5:      0   z6   z5   z4   z3   z2    z1    z0
- *
- * ?'s are not defined in the protocol spec, may vary between models.
- *
- * swr/swl are the left/right buttons.
- *
- * pt-dsw/gs-dsw indicate that the pt/gs sensor is detecting a
- * pen/finger
- */
-static bool hgpk_is_byte_valid(struct psmouse *psmouse, unsigned char *packet)
-{
-	struct hgpk_data *priv = psmouse->private;
-	int pktcnt = psmouse->pktcnt;
-	bool valid;
-
-	switch (priv->mode) {
-	case HGPK_MODE_MOUSE:
-		valid = (packet[0] & 0x0C) == 0x08;
-		break;
-
-	case HGPK_MODE_GLIDESENSOR:
-		valid = pktcnt == 1 ?
-			packet[0] == HGPK_GS : !(packet[pktcnt - 1] & 0x80);
-		break;
-
-	case HGPK_MODE_PENTABLET:
-		valid = pktcnt == 1 ?
-			packet[0] == HGPK_PT : !(packet[pktcnt - 1] & 0x80);
-		break;
-
-	default:
-		valid = false;
-		break;
-	}
-
-	if (!valid)
-		psmouse_dbg(psmouse,
-			    "bad data, mode %d (%d) %*ph\n",
-			    priv->mode, pktcnt, 6, psmouse->packet);
-
-	return valid;
-}
-
-static void hgpk_process_advanced_packet(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-	struct input_dev *idev = psmouse->dev;
-	unsigned char *packet = psmouse->packet;
-	int down = !!(packet[2] & 2);
-	int left = !!(packet[3] & 1);
-	int right = !!(packet[3] & 2);
-	int x = packet[1] | ((packet[2] & 0x78) << 4);
-	int y = packet[4] | ((packet[3] & 0x70) << 3);
-
-	if (priv->mode == HGPK_MODE_GLIDESENSOR) {
-		int pt_down = !!(packet[2] & 1);
-		int finger_down = !!(packet[2] & 2);
-		int z = packet[5];
-
-		input_report_abs(idev, ABS_PRESSURE, z);
-		if (tpdebug)
-			psmouse_dbg(psmouse, "pd=%d fd=%d z=%d",
-				    pt_down, finger_down, z);
-	} else {
-		/*
-		 * PenTablet mode does not report pressure, so we don't
-		 * report it here
-		 */
-		if (tpdebug)
-			psmouse_dbg(psmouse, "pd=%d ", down);
-	}
-
-	if (tpdebug)
-		psmouse_dbg(psmouse, "l=%d r=%d x=%d y=%d\n",
-			    left, right, x, y);
-
-	input_report_key(idev, BTN_TOUCH, down);
-	input_report_key(idev, BTN_LEFT, left);
-	input_report_key(idev, BTN_RIGHT, right);
-
-	/*
-	 * If this packet says that the finger was removed, reset our position
-	 * tracking so that we don't erroneously detect a jump on next press.
-	 */
-	if (!down) {
-		hgpk_reset_hack_state(psmouse);
-		goto done;
-	}
-
-	/*
-	 * Weed out duplicate packets (we get quite a few, and they mess up
-	 * our jump detection)
-	 */
-	if (x == priv->abs_x && y == priv->abs_y) {
-		if (++priv->dupe_count > SPEW_WATCH_COUNT) {
-			if (tpdebug)
-				psmouse_dbg(psmouse, "hard spew detected\n");
-			priv->spew_flag = RECALIBRATING;
-			psmouse_queue_work(psmouse, &priv->recalib_wq,
-					   msecs_to_jiffies(spew_delay));
-		}
-		goto done;
-	}
-
-	/* not a duplicate, continue with position reporting */
-	priv->dupe_count = 0;
-
-	/* Don't apply hacks in PT mode, it seems reliable */
-	if (priv->mode != HGPK_MODE_PENTABLET && priv->abs_x != -1) {
-		int x_diff = priv->abs_x - x;
-		int y_diff = priv->abs_y - y;
-		if (hgpk_discard_decay_hack(psmouse, x_diff, y_diff)) {
-			if (tpdebug)
-				psmouse_dbg(psmouse, "discarding\n");
-			goto done;
-		}
-		hgpk_spewing_hack(psmouse, left, right, x_diff, y_diff);
-	}
-
-	input_report_abs(idev, ABS_X, x);
-	input_report_abs(idev, ABS_Y, y);
-	priv->abs_x = x;
-	priv->abs_y = y;
-
-done:
-	input_sync(idev);
-}
-
-static void hgpk_process_simple_packet(struct psmouse *psmouse)
-{
-	struct input_dev *dev = psmouse->dev;
-	unsigned char *packet = psmouse->packet;
-	int left = packet[0] & 1;
-	int right = (packet[0] >> 1) & 1;
-	int x = packet[1] - ((packet[0] << 4) & 0x100);
-	int y = ((packet[0] << 3) & 0x100) - packet[2];
-
-	if (packet[0] & 0xc0)
-		psmouse_dbg(psmouse,
-			    "overflow -- 0x%02x 0x%02x 0x%02x\n",
-			    packet[0], packet[1], packet[2]);
-
-	if (hgpk_discard_decay_hack(psmouse, x, y)) {
-		if (tpdebug)
-			psmouse_dbg(psmouse, "discarding\n");
-		return;
-	}
-
-	hgpk_spewing_hack(psmouse, left, right, x, y);
-
-	if (tpdebug)
-		psmouse_dbg(psmouse, "l=%d r=%d x=%d y=%d\n",
-			    left, right, x, y);
-
-	input_report_key(dev, BTN_LEFT, left);
-	input_report_key(dev, BTN_RIGHT, right);
-
-	input_report_rel(dev, REL_X, x);
-	input_report_rel(dev, REL_Y, y);
-
-	input_sync(dev);
-}
-
-static psmouse_ret_t hgpk_process_byte(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	if (!hgpk_is_byte_valid(psmouse, psmouse->packet))
-		return PSMOUSE_BAD_DATA;
-
-	if (psmouse->pktcnt >= psmouse->pktsize) {
-		if (priv->mode == HGPK_MODE_MOUSE)
-			hgpk_process_simple_packet(psmouse);
-		else
-			hgpk_process_advanced_packet(psmouse);
-		return PSMOUSE_FULL_PACKET;
-	}
-
-	if (priv->recalib_window) {
-		if (time_before(jiffies, priv->recalib_window)) {
-			/*
-			 * ugh, got a packet inside our recalibration
-			 * window, schedule another recalibration.
-			 */
-			psmouse_dbg(psmouse,
-				    "packet inside calibration window, queueing another recalibration\n");
-			psmouse_queue_work(psmouse, &priv->recalib_wq,
-					msecs_to_jiffies(post_interrupt_delay));
-		}
-		priv->recalib_window = 0;
-	}
-
-	return PSMOUSE_GOOD_DATA;
-}
-
-static int hgpk_select_mode(struct psmouse *psmouse)
-{
-	struct ps2dev *ps2dev = &psmouse->ps2dev;
-	struct hgpk_data *priv = psmouse->private;
-	int i;
-	int cmd;
-
-	/*
-	 * 4 disables to enable advanced mode
-	 * then 3 0xf2 bytes as the preamble for GS/PT selection
-	 */
-	const int advanced_init[] = {
-		PSMOUSE_CMD_DISABLE, PSMOUSE_CMD_DISABLE,
-		PSMOUSE_CMD_DISABLE, PSMOUSE_CMD_DISABLE,
-		0xf2, 0xf2, 0xf2,
-	};
-
-	switch (priv->mode) {
-	case HGPK_MODE_MOUSE:
-		psmouse->pktsize = 3;
-		break;
-
-	case HGPK_MODE_GLIDESENSOR:
-	case HGPK_MODE_PENTABLET:
-		psmouse->pktsize = 6;
-
-		/* Switch to 'Advanced mode.', four disables in a row. */
-		for (i = 0; i < ARRAY_SIZE(advanced_init); i++)
-			if (ps2_command(ps2dev, NULL, advanced_init[i]))
-				return -EIO;
-
-		/* select between GlideSensor (mouse) or PenTablet */
-		cmd = priv->mode == HGPK_MODE_GLIDESENSOR ?
-			PSMOUSE_CMD_SETSCALE11 : PSMOUSE_CMD_SETSCALE21;
-
-		if (ps2_command(ps2dev, NULL, cmd))
-			return -EIO;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static void hgpk_setup_input_device(struct input_dev *input,
-				    struct input_dev *old_input,
-				    enum hgpk_mode mode)
-{
-	if (old_input) {
-		input->name = old_input->name;
-		input->phys = old_input->phys;
-		input->id = old_input->id;
-		input->dev.parent = old_input->dev.parent;
-	}
-
-	memset(input->evbit, 0, sizeof(input->evbit));
-	memset(input->relbit, 0, sizeof(input->relbit));
-	memset(input->keybit, 0, sizeof(input->keybit));
-
-	/* All modes report left and right buttons */
-	__set_bit(EV_KEY, input->evbit);
-	__set_bit(BTN_LEFT, input->keybit);
-	__set_bit(BTN_RIGHT, input->keybit);
-
-	switch (mode) {
-	case HGPK_MODE_MOUSE:
-		__set_bit(EV_REL, input->evbit);
-		__set_bit(REL_X, input->relbit);
-		__set_bit(REL_Y, input->relbit);
-		break;
-
-	case HGPK_MODE_GLIDESENSOR:
-		__set_bit(BTN_TOUCH, input->keybit);
-		__set_bit(BTN_TOOL_FINGER, input->keybit);
-
-		__set_bit(EV_ABS, input->evbit);
-
-		/* GlideSensor has pressure sensor, PenTablet does not */
-		input_set_abs_params(input, ABS_PRESSURE, 0, 15, 0, 0);
-
-		/* From device specs */
-		input_set_abs_params(input, ABS_X, 0, 399, 0, 0);
-		input_set_abs_params(input, ABS_Y, 0, 290, 0, 0);
-
-		/* Calculated by hand based on usable size (52mm x 38mm) */
-		input_abs_set_res(input, ABS_X, 8);
-		input_abs_set_res(input, ABS_Y, 8);
-		break;
-
-	case HGPK_MODE_PENTABLET:
-		__set_bit(BTN_TOUCH, input->keybit);
-		__set_bit(BTN_TOOL_FINGER, input->keybit);
-
-		__set_bit(EV_ABS, input->evbit);
-
-		/* From device specs */
-		input_set_abs_params(input, ABS_X, 0, 999, 0, 0);
-		input_set_abs_params(input, ABS_Y, 5, 239, 0, 0);
-
-		/* Calculated by hand based on usable size (156mm x 38mm) */
-		input_abs_set_res(input, ABS_X, 6);
-		input_abs_set_res(input, ABS_Y, 8);
-		break;
-
-	default:
-		BUG();
-	}
-}
-
-static int hgpk_reset_device(struct psmouse *psmouse, bool recalibrate)
-{
-	int err;
-
-	psmouse_reset(psmouse);
-
-	if (recalibrate) {
-		struct ps2dev *ps2dev = &psmouse->ps2dev;
-
-		/* send the recalibrate request */
-		if (ps2_command(ps2dev, NULL, 0xf5) ||
-		    ps2_command(ps2dev, NULL, 0xf5) ||
-		    ps2_command(ps2dev, NULL, 0xe6) ||
-		    ps2_command(ps2dev, NULL, 0xf5)) {
-			return -1;
-		}
-
-		/* according to ALPS, 150mS is required for recalibration */
-		msleep(150);
-	}
-
-	err = hgpk_select_mode(psmouse);
-	if (err) {
-		psmouse_err(psmouse, "failed to select mode\n");
-		return err;
-	}
-
-	hgpk_reset_hack_state(psmouse);
-
-	return 0;
-}
-
-static int hgpk_force_recalibrate(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-	int err;
-
-	/* C-series touchpads added the recalibrate command */
-	if (psmouse->model < HGPK_MODEL_C)
-		return 0;
-
-	if (!autorecal) {
-		psmouse_dbg(psmouse, "recalibration disabled, ignoring\n");
-		return 0;
-	}
-
-	psmouse_dbg(psmouse, "recalibrating touchpad..\n");
-
-	/* we don't want to race with the irq handler, nor with resyncs */
-	psmouse_set_state(psmouse, PSMOUSE_INITIALIZING);
-
-	/* start by resetting the device */
-	err = hgpk_reset_device(psmouse, true);
-	if (err)
-		return err;
-
-	/*
-	 * XXX: If a finger is down during this delay, recalibration will
-	 * detect capacitance incorrectly.  This is a hardware bug, and
-	 * we don't have a good way to deal with it.  The 2s window stuff
-	 * (below) is our best option for now.
-	 */
-	if (psmouse_activate(psmouse))
-		return -1;
-
-	if (tpdebug)
-		psmouse_dbg(psmouse, "touchpad reactivated\n");
-
-	/*
-	 * If we get packets right away after recalibrating, it's likely
-	 * that a finger was on the touchpad.  If so, it's probably
-	 * miscalibrated, so we optionally schedule another.
-	 */
-	if (recal_guard_time)
-		priv->recalib_window = jiffies +
-			msecs_to_jiffies(recal_guard_time);
-
-	return 0;
-}
-
-/*
- * This puts the touchpad in a power saving mode; according to ALPS, current
- * consumption goes down to 50uA after running this.  To turn power back on,
- * we drive MS-DAT low.  Measuring with a 1mA resolution ammeter says that
- * the current on the SUS_3.3V rail drops from 3mA or 4mA to 0 when we do this.
- *
- * We have no formal spec that details this operation -- the low-power
- * sequence came from a long-lost email trail.
- */
-static int hgpk_toggle_powersave(struct psmouse *psmouse, int enable)
-{
-	struct ps2dev *ps2dev = &psmouse->ps2dev;
-	int timeo;
-	int err;
-
-	/* Added on D-series touchpads */
-	if (psmouse->model < HGPK_MODEL_D)
-		return 0;
-
-	if (enable) {
-		psmouse_set_state(psmouse, PSMOUSE_INITIALIZING);
-
-		/*
-		 * Sending a byte will drive MS-DAT low; this will wake up
-		 * the controller.  Once we get an ACK back from it, it
-		 * means we can continue with the touchpad re-init.  ALPS
-		 * tells us that 1s should be long enough, so set that as
-		 * the upper bound. (in practice, it takes about 3 loops.)
-		 */
-		for (timeo = 20; timeo > 0; timeo--) {
-			if (!ps2_sendbyte(ps2dev, PSMOUSE_CMD_DISABLE, 20))
-				break;
-			msleep(25);
-		}
-
-		err = hgpk_reset_device(psmouse, false);
-		if (err) {
-			psmouse_err(psmouse, "Failed to reset device!\n");
-			return err;
-		}
-
-		/* should be all set, enable the touchpad */
-		psmouse_activate(psmouse);
-		psmouse_dbg(psmouse, "Touchpad powered up.\n");
-	} else {
-		psmouse_dbg(psmouse, "Powering off touchpad.\n");
-
-		if (ps2_command(ps2dev, NULL, 0xec) ||
-		    ps2_command(ps2dev, NULL, 0xec) ||
-		    ps2_command(ps2dev, NULL, 0xea)) {
-			return -1;
-		}
-
-		psmouse_set_state(psmouse, PSMOUSE_IGNORE);
-
-		/* probably won't see an ACK, the touchpad will be off */
-		ps2_sendbyte(ps2dev, 0xec, 20);
-	}
-
-	return 0;
-}
-
-static int hgpk_poll(struct psmouse *psmouse)
-{
-	/* We can't poll, so always return failure. */
-	return -1;
-}
-
-static int hgpk_reconnect(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	/*
-	 * During suspend/resume the ps2 rails remain powered.  We don't want
-	 * to do a reset because it's flush data out of buffers; however,
-	 * earlier prototypes (B1) had some brokenness that required a reset.
-	 */
-	if (olpc_board_at_least(olpc_board(0xb2)))
-		if (psmouse->ps2dev.serio->dev.power.power_state.event !=
-				PM_EVENT_ON)
-			return 0;
-
-	priv->powered = 1;
-	return hgpk_reset_device(psmouse, false);
-}
-
-static ssize_t hgpk_show_powered(struct psmouse *psmouse, void *data, char *buf)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	return sprintf(buf, "%d\n", priv->powered);
-}
-
-static ssize_t hgpk_set_powered(struct psmouse *psmouse, void *data,
-				const char *buf, size_t count)
-{
-	struct hgpk_data *priv = psmouse->private;
-	unsigned int value;
-	int err;
-
-	err = kstrtouint(buf, 10, &value);
-	if (err)
-		return err;
-
-	if (value > 1)
-		return -EINVAL;
-
-	if (value != priv->powered) {
-		/*
-		 * hgpk_toggle_power will deal w/ state so
-		 * we're not racing w/ irq
-		 */
-		err = hgpk_toggle_powersave(psmouse, value);
-		if (!err)
-			priv->powered = value;
-	}
-
-	return err ? err : count;
-}
-
-__PSMOUSE_DEFINE_ATTR(powered, S_IWUSR | S_IRUGO, NULL,
-		      hgpk_show_powered, hgpk_set_powered, false);
-
-static ssize_t attr_show_mode(struct psmouse *psmouse, void *data, char *buf)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	return sprintf(buf, "%s\n", hgpk_mode_names[priv->mode]);
-}
-
-static ssize_t attr_set_mode(struct psmouse *psmouse, void *data,
-			     const char *buf, size_t len)
-{
-	struct hgpk_data *priv = psmouse->private;
-	enum hgpk_mode old_mode = priv->mode;
-	enum hgpk_mode new_mode = hgpk_mode_from_name(buf, len);
-	struct input_dev *old_dev = psmouse->dev;
-	struct input_dev *new_dev;
-	int err;
-
-	if (new_mode == HGPK_MODE_INVALID)
-		return -EINVAL;
-
-	if (old_mode == new_mode)
-		return len;
-
-	new_dev = input_allocate_device();
-	if (!new_dev)
-		return -ENOMEM;
-
-	psmouse_set_state(psmouse, PSMOUSE_INITIALIZING);
-
-	/* Switch device into the new mode */
-	priv->mode = new_mode;
-	err = hgpk_reset_device(psmouse, false);
-	if (err)
-		goto err_try_restore;
-
-	hgpk_setup_input_device(new_dev, old_dev, new_mode);
-
-	psmouse_set_state(psmouse, PSMOUSE_CMD_MODE);
-
-	err = input_register_device(new_dev);
-	if (err)
-		goto err_try_restore;
-
-	psmouse->dev = new_dev;
-	input_unregister_device(old_dev);
-
-	return len;
-
-err_try_restore:
-	input_free_device(new_dev);
-	priv->mode = old_mode;
-	hgpk_reset_device(psmouse, false);
-
-	return err;
-}
-
-PSMOUSE_DEFINE_ATTR(hgpk_mode, S_IWUSR | S_IRUGO, NULL,
-		    attr_show_mode, attr_set_mode);
-
-static ssize_t hgpk_trigger_recal_show(struct psmouse *psmouse,
-		void *data, char *buf)
-{
-	return -EINVAL;
-}
-
-static ssize_t hgpk_trigger_recal(struct psmouse *psmouse, void *data,
-				const char *buf, size_t count)
-{
-	struct hgpk_data *priv = psmouse->private;
-	unsigned int value;
-	int err;
-
-	err = kstrtouint(buf, 10, &value);
-	if (err)
-		return err;
-
-	if (value != 1)
-		return -EINVAL;
-
-	/*
-	 * We queue work instead of doing recalibration right here
-	 * to avoid adding locking to hgpk_force_recalibrate()
-	 * since workqueue provides serialization.
-	 */
-	psmouse_queue_work(psmouse, &priv->recalib_wq, 0);
-	return count;
-}
-
-__PSMOUSE_DEFINE_ATTR(recalibrate, S_IWUSR | S_IRUGO, NULL,
-		      hgpk_trigger_recal_show, hgpk_trigger_recal, false);
-
-static void hgpk_disconnect(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-
-	device_remove_file(&psmouse->ps2dev.serio->dev,
-			   &psmouse_attr_powered.dattr);
-	device_remove_file(&psmouse->ps2dev.serio->dev,
-			   &psmouse_attr_hgpk_mode.dattr);
-
-	if (psmouse->model >= HGPK_MODEL_C)
-		device_remove_file(&psmouse->ps2dev.serio->dev,
-				   &psmouse_attr_recalibrate.dattr);
-
-	psmouse_reset(psmouse);
-	kfree(priv);
-}
-
-static void hgpk_recalib_work(struct work_struct *work)
-{
-	struct delayed_work *w = to_delayed_work(work);
-	struct hgpk_data *priv = container_of(w, struct hgpk_data, recalib_wq);
-	struct psmouse *psmouse = priv->psmouse;
-
-	if (hgpk_force_recalibrate(psmouse))
-		psmouse_err(psmouse, "recalibration failed!\n");
-}
-
-static int hgpk_register(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv = psmouse->private;
-	int err;
-
-	/* register handlers */
-	psmouse->protocol_handler = hgpk_process_byte;
-	psmouse->poll = hgpk_poll;
-	psmouse->disconnect = hgpk_disconnect;
-	psmouse->reconnect = hgpk_reconnect;
-
-	/* Disable the idle resync. */
-	psmouse->resync_time = 0;
-	/* Reset after a lot of bad bytes. */
-	psmouse->resetafter = 1024;
-
-	hgpk_setup_input_device(psmouse->dev, NULL, priv->mode);
-
-	err = device_create_file(&psmouse->ps2dev.serio->dev,
-				 &psmouse_attr_powered.dattr);
-	if (err) {
-		psmouse_err(psmouse, "Failed creating 'powered' sysfs node\n");
-		return err;
-	}
-
-	err = device_create_file(&psmouse->ps2dev.serio->dev,
-				 &psmouse_attr_hgpk_mode.dattr);
-	if (err) {
-		psmouse_err(psmouse,
-			    "Failed creating 'hgpk_mode' sysfs node\n");
-		goto err_remove_powered;
-	}
-
-	/* C-series touchpads added the recalibrate command */
-	if (psmouse->model >= HGPK_MODEL_C) {
-		err = device_create_file(&psmouse->ps2dev.serio->dev,
-					 &psmouse_attr_recalibrate.dattr);
-		if (err) {
-			psmouse_err(psmouse,
-				    "Failed creating 'recalibrate' sysfs node\n");
-			goto err_remove_mode;
-		}
-	}
-
-	return 0;
-
-err_remove_mode:
-	device_remove_file(&psmouse->ps2dev.serio->dev,
-			   &psmouse_attr_hgpk_mode.dattr);
-err_remove_powered:
-	device_remove_file(&psmouse->ps2dev.serio->dev,
-			   &psmouse_attr_powered.dattr);
-	return err;
-}
-
-int hgpk_init(struct psmouse *psmouse)
-{
-	struct hgpk_data *priv;
-	int err;
-
-	priv = kzalloc_obj(*priv);
-	if (!priv) {
-		err = -ENOMEM;
-		goto alloc_fail;
-	}
-
-	psmouse->private = priv;
-
-	priv->psmouse = psmouse;
-	priv->powered = true;
-	priv->mode = hgpk_default_mode;
-	INIT_DELAYED_WORK(&priv->recalib_wq, hgpk_recalib_work);
-
-	err = hgpk_reset_device(psmouse, false);
-	if (err)
-		goto init_fail;
-
-	err = hgpk_register(psmouse);
-	if (err)
-		goto init_fail;
-
-	return 0;
-
-init_fail:
-	kfree(priv);
-alloc_fail:
-	return err;
-}
-
-static enum hgpk_model_t hgpk_get_model(struct psmouse *psmouse)
-{
-	struct ps2dev *ps2dev = &psmouse->ps2dev;
-	unsigned char param[3];
-
-	/* E7, E7, E7, E9 gets us a 3 byte identifier */
-	if (ps2_command(ps2dev,  NULL, PSMOUSE_CMD_SETSCALE21) ||
-	    ps2_command(ps2dev,  NULL, PSMOUSE_CMD_SETSCALE21) ||
-	    ps2_command(ps2dev,  NULL, PSMOUSE_CMD_SETSCALE21) ||
-	    ps2_command(ps2dev, param, PSMOUSE_CMD_GETINFO)) {
-		return -EIO;
-	}
-
-	psmouse_dbg(psmouse, "ID: %*ph\n", 3, param);
-
-	/* HGPK signature: 0x67, 0x00, 0x<model> */
-	if (param[0] != 0x67 || param[1] != 0x00)
-		return -ENODEV;
-
-	psmouse_info(psmouse, "OLPC touchpad revision 0x%x\n", param[2]);
-
-	return param[2];
-}
-
-int hgpk_detect(struct psmouse *psmouse, bool set_properties)
-{
-	int version;
-
-	version = hgpk_get_model(psmouse);
-	if (version < 0)
-		return version;
-
-	if (set_properties) {
-		psmouse->vendor = "ALPS";
-		psmouse->name = "HGPK";
-		psmouse->model = version;
-	}
-
-	return 0;
-}
-
-void hgpk_module_init(void)
-{
-	hgpk_default_mode = hgpk_mode_from_name(hgpk_mode_name,
-						strlen(hgpk_mode_name));
-	if (hgpk_default_mode == HGPK_MODE_INVALID) {
-		hgpk_default_mode = HGPK_MODE_MOUSE;
-		strscpy(hgpk_mode_name, hgpk_mode_names[HGPK_MODE_MOUSE],
-			sizeof(hgpk_mode_name));
-	}
-}
diff --git a/drivers/input/mouse/hgpk.h b/drivers/input/mouse/hgpk.h
deleted file mode 100644
index ce041591f1a8b..0000000000000
--- a/drivers/input/mouse/hgpk.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * OLPC HGPK (XO-1) touchpad PS/2 mouse driver
- */
-
-#ifndef _HGPK_H
-#define _HGPK_H
-
-#define HGPK_GS		0xff       /* The GlideSensor */
-#define HGPK_PT		0xcf       /* The PenTablet */
-
-enum hgpk_model_t {
-	HGPK_MODEL_PREA = 0x0a,	/* pre-B1s */
-	HGPK_MODEL_A = 0x14,	/* found on B1s, PT disabled in hardware */
-	HGPK_MODEL_B = 0x28,	/* B2s, has capacitance issues */
-	HGPK_MODEL_C = 0x3c,
-	HGPK_MODEL_D = 0x50,	/* C1, mass production */
-};
-
-enum hgpk_spew_flag {
-	NO_SPEW,
-	MAYBE_SPEWING,
-	SPEW_DETECTED,
-	RECALIBRATING,
-};
-
-#define SPEW_WATCH_COUNT 42  /* at 12ms/packet, this is 1/2 second */
-
-enum hgpk_mode {
-	HGPK_MODE_MOUSE,
-	HGPK_MODE_GLIDESENSOR,
-	HGPK_MODE_PENTABLET,
-	HGPK_MODE_INVALID
-};
-
-struct hgpk_data {
-	struct psmouse *psmouse;
-	enum hgpk_mode mode;
-	bool powered;
-	enum hgpk_spew_flag spew_flag;
-	int spew_count, x_tally, y_tally;	/* spew detection */
-	unsigned long recalib_window;
-	struct delayed_work recalib_wq;
-	int abs_x, abs_y;
-	int dupe_count;
-	int xbigj, ybigj, xlast, ylast; /* jumpiness detection */
-	int xsaw_secondary, ysaw_secondary; /* jumpiness detection */
-};
-
-int hgpk_detect(struct psmouse *psmouse, bool set_properties);
-int hgpk_init(struct psmouse *psmouse);
-
-#ifdef CONFIG_MOUSE_PS2_OLPC
-void hgpk_module_init(void);
-#else
-static inline void hgpk_module_init(void)
-{
-}
-#endif
-
-#endif
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index ff8cd2d68a3cb..6ab5f1d96eaea 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -26,7 +26,6 @@
 #include "synaptics.h"
 #include "logips2pp.h"
 #include "alps.h"
-#include "hgpk.h"
 #include "lifebook.h"
 #include "trackpoint.h"
 #include "touchkit_ps2.h"
@@ -114,8 +113,6 @@ ATTRIBUTE_GROUPS(psmouse_dev);
  */
 static DEFINE_MUTEX(psmouse_mutex);
 
-static struct workqueue_struct *kpsmoused_wq;
-
 struct psmouse *psmouse_from_serio(struct serio *serio)
 {
 	struct ps2dev *ps2dev = serio_get_drvdata(serio);
@@ -241,12 +238,6 @@ psmouse_ret_t psmouse_process_byte(struct psmouse *psmouse)
 	return PSMOUSE_FULL_PACKET;
 }
 
-void psmouse_queue_work(struct psmouse *psmouse, struct delayed_work *work,
-		unsigned long delay)
-{
-	queue_delayed_work(kpsmoused_wq, work, delay);
-}
-
 /*
  * __psmouse_set_state() sets new psmouse state and resets all flags.
  */
@@ -380,7 +371,7 @@ static void psmouse_receive_byte(struct ps2dev *ps2dev, u8 data)
 			     psmouse->name, psmouse->phys, psmouse->pktcnt);
 		psmouse->badbyte = psmouse->packet[0];
 		__psmouse_set_state(psmouse, PSMOUSE_RESYNCING);
-		psmouse_queue_work(psmouse, &psmouse->resync_work, 0);
+		schedule_work(&psmouse->resync_work);
 		return;
 	}
 
@@ -393,9 +384,7 @@ static void psmouse_receive_byte(struct ps2dev *ps2dev, u8 data)
 			return;
 		}
 
-		if (psmouse->packet[1] == PSMOUSE_RET_ID ||
-		    (psmouse->protocol->type == PSMOUSE_HGPK &&
-		     psmouse->packet[1] == PSMOUSE_RET_BAT)) {
+		if (psmouse->packet[1] == PSMOUSE_RET_ID) {
 			__psmouse_set_state(psmouse, PSMOUSE_IGNORE);
 			serio_reconnect(ps2dev->serio);
 			return;
@@ -418,7 +407,7 @@ static void psmouse_receive_byte(struct ps2dev *ps2dev, u8 data)
 	    time_after(jiffies, psmouse->last + psmouse->resync_time * HZ)) {
 		psmouse->badbyte = psmouse->packet[0];
 		__psmouse_set_state(psmouse, PSMOUSE_RESYNCING);
-		psmouse_queue_work(psmouse, &psmouse->resync_work, 0);
+		schedule_work(&psmouse->resync_work);
 		return;
 	}
 
@@ -837,14 +826,6 @@ static const struct psmouse_protocol psmouse_protocols[] = {
 		.detect		= touchkit_ps2_detect,
 	},
 #endif
-#ifdef CONFIG_MOUSE_PS2_OLPC
-	{
-		.type		= PSMOUSE_HGPK,
-		.name		= "OLPC HGPK",
-		.alias		= "hgpk",
-		.detect		= hgpk_detect,
-	},
-#endif
 #ifdef CONFIG_MOUSE_PS2_ELANTECH
 	{
 		.type		= PSMOUSE_ELANTECH,
@@ -1153,13 +1134,6 @@ static int psmouse_extensions(struct psmouse *psmouse,
 			return PSMOUSE_ALPS;
 	}
 
-	/* Try OLPC HGPK touchpad */
-	if (max_proto > PSMOUSE_IMEX &&
-	    psmouse_try_protocol(psmouse, PSMOUSE_HGPK, &max_proto,
-				 set_properties, true)) {
-		return PSMOUSE_HGPK;
-	}
-
 	/* Try Elantech touchpad */
 	if (max_proto > PSMOUSE_IMEX &&
 	    psmouse_try_protocol(psmouse, PSMOUSE_ELANTECH,
@@ -1331,7 +1305,7 @@ int psmouse_deactivate(struct psmouse *psmouse)
 static void psmouse_resync(struct work_struct *work)
 {
 	struct psmouse *parent = NULL, *psmouse =
-		container_of(work, struct psmouse, resync_work.work);
+		container_of(work, struct psmouse, resync_work);
 	struct serio *serio = psmouse->ps2dev.serio;
 	psmouse_ret_t rc = PSMOUSE_GOOD_DATA;
 	bool failed = false, enabled = false;
@@ -1484,7 +1458,7 @@ static void psmouse_disconnect(struct serio *serio)
 
 	/* make sure we don't have a resync in progress */
 	mutex_unlock(&psmouse_mutex);
-	flush_workqueue(kpsmoused_wq);
+	disable_work_sync(&psmouse->resync_work);
 	mutex_lock(&psmouse_mutex);
 
 	if (serio->parent && serio->id.type == SERIO_PS_PSTHRU) {
@@ -1598,7 +1572,7 @@ static int psmouse_connect(struct serio *serio, struct serio_driver *drv)
 
 	ps2_init(&psmouse->ps2dev, serio,
 		 psmouse_pre_receive_byte, psmouse_receive_byte);
-	INIT_DELAYED_WORK(&psmouse->resync_work, psmouse_resync);
+	INIT_WORK(&psmouse->resync_work, psmouse_resync);
 	psmouse->dev = input_dev;
 	scnprintf(psmouse->phys, sizeof(psmouse->phys), "%s/input0", serio->phys);
 
@@ -2035,27 +2009,17 @@ static int __init psmouse_init(void)
 
 	lifebook_module_init();
 	synaptics_module_init();
-	hgpk_module_init();
 
 	err = psmouse_smbus_module_init();
 	if (err)
 		return err;
 
-	kpsmoused_wq = alloc_ordered_workqueue("kpsmoused", 0);
-	if (!kpsmoused_wq) {
-		pr_err("failed to create kpsmoused workqueue\n");
-		err = -ENOMEM;
-		goto err_smbus_exit;
-	}
-
 	err = serio_register_driver(&psmouse_drv);
 	if (err)
-		goto err_destroy_wq;
+		goto err_smbus_exit;
 
 	return 0;
 
-err_destroy_wq:
-	destroy_workqueue(kpsmoused_wq);
 err_smbus_exit:
 	psmouse_smbus_module_exit();
 	return err;
@@ -2064,7 +2028,6 @@ static int __init psmouse_init(void)
 static void __exit psmouse_exit(void)
 {
 	serio_unregister_driver(&psmouse_drv);
-	destroy_workqueue(kpsmoused_wq);
 	psmouse_smbus_module_exit();
 }
 
diff --git a/drivers/input/mouse/psmouse.h b/drivers/input/mouse/psmouse.h
index 4d8acfe0d82aa..90ed8cd15d857 100644
--- a/drivers/input/mouse/psmouse.h
+++ b/drivers/input/mouse/psmouse.h
@@ -59,7 +59,7 @@ enum psmouse_type {
 	PSMOUSE_TRACKPOINT,
 	PSMOUSE_TOUCHKIT_PS2,
 	PSMOUSE_CORTRON,
-	PSMOUSE_HGPK,
+	PSMOUSE_HGPK,		/* No longer used */
 	PSMOUSE_ELANTECH,
 	PSMOUSE_FSP,
 	PSMOUSE_SYNAPTICS_RELATIVE,
@@ -90,7 +90,7 @@ struct psmouse {
 	void *private;
 	struct input_dev *dev;
 	struct ps2dev ps2dev;
-	struct delayed_work resync_work;
+	struct work_struct resync_work;
 	const char *vendor;
 	const char *name;
 	const struct psmouse_protocol *protocol;
@@ -132,8 +132,6 @@ struct psmouse {
 
 struct psmouse *psmouse_from_serio(struct serio *serio);
 
-void psmouse_queue_work(struct psmouse *psmouse, struct delayed_work *work,
-		unsigned long delay);
 int psmouse_reset(struct psmouse *psmouse);
 void psmouse_set_state(struct psmouse *psmouse, enum psmouse_state new_state);
 void psmouse_set_resolution(struct psmouse *psmouse, unsigned int resolution);
diff --git a/drivers/input/mouse/trackpoint.c b/drivers/input/mouse/trackpoint.c
index b06c7ad721fe9..3bd8fdf56cd3d 100644
--- a/drivers/input/mouse/trackpoint.c
+++ b/drivers/input/mouse/trackpoint.c
@@ -5,6 +5,7 @@
  * Trademarks are the property of their respective owners.
  */
 
+#include <linux/array_size.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/serio.h>
@@ -12,6 +13,7 @@
 #include <linux/input.h>
 #include <linux/libps2.h>
 #include <linux/proc_fs.h>
+#include <linux/string.h>
 #include <linux/uaccess.h>
 #include "psmouse.h"
 #include "trackpoint.h"
@@ -393,6 +395,44 @@ static int trackpoint_reconnect(struct psmouse *psmouse)
 	return 0;
 }
 
+/* List of known incapable device PNP IDs */
+static const char * const dt_incompatible_devices[] = {
+	"LEN0304",
+	"LEN0306",
+	"LEN0317",
+	"LEN031A",
+	"LEN031B",
+	"LEN031C",
+	"LEN031D",
+};
+
+/*
+ * Checks if it's a doubletap capable device.
+ * The PNP ID format is "PNP: LEN030d PNP0f13".
+ */
+static bool trackpoint_is_dt_capable(const char *pnp_id)
+{
+	size_t i;
+
+	if (!pnp_id)
+		return false;
+
+	/* Must start with "PNP: LEN03" */
+	if (!strstarts(pnp_id, "PNP: LEN03"))
+		return false;
+
+	/* Ensure enough length before comparing */
+	if (strlen(pnp_id) < 12)
+		return false;
+
+	/* Check deny-list */
+	for (i = 0; i < ARRAY_SIZE(dt_incompatible_devices); i++) {
+		if (!strncmp(pnp_id + 5, dt_incompatible_devices[i], 7))
+			return false;
+	}
+	return true;
+}
+
 int trackpoint_detect(struct psmouse *psmouse, bool set_properties)
 {
 	struct ps2dev *ps2dev = &psmouse->ps2dev;
@@ -470,6 +510,12 @@ int trackpoint_detect(struct psmouse *psmouse, bool set_properties)
 		     psmouse->vendor, firmware_id,
 		     (button_info & 0xf0) >> 4, button_info & 0x0f);
 
+	if (trackpoint_is_dt_capable(ps2dev->serio->firmware_id)) {
+		error = trackpoint_write(ps2dev, TP_DOUBLETAP, TP_DOUBLETAP_ENABLE);
+		if (error)
+			psmouse_warn(psmouse, "Failed to enable doubletap: %d\n", error);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/input/mouse/trackpoint.h b/drivers/input/mouse/trackpoint.h
index eb5412904fe07..3e03cdb394498 100644
--- a/drivers/input/mouse/trackpoint.h
+++ b/drivers/input/mouse/trackpoint.h
@@ -69,6 +69,8 @@
 					/* (how hard it is to drag */
 					/* with Z-axis pressed) */
 
+#define TP_DOUBLETAP		0x58	/* TrackPoint doubletap register */
+
 #define TP_MINDRAG		0x59	/* Minimum amount of force needed */
 					/* to trigger dragging */
 
@@ -110,6 +112,9 @@
 					   external device will be forced to 1 */
 #define TP_MASK_EXT_TAG			0x04
 
+/* Doubletap register values */
+#define TP_DOUBLETAP_ENABLE	0xFF	/* Enable value */
+#define TP_DOUBLETAP_DISABLE	0xFE	/* Disable value */
 
 /* Power on Self Test Results */
 #define TP_POR_SUCCESS		0x3B
diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c
index ac4041a69fcd3..61909e1a39e24 100644
--- a/drivers/input/rmi4/rmi_f54.c
+++ b/drivers/input/rmi4/rmi_f54.c
@@ -538,6 +538,8 @@ static void rmi_f54_work(struct work_struct *work)
 	int error;
 	int i;
 
+	mutex_lock(&f54->data_mutex);
+
 	report_size = rmi_f54_get_report_size(f54);
 	if (report_size == 0) {
 		dev_err(&fn->dev, "Bad report size, report type=%d\n",
@@ -546,8 +548,6 @@ static void rmi_f54_work(struct work_struct *work)
 		goto error;     /* retry won't help */
 	}
 
-	mutex_lock(&f54->data_mutex);
-
 	/*
 	 * Need to check if command has completed.
 	 * If not try again later.
diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h
index d2cf940b105a6..8ebdf4fb90308 100644
--- a/drivers/input/serio/i8042-acpipnpio.h
+++ b/drivers/input/serio/i8042-acpipnpio.h
@@ -1187,6 +1187,13 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = {
 		.driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS |
 					SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP)
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "X6KK45xU_X6SP45xU"),
+		},
+		.driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS |
+					SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP)
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "WUJIE Series-X5SP4NAG"),
diff --git a/drivers/input/serio/libps2.c b/drivers/input/serio/libps2.c
index 269df83a167d9..05b64277aecd9 100644
--- a/drivers/input/serio/libps2.c
+++ b/drivers/input/serio/libps2.c
@@ -154,10 +154,8 @@ EXPORT_SYMBOL(ps2_end_command);
  */
 void ps2_drain(struct ps2dev *ps2dev, size_t maxbytes, unsigned int timeout)
 {
-	if (maxbytes > sizeof(ps2dev->cmdbuf)) {
-		WARN_ON(1);
+	if (WARN_ON(maxbytes > sizeof(ps2dev->cmdbuf)))
 		maxbytes = sizeof(ps2dev->cmdbuf);
-	}
 
 	ps2_begin_command(ps2dev);
 
@@ -270,15 +268,11 @@ int __ps2_command(struct ps2dev *ps2dev, u8 *param, unsigned int command)
 	int i;
 	u8 send_param[16];
 
-	if (receive > sizeof(ps2dev->cmdbuf)) {
-		WARN_ON(1);
+	if (WARN_ON(receive > sizeof(ps2dev->cmdbuf)))
 		return -EINVAL;
-	}
 
-	if (send && !param) {
-		WARN_ON(1);
+	if (WARN_ON(send && !param))
 		return -EINVAL;
-	}
 
 	memcpy(send_param, param, send);
 
diff --git a/drivers/input/touchscreen/st1232.c b/drivers/input/touchscreen/st1232.c
index 9b3901eec0a5e..9b266927b7fe3 100644
--- a/drivers/input/touchscreen/st1232.c
+++ b/drivers/input/touchscreen/st1232.c
@@ -15,6 +15,7 @@
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
+#include <linux/input/touch-overlay.h>
 #include <linux/input/touchscreen.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
@@ -22,11 +23,14 @@
 #include <linux/pm_qos.h>
 #include <linux/slab.h>
 #include <linux/types.h>
-#include <linux/input/touch-overlay.h>
+#include <asm/byteorder.h>
 
 #define ST1232_TS_NAME	"st1232-ts"
 #define ST1633_TS_NAME	"st1633-ts"
 
+#define REG_FIRMWARE_VERSION	0x00
+#define REG_FIRMWARE_REVISION_3	0x0C
+
 #define REG_STATUS		0x01	/* Device Status | Error Code */
 
 #define STATUS_NORMAL		0x00
@@ -61,8 +65,38 @@ struct st1232_ts_data {
 	struct list_head touch_overlay_list;
 	int read_buf_len;
 	u8 *read_buf;
+	u8 fw_version;
+	u32 fw_revision;
 };
 
+static ssize_t fw_version_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct st1232_ts_data *st1232_ts = i2c_get_clientdata(client);
+
+	return sysfs_emit(buf, "%u\n", st1232_ts->fw_version);
+}
+
+static ssize_t fw_revision_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct st1232_ts_data *st1232_ts = i2c_get_clientdata(client);
+
+	return sysfs_emit(buf, "%08x\n", st1232_ts->fw_revision);
+}
+
+static DEVICE_ATTR_RO(fw_version);
+static DEVICE_ATTR_RO(fw_revision);
+
+static struct attribute *st1232_attrs[] = {
+	&dev_attr_fw_version.attr,
+	&dev_attr_fw_revision.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(st1232);
+
 static int st1232_ts_read_data(struct st1232_ts_data *ts, u8 reg,
 			       unsigned int n)
 {
@@ -110,6 +144,26 @@ static int st1232_ts_wait_ready(struct st1232_ts_data *ts)
 	return -ENXIO;
 }
 
+static int st1232_ts_read_fw_version(struct st1232_ts_data *ts,
+				     u8 *fw_version, u32 *fw_revision)
+{
+	int error;
+
+	/* select firmware version register */
+	error = st1232_ts_read_data(ts, REG_FIRMWARE_VERSION, 1);
+	if (error)
+		return error;
+	*fw_version = ts->read_buf[0];
+
+	/* select firmware revision register */
+	error = st1232_ts_read_data(ts, REG_FIRMWARE_REVISION_3, 4);
+	if (error)
+		return error;
+	*fw_revision = le32_to_cpup((__le32 *)ts->read_buf);
+
+	return 0;
+}
+
 static int st1232_ts_read_resolution(struct st1232_ts_data *ts, u16 *max_x,
 				     u16 *max_y)
 {
@@ -299,6 +353,16 @@ static int st1232_ts_probe(struct i2c_client *client)
 	if (error)
 		return error;
 
+	/* Read firmware version from the chip */
+	error = st1232_ts_read_fw_version(ts, &ts->fw_version, &ts->fw_revision);
+	if (error) {
+		dev_err(&client->dev,
+			"Failed to read firmware version: %d\n", error);
+		return error;
+	}
+	dev_dbg(&client->dev, "Detected firmware version %u, rev %08x\n",
+		ts->fw_version, ts->fw_revision);
+
 	if (ts->chip_info->have_z)
 		input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0,
 				     ts->chip_info->max_area, 0, 0);
@@ -408,6 +472,7 @@ static struct i2c_driver st1232_ts_driver = {
 	.driver = {
 		.name	= ST1232_TS_NAME,
 		.of_match_table = st1232_ts_dt_ids,
+		.dev_groups	= st1232_groups,
 		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.pm	= pm_sleep_ptr(&st1232_ts_pm_ops),
 	},
diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig
index bb1cb8a640c17..786b4eda44b4f 100644
--- a/drivers/interconnect/qcom/Kconfig
+++ b/drivers/interconnect/qcom/Kconfig
@@ -8,8 +8,17 @@ config INTERCONNECT_QCOM
 config INTERCONNECT_QCOM_BCM_VOTER
 	tristate
 
+config INTERCONNECT_QCOM_ELIZA
+       tristate "Qualcomm Eliza interconnect driver"
+       depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+       select INTERCONNECT_QCOM_RPMH
+       select INTERCONNECT_QCOM_BCM_VOTER
+       help
+         This is a driver for the Qualcomm Network-on-Chip on Eliza-based
+         platforms.
+
 config INTERCONNECT_QCOM_GLYMUR
-       tristate "Qualcomm GLYMUR interconnect driver"
+       tristate "Qualcomm Glymur interconnect driver"
        depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
        select INTERCONNECT_QCOM_RPMH
        select INTERCONNECT_QCOM_BCM_VOTER
@@ -18,7 +27,7 @@ config INTERCONNECT_QCOM_GLYMUR
          platforms.
 
 config INTERCONNECT_QCOM_KAANAPALI
-	tristate "Qualcomm KAANAPALI interconnect driver"
+	tristate "Qualcomm Kaanapali interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile
index 6eedff043b412..cdf2c6c9fbf32 100644
--- a/drivers/interconnect/qcom/Makefile
+++ b/drivers/interconnect/qcom/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM) += interconnect_qcom.o
 
 interconnect_qcom-y			:= icc-common.o
 icc-bcm-voter-objs			:= bcm-voter.o
+qnoc-eliza-objs				:= eliza.o
 qnoc-glymur-objs			:= glymur.o
 qnoc-kaanapali-objs			:= kaanapali.o
 qnoc-milos-objs				:= milos.o
@@ -48,6 +49,7 @@ qnoc-x1e80100-objs			:= x1e80100.o
 icc-smd-rpm-objs			:= smd-rpm.o icc-rpm.o icc-rpm-clocks.o
 
 obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o
+obj-$(CONFIG_INTERCONNECT_QCOM_ELIZA) += qnoc-eliza.o
 obj-$(CONFIG_INTERCONNECT_QCOM_GLYMUR) += qnoc-glymur.o
 obj-$(CONFIG_INTERCONNECT_QCOM_KAANAPALI) += qnoc-kaanapali.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MILOS) += qnoc-milos.o
diff --git a/drivers/interconnect/qcom/eliza.c b/drivers/interconnect/qcom/eliza.c
new file mode 100644
index 0000000000000..a4f7903f0524d
--- /dev/null
+++ b/drivers/interconnect/qcom/eliza.c
@@ -0,0 +1,1585 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <dt-bindings/interconnect/qcom,eliza-rpmh.h>
+
+#include "bcm-voter.h"
+#include "icc-rpmh.h"
+
+static struct qcom_icc_node qup1_core_slave = {
+	.name = "qup1_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup2_core_slave = {
+	.name = "qup2_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy0 = {
+	.name = "qhs_ahb2phy0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy1 = {
+	.name = "qhs_ahb2phy1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_camera_cfg = {
+	.name = "qhs_camera_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto0_cfg = {
+	.name = "qhs_crypto0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_display_cfg = {
+	.name = "qhs_display_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_gpuss_cfg = {
+	.name = "qhs_gpuss_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_i3c_ibi0_cfg = {
+	.name = "qhs_i3c_ibi0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_i3c_ibi1_cfg = {
+	.name = "qhs_i3c_ibi1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_mss_cfg = {
+	.name = "qhs_mss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_0_cfg = {
+	.name = "qhs_pcie_0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_prng = {
+	.name = "qhs_prng",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qdss_cfg = {
+	.name = "qhs_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qspi = {
+	.name = "qhs_qspi",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup1 = {
+	.name = "qhs_qup1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup2 = {
+	.name = "qhs_qup2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc2 = {
+	.name = "qhs_sdc2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tcsr = {
+	.name = "qhs_tcsr",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ufs_mem_cfg = {
+	.name = "qhs_ufs_mem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb3_0 = {
+	.name = "qhs_usb3_0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_venus_cfg = {
+	.name = "qhs_venus_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_vsense_ctrl_cfg = {
+	.name = "qhs_vsense_ctrl_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_qdss_stm = {
+	.name = "xs_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_sys_tcu_cfg = {
+	.name = "xs_sys_tcu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_aoss = {
+	.name = "qhs_aoss",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipa = {
+	.name = "qhs_ipa",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router = {
+	.name = "qhs_ipc_router",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_soccp = {
+	.name = "qhs_soccp",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tme_cfg = {
+	.name = "qhs_tme_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_apss = {
+	.name = "qss_apss",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_ddrss_cfg = {
+	.name = "qss_ddrss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qxs_boot_imem = {
+	.name = "qxs_boot_imem",
+	.channels = 1,
+	.buswidth = 16,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qxs_modem_boot_imem = {
+	.name = "qxs_modem_boot_imem",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node srvc_cnoc_main = {
+	.name = "srvc_cnoc_main",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_pcie_0 = {
+	.name = "xs_pcie_0",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie_1 = {
+	.name = "xs_pcie_1",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node ebi = {
+	.name = "ebi",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_mnoc_sf = {
+	.name = "srvc_mnoc_sf",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_mnoc_hf = {
+	.name = "srvc_mnoc_hf",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_pcie_aggre_noc = {
+	.name = "srvc_pcie_aggre_noc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup1_core_master = {
+	.name = "qup1_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_node qup2_core_master = {
+	.name = "qup2_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_node qnm_gemnoc_pcie = {
+	.name = "qnm_gemnoc_pcie",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 2,
+	.link_nodes = { &xs_pcie_0, &xs_pcie_1 },
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.name = "llcc_mc",
+	.channels = 4,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ebi },
+};
+
+static struct qcom_icc_node qsm_sf_mnoc_cfg = {
+	.name = "qsm_sf_mnoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_mnoc_sf },
+};
+
+static struct qcom_icc_node qsm_hf_mnoc_cfg = {
+	.name = "qsm_hf_mnoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_mnoc_hf },
+};
+
+static struct qcom_icc_node qsm_pcie_anoc_cfg = {
+	.name = "qsm_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_pcie_aggre_noc },
+};
+
+static struct qcom_icc_node qss_mnoc_hf_cfg = {
+	.name = "qss_mnoc_hf_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_hf_mnoc_cfg },
+};
+
+static struct qcom_icc_node qss_mnoc_sf_cfg = {
+	.name = "qss_mnoc_sf_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_sf_mnoc_cfg },
+};
+
+static struct qcom_icc_node qss_pcie_anoc_cfg = {
+	.name = "qss_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_pcie_anoc_cfg },
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.name = "qns_llcc",
+	.channels = 2,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &llcc_mc },
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_pcie },
+};
+
+static struct qcom_icc_node qsm_cfg = {
+	.name = "qsm_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 29,
+	.link_nodes = { &qhs_ahb2phy0, &qhs_ahb2phy1,
+			&qhs_camera_cfg, &qhs_clk_ctl,
+			&qhs_crypto0_cfg, &qhs_display_cfg,
+			&qhs_gpuss_cfg, &qhs_i3c_ibi0_cfg,
+			&qhs_i3c_ibi1_cfg, &qhs_imem_cfg,
+			&qhs_mss_cfg, &qhs_pcie_0_cfg,
+			&qhs_prng, &qhs_qdss_cfg,
+			&qhs_qspi, &qhs_qup1,
+			&qhs_qup2, &qhs_sdc2,
+			&qhs_tcsr, &qhs_tlmm,
+			&qhs_ufs_mem_cfg, &qhs_usb3_0,
+			&qhs_venus_cfg, &qhs_vsense_ctrl_cfg,
+			&qss_mnoc_hf_cfg, &qss_mnoc_sf_cfg,
+			&qss_pcie_anoc_cfg, &xs_qdss_stm,
+			&xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_node xm_gic = {
+	.name = "xm_gic",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15d000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qss_cfg = {
+	.name = "qss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg },
+};
+
+static struct qcom_icc_node qnm_gemnoc_cnoc = {
+	.name = "qnm_gemnoc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 12,
+	.link_nodes = { &qhs_aoss, &qhs_ipa,
+			&qhs_ipc_router, &qhs_soccp,
+			&qhs_tme_cfg, &qss_apss,
+			&qss_cfg, &qss_ddrss_cfg,
+			&qxs_boot_imem, &qxs_imem,
+			&qxs_modem_boot_imem, &srvc_cnoc_main },
+};
+
+static struct qcom_icc_node qns_gem_noc_cnoc = {
+	.name = "qns_gem_noc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_cnoc },
+};
+
+static struct qcom_icc_qosbox alm_gpu_tcu_qos = {
+	.num_ports = 1,
+	.port_offsets = { 0x155000 },
+	.prio = 1,
+	.urg_fwd = 0,
+	.prio_fwd_disable = 1,
+};
+
+static struct qcom_icc_node alm_gpu_tcu = {
+	.name = "alm_gpu_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &alm_gpu_tcu_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_qosbox alm_sys_tcu_qos = {
+	.num_ports = 1,
+	.port_offsets = { 0x157000 },
+	.prio = 6,
+	.urg_fwd = 0,
+	.prio_fwd_disable = 1,
+};
+
+static struct qcom_icc_node alm_sys_tcu = {
+	.name = "alm_sys_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &alm_sys_tcu_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node chm_apps = {
+	.name = "chm_apps",
+	.channels = 3,
+	.buswidth = 32,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_qosbox qnm_gpu_qos = {
+	.num_ports = 2,
+	.port_offsets = { 0x31000, 0xb1000 },
+	.prio = 0,
+	.urg_fwd = 1,
+	.prio_fwd_disable = 1,
+};
+
+static struct qcom_icc_node qnm_gpu = {
+	.name = "qnm_gpu",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &qnm_gpu_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
+	.num_ports = 1,
+	.port_offsets = { 0x159000 },
+	.prio = 0,
+	.urg_fwd = 1,
+	.prio_fwd_disable = 0,
+};
+
+static struct qcom_icc_node qnm_lpass_gemnoc = {
+	.name = "qnm_lpass_gemnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &qnm_lpass_gemnoc_qos,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mdsp = {
+	.name = "qnm_mdsp",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
+	.num_ports = 2,
+	.port_offsets = { 0x33000, 0xb3000 },
+	.prio = 0,
+	.urg_fwd = 1,
+	.prio_fwd_disable = 0,
+};
+
+static struct qcom_icc_node qnm_mnoc_hf = {
+	.name = "qnm_mnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &qnm_mnoc_hf_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
+	.num_ports = 2,
+	.port_offsets = { 0x35000, 0xb5000 },
+	.prio = 0,
+	.urg_fwd = 0,
+	.prio_fwd_disable = 0,
+};
+
+static struct qcom_icc_node qnm_mnoc_sf = {
+	.name = "qnm_mnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &qnm_mnoc_sf_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
+	.num_ports = 2,
+	.port_offsets = { 0x37000, 0xb7000 },
+	.prio = 0,
+	.urg_fwd = 1,
+	.prio_fwd_disable = 1,
+};
+
+static struct qcom_icc_node qnm_nsp_gemnoc = {
+	.name = "qnm_nsp_gemnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &qnm_nsp_gemnoc_qos,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_qosbox qnm_pcie_qos = {
+	.num_ports = 1,
+	.port_offsets = { 0x15b000 },
+	.prio = 2,
+	.urg_fwd = 1,
+	.prio_fwd_disable = 0,
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.name = "qnm_pcie",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &qnm_pcie_qos,
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.name = "qnm_snoc_sf",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15f000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qxm_wlan_q6 = {
+	.name = "qxm_wlan_q6",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qns_lpass_ag_noc_gemnoc = {
+	.name = "qns_lpass_ag_noc_gemnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_gemnoc },
+};
+
+static struct qcom_icc_node qns_mem_noc_sf = {
+	.name = "qns_mem_noc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_sf },
+};
+
+static struct qcom_icc_node qns_mem_noc_hf = {
+	.name = "qns_mem_noc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_hf },
+};
+
+static struct qcom_icc_node qns_nsp_gemnoc = {
+	.name = "qns_nsp_gemnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp_gemnoc },
+};
+
+static struct qcom_icc_node qns_pcie_mem_noc = {
+	.name = "qns_pcie_mem_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie },
+};
+
+static struct qcom_icc_node qns_gemnoc_sf = {
+	.name = "qns_gemnoc_sf",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_snoc_sf },
+};
+
+static struct qcom_icc_node qnm_lpiaon_noc = {
+	.name = "qnm_lpiaon_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_lpass_ag_noc_gemnoc },
+};
+
+static struct qcom_icc_qosbox qnm_camnoc_nrt_icp_sf_qos = {
+	.num_ports = 1,
+	.port_offsets = { 0x25000 },
+	.prio = 4,
+	.urg_fwd = 0,
+	.prio_fwd_disable = 1,
+};
+
+static struct qcom_icc_node qnm_camnoc_nrt_icp_sf = {
+	.name = "qnm_camnoc_nrt_icp_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &qnm_camnoc_nrt_icp_sf_qos,
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_rt_cdm_sf = {
+	.name = "qnm_camnoc_rt_cdm_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_sf = {
+	.name = "qnm_camnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x26000, 0x27000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_mvp = {
+	.name = "qnm_video_mvp",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x28000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_v_cpu = {
+	.name = "qnm_video_v_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2b000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_hf = {
+	.name = "qnm_camnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x64000, 0x65000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_mdp = {
+	.name = "qnm_mdp",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x66000, 0x67000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qxm_nsp = {
+	.name = "qxm_nsp",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_node xm_pcie3_0 = {
+	.name = "xm_pcie3_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_mem_noc },
+};
+
+static struct qcom_icc_node xm_pcie3_1 = {
+	.name = "xm_pcie3_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_mem_noc },
+};
+
+static struct qcom_icc_node qnm_aggre1_noc = {
+	.name = "qnm_aggre1_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_aggre2_noc = {
+	.name = "qnm_aggre2_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_cnoc_data = {
+	.name = "qnm_cnoc_data",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1d000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_nsinoc_snoc = {
+	.name = "qnm_nsinoc_snoc",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qns_a1noc_snoc = {
+	.name = "qns_a1noc_snoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre1_noc },
+};
+
+static struct qcom_icc_node qns_a2noc_snoc = {
+	.name = "qns_a2noc_snoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre2_noc },
+};
+
+static struct qcom_icc_node qns_lpass_aggnoc = {
+	.name = "qns_lpass_aggnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpiaon_noc },
+};
+
+static struct qcom_icc_node qhm_qspi = {
+	.name = "qhm_qspi",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup1 = {
+	.name = "qhm_qup1",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_ufs_mem = {
+	.name = "xm_ufs_mem",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_usb3_0 = {
+	.name = "xm_usb3_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x10000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup2 = {
+	.name = "qhm_qup2",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node qxm_crypto = {
+	.name = "qxm_crypto",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node qxm_ipa = {
+	.name = "qxm_ipa",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node qxm_soccp = {
+	.name = "qxm_soccp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_0 = {
+	.name = "xm_qdss_etr_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_1 = {
+	.name = "xm_qdss_etr_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x18000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc1 = {
+	.name = "xm_sdc1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc2 = {
+	.name = "xm_sdc2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x19000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node qnm_lpass_lpinoc = {
+	.name = "qnm_lpass_lpinoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_node qns_lpi_aon_noc = {
+	.name = "qns_lpi_aon_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_lpinoc },
+};
+
+static struct qcom_icc_node qxm_lpinoc_dsp_axim = {
+	.name = "qxm_lpinoc_dsp_axim",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_lpi_aon_noc },
+};
+
+static struct qcom_icc_bcm bcm_ce0 = {
+	.name = "CE0",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto },
+};
+
+static struct qcom_icc_bcm bcm_cn0 = {
+	.name = "CN0",
+	.enable_mask = BIT(0),
+	.keepalive = true,
+	.num_nodes = 43,
+	.nodes = { &qsm_cfg, &qhs_ahb2phy0,
+		   &qhs_ahb2phy1, &qhs_camera_cfg,
+		   &qhs_clk_ctl, &qhs_crypto0_cfg,
+		   &qhs_gpuss_cfg, &qhs_i3c_ibi0_cfg,
+		   &qhs_i3c_ibi1_cfg, &qhs_imem_cfg,
+		   &qhs_mss_cfg, &qhs_pcie_0_cfg,
+		   &qhs_prng, &qhs_qdss_cfg,
+		   &qhs_qspi, &qhs_sdc2,
+		   &qhs_tcsr, &qhs_tlmm,
+		   &qhs_ufs_mem_cfg, &qhs_usb3_0,
+		   &qhs_venus_cfg, &qhs_vsense_ctrl_cfg,
+		   &qss_mnoc_hf_cfg, &qss_mnoc_sf_cfg,
+		   &qss_pcie_anoc_cfg, &xs_qdss_stm,
+		   &xs_sys_tcu_cfg, &qnm_gemnoc_cnoc,
+		   &qnm_gemnoc_pcie, &qhs_aoss,
+		   &qhs_ipa, &qhs_ipc_router,
+		   &qhs_soccp, &qhs_tme_cfg,
+		   &qss_apss, &qss_cfg,
+		   &qss_ddrss_cfg, &qxs_boot_imem,
+		   &qxs_imem, &qxs_modem_boot_imem,
+		   &srvc_cnoc_main, &xs_pcie_0,
+		   &xs_pcie_1 },
+};
+
+static struct qcom_icc_bcm bcm_cn1 = {
+	.name = "CN1",
+	.num_nodes = 3,
+	.nodes = { &qhs_display_cfg, &qhs_qup1,
+			   &qhs_qup2 },
+};
+
+static struct qcom_icc_bcm bcm_co0 = {
+	.name = "CO0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qxm_nsp, &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_bcm bcm_lp0 = {
+	.name = "LP0",
+	.num_nodes = 2,
+	.nodes = { &qnm_lpass_lpinoc, &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_bcm bcm_mc0 = {
+	.name = "MC0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_mm0 = {
+	.name = "MM0",
+	.num_nodes = 1,
+	.nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_bcm bcm_mm1 = {
+	.name = "MM1",
+	.enable_mask = BIT(0),
+	.num_nodes = 7,
+	.nodes = { &qnm_camnoc_nrt_icp_sf, &qnm_camnoc_rt_cdm_sf,
+		   &qnm_camnoc_sf, &qnm_video_mvp,
+		   &qnm_video_v_cpu, &qnm_camnoc_hf,
+		   &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_bcm bcm_qup1 = {
+	.name = "QUP1",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup2 = {
+	.name = "QUP2",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_sh0 = {
+	.name = "SH0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_bcm bcm_sh1 = {
+	.name = "SH1",
+	.enable_mask = BIT(0),
+	.num_nodes = 14,
+	.nodes = { &alm_gpu_tcu, &alm_sys_tcu,
+		   &chm_apps, &qnm_gpu,
+		   &qnm_mdsp, &qnm_mnoc_hf,
+		   &qnm_mnoc_sf, &qnm_nsp_gemnoc,
+		   &qnm_pcie, &qnm_snoc_sf,
+		   &qxm_wlan_q6, &xm_gic,
+		   &qns_gem_noc_cnoc, &qns_pcie },
+};
+
+static struct qcom_icc_bcm bcm_sn0 = {
+	.name = "SN0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_bcm bcm_sn2 = {
+	.name = "SN2",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre1_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn3 = {
+	.name = "SN3",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre2_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn4 = {
+	.name = "SN4",
+	.num_nodes = 1,
+	.nodes = { &qns_pcie_mem_noc },
+};
+
+static struct qcom_icc_node * const aggre1_noc_nodes[] = {
+	[MASTER_QSPI_0] = &qhm_qspi,
+	[MASTER_QUP_1] = &qhm_qup1,
+	[MASTER_UFS_MEM] = &xm_ufs_mem,
+	[MASTER_USB3_0] = &xm_usb3_0,
+	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
+};
+
+static const struct qcom_icc_desc eliza_aggre1_noc = {
+	.nodes = aggre1_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
+	&bcm_ce0,
+};
+
+static struct qcom_icc_node * const aggre2_noc_nodes[] = {
+	[MASTER_QUP_2] = &qhm_qup2,
+	[MASTER_CRYPTO] = &qxm_crypto,
+	[MASTER_IPA] = &qxm_ipa,
+	[MASTER_SOCCP_AGGR_NOC] = &qxm_soccp,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr_0,
+	[MASTER_QDSS_ETR_1] = &xm_qdss_etr_1,
+	[MASTER_SDCC_1] = &xm_sdc1,
+	[MASTER_SDCC_2] = &xm_sdc2,
+	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
+};
+
+static const struct qcom_icc_desc eliza_aggre2_noc = {
+	.nodes = aggre2_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
+	.bcms = aggre2_noc_bcms,
+	.num_bcms = ARRAY_SIZE(aggre2_noc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const clk_virt_bcms[] = {
+	&bcm_qup1,
+	&bcm_qup2,
+};
+
+static struct qcom_icc_node * const clk_virt_nodes[] = {
+	[MASTER_QUP_CORE_1] = &qup1_core_master,
+	[MASTER_QUP_CORE_2] = &qup2_core_master,
+	[SLAVE_QUP_CORE_1] = &qup1_core_slave,
+	[SLAVE_QUP_CORE_2] = &qup2_core_slave,
+};
+
+static const struct qcom_icc_desc eliza_clk_virt = {
+	.nodes = clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(clk_virt_nodes),
+	.bcms = clk_virt_bcms,
+	.num_bcms = ARRAY_SIZE(clk_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_cfg_bcms[] = {
+	&bcm_cn0,
+	&bcm_cn1,
+};
+
+static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
+	[MASTER_CNOC_CFG] = &qsm_cfg,
+	[SLAVE_AHB2PHY_SOUTH] = &qhs_ahb2phy0,
+	[SLAVE_AHB2PHY_NORTH] = &qhs_ahb2phy1,
+	[SLAVE_CAMERA_CFG] = &qhs_camera_cfg,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg,
+	[SLAVE_DISPLAY_CFG] = &qhs_display_cfg,
+	[SLAVE_GFX3D_CFG] = &qhs_gpuss_cfg,
+	[SLAVE_I3C_IBI0_CFG] = &qhs_i3c_ibi0_cfg,
+	[SLAVE_I3C_IBI1_CFG] = &qhs_i3c_ibi1_cfg,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_CNOC_MSS] = &qhs_mss_cfg,
+	[SLAVE_PCIE_0_CFG] = &qhs_pcie_0_cfg,
+	[SLAVE_PRNG] = &qhs_prng,
+	[SLAVE_QDSS_CFG] = &qhs_qdss_cfg,
+	[SLAVE_QSPI_0] = &qhs_qspi,
+	[SLAVE_QUP_1] = &qhs_qup1,
+	[SLAVE_QUP_2] = &qhs_qup2,
+	[SLAVE_SDCC_2] = &qhs_sdc2,
+	[SLAVE_TCSR] = &qhs_tcsr,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_UFS_MEM_CFG] = &qhs_ufs_mem_cfg,
+	[SLAVE_USB3_0] = &qhs_usb3_0,
+	[SLAVE_VENUS_CFG] = &qhs_venus_cfg,
+	[SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg,
+	[SLAVE_CNOC_MNOC_HF_CFG] = &qss_mnoc_hf_cfg,
+	[SLAVE_CNOC_MNOC_SF_CFG] = &qss_mnoc_sf_cfg,
+	[SLAVE_PCIE_ANOC_CFG] = &qss_pcie_anoc_cfg,
+	[SLAVE_QDSS_STM] = &xs_qdss_stm,
+	[SLAVE_TCU] = &xs_sys_tcu_cfg,
+};
+
+static const struct qcom_icc_desc eliza_cnoc_cfg = {
+	.nodes = cnoc_cfg_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_cfg_nodes),
+	.bcms = cnoc_cfg_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_cfg_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_main_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const cnoc_main_nodes[] = {
+	[MASTER_GEM_NOC_CNOC] = &qnm_gemnoc_cnoc,
+	[MASTER_GEM_NOC_PCIE_SNOC] = &qnm_gemnoc_pcie,
+	[SLAVE_AOSS] = &qhs_aoss,
+	[SLAVE_IPA_CFG] = &qhs_ipa,
+	[SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router,
+	[SLAVE_SOCCP] = &qhs_soccp,
+	[SLAVE_TME_CFG] = &qhs_tme_cfg,
+	[SLAVE_APPSS] = &qss_apss,
+	[SLAVE_CNOC_CFG] = &qss_cfg,
+	[SLAVE_DDRSS_CFG] = &qss_ddrss_cfg,
+	[SLAVE_BOOT_IMEM] = &qxs_boot_imem,
+	[SLAVE_IMEM] = &qxs_imem,
+	[SLAVE_BOOT_IMEM_2] = &qxs_modem_boot_imem,
+	[SLAVE_SERVICE_CNOC] = &srvc_cnoc_main,
+	[SLAVE_PCIE_0] = &xs_pcie_0,
+	[SLAVE_PCIE_1] = &xs_pcie_1,
+};
+
+static const struct qcom_icc_desc eliza_cnoc_main = {
+	.nodes = cnoc_main_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
+	.bcms = cnoc_main_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_main_bcms),
+};
+
+static struct qcom_icc_bcm * const gem_noc_bcms[] = {
+	&bcm_sh0,
+	&bcm_sh1,
+};
+
+static struct qcom_icc_node * const gem_noc_nodes[] = {
+	[MASTER_GPU_TCU] = &alm_gpu_tcu,
+	[MASTER_SYS_TCU] = &alm_sys_tcu,
+	[MASTER_APPSS_PROC] = &chm_apps,
+	[MASTER_GFX3D] = &qnm_gpu,
+	[MASTER_LPASS_GEM_NOC] = &qnm_lpass_gemnoc,
+	[MASTER_MSS_PROC] = &qnm_mdsp,
+	[MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf,
+	[MASTER_MNOC_SF_MEM_NOC] = &qnm_mnoc_sf,
+	[MASTER_COMPUTE_NOC] = &qnm_nsp_gemnoc,
+	[MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_WLAN_Q6] = &qxm_wlan_q6,
+	[MASTER_GIC] = &xm_gic,
+	[SLAVE_GEM_NOC_CNOC] = &qns_gem_noc_cnoc,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+};
+
+static const struct qcom_icc_desc eliza_gem_noc = {
+	.nodes = gem_noc_nodes,
+	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
+	.bcms = gem_noc_bcms,
+	.num_bcms = ARRAY_SIZE(gem_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
+	[MASTER_LPIAON_NOC] = &qnm_lpiaon_noc,
+	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
+};
+
+static const struct qcom_icc_desc eliza_lpass_ag_noc = {
+	.nodes = lpass_ag_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const lpass_lpiaon_noc_bcms[] = {
+	&bcm_lp0,
+};
+
+static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
+	[MASTER_LPASS_LPINOC] = &qnm_lpass_lpinoc,
+	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
+};
+
+static const struct qcom_icc_desc eliza_lpass_lpiaon_noc = {
+	.nodes = lpass_lpiaon_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
+	.bcms = lpass_lpiaon_noc_bcms,
+	.num_bcms = ARRAY_SIZE(lpass_lpiaon_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
+	[MASTER_LPASS_PROC] = &qxm_lpinoc_dsp_axim,
+	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
+};
+
+static const struct qcom_icc_desc eliza_lpass_lpicx_noc = {
+	.nodes = lpass_lpicx_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const mc_virt_bcms[] = {
+	&bcm_mc0,
+};
+
+static struct qcom_icc_node * const mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[SLAVE_EBI1] = &ebi,
+};
+
+static const struct qcom_icc_desc eliza_mc_virt = {
+	.nodes = mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(mc_virt_nodes),
+	.bcms = mc_virt_bcms,
+	.num_bcms = ARRAY_SIZE(mc_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const mmss_noc_bcms[] = {
+	&bcm_mm0,
+	&bcm_mm1,
+};
+
+static struct qcom_icc_node * const mmss_noc_nodes[] = {
+	[MASTER_CAMNOC_NRT_ICP_SF] = &qnm_camnoc_nrt_icp_sf,
+	[MASTER_CAMNOC_RT_CDM_SF] = &qnm_camnoc_rt_cdm_sf,
+	[MASTER_CAMNOC_SF] = &qnm_camnoc_sf,
+	[MASTER_VIDEO_MVP] = &qnm_video_mvp,
+	[MASTER_VIDEO_V_PROC] = &qnm_video_v_cpu,
+	[MASTER_CNOC_MNOC_SF_CFG] = &qsm_sf_mnoc_cfg,
+	[MASTER_CAMNOC_HF] = &qnm_camnoc_hf,
+	[MASTER_MDP] = &qnm_mdp,
+	[MASTER_CNOC_MNOC_HF_CFG] = &qsm_hf_mnoc_cfg,
+	[SLAVE_MNOC_SF_MEM_NOC] = &qns_mem_noc_sf,
+	[SLAVE_SERVICE_MNOC_SF] = &srvc_mnoc_sf,
+	[SLAVE_MNOC_HF_MEM_NOC] = &qns_mem_noc_hf,
+	[SLAVE_SERVICE_MNOC_HF] = &srvc_mnoc_hf,
+};
+
+static const struct qcom_icc_desc eliza_mmss_noc = {
+	.nodes = mmss_noc_nodes,
+	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
+	.bcms = mmss_noc_bcms,
+	.num_bcms = ARRAY_SIZE(mmss_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_noc_bcms[] = {
+	&bcm_co0,
+};
+
+static struct qcom_icc_node * const nsp_noc_nodes[] = {
+	[MASTER_CDSP_PROC] = &qxm_nsp,
+	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
+};
+
+static const struct qcom_icc_desc eliza_nsp_noc = {
+	.nodes = nsp_noc_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
+	.bcms = nsp_noc_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const pcie_anoc_bcms[] = {
+	&bcm_sn4,
+};
+
+static struct qcom_icc_node * const pcie_anoc_nodes[] = {
+	[MASTER_PCIE_ANOC_CFG] = &qsm_pcie_anoc_cfg,
+	[MASTER_PCIE_0] = &xm_pcie3_0,
+	[MASTER_PCIE_1] = &xm_pcie3_1,
+	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc,
+	[SLAVE_SERVICE_PCIE_ANOC] = &srvc_pcie_aggre_noc,
+};
+
+static const struct qcom_icc_desc eliza_pcie_anoc = {
+	.nodes = pcie_anoc_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_anoc_nodes),
+	.bcms = pcie_anoc_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_anoc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const system_noc_bcms[] = {
+	&bcm_sn0,
+	&bcm_sn2,
+	&bcm_sn3,
+};
+
+static struct qcom_icc_node * const system_noc_nodes[] = {
+	[MASTER_A1NOC_SNOC] = &qnm_aggre1_noc,
+	[MASTER_A2NOC_SNOC] = &qnm_aggre2_noc,
+	[MASTER_CNOC_SNOC] = &qnm_cnoc_data,
+	[MASTER_NSINOC_SNOC] = &qnm_nsinoc_snoc,
+	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
+};
+
+static const struct qcom_icc_desc eliza_system_noc = {
+	.nodes = system_noc_nodes,
+	.num_nodes = ARRAY_SIZE(system_noc_nodes),
+	.bcms = system_noc_bcms,
+	.num_bcms = ARRAY_SIZE(system_noc_bcms),
+};
+
+static const struct of_device_id qnoc_of_match[] = {
+	{ .compatible = "qcom,eliza-aggre1-noc", .data = &eliza_aggre1_noc },
+	{ .compatible = "qcom,eliza-aggre2-noc", .data = &eliza_aggre2_noc },
+	{ .compatible = "qcom,eliza-clk-virt", .data = &eliza_clk_virt },
+	{ .compatible = "qcom,eliza-cnoc-cfg", .data = &eliza_cnoc_cfg },
+	{ .compatible = "qcom,eliza-cnoc-main", .data = &eliza_cnoc_main },
+	{ .compatible = "qcom,eliza-gem-noc", .data = &eliza_gem_noc },
+	{ .compatible = "qcom,eliza-lpass-ag-noc", .data = &eliza_lpass_ag_noc },
+	{ .compatible = "qcom,eliza-lpass-lpiaon-noc", .data = &eliza_lpass_lpiaon_noc },
+	{ .compatible = "qcom,eliza-lpass-lpicx-noc", .data = &eliza_lpass_lpicx_noc },
+	{ .compatible = "qcom,eliza-mc-virt", .data = &eliza_mc_virt },
+	{ .compatible = "qcom,eliza-mmss-noc", .data = &eliza_mmss_noc },
+	{ .compatible = "qcom,eliza-nsp-noc", .data = &eliza_nsp_noc },
+	{ .compatible = "qcom,eliza-pcie-anoc", .data = &eliza_pcie_anoc },
+	{ .compatible = "qcom,eliza-system-noc", .data = &eliza_system_noc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qnoc_of_match);
+
+static struct platform_driver qnoc_driver = {
+	.probe = qcom_icc_rpmh_probe,
+	.remove = qcom_icc_rpmh_remove,
+	.driver = {
+		.name = "qnoc-eliza",
+		.of_match_table = qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION(" Qualcomm Eliza NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/glymur.c b/drivers/interconnect/qcom/glymur.c
index e5c07795a6c67..9b9ed99ef242e 100644
--- a/drivers/interconnect/qcom/glymur.c
+++ b/drivers/interconnect/qcom/glymur.c
@@ -9,6 +9,7 @@
 #include <linux/interconnect-provider.h>
 #include <linux/module.h>
 #include <linux/of_platform.h>
+#include <linux/property.h>
 #include <dt-bindings/interconnect/qcom,glymur-rpmh.h>
 
 #include "bcm-voter.h"
@@ -1985,7 +1986,7 @@ static struct qcom_icc_bcm * const cnoc_cfg_bcms[] = {
 	&bcm_cn1,
 };
 
-static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
+static struct qcom_icc_node *cnoc_cfg_nodes[] = {
 	[MASTER_CNOC_CFG] = &qsm_cfg,
 	[SLAVE_AHB2PHY_SOUTH] = &qhs_ahb2phy0,
 	[SLAVE_AHB2PHY_NORTH] = &qhs_ahb2phy1,
@@ -2093,7 +2094,7 @@ static struct qcom_icc_bcm * const hscnoc_bcms[] = {
 	&bcm_sh1,
 };
 
-static struct qcom_icc_node * const hscnoc_nodes[] = {
+static struct qcom_icc_node *hscnoc_nodes[] = {
 	[MASTER_GPU_TCU] = &alm_gpu_tcu,
 	[MASTER_PCIE_TCU] = &alm_pcie_qtc,
 	[MASTER_SYS_TCU] = &alm_sys_tcu,
@@ -2377,7 +2378,7 @@ static struct qcom_icc_bcm * const pcie_west_anoc_bcms[] = {
 	&bcm_sn6,
 };
 
-static struct qcom_icc_node * const pcie_west_anoc_nodes[] = {
+static struct qcom_icc_node *pcie_west_anoc_nodes[] = {
 	[MASTER_PCIE_WEST_ANOC_CFG] = &qsm_pcie_west_anoc_cfg,
 	[MASTER_PCIE_2] = &xm_pcie_2,
 	[MASTER_PCIE_3A] = &xm_pcie_3a,
@@ -2409,7 +2410,7 @@ static struct qcom_icc_bcm * const pcie_west_slv_noc_bcms[] = {
 	&bcm_sn6,
 };
 
-static struct qcom_icc_node * const pcie_west_slv_noc_nodes[] = {
+static struct qcom_icc_node *pcie_west_slv_noc_nodes[] = {
 	[MASTER_HSCNOC_PCIE_WEST] = &qnm_hscnoc_pcie_west,
 	[MASTER_CNOC_PCIE_WEST_SLAVE_CFG] = &qsm_cnoc_pcie_west_slave_cfg,
 	[SLAVE_HSCNOC_PCIE_WEST_MS_MPU_CFG] = &qhs_hscnoc_pcie_west_ms_mpu_cfg,
@@ -2470,6 +2471,28 @@ static const struct qcom_icc_desc glymur_system_noc = {
 	.num_bcms = ARRAY_SIZE(system_noc_bcms),
 };
 
+static int glymur_qnoc_probe(struct platform_device *pdev)
+{
+	if (device_is_compatible(&pdev->dev, "qcom,mahua-mc-virt")) {
+		llcc_mc.channels = 8;
+		ebi.channels = 8;
+	} else if (device_is_compatible(&pdev->dev, "qcom,mahua-hscnoc")) {
+		qns_llcc.channels = 8;
+		chm_apps.channels = 4;
+		qnm_pcie_west.buswidth = 32;
+		hscnoc_nodes[MASTER_WLAN_Q6] = NULL;
+	} else if (device_is_compatible(&pdev->dev, "qcom,mahua-pcie-west-anoc")) {
+		qns_pcie_west_mem_noc.buswidth = 32;
+		pcie_west_anoc_nodes[MASTER_PCIE_3A] = NULL;
+	} else if (device_is_compatible(&pdev->dev, "qcom,mahua-cnoc-cfg")) {
+		cnoc_cfg_nodes[SLAVE_PCIE_3A_CFG] = NULL;
+	} else if (device_is_compatible(&pdev->dev, "qcom,mahua-pcie-west-slv-noc")) {
+		pcie_west_slv_noc_nodes[SLAVE_PCIE_3A] = NULL;
+	}
+
+	return qcom_icc_rpmh_probe(pdev);
+}
+
 static const struct of_device_id qnoc_of_match[] = {
 	{ .compatible = "qcom,glymur-aggre1-noc", .data = &glymur_aggre1_noc},
 	{ .compatible = "qcom,glymur-aggre2-noc", .data = &glymur_aggre2_noc},
@@ -2477,12 +2500,15 @@ static const struct of_device_id qnoc_of_match[] = {
 	{ .compatible = "qcom,glymur-aggre4-noc", .data = &glymur_aggre4_noc},
 	{ .compatible = "qcom,glymur-clk-virt", .data = &glymur_clk_virt},
 	{ .compatible = "qcom,glymur-cnoc-cfg", .data = &glymur_cnoc_cfg},
+	{ .compatible = "qcom,mahua-cnoc-cfg", .data = &glymur_cnoc_cfg},
 	{ .compatible = "qcom,glymur-cnoc-main", .data = &glymur_cnoc_main},
 	{ .compatible = "qcom,glymur-hscnoc", .data = &glymur_hscnoc},
+	{ .compatible = "qcom,mahua-hscnoc", .data = &glymur_hscnoc},
 	{ .compatible = "qcom,glymur-lpass-ag-noc", .data = &glymur_lpass_ag_noc},
 	{ .compatible = "qcom,glymur-lpass-lpiaon-noc", .data = &glymur_lpass_lpiaon_noc},
 	{ .compatible = "qcom,glymur-lpass-lpicx-noc", .data = &glymur_lpass_lpicx_noc},
 	{ .compatible = "qcom,glymur-mc-virt", .data = &glymur_mc_virt},
+	{ .compatible = "qcom,mahua-mc-virt", .data = &glymur_mc_virt},
 	{ .compatible = "qcom,glymur-mmss-noc", .data = &glymur_mmss_noc},
 	{ .compatible = "qcom,glymur-nsinoc", .data = &glymur_nsinoc},
 	{ .compatible = "qcom,glymur-nsp-noc", .data = &glymur_nsp_noc},
@@ -2490,14 +2516,16 @@ static const struct of_device_id qnoc_of_match[] = {
 	{ .compatible = "qcom,glymur-pcie-east-anoc", .data = &glymur_pcie_east_anoc},
 	{ .compatible = "qcom,glymur-pcie-east-slv-noc", .data = &glymur_pcie_east_slv_noc},
 	{ .compatible = "qcom,glymur-pcie-west-anoc", .data = &glymur_pcie_west_anoc},
+	{ .compatible = "qcom,mahua-pcie-west-anoc", .data = &glymur_pcie_west_anoc},
 	{ .compatible = "qcom,glymur-pcie-west-slv-noc", .data = &glymur_pcie_west_slv_noc},
+	{ .compatible = "qcom,mahua-pcie-west-slv-noc", .data = &glymur_pcie_west_slv_noc},
 	{ .compatible = "qcom,glymur-system-noc", .data = &glymur_system_noc},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qnoc_of_match);
 
 static struct platform_driver qnoc_driver = {
-	.probe = qcom_icc_rpmh_probe,
+	.probe = glymur_qnoc_probe,
 	.remove = qcom_icc_rpmh_remove,
 	.driver = {
 		.name = "qnoc-glymur",
@@ -2518,5 +2546,5 @@ static void __exit qnoc_driver_exit(void)
 }
 module_exit(qnoc_driver_exit);
 
-MODULE_DESCRIPTION("GLYMUR NoC driver");
+MODULE_DESCRIPTION("Glymur NoC driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/qcs615.c b/drivers/interconnect/qcom/qcs615.c
index 797956eb6ff54..017a6017421ff 100644
--- a/drivers/interconnect/qcom/qcs615.c
+++ b/drivers/interconnect/qcom/qcs615.c
@@ -142,6 +142,12 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.name = "qhm_qdss_bam",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -150,6 +156,12 @@ static struct qcom_icc_node qhm_qspi = {
 	.name = "qhm_qspi",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -158,6 +170,12 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x10000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -166,6 +184,12 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -174,6 +198,12 @@ static struct qcom_icc_node qnm_cnoc = {
 	.name = "qnm_cnoc",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -182,6 +212,12 @@ static struct qcom_icc_node qxm_crypto = {
 	.name = "qxm_crypto",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x5000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -190,6 +226,12 @@ static struct qcom_icc_node qxm_ipa = {
 	.name = "qxm_ipa",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x6000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_lpass_snoc },
 };
@@ -198,6 +240,12 @@ static struct qcom_icc_node xm_emac_avb = {
 	.name = "xm_emac_avb",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -206,6 +254,12 @@ static struct qcom_icc_node xm_pcie = {
 	.name = "xm_pcie",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 0,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_snoc },
 };
@@ -214,6 +268,12 @@ static struct qcom_icc_node xm_qdss_etr = {
 	.name = "xm_qdss_etr",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -222,6 +282,12 @@ static struct qcom_icc_node xm_sdc1 = {
 	.name = "xm_sdc1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -230,6 +296,12 @@ static struct qcom_icc_node xm_sdc2 = {
 	.name = "xm_sdc2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -238,6 +310,12 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -246,6 +324,12 @@ static struct qcom_icc_node xm_usb2 = {
 	.name = "xm_usb2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -254,6 +338,12 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -356,6 +446,12 @@ static struct qcom_icc_node acm_apps = {
 	.name = "acm_apps",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2e000, 0x2e100 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_snoc, &qns_llcc,
 			&qns_sys_pcie },
@@ -365,6 +461,12 @@ static struct qcom_icc_node acm_gpu_tcu = {
 	.name = "acm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x36000 },
+		.prio = 6,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_snoc, &qns_llcc },
 };
@@ -373,6 +475,12 @@ static struct qcom_icc_node acm_sys_tcu = {
 	.name = "acm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x37000 },
+		.prio = 6,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_snoc, &qns_llcc },
 };
@@ -389,6 +497,12 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x34000, 0x34080 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_snoc, &qns_llcc },
 };
@@ -397,6 +511,12 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2f000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -405,6 +525,12 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x35000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_snoc, &qns_llcc },
 };
@@ -413,6 +539,12 @@ static struct qcom_icc_node qnm_snoc_gc = {
 	.name = "qnm_snoc_gc",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x31000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -421,6 +553,12 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x30000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -445,6 +583,12 @@ static struct qcom_icc_node qxm_camnoc_hf0 = {
 	.name = "qxm_camnoc_hf0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -453,6 +597,12 @@ static struct qcom_icc_node qxm_camnoc_hf1 = {
 	.name = "qxm_camnoc_hf1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -461,6 +611,12 @@ static struct qcom_icc_node qxm_camnoc_sf = {
 	.name = "qxm_camnoc_sf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns2_mem_noc },
 };
@@ -469,6 +625,12 @@ static struct qcom_icc_node qxm_mdp0 = {
 	.name = "qxm_mdp0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -477,6 +639,12 @@ static struct qcom_icc_node qxm_rot = {
 	.name = "qxm_rot",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns2_mem_noc },
 };
@@ -485,6 +653,12 @@ static struct qcom_icc_node qxm_venus0 = {
 	.name = "qxm_venus0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns2_mem_noc },
 };
@@ -493,6 +667,12 @@ static struct qcom_icc_node qxm_venus_arm9 = {
 	.name = "qxm_venus_arm9",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns2_mem_noc },
 };
@@ -559,6 +739,12 @@ static struct qcom_icc_node qxm_pimem = {
 	.name = "qxm_pimem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_memnoc_gc, &qxs_imem },
 };
@@ -567,6 +753,12 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_memnoc_gc, &qxs_imem },
 };
@@ -1213,11 +1405,21 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_SERVICE_A2NOC] = &srvc_aggre2_noc,
 };
 
+static const struct regmap_config qcs615_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3f200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_aggre1_noc = {
+	.config = &qcs615_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const camnoc_virt_bcms[] = {
@@ -1289,7 +1491,16 @@ static struct qcom_icc_node * const config_noc_nodes[] = {
 	[SLAVE_SERVICE_CNOC] = &srvc_cnoc,
 };
 
+static const struct regmap_config qcs615_config_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_config_noc = {
+	.config = &qcs615_config_noc_regmap_config,
 	.nodes = config_noc_nodes,
 	.num_nodes = ARRAY_SIZE(config_noc_nodes),
 	.bcms = config_noc_bcms,
@@ -1302,7 +1513,16 @@ static struct qcom_icc_node * const dc_noc_nodes[] = {
 	[SLAVE_LLCC_CFG] = &qhs_llcc,
 };
 
+static const struct regmap_config qcs615_dc_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_dc_noc = {
+	.config = &qcs615_dc_noc_regmap_config,
 	.nodes = dc_noc_nodes,
 	.num_nodes = ARRAY_SIZE(dc_noc_nodes),
 };
@@ -1331,7 +1551,16 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_SERVICE_GEM_NOC] = &srvc_gemnoc,
 };
 
+static const struct regmap_config qcs615_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3e200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_gem_noc = {
+	.config = &qcs615_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
@@ -1376,7 +1605,16 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC] = &srvc_mnoc,
 };
 
+static const struct regmap_config qcs615_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c100,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_mmss_noc = {
+	.config = &qcs615_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -1418,7 +1656,16 @@ static struct qcom_icc_node * const system_noc_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
+static const struct regmap_config qcs615_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1f300,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs615_system_noc = {
+	.config = &qcs615_system_noc_regmap_config,
 	.nodes = system_noc_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_nodes),
 	.bcms = system_noc_bcms,
diff --git a/drivers/interconnect/qcom/qcs8300.c b/drivers/interconnect/qcom/qcs8300.c
index bc403a9bf68c6..ebf1671825721 100644
--- a/drivers/interconnect/qcom/qcs8300.c
+++ b/drivers/interconnect/qcom/qcs8300.c
@@ -186,6 +186,13 @@ static struct qcom_icc_node qxm_qup3 = {
 	.name = "qxm_qup3",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -194,6 +201,13 @@ static struct qcom_icc_node xm_emac_0 = {
 	.name = "xm_emac_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -202,6 +216,13 @@ static struct qcom_icc_node xm_sdc1 = {
 	.name = "xm_sdc1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -210,6 +231,13 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -218,6 +246,13 @@ static struct qcom_icc_node xm_usb2_2 = {
 	.name = "xm_usb2_2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -226,6 +261,13 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -234,6 +276,13 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.name = "qhm_qdss_bam",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -242,6 +291,13 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -250,6 +306,13 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -258,6 +321,13 @@ static struct qcom_icc_node qnm_cnoc_datapath = {
 	.name = "qnm_cnoc_datapath",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -266,6 +336,13 @@ static struct qcom_icc_node qxm_crypto_0 = {
 	.name = "qxm_crypto_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x18000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -274,6 +351,13 @@ static struct qcom_icc_node qxm_crypto_1 = {
 	.name = "qxm_crypto_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -282,6 +366,13 @@ static struct qcom_icc_node qxm_ipa = {
 	.name = "qxm_ipa",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -290,6 +381,13 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.name = "xm_qdss_etr_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -298,6 +396,13 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.name = "xm_qdss_etr_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x19000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -390,6 +495,13 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.name = "alm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xaf000 },
+		.prio_fwd_disable = 1,
+		.prio = 1,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -398,6 +510,13 @@ static struct qcom_icc_node alm_pcie_tcu = {
 	.name = "alm_pcie_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb0000 },
+		.prio_fwd_disable = 1,
+		.prio = 3,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -406,6 +525,13 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.name = "alm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb1000 },
+		.prio_fwd_disable = 1,
+		.prio = 6,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -423,6 +549,13 @@ static struct qcom_icc_node qnm_cmpnoc0 = {
 	.name = "qnm_cmpnoc0",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0xf6000, 0xf7000 },
+		.prio_fwd_disable = 1,
+		.prio = 0,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -448,6 +581,13 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0xf0000, 0xf1000 },
+		.prio_fwd_disable = 1,
+		.prio = 0,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -456,6 +596,13 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0xf2000, 0xf3000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_llcc, &qns_pcie },
 };
@@ -464,6 +611,13 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0xf4000, 0xf5000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -473,6 +627,13 @@ static struct qcom_icc_node qnm_pcie = {
 	.name = "qnm_pcie",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb3000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -481,6 +642,13 @@ static struct qcom_icc_node qnm_snoc_gc = {
 	.name = "qnm_snoc_gc",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb4000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -489,6 +657,13 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb5000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -541,6 +716,13 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.name = "qnm_camnoc_hf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -549,6 +731,13 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.name = "qnm_camnoc_icp",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -557,6 +746,13 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.name = "qnm_camnoc_sf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a080 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -565,6 +761,13 @@ static struct qcom_icc_node qnm_mdp0_0 = {
 	.name = "qnm_mdp0_0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa080 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -573,6 +776,13 @@ static struct qcom_icc_node qnm_mdp0_1 = {
 	.name = "qnm_mdp0_1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa180 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -597,6 +807,13 @@ static struct qcom_icc_node qnm_video0 = {
 	.name = "qnm_video0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a100 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -605,6 +822,13 @@ static struct qcom_icc_node qnm_video_cvp = {
 	.name = "qnm_video_cvp",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a200 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -613,6 +837,13 @@ static struct qcom_icc_node qnm_video_v_cpu = {
 	.name = "qnm_video_v_cpu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a280 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -637,6 +868,13 @@ static struct qcom_icc_node xm_pcie3_0 = {
 	.name = "xm_pcie3_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_mem_noc },
 };
@@ -645,6 +883,13 @@ static struct qcom_icc_node xm_pcie3_1 = {
 	.name = "xm_pcie3_1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_mem_noc },
 };
@@ -653,6 +898,13 @@ static struct qcom_icc_node qhm_gic = {
 	.name = "qhm_gic",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -677,6 +929,13 @@ static struct qcom_icc_node qnm_lpass_noc = {
 	.name = "qnm_lpass_noc",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio_fwd_disable = 0,
+		.prio = 0,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -693,6 +952,13 @@ static struct qcom_icc_node qxm_pimem = {
 	.name = "qxm_pimem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_gc },
 };
@@ -701,6 +967,13 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio_fwd_disable = 1,
+		.prio = 2,
+		.urg_fwd = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_gc },
 };
@@ -1599,11 +1872,21 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
 };
 
+static const struct regmap_config qcs8300_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_aggre1_noc = {
+	.config = &qcs8300_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
@@ -1624,11 +1907,21 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = {
 	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
 };
 
+static const struct regmap_config qcs8300_aggre2_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1a080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_aggre2_noc = {
+	.config = &qcs8300_aggre2_noc_regmap_config,
 	.nodes = aggre2_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
 	.bcms = aggre2_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre2_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const clk_virt_bcms[] = {
@@ -1740,7 +2033,16 @@ static struct qcom_icc_node * const config_noc_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
+static const struct regmap_config qcs8300_config_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x13080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_config_noc = {
+	.config = &qcs8300_config_noc_regmap_config,
 	.nodes = config_noc_nodes,
 	.num_nodes = ARRAY_SIZE(config_noc_nodes),
 	.bcms = config_noc_bcms,
@@ -1753,7 +2055,16 @@ static struct qcom_icc_node * const dc_noc_nodes[] = {
 	[SLAVE_GEM_NOC_CFG] = &qns_gemnoc,
 };
 
+static const struct regmap_config qcs8300_dc_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_dc_noc = {
+	.config = &qcs8300_dc_noc_regmap_config,
 	.nodes = dc_noc_nodes,
 	.num_nodes = ARRAY_SIZE(dc_noc_nodes),
 };
@@ -1786,11 +2097,21 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_SERVICE_GEM_NOC2] = &srvc_sys_gemnoc_2,
 };
 
+static const struct regmap_config qcs8300_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xf7080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_gem_noc = {
+	.config = &qcs8300_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
 	.num_bcms = ARRAY_SIZE(gem_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const gpdsp_anoc_bcms[] = {
@@ -1803,7 +2124,16 @@ static struct qcom_icc_node * const gpdsp_anoc_nodes[] = {
 	[SLAVE_GP_DSP_SAIL_NOC] = &qns_gp_dsp_sail_noc,
 };
 
+static const struct regmap_config qcs8300_gpdsp_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xd080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_gpdsp_anoc = {
+	.config = &qcs8300_gpdsp_anoc_regmap_config,
 	.nodes = gpdsp_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(gpdsp_anoc_nodes),
 	.bcms = gpdsp_anoc_bcms,
@@ -1826,7 +2156,16 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
 	[SLAVE_SERVICE_LPASS_AG_NOC] = &srvc_niu_lpass_agnoc,
 };
 
+static const struct regmap_config qcs8300_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_lpass_ag_noc = {
+	.config = &qcs8300_lpass_ag_noc_regmap_config,
 	.nodes = lpass_ag_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
 	.bcms = lpass_ag_noc_bcms,
@@ -1872,7 +2211,16 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC_SF] = &srvc_mnoc_sf,
 };
 
+static const struct regmap_config qcs8300_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x40000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_mmss_noc = {
+	.config = &qcs8300_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -1892,7 +2240,16 @@ static struct qcom_icc_node * const nspa_noc_nodes[] = {
 	[SLAVE_SERVICE_NSP_NOC] = &service_nsp_noc,
 };
 
+static const struct regmap_config qcs8300_nspa_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x16080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_nspa_noc = {
+	.config = &qcs8300_nspa_noc_regmap_config,
 	.nodes = nspa_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nspa_noc_nodes),
 	.bcms = nspa_noc_bcms,
@@ -1909,7 +2266,16 @@ static struct qcom_icc_node * const pcie_anoc_nodes[] = {
 	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc,
 };
 
+static const struct regmap_config qcs8300_pcie_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xc080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_pcie_anoc = {
+	.config = &qcs8300_pcie_anoc_regmap_config,
 	.nodes = pcie_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_anoc_nodes),
 	.bcms = pcie_anoc_bcms,
@@ -1937,7 +2303,16 @@ static struct qcom_icc_node * const system_noc_nodes[] = {
 	[SLAVE_SERVICE_SNOC] = &srvc_snoc,
 };
 
+static const struct regmap_config qcs8300_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x15080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc qcs8300_system_noc = {
+	.config = &qcs8300_system_noc_regmap_config,
 	.nodes = system_noc_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_nodes),
 	.bcms = system_noc_bcms,
diff --git a/drivers/interconnect/qcom/sm8450.c b/drivers/interconnect/qcom/sm8450.c
index 669a638bf3efc..c88327d200acc 100644
--- a/drivers/interconnect/qcom/sm8450.c
+++ b/drivers/interconnect/qcom/sm8450.c
@@ -800,7 +800,7 @@ static struct qcom_icc_node qhs_compute_cfg = {
 	.channels = 1,
 	.buswidth = 4,
 	.num_links = 1,
-	.link_nodes = { MASTER_CDSP_NOC_CFG },
+	.link_nodes = { &qhm_nsp_noc_config },
 };
 
 static struct qcom_icc_node qhs_cpr_cx = {
@@ -874,7 +874,7 @@ static struct qcom_icc_node qhs_lpass_cfg = {
 	.channels = 1,
 	.buswidth = 4,
 	.num_links = 1,
-	.link_nodes = { MASTER_CNOC_LPASS_AG_NOC },
+	.link_nodes = { &qhm_config_noc },
 };
 
 static struct qcom_icc_node qhs_mss_cfg = {
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index f3fd7f39efb48..56ad020df4949 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -848,10 +848,11 @@ static void __init free_command_buffer(struct amd_iommu *iommu)
 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
 				  size_t size)
 {
+	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
 	void *buf;
 
 	size = PAGE_ALIGN(size);
-	buf = iommu_alloc_pages_sz(gfp, size);
+	buf = iommu_alloc_pages_node_sz(nid, gfp, size);
 	if (!buf)
 		return NULL;
 	if (check_feature(FEATURE_SNP) &&
@@ -954,14 +955,16 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
 
 static int iommu_init_ga_log(struct amd_iommu *iommu)
 {
+	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
+
 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
 		return 0;
 
-	iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE);
+	iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
 	if (!iommu->ga_log)
 		goto err_out;
 
-	iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8);
+	iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8);
 	if (!iommu->ga_log_tail)
 		goto err_out;
 
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 81c4d77338722..528ebdd26d71c 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2909,8 +2909,21 @@ static struct iommu_domain blocked_domain = {
 
 static struct protection_domain identity_domain;
 
+static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev,
+				     struct iommu_domain *old)
+{
+	/*
+	 * Don't allow attaching a device to the identity domain if SNP is
+	 * enabled.
+	 */
+	if (amd_iommu_snp_en)
+		return -EINVAL;
+
+	return amd_iommu_attach_device(dom, dev, old);
+}
+
 static const struct iommu_domain_ops identity_domain_ops = {
-	.attach_dev = amd_iommu_attach_device,
+	.attach_dev = amd_iommu_identity_attach,
 };
 
 void amd_iommu_init_identity_domain(void)
@@ -2985,6 +2998,12 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)
 
 		return amd_iommu_hd_support(iommu);
 	}
+	case IOMMU_CAP_PCI_ATS_SUPPORTED: {
+		struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+
+		return amd_iommu_iotlb_sup &&
+			 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP);
+	}
 	default:
 		break;
 	}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 59a480974d80f..f1f8e01a7e914 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -122,15 +122,6 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
 }
 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd);
 
-/*
- * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this
- * is used as a threshold to replace per-page TLBI commands to issue in the
- * command queue with an address-space TLBI command, when SMMU w/o a range
- * invalidation feature handles too many per-page TLBI commands, which will
- * otherwise result in a soft lockup.
- */
-#define CMDQ_MAX_TLBI_OPS		(1 << (PAGE_SHIFT - 3))
-
 static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
 						struct mm_struct *mm,
 						unsigned long start,
@@ -146,21 +137,8 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
 	 * range. So do a simple translation here by calculating size correctly.
 	 */
 	size = end - start;
-	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) {
-		if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE)
-			size = 0;
-	} else {
-		if (size == ULONG_MAX)
-			size = 0;
-	}
-
-	if (!size)
-		arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
-	else
-		arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid,
-					    PAGE_SIZE, false, smmu_domain);
 
-	arm_smmu_atc_inv_domain(smmu_domain, start, size);
+	arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false);
 }
 
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -191,13 +169,13 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	}
 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
-	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
-	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
+	arm_smmu_domain_inv(smmu_domain);
 }
 
 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
 {
-	kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier));
+	arm_smmu_domain_free(
+		container_of(mn, struct arm_smmu_domain, mmu_notifier));
 }
 
 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
@@ -301,7 +279,7 @@ static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
 	/*
 	 * Ensure the ASID is empty in the iommu cache before allowing reuse.
 	 */
-	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
+	arm_smmu_domain_inv(smmu_domain);
 
 	/*
 	 * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can
@@ -346,6 +324,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
 	 * ARM_SMMU_FEAT_RANGE_INV is present
 	 */
 	smmu_domain->domain.pgsize_bitmap = PAGE_SIZE;
+	smmu_domain->stage = ARM_SMMU_DOMAIN_SVA;
 	smmu_domain->smmu = smmu;
 
 	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
@@ -364,6 +343,6 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
 err_asid:
 	xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
 err_free:
-	kfree(smmu_domain);
+	arm_smmu_domain_free(smmu_domain);
 	return ERR_PTR(ret);
 }
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
index 69c9ef441fc1d..add671363c828 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
@@ -637,6 +637,140 @@ static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test)
 						      NUM_EXPECTED_SYNCS(2));
 }
 
+static void arm_smmu_v3_invs_test_verify(struct kunit *test,
+					 struct arm_smmu_invs *invs,
+					 int num_invs, const int num_trashes,
+					 const int *ids, const int *users,
+					 const int *ssids)
+{
+	KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs);
+	KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes);
+	while (num_invs--) {
+		KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]);
+		KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users),
+				users[num_invs]);
+		KUNIT_EXPECT_EQ(test, invs->inv[num_invs].ssid, ssids[num_invs]);
+	}
+}
+
+static struct arm_smmu_invs invs1 = {
+	.num_invs = 3,
+	.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, },
+		 { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, },
+		 { .type = INV_TYPE_ATS, .id = 3, }, },
+};
+
+static struct arm_smmu_invs invs2 = {
+	.num_invs = 3,
+	.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
+		 { .type = INV_TYPE_ATS, .id = 4, },
+		 { .type = INV_TYPE_ATS, .id = 5, }, },
+};
+
+static struct arm_smmu_invs invs3 = {
+	.num_invs = 3,
+	.inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
+		 { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */
+		 { .type = INV_TYPE_ATS, .id = 6, }, },
+};
+
+static struct arm_smmu_invs invs4 = {
+	.num_invs = 3,
+	.inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 },
+		 { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 },
+		 { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, },
+};
+
+static struct arm_smmu_invs invs5 = {
+	.num_invs = 3,
+	.inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 },
+		 { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */
+		 { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, },
+};
+
+static void arm_smmu_v3_invs_test(struct kunit *test)
+{
+	const int results1[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } };
+	const int results2[3][5] = { { 1, 1, 3, 4, 5, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } };
+	const int results3[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } };
+	const int results4[3][5] = { { 1, 1, 3, 5, 6, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } };
+	const int results5[3][5] = { { 1, 1, 3, 5, 6, }, { 1, 0, 0, 1, 1, }, { 0, 0, 0, 0, 0, } };
+	const int results6[3][3] = { { 1, 5, 6, }, { 1, 1, 1, }, { 0, 0, 0, } };
+	const int results7[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } };
+	const int results8[3][5] = { { 10, 10, 10, 12, 12, }, { 1, 1, 2, 1, 1, }, { 1, 2, 3, 1, 2, } };
+	const int results9[3][4] = { { 10, 10, 10, 12, }, { 1, 0, 1, 1, }, { 1, 2, 3, 1, } };
+	const int results10[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } };
+	struct arm_smmu_invs *test_a, *test_b;
+
+	/* New array */
+	test_a = arm_smmu_invs_alloc(0);
+	KUNIT_EXPECT_EQ(test, test_a->num_invs, 0);
+
+	/* Test1: merge invs1 (new array) */
+	test_b = arm_smmu_invs_merge(test_a, &invs1);
+	kfree(test_a);
+	arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0,
+				     results1[0], results1[1], results1[2]);
+
+	/* Test2: merge invs2 (new array) */
+	test_a = arm_smmu_invs_merge(test_b, &invs2);
+	kfree(test_b);
+	arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0,
+				     results2[0], results2[1], results2[2]);
+
+	/* Test3: unref invs2 (same array) */
+	arm_smmu_invs_unref(test_a, &invs2);
+	arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results3[0]), 0,
+				     results3[0], results3[1], results3[2]);
+
+	/* Test4: merge invs3 (new array) */
+	test_b = arm_smmu_invs_merge(test_a, &invs3);
+	kfree(test_a);
+	arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0,
+				     results4[0], results4[1], results4[2]);
+
+	/* Test5: unref invs1 (same array) */
+	arm_smmu_invs_unref(test_b, &invs1);
+	arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results5[0]), 2,
+				     results5[0], results5[1], results5[2]);
+
+	/* Test6: purge test_b (new array) */
+	test_a = arm_smmu_invs_purge(test_b);
+	kfree(test_b);
+	arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0,
+				     results6[0], results6[1], results6[2]);
+
+	/* Test7: unref invs3 (same array) */
+	arm_smmu_invs_unref(test_a, &invs3);
+	KUNIT_EXPECT_EQ(test, test_a->num_invs, 0);
+	KUNIT_EXPECT_EQ(test, test_a->num_trashes, 0);
+
+	/* Test8: merge invs4 (new array) */
+	test_b = arm_smmu_invs_merge(test_a, &invs4);
+	kfree(test_a);
+	arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0,
+				     results7[0], results7[1], results7[2]);
+
+	/* Test9: merge invs5 (new array) */
+	test_a = arm_smmu_invs_merge(test_b, &invs5);
+	kfree(test_b);
+	arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0,
+				     results8[0], results8[1], results8[2]);
+
+	/* Test10: unref invs5 (same array) */
+	arm_smmu_invs_unref(test_a, &invs5);
+	arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results9[0]), 1,
+				     results9[0], results9[1], results9[2]);
+
+	/* Test11: purge test_a (new array) */
+	test_b = arm_smmu_invs_purge(test_a);
+	kfree(test_a);
+	arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0,
+				     results10[0], results10[1], results10[2]);
+
+	kfree(test_b);
+}
+
 static struct kunit_case arm_smmu_v3_test_cases[] = {
 	KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort),
 	KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass),
@@ -662,6 +796,7 @@ static struct kunit_case arm_smmu_v3_test_cases[] = {
 	KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass),
 	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear),
 	KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release),
+	KUNIT_CASE(arm_smmu_v3_invs_test),
 	{},
 };
 
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 4d00d796f0783..8bd9717c7db85 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -26,6 +26,7 @@
 #include <linux/pci.h>
 #include <linux/pci-ats.h>
 #include <linux/platform_device.h>
+#include <linux/sort.h>
 #include <linux/string_choices.h>
 #include <kunit/visibility.h>
 #include <uapi/linux/iommufd.h>
@@ -107,6 +108,7 @@ static const char * const event_class_str[] = {
 };
 
 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
+static bool arm_smmu_ats_supported(struct arm_smmu_master *master);
 
 static void parse_driver_options(struct arm_smmu_device *smmu)
 {
@@ -1026,18 +1028,269 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused
 	 */
 }
 
-/* Context descriptor manipulation functions */
-void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
+/* Invalidation array manipulation functions */
+static inline struct arm_smmu_inv *
+arm_smmu_invs_iter_next(struct arm_smmu_invs *invs, size_t next, size_t *idx)
 {
-	struct arm_smmu_cmdq_ent cmd = {
-		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
-			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
-		.tlbi.asid = asid,
-	};
+	while (true) {
+		if (next >= invs->num_invs) {
+			*idx = next;
+			return NULL;
+		}
+		if (!READ_ONCE(invs->inv[next].users)) {
+			next++;
+			continue;
+		}
+		*idx = next;
+		return &invs->inv[next];
+	}
+}
 
-	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
+/**
+ * arm_smmu_invs_for_each_entry - Iterate over all non-trash entries in invs
+ * @invs: the base invalidation array
+ * @idx: a stack variable of 'size_t', to store the array index
+ * @cur: a stack variable of 'struct arm_smmu_inv *'
+ */
+#define arm_smmu_invs_for_each_entry(invs, idx, cur)                           \
+	for (cur = arm_smmu_invs_iter_next(invs, 0, &(idx)); cur;              \
+	     cur = arm_smmu_invs_iter_next(invs, idx + 1, &(idx)))
+
+static int arm_smmu_inv_cmp(const struct arm_smmu_inv *inv_l,
+			    const struct arm_smmu_inv *inv_r)
+{
+	if (inv_l->smmu != inv_r->smmu)
+		return cmp_int((uintptr_t)inv_l->smmu, (uintptr_t)inv_r->smmu);
+	if (inv_l->type != inv_r->type)
+		return cmp_int(inv_l->type, inv_r->type);
+	if (inv_l->id != inv_r->id)
+		return cmp_int(inv_l->id, inv_r->id);
+	if (arm_smmu_inv_is_ats(inv_l))
+		return cmp_int(inv_l->ssid, inv_r->ssid);
+	return 0;
 }
 
+static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l,
+					      size_t next_l, size_t *idx_l,
+					      struct arm_smmu_invs *invs_r,
+					      size_t next_r, size_t *idx_r)
+{
+	struct arm_smmu_inv *cur_l =
+		arm_smmu_invs_iter_next(invs_l, next_l, idx_l);
+
+	/*
+	 * We have to update the idx_r manually, because the invs_r cannot call
+	 * arm_smmu_invs_iter_next() as the invs_r never sets any users counter.
+	 */
+	*idx_r = next_r;
+
+	/*
+	 * Compare of two sorted arrays items. If one side is past the end of
+	 * the array, return the other side to let it run out the iteration.
+	 *
+	 * If the left entry is empty, return 1 to pick the right entry.
+	 * If the right entry is empty, return -1 to pick the left entry.
+	 */
+	if (!cur_l)
+		return 1;
+	if (next_r >= invs_r->num_invs)
+		return -1;
+	return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]);
+}
+
+/**
+ * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for
+ *                              arm_smmu_invs_merge() or arm_smmu_invs_unref()
+ * @invs_l: the base invalidation array
+ * @idx_l: a stack variable of 'size_t', to store the base array index
+ * @invs_r: the build_invs array as to_merge or to_unref
+ * @idx_r: a stack variable of 'size_t', to store the build_invs index
+ * @cmp: a stack variable of 'int', to store return value (-1, 0, or 1)
+ */
+#define arm_smmu_invs_for_each_cmp(invs_l, idx_l, invs_r, idx_r, cmp)          \
+	for (idx_l = idx_r = 0,                                                \
+	     cmp = arm_smmu_invs_iter_next_cmp(invs_l, 0, &(idx_l),            \
+					       invs_r, 0, &(idx_r));           \
+	     idx_l < invs_l->num_invs || idx_r < invs_r->num_invs;             \
+	     cmp = arm_smmu_invs_iter_next_cmp(                                \
+		     invs_l, idx_l + (cmp <= 0 ? 1 : 0), &(idx_l),             \
+		     invs_r, idx_r + (cmp >= 0 ? 1 : 0), &(idx_r)))
+
+/**
+ * arm_smmu_invs_merge() - Merge @to_merge into @invs and generate a new array
+ * @invs: the base invalidation array
+ * @to_merge: an array of invalidations to merge
+ *
+ * Return: a newly allocated array on success, or ERR_PTR
+ *
+ * This function must be locked and serialized with arm_smmu_invs_unref() and
+ * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test.
+ *
+ * Both @invs and @to_merge must be sorted, to ensure the returned array will be
+ * sorted as well.
+ *
+ * Caller is resposible for freeing the @invs and the returned new one.
+ *
+ * Entries marked as trash will be purged in the returned array.
+ */
+VISIBLE_IF_KUNIT
+struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
+					  struct arm_smmu_invs *to_merge)
+{
+	struct arm_smmu_invs *new_invs;
+	struct arm_smmu_inv *new;
+	size_t num_invs = 0;
+	size_t i, j;
+	int cmp;
+
+	arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp)
+		num_invs++;
+
+	new_invs = arm_smmu_invs_alloc(num_invs);
+	if (!new_invs)
+		return ERR_PTR(-ENOMEM);
+
+	new = new_invs->inv;
+	arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) {
+		if (cmp < 0) {
+			*new = invs->inv[i];
+		} else if (cmp == 0) {
+			*new = invs->inv[i];
+			WRITE_ONCE(new->users, READ_ONCE(new->users) + 1);
+		} else {
+			*new = to_merge->inv[j];
+			WRITE_ONCE(new->users, 1);
+		}
+
+		/*
+		 * Check that the new array is sorted. This also validates that
+		 * to_merge is sorted.
+		 */
+		if (new != new_invs->inv)
+			WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1);
+		if (arm_smmu_inv_is_ats(new))
+			new_invs->has_ats = true;
+		new++;
+	}
+
+	WARN_ON(new != new_invs->inv + new_invs->num_invs);
+
+	return new_invs;
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_merge);
+
+/**
+ * arm_smmu_invs_unref() - Find in @invs for all entries in @to_unref, decrease
+ *                         the user counts without deletions
+ * @invs: the base invalidation array
+ * @to_unref: an array of invalidations to decrease their user counts
+ *
+ * Return: the number of trash entries in the array, for arm_smmu_invs_purge()
+ *
+ * This function will not fail. Any entry with users=0 will be marked as trash,
+ * and caller will be notified about the trashed entry via @to_unref by setting
+ * a users=0.
+ *
+ * All tailing trash entries in the array will be dropped. And the size of the
+ * array will be trimmed properly. All trash entries in-between will remain in
+ * the @invs until being completely deleted by the next arm_smmu_invs_merge()
+ * or an arm_smmu_invs_purge() function call.
+ *
+ * This function must be locked and serialized with arm_smmu_invs_merge() and
+ * arm_smmu_invs_purge(), but do not lockdep on any mutex for KUNIT test.
+ *
+ * Note that the final @invs->num_invs might not reflect the actual number of
+ * invalidations due to trash entries. Any reader should take the read lock to
+ * iterate each entry and check its users counter till the last entry.
+ */
+VISIBLE_IF_KUNIT
+void arm_smmu_invs_unref(struct arm_smmu_invs *invs,
+			 struct arm_smmu_invs *to_unref)
+{
+	unsigned long flags;
+	size_t num_invs = 0;
+	size_t i, j;
+	int cmp;
+
+	arm_smmu_invs_for_each_cmp(invs, i, to_unref, j, cmp) {
+		if (cmp < 0) {
+			/* not found in to_unref, leave alone */
+			num_invs = i + 1;
+		} else if (cmp == 0) {
+			int users = READ_ONCE(invs->inv[i].users) - 1;
+
+			if (WARN_ON(users < 0))
+				continue;
+
+			/* same item */
+			WRITE_ONCE(invs->inv[i].users, users);
+			if (users) {
+				WRITE_ONCE(to_unref->inv[j].users, 1);
+				num_invs = i + 1;
+				continue;
+			}
+
+			/* Notify the caller about the trash entry */
+			WRITE_ONCE(to_unref->inv[j].users, 0);
+			invs->num_trashes++;
+		} else {
+			/* item in to_unref is not in invs or already a trash */
+			WARN_ON(true);
+		}
+	}
+
+	/* Exclude any tailing trash */
+	invs->num_trashes -= invs->num_invs - num_invs;
+
+	/* The lock is required to fence concurrent ATS operations. */
+	write_lock_irqsave(&invs->rwlock, flags);
+	WRITE_ONCE(invs->num_invs, num_invs); /* Remove tailing trash entries */
+	write_unlock_irqrestore(&invs->rwlock, flags);
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref);
+
+/**
+ * arm_smmu_invs_purge() - Purge all the trash entries in the @invs
+ * @invs: the base invalidation array
+ *
+ * Return: a newly allocated array on success removing all the trash entries, or
+ *         NULL if there is no trash entry in the array or if allocation failed
+ *
+ * This function must be locked and serialized with arm_smmu_invs_merge() and
+ * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test.
+ *
+ * Caller is resposible for freeing the @invs and the returned new one.
+ */
+VISIBLE_IF_KUNIT
+struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs)
+{
+	struct arm_smmu_invs *new_invs;
+	struct arm_smmu_inv *inv;
+	size_t i, num_invs = 0;
+
+	if (WARN_ON(invs->num_invs < invs->num_trashes))
+		return NULL;
+	if (!invs->num_invs || !invs->num_trashes)
+		return NULL;
+
+	new_invs = arm_smmu_invs_alloc(invs->num_invs - invs->num_trashes);
+	if (!new_invs)
+		return NULL;
+
+	arm_smmu_invs_for_each_entry(invs, i, inv) {
+		new_invs->inv[num_invs] = *inv;
+		if (arm_smmu_inv_is_ats(inv))
+			new_invs->has_ats = true;
+		num_invs++;
+	}
+
+	WARN_ON(num_invs != new_invs->num_invs);
+	return new_invs;
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge);
+
+/* Context descriptor manipulation functions */
+
 /*
  * Based on the value of ent report which bits of the STE the HW will access. It
  * would be nice if this was complete according to the spec, but minimally it
@@ -1236,6 +1489,13 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
 	__le64 unused_update[NUM_ENTRY_QWORDS];
 	u8 used_qword_diff;
 
+	/*
+	 * Many of the entry structures have pointers to other structures that
+	 * need to have their updates be visible before any writes of the entry
+	 * happen.
+	 */
+	dma_wmb();
+
 	used_qword_diff =
 		arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
 	if (hweight8(used_qword_diff) == 1) {
@@ -2240,109 +2500,42 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
 }
 
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
-			    unsigned long iova, size_t size)
-{
-	struct arm_smmu_master_domain *master_domain;
-	int i;
-	unsigned long flags;
-	struct arm_smmu_cmdq_ent cmd = {
-		.opcode = CMDQ_OP_ATC_INV,
-	};
-	struct arm_smmu_cmdq_batch cmds;
-
-	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
-		return 0;
-
-	/*
-	 * Ensure that we've completed prior invalidation of the main TLBs
-	 * before we read 'nr_ats_masters' in case of a concurrent call to
-	 * arm_smmu_enable_ats():
-	 *
-	 *	// unmap()			// arm_smmu_enable_ats()
-	 *	TLBI+SYNC			atomic_inc(&nr_ats_masters);
-	 *	smp_mb();			[...]
-	 *	atomic_read(&nr_ats_masters);	pci_enable_ats() // writel()
-	 *
-	 * Ensures that we always see the incremented 'nr_ats_masters' count if
-	 * ATS was enabled at the PCI device before completion of the TLBI.
-	 */
-	smp_mb();
-	if (!atomic_read(&smmu_domain->nr_ats_masters))
-		return 0;
-
-	arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
-
-	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	list_for_each_entry(master_domain, &smmu_domain->devices,
-			    devices_elm) {
-		struct arm_smmu_master *master = master_domain->master;
-
-		if (!master->ats_enabled)
-			continue;
-
-		if (master_domain->nested_ats_flush) {
-			/*
-			 * If a S2 used as a nesting parent is changed we have
-			 * no option but to completely flush the ATC.
-			 */
-			arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
-		} else {
-			arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
-						&cmd);
-		}
-
-		for (i = 0; i < master->num_streams; i++) {
-			cmd.atc.sid = master->streams[i].id;
-			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
-		}
-	}
-	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
-
-	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
-}
-
 /* IO_PGTABLE API */
 static void arm_smmu_tlb_inv_context(void *cookie)
 {
 	struct arm_smmu_domain *smmu_domain = cookie;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	struct arm_smmu_cmdq_ent cmd;
 
 	/*
-	 * NOTE: when io-pgtable is in non-strict mode, we may get here with
-	 * PTEs previously cleared by unmaps on the current CPU not yet visible
-	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
-	 * insertion to guarantee those are observed before the TLBI. Do be
-	 * careful, 007.
+	 * If the DMA API is running in non-strict mode then another CPU could
+	 * have changed the page table and not invoked any flush op. Instead the
+	 * other CPU will do an atomic_read() and this CPU will have done an
+	 * atomic_write(). That handshake is enough to acquire the page table
+	 * writes from the other CPU.
+	 *
+	 * All command execution has a dma_wmb() to release all the in-memory
+	 * structures written by this CPU, that barrier must also release the
+	 * writes acquired from all the other CPUs too.
+	 *
+	 * There are other barriers and atomics on this path, but the above is
+	 * the essential mechanism for ensuring that HW sees the page table
+	 * writes from another CPU before it executes the IOTLB invalidation.
 	 */
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
-	} else {
-		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
-		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
-		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
-	}
-	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
+	arm_smmu_domain_inv(smmu_domain);
 }
 
-static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
-				     unsigned long iova, size_t size,
-				     size_t granule,
-				     struct arm_smmu_domain *smmu_domain)
+static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
+					  struct arm_smmu_cmdq_batch *cmds,
+					  struct arm_smmu_cmdq_ent *cmd,
+					  unsigned long iova, size_t size,
+					  size_t granule, size_t pgsize)
 {
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	unsigned long end = iova + size, num_pages = 0, tg = 0;
+	unsigned long end = iova + size, num_pages = 0, tg = pgsize;
 	size_t inv_range = granule;
-	struct arm_smmu_cmdq_batch cmds;
 
-	if (!size)
+	if (WARN_ON_ONCE(!size))
 		return;
 
 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
-		/* Get the leaf page size */
-		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
-
 		num_pages = size >> tg;
 
 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
@@ -2362,8 +2555,6 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
 			num_pages++;
 	}
 
-	arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
-
 	while (iova < end) {
 		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
 			/*
@@ -2391,62 +2582,197 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
 		}
 
 		cmd->tlbi.addr = iova;
-		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
+		arm_smmu_cmdq_batch_add(smmu, cmds, cmd);
 		iova += inv_range;
 	}
-	arm_smmu_cmdq_batch_submit(smmu, &cmds);
 }
 
-static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
-					  size_t granule, bool leaf,
-					  struct arm_smmu_domain *smmu_domain)
+static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
+				      size_t granule)
 {
-	struct arm_smmu_cmdq_ent cmd = {
-		.tlbi = {
-			.leaf	= leaf,
-		},
-	};
+	size_t max_tlbi_ops;
 
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
-				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
-		cmd.tlbi.asid	= smmu_domain->cd.asid;
-	} else {
-		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
-		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
-	}
-	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
+	/* 0 size means invalidate all */
+	if (!size || size == SIZE_MAX)
+		return true;
 
-	if (smmu_domain->nest_parent) {
-		/*
-		 * When the S2 domain changes all the nested S1 ASIDs have to be
-		 * flushed too.
-		 */
-		cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
-		arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
-	}
+	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV)
+		return false;
 
 	/*
-	 * Unfortunately, this can't be leaf-only since we may have
-	 * zapped an entire table.
+	 * Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h,
+	 * this is used as a threshold to replace "size_opcode" commands with a
+	 * single "nsize_opcode" command, when SMMU doesn't implement the range
+	 * invalidation feature, where there can be too many per-granule TLBIs,
+	 * resulting in a soft lockup.
 	 */
-	arm_smmu_atc_inv_domain(smmu_domain, iova, size);
+	max_tlbi_ops = 1 << (ilog2(granule) - 3);
+	return size >= max_tlbi_ops * granule;
 }
 
-void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
-				 size_t granule, bool leaf,
-				 struct arm_smmu_domain *smmu_domain)
+/* Used by non INV_TYPE_ATS* invalidations */
+static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
+				       struct arm_smmu_cmdq_batch *cmds,
+				       struct arm_smmu_cmdq_ent *cmd,
+				       unsigned long iova, size_t size,
+				       unsigned int granule)
 {
-	struct arm_smmu_cmdq_ent cmd = {
-		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
-			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
-		.tlbi = {
-			.asid	= asid,
-			.leaf	= leaf,
-		},
-	};
+	if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) {
+		cmd->opcode = inv->nsize_opcode;
+		arm_smmu_cmdq_batch_add(inv->smmu, cmds, cmd);
+		return;
+	}
+
+	cmd->opcode = inv->size_opcode;
+	arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, iova, size, granule,
+				      inv->pgsize);
+}
+
+static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
+					   struct arm_smmu_inv *next)
+{
+	/* Changing smmu means changing command queue */
+	if (cur->smmu != next->smmu)
+		return true;
+	/* The batch for S2 TLBI must be done before nested S1 ASIDs */
+	if (cur->type != INV_TYPE_S2_VMID_S1_CLEAR &&
+	    next->type == INV_TYPE_S2_VMID_S1_CLEAR)
+		return true;
+	/* ATS must be after a sync of the S1/S2 invalidations */
+	if (!arm_smmu_inv_is_ats(cur) && arm_smmu_inv_is_ats(next))
+		return true;
+	return false;
+}
+
+static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
+					unsigned long iova, size_t size,
+					unsigned int granule, bool leaf)
+{
+	struct arm_smmu_cmdq_batch cmds = {};
+	struct arm_smmu_inv *cur;
+	struct arm_smmu_inv *end;
+
+	cur = invs->inv;
+	end = cur + READ_ONCE(invs->num_invs);
+	/* Skip any leading entry marked as a trash */
+	for (; cur != end; cur++)
+		if (READ_ONCE(cur->users))
+			break;
+	while (cur != end) {
+		struct arm_smmu_device *smmu = cur->smmu;
+		struct arm_smmu_cmdq_ent cmd = {
+			/*
+			 * Pick size_opcode to run arm_smmu_get_cmdq(). This can
+			 * be changed to nsize_opcode, which would result in the
+			 * same CMDQ pointer.
+			 */
+			.opcode = cur->size_opcode,
+		};
+		struct arm_smmu_inv *next;
+
+		if (!cmds.num)
+			arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
+
+		switch (cur->type) {
+		case INV_TYPE_S1_ASID:
+			cmd.tlbi.asid = cur->id;
+			cmd.tlbi.leaf = leaf;
+			arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size,
+						   granule);
+			break;
+		case INV_TYPE_S2_VMID:
+			cmd.tlbi.vmid = cur->id;
+			cmd.tlbi.leaf = leaf;
+			arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size,
+						   granule);
+			break;
+		case INV_TYPE_S2_VMID_S1_CLEAR:
+			/* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
+			if (arm_smmu_inv_size_too_big(cur->smmu, size, granule))
+				continue;
+			cmd.tlbi.vmid = cur->id;
+			arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+			break;
+		case INV_TYPE_ATS:
+			arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd);
+			cmd.atc.sid = cur->id;
+			arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+			break;
+		case INV_TYPE_ATS_FULL:
+			arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
+			cmd.atc.sid = cur->id;
+			arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+			break;
+		default:
+			WARN_ON_ONCE(1);
+			continue;
+		}
+
+		/* Skip any trash entry in-between */
+		for (next = cur + 1; next != end; next++)
+			if (READ_ONCE(next->users))
+				break;
 
-	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
+		if (cmds.num &&
+		    (next == end || arm_smmu_invs_end_batch(cur, next))) {
+			arm_smmu_cmdq_batch_submit(smmu, &cmds);
+			cmds.num = 0;
+		}
+		cur = next;
+	}
+}
+
+void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
+			       unsigned long iova, size_t size,
+			       unsigned int granule, bool leaf)
+{
+	struct arm_smmu_invs *invs;
+
+	/*
+	 * An invalidation request must follow some IOPTE change and then load
+	 * an invalidation array. In the meantime, a domain attachment mutates
+	 * the array and then stores an STE/CD asking SMMU HW to acquire those
+	 * changed IOPTEs.
+	 *
+	 * When running alone, a domain attachment relies on the dma_wmb() in
+	 * arm_smmu_write_entry() used by arm_smmu_install_ste_for_dev().
+	 *
+	 * But in a race, these two can be interdependent, making it a special
+	 * case requiring an additional smp_mb() for the write->read ordering.
+	 * Pairing with the dma_wmb() in arm_smmu_install_ste_for_dev(), this
+	 * makes sure that IOPTE update prior to this point is visable to SMMU
+	 * hardware before we load the updated invalidation array.
+	 *
+	 *  [CPU0]                        | [CPU1]
+	 *  change IOPTE on new domain:   |
+	 *  arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs()
+	 *    smp_mb(); // ensures IOPTE  | arm_smmu_install_ste_for_dev {
+	 *              // seen by SMMU   |   dma_wmb(); // ensures invs update
+	 *    // load the updated invs    |              // before updating STE
+	 *    invs = rcu_dereference();   |   STE = TTB0;
+	 *    ...                         |   ...
+	 *  }                             | }
+	 */
+	smp_mb();
+
+	rcu_read_lock();
+	invs = rcu_dereference(smmu_domain->invs);
+
+	/*
+	 * Avoid locking unless ATS is being used. No ATC invalidation can be
+	 * going on after a domain is detached.
+	 */
+	if (invs->has_ats) {
+		unsigned long flags;
+
+		read_lock_irqsave(&invs->rwlock, flags);
+		__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
+		read_unlock_irqrestore(&invs->rwlock, flags);
+	} else {
+		__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
+	}
+
+	rcu_read_unlock();
 }
 
 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
@@ -2462,7 +2788,9 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
 				  size_t granule, void *cookie)
 {
-	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
+	struct arm_smmu_domain *smmu_domain = cookie;
+
+	arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false);
 }
 
 static const struct iommu_flush_ops arm_smmu_flush_ops = {
@@ -2494,6 +2822,8 @@ static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
 		return true;
 	case IOMMU_CAP_DIRTY_TRACKING:
 		return arm_smmu_dbm_capable(master->smmu);
+	case IOMMU_CAP_PCI_ATS_SUPPORTED:
+		return arm_smmu_ats_supported(master);
 	default:
 		return false;
 	}
@@ -2522,13 +2852,21 @@ static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
 struct arm_smmu_domain *arm_smmu_domain_alloc(void)
 {
 	struct arm_smmu_domain *smmu_domain;
+	struct arm_smmu_invs *new_invs;
 
 	smmu_domain = kzalloc_obj(*smmu_domain);
 	if (!smmu_domain)
 		return ERR_PTR(-ENOMEM);
 
+	new_invs = arm_smmu_invs_alloc(0);
+	if (!new_invs) {
+		kfree(smmu_domain);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	INIT_LIST_HEAD(&smmu_domain->devices);
 	spin_lock_init(&smmu_domain->devices_lock);
+	rcu_assign_pointer(smmu_domain->invs, new_invs);
 
 	return smmu_domain;
 }
@@ -2552,7 +2890,7 @@ static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
 			ida_free(&smmu->vmid_map, cfg->vmid);
 	}
 
-	kfree(smmu_domain);
+	arm_smmu_domain_free(smmu_domain);
 }
 
 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
@@ -2870,6 +3208,121 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
 		iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev);
 }
 
+static struct arm_smmu_inv *
+arm_smmu_master_build_inv(struct arm_smmu_master *master,
+			  enum arm_smmu_inv_type type, u32 id, ioasid_t ssid,
+			  size_t pgsize)
+{
+	struct arm_smmu_invs *build_invs = master->build_invs;
+	struct arm_smmu_inv *cur, inv = {
+		.smmu = master->smmu,
+		.type = type,
+		.id = id,
+		.pgsize = pgsize,
+	};
+
+	if (WARN_ON(build_invs->num_invs >= build_invs->max_invs))
+		return NULL;
+	cur = &build_invs->inv[build_invs->num_invs];
+	build_invs->num_invs++;
+
+	*cur = inv;
+	switch (type) {
+	case INV_TYPE_S1_ASID:
+		/*
+		 * For S1 page tables the driver always uses VMID=0, and the
+		 * invalidation logic for this type will set it as well.
+		 */
+		if (master->smmu->features & ARM_SMMU_FEAT_E2H) {
+			cur->size_opcode = CMDQ_OP_TLBI_EL2_VA;
+			cur->nsize_opcode = CMDQ_OP_TLBI_EL2_ASID;
+		} else {
+			cur->size_opcode = CMDQ_OP_TLBI_NH_VA;
+			cur->nsize_opcode = CMDQ_OP_TLBI_NH_ASID;
+		}
+		break;
+	case INV_TYPE_S2_VMID:
+		cur->size_opcode = CMDQ_OP_TLBI_S2_IPA;
+		cur->nsize_opcode = CMDQ_OP_TLBI_S12_VMALL;
+		break;
+	case INV_TYPE_S2_VMID_S1_CLEAR:
+		cur->size_opcode = cur->nsize_opcode = CMDQ_OP_TLBI_NH_ALL;
+		break;
+	case INV_TYPE_ATS:
+	case INV_TYPE_ATS_FULL:
+		cur->size_opcode = cur->nsize_opcode = CMDQ_OP_ATC_INV;
+		cur->ssid = ssid;
+		break;
+	}
+
+	return cur;
+}
+
+/*
+ * Use the preallocated scratch array at master->build_invs, to build a to_merge
+ * or to_unref array, to pass into a following arm_smmu_invs_merge/unref() call.
+ *
+ * Do not free the returned invs array. It is reused, and will be overwritten by
+ * the next arm_smmu_master_build_invs() call.
+ */
+static struct arm_smmu_invs *
+arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled,
+			   ioasid_t ssid, struct arm_smmu_domain *smmu_domain)
+{
+	const bool nesting = smmu_domain->nest_parent;
+	size_t pgsize = 0, i;
+
+	iommu_group_mutex_assert(master->dev);
+
+	master->build_invs->num_invs = 0;
+
+	/* Range-based invalidation requires the leaf pgsize for calculation */
+	if (master->smmu->features & ARM_SMMU_FEAT_RANGE_INV)
+		pgsize = __ffs(smmu_domain->domain.pgsize_bitmap);
+
+	switch (smmu_domain->stage) {
+	case ARM_SMMU_DOMAIN_SVA:
+	case ARM_SMMU_DOMAIN_S1:
+		if (!arm_smmu_master_build_inv(master, INV_TYPE_S1_ASID,
+					       smmu_domain->cd.asid,
+					       IOMMU_NO_PASID, pgsize))
+			return NULL;
+		break;
+	case ARM_SMMU_DOMAIN_S2:
+		if (!arm_smmu_master_build_inv(master, INV_TYPE_S2_VMID,
+					       smmu_domain->s2_cfg.vmid,
+					       IOMMU_NO_PASID, pgsize))
+			return NULL;
+		break;
+	default:
+		WARN_ON(true);
+		return NULL;
+	}
+
+	/* All the nested S1 ASIDs have to be flushed when S2 parent changes */
+	if (nesting) {
+		if (!arm_smmu_master_build_inv(
+			    master, INV_TYPE_S2_VMID_S1_CLEAR,
+			    smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID, 0))
+			return NULL;
+	}
+
+	for (i = 0; ats_enabled && i < master->num_streams; i++) {
+		/*
+		 * If an S2 used as a nesting parent is changed we have no
+		 * option but to completely flush the ATC.
+		 */
+		if (!arm_smmu_master_build_inv(
+			    master, nesting ? INV_TYPE_ATS_FULL : INV_TYPE_ATS,
+			    master->streams[i].id, ssid, 0))
+			return NULL;
+	}
+
+	/* Note this build_invs must have been sorted */
+
+	return master->build_invs;
+}
+
 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
 					  struct iommu_domain *domain,
 					  ioasid_t ssid)
@@ -2899,6 +3352,135 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
 	kfree(master_domain);
 }
 
+/*
+ * During attachment, the updates of the two domain->invs arrays are sequenced:
+ *  1. new domain updates its invs array, merging master->build_invs
+ *  2. new domain starts to include the master during its invalidation
+ *  3. master updates its STE switching from the old domain to the new domain
+ *  4. old domain still includes the master during its invalidation
+ *  5. old domain updates its invs array, unreferencing master->build_invs
+ *
+ * For 1 and 5, prepare the two updated arrays in advance, handling any changes
+ * that can possibly failure. So the actual update of either 1 or 5 won't fail.
+ * arm_smmu_asid_lock ensures that the old invs in the domains are intact while
+ * we are sequencing to update them.
+ */
+static int arm_smmu_attach_prepare_invs(struct arm_smmu_attach_state *state,
+					struct iommu_domain *new_domain)
+{
+	struct arm_smmu_domain *old_smmu_domain =
+		to_smmu_domain_devices(state->old_domain);
+	struct arm_smmu_domain *new_smmu_domain =
+		to_smmu_domain_devices(new_domain);
+	struct arm_smmu_master *master = state->master;
+	ioasid_t ssid = state->ssid;
+
+	/*
+	 * At this point a NULL domain indicates the domain doesn't use the
+	 * IOTLB, see to_smmu_domain_devices().
+	 */
+	if (new_smmu_domain) {
+		struct arm_smmu_inv_state *invst = &state->new_domain_invst;
+		struct arm_smmu_invs *build_invs;
+
+		invst->invs_ptr = &new_smmu_domain->invs;
+		invst->old_invs = rcu_dereference_protected(
+			new_smmu_domain->invs,
+			lockdep_is_held(&arm_smmu_asid_lock));
+		build_invs = arm_smmu_master_build_invs(
+			master, state->ats_enabled, ssid, new_smmu_domain);
+		if (!build_invs)
+			return -EINVAL;
+
+		invst->new_invs =
+			arm_smmu_invs_merge(invst->old_invs, build_invs);
+		if (IS_ERR(invst->new_invs))
+			return PTR_ERR(invst->new_invs);
+	}
+
+	if (old_smmu_domain) {
+		struct arm_smmu_inv_state *invst = &state->old_domain_invst;
+
+		invst->invs_ptr = &old_smmu_domain->invs;
+		/* A re-attach case might have a different ats_enabled state */
+		if (new_smmu_domain == old_smmu_domain)
+			invst->old_invs = state->new_domain_invst.new_invs;
+		else
+			invst->old_invs = rcu_dereference_protected(
+				old_smmu_domain->invs,
+				lockdep_is_held(&arm_smmu_asid_lock));
+		/* For old_smmu_domain, new_invs points to master->build_invs */
+		invst->new_invs = arm_smmu_master_build_invs(
+			master, master->ats_enabled, ssid, old_smmu_domain);
+	}
+
+	return 0;
+}
+
+/* Must be installed before arm_smmu_install_ste_for_dev() */
+static void
+arm_smmu_install_new_domain_invs(struct arm_smmu_attach_state *state)
+{
+	struct arm_smmu_inv_state *invst = &state->new_domain_invst;
+
+	if (!invst->invs_ptr)
+		return;
+
+	rcu_assign_pointer(*invst->invs_ptr, invst->new_invs);
+	kfree_rcu(invst->old_invs, rcu);
+}
+
+static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv)
+{
+	struct arm_smmu_cmdq_ent cmd = {};
+
+	switch (inv->type) {
+	case INV_TYPE_S1_ASID:
+		cmd.tlbi.asid = inv->id;
+		break;
+	case INV_TYPE_S2_VMID:
+		/* S2_VMID using nsize_opcode covers S2_VMID_S1_CLEAR */
+		cmd.tlbi.vmid = inv->id;
+		break;
+	default:
+		return;
+	}
+
+	cmd.opcode = inv->nsize_opcode;
+	arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &cmd);
+}
+
+/* Should be installed after arm_smmu_install_ste_for_dev() */
+static void
+arm_smmu_install_old_domain_invs(struct arm_smmu_attach_state *state)
+{
+	struct arm_smmu_inv_state *invst = &state->old_domain_invst;
+	struct arm_smmu_invs *old_invs = invst->old_invs;
+	struct arm_smmu_invs *new_invs;
+
+	lockdep_assert_held(&arm_smmu_asid_lock);
+
+	if (!invst->invs_ptr)
+		return;
+
+	arm_smmu_invs_unref(old_invs, invst->new_invs);
+	/*
+	 * When an IOTLB tag (the first entry in invs->new_invs) is no longer used,
+	 * it means the ASID or VMID will no longer be invalidated by map/unmap and
+	 * must be cleaned right now. The rule is that any ASID/VMID not in an invs
+	 * array must be left cleared in the IOTLB.
+	 */
+	if (!READ_ONCE(invst->new_invs->inv[0].users))
+		arm_smmu_inv_flush_iotlb_tag(&invst->new_invs->inv[0]);
+
+	new_invs = arm_smmu_invs_purge(old_invs);
+	if (!new_invs)
+		return;
+
+	rcu_assign_pointer(*invst->invs_ptr, new_invs);
+	kfree_rcu(old_invs, rcu);
+}
+
 /*
  * Start the sequence to attach a domain to a master. The sequence contains three
  * steps:
@@ -2956,12 +3538,16 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
 				     arm_smmu_ats_supported(master);
 	}
 
+	ret = arm_smmu_attach_prepare_invs(state, new_domain);
+	if (ret)
+		return ret;
+
 	if (smmu_domain) {
 		if (new_domain->type == IOMMU_DOMAIN_NESTED) {
 			ret = arm_smmu_attach_prepare_vmaster(
 				state, to_smmu_nested_domain(new_domain));
 			if (ret)
-				return ret;
+				goto err_unprepare_invs;
 		}
 
 		master_domain = kzalloc_obj(*master_domain);
@@ -3009,6 +3595,8 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
 			atomic_inc(&smmu_domain->nr_ats_masters);
 		list_add(&master_domain->devices_elm, &smmu_domain->devices);
 		spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+
+		arm_smmu_install_new_domain_invs(state);
 	}
 
 	if (!state->ats_enabled && master->ats_enabled) {
@@ -3028,6 +3616,8 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
 	kfree(master_domain);
 err_free_vmaster:
 	kfree(state->vmaster);
+err_unprepare_invs:
+	kfree(state->new_domain_invst.new_invs);
 	return ret;
 }
 
@@ -3059,6 +3649,7 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
 	}
 
 	arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
+	arm_smmu_install_old_domain_invs(state);
 	master->ats_enabled = state->ats_enabled;
 }
 
@@ -3125,6 +3716,9 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev,
 		arm_smmu_install_ste_for_dev(master, &target);
 		arm_smmu_clear_cd(master, IOMMU_NO_PASID);
 		break;
+	default:
+		WARN_ON(true);
+		break;
 	}
 
 	arm_smmu_attach_commit(&state);
@@ -3238,12 +3832,19 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
 {
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	struct arm_smmu_attach_state state = {
+		.master = master,
+		.old_domain = old_domain,
+		.ssid = pasid,
+	};
 
 	mutex_lock(&arm_smmu_asid_lock);
+	arm_smmu_attach_prepare_invs(&state, NULL);
 	arm_smmu_clear_cd(master, pasid);
 	if (master->ats_enabled)
 		arm_smmu_atc_inv_master(master, pasid);
 	arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
+	arm_smmu_install_old_domain_invs(&state);
 	mutex_unlock(&arm_smmu_asid_lock);
 
 	/*
@@ -3417,7 +4018,7 @@ arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
 	return &smmu_domain->domain;
 
 err_free:
-	kfree(smmu_domain);
+	arm_smmu_domain_free(smmu_domain);
 	return ERR_PTR(ret);
 }
 
@@ -3462,9 +4063,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
 	if (!gather->pgsize)
 		return;
 
-	arm_smmu_tlb_inv_range_domain(gather->start,
-				      gather->end - gather->start + 1,
-				      gather->pgsize, true, smmu_domain);
+	arm_smmu_domain_inv_range(smmu_domain, gather->start,
+				  gather->end - gather->start + 1,
+				  gather->pgsize, true);
 }
 
 static phys_addr_t
@@ -3509,26 +4110,57 @@ static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
 	return 0;
 }
 
+static int arm_smmu_stream_id_cmp(const void *_l, const void *_r)
+{
+	const typeof_member(struct arm_smmu_stream, id) *l = _l;
+	const typeof_member(struct arm_smmu_stream, id) *r = _r;
+
+	return cmp_int(*l, *r);
+}
+
 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
 				  struct arm_smmu_master *master)
 {
 	int i;
 	int ret = 0;
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
+	bool ats_supported = dev_is_pci(master->dev) &&
+			     pci_ats_supported(to_pci_dev(master->dev));
 
 	master->streams = kzalloc_objs(*master->streams, fwspec->num_ids);
 	if (!master->streams)
 		return -ENOMEM;
 	master->num_streams = fwspec->num_ids;
 
-	mutex_lock(&smmu->streams_mutex);
+	if (!ats_supported) {
+		/* Base case has 1 ASID entry or maximum 2 VMID entries */
+		master->build_invs = arm_smmu_invs_alloc(2);
+	} else {
+		/* ATS case adds num_ids of entries, on top of the base case */
+		master->build_invs = arm_smmu_invs_alloc(2 + fwspec->num_ids);
+	}
+	if (!master->build_invs) {
+		kfree(master->streams);
+		return -ENOMEM;
+	}
+
 	for (i = 0; i < fwspec->num_ids; i++) {
 		struct arm_smmu_stream *new_stream = &master->streams[i];
-		struct rb_node *existing;
-		u32 sid = fwspec->ids[i];
 
-		new_stream->id = sid;
+		new_stream->id = fwspec->ids[i];
 		new_stream->master = master;
+	}
+
+	/* Put the ids into order for sorted to_merge/to_unref arrays */
+	sort_nonatomic(master->streams, master->num_streams,
+		       sizeof(master->streams[0]), arm_smmu_stream_id_cmp,
+		       NULL);
+
+	mutex_lock(&smmu->streams_mutex);
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct arm_smmu_stream *new_stream = &master->streams[i];
+		struct rb_node *existing;
+		u32 sid = new_stream->id;
 
 		ret = arm_smmu_init_sid_strtab(smmu, sid);
 		if (ret)
@@ -3558,6 +4190,7 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
 		for (i--; i >= 0; i--)
 			rb_erase(&master->streams[i].node, &smmu->streams);
 		kfree(master->streams);
+		kfree(master->build_invs);
 	}
 	mutex_unlock(&smmu->streams_mutex);
 
@@ -3579,6 +4212,7 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master)
 	mutex_unlock(&smmu->streams_mutex);
 
 	kfree(master->streams);
+	kfree(master->build_invs);
 }
 
 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 3c6d65d36164f..36de2b0b2ebe6 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -648,6 +648,93 @@ struct arm_smmu_cmdq_batch {
 	int				num;
 };
 
+/*
+ * The order here also determines the sequence in which commands are sent to the
+ * command queue. E.g. TLBI must be done before ATC_INV.
+ */
+enum arm_smmu_inv_type {
+	INV_TYPE_S1_ASID,
+	INV_TYPE_S2_VMID,
+	INV_TYPE_S2_VMID_S1_CLEAR,
+	INV_TYPE_ATS,
+	INV_TYPE_ATS_FULL,
+};
+
+struct arm_smmu_inv {
+	struct arm_smmu_device *smmu;
+	u8 type;
+	u8 size_opcode;
+	u8 nsize_opcode;
+	u32 id; /* ASID or VMID or SID */
+	union {
+		size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */
+		u32 ssid; /* INV_TYPE_ATS */
+	};
+
+	int users; /* users=0 to mark as a trash to be purged */
+};
+
+static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv)
+{
+	return inv->type == INV_TYPE_ATS || inv->type == INV_TYPE_ATS_FULL;
+}
+
+/**
+ * struct arm_smmu_invs - Per-domain invalidation array
+ * @max_invs: maximum capacity of the flexible array
+ * @num_invs: number of invalidations in the flexible array. May be smaller than
+ *            @max_invs after a tailing trash entry is excluded, but must not be
+ *            greater than @max_invs
+ * @num_trashes: number of trash entries in the array for arm_smmu_invs_purge().
+ *               Must not be greater than @num_invs
+ * @rwlock: optional rwlock to fench ATS operations
+ * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL
+ * @rcu: rcu head for kfree_rcu()
+ * @inv: flexible invalidation array
+ *
+ * The arm_smmu_invs is an RCU data structure. During a ->attach_dev callback,
+ * arm_smmu_invs_merge(), arm_smmu_invs_unref() and arm_smmu_invs_purge() will
+ * be used to allocate a new copy of an old array for addition and deletion in
+ * the old domain's and new domain's invs arrays.
+ *
+ * The arm_smmu_invs_unref() mutates a given array, by internally reducing the
+ * users counts of some given entries. This exists to support a no-fail routine
+ * like attaching to an IOMMU_DOMAIN_BLOCKED. And it could pair with a followup
+ * arm_smmu_invs_purge() call to generate a new clean array.
+ *
+ * Concurrent invalidation thread will push every invalidation described in the
+ * array into the command queue for each invalidation event. It is designed like
+ * this to optimize the invalidation fast path by avoiding locks.
+ *
+ * A domain can be shared across SMMU instances. When an instance gets removed,
+ * it would delete all the entries that belong to that SMMU instance. Then, a
+ * synchronize_rcu() would have to be called to sync the array, to prevent any
+ * concurrent invalidation thread accessing the old array from issuing commands
+ * to the command queue of a removed SMMU instance.
+ */
+struct arm_smmu_invs {
+	size_t max_invs;
+	size_t num_invs;
+	size_t num_trashes;
+	rwlock_t rwlock;
+	bool has_ats;
+	struct rcu_head rcu;
+	struct arm_smmu_inv inv[] __counted_by(max_invs);
+};
+
+static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
+{
+	struct arm_smmu_invs *new_invs;
+
+	new_invs = kzalloc(struct_size(new_invs, inv, num_invs), GFP_KERNEL);
+	if (!new_invs)
+		return NULL;
+	new_invs->max_invs = num_invs;
+	new_invs->num_invs = num_invs;
+	rwlock_init(&new_invs->rwlock);
+	return new_invs;
+}
+
 struct arm_smmu_evtq {
 	struct arm_smmu_queue		q;
 	struct iopf_queue		*iopf;
@@ -841,6 +928,14 @@ struct arm_smmu_master {
 	struct arm_smmu_device		*smmu;
 	struct device			*dev;
 	struct arm_smmu_stream		*streams;
+	/*
+	 * Scratch memory for a to_merge or to_unref array to build a per-domain
+	 * invalidation array. It'll be pre-allocated with enough enries for all
+	 * possible build scenarios. It can be used by only one caller at a time
+	 * until the arm_smmu_invs_merge/unref() finishes. Must be locked by the
+	 * iommu_group mutex.
+	 */
+	struct arm_smmu_invs		*build_invs;
 	struct arm_smmu_vmaster		*vmaster; /* use smmu->streams_mutex */
 	/* Locked by the iommu core using the group mutex */
 	struct arm_smmu_ctx_desc_cfg	cd_table;
@@ -856,6 +951,7 @@ struct arm_smmu_master {
 enum arm_smmu_domain_stage {
 	ARM_SMMU_DOMAIN_S1 = 0,
 	ARM_SMMU_DOMAIN_S2,
+	ARM_SMMU_DOMAIN_SVA,
 };
 
 struct arm_smmu_domain {
@@ -872,6 +968,8 @@ struct arm_smmu_domain {
 
 	struct iommu_domain		domain;
 
+	struct arm_smmu_invs __rcu	*invs;
+
 	/* List of struct arm_smmu_master_domain */
 	struct list_head		devices;
 	spinlock_t			devices_lock;
@@ -924,6 +1022,12 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
 			  struct arm_smmu_master *master, struct mm_struct *mm,
 			  u16 asid);
+
+struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
+					  struct arm_smmu_invs *to_merge);
+void arm_smmu_invs_unref(struct arm_smmu_invs *invs,
+			 struct arm_smmu_invs *to_unref);
+struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs);
 #endif
 
 struct arm_smmu_master_domain {
@@ -955,6 +1059,13 @@ extern struct mutex arm_smmu_asid_lock;
 
 struct arm_smmu_domain *arm_smmu_domain_alloc(void);
 
+static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain)
+{
+	/* No concurrency with invalidation is possible at this point */
+	kfree(rcu_dereference_protected(smmu_domain->invs, true));
+	kfree(smmu_domain);
+}
+
 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid);
 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
 					u32 ssid);
@@ -969,12 +1080,14 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
 		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
 		       struct arm_smmu_cd *cd, struct iommu_domain *old);
 
-void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
-void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
-				 size_t granule, bool leaf,
-				 struct arm_smmu_domain *smmu_domain);
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
-			    unsigned long iova, size_t size);
+void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
+			       unsigned long iova, size_t size,
+			       unsigned int granule, bool leaf);
+
+static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain)
+{
+	arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false);
+}
 
 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
 			      struct arm_smmu_cmdq *cmdq);
@@ -991,6 +1104,21 @@ static inline bool arm_smmu_master_canwbs(struct arm_smmu_master *master)
 	       IOMMU_FWSPEC_PCI_RC_CANWBS;
 }
 
+/**
+ * struct arm_smmu_inv_state - Per-domain invalidation array state
+ * @invs_ptr: points to the domain->invs (unwinding nesting/etc.) or is NULL if
+ *            no change should be made
+ * @old_invs: the original invs array
+ * @new_invs: for new domain, this is the new invs array to update domain->invs;
+ *            for old domain, this is the master->build_invs to pass in as the
+ *            to_unref argument to an arm_smmu_invs_unref() call
+ */
+struct arm_smmu_inv_state {
+	struct arm_smmu_invs __rcu **invs_ptr;
+	struct arm_smmu_invs *old_invs;
+	struct arm_smmu_invs *new_invs;
+};
+
 struct arm_smmu_attach_state {
 	/* Inputs */
 	struct iommu_domain *old_domain;
@@ -1000,6 +1128,8 @@ struct arm_smmu_attach_state {
 	ioasid_t ssid;
 	/* Resulting state */
 	struct arm_smmu_vmaster *vmaster;
+	struct arm_smmu_inv_state old_domain_invst;
+	struct arm_smmu_inv_state new_domain_invst;
 	bool ats_enabled;
 };
 
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 5dac64be61bb2..68b44d9320d3c 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1095,8 +1095,10 @@ void iommu_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
 		return;
 
 	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
-	if (!dev_is_dma_coherent(dev))
+	if (!dev_is_dma_coherent(dev)) {
 		arch_sync_dma_for_cpu(phys, size, dir);
+		arch_sync_dma_flush();
+	}
 
 	swiotlb_sync_single_for_cpu(dev, phys, size, dir);
 }
@@ -1112,8 +1114,10 @@ void iommu_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
 	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
 	swiotlb_sync_single_for_device(dev, phys, size, dir);
 
-	if (!dev_is_dma_coherent(dev))
+	if (!dev_is_dma_coherent(dev)) {
 		arch_sync_dma_for_device(phys, size, dir);
+		arch_sync_dma_flush();
+	}
 }
 
 void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
@@ -1122,13 +1126,15 @@ void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
 	struct scatterlist *sg;
 	int i;
 
-	if (sg_dma_is_swiotlb(sgl))
+	if (sg_dma_is_swiotlb(sgl)) {
 		for_each_sg(sgl, sg, nelems, i)
 			iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
 						      sg->length, dir);
-	else if (!dev_is_dma_coherent(dev))
+	} else if (!dev_is_dma_coherent(dev)) {
 		for_each_sg(sgl, sg, nelems, i)
 			arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
+		arch_sync_dma_flush();
+	}
 }
 
 void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
@@ -1137,14 +1143,16 @@ void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
 	struct scatterlist *sg;
 	int i;
 
-	if (sg_dma_is_swiotlb(sgl))
+	if (sg_dma_is_swiotlb(sgl)) {
 		for_each_sg(sgl, sg, nelems, i)
 			iommu_dma_sync_single_for_device(dev,
 							 sg_dma_address(sg),
 							 sg->length, dir);
-	else if (!dev_is_dma_coherent(dev))
+	} else if (!dev_is_dma_coherent(dev)) {
 		for_each_sg(sgl, sg, nelems, i)
 			arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
+		arch_sync_dma_flush();
+	}
 }
 
 static phys_addr_t iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
@@ -1211,7 +1219,7 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	 */
 	if (dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
-		if (attrs & DMA_ATTR_MMIO)
+		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
 			return DMA_MAPPING_ERROR;
 
 		phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs);
@@ -1219,11 +1227,14 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 			return DMA_MAPPING_ERROR;
 	}
 
-	if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))
+	if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
 		arch_sync_dma_for_device(phys, size, dir);
+		arch_sync_dma_flush();
+	}
 
 	iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
-	if (iova == DMA_MAPPING_ERROR && !(attrs & DMA_ATTR_MMIO))
+	if (iova == DMA_MAPPING_ERROR &&
+	    !(attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)))
 		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
 	return iova;
 }
@@ -1233,7 +1244,7 @@ void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle,
 {
 	phys_addr_t phys;
 
-	if (attrs & DMA_ATTR_MMIO) {
+	if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) {
 		__iommu_dma_unmap(dev, dma_handle, size);
 		return;
 	}
@@ -1242,8 +1253,10 @@ void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle,
 	if (WARN_ON(!phys))
 		return;
 
-	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev))
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev)) {
 		arch_sync_dma_for_cpu(phys, size, dir);
+		arch_sync_dma_flush();
+	}
 
 	__iommu_dma_unmap(dev, dma_handle, size);
 
@@ -1945,9 +1958,21 @@ int dma_iova_link(struct device *dev, struct dma_iova_state *state,
 	if (WARN_ON_ONCE(iova_start_pad && offset > 0))
 		return -EIO;
 
+	/*
+	 * DMA_IOVA_USE_SWIOTLB is set on state after some entry
+	 * took SWIOTLB path, which we were supposed to prevent
+	 * for DMA_ATTR_REQUIRE_COHERENT attribute.
+	 */
+	if (WARN_ON_ONCE((state->__size & DMA_IOVA_USE_SWIOTLB) &&
+			 (attrs & DMA_ATTR_REQUIRE_COHERENT)))
+		return -EOPNOTSUPP;
+
+	if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
+		return -EOPNOTSUPP;
+
 	if (dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
-		if (attrs & DMA_ATTR_MMIO)
+		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
 			return -EPERM;
 
 		return iommu_dma_iova_link_swiotlb(dev, state, phys, offset,
@@ -1980,6 +2005,8 @@ int dma_iova_sync(struct device *dev, struct dma_iova_state *state,
 	dma_addr_t addr = state->addr + offset;
 	size_t iova_start_pad = iova_offset(iovad, addr);
 
+	if (!dev_is_dma_coherent(dev))
+		arch_sync_dma_flush();
 	return iommu_sync_map(domain, addr - iova_start_pad,
 		      iova_align(iovad, size + iova_start_pad));
 }
@@ -1993,6 +2020,8 @@ static void iommu_dma_iova_unlink_range_slow(struct device *dev,
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 	struct iova_domain *iovad = &cookie->iovad;
 	size_t iova_start_pad = iova_offset(iovad, addr);
+	bool need_sync_dma = !dev_is_dma_coherent(dev) &&
+			!(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO));
 	dma_addr_t end = addr + size;
 
 	do {
@@ -2016,6 +2045,9 @@ static void iommu_dma_iova_unlink_range_slow(struct device *dev,
 		addr += len;
 		iova_start_pad = 0;
 	} while (addr < end);
+
+	if (need_sync_dma)
+		arch_sync_dma_flush();
 }
 
 static void __iommu_dma_iova_unlink(struct device *dev,
diff --git a/drivers/iommu/generic_pt/.kunitconfig b/drivers/iommu/generic_pt/.kunitconfig
index a78b295f264d2..0bb98fe581fe9 100644
--- a/drivers/iommu/generic_pt/.kunitconfig
+++ b/drivers/iommu/generic_pt/.kunitconfig
@@ -5,6 +5,7 @@ CONFIG_DEBUG_GENERIC_PT=y
 CONFIG_IOMMU_PT=y
 CONFIG_IOMMU_PT_AMDV1=y
 CONFIG_IOMMU_PT_VTDSS=y
+CONFIG_IOMMU_PT_RISCV64=y
 CONFIG_IOMMU_PT_X86_64=y
 CONFIG_IOMMU_PT_KUNIT_TEST=y
 
diff --git a/drivers/iommu/generic_pt/Kconfig b/drivers/iommu/generic_pt/Kconfig
index ce4fb47869145..f4ed1add58b74 100644
--- a/drivers/iommu/generic_pt/Kconfig
+++ b/drivers/iommu/generic_pt/Kconfig
@@ -52,6 +52,16 @@ config IOMMU_PT_VTDSS
 
 	  Selected automatically by an IOMMU driver that uses this format.
 
+config IOMMU_PT_RISCV64
+       tristate "IOMMU page table for RISC-V 64 bit Sv57/Sv48/Sv39"
+	depends on !GENERIC_ATOMIC64 # for cmpxchg64
+	help
+	  iommu_domain implementation for RISC-V 64 bit 3/4/5 level page table.
+	  It supports 4K/2M/1G/512G/256T page sizes and can decode a sign
+	  extended portion of the 64 bit IOVA space.
+
+	  Selected automatically by an IOMMU driver that uses this format.
+
 config IOMMU_PT_X86_64
 	tristate "IOMMU page table for x86 64-bit, 4/5 levels"
 	depends on !GENERIC_ATOMIC64 # for cmpxchg64
@@ -66,6 +76,7 @@ config IOMMU_PT_KUNIT_TEST
 	tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	depends on IOMMU_PT_AMDV1 || !IOMMU_PT_AMDV1
+	depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64
 	depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64
 	depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS
 	default KUNIT_ALL_TESTS
diff --git a/drivers/iommu/generic_pt/fmt/Makefile b/drivers/iommu/generic_pt/fmt/Makefile
index 976b49ec97dca..ea024d582594e 100644
--- a/drivers/iommu/generic_pt/fmt/Makefile
+++ b/drivers/iommu/generic_pt/fmt/Makefile
@@ -5,6 +5,8 @@ iommu_pt_fmt-$(CONFIG_IOMMUFD_TEST) += mock
 
 iommu_pt_fmt-$(CONFIG_IOMMU_PT_VTDSS) += vtdss
 
+iommu_pt_fmt-$(CONFIG_IOMMU_PT_RISCV64) += riscv64
+
 iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64
 
 IOMMU_PT_KUNIT_TEST :=
diff --git a/drivers/iommu/generic_pt/fmt/defs_riscv.h b/drivers/iommu/generic_pt/fmt/defs_riscv.h
new file mode 100644
index 0000000000000..cf67474d5ebae
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/defs_riscv.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES
+ *
+ */
+#ifndef __GENERIC_PT_FMT_DEFS_RISCV_H
+#define __GENERIC_PT_FMT_DEFS_RISCV_H
+
+#include <linux/generic_pt/common.h>
+#include <linux/types.h>
+
+#ifdef PT_RISCV_32BIT
+typedef u32 pt_riscv_entry_t;
+#define riscvpt_write_attrs riscv32pt_write_attrs
+#else
+typedef u64 pt_riscv_entry_t;
+#define riscvpt_write_attrs riscv64pt_write_attrs
+#endif
+
+typedef pt_riscv_entry_t pt_vaddr_t;
+typedef u64 pt_oaddr_t;
+
+struct riscvpt_write_attrs {
+	pt_riscv_entry_t descriptor_bits;
+	gfp_t gfp;
+};
+#define pt_write_attrs riscvpt_write_attrs
+
+#endif
diff --git a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
new file mode 100644
index 0000000000000..cbf60fffa9bf7
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES
+ */
+#define PT_FMT riscv
+#define PT_FMT_VARIANT 64
+#define PT_SUPPORTED_FEATURES                                  \
+	(BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \
+	 BIT(PT_FEAT_RISCV_SVNAPOT_64K))
+
+#include "iommu_template.h"
diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h
new file mode 100644
index 0000000000000..a7fef6266a36a
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/riscv.h
@@ -0,0 +1,313 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES
+ *
+ * RISC-V page table
+ *
+ * This is described in Sections:
+ *  12.3. Sv32: Page-Based 32-bit Virtual-Memory Systems
+ *  12.4. Sv39: Page-Based 39-bit Virtual-Memory System
+ *  12.5. Sv48: Page-Based 48-bit Virtual-Memory System
+ *  12.6. Sv57: Page-Based 57-bit Virtual-Memory System
+ * of the "The RISC-V Instruction Set Manual: Volume II"
+ *
+ * This includes the contiguous page extension from:
+ *  Chapter 13. "Svnapot" Extension for NAPOT Translation Contiguity,
+ *     Version 1.0
+ *
+ * The table format is sign extended and supports leafs in every level. The spec
+ * doesn't talk a lot about levels, but level here is the same as i=LEVELS-1 in
+ * the spec.
+ */
+#ifndef __GENERIC_PT_FMT_RISCV_H
+#define __GENERIC_PT_FMT_RISCV_H
+
+#include "defs_riscv.h"
+#include "../pt_defs.h"
+
+#include <linux/bitfield.h>
+#include <linux/container_of.h>
+#include <linux/log2.h>
+#include <linux/sizes.h>
+
+enum {
+	PT_ITEM_WORD_SIZE = sizeof(pt_riscv_entry_t),
+#ifdef PT_RISCV_32BIT
+	PT_MAX_VA_ADDRESS_LG2 = 32,
+	PT_MAX_OUTPUT_ADDRESS_LG2 = 34,
+	PT_MAX_TOP_LEVEL = 1,
+#else
+	PT_MAX_VA_ADDRESS_LG2 = 57,
+	PT_MAX_OUTPUT_ADDRESS_LG2 = 56,
+	PT_MAX_TOP_LEVEL = 4,
+#endif
+	PT_GRANULE_LG2SZ = 12,
+	PT_TABLEMEM_LG2SZ = 12,
+
+	/* fsc.PPN is 44 bits wide, all PPNs are 4k aligned */
+	PT_TOP_PHYS_MASK = GENMASK_ULL(55, 12),
+};
+
+/* PTE bits */
+enum {
+	RISCVPT_V = BIT(0),
+	RISCVPT_R = BIT(1),
+	RISCVPT_W = BIT(2),
+	RISCVPT_X = BIT(3),
+	RISCVPT_U = BIT(4),
+	RISCVPT_G = BIT(5),
+	RISCVPT_A = BIT(6),
+	RISCVPT_D = BIT(7),
+	RISCVPT_RSW = GENMASK(9, 8),
+	RISCVPT_PPN32 = GENMASK(31, 10),
+
+	RISCVPT_PPN64 = GENMASK_ULL(53, 10),
+	RISCVPT_PPN64_64K = GENMASK_ULL(53, 14),
+	RISCVPT_PBMT = GENMASK_ULL(62, 61),
+	RISCVPT_N = BIT_ULL(63),
+
+	/* Svnapot encodings for ppn[0] */
+	RISCVPT_PPN64_64K_SZ = BIT(13),
+};
+
+#ifdef PT_RISCV_32BIT
+#define RISCVPT_PPN RISCVPT_PPN32
+#define pt_riscv pt_riscv_32
+#else
+#define RISCVPT_PPN RISCVPT_PPN64
+#define pt_riscv pt_riscv_64
+#endif
+
+#define common_to_riscvpt(common_ptr) \
+	container_of_const(common_ptr, struct pt_riscv, common)
+#define to_riscvpt(pts) common_to_riscvpt((pts)->range->common)
+
+static inline pt_oaddr_t riscvpt_table_pa(const struct pt_state *pts)
+{
+	return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ);
+}
+#define pt_table_pa riscvpt_table_pa
+
+static inline pt_oaddr_t riscvpt_entry_oa(const struct pt_state *pts)
+{
+	if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) &&
+	    pts->entry & RISCVPT_N) {
+		PT_WARN_ON(pts->level != 0);
+		return oalog2_mul(FIELD_GET(RISCVPT_PPN64_64K, pts->entry),
+				  ilog2(SZ_64K));
+	}
+	return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ);
+}
+#define pt_entry_oa riscvpt_entry_oa
+
+static inline bool riscvpt_can_have_leaf(const struct pt_state *pts)
+{
+	return true;
+}
+#define pt_can_have_leaf riscvpt_can_have_leaf
+
+/* Body in pt_fmt_defaults.h */
+static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts);
+
+static inline unsigned int
+riscvpt_entry_num_contig_lg2(const struct pt_state *pts)
+{
+	if (PT_SUPPORTED_FEATURE(PT_FEAT_RISCV_SVNAPOT_64K) &&
+	    pts->entry & RISCVPT_N) {
+		PT_WARN_ON(!pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K));
+		PT_WARN_ON(pts->level);
+		return ilog2(16);
+	}
+	return ilog2(1);
+}
+#define pt_entry_num_contig_lg2 riscvpt_entry_num_contig_lg2
+
+static inline unsigned int riscvpt_num_items_lg2(const struct pt_state *pts)
+{
+	return PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64));
+}
+#define pt_num_items_lg2 riscvpt_num_items_lg2
+
+static inline unsigned short
+riscvpt_contig_count_lg2(const struct pt_state *pts)
+{
+	if (pts->level == 0 && pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K))
+		return ilog2(16);
+	return ilog2(1);
+}
+#define pt_contig_count_lg2 riscvpt_contig_count_lg2
+
+static inline enum pt_entry_type riscvpt_load_entry_raw(struct pt_state *pts)
+{
+	const pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t);
+	pt_riscv_entry_t entry;
+
+	pts->entry = entry = READ_ONCE(tablep[pts->index]);
+	if (!(entry & RISCVPT_V))
+		return PT_ENTRY_EMPTY;
+	if (pts->level == 0 ||
+	    ((entry & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) != 0))
+		return PT_ENTRY_OA;
+	return PT_ENTRY_TABLE;
+}
+#define pt_load_entry_raw riscvpt_load_entry_raw
+
+static inline void
+riscvpt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
+			   unsigned int oasz_lg2,
+			   const struct pt_write_attrs *attrs)
+{
+	pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t);
+	pt_riscv_entry_t entry;
+
+	if (!pt_check_install_leaf_args(pts, oa, oasz_lg2))
+		return;
+
+	entry = RISCVPT_V |
+		FIELD_PREP(RISCVPT_PPN, log2_div(oa, PT_GRANULE_LG2SZ)) |
+		attrs->descriptor_bits;
+
+	if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) && pts->level == 0 &&
+	    oasz_lg2 != PT_GRANULE_LG2SZ) {
+		u64 *end;
+
+		entry |= RISCVPT_N | RISCVPT_PPN64_64K_SZ;
+		tablep += pts->index;
+		end = tablep + log2_div(SZ_64K, PT_GRANULE_LG2SZ);
+		for (; tablep != end; tablep++)
+			WRITE_ONCE(*tablep, entry);
+	} else {
+		/* FIXME does riscv need this to be cmpxchg? */
+		WRITE_ONCE(tablep[pts->index], entry);
+	}
+	pts->entry = entry;
+}
+#define pt_install_leaf_entry riscvpt_install_leaf_entry
+
+static inline bool riscvpt_install_table(struct pt_state *pts,
+					 pt_oaddr_t table_pa,
+					 const struct pt_write_attrs *attrs)
+{
+	pt_riscv_entry_t entry;
+
+	entry = RISCVPT_V |
+		FIELD_PREP(RISCVPT_PPN, log2_div(table_pa, PT_GRANULE_LG2SZ));
+	return pt_table_install64(pts, entry);
+}
+#define pt_install_table riscvpt_install_table
+
+static inline void riscvpt_attr_from_entry(const struct pt_state *pts,
+					   struct pt_write_attrs *attrs)
+{
+	attrs->descriptor_bits =
+		pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U |
+			      RISCVPT_G | RISCVPT_A | RISCVPT_D);
+}
+#define pt_attr_from_entry riscvpt_attr_from_entry
+
+/* --- iommu */
+#include <linux/generic_pt/iommu.h>
+#include <linux/iommu.h>
+
+#define pt_iommu_table pt_iommu_riscv_64
+
+/* The common struct is in the per-format common struct */
+static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table)
+{
+	return &container_of(iommu_table, struct pt_iommu_table, iommu)
+			->riscv_64pt.common;
+}
+
+static inline struct pt_iommu *iommu_from_common(struct pt_common *common)
+{
+	return &container_of(common, struct pt_iommu_table, riscv_64pt.common)
+			->iommu;
+}
+
+static inline int riscvpt_iommu_set_prot(struct pt_common *common,
+					 struct pt_write_attrs *attrs,
+					 unsigned int iommu_prot)
+{
+	u64 pte;
+
+	pte = RISCVPT_A | RISCVPT_U;
+	if (iommu_prot & IOMMU_WRITE)
+		pte |= RISCVPT_W | RISCVPT_R | RISCVPT_D;
+	if (iommu_prot & IOMMU_READ)
+		pte |= RISCVPT_R;
+	if (!(iommu_prot & IOMMU_NOEXEC))
+		pte |= RISCVPT_X;
+
+	/* Caller must specify a supported combination of flags */
+	if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0))
+		return -EOPNOTSUPP;
+
+	attrs->descriptor_bits = pte;
+	return 0;
+}
+#define pt_iommu_set_prot riscvpt_iommu_set_prot
+
+static inline int
+riscvpt_iommu_fmt_init(struct pt_iommu_riscv_64 *iommu_table,
+		       const struct pt_iommu_riscv_64_cfg *cfg)
+{
+	struct pt_riscv *table = &iommu_table->riscv_64pt;
+
+	switch (cfg->common.hw_max_vasz_lg2) {
+	case 39:
+		pt_top_set_level(&table->common, 2);
+		break;
+	case 48:
+		pt_top_set_level(&table->common, 3);
+		break;
+	case 57:
+		pt_top_set_level(&table->common, 4);
+		break;
+	default:
+		return -EINVAL;
+	}
+	table->common.max_oasz_lg2 =
+		min(PT_MAX_OUTPUT_ADDRESS_LG2, cfg->common.hw_max_oasz_lg2);
+	return 0;
+}
+#define pt_iommu_fmt_init riscvpt_iommu_fmt_init
+
+static inline void
+riscvpt_iommu_fmt_hw_info(struct pt_iommu_riscv_64 *table,
+			  const struct pt_range *top_range,
+			  struct pt_iommu_riscv_64_hw_info *info)
+{
+	phys_addr_t top_phys = virt_to_phys(top_range->top_table);
+
+	info->ppn = oalog2_div(top_phys, PT_GRANULE_LG2SZ);
+	PT_WARN_ON(top_phys & ~PT_TOP_PHYS_MASK);
+
+	/*
+	 * See Table 3. Encodings of iosatp.MODE field" for DC.tx.SXL = 0:
+	 *  8 = Sv39 = top level 2
+	 *  9 = Sv38 = top level 3
+	 *  10 = Sv57 = top level 4
+	 */
+	info->fsc_iosatp_mode = top_range->top_level + 6;
+}
+#define pt_iommu_fmt_hw_info riscvpt_iommu_fmt_hw_info
+
+#if defined(GENERIC_PT_KUNIT)
+static const struct pt_iommu_riscv_64_cfg riscv_64_kunit_fmt_cfgs[] = {
+	[0] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
+		.common.hw_max_oasz_lg2 = 56,
+		.common.hw_max_vasz_lg2 = 39 },
+	[1] = { .common.features = 0,
+		.common.hw_max_oasz_lg2 = 56,
+		.common.hw_max_vasz_lg2 = 48 },
+	[2] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
+		.common.hw_max_oasz_lg2 = 56,
+		.common.hw_max_vasz_lg2 = 57 },
+};
+#define kunit_fmt_cfgs riscv_64_kunit_fmt_cfgs
+enum {
+	KUNIT_FMT_FEATURES = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
+};
+#endif
+
+#endif
diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 3e33fe64feab2..17b72dbd7d51c 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -51,16 +51,27 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
 		iommu_pages_stop_incoherent_list(free_list,
 						 iommu_table->iommu_device);
 
-	if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) &&
-	    iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) {
-		iommu_iotlb_sync(&iommu_table->domain, iotlb_gather);
-		/*
-		 * Note that the sync frees the gather's free list, so we must
-		 * not have any pages on that list that are covered by iova/len
-		 */
+	/*
+	 * If running in DMA-FQ mode then the unmap will be followed by an IOTLB
+	 * flush all so we need to optimize by never flushing the IOTLB here.
+	 *
+	 * For NO_GAPS the user gets to pick if flushing all or doing micro
+	 * flushes is better for their work load by choosing DMA vs DMA-FQ
+	 * operation. Drivers should also see shadow_on_flush.
+	 */
+	if (!iommu_iotlb_gather_queued(iotlb_gather)) {
+		if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) &&
+		    iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) {
+			iommu_iotlb_sync(&iommu_table->domain, iotlb_gather);
+			/*
+			 * Note that the sync frees the gather's free list, so
+			 * we must not have any pages on that list that are
+			 * covered by iova/len
+			 */
+		}
+		iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
 	}
 
-	iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
 	iommu_pages_list_splice(free_list, &iotlb_gather->freelist);
 }
 
@@ -466,6 +477,7 @@ struct pt_iommu_map_args {
 	pt_oaddr_t oa;
 	unsigned int leaf_pgsize_lg2;
 	unsigned int leaf_level;
+	pt_vaddr_t num_leaves;
 };
 
 /*
@@ -518,11 +530,15 @@ static int clear_contig(const struct pt_state *start_pts,
 static int __map_range_leaf(struct pt_range *range, void *arg,
 			    unsigned int level, struct pt_table_p *table)
 {
+	struct pt_iommu *iommu_table = iommu_from_common(range->common);
 	struct pt_state pts = pt_init(range, level, table);
 	struct pt_iommu_map_args *map = arg;
 	unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2;
 	unsigned int start_index;
 	pt_oaddr_t oa = map->oa;
+	unsigned int num_leaves;
+	unsigned int orig_end;
+	pt_vaddr_t last_va;
 	unsigned int step;
 	bool need_contig;
 	int ret = 0;
@@ -536,6 +552,15 @@ static int __map_range_leaf(struct pt_range *range, void *arg,
 
 	_pt_iter_first(&pts);
 	start_index = pts.index;
+	orig_end = pts.end_index;
+	if (pts.index + map->num_leaves < pts.end_index) {
+		/* Need to stop in the middle of the table to change sizes */
+		pts.end_index = pts.index + map->num_leaves;
+		num_leaves = 0;
+	} else {
+		num_leaves = map->num_leaves - (pts.end_index - pts.index);
+	}
+
 	do {
 		pts.type = pt_load_entry_raw(&pts);
 		if (pts.type != PT_ENTRY_EMPTY || need_contig) {
@@ -561,7 +586,40 @@ static int __map_range_leaf(struct pt_range *range, void *arg,
 	flush_writes_range(&pts, start_index, pts.index);
 
 	map->oa = oa;
-	return ret;
+	map->num_leaves = num_leaves;
+	if (ret || num_leaves)
+		return ret;
+
+	/* range->va is not valid if we reached the end of the table */
+	pts.index -= step;
+	pt_index_to_va(&pts);
+	pts.index += step;
+	last_va = range->va + log2_to_int(leaf_pgsize_lg2);
+
+	if (last_va - 1 == range->last_va) {
+		PT_WARN_ON(pts.index != orig_end);
+		return 0;
+	}
+
+	/*
+	 * Reached a point where the page size changed, compute the new
+	 * parameters.
+	 */
+	map->leaf_pgsize_lg2 = pt_compute_best_pgsize(
+		iommu_table->domain.pgsize_bitmap, last_va, range->last_va, oa);
+	map->leaf_level =
+		pt_pgsz_lg2_to_level(range->common, map->leaf_pgsize_lg2);
+	map->num_leaves = pt_pgsz_count(iommu_table->domain.pgsize_bitmap,
+					last_va, range->last_va, oa,
+					map->leaf_pgsize_lg2);
+
+	/* Didn't finish this table level, caller will repeat it */
+	if (pts.index != orig_end) {
+		if (pts.index != start_index)
+			pt_index_to_va(&pts);
+		return -EAGAIN;
+	}
+	return 0;
 }
 
 static int __map_range(struct pt_range *range, void *arg, unsigned int level,
@@ -584,14 +642,9 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level,
 			if (pts.type != PT_ENTRY_EMPTY)
 				return -EADDRINUSE;
 			ret = pt_iommu_new_table(&pts, &map->attrs);
-			if (ret) {
-				/*
-				 * Racing with another thread installing a table
-				 */
-				if (ret == -EAGAIN)
-					continue;
+			/* EAGAIN on a race will loop again */
+			if (ret)
 				return ret;
-			}
 		} else {
 			pts.table_lower = pt_table_ptr(&pts);
 			/*
@@ -615,10 +668,12 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level,
 		 * The already present table can possibly be shared with another
 		 * concurrent map.
 		 */
-		if (map->leaf_level == level - 1)
-			ret = pt_descend(&pts, arg, __map_range_leaf);
-		else
-			ret = pt_descend(&pts, arg, __map_range);
+		do {
+			if (map->leaf_level == level - 1)
+				ret = pt_descend(&pts, arg, __map_range_leaf);
+			else
+				ret = pt_descend(&pts, arg, __map_range);
+		} while (ret == -EAGAIN);
 		if (ret)
 			return ret;
 
@@ -626,6 +681,14 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level,
 		pt_index_to_va(&pts);
 		if (pts.index >= pts.end_index)
 			break;
+
+		/*
+		 * This level is currently running __map_range_leaf() which is
+		 * not correct if the target level has been updated to this
+		 * level. Have the caller invoke __map_range_leaf.
+		 */
+		if (map->leaf_level == level)
+			return -EAGAIN;
 	} while (true);
 	return 0;
 }
@@ -797,12 +860,13 @@ static int check_map_range(struct pt_iommu *iommu_table, struct pt_range *range,
 static int do_map(struct pt_range *range, struct pt_common *common,
 		  bool single_page, struct pt_iommu_map_args *map)
 {
+	int ret;
+
 	/*
 	 * The __map_single_page() fast path does not support DMA_INCOHERENT
 	 * flushing to keep its .text small.
 	 */
 	if (single_page && !pt_feature(common, PT_FEAT_DMA_INCOHERENT)) {
-		int ret;
 
 		ret = pt_walk_range(range, __map_single_page, map);
 		if (ret != -EAGAIN)
@@ -810,50 +874,25 @@ static int do_map(struct pt_range *range, struct pt_common *common,
 		/* EAGAIN falls through to the full path */
 	}
 
-	if (map->leaf_level == range->top_level)
-		return pt_walk_range(range, __map_range_leaf, map);
-	return pt_walk_range(range, __map_range, map);
+	do {
+		if (map->leaf_level == range->top_level)
+			ret = pt_walk_range(range, __map_range_leaf, map);
+		else
+			ret = pt_walk_range(range, __map_range, map);
+	} while (ret == -EAGAIN);
+	return ret;
 }
 
-/**
- * map_pages() - Install translation for an IOVA range
- * @domain: Domain to manipulate
- * @iova: IO virtual address to start
- * @paddr: Physical/Output address to start
- * @pgsize: Length of each page
- * @pgcount: Length of the range in pgsize units starting from @iova
- * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO
- * @gfp: GFP flags for any memory allocations
- * @mapped: Total bytes successfully mapped
- *
- * The range starting at IOVA will have paddr installed into it. The caller
- * must specify a valid pgsize and pgcount to segment the range into compatible
- * blocks.
- *
- * On error the caller will probably want to invoke unmap on the range from iova
- * up to the amount indicated by @mapped to return the table back to an
- * unchanged state.
- *
- * Context: The caller must hold a write range lock that includes the whole
- * range.
- *
- * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA that were
- * mapped are added to @mapped, @mapped is not zerod first.
- */
-int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova,
-			 phys_addr_t paddr, size_t pgsize, size_t pgcount,
-			 int prot, gfp_t gfp, size_t *mapped)
+static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+			 phys_addr_t paddr, dma_addr_t len, unsigned int prot,
+			 gfp_t gfp, size_t *mapped)
 {
-	struct pt_iommu *iommu_table =
-		container_of(domain, struct pt_iommu, domain);
 	pt_vaddr_t pgsize_bitmap = iommu_table->domain.pgsize_bitmap;
 	struct pt_common *common = common_from_iommu(iommu_table);
 	struct iommu_iotlb_gather iotlb_gather;
-	pt_vaddr_t len = pgsize * pgcount;
 	struct pt_iommu_map_args map = {
 		.iotlb_gather = &iotlb_gather,
 		.oa = paddr,
-		.leaf_pgsize_lg2 = vaffs(pgsize),
 	};
 	bool single_page = false;
 	struct pt_range range;
@@ -881,13 +920,13 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova,
 		return ret;
 
 	/* Calculate target page size and level for the leaves */
-	if (pt_has_system_page_size(common) && pgsize == PAGE_SIZE &&
-	    pgcount == 1) {
+	if (pt_has_system_page_size(common) && len == PAGE_SIZE) {
 		PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE));
 		if (log2_mod(iova | paddr, PAGE_SHIFT))
 			return -ENXIO;
 		map.leaf_pgsize_lg2 = PAGE_SHIFT;
 		map.leaf_level = 0;
+		map.num_leaves = 1;
 		single_page = true;
 	} else {
 		map.leaf_pgsize_lg2 = pt_compute_best_pgsize(
@@ -896,6 +935,9 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova,
 			return -ENXIO;
 		map.leaf_level =
 			pt_pgsz_lg2_to_level(common, map.leaf_pgsize_lg2);
+		map.num_leaves = pt_pgsz_count(pgsize_bitmap, range.va,
+					       range.last_va, paddr,
+					       map.leaf_pgsize_lg2);
 	}
 
 	ret = check_map_range(iommu_table, &range, &map);
@@ -918,7 +960,6 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova,
 	*mapped += map.oa - paddr;
 	return ret;
 }
-EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(map_pages), "GENERIC_PT_IOMMU");
 
 struct pt_unmap_args {
 	struct iommu_pages_list free_list;
@@ -1020,34 +1061,12 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg,
 	return ret;
 }
 
-/**
- * unmap_pages() - Make a range of IOVA empty/not present
- * @domain: Domain to manipulate
- * @iova: IO virtual address to start
- * @pgsize: Length of each page
- * @pgcount: Length of the range in pgsize units starting from @iova
- * @iotlb_gather: Gather struct that must be flushed on return
- *
- * unmap_pages() will remove a translation created by map_pages(). It cannot
- * subdivide a mapping created by map_pages(), so it should be called with IOVA
- * ranges that match those passed to map_pages(). The IOVA range can aggregate
- * contiguous map_pages() calls so long as no individual range is split.
- *
- * Context: The caller must hold a write range lock that includes
- * the whole range.
- *
- * Returns: Number of bytes of VA unmapped. iova + res will be the point
- * unmapping stopped.
- */
-size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova,
-			      size_t pgsize, size_t pgcount,
+static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+			      dma_addr_t len,
 			      struct iommu_iotlb_gather *iotlb_gather)
 {
-	struct pt_iommu *iommu_table =
-		container_of(domain, struct pt_iommu, domain);
 	struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT(
 					       unmap.free_list) };
-	pt_vaddr_t len = pgsize * pgcount;
 	struct pt_range range;
 	int ret;
 
@@ -1062,7 +1081,6 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova,
 
 	return unmap.unmapped;
 }
-EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(unmap_pages), "GENERIC_PT_IOMMU");
 
 static void NS(get_info)(struct pt_iommu *iommu_table,
 			 struct pt_iommu_info *info)
@@ -1110,6 +1128,8 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
 }
 
 static const struct pt_iommu_ops NS(ops) = {
+	.map_range = NS(map_range),
+	.unmap_range = NS(unmap_range),
 #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \
 	IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty)
 	.set_dirty = NS(set_dirty),
@@ -1172,6 +1192,7 @@ static int pt_iommu_init_domain(struct pt_iommu *iommu_table,
 
 	domain->type = __IOMMU_DOMAIN_PAGING;
 	domain->pgsize_bitmap = info.pgsize_bitmap;
+	domain->is_iommupt = true;
 
 	if (pt_feature(common, PT_FEAT_DYNAMIC_TOP))
 		range = _pt_top_range(common,
diff --git a/drivers/iommu/generic_pt/kunit_generic_pt.h b/drivers/iommu/generic_pt/kunit_generic_pt.h
index 68278bf15cfe0..374e475f591e1 100644
--- a/drivers/iommu/generic_pt/kunit_generic_pt.h
+++ b/drivers/iommu/generic_pt/kunit_generic_pt.h
@@ -312,6 +312,17 @@ static void test_best_pgsize(struct kunit *test)
 	}
 }
 
+static void test_pgsz_count(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test,
+			pt_pgsz_count(SZ_4K, 0, SZ_1G - 1, 0, ilog2(SZ_4K)),
+			SZ_1G / SZ_4K);
+	KUNIT_EXPECT_EQ(test,
+			pt_pgsz_count(SZ_2M | SZ_4K, SZ_4K, SZ_1G - 1, SZ_4K,
+				      ilog2(SZ_4K)),
+			(SZ_2M - SZ_4K) / SZ_4K);
+}
+
 /*
  * Check that pt_install_table() and pt_table_pa() match
  */
@@ -770,6 +781,7 @@ static struct kunit_case generic_pt_test_cases[] = {
 	KUNIT_CASE_FMT(test_init),
 	KUNIT_CASE_FMT(test_bitops),
 	KUNIT_CASE_FMT(test_best_pgsize),
+	KUNIT_CASE_FMT(test_pgsz_count),
 	KUNIT_CASE_FMT(test_table_ptr),
 	KUNIT_CASE_FMT(test_max_va),
 	KUNIT_CASE_FMT(test_table_radix),
diff --git a/drivers/iommu/generic_pt/pt_iter.h b/drivers/iommu/generic_pt/pt_iter.h
index c0d8617cce292..3e45dbde6b832 100644
--- a/drivers/iommu/generic_pt/pt_iter.h
+++ b/drivers/iommu/generic_pt/pt_iter.h
@@ -569,6 +569,28 @@ static inline unsigned int pt_compute_best_pgsize(pt_vaddr_t pgsz_bitmap,
 	return pgsz_lg2;
 }
 
+/*
+ * Return the number of pgsize_lg2 leaf entries that can be mapped for
+ * va to oa. This accounts for any requirement to reduce or increase the page
+ * size across the VA range.
+ */
+static inline pt_vaddr_t pt_pgsz_count(pt_vaddr_t pgsz_bitmap, pt_vaddr_t va,
+				       pt_vaddr_t last_va, pt_oaddr_t oa,
+				       unsigned int pgsize_lg2)
+{
+	pt_vaddr_t len = last_va - va + 1;
+	pt_vaddr_t next_pgsizes = log2_set_mod(pgsz_bitmap, 0, pgsize_lg2 + 1);
+
+	if (next_pgsizes) {
+		unsigned int next_pgsize_lg2 = vaffs(next_pgsizes);
+
+		if (log2_mod(va ^ oa, next_pgsize_lg2) == 0)
+			len = min(len, log2_set_mod_max(va, next_pgsize_lg2) -
+					       va + 1);
+	}
+	return log2_div(len, pgsize_lg2);
+}
+
 #define _PT_MAKE_CALL_LEVEL(fn)                                          \
 	static __always_inline int fn(struct pt_range *range, void *arg, \
 				      unsigned int level,                \
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index d68c06025cac2..69222dbd2af0e 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -1314,7 +1314,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
 	if (fault & DMA_FSTS_ITE) {
 		head = readl(iommu->reg + DMAR_IQH_REG);
 		head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
-		head |= 1;
 		tail = readl(iommu->reg + DMAR_IQT_REG);
 		tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
 
@@ -1331,7 +1330,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
 		do {
 			if (qi->desc_status[head] == QI_IN_USE)
 				qi->desc_status[head] = QI_ABORT;
-			head = (head - 2 + QI_LENGTH) % QI_LENGTH;
+			head = (head - 1 + QI_LENGTH) % QI_LENGTH;
 		} while (head != tail);
 
 		/*
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index ef7613b177b9a..5dca8e525c73c 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -3220,6 +3220,8 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
 		return ecap_sc_support(info->iommu->ecap);
 	case IOMMU_CAP_DIRTY_TRACKING:
 		return ssads_supported(info->iommu);
+	case IOMMU_CAP_PCI_ATS_SUPPORTED:
+		return info->ats_supported;
 	default:
 		return false;
 	}
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index fea10acd4f021..57cd1db7207ab 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -164,9 +164,12 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
 	if (IS_ERR(dev_pasid))
 		return PTR_ERR(dev_pasid);
 
-	ret = iopf_for_domain_replace(domain, old, dev);
-	if (ret)
-		goto out_remove_dev_pasid;
+	/* SVA with non-IOMMU/PRI IOPF handling is allowed. */
+	if (info->pri_supported) {
+		ret = iopf_for_domain_replace(domain, old, dev);
+		if (ret)
+			goto out_remove_dev_pasid;
+	}
 
 	/* Setup the pasid table: */
 	sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
@@ -181,7 +184,8 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
 
 	return 0;
 out_unwind_iopf:
-	iopf_for_domain_replace(old, domain, dev);
+	if (info->pri_supported)
+		iopf_for_domain_replace(old, domain, dev);
 out_remove_dev_pasid:
 	domain_remove_dev_pasid(domain, dev, pasid);
 	return ret;
diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c
index 07d64908a05fd..bc7c7232a43e2 100644
--- a/drivers/iommu/iommu-sva.c
+++ b/drivers/iommu/iommu-sva.c
@@ -182,13 +182,13 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
 	iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
 	if (--domain->users == 0) {
 		list_del(&domain->next);
-		iommu_domain_free(domain);
-	}
+		if (list_empty(&iommu_mm->sva_domains)) {
+			list_del(&iommu_mm->mm_list_elm);
+			if (list_empty(&iommu_sva_mms))
+				iommu_sva_present = false;
+		}
 
-	if (list_empty(&iommu_mm->sva_domains)) {
-		list_del(&iommu_mm->mm_list_elm);
-		if (list_empty(&iommu_sva_mms))
-			iommu_sva_present = false;
+		iommu_domain_free(domain);
 	}
 
 	mutex_unlock(&iommu_sva_lock);
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 35db517809540..78756c3f3c405 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -34,6 +34,7 @@
 #include <linux/sched/mm.h>
 #include <linux/msi.h>
 #include <uapi/linux/iommufd.h>
+#include <linux/generic_pt/iommu.h>
 
 #include "dma-iommu.h"
 #include "iommu-priv.h"
@@ -1213,7 +1214,11 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
 			if (addr == end)
 				goto map_end;
 
-			phys_addr = iommu_iova_to_phys(domain, addr);
+			/*
+			 * Return address by iommu_iova_to_phys for 0 is
+			 * ambiguous. Offset to address 1 if addr is 0.
+			 */
+			phys_addr = iommu_iova_to_phys(domain, addr ? addr : 1);
 			if (!phys_addr) {
 				map_size += pg_size;
 				continue;
@@ -2568,14 +2573,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
 	return pgsize;
 }
 
-int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
-		phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
+static int __iommu_map_domain_pgtbl(struct iommu_domain *domain,
+				    unsigned long iova, phys_addr_t paddr,
+				    size_t size, int prot, gfp_t gfp)
 {
 	const struct iommu_domain_ops *ops = domain->ops;
 	unsigned long orig_iova = iova;
 	unsigned int min_pagesz;
 	size_t orig_size = size;
-	phys_addr_t orig_paddr = paddr;
 	int ret = 0;
 
 	might_sleep_if(gfpflags_allow_blocking(gfp));
@@ -2632,12 +2637,9 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
 	/* unroll mapping in case something went wrong */
 	if (ret) {
 		iommu_unmap(domain, orig_iova, orig_size - size);
-	} else {
-		trace_map(orig_iova, orig_paddr, orig_size);
-		iommu_debug_map(domain, orig_paddr, orig_size);
+		return ret;
 	}
-
-	return ret;
+	return 0;
 }
 
 int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size)
@@ -2649,6 +2651,32 @@ int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size)
 	return ops->iotlb_sync_map(domain, iova, size);
 }
 
+int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
+		phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
+{
+	struct pt_iommu *pt = iommupt_from_domain(domain);
+	int ret;
+
+	if (pt) {
+		size_t mapped = 0;
+
+		ret = pt->ops->map_range(pt, iova, paddr, size, prot, gfp,
+					 &mapped);
+		if (ret) {
+			iommu_unmap(domain, iova, mapped);
+			return ret;
+		}
+		return 0;
+	}
+	ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp);
+	if (!ret)
+		return ret;
+
+	trace_map(iova, paddr, size);
+	iommu_debug_map(domain, paddr, size);
+	return 0;
+}
+
 int iommu_map(struct iommu_domain *domain, unsigned long iova,
 	      phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
@@ -2666,13 +2694,12 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 EXPORT_SYMBOL_GPL(iommu_map);
 
-static size_t __iommu_unmap(struct iommu_domain *domain,
-			    unsigned long iova, size_t size,
-			    struct iommu_iotlb_gather *iotlb_gather)
+static size_t
+__iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova,
+			   size_t size, struct iommu_iotlb_gather *iotlb_gather)
 {
 	const struct iommu_domain_ops *ops = domain->ops;
 	size_t unmapped_page, unmapped = 0;
-	unsigned long orig_iova = iova;
 	unsigned int min_pagesz;
 
 	if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
@@ -2718,8 +2745,23 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
 		unmapped += unmapped_page;
 	}
 
-	trace_unmap(orig_iova, size, unmapped);
-	iommu_debug_unmap_end(domain, orig_iova, size, unmapped);
+	return unmapped;
+}
+
+static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+			    size_t size,
+			    struct iommu_iotlb_gather *iotlb_gather)
+{
+	struct pt_iommu *pt = iommupt_from_domain(domain);
+	size_t unmapped;
+
+	if (pt)
+		unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather);
+	else
+		unmapped = __iommu_unmap_domain_pgtbl(domain, iova, size,
+						      iotlb_gather);
+	trace_unmap(iova, size, unmapped);
+	iommu_debug_unmap_end(domain, iova, size, unmapped);
 	return unmapped;
 }
 
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 344d620cdecc4..92c5d5ef8d009 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -1624,6 +1624,10 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
 	if (device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING))
 		cmd->out_capabilities |= IOMMU_HW_CAP_DIRTY_TRACKING;
 
+	/* Report when ATS cannot be used for this device */
+	if (!device_iommu_capable(idev->dev, IOMMU_CAP_PCI_ATS_SUPPORTED))
+		cmd->out_capabilities |= IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED;
+
 	cmd->out_max_pasid_log2 = 0;
 	/*
 	 * Currently, all iommu drivers enable PASID in the probe_device()
diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c
index 9b49f0c5b4599..9bdb2945afe1e 100644
--- a/drivers/iommu/iommufd/pages.c
+++ b/drivers/iommu/iommufd/pages.c
@@ -1450,9 +1450,9 @@ static void iopt_revoke_notify(struct dma_buf_attachment *attach)
 	pages->dmabuf.phys.len = 0;
 }
 
-static struct dma_buf_attach_ops iopt_dmabuf_attach_revoke_ops = {
+static const struct dma_buf_attach_ops iopt_dmabuf_attach_revoke_ops = {
 	.allow_peer2peer = true,
-	.move_notify = iopt_revoke_notify,
+	.invalidate_mappings = iopt_revoke_notify,
 };
 
 /*
@@ -1502,16 +1502,22 @@ static int iopt_map_dmabuf(struct iommufd_ctx *ictx, struct iopt_pages *pages,
 		mutex_unlock(&pages->mutex);
 	}
 
-	rc = sym_vfio_pci_dma_buf_iommufd_map(attach, &pages->dmabuf.phys);
+	rc = dma_buf_pin(attach);
 	if (rc)
 		goto err_detach;
 
+	rc = sym_vfio_pci_dma_buf_iommufd_map(attach, &pages->dmabuf.phys);
+	if (rc)
+		goto err_unpin;
+
 	dma_resv_unlock(dmabuf->resv);
 
 	/* On success iopt_release_pages() will detach and put the dmabuf. */
 	pages->dmabuf.attach = attach;
 	return 0;
 
+err_unpin:
+	dma_buf_unpin(attach);
 err_detach:
 	dma_resv_unlock(dmabuf->resv);
 	dma_buf_detach(dmabuf, attach);
@@ -1657,6 +1663,7 @@ void iopt_release_pages(struct kref *kref)
 	if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) {
 		struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf;
 
+		dma_buf_unpin(pages->dmabuf.attach);
 		dma_buf_detach(dmabuf, pages->dmabuf.attach);
 		dma_buf_put(dmabuf);
 		WARN_ON(!list_empty(&pages->dmabuf.tracker));
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index 7823142097d47..9607416f80695 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -636,7 +636,7 @@ static void mock_viommu_destroy(struct iommufd_viommu *viommu)
 	if (mock_viommu->mmap_offset)
 		iommufd_viommu_destroy_mmap(&mock_viommu->core,
 					    mock_viommu->mmap_offset);
-	free_page((unsigned long)mock_viommu->page);
+	free_pages((unsigned long)mock_viommu->page, 1);
 	mutex_destroy(&mock_viommu->queue_mutex);
 
 	/* iommufd core frees mock_viommu and viommu */
@@ -870,7 +870,7 @@ static int mock_viommu_init(struct iommufd_viommu *viommu,
 	iommufd_viommu_destroy_mmap(&mock_viommu->core,
 				    mock_viommu->mmap_offset);
 err_free_page:
-	free_page((unsigned long)mock_viommu->page);
+	free_pages((unsigned long)mock_viommu->page, 1);
 	return rc;
 }
 
@@ -2081,7 +2081,7 @@ static int iommufd_test_dmabuf_revoke(struct iommufd_ucmd *ucmd, int fd,
 	priv = dmabuf->priv;
 	dma_resv_lock(dmabuf->resv, NULL);
 	priv->revoked = revoked;
-	dma_buf_move_notify(dmabuf);
+	dma_buf_invalidate_mappings(dmabuf);
 	dma_resv_unlock(dmabuf->resv);
 
 err_put:
diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c
index a258ee2f4579f..acb48cdd3b005 100644
--- a/drivers/iommu/iommufd/vfio_compat.c
+++ b/drivers/iommu/iommufd/vfio_compat.c
@@ -283,7 +283,7 @@ static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx,
 	case VFIO_TYPE1_IOMMU:
 	case VFIO_TYPE1v2_IOMMU:
 	case VFIO_UNMAP_ALL:
-		return 1;
+		return !ictx->no_iommu_mode;
 
 	case VFIO_NOIOMMU_IOMMU:
 		return IS_ENABLED(CONFIG_VFIO_NOIOMMU);
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index f9cd18316d16b..021daf6528de1 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -611,7 +611,8 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
 
 static void iova_magazine_free(struct iova_magazine *mag)
 {
-	kmem_cache_free(iova_magazine_cache, mag);
+	if (mag)
+		kmem_cache_free(iova_magazine_cache, mag);
 }
 
 static void
diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
index c071816f59a67..b86e5ab941839 100644
--- a/drivers/iommu/riscv/Kconfig
+++ b/drivers/iommu/riscv/Kconfig
@@ -3,9 +3,13 @@
 
 config RISCV_IOMMU
 	bool "RISC-V IOMMU Support"
-	depends on RISCV && 64BIT
-	default y
+	default RISCV
+	depends on GENERIC_MSI_IRQ
+	depends on (RISCV || COMPILE_TEST) && 64BIT
 	select IOMMU_API
+	select GENERIC_PT
+	select IOMMU_PT
+	select IOMMU_PT_RISCV64
 	help
 	  Support for implementations of the RISC-V IOMMU architecture that
 	  complements the RISC-V MMU capabilities, providing similar address
diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
index 98daf0e1a3069..29a0040b1c32e 100644
--- a/drivers/iommu/riscv/iommu-bits.h
+++ b/drivers/iommu/riscv/iommu-bits.h
@@ -17,6 +17,7 @@
 #include <linux/types.h>
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <asm/page.h>
 
 /*
  * Chapter 5: Memory Mapped register interface
@@ -718,7 +719,8 @@ static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd)
 static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cmd,
 						  u64 addr)
 {
-	cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, phys_to_pfn(addr));
+	cmd->dword1 =
+		FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, PHYS_PFN(addr));
 	cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
 }
 
diff --git a/drivers/iommu/riscv/iommu-platform.c b/drivers/iommu/riscv/iommu-platform.c
index 83a28c83f9914..8f15b06e84997 100644
--- a/drivers/iommu/riscv/iommu-platform.c
+++ b/drivers/iommu/riscv/iommu-platform.c
@@ -68,12 +68,7 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev)
 	iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
 	iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
 
-	iommu->irqs_count = platform_irq_count(pdev);
-	if (iommu->irqs_count <= 0)
-		return dev_err_probe(dev, -ENODEV,
-				     "no IRQ resources provided\n");
-	if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)
-		iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
+	iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
 
 	igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps);
 	switch (igs) {
@@ -120,6 +115,13 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev)
 		fallthrough;
 
 	case RISCV_IOMMU_CAPABILITIES_IGS_WSI:
+		iommu->irqs_count = platform_irq_count(pdev);
+		if (iommu->irqs_count <= 0)
+			return dev_err_probe(dev, -ENODEV,
+					     "no IRQ resources provided\n");
+		if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)
+			iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
+
 		for (vec = 0; vec < iommu->irqs_count; vec++)
 			iommu->irqs[vec] = platform_get_irq(pdev, vec);
 
diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index fa2ebfd2f912e..368f3cbd2d0a9 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -21,6 +21,7 @@
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
+#include <linux/generic_pt/iommu.h>
 
 #include "../iommu-pages.h"
 #include "iommu-bits.h"
@@ -159,7 +160,7 @@ static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu,
 	if (FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb)) {
 		const size_t queue_size = entry_size << (logsz + 1);
 
-		queue->phys = pfn_to_phys(FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb));
+		queue->phys = PFN_PHYS(FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb));
 		queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size);
 	} else {
 		do {
@@ -368,6 +369,8 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue,
 				  unsigned int timeout_us)
 {
 	unsigned int cons = atomic_read(&queue->head);
+	unsigned int flags = RISCV_IOMMU_CQCSR_CQMF | RISCV_IOMMU_CQCSR_CMD_TO |
+			     RISCV_IOMMU_CQCSR_CMD_ILL;
 
 	/* Already processed by the consumer */
 	if ((int)(cons - index) > 0)
@@ -375,6 +378,7 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue,
 
 	/* Monitor consumer index */
 	return readx_poll_timeout(riscv_iommu_queue_cons, queue, cons,
+				 (riscv_iommu_readl(queue->iommu, queue->qcr) & flags) ||
 				 (int)(cons - index) > 0, 0, timeout_us);
 }
 
@@ -435,7 +439,9 @@ static unsigned int riscv_iommu_queue_send(struct riscv_iommu_queue *queue,
 	 * 6. Make sure the doorbell write to the device has finished before updating
 	 *    the shadow tail index in normal memory. 'fence o, w'
 	 */
+#ifdef CONFIG_MMIOWB
 	mmiowb();
+#endif
 	atomic_inc(&queue->tail);
 
 	/* 7. Complete submission and restore local interrupts */
@@ -806,15 +812,15 @@ static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu,
 
 /* This struct contains protection domain specific IOMMU driver data. */
 struct riscv_iommu_domain {
-	struct iommu_domain domain;
+	union {
+		struct iommu_domain domain;
+		struct pt_iommu_riscv_64 riscvpt;
+	};
 	struct list_head bonds;
 	spinlock_t lock;		/* protect bonds list updates. */
 	int pscid;
-	bool amo_enabled;
-	int numa_node;
-	unsigned int pgd_mode;
-	unsigned long *pgd_root;
 };
+PT_IOMMU_CHECK_DOMAIN(struct riscv_iommu_domain, riscvpt.iommu, domain);
 
 #define iommu_domain_to_riscv(iommu_domain) \
 	container_of(iommu_domain, struct riscv_iommu_domain, domain)
@@ -996,7 +1002,67 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
 }
 
 #define RISCV_IOMMU_FSC_BARE 0
+/*
+ * This function sends IOTINVAL commands as required by the RISC-V
+ * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version)
+ * after modifying DDT or PDT entries
+ */
+static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu,
+				       bool inval_pdt, unsigned long iohgatp,
+				       struct riscv_iommu_dc *dc,
+				       struct riscv_iommu_pc *pc)
+{
+	struct riscv_iommu_command cmd;
 
+	riscv_iommu_cmd_inval_vma(&cmd);
+
+	if (FIELD_GET(RISCV_IOMMU_DC_IOHGATP_MODE, iohgatp) ==
+	    RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
+		if (inval_pdt) {
+			/*
+			 * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
+			 * PSCID=PC.PSCID
+			 */
+			riscv_iommu_cmd_inval_set_pscid(&cmd,
+				FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
+		} else {
+			if (!FIELD_GET(RISCV_IOMMU_DC_TC_PDTV, dc->tc) &&
+			    FIELD_GET(RISCV_IOMMU_DC_FSC_MODE, dc->fsc) !=
+			    RISCV_IOMMU_DC_FSC_MODE_BARE) {
+				/*
+				 * DC.tc.PDTV == 0 && DC.fsc.MODE != Bare
+				 * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
+				 * PSCID=DC.ta.PSCID
+				 */
+				riscv_iommu_cmd_inval_set_pscid(&cmd,
+					FIELD_GET(RISCV_IOMMU_DC_TA_PSCID, dc->ta));
+			}
+			/* else: IOTINVAL.VMA with GV=AV=PSCV=0 */
+		}
+	} else {
+		riscv_iommu_cmd_inval_set_gscid(&cmd,
+			FIELD_GET(RISCV_IOMMU_DC_IOHGATP_GSCID, iohgatp));
+
+		if (inval_pdt) {
+			/*
+			 * IOTINVAL.VMA with GV=1, AV=0, and PSCV=1, and
+			 * GSCID=DC.iohgatp.GSCID, PSCID=PC.PSCID
+			 */
+			riscv_iommu_cmd_inval_set_pscid(&cmd,
+				FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
+		}
+		/*
+		 * else: IOTINVAL.VMA with GV=1,AV=PSCV=0,and
+		 * GSCID=DC.iohgatp.GSCID
+		 *
+		 * IOTINVAL.GVMA with GV=1,AV=0,and
+		 * GSCID=DC.iohgatp.GSCID
+		 * TODO: For now, the Second-Stage feature have not yet been merged,
+		 * also issue IOTINVAL.GVMA once second-stage support is merged.
+		 */
+	}
+	riscv_iommu_cmd_send(iommu, &cmd);
+}
 /*
  * Update IODIR for the device.
  *
@@ -1031,6 +1097,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 		riscv_iommu_cmd_iodir_inval_ddt(&cmd);
 		riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]);
 		riscv_iommu_cmd_send(iommu, &cmd);
+		/*
+		 * For now, the SVA and PASID features have not yet been merged, the
+		 * default configuration is inval_pdt=false and pc=NULL.
+		 */
+		riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);
 		sync_required = true;
 	}
 
@@ -1056,6 +1127,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 		riscv_iommu_cmd_iodir_inval_ddt(&cmd);
 		riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]);
 		riscv_iommu_cmd_send(iommu, &cmd);
+		/*
+		 * For now, the SVA and PASID features have not yet been merged, the
+		 * default configuration is inval_pdt=false and pc=NULL.
+		 */
+		riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);
 	}
 
 	riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
@@ -1077,158 +1153,9 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
 {
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
 
-	riscv_iommu_iotlb_inval(domain, gather->start, gather->end);
-}
-
-#define PT_SHIFT (PAGE_SHIFT - ilog2(sizeof(pte_t)))
-
-#define _io_pte_present(pte)	((pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE))
-#define _io_pte_leaf(pte)	((pte) & _PAGE_LEAF)
-#define _io_pte_none(pte)	((pte) == 0)
-#define _io_pte_entry(pn, prot)	((_PAGE_PFN_MASK & ((pn) << _PAGE_PFN_SHIFT)) | (prot))
-
-static void riscv_iommu_pte_free(struct riscv_iommu_domain *domain,
-				 unsigned long pte,
-				 struct iommu_pages_list *freelist)
-{
-	unsigned long *ptr;
-	int i;
-
-	if (!_io_pte_present(pte) || _io_pte_leaf(pte))
-		return;
-
-	ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte));
-
-	/* Recursively free all sub page table pages */
-	for (i = 0; i < PTRS_PER_PTE; i++) {
-		pte = READ_ONCE(ptr[i]);
-		if (!_io_pte_none(pte) && cmpxchg_relaxed(ptr + i, pte, 0) == pte)
-			riscv_iommu_pte_free(domain, pte, freelist);
-	}
-
-	if (freelist)
-		iommu_pages_list_add(freelist, ptr);
-	else
-		iommu_free_pages(ptr);
-}
-
-static unsigned long *riscv_iommu_pte_alloc(struct riscv_iommu_domain *domain,
-					    unsigned long iova, size_t pgsize,
-					    gfp_t gfp)
-{
-	unsigned long *ptr = domain->pgd_root;
-	unsigned long pte, old;
-	int level = domain->pgd_mode - RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 + 2;
-	void *addr;
-
-	do {
-		const int shift = PAGE_SHIFT + PT_SHIFT * level;
-
-		ptr += ((iova >> shift) & (PTRS_PER_PTE - 1));
-		/*
-		 * Note: returned entry might be a non-leaf if there was
-		 * existing mapping with smaller granularity. Up to the caller
-		 * to replace and invalidate.
-		 */
-		if (((size_t)1 << shift) == pgsize)
-			return ptr;
-pte_retry:
-		pte = READ_ONCE(*ptr);
-		/*
-		 * This is very likely incorrect as we should not be adding
-		 * new mapping with smaller granularity on top
-		 * of existing 2M/1G mapping. Fail.
-		 */
-		if (_io_pte_present(pte) && _io_pte_leaf(pte))
-			return NULL;
-		/*
-		 * Non-leaf entry is missing, allocate and try to add to the
-		 * page table. This might race with other mappings, retry.
-		 */
-		if (_io_pte_none(pte)) {
-			addr = iommu_alloc_pages_node_sz(domain->numa_node, gfp,
-							 SZ_4K);
-			if (!addr)
-				return NULL;
-			old = pte;
-			pte = _io_pte_entry(virt_to_pfn(addr), _PAGE_TABLE);
-			if (cmpxchg_relaxed(ptr, old, pte) != old) {
-				iommu_free_pages(addr);
-				goto pte_retry;
-			}
-		}
-		ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte));
-	} while (level-- > 0);
-
-	return NULL;
-}
-
-static unsigned long *riscv_iommu_pte_fetch(struct riscv_iommu_domain *domain,
-					    unsigned long iova, size_t *pte_pgsize)
-{
-	unsigned long *ptr = domain->pgd_root;
-	unsigned long pte;
-	int level = domain->pgd_mode - RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 + 2;
-
-	do {
-		const int shift = PAGE_SHIFT + PT_SHIFT * level;
-
-		ptr += ((iova >> shift) & (PTRS_PER_PTE - 1));
-		pte = READ_ONCE(*ptr);
-		if (_io_pte_present(pte) && _io_pte_leaf(pte)) {
-			*pte_pgsize = (size_t)1 << shift;
-			return ptr;
-		}
-		if (_io_pte_none(pte))
-			return NULL;
-		ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte));
-	} while (level-- > 0);
-
-	return NULL;
-}
-
-static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain,
-				 unsigned long iova, phys_addr_t phys,
-				 size_t pgsize, size_t pgcount, int prot,
-				 gfp_t gfp, size_t *mapped)
-{
-	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
-	size_t size = 0;
-	unsigned long *ptr;
-	unsigned long pte, old, pte_prot;
-	int rc = 0;
-	struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist);
-
-	if (!(prot & IOMMU_WRITE))
-		pte_prot = _PAGE_BASE | _PAGE_READ;
-	else if (domain->amo_enabled)
-		pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE;
-	else
-		pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY;
-
-	while (pgcount) {
-		ptr = riscv_iommu_pte_alloc(domain, iova, pgsize, gfp);
-		if (!ptr) {
-			rc = -ENOMEM;
-			break;
-		}
-
-		old = READ_ONCE(*ptr);
-		pte = _io_pte_entry(phys_to_pfn(phys), pte_prot);
-		if (cmpxchg_relaxed(ptr, old, pte) != old)
-			continue;
-
-		riscv_iommu_pte_free(domain, old, &freelist);
-
-		size += pgsize;
-		iova += pgsize;
-		phys += pgsize;
-		--pgcount;
-	}
-
-	*mapped = size;
-
-	if (!iommu_pages_list_empty(&freelist)) {
+	if (iommu_pages_list_empty(&gather->freelist)) {
+		riscv_iommu_iotlb_inval(domain, gather->start, gather->end);
+	} else {
 		/*
 		 * In 1.0 spec version, the smallest scope we can use to
 		 * invalidate all levels of page table (i.e. leaf and non-leaf)
@@ -1237,71 +1164,20 @@ static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain,
 		 * capability.NL (non-leaf) IOTINVAL command.
 		 */
 		riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
-		iommu_put_pages_list(&freelist);
+		iommu_put_pages_list(&gather->freelist);
 	}
-
-	return rc;
-}
-
-static size_t riscv_iommu_unmap_pages(struct iommu_domain *iommu_domain,
-				      unsigned long iova, size_t pgsize,
-				      size_t pgcount,
-				      struct iommu_iotlb_gather *gather)
-{
-	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
-	size_t size = pgcount << __ffs(pgsize);
-	unsigned long *ptr, old;
-	size_t unmapped = 0;
-	size_t pte_size;
-
-	while (unmapped < size) {
-		ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size);
-		if (!ptr)
-			return unmapped;
-
-		/* partial unmap is not allowed, fail. */
-		if (iova & (pte_size - 1))
-			return unmapped;
-
-		old = READ_ONCE(*ptr);
-		if (cmpxchg_relaxed(ptr, old, 0) != old)
-			continue;
-
-		iommu_iotlb_gather_add_page(&domain->domain, gather, iova,
-					    pte_size);
-
-		iova += pte_size;
-		unmapped += pte_size;
-	}
-
-	return unmapped;
-}
-
-static phys_addr_t riscv_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
-					    dma_addr_t iova)
-{
-	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
-	size_t pte_size;
-	unsigned long *ptr;
-
-	ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size);
-	if (!ptr)
-		return 0;
-
-	return pfn_to_phys(__page_val_to_pfn(*ptr)) | (iova & (pte_size - 1));
 }
 
 static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
 {
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
-	const unsigned long pfn = virt_to_pfn(domain->pgd_root);
 
 	WARN_ON(!list_empty(&domain->bonds));
 
 	if ((int)domain->pscid > 0)
 		ida_free(&riscv_iommu_pscids, domain->pscid);
 
-	riscv_iommu_pte_free(domain, _io_pte_entry(pfn, _PAGE_TABLE), NULL);
+	pt_iommu_deinit(&domain->riscvpt.iommu);
 	kfree(domain);
 }
 
@@ -1327,13 +1203,16 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain,
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);
 	struct riscv_iommu_info *info = dev_iommu_priv_get(dev);
+	struct pt_iommu_riscv_64_hw_info pt_info;
 	u64 fsc, ta;
 
-	if (!riscv_iommu_pt_supported(iommu, domain->pgd_mode))
+	pt_iommu_riscv_64_hw_info(&domain->riscvpt, &pt_info);
+
+	if (!riscv_iommu_pt_supported(iommu, pt_info.fsc_iosatp_mode))
 		return -ENODEV;
 
-	fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, domain->pgd_mode) |
-	      FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, virt_to_pfn(domain->pgd_root));
+	fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, pt_info.fsc_iosatp_mode) |
+	      FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, pt_info.ppn);
 	ta = FIELD_PREP(RISCV_IOMMU_PC_TA_PSCID, domain->pscid) |
 	     RISCV_IOMMU_PC_TA_V;
 
@@ -1348,37 +1227,32 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain,
 }
 
 static const struct iommu_domain_ops riscv_iommu_paging_domain_ops = {
+	IOMMU_PT_DOMAIN_OPS(riscv_64),
 	.attach_dev = riscv_iommu_attach_paging_domain,
 	.free = riscv_iommu_free_paging_domain,
-	.map_pages = riscv_iommu_map_pages,
-	.unmap_pages = riscv_iommu_unmap_pages,
-	.iova_to_phys = riscv_iommu_iova_to_phys,
 	.iotlb_sync = riscv_iommu_iotlb_sync,
 	.flush_iotlb_all = riscv_iommu_iotlb_flush_all,
 };
 
 static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
 {
+	struct pt_iommu_riscv_64_cfg cfg = {};
 	struct riscv_iommu_domain *domain;
 	struct riscv_iommu_device *iommu;
-	unsigned int pgd_mode;
-	dma_addr_t va_mask;
-	int va_bits;
+	int ret;
 
 	iommu = dev_to_iommu(dev);
 	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) {
-		pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57;
-		va_bits = 57;
+		cfg.common.hw_max_vasz_lg2 = 57;
 	} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) {
-		pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48;
-		va_bits = 48;
+		cfg.common.hw_max_vasz_lg2 = 48;
 	} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) {
-		pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39;
-		va_bits = 39;
+		cfg.common.hw_max_vasz_lg2 = 39;
 	} else {
 		dev_err(dev, "cannot find supported page table mode\n");
 		return ERR_PTR(-ENODEV);
 	}
+	cfg.common.hw_max_oasz_lg2 = 56;
 
 	domain = kzalloc_obj(*domain);
 	if (!domain)
@@ -1386,43 +1260,28 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
 
 	INIT_LIST_HEAD_RCU(&domain->bonds);
 	spin_lock_init(&domain->lock);
-	domain->numa_node = dev_to_node(iommu->dev);
-	domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD);
-	domain->pgd_mode = pgd_mode;
-	domain->pgd_root = iommu_alloc_pages_node_sz(domain->numa_node,
-						     GFP_KERNEL_ACCOUNT, SZ_4K);
-	if (!domain->pgd_root) {
-		kfree(domain);
-		return ERR_PTR(-ENOMEM);
-	}
+	/*
+	 * 6.4 IOMMU capabilities [..] IOMMU implementations must support the
+	 * Svnapot standard extension for NAPOT Translation Contiguity.
+	 */
+	cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) |
+			      BIT(PT_FEAT_FLUSH_RANGE) |
+			      BIT(PT_FEAT_RISCV_SVNAPOT_64K);
+	domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);
+	domain->domain.ops = &riscv_iommu_paging_domain_ops;
 
 	domain->pscid = ida_alloc_range(&riscv_iommu_pscids, 1,
 					RISCV_IOMMU_MAX_PSCID, GFP_KERNEL);
 	if (domain->pscid < 0) {
-		iommu_free_pages(domain->pgd_root);
-		kfree(domain);
+		riscv_iommu_free_paging_domain(&domain->domain);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	/*
-	 * Note: RISC-V Privilege spec mandates that virtual addresses
-	 * need to be sign-extended, so if (VA_BITS - 1) is set, all
-	 * bits >= VA_BITS need to also be set or else we'll get a
-	 * page fault. However the code that creates the mappings
-	 * above us (e.g. iommu_dma_alloc_iova()) won't do that for us
-	 * for now, so we'll end up with invalid virtual addresses
-	 * to map. As a workaround until we get this sorted out
-	 * limit the available virtual addresses to VA_BITS - 1.
-	 */
-	va_mask = DMA_BIT_MASK(va_bits - 1);
-
-	domain->domain.geometry.aperture_start = 0;
-	domain->domain.geometry.aperture_end = va_mask;
-	domain->domain.geometry.force_aperture = true;
-	domain->domain.pgsize_bitmap = va_mask & (SZ_4K | SZ_2M | SZ_1G | SZ_512G);
-
-	domain->domain.ops = &riscv_iommu_paging_domain_ops;
-
+	ret = pt_iommu_riscv_64_init(&domain->riscvpt, &cfg, GFP_KERNEL);
+	if (ret) {
+		riscv_iommu_free_paging_domain(&domain->domain);
+		return ERR_PTR(ret);
+	}
 	return &domain->domain;
 }
 
@@ -1512,8 +1371,6 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev)
 	 * the device directory. Do not mark the context valid yet.
 	 */
 	tc = 0;
-	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD)
-		tc |= RISCV_IOMMU_DC_TC_SADE;
 	for (i = 0; i < fwspec->num_ids; i++) {
 		dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);
 		if (!dc) {
@@ -1680,3 +1537,5 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu)
 	riscv_iommu_queue_disable(&iommu->cmdq);
 	return rc;
 }
+
+MODULE_IMPORT_NS("GENERIC_PT_IOMMU");
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index f07b00d7fef90..d631dd796f0ae 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -51,12 +51,6 @@ config ARM_GIC_V3_ITS
 	default ARM_GIC_V3
 	select IRQ_MSI_IOMMU
 
-config ARM_GIC_V3_ITS_FSL_MC
-	bool
-	depends on ARM_GIC_V3_ITS
-	depends on FSL_MC_BUS
-	default ARM_GIC_V3_ITS
-
 config ARM_GIC_V5
 	bool
 	select IRQ_DOMAIN_HIERARCHY
@@ -252,9 +246,12 @@ config ORION_IRQCHIP
 	select IRQ_DOMAIN
 
 config PIC32_EVIC
-	bool
+	def_bool MACH_PIC32 || COMPILE_TEST
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
+	help
+	  Enable support for the interrupt controller on the Microchip PIC32
+	  family of platforms.
 
 config JCORE_AIC
 	bool "J-Core integrated AIC" if COMPILE_TEST
@@ -541,11 +538,11 @@ config CSKY_APB_INTC
 
 config IMX_IRQSTEER
 	bool "i.MX IRQSTEER support"
-	depends on ARCH_MXC || COMPILE_TEST
-	default ARCH_MXC
+	depends on ARCH_MXC || ARCH_S32 || COMPILE_TEST
+	default y if ARCH_MXC || ARCH_S32
 	select IRQ_DOMAIN
 	help
-	  Support for the i.MX IRQSTEER interrupt multiplexer/remapper.
+	  Support for the i.MX and S32 IRQSTEER interrupt multiplexer/remapper.
 
 config IMX_INTMUX
 	bool "i.MX INTMUX support" if COMPILE_TEST
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 26aa3b6ec99fd..d5a28cee0d8eb 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -35,7 +35,6 @@ obj-$(CONFIG_ARM_GIC_V2M)		+= irq-gic-v2m.o
 obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_ITS_PARENT)	+= irq-gic-its-msi-parent.o
 obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v4.o
-obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC)	+= irq-gic-v3-its-fsl-mc-msi.o
 obj-$(CONFIG_ARM_GIC_V5)		+= irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \
 					   irq-gic-v5-iwb.o
 obj-$(CONFIG_HISILICON_IRQ_MBIGEN)	+= irq-mbigen.o
diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c
index 2b24c82bb0df5..4a3141d9f9141 100644
--- a/drivers/irqchip/irq-apple-aic.c
+++ b/drivers/irqchip/irq-apple-aic.c
@@ -134,8 +134,12 @@
 
 #define AIC2_IRQ_CFG		0x2000
 
+/* AIC v3 registers (MMIO) */
+#define AIC3_IRQ_CFG		0x10000
+
 /*
  * AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG:
+ * AIC3 registers use the same layout but start at AIC3_IRQ_CFG:
  *
  * Repeat for each die:
  *   IRQ_CFG: u32 * MAX_IRQS
@@ -293,6 +297,15 @@ static const struct aic_info aic2_info __initconst = {
 	.local_fast_ipi = true,
 };
 
+static const struct aic_info aic3_info __initconst = {
+	.version	= 3,
+
+	.irq_cfg	= AIC3_IRQ_CFG,
+
+	.fast_ipi	= true,
+	.local_fast_ipi = true,
+};
+
 static const struct of_device_id aic_info_match[] = {
 	{
 		.compatible = "apple,t8103-aic",
@@ -310,6 +323,10 @@ static const struct of_device_id aic_info_match[] = {
 		.compatible = "apple,aic2",
 		.data = &aic2_info,
 	},
+	{
+		.compatible = "apple,t8122-aic3",
+		.data = &aic3_info,
+	},
 	{}
 };
 
@@ -620,7 +637,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq,
 	u32 type = FIELD_GET(AIC_EVENT_TYPE, hw);
 	struct irq_chip *chip = &aic_chip;
 
-	if (ic->info.version == 2)
+	if (ic->info.version == 2 || ic->info.version == 3)
 		chip = &aic2_chip;
 
 	if (type == AIC_EVENT_TYPE_IRQ) {
@@ -991,7 +1008,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p
 
 		break;
 	}
-	case 2: {
+	case 2 ... 3: {
 		u32 info1, info3;
 
 		info1 = aic_ic_read(irqc, AIC2_INFO1);
@@ -1065,7 +1082,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p
 		off += irqc->info.die_stride;
 	}
 
-	if (irqc->info.version == 2) {
+	if (irqc->info.version == 2 || irqc->info.version == 3) {
 		u32 config = aic_ic_read(irqc, AIC2_CONFIG);
 
 		config |= AIC2_CONFIG_ENABLE;
@@ -1116,3 +1133,4 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p
 
 IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init);
 IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init);
+IRQCHIP_DECLARE(apple_aic3, "apple,t8122-aic3", aic_of_ic_init);
diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c
index a832cdb2e6978..d36b278ae66c5 100644
--- a/drivers/irqchip/irq-gic-its-msi-parent.c
+++ b/drivers/irqchip/irq-gic-its-msi-parent.c
@@ -5,6 +5,7 @@
 // Copyright (C) 2022 Intel
 
 #include <linux/acpi_iort.h>
+#include <linux/fsl/mc.h>
 #include <linux/of_address.h>
 #include <linux/pci.h>
 
@@ -187,9 +188,11 @@ static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev,
 {
 	struct msi_domain_info *msi_info;
 	u32 dev_id;
-	int ret;
+	int ret = 0;
 
-	if (dev->of_node)
+	if (dev_is_fsl_mc(dev))
+		dev_id = fsl_mc_get_msi_id(dev);
+	else if (dev->of_node)
 		ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, NULL);
 	else
 		ret = iort_pmsi_get_msi_info(dev, &dev_id, NULL);
diff --git a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
deleted file mode 100644
index b5785472765a3..0000000000000
--- a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
+++ /dev/null
@@ -1,168 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Freescale Management Complex (MC) bus driver MSI support
- *
- * Copyright (C) 2015-2016 Freescale Semiconductor, Inc.
- * Author: German Rivera <German.Rivera@freescale.com>
- *
- */
-
-#include <linux/acpi.h>
-#include <linux/acpi_iort.h>
-#include <linux/irq.h>
-#include <linux/msi.h>
-#include <linux/of.h>
-#include <linux/of_irq.h>
-#include <linux/fsl/mc.h>
-
-static struct irq_chip its_msi_irq_chip = {
-	.name = "ITS-fMSI",
-	.irq_mask = irq_chip_mask_parent,
-	.irq_unmask = irq_chip_unmask_parent,
-	.irq_eoi = irq_chip_eoi_parent,
-	.irq_set_affinity = msi_domain_set_affinity
-};
-
-static u32 fsl_mc_msi_domain_get_msi_id(struct irq_domain *domain,
-					struct fsl_mc_device *mc_dev)
-{
-	struct device_node *of_node;
-	u32 out_id;
-
-	of_node = irq_domain_get_of_node(domain);
-	out_id = of_node ? of_msi_xlate(&mc_dev->dev, &of_node, mc_dev->icid) :
-			iort_msi_map_id(&mc_dev->dev, mc_dev->icid);
-
-	return out_id;
-}
-
-static int its_fsl_mc_msi_prepare(struct irq_domain *msi_domain,
-				  struct device *dev,
-				  int nvec, msi_alloc_info_t *info)
-{
-	struct fsl_mc_device *mc_bus_dev;
-	struct msi_domain_info *msi_info;
-
-	if (!dev_is_fsl_mc(dev))
-		return -EINVAL;
-
-	mc_bus_dev = to_fsl_mc_device(dev);
-	if (!(mc_bus_dev->flags & FSL_MC_IS_DPRC))
-		return -EINVAL;
-
-	/*
-	 * Set the device Id to be passed to the GIC-ITS:
-	 *
-	 * NOTE: This device id corresponds to the IOMMU stream ID
-	 * associated with the DPRC object (ICID).
-	 */
-	info->scratchpad[0].ul = fsl_mc_msi_domain_get_msi_id(msi_domain,
-							      mc_bus_dev);
-	msi_info = msi_get_domain_info(msi_domain->parent);
-
-	/* Allocate at least 32 MSIs, and always as a power of 2 */
-	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
-	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
-}
-
-static struct msi_domain_ops its_fsl_mc_msi_ops __ro_after_init = {
-	.msi_prepare = its_fsl_mc_msi_prepare,
-};
-
-static struct msi_domain_info its_fsl_mc_msi_domain_info = {
-	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
-	.ops	= &its_fsl_mc_msi_ops,
-	.chip	= &its_msi_irq_chip,
-};
-
-static const struct of_device_id its_device_id[] = {
-	{	.compatible	= "arm,gic-v3-its",	},
-	{},
-};
-
-static void __init its_fsl_mc_msi_init_one(struct fwnode_handle *handle,
-					  const char *name)
-{
-	struct irq_domain *parent;
-	struct irq_domain *mc_msi_domain;
-
-	parent = irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
-	if (!parent || !msi_get_domain_info(parent)) {
-		pr_err("%s: unable to locate ITS domain\n", name);
-		return;
-	}
-
-	mc_msi_domain = fsl_mc_msi_create_irq_domain(handle,
-						&its_fsl_mc_msi_domain_info,
-						parent);
-	if (!mc_msi_domain) {
-		pr_err("%s: unable to create fsl-mc domain\n", name);
-		return;
-	}
-
-	pr_info("fsl-mc MSI: %s domain created\n", name);
-}
-
-#ifdef CONFIG_ACPI
-static int __init
-its_fsl_mc_msi_parse_madt(union acpi_subtable_headers *header,
-			  const unsigned long end)
-{
-	struct acpi_madt_generic_translator *its_entry;
-	struct fwnode_handle *dom_handle;
-	const char *node_name;
-	int err = 0;
-
-	its_entry = (struct acpi_madt_generic_translator *)header;
-	node_name = kasprintf(GFP_KERNEL, "ITS@0x%lx",
-			      (long)its_entry->base_address);
-
-	dom_handle = iort_find_domain_token(its_entry->translation_id);
-	if (!dom_handle) {
-		pr_err("%s: Unable to locate ITS domain handle\n", node_name);
-		err = -ENXIO;
-		goto out;
-	}
-
-	its_fsl_mc_msi_init_one(dom_handle, node_name);
-
-out:
-	kfree(node_name);
-	return err;
-}
-
-
-static void __init its_fsl_mc_acpi_msi_init(void)
-{
-	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
-			      its_fsl_mc_msi_parse_madt, 0);
-}
-#else
-static inline void its_fsl_mc_acpi_msi_init(void) { }
-#endif
-
-static void __init its_fsl_mc_of_msi_init(void)
-{
-	struct device_node *np;
-
-	for (np = of_find_matching_node(NULL, its_device_id); np;
-	     np = of_find_matching_node(np, its_device_id)) {
-		if (!of_device_is_available(np))
-			continue;
-		if (!of_property_read_bool(np, "msi-controller"))
-			continue;
-
-		its_fsl_mc_msi_init_one(of_fwnode_handle(np),
-					np->full_name);
-	}
-}
-
-static int __init its_fsl_mc_msi_init(void)
-{
-	its_fsl_mc_of_msi_init();
-	its_fsl_mc_acpi_msi_init();
-
-	return 0;
-}
-
-early_initcall(its_fsl_mc_msi_init);
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 20f13b686ab22..6dc9827357a21 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -2252,7 +2252,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 
 out_unmap_rdist:
 	for (i = 0; i < nr_redist_regions; i++)
-		if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base))
+		if (!IS_ERR_OR_NULL(rdist_regs[i].redist_base))
 			iounmap(rdist_regs[i].redist_base);
 	kfree(rdist_regs);
 out_unmap_dist:
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c
index 405a5eee847b6..6b0903be8ebfd 100644
--- a/drivers/irqchip/irq-gic-v5.c
+++ b/drivers/irqchip/irq-gic-v5.c
@@ -511,6 +511,23 @@ static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq)
 	return !!(read_ppi_sysreg_s(hwirq, PPI_HM) & bit);
 }
 
+static int gicv5_ppi_irq_set_type(struct irq_data *d, unsigned int type)
+{
+	/*
+	 * GICv5's PPIs do not have a configurable trigger or handling
+	 * mode. Check that the attempt to set a type matches what the
+	 * hardware reports in the HMR, and error on a mismatch.
+	 */
+
+	if (type & IRQ_TYPE_EDGE_BOTH && gicv5_ppi_irq_is_level(d->hwirq))
+		return -EINVAL;
+
+	if (type & IRQ_TYPE_LEVEL_MASK && !gicv5_ppi_irq_is_level(d->hwirq))
+		return -EINVAL;
+
+	return 0;
+}
+
 static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 {
 	if (vcpu)
@@ -526,6 +543,7 @@ static const struct irq_chip gicv5_ppi_irq_chip = {
 	.irq_mask		= gicv5_ppi_irq_mask,
 	.irq_unmask		= gicv5_ppi_irq_unmask,
 	.irq_eoi		= gicv5_ppi_irq_eoi,
+	.irq_set_type		= gicv5_ppi_irq_set_type,
 	.irq_get_irqchip_state	= gicv5_ppi_irq_get_irqchip_state,
 	.irq_set_irqchip_state	= gicv5_ppi_irq_set_irqchip_state,
 	.irq_set_vcpu_affinity	= gicv5_ppi_irq_set_vcpu_affinity,
diff --git a/drivers/irqchip/irq-imx-irqsteer.c b/drivers/irqchip/irq-imx-irqsteer.c
index 4682ce5bf8d3c..87b07f517be35 100644
--- a/drivers/irqchip/irq-imx-irqsteer.c
+++ b/drivers/irqchip/irq-imx-irqsteer.c
@@ -26,19 +26,38 @@
 
 #define CHAN_MAX_OUTPUT_INT	0xF
 
+/* SoC does not implement the CHANCTRL register */
+#define IRQSTEER_QUIRK_NO_CHANCTRL	BIT(0)
+
+struct irqsteer_devtype_data {
+	u32	quirks;
+};
+
 struct irqsteer_data {
-	void __iomem		*regs;
-	struct clk		*ipg_clk;
-	int			irq[CHAN_MAX_OUTPUT_INT];
-	int			irq_count;
-	raw_spinlock_t		lock;
-	int			reg_num;
-	int			channel;
-	struct irq_domain	*domain;
-	u32			*saved_reg;
-	struct device		*dev;
+	void __iomem				*regs;
+	struct clk				*ipg_clk;
+	int					irq[CHAN_MAX_OUTPUT_INT];
+	int					irq_count;
+	raw_spinlock_t				lock;
+	int					reg_num;
+	int					channel;
+	struct irq_domain			*domain;
+	u32					*saved_reg;
+	struct device				*dev;
+	const struct irqsteer_devtype_data	*devtype_data;
+};
+
+static const struct irqsteer_devtype_data imx_data = { };
+
+static const struct irqsteer_devtype_data s32n79_data = {
+	.quirks	= IRQSTEER_QUIRK_NO_CHANCTRL,
 };
 
+static bool irqsteer_has_chanctrl(const struct irqsteer_devtype_data *data)
+{
+	return !(data->quirks & IRQSTEER_QUIRK_NO_CHANCTRL);
+}
+
 static int imx_irqsteer_get_reg_index(struct irqsteer_data *data,
 				      unsigned long irqnum)
 {
@@ -188,6 +207,10 @@ static int imx_irqsteer_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	data->devtype_data = device_get_match_data(&pdev->dev);
+	if (!data->devtype_data)
+		return dev_err_probe(&pdev->dev, -ENODEV, "failed to match device data\n");
+
 	/*
 	 * There is one output irq for each group of 64 inputs.
 	 * One register bit map can represent 32 input interrupts.
@@ -210,7 +233,8 @@ static int imx_irqsteer_probe(struct platform_device *pdev)
 	}
 
 	/* steer all IRQs into configured channel */
-	writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
+	if (irqsteer_has_chanctrl(data->devtype_data))
+		writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
 
 	data->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), data->reg_num * 32,
 						&imx_irqsteer_domain_ops, data);
@@ -279,7 +303,9 @@ static void imx_irqsteer_restore_regs(struct irqsteer_data *data)
 {
 	int i;
 
-	writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
+	if (irqsteer_has_chanctrl(data->devtype_data))
+		writel_relaxed(BIT(data->channel), data->regs + CHANCTRL);
+
 	for (i = 0; i < data->reg_num; i++)
 		writel_relaxed(data->saved_reg[i],
 			       data->regs + CHANMASK(i, data->reg_num));
@@ -319,7 +345,8 @@ static const struct dev_pm_ops imx_irqsteer_pm_ops = {
 };
 
 static const struct of_device_id imx_irqsteer_dt_ids[] = {
-	{ .compatible = "fsl,imx-irqsteer", },
+	{ .compatible = "fsl,imx-irqsteer",	.data = &imx_data },
+	{ .compatible = "nxp,s32n79-irqsteer",	.data = &s32n79_data },
 	{},
 };
 
diff --git a/drivers/irqchip/irq-msi-lib.c b/drivers/irqchip/irq-msi-lib.c
index d5eefc3d72155..45e0ed3134ce1 100644
--- a/drivers/irqchip/irq-msi-lib.c
+++ b/drivers/irqchip/irq-msi-lib.c
@@ -48,6 +48,9 @@ bool msi_lib_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
 		return false;
 	}
 
+	if (WARN_ON_ONCE(!chip->irq_write_msi_msg))
+		return false;
+
 	required_flags = pops->required_flags;
 
 	/* Is the target domain bus token supported? */
diff --git a/drivers/irqchip/irq-mvebu-odmi.c b/drivers/irqchip/irq-mvebu-odmi.c
index b99ab9dcc14b3..94e7eda46e818 100644
--- a/drivers/irqchip/irq-mvebu-odmi.c
+++ b/drivers/irqchip/irq-mvebu-odmi.c
@@ -217,7 +217,7 @@ static int __init mvebu_odmi_init(struct device_node *node,
 	for (i = 0; i < odmis_count; i++) {
 		struct odmi_data *odmi = &odmis[i];
 
-		if (odmi->base && !IS_ERR(odmi->base))
+		if (!IS_ERR_OR_NULL(odmi->base))
 			iounmap(odmis[i].base);
 	}
 	bitmap_free(odmis_bm);
diff --git a/drivers/irqchip/irq-pic32-evic.c b/drivers/irqchip/irq-pic32-evic.c
index e85c3e3007018..ecea7cb3ba849 100644
--- a/drivers/irqchip/irq-pic32-evic.c
+++ b/drivers/irqchip/irq-pic32-evic.c
@@ -15,8 +15,10 @@
 #include <linux/irq.h>
 #include <linux/platform_data/pic32.h>
 
+#ifdef CONFIG_MIPS
 #include <asm/irq.h>
 #include <asm/traps.h>
+#endif
 
 #define REG_INTCON	0x0000
 #define REG_INTSTAT	0x0020
@@ -40,6 +42,7 @@ struct evic_chip_data {
 static struct irq_domain *evic_irq_domain;
 static void __iomem *evic_base;
 
+#ifdef CONFIG_MIPS
 asmlinkage void __weak plat_irq_dispatch(void)
 {
 	unsigned int hwirq;
@@ -47,6 +50,9 @@ asmlinkage void __weak plat_irq_dispatch(void)
 	hwirq = readl(evic_base + REG_INTSTAT) & 0xFF;
 	do_domain_IRQ(evic_irq_domain, hwirq);
 }
+#else
+static void (*board_bind_eic_interrupt)(int irq, int regset);
+#endif
 
 static struct evic_chip_data *irqd_to_priv(struct irq_data *data)
 {
@@ -196,7 +202,7 @@ static void __init pic32_ext_irq_of_init(struct irq_domain *domain)
 
 	of_property_for_each_u32(node, pname, hwirq) {
 		if (i >= ARRAY_SIZE(priv->ext_irqs)) {
-			pr_warn("More than %d external irq, skip rest\n",
+			pr_warn("More than %zu external irq, skip rest\n",
 				ARRAY_SIZE(priv->ext_irqs));
 			break;
 		}
diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c
index da2bc43a0e122..ce790590f7ca9 100644
--- a/drivers/irqchip/irq-renesas-rzv2h.c
+++ b/drivers/irqchip/irq-renesas-rzv2h.c
@@ -12,6 +12,7 @@
 #include <linux/bitfield.h>
 #include <linux/cleanup.h>
 #include <linux/err.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irqchip.h>
 #include <linux/irqchip/irq-renesas-rzv2h.h>
@@ -25,9 +26,17 @@
 /* DT "interrupts" indexes */
 #define ICU_IRQ_START				1
 #define ICU_IRQ_COUNT				16
-#define ICU_TINT_START				(ICU_IRQ_START + ICU_IRQ_COUNT)
+#define ICU_IRQ_LAST				(ICU_IRQ_START + ICU_IRQ_COUNT - 1)
+#define ICU_TINT_START				(ICU_IRQ_LAST + 1)
 #define ICU_TINT_COUNT				32
-#define ICU_NUM_IRQ				(ICU_TINT_START + ICU_TINT_COUNT)
+#define ICU_TINT_LAST				(ICU_TINT_START + ICU_TINT_COUNT - 1)
+#define ICU_CA55_INT_START			(ICU_TINT_LAST + 1)
+#define ICU_CA55_INT_COUNT			4
+#define ICU_CA55_INT_LAST			(ICU_CA55_INT_START + ICU_CA55_INT_COUNT - 1)
+#define ICU_ERR_INT_START			(ICU_CA55_INT_LAST + 1)
+#define ICU_ERR_INT_COUNT			1
+#define ICU_ERR_INT_LAST			(ICU_ERR_INT_START + ICU_ERR_INT_COUNT - 1)
+#define ICU_NUM_IRQ				(ICU_ERR_INT_LAST + 1)
 
 /* Registers */
 #define ICU_NSCNT				0x00
@@ -40,6 +49,15 @@
 #define ICU_TSCLR				0x24
 #define ICU_TITSR(k)				(0x28 + (k) * 4)
 #define ICU_TSSR(k)				(0x30 + (k) * 4)
+#define ICU_BEISR(k)				(0x70 + (k) * 4)
+#define ICU_BECLR(k)				(0x80 + (k) * 4)
+#define ICU_EREISR(k)				(0x90 + (k) * 4)
+#define ICU_ERCLR(k)				(0xE0 + (k) * 4)
+#define ICU_SWINT				0x130
+#define ICU_ERINTA55CTL(k)			(0x338 + (k) * 4)
+#define ICU_ERINTA55CRL(k)			(0x348 + (k) * 4)
+#define ICU_ERINTA55MSK(k)			(0x358 + (k) * 4)
+#define ICU_SWPE				0x370
 #define ICU_DMkSELy(k, y)			(0x420 + (k) * 0x20 + (y) * 4)
 #define ICU_DMACKSELk(k)			(0x500 + (k) * 4)
 
@@ -90,6 +108,10 @@
 #define ICU_RZG3E_TSSEL_MAX_VAL			0x8c
 #define ICU_RZV2H_TSSEL_MAX_VAL			0x55
 
+#define ICU_SWPE_NUM				16
+#define ICU_NUM_BE				4
+#define ICU_NUM_A55ERR				4
+
 /**
  * struct rzv2h_irqc_reg_cache - registers cache (necessary for suspend/resume)
  * @nitsr: ICU_NITSR register
@@ -108,12 +130,16 @@ struct rzv2h_irqc_reg_cache {
  * @t_offs:		TINT offset
  * @max_tssel:		TSSEL max value
  * @field_width:	TSSR field width
+ * @ecc_start:		Start index of ECC RAM interrupts
+ * @ecc_end:		End index of ECC RAM interrupts
  */
 struct rzv2h_hw_info {
 	const u8	*tssel_lut;
 	u16		t_offs;
 	u8		max_tssel;
 	u8		field_width;
+	u8		ecc_start;
+	u8		ecc_end;
 };
 
 /* DMAC */
@@ -167,32 +193,47 @@ static inline struct rzv2h_icu_priv *irq_data_to_priv(struct irq_data *data)
 	return data->domain->host_data;
 }
 
-static void rzv2h_icu_eoi(struct irq_data *d)
+static void rzv2h_icu_tint_eoi(struct irq_data *d)
 {
 	struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
 	unsigned int hw_irq = irqd_to_hwirq(d);
 	unsigned int tintirq_nr;
 	u32 bit;
 
-	scoped_guard(raw_spinlock, &priv->lock) {
-		if (hw_irq >= ICU_TINT_START) {
-			tintirq_nr = hw_irq - ICU_TINT_START;
-			bit = BIT(tintirq_nr);
-			if (!irqd_is_level_type(d))
-				writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR);
-		} else if (hw_irq >= ICU_IRQ_START) {
-			tintirq_nr = hw_irq - ICU_IRQ_START;
-			bit = BIT(tintirq_nr);
-			if (!irqd_is_level_type(d))
-				writel_relaxed(bit, priv->base + ICU_ISCLR);
-		} else {
-			writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR);
-		}
+	if (!irqd_is_level_type(d)) {
+		tintirq_nr = hw_irq - ICU_TINT_START;
+		bit = BIT(tintirq_nr);
+		writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR);
 	}
 
 	irq_chip_eoi_parent(d);
 }
 
+static void rzv2h_icu_irq_eoi(struct irq_data *d)
+{
+	struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+	unsigned int hw_irq = irqd_to_hwirq(d);
+	unsigned int tintirq_nr;
+	u32 bit;
+
+	if (!irqd_is_level_type(d)) {
+		tintirq_nr = hw_irq - ICU_IRQ_START;
+		bit = BIT(tintirq_nr);
+		writel_relaxed(bit, priv->base + ICU_ISCLR);
+	}
+
+	irq_chip_eoi_parent(d);
+}
+
+static void rzv2h_icu_nmi_eoi(struct irq_data *d)
+{
+	struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+
+	writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR);
+
+	irq_chip_eoi_parent(d);
+}
+
 static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
 {
 	struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
@@ -200,9 +241,6 @@ static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
 	u32 tint_nr, tssel_n, k, tssr;
 	u8 nr_tint;
 
-	if (hw_irq < ICU_TINT_START)
-		return;
-
 	tint_nr = hw_irq - ICU_TINT_START;
 	nr_tint = 32 / priv->info->field_width;
 	k = tint_nr / nr_tint;
@@ -225,13 +263,13 @@ static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
 	writel_relaxed(BIT(tint_nr), priv->base + priv->info->t_offs + ICU_TSCLR);
 }
 
-static void rzv2h_icu_irq_disable(struct irq_data *d)
+static void rzv2h_icu_tint_disable(struct irq_data *d)
 {
 	irq_chip_disable_parent(d);
 	rzv2h_tint_irq_endisable(d, false);
 }
 
-static void rzv2h_icu_irq_enable(struct irq_data *d)
+static void rzv2h_icu_tint_enable(struct irq_data *d)
 {
 	rzv2h_tint_irq_endisable(d, true);
 	irq_chip_enable_parent(d);
@@ -257,7 +295,7 @@ static int rzv2h_nmi_set_type(struct irq_data *d, unsigned int type)
 
 	writel_relaxed(sense, priv->base + ICU_NITSR);
 
-	return 0;
+	return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
 }
 
 static void rzv2h_clear_irq_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
@@ -307,14 +345,15 @@ static int rzv2h_irq_set_type(struct irq_data *d, unsigned int type)
 		return -EINVAL;
 	}
 
-	guard(raw_spinlock)(&priv->lock);
-	iitsr = readl_relaxed(priv->base + ICU_IITSR);
-	iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr);
-	iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr);
-	rzv2h_clear_irq_int(priv, hwirq);
-	writel_relaxed(iitsr, priv->base + ICU_IITSR);
+	scoped_guard(raw_spinlock, &priv->lock) {
+		iitsr = readl_relaxed(priv->base + ICU_IITSR);
+		iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr);
+		iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr);
+		rzv2h_clear_irq_int(priv, hwirq);
+		writel_relaxed(iitsr, priv->base + ICU_IITSR);
+	}
 
-	return 0;
+	return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
 }
 
 static void rzv2h_clear_tint_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
@@ -389,49 +428,82 @@ static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type)
 	titsr_k = ICU_TITSR_K(tint_nr);
 	titsel_n = ICU_TITSR_TITSEL_N(tint_nr);
 
-	guard(raw_spinlock)(&priv->lock);
+	scoped_guard(raw_spinlock, &priv->lock) {
+		tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
+		titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
 
-	tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
-	titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
+		tssr_cur = field_get(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width), tssr);
+		titsr_cur = field_get(ICU_TITSR_TITSEL_MASK(titsel_n), titsr);
+		if (tssr_cur == tint && titsr_cur == sense)
+			goto set_parent_type;
 
-	tssr_cur = field_get(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width), tssr);
-	titsr_cur = field_get(ICU_TITSR_TITSEL_MASK(titsel_n), titsr);
-	if (tssr_cur == tint && titsr_cur == sense)
-		return 0;
+		tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien);
+		tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width);
+
+		writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
+
+		titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n);
+		titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n);
 
-	tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien);
-	tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width);
+		writel_relaxed(titsr, priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
+
+		rzv2h_clear_tint_int(priv, hwirq);
+
+		writel_relaxed(tssr | tien, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
+	}
+set_parent_type:
+	return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
+}
 
-	writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
+static int rzv2h_icu_swint_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which,
+					     bool state)
+{
+	unsigned int hwirq = irqd_to_hwirq(d);
+	struct rzv2h_icu_priv *priv;
+	unsigned int bit;
 
-	titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n);
-	titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n);
+	if (which != IRQCHIP_STATE_PENDING)
+		return irq_chip_set_parent_state(d, which, state);
 
-	writel_relaxed(titsr, priv->base + priv->info->t_offs + ICU_TITSR(titsr_k));
+	if (!state)
+		return 0;
 
-	rzv2h_clear_tint_int(priv, hwirq);
+	priv = irq_data_to_priv(d);
+	bit = BIT(hwirq - ICU_CA55_INT_START);
 
-	writel_relaxed(tssr | tien, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k));
+	/* Trigger the software interrupt */
+	writel_relaxed(bit, priv->base + ICU_SWINT);
 
 	return 0;
 }
 
-static int rzv2h_icu_set_type(struct irq_data *d, unsigned int type)
+static int rzv2h_icu_swpe_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which,
+					    bool state)
 {
-	unsigned int hw_irq = irqd_to_hwirq(d);
-	int ret;
+	struct rzv2h_icu_priv *priv;
+	unsigned int bit;
+	static u8 swpe;
 
-	if (hw_irq >= ICU_TINT_START)
-		ret = rzv2h_tint_set_type(d, type);
-	else if (hw_irq >= ICU_IRQ_START)
-		ret = rzv2h_irq_set_type(d, type);
-	else
-		ret = rzv2h_nmi_set_type(d, type);
+	if (which != IRQCHIP_STATE_PENDING)
+		return irq_chip_set_parent_state(d, which, state);
 
-	if (ret)
-		return ret;
+	if (!state)
+		return 0;
 
-	return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
+	priv = irq_data_to_priv(d);
+
+	bit = BIT(swpe);
+	/*
+	 * SWPE has 16 bits; the bit position is rotated on each trigger
+	 * and wraps around once all bits have been used.
+	 */
+	if (++swpe >= ICU_SWPE_NUM)
+		swpe = 0;
+
+	/* Trigger the pseudo error interrupt */
+	writel_relaxed(bit, priv->base + ICU_SWPE);
+
+	return 0;
 }
 
 static int rzv2h_irqc_irq_suspend(void *data)
@@ -472,27 +544,98 @@ static struct syscore rzv2h_irqc_syscore = {
 	.ops = &rzv2h_irqc_syscore_ops,
 };
 
-static const struct irq_chip rzv2h_icu_chip = {
+static const struct irq_chip rzv2h_icu_tint_chip = {
+	.name			= "rzv2h-icu",
+	.irq_eoi		= rzv2h_icu_tint_eoi,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_disable		= rzv2h_icu_tint_disable,
+	.irq_enable		= rzv2h_icu_tint_enable,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= irq_chip_set_parent_state,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= rzv2h_tint_set_type,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE,
+};
+
+static const struct irq_chip rzv2h_icu_irq_chip = {
 	.name			= "rzv2h-icu",
-	.irq_eoi		= rzv2h_icu_eoi,
+	.irq_eoi		= rzv2h_icu_irq_eoi,
 	.irq_mask		= irq_chip_mask_parent,
 	.irq_unmask		= irq_chip_unmask_parent,
-	.irq_disable		= rzv2h_icu_irq_disable,
-	.irq_enable		= rzv2h_icu_irq_enable,
+	.irq_disable		= irq_chip_disable_parent,
+	.irq_enable		= irq_chip_enable_parent,
 	.irq_get_irqchip_state	= irq_chip_get_parent_state,
 	.irq_set_irqchip_state	= irq_chip_set_parent_state,
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
-	.irq_set_type		= rzv2h_icu_set_type,
+	.irq_set_type		= rzv2h_irq_set_type,
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 	.flags			= IRQCHIP_MASK_ON_SUSPEND |
 				  IRQCHIP_SET_TYPE_MASKED |
 				  IRQCHIP_SKIP_SET_WAKE,
 };
 
+static const struct irq_chip rzv2h_icu_nmi_chip = {
+	.name			= "rzv2h-icu",
+	.irq_eoi		= rzv2h_icu_nmi_eoi,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_disable		= irq_chip_disable_parent,
+	.irq_enable		= irq_chip_enable_parent,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= irq_chip_set_parent_state,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= rzv2h_nmi_set_type,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE,
+};
+
+static const struct irq_chip rzv2h_icu_swint_chip = {
+	.name			= "rzv2h-icu",
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_disable		= irq_chip_disable_parent,
+	.irq_enable		= irq_chip_enable_parent,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= rzv2h_icu_swint_set_irqchip_state,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE,
+};
+
+static const struct irq_chip rzv2h_icu_swpe_err_chip = {
+	.name			= "rzv2h-icu",
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_disable		= irq_chip_disable_parent,
+	.irq_enable		= irq_chip_enable_parent,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= rzv2h_icu_swpe_set_irqchip_state,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= irq_chip_set_type_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE,
+};
+
+#define hwirq_within(hwirq, which)	((hwirq) >= which##_START && (hwirq) <= which##_LAST)
+
 static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
 			   void *arg)
 {
 	struct rzv2h_icu_priv *priv = domain->host_data;
+	const struct irq_chip *chip;
 	unsigned long tint = 0;
 	irq_hw_number_t hwirq;
 	unsigned int type;
@@ -508,19 +651,27 @@ static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigne
 	 * hwirq is embedded in bits 0-15.
 	 * TINT is embedded in bits 16-31.
 	 */
-	if (hwirq >= ICU_TINT_START) {
-		tint = ICU_TINT_EXTRACT_GPIOINT(hwirq);
+	tint = ICU_TINT_EXTRACT_GPIOINT(hwirq);
+	if (tint || hwirq_within(hwirq, ICU_TINT)) {
 		hwirq = ICU_TINT_EXTRACT_HWIRQ(hwirq);
 
-		if (hwirq < ICU_TINT_START)
+		if (!hwirq_within(hwirq, ICU_TINT))
 			return -EINVAL;
+		chip = &rzv2h_icu_tint_chip;
+	} else if (hwirq_within(hwirq, ICU_IRQ)) {
+		chip = &rzv2h_icu_irq_chip;
+	} else if (hwirq_within(hwirq, ICU_CA55_INT)) {
+		chip = &rzv2h_icu_swint_chip;
+	} else if (hwirq_within(hwirq, ICU_ERR_INT)) {
+		chip = &rzv2h_icu_swpe_err_chip;
+	} else {
+		chip = &rzv2h_icu_nmi_chip;
 	}
 
 	if (hwirq > (ICU_NUM_IRQ - 1))
 		return -EINVAL;
 
-	ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &rzv2h_icu_chip,
-					    (void *)(uintptr_t)tint);
+	ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, (void *)(uintptr_t)tint);
 	if (ret)
 		return ret;
 
@@ -550,62 +701,170 @@ static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device
 	return 0;
 }
 
+static irqreturn_t rzv2h_icu_error_irq(int irq, void *data)
+{
+	struct rzv2h_icu_priv *priv = data;
+	const struct rzv2h_hw_info *hw_info = priv->info;
+	void __iomem *base = priv->base;
+	unsigned int k;
+	u32 st;
+
+	/* 1) Bus errors (BEISR0..3) */
+	for (k = 0; k < ICU_NUM_BE; k++) {
+		st = readl(base + ICU_BEISR(k));
+		if (!st)
+			continue;
+
+		writel_relaxed(st, base + ICU_BECLR(k));
+		pr_warn("rzv2h-icu: BUS error k=%u status=0x%08x\n", k, st);
+	}
+
+	/* 2) ECC RAM errors (EREISR0..X) */
+	for (k = hw_info->ecc_start; k <= hw_info->ecc_end; k++) {
+		st = readl(base + ICU_EREISR(k));
+		if (!st)
+			continue;
+
+		writel_relaxed(st, base + ICU_ERCLR(k));
+		pr_warn("rzv2h-icu: ECC error k=%u status=0x%08x\n", k, st);
+	}
+
+	/* 3) IP/CA55 error interrupt status (ERINTA55CTL0..3) */
+	for (k = 0; k < ICU_NUM_A55ERR; k++) {
+		st = readl(base + ICU_ERINTA55CTL(k));
+		if (!st)
+			continue;
+
+		/* there is no relation with status bits so clear all the interrupts */
+		writel_relaxed(0xffffffff, base + ICU_ERINTA55CRL(k));
+		pr_warn("rzv2h-icu: IP/CA55 error k=%u status=0x%08x\n", k, st);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t rzv2h_icu_swint_irq(int irq, void *data)
+{
+	u8 cpu = *(u8 *)data;
+
+	pr_info("SWINT interrupt for CA55 core %u\n", cpu);
+	return IRQ_HANDLED;
+}
+
+static int rzv2h_icu_setup_irqs(struct platform_device *pdev, struct irq_domain *irq_domain)
+{
+	const struct rzv2h_hw_info *hw_info = rzv2h_icu_data->info;
+	bool irq_inject = IS_ENABLED(CONFIG_GENERIC_IRQ_INJECTION);
+	static const char * const rzv2h_swint_names[] = {
+		"int-ca55-0", "int-ca55-1",
+		"int-ca55-2", "int-ca55-3",
+	};
+	static const char *icu_err = "icu-error-ca55";
+	static const u8 swint_idx[] = { 0, 1, 2, 3 };
+	void __iomem *base = rzv2h_icu_data->base;
+	struct device *dev = &pdev->dev;
+	struct irq_fwspec fwspec;
+	unsigned int i, virq;
+	int ret;
+
+	for (i = 0; i < ICU_CA55_INT_COUNT && irq_inject; i++) {
+		fwspec.fwnode = irq_domain->fwnode;
+		fwspec.param_count = 2;
+		fwspec.param[0] = ICU_CA55_INT_START + i;
+		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
+
+		virq = irq_create_fwspec_mapping(&fwspec);
+		if (!virq) {
+			return dev_err_probe(dev, -EINVAL, "failed to create IRQ mapping for %s\n",
+					     rzv2h_swint_names[i]);
+		}
+
+		ret = devm_request_irq(dev, virq, rzv2h_icu_swint_irq, 0, dev_name(dev),
+				       (void *)&swint_idx[i]);
+		if (ret) {
+			return dev_err_probe(dev, ret, "Failed to request %s IRQ\n",
+					     rzv2h_swint_names[i]);
+		}
+	}
+
+	/* Unmask and clear all IP/CA55 error interrupts */
+	for (i = 0; i < ICU_NUM_A55ERR; i++) {
+		writel_relaxed(0xffffff, base + ICU_ERINTA55CRL(i));
+		writel_relaxed(0x0, base + ICU_ERINTA55MSK(i));
+	}
+
+	/* Clear all Bus errors */
+	for (i = 0; i < ICU_NUM_BE; i++)
+		writel_relaxed(0xffffffff, base + ICU_BECLR(i));
+
+	/* Clear all ECCRAM errors */
+	for (i = hw_info->ecc_start; i <= hw_info->ecc_end; i++)
+		writel_relaxed(0xffffffff, base + ICU_ERCLR(i));
+
+	fwspec.fwnode = irq_domain->fwnode;
+	fwspec.param_count = 2;
+	fwspec.param[0] = ICU_ERR_INT_START;
+	fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH;
+
+	virq = irq_create_fwspec_mapping(&fwspec);
+	if (!virq) {
+		return dev_err_probe(dev, -EINVAL, "failed to create IRQ mapping for %s\n",
+				     icu_err);
+	}
+
+	ret = devm_request_irq(dev, virq, rzv2h_icu_error_irq, 0, dev_name(dev), rzv2h_icu_data);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to request %s IRQ\n", icu_err);
+
+	return 0;
+}
+
 static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_node *parent,
 				  const struct rzv2h_hw_info *hw_info)
 {
 	struct irq_domain *irq_domain, *parent_domain;
 	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
 	struct reset_control *resetn;
 	int ret;
 
 	parent_domain = irq_find_host(parent);
-	if (!parent_domain) {
-		dev_err(&pdev->dev, "cannot find parent domain\n");
-		return -ENODEV;
-	}
+	if (!parent_domain)
+		return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n");
 
-	rzv2h_icu_data = devm_kzalloc(&pdev->dev, sizeof(*rzv2h_icu_data), GFP_KERNEL);
+	rzv2h_icu_data = devm_kzalloc(dev, sizeof(*rzv2h_icu_data), GFP_KERNEL);
 	if (!rzv2h_icu_data)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, rzv2h_icu_data);
 
-	rzv2h_icu_data->base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL);
+	rzv2h_icu_data->base = devm_of_iomap(dev, node, 0, NULL);
 	if (IS_ERR(rzv2h_icu_data->base))
 		return PTR_ERR(rzv2h_icu_data->base);
 
 	ret = rzv2h_icu_parse_interrupts(rzv2h_icu_data, node);
-	if (ret) {
-		dev_err(&pdev->dev, "cannot parse interrupts: %d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "cannot parse interrupts\n");
 
-	resetn = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL);
-	if (IS_ERR(resetn)) {
-		ret = PTR_ERR(resetn);
-		dev_err(&pdev->dev, "failed to acquire deasserted reset: %d\n", ret);
-		return ret;
-	}
+	resetn = devm_reset_control_get_exclusive_deasserted(dev, NULL);
+	if (IS_ERR(resetn))
+		return dev_err_probe(dev, PTR_ERR(resetn), "failed to acquire deasserted reset\n");
 
-	ret = devm_pm_runtime_enable(&pdev->dev);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "devm_pm_runtime_enable failed, %d\n", ret);
-		return ret;
-	}
+	ret = devm_pm_runtime_enable(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "devm_pm_runtime_enable failed\n");
 
-	ret = pm_runtime_resume_and_get(&pdev->dev);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "pm_runtime_resume_and_get failed: %d\n", ret);
-		return ret;
-	}
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "pm_runtime_resume_and_get failed\n");
 
 	raw_spin_lock_init(&rzv2h_icu_data->lock);
 
 	irq_domain = irq_domain_create_hierarchy(parent_domain, 0, ICU_NUM_IRQ,
-						 dev_fwnode(&pdev->dev), &rzv2h_icu_domain_ops,
+						 dev_fwnode(dev), &rzv2h_icu_domain_ops,
 						 rzv2h_icu_data);
 	if (!irq_domain) {
-		dev_err(&pdev->dev, "failed to add irq domain\n");
+		dev_err(dev, "failed to add irq domain\n");
 		ret = -ENOMEM;
 		goto pm_put;
 	}
@@ -614,14 +873,18 @@ static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_no
 
 	register_syscore(&rzv2h_irqc_syscore);
 
+	ret = rzv2h_icu_setup_irqs(pdev, irq_domain);
+	if (ret)
+		goto pm_put;
+
 	/*
 	 * coccicheck complains about a missing put_device call before returning, but it's a false
-	 * positive. We still need &pdev->dev after successfully returning from this function.
+	 * positive. We still need dev after successfully returning from this function.
 	 */
 	return 0;
 
 pm_put:
-	pm_runtime_put(&pdev->dev);
+	pm_runtime_put(dev);
 
 	return ret;
 }
@@ -657,12 +920,24 @@ static const struct rzv2h_hw_info rzg3e_hw_params = {
 	.t_offs		= ICU_RZG3E_TINT_OFFSET,
 	.max_tssel	= ICU_RZG3E_TSSEL_MAX_VAL,
 	.field_width	= 16,
+	.ecc_start	= 1,
+	.ecc_end	= 4,
+};
+
+static const struct rzv2h_hw_info rzv2n_hw_params = {
+	.t_offs		= 0,
+	.max_tssel	= ICU_RZV2H_TSSEL_MAX_VAL,
+	.field_width	= 8,
+	.ecc_start	= 0,
+	.ecc_end	= 2,
 };
 
 static const struct rzv2h_hw_info rzv2h_hw_params = {
 	.t_offs		= 0,
 	.max_tssel	= ICU_RZV2H_TSSEL_MAX_VAL,
 	.field_width	= 8,
+	.ecc_start	= 0,
+	.ecc_end	= 11,
 };
 
 static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *parent)
@@ -670,6 +945,11 @@ static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *par
 	return rzv2h_icu_probe_common(pdev, parent, &rzg3e_hw_params);
 }
 
+static int rzv2n_icu_probe(struct platform_device *pdev, struct device_node *parent)
+{
+	return rzv2h_icu_probe_common(pdev, parent, &rzv2n_hw_params);
+}
+
 static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *parent)
 {
 	return rzv2h_icu_probe_common(pdev, parent, &rzv2h_hw_params);
@@ -677,7 +957,7 @@ static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *par
 
 IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu)
 IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe)
-IRQCHIP_MATCH("renesas,r9a09g056-icu", rzv2h_icu_probe)
+IRQCHIP_MATCH("renesas,r9a09g056-icu", rzv2n_icu_probe)
 IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe)
 IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu)
 MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>");
diff --git a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c
index 5c74c561ce316..612f3972f7af0 100644
--- a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c
+++ b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c
@@ -250,6 +250,7 @@ static int rpmi_sysmsi_probe(struct platform_device *pdev)
 		rc = riscv_acpi_get_gsi_info(fwnode, &priv->gsi_base, &id,
 					     &nr_irqs, NULL);
 		if (rc) {
+			mbox_free_channel(priv->chan);
 			dev_err(dev, "failed to find GSI mapping\n");
 			return rc;
 		}
diff --git a/drivers/leds/blink/leds-lgm-sso.c b/drivers/leds/blink/leds-lgm-sso.c
index 8923d2df47049..3d9ef9a54805c 100644
--- a/drivers/leds/blink/leds-lgm-sso.c
+++ b/drivers/leds/blink/leds-lgm-sso.c
@@ -808,8 +808,6 @@ static int intel_sso_led_probe(struct platform_device *pdev)
 
 	priv->fpid_clkrate = clk_get_rate(priv->clocks[1].clk);
 
-	priv->mmap = syscon_node_to_regmap(dev->of_node);
-
 	priv->mmap = syscon_node_to_regmap(dev->of_node);
 	if (IS_ERR(priv->mmap)) {
 		dev_err(dev, "Failed to map iomem!\n");
diff --git a/drivers/leds/flash/leds-ktd2692.c b/drivers/leds/flash/leds-ktd2692.c
index 0f16eefcfe4cd..22fbfccd48735 100644
--- a/drivers/leds/flash/leds-ktd2692.c
+++ b/drivers/leds/flash/leds-ktd2692.c
@@ -59,7 +59,7 @@ struct ktd2692_led_config_data {
 	enum led_brightness max_brightness;
 };
 
-const struct expresswire_timing ktd2692_timing = {
+static const struct expresswire_timing ktd2692_timing = {
 	.poweroff_us = 700,
 	.data_start_us = 10,
 	.end_of_data_low_us = 10,
diff --git a/drivers/leds/led-class-multicolor.c b/drivers/leds/led-class-multicolor.c
index fd66d2bdeace8..6b671f3f9c617 100644
--- a/drivers/leds/led-class-multicolor.c
+++ b/drivers/leds/led-class-multicolor.c
@@ -34,14 +34,14 @@ static ssize_t multi_intensity_store(struct device *dev,
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct led_classdev_mc *mcled_cdev = lcdev_to_mccdev(led_cdev);
 	int nrchars, offset = 0;
-	int intensity_value[LED_COLOR_ID_MAX];
+	unsigned int intensity_value[LED_COLOR_ID_MAX];
 	int i;
 	ssize_t ret;
 
 	mutex_lock(&led_cdev->led_access);
 
 	for (i = 0; i < mcled_cdev->num_colors; i++) {
-		ret = sscanf(buf + offset, "%i%n",
+		ret = sscanf(buf + offset, "%u%n",
 			     &intensity_value[i], &nrchars);
 		if (ret != 1) {
 			ret = -EINVAL;
diff --git a/drivers/leds/leds-lp5569.c b/drivers/leds/leds-lp5569.c
index 786f2aa35319e..a252ba6c455d5 100644
--- a/drivers/leds/leds-lp5569.c
+++ b/drivers/leds/leds-lp5569.c
@@ -410,12 +410,12 @@ static ssize_t lp5569_selftest(struct device *dev,
 	/* Test LED Open */
 	pos = lp5569_led_open_test(led, buf);
 	if (pos < 0)
-		return sprintf(buf, "FAIL\n");
+		return sysfs_emit(buf, "FAIL\n");
 
 	/* Test LED Shorted */
 	pos += lp5569_led_short_test(led, buf);
 	if (pos < 0)
-		return sprintf(buf, "FAIL\n");
+		return sysfs_emit(buf, "FAIL\n");
 
 	for (i = 0; i < chip->pdata->num_channels; i++) {
 		/* Restore current */
diff --git a/drivers/leds/leds-lp8860.c b/drivers/leds/leds-lp8860.c
index 0962c00c215a1..7a436861c4b71 100644
--- a/drivers/leds/leds-lp8860.c
+++ b/drivers/leds/leds-lp8860.c
@@ -89,16 +89,24 @@
  * @client: Pointer to the I2C client
  * @led_dev: led class device pointer
  * @regmap: Devices register map
- * @eeprom_regmap: EEPROM register map
  */
 struct lp8860_led {
 	struct mutex lock;
 	struct i2c_client *client;
 	struct led_classdev led_dev;
 	struct regmap *regmap;
-	struct regmap *eeprom_regmap;
 };
 
+static bool program_eeprom;
+module_param(program_eeprom, bool, 0644);
+MODULE_PARM_DESC(program_eeprom, "Program the configuration EEPROM on device startup");
+
+/*
+ * EEPROM bits are intended to be set/programmed before normal operation only
+ * once during silicon production, but can be reprogrammed for evaluation purposes
+ * up to 1000 cycles. To program this EEPROM using this driver, update the below
+ * table and set the module param "program_eeprom" to 1
+ */
 static const struct reg_sequence lp8860_eeprom_disp_regs[] = {
 	{ LP8860_EEPROM_REG_0, 0xed },
 	{ LP8860_EEPROM_REG_1, 0xdf },
@@ -127,32 +135,6 @@ static const struct reg_sequence lp8860_eeprom_disp_regs[] = {
 	{ LP8860_EEPROM_REG_24, 0x3E },
 };
 
-static int lp8860_unlock_eeprom(struct lp8860_led *led)
-{
-	int ret;
-
-	guard(mutex)(&led->lock);
-
-	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_1);
-	if (ret) {
-		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
-		return ret;
-	}
-
-	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_2);
-	if (ret) {
-		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
-		return ret;
-	}
-	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_3);
-	if (ret) {
-		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
-		return ret;
-	}
-
-	return ret;
-}
-
 static int lp8860_fault_check(struct lp8860_led *led)
 {
 	int ret, fault;
@@ -211,61 +193,72 @@ static int lp8860_brightness_set(struct led_classdev *led_cdev,
 	return 0;
 }
 
-static int lp8860_init(struct lp8860_led *led)
+static int lp8860_program_eeprom(struct lp8860_led *led)
 {
-	unsigned int read_buf;
 	int ret, reg_count;
 
+	guard(mutex)(&led->lock);
+
 	ret = lp8860_fault_check(led);
-	if (ret)
-		goto out;
+	if (ret) {
+		dev_err(&led->client->dev, "Cannot read/clear faults\n");
+		return ret;
+	}
 
-	ret = regmap_read(led->regmap, LP8860_STATUS, &read_buf);
-	if (ret)
-		goto out;
+	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_1);
+	if (ret) {
+		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
+		return ret;
+	}
 
-	ret = lp8860_unlock_eeprom(led);
+	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_2);
 	if (ret) {
-		dev_err(&led->client->dev, "Failed unlocking EEPROM\n");
-		goto out;
+		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
+		return ret;
+	}
+	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_EEPROM_CODE_3);
+	if (ret) {
+		dev_err(&led->client->dev, "EEPROM Unlock failed\n");
+		return ret;
 	}
 
 	reg_count = ARRAY_SIZE(lp8860_eeprom_disp_regs);
-	ret = regmap_multi_reg_write(led->eeprom_regmap, lp8860_eeprom_disp_regs, reg_count);
+	ret = regmap_multi_reg_write(led->regmap, lp8860_eeprom_disp_regs, reg_count);
 	if (ret) {
 		dev_err(&led->client->dev, "Failed writing EEPROM\n");
-		goto out;
+		return ret;
 	}
 
 	ret = regmap_write(led->regmap, LP8860_EEPROM_UNLOCK, LP8860_LOCK_EEPROM);
 	if (ret)
-		goto out;
+		return ret;
 
 	ret = regmap_write(led->regmap,
 			LP8860_EEPROM_CNTRL,
 			LP8860_PROGRAM_EEPROM);
 	if (ret) {
 		dev_err(&led->client->dev, "Failed programming EEPROM\n");
-		goto out;
+		return ret;
 	}
 
-	return ret;
-
-out:
-	return ret;
+	return 0;
 }
 
-static const struct regmap_config lp8860_regmap_config = {
-	.reg_bits = 8,
-	.val_bits = 8,
+static const struct regmap_range lp8860_reg_ranges[] = {
+	regmap_reg_range(LP8860_DISP_CL1_BRT_MSB, LP8860_EEPROM_UNLOCK),
+	regmap_reg_range(LP8860_EEPROM_REG_0, LP8860_EEPROM_REG_24),
+};
 
-	.max_register = LP8860_EEPROM_UNLOCK,
+static const struct regmap_access_table lp8860_reg_table = {
+	.yes_ranges = lp8860_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(lp8860_reg_ranges),
 };
 
-static const struct regmap_config lp8860_eeprom_regmap_config = {
+static const struct regmap_config lp8860_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
-
+	.rd_table = &lp8860_reg_table,
+	.wr_table = &lp8860_reg_table,
 	.max_register = LP8860_EEPROM_REG_24,
 };
 
@@ -319,18 +312,12 @@ static int lp8860_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	led->eeprom_regmap = devm_regmap_init_i2c(client, &lp8860_eeprom_regmap_config);
-	if (IS_ERR(led->eeprom_regmap)) {
-		ret = PTR_ERR(led->eeprom_regmap);
-		dev_err(&client->dev, "Failed to allocate register map: %d\n",
-			ret);
-		return ret;
+	if (program_eeprom) {
+		ret = lp8860_program_eeprom(led);
+		if (ret)
+			return ret;
 	}
 
-	ret = lp8860_init(led);
-	if (ret)
-		return ret;
-
 	init_data.fwnode = of_fwnode_handle(child_node);
 	init_data.devicename = LP8860_NAME;
 	init_data.default_label = ":display_cluster";
diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c
index 016bf468e0941..f6061c47f863d 100644
--- a/drivers/leds/rgb/leds-qcom-lpg.c
+++ b/drivers/leds/rgb/leds-qcom-lpg.c
@@ -1273,7 +1273,12 @@ static int lpg_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
 		return ret;
 
 	if (chan->subtype == LPG_SUBTYPE_HI_RES_PWM) {
-		refclk = lpg_clk_rates_hi_res[FIELD_GET(PWM_CLK_SELECT_HI_RES_MASK, val)];
+		unsigned int clk_idx = FIELD_GET(PWM_CLK_SELECT_HI_RES_MASK, val);
+
+		if (clk_idx >= ARRAY_SIZE(lpg_clk_rates_hi_res))
+			return -EINVAL;
+
+		refclk = lpg_clk_rates_hi_res[clk_idx];
 		resolution = lpg_pwm_resolution_hi_res[FIELD_GET(PWM_SIZE_HI_RES_MASK, val)];
 	} else {
 		refclk = lpg_clk_rates[FIELD_GET(PWM_CLK_SELECT_MASK, val)];
diff --git a/drivers/leds/trigger/ledtrig-tty.c b/drivers/leds/trigger/ledtrig-tty.c
index 8eb6286b33ac3..3725571144d91 100644
--- a/drivers/leds/trigger/ledtrig-tty.c
+++ b/drivers/leds/trigger/ledtrig-tty.c
@@ -220,7 +220,7 @@ static void ledtrig_tty_work(struct work_struct *work)
 			goto out;
 
 		tty = tty_kopen_shared(devno);
-		if (IS_ERR(tty) || !tty)
+		if (IS_ERR_OR_NULL(tty))
 			/* What to do? retry or abort */
 			goto out;
 
diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c
index 617ba505691d3..354434cd12097 100644
--- a/drivers/mailbox/mailbox.c
+++ b/drivers/mailbox/mailbox.c
@@ -218,6 +218,29 @@ bool mbox_client_peek_data(struct mbox_chan *chan)
 }
 EXPORT_SYMBOL_GPL(mbox_client_peek_data);
 
+/**
+ * mbox_chan_tx_slots_available - Query the number of available TX queue slots.
+ * @chan: Mailbox channel to query.
+ *
+ * Clients may call this to check how many messages can be queued via
+ * mbox_send_message() before the channel's TX queue is full. This helps
+ * clients avoid the -ENOBUFS error without needing to increase
+ * MBOX_TX_QUEUE_LEN.
+ * This can be called from atomic context.
+ *
+ * Return: Number of available slots in the channel's TX queue.
+ */
+unsigned int mbox_chan_tx_slots_available(struct mbox_chan *chan)
+{
+	unsigned int ret;
+
+	guard(spinlock_irqsave)(&chan->lock);
+	ret = MBOX_TX_QUEUE_LEN - chan->msg_count;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mbox_chan_tx_slots_available);
+
 /**
  * mbox_send_message -	For client to submit a message to be
  *				sent to the remote.
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index c58a9a8ea54e9..53351048d3ec6 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -226,6 +226,7 @@ config BLK_DEV_DM
 	select BLOCK_HOLDER_DEPRECATED if SYSFS
 	select BLK_DEV_DM_BUILTIN
 	select BLK_MQ_STACKING
+	select CRYPTO_LIB_SHA256 if IMA
 	depends on DAX || DAX=n
 	help
 	  Device-mapper is a low level volume manager.  It works by allowing
diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c
index 57158c02d096e..acd9b179fcb3f 100644
--- a/drivers/md/dm-cache-metadata.c
+++ b/drivers/md/dm-cache-metadata.c
@@ -1023,6 +1023,12 @@ static bool cmd_write_lock(struct dm_cache_metadata *cmd)
 			return;			\
 	} while (0)
 
+#define WRITE_LOCK_OR_GOTO(cmd, label)		\
+	do {					\
+		if (!cmd_write_lock((cmd)))	\
+			goto label;		\
+	} while (0)
+
 #define WRITE_UNLOCK(cmd) \
 	up_write(&(cmd)->root_lock)
 
@@ -1714,17 +1720,6 @@ int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *
 	return r;
 }
 
-int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
-{
-	int r;
-
-	READ_LOCK(cmd);
-	r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
-	READ_UNLOCK(cmd);
-
-	return r;
-}
-
 void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
 {
 	WRITE_LOCK_VOID(cmd);
@@ -1791,11 +1786,8 @@ int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
 	new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
 					 CACHE_MAX_CONCURRENT_LOCKS);
 
-	WRITE_LOCK(cmd);
-	if (cmd->fail_io) {
-		WRITE_UNLOCK(cmd);
-		goto out;
-	}
+	/* cmd_write_lock() already checks fail_io with cmd->root_lock held */
+	WRITE_LOCK_OR_GOTO(cmd, out);
 
 	__destroy_persistent_data_objects(cmd, false);
 	old_bm = cmd->bm;
@@ -1824,3 +1816,12 @@ int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
 
 	return r;
 }
+
+int dm_cache_metadata_clean_when_opened(struct dm_cache_metadata *cmd, bool *result)
+{
+	READ_LOCK(cmd);
+	*result = cmd->clean_when_opened;
+	READ_UNLOCK(cmd);
+
+	return 0;
+}
diff --git a/drivers/md/dm-cache-metadata.h b/drivers/md/dm-cache-metadata.h
index 5f77890207fed..91f8706b41fdd 100644
--- a/drivers/md/dm-cache-metadata.h
+++ b/drivers/md/dm-cache-metadata.h
@@ -135,17 +135,17 @@ int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
  */
 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *p);
 
-/*
- * Query method.  Are all the blocks in the cache clean?
- */
-int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result);
-
 int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result);
 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd);
 void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd);
 void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd);
 int dm_cache_metadata_abort(struct dm_cache_metadata *cmd);
 
+/*
+ * Query method.  Was the metadata cleanly shut down when opened?
+ */
+int dm_cache_metadata_clean_when_opened(struct dm_cache_metadata *cmd, bool *result);
+
 /*----------------------------------------------------------------*/
 
 #endif /* DM_CACHE_METADATA_H */
diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c
index b328d9601046b..dd77a93fd68d2 100644
--- a/drivers/md/dm-cache-policy-smq.c
+++ b/drivers/md/dm-cache-policy-smq.c
@@ -1589,14 +1589,18 @@ static int smq_invalidate_mapping(struct dm_cache_policy *p, dm_cblock_t cblock)
 {
 	struct smq_policy *mq = to_smq_policy(p);
 	struct entry *e = get_entry(&mq->cache_alloc, from_cblock(cblock));
+	unsigned long flags;
 
 	if (!e->allocated)
 		return -ENODATA;
 
+	spin_lock_irqsave(&mq->lock, flags);
 	// FIXME: what if this block has pending background work?
 	del_queue(mq, e);
 	h_remove(&mq->table, e);
 	free_entry(&mq->cache_alloc, e);
+	spin_unlock_irqrestore(&mq->lock, flags);
+
 	return 0;
 }
 
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 935ab79b1d0cd..f8200c1548052 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -1462,11 +1462,19 @@ static void invalidate_complete(struct dm_cache_migration *mg, bool success)
 	struct cache *cache = mg->cache;
 
 	bio_list_init(&bios);
-	if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
-		free_prison_cell(cache, mg->cell);
+	if (mg->cell) {
+		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
+			free_prison_cell(cache, mg->cell);
+	}
 
-	if (!success && mg->overwrite_bio)
-		bio_io_error(mg->overwrite_bio);
+	if (mg->overwrite_bio) {
+		// Set generic error if the bio hasn't been issued yet,
+		// e.g., invalidation or metadata commit failed before bio
+		// submission. Otherwise preserve the bio's own error status.
+		if (!success && !mg->overwrite_bio->bi_status)
+			mg->overwrite_bio->bi_status = BLK_STS_IOERR;
+		bio_endio(mg->overwrite_bio);
+	}
 
 	free_migration(mg);
 	defer_bios(cache, &bios);
@@ -1506,6 +1514,22 @@ static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
 	return r;
 }
 
+static void invalidate_committed(struct work_struct *ws)
+{
+	struct dm_cache_migration *mg = ws_to_mg(ws);
+	struct cache *cache = mg->cache;
+	struct bio *bio = mg->overwrite_bio;
+	struct per_bio_data *pb = get_per_bio_data(bio);
+
+	if (mg->k.input)
+		invalidate_complete(mg, false);
+
+	init_continuation(&mg->k, invalidate_completed);
+	remap_to_origin_clear_discard(cache, bio, mg->invalidate_oblock);
+	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
+	dm_submit_bio_remap(bio, NULL);
+}
+
 static void invalidate_remove(struct work_struct *ws)
 {
 	int r;
@@ -1518,10 +1542,8 @@ static void invalidate_remove(struct work_struct *ws)
 		return;
 	}
 
-	init_continuation(&mg->k, invalidate_completed);
+	init_continuation(&mg->k, invalidate_committed);
 	continue_after_commit(&cache->committer, &mg->k);
-	remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
-	mg->overwrite_bio = NULL;
 	schedule_commit(&cache->committer);
 }
 
@@ -1539,6 +1561,15 @@ static int invalidate_lock(struct dm_cache_migration *mg)
 			    READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
 	if (r < 0) {
 		free_prison_cell(cache, prealloc);
+
+		/* Defer the bio for retrying the cell lock */
+		if (mg->overwrite_bio) {
+			struct bio *bio = mg->overwrite_bio;
+
+			mg->overwrite_bio = NULL;
+			defer_bio(cache, bio);
+		}
+
 		invalidate_complete(mg, false);
 		return r;
 	}
@@ -1701,6 +1732,7 @@ static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
 				bio_drop_shared_lock(cache, bio);
 				atomic_inc(&cache->stats.demotion);
 				invalidate_start(cache, cblock, block, bio);
+				return DM_MAPIO_SUBMITTED;
 			} else
 				remap_to_origin_clear_discard(cache, bio, block);
 		} else {
@@ -2467,23 +2499,8 @@ static int cache_create(struct cache_args *ca, struct cache **result)
 		goto bad;
 	}
 
-	if (passthrough_mode(cache)) {
-		bool all_clean;
-
-		r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
-		if (r) {
-			*error = "dm_cache_metadata_all_clean() failed";
-			goto bad;
-		}
-
-		if (!all_clean) {
-			*error = "Cannot enter passthrough mode unless all blocks are clean";
-			r = -EINVAL;
-			goto bad;
-		}
-
+	if (passthrough_mode(cache))
 		policy_allow_migrations(cache->policy, false);
-	}
 
 	spin_lock_init(&cache->lock);
 	bio_list_init(&cache->deferred_bios);
@@ -2810,6 +2827,12 @@ static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
 	struct cache *cache = context;
 
 	if (dirty) {
+		if (passthrough_mode(cache)) {
+			DMERR("%s: cannot enter passthrough mode unless all blocks are clean",
+			      cache_device_name(cache));
+			return -EBUSY;
+		}
+
 		set_bit(from_cblock(cblock), cache->dirty_bitset);
 		atomic_inc(&cache->nr_dirty);
 	} else
@@ -2929,6 +2952,9 @@ static dm_cblock_t get_cache_dev_size(struct cache *cache)
 
 static bool can_resume(struct cache *cache)
 {
+	bool clean_when_opened;
+	int r;
+
 	/*
 	 * Disallow retrying the resume operation for devices that failed the
 	 * first resume attempt, as the failure leaves the policy object partially
@@ -2945,6 +2971,20 @@ static bool can_resume(struct cache *cache)
 		return false;
 	}
 
+	if (passthrough_mode(cache)) {
+		r = dm_cache_metadata_clean_when_opened(cache->cmd, &clean_when_opened);
+		if (r) {
+			DMERR("%s: failed to query metadata flags", cache_device_name(cache));
+			return false;
+		}
+
+		if (!clean_when_opened) {
+			DMERR("%s: unable to resume into passthrough mode after unclean shutdown",
+			      cache_device_name(cache));
+			return false;
+		}
+	}
+
 	return true;
 }
 
@@ -3043,7 +3083,7 @@ static int cache_preresume(struct dm_target *ti)
 					   load_filtered_mapping, cache);
 		if (r) {
 			DMERR("%s: could not load cache mappings", cache_device_name(cache));
-			if (r != -EFBIG)
+			if (r != -EFBIG && r != -EBUSY)
 				metadata_operation_failed(cache, "dm_cache_load_mappings", r);
 			return r;
 		}
@@ -3510,7 +3550,7 @@ static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
 
 static struct target_type cache_target = {
 	.name = "cache",
-	.version = {2, 3, 0},
+	.version = {2, 4, 0},
 	.module = THIS_MODULE,
 	.ctr = cache_ctr,
 	.dtr = cache_dtr,
diff --git a/drivers/md/dm-ima.c b/drivers/md/dm-ima.c
index efb3cd4f9cd45..9495ca0350563 100644
--- a/drivers/md/dm-ima.c
+++ b/drivers/md/dm-ima.c
@@ -12,9 +12,7 @@
 
 #include <linux/ima.h>
 #include <linux/sched/mm.h>
-#include <crypto/hash.h>
-#include <linux/crypto.h>
-#include <crypto/hash_info.h>
+#include <crypto/sha2.h>
 
 #define DM_MSG_PREFIX "ima"
 
@@ -178,19 +176,13 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	size_t device_data_buf_len, target_metadata_buf_len, target_data_buf_len, l = 0;
 	char *target_metadata_buf = NULL, *target_data_buf = NULL, *digest_buf = NULL;
 	char *ima_buf = NULL, *device_data_buf = NULL;
-	int digest_size, last_target_measured = -1, r;
+	int last_target_measured = -1;
 	status_type_t type = STATUSTYPE_IMA;
 	size_t cur_total_buf_len = 0;
 	unsigned int num_targets, i;
-	SHASH_DESC_ON_STACK(shash, NULL);
-	struct crypto_shash *tfm = NULL;
-	u8 *digest = NULL;
+	struct sha256_ctx hash_ctx;
+	u8 digest[SHA256_DIGEST_SIZE];
 	bool noio = false;
-	/*
-	 * In below hash_alg_prefix_len assignment +1 is for the additional char (':'),
-	 * when prefixing the hash value with the hash algorithm name. e.g. sha256:<hash_value>.
-	 */
-	const size_t hash_alg_prefix_len = strlen(DM_IMA_TABLE_HASH_ALG) + 1;
 	char table_load_event_name[] = "dm_table_load";
 
 	ima_buf = dm_ima_alloc(DM_IMA_MEASUREMENT_BUF_LEN, noio);
@@ -210,19 +202,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio))
 		goto error;
 
-	tfm = crypto_alloc_shash(DM_IMA_TABLE_HASH_ALG, 0, 0);
-	if (IS_ERR(tfm))
-		goto error;
-
-	shash->tfm = tfm;
-	digest_size = crypto_shash_digestsize(tfm);
-	digest = dm_ima_alloc(digest_size, noio);
-	if (!digest)
-		goto error;
-
-	r = crypto_shash_init(shash);
-	if (r)
-		goto error;
+	sha256_init(&hash_ctx);
 
 	memcpy(ima_buf + l, DM_IMA_VERSION_STR, table->md->ima.dm_version_str_len);
 	l += table->md->ima.dm_version_str_len;
@@ -270,9 +250,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 		 */
 		if (unlikely(cur_total_buf_len >= DM_IMA_MEASUREMENT_BUF_LEN)) {
 			dm_ima_measure_data(table_load_event_name, ima_buf, l, noio);
-			r = crypto_shash_update(shash, (const u8 *)ima_buf, l);
-			if (r < 0)
-				goto error;
+			sha256_update(&hash_ctx, (const u8 *)ima_buf, l);
 
 			memset(ima_buf, 0, DM_IMA_MEASUREMENT_BUF_LEN);
 			l = 0;
@@ -311,9 +289,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	if (!last_target_measured) {
 		dm_ima_measure_data(table_load_event_name, ima_buf, l, noio);
 
-		r = crypto_shash_update(shash, (const u8 *)ima_buf, l);
-		if (r < 0)
-			goto error;
+		sha256_update(&hash_ctx, (const u8 *)ima_buf, l);
 	}
 
 	/*
@@ -321,20 +297,13 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	 * so that the table data can be verified against the future device state change
 	 * events, e.g. resume, rename, remove, table-clear etc.
 	 */
-	r = crypto_shash_final(shash, digest);
-	if (r < 0)
-		goto error;
-
-	digest_buf = dm_ima_alloc((digest_size*2) + hash_alg_prefix_len + 1, noio);
+	sha256_final(&hash_ctx, digest);
 
+	digest_buf = kasprintf(GFP_KERNEL, "sha256:%*phN", SHA256_DIGEST_SIZE,
+			       digest);
 	if (!digest_buf)
 		goto error;
 
-	snprintf(digest_buf, hash_alg_prefix_len + 1, "%s:", DM_IMA_TABLE_HASH_ALG);
-
-	for (i = 0; i < digest_size; i++)
-		snprintf((digest_buf + hash_alg_prefix_len + (i*2)), 3, "%02x", digest[i]);
-
 	if (table->md->ima.active_table.hash != table->md->ima.inactive_table.hash)
 		kfree(table->md->ima.inactive_table.hash);
 
@@ -354,9 +323,6 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	kfree(digest_buf);
 	kfree(device_data_buf);
 exit:
-	kfree(digest);
-	if (tfm)
-		crypto_free_shash(tfm);
 	kfree(ima_buf);
 	kfree(target_metadata_buf);
 	kfree(target_data_buf);
diff --git a/drivers/md/dm-ima.h b/drivers/md/dm-ima.h
index 568870a1a1456..a403deca6093e 100644
--- a/drivers/md/dm-ima.h
+++ b/drivers/md/dm-ima.h
@@ -15,7 +15,6 @@
 #define DM_IMA_TARGET_METADATA_BUF_LEN	128
 #define DM_IMA_TARGET_DATA_BUF_LEN	2048
 #define DM_IMA_DEVICE_CAPACITY_BUF_LEN	128
-#define DM_IMA_TABLE_HASH_ALG		"sha256"
 
 #define __dm_ima_stringify(s) #s
 #define __dm_ima_str(s) __dm_ima_stringify(s)
diff --git a/drivers/md/dm-init.c b/drivers/md/dm-init.c
index 7403823384c57..c1bacba92c65c 100644
--- a/drivers/md/dm-init.c
+++ b/drivers/md/dm-init.c
@@ -303,8 +303,10 @@ static int __init dm_init_init(void)
 		}
 	}
 
-	if (waitfor[0])
+	if (waitfor[0]) {
+		wait_for_device_probe();
 		DMINFO("all devices available");
+	}
 
 	list_for_each_entry(dev, &devices, list) {
 		if (dm_early_create(&dev->dmi, dev->table,
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 3ab8b4beff864..405acc14d7183 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -64,7 +64,11 @@ struct vers_iter {
 static struct rb_root name_rb_tree = RB_ROOT;
 static struct rb_root uuid_rb_tree = RB_ROOT;
 
-static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred);
+#define DM_REMOVE_KEEP_OPEN_DEVICES	1
+#define DM_REMOVE_MARK_DEFERRED		2
+#define DM_REMOVE_ONLY_DEFERRED		4
+#define DM_REMOVE_INTERRUPTIBLE		8
+static int dm_hash_remove_all(unsigned flags);
 
 /*
  * Guards access to both hash tables.
@@ -78,7 +82,7 @@ static DEFINE_MUTEX(dm_hash_cells_mutex);
 
 static void dm_hash_exit(void)
 {
-	dm_hash_remove_all(false, false, false);
+	dm_hash_remove_all(0);
 }
 
 /*
@@ -333,7 +337,7 @@ static struct dm_table *__hash_remove(struct hash_cell *hc)
 	return table;
 }
 
-static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred)
+static int dm_hash_remove_all(unsigned flags)
 {
 	int dev_skipped;
 	struct rb_node *n;
@@ -347,12 +351,17 @@ static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool
 	down_write(&_hash_lock);
 
 	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
+		if (flags & DM_REMOVE_INTERRUPTIBLE && fatal_signal_pending(current)) {
+			up_write(&_hash_lock);
+			return -EINTR;
+		}
+
 		hc = container_of(n, struct hash_cell, name_node);
 		md = hc->md;
 		dm_get(md);
 
-		if (keep_open_devices &&
-		    dm_lock_for_deletion(md, mark_deferred, only_deferred)) {
+		if (flags & DM_REMOVE_KEEP_OPEN_DEVICES &&
+		    dm_lock_for_deletion(md, !!(flags & DM_REMOVE_MARK_DEFERRED), !!(flags & DM_REMOVE_ONLY_DEFERRED))) {
 			dm_put(md);
 			dev_skipped++;
 			continue;
@@ -368,7 +377,7 @@ static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool
 		}
 		dm_ima_measure_on_device_remove(md, true);
 		dm_put(md);
-		if (likely(keep_open_devices))
+		if (likely(flags & DM_REMOVE_KEEP_OPEN_DEVICES))
 			dm_destroy(md);
 		else
 			dm_destroy_immediate(md);
@@ -384,8 +393,10 @@ static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool
 
 	up_write(&_hash_lock);
 
-	if (dev_skipped)
+	if (dev_skipped && !(flags & DM_REMOVE_ONLY_DEFERRED))
 		DMWARN("remove_all left %d open device(s)", dev_skipped);
+
+	return 0;
 }
 
 /*
@@ -513,7 +524,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 
 void dm_deferred_remove(void)
 {
-	dm_hash_remove_all(true, false, true);
+	dm_hash_remove_all(DM_REMOVE_KEEP_OPEN_DEVICES | DM_REMOVE_ONLY_DEFERRED);
 }
 
 /*
@@ -529,9 +540,13 @@ typedef int (*ioctl_fn)(struct file *filp, struct dm_ioctl *param, size_t param_
 
 static int remove_all(struct file *filp, struct dm_ioctl *param, size_t param_size)
 {
-	dm_hash_remove_all(true, !!(param->flags & DM_DEFERRED_REMOVE), false);
+	int r;
+	int flags = DM_REMOVE_KEEP_OPEN_DEVICES | DM_REMOVE_INTERRUPTIBLE;
+	if (param->flags & DM_DEFERRED_REMOVE)
+		flags |= DM_REMOVE_MARK_DEFERRED;
+	r = dm_hash_remove_all(flags);
 	param->data_size = 0;
-	return 0;
+	return r;
 }
 
 /*
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index 1aa6a4a7d2323..d316757a328be 100644
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -373,7 +373,7 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti,
 
 	struct log_c *lc;
 	uint32_t region_size;
-	unsigned int region_count;
+	sector_t region_count;
 	size_t bitset_size, buf_size;
 	int r;
 	char dummy;
@@ -401,6 +401,10 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti,
 	}
 
 	region_count = dm_sector_div_up(ti->len, region_size);
+	if (region_count > UINT_MAX) {
+		DMWARN("region count exceeds limit of %u", UINT_MAX);
+		return -EINVAL;
+	}
 
 	lc = kmalloc_obj(*lc);
 	if (!lc) {
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 8f4ae2f515453..7cb7bb6233b64 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -102,7 +102,6 @@ struct multipath {
 	struct bio_list queued_bios;
 
 	struct timer_list nopath_timer;	/* Timeout for queue_if_no_path */
-	bool is_suspending;
 };
 
 /*
@@ -1749,9 +1748,6 @@ static void multipath_presuspend(struct dm_target *ti)
 {
 	struct multipath *m = ti->private;
 
-	spin_lock_irq(&m->lock);
-	m->is_suspending = true;
-	spin_unlock_irq(&m->lock);
 	/* FIXME: bio-based shouldn't need to always disable queue_if_no_path */
 	if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti))
 		queue_if_no_path(m, false, true, __func__);
@@ -1774,7 +1770,6 @@ static void multipath_resume(struct dm_target *ti)
 	struct multipath *m = ti->private;
 
 	spin_lock_irq(&m->lock);
-	m->is_suspending = false;
 	if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) {
 		set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
 		clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
@@ -2098,7 +2093,7 @@ static int probe_active_paths(struct multipath *m)
 		if (m->current_pg == m->last_probed_pg)
 			goto skip_probe;
 	}
-	if (!m->current_pg || m->is_suspending ||
+	if (!m->current_pg || dm_suspended(m->ti) ||
 	    test_bit(MPATHF_QUEUE_IO, &m->flags))
 		goto skip_probe;
 	set_bit(MPATHF_DELAY_PG_SWITCH, &m->flags);
@@ -2107,7 +2102,7 @@ static int probe_active_paths(struct multipath *m)
 
 	list_for_each_entry(pgpath, &pg->pgpaths, list) {
 		if (pg != READ_ONCE(m->current_pg) ||
-		    READ_ONCE(m->is_suspending))
+		    dm_suspended(m->ti))
 			goto out;
 		if (!pgpath->is_active)
 			continue;
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 80a5c41277079..de5c00704e69c 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -993,13 +993,13 @@ static struct dm_dirty_log *create_dirty_log(struct dm_target *ti,
 		return NULL;
 	}
 
-	*args_used = 2 + param_count;
-
-	if (argc < *args_used) {
+	if (param_count > argc - 2) {
 		ti->error = "Insufficient mirror log arguments";
 		return NULL;
 	}
 
+	*args_used = 2 + param_count;
+
 	dl = dm_dirty_log_create(argv[0], ti, mirror_flush, param_count,
 				 argv + 2);
 	if (!dl) {
diff --git a/drivers/md/dm-vdo/action-manager.c b/drivers/md/dm-vdo/action-manager.c
index e3bba0b28aad8..b8a3977b815d9 100644
--- a/drivers/md/dm-vdo/action-manager.c
+++ b/drivers/md/dm-vdo/action-manager.c
@@ -107,7 +107,7 @@ int vdo_make_action_manager(zone_count_t zones,
 			    struct action_manager **manager_ptr)
 {
 	struct action_manager *manager;
-	int result = vdo_allocate(1, struct action_manager, __func__, &manager);
+	int result = vdo_allocate(1, __func__, &manager);
 
 	if (result != VDO_SUCCESS)
 		return result;
diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c
index a7db5b41155e9..5ffc360540ed1 100644
--- a/drivers/md/dm-vdo/block-map.c
+++ b/drivers/md/dm-vdo/block-map.c
@@ -221,8 +221,7 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache)
 	u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE;
 	int result;
 
-	result = vdo_allocate(cache->page_count, struct page_info, "page infos",
-			      &cache->infos);
+	result = vdo_allocate(cache->page_count, "page infos", &cache->infos);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2364,18 +2363,15 @@ static int make_segment(struct forest *old_forest, block_count_t new_pages,
 
 	forest->segments = index + 1;
 
-	result = vdo_allocate(forest->segments, struct boundary,
-			      "forest boundary array", &forest->boundaries);
+	result = vdo_allocate(forest->segments, "forest boundary array", &forest->boundaries);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(forest->segments, struct tree_page *,
-			      "forest page pointers", &forest->pages);
+	result = vdo_allocate(forest->segments, "forest page pointers", &forest->pages);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(new_pages, struct tree_page,
-			      "new forest pages", &forest->pages[index]);
+	result = vdo_allocate(new_pages, "new forest pages", &forest->pages[index]);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2400,9 +2396,7 @@ static int make_segment(struct forest *old_forest, block_count_t new_pages,
 		struct block_map_tree *tree = &(forest->trees[root]);
 		height_t height;
 
-		int result = vdo_allocate(forest->segments,
-					  struct block_map_tree_segment,
-					  "tree root segments", &tree->segments);
+		result = vdo_allocate(forest->segments, "tree root segments", &tree->segments);
 		if (result != VDO_SUCCESS)
 			return result;
 
@@ -2478,9 +2472,7 @@ static int make_forest(struct block_map *map, block_count_t entries)
 		return VDO_SUCCESS;
 	}
 
-	result = vdo_allocate_extended(struct forest, map->root_count,
-				       struct block_map_tree, __func__,
-				       &forest);
+	result = vdo_allocate_extended(map->root_count, trees, __func__, &forest);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2707,8 +2699,7 @@ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback,
 	struct cursors *cursors;
 	int result;
 
-	result = vdo_allocate_extended(struct cursors, map->root_count,
-				       struct cursor, __func__, &cursors);
+	result = vdo_allocate_extended(map->root_count, cursors, __func__, &cursors);
 	if (result != VDO_SUCCESS) {
 		vdo_fail_completion(completion, result);
 		return;
@@ -2758,9 +2749,7 @@ static int __must_check initialize_block_map_zone(struct block_map *map,
 	zone->thread_id = vdo->thread_config.logical_threads[zone_number];
 	zone->block_map = map;
 
-	result = vdo_allocate_extended(struct dirty_lists, maximum_age,
-				       dirty_era_t, __func__,
-				       &zone->dirty_lists);
+	result = vdo_allocate_extended(maximum_age, eras, __func__, &zone->dirty_lists);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2900,9 +2889,8 @@ int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate_extended(struct block_map,
-				       vdo->thread_config.logical_zone_count,
-				       struct block_map_zone, __func__, &map);
+	result = vdo_allocate_extended(vdo->thread_config.logical_zone_count,
+				       zones, __func__, &map);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h
index 39a13039e4a35..4fd24043b0d76 100644
--- a/drivers/md/dm-vdo/block-map.h
+++ b/drivers/md/dm-vdo/block-map.h
@@ -276,7 +276,7 @@ struct block_map {
 	block_count_t next_entry_count;
 
 	zone_count_t zone_count;
-	struct block_map_zone zones[];
+	struct block_map_zone zones[] __counted_by(zone_count);
 };
 
 /**
diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c
index 3333e1e5b02ee..370d4239ba315 100644
--- a/drivers/md/dm-vdo/data-vio.c
+++ b/drivers/md/dm-vdo/data-vio.c
@@ -842,8 +842,7 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size,
 	struct data_vio_pool *pool;
 	data_vio_count_t i;
 
-	result = vdo_allocate_extended(struct data_vio_pool, pool_size, struct data_vio,
-				       __func__, &pool);
+	result = vdo_allocate_extended(pool_size, data_vios, __func__, &pool);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c
index 75a26f3f44610..5f5639d89bc6d 100644
--- a/drivers/md/dm-vdo/dedupe.c
+++ b/drivers/md/dm-vdo/dedupe.c
@@ -296,7 +296,7 @@ struct hash_zones {
 	/* The number of zones */
 	zone_count_t zone_count;
 	/* The hash zones themselves */
-	struct hash_zone zones[];
+	struct hash_zone zones[] __counted_by(zone_count);
 };
 
 /* These are in milliseconds. */
@@ -2364,8 +2364,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone
 	vdo_set_completion_callback(&zone->completion, timeout_index_operations_callback,
 				    zone->thread_id);
 	INIT_LIST_HEAD(&zone->lock_pool);
-	result = vdo_allocate(LOCK_POOL_CAPACITY, struct hash_lock, "hash_lock array",
-			      &zone->lock_array);
+	result = vdo_allocate(LOCK_POOL_CAPACITY, "hash_lock array", &zone->lock_array);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2418,8 +2417,7 @@ int vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr)
 	if (zone_count == 0)
 		return VDO_SUCCESS;
 
-	result = vdo_allocate_extended(struct hash_zones, zone_count, struct hash_zone,
-				       __func__, &zones);
+	result = vdo_allocate_extended(zone_count, zones, __func__, &zones);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c
index 6af40d40f2550..7eb676e58ed53 100644
--- a/drivers/md/dm-vdo/dm-vdo-target.c
+++ b/drivers/md/dm-vdo/dm-vdo-target.c
@@ -273,8 +273,7 @@ static int split_string(const char *string, char separator, char ***substring_ar
 			substring_count++;
 	}
 
-	result = vdo_allocate(substring_count + 1, char *, "string-splitting array",
-			      &substrings);
+	result = vdo_allocate(substring_count + 1, "string-splitting array", &substrings);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -282,7 +281,7 @@ static int split_string(const char *string, char separator, char ***substring_ar
 		if (*s == separator) {
 			ptrdiff_t length = s - string;
 
-			result = vdo_allocate(length + 1, char, "split string",
+			result = vdo_allocate(length + 1, "split string",
 					      &substrings[current_substring]);
 			if (result != VDO_SUCCESS) {
 				free_string_array(substrings);
@@ -303,8 +302,7 @@ static int split_string(const char *string, char separator, char ***substring_ar
 	BUG_ON(current_substring != (substring_count - 1));
 	length = strlen(string);
 
-	result = vdo_allocate(length + 1, char, "split string",
-			      &substrings[current_substring]);
+	result = vdo_allocate(length + 1, "split string", &substrings[current_substring]);
 	if (result != VDO_SUCCESS) {
 		free_string_array(substrings);
 		return result;
@@ -332,7 +330,7 @@ static int join_strings(char **substring_array, size_t array_length, char separa
 	for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++)
 		string_length += strlen(substring_array[i]) + 1;
 
-	result = vdo_allocate(string_length, char, __func__, &output);
+	result = vdo_allocate(string_length, __func__, &output);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -726,7 +724,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti,
 		return VDO_BAD_CONFIGURATION;
 	}
 
-	result = vdo_allocate(1, struct device_config, "device_config", &config);
+	result = vdo_allocate(1, "device_config", &config);
 	if (result != VDO_SUCCESS) {
 		handle_parse_error(config, error_ptr,
 				   "Could not allocate config structure");
diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c
index bd60f4b3a0d0e..ec98c539701e3 100644
--- a/drivers/md/dm-vdo/encodings.c
+++ b/drivers/md/dm-vdo/encodings.c
@@ -798,7 +798,7 @@ static int allocate_partition(struct layout *layout, u8 id,
 	struct partition *partition;
 	int result;
 
-	result = vdo_allocate(1, struct partition, __func__, &partition);
+	result = vdo_allocate(1, __func__, &partition);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c
index 82a259ef16012..6c1610ba91b61 100644
--- a/drivers/md/dm-vdo/flush.c
+++ b/drivers/md/dm-vdo/flush.c
@@ -105,7 +105,7 @@ static void *allocate_flush(gfp_t gfp_mask, void *pool_data)
 	if ((gfp_mask & GFP_NOWAIT) == GFP_NOWAIT) {
 		flush = vdo_allocate_memory_nowait(sizeof(struct vdo_flush), __func__);
 	} else {
-		int result = vdo_allocate(1, struct vdo_flush, __func__, &flush);
+		int result = vdo_allocate(1, __func__, &flush);
 
 		if (result != VDO_SUCCESS)
 			vdo_log_error_strerror(result, "failed to allocate spare flush");
@@ -134,7 +134,7 @@ static void free_flush(void *element, void *pool_data __always_unused)
  */
 int vdo_make_flusher(struct vdo *vdo)
 {
-	int result = vdo_allocate(1, struct flusher, __func__, &vdo->flusher);
+	int result = vdo_allocate(1, __func__, &vdo->flusher);
 
 	if (result != VDO_SUCCESS)
 		return result;
diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c
index a63b2f2bfd7de..7011963c90734 100644
--- a/drivers/md/dm-vdo/funnel-queue.c
+++ b/drivers/md/dm-vdo/funnel-queue.c
@@ -14,7 +14,7 @@ int vdo_make_funnel_queue(struct funnel_queue **queue_ptr)
 	int result;
 	struct funnel_queue *queue;
 
-	result = vdo_allocate(1, struct funnel_queue, "funnel queue", &queue);
+	result = vdo_allocate(1, "funnel queue", &queue);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c
index 8a79b33b8b090..62d300f70de94 100644
--- a/drivers/md/dm-vdo/funnel-workqueue.c
+++ b/drivers/md/dm-vdo/funnel-workqueue.c
@@ -322,7 +322,7 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na
 			    "queue priority count %u within limit %u", type->max_priority,
 			    VDO_WORK_Q_MAX_PRIORITY);
 
-	result = vdo_allocate(1, struct simple_work_queue, "simple work queue", &queue);
+	result = vdo_allocate(1, "simple work queue", &queue);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -405,13 +405,11 @@ int vdo_make_work_queue(const char *thread_name_prefix, const char *name,
 		return result;
 	}
 
-	result = vdo_allocate(1, struct round_robin_work_queue, "round-robin work queue",
-			      &queue);
+	result = vdo_allocate(1, "round-robin work queue", &queue);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(thread_count, struct simple_work_queue *,
-			      "subordinate work queues", &queue->service_queues);
+	result = vdo_allocate(thread_count, "subordinate work queues", &queue->service_queues);
 	if (result != VDO_SUCCESS) {
 		vdo_free(queue);
 		return result;
diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c
index fb1db41c794b3..bb3b0ab5d50dc 100644
--- a/drivers/md/dm-vdo/indexer/chapter-index.c
+++ b/drivers/md/dm-vdo/indexer/chapter-index.c
@@ -20,7 +20,7 @@ int uds_make_open_chapter_index(struct open_chapter_index **chapter_index,
 	size_t memory_size;
 	struct open_chapter_index *index;
 
-	result = vdo_allocate(1, struct open_chapter_index, "open chapter index", &index);
+	result = vdo_allocate(1, "open chapter index", &index);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c
index 5532371b952fe..4a2cc66cfd60a 100644
--- a/drivers/md/dm-vdo/indexer/config.c
+++ b/drivers/md/dm-vdo/indexer/config.c
@@ -325,7 +325,7 @@ int uds_make_configuration(const struct uds_parameters *params,
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = vdo_allocate(1, struct uds_configuration, __func__, &config);
+	result = vdo_allocate(1, __func__, &config);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c
index 0ac2443f0df36..b288749067dee 100644
--- a/drivers/md/dm-vdo/indexer/delta-index.c
+++ b/drivers/md/dm-vdo/indexer/delta-index.c
@@ -311,18 +311,16 @@ static int initialize_delta_zone(struct delta_zone *delta_zone, size_t size,
 {
 	int result;
 
-	result = vdo_allocate(size, u8, "delta list", &delta_zone->memory);
+	result = vdo_allocate(size, "delta list", &delta_zone->memory);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(list_count + 2, u64, "delta list temp",
-			      &delta_zone->new_offsets);
+	result = vdo_allocate(list_count + 2, "delta list temp", &delta_zone->new_offsets);
 	if (result != VDO_SUCCESS)
 		return result;
 
 	/* Allocate the delta lists. */
-	result = vdo_allocate(list_count + 2, struct delta_list, "delta lists",
-			      &delta_zone->delta_lists);
+	result = vdo_allocate(list_count + 2, "delta lists", &delta_zone->delta_lists);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -352,8 +350,7 @@ int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zon
 	unsigned int z;
 	size_t zone_memory;
 
-	result = vdo_allocate(zone_count, struct delta_zone, "Delta Index Zones",
-			      &delta_index->delta_zones);
+	result = vdo_allocate(zone_count, "Delta Index Zones", &delta_index->delta_zones);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1047,7 +1044,7 @@ int uds_finish_restoring_delta_index(struct delta_index *delta_index,
 	unsigned int z;
 	u8 *data;
 
-	result = vdo_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data);
+	result = vdo_allocate(DELTA_LIST_MAX_BYTE_COUNT, __func__, &data);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c
index 1a5735375ddc0..03797cf87b914 100644
--- a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c
+++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c
@@ -198,7 +198,7 @@ int uds_make_request_queue(const char *queue_name,
 	int result;
 	struct uds_request_queue *queue;
 
-	result = vdo_allocate(1, struct uds_request_queue, __func__, &queue);
+	result = vdo_allocate(1, __func__, &queue);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/geometry.c b/drivers/md/dm-vdo/indexer/geometry.c
index c0575612e820b..49f122a223d51 100644
--- a/drivers/md/dm-vdo/indexer/geometry.c
+++ b/drivers/md/dm-vdo/indexer/geometry.c
@@ -61,7 +61,7 @@ int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter,
 	int result;
 	struct index_geometry *geometry;
 
-	result = vdo_allocate(1, struct index_geometry, "geometry", &geometry);
+	result = vdo_allocate(1, "geometry", &geometry);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c
index 61edf2b72427d..7a1209b21c03c 100644
--- a/drivers/md/dm-vdo/indexer/index-layout.c
+++ b/drivers/md/dm-vdo/indexer/index-layout.c
@@ -459,8 +459,7 @@ static int __must_check make_index_save_region_table(struct index_save_layout *i
 		type = RH_TYPE_UNSAVED;
 	}
 
-	result = vdo_allocate_extended(struct region_table, region_count,
-				       struct layout_region,
+	result = vdo_allocate_extended(region_count, regions,
 				       "layout region table for ISL", &table);
 	if (result != VDO_SUCCESS)
 		return result;
@@ -520,7 +519,7 @@ static int __must_check write_index_save_header(struct index_save_layout *isl,
 	u8 *buffer;
 	size_t offset = 0;
 
-	result = vdo_allocate(table->encoded_size, u8, "index save data", &buffer);
+	result = vdo_allocate(table->encoded_size, "index save data", &buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -642,9 +641,8 @@ static int __must_check make_layout_region_table(struct index_layout *layout,
 	struct region_table *table;
 	struct layout_region *lr;
 
-	result = vdo_allocate_extended(struct region_table, region_count,
-				       struct layout_region, "layout region table",
-				       &table);
+	result = vdo_allocate_extended(region_count, regions,
+				       "layout region table", &table);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -690,7 +688,7 @@ static int __must_check write_layout_header(struct index_layout *layout,
 	u8 *buffer;
 	size_t offset = 0;
 
-	result = vdo_allocate(table->encoded_size, u8, "layout data", &buffer);
+	result = vdo_allocate(table->encoded_size, "layout data", &buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -780,8 +778,7 @@ static int create_index_layout(struct index_layout *layout, struct uds_configura
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = vdo_allocate(sizes.save_count, struct index_save_layout, __func__,
-			      &layout->index.saves);
+	result = vdo_allocate(sizes.save_count, __func__, &layout->index.saves);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1138,8 +1135,7 @@ static int __must_check load_region_table(struct buffered_reader *reader,
 					      header.version);
 	}
 
-	result = vdo_allocate_extended(struct region_table, header.region_count,
-				       struct layout_region,
+	result = vdo_allocate_extended(header.region_count, regions,
 				       "single file layout region table", &table);
 	if (result != VDO_SUCCESS)
 		return result;
@@ -1177,7 +1173,7 @@ static int __must_check read_super_block_data(struct buffered_reader *reader,
 	u8 *buffer;
 	size_t offset = 0;
 
-	result = vdo_allocate(saved_size, u8, "super block data", &buffer);
+	result = vdo_allocate(saved_size, "super block data", &buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1311,8 +1307,7 @@ static int __must_check reconstitute_layout(struct index_layout *layout,
 	int result;
 	u64 next_block = first_block;
 
-	result = vdo_allocate(layout->super.max_saves, struct index_save_layout,
-			      __func__, &layout->index.saves);
+	result = vdo_allocate(layout->super.max_saves, __func__, &layout->index.saves);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1445,6 +1440,9 @@ static int __must_check reconstruct_index_save(struct index_save_layout *isl,
 	u64 last_block = next_block + isl->index_save.block_count;
 
 	isl->zone_count = table->header.region_count - 3;
+	if (isl->zone_count > MAX_ZONES)
+		return vdo_log_error_strerror(UDS_CORRUPT_DATA,
+					      "invalid zone count");
 
 	last_region = &table->regions[table->header.region_count - 1];
 	if (last_region->kind == RL_KIND_EMPTY) {
@@ -1672,7 +1670,7 @@ int uds_make_index_layout(struct uds_configuration *config, bool new_layout,
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = vdo_allocate(1, struct index_layout, __func__, &layout);
+	result = vdo_allocate(1, __func__, &layout);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c
index 00b44e07d0c1f..1d45d466d07f1 100644
--- a/drivers/md/dm-vdo/indexer/index-page-map.c
+++ b/drivers/md/dm-vdo/indexer/index-page-map.c
@@ -38,13 +38,13 @@ int uds_make_index_page_map(const struct index_geometry *geometry,
 	int result;
 	struct index_page_map *map;
 
-	result = vdo_allocate(1, struct index_page_map, "page map", &map);
+	result = vdo_allocate(1, "page map", &map);
 	if (result != VDO_SUCCESS)
 		return result;
 
 	map->geometry = geometry;
 	map->entries_per_chapter = geometry->index_pages_per_chapter - 1;
-	result = vdo_allocate(get_entry_count(geometry), u16, "Index Page Map Entries",
+	result = vdo_allocate(get_entry_count(geometry), "Index Page Map Entries",
 			      &map->entries);
 	if (result != VDO_SUCCESS) {
 		uds_free_index_page_map(map);
@@ -118,7 +118,7 @@ int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer
 	u64 saved_size = uds_compute_index_page_map_save_size(map->geometry);
 	u32 i;
 
-	result = vdo_allocate(saved_size, u8, "page map data", &buffer);
+	result = vdo_allocate(saved_size, "page map data", &buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -145,7 +145,7 @@ int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader *
 	u64 saved_size = uds_compute_index_page_map_save_size(map->geometry);
 	u32 i;
 
-	result = vdo_allocate(saved_size, u8, "page map data", &buffer);
+	result = vdo_allocate(saved_size, "page map data", &buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c
index aa575a24e0b2f..6c78070e1a055 100644
--- a/drivers/md/dm-vdo/indexer/index-session.c
+++ b/drivers/md/dm-vdo/indexer/index-session.c
@@ -217,7 +217,7 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde
 	int result;
 	struct uds_index_session *session;
 
-	result = vdo_allocate(1, struct uds_index_session, __func__, &session);
+	result = vdo_allocate(1, __func__, &session);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c
index df49348462441..793bd32c1179d 100644
--- a/drivers/md/dm-vdo/indexer/index.c
+++ b/drivers/md/dm-vdo/indexer/index.c
@@ -88,7 +88,7 @@ static int launch_zone_message(struct uds_zone_message message, unsigned int zon
 	int result;
 	struct uds_request *request;
 
-	result = vdo_allocate(1, struct uds_request, __func__, &request);
+	result = vdo_allocate(1, __func__, &request);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -764,9 +764,7 @@ static int make_chapter_writer(struct uds_index *index,
 	size_t collated_records_size =
 		(sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter);
 
-	result = vdo_allocate_extended(struct chapter_writer, index->zone_count,
-				       struct open_chapter_zone *, "Chapter Writer",
-				       &writer);
+	result = vdo_allocate_extended(index->zone_count, chapters, "Chapter Writer", &writer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1123,7 +1121,7 @@ static int make_index_zone(struct uds_index *index, unsigned int zone_number)
 	int result;
 	struct index_zone *zone;
 
-	result = vdo_allocate(1, struct index_zone, "index zone", &zone);
+	result = vdo_allocate(1, "index zone", &zone);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1160,8 +1158,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op
 	u64 nonce;
 	unsigned int z;
 
-	result = vdo_allocate_extended(struct uds_index, config->zone_count,
-				       struct uds_request_queue *, "index", &index);
+	result = vdo_allocate_extended(config->zone_count, zone_queues, "index", &index);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1173,8 +1170,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op
 		return result;
 	}
 
-	result = vdo_allocate(index->zone_count, struct index_zone *, "zones",
-			      &index->zones);
+	result = vdo_allocate(index->zone_count, "zones", &index->zones);
 	if (result != VDO_SUCCESS) {
 		uds_free_index(index);
 		return result;
diff --git a/drivers/md/dm-vdo/indexer/index.h b/drivers/md/dm-vdo/indexer/index.h
index edabb239548ec..1891f2de508ee 100644
--- a/drivers/md/dm-vdo/indexer/index.h
+++ b/drivers/md/dm-vdo/indexer/index.h
@@ -53,7 +53,7 @@ struct uds_index {
 
 	index_callback_fn callback;
 	struct uds_request_queue *triage_queue;
-	struct uds_request_queue *zone_queues[];
+	struct uds_request_queue *zone_queues[] __counted_by(zone_count);
 };
 
 enum request_stage {
diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c
index 1bee9d63dc0a6..f428613720301 100644
--- a/drivers/md/dm-vdo/indexer/io-factory.c
+++ b/drivers/md/dm-vdo/indexer/io-factory.c
@@ -64,7 +64,7 @@ int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_p
 	int result;
 	struct io_factory *factory;
 
-	result = vdo_allocate(1, struct io_factory, __func__, &factory);
+	result = vdo_allocate(1, __func__, &factory);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -144,7 +144,7 @@ int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader);
+	result = vdo_allocate(1, "buffered reader", &reader);
 	if (result != VDO_SUCCESS) {
 		dm_bufio_client_destroy(client);
 		return result;
@@ -282,7 +282,7 @@ int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block
 	if (result != UDS_SUCCESS)
 		return result;
 
-	result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer);
+	result = vdo_allocate(1, "buffered writer", &writer);
 	if (result != VDO_SUCCESS) {
 		dm_bufio_client_destroy(client);
 		return result;
diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c
index 4a67bcadaae0a..89b91c600bfd6 100644
--- a/drivers/md/dm-vdo/indexer/open-chapter.c
+++ b/drivers/md/dm-vdo/indexer/open-chapter.c
@@ -68,9 +68,7 @@ int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zo
 	size_t capacity = geometry->records_per_chapter / zone_count;
 	size_t slot_count = (1 << bits_per(capacity * LOAD_RATIO));
 
-	result = vdo_allocate_extended(struct open_chapter_zone, slot_count,
-				       struct open_chapter_zone_slot, "open chapter",
-				       &open_chapter);
+	result = vdo_allocate_extended(slot_count, slots, "open chapter", &open_chapter);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/open-chapter.h b/drivers/md/dm-vdo/indexer/open-chapter.h
index a4250bb19525e..ea6d7336aea01 100644
--- a/drivers/md/dm-vdo/indexer/open-chapter.h
+++ b/drivers/md/dm-vdo/indexer/open-chapter.h
@@ -40,7 +40,7 @@ struct open_chapter_zone {
 	/* The number of slots in the hash table */
 	unsigned int slot_count;
 	/* The hash table slots, referencing virtual record numbers */
-	struct open_chapter_zone_slot slots[];
+	struct open_chapter_zone_slot slots[] __counted_by(slot_count);
 };
 
 int __must_check uds_make_open_chapter(const struct index_geometry *geometry,
diff --git a/drivers/md/dm-vdo/indexer/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c
index 66b8c706a1ef9..4b81e130d18a5 100644
--- a/drivers/md/dm-vdo/indexer/radix-sort.c
+++ b/drivers/md/dm-vdo/indexer/radix-sort.c
@@ -211,8 +211,7 @@ int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter)
 	unsigned int stack_size = count / INSERTION_SORT_THRESHOLD;
 	struct radix_sorter *radix_sorter;
 
-	result = vdo_allocate_extended(struct radix_sorter, stack_size, struct task,
-				       __func__, &radix_sorter);
+	result = vdo_allocate_extended(stack_size, stack, __func__, &radix_sorter);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c
index 28920167827c4..eb62d3f018341 100644
--- a/drivers/md/dm-vdo/indexer/sparse-cache.c
+++ b/drivers/md/dm-vdo/indexer/sparse-cache.c
@@ -222,13 +222,12 @@ static int __must_check initialize_cached_chapter_index(struct cached_chapter_in
 	chapter->virtual_chapter = NO_CHAPTER;
 	chapter->index_pages_count = geometry->index_pages_per_chapter;
 
-	result = vdo_allocate(chapter->index_pages_count, struct delta_index_page,
-			      __func__, &chapter->index_pages);
+	result = vdo_allocate(chapter->index_pages_count, __func__, &chapter->index_pages);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	return vdo_allocate(chapter->index_pages_count, struct dm_buffer *,
-			    "sparse index volume pages", &chapter->page_buffers);
+	return vdo_allocate(chapter->index_pages_count, "sparse index volume pages",
+			    &chapter->page_buffers);
 }
 
 static int __must_check make_search_list(struct sparse_cache *cache,
@@ -294,8 +293,7 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca
 	}
 
 	/* purge_search_list() needs some temporary lists for sorting. */
-	result = vdo_allocate(capacity * 2, struct cached_chapter_index *,
-			      "scratch entries", &cache->scratch_entries);
+	result = vdo_allocate(capacity * 2, "scratch entries", &cache->scratch_entries);
 	if (result != VDO_SUCCESS)
 		goto out;
 
diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c
index afb062e1f1fb4..e78d2725ce8b6 100644
--- a/drivers/md/dm-vdo/indexer/volume-index.c
+++ b/drivers/md/dm-vdo/indexer/volume-index.c
@@ -1211,13 +1211,12 @@ static int initialize_volume_sub_index(const struct uds_configuration *config,
 				  (zone_count * sizeof(struct volume_sub_index_zone)));
 
 	/* The following arrays are initialized to all zeros. */
-	result = vdo_allocate(params.list_count, u64, "first chapter to flush",
+	result = vdo_allocate(params.list_count, "first chapter to flush",
 			      &sub_index->flush_chapters);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	return vdo_allocate(zone_count, struct volume_sub_index_zone,
-			    "volume index zones", &sub_index->zones);
+	return vdo_allocate(zone_count, "volume index zones", &sub_index->zones);
 }
 
 int uds_make_volume_index(const struct uds_configuration *config, u64 volume_nonce,
@@ -1228,7 +1227,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non
 	struct volume_index *volume_index;
 	int result;
 
-	result = vdo_allocate(1, struct volume_index, "volume index", &volume_index);
+	result = vdo_allocate(1, "volume index", &volume_index);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1249,8 +1248,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non
 
 	volume_index->sparse_sample_rate = config->sparse_sample_rate;
 
-	result = vdo_allocate(config->zone_count, struct volume_index_zone,
-			      "volume index zones", &volume_index->zones);
+	result = vdo_allocate(config->zone_count, "volume index zones", &volume_index->zones);
 	if (result != VDO_SUCCESS) {
 		uds_free_volume_index(volume_index);
 		return result;
diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c
index 425b3a74f4dba..af97c0cbeede4 100644
--- a/drivers/md/dm-vdo/indexer/volume.c
+++ b/drivers/md/dm-vdo/indexer/volume.c
@@ -1509,23 +1509,21 @@ static int __must_check initialize_page_cache(struct page_cache *cache,
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read,
-			      "volume read queue", &cache->read_queue);
+	result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, "volume read queue",
+			      &cache->read_queue);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(cache->zone_count, struct search_pending_counter,
-			      "Volume Cache Zones", &cache->search_pending_counters);
+	result = vdo_allocate(cache->zone_count, "Volume Cache Zones",
+			      &cache->search_pending_counters);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(cache->indexable_pages, u16, "page cache index",
-			      &cache->index);
+	result = vdo_allocate(cache->indexable_pages, "page cache index", &cache->index);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(cache->cache_slots, struct cached_page, "page cache cache",
-			      &cache->cache);
+	result = vdo_allocate(cache->cache_slots, "page cache cache", &cache->cache);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1548,7 +1546,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout
 	unsigned int reserved_buffers;
 	int result;
 
-	result = vdo_allocate(1, struct volume, "volume", &volume);
+	result = vdo_allocate(1, "volume", &volume);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1585,8 +1583,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout
 		return result;
 	}
 
-	result = vdo_allocate(geometry->records_per_page,
-			      const struct uds_volume_record *, "record pointers",
+	result = vdo_allocate(geometry->records_per_page, "record pointers",
 			      &volume->record_pointers);
 	if (result != VDO_SUCCESS) {
 		uds_free_volume(volume);
@@ -1626,8 +1623,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout
 	uds_init_cond(&volume->read_threads_read_done_cond);
 	uds_init_cond(&volume->read_threads_cond);
 
-	result = vdo_allocate(config->read_threads, struct thread *, "reader threads",
-			      &volume->reader_threads);
+	result = vdo_allocate(config->read_threads, "reader threads", &volume->reader_threads);
 	if (result != VDO_SUCCESS) {
 		uds_free_volume(volume);
 		return result;
diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c
index aeb690415dbd1..28d8af1f9be22 100644
--- a/drivers/md/dm-vdo/int-map.c
+++ b/drivers/md/dm-vdo/int-map.c
@@ -164,8 +164,7 @@ static int allocate_buckets(struct int_map *map, size_t capacity)
 	 * without have to wrap back around to element zero.
 	 */
 	map->bucket_count = capacity + (NEIGHBORHOOD - 1);
-	return vdo_allocate(map->bucket_count, struct bucket,
-			    "struct int_map buckets", &map->buckets);
+	return vdo_allocate(map->bucket_count, "struct int_map buckets", &map->buckets);
 }
 
 /**
@@ -182,7 +181,7 @@ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr)
 	int result;
 	size_t capacity;
 
-	result = vdo_allocate(1, struct int_map, "struct int_map", &map);
+	result = vdo_allocate(1, "struct int_map", &map);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c
index e26d75f8366dd..0e9932929fee7 100644
--- a/drivers/md/dm-vdo/io-submitter.c
+++ b/drivers/md/dm-vdo/io-submitter.c
@@ -383,8 +383,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter
 	struct io_submitter *io_submitter;
 	int result;
 
-	result = vdo_allocate_extended(struct io_submitter, thread_count,
-				       struct bio_queue_data, "bio submission data",
+	result = vdo_allocate_extended(thread_count, bio_queue_data, "bio submission data",
 				       &io_submitter);
 	if (result != VDO_SUCCESS)
 		return result;
diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c
index 0a27e60a9dfd7..fa7c3eb7ee6bf 100644
--- a/drivers/md/dm-vdo/logical-zone.c
+++ b/drivers/md/dm-vdo/logical-zone.c
@@ -94,8 +94,7 @@ int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr)
 	if (zone_count == 0)
 		return VDO_SUCCESS;
 
-	result = vdo_allocate_extended(struct logical_zones, zone_count,
-				       struct logical_zone, __func__, &zones);
+	result = vdo_allocate_extended(zone_count, zones, __func__, &zones);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/logical-zone.h b/drivers/md/dm-vdo/logical-zone.h
index 1b666c84a1935..a36a864c6836e 100644
--- a/drivers/md/dm-vdo/logical-zone.h
+++ b/drivers/md/dm-vdo/logical-zone.h
@@ -60,7 +60,7 @@ struct logical_zones {
 	/* The number of zones */
 	zone_count_t zone_count;
 	/* The logical zones themselves */
-	struct logical_zone zones[];
+	struct logical_zone zones[] __counted_by(zone_count);
 };
 
 int __must_check vdo_make_logical_zones(struct vdo *vdo,
diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c
index 185f259c7245d..a7f07522110d9 100644
--- a/drivers/md/dm-vdo/memory-alloc.c
+++ b/drivers/md/dm-vdo/memory-alloc.c
@@ -245,7 +245,7 @@ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr)
 	} else {
 		struct vmalloc_block_info *block;
 
-		if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == VDO_SUCCESS) {
+		if (vdo_allocate(1, __func__, &block) == VDO_SUCCESS) {
 			/*
 			 * It is possible for __vmalloc to fail to allocate memory because there
 			 * are no pages available. A short sleep may allow the page reclaimer
@@ -341,6 +341,7 @@ int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *w
 			  void *new_ptr)
 {
 	int result;
+	char *temp_ptr;
 
 	if (size == 0) {
 		vdo_free(ptr);
@@ -348,9 +349,10 @@ int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *w
 		return VDO_SUCCESS;
 	}
 
-	result = vdo_allocate(size, char, what, new_ptr);
+	result = vdo_allocate(size, what, &temp_ptr);
 	if (result != VDO_SUCCESS)
 		return result;
+	*(void **) new_ptr = temp_ptr;
 
 	if (ptr != NULL) {
 		if (old_size < size)
@@ -368,7 +370,7 @@ int vdo_duplicate_string(const char *string, const char *what, char **new_string
 	int result;
 	u8 *dup;
 
-	result = vdo_allocate(strlen(string) + 1, u8, what, &dup);
+	result = vdo_allocate(strlen(string) + 1, what, &dup);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h
index 0093d9f940d98..bc5527327ed80 100644
--- a/drivers/md/dm-vdo/memory-alloc.h
+++ b/drivers/md/dm-vdo/memory-alloc.h
@@ -8,6 +8,7 @@
 
 #include <linux/cache.h>
 #include <linux/io.h> /* for PAGE_SIZE */
+#include <linux/overflow.h>
 
 #include "permassert.h"
 #include "thread-registry.h"
@@ -15,87 +16,36 @@
 /* Custom memory allocation function that tracks memory usage */
 int __must_check vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr);
 
-/*
- * Allocate storage based on element counts, sizes, and alignment.
- *
- * This is a generalized form of our allocation use case: It allocates an array of objects,
- * optionally preceded by one object of another type (i.e., a struct with trailing variable-length
- * array), with the alignment indicated.
- *
- * Why is this inline? The sizes and alignment will always be constant, when invoked through the
- * macros below, and often the count will be a compile-time constant 1 or the number of extra bytes
- * will be a compile-time constant 0. So at least some of the arithmetic can usually be optimized
- * away, and the run-time selection between allocation functions always can. In many cases, it'll
- * boil down to just a function call with a constant size.
- *
- * @count: The number of objects to allocate
- * @size: The size of an object
- * @extra: The number of additional bytes to allocate
- * @align: The required alignment
- * @what: What is being allocated (for error logging)
- * @ptr: A pointer to hold the allocated memory
- *
- * Return: VDO_SUCCESS or an error code
- */
-static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra,
-				      size_t align, const char *what, void *ptr)
-{
-	size_t total_size = count * size + extra;
-
-	/* Overflow check: */
-	if ((size > 0) && (count > ((SIZE_MAX - extra) / size))) {
-		/*
-		 * This is kind of a hack: We rely on the fact that SIZE_MAX would cover the entire
-		 * address space (minus one byte) and thus the system can never allocate that much
-		 * and the call will always fail. So we can report an overflow as "out of memory"
-		 * by asking for "merely" SIZE_MAX bytes.
-		 */
-		total_size = SIZE_MAX;
-	}
-
-	return vdo_allocate_memory(total_size, align, what, ptr);
-}
-
 /*
  * Allocate one or more elements of the indicated type, logging an error if the allocation fails.
  * The memory will be zeroed.
  *
  * @COUNT: The number of objects to allocate
- * @TYPE: The type of objects to allocate. This type determines the alignment of the allocation.
  * @WHAT: What is being allocated (for error logging)
  * @PTR: A pointer to hold the allocated memory
  *
  * Return: VDO_SUCCESS or an error code
  */
-#define vdo_allocate(COUNT, TYPE, WHAT, PTR) \
-	__vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR)
+#define vdo_allocate(COUNT, WHAT, PTR)					\
+	vdo_allocate_memory(size_mul((COUNT), sizeof(typeof(**(PTR)))),	\
+			    __alignof__(typeof(**(PTR))), WHAT, PTR)
 
 /*
- * Allocate one object of an indicated type, followed by one or more elements of a second type,
- * logging an error if the allocation fails. The memory will be zeroed.
+ * Allocate a structure with a flexible array member, with a specified number of elements, logging
+ * an error if the allocation fails. The memory will be zeroed.
  *
- * @TYPE1: The type of the primary object to allocate. This type determines the alignment of the
- *         allocated memory.
  * @COUNT: The number of objects to allocate
- * @TYPE2: The type of array objects to allocate
+ * @FIELD: The flexible array field at the end of the structure
  * @WHAT: What is being allocated (for error logging)
  * @PTR: A pointer to hold the allocated memory
  *
  * Return: VDO_SUCCESS or an error code
  */
-#define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR)		\
-	__extension__({							\
-		int _result;						\
-		TYPE1 **_ptr = (PTR);					\
-		BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2));	\
-		_result = __vdo_do_allocation(COUNT,			\
-					      sizeof(TYPE2),		\
-					      sizeof(TYPE1),		\
-					      __alignof__(TYPE1),	\
-					      WHAT,			\
-					      _ptr);			\
-		_result;						\
-	})
+#define vdo_allocate_extended(COUNT, FIELD, WHAT, PTR)			\
+	vdo_allocate_memory(struct_size(*(PTR), FIELD, (COUNT)),	\
+			    __alignof__(typeof(**(PTR))),		\
+			    WHAT,					\
+			    (PTR))
 
 /*
  * Allocate memory starting on a cache line boundary, logging an error if the allocation fails. The
diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c
index 75dfcd7c5f631..b4c919780c226 100644
--- a/drivers/md/dm-vdo/message-stats.c
+++ b/drivers/md/dm-vdo/message-stats.c
@@ -420,7 +420,7 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen)
 	struct vdo_statistics *stats;
 	int result;
 
-	result = vdo_allocate(1, struct vdo_statistics, __func__, &stats);
+	result = vdo_allocate(1, __func__, &stats);
 	if (result != VDO_SUCCESS) {
 		vdo_log_error("Cannot allocate memory to write VDO statistics");
 		return result;
diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c
index 666be6d557e10..ea2d8d14495c0 100644
--- a/drivers/md/dm-vdo/packer.c
+++ b/drivers/md/dm-vdo/packer.c
@@ -120,8 +120,7 @@ static int __must_check make_bin(struct packer *packer)
 	struct packer_bin *bin;
 	int result;
 
-	result = vdo_allocate_extended(struct packer_bin, VDO_MAX_COMPRESSION_SLOTS,
-				       struct vio *, __func__, &bin);
+	result = vdo_allocate_extended(VDO_MAX_COMPRESSION_SLOTS, incoming, __func__, &bin);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -146,7 +145,7 @@ int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **pa
 	block_count_t i;
 	int result;
 
-	result = vdo_allocate(1, struct packer, __func__, &packer);
+	result = vdo_allocate(1, __func__, &packer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -168,8 +167,8 @@ int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **pa
 	 * bin must have a canceler for which it is waiting, and any canceler will only have
 	 * canceled one lock holder at a time.
 	 */
-	result = vdo_allocate_extended(struct packer_bin, MAXIMUM_VDO_USER_VIOS / 2,
-				       struct vio *, __func__, &packer->canceled_bin);
+	result = vdo_allocate_extended(MAXIMUM_VDO_USER_VIOS / 2, incoming, __func__,
+				       &packer->canceled_bin);
 	if (result != VDO_SUCCESS) {
 		vdo_free_packer(packer);
 		return result;
diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c
index 686eb7d714e60..d6ad8f1a33bbf 100644
--- a/drivers/md/dm-vdo/physical-zone.c
+++ b/drivers/md/dm-vdo/physical-zone.c
@@ -200,7 +200,7 @@ struct pbn_lock_pool {
 	/** @idle_list: A list containing all idle PBN lock instances. */
 	struct list_head idle_list;
 	/** @locks: The memory for all the locks allocated by this pool. */
-	idle_pbn_lock locks[];
+	idle_pbn_lock locks[] __counted_by(capacity);
 };
 
 /**
@@ -240,8 +240,7 @@ static int make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr)
 	struct pbn_lock_pool *pool;
 	int result;
 
-	result = vdo_allocate_extended(struct pbn_lock_pool, capacity, idle_pbn_lock,
-				       __func__, &pool);
+	result = vdo_allocate_extended(capacity, locks, __func__, &pool);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -368,8 +367,7 @@ int vdo_make_physical_zones(struct vdo *vdo, struct physical_zones **zones_ptr)
 	if (zone_count == 0)
 		return VDO_SUCCESS;
 
-	result = vdo_allocate_extended(struct physical_zones, zone_count,
-				       struct physical_zone, __func__, &zones);
+	result = vdo_allocate_extended(zone_count, zones, __func__, &zones);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c
index 9bae8256ba4e6..bb8a878ce4e53 100644
--- a/drivers/md/dm-vdo/priority-table.c
+++ b/drivers/md/dm-vdo/priority-table.c
@@ -60,8 +60,7 @@ int vdo_make_priority_table(unsigned int max_priority, struct priority_table **t
 	if (max_priority > MAX_PRIORITY)
 		return UDS_INVALID_ARGUMENT;
 
-	result = vdo_allocate_extended(struct priority_table, max_priority + 1,
-				       struct bucket, __func__, &table);
+	result = vdo_allocate_extended(max_priority + 1, buckets, __func__, &table);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c
index 9cc0f0ff16645..f03939cc89e38 100644
--- a/drivers/md/dm-vdo/recovery-journal.c
+++ b/drivers/md/dm-vdo/recovery-journal.c
@@ -593,32 +593,29 @@ static int __must_check initialize_lock_counter(struct recovery_journal *journal
 	struct thread_config *config = &vdo->thread_config;
 	struct lock_counter *counter = &journal->lock_counter;
 
-	result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters);
+	result = vdo_allocate(journal->size, __func__, &counter->journal_counters);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(journal->size, atomic_t, __func__,
-			      &counter->journal_decrement_counts);
+	result = vdo_allocate(journal->size, __func__, &counter->journal_decrement_counts);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__,
+	result = vdo_allocate(journal->size * config->logical_zone_count, __func__,
 			      &counter->logical_counters);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(journal->size, atomic_t, __func__,
-			      &counter->logical_zone_counts);
+	result = vdo_allocate(journal->size, __func__, &counter->logical_zone_counts);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__,
+	result = vdo_allocate(journal->size * config->physical_zone_count, __func__,
 			      &counter->physical_counters);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(journal->size, atomic_t, __func__,
-			      &counter->physical_zone_counts);
+	result = vdo_allocate(journal->size, __func__, &counter->physical_zone_counts);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -672,7 +669,7 @@ static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *j
 	 * Allocate a full block for the journal block even though not all of the space is used
 	 * since the VIO needs to write a full disk block.
 	 */
-	result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &data);
+	result = vdo_allocate(VDO_BLOCK_SIZE, __func__, &data);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -711,10 +708,8 @@ int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t
 	struct recovery_journal *journal;
 	int result;
 
-	result = vdo_allocate_extended(struct recovery_journal,
-				       RECOVERY_JOURNAL_RESERVED_BLOCKS,
-				       struct recovery_journal_block, __func__,
-				       &journal);
+	result = vdo_allocate_extended(RECOVERY_JOURNAL_RESERVED_BLOCKS, blocks,
+				       __func__, &journal);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c
index 8c006fb3afcf0..bfed622602803 100644
--- a/drivers/md/dm-vdo/repair.c
+++ b/drivers/md/dm-vdo/repair.c
@@ -127,7 +127,7 @@ struct repair_completion {
 	 * The page completions used for playing the journal into the block map, and, during
 	 * read-only rebuild, for rebuilding the reference counts from the block map.
 	 */
-	struct vdo_page_completion page_completions[];
+	struct vdo_page_completion page_completions[] __counted_by(page_count);
 };
 
 /*
@@ -1417,8 +1417,7 @@ static int parse_journal_for_rebuild(struct repair_completion *repair)
 	 * packed_recovery_journal_entry from every valid journal block.
 	 */
 	count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block);
-	result = vdo_allocate(count, struct numbered_block_mapping, __func__,
-			      &repair->entries);
+	result = vdo_allocate(count, __func__, &repair->entries);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1464,8 +1463,7 @@ static int extract_new_mappings(struct repair_completion *repair)
 	 * Allocate an array of numbered_block_mapping structs just large enough to transcribe
 	 * every packed_recovery_journal_entry from every valid journal block.
 	 */
-	result = vdo_allocate(repair->entry_count, struct numbered_block_mapping,
-			      __func__, &repair->entries);
+	result = vdo_allocate(repair->entry_count, __func__, &repair->entries);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -1715,9 +1713,7 @@ void vdo_repair(struct vdo_completion *parent)
 		vdo_log_warning("Device was dirty, rebuilding reference counts");
 	}
 
-	result = vdo_allocate_extended(struct repair_completion, page_count,
-				       struct vdo_page_completion, __func__,
-				       &repair);
+	result = vdo_allocate_extended(page_count, page_completions, __func__, &repair);
 	if (result != VDO_SUCCESS) {
 		vdo_fail_completion(parent, result);
 		return;
@@ -1729,12 +1725,11 @@ void vdo_repair(struct vdo_completion *parent)
 	prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN);
 	repair->page_count = page_count;
 
-	result = vdo_allocate(remaining * VDO_BLOCK_SIZE, char, __func__,
-			      &repair->journal_data);
+	result = vdo_allocate(remaining * VDO_BLOCK_SIZE, __func__, &repair->journal_data);
 	if (abort_on_error(result, repair))
 		return;
 
-	result = vdo_allocate(vio_count, struct vio, __func__, &repair->vios);
+	result = vdo_allocate(vio_count, __func__, &repair->vios);
 	if (abort_on_error(result, repair))
 		return;
 
diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c
index 034ecaa51f481..7fcbb361b38d1 100644
--- a/drivers/md/dm-vdo/slab-depot.c
+++ b/drivers/md/dm-vdo/slab-depot.c
@@ -2453,8 +2453,7 @@ static int allocate_slab_counters(struct vdo_slab *slab)
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(slab->reference_block_count, struct reference_block,
-			      __func__, &slab->reference_blocks);
+	result = vdo_allocate(slab->reference_block_count, __func__, &slab->reference_blocks);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -2463,8 +2462,7 @@ static int allocate_slab_counters(struct vdo_slab *slab)
 	 * so we can word-search even at the very end.
 	 */
 	bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD);
-	result = vdo_allocate(bytes, vdo_refcount_t, "ref counts array",
-			      &slab->counters);
+	result = vdo_allocate(bytes, "ref counts array", &slab->counters);
 	if (result != VDO_SUCCESS) {
 		vdo_free(vdo_forget(slab->reference_blocks));
 		return result;
@@ -3563,8 +3561,7 @@ static int get_slab_statuses(struct block_allocator *allocator,
 	struct slab_status *statuses;
 	struct slab_iterator iterator = get_slab_iterator(allocator);
 
-	result = vdo_allocate(allocator->slab_count, struct slab_status, __func__,
-			      &statuses);
+	result = vdo_allocate(allocator->slab_count, __func__, &statuses);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -3739,13 +3736,12 @@ static int initialize_slab_journal(struct vdo_slab *slab)
 	const struct slab_config *slab_config = &slab->allocator->depot->slab_config;
 	int result;
 
-	result = vdo_allocate(slab_config->slab_journal_blocks, struct journal_lock,
-			      __func__, &journal->locks);
+	result = vdo_allocate(slab_config->slab_journal_blocks, __func__, &journal->locks);
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block",
-			      (char **) &journal->block);
+	BUILD_BUG_ON(sizeof(*journal->block) != VDO_BLOCK_SIZE);
+	result = vdo_allocate(1, "struct packed_slab_journal_block", &journal->block);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -3800,7 +3796,7 @@ static int __must_check make_slab(physical_block_number_t slab_origin,
 	struct vdo_slab *slab;
 	int result;
 
-	result = vdo_allocate(1, struct vdo_slab, __func__, &slab);
+	result = vdo_allocate(1, __func__, &slab);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -3857,8 +3853,7 @@ static int allocate_slabs(struct slab_depot *depot, slab_count_t slab_count)
 	physical_block_number_t slab_origin;
 	int result;
 
-	result = vdo_allocate(slab_count, struct vdo_slab *,
-			      "slab pointer array", &depot->new_slabs);
+	result = vdo_allocate(slab_count, "slab pointer array", &depot->new_slabs);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -4011,8 +4006,7 @@ static int initialize_slab_scrubber(struct block_allocator *allocator)
 	char *journal_data;
 	int result;
 
-	result = vdo_allocate(VDO_BLOCK_SIZE * slab_journal_size,
-			      char, __func__, &journal_data);
+	result = vdo_allocate(VDO_BLOCK_SIZE * slab_journal_size, __func__, &journal_data);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -4045,7 +4039,7 @@ static int __must_check initialize_slab_summary_block(struct block_allocator *al
 	struct slab_summary_block *block = &allocator->summary_blocks[index];
 	int result;
 
-	result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries);
+	result = vdo_allocate(VDO_BLOCK_SIZE, __func__, &block->outgoing_entries);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -4114,8 +4108,7 @@ static int __must_check initialize_block_allocator(struct slab_depot *depot,
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE,
-			      struct slab_summary_block, __func__,
+	result = vdo_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, __func__,
 			      &allocator->summary_blocks);
 	if (result != VDO_SUCCESS)
 		return result;
@@ -4174,8 +4167,7 @@ static int allocate_components(struct slab_depot *depot,
 
 	depot->summary_origin = summary_partition->offset;
 	depot->hint_shift = vdo_get_slab_summary_hint_shift(depot->slab_size_shift);
-	result = vdo_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES,
-			      struct slab_summary_entry, __func__,
+	result = vdo_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, __func__,
 			      &depot->summary_entries);
 	if (result != VDO_SUCCESS)
 		return result;
@@ -4262,9 +4254,12 @@ int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo,
 	}
 	slab_size_shift = ilog2(slab_size);
 
-	result = vdo_allocate_extended(struct slab_depot,
-				       vdo->thread_config.physical_zone_count,
-				       struct block_allocator, __func__, &depot);
+	if (state.zone_count > MAX_VDO_PHYSICAL_ZONES)
+		return vdo_log_error_strerror(UDS_CORRUPT_DATA,
+					      "invalid zone count");
+
+	result = vdo_allocate_extended(vdo->thread_config.physical_zone_count,
+				       allocators, __func__, &depot);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h
index fadc0c9d4dc48..6bfd61c937b62 100644
--- a/drivers/md/dm-vdo/slab-depot.h
+++ b/drivers/md/dm-vdo/slab-depot.h
@@ -509,7 +509,7 @@ struct slab_depot {
 	struct slab_summary_entry *summary_entries;
 
 	/* The block allocators for this depot */
-	struct block_allocator allocators[];
+	struct block_allocator allocators[] __counted_by(zone_count);
 };
 
 struct reference_updater;
diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c
index ec08478dd0139..826afc952b56c 100644
--- a/drivers/md/dm-vdo/thread-utils.c
+++ b/drivers/md/dm-vdo/thread-utils.c
@@ -56,7 +56,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data,
 	struct thread *thread;
 	int result;
 
-	result = vdo_allocate(1, struct thread, __func__, &thread);
+	result = vdo_allocate(1, __func__, &thread);
 	if (result != VDO_SUCCESS) {
 		vdo_log_warning("Error allocating memory for %s", name);
 		return result;
diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c
index 09fd0628d18c7..167cf93a284ac 100644
--- a/drivers/md/dm-vdo/vdo.c
+++ b/drivers/md/dm-vdo/vdo.c
@@ -207,29 +207,28 @@ static int __must_check initialize_thread_config(struct thread_count_config coun
 		config->hash_zone_count = counts.hash_zones;
 	}
 
-	result = vdo_allocate(config->logical_zone_count, thread_id_t,
-			      "logical thread array", &config->logical_threads);
+	result = vdo_allocate(config->logical_zone_count, "logical thread array",
+			      &config->logical_threads);
 	if (result != VDO_SUCCESS) {
 		uninitialize_thread_config(config);
 		return result;
 	}
 
-	result = vdo_allocate(config->physical_zone_count, thread_id_t,
-			      "physical thread array", &config->physical_threads);
+	result = vdo_allocate(config->physical_zone_count, "physical thread array",
+			      &config->physical_threads);
 	if (result != VDO_SUCCESS) {
 		uninitialize_thread_config(config);
 		return result;
 	}
 
-	result = vdo_allocate(config->hash_zone_count, thread_id_t,
-			      "hash thread array", &config->hash_zone_threads);
+	result = vdo_allocate(config->hash_zone_count, "hash thread array",
+			      &config->hash_zone_threads);
 	if (result != VDO_SUCCESS) {
 		uninitialize_thread_config(config);
 		return result;
 	}
 
-	result = vdo_allocate(config->bio_thread_count, thread_id_t,
-			      "bio thread array", &config->bio_threads);
+	result = vdo_allocate(config->bio_thread_count, "bio thread array", &config->bio_threads);
 	if (result != VDO_SUCCESS) {
 		uninitialize_thread_config(config);
 		return result;
@@ -269,7 +268,7 @@ static int __must_check read_geometry_block(struct vdo *vdo)
 	char *block;
 	int result;
 
-	result = vdo_allocate(VDO_BLOCK_SIZE, u8, __func__, &block);
+	result = vdo_allocate(VDO_BLOCK_SIZE, __func__, &block);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -493,7 +492,7 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config,
 		     config->thread_counts.hash_zones, vdo->thread_config.thread_count);
 
 	/* Compression context storage */
-	result = vdo_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context",
+	result = vdo_allocate(config->thread_counts.cpu_threads, "LZ4 context",
 			      &vdo->compression_context);
 	if (result != VDO_SUCCESS) {
 		*reason = "cannot allocate LZ4 context";
@@ -501,7 +500,7 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config,
 	}
 
 	for (i = 0; i < config->thread_counts.cpu_threads; i++) {
-		result = vdo_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context",
+		result = vdo_allocate(LZ4_MEM_COMPRESS, "LZ4 context",
 				      &vdo->compression_context[i]);
 		if (result != VDO_SUCCESS) {
 			*reason = "cannot allocate LZ4 context";
@@ -537,7 +536,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason,
 	/* Initialize with a generic failure reason to prevent returning garbage. */
 	*reason = "Unspecified error";
 
-	result = vdo_allocate(1, struct vdo, __func__, &vdo);
+	result = vdo_allocate(1, __func__, &vdo);
 	if (result != VDO_SUCCESS) {
 		*reason = "Cannot allocate VDO";
 		return result;
@@ -554,8 +553,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason,
 
 	snprintf(vdo->thread_name_prefix, sizeof(vdo->thread_name_prefix),
 		 "vdo%u", instance);
-	result = vdo_allocate(vdo->thread_config.thread_count,
-			      struct vdo_thread, __func__, &vdo->threads);
+	result = vdo_allocate(vdo->thread_config.thread_count, __func__, &vdo->threads);
 	if (result != VDO_SUCCESS) {
 		*reason = "Cannot allocate thread structures";
 		return result;
@@ -724,8 +722,7 @@ static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super
 {
 	int result;
 
-	result = vdo_allocate(VDO_BLOCK_SIZE, char, "encoded super block",
-			      (char **) &vdo->super_block.buffer);
+	result = vdo_allocate(VDO_BLOCK_SIZE, "encoded super block", &vdo->super_block.buffer);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -997,8 +994,7 @@ int vdo_register_read_only_listener(struct vdo *vdo, void *listener,
 	if (result != VDO_SUCCESS)
 		return result;
 
-	result = vdo_allocate(1, struct read_only_listener, __func__,
-			      &read_only_listener);
+	result = vdo_allocate(1, __func__, &read_only_listener);
 	if (result != VDO_SUCCESS)
 		return result;
 
diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c
index 5ffc867d9c5e5..ea8ac619ff1b3 100644
--- a/drivers/md/dm-vdo/vio.c
+++ b/drivers/md/dm-vdo/vio.c
@@ -52,8 +52,8 @@ static int create_multi_block_bio(block_count_t size, struct bio **bio_ptr)
 	struct bio *bio = NULL;
 	int result;
 
-	result = vdo_allocate_extended(struct bio, size + 1, struct bio_vec,
-				       "bio", &bio);
+	result = vdo_allocate_memory(sizeof(struct bio) + sizeof(struct bio_vec) * (size + 1),
+				     __alignof__(struct bio), "bio", &bio);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -129,7 +129,7 @@ int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type,
 	 * Metadata vios should use direct allocation and not use the buffer pool, which is
 	 * reserved for submissions from the linux block layer.
 	 */
-	result = vdo_allocate(1, struct vio, __func__, &vio);
+	result = vdo_allocate(1, __func__, &vio);
 	if (result != VDO_SUCCESS) {
 		vdo_log_error("metadata vio allocation failure %d", result);
 		return result;
@@ -327,8 +327,7 @@ int make_vio_pool(struct vdo *vdo, size_t pool_size, size_t block_count, thread_
 	int result;
 	size_t per_vio_size = VDO_BLOCK_SIZE * block_count;
 
-	result = vdo_allocate_extended(struct vio_pool, pool_size, struct pooled_vio,
-				       __func__, &pool);
+	result = vdo_allocate_extended(pool_size, vios, __func__, &pool);
 	if (result != VDO_SUCCESS)
 		return result;
 
@@ -336,8 +335,7 @@ int make_vio_pool(struct vdo *vdo, size_t pool_size, size_t block_count, thread_
 	INIT_LIST_HEAD(&pool->available);
 	INIT_LIST_HEAD(&pool->busy);
 
-	result = vdo_allocate(pool_size * per_vio_size, char,
-			      "VIO pool buffer", &pool->buffer);
+	result = vdo_allocate(pool_size * per_vio_size, "VIO pool buffer", &pool->buffer);
 	if (result != VDO_SUCCESS) {
 		free_vio_pool(pool);
 		return result;
diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c
index 14be4d888af38..fb71e83e6404e 100644
--- a/drivers/md/dm-verity-fec.c
+++ b/drivers/md/dm-verity-fec.c
@@ -11,163 +11,131 @@
 #define DM_MSG_PREFIX	"verity-fec"
 
 /*
- * When correcting a data block, the FEC code performs optimally when it can
- * collect all the associated RS blocks at the same time.  As each byte is part
- * of a different RS block, there are '1 << data_dev_block_bits' RS blocks.
- * There are '1 << DM_VERITY_FEC_BUF_RS_BITS' RS blocks per buffer, so that
- * gives '1 << (data_dev_block_bits - DM_VERITY_FEC_BUF_RS_BITS)' buffers.
+ * When correcting a block, the FEC implementation performs optimally when it
+ * can collect all the associated RS codewords at the same time.  As each byte
+ * is part of a different codeword, there are '1 << data_dev_block_bits'
+ * codewords.  Each buffer has space for the message bytes for
+ * '1 << DM_VERITY_FEC_BUF_RS_BITS' codewords, so that gives
+ * '1 << (data_dev_block_bits - DM_VERITY_FEC_BUF_RS_BITS)' buffers.
  */
 static inline unsigned int fec_max_nbufs(struct dm_verity *v)
 {
 	return 1 << (v->data_dev_block_bits - DM_VERITY_FEC_BUF_RS_BITS);
 }
 
-/*
- * Return an interleaved offset for a byte in RS block.
- */
-static inline u64 fec_interleave(struct dm_verity *v, u64 offset)
-{
-	u32 mod;
-
-	mod = do_div(offset, v->fec->rsn);
-	return offset + mod * (v->fec->rounds << v->data_dev_block_bits);
-}
-
-/*
- * Read error-correcting codes for the requested RS block. Returns a pointer
- * to the data block. Caller is responsible for releasing buf.
- */
-static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,
-			   unsigned int *offset, unsigned int par_buf_offset,
-			   struct dm_buffer **buf, unsigned short ioprio)
-{
-	u64 position, block, rem;
-	u8 *res;
-
-	/* We have already part of parity bytes read, skip to the next block */
-	if (par_buf_offset)
-		index++;
-
-	position = (index + rsb) * v->fec->roots;
-	block = div64_u64_rem(position, v->fec->io_size, &rem);
-	*offset = par_buf_offset ? 0 : (unsigned int)rem;
-
-	res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio);
-	if (IS_ERR(res)) {
-		DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
-		      v->data_dev->name, (unsigned long long)rsb,
-		      (unsigned long long)block, PTR_ERR(res));
-		*buf = NULL;
-	}
-
-	return res;
-}
-
 /* Loop over each allocated buffer. */
 #define fec_for_each_buffer(io, __i) \
 	for (__i = 0; __i < (io)->nbufs; __i++)
 
-/* Loop over each RS block in each allocated buffer. */
-#define fec_for_each_buffer_rs_block(io, __i, __j) \
+/* Loop over each RS message in each allocated buffer. */
+/* To stop early, use 'goto', not 'break' (since this uses nested loops). */
+#define fec_for_each_buffer_rs_message(io, __i, __j) \
 	fec_for_each_buffer(io, __i) \
 		for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++)
 
 /*
- * Return a pointer to the current RS block when called inside
- * fec_for_each_buffer_rs_block.
+ * Return a pointer to the current RS message when called inside
+ * fec_for_each_buffer_rs_message.
  */
-static inline u8 *fec_buffer_rs_block(struct dm_verity *v,
-				      struct dm_verity_fec_io *fio,
-				      unsigned int i, unsigned int j)
+static inline u8 *fec_buffer_rs_message(struct dm_verity *v,
+					struct dm_verity_fec_io *fio,
+					unsigned int i, unsigned int j)
 {
-	return &fio->bufs[i][j * v->fec->rsn];
+	return &fio->bufs[i][j * v->fec->rs_k];
 }
 
 /*
- * Return an index to the current RS block when called inside
- * fec_for_each_buffer_rs_block.
- */
-static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j)
-{
-	return (i << DM_VERITY_FEC_BUF_RS_BITS) + j;
-}
-
-/*
- * Decode all RS blocks from buffers and copy corrected bytes into fio->output
- * starting from block_offset.
+ * Decode all RS codewords whose message bytes were loaded into fio->bufs.  Copy
+ * the corrected bytes into fio->output starting from out_pos.
  */
 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io,
-			   struct dm_verity_fec_io *fio, u64 rsb, int byte_index,
-			   unsigned int block_offset, int neras)
+			   struct dm_verity_fec_io *fio, u64 target_block,
+			   unsigned int target_region, u64 index_in_region,
+			   unsigned int out_pos, int neras)
 {
-	int r, corrected = 0, res;
+	int r = 0, corrected = 0, res;
 	struct dm_buffer *buf;
-	unsigned int n, i, j, offset, par_buf_offset = 0;
-	uint16_t par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN];
-	u8 *par, *block;
+	unsigned int n, i, j, parity_pos, to_copy;
+	uint16_t par_buf[DM_VERITY_FEC_MAX_ROOTS];
+	u8 *par, *msg_buf;
+	u64 parity_block;
 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
 
-	par = fec_read_parity(v, rsb, block_offset, &offset,
-			      par_buf_offset, &buf, bio->bi_ioprio);
-	if (IS_ERR(par))
+	/*
+	 * Compute the index of the first parity block that will be needed and
+	 * the starting position in that block.  Then read that block.
+	 *
+	 * block_size is always a power of 2, but roots might not be.  Note that
+	 * when it's not, a codeword's parity bytes can span a block boundary.
+	 */
+	parity_block = ((index_in_region << v->data_dev_block_bits) + out_pos) *
+		       v->fec->roots;
+	parity_pos = parity_block & (v->fec->block_size - 1);
+	parity_block >>= v->data_dev_block_bits;
+	par = dm_bufio_read_with_ioprio(v->fec->bufio, parity_block, &buf,
+					bio->bi_ioprio);
+	if (IS_ERR(par)) {
+		DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
+		      v->data_dev->name, target_block, parity_block,
+		      PTR_ERR(par));
 		return PTR_ERR(par);
+	}
 
 	/*
-	 * Decode the RS blocks we have in bufs. Each RS block results in
-	 * one corrected target byte and consumes fec->roots parity bytes.
+	 * Decode the RS codewords whose message bytes are in bufs. Each RS
+	 * codeword results in one corrected target byte and consumes fec->roots
+	 * parity bytes.
 	 */
-	fec_for_each_buffer_rs_block(fio, n, i) {
-		block = fec_buffer_rs_block(v, fio, n, i);
-		for (j = 0; j < v->fec->roots - par_buf_offset; j++)
-			par_buf[par_buf_offset + j] = par[offset + j];
-		/* Decode an RS block using Reed-Solomon */
-		res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn,
+	fec_for_each_buffer_rs_message(fio, n, i) {
+		msg_buf = fec_buffer_rs_message(v, fio, n, i);
+
+		/*
+		 * Copy the next 'roots' parity bytes to 'par_buf', reading
+		 * another parity block if needed.
+		 */
+		to_copy = min(v->fec->block_size - parity_pos, v->fec->roots);
+		for (j = 0; j < to_copy; j++)
+			par_buf[j] = par[parity_pos++];
+		if (to_copy < v->fec->roots) {
+			parity_block++;
+			parity_pos = 0;
+
+			dm_bufio_release(buf);
+			par = dm_bufio_read_with_ioprio(v->fec->bufio,
+							parity_block, &buf,
+							bio->bi_ioprio);
+			if (IS_ERR(par)) {
+				DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
+				      v->data_dev->name, target_block,
+				      parity_block, PTR_ERR(par));
+				return PTR_ERR(par);
+			}
+			for (; j < v->fec->roots; j++)
+				par_buf[j] = par[parity_pos++];
+		}
+
+		/* Decode an RS codeword using the Reed-Solomon library. */
+		res = decode_rs8(fio->rs, msg_buf, par_buf, v->fec->rs_k,
 				 NULL, neras, fio->erasures, 0, NULL);
 		if (res < 0) {
 			r = res;
-			goto error;
+			goto done;
 		}
-
 		corrected += res;
-		fio->output[block_offset] = block[byte_index];
+		fio->output[out_pos++] = msg_buf[target_region];
 
-		block_offset++;
-		if (block_offset >= 1 << v->data_dev_block_bits)
+		if (out_pos >= v->fec->block_size)
 			goto done;
-
-		/* Read the next block when we run out of parity bytes */
-		offset += (v->fec->roots - par_buf_offset);
-		/* Check if parity bytes are split between blocks */
-		if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) {
-			par_buf_offset = v->fec->io_size - offset;
-			for (j = 0; j < par_buf_offset; j++)
-				par_buf[j] = par[offset + j];
-			offset += par_buf_offset;
-		} else
-			par_buf_offset = 0;
-
-		if (offset >= v->fec->io_size) {
-			dm_bufio_release(buf);
-
-			par = fec_read_parity(v, rsb, block_offset, &offset,
-					      par_buf_offset, &buf, bio->bi_ioprio);
-			if (IS_ERR(par))
-				return PTR_ERR(par);
-		}
 	}
 done:
-	r = corrected;
-error:
 	dm_bufio_release(buf);
 
 	if (r < 0 && neras)
 		DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",
-			    v->data_dev->name, (unsigned long long)rsb, r);
-	else if (r > 0) {
+			    v->data_dev->name, target_block, r);
+	else if (r == 0 && corrected > 0)
 		DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",
-			     v->data_dev->name, (unsigned long long)rsb, r);
-		atomic64_inc(&v->fec->corrected);
-	}
+			     v->data_dev->name, target_block, corrected);
 
 	return r;
 }
@@ -178,7 +146,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io,
 static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
 			  const u8 *want_digest, const u8 *data)
 {
-	if (unlikely(verity_hash(v, io, data, 1 << v->data_dev_block_bits,
+	if (unlikely(verity_hash(v, io, data, v->fec->block_size,
 				 io->tmp_digest)))
 		return 0;
 
@@ -186,22 +154,35 @@ static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
 }
 
 /*
- * Read data blocks that are part of the RS block and deinterleave as much as
- * fits into buffers. Check for erasure locations if @neras is non-NULL.
+ * Read the message block at index @index_in_region within each of the
+ * @v->fec->rs_k regions and deinterleave their contents into @io->fec_io->bufs.
+ *
+ * @target_block gives the index of specific block within this sequence that is
+ * being corrected, relative to the start of all the FEC message blocks.
+ *
+ * @out_pos gives the current output position, i.e. the position in (each) block
+ * from which to start the deinterleaving.  Deinterleaving continues until
+ * either end-of-block is reached or there's no more buffer space.
+ *
+ * If @neras is non-NULL, then also use verity hashes and the presence/absence
+ * of I/O errors to determine which of the message blocks in the sequence are
+ * likely to be incorrect.  Write the number of such blocks to *@neras and the
+ * indices of the corresponding RS message bytes in [0, k - 1] to
+ * @io->fec_io->erasures, up to a limit of @v->fec->roots + 1 such blocks.
  */
 static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
-			 u64 rsb, u64 target, unsigned int block_offset,
-			 int *neras)
+			 u64 target_block, u64 index_in_region,
+			 unsigned int out_pos, int *neras)
 {
 	bool is_zero;
-	int i, j, target_index = -1;
+	int i, j;
 	struct dm_buffer *buf;
 	struct dm_bufio_client *bufio;
 	struct dm_verity_fec_io *fio = io->fec_io;
-	u64 block, ileaved;
-	u8 *bbuf, *rs_block;
+	u64 block;
+	u8 *bbuf;
 	u8 want_digest[HASH_MAX_DIGESTSIZE];
-	unsigned int n, k;
+	unsigned int n, src_pos;
 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
 
 	if (neras)
@@ -210,21 +191,12 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
 	if (WARN_ON(v->digest_size > sizeof(want_digest)))
 		return -EINVAL;
 
-	/*
-	 * read each of the rsn data blocks that are part of the RS block, and
-	 * interleave contents to available bufs
-	 */
-	for (i = 0; i < v->fec->rsn; i++) {
-		ileaved = fec_interleave(v, rsb * v->fec->rsn + i);
-
+	for (i = 0; i < v->fec->rs_k; i++) {
 		/*
-		 * target is the data block we want to correct, target_index is
-		 * the index of this block within the rsn RS blocks
+		 * Read the block from region i.  It contains the i'th message
+		 * byte of the target block's RS codewords.
 		 */
-		if (ileaved == target)
-			target_index = i;
-
-		block = ileaved >> v->data_dev_block_bits;
+		block = i * v->fec->region_blocks + index_in_region;
 		bufio = v->fec->data_bufio;
 
 		if (block >= v->data_blocks) {
@@ -244,9 +216,8 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
 		bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio->bi_ioprio);
 		if (IS_ERR(bbuf)) {
 			DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",
-				     v->data_dev->name,
-				     (unsigned long long)rsb,
-				     (unsigned long long)block, PTR_ERR(bbuf));
+				     v->data_dev->name, target_block, block,
+				     PTR_ERR(bbuf));
 
 			/* assume the block is corrupted */
 			if (neras && *neras <= v->fec->roots)
@@ -273,23 +244,20 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
 		}
 
 		/*
-		 * deinterleave and copy the bytes that fit into bufs,
-		 * starting from block_offset
+		 * Deinterleave the bytes of the block, starting from 'out_pos',
+		 * into the i'th byte of the RS message buffers.  Stop when
+		 * end-of-block is reached or there are no more buffers.
 		 */
-		fec_for_each_buffer_rs_block(fio, n, j) {
-			k = fec_buffer_rs_index(n, j) + block_offset;
-
-			if (k >= 1 << v->data_dev_block_bits)
+		src_pos = out_pos;
+		fec_for_each_buffer_rs_message(fio, n, j) {
+			if (src_pos >= v->fec->block_size)
 				goto done;
-
-			rs_block = fec_buffer_rs_block(v, fio, n, j);
-			rs_block[i] = bbuf[k];
+			fec_buffer_rs_message(v, fio, n, j)[i] = bbuf[src_pos++];
 		}
 done:
 		dm_bufio_release(buf);
 	}
-
-	return target_index;
+	return 0;
 }
 
 /*
@@ -336,47 +304,65 @@ static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
 	unsigned int n;
 
 	fec_for_each_buffer(fio, n)
-		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);
+		memset(fio->bufs[n], 0, v->fec->rs_k << DM_VERITY_FEC_BUF_RS_BITS);
 
 	memset(fio->erasures, 0, sizeof(fio->erasures));
 }
 
 /*
- * Decode all RS blocks in a single data block and return the target block
- * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
- * hashes to locate erasures.
+ * Try to correct the message (data or hash) block at index @target_block.
+ *
+ * If @use_erasures is true, use verity hashes to locate erasures.  This makes
+ * the error correction slower but up to twice as capable.
+ *
+ * On success, return 0 and write the corrected block to @fio->output.  0 is
+ * returned only if the digest of the corrected block matches @want_digest; this
+ * is critical to ensure that FEC can't cause dm-verity to return bad data.
  */
-static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
-			  struct dm_verity_fec_io *fio, u64 rsb, u64 offset,
-			  const u8 *want_digest, bool use_erasures)
+static int fec_decode(struct dm_verity *v, struct dm_verity_io *io,
+		      struct dm_verity_fec_io *fio, u64 target_block,
+		      const u8 *want_digest, bool use_erasures)
 {
 	int r, neras = 0;
-	unsigned int pos;
+	unsigned int target_region, out_pos;
+	u64 index_in_region;
 
-	for (pos = 0; pos < 1 << v->data_dev_block_bits; ) {
+	/*
+	 * Compute 'target_region', the index of the region the target block is
+	 * in; and 'index_in_region', the index of the target block within its
+	 * region.  The latter value is also the index within its region of each
+	 * message block that shares its RS codewords with the target block.
+	 */
+	target_region = div64_u64_rem(target_block, v->fec->region_blocks,
+				      &index_in_region);
+	if (WARN_ON_ONCE(target_region >= v->fec->rs_k))
+		/* target_block is out-of-bounds.  Should never happen. */
+		return -EIO;
+
+	for (out_pos = 0; out_pos < v->fec->block_size;) {
 		fec_init_bufs(v, fio);
 
-		r = fec_read_bufs(v, io, rsb, offset, pos,
+		r = fec_read_bufs(v, io, target_block, index_in_region, out_pos,
 				  use_erasures ? &neras : NULL);
 		if (unlikely(r < 0))
 			return r;
 
-		r = fec_decode_bufs(v, io, fio, rsb, r, pos, neras);
+		r = fec_decode_bufs(v, io, fio, target_block, target_region,
+				    index_in_region, out_pos, neras);
 		if (r < 0)
 			return r;
 
-		pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;
+		out_pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;
 	}
 
 	/* Always re-validate the corrected block against the expected hash */
-	r = verity_hash(v, io, fio->output, 1 << v->data_dev_block_bits,
-			io->tmp_digest);
+	r = verity_hash(v, io, fio->output, v->fec->block_size, io->tmp_digest);
 	if (unlikely(r < 0))
 		return r;
 
 	if (memcmp(io->tmp_digest, want_digest, v->digest_size)) {
 		DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",
-			    v->data_dev->name, (unsigned long long)rsb, neras);
+			    v->data_dev->name, target_block, neras);
 		return -EILSEQ;
 	}
 
@@ -390,7 +376,6 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
 {
 	int r;
 	struct dm_verity_fec_io *fio;
-	u64 offset, res, rsb;
 
 	if (!verity_fec_is_enabled(v))
 		return -EOPNOTSUPP;
@@ -407,38 +392,20 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
 	if (type == DM_VERITY_BLOCK_TYPE_METADATA)
 		block = block - v->hash_start + v->data_blocks;
 
-	/*
-	 * For RS(M, N), the continuous FEC data is divided into blocks of N
-	 * bytes. Since block size may not be divisible by N, the last block
-	 * is zero padded when decoding.
-	 *
-	 * Each byte of the block is covered by a different RS(M, N) code,
-	 * and each code is interleaved over N blocks to make it less likely
-	 * that bursty corruption will leave us in unrecoverable state.
-	 */
-
-	offset = block << v->data_dev_block_bits;
-	res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);
-
-	/*
-	 * The base RS block we can feed to the interleaver to find out all
-	 * blocks required for decoding.
-	 */
-	rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);
-
 	/*
 	 * Locating erasures is slow, so attempt to recover the block without
 	 * them first. Do a second attempt with erasures if the corruption is
 	 * bad enough.
 	 */
-	r = fec_decode_rsb(v, io, fio, rsb, offset, want_digest, false);
+	r = fec_decode(v, io, fio, block, want_digest, false);
 	if (r < 0) {
-		r = fec_decode_rsb(v, io, fio, rsb, offset, want_digest, true);
+		r = fec_decode(v, io, fio, block, want_digest, true);
 		if (r < 0)
 			goto done;
 	}
 
-	memcpy(dest, fio->output, 1 << v->data_dev_block_bits);
+	memcpy(dest, fio->output, v->fec->block_size);
+	atomic64_inc(&v->fec->corrected);
 
 done:
 	fio->level--;
@@ -585,8 +552,8 @@ int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
 
 	} else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) {
 		if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c ||
-		    num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) ||
-		    num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) {
+		    num_c < DM_VERITY_FEC_MIN_ROOTS ||
+		    num_c > DM_VERITY_FEC_MAX_ROOTS) {
 			ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS;
 			return -EINVAL;
 		}
@@ -625,7 +592,7 @@ int verity_fec_ctr(struct dm_verity *v)
 {
 	struct dm_verity_fec *f = v->fec;
 	struct dm_target *ti = v->ti;
-	u64 hash_blocks, fec_blocks;
+	u64 hash_blocks;
 	int ret;
 
 	if (!verity_fec_is_enabled(v)) {
@@ -648,7 +615,7 @@ int verity_fec_ctr(struct dm_verity *v)
 	 * hash device after the hash blocks.
 	 */
 
-	hash_blocks = v->hash_blocks - v->hash_start;
+	hash_blocks = v->hash_end - v->hash_start;
 
 	/*
 	 * Require matching block sizes for data and hash devices for
@@ -658,27 +625,28 @@ int verity_fec_ctr(struct dm_verity *v)
 		ti->error = "Block sizes must match to use FEC";
 		return -EINVAL;
 	}
+	f->block_size = 1 << v->data_dev_block_bits;
 
 	if (!f->roots) {
 		ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS;
 		return -EINVAL;
 	}
-	f->rsn = DM_VERITY_FEC_RSM - f->roots;
+	f->rs_k = DM_VERITY_FEC_RS_N - f->roots;
 
 	if (!f->blocks) {
 		ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS;
 		return -EINVAL;
 	}
 
-	f->rounds = f->blocks;
-	if (sector_div(f->rounds, f->rsn))
-		f->rounds++;
+	f->region_blocks = f->blocks;
+	if (sector_div(f->region_blocks, f->rs_k))
+		f->region_blocks++;
 
 	/*
 	 * Due to optional metadata, f->blocks can be larger than
 	 * data_blocks and hash_blocks combined.
 	 */
-	if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) {
+	if (f->blocks < v->data_blocks + hash_blocks || !f->region_blocks) {
 		ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
 		return -EINVAL;
 	}
@@ -688,16 +656,14 @@ int verity_fec_ctr(struct dm_verity *v)
 	 * it to be large enough.
 	 */
 	f->hash_blocks = f->blocks - v->data_blocks;
-	if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) {
+	if (dm_bufio_get_device_size(v->bufio) <
+	    v->hash_start + f->hash_blocks) {
 		ti->error = "Hash device is too small for "
 			DM_VERITY_OPT_FEC_BLOCKS;
 		return -E2BIG;
 	}
 
-	f->io_size = 1 << v->data_dev_block_bits;
-
-	f->bufio = dm_bufio_client_create(f->dev->bdev,
-					  f->io_size,
+	f->bufio = dm_bufio_client_create(f->dev->bdev, f->block_size,
 					  1, 0, NULL, NULL, 0);
 	if (IS_ERR(f->bufio)) {
 		ti->error = "Cannot initialize FEC bufio client";
@@ -706,14 +672,12 @@ int verity_fec_ctr(struct dm_verity *v)
 
 	dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT));
 
-	fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);
-	if (dm_bufio_get_device_size(f->bufio) < fec_blocks) {
+	if (dm_bufio_get_device_size(f->bufio) < f->region_blocks * f->roots) {
 		ti->error = "FEC device is too small";
 		return -E2BIG;
 	}
 
-	f->data_bufio = dm_bufio_client_create(v->data_dev->bdev,
-					       1 << v->data_dev_block_bits,
+	f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, f->block_size,
 					       1, 0, NULL, NULL, 0);
 	if (IS_ERR(f->data_bufio)) {
 		ti->error = "Cannot initialize FEC data bufio client";
@@ -743,7 +707,7 @@ int verity_fec_ctr(struct dm_verity *v)
 	}
 
 	f->cache = kmem_cache_create("dm_verity_fec_buffers",
-				     f->rsn << DM_VERITY_FEC_BUF_RS_BITS,
+				     f->rs_k << DM_VERITY_FEC_BUF_RS_BITS,
 				     0, 0, NULL);
 	if (!f->cache) {
 		ti->error = "Cannot create FEC buffer cache";
@@ -760,7 +724,7 @@ int verity_fec_ctr(struct dm_verity *v)
 
 	/* Preallocate an output buffer for each thread */
 	ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(),
-					1 << v->data_dev_block_bits);
+					f->block_size);
 	if (ret) {
 		ti->error = "Cannot allocate FEC output pool";
 		return ret;
diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h
index 35d28d9f8a9b0..50b5e187d5cc1 100644
--- a/drivers/md/dm-verity-fec.h
+++ b/drivers/md/dm-verity-fec.h
@@ -11,13 +11,13 @@
 #include "dm-verity.h"
 #include <linux/rslib.h>
 
-/* Reed-Solomon(M, N) parameters */
-#define DM_VERITY_FEC_RSM		255
-#define DM_VERITY_FEC_MAX_RSN		253
-#define DM_VERITY_FEC_MIN_RSN		231	/* ~10% space overhead */
+/* Reed-Solomon(n, k) parameters */
+#define DM_VERITY_FEC_RS_N	255
+#define DM_VERITY_FEC_MIN_ROOTS	2	/* RS(255, 253): ~0.8% space overhead */
+#define DM_VERITY_FEC_MAX_ROOTS	24	/* RS(255, 231): ~10% space overhead */
 
 /* buffers for deinterleaving and decoding */
-#define DM_VERITY_FEC_BUF_RS_BITS	4	/* 1 << RS blocks per buffer */
+#define DM_VERITY_FEC_BUF_RS_BITS	4 /* log2(RS messages per buffer) */
 
 #define DM_VERITY_OPT_FEC_DEV		"use_fec_from_device"
 #define DM_VERITY_OPT_FEC_BLOCKS	"fec_blocks"
@@ -29,13 +29,13 @@ struct dm_verity_fec {
 	struct dm_dev *dev;	/* parity data device */
 	struct dm_bufio_client *data_bufio;	/* for data dev access */
 	struct dm_bufio_client *bufio;		/* for parity data access */
-	size_t io_size;		/* IO size for roots */
+	size_t block_size;	/* size of data, hash, and parity blocks in bytes */
 	sector_t start;		/* parity data start in blocks */
 	sector_t blocks;	/* number of blocks covered */
-	sector_t rounds;	/* number of interleaving rounds */
+	sector_t region_blocks; /* blocks per region: ceil(blocks / rs_k) */
 	sector_t hash_blocks;	/* blocks covered after v->hash_start */
-	unsigned char roots;	/* number of parity bytes, M-N of RS(M, N) */
-	unsigned char rsn;	/* N of RS(M, N) */
+	unsigned char roots;	/* parity bytes per RS codeword, n-k of RS(n, k) */
+	unsigned char rs_k;	/* message bytes per RS codeword, k of RS(n, k) */
 	mempool_t fio_pool;	/* mempool for dm_verity_fec_io */
 	mempool_t rs_pool;	/* mempool for fio->rs */
 	mempool_t prealloc_pool;	/* mempool for preallocated buffers */
@@ -47,15 +47,15 @@ struct dm_verity_fec {
 /* per-bio data */
 struct dm_verity_fec_io {
 	struct rs_control *rs;	/* Reed-Solomon state */
-	int erasures[DM_VERITY_FEC_MAX_RSN];	/* erasures for decode_rs8 */
+	int erasures[DM_VERITY_FEC_MAX_ROOTS + 1]; /* erasures for decode_rs8 */
 	u8 *output;		/* buffer for corrected output */
 	unsigned int level;		/* recursion level */
 	unsigned int nbufs;		/* number of buffers allocated */
 	/*
-	 * Buffers for deinterleaving RS blocks.  Each buffer has space for
-	 * the data bytes of (1 << DM_VERITY_FEC_BUF_RS_BITS) RS blocks.  The
-	 * array length is fec_max_nbufs(v), and we try to allocate that many
-	 * buffers.  However, in low-memory situations we may be unable to
+	 * Buffers for deinterleaving RS codewords.  Each buffer has space for
+	 * the message bytes of (1 << DM_VERITY_FEC_BUF_RS_BITS) RS codewords.
+	 * The array length is fec_max_nbufs(v), and we try to allocate that
+	 * many buffers.  However, in low-memory situations we may be unable to
 	 * allocate all buffers.  'nbufs' holds the number actually allocated.
 	 */
 	u8 *bufs[];
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 61073cd01d131..e1d435c79e96c 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -733,8 +733,8 @@ static void verity_prefetch_io(struct work_struct *work)
 
 			hash_block_start &= ~(sector_t)(cluster - 1);
 			hash_block_end |= cluster - 1;
-			if (unlikely(hash_block_end >= v->hash_blocks))
-				hash_block_end = v->hash_blocks - 1;
+			if (unlikely(hash_block_end >= v->hash_end))
+				hash_block_end = v->hash_end - 1;
 		}
 no_prefetch_cluster:
 		dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start,
@@ -1607,7 +1607,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		}
 		hash_position += s;
 	}
-	v->hash_blocks = hash_position;
+	v->hash_end = hash_position;
 
 	r = mempool_init_page_pool(&v->recheck_pool, 1, 0);
 	if (unlikely(r)) {
@@ -1634,7 +1634,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
-	if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
+	if (dm_bufio_get_device_size(v->bufio) < v->hash_end) {
 		ti->error = "Hash device is too small";
 		r = -E2BIG;
 		goto bad;
diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h
index d6bfabb27113b..2922263501f68 100644
--- a/drivers/md/dm-verity.h
+++ b/drivers/md/dm-verity.h
@@ -53,9 +53,9 @@ struct dm_verity {
 	unsigned int sig_size;	/* root digest signature size */
 #endif /* CONFIG_SECURITY */
 	unsigned int salt_size;
-	sector_t hash_start;	/* hash start in blocks */
+	sector_t hash_start;	/* index of first hash block on hash_dev */
+	sector_t hash_end;	/* 1 + index of last hash block on hash dev */
 	sector_t data_blocks;	/* the number of data blocks */
-	sector_t hash_blocks;	/* the number of hash blocks */
 	unsigned char data_dev_block_bits;	/* log2(data blocksize) */
 	unsigned char hash_dev_block_bits;	/* log2(hash blocksize) */
 	unsigned char hash_per_block_bits;	/* log2(hashes in hash block) */
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 181400e147c0d..cda6af0712b92 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1878,7 +1878,7 @@ static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk,
 	if (info->rdev)
 		return false;
 
-	if (bdev_nonrot(rdev->bdev)) {
+	if (!bdev_rot(rdev->bdev)) {
 		set_bit(Nonrot, &rdev->flags);
 		WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1);
 	}
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 0653b5d8545a6..cfbd345805ca2 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -806,7 +806,7 @@ static struct md_rdev *read_balance(struct r10conf *conf,
 		if (!do_balance)
 			break;
 
-		nonrot = bdev_nonrot(rdev->bdev);
+		nonrot = !bdev_rot(rdev->bdev);
 		has_nonrot_disk |= nonrot;
 		pending = atomic_read(&rdev->nr_pending);
 		if (min_pending > pending && nonrot) {
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index a8e8d431071ba..ba9d6d05b0895 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -7541,7 +7541,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	rdev_for_each(rdev, mddev) {
 		if (test_bit(Journal, &rdev->flags))
 			continue;
-		if (bdev_nonrot(rdev->bdev)) {
+		if (!bdev_rot(rdev->bdev)) {
 			conf->batch_bio_dispatch = false;
 			break;
 		}
diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c
index 1953ce559ecaf..0fcd3b5e60c8d 100644
--- a/drivers/media/cec/core/cec-core.c
+++ b/drivers/media/cec/core/cec-core.c
@@ -338,8 +338,8 @@ int cec_register_adapter(struct cec_adapter *adap,
 	res = cec_devnode_register(&adap->devnode, adap->owner);
 	if (res) {
 #ifdef CONFIG_MEDIA_CEC_RC
-		/* Note: rc_unregister also calls rc_free */
 		rc_unregister_device(adap->rc);
+		rc_free_device(adap->rc);
 		adap->rc = NULL;
 #endif
 		return res;
diff --git a/drivers/media/cec/usb/pulse8/pulse8-cec.c b/drivers/media/cec/usb/pulse8/pulse8-cec.c
index 0df3af152762c..fa5df10627539 100644
--- a/drivers/media/cec/usb/pulse8/pulse8-cec.c
+++ b/drivers/media/cec/usb/pulse8/pulse8-cec.c
@@ -235,6 +235,9 @@ static int pulse8_send_and_wait_once(struct pulse8 *pulse8,
 {
 	int err;
 
+	if (!pulse8->serio)
+		return -ENODEV;
+
 	if (debug > 1)
 		dev_info(pulse8->dev, "transmit %s: %*ph\n",
 			 pulse8_msgname(cmd[0]), cmd_len, cmd);
@@ -655,6 +658,10 @@ static void pulse8_disconnect(struct serio *serio)
 {
 	struct pulse8 *pulse8 = serio_get_drvdata(serio);
 
+	cancel_delayed_work_sync(&pulse8->ping_eeprom_work);
+	mutex_lock(&pulse8->lock);
+	pulse8->serio = NULL;
+	mutex_unlock(&pulse8->lock);
 	cec_unregister_adapter(pulse8->adap);
 	serio_set_drvdata(serio, NULL);
 	serio_close(serio);
diff --git a/drivers/media/common/b2c2/flexcop.c b/drivers/media/common/b2c2/flexcop.c
index abd3d21a18cab..f7766a9b31dae 100644
--- a/drivers/media/common/b2c2/flexcop.c
+++ b/drivers/media/common/b2c2/flexcop.c
@@ -291,20 +291,6 @@ void flexcop_device_exit(struct flexcop_device *fc)
 }
 EXPORT_SYMBOL(flexcop_device_exit);
 
-static int flexcop_module_init(void)
-{
-	info(DRIVER_NAME " loaded successfully");
-	return 0;
-}
-
-static void flexcop_module_cleanup(void)
-{
-	info(DRIVER_NAME " unloaded successfully");
-}
-
-module_init(flexcop_module_init);
-module_exit(flexcop_module_cleanup);
-
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_NAME);
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/common/saa7146/saa7146_fops.c b/drivers/media/common/saa7146/saa7146_fops.c
index fd12ef10409ca..fe5c376d397ad 100644
--- a/drivers/media/common/saa7146/saa7146_fops.c
+++ b/drivers/media/common/saa7146/saa7146_fops.c
@@ -417,19 +417,6 @@ int saa7146_unregister_device(struct video_device *vfd, struct saa7146_dev *dev)
 }
 EXPORT_SYMBOL_GPL(saa7146_unregister_device);
 
-static int __init saa7146_vv_init_module(void)
-{
-	return 0;
-}
-
-
-static void __exit saa7146_vv_cleanup_module(void)
-{
-}
-
-module_init(saa7146_vv_init_module);
-module_exit(saa7146_vv_cleanup_module);
-
 MODULE_AUTHOR("Michael Hunold <michael@mihu.de>");
 MODULE_DESCRIPTION("video4linux driver for saa7146-based hardware");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/common/siano/smsir.c b/drivers/media/common/siano/smsir.c
index af07fed21ae12..283770d583d56 100644
--- a/drivers/media/common/siano/smsir.c
+++ b/drivers/media/common/siano/smsir.c
@@ -92,6 +92,7 @@ int sms_ir_init(struct smscore_device_t *coredev)
 void sms_ir_exit(struct smscore_device_t *coredev)
 {
 	rc_unregister_device(coredev->ir.dev);
+	rc_free_device(coredev->ir.dev);
 
 	pr_debug("\n");
 }
diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
index 982021d547e53..b1d0695cda260 100644
--- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c
+++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
@@ -345,6 +345,7 @@ static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma)
 		return err;
 	}
 
+	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
 	/*
 	 * Use common vm_area operations to track buffer refcount.
 	 */
diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c
index 58c4c489bf973..58b959b272c6f 100644
--- a/drivers/media/dvb-frontends/au8522_decoder.c
+++ b/drivers/media/dvb-frontends/au8522_decoder.c
@@ -567,6 +567,7 @@ static int au8522_s_video_routing(struct v4l2_subdev *sd,
 	case AU8522_COMPOSITE_CH1:
 	case AU8522_SVIDEO_CH13:
 	case AU8522_COMPOSITE_CH4_SIF:
+	case AU8522_COMPOSITE_CH4:
 		state->vid_input = input;
 		break;
 	default:
diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c
index ebef27bcc9899..d291113291e04 100644
--- a/drivers/media/dvb-frontends/dib8000.c
+++ b/drivers/media/dvb-frontends/dib8000.c
@@ -2695,7 +2695,7 @@ static void dib8000_viterbi_state(struct dib8000_state *state, u8 onoff)
 
 static void dib8000_set_dds(struct dib8000_state *state, s32 offset_khz)
 {
-	s16 unit_khz_dds_val;
+	s32 unit_khz_dds_val;
 	u32 abs_offset_khz = abs(offset_khz);
 	u32 dds = state->cfg.pll->ifreq & 0x1ffffff;
 	u8 invert = !!(state->cfg.pll->ifreq & (1 << 25));
@@ -2716,7 +2716,7 @@ static void dib8000_set_dds(struct dib8000_state *state, s32 offset_khz)
 			dds = (1<<26) - dds;
 	} else {
 		ratio = 2;
-		unit_khz_dds_val = (u16) (67108864 / state->cfg.pll->internal);
+		unit_khz_dds_val = 67108864 / state->cfg.pll->internal;
 
 		if (offset_khz < 0)
 			unit_khz_dds_val *= -1;
diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c
index c4bbcd127caca..9c5bac8cda477 100644
--- a/drivers/media/dvb-frontends/si2168.c
+++ b/drivers/media/dvb-frontends/si2168.c
@@ -40,7 +40,7 @@ static int si2168_cmd_execute(struct i2c_client *client, struct si2168_cmd *cmd)
 
 	if (cmd->rlen) {
 		/* wait cmd execution terminate */
-		#define TIMEOUT 70
+		#define TIMEOUT 140
 		timeout = jiffies + msecs_to_jiffies(TIMEOUT);
 		while (!time_after(jiffies, timeout)) {
 			ret = i2c_master_recv(client, cmd->args, cmd->rlen);
@@ -54,6 +54,8 @@ static int si2168_cmd_execute(struct i2c_client *client, struct si2168_cmd *cmd)
 			/* firmware ready? */
 			if ((cmd->args[0] >> 7) & 0x01)
 				break;
+
+			usleep_range(2500, 3500);
 		}
 
 		dev_dbg(&client->dev, "cmd execution took %d ms\n",
@@ -572,8 +574,8 @@ static int si2168_sleep(struct dvb_frontend *fe)
 	if (ret)
 		goto err;
 
-	/* Firmware later than B 4.0-11 loses warm state during sleep */
-	if (dev->version > ('B' << 24 | 4 << 16 | 0 << 8 | 11 << 0))
+	/* Firmware B 4.0-11 and later lose warm state during sleep */
+	if (dev->version >= ('B' << 24 | 4 << 16 | 0 << 8 | 11 << 0))
 		dev->warm = false;
 
 	cmd_init(&cmd, "\x13", 1, 0);
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 5eb1e0e0a87ab..20482be35f268 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -355,6 +355,7 @@ config VIDEO_MT9V111
 
 config VIDEO_OG01A1B
 	tristate "OmniVision OG01A1B sensor support"
+	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OG01A1B camera.
@@ -690,6 +691,7 @@ config VIDEO_OV8865
 config VIDEO_OV9282
 	tristate "OmniVision OV9282 sensor support"
 	depends on OF_GPIO
+	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV9282 camera sensor.
@@ -1724,8 +1726,8 @@ config VIDEO_DS90UB960
 	select V4L2_FWNODE
 	select VIDEO_V4L2_SUBDEV_API
 	help
-	  Device driver for the Texas Instruments DS90UB960
-	  FPD-Link III Deserializer and DS90UB9702 FPD-Link IV Deserializer.
+	  Device driver for the Texas Instruments DS90UB954, DS90UB960
+	  FPD-Link III Deserializers and DS90UB9702 FPD-Link IV Deserializer.
 
 config VIDEO_MAX96714
 	tristate "Maxim MAX96714 GMSL2 deserializer"
diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c
index e6fa2f9611770..955b7072a5605 100644
--- a/drivers/media/i2c/alvium-csi2.c
+++ b/drivers/media/i2c/alvium-csi2.c
@@ -2541,6 +2541,6 @@ module_i2c_driver(alvium_i2c_driver);
 
 MODULE_DESCRIPTION("Allied Vision's Alvium Camera Driver");
 MODULE_AUTHOR("Tommaso Merciai <tomm.merciai@gmail.com>");
-MODULE_AUTHOR("Martin Hecht <martin.hecht@avnet.eu>");
+MODULE_AUTHOR("Martin Hecht <mhecht73@gmail.com>");
 MODULE_AUTHOR("Avnet Silica Software & Services EMEA");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/ar0521.c b/drivers/media/i2c/ar0521.c
index f156058500e3d..ed324c2d87aa2 100644
--- a/drivers/media/i2c/ar0521.c
+++ b/drivers/media/i2c/ar0521.c
@@ -1094,6 +1094,9 @@ static int ar0521_probe(struct i2c_client *client)
 	/* Request optional reset pin (usually active low) and assert it */
 	sensor->reset_gpio = devm_gpiod_get_optional(dev, "reset",
 						     GPIOD_OUT_HIGH);
+	if (IS_ERR(sensor->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(sensor->reset_gpio),
+				     "failed to get reset gpio\n");
 
 	v4l2_i2c_subdev_init(&sensor->sd, client, &ar0521_subdev_ops);
 
diff --git a/drivers/media/i2c/ccs-pll.c b/drivers/media/i2c/ccs-pll.c
index 4eb83636e1027..1605cfa5db19d 100644
--- a/drivers/media/i2c/ccs-pll.c
+++ b/drivers/media/i2c/ccs-pll.c
@@ -824,9 +824,8 @@ int ccs_pll_calculate(struct device *dev, const struct ccs_pll_limits *lim,
 				   op_lim_fr->min_pll_ip_clk_freq_hz));
 	min_op_pre_pll_clk_div =
 		max_t(u16, op_lim_fr->min_pre_pll_clk_div,
-		      clk_div_even_up(
-			      DIV_ROUND_UP(pll->ext_clk_freq_hz,
-					   op_lim_fr->max_pll_ip_clk_freq_hz)));
+		      DIV_ROUND_UP(pll->ext_clk_freq_hz,
+				   op_lim_fr->max_pll_ip_clk_freq_hz));
 	dev_dbg(dev, "pre-pll check: min / max op_pre_pll_clk_div: %u / %u\n",
 		min_op_pre_pll_clk_div, max_op_pre_pll_clk_div);
 
diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c
index a863063043303..69d5cc648c0fc 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.c
+++ b/drivers/media/i2c/cx25840/cx25840-core.c
@@ -1652,10 +1652,14 @@ static int set_v4lstd(struct i2c_client *client)
 	struct cx25840_state *state = to_state(i2c_get_clientdata(client));
 	u8 fmt = 0;	/* zero is autodetect */
 	u8 pal_m = 0;
+	u8 pal_n = 0;
+	u8 ntsc_j = 0;
+	u8 tmp_reg = 0;
 
 	/* First tests should be against specific std */
 	if (state->std == V4L2_STD_NTSC_M_JP) {
 		fmt = 0x2;
+		ntsc_j = 0x80;
 	} else if (state->std == V4L2_STD_NTSC_443) {
 		fmt = 0x3;
 	} else if (state->std == V4L2_STD_PAL_M) {
@@ -1663,6 +1667,7 @@ static int set_v4lstd(struct i2c_client *client)
 		fmt = 0x5;
 	} else if (state->std == V4L2_STD_PAL_N) {
 		fmt = 0x6;
+		pal_n = 0x40;
 	} else if (state->std == V4L2_STD_PAL_Nc) {
 		fmt = 0x7;
 	} else if (state->std == V4L2_STD_PAL_60) {
@@ -1689,10 +1694,30 @@ static int set_v4lstd(struct i2c_client *client)
 		/* Set format to NTSC-M */
 		cx25840_and_or(client, 0x400, ~0xf, 1);
 		/* Turn off LCOMB */
-		cx25840_and_or(client, 0x47b, ~6, 0);
+		cx25840_and_or(client, 0x47b, ~0x6, 0);
+	} else if (fmt == 0xc) { /* SECAM - Step 9c - toggle CKILLEN */
+		tmp_reg = cx25840_read(client, 0x401);
+		cx25840_and_or(client, 0x401, ~0x20, tmp_reg & 0x20 ? 0x00 : 0x20);
+		cx25840_and_or(client, 0x401, ~0x20, tmp_reg & 0x20 ? 0x20 : 0x00);
 	}
+
 	cx25840_and_or(client, 0x400, ~0xf, fmt);
-	cx25840_and_or(client, 0x403, ~0x3, pal_m);
+
+	if (fmt >= 4 && fmt < 8) {
+		tmp_reg = cx25840_read(client, 0x401);
+		cx25840_and_or(client, 0x401, ~0x40, tmp_reg & 0x40 ? 0x00 : 0x40); /* CAGCEN */
+		cx25840_and_or(client, 0x401, ~0x40, tmp_reg & 0x40 ? 0x40 : 0x00);
+		cx25840_and_or(client, 0x401, ~0x20, tmp_reg & 0x20 ? 0x00 : 0x20); /* CKILLEN */
+		cx25840_and_or(client, 0x401, ~0x20, tmp_reg & 0x20 ? 0x20 : 0x00);
+	}
+
+	if (pal_m)
+		cx25840_and_or(client, 0x403, ~0x3, pal_m);
+	else if (pal_n)         /* cx25840 datasheet table 3-19 */
+		cx25840_and_or(client, 0x403, ~0x40, pal_n);
+	else if (ntsc_j)        /* cx25840 datasheet table 3-19 */
+		cx25840_and_or(client, 0x403, ~0x80, ntsc_j);
+
 	if (is_cx23888(state))
 		cx23888_std_setup(client);
 	else
diff --git a/drivers/media/i2c/ds90ub913.c b/drivers/media/i2c/ds90ub913.c
index e97e499b04e6c..49aa5f4a172c0 100644
--- a/drivers/media/i2c/ds90ub913.c
+++ b/drivers/media/i2c/ds90ub913.c
@@ -372,63 +372,6 @@ static int ub913_set_routing(struct v4l2_subdev *sd,
 	return _ub913_set_routing(sd, state, routing);
 }
 
-static int ub913_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
-				struct v4l2_mbus_frame_desc *fd)
-{
-	struct ub913_data *priv = sd_to_ub913(sd);
-	const struct v4l2_subdev_krouting *routing;
-	struct v4l2_mbus_frame_desc source_fd;
-	struct v4l2_subdev_route *route;
-	struct v4l2_subdev_state *state;
-	int ret;
-
-	if (pad != UB913_PAD_SOURCE)
-		return -EINVAL;
-
-	ret = v4l2_subdev_call(priv->source_sd, pad, get_frame_desc,
-			       priv->source_sd_pad, &source_fd);
-	if (ret)
-		return ret;
-
-	fd->type = V4L2_MBUS_FRAME_DESC_TYPE_PARALLEL;
-
-	state = v4l2_subdev_lock_and_get_active_state(sd);
-
-	routing = &state->routing;
-
-	for_each_active_route(routing, route) {
-		unsigned int i;
-
-		if (route->source_pad != pad)
-			continue;
-
-		for (i = 0; i < source_fd.num_entries; i++) {
-			if (source_fd.entry[i].stream == route->sink_stream)
-				break;
-		}
-
-		if (i == source_fd.num_entries) {
-			dev_err(&priv->client->dev,
-				"Failed to find stream from source frame desc\n");
-			ret = -EPIPE;
-			goto out_unlock;
-		}
-
-		fd->entry[fd->num_entries].stream = route->source_stream;
-		fd->entry[fd->num_entries].flags = source_fd.entry[i].flags;
-		fd->entry[fd->num_entries].length = source_fd.entry[i].length;
-		fd->entry[fd->num_entries].pixelcode =
-			source_fd.entry[i].pixelcode;
-
-		fd->num_entries++;
-	}
-
-out_unlock:
-	v4l2_subdev_unlock_state(state);
-
-	return ret;
-}
-
 static int ub913_set_fmt(struct v4l2_subdev *sd,
 			 struct v4l2_subdev_state *state,
 			 struct v4l2_subdev_format *format)
@@ -544,7 +487,7 @@ static const struct v4l2_subdev_pad_ops ub913_pad_ops = {
 	.enable_streams = ub913_enable_streams,
 	.disable_streams = ub913_disable_streams,
 	.set_routing = ub913_set_routing,
-	.get_frame_desc = ub913_get_frame_desc,
+	.get_frame_desc = v4l2_subdev_get_frame_desc_passthrough,
 	.get_fmt = v4l2_subdev_get_fmt,
 	.set_fmt = ub913_set_fmt,
 };
diff --git a/drivers/media/i2c/ds90ub953.c b/drivers/media/i2c/ds90ub953.c
index daefdb108fbf4..a8ab67f4137f3 100644
--- a/drivers/media/i2c/ds90ub953.c
+++ b/drivers/media/i2c/ds90ub953.c
@@ -424,65 +424,6 @@ static int ub953_set_routing(struct v4l2_subdev *sd,
 	return _ub953_set_routing(sd, state, routing);
 }
 
-static int ub953_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
-				struct v4l2_mbus_frame_desc *fd)
-{
-	struct ub953_data *priv = sd_to_ub953(sd);
-	struct v4l2_mbus_frame_desc source_fd;
-	struct v4l2_subdev_route *route;
-	struct v4l2_subdev_state *state;
-	int ret;
-
-	if (pad != UB953_PAD_SOURCE)
-		return -EINVAL;
-
-	ret = v4l2_subdev_call(priv->source_sd, pad, get_frame_desc,
-			       priv->source_sd_pad, &source_fd);
-	if (ret)
-		return ret;
-
-	fd->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
-
-	state = v4l2_subdev_lock_and_get_active_state(sd);
-
-	for_each_active_route(&state->routing, route) {
-		struct v4l2_mbus_frame_desc_entry *source_entry = NULL;
-		unsigned int i;
-
-		if (route->source_pad != pad)
-			continue;
-
-		for (i = 0; i < source_fd.num_entries; i++) {
-			if (source_fd.entry[i].stream == route->sink_stream) {
-				source_entry = &source_fd.entry[i];
-				break;
-			}
-		}
-
-		if (!source_entry) {
-			dev_err(&priv->client->dev,
-				"Failed to find stream from source frame desc\n");
-			ret = -EPIPE;
-			goto out_unlock;
-		}
-
-		fd->entry[fd->num_entries].stream = route->source_stream;
-		fd->entry[fd->num_entries].flags = source_entry->flags;
-		fd->entry[fd->num_entries].length = source_entry->length;
-		fd->entry[fd->num_entries].pixelcode = source_entry->pixelcode;
-		fd->entry[fd->num_entries].bus.csi2.vc =
-			source_entry->bus.csi2.vc;
-		fd->entry[fd->num_entries].bus.csi2.dt =
-			source_entry->bus.csi2.dt;
-
-		fd->num_entries++;
-	}
-
-out_unlock:
-	v4l2_subdev_unlock_state(state);
-
-	return ret;
-}
 
 static int ub953_set_fmt(struct v4l2_subdev *sd,
 			 struct v4l2_subdev_state *state,
@@ -694,7 +635,7 @@ static const struct v4l2_subdev_pad_ops ub953_pad_ops = {
 	.enable_streams = ub953_enable_streams,
 	.disable_streams = ub953_disable_streams,
 	.set_routing = ub953_set_routing,
-	.get_frame_desc = ub953_get_frame_desc,
+	.get_frame_desc = v4l2_subdev_get_frame_desc_passthrough,
 	.get_fmt = v4l2_subdev_get_fmt,
 	.set_fmt = ub953_set_fmt,
 };
diff --git a/drivers/media/i2c/ds90ub960.c b/drivers/media/i2c/ds90ub960.c
index e133127397f48..d50e977cf6ce4 100644
--- a/drivers/media/i2c/ds90ub960.c
+++ b/drivers/media/i2c/ds90ub960.c
@@ -396,6 +396,13 @@
 #define UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY	BIT(3)
 #define UB960_IR_RX_ANA_STROBE_SET_DATA_DELAY_MASK	GENMASK(2, 0)
 
+#define UB954_IR_RX_ANA_STROBE_SET_CLK_DATA		0x08
+#define UB954_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY	BIT(3)
+#define UB954_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY	BIT(7)
+#define UB954_IR_RX_ANA_STROBE_SET_CLK_DELAY_MASK	GENMASK(2, 0)
+#define UB954_IR_RX_ANA_STROBE_SET_DATA_DELAY_MASK	GENMASK(4, 6)
+#define UB954_IR_RX_ANA_STROBE_SET_DATA_DELAY_SHIFT	4
+
 /* UB9702 Registers */
 
 #define UB9702_SR_CSI_EXCLUSIVE_FWD2		0x3c
@@ -454,12 +461,23 @@
 #define UB960_MAX_EQ_LEVEL  14
 #define UB960_NUM_EQ_LEVELS (UB960_MAX_EQ_LEVEL - UB960_MIN_EQ_LEVEL + 1)
 
+enum chip_type {
+	UB954,
+	UB960,
+	UB9702,
+};
+
+enum chip_family {
+	FAMILY_FPD3,
+	FAMILY_FPD4,
+};
+
 struct ub960_hw_data {
 	const char *model;
+	enum chip_type chip_type;
+	enum chip_family chip_family;
 	u8 num_rxports;
 	u8 num_txports;
-	bool is_ub9702;
-	bool is_fpdlink4;
 };
 
 enum ub960_rxport_mode {
@@ -982,6 +1000,10 @@ static int ub960_txport_select(struct ub960_data *priv, u8 nport)
 
 	lockdep_assert_held(&priv->reg_lock);
 
+	/* UB954 has only 1 CSI TX. Hence, no need to select */
+	if (priv->hw_data->chip_type == UB954)
+		return 0;
+
 	if (priv->reg_current.txport == nport)
 		return 0;
 
@@ -1406,10 +1428,11 @@ static int ub960_parse_dt_txport(struct ub960_data *priv,
 	priv->tx_link_freq[0] = vep.link_frequencies[0];
 	priv->tx_data_rate = priv->tx_link_freq[0] * 2;
 
-	if (priv->tx_data_rate != MHZ(1600) &&
-	    priv->tx_data_rate != MHZ(1200) &&
-	    priv->tx_data_rate != MHZ(800) &&
-	    priv->tx_data_rate != MHZ(400)) {
+	if ((priv->tx_data_rate != MHZ(1600) &&
+	     priv->tx_data_rate != MHZ(1200) &&
+	     priv->tx_data_rate != MHZ(800) &&
+	     priv->tx_data_rate != MHZ(400)) ||
+	     (priv->hw_data->chip_type == UB954 && priv->tx_data_rate == MHZ(1200))) {
 		dev_err(dev, "tx%u: invalid 'link-frequencies' value\n", nport);
 		ret = -EINVAL;
 		goto err_free_vep;
@@ -1533,22 +1556,35 @@ static int ub960_rxport_get_strobe_pos(struct ub960_data *priv,
 	u8 clk_delay, data_delay;
 	int ret;
 
-	ret = ub960_read_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
-			     UB960_IR_RX_ANA_STROBE_SET_CLK, &v, NULL);
-	if (ret)
-		return ret;
+	if (priv->hw_data->chip_type == UB954) {
+		ret = ub960_read_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				     UB954_IR_RX_ANA_STROBE_SET_CLK_DATA, &v, NULL);
+		if (ret)
+			return ret;
 
-	clk_delay = (v & UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY) ?
-			    0 : UB960_MANUAL_STROBE_EXTRA_DELAY;
+		clk_delay = (v & UB954_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY) ?
+			     0 : UB960_MANUAL_STROBE_EXTRA_DELAY;
 
-	ret = ub960_read_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
-			     UB960_IR_RX_ANA_STROBE_SET_DATA, &v, NULL);
-	if (ret)
-		return ret;
+		data_delay = (v & UB954_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY) ?
+			      0 : UB960_MANUAL_STROBE_EXTRA_DELAY;
+	} else {
+		ret = ub960_read_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				     UB960_IR_RX_ANA_STROBE_SET_CLK, &v, NULL);
+		if (ret)
+			return ret;
 
-	data_delay = (v & UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY) ?
+		clk_delay = (v & UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY) ?
 			     0 : UB960_MANUAL_STROBE_EXTRA_DELAY;
 
+		ret = ub960_read_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				     UB960_IR_RX_ANA_STROBE_SET_DATA, &v, NULL);
+		if (ret)
+			return ret;
+
+		data_delay = (v & UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY) ?
+			      0 : UB960_MANUAL_STROBE_EXTRA_DELAY;
+	}
+
 	ret = ub960_rxport_read(priv, nport, UB960_RR_SFILTER_STS_0, &v, NULL);
 	if (ret)
 		return ret;
@@ -1569,26 +1605,49 @@ static int ub960_rxport_get_strobe_pos(struct ub960_data *priv,
 static int ub960_rxport_set_strobe_pos(struct ub960_data *priv,
 				       unsigned int nport, s8 strobe_pos)
 {
-	u8 clk_delay, data_delay;
 	int ret = 0;
 
-	clk_delay = UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY;
-	data_delay = UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY;
-
-	if (strobe_pos < UB960_MIN_AEQ_STROBE_POS)
-		clk_delay = abs(strobe_pos) - UB960_MANUAL_STROBE_EXTRA_DELAY;
-	else if (strobe_pos > UB960_MAX_AEQ_STROBE_POS)
-		data_delay = strobe_pos - UB960_MANUAL_STROBE_EXTRA_DELAY;
-	else if (strobe_pos < 0)
-		clk_delay = abs(strobe_pos) | UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY;
-	else if (strobe_pos > 0)
-		data_delay = strobe_pos | UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY;
-
-	ub960_write_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
-			UB960_IR_RX_ANA_STROBE_SET_CLK, clk_delay, &ret);
-
-	ub960_write_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
-			UB960_IR_RX_ANA_STROBE_SET_DATA, data_delay, &ret);
+	if (priv->hw_data->chip_type == UB954) {
+		u8 clk_data_delay;
+
+		clk_data_delay = UB954_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY |
+				 UB954_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY;
+
+		if (strobe_pos < UB960_MIN_AEQ_STROBE_POS)
+			clk_data_delay = abs(strobe_pos) - UB960_MANUAL_STROBE_EXTRA_DELAY;
+		else if (strobe_pos > UB960_MAX_AEQ_STROBE_POS)
+			clk_data_delay = (strobe_pos - UB960_MANUAL_STROBE_EXTRA_DELAY) <<
+					  UB954_IR_RX_ANA_STROBE_SET_DATA_DELAY_SHIFT;
+		else if (strobe_pos < 0)
+			clk_data_delay = abs(strobe_pos) |
+					 UB954_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY;
+		else if (strobe_pos > 0)
+			clk_data_delay = (strobe_pos |
+					  UB954_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY) <<
+					  UB954_IR_RX_ANA_STROBE_SET_DATA_DELAY_SHIFT;
+
+		ub960_write_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				UB954_IR_RX_ANA_STROBE_SET_CLK_DATA, clk_data_delay, &ret);
+	} else {
+		u8 clk_delay, data_delay;
+
+		clk_delay = UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY;
+		data_delay = UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY;
+
+		if (strobe_pos < UB960_MIN_AEQ_STROBE_POS)
+			clk_delay = abs(strobe_pos) - UB960_MANUAL_STROBE_EXTRA_DELAY;
+		else if (strobe_pos > UB960_MAX_AEQ_STROBE_POS)
+			data_delay = strobe_pos - UB960_MANUAL_STROBE_EXTRA_DELAY;
+		else if (strobe_pos < 0)
+			clk_delay = abs(strobe_pos) | UB960_IR_RX_ANA_STROBE_SET_CLK_NO_EXTRA_DELAY;
+		else if (strobe_pos > 0)
+			data_delay = strobe_pos | UB960_IR_RX_ANA_STROBE_SET_DATA_NO_EXTRA_DELAY;
+
+		ub960_write_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				UB960_IR_RX_ANA_STROBE_SET_CLK, clk_delay, &ret);
+		ub960_write_ind(priv, UB960_IND_TARGET_RX_ANA(nport),
+				UB960_IR_RX_ANA_STROBE_SET_DATA, data_delay, &ret);
+	}
 
 	return ret;
 }
@@ -1924,7 +1983,7 @@ static int ub960_rxport_wait_locks(struct ub960_data *priv,
 		if (ret)
 			return ret;
 
-		if (priv->hw_data->is_ub9702) {
+		if (priv->hw_data->chip_type == UB9702) {
 			dev_dbg(dev, "\trx%u: locked, freq %llu Hz\n",
 				nport, ((u64)v * HZ_PER_MHZ) >> 8);
 		} else {
@@ -2186,7 +2245,7 @@ static int ub960_rxport_add_serializer(struct ub960_data *priv, u8 nport)
 
 	ser_pdata->port = nport;
 	ser_pdata->atr = priv->atr;
-	if (priv->hw_data->is_ub9702)
+	if (priv->hw_data->chip_type == UB9702)
 		ser_pdata->bc_rate = ub960_calc_bc_clk_rate_ub9702(priv, rxport);
 	else
 		ser_pdata->bc_rate = ub960_calc_bc_clk_rate_ub960(priv, rxport);
@@ -2352,7 +2411,7 @@ static int ub960_init_tx_ports(struct ub960_data *priv)
 {
 	int ret;
 
-	if (priv->hw_data->is_ub9702)
+	if (priv->hw_data->chip_type == UB9702)
 		ret = ub960_init_tx_ports_ub9702(priv);
 	else
 		ret = ub960_init_tx_ports_ub960(priv);
@@ -3624,7 +3683,8 @@ static int ub960_configure_ports_for_streaming(struct ub960_data *priv,
 
 		case RXPORT_MODE_CSI2_SYNC:
 		case RXPORT_MODE_CSI2_NONSYNC:
-			if (!priv->hw_data->is_ub9702) {
+			if (priv->hw_data->chip_type == UB960 ||
+			    priv->hw_data->chip_type == UB954) {
 				/* Map all VCs from this port to the same VC */
 				ub960_rxport_write(priv, nport, UB960_RR_CSI_VC_MAP,
 						   (vc << UB960_RR_CSI_VC_MAP_SHIFT(3)) |
@@ -4158,33 +4218,40 @@ static int ub960_log_status(struct v4l2_subdev *sd)
 		dev_info(dev, "\tsync %u, pass %u\n", v & (u8)BIT(1),
 			 v & (u8)BIT(0));
 
-		ret = ub960_read16(priv, UB960_SR_CSI_FRAME_COUNT_HI(nport),
-				   &v16, NULL);
-		if (ret)
-			return ret;
+		/*
+		 * Frame counter, frame error counter, line counter and line error counter
+		 * registers are marked as reserved in the UB954 datasheet. Hence restrict
+		 * the following register reads only for UB960 and UB9702.
+		 */
+		if (priv->hw_data->chip_type == UB960 || priv->hw_data->chip_type == UB9702) {
+			ret = ub960_read16(priv, UB960_SR_CSI_FRAME_COUNT_HI(nport),
+					   &v16, NULL);
+			if (ret)
+				return ret;
 
-		dev_info(dev, "\tframe counter %u\n", v16);
+			dev_info(dev, "\tframe counter %u\n", v16);
 
-		ret = ub960_read16(priv, UB960_SR_CSI_FRAME_ERR_COUNT_HI(nport),
-				   &v16, NULL);
-		if (ret)
-			return ret;
+			ret = ub960_read16(priv, UB960_SR_CSI_FRAME_ERR_COUNT_HI(nport),
+					   &v16, NULL);
+			if (ret)
+				return ret;
 
-		dev_info(dev, "\tframe error counter %u\n", v16);
+			dev_info(dev, "\tframe error counter %u\n", v16);
 
-		ret = ub960_read16(priv, UB960_SR_CSI_LINE_COUNT_HI(nport),
-				   &v16, NULL);
-		if (ret)
-			return ret;
+			ret = ub960_read16(priv, UB960_SR_CSI_LINE_COUNT_HI(nport),
+					   &v16, NULL);
+			if (ret)
+				return ret;
 
-		dev_info(dev, "\tline counter %u\n", v16);
+			dev_info(dev, "\tline counter %u\n", v16);
 
-		ret = ub960_read16(priv, UB960_SR_CSI_LINE_ERR_COUNT_HI(nport),
-				   &v16, NULL);
-		if (ret)
-			return ret;
+			ret = ub960_read16(priv, UB960_SR_CSI_LINE_ERR_COUNT_HI(nport),
+					   &v16, NULL);
+			if (ret)
+				return ret;
 
-		dev_info(dev, "\tline error counter %u\n", v16);
+			dev_info(dev, "\tline error counter %u\n", v16);
+		}
 	}
 
 	for_each_rxport(priv, it) {
@@ -4250,7 +4317,7 @@ static int ub960_log_status(struct v4l2_subdev *sd)
 
 		dev_info(dev, "\tcsi_err_counter %u\n", v);
 
-		if (!priv->hw_data->is_ub9702) {
+		if (priv->hw_data->chip_type == UB960 || priv->hw_data->chip_type == UB954) {
 			ret = ub960_log_status_ub960_sp_eq(priv, nport);
 			if (ret)
 				return ret;
@@ -4408,7 +4475,7 @@ ub960_parse_dt_rxport_link_properties(struct ub960_data *priv,
 		return -EINVAL;
 	}
 
-	if (!priv->hw_data->is_fpdlink4 && cdr_mode == RXPORT_CDR_FPD4) {
+	if (priv->hw_data->chip_family != FAMILY_FPD4 && cdr_mode == RXPORT_CDR_FPD4) {
 		dev_err(dev, "rx%u: FPD-Link 4 CDR not supported\n", nport);
 		return -EINVAL;
 	}
@@ -5010,7 +5077,12 @@ static int ub960_enable_core_hw(struct ub960_data *priv)
 	if (ret)
 		goto err_pd_gpio;
 
-	if (priv->hw_data->is_ub9702)
+	/*
+	 * UB954 REFCLK_FREQ is not synchronized, so multiple reads are recommended
+	 * by the datasheet. However, a single read is practically seen to be
+	 * sufficient and moreover it is only used for a debug print.
+	 */
+	if (priv->hw_data->chip_type == UB9702)
 		ret = ub960_read(priv, UB9702_SR_REFCLK_FREQ, &refclk_freq,
 				 NULL);
 	else
@@ -5029,7 +5101,7 @@ static int ub960_enable_core_hw(struct ub960_data *priv)
 		goto err_pd_gpio;
 
 	/* release GPIO lock */
-	if (priv->hw_data->is_ub9702) {
+	if (priv->hw_data->chip_type == UB9702) {
 		ret = ub960_update_bits(priv, UB960_SR_RESET,
 					UB960_SR_RESET_GPIO_LOCK_RELEASE,
 					UB960_SR_RESET_GPIO_LOCK_RELEASE,
@@ -5102,7 +5174,7 @@ static int ub960_probe(struct i2c_client *client)
 	if (ret)
 		goto err_free_ports;
 
-	if (priv->hw_data->is_ub9702)
+	if (priv->hw_data->chip_type == UB9702)
 		ret = ub960_init_rx_ports_ub9702(priv);
 	else
 		ret = ub960_init_rx_ports_ub960(priv);
@@ -5169,21 +5241,32 @@ static void ub960_remove(struct i2c_client *client)
 	mutex_destroy(&priv->reg_lock);
 }
 
+static const struct ub960_hw_data ds90ub954_hw = {
+	.model = "ub954",
+	.chip_type = UB954,
+	.chip_family = FAMILY_FPD3,
+	.num_rxports = 2,
+	.num_txports = 1,
+};
+
 static const struct ub960_hw_data ds90ub960_hw = {
 	.model = "ub960",
+	.chip_type = UB960,
+	.chip_family = FAMILY_FPD3,
 	.num_rxports = 4,
 	.num_txports = 2,
 };
 
 static const struct ub960_hw_data ds90ub9702_hw = {
 	.model = "ub9702",
+	.chip_type = UB9702,
+	.chip_family = FAMILY_FPD4,
 	.num_rxports = 4,
 	.num_txports = 2,
-	.is_ub9702 = true,
-	.is_fpdlink4 = true,
 };
 
 static const struct i2c_device_id ub960_id[] = {
+	{ "ds90ub954-q1", (kernel_ulong_t)&ds90ub954_hw },
 	{ "ds90ub960-q1", (kernel_ulong_t)&ds90ub960_hw },
 	{ "ds90ub9702-q1", (kernel_ulong_t)&ds90ub9702_hw },
 	{}
@@ -5191,6 +5274,7 @@ static const struct i2c_device_id ub960_id[] = {
 MODULE_DEVICE_TABLE(i2c, ub960_id);
 
 static const struct of_device_id ub960_dt_ids[] = {
+	{ .compatible = "ti,ds90ub954-q1", .data = &ds90ub954_hw },
 	{ .compatible = "ti,ds90ub960-q1", .data = &ds90ub960_hw },
 	{ .compatible = "ti,ds90ub9702-q1", .data = &ds90ub9702_hw },
 	{}
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index fee63bc106d9e..7da02ce5da154 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -1218,6 +1218,9 @@ static int imx219_probe(struct i2c_client *client)
 	/* Request optional enable pin */
 	imx219->reset_gpio = devm_gpiod_get_optional(dev, "reset",
 						     GPIOD_OUT_HIGH);
+	if (IS_ERR(imx219->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(imx219->reset_gpio),
+				     "failed to get reset gpio\n");
 
 	/*
 	 * The sensor must be powered for imx219_identify_module()
diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
index e50dcfd830f52..bc9ee449a87c5 100644
--- a/drivers/media/i2c/imx258.c
+++ b/drivers/media/i2c/imx258.c
@@ -709,12 +709,16 @@ static int imx258_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 		v4l2_subdev_state_get_format(fh->state, 0);
 	struct v4l2_rect *try_crop;
 
+	mutex_lock(&imx258->mutex);
+
 	/* Initialize try_fmt */
 	try_fmt->width = supported_modes[0].width;
 	try_fmt->height = supported_modes[0].height;
 	try_fmt->code = imx258_get_format_code(imx258);
 	try_fmt->field = V4L2_FIELD_NONE;
 
+	mutex_unlock(&imx258->mutex);
+
 	/* Initialize try_crop */
 	try_crop = v4l2_subdev_state_get_crop(fh->state, 0);
 	try_crop->left = IMX258_PIXEL_ARRAY_LEFT;
@@ -839,7 +843,9 @@ static int imx258_enum_mbus_code(struct v4l2_subdev *sd,
 	if (code->index > 0)
 		return -EINVAL;
 
+	mutex_lock(&imx258->mutex);
 	code->code = imx258_get_format_code(imx258);
+	mutex_unlock(&imx258->mutex);
 
 	return 0;
 }
@@ -849,10 +855,16 @@ static int imx258_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_frame_size_enum *fse)
 {
 	struct imx258 *imx258 = to_imx258(sd);
+	u32 code;
+
 	if (fse->index >= ARRAY_SIZE(supported_modes))
 		return -EINVAL;
 
-	if (fse->code != imx258_get_format_code(imx258))
+	mutex_lock(&imx258->mutex);
+	code = imx258_get_format_code(imx258);
+	mutex_unlock(&imx258->mutex);
+
+	if (fse->code != code)
 		return -EINVAL;
 
 	fse->min_width = supported_modes[fse->index].width;
diff --git a/drivers/media/i2c/imx283.c b/drivers/media/i2c/imx283.c
index 8ab63ad8f385f..1be6164c2d155 100644
--- a/drivers/media/i2c/imx283.c
+++ b/drivers/media/i2c/imx283.c
@@ -129,7 +129,8 @@
 
 /* Master Mode Operation Control */
 #define IMX283_REG_XMSTA		CCI_REG8(0x3105)
-#define   IMX283_XMSTA			BIT(0)
+#define   IMX283_XMSTA_START		0
+#define   IMX283_XMSTA_STOP		BIT(0)
 
 #define IMX283_REG_SYNCDRV		CCI_REG8(0x3107)
 #define   IMX283_SYNCDRV_XHS_XVS	(0xa0 | 0x02)
@@ -1023,8 +1024,6 @@ static int imx283_standby_cancel(struct imx283 *imx283)
 	usleep_range(19000, 20000);
 
 	cci_write(imx283->cci, IMX283_REG_CLAMP, IMX283_CLPSQRST, &ret);
-	cci_write(imx283->cci, IMX283_REG_XMSTA, 0, &ret);
-	cci_write(imx283->cci, IMX283_REG_SYNCDRV, IMX283_SYNCDRV_XHS_XVS, &ret);
 
 	return ret;
 }
@@ -1117,6 +1116,10 @@ static int imx283_start_streaming(struct imx283 *imx283,
 	/* Apply customized values from controls (HMAX/VMAX/SHR) */
 	ret =  __v4l2_ctrl_handler_setup(imx283->sd.ctrl_handler);
 
+	/* Start master mode */
+	cci_write(imx283->cci, IMX283_REG_XMSTA, IMX283_XMSTA_START, &ret);
+	cci_write(imx283->cci, IMX283_REG_SYNCDRV, IMX283_SYNCDRV_XHS_XVS, &ret);
+
 	return ret;
 }
 
@@ -1153,12 +1156,14 @@ static int imx283_disable_streams(struct v4l2_subdev *sd,
 				  u64 streams_mask)
 {
 	struct imx283 *imx283 = to_imx283(sd);
-	int ret;
+	int ret = 0;
 
 	if (pad != IMAGE_PAD)
 		return -EINVAL;
 
-	ret = cci_write(imx283->cci, IMX283_REG_STANDBY, IMX283_STBLOGIC, NULL);
+	cci_write(imx283->cci, IMX283_REG_XMSTA, IMX283_XMSTA_STOP, &ret);
+	cci_write(imx283->cci, IMX283_REG_STANDBY, IMX283_STANDBY, &ret);
+
 	if (ret)
 		dev_err(imx283->dev, "Failed to stop stream\n");
 
diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c
index 776107efe3861..9ca87488c933d 100644
--- a/drivers/media/i2c/imx355.c
+++ b/drivers/media/i2c/imx355.c
@@ -3,9 +3,13 @@
 
 #include <linux/acpi.h>
 #include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
 #include <linux/unaligned.h>
 
 #include <media/v4l2-ctrls.h>
@@ -125,6 +129,15 @@ struct imx355 {
 	 * Protect access to sensor v4l2 controls.
 	 */
 	struct mutex mutex;
+
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data *supplies;
+};
+
+static const struct regulator_bulk_data imx355_supplies[] = {
+	{ .supply = "avdd" },
+	{ .supply = "dvdd" },
+	{ .supply = "dovdd" },
 };
 
 static const struct imx355_reg imx355_global_regs[] = {
@@ -1515,6 +1528,52 @@ static const struct v4l2_subdev_internal_ops imx355_internal_ops = {
 	.open = imx355_open,
 };
 
+static int imx355_power_off(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx355 *imx355 = to_imx355(sd);
+
+	gpiod_set_value_cansleep(imx355->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(imx355_supplies), imx355->supplies);
+	clk_disable_unprepare(imx355->clk);
+
+	return 0;
+}
+
+static int imx355_power_on(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx355 *imx355 = to_imx355(sd);
+	int ret;
+
+	ret = clk_prepare_enable(imx355->clk);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable clocks");
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(imx355_supplies),
+				    imx355->supplies);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to enable regulators");
+		goto error_disable_clocks;
+	}
+
+	usleep_range(1000, 2000);
+	gpiod_set_value_cansleep(imx355->reset_gpio, 0);
+	usleep_range(10000, 11000);
+
+	return 0;
+
+error_disable_clocks:
+	clk_disable_unprepare(imx355->clk);
+	return ret;
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(imx355_pm_ops, imx355_power_off,
+				 imx355_power_on, NULL);
+
 /* Initialize control handlers */
 static int imx355_init_controls(struct imx355 *imx355)
 {
@@ -1689,16 +1748,26 @@ static int imx355_probe(struct i2c_client *client)
 				     "external clock %lu is not supported\n",
 				     freq);
 
-	/* Initialize subdev */
-	v4l2_i2c_subdev_init(&imx355->sd, client, &imx355_subdev_ops);
-
-	/* Check module identity */
-	ret = imx355_identify_module(imx355);
+	ret = devm_regulator_bulk_get_const(imx355->dev,
+					    ARRAY_SIZE(imx355_supplies),
+					    imx355_supplies,
+					    &imx355->supplies);
 	if (ret) {
-		dev_err(imx355->dev, "failed to find sensor: %d", ret);
+		dev_err_probe(imx355->dev, ret, "could not get regulators");
 		goto error_probe;
 	}
 
+	imx355->reset_gpio = devm_gpiod_get_optional(imx355->dev, "reset",
+						     GPIOD_OUT_HIGH);
+	if (IS_ERR(imx355->reset_gpio)) {
+		ret = dev_err_probe(imx355->dev, PTR_ERR(imx355->reset_gpio),
+				    "failed to get gpios");
+		goto error_probe;
+	}
+
+	/* Initialize subdev */
+	v4l2_i2c_subdev_init(&imx355->sd, client, &imx355_subdev_ops);
+
 	imx355->hwcfg = imx355_get_hwcfg(imx355->dev);
 	if (!imx355->hwcfg) {
 		dev_err(imx355->dev, "failed to get hwcfg");
@@ -1706,13 +1775,24 @@ static int imx355_probe(struct i2c_client *client)
 		goto error_probe;
 	}
 
+	ret = imx355_power_on(imx355->dev);
+	if (ret)
+		goto error_probe;
+
+	/* Check module identity */
+	ret = imx355_identify_module(imx355);
+	if (ret) {
+		dev_err(imx355->dev, "failed to find sensor: %d", ret);
+		goto error_power_off;
+	}
+
 	/* Set default mode to max resolution */
 	imx355->cur_mode = &supported_modes[0];
 
 	ret = imx355_init_controls(imx355);
 	if (ret) {
 		dev_err(imx355->dev, "failed to init controls: %d", ret);
-		goto error_probe;
+		goto error_power_off;
 	}
 
 	/* Initialize subdev */
@@ -1752,6 +1832,9 @@ static int imx355_probe(struct i2c_client *client)
 error_handler_free:
 	v4l2_ctrl_handler_free(imx355->sd.ctrl_handler);
 
+error_power_off:
+	imx355_power_off(imx355->dev);
+
 error_probe:
 	mutex_destroy(&imx355->mutex);
 
@@ -1768,7 +1851,11 @@ static void imx355_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 
 	pm_runtime_disable(imx355->dev);
-	pm_runtime_set_suspended(imx355->dev);
+
+	if (!pm_runtime_status_suspended(imx355->dev)) {
+		imx355_power_off(imx355->dev);
+		pm_runtime_set_suspended(imx355->dev);
+	}
 
 	mutex_destroy(&imx355->mutex);
 }
@@ -1779,10 +1866,18 @@ static const struct acpi_device_id imx355_acpi_ids[] __maybe_unused = {
 };
 MODULE_DEVICE_TABLE(acpi, imx355_acpi_ids);
 
+static const struct of_device_id imx355_match_table[] = {
+	{ .compatible = "sony,imx355", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, imx355_match_table);
+
 static struct i2c_driver imx355_i2c_driver = {
 	.driver = {
 		.name = "imx355",
 		.acpi_match_table = ACPI_PTR(imx355_acpi_ids),
+		.of_match_table = imx355_match_table,
+		.pm = &imx355_pm_ops,
 	},
 	.probe = imx355_probe,
 	.remove = imx355_remove,
diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c
index b3826f8035470..e25e0a9ff65c3 100644
--- a/drivers/media/i2c/imx412.c
+++ b/drivers/media/i2c/imx412.c
@@ -925,7 +925,7 @@ static int imx412_parse_hw_config(struct imx412 *imx412)
 
 	/* Request optional reset pin */
 	imx412->reset_gpio = devm_gpiod_get_optional(imx412->dev, "reset",
-						     GPIOD_OUT_LOW);
+						     GPIOD_OUT_HIGH);
 	if (IS_ERR(imx412->reset_gpio)) {
 		dev_err(imx412->dev, "failed to get reset gpio %pe\n",
 			imx412->reset_gpio);
@@ -1037,7 +1037,11 @@ static int imx412_power_on(struct device *dev)
 		goto error_reset;
 	}
 
-	usleep_range(1000, 1200);
+	/*
+	 * Certain Arducam IMX577 module variants require a longer reset settle
+	 * time. Increasing the delay from 1ms to 10ms ensures reliable startup.
+	 */
+	usleep_range(10000, 12000);
 
 	return 0;
 
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index 5588cdd7ec20d..6047453170043 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -355,6 +355,7 @@ static void ir_work(struct work_struct *work)
 		mutex_unlock(&ir->lock);
 		if (rc == -ENODEV) {
 			rc_unregister_device(ir->rc);
+			rc_free_device(ir->rc);
 			ir->rc = NULL;
 			return;
 		}
@@ -972,6 +973,7 @@ static void ir_remove(struct i2c_client *client)
 	i2c_unregister_device(ir->tx_c);
 
 	rc_unregister_device(ir->rc);
+	rc_free_device(ir->rc);
 }
 
 static const struct i2c_device_id ir_kbd_id[] = {
diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c
index e6e214f8294b8..ac0712ce1e658 100644
--- a/drivers/media/i2c/max9286.c
+++ b/drivers/media/i2c/max9286.c
@@ -1205,7 +1205,7 @@ static int max9286_gpiochip_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct max9286_priv *priv = gpiochip_get_data(chip);
 
-	return priv->gpio_state & BIT(offset);
+	return !!(priv->gpio_state & BIT(offset));
 }
 
 static int max9286_register_gpio(struct max9286_priv *priv)
diff --git a/drivers/media/i2c/mt9m114.c b/drivers/media/i2c/mt9m114.c
index 16b0ace158137..e395e2d14e979 100644
--- a/drivers/media/i2c/mt9m114.c
+++ b/drivers/media/i2c/mt9m114.c
@@ -368,6 +368,10 @@
  * Data Structures
  */
 
+struct mt9m114_model_info {
+	bool state_standby_polling;
+};
+
 enum mt9m114_format_flag {
 	MT9M114_FMT_FLAG_PARALLEL = BIT(0),
 	MT9M114_FMT_FLAG_CSI2 = BIT(1),
@@ -417,6 +421,8 @@ struct mt9m114 {
 
 		struct v4l2_ctrl *tpg[4];
 	} ifp;
+
+	const struct mt9m114_model_info *info;
 };
 
 /* -----------------------------------------------------------------------------
@@ -2284,9 +2290,11 @@ static int mt9m114_power_on(struct mt9m114 *sensor)
 	 * reaches the standby mode (either initiated manually above in
 	 * parallel mode, or automatically after reset in MIPI mode).
 	 */
-	ret = mt9m114_poll_state(sensor, MT9M114_SYS_STATE_STANDBY);
-	if (ret < 0)
-		goto error_clock;
+	if (sensor->info->state_standby_polling) {
+		ret = mt9m114_poll_state(sensor, MT9M114_SYS_STATE_STANDBY);
+		if (ret < 0)
+			goto error_clock;
+	}
 
 	return 0;
 
@@ -2532,6 +2540,10 @@ static int mt9m114_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
+	sensor->info = device_get_match_data(dev);
+	if (!sensor->info)
+		return -ENODEV;
+
 	/* Acquire clocks, GPIOs and regulators. */
 	sensor->clk = devm_v4l2_sensor_clk_get(dev, NULL);
 	if (IS_ERR(sensor->clk)) {
@@ -2646,15 +2658,24 @@ static void mt9m114_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(dev);
 }
 
+static const struct mt9m114_model_info mt9m114_models_default = {
+	.state_standby_polling = true,
+};
+
+static const struct mt9m114_model_info mt9m114_models_aptina = {
+	.state_standby_polling = false,
+};
+
 static const struct of_device_id mt9m114_of_ids[] = {
-	{ .compatible = "onnn,mt9m114" },
-	{ /* sentinel */ },
+	{ .compatible = "onnn,mt9m114", .data = &mt9m114_models_default },
+	{ .compatible = "aptina,mi1040", .data = &mt9m114_models_aptina },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, mt9m114_of_ids);
 
 static const struct acpi_device_id mt9m114_acpi_ids[] = {
-	{ "INT33F0" },
-	{ /* sentinel */ },
+	{ "INT33F0", (kernel_ulong_t)&mt9m114_models_default },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(acpi, mt9m114_acpi_ids);
 
diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c
index 1500ee4db47ec..ea5d43d925ffa 100644
--- a/drivers/media/i2c/mt9p031.c
+++ b/drivers/media/i2c/mt9p031.c
@@ -1183,6 +1183,10 @@ static int mt9p031_probe(struct i2c_client *client)
 
 	mt9p031->reset = devm_gpiod_get_optional(&client->dev, "reset",
 						 GPIOD_OUT_HIGH);
+	if (IS_ERR(mt9p031->reset)) {
+		ret = PTR_ERR(mt9p031->reset);
+		goto done;
+	}
 
 	ret = mt9p031_clk_setup(mt9p031);
 	if (ret)
diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c
index c7184de6251ae..1675f0460969a 100644
--- a/drivers/media/i2c/og01a1b.c
+++ b/drivers/media/i2c/og01a1b.c
@@ -9,65 +9,53 @@
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
-#include <linux/unaligned.h>
-
+#include <media/v4l2-cci.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-fwnode.h>
 
-#define OG01A1B_REG_VALUE_08BIT		1
-#define OG01A1B_REG_VALUE_16BIT		2
-#define OG01A1B_REG_VALUE_24BIT		3
-
 #define OG01A1B_LINK_FREQ_500MHZ	500000000ULL
 #define OG01A1B_SCLK			120000000LL
 #define OG01A1B_MCLK			19200000
 #define OG01A1B_DATA_LANES		2
-#define OG01A1B_RGB_DEPTH		10
 
-#define OG01A1B_REG_CHIP_ID		0x300a
+#define OG01A1B_REG_CHIP_ID		CCI_REG24(0x300a)
 #define OG01A1B_CHIP_ID			0x470141
 
-#define OG01A1B_REG_MODE_SELECT		0x0100
+#define OG01A1B_REG_MODE_SELECT		CCI_REG8(0x0100)
 #define OG01A1B_MODE_STANDBY		0x00
 #define OG01A1B_MODE_STREAMING		0x01
 
 /* vertical-timings from sensor */
-#define OG01A1B_REG_VTS			0x380e
+#define OG01A1B_REG_VTS			CCI_REG16(0x380e)
 #define OG01A1B_VTS_120FPS		0x0498
 #define OG01A1B_VTS_120FPS_MIN		0x0498
 #define OG01A1B_VTS_MAX			0x7fff
 
 /* horizontal-timings from sensor */
-#define OG01A1B_REG_HTS			0x380c
+#define OG01A1B_REG_HTS			CCI_REG16(0x380c)
 
 /* Exposure controls from sensor */
-#define OG01A1B_REG_EXPOSURE		0x3501
+#define OG01A1B_REG_EXPOSURE		CCI_REG16(0x3501)
 #define	OG01A1B_EXPOSURE_MIN		1
 #define OG01A1B_EXPOSURE_MAX_MARGIN	14
 #define	OG01A1B_EXPOSURE_STEP		1
 
 /* Analog gain controls from sensor */
-#define OG01A1B_REG_ANALOG_GAIN		0x3508
+#define OG01A1B_REG_ANALOG_GAIN		CCI_REG16(0x3508)
 #define	OG01A1B_ANAL_GAIN_MIN		16
 #define	OG01A1B_ANAL_GAIN_MAX		248 /* Max = 15.5x */
 #define	OG01A1B_ANAL_GAIN_STEP		1
 
 /* Digital gain controls from sensor */
-#define OG01A1B_REG_DIG_GAIN		0x350a
+#define OG01A1B_REG_DIG_GAIN		CCI_REG24(0x350a)
 #define OG01A1B_DGTL_GAIN_MIN		1024
 #define OG01A1B_DGTL_GAIN_MAX		16384 /* Max = 16x */
 #define OG01A1B_DGTL_GAIN_STEP		1
 #define OG01A1B_DGTL_GAIN_DEFAULT	1024
 
-/* Group Access */
-#define OG01A1B_REG_GROUP_ACCESS	0x3208
-#define OG01A1B_GROUP_HOLD_START	0x0
-#define OG01A1B_GROUP_HOLD_END		0x10
-#define OG01A1B_GROUP_HOLD_LAUNCH	0xa0
-
 /* Test Pattern Control */
-#define OG01A1B_REG_TEST_PATTERN	0x5100
+#define OG01A1B_REG_TEST_PATTERN	CCI_REG8(0x5100)
 #define OG01A1B_TEST_PATTERN_ENABLE	BIT(7)
 #define OG01A1B_TEST_PATTERN_BAR_SHIFT	2
 
@@ -77,14 +65,9 @@ enum {
 	OG01A1B_LINK_FREQ_1000MBPS,
 };
 
-struct og01a1b_reg {
-	u16 address;
-	u8 val;
-};
-
 struct og01a1b_reg_list {
+	const struct cci_reg_sequence *regs;
 	u32 num_of_regs;
-	const struct og01a1b_reg *regs;
 };
 
 struct og01a1b_link_freq_config {
@@ -114,275 +97,279 @@ struct og01a1b_mode {
 	const struct og01a1b_reg_list reg_list;
 };
 
-static const struct og01a1b_reg mipi_data_rate_1000mbps[] = {
-	{0x0103, 0x01},
-	{0x0303, 0x02},
-	{0x0304, 0x00},
-	{0x0305, 0xd2},
-	{0x0323, 0x02},
-	{0x0324, 0x01},
-	{0x0325, 0x77},
+static const u32 og01a1b_mbus_formats[] = {
+	MEDIA_BUS_FMT_Y10_1X10,
+	MEDIA_BUS_FMT_Y8_1X8,
 };
 
-static const struct og01a1b_reg mode_1280x1024_regs[] = {
-	{0x0300, 0x0a},
-	{0x0301, 0x29},
-	{0x0302, 0x31},
-	{0x0303, 0x02},
-	{0x0304, 0x00},
-	{0x0305, 0xd2},
-	{0x0306, 0x00},
-	{0x0307, 0x01},
-	{0x0308, 0x02},
-	{0x0309, 0x00},
-	{0x0310, 0x00},
-	{0x0311, 0x00},
-	{0x0312, 0x07},
-	{0x0313, 0x00},
-	{0x0314, 0x00},
-	{0x0315, 0x00},
-	{0x0320, 0x02},
-	{0x0321, 0x01},
-	{0x0322, 0x01},
-	{0x0323, 0x02},
-	{0x0324, 0x01},
-	{0x0325, 0x77},
-	{0x0326, 0xce},
-	{0x0327, 0x04},
-	{0x0329, 0x02},
-	{0x032a, 0x04},
-	{0x032b, 0x04},
-	{0x032c, 0x02},
-	{0x032d, 0x01},
-	{0x032e, 0x00},
-	{0x300d, 0x02},
-	{0x300e, 0x04},
-	{0x3021, 0x08},
-	{0x301e, 0x03},
-	{0x3103, 0x00},
-	{0x3106, 0x08},
-	{0x3107, 0x40},
-	{0x3216, 0x01},
-	{0x3217, 0x00},
-	{0x3218, 0xc0},
-	{0x3219, 0x55},
-	{0x3500, 0x00},
-	{0x3501, 0x04},
-	{0x3502, 0x8a},
-	{0x3506, 0x01},
-	{0x3507, 0x72},
-	{0x3508, 0x01},
-	{0x3509, 0x00},
-	{0x350a, 0x01},
-	{0x350b, 0x00},
-	{0x350c, 0x00},
-	{0x3541, 0x00},
-	{0x3542, 0x40},
-	{0x3605, 0xe0},
-	{0x3606, 0x41},
-	{0x3614, 0x20},
-	{0x3620, 0x0b},
-	{0x3630, 0x07},
-	{0x3636, 0xa0},
-	{0x3637, 0xf9},
-	{0x3638, 0x09},
-	{0x3639, 0x38},
-	{0x363f, 0x09},
-	{0x3640, 0x17},
-	{0x3662, 0x04},
-	{0x3665, 0x80},
-	{0x3670, 0x68},
-	{0x3674, 0x00},
-	{0x3677, 0x3f},
-	{0x3679, 0x00},
-	{0x369f, 0x19},
-	{0x36a0, 0x03},
-	{0x36a2, 0x19},
-	{0x36a3, 0x03},
-	{0x370d, 0x66},
-	{0x370f, 0x00},
-	{0x3710, 0x03},
-	{0x3715, 0x03},
-	{0x3716, 0x03},
-	{0x3717, 0x06},
-	{0x3733, 0x00},
-	{0x3778, 0x00},
-	{0x37a8, 0x0f},
-	{0x37a9, 0x01},
-	{0x37aa, 0x07},
-	{0x37bd, 0x1c},
-	{0x37c1, 0x2f},
-	{0x37c3, 0x09},
-	{0x37c8, 0x1d},
-	{0x37ca, 0x30},
-	{0x37df, 0x00},
-	{0x3800, 0x00},
-	{0x3801, 0x00},
-	{0x3802, 0x00},
-	{0x3803, 0x00},
-	{0x3804, 0x05},
-	{0x3805, 0x0f},
-	{0x3806, 0x04},
-	{0x3807, 0x0f},
-	{0x3808, 0x05},
-	{0x3809, 0x00},
-	{0x380a, 0x04},
-	{0x380b, 0x00},
-	{0x380c, 0x03},
-	{0x380d, 0x50},
-	{0x380e, 0x04},
-	{0x380f, 0x98},
-	{0x3810, 0x00},
-	{0x3811, 0x08},
-	{0x3812, 0x00},
-	{0x3813, 0x08},
-	{0x3814, 0x11},
-	{0x3815, 0x11},
-	{0x3820, 0x40},
-	{0x3821, 0x04},
-	{0x3826, 0x00},
-	{0x3827, 0x00},
-	{0x382a, 0x08},
-	{0x382b, 0x52},
-	{0x382d, 0xba},
-	{0x383d, 0x14},
-	{0x384a, 0xa2},
-	{0x3866, 0x0e},
-	{0x3867, 0x07},
-	{0x3884, 0x00},
-	{0x3885, 0x08},
-	{0x3893, 0x68},
-	{0x3894, 0x2a},
-	{0x3898, 0x00},
-	{0x3899, 0x31},
-	{0x389a, 0x04},
-	{0x389b, 0x00},
-	{0x389c, 0x0b},
-	{0x389d, 0xad},
-	{0x389f, 0x08},
-	{0x38a0, 0x00},
-	{0x38a1, 0x00},
-	{0x38a8, 0x70},
-	{0x38ac, 0xea},
-	{0x38b2, 0x00},
-	{0x38b3, 0x08},
-	{0x38bc, 0x20},
-	{0x38c4, 0x0c},
-	{0x38c5, 0x3a},
-	{0x38c7, 0x3a},
-	{0x38e1, 0xc0},
-	{0x38ec, 0x3c},
-	{0x38f0, 0x09},
-	{0x38f1, 0x6f},
-	{0x38fe, 0x3c},
-	{0x391e, 0x00},
-	{0x391f, 0x00},
-	{0x3920, 0xa5},
-	{0x3921, 0x00},
-	{0x3922, 0x00},
-	{0x3923, 0x00},
-	{0x3924, 0x05},
-	{0x3925, 0x00},
-	{0x3926, 0x00},
-	{0x3927, 0x00},
-	{0x3928, 0x1a},
-	{0x3929, 0x01},
-	{0x392a, 0xb4},
-	{0x392b, 0x00},
-	{0x392c, 0x10},
-	{0x392f, 0x40},
-	{0x4000, 0xcf},
-	{0x4003, 0x40},
-	{0x4008, 0x00},
-	{0x4009, 0x07},
-	{0x400a, 0x02},
-	{0x400b, 0x54},
-	{0x400c, 0x00},
-	{0x400d, 0x07},
-	{0x4010, 0xc0},
-	{0x4012, 0x02},
-	{0x4014, 0x04},
-	{0x4015, 0x04},
-	{0x4017, 0x02},
-	{0x4042, 0x01},
-	{0x4306, 0x04},
-	{0x4307, 0x12},
-	{0x4509, 0x00},
-	{0x450b, 0x83},
-	{0x4604, 0x68},
-	{0x4608, 0x0a},
-	{0x4700, 0x06},
-	{0x4800, 0x64},
-	{0x481b, 0x3c},
-	{0x4825, 0x32},
-	{0x4833, 0x18},
-	{0x4837, 0x0f},
-	{0x4850, 0x40},
-	{0x4860, 0x00},
-	{0x4861, 0xec},
-	{0x4864, 0x00},
-	{0x4883, 0x00},
-	{0x4888, 0x90},
-	{0x4889, 0x05},
-	{0x488b, 0x04},
-	{0x4f00, 0x04},
-	{0x4f10, 0x04},
-	{0x4f21, 0x01},
-	{0x4f22, 0x40},
-	{0x4f23, 0x44},
-	{0x4f24, 0x51},
-	{0x4f25, 0x41},
-	{0x5000, 0x1f},
-	{0x500a, 0x00},
-	{0x5100, 0x00},
-	{0x5111, 0x20},
-	{0x3020, 0x20},
-	{0x3613, 0x03},
-	{0x38c9, 0x02},
-	{0x5304, 0x01},
-	{0x3620, 0x08},
-	{0x3639, 0x58},
-	{0x363a, 0x10},
-	{0x3674, 0x04},
-	{0x3780, 0xff},
-	{0x3781, 0xff},
-	{0x3782, 0x00},
-	{0x3783, 0x01},
-	{0x3798, 0xa3},
-	{0x37aa, 0x10},
-	{0x38a8, 0xf0},
-	{0x38c4, 0x09},
-	{0x38c5, 0xb0},
-	{0x38df, 0x80},
-	{0x38ff, 0x05},
-	{0x4010, 0xf1},
-	{0x4011, 0x70},
-	{0x3667, 0x80},
-	{0x4d00, 0x4a},
-	{0x4d01, 0x18},
-	{0x4d02, 0xbb},
-	{0x4d03, 0xde},
-	{0x4d04, 0x93},
-	{0x4d05, 0xff},
-	{0x4d09, 0x0a},
-	{0x37aa, 0x16},
-	{0x3606, 0x42},
-	{0x3605, 0x00},
-	{0x36a2, 0x17},
-	{0x300d, 0x0a},
-	{0x4d00, 0x4d},
-	{0x4d01, 0x95},
-	{0x3d8C, 0x70},
-	{0x3d8d, 0xE9},
-	{0x5300, 0x00},
-	{0x5301, 0x10},
-	{0x5302, 0x00},
-	{0x5303, 0xE3},
-	{0x3d88, 0x00},
-	{0x3d89, 0x10},
-	{0x3d8a, 0x00},
-	{0x3d8b, 0xE3},
-	{0x4f22, 0x00},
+static const struct cci_reg_sequence mipi_data_rate_1000mbps[] = {
+	{ CCI_REG8(0x0103), 0x01 },
+	{ CCI_REG8(0x0303), 0x02 },
+	{ CCI_REG8(0x0304), 0x00 },
+	{ CCI_REG8(0x0305), 0xd2 },
+	{ CCI_REG8(0x0323), 0x02 },
+	{ CCI_REG8(0x0324), 0x01 },
+	{ CCI_REG8(0x0325), 0x77 },
+};
+
+static const struct cci_reg_sequence mode_1280x1024_regs[] = {
+	{ CCI_REG8(0x0300), 0x0a },
+	{ CCI_REG8(0x0301), 0x29 },
+	{ CCI_REG8(0x0302), 0x31 },
+	{ CCI_REG8(0x0303), 0x02 },
+	{ CCI_REG8(0x0304), 0x00 },
+	{ CCI_REG8(0x0305), 0xd2 },
+	{ CCI_REG8(0x0306), 0x00 },
+	{ CCI_REG8(0x0307), 0x01 },
+	{ CCI_REG8(0x0308), 0x02 },
+	{ CCI_REG8(0x0309), 0x00 },
+	{ CCI_REG8(0x0310), 0x00 },
+	{ CCI_REG8(0x0311), 0x00 },
+	{ CCI_REG8(0x0312), 0x07 },
+	{ CCI_REG8(0x0313), 0x00 },
+	{ CCI_REG8(0x0314), 0x00 },
+	{ CCI_REG8(0x0315), 0x00 },
+	{ CCI_REG8(0x0320), 0x02 },
+	{ CCI_REG8(0x0321), 0x01 },
+	{ CCI_REG8(0x0322), 0x01 },
+	{ CCI_REG8(0x0323), 0x02 },
+	{ CCI_REG8(0x0324), 0x01 },
+	{ CCI_REG8(0x0325), 0x77 },
+	{ CCI_REG8(0x0326), 0xce },
+	{ CCI_REG8(0x0327), 0x04 },
+	{ CCI_REG8(0x0329), 0x02 },
+	{ CCI_REG8(0x032a), 0x04 },
+	{ CCI_REG8(0x032b), 0x04 },
+	{ CCI_REG8(0x032c), 0x02 },
+	{ CCI_REG8(0x032d), 0x01 },
+	{ CCI_REG8(0x032e), 0x00 },
+	{ CCI_REG8(0x300d), 0x02 },
+	{ CCI_REG8(0x300e), 0x04 },
+	{ CCI_REG8(0x3021), 0x08 },
+	{ CCI_REG8(0x301e), 0x03 },
+	{ CCI_REG8(0x3103), 0x00 },
+	{ CCI_REG8(0x3106), 0x08 },
+	{ CCI_REG8(0x3107), 0x40 },
+	{ CCI_REG8(0x3216), 0x01 },
+	{ CCI_REG8(0x3217), 0x00 },
+	{ CCI_REG8(0x3218), 0xc0 },
+	{ CCI_REG8(0x3219), 0x55 },
+	{ CCI_REG8(0x3500), 0x00 },
+	{ CCI_REG8(0x3501), 0x04 },
+	{ CCI_REG8(0x3502), 0x8a },
+	{ CCI_REG8(0x3506), 0x01 },
+	{ CCI_REG8(0x3507), 0x72 },
+	{ CCI_REG8(0x3508), 0x01 },
+	{ CCI_REG8(0x3509), 0x00 },
+	{ CCI_REG8(0x350a), 0x01 },
+	{ CCI_REG8(0x350b), 0x00 },
+	{ CCI_REG8(0x350c), 0x00 },
+	{ CCI_REG8(0x3541), 0x00 },
+	{ CCI_REG8(0x3542), 0x40 },
+	{ CCI_REG8(0x3605), 0xe0 },
+	{ CCI_REG8(0x3606), 0x41 },
+	{ CCI_REG8(0x3614), 0x20 },
+	{ CCI_REG8(0x3620), 0x0b },
+	{ CCI_REG8(0x3630), 0x07 },
+	{ CCI_REG8(0x3636), 0xa0 },
+	{ CCI_REG8(0x3637), 0xf9 },
+	{ CCI_REG8(0x3638), 0x09 },
+	{ CCI_REG8(0x3639), 0x38 },
+	{ CCI_REG8(0x363f), 0x09 },
+	{ CCI_REG8(0x3640), 0x17 },
+	{ CCI_REG8(0x3665), 0x80 },
+	{ CCI_REG8(0x3670), 0x68 },
+	{ CCI_REG8(0x3674), 0x00 },
+	{ CCI_REG8(0x3677), 0x3f },
+	{ CCI_REG8(0x3679), 0x00 },
+	{ CCI_REG8(0x369f), 0x19 },
+	{ CCI_REG8(0x36a0), 0x03 },
+	{ CCI_REG8(0x36a2), 0x19 },
+	{ CCI_REG8(0x36a3), 0x03 },
+	{ CCI_REG8(0x370d), 0x66 },
+	{ CCI_REG8(0x370f), 0x00 },
+	{ CCI_REG8(0x3710), 0x03 },
+	{ CCI_REG8(0x3715), 0x03 },
+	{ CCI_REG8(0x3716), 0x03 },
+	{ CCI_REG8(0x3717), 0x06 },
+	{ CCI_REG8(0x3733), 0x00 },
+	{ CCI_REG8(0x3778), 0x00 },
+	{ CCI_REG8(0x37a8), 0x0f },
+	{ CCI_REG8(0x37a9), 0x01 },
+	{ CCI_REG8(0x37aa), 0x07 },
+	{ CCI_REG8(0x37bd), 0x1c },
+	{ CCI_REG8(0x37c1), 0x2f },
+	{ CCI_REG8(0x37c3), 0x09 },
+	{ CCI_REG8(0x37c8), 0x1d },
+	{ CCI_REG8(0x37ca), 0x30 },
+	{ CCI_REG8(0x37df), 0x00 },
+	{ CCI_REG8(0x3800), 0x00 },
+	{ CCI_REG8(0x3801), 0x00 },
+	{ CCI_REG8(0x3802), 0x00 },
+	{ CCI_REG8(0x3803), 0x00 },
+	{ CCI_REG8(0x3804), 0x05 },
+	{ CCI_REG8(0x3805), 0x0f },
+	{ CCI_REG8(0x3806), 0x04 },
+	{ CCI_REG8(0x3807), 0x0f },
+	{ CCI_REG8(0x3808), 0x05 },
+	{ CCI_REG8(0x3809), 0x00 },
+	{ CCI_REG8(0x380a), 0x04 },
+	{ CCI_REG8(0x380b), 0x00 },
+	{ CCI_REG8(0x380c), 0x03 },
+	{ CCI_REG8(0x380d), 0x50 },
+	{ CCI_REG8(0x380e), 0x04 },
+	{ CCI_REG8(0x380f), 0x98 },
+	{ CCI_REG8(0x3810), 0x00 },
+	{ CCI_REG8(0x3811), 0x08 },
+	{ CCI_REG8(0x3812), 0x00 },
+	{ CCI_REG8(0x3813), 0x08 },
+	{ CCI_REG8(0x3814), 0x11 },
+	{ CCI_REG8(0x3815), 0x11 },
+	{ CCI_REG8(0x3820), 0x40 },
+	{ CCI_REG8(0x3821), 0x04 },
+	{ CCI_REG8(0x3826), 0x00 },
+	{ CCI_REG8(0x3827), 0x00 },
+	{ CCI_REG8(0x382a), 0x08 },
+	{ CCI_REG8(0x382b), 0x52 },
+	{ CCI_REG8(0x382d), 0xba },
+	{ CCI_REG8(0x383d), 0x14 },
+	{ CCI_REG8(0x384a), 0xa2 },
+	{ CCI_REG8(0x3866), 0x0e },
+	{ CCI_REG8(0x3867), 0x07 },
+	{ CCI_REG8(0x3884), 0x00 },
+	{ CCI_REG8(0x3885), 0x08 },
+	{ CCI_REG8(0x3893), 0x68 },
+	{ CCI_REG8(0x3894), 0x2a },
+	{ CCI_REG8(0x3898), 0x00 },
+	{ CCI_REG8(0x3899), 0x31 },
+	{ CCI_REG8(0x389a), 0x04 },
+	{ CCI_REG8(0x389b), 0x00 },
+	{ CCI_REG8(0x389c), 0x0b },
+	{ CCI_REG8(0x389d), 0xad },
+	{ CCI_REG8(0x389f), 0x08 },
+	{ CCI_REG8(0x38a0), 0x00 },
+	{ CCI_REG8(0x38a1), 0x00 },
+	{ CCI_REG8(0x38a8), 0x70 },
+	{ CCI_REG8(0x38ac), 0xea },
+	{ CCI_REG8(0x38b2), 0x00 },
+	{ CCI_REG8(0x38b3), 0x08 },
+	{ CCI_REG8(0x38bc), 0x20 },
+	{ CCI_REG8(0x38c4), 0x0c },
+	{ CCI_REG8(0x38c5), 0x3a },
+	{ CCI_REG8(0x38c7), 0x3a },
+	{ CCI_REG8(0x38e1), 0xc0 },
+	{ CCI_REG8(0x38ec), 0x3c },
+	{ CCI_REG8(0x38f0), 0x09 },
+	{ CCI_REG8(0x38f1), 0x6f },
+	{ CCI_REG8(0x38fe), 0x3c },
+	{ CCI_REG8(0x391e), 0x00 },
+	{ CCI_REG8(0x391f), 0x00 },
+	{ CCI_REG8(0x3920), 0xa5 },
+	{ CCI_REG8(0x3921), 0x00 },
+	{ CCI_REG8(0x3922), 0x00 },
+	{ CCI_REG8(0x3923), 0x00 },
+	{ CCI_REG8(0x3924), 0x05 },
+	{ CCI_REG8(0x3925), 0x00 },
+	{ CCI_REG8(0x3926), 0x00 },
+	{ CCI_REG8(0x3927), 0x00 },
+	{ CCI_REG8(0x3928), 0x1a },
+	{ CCI_REG8(0x3929), 0x01 },
+	{ CCI_REG8(0x392a), 0xb4 },
+	{ CCI_REG8(0x392b), 0x00 },
+	{ CCI_REG8(0x392c), 0x10 },
+	{ CCI_REG8(0x392f), 0x40 },
+	{ CCI_REG8(0x4000), 0xcf },
+	{ CCI_REG8(0x4003), 0x40 },
+	{ CCI_REG8(0x4008), 0x00 },
+	{ CCI_REG8(0x4009), 0x07 },
+	{ CCI_REG8(0x400a), 0x02 },
+	{ CCI_REG8(0x400b), 0x54 },
+	{ CCI_REG8(0x400c), 0x00 },
+	{ CCI_REG8(0x400d), 0x07 },
+	{ CCI_REG8(0x4010), 0xc0 },
+	{ CCI_REG8(0x4012), 0x02 },
+	{ CCI_REG8(0x4014), 0x04 },
+	{ CCI_REG8(0x4015), 0x04 },
+	{ CCI_REG8(0x4017), 0x02 },
+	{ CCI_REG8(0x4042), 0x01 },
+	{ CCI_REG8(0x4306), 0x04 },
+	{ CCI_REG8(0x4307), 0x12 },
+	{ CCI_REG8(0x4509), 0x00 },
+	{ CCI_REG8(0x450b), 0x83 },
+	{ CCI_REG8(0x4604), 0x68 },
+	{ CCI_REG8(0x4608), 0x0a },
+	{ CCI_REG8(0x4700), 0x06 },
+	{ CCI_REG8(0x4800), 0x64 },
+	{ CCI_REG8(0x481b), 0x3c },
+	{ CCI_REG8(0x4825), 0x32 },
+	{ CCI_REG8(0x4833), 0x18 },
+	{ CCI_REG8(0x4837), 0x0f },
+	{ CCI_REG8(0x4850), 0x40 },
+	{ CCI_REG8(0x4860), 0x00 },
+	{ CCI_REG8(0x4861), 0xec },
+	{ CCI_REG8(0x4864), 0x00 },
+	{ CCI_REG8(0x4883), 0x00 },
+	{ CCI_REG8(0x4888), 0x90 },
+	{ CCI_REG8(0x4889), 0x05 },
+	{ CCI_REG8(0x488b), 0x04 },
+	{ CCI_REG8(0x4f00), 0x04 },
+	{ CCI_REG8(0x4f10), 0x04 },
+	{ CCI_REG8(0x4f21), 0x01 },
+	{ CCI_REG8(0x4f22), 0x40 },
+	{ CCI_REG8(0x4f23), 0x44 },
+	{ CCI_REG8(0x4f24), 0x51 },
+	{ CCI_REG8(0x4f25), 0x41 },
+	{ CCI_REG8(0x5000), 0x1f },
+	{ CCI_REG8(0x500a), 0x00 },
+	{ CCI_REG8(0x5100), 0x00 },
+	{ CCI_REG8(0x5111), 0x20 },
+	{ CCI_REG8(0x3020), 0x20 },
+	{ CCI_REG8(0x3613), 0x03 },
+	{ CCI_REG8(0x38c9), 0x02 },
+	{ CCI_REG8(0x5304), 0x01 },
+	{ CCI_REG8(0x3620), 0x08 },
+	{ CCI_REG8(0x3639), 0x58 },
+	{ CCI_REG8(0x363a), 0x10 },
+	{ CCI_REG8(0x3674), 0x04 },
+	{ CCI_REG8(0x3780), 0xff },
+	{ CCI_REG8(0x3781), 0xff },
+	{ CCI_REG8(0x3782), 0x00 },
+	{ CCI_REG8(0x3783), 0x01 },
+	{ CCI_REG8(0x3798), 0xa3 },
+	{ CCI_REG8(0x37aa), 0x10 },
+	{ CCI_REG8(0x38a8), 0xf0 },
+	{ CCI_REG8(0x38c4), 0x09 },
+	{ CCI_REG8(0x38c5), 0xb0 },
+	{ CCI_REG8(0x38df), 0x80 },
+	{ CCI_REG8(0x38ff), 0x05 },
+	{ CCI_REG8(0x4010), 0xf1 },
+	{ CCI_REG8(0x4011), 0x70 },
+	{ CCI_REG8(0x3667), 0x80 },
+	{ CCI_REG8(0x4d00), 0x4a },
+	{ CCI_REG8(0x4d01), 0x18 },
+	{ CCI_REG8(0x4d02), 0xbb },
+	{ CCI_REG8(0x4d03), 0xde },
+	{ CCI_REG8(0x4d04), 0x93 },
+	{ CCI_REG8(0x4d05), 0xff },
+	{ CCI_REG8(0x4d09), 0x0a },
+	{ CCI_REG8(0x37aa), 0x16 },
+	{ CCI_REG8(0x3606), 0x42 },
+	{ CCI_REG8(0x3605), 0x00 },
+	{ CCI_REG8(0x36a2), 0x17 },
+	{ CCI_REG8(0x300d), 0x0a },
+	{ CCI_REG8(0x4d00), 0x4d },
+	{ CCI_REG8(0x4d01), 0x95 },
+	{ CCI_REG8(0x3d8c), 0x70 },
+	{ CCI_REG8(0x3d8d), 0xe9 },
+	{ CCI_REG8(0x5300), 0x00 },
+	{ CCI_REG8(0x5301), 0x10 },
+	{ CCI_REG8(0x5302), 0x00 },
+	{ CCI_REG8(0x5303), 0xe3 },
+	{ CCI_REG8(0x3d88), 0x00 },
+	{ CCI_REG8(0x3d89), 0x10 },
+	{ CCI_REG8(0x3d8a), 0x00 },
+	{ CCI_REG8(0x3d8b), 0xe3 },
+	{ CCI_REG8(0x4f22), 0x00 },
 };
 
 static const char * const og01a1b_test_pattern_menu[] = {
@@ -423,6 +410,7 @@ static const struct og01a1b_mode supported_modes[] = {
 
 struct og01a1b {
 	struct device *dev;
+	struct regmap *regmap;
 	struct clk *xvclk;
 	struct gpio_desc *reset_gpio;
 	struct regulator *avdd;
@@ -443,102 +431,36 @@ struct og01a1b {
 	/* Current mode */
 	const struct og01a1b_mode *cur_mode;
 
-	/* To serialize asynchronus callbacks */
-	struct mutex mutex;
+	/* Selected media bus format output */
+	u32 code;
 };
 
-static u64 to_pixel_rate(u32 f_index)
+static u64 to_pixel_rate(u32 f_index, u32 bpp)
 {
 	u64 pixel_rate = link_freq_menu_items[f_index] * 2 * OG01A1B_DATA_LANES;
 
-	do_div(pixel_rate, OG01A1B_RGB_DEPTH);
+	do_div(pixel_rate, bpp);
 
 	return pixel_rate;
 }
 
-static u64 to_pixels_per_line(u32 hts, u32 f_index)
+static u64 to_pixels_per_line(u32 hts, u32 f_index, u32 bpp)
 {
-	u64 ppl = hts * to_pixel_rate(f_index);
+	u64 ppl = hts * to_pixel_rate(f_index, bpp);
 
 	do_div(ppl, OG01A1B_SCLK);
 
 	return ppl;
 }
 
-static int og01a1b_read_reg(struct og01a1b *og01a1b, u16 reg, u16 len, u32 *val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&og01a1b->sd);
-	struct i2c_msg msgs[2];
-	u8 addr_buf[2];
-	u8 data_buf[4] = {0};
-	int ret;
-
-	if (len > 4)
-		return -EINVAL;
-
-	put_unaligned_be16(reg, addr_buf);
-	msgs[0].addr = client->addr;
-	msgs[0].flags = 0;
-	msgs[0].len = sizeof(addr_buf);
-	msgs[0].buf = addr_buf;
-	msgs[1].addr = client->addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = len;
-	msgs[1].buf = &data_buf[4 - len];
-
-	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
-	if (ret != ARRAY_SIZE(msgs))
-		return -EIO;
-
-	*val = get_unaligned_be32(data_buf);
-
-	return 0;
-}
-
-static int og01a1b_write_reg(struct og01a1b *og01a1b, u16 reg, u16 len, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&og01a1b->sd);
-	u8 buf[6];
-
-	if (len > 4)
-		return -EINVAL;
-
-	put_unaligned_be16(reg, buf);
-	put_unaligned_be32(val << 8 * (4 - len), buf + 2);
-	if (i2c_master_send(client, buf, len + 2) != len + 2)
-		return -EIO;
-
-	return 0;
-}
-
-static int og01a1b_write_reg_list(struct og01a1b *og01a1b,
-				  const struct og01a1b_reg_list *r_list)
-{
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < r_list->num_of_regs; i++) {
-		ret = og01a1b_write_reg(og01a1b, r_list->regs[i].address, 1,
-					r_list->regs[i].val);
-		if (ret) {
-			dev_err_ratelimited(og01a1b->dev,
-					    "failed to write reg 0x%4.4x. error = %d",
-					    r_list->regs[i].address, ret);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
 static int og01a1b_test_pattern(struct og01a1b *og01a1b, u32 pattern)
 {
 	if (pattern)
 		pattern = (pattern - 1) << OG01A1B_TEST_PATTERN_BAR_SHIFT |
 			  OG01A1B_TEST_PATTERN_ENABLE;
 
-	return og01a1b_write_reg(og01a1b, OG01A1B_REG_TEST_PATTERN,
-				 OG01A1B_REG_VALUE_08BIT, pattern);
+	return cci_write(og01a1b->regmap, OG01A1B_REG_TEST_PATTERN,
+			 pattern, NULL);
 }
 
 static int og01a1b_set_ctrl(struct v4l2_ctrl *ctrl)
@@ -565,26 +487,23 @@ static int og01a1b_set_ctrl(struct v4l2_ctrl *ctrl)
 
 	switch (ctrl->id) {
 	case V4L2_CID_ANALOGUE_GAIN:
-		ret = og01a1b_write_reg(og01a1b, OG01A1B_REG_ANALOG_GAIN,
-					OG01A1B_REG_VALUE_16BIT,
-					ctrl->val << 4);
+		ret = cci_write(og01a1b->regmap, OG01A1B_REG_ANALOG_GAIN,
+				ctrl->val << 4, NULL);
 		break;
 
 	case V4L2_CID_DIGITAL_GAIN:
-		ret = og01a1b_write_reg(og01a1b, OG01A1B_REG_DIG_GAIN,
-					OG01A1B_REG_VALUE_24BIT,
-					ctrl->val << 6);
+		ret = cci_write(og01a1b->regmap, OG01A1B_REG_DIG_GAIN,
+				ctrl->val << 6, NULL);
 		break;
 
 	case V4L2_CID_EXPOSURE:
-		ret = og01a1b_write_reg(og01a1b, OG01A1B_REG_EXPOSURE,
-					OG01A1B_REG_VALUE_16BIT, ctrl->val);
+		ret = cci_write(og01a1b->regmap, OG01A1B_REG_EXPOSURE,
+				ctrl->val, NULL);
 		break;
 
 	case V4L2_CID_VBLANK:
-		ret = og01a1b_write_reg(og01a1b, OG01A1B_REG_VTS,
-					OG01A1B_REG_VALUE_16BIT,
-					og01a1b->cur_mode->height + ctrl->val);
+		ret = cci_write(og01a1b->regmap, OG01A1B_REG_VTS,
+				og01a1b->cur_mode->height + ctrl->val, NULL);
 		break;
 
 	case V4L2_CID_TEST_PATTERN:
@@ -609,6 +528,7 @@ static int og01a1b_init_controls(struct og01a1b *og01a1b)
 {
 	struct v4l2_ctrl_handler *ctrl_hdlr;
 	s64 exposure_max, h_blank;
+	u32 bpp;
 	int ret;
 
 	ctrl_hdlr = &og01a1b->ctrl_handler;
@@ -616,7 +536,6 @@ static int og01a1b_init_controls(struct og01a1b *og01a1b)
 	if (ret)
 		return ret;
 
-	ctrl_hdlr->lock = &og01a1b->mutex;
 	og01a1b->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr,
 						    &og01a1b_ctrl_ops,
 						    V4L2_CID_LINK_FREQ,
@@ -626,13 +545,12 @@ static int og01a1b_init_controls(struct og01a1b *og01a1b)
 	if (og01a1b->link_freq)
 		og01a1b->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
+	bpp = (og01a1b->code == MEDIA_BUS_FMT_Y10_1X10 ? 10 : 8);
 	og01a1b->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &og01a1b_ctrl_ops,
-						V4L2_CID_PIXEL_RATE, 0,
-						to_pixel_rate
-						(OG01A1B_LINK_FREQ_1000MBPS),
-						1,
-						to_pixel_rate
-						(OG01A1B_LINK_FREQ_1000MBPS));
+				V4L2_CID_PIXEL_RATE, 0,
+				to_pixel_rate(OG01A1B_LINK_FREQ_1000MBPS, bpp),
+				1,
+				to_pixel_rate(OG01A1B_LINK_FREQ_1000MBPS, bpp));
 	og01a1b->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &og01a1b_ctrl_ops,
 					    V4L2_CID_VBLANK,
 					    og01a1b->cur_mode->vts_min -
@@ -642,7 +560,7 @@ static int og01a1b_init_controls(struct og01a1b *og01a1b)
 					    og01a1b->cur_mode->vts_def -
 					    og01a1b->cur_mode->height);
 	h_blank = to_pixels_per_line(og01a1b->cur_mode->hts,
-				     og01a1b->cur_mode->link_freq_index) -
+				     og01a1b->cur_mode->link_freq_index, bpp) -
 				     og01a1b->cur_mode->width;
 	og01a1b->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &og01a1b_ctrl_ops,
 					    V4L2_CID_HBLANK, h_blank, h_blank,
@@ -682,78 +600,78 @@ static void og01a1b_update_pad_format(const struct og01a1b_mode *mode,
 {
 	fmt->width = mode->width;
 	fmt->height = mode->height;
-	fmt->code = MEDIA_BUS_FMT_Y10_1X10;
 	fmt->field = V4L2_FIELD_NONE;
 }
 
-static int og01a1b_start_streaming(struct og01a1b *og01a1b)
+static int og01a1b_enable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
 {
+	struct og01a1b *og01a1b = to_og01a1b(sd);
+	unsigned int link_freq_index = og01a1b->cur_mode->link_freq_index;
 	const struct og01a1b_reg_list *reg_list;
-	int link_freq_index, ret;
+	int ret;
 
-	link_freq_index = og01a1b->cur_mode->link_freq_index;
-	reg_list = &link_freq_configs[link_freq_index].reg_list;
+	ret = pm_runtime_resume_and_get(og01a1b->dev);
+	if (ret)
+		return ret;
 
-	ret = og01a1b_write_reg_list(og01a1b, reg_list);
+	reg_list = &link_freq_configs[link_freq_index].reg_list;
+	ret = cci_multi_reg_write(og01a1b->regmap, reg_list->regs,
+				  reg_list->num_of_regs, NULL);
 	if (ret) {
-		dev_err(og01a1b->dev, "failed to set plls");
-		return ret;
+		dev_err(og01a1b->dev, "failed to set plls: %d\n", ret);
+		goto error;
 	}
 
 	reg_list = &og01a1b->cur_mode->reg_list;
-	ret = og01a1b_write_reg_list(og01a1b, reg_list);
+	ret = cci_multi_reg_write(og01a1b->regmap, reg_list->regs,
+				  reg_list->num_of_regs, NULL);
 	if (ret) {
-		dev_err(og01a1b->dev, "failed to set mode");
-		return ret;
+		dev_err(og01a1b->dev, "failed to set mode: %d\n", ret);
+		goto error;
+	}
+
+	ret = cci_write(og01a1b->regmap, CCI_REG8(0x3662),
+			(og01a1b->code == MEDIA_BUS_FMT_Y10_1X10 ? 0x4 : 0x6),
+			NULL);
+	if (ret) {
+		dev_err(og01a1b->dev, "failed to set output format: %d\n", ret);
+		goto error;
 	}
 
 	ret = __v4l2_ctrl_handler_setup(og01a1b->sd.ctrl_handler);
 	if (ret)
-		return ret;
+		goto error;
 
-	ret = og01a1b_write_reg(og01a1b, OG01A1B_REG_MODE_SELECT,
-				OG01A1B_REG_VALUE_08BIT,
-				OG01A1B_MODE_STREAMING);
+	ret = cci_write(og01a1b->regmap, OG01A1B_REG_MODE_SELECT,
+			OG01A1B_MODE_STREAMING, NULL);
 	if (ret) {
-		dev_err(og01a1b->dev, "failed to set stream");
-		return ret;
+		dev_err(og01a1b->dev, "failed to start streaming: %d\n", ret);
+		goto error;
 	}
 
 	return 0;
-}
 
-static void og01a1b_stop_streaming(struct og01a1b *og01a1b)
-{
-	if (og01a1b_write_reg(og01a1b, OG01A1B_REG_MODE_SELECT,
-			      OG01A1B_REG_VALUE_08BIT, OG01A1B_MODE_STANDBY))
-		dev_err(og01a1b->dev, "failed to set stream");
+error:
+	pm_runtime_put_autosuspend(og01a1b->dev);
+
+	return ret;
 }
 
-static int og01a1b_set_stream(struct v4l2_subdev *sd, int enable)
+static int og01a1b_disable_streams(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_state *state, u32 pad,
+				   u64 streams_mask)
 {
 	struct og01a1b *og01a1b = to_og01a1b(sd);
-	int ret = 0;
-
-	mutex_lock(&og01a1b->mutex);
-	if (enable) {
-		ret = pm_runtime_resume_and_get(og01a1b->dev);
-		if (ret) {
-			mutex_unlock(&og01a1b->mutex);
-			return ret;
-		}
+	int ret;
 
-		ret = og01a1b_start_streaming(og01a1b);
-		if (ret) {
-			enable = 0;
-			og01a1b_stop_streaming(og01a1b);
-			pm_runtime_put(og01a1b->dev);
-		}
-	} else {
-		og01a1b_stop_streaming(og01a1b);
-		pm_runtime_put(og01a1b->dev);
-	}
+	ret = cci_write(og01a1b->regmap, OG01A1B_REG_MODE_SELECT,
+			OG01A1B_MODE_STANDBY, NULL);
+	if (ret)
+		dev_err(og01a1b->dev, "failed to stop streaming: %d\n", ret);
 
-	mutex_unlock(&og01a1b->mutex);
+	pm_runtime_put_autosuspend(og01a1b->dev);
 
 	return ret;
 }
@@ -764,55 +682,38 @@ static int og01a1b_set_format(struct v4l2_subdev *sd,
 {
 	struct og01a1b *og01a1b = to_og01a1b(sd);
 	const struct og01a1b_mode *mode;
-	s32 vblank_def, h_blank;
+	s32 vblank_def, h_blank, bpp;
 
 	mode = v4l2_find_nearest_size(supported_modes,
 				      ARRAY_SIZE(supported_modes), width,
 				      height, fmt->format.width,
 				      fmt->format.height);
 
-	mutex_lock(&og01a1b->mutex);
 	og01a1b_update_pad_format(mode, &fmt->format);
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		*v4l2_subdev_state_get_format(sd_state, fmt->pad) = fmt->format;
-	} else {
-		og01a1b->cur_mode = mode;
-		__v4l2_ctrl_s_ctrl(og01a1b->link_freq, mode->link_freq_index);
-		__v4l2_ctrl_s_ctrl_int64(og01a1b->pixel_rate,
-					 to_pixel_rate(mode->link_freq_index));
-
-		/* Update limits and set FPS to default */
-		vblank_def = mode->vts_def - mode->height;
-		__v4l2_ctrl_modify_range(og01a1b->vblank,
-					 mode->vts_min - mode->height,
-					 OG01A1B_VTS_MAX - mode->height, 1,
-					 vblank_def);
-		__v4l2_ctrl_s_ctrl(og01a1b->vblank, vblank_def);
-		h_blank = to_pixels_per_line(mode->hts, mode->link_freq_index) -
-			  mode->width;
-		__v4l2_ctrl_modify_range(og01a1b->hblank, h_blank, h_blank, 1,
-					 h_blank);
-	}
 
-	mutex_unlock(&og01a1b->mutex);
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		goto set_format;
 
-	return 0;
-}
+	bpp = (fmt->format.code == MEDIA_BUS_FMT_Y10_1X10 ? 10 : 8);
+	__v4l2_ctrl_s_ctrl(og01a1b->link_freq, mode->link_freq_index);
+	__v4l2_ctrl_s_ctrl_int64(og01a1b->pixel_rate,
+				 to_pixel_rate(mode->link_freq_index, bpp));
 
-static int og01a1b_get_format(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_state *sd_state,
-			      struct v4l2_subdev_format *fmt)
-{
-	struct og01a1b *og01a1b = to_og01a1b(sd);
+	/* Update limits and set FPS to default */
+	vblank_def = mode->vts_def - mode->height;
+	__v4l2_ctrl_modify_range(og01a1b->vblank,
+				 mode->vts_min - mode->height,
+				 OG01A1B_VTS_MAX - mode->height, 1, vblank_def);
+	__v4l2_ctrl_s_ctrl(og01a1b->vblank, vblank_def);
+	h_blank = to_pixels_per_line(mode->hts, mode->link_freq_index,
+				     bpp) - mode->width;
+	__v4l2_ctrl_modify_range(og01a1b->hblank, h_blank, h_blank, 1, h_blank);
 
-	mutex_lock(&og01a1b->mutex);
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
-		fmt->format = *v4l2_subdev_state_get_format(sd_state,
-							    fmt->pad);
-	else
-		og01a1b_update_pad_format(og01a1b->cur_mode, &fmt->format);
+	og01a1b->cur_mode = mode;
+	og01a1b->code = fmt->format.code;
 
-	mutex_unlock(&og01a1b->mutex);
+set_format:
+	*v4l2_subdev_state_get_format(sd_state, fmt->pad) = fmt->format;
 
 	return 0;
 }
@@ -821,10 +722,10 @@ static int og01a1b_enum_mbus_code(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_mbus_code_enum *code)
 {
-	if (code->index > 0)
+	if (code->index > ARRAY_SIZE(og01a1b_mbus_formats) - 1)
 		return -EINVAL;
 
-	code->code = MEDIA_BUS_FMT_Y10_1X10;
+	code->code = og01a1b_mbus_formats[code->index];
 
 	return 0;
 }
@@ -836,7 +737,8 @@ static int og01a1b_enum_frame_size(struct v4l2_subdev *sd,
 	if (fse->index >= ARRAY_SIZE(supported_modes))
 		return -EINVAL;
 
-	if (fse->code != MEDIA_BUS_FMT_Y10_1X10)
+	if (fse->code != MEDIA_BUS_FMT_Y10_1X10 &&
+	    fse->code != MEDIA_BUS_FMT_Y8_1X8)
 		return -EINVAL;
 
 	fse->min_width = supported_modes[fse->index].width;
@@ -847,27 +749,36 @@ static int og01a1b_enum_frame_size(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int og01a1b_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+static int og01a1b_init_state(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_state *state)
 {
 	struct og01a1b *og01a1b = to_og01a1b(sd);
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+		.pad = 0,
+		.format = {
+			.width = og01a1b->cur_mode->width,
+			.height = og01a1b->cur_mode->height,
+			.code = og01a1b->code,
+		},
+	};
 
-	mutex_lock(&og01a1b->mutex);
-	og01a1b_update_pad_format(&supported_modes[0],
-				  v4l2_subdev_state_get_format(fh->state, 0));
-	mutex_unlock(&og01a1b->mutex);
+	og01a1b_set_format(sd, state, &fmt);
 
 	return 0;
 }
 
 static const struct v4l2_subdev_video_ops og01a1b_video_ops = {
-	.s_stream = og01a1b_set_stream,
+	.s_stream = v4l2_subdev_s_stream_helper,
 };
 
 static const struct v4l2_subdev_pad_ops og01a1b_pad_ops = {
 	.set_fmt = og01a1b_set_format,
-	.get_fmt = og01a1b_get_format,
+	.get_fmt = v4l2_subdev_get_fmt,
 	.enum_mbus_code = og01a1b_enum_mbus_code,
 	.enum_frame_size = og01a1b_enum_frame_size,
+	.enable_streams = og01a1b_enable_streams,
+	.disable_streams = og01a1b_disable_streams,
 };
 
 static const struct v4l2_subdev_ops og01a1b_subdev_ops = {
@@ -880,21 +791,20 @@ static const struct media_entity_operations og01a1b_subdev_entity_ops = {
 };
 
 static const struct v4l2_subdev_internal_ops og01a1b_internal_ops = {
-	.open = og01a1b_open,
+	.init_state = og01a1b_init_state,
 };
 
 static int og01a1b_identify_module(struct og01a1b *og01a1b)
 {
 	int ret;
-	u32 val;
+	u64 val;
 
-	ret = og01a1b_read_reg(og01a1b, OG01A1B_REG_CHIP_ID,
-			       OG01A1B_REG_VALUE_24BIT, &val);
+	ret = cci_read(og01a1b->regmap, OG01A1B_REG_CHIP_ID, &val, NULL);
 	if (ret)
 		return ret;
 
 	if (val != OG01A1B_CHIP_ID) {
-		dev_err(og01a1b->dev, "chip id mismatch: %x!=%x",
+		dev_err(og01a1b->dev, "chip id mismatch: %x!=%llx",
 			OG01A1B_CHIP_ID, val);
 		return -ENXIO;
 	}
@@ -1042,10 +952,10 @@ static void og01a1b_remove(struct i2c_client *client)
 	struct og01a1b *og01a1b = to_og01a1b(sd);
 
 	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(&og01a1b->sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(og01a1b->dev);
-	mutex_destroy(&og01a1b->mutex);
 }
 
 static int og01a1b_probe(struct i2c_client *client)
@@ -1062,6 +972,11 @@ static int og01a1b_probe(struct i2c_client *client)
 
 	v4l2_i2c_subdev_init(&og01a1b->sd, client, &og01a1b_subdev_ops);
 
+	og01a1b->regmap = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(og01a1b->regmap))
+		return dev_err_probe(og01a1b->dev, PTR_ERR(og01a1b->regmap),
+				     "failed to init CCI\n");
+
 	og01a1b->xvclk = devm_v4l2_sensor_clk_get(og01a1b->dev, NULL);
 	if (IS_ERR(og01a1b->xvclk))
 		return dev_err_probe(og01a1b->dev, PTR_ERR(og01a1b->xvclk),
@@ -1134,14 +1049,15 @@ static int og01a1b_probe(struct i2c_client *client)
 		goto power_off;
 	}
 
-	mutex_init(&og01a1b->mutex);
 	og01a1b->cur_mode = &supported_modes[0];
+	og01a1b->code = og01a1b_mbus_formats[0];
 	ret = og01a1b_init_controls(og01a1b);
 	if (ret) {
 		dev_err(og01a1b->dev, "failed to init controls: %d", ret);
 		goto probe_error_v4l2_ctrl_handler_free;
 	}
 
+	og01a1b->sd.state_lock = og01a1b->ctrl_handler.lock;
 	og01a1b->sd.internal_ops = &og01a1b_internal_ops;
 	og01a1b->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	og01a1b->sd.entity.ops = &og01a1b_subdev_entity_ops;
@@ -1153,11 +1069,18 @@ static int og01a1b_probe(struct i2c_client *client)
 		goto probe_error_v4l2_ctrl_handler_free;
 	}
 
+	ret = v4l2_subdev_init_finalize(&og01a1b->sd);
+	if (ret < 0) {
+		dev_err_probe(og01a1b->dev, ret,
+			      "failed to finalize subdevice init\n");
+		goto probe_error_media_entity_cleanup;
+	}
+
 	ret = v4l2_async_register_subdev_sensor(&og01a1b->sd);
 	if (ret < 0) {
 		dev_err(og01a1b->dev, "failed to register V4L2 subdev: %d",
 			ret);
-		goto probe_error_media_entity_cleanup;
+		goto probe_error_v4l2_subdev_cleanup;
 	}
 
 	/* Enable runtime PM and turn off the device */
@@ -1167,12 +1090,14 @@ static int og01a1b_probe(struct i2c_client *client)
 
 	return 0;
 
+probe_error_v4l2_subdev_cleanup:
+	v4l2_subdev_cleanup(&og01a1b->sd);
+
 probe_error_media_entity_cleanup:
 	media_entity_cleanup(&og01a1b->sd.entity);
 
 probe_error_v4l2_ctrl_handler_free:
 	v4l2_ctrl_handler_free(og01a1b->sd.ctrl_handler);
-	mutex_destroy(&og01a1b->mutex);
 
 power_off:
 	og01a1b_power_off(og01a1b->dev);
diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c
index 6a46ef7233ac3..db9bd2892140d 100644
--- a/drivers/media/i2c/ov5647.c
+++ b/drivers/media/i2c/ov5647.c
@@ -967,21 +967,21 @@ static int ov5647_s_ctrl(struct v4l2_ctrl *ctrl)
 		break;
 	case V4L2_CID_AUTOGAIN:
 		/* Non-zero turns on AGC by clearing bit 1.*/
-		return cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(1),
-				       ctrl->val ? 0 : BIT(1), NULL);
+		ret = cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(1),
+				      ctrl->val ? 0 : BIT(1), NULL);
 		break;
 	case V4L2_CID_EXPOSURE_AUTO:
 		/*
 		 * Everything except V4L2_EXPOSURE_MANUAL turns on AEC by
 		 * clearing bit 0.
 		 */
-		return cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(0),
-				       ctrl->val == V4L2_EXPOSURE_MANUAL ? BIT(0) : 0, NULL);
+		ret = cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(0),
+				      ctrl->val == V4L2_EXPOSURE_MANUAL ? BIT(0) : 0, NULL);
 		break;
 	case V4L2_CID_ANALOGUE_GAIN:
 		/* 10 bits of gain, 2 in the high register. */
-		return cci_write(sensor->regmap, OV5647_REG_GAIN,
-				 ctrl->val & 0x3ff, NULL);
+		ret = cci_write(sensor->regmap, OV5647_REG_GAIN,
+				ctrl->val & 0x3ff, NULL);
 		break;
 	case V4L2_CID_EXPOSURE:
 		/*
diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c
index ded9b2044ff8f..2167fb73ea415 100644
--- a/drivers/media/i2c/ov9282.c
+++ b/drivers/media/i2c/ov9282.c
@@ -12,38 +12,40 @@
 #include <linux/math.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 
+#include <media/v4l2-cci.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-subdev.h>
 
 /* Streaming Mode */
-#define OV9282_REG_MODE_SELECT	0x0100
+#define OV9282_REG_MODE_SELECT	CCI_REG8(0x0100)
 #define OV9282_MODE_STANDBY	0x00
 #define OV9282_MODE_STREAMING	0x01
 
-#define OV9282_REG_PLL_CTRL_0D	0x030d
+#define OV9282_REG_PLL_CTRL_0D	CCI_REG8(0x030d)
 #define OV9282_PLL_CTRL_0D_RAW8		0x60
 #define OV9282_PLL_CTRL_0D_RAW10	0x50
 
-#define OV9282_REG_TIMING_HTS	0x380c
+#define OV9282_REG_TIMING_HTS	CCI_REG16(0x380c)
 #define OV9282_TIMING_HTS_MAX	0x7fff
 
 /* Lines per frame */
-#define OV9282_REG_LPFR		0x380e
+#define OV9282_REG_LPFR		CCI_REG16(0x380e)
 
 /* Chip ID */
-#define OV9282_REG_ID		0x300a
+#define OV9282_REG_ID		CCI_REG16(0x300a)
 #define OV9282_ID		0x9281
 
 /* Output enable registers */
-#define OV9282_REG_OUTPUT_ENABLE4	0x3004
+#define OV9282_REG_OUTPUT_ENABLE4	CCI_REG8(0x3004)
 #define OV9282_OUTPUT_ENABLE4_GPIO2	BIT(1)
 #define OV9282_OUTPUT_ENABLE4_D9	BIT(0)
 
-#define OV9282_REG_OUTPUT_ENABLE5	0x3005
+#define OV9282_REG_OUTPUT_ENABLE5	CCI_REG8(0x3005)
 #define OV9282_OUTPUT_ENABLE5_D8	BIT(7)
 #define OV9282_OUTPUT_ENABLE5_D7	BIT(6)
 #define OV9282_OUTPUT_ENABLE5_D6	BIT(5)
@@ -53,7 +55,7 @@
 #define OV9282_OUTPUT_ENABLE5_D2	BIT(1)
 #define OV9282_OUTPUT_ENABLE5_D1	BIT(0)
 
-#define OV9282_REG_OUTPUT_ENABLE6	0x3006
+#define OV9282_REG_OUTPUT_ENABLE6	CCI_REG8(0x3006)
 #define OV9282_OUTPUT_ENABLE6_D0	BIT(7)
 #define OV9282_OUTPUT_ENABLE6_PCLK	BIT(6)
 #define OV9282_OUTPUT_ENABLE6_HREF	BIT(5)
@@ -62,14 +64,14 @@
 #define OV9282_OUTPUT_ENABLE6_VSYNC	BIT(1)
 
 /* Exposure control */
-#define OV9282_REG_EXPOSURE	0x3500
+#define OV9282_REG_EXPOSURE	CCI_REG24(0x3500)
 #define OV9282_EXPOSURE_MIN	1
 #define OV9282_EXPOSURE_OFFSET	25
 #define OV9282_EXPOSURE_STEP	1
 #define OV9282_EXPOSURE_DEFAULT	0x0282
 
 /* AEC/AGC manual */
-#define OV9282_REG_AEC_MANUAL		0x3503
+#define OV9282_REG_AEC_MANUAL		CCI_REG8(0x3503)
 #define OV9282_DIGFRAC_GAIN_DELAY	BIT(6)
 #define OV9282_GAIN_CHANGE_DELAY	BIT(5)
 #define OV9282_GAIN_DELAY		BIT(4)
@@ -78,28 +80,28 @@
 #define OV9282_AEC_MANUAL_DEFAULT	0x00
 
 /* Analog gain control */
-#define OV9282_REG_AGAIN	0x3509
+#define OV9282_REG_AGAIN	CCI_REG8(0x3509)
 #define OV9282_AGAIN_MIN	0x10
 #define OV9282_AGAIN_MAX	0xff
 #define OV9282_AGAIN_STEP	1
 #define OV9282_AGAIN_DEFAULT	0x10
 
 /* Group hold register */
-#define OV9282_REG_HOLD		0x3308
+#define OV9282_REG_HOLD		CCI_REG8(0x3308)
 
-#define OV9282_REG_ANA_CORE_2	0x3662
+#define OV9282_REG_ANA_CORE_2	CCI_REG8(0x3662)
 #define OV9282_ANA_CORE2_RAW8	0x07
 #define OV9282_ANA_CORE2_RAW10	0x05
 
-#define OV9282_REG_TIMING_FORMAT_1	0x3820
-#define OV9282_REG_TIMING_FORMAT_2	0x3821
+#define OV9282_REG_TIMING_FORMAT_1	CCI_REG8(0x3820)
+#define OV9282_REG_TIMING_FORMAT_2	CCI_REG8(0x3821)
 #define OV9282_FLIP_BIT			BIT(2)
 
-#define OV9282_REG_MIPI_CTRL00	0x4800
+#define OV9282_REG_MIPI_CTRL00	CCI_REG8(0x4800)
 #define OV9282_GATED_CLOCK	BIT(5)
 
 /* Flash/Strobe control registers */
-#define OV9282_REG_STROBE_FRAME_SPAN		0x3925
+#define OV9282_REG_STROBE_FRAME_SPAN		CCI_REG32(0x3925)
 #define OV9282_STROBE_FRAME_SPAN_DEFAULT	0x0000001a
 
 /* Input clock rate */
@@ -139,16 +141,6 @@ static const char * const ov9282_supply_names[] = {
 
 #define OV9282_NUM_SUPPLIES ARRAY_SIZE(ov9282_supply_names)
 
-/**
- * struct ov9282_reg - ov9282 sensor register
- * @address: Register address
- * @val: Register value
- */
-struct ov9282_reg {
-	u16 address;
-	u8 val;
-};
-
 /**
  * struct ov9282_reg_list - ov9282 sensor register list
  * @num_of_regs: Number of registers in the list
@@ -156,7 +148,7 @@ struct ov9282_reg {
  */
 struct ov9282_reg_list {
 	u32 num_of_regs;
-	const struct ov9282_reg *regs;
+	const struct cci_reg_sequence *regs;
 };
 
 /**
@@ -188,6 +180,7 @@ struct ov9282_mode {
  * struct ov9282 - ov9282 sensor device structure
  * @dev: Pointer to generic device
  * @sd: V4L2 sub-device
+ * @regmap: Regmap for sensor register access
  * @pad: Media pad. Only one pad supported
  * @reset_gpio: Sensor reset gpio
  * @inclk: Sensor input clock
@@ -209,6 +202,7 @@ struct ov9282_mode {
 struct ov9282 {
 	struct device *dev;
 	struct v4l2_subdev sd;
+	struct regmap *regmap;
 	struct media_pad pad;
 	struct gpio_desc *reset_gpio;
 	struct clk *inclk;
@@ -227,7 +221,6 @@ struct ov9282 {
 	bool noncontinuous_clock;
 	const struct ov9282_mode *cur_mode;
 	u32 code;
-	struct mutex mutex;
 };
 
 static const s64 link_freq[] = {
@@ -241,73 +234,68 @@ static const s64 link_freq[] = {
  * register arrays as some settings are written as part of ov9282_power_on,
  * and the reset will clear them.
  */
-static const struct ov9282_reg common_regs[] = {
-	{0x0302, 0x32},
-	{0x030e, 0x02},
-	{0x3001, 0x00},
+static const struct cci_reg_sequence common_regs[] = {
+	{CCI_REG8(0x0302), 0x32},
+	{CCI_REG8(0x030e), 0x02},
+	{CCI_REG8(0x3001), 0x00},
 	{OV9282_REG_OUTPUT_ENABLE4, 0x00},
 	{OV9282_REG_OUTPUT_ENABLE5, 0x00},
 	{OV9282_REG_OUTPUT_ENABLE6, OV9282_OUTPUT_ENABLE6_ILPWM},
-	{0x3011, 0x0a},
-	{0x3013, 0x18},
-	{0x301c, 0xf0},
-	{0x3022, 0x01},
-	{0x3030, 0x10},
-	{0x3039, 0x32},
-	{0x303a, 0x00},
+	{CCI_REG8(0x3011), 0x0a},
+	{CCI_REG8(0x3013), 0x18},
+	{CCI_REG8(0x301c), 0xf0},
+	{CCI_REG8(0x3022), 0x01},
+	{CCI_REG8(0x3030), 0x10},
+	{CCI_REG8(0x3039), 0x32},
+	{CCI_REG8(0x303a), 0x00},
 	{OV9282_REG_AEC_MANUAL, OV9282_GAIN_PREC16_EN},
-	{0x3505, 0x8c},
-	{0x3507, 0x03},
-	{0x3508, 0x00},
-	{0x3610, 0x80},
-	{0x3611, 0xa0},
-	{0x3620, 0x6e},
-	{0x3632, 0x56},
-	{0x3633, 0x78},
-	{0x3666, 0x00},
-	{0x366f, 0x5a},
-	{0x3680, 0x84},
-	{0x3712, 0x80},
-	{0x372d, 0x22},
-	{0x3731, 0x80},
-	{0x3732, 0x30},
-	{0x377d, 0x22},
-	{0x3788, 0x02},
-	{0x3789, 0xa4},
-	{0x378a, 0x00},
-	{0x378b, 0x4a},
-	{0x3799, 0x20},
-	{0x3881, 0x42},
-	{0x38a8, 0x02},
-	{0x38a9, 0x80},
-	{0x38b1, 0x00},
-	{0x38c4, 0x00},
-	{0x38c5, 0xc0},
-	{0x38c6, 0x04},
-	{0x38c7, 0x80},
-	{0x3920, 0xff},
-	{0x4010, 0x40},
-	{0x4043, 0x40},
-	{0x4307, 0x30},
-	{0x4317, 0x00},
-	{0x4501, 0x00},
-	{0x450a, 0x08},
-	{0x4601, 0x04},
-	{0x470f, 0x00},
-	{0x4f07, 0x00},
-	{0x5000, 0x9f},
-	{0x5001, 0x00},
-	{0x5e00, 0x00},
-	{0x5d00, 0x07},
-	{0x5d01, 0x00},
-	{0x0101, 0x01},
-	{0x1000, 0x03},
-	{0x5a08, 0x84},
-};
-
-static struct ov9282_reg_list common_regs_list = {
-	.num_of_regs = ARRAY_SIZE(common_regs),
-	.regs = common_regs,
+	{CCI_REG8(0x3505), 0x8c},
+	{CCI_REG8(0x3507), 0x03},
+	{CCI_REG8(0x3508), 0x00},
+	{CCI_REG8(0x3610), 0x80},
+	{CCI_REG8(0x3611), 0xa0},
+	{CCI_REG8(0x3620), 0x6e},
+	{CCI_REG8(0x3632), 0x56},
+	{CCI_REG8(0x3633), 0x78},
+	{CCI_REG8(0x3666), 0x00},
+	{CCI_REG8(0x366f), 0x5a},
+	{CCI_REG8(0x3680), 0x84},
+	{CCI_REG8(0x3712), 0x80},
+	{CCI_REG8(0x372d), 0x22},
+	{CCI_REG8(0x3731), 0x80},
+	{CCI_REG8(0x3732), 0x30},
+	{CCI_REG8(0x377d), 0x22},
+	{CCI_REG8(0x3788), 0x02},
+	{CCI_REG8(0x3789), 0xa4},
+	{CCI_REG8(0x378a), 0x00},
+	{CCI_REG8(0x378b), 0x4a},
+	{CCI_REG8(0x3799), 0x20},
+	{CCI_REG8(0x3881), 0x42},
+	{CCI_REG8(0x38a8), 0x02},
+	{CCI_REG8(0x38a9), 0x80},
+	{CCI_REG8(0x38b1), 0x00},
+	{CCI_REG8(0x38c4), 0x00},
+	{CCI_REG8(0x38c5), 0xc0},
+	{CCI_REG8(0x38c6), 0x04},
+	{CCI_REG8(0x38c7), 0x80},
+	{CCI_REG8(0x3920), 0xff},
+	{CCI_REG8(0x4010), 0x40},
+	{CCI_REG8(0x4043), 0x40},
+	{CCI_REG8(0x4307), 0x30},
+	{CCI_REG8(0x4317), 0x00},
+	{CCI_REG8(0x4501), 0x00},
+	{CCI_REG8(0x450a), 0x08},
+	{CCI_REG8(0x4601), 0x04},
+	{CCI_REG8(0x470f), 0x00},
+	{CCI_REG8(0x4f07), 0x00},
+	{CCI_REG8(0x5000), 0x9f},
+	{CCI_REG8(0x5001), 0x00},
+	{CCI_REG8(0x5e00), 0x00},
+	{CCI_REG8(0x5d00), 0x07},
+	{CCI_REG8(0x5d01), 0x00},
+	{CCI_REG8(0x0101), 0x01},
+	{CCI_REG8(0x1000), 0x03},
+	{CCI_REG8(0x5a08), 0x84},
 };
 
 #define MODE_1280_800		0
@@ -317,96 +305,96 @@ static struct ov9282_reg_list common_regs_list = {
 #define DEFAULT_MODE		MODE_1280_720
 
 /* Sensor mode registers */
-static const struct ov9282_reg mode_1280x800_regs[] = {
-	{0x3778, 0x00},
-	{0x3800, 0x00},
-	{0x3801, 0x00},
-	{0x3802, 0x00},
-	{0x3803, 0x00},
-	{0x3804, 0x05},
-	{0x3805, 0x0f},
-	{0x3806, 0x03},
-	{0x3807, 0x2f},
-	{0x3808, 0x05},
-	{0x3809, 0x00},
-	{0x380a, 0x03},
-	{0x380b, 0x20},
-	{0x3810, 0x00},
-	{0x3811, 0x08},
-	{0x3812, 0x00},
-	{0x3813, 0x08},
-	{0x3814, 0x11},
-	{0x3815, 0x11},
+static const struct cci_reg_sequence mode_1280x800_regs[] = {
+	{CCI_REG8(0x3778), 0x00},
+	{CCI_REG8(0x3800), 0x00},
+	{CCI_REG8(0x3801), 0x00},
+	{CCI_REG8(0x3802), 0x00},
+	{CCI_REG8(0x3803), 0x00},
+	{CCI_REG8(0x3804), 0x05},
+	{CCI_REG8(0x3805), 0x0f},
+	{CCI_REG8(0x3806), 0x03},
+	{CCI_REG8(0x3807), 0x2f},
+	{CCI_REG8(0x3808), 0x05},
+	{CCI_REG8(0x3809), 0x00},
+	{CCI_REG8(0x380a), 0x03},
+	{CCI_REG8(0x380b), 0x20},
+	{CCI_REG8(0x3810), 0x00},
+	{CCI_REG8(0x3811), 0x08},
+	{CCI_REG8(0x3812), 0x00},
+	{CCI_REG8(0x3813), 0x08},
+	{CCI_REG8(0x3814), 0x11},
+	{CCI_REG8(0x3815), 0x11},
 	{OV9282_REG_TIMING_FORMAT_1, 0x40},
 	{OV9282_REG_TIMING_FORMAT_2, 0x00},
-	{0x4003, 0x40},
-	{0x4008, 0x04},
-	{0x4009, 0x0b},
-	{0x400c, 0x00},
-	{0x400d, 0x07},
-	{0x4507, 0x00},
-	{0x4509, 0x00},
+	{CCI_REG8(0x4003), 0x40},
+	{CCI_REG8(0x4008), 0x04},
+	{CCI_REG8(0x4009), 0x0b},
+	{CCI_REG8(0x400c), 0x00},
+	{CCI_REG8(0x400d), 0x07},
+	{CCI_REG8(0x4507), 0x00},
+	{CCI_REG8(0x4509), 0x00},
 };
 
-static const struct ov9282_reg mode_1280x720_regs[] = {
-	{0x3778, 0x00},
-	{0x3800, 0x00},
-	{0x3801, 0x00},
-	{0x3802, 0x00},
-	{0x3803, 0x00},
-	{0x3804, 0x05},
-	{0x3805, 0x0f},
-	{0x3806, 0x02},
-	{0x3807, 0xdf},
-	{0x3808, 0x05},
-	{0x3809, 0x00},
-	{0x380a, 0x02},
-	{0x380b, 0xd0},
-	{0x3810, 0x00},
-	{0x3811, 0x08},
-	{0x3812, 0x00},
-	{0x3813, 0x08},
-	{0x3814, 0x11},
-	{0x3815, 0x11},
+static const struct cci_reg_sequence mode_1280x720_regs[] = {
+	{CCI_REG8(0x3778), 0x00},
+	{CCI_REG8(0x3800), 0x00},
+	{CCI_REG8(0x3801), 0x00},
+	{CCI_REG8(0x3802), 0x00},
+	{CCI_REG8(0x3803), 0x00},
+	{CCI_REG8(0x3804), 0x05},
+	{CCI_REG8(0x3805), 0x0f},
+	{CCI_REG8(0x3806), 0x02},
+	{CCI_REG8(0x3807), 0xdf},
+	{CCI_REG8(0x3808), 0x05},
+	{CCI_REG8(0x3809), 0x00},
+	{CCI_REG8(0x380a), 0x02},
+	{CCI_REG8(0x380b), 0xd0},
+	{CCI_REG8(0x3810), 0x00},
+	{CCI_REG8(0x3811), 0x08},
+	{CCI_REG8(0x3812), 0x00},
+	{CCI_REG8(0x3813), 0x08},
+	{CCI_REG8(0x3814), 0x11},
+	{CCI_REG8(0x3815), 0x11},
 	{OV9282_REG_TIMING_FORMAT_1, 0x3c},
 	{OV9282_REG_TIMING_FORMAT_2, 0x84},
-	{0x4003, 0x40},
-	{0x4008, 0x02},
-	{0x4009, 0x05},
-	{0x400c, 0x00},
-	{0x400d, 0x03},
-	{0x4507, 0x00},
-	{0x4509, 0x80},
+	{CCI_REG8(0x4003), 0x40},
+	{CCI_REG8(0x4008), 0x02},
+	{CCI_REG8(0x4009), 0x05},
+	{CCI_REG8(0x400c), 0x00},
+	{CCI_REG8(0x400d), 0x03},
+	{CCI_REG8(0x4507), 0x00},
+	{CCI_REG8(0x4509), 0x80},
 };
 
-static const struct ov9282_reg mode_640x400_regs[] = {
-	{0x3778, 0x10},
-	{0x3800, 0x00},
-	{0x3801, 0x00},
-	{0x3802, 0x00},
-	{0x3803, 0x00},
-	{0x3804, 0x05},
-	{0x3805, 0x0f},
-	{0x3806, 0x03},
-	{0x3807, 0x2f},
-	{0x3808, 0x02},
-	{0x3809, 0x80},
-	{0x380a, 0x01},
-	{0x380b, 0x90},
-	{0x3810, 0x00},
-	{0x3811, 0x04},
-	{0x3812, 0x00},
-	{0x3813, 0x04},
-	{0x3814, 0x31},
-	{0x3815, 0x22},
+static const struct cci_reg_sequence mode_640x400_regs[] = {
+	{CCI_REG8(0x3778), 0x10},
+	{CCI_REG8(0x3800), 0x00},
+	{CCI_REG8(0x3801), 0x00},
+	{CCI_REG8(0x3802), 0x00},
+	{CCI_REG8(0x3803), 0x00},
+	{CCI_REG8(0x3804), 0x05},
+	{CCI_REG8(0x3805), 0x0f},
+	{CCI_REG8(0x3806), 0x03},
+	{CCI_REG8(0x3807), 0x2f},
+	{CCI_REG8(0x3808), 0x02},
+	{CCI_REG8(0x3809), 0x80},
+	{CCI_REG8(0x380a), 0x01},
+	{CCI_REG8(0x380b), 0x90},
+	{CCI_REG8(0x3810), 0x00},
+	{CCI_REG8(0x3811), 0x04},
+	{CCI_REG8(0x3812), 0x00},
+	{CCI_REG8(0x3813), 0x04},
+	{CCI_REG8(0x3814), 0x31},
+	{CCI_REG8(0x3815), 0x22},
 	{OV9282_REG_TIMING_FORMAT_1, 0x60},
 	{OV9282_REG_TIMING_FORMAT_2, 0x01},
-	{0x4008, 0x02},
-	{0x4009, 0x05},
-	{0x400c, 0x00},
-	{0x400d, 0x03},
-	{0x4507, 0x03},
-	{0x4509, 0x80},
+	{CCI_REG8(0x4008), 0x02},
+	{CCI_REG8(0x4009), 0x05},
+	{CCI_REG8(0x400c), 0x00},
+	{CCI_REG8(0x400d), 0x03},
+	{CCI_REG8(0x4507), 0x03},
+	{CCI_REG8(0x4509), 0x80},
 };
 
 /* Supported sensor mode configurations */
@@ -485,97 +473,6 @@ static inline struct ov9282 *to_ov9282(struct v4l2_subdev *subdev)
 	return container_of(subdev, struct ov9282, sd);
 }
 
-/**
- * ov9282_read_reg() - Read registers.
- * @ov9282: pointer to ov9282 device
- * @reg: register address
- * @len: length of bytes to read. Max supported bytes is 4
- * @val: pointer to register value to be filled.
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_read_reg(struct ov9282 *ov9282, u16 reg, u32 len, u32 *val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&ov9282->sd);
-	struct i2c_msg msgs[2] = {0};
-	u8 addr_buf[2] = {0};
-	u8 data_buf[4] = {0};
-	int ret;
-
-	if (WARN_ON(len > 4))
-		return -EINVAL;
-
-	put_unaligned_be16(reg, addr_buf);
-
-	/* Write register address */
-	msgs[0].addr = client->addr;
-	msgs[0].flags = 0;
-	msgs[0].len = ARRAY_SIZE(addr_buf);
-	msgs[0].buf = addr_buf;
-
-	/* Read data from register */
-	msgs[1].addr = client->addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = len;
-	msgs[1].buf = &data_buf[4 - len];
-
-	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
-	if (ret != ARRAY_SIZE(msgs))
-		return -EIO;
-
-	*val = get_unaligned_be32(data_buf);
-
-	return 0;
-}
-
-/**
- * ov9282_write_reg() - Write register
- * @ov9282: pointer to ov9282 device
- * @reg: register address
- * @len: length of bytes. Max supported bytes is 4
- * @val: register value
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_write_reg(struct ov9282 *ov9282, u16 reg, u32 len, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&ov9282->sd);
-	u8 buf[6] = {0};
-
-	if (WARN_ON(len > 4))
-		return -EINVAL;
-
-	put_unaligned_be16(reg, buf);
-	put_unaligned_be32(val << (8 * (4 - len)), buf + 2);
-	if (i2c_master_send(client, buf, len + 2) != len + 2)
-		return -EIO;
-
-	return 0;
-}
-
-/**
- * ov9282_write_regs() - Write a list of registers
- * @ov9282: pointer to ov9282 device
- * @regs: list of registers to be written
- * @len: length of registers array
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_write_regs(struct ov9282 *ov9282,
-			     const struct ov9282_reg *regs, u32 len)
-{
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < len; i++) {
-		ret = ov9282_write_reg(ov9282, regs[i].address, 1, regs[i].val);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
 /**
  * ov9282_update_controls() - Update control ranges based on streaming mode
  * @ov9282: pointer to ov9282 device
@@ -633,21 +530,21 @@ static u32 ov9282_exposure_to_us(struct ov9282 *ov9282, u32 exposure)
  */
 static int ov9282_update_exp_gain(struct ov9282 *ov9282, u32 exposure, u32 gain)
 {
-	int ret;
 	u32 exposure_us = ov9282_exposure_to_us(ov9282, exposure);
+	int ret, ret_hold;
 
 	dev_dbg(ov9282->dev, "Set exp %u (~%u us), analog gain %u",
 		exposure, exposure_us, gain);
 
-	ret = ov9282_write_reg(ov9282, OV9282_REG_HOLD, 1, 1);
+	ret = cci_write(ov9282->regmap, OV9282_REG_HOLD, 0x01, NULL);
 	if (ret)
 		return ret;
 
-	ret = ov9282_write_reg(ov9282, OV9282_REG_EXPOSURE, 3, exposure << 4);
+	ret = cci_write(ov9282->regmap, OV9282_REG_EXPOSURE, exposure << 4, NULL);
 	if (ret)
 		goto error_release_group_hold;
 
-	ret = ov9282_write_reg(ov9282, OV9282_REG_AGAIN, 1, gain);
+	ret = cci_write(ov9282->regmap, OV9282_REG_AGAIN, gain, NULL);
 	if (ret)
 		goto error_release_group_hold;
 
@@ -656,60 +553,9 @@ static int ov9282_update_exp_gain(struct ov9282 *ov9282, u32 exposure, u32 gain)
 				       OV9282_STROBE_FRAME_SPAN_DEFAULT);
 
 error_release_group_hold:
-	ov9282_write_reg(ov9282, OV9282_REG_HOLD, 1, 0);
-
-	return ret;
-}
+	ret_hold = cci_write(ov9282->regmap, OV9282_REG_HOLD, 0, NULL);
 
-static int ov9282_set_ctrl_hflip(struct ov9282 *ov9282, int value)
-{
-	u32 current_val;
-	int ret = ov9282_read_reg(ov9282, OV9282_REG_TIMING_FORMAT_2, 1,
-				  &current_val);
-	if (ret)
-		return ret;
-
-	if (value)
-		current_val |= OV9282_FLIP_BIT;
-	else
-		current_val &= ~OV9282_FLIP_BIT;
-
-	return ov9282_write_reg(ov9282, OV9282_REG_TIMING_FORMAT_2, 1,
-				current_val);
-}
-
-static int ov9282_set_ctrl_vflip(struct ov9282 *ov9282, int value)
-{
-	u32 current_val;
-	int ret = ov9282_read_reg(ov9282, OV9282_REG_TIMING_FORMAT_1, 1,
-				  &current_val);
-	if (ret)
-		return ret;
-
-	if (value)
-		current_val |= OV9282_FLIP_BIT;
-	else
-		current_val &= ~OV9282_FLIP_BIT;
-
-	return ov9282_write_reg(ov9282, OV9282_REG_TIMING_FORMAT_1, 1,
-				current_val);
-}
-
-static int ov9282_set_ctrl_flash_strobe_oe(struct ov9282 *ov9282, bool enable)
-{
-	u32 current_val;
-	int ret;
-
-	ret = ov9282_read_reg(ov9282, OV9282_REG_OUTPUT_ENABLE6, 1, &current_val);
-	if (ret)
-		return ret;
-
-	if (enable)
-		current_val |= OV9282_OUTPUT_ENABLE6_STROBE;
-	else
-		current_val &= ~OV9282_OUTPUT_ENABLE6_STROBE;
-
-	return ov9282_write_reg(ov9282, OV9282_REG_OUTPUT_ENABLE6, 1, current_val);
+	return ret ? ret : ret_hold;
 }
 
 static u32 ov9282_us_to_flash_duration(struct ov9282 *ov9282, u32 value)
@@ -740,30 +586,6 @@ static u32 ov9282_flash_duration_to_us(struct ov9282 *ov9282, u32 value)
 	return DIV_ROUND_UP(value * frame_width, OV9282_STROBE_SPAN_FACTOR);
 }
 
-static int ov9282_set_ctrl_flash_duration(struct ov9282 *ov9282, u32 value)
-{
-	u32 val = ov9282_us_to_flash_duration(ov9282, value);
-	int ret;
-
-	ret = ov9282_write_reg(ov9282, OV9282_REG_STROBE_FRAME_SPAN, 1,
-			       (val >> 24) & 0xff);
-	if (ret)
-		return ret;
-
-	ret = ov9282_write_reg(ov9282, OV9282_REG_STROBE_FRAME_SPAN + 1, 1,
-			       (val >> 16) & 0xff);
-	if (ret)
-		return ret;
-
-	ret = ov9282_write_reg(ov9282, OV9282_REG_STROBE_FRAME_SPAN + 2, 1,
-			       (val >> 8) & 0xff);
-	if (ret)
-		return ret;
-
-	return ov9282_write_reg(ov9282, OV9282_REG_STROBE_FRAME_SPAN + 3, 1,
-				val & 0xff);
-}
-
 /**
  * ov9282_set_ctrl() - Set subdevice control
  * @ctrl: pointer to v4l2_ctrl structure
@@ -818,23 +640,27 @@ static int ov9282_set_ctrl(struct v4l2_ctrl *ctrl)
 		break;
 	case V4L2_CID_VBLANK:
 		lpfr = ov9282->vblank + ov9282->cur_mode->height;
-		ret = ov9282_write_reg(ov9282, OV9282_REG_LPFR, 2, lpfr);
+		ret = cci_write(ov9282->regmap, OV9282_REG_LPFR, lpfr, NULL);
 		break;
 	case V4L2_CID_HFLIP:
-		ret = ov9282_set_ctrl_hflip(ov9282, ctrl->val);
+		ret = cci_update_bits(ov9282->regmap, OV9282_REG_TIMING_FORMAT_2,
+				      OV9282_FLIP_BIT, ctrl->val ? OV9282_FLIP_BIT : 0, NULL);
 		break;
 	case V4L2_CID_VFLIP:
-		ret = ov9282_set_ctrl_vflip(ov9282, ctrl->val);
+		ret = cci_update_bits(ov9282->regmap, OV9282_REG_TIMING_FORMAT_1,
+				      OV9282_FLIP_BIT, ctrl->val ? OV9282_FLIP_BIT : 0, NULL);
 		break;
 	case V4L2_CID_HBLANK:
-		ret = ov9282_write_reg(ov9282, OV9282_REG_TIMING_HTS, 2,
-				       (ctrl->val + ov9282->cur_mode->width) >> 1);
+		ret = cci_write(ov9282->regmap, OV9282_REG_TIMING_HTS,
+				(ctrl->val + ov9282->cur_mode->width) >> 1, NULL);
 		break;
 	case V4L2_CID_FLASH_STROBE_OE:
-		ret = ov9282_set_ctrl_flash_strobe_oe(ov9282, ctrl->val);
+		ret = cci_update_bits(ov9282->regmap, OV9282_REG_OUTPUT_ENABLE6,
+				      OV9282_OUTPUT_ENABLE6_STROBE,
+				      ctrl->val ? OV9282_OUTPUT_ENABLE6_STROBE : 0, NULL);
 		break;
 	case V4L2_CID_FLASH_DURATION:
-		ret = ov9282_set_ctrl_flash_duration(ov9282, ctrl->val);
+		ret = cci_write(ov9282->regmap, OV9282_REG_STROBE_FRAME_SPAN, ctrl->val, NULL);
 		break;
 	default:
 		dev_err(ov9282->dev, "Invalid control %d", ctrl->id);
@@ -968,8 +794,6 @@ static int ov9282_get_pad_format(struct v4l2_subdev *sd,
 {
 	struct ov9282 *ov9282 = to_ov9282(sd);
 
-	mutex_lock(&ov9282->mutex);
-
 	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
 		struct v4l2_mbus_framefmt *framefmt;
 
@@ -980,8 +804,6 @@ static int ov9282_get_pad_format(struct v4l2_subdev *sd,
 				       fmt);
 	}
 
-	mutex_unlock(&ov9282->mutex);
-
 	return 0;
 }
 
@@ -1002,8 +824,6 @@ static int ov9282_set_pad_format(struct v4l2_subdev *sd,
 	u32 code;
 	int ret = 0;
 
-	mutex_lock(&ov9282->mutex);
-
 	mode = v4l2_find_nearest_size(supported_modes,
 				      ARRAY_SIZE(supported_modes),
 				      width, height,
@@ -1029,8 +849,6 @@ static int ov9282_set_pad_format(struct v4l2_subdev *sd,
 		}
 	}
 
-	mutex_unlock(&ov9282->mutex);
-
 	return ret;
 }
 
@@ -1077,10 +895,8 @@ static int ov9282_get_selection(struct v4l2_subdev *sd,
 	case V4L2_SEL_TGT_CROP: {
 		struct ov9282 *ov9282 = to_ov9282(sd);
 
-		mutex_lock(&ov9282->mutex);
 		sel->r = *__ov9282_get_pad_crop(ov9282, sd_state, sel->pad,
 						sel->which);
-		mutex_unlock(&ov9282->mutex);
 
 		return 0;
 	}
@@ -1106,15 +922,11 @@ static int ov9282_get_selection(struct v4l2_subdev *sd,
 	return -EINVAL;
 }
 
-/**
- * ov9282_start_streaming() - Start sensor stream
- * @ov9282: pointer to ov9282 device
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_start_streaming(struct ov9282 *ov9282)
+static int ov9282_enable_streams(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state, u32 pad,
+				 u64 streams_mask)
 {
-	const struct ov9282_reg bitdepth_regs[2][2] = {
+	const struct cci_reg_sequence bitdepth_regs[2][2] = {
 		{
 			{OV9282_REG_PLL_CTRL_0D, OV9282_PLL_CTRL_0D_RAW10},
 			{OV9282_REG_ANA_CORE_2, OV9282_ANA_CORE2_RAW10},
@@ -1123,98 +935,74 @@ static int ov9282_start_streaming(struct ov9282 *ov9282)
 			{OV9282_REG_ANA_CORE_2, OV9282_ANA_CORE2_RAW8},
 		}
 	};
+	struct ov9282 *ov9282 = to_ov9282(sd);
 	const struct ov9282_reg_list *reg_list;
 	int bitdepth_index;
 	int ret;
 
+	ret = pm_runtime_resume_and_get(ov9282->dev);
+	if (ret)
+		return ret;
+
 	/* Write common registers */
-	ret = ov9282_write_regs(ov9282, common_regs_list.regs,
-				common_regs_list.num_of_regs);
+	ret = cci_multi_reg_write(ov9282->regmap, common_regs,
+				  ARRAY_SIZE(common_regs), NULL);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to write common registers");
-		return ret;
+		goto err_pm_put;
 	}
 
 	bitdepth_index = ov9282->code == MEDIA_BUS_FMT_Y10_1X10 ? 0 : 1;
-	ret = ov9282_write_regs(ov9282, bitdepth_regs[bitdepth_index], 2);
+	ret = cci_multi_reg_write(ov9282->regmap,
+				  bitdepth_regs[bitdepth_index], 2, NULL);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to write bitdepth regs");
-		return ret;
+		goto err_pm_put;
 	}
 
 	/* Write sensor mode registers */
 	reg_list = &ov9282->cur_mode->reg_list;
-	ret = ov9282_write_regs(ov9282, reg_list->regs, reg_list->num_of_regs);
+	ret = cci_multi_reg_write(ov9282->regmap, reg_list->regs,
+				  reg_list->num_of_regs, NULL);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to write initial registers");
-		return ret;
+		goto err_pm_put;
 	}
 
 	/* Setup handler will write actual exposure and gain */
 	ret =  __v4l2_ctrl_handler_setup(ov9282->sd.ctrl_handler);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to setup handler");
-		return ret;
+		goto err_pm_put;
 	}
 
 	/* Start streaming */
-	ret = ov9282_write_reg(ov9282, OV9282_REG_MODE_SELECT,
-			       1, OV9282_MODE_STREAMING);
+	ret = cci_write(ov9282->regmap, OV9282_REG_MODE_SELECT,
+			OV9282_MODE_STREAMING, NULL);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to start streaming");
-		return ret;
+		goto err_pm_put;
 	}
 
 	return 0;
-}
 
-/**
- * ov9282_stop_streaming() - Stop sensor stream
- * @ov9282: pointer to ov9282 device
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_stop_streaming(struct ov9282 *ov9282)
-{
-	return ov9282_write_reg(ov9282, OV9282_REG_MODE_SELECT,
-				1, OV9282_MODE_STANDBY);
+err_pm_put:
+	pm_runtime_put(ov9282->dev);
+
+	return ret;
 }
 
-/**
- * ov9282_set_stream() - Enable sensor streaming
- * @sd: pointer to ov9282 subdevice
- * @enable: set to enable sensor streaming
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int ov9282_set_stream(struct v4l2_subdev *sd, int enable)
+static int ov9282_disable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
 {
 	struct ov9282 *ov9282 = to_ov9282(sd);
 	int ret;
 
-	mutex_lock(&ov9282->mutex);
-
-	if (enable) {
-		ret = pm_runtime_resume_and_get(ov9282->dev);
-		if (ret)
-			goto error_unlock;
-
-		ret = ov9282_start_streaming(ov9282);
-		if (ret)
-			goto error_power_off;
-	} else {
-		ov9282_stop_streaming(ov9282);
-		pm_runtime_put(ov9282->dev);
-	}
-
-	mutex_unlock(&ov9282->mutex);
-
-	return 0;
+	ret = cci_write(ov9282->regmap, OV9282_REG_MODE_SELECT,
+			OV9282_MODE_STANDBY, NULL);
 
-error_power_off:
 	pm_runtime_put(ov9282->dev);
-error_unlock:
-	mutex_unlock(&ov9282->mutex);
 
 	return ret;
 }
@@ -1228,14 +1016,14 @@ static int ov9282_set_stream(struct v4l2_subdev *sd, int enable)
 static int ov9282_detect(struct ov9282 *ov9282)
 {
 	int ret;
-	u32 val;
+	u64 val;
 
-	ret = ov9282_read_reg(ov9282, OV9282_REG_ID, 2, &val);
+	ret = cci_read(ov9282->regmap, OV9282_REG_ID, &val, NULL);
 	if (ret)
 		return ret;
 
 	if (val != OV9282_ID) {
-		dev_err(ov9282->dev, "chip id mismatch: %x!=%x",
+		dev_err(ov9282->dev, "chip id mismatch: %x!=%llx",
 			OV9282_ID, val);
 		return -ENXIO;
 	}
@@ -1346,7 +1134,7 @@ static const struct v4l2_subdev_core_ops ov9282_core_ops = {
 };
 
 static const struct v4l2_subdev_video_ops ov9282_video_ops = {
-	.s_stream = ov9282_set_stream,
+	.s_stream = v4l2_subdev_s_stream_helper,
 };
 
 static const struct v4l2_subdev_pad_ops ov9282_pad_ops = {
@@ -1355,6 +1143,8 @@ static const struct v4l2_subdev_pad_ops ov9282_pad_ops = {
 	.get_fmt = ov9282_get_pad_format,
 	.set_fmt = ov9282_set_pad_format,
 	.get_selection = ov9282_get_selection,
+	.enable_streams = ov9282_enable_streams,
+	.disable_streams = ov9282_disable_streams,
 };
 
 static const struct v4l2_subdev_ops ov9282_subdev_ops = {
@@ -1397,9 +1187,8 @@ static int ov9282_power_on(struct device *dev)
 
 	usleep_range(400, 600);
 
-	ret = ov9282_write_reg(ov9282, OV9282_REG_MIPI_CTRL00, 1,
-			       ov9282->noncontinuous_clock ?
-					OV9282_GATED_CLOCK : 0);
+	ret = cci_write(ov9282->regmap, OV9282_REG_MIPI_CTRL00,
+			ov9282->noncontinuous_clock ? OV9282_GATED_CLOCK : 0, NULL);
 	if (ret) {
 		dev_err(ov9282->dev, "fail to write MIPI_CTRL00");
 		goto error_clk;
@@ -1457,9 +1246,6 @@ static int ov9282_init_controls(struct ov9282 *ov9282)
 	if (ret)
 		return ret;
 
-	/* Serialize controls with sensor device */
-	ctrl_hdlr->lock = &ov9282->mutex;
-
 	/* Initialize exposure and gain */
 	lpfr = mode->vblank + mode->height;
 	ov9282->exp_ctrl = v4l2_ctrl_new_std(ctrl_hdlr,
@@ -1576,13 +1362,15 @@ static int ov9282_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	mutex_init(&ov9282->mutex);
+	ov9282->regmap = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(ov9282->regmap))
+		return dev_err_probe(ov9282->dev, PTR_ERR(ov9282->regmap),
+				     "Failed to init CCI\n");
 
 	ret = ov9282_power_on(ov9282->dev);
-	if (ret) {
-		dev_err(ov9282->dev, "failed to power-on the sensor");
-		goto error_mutex_destroy;
-	}
+	if (ret)
+		return dev_err_probe(ov9282->dev, ret,
+				     "failed to power-on the sensor");
 
 	/* Check module identity */
 	ret = ov9282_detect(ov9282);
@@ -1615,27 +1403,34 @@ static int ov9282_probe(struct i2c_client *client)
 		goto error_handler_free;
 	}
 
-	ret = v4l2_async_register_subdev_sensor(&ov9282->sd);
+	ov9282->sd.state_lock = ov9282->ctrl_handler.lock;
+	ret = v4l2_subdev_init_finalize(&ov9282->sd);
 	if (ret < 0) {
-		dev_err(ov9282->dev,
-			"failed to register async subdev: %d", ret);
+		dev_err_probe(ov9282->dev, ret, "failed to init subdev\n");
 		goto error_media_entity;
 	}
 
 	pm_runtime_set_active(ov9282->dev);
 	pm_runtime_enable(ov9282->dev);
+
+	ret = v4l2_async_register_subdev_sensor(&ov9282->sd);
+	if (ret < 0)
+		goto v4l2_subdev_cleanup;
+
 	pm_runtime_idle(ov9282->dev);
 
 	return 0;
 
+v4l2_subdev_cleanup:
+	v4l2_subdev_cleanup(&ov9282->sd);
+	pm_runtime_disable(ov9282->dev);
+	pm_runtime_set_suspended(ov9282->dev);
 error_media_entity:
 	media_entity_cleanup(&ov9282->sd.entity);
 error_handler_free:
 	v4l2_ctrl_handler_free(ov9282->sd.ctrl_handler);
 error_power_off:
 	ov9282_power_off(ov9282->dev);
-error_mutex_destroy:
-	mutex_destroy(&ov9282->mutex);
 
 	return ret;
 }
@@ -1649,9 +1444,9 @@ static int ov9282_probe(struct i2c_client *client)
 static void ov9282_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct ov9282 *ov9282 = to_ov9282(sd);
 
 	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 
@@ -1659,8 +1454,6 @@ static void ov9282_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		ov9282_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	mutex_destroy(&ov9282->mutex);
 }
 
 static const struct dev_pm_ops ov9282_pm_ops = {
diff --git a/drivers/media/i2c/vgxy61.c b/drivers/media/i2c/vgxy61.c
index d64d0099e6fe2..3fb2166c81ef0 100644
--- a/drivers/media/i2c/vgxy61.c
+++ b/drivers/media/i2c/vgxy61.c
@@ -1802,6 +1802,9 @@ static int vgxy61_probe(struct i2c_client *client)
 
 	sensor->reset_gpio = devm_gpiod_get_optional(dev, "reset",
 						     GPIOD_OUT_HIGH);
+	if (IS_ERR(sensor->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(sensor->reset_gpio),
+				     "failed to get reset gpio\n");
 
 	ret = vgxy61_get_regulators(sensor);
 	if (ret) {
diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c
index c7c7d4a86c6bc..13e77648807c2 100644
--- a/drivers/media/mc/mc-request.c
+++ b/drivers/media/mc/mc-request.c
@@ -192,6 +192,8 @@ static long media_request_ioctl_reinit(struct media_request *req)
 	struct media_device *mdev = req->mdev;
 	unsigned long flags;
 
+	mutex_lock(&mdev->req_queue_mutex);
+
 	spin_lock_irqsave(&req->lock, flags);
 	if (req->state != MEDIA_REQUEST_STATE_IDLE &&
 	    req->state != MEDIA_REQUEST_STATE_COMPLETE) {
@@ -199,6 +201,7 @@ static long media_request_ioctl_reinit(struct media_request *req)
 			"request: %s not in idle or complete state, cannot reinit\n",
 			req->debug_str);
 		spin_unlock_irqrestore(&req->lock, flags);
+		mutex_unlock(&mdev->req_queue_mutex);
 		return -EBUSY;
 	}
 	if (req->access_count) {
@@ -206,6 +209,7 @@ static long media_request_ioctl_reinit(struct media_request *req)
 			"request: %s is being accessed, cannot reinit\n",
 			req->debug_str);
 		spin_unlock_irqrestore(&req->lock, flags);
+		mutex_unlock(&mdev->req_queue_mutex);
 		return -EBUSY;
 	}
 	req->state = MEDIA_REQUEST_STATE_CLEANING;
@@ -216,6 +220,7 @@ static long media_request_ioctl_reinit(struct media_request *req)
 	spin_lock_irqsave(&req->lock, flags);
 	req->state = MEDIA_REQUEST_STATE_IDLE;
 	spin_unlock_irqrestore(&req->lock, flags);
+	mutex_unlock(&mdev->req_queue_mutex);
 
 	return 0;
 }
diff --git a/drivers/media/pci/bt8xx/bttv-input.c b/drivers/media/pci/bt8xx/bttv-input.c
index 373b6c6817d76..f704476506e07 100644
--- a/drivers/media/pci/bt8xx/bttv-input.c
+++ b/drivers/media/pci/bt8xx/bttv-input.c
@@ -572,8 +572,9 @@ void bttv_input_fini(struct bttv *btv)
 	if (btv->remote == NULL)
 		return;
 
-	bttv_ir_stop(btv);
 	rc_unregister_device(btv->remote->dev);
+	bttv_ir_stop(btv);
+	rc_free_device(btv->remote->dev);
 	kfree(btv->remote);
 	btv->remote = NULL;
 }
diff --git a/drivers/media/pci/cx23885/cx23885-input.c b/drivers/media/pci/cx23885/cx23885-input.c
index ffbbeca8a8e5f..554767b8ef2bf 100644
--- a/drivers/media/pci/cx23885/cx23885-input.c
+++ b/drivers/media/pci/cx23885/cx23885-input.c
@@ -402,6 +402,7 @@ void cx23885_input_fini(struct cx23885_dev *dev)
 	if (dev->kernel_ir == NULL)
 		return;
 	rc_unregister_device(dev->kernel_ir->rc);
+	rc_free_device(dev->kernel_ir->rc);
 	kfree(dev->kernel_ir->phys);
 	kfree(dev->kernel_ir->name);
 	kfree(dev->kernel_ir);
diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c
index 3f73391f90987..4fdc59aaed8ee 100644
--- a/drivers/media/pci/cx25821/cx25821-alsa.c
+++ b/drivers/media/pci/cx25821/cx25821-alsa.c
@@ -397,7 +397,7 @@ static int dsp_buffer_free(struct cx25821_audio_dev *chip)
 {
 	struct cx25821_riscmem *risc = &chip->buf->risc;
 
-	BUG_ON(!chip->dma_size);
+	WARN_ON(!chip->dma_size);
 
 	dprintk(2, "Freeing buffer\n");
 	cx25821_alsa_dma_unmap(chip);
@@ -509,8 +509,8 @@ static int snd_cx25821_hw_params(struct snd_pcm_substream *substream,
 	chip->num_periods = params_periods(hw_params);
 	chip->dma_size = chip->period_size * params_periods(hw_params);
 
-	BUG_ON(!chip->dma_size);
-	BUG_ON(chip->num_periods & (chip->num_periods - 1));
+	WARN_ON(!chip->dma_size);
+	WARN_ON(chip->num_periods & (chip->num_periods - 1));
 
 	buf = kzalloc_obj(*buf);
 	if (NULL == buf)
diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c
index e958eecb29c5c..5d9ce4f9af011 100644
--- a/drivers/media/pci/cx88/cx88-input.c
+++ b/drivers/media/pci/cx88/cx88-input.c
@@ -509,8 +509,9 @@ int cx88_ir_fini(struct cx88_core *core)
 	if (!ir)
 		return 0;
 
-	cx88_ir_stop(core);
 	rc_unregister_device(ir->dev);
+	cx88_ir_stop(core);
+	rc_free_device(ir->dev);
 	kfree(ir);
 
 	/* done */
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index de05d8b0f9dc5..bbd24769ae56b 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -763,6 +763,7 @@ static int dm1105_ir_init(struct dm1105_dev *dm1105)
 static void dm1105_ir_exit(struct dm1105_dev *dm1105)
 {
 	rc_unregister_device(dm1105->ir.dev);
+	rc_free_device(dm1105->ir.dev);
 }
 
 static int dm1105_hw_init(struct dm1105_dev *dev)
diff --git a/drivers/media/pci/intel/ipu-bridge.c b/drivers/media/pci/intel/ipu-bridge.c
index 32cc95a766b71..fc6608e33de40 100644
--- a/drivers/media/pci/intel/ipu-bridge.c
+++ b/drivers/media/pci/intel/ipu-bridge.c
@@ -91,6 +91,8 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	IPU_SENSOR_CONFIG("OVTIDB10", 1, 560000000),
 	/* Omnivision OV2680 */
 	IPU_SENSOR_CONFIG("OVTI2680", 1, 331200000),
+	/* Omnivision OV5675 */
+	IPU_SENSOR_CONFIG("OVTI5675", 1, 450000000),
 	/* Omnivision OV8856 */
 	IPU_SENSOR_CONFIG("OVTI8856", 3, 180000000, 360000000, 720000000),
 	/* Sony IMX471 */
@@ -104,6 +106,13 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
  * without reporting a rotation of 180° in neither the SSDB nor the _PLD.
  */
 static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 13 9340"),
+		},
+		.driver_data = "OVTI02C1",
+	},
 	{
 		.matches = {
 			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
@@ -111,6 +120,13 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "OVTI02C1",
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 14 9440"),
+		},
+		.driver_data = "OVTI02C1",
+	},
 	{
 		.matches = {
 			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c
index 34f67f4f1bb56..d033d46181692 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.c
+++ b/drivers/media/pci/intel/ipu6/ipu6.c
@@ -686,7 +686,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 out_ipu6_rpm_put:
 	pm_runtime_put_sync(&isp->psys->auxdev.dev);
 out_ipu6_bus_del_devices:
-	if (isp->psys) {
+	if (!IS_ERR_OR_NULL(isp->psys)) {
 		ipu6_cpd_free_pkg_dir(isp->psys);
 		ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
 	}
diff --git a/drivers/media/pci/mantis/mantis_input.c b/drivers/media/pci/mantis/mantis_input.c
index 34c0d979240fd..edb4cacf55d22 100644
--- a/drivers/media/pci/mantis/mantis_input.c
+++ b/drivers/media/pci/mantis/mantis_input.c
@@ -72,5 +72,6 @@ EXPORT_SYMBOL_GPL(mantis_input_init);
 void mantis_input_exit(struct mantis_pci *mantis)
 {
 	rc_unregister_device(mantis->rc);
+	rc_free_device(mantis->rc);
 }
 EXPORT_SYMBOL_GPL(mantis_input_exit);
diff --git a/drivers/media/pci/mgb4/mgb4_core.c b/drivers/media/pci/mgb4/mgb4_core.c
index a7351a4693863..d23d854581c5f 100644
--- a/drivers/media/pci/mgb4/mgb4_core.c
+++ b/drivers/media/pci/mgb4/mgb4_core.c
@@ -84,7 +84,8 @@ static int temp_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		     int channel, long *val)
 {
 	struct mgb4_dev *mgbdev = dev_get_drvdata(dev);
-	u32 val10, raw;
+	u32 raw;
+	int val10;
 
 	if (type != hwmon_temp || attr != hwmon_temp_input)
 		return -EOPNOTSUPP;
diff --git a/drivers/media/pci/mgb4/mgb4_sysfs_in.c b/drivers/media/pci/mgb4/mgb4_sysfs_in.c
index 4cd4addcd0a57..80ff05450e985 100644
--- a/drivers/media/pci/mgb4/mgb4_sysfs_in.c
+++ b/drivers/media/pci/mgb4/mgb4_sysfs_in.c
@@ -128,7 +128,16 @@ static ssize_t color_mapping_show(struct device *dev,
 	u32 config = mgb4_read_reg(&vindev->mgbdev->video,
 	  vindev->config->regs.config);
 
-	return sprintf(buf, "%s\n", config & (1U << 8) ? "0" : "1");
+	switch ((config >> 7) & 3) {
+	case 0: /* SPWG/VESA */
+		return sprintf(buf, "1\n");
+	case 1: /* ZDML */
+		return sprintf(buf, "2\n");
+	case 2: /* OLDI/JEIDA */
+		return sprintf(buf, "0\n");
+	default:
+		return -EIO;
+	}
 }
 
 /*
@@ -151,17 +160,20 @@ static ssize_t color_mapping_store(struct device *dev,
 
 	switch (val) {
 	case 0: /* OLDI/JEIDA */
-		fpga_data = (1U << 8);
+		fpga_data = 2;
 		break;
 	case 1: /* SPWG/VESA */
 		fpga_data = 0;
 		break;
+	case 2: /* ZDML */
+		fpga_data = 1;
+		break;
 	default:
 		return -EINVAL;
 	}
 
 	mgb4_mask_reg(&vindev->mgbdev->video, vindev->config->regs.config,
-		      1U << 8, fpga_data);
+		      3U << 7, fpga_data << 7);
 
 	return count;
 }
diff --git a/drivers/media/pci/mgb4/mgb4_sysfs_out.c b/drivers/media/pci/mgb4/mgb4_sysfs_out.c
index 5769f3ca6c2ff..d807218e28ca2 100644
--- a/drivers/media/pci/mgb4/mgb4_sysfs_out.c
+++ b/drivers/media/pci/mgb4/mgb4_sysfs_out.c
@@ -143,6 +143,64 @@ static ssize_t video_source_store(struct device *dev,
 	return ret;
 }
 
+static ssize_t color_mapping_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct video_device *vdev = to_video_device(dev);
+	struct mgb4_vout_dev *voutdev = video_get_drvdata(vdev);
+	u32 config = mgb4_read_reg(&voutdev->mgbdev->video,
+	  voutdev->config->regs.config);
+
+	switch ((config >> 6) & 3) {
+	case 0: /* SPWG/VESA */
+		return sprintf(buf, "1\n");
+	case 1: /* ZDML */
+		return sprintf(buf, "2\n");
+	case 2: /* OLDI/JEIDA */
+		return sprintf(buf, "0\n");
+	default:
+		return -EIO;
+	}
+}
+
+/*
+ * Color mapping change is expected to be called on live streams. Video device
+ * locking/queue check is not needed.
+ */
+static ssize_t color_mapping_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	struct video_device *vdev = to_video_device(dev);
+	struct mgb4_vout_dev *voutdev = video_get_drvdata(vdev);
+	u32 fpga_data;
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	switch (val) {
+	case 0: /* OLDI/JEIDA */
+		fpga_data = 2;
+		break;
+	case 1: /* SPWG/VESA */
+		fpga_data = 0;
+		break;
+	case 2: /* ZDML */
+		fpga_data = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mgb4_mask_reg(&voutdev->mgbdev->video, voutdev->config->regs.config,
+		      3U << 6, fpga_data << 6);
+
+	return count;
+}
+
 static ssize_t display_width_show(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
@@ -711,6 +769,7 @@ static DEVICE_ATTR_RW(hback_porch);
 static DEVICE_ATTR_RW(hfront_porch);
 static DEVICE_ATTR_RW(vback_porch);
 static DEVICE_ATTR_RW(vfront_porch);
+static DEVICE_ATTR_RW(color_mapping);
 
 static DEVICE_ATTR_RW(fpdl3_output_width);
 
@@ -731,6 +790,7 @@ struct attribute *mgb4_fpdl3_out_attrs[] = {
 	&dev_attr_vback_porch.attr,
 	&dev_attr_vfront_porch.attr,
 	&dev_attr_fpdl3_output_width.attr,
+	&dev_attr_color_mapping.attr,
 	NULL
 };
 
@@ -740,6 +800,7 @@ struct attribute *mgb4_gmsl3_out_attrs[] = {
 	&dev_attr_display_width.attr,
 	&dev_attr_display_height.attr,
 	&dev_attr_frame_rate.attr,
+	&dev_attr_color_mapping.attr,
 	NULL
 };
 
@@ -759,5 +820,6 @@ struct attribute *mgb4_gmsl1_out_attrs[] = {
 	&dev_attr_hfront_porch.attr,
 	&dev_attr_vback_porch.attr,
 	&dev_attr_vfront_porch.attr,
+	&dev_attr_color_mapping.attr,
 	NULL
 };
diff --git a/drivers/media/pci/saa7134/saa7134-i2c.c b/drivers/media/pci/saa7134/saa7134-i2c.c
index 04e85765373ec..1164e91cbb7b7 100644
--- a/drivers/media/pci/saa7134/saa7134-i2c.c
+++ b/drivers/media/pci/saa7134/saa7134-i2c.c
@@ -28,7 +28,7 @@ static unsigned int i2c_scan;
 module_param(i2c_scan, int, 0444);
 MODULE_PARM_DESC(i2c_scan,"scan i2c bus at insmod time");
 
-#define i2c_dbg(level, fmt, arg...) do { \
+#define saa7134_i2c_dbg(level, fmt, arg...) do { \
 	if (i2c_debug == level) \
 		printk(KERN_DEBUG pr_fmt("i2c: " fmt), ## arg); \
 	} while (0)
@@ -84,20 +84,20 @@ static inline enum i2c_status i2c_get_status(struct saa7134_dev *dev)
 	enum i2c_status status;
 
 	status = saa_readb(SAA7134_I2C_ATTR_STATUS) & 0x0f;
-	i2c_dbg(2, "i2c stat <= %s\n", str_i2c_status[status]);
+	saa7134_i2c_dbg(2, "i2c stat <= %s\n", str_i2c_status[status]);
 	return status;
 }
 
 static inline void i2c_set_status(struct saa7134_dev *dev,
 				  enum i2c_status status)
 {
-	i2c_dbg(2, "i2c stat => %s\n", str_i2c_status[status]);
+	saa7134_i2c_dbg(2, "i2c stat => %s\n", str_i2c_status[status]);
 	saa_andorb(SAA7134_I2C_ATTR_STATUS,0x0f,status);
 }
 
 static inline void i2c_set_attr(struct saa7134_dev *dev, enum i2c_attr attr)
 {
-	i2c_dbg(2, "i2c attr => %s\n", str_i2c_attr[attr]);
+	saa7134_i2c_dbg(2, "i2c attr => %s\n", str_i2c_attr[attr]);
 	saa_andorb(SAA7134_I2C_ATTR_STATUS,0xc0,attr << 6);
 }
 
@@ -160,7 +160,7 @@ static int i2c_reset(struct saa7134_dev *dev)
 	enum i2c_status status;
 	int count;
 
-	i2c_dbg(2, "i2c reset\n");
+	saa7134_i2c_dbg(2, "i2c reset\n");
 	status = i2c_get_status(dev);
 	if (!i2c_is_error(status))
 		return true;
@@ -198,7 +198,7 @@ static inline int i2c_send_byte(struct saa7134_dev *dev,
 //	dword |= 0x40 << 16;  /* 400 kHz */
 	dword |= 0xf0 << 24;
 	saa_writel(SAA7134_I2C_ATTR_STATUS >> 2, dword);
-	i2c_dbg(2, "i2c data => 0x%x\n", data);
+	saa7134_i2c_dbg(2, "i2c data => 0x%x\n", data);
 
 	if (!i2c_is_busy_wait(dev))
 		return -EIO;
@@ -220,7 +220,7 @@ static inline int i2c_recv_byte(struct saa7134_dev *dev)
 	if (i2c_is_error(status))
 		return -EIO;
 	data = saa_readb(SAA7134_I2C_DATA);
-	i2c_dbg(2, "i2c data <= 0x%x\n", data);
+	saa7134_i2c_dbg(2, "i2c data <= 0x%x\n", data);
 	return data;
 }
 
@@ -237,12 +237,12 @@ static int saa7134_i2c_xfer(struct i2c_adapter *i2c_adap,
 		if (!i2c_reset(dev))
 			return -EIO;
 
-	i2c_dbg(2, "start xfer\n");
-	i2c_dbg(1, "i2c xfer:");
+	saa7134_i2c_dbg(2, "start xfer\n");
+	saa7134_i2c_dbg(1, "i2c xfer:");
 	for (i = 0; i < num; i++) {
 		if (!(msgs[i].flags & I2C_M_NOSTART) || 0 == i) {
 			/* send address */
-			i2c_dbg(2, "send address\n");
+			saa7134_i2c_dbg(2, "send address\n");
 			addr  = msgs[i].addr << 1;
 			if (msgs[i].flags & I2C_M_RD)
 				addr |= 1;
@@ -265,7 +265,7 @@ static int saa7134_i2c_xfer(struct i2c_adapter *i2c_adap,
 		}
 		if (msgs[i].flags & I2C_M_RD) {
 			/* read bytes */
-			i2c_dbg(2, "read bytes\n");
+			saa7134_i2c_dbg(2, "read bytes\n");
 			for (byte = 0; byte < msgs[i].len; byte++) {
 				i2c_cont(1, " =");
 				rc = i2c_recv_byte(dev);
@@ -286,7 +286,7 @@ static int saa7134_i2c_xfer(struct i2c_adapter *i2c_adap,
 			}
 		} else {
 			/* write bytes */
-			i2c_dbg(2, "write bytes\n");
+			saa7134_i2c_dbg(2, "write bytes\n");
 			for (byte = 0; byte < msgs[i].len; byte++) {
 				data = msgs[i].buf[byte];
 				i2c_cont(1, " %02x", data);
@@ -296,7 +296,7 @@ static int saa7134_i2c_xfer(struct i2c_adapter *i2c_adap,
 			}
 		}
 	}
-	i2c_dbg(2, "xfer done\n");
+	saa7134_i2c_dbg(2, "xfer done\n");
 	i2c_cont(1, " >");
 	i2c_set_attr(dev,STOP);
 	rc = -EIO;
diff --git a/drivers/media/pci/saa7134/saa7134-input.c b/drivers/media/pci/saa7134/saa7134-input.c
index 5b71014157808..7f6680de31564 100644
--- a/drivers/media/pci/saa7134/saa7134-input.c
+++ b/drivers/media/pci/saa7134/saa7134-input.c
@@ -834,6 +834,7 @@ void saa7134_input_fini(struct saa7134_dev *dev)
 		return;
 
 	rc_unregister_device(dev->remote->dev);
+	rc_free_device(dev->remote->dev);
 	kfree(dev->remote);
 	dev->remote = NULL;
 }
diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c
index 74406d5ea0a50..6bcde506adf50 100644
--- a/drivers/media/pci/saa7164/saa7164-core.c
+++ b/drivers/media/pci/saa7164/saa7164-core.c
@@ -888,6 +888,15 @@ static int get_resources(struct saa7164_dev *dev)
 	return -EBUSY;
 }
 
+static void release_resources(struct saa7164_dev *dev)
+{
+	release_mem_region(pci_resource_start(dev->pci, 0),
+			   pci_resource_len(dev->pci, 0));
+
+	release_mem_region(pci_resource_start(dev->pci, 2),
+			   pci_resource_len(dev->pci, 2));
+}
+
 static int saa7164_port_init(struct saa7164_dev *dev, int portnr)
 {
 	struct saa7164_port *port = NULL;
@@ -947,9 +956,9 @@ static int saa7164_dev_setup(struct saa7164_dev *dev)
 
 	snprintf(dev->name, sizeof(dev->name), "saa7164[%d]", dev->nr);
 
-	mutex_lock(&devlist);
-	list_add_tail(&dev->devlist, &saa7164_devlist);
-	mutex_unlock(&devlist);
+	scoped_guard(mutex, &devlist) {
+		list_add_tail(&dev->devlist, &saa7164_devlist);
+	}
 
 	/* board config */
 	dev->board = UNSET;
@@ -996,11 +1005,17 @@ static int saa7164_dev_setup(struct saa7164_dev *dev)
 	}
 
 	/* PCI/e allocations */
-	dev->lmmio = ioremap(pci_resource_start(dev->pci, 0),
-			     pci_resource_len(dev->pci, 0));
+	dev->lmmio = pci_ioremap_bar(dev->pci, 0);
+	if (!dev->lmmio) {
+		dev_err(&dev->pci->dev, "Failed to remap MMIO BAR 0\n");
+		goto err_ioremap_bar0;
+	}
 
-	dev->lmmio2 = ioremap(pci_resource_start(dev->pci, 2),
-			     pci_resource_len(dev->pci, 2));
+	dev->lmmio2 = pci_ioremap_bar(dev->pci, 2);
+	if (!dev->lmmio2) {
+		dev_err(&dev->pci->dev, "Failed to remap MMIO BAR 2\n");
+		goto err_ioremap_bar2;
+	}
 
 	dev->bmmio = (u8 __iomem *)dev->lmmio;
 	dev->bmmio2 = (u8 __iomem *)dev->lmmio2;
@@ -1019,17 +1034,25 @@ static int saa7164_dev_setup(struct saa7164_dev *dev)
 	saa7164_pci_quirks(dev);
 
 	return 0;
+
+err_ioremap_bar2:
+	iounmap(dev->lmmio);
+err_ioremap_bar0:
+	release_resources(dev);
+
+	scoped_guard(mutex, &devlist) {
+		list_del(&dev->devlist);
+	}
+	saa7164_devcount--;
+
+	return -ENODEV;
 }
 
 static void saa7164_dev_unregister(struct saa7164_dev *dev)
 {
 	dprintk(1, "%s()\n", __func__);
 
-	release_mem_region(pci_resource_start(dev->pci, 0),
-		pci_resource_len(dev->pci, 0));
-
-	release_mem_region(pci_resource_start(dev->pci, 2),
-		pci_resource_len(dev->pci, 2));
+	release_resources(dev);
 
 	if (!atomic_dec_and_test(&dev->refcount))
 		return;
diff --git a/drivers/media/pci/saa7164/saa7164-fw.c b/drivers/media/pci/saa7164/saa7164-fw.c
index cc9f384f7f1e9..341cef62452f1 100644
--- a/drivers/media/pci/saa7164/saa7164-fw.c
+++ b/drivers/media/pci/saa7164/saa7164-fw.c
@@ -10,8 +10,8 @@
 
 #include "saa7164.h"
 
-#define SAA7164_REV2_FIRMWARE		"NXP7164-2010-03-10.1.fw"
-#define SAA7164_REV2_FIRMWARE_SIZE	4019072
+#define SAA7164_REV2_FIRMWARE		"v4l-saa7164-1.0.2-3.fw"
+#define SAA7164_REV2_FIRMWARE_SIZE	4038864
 
 #define SAA7164_REV3_FIRMWARE		"NXP7164-2010-03-10.1.fw"
 #define SAA7164_REV3_FIRMWARE_SIZE	4019072
diff --git a/drivers/media/pci/smipcie/smipcie-ir.c b/drivers/media/pci/smipcie/smipcie-ir.c
index c0604d9c70119..0bbe4fa2d5a84 100644
--- a/drivers/media/pci/smipcie/smipcie-ir.c
+++ b/drivers/media/pci/smipcie/smipcie-ir.c
@@ -181,5 +181,6 @@ void smi_ir_exit(struct smi_dev *dev)
 
 	rc_unregister_device(rc_dev);
 	smi_ir_stop(ir);
+	rc_free_device(rc_dev);
 	ir->rc_dev = NULL;
 }
diff --git a/drivers/media/pci/ttpci/budget-ci.c b/drivers/media/pci/ttpci/budget-ci.c
index 3709c0fb23b07..8b496b959d7ea 100644
--- a/drivers/media/pci/ttpci/budget-ci.c
+++ b/drivers/media/pci/ttpci/budget-ci.c
@@ -249,6 +249,7 @@ static void msp430_ir_deinit(struct budget_ci *budget_ci)
 	cancel_work_sync(&budget_ci->ir.msp430_irq_bh_work);
 
 	rc_unregister_device(budget_ci->ir.dev);
+	rc_free_device(budget_ci->ir.dev);
 }
 
 static int ciintf_read_attribute_mem(struct dvb_ca_en50221 *ca, int slot, int address)
diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c
index d81facf735d92..f707bdc1fb0f1 100644
--- a/drivers/media/pci/zoran/zoran_card.c
+++ b/drivers/media/pci/zoran/zoran_card.c
@@ -1373,7 +1373,7 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		}
 		if (zr->codec->type != zr->card.video_codec) {
 			pci_err(pdev, "%s - wrong codec\n", __func__);
-			goto zr_unreg_videocodec;
+			goto zr_detach_codec;
 		}
 	}
 	if (zr->card.video_vfe != 0) {
diff --git a/drivers/media/platform/broadcom/Kconfig b/drivers/media/platform/broadcom/Kconfig
index 32b76ebfcd9a1..93ba565d9a027 100644
--- a/drivers/media/platform/broadcom/Kconfig
+++ b/drivers/media/platform/broadcom/Kconfig
@@ -14,10 +14,5 @@ config VIDEO_BCM2835_UNICAM
 	  This is a V4L2 driver that controls the CSI-2 receiver directly,
 	  independently from the VC4 firmware.
 
-	  This driver is mutually exclusive with the use of bcm2835-camera. The
-	  firmware will disable all access to the peripheral from within the
-	  firmware if it finds a DT node using it, and bcm2835-camera will
-	  therefore fail to probe.
-
 	  To compile this driver as a module, choose M here. The module will be
 	  called bcm2835-unicam.
diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c
index 53a342853be6e..f0a41f147dfeb 100644
--- a/drivers/media/platform/broadcom/bcm2835-unicam.c
+++ b/drivers/media/platform/broadcom/bcm2835-unicam.c
@@ -14,10 +14,6 @@
  * and TI CAL camera interface driver by
  *    Benoit Parrot <bparrot@ti.com>
  *
- *
- * There are two camera drivers in the kernel for BCM283x - this one and
- * bcm2835-camera (currently in staging).
- *
  * This driver directly controls the Unicam peripheral - there is no
  * involvement with the VideoCore firmware. Unicam receives CSI-2 or CCP2 data
  * and writes it into SDRAM. The only potential processing options are to
diff --git a/drivers/media/platform/renesas/rcar-csi2.c b/drivers/media/platform/renesas/rcar-csi2.c
index d1b31ab8b8c4a..7305cc4a04cb1 100644
--- a/drivers/media/platform/renesas/rcar-csi2.c
+++ b/drivers/media/platform/renesas/rcar-csi2.c
@@ -71,10 +71,7 @@ struct rcar_csi2;
 #define FLD_REG				0x1c
 #define FLD_FLD_NUM(n)			(((n) & 0xff) << 16)
 #define FLD_DET_SEL(n)			(((n) & 0x3) << 4)
-#define FLD_FLD_EN4			BIT(3)
-#define FLD_FLD_EN3			BIT(2)
-#define FLD_FLD_EN2			BIT(1)
-#define FLD_FLD_EN			BIT(0)
+#define FLD_FLD_EN(ch)			BIT(ch)
 
 /* Automatic Standby Control */
 #define ASTBY_REG			0x20
@@ -959,35 +956,6 @@ static int rcsi2_set_phypll(struct rcar_csi2 *priv, unsigned int mbps)
 	return 0;
 }
 
-static int rcsi2_calc_mbps(struct rcar_csi2 *priv, unsigned int bpp,
-			   unsigned int lanes)
-{
-	struct media_pad *remote_pad;
-	struct v4l2_subdev *source;
-	s64 freq;
-	u64 mbps;
-
-	if (!priv->remote)
-		return -ENODEV;
-
-	source = priv->remote;
-	remote_pad = &source->entity.pads[priv->remote_pad];
-
-	freq = v4l2_get_link_freq(remote_pad, bpp, 2 * lanes);
-	if (freq < 0) {
-		int ret = (int)freq;
-
-		dev_err(priv->dev, "failed to get link freq for %s: %d\n",
-			source->name, ret);
-
-		return ret;
-	}
-
-	mbps = div_u64(freq * 2, MEGA);
-
-	return mbps;
-}
-
 static int rcsi2_get_active_lanes(struct rcar_csi2 *priv,
 				  unsigned int *lanes)
 {
@@ -1035,6 +1003,62 @@ static int rcsi2_get_active_lanes(struct rcar_csi2 *priv,
 	return 0;
 }
 
+static int rcsi2_calc_mbps(struct rcar_csi2 *priv,
+			   struct v4l2_subdev_state *state)
+{
+	struct media_pad *remote_pad;
+	struct v4l2_subdev *source;
+	s64 freq;
+	u64 mbps;
+
+	if (!priv->remote)
+		return -ENODEV;
+
+	source = priv->remote;
+	remote_pad = &source->entity.pads[priv->remote_pad];
+
+	/*
+	 * First try to get the real link freq. If that fails, try the heuristic
+	 * method with bpp and lanes (but that only works for one route).
+	 */
+	freq = v4l2_get_link_freq(remote_pad, 0, 0);
+	if (freq < 0) {
+		const struct rcar_csi2_format *format;
+		const struct v4l2_mbus_framefmt *fmt;
+		unsigned int lanes;
+		unsigned int bpp;
+		int ret;
+
+		ret = rcsi2_get_active_lanes(priv, &lanes);
+		if (ret)
+			return ret;
+
+		fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK);
+		if (!fmt)
+			return -EINVAL;
+
+		format = rcsi2_code_to_fmt(fmt->code);
+		if (!format)
+			return -EINVAL;
+
+		bpp = format->bpp;
+
+		freq = v4l2_get_link_freq(remote_pad, bpp, 2 * lanes);
+		if (freq < 0) {
+			int ret = (int)freq;
+
+			dev_err(priv->dev, "failed to get link freq for %s: %d\n",
+				source->name, ret);
+
+			return ret;
+		}
+	}
+
+	mbps = div_u64(freq * 2, MEGA);
+
+	return mbps;
+}
+
 static int rcsi2_start_receiver_gen3(struct rcar_csi2 *priv,
 				     struct v4l2_subdev_state *state)
 {
@@ -1082,8 +1106,8 @@ static int rcsi2_start_receiver_gen3(struct rcar_csi2 *priv,
 	}
 
 	if (fmt->field == V4L2_FIELD_ALTERNATE)
-		fld = FLD_DET_SEL(1) | FLD_FLD_EN4 | FLD_FLD_EN3 | FLD_FLD_EN2
-			| FLD_FLD_EN;
+		fld = FLD_DET_SEL(1) | FLD_FLD_EN(3) | FLD_FLD_EN(2) |
+		      FLD_FLD_EN(1) | FLD_FLD_EN(0);
 
 	/*
 	 * Get the number of active data lanes inspecting the remote mbus
@@ -1096,7 +1120,7 @@ static int rcsi2_start_receiver_gen3(struct rcar_csi2 *priv,
 	phycnt = PHYCNT_ENABLECLK;
 	phycnt |= (1 << lanes) - 1;
 
-	mbps = rcsi2_calc_mbps(priv, format->bpp, lanes);
+	mbps = rcsi2_calc_mbps(priv, state);
 	if (mbps < 0)
 		return mbps;
 
@@ -1478,23 +1502,15 @@ static int rcsi2_start_receiver_v4h(struct rcar_csi2 *priv,
 				    struct v4l2_subdev_state *state)
 {
 	const struct rcsi2_cphy_setting *cphy = NULL;
-	const struct rcar_csi2_format *format;
-	const struct v4l2_mbus_framefmt *fmt;
 	unsigned int lanes;
 	int mbps;
 	int ret;
 
-	/* Use the format on the sink pad to compute the receiver config. */
-	fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK);
-	format = rcsi2_code_to_fmt(fmt->code);
-	if (!format)
-		return -EINVAL;
-
 	ret = rcsi2_get_active_lanes(priv, &lanes);
 	if (ret)
 		return ret;
 
-	mbps = rcsi2_calc_mbps(priv, format->bpp, lanes);
+	mbps = rcsi2_calc_mbps(priv, state);
 	if (mbps < 0)
 		return mbps;
 
@@ -1735,23 +1751,15 @@ static int rcsi2_init_common_v4m(struct rcar_csi2 *priv, unsigned int mbps)
 static int rcsi2_start_receiver_v4m(struct rcar_csi2 *priv,
 				    struct v4l2_subdev_state *state)
 {
-	const struct rcar_csi2_format *format;
-	const struct v4l2_mbus_framefmt *fmt;
 	unsigned int lanes;
 	int mbps;
 	int ret;
 
-	/* Calculate parameters */
-	fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK);
-	format = rcsi2_code_to_fmt(fmt->code);
-	if (!format)
-		return -EINVAL;
-
 	ret = rcsi2_get_active_lanes(priv, &lanes);
 	if (ret)
 		return ret;
 
-	mbps = rcsi2_calc_mbps(priv, format->bpp, lanes);
+	mbps = rcsi2_calc_mbps(priv, state);
 	if (mbps < 0)
 		return mbps;
 
diff --git a/drivers/media/platform/renesas/rcar-isp/csisp.c b/drivers/media/platform/renesas/rcar-isp/csisp.c
index 1eb29a0b774a9..8fb2cc3b56503 100644
--- a/drivers/media/platform/renesas/rcar-isp/csisp.c
+++ b/drivers/media/platform/renesas/rcar-isp/csisp.c
@@ -28,10 +28,7 @@
 #define ISPSTART_STOP					0x0000
 
 #define ISPPROCMODE_DT_REG(n)				(0x1100 + (0x4 * (n)))
-#define ISPPROCMODE_DT_PROC_MODE_VC3(pm)		(((pm) & 0x3f) << 24)
-#define ISPPROCMODE_DT_PROC_MODE_VC2(pm)		(((pm) & 0x3f) << 16)
-#define ISPPROCMODE_DT_PROC_MODE_VC1(pm)		(((pm) & 0x3f) << 8)
-#define ISPPROCMODE_DT_PROC_MODE_VC0(pm)		((pm) & 0x3f)
+#define ISPPROCMODE_DT_PROC_MODE_VCn(vc, pm)		(((pm) & 0x3f) << (8 * (vc)))
 
 #define ISPCS_FILTER_ID_CH_REG(n)			(0x3000 + (0x0100 * (n)))
 
@@ -263,10 +260,10 @@ static int risp_start(struct rcar_isp *isp, struct v4l2_subdev_state *state)
 
 	/* Setup processing method. */
 	risp_write_cs(isp, ISPPROCMODE_DT_REG(format->datatype),
-		      ISPPROCMODE_DT_PROC_MODE_VC3(format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VC2(format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VC1(format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VC0(format->procmode));
+		      ISPPROCMODE_DT_PROC_MODE_VCn(3, format->procmode) |
+		      ISPPROCMODE_DT_PROC_MODE_VCn(2, format->procmode) |
+		      ISPPROCMODE_DT_PROC_MODE_VCn(1, format->procmode) |
+		      ISPPROCMODE_DT_PROC_MODE_VCn(0, format->procmode));
 
 	/* Start ISP. */
 	risp_write_cs(isp, ISPSTART_REG, ISPSTART_START);
diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c b/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c
index 1b81bcc067efd..9e67160a16e46 100644
--- a/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c
+++ b/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c
@@ -489,8 +489,8 @@ static inline unsigned int rkcif_mipi_get_reg(struct rkcif_interface *interface,
 
 	block = interface->index - RKCIF_MIPI_BASE;
 
-	if (WARN_ON_ONCE(block > RKCIF_MIPI_MAX - RKCIF_MIPI_BASE) ||
-	    WARN_ON_ONCE(index > RKCIF_MIPI_REGISTER_MAX))
+	if (WARN_ON_ONCE(block >= ARRAY_SIZE(rkcif->match_data->mipi->blocks)) ||
+	    WARN_ON_ONCE(index >= ARRAY_SIZE(rkcif->match_data->mipi->regs)))
 		return RKCIF_REGISTER_NOTSUPPORTED;
 
 	offset = rkcif->match_data->mipi->blocks[block].offset;
@@ -510,9 +510,9 @@ static inline unsigned int rkcif_mipi_id_get_reg(struct rkcif_stream *stream,
 	block = stream->interface->index - RKCIF_MIPI_BASE;
 	id = stream->id;
 
-	if (WARN_ON_ONCE(block > RKCIF_MIPI_MAX - RKCIF_MIPI_BASE) ||
-	    WARN_ON_ONCE(id > RKCIF_ID_MAX) ||
-	    WARN_ON_ONCE(index > RKCIF_MIPI_ID_REGISTER_MAX))
+	if (WARN_ON_ONCE(block >= ARRAY_SIZE(rkcif->match_data->mipi->blocks)) ||
+	    WARN_ON_ONCE(id >= ARRAY_SIZE(rkcif->match_data->mipi->regs_id)) ||
+	    WARN_ON_ONCE(index >= ARRAY_SIZE(rkcif->match_data->mipi->regs_id[id])))
 		return RKCIF_REGISTER_NOTSUPPORTED;
 
 	offset = rkcif->match_data->mipi->blocks[block].offset;
diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
index 523103872b7a1..414a9980cf2e5 100644
--- a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
+++ b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
@@ -378,7 +378,8 @@ int rkcif_interface_register(struct rkcif_device *rkcif,
 		snprintf(sd->name, sizeof(sd->name), "rkcif-mipi%d",
 			 interface->index - RKCIF_MIPI_BASE);
 
-	pads[RKCIF_IF_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+	pads[RKCIF_IF_PAD_SINK].flags = MEDIA_PAD_FL_SINK |
+					MEDIA_PAD_FL_MUST_CONNECT;
 	pads[RKCIF_IF_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
 	ret = media_entity_pads_init(&sd->entity, RKCIF_IF_PAD_MAX, pads);
 	if (ret)
diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-stream.c b/drivers/media/platform/rockchip/rkcif/rkcif-stream.c
index e00010a91e8be..3130d420ad559 100644
--- a/drivers/media/platform/rockchip/rkcif/rkcif-stream.c
+++ b/drivers/media/platform/rockchip/rkcif/rkcif-stream.c
@@ -106,42 +106,42 @@ static int rkcif_stream_init_buffers(struct rkcif_stream *stream)
 {
 	struct v4l2_pix_format_mplane *pix = &stream->pix;
 
-	stream->buffers[0] = rkcif_stream_pop_buffer(stream);
-	if (!stream->buffers[0])
-		goto err_buff_0;
-
-	stream->buffers[1] = rkcif_stream_pop_buffer(stream);
-	if (!stream->buffers[1])
-		goto err_buff_1;
-
-	if (stream->queue_buffer) {
-		stream->queue_buffer(stream, 0);
-		stream->queue_buffer(stream, 1);
-	}
-
 	stream->dummy.size = pix->num_planes * pix->plane_fmt[0].sizeimage;
 	stream->dummy.vaddr =
 		dma_alloc_attrs(stream->rkcif->dev, stream->dummy.size,
 				&stream->dummy.buffer.buff_addr[0], GFP_KERNEL,
 				DMA_ATTR_NO_KERNEL_MAPPING);
 	if (!stream->dummy.vaddr)
-		goto err_dummy;
+		return -ENOMEM;
 
 	for (unsigned int i = 1; i < pix->num_planes; i++)
 		stream->dummy.buffer.buff_addr[i] =
 			stream->dummy.buffer.buff_addr[i - 1] +
 			pix->plane_fmt[i - 1].bytesperline * pix->height;
 
-	return 0;
+	stream->buffers[0] = rkcif_stream_pop_buffer(stream);
+	if (!stream->buffers[0])
+		goto err_dummy_free;
+
+	stream->buffers[1] = rkcif_stream_pop_buffer(stream);
+	if (!stream->buffers[1]) {
+		stream->buffers[1] = &stream->dummy.buffer;
+		stream->buffers[1]->is_dummy = true;
+	}
 
-err_dummy:
-	rkcif_stream_return_buffer(stream->buffers[1], VB2_BUF_STATE_QUEUED);
-	stream->buffers[1] = NULL;
+	if (stream->queue_buffer) {
+		stream->queue_buffer(stream, 0);
+		stream->queue_buffer(stream, 1);
+	}
+
+	return 0;
 
-err_buff_1:
-	rkcif_stream_return_buffer(stream->buffers[0], VB2_BUF_STATE_QUEUED);
-	stream->buffers[0] = NULL;
-err_buff_0:
+err_dummy_free:
+	dma_free_attrs(stream->rkcif->dev, stream->dummy.size,
+		       stream->dummy.vaddr,
+		       stream->dummy.buffer.buff_addr[0],
+		       DMA_ATTR_NO_KERNEL_MAPPING);
+	stream->dummy.vaddr = NULL;
 	return -EINVAL;
 }
 
@@ -555,7 +555,7 @@ int rkcif_stream_register(struct rkcif_device *rkcif,
 	vdev->vfl_dir = VFL_DIR_RX;
 	video_set_drvdata(vdev, stream);
 
-	stream->pad.flags = MEDIA_PAD_FL_SINK;
+	stream->pad.flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT;
 
 	stream->pix.height = CIF_MIN_HEIGHT;
 	stream->pix.width = CIF_MIN_WIDTH;
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
index 28267ee301903..3119f3bc9f98b 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
@@ -500,11 +500,15 @@ void rkvdec_hevc_run_preamble(struct rkvdec_ctx *ctx,
 		ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl,
 				      V4L2_CID_STATELESS_HEVC_EXT_SPS_ST_RPS);
 		run->ext_sps_st_rps = ctrl ? ctrl->p_cur.p : NULL;
+	} else {
+		run->ext_sps_st_rps = NULL;
 	}
 	if (ctx->has_sps_lt_rps) {
 		ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl,
 				      V4L2_CID_STATELESS_HEVC_EXT_SPS_LT_RPS);
 		run->ext_sps_lt_rps = ctrl ? ctrl->p_cur.p : NULL;
+	} else {
+		run->ext_sps_lt_rps = NULL;
 	}
 
 	rkvdec_run_preamble(ctx, &run->base);
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
index 97f1efde2e47e..fb4f849d73668 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
@@ -130,7 +130,7 @@ struct rkvdec_h264_ctx {
 	struct vdpu383_regs_h26x regs;
 };
 
-static void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_dpb_entry *dpb)
+static noinline_for_stack void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_dpb_entry *dpb)
 {
 	pps->top_field_order_cnt0 = dpb[0].top_field_order_cnt;
 	pps->bot_field_order_cnt0 = dpb[0].bottom_field_order_cnt;
@@ -166,6 +166,31 @@ static void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_d
 	pps->bot_field_order_cnt15 = dpb[15].bottom_field_order_cnt;
 }
 
+static noinline_for_stack void set_dec_params(struct rkvdec_pps *pps, const struct v4l2_ctrl_h264_decode_params *dec_params)
+{
+	const struct v4l2_h264_dpb_entry *dpb = dec_params->dpb;
+
+	for (int i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
+		if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM)
+			pps->is_longterm |= (1 << i);
+		pps->ref_field_flags |=
+		 (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)) << i;
+		pps->ref_colmv_use_flag |=
+		 (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) << i;
+		pps->ref_topfield_used |=
+		 (!!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF)) << i;
+		pps->ref_botfield_used |=
+			(!!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)) << i;
+	}
+	pps->pic_field_flag =
+		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC);
+	pps->pic_associated_flag =
+		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD);
+
+	pps->cur_top_field = dec_params->top_field_order_cnt;
+	pps->cur_bot_field = dec_params->bottom_field_order_cnt;
+}
+
 static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 			    struct rkvdec_h264_run *run)
 {
@@ -177,7 +202,6 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	struct rkvdec_h264_priv_tbl *priv_tbl = h264_ctx->priv_tbl.cpu;
 	struct rkvdec_sps_pps *hw_ps;
 	u32 pic_width, pic_height;
-	u32 i;
 
 	/*
 	 * HW read the SPS/PPS information from PPS packet index by PPS id.
@@ -261,28 +285,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 		!!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT);
 
 	set_field_order_cnt(&hw_ps->pps, dpb);
+	set_dec_params(&hw_ps->pps, dec_params);
 
-	for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
-		if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM)
-			hw_ps->pps.is_longterm |= (1 << i);
-
-		hw_ps->pps.ref_field_flags |=
-			(!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)) << i;
-		hw_ps->pps.ref_colmv_use_flag |=
-			(!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) << i;
-		hw_ps->pps.ref_topfield_used |=
-			(!!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF)) << i;
-		hw_ps->pps.ref_botfield_used |=
-			(!!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)) << i;
-	}
-
-	hw_ps->pps.pic_field_flag =
-		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC);
-	hw_ps->pps.pic_associated_flag =
-		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD);
-
-	hw_ps->pps.cur_top_field = dec_params->top_field_order_cnt;
-	hw_ps->pps.cur_bot_field = dec_params->bottom_field_order_cnt;
 }
 
 static void rkvdec_write_regs(struct rkvdec_ctx *ctx)
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c
index e4cdd2122873e..2751f5396ee80 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vp9.c
@@ -893,7 +893,8 @@ static void rkvdec_vp9_done(struct rkvdec_ctx *ctx,
 	update_ctx_last_info(vp9_ctx);
 }
 
-static void rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx)
+static noinline_for_stack void
+rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx)
 {
 	struct rkvdec_vp9_ctx *vp9_ctx = ctx->priv;
 	struct rkvdec_vp9_intra_frame_symbol_counts *intra_cnts = vp9_ctx->count_tbl.cpu;
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-core.c b/drivers/media/platform/samsung/exynos4-is/fimc-core.c
index 2c9edd0a559b8..57e573949e54d 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-core.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-core.c
@@ -1020,11 +1020,15 @@ static int fimc_probe(struct platform_device *pdev)
 static int fimc_runtime_resume(struct device *dev)
 {
 	struct fimc_dev *fimc =	dev_get_drvdata(dev);
+	int ret;
 
 	dbg("fimc%d: state: 0x%lx", fimc->id, fimc->state);
 
 	/* Enable clocks and perform basic initialization */
-	clk_enable(fimc->clock[CLK_GATE]);
+	ret = clk_enable(fimc->clock[CLK_GATE]);
+	if (ret)
+		return ret;
+
 	fimc_hw_reset(fimc);
 
 	/* Resume the capture or mem-to-mem device */
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is.c b/drivers/media/platform/samsung/exynos4-is/fimc-is.c
index 0827fdaf455ad..b7d9bfedd590f 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-is.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-is.c
@@ -200,22 +200,19 @@ static int fimc_is_parse_sensor_config(struct fimc_is *is, unsigned int index,
 
 static int fimc_is_register_subdevs(struct fimc_is *is)
 {
-	struct device_node *i2c_bus, *child;
 	int ret, index = 0;
 
 	ret = fimc_isp_subdev_create(&is->isp);
 	if (ret < 0)
 		return ret;
 
-	for_each_compatible_node(i2c_bus, NULL, "samsung,exynos4212-i2c-isp") {
-		for_each_available_child_of_node(i2c_bus, child) {
+	for_each_compatible_node_scoped(i2c_bus, NULL, "samsung,exynos4212-i2c-isp") {
+		for_each_available_child_of_node_scoped(i2c_bus, child) {
 			ret = fimc_is_parse_sensor_config(is, index, child);
 
-			if (ret < 0 || index >= FIMC_IS_SENSORS_NUM) {
-				of_node_put(child);
-				of_node_put(i2c_bus);
+			if (ret < 0 || index >= FIMC_IS_SENSORS_NUM)
 				return ret;
-			}
+
 			index++;
 		}
 	}
diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
index 13762861b7694..e5663fbe64220 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
@@ -15,6 +15,7 @@
 #include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/dmaengine.h>
+#include <linux/genalloc.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
@@ -113,6 +114,9 @@ struct dcmi_buf {
 	struct vb2_v4l2_buffer	vb;
 	bool			prepared;
 	struct sg_table		sgt;
+	struct sg_table		sgt_mdma;
+	struct dma_async_tx_descriptor *dma_desc;
+	struct dma_async_tx_descriptor *mdma_desc;
 	size_t			size;
 	struct list_head	list;
 };
@@ -158,14 +162,20 @@ struct stm32_dcmi {
 	struct dma_chan			*dma_chan;
 	dma_cookie_t			dma_cookie;
 	u32				dma_max_burst;
+
+	/* Elements for the MDMA - DMA chaining */
+	struct gen_pool			*sram_pool;
+	struct dma_chan			*mdma_chan;
+	void				*sram_buf;
+	u32				sram_buf_size;
+	dma_addr_t			sram_dma_buf;
+	dma_cookie_t			mdma_cookie;
+
 	u32				misr;
 	int				errors_count;
 	int				overrun_count;
 	int				buffers_count;
 
-	/* Ensure DMA operations atomicity */
-	struct mutex			dma_lock;
-
 	struct media_device		mdev;
 	struct media_pad		vid_cap_pad;
 	struct media_pipeline		pipeline;
@@ -229,127 +239,42 @@ static int dcmi_restart_capture(struct stm32_dcmi *dcmi)
 {
 	struct dcmi_buf *buf;
 
-	spin_lock_irq(&dcmi->irqlock);
-
-	if (dcmi->state != RUNNING) {
-		spin_unlock_irq(&dcmi->irqlock);
-		return -EINVAL;
-	}
+	/* Nothing to do if we are not running */
+	if (dcmi->state != RUNNING)
+		return 0;
 
 	/* Restart a new DMA transfer with next buffer */
 	if (list_empty(&dcmi->buffers)) {
 		dev_dbg(dcmi->dev, "Capture restart is deferred to next buffer queueing\n");
 		dcmi->state = WAIT_FOR_BUFFER;
-		spin_unlock_irq(&dcmi->irqlock);
 		return 0;
 	}
 	buf = list_entry(dcmi->buffers.next, struct dcmi_buf, list);
 	dcmi->active = buf;
 
-	spin_unlock_irq(&dcmi->irqlock);
-
 	return dcmi_start_capture(dcmi, buf);
 }
 
-static void dcmi_dma_callback(void *param)
-{
-	struct stm32_dcmi *dcmi = (struct stm32_dcmi *)param;
-	struct dma_tx_state state;
-	enum dma_status status;
-	struct dcmi_buf *buf = dcmi->active;
-
-	spin_lock_irq(&dcmi->irqlock);
-
-	/* Check DMA status */
-	status = dmaengine_tx_status(dcmi->dma_chan, dcmi->dma_cookie, &state);
-
-	switch (status) {
-	case DMA_IN_PROGRESS:
-		dev_dbg(dcmi->dev, "%s: Received DMA_IN_PROGRESS\n", __func__);
-		break;
-	case DMA_PAUSED:
-		dev_err(dcmi->dev, "%s: Received DMA_PAUSED\n", __func__);
-		break;
-	case DMA_ERROR:
-		dev_err(dcmi->dev, "%s: Received DMA_ERROR\n", __func__);
-
-		/* Return buffer to V4L2 in error state */
-		dcmi_buffer_done(dcmi, buf, 0, -EIO);
-		break;
-	case DMA_COMPLETE:
-		dev_dbg(dcmi->dev, "%s: Received DMA_COMPLETE\n", __func__);
-
-		/* Return buffer to V4L2 */
-		dcmi_buffer_done(dcmi, buf, buf->size, 0);
-
-		spin_unlock_irq(&dcmi->irqlock);
-
-		/* Restart capture */
-		if (dcmi_restart_capture(dcmi))
-			dev_err(dcmi->dev, "%s: Cannot restart capture on DMA complete\n",
-				__func__);
-		return;
-	default:
-		dev_err(dcmi->dev, "%s: Received unknown status\n", __func__);
-		break;
-	}
-
-	spin_unlock_irq(&dcmi->irqlock);
-}
-
 static int dcmi_start_dma(struct stm32_dcmi *dcmi,
 			  struct dcmi_buf *buf)
 {
-	struct dma_async_tx_descriptor *desc = NULL;
-	struct dma_slave_config config;
-	int ret;
-
-	memset(&config, 0, sizeof(config));
-
-	config.src_addr = (dma_addr_t)dcmi->res->start + DCMI_DR;
-	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	config.dst_maxburst = 4;
-
-	/* Configure DMA channel */
-	ret = dmaengine_slave_config(dcmi->dma_chan, &config);
-	if (ret < 0) {
-		dev_err(dcmi->dev, "%s: DMA channel config failed (%d)\n",
-			__func__, ret);
-		return ret;
-	}
-
-	/*
-	 * Avoid call of dmaengine_terminate_sync() between
-	 * dmaengine_prep_slave_single() and dmaengine_submit()
-	 * by locking the whole DMA submission sequence
-	 */
-	mutex_lock(&dcmi->dma_lock);
-
-	/* Prepare a DMA transaction */
-	desc = dmaengine_prep_slave_sg(dcmi->dma_chan, buf->sgt.sgl, buf->sgt.nents,
-				       DMA_DEV_TO_MEM,
-				       DMA_PREP_INTERRUPT);
-	if (!desc) {
-		dev_err(dcmi->dev, "%s: DMA dmaengine_prep_slave_sg failed\n", __func__);
-		mutex_unlock(&dcmi->dma_lock);
-		return -EINVAL;
+	/* Push current DMA transaction in the pending queue */
+	if (dcmi->mdma_chan) {
+		dcmi->mdma_cookie = dmaengine_submit(buf->mdma_desc);
+		if (dma_submit_error(dcmi->mdma_cookie)) {
+			dev_err(dcmi->dev, "%s: MDMA submission failed\n", __func__);
+			return -ENXIO;
+		}
 	}
 
-	/* Set completion callback routine for notification */
-	desc->callback = dcmi_dma_callback;
-	desc->callback_param = dcmi;
-
-	/* Push current DMA transaction in the pending queue */
-	dcmi->dma_cookie = dmaengine_submit(desc);
+	dcmi->dma_cookie = dmaengine_submit(buf->dma_desc);
 	if (dma_submit_error(dcmi->dma_cookie)) {
 		dev_err(dcmi->dev, "%s: DMA submission failed\n", __func__);
-		mutex_unlock(&dcmi->dma_lock);
 		return -ENXIO;
 	}
 
-	mutex_unlock(&dcmi->dma_lock);
-
+	if (dcmi->mdma_chan)
+		dma_async_issue_pending(dcmi->mdma_chan);
 	dma_async_issue_pending(dcmi->dma_chan);
 
 	return 0;
@@ -396,9 +321,11 @@ static void dcmi_set_crop(struct stm32_dcmi *dcmi)
 	reg_set(dcmi->regs, DCMI_CR, CR_CROP);
 }
 
-static void dcmi_process_jpeg(struct stm32_dcmi *dcmi)
+static void dcmi_process_frame(struct stm32_dcmi *dcmi)
 {
-	struct dma_tx_state state;
+	struct dma_tx_state state, state_dma;
+	size_t bytes_used;
+
 	enum dma_status status;
 	struct dcmi_buf *buf = dcmi->active;
 
@@ -406,38 +333,48 @@ static void dcmi_process_jpeg(struct stm32_dcmi *dcmi)
 		return;
 
 	/*
-	 * Because of variable JPEG buffer size sent by sensor,
-	 * DMA transfer never completes due to transfer size never reached.
-	 * In order to ensure that all the JPEG data are transferred
-	 * in active buffer memory, DMA is drained.
-	 * Then DMA tx status gives the amount of data transferred
-	 * to memory, which is then returned to V4L2 through the active
-	 * buffer payload.
+	 * Pause the DMA transfer, leading to trigger of the MDMA channel read while
+	 * keeping a valid residue value on the dma channel
 	 */
+	if (dcmi->mdma_chan) {
+		spin_unlock_irq(&dcmi->irqlock);
+		dmaengine_pause(dcmi->dma_chan);
+		spin_lock_irq(&dcmi->irqlock);
+
+		do {
+			status = dmaengine_tx_status(dcmi->mdma_chan, dcmi->mdma_cookie, &state);
+			cpu_relax();
+		} while (status != DMA_ERROR && status != DMA_COMPLETE &&
+			 state.residue && state.in_flight_bytes);
+	} else {
+		status = dmaengine_tx_status(dcmi->dma_chan, dcmi->dma_cookie, &state);
+	}
 
-	/* Drain DMA */
-	dmaengine_synchronize(dcmi->dma_chan);
-
-	/* Get DMA residue to get JPEG size */
-	status = dmaengine_tx_status(dcmi->dma_chan, dcmi->dma_cookie, &state);
 	if (status != DMA_ERROR && state.residue < buf->size) {
-		/* Return JPEG buffer to V4L2 with received JPEG buffer size */
-		dcmi_buffer_done(dcmi, buf, buf->size - state.residue, 0);
+		bytes_used = buf->size - state.residue;
+
+		if (state.residue && dcmi->mdma_chan) {
+			dmaengine_tx_status(dcmi->dma_chan, dcmi->dma_cookie, &state_dma);
+			/* Getting full size residue equal to no residue */
+			if (state_dma.residue == dcmi->sram_buf_size)
+				state_dma.residue = 0;
+			bytes_used -= state_dma.residue;
+		}
+
+		/* Return buffer to V4L2 with received data size */
+		dcmi_buffer_done(dcmi, buf, bytes_used, 0);
 	} else {
 		dcmi->errors_count++;
-		dev_err(dcmi->dev, "%s: Cannot get JPEG size from DMA\n",
-			__func__);
-		/* Return JPEG buffer to V4L2 in ERROR state */
+		dev_err(dcmi->dev, "%s: DMA error. status: 0x%x, residue: %d\n",
+			__func__, status, state.residue);
+		/* Return buffer to V4L2 in ERROR state */
 		dcmi_buffer_done(dcmi, buf, 0, -EIO);
 	}
 
 	/* Abort DMA operation */
-	dmaengine_terminate_sync(dcmi->dma_chan);
-
-	/* Restart capture */
-	if (dcmi_restart_capture(dcmi))
-		dev_err(dcmi->dev, "%s: Cannot restart capture on JPEG received\n",
-			__func__);
+	dmaengine_terminate_async(dcmi->dma_chan);
+	if (dcmi->mdma_chan)
+		dmaengine_terminate_async(dcmi->mdma_chan);
 }
 
 static irqreturn_t dcmi_irq_thread(int irq, void *arg)
@@ -447,19 +384,31 @@ static irqreturn_t dcmi_irq_thread(int irq, void *arg)
 	spin_lock_irq(&dcmi->irqlock);
 
 	if (dcmi->misr & IT_OVR) {
+		/* Disable capture */
+		reg_clear(dcmi->regs, DCMI_CR, CR_CAPTURE);
+
 		dcmi->overrun_count++;
+
 		if (dcmi->overrun_count > OVERRUN_ERROR_THRESHOLD)
 			dcmi->errors_count++;
+
+		dmaengine_terminate_async(dcmi->dma_chan);
+		if (dcmi->mdma_chan)
+			dmaengine_terminate_async(dcmi->mdma_chan);
+
+		if (dcmi_restart_capture(dcmi))
+			dev_err(dcmi->dev, "%s: Cannot restart capture\n", __func__);
+		spin_unlock_irq(&dcmi->irqlock);
+		return IRQ_HANDLED;
 	}
+
 	if (dcmi->misr & IT_ERR)
 		dcmi->errors_count++;
 
-	if (dcmi->sd_format->fourcc == V4L2_PIX_FMT_JPEG &&
-	    dcmi->misr & IT_FRAME) {
-		/* JPEG received */
-		spin_unlock_irq(&dcmi->irqlock);
-		dcmi_process_jpeg(dcmi);
-		return IRQ_HANDLED;
+	if (dcmi->misr & IT_FRAME) {
+		dcmi_process_frame(dcmi);
+		if (dcmi_restart_capture(dcmi))
+			dev_err(dcmi->dev, "%s: Cannot restart capture\n", __func__);
 	}
 
 	spin_unlock_irq(&dcmi->irqlock);
@@ -539,31 +488,134 @@ static int dcmi_buf_prepare(struct vb2_buffer *vb)
 	vb2_set_plane_payload(vb, 0, size);
 
 	if (!buf->prepared) {
+		u32 max_size = dcmi->dma_max_burst;
+		unsigned int dma_nents;
+
 		/* Get memory addresses */
 		buf->size = vb2_plane_size(&buf->vb.vb2_buf, 0);
-		if (buf->size > dcmi->dma_max_burst)
-			num_sgs = DIV_ROUND_UP(buf->size, dcmi->dma_max_burst);
+		if (dcmi->mdma_chan)
+			max_size = dcmi->sram_buf_size / 2;
+
+		if (buf->size > max_size)
+			num_sgs = DIV_ROUND_UP(buf->size, max_size);
+
+		/*
+		 * If we use MDMA chaining, DMA is used in cyclic mode and the scatterlist
+		 * maximum size is thus 2
+		 */
+		dma_nents = num_sgs;
+		if (dcmi->mdma_chan)
+			dma_nents = min_t(unsigned int, num_sgs, 2);
 
-		ret = sg_alloc_table(&buf->sgt, num_sgs, GFP_ATOMIC);
+		ret = sg_alloc_table(&buf->sgt, dma_nents, GFP_ATOMIC);
 		if (ret) {
-			dev_err(dcmi->dev, "sg table alloc failed\n");
+			dev_err(dcmi->dev, "sg table alloc failed for DMA\n");
 			return ret;
 		}
 
+		if (dcmi->mdma_chan) {
+			ret = sg_alloc_table(&buf->sgt_mdma, num_sgs, GFP_ATOMIC);
+			if (ret) {
+				dev_err(dcmi->dev, "sg table alloc failed for MDMA\n");
+				return ret;
+			}
+		}
+
 		dma_buf = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
 
 		dev_dbg(dcmi->dev, "buffer[%d] phy=%pad size=%zu\n",
 			vb->index, &dma_buf, buf->size);
 
-		for_each_sg(buf->sgt.sgl, sg, num_sgs, i) {
-			size_t bytes = min_t(size_t, size, dcmi->dma_max_burst);
+		for_each_sg(buf->sgt.sgl, sg, dma_nents, i) {
+			size_t bytes = min_t(size_t, size, max_size);
+
+			if (!dcmi->mdma_chan) {
+				sg_dma_address(sg) = dma_buf;
+				dma_buf += bytes;
+			} else {
+				/* Targets the beginning = first half of the sram_buf */
+				sg_dma_address(sg) = dcmi->sram_dma_buf;
+				/*
+				 * Targets the second half of the sram_bubf
+				 * for odd indexes of the item of the sg_list
+				 */
+				if (i & 1)
+					sg->dma_address += (dcmi->sram_buf_size / 2);
+			}
+			/*
+			 * In case of DMA-MDMA chaining, since the DMA is working in cyclic mode,
+			 * we need to provide two linked-list node of same size in order to have
+			 * a correct residue value computed.
+			 */
+			if (!dcmi->mdma_chan)
+				sg_dma_len(sg) = bytes;
+			else
+				sg_dma_len(sg) = dcmi->sram_buf_size / 2;
 
-			sg_dma_address(sg) = dma_buf;
-			sg_dma_len(sg) = bytes;
-			dma_buf += bytes;
 			size -= bytes;
 		}
 
+		/* Prepare a DMA transaction */
+		buf->dma_desc = dmaengine_prep_slave_sg(dcmi->dma_chan,
+							buf->sgt.sgl,
+							buf->sgt.nents,
+							DMA_DEV_TO_MEM,
+							DMA_PREP_INTERRUPT);
+		if (!buf->dma_desc) {
+			dev_err(dcmi->dev, "%s: DMA dmaengine_prep_slave_sg failed\n", __func__);
+			if (dcmi->mdma_chan)
+				sg_free_table(&buf->sgt_mdma);
+			sg_free_table(&buf->sgt);
+			return -EIO;
+		}
+
+		/* Mark the descriptor as reusable to avoid having to prepare it */
+		ret = dmaengine_desc_set_reuse(buf->dma_desc);
+		if (ret) {
+			dev_err(dcmi->dev, "%s: DMA dmaengine_desc_set_reuse failed\n", __func__);
+			dmaengine_desc_free(buf->dma_desc);
+			if (dcmi->mdma_chan)
+				sg_free_table(&buf->sgt_mdma);
+			sg_free_table(&buf->sgt);
+			return -EIO;
+		}
+
+		if (dcmi->mdma_chan) {
+			size = dcmi->fmt.fmt.pix.sizeimage;
+			for_each_sg(buf->sgt_mdma.sgl, sg, num_sgs, i) {
+				size_t bytes = min_t(size_t, size, max_size);
+
+				sg_dma_address(sg) = dma_buf;
+				sg_dma_len(sg) = bytes;
+				dma_buf += bytes;
+				size -= bytes;
+			}
+
+			buf->mdma_desc = dmaengine_prep_slave_sg(dcmi->mdma_chan, buf->sgt_mdma.sgl,
+								 buf->sgt_mdma.nents,
+								 DMA_DEV_TO_MEM,
+								 DMA_PREP_INTERRUPT);
+			if (!buf->mdma_desc) {
+				dev_err(dcmi->dev, "%s: failed dmaengine_prep_slave_sg for MDMA\n",
+					__func__);
+				dmaengine_desc_free(buf->dma_desc);
+				sg_free_table(&buf->sgt_mdma);
+				sg_free_table(&buf->sgt);
+				return -EIO;
+			}
+
+			ret = dmaengine_desc_set_reuse(buf->mdma_desc);
+			if (ret) {
+				dev_err(dcmi->dev, "%s: failed to set reuse for mdma desc\n",
+					__func__);
+				dmaengine_desc_free(buf->mdma_desc);
+				dmaengine_desc_free(buf->dma_desc);
+				sg_free_table(&buf->sgt_mdma);
+				sg_free_table(&buf->sgt);
+				return -EIO;
+			}
+		}
+
 		buf->prepared = true;
 
 		vb2_set_plane_payload(&buf->vb.vb2_buf, 0, buf->size);
@@ -572,6 +624,31 @@ static int dcmi_buf_prepare(struct vb2_buffer *vb)
 	return 0;
 }
 
+static void dcmi_buf_cleanup(struct vb2_buffer *vb)
+{
+	struct stm32_dcmi *dcmi =  vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct dcmi_buf *buf = container_of(vbuf, struct dcmi_buf, vb);
+	int ret;
+
+	if (!buf->prepared)
+		return;
+
+	if (dcmi->mdma_chan) {
+		ret = dmaengine_desc_free(buf->mdma_desc);
+		if (ret)
+			dev_err(dcmi->dev, "%s: Failed to free the mdma descriptor (0x%x)\n",
+				__func__, ret);
+		sg_free_table(&buf->sgt_mdma);
+	}
+
+	ret = dmaengine_desc_free(buf->dma_desc);
+	if (ret)
+		dev_err(dcmi->dev, "%s: Failed to free the mdma descriptor (0x%x)\n",
+			__func__, ret);
+	sg_free_table(&buf->sgt);
+}
+
 static void dcmi_buf_queue(struct vb2_buffer *vb)
 {
 	struct stm32_dcmi *dcmi =  vb2_get_drv_priv(vb->vb2_queue);
@@ -590,11 +667,9 @@ static void dcmi_buf_queue(struct vb2_buffer *vb)
 		dev_dbg(dcmi->dev, "Starting capture on buffer[%d] queued\n",
 			buf->vb.vb2_buf.index);
 
-		spin_unlock_irq(&dcmi->irqlock);
 		if (dcmi_start_capture(dcmi, buf))
 			dev_err(dcmi->dev, "%s: Cannot restart capture on overflow or error\n",
 				__func__);
-		return;
 	}
 
 	spin_unlock_irq(&dcmi->irqlock);
@@ -773,10 +848,6 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 	if (dcmi->do_crop)
 		dcmi_set_crop(dcmi);
 
-	/* Enable jpeg capture */
-	if (dcmi->sd_format->fourcc == V4L2_PIX_FMT_JPEG)
-		reg_set(dcmi->regs, DCMI_CR, CR_CM);/* Snapshot mode */
-
 	/* Enable dcmi */
 	reg_set(dcmi->regs, DCMI_CR, CR_ENABLE);
 
@@ -803,19 +874,18 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	dev_dbg(dcmi->dev, "Start streaming, starting capture\n");
 
-	spin_unlock_irq(&dcmi->irqlock);
 	ret = dcmi_start_capture(dcmi, buf);
 	if (ret) {
 		dev_err(dcmi->dev, "%s: Start streaming failed, cannot start capture\n",
 			__func__);
+		spin_unlock_irq(&dcmi->irqlock);
 		goto err_pipeline_stop;
 	}
 
 	/* Enable interruptions */
-	if (dcmi->sd_format->fourcc == V4L2_PIX_FMT_JPEG)
-		reg_set(dcmi->regs, DCMI_IER, IT_FRAME | IT_OVR | IT_ERR);
-	else
-		reg_set(dcmi->regs, DCMI_IER, IT_OVR | IT_ERR);
+	reg_set(dcmi->regs, DCMI_IER, IT_FRAME | IT_OVR | IT_ERR);
+
+	spin_unlock_irq(&dcmi->irqlock);
 
 	return 0;
 
@@ -876,9 +946,9 @@ static void dcmi_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irq(&dcmi->irqlock);
 
 	/* Stop all pending DMA operations */
-	mutex_lock(&dcmi->dma_lock);
 	dmaengine_terminate_sync(dcmi->dma_chan);
-	mutex_unlock(&dcmi->dma_lock);
+	if (dcmi->mdma_chan)
+		dmaengine_terminate_sync(dcmi->mdma_chan);
 
 	pm_runtime_put(dcmi->dev);
 
@@ -895,6 +965,7 @@ static const struct vb2_ops dcmi_video_qops = {
 	.queue_setup		= dcmi_queue_setup,
 	.buf_init		= dcmi_buf_init,
 	.buf_prepare		= dcmi_buf_prepare,
+	.buf_cleanup		= dcmi_buf_cleanup,
 	.buf_queue		= dcmi_buf_queue,
 	.start_streaming	= dcmi_start_streaming,
 	.stop_streaming		= dcmi_stop_streaming,
@@ -1886,8 +1957,9 @@ static int dcmi_probe(struct platform_device *pdev)
 	struct v4l2_fwnode_endpoint ep = { .bus_type = 0 };
 	struct stm32_dcmi *dcmi;
 	struct vb2_queue *q;
-	struct dma_chan *chan;
+	struct dma_chan *chan, *mdma_chan;
 	struct dma_slave_caps caps;
+	struct dma_slave_config dma_config, mdma_config;
 	struct clk *mclk;
 	int ret = 0;
 
@@ -1949,14 +2021,73 @@ static int dcmi_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, PTR_ERR(chan),
 				     "Failed to request DMA channel\n");
 
+	mdma_chan = dma_request_chan(&pdev->dev, "mdma_tx");
+	if (IS_ERR(mdma_chan)) {
+		ret = PTR_ERR(mdma_chan);
+		if (ret != -ENODEV)
+			return dev_err_probe(&pdev->dev, ret, "Failed to request MDMA channel\n");
+		mdma_chan = NULL;
+	}
+
+	/* Configure the DMA channel */
+	memset(&dma_config, 0, sizeof(dma_config));
+
+	dma_config.src_addr = (dma_addr_t)dcmi->res->start + DCMI_DR;
+	dma_config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	if (mdma_chan)
+		dma_config.peripheral_size = 1; /* Indicates chaining */
+
+	/* Configure DMA channel */
+	ret = dmaengine_slave_config(chan, &dma_config);
+	if (ret < 0) {
+		dev_err(dcmi->dev, "%s: DMA channel config failed (%d)\n",
+			__func__, ret);
+		goto err_dma_slave_config;
+	}
+
+	/* If we want MDMA, we also need a sram pool */
+	if (mdma_chan) {
+		dcmi->sram_pool = of_gen_pool_get(pdev->dev.of_node, "sram", 0);
+		if (!dcmi->sram_pool) {
+			dev_info(&pdev->dev, "No SRAM pool, can't use MDMA chaining\n");
+			goto err_dma_slave_config;
+		}
+
+		dev_info(&pdev->dev, "SRAM pool: %zu KiB for DMA/MDMA chaining\n",
+			 gen_pool_size(dcmi->sram_pool) / 1024);
+
+		dcmi->sram_buf_size = gen_pool_size(dcmi->sram_pool);
+		dcmi->sram_buf = gen_pool_dma_zalloc(dcmi->sram_pool, dcmi->sram_buf_size,
+						     &dcmi->sram_dma_buf);
+		if (!dcmi->sram_buf) {
+			dev_err(dcmi->dev, "Failed to allocate from SRAM\n");
+			goto err_dma_slave_config;
+		}
+
+		/* Configure the MDMA Channel */
+		memset(&mdma_config, 0, sizeof(mdma_config));
+		mdma_config.direction = DMA_DEV_TO_MEM;
+		mdma_config.src_addr = dcmi->sram_dma_buf;
+		mdma_config.peripheral_size = dma_config.peripheral_size;
+		mdma_config.peripheral_config = dma_config.peripheral_config;
+		ret = dmaengine_slave_config(mdma_chan, &mdma_config);
+		if (ret < 0) {
+			dev_err(dcmi->dev, "%s: MDMA channel config failed (%d)\n",
+				__func__, ret);
+			goto err_mdma_slave_config;
+		}
+	}
+
 	dcmi->dma_max_burst = UINT_MAX;
-	ret = dma_get_slave_caps(chan, &caps);
-	if (!ret && caps.max_sg_burst)
-		dcmi->dma_max_burst = caps.max_sg_burst	* DMA_SLAVE_BUSWIDTH_4_BYTES;
+	/* In case of using DMA-MDMA chaining we consider the maximum infini */
+	if (!mdma_chan) {
+		ret = dma_get_slave_caps(chan, &caps);
+		if (!ret && caps.max_sg_burst)
+			dcmi->dma_max_burst = caps.max_sg_burst * DMA_SLAVE_BUSWIDTH_4_BYTES;
+	}
 
 	spin_lock_init(&dcmi->irqlock);
 	mutex_init(&dcmi->lock);
-	mutex_init(&dcmi->dma_lock);
 	init_completion(&dcmi->complete);
 	INIT_LIST_HEAD(&dcmi->buffers);
 
@@ -1964,6 +2095,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	dcmi->mclk = mclk;
 	dcmi->state = STOPPED;
 	dcmi->dma_chan = chan;
+	dcmi->mdma_chan = mdma_chan;
 
 	q = &dcmi->queue;
 
@@ -2072,7 +2204,13 @@ static int dcmi_probe(struct platform_device *pdev)
 	v4l2_device_unregister(&dcmi->v4l2_dev);
 err_media_device_cleanup:
 	media_device_cleanup(&dcmi->mdev);
+err_mdma_slave_config:
+	if (dcmi->mdma_chan)
+		gen_pool_free(dcmi->sram_pool, (unsigned long)dcmi->sram_buf, dcmi->sram_buf_size);
+err_dma_slave_config:
 	dma_release_channel(dcmi->dma_chan);
+	if (dcmi->mdma_chan)
+		dma_release_channel(mdma_chan);
 
 	return ret;
 }
@@ -2090,9 +2228,13 @@ static void dcmi_remove(struct platform_device *pdev)
 	media_device_cleanup(&dcmi->mdev);
 
 	dma_release_channel(dcmi->dma_chan);
+	if (dcmi->mdma_chan) {
+		gen_pool_free(dcmi->sram_pool, (unsigned long)dcmi->sram_buf, dcmi->sram_buf_size);
+		dma_release_channel(dcmi->mdma_chan);
+	}
 }
 
-static __maybe_unused int dcmi_runtime_suspend(struct device *dev)
+static int dcmi_runtime_suspend(struct device *dev)
 {
 	struct stm32_dcmi *dcmi = dev_get_drvdata(dev);
 
@@ -2101,7 +2243,7 @@ static __maybe_unused int dcmi_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static __maybe_unused int dcmi_runtime_resume(struct device *dev)
+static int dcmi_runtime_resume(struct device *dev)
 {
 	struct stm32_dcmi *dcmi = dev_get_drvdata(dev);
 	int ret;
@@ -2113,7 +2255,7 @@ static __maybe_unused int dcmi_runtime_resume(struct device *dev)
 	return ret;
 }
 
-static __maybe_unused int dcmi_suspend(struct device *dev)
+static int dcmi_suspend(struct device *dev)
 {
 	/* disable clock */
 	pm_runtime_force_suspend(dev);
@@ -2124,7 +2266,7 @@ static __maybe_unused int dcmi_suspend(struct device *dev)
 	return 0;
 }
 
-static __maybe_unused int dcmi_resume(struct device *dev)
+static int dcmi_resume(struct device *dev)
 {
 	/* restore pinctl default state */
 	pinctrl_pm_select_default_state(dev);
@@ -2136,9 +2278,8 @@ static __maybe_unused int dcmi_resume(struct device *dev)
 }
 
 static const struct dev_pm_ops dcmi_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(dcmi_suspend, dcmi_resume)
-	SET_RUNTIME_PM_OPS(dcmi_runtime_suspend,
-			   dcmi_runtime_resume, NULL)
+	SYSTEM_SLEEP_PM_OPS(dcmi_suspend, dcmi_resume)
+	RUNTIME_PM_OPS(dcmi_runtime_suspend, dcmi_runtime_resume, NULL)
 };
 
 static struct platform_driver stm32_dcmi_driver = {
@@ -2147,7 +2288,7 @@ static struct platform_driver stm32_dcmi_driver = {
 	.driver		= {
 		.name = DRV_NAME,
 		.of_match_table = of_match_ptr(stm32_dcmi_of_match),
-		.pm = &dcmi_pm_ops,
+		.pm = pm_ptr(&dcmi_pm_ops),
 	},
 };
 
diff --git a/drivers/media/platform/synopsys/Kconfig b/drivers/media/platform/synopsys/Kconfig
index e798ec00b1897..b109de2c8111c 100644
--- a/drivers/media/platform/synopsys/Kconfig
+++ b/drivers/media/platform/synopsys/Kconfig
@@ -4,9 +4,11 @@ source "drivers/media/platform/synopsys/hdmirx/Kconfig"
 
 config VIDEO_DW_MIPI_CSI2RX
 	tristate "Synopsys DesignWare MIPI CSI-2 Receiver"
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	depends on VIDEO_DEV
 	depends on V4L_PLATFORM_DRIVERS
 	depends on PM && COMMON_CLK
+	select GENERIC_PHY_MIPI_DPHY
 	select MEDIA_CONTROLLER
 	select V4L2_FWNODE
 	select VIDEO_V4L2_SUBDEV_API
diff --git a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
index 170346ae1a599..4d96171a650b6 100644
--- a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
+++ b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
@@ -301,7 +301,7 @@ dw_mipi_csi2rx_enum_mbus_code(struct v4l2_subdev *sd,
 
 		return 0;
 	case DW_MIPI_CSI2RX_PAD_SINK:
-		if (code->index > csi2->formats_num)
+		if (code->index >= csi2->formats_num)
 			return -EINVAL;
 
 		code->code = csi2->formats[code->index].code;
diff --git a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
index 9cceffa4ce250..61ad20b18b8d6 100644
--- a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
+++ b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
@@ -232,7 +232,7 @@ static bool tx_5v_power_present(struct snps_hdmirx_dev *hdmirx_dev)
 
 	for (i = 0; i < 10; i++) {
 		usleep_range(1000, 1100);
-		val = gpiod_get_value(hdmirx_dev->detect_5v_gpio);
+		val = gpiod_get_value_cansleep(hdmirx_dev->detect_5v_gpio);
 		if (val > 0)
 			cnt++;
 		if (cnt >= detection_threshold)
@@ -2252,10 +2252,6 @@ static void hdmirx_delayed_work_res_change(struct work_struct *work)
 static irqreturn_t hdmirx_5v_det_irq_handler(int irq, void *dev_id)
 {
 	struct snps_hdmirx_dev *hdmirx_dev = dev_id;
-	u32 val;
-
-	val = gpiod_get_value(hdmirx_dev->detect_5v_gpio);
-	v4l2_dbg(3, debug, &hdmirx_dev->v4l2_dev, "%s: 5v:%d\n", __func__, val);
 
 	queue_delayed_work(system_unbound_wq,
 			   &hdmirx_dev->delayed_work_hotplug,
diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c
index 64e76e3576a86..b946c8087c778 100644
--- a/drivers/media/platform/ti/omap3isp/ispvideo.c
+++ b/drivers/media/platform/ti/omap3isp/ispvideo.c
@@ -1403,6 +1403,7 @@ static int isp_video_open(struct file *file)
 
 	ret = vb2_queue_init(&handle->queue);
 	if (ret < 0) {
+		v4l2_pipeline_pm_put(&video->video.entity);
 		omap3isp_put(video->isp);
 		goto done;
 	}
diff --git a/drivers/media/platform/ti/vpe/vip.c b/drivers/media/platform/ti/vpe/vip.c
index a4b616a5ece75..0e91e87bda9b9 100644
--- a/drivers/media/platform/ti/vpe/vip.c
+++ b/drivers/media/platform/ti/vpe/vip.c
@@ -3641,6 +3641,7 @@ static void vip_remove(struct platform_device *pdev)
 	}
 
 	v4l2_ctrl_handler_free(&shared->ctrl_handler);
+	v4l2_device_unregister(&shared->v4l2_dev);
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
index 6f8e43b7f1575..fa4224de4b996 100644
--- a/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
+++ b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c
@@ -343,7 +343,7 @@ const struct hantro_variant imx8mq_vpu_variant = {
 	.num_regs = ARRAY_SIZE(imx8mq_reg_names)
 };
 
-static const struct of_device_id imx8mq_vpu_shared_resources[] __initconst = {
+static const struct of_device_id imx8mq_vpu_shared_resources[] = {
 	{ .compatible = "nxp,imx8mq-vpu-g1", },
 	{ .compatible = "nxp,imx8mq-vpu-g2", },
 	{ /* sentinel */ }
diff --git a/drivers/media/rc/ati_remote.c b/drivers/media/rc/ati_remote.c
index 78abe810a88e7..51d85de24fae3 100644
--- a/drivers/media/rc/ati_remote.c
+++ b/drivers/media/rc/ati_remote.c
@@ -921,7 +921,6 @@ static int ati_remote_probe(struct usb_interface *interface,
 	input_free_device(input_dev);
  exit_unregister_device:
 	rc_unregister_device(rc_dev);
-	rc_dev = NULL;
  exit_kill_urbs:
 	usb_kill_urb(ati_remote->irq_urb);
 	usb_kill_urb(ati_remote->out_urb);
@@ -941,18 +940,19 @@ static void ati_remote_disconnect(struct usb_interface *interface)
 	struct ati_remote *ati_remote;
 
 	ati_remote = usb_get_intfdata(interface);
-	usb_set_intfdata(interface, NULL);
 	if (!ati_remote) {
 		dev_warn(&interface->dev, "%s - null device?\n", __func__);
 		return;
 	}
 
+	rc_unregister_device(ati_remote->rdev);
+	usb_set_intfdata(interface, NULL);
 	usb_kill_urb(ati_remote->irq_urb);
 	usb_kill_urb(ati_remote->out_urb);
 	if (ati_remote->idev)
 		input_unregister_device(ati_remote->idev);
-	rc_unregister_device(ati_remote->rdev);
 	ati_remote_free_buffers(ati_remote);
+	rc_free_device(ati_remote->rdev);
 	kfree(ati_remote);
 }
 
diff --git a/drivers/media/rc/ene_ir.c b/drivers/media/rc/ene_ir.c
index f8120605501ab..6f7dccc965e7f 100644
--- a/drivers/media/rc/ene_ir.c
+++ b/drivers/media/rc/ene_ir.c
@@ -1090,7 +1090,6 @@ static int ene_probe(struct pnp_dev *pnp_dev, const struct pnp_device_id *id)
 	release_region(dev->hw_io, ENE_IO_SIZE);
 exit_unregister_device:
 	rc_unregister_device(rdev);
-	rdev = NULL;
 exit_free_dev_rdev:
 	rc_free_device(rdev);
 	kfree(dev);
@@ -1110,6 +1109,7 @@ static void ene_remove(struct pnp_dev *pnp_dev)
 	ene_rx_restore_hw_buffer(dev);
 	spin_unlock_irqrestore(&dev->hw_lock, flags);
 
+	rc_free_device(dev->rdev);
 	free_irq(dev->irq, dev);
 	release_region(dev->hw_io, ENE_IO_SIZE);
 	kfree(dev);
diff --git a/drivers/media/rc/fintek-cir.c b/drivers/media/rc/fintek-cir.c
index f7cfa8a073ebc..5055dfc3f4651 100644
--- a/drivers/media/rc/fintek-cir.c
+++ b/drivers/media/rc/fintek-cir.c
@@ -568,6 +568,7 @@ static void fintek_remove(struct pnp_dev *pdev)
 	struct fintek_dev *fintek = pnp_get_drvdata(pdev);
 	unsigned long flags;
 
+	rc_unregister_device(fintek->rdev);
 	spin_lock_irqsave(&fintek->fintek_lock, flags);
 	/* disable CIR */
 	fintek_disable_cir(fintek);
@@ -580,7 +581,7 @@ static void fintek_remove(struct pnp_dev *pdev)
 	free_irq(fintek->cir_irq, fintek);
 	release_region(fintek->cir_addr, fintek->cir_port_len);
 
-	rc_unregister_device(fintek->rdev);
+	rc_free_device(fintek->rdev);
 
 	kfree(fintek);
 }
diff --git a/drivers/media/rc/gpio-ir-tx.c b/drivers/media/rc/gpio-ir-tx.c
index e185ead404644..d15fb17fa0fa1 100644
--- a/drivers/media/rc/gpio-ir-tx.c
+++ b/drivers/media/rc/gpio-ir-tx.c
@@ -51,7 +51,7 @@ static int gpio_ir_tx_set_carrier(struct rc_dev *dev, u32 carrier)
 static void delay_until(ktime_t until)
 {
 	/*
-	 * delta should never exceed 0.5 seconds (IR_MAX_DURATION) and on
+	 * delta should never exceed 1 second (IR_MAX_DURATION) and on
 	 * m68k ndelay(s64) does not compile; so use s32 rather than s64.
 	 */
 	s32 delta;
@@ -95,7 +95,7 @@ static void gpio_ir_tx_modulated(struct gpio_ir *gpio_ir, uint *txbuf,
 {
 	ktime_t edge;
 	/*
-	 * delta should never exceed 0.5 seconds (IR_MAX_DURATION) and on
+	 * delta should never exceed 1 second (IR_MAX_DURATION) and on
 	 * m68k ndelay(s64) does not compile; so use s32 rather than s64.
 	 */
 	s32 delta;
diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c
index e034c93d57cf0..3e10f6fe89f83 100644
--- a/drivers/media/rc/igorplugusb.c
+++ b/drivers/media/rc/igorplugusb.c
@@ -34,7 +34,7 @@ struct igorplugusb {
 	struct device *dev;
 
 	struct urb *urb;
-	struct usb_ctrlrequest request;
+	struct usb_ctrlrequest *request;
 
 	struct timer_list timer;
 
@@ -122,7 +122,7 @@ static void igorplugusb_cmd(struct igorplugusb *ir, int cmd)
 {
 	int ret;
 
-	ir->request.bRequest = cmd;
+	ir->request->bRequest = cmd;
 	ir->urb->transfer_flags = 0;
 	ret = usb_submit_urb(ir->urb, GFP_ATOMIC);
 	if (ret && ret != -EPERM)
@@ -164,13 +164,17 @@ static int igorplugusb_probe(struct usb_interface *intf,
 	if (!ir)
 		return -ENOMEM;
 
+	ir->request = kzalloc_obj(*ir->request, GFP_KERNEL);
+	if (!ir->request)
+		goto fail;
+
 	ir->dev = &intf->dev;
 
 	timer_setup(&ir->timer, igorplugusb_timer, 0);
 
-	ir->request.bRequest = GET_INFRACODE;
-	ir->request.bRequestType = USB_TYPE_VENDOR | USB_DIR_IN;
-	ir->request.wLength = cpu_to_le16(MAX_PACKET);
+	ir->request->bRequest = GET_INFRACODE;
+	ir->request->bRequestType = USB_TYPE_VENDOR | USB_DIR_IN;
+	ir->request->wLength = cpu_to_le16(MAX_PACKET);
 
 	ir->urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!ir->urb)
@@ -228,6 +232,7 @@ static int igorplugusb_probe(struct usb_interface *intf,
 	usb_free_urb(ir->urb);
 	rc_free_device(ir->rc);
 	kfree(ir->buf_in);
+	kfree(ir->request);
 
 	return ret;
 }
@@ -242,7 +247,9 @@ static void igorplugusb_disconnect(struct usb_interface *intf)
 	usb_set_intfdata(intf, NULL);
 	usb_unpoison_urb(ir->urb);
 	usb_free_urb(ir->urb);
+	rc_free_device(ir->rc);
 	kfree(ir->buf_in);
+	kfree(ir->request);
 }
 
 static const struct usb_device_id igorplugusb_table[] = {
diff --git a/drivers/media/rc/iguanair.c b/drivers/media/rc/iguanair.c
index c508f2536243e..0c5b8befb0af3 100644
--- a/drivers/media/rc/iguanair.c
+++ b/drivers/media/rc/iguanair.c
@@ -500,6 +500,7 @@ static void iguanair_disconnect(struct usb_interface *intf)
 	usb_set_intfdata(intf, NULL);
 	usb_kill_urb(ir->urb_in);
 	usb_kill_urb(ir->urb_out);
+	rc_free_device(ir->rc);
 	usb_free_urb(ir->urb_in);
 	usb_free_urb(ir->urb_out);
 	usb_free_coherent(ir->udev, MAX_IN_PACKET, ir->buf_in, ir->dma_in);
diff --git a/drivers/media/rc/img-ir/img-ir-hw.c b/drivers/media/rc/img-ir/img-ir-hw.c
index 63f6f5b36838d..f30adf4d8444d 100644
--- a/drivers/media/rc/img-ir/img-ir-hw.c
+++ b/drivers/media/rc/img-ir/img-ir-hw.c
@@ -1118,9 +1118,10 @@ void img_ir_remove_hw(struct img_ir_priv *priv)
 	struct rc_dev *rdev = hw->rdev;
 	if (!rdev)
 		return;
+	rc_unregister_device(rdev);
 	img_ir_set_decoder(priv, NULL, 0);
 	hw->rdev = NULL;
-	rc_unregister_device(rdev);
+	rc_free_device(rdev);
 #ifdef CONFIG_COMMON_CLK
 	if (!IS_ERR(priv->clk))
 		clk_notifier_unregister(priv->clk, &hw->clk_nb);
diff --git a/drivers/media/rc/img-ir/img-ir-raw.c b/drivers/media/rc/img-ir/img-ir-raw.c
index 92fb7b555a0f6..f1460d4acf3e8 100644
--- a/drivers/media/rc/img-ir/img-ir-raw.c
+++ b/drivers/media/rc/img-ir/img-ir-raw.c
@@ -136,6 +136,7 @@ void img_ir_remove_raw(struct img_ir_priv *priv)
 	if (!rdev)
 		return;
 
+	rc_unregister_device(rdev);
 	/* switch off and disable raw (edge) interrupts */
 	spin_lock_irq(&priv->lock);
 	raw->rdev = NULL;
@@ -145,7 +146,7 @@ void img_ir_remove_raw(struct img_ir_priv *priv)
 	img_ir_write(priv, IMG_IR_IRQ_CLEAR, IMG_IR_IRQ_EDGE);
 	spin_unlock_irq(&priv->lock);
 
-	rc_unregister_device(rdev);
+	rc_free_device(rdev);
 
 	timer_delete_sync(&raw->timer);
 }
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index 7e92161105d53..9bb27ba8240f8 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -2250,7 +2250,7 @@ static struct imon_context *imon_init_intf0(struct usb_interface *intf,
 	mutex_lock(&ictx->lock);
 
 	ictx->dev = dev;
-	ictx->usbdev_intf0 = usb_get_dev(interface_to_usbdev(intf));
+	ictx->usbdev_intf0 = interface_to_usbdev(intf);
 	ictx->rx_urb_intf0 = rx_urb;
 	ictx->tx_urb = tx_urb;
 	ictx->rf_device = false;
@@ -2308,7 +2308,6 @@ static struct imon_context *imon_init_intf0(struct usb_interface *intf,
 	usb_kill_urb(ictx->rx_urb_intf0);
 urb_submit_failed:
 find_endpoint_failed:
-	usb_put_dev(ictx->usbdev_intf0);
 	mutex_unlock(&ictx->lock);
 	usb_free_urb(tx_urb);
 tx_urb_alloc_failed:
@@ -2338,7 +2337,7 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 		timer_setup(&ictx->ttimer, imon_touch_display_timeout, 0);
 	}
 
-	ictx->usbdev_intf1 = usb_get_dev(interface_to_usbdev(intf));
+	ictx->usbdev_intf1 = interface_to_usbdev(intf);
 	ictx->rx_urb_intf1 = rx_urb;
 
 	ret = -ENODEV;
@@ -2377,7 +2376,6 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 		input_unregister_device(ictx->touch);
 touch_setup_failed:
 find_endpoint_failed:
-	usb_put_dev(ictx->usbdev_intf1);
 	ictx->usbdev_intf1 = NULL;
 	mutex_unlock(&ictx->lock);
 	usb_free_urb(rx_urb);
@@ -2426,7 +2424,7 @@ static int imon_probe(struct usb_interface *interface,
 	struct imon_context *ictx = NULL;
 	u16 vendor, product;
 
-	usbdev     = usb_get_dev(interface_to_usbdev(interface));
+	usbdev     = interface_to_usbdev(interface);
 	iface_desc = interface->cur_altsetting;
 	ifnum      = iface_desc->desc.bInterfaceNumber;
 	vendor     = le16_to_cpu(usbdev->descriptor.idVendor);
@@ -2495,12 +2493,9 @@ static int imon_probe(struct usb_interface *interface,
 		 vendor, product, ifnum,
 		 usbdev->bus->busnum, usbdev->devnum);
 
-	usb_put_dev(usbdev);
-
 	return 0;
 
 fail:
-	usb_put_dev(usbdev);
 	dev_err(dev, "unable to register, err %d\n", ret);
 
 	return ret;
@@ -2541,16 +2536,16 @@ static void imon_disconnect(struct usb_interface *interface)
 
 	if (ifnum == 0) {
 		ictx->dev_present_intf0 = false;
+		rc_unregister_device(ictx->rdev);
 		usb_kill_urb(ictx->rx_urb_intf0);
 		input_unregister_device(ictx->idev);
-		rc_unregister_device(ictx->rdev);
+		rc_free_device(ictx->rdev);
 		if (ictx->display_supported) {
 			if (ictx->display_type == IMON_DISPLAY_TYPE_LCD)
 				usb_deregister_dev(interface, &imon_lcd_class);
 			else if (ictx->display_type == IMON_DISPLAY_TYPE_VFD)
 				usb_deregister_dev(interface, &imon_vfd_class);
 		}
-		usb_put_dev(ictx->usbdev_intf0);
 	} else {
 		ictx->dev_present_intf1 = false;
 		usb_kill_urb(ictx->rx_urb_intf1);
@@ -2558,7 +2553,6 @@ static void imon_disconnect(struct usb_interface *interface)
 			timer_delete_sync(&ictx->ttimer);
 			input_unregister_device(ictx->touch);
 		}
-		usb_put_dev(ictx->usbdev_intf1);
 	}
 
 	if (refcount_dec_and_test(&ictx->users))
diff --git a/drivers/media/rc/ir-hix5hd2.c b/drivers/media/rc/ir-hix5hd2.c
index edc46828509c8..1b061e4a3dcfa 100644
--- a/drivers/media/rc/ir-hix5hd2.c
+++ b/drivers/media/rc/ir-hix5hd2.c
@@ -331,7 +331,6 @@ static int hix5hd2_ir_probe(struct platform_device *pdev)
 
 regerr:
 	rc_unregister_device(rdev);
-	rdev = NULL;
 clkerr:
 	clk_disable_unprepare(priv->clock);
 err:
@@ -346,6 +345,7 @@ static void hix5hd2_ir_remove(struct platform_device *pdev)
 
 	clk_disable_unprepare(priv->clock);
 	rc_unregister_device(priv->rdev);
+	rc_free_device(priv->rdev);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/media/rc/ir_toy.c b/drivers/media/rc/ir_toy.c
index d6472de5da87d..089833e411786 100644
--- a/drivers/media/rc/ir_toy.c
+++ b/drivers/media/rc/ir_toy.c
@@ -536,6 +536,7 @@ static void irtoy_disconnect(struct usb_interface *intf)
 	usb_free_urb(ir->urb_out);
 	usb_kill_urb(ir->urb_in);
 	usb_free_urb(ir->urb_in);
+	rc_free_device(ir->rc);
 	kfree(ir->in);
 	kfree(ir->out);
 	kfree(ir);
diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
index bf544517c67a9..bde2a70512310 100644
--- a/drivers/media/rc/ite-cir.c
+++ b/drivers/media/rc/ite-cir.c
@@ -1414,7 +1414,6 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 	release_region(itdev->cir_addr, itdev->params->io_region_size);
 exit_unregister_device:
 	rc_unregister_device(rdev);
-	rdev = NULL;
 exit_free_dev_rdev:
 	rc_free_device(rdev);
 	kfree(itdev);
@@ -1439,6 +1438,7 @@ static void ite_remove(struct pnp_dev *pdev)
 	release_region(dev->cir_addr, dev->params->io_region_size);
 
 	rc_unregister_device(dev->rdev);
+	rc_free_device(dev->rdev);
 
 	kfree(dev);
 }
diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
index ed55e9ec3c570..6a9e4382a2243 100644
--- a/drivers/media/rc/mceusb.c
+++ b/drivers/media/rc/mceusb.c
@@ -1729,7 +1729,7 @@ static int mceusb_dev_probe(struct usb_interface *intf,
 		goto urb_in_alloc_fail;
 
 	ir->usbintf = intf;
-	ir->usbdev = usb_get_dev(dev);
+	ir->usbdev = dev;
 	ir->dev = &intf->dev;
 	ir->len_in = maxp;
 	ir->flags.microsoft_gen1 = is_microsoft_gen1;
@@ -1817,7 +1817,6 @@ static int mceusb_dev_probe(struct usb_interface *intf,
 	/* Error-handling path */
 rc_dev_fail:
 	cancel_work_sync(&ir->kevent);
-	usb_put_dev(ir->usbdev);
 	usb_kill_urb(ir->urb_in);
 	usb_free_urb(ir->urb_in);
 urb_in_alloc_fail:
@@ -1849,7 +1848,7 @@ static void mceusb_dev_disconnect(struct usb_interface *intf)
 	usb_kill_urb(ir->urb_in);
 	usb_free_urb(ir->urb_in);
 	usb_free_coherent(dev, ir->len_in, ir->buf_in, ir->dma_in);
-	usb_put_dev(dev);
+	rc_free_device(ir->rc);
 
 	kfree(ir);
 }
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index 2e269ef5e26be..ba24c2f22d39f 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -648,9 +648,6 @@ int ir_raw_event_register(struct rc_dev *dev)
 
 void ir_raw_event_free(struct rc_dev *dev)
 {
-	if (!dev)
-		return;
-
 	kfree(dev->raw);
 	dev->raw = NULL;
 }
@@ -674,8 +671,6 @@ void ir_raw_event_unregister(struct rc_dev *dev)
 
 	lirc_bpf_free(dev);
 
-	ir_raw_event_free(dev);
-
 	/*
 	 * A user can be calling bpf(BPF_PROG_{QUERY|ATTACH|DETACH}), so
 	 * ensure that the raw member is null on unlock; this is how
diff --git a/drivers/media/rc/rc-loopback.c b/drivers/media/rc/rc-loopback.c
index 78ac09b3cbd34..53d0540717b36 100644
--- a/drivers/media/rc/rc-loopback.c
+++ b/drivers/media/rc/rc-loopback.c
@@ -263,6 +263,7 @@ static int __init loop_init(void)
 static void __exit loop_exit(void)
 {
 	rc_unregister_device(loopdev.dev);
+	rc_free_device(loopdev.dev);
 }
 
 module_init(loop_init);
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index 821607504008a..dda3479ea3add 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -1611,6 +1611,7 @@ static void rc_dev_release(struct device *device)
 {
 	struct rc_dev *dev = to_rc_dev(device);
 
+	ir_raw_event_free(dev);
 	kfree(dev);
 }
 
@@ -1773,7 +1774,6 @@ struct rc_dev *devm_rc_allocate_device(struct device *dev,
 	}
 
 	rc->dev.parent = dev;
-	rc->managed_alloc = true;
 	*dr = rc;
 	devres_add(dev, dr);
 
@@ -2042,11 +2042,7 @@ void rc_unregister_device(struct rc_dev *dev)
 	device_del(&dev->dev);
 
 	ida_free(&rc_ida, dev->minor);
-
-	if (!dev->managed_alloc)
-		rc_free_device(dev);
 }
-
 EXPORT_SYMBOL_GPL(rc_unregister_device);
 
 /*
diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c
index 3b917a2a89188..3f828a564e192 100644
--- a/drivers/media/rc/redrat3.c
+++ b/drivers/media/rc/redrat3.c
@@ -1131,11 +1131,13 @@ static void redrat3_dev_disconnect(struct usb_interface *intf)
 {
 	struct usb_device *udev = interface_to_usbdev(intf);
 	struct redrat3_dev *rr3 = usb_get_intfdata(intf);
+	struct rc_dev *rc = rr3->rc;
 
 	usb_set_intfdata(intf, NULL);
-	rc_unregister_device(rr3->rc);
+	rc_unregister_device(rc);
 	led_classdev_unregister(&rr3->led);
 	redrat3_delete(rr3, udev);
+	rc_free_device(rc);
 }
 
 static int redrat3_dev_suspend(struct usb_interface *intf, pm_message_t message)
diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c
index 6b70bac5f45d6..0ba06bfc9e14b 100644
--- a/drivers/media/rc/st_rc.c
+++ b/drivers/media/rc/st_rc.c
@@ -203,6 +203,7 @@ static void st_rc_remove(struct platform_device *pdev)
 	device_init_wakeup(&pdev->dev, false);
 	clk_disable_unprepare(rc_dev->sys_clock);
 	rc_unregister_device(rc_dev->rdev);
+	rc_free_device(rc_dev->rdev);
 }
 
 static int st_rc_open(struct rc_dev *rdev)
@@ -334,7 +335,6 @@ static int st_rc_probe(struct platform_device *pdev)
 	return ret;
 rcerr:
 	rc_unregister_device(rdev);
-	rdev = NULL;
 clkerr:
 	clk_disable_unprepare(rc_dev->sys_clock);
 err:
diff --git a/drivers/media/rc/streamzap.c b/drivers/media/rc/streamzap.c
index 5a18603f9a95c..307985d74fe8a 100644
--- a/drivers/media/rc/streamzap.c
+++ b/drivers/media/rc/streamzap.c
@@ -219,9 +219,8 @@ static void streamzap_callback(struct urb *urb)
 	case -ESHUTDOWN:
 		/*
 		 * this urb is terminated, clean up.
-		 * sz might already be invalid at this point
 		 */
-		dev_err(sz->dev, "urb terminated, status: %d\n", urb->status);
+		dev_dbg(sz->dev, "urb terminated, status: %d\n", urb->status);
 		return;
 	default:
 		break;
@@ -358,11 +357,16 @@ static int streamzap_probe(struct usb_interface *intf,
 
 	usb_set_intfdata(intf, sz);
 
-	if (usb_submit_urb(sz->urb_in, GFP_ATOMIC))
+	retval = usb_submit_urb(sz->urb_in, GFP_ATOMIC);
+	if (retval < 0) {
 		dev_err(sz->dev, "urb submit failed\n");
+		goto rc_submit_fail;
+	}
 
 	return 0;
-
+rc_submit_fail:
+	rc_free_device(sz->rdev);
+	usb_set_intfdata(intf, NULL);
 rc_dev_fail:
 	usb_free_urb(sz->urb_in);
 free_buf_in:
@@ -388,15 +392,16 @@ static void streamzap_disconnect(struct usb_interface *interface)
 	struct streamzap_ir *sz = usb_get_intfdata(interface);
 	struct usb_device *usbdev = interface_to_usbdev(interface);
 
-	usb_set_intfdata(interface, NULL);
-
 	if (!sz)
 		return;
 
-	usb_kill_urb(sz->urb_in);
 	rc_unregister_device(sz->rdev);
+	usb_set_intfdata(interface, NULL);
+
+	usb_kill_urb(sz->urb_in);
 	usb_free_urb(sz->urb_in);
 	usb_free_coherent(usbdev, sz->buf_in_len, sz->buf_in, sz->dma_in);
+	rc_free_device(sz->rdev);
 
 	kfree(sz);
 }
diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c
index 92ef4e7c6f69f..cb4c56bf0752a 100644
--- a/drivers/media/rc/sunxi-cir.c
+++ b/drivers/media/rc/sunxi-cir.c
@@ -371,6 +371,7 @@ static void sunxi_ir_remove(struct platform_device *pdev)
 	struct sunxi_ir *ir = platform_get_drvdata(pdev);
 
 	rc_unregister_device(ir->rc);
+	rc_free_device(ir->rc);
 	sunxi_ir_hw_exit(&pdev->dev);
 }
 
diff --git a/drivers/media/rc/ttusbir.c b/drivers/media/rc/ttusbir.c
index 110a469001146..3848ad3a6b85f 100644
--- a/drivers/media/rc/ttusbir.c
+++ b/drivers/media/rc/ttusbir.c
@@ -32,7 +32,7 @@ struct ttusbir {
 
 	struct led_classdev led;
 	struct urb *bulk_urb;
-	uint8_t bulk_buffer[5];
+	u8 *bulk_buffer;
 	int bulk_out_endp, iso_in_endp;
 	bool led_on, is_led_on;
 	atomic_t led_complete;
@@ -186,13 +186,16 @@ static int ttusbir_probe(struct usb_interface *intf,
 	struct rc_dev *rc;
 	int i, j, ret;
 	int altsetting = -1;
+	u8 *buffer;
 
 	tt = kzalloc_obj(*tt);
+	buffer = kzalloc(5, GFP_KERNEL);
 	rc = rc_allocate_device(RC_DRIVER_IR_RAW);
-	if (!tt || !rc) {
+	if (!tt || !rc || buffer) {
 		ret = -ENOMEM;
 		goto out;
 	}
+	tt->bulk_buffer = buffer;
 
 	/* find the correct alt setting */
 	for (i = 0; i < intf->num_altsetting && altsetting == -1; i++) {
@@ -281,8 +284,8 @@ static int ttusbir_probe(struct usb_interface *intf,
 	tt->bulk_buffer[3] = 0x01;
 
 	usb_fill_bulk_urb(tt->bulk_urb, tt->udev, usb_sndbulkpipe(tt->udev,
-		tt->bulk_out_endp), tt->bulk_buffer, sizeof(tt->bulk_buffer),
-						ttusbir_bulk_complete, tt);
+			  tt->bulk_out_endp), tt->bulk_buffer, 5,
+			  ttusbir_bulk_complete, tt);
 
 	tt->led.name = "ttusbir:green:power";
 	tt->led.default_trigger = "rc-feedback";
@@ -333,7 +336,6 @@ static int ttusbir_probe(struct usb_interface *intf,
 	return 0;
 out3:
 	rc_unregister_device(rc);
-	rc = NULL;
 out2:
 	led_classdev_unregister(&tt->led);
 out:
@@ -351,6 +353,7 @@ static int ttusbir_probe(struct usb_interface *intf,
 		kfree(tt);
 	}
 	rc_free_device(rc);
+	kfree(buffer);
 
 	return ret;
 }
@@ -373,6 +376,8 @@ static void ttusbir_disconnect(struct usb_interface *intf)
 	}
 	usb_kill_urb(tt->bulk_urb);
 	usb_free_urb(tt->bulk_urb);
+	rc_free_device(tt->rc);
+	kfree(tt->bulk_buffer);
 	usb_set_intfdata(intf, NULL);
 	kfree(tt);
 }
diff --git a/drivers/media/rc/winbond-cir.c b/drivers/media/rc/winbond-cir.c
index 515469dd82d4c..8e804661a6215 100644
--- a/drivers/media/rc/winbond-cir.c
+++ b/drivers/media/rc/winbond-cir.c
@@ -1132,7 +1132,6 @@ wbcir_probe(struct pnp_dev *device, const struct pnp_device_id *dev_id)
 	release_region(data->wbase, WAKEUP_IOMEM_LEN);
 exit_unregister_device:
 	rc_unregister_device(data->dev);
-	data->dev = NULL;
 exit_free_rc:
 	rc_free_device(data->dev);
 exit_unregister_led:
@@ -1163,6 +1162,7 @@ wbcir_remove(struct pnp_dev *device)
 	wbcir_set_bits(data->wbase + WBCIR_REG_WCEIR_EV_EN, 0x00, 0x07);
 
 	rc_unregister_device(data->dev);
+	rc_free_device(data->dev);
 
 	led_classdev_unregister(&data->led);
 
diff --git a/drivers/media/rc/xbox_remote.c b/drivers/media/rc/xbox_remote.c
index 3e3da70cf8da0..d2cb88b8f1ca8 100644
--- a/drivers/media/rc/xbox_remote.c
+++ b/drivers/media/rc/xbox_remote.c
@@ -55,7 +55,7 @@ struct xbox_remote {
 	struct usb_interface *interface;
 
 	struct urb *irq_urb;
-	unsigned char inbuf[DATA_BUFSIZE] __aligned(sizeof(u16));
+	u8 *inbuf;
 
 	char rc_name[NAME_BUFSIZE];
 	char rc_phys[NAME_BUFSIZE];
@@ -218,6 +218,10 @@ static int xbox_remote_probe(struct usb_interface *interface,
 	if (!xbox_remote || !rc_dev)
 		goto exit_free_dev_rdev;
 
+	xbox_remote->inbuf = kzalloc(DATA_BUFSIZE, GFP_KERNEL);
+	if (!xbox_remote->inbuf)
+		goto exit_free_inbuf;
+
 	/* Allocate URB buffer */
 	xbox_remote->irq_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!xbox_remote->irq_urb)
@@ -262,6 +266,8 @@ static int xbox_remote_probe(struct usb_interface *interface,
 	usb_kill_urb(xbox_remote->irq_urb);
 exit_free_buffers:
 	usb_free_urb(xbox_remote->irq_urb);
+exit_free_inbuf:
+	kfree(xbox_remote->inbuf);
 exit_free_dev_rdev:
 	rc_free_device(rc_dev);
 	kfree(xbox_remote);
@@ -277,15 +283,17 @@ static void xbox_remote_disconnect(struct usb_interface *interface)
 	struct xbox_remote *xbox_remote;
 
 	xbox_remote = usb_get_intfdata(interface);
-	usb_set_intfdata(interface, NULL);
 	if (!xbox_remote) {
 		dev_warn(&interface->dev, "%s - null device?\n", __func__);
 		return;
 	}
 
-	usb_kill_urb(xbox_remote->irq_urb);
 	rc_unregister_device(xbox_remote->rdev);
+	usb_set_intfdata(interface, NULL);
+	usb_kill_urb(xbox_remote->irq_urb);
+	rc_free_device(xbox_remote->rdev);
 	usb_free_urb(xbox_remote->irq_urb);
+	kfree(xbox_remote->inbuf);
 	kfree(xbox_remote);
 }
 
diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
index b6203e10e37aa..a8a76434989c2 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
@@ -237,8 +237,10 @@ static int vidtv_start_feed(struct dvb_demux_feed *feed)
 
 	if (dvb->nfeeds == 1) {
 		ret = vidtv_start_streaming(dvb);
-		if (ret < 0)
+		if (ret < 0) {
+			dvb->nfeeds--;
 			rc = ret;
+		}
 	}
 
 	mutex_unlock(&dvb->feed_lock);
diff --git a/drivers/media/test-drivers/vidtv/vidtv_channel.c b/drivers/media/test-drivers/vidtv/vidtv_channel.c
index da20657adc747..5f8c3af871711 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_channel.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_channel.c
@@ -341,6 +341,10 @@ vidtv_channel_pmt_match_sections(struct vidtv_channel *channels,
 					tail = vidtv_psi_pmt_stream_init(tail,
 									 s->type,
 									 e_pid);
+					if (!tail) {
+						vidtv_psi_pmt_stream_destroy(head);
+						return;
+					}
 
 					if (!head)
 						head = tail;
diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c
index 403fbedb86636..f0134e38a1fb8 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_mux.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c
@@ -233,7 +233,7 @@ static u32 vidtv_mux_push_pcr(struct vidtv_mux *m)
 	/* the 27Mhz clock will feed both parts of the PCR bitfield */
 	args.pcr = m->timing.clk;
 
-	nbytes += vidtv_ts_pcr_write_into(args);
+	nbytes += vidtv_ts_pcr_write_into(&args);
 	m->mux_buf_offset += nbytes;
 
 	m->num_streamed_pcr++;
@@ -363,7 +363,7 @@ static u32 vidtv_mux_pad_with_nulls(struct vidtv_mux *m, u32 npkts)
 	args.continuity_counter = &ctx->cc;
 
 	for (i = 0; i < npkts; ++i) {
-		m->mux_buf_offset += vidtv_ts_null_write_into(args);
+		m->mux_buf_offset += vidtv_ts_null_write_into(&args);
 		args.dest_offset  = m->mux_buf_offset;
 	}
 
diff --git a/drivers/media/test-drivers/vidtv/vidtv_s302m.c b/drivers/media/test-drivers/vidtv/vidtv_s302m.c
index 5814976449407..3c08e442b2d15 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_s302m.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_s302m.c
@@ -45,7 +45,7 @@
 #define FF_S302M_DEFAULT_PTS_OFFSET 100000
 
 /* Used by the tone generator: number of samples for PI */
-#define PI		180
+#define PI_SAMPLES		180
 
 static const u8 reverse[256] = {
 	/* from ffmpeg */
@@ -259,10 +259,10 @@ static u16 vidtv_s302m_get_sample(struct vidtv_encoder *e)
 		if (!ctx->last_tone)
 			return 0x8000;
 
-		pos = (2 * PI * ctx->note_offset * ctx->last_tone) / S302M_SAMPLING_RATE_HZ;
+		pos = (2 * PI_SAMPLES * ctx->note_offset * ctx->last_tone) / S302M_SAMPLING_RATE_HZ;
 		ctx->note_offset++;
 
-		return (fixp_sin32(pos % (2 * PI)) >> 16) + 0x8000;
+		return (fixp_sin32(pos % (2 * PI_SAMPLES)) >> 16) + 0x8000;
 	}
 
 	/* bug somewhere */
diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.c b/drivers/media/test-drivers/vidtv/vidtv_ts.c
index ca4bb9c40b78e..cbe9aff9ffb56 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_ts.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_ts.c
@@ -48,7 +48,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter)
 		*continuity_counter = 0;
 }
 
-u32 vidtv_ts_null_write_into(struct null_packet_write_args args)
+u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args)
 {
 	u32 nbytes = 0;
 	struct vidtv_mpeg_ts ts_header = {};
@@ -56,21 +56,21 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args)
 	ts_header.sync_byte          = TS_SYNC_BYTE;
 	ts_header.bitfield           = cpu_to_be16(TS_NULL_PACKET_PID);
 	ts_header.payload            = 1;
-	ts_header.continuity_counter = *args.continuity_counter;
+	ts_header.continuity_counter = *args->continuity_counter;
 
 	/* copy TS header */
-	nbytes += vidtv_memcpy(args.dest_buf,
-			       args.dest_offset + nbytes,
-			       args.buf_sz,
+	nbytes += vidtv_memcpy(args->dest_buf,
+			       args->dest_offset + nbytes,
+			       args->buf_sz,
 			       &ts_header,
 			       sizeof(ts_header));
 
-	vidtv_ts_inc_cc(args.continuity_counter);
+	vidtv_ts_inc_cc(args->continuity_counter);
 
 	/* fill the rest with empty data */
-	nbytes += vidtv_memset(args.dest_buf,
-			       args.dest_offset + nbytes,
-			       args.buf_sz,
+	nbytes += vidtv_memset(args->dest_buf,
+			       args->dest_offset + nbytes,
+			       args->buf_sz,
 			       TS_FILL_BYTE,
 			       TS_PACKET_LEN - nbytes);
 
@@ -83,17 +83,17 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args)
 	return nbytes;
 }
 
-u32 vidtv_ts_pcr_write_into(struct pcr_write_args args)
+u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args)
 {
 	u32 nbytes = 0;
 	struct vidtv_mpeg_ts ts_header = {};
 	struct vidtv_mpeg_ts_adaption ts_adap = {};
 
 	ts_header.sync_byte     = TS_SYNC_BYTE;
-	ts_header.bitfield      = cpu_to_be16(args.pid);
+	ts_header.bitfield      = cpu_to_be16(args->pid);
 	ts_header.scrambling    = 0;
 	/* cc is not incremented, but it is needed. see 13818-1 clause 2.4.3.3 */
-	ts_header.continuity_counter = *args.continuity_counter;
+	ts_header.continuity_counter = *args->continuity_counter;
 	ts_header.payload            = 0;
 	ts_header.adaptation_field   = 1;
 
@@ -102,27 +102,27 @@ u32 vidtv_ts_pcr_write_into(struct pcr_write_args args)
 	ts_adap.PCR    = 1;
 
 	/* copy TS header */
-	nbytes += vidtv_memcpy(args.dest_buf,
-			       args.dest_offset + nbytes,
-			       args.buf_sz,
+	nbytes += vidtv_memcpy(args->dest_buf,
+			       args->dest_offset + nbytes,
+			       args->buf_sz,
 			       &ts_header,
 			       sizeof(ts_header));
 
 	/* write the adap after the TS header */
-	nbytes += vidtv_memcpy(args.dest_buf,
-			       args.dest_offset + nbytes,
-			       args.buf_sz,
+	nbytes += vidtv_memcpy(args->dest_buf,
+			       args->dest_offset + nbytes,
+			       args->buf_sz,
 			       &ts_adap,
 			       sizeof(ts_adap));
 
 	/* write the PCR optional */
-	nbytes += vidtv_ts_write_pcr_bits(args.dest_buf,
-					  args.dest_offset + nbytes,
-					  args.pcr);
+	nbytes += vidtv_ts_write_pcr_bits(args->dest_buf,
+					  args->dest_offset + nbytes,
+					  args->pcr);
 
-	nbytes += vidtv_memset(args.dest_buf,
-			       args.dest_offset + nbytes,
-			       args.buf_sz,
+	nbytes += vidtv_memset(args->dest_buf,
+			       args->dest_offset + nbytes,
+			       args->buf_sz,
 			       TS_FILL_BYTE,
 			       TS_PACKET_LEN - nbytes);
 
diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.h b/drivers/media/test-drivers/vidtv/vidtv_ts.h
index 09b4ffd028292..3606398e160d9 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_ts.h
+++ b/drivers/media/test-drivers/vidtv/vidtv_ts.h
@@ -90,7 +90,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter);
  *
  * Return: The number of bytes written into the buffer.
  */
-u32 vidtv_ts_null_write_into(struct null_packet_write_args args);
+u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args);
 
 /**
  * vidtv_ts_pcr_write_into - Write a PCR  packet into a buffer.
@@ -101,6 +101,6 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args);
  *
  * Return: The number of bytes written into the buffer.
  */
-u32 vidtv_ts_pcr_write_into(struct pcr_write_args args);
+u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args);
 
 #endif //VIDTV_TS_H
diff --git a/drivers/media/test-drivers/vimc/vimc-common.h b/drivers/media/test-drivers/vimc/vimc-common.h
index 7a45a2117748e..861b334ffc65a 100644
--- a/drivers/media/test-drivers/vimc/vimc-common.h
+++ b/drivers/media/test-drivers/vimc/vimc-common.h
@@ -12,6 +12,8 @@
 #include <linux/slab.h>
 #include <media/media-device.h>
 #include <media/v4l2-device.h>
+#include <media/tpg/v4l2-tpg.h>
+#include <media/v4l2-ctrls.h>
 
 #define VIMC_PDEV_NAME "vimc"
 
@@ -27,6 +29,15 @@
 #define VIMC_FRAME_MIN_WIDTH 16
 #define VIMC_FRAME_MIN_HEIGHT 16
 
+#define VIMC_PIXEL_RATE_FIXED		160000000	/* 160 MHz */
+#define VIMC_HBLANK_FIXED		800
+/* VBLANK - vertical blanking (primary FPS control) */
+#define VIMC_VBLANK_MIN			4
+#define VIMC_VBLANK_MAX			65535
+#define VIMC_VBLANK_STEP		1
+#define VIMC_VBLANK_DEFAULT	        3223           /* 30fps vga */
+#define VIMC_PIXELS_THRESHOLD_30FPS	(1920 * 1080) /* 2073600 pixels */
+
 #define VIMC_FRAME_INDEX(lin, col, width, bpp) ((lin * width + col) * bpp)
 
 /* Source and sink pad checks */
@@ -159,6 +170,36 @@ struct vimc_ent_config {
 	const struct vimc_ent_type *type;
 };
 
+enum vimc_sensor_osd_mode {
+	VIMC_SENSOR_OSD_SHOW_ALL = 0,
+	VIMC_SENSOR_OSD_SHOW_COUNTERS = 1,
+	VIMC_SENSOR_OSD_SHOW_NONE = 2
+};
+
+struct vimc_sensor_device {
+	struct vimc_ent_device ved;
+	struct v4l2_subdev sd;
+	struct tpg_data tpg;
+	struct v4l2_ctrl_handler hdl;
+	struct media_pad pad;
+	struct v4l2_ctrl *pixel_rate;
+	struct v4l2_ctrl *hblank;
+	struct v4l2_ctrl *vblank;
+
+	u8 *frame;
+
+	/*
+	 * Virtual "hardware" configuration, filled when the stream starts or
+	 * when controls are set.
+	 */
+	struct {
+		struct v4l2_area size;
+		enum vimc_sensor_osd_mode osd_value;
+		u64 start_stream_ts;
+		unsigned long fps_jiffies;
+	} hw;
+};
+
 /**
  * vimc_is_source - returns true if the entity has only source pads
  *
diff --git a/drivers/media/test-drivers/vimc/vimc-sensor.c b/drivers/media/test-drivers/vimc/vimc-sensor.c
index 69e714d4f2281..5deebcc78a33b 100644
--- a/drivers/media/test-drivers/vimc/vimc-sensor.c
+++ b/drivers/media/test-drivers/vimc/vimc-sensor.c
@@ -14,32 +14,6 @@
 
 #include "vimc-common.h"
 
-enum vimc_sensor_osd_mode {
-	VIMC_SENSOR_OSD_SHOW_ALL = 0,
-	VIMC_SENSOR_OSD_SHOW_COUNTERS = 1,
-	VIMC_SENSOR_OSD_SHOW_NONE = 2
-};
-
-struct vimc_sensor_device {
-	struct vimc_ent_device ved;
-	struct v4l2_subdev sd;
-	struct tpg_data tpg;
-	struct v4l2_ctrl_handler hdl;
-	struct media_pad pad;
-
-	u8 *frame;
-
-	/*
-	 * Virtual "hardware" configuration, filled when the stream starts or
-	 * when controls are set.
-	 */
-	struct {
-		struct v4l2_area size;
-		enum vimc_sensor_osd_mode osd_value;
-		u64 start_stream_ts;
-	} hw;
-};
-
 static const struct v4l2_mbus_framefmt fmt_default = {
 	.width = 640,
 	.height = 480,
@@ -51,10 +25,15 @@ static const struct v4l2_mbus_framefmt fmt_default = {
 static int vimc_sensor_init_state(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state)
 {
+	struct vimc_sensor_device *vsensor =
+		container_of(sd, struct vimc_sensor_device, sd);
+
 	struct v4l2_mbus_framefmt *mf;
 
 	mf = v4l2_subdev_state_get_format(sd_state, 0);
 	*mf = fmt_default;
+	vsensor->hw.size.width = fmt_default.width;
+	vsensor->hw.size.height = fmt_default.height;
 
 	return 0;
 }
@@ -113,6 +92,26 @@ static void vimc_sensor_tpg_s_format(struct vimc_sensor_device *vsensor,
 	tpg_s_xfer_func(&vsensor->tpg, format->xfer_func);
 }
 
+static int vimc_sensor_update_frame_timing(struct v4l2_subdev *sd,
+					   u32 width, u32 height)
+{
+	struct vimc_sensor_device *vsensor =
+		container_of(sd, struct vimc_sensor_device, sd);
+	u64 pixel_rate = vsensor->pixel_rate->val;
+	u32 hts = width + vsensor->hblank->val;
+	u32 vts = height + vsensor->vblank->val;
+	u64 total_pixels = (u64)hts * vts;
+	u64 frame_interval_ns;
+
+	frame_interval_ns = total_pixels * NSEC_PER_SEC;
+	do_div(frame_interval_ns, pixel_rate);
+	vsensor->hw.fps_jiffies = nsecs_to_jiffies(frame_interval_ns);
+	if (vsensor->hw.fps_jiffies == 0)
+		vsensor->hw.fps_jiffies = 1;
+
+	return 0;
+}
+
 static void vimc_sensor_adjust_fmt(struct v4l2_mbus_framefmt *fmt)
 {
 	const struct vimc_pix_map *vpix;
@@ -134,6 +133,24 @@ static void vimc_sensor_adjust_fmt(struct v4l2_mbus_framefmt *fmt)
 	vimc_colorimetry_clamp(fmt);
 }
 
+static u32 vimc_calc_vblank(u32 width, u32 height,
+			    s64 pixel_rate, s32 hblank)
+{
+	u32 hts = width + hblank;
+	u32 target_fps;
+	u64 vts;
+
+	target_fps = (width * height <= VIMC_PIXELS_THRESHOLD_30FPS) ? 30 : 10;
+
+	vts = (u64)pixel_rate;
+	do_div(vts, target_fps * hts);
+
+	if (vts > height)
+		return clamp((u32)(vts - height), VIMC_VBLANK_MIN, VIMC_VBLANK_MAX);
+
+	return VIMC_VBLANK_MIN;
+}
+
 static int vimc_sensor_set_fmt(struct v4l2_subdev *sd,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
@@ -163,6 +180,20 @@ static int vimc_sensor_set_fmt(struct v4l2_subdev *sd,
 		fmt->format.xfer_func, fmt->format.ycbcr_enc);
 
 	*mf = fmt->format;
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		u32 vblank_def = vimc_calc_vblank(fmt->format.width,
+						  fmt->format.height,
+						  vsensor->pixel_rate->val,
+						  vsensor->hblank->val);
+		vsensor->hw.size.width = fmt->format.width;
+		vsensor->hw.size.height = fmt->format.height;
+		__v4l2_ctrl_modify_range(vsensor->vblank,
+					 VIMC_VBLANK_MIN,
+					 VIMC_VBLANK_MAX,
+					 VIMC_VBLANK_STEP,
+					 vblank_def);
+		__v4l2_ctrl_s_ctrl(vsensor->vblank, vblank_def);
+	}
 
 	return 0;
 }
@@ -248,6 +279,8 @@ static int vimc_sensor_s_stream(struct v4l2_subdev *sd, int enable)
 
 		vsensor->hw.size.width = format->width;
 		vsensor->hw.size.height = format->height;
+		vimc_sensor_update_frame_timing(sd, format->width,
+						format->height);
 
 		v4l2_subdev_unlock_state(state);
 
@@ -319,6 +352,15 @@ static int vimc_sensor_s_ctrl(struct v4l2_ctrl *ctrl)
 	case VIMC_CID_OSD_TEXT_MODE:
 		vsensor->hw.osd_value = ctrl->val;
 		break;
+	case V4L2_CID_PIXEL_RATE:
+		break;
+	case V4L2_CID_HBLANK:
+		break;
+	case V4L2_CID_VBLANK:
+		vimc_sensor_update_frame_timing(&vsensor->sd,
+						vsensor->hw.size.width,
+						vsensor->hw.size.height);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -403,6 +445,26 @@ static struct vimc_ent_device *vimc_sensor_add(struct vimc_device *vimc,
 			  V4L2_CID_HUE, -128, 127, 1, 0);
 	v4l2_ctrl_new_std(&vsensor->hdl, &vimc_sensor_ctrl_ops,
 			  V4L2_CID_SATURATION, 0, 255, 1, 128);
+	/* Timing controls for frame interval configuration */
+	vsensor->pixel_rate = v4l2_ctrl_new_std(&vsensor->hdl, &vimc_sensor_ctrl_ops,
+						V4L2_CID_PIXEL_RATE,
+						VIMC_PIXEL_RATE_FIXED, VIMC_PIXEL_RATE_FIXED,
+						1, VIMC_PIXEL_RATE_FIXED);
+	if (vsensor->pixel_rate)
+		vsensor->pixel_rate->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	vsensor->hblank = v4l2_ctrl_new_std(&vsensor->hdl, &vimc_sensor_ctrl_ops,
+					    V4L2_CID_HBLANK,
+					    VIMC_HBLANK_FIXED, VIMC_HBLANK_FIXED,
+					    1, VIMC_HBLANK_FIXED);
+	if (vsensor->hblank)
+		vsensor->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	vsensor->vblank = v4l2_ctrl_new_std(&vsensor->hdl, &vimc_sensor_ctrl_ops,
+					    V4L2_CID_VBLANK,
+					    VIMC_VBLANK_MIN, VIMC_VBLANK_MAX,
+					    VIMC_VBLANK_STEP, VIMC_VBLANK_DEFAULT);
+
 	vsensor->sd.ctrl_handler = &vsensor->hdl;
 	if (vsensor->hdl.error) {
 		ret = vsensor->hdl.error;
diff --git a/drivers/media/test-drivers/vimc/vimc-streamer.c b/drivers/media/test-drivers/vimc/vimc-streamer.c
index 15d863f97cbf9..3ebf5798fa3d5 100644
--- a/drivers/media/test-drivers/vimc/vimc-streamer.c
+++ b/drivers/media/test-drivers/vimc/vimc-streamer.c
@@ -139,6 +139,29 @@ static int vimc_streamer_pipeline_init(struct vimc_stream *stream,
 	return -EINVAL;
 }
 
+/**
+ * vimc_streamer_get_sensor() - Get sensor from pipeline
+ * @stream: the pipeline
+ *
+ * Helper function to find the sensor device in the pipeline.
+ * Returns pointer to sensor device or NULL if not found.
+ */
+static struct vimc_sensor_device *vimc_streamer_get_sensor(struct vimc_stream *stream)
+{
+	int i;
+
+	for (i = 0; i < stream->pipe_size; i++) {
+		struct vimc_ent_device *ved = stream->ved_pipeline[i];
+
+		if (ved && ved->ent &&
+		    ved->ent->function == MEDIA_ENT_F_CAM_SENSOR) {
+			return container_of(ved, struct vimc_sensor_device, ved);
+		}
+	}
+
+	return NULL;
+}
+
 /**
  * vimc_streamer_thread - Process frames through the pipeline
  *
@@ -154,25 +177,31 @@ static int vimc_streamer_pipeline_init(struct vimc_stream *stream,
 static int vimc_streamer_thread(void *data)
 {
 	struct vimc_stream *stream = data;
+	struct vimc_sensor_device *vsensor;
 	u8 *frame = NULL;
 	int i;
+	unsigned long fps_jiffies;
+	const unsigned long default_jiffies = HZ / 30;
 
 	set_freezable();
+	vsensor = vimc_streamer_get_sensor(stream);
 
 	for (;;) {
 		try_to_freeze();
 		if (kthread_should_stop())
 			break;
 
+		/* Read from hardware configuration */
+		fps_jiffies = vsensor ? vsensor->hw.fps_jiffies : default_jiffies;
+
 		for (i = stream->pipe_size - 1; i >= 0; i--) {
 			frame = stream->ved_pipeline[i]->process_frame(
 					stream->ved_pipeline[i], frame);
 			if (!frame || IS_ERR(frame))
 				break;
 		}
-		//wait for 60hz
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(HZ / 60);
+		schedule_timeout(fps_jiffies);
 	}
 
 	return 0;
diff --git a/drivers/media/tuners/mxl5005s.c b/drivers/media/tuners/mxl5005s.c
index ba22bf594ac71..d57570290e931 100644
--- a/drivers/media/tuners/mxl5005s.c
+++ b/drivers/media/tuners/mxl5005s.c
@@ -1174,7 +1174,12 @@ static u16 MXL5005_ControlInit(struct dvb_frontend *fe)
 	state->Init_Ctrl[39].bit[0] = 3;
 	state->Init_Ctrl[39].val[0] = 1;
 
+	return 0;
+}
 
+static u16 MXL5005_ControlInitCH(struct dvb_frontend *fe)
+{
+	struct mxl5005s_state *state = fe->tuner_priv;
 	state->CH_Ctrl_Num = CHCTRL_NUM ;
 
 	state->CH_Ctrl[0].Ctrl_Num = DN_POLY ;
@@ -1663,6 +1668,7 @@ static void InitTunerControls(struct dvb_frontend *fe)
 {
 	MXL5005_RegisterInit(fe);
 	MXL5005_ControlInit(fe);
+	MXL5005_ControlInitCH(fe);
 #ifdef _MXL_INTERNAL
 	MXL5005_MXLControlInit(fe);
 #endif
diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c
index 4ca42114da8a1..b041cd8547324 100644
--- a/drivers/media/tuners/si2157.c
+++ b/drivers/media/tuners/si2157.c
@@ -638,20 +638,27 @@ static int si2157_set_analog_params(struct dvb_frontend *fe,
 				color = 0x10;
 			}
 		}
-	} else if (params->std & V4L2_STD_MN) {
+	} else if (params->std & (V4L2_STD_MN | V4L2_STD_NTSC_443)) {
 		std = "MN";
 		bandwidth = 6000000;
 		if_frequency = 5400000;
 		system = 2;
+		if (params->std & V4L2_STD_PAL_N) {
+			std = "palN";
+			color = 0x10;
+		} else if (params->std & V4L2_STD_PAL_Nc) {
+			std = "palNc";
+			color = 0x10;
+		}
 	} else if (params->std & V4L2_STD_PAL_I) {
 		std = "palI";
 		bandwidth = 8000000;
-		if_frequency = 7250000; /* TODO: does not work yet */
+		if_frequency = 7250000;
 		system = 4;
 	} else if (params->std & V4L2_STD_DK) {
 		std = "palDK";
 		bandwidth = 8000000;
-		if_frequency = 6900000; /* TODO: does not work yet */
+		if_frequency = 6900000;
 		system = 5;
 		if (params->std & V4L2_STD_SECAM_DK) {
 			std = "secamDK";
@@ -660,7 +667,7 @@ static int si2157_set_analog_params(struct dvb_frontend *fe,
 	} else if (params->std & V4L2_STD_SECAM_L) {
 		std = "secamL";
 		bandwidth = 8000000;
-		if_frequency = 6750000; /* TODO: untested */
+		if_frequency = 6900000;
 		system = 6;
 		color = 0x10;
 	} else if (params->std & V4L2_STD_SECAM_LC) {
diff --git a/drivers/media/tuners/xc5000.c b/drivers/media/tuners/xc5000.c
index a28481edd22ed..1a9c4574a8e8e 100644
--- a/drivers/media/tuners/xc5000.c
+++ b/drivers/media/tuners/xc5000.c
@@ -1333,6 +1333,15 @@ static int xc5000_set_config(struct dvb_frontend *fe, void *priv_cfg)
 }
 
 
+static int xc5000_get_rf_strength(struct dvb_frontend *fe, u16 *rssi)
+{
+	struct xc5000_priv *priv = fe->tuner_priv;
+
+	dprintk(1, "%s()\n", __func__);
+
+	return xc_get_lock_status(priv, rssi);
+}
+
 static const struct dvb_tuner_ops xc5000_tuner_ops = {
 	.info = {
 		.name              = "Xceive XC5000",
@@ -1353,7 +1362,8 @@ static const struct dvb_tuner_ops xc5000_tuner_ops = {
 	.get_frequency	   = xc5000_get_frequency,
 	.get_if_frequency  = xc5000_get_if_frequency,
 	.get_bandwidth	   = xc5000_get_bandwidth,
-	.get_status	   = xc5000_get_status
+	.get_status	   = xc5000_get_status,
+	.get_rf_strength   = xc5000_get_rf_strength,
 };
 
 struct dvb_frontend *xc5000_attach(struct dvb_frontend *fe,
diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c
index 8e480ab78f9b2..a11024451cebd 100644
--- a/drivers/media/usb/as102/as102_usb_drv.c
+++ b/drivers/media/usb/as102/as102_usb_drv.c
@@ -403,7 +403,9 @@ static int as102_usb_probe(struct usb_interface *intf,
 failed_dvb:
 	as102_free_usb_stream_buffer(as102_dev);
 failed_stream:
+	usb_set_intfdata(intf, NULL);
 	usb_deregister_dev(intf, &as102_usb_class_driver);
+	return ret;
 failed:
 	usb_put_dev(as102_dev->bus_adap.usb_dev);
 	usb_set_intfdata(intf, NULL);
diff --git a/drivers/media/usb/au0828/au0828-cards.c b/drivers/media/usb/au0828/au0828-cards.c
index 9ee21f8bf6fa0..9929ce5dcdcd3 100644
--- a/drivers/media/usb/au0828/au0828-cards.c
+++ b/drivers/media/usb/au0828/au0828-cards.c
@@ -105,6 +105,46 @@ struct au0828_board au0828_boards[] = {
 		.tuner_addr = 0x60,
 		.i2c_clk_divider = AU0828_I2C_CLK_250KHZ,
 	},
+	[AU0828_BOARD_HAUPPAUGE_IMPACTVCBE] = {
+		.name	= "Hauppauge Impact VCB-e",
+		.tuner_type = TUNER_ABSENT,
+		.i2c_clk_divider = AU0828_I2C_CLK_250KHZ,
+		.input = {
+			{
+				.type = AU0828_VMUX_COMPOSITE,
+				.vmux = AU8522_COMPOSITE_CH4,
+				.amux = AU8522_AUDIO_NONE,
+				.audio_setup = hvr950q_cs5340_audio,
+			},
+			{
+				.type = AU0828_VMUX_SVIDEO,
+				.vmux = AU8522_SVIDEO_CH13,
+				.amux = AU8522_AUDIO_NONE,
+				.audio_setup = hvr950q_cs5340_audio,
+			},
+		},
+	},
+	[AU0828_BOARD_HAUPPAUGE_HVR1265] = {
+		.name	= "Hauppauge HVR1265",
+		.tuner_type = TUNER_XC5000,
+		.tuner_addr = 0x61,
+		.has_ir_i2c = 1,
+		.has_analog = 1,
+		.i2c_clk_divider = AU0828_I2C_CLK_250KHZ,
+		.input = {
+			{
+				.type = AU0828_VMUX_TELEVISION,
+				.vmux = AU8522_COMPOSITE_CH4_SIF,
+				.amux = AU8522_AUDIO_SIF,
+			},
+			{
+				.type = AU0828_VMUX_SVIDEO,
+				.vmux = AU8522_SVIDEO_CH13,
+				.amux = AU8522_AUDIO_NONE,
+				.audio_setup = hvr950q_cs5340_audio,
+			},
+		},
+	},
 };
 
 /* Tuner callback function for au0828 boards. Currently only needed
@@ -120,6 +160,8 @@ int au0828_tuner_callback(void *priv, int component, int command, int arg)
 	case AU0828_BOARD_HAUPPAUGE_HVR850:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL:
+	case AU0828_BOARD_HAUPPAUGE_HVR1265:
+	case AU0828_BOARD_HAUPPAUGE_IMPACTVCBE:
 	case AU0828_BOARD_DVICO_FUSIONHDTV7:
 		if (command == 0) {
 			/* Tuner Reset Command from xc5000 */
@@ -190,6 +232,8 @@ void au0828_card_setup(struct au0828_dev *dev)
 	case AU0828_BOARD_HAUPPAUGE_HVR850:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL:
+	case AU0828_BOARD_HAUPPAUGE_HVR1265:
+	case AU0828_BOARD_HAUPPAUGE_IMPACTVCBE:
 	case AU0828_BOARD_HAUPPAUGE_WOODBURY:
 		if (dev->i2c_rc == 0)
 			hauppauge_eeprom(dev, eeprom+0xa0);
@@ -248,6 +292,8 @@ void au0828_gpio_setup(struct au0828_dev *dev)
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL:
 	case AU0828_BOARD_HAUPPAUGE_WOODBURY:
+	case AU0828_BOARD_HAUPPAUGE_HVR1265:
+	case AU0828_BOARD_HAUPPAUGE_IMPACTVCBE:
 		/* GPIO's
 		 * 4 - CS5340
 		 * 5 - AU8522 Demodulator
@@ -340,6 +386,10 @@ struct usb_device_id au0828_usb_id_table[] = {
 		.driver_info = AU0828_BOARD_HAUPPAUGE_HVR950Q },
 	{ USB_DEVICE(0x2040, 0x7270),
 		.driver_info = AU0828_BOARD_HAUPPAUGE_HVR950Q },
+	{ USB_DEVICE(0x2040, 0x72b0),
+		.driver_info = AU0828_BOARD_HAUPPAUGE_IMPACTVCBE },
+	{ USB_DEVICE(0x2040, 0x72a0),
+		.driver_info = AU0828_BOARD_HAUPPAUGE_HVR1265 },
 	{ },
 };
 
diff --git a/drivers/media/usb/au0828/au0828-cards.h b/drivers/media/usb/au0828/au0828-cards.h
index a9cdf85f98f5e..a438aeb334a08 100644
--- a/drivers/media/usb/au0828/au0828-cards.h
+++ b/drivers/media/usb/au0828/au0828-cards.h
@@ -11,3 +11,5 @@
 #define AU0828_BOARD_DVICO_FUSIONHDTV7	3
 #define AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL	4
 #define AU0828_BOARD_HAUPPAUGE_WOODBURY	5
+#define AU0828_BOARD_HAUPPAUGE_IMPACTVCBE	6
+#define AU0828_BOARD_HAUPPAUGE_HVR1265		7
diff --git a/drivers/media/usb/au0828/au0828-dvb.c b/drivers/media/usb/au0828/au0828-dvb.c
index 5d0447ff7d06e..31123e6f9fc31 100644
--- a/drivers/media/usb/au0828/au0828-dvb.c
+++ b/drivers/media/usb/au0828/au0828-dvb.c
@@ -571,6 +571,7 @@ int au0828_dvb_register(struct au0828_dev *dev)
 	switch (dev->boardnr) {
 	case AU0828_BOARD_HAUPPAUGE_HVR850:
 	case AU0828_BOARD_HAUPPAUGE_HVR950Q:
+	case AU0828_BOARD_HAUPPAUGE_HVR1265:
 		dvb->frontend = dvb_attach(au8522_attach,
 				&hauppauge_hvr950q_config,
 				&dev->i2c_adap);
diff --git a/drivers/media/usb/au0828/au0828-input.c b/drivers/media/usb/au0828/au0828-input.c
index 7dec1a360da6a..b156afb1a0aea 100644
--- a/drivers/media/usb/au0828/au0828-input.c
+++ b/drivers/media/usb/au0828/au0828-input.c
@@ -300,6 +300,7 @@ int au0828_rc_register(struct au0828_dev *dev)
 	if (dev->board.has_ir_i2c) {	/* external i2c device */
 		switch (dev->boardnr) {
 		case AU0828_BOARD_HAUPPAUGE_HVR950Q:
+		case AU0828_BOARD_HAUPPAUGE_HVR1265:
 			rc->map_name = RC_MAP_HAUPPAUGE;
 			ir->get_key_i2c = au0828_get_key_au8522;
 			break;
@@ -357,6 +358,7 @@ void au0828_rc_unregister(struct au0828_dev *dev)
 		return;
 
 	rc_unregister_device(ir->rc);
+	rc_free_device(ir->rc);
 
 	/* done */
 	kfree(ir);
diff --git a/drivers/media/usb/cx231xx/cx231xx-avcore.c b/drivers/media/usb/cx231xx/cx231xx-avcore.c
index 1cfec76b72f37..0a5c635da0406 100644
--- a/drivers/media/usb/cx231xx/cx231xx-avcore.c
+++ b/drivers/media/usb/cx231xx/cx231xx-avcore.c
@@ -2027,10 +2027,9 @@ int cx231xx_dif_set_standard(struct cx231xx *dev, u32 standard)
 		status = vid_blk_write_word(dev, DIF_SRC_GAIN_CONTROL,
 						0x000035e8);
 
-		status = vid_blk_write_word(dev, DIF_AGC_CTRL_IF, 0xC2262600);
-		status = vid_blk_write_word(dev, DIF_AGC_CTRL_INT,
-						0xC2262600);
-		status = vid_blk_write_word(dev, DIF_AGC_CTRL_RF, 0xC2262600);
+		status = vid_blk_write_word(dev, DIF_AGC_CTRL_IF,  0xC2262600);
+		status = vid_blk_write_word(dev, DIF_AGC_CTRL_INT, 0xC2260000);
+		status = vid_blk_write_word(dev, DIF_AGC_CTRL_RF,  0xC2260000);
 
 		/* Save the Spec Inversion value */
 		dif_misc_ctrl_value &= FLD_DIF_SPEC_INV;
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index b64a37d1acf40..8bf2ab9c2e8ec 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -1027,6 +1027,8 @@ struct usb_device_id cx231xx_id_table[] = {
 	 .driver_info = CX231XX_BOARD_HAUPPAUGE_955Q},
 	{USB_DEVICE(0x2040, 0xb151),
 	 .driver_info = CX231XX_BOARD_HAUPPAUGE_935C},
+	{USB_DEVICE(0x2040, 0x8360),
+	 .driver_info = CX231XX_BOARD_HAUPPAUGE_935C},
 	{USB_DEVICE(0x2040, 0xb150),
 	 .driver_info = CX231XX_BOARD_HAUPPAUGE_975},
 	{USB_DEVICE(0x2040, 0xb130),
@@ -1369,8 +1371,6 @@ void cx231xx_release_resources(struct cx231xx *dev)
 
 	cx231xx_unregister_media_device(dev);
 
-	usb_put_dev(dev->udev);
-
 	/* Mark device as unused */
 	clear_bit(dev->devno, &cx231xx_devused);
 }
@@ -1719,7 +1719,7 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
 		}
 	} while (test_and_set_bit(nr, &cx231xx_devused));
 
-	udev = usb_get_dev(interface_to_usbdev(interface));
+	udev = interface_to_usbdev(interface);
 
 	/* allocate memory for our device state and initialize it */
 	dev = devm_kzalloc(&udev->dev, sizeof(*dev), GFP_KERNEL);
@@ -1915,7 +1915,6 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
 err_media_init:
 	usb_set_intfdata(interface, NULL);
 err_if:
-	usb_put_dev(udev);
 	clear_bit(nr, &cx231xx_devused);
 	return retval;
 }
diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
index 600cff8a4abdc..bd86d250433df 100644
--- a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
+++ b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
@@ -187,6 +187,7 @@ static int dvb_usbv2_remote_exit(struct dvb_usb_device *d)
 	if (d->rc_dev) {
 		cancel_delayed_work_sync(&d->rc_query_work);
 		rc_unregister_device(d->rc_dev);
+		rc_free_device(d->rc_dev);
 		d->rc_dev = NULL;
 	}
 
diff --git a/drivers/media/usb/dvb-usb/dvb-usb-remote.c b/drivers/media/usb/dvb-usb/dvb-usb-remote.c
index 65e2c9e2cdc99..6dc11718dfb98 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb-remote.c
+++ b/drivers/media/usb/dvb-usb/dvb-usb-remote.c
@@ -347,10 +347,12 @@ int dvb_usb_remote_exit(struct dvb_usb_device *d)
 {
 	if (d->state & DVB_USB_STATE_REMOTE) {
 		cancel_delayed_work_sync(&d->rc_query_work);
-		if (d->props.rc.mode == DVB_RC_LEGACY)
+		if (d->props.rc.mode == DVB_RC_LEGACY) {
 			input_unregister_device(d->input_dev);
-		else
+		} else {
 			rc_unregister_device(d->rc_dev);
+			rc_free_device(d->rc_dev);
+		}
 	}
 	d->state &= ~DVB_USB_STATE_REMOTE;
 	return 0;
diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c
index 59a2e4db75b72..d7075ebabceb8 100644
--- a/drivers/media/usb/em28xx/em28xx-cards.c
+++ b/drivers/media/usb/em28xx/em28xx-cards.c
@@ -2522,17 +2522,12 @@ const struct em28xx_board em28xx_boards[] = {
 		.def_i2c_bus   = 1,
 		.i2c_speed     = EM28XX_I2C_CLK_WAIT_ENABLE |
 				 EM28XX_I2C_FREQ_400_KHZ,
-		.tuner_type    = TUNER_SI2157,
+		.tuner_type    = TUNER_ABSENT,
 		.tuner_gpio    = hauppauge_dualhd_dvb,
 		.has_dvb       = 1,
 		.has_dual_ts   = 1,
 		.ir_codes      = RC_MAP_HAUPPAUGE,
 		.leds          = hauppauge_dualhd_leds,
-		.input         = { {
-			.type     = EM28XX_VMUX_COMPOSITE,
-			.vmux     = TVP5150_COMPOSITE1,
-			.amux     = EM28XX_AMUX_LINE_IN,
-		} },
 	},
 	/*
 	 * 2040:026d Hauppauge WinTV-dualHD (model 01595 - ATSC/QAM) Isoc.
@@ -2757,10 +2752,22 @@ struct usb_device_id em28xx_id_table[] = {
 			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_DVB },
 	{ USB_DEVICE(0x2040, 0x8265),
 			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_DVB },
+	{ USB_DEVICE(0x2040, 0x8269),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_DVB },
+	{ USB_DEVICE(0x2040, 0x8278),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_DVB },
 	{ USB_DEVICE(0x2040, 0x026d),
 			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
 	{ USB_DEVICE(0x2040, 0x826d),
 			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
+	{ USB_DEVICE(0x2040, 0x826e),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
+	{ USB_DEVICE(0x2040, 0x826f),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
+	{ USB_DEVICE(0x2040, 0x8270),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
+	{ USB_DEVICE(0x2040, 0x8271),
+			.driver_info = EM28174_BOARD_HAUPPAUGE_WINTV_DUALHD_01595 },
 	{ USB_DEVICE(0x2040, 0x846d),
 			.driver_info = EM2874_BOARD_HAUPPAUGE_USB_QUADHD },
 	{ USB_DEVICE(0x0438, 0xb002),
diff --git a/drivers/media/usb/em28xx/em28xx-input.c b/drivers/media/usb/em28xx/em28xx-input.c
index 20fdd59b55185..ab61d9a29b10e 100644
--- a/drivers/media/usb/em28xx/em28xx-input.c
+++ b/drivers/media/usb/em28xx/em28xx-input.c
@@ -853,6 +853,7 @@ static int em28xx_ir_fini(struct em28xx *dev)
 		goto ref_put;
 
 	rc_unregister_device(ir->rc);
+	rc_free_device(ir->rc);
 
 	kfree(ir->i2c_client);
 
diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c
index b0c184f237a72..5f13f63fbdee2 100644
--- a/drivers/media/usb/em28xx/em28xx-video.c
+++ b/drivers/media/usb/em28xx/em28xx-video.c
@@ -2126,7 +2126,7 @@ static int em28xx_v4l2_open(struct file *filp)
 {
 	struct video_device *vdev = video_devdata(filp);
 	struct em28xx *dev = video_drvdata(filp);
-	struct em28xx_v4l2 *v4l2 = dev->v4l2;
+	struct em28xx_v4l2 *v4l2;
 	enum v4l2_buf_type fh_type = 0;
 	int ret;
 
@@ -2143,13 +2143,19 @@ static int em28xx_v4l2_open(struct file *filp)
 		return -EINVAL;
 	}
 
+	if (mutex_lock_interruptible(&dev->lock))
+		return -ERESTARTSYS;
+
+	v4l2 = dev->v4l2;
+	if (!v4l2) {
+		mutex_unlock(&dev->lock);
+		return -ENODEV;
+	}
+
 	em28xx_videodbg("open dev=%s type=%s users=%d\n",
 			video_device_node_name(vdev), v4l2_type_names[fh_type],
 			v4l2->users);
 
-	if (mutex_lock_interruptible(&dev->lock))
-		return -ERESTARTSYS;
-
 	ret = v4l2_fh_open(filp);
 	if (ret) {
 		dev_err(&dev->intf->dev,
diff --git a/drivers/media/usb/go7007/go7007-loader.c b/drivers/media/usb/go7007/go7007-loader.c
index 243aa0ad074c8..5747e6e4f2d64 100644
--- a/drivers/media/usb/go7007/go7007-loader.c
+++ b/drivers/media/usb/go7007/go7007-loader.c
@@ -41,9 +41,7 @@ static int go7007_loader_probe(struct usb_interface *interface,
 	int ret;
 	int i;
 
-	usbdev = usb_get_dev(interface_to_usbdev(interface));
-	if (!usbdev)
-		goto failed2;
+	usbdev = interface_to_usbdev(interface);
 
 	if (usbdev->descriptor.bNumConfigurations != 1) {
 		dev_err(&interface->dev, "can't handle multiple config\n");
@@ -96,7 +94,6 @@ static int go7007_loader_probe(struct usb_interface *interface,
 	return 0;
 
 failed2:
-	usb_put_dev(usbdev);
 	dev_err(&interface->dev, "probe failed\n");
 	return -ENODEV;
 }
@@ -104,7 +101,6 @@ static int go7007_loader_probe(struct usb_interface *interface,
 static void go7007_loader_disconnect(struct usb_interface *interface)
 {
 	dev_info(&interface->dev, "disconnect\n");
-	usb_put_dev(interface_to_usbdev(interface));
 	usb_set_intfdata(interface, NULL);
 }
 
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index 3fc15d16df8e1..f3d3f441c8510 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -1701,19 +1701,6 @@ int gspca_resume(struct usb_interface *intf)
 EXPORT_SYMBOL(gspca_resume);
 #endif
 
-/* -- module insert / remove -- */
-static int __init gspca_init(void)
-{
-	pr_info("v" GSPCA_VERSION " registered\n");
-	return 0;
-}
-static void __exit gspca_exit(void)
-{
-}
-
-module_init(gspca_init);
-module_exit(gspca_exit);
-
 module_param_named(debug, gspca_debug, int, 0644);
 MODULE_PARM_DESC(debug,
 		"1:probe 2:config 3:stream 4:frame 5:packet 6:usbi 7:usbo");
diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c
index 94d356fba6122..a15829a60e883 100644
--- a/drivers/media/usb/hackrf/hackrf.c
+++ b/drivers/media/usb/hackrf/hackrf.c
@@ -1485,7 +1485,7 @@ static int hackrf_probe(struct usb_interface *intf,
 	if (ret) {
 		dev_err(dev->dev,
 			"Failed to register as video device (%d)\n", ret);
-		goto err_v4l2_device_unregister;
+		goto err_v4l2_device_put;
 	}
 	dev_info(dev->dev, "Registered as %s\n",
 		 video_device_node_name(&dev->rx_vdev));
@@ -1513,8 +1513,9 @@ static int hackrf_probe(struct usb_interface *intf,
 	return 0;
 err_video_unregister_device_rx:
 	video_unregister_device(&dev->rx_vdev);
-err_v4l2_device_unregister:
-	v4l2_device_unregister(&dev->v4l2_dev);
+err_v4l2_device_put:
+	v4l2_device_put(&dev->v4l2_dev);
+	return ret;
 err_v4l2_ctrl_handler_free_tx:
 	v4l2_ctrl_handler_free(&dev->tx_ctrl_handler);
 err_v4l2_ctrl_handler_free_rx:
diff --git a/drivers/media/usb/usbtv/usbtv-core.c b/drivers/media/usb/usbtv/usbtv-core.c
index c4bfadbe63c90..6c4facf4f41a7 100644
--- a/drivers/media/usb/usbtv/usbtv-core.c
+++ b/drivers/media/usb/usbtv/usbtv-core.c
@@ -91,7 +91,7 @@ static int usbtv_probe(struct usb_interface *intf,
 	if (usbtv == NULL)
 		return -ENOMEM;
 	usbtv->dev = dev;
-	usbtv->udev = usb_get_dev(interface_to_usbdev(intf));
+	usbtv->udev = interface_to_usbdev(intf);
 
 	usbtv->iso_size = size;
 
@@ -119,7 +119,6 @@ static int usbtv_probe(struct usb_interface *intf,
 
 usbtv_video_fail:
 	usb_set_intfdata(intf, NULL);
-	usb_put_dev(usbtv->udev);
 	kfree(usbtv);
 
 	return ret;
@@ -137,7 +136,6 @@ static void usbtv_disconnect(struct usb_interface *intf)
 	usbtv_audio_free(usbtv);
 	usbtv_video_free(usbtv);
 
-	usb_put_dev(usbtv->udev);
 	usbtv->udev = NULL;
 
 	/* the usbtv structure will be deallocated when v4l2 will be
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index 37d33d4a363d7..a2b650f4ec3c3 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -3082,13 +3082,14 @@ static long __video_do_ioctl(struct file *file,
 	}
 
 	/*
-	 * We need to serialize streamon/off with queueing new requests.
+	 * We need to serialize streamon/off/reqbufs with queueing new requests.
 	 * These ioctls may trigger the cancellation of a streaming
 	 * operation, and that should not be mixed with queueing a new
 	 * request at the same time.
 	 */
 	if (v4l2_device_supports_requests(vfd->v4l2_dev) &&
-	    (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF)) {
+	    (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF ||
+	     cmd == VIDIOC_REQBUFS)) {
 		req_queue_lock = &vfd->v4l2_dev->mdev->req_queue_mutex;
 
 		if (mutex_lock_interruptible(req_queue_lock))
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index 32e6f60e26c71..9efd14d4026f2 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -2545,6 +2545,119 @@ int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable)
 }
 EXPORT_SYMBOL_GPL(v4l2_subdev_s_stream_helper);
 
+int v4l2_subdev_get_frame_desc_passthrough(struct v4l2_subdev *sd,
+					   unsigned int pad,
+					   struct v4l2_mbus_frame_desc *fd)
+{
+	const struct media_pad *pads = sd->entity.pads;
+	struct media_pad *local_sink_pad;
+	struct v4l2_subdev_route *route;
+	struct v4l2_subdev_state *state;
+	struct device *dev = sd->dev;
+	int ret = 0;
+
+	if (WARN_ON(!(pads[pad].flags & MEDIA_PAD_FL_SOURCE)))
+		return -EINVAL;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+
+	/* Iterate over sink pads */
+	media_entity_for_each_pad(&sd->entity, local_sink_pad) {
+		struct v4l2_mbus_frame_desc source_fd;
+		bool have_source_fd = false;
+
+		if (!(local_sink_pad->flags & MEDIA_PAD_FL_SINK))
+			continue;
+
+		/*
+		 * Copy frame desc entries for the streams going from the sink
+		 * pad to the requested pad
+		 */
+		for_each_active_route(&state->routing, route) {
+			struct v4l2_mbus_frame_desc_entry *source_entry = NULL;
+			struct media_pad *remote_source_pad;
+			struct v4l2_subdev *remote_sd;
+			unsigned int i;
+
+			if (route->source_pad != pad ||
+			    route->sink_pad != local_sink_pad->index)
+				continue;
+
+			if (!have_source_fd) {
+				remote_source_pad = media_pad_remote_pad_unique(local_sink_pad);
+				if (!remote_source_pad) {
+					dev_dbg(dev, "Failed to find remote pad for sink pad %u\n",
+						local_sink_pad->index);
+					ret = -EINVAL;
+					goto out_unlock;
+				}
+
+				remote_sd = media_entity_to_v4l2_subdev(remote_source_pad->entity);
+				if (!remote_sd) {
+					ret = -EINVAL;
+					goto out_unlock;
+				}
+
+				ret = v4l2_subdev_call(remote_sd, pad,
+						       get_frame_desc,
+						       remote_source_pad->index,
+						       &source_fd);
+				if (ret) {
+					dev_err(dev,
+						"Failed to get frame desc from remote subdev %s\n",
+						remote_sd->name);
+					goto out_unlock;
+				}
+
+				have_source_fd = true;
+
+				if (fd->num_entries == 0) {
+					fd->type = source_fd.type;
+				} else if (fd->type != source_fd.type) {
+					dev_err(dev,
+						"Frame desc type mismatch: %u != %u\n",
+						fd->type, source_fd.type);
+					ret = -EPIPE;
+					goto out_unlock;
+				}
+			}
+
+			for (i = 0; i < source_fd.num_entries; i++) {
+				if (source_fd.entry[i].stream == route->sink_stream) {
+					source_entry = &source_fd.entry[i];
+					break;
+				}
+			}
+
+			if (!source_entry) {
+				dev_dbg(sd->dev,
+					"Failed to find stream %u from source frame desc\n",
+					route->sink_stream);
+				ret = -EPIPE;
+				goto out_unlock;
+			}
+
+			if (fd->num_entries >= V4L2_FRAME_DESC_ENTRY_MAX) {
+				dev_dbg(sd->dev, "Frame desc entry limit reached\n");
+				ret = -ENOSPC;
+				goto out_unlock;
+			}
+
+			fd->entry[fd->num_entries] = *source_entry;
+
+			fd->entry[fd->num_entries].stream = route->source_stream;
+
+			fd->num_entries++;
+		}
+	}
+
+out_unlock:
+	v4l2_subdev_unlock_state(state);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(v4l2_subdev_get_frame_desc_passthrough);
+
 #endif /* CONFIG_VIDEO_V4L2_SUBDEV_API */
 
 #endif /* CONFIG_MEDIA_CONTROLLER */
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index 79df0d22e218a..e5527020ff337 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -64,17 +64,6 @@ config BRCMSTB_MEMC
 	  controller and specifically control the Self Refresh Power Down
 	  (SRPD) inactivity timeout.
 
-config BT1_L2_CTL
-	bool "Baikal-T1 CM2 L2-RAM Cache Control Block"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
-	select MFD_SYSCON
-	help
-	  Baikal-T1 CPU is based on the MIPS P5600 Warrior IP-core. The CPU
-	  resides Coherency Manager v2 with embedded 1MB L2-cache. It's
-	  possible to tune the L2 cache performance up by setting the data,
-	  tags and way-select latencies of RAM access. This driver provides a
-	  dt properties-based and sysfs interface for it.
-
 config TI_AEMIF
 	tristate "Texas Instruments AEMIF driver"
 	depends on ARCH_DAVINCI || ARCH_KEYSTONE || COMPILE_TEST
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index c1959661bf637..3ee883c8759a9 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -11,7 +11,6 @@ obj-$(CONFIG_ARM_PL172_MPMC)	+= pl172.o
 obj-$(CONFIG_ATMEL_EBI)		+= atmel-ebi.o
 obj-$(CONFIG_BRCMSTB_DPFE)	+= brcmstb_dpfe.o
 obj-$(CONFIG_BRCMSTB_MEMC)	+= brcmstb_memc.o
-obj-$(CONFIG_BT1_L2_CTL)	+= bt1-l2-ctl.o
 obj-$(CONFIG_TI_AEMIF)		+= ti-aemif.o
 obj-$(CONFIG_TI_EMIF)		+= emif.o
 obj-$(CONFIG_OMAP_GPMC)		+= omap-gpmc.o
diff --git a/drivers/memory/brcmstb_memc.c b/drivers/memory/brcmstb_memc.c
index ba73470b1b134..c28fe90936168 100644
--- a/drivers/memory/brcmstb_memc.c
+++ b/drivers/memory/brcmstb_memc.c
@@ -14,6 +14,7 @@
 
 #define REG_MEMC_CNTRLR_CONFIG		0x00
 #define  CNTRLR_CONFIG_LPDDR4_SHIFT	5
+#define  CNTRLR_CONFIG_LPDDR5_SHIFT	6
 #define  CNTRLR_CONFIG_MASK		0xf
 #define REG_MEMC_SRPD_CFG_21		0x20
 #define REG_MEMC_SRPD_CFG_20		0x34
@@ -34,14 +35,15 @@ struct brcmstb_memc {
 	u32 srpd_offset;
 };
 
-static int brcmstb_memc_uses_lpddr4(struct brcmstb_memc *memc)
+static int brcmstb_memc_uses_lpddr45(struct brcmstb_memc *memc)
 {
 	void __iomem *config = memc->ddr_ctrl + REG_MEMC_CNTRLR_CONFIG;
 	u32 reg;
 
 	reg = readl_relaxed(config) & CNTRLR_CONFIG_MASK;
 
-	return reg == CNTRLR_CONFIG_LPDDR4_SHIFT;
+	return reg == CNTRLR_CONFIG_LPDDR4_SHIFT ||
+	       reg == CNTRLR_CONFIG_LPDDR5_SHIFT;
 }
 
 static int brcmstb_memc_srpd_config(struct brcmstb_memc *memc,
@@ -95,7 +97,7 @@ static ssize_t srpd_store(struct device *dev, struct device_attribute *attr,
 	 * dynamic tuning process will also get affected by the inactivity
 	 * timeout, thus making it non functional.
 	 */
-	if (brcmstb_memc_uses_lpddr4(memc))
+	if (brcmstb_memc_uses_lpddr45(memc))
 		return -EOPNOTSUPP;
 
 	ret = kstrtouint(buf, 10, &val);
diff --git a/drivers/memory/bt1-l2-ctl.c b/drivers/memory/bt1-l2-ctl.c
deleted file mode 100644
index 0fd96abc172a1..0000000000000
--- a/drivers/memory/bt1-l2-ctl.c
+++ /dev/null
@@ -1,323 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 CM2 L2-cache Control Block driver.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/bitfield.h>
-#include <linux/types.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/regmap.h>
-#include <linux/mfd/syscon.h>
-#include <linux/sysfs.h>
-#include <linux/of.h>
-
-#define L2_CTL_REG			0x028
-#define L2_CTL_DATA_STALL_FLD		0
-#define L2_CTL_DATA_STALL_MASK		GENMASK(1, L2_CTL_DATA_STALL_FLD)
-#define L2_CTL_TAG_STALL_FLD		2
-#define L2_CTL_TAG_STALL_MASK		GENMASK(3, L2_CTL_TAG_STALL_FLD)
-#define L2_CTL_WS_STALL_FLD		4
-#define L2_CTL_WS_STALL_MASK		GENMASK(5, L2_CTL_WS_STALL_FLD)
-#define L2_CTL_SET_CLKRATIO		BIT(13)
-#define L2_CTL_CLKRATIO_LOCK		BIT(31)
-
-#define L2_CTL_STALL_MIN		0
-#define L2_CTL_STALL_MAX		3
-#define L2_CTL_STALL_SET_DELAY_US	1
-#define L2_CTL_STALL_SET_TOUT_US	1000
-
-/*
- * struct l2_ctl - Baikal-T1 L2 Control block private data.
- * @dev: Pointer to the device structure.
- * @sys_regs: Baikal-T1 System Controller registers map.
- */
-struct l2_ctl {
-	struct device *dev;
-
-	struct regmap *sys_regs;
-};
-
-/*
- * enum l2_ctl_stall - Baikal-T1 L2-cache-RAM stall identifier.
- * @L2_WSSTALL: Way-select latency.
- * @L2_TAGSTALL: Tag latency.
- * @L2_DATASTALL: Data latency.
- */
-enum l2_ctl_stall {
-	L2_WS_STALL,
-	L2_TAG_STALL,
-	L2_DATA_STALL
-};
-
-/*
- * struct l2_ctl_device_attribute - Baikal-T1 L2-cache device attribute.
- * @dev_attr: Actual sysfs device attribute.
- * @id: L2-cache stall field identifier.
- */
-struct l2_ctl_device_attribute {
-	struct device_attribute dev_attr;
-	enum l2_ctl_stall id;
-};
-
-#define to_l2_ctl_dev_attr(_dev_attr) \
-	container_of(_dev_attr, struct l2_ctl_device_attribute, dev_attr)
-
-#define L2_CTL_ATTR_RW(_name, _prefix, _id) \
-	struct l2_ctl_device_attribute l2_ctl_attr_##_name = \
-		{ __ATTR(_name, 0644, _prefix##_show, _prefix##_store),	_id }
-
-static int l2_ctl_get_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 *val)
-{
-	u32 data = 0;
-	int ret;
-
-	ret = regmap_read(l2->sys_regs, L2_CTL_REG, &data);
-	if (ret)
-		return ret;
-
-	switch (id) {
-	case L2_WS_STALL:
-		*val = FIELD_GET(L2_CTL_WS_STALL_MASK, data);
-		break;
-	case L2_TAG_STALL:
-		*val = FIELD_GET(L2_CTL_TAG_STALL_MASK, data);
-		break;
-	case L2_DATA_STALL:
-		*val = FIELD_GET(L2_CTL_DATA_STALL_MASK, data);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int l2_ctl_set_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 val)
-{
-	u32 mask = 0, data = 0;
-	int ret;
-
-	val = clamp_val(val, L2_CTL_STALL_MIN, L2_CTL_STALL_MAX);
-
-	switch (id) {
-	case L2_WS_STALL:
-		data = FIELD_PREP(L2_CTL_WS_STALL_MASK, val);
-		mask = L2_CTL_WS_STALL_MASK;
-		break;
-	case L2_TAG_STALL:
-		data = FIELD_PREP(L2_CTL_TAG_STALL_MASK, val);
-		mask = L2_CTL_TAG_STALL_MASK;
-		break;
-	case L2_DATA_STALL:
-		data = FIELD_PREP(L2_CTL_DATA_STALL_MASK, val);
-		mask = L2_CTL_DATA_STALL_MASK;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	data |= L2_CTL_SET_CLKRATIO;
-	mask |= L2_CTL_SET_CLKRATIO;
-
-	ret = regmap_update_bits(l2->sys_regs, L2_CTL_REG, mask, data);
-	if (ret)
-		return ret;
-
-	return regmap_read_poll_timeout(l2->sys_regs, L2_CTL_REG, data,
-					data & L2_CTL_CLKRATIO_LOCK,
-					L2_CTL_STALL_SET_DELAY_US,
-					L2_CTL_STALL_SET_TOUT_US);
-}
-
-static void l2_ctl_clear_data(void *data)
-{
-	struct l2_ctl *l2 = data;
-	struct platform_device *pdev = to_platform_device(l2->dev);
-
-	platform_set_drvdata(pdev, NULL);
-}
-
-static struct l2_ctl *l2_ctl_create_data(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct l2_ctl *l2;
-	int ret;
-
-	l2 = devm_kzalloc(dev, sizeof(*l2), GFP_KERNEL);
-	if (!l2)
-		return ERR_PTR(-ENOMEM);
-
-	ret = devm_add_action(dev, l2_ctl_clear_data, l2);
-	if (ret) {
-		dev_err(dev, "Can't add L2 CTL data clear action\n");
-		return ERR_PTR(ret);
-	}
-
-	l2->dev = dev;
-	platform_set_drvdata(pdev, l2);
-
-	return l2;
-}
-
-static int l2_ctl_find_sys_regs(struct l2_ctl *l2)
-{
-	l2->sys_regs = syscon_node_to_regmap(l2->dev->of_node->parent);
-	if (IS_ERR(l2->sys_regs)) {
-		dev_err(l2->dev, "Couldn't get L2 CTL register map\n");
-		return PTR_ERR(l2->sys_regs);
-	}
-
-	return 0;
-}
-
-static int l2_ctl_of_parse_property(struct l2_ctl *l2, enum l2_ctl_stall id,
-				    const char *propname)
-{
-	int ret = 0;
-	u32 data;
-
-	if (!of_property_read_u32(l2->dev->of_node, propname, &data)) {
-		ret = l2_ctl_set_latency(l2, id, data);
-		if (ret)
-			dev_err(l2->dev, "Invalid value of '%s'\n", propname);
-	}
-
-	return ret;
-}
-
-static int l2_ctl_of_parse(struct l2_ctl *l2)
-{
-	int ret;
-
-	ret = l2_ctl_of_parse_property(l2, L2_WS_STALL, "baikal,l2-ws-latency");
-	if (ret)
-		return ret;
-
-	ret = l2_ctl_of_parse_property(l2, L2_TAG_STALL, "baikal,l2-tag-latency");
-	if (ret)
-		return ret;
-
-	return l2_ctl_of_parse_property(l2, L2_DATA_STALL,
-					"baikal,l2-data-latency");
-}
-
-static ssize_t l2_ctl_latency_show(struct device *dev,
-				   struct device_attribute *attr,
-				   char *buf)
-{
-	struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr);
-	struct l2_ctl *l2 = dev_get_drvdata(dev);
-	u32 data;
-	int ret;
-
-	ret = l2_ctl_get_latency(l2, devattr->id, &data);
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%u\n", data);
-}
-
-static ssize_t l2_ctl_latency_store(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
-{
-	struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr);
-	struct l2_ctl *l2 = dev_get_drvdata(dev);
-	u32 data;
-	int ret;
-
-	if (kstrtouint(buf, 0, &data) < 0)
-		return -EINVAL;
-
-	ret = l2_ctl_set_latency(l2, devattr->id, data);
-	if (ret)
-		return ret;
-
-	return count;
-}
-
-static L2_CTL_ATTR_RW(l2_ws_latency, l2_ctl_latency, L2_WS_STALL);
-static L2_CTL_ATTR_RW(l2_tag_latency, l2_ctl_latency, L2_TAG_STALL);
-static L2_CTL_ATTR_RW(l2_data_latency, l2_ctl_latency, L2_DATA_STALL);
-
-static struct attribute *l2_ctl_sysfs_attrs[] = {
-	&l2_ctl_attr_l2_ws_latency.dev_attr.attr,
-	&l2_ctl_attr_l2_tag_latency.dev_attr.attr,
-	&l2_ctl_attr_l2_data_latency.dev_attr.attr,
-	NULL
-};
-ATTRIBUTE_GROUPS(l2_ctl_sysfs);
-
-static void l2_ctl_remove_sysfs(void *data)
-{
-	struct l2_ctl *l2 = data;
-
-	device_remove_groups(l2->dev, l2_ctl_sysfs_groups);
-}
-
-static int l2_ctl_init_sysfs(struct l2_ctl *l2)
-{
-	int ret;
-
-	ret = device_add_groups(l2->dev, l2_ctl_sysfs_groups);
-	if (ret) {
-		dev_err(l2->dev, "Failed to create L2 CTL sysfs nodes\n");
-		return ret;
-	}
-
-	ret = devm_add_action_or_reset(l2->dev, l2_ctl_remove_sysfs, l2);
-	if (ret)
-		dev_err(l2->dev, "Can't add L2 CTL sysfs remove action\n");
-
-	return ret;
-}
-
-static int l2_ctl_probe(struct platform_device *pdev)
-{
-	struct l2_ctl *l2;
-	int ret;
-
-	l2 = l2_ctl_create_data(pdev);
-	if (IS_ERR(l2))
-		return PTR_ERR(l2);
-
-	ret = l2_ctl_find_sys_regs(l2);
-	if (ret)
-		return ret;
-
-	ret = l2_ctl_of_parse(l2);
-	if (ret)
-		return ret;
-
-	ret = l2_ctl_init_sysfs(l2);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static const struct of_device_id l2_ctl_of_match[] = {
-	{ .compatible = "baikal,bt1-l2-ctl" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, l2_ctl_of_match);
-
-static struct platform_driver l2_ctl_driver = {
-	.probe = l2_ctl_probe,
-	.driver = {
-		.name = "bt1-l2-ctl",
-		.of_match_table = l2_ctl_of_match
-	}
-};
-module_platform_driver(l2_ctl_driver);
-
-MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
-MODULE_DESCRIPTION("Baikal-T1 L2-cache driver");
diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c
index 58ccc1c02e906..0fb5684561648 100644
--- a/drivers/memory/renesas-rpc-if.c
+++ b/drivers/memory/renesas-rpc-if.c
@@ -1005,11 +1005,9 @@ static int rpcif_probe(struct platform_device *pdev)
 		return PTR_ERR(rpc->base);
 	rpc->info = of_device_get_match_data(dev);
 	rpc->regmap = devm_regmap_init(dev, NULL, rpc, rpc->info->regmap_config);
-	if (IS_ERR(rpc->regmap)) {
-		dev_err(dev, "failed to init regmap for rpcif, error %ld\n",
-			PTR_ERR(rpc->regmap));
-		return	PTR_ERR(rpc->regmap);
-	}
+	if (IS_ERR(rpc->regmap))
+		return dev_err_probe(dev, PTR_ERR(rpc->regmap),
+				     "failed to init regmap for rpcif\n");
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dirmap");
 	rpc->dirmap = devm_ioremap_resource(dev, res);
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c
index 6edb210287dca..d620660da3311 100644
--- a/drivers/memory/tegra/mc.c
+++ b/drivers/memory/tegra/mc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2014-2025 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/clk.h>
@@ -56,6 +56,23 @@ static const struct of_device_id tegra_mc_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
 
+const struct tegra_mc_regs tegra20_mc_regs = {
+	.cfg_channel_enable = 0xdf8,
+	.err_status = 0x08,
+	.err_add = 0x0c,
+	.err_add_hi = 0x11fc,
+	.err_vpr_status = 0x654,
+	.err_vpr_add = 0x658,
+	.err_sec_status = 0x67c,
+	.err_sec_add = 0x680,
+	.err_mts_status = 0x9b0,
+	.err_mts_add = 0x9b4,
+	.err_gen_co_status = 0xc00,
+	.err_gen_co_add = 0xc04,
+	.err_route_status = 0x9c0,
+	.err_route_add = 0x9c4,
+};
+
 static void tegra_mc_devm_action_put_device(void *data)
 {
 	struct tegra_mc *mc = data;
@@ -381,12 +398,16 @@ unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 }
 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 
+const irq_handler_t tegra30_mc_irq_handlers[] = {
+	tegra30_mc_handle_irq
+};
+
 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
     defined(CONFIG_ARCH_TEGRA_114_SOC) || \
     defined(CONFIG_ARCH_TEGRA_124_SOC) || \
     defined(CONFIG_ARCH_TEGRA_132_SOC) || \
     defined(CONFIG_ARCH_TEGRA_210_SOC)
-static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
+static void tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 {
 	unsigned long long tick;
 	unsigned int i;
@@ -414,8 +435,6 @@ static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 
 	/* latch new values */
 	mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
-
-	return 0;
 }
 
 static int load_one_timing(struct tegra_mc *mc,
@@ -509,32 +528,24 @@ int tegra30_mc_probe(struct tegra_mc *mc)
 	int err;
 
 	mc->clk = devm_clk_get_optional(mc->dev, "mc");
-	if (IS_ERR(mc->clk)) {
-		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
-		return PTR_ERR(mc->clk);
-	}
+	if (IS_ERR(mc->clk))
+		return dev_err_probe(mc->dev, PTR_ERR(mc->clk),
+				     "failed to get MC clock\n");
 
 	/* ensure that debug features are disabled */
 	mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 
-	err = tegra_mc_setup_latency_allowance(mc);
-	if (err < 0) {
-		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
-		return err;
-	}
+	tegra_mc_setup_latency_allowance(mc);
 
 	err = tegra_mc_setup_timings(mc);
-	if (err < 0) {
-		dev_err(mc->dev, "failed to setup timings: %d\n", err);
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(mc->dev, err, "failed to setup timings\n");
 
 	return 0;
 }
 
 const struct tegra_mc_ops tegra30_mc_ops = {
 	.probe = tegra30_mc_probe,
-	.handle_irq = tegra30_mc_handle_irq,
 };
 #endif
 
@@ -575,9 +586,9 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 		}
 
 		/* mask all interrupts to avoid flooding */
-		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
+		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmasks[0].mask;
 	} else {
-		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
+		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmasks[0].mask;
 	}
 
 	if (!status)
@@ -600,37 +611,37 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 
 		switch (intmask) {
 		case MC_INT_DECERR_VPR:
-			status_reg = MC_ERR_VPR_STATUS;
-			addr_reg = MC_ERR_VPR_ADR;
+			status_reg = mc->soc->regs->err_vpr_status;
+			addr_reg = mc->soc->regs->err_vpr_add;
 			break;
 
 		case MC_INT_SECERR_SEC:
-			status_reg = MC_ERR_SEC_STATUS;
-			addr_reg = MC_ERR_SEC_ADR;
+			status_reg = mc->soc->regs->err_sec_status;
+			addr_reg = mc->soc->regs->err_sec_add;
 			break;
 
 		case MC_INT_DECERR_MTS:
-			status_reg = MC_ERR_MTS_STATUS;
-			addr_reg = MC_ERR_MTS_ADR;
+			status_reg = mc->soc->regs->err_mts_status;
+			addr_reg = mc->soc->regs->err_mts_add;
 			break;
 
 		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
-			status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
-			addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
+			status_reg = mc->soc->regs->err_gen_co_status;
+			addr_reg = mc->soc->regs->err_gen_co_add;
 			break;
 
 		case MC_INT_DECERR_ROUTE_SANITY:
-			status_reg = MC_ERR_ROUTE_SANITY_STATUS;
-			addr_reg = MC_ERR_ROUTE_SANITY_ADR;
+			status_reg = mc->soc->regs->err_route_status;
+			addr_reg = mc->soc->regs->err_route_add;
 			break;
 
 		default:
-			status_reg = MC_ERR_STATUS;
-			addr_reg = MC_ERR_ADR;
+			status_reg = mc->soc->regs->err_status;
+			addr_reg = mc->soc->regs->err_add;
 
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 			if (mc->soc->has_addr_hi_reg)
-				addr_hi_reg = MC_ERR_ADR_HI;
+				addr_hi_reg = mc->soc->regs->err_add_hi;
 #endif
 			break;
 		}
@@ -647,9 +658,12 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 					addr = mc_ch_readl(mc, channel, addr_hi_reg);
 				else
 					addr = mc_readl(mc, addr_hi_reg);
-			} else {
+			} else if (mc->soc->mc_addr_hi_mask) {
 				addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
-					MC_ERR_STATUS_ADR_HI_MASK);
+					mc->soc->mc_addr_hi_mask);
+			} else {
+				dev_err_ratelimited(mc->dev, "Unable to determine high address!");
+				return IRQ_NONE;
 			}
 			addr <<= 32;
 		}
@@ -674,11 +688,11 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 			}
 		}
 
-		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
+		type = (value & mc->soc->mc_err_status_type_mask) >>
 		       MC_ERR_STATUS_TYPE_SHIFT;
-		desc = tegra_mc_error_names[type];
+		desc = tegra20_mc_error_names[type];
 
-		switch (value & MC_ERR_STATUS_TYPE_MASK) {
+		switch (value & mc->soc->mc_err_status_type_mask) {
 		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 			perm[0] = ' ';
 			perm[1] = '[';
@@ -744,9 +758,10 @@ const char *const tegra_mc_status_names[32] = {
 	[16] = "MTS carveout violation",
 	[17] = "Generalized carveout violation",
 	[20] = "Route Sanity error",
+	[21] = "GIC_MSI error",
 };
 
-const char *const tegra_mc_error_names[8] = {
+const char *const tegra20_mc_error_names[8] = {
 	[2] = "EMEM decode error",
 	[3] = "TrustZone violation",
 	[4] = "Carveout violation",
@@ -883,7 +898,7 @@ static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
 	unsigned int i;
 	u32 value;
 
-	value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
+	value = mc_ch_readl(mc, 0, mc->soc->regs->cfg_channel_enable);
 	if (value <= 0) {
 		mc->num_channels = mc->soc->num_channels;
 		return;
@@ -935,25 +950,32 @@ static int tegra_mc_probe(struct platform_device *pdev)
 
 	tegra_mc_num_channel_enabled(mc);
 
-	if (mc->soc->ops && mc->soc->ops->handle_irq) {
-		mc->irq = platform_get_irq(pdev, 0);
-		if (mc->irq < 0)
-			return mc->irq;
+	if (mc->soc->handle_irq) {
+		unsigned int i;
 
 		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 
-		if (mc->soc->num_channels)
-			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
-				     MC_INTMASK);
-		else
-			mc_writel(mc, mc->soc->intmask, MC_INTMASK);
+		for (i = 0; i < mc->soc->num_interrupts; i++) {
+			int irq;
 
-		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
-				       dev_name(&pdev->dev), mc);
-		if (err < 0) {
-			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
-				err);
-			return err;
+			irq = platform_get_irq(pdev, i);
+			if (irq < 0)
+				return irq;
+
+			err = devm_request_irq(&pdev->dev, irq, mc->soc->handle_irq[i], 0,
+					       dev_name(&pdev->dev), mc);
+			if (err < 0) {
+				dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err);
+				return err;
+			}
+		}
+
+		for (i = 0; i < mc->soc->num_intmasks; i++) {
+			if (mc->soc->num_channels)
+				mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmasks[i].mask,
+					     mc->soc->intmasks[i].reg);
+			else
+				mc_writel(mc, mc->soc->intmasks[i].mask, mc->soc->intmasks[i].reg);
 		}
 	}
 
@@ -971,8 +993,7 @@ static int tegra_mc_probe(struct platform_device *pdev)
 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 		if (IS_ERR(mc->smmu)) {
-			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
-				PTR_ERR(mc->smmu));
+			dev_err(&pdev->dev, "failed to probe SMMU: %pe\n", mc->smmu);
 			mc->smmu = NULL;
 		}
 	}
diff --git a/drivers/memory/tegra/mc.h b/drivers/memory/tegra/mc.h
index 1d97cf4d3a944..649b54369263a 100644
--- a/drivers/memory/tegra/mc.h
+++ b/drivers/memory/tegra/mc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2014-2025 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #ifndef MEMORY_TEGRA_MC_H
@@ -13,15 +13,36 @@
 #include <soc/tegra/mc.h>
 
 #define MC_INTSTATUS					0x00
+/* Bit field of MC_INTSTATUS register */
+#define MC_INT_DECERR_EMEM				BIT(6)
+#define MC_INT_INVALID_GART_PAGE			BIT(7)
+#define MC_INT_SECURITY_VIOLATION			BIT(8)
+#define MC_INT_ARBITRATION_EMEM				BIT(9)
+#define MC_INT_INVALID_SMMU_PAGE			BIT(10)
+#define MC_INT_INVALID_APB_ASID_UPDATE			BIT(11)
+#define MC_INT_DECERR_VPR				BIT(12)
+#define MC_INT_SECERR_SEC				BIT(13)
+#define MC_INT_DECERR_MTS				BIT(16)
+#define MC_INT_DECERR_GENERALIZED_CARVEOUT		BIT(17)
+#define MC_INT_DECERR_ROUTE_SANITY			BIT(20)
+#define MC_INT_DECERR_ROUTE_SANITY_GIC_MSI		BIT(21)
+
 #define MC_INTMASK					0x04
-#define MC_ERR_STATUS					0x08
-#define MC_ERR_ADR					0x0c
 #define MC_GART_ERROR_REQ				0x30
 #define MC_EMEM_ADR_CFG					0x54
+#define MC_EMEM_ADR_CFG_EMEM_NUMDEV			BIT(0)
+
 #define MC_DECERR_EMEM_OTHERS_STATUS			0x58
 #define MC_SECURITY_VIOLATION_STATUS			0x74
 #define MC_EMEM_ARB_CFG					0x90
+#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x)		((x) & 0x1ff)
+#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK		0x1ff
+
 #define MC_EMEM_ARB_OUTSTANDING_REQ			0x94
+#define MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE	BIT(30)
+#define MC_EMEM_ARB_OUTSTANDING_REQ_LIMIT_ENABLE	BIT(31)
+#define MC_EMEM_ARB_OUTSTANDING_REQ_MAX_MASK		0x1ff
+
 #define MC_EMEM_ARB_TIMING_RCD				0x98
 #define MC_EMEM_ARB_TIMING_RP				0x9c
 #define MC_EMEM_ARB_TIMING_RC				0xa0
@@ -41,60 +62,97 @@
 #define MC_EMEM_ARB_MISC1				0xdc
 #define MC_EMEM_ARB_RING1_THROTTLE			0xe0
 #define MC_EMEM_ARB_OVERRIDE				0xe8
+#define MC_EMEM_ARB_OVERRIDE_EACK_MASK			0x3
+
 #define MC_TIMING_CONTROL_DBG				0xf8
 #define MC_TIMING_CONTROL				0xfc
-#define MC_ERR_VPR_STATUS				0x654
-#define MC_ERR_VPR_ADR					0x658
-#define MC_ERR_SEC_STATUS				0x67c
-#define MC_ERR_SEC_ADR					0x680
-#define MC_ERR_MTS_STATUS				0x9b0
-#define MC_ERR_MTS_ADR					0x9b4
-#define MC_ERR_ROUTE_SANITY_STATUS			0x9c0
-#define MC_ERR_ROUTE_SANITY_ADR				0x9c4
-#define MC_ERR_GENERALIZED_CARVEOUT_STATUS		0xc00
-#define MC_ERR_GENERALIZED_CARVEOUT_ADR			0xc04
-#define MC_EMEM_ADR_CFG_CHANNEL_ENABLE			0xdf8
-#define MC_GLOBAL_INTSTATUS				0xf24
-#define MC_ERR_ADR_HI					0x11fc
+#define MC_TIMING_UPDATE				BIT(0)
 
-#define MC_INT_DECERR_ROUTE_SANITY			BIT(20)
-#define MC_INT_DECERR_GENERALIZED_CARVEOUT		BIT(17)
-#define MC_INT_DECERR_MTS				BIT(16)
-#define MC_INT_SECERR_SEC				BIT(13)
-#define MC_INT_DECERR_VPR				BIT(12)
-#define MC_INT_INVALID_APB_ASID_UPDATE			BIT(11)
-#define MC_INT_INVALID_SMMU_PAGE			BIT(10)
-#define MC_INT_ARBITRATION_EMEM				BIT(9)
-#define MC_INT_SECURITY_VIOLATION			BIT(8)
-#define MC_INT_INVALID_GART_PAGE			BIT(7)
-#define MC_INT_DECERR_EMEM				BIT(6)
+#define MC_GLOBAL_INTSTATUS				0xf24
 
-#define MC_ERR_STATUS_TYPE_SHIFT			28
-#define MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE		(0x6 << 28)
-#define MC_ERR_STATUS_TYPE_MASK				(0x7 << 28)
-#define MC_ERR_STATUS_READABLE				BIT(27)
-#define MC_ERR_STATUS_WRITABLE				BIT(26)
-#define MC_ERR_STATUS_NONSECURE				BIT(25)
-#define MC_ERR_STATUS_ADR_HI_SHIFT			20
-#define MC_ERR_STATUS_ADR_HI_MASK			0x3
-#define MC_ERR_STATUS_SECURITY				BIT(17)
+/* Bit field of MC_ERR_STATUS_0 register */
 #define MC_ERR_STATUS_RW				BIT(16)
+#define MC_ERR_STATUS_SECURITY				BIT(17)
+#define MC_ERR_STATUS_NONSECURE				BIT(25)
+#define MC_ERR_STATUS_WRITABLE				BIT(26)
+#define MC_ERR_STATUS_READABLE				BIT(27)
 
-#define MC_EMEM_ADR_CFG_EMEM_NUMDEV			BIT(0)
-
-#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x)		((x) & 0x1ff)
-#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK		0x1ff
-
-#define MC_EMEM_ARB_OUTSTANDING_REQ_MAX_MASK		0x1ff
-#define MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE	BIT(30)
-#define MC_EMEM_ARB_OUTSTANDING_REQ_LIMIT_ENABLE	BIT(31)
+#define MC_ERR_STATUS_GSC_ADR_HI_MASK			0xffff
+#define MC_ERR_STATUS_GSC_ADR_HI_SHIFT			16
+#define MC_ERR_STATUS_RT_ADR_HI_SHIFT			15
 
-#define MC_EMEM_ARB_OVERRIDE_EACK_MASK			0x3
+#define MC_ERR_STATUS_TYPE_SHIFT			28
+#define MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE		(0x6 << 28)
+#define MC_ERR_STATUS_RT_TYPE_MASK			(0xf << 28)
+#define MC_ERR_STATUS_RT_TYPE_SHIFT			28
 
-#define MC_TIMING_UPDATE				BIT(0)
+#define MC_ERR_STATUS_ADR_HI_SHIFT			20
 
 #define MC_BROADCAST_CHANNEL				~0
 
+/* Tegra264 specific registers */
+
+/* Registers for MSS HUB */
+#define MSS_HUB_GLOBAL_INTSTATUS_0			0x6000
+#define MSS_HUBC_INTR					BIT(0)
+#define MSS_HUB_GLOBAL_MASK				0x7F00
+#define MSS_HUB_GLOBAL_SHIFT				8
+
+#define MSS_HUB_HUBC_INTSTATUS_0			0x6008
+#define MSS_HUB_INTRSTATUS_0				0x600c
+#define MSS_HUB_HUBC_INTMASK_0				0x6010
+#define MSS_HUB_HUBC_SCRUB_DONE_INTMASK			BIT(0)
+
+#define MSS_HUB_HUBC_INTPRIORITY_0			0x6014
+#define MSS_HUB_INTRMASK_0				0x6018
+#define MSS_HUB_COALESCER_ERR_INTMASK			BIT(0)
+#define MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK		BIT(1)
+#define MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK		BIT(2)
+#define MSS_HUB_MSI_ERR_INTMASK				BIT(3)
+#define MSS_HUB_POISON_RSP_INTMASK			BIT(4)
+#define MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK		BIT(5)
+#define MSS_HUB_RESERVED_PA_ERR_INTMASK			BIT(6)
+
+#define MSS_HUB_INTRPRIORITY_0				0x601c
+#define MSS_HUB_SMMU_BYPASS_ALLOW_ERR_STATUS_0		0x6020
+#define MSS_HUB_MSI_ERR_STATUS_0			0x6024
+#define MSS_HUB_POISON_RSP_STATUS_0			0x6028
+#define MSS_HUB_COALESCE_ERR_STATUS_0			0x60e0
+#define MSS_HUB_COALESCE_ERR_ADR_HI_0			0x60e4
+#define MSS_HUB_COALESCE_ERR_ADR_0			0x60e8
+#define MSS_HUB_RESTRICTED_ACCESS_ERR_STATUS_0		0x638c
+#define MSS_HUB_RESERVED_PA_ERR_STATUS_0		0x6390
+#define MSS_HUB_ILLEGAL_TBUGRP_ID_ERR_STATUS_0		0x63b0
+
+/* Registers for channels */
+#define MC_CH_INTSTATUS_0				0x82d4
+#define MC_CH_INTMASK_0					0x82d8
+#define WCAM_ERR_INTMASK				BIT(19)
+
+#define MC_ERR_GENERALIZED_CARVEOUT_STATUS_1_0		0xbc74
+
+/* Registers for MCF */
+#define MCF_COMMON_INTSTATUS0_0_0			0xce04
+#define MCF_INTSTATUS_0					0xce2c
+#define MCF_INTMASK_0					0xce30
+#define MCF_INTPRIORITY_0				0xce34
+
+/* Registers for SBS */
+#define MSS_SBS_INTSTATUS_0				0xec08
+#define MSS_SBS_INTMASK_0				0xec0c
+#define MSS_SBS_FILL_FIFO_ISO_OVERFLOW_INTMASK		BIT(0)
+#define MSS_SBS_FILL_FIFO_SISO_OVERFLOW_INTMASK		BIT(1)
+#define MSS_SBS_FILL_FIFO_NISO_OVERFLOW_INTMASK		BIT(2)
+
+/* Bit field of MC_ERR_ROUTE_SANITY_STATUS_0 register */
+#define MC_ERR_ROUTE_SANITY_RW				BIT(12)
+#define MC_ERR_ROUTE_SANITY_SEC				BIT(13)
+
+#define ERR_GENERALIZED_APERTURE_ID_SHIFT		0
+#define ERR_GENERALIZED_APERTURE_ID_MASK		0x1F
+#define ERR_GENERALIZED_CARVEOUT_APERTURE_ID_SHIFT	5
+#define ERR_GENERALIZED_CARVEOUT_APERTURE_ID_MASK	0x1F
+
 static inline u32 tegra_mc_scale_percents(u64 val, unsigned int percents)
 {
 	val = val * percents;
@@ -203,8 +261,9 @@ extern const struct tegra_mc_ops tegra186_mc_ops;
 #endif
 
 irqreturn_t tegra30_mc_handle_irq(int irq, void *data);
+extern const irq_handler_t tegra30_mc_irq_handlers[1];
 extern const char * const tegra_mc_status_names[32];
-extern const char * const tegra_mc_error_names[8];
+extern const char * const tegra20_mc_error_names[8];
 
 /*
  * These IDs are for internal use of Tegra ICC drivers. The ID numbers are
diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c
index 41350570c815c..02dd4e26288a1 100644
--- a/drivers/memory/tegra/tegra114.c
+++ b/drivers/memory/tegra/tegra114.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/of.h>
@@ -1101,6 +1101,14 @@ static const struct tegra_mc_reset tegra114_mc_resets[] = {
 	TEGRA114_MC_RESET(VI,       0x200, 0x204, 17),
 };
 
+static const struct tegra_mc_intmask tegra114_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
+			MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra114_mc_soc = {
 	.clients = tegra114_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra114_mc_clients),
@@ -1108,10 +1116,14 @@ const struct tegra_mc_soc tegra114_mc_soc = {
 	.atom_size = 32,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra114_smmu_soc,
-	.intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
-		   MC_INT_DECERR_EMEM,
+	.intmasks = tegra114_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra114_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra114_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra114_mc_resets),
 	.ops = &tegra30_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c
index ff26815e51f17..5cfbc169c5f98 100644
--- a/drivers/memory/tegra/tegra124-emc.c
+++ b/drivers/memory/tegra/tegra124-emc.c
@@ -608,7 +608,7 @@ static int tegra124_emc_prepare_timing_change(struct tegra_emc *emc,
 
 	if ((last->emc_mode_1 & 0x1) == (timing->emc_mode_1 & 0x1))
 		dll_change = DLL_CHANGE_NONE;
-	else if (timing->emc_mode_1 & 0x1)
+	else if (!(timing->emc_mode_1 & 0x1))
 		dll_change = DLL_CHANGE_ON;
 	else
 		dll_change = DLL_CHANGE_OFF;
diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c
index 991d4f7bc070f..df87c50386252 100644
--- a/drivers/memory/tegra/tegra124.c
+++ b/drivers/memory/tegra/tegra124.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/of.h>
@@ -1258,6 +1258,15 @@ static const struct tegra_smmu_soc tegra124_smmu_soc = {
 	.num_asids = 128,
 };
 
+static const struct tegra_mc_intmask tegra124_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra124_mc_soc = {
 	.clients = tegra124_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra124_mc_clients),
@@ -1267,14 +1276,18 @@ const struct tegra_mc_soc tegra124_mc_soc = {
 	.smmu = &tegra124_smmu_soc,
 	.emem_regs = tegra124_mc_emem_regs,
 	.num_emem_regs = ARRAY_SIZE(tegra124_mc_emem_regs),
-	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra124_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra124_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra124_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra124_mc_resets),
 	.icc_ops = &tegra124_mc_icc_ops,
 	.ops = &tegra30_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
 #endif /* CONFIG_ARCH_TEGRA_124_SOC */
 
@@ -1292,6 +1305,15 @@ static const struct tegra_smmu_soc tegra132_smmu_soc = {
 	.num_asids = 128,
 };
 
+static const struct tegra_mc_intmask tegra132_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra132_mc_soc = {
 	.clients = tegra124_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra124_mc_clients),
@@ -1299,13 +1321,17 @@ const struct tegra_mc_soc tegra132_mc_soc = {
 	.atom_size = 32,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra132_smmu_soc,
-	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra132_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra132_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra124_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra124_mc_resets),
 	.icc_ops = &tegra124_mc_icc_ops,
 	.ops = &tegra30_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
 #endif /* CONFIG_ARCH_TEGRA_132_SOC */
diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c
index dfddceecdd1aa..03ebab6fbe68f 100644
--- a/drivers/memory/tegra/tegra186-emc.c
+++ b/drivers/memory/tegra/tegra186-emc.c
@@ -22,6 +22,7 @@ struct tegra186_emc {
 	struct tegra_bpmp *bpmp;
 	struct device *dev;
 	struct clk *clk;
+	struct clk *clk_dbb;
 
 	struct tegra186_emc_dvfs *dvfs;
 	unsigned int num_dvfs;
@@ -328,6 +329,13 @@ static int tegra186_emc_probe(struct platform_device *pdev)
 		goto put_bpmp;
 	}
 
+	emc->clk_dbb = devm_clk_get_optional_enabled(&pdev->dev, "dbb");
+	if (IS_ERR(emc->clk_dbb)) {
+		err = dev_err_probe(&pdev->dev, PTR_ERR(emc->clk_dbb),
+				    "failed to get DBB clock\n");
+		goto put_bpmp;
+	}
+
 	platform_set_drvdata(pdev, emc);
 	emc->dev = &pdev->dev;
 
diff --git a/drivers/memory/tegra/tegra186.c b/drivers/memory/tegra/tegra186.c
index aee11457bf8e0..91d56165605f6 100644
--- a/drivers/memory/tegra/tegra186.c
+++ b/drivers/memory/tegra/tegra186.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2017-2025 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2017-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/io.h>
@@ -174,7 +174,6 @@ const struct tegra_mc_ops tegra186_mc_ops = {
 	.remove = tegra186_mc_remove,
 	.resume = tegra186_mc_resume,
 	.probe_device = tegra186_mc_probe_device,
-	.handle_irq = tegra30_mc_handle_irq,
 };
 
 #if defined(CONFIG_ARCH_TEGRA_186_SOC)
@@ -902,17 +901,30 @@ static const struct tegra_mc_client tegra186_mc_clients[] = {
 	},
 };
 
+static const struct tegra_mc_intmask tegra186_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
+			MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra186_mc_soc = {
 	.num_clients = ARRAY_SIZE(tegra186_mc_clients),
 	.clients = tegra186_mc_clients,
 	.num_address_bits = 40,
 	.num_channels = 4,
 	.client_id_mask = 0xff,
-	.intmask = MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
-		   MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra186_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra186_mc_intmasks),
 	.ops = &tegra186_mc_ops,
 	.ch_intmask = 0x0000000f,
 	.global_intstatus_channel_shift = 0,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
 #endif
diff --git a/drivers/memory/tegra/tegra194.c b/drivers/memory/tegra/tegra194.c
index 26035ac3a1eb5..a8cc57690696d 100644
--- a/drivers/memory/tegra/tegra194.c
+++ b/drivers/memory/tegra/tegra194.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2017-2021 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2017-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <soc/tegra/mc.h>
@@ -1343,19 +1343,31 @@ static const struct tegra_mc_client tegra194_mc_clients[] = {
 	},
 };
 
+static const struct tegra_mc_intmask tegra194_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_ROUTE_SANITY | MC_INT_DECERR_GENERALIZED_CARVEOUT |
+			MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra194_mc_soc = {
 	.num_clients = ARRAY_SIZE(tegra194_mc_clients),
 	.clients = tegra194_mc_clients,
 	.num_address_bits = 40,
 	.num_channels = 16,
 	.client_id_mask = 0xff,
-	.intmask = MC_INT_DECERR_ROUTE_SANITY |
-		   MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
-		   MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra194_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra194_mc_intmasks),
 	.has_addr_hi_reg = true,
 	.ops = &tegra186_mc_ops,
 	.icc_ops = &tegra_mc_icc_ops,
 	.ch_intmask = 0x00000f00,
 	.global_intstatus_channel_shift = 8,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/memory/tegra/tegra20.c b/drivers/memory/tegra/tegra20.c
index 4748113bfe9d1..27dd6886f86e8 100644
--- a/drivers/memory/tegra/tegra20.c
+++ b/drivers/memory/tegra/tegra20.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2012-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/bitfield.h>
@@ -695,7 +695,7 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 	unsigned int bit;
 
 	/* mask all interrupts to avoid flooding */
-	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
+	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmasks[0].mask;
 	if (!status)
 		return IRQ_NONE;
 
@@ -713,7 +713,7 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 			value = mc_readl(mc, reg);
 
 			id = value & mc->soc->client_id_mask;
-			desc = tegra_mc_error_names[2];
+			desc = tegra20_mc_error_names[2];
 
 			if (value & BIT(31))
 				direction = "write";
@@ -724,7 +724,7 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 			value = mc_readl(mc, reg);
 
 			id = (value >> 1) & mc->soc->client_id_mask;
-			desc = tegra_mc_error_names[2];
+			desc = tegra20_mc_error_names[2];
 
 			if (value & BIT(0))
 				direction = "write";
@@ -736,7 +736,7 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 
 			id = value & mc->soc->client_id_mask;
 			type = (value & BIT(30)) ? 4 : 3;
-			desc = tegra_mc_error_names[type];
+			desc = tegra20_mc_error_names[type];
 			secure = "secure ";
 
 			if (value & BIT(31))
@@ -761,9 +761,20 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static const irq_handler_t tegra20_mc_irq_handlers[] = {
+	tegra20_mc_handle_irq
+};
+
 static const struct tegra_mc_ops tegra20_mc_ops = {
 	.probe = tegra20_mc_probe,
-	.handle_irq = tegra20_mc_handle_irq,
+};
+
+static const struct tegra_mc_intmask tegra20_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
+			MC_INT_DECERR_EMEM,
+	},
 };
 
 const struct tegra_mc_soc tegra20_mc_soc = {
@@ -771,11 +782,15 @@ const struct tegra_mc_soc tegra20_mc_soc = {
 	.num_clients = ARRAY_SIZE(tegra20_mc_clients),
 	.num_address_bits = 32,
 	.client_id_mask = 0x3f,
-	.intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
-		   MC_INT_DECERR_EMEM,
+	.intmasks = tegra20_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra20_mc_intmasks),
 	.reset_ops = &tegra20_mc_reset_ops,
 	.resets = tegra20_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra20_mc_resets),
 	.icc_ops = &tegra20_mc_icc_ops,
 	.ops = &tegra20_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra20_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra20_mc_irq_handlers),
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/memory/tegra/tegra210.c b/drivers/memory/tegra/tegra210.c
index 3c2949c16fdef..f58f3ef6f6819 100644
--- a/drivers/memory/tegra/tegra210.c
+++ b/drivers/memory/tegra/tegra210.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2015 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2015-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <dt-bindings/memory/tegra210-mc.h>
@@ -1273,6 +1273,15 @@ static const struct tegra_mc_reset tegra210_mc_resets[] = {
 	TEGRA210_MC_RESET(TSECB,     0x970, 0x974, 13),
 };
 
+static const struct tegra_mc_intmask tegra210_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra210_mc_soc = {
 	.clients = tegra210_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra210_mc_clients),
@@ -1280,11 +1289,15 @@ const struct tegra_mc_soc tegra210_mc_soc = {
 	.atom_size = 64,
 	.client_id_mask = 0xff,
 	.smmu = &tegra210_smmu_soc,
-	.intmask = MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_INVALID_APB_ASID_UPDATE | MC_INT_INVALID_SMMU_PAGE |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra210_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra210_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra210_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra210_mc_resets),
 	.ops = &tegra30_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/memory/tegra/tegra234.c b/drivers/memory/tegra/tegra234.c
index 5f57cea48b629..87b22038a5fbf 100644
--- a/drivers/memory/tegra/tegra234.c
+++ b/drivers/memory/tegra/tegra234.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2022-2023, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2022-2026, NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <soc/tegra/mc.h>
@@ -1132,16 +1132,23 @@ static const struct tegra_mc_icc_ops tegra234_mc_icc_ops = {
 	.set = tegra234_mc_icc_set,
 };
 
+static const struct tegra_mc_intmask tegra234_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_ROUTE_SANITY | MC_INT_DECERR_GENERALIZED_CARVEOUT |
+			MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra234_mc_soc = {
 	.num_clients = ARRAY_SIZE(tegra234_mc_clients),
 	.clients = tegra234_mc_clients,
 	.num_address_bits = 40,
 	.num_channels = 16,
 	.client_id_mask = 0x1ff,
-	.intmask = MC_INT_DECERR_ROUTE_SANITY |
-		   MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
-		   MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra234_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra234_mc_intmasks),
 	.has_addr_hi_reg = true,
 	.ops = &tegra186_mc_ops,
 	.icc_ops = &tegra234_mc_icc_ops,
@@ -1152,4 +1159,9 @@ const struct tegra_mc_soc tegra234_mc_soc = {
 	 * supported.
 	 */
 	.num_carveouts = 32,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/memory/tegra/tegra264.c b/drivers/memory/tegra/tegra264.c
index 5203e6c113727..e43ef14da1ee7 100644
--- a/drivers/memory/tegra/tegra264.c
+++ b/drivers/memory/tegra/tegra264.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2025, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2025-2026, NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <dt-bindings/memory/nvidia,tegra264.h>
@@ -188,6 +188,41 @@ static const struct tegra_mc_client tegra264_mc_clients[] = {
 	},
 };
 
+static const char *const tegra264_hub_error_names[32] = {
+	[0] = "coalescer error",
+	[1] = "SMMU BYPASS ALLOW error",
+	[2] = "Illegal tbugrp_id error",
+	[3] = "Malformed MSI request error",
+	[4] = "Read response with poison bit error",
+	[5] = "Restricted access violation error",
+	[6] = "Reserved PA error",
+};
+
+static const char *const tegra264_mc_error_names[4] = {
+	[1] = "EMEM decode error",
+	[2] = "TrustZone violation",
+	[3] = "Carveout violation",
+};
+
+static const char *const tegra264_rt_error_names[16] = {
+	[1] = "DECERR_PARTIAL_POPULATED",
+	[2] = "DECERR_SMMU_BYPASS",
+	[3] = "DECERR_INVALID_MMIO",
+	[4] = "DECERR_INVALID_GIC_MSI",
+	[5] = "DECERR_ATOMIC_SYSRAM",
+	[9] = "DECERR_REMOTE_REQ_PRE_BOOT",
+	[10] = "DECERR_ISO_OVER_C2C",
+	[11] = "DECERR_UNSUPPORTED_SBS_OPCODE",
+	[12] = "DECERR_SBS_REQ_OVER_SISO_LL",
+};
+
+/*
+ * MC instance aperture mapping for hubc registers
+ */
+static const int mc_hubc_aperture_number[5] = {
+	7, 8, 9, 10, 11
+};
+
 /*
  * tegra264_mc_icc_set() - Pass MC client info to the BPMP-FW
  * @src: ICC node for Memory Controller's (MC) Client
@@ -283,6 +318,312 @@ static int tegra264_mc_icc_get_init_bw(struct icc_node *node, u32 *avg, u32 *pea
 	return 0;
 }
 
+static void mcf_log_fault(struct tegra_mc *mc, u32 channel, unsigned long mcf_ch_intstatus)
+{
+	unsigned int bit;
+
+	for_each_set_bit(bit, &mcf_ch_intstatus, 32) {
+		const char *client = "unknown", *desc = "NA";
+		u32 status_reg, status1_reg = 0, addr_reg, addr_hi_reg = 0, err_type_mask = 0;
+		u32 value, client_id, i, addr_hi_shift = 0, addr_hi_mask = 0, status1;
+		u32 mc_rw_bit = MC_ERR_STATUS_RW, mc_sec_bit = MC_ERR_STATUS_SECURITY;
+		phys_addr_t addr = 0;
+		u8 type;
+
+		switch (BIT(bit)) {
+		case MC_INT_DECERR_EMEM:
+		case MC_INT_SECURITY_VIOLATION:
+			status_reg = mc->soc->regs->err_status;
+			addr_reg = mc->soc->regs->err_add;
+			addr_hi_reg = mc->soc->regs->err_add_hi;
+			err_type_mask = mc->soc->mc_err_status_type_mask;
+			break;
+
+		case MC_INT_DECERR_VPR:
+			status_reg = mc->soc->regs->err_vpr_status;
+			addr_reg = mc->soc->regs->err_vpr_add;
+			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+			addr_hi_mask = mc->soc->mc_addr_hi_mask;
+			break;
+
+		case MC_INT_SECERR_SEC:
+			status_reg = mc->soc->regs->err_sec_status;
+			addr_reg = mc->soc->regs->err_sec_add;
+			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+			addr_hi_mask = mc->soc->mc_addr_hi_mask;
+			break;
+
+		case MC_INT_DECERR_MTS:
+			status_reg = mc->soc->regs->err_mts_status;
+			addr_reg = mc->soc->regs->err_mts_add;
+			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+			addr_hi_mask = mc->soc->mc_addr_hi_mask;
+			break;
+
+		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
+			status_reg = mc->soc->regs->err_gen_co_status;
+			status1_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS_1_0;
+			addr_reg = mc->soc->regs->err_gen_co_add;
+			addr_hi_shift = MC_ERR_STATUS_GSC_ADR_HI_SHIFT;
+			addr_hi_mask = MC_ERR_STATUS_GSC_ADR_HI_MASK;
+			break;
+
+		case MC_INT_DECERR_ROUTE_SANITY:
+		case MC_INT_DECERR_ROUTE_SANITY_GIC_MSI:
+			status_reg = mc->soc->regs->err_route_status;
+			addr_reg = mc->soc->regs->err_route_add;
+			addr_hi_shift = MC_ERR_STATUS_RT_ADR_HI_SHIFT;
+			addr_hi_mask = mc->soc->mc_addr_hi_mask;
+			mc_sec_bit = MC_ERR_ROUTE_SANITY_SEC;
+			mc_rw_bit = MC_ERR_ROUTE_SANITY_RW;
+			err_type_mask = MC_ERR_STATUS_RT_TYPE_MASK;
+			break;
+
+		default:
+			dev_err_ratelimited(mc->dev, "Incorrect MC interrupt mask\n");
+			return;
+		}
+
+		value = mc_ch_readl(mc, channel, status_reg);
+		if (addr_hi_reg) {
+			addr = mc_ch_readl(mc, channel, addr_hi_reg);
+		} else {
+			if (!status1_reg) {
+				addr = ((value >> addr_hi_shift) & addr_hi_mask);
+			} else {
+				status1 = mc_ch_readl(mc, channel, status1_reg);
+				addr = ((status1 >> addr_hi_shift) & addr_hi_mask);
+			}
+		}
+
+		addr <<= 32;
+		addr |= mc_ch_readl(mc, channel, addr_reg);
+
+		client_id = value & mc->soc->client_id_mask;
+		for (i = 0; i < mc->soc->num_clients; i++) {
+			if (mc->soc->clients[i].id == client_id) {
+				client = mc->soc->clients[i].name;
+				break;
+			}
+		}
+
+		if (err_type_mask == MC_ERR_STATUS_RT_TYPE_MASK) {
+			type = (value & err_type_mask) >>
+				MC_ERR_STATUS_RT_TYPE_SHIFT;
+			desc = tegra264_rt_error_names[type];
+		} else if (err_type_mask) {
+			type = (value & err_type_mask) >>
+					MC_ERR_STATUS_TYPE_SHIFT;
+			desc = tegra264_mc_error_names[type];
+		}
+
+		dev_err_ratelimited(mc->dev, "%s: %s %s @%pa: %s (%s)\n",
+				    client, value & mc_sec_bit ? "secure" : "non-secure",
+				    value & mc_rw_bit ? "write" : "read", &addr,
+				    tegra_mc_status_names[bit] ?: "unknown", desc);
+		if (status1_reg)
+			dev_err_ratelimited(mc->dev, "gsc_apr_id=%u gsc_co_apr_id=%u\n",
+					    ((status1 >> ERR_GENERALIZED_APERTURE_ID_SHIFT)
+					    & ERR_GENERALIZED_APERTURE_ID_MASK),
+					    ((status1 >> ERR_GENERALIZED_CARVEOUT_APERTURE_ID_SHIFT)
+					    & ERR_GENERALIZED_CARVEOUT_APERTURE_ID_MASK));
+	}
+
+	/* clear interrupts */
+	mc_ch_writel(mc, channel, mcf_ch_intstatus, MCF_INTSTATUS_0);
+}
+
+static irqreturn_t handle_mcf_irq(int irq, void *data)
+{
+	struct tegra_mc *mc = data;
+	unsigned long common_intstat, intstatus;
+	u32 slice;
+
+	/* Read MCF_COMMON_INTSTATUS0_0_0 from MCB block */
+	common_intstat = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MCF_COMMON_INTSTATUS0_0_0);
+	if (common_intstat == 0) {
+		dev_warn(mc->dev, "No interrupt in MCF\n");
+		return IRQ_NONE;
+	}
+
+	for_each_set_bit(slice, &common_intstat, 32) {
+		/* Find out the slice number on which interrupt occurred */
+		if (slice > 4) {
+			dev_err(mc->dev, "Slice index out of bounds: %u\n", slice);
+			return IRQ_NONE;
+		}
+
+		intstatus = mc_ch_readl(mc, slice, MCF_INTSTATUS_0);
+		if (intstatus != 0)
+			mcf_log_fault(mc, slice, intstatus);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void hub_log_fault(struct tegra_mc *mc, u32 hub, unsigned long hub_intstat)
+{
+	unsigned int bit;
+
+	for_each_set_bit(bit, &hub_intstat, 32) {
+		const char *client = "unknown";
+		u32 client_id, status_reg, value, i;
+		phys_addr_t addr = 0;
+
+		switch (BIT(bit)) {
+		case MSS_HUB_COALESCER_ERR_INTMASK:
+			status_reg = MSS_HUB_COALESCE_ERR_STATUS_0;
+			addr = mc_ch_readl(mc, hub, MSS_HUB_COALESCE_ERR_ADR_HI_0);
+			addr <<= 32;
+			addr |= mc_ch_readl(mc, hub, MSS_HUB_COALESCE_ERR_ADR_0);
+			break;
+
+		case MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK:
+			status_reg = MSS_HUB_SMMU_BYPASS_ALLOW_ERR_STATUS_0;
+			break;
+
+		case MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK:
+			status_reg = MSS_HUB_ILLEGAL_TBUGRP_ID_ERR_STATUS_0;
+			break;
+
+		case MSS_HUB_MSI_ERR_INTMASK:
+			status_reg = MSS_HUB_MSI_ERR_STATUS_0;
+			break;
+
+		case MSS_HUB_POISON_RSP_INTMASK:
+			status_reg = MSS_HUB_POISON_RSP_STATUS_0;
+			break;
+
+		case MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK:
+			status_reg = MSS_HUB_RESTRICTED_ACCESS_ERR_STATUS_0;
+			break;
+
+		case MSS_HUB_RESERVED_PA_ERR_INTMASK:
+			status_reg = MSS_HUB_RESERVED_PA_ERR_STATUS_0;
+			break;
+
+		default:
+			dev_err_ratelimited(mc->dev, "Incorrect HUB interrupt mask\n");
+			return;
+		}
+
+		value = mc_ch_readl(mc, hub, status_reg);
+
+		client_id = value & mc->soc->client_id_mask;
+		for (i = 0; i < mc->soc->num_clients; i++) {
+			if (mc->soc->clients[i].id == client_id) {
+				client = mc->soc->clients[i].name;
+				break;
+			}
+		}
+
+		dev_err_ratelimited(mc->dev, "%s: @%pa: %s status: 0x%x\n",
+				    client, &addr, tegra264_hub_error_names[bit] ?: "unknown",
+				    value);
+	}
+
+	/* clear interrupts */
+	mc_ch_writel(mc, hub, hub_intstat, MSS_HUB_INTRSTATUS_0);
+}
+
+static irqreturn_t handle_hub_irq(int irq, void *data, int mc_hubc_aperture_number)
+{
+	struct tegra_mc *mc = data;
+	u32 global_intstat;
+	unsigned long hub_interrupt, intstat, hub;
+
+	/* Read MSS_HUB_GLOBAL_INTSTATUS_0 from mc_hubc_aperture_number block */
+	global_intstat = mc_ch_readl(mc, mc_hubc_aperture_number, MSS_HUB_GLOBAL_INTSTATUS_0);
+	if (global_intstat == 0) {
+		dev_warn(mc->dev, "No interrupt in HUB/HUBC\n");
+		return IRQ_NONE;
+	}
+
+	/* Handle interrupt from hubc */
+	if (global_intstat & MSS_HUBC_INTR) {
+		/* Read MSS_HUB_HUBC_INTSTATUS_0 from block mc_hubc_aperture_number */
+		intstat = mc_ch_readl(mc, mc_hubc_aperture_number, MSS_HUB_HUBC_INTSTATUS_0);
+		if (intstat != 0) {
+			dev_err_ratelimited(mc->dev, "Scrubber operation status: 0x%lx\n",
+					    intstat);
+			/* Clear hubc interrupt */
+			mc_ch_writel(mc, mc_hubc_aperture_number, intstat,
+				     MSS_HUB_HUBC_INTSTATUS_0);
+		}
+	}
+
+	hub_interrupt = (global_intstat & MSS_HUB_GLOBAL_MASK) >> MSS_HUB_GLOBAL_SHIFT;
+	/* Handle interrupt from hub */
+	for_each_set_bit(hub, &hub_interrupt, 32) {
+		/* Read MSS_HUB_INTRSTATUS_0 from block MCi */
+		intstat = mc_ch_readl(mc, hub, MSS_HUB_INTRSTATUS_0);
+		if (intstat != 0)
+			hub_log_fault(mc, hub, intstat);
+	}
+
+	/* Clear global interrupt status register */
+	mc_ch_writel(mc, mc_hubc_aperture_number, global_intstat, MSS_HUB_GLOBAL_INTSTATUS_0);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t handle_disp_hub_irq(int irq, void *data)
+{
+	return handle_hub_irq(irq, data, mc_hubc_aperture_number[0]);
+}
+
+static irqreturn_t handle_system_hub_irq(int irq, void *data)
+{
+	return handle_hub_irq(irq, data, mc_hubc_aperture_number[1]);
+}
+
+static irqreturn_t handle_vision_hub_irq(int irq, void *data)
+{
+	return handle_hub_irq(irq, data, mc_hubc_aperture_number[2]);
+}
+
+static irqreturn_t handle_uphy_hub_irq(int irq, void *data)
+{
+	return handle_hub_irq(irq, data, mc_hubc_aperture_number[3]);
+}
+
+static irqreturn_t handle_top_hub_irq(int irq, void *data)
+{
+	return handle_hub_irq(irq, data, mc_hubc_aperture_number[4]);
+}
+
+static irqreturn_t handle_generic_irq(struct tegra_mc *mc, unsigned long intstat_reg)
+{
+	u32 intstat, i;
+
+	/* Iterate over all MC blocks to read INTSTATUS */
+	for (i = 0; i < mc->num_channels; i++) {
+		intstat = mc_ch_readl(mc, i, intstat_reg);
+		if (intstat) {
+			dev_err_ratelimited(mc->dev, "channel: %i status: 0x%x\n", i, intstat);
+			/* Clear interrupt */
+			mc_ch_writel(mc, i, intstat, intstat_reg);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t handle_sbs_irq(int irq, void *data)
+{
+	return handle_generic_irq((struct tegra_mc *)data, MSS_SBS_INTSTATUS_0);
+}
+
+static irqreturn_t handle_channel_irq(int irq, void *data)
+{
+	return handle_generic_irq((struct tegra_mc *)data, MC_CH_INTSTATUS_0);
+}
+
+static const irq_handler_t tegra264_mc_irq_handlers[8] = {
+	handle_mcf_irq, handle_disp_hub_irq, handle_vision_hub_irq,
+	handle_system_hub_irq, handle_uphy_hub_irq, handle_top_hub_irq,
+	handle_sbs_irq, handle_channel_irq
+};
+
 static const struct tegra_mc_icc_ops tegra264_mc_icc_ops = {
 	.xlate = tegra_mc_icc_xlate,
 	.aggregate = tegra264_mc_icc_aggregate,
@@ -290,16 +631,80 @@ static const struct tegra_mc_icc_ops tegra264_mc_icc_ops = {
 	.set = tegra264_mc_icc_set,
 };
 
+static const struct tegra_mc_regs tegra264_mc_regs = {
+	.cfg_channel_enable = 0x8870,
+	.err_status = 0xbc00,
+	.err_add = 0xbc04,
+	.err_add_hi = 0xbc08,
+	.err_vpr_status = 0xbc20,
+	.err_vpr_add = 0xbc24,
+	.err_sec_status = 0xbc3c,
+	.err_sec_add = 0xbc40,
+	.err_mts_status = 0xbc5c,
+	.err_mts_add = 0xbc60,
+	.err_gen_co_status = 0xbc78,
+	.err_gen_co_add = 0xbc7c,
+	.err_route_status = 0xbc64,
+	.err_route_add = 0xbc68,
+};
+
+static const struct tegra_mc_intmask tegra264_mc_intmasks[] = {
+	{
+		.reg = MCF_INTMASK_0,
+		.mask = MC_INT_DECERR_ROUTE_SANITY_GIC_MSI | MC_INT_DECERR_ROUTE_SANITY |
+			MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
+			MC_INT_SECERR_SEC | MC_INT_DECERR_VPR | MC_INT_SECURITY_VIOLATION |
+			MC_INT_DECERR_EMEM,
+	},
+	{
+		.reg = MCF_INTPRIORITY_0,
+		.mask = MC_INT_DECERR_ROUTE_SANITY_GIC_MSI | MC_INT_DECERR_ROUTE_SANITY |
+			MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
+			MC_INT_SECERR_SEC | MC_INT_DECERR_VPR | MC_INT_SECURITY_VIOLATION |
+			MC_INT_DECERR_EMEM,
+	},
+	{
+		.reg = MSS_HUB_INTRMASK_0,
+		.mask = MSS_HUB_COALESCER_ERR_INTMASK | MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK |
+			MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK | MSS_HUB_MSI_ERR_INTMASK |
+			MSS_HUB_POISON_RSP_INTMASK | MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK |
+			MSS_HUB_RESERVED_PA_ERR_INTMASK,
+	},
+	{
+		.reg = MSS_HUB_INTRPRIORITY_0,
+		.mask = MSS_HUB_COALESCER_ERR_INTMASK | MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK |
+			MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK | MSS_HUB_MSI_ERR_INTMASK |
+			MSS_HUB_POISON_RSP_INTMASK | MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK |
+			MSS_HUB_RESERVED_PA_ERR_INTMASK,
+	},
+	{
+		.reg = MSS_HUB_HUBC_INTMASK_0,
+		.mask = MSS_HUB_HUBC_SCRUB_DONE_INTMASK,
+	},
+	{
+		.reg = MSS_HUB_HUBC_INTPRIORITY_0,
+		.mask = MSS_HUB_HUBC_SCRUB_DONE_INTMASK,
+	},
+	{
+		.reg = MSS_SBS_INTMASK_0,
+		.mask = MSS_SBS_FILL_FIFO_ISO_OVERFLOW_INTMASK |
+			MSS_SBS_FILL_FIFO_SISO_OVERFLOW_INTMASK |
+			MSS_SBS_FILL_FIFO_NISO_OVERFLOW_INTMASK,
+	},
+	{
+		.reg = MC_CH_INTMASK_0,
+		.mask = WCAM_ERR_INTMASK,
+	}
+};
+
 const struct tegra_mc_soc tegra264_mc_soc = {
 	.num_clients = ARRAY_SIZE(tegra264_mc_clients),
 	.clients = tegra264_mc_clients,
 	.num_address_bits = 40,
 	.num_channels = 16,
 	.client_id_mask = 0x1ff,
-	.intmask = MC_INT_DECERR_ROUTE_SANITY |
-		   MC_INT_DECERR_GENERALIZED_CARVEOUT | MC_INT_DECERR_MTS |
-		   MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
-		   MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	.intmasks = tegra264_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra264_mc_intmasks),
 	.has_addr_hi_reg = true,
 	.ops = &tegra186_mc_ops,
 	.icc_ops = &tegra264_mc_icc_ops,
@@ -310,4 +715,9 @@ const struct tegra_mc_soc tegra264_mc_soc = {
 	 * supported.
 	 */
 	.num_carveouts = 32,
+	.mc_addr_hi_mask = 0xff,
+	.mc_err_status_type_mask = (0x3 << 28),
+	.regs = &tegra264_mc_regs,
+	.handle_irq = tegra264_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra264_mc_irq_handlers),
 };
diff --git a/drivers/memory/tegra/tegra30-emc.c b/drivers/memory/tegra/tegra30-emc.c
index 606106dd2b32e..5812c8cd6ce4f 100644
--- a/drivers/memory/tegra/tegra30-emc.c
+++ b/drivers/memory/tegra/tegra30-emc.c
@@ -554,14 +554,14 @@ static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate)
 	emc->emc_cfg = readl_relaxed(emc->regs + EMC_CFG);
 	emc_dbg = readl_relaxed(emc->regs + EMC_DBG);
 
-	if (emc->dll_on == !!(timing->emc_mode_1 & 0x1))
+	if (emc->dll_on == !(timing->emc_mode_1 & 0x1))
 		dll_change = DLL_CHANGE_NONE;
-	else if (timing->emc_mode_1 & 0x1)
+	else if (!(timing->emc_mode_1 & 0x1))
 		dll_change = DLL_CHANGE_ON;
 	else
 		dll_change = DLL_CHANGE_OFF;
 
-	emc->dll_on = !!(timing->emc_mode_1 & 0x1);
+	emc->dll_on = !(timing->emc_mode_1 & 0x1);
 
 	if (timing->data[80] && !readl_relaxed(emc->regs + EMC_ZCAL_INTERVAL))
 		emc->zcal_long = true;
diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c
index a6bcde4b92c03..8389e3af01213 100644
--- a/drivers/memory/tegra/tegra30.c
+++ b/drivers/memory/tegra/tegra30.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
 #include <linux/device.h>
@@ -1384,6 +1384,14 @@ static const struct tegra_mc_icc_ops tegra30_mc_icc_ops = {
 	.set = tegra30_mc_icc_set,
 };
 
+static const struct tegra_mc_intmask tegra30_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
+			MC_INT_DECERR_EMEM,
+	},
+};
+
 const struct tegra_mc_soc tegra30_mc_soc = {
 	.clients = tegra30_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra30_mc_clients),
@@ -1393,11 +1401,15 @@ const struct tegra_mc_soc tegra30_mc_soc = {
 	.smmu = &tegra30_smmu_soc,
 	.emem_regs = tegra30_mc_emem_regs,
 	.num_emem_regs = ARRAY_SIZE(tegra30_mc_emem_regs),
-	.intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
-		   MC_INT_DECERR_EMEM,
+	.intmasks = tegra30_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra30_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra30_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra30_mc_resets),
 	.icc_ops = &tegra30_mc_icc_ops,
 	.ops = &tegra30_mc_ops,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_err_status_type_mask = (0x7 << 28),
 };
diff --git a/drivers/mfd/atmel-hlcdc.c b/drivers/mfd/atmel-hlcdc.c
index c3f3d39bf5845..0b541c0d3b1b8 100644
--- a/drivers/mfd/atmel-hlcdc.c
+++ b/drivers/mfd/atmel-hlcdc.c
@@ -108,10 +108,19 @@ static int atmel_hlcdc_probe(struct platform_device *pdev)
 		return PTR_ERR(hlcdc->periph_clk);
 	}
 
+	/*
+	 * Retrieve one of the primary clocks required for LCD operation:
+	 * prefer sys_clk (for RGB/MIPI), and fall back to lvds_pll_clk
+	 * (for LVDS) if needed.
+	 */
 	hlcdc->sys_clk = devm_clk_get(dev, "sys_clk");
 	if (IS_ERR(hlcdc->sys_clk)) {
-		dev_err(dev, "failed to get system clock\n");
-		return PTR_ERR(hlcdc->sys_clk);
+		hlcdc->sys_clk = NULL;
+		hlcdc->lvds_pll_clk = devm_clk_get(dev, "lvds_pll_clk");
+		if (IS_ERR(hlcdc->lvds_pll_clk)) {
+			dev_err(dev, "Failed to obtain both the LCDC (generic) and LVDS PLL clocks\n");
+			return PTR_ERR(hlcdc->lvds_pll_clk);
+		}
 	}
 
 	hlcdc->slow_clk = devm_clk_get(dev, "slow_clk");
diff --git a/drivers/mfd/cs42l43-i2c.c b/drivers/mfd/cs42l43-i2c.c
index a2ab001a600ae..0a0ab5e549a5a 100644
--- a/drivers/mfd/cs42l43-i2c.c
+++ b/drivers/mfd/cs42l43-i2c.c
@@ -47,6 +47,7 @@ static int cs42l43_i2c_probe(struct i2c_client *i2c)
 	cs42l43->irq = i2c->irq;
 	/* A device on an I2C is always attached by definition. */
 	cs42l43->attached = true;
+	cs42l43->variant_id = (long)device_get_match_data(cs42l43->dev);
 
 	cs42l43->regmap = devm_regmap_init_i2c(i2c, &cs42l43_i2c_regmap);
 	if (IS_ERR(cs42l43->regmap))
@@ -58,7 +59,8 @@ static int cs42l43_i2c_probe(struct i2c_client *i2c)
 
 #if IS_ENABLED(CONFIG_OF)
 static const struct of_device_id cs42l43_of_match[] = {
-	{ .compatible = "cirrus,cs42l43", },
+	{ .compatible = "cirrus,cs42l43", .data = (void *)CS42L43_DEVID_VAL },
+	{ .compatible = "cirrus,cs42l43b", .data = (void *)CS42L43B_DEVID_VAL },
 	{}
 };
 MODULE_DEVICE_TABLE(of, cs42l43_of_match);
@@ -66,7 +68,8 @@ MODULE_DEVICE_TABLE(of, cs42l43_of_match);
 
 #if IS_ENABLED(CONFIG_ACPI)
 static const struct acpi_device_id cs42l43_acpi_match[] = {
-	{ "CSC4243", 0 },
+	{ "CSC4243", CS42L43_DEVID_VAL },
+	{ "CSC2A3B", CS42L43B_DEVID_VAL },
 	{}
 };
 MODULE_DEVICE_TABLE(acpi, cs42l43_acpi_match);
diff --git a/drivers/mfd/cs42l43-sdw.c b/drivers/mfd/cs42l43-sdw.c
index 023f7e1a30f8c..794c983781750 100644
--- a/drivers/mfd/cs42l43-sdw.c
+++ b/drivers/mfd/cs42l43-sdw.c
@@ -178,6 +178,7 @@ static int cs42l43_sdw_probe(struct sdw_slave *sdw, const struct sdw_device_id *
 
 	cs42l43->dev = dev;
 	cs42l43->sdw = sdw;
+	cs42l43->variant_id = (long)id->driver_data;
 
 	cs42l43->regmap = devm_regmap_init_sdw(sdw, &cs42l43_sdw_regmap);
 	if (IS_ERR(cs42l43->regmap))
@@ -188,7 +189,8 @@ static int cs42l43_sdw_probe(struct sdw_slave *sdw, const struct sdw_device_id *
 }
 
 static const struct sdw_device_id cs42l43_sdw_id[] = {
-	SDW_SLAVE_ENTRY(0x01FA, 0x4243, 0),
+	SDW_SLAVE_ENTRY(0x01FA, 0x4243, (void *) CS42L43_DEVID_VAL),
+	SDW_SLAVE_ENTRY(0x01FA, 0x2A3B, (void *) CS42L43B_DEVID_VAL),
 	{}
 };
 MODULE_DEVICE_TABLE(sdw, cs42l43_sdw_id);
diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c
index 107cfb983fec4..166881751e698 100644
--- a/drivers/mfd/cs42l43.c
+++ b/drivers/mfd/cs42l43.c
@@ -115,9 +115,14 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = {
 	{ CS42L43_DECIM_HPF_WNF_CTRL2,			0x00000001 },
 	{ CS42L43_DECIM_HPF_WNF_CTRL3,			0x00000001 },
 	{ CS42L43_DECIM_HPF_WNF_CTRL4,			0x00000001 },
+	{ CS42L43B_DECIM_HPF_WNF_CTRL5,			0x00000001 },
+	{ CS42L43B_DECIM_HPF_WNF_CTRL6,			0x00000001 },
 	{ CS42L43_DMIC_PDM_CTRL,			0x00000000 },
 	{ CS42L43_DECIM_VOL_CTRL_CH1_CH2,		0x20122012 },
 	{ CS42L43_DECIM_VOL_CTRL_CH3_CH4,		0x20122012 },
+	{ CS42L43B_DECIM_VOL_CTRL_CH1_CH2,		0x20122012 },
+	{ CS42L43B_DECIM_VOL_CTRL_CH3_CH4,		0x20122012 },
+	{ CS42L43B_DECIM_VOL_CTRL_CH5_CH6,		0x20122012 },
 	{ CS42L43_INTP_VOLUME_CTRL1,			0x00000180 },
 	{ CS42L43_INTP_VOLUME_CTRL2,			0x00000180 },
 	{ CS42L43_AMP1_2_VOL_RAMP,			0x00000022 },
@@ -155,8 +160,12 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = {
 	{ CS42L43_SWIRE_DP2_CH2_INPUT,			0x00000000 },
 	{ CS42L43_SWIRE_DP3_CH1_INPUT,			0x00000000 },
 	{ CS42L43_SWIRE_DP3_CH2_INPUT,			0x00000000 },
+	{ CS42L43B_SWIRE_DP3_CH3_INPUT,			0x00000000 },
+	{ CS42L43B_SWIRE_DP3_CH4_INPUT,			0x00000000 },
 	{ CS42L43_SWIRE_DP4_CH1_INPUT,			0x00000000 },
 	{ CS42L43_SWIRE_DP4_CH2_INPUT,			0x00000000 },
+	{ CS42L43B_SWIRE_DP4_CH3_INPUT,			0x00000000 },
+	{ CS42L43B_SWIRE_DP4_CH4_INPUT,			0x00000000 },
 	{ CS42L43_ASRC_INT1_INPUT1,			0x00000000 },
 	{ CS42L43_ASRC_INT2_INPUT1,			0x00000000 },
 	{ CS42L43_ASRC_INT3_INPUT1,			0x00000000 },
@@ -169,10 +178,14 @@ const struct reg_default cs42l43_reg_default[CS42L43_N_DEFAULTS] = {
 	{ CS42L43_ISRC1INT2_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC1DEC1_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC1DEC2_INPUT1,			0x00000000 },
+	{ CS42L43B_ISRC1DEC3_INPUT1,			0x00000000 },
+	{ CS42L43B_ISRC1DEC4_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC2INT1_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC2INT2_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC2DEC1_INPUT1,			0x00000000 },
 	{ CS42L43_ISRC2DEC2_INPUT1,			0x00000000 },
+	{ CS42L43B_ISRC2DEC3_INPUT1,			0x00000000 },
+	{ CS42L43B_ISRC2DEC4_INPUT1,			0x00000000 },
 	{ CS42L43_EQ1MIX_INPUT1,			0x00800000 },
 	{ CS42L43_EQ1MIX_INPUT2,			0x00800000 },
 	{ CS42L43_EQ1MIX_INPUT3,			0x00800000 },
@@ -269,6 +282,8 @@ EXPORT_SYMBOL_NS_GPL(cs42l43_reg_default, "MFD_CS42L43");
 
 bool cs42l43_readable_register(struct device *dev, unsigned int reg)
 {
+	struct cs42l43 *cs42l43 = dev_get_drvdata(dev);
+
 	switch (reg) {
 	case CS42L43_DEVID:
 	case CS42L43_REVID:
@@ -292,7 +307,6 @@ bool cs42l43_readable_register(struct device *dev, unsigned int reg)
 	case CS42L43_ADC_B_CTRL1 ...  CS42L43_ADC_B_CTRL2:
 	case CS42L43_DECIM_HPF_WNF_CTRL1 ... CS42L43_DECIM_HPF_WNF_CTRL4:
 	case CS42L43_DMIC_PDM_CTRL:
-	case CS42L43_DECIM_VOL_CTRL_CH1_CH2 ... CS42L43_DECIM_VOL_CTRL_CH3_CH4:
 	case CS42L43_INTP_VOLUME_CTRL1 ... CS42L43_INTP_VOLUME_CTRL2:
 	case CS42L43_AMP1_2_VOL_RAMP:
 	case CS42L43_ASP_CTRL:
@@ -387,8 +401,16 @@ bool cs42l43_readable_register(struct device *dev, unsigned int reg)
 	case CS42L43_BOOT_CONTROL:
 	case CS42L43_BLOCK_EN:
 	case CS42L43_SHUTTER_CONTROL:
-	case CS42L43_MCU_SW_REV ... CS42L43_MCU_RAM_MAX:
-		return true;
+	case CS42L43B_MCU_SW_REV ... CS42L43B_MCU_RAM_MAX:
+		return true; // registers present on all variants
+	case CS42L43_MCU_SW_REV ... CS42L43B_MCU_SW_REV - 1:
+	case CS42L43B_MCU_RAM_MAX + 1 ... CS42L43_MCU_RAM_MAX:
+	case CS42L43_DECIM_VOL_CTRL_CH1_CH2 ... CS42L43_DECIM_VOL_CTRL_CH3_CH4:
+		return cs42l43->variant_id == CS42L43_DEVID_VAL; // regs only in CS42L43 variant
+	case CS42L43B_DECIM_VOL_CTRL_CH1_CH2 ... CS42L43B_DECIM_HPF_WNF_CTRL6:
+	case CS42L43B_SWIRE_DP3_CH3_INPUT ... CS42L43B_SWIRE_DP4_CH4_INPUT:
+	case CS42L43B_ISRC1DEC3_INPUT1 ... CS42L43B_ISRC2DEC4_INPUT1:
+		return cs42l43->variant_id == CS42L43B_DEVID_VAL; // regs only in CS42L43B variant
 	default:
 		return false;
 	}
@@ -597,15 +619,27 @@ static int cs42l43_wait_for_attach(struct cs42l43 *cs42l43)
 static int cs42l43_mcu_stage_2_3(struct cs42l43 *cs42l43, bool shadow)
 {
 	unsigned int need_reg = CS42L43_NEED_CONFIGS;
+	unsigned int boot_reg;
 	unsigned int val;
 	int ret;
 
-	if (shadow)
-		need_reg = CS42L43_FW_SH_BOOT_CFG_NEED_CONFIGS;
+	switch (cs42l43->variant_id) {
+	case CS42L43_DEVID_VAL:
+		if (shadow)
+			need_reg = CS42L43_FW_SH_BOOT_CFG_NEED_CONFIGS;
+		boot_reg = CS42L43_BOOT_STATUS;
+		break;
+	case CS42L43B_DEVID_VAL:
+		need_reg = CS42L43B_NEED_CONFIGS;
+		boot_reg = CS42L43B_BOOT_STATUS;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	regmap_write(cs42l43->regmap, need_reg, 0);
 
-	ret = regmap_read_poll_timeout(cs42l43->regmap, CS42L43_BOOT_STATUS,
+	ret = regmap_read_poll_timeout(cs42l43->regmap, boot_reg,
 				       val, (val == CS42L43_MCU_BOOT_STAGE3),
 				       CS42L43_MCU_POLL_US, CS42L43_MCU_CMD_TIMEOUT_US);
 	if (ret) {
@@ -644,13 +678,25 @@ static int cs42l43_mcu_stage_3_2(struct cs42l43 *cs42l43)
  */
 static int cs42l43_mcu_disable(struct cs42l43 *cs42l43)
 {
-	unsigned int val;
+	unsigned int val, cfg_reg, ctrl_reg;
 	int ret;
 
-	regmap_write(cs42l43->regmap, CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_REG,
-		     CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_DISABLE_VAL);
-	regmap_write(cs42l43->regmap, CS42L43_FW_MISSION_CTRL_MM_CTRL_SELECTION,
-		     CS42L43_FW_MM_CTRL_MCU_SEL_MASK);
+	switch (cs42l43->variant_id) {
+	case CS42L43_DEVID_VAL:
+		cfg_reg = CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_REG;
+		ctrl_reg = CS42L43_FW_MISSION_CTRL_MM_CTRL_SELECTION;
+		break;
+	case CS42L43B_DEVID_VAL:
+		cfg_reg = CS42L43B_FW_MISSION_CTRL_MM_MCU_CFG_REG;
+		ctrl_reg = CS42L43B_FW_MISSION_CTRL_MM_CTRL_SELECTION;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	regmap_write(cs42l43->regmap, cfg_reg, CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_DISABLE_VAL);
+	regmap_write(cs42l43->regmap, ctrl_reg, CS42L43_FW_MM_CTRL_MCU_SEL_MASK);
+
 	regmap_write(cs42l43->regmap, CS42L43_MCU_SW_INTERRUPT, CS42L43_CONTROL_IND_MASK);
 	regmap_write(cs42l43->regmap, CS42L43_MCU_SW_INTERRUPT, 0);
 
@@ -740,18 +786,32 @@ static int cs42l43_mcu_update_step(struct cs42l43 *cs42l43)
 {
 	unsigned int mcu_rev, bios_rev, boot_status, secure_cfg;
 	bool patched, shadow;
+	int boot_status_reg, mcu_sw_rev_reg;
 	int ret;
 
+	switch (cs42l43->variant_id) {
+	case CS42L43_DEVID_VAL:
+		boot_status_reg = CS42L43_BOOT_STATUS;
+		mcu_sw_rev_reg = CS42L43_MCU_SW_REV;
+		break;
+	case CS42L43B_DEVID_VAL:
+		boot_status_reg = CS42L43B_BOOT_STATUS;
+		mcu_sw_rev_reg = CS42L43B_MCU_SW_REV;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	/* Clear any stale software interrupt bits. */
 	regmap_read(cs42l43->regmap, CS42L43_SOFT_INT, &mcu_rev);
 
-	ret = regmap_read(cs42l43->regmap, CS42L43_BOOT_STATUS, &boot_status);
+	ret = regmap_read(cs42l43->regmap, boot_status_reg, &boot_status);
 	if (ret) {
 		dev_err(cs42l43->dev, "Failed to read boot status: %d\n", ret);
 		return ret;
 	}
 
-	ret = regmap_read(cs42l43->regmap, CS42L43_MCU_SW_REV, &mcu_rev);
+	ret = regmap_read(cs42l43->regmap, mcu_sw_rev_reg, &mcu_rev);
 	if (ret) {
 		dev_err(cs42l43->dev, "Failed to read firmware revision: %d\n", ret);
 		return ret;
@@ -918,6 +978,13 @@ static void cs42l43_boot_work(struct work_struct *work)
 
 	switch (devid) {
 	case CS42L43_DEVID_VAL:
+	case CS42L43B_DEVID_VAL:
+		if (devid != cs42l43->variant_id) {
+			dev_err(cs42l43->dev,
+				"Device ID (0x%06x) does not match variant ID (0x%06lx)\n",
+				devid, cs42l43->variant_id);
+			goto err;
+		}
 		break;
 	default:
 		dev_err(cs42l43->dev, "Unrecognised devid: 0x%06x\n", devid);
diff --git a/drivers/mfd/cs42l43.h b/drivers/mfd/cs42l43.h
index f3da783930f53..a0068f6572e2c 100644
--- a/drivers/mfd/cs42l43.h
+++ b/drivers/mfd/cs42l43.h
@@ -9,7 +9,7 @@
 #ifndef CS42L43_CORE_INT_H
 #define CS42L43_CORE_INT_H
 
-#define CS42L43_N_DEFAULTS 176
+#define CS42L43_N_DEFAULTS 189
 
 struct dev_pm_ops;
 struct device;
diff --git a/drivers/mfd/dln2.c b/drivers/mfd/dln2.c
index d12510e391c81..0b1e7fd7f5d73 100644
--- a/drivers/mfd/dln2.c
+++ b/drivers/mfd/dln2.c
@@ -424,8 +424,8 @@ static void free_rx_slot(struct dln2_dev *dln2, u16 handle, int slot)
 }
 
 static int _dln2_transfer(struct dln2_dev *dln2, u16 handle, u16 cmd,
-			  const void *obuf, unsigned obuf_len,
-			  void *ibuf, unsigned *ibuf_len)
+			  const void *obuf, unsigned int obuf_len,
+			  void *ibuf, unsigned int *ibuf_len)
 {
 	int ret = 0;
 	int rx_slot;
@@ -511,8 +511,8 @@ static int _dln2_transfer(struct dln2_dev *dln2, u16 handle, u16 cmd,
 }
 
 int dln2_transfer(struct platform_device *pdev, u16 cmd,
-		  const void *obuf, unsigned obuf_len,
-		  void *ibuf, unsigned *ibuf_len)
+		  const void *obuf, unsigned int obuf_len,
+		  void *ibuf, unsigned int *ibuf_len)
 {
 	struct dln2_platform_data *dln2_pdata;
 	struct dln2_dev *dln2;
@@ -583,10 +583,8 @@ static void dln2_free_rx_urbs(struct dln2_dev *dln2)
 {
 	int i;
 
-	for (i = 0; i < DLN2_MAX_URBS; i++) {
+	for (i = 0; i < DLN2_MAX_URBS; i++)
 		usb_free_urb(dln2->rx_urb[i]);
-		kfree(dln2->rx_buf[i]);
-	}
 }
 
 static void dln2_stop_rx_urbs(struct dln2_dev *dln2)
@@ -600,8 +598,6 @@ static void dln2_stop_rx_urbs(struct dln2_dev *dln2)
 static void dln2_free(struct dln2_dev *dln2)
 {
 	dln2_free_rx_urbs(dln2);
-	usb_put_dev(dln2->usb_dev);
-	kfree(dln2);
 }
 
 static int dln2_setup_rx_urbs(struct dln2_dev *dln2,
@@ -609,9 +605,10 @@ static int dln2_setup_rx_urbs(struct dln2_dev *dln2,
 {
 	int i;
 	const int rx_max_size = DLN2_RX_BUF_SIZE;
+	struct device *dev = &dln2->interface->dev;
 
 	for (i = 0; i < DLN2_MAX_URBS; i++) {
-		dln2->rx_buf[i] = kmalloc(rx_max_size, GFP_KERNEL);
+		dln2->rx_buf[i] = devm_kmalloc(dev, rx_max_size, GFP_KERNEL);
 		if (!dln2->rx_buf[i])
 			return -ENOMEM;
 
@@ -778,13 +775,13 @@ static int dln2_probe(struct usb_interface *interface,
 	if (ret)
 		return ret;
 
-	dln2 = kzalloc_obj(*dln2);
+	dln2 = devm_kzalloc(dev, sizeof(*dln2), GFP_KERNEL);
 	if (!dln2)
 		return -ENOMEM;
 
 	dln2->ep_out = epout->bEndpointAddress;
 	dln2->ep_in = epin->bEndpointAddress;
-	dln2->usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	dln2->usb_dev = interface_to_usbdev(interface);
 	dln2->interface = interface;
 	usb_set_intfdata(interface, dln2);
 	init_waitqueue_head(&dln2->disconnect_wq);
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index 4b7d0cb9340f1..5a3d79f339dd2 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -45,6 +45,7 @@
 #include <linux/acpi.h>
 #include <linux/pci.h>
 #include <linux/pinctrl/pinctrl.h>
+#include <linux/property.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/lpc_ich.h>
 #include <linux/platform_data/itco_wdt.h>
@@ -125,11 +126,17 @@ static struct mfd_cell lpc_ich_wdt_cell = {
 	.ignore_resource_conflicts = true,
 };
 
+const struct software_node lpc_ich_gpio_swnode = {
+	.name = "gpio_ich",
+};
+EXPORT_SYMBOL_NS(lpc_ich_gpio_swnode, "LPC_ICH");
+
 static struct mfd_cell lpc_ich_gpio_cell = {
 	.name = "gpio_ich",
 	.num_resources = ARRAY_SIZE(gpio_ich_res),
 	.resources = gpio_ich_res,
 	.ignore_resource_conflicts = true,
+	.swnode = &lpc_ich_gpio_swnode,
 };
 
 #define INTEL_GPIO_RESOURCE_SIZE	0x1000
diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c
index 1b7e7b3e785f2..358feec2d088f 100644
--- a/drivers/mfd/macsmc.c
+++ b/drivers/mfd/macsmc.c
@@ -46,6 +46,7 @@
 
 static const struct mfd_cell apple_smc_devs[] = {
 	MFD_CELL_NAME("macsmc-input"),
+	MFD_CELL_NAME("macsmc-power"),
 	MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"),
 	MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"),
 	MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"),
diff --git a/drivers/mfd/max77705.c b/drivers/mfd/max77705.c
index e1a9bfd658560..e98c76d6b6990 100644
--- a/drivers/mfd/max77705.c
+++ b/drivers/mfd/max77705.c
@@ -157,7 +157,7 @@ static int max77705_resume(struct device *dev)
 
 	return 0;
 }
-DEFINE_SIMPLE_DEV_PM_OPS(max77705_pm_ops, max77705_suspend, max77705_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(max77705_pm_ops, max77705_suspend, max77705_resume);
 
 static const struct of_device_id max77705_i2c_of_match[] = {
 	{ .compatible = "maxim,max77705" },
diff --git a/drivers/mfd/mc13xxx-core.c b/drivers/mfd/mc13xxx-core.c
index 920797b806ced..786eab3b2d03c 100644
--- a/drivers/mfd/mc13xxx-core.c
+++ b/drivers/mfd/mc13xxx-core.c
@@ -377,7 +377,7 @@ static int mc13xxx_add_subdevice_pdata(struct mc13xxx *mc13xxx,
 	if (snprintf(buf, sizeof(buf), format, name) > sizeof(buf))
 		return -E2BIG;
 
-	cell.name = kmemdup(buf, strlen(buf) + 1, GFP_KERNEL);
+	cell.name = devm_kmemdup(mc13xxx->dev, buf, strlen(buf) + 1, GFP_KERNEL);
 	if (!cell.name)
 		return -ENOMEM;
 
diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c
index 3e58d0764c7e0..1bdacda9a933f 100644
--- a/drivers/mfd/mt6397-core.c
+++ b/drivers/mfd/mt6397-core.c
@@ -297,7 +297,7 @@ static const struct chip_data mt6323_core = {
 
 static const struct chip_data mt6328_core = {
 	.cid_addr = MT6328_HWCID,
-	.cid_shift = 0,
+	.cid_shift = 8,
 	.cells = mt6328_devs,
 	.cell_size = ARRAY_SIZE(mt6328_devs),
 	.irq_init = mt6397_irq_init,
@@ -313,7 +313,7 @@ static const struct chip_data mt6357_core = {
 
 static const struct chip_data mt6331_mt6332_core = {
 	.cid_addr = MT6331_HWCID,
-	.cid_shift = 0,
+	.cid_shift = 8,
 	.cells = mt6331_mt6332_devs,
 	.cell_size = ARRAY_SIZE(mt6331_mt6332_devs),
 	.irq_init = mt6397_irq_init,
diff --git a/drivers/mfd/rohm-bd71828.c b/drivers/mfd/rohm-bd71828.c
index e54152a035106..a79f354bf5cb6 100644
--- a/drivers/mfd/rohm-bd71828.c
+++ b/drivers/mfd/rohm-bd71828.c
@@ -41,6 +41,7 @@ static struct gpio_keys_button button = {
 	.code = KEY_POWER,
 	.gpio = -1,
 	.type = EV_KEY,
+	.wakeup = 1,
 };
 
 static const struct gpio_keys_platform_data bd71828_powerkey_data = {
diff --git a/drivers/mfd/sec-acpm.c b/drivers/mfd/sec-acpm.c
index 537ea65685bfe..0e23b9d9f7ee7 100644
--- a/drivers/mfd/sec-acpm.c
+++ b/drivers/mfd/sec-acpm.c
@@ -367,7 +367,7 @@ static const struct regmap_config s2mpg11_regmap_config_meter = {
 };
 
 struct sec_pmic_acpm_shared_bus_context {
-	const struct acpm_handle *acpm;
+	struct acpm_handle *acpm;
 	unsigned int acpm_chan_id;
 	u8 speedy_channel;
 };
@@ -390,7 +390,7 @@ static int sec_pmic_acpm_bus_write(void *context, const void *data,
 				   size_t count)
 {
 	struct sec_pmic_acpm_bus_context *ctx = context;
-	const struct acpm_handle *acpm = ctx->shared->acpm;
+	struct acpm_handle *acpm = ctx->shared->acpm;
 	const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops;
 	size_t val_count = count - BITS_TO_BYTES(ACPM_ADDR_BITS);
 	const u8 *d = data;
@@ -410,7 +410,7 @@ static int sec_pmic_acpm_bus_read(void *context, const void *reg_buf, size_t reg
 				  void *val_buf, size_t val_size)
 {
 	struct sec_pmic_acpm_bus_context *ctx = context;
-	const struct acpm_handle *acpm = ctx->shared->acpm;
+	struct acpm_handle *acpm = ctx->shared->acpm;
 	const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops;
 	const u8 *r = reg_buf;
 	u8 reg;
@@ -429,7 +429,7 @@ static int sec_pmic_acpm_bus_reg_update_bits(void *context, unsigned int reg, un
 					     unsigned int val)
 {
 	struct sec_pmic_acpm_bus_context *ctx = context;
-	const struct acpm_handle *acpm = ctx->shared->acpm;
+	struct acpm_handle *acpm = ctx->shared->acpm;
 	const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops;
 
 	return pmic_ops->update_reg(acpm, ctx->shared->acpm_chan_id, ctx->type, reg & 0xff,
@@ -480,7 +480,7 @@ static int sec_pmic_acpm_probe(struct platform_device *pdev)
 	struct regmap *regmap_common, *regmap_pmic, *regmap;
 	const struct sec_pmic_acpm_platform_data *pdata;
 	struct sec_pmic_acpm_shared_bus_context *shared_ctx;
-	const struct acpm_handle *acpm;
+	struct acpm_handle *acpm;
 	struct device *dev = &pdev->dev;
 	int ret, irq;
 
diff --git a/drivers/mfd/stpmic1.c b/drivers/mfd/stpmic1.c
index 081827bc05961..7c677b0344c60 100644
--- a/drivers/mfd/stpmic1.c
+++ b/drivers/mfd/stpmic1.c
@@ -16,6 +16,8 @@
 
 #include <dt-bindings/mfd/st,stpmic1.h>
 
+#define STPMIC1_MAX_RETRIES 2
+
 #define STPMIC1_MAIN_IRQ 0
 
 static const struct regmap_range stpmic1_readable_ranges[] = {
@@ -121,9 +123,23 @@ static const struct regmap_irq_chip stpmic1_regmap_irq_chip = {
 static int stpmic1_power_off(struct sys_off_data *data)
 {
 	struct stpmic1 *ddata = data->cb_data;
+	int ret;
+
+	/*
+	 * Attempt to shut down again, in case the first attempt failed.
+	 * The STPMIC1 might get confused and the first regmap_update_bits()
+	 * returns with -ETIMEDOUT / -110 . If that or similar transient
+	 * failure occurs, try to shut down again. If the second attempt
+	 * fails, there is some bigger problem, report it to user.
+	 */
+	for (int retries = 0; retries < STPMIC1_MAX_RETRIES; retries++) {
+		ret = regmap_update_bits(ddata->regmap, MAIN_CR, SOFTWARE_SWITCH_OFF,
+					 SOFTWARE_SWITCH_OFF);
+		if (!ret)
+			return NOTIFY_DONE;
+	}
 
-	regmap_update_bits(ddata->regmap, MAIN_CR,
-			   SOFTWARE_SWITCH_OFF, SOFTWARE_SWITCH_OFF);
+	dev_err(ddata->dev, "Failed to access PMIC I2C bus (%d)\n", ret);
 
 	return NOTIFY_DONE;
 }
diff --git a/drivers/mfd/viperboard.c b/drivers/mfd/viperboard.c
index f964bcfa07966..888737b8e7be8 100644
--- a/drivers/mfd/viperboard.c
+++ b/drivers/mfd/viperboard.c
@@ -59,7 +59,7 @@ static int vprbrd_probe(struct usb_interface *interface,
 
 	mutex_init(&vb->lock);
 
-	vb->usb_dev = usb_get_dev(interface_to_usbdev(interface));
+	vb->usb_dev = interface_to_usbdev(interface);
 
 	/* save our data pointer in this interface device */
 	usb_set_intfdata(interface, vb);
@@ -96,10 +96,8 @@ static int vprbrd_probe(struct usb_interface *interface,
 	return 0;
 
 error:
-	if (vb) {
-		usb_put_dev(vb->usb_dev);
+	if (vb)
 		kfree(vb);
-	}
 
 	return ret;
 }
@@ -110,7 +108,6 @@ static void vprbrd_disconnect(struct usb_interface *interface)
 
 	mfd_remove_devices(&interface->dev);
 	usb_set_intfdata(interface, NULL);
-	usb_put_dev(vb->usb_dev);
 	kfree(vb);
 
 	dev_dbg(&interface->dev, "disconnected\n");
diff --git a/drivers/misc/open-dice.c b/drivers/misc/open-dice.c
index 24c29e0f00ef1..45060fb4ea279 100644
--- a/drivers/misc/open-dice.c
+++ b/drivers/misc/open-dice.c
@@ -86,29 +86,32 @@ static ssize_t open_dice_write(struct file *filp, const char __user *ptr,
 /*
  * Creates a mapping of the reserved memory region in user address space.
  */
-static int open_dice_mmap(struct file *filp, struct vm_area_struct *vma)
+static int open_dice_mmap_prepare(struct vm_area_desc *desc)
 {
+	struct file *filp = desc->file;
 	struct open_dice_drvdata *drvdata = to_open_dice_drvdata(filp);
 
-	if (vma->vm_flags & VM_MAYSHARE) {
+	if (vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
 		/* Do not allow userspace to modify the underlying data. */
-		if (vma->vm_flags & VM_WRITE)
+		if (vma_desc_test(desc, VMA_WRITE_BIT))
 			return -EPERM;
 		/* Ensure userspace cannot acquire VM_WRITE later. */
-		vm_flags_clear(vma, VM_MAYWRITE);
+		vma_desc_clear_flags(desc, VMA_MAYWRITE_BIT);
 	}
 
 	/* Create write-combine mapping so all clients observe a wipe. */
-	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
-	vm_flags_set(vma, VM_DONTCOPY | VM_DONTDUMP);
-	return vm_iomap_memory(vma, drvdata->rmem->base, drvdata->rmem->size);
+	desc->page_prot = pgprot_writecombine(desc->page_prot);
+	vma_desc_set_flags(desc, VMA_DONTCOPY_BIT, VMA_DONTDUMP_BIT);
+	mmap_action_simple_ioremap(desc, drvdata->rmem->base,
+				   drvdata->rmem->size);
+	return 0;
 }
 
 static const struct file_operations open_dice_fops = {
 	.owner = THIS_MODULE,
 	.read = open_dice_read,
 	.write = open_dice_write,
-	.mmap = open_dice_mmap,
+	.mmap_prepare = open_dice_mmap_prepare,
 };
 
 static int __init open_dice_probe(struct platform_device *pdev)
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index 74ab5b5b90110..55e128ed82f00 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -84,6 +84,13 @@
 #define CAP_MSIX				BIT(2)
 #define CAP_INTX				BIT(3)
 #define CAP_SUBRANGE_MAPPING			BIT(4)
+#define CAP_DYNAMIC_INBOUND_MAPPING		BIT(5)
+#define CAP_BAR0_RESERVED			BIT(6)
+#define CAP_BAR1_RESERVED			BIT(7)
+#define CAP_BAR2_RESERVED			BIT(8)
+#define CAP_BAR3_RESERVED			BIT(9)
+#define CAP_BAR4_RESERVED			BIT(10)
+#define CAP_BAR5_RESERVED			BIT(11)
 
 #define PCI_ENDPOINT_TEST_DB_BAR		0x34
 #define PCI_ENDPOINT_TEST_DB_OFFSET		0x38
@@ -275,6 +282,11 @@ static int pci_endpoint_test_request_irq(struct pci_endpoint_test *test)
 	return ret;
 }
 
+static bool bar_is_reserved(struct pci_endpoint_test *test, enum pci_barno bar)
+{
+	return test->ep_caps & BIT(bar + __fls(CAP_BAR0_RESERVED));
+}
+
 static const u32 bar_test_pattern[] = {
 	0xA0A0A0A0,
 	0xA1A1A1A1,
@@ -403,7 +415,7 @@ static int pci_endpoint_test_bars(struct pci_endpoint_test *test)
 
 	/* Write all BARs in order (without reading). */
 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		if (test->bar[bar])
+		if (test->bar[bar] && !bar_is_reserved(test, bar))
 			pci_endpoint_test_bars_write_bar(test, bar);
 
 	/*
@@ -413,7 +425,7 @@ static int pci_endpoint_test_bars(struct pci_endpoint_test *test)
 	 * (Reading back the BAR directly after writing can not detect this.)
 	 */
 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
-		if (test->bar[bar]) {
+		if (test->bar[bar] && !bar_is_reserved(test, bar)) {
 			ret = pci_endpoint_test_bars_read_bar(test, bar);
 			if (ret)
 				return ret;
@@ -1060,6 +1072,9 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
 	u32 addr;
 	int left;
 
+	if (!(test->ep_caps & CAP_DYNAMIC_INBOUND_MAPPING))
+		return -EOPNOTSUPP;
+
 	if (irq_type < PCITEST_IRQ_TYPE_INTX ||
 	    irq_type > PCITEST_IRQ_TYPE_MSIX) {
 		dev_err(dev, "Invalid IRQ type\n");
@@ -1139,6 +1154,11 @@ static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
 		if (is_am654_pci_dev(pdev) && bar == BAR_0)
 			goto ret;
 
+		if (bar_is_reserved(test, bar)) {
+			ret = -ENOBUFS;
+			goto ret;
+		}
+
 		if (cmd == PCITEST_BAR)
 			ret = pci_endpoint_test_bar(test, bar);
 		else
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 8d749f3452468..278b76cbd2814 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -542,7 +542,7 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate)
 	int ctxnum = gts->ts_ctxnum;
 
 	if (!is_kernel_context(gts))
-		zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
+		zap_special_vma_range(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
 	gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h
index 1200951bab08c..a7c364d0030ad 100644
--- a/drivers/mmc/core/card.h
+++ b/drivers/mmc/core/card.h
@@ -89,6 +89,7 @@ struct mmc_fixup {
 #define CID_MANFID_MICRON       0x13
 #define CID_MANFID_SAMSUNG      0x15
 #define CID_MANFID_APACER       0x27
+#define CID_MANFID_SANDISK_MMC  0x45
 #define CID_MANFID_SWISSBIT     0x5D
 #define CID_MANFID_KINGSTON     0x70
 #define CID_MANFID_HYNIX	0x90
@@ -305,4 +306,14 @@ static inline int mmc_card_no_uhs_ddr50_tuning(const struct mmc_card *c)
 	return c->quirks & MMC_QUIRK_NO_UHS_DDR50_TUNING;
 }
 
+static inline int mmc_card_broken_mdt(const struct mmc_card *c)
+{
+	return c->quirks & MMC_QUIRK_BROKEN_MDT;
+}
+
+static inline int mmc_card_fixed_secure_erase_trim_time(const struct mmc_card *c)
+{
+	return c->quirks & MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME;
+}
+
 #endif
diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
index 88c95dbfd9cfd..a457c88fdcbc7 100644
--- a/drivers/mmc/core/host.c
+++ b/drivers/mmc/core/host.c
@@ -624,12 +624,24 @@ static int mmc_validate_host_caps(struct mmc_host *host)
 		return -EINVAL;
 	}
 
+	/* UHS/DDR/HS200 modes require at least 4-bit bus */
+	if (!(caps & (MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA)) &&
+	    ((caps & (MMC_CAP_UHS | MMC_CAP_DDR)) || (caps2 & MMC_CAP2_HS200))) {
+		dev_warn(dev, "drop UHS/DDR/HS200 support since 1-bit bus only\n");
+		caps &= ~(MMC_CAP_UHS | MMC_CAP_DDR);
+		caps2 &= ~MMC_CAP2_HS200;
+	}
+
+	/* HS400 and HS400ES modes require 8-bit bus */
 	if (caps2 & (MMC_CAP2_HS400_ES | MMC_CAP2_HS400) &&
 	    !(caps & MMC_CAP_8_BIT_DATA) && !(caps2 & MMC_CAP2_NO_MMC)) {
 		dev_warn(dev, "drop HS400 support since no 8-bit bus\n");
-		host->caps2 = caps2 & ~MMC_CAP2_HS400_ES & ~MMC_CAP2_HS400;
+		caps2 &= ~(MMC_CAP2_HS400_ES | MMC_CAP2_HS400);
 	}
 
+	host->caps = caps;
+	host->caps2 = caps2;
+
 	return 0;
 }
 
diff --git a/drivers/mmc/core/host.h b/drivers/mmc/core/host.h
index 5941d68ff9894..6bce5a4798e2f 100644
--- a/drivers/mmc/core/host.h
+++ b/drivers/mmc/core/host.h
@@ -56,11 +56,7 @@ static inline int mmc_host_can_access_boot(struct mmc_host *host)
 
 static inline int mmc_host_can_uhs(struct mmc_host *host)
 {
-	return host->caps &
-		(MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
-		 MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
-		 MMC_CAP_UHS_DDR50) &&
-	       host->caps & MMC_CAP_4_BIT_DATA;
+	return host->caps & MMC_CAP_UHS;
 }
 
 static inline bool mmc_card_hs200(struct mmc_card *card)
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 7c86efb1044a3..8846550a8892a 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -671,7 +671,19 @@ static int mmc_decode_ext_csd(struct mmc_card *card, u8 *ext_csd)
 		card->ext_csd.enhanced_rpmb_supported =
 					(card->ext_csd.rel_param &
 					 EXT_CSD_WR_REL_PARAM_EN_RPMB_REL_WR);
+
+		if (card->ext_csd.rev >= 9) {
+			/* Adjust production date as per JEDEC JESD84-B51B September 2025 */
+			if (card->cid.year < 2023)
+				card->cid.year += 16;
+		} else {
+			/* Handle vendors with broken MDT reporting */
+			if (mmc_card_broken_mdt(card) && card->cid.year >= 2010 &&
+			    card->cid.year <= 2012)
+				card->cid.year += 16;
+		}
 	}
+
 out:
 	return err;
 }
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index 13000fc57e2e1..39fcb662c43fc 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -184,8 +184,13 @@ static void mmc_queue_setup_discard(struct mmc_card *card,
 		return;
 
 	lim->max_hw_discard_sectors = max_discard;
-	if (mmc_card_can_secure_erase_trim(card))
-		lim->max_secure_erase_sectors = max_discard;
+	if (mmc_card_can_secure_erase_trim(card)) {
+		if (mmc_card_fixed_secure_erase_trim_time(card))
+			lim->max_secure_erase_sectors = UINT_MAX >> card->erase_shift;
+		else
+			lim->max_secure_erase_sectors = max_discard;
+	}
+
 	if (mmc_card_can_trim(card) && card->erased_byte == 0)
 		lim->max_write_zeroes_sectors = max_discard;
 
diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h
index c417ed34c0576..6f727b4a60a52 100644
--- a/drivers/mmc/core/quirks.h
+++ b/drivers/mmc/core/quirks.h
@@ -153,6 +153,15 @@ static const struct mmc_fixup __maybe_unused mmc_blk_fixups[] = {
 	MMC_FIXUP("M62704", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc,
 		  MMC_QUIRK_TRIM_BROKEN),
 
+	/*
+	 * On Some Kingston eMMCs, secure erase/trim time is independent
+	 * of erase size, fixed at approximately 2 seconds.
+	 */
+	MMC_FIXUP("IY2964", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc,
+		  MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME),
+	MMC_FIXUP("IB2932", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc,
+		  MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME),
+
 	END_FIXUP
 };
 
@@ -170,6 +179,9 @@ static const struct mmc_fixup __maybe_unused mmc_ext_csd_fixups[] = {
 	MMC_FIXUP_EXT_CSD_REV(CID_NAME_ANY, CID_MANFID_NUMONYX,
 			      0x014e, add_quirk, MMC_QUIRK_BROKEN_HPI, 6),
 
+	MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_MMC, CID_OEMID_ANY, add_quirk_mmc,
+		  MMC_QUIRK_BROKEN_MDT),
+
 	END_FIXUP
 };
 
diff --git a/drivers/mmc/core/sdio_io.c b/drivers/mmc/core/sdio_io.c
index b774bf51981d0..12716ec0e35d6 100644
--- a/drivers/mmc/core/sdio_io.c
+++ b/drivers/mmc/core/sdio_io.c
@@ -163,10 +163,8 @@ int sdio_set_block_size(struct sdio_func *func, unsigned blksz)
 	if (blksz > func->card->host->max_blk_size)
 		return -EINVAL;
 
-	if (blksz == 0) {
-		blksz = min(func->max_blksize, func->card->host->max_blk_size);
-		blksz = min(blksz, 512u);
-	}
+	if (blksz == 0)
+		blksz = min3(func->max_blksize, func->card->host->max_blk_size, 512u);
 
 	ret = mmc_io_rw_direct(func->card, 1, 0,
 		SDIO_FBR_BASE(func->num) + SDIO_FBR_BLKSIZE,
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 6d79cc9a79e22..4f060d3e56365 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -429,6 +429,20 @@ config MMC_SDHCI_BCM_KONA
 
 	  If you have a controller with this interface, say Y or M here.
 
+config MMC_SDHCI_BST
+	tristate "SDHCI support for Black Sesame Technologies BST C1200 controller"
+	depends on ARCH_BST || COMPILE_TEST
+	depends on MMC_SDHCI_PLTFM
+	depends on OF
+	help
+	  This selects the Secure Digital Host Controller Interface (SDHCI)
+	  for Black Sesame Technologies BST C1200 SoC. The controller is
+	  based on Synopsys DesignWare Cores Mobile Storage Controller but
+	  requires platform-specific workarounds for hardware limitations.
+
+	  If you have a controller with this interface, say Y or M here.
+	  If unsure, say N.
+
 config MMC_SDHCI_F_SDH30
 	tristate "SDHCI support for Fujitsu Semiconductor F_SDH30"
 	depends on MMC_SDHCI_PLTFM
@@ -1044,7 +1058,7 @@ config MMC_MTK
 
 config MMC_SDHCI_MICROCHIP_PIC32
 	tristate "Microchip PIC32MZDA SDHCI support"
-	depends on MMC_SDHCI && PIC32MZDA && MMC_SDHCI_PLTFM
+	depends on MMC_SDHCI && MMC_SDHCI_PLTFM && (PIC32MZDA || COMPILE_TEST)
 	help
 	  This selects the Secure Digital Host Controller Interface (SDHCI)
 	  for PIC32MZDA platform.
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index 5057fea8afb69..ee412e6b84d6c 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_MMC_MXS)		+= mxs-mmc.o
 obj-$(CONFIG_MMC_SDHCI)		+= sdhci.o
 obj-$(CONFIG_MMC_SDHCI_UHS2)	+= sdhci-uhs2.o
 obj-$(CONFIG_MMC_SDHCI_PCI)	+= sdhci-pci.o
+obj-$(CONFIG_MMC_SDHCI_BST)	        += sdhci-of-bst.o
 sdhci-pci-y			+= sdhci-pci-core.o sdhci-pci-o2micro.o sdhci-pci-arasan.o \
 				   sdhci-pci-dwc-mshc.o sdhci-pci-gli.o
 obj-$(CONFIG_MMC_SDHCI_ACPI)	+= sdhci-acpi.o
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index fdf6926ea4682..3b4928f5b9b2e 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -629,14 +629,13 @@ static int atmci_of_init(struct atmel_mci *host)
 {
 	struct device *dev = host->dev;
 	struct device_node *np = dev->of_node;
-	struct device_node *cnp;
 	u32 slot_id;
 	int err;
 
 	if (!np)
 		return dev_err_probe(dev, -EINVAL, "device node not found\n");
 
-	for_each_child_of_node(np, cnp) {
+	for_each_child_of_node_scoped(np, cnp) {
 		if (of_property_read_u32(cnp, "reg", &slot_id)) {
 			dev_warn(dev, "reg property is missing for %pOF\n", cnp);
 			continue;
@@ -645,7 +644,6 @@ static int atmci_of_init(struct atmel_mci *host)
 		if (slot_id >= ATMCI_MAX_NR_SLOTS) {
 			dev_warn(dev, "can't have more than %d slots\n",
 			         ATMCI_MAX_NR_SLOTS);
-			of_node_put(cnp);
 			break;
 		}
 
@@ -658,10 +656,8 @@ static int atmci_of_init(struct atmel_mci *host)
 					      "cd", GPIOD_IN, "cd-gpios");
 		err = PTR_ERR_OR_ZERO(host->pdata[slot_id].detect_pin);
 		if (err) {
-			if (err != -ENOENT) {
-				of_node_put(cnp);
+			if (err != -ENOENT)
 				return err;
-			}
 			host->pdata[slot_id].detect_pin = NULL;
 		}
 
@@ -673,10 +669,8 @@ static int atmci_of_init(struct atmel_mci *host)
 					      "wp", GPIOD_IN, "wp-gpios");
 		err = PTR_ERR_OR_ZERO(host->pdata[slot_id].wp_pin);
 		if (err) {
-			if (err != -ENOENT) {
-				of_node_put(cnp);
+			if (err != -ENOENT)
 				return err;
-			}
 			host->pdata[slot_id].wp_pin = NULL;
 		}
 	}
diff --git a/drivers/mmc/host/cavium-octeon.c b/drivers/mmc/host/cavium-octeon.c
index 0592f356b1e5a..8a0daddd9200e 100644
--- a/drivers/mmc/host/cavium-octeon.c
+++ b/drivers/mmc/host/cavium-octeon.c
@@ -148,7 +148,7 @@ static void octeon_mmc_dmar_fixup_done(struct cvm_mmc_host *host)
 
 static int octeon_mmc_probe(struct platform_device *pdev)
 {
-	struct device_node *cn, *node = pdev->dev.of_node;
+	struct device_node *node = pdev->dev.of_node;
 	struct cvm_mmc_host *host;
 	void __iomem *base;
 	int mmc_irq[9];
@@ -268,7 +268,7 @@ static int octeon_mmc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, host);
 
 	i = 0;
-	for_each_child_of_node(node, cn) {
+	for_each_child_of_node_scoped(node, cn) {
 		host->slot_pdev[i] =
 			of_platform_device_create(cn, NULL, &pdev->dev);
 		if (!host->slot_pdev[i]) {
@@ -279,7 +279,6 @@ static int octeon_mmc_probe(struct platform_device *pdev)
 		if (ret) {
 			dev_err(&pdev->dev, "Error populating slots\n");
 			octeon_mmc_set_shared_power(host, 0);
-			of_node_put(cn);
 			goto error;
 		}
 		i++;
diff --git a/drivers/mmc/host/cavium.c b/drivers/mmc/host/cavium.c
index 9a55db0e657ce..37a88f2a0c86f 100644
--- a/drivers/mmc/host/cavium.c
+++ b/drivers/mmc/host/cavium.c
@@ -905,9 +905,7 @@ static void cvm_mmc_set_clock(struct cvm_mmc_slot *slot, unsigned int clock)
 {
 	struct mmc_host *mmc = slot->mmc;
 
-	clock = min(clock, mmc->f_max);
-	clock = max(clock, mmc->f_min);
-	slot->clock = clock;
+	slot->clock = clamp(clock, mmc->f_min, mmc->f_max);
 }
 
 static int cvm_mmc_init_lowlevel(struct cvm_mmc_slot *slot)
diff --git a/drivers/mmc/host/dw_mmc-bluefield.c b/drivers/mmc/host/dw_mmc-bluefield.c
index 3cf526ab0387b..81dc072d0e3f6 100644
--- a/drivers/mmc/host/dw_mmc-bluefield.c
+++ b/drivers/mmc/host/dw_mmc-bluefield.c
@@ -73,7 +73,7 @@ static struct platform_driver dw_mci_bluefield_pltfm_driver = {
 		.name		= "dwmmc_bluefield",
 		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= dw_mci_bluefield_match,
-		.pm		= &dw_mci_pltfm_pmops,
+		.pm		= pm_ptr(&dw_mci_pmops),
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index 384609671a9ab..261344d3a8cfe 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -185,8 +185,8 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing)
 	 * HOLD register should be bypassed in case there is no phase shift
 	 * applied on CMD/DATA that is sent to the card.
 	 */
-	if (!SDMMC_CLKSEL_GET_DRV_WD3(clksel) && host->slot)
-		set_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags);
+	if (!SDMMC_CLKSEL_GET_DRV_WD3(clksel))
+		set_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags);
 }
 
 static int dw_mci_exynos_runtime_resume(struct device *dev)
@@ -530,11 +530,10 @@ static s8 dw_mci_exynos_get_best_clksmpl(u8 candidates)
 	return loc;
 }
 
-static int dw_mci_exynos_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_exynos_execute_tuning(struct dw_mci *host, u32 opcode)
 {
-	struct dw_mci *host = slot->host;
 	struct dw_mci_exynos_priv_data *priv = host->priv;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	u8 start_smpl, smpl, candidates = 0;
 	s8 found;
 	int ret = 0;
diff --git a/drivers/mmc/host/dw_mmc-hi3798cv200.c b/drivers/mmc/host/dw_mmc-hi3798cv200.c
index 0ccfae1b2dbe6..4b723ed3259ca 100644
--- a/drivers/mmc/host/dw_mmc-hi3798cv200.c
+++ b/drivers/mmc/host/dw_mmc-hi3798cv200.c
@@ -57,11 +57,9 @@ static void dw_mci_hi3798cv200_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 		clk_set_phase(priv->drive_clk, 135);
 }
 
-static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci_slot *slot,
-					     u32 opcode)
+static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 };
-	struct dw_mci *host = slot->host;
 	struct hi3798cv200_priv *priv = host->priv;
 	int raise_point = -1, fall_point = -1;
 	int err, prev_err = -1;
@@ -72,7 +70,7 @@ static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci_slot *slot,
 		clk_set_phase(priv->sample_clk, degrees[i]);
 		mci_writel(host, RINTSTS, ALL_INT_CLR);
 
-		err = mmc_send_tuning(slot->mmc, opcode, NULL);
+		err = mmc_send_tuning(host->mmc, opcode, NULL);
 		if (!err)
 			found = 1;
 
diff --git a/drivers/mmc/host/dw_mmc-hi3798mv200.c b/drivers/mmc/host/dw_mmc-hi3798mv200.c
index 5791a975a944d..fda417888be97 100644
--- a/drivers/mmc/host/dw_mmc-hi3798mv200.c
+++ b/drivers/mmc/host/dw_mmc-hi3798mv200.c
@@ -30,13 +30,12 @@ struct dw_mci_hi3798mv200_priv {
 	struct clk *drive_clk;
 	struct regmap *crg_reg;
 	u32 sap_dll_offset;
-	struct mmc_clk_phase_map phase_map;
 };
 
 static void dw_mci_hi3798mv200_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 {
 	struct dw_mci_hi3798mv200_priv *priv = host->priv;
-	struct mmc_clk_phase phase = priv->phase_map.phase[ios->timing];
+	struct mmc_clk_phase phase = host->phase_map.phase[ios->timing];
 	u32 val;
 
 	val = mci_readl(host, ENABLE_SHIFT);
@@ -74,25 +73,24 @@ static void dw_mci_hi3798mv200_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	}
 }
 
-static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci_slot *slot)
+static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci *host)
 {
-	struct dw_mci_hi3798mv200_priv *priv = slot->host->priv;
+	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 
 	return regmap_clear_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE);
 }
 
-static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci_slot *slot)
+static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci *host)
 {
-	struct dw_mci_hi3798mv200_priv *priv = slot->host->priv;
+	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 
 	return regmap_set_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE);
 }
 
-static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
+static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci *host,
 					     u32 opcode)
 {
 	static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 };
-	struct dw_mci *host = slot->host;
 	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 	int raise_point = -1, fall_point = -1, mid;
 	int err, prev_err = -1;
@@ -101,7 +99,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 	int i;
 	int ret;
 
-	ret = dw_mci_hi3798mv200_enable_tuning(slot);
+	ret = dw_mci_hi3798mv200_enable_tuning(host);
 	if (ret < 0)
 		return ret;
 
@@ -115,7 +113,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 		 *
 		 * Treat edge(flip) found as an error too.
 		 */
-		err = mmc_send_tuning(slot->mmc, opcode, NULL);
+		err = mmc_send_tuning(host->mmc, opcode, NULL);
 		regval = mci_readl(host, TUNING_CTRL);
 		if (err || (regval & SDMMC_TUNING_FIND_EDGE))
 			err = 1;
@@ -136,7 +134,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 	}
 
 tuning_out:
-	ret = dw_mci_hi3798mv200_disable_tuning(slot);
+	ret = dw_mci_hi3798mv200_disable_tuning(host);
 	if (ret < 0)
 		return ret;
 
@@ -159,9 +157,9 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 		 * We don't care what timing we are tuning for,
 		 * simply use the same phase for all timing needs tuning.
 		 */
-		priv->phase_map.phase[MMC_TIMING_MMC_HS200].in_deg = degrees[mid];
-		priv->phase_map.phase[MMC_TIMING_MMC_HS400].in_deg = degrees[mid];
-		priv->phase_map.phase[MMC_TIMING_UHS_SDR104].in_deg = degrees[mid];
+		host->phase_map.phase[MMC_TIMING_MMC_HS200].in_deg = degrees[mid];
+		host->phase_map.phase[MMC_TIMING_MMC_HS400].in_deg = degrees[mid];
+		host->phase_map.phase[MMC_TIMING_UHS_SDR104].in_deg = degrees[mid];
 
 		clk_set_phase(priv->sample_clk, degrees[mid]);
 		dev_dbg(host->dev, "Tuning clk_sample[%d, %d], set[%d]\n",
@@ -186,8 +184,6 @@ static int dw_mci_hi3798mv200_init(struct dw_mci *host)
 	if (!priv)
 		return -ENOMEM;
 
-	mmc_of_parse_clk_phase(host->dev, &priv->phase_map);
-
 	priv->sample_clk = devm_clk_get_enabled(host->dev, "ciu-sample");
 	if (IS_ERR(priv->sample_clk))
 		return dev_err_probe(host->dev, PTR_ERR(priv->sample_clk),
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index ad6aa1aea5491..59b802c9f2b86 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -53,7 +53,6 @@
 #define USE_DLY_MAX_SMPL (14)
 
 struct k3_priv {
-	int ctrl_id;
 	u32 cur_speed;
 	struct regmap	*reg;
 };
@@ -127,26 +126,17 @@ static int dw_mci_hi6220_parse_dt(struct dw_mci *host)
 	if (IS_ERR(priv->reg))
 		priv->reg = NULL;
 
-	priv->ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
-	if (priv->ctrl_id < 0)
-		priv->ctrl_id = 0;
-
-	if (priv->ctrl_id >= TIMING_MODE)
-		return -EINVAL;
-
 	host->priv = priv;
 	return 0;
 }
 
-static int dw_mci_hi6220_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
+static int dw_mci_hi6220_switch_voltage(struct dw_mci *host, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct k3_priv *priv;
-	struct dw_mci *host;
+	struct mmc_host *mmc = host->mmc;
 	int min_uv, max_uv;
 	int ret;
 
-	host = slot->host;
 	priv = host->priv;
 
 	if (!priv || !priv->reg)
@@ -199,7 +189,7 @@ static void dw_mci_hi6220_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	host->bus_hz = clk_get_rate(host->biu_clk);
 }
 
-static int dw_mci_hi6220_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_hi6220_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	return 0;
 }
@@ -213,7 +203,7 @@ static const struct dw_mci_drv_data hi6220_data = {
 	.execute_tuning		= dw_mci_hi6220_execute_tuning,
 };
 
-static void dw_mci_hs_set_timing(struct dw_mci *host, int timing,
+static int dw_mci_hs_set_timing(struct dw_mci *host, int timing,
 				     int smpl_phase)
 {
 	u32 drv_phase;
@@ -222,10 +212,10 @@ static void dw_mci_hs_set_timing(struct dw_mci *host, int timing,
 	u32 enable_shift = 0;
 	u32 reg_value;
 	int ctrl_id;
-	struct k3_priv *priv;
 
-	priv = host->priv;
-	ctrl_id = priv->ctrl_id;
+	ctrl_id = host->mmc->index;
+	if (ctrl_id >= TIMING_MODE)
+		return -EINVAL;
 
 	drv_phase = hs_timing_cfg[ctrl_id][timing].drv_phase;
 	smpl_dly   = hs_timing_cfg[ctrl_id][timing].smpl_dly;
@@ -262,6 +252,8 @@ static void dw_mci_hs_set_timing(struct dw_mci *host, int timing,
 
 	/* We should delay 1ms wait for timing setting finished. */
 	usleep_range(1000, 2000);
+
+	return 0;
 }
 
 static int dw_mci_hi3660_init(struct dw_mci *host)
@@ -269,10 +261,9 @@ static int dw_mci_hi3660_init(struct dw_mci *host)
 	mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(SDCARD_RD_THRESHOLD,
 		    SDMMC_CARD_RD_THR_EN));
 
-	dw_mci_hs_set_timing(host, MMC_TIMING_LEGACY, -1);
 	host->bus_hz /= (GENCLK_DIV + 1);
 
-	return 0;
+	return dw_mci_hs_set_timing(host, MMC_TIMING_LEGACY, -1);
 }
 
 static int dw_mci_set_sel18(struct dw_mci *host, bool set)
@@ -364,11 +355,10 @@ static int dw_mci_get_best_clksmpl(unsigned int sample_flag)
 	return middle_range;
 }
 
-static int dw_mci_hi3660_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_hi3660_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	int i = 0;
-	struct dw_mci *host = slot->host;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int smpl_phase = 0;
 	u32 tuning_sample_flag = 0;
 	int best_clksmpl = 0;
@@ -398,21 +388,19 @@ static int dw_mci_hi3660_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
 	return 0;
 }
 
-static int dw_mci_hi3660_switch_voltage(struct mmc_host *mmc,
+static int dw_mci_hi3660_switch_voltage(struct dw_mci *host,
 					struct mmc_ios *ios)
 {
-	int ret = 0;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct k3_priv *priv;
-	struct dw_mci *host;
+	struct mmc_host *mmc = host->mmc;
+	int ret = 0;
 
-	host = slot->host;
 	priv = host->priv;
 
 	if (!priv || !priv->reg)
 		return 0;
 
-	if (priv->ctrl_id == DWMMC_SDIO_ID)
+	if (mmc->index == DWMMC_SDIO_ID)
 		return 0;
 
 	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
@@ -460,11 +448,6 @@ static int dw_mci_k3_probe(struct platform_device *pdev)
 	return dw_mci_pltfm_register(pdev, drv_data);
 }
 
-static const struct dev_pm_ops dw_mci_k3_dev_pm_ops = {
-	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
-	RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL)
-};
-
 static struct platform_driver dw_mci_k3_pltfm_driver = {
 	.probe		= dw_mci_k3_probe,
 	.remove		= dw_mci_pltfm_remove,
@@ -472,7 +455,7 @@ static struct platform_driver dw_mci_k3_pltfm_driver = {
 		.name		= "dwmmc_k3",
 		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= dw_mci_k3_match,
-		.pm		= pm_ptr(&dw_mci_k3_dev_pm_ops),
+		.pm		= pm_ptr(&dw_mci_pmops),
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-pci.c b/drivers/mmc/host/dw_mmc-pci.c
index 092cc99175af0..a2e1b4d75a5ee 100644
--- a/drivers/mmc/host/dw_mmc-pci.c
+++ b/drivers/mmc/host/dw_mmc-pci.c
@@ -10,13 +10,14 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/pci.h>
+#include <linux/pci-epf.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
 #include "dw_mmc.h"
+#include "dw_mmc-pltfm.h"
 
-#define PCI_BAR_NO 2
 #define SYNOPSYS_DW_MCI_VENDOR_ID 0x700
 #define SYNOPSYS_DW_MCI_DEVICE_ID 0x1107
 /* Defining the Capabilities */
@@ -24,11 +25,8 @@
 				MMC_CAP_SD_HIGHSPEED | MMC_CAP_8_BIT_DATA |\
 				MMC_CAP_SDIO_IRQ)
 
-static struct dw_mci_board pci_board_data = {
-	.caps				= DW_MCI_CAPABILITIES,
-	.bus_hz				= 33 * 1000 * 1000,
-	.detect_delay_ms		= 200,
-	.fifo_depth			= 32,
+static const struct dw_mci_drv_data pci_drv_data = {
+	.common_caps = DW_MCI_CAPABILITIES,
 };
 
 static int dw_mci_pci_probe(struct pci_dev *pdev,
@@ -41,20 +39,20 @@ static int dw_mci_pci_probe(struct pci_dev *pdev,
 	if (ret)
 		return ret;
 
-	host = devm_kzalloc(&pdev->dev, sizeof(struct dw_mci), GFP_KERNEL);
-	if (!host)
-		return -ENOMEM;
+	host = dw_mci_alloc_host(&pdev->dev);
+	if (IS_ERR(host))
+		return PTR_ERR(host);
 
 	host->irq = pdev->irq;
 	host->irq_flags = IRQF_SHARED;
-	host->dev = &pdev->dev;
-	host->pdata = &pci_board_data;
+	host->fifo_depth = 32;
+	host->detect_delay_ms = 200;
+	host->bus_hz = 33 * 1000 * 1000;
+	host->drv_data = &pci_drv_data;
 
-	ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_NO, pci_name(pdev));
-	if (ret)
-		return ret;
-
-	host->regs = pcim_iomap_table(pdev)[PCI_BAR_NO];
+	host->regs = pcim_iomap_region(pdev, BAR_2, pci_name(pdev));
+	if (IS_ERR(host->regs))
+		return PTR_ERR(host->regs);
 
 	pci_set_master(pdev);
 
@@ -74,11 +72,6 @@ static void dw_mci_pci_remove(struct pci_dev *pdev)
 	dw_mci_remove(host);
 }
 
-static const struct dev_pm_ops dw_mci_pci_dev_pm_ops = {
-	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
-	RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL)
-};
-
 static const struct pci_device_id dw_mci_pci_id[] = {
 	{ PCI_DEVICE(SYNOPSYS_DW_MCI_VENDOR_ID, SYNOPSYS_DW_MCI_DEVICE_ID) },
 	{}
@@ -91,7 +84,7 @@ static struct pci_driver dw_mci_pci_driver = {
 	.probe		= dw_mci_pci_probe,
 	.remove		= dw_mci_pci_remove,
 	.driver		=	{
-		.pm =   pm_ptr(&dw_mci_pci_dev_pm_ops),
+		.pm =   pm_ptr(&dw_mci_pmops),
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-pltfm.c b/drivers/mmc/host/dw_mmc-pltfm.c
index de820ffd21333..68770aaff8d99 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.c
+++ b/drivers/mmc/host/dw_mmc-pltfm.c
@@ -33,18 +33,16 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 	struct dw_mci *host;
 	struct resource	*regs;
 
-	host = devm_kzalloc(&pdev->dev, sizeof(struct dw_mci), GFP_KERNEL);
-	if (!host)
-		return -ENOMEM;
+	host = dw_mci_alloc_host(&pdev->dev);
+	if (IS_ERR(host))
+		return PTR_ERR(host);
 
 	host->irq = platform_get_irq(pdev, 0);
 	if (host->irq < 0)
 		return host->irq;
 
 	host->drv_data = drv_data;
-	host->dev = &pdev->dev;
 	host->irq_flags = 0;
-	host->pdata = pdev->dev.platform_data;
 
 	host->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
 	if (IS_ERR(host->regs))
@@ -58,15 +56,6 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 }
 EXPORT_SYMBOL_GPL(dw_mci_pltfm_register);
 
-const struct dev_pm_ops dw_mci_pltfm_pmops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
-	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
-			   dw_mci_runtime_resume,
-			   NULL)
-};
-EXPORT_SYMBOL_GPL(dw_mci_pltfm_pmops);
-
 static int dw_mci_socfpga_priv_init(struct dw_mci *host)
 {
 	struct device_node *np = host->dev->of_node;
@@ -136,7 +125,7 @@ static struct platform_driver dw_mci_pltfm_driver = {
 		.name		= "dw_mmc",
 		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table	= dw_mci_pltfm_match,
-		.pm		= &dw_mci_pltfm_pmops,
+		.pm		= pm_ptr(&dw_mci_pmops),
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-pltfm.h b/drivers/mmc/host/dw_mmc-pltfm.h
index 64cf7522a3d49..ef1b05d484c39 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.h
+++ b/drivers/mmc/host/dw_mmc-pltfm.h
@@ -11,6 +11,6 @@
 extern int dw_mci_pltfm_register(struct platform_device *pdev,
 				const struct dw_mci_drv_data *drv_data);
 extern void dw_mci_pltfm_remove(struct platform_device *pdev);
-extern const struct dev_pm_ops dw_mci_pltfm_pmops;
+extern const struct dev_pm_ops dw_mci_pmops;
 
 #endif /* _DW_MMC_PLTFM_H_ */
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index ac069d0c42b26..c6eece4ec3fda 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -179,7 +179,8 @@ static int rockchip_mmc_set_phase(struct dw_mci *host, bool sample, int degrees)
 static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 {
 	struct dw_mci_rockchip_priv_data *priv = host->priv;
-	int ret;
+	struct mmc_clk_phase phase = host->phase_map.phase[ios->timing];
+	int ret, sample_phase, drv_phase;
 	unsigned int cclkin;
 	u32 bus_hz;
 
@@ -213,8 +214,15 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	}
 
 	/* Make sure we use phases which we can enumerate with */
-	if (!IS_ERR(priv->sample_clk) && ios->timing <= MMC_TIMING_SD_HS)
-		rockchip_mmc_set_phase(host, true, priv->default_sample_phase);
+	if (!IS_ERR(priv->sample_clk)) {
+		/* Keep backward compatibility */
+		if (ios->timing <= MMC_TIMING_SD_HS) {
+			sample_phase = phase.valid ? phase.in_deg : priv->default_sample_phase;
+			rockchip_mmc_set_phase(host, true, sample_phase);
+		} else if (phase.valid) {
+			rockchip_mmc_set_phase(host, true, phase.in_deg);
+		}
+	}
 
 	/*
 	 * Set the drive phase offset based on speed mode to achieve hold times.
@@ -243,15 +251,13 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	 * same results, for instance).
 	 */
 	if (!IS_ERR(priv->drv_clk)) {
-		int phase;
-
 		/*
 		 * In almost all cases a 90 degree phase offset will provide
 		 * sufficient hold times across all valid input clock rates
 		 * assuming delay_o is not absurd for a given SoC.  We'll use
 		 * that as a default.
 		 */
-		phase = 90;
+		drv_phase = 90;
 
 		switch (ios->timing) {
 		case MMC_TIMING_MMC_DDR52:
@@ -261,7 +267,7 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 			 * to get the same timings.
 			 */
 			if (ios->bus_width == MMC_BUS_WIDTH_8)
-				phase = 180;
+				drv_phase = 180;
 			break;
 		case MMC_TIMING_UHS_SDR104:
 		case MMC_TIMING_MMC_HS200:
@@ -273,22 +279,24 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 			 * SoCs measured this seems to be OK, but it doesn't
 			 * hurt to give margin here, so we use 180.
 			 */
-			phase = 180;
+			drv_phase = 180;
 			break;
 		}
 
-		rockchip_mmc_set_phase(host, false, phase);
+		/* Use out phase from phase map first */
+		if (phase.valid)
+			drv_phase = phase.out_deg;
+		rockchip_mmc_set_phase(host, false, drv_phase);
 	}
 }
 
 #define TUNING_ITERATION_TO_PHASE(i, num_phases) \
 		(DIV_ROUND_UP((i) * 360, num_phases))
 
-static int dw_mci_rk3288_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_rk3288_execute_tuning(struct dw_mci *host, u32 opcode)
 {
-	struct dw_mci *host = slot->host;
 	struct dw_mci_rockchip_priv_data *priv = host->priv;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int ret = 0;
 	int i;
 	bool v, prev_v = 0, first_v;
@@ -476,8 +484,8 @@ static int dw_mci_rockchip_init(struct dw_mci *host)
 	struct dw_mci_rockchip_priv_data *priv = host->priv;
 	int ret, i;
 
-	/* It is slot 8 on Rockchip SoCs */
-	host->sdio_id0 = 8;
+	/* SDIO irq is the 8th on Rockchip SoCs */
+	host->sdio_irq = 8;
 
 	if (of_device_is_compatible(host->dev->of_node, "rockchip,rk3288-dw-mshc")) {
 		host->bus_hz /= RK3288_CLKGEN_DIV;
diff --git a/drivers/mmc/host/dw_mmc-starfive.c b/drivers/mmc/host/dw_mmc-starfive.c
index 34964b0dab21f..11472a641cbe9 100644
--- a/drivers/mmc/host/dw_mmc-starfive.c
+++ b/drivers/mmc/host/dw_mmc-starfive.c
@@ -53,11 +53,10 @@ static void dw_mci_starfive_set_sample_phase(struct dw_mci *host, u32 smpl_phase
 	mdelay(1);
 }
 
-static int dw_mci_starfive_execute_tuning(struct dw_mci_slot *slot,
+static int dw_mci_starfive_execute_tuning(struct dw_mci *host,
 					     u32 opcode)
 {
 	static const int grade  = MAX_DELAY_CHAIN;
-	struct dw_mci *host = slot->host;
 	int smpl_phase, smpl_raise = -1, smpl_fall = -1;
 	int ret;
 
@@ -65,7 +64,7 @@ static int dw_mci_starfive_execute_tuning(struct dw_mci_slot *slot,
 		dw_mci_starfive_set_sample_phase(host, smpl_phase);
 		mci_writel(host, RINTSTS, ALL_INT_CLR);
 
-		ret = mmc_send_tuning(slot->mmc, opcode, NULL);
+		ret = mmc_send_tuning(host->mmc, opcode, NULL);
 
 		if (!ret && smpl_raise < 0) {
 			smpl_raise = smpl_phase;
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 07eb9f23b9d68..bbdd7594ef01b 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -7,35 +7,28 @@
  * Copyright (C) 2009, 2010 Imagination Technologies Ltd.
  */
 
-#include <linux/blkdev.h>
+#include <linux/bitops.h>
 #include <linux/clk.h>
 #include <linux/debugfs.h>
+#include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
-#include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
-#include <linux/ioport.h>
-#include <linux/ktime.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/prandom.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/stat.h>
-#include <linux/delay.h>
 #include <linux/irq.h>
+#include <linux/ktime.h>
 #include <linux/mmc/card.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
 #include <linux/mmc/sd.h>
 #include <linux/mmc/sdio.h>
-#include <linux/bitops.h>
-#include <linux/regulator/consumer.h>
-#include <linux/of.h>
 #include <linux/mmc/slot-gpio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
 
 #include "dw_mmc.h"
 
@@ -47,8 +40,6 @@
 				 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE)
 #define DW_MCI_ERROR_FLAGS	(DW_MCI_DATA_ERROR_FLAGS | \
 				 DW_MCI_CMD_ERROR_FLAGS)
-#define DW_MCI_SEND_STATUS	1
-#define DW_MCI_RECV_STATUS	2
 #define DW_MCI_DMA_THRESHOLD	16
 
 #define DW_MCI_FREQ_MAX	200000000	/* unit: HZ */
@@ -107,15 +98,15 @@ struct idmac_desc {
 #if defined(CONFIG_DEBUG_FS)
 static int dw_mci_req_show(struct seq_file *s, void *v)
 {
-	struct dw_mci_slot *slot = s->private;
+	struct dw_mci *host = s->private;
 	struct mmc_request *mrq;
 	struct mmc_command *cmd;
 	struct mmc_command *stop;
 	struct mmc_data	*data;
 
 	/* Make sure we get a consistent snapshot */
-	spin_lock_bh(&slot->host->lock);
-	mrq = slot->mrq;
+	spin_lock_bh(&host->lock);
+	mrq = host->mrq;
 
 	if (mrq) {
 		cmd = mrq->cmd;
@@ -140,7 +131,7 @@ static int dw_mci_req_show(struct seq_file *s, void *v)
 				   stop->resp[2], stop->error);
 	}
 
-	spin_unlock_bh(&slot->host->lock);
+	spin_unlock_bh(&host->lock);
 
 	return 0;
 }
@@ -165,10 +156,9 @@ static int dw_mci_regs_show(struct seq_file *s, void *v)
 }
 DEFINE_SHOW_ATTRIBUTE(dw_mci_regs);
 
-static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
+static void dw_mci_init_debugfs(struct dw_mci *host)
 {
-	struct mmc_host	*mmc = slot->mmc;
-	struct dw_mci *host = slot->host;
+	struct mmc_host *mmc = host->mmc;
 	struct dentry *root;
 
 	root = mmc->debugfs_root;
@@ -176,7 +166,7 @@ static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
 		return;
 
 	debugfs_create_file("regs", 0400, root, host, &dw_mci_regs_fops);
-	debugfs_create_file("req", 0400, root, slot, &dw_mci_req_fops);
+	debugfs_create_file("req", 0400, root, host, &dw_mci_req_fops);
 	debugfs_create_u32("state", 0400, root, &host->state);
 	debugfs_create_xul("pending_events", 0400, root,
 			   &host->pending_events);
@@ -231,9 +221,8 @@ static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags)
 	}
 }
 
-static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)
+static void mci_send_cmd(struct dw_mci *host, u32 cmd, u32 arg)
 {
-	struct dw_mci *host = slot->host;
 	unsigned int cmd_status = 0;
 
 	mci_writel(host, CMDARG, arg);
@@ -244,15 +233,14 @@ static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)
 	if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status,
 				      !(cmd_status & SDMMC_CMD_START),
 				      1, 500 * USEC_PER_MSEC))
-		dev_err(&slot->mmc->class_dev,
+		dev_err(&host->mmc->class_dev,
 			"Timeout sending command (cmd %#x arg %#x status %#x)\n",
 			cmd, arg, cmd_status);
 }
 
 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	u32 cmdr;
 
 	cmd->error = -EINPROGRESS;
@@ -274,8 +262,8 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 		cmdr |= SDMMC_CMD_VOLT_SWITCH;
 
 		/* Change state to continue to handle CMD11 weirdness */
-		WARN_ON(slot->host->state != STATE_SENDING_CMD);
-		slot->host->state = STATE_SENDING_CMD11;
+		WARN_ON(host->state != STATE_SENDING_CMD);
+		host->state = STATE_SENDING_CMD11;
 
 		/*
 		 * We need to disable low power mode (automatic clock stop)
@@ -289,9 +277,9 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 		 * until the voltage change is all done.
 		 */
 		clk_en_a = mci_readl(host, CLKENA);
-		clk_en_a &= ~(SDMMC_CLKEN_LOW_PWR << slot->id);
+		clk_en_a &= ~SDMMC_CLKEN_LOW_PWR;
 		mci_writel(host, CLKENA, clk_en_a);
-		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK |
+		mci_send_cmd(host, SDMMC_CMD_UPD_CLK |
 			     SDMMC_CMD_PRV_DAT_WAIT, 0);
 	}
 
@@ -311,7 +299,7 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 			cmdr |= SDMMC_CMD_DAT_WR;
 	}
 
-	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &slot->flags))
+	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags))
 		cmdr |= SDMMC_CMD_USE_HOLD_REG;
 
 	return cmdr;
@@ -350,7 +338,7 @@ static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)
 	cmdr = stop->opcode | SDMMC_CMD_STOP |
 		SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP;
 
-	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags))
+	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags))
 		cmdr |= SDMMC_CMD_USE_HOLD_REG;
 
 	return cmdr;
@@ -480,7 +468,7 @@ static void dw_mci_dmac_complete_dma(void *arg)
 	if ((host->use_dma == TRANS_MODE_EDMAC) &&
 	    data && (data->flags & MMC_DATA_READ))
 		/* Invalidate cache after read */
-		dma_sync_sg_for_cpu(mmc_dev(host->slot->mmc),
+		dma_sync_sg_for_cpu(mmc_dev(host->mmc),
 				    data->sg,
 				    data->sg_len,
 				    DMA_FROM_DEVICE);
@@ -575,23 +563,26 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 	return 0;
 }
 
-static inline int dw_mci_prepare_desc64(struct dw_mci *host,
-					 struct mmc_data *data,
-					 unsigned int sg_len)
+static inline int dw_mci_prepare_desc(struct dw_mci *host, struct mmc_data *data,
+				      unsigned int sg_len, bool is_64bit)
 {
 	unsigned int desc_len;
-	struct idmac_desc_64addr *desc_first, *desc_last, *desc;
-	u32 val;
-	int i;
+	struct idmac_desc *desc_first, *desc_last, *desc;
+	struct idmac_desc_64addr *desc64_first, *desc64_last, *desc64;
+	u32 val, des0;
+	int i, err;
 
-	desc_first = desc_last = desc = host->sg_cpu;
+	if (is_64bit)
+		desc64_first = desc64_last = desc64 = host->sg_cpu;
+	else
+		desc_first = desc_last = desc = host->sg_cpu;
 
 	for (i = 0; i < sg_len; i++) {
 		unsigned int length = sg_dma_len(&data->sg[i]);
 
 		u64 mem_addr = sg_dma_address(&data->sg[i]);
 
-		for ( ; length ; desc++) {
+		while (length > 0) {
 			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
 				   length : DW_MCI_DESC_DATA_LENGTH;
 
@@ -603,113 +594,60 @@ static inline int dw_mci_prepare_desc64(struct dw_mci *host,
 			 * isn't still owned by IDMAC as IDMAC's write
 			 * ops and CPU's read ops are asynchronous.
 			 */
-			if (readl_poll_timeout_atomic(&desc->des0, val,
-						!(val & IDMAC_DES0_OWN),
-						10, 100 * USEC_PER_MSEC))
+			if (is_64bit)
+				err = readl_poll_timeout_atomic(&desc64->des0, val,
+					IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC);
+			else
+				err = readl_poll_timeout_atomic(&desc->des0, val,
+					IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC);
+			if (err)
 				goto err_own_bit;
 
-			/*
-			 * Set the OWN bit and disable interrupts
-			 * for this descriptor
-			 */
-			desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |
-						IDMAC_DES0_CH;
-
-			/* Buffer length */
-			IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);
-
-			/* Physical address to DMA to/from */
-			desc->des4 = mem_addr & 0xffffffff;
-			desc->des5 = mem_addr >> 32;
-
-			/* Update physical address for the next desc */
-			mem_addr += desc_len;
-
-			/* Save pointer to the last descriptor */
-			desc_last = desc;
-		}
-	}
-
-	/* Set first descriptor */
-	desc_first->des0 |= IDMAC_DES0_FD;
-
-	/* Set last descriptor */
-	desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
-	desc_last->des0 |= IDMAC_DES0_LD;
-
-	return 0;
-err_own_bit:
-	/* restore the descriptor chain as it's polluted */
-	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
-	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
-	dw_mci_idmac_init(host);
-	return -EINVAL;
-}
-
-
-static inline int dw_mci_prepare_desc32(struct dw_mci *host,
-					 struct mmc_data *data,
-					 unsigned int sg_len)
-{
-	unsigned int desc_len;
-	struct idmac_desc *desc_first, *desc_last, *desc;
-	u32 val;
-	int i;
-
-	desc_first = desc_last = desc = host->sg_cpu;
-
-	for (i = 0; i < sg_len; i++) {
-		unsigned int length = sg_dma_len(&data->sg[i]);
-
-		u32 mem_addr = sg_dma_address(&data->sg[i]);
-
-		for ( ; length ; desc++) {
-			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
-				   length : DW_MCI_DESC_DATA_LENGTH;
-
-			length -= desc_len;
-
-			/*
-			 * Wait for the former clear OWN bit operation
-			 * of IDMAC to make sure that this descriptor
-			 * isn't still owned by IDMAC as IDMAC's write
-			 * ops and CPU's read ops are asynchronous.
-			 */
-			if (readl_poll_timeout_atomic(&desc->des0, val,
-						      IDMAC_OWN_CLR64(val),
-						      10,
-						      100 * USEC_PER_MSEC))
-				goto err_own_bit;
+			des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH;
+			if (is_64bit)
+				desc64->des0 = des0;
+			else
+				desc->des0 = cpu_to_le32(des0);
 
 			/*
-			 * Set the OWN bit and disable interrupts
-			 * for this descriptor
+			 * 1. Set OWN bit and disable interrupts for this descriptor
+			 * 2. Set Buffer length
+			 * Set physical address to DMA to/from
 			 */
-			desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |
-						 IDMAC_DES0_DIC |
-						 IDMAC_DES0_CH);
-
-			/* Buffer length */
-			IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
-
-			/* Physical address to DMA to/from */
-			desc->des2 = cpu_to_le32(mem_addr);
+			if (is_64bit) {
+				desc64->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH;
+				IDMAC_64ADDR_SET_BUFFER1_SIZE(desc64, desc_len);
+				desc64->des4 = mem_addr & 0xffffffff;
+				desc64->des5 = mem_addr >> 32;
+			} else {
+				IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
+				desc->des2 = cpu_to_le32(mem_addr);
+			}
 
 			/* Update physical address for the next desc */
 			mem_addr += desc_len;
 
 			/* Save pointer to the last descriptor */
-			desc_last = desc;
+			if (is_64bit) {
+				desc64_last = desc64;
+				desc64++;
+			} else {
+				desc_last = desc;
+				desc++;
+			}
 		}
 	}
 
-	/* Set first descriptor */
-	desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
-
-	/* Set last descriptor */
-	desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |
-				       IDMAC_DES0_DIC));
-	desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
+	/* Set the first descriptor and the last descriptor */
+	if (is_64bit) {
+		desc64_first->des0 |= IDMAC_DES0_FD;
+		desc64_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
+		desc64_last->des0 |= IDMAC_DES0_LD;
+	} else {
+		desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
+		desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC));
+		desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
+	}
 
 	return 0;
 err_own_bit:
@@ -725,11 +663,7 @@ static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)
 	u32 temp;
 	int ret;
 
-	if (host->dma_64bit_address == 1)
-		ret = dw_mci_prepare_desc64(host, host->data, sg_len);
-	else
-		ret = dw_mci_prepare_desc32(host, host->data, sg_len);
-
+	ret = dw_mci_prepare_desc(host, host->data, sg_len, host->dma_64bit_address);
 	if (ret)
 		goto out;
 
@@ -823,7 +757,7 @@ static int dw_mci_edmac_start_dma(struct dw_mci *host,
 
 	/* Flush cache before write */
 	if (host->data->flags & MMC_DATA_WRITE)
-		dma_sync_sg_for_device(mmc_dev(host->slot->mmc), sgl,
+		dma_sync_sg_for_device(mmc_dev(host->mmc), sgl,
 				       sg_elems, DMA_TO_DEVICE);
 
 	dma_async_issue_pending(host->dms->ch);
@@ -913,16 +847,16 @@ static int dw_mci_pre_dma_transfer(struct dw_mci *host,
 static void dw_mci_pre_req(struct mmc_host *mmc,
 			   struct mmc_request *mrq)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
 
-	if (!slot->host->use_dma || !data)
+	if (!host->use_dma || !data)
 		return;
 
 	/* This data might be unmapped at this time */
 	data->host_cookie = COOKIE_UNMAPPED;
 
-	if (dw_mci_pre_dma_transfer(slot->host, mrq->data,
+	if (dw_mci_pre_dma_transfer(host, mrq->data,
 				COOKIE_PRE_MAPPED) < 0)
 		data->host_cookie = COOKIE_UNMAPPED;
 }
@@ -931,14 +865,14 @@ static void dw_mci_post_req(struct mmc_host *mmc,
 			    struct mmc_request *mrq,
 			    int err)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
 
-	if (!slot->host->use_dma || !data)
+	if (!host->use_dma || !data)
 		return;
 
 	if (data->host_cookie != COOKIE_UNMAPPED)
-		dma_unmap_sg(slot->host->dev,
+		dma_unmap_sg(host->dev,
 			     data->sg,
 			     data->sg_len,
 			     mmc_get_dma_dir(data));
@@ -947,43 +881,21 @@ static void dw_mci_post_req(struct mmc_host *mmc,
 
 static int dw_mci_get_cd(struct mmc_host *mmc)
 {
-	int present;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	int gpio_cd = mmc_gpio_get_cd(mmc);
 
-	/* Use platform get_cd function, else try onboard card detect */
-	if (((mmc->caps & MMC_CAP_NEEDS_POLL)
-				|| !mmc_card_is_removable(mmc))) {
-		present = 1;
+	if (mmc->caps & MMC_CAP_NEEDS_POLL)
+		return 1;
 
-		if (!test_bit(DW_MMC_CARD_PRESENT, &slot->flags)) {
-			if (mmc->caps & MMC_CAP_NEEDS_POLL) {
-				dev_info(&mmc->class_dev,
-					"card is polling.\n");
-			} else {
-				dev_info(&mmc->class_dev,
-					"card is non-removable.\n");
-			}
-			set_bit(DW_MMC_CARD_PRESENT, &slot->flags);
-		}
-
-		return present;
-	} else if (gpio_cd >= 0)
-		present = gpio_cd;
-	else
-		present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))
-			== 0 ? 1 : 0;
+	if (!mmc_card_is_removable(mmc))
+		return 1;
 
-	spin_lock_bh(&host->lock);
-	if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &slot->flags))
-		dev_dbg(&mmc->class_dev, "card is present\n");
-	else if (!present &&
-			!test_and_clear_bit(DW_MMC_CARD_PRESENT, &slot->flags))
-		dev_dbg(&mmc->class_dev, "card is not present\n");
-	spin_unlock_bh(&host->lock);
+	/* Try slot gpio detection */
+	if (gpio_cd >= 0)
+		return !!gpio_cd;
 
-	return present;
+	/* Host native card detect */
+	return !(mci_readl(host, CDETECT) & BIT(0));
 }
 
 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)
@@ -1150,9 +1062,9 @@ static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)
 	host->data = data;
 
 	if (data->flags & MMC_DATA_READ)
-		host->dir_status = DW_MCI_RECV_STATUS;
+		host->dir_status = MMC_DATA_READ;
 	else
-		host->dir_status = DW_MCI_SEND_STATUS;
+		host->dir_status = MMC_DATA_WRITE;
 
 	dw_mci_ctrl_thld(host, data);
 
@@ -1200,10 +1112,9 @@ static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)
 	}
 }
 
-static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
+static void dw_mci_setup_bus(struct dw_mci *host, bool force_clkinit)
 {
-	struct dw_mci *host = slot->host;
-	unsigned int clock = slot->clock;
+	unsigned int clock = host->clock;
 	u32 div;
 	u32 clk_en_a;
 	u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT;
@@ -1212,11 +1123,11 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 	if (host->state == STATE_WAITING_CMD11_DONE)
 		sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH;
 
-	slot->mmc->actual_clock = 0;
+	host->mmc->actual_clock = 0;
 
 	if (!clock) {
 		mci_writel(host, CLKENA, 0);
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 	} else if (clock != host->current_speed || force_clkinit) {
 		div = host->bus_hz / clock;
 		if (host->bus_hz % clock && host->bus_hz > clock)
@@ -1228,14 +1139,14 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;
 
-		if ((clock != slot->__clk_old &&
-			!test_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags)) ||
+		if ((clock != host->clk_old &&
+			!test_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags)) ||
 			force_clkinit) {
 			/* Silent the verbose log if calling from PM context */
 			if (!force_clkinit)
-				dev_info(&slot->mmc->class_dev,
-					 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
-					 slot->id, host->bus_hz, clock,
+				dev_info(&host->mmc->class_dev,
+					 "Bus speed = %dHz (req %dHz, actual %dHZ div = %d)\n",
+					 host->bus_hz, clock,
 					 div ? ((host->bus_hz / div) >> 1) :
 					 host->bus_hz, div);
 
@@ -1243,9 +1154,9 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 			 * If card is polling, display the message only
 			 * one time at boot time.
 			 */
-			if (slot->mmc->caps & MMC_CAP_NEEDS_POLL &&
-					slot->mmc->f_min == clock)
-				set_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags);
+			if (host->mmc->caps & MMC_CAP_NEEDS_POLL &&
+					host->mmc->f_min == clock)
+				set_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags);
 		}
 
 		/* disable clock */
@@ -1253,33 +1164,33 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 		mci_writel(host, CLKSRC, 0);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* set clock to desired speed */
 		mci_writel(host, CLKDIV, div);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* enable clock; only low power if no SDIO */
-		clk_en_a = SDMMC_CLKEN_ENABLE << slot->id;
-		if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags))
-			clk_en_a |= SDMMC_CLKEN_LOW_PWR << slot->id;
+		clk_en_a = SDMMC_CLKEN_ENABLE;
+		if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags))
+			clk_en_a |= SDMMC_CLKEN_LOW_PWR;
 		mci_writel(host, CLKENA, clk_en_a);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* keep the last clock value that was requested from core */
-		slot->__clk_old = clock;
-		slot->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :
+		host->clk_old = clock;
+		host->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :
 					  host->bus_hz;
 	}
 
 	host->current_speed = clock;
 
-	/* Set the current slot bus width */
-	mci_writel(host, CTYPE, (slot->ctype << slot->id));
+	/* Set the current bus width */
+	mci_writel(host, CTYPE, host->ctype);
 }
 
 static void dw_mci_set_data_timeout(struct dw_mci *host,
@@ -1314,14 +1225,13 @@ static void dw_mci_set_data_timeout(struct dw_mci *host,
 }
 
 static void __dw_mci_start_request(struct dw_mci *host,
-				   struct dw_mci_slot *slot,
 				   struct mmc_command *cmd)
 {
 	struct mmc_request *mrq;
 	struct mmc_data	*data;
 	u32 cmdflags;
 
-	mrq = slot->mrq;
+	mrq = host->mrq;
 
 	host->mrq = mrq;
 
@@ -1338,10 +1248,10 @@ static void __dw_mci_start_request(struct dw_mci *host,
 		mci_writel(host, BLKSIZ, data->blksz);
 	}
 
-	cmdflags = dw_mci_prepare_command(slot->mmc, cmd);
+	cmdflags = dw_mci_prepare_command(host->mmc, cmd);
 
 	/* this is the first command, send the initialization clock */
-	if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &slot->flags))
+	if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &host->flags))
 		cmdflags |= SDMMC_CMD_INIT;
 
 	if (data) {
@@ -1374,27 +1284,25 @@ static void __dw_mci_start_request(struct dw_mci *host,
 	host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);
 }
 
-static void dw_mci_start_request(struct dw_mci *host,
-				 struct dw_mci_slot *slot)
+static void dw_mci_start_request(struct dw_mci *host)
 {
-	struct mmc_request *mrq = slot->mrq;
+	struct mmc_request *mrq = host->mrq;
 	struct mmc_command *cmd;
 
 	cmd = mrq->sbc ? mrq->sbc : mrq->cmd;
-	__dw_mci_start_request(host, slot, cmd);
+	__dw_mci_start_request(host, cmd);
 }
 
 /* must be called with host->lock held */
-static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,
-				 struct mmc_request *mrq)
+static void dw_mci_queue_request(struct dw_mci *host, struct mmc_request *mrq)
 {
-	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
+	dev_vdbg(&host->mmc->class_dev, "queue request: state=%d\n",
 		 host->state);
 
-	slot->mrq = mrq;
+	host->mrq = mrq;
 
 	if (host->state == STATE_WAITING_CMD11_DONE) {
-		dev_warn(&slot->mmc->class_dev,
+		dev_warn(&host->mmc->class_dev,
 			 "Voltage change didn't complete\n");
 		/*
 		 * this case isn't expected to happen, so we can
@@ -1406,18 +1314,15 @@ static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,
 
 	if (host->state == STATE_IDLE) {
 		host->state = STATE_SENDING_CMD;
-		dw_mci_start_request(host, slot);
-	} else {
-		list_add_tail(&slot->queue_node, &host->queue);
+		dw_mci_start_request(host);
 	}
 }
 
 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 
-	WARN_ON(slot->mrq);
+	WARN_ON(host->mrq);
 
 	/*
 	 * The check for card presence and queueing of the request must be
@@ -1433,141 +1338,117 @@ static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 
 	spin_lock_bh(&host->lock);
 
-	dw_mci_queue_request(host, slot, mrq);
+	dw_mci_queue_request(host, mrq);
 
 	spin_unlock_bh(&host->lock);
 }
 
 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	const struct dw_mci_drv_data *drv_data = slot->host->drv_data;
+	struct dw_mci *host = mmc_priv(mmc);
+	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	u32 regs;
 	int ret;
 
 	switch (ios->bus_width) {
 	case MMC_BUS_WIDTH_4:
-		slot->ctype = SDMMC_CTYPE_4BIT;
+		host->ctype = SDMMC_CTYPE_4BIT;
 		break;
 	case MMC_BUS_WIDTH_8:
-		slot->ctype = SDMMC_CTYPE_8BIT;
+		host->ctype = SDMMC_CTYPE_8BIT;
 		break;
 	default:
 		/* set default 1 bit mode */
-		slot->ctype = SDMMC_CTYPE_1BIT;
+		host->ctype = SDMMC_CTYPE_1BIT;
 	}
 
-	regs = mci_readl(slot->host, UHS_REG);
+	regs = mci_readl(host, UHS_REG);
 
 	/* DDR mode set */
 	if (ios->timing == MMC_TIMING_MMC_DDR52 ||
 	    ios->timing == MMC_TIMING_UHS_DDR50 ||
 	    ios->timing == MMC_TIMING_MMC_HS400)
-		regs |= ((0x1 << slot->id) << 16);
+		regs |= BIT(16);
 	else
-		regs &= ~((0x1 << slot->id) << 16);
+		regs &= ~BIT(16);
 
-	mci_writel(slot->host, UHS_REG, regs);
-	slot->host->timing = ios->timing;
+	mci_writel(host, UHS_REG, regs);
+	host->timing = ios->timing;
 
 	/*
 	 * Use mirror of ios->clock to prevent race with mmc
 	 * core ios update when finding the minimum.
 	 */
-	slot->clock = ios->clock;
+	host->clock = ios->clock;
 
 	if (drv_data && drv_data->set_ios)
-		drv_data->set_ios(slot->host, ios);
+		drv_data->set_ios(host, ios);
 
 	switch (ios->power_mode) {
 	case MMC_POWER_UP:
-		if (!IS_ERR(mmc->supply.vmmc)) {
-			ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,
-					ios->vdd);
-			if (ret) {
-				dev_err(slot->host->dev,
-					"failed to enable vmmc regulator\n");
-				/*return, if failed turn on vmmc*/
-				return;
-			}
+		ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
+		if (ret) {
+			dev_err(host->dev, "failed to enable vmmc regulator\n");
+			return;
 		}
-		set_bit(DW_MMC_CARD_NEED_INIT, &slot->flags);
-		regs = mci_readl(slot->host, PWREN);
-		regs |= (1 << slot->id);
-		mci_writel(slot->host, PWREN, regs);
+		set_bit(DW_MMC_CARD_NEED_INIT, &host->flags);
+		regs = mci_readl(host, PWREN);
+		regs |= BIT(0);
+		mci_writel(host, PWREN, regs);
 		break;
 	case MMC_POWER_ON:
-		if (!slot->host->vqmmc_enabled) {
-			if (!IS_ERR(mmc->supply.vqmmc)) {
-				ret = regulator_enable(mmc->supply.vqmmc);
-				if (ret < 0)
-					dev_err(slot->host->dev,
-						"failed to enable vqmmc\n");
-				else
-					slot->host->vqmmc_enabled = true;
-
-			} else {
-				/* Keep track so we don't reset again */
-				slot->host->vqmmc_enabled = true;
-			}
-
-			/* Reset our state machine after powering on */
-			dw_mci_ctrl_reset(slot->host,
-					  SDMMC_CTRL_ALL_RESET_FLAGS);
-		}
-
+		mmc_regulator_enable_vqmmc(mmc);
 		/* Adjust clock / bus width after power is up */
-		dw_mci_setup_bus(slot, false);
+		dw_mci_setup_bus(host, false);
 
 		break;
 	case MMC_POWER_OFF:
 		/* Turn clock off before power goes down */
-		dw_mci_setup_bus(slot, false);
+		dw_mci_setup_bus(host, false);
 
 		if (!IS_ERR(mmc->supply.vmmc))
 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
 
-		if (!IS_ERR(mmc->supply.vqmmc) && slot->host->vqmmc_enabled)
-			regulator_disable(mmc->supply.vqmmc);
-		slot->host->vqmmc_enabled = false;
+		mmc_regulator_disable_vqmmc(mmc);
 
-		regs = mci_readl(slot->host, PWREN);
-		regs &= ~(1 << slot->id);
-		mci_writel(slot->host, PWREN, regs);
+		regs = mci_readl(host, PWREN);
+		regs &= ~BIT(0);
+		mci_writel(host, PWREN, regs);
+		/* Reset our state machine after powering off */
+		dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS);
 		break;
 	default:
 		break;
 	}
 
-	if (slot->host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)
-		slot->host->state = STATE_IDLE;
+	if (host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)
+		host->state = STATE_IDLE;
 }
 
 static int dw_mci_card_busy(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	u32 status;
 
 	/*
 	 * Check the busy bit which is low when DAT[3:0]
 	 * (the data lines) are 0000
 	 */
-	status = mci_readl(slot->host, STATUS);
+	status = mci_readl(host, STATUS);
 
 	return !!(status & SDMMC_STATUS_BUSY);
 }
 
 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	u32 uhs;
-	u32 v18 = SDMMC_UHS_18V << slot->id;
+	u32 v18 = SDMMC_UHS_18V;
 	int ret;
 
 	if (drv_data && drv_data->switch_voltage)
-		return drv_data->switch_voltage(mmc, ios);
+		return drv_data->switch_voltage(host, ios);
 
 	/*
 	 * Program the voltage.  Note that some instances of dw_mmc may use
@@ -1597,7 +1478,7 @@ static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 static int dw_mci_get_ro(struct mmc_host *mmc)
 {
 	int read_only;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	int gpio_ro = mmc_gpio_get_ro(mmc);
 
 	/* Use platform get_ro function, else try on board write protect */
@@ -1605,7 +1486,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc)
 		read_only = gpio_ro;
 	else
 		read_only =
-			mci_readl(slot->host, WRTPRT) & (1 << slot->id) ? 1 : 0;
+			mci_readl(host, WRTPRT) & BIT(0) ? 1 : 0;
 
 	dev_dbg(&mmc->class_dev, "card is %s\n",
 		read_only ? "read-only" : "read-write");
@@ -1615,8 +1496,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc)
 
 static void dw_mci_hw_reset(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int reset;
 
@@ -1639,18 +1519,17 @@ static void dw_mci_hw_reset(struct mmc_host *mmc)
 	 * tRSTH >= 1us:   RST_n high period
 	 */
 	reset = mci_readl(host, RST_N);
-	reset &= ~(SDMMC_RST_HWACTIVE << slot->id);
+	reset &= ~SDMMC_RST_HWACTIVE;
 	mci_writel(host, RST_N, reset);
 	usleep_range(1, 2);
-	reset |= SDMMC_RST_HWACTIVE << slot->id;
+	reset |= SDMMC_RST_HWACTIVE;
 	mci_writel(host, RST_N, reset);
 	usleep_range(200, 300);
 }
 
-static void dw_mci_prepare_sdio_irq(struct dw_mci_slot *slot, bool prepare)
+static void dw_mci_prepare_sdio_irq(struct dw_mci *host, bool prepare)
 {
-	struct dw_mci *host = slot->host;
-	const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR << slot->id;
+	const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR;
 	u32 clk_en_a_old;
 	u32 clk_en_a;
 
@@ -1662,23 +1541,22 @@ static void dw_mci_prepare_sdio_irq(struct dw_mci_slot *slot, bool prepare)
 
 	clk_en_a_old = mci_readl(host, CLKENA);
 	if (prepare) {
-		set_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
+		set_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags);
 		clk_en_a = clk_en_a_old & ~clken_low_pwr;
 	} else {
-		clear_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
+		clear_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags);
 		clk_en_a = clk_en_a_old | clken_low_pwr;
 	}
 
 	if (clk_en_a != clk_en_a_old) {
 		mci_writel(host, CLKENA, clk_en_a);
-		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT,
+		mci_send_cmd(host, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT,
 			     0);
 	}
 }
 
-static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
+static void __dw_mci_enable_sdio_irq(struct dw_mci *host, int enb)
 {
-	struct dw_mci *host = slot->host;
 	unsigned long irqflags;
 	u32 int_mask;
 
@@ -1687,9 +1565,9 @@ static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
 	/* Enable/disable Slot Specific SDIO interrupt */
 	int_mask = mci_readl(host, INTMASK);
 	if (enb)
-		int_mask |= SDMMC_INT_SDIO(slot->sdio_id);
+		int_mask |= SDMMC_INT_SDIO(host->sdio_irq);
 	else
-		int_mask &= ~SDMMC_INT_SDIO(slot->sdio_id);
+		int_mask &= ~SDMMC_INT_SDIO(host->sdio_irq);
 	mci_writel(host, INTMASK, int_mask);
 
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
@@ -1697,11 +1575,10 @@ static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
 
 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 
-	dw_mci_prepare_sdio_irq(slot, enb);
-	__dw_mci_enable_sdio_irq(slot, enb);
+	dw_mci_prepare_sdio_irq(host, enb);
+	__dw_mci_enable_sdio_irq(host, enb);
 
 	/* Avoid runtime suspending the device when SDIO IRQ is enabled */
 	if (enb)
@@ -1712,28 +1589,26 @@ static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)
 
 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 
-	__dw_mci_enable_sdio_irq(slot, 1);
+	__dw_mci_enable_sdio_irq(host, 1);
 }
 
 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int err = -EINVAL;
 
 	if (drv_data && drv_data->execute_tuning)
-		err = drv_data->execute_tuning(slot, opcode);
+		err = drv_data->execute_tuning(host, opcode);
 	return err;
 }
 
 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc,
 				       struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 
 	if (drv_data && drv_data->prepare_hs400_tuning)
@@ -1804,7 +1679,7 @@ static bool dw_mci_reset(struct dw_mci *host)
 
 ciu_out:
 	/* After a CTRL reset we need to have CIU set clock registers  */
-	mci_send_cmd(host->slot, SDMMC_CMD_UPD_CLK, 0);
+	mci_send_cmd(host, SDMMC_CMD_UPD_CLK, 0);
 
 	return ret;
 }
@@ -1895,29 +1770,16 @@ static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
 	__releases(&host->lock)
 	__acquires(&host->lock)
 {
-	struct dw_mci_slot *slot;
-	struct mmc_host	*prev_mmc = host->slot->mmc;
+	struct mmc_host	*prev_mmc = host->mmc;
 
 	WARN_ON(host->cmd || host->data);
 
-	host->slot->mrq = NULL;
 	host->mrq = NULL;
-	if (!list_empty(&host->queue)) {
-		slot = list_entry(host->queue.next,
-				  struct dw_mci_slot, queue_node);
-		list_del(&slot->queue_node);
-		dev_vdbg(host->dev, "list not empty: %s is next\n",
-			 mmc_hostname(slot->mmc));
-		host->state = STATE_SENDING_CMD;
-		dw_mci_start_request(host, slot);
-	} else {
-		dev_vdbg(host->dev, "list empty\n");
 
-		if (host->state == STATE_SENDING_CMD11)
-			host->state = STATE_WAITING_CMD11_DONE;
-		else
-			host->state = STATE_IDLE;
-	}
+	if (host->state == STATE_SENDING_CMD11)
+		host->state = STATE_WAITING_CMD11_DONE;
+	else
+		host->state = STATE_IDLE;
 
 	spin_unlock(&host->lock);
 	mmc_request_done(prev_mmc, mrq);
@@ -1968,7 +1830,7 @@ static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)
 			data->error = -EILSEQ;
 		} else if (status & SDMMC_INT_EBE) {
 			if (host->dir_status ==
-				DW_MCI_SEND_STATUS) {
+				MMC_DATA_WRITE) {
 				/*
 				 * No data CRC status was returned.
 				 * The number of bytes transferred
@@ -1977,7 +1839,7 @@ static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)
 				data->bytes_xfered = 0;
 				data->error = -ETIMEDOUT;
 			} else if (host->dir_status ==
-					DW_MCI_RECV_STATUS) {
+					MMC_DATA_READ) {
 				data->error = -EILSEQ;
 			}
 		} else {
@@ -2095,8 +1957,7 @@ static void dw_mci_work_func(struct work_struct *t)
 			set_bit(EVENT_CMD_COMPLETE, &host->completed_events);
 			err = dw_mci_command_complete(host, cmd);
 			if (cmd == mrq->sbc && !err) {
-				__dw_mci_start_request(host, host->slot,
-						       mrq->cmd);
+				__dw_mci_start_request(host, mrq->cmd);
 				goto unlock;
 			}
 
@@ -2123,7 +1984,7 @@ static void dw_mci_work_func(struct work_struct *t)
 				 * avoids races and keeps things simple.
 				 */
 				if (err != -ETIMEDOUT &&
-				    host->dir_status == DW_MCI_RECV_STATUS) {
+				    host->dir_status == MMC_DATA_READ) {
 					state = STATE_SENDING_DATA;
 					continue;
 				}
@@ -2167,7 +2028,7 @@ static void dw_mci_work_func(struct work_struct *t)
 				 * If all data-related interrupts don't come
 				 * within the given time in reading data state.
 				 */
-				if (host->dir_status == DW_MCI_RECV_STATUS)
+				if (host->dir_status == MMC_DATA_READ)
 					dw_mci_set_drto(host);
 				break;
 			}
@@ -2207,7 +2068,7 @@ static void dw_mci_work_func(struct work_struct *t)
 				 * interrupt doesn't come within the given time.
 				 * in reading data state.
 				 */
-				if (host->dir_status == DW_MCI_RECV_STATUS)
+				if (host->dir_status == MMC_DATA_READ)
 					dw_mci_set_drto(host);
 				break;
 			}
@@ -2803,17 +2664,14 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
 
 static void dw_mci_handle_cd(struct dw_mci *host)
 {
-	struct dw_mci_slot *slot = host->slot;
-
-	mmc_detect_change(slot->mmc,
-		msecs_to_jiffies(host->pdata->detect_delay_ms));
+	mmc_detect_change(host->mmc,
+		msecs_to_jiffies(host->detect_delay_ms));
 }
 
 static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 {
 	struct dw_mci *host = dev_id;
 	u32 pending;
-	struct dw_mci_slot *slot = host->slot;
 
 	pending = mci_readl(host, MINTSTS); /* read-only mask reg */
 
@@ -2878,7 +2736,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 			if (!host->data_status)
 				host->data_status = pending;
 			smp_wmb(); /* drain writebuffer */
-			if (host->dir_status == DW_MCI_RECV_STATUS) {
+			if (host->dir_status == MMC_DATA_READ) {
 				if (host->sg != NULL)
 					dw_mci_read_data_pio(host, true);
 			}
@@ -2890,13 +2748,13 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 
 		if (pending & SDMMC_INT_RXDR) {
 			mci_writel(host, RINTSTS, SDMMC_INT_RXDR);
-			if (host->dir_status == DW_MCI_RECV_STATUS && host->sg)
+			if (host->dir_status == MMC_DATA_READ && host->sg)
 				dw_mci_read_data_pio(host, false);
 		}
 
 		if (pending & SDMMC_INT_TXDR) {
 			mci_writel(host, RINTSTS, SDMMC_INT_TXDR);
-			if (host->dir_status == DW_MCI_SEND_STATUS && host->sg)
+			if (host->dir_status == MMC_DATA_WRITE && host->sg)
 				dw_mci_write_data_pio(host);
 		}
 
@@ -2914,11 +2772,11 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 			dw_mci_handle_cd(host);
 		}
 
-		if (pending & SDMMC_INT_SDIO(slot->sdio_id)) {
+		if (pending & SDMMC_INT_SDIO(host->sdio_irq)) {
 			mci_writel(host, RINTSTS,
-				   SDMMC_INT_SDIO(slot->sdio_id));
-			__dw_mci_enable_sdio_irq(slot, 0);
-			sdio_signal_irq(slot->mmc);
+				   SDMMC_INT_SDIO(host->sdio_irq));
+			__dw_mci_enable_sdio_irq(host, 0);
+			sdio_signal_irq(host->mmc);
 		}
 
 	}
@@ -2950,29 +2808,19 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
+static int dw_mci_init_host_caps(struct dw_mci *host)
 {
-	struct dw_mci *host = slot->host;
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int ctrl_id;
 
-	if (host->pdata->caps)
-		mmc->caps = host->pdata->caps;
-
-	if (host->pdata->pm_caps)
-		mmc->pm_caps = host->pdata->pm_caps;
-
 	if (drv_data)
 		mmc->caps |= drv_data->common_caps;
 
-	if (host->dev->of_node) {
-		ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
-		if (ctrl_id < 0)
-			ctrl_id = 0;
-	} else {
+	if (host->dev->of_node)
+		ctrl_id = mmc->index;
+	else
 		ctrl_id = to_platform_device(host->dev)->id;
-	}
 
 	if (drv_data && drv_data->caps) {
 		if (ctrl_id >= drv_data->num_caps) {
@@ -2983,9 +2831,6 @@ static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
 		mmc->caps |= drv_data->caps[ctrl_id];
 	}
 
-	if (host->pdata->caps2)
-		mmc->caps2 = host->pdata->caps2;
-
 	/* if host has set a minimum_freq, we should respect it */
 	if (host->minimum_speed)
 		mmc->f_min = host->minimum_speed;
@@ -3002,23 +2847,11 @@ static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
 	return 0;
 }
 
-static int dw_mci_init_slot(struct dw_mci *host)
+static int dw_mci_init_host(struct dw_mci *host)
 {
-	struct mmc_host *mmc;
-	struct dw_mci_slot *slot;
+	struct mmc_host *mmc = host->mmc;
 	int ret;
 
-	mmc = devm_mmc_alloc_host(host->dev, sizeof(*slot));
-	if (!mmc)
-		return -ENOMEM;
-
-	slot = mmc_priv(mmc);
-	slot->id = 0;
-	slot->sdio_id = host->sdio_id0 + slot->id;
-	slot->mmc = mmc;
-	slot->host = host;
-	host->slot = slot;
-
 	mmc->ops = &dw_mci_ops;
 
 	/*if there are external regulators, get them*/
@@ -3033,7 +2866,9 @@ static int dw_mci_init_slot(struct dw_mci *host)
 	if (ret)
 		return ret;
 
-	ret = dw_mci_init_slot_caps(slot);
+	mmc_of_parse_clk_phase(host->dev, &host->phase_map);
+
+	ret = dw_mci_init_host_caps(host);
 	if (ret)
 		return ret;
 
@@ -3061,6 +2896,11 @@ static int dw_mci_init_slot(struct dw_mci *host)
 		mmc->max_seg_size = mmc->max_req_size;
 	}
 
+	if (mmc->caps & MMC_CAP_NEEDS_POLL)
+		dev_info(&mmc->class_dev, "card is polling.\n");
+	else if (!mmc_card_is_removable(mmc))
+		dev_info(&mmc->class_dev, "card is non-removable.\n");
+
 	dw_mci_get_cd(mmc);
 
 	ret = mmc_add_host(mmc);
@@ -3068,17 +2908,16 @@ static int dw_mci_init_slot(struct dw_mci *host)
 		return ret;
 
 #if defined(CONFIG_DEBUG_FS)
-	dw_mci_init_debugfs(slot);
+	dw_mci_init_debugfs(host);
 #endif
 
 	return 0;
 }
 
-static void dw_mci_cleanup_slot(struct dw_mci_slot *slot)
+static void dw_mci_cleanup_host(struct dw_mci *host)
 {
 	/* Debugfs stuff is cleaned up by mmc core */
-	mmc_remove_host(slot->mmc);
-	slot->host->slot = NULL;
+	mmc_remove_host(host->mmc);
 }
 
 static void dw_mci_init_dma(struct dw_mci *host)
@@ -3291,54 +3130,43 @@ static void dw_mci_dto_timer(struct timer_list *t)
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
 }
 
-#ifdef CONFIG_OF
-static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
+static int dw_mci_parse_dt(struct dw_mci *host)
 {
-	struct dw_mci_board *pdata;
 	struct device *dev = host->dev;
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int ret;
 	u32 clock_frequency;
 
-	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata)
-		return ERR_PTR(-ENOMEM);
-
 	/* find reset controller when exist */
-	pdata->rstc = devm_reset_control_get_optional_exclusive(dev, "reset");
-	if (IS_ERR(pdata->rstc))
-		return ERR_CAST(pdata->rstc);
+	host->rstc = devm_reset_control_get_optional_exclusive(dev, "reset");
+	if (IS_ERR(host->rstc))
+		return PTR_ERR(host->rstc);
 
-	if (device_property_read_u32(dev, "fifo-depth", &pdata->fifo_depth))
+	if (!host->fifo_depth && device_property_read_u32(dev, "fifo-depth", &host->fifo_depth))
 		dev_info(dev,
 			 "fifo-depth property not found, using value of FIFOTH register as default\n");
 
-	device_property_read_u32(dev, "card-detect-delay",
-				 &pdata->detect_delay_ms);
+	if (!host->detect_delay_ms)
+		device_property_read_u32(dev, "card-detect-delay",
+					 &host->detect_delay_ms);
 
-	device_property_read_u32(dev, "data-addr", &host->data_addr_override);
+	if (!host->data_addr_override)
+		device_property_read_u32(dev, "data-addr", &host->data_addr_override);
 
 	if (device_property_present(dev, "fifo-watermark-aligned"))
 		host->wm_aligned = true;
 
-	if (!device_property_read_u32(dev, "clock-frequency", &clock_frequency))
-		pdata->bus_hz = clock_frequency;
+	if (!host->bus_hz && !device_property_read_u32(dev, "clock-frequency", &clock_frequency))
+		host->bus_hz = clock_frequency;
 
 	if (drv_data && drv_data->parse_dt) {
 		ret = drv_data->parse_dt(host);
 		if (ret)
-			return ERR_PTR(ret);
+			return ret;
 	}
 
-	return pdata;
-}
-
-#else /* CONFIG_OF */
-static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
-{
-	return ERR_PTR(-EINVAL);
+	return 0;
 }
-#endif /* CONFIG_OF */
 
 static void dw_mci_enable_cd(struct dw_mci *host)
 {
@@ -3346,13 +3174,13 @@ static void dw_mci_enable_cd(struct dw_mci *host)
 	u32 temp;
 
 	/*
-	 * No need for CD if all slots have a non-error GPIO
+	 * No need for CD if host has a non-error GPIO
 	 * as well as broken card detection is found.
 	 */
-	if (host->slot->mmc->caps & MMC_CAP_NEEDS_POLL)
+	if (host->mmc->caps & MMC_CAP_NEEDS_POLL)
 		return;
 
-	if (mmc_gpio_get_cd(host->slot->mmc) < 0) {
+	if (mmc_gpio_get_cd(host->mmc) < 0) {
 		spin_lock_irqsave(&host->irq_lock, irqflags);
 		temp = mci_readl(host, INTMASK);
 		temp  |= SDMMC_INT_CD;
@@ -3361,18 +3189,32 @@ static void dw_mci_enable_cd(struct dw_mci *host)
 	}
 }
 
+struct dw_mci *dw_mci_alloc_host(struct device *dev)
+{
+	struct mmc_host *mmc;
+	struct dw_mci *host;
+
+	mmc = devm_mmc_alloc_host(dev, sizeof(struct dw_mci));
+	if (!mmc)
+		return ERR_PTR(-ENOMEM);
+
+	host = mmc_priv(mmc);
+	host->mmc = mmc;
+	host->dev = dev;
+
+	return host;
+}
+EXPORT_SYMBOL(dw_mci_alloc_host);
+
 int dw_mci_probe(struct dw_mci *host)
 {
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int width, i, ret = 0;
 	u32 fifo_size;
 
-	if (!host->pdata) {
-		host->pdata = dw_mci_parse_dt(host);
-		if (IS_ERR(host->pdata))
-			return dev_err_probe(host->dev, PTR_ERR(host->pdata),
-					     "platform data not available\n");
-	}
+	ret = dw_mci_parse_dt(host);
+	if (ret)
+		return dev_err_probe(host->dev, ret, "parse dt failed\n");
 
 	host->biu_clk = devm_clk_get(host->dev, "biu");
 	if (IS_ERR(host->biu_clk)) {
@@ -3395,8 +3237,6 @@ int dw_mci_probe(struct dw_mci *host)
 		ret = PTR_ERR(host->ciu_clk);
 		if (ret == -EPROBE_DEFER)
 			goto err_clk_biu;
-
-		host->bus_hz = host->pdata->bus_hz;
 	} else {
 		ret = clk_prepare_enable(host->ciu_clk);
 		if (ret) {
@@ -3404,12 +3244,12 @@ int dw_mci_probe(struct dw_mci *host)
 			goto err_clk_biu;
 		}
 
-		if (host->pdata->bus_hz) {
-			ret = clk_set_rate(host->ciu_clk, host->pdata->bus_hz);
+		if (host->bus_hz) {
+			ret = clk_set_rate(host->ciu_clk, host->bus_hz);
 			if (ret)
 				dev_warn(host->dev,
 					 "Unable to set bus rate to %uHz\n",
-					 host->pdata->bus_hz);
+					 host->bus_hz);
 		}
 		host->bus_hz = clk_get_rate(host->ciu_clk);
 	}
@@ -3421,10 +3261,10 @@ int dw_mci_probe(struct dw_mci *host)
 		goto err_clk_ciu;
 	}
 
-	if (host->pdata->rstc) {
-		reset_control_assert(host->pdata->rstc);
+	if (host->rstc) {
+		reset_control_assert(host->rstc);
 		usleep_range(10, 50);
-		reset_control_deassert(host->pdata->rstc);
+		reset_control_deassert(host->rstc);
 	}
 
 	if (drv_data && drv_data->init) {
@@ -3442,7 +3282,6 @@ int dw_mci_probe(struct dw_mci *host)
 
 	spin_lock_init(&host->lock);
 	spin_lock_init(&host->irq_lock);
-	INIT_LIST_HEAD(&host->queue);
 
 	dw_mci_init_fault(host);
 
@@ -3483,7 +3322,6 @@ int dw_mci_probe(struct dw_mci *host)
 		goto err_clk_ciu;
 	}
 
-	host->dma_ops = host->pdata->dma_ops;
 	dw_mci_init_dma(host);
 
 	/* Clear the interrupts for the host controller */
@@ -3497,7 +3335,7 @@ int dw_mci_probe(struct dw_mci *host)
 	 * FIFO threshold settings  RxMark  = fifo_size / 2 - 1,
 	 *                          Tx Mark = fifo_size / 2 DMA Size = 8
 	 */
-	if (!host->pdata->fifo_depth) {
+	if (!host->fifo_depth) {
 		/*
 		 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may
 		 * have been overwritten by the bootloader, just like we're
@@ -3507,7 +3345,7 @@ int dw_mci_probe(struct dw_mci *host)
 		fifo_size = mci_readl(host, FIFOTH);
 		fifo_size = 1 + ((fifo_size >> 16) & 0xfff);
 	} else {
-		fifo_size = host->pdata->fifo_depth;
+		fifo_size = host->fifo_depth;
 	}
 	host->fifo_depth = fifo_size;
 	host->fifoth_val =
@@ -3552,14 +3390,13 @@ int dw_mci_probe(struct dw_mci *host)
 		 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n",
 		 host->irq, width, fifo_size);
 
-	/* We need at least one slot to succeed */
-	ret = dw_mci_init_slot(host);
+	ret = dw_mci_init_host(host);
 	if (ret) {
-		dev_dbg(host->dev, "slot %d init failed\n", i);
+		dev_dbg(host->dev, "host init failed\n");
 		goto err_dmaunmap;
 	}
 
-	/* Now that slots are all setup, we can enable card detect */
+	/* Now that host is setup, we can enable card detect */
 	dw_mci_enable_cd(host);
 
 	return 0;
@@ -3568,7 +3405,7 @@ int dw_mci_probe(struct dw_mci *host)
 	if (host->use_dma && host->dma_ops->exit)
 		host->dma_ops->exit(host);
 
-	reset_control_assert(host->pdata->rstc);
+	reset_control_assert(host->rstc);
 
 err_clk_ciu:
 	clk_disable_unprepare(host->ciu_clk);
@@ -3582,9 +3419,8 @@ EXPORT_SYMBOL(dw_mci_probe);
 
 void dw_mci_remove(struct dw_mci *host)
 {
-	dev_dbg(host->dev, "remove slot\n");
-	if (host->slot)
-		dw_mci_cleanup_slot(host->slot);
+	dev_dbg(host->dev, "remove host\n");
+	dw_mci_cleanup_host(host);
 
 	mci_writel(host, RINTSTS, 0xFFFFFFFF);
 	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */
@@ -3596,16 +3432,13 @@ void dw_mci_remove(struct dw_mci *host)
 	if (host->use_dma && host->dma_ops->exit)
 		host->dma_ops->exit(host);
 
-	reset_control_assert(host->pdata->rstc);
+	reset_control_assert(host->rstc);
 
 	clk_disable_unprepare(host->ciu_clk);
 	clk_disable_unprepare(host->biu_clk);
 }
 EXPORT_SYMBOL(dw_mci_remove);
 
-
-
-#ifdef CONFIG_PM
 int dw_mci_runtime_suspend(struct device *dev)
 {
 	struct dw_mci *host = dev_get_drvdata(dev);
@@ -3615,9 +3448,8 @@ int dw_mci_runtime_suspend(struct device *dev)
 
 	clk_disable_unprepare(host->ciu_clk);
 
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc)))
+	if (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc))
 		clk_disable_unprepare(host->biu_clk);
 
 	return 0;
@@ -3629,9 +3461,8 @@ int dw_mci_runtime_resume(struct device *dev)
 	int ret = 0;
 	struct dw_mci *host = dev_get_drvdata(dev);
 
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc))) {
+	if (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc)) {
 		ret = clk_prepare_enable(host->biu_clk);
 		if (ret)
 			return ret;
@@ -3647,8 +3478,11 @@ int dw_mci_runtime_resume(struct device *dev)
 		goto err;
 	}
 
-	if (host->use_dma && host->dma_ops->init)
-		host->dma_ops->init(host);
+	if (host->use_dma && host->dma_ops->init) {
+		ret = host->dma_ops->init(host);
+		if (ret)
+			return ret;
+	}
 
 	/*
 	 * Restore the initial value at FIFOTH register
@@ -3667,44 +3501,35 @@ int dw_mci_runtime_resume(struct device *dev)
 	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);
 
 
-	if (host->slot && host->slot->mmc->pm_flags & MMC_PM_KEEP_POWER)
-		dw_mci_set_ios(host->slot->mmc, &host->slot->mmc->ios);
+	if (host->mmc->pm_flags & MMC_PM_KEEP_POWER)
+		dw_mci_set_ios(host->mmc, &host->mmc->ios);
 
 	/* Force setup bus to guarantee available clock output */
-	dw_mci_setup_bus(host->slot, true);
+	dw_mci_setup_bus(host, true);
 
 	/* Re-enable SDIO interrupts. */
-	if (sdio_irq_claimed(host->slot->mmc))
-		__dw_mci_enable_sdio_irq(host->slot, 1);
+	if (sdio_irq_claimed(host->mmc))
+		__dw_mci_enable_sdio_irq(host, 1);
 
-	/* Now that slots are all setup, we can enable card detect */
+	/* Now that host is setup, we can enable card detect */
 	dw_mci_enable_cd(host);
 
 	return 0;
 
 err:
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc)))
+	if (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc))
 		clk_disable_unprepare(host->biu_clk);
 
 	return ret;
 }
 EXPORT_SYMBOL(dw_mci_runtime_resume);
-#endif /* CONFIG_PM */
-
-static int __init dw_mci_init(void)
-{
-	pr_info("Synopsys Designware Multimedia Card Interface Driver\n");
-	return 0;
-}
 
-static void __exit dw_mci_exit(void)
-{
-}
-
-module_init(dw_mci_init);
-module_exit(dw_mci_exit);
+const struct dev_pm_ops dw_mci_pmops = {
+	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL)
+};
+EXPORT_SYMBOL_GPL(dw_mci_pmops);
 
 MODULE_DESCRIPTION("DW Multimedia Card Interface driver");
 MODULE_AUTHOR("NXP Semiconductor VietNam");
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 648b4a5641bf2..42e58be74ce09 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -43,8 +43,6 @@ enum dw_mci_cookie {
 	COOKIE_MAPPED,		/* mapped by prepare_data() of dwmmc */
 };
 
-struct mmc_data;
-
 enum {
 	TRANS_MODE_PIO = 0,
 	TRANS_MODE_IDMAC,
@@ -57,14 +55,14 @@ struct dw_mci_dma_slave {
 };
 
 /**
- * struct dw_mci - MMC controller state shared between all slots
+ * struct dw_mci - MMC controller state
  * @lock: Spinlock protecting the queue and associated data.
  * @irq_lock: Spinlock protecting the INTMASK setting.
  * @regs: Pointer to MMIO registers.
  * @fifo_reg: Pointer to MMIO registers for data FIFO
  * @sg: Scatterlist entry currently being processed by PIO code, if any.
  * @sg_miter: PIO mapping scatterlist iterator.
- * @mrq: The request currently being processed on @slot,
+ * @mrq: The request currently being processed on @host,
  *	or NULL if the controller is idle.
  * @cmd: The command currently being sent to the card, or NULL.
  * @data: The data currently being transferred, or NULL if no data
@@ -78,7 +76,7 @@ struct dw_mci_dma_slave {
  * @dma_64bit_address: Whether DMA supports 64-bit address mode or not.
  * @sg_dma: Bus address of DMA buffer.
  * @sg_cpu: Virtual address of DMA buffer.
- * @dma_ops: Pointer to platform-specific DMA callbacks.
+ * @dma_ops: Pointer to DMA callbacks.
  * @cmd_status: Snapshot of SR taken upon completion of the current
  * @ring_size: Buffer size for idma descriptors.
  *	command. Only valid when EVENT_CMD_COMPLETE is pending.
@@ -96,7 +94,6 @@ struct dw_mci_dma_slave {
  * @completed_events: Bitmask of events which the state machine has
  *	processed.
  * @state: BH work state.
- * @queue: List of slots waiting for access to the controller.
  * @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus
  *	rate and timeout calculations.
  * @current_speed: Configured rate of the controller.
@@ -104,12 +101,10 @@ struct dw_mci_dma_slave {
  * @fifoth_val: The value of FIFOTH register.
  * @verid: Denote Version ID.
  * @dev: Device associated with the MMC controller.
- * @pdata: Platform data associated with the MMC controller.
  * @drv_data: Driver specific data for identified variant of the controller
  * @priv: Implementation defined private data.
  * @biu_clk: Pointer to bus interface unit clock instance.
  * @ciu_clk: Pointer to card interface unit clock instance.
- * @slot: Slots sharing this MMC controller.
  * @fifo_depth: depth of FIFO.
  * @data_addr_override: override fifo reg offset with this value.
  * @wm_aligned: force fifo watermark equal with data length in PIO mode.
@@ -121,23 +116,28 @@ struct dw_mci_dma_slave {
  * @push_data: Pointer to FIFO push function.
  * @pull_data: Pointer to FIFO pull function.
  * @quirks: Set of quirks that apply to specific versions of the IP.
- * @vqmmc_enabled: Status of vqmmc, should be true or false.
  * @irq_flags: The flags to be passed to request_irq.
  * @irq: The irq value to be passed to request_irq.
- * @sdio_id0: Number of slot0 in the SDIO interrupt registers.
+ * @sdio_irq: SDIO interrupt bit in interrupt registers.
  * @cmd11_timer: Timer for SD3.0 voltage switch over scheme.
  * @cto_timer: Timer for broken command transfer over scheme.
  * @dto_timer: Timer for broken data transfer over scheme.
+ * @mmc: The mmc_host representing this dw_mci.
+ * @flags: Random state bits associated with the host.
+ * @ctype: Card type for this host.
+ * @clock: Clock rate configured by set_ios(). Protected by host->lock.
+ * @clk_old: The last clock value that was requested from core.
+ * @pdev: platform_device registered
+ * @rstc: Reset controller for this host.
+ * @detect_delay_ms: Delay in mS before detecting cards after interrupt.
+ * @phase_map: The map for recording in and out phases for each timing
  *
  * Locking
  * =======
  *
- * @lock is a softirq-safe spinlock protecting @queue as well as
- * @slot, @mrq and @state. These must always be updated
+ * @lock is a softirq-safe spinlock protecting as well as
+ * @mrq and @state. These must always be updated
  * at the same time while holding @lock.
- * The @mrq field of struct dw_mci_slot is also protected by @lock,
- * and must always be written at the same time as the slot is added to
- * @queue.
  *
  * @irq_lock is an irq-safe spinlock protecting the INTMASK register
  * to allow the interrupt handler to modify it directly.  Held for only long
@@ -199,7 +199,6 @@ struct dw_mci {
 	unsigned long		pending_events;
 	unsigned long		completed_events;
 	enum dw_mci_state	state;
-	struct list_head	queue;
 
 	u32			bus_hz;
 	u32			current_speed;
@@ -207,7 +206,6 @@ struct dw_mci {
 	u32			fifoth_val;
 	u16			verid;
 	struct device		*dev;
-	struct dw_mci_board	*pdata;
 	const struct dw_mci_drv_data	*drv_data;
 	void			*priv;
 	struct clk		*biu_clk;
@@ -228,11 +226,10 @@ struct dw_mci {
 	void (*pull_data)(struct dw_mci *host, void *buf, int cnt);
 
 	u32			quirks;
-	bool			vqmmc_enabled;
 	unsigned long		irq_flags; /* IRQ flags */
 	int			irq;
 
-	int			sdio_id0;
+	int			sdio_irq;
 
 	struct timer_list       cmd11_timer;
 	struct timer_list       cto_timer;
@@ -242,6 +239,19 @@ struct dw_mci {
 	struct fault_attr	fail_data_crc;
 	struct hrtimer		fault_timer;
 #endif
+	struct mmc_host		*mmc;
+	unsigned long		flags;
+#define DW_MMC_CARD_NEED_INIT	0
+#define DW_MMC_CARD_NO_LOW_PWR	1
+#define DW_MMC_CARD_NO_USE_HOLD 2
+#define DW_MMC_CARD_NEEDS_POLL	3
+	u32			ctype;
+	unsigned int		clock;
+	unsigned int		clk_old;
+	struct platform_device	*pdev;
+	struct reset_control *rstc;
+	u32 detect_delay_ms;
+	struct mmc_clk_phase_map phase_map;
 };
 
 /* DMA ops for Internal/External DMAC interface */
@@ -255,30 +265,6 @@ struct dw_mci_dma_ops {
 	void (*exit)(struct dw_mci *host);
 };
 
-struct dma_pdata;
-
-/* Board platform data */
-struct dw_mci_board {
-	unsigned int bus_hz; /* Clock speed at the cclk_in pad */
-
-	u32 caps;	/* Capabilities */
-	u32 caps2;	/* More capabilities */
-	u32 pm_caps;	/* PM capabilities */
-	/*
-	 * Override fifo depth. If 0, autodetect it from the FIFOTH register,
-	 * but note that this may not be reliable after a bootloader has used
-	 * it.
-	 */
-	unsigned int fifo_depth;
-
-	/* delay in mS before detecting cards after interrupt */
-	u32 detect_delay_ms;
-
-	struct reset_control *rstc;
-	struct dw_mci_dma_ops *dma_ops;
-	struct dma_pdata *data;
-};
-
 /* Support for longer data read timeout */
 #define DW_MMC_QUIRK_EXTENDED_TMOUT            BIT(0)
 /* Force 32-bit access to the FIFO */
@@ -396,7 +382,6 @@ struct dw_mci_board {
 #define SDMMC_INT_CMD_DONE		BIT(2)
 #define SDMMC_INT_RESP_ERR		BIT(1)
 #define SDMMC_INT_CD			BIT(0)
-#define SDMMC_INT_ERROR			0xbfc2
 /* Command register defines */
 #define SDMMC_CMD_START			BIT(31)
 #define SDMMC_CMD_USE_HOLD_REG	BIT(29)
@@ -505,84 +490,17 @@ static inline void mci_fifo_l_writeq(void __iomem *addr, u64 value)
 #define mci_writel(dev, reg, value)			\
 	writel_relaxed((value), (dev)->regs + SDMMC_##reg)
 
-/* 16-bit FIFO access macros */
-#define mci_readw(dev, reg)			\
-	readw_relaxed((dev)->regs + SDMMC_##reg)
-#define mci_writew(dev, reg, value)			\
-	writew_relaxed((value), (dev)->regs + SDMMC_##reg)
-
-/* 64-bit FIFO access macros */
-#ifdef readq
-#define mci_readq(dev, reg)			\
-	readq_relaxed((dev)->regs + SDMMC_##reg)
-#define mci_writeq(dev, reg, value)			\
-	writeq_relaxed((value), (dev)->regs + SDMMC_##reg)
-#else
-/*
- * Dummy readq implementation for architectures that don't define it.
- *
- * We would assume that none of these architectures would configure
- * the IP block with a 64bit FIFO width, so this code will never be
- * executed on those machines. Defining these macros here keeps the
- * rest of the code free from ifdefs.
- */
-#define mci_readq(dev, reg)			\
-	(*(volatile u64 __force *)((dev)->regs + SDMMC_##reg))
-#define mci_writeq(dev, reg, value)			\
-	(*(volatile u64 __force *)((dev)->regs + SDMMC_##reg) = (value))
-
+#ifndef readq
 #define __raw_writeq(__value, __reg) \
 	(*(volatile u64 __force *)(__reg) = (__value))
 #define __raw_readq(__reg) (*(volatile u64 __force *)(__reg))
 #endif
 
+extern struct dw_mci *dw_mci_alloc_host(struct device *device);
 extern int dw_mci_probe(struct dw_mci *host);
 extern void dw_mci_remove(struct dw_mci *host);
-#ifdef CONFIG_PM
 extern int dw_mci_runtime_suspend(struct device *device);
 extern int dw_mci_runtime_resume(struct device *device);
-#else
-static inline int dw_mci_runtime_suspend(struct device *device) { return -EOPNOTSUPP; }
-static inline int dw_mci_runtime_resume(struct device *device) { return -EOPNOTSUPP; }
-#endif
-
-/**
- * struct dw_mci_slot - MMC slot state
- * @mmc: The mmc_host representing this slot.
- * @host: The MMC controller this slot is using.
- * @ctype: Card type for this slot.
- * @mrq: mmc_request currently being processed or waiting to be
- *	processed, or NULL when the slot is idle.
- * @queue_node: List node for placing this node in the @queue list of
- *	&struct dw_mci.
- * @clock: Clock rate configured by set_ios(). Protected by host->lock.
- * @__clk_old: The last clock value that was requested from core.
- *	Keeping track of this helps us to avoid spamming the console.
- * @flags: Random state bits associated with the slot.
- * @id: Number of this slot.
- * @sdio_id: Number of this slot in the SDIO interrupt registers.
- */
-struct dw_mci_slot {
-	struct mmc_host		*mmc;
-	struct dw_mci		*host;
-
-	u32			ctype;
-
-	struct mmc_request	*mrq;
-	struct list_head	queue_node;
-
-	unsigned int		clock;
-	unsigned int		__clk_old;
-
-	unsigned long		flags;
-#define DW_MMC_CARD_PRESENT	0
-#define DW_MMC_CARD_NEED_INIT	1
-#define DW_MMC_CARD_NO_LOW_PWR	2
-#define DW_MMC_CARD_NO_USE_HOLD 3
-#define DW_MMC_CARD_NEEDS_POLL	4
-	int			id;
-	int			sdio_id;
-};
 
 /**
  * dw_mci driver data - dw-mshc implementation specific driver data.
@@ -609,10 +527,10 @@ struct dw_mci_drv_data {
 	int		(*init)(struct dw_mci *host);
 	void		(*set_ios)(struct dw_mci *host, struct mmc_ios *ios);
 	int		(*parse_dt)(struct dw_mci *host);
-	int		(*execute_tuning)(struct dw_mci_slot *slot, u32 opcode);
+	int		(*execute_tuning)(struct dw_mci *host, u32 opcode);
 	int		(*prepare_hs400_tuning)(struct dw_mci *host,
 						struct mmc_ios *ios);
-	int		(*switch_voltage)(struct mmc_host *mmc,
+	int		(*switch_voltage)(struct dw_mci *host,
 					  struct mmc_ios *ios);
 	void		(*set_data_timeout)(struct dw_mci *host,
 					  unsigned int timeout_ns);
diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c
index 6a0d0250d47b6..6a3c26b7c82d8 100644
--- a/drivers/mmc/host/jz4740_mmc.c
+++ b/drivers/mmc/host/jz4740_mmc.c
@@ -1052,7 +1052,7 @@ static int jz4740_mmc_probe(struct platform_device* pdev)
 	host = mmc_priv(mmc);
 
 	/* Default if no match is JZ4740 */
-	host->version = (enum jz4740_mmc_version)device_get_match_data(&pdev->dev);
+	host->version = (unsigned long)device_get_match_data(&pdev->dev);
 
 	ret = mmc_of_parse(mmc);
 	if (ret)
diff --git a/drivers/mmc/host/loongson2-mmc.c b/drivers/mmc/host/loongson2-mmc.c
index da3daab5f3d6d..f553e92fd9e54 100644
--- a/drivers/mmc/host/loongson2-mmc.c
+++ b/drivers/mmc/host/loongson2-mmc.c
@@ -189,6 +189,12 @@
 #define LOONGSON2_MMC_DLLVAL_TIMEOUT_US		4000
 #define LOONGSON2_MMC_TXFULL_TIMEOUT_US		500
 
+/*
+ * Due to a hardware design flaw, the Loongson-2K0300 may fail to recognize the
+ * CMD48 (SD_READ_EXTR_SINGLE) interrupt.
+ */
+#define LOONGSON2_MMC_CMD48_QUIRK	BIT(0)
+
 /* Loongson-2K1000 SDIO2 DMA routing register */
 #define LS2K1000_SDIO_DMA_MASK		GENMASK(17, 15)
 #define LS2K1000_DMA0_CONF		0x0
@@ -245,6 +251,7 @@ struct loongson2_mmc_host {
 };
 
 struct loongson2_mmc_pdata {
+	u32 flags;
 	const struct regmap_config *regmap_config;
 	void (*reorder_cmd_data)(struct loongson2_mmc_host *host, struct mmc_command *cmd);
 	void (*fix_data_timeout)(struct loongson2_mmc_host *host, struct mmc_command *cmd);
@@ -568,6 +575,12 @@ static void loongson2_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct loongson2_mmc_host *host = mmc_priv(mmc);
 
+	if ((host->pdata->flags & LOONGSON2_MMC_CMD48_QUIRK) &&
+	    mrq->cmd->opcode == SD_READ_EXTR_SINGLE) {
+		mmc_request_done(mmc, mrq);
+		return;
+	}
+
 	host->cmd_is_stop = 0;
 	host->mrq = mrq;
 	loongson2_mmc_send_request(mmc);
@@ -703,14 +716,6 @@ static int ls2k0500_mmc_set_external_dma(struct loongson2_mmc_host *host,
 	return 0;
 }
 
-static struct loongson2_mmc_pdata ls2k0500_mmc_pdata = {
-	.regmap_config		= &ls2k0500_mmc_regmap_config,
-	.reorder_cmd_data	= ls2k0500_mmc_reorder_cmd_data,
-	.setting_dma		= ls2k0500_mmc_set_external_dma,
-	.prepare_dma		= loongson2_mmc_prepare_external_dma,
-	.release_dma		= loongson2_mmc_release_external_dma,
-};
-
 static int ls2k1000_mmc_set_external_dma(struct loongson2_mmc_host *host,
 					 struct platform_device *pdev)
 {
@@ -735,14 +740,6 @@ static int ls2k1000_mmc_set_external_dma(struct loongson2_mmc_host *host,
 	return 0;
 }
 
-static struct loongson2_mmc_pdata ls2k1000_mmc_pdata = {
-	.regmap_config		= &ls2k0500_mmc_regmap_config,
-	.reorder_cmd_data	= ls2k0500_mmc_reorder_cmd_data,
-	.setting_dma		= ls2k1000_mmc_set_external_dma,
-	.prepare_dma		= loongson2_mmc_prepare_external_dma,
-	.release_dma		= loongson2_mmc_release_external_dma,
-};
-
 static const struct regmap_config ls2k2000_mmc_regmap_config = {
 	.reg_bits = 32,
 	.val_bits = 32,
@@ -846,7 +843,6 @@ static int ls2k2000_mmc_set_internal_dma(struct loongson2_mmc_host *host,
 	if (!host->sg_cpu)
 		return -ENOMEM;
 
-	memset(host->sg_cpu, 0, PAGE_SIZE);
 	return 0;
 }
 
@@ -856,7 +852,36 @@ static void loongson2_mmc_release_internal_dma(struct loongson2_mmc_host *host,
 	dma_free_coherent(dev, PAGE_SIZE, host->sg_cpu, host->sg_dma);
 }
 
+static struct loongson2_mmc_pdata ls2k0300_mmc_pdata = {
+	.flags			= LOONGSON2_MMC_CMD48_QUIRK,
+	.regmap_config		= &ls2k2000_mmc_regmap_config,
+	.reorder_cmd_data	= ls2k2000_mmc_reorder_cmd_data,
+	.fix_data_timeout	= ls2k2000_mmc_fix_data_timeout,
+	.setting_dma		= ls2k2000_mmc_set_internal_dma,
+	.prepare_dma		= loongson2_mmc_prepare_internal_dma,
+	.release_dma		= loongson2_mmc_release_internal_dma,
+};
+
+static struct loongson2_mmc_pdata ls2k0500_mmc_pdata = {
+	.flags			= 0,
+	.regmap_config		= &ls2k0500_mmc_regmap_config,
+	.reorder_cmd_data	= ls2k0500_mmc_reorder_cmd_data,
+	.setting_dma		= ls2k0500_mmc_set_external_dma,
+	.prepare_dma		= loongson2_mmc_prepare_external_dma,
+	.release_dma		= loongson2_mmc_release_external_dma,
+};
+
+static struct loongson2_mmc_pdata ls2k1000_mmc_pdata = {
+	.flags			= 0,
+	.regmap_config		= &ls2k0500_mmc_regmap_config,
+	.reorder_cmd_data	= ls2k0500_mmc_reorder_cmd_data,
+	.setting_dma		= ls2k1000_mmc_set_external_dma,
+	.prepare_dma		= loongson2_mmc_prepare_external_dma,
+	.release_dma		= loongson2_mmc_release_external_dma,
+};
+
 static struct loongson2_mmc_pdata ls2k2000_mmc_pdata = {
+	.flags			= 0,
 	.regmap_config		= &ls2k2000_mmc_regmap_config,
 	.reorder_cmd_data	= ls2k2000_mmc_reorder_cmd_data,
 	.fix_data_timeout	= ls2k2000_mmc_fix_data_timeout,
@@ -985,6 +1010,7 @@ static void loongson2_mmc_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id loongson2_mmc_of_ids[] = {
+	{ .compatible = "loongson,ls2k0300-mmc", .data = &ls2k0300_mmc_pdata },
 	{ .compatible = "loongson,ls2k0500-mmc", .data = &ls2k0500_mmc_pdata },
 	{ .compatible = "loongson,ls2k1000-mmc", .data = &ls2k1000_mmc_pdata },
 	{ .compatible = "loongson,ls2k2000-mmc", .data = &ls2k2000_mmc_pdata },
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index daed659f63f68..302ac8529c4f0 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -203,9 +203,10 @@
 #define SDC_CFG_DTOC            GENMASK(31, 24)	/* RW */
 
 /* SDC_STS mask */
-#define SDC_STS_SDCBUSY         BIT(0)	/* RW */
-#define SDC_STS_CMDBUSY         BIT(1)	/* RW */
-#define SDC_STS_SWR_COMPL       BIT(31)	/* RW */
+#define SDC_STS_SDCBUSY			BIT(0)  /* RW */
+#define SDC_STS_CMDBUSY			BIT(1)  /* RW */
+#define SDC_STS_SPM_RESOURCE_RELEASE	BIT(3)  /* RW */
+#define SDC_STS_SWR_COMPL		BIT(31) /* RW */
 
 /* SDC_ADV_CFG0 mask */
 #define SDC_DAT1_IRQ_TRIGGER	BIT(19)	/* RW */
@@ -448,6 +449,7 @@ struct mtk_mmc_compatible {
 	bool use_internal_cd;
 	bool support_new_tx;
 	bool support_new_rx;
+	bool support_spm_res_release;
 };
 
 struct msdc_tune_para {
@@ -673,6 +675,25 @@ static const struct mtk_mmc_compatible mt8516_compat = {
 	.stop_dly_sel = 3,
 };
 
+static const struct mtk_mmc_compatible mt8189_compat = {
+	.clk_div_bits = 12,
+	.recheck_sdio_irq = false,
+	.hs400_tune = false,
+	.needs_top_base = true,
+	.pad_tune_reg = MSDC_PAD_TUNE0,
+	.async_fifo = true,
+	.data_tune = true,
+	.busy_check = true,
+	.stop_clk_fix = true,
+	.stop_dly_sel = 1,
+	.pop_en_cnt = 2,
+	.enhance_rx = true,
+	.support_64g = true,
+	.support_new_tx = true,
+	.support_new_rx = true,
+	.support_spm_res_release = true,
+};
+
 static const struct mtk_mmc_compatible mt8196_compat = {
 	.clk_div_bits = 12,
 	.recheck_sdio_irq = false,
@@ -703,6 +724,7 @@ static const struct of_device_id msdc_of_ids[] = {
 	{ .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat},
 	{ .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat},
 	{ .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat},
+	{ .compatible = "mediatek,mt8189-mmc", .data = &mt8189_compat},
 	{ .compatible = "mediatek,mt8196-mmc", .data = &mt8196_compat},
 	{ .compatible = "mediatek,mt8516-mmc", .data = &mt8516_compat},
 
@@ -3296,6 +3318,10 @@ static int msdc_runtime_suspend(struct device *dev)
 
 		__msdc_enable_sdio_irq(host, 0);
 	}
+
+	if (host->dev_comp->support_spm_res_release)
+		sdr_set_bits(host->base + SDC_STS, SDC_STS_SPM_RESOURCE_RELEASE);
+
 	msdc_gate_clock(host);
 	return 0;
 }
diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c
index 2a310a1457855..f9ec78d699f45 100644
--- a/drivers/mmc/host/renesas_sdhi_core.c
+++ b/drivers/mmc/host/renesas_sdhi_core.c
@@ -26,6 +26,7 @@
 #include <linux/mmc/mmc.h>
 #include <linux/mmc/slot-gpio.h>
 #include <linux/module.h>
+#include <linux/mux/consumer.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/pinctrl/pinctrl-state.h>
 #include <linux/platform_data/tmio.h>
@@ -1062,6 +1063,7 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 	struct regulator_dev *rdev;
 	struct renesas_sdhi_dma *dma_priv;
 	struct device *dev = &pdev->dev;
+	struct mux_state *mux_state;
 	struct tmio_mmc_host *host;
 	struct renesas_sdhi *priv;
 	int num_irqs, irq, ret, i;
@@ -1116,6 +1118,10 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 						"state_uhs");
 	}
 
+	mux_state = devm_mux_state_get_optional_selected(&pdev->dev, NULL);
+	if (IS_ERR(mux_state))
+		return PTR_ERR(mux_state);
+
 	host = tmio_mmc_host_alloc(pdev, mmc_data);
 	if (IS_ERR(host))
 		return PTR_ERR(host);
diff --git a/drivers/mmc/host/renesas_sdhi_sys_dmac.c b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
index 543ad1d0ed1cf..9215600f03a23 100644
--- a/drivers/mmc/host/renesas_sdhi_sys_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
@@ -456,19 +456,15 @@ static int renesas_sdhi_sys_dmac_probe(struct platform_device *pdev)
 				  of_device_get_match_data(&pdev->dev), NULL);
 }
 
-static const struct dev_pm_ops renesas_sdhi_sys_dmac_dev_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
-	SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend,
-			   tmio_mmc_host_runtime_resume,
-			   NULL)
-};
+static DEFINE_RUNTIME_DEV_PM_OPS(renesas_sdhi_sys_dmac_dev_pm_ops,
+				 tmio_mmc_host_runtime_suspend,
+				 tmio_mmc_host_runtime_resume, NULL);
 
 static struct platform_driver renesas_sys_dmac_sdhi_driver = {
 	.driver		= {
 		.name	= "sh_mobile_sdhi",
 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
-		.pm	= &renesas_sdhi_sys_dmac_dev_pm_ops,
+		.pm	= pm_ptr(&renesas_sdhi_sys_dmac_dev_pm_ops),
 		.of_match_table = renesas_sdhi_sys_dmac_of_match,
 	},
 	.probe		= renesas_sdhi_sys_dmac_probe,
diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c
index 4db3328f46dfb..8dfbc62f165bc 100644
--- a/drivers/mmc/host/rtsx_pci_sdmmc.c
+++ b/drivers/mmc/host/rtsx_pci_sdmmc.c
@@ -1181,79 +1181,6 @@ static int sdmmc_get_cd(struct mmc_host *mmc)
 	return cd;
 }
 
-static int sd_wait_voltage_stable_1(struct realtek_pci_sdmmc *host)
-{
-	struct rtsx_pcr *pcr = host->pcr;
-	int err;
-	u8 stat;
-
-	/* Reference to Signal Voltage Switch Sequence in SD spec.
-	 * Wait for a period of time so that the card can drive SD_CMD and
-	 * SD_DAT[3:0] to low after sending back CMD11 response.
-	 */
-	mdelay(1);
-
-	/* SD_CMD, SD_DAT[3:0] should be driven to low by card;
-	 * If either one of SD_CMD,SD_DAT[3:0] is not low,
-	 * abort the voltage switch sequence;
-	 */
-	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
-	if (err < 0)
-		return err;
-
-	if (stat & (SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
-				SD_DAT1_STATUS | SD_DAT0_STATUS))
-		return -EINVAL;
-
-	/* Stop toggle SD clock */
-	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
-			0xFF, SD_CLK_FORCE_STOP);
-	if (err < 0)
-		return err;
-
-	return 0;
-}
-
-static int sd_wait_voltage_stable_2(struct realtek_pci_sdmmc *host)
-{
-	struct rtsx_pcr *pcr = host->pcr;
-	int err;
-	u8 stat, mask, val;
-
-	/* Wait 1.8V output of voltage regulator in card stable */
-	msleep(50);
-
-	/* Toggle SD clock again */
-	err = rtsx_pci_write_register(pcr, SD_BUS_STAT, 0xFF, SD_CLK_TOGGLE_EN);
-	if (err < 0)
-		return err;
-
-	/* Wait for a period of time so that the card can drive
-	 * SD_DAT[3:0] to high at 1.8V
-	 */
-	msleep(20);
-
-	/* SD_CMD, SD_DAT[3:0] should be pulled high by host */
-	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
-	if (err < 0)
-		return err;
-
-	mask = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
-		SD_DAT1_STATUS | SD_DAT0_STATUS;
-	val = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
-		SD_DAT1_STATUS | SD_DAT0_STATUS;
-	if ((stat & mask) != val) {
-		dev_dbg(sdmmc_dev(host),
-			"%s: SD_BUS_STAT = 0x%x\n", __func__, stat);
-		rtsx_pci_write_register(pcr, SD_BUS_STAT,
-				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
-		rtsx_pci_write_register(pcr, CARD_CLK_EN, 0xFF, 0);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 {
 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
@@ -1281,7 +1208,9 @@ static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 		voltage = OUTPUT_1V8;
 
 	if (voltage == OUTPUT_1V8) {
-		err = sd_wait_voltage_stable_1(host);
+		/* Stop toggle SD clock */
+		err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
+				0xFF, SD_CLK_FORCE_STOP);
 		if (err < 0)
 			goto out;
 	}
@@ -1290,16 +1219,11 @@ static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 	if (err < 0)
 		goto out;
 
-	if (voltage == OUTPUT_1V8) {
-		err = sd_wait_voltage_stable_2(host);
-		if (err < 0)
-			goto out;
-	}
-
 out:
 	/* Stop toggle SD clock in idle */
-	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
-			SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
+	if (err < 0)
+		rtsx_pci_write_register(pcr, SD_BUS_STAT,
+				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
 
 	mutex_unlock(&pcr->pcr_mutex);
 
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index a7a5df673b0f6..18ecddd6df6f2 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -216,6 +216,8 @@
 #define ESDHC_FLAG_DUMMY_PAD		BIT(19)
 
 #define ESDHC_AUTO_TUNING_WINDOW	3
+/* 100ms timeout for data inhibit */
+#define ESDHC_DATA_INHIBIT_WAIT_US	100000
 
 enum wp_types {
 	ESDHC_WP_NONE,		/* no WP, neither controller nor gpio */
@@ -321,6 +323,14 @@ static struct esdhc_soc_data usdhc_s32g2_data = {
 	.quirks = SDHCI_QUIRK_NO_LED,
 };
 
+static struct esdhc_soc_data usdhc_s32n79_data = {
+	.flags = ESDHC_FLAG_USDHC | ESDHC_FLAG_MAN_TUNING
+			| ESDHC_FLAG_HAVE_CAP1 | ESDHC_FLAG_HS200
+			| ESDHC_FLAG_HS400 | ESDHC_FLAG_HS400_ES
+			| ESDHC_FLAG_SKIP_ERR004536,
+	.quirks = SDHCI_QUIRK_NO_LED,
+};
+
 static struct esdhc_soc_data usdhc_imx7ulp_data = {
 	.flags = ESDHC_FLAG_USDHC | ESDHC_FLAG_MAN_TUNING
 			| ESDHC_FLAG_HAVE_CAP1 | ESDHC_FLAG_HS200
@@ -408,6 +418,7 @@ static const struct of_device_id imx_esdhc_dt_ids[] = {
 	{ .compatible = "fsl,imx95-usdhc", .data = &usdhc_imx95_data, },
 	{ .compatible = "fsl,imxrt1050-usdhc", .data = &usdhc_imxrt1050_data, },
 	{ .compatible = "nxp,s32g2-usdhc", .data = &usdhc_s32g2_data, },
+	{ .compatible = "nxp,s32n79-usdhc", .data = &usdhc_s32n79_data, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, imx_esdhc_dt_ids);
@@ -1453,6 +1464,22 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
 
 static void esdhc_reset(struct sdhci_host *host, u8 mask)
 {
+	u32 present_state;
+	int ret;
+
+	/*
+	 * For data or full reset, ensure any active data transfer completes
+	 * before resetting to avoid system hang.
+	 */
+	if (mask & (SDHCI_RESET_DATA | SDHCI_RESET_ALL)) {
+		ret = readl_poll_timeout_atomic(host->ioaddr + ESDHC_PRSSTAT, present_state,
+						!(present_state & SDHCI_DATA_INHIBIT), 2,
+						ESDHC_DATA_INHIBIT_WAIT_US);
+		if (ret == -ETIMEDOUT)
+			dev_warn(mmc_dev(host->mmc),
+				 "timeout waiting for data transfer completion\n");
+	}
+
 	sdhci_and_cqhci_reset(host, mask);
 
 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
@@ -1795,8 +1822,6 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev,
 
 	of_property_read_u32(np, "fsl,strobe-dll-delay-target",
 				&boarddata->strobe_dll_delay_target);
-	if (of_property_read_bool(np, "no-1-8-v"))
-		host->quirks2 |= SDHCI_QUIRK2_NO_1_8_V;
 
 	if (of_property_read_u32(np, "fsl,delay-line", &boarddata->delay_line))
 		boarddata->delay_line = 0;
@@ -1815,9 +1840,7 @@ sdhci_esdhc_imx_probe_dt(struct platform_device *pdev,
 	if (ret)
 		return ret;
 
-	/* HS400/HS400ES require 8 bit bus */
-	if (!(host->mmc->caps & MMC_CAP_8_BIT_DATA))
-		host->mmc->caps2 &= ~(MMC_CAP2_HS400 | MMC_CAP2_HS400_ES);
+	sdhci_get_property(pdev);
 
 	if (mmc_gpio_get_cd(host->mmc) >= 0)
 		host->quirks &= ~SDHCI_QUIRK_BROKEN_CARD_DETECTION;
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 3b85233131b38..da356627d9de6 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -157,6 +157,17 @@
 #define CQHCI_VENDOR_CFG1	0xA00
 #define CQHCI_VENDOR_DIS_RST_ON_CQ_EN	(0x3 << 13)
 
+/* non command queue crypto enable register*/
+#define NONCQ_CRYPTO_PARM		0x70
+#define NONCQ_CRYPTO_DUN		0x74
+
+#define DISABLE_CRYPTO			BIT(15)
+#define CRYPTO_GENERAL_ENABLE		BIT(1)
+#define HC_VENDOR_SPECIFIC_FUNC4	0x260
+
+#define ICE_HCI_PARAM_CCI	GENMASK(7, 0)
+#define ICE_HCI_PARAM_CE	GENMASK(8, 8)
+
 struct sdhci_msm_offset {
 	u32 core_hc_mode;
 	u32 core_mci_data_cnt;
@@ -300,6 +311,7 @@ struct sdhci_msm_host {
 	u32 dll_config;
 	u32 ddr_config;
 	bool vqmmc_enabled;
+	bool non_cqe_ice_init_done;
 };
 
 static const struct sdhci_msm_offset *sdhci_priv_msm_offset(struct sdhci_host *host)
@@ -2012,6 +2024,68 @@ static int sdhci_msm_ice_keyslot_evict(struct blk_crypto_profile *profile,
 	return qcom_ice_evict_key(msm_host->ice, slot);
 }
 
+static void sdhci_msm_non_cqe_ice_init(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	struct mmc_host *mmc = msm_host->mmc;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	u32 config;
+
+	config = sdhci_readl(host, HC_VENDOR_SPECIFIC_FUNC4);
+	config &= ~DISABLE_CRYPTO;
+	sdhci_writel(host, config, HC_VENDOR_SPECIFIC_FUNC4);
+	config = cqhci_readl(cq_host, CQHCI_CFG);
+	config |= CRYPTO_GENERAL_ENABLE;
+	cqhci_writel(cq_host, config, CQHCI_CFG);
+}
+
+static void sdhci_msm_ice_cfg(struct sdhci_host *host, struct mmc_request *mrq)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	struct mmc_host *mmc = msm_host->mmc;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	unsigned int crypto_params = 0;
+	int key_index;
+
+	if (mrq->crypto_ctx) {
+		if (!msm_host->non_cqe_ice_init_done) {
+			sdhci_msm_non_cqe_ice_init(host);
+			msm_host->non_cqe_ice_init_done = true;
+		}
+
+		key_index = mrq->crypto_key_slot;
+		crypto_params = FIELD_PREP(ICE_HCI_PARAM_CE, 1) |
+				FIELD_PREP(ICE_HCI_PARAM_CCI, key_index);
+
+		cqhci_writel(cq_host, crypto_params, NONCQ_CRYPTO_PARM);
+		cqhci_writel(cq_host, lower_32_bits(mrq->crypto_ctx->bc_dun[0]),
+			     NONCQ_CRYPTO_DUN);
+	} else {
+		cqhci_writel(cq_host, crypto_params, NONCQ_CRYPTO_PARM);
+	}
+
+	/* Ensure crypto configuration is written before proceeding */
+	wmb();
+}
+
+/*
+ * Handle non-CQE MMC requests with ICE crypto support.
+ * Configures ICE registers before passing the request to
+ * the standard SDHCI handler.
+ */
+static void sdhci_msm_request(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	/* Only need to handle non-CQE crypto requests in this path */
+	if (mmc->caps2 & MMC_CAP2_CRYPTO)
+		sdhci_msm_ice_cfg(host, mrq);
+
+	sdhci_request(mmc, mrq);
+}
+
 static const struct blk_crypto_ll_ops sdhci_msm_crypto_ops = {
 	.keyslot_program	= sdhci_msm_ice_keyslot_program,
 	.keyslot_evict		= sdhci_msm_ice_keyslot_evict,
@@ -2762,6 +2836,9 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 
 	msm_host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_NEED_RSP_BUSY;
 
+#ifdef CONFIG_MMC_CRYPTO
+	host->mmc_host_ops.request = sdhci_msm_request;
+#endif
 	/* Set the timeout value to max possible */
 	host->max_timeout_count = 0xF;
 
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index ab7f0ffe7b4f0..caf97238a58b4 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -1512,6 +1512,17 @@ static struct sdhci_arasan_of_data intel_keembay_sdio_data = {
 	.clk_ops = &arasan_clk_ops,
 };
 
+static const struct sdhci_pltfm_data sdhci_arasan_axiado_pdata = {
+	.ops = &sdhci_arasan_ops,
+	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+			SDHCI_QUIRK_BROKEN_CQE,
+};
+
+static struct sdhci_arasan_of_data sdhci_arasan_axiado_data = {
+	.pdata = &sdhci_arasan_axiado_pdata,
+	.clk_ops = &arasan_clk_ops,
+};
+
 static const struct of_device_id sdhci_arasan_of_match[] = {
 	/* SoC-specific compatible strings w/ soc_ctl_map */
 	{
@@ -1538,6 +1549,10 @@ static const struct of_device_id sdhci_arasan_of_match[] = {
 		.compatible = "intel,keembay-sdhci-5.1-sdio",
 		.data = &intel_keembay_sdio_data,
 	},
+	{
+		.compatible = "axiado,ax3000-sdhci-5.1-emmc",
+		.data = &sdhci_arasan_axiado_data,
+	},
 	/* Generic compatible below here */
 	{
 		.compatible = "arasan,sdhci-8.9a",
diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c
index ca97b01996b10..4296def694369 100644
--- a/drivers/mmc/host/sdhci-of-aspeed.c
+++ b/drivers/mmc/host/sdhci-of-aspeed.c
@@ -519,7 +519,7 @@ static struct platform_driver aspeed_sdhci_driver = {
 static int aspeed_sdc_probe(struct platform_device *pdev)
 
 {
-	struct device_node *parent, *child;
+	struct device_node *parent;
 	struct aspeed_sdc *sdc;
 	int ret;
 
@@ -548,12 +548,11 @@ static int aspeed_sdc_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, sdc);
 
 	parent = pdev->dev.of_node;
-	for_each_available_child_of_node(parent, child) {
+	for_each_available_child_of_node_scoped(parent, child) {
 		struct platform_device *cpdev;
 
 		cpdev = of_platform_device_create(child, NULL, &pdev->dev);
 		if (!cpdev) {
-			of_node_put(child);
 			ret = -ENODEV;
 			goto err_clk;
 		}
diff --git a/drivers/mmc/host/sdhci-of-bst.c b/drivers/mmc/host/sdhci-of-bst.c
new file mode 100644
index 0000000000000..f8d3df715e1a0
--- /dev/null
+++ b/drivers/mmc/host/sdhci-of-bst.c
@@ -0,0 +1,523 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * SDHCI driver for Black Sesame Technologies C1200 controller
+ *
+ * Copyright (c) 2025 Black Sesame Technologies
+ */
+
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include "sdhci.h"
+#include "sdhci-pltfm.h"
+
+/* SDHCI register extensions */
+#define SDHCI_CLOCK_PLL_EN		0x0008
+#define SDHCI_VENDOR_PTR_R		0xE8
+
+/* BST-specific tuning parameters */
+#define BST_TUNING_COUNT		0x20
+
+/* Synopsys vendor specific registers */
+#define SDHC_EMMC_CTRL_R_OFFSET		0x2C
+#define MBIU_CTRL			0x510
+
+/* MBIU burst control bits */
+#define BURST_INCR16_EN			BIT(3)
+#define BURST_INCR8_EN			BIT(2)
+#define BURST_INCR4_EN			BIT(1)
+#define BURST_EN			(BURST_INCR16_EN | BURST_INCR8_EN | BURST_INCR4_EN)
+#define MBIU_BURST_MASK			GENMASK(3, 0)
+
+/* CRM (Clock/Reset/Management) register offsets */
+#define SDEMMC_CRM_BCLK_DIV_CTRL	0x08
+#define SDEMMC_CRM_TIMER_DIV_CTRL	0x0C
+#define SDEMMC_CRM_RX_CLK_CTRL		0x14
+#define SDEMMC_CRM_VOL_CTRL		0x1C
+#define REG_WR_PROTECT			0x88
+#define DELAY_CHAIN_SEL			0x94
+
+/* CRM register values and bit definitions */
+#define REG_WR_PROTECT_KEY		0x1234abcd
+#define BST_VOL_STABLE_ON		BIT(7)
+#define BST_TIMER_DIV_MASK		GENMASK(7, 0)
+#define BST_TIMER_DIV_VAL		0x20
+#define BST_TIMER_LOAD_BIT		BIT(8)
+#define BST_BCLK_EN_BIT			BIT(10)
+#define BST_RX_UPDATE_BIT		BIT(11)
+#define BST_EMMC_CTRL_RST_N		BIT(2)	/* eMMC card reset control */
+
+/* Clock frequency limits */
+#define BST_DEFAULT_MAX_FREQ		200000000UL	/* 200 MHz */
+#define BST_DEFAULT_MIN_FREQ		400000UL	/* 400 kHz */
+
+/* Clock control bit definitions */
+#define BST_CLOCK_DIV_MASK		GENMASK(7, 0)
+#define BST_CLOCK_DIV_SHIFT		8
+#define BST_BCLK_DIV_MASK		GENMASK(9, 0)
+
+/* Clock frequency thresholds */
+#define BST_CLOCK_THRESHOLD_LOW		1500
+
+/* Clock stability polling parameters */
+#define BST_CLK_STABLE_POLL_US		1000	/* Poll interval in microseconds */
+#define BST_CLK_STABLE_TIMEOUT_US	20000	/* Timeout for internal clock stabilization (us) */
+
+struct sdhci_bst_priv {
+	void __iomem *crm_reg_base;
+};
+
+union sdhci_bst_rx_ctrl {
+	struct {
+		u32 rx_revert:1,
+		    rx_clk_sel_sec:1,
+		    rx_clk_div:4,
+		    rx_clk_phase_inner:2,
+		    rx_clk_sel_first:1,
+		    rx_clk_phase_out:2,
+		    rx_clk_en:1,
+		    res0:20;
+	};
+	u32 reg;
+};
+
+static u32 sdhci_bst_crm_read(struct sdhci_pltfm_host *pltfm_host, u32 offset)
+{
+	struct sdhci_bst_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+	return readl(priv->crm_reg_base + offset);
+}
+
+static void sdhci_bst_crm_write(struct sdhci_pltfm_host *pltfm_host, u32 offset, u32 value)
+{
+	struct sdhci_bst_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+	writel(value, priv->crm_reg_base + offset);
+}
+
+static int sdhci_bst_wait_int_clk(struct sdhci_host *host)
+{
+	u16 clk;
+
+	if (read_poll_timeout(sdhci_readw, clk, (clk & SDHCI_CLOCK_INT_STABLE),
+			      BST_CLK_STABLE_POLL_US, BST_CLK_STABLE_TIMEOUT_US, false,
+			      host, SDHCI_CLOCK_CONTROL))
+		return -EBUSY;
+	return 0;
+}
+
+static unsigned int sdhci_bst_get_max_clock(struct sdhci_host *host)
+{
+	return BST_DEFAULT_MAX_FREQ;
+}
+
+static unsigned int sdhci_bst_get_min_clock(struct sdhci_host *host)
+{
+	return BST_DEFAULT_MIN_FREQ;
+}
+
+static void sdhci_bst_enable_clk(struct sdhci_host *host, unsigned int clk)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	unsigned int div;
+	u32 val;
+	union sdhci_bst_rx_ctrl rx_reg;
+
+	pltfm_host = sdhci_priv(host);
+
+	/* Calculate clock divider based on target frequency */
+	if (clk == 0) {
+		div = 0;
+	} else if (clk < BST_DEFAULT_MIN_FREQ) {
+		/* Below minimum: use max divider to get closest to min freq */
+		div = BST_DEFAULT_MAX_FREQ / BST_DEFAULT_MIN_FREQ;
+	} else if (clk <= BST_DEFAULT_MAX_FREQ) {
+		/* Normal range: calculate divider directly */
+		div = BST_DEFAULT_MAX_FREQ / clk;
+	} else {
+		/* Above maximum: no division needed */
+		div = 1;
+	}
+
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	clk &= ~SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	clk &= ~SDHCI_CLOCK_PLL_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL);
+	val &= ~BST_TIMER_LOAD_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL, val);
+
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL);
+	val &= ~BST_TIMER_DIV_MASK;
+	val |= BST_TIMER_DIV_VAL;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL, val);
+
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL);
+	val |= BST_TIMER_LOAD_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_TIMER_DIV_CTRL, val);
+
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL);
+	val &= ~BST_RX_UPDATE_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL, val);
+
+	rx_reg.reg = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL);
+
+	rx_reg.rx_revert = 0;
+	rx_reg.rx_clk_sel_sec = 1;
+	rx_reg.rx_clk_div = 4;
+	rx_reg.rx_clk_phase_inner = 2;
+	rx_reg.rx_clk_sel_first = 0;
+	rx_reg.rx_clk_phase_out = 2;
+
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL, rx_reg.reg);
+
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL);
+	val |= BST_RX_UPDATE_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL, val);
+
+	/* Disable clock first */
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL);
+	val &= ~BST_BCLK_EN_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL, val);
+
+	/* Setup clock divider */
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL);
+	val &= ~BST_BCLK_DIV_MASK;
+	val |= div;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL, val);
+
+	/* Enable clock */
+	val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL);
+	val |= BST_BCLK_EN_BIT;
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL, val);
+
+	/* RMW the clock divider bits to avoid clobbering other fields */
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	clk &= ~(BST_CLOCK_DIV_MASK << BST_CLOCK_DIV_SHIFT);
+	clk |= (div & BST_CLOCK_DIV_MASK) << BST_CLOCK_DIV_SHIFT;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	clk |= SDHCI_CLOCK_PLL_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	clk |= SDHCI_CLOCK_CARD_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+
+	clk |= SDHCI_CLOCK_INT_EN;
+	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
+}
+
+static void sdhci_bst_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	/* Turn off card/internal/PLL clocks when clock==0 to avoid idle power */
+	u32 clk_reg = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+
+	if (!clock) {
+		clk_reg &= ~(SDHCI_CLOCK_CARD_EN | SDHCI_CLOCK_INT_EN | SDHCI_CLOCK_PLL_EN);
+		sdhci_writew(host, clk_reg, SDHCI_CLOCK_CONTROL);
+		return;
+	}
+	sdhci_bst_enable_clk(host, clock);
+}
+
+/*
+ * sdhci_bst_reset - Reset the SDHCI host controller with special
+ * handling for eMMC card reset control.
+ */
+static void sdhci_bst_reset(struct sdhci_host *host, u8 mask)
+{
+	u16 vendor_ptr, emmc_ctrl_reg;
+	u32 reg;
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_SD) {
+		vendor_ptr = sdhci_readw(host, SDHCI_VENDOR_PTR_R);
+		emmc_ctrl_reg = vendor_ptr + SDHC_EMMC_CTRL_R_OFFSET;
+
+		reg = sdhci_readw(host, emmc_ctrl_reg);
+		reg &= ~BST_EMMC_CTRL_RST_N;
+		sdhci_writew(host, reg, emmc_ctrl_reg);
+		sdhci_reset(host, mask);
+		usleep_range(10, 20);
+		reg = sdhci_readw(host, emmc_ctrl_reg);
+		reg |= BST_EMMC_CTRL_RST_N;
+		sdhci_writew(host, reg, emmc_ctrl_reg);
+	} else {
+		sdhci_reset(host, mask);
+	}
+}
+
+/* Set timeout control register to maximum value (0xE) */
+static void sdhci_bst_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
+{
+	sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
+}
+
+/*
+ * sdhci_bst_set_power - Set power mode and voltage, also configures
+ * MBIU burst mode control based on power state.
+ */
+static void sdhci_bst_set_power(struct sdhci_host *host, unsigned char mode,
+				unsigned short vdd)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	u32 reg;
+	u32 val;
+
+	sdhci_set_power(host, mode, vdd);
+
+	if (mode == MMC_POWER_OFF) {
+		/* Disable MBIU burst mode */
+		reg = sdhci_readw(host, MBIU_CTRL);
+		reg &= ~BURST_EN; /* Clear all burst enable bits */
+		sdhci_writew(host, reg, MBIU_CTRL);
+
+		/* Disable CRM BCLK */
+		val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL);
+		val &= ~BST_BCLK_EN_BIT;
+		sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_BCLK_DIV_CTRL, val);
+
+		/* Disable RX clock */
+		val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL);
+		val &= ~BST_RX_UPDATE_BIT;
+		sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_RX_CLK_CTRL, val);
+
+		/* Turn off voltage stable power */
+		val = sdhci_bst_crm_read(pltfm_host, SDEMMC_CRM_VOL_CTRL);
+		val &= ~BST_VOL_STABLE_ON;
+		sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_VOL_CTRL, val);
+	} else {
+		/* Configure burst mode only when powered on */
+		reg = sdhci_readw(host, MBIU_CTRL);
+		reg &= ~MBIU_BURST_MASK; /* Clear burst related bits */
+		reg |= BURST_EN; /* Enable burst mode for better bandwidth */
+		sdhci_writew(host, reg, MBIU_CTRL);
+	}
+}
+
+/*
+ * sdhci_bst_execute_tuning - Execute tuning procedure by trying different
+ * delay chain values and selecting the optimal one.
+ */
+static int sdhci_bst_execute_tuning(struct sdhci_host *host, u32 opcode)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	int ret = 0, error;
+	int first_start = -1, first_end = -1, best = 0;
+	int second_start = -1, second_end = -1, has_failure = 0;
+	int i;
+
+	pltfm_host = sdhci_priv(host);
+
+	for (i = 0; i < BST_TUNING_COUNT; i++) {
+		/* Protected write */
+		sdhci_bst_crm_write(pltfm_host, REG_WR_PROTECT, REG_WR_PROTECT_KEY);
+		/* Write tuning value */
+		sdhci_bst_crm_write(pltfm_host, DELAY_CHAIN_SEL, (1ul << i) - 1);
+
+		/* Wait for internal clock stable before tuning */
+		if (sdhci_bst_wait_int_clk(host)) {
+			dev_err(mmc_dev(host->mmc), "Internal clock never stabilised\n");
+			return -EBUSY;
+		}
+
+		ret = mmc_send_tuning(host->mmc, opcode, &error);
+		if (ret != 0) {
+			has_failure = 1;
+		} else {
+			if (has_failure == 0) {
+				if (first_start == -1)
+					first_start = i;
+				first_end = i;
+			} else {
+				if (second_start == -1)
+					second_start = i;
+				second_end = i;
+			}
+		}
+	}
+
+	/* Calculate best tuning value */
+	if (first_end - first_start >= second_end - second_start)
+		best = ((first_end - first_start) >> 1) + first_start;
+	else
+		best = ((second_end - second_start) >> 1) + second_start;
+
+	if (best < 0)
+		best = 0;
+
+	sdhci_bst_crm_write(pltfm_host, DELAY_CHAIN_SEL, (1ul << best) - 1);
+	/* Confirm internal clock stable after setting best tuning value */
+	if (sdhci_bst_wait_int_clk(host)) {
+		dev_err(mmc_dev(host->mmc), "Internal clock never stabilised\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+/* Enable voltage stable power for voltage switch */
+static void sdhci_bst_voltage_switch(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+
+	/* Enable voltage stable power */
+	sdhci_bst_crm_write(pltfm_host, SDEMMC_CRM_VOL_CTRL, BST_VOL_STABLE_ON);
+}
+
+static const struct sdhci_ops sdhci_bst_ops = {
+	.set_clock		= sdhci_bst_set_clock,
+	.set_bus_width		= sdhci_set_bus_width,
+	.set_uhs_signaling	= sdhci_set_uhs_signaling,
+	.get_min_clock		= sdhci_bst_get_min_clock,
+	.get_max_clock		= sdhci_bst_get_max_clock,
+	.reset			= sdhci_bst_reset,
+	.set_power		= sdhci_bst_set_power,
+	.set_timeout		= sdhci_bst_set_timeout,
+	.platform_execute_tuning = sdhci_bst_execute_tuning,
+	.voltage_switch		= sdhci_bst_voltage_switch,
+};
+
+static const struct sdhci_pltfm_data sdhci_bst_pdata = {
+	.ops = &sdhci_bst_ops,
+	.quirks = SDHCI_QUIRK_BROKEN_ADMA |
+		  SDHCI_QUIRK_DELAY_AFTER_POWER |
+		  SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+		  SDHCI_QUIRK_INVERTED_WRITE_PROTECT,
+	.quirks2 = SDHCI_QUIRK2_BROKEN_DDR50 |
+		   SDHCI_QUIRK2_TUNING_WORK_AROUND |
+		   SDHCI_QUIRK2_ACMD23_BROKEN,
+};
+
+static void sdhci_bst_free_bounce_buffer(struct sdhci_host *host)
+{
+	if (host->bounce_buffer) {
+		dma_free_coherent(mmc_dev(host->mmc), host->bounce_buffer_size,
+				  host->bounce_buffer, host->bounce_addr);
+		host->bounce_buffer = NULL;
+	}
+	of_reserved_mem_device_release(mmc_dev(host->mmc));
+}
+
+static int sdhci_bst_alloc_bounce_buffer(struct sdhci_host *host)
+{
+	struct mmc_host *mmc = host->mmc;
+	unsigned int bounce_size;
+	int ret;
+
+	/* Fixed SRAM bounce size to 32KB: verified config under 32-bit DMA addressing limit */
+	bounce_size = SZ_32K;
+
+	ret = of_reserved_mem_device_init_by_idx(mmc_dev(mmc), mmc_dev(mmc)->of_node, 0);
+	if (ret) {
+		dev_err(mmc_dev(mmc), "Failed to initialize reserved memory\n");
+		return ret;
+	}
+
+	host->bounce_buffer = dma_alloc_coherent(mmc_dev(mmc), bounce_size,
+						 &host->bounce_addr, GFP_KERNEL);
+	if (!host->bounce_buffer) {
+		of_reserved_mem_device_release(mmc_dev(mmc));
+		return -ENOMEM;
+	}
+
+	host->bounce_buffer_size = bounce_size;
+
+	return 0;
+}
+
+static int sdhci_bst_probe(struct platform_device *pdev)
+{
+	struct sdhci_pltfm_host *pltfm_host;
+	struct sdhci_host *host;
+	struct sdhci_bst_priv *priv;
+	int err;
+
+	host = sdhci_pltfm_init(pdev, &sdhci_bst_pdata, sizeof(struct sdhci_bst_priv));
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	priv = sdhci_pltfm_priv(pltfm_host); /* Get platform private data */
+
+	err = mmc_of_parse(host->mmc);
+	if (err)
+		return err;
+
+	sdhci_get_of_property(pdev);
+
+	/* Get CRM registers from the second reg entry */
+	priv->crm_reg_base = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(priv->crm_reg_base)) {
+		err = PTR_ERR(priv->crm_reg_base);
+		return err;
+	}
+
+	/*
+	 * Silicon constraints for BST C1200:
+	 * - System RAM base is 0x800000000 (above 32-bit addressable range)
+	 * - The eMMC controller DMA engine is limited to 32-bit addressing
+	 * - SMMU cannot be used on this path due to hardware design flaws
+	 * - These are fixed in silicon and cannot be changed in software
+	 *
+	 * Bus/controller mapping:
+	 * - No registers are available to reprogram the address mapping
+	 * - The 32-bit DMA limit is a hard constraint of the controller IP
+	 *
+	 * Given these constraints, an SRAM-based bounce buffer in the 32-bit
+	 * address space is required to enable eMMC DMA on this platform.
+	 */
+	err = sdhci_bst_alloc_bounce_buffer(host);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate bounce buffer: %d\n", err);
+		return err;
+	}
+
+	err = sdhci_add_host(host);
+	if (err)
+		goto err_free_bounce_buffer;
+
+	return 0;
+
+err_free_bounce_buffer:
+	sdhci_bst_free_bounce_buffer(host);
+
+	return err;
+}
+
+static void sdhci_bst_remove(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+
+	sdhci_bst_free_bounce_buffer(host);
+	sdhci_pltfm_remove(pdev);
+}
+
+static const struct of_device_id sdhci_bst_ids[] = {
+	{ .compatible = "bst,c1200-sdhci" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sdhci_bst_ids);
+
+static struct platform_driver sdhci_bst_driver = {
+	.driver = {
+		.name = "sdhci-bst",
+		.of_match_table = sdhci_bst_ids,
+	},
+	.probe = sdhci_bst_probe,
+	.remove = sdhci_bst_remove,
+};
+module_platform_driver(sdhci_bst_driver);
+
+MODULE_DESCRIPTION("Black Sesame Technologies SDHCI driver (BST)");
+MODULE_AUTHOR("Black Sesame Technologies Co., Ltd.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-of-k1.c b/drivers/mmc/host/sdhci-of-k1.c
index 0cc97e23a2f9c..455656f9842df 100644
--- a/drivers/mmc/host/sdhci-of-k1.c
+++ b/drivers/mmc/host/sdhci-of-k1.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/reset.h>
 #include <linux/platform_device.h>
 
 #include "sdhci.h"
@@ -223,6 +224,21 @@ static inline int spacemit_sdhci_get_clocks(struct device *dev,
 	return 0;
 }
 
+static inline int spacemit_sdhci_get_resets(struct device *dev)
+{
+	struct reset_control *rst;
+
+	rst = devm_reset_control_get_optional_shared_deasserted(dev, "axi");
+	if (IS_ERR(rst))
+		return PTR_ERR(rst);
+
+	rst = devm_reset_control_get_optional_exclusive_deasserted(dev, "sdh");
+	if (IS_ERR(rst))
+		return PTR_ERR(rst);
+
+	return 0;
+}
+
 static const struct sdhci_ops spacemit_sdhci_ops = {
 	.get_max_clock		= spacemit_sdhci_clk_get_max_clock,
 	.reset			= spacemit_sdhci_reset,
@@ -243,8 +259,20 @@ static const struct sdhci_pltfm_data spacemit_sdhci_k1_pdata = {
 		   SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
 };
 
+static const struct sdhci_pltfm_data spacemit_sdhci_k3_pdata = {
+	.ops = &spacemit_sdhci_ops,
+	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
+		  SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC |
+		  SDHCI_QUIRK_32BIT_ADMA_SIZE |
+		  SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+		  SDHCI_QUIRK_BROKEN_CARD_DETECTION |
+		  SDHCI_QUIRK_BROKEN_TIMEOUT_VAL,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
 static const struct of_device_id spacemit_sdhci_of_match[] = {
-	{ .compatible = "spacemit,k1-sdhci" },
+	{ .compatible = "spacemit,k1-sdhci", .data = &spacemit_sdhci_k1_pdata },
+	{ .compatible = "spacemit,k3-sdhci", .data = &spacemit_sdhci_k3_pdata },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, spacemit_sdhci_of_match);
@@ -255,10 +283,13 @@ static int spacemit_sdhci_probe(struct platform_device *pdev)
 	struct spacemit_sdhci_host *sdhst;
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_host *host;
+	const struct sdhci_pltfm_data *data;
 	struct mmc_host_ops *mops;
 	int ret;
 
-	host = sdhci_pltfm_init(pdev, &spacemit_sdhci_k1_pdata, sizeof(*sdhst));
+	data = of_device_get_match_data(&pdev->dev);
+
+	host = sdhci_pltfm_init(pdev, data, sizeof(*sdhst));
 	if (IS_ERR(host))
 		return PTR_ERR(host);
 
@@ -284,6 +315,10 @@ static int spacemit_sdhci_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_pltfm;
 
+	ret = spacemit_sdhci_get_resets(dev);
+	if (ret)
+		goto err_pltfm;
+
 	ret = sdhci_add_host(host);
 	if (ret)
 		goto err_pltfm;
diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c
index b0f91cc9e40e4..6e4084407662a 100644
--- a/drivers/mmc/host/sdhci-pci-gli.c
+++ b/drivers/mmc/host/sdhci-pci-gli.c
@@ -68,6 +68,9 @@
 #define   GLI_9750_MISC_TX1_DLY_VALUE    0x5
 #define   SDHCI_GLI_9750_MISC_SSC_OFF    BIT(26)
 
+#define SDHCI_GLI_9750_GM_BURST_SIZE		  0x510
+#define   SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT  GENMASK(17, 16)
+
 #define SDHCI_GLI_9750_TUNING_CONTROL	          0x540
 #define   SDHCI_GLI_9750_TUNING_CONTROL_EN          BIT(4)
 #define   GLI_9750_TUNING_CONTROL_EN_ON             0x1
@@ -345,10 +348,16 @@ static void gli_set_9750(struct sdhci_host *host)
 	u32 misc_value;
 	u32 parameter_value;
 	u32 control_value;
+	u32 burst_value;
 	u16 ctrl2;
 
 	gl9750_wt_on(host);
 
+	/* clear R_OSRC_Lmt to avoid DMA write corruption */
+	burst_value = sdhci_readl(host, SDHCI_GLI_9750_GM_BURST_SIZE);
+	burst_value &= ~SDHCI_GLI_9750_GM_BURST_SIZE_R_OSRC_LMT;
+	sdhci_writel(host, burst_value, SDHCI_GLI_9750_GM_BURST_SIZE);
+
 	driving_value = sdhci_readl(host, SDHCI_GLI_9750_DRIVING);
 	pll_value = sdhci_readl(host, SDHCI_GLI_9750_PLL);
 	sw_ctrl_value = sdhci_readl(host, SDHCI_GLI_9750_SW_CTRL);
diff --git a/drivers/mmc/host/sdhci-pic32.c b/drivers/mmc/host/sdhci-pic32.c
index 2cc632e91fe45..6899cf6c65e7d 100644
--- a/drivers/mmc/host/sdhci-pic32.c
+++ b/drivers/mmc/host/sdhci-pic32.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * Support of SDHCI platform devices for Microchip PIC32.
  *
@@ -5,10 +6,6 @@
  * Andrei Pistirica, Paul Thacker
  *
  * Inspired by sdhci-pltfm.c
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/mmc/host/sdhci-pltfm.c b/drivers/mmc/host/sdhci-pltfm.c
index 7f6ac636f040d..933fafe0a0efd 100644
--- a/drivers/mmc/host/sdhci-pltfm.c
+++ b/drivers/mmc/host/sdhci-pltfm.c
@@ -95,13 +95,6 @@ void sdhci_get_property(struct platform_device *pdev)
 	sdhci_get_compatibility(pdev);
 
 	device_property_read_u32(dev, "clock-frequency", &pltfm_host->clock);
-
-	if (device_property_present(dev, "keep-power-in-suspend"))
-		host->mmc->pm_caps |= MMC_PM_KEEP_POWER;
-
-	if (device_property_read_bool(dev, "wakeup-source") ||
-	    device_property_read_bool(dev, "enable-sdio-wakeup")) /* legacy */
-		host->mmc->pm_caps |= MMC_PM_WAKE_SDIO_IRQ;
 }
 EXPORT_SYMBOL_GPL(sdhci_get_property);
 
@@ -215,19 +208,6 @@ const struct dev_pm_ops sdhci_pltfm_pmops = {
 };
 EXPORT_SYMBOL_GPL(sdhci_pltfm_pmops);
 
-static int __init sdhci_pltfm_drv_init(void)
-{
-	pr_info("sdhci-pltfm: SDHCI platform and OF driver helper\n");
-
-	return 0;
-}
-module_init(sdhci_pltfm_drv_init);
-
-static void __exit sdhci_pltfm_drv_exit(void)
-{
-}
-module_exit(sdhci_pltfm_drv_exit);
-
 MODULE_DESCRIPTION("SDHCI platform and OF driver helper");
 MODULE_AUTHOR("Intel Corporation");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/sdhci-uhs2.c b/drivers/mmc/host/sdhci-uhs2.c
index c459a08d01da5..41e49c6cc7511 100644
--- a/drivers/mmc/host/sdhci-uhs2.c
+++ b/drivers/mmc/host/sdhci-uhs2.c
@@ -1126,7 +1126,7 @@ static irqreturn_t sdhci_uhs2_thread_irq(int irq, void *dev_id)
 
 /*****************************************************************************\
  *                                                                           *
- * Driver init/exit                                                          *
+ * Driver init                                                               *
  *                                                                           *
 \*****************************************************************************/
 
@@ -1138,17 +1138,6 @@ static int sdhci_uhs2_host_ops_init(struct sdhci_host *host)
 	return 0;
 }
 
-static int __init sdhci_uhs2_mod_init(void)
-{
-	return 0;
-}
-module_init(sdhci_uhs2_mod_init);
-
-static void __exit sdhci_uhs2_mod_exit(void)
-{
-}
-module_exit(sdhci_uhs2_mod_exit);
-
 /*****************************************************************************\
  *
  * Device allocation/registration                                            *
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index ac7e11f37af71..605be55f8d2d0 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -4193,6 +4193,12 @@ static void sdhci_allocate_bounce_buffer(struct sdhci_host *host)
 	unsigned int bounce_size;
 	int ret;
 
+	/* Drivers may have already allocated the buffer */
+	if (host->bounce_buffer) {
+		bounce_size = host->bounce_buffer_size;
+		max_blocks = bounce_size / 512;
+		goto out;
+	}
 	/*
 	 * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
 	 * has diminishing returns, this is probably because SD/MMC
@@ -4241,6 +4247,7 @@ static void sdhci_allocate_bounce_buffer(struct sdhci_host *host)
 
 	host->bounce_buffer_size = bounce_size;
 
+out:
 	/* Lie about this since we're bouncing */
 	mmc->max_segs = max_blocks;
 	mmc->max_seg_size = bounce_size;
@@ -4532,8 +4539,15 @@ int sdhci_setup_host(struct sdhci_host *host)
 	 * their platform code before calling sdhci_add_host(), and we
 	 * won't assume 8-bit width for hosts without that CAP.
 	 */
-	if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
+	if (host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA) {
+		host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_DDR50);
+		if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400)
+			host->caps1 &= ~SDHCI_SUPPORT_HS400;
+		mmc->caps2 &= ~(MMC_CAP2_HS200 | MMC_CAP2_HS400 | MMC_CAP2_HS400_ES);
+		mmc->caps &= ~(MMC_CAP_DDR | MMC_CAP_UHS);
+	} else {
 		mmc->caps |= MMC_CAP_4_BIT_DATA;
+	}
 
 	if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
 		mmc->caps &= ~MMC_CAP_CMD23;
@@ -4997,22 +5011,6 @@ EXPORT_SYMBOL_GPL(sdhci_remove_host);
  *                                                                           *
 \*****************************************************************************/
 
-static int __init sdhci_drv_init(void)
-{
-	pr_info(DRIVER_NAME
-		": Secure Digital Host Controller Interface driver\n");
-	pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
-
-	return 0;
-}
-
-static void __exit sdhci_drv_exit(void)
-{
-}
-
-module_init(sdhci_drv_init);
-module_exit(sdhci_drv_exit);
-
 module_param(debug_quirks, uint, 0444);
 module_param(debug_quirks2, uint, 0444);
 
diff --git a/drivers/mmc/host/tifm_sd.c b/drivers/mmc/host/tifm_sd.c
index 2cd69c9e9571b..c1f7d5b37911d 100644
--- a/drivers/mmc/host/tifm_sd.c
+++ b/drivers/mmc/host/tifm_sd.c
@@ -193,9 +193,7 @@ static void tifm_sd_transfer_data(struct tifm_sd *host)
 
 		pg = sg_page(&sg[host->sg_pos]) + (off >> PAGE_SHIFT);
 		p_off = offset_in_page(off);
-		p_cnt = PAGE_SIZE - p_off;
-		p_cnt = min(p_cnt, cnt);
-		p_cnt = min(p_cnt, t_size);
+		p_cnt = min3(PAGE_SIZE - p_off, cnt, t_size);
 
 		if (r_data->flags & MMC_DATA_READ)
 			tifm_sd_read_fifo(host, pg, p_off, p_cnt);
diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig
index 796a2eccbef0b..0421c6208de77 100644
--- a/drivers/mtd/Kconfig
+++ b/drivers/mtd/Kconfig
@@ -206,6 +206,15 @@ config MTD_PARTITIONED_MASTER
 	  the parent of the partition device be the master device, rather than
 	  what lies behind the master.
 
+config MTD_VIRT_CONCAT
+	bool "Virtual concatenated MTD devices"
+	depends on MTD_PARTITIONED_MASTER
+	help
+	  The driver enables the creation of virtual MTD device by
+	  concatenating multiple physical MTD devices into a single
+	  entity. This allows for the creation of partitions larger than
+	  the individual physical chips, extending across chip boundaries.
+
 source "drivers/mtd/chips/Kconfig"
 
 source "drivers/mtd/maps/Kconfig"
diff --git a/drivers/mtd/Makefile b/drivers/mtd/Makefile
index 593d0593a038a..7b6dd53e81504 100644
--- a/drivers/mtd/Makefile
+++ b/drivers/mtd/Makefile
@@ -6,6 +6,7 @@
 # Core functionality.
 obj-$(CONFIG_MTD)		+= mtd.o
 mtd-y				:= mtdcore.o mtdsuper.o mtdconcat.o mtdpart.o mtdchar.o
+mtd-$(CONFIG_MTD_VIRT_CONCAT)	+= mtd_virt_concat.o
 
 obj-y				+= parsers/
 
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index 33050a2a80f79..603fd0efc2ea2 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -2049,7 +2049,6 @@ static int __init docg3_probe(struct platform_device *pdev)
 static void docg3_release(struct platform_device *pdev)
 {
 	struct docg3_cascade *cascade = platform_get_drvdata(pdev);
-	struct docg3 *docg3 = cascade->floors[0]->priv;
 	int floor;
 
 	doc_unregister_sysfs(pdev, cascade);
@@ -2057,7 +2056,7 @@ static void docg3_release(struct platform_device *pdev)
 		if (cascade->floors[floor])
 			doc_release_device(cascade->floors[floor]);
 
-	bch_free(docg3->cascade->bch);
+	bch_free(cascade->bch);
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index 8a8b19874e239..99d5ff9a1fbe6 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -75,17 +75,6 @@ config MTD_PHYSMAP_OF
 	  physically into the CPU's memory. The mapping description here is
 	  taken from OF device tree.
 
-config MTD_PHYSMAP_BT1_ROM
-	bool "Baikal-T1 Boot ROMs OF-based physical memory map handling"
-	depends on MTD_PHYSMAP_OF
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
-	select MTD_COMPLEX_MAPPINGS
-	select MULTIPLEXER
-	select MUX_MMIO
-	help
-	  This provides some extra DT physmap parsing for the Baikal-T1
-	  platforms, some detection and setting up ROMs-specific accessors.
-
 config MTD_PHYSMAP_VERSATILE
 	bool "ARM Versatile OF-based physical memory map handling"
 	depends on MTD_PHYSMAP_OF
diff --git a/drivers/mtd/maps/Makefile b/drivers/mtd/maps/Makefile
index 019f1e92cc414..51af1d2ebd521 100644
--- a/drivers/mtd/maps/Makefile
+++ b/drivers/mtd/maps/Makefile
@@ -19,7 +19,6 @@ obj-$(CONFIG_MTD_TSUNAMI)	+= tsunami_flash.o
 obj-$(CONFIG_MTD_PXA2XX)	+= pxa2xx-flash.o
 obj-$(CONFIG_MTD_PHYSMAP)	+= physmap.o
 physmap-y			:= physmap-core.o
-physmap-$(CONFIG_MTD_PHYSMAP_BT1_ROM) += physmap-bt1-rom.o
 physmap-$(CONFIG_MTD_PHYSMAP_VERSATILE) += physmap-versatile.o
 physmap-$(CONFIG_MTD_PHYSMAP_GEMINI) += physmap-gemini.o
 physmap-$(CONFIG_MTD_PHYSMAP_IXP4XX) += physmap-ixp4xx.o
diff --git a/drivers/mtd/maps/physmap-bt1-rom.c b/drivers/mtd/maps/physmap-bt1-rom.c
deleted file mode 100644
index 60dccc48f99e0..0000000000000
--- a/drivers/mtd/maps/physmap-bt1-rom.c
+++ /dev/null
@@ -1,125 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 Physically Mapped Internal ROM driver
- */
-#include <linux/bits.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/mtd/map.h>
-#include <linux/mtd/xip.h>
-#include <linux/mux/consumer.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/string.h>
-#include <linux/types.h>
-
-#include "physmap-bt1-rom.h"
-
-/*
- * Baikal-T1 SoC ROMs are only accessible by the dword-aligned instructions.
- * We have to take this into account when implementing the data read-methods.
- * Note there is no need in bothering with endianness, since both Baikal-T1
- * CPU and MMIO are LE.
- */
-static map_word __xipram bt1_rom_map_read(struct map_info *map,
-					  unsigned long ofs)
-{
-	void __iomem *src = map->virt + ofs;
-	unsigned int shift;
-	map_word ret;
-	u32 data;
-
-	/* Read data within offset dword. */
-	shift = (uintptr_t)src & 0x3;
-	data = readl_relaxed(src - shift);
-	if (!shift) {
-		ret.x[0] = data;
-		return ret;
-	}
-	ret.x[0] = data >> (shift * BITS_PER_BYTE);
-
-	/* Read data from the next dword. */
-	shift = 4 - shift;
-	if (ofs + shift >= map->size)
-		return ret;
-
-	data = readl_relaxed(src + shift);
-	ret.x[0] |= data << (shift * BITS_PER_BYTE);
-
-	return ret;
-}
-
-static void __xipram bt1_rom_map_copy_from(struct map_info *map,
-					   void *to, unsigned long from,
-					   ssize_t len)
-{
-	void __iomem *src = map->virt + from;
-	unsigned int shift, chunk;
-	u32 data;
-
-	if (len <= 0 || from >= map->size)
-		return;
-
-	/* Make sure we don't go over the map limit. */
-	len = min_t(ssize_t, map->size - from, len);
-
-	/*
-	 * Since requested data size can be pretty big we have to implement
-	 * the copy procedure as optimal as possible. That's why it's split
-	 * up into the next three stages: unaligned head, aligned body,
-	 * unaligned tail.
-	 */
-	shift = (uintptr_t)src & 0x3;
-	if (shift) {
-		chunk = min_t(ssize_t, 4 - shift, len);
-		data = readl_relaxed(src - shift);
-		memcpy(to, (char *)&data + shift, chunk);
-		src += chunk;
-		to += chunk;
-		len -= chunk;
-	}
-
-	while (len >= 4) {
-		data = readl_relaxed(src);
-		memcpy(to, &data, 4);
-		src += 4;
-		to += 4;
-		len -= 4;
-	}
-
-	if (len) {
-		data = readl_relaxed(src);
-		memcpy(to, &data, len);
-	}
-}
-
-int of_flash_probe_bt1_rom(struct platform_device *pdev,
-			   struct device_node *np,
-			   struct map_info *map)
-{
-	struct device *dev = &pdev->dev;
-
-	/* It's supposed to be read-only MTD. */
-	if (!of_device_is_compatible(np, "mtd-rom")) {
-		dev_info(dev, "No mtd-rom compatible string\n");
-		return 0;
-	}
-
-	/* Multiplatform guard. */
-	if (!of_device_is_compatible(np, "baikal,bt1-int-rom"))
-		return 0;
-
-	/* Sanity check the device parameters retrieved from DTB. */
-	if (map->bankwidth != 4)
-		dev_warn(dev, "Bank width is supposed to be 32 bits wide\n");
-
-	map->read = bt1_rom_map_read;
-	map->copy_from = bt1_rom_map_copy_from;
-
-	return 0;
-}
diff --git a/drivers/mtd/maps/physmap-bt1-rom.h b/drivers/mtd/maps/physmap-bt1-rom.h
deleted file mode 100644
index 6782899598a4c..0000000000000
--- a/drivers/mtd/maps/physmap-bt1-rom.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include <linux/mtd/map.h>
-#include <linux/of.h>
-
-#ifdef CONFIG_MTD_PHYSMAP_BT1_ROM
-int of_flash_probe_bt1_rom(struct platform_device *pdev,
-			   struct device_node *np,
-			   struct map_info *map);
-#else
-static inline
-int of_flash_probe_bt1_rom(struct platform_device *pdev,
-			   struct device_node *np,
-			   struct map_info *map)
-{
-	return 0;
-}
-#endif
diff --git a/drivers/mtd/maps/physmap-core.c b/drivers/mtd/maps/physmap-core.c
index 0dcc25b7ff988..dcda7685fc99b 100644
--- a/drivers/mtd/maps/physmap-core.c
+++ b/drivers/mtd/maps/physmap-core.c
@@ -42,7 +42,6 @@
 #include <linux/pm_runtime.h>
 #include <linux/gpio/consumer.h>
 
-#include "physmap-bt1-rom.h"
 #include "physmap-gemini.h"
 #include "physmap-ixp4xx.h"
 #include "physmap-versatile.h"
@@ -365,10 +364,6 @@ static int physmap_flash_of_init(struct platform_device *dev)
 		info->maps[i].bankwidth = bankwidth;
 		info->maps[i].device_node = dp;
 
-		err = of_flash_probe_bt1_rom(dev, dp, &info->maps[i]);
-		if (err)
-			return err;
-
 		err = of_flash_probe_gemini(dev, dp, &info->maps[i]);
 		if (err)
 			return err;
diff --git a/drivers/mtd/maps/physmap-gemini.c b/drivers/mtd/maps/physmap-gemini.c
index 9d3b4bf84a1ad..1c34b4ef77ea3 100644
--- a/drivers/mtd/maps/physmap-gemini.c
+++ b/drivers/mtd/maps/physmap-gemini.c
@@ -181,7 +181,7 @@ int of_flash_probe_gemini(struct platform_device *pdev,
 		dev_err(dev, "no enabled pin control state\n");
 
 	gf->disabled_state = pinctrl_lookup_state(gf->p, "disabled");
-	if (IS_ERR(gf->enabled_state)) {
+	if (IS_ERR(gf->disabled_state)) {
 		dev_err(dev, "no disabled pin control state\n");
 	} else {
 		ret = pinctrl_select_state(gf->p, gf->disabled_state);
diff --git a/drivers/mtd/mtd_virt_concat.c b/drivers/mtd/mtd_virt_concat.c
new file mode 100644
index 0000000000000..37075ead0f337
--- /dev/null
+++ b/drivers/mtd/mtd_virt_concat.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Virtual concat MTD device driver
+ *
+ * Copyright (C) 2018 Bernhard Frauendienst
+ * Author: Bernhard Frauendienst <kernel@nospam.obeliks.de>
+ */
+
+#include <linux/device.h>
+#include <linux/mtd/mtd.h>
+#include "mtdcore.h"
+#include <linux/mtd/partitions.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/mtd/concat.h>
+
+#define CONCAT_PROP "part-concat-next"
+#define CONCAT_POSTFIX "concat"
+#define MIN_DEV_PER_CONCAT 1
+
+static LIST_HEAD(concat_node_list);
+
+/**
+ * struct mtd_virt_concat_node - components of a concatenation
+ * @head: List handle
+ * @count: Number of nodes
+ * @nodes: Pointer to the nodes (partitions) to concatenate
+ * @concat: Concatenation container
+ */
+struct mtd_virt_concat_node {
+	struct list_head head;
+	unsigned int count;
+	struct mtd_concat *concat;
+	struct device_node *nodes[] __counted_by(count);
+};
+
+/**
+ * mtd_is_part_concat - Check if the device is already part
+ *                       of a concatenated device
+ * @dev:        pointer to 'device_node'
+ *
+ * Return: true if the device is already part of a concatenation,
+ *         false otherwise.
+ */
+static bool mtd_is_part_concat(struct device_node *dev)
+{
+	struct mtd_virt_concat_node *item;
+	int idx;
+
+	list_for_each_entry(item, &concat_node_list, head) {
+		for (idx = 0; idx < item->count; idx++) {
+			if (item->nodes[idx] == dev)
+				return true;
+		}
+	}
+	return false;
+}
+
+static void mtd_virt_concat_put_mtd_devices(struct mtd_concat *concat)
+{
+	int i;
+
+	for (i = 0; i < concat->num_subdev; i++)
+		put_mtd_device(concat->subdev[i]);
+}
+
+void mtd_virt_concat_destroy_joins(void)
+{
+	struct mtd_virt_concat_node *item, *tmp;
+	struct mtd_info *mtd;
+
+	list_for_each_entry_safe(item, tmp, &concat_node_list, head) {
+		mtd = &item->concat->mtd;
+		if (item->concat) {
+			mtd_device_unregister(mtd);
+			kfree(mtd->name);
+			mtd_concat_destroy(mtd);
+			mtd_virt_concat_put_mtd_devices(item->concat);
+		}
+	}
+}
+
+/**
+ * mtd_virt_concat_destroy - Destroy the concat that includes the mtd object
+ * @mtd:        pointer to 'mtd_info'
+ *
+ * Return: 0 on success, -error otherwise.
+ */
+int mtd_virt_concat_destroy(struct mtd_info *mtd)
+{
+	struct mtd_info *child, *master = mtd_get_master(mtd);
+	struct mtd_virt_concat_node *item, *tmp;
+	struct mtd_concat *concat;
+	int idx, ret = 0;
+	bool is_mtd_found;
+
+	list_for_each_entry_safe(item, tmp, &concat_node_list, head) {
+		is_mtd_found = false;
+
+		/* Find the concat item that hold the mtd device */
+		for (idx = 0; idx < item->count; idx++) {
+			if (item->nodes[idx] == mtd->dev.of_node) {
+				is_mtd_found = true;
+				break;
+			}
+		}
+		if (!is_mtd_found)
+			continue;
+		concat = item->concat;
+
+		/*
+		 * Since this concatenated device is being removed, retrieve
+		 * all MTD devices that are part of it and register them
+		 * individually.
+		 */
+		for (idx = 0; idx < concat->num_subdev; idx++) {
+			child = concat->subdev[idx];
+			if (child->dev.of_node != mtd->dev.of_node) {
+				ret = add_mtd_device(child);
+				if (ret)
+					goto out;
+			}
+		}
+		/* Destroy the concat */
+		if (concat->mtd.name) {
+			del_mtd_device(&concat->mtd);
+			kfree(concat->mtd.name);
+			mtd_concat_destroy(&concat->mtd);
+			mtd_virt_concat_put_mtd_devices(item->concat);
+		}
+
+		for (idx = 0; idx < item->count; idx++)
+			of_node_put(item->nodes[idx]);
+
+		kfree(item);
+	}
+	return 0;
+out:
+	mutex_lock(&master->master.partitions_lock);
+	list_del(&child->part.node);
+	mutex_unlock(&master->master.partitions_lock);
+	kfree(mtd->name);
+	kfree(mtd);
+
+	return ret;
+}
+
+/**
+ * mtd_virt_concat_create_item - Create a concat item
+ * @parts:        pointer to 'device_node'
+ * @count:        number of mtd devices that make up
+ *                the concatenated device.
+ *
+ * Return: 0 on success, -error otherwise.
+ */
+static int mtd_virt_concat_create_item(struct device_node *parts,
+				       unsigned int count)
+{
+	struct mtd_virt_concat_node *item;
+	struct mtd_concat *concat;
+	int i;
+
+	for (i = 0; i < (count - 1); i++) {
+		if (mtd_is_part_concat(of_parse_phandle(parts, CONCAT_PROP, i)))
+			return 0;
+	}
+
+	item = kzalloc_flex(*item, nodes, count, GFP_KERNEL);
+	if (!item)
+		return -ENOMEM;
+
+	item->count = count;
+
+	/*
+	 * The partition in which "part-concat-next" property
+	 * is defined is the first device in the list of concat
+	 * devices.
+	 */
+	item->nodes[0] = parts;
+
+	for (i = 1; i < count; i++)
+		item->nodes[i] = of_parse_phandle(parts, CONCAT_PROP, (i - 1));
+
+	concat = kzalloc_flex(*concat, subdev, count, GFP_KERNEL);
+	if (!concat) {
+		kfree(item);
+		return -ENOMEM;
+	}
+
+	item->concat = concat;
+
+	list_add_tail(&item->head, &concat_node_list);
+
+	return 0;
+}
+
+void mtd_virt_concat_destroy_items(void)
+{
+	struct mtd_virt_concat_node *item, *temp;
+	int i;
+
+	list_for_each_entry_safe(item, temp, &concat_node_list, head) {
+		for (i = 0; i < item->count; i++)
+			of_node_put(item->nodes[i]);
+
+		kfree(item);
+	}
+}
+
+/**
+ * mtd_virt_concat_add - Add a mtd device to the concat list
+ * @mtd:        pointer to 'mtd_info'
+ *
+ * Return: true on success, false otherwise.
+ */
+bool mtd_virt_concat_add(struct mtd_info *mtd)
+{
+	struct mtd_virt_concat_node *item;
+	struct mtd_concat *concat;
+	int idx;
+
+	list_for_each_entry(item, &concat_node_list, head) {
+		concat = item->concat;
+		for (idx = 0; idx < item->count; idx++) {
+			if (item->nodes[idx] == mtd->dev.of_node) {
+				concat->subdev[concat->num_subdev++] = mtd;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+/**
+ * mtd_virt_concat_node_create - List all the concatenations found in DT
+ *
+ * Return: 0 on success, -error otherwise.
+ */
+int mtd_virt_concat_node_create(void)
+{
+	struct device_node *parts = NULL;
+	int ret = 0, count = 0;
+
+	/* List all the concatenations found in DT */
+	do {
+		parts = of_find_node_with_property(parts, CONCAT_PROP);
+		if (!of_device_is_available(parts))
+			continue;
+
+		if (mtd_is_part_concat(parts))
+			continue;
+
+		count = of_count_phandle_with_args(parts, CONCAT_PROP, NULL);
+		if (count < MIN_DEV_PER_CONCAT)
+			continue;
+
+		/*
+		 * The partition in which "part-concat-next" property is defined
+		 * is also part of the concat device, so increament count by 1.
+		 */
+		count++;
+
+		ret = mtd_virt_concat_create_item(parts, count);
+		if (ret) {
+			of_node_put(parts);
+			goto destroy_items;
+		}
+	} while (parts);
+
+	return ret;
+
+destroy_items:
+	mtd_virt_concat_destroy_items();
+
+	return ret;
+}
+
+/**
+ * mtd_virt_concat_create_join - Create and register the concatenated
+ *                                 MTD device.
+ *
+ * Return: 0 on success, -error otherwise.
+ */
+int mtd_virt_concat_create_join(void)
+{
+	struct mtd_virt_concat_node *item;
+	struct mtd_concat *concat;
+	struct mtd_info *mtd;
+	ssize_t name_sz;
+	int ret, idx;
+	char *name;
+
+	list_for_each_entry(item, &concat_node_list, head) {
+		concat = item->concat;
+		/*
+		 * Check if item->count != concat->num_subdev, it indicates
+		 * that the MTD information for all devices included in the
+		 * concatenation are not handy, concat MTD device can't be
+		 * created hence switch to next concat device.
+		 */
+		if (item->count != concat->num_subdev) {
+			continue;
+		} else {
+			/* Calculate the legth of the name of the virtual device */
+			for (idx = 0, name_sz = 0; idx < concat->num_subdev; idx++)
+				name_sz += (strlen(concat->subdev[idx]->name) + 1);
+			name_sz += strlen(CONCAT_POSTFIX);
+			name = kmalloc(name_sz + 1, GFP_KERNEL);
+			if (!name) {
+				mtd_virt_concat_put_mtd_devices(concat);
+				return -ENOMEM;
+			}
+
+			ret = 0;
+			for (idx = 0; idx < concat->num_subdev; idx++) {
+				ret += sprintf((name + ret), "%s-",
+					       concat->subdev[idx]->name);
+			}
+			sprintf((name + ret), CONCAT_POSTFIX);
+
+			if (concat->mtd.name) {
+				ret = memcmp(concat->mtd.name, name, name_sz);
+				if (ret == 0)
+					continue;
+			}
+			mtd = mtd_concat_create(concat->subdev, concat->num_subdev, name);
+			if (!mtd) {
+				kfree(name);
+				return -ENXIO;
+			}
+			concat->mtd = *mtd;
+			/* Arbitrary set the first device as parent */
+			concat->mtd.dev.parent = concat->subdev[0]->dev.parent;
+			concat->mtd.dev = concat->subdev[0]->dev;
+
+			/* Add the mtd device */
+			ret = add_mtd_device(&concat->mtd);
+			if (ret)
+				goto destroy_concat;
+		}
+	}
+
+	return 0;
+
+destroy_concat:
+	mtd_concat_destroy(mtd);
+
+	return ret;
+}
diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c
index 55a43682c567e..bf01e6ac72931 100644
--- a/drivers/mtd/mtdchar.c
+++ b/drivers/mtd/mtdchar.c
@@ -1376,27 +1376,12 @@ static unsigned mtdchar_mmap_capabilities(struct file *file)
 /*
  * set up a mapping for shared memory segments
  */
-static int mtdchar_mmap(struct file *file, struct vm_area_struct *vma)
+static int mtdchar_mmap_prepare(struct vm_area_desc *desc)
 {
 #ifdef CONFIG_MMU
-	struct mtd_file_info *mfi = file->private_data;
-	struct mtd_info *mtd = mfi->mtd;
-	struct map_info *map = mtd->priv;
-
-        /* This is broken because it assumes the MTD device is map-based
-	   and that mtd->priv is a valid struct map_info.  It should be
-	   replaced with something that uses the mtd_get_unmapped_area()
-	   operation properly. */
-	if (0 /*mtd->type == MTD_RAM || mtd->type == MTD_ROM*/) {
-#ifdef pgprot_noncached
-		if (file->f_flags & O_DSYNC || map->phys >= __pa(high_memory))
-			vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-#endif
-		return vm_iomap_memory(vma, map->phys, map->size);
-	}
 	return -ENODEV;
 #else
-	return vma->vm_flags & VM_SHARED ? 0 : -EACCES;
+	return vma_desc_test(desc, VMA_SHARED_BIT) ? 0 : -EACCES;
 #endif
 }
 
@@ -1411,7 +1396,7 @@ static const struct file_operations mtd_fops = {
 #endif
 	.open		= mtdchar_open,
 	.release	= mtdchar_close,
-	.mmap		= mtdchar_mmap,
+	.mmap_prepare	= mtdchar_mmap_prepare,
 #ifndef CONFIG_MMU
 	.get_unmapped_area = mtdchar_get_unmapped_area,
 	.mmap_capabilities = mtdchar_mmap_capabilities,
diff --git a/drivers/mtd/mtdconcat.c b/drivers/mtd/mtdconcat.c
index 9eb5d919d9bad..c97167d51fe2d 100644
--- a/drivers/mtd/mtdconcat.c
+++ b/drivers/mtd/mtdconcat.c
@@ -20,18 +20,6 @@
 
 #include <asm/div64.h>
 
-/*
- * Our storage structure:
- * Subdev points to an array of pointers to struct mtd_info objects
- * which is allocated along with this structure
- *
- */
-struct mtd_concat {
-	struct mtd_info mtd;
-	int num_subdev;
-	struct mtd_info **subdev;
-};
-
 /*
  * how to calculate the size required for the above structure,
  * including the pointer array subdev points to:
@@ -639,7 +627,6 @@ struct mtd_info *mtd_concat_create(struct mtd_info *subdev[],	/* subdevices to c
 				   const char *name)
 {				/* name for the new device   */
 	int i;
-	size_t size;
 	struct mtd_concat *concat;
 	struct mtd_info *subdev_master = NULL;
 	uint32_t max_erasesize, curr_erasesize;
@@ -652,15 +639,13 @@ struct mtd_info *mtd_concat_create(struct mtd_info *subdev[],	/* subdevices to c
 	printk(KERN_NOTICE "into device \"%s\"\n", name);
 
 	/* allocate the device structure */
-	size = SIZEOF_STRUCT_MTD_CONCAT(num_devs);
-	concat = kzalloc(size, GFP_KERNEL);
+	concat = kzalloc_flex(*concat, subdev, num_devs, GFP_KERNEL);
 	if (!concat) {
 		printk
 		    ("memory allocation error while creating concatenated device \"%s\"\n",
 		     name);
 		return NULL;
 	}
-	concat->subdev = (struct mtd_info **) (concat + 1);
 
 	/*
 	 * Set up the new "super" device's MTD object structure, check for
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 64808493b4f5e..5765377746288 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -34,6 +34,7 @@
 
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
+#include <linux/mtd/concat.h>
 
 #include "mtdcore.h"
 
@@ -1120,6 +1121,12 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 			goto out;
 	}
 
+	if (IS_REACHABLE(CONFIG_MTD_VIRT_CONCAT)) {
+		ret = mtd_virt_concat_node_create();
+		if (ret < 0)
+			goto out;
+	}
+
 	/* Prefer parsed partitions over driver-provided fallback */
 	ret = parse_mtd_partitions(mtd, types, parser_data);
 	if (ret == -EPROBE_DEFER)
@@ -1137,6 +1144,11 @@ int mtd_device_parse_register(struct mtd_info *mtd, const char * const *types,
 	if (ret)
 		goto out;
 
+	if (IS_REACHABLE(CONFIG_MTD_VIRT_CONCAT)) {
+		ret = mtd_virt_concat_create_join();
+		if (ret < 0)
+			goto out;
+	}
 	/*
 	 * FIXME: some drivers unfortunately call this function more than once.
 	 * So we have to check if we've already assigned the reboot notifier.
@@ -1186,6 +1198,11 @@ int mtd_device_unregister(struct mtd_info *master)
 	nvmem_unregister(master->otp_user_nvmem);
 	nvmem_unregister(master->otp_factory_nvmem);
 
+	if (IS_REACHABLE(CONFIG_MTD_VIRT_CONCAT)) {
+		err = mtd_virt_concat_destroy(master);
+		if (err)
+			return err;
+	}
 	err = del_mtd_partitions(master);
 	if (err)
 		return err;
@@ -2621,6 +2638,10 @@ static int __init init_mtd(void)
 
 static void __exit cleanup_mtd(void)
 {
+	if (IS_REACHABLE(CONFIG_MTD_VIRT_CONCAT)) {
+		mtd_virt_concat_destroy_joins();
+		mtd_virt_concat_destroy_items();
+	}
 	debugfs_remove_recursive(dfs_dir_mtd);
 	cleanup_mtdchar();
 	if (proc_mtd)
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index e016cfbc72246..795a94e6b482f 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -18,6 +18,7 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/mtd/concat.h>
 
 #include "mtdcore.h"
 
@@ -409,6 +410,11 @@ int add_mtd_partitions(struct mtd_info *parent,
 			goto err_del_partitions;
 		}
 
+		if (IS_REACHABLE(CONFIG_MTD_VIRT_CONCAT)) {
+			if (mtd_virt_concat_add(child))
+				continue;
+		}
+
 		mutex_lock(&master->master.partitions_lock);
 		list_add_tail(&child->part.node, &parent->partitions);
 		mutex_unlock(&master->master.partitions_lock);
diff --git a/drivers/mtd/nand/ecc-realtek.c b/drivers/mtd/nand/ecc-realtek.c
index 0046da37ea3e5..7d003fd72027a 100644
--- a/drivers/mtd/nand/ecc-realtek.c
+++ b/drivers/mtd/nand/ecc-realtek.c
@@ -17,10 +17,12 @@
  * - BCH12 : Generate 20 ECC bytes from 512 data bytes plus 6 free bytes
  *
  * It can run for arbitrary NAND flash chips with different block and OOB sizes. Currently there
- * are only two known devices in the wild that have NAND flash and make use of this ECC engine
- * (Linksys LGS328C & LGS352C). To keep compatibility with vendor firmware, new modes can only
- * be added when new data layouts have been analyzed. For now allow BCH6 on flash with 2048 byte
- * blocks and 64 bytes oob.
+ * are a few known devices in the wild that make use of this ECC engine
+ * (Linksys LGS328C, LGS352C & Netlink HG323DAC). To keep compatibility with vendor firmware,
+ * new modes can only be added when new data layouts have been analyzed. For now allow BCH6 on
+ * flash with 2048 byte blocks and at least 64 bytes oob. Some vendors make use of
+ * 128 bytes OOB NAND chips (e.g. Macronix MX35LF1G24AD) but only use BCH6 and thus the first
+ * 64 bytes of the OOB area. In this case the engine leaves any extra bytes unused.
  *
  * This driver aligns with kernel ECC naming conventions. Neverthless a short notice on the
  * Realtek naming conventions for the different structures in the OOB area.
@@ -39,7 +41,7 @@
  */
 
 #define RTL_ECC_ALLOWED_PAGE_SIZE 	2048
-#define RTL_ECC_ALLOWED_OOB_SIZE	64
+#define RTL_ECC_ALLOWED_MIN_OOB_SIZE	64
 #define RTL_ECC_ALLOWED_STRENGTH	6
 
 #define RTL_ECC_BLOCK_SIZE 		512
@@ -310,10 +312,10 @@ static int rtl_ecc_check_support(struct nand_device *nand)
 	struct mtd_info *mtd = nanddev_to_mtd(nand);
 	struct device *dev = nand->ecc.engine->dev;
 
-	if (mtd->oobsize != RTL_ECC_ALLOWED_OOB_SIZE ||
+	if (mtd->oobsize < RTL_ECC_ALLOWED_MIN_OOB_SIZE ||
 	    mtd->writesize != RTL_ECC_ALLOWED_PAGE_SIZE) {
-		dev_err(dev, "only flash geometry data=%d, oob=%d supported\n",
-			RTL_ECC_ALLOWED_PAGE_SIZE, RTL_ECC_ALLOWED_OOB_SIZE);
+		dev_err(dev, "only flash geometry data=%d, oob>=%d supported\n",
+			RTL_ECC_ALLOWED_PAGE_SIZE, RTL_ECC_ALLOWED_MIN_OOB_SIZE);
 		return -EINVAL;
 	}
 
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
index 0427d76f45d0a..5b9dadd5405e3 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
@@ -2350,14 +2350,12 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip,
 	for (i = 0; i < ctrl->max_oob; i += 4)
 		oob_reg_write(ctrl, i, 0xffffffff);
 
-	if (mtd->oops_panic_write)
+	if (mtd->oops_panic_write) {
 		/* switch to interrupt polling and PIO mode */
 		disable_ctrl_irqs(ctrl);
-
-	if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) {
+	} else if (use_dma(ctrl) && (has_edu(ctrl) || !oob) && flash_dma_buf_ok(buf)) {
 		if (ctrl->dma_trans(host, addr, (u32 *)buf, oob, mtd->writesize,
 				    CMD_PROGRAM_PAGE))
-
 			ret = -EIO;
 
 		goto out;
diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c
index 99135ec230105..d53b35a8b3cb2 100644
--- a/drivers/mtd/nand/raw/cadence-nand-controller.c
+++ b/drivers/mtd/nand/raw/cadence-nand-controller.c
@@ -3133,7 +3133,7 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
 						  sizeof(*cdns_ctrl->cdma_desc),
 						  &cdns_ctrl->dma_cdma_desc,
 						  GFP_KERNEL);
-	if (!cdns_ctrl->dma_cdma_desc)
+	if (!cdns_ctrl->cdma_desc)
 		return -ENOMEM;
 
 	cdns_ctrl->buf_size = SZ_16K;
diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c
index 65a36d5de742f..c4018bc59670d 100644
--- a/drivers/mtd/nand/raw/cafe_nand.c
+++ b/drivers/mtd/nand/raw/cafe_nand.c
@@ -837,9 +837,10 @@ static const struct pci_device_id cafe_nand_tbl[] = {
 
 MODULE_DEVICE_TABLE(pci, cafe_nand_tbl);
 
-static int cafe_nand_resume(struct pci_dev *pdev)
+static int cafe_nand_resume(struct device *dev)
 {
 	uint32_t ctrl;
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct mtd_info *mtd = pci_get_drvdata(pdev);
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct cafe_priv *cafe = nand_get_controller_data(chip);
@@ -877,12 +878,14 @@ static int cafe_nand_resume(struct pci_dev *pdev)
 	return 0;
 }
 
+static DEFINE_SIMPLE_DEV_PM_OPS(cafe_nand_ops, NULL, cafe_nand_resume);
+
 static struct pci_driver cafe_nand_pci_driver = {
 	.name = "CAFÉ NAND",
 	.id_table = cafe_nand_tbl,
 	.probe = cafe_nand_probe,
 	.remove = cafe_nand_remove,
-	.resume = cafe_nand_resume,
+	.driver.pm = &cafe_nand_ops,
 };
 
 module_pci_driver(cafe_nand_pci_driver);
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index dd88b22a91bd1..fad0334f759dd 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -7,6 +7,7 @@
  * Author: Dipen Dudhat <Dipen.Dudhat@freescale.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
@@ -863,7 +864,14 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv)
 
 	/* Fill in fsl_ifc_mtd structure */
 	mtd->dev.parent = priv->dev;
-	nand_set_flash_node(chip, priv->dev->of_node);
+
+	struct device_node *np __free(device_node) =
+		of_get_next_child_with_prefix(priv->dev->of_node, NULL, "nand");
+
+	if (np)
+		nand_set_flash_node(chip, np);
+	else
+		nand_set_flash_node(chip, priv->dev->of_node);
 
 	/* fill in nand_chip structure */
 	/* set up function call table */
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
index 51f595fbc834e..c1f766cb225aa 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.
  * Copyright (C) 2008 Embedded Alley Solutions, Inc.
  */
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
@@ -2688,7 +2689,15 @@ static int gpmi_nand_init(struct gpmi_nand_data *this)
 
 	/* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
 	nand_set_controller_data(chip, this);
-	nand_set_flash_node(chip, this->pdev->dev.of_node);
+
+	struct device_node *np __free(device_node) =
+		of_get_next_child_with_prefix(this->pdev->dev.of_node, NULL, "nand");
+
+	if (np)
+		nand_set_flash_node(chip, np);
+	else
+		nand_set_flash_node(chip, this->pdev->dev.of_node);
+
 	chip->legacy.block_markbad = gpmi_block_markbad;
 	chip->badblock_pattern	= &gpmi_bbt_descr;
 	chip->options		|= NAND_NO_SUBPAGE_WRITE;
diff --git a/drivers/mtd/nand/raw/mxc_nand.c b/drivers/mtd/nand/raw/mxc_nand.c
index 8c56b685bf91a..4d8b92e7e6725 100644
--- a/drivers/mtd/nand/raw/mxc_nand.c
+++ b/drivers/mtd/nand/raw/mxc_nand.c
@@ -4,6 +4,7 @@
  * Copyright 2008 Sascha Hauer, kernel@pengutronix.de
  */
 
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/init.h>
@@ -1714,7 +1715,14 @@ static int mxcnd_probe(struct platform_device *pdev)
 	this->legacy.chip_delay = 5;
 
 	nand_set_controller_data(this, host);
-	nand_set_flash_node(this, pdev->dev.of_node);
+
+	struct device_node *np __free(device_node) =
+		of_get_next_child_with_prefix(pdev->dev.of_node, NULL, "nand");
+
+	if (np)
+		nand_set_flash_node(this, np);
+	else
+		nand_set_flash_node(this, pdev->dev.of_node);
 
 	host->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(host->clk))
diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c
index 38429363251ca..d6d3e17ab4079 100644
--- a/drivers/mtd/nand/raw/nand_base.c
+++ b/drivers/mtd/nand/raw/nand_base.c
@@ -43,6 +43,7 @@
 #include <linux/mtd/partitions.h>
 #include <linux/of.h>
 #include <linux/gpio/consumer.h>
+#include <linux/cleanup.h>
 
 #include "internals.h"
 
@@ -4704,16 +4705,16 @@ static void nand_resume(struct mtd_info *mtd)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 
-	mutex_lock(&chip->lock);
-	if (chip->suspended) {
-		if (chip->ops.resume)
-			chip->ops.resume(chip);
-		chip->suspended = 0;
-	} else {
-		pr_err("%s called for a chip which is not in suspended state\n",
-			__func__);
+	scoped_guard(mutex, &chip->lock) {
+		if (chip->suspended) {
+			if (chip->ops.resume)
+				chip->ops.resume(chip);
+			chip->suspended = 0;
+		} else {
+			pr_err("%s called for a chip which is not in suspended state\n",
+				__func__);
+		}
 	}
-	mutex_unlock(&chip->lock);
 
 	wake_up_all(&chip->resume_wq);
 }
@@ -4737,11 +4738,16 @@ static void nand_shutdown(struct mtd_info *mtd)
 static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
+	int ret;
 
 	if (!chip->ops.lock_area)
 		return -ENOTSUPP;
 
-	return chip->ops.lock_area(chip, ofs, len);
+	nand_get_device(chip);
+	ret = chip->ops.lock_area(chip, ofs, len);
+	nand_release_device(chip);
+
+	return ret;
 }
 
 /**
@@ -4753,11 +4759,16 @@ static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
+	int ret;
 
 	if (!chip->ops.unlock_area)
 		return -ENOTSUPP;
 
-	return chip->ops.unlock_area(chip, ofs, len);
+	nand_get_device(chip);
+	ret = chip->ops.unlock_area(chip, ofs, len);
+	nand_release_device(chip);
+
+	return ret;
 }
 
 /* Set default functions */
diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c
index 947fd86ac5fa2..f2c65eb7a8d9a 100644
--- a/drivers/mtd/nand/raw/pl35x-nand-controller.c
+++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c
@@ -862,6 +862,9 @@ static int pl35x_nfc_setup_interface(struct nand_chip *chip, int cs,
 			  PL35X_SMC_NAND_TAR_CYCLES(tmgs.t_ar) |
 			  PL35X_SMC_NAND_TRR_CYCLES(tmgs.t_rr);
 
+	writel(plnand->timings, nfc->conf_regs + PL35X_SMC_CYCLES);
+	pl35x_smc_update_regs(nfc);
+
 	return 0;
 }
 
diff --git a/drivers/mtd/parsers/cmdlinepart.c b/drivers/mtd/parsers/cmdlinepart.c
index 504e5fa2b45b0..4caf1b3804f29 100644
--- a/drivers/mtd/parsers/cmdlinepart.c
+++ b/drivers/mtd/parsers/cmdlinepart.c
@@ -50,9 +50,9 @@
 
 struct cmdline_mtd_partition {
 	struct cmdline_mtd_partition *next;
-	char *mtd_id;
 	int num_parts;
 	struct mtd_partition *parts;
+	char mtd_id[];
 };
 
 /* mtdpart_setup() parses into here */
@@ -289,7 +289,6 @@ static int mtdpart_setup_real(char *s)
 		/* enter results */
 		this_mtd->parts = parts;
 		this_mtd->num_parts = num_parts;
-		this_mtd->mtd_id = (char*)(this_mtd + 1);
 		strscpy(this_mtd->mtd_id, mtd_id, mtd_id_len + 1);
 
 		/* link into chain */
diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c
index 0029bda165bde..262c4221d23f7 100644
--- a/drivers/mtd/parsers/ofpart_core.c
+++ b/drivers/mtd/parsers/ofpart_core.c
@@ -75,7 +75,7 @@ static int parse_fixed_partitions(struct mtd_info *master,
 			dedicated = false;
 		}
 	} else { /* Partition */
-		ofpart_node = mtd_node;
+		ofpart_node = of_node_get(mtd_node);
 	}
 
 	of_id = of_match_node(parse_ofpart_match_table, ofpart_node);
@@ -195,11 +195,11 @@ static int parse_fixed_partitions(struct mtd_info *master,
 ofpart_fail:
 	pr_err("%s: error parsing ofpart partition %pOF (%pOF)\n",
 	       master->name, pp, mtd_node);
+	of_node_put(pp);
 	ret = -EINVAL;
 ofpart_none:
 	if (dedicated)
 		of_node_put(ofpart_node);
-	of_node_put(pp);
 	kfree(parts);
 	return ret;
 }
diff --git a/drivers/mtd/parsers/redboot.c b/drivers/mtd/parsers/redboot.c
index 558905160ddba..bf162c44eafe0 100644
--- a/drivers/mtd/parsers/redboot.c
+++ b/drivers/mtd/parsers/redboot.c
@@ -270,9 +270,9 @@ static int parse_redboot_partitions(struct mtd_info *master,
 
 		strcpy(names, fl->img->name);
 #ifdef CONFIG_MTD_REDBOOT_PARTS_READONLY
-		if (!memcmp(names, "RedBoot", 8) ||
-		    !memcmp(names, "RedBoot config", 15) ||
-		    !memcmp(names, "FIS directory", 14)) {
+		if (!strcmp(names, "RedBoot") ||
+		    !strcmp(names, "RedBoot config") ||
+		    !strcmp(names, "FIS directory")) {
 			parts[i].mask_flags = MTD_WRITEABLE;
 		}
 #endif
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index 8ffeb41c3e08a..5dd0b3cb52503 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -2345,15 +2345,15 @@ int spi_nor_hwcaps_pp2cmd(u32 hwcaps)
 }
 
 /**
- * spi_nor_spimem_check_op - check if the operation is supported
- *                           by controller
+ * spi_nor_spimem_check_read_pp_op - check if a read or a page program operation is
+ *                                   supported by controller
  *@nor:        pointer to a 'struct spi_nor'
  *@op:         pointer to op template to be checked
  *
  * Returns 0 if operation is supported, -EOPNOTSUPP otherwise.
  */
-static int spi_nor_spimem_check_op(struct spi_nor *nor,
-				   struct spi_mem_op *op)
+static int spi_nor_spimem_check_read_pp_op(struct spi_nor *nor,
+					   struct spi_mem_op *op)
 {
 	/*
 	 * First test with 4 address bytes. The opcode itself might
@@ -2393,10 +2393,10 @@ static int spi_nor_spimem_check_readop(struct spi_nor *nor,
 	/* convert the dummy cycles to the number of bytes */
 	op.dummy.nbytes = (read->num_mode_clocks + read->num_wait_states) *
 			  op.dummy.buswidth / 8;
-	if (spi_nor_protocol_is_dtr(nor->read_proto))
+	if (spi_nor_protocol_is_dtr(read->proto))
 		op.dummy.nbytes *= 2;
 
-	return spi_nor_spimem_check_op(nor, &op);
+	return spi_nor_spimem_check_read_pp_op(nor, &op);
 }
 
 /**
@@ -2414,7 +2414,7 @@ static int spi_nor_spimem_check_pp(struct spi_nor *nor,
 
 	spi_nor_spimem_setup_op(nor, &op, pp->proto);
 
-	return spi_nor_spimem_check_op(nor, &op);
+	return spi_nor_spimem_check_read_pp_op(nor, &op);
 }
 
 /**
@@ -2466,7 +2466,7 @@ spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps)
 
 		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
 
-		if (spi_nor_spimem_check_op(nor, &op))
+		if (!spi_mem_supports_op(nor->spimem, &op))
 			nor->flags |= SNOR_F_NO_READ_CR;
 	}
 }
diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h
index 16b382d4f04f2..e838c40a25897 100644
--- a/drivers/mtd/spi-nor/core.h
+++ b/drivers/mtd/spi-nor/core.h
@@ -413,7 +413,7 @@ struct spi_nor_flash_parameter {
  *                   number of dummy cycles in read register ops.
  * @smpt_map_id: called after map ID in SMPT table has been determined for the
  *               case the map ID is wrong and needs to be fixed.
- * @post_sfdp: called after SFDP has been parsed (is also called for SPI NORs
+ * @post_sfdp: called after SFDP has been parsed (is not called for SPI NORs
  *             that do not support RDSFDP). Typically used to tweak various
  *             parameters that could not be extracted by other means (i.e.
  *             when information provided by the SFDP/flash_info tables are
diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c
index 88033384a71e5..b2b473501d023 100644
--- a/drivers/mtd/spi-nor/micron-st.c
+++ b/drivers/mtd/spi-nor/micron-st.c
@@ -167,6 +167,16 @@ static int mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor)
 				  0, 20, SPINOR_OP_MT_DTR_RD,
 				  SNOR_PROTO_8_8_8_DTR);
 
+	/*
+	 * Some batches of mt35xu512aba do not contain the OCT DTR command
+	 * information, but do support OCT DTR mode. Add the settings for
+	 * SNOR_CMD_PP_8_8_8_DTR here. This also makes sure the flash can switch
+	 * to OCT DTR mode.
+	 */
+	nor->params->hwcaps.mask |= SNOR_HWCAPS_PP_8_8_8_DTR;
+	spi_nor_set_pp_settings(&nor->params->page_programs[SNOR_CMD_PP_8_8_8_DTR],
+				SPINOR_OP_PP_4B, SNOR_PROTO_8_8_8_DTR);
+
 	nor->cmd_ext_type = SPI_NOR_EXT_REPEAT;
 	nor->params->rdsr_dummy = 8;
 	nor->params->rdsr_addr_nbytes = 0;
diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c
index 9b07f83aeac76..e67a81dbb6bf6 100644
--- a/drivers/mtd/spi-nor/swp.c
+++ b/drivers/mtd/spi-nor/swp.c
@@ -28,8 +28,10 @@ static u8 spi_nor_get_sr_tb_mask(struct spi_nor *nor)
 {
 	if (nor->flags & SNOR_F_HAS_SR_TB_BIT6)
 		return SR_TB_BIT6;
-	else
+	else if (nor->flags & SNOR_F_HAS_SR_TB)
 		return SR_TB_BIT5;
+	else
+		return 0;
 }
 
 static u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor)
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index af466cd83ae07..95548d5aa9308 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -160,6 +160,7 @@ struct ubi_vid_io_buf {
  * struct ubi_wl_entry - wear-leveling entry.
  * @u.rb: link in the corresponding (free/used) RB-tree
  * @u.list: link in the protection queue
+ * @u: union of rb_node and list_head
  * @ec: erase counter
  * @pnum: physical eraseblock number
  *
@@ -282,6 +283,7 @@ struct ubi_eba_leb_desc {
  * @writers: number of users holding this volume in read-write mode
  * @exclusive: whether somebody holds this volume in exclusive mode
  * @metaonly: whether somebody is altering only meta data of this volume
+ * @is_dead: prevents taking a reference to the volume
  *
  * @reserved_pebs: how many physical eraseblocks are reserved for this volume
  * @vol_type: volume type (%UBI_DYNAMIC_VOLUME or %UBI_STATIC_VOLUME)
@@ -449,6 +451,7 @@ struct ubi_debug_info {
  *                @vol->eba_tbl.
  * @ref_count: count of references on the UBI device
  * @image_seq: image sequence number recorded on EC headers
+ * @is_dead: prevents taking a reference to the volume
  *
  * @rsvd_pebs: count of reserved physical eraseblocks
  * @avail_pebs: count of available physical eraseblocks
@@ -741,7 +744,7 @@ struct ubi_ainf_volume {
  * @highest_vol_id: highest volume ID
  * @is_empty: flag indicating whether the MTD device is empty or not
  * @force_full_scan: flag indicating whether we need to do a full scan and drop
-		     all existing Fastmap data structures
+ *		     all existing Fastmap data structures
  * @min_ec: lowest erase counter value
  * @max_ec: highest erase counter value
  * @max_sqnum: highest sequence number value
@@ -750,7 +753,7 @@ struct ubi_ainf_volume {
  * @ec_count: a temporary variable used when calculating @mean_ec
  * @aeb_slab_cache: slab cache for &struct ubi_ainf_peb objects
  * @ech: temporary EC header. Only available during scan
- * @vidh: temporary VID buffer. Only available during scan
+ * @vidb: temporary VID buffer. Only available during scan
  *
  * This data structure contains the result of attaching an MTD device and may
  * be used by other UBI sub-systems to build final UBI data structures, further
@@ -1090,7 +1093,7 @@ static inline void ubi_init_vid_buf(const struct ubi_device *ubi,
 }
 
 /**
- * ubi_init_vid_buf - Allocate a VID buffer
+ * ubi_alloc_vid_buf - Allocate a VID buffer
  * @ubi: the UBI device
  * @gfp_flags: GFP flags to use for the allocation
  */
diff --git a/drivers/mux/Kconfig b/drivers/mux/Kconfig
index c68132e38138e..6d17dfa25dad6 100644
--- a/drivers/mux/Kconfig
+++ b/drivers/mux/Kconfig
@@ -4,10 +4,21 @@
 #
 
 config MULTIPLEXER
-	tristate
+	bool
+
+config MUX_CORE
+	bool "Generic Multiplexer Support"
+	select MULTIPLEXER
+	help
+	  This framework is designed to abstract multiplexer handling for
+	  devices via various GPIO-, MMIO/Regmap or specific multiplexer
+	  controller chips.
+
+	  If unsure, say no.
+
+if MULTIPLEXER
 
 menu "Multiplexer drivers"
-	depends on MULTIPLEXER
 
 config MUX_ADG792A
 	tristate "Analog Devices ADG792A/ADG792G Multiplexers"
@@ -60,3 +71,5 @@ config MUX_MMIO
 	  be called mux-mmio.
 
 endmenu
+
+endif # MULTIPLEXER
diff --git a/drivers/mux/core.c b/drivers/mux/core.c
index f09ee8782e3de..23538de2c91b4 100644
--- a/drivers/mux/core.c
+++ b/drivers/mux/core.c
@@ -46,6 +46,16 @@ static const struct class mux_class = {
 	.name = "mux",
 };
 
+/**
+ * struct devm_mux_state_state -	Tracks managed resources for mux-state objects.
+ * @mstate:				Pointer to a mux state.
+ * @exit:				An optional callback to execute before free.
+ */
+struct devm_mux_state_state {
+	struct mux_state *mstate;
+	int (*exit)(struct mux_state *mstate);
+};
+
 static DEFINE_IDA(mux_ida);
 
 static int __init mux_init(void)
@@ -516,17 +526,19 @@ static struct mux_chip *of_find_mux_chip_by_node(struct device_node *np)
 	return dev ? to_mux_chip(dev) : NULL;
 }
 
-/*
+/**
  * mux_get() - Get the mux-control for a device.
  * @dev: The device that needs a mux-control.
  * @mux_name: The name identifying the mux-control.
  * @state: Pointer to where the requested state is returned, or NULL when
  *         the required multiplexer states are handled by other means.
+ * @optional: Whether to return NULL and silence errors when mux doesn't exist.
  *
- * Return: A pointer to the mux-control, or an ERR_PTR with a negative errno.
+ * Return: Pointer to the mux-control on success, an ERR_PTR with a negative
+ * errno on error, or NULL if optional is true and mux doesn't exist.
  */
 static struct mux_control *mux_get(struct device *dev, const char *mux_name,
-				   unsigned int *state)
+				   unsigned int *state, bool optional)
 {
 	struct device_node *np = dev->of_node;
 	struct of_phandle_args args;
@@ -542,7 +554,9 @@ static struct mux_control *mux_get(struct device *dev, const char *mux_name,
 		else
 			index = of_property_match_string(np, "mux-control-names",
 							 mux_name);
-		if (index < 0) {
+		if (index < 0 && optional) {
+			return NULL;
+		} else if (index < 0) {
 			dev_err(dev, "mux controller '%s' not found\n",
 				mux_name);
 			return ERR_PTR(index);
@@ -558,8 +572,12 @@ static struct mux_control *mux_get(struct device *dev, const char *mux_name,
 						 "mux-controls", "#mux-control-cells",
 						 index, &args);
 	if (ret) {
+		if (optional && ret == -ENOENT)
+			return NULL;
+
 		dev_err(dev, "%pOF: failed to get mux-%s %s(%i)\n",
-			np, state ? "state" : "control", mux_name ?: "", index);
+			np, state ? "state" : "control",
+			mux_name ?: "", index);
 		return ERR_PTR(ret);
 	}
 
@@ -617,10 +635,29 @@ static struct mux_control *mux_get(struct device *dev, const char *mux_name,
  */
 struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 {
-	return mux_get(dev, mux_name, NULL);
+	struct mux_control *mux = mux_get(dev, mux_name, NULL, false);
+
+	if (!mux)
+		return ERR_PTR(-ENOENT);
+
+	return mux;
 }
 EXPORT_SYMBOL_GPL(mux_control_get);
 
+/**
+ * mux_control_get_optional() - Get the optional mux-control for a device.
+ * @dev: The device that needs a mux-control.
+ * @mux_name: The name identifying the mux-control.
+ *
+ * Return: Pointer to the mux-control on success, an ERR_PTR with a negative
+ * errno on error, or NULL if mux doesn't exist.
+ */
+struct mux_control *mux_control_get_optional(struct device *dev, const char *mux_name)
+{
+	return mux_get(dev, mux_name, NULL, true);
+}
+EXPORT_SYMBOL_GPL(mux_control_get_optional);
+
 /**
  * mux_control_put() - Put away the mux-control for good.
  * @mux: The mux-control to put away.
@@ -670,14 +707,16 @@ struct mux_control *devm_mux_control_get(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_mux_control_get);
 
-/*
+/**
  * mux_state_get() - Get the mux-state for a device.
  * @dev: The device that needs a mux-state.
  * @mux_name: The name identifying the mux-state.
+ * @optional: Whether to return NULL and silence errors when mux doesn't exist.
  *
- * Return: A pointer to the mux-state, or an ERR_PTR with a negative errno.
+ * Return: Pointer to the mux-state on success, an ERR_PTR with a negative
+ * errno on error, or NULL if optional is true and mux doesn't exist.
  */
-static struct mux_state *mux_state_get(struct device *dev, const char *mux_name)
+static struct mux_state *mux_state_get(struct device *dev, const char *mux_name, bool optional)
 {
 	struct mux_state *mstate;
 
@@ -685,12 +724,15 @@ static struct mux_state *mux_state_get(struct device *dev, const char *mux_name)
 	if (!mstate)
 		return ERR_PTR(-ENOMEM);
 
-	mstate->mux = mux_get(dev, mux_name, &mstate->state);
+	mstate->mux = mux_get(dev, mux_name, &mstate->state, optional);
 	if (IS_ERR(mstate->mux)) {
 		int err = PTR_ERR(mstate->mux);
 
 		kfree(mstate);
 		return ERR_PTR(err);
+	} else if (!mstate->mux) {
+		kfree(mstate);
+		return optional ? NULL : ERR_PTR(-ENOENT);
 	}
 
 	return mstate;
@@ -710,9 +752,66 @@ static void mux_state_put(struct mux_state *mstate)
 
 static void devm_mux_state_release(struct device *dev, void *res)
 {
-	struct mux_state *mstate = *(struct mux_state **)res;
+	struct devm_mux_state_state *devm_state = res;
+
+	if (devm_state->exit)
+		devm_state->exit(devm_state->mstate);
+
+	mux_state_put(devm_state->mstate);
+}
+
+/**
+ * __devm_mux_state_get() - Get the optional mux-state for a device,
+ *			    with resource management.
+ * @dev: The device that needs a mux-state.
+ * @mux_name: The name identifying the mux-state.
+ * @optional: Whether to return NULL and silence errors when mux doesn't exist.
+ * @init: Optional function pointer for mux-state object initialisation.
+ * @exit: Optional function pointer for mux-state object cleanup on release.
+ *
+ * Return: Pointer to the mux-state on success, an ERR_PTR with a negative
+ * errno on error, or NULL if optional is true and mux doesn't exist.
+ */
+static struct mux_state *__devm_mux_state_get(struct device *dev, const char *mux_name,
+					      bool optional,
+					      int (*init)(struct mux_state *mstate),
+					      int (*exit)(struct mux_state *mstate))
+{
+	struct devm_mux_state_state *devm_state;
+	struct mux_state *mstate;
+	int ret;
+
+	mstate = mux_state_get(dev, mux_name, optional);
+	if (IS_ERR(mstate))
+		return ERR_CAST(mstate);
+	else if (optional && !mstate)
+		return NULL;
+	else if (!mstate)
+		return ERR_PTR(-ENOENT);
+
+	devm_state = devres_alloc(devm_mux_state_release, sizeof(*devm_state), GFP_KERNEL);
+	if (!devm_state) {
+		ret = -ENOMEM;
+		goto err_devres_alloc;
+	}
+
+	if (init) {
+		ret = init(mstate);
+		if (ret)
+			goto err_mux_state_init;
+	}
+
+	devm_state->mstate = mstate;
+	devm_state->exit = exit;
+	devres_add(dev, devm_state);
 
+	return mstate;
+
+err_mux_state_init:
+	devres_free(devm_state);
+err_devres_alloc:
 	mux_state_put(mstate);
+	return ERR_PTR(ret);
 }
 
 /**
@@ -722,28 +821,69 @@ static void devm_mux_state_release(struct device *dev, void *res)
  * @mux_name: The name identifying the mux-control.
  *
  * Return: Pointer to the mux-state, or an ERR_PTR with a negative errno.
+ *
+ * The mux-state will automatically be freed on release.
  */
-struct mux_state *devm_mux_state_get(struct device *dev,
-				     const char *mux_name)
+struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name)
 {
-	struct mux_state **ptr, *mstate;
-
-	ptr = devres_alloc(devm_mux_state_release, sizeof(*ptr), GFP_KERNEL);
-	if (!ptr)
-		return ERR_PTR(-ENOMEM);
+	return __devm_mux_state_get(dev, mux_name, false, NULL, NULL);
+}
+EXPORT_SYMBOL_GPL(devm_mux_state_get);
 
-	mstate = mux_state_get(dev, mux_name);
-	if (IS_ERR(mstate)) {
-		devres_free(ptr);
-		return mstate;
-	}
+/**
+ * devm_mux_state_get_optional() - Get the optional mux-state for a device,
+ *				   with resource management.
+ * @dev: The device that needs a mux-state.
+ * @mux_name: The name identifying the mux-state.
+ *
+ * Return: Pointer to the mux-state on success, an ERR_PTR with a negative
+ * errno on error, or NULL if mux doesn't exist.
+ *
+ * The mux-state will automatically be freed on release.
+ */
+struct mux_state *devm_mux_state_get_optional(struct device *dev, const char *mux_name)
+{
+	return __devm_mux_state_get(dev, mux_name, true, NULL, NULL);
+}
+EXPORT_SYMBOL_GPL(devm_mux_state_get_optional);
 
-	*ptr = mstate;
-	devres_add(dev, ptr);
+/**
+ * devm_mux_state_get_selected() - Get the mux-state for a device, with
+ *				   resource management.
+ * @dev: The device that needs a mux-state.
+ * @mux_name: The name identifying the mux-state.
+ *
+ * Return: Pointer to the mux-state, or an ERR_PTR with a negative errno.
+ *
+ * The returned mux-state (if valid) is already selected.
+ *
+ * The mux-state will automatically be deselected and freed on release.
+ */
+struct mux_state *devm_mux_state_get_selected(struct device *dev, const char *mux_name)
+{
+	return __devm_mux_state_get(dev, mux_name, false, mux_state_select, mux_state_deselect);
+}
+EXPORT_SYMBOL_GPL(devm_mux_state_get_selected);
 
-	return mstate;
+/**
+ * devm_mux_state_get_optional_selected() - Get the optional mux-state for
+ *					    a device, with resource management.
+ * @dev: The device that needs a mux-state.
+ * @mux_name: The name identifying the mux-state.
+ *
+ * Return: Pointer to the mux-state on success, an ERR_PTR with a negative
+ * errno on error, or NULL if mux doesn't exist.
+ *
+ * The returned mux-state (if valid) is already selected.
+ *
+ * The mux-state will automatically be deselected and freed on release.
+ */
+struct mux_state *devm_mux_state_get_optional_selected(struct device *dev,
+						       const char *mux_name)
+{
+	return __devm_mux_state_get(dev, mux_name, true, mux_state_select, mux_state_deselect);
 }
-EXPORT_SYMBOL_GPL(devm_mux_state_get);
+EXPORT_SYMBOL_GPL(devm_mux_state_get_optional_selected);
 
 /*
  * Using subsys_initcall instead of module_init here to try to ensure - for
diff --git a/drivers/mux/mmio.c b/drivers/mux/mmio.c
index 0611ef28bb69a..b61e590f2ac9e 100644
--- a/drivers/mux/mmio.c
+++ b/drivers/mux/mmio.c
@@ -100,12 +100,14 @@ static int mux_mmio_probe(struct platform_device *pdev)
 
 	mux_mmio = mux_chip_priv(mux_chip);
 
-	mux_mmio->fields = devm_kmalloc(dev, num_fields * sizeof(*mux_mmio->fields), GFP_KERNEL);
+	mux_mmio->fields = devm_kcalloc(dev, num_fields, sizeof(*mux_mmio->fields),
+					GFP_KERNEL);
 	if (!mux_mmio->fields)
 		return -ENOMEM;
 
-	mux_mmio->hardware_states = devm_kmalloc(dev, num_fields *
-						 sizeof(*mux_mmio->hardware_states), GFP_KERNEL);
+	mux_mmio->hardware_states = devm_kcalloc(dev, num_fields,
+						 sizeof(*mux_mmio->hardware_states),
+						 GFP_KERNEL);
 	if (!mux_mmio->hardware_states)
 		return -ENOMEM;
 
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 17108c3592166..b2fd90466babd 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -452,6 +452,7 @@ config VIRTIO_NET
 	depends on VIRTIO
 	select NET_FAILOVER
 	select DIMLIB
+	select PAGE_POOL
 	help
 	  This is the virtual network driver for virtio.  It can be used with
 	  QEMU based VMMs (like KVM or Xen).  Say Y or M.
diff --git a/drivers/net/bonding/bond_debugfs.c b/drivers/net/bonding/bond_debugfs.c
index 8adbec7c5084a..8967b65f6d840 100644
--- a/drivers/net/bonding/bond_debugfs.c
+++ b/drivers/net/bonding/bond_debugfs.c
@@ -34,11 +34,17 @@ static int bond_debug_rlb_hash_show(struct seq_file *m, void *v)
 	for (; hash_index != RLB_NULL_INDEX;
 	     hash_index = client_info->used_next) {
 		client_info = &(bond_info->rx_hashtbl[hash_index]);
-		seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n",
-			&client_info->ip_src,
-			&client_info->ip_dst,
-			&client_info->mac_dst,
-			client_info->slave->dev->name);
+		if (client_info->slave)
+			seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n",
+				   &client_info->ip_src,
+				   &client_info->ip_dst,
+				   &client_info->mac_dst,
+				   client_info->slave->dev->name);
+		else
+			seq_printf(m, "%-15pI4 %-15pI4 %-17pM (none)\n",
+				   &client_info->ip_src,
+				   &client_info->ip_dst,
+				   &client_info->mac_dst);
 	}
 
 	spin_unlock_bh(&bond->mode_lock);
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 707419270ebf2..7fb10e93883da 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -206,7 +206,7 @@ MODULE_PARM_DESC(lp_interval, "The number of seconds between instances where "
 /*----------------------------- Global variables ----------------------------*/
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
-atomic_t netpoll_block_tx = ATOMIC_INIT(0);
+DEFINE_STATIC_KEY_FALSE(netpoll_block_tx);
 #endif
 
 unsigned int bond_net_id __read_mostly;
@@ -1530,9 +1530,11 @@ static int bond_header_create(struct sk_buff *skb, struct net_device *bond_dev,
 	return ret;
 }
 
-static int bond_header_parse(const struct sk_buff *skb, unsigned char *haddr)
+static int bond_header_parse(const struct sk_buff *skb,
+			     const struct net_device *dev,
+			     unsigned char *haddr)
 {
-	struct bonding *bond = netdev_priv(skb->dev);
+	struct bonding *bond = netdev_priv(dev);
 	const struct header_ops *slave_ops;
 	struct slave *slave;
 	int ret = 0;
@@ -1542,7 +1544,7 @@ static int bond_header_parse(const struct sk_buff *skb, unsigned char *haddr)
 	if (slave) {
 		slave_ops = READ_ONCE(slave->dev->header_ops);
 		if (slave_ops && slave_ops->parse)
-			ret = slave_ops->parse(skb, haddr);
+			ret = slave_ops->parse(skb, slave->dev, haddr);
 	}
 	rcu_read_unlock();
 	return ret;
@@ -6647,7 +6649,7 @@ static void __exit bonding_exit(void)
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	/* Make sure we don't have an imbalance on our netpoll blocking */
-	WARN_ON(atomic_read(&netpoll_block_tx));
+	WARN_ON(static_branch_unlikely(&netpoll_block_tx));
 #endif
 }
 
diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c
index 286f11c517f76..ea1a80e658aeb 100644
--- a/drivers/net/bonding/bond_netlink.c
+++ b/drivers/net/bonding/bond_netlink.c
@@ -29,6 +29,8 @@ static size_t bond_get_slave_size(const struct net_device *bond_dev,
 		nla_total_size(sizeof(u16)) +	/* IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE */
 		nla_total_size(sizeof(s32)) +	/* IFLA_BOND_SLAVE_PRIO */
 		nla_total_size(sizeof(u16)) +	/* IFLA_BOND_SLAVE_ACTOR_PORT_PRIO */
+		nla_total_size(sizeof(u8)) +	/* IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE */
+		nla_total_size(sizeof(u8)) +	/* IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE */
 		0;
 }
 
@@ -77,6 +79,13 @@ static int bond_fill_slave_info(struct sk_buff *skb,
 					IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE,
 					ad_port->partner_oper.port_state))
 				goto nla_put_failure;
+
+			if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE,
+				       ad_port->sm_churn_actor_state))
+				goto nla_put_failure;
+			if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE,
+				       ad_port->sm_churn_partner_state))
+				goto nla_put_failure;
 		}
 
 		if (nla_put_u16(skb, IFLA_BOND_SLAVE_ACTOR_PORT_PRIO,
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index 960685596093b..de3efa3ce9a75 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -980,15 +980,19 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds)
 	ret = bcm_sf2_sw_rst(priv);
 	if (ret) {
 		pr_err("%s: failed to software reset switch\n", __func__);
+		if (!priv->wol_ports_mask)
+			clk_disable_unprepare(priv->clk);
 		return ret;
 	}
 
 	bcm_sf2_crossbar_setup(priv);
 
 	ret = bcm_sf2_cfp_resume(ds);
-	if (ret)
+	if (ret) {
+		if (!priv->wol_ports_mask)
+			clk_disable_unprepare(priv->clk);
 		return ret;
-
+	}
 	if (priv->hw_params.num_gphy == 1)
 		bcm_sf2_gphy_enable_set(ds, true);
 
diff --git a/drivers/net/dsa/mv88e6xxx/Makefile b/drivers/net/dsa/mv88e6xxx/Makefile
index dd961081d6313..b0b08c6f159c6 100644
--- a/drivers/net/dsa/mv88e6xxx/Makefile
+++ b/drivers/net/dsa/mv88e6xxx/Makefile
@@ -21,6 +21,8 @@ mv88e6xxx-objs += serdes.o
 mv88e6xxx-objs += smi.o
 mv88e6xxx-objs += switchdev.o
 mv88e6xxx-objs += trace.o
+mv88e6xxx-objs += tcflower.o
+mv88e6xxx-objs += tcam.o
 
 # for tracing framework to find trace.h
 CFLAGS_trace.o := -I$(src)
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 6fcd7181116a9..8ca5fd40df92e 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -43,6 +43,8 @@
 #include "ptp.h"
 #include "serdes.h"
 #include "smi.h"
+#include "tcam.h"
+#include "tcflower.h"
 
 static void assert_reg_lock(struct mv88e6xxx_chip *chip)
 {
@@ -3560,6 +3562,11 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 		if (err)
 			return err;
 	}
+	if (chip->info->ops->port_enable_tcam) {
+		err = chip->info->ops->port_enable_tcam(chip, port);
+		if (err)
+			return err;
+	}
 
 	if (chip->info->ops->port_tag_remap) {
 		err = chip->info->ops->port_tag_remap(chip, port);
@@ -3938,6 +3945,14 @@ static int mv88e6xxx_mdios_register(struct mv88e6xxx_chip *chip)
 	return 0;
 }
 
+static int mv88e6xxx_tcam_setup(struct mv88e6xxx_chip *chip)
+{
+	if (!mv88e6xxx_has_tcam(chip))
+		return 0;
+
+	return chip->info->ops->tcam_ops->flush_tcam(chip);
+}
+
 static void mv88e6xxx_teardown(struct dsa_switch *ds)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
@@ -3947,6 +3962,7 @@ static void mv88e6xxx_teardown(struct dsa_switch *ds)
 	mv88e6xxx_teardown_devlink_regions_global(ds);
 	mv88e6xxx_hwtstamp_free(chip);
 	mv88e6xxx_ptp_free(chip);
+	mv88e6xxx_flower_teardown(chip);
 	mv88e6xxx_mdios_unregister(chip);
 }
 
@@ -4083,6 +4099,10 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 	if (err)
 		goto unlock;
 
+	err = mv88e6xxx_tcam_setup(chip);
+	if (err)
+		goto unlock;
+
 unlock:
 	mv88e6xxx_reg_unlock(chip);
 
@@ -5134,6 +5154,7 @@ static const struct mv88e6xxx_ops mv88e6290_ops = {
 	.ptp_ops = &mv88e6390_ptp_ops,
 	.phylink_get_caps = mv88e6390_phylink_get_caps,
 	.pcs_ops = &mv88e6390_pcs_ops,
+	.tcam_ops = &mv88e6390_tcam_ops,
 };
 
 static const struct mv88e6xxx_ops mv88e6320_ops = {
@@ -5525,6 +5546,7 @@ static const struct mv88e6xxx_ops mv88e6390_ops = {
 	.serdes_get_regs = mv88e6390_serdes_get_regs,
 	.phylink_get_caps = mv88e6390_phylink_get_caps,
 	.pcs_ops = &mv88e6390_pcs_ops,
+	.tcam_ops = &mv88e6390_tcam_ops,
 };
 
 static const struct mv88e6xxx_ops mv88e6390x_ops = {
@@ -5621,6 +5643,7 @@ static const struct mv88e6xxx_ops mv88e6393x_ops = {
 	.port_set_cmode = mv88e6393x_port_set_cmode,
 	.port_setup_message_port = mv88e6xxx_setup_message_port,
 	.port_set_upstream_port = mv88e6393x_port_set_upstream_port,
+	.port_enable_tcam = mv88e6xxx_port_enable_tcam,
 	.stats_snapshot = mv88e6390_g1_stats_snapshot,
 	.stats_set_histogram = mv88e6390_g1_stats_set_histogram,
 	.stats_get_sset_count = mv88e6320_stats_get_sset_count,
@@ -5652,6 +5675,7 @@ static const struct mv88e6xxx_ops mv88e6393x_ops = {
 	.ptp_ops = &mv88e6352_ptp_ops,
 	.phylink_get_caps = mv88e6393x_phylink_get_caps,
 	.pcs_ops = &mv88e6393x_pcs_ops,
+	.tcam_ops = &mv88e6393_tcam_ops,
 };
 
 static const struct mv88e6xxx_info mv88e6xxx_table[] = {
@@ -6125,6 +6149,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 11,	/* 10 + Z80 */
 		.num_internal_phys = 8,
+		.num_tcam_entries = 256,
 		.internal_phys_offset = 1,
 		.max_vid = 8191,
 		.max_sid = 63,
@@ -6132,6 +6157,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.phy_base_addr = 0x0,
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
+		.tcam_addr = 0x1f,
 		.age_time_coeff = 3750,
 		.g1_irqs = 10,
 		.g2_irqs = 14,
@@ -6227,12 +6253,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_ports = 11,	/* 10 + Z80 */
 		.num_internal_phys = 9,
 		.num_gpio = 16,
+		.num_tcam_entries = 256,
 		.max_vid = 8191,
 		.max_sid = 63,
 		.port_base_addr = 0x0,
 		.phy_base_addr = 0x0,
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
+		.tcam_addr = 0x1f,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
 		.g2_irqs = 14,
@@ -6439,12 +6467,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_ports = 11,	/* 10 + Z80 */
 		.num_internal_phys = 9,
 		.num_gpio = 16,
+		.num_tcam_entries = 256,
 		.max_vid = 8191,
 		.max_sid = 63,
 		.port_base_addr = 0x0,
 		.phy_base_addr = 0x0,
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
+		.tcam_addr = 0x1f,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
 		.g2_irqs = 14,
@@ -6490,6 +6520,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 11,	/* 10 + Z80 */
 		.num_internal_phys = 8,
+		.num_tcam_entries = 256,
 		.internal_phys_offset = 1,
 		.max_vid = 8191,
 		.max_sid = 63,
@@ -6497,6 +6528,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.phy_base_addr = 0x0,
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
+		.tcam_addr = 0x1f,
 		.age_time_coeff = 3750,
 		.g1_irqs = 10,
 		.g2_irqs = 14,
@@ -6589,6 +6621,7 @@ static struct mv88e6xxx_chip *mv88e6xxx_alloc_chip(struct device *dev)
 	INIT_LIST_HEAD(&chip->mdios);
 	idr_init(&chip->policies);
 	INIT_LIST_HEAD(&chip->msts);
+	INIT_LIST_HEAD(&chip->tcam.entries);
 
 	return chip;
 }
@@ -7184,6 +7217,8 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.port_hwtstamp_get	= mv88e6xxx_port_hwtstamp_get,
 	.port_txtstamp		= mv88e6xxx_port_txtstamp,
 	.port_rxtstamp		= mv88e6xxx_port_rxtstamp,
+	.cls_flower_add		= mv88e6xxx_cls_flower_add,
+	.cls_flower_del         = mv88e6xxx_cls_flower_del,
 	.get_ts_info		= mv88e6xxx_get_ts_info,
 	.devlink_param_get	= mv88e6xxx_devlink_param_get,
 	.devlink_param_set	= mv88e6xxx_devlink_param_set,
diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index e073446ee7d02..2b235ac2c5df5 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -135,12 +135,14 @@ struct mv88e6xxx_info {
 	unsigned int num_ports;
 	unsigned int num_internal_phys;
 	unsigned int num_gpio;
+	unsigned int num_tcam_entries;
 	unsigned int max_vid;
 	unsigned int max_sid;
 	unsigned int port_base_addr;
 	unsigned int phy_base_addr;
 	unsigned int global1_addr;
 	unsigned int global2_addr;
+	unsigned int tcam_addr;
 	unsigned int age_time_coeff;
 	unsigned int g1_irqs;
 	unsigned int g2_irqs;
@@ -210,6 +212,7 @@ struct mv88e6xxx_avb_ops;
 struct mv88e6xxx_ptp_ops;
 struct mv88e6xxx_pcs_ops;
 struct mv88e6xxx_cc_coeffs;
+struct mv88e6xxx_tcam_ops;
 
 struct mv88e6xxx_irq {
 	u16 masked;
@@ -339,6 +342,10 @@ struct mv88e6xxx_hw_stat {
 	int type;
 };
 
+struct mv88e6xxx_tcam {
+	struct list_head entries;
+};
+
 struct mv88e6xxx_chip {
 	const struct mv88e6xxx_info *info;
 
@@ -444,6 +451,35 @@ struct mv88e6xxx_chip {
 
 	/* FID map */
 	DECLARE_BITMAP(fid_bitmap, MV88E6XXX_N_FID);
+
+	/* TCAM entries */
+	struct mv88e6xxx_tcam tcam;
+};
+
+#define TCAM_MATCH_SIZE 96
+
+struct mv88e6xxx_tcam_key {
+	u16 spv;
+	u16 spv_mask;
+
+	u8 frame_data[TCAM_MATCH_SIZE];
+	u8 frame_mask[TCAM_MATCH_SIZE];
+};
+
+struct mv88e6xxx_tcam_action {
+	u8 dpv_mode;
+	u16 dpv;
+};
+
+struct mv88e6xxx_tcam_entry {
+	struct list_head list;
+	unsigned long cookie;
+	u32 prio;
+	u8 hw_idx;
+
+	struct mv88e6xxx_tcam_key key;
+	struct mv88e6xxx_tcam_action action;
+
 };
 
 struct mv88e6xxx_bus_ops {
@@ -678,6 +714,11 @@ struct mv88e6xxx_ops {
 
 	/* Max Frame Size */
 	int (*set_max_frame_size)(struct mv88e6xxx_chip *chip, int mtu);
+
+	int (*port_enable_tcam)(struct mv88e6xxx_chip *chip, int port);
+
+	/* Ternary Content Addressable Memory operations */
+	const struct mv88e6xxx_tcam_ops *tcam_ops;
 };
 
 struct mv88e6xxx_irq_ops {
@@ -752,6 +793,12 @@ struct mv88e6xxx_pcs_ops {
 
 };
 
+struct mv88e6xxx_tcam_ops {
+	int (*entry_add)(struct mv88e6xxx_chip *chip,
+			 struct mv88e6xxx_tcam_entry *entry, u8 idx);
+	int (*flush_tcam)(struct mv88e6xxx_chip *chip);
+};
+
 static inline bool mv88e6xxx_has_stu(struct mv88e6xxx_chip *chip)
 {
 	return chip->info->max_sid > 0 &&
@@ -769,6 +816,11 @@ static inline bool mv88e6xxx_has_lag(struct mv88e6xxx_chip *chip)
 	return !!chip->info->global2_addr;
 }
 
+static inline bool mv88e6xxx_has_tcam(struct mv88e6xxx_chip *chip)
+{
+	return !!chip->info->tcam_addr;
+}
+
 static inline unsigned int mv88e6xxx_num_databases(struct mv88e6xxx_chip *chip)
 {
 	return chip->info->num_databases;
diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c
index 66b1b72772810..49cd82930b7a5 100644
--- a/drivers/net/dsa/mv88e6xxx/port.c
+++ b/drivers/net/dsa/mv88e6xxx/port.c
@@ -1380,7 +1380,33 @@ int mv88e6xxx_port_disable_learn_limit(struct mv88e6xxx_chip *chip, int port)
 
 int mv88e6xxx_port_disable_pri_override(struct mv88e6xxx_chip *chip, int port)
 {
-	return mv88e6xxx_port_write(chip, port, MV88E6XXX_PORT_PRI_OVERRIDE, 0);
+	u16 reg;
+	int err;
+
+	err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_PRI_OVERRIDE,
+				  &reg);
+	if (err)
+		return err;
+
+	reg &= MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_MASK;
+	return mv88e6xxx_port_write(chip, port, MV88E6XXX_PORT_PRI_OVERRIDE,
+				    reg);
+}
+
+int mv88e6xxx_port_enable_tcam(struct mv88e6xxx_chip *chip, int port)
+{
+	u16 reg;
+	int err;
+
+	err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_PRI_OVERRIDE,
+				  &reg);
+	if (err)
+		return err;
+
+	reg &= ~MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_MASK;
+	reg |= MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_96_BYTE;
+	return mv88e6xxx_port_write(chip, port, MV88E6XXX_PORT_PRI_OVERRIDE,
+				    reg);
 }
 
 /* Offset 0x0E: Policy & MGMT Control Register for FAMILY 6191X 6193X 6393X */
diff --git a/drivers/net/dsa/mv88e6xxx/port.h b/drivers/net/dsa/mv88e6xxx/port.h
index c1d2f99efb1c6..a2492cf4d920f 100644
--- a/drivers/net/dsa/mv88e6xxx/port.h
+++ b/drivers/net/dsa/mv88e6xxx/port.h
@@ -254,7 +254,10 @@
 #define MV88E6XXX_PORT_ATU_CTL		0x0c
 
 /* Offset 0x0D: Priority Override Register */
-#define MV88E6XXX_PORT_PRI_OVERRIDE	0x0d
+#define MV88E6XXX_PORT_PRI_OVERRIDE			0x0d
+#define MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_MASK	0x0003
+#define MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_48_BYTE	0x0001
+#define MV88E6XXX_PORT_PRI_OVERRIDE_TCAM_MODE_96_BYTE	0x0002
 
 /* Offset 0x0E: Policy Control Register */
 #define MV88E6XXX_PORT_POLICY_CTL		0x0e
@@ -608,4 +611,6 @@ int mv88e6xxx_port_hidden_wait(struct mv88e6xxx_chip *chip);
 int mv88e6xxx_port_hidden_read(struct mv88e6xxx_chip *chip, int block, int port,
 			       int reg, u16 *val);
 
+int mv88e6xxx_port_enable_tcam(struct mv88e6xxx_chip *chip, int port);
+
 #endif /* _MV88E6XXX_PORT_H */
diff --git a/drivers/net/dsa/mv88e6xxx/tcam.c b/drivers/net/dsa/mv88e6xxx/tcam.c
new file mode 100644
index 0000000000000..fc16bab8e7725
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/tcam.c
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Marvell 88E6xxx Switch TCAM support
+ *
+ * Copyright (c) 2026 Luminex Network Intelligence
+ */
+
+#include "linux/list.h"
+
+#include "chip.h"
+#include "tcam.h"
+
+/* TCAM operatation register */
+#define MV88E6XXX_TCAM_OP			0x00
+#define MV88E6XXX_TCAM_OP_BUSY			0x8000
+#define MV88E6XXX_TCAM_OP_OP_MASK		0x7000
+#define MV88E6XXX_TCAM_OP_OP_FLUSH_ALL		0x1000
+#define MV88E6XXX_TCAM_OP_OP_FLUSH		0x2000
+#define MV88E6XXX_TCAM_OP_OP_LOAD		0x3000
+#define MV88E6XXX_TCAM_OP_OP_GET_NEXT		0x4000
+#define MV88E6XXX_TCAM_OP_OP_READ		0x5000
+
+/* TCAM extension register */
+#define MV88E6XXX_TCAM_EXTENSION		0x01
+
+/* TCAM keys register 1 */
+#define MV88E6XXX_TCAM_KEYS1			0x02
+#define MV88E6XXX_TCAM_KEYS1_FT_MASK		0xC000
+#define MV88E6XXX_TCAM_KEYS1_SPV_MASK		0x0007
+#define MV88E6XXX_TCAM_KEYS1_SPV_MASK_MASK	0x0700
+
+/* TCAM keys register 2 */
+#define MV88E6XXX_TCAM_KEYS2			0x03
+#define MV88E6XXX_TCAM_KEYS2_SPV_MASK		0x00ff
+#define MV88E6XXX_TCAM_KEYS2_SPV_MASK_MASK	0xff00
+
+#define MV88E6XXX_TCAM_ING_ACT3			0x04
+#define MV88E6XXX_TCAM_ING_ACT3_SF		0x0800
+#define MV88E6XXX_TCAM_ING_ACT3_DPV_MASK	0x07ff
+
+#define MV88E6XXX_TCAM_ING_ACT5			0x06
+#define MV88E6XXX_TCAM_ING_ACT5_DPV_MODE_MASK	0xc000
+
+static int mv88e6xxx_tcam_write(struct mv88e6xxx_chip *chip, int reg, u16 val)
+{
+	return mv88e6xxx_write(chip, chip->info->tcam_addr, reg, val);
+}
+
+static int mv88e6xxx_tcam_wait(struct mv88e6xxx_chip *chip)
+{
+	int bit = __bf_shf(MV88E6XXX_TCAM_OP_BUSY);
+
+	return mv88e6xxx_wait_bit(chip, chip->info->tcam_addr,
+				  MV88E6XXX_TCAM_OP, bit, 0);
+}
+
+static int mv88e6xxx_tcam_read_page(struct mv88e6xxx_chip *chip, u8 page,
+				    u8 entry)
+
+{
+	u16 val = MV88E6XXX_TCAM_OP_BUSY | MV88E6XXX_TCAM_OP_OP_READ |
+		  (page & 0x3) << 10 | entry;
+	int err;
+
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_OP, val);
+	if (err)
+		return err;
+
+	return mv88e6xxx_tcam_wait(chip);
+}
+
+static int mv88e6xxx_tcam_load_page(struct mv88e6xxx_chip *chip, u8 page,
+				    u8 entry)
+{
+	u16 val = MV88E6XXX_TCAM_OP_BUSY | MV88E6XXX_TCAM_OP_OP_LOAD |
+		  (page & 0x3) << 10 | entry;
+	int err;
+
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_OP, val);
+	if (err)
+		return err;
+
+	return mv88e6xxx_tcam_wait(chip);
+}
+
+static int mv88e6xxx_tcam_flush_entry(struct mv88e6xxx_chip *chip, u8 entry)
+{
+	u16 val = MV88E6XXX_TCAM_OP_BUSY | MV88E6XXX_TCAM_OP_OP_FLUSH | entry;
+	int err;
+
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_OP, val);
+	if (err)
+		return err;
+
+	return mv88e6xxx_tcam_wait(chip);
+}
+
+static int mv88e6xxx_tcam_flush_all(struct mv88e6xxx_chip *chip)
+{
+	u16 val = MV88E6XXX_TCAM_OP_BUSY | MV88E6XXX_TCAM_OP_OP_FLUSH_ALL;
+	int err;
+
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_OP, val);
+	if (err)
+		return err;
+
+	return mv88e6xxx_tcam_wait(chip);
+}
+
+struct mv88e6xxx_tcam_entry *
+mv88e6xxx_tcam_entry_find(struct mv88e6xxx_chip *chip, unsigned long cookie)
+{
+	struct mv88e6xxx_tcam_entry *entry;
+
+	list_for_each_entry(entry, &chip->tcam.entries, list)
+		if (entry->cookie == cookie)
+			return entry;
+
+	return NULL;
+}
+
+/* insert tcam entry in ordered list and move existing entries if necessary */
+static int mv88e6xxx_tcam_insert_entry(struct mv88e6xxx_chip *chip,
+				       struct mv88e6xxx_tcam_entry *entry)
+{
+	struct mv88e6xxx_tcam_entry *elem;
+	struct list_head *hpos;
+	int err;
+
+	list_for_each_prev(hpos, &chip->tcam.entries) {
+		u8 move_idx;
+
+		elem = list_entry(hpos, struct mv88e6xxx_tcam_entry, list);
+		if (entry->prio >= elem->prio)
+			break;
+
+		move_idx = elem->hw_idx + 1;
+
+		err = mv88e6xxx_tcam_flush_entry(chip, move_idx);
+		if (err)
+			return err;
+
+		err = chip->info->ops->tcam_ops->entry_add(chip, elem,
+							   move_idx);
+		if (err)
+			return err;
+
+		elem->hw_idx = move_idx;
+	}
+
+	if (list_is_head(hpos, &chip->tcam.entries)) {
+		entry->hw_idx = 0;
+	} else {
+		elem = list_entry(hpos, struct mv88e6xxx_tcam_entry, list);
+		entry->hw_idx = elem->hw_idx + 1;
+	}
+	list_add(&entry->list, hpos);
+	return 0;
+}
+
+int mv88e6xxx_tcam_entry_add(struct mv88e6xxx_chip *chip,
+			     struct mv88e6xxx_tcam_entry *entry)
+{
+	int err;
+	struct mv88e6xxx_tcam_entry *last;
+
+	last = list_last_entry_or_null(&chip->tcam.entries,
+				       struct mv88e6xxx_tcam_entry, list);
+	if (last && last->hw_idx == chip->info->num_tcam_entries - 1)
+		return -ENOSPC;
+
+	err = mv88e6xxx_tcam_insert_entry(chip, entry);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_tcam_flush_entry(chip, entry->hw_idx);
+	if (err)
+		goto unlink_out;
+
+	err = chip->info->ops->tcam_ops->entry_add(chip, entry, entry->hw_idx);
+	if (err)
+		goto unlink_out;
+
+	return 0;
+
+unlink_out:
+	list_del(&entry->list);
+	return err;
+}
+
+int mv88e6xxx_tcam_entry_del(struct mv88e6xxx_chip *chip,
+			     struct mv88e6xxx_tcam_entry *entry)
+{
+	struct mv88e6xxx_tcam_entry *elem = entry;
+	u8 move_idx = entry->hw_idx;
+	int err;
+
+	err = mv88e6xxx_tcam_flush_entry(chip, entry->hw_idx);
+	if (err)
+		return err;
+
+	/* move entries that come after the deleted entry forward */
+	list_for_each_entry_continue(elem, &chip->tcam.entries, list) {
+		u8 tmp_idx = elem->hw_idx;
+
+		err = mv88e6xxx_tcam_flush_entry(chip, move_idx);
+		if (err)
+			break;
+
+		err = chip->info->ops->tcam_ops->entry_add(chip, elem,
+							   move_idx);
+		if (err)
+			break;
+
+		elem->hw_idx =  move_idx;
+		move_idx = tmp_idx;
+
+		/* flush the last entry after moving entries */
+		if (list_is_last(&elem->list, &chip->tcam.entries))
+			err = mv88e6xxx_tcam_flush_entry(chip, tmp_idx);
+	}
+
+	list_del(&entry->list);
+	kfree(entry);
+	return err;
+}
+
+static int mv88e6390_tcam_entry_add(struct mv88e6xxx_chip *chip,
+				    struct mv88e6xxx_tcam_entry *entry, u8 idx)
+{
+	int err = 0;
+	int i;
+	u16 val, spv_mask, spv;
+
+	err = mv88e6xxx_tcam_read_page(chip, 2, idx);
+	if (err)
+		return err;
+	if (entry->action.dpv_mode != 0) {
+		val = MV88E6XXX_TCAM_ING_ACT3_SF |
+		      (entry->action.dpv & MV88E6XXX_TCAM_ING_ACT3_DPV_MASK);
+
+		err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_ING_ACT3, val);
+		if (err)
+			return err;
+
+		val = entry->action.dpv_mode << 14;
+		err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_ING_ACT5, val);
+		if (err)
+			return err;
+	}
+	err = mv88e6xxx_tcam_load_page(chip, 2, idx);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_tcam_read_page(chip, 1, idx);
+	if (err)
+		return err;
+
+	for (i = PAGE0_MATCH_SIZE;
+	     i < PAGE0_MATCH_SIZE + PAGE1_MATCH_SIZE; i++) {
+		if (entry->key.frame_mask[i]) {
+			val = entry->key.frame_mask[i] << 8 |
+			      entry->key.frame_data[i];
+
+			err = mv88e6xxx_tcam_write(chip,
+						   i - PAGE0_MATCH_SIZE + 2,
+						   val);
+			if (err)
+				return err;
+		}
+	}
+	err = mv88e6xxx_tcam_load_page(chip, 1, idx);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_tcam_read_page(chip, 0, idx);
+	if (err)
+		return err;
+
+	for (i = 0; i < PAGE0_MATCH_SIZE; i++) {
+		if (entry->key.frame_mask[i]) {
+			val = entry->key.frame_mask[i] << 8 |
+			      entry->key.frame_data[i];
+
+			err = mv88e6xxx_tcam_write(chip, i + 6, val);
+			if (err)
+				return err;
+		}
+	}
+
+	spv_mask = entry->key.spv_mask & mv88e6xxx_port_mask(chip);
+	spv = entry->key.spv & mv88e6xxx_port_mask(chip);
+	/* frame type mask bits must be set for a valid entry */
+	val = MV88E6XXX_TCAM_KEYS1_FT_MASK |
+	      (spv_mask & MV88E6XXX_TCAM_KEYS1_SPV_MASK_MASK) |
+	      ((spv >> 8) & MV88E6XXX_TCAM_KEYS1_SPV_MASK);
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_KEYS1, val);
+	if (err)
+		return err;
+
+	val = ((spv_mask << 8) & MV88E6XXX_TCAM_KEYS2_SPV_MASK_MASK) |
+	      (spv & MV88E6XXX_TCAM_KEYS2_SPV_MASK);
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_KEYS2, val);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_tcam_load_page(chip, 0, idx);
+	if (err)
+		return err;
+
+	entry->hw_idx = idx;
+	return 0;
+}
+
+static int mv88e6393_tcam_entry_add(struct mv88e6xxx_chip *chip,
+				    struct mv88e6xxx_tcam_entry *entry, u8 idx)
+{
+	int err;
+
+	/* select block 0 port 0, then adding an entry is the same as 6390 as
+	 * other blocks aren't used at the moment
+	 */
+	err = mv88e6xxx_tcam_write(chip, MV88E6XXX_TCAM_EXTENSION, 0x00);
+	if (err)
+		return err;
+
+	return mv88e6390_tcam_entry_add(chip, entry, idx);
+}
+
+const struct mv88e6xxx_tcam_ops mv88e6390_tcam_ops = {
+	.entry_add = mv88e6390_tcam_entry_add,
+	.flush_tcam = mv88e6xxx_tcam_flush_all,
+};
+
+const struct mv88e6xxx_tcam_ops mv88e6393_tcam_ops = {
+	.entry_add = mv88e6393_tcam_entry_add,
+	.flush_tcam = mv88e6xxx_tcam_flush_all,
+};
diff --git a/drivers/net/dsa/mv88e6xxx/tcam.h b/drivers/net/dsa/mv88e6xxx/tcam.h
new file mode 100644
index 0000000000000..d82bc9bc07686
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/tcam.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * Copyright (c) 2026 Luminex Network Intelligence
+ */
+#ifndef _MV88E6XXX_TCAM_H_
+#define _MV88E6XXX_TCAM_H_
+
+#define PAGE0_MATCH_SIZE 22
+#define PAGE1_MATCH_SIZE 26
+
+#define DPV_MODE_NOP		0x0
+#define DPV_MODE_AND		0x1
+#define DPV_MODE_OR		0x2
+#define DPV_MODE_REPLACE	0x3
+
+int mv88e6xxx_tcam_entry_add(struct mv88e6xxx_chip *chip,
+			     struct mv88e6xxx_tcam_entry *entry);
+int mv88e6xxx_tcam_entry_del(struct mv88e6xxx_chip *chip,
+			     struct mv88e6xxx_tcam_entry *entry);
+struct mv88e6xxx_tcam_entry *
+mv88e6xxx_tcam_entry_find(struct mv88e6xxx_chip *chip, unsigned long cookie);
+#define mv88e6xxx_tcam_match_set(key, _offset, data, mask) \
+	do { \
+		typeof(_offset) (offset) = (_offset); \
+		BUILD_BUG_ON((offset) + sizeof((data)) > TCAM_MATCH_SIZE); \
+		__mv88e6xxx_tcam_match_set(key, offset, sizeof(data), \
+					   (u8 *)&(data), (u8 *)&(mask)); \
+	} while (0)
+
+static inline void __mv88e6xxx_tcam_match_set(struct mv88e6xxx_tcam_key *key,
+					      unsigned int offset, size_t size,
+					      u8 *data, u8 *mask)
+{
+	memcpy(&key->frame_data[offset], data, size);
+	memcpy(&key->frame_mask[offset], mask, size);
+}
+
+extern const struct mv88e6xxx_tcam_ops mv88e6390_tcam_ops;
+extern const struct mv88e6xxx_tcam_ops mv88e6393_tcam_ops;
+#endif
diff --git a/drivers/net/dsa/mv88e6xxx/tcflower.c b/drivers/net/dsa/mv88e6xxx/tcflower.c
new file mode 100644
index 0000000000000..d67604a55b9f9
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/tcflower.c
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Marvell 88E6xxx Switch flower support
+ *
+ * Copyright (c) 2026 Luminex Network Intelligence
+ */
+
+#include "chip.h"
+#include "tcflower.h"
+#include "tcam.h"
+
+#define MV88E6XXX_ETHTYPE_OFFSET 16
+#define MV88E6XXX_IP_PROTO_OFFSET 27
+#define MV88E6XXX_IPV4_SRC_OFFSET 30
+#define MV88E6XXX_IPV4_DST_OFFSET 34
+
+static int mv88e6xxx_flower_parse_key(struct mv88e6xxx_chip *chip,
+				      struct netlink_ext_ack *extack,
+				      struct flow_cls_offload *cls,
+				      struct mv88e6xxx_tcam_key *key)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	u16 addr_type = 0;
+
+	if (dissector->used_keys &
+	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS))) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Unsupported keys used");
+		return -EOPNOTSUPP;
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
+		struct flow_match_control match;
+
+		flow_rule_match_control(rule, &match);
+		addr_type = match.key->addr_type;
+
+		if (flow_rule_has_control_flags(match.mask->flags,
+						cls->common.extack))
+			return -EOPNOTSUPP;
+	}
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_match_basic match;
+
+		flow_rule_match_basic(rule, &match);
+		mv88e6xxx_tcam_match_set(key, MV88E6XXX_ETHTYPE_OFFSET,
+					 match.key->n_proto,
+					 match.mask->n_proto);
+		mv88e6xxx_tcam_match_set(key, MV88E6XXX_IP_PROTO_OFFSET,
+					 match.key->ip_proto,
+					 match.mask->ip_proto);
+	}
+
+	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+		struct flow_match_ipv4_addrs match;
+
+		flow_rule_match_ipv4_addrs(cls->rule, &match);
+		mv88e6xxx_tcam_match_set(key, MV88E6XXX_IPV4_SRC_OFFSET,
+					 match.key->src,
+					 match.mask->src);
+		mv88e6xxx_tcam_match_set(key, MV88E6XXX_IPV4_DST_OFFSET,
+					 match.key->dst,
+					 match.mask->dst);
+	}
+
+	return 0;
+}
+
+int mv88e6xxx_cls_flower_add(struct dsa_switch *ds, int port,
+			     struct flow_cls_offload *cls, bool ingress)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct netlink_ext_ack *extack = cls->common.extack;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	struct mv88e6xxx_tcam_key key = { 0 };
+	const struct flow_action_entry *act;
+	unsigned long cookie = cls->cookie;
+	struct mv88e6xxx_tcam_entry *entry;
+	int err, i;
+
+	if (!mv88e6xxx_has_tcam(chip)) {
+		NL_SET_ERR_MSG_MOD(extack, "hardware offload not supported");
+		return -EOPNOTSUPP;
+	}
+
+	err = mv88e6xxx_flower_parse_key(chip, extack, cls, &key);
+	if (err)
+		return err;
+
+	mv88e6xxx_reg_lock(chip);
+	entry = mv88e6xxx_tcam_entry_find(chip, cookie);
+	if (entry) {
+		err = -EEXIST;
+		goto err_unlock;
+	}
+
+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry) {
+		err = -ENOMEM;
+		goto err_unlock;
+	}
+
+	entry->cookie = cookie;
+	entry->prio = cls->common.prio;
+	entry->key = key;
+
+	flow_action_for_each(i, act, &rule->action) {
+		switch (act->id) {
+		case FLOW_ACTION_TRAP: {
+			int cpu = dsa_upstream_port(ds, port);
+
+			entry->action.dpv_mode = DPV_MODE_REPLACE;
+			entry->action.dpv = BIT(cpu);
+			break;
+		}
+		default:
+			NL_SET_ERR_MSG_MOD(extack, "action not supported");
+			err = -EOPNOTSUPP;
+			goto err_free_entry;
+		}
+	}
+
+	entry->key.spv = BIT(port);
+	entry->key.spv_mask = mv88e6xxx_port_mask(chip);
+
+	err = mv88e6xxx_tcam_entry_add(chip, entry);
+	if (err)
+		goto err_free_entry;
+
+	mv88e6xxx_reg_unlock(chip);
+	return  0;
+
+err_free_entry:
+	kfree(entry);
+err_unlock:
+	mv88e6xxx_reg_unlock(chip);
+	return err;
+}
+
+int mv88e6xxx_cls_flower_del(struct dsa_switch *ds, int port,
+			     struct flow_cls_offload *cls, bool ingress)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	struct mv88e6xxx_tcam_entry *entry;
+	int err = 0;
+
+	mv88e6xxx_reg_lock(chip);
+	entry = mv88e6xxx_tcam_entry_find(chip, cls->cookie);
+
+	if (entry)
+		err = mv88e6xxx_tcam_entry_del(chip, entry);
+	mv88e6xxx_reg_unlock(chip);
+	return  err;
+}
+
+void mv88e6xxx_flower_teardown(struct mv88e6xxx_chip *chip)
+{
+	struct mv88e6xxx_tcam_entry *pos, *n;
+
+	list_for_each_entry_safe(pos, n, &chip->tcam.entries, list) {
+		list_del(&pos->list);
+		kfree(pos);
+	}
+}
diff --git a/drivers/net/dsa/mv88e6xxx/tcflower.h b/drivers/net/dsa/mv88e6xxx/tcflower.h
new file mode 100644
index 0000000000000..fb79f12fe18c4
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/tcflower.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * Copyright (c) 2026 Luminex Network Intelligence
+ */
+#ifndef _MV88E6XXX_TCFLOWER_H_
+#define _MV88E6XXX_TCFLOWER_H_
+
+int mv88e6xxx_cls_flower_add(struct dsa_switch *ds, int port,
+			     struct flow_cls_offload *cls, bool ingress);
+int mv88e6xxx_cls_flower_del(struct dsa_switch *ds, int port,
+			     struct flow_cls_offload *cls, bool ingress);
+void mv88e6xxx_flower_teardown(struct mv88e6xxx_chip *chip);
+#endif
diff --git a/drivers/net/dsa/mxl862xx/mxl862xx.c b/drivers/net/dsa/mxl862xx/mxl862xx.c
index d7ab04f5afefd..c825704c9e9ec 100644
--- a/drivers/net/dsa/mxl862xx/mxl862xx.c
+++ b/drivers/net/dsa/mxl862xx/mxl862xx.c
@@ -44,13 +44,13 @@ static enum dsa_tag_protocol mxl862xx_get_tag_protocol(struct dsa_switch *ds,
 }
 
 /* PHY access via firmware relay */
-static int mxl862xx_phy_read_mmd(struct mxl862xx_priv *priv, int port,
-				 int devadd, int reg)
+static int mxl862xx_phy_read_mmd(struct mxl862xx_priv *priv, int addr,
+				 int devadd, int regnum)
 {
 	struct mdio_relay_data param = {
-		.phy = port,
+		.phy = addr,
 		.mmd = devadd,
-		.reg = cpu_to_le16(reg),
+		.reg = cpu_to_le16(regnum),
 	};
 	int ret;
 
@@ -61,40 +61,40 @@ static int mxl862xx_phy_read_mmd(struct mxl862xx_priv *priv, int port,
 	return le16_to_cpu(param.data);
 }
 
-static int mxl862xx_phy_write_mmd(struct mxl862xx_priv *priv, int port,
-				  int devadd, int reg, u16 data)
+static int mxl862xx_phy_write_mmd(struct mxl862xx_priv *priv, int addr,
+				  int devadd, int regnum, u16 data)
 {
 	struct mdio_relay_data param = {
-		.phy = port,
+		.phy = addr,
 		.mmd = devadd,
-		.reg = cpu_to_le16(reg),
+		.reg = cpu_to_le16(regnum),
 		.data = cpu_to_le16(data),
 	};
 
 	return MXL862XX_API_WRITE(priv, INT_GPHY_WRITE, param);
 }
 
-static int mxl862xx_phy_read_mii_bus(struct mii_bus *bus, int port, int regnum)
+static int mxl862xx_phy_read_mii_bus(struct mii_bus *bus, int addr, int regnum)
 {
-	return mxl862xx_phy_read_mmd(bus->priv, port, 0, regnum);
+	return mxl862xx_phy_read_mmd(bus->priv, addr, 0, regnum);
 }
 
-static int mxl862xx_phy_write_mii_bus(struct mii_bus *bus, int port,
+static int mxl862xx_phy_write_mii_bus(struct mii_bus *bus, int addr,
 				      int regnum, u16 val)
 {
-	return mxl862xx_phy_write_mmd(bus->priv, port, 0, regnum, val);
+	return mxl862xx_phy_write_mmd(bus->priv, addr, 0, regnum, val);
 }
 
-static int mxl862xx_phy_read_c45_mii_bus(struct mii_bus *bus, int port,
+static int mxl862xx_phy_read_c45_mii_bus(struct mii_bus *bus, int addr,
 					 int devadd, int regnum)
 {
-	return mxl862xx_phy_read_mmd(bus->priv, port, devadd, regnum);
+	return mxl862xx_phy_read_mmd(bus->priv, addr, devadd, regnum);
 }
 
-static int mxl862xx_phy_write_c45_mii_bus(struct mii_bus *bus, int port,
+static int mxl862xx_phy_write_c45_mii_bus(struct mii_bus *bus, int addr,
 					  int devadd, int regnum, u16 val)
 {
-	return mxl862xx_phy_write_mmd(bus->priv, port, devadd, regnum, val);
+	return mxl862xx_phy_write_mmd(bus->priv, addr, devadd, regnum, val);
 }
 
 static int mxl862xx_wait_ready(struct dsa_switch *ds)
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index 62bcbbbe2a956..db9d9531711e3 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -76,7 +76,7 @@ static void airoha_set_macaddr(struct airoha_gdm_port *port, const u8 *addr)
 	struct airoha_eth *eth = port->qdma->eth;
 	u32 val, reg;
 
-	reg = airhoa_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H
+	reg = airoha_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H
 					   : REG_FE_WAN_MAC_H;
 	val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
 	airoha_fe_wr(eth, reg, val);
@@ -1611,6 +1611,7 @@ static int airoha_dev_open(struct net_device *dev)
 	int err, len = ETH_HLEN + dev->mtu + ETH_FCS_LEN;
 	struct airoha_gdm_port *port = netdev_priv(dev);
 	struct airoha_qdma *qdma = port->qdma;
+	u32 pse_port = FE_PSE_PORT_PPE1;
 
 	netif_tx_start_all_queues(dev);
 	err = airoha_set_vip_for_gdm_port(port, true);
@@ -1634,6 +1635,14 @@ static int airoha_dev_open(struct net_device *dev)
 			GLOBAL_CFG_RX_DMA_EN_MASK);
 	atomic_inc(&qdma->users);
 
+	if (port->id == AIROHA_GDM2_IDX &&
+	    airoha_ppe_is_enabled(qdma->eth, 1)) {
+		/* For PPE2 always use secondary cpu port. */
+		pse_port = FE_PSE_PORT_PPE2;
+	}
+	airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id),
+				    pse_port);
+
 	return 0;
 }
 
@@ -1651,6 +1660,9 @@ static int airoha_dev_stop(struct net_device *dev)
 	for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++)
 		netdev_tx_reset_subqueue(dev, i);
 
+	airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id),
+				    FE_PSE_PORT_DROP);
+
 	if (atomic_dec_and_test(&qdma->users)) {
 		airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
 				  GLOBAL_CFG_TX_DMA_EN_MASK |
@@ -1727,12 +1739,14 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port)
 	airoha_fe_rmw(eth,
 		      REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)),
 		      SP_CPORT_MASK(val),
-		      FE_PSE_PORT_CDM2 << __ffs(SP_CPORT_MASK(val)));
+		      __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2));
+
+	if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) {
+		u32 mask = FC_ID_OF_SRC_PORT_MASK(nbq);
 
-	if (port->id != AIROHA_GDM3_IDX && airoha_is_7581(eth))
-		airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6,
-			      FC_ID_OF_SRC_PORT24_MASK,
-			      FIELD_PREP(FC_ID_OF_SRC_PORT24_MASK, 2));
+		airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask,
+			      __field_prep(mask, AIROHA_GDM2_IDX));
+	}
 
 	return 0;
 }
@@ -1740,11 +1754,13 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port)
 static int airoha_dev_init(struct net_device *dev)
 {
 	struct airoha_gdm_port *port = netdev_priv(dev);
-	struct airoha_qdma *qdma = port->qdma;
-	struct airoha_eth *eth = qdma->eth;
-	u32 pse_port, fe_cpu_port;
+	struct airoha_eth *eth = port->eth;
+	u32 fe_cpu_port;
 	u8 ppe_id;
 
+	/* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */
+	port->qdma = &eth->qdma[!airoha_is_lan_gdm_port(port)];
+	port->dev->irq = port->qdma->irq_banks[0].irq;
 	airoha_set_macaddr(port, dev->dev_addr);
 
 	switch (port->id) {
@@ -1763,25 +1779,23 @@ static int airoha_dev_init(struct net_device *dev)
 		if (airoha_ppe_is_enabled(eth, 1)) {
 			/* For PPE2 always use secondary cpu port. */
 			fe_cpu_port = FE_PSE_PORT_CDM2;
-			pse_port = FE_PSE_PORT_PPE2;
+			ppe_id = 1;
 			break;
 		}
 		fallthrough;
 	default: {
-		u8 qdma_id = qdma - &eth->qdma[0];
+		u8 qdma_id = port->qdma - &eth->qdma[0];
 
 		/* For PPE1 select cpu port according to the running QDMA. */
 		fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1;
-		pse_port = FE_PSE_PORT_PPE1;
+		ppe_id = 0;
 		break;
 	}
 	}
 
-	airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(port->id), pse_port);
-	ppe_id = pse_port == FE_PSE_PORT_PPE2 ? 1 : 0;
 	airoha_fe_rmw(eth, REG_PPE_DFT_CPORT0(ppe_id),
 		      DFT_CPORT_MASK(port->id),
-		      fe_cpu_port << __ffs(DFT_CPORT_MASK(port->id)));
+		      __field_prep(DFT_CPORT_MASK(port->id), fe_cpu_port));
 
 	return 0;
 }
@@ -2138,7 +2152,7 @@ static int airoha_qdma_set_chan_tx_sched(struct airoha_gdm_port *port,
 
 	airoha_qdma_rmw(port->qdma, REG_CHAN_QOS_MODE(channel >> 3),
 			CHAN_QOS_MODE_MASK(channel),
-			mode << __ffs(CHAN_QOS_MODE_MASK(channel)));
+			__field_prep(CHAN_QOS_MODE_MASK(channel), mode));
 
 	return 0;
 }
@@ -2854,11 +2868,10 @@ bool airoha_is_valid_gdm_port(struct airoha_eth *eth,
 }
 
 static int airoha_alloc_gdm_port(struct airoha_eth *eth,
-				 struct device_node *np, int index)
+				 struct device_node *np)
 {
 	const __be32 *id_ptr = of_get_property(np, "reg", NULL);
 	struct airoha_gdm_port *port;
-	struct airoha_qdma *qdma;
 	struct net_device *dev;
 	int err, p;
 	u32 id;
@@ -2889,7 +2902,6 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth,
 		return -ENOMEM;
 	}
 
-	qdma = &eth->qdma[index % AIROHA_MAX_NUM_QDMA];
 	dev->netdev_ops = &airoha_netdev_ops;
 	dev->ethtool_ops = &airoha_ethtool_ops;
 	dev->max_mtu = AIROHA_MAX_MTU;
@@ -2901,7 +2913,6 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth,
 	dev->features |= dev->hw_features;
 	dev->vlan_features = dev->hw_features;
 	dev->dev.of_node = np;
-	dev->irq = qdma->irq_banks[0].irq;
 	SET_NETDEV_DEV(dev, eth->dev);
 
 	/* reserve hw queues for HTB offloading */
@@ -2922,7 +2933,7 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth,
 	port = netdev_priv(dev);
 	u64_stats_init(&port->stats.syncp);
 	spin_lock_init(&port->stats.lock);
-	port->qdma = qdma;
+	port->eth = eth;
 	port->dev = dev;
 	port->id = id;
 	eth->ports[p] = port;
@@ -3022,7 +3033,6 @@ static int airoha_probe(struct platform_device *pdev)
 	for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
 		airoha_qdma_start_napi(&eth->qdma[i]);
 
-	i = 0;
 	for_each_child_of_node(pdev->dev.of_node, np) {
 		if (!of_device_is_compatible(np, "airoha,eth-mac"))
 			continue;
@@ -3030,7 +3040,7 @@ static int airoha_probe(struct platform_device *pdev)
 		if (!of_device_is_available(np))
 			continue;
 
-		err = airoha_alloc_gdm_port(eth, np, i++);
+		err = airoha_alloc_gdm_port(eth, np);
 		if (err) {
 			of_node_put(np);
 			goto error_napi_stop;
@@ -3083,7 +3093,6 @@ static void airoha_remove(struct platform_device *pdev)
 		if (!port)
 			continue;
 
-		airoha_dev_stop(port->dev);
 		unregister_netdev(port->dev);
 		airoha_metadata_dst_free(port);
 	}
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index 20e602d61e615..8cfa94ec084a1 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -533,6 +533,7 @@ struct airoha_qdma {
 
 struct airoha_gdm_port {
 	struct airoha_qdma *qdma;
+	struct airoha_eth *eth;
 	struct net_device *dev;
 	int id;
 
@@ -626,7 +627,7 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
 #define airoha_qdma_clear(qdma, offset, val)			\
 	airoha_rmw((qdma)->regs, (offset), (val), 0)
 
-static inline bool airhoa_is_lan_gdm_port(struct airoha_gdm_port *port)
+static inline bool airoha_is_lan_gdm_port(struct airoha_gdm_port *port)
 {
 	/* GDM1 port on EN7581 SoC is connected to the lan dsa switch.
 	 * GDM{2,3,4} can be used as wan port connected to an external
diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c
index 42dbe8f93231e..ec5ce41dad808 100644
--- a/drivers/net/ethernet/airoha/airoha_ppe.c
+++ b/drivers/net/ethernet/airoha/airoha_ppe.c
@@ -321,7 +321,7 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
 			/* For downlink traffic consume SRAM memory for hw
 			 * forwarding descriptors queue.
 			 */
-			if (airhoa_is_lan_gdm_port(port))
+			if (airoha_is_lan_gdm_port(port))
 				val |= AIROHA_FOE_IB2_FAST_PATH;
 			if (dsa_port >= 0)
 				val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ,
diff --git a/drivers/net/ethernet/airoha/airoha_regs.h b/drivers/net/ethernet/airoha/airoha_regs.h
index ed4e3407f4a0e..29878b954c77c 100644
--- a/drivers/net/ethernet/airoha/airoha_regs.h
+++ b/drivers/net/ethernet/airoha/airoha_regs.h
@@ -376,10 +376,7 @@
 #define SP_CPORT_MASK(_n)		GENMASK(3 + ((_n) << 2), ((_n) << 2))
 
 #define REG_SRC_PORT_FC_MAP6		0x2298
-#define FC_ID_OF_SRC_PORT27_MASK	GENMASK(28, 24)
-#define FC_ID_OF_SRC_PORT26_MASK	GENMASK(20, 16)
-#define FC_ID_OF_SRC_PORT25_MASK	GENMASK(12, 8)
-#define FC_ID_OF_SRC_PORT24_MASK	GENMASK(4, 0)
+#define FC_ID_OF_SRC_PORT_MASK(_n)	GENMASK(4 + ((_n) << 3), ((_n) << 3))
 
 #define REG_CDM5_RX_OQ1_DROP_CNT	0x29d4
 
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
index 80c2c27ac9dc0..c17900a49595e 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
@@ -832,6 +832,8 @@
 #define PCS_V2_YC_WINDOW_SELECT		0x18064
 #define PCS_V3_RN_WINDOW_DEF		0xf8078
 #define PCS_V3_RN_WINDOW_SELECT		0xf807c
+#define PCS_P100a_WINDOW_DEF		0x8060
+#define PCS_P100a_WINDOW_SELECT		0x8080
 
 #define PCS_RN_SMN_BASE_ADDR		0x11e00000
 #define PCS_RN_PORT_ADDR_SIZE		0x100000
@@ -968,7 +970,7 @@
 #define XP_PROP_0_PORT_MODE_INDEX		8
 #define XP_PROP_0_PORT_MODE_WIDTH		4
 #define XP_PROP_0_PORT_SPEEDS_INDEX		22
-#define XP_PROP_0_PORT_SPEEDS_WIDTH		5
+#define XP_PROP_0_PORT_SPEEDS_WIDTH		6
 #define XP_PROP_1_MAX_RX_DMA_INDEX		24
 #define XP_PROP_1_MAX_RX_DMA_WIDTH		5
 #define XP_PROP_1_MAX_RX_QUEUES_INDEX		8
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
index c04a9c76bd40a..f1357619097ea 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
@@ -718,20 +718,25 @@ static void xgbe_disable_ecc_sec(struct xgbe_prv_data *pdata,
 
 static int xgbe_set_speed(struct xgbe_prv_data *pdata, int speed)
 {
-	unsigned int ss;
+	unsigned int ss, ver;
 
 	switch (speed) {
 	case SPEED_10:
-		ss = 0x07;
+		ss = XGBE_MAC_SS_10M;
 		break;
 	case SPEED_1000:
-		ss = 0x03;
+		ss = XGBE_MAC_SS_1G;
 		break;
 	case SPEED_2500:
-		ss = 0x02;
+		/* P100a uses XGMII mode for 2.5G, older platforms use GMII */
+		ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER);
+		if (ver == XGBE_MAC_VER_33)
+			ss = XGBE_MAC_SS_2_5G_XGMII;
+		else
+			ss = XGBE_MAC_SS_2_5G_GMII;
 		break;
 	case SPEED_10000:
-		ss = 0x00;
+		ss = XGBE_MAC_SS_10G;
 		break;
 	default:
 		return -EINVAL;
@@ -1070,6 +1075,8 @@ static void xgbe_get_pcs_index_and_offset(struct xgbe_prv_data *pdata,
 					  unsigned int *index,
 					  unsigned int *offset)
 {
+	unsigned int ver;
+
 	/* The PCS registers are accessed using mmio. The underlying
 	 * management interface uses indirect addressing to access the MMD
 	 * register sets. This requires accessing of the PCS register in two
@@ -1081,7 +1088,27 @@ static void xgbe_get_pcs_index_and_offset(struct xgbe_prv_data *pdata,
 	 */
 	mmd_address <<= 1;
 	*index = mmd_address & ~pdata->xpcs_window_mask;
-	*offset = pdata->xpcs_window + (mmd_address & pdata->xpcs_window_mask);
+
+	ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER);
+
+	/* The P100a platform uses a different memory mapping scheme for XPCS
+	 * register access. The offset calculation differs between platforms:
+	 *
+	 * For P100a platforms: The offset is calculated by adding the
+	 * mmd_address to the xpcs_window and then applying the xpcs_window_mask
+	 * For older platforms: The offset is calculated by applying the
+	 * xpcs_window_mask to the mmd_address and then adding it to the
+	 * xpcs_window.
+	 *
+	 * This is critical because using the wrong calculation causes register
+	 * accesses to target the wrong registers, leading to incorrect behavior
+	 */
+	if (ver == XGBE_MAC_VER_33)
+		*offset = (pdata->xpcs_window + mmd_address) &
+			  pdata->xpcs_window_mask;
+	else
+		*offset = pdata->xpcs_window +
+			  (mmd_address & pdata->xpcs_window_mask);
 }
 
 static int xgbe_read_mmd_regs_v3(struct xgbe_prv_data *pdata, int prtad,
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index 23beea48ae26b..2f39f38fecf9f 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -1116,69 +1116,62 @@ static int xgbe_phy_reset(struct xgbe_prv_data *pdata)
 	return pdata->phy_if.phy_reset(pdata);
 }
 
-int xgbe_powerdown(struct net_device *netdev, unsigned int caller)
+int xgbe_powerdown(struct net_device *netdev)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 
-	DBGPR("-->xgbe_powerdown\n");
-
-	if (!netif_running(netdev) ||
-	    (caller == XGMAC_IOCTL_CONTEXT && pdata->power_down)) {
-		netdev_alert(netdev, "Device is already powered down\n");
-		DBGPR("<--xgbe_powerdown\n");
+	if (!netif_running(netdev)) {
+		netdev_dbg(netdev, "Device is not running, skipping powerdown\n");
 		return -EINVAL;
 	}
 
-	if (caller == XGMAC_DRIVER_CONTEXT)
-		netif_device_detach(netdev);
+	if (pdata->power_down) {
+		netdev_dbg(netdev, "Device is already powered down\n");
+		return -EINVAL;
+	}
 
+	netif_device_detach(netdev);
 	netif_tx_stop_all_queues(netdev);
 
 	xgbe_stop_timers(pdata);
 	flush_workqueue(pdata->dev_workqueue);
 
+	xgbe_napi_disable(pdata, 0);
+
 	hw_if->powerdown_tx(pdata);
 	hw_if->powerdown_rx(pdata);
 
-	xgbe_napi_disable(pdata, 0);
-
 	pdata->power_down = 1;
 
-	DBGPR("<--xgbe_powerdown\n");
-
 	return 0;
 }
 
-int xgbe_powerup(struct net_device *netdev, unsigned int caller)
+int xgbe_powerup(struct net_device *netdev)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 
-	DBGPR("-->xgbe_powerup\n");
-
-	if (!netif_running(netdev) ||
-	    (caller == XGMAC_IOCTL_CONTEXT && !pdata->power_down)) {
-		netdev_alert(netdev, "Device is already powered up\n");
-		DBGPR("<--xgbe_powerup\n");
+	if (!netif_running(netdev)) {
+		netdev_dbg(netdev, "Device is not running, skipping powerup\n");
 		return -EINVAL;
 	}
 
-	pdata->power_down = 0;
-
-	xgbe_napi_enable(pdata, 0);
+	if (!pdata->power_down) {
+		netdev_dbg(netdev, "Device is already powered up\n");
+		return -EINVAL;
+	}
 
 	hw_if->powerup_tx(pdata);
 	hw_if->powerup_rx(pdata);
 
-	if (caller == XGMAC_DRIVER_CONTEXT)
-		netif_device_attach(netdev);
+	xgbe_napi_enable(pdata, 0);
 
 	netif_tx_start_all_queues(netdev);
-
 	xgbe_start_timers(pdata);
+	netif_device_attach(netdev);
 
-	DBGPR("<--xgbe_powerup\n");
+	pdata->power_down = 0;
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
index e3e1dca9856ac..dbdd791380a47 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
@@ -168,6 +168,14 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
 	if (rdev && rdev->vendor == PCI_VENDOR_ID_AMD) {
 		switch (rdev->device) {
+		case XGBE_P100a_PCI_DEVICE_ID:
+			pdata->xpcs_window_def_reg = PCS_P100a_WINDOW_DEF;
+			pdata->xpcs_window_sel_reg = PCS_P100a_WINDOW_SELECT;
+
+			/* P100a devices do not need rrc and cdr workaround */
+			pdata->vdata->an_cdr_workaround = 0;
+			pdata->vdata->enable_rrc = 0;
+			break;
 		case XGBE_RV_PCI_DEVICE_ID:
 			pdata->xpcs_window_def_reg = PCS_V2_RV_WINDOW_DEF;
 			pdata->xpcs_window_sel_reg = PCS_V2_RV_WINDOW_SELECT;
@@ -352,35 +360,102 @@ static void xgbe_pci_remove(struct pci_dev *pdev)
 	xgbe_free_pdata(pdata);
 }
 
-static int __maybe_unused xgbe_pci_suspend(struct device *dev)
+static void xgbe_pci_synchronize_irqs(struct xgbe_prv_data *pdata)
+{
+	unsigned int i;
+
+	/* Synchronize main device interrupt */
+	synchronize_irq(pdata->dev_irq);
+
+	/* Synchronize ECC interrupt if separate from main device interrupt */
+	if (pdata->vdata->ecc_support && pdata->dev_irq != pdata->ecc_irq)
+		synchronize_irq(pdata->ecc_irq);
+
+	/* Synchronize I2C interrupt if separate from main device interrupt */
+	if (pdata->vdata->i2c_support && pdata->dev_irq != pdata->i2c_irq)
+		synchronize_irq(pdata->i2c_irq);
+
+	/* Synchronize AN interrupt if separate from main device interrupt */
+	if (pdata->dev_irq != pdata->an_irq)
+		synchronize_irq(pdata->an_irq);
+
+	/* Synchronize per-channel DMA interrupts */
+	if (pdata->per_channel_irq) {
+		for (i = 0; i < pdata->channel_count; i++)
+			synchronize_irq(pdata->channel[i]->dma_irq);
+	}
+}
+
+static int xgbe_pci_suspend(struct device *dev)
 {
 	struct xgbe_prv_data *pdata = dev_get_drvdata(dev);
 	struct net_device *netdev = pdata->netdev;
+	struct pci_dev *pdev = to_pci_dev(dev);
 	int ret = 0;
 
 	if (netif_running(netdev))
-		ret = xgbe_powerdown(netdev, XGMAC_DRIVER_CONTEXT);
+		ret = xgbe_powerdown(netdev);
+
+	/* Disable all device interrupts to prevent spurious wakeups */
+	XP_IOWRITE(pdata, XP_INT_EN, 0x0);
+
+	/* Ensure no IRQ handlers are still executing before powering down.
+	 * This prevents race conditions where an IRQ handler could access
+	 * invalid register state after the device is disabled.
+	 */
+	xgbe_pci_synchronize_irqs(pdata);
 
+	/* Set PHY to low-power mode */
 	pdata->lpm_ctrl = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_CTRL1);
 	pdata->lpm_ctrl |= MDIO_CTRL1_LPOWER;
 	XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl);
 
+	/* Disable bus mastering to prevent DMA activity */
+	pci_clear_master(pdev);
+
+	/* Save PCI configuration state and disable device */
+	pci_save_state(pdev);
+	pci_disable_device(pdev);
+
+	/* Disable wake from D3 - required for S0i3 deep sleep */
+	pci_wake_from_d3(pdev, false);
+	pci_set_power_state(pdev, PCI_D3hot);
+
 	return ret;
 }
 
-static int __maybe_unused xgbe_pci_resume(struct device *dev)
+static int xgbe_pci_resume(struct device *dev)
 {
 	struct xgbe_prv_data *pdata = dev_get_drvdata(dev);
 	struct net_device *netdev = pdata->netdev;
+	struct pci_dev *pdev = to_pci_dev(dev);
 	int ret = 0;
 
+	/* Restore PCI power state */
+	pci_set_power_state(pdev, PCI_D0);
+
+	/* Restore PCI configuration state */
+	pci_restore_state(pdev);
+
+	/* Enable PCI device */
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(dev, "pci_enable_device failed: %d\n", ret);
+		return ret;
+	}
+
+	/* Re-enable bus mastering */
+	pci_set_master(pdev);
+
+	/* Re-enable all device interrupts */
 	XP_IOWRITE(pdata, XP_INT_EN, 0x1fffff);
 
+	/* Clear PHY low-power mode */
 	pdata->lpm_ctrl &= ~MDIO_CTRL1_LPOWER;
 	XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl);
 
 	if (netif_running(netdev)) {
-		ret = xgbe_powerup(netdev, XGMAC_DRIVER_CONTEXT);
+		ret = xgbe_powerup(netdev);
 
 		/* Schedule a restart in case the link or phy state changed
 		 * while we were powered down.
@@ -453,16 +528,16 @@ static const struct pci_device_id xgbe_pci_table[] = {
 };
 MODULE_DEVICE_TABLE(pci, xgbe_pci_table);
 
-static SIMPLE_DEV_PM_OPS(xgbe_pci_pm_ops, xgbe_pci_suspend, xgbe_pci_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(xgbe_pci_pm_ops,
+				xgbe_pci_suspend,
+				xgbe_pci_resume);
 
 static struct pci_driver xgbe_driver = {
 	.name = XGBE_DRV_NAME,
 	.id_table = xgbe_pci_table,
 	.probe = xgbe_pci_probe,
 	.remove = xgbe_pci_remove,
-	.driver = {
-		.pm = &xgbe_pci_pm_ops,
-	}
+	.driver.pm = pm_sleep_ptr(&xgbe_pci_pm_ops),
 };
 
 int xgbe_pci_init(void)
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
index b8cf6ccfe6414..59a074ed312a5 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
@@ -2183,7 +2183,11 @@ static bool enable_rx_adap(struct xgbe_prv_data *pdata, enum xgbe_mode mode)
 
 	/* Rx-Adaptation is not supported on older platforms(< 0x30H) */
 	ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER);
-	if (ver < 0x30)
+	if (ver < XGBE_MAC_VER_30)
+		return false;
+
+	/* Rx adaptation not yet supported on P100a */
+	if (ver == XGBE_MAC_VER_33)
 		return false;
 
 	/* Re-driver models 4223 && 4227 do not support Rx-Adaptation */
@@ -2309,12 +2313,24 @@ static void xgbe_phy_kr_mode(struct xgbe_prv_data *pdata)
 
 static void xgbe_phy_kx_2500_mode(struct xgbe_prv_data *pdata)
 {
+	unsigned int ver;
+
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 
 	xgbe_phy_set_redrv_mode(pdata);
 
-	/* 2.5G/KX */
-	xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G, XGBE_MB_SUBCMD_NONE);
+	ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER);
+
+	/*
+	 * P100a uses XGMII mode for 2.5G which requires the 2.5G_KX subcommand.
+	 * Older platforms use GMII mode.
+	 */
+	if (ver == XGBE_MAC_VER_33)
+		xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G,
+					    XGBE_MB_SUBCMD_2_5G_KX);
+	else
+		xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G,
+					    XGBE_MB_SUBCMD_NONE);
 
 	phy_data->cur_mode = XGBE_MODE_KX_2500;
 
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-platform.c b/drivers/net/ethernet/amd/xgbe/xgbe-platform.c
index 47d53e59ccf63..bca0dbff80c17 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-platform.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-platform.c
@@ -252,13 +252,9 @@ static int xgbe_platform_probe(struct platform_device *pdev)
 		dev_dbg(dev, "sir1_regs  = %p\n", pdata->sir1_regs);
 
 	/* Retrieve the MAC address */
-	ret = device_property_read_u8_array(dev, XGBE_MAC_ADDR_PROPERTY,
-					    pdata->mac_addr,
-					    sizeof(pdata->mac_addr));
-	if (ret || !is_valid_ether_addr(pdata->mac_addr)) {
-		dev_err(dev, "invalid %s property\n", XGBE_MAC_ADDR_PROPERTY);
-		if (!ret)
-			ret = -EINVAL;
+	ret = device_get_mac_address(dev, pdata->mac_addr);
+	if (ret) {
+		dev_err(dev, "invalid MAC address property\n");
 		goto err_io;
 	}
 
@@ -384,7 +380,7 @@ static int xgbe_platform_suspend(struct device *dev)
 	DBGPR("-->xgbe_suspend\n");
 
 	if (netif_running(netdev))
-		ret = xgbe_powerdown(netdev, XGMAC_DRIVER_CONTEXT);
+		ret = xgbe_powerdown(netdev);
 
 	pdata->lpm_ctrl = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_CTRL1);
 	pdata->lpm_ctrl |= MDIO_CTRL1_LPOWER;
@@ -407,7 +403,7 @@ static int xgbe_platform_resume(struct device *dev)
 	XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl);
 
 	if (netif_running(netdev)) {
-		ret = xgbe_powerup(netdev, XGMAC_DRIVER_CONTEXT);
+		ret = xgbe_powerup(netdev);
 
 		/* Schedule a restart in case the link or phy state changed
 		 * while we were powered down.
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h
index 438033a715238..60b7e53206d1e 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe.h
@@ -96,7 +96,6 @@
 	min_t(unsigned int, IEEE_8021QAZ_MAX_TCS, (_cnt))
 
 /* Common property names */
-#define XGBE_MAC_ADDR_PROPERTY	"mac-address"
 #define XGBE_PHY_MODE_PROPERTY	"phy-mode"
 #define XGBE_DMA_IRQS_PROPERTY	"amd,per-channel-interrupt"
 #define XGBE_SPEEDSET_PROPERTY	"amd,speed-set"
@@ -146,10 +145,6 @@
 #define XGBE_MAX_PPS_OUT	4
 #define XGBE_MAX_AUX_SNAP	4
 
-/* Driver PMT macros */
-#define XGMAC_DRIVER_CONTEXT	1
-#define XGMAC_IOCTL_CONTEXT	2
-
 #define XGMAC_FIFO_MIN_ALLOC	2048
 #define XGMAC_FIFO_UNIT		256
 #define XGMAC_FIFO_ALIGN(_x)				\
@@ -262,11 +257,31 @@
 #define XGBE_RV_PCI_DEVICE_ID	0x15d0
 #define XGBE_YC_PCI_DEVICE_ID	0x14b5
 #define XGBE_RN_PCI_DEVICE_ID	0x1630
+#define XGBE_P100a_PCI_DEVICE_ID	0x1122
 
  /* Generic low and high masks */
 #define XGBE_GEN_HI_MASK	GENMASK(31, 16)
 #define XGBE_GEN_LO_MASK	GENMASK(15, 0)
 
+/* MAC hardware version numbers (SNPSVER field in MAC_VR register) */
+#define XGBE_MAC_VER_30		0x30	/* Baseline Rx adaptation support */
+#define XGBE_MAC_VER_33		0x33	/* P100a platform */
+
+/* MAC Speed Select (SS) values for MAC_TCR register
+ * These values are written to the SS field to configure link speed.
+ * Note: P100a uses XGMII mode (0x06) for 2.5G instead of GMII (0x02)
+ */
+/* Note: 100M and 2.5G GMII share the same value (0x02) but are
+ * differentiated by the mode/interface type at the PHY level
+ */
+
+#define XGBE_MAC_SS_10G		0x00	/* 10Gbps - XGMII mode */
+#define XGBE_MAC_SS_2_5G_GMII	0x02	/* 2.5Gbps - GMII mode (YC) */
+#define XGBE_MAC_SS_2_5G_XGMII	0x06	/* 2.5Gbps - XGMII mode (P100a) */
+#define XGBE_MAC_SS_1G		0x03	/* 1Gbps */
+#define XGBE_MAC_SS_100M	0x02	/* 100Mbps */
+#define XGBE_MAC_SS_10M		0x07	/* 10Mbps */
+
 struct xgbe_prv_data;
 
 struct xgbe_packet_data {
@@ -558,7 +573,10 @@ enum xgbe_mb_subcmd {
 	XGBE_MB_SUBCMD_10MBITS = 0,
 	XGBE_MB_SUBCMD_100MBITS,
 	XGBE_MB_SUBCMD_1G_SGMII,
-	XGBE_MB_SUBCMD_1G_KX
+	XGBE_MB_SUBCMD_1G_KX,
+
+	/* 2.5GbE Mode subcommands */
+	XGBE_MB_SUBCMD_2_5G_KX = 1
 };
 
 struct xgbe_phy {
@@ -1290,8 +1308,8 @@ void xgbe_dump_rx_desc(struct xgbe_prv_data *, struct xgbe_ring *,
 		       unsigned int);
 void xgbe_print_pkt(struct net_device *, struct sk_buff *, bool);
 void xgbe_get_all_hw_features(struct xgbe_prv_data *);
-int xgbe_powerup(struct net_device *, unsigned int);
-int xgbe_powerdown(struct net_device *, unsigned int);
+int xgbe_powerup(struct net_device *netdev);
+int xgbe_powerdown(struct net_device *netdev);
 void xgbe_init_rx_coalesce(struct xgbe_prv_data *);
 void xgbe_init_tx_coalesce(struct xgbe_prv_data *);
 void xgbe_restart_dev(struct xgbe_prv_data *pdata);
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
index a6e1826dd5d7e..420af958d4861 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
@@ -983,6 +983,38 @@ static int aq_ethtool_set_phy_tunable(struct net_device *ndev,
 	return err;
 }
 
+static bool aq_ethtool_can_read_module_eeprom(struct aq_nic_s *aq_nic)
+{
+	return aq_nic->aq_fw_ops->read_module_eeprom ||
+		aq_nic->aq_hw_ops->hw_read_module_eeprom;
+}
+
+static int aq_ethtool_read_module_eeprom(struct aq_nic_s *aq_nic, u8 dev_addr,
+					 u8 reg_start_addr, int len, u8 *data)
+{
+	const struct aq_fw_ops *fw_ops = aq_nic->aq_fw_ops;
+	const struct aq_hw_ops *hw_ops = aq_nic->aq_hw_ops;
+	int err = -EOPNOTSUPP;
+
+	if (fw_ops->read_module_eeprom) {
+		err = fw_ops->read_module_eeprom(aq_nic->aq_hw, dev_addr,
+						 reg_start_addr, len, data);
+
+		/* If the only error is that the firmware version doesn't
+		 * support reading EEPROM, we can still attempt to read it
+		 * directly from the hardware if supported.
+		 */
+		if (err != -EOPNOTSUPP)
+			return err;
+	}
+
+	if (hw_ops->hw_read_module_eeprom)
+		err = hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, dev_addr,
+						    reg_start_addr, len, data);
+
+	return err;
+}
+
 static int aq_ethtool_get_module_info(struct net_device *ndev,
 				      struct ethtool_modinfo *modinfo)
 {
@@ -992,16 +1024,18 @@ static int aq_ethtool_get_module_info(struct net_device *ndev,
 
 	/* Module EEPROM is only supported for controllers with external PHY */
 	if (aq_nic->aq_nic_cfg.aq_hw_caps->media_type != AQ_HW_MEDIA_TYPE_FIBRE ||
-	    !aq_nic->aq_hw_ops->hw_read_module_eeprom)
+	    !aq_ethtool_can_read_module_eeprom(aq_nic))
 		return -EOPNOTSUPP;
 
-	err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw,
-		SFF_8472_ID_ADDR, SFF_8472_COMP_ADDR, 1, &compliance_val);
+	err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR,
+					    SFF_8472_COMP_ADDR, 1,
+					    &compliance_val);
 	if (err)
 		return err;
 
-	err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw,
-		SFF_8472_ID_ADDR, SFF_8472_DOM_TYPE_ADDR, 1, &dom_type);
+	err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR,
+					    SFF_8472_DOM_TYPE_ADDR, 1,
+					    &dom_type);
 	if (err)
 		return err;
 
@@ -1022,7 +1056,7 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev,
 	unsigned int first, last, len;
 	int err;
 
-	if (!aq_nic->aq_hw_ops->hw_read_module_eeprom)
+	if (!aq_ethtool_can_read_module_eeprom(aq_nic))
 		return -EOPNOTSUPP;
 
 	first = ee->offset;
@@ -1032,8 +1066,8 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev,
 		len = min(last, ETH_MODULE_SFF_8079_LEN);
 		len -= first;
 
-		err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw,
-			SFF_8472_ID_ADDR, first, len, data);
+		err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR,
+						    first, len, data);
 		if (err)
 			return err;
 
@@ -1045,8 +1079,9 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev,
 		len -= first;
 		first -= ETH_MODULE_SFF_8079_LEN;
 
-		err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw,
-			SFF_8472_DIAGNOSTICS_ADDR, first, len, data);
+		err = aq_ethtool_read_module_eeprom(aq_nic,
+						    SFF_8472_DIAGNOSTICS_ADDR,
+						    first, len, data);
 		if (err)
 			return err;
 	}
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
index 4e66fd9b2ab1d..57ea59026a2c1 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
@@ -404,6 +404,9 @@ struct aq_fw_ops {
 	int (*send_macsec_req)(struct aq_hw_s *self,
 			       struct macsec_msg_fw_request *msg,
 			       struct macsec_msg_fw_response *resp);
+
+	int (*read_module_eeprom)(struct aq_hw_s *self, u8 dev_addr,
+				  u8 reg_start_addr, int len, u8 *data);
 };
 
 #endif /* AQ_HW_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
index f6b990b7f5b47..404c84adad4a6 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
@@ -319,6 +319,13 @@ struct __packed hw_atl_utils_settings {
 	u32 media_detect;
 };
 
+struct __packed smbus_request {
+	u32 msg_id;
+	u32 device_id;
+	u32 address;
+	u32 length;
+};
+
 enum macsec_msg_type {
 	macsec_cfg_msg = 0,
 	macsec_add_rx_sc_msg,
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
index 4d4cfbc91e19c..2cac0d9670bf0 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
@@ -703,6 +703,85 @@ static int aq_fw2x_send_macsec_req(struct aq_hw_s *hw,
 	return err;
 }
 
+static int aq_fw2x_read_module_eeprom(struct aq_hw_s *self, u8 dev_addr,
+				      u8 reg_start_addr, int len, u8 *data)
+{
+	u32 low_status, orig_low_status, low_req = 0;
+	u32 res_bytes_remain_cnt = len % sizeof(u32);
+	u32 res_dword_cnt = len / sizeof(u32);
+	struct smbus_request request = { 0 };
+	u32 req_dword_cnt;
+	u32 result = 0;
+	u32 caps_lo;
+	u32 offset;
+	int err;
+
+	caps_lo = aq_fw2x_get_link_capabilities(self);
+	if (!(caps_lo & BIT(CAPS_LO_SMBUS_READ)))
+		return -EOPNOTSUPP;
+
+	request.msg_id = 0;
+	request.device_id = dev_addr;
+	request.address = reg_start_addr;
+	request.length = len;
+
+	/* Write SMBUS request to cfg memory */
+	req_dword_cnt = DIV_ROUND_UP(sizeof(request), sizeof(u32));
+	err = hw_atl_write_fwcfg_dwords(self, (void *)&request, req_dword_cnt);
+	if (err < 0)
+		return err;
+
+	/* Toggle 0x368.CAPS_LO_SMBUS_READ bit */
+	low_req = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR);
+	orig_low_status = low_req & BIT(CAPS_LO_SMBUS_READ);
+	low_req ^= BIT(CAPS_LO_SMBUS_READ);
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, low_req);
+
+	/* Wait FW to report back */
+	err = readx_poll_timeout_atomic(aq_fw2x_state_get, self, low_status,
+					orig_low_status != (low_status &
+					BIT(CAPS_LO_SMBUS_READ)),
+					10U, 100000U);
+	if (err)
+		return err;
+
+	/* Read status of read operation */
+	offset = self->rpc_addr + sizeof(u32);
+	err = hw_atl_utils_fw_downld_dwords(self, offset, &result,
+					    sizeof(result) / sizeof(u32));
+	if (err < 0)
+		return err;
+	if (result)
+		return -EIO;
+
+	/* Read response full DWORD data */
+	if (res_dword_cnt) {
+		offset = self->rpc_addr + sizeof(u32) * 2;
+		err = hw_atl_utils_fw_downld_dwords(self, offset, (u32 *)data,
+						    res_dword_cnt);
+		if (err < 0)
+			return err;
+	}
+
+	/* Read response trailing bytes data */
+	if (res_bytes_remain_cnt) {
+		u32 bytes_remain_val = 0;
+
+		offset = self->rpc_addr +
+			 (sizeof(u32) * 2) +
+			 (res_dword_cnt * sizeof(u32));
+		err = hw_atl_utils_fw_downld_dwords(self, offset,
+						    &bytes_remain_val, 1);
+		if (err < 0)
+			return err;
+
+		memcpy(data + len - res_bytes_remain_cnt,
+		       &bytes_remain_val, res_bytes_remain_cnt);
+	}
+
+	return 0;
+}
+
 const struct aq_fw_ops aq_fw_2x_ops = {
 	.init               = aq_fw2x_init,
 	.deinit             = aq_fw2x_deinit,
@@ -729,4 +808,5 @@ const struct aq_fw_ops aq_fw_2x_ops = {
 	.adjust_ptp         = aq_fw3x_adjust_ptp,
 	.get_link_capabilities = aq_fw2x_get_link_capabilities,
 	.send_macsec_req    = aq_fw2x_send_macsec_req,
+	.read_module_eeprom = aq_fw2x_read_module_eeprom,
 };
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index c426a41c36639..604966a398f58 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -2929,6 +2929,8 @@ static int bnxt_async_event_process(struct bnxt *bp,
 		u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1);
 		u32 offset =  BNXT_EVENT_BUF_PRODUCER_OFFSET(data2);
 
+		if (type >= ARRAY_SIZE(bp->bs_trace))
+			goto async_event_process_exit;
 		bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset);
 		goto async_event_process_exit;
 	}
@@ -13243,7 +13245,7 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
 	/* Save ring stats before shutdown */
 	if (bp->bnapi && irq_re_init) {
 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
-		bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
+		bnxt_get_ring_drv_stats(bp, &bp->ring_drv_stats_prev);
 	}
 	if (irq_re_init) {
 		bnxt_free_irq(bp);
@@ -13488,8 +13490,8 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
 }
 
-static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
-					struct bnxt_total_ring_err_stats *stats,
+static void bnxt_get_one_ring_drv_stats(struct bnxt *bp,
+					struct bnxt_total_ring_drv_stats *stats,
 					struct bnxt_cp_ring_info *cpr)
 {
 	struct bnxt_sw_stats *sw_stats = cpr->sw_stats;
@@ -13510,13 +13512,13 @@ static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
 	stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
 }
 
-void bnxt_get_ring_err_stats(struct bnxt *bp,
-			     struct bnxt_total_ring_err_stats *stats)
+void bnxt_get_ring_drv_stats(struct bnxt *bp,
+			     struct bnxt_total_ring_drv_stats *stats)
 {
 	int i;
 
 	for (i = 0; i < bp->cp_nr_rings; i++)
-		bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
+		bnxt_get_one_ring_drv_stats(bp, stats, &bp->bnapi[i]->cp_ring);
 }
 
 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
@@ -15961,9 +15963,9 @@ static void bnxt_get_base_stats(struct net_device *dev,
 
 	rx->packets = bp->net_stats_prev.rx_packets;
 	rx->bytes = bp->net_stats_prev.rx_bytes;
-	rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards;
-	rx->hw_gro_packets = bp->ring_err_stats_prev.rx_total_hw_gro_packets;
-	rx->hw_gro_wire_packets = bp->ring_err_stats_prev.rx_total_hw_gro_wire_packets;
+	rx->alloc_fail = bp->ring_drv_stats_prev.rx_total_oom_discards;
+	rx->hw_gro_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_packets;
+	rx->hw_gro_wire_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_wire_packets;
 
 	tx->packets = bp->net_stats_prev.tx_packets;
 	tx->bytes = bp->net_stats_prev.tx_bytes;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 9a41b9e0423c7..dd0f6743acf5b 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -1147,7 +1147,7 @@ struct bnxt_sw_stats {
 	struct bnxt_cmn_sw_stats cmn;
 };
 
-struct bnxt_total_ring_err_stats {
+struct bnxt_total_ring_drv_stats {
 	u64			rx_total_l4_csum_errors;
 	u64			rx_total_resets;
 	u64			rx_total_buf_errors;
@@ -2146,7 +2146,7 @@ enum board_idx {
 };
 
 #define BNXT_TRACE_BUF_MAGIC_BYTE ((u8)0xbc)
-#define BNXT_TRACE_MAX 11
+#define BNXT_TRACE_MAX (DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE + 1)
 
 struct bnxt_bs_trace_info {
 	u8 *magic_byte;
@@ -2572,7 +2572,7 @@ struct bnxt {
 	u8			pri2cos_idx[8];
 	u8			pri2cos_valid;
 
-	struct bnxt_total_ring_err_stats ring_err_stats_prev;
+	struct bnxt_total_ring_drv_stats ring_drv_stats_prev;
 
 	u16			hwrm_max_req_len;
 	u16			hwrm_max_ext_req_len;
@@ -2974,8 +2974,8 @@ int bnxt_half_open_nic(struct bnxt *bp);
 void bnxt_half_close_nic(struct bnxt *bp);
 void bnxt_reenable_sriov(struct bnxt *bp);
 void bnxt_close_nic(struct bnxt *, bool, bool);
-void bnxt_get_ring_err_stats(struct bnxt *bp,
-			     struct bnxt_total_ring_err_stats *stats);
+void bnxt_get_ring_drv_stats(struct bnxt *bp,
+			     struct bnxt_total_ring_drv_stats *stats);
 bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx);
 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
 			 u32 *reg_buf);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 28d0ece2e7b1f..48e8e3be70d38 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -340,7 +340,7 @@ enum {
 	RX_NETPOLL_DISCARDS,
 };
 
-static const char *const bnxt_ring_err_stats_arr[] = {
+static const char *const bnxt_ring_drv_stats_arr[] = {
 	"rx_total_l4_csum_errors",
 	"rx_total_resets",
 	"rx_total_buf_errors",
@@ -500,7 +500,7 @@ static const struct {
 	BNXT_TX_STATS_PRI_ENTRIES(tx_packets),
 };
 
-#define BNXT_NUM_RING_ERR_STATS	ARRAY_SIZE(bnxt_ring_err_stats_arr)
+#define BNXT_NUM_RING_DRV_STATS	ARRAY_SIZE(bnxt_ring_drv_stats_arr)
 #define BNXT_NUM_PORT_STATS ARRAY_SIZE(bnxt_port_stats_arr)
 #define BNXT_NUM_STATS_PRI			\
 	(ARRAY_SIZE(bnxt_rx_bytes_pri_arr) +	\
@@ -539,7 +539,7 @@ static int bnxt_get_num_stats(struct bnxt *bp)
 	int num_stats = bnxt_get_num_ring_stats(bp);
 	int len;
 
-	num_stats += BNXT_NUM_RING_ERR_STATS;
+	num_stats += BNXT_NUM_RING_DRV_STATS;
 
 	if (bp->flags & BNXT_FLAG_PORT_STATS)
 		num_stats += BNXT_NUM_PORT_STATS;
@@ -594,7 +594,7 @@ static bool is_tx_ring(struct bnxt *bp, int ring_num)
 static void bnxt_get_ethtool_stats(struct net_device *dev,
 				   struct ethtool_stats *stats, u64 *buf)
 {
-	struct bnxt_total_ring_err_stats ring_err_stats = {0};
+	struct bnxt_total_ring_drv_stats ring_drv_stats = {0};
 	struct bnxt *bp = netdev_priv(dev);
 	u64 *curr, *prev;
 	u32 tpa_stats;
@@ -643,12 +643,12 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,
 			buf[j] = sw[k];
 	}
 
-	bnxt_get_ring_err_stats(bp, &ring_err_stats);
+	bnxt_get_ring_drv_stats(bp, &ring_drv_stats);
 
 skip_ring_stats:
-	curr = &ring_err_stats.rx_total_l4_csum_errors;
-	prev = &bp->ring_err_stats_prev.rx_total_l4_csum_errors;
-	for (i = 0; i < BNXT_NUM_RING_ERR_STATS; i++, j++, curr++, prev++)
+	curr = &ring_drv_stats.rx_total_l4_csum_errors;
+	prev = &bp->ring_drv_stats_prev.rx_total_l4_csum_errors;
+	for (i = 0; i < BNXT_NUM_RING_DRV_STATS; i++, j++, curr++, prev++)
 		buf[j] = *curr + *prev;
 
 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
@@ -758,8 +758,8 @@ static void bnxt_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
 				ethtool_sprintf(&buf, "[%d]: %s", i, str);
 			}
 		}
-		for (i = 0; i < BNXT_NUM_RING_ERR_STATS; i++)
-			ethtool_puts(&buf, bnxt_ring_err_stats_arr[i]);
+		for (i = 0; i < BNXT_NUM_RING_DRV_STATS; i++)
+			ethtool_puts(&buf, bnxt_ring_drv_stats_arr[i]);
 
 		if (bp->flags & BNXT_FLAG_PORT_STATS)
 			for (i = 0; i < BNXT_NUM_PORT_STATS; i++) {
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
index 8fb5512882980..96d5d4f7f51fe 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c
@@ -123,7 +123,7 @@ static int bcmgenet_poll_wol_status(struct bcmgenet_priv *priv)
 	while (!(bcmgenet_rbuf_readl(priv, RBUF_STATUS)
 		& RBUF_STATUS_WOL)) {
 		retries++;
-		if (retries > 5) {
+		if (retries > 50) {
 			netdev_crit(dev, "polling wol mode timeout\n");
 			return -ETIMEDOUT;
 		}
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index 2328fce336447..21a5dd3427244 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -17029,6 +17029,13 @@ static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent)
 	return err;
 }
 
+static int tg3_is_default_mac_address(u8 *addr)
+{
+	static const u8 default_mac_address[ETH_ALEN] = { 0x00, 0x10, 0x18, 0x00, 0x00, 0x00 };
+
+	return ether_addr_equal(default_mac_address, addr);
+}
+
 static int tg3_get_device_address(struct tg3 *tp, u8 *addr)
 {
 	u32 hi, lo, mac_offset;
@@ -17102,6 +17109,10 @@ static int tg3_get_device_address(struct tg3 *tp, u8 *addr)
 
 	if (!is_valid_ether_addr(addr))
 		return -EINVAL;
+
+	if (tg3_is_default_mac_address(addr))
+		return device_get_mac_address(&tp->pdev->dev, addr);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index 87414a2ddf6e3..c69828b27dae6 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -170,6 +170,10 @@
 #define GEM_PCSANNPTX		0x021c /* PCS AN Next Page TX */
 #define GEM_PCSANNPLP		0x0220 /* PCS AN Next Page LP */
 #define GEM_PCSANEXTSTS		0x023c /* PCS AN Extended Status */
+#define GEM_RXLPI		0x0270 /* RX LPI Transitions */
+#define GEM_RXLPITIME		0x0274 /* RX LPI Time */
+#define GEM_TXLPI		0x0278 /* TX LPI Transitions */
+#define GEM_TXLPITIME		0x027c /* TX LPI Time */
 #define GEM_DCFG1		0x0280 /* Design Config 1 */
 #define GEM_DCFG2		0x0284 /* Design Config 2 */
 #define GEM_DCFG3		0x0288 /* Design Config 3 */
@@ -305,6 +309,8 @@
 #define MACB_IRXFCS_SIZE	1
 
 /* GEM specific NCR bitfields. */
+#define GEM_TXLPIEN_OFFSET		19
+#define GEM_TXLPIEN_SIZE		1
 #define GEM_ENABLE_HS_MAC_OFFSET	31
 #define GEM_ENABLE_HS_MAC_SIZE		1
 
@@ -779,6 +785,7 @@
 #define MACB_CAPS_DMA_PTP			BIT(22)
 #define MACB_CAPS_RSC				BIT(23)
 #define MACB_CAPS_NO_LSO			BIT(24)
+#define MACB_CAPS_EEE				BIT(25)
 
 /* LSO settings */
 #define MACB_LSO_UFO_ENABLE			0x01
@@ -1043,6 +1050,10 @@ struct gem_stats {
 	u64	rx_ip_header_checksum_errors;
 	u64	rx_tcp_checksum_errors;
 	u64	rx_udp_checksum_errors;
+	u64	rx_lpi_transitions;
+	u64	rx_lpi_time;
+	u64	tx_lpi_transitions;
+	u64	tx_lpi_time;
 };
 
 /* Describes the name and offset of an individual statistic register, as
@@ -1142,6 +1153,10 @@ static const struct gem_statistic gem_statistics[] = {
 			    GEM_BIT(NDS_RXERR)),
 	GEM_STAT_TITLE_BITS(RXUDPCCNT, "rx_udp_checksum_errors",
 			    GEM_BIT(NDS_RXERR)),
+	GEM_STAT_TITLE(RXLPI, "rx_lpi_transitions"),
+	GEM_STAT_TITLE(RXLPITIME, "rx_lpi_time"),
+	GEM_STAT_TITLE(TXLPI, "tx_lpi_transitions"),
+	GEM_STAT_TITLE(TXLPITIME, "tx_lpi_time"),
 };
 
 #define GEM_STATS_LEN ARRAY_SIZE(gem_statistics)
@@ -1357,6 +1372,11 @@ struct macb {
 
 	struct work_struct	hresp_err_bh_work;
 
+	/* EEE / LPI state */
+	bool			eee_active;
+	struct delayed_work	tx_lpi_work;
+	u32			tx_lpi_timer;
+
 	int	rx_bd_rd_prefetch;
 	int	tx_bd_rd_prefetch;
 
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index f290d608b4094..1cb49252abf5a 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -10,6 +10,7 @@
 #include <linux/clk-provider.h>
 #include <linux/clk.h>
 #include <linux/crc32.h>
+#include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/etherdevice.h>
 #include <linux/firmware/xlnx-zynqmp.h>
@@ -49,6 +50,7 @@ struct sifive_fu540_macb_mgmt {
 
 #define MACB_RX_BUFFER_SIZE	128
 #define RX_BUFFER_MULTIPLE	64  /* bytes */
+#define RX_BUFFER_MAX		(0xFF * RX_BUFFER_MULTIPLE) /* 16320 bytes */
 
 #define DEFAULT_RX_RING_SIZE	512 /* must be power of 2 */
 #define MIN_RX_RING_SIZE	64
@@ -558,10 +560,21 @@ static int macb_usx_pcs_config(struct phylink_pcs *pcs,
 	return 0;
 }
 
+static unsigned int macb_pcs_inband_caps(struct phylink_pcs *pcs,
+					 phy_interface_t interface)
+{
+	return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE;
+}
+
 static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
 			       struct phylink_link_state *state)
 {
-	state->link = 0;
+	struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs);
+	u16 bmsr, lpa;
+
+	bmsr = gem_readl(bp, PCSSTS);
+	lpa = gem_readl(bp, PCSANLPBASE);
+	phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa);
 }
 
 static void macb_pcs_an_restart(struct phylink_pcs *pcs)
@@ -575,6 +588,26 @@ static int macb_pcs_config(struct phylink_pcs *pcs,
 			   const unsigned long *advertising,
 			   bool permit_pause_to_mac)
 {
+	struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs);
+	u32 old, new;
+
+	old = gem_readl(bp, PCSANADV);
+	new = phylink_mii_c22_pcs_encode_advertisement(interface, advertising);
+	if (new != -EINVAL && old != new)
+		gem_writel(bp, PCSANADV, new);
+
+	/* Disable AN if it's not to be used, enable otherwise.
+	 * Must be written after PCSSEL is set in NCFGR which is done in
+	 * macb_mac_config(), otherwise writes will not take effect.
+	 */
+	old = gem_readl(bp, PCSCNTRL);
+	if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED)
+		new = old | BMCR_ANENABLE;
+	else
+		new = old & ~BMCR_ANENABLE;
+	if (old != new)
+		gem_writel(bp, PCSCNTRL, new);
+
 	return 0;
 }
 
@@ -585,11 +618,113 @@ static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {
 };
 
 static const struct phylink_pcs_ops macb_phylink_pcs_ops = {
+	.pcs_inband_caps = macb_pcs_inband_caps,
 	.pcs_get_state = macb_pcs_get_state,
 	.pcs_an_restart = macb_pcs_an_restart,
 	.pcs_config = macb_pcs_config,
 };
 
+static bool macb_tx_lpi_set(struct macb *bp, bool enable)
+{
+	u32 old, ncr;
+
+	lockdep_assert_held(&bp->lock);
+
+	ncr = macb_readl(bp, NCR);
+	old = ncr;
+	if (enable)
+		ncr |= GEM_BIT(TXLPIEN);
+	else
+		ncr &= ~GEM_BIT(TXLPIEN);
+	if (old != ncr)
+		macb_writel(bp, NCR, ncr);
+
+	return old != ncr;
+}
+
+static bool macb_tx_all_queues_idle(struct macb *bp)
+{
+	struct macb_queue *queue;
+	unsigned int q;
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		if (READ_ONCE(queue->tx_head) != READ_ONCE(queue->tx_tail))
+			return false;
+	}
+	return true;
+}
+
+static void macb_tx_lpi_work_fn(struct work_struct *work)
+{
+	struct macb *bp = container_of(work, struct macb, tx_lpi_work.work);
+	unsigned long flags;
+
+	spin_lock_irqsave(&bp->lock, flags);
+	if (bp->eee_active && macb_tx_all_queues_idle(bp))
+		macb_tx_lpi_set(bp, true);
+	spin_unlock_irqrestore(&bp->lock, flags);
+}
+
+static void macb_tx_lpi_schedule(struct macb *bp)
+{
+	if (bp->eee_active)
+		mod_delayed_work(system_wq, &bp->tx_lpi_work,
+				 usecs_to_jiffies(bp->tx_lpi_timer));
+}
+
+/* Wake from LPI before transmitting. The MAC must deassert TXLPIEN
+ * and wait for the PHY to exit LPI before any frame can be sent.
+ * IEEE 802.3az Tw_sys is ~17us for 1000BASE-T, ~30us for 100BASE-TX;
+ * we use a conservative 50us.
+ */
+static void macb_tx_lpi_wake(struct macb *bp)
+{
+	lockdep_assert_held(&bp->lock);
+
+	if (!bp->eee_active)
+		return;
+
+	if (!macb_tx_lpi_set(bp, false))
+		return;
+
+	cancel_delayed_work(&bp->tx_lpi_work);
+	udelay(50);
+}
+
+static void macb_mac_disable_tx_lpi(struct phylink_config *config)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct macb *bp = netdev_priv(ndev);
+	unsigned long flags;
+
+	cancel_delayed_work_sync(&bp->tx_lpi_work);
+
+	spin_lock_irqsave(&bp->lock, flags);
+	bp->eee_active = false;
+	macb_tx_lpi_set(bp, false);
+	spin_unlock_irqrestore(&bp->lock, flags);
+}
+
+static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer,
+				  bool tx_clk_stop)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct macb *bp = netdev_priv(ndev);
+	unsigned long flags;
+
+	spin_lock_irqsave(&bp->lock, flags);
+	bp->tx_lpi_timer = timer;
+	bp->eee_active = true;
+	spin_unlock_irqrestore(&bp->lock, flags);
+
+	/* Defer initial LPI entry by 1 second after link-up per
+	 * IEEE 802.3az section 22.7a.
+	 */
+	mod_delayed_work(system_wq, &bp->tx_lpi_work, msecs_to_jiffies(1000));
+
+	return 0;
+}
+
 static void macb_mac_config(struct phylink_config *config, unsigned int mode,
 			    const struct phylink_link_state *state)
 {
@@ -629,22 +764,6 @@ static void macb_mac_config(struct phylink_config *config, unsigned int mode,
 	if (old_ncr ^ ncr)
 		macb_or_gem_writel(bp, NCR, ncr);
 
-	/* Disable AN for SGMII fixed link configuration, enable otherwise.
-	 * Must be written after PCSSEL is set in NCFGR,
-	 * otherwise writes will not take effect.
-	 */
-	if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {
-		u32 pcsctrl, old_pcsctrl;
-
-		old_pcsctrl = gem_readl(bp, PCSCNTRL);
-		if (mode == MLO_AN_FIXED)
-			pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG);
-		else
-			pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG);
-		if (old_pcsctrl != pcsctrl)
-			gem_writel(bp, PCSCNTRL, pcsctrl);
-	}
-
 	spin_unlock_irqrestore(&bp->lock, flags);
 }
 
@@ -845,6 +964,8 @@ static const struct phylink_mac_ops macb_phylink_ops = {
 	.mac_config = macb_mac_config,
 	.mac_link_down = macb_mac_link_down,
 	.mac_link_up = macb_mac_link_up,
+	.mac_disable_tx_lpi = macb_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = macb_mac_enable_tx_lpi,
 };
 
 static bool macb_phy_handle_exists(struct device_node *dn)
@@ -909,6 +1030,12 @@ static int macb_mii_probe(struct net_device *dev)
 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
 		bp->phylink_config.poll_fixed_state = true;
 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;
+		/* The PCSAUTONEG bit in PCSCNTRL is on out of reset. Setting
+		 * default_an_inband to true tells phylink to turn it off only
+		 * if necessary (e.g. a fixed link or a PHY that doesn't support
+		 * inband).
+		 */
+		bp->phylink_config.default_an_inband = true;
 	}
 
 	bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |
@@ -940,6 +1067,18 @@ static int macb_mii_probe(struct net_device *dev)
 		}
 	}
 
+	/* Configure EEE LPI if supported */
+	if (bp->caps & MACB_CAPS_EEE) {
+		__set_bit(PHY_INTERFACE_MODE_MII,
+			  bp->phylink_config.lpi_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_GMII,
+			  bp->phylink_config.lpi_interfaces);
+		phy_interface_set_rgmii(bp->phylink_config.lpi_interfaces);
+		bp->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD;
+		bp->phylink_config.lpi_timer_default = 250000;
+		bp->phylink_config.eee_enabled_default = true;
+	}
+
 	bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,
 				     bp->phy_interface, &macb_phylink_ops);
 	if (IS_ERR(bp->phylink)) {
@@ -1336,6 +1475,9 @@ static int macb_tx_complete(struct macb_queue *queue, int budget)
 		netif_wake_subqueue(bp->dev, queue_index);
 	spin_unlock_irqrestore(&queue->tx_ptr_lock, flags);
 
+	if (packets)
+		macb_tx_lpi_schedule(bp);
+
 	return packets;
 }
 
@@ -2442,6 +2584,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
 			     skb->len);
 
 	spin_lock(&bp->lock);
+	macb_tx_lpi_wake(bp);
 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
 	spin_unlock(&bp->lock);
 
@@ -2459,7 +2602,7 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size)
 	if (!macb_is_gem(bp)) {
 		bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;
 	} else {
-		bp->rx_buffer_size = size;
+		bp->rx_buffer_size = MIN(size, RX_BUFFER_MAX);
 
 		if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {
 			netdev_dbg(bp->dev,
@@ -2669,6 +2812,14 @@ static void macb_init_tieoff(struct macb *bp)
 	desc->ctrl = 0;
 }
 
+static void gem_init_rx_ring(struct macb_queue *queue)
+{
+	queue->rx_tail = 0;
+	queue->rx_prepared_head = 0;
+
+	gem_rx_refill(queue);
+}
+
 static void gem_init_rings(struct macb *bp)
 {
 	struct macb_queue *queue;
@@ -2686,10 +2837,7 @@ static void gem_init_rings(struct macb *bp)
 		queue->tx_head = 0;
 		queue->tx_tail = 0;
 
-		queue->rx_tail = 0;
-		queue->rx_prepared_head = 0;
-
-		gem_rx_refill(queue);
+		gem_init_rx_ring(queue);
 	}
 
 	macb_init_tieoff(bp);
@@ -3102,6 +3250,8 @@ static int macb_close(struct net_device *dev)
 		netdev_tx_reset_queue(netdev_get_tx_queue(dev, q));
 	}
 
+	cancel_delayed_work_sync(&bp->tx_lpi_work);
+
 	phylink_stop(bp->phylink);
 	phylink_disconnect_phy(bp->phylink);
 
@@ -3237,7 +3387,6 @@ static int gem_get_sset_count(struct net_device *dev, int sset)
 
 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
 {
-	char stat_string[ETH_GSTRING_LEN];
 	struct macb *bp = netdev_priv(dev);
 	struct macb_queue *queue;
 	unsigned int i;
@@ -3250,11 +3399,8 @@ static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
 			       ETH_GSTRING_LEN);
 
 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
-			for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
-				snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
-						q, queue_statistics[i].stat_string);
-				memcpy(p, stat_string, ETH_GSTRING_LEN);
-			}
+			for (i = 0; i < QUEUE_STATS_LEN; i++)
+				ethtool_sprintf(&p, "q%u_%s", q, queue_statistics[i].stat_string);
 		}
 		break;
 	}
@@ -3978,6 +4124,9 @@ static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
 	struct macb *bp = netdev_priv(netdev);
 	int ret;
 
+	if (!(netdev->hw_features & NETIF_F_NTUPLE))
+		return -EOPNOTSUPP;
+
 	switch (cmd->cmd) {
 	case ETHTOOL_SRXCLSRLINS:
 		if ((cmd->fs.location >= bp->max_tuples)
@@ -4016,6 +4165,20 @@ static const struct ethtool_ops macb_ethtool_ops = {
 	.set_ringparam		= macb_set_ringparam,
 };
 
+static int macb_get_eee(struct net_device *dev, struct ethtool_keee *eee)
+{
+	struct macb *bp = netdev_priv(dev);
+
+	return phylink_ethtool_get_eee(bp->phylink, eee);
+}
+
+static int macb_set_eee(struct net_device *dev, struct ethtool_keee *eee)
+{
+	struct macb *bp = netdev_priv(dev);
+
+	return phylink_ethtool_set_eee(bp->phylink, eee);
+}
+
 static const struct ethtool_ops gem_ethtool_ops = {
 	.get_regs_len		= macb_get_regs_len,
 	.get_regs		= macb_get_regs,
@@ -4037,6 +4200,9 @@ static const struct ethtool_ops gem_ethtool_ops = {
 	.get_rxnfc			= gem_get_rxnfc,
 	.set_rxnfc			= gem_set_rxnfc,
 	.get_rx_ring_count		= gem_get_rx_ring_count,
+	.nway_reset			= phy_ethtool_nway_reset,
+	.get_eee		= macb_get_eee,
+	.set_eee		= macb_set_eee,
 };
 
 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
@@ -4489,9 +4655,9 @@ static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx
 	clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks);
 }
 
-static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
-			 struct clk **hclk, struct clk **tx_clk,
-			 struct clk **rx_clk, struct clk **tsu_clk)
+static int macb_clk_init_dflt(struct platform_device *pdev, struct clk **pclk,
+			      struct clk **hclk, struct clk **tx_clk,
+			      struct clk **rx_clk, struct clk **tsu_clk)
 {
 	struct macb_platform_data *pdata;
 	int err;
@@ -4574,7 +4740,20 @@ static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
 	return err;
 }
 
-static int macb_init(struct platform_device *pdev)
+static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
+			 struct clk **hclk, struct clk **tx_clk,
+			 struct clk **rx_clk, struct clk **tsu_clk,
+			 const struct macb_config *config)
+{
+	if (config->clk_init)
+		return config->clk_init(pdev, pclk, hclk, tx_clk, rx_clk,
+					tsu_clk);
+	else
+		return macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk,
+					  tsu_clk);
+}
+
+static int macb_init_dflt(struct platform_device *pdev)
 {
 	struct net_device *dev = platform_get_drvdata(pdev);
 	unsigned int hw_q, q;
@@ -4729,6 +4908,15 @@ static int macb_init(struct platform_device *pdev)
 	return 0;
 }
 
+static int macb_init(struct platform_device *pdev,
+		     const struct macb_config *config)
+{
+	if (config->init)
+		return config->init(pdev);
+	else
+		return macb_init_dflt(pdev);
+}
+
 static const struct macb_usrio_config macb_default_usrio = {
 	.mii = MACB_BIT(MII),
 	.rmii = MACB_BIT(RMII),
@@ -5182,7 +5370,7 @@ static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
 	struct clk_init_data init;
 	int err = 0;
 
-	err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
+	err = macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
 	if (err)
 		return err;
 
@@ -5229,7 +5417,7 @@ static int fu540_c000_init(struct platform_device *pdev)
 	if (IS_ERR(mgmt->reg))
 		return PTR_ERR(mgmt->reg);
 
-	return macb_init(pdev);
+	return macb_init_dflt(pdev);
 }
 
 static int init_reset_optional(struct platform_device *pdev)
@@ -5279,7 +5467,7 @@ static int init_reset_optional(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, ret, "failed to reset controller");
 	}
 
-	ret = macb_init(pdev);
+	ret = macb_init_dflt(pdev);
 
 err_out_phy_exit:
 	if (ret)
@@ -5304,7 +5492,7 @@ static int eyeq5_init(struct platform_device *pdev)
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to init PHY\n");
 
-	ret = macb_init(pdev);
+	ret = macb_init_dflt(pdev);
 	if (ret)
 		phy_exit(bp->phy);
 	return ret;
@@ -5325,79 +5513,53 @@ static const struct macb_config fu540_c000_config = {
 	.clk_init = fu540_c000_clk_init,
 	.init = fu540_c000_init,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config at91sam9260_config = {
 	.caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config sama5d3macb_config = {
 	.caps = MACB_CAPS_SG_DISABLED |
 		MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config pc302gem_config = {
 	.caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config sama5d2_config = {
 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config sama5d29_config = {
 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config sama5d3_config = {
 	.caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE |
 		MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config sama5d4_config = {
 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
 	.dma_burst_length = 4,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config emac_config = {
 	.caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC,
 	.clk_init = at91ether_clk_init,
 	.init = at91ether_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config np4_config = {
 	.caps = MACB_CAPS_USRIO_DISABLED,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config zynqmp_config = {
@@ -5405,19 +5567,14 @@ static const struct macb_config zynqmp_config = {
 		MACB_CAPS_JUMBO |
 		MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
 	.init = init_reset_optional,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config zynq_config = {
 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |
 		MACB_CAPS_NEEDS_RSTONUBR,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config mpfs_config = {
@@ -5425,9 +5582,7 @@ static const struct macb_config mpfs_config = {
 		MACB_CAPS_JUMBO |
 		MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
 	.init = init_reset_optional,
-	.usrio = &macb_default_usrio,
 	.max_tx_length = 4040, /* Cadence Erratum 1686 */
 	.jumbo_max_len = 4040,
 };
@@ -5437,8 +5592,6 @@ static const struct macb_config sama7g5_gem_config = {
 		MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |
 		MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
 	.usrio = &sama7g5_usrio,
 };
 
@@ -5447,8 +5600,6 @@ static const struct macb_config sama7g5_emac_config = {
 		MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII |
 		MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
 	.usrio = &sama7g5_usrio,
 };
 
@@ -5458,34 +5609,36 @@ static const struct macb_config versal_config = {
 		MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE |
 		MACB_CAPS_QBV,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
 	.init = init_reset_optional,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config eyeq5_config = {
 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
 		MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE |
-		MACB_CAPS_NO_LSO,
+		MACB_CAPS_NO_LSO | MACB_CAPS_EEE,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
 	.init = eyeq5_init,
 	.jumbo_max_len = 10240,
-	.usrio = &macb_default_usrio,
 };
 
 static const struct macb_config raspberrypi_rp1_config = {
 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |
 		MACB_CAPS_JUMBO |
-		MACB_CAPS_GEM_HAS_PTP,
+		MACB_CAPS_GEM_HAS_PTP |
+		MACB_CAPS_EEE,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 	.jumbo_max_len = 10240,
 };
 
+static const struct macb_config pic64hpsc_config = {
+	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
+		MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_DISABLED,
+	.dma_burst_length = 16,
+	.init = init_reset_optional,
+	.jumbo_max_len = 16383,
+};
+
 static const struct of_device_id macb_dt_ids[] = {
 	{ .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
 	{ .compatible = "cdns,macb" },
@@ -5504,6 +5657,7 @@ static const struct of_device_id macb_dt_ids[] = {
 	{ .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
 	{ .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
 	{ .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
+	{ .compatible = "microchip,pic64hpsc-gem", .data = &pic64hpsc_config},
 	{ .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
 	{ .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
 	{ .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config },
@@ -5521,9 +5675,6 @@ static const struct macb_config default_gem_config = {
 		MACB_CAPS_JUMBO |
 		MACB_CAPS_GEM_HAS_PTP,
 	.dma_burst_length = 16,
-	.clk_init = macb_clk_init,
-	.init = macb_init,
-	.usrio = &macb_default_usrio,
 	.jumbo_max_len = 10240,
 };
 
@@ -5551,7 +5702,8 @@ static int macb_probe(struct platform_device *pdev)
 	if (!macb_config)
 		macb_config = &default_gem_config;
 
-	err = macb_config->clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);
+	err = macb_clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk,
+			    macb_config);
 	if (err)
 		return err;
 
@@ -5609,7 +5761,7 @@ static int macb_probe(struct platform_device *pdev)
 	bp->wol = 0;
 	device_set_wakeup_capable(&pdev->dev, 1);
 
-	bp->usrio = macb_config->usrio;
+	bp->usrio = macb_config->usrio ? : &macb_default_usrio;
 
 	/* By default we set to partial store and forward mode for zynqmp.
 	 * Disable if not set in devicetree.
@@ -5657,7 +5809,8 @@ static int macb_probe(struct platform_device *pdev)
 	/* MTU range: 68 - 1518 or 10240 */
 	dev->min_mtu = GEM_MTU_MIN_SIZE;
 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
-		dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;
+		dev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) -
+				ETH_HLEN - ETH_FCS_LEN;
 	else
 		dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN;
 
@@ -5691,7 +5844,7 @@ static int macb_probe(struct platform_device *pdev)
 		bp->phy_interface = interface;
 
 	/* IP specific init */
-	err = macb_config->init(pdev);
+	err = macb_init(pdev, macb_config);
 	if (err)
 		goto err_out_free_netdev;
 
@@ -5708,6 +5861,7 @@ static int macb_probe(struct platform_device *pdev)
 	}
 
 	INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task);
+	INIT_DELAYED_WORK(&bp->tx_lpi_work, macb_tx_lpi_work_fn);
 
 	netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
 		    macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),
@@ -5751,6 +5905,7 @@ static void macb_remove(struct platform_device *pdev)
 		mdiobus_free(bp->mii_bus);
 
 		device_set_wakeup_enable(&bp->pdev->dev, 0);
+		cancel_delayed_work_sync(&bp->tx_lpi_work);
 		cancel_work_sync(&bp->hresp_err_bh_work);
 		pm_runtime_disable(&pdev->dev);
 		pm_runtime_dont_use_autosuspend(&pdev->dev);
@@ -5947,8 +6102,18 @@ static int __maybe_unused macb_resume(struct device *dev)
 		rtnl_unlock();
 	}
 
+	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
+		macb_init_buffers(bp);
+
 	for (q = 0, queue = bp->queues; q < bp->num_queues;
 	     ++q, ++queue) {
+		if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {
+			if (macb_is_gem(bp))
+				gem_init_rx_ring(queue);
+			else
+				macb_init_rx_ring(queue);
+		}
+
 		napi_enable(&queue->napi_rx);
 		napi_enable(&queue->napi_tx);
 	}
diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c
index c9e77819196e1..d91f7b1aa39ca 100644
--- a/drivers/net/ethernet/cadence/macb_ptp.c
+++ b/drivers/net/ethernet/cadence/macb_ptp.c
@@ -357,8 +357,10 @@ void gem_ptp_remove(struct net_device *ndev)
 {
 	struct macb *bp = netdev_priv(ndev);
 
-	if (bp->ptp_clock)
+	if (bp->ptp_clock) {
 		ptp_clock_unregister(bp->ptp_clock);
+		bp->ptp_clock = NULL;
+	}
 
 	gem_ptp_clear_timer(bp);
 
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
index 21e0dfeff1585..1de5744a49b07 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
+++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
@@ -567,7 +567,7 @@ void chtls_shutdown(struct sock *sk, int how);
 void chtls_destroy_sock(struct sock *sk);
 int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
 int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
-		  size_t len, int flags, int *addr_len);
+		  size_t len, int flags);
 void chtls_splice_eof(struct socket *sock);
 int send_tx_flowc_wr(struct sock *sk, int compl,
 		     u32 snd_nxt, u32 rcv_nxt);
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
index ee19933e2cca5..c8e99409a52ad 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
@@ -1332,7 +1332,7 @@ static void chtls_cleanup_rbuf(struct sock *sk, int copied)
 }
 
 static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-			    int flags, int *addr_len)
+			    int flags)
 {
 	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 	struct chtls_hws *hws = &csk->tlshws;
@@ -1656,7 +1656,7 @@ static int peekmsg(struct sock *sk, struct msghdr *msg,
 }
 
 int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-		  int flags, int *addr_len)
+		  int flags)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct chtls_sock *csk;
@@ -1670,7 +1670,7 @@ int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 	buffers_freed = 0;
 
 	if (unlikely(flags & MSG_OOB))
-		return tcp_prot.recvmsg(sk, msg, len, flags, addr_len);
+		return tcp_prot.recvmsg(sk, msg, len, flags);
 
 	if (unlikely(flags & MSG_PEEK))
 		return peekmsg(sk, msg, len, flags);
@@ -1684,7 +1684,7 @@ int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
 	csk = rcu_dereference_sk_user_data(sk);
 
 	if (is_tls_rx(csk))
-		return chtls_pt_recvmsg(sk, msg, len, flags, addr_len);
+		return chtls_pt_recvmsg(sk, msg, len, flags);
 
 	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h
index 301b3f3114afa..6959e85ab516c 100644
--- a/drivers/net/ethernet/cisco/enic/enic.h
+++ b/drivers/net/ethernet/cisco/enic/enic.h
@@ -22,6 +22,45 @@
 #define DRV_NAME		"enic"
 #define DRV_DESCRIPTION		"Cisco VIC Ethernet NIC Driver"
 
+#define PCI_SUBDEV_ID_CISCO_VIC_1225	0x085
+#define PCI_SUBDEV_ID_CISCO_VIC_1225T	0x0CE
+#define PCI_SUBDEV_ID_CISCO_VIC_1227	0x12E
+#define PCI_SUBDEV_ID_CISCO_VIC_1227T	0x139
+#define PCI_SUBDEV_ID_CISCO_VIC_1240	0x084
+#define PCI_SUBDEV_ID_CISCO_VIC_1280	0x04F
+#define PCI_SUBDEV_ID_CISCO_VIC_1285	0x0CD
+
+#define PCI_SUBDEV_ID_CISCO_VIC_1340	0x12C
+#define PCI_SUBDEV_ID_CISCO_VIC_1380	0x137
+#define PCI_SUBDEV_ID_CISCO_VIC_1385	0x14D
+#define PCI_SUBDEV_ID_CISCO_VIC_1387	0x15D
+
+#define PCI_SUBDEV_ID_CISCO_VIC_1440	0x0215
+#define PCI_SUBDEV_ID_CISCO_VIC_1455	0x0217
+#define PCI_SUBDEV_ID_CISCO_VIC_1457	0x0218
+#define PCI_SUBDEV_ID_CISCO_VIC_1467	0x02AF
+#define PCI_SUBDEV_ID_CISCO_VIC_1477	0x2B0
+#define PCI_SUBDEV_ID_CISCO_VIC_1480	0x0216
+#define PCI_SUBDEV_ID_CISCO_VIC_1485	0x0219
+#define PCI_SUBDEV_ID_CISCO_VIC_1487	0x021A
+#define PCI_SUBDEV_ID_CISCO_VIC_1495	0x024A
+#define PCI_SUBDEV_ID_CISCO_VIC_1497	0x024B
+#define PCI_SUBDEV_ID_CISCO_VIC_14425	0x02CF
+#define PCI_SUBDEV_ID_CISCO_VIC_14825	0x02D0
+
+#define PCI_SUBDEV_ID_CISCO_VIC_15230	0x02DF
+#define PCI_SUBDEV_ID_CISCO_VIC_15231	0x02DB
+#define PCI_SUBDEV_ID_CISCO_VIC_15235	0x02E4
+#define PCI_SUBDEV_ID_CISCO_VIC_15237	0x02F3
+#define PCI_SUBDEV_ID_CISCO_VIC_15238	0x02E8
+#define PCI_SUBDEV_ID_CISCO_VIC_15411	0x02DC
+#define PCI_SUBDEV_ID_CISCO_VIC_15412	0x02E2
+#define PCI_SUBDEV_ID_CISCO_VIC_15420	0x02DE
+#define PCI_SUBDEV_ID_CISCO_VIC_15422	0x02E1
+#define PCI_SUBDEV_ID_CISCO_VIC_15425	0x02F2
+#define PCI_SUBDEV_ID_CISCO_VIC_15427	0x02E0
+#define PCI_SUBDEV_ID_CISCO_VIC_15428	0x02DD
+
 #define ENIC_BARS_MAX		6
 
 #define ENIC_WQ_MAX		256
diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
index 471613899ec0e..61541fb8db7b7 100644
--- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c
+++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
@@ -126,19 +126,160 @@ static void enic_intr_coal_set_rx(struct enic *enic, u32 timer)
 	}
 }
 
+static void enic_get_supp_adv_media_type(struct net_device *netdev,
+					 struct ethtool_link_ksettings *ecmd)
+{
+	struct enic *enic = netdev_priv(netdev);
+	struct ethtool_link_settings *base = &ecmd->base;
+	u16 sub_dev_id = 0;
+
+	base->port = PORT_OTHER;
+
+	if (enic->pdev)
+		sub_dev_id = enic->pdev->subsystem_device;
+
+	switch (sub_dev_id) {
+	case PCI_SUBDEV_ID_CISCO_VIC_1225:
+	case PCI_SUBDEV_ID_CISCO_VIC_1227:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseSR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1285:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1225T:
+	case PCI_SUBDEV_ID_CISCO_VIC_1227T:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseT_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1385:
+	case PCI_SUBDEV_ID_CISCO_VIC_1387:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseSR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseLR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseLR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1477:
+	case PCI_SUBDEV_ID_CISCO_VIC_1485:
+	case PCI_SUBDEV_ID_CISCO_VIC_1487:
+	case PCI_SUBDEV_ID_CISCO_VIC_1495:
+	case PCI_SUBDEV_ID_CISCO_VIC_1497:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseLR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     100000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     100000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_15235:
+	case PCI_SUBDEV_ID_CISCO_VIC_15237:
+	case PCI_SUBDEV_ID_CISCO_VIC_15238:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseLR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     100000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     100000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     200000baseSR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     200000baseDR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     200000baseLR4_ER4_FR4_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1455:
+	case PCI_SUBDEV_ID_CISCO_VIC_1457:
+	case PCI_SUBDEV_ID_CISCO_VIC_1467:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseT_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     25000baseSR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_15428:
+	case PCI_SUBDEV_ID_CISCO_VIC_15427:
+	case PCI_SUBDEV_ID_CISCO_VIC_15425:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseT_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     25000baseSR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     50000baseSR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
+		base->port = PORT_FIBRE;
+		break;
+	/* Do not mention port type as FIBRE for blade VICs */
+	case PCI_SUBDEV_ID_CISCO_VIC_1240:
+	case PCI_SUBDEV_ID_CISCO_VIC_1280:
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseKR_Full);
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_1340:
+	case PCI_SUBDEV_ID_CISCO_VIC_1380:
+	case PCI_SUBDEV_ID_CISCO_VIC_1440: /* 10G/40G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_1480: /* 10G/40G KR */
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseKR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     40000baseKR4_Full);
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_14425: /* 25G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_14825: /* 25G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_15420: /* 25G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_15422: /* 25G KR */
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     25000baseKR_Full);
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_15411: /* 10G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_15412: /* 10G KR */
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     10000baseKR_Full);
+		break;
+	case PCI_SUBDEV_ID_CISCO_VIC_15231: /* 25G/100G/200G KR */
+	case PCI_SUBDEV_ID_CISCO_VIC_15230: /* 25G/100G/200G KR */
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     25000baseKR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     100000baseKR_Full);
+		ethtool_link_ksettings_add_link_mode(ecmd, supported,
+						     200000baseKR4_Full);
+		break;
+	}
+}
+
 static int enic_get_ksettings(struct net_device *netdev,
 			      struct ethtool_link_ksettings *ecmd)
 {
 	struct enic *enic = netdev_priv(netdev);
 	struct ethtool_link_settings *base = &ecmd->base;
 
-	ethtool_link_ksettings_add_link_mode(ecmd, supported,
-					     10000baseT_Full);
-	ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
-	ethtool_link_ksettings_add_link_mode(ecmd, advertising,
-					     10000baseT_Full);
-	ethtool_link_ksettings_add_link_mode(ecmd, advertising, FIBRE);
-	base->port = PORT_FIBRE;
+	enic_get_supp_adv_media_type(netdev, ecmd);
 
 	if (netif_carrier_ok(netdev)) {
 		base->speed = vnic_dev_port_speed(enic->vdev);
diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile
index de7b318422330..d0a259e47960f 100644
--- a/drivers/net/ethernet/freescale/Makefile
+++ b/drivers/net/ethernet/freescale/Makefile
@@ -22,6 +22,5 @@ ucc_geth_driver-objs := ucc_geth.o ucc_geth_ethtool.o
 obj-$(CONFIG_FSL_FMAN) += fman/
 obj-$(CONFIG_FSL_DPAA_ETH) += dpaa/
 
-obj-$(CONFIG_FSL_DPAA2_ETH) += dpaa2/
-
+obj-y += dpaa2/
 obj-y += enetc/
diff --git a/drivers/net/ethernet/freescale/dpaa2/Kconfig b/drivers/net/ethernet/freescale/dpaa2/Kconfig
index d029b69c3f183..36280e5d99e1f 100644
--- a/drivers/net/ethernet/freescale/dpaa2/Kconfig
+++ b/drivers/net/ethernet/freescale/dpaa2/Kconfig
@@ -34,6 +34,10 @@ config FSL_DPAA2_SWITCH
 	tristate "Freescale DPAA2 Ethernet Switch"
 	depends on BRIDGE || BRIDGE=n
 	depends on NET_SWITCHDEV
+	depends on FSL_MC_BUS && FSL_MC_DPIO
+	select PHYLINK
+	select PCS_LYNX
+	select FSL_XGMAC_MDIO
 	help
 	  Driver for Freescale DPAA2 Ethernet Switch. This driver manages
 	  switch objects discovered on the Freeescale MC bus.
diff --git a/drivers/net/ethernet/freescale/enetc/ntmp.c b/drivers/net/ethernet/freescale/enetc/ntmp.c
index 0c1d343253bfb..703752995e931 100644
--- a/drivers/net/ethernet/freescale/enetc/ntmp.c
+++ b/drivers/net/ethernet/freescale/enetc/ntmp.c
@@ -227,7 +227,7 @@ static const char *ntmp_table_name(int tbl_id)
 		return "RSS Table";
 	default:
 		return "Unknown Table";
-	};
+	}
 }
 
 static int ntmp_delete_entry_by_id(struct ntmp_user *user, int tbl_id,
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index 2cbcc29408e3c..d3c772ed5fc9b 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -951,7 +951,9 @@ static int fs_enet_probe(struct platform_device *ofdev)
 	spin_lock_init(&fep->lock);
 	spin_lock_init(&fep->tx_lock);
 
-	of_get_ethdev_address(ofdev->dev.of_node, ndev);
+	ret = of_get_ethdev_address(ofdev->dev.of_node, ndev);
+	if (ret == -EPROBE_DEFER)
+		goto out_cleanup_data;
 
 	ret = fep->ops->allocate_bd(ndev);
 	if (ret)
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index 3607681650ce8..7af4b5e3f38ea 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -3470,14 +3470,13 @@ static int ucc_geth_probe(struct platform_device* ofdev)
 	phy_node = of_parse_phandle(np, "phy-handle", 0);
 	if (phy_node) {
 		prop = of_get_property(phy_node, "interface", NULL);
+		of_node_put(phy_node);
 		if (prop) {
 			dev_err(&ofdev->dev,
 				"Device-tree property 'interface' is no longer supported. Please use 'phy-connection-type' instead.");
-			of_node_put(phy_node);
 			err = -EINVAL;
 			goto err_put_tbi;
 		}
-		of_node_put(phy_node);
 	}
 
 	err = of_get_phy_mode(np, &phy_interface);
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index cbdf3a842cfe0..1d66d3834f7e6 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -79,8 +79,6 @@
 
 #define GVE_DEFAULT_HEADER_BUFFER_SIZE 128
 
-#define DQO_QPL_DEFAULT_TX_PAGES 512
-
 /* Maximum TSO size supported on DQO */
 #define GVE_DQO_TX_MAX	0x3FFFF
 
@@ -711,6 +709,7 @@ struct gve_ptype_lut {
 /* Parameters for allocating resources for tx queues */
 struct gve_tx_alloc_rings_cfg {
 	struct gve_tx_queue_config *qcfg;
+	u16 pages_per_qpl;
 
 	u16 num_xdp_rings;
 
@@ -726,6 +725,7 @@ struct gve_rx_alloc_rings_cfg {
 	/* tx config is also needed to determine QPL ids */
 	struct gve_rx_queue_config *qcfg_rx;
 	struct gve_tx_queue_config *qcfg_tx;
+	u16 pages_per_qpl;
 
 	u16 ring_size;
 	u16 packet_buffer_size;
@@ -816,7 +816,8 @@ struct gve_priv {
 	u16 min_rx_desc_cnt;
 	bool modify_ring_size_enabled;
 	bool default_min_ring_size;
-	u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */
+	u16 tx_pages_per_qpl;
+	u16 rx_pages_per_qpl;
 	u64 max_registered_pages;
 	u64 num_registered_pages; /* num pages registered with NIC */
 	struct bpf_prog *xdp_prog; /* XDP BPF program */
@@ -1150,14 +1151,6 @@ static inline u32 gve_rx_start_qpl_id(const struct gve_tx_queue_config *tx_cfg)
 	return gve_get_rx_qpl_id(tx_cfg, 0);
 }
 
-static inline u32 gve_get_rx_pages_per_qpl_dqo(u32 rx_desc_cnt)
-{
-	/* For DQO, page count should be more than ring size for
-	 * out-of-order completions. Set it to two times of ring size.
-	 */
-	return 2 * rx_desc_cnt;
-}
-
 /* Returns the correct dma direction for tx and rx qpls */
 static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
 						      int id)
@@ -1308,6 +1301,9 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown);
 void gve_get_curr_alloc_cfgs(struct gve_priv *priv,
 			     struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
 			     struct gve_rx_alloc_rings_cfg *rx_alloc_cfg);
+void gve_update_num_qpl_pages(struct gve_priv *priv,
+			      struct gve_rx_alloc_rings_cfg *rx_alloc_cfg,
+			      struct gve_tx_alloc_rings_cfg *tx_alloc_cfg);
 int gve_adjust_config(struct gve_priv *priv,
 		      struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
 		      struct gve_rx_alloc_rings_cfg *rx_alloc_cfg);
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index b72cc0fa2ba2b..08587bf40ed4a 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -791,7 +791,7 @@ static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv,
 		cmd->create_rx_queue.rx_buff_ring_size =
 			cpu_to_be16(priv->rx_desc_cnt);
 		cmd->create_rx_queue.enable_rsc =
-			!!(priv->dev->features & NETIF_F_LRO);
+			!!(priv->dev->features & NETIF_F_GRO_HW);
 		if (priv->header_split_enabled)
 			cmd->create_rx_queue.header_buffer_size =
 				cpu_to_be16(priv->header_buf_size);
@@ -970,14 +970,6 @@ static void gve_enable_supported_features(struct gve_priv *priv,
 		priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
 	}
 
-	/* Override pages for qpl for DQO-QPL */
-	if (dev_op_dqo_qpl) {
-		priv->tx_pages_per_qpl =
-			be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
-		if (priv->tx_pages_per_qpl == 0)
-			priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
-	}
-
 	if (dev_op_buffer_sizes &&
 	    (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
 		priv->max_rx_buffer_size =
@@ -997,12 +989,10 @@ static void gve_enable_supported_features(struct gve_priv *priv,
 	if (dev_op_modify_ring &&
 	    (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
 		priv->modify_ring_size_enabled = true;
-
-		/* max ring size for DQO QPL should not be overwritten because of device limit */
-		if (priv->queue_format != GVE_DQO_QPL_FORMAT) {
-			priv->max_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
-			priv->max_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
-		}
+		priv->max_rx_desc_cnt =
+			be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
+		priv->max_tx_desc_cnt =
+			be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
 		if (priv->default_min_ring_size) {
 			/* If device hasn't provided minimums, use default minimums */
 			priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
@@ -1127,9 +1117,13 @@ int gve_adminq_describe_device(struct gve_priv *priv)
 
 	gve_set_default_rss_sizes(priv);
 
-	/* DQO supports LRO. */
-	if (!gve_is_gqi(priv))
-		priv->dev->hw_features |= NETIF_F_LRO;
+	/* DQO supports HW-GRO and UDP_GSO */
+	if (gve_is_dqo(priv)) {
+		u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4;
+
+		priv->dev->hw_features |= additional_features;
+		priv->dev->features |= additional_features;
+	}
 
 	priv->max_registered_pages =
 				be64_to_cpu(descriptor->max_registered_pages);
diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
index 0e2b703c673ac..6880d1531ba5e 100644
--- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
@@ -133,7 +133,7 @@ int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx,
 	u32 idx;
 
 	idx = rx->dqo.next_qpl_page_idx;
-	if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) {
+	if (idx >= priv->rx_pages_per_qpl) {
 		net_err_ratelimited("%s: Out of QPL pages\n",
 				    priv->dev->name);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 9eb4b3614c4f5..424d973c97f24 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -11,6 +11,7 @@
 #include <linux/filter.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/sched.h>
@@ -966,6 +967,7 @@ static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv,
 	cfg->qcfg = &priv->tx_cfg;
 	cfg->raw_addressing = !gve_is_qpl(priv);
 	cfg->ring_size = priv->tx_desc_cnt;
+	cfg->pages_per_qpl = priv->tx_pages_per_qpl;
 	cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues;
 	cfg->tx = priv->tx;
 }
@@ -997,12 +999,48 @@ static void gve_tx_start_rings(struct gve_priv *priv, int num_rings)
 	}
 }
 
+void gve_update_num_qpl_pages(struct gve_priv *priv,
+			      struct gve_rx_alloc_rings_cfg *rx_alloc_cfg,
+			      struct gve_tx_alloc_rings_cfg *tx_alloc_cfg)
+{
+	u64 ideal_tx_pages, ideal_rx_pages;
+	u16 tx_num_queues, rx_num_queues;
+	u64 max_pages, tx_pages;
+
+	if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+		rx_alloc_cfg->pages_per_qpl = rx_alloc_cfg->ring_size;
+	} else if (priv->queue_format == GVE_DQO_QPL_FORMAT) {
+		/*
+		 * We want 2 pages per RX descriptor and half a page per TX
+		 * descriptor, which means the fraction ideal_tx_pages /
+		 * (ideal_tx_pages + ideal_rx_pages) of the pages we allocate
+		 * should be for TX. Shrink proportionally as necessary to avoid
+		 * allocating more than max_registered_pages total pages.
+		 */
+		tx_num_queues = tx_alloc_cfg->qcfg->num_queues;
+		rx_num_queues = rx_alloc_cfg->qcfg_rx->num_queues;
+
+		ideal_tx_pages = tx_alloc_cfg->ring_size * tx_num_queues / 2;
+		ideal_rx_pages = rx_alloc_cfg->ring_size * rx_num_queues * 2;
+		max_pages = min(priv->max_registered_pages,
+				ideal_tx_pages + ideal_rx_pages);
+
+		tx_pages = div64_u64(max_pages * ideal_tx_pages,
+				     ideal_tx_pages + ideal_rx_pages);
+		tx_alloc_cfg->pages_per_qpl = div_u64(tx_pages, tx_num_queues);
+		rx_alloc_cfg->pages_per_qpl = div_u64(max_pages - tx_pages,
+						      rx_num_queues);
+	}
+}
+
 static int gve_queues_mem_alloc(struct gve_priv *priv,
 				struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
 				struct gve_rx_alloc_rings_cfg *rx_alloc_cfg)
 {
 	int err;
 
+	gve_update_num_qpl_pages(priv, rx_alloc_cfg, tx_alloc_cfg);
+
 	if (gve_is_gqi(priv))
 		err = gve_tx_alloc_rings_gqi(priv, tx_alloc_cfg);
 	else
@@ -1293,6 +1331,7 @@ static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv,
 	cfg->raw_addressing = !gve_is_qpl(priv);
 	cfg->enable_header_split = priv->header_split_enabled;
 	cfg->ring_size = priv->rx_desc_cnt;
+	cfg->pages_per_qpl = priv->rx_pages_per_qpl;
 	cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size;
 	cfg->rx = priv->rx;
 	cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues;
@@ -1372,6 +1411,8 @@ static int gve_queues_start(struct gve_priv *priv,
 	priv->rx_cfg = *rx_alloc_cfg->qcfg_rx;
 	priv->tx_desc_cnt = tx_alloc_cfg->ring_size;
 	priv->rx_desc_cnt = rx_alloc_cfg->ring_size;
+	priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl;
+	priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl;
 
 	gve_tx_start_rings(priv, gve_num_tx_queues(priv));
 	gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues);
@@ -1717,9 +1758,9 @@ static int gve_verify_xdp_configuration(struct net_device *dev,
 	struct gve_priv *priv = netdev_priv(dev);
 	u16 max_xdp_mtu;
 
-	if (dev->features & NETIF_F_LRO) {
+	if (dev->features & NETIF_F_GRO_HW) {
 		NL_SET_ERR_MSG_MOD(extack,
-				   "XDP is not supported when LRO is on.");
+				   "XDP is not supported when HW-GRO is on.");
 		return -EOPNOTSUPP;
 	}
 
@@ -2136,12 +2177,13 @@ static int gve_set_features(struct net_device *netdev,
 
 	gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
 
-	if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) {
-		netdev->features ^= NETIF_F_LRO;
-		if (priv->xdp_prog && (netdev->features & NETIF_F_LRO)) {
+	if ((netdev->features & NETIF_F_GRO_HW) !=
+	    (features & NETIF_F_GRO_HW)) {
+		netdev->features ^= NETIF_F_GRO_HW;
+		if (priv->xdp_prog && (netdev->features & NETIF_F_GRO_HW)) {
 			netdev_warn(netdev,
-				    "XDP is not supported when LRO is on.\n");
-			err =  -EOPNOTSUPP;
+				    "HW-GRO is not supported when XDP is on.");
+			err = -EOPNOTSUPP;
 			goto revert_features;
 		}
 		if (netif_running(netdev)) {
diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
index 4eb6ed29d3cc6..81ea800e66e94 100644
--- a/drivers/net/ethernet/google/gve/gve_rx.c
+++ b/drivers/net/ethernet/google/gve/gve_rx.c
@@ -278,7 +278,6 @@ int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
 	struct device *hdev = &priv->pdev->dev;
 	u32 slots = cfg->ring_size;
 	int filled_pages;
-	int qpl_page_cnt;
 	u32 qpl_id = 0;
 	size_t bytes;
 	int err;
@@ -314,10 +313,8 @@ int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
 
 	if (!rx->data.raw_addressing) {
 		qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
-		qpl_page_cnt = cfg->ring_size;
-
 		rx->data.qpl = gve_alloc_queue_page_list(priv, qpl_id,
-							 qpl_page_cnt);
+							 cfg->pages_per_qpl);
 		if (!rx->data.qpl) {
 			err = -ENOMEM;
 			goto abort_with_copy_pool;
diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index c706c79321594..7924dce719e29 100644
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -218,7 +218,6 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 {
 	struct device *hdev = &priv->pdev->dev;
 	struct page_pool *pool;
-	int qpl_page_cnt;
 	size_t size;
 	u32 qpl_id;
 
@@ -246,7 +245,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 	XSK_CHECK_PRIV_TYPE(struct gve_xdp_buff);
 
 	rx->dqo.num_buf_states = cfg->raw_addressing ? buffer_queue_slots :
-		gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
+		cfg->pages_per_qpl;
 	rx->dqo.buf_states = kvcalloc_node(rx->dqo.num_buf_states,
 					   sizeof(rx->dqo.buf_states[0]),
 					   GFP_KERNEL, priv->numa_node);
@@ -281,10 +280,9 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
 		rx->dqo.page_pool = pool;
 	} else {
 		qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
-		qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
 
 		rx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id,
-							qpl_page_cnt);
+							cfg->pages_per_qpl);
 		if (!rx->dqo.qpl)
 			goto err;
 		rx->dqo.next_qpl_page_idx = 0;
@@ -944,11 +942,18 @@ static int gve_rx_complete_rsc(struct sk_buff *skb,
 			       struct gve_ptype ptype)
 {
 	struct skb_shared_info *shinfo = skb_shinfo(skb);
+	int rsc_segments, rsc_seg_len, hdr_len;
+	skb_frag_t *frag;
+	void *va;
 
-	/* Only TCP is supported right now. */
+	/* HW-GRO only coalesces TCP. */
 	if (ptype.l4_type != GVE_L4_TYPE_TCP)
 		return -EINVAL;
 
+	rsc_seg_len = le16_to_cpu(desc->rsc_seg_len);
+	if (!rsc_seg_len)
+		return 0;
+
 	switch (ptype.l3_type) {
 	case GVE_L3_TYPE_IPV4:
 		shinfo->gso_type = SKB_GSO_TCPV4;
@@ -960,7 +965,31 @@ static int gve_rx_complete_rsc(struct sk_buff *skb,
 		return -EINVAL;
 	}
 
-	shinfo->gso_size = le16_to_cpu(desc->rsc_seg_len);
+	if (skb_headlen(skb)) {
+		/* With header-split, payload is in the non-linear part */
+		rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len);
+	} else {
+		/* HW-GRO packets are guaranteed to have complete TCP/IP
+		 * headers in frag[0] when header-split is not enabled.
+		 */
+		frag = &skb_shinfo(skb)->frags[0];
+		va = skb_frag_address(frag);
+		hdr_len =
+			eth_get_headlen(skb->dev, va, skb_frag_size(frag));
+		rsc_segments = DIV_ROUND_UP(skb->len - hdr_len, rsc_seg_len);
+		skb_copy_to_linear_data(skb, va, hdr_len);
+		skb_frag_size_sub(frag, hdr_len);
+		/* Verify we didn't empty the fragment completely as that could
+		 * otherwise lead to page leaks.
+		 */
+		DEBUG_NET_WARN_ON_ONCE(!skb_frag_size(frag));
+		skb_frag_off_add(frag, hdr_len);
+		skb->data_len -= hdr_len;
+		skb->tail += hdr_len;
+	}
+	shinfo->gso_size = rsc_seg_len;
+	shinfo->gso_segs = rsc_segments;
+
 	return 0;
 }
 
@@ -993,7 +1022,7 @@ static int gve_rx_complete_skb(struct gve_rx_ring *rx, struct napi_struct *napi,
 			return err;
 	}
 
-	if (skb_headlen(rx->ctx.skb_head) == 0)
+	if (rx->ctx.skb_head == napi->skb)
 		napi_gro_frags(napi);
 	else
 		napi_gro_receive(napi, rx->ctx.skb_head);
diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c
index 0b856e269eac5..79338fd9bf66e 100644
--- a/drivers/net/ethernet/google/gve/gve_tx.c
+++ b/drivers/net/ethernet/google/gve/gve_tx.c
@@ -264,7 +264,6 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv,
 				 int idx)
 {
 	struct device *hdev = &priv->pdev->dev;
-	int qpl_page_cnt;
 	u32 qpl_id = 0;
 	size_t bytes;
 
@@ -291,10 +290,8 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv,
 	tx->dev = hdev;
 	if (!tx->raw_addressing) {
 		qpl_id = gve_tx_qpl_id(priv, tx->q_num);
-		qpl_page_cnt = priv->tx_pages_per_qpl;
-
 		tx->tx_fifo.qpl = gve_alloc_queue_page_list(priv, qpl_id,
-							    qpl_page_cnt);
+							    cfg->pages_per_qpl);
 		if (!tx->tx_fifo.qpl)
 			goto abort_with_desc;
 
diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
index 4db9d4f9ed6f9..80ab0a449ff54 100644
--- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
@@ -311,7 +311,6 @@ static int gve_tx_alloc_ring_dqo(struct gve_priv *priv,
 {
 	struct device *hdev = &priv->pdev->dev;
 	int num_pending_packets;
-	int qpl_page_cnt;
 	size_t bytes;
 	u32 qpl_id;
 	int i;
@@ -392,10 +391,9 @@ static int gve_tx_alloc_ring_dqo(struct gve_priv *priv,
 
 	if (!cfg->raw_addressing) {
 		qpl_id = gve_tx_qpl_id(priv, tx->q_num);
-		qpl_page_cnt = priv->tx_pages_per_qpl;
 
 		tx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id,
-							qpl_page_cnt);
+							cfg->pages_per_qpl);
 		if (!tx->dqo.qpl)
 			goto err;
 
@@ -572,9 +570,11 @@ static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx,
  */
 static int gve_prep_tso(struct sk_buff *skb)
 {
+	struct skb_shared_info *shinfo = skb_shinfo(skb);
+	u32 paylen, l4_start;
 	struct tcphdr *tcp;
+	struct udphdr *udp;
 	int header_len;
-	u32 paylen;
 	int err;
 
 	/* Note: HW requires MSS (gso_size) to be <= 9728 and the total length
@@ -585,21 +585,34 @@ static int gve_prep_tso(struct sk_buff *skb)
 	 * - Kernel will not produce a TSO larger than 64k
 	 */
 
-	if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO))
+	if (unlikely(shinfo->gso_size < GVE_TX_MIN_TSO_MSS_DQO))
 		return -1;
 
-	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
-		return -EINVAL;
-
 	/* Needed because we will modify header. */
 	err = skb_cow_head(skb, 0);
 	if (err < 0)
 		return err;
 
-	tcp = tcp_hdr(skb);
-	paylen = skb->len - skb_transport_offset(skb);
-	csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen));
-	header_len = skb_tcp_all_headers(skb);
+	l4_start = skb_transport_offset(skb);
+	paylen = skb->len - l4_start;
+
+	switch (shinfo->gso_type) {
+	case SKB_GSO_TCPV4:
+	case SKB_GSO_TCPV6:
+		tcp = tcp_hdr(skb);
+		csum_replace_by_diff(&tcp->check,
+				     (__force __wsum)htonl(paylen));
+		header_len = skb_tcp_all_headers(skb);
+		break;
+	case SKB_GSO_UDP_L4:
+		udp = udp_hdr(skb);
+		csum_replace_by_diff(&udp->check,
+				     (__force __wsum)htonl(paylen));
+		header_len = sizeof(struct udphdr) + l4_start;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO))
 		return -EINVAL;
diff --git a/drivers/net/ethernet/huawei/hinic3/Makefile b/drivers/net/ethernet/huawei/hinic3/Makefile
index 26c05ecf31c90..4ebad3a4f9439 100644
--- a/drivers/net/ethernet/huawei/hinic3/Makefile
+++ b/drivers/net/ethernet/huawei/hinic3/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_HINIC3) += hinic3.o
 hinic3-objs := hinic3_cmdq.o \
 	       hinic3_common.o \
 	       hinic3_eqs.o \
+	       hinic3_ethtool.o \
 	       hinic3_filter.o \
 	       hinic3_hw_cfg.o \
 	       hinic3_hw_comm.o \
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c
index bc5b80c31693f..1b0ba244a68f3 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c
@@ -6,12 +6,14 @@
 #include <linux/dma-mapping.h>
 
 #include "hinic3_cmdq.h"
+#include "hinic3_eqs.h"
 #include "hinic3_hwdev.h"
 #include "hinic3_hwif.h"
 #include "hinic3_mbox.h"
 
 #define CMDQ_BUF_SIZE             2048
 #define CMDQ_WQEBB_SIZE           64
+#define CMDQ_WQE_HEAD_LEN         32
 
 #define CMDQ_CMD_TIMEOUT          5000
 #define CMDQ_ENABLE_WAIT_TIMEOUT  300
@@ -61,6 +63,10 @@
 #define CMDQ_DB_HEAD_SET(val, member)  \
 	FIELD_PREP(CMDQ_DB_HEAD_##member##_MASK, val)
 
+#define SAVED_DATA_ARM_MASK          BIT(31)
+#define SAVED_DATA_SET(val, member)  \
+	FIELD_PREP(SAVED_DATA_##member##_MASK, val)
+
 #define CMDQ_CEQE_TYPE_MASK            GENMASK(2, 0)
 #define CMDQ_CEQE_GET(val, member)  \
 	FIELD_GET(CMDQ_CEQE_##member##_MASK, le32_to_cpu(val))
@@ -84,6 +90,10 @@ enum cmdq_data_format {
 	CMDQ_DATA_DIRECT = 1,
 };
 
+enum cmdq_scmd_type {
+	CMDQ_SET_ARM_CMD = 2,
+};
+
 enum cmdq_ctrl_sect_len {
 	CMDQ_CTRL_SECT_LEN        = 1,
 	CMDQ_CTRL_DIRECT_SECT_LEN = 2,
@@ -106,6 +116,20 @@ enum cmdq_cmd_type {
 
 #define CMDQ_WQE_NUM_WQEBBS  1
 
+static void hinic3_dump_cmdq_wqe_head(struct hinic3_hwdev *hwdev,
+				      struct cmdq_wqe *wqe)
+{
+	u32 *data = (u32 *)wqe;
+	u32 i;
+
+	for (i = 0; i < (CMDQ_WQE_HEAD_LEN / sizeof(u32)); i += 0x4) {
+		dev_dbg(hwdev->dev,
+			"wqe data: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
+			*(data + i), *(data + i + 0x1), *(data + i + 0x2),
+			*(data + i + 0x3));
+	}
+}
+
 static struct cmdq_wqe *cmdq_read_wqe(struct hinic3_wq *wq, u16 *ci)
 {
 	if (hinic3_wq_get_used(wq) == 0)
@@ -166,6 +190,11 @@ static void cmdq_clear_cmd_buf(struct hinic3_cmdq_cmd_info *cmd_info,
 		hinic3_free_cmd_buf(hwdev, cmd_info->buf_in);
 		cmd_info->buf_in = NULL;
 	}
+
+	if (cmd_info->buf_out) {
+		hinic3_free_cmd_buf(hwdev, cmd_info->buf_out);
+		cmd_info->buf_out = NULL;
+	}
 }
 
 static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq,
@@ -189,6 +218,20 @@ static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq,
 	hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS);
 }
 
+static int cmdq_arm_ceq_handler(struct hinic3_cmdq *cmdq,
+				struct cmdq_wqe *wqe, u16 ci)
+{
+	struct cmdq_ctrl *ctrl = &wqe->wqe_scmd.ctrl;
+	__le32 ctrl_info = ctrl->ctrl_info;
+
+	if (!CMDQ_WQE_COMPLETED(ctrl_info))
+		return -EBUSY;
+
+	clear_wqe_complete_bit(cmdq, wqe, ci);
+
+	return 0;
+}
+
 static void cmdq_update_cmd_status(struct hinic3_cmdq *cmdq, u16 prod_idx,
 				   struct cmdq_wqe *wqe)
 {
@@ -252,11 +295,17 @@ void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data)
 		case HINIC3_CMD_TYPE_TIMEOUT:
 			dev_warn(hwdev->dev, "Cmdq timeout, q_id: %u, ci: %u\n",
 				 cmdq_type, ci);
+			hinic3_dump_cmdq_wqe_head(hwdev, wqe);
 			fallthrough;
 		case HINIC3_CMD_TYPE_FAKE_TIMEOUT:
 			cmdq_clear_cmd_buf(cmd_info, hwdev);
 			clear_wqe_complete_bit(cmdq, wqe, ci);
 			break;
+		case HINIC3_CMD_TYPE_SET_ARM:
+			/* arm_bit was set until here */
+			if (cmdq_arm_ceq_handler(cmdq, wqe, ci))
+				return;
+			break;
 		default:
 			/* only arm bit is using scmd wqe,
 			 * the other wqe is lcmd
@@ -283,6 +332,18 @@ void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data)
 	}
 }
 
+static int cmdq_params_valid(const struct hinic3_hwdev *hwdev,
+			     const struct hinic3_cmd_buf *buf_in)
+{
+	if (le16_to_cpu(buf_in->size) > CMDQ_BUF_SIZE) {
+		dev_err(hwdev->dev, "Invalid CMDQ buffer size: 0x%x\n",
+			le16_to_cpu(buf_in->size));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int wait_cmdqs_enable(struct hinic3_cmdqs *cmdqs)
 {
 	unsigned long end;
@@ -292,7 +353,8 @@ static int wait_cmdqs_enable(struct hinic3_cmdqs *cmdqs)
 		if (cmdqs->status & HINIC3_CMDQ_ENABLE)
 			return 0;
 		usleep_range(1000, 2000);
-	} while (time_before(jiffies, end) && !cmdqs->disable_flag);
+	} while (time_before(jiffies, end) && !cmdqs->disable_flag &&
+		 cmdqs->hwdev->chip_present_flag);
 
 	cmdqs->disable_flag = 1;
 
@@ -356,6 +418,7 @@ static void cmdq_prepare_wqe_ctrl(struct cmdq_wqe *wqe, u8 wrapped,
 				  enum cmdq_bufdesc_len buf_len)
 {
 	struct cmdq_header *hdr = CMDQ_WQE_HEADER(wqe);
+	__le32 saved_data = hdr->saved_data;
 	enum cmdq_ctrl_sect_len ctrl_len;
 	struct cmdq_wqe_lcmd *wqe_lcmd;
 	struct cmdq_wqe_scmd *wqe_scmd;
@@ -386,6 +449,11 @@ static void cmdq_prepare_wqe_ctrl(struct cmdq_wqe *wqe, u8 wrapped,
 			    CMDQ_WQE_HDR_SET(3, COMPLETE_SECT_LEN) |
 			    CMDQ_WQE_HDR_SET(ctrl_len, CTRL_LEN) |
 			    CMDQ_WQE_HDR_SET(wrapped, HW_BUSY_BIT));
+
+	saved_data &= ~cpu_to_le32(SAVED_DATA_ARM_MASK);
+	if (cmd == CMDQ_SET_ARM_CMD && mod == MGMT_MOD_COMM)
+		saved_data |= cpu_to_le32(SAVED_DATA_SET(1, ARM));
+	hdr->saved_data = saved_data;
 }
 
 static void cmdq_set_lcmd_wqe(struct cmdq_wqe *wqe,
@@ -485,12 +553,15 @@ static int wait_cmdq_sync_cmd_completion(struct hinic3_cmdq *cmdq,
 	clear_cmd_info(cmd_info, saved_cmd_info);
 	spin_unlock_bh(&cmdq->cmdq_lock);
 
+	hinic3_dump_ceq_info(cmdq->hwdev);
+
 	return err;
 }
 
-static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
-				     struct hinic3_cmd_buf *buf_in,
-				     __le64 *out_param)
+static int cmdq_sync_cmd_exec(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
+			      struct hinic3_cmd_buf *buf_in,
+			      struct hinic3_cmd_buf *buf_out,
+			      __le64 *out_param, u8 cmd_type, u8 wqe_cmd)
 {
 	struct hinic3_cmdq_cmd_info *cmd_info, saved_cmd_info;
 	int cmpt_code = CMDQ_SEND_CMPT_CODE;
@@ -520,31 +591,35 @@ static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
 	cmd_info = &cmdq->cmd_infos[curr_prod_idx];
 	init_completion(&done);
 	refcount_inc(&buf_in->ref_cnt);
-	cmd_info->cmd_type = HINIC3_CMD_TYPE_DIRECT_RESP;
+	if (buf_out)
+		refcount_inc(&buf_out->ref_cnt);
+
+	cmd_info->cmd_type = cmd_type;
 	cmd_info->done = &done;
 	cmd_info->errcode = &errcode;
 	cmd_info->direct_resp = out_param;
 	cmd_info->cmpt_code = &cmpt_code;
 	cmd_info->buf_in = buf_in;
+	if (buf_out)
+		cmd_info->buf_out = buf_out;
+
 	saved_cmd_info = *cmd_info;
-	cmdq_set_lcmd_wqe(&wqe, CMDQ_CMD_DIRECT_RESP, buf_in, NULL,
+	cmdq_set_lcmd_wqe(&wqe, wqe_cmd, buf_in, buf_out,
 			  wrapped, mod, cmd, curr_prod_idx);
 
 	cmdq_wqe_fill(curr_wqe, &wqe);
 	(cmd_info->cmdq_msg_id)++;
 	curr_msg_id = cmd_info->cmdq_msg_id;
-	cmdq_set_db(cmdq, HINIC3_CMDQ_SYNC, next_prod_idx);
+	cmdq_set_db(cmdq, cmdq->cmdq_type, next_prod_idx);
 	spin_unlock_bh(&cmdq->cmdq_lock);
 
 	err = wait_cmdq_sync_cmd_completion(cmdq, cmd_info, &saved_cmd_info,
 					    curr_msg_id, curr_prod_idx,
 					    curr_wqe, CMDQ_CMD_TIMEOUT);
-	if (err) {
+	if (err)
 		dev_err(cmdq->hwdev->dev,
 			"Cmdq sync command timeout, mod: %u, cmd: %u, prod idx: 0x%x\n",
 			mod, cmd, curr_prod_idx);
-		err = -ETIMEDOUT;
-	}
 
 	if (cmpt_code == CMDQ_FORCE_STOP_CMPT_CODE) {
 		dev_dbg(cmdq->hwdev->dev,
@@ -557,22 +632,114 @@ static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
 	return err ? err : errcode;
 }
 
+static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
+				     struct hinic3_cmd_buf *buf_in,
+				     __le64 *out_param)
+{
+	return cmdq_sync_cmd_exec(cmdq, mod, cmd, buf_in, NULL, out_param,
+				    HINIC3_CMD_TYPE_DIRECT_RESP,
+				    CMDQ_CMD_DIRECT_RESP);
+}
+
+static int cmdq_sync_cmd_detail_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
+				     struct hinic3_cmd_buf *buf_in,
+				     struct hinic3_cmd_buf *buf_out,
+				     __le64 *out_param)
+{
+	return cmdq_sync_cmd_exec(cmdq, mod, cmd, buf_in, buf_out, out_param,
+				    HINIC3_CMD_TYPE_SGE_RESP,
+				    CMDQ_CMD_SGE_RESP);
+}
+
+int hinic3_cmd_buf_pair_init(struct hinic3_hwdev *hwdev,
+			     struct hinic3_cmd_buf_pair *pair)
+{
+	pair->in = hinic3_alloc_cmd_buf(hwdev);
+	if (!pair->in)
+		goto err_out;
+
+	pair->out = hinic3_alloc_cmd_buf(hwdev);
+	if (!pair->out)
+		goto err_free_cmd_buf_in;
+
+	return 0;
+
+err_free_cmd_buf_in:
+	hinic3_free_cmd_buf(hwdev, pair->in);
+err_out:
+	return -ENOMEM;
+}
+
+void hinic3_cmd_buf_pair_uninit(struct hinic3_hwdev *hwdev,
+				struct hinic3_cmd_buf_pair *pair)
+{
+	hinic3_free_cmd_buf(hwdev, pair->in);
+	hinic3_free_cmd_buf(hwdev, pair->out);
+}
+
 int hinic3_cmdq_direct_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd,
 			    struct hinic3_cmd_buf *buf_in, __le64 *out_param)
 {
 	struct hinic3_cmdqs *cmdqs;
 	int err;
 
+	if (!hwdev->chip_present_flag)
+		return -ETIMEDOUT;
+
+	err = cmdq_params_valid(hwdev, buf_in);
+	if (err) {
+		dev_err(hwdev->dev, "Invalid CMDQ parameters\n");
+		goto err_out;
+	}
+
 	cmdqs = hwdev->cmdqs;
 	err = wait_cmdqs_enable(cmdqs);
 	if (err) {
 		dev_err(hwdev->dev, "Cmdq is disabled\n");
-		return err;
+		goto err_out;
 	}
 
 	err = cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC3_CMDQ_SYNC],
 					mod, cmd, buf_in, out_param);
+	if (err)
+		goto err_out;
+
+	return 0;
 
+err_out:
+	return err;
+}
+
+int hinic3_cmdq_detail_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd,
+			    struct hinic3_cmd_buf *buf_in,
+			    struct hinic3_cmd_buf *buf_out, __le64 *out_param)
+{
+	struct hinic3_cmdqs *cmdqs;
+	int err;
+
+	if (!hwdev->chip_present_flag)
+		return -ETIMEDOUT;
+
+	err = cmdq_params_valid(hwdev, buf_in);
+	if (err)
+		goto err_out;
+
+	cmdqs = hwdev->cmdqs;
+
+	err = wait_cmdqs_enable(cmdqs);
+	if (err) {
+		dev_err(hwdev->dev, "Cmdq is disabled\n");
+		goto err_out;
+	}
+
+	err = cmdq_sync_cmd_detail_resp(&cmdqs->cmdq[HINIC3_CMDQ_SYNC],
+					mod, cmd, buf_in, buf_out, out_param);
+	if (err)
+		goto err_out;
+
+	return 0;
+
+err_out:
 	return err;
 }
 
@@ -758,7 +925,8 @@ static int init_cmdqs(struct hinic3_hwdev *hwdev)
 
 static void cmdq_flush_sync_cmd(struct hinic3_cmdq_cmd_info *cmd_info)
 {
-	if (cmd_info->cmd_type != HINIC3_CMD_TYPE_DIRECT_RESP)
+	if (cmd_info->cmd_type != HINIC3_CMD_TYPE_DIRECT_RESP &&
+	    cmd_info->cmd_type != HINIC3_CMD_TYPE_SGE_RESP)
 		return;
 
 	cmd_info->cmd_type = HINIC3_CMD_TYPE_FORCE_STOP;
@@ -785,7 +953,8 @@ static void hinic3_cmdq_flush_cmd(struct hinic3_cmdq *cmdq)
 	while (cmdq_read_wqe(&cmdq->wq, &ci)) {
 		hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS);
 		cmd_info = &cmdq->cmd_infos[ci];
-		if (cmd_info->cmd_type == HINIC3_CMD_TYPE_DIRECT_RESP)
+		if (cmd_info->cmd_type == HINIC3_CMD_TYPE_DIRECT_RESP ||
+		    cmd_info->cmd_type == HINIC3_CMD_TYPE_SGE_RESP)
 			cmdq_flush_sync_cmd(cmd_info);
 	}
 	spin_unlock_bh(&cmdq->cmdq_lock);
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h
index f99c386a2780f..29bad2fae7ba5 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h
@@ -85,7 +85,9 @@ enum hinic3_cmdq_status {
 
 enum hinic3_cmdq_cmd_type {
 	HINIC3_CMD_TYPE_NONE,
+	HINIC3_CMD_TYPE_SET_ARM,
 	HINIC3_CMD_TYPE_DIRECT_RESP,
+	HINIC3_CMD_TYPE_SGE_RESP,
 	HINIC3_CMD_TYPE_FAKE_TIMEOUT,
 	HINIC3_CMD_TYPE_TIMEOUT,
 	HINIC3_CMD_TYPE_FORCE_STOP,
@@ -98,6 +100,11 @@ struct hinic3_cmd_buf {
 	refcount_t ref_cnt;
 };
 
+struct hinic3_cmd_buf_pair {
+	struct hinic3_cmd_buf *in;
+	struct hinic3_cmd_buf *out;
+};
+
 struct hinic3_cmdq_cmd_info {
 	enum hinic3_cmdq_cmd_type cmd_type;
 	struct completion         *done;
@@ -107,6 +114,7 @@ struct hinic3_cmdq_cmd_info {
 	__le64                    *direct_resp;
 	u64                       cmdq_msg_id;
 	struct hinic3_cmd_buf     *buf_in;
+	struct hinic3_cmd_buf     *buf_out;
 };
 
 struct hinic3_cmdq {
@@ -146,8 +154,15 @@ void hinic3_free_cmd_buf(struct hinic3_hwdev *hwdev,
 			 struct hinic3_cmd_buf *cmd_buf);
 void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data);
 
+int hinic3_cmd_buf_pair_init(struct hinic3_hwdev *hwdev,
+			     struct hinic3_cmd_buf_pair *pair);
+void hinic3_cmd_buf_pair_uninit(struct hinic3_hwdev *hwdev,
+				struct hinic3_cmd_buf_pair *pair);
 int hinic3_cmdq_direct_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd,
 			    struct hinic3_cmd_buf *buf_in, __le64 *out_param);
+int hinic3_cmdq_detail_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd,
+			    struct hinic3_cmd_buf *buf_in,
+			    struct hinic3_cmd_buf *buf_out, __le64 *out_param);
 
 void hinic3_cmdq_flush_sync_cmd(struct hinic3_hwdev *hwdev);
 int hinic3_reinit_cmdq_ctxts(struct hinic3_hwdev *hwdev);
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_common.c b/drivers/net/ethernet/huawei/hinic3/hinic3_common.c
index fe4778d152cff..b28576debdc81 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_common.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_common.c
@@ -59,10 +59,14 @@ int hinic3_wait_for_timeout(void *priv_data, wait_cpl_handler handler,
 	enum hinic3_wait_return ret;
 	int err;
 
-	err = read_poll_timeout(handler, ret, ret == HINIC3_WAIT_PROCESS_CPL,
+	err = read_poll_timeout(handler, ret,
+				!(ret & HINIC3_WAIT_PROCESS_WAITING),
 				wait_once_us, wait_total_ms * USEC_PER_MSEC,
 				false, priv_data);
 
+	if (ret == HINIC3_WAIT_PROCESS_ERR)
+		return -EIO;
+
 	return err;
 }
 
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_common.h b/drivers/net/ethernet/huawei/hinic3/hinic3_common.h
index a8fabfae90fb5..c892439fa3cd4 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_common.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_common.h
@@ -21,6 +21,7 @@ struct hinic3_dma_addr_align {
 enum hinic3_wait_return {
 	HINIC3_WAIT_PROCESS_CPL     = 0,
 	HINIC3_WAIT_PROCESS_WAITING = 1,
+	HINIC3_WAIT_PROCESS_ERR     = 2,
 };
 
 struct hinic3_sge {
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h b/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h
index f7083a6e7df9e..0e32ff34919ea 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h
@@ -76,9 +76,11 @@
 
 #define HINIC3_CSR_AEQ_CTRL_0_ADDR           (HINIC3_CFG_REGS_FLAG + 0x200)
 #define HINIC3_CSR_AEQ_CTRL_1_ADDR           (HINIC3_CFG_REGS_FLAG + 0x204)
+#define HINIC3_CSR_AEQ_CONS_IDX_ADDR         (HINIC3_CFG_REGS_FLAG + 0x208)
 #define HINIC3_CSR_AEQ_PROD_IDX_ADDR         (HINIC3_CFG_REGS_FLAG + 0x20C)
 #define HINIC3_CSR_AEQ_CI_SIMPLE_INDIR_ADDR  (HINIC3_CFG_REGS_FLAG + 0x50)
 
+#define HINIC3_CSR_CEQ_CONS_IDX_ADDR         (HINIC3_CFG_REGS_FLAG + 0x288)
 #define HINIC3_CSR_CEQ_PROD_IDX_ADDR         (HINIC3_CFG_REGS_FLAG + 0x28c)
 #define HINIC3_CSR_CEQ_CI_SIMPLE_INDIR_ADDR  (HINIC3_CFG_REGS_FLAG + 0x54)
 
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c
index 13a0c6b076601..b8ac1d7bd82bf 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c
@@ -56,6 +56,10 @@
 #define EQ_CI_SIMPLE_INDIR_SET(val, member)  \
 	FIELD_PREP(EQ_CI_SIMPLE_INDIR_##member##_MASK, val)
 
+#define EQ_CONS_IDX_REG_ADDR(eq)  \
+	(((eq)->type == HINIC3_AEQ) ?  \
+	 HINIC3_CSR_AEQ_CONS_IDX_ADDR : HINIC3_CSR_CEQ_CONS_IDX_ADDR)
+
 #define EQ_CI_SIMPLE_INDIR_REG_ADDR(eq)  \
 	(((eq)->type == HINIC3_AEQ) ?  \
 	 HINIC3_CSR_AEQ_CI_SIMPLE_INDIR_ADDR :  \
@@ -353,6 +357,7 @@ static irqreturn_t ceq_interrupt(int irq, void *data)
 	struct hinic3_eq *ceq = data;
 	int err;
 
+	ceq->soft_intr_jif = jiffies;
 	/* clear resend timer counters */
 	hinic3_msix_intr_clear_resend_bit(ceq->hwdev, ceq->msix_entry_idx,
 					  EQ_MSIX_RESEND_TIMER_CLEAR);
@@ -713,6 +718,39 @@ void hinic3_aeqs_free(struct hinic3_hwdev *hwdev)
 	kfree(aeqs);
 }
 
+void hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev)
+{
+	const struct hinic3_aeq_elem *aeqe_pos;
+	u32 addr, ci, pi, ctrl0, idx;
+	struct hinic3_eq *eq;
+	int q_id;
+
+	for (q_id = 0; q_id < hwdev->aeqs->num_aeqs; q_id++) {
+		eq = &hwdev->aeqs->aeq[q_id];
+		/* Indirect access should set q_id first */
+		hinic3_hwif_write_reg(eq->hwdev->hwif,
+				      HINIC3_EQ_INDIR_IDX_ADDR(eq->type),
+				      eq->q_id);
+
+		addr = HINIC3_CSR_AEQ_CTRL_0_ADDR;
+
+		ctrl0 = hinic3_hwif_read_reg(hwdev->hwif, addr);
+
+		idx = hinic3_hwif_read_reg(hwdev->hwif,
+					   HINIC3_EQ_INDIR_IDX_ADDR(eq->type));
+
+		addr = EQ_CONS_IDX_REG_ADDR(eq);
+		ci = hinic3_hwif_read_reg(hwdev->hwif, addr);
+		addr = EQ_PROD_IDX_REG_ADDR(eq);
+		pi = hinic3_hwif_read_reg(hwdev->hwif, addr);
+		aeqe_pos = get_curr_aeq_elem(eq);
+		dev_err(hwdev->dev,
+			"Aeq id: %d, idx: %u, ctrl0: 0x%08x, ci: 0x%08x, pi: 0x%x, work_state: 0x%x, wrap: %u, desc: 0x%x swci:0x%x\n",
+			q_id, idx, ctrl0, ci, pi, work_busy(&eq->aeq_work),
+			eq->wrapped, be32_to_cpu(aeqe_pos->desc), eq->cons_idx);
+	}
+}
+
 int hinic3_ceqs_init(struct hinic3_hwdev *hwdev, u16 num_ceqs,
 		     struct msix_entry *msix_entries)
 {
@@ -773,3 +811,30 @@ void hinic3_ceqs_free(struct hinic3_hwdev *hwdev)
 
 	kfree(ceqs);
 }
+
+void hinic3_dump_ceq_info(struct hinic3_hwdev *hwdev)
+{
+	struct hinic3_eq *eq;
+	u32 addr, ci, pi;
+	int q_id;
+
+	for (q_id = 0; q_id < hwdev->ceqs->num_ceqs; q_id++) {
+		eq = &hwdev->ceqs->ceq[q_id];
+		/* Indirect access should set q_id first */
+		hinic3_hwif_write_reg(eq->hwdev->hwif,
+				      HINIC3_EQ_INDIR_IDX_ADDR(eq->type),
+				      eq->q_id);
+
+		addr = EQ_CONS_IDX_REG_ADDR(eq);
+		ci = hinic3_hwif_read_reg(hwdev->hwif, addr);
+		addr = EQ_PROD_IDX_REG_ADDR(eq);
+		pi = hinic3_hwif_read_reg(hwdev->hwif, addr);
+		dev_err(hwdev->dev,
+			"Ceq id: %d, ci: 0x%08x, sw_ci: 0x%08x, pi: 0x%x, wrap: %u, ceqe: 0x%x\n",
+			q_id, ci, eq->cons_idx, pi,
+			eq->wrapped, be32_to_cpu(*get_curr_ceq_elem(eq)));
+
+		dev_err(hwdev->dev, "Ceq last response soft interrupt time: %u\n",
+			jiffies_to_msecs(jiffies - eq->soft_intr_jif));
+	}
+}
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h
index 005a6e0745b37..c0fa237b270b6 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h
@@ -56,6 +56,8 @@ struct hinic3_eq {
 	u16                       msix_entry_idx;
 	char                      irq_name[HINIC3_EQ_IRQ_NAME_LEN];
 	struct work_struct        aeq_work;
+
+	u64                       soft_intr_jif;
 };
 
 struct hinic3_aeq_elem {
@@ -110,6 +112,8 @@ int hinic3_aeq_register_cb(struct hinic3_hwdev *hwdev,
 			   hinic3_aeq_event_cb hwe_cb);
 void hinic3_aeq_unregister_cb(struct hinic3_hwdev *hwdev,
 			      enum hinic3_aeq_type event);
+void hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev);
+
 int hinic3_ceqs_init(struct hinic3_hwdev *hwdev, u16 num_ceqs,
 		     struct msix_entry *msix_entries);
 void hinic3_ceqs_free(struct hinic3_hwdev *hwdev);
@@ -118,5 +122,6 @@ int hinic3_ceq_register_cb(struct hinic3_hwdev *hwdev,
 			   hinic3_ceq_event_cb callback);
 void hinic3_ceq_unregister_cb(struct hinic3_hwdev *hwdev,
 			      enum hinic3_ceq_event event);
+void hinic3_dump_ceq_info(struct hinic3_hwdev *hwdev);
 
 #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c b/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c
new file mode 100644
index 0000000000000..90fc16288de95
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c
@@ -0,0 +1,425 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/netdevice.h>
+#include <linux/ethtool.h>
+
+#include "hinic3_lld.h"
+#include "hinic3_hw_comm.h"
+#include "hinic3_nic_dev.h"
+#include "hinic3_nic_cfg.h"
+
+#define HINIC3_MGMT_VERSION_MAX_LEN     32
+
+static void hinic3_get_drvinfo(struct net_device *netdev,
+			       struct ethtool_drvinfo *info)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+	u8 mgmt_ver[HINIC3_MGMT_VERSION_MAX_LEN];
+	struct pci_dev *pdev = nic_dev->pdev;
+	int err;
+
+	strscpy(info->driver, HINIC3_NIC_DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info));
+
+	err = hinic3_get_mgmt_version(nic_dev->hwdev, mgmt_ver,
+				      HINIC3_MGMT_VERSION_MAX_LEN);
+	if (err) {
+		netdev_err(netdev, "Failed to get fw version\n");
+		return;
+	}
+
+	snprintf(info->fw_version, sizeof(info->fw_version), "%s", mgmt_ver);
+}
+
+static u32 hinic3_get_msglevel(struct net_device *netdev)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+
+	return nic_dev->msg_enable;
+}
+
+static void hinic3_set_msglevel(struct net_device *netdev, u32 data)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+
+	nic_dev->msg_enable = data;
+
+	netdev_dbg(netdev, "Set message level: 0x%x\n", data);
+}
+
+static const u32 hinic3_link_mode_ge[] = {
+	ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
+	ETHTOOL_LINK_MODE_1000baseKX_Full_BIT,
+	ETHTOOL_LINK_MODE_1000baseX_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_10ge_base_r[] = {
+	ETHTOOL_LINK_MODE_10000baseKR_Full_BIT,
+	ETHTOOL_LINK_MODE_10000baseR_FEC_BIT,
+	ETHTOOL_LINK_MODE_10000baseCR_Full_BIT,
+	ETHTOOL_LINK_MODE_10000baseSR_Full_BIT,
+	ETHTOOL_LINK_MODE_10000baseLR_Full_BIT,
+	ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_25ge_base_r[] = {
+	ETHTOOL_LINK_MODE_25000baseCR_Full_BIT,
+	ETHTOOL_LINK_MODE_25000baseKR_Full_BIT,
+	ETHTOOL_LINK_MODE_25000baseSR_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_40ge_base_r4[] = {
+	ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT,
+	ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT,
+	ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT,
+	ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_50ge_base_r[] = {
+	ETHTOOL_LINK_MODE_50000baseKR_Full_BIT,
+	ETHTOOL_LINK_MODE_50000baseSR_Full_BIT,
+	ETHTOOL_LINK_MODE_50000baseCR_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_50ge_base_r2[] = {
+	ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT,
+	ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT,
+	ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_100ge_base_r[] = {
+	ETHTOOL_LINK_MODE_100000baseKR_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseSR_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseCR_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_100ge_base_r2[] = {
+	ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_100ge_base_r4[] = {
+	ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT,
+	ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_200ge_base_r2[] = {
+	ETHTOOL_LINK_MODE_200000baseKR2_Full_BIT,
+	ETHTOOL_LINK_MODE_200000baseSR2_Full_BIT,
+	ETHTOOL_LINK_MODE_200000baseCR2_Full_BIT,
+};
+
+static const u32 hinic3_link_mode_200ge_base_r4[] = {
+	ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT,
+	ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT,
+	ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT,
+};
+
+struct hw2ethtool_link_mode {
+	const u32 *link_mode_bit_arr;
+	u32       arr_size;
+	u32       speed;
+};
+
+static const struct hw2ethtool_link_mode
+	hw2ethtool_link_mode_table[LINK_MODE_MAX_NUMBERS] = {
+	[LINK_MODE_GE] = {
+		.link_mode_bit_arr = hinic3_link_mode_ge,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_ge),
+		.speed             = SPEED_1000,
+	},
+	[LINK_MODE_10GE_BASE_R] = {
+		.link_mode_bit_arr = hinic3_link_mode_10ge_base_r,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_10ge_base_r),
+		.speed             = SPEED_10000,
+	},
+	[LINK_MODE_25GE_BASE_R] = {
+		.link_mode_bit_arr = hinic3_link_mode_25ge_base_r,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_25ge_base_r),
+		.speed             = SPEED_25000,
+	},
+	[LINK_MODE_40GE_BASE_R4] = {
+		.link_mode_bit_arr = hinic3_link_mode_40ge_base_r4,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_40ge_base_r4),
+		.speed             = SPEED_40000,
+	},
+	[LINK_MODE_50GE_BASE_R] = {
+		.link_mode_bit_arr = hinic3_link_mode_50ge_base_r,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_50ge_base_r),
+		.speed             = SPEED_50000,
+	},
+	[LINK_MODE_50GE_BASE_R2] = {
+		.link_mode_bit_arr = hinic3_link_mode_50ge_base_r2,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_50ge_base_r2),
+		.speed             = SPEED_50000,
+	},
+	[LINK_MODE_100GE_BASE_R] = {
+		.link_mode_bit_arr = hinic3_link_mode_100ge_base_r,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_100ge_base_r),
+		.speed             = SPEED_100000,
+	},
+	[LINK_MODE_100GE_BASE_R2] = {
+		.link_mode_bit_arr = hinic3_link_mode_100ge_base_r2,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_100ge_base_r2),
+		.speed             = SPEED_100000,
+	},
+	[LINK_MODE_100GE_BASE_R4] = {
+		.link_mode_bit_arr = hinic3_link_mode_100ge_base_r4,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_100ge_base_r4),
+		.speed             = SPEED_100000,
+	},
+	[LINK_MODE_200GE_BASE_R2] = {
+		.link_mode_bit_arr = hinic3_link_mode_200ge_base_r2,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_200ge_base_r2),
+		.speed             = SPEED_200000,
+	},
+	[LINK_MODE_200GE_BASE_R4] = {
+		.link_mode_bit_arr = hinic3_link_mode_200ge_base_r4,
+		.arr_size          = ARRAY_SIZE(hinic3_link_mode_200ge_base_r4),
+		.speed             = SPEED_200000,
+	},
+};
+
+#define GET_SUPPORTED_MODE     0
+#define GET_ADVERTISED_MODE    1
+
+struct hinic3_link_settings {
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
+
+	u32 speed;
+	u8  duplex;
+	u8  port;
+	u8  autoneg;
+};
+
+#define HINIC3_ADD_SUPPORTED_LINK_MODE(ecmd, mode) \
+	set_bit(ETHTOOL_LINK_##mode##_BIT, (ecmd)->supported)
+#define HINIC3_ADD_ADVERTISED_LINK_MODE(ecmd, mode) \
+	set_bit(ETHTOOL_LINK_##mode##_BIT, (ecmd)->advertising)
+
+static void hinic3_add_speed_link_mode(unsigned long *bitmap, u32 mode)
+{
+	u32 i;
+
+	for (i = 0; i < hw2ethtool_link_mode_table[mode].arr_size; i++) {
+		if (hw2ethtool_link_mode_table[mode].link_mode_bit_arr[i] >=
+		    __ETHTOOL_LINK_MODE_MASK_NBITS)
+			continue;
+
+		set_bit(hw2ethtool_link_mode_table[mode].link_mode_bit_arr[i],
+			bitmap);
+	}
+}
+
+/* Related to enum mag_cmd_port_speed */
+static const u32 hw_to_ethtool_speed[] = {
+	(u32)SPEED_UNKNOWN, SPEED_10,    SPEED_100,   SPEED_1000,   SPEED_10000,
+	SPEED_25000,        SPEED_40000, SPEED_50000, SPEED_100000, SPEED_200000
+};
+
+static void
+hinic3_add_ethtool_link_mode(struct hinic3_link_settings *link_settings,
+			     u32 hw_link_mode, u32 name)
+{
+	unsigned long *advertising_mask = link_settings->advertising;
+	unsigned long *supported_mask = link_settings->supported;
+	u32 link_mode;
+
+	for (link_mode = 0; link_mode < LINK_MODE_MAX_NUMBERS; link_mode++) {
+		if (hw_link_mode & BIT(link_mode)) {
+			if (name == GET_SUPPORTED_MODE)
+				hinic3_add_speed_link_mode(supported_mask,
+							   link_mode);
+			else
+				hinic3_add_speed_link_mode(advertising_mask,
+							   link_mode);
+		}
+	}
+}
+
+static void
+hinic3_link_speed_set(struct net_device *netdev,
+		      struct hinic3_link_settings *link_settings,
+		      struct hinic3_nic_port_info *port_info)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+	bool link_status_up;
+	int err;
+
+	if (port_info->supported_mode != LINK_MODE_UNKNOWN)
+		hinic3_add_ethtool_link_mode(link_settings,
+					     port_info->supported_mode,
+					     GET_SUPPORTED_MODE);
+	if (port_info->advertised_mode != LINK_MODE_UNKNOWN)
+		hinic3_add_ethtool_link_mode(link_settings,
+					     port_info->advertised_mode,
+					     GET_ADVERTISED_MODE);
+
+	err = hinic3_get_link_status(nic_dev->hwdev, &link_status_up);
+	if (!err && link_status_up) {
+		link_settings->speed =
+			port_info->speed < ARRAY_SIZE(hw_to_ethtool_speed) ?
+			hw_to_ethtool_speed[port_info->speed] :
+			(u32)SPEED_UNKNOWN;
+
+		link_settings->duplex = port_info->duplex;
+	} else {
+		link_settings->speed = (u32)SPEED_UNKNOWN;
+		link_settings->duplex = DUPLEX_UNKNOWN;
+	}
+}
+
+static void
+hinic3_link_port_type_set(struct hinic3_link_settings *link_settings,
+			  u8 port_type)
+{
+	switch (port_type) {
+	case MAG_CMD_WIRE_TYPE_ELECTRIC:
+		HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_TP);
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_TP);
+		link_settings->port = PORT_TP;
+		break;
+
+	case MAG_CMD_WIRE_TYPE_AOC:
+	case MAG_CMD_WIRE_TYPE_MM:
+	case MAG_CMD_WIRE_TYPE_SM:
+		HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_FIBRE);
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_FIBRE);
+		link_settings->port = PORT_FIBRE;
+		break;
+
+	case MAG_CMD_WIRE_TYPE_COPPER:
+		HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_FIBRE);
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_FIBRE);
+		link_settings->port = PORT_DA;
+		break;
+
+	case MAG_CMD_WIRE_TYPE_BACKPLANE:
+		HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Backplane);
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Backplane);
+		link_settings->port = PORT_NONE;
+		break;
+
+	default:
+		link_settings->port = PORT_OTHER;
+		break;
+	}
+}
+
+static int
+hinic3_get_link_pause_settings(struct net_device *netdev,
+			       struct hinic3_link_settings *link_settings)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic3_nic_pause_config nic_pause = {};
+	int err;
+
+	err = hinic3_get_pause_info(nic_dev, &nic_pause);
+	if (err) {
+		netdev_err(netdev, "Failed to get pause param from hw\n");
+		return err;
+	}
+
+	HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Pause);
+	if (nic_pause.rx_pause && nic_pause.tx_pause) {
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Pause);
+	} else if (nic_pause.tx_pause) {
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings,
+						MODE_Asym_Pause);
+	} else if (nic_pause.rx_pause) {
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Pause);
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings,
+						MODE_Asym_Pause);
+	}
+
+	return 0;
+}
+
+static int
+hinic3_get_link_settings(struct net_device *netdev,
+			 struct hinic3_link_settings *link_settings)
+{
+	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic3_nic_port_info port_info = {};
+	int err;
+
+	err = hinic3_get_port_info(nic_dev->hwdev, &port_info);
+	if (err) {
+		netdev_err(netdev, "Failed to get port info\n");
+		return err;
+	}
+
+	hinic3_link_speed_set(netdev, link_settings, &port_info);
+
+	hinic3_link_port_type_set(link_settings, port_info.port_type);
+
+	link_settings->autoneg = port_info.autoneg_state == PORT_CFG_AN_ON ?
+				 AUTONEG_ENABLE : AUTONEG_DISABLE;
+	if (port_info.autoneg_cap)
+		HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Autoneg);
+	if (port_info.autoneg_state == PORT_CFG_AN_ON)
+		HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Autoneg);
+
+	if (!HINIC3_IS_VF(nic_dev->hwdev)) {
+		err = hinic3_get_link_pause_settings(netdev, link_settings);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int
+hinic3_get_link_ksettings(struct net_device *netdev,
+			  struct ethtool_link_ksettings *link_settings)
+{
+	struct ethtool_link_settings *base = &link_settings->base;
+	struct hinic3_link_settings settings = {};
+	int err;
+
+	ethtool_link_ksettings_zero_link_mode(link_settings, supported);
+	ethtool_link_ksettings_zero_link_mode(link_settings, advertising);
+
+	err = hinic3_get_link_settings(netdev, &settings);
+	if (err)
+		return err;
+
+	bitmap_copy(link_settings->link_modes.supported, settings.supported,
+		    __ETHTOOL_LINK_MODE_MASK_NBITS);
+	bitmap_copy(link_settings->link_modes.advertising, settings.advertising,
+		    __ETHTOOL_LINK_MODE_MASK_NBITS);
+
+	base->autoneg = settings.autoneg;
+	base->speed = settings.speed;
+	base->duplex = settings.duplex;
+	base->port = settings.port;
+
+	return 0;
+}
+
+static const struct ethtool_ops hinic3_ethtool_ops = {
+	.supported_coalesce_params      = ETHTOOL_COALESCE_USECS |
+					  ETHTOOL_COALESCE_PKT_RATE_RX_USECS,
+	.get_link_ksettings             = hinic3_get_link_ksettings,
+	.get_drvinfo                    = hinic3_get_drvinfo,
+	.get_msglevel                   = hinic3_get_msglevel,
+	.set_msglevel                   = hinic3_set_msglevel,
+	.get_link                       = ethtool_op_get_link,
+};
+
+void hinic3_set_ethtool_ops(struct net_device *netdev)
+{
+	netdev->ethtool_ops = &hinic3_ethtool_ops;
+}
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c
index ca60cf4b6282a..c5dd53a8fa96c 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c
@@ -17,6 +17,17 @@ static void hinic3_parse_pub_res_cap(struct hinic3_hwdev *hwdev,
 {
 	cap->port_id = dev_cap->port_id;
 	cap->supp_svcs_bitmap = dev_cap->svc_cap_en;
+
+	cap->cos_valid_bitmap = dev_cap->valid_cos_bitmap;
+	cap->port_cos_valid_bitmap = dev_cap->port_cos_valid_bitmap;
+
+	if (type != HINIC3_FUNC_TYPE_VF)
+		cap->max_vf = dev_cap->max_vf;
+	else
+		cap->max_vf = 0;
+
+	dev_dbg(hwdev->dev, "Port_id: 0x%x, cos_bitmap: 0x%x, Max_vf: 0x%x\n",
+		cap->port_id, cap->cos_valid_bitmap, cap->max_vf);
 }
 
 static void hinic3_parse_l2nic_res_cap(struct hinic3_hwdev *hwdev,
@@ -28,6 +39,13 @@ static void hinic3_parse_l2nic_res_cap(struct hinic3_hwdev *hwdev,
 
 	nic_svc_cap->max_sqs = min(dev_cap->nic_max_sq_id + 1,
 				   HINIC3_CFG_MAX_QP);
+
+	nic_svc_cap->max_rqs = min(dev_cap->nic_max_rq_id + 1,
+				   HINIC3_CFG_MAX_QP);
+	nic_svc_cap->default_num_queues = dev_cap->nic_default_num_queues;
+
+	dev_dbg(hwdev->dev, "L2nic resource capability, max_sqs: 0x%x, max_rqs: 0x%x\n",
+		nic_svc_cap->max_sqs, nic_svc_cap->max_rqs);
 }
 
 static void hinic3_parse_dev_cap(struct hinic3_hwdev *hwdev,
@@ -44,8 +62,8 @@ static void hinic3_parse_dev_cap(struct hinic3_hwdev *hwdev,
 		hinic3_parse_l2nic_res_cap(hwdev, cap, dev_cap, type);
 }
 
-static int get_cap_from_fw(struct hinic3_hwdev *hwdev,
-			   enum hinic3_func_type type)
+static int hinic3_get_cap_from_fw(struct hinic3_hwdev *hwdev,
+				  enum hinic3_func_type type)
 {
 	struct mgmt_msg_params msg_params = {};
 	struct cfg_cmd_dev_cap dev_cap = {};
@@ -69,6 +87,29 @@ static int get_cap_from_fw(struct hinic3_hwdev *hwdev,
 	return 0;
 }
 
+static int hinic3_get_dev_cap(struct hinic3_hwdev *hwdev)
+{
+	enum hinic3_func_type type = HINIC3_FUNC_TYPE(hwdev);
+	int err;
+
+	switch (type) {
+	case HINIC3_FUNC_TYPE_PF:
+	case HINIC3_FUNC_TYPE_VF:
+		err = hinic3_get_cap_from_fw(hwdev, type);
+		if (err) {
+			dev_err(hwdev->dev, "Failed to get FW capability\n");
+			return err;
+		}
+		break;
+	default:
+		dev_err(hwdev->dev, "Unsupported PCI Function type: %d\n",
+			type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int hinic3_init_irq_info(struct hinic3_hwdev *hwdev)
 {
 	struct hinic3_cfg_mgmt_info *cfg_mgmt = hwdev->cfg_mgmt;
@@ -215,7 +256,7 @@ void hinic3_free_irq(struct hinic3_hwdev *hwdev, u32 irq_id)
 
 int hinic3_init_capability(struct hinic3_hwdev *hwdev)
 {
-	return get_cap_from_fw(hwdev, HINIC3_FUNC_TYPE_VF);
+	return hinic3_get_dev_cap(hwdev);
 }
 
 bool hinic3_support_nic(struct hinic3_hwdev *hwdev)
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h
index 58806199bf547..361c0b6a70f09 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h
@@ -26,6 +26,8 @@ struct hinic3_irq_info {
 
 struct hinic3_nic_service_cap {
 	u16 max_sqs;
+	u16 max_rqs;
+	u16 default_num_queues;
 };
 
 /* Device capabilities */
@@ -34,6 +36,12 @@ struct hinic3_dev_cap {
 	u16                           supp_svcs_bitmap;
 	/* Physical port */
 	u8                            port_id;
+
+	u8                            cos_valid_bitmap;
+	u8                            port_cos_valid_bitmap;
+	/* max number of VFs that PF supports */
+	u16                           max_vf;
+
 	struct hinic3_nic_service_cap nic_svc_cap;
 };
 
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c
index ecfe6265954e7..0dbf9cbe7f9ab 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c
@@ -292,6 +292,32 @@ int hinic3_set_cmdq_depth(struct hinic3_hwdev *hwdev, u16 cmdq_depth)
 	return 0;
 }
 
+#define HINIC3_FLR_TIMEOUT    1000
+
+static enum hinic3_wait_return hinic3_check_flr_finish_handler(void *priv_data)
+{
+	struct hinic3_hwdev *hwdev = priv_data;
+	struct hinic3_hwif *hwif = hwdev->hwif;
+	enum hinic3_pf_status status;
+
+	if (!hwdev->chip_present_flag)
+		return HINIC3_WAIT_PROCESS_ERR;
+
+	status = hinic3_get_pf_status(hwif);
+	if (status == HINIC3_PF_STATUS_FLR_FINISH_FLAG) {
+		hinic3_set_pf_status(hwif, HINIC3_PF_STATUS_ACTIVE_FLAG);
+		return HINIC3_WAIT_PROCESS_CPL;
+	}
+
+	return HINIC3_WAIT_PROCESS_WAITING;
+}
+
+static int hinic3_wait_for_flr_finish(struct hinic3_hwdev *hwdev)
+{
+	return hinic3_wait_for_timeout(hwdev, hinic3_check_flr_finish_handler,
+				       HINIC3_FLR_TIMEOUT, 0xa * USEC_PER_MSEC);
+}
+
 #define HINIC3_WAIT_CMDQ_IDLE_TIMEOUT    5000
 
 static enum hinic3_wait_return check_cmdq_stop_handler(void *priv_data)
@@ -300,6 +326,10 @@ static enum hinic3_wait_return check_cmdq_stop_handler(void *priv_data)
 	enum hinic3_cmdq_type cmdq_type;
 	struct hinic3_cmdqs *cmdqs;
 
+	/* Stop waiting when card unpresent */
+	if (!hwdev->chip_present_flag)
+		return HINIC3_WAIT_PROCESS_ERR;
+
 	cmdqs = hwdev->cmdqs;
 	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
 		if (!hinic3_cmdq_idle(&cmdqs->cmdq[cmdq_type]))
@@ -347,6 +377,9 @@ int hinic3_func_rx_tx_flush(struct hinic3_hwdev *hwdev)
 	int ret = 0;
 	int err;
 
+	if (!hwdev->chip_present_flag)
+		return 0;
+
 	err = wait_cmdq_stop(hwdev);
 	if (err) {
 		dev_warn(hwdev->dev, "CMDQ is still working, CMDQ timeout value is unreasonable\n");
@@ -382,6 +415,14 @@ int hinic3_func_rx_tx_flush(struct hinic3_hwdev *hwdev)
 		ret = err;
 	}
 
+	if (HINIC3_FUNC_TYPE(hwdev) != HINIC3_FUNC_TYPE_VF) {
+		err = hinic3_wait_for_flr_finish(hwdev);
+		if (err) {
+			dev_warn(hwdev->dev, "Wait firmware FLR timeout\n");
+			ret = err;
+		}
+	}
+
 	hinic3_toggle_doorbell(hwif, ENABLE_DOORBELL);
 
 	err = hinic3_reinit_cmdq_ctxts(hwdev);
@@ -539,3 +580,31 @@ int hinic3_clean_root_ctxt(struct hinic3_hwdev *hwdev)
 
 	return 0;
 }
+
+#define HINIC3_FW_VER_TYPE_MPU  1
+
+int hinic3_get_mgmt_version(struct hinic3_hwdev *hwdev, u8 *mgmt_ver,
+			    u8 version_size)
+{
+	struct comm_cmd_get_fw_version fw_ver = {};
+	struct mgmt_msg_params msg_params = {};
+	int err;
+
+	fw_ver.fw_type = HINIC3_FW_VER_TYPE_MPU;
+
+	mgmt_msg_params_init_default(&msg_params, &fw_ver, sizeof(fw_ver));
+
+	err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_COMM,
+				       COMM_CMD_GET_FW_VERSION, &msg_params);
+
+	if (err || fw_ver.head.status) {
+		dev_err(hwdev->dev,
+			"Failed to get fw version, err: %d, status: 0x%x\n",
+			err, fw_ver.head.status);
+		return -EFAULT;
+	}
+
+	snprintf(mgmt_ver, version_size, "%s", fw_ver.ver);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h
index 8e4737c486b7f..e672f9af5cb14 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h
@@ -49,5 +49,7 @@ void hinic3_sync_time_to_fw(struct hinic3_hwdev *hwdev);
 int hinic3_set_root_ctxt(struct hinic3_hwdev *hwdev, u32 rq_depth, u32 sq_depth,
 			 int rx_buf_sz);
 int hinic3_clean_root_ctxt(struct hinic3_hwdev *hwdev);
+int hinic3_get_mgmt_version(struct hinic3_hwdev *hwdev, u8 *mgmt_ver,
+			    u8 version_size);
 
 #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h
index 329a9c464ff95..cfc9daa3034fe 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h
@@ -114,6 +114,7 @@ enum comm_cmd {
 	COMM_CMD_SET_DMA_ATTR            = 25,
 
 	/* Commands for obtaining information */
+	COMM_CMD_GET_FW_VERSION          = 60,
 	COMM_CMD_SYNC_TIME               = 62,
 	COMM_CMD_SEND_BDF_INFO           = 64,
 };
@@ -275,6 +276,17 @@ struct comm_cmd_bdf_info {
 	u8                   rsvd2[5];
 };
 
+#define COMM_FW_VERSION_LEN       16
+#define COMM_FW_COMPILE_TIME_LEN  20
+struct comm_cmd_get_fw_version {
+	struct mgmt_msg_head head;
+
+	u16                  fw_type;
+	u16                  rsvd1;
+	u8                   ver[COMM_FW_VERSION_LEN];
+	u8                   time[COMM_FW_COMPILE_TIME_LEN];
+};
+
 /* Services supported by HW. HW uses these values when delivering events.
  * HW supports multiple services that are not yet supported by driver
  * (e.g. RoCE).
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
index 0074d0c6dbaf5..f44b3064ab2ea 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
@@ -32,6 +32,9 @@
 #define HINIC3_PCIE_PH_DISABLE       0
 #define HINIC3_PCIE_MSIX_ATTR_ENTRY  0
 
+#define HINIC3_CHIP_PRESENT          1
+#define HINIC3_CHIP_ABSENT           0
+
 #define HINIC3_DEFAULT_EQ_MSIX_PENDING_LIMIT      0
 #define HINIC3_DEFAULT_EQ_MSIX_COALESC_TIMER_CFG  0xFF
 #define HINIC3_DEFAULT_EQ_MSIX_RESEND_TIMER_CFG   7
@@ -431,6 +434,10 @@ static int init_cmdqs_channel(struct hinic3_hwdev *hwdev)
 static void hinic3_free_cmdqs_channel(struct hinic3_hwdev *hwdev)
 {
 	hinic3_comm_cmdqs_free(hwdev);
+
+	hinic3_set_wq_page_size(hwdev, hinic3_global_func_id(hwdev),
+				HINIC3_MIN_PAGE_SIZE);
+
 	hinic3_ceqs_free(hwdev);
 }
 
@@ -545,6 +552,7 @@ int hinic3_init_hwdev(struct pci_dev *pdev)
 		dev_err(hwdev->dev, "Failed to init hwif\n");
 		goto err_free_hwdev;
 	}
+	hwdev->chip_present_flag = HINIC3_CHIP_PRESENT;
 
 	hwdev->workq = alloc_workqueue(HINIC3_HWDEV_WQ_NAME, WQ_MEM_RECLAIM | WQ_PERCPU,
 				       HINIC3_WQ_MAX_REQ);
@@ -621,6 +629,7 @@ void hinic3_set_api_stop(struct hinic3_hwdev *hwdev)
 	struct hinic3_recv_msg *recv_resp_msg;
 	struct hinic3_mbox *mbox;
 
+	hwdev->chip_present_flag = HINIC3_CHIP_ABSENT;
 	spin_lock_bh(&hwdev->channel_lock);
 	if (HINIC3_IS_PF(hwdev) &&
 	    test_bit(HINIC3_HWDEV_MGMT_INITED, &hwdev->func_state)) {
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h
index 9686c2600b467..4276ac136464b 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h
@@ -76,6 +76,7 @@ struct hinic3_hwdev {
 	u32                         wq_page_size;
 	u8                          max_cmdq;
 	ulong                       func_state;
+	int                         chip_present_flag;
 };
 
 struct hinic3_event_info {
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c
index 771883174b3be..90feaa2250801 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c
@@ -70,11 +70,19 @@
 #define HINIC3_PPF_ELECTION_GET(val, member) \
 	FIELD_GET(HINIC3_PPF_ELECTION_##member##_MASK, val)
 
+#define HINIC3_GET_REG_FLAG(reg)  ((reg) & (~(HINIC3_REGS_FLAG_MASK)))
 #define HINIC3_GET_REG_ADDR(reg)  ((reg) & (HINIC3_REGS_FLAG_MASK))
 
 static void __iomem *hinic3_reg_addr(struct hinic3_hwif *hwif, u32 reg)
 {
-	return hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg);
+	void __iomem *addr;
+
+	if (HINIC3_GET_REG_FLAG(reg) == HINIC3_MGMT_REGS_FLAG)
+		addr = hwif->mgmt_regs_base + HINIC3_GET_REG_ADDR(reg);
+	else
+		addr = hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg);
+
+	return addr;
 }
 
 u32 hinic3_hwif_read_reg(struct hinic3_hwif *hwif, u32 reg)
@@ -97,6 +105,8 @@ static enum hinic3_wait_return check_hwif_ready_handler(void *priv_data)
 	u32 attr1;
 
 	attr1 = hinic3_hwif_read_reg(hwdev->hwif, HINIC3_CSR_FUNC_ATTR1_ADDR);
+	if (attr1 == HINIC3_PCIE_LINK_DOWN)
+		return HINIC3_WAIT_PROCESS_ERR;
 
 	return HINIC3_AF1_GET(attr1, MGMT_INIT_STATUS) ?
 	       HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING;
@@ -135,6 +145,7 @@ static void set_hwif_attr(struct hinic3_func_attr *attr, u32 attr0, u32 attr1,
 static int init_hwif_attr(struct hinic3_hwdev *hwdev)
 {
 	u32 attr0, attr1, attr2, attr3, attr6;
+	struct hinic3_func_attr *attr;
 	struct hinic3_hwif *hwif;
 
 	hwif = hwdev->hwif;
@@ -158,7 +169,15 @@ static int init_hwif_attr(struct hinic3_hwdev *hwdev)
 	if (attr6 == HINIC3_PCIE_LINK_DOWN)
 		return -EFAULT;
 
-	set_hwif_attr(&hwif->attr, attr0, attr1, attr2, attr3, attr6);
+	attr = &hwif->attr;
+	set_hwif_attr(attr, attr0, attr1, attr2, attr3, attr6);
+
+	if (attr->func_type != HINIC3_FUNC_TYPE_VF &&
+	    attr->func_type != HINIC3_FUNC_TYPE_PF) {
+		dev_err(hwdev->dev, "unexpected func_type %u\n",
+			attr->func_type);
+		return -EINVAL;
+	}
 
 	if (!hwif->attr.num_ceqs) {
 		dev_err(hwdev->dev, "Ceq num cfg in fw is zero\n");
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
index a8b89eeed753f..f83d5bd1041e3 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
@@ -426,6 +426,7 @@ static void hinic3_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id hinic3_pci_table[] = {
+	{PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_PF), 0},
 	{PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_VF), 0},
 	{0, 0}
 
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_main.c b/drivers/net/ethernet/huawei/hinic3/hinic3_main.c
index 6039fcf3c1dd1..0a888fe4c975b 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_main.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_main.c
@@ -18,6 +18,7 @@
 
 #define HINIC3_NIC_DRV_DESC  "Intelligent Network Interface Card Driver"
 
+#define HINIC3_DEFAULT_MSG_ENABLE  (NETIF_MSG_DRV | NETIF_MSG_LINK)
 #define HINIC3_RX_BUF_LEN          2048
 #define HINIC3_LRO_REPLENISH_THLD  256
 #define HINIC3_NIC_DEV_WQ_NAME     "hinic3_nic_dev_wq"
@@ -143,6 +144,7 @@ static int hinic3_init_nic_dev(struct net_device *netdev,
 	nic_dev->hwdev = hwdev;
 	nic_dev->pdev = pdev;
 
+	nic_dev->msg_enable = HINIC3_DEFAULT_MSG_ENABLE;
 	nic_dev->rx_buf_len = HINIC3_RX_BUF_LEN;
 	nic_dev->lro_replenish_thld = HINIC3_LRO_REPLENISH_THLD;
 	nic_dev->vlan_bitmap = kzalloc(HINIC3_VLAN_BITMAP_SIZE(nic_dev),
@@ -241,6 +243,7 @@ static void hinic3_sw_uninit(struct net_device *netdev)
 static void hinic3_assign_netdev_ops(struct net_device *netdev)
 {
 	hinic3_set_netdev_ops(netdev);
+	hinic3_set_ethtool_ops(netdev);
 }
 
 static void netdev_feature_init(struct net_device *netdev)
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c
index 826fa8879a113..c1bc64a50dd3b 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c
@@ -5,6 +5,7 @@
 
 #include "hinic3_common.h"
 #include "hinic3_csr.h"
+#include "hinic3_eqs.h"
 #include "hinic3_hwdev.h"
 #include "hinic3_hwif.h"
 #include "hinic3_mbox.h"
@@ -50,9 +51,9 @@
 #define MBOX_WB_STATUS_NOT_FINISHED      0x00
 
 #define MBOX_STATUS_FINISHED(wb)  \
-	((FIELD_PREP(MBOX_WB_STATUS_MASK, (wb))) != MBOX_WB_STATUS_NOT_FINISHED)
+	((FIELD_GET(MBOX_WB_STATUS_MASK, (wb))) != MBOX_WB_STATUS_NOT_FINISHED)
 #define MBOX_STATUS_SUCCESS(wb)  \
-	((FIELD_PREP(MBOX_WB_STATUS_MASK, (wb))) ==  \
+	((FIELD_GET(MBOX_WB_STATUS_MASK, (wb))) ==  \
 	MBOX_WB_STATUS_FINISHED_SUCCESS)
 #define MBOX_STATUS_ERRCODE(wb)  \
 	((wb) & MBOX_WB_ERROR_CODE_MASK)
@@ -395,6 +396,7 @@ static int hinic3_mbox_pre_init(struct hinic3_hwdev *hwdev,
 {
 	mbox->hwdev = hwdev;
 	mutex_init(&mbox->mbox_send_lock);
+	mutex_init(&mbox->mbox_seg_send_lock);
 	spin_lock_init(&mbox->mbox_lock);
 
 	mbox->workq = create_singlethread_workqueue(HINIC3_MBOX_WQ_NAME);
@@ -460,7 +462,8 @@ void hinic3_free_mbox(struct hinic3_hwdev *hwdev)
 
 	destroy_workqueue(mbox->workq);
 	free_mbox_wb_status(mbox);
-	hinic3_uninit_func_mbox_msg_channel(hwdev);
+	if (HINIC3_IS_VF(hwdev))
+		hinic3_uninit_func_mbox_msg_channel(hwdev);
 	uninit_mgmt_msg_channel(mbox);
 	kfree(mbox);
 }
@@ -616,6 +619,18 @@ static void write_mbox_msg_attr(struct hinic3_mbox *mbox,
 			      mbox_ctrl);
 }
 
+static void hinic3_dump_mbox_reg(struct hinic3_hwdev *hwdev)
+{
+	u32 val;
+
+	val = hinic3_hwif_read_reg(hwdev->hwif,
+				   HINIC3_FUNC_CSR_MAILBOX_CONTROL_OFF);
+	dev_err(hwdev->dev, "Mailbox control reg: 0x%x\n", val);
+	val = hinic3_hwif_read_reg(hwdev->hwif,
+				   HINIC3_FUNC_CSR_MAILBOX_INT_OFF);
+	dev_err(hwdev->dev, "Mailbox interrupt offset: 0x%x\n", val);
+}
+
 static u16 get_mbox_status(const struct hinic3_send_mbox *mbox)
 {
 	__be64 *wb_status = mbox->wb_vaddr;
@@ -633,6 +648,9 @@ static enum hinic3_wait_return check_mbox_wb_status(void *priv_data)
 	struct hinic3_mbox *mbox = priv_data;
 	u16 wb_status;
 
+	if (!mbox->hwdev->chip_present_flag)
+		return HINIC3_WAIT_PROCESS_ERR;
+
 	wb_status = get_mbox_status(&mbox->send_mbox);
 
 	return MBOX_STATUS_FINISHED(wb_status) ?
@@ -670,6 +688,7 @@ static int send_mbox_seg(struct hinic3_mbox *mbox, __le64 header,
 	if (err) {
 		dev_err(hwdev->dev, "Send mailbox segment timeout, wb status: 0x%x\n",
 			wb_status);
+		hinic3_dump_mbox_reg(hwdev);
 		return err;
 	}
 
@@ -706,6 +725,8 @@ static int send_mbox_msg(struct hinic3_mbox *mbox, u8 mod, u16 cmd,
 	else
 		rsp_aeq_id = 0;
 
+	mutex_lock(&mbox->mbox_seg_send_lock);
+
 	if (dst_func == MBOX_MGMT_FUNC_ID &&
 	    !(hwdev->features[0] & MBOX_COMM_F_MBOX_SEGMENT)) {
 		err = mbox_prepare_dma_msg(mbox, ack_type, &dma_msg,
@@ -759,6 +780,8 @@ static int send_mbox_msg(struct hinic3_mbox *mbox, u8 mod, u16 cmd,
 	}
 
 err_send:
+	mutex_unlock(&mbox->mbox_seg_send_lock);
+
 	return err;
 }
 
@@ -774,6 +797,9 @@ static enum hinic3_wait_return check_mbox_msg_finish(void *priv_data)
 {
 	struct hinic3_mbox *mbox = priv_data;
 
+	if (!mbox->hwdev->chip_present_flag)
+		return HINIC3_WAIT_PROCESS_ERR;
+
 	return (mbox->event_flag == MBOX_EVENT_SUCCESS) ?
 		HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING;
 }
@@ -805,6 +831,9 @@ int hinic3_send_mbox_to_mgmt(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd,
 	u32 msg_len;
 	int err;
 
+	if (!hwdev->chip_present_flag)
+		return -EPERM;
+
 	/* expect response message */
 	msg_desc = get_mbox_msg_desc(mbox, MBOX_MSG_RESP, MBOX_MGMT_FUNC_ID);
 	mutex_lock(&mbox->mbox_send_lock);
@@ -825,6 +854,7 @@ int hinic3_send_mbox_to_mgmt(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd,
 	if (wait_mbox_msg_completion(mbox, msg_params->timeout_ms)) {
 		dev_err(hwdev->dev,
 			"Send mbox msg timeout, msg_id: %u\n", msg_info.msg_id);
+		hinic3_dump_aeq_info(mbox->hwdev);
 		err = -ETIMEDOUT;
 		goto err_send;
 	}
@@ -882,6 +912,9 @@ int hinic3_send_mbox_to_mgmt_no_ack(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd,
 	struct mbox_msg_info msg_info = {};
 	int err;
 
+	if (!hwdev->chip_present_flag)
+		return -EPERM;
+
 	mutex_lock(&mbox->mbox_send_lock);
 	err = send_mbox_msg(mbox, mod, cmd, msg_params->buf_in,
 			    msg_params->in_size, MBOX_MGMT_FUNC_ID,
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h
index e26f22d1d5641..30de0c1295038 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h
@@ -114,6 +114,10 @@ struct hinic3_mbox {
 	struct hinic3_hwdev       *hwdev;
 	/* lock for send mbox message and ack message */
 	struct mutex              mbox_send_lock;
+	/* lock for send message transmission.
+	 * The lock hierarchy is mbox_send_lock -> mbox_seg_send_lock.
+	 */
+	struct mutex              mbox_seg_send_lock;
 	struct hinic3_send_mbox   send_mbox;
 	struct mbox_dma_queue     sync_msg_queue;
 	struct mbox_dma_queue     async_msg_queue;
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h
index c0c87a8c21989..c5bca3c4af961 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h
@@ -183,7 +183,18 @@ struct l2nic_cmd_lro_timer {
 /* IEEE 802.1Qaz std */
 #define L2NIC_DCB_COS_MAX     0x8
 
-struct l2nic_cmd_set_rss_ctx_tbl {
+struct l2nic_cmd_pause_config {
+	struct mgmt_msg_head msg_head;
+	u8                   port_id;
+	u8                   opcode;
+	u16                  rsvd1;
+	u8                   auto_neg;
+	u8                   rx_pause;
+	u8                   tx_pause;
+	u8                   rsvd2[5];
+};
+
+struct l2nic_cmd_rss_ctx_tbl {
 	struct mgmt_msg_head msg_head;
 	u16                  func_id;
 	u16                  rsvd1;
@@ -238,6 +249,7 @@ enum l2nic_cmd {
 	L2NIC_CMD_CFG_RSS_HASH_KEY    = 63,
 	L2NIC_CMD_CFG_RSS_HASH_ENGINE = 64,
 	L2NIC_CMD_SET_RSS_CTX_TBL     = 65,
+	L2NIC_CMD_CFG_PAUSE_INFO      = 101,
 	L2NIC_CMD_QOS_DCB_STATE       = 110,
 	L2NIC_CMD_FORCE_PKT_DROP      = 113,
 	L2NIC_CMD_MAX                 = 256,
@@ -259,6 +271,8 @@ enum l2nic_ucode_cmd {
 enum mag_cmd {
 	MAG_CMD_SET_PORT_ENABLE = 6,
 	MAG_CMD_GET_LINK_STATUS = 7,
+
+	MAG_CMD_GET_PORT_INFO   = 153,
 };
 
 /* firmware also use this cmd report link event to driver */
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c b/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c
index cabb8523f246c..da73811641a9d 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c
@@ -415,13 +415,17 @@ static void hinic3_vport_down(struct net_device *netdev)
 	netif_carrier_off(netdev);
 	netif_tx_disable(netdev);
 
-	glb_func_id = hinic3_global_func_id(nic_dev->hwdev);
-	hinic3_set_vport_enable(nic_dev->hwdev, glb_func_id, false);
+	if (nic_dev->hwdev->chip_present_flag) {
+		hinic3_maybe_set_port_state(netdev, false);
 
-	hinic3_flush_txqs(netdev);
-	/* wait to guarantee that no packets will be sent to host */
-	msleep(100);
-	hinic3_flush_qps_res(nic_dev->hwdev);
+		glb_func_id = hinic3_global_func_id(nic_dev->hwdev);
+		hinic3_set_vport_enable(nic_dev->hwdev, glb_func_id, false);
+
+		hinic3_flush_txqs(netdev);
+		/* wait to guarantee that no packets will be sent to host */
+		msleep(100);
+		hinic3_flush_qps_res(nic_dev->hwdev);
+	}
 }
 
 static int hinic3_open(struct net_device *netdev)
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c
index 44abccf9cb29f..de5a7984d2cb9 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c
@@ -639,6 +639,39 @@ int hinic3_get_link_status(struct hinic3_hwdev *hwdev, bool *link_status_up)
 	return 0;
 }
 
+int hinic3_get_port_info(struct hinic3_hwdev *hwdev,
+			 struct hinic3_nic_port_info *port_info)
+{
+	struct mag_cmd_get_port_info port_msg = {};
+	struct mgmt_msg_params msg_params = {};
+	int err;
+
+	port_msg.port_id = hinic3_physical_port_id(hwdev);
+
+	mgmt_msg_params_init_default(&msg_params, &port_msg, sizeof(port_msg));
+
+	err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_HILINK,
+				       MAG_CMD_GET_PORT_INFO, &msg_params);
+
+	if (err || port_msg.head.status) {
+		dev_err(hwdev->dev,
+			"Failed to get port info, err: %d, status: 0x%x\n",
+			err, port_msg.head.status);
+		return -EFAULT;
+	}
+
+	port_info->autoneg_cap = port_msg.an_support;
+	port_info->autoneg_state = port_msg.an_en;
+	port_info->duplex = port_msg.duplex;
+	port_info->port_type = port_msg.wire_type;
+	port_info->speed = port_msg.speed;
+	port_info->fec = port_msg.fec;
+	port_info->supported_mode = port_msg.supported_mode;
+	port_info->advertised_mode = port_msg.advertised_mode;
+
+	return 0;
+}
+
 int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id,
 			    bool enable)
 {
@@ -661,3 +694,47 @@ int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id,
 
 	return 0;
 }
+
+static int hinic3_cfg_hw_pause(struct hinic3_hwdev *hwdev, u8 opcode,
+			       struct hinic3_nic_pause_config *nic_pause)
+{
+	struct l2nic_cmd_pause_config pause_info = {};
+	struct mgmt_msg_params msg_params = {};
+	int err;
+
+	pause_info.port_id = hinic3_physical_port_id(hwdev);
+	pause_info.opcode = opcode;
+	if (opcode == MGMT_MSG_CMD_OP_SET) {
+		pause_info.auto_neg = nic_pause->auto_neg;
+		pause_info.rx_pause = nic_pause->rx_pause;
+		pause_info.tx_pause = nic_pause->tx_pause;
+	}
+
+	mgmt_msg_params_init_default(&msg_params, &pause_info,
+				     sizeof(pause_info));
+
+	err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_L2NIC,
+				       L2NIC_CMD_CFG_PAUSE_INFO, &msg_params);
+
+	if (err || pause_info.msg_head.status) {
+		dev_err(hwdev->dev, "Failed to %s pause info, err: %d, status: 0x%x\n",
+			opcode == MGMT_MSG_CMD_OP_SET ? "set" : "get",
+			err, pause_info.msg_head.status);
+		return -EFAULT;
+	}
+
+	if (opcode == MGMT_MSG_CMD_OP_GET) {
+		nic_pause->auto_neg = pause_info.auto_neg;
+		nic_pause->rx_pause = pause_info.rx_pause;
+		nic_pause->tx_pause = pause_info.tx_pause;
+	}
+
+	return 0;
+}
+
+int hinic3_get_pause_info(struct hinic3_nic_dev *nic_dev,
+			  struct hinic3_nic_pause_config *nic_pause)
+{
+	return hinic3_cfg_hw_pause(nic_dev->hwdev, MGMT_MSG_CMD_OP_GET,
+				   nic_pause);
+}
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h
index c32eaa886e179..5d52202a8d4eb 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h
@@ -6,6 +6,7 @@
 
 #include <linux/types.h>
 
+#include "hinic3_hwif.h"
 #include "hinic3_hw_intf.h"
 #include "hinic3_mgmt_interface.h"
 
@@ -35,6 +36,49 @@ struct hinic3_sq_attr {
 	u64 ci_dma_base;
 };
 
+enum mag_cmd_port_an {
+	PORT_CFG_AN_ON  = 1,
+};
+
+/* mag supported/advertised link mode bitmap */
+enum mag_cmd_link_mode {
+	LINK_MODE_GE            = 0,
+	LINK_MODE_10GE_BASE_R   = 1,
+	LINK_MODE_25GE_BASE_R   = 2,
+	LINK_MODE_40GE_BASE_R4  = 3,
+	LINK_MODE_50GE_BASE_R   = 4,
+	LINK_MODE_50GE_BASE_R2  = 5,
+	LINK_MODE_100GE_BASE_R  = 6,
+	LINK_MODE_100GE_BASE_R2 = 7,
+	LINK_MODE_100GE_BASE_R4 = 8,
+	LINK_MODE_200GE_BASE_R2 = 9,
+	LINK_MODE_200GE_BASE_R4 = 10,
+	LINK_MODE_MAX_NUMBERS,
+
+	LINK_MODE_UNKNOWN       = 0xFFFF
+};
+
+struct mag_cmd_get_port_info {
+	struct mgmt_msg_head head;
+
+	u8                   port_id;
+	u8                   rsvd0[3];
+
+	u8                   wire_type;
+	u8                   an_support;
+	u8                   an_en;
+	u8                   duplex;
+
+	u8                   speed;
+	u8                   fec;
+	u8                   lanes;
+	u8                   rsvd1;
+
+	u32                  supported_mode;
+	u32                  advertised_mode;
+	u8                   rsvd2[8];
+};
+
 #define MAG_CMD_PORT_DISABLE    0x0
 #define MAG_CMD_TX_ENABLE       0x1
 #define MAG_CMD_RX_ENABLE       0x2
@@ -52,6 +96,39 @@ struct mag_cmd_set_port_enable {
 	u8                   rsvd1[3];
 };
 
+/* xsfp wire type, refers to cmis protocol definition */
+enum mag_wire_type {
+	MAG_CMD_WIRE_TYPE_UNKNOWN   = 0x0,
+	MAG_CMD_WIRE_TYPE_MM        = 0x1,
+	MAG_CMD_WIRE_TYPE_SM        = 0x2,
+	MAG_CMD_WIRE_TYPE_COPPER    = 0x3,
+	MAG_CMD_WIRE_TYPE_ACC       = 0x4,
+	MAG_CMD_WIRE_TYPE_BASET     = 0x5,
+	MAG_CMD_WIRE_TYPE_AOC       = 0x40,
+	MAG_CMD_WIRE_TYPE_ELECTRIC  = 0x41,
+	MAG_CMD_WIRE_TYPE_BACKPLANE = 0x42
+};
+
+#define XSFP_INFO_MAX_SIZE    640
+struct mag_cmd_get_xsfp_info {
+	struct mgmt_msg_head head;
+
+	u8                   port_id;
+	u8                   wire_type;
+	u16                  out_len;
+	u32                  rsvd;
+	u8                   sfp_info[XSFP_INFO_MAX_SIZE];
+};
+
+struct mag_cmd_get_xsfp_present {
+	struct mgmt_msg_head head;
+
+	u8                   port_id;
+	/* 0:present, 1:absent */
+	u8                   abs_status;
+	u8                   rsvd[2];
+};
+
 enum link_err_type {
 	LINK_ERR_MODULE_UNRECOGENIZED,
 	LINK_ERR_NUM,
@@ -69,6 +146,34 @@ struct hinic3_port_module_event {
 	enum link_err_type          err_type;
 };
 
+struct hinic3_nic_port_info {
+	u8  port_type;
+	u8  autoneg_cap;
+	u8  autoneg_state;
+	u8  duplex;
+	u8  speed;
+	u8  fec;
+	u32 supported_mode;
+	u32 advertised_mode;
+};
+
+struct hinic3_nic_pause_config {
+	u8 auto_neg;
+	u8 rx_pause;
+	u8 tx_pause;
+};
+
+struct hinic3_nic_cfg {
+	/* Valid when pfc is disabled */
+	bool                           pause_set;
+	struct hinic3_nic_pause_config nic_pause;
+
+	u8                             pfc_en;
+	u8                             pfc_bitmap;
+
+	struct hinic3_nic_port_info    port_info;
+};
+
 int hinic3_get_nic_feature_from_hw(struct hinic3_nic_dev *nic_dev);
 int hinic3_set_nic_feature_to_hw(struct hinic3_nic_dev *nic_dev);
 bool hinic3_test_support(struct hinic3_nic_dev *nic_dev,
@@ -100,9 +205,14 @@ int hinic3_set_rx_mode(struct hinic3_hwdev *hwdev, u32 rx_mode);
 int hinic3_sync_dcb_state(struct hinic3_hwdev *hwdev, u8 op_code, u8 state);
 int hinic3_set_port_enable(struct hinic3_hwdev *hwdev, bool enable);
 int hinic3_get_link_status(struct hinic3_hwdev *hwdev, bool *link_status_up);
+int hinic3_get_port_info(struct hinic3_hwdev *hwdev,
+			 struct hinic3_nic_port_info *port_info);
 int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id,
 			    bool enable);
 int hinic3_add_vlan(struct hinic3_hwdev *hwdev, u16 vlan_id, u16 func_id);
 int hinic3_del_vlan(struct hinic3_hwdev *hwdev, u16 vlan_id, u16 func_id);
 
+int hinic3_get_pause_info(struct hinic3_nic_dev *nic_dev,
+			  struct hinic3_nic_pause_config *nic_pause);
+
 #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h
index 29189241f446c..9502293ff710c 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h
@@ -101,6 +101,7 @@ struct hinic3_nic_dev {
 	struct hinic3_hwdev             *hwdev;
 	struct hinic3_nic_io            *nic_io;
 
+	u32                             msg_enable;
 	u16                             max_qps;
 	u16                             rx_buf_len;
 	u32                             lro_replenish_thld;
@@ -148,4 +149,6 @@ void hinic3_qps_irq_uninit(struct net_device *netdev);
 void hinic3_set_rx_mode_work(struct work_struct *work);
 void hinic3_clean_mac_list_filter(struct net_device *netdev);
 
+void hinic3_set_ethtool_ops(struct net_device *netdev);
+
 #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h b/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h
index 86c88d0bb4bd0..02b2b0fbecc75 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h
@@ -4,6 +4,7 @@
 #ifndef _HINIC3_PCI_ID_TBL_H_
 #define _HINIC3_PCI_ID_TBL_H_
 
+#define PCI_DEV_ID_HINIC3_PF    0x0222
 #define PCI_DEV_ID_HINIC3_VF    0x375F
 
 #endif
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c b/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c
index 4ff1b2f79838a..25db74d8c7ddb 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c
@@ -132,7 +132,7 @@ static int hinic3_rss_set_indir_tbl(struct hinic3_hwdev *hwdev,
 static int hinic3_set_rss_type(struct hinic3_hwdev *hwdev,
 			       struct hinic3_rss_type rss_type)
 {
-	struct l2nic_cmd_set_rss_ctx_tbl ctx_tbl = {};
+	struct l2nic_cmd_rss_ctx_tbl ctx_tbl = {};
 	struct mgmt_msg_params msg_params = {};
 	u32 ctx;
 	int err;
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c
index 1236ec233b7fc..309ab59013797 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c
@@ -328,6 +328,7 @@ static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type,
 	u32 ip_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, IP_TYPE);
 	u32 csum_err = RQ_CQE_STATUS_GET(status, CSUM_ERR);
 	struct net_device *netdev = rxq->netdev;
+	bool l2_tunnel;
 
 	if (!(netdev->features & NETIF_F_RXCSUM))
 		return;
@@ -350,6 +351,12 @@ static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type,
 	case HINIC3_RX_UDP_PKT:
 	case HINIC3_RX_SCTP_PKT:
 		skb->ip_summed = CHECKSUM_UNNECESSARY;
+		l2_tunnel = HINIC3_GET_RX_TUNNEL_PKT_FORMAT(offload_type) ==
+			    HINIC3_RX_PKT_FORMAT_VXLAN ? 1 : 0;
+		if (l2_tunnel) {
+			/* If we checked the outer header let the stack know */
+			skb->csum_level = 1;
+		}
 		break;
 	default:
 		skb->ip_summed = CHECKSUM_NONE;
@@ -390,6 +397,14 @@ static int recv_one_pkt(struct hinic3_rxq *rxq, struct hinic3_rq_cqe *rx_cqe,
 	offload_type = le32_to_cpu(rx_cqe->offload_type);
 	hinic3_rx_csum(rxq, offload_type, status, skb);
 
+	if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+	    RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)) {
+		u16 vid = RQ_CQE_SGE_GET(vlan_len, VLAN);
+
+		/* if the packet is a vlan pkt, the vid may be 0 */
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
+	}
+
 	num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO);
 	if (num_lro)
 		hinic3_lro_set_gso_params(skb, num_lro);
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h
index 31622e0a63d06..06d1b3299e7cf 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h
@@ -15,6 +15,9 @@
 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member) \
 	FIELD_GET(RQ_CQE_OFFOLAD_TYPE_##member##_MASK, val)
 
+#define HINIC3_GET_RX_TUNNEL_PKT_FORMAT(offload_type) \
+	RQ_CQE_OFFOLAD_TYPE_GET(offload_type, TUNNEL_PKT_FORMAT)
+
 #define RQ_CQE_SGE_VLAN_MASK  GENMASK(15, 0)
 #define RQ_CQE_SGE_LEN_MASK   GENMASK(31, 16)
 #define RQ_CQE_SGE_GET(val, member) \
diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
index 4d6b9e83e3412..58cc3147afe26 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -355,7 +355,7 @@ static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,
 
 		/*
 		 * If multi rx buffers hcall is no longer supported by FW
-		 * e.g. in the case of Live Parttion Migration
+		 * e.g. in the case of Live Partition Migration
 		 */
 		if (batch > 1 && lpar_rc == H_FUNCTION) {
 			/*
@@ -480,7 +480,7 @@ static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
 	 */
 	if (!reuse) {
 		/* remove the skb pointer to mark free. actual freeing is done
-		 * by upper level networking after gro_recieve
+		 * by upper level networking after gro_receive
 		 */
 		adapter->rx_buff_pool[pool].skbuff[index] = NULL;
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 8c27f7ba4adc5..3da9ec49cc745 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -3624,6 +3624,7 @@ static int i40e_set_rxfh_fields(struct net_device *netdev,
 		   ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
 	DECLARE_BITMAP(flow_pctypes, FLOW_PCTYPES_SIZE);
 	u64 i_set, i_setc;
+	u8 flow_id;
 
 	bitmap_zero(flow_pctypes, FLOW_PCTYPES_SIZE);
 
@@ -3707,20 +3708,14 @@ static int i40e_set_rxfh_fields(struct net_device *netdev,
 		return -EINVAL;
 	}
 
-	if (bitmap_weight(flow_pctypes, FLOW_PCTYPES_SIZE)) {
-		u8 flow_id;
+	for_each_set_bit(flow_id, flow_pctypes, FLOW_PCTYPES_SIZE) {
+		i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id)) |
+			 ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id)) << 32);
+		i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc);
 
-		for_each_set_bit(flow_id, flow_pctypes, FLOW_PCTYPES_SIZE) {
-			i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id)) |
-				 ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id)) << 32);
-			i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc);
-
-			i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id),
-					  (u32)i_set);
-			i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id),
-					  (u32)(i_set >> 32));
-			hena |= BIT_ULL(flow_id);
-		}
+		i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id), (u32)i_set);
+		i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id), (u32)(i_set >> 32));
+		hena |= BIT_ULL(flow_id);
 	}
 
 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_hmc.h b/drivers/net/ethernet/intel/i40e/i40e_hmc.h
index 480e3a883cc7a..9677114059197 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_hmc.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_hmc.h
@@ -4,6 +4,8 @@
 #ifndef _I40E_HMC_H_
 #define _I40E_HMC_H_
 
+#include <linux/wordpart.h>
+
 #include "i40e_alloc.h"
 #include "i40e_io.h"
 #include "i40e_register.h"
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index 7925ee152c760..dad001abc9086 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -757,10 +757,13 @@ iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
 		adapter->num_vlan_filters++;
 		iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
 	} else if (f->state == IAVF_VLAN_REMOVE) {
-		/* IAVF_VLAN_REMOVE means that VLAN wasn't yet removed.
-		 * We can safely only change the state here.
+		/* Re-add the filter since we cannot tell whether the
+		 * pending delete has already been processed by the PF.
+		 * A duplicate add is harmless.
 		 */
-		f->state = IAVF_VLAN_ACTIVE;
+		f->state = IAVF_VLAN_ADD;
+		iavf_schedule_aq_request(adapter,
+					 IAVF_FLAG_AQ_ADD_VLAN_FILTER);
 	}
 
 clearout:
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index e7308e381e2f0..ee5e3b1229a8e 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -5028,7 +5028,7 @@ static int ice_init(struct ice_pf *pf)
 	}
 
 	if (pf->hw.mac_type == ICE_MAC_E830) {
-		err = pci_enable_ptm(pf->pdev, NULL);
+		err = pci_enable_ptm(pf->pdev);
 		if (err)
 			dev_dbg(dev, "PCIe PTM not supported by PCIe bus/controller\n");
 	}
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
index 5896b346e5790..9bfd3e79c580d 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
+++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
@@ -258,7 +258,7 @@ enum ice_si_cgu_out_pins {
 };
 
 struct ice_cgu_pin_desc {
-	char *name;
+	const char *name;
 	u8 index;
 	enum dpll_pin_type type;
 	u32 freq_supp_num;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_main.c b/drivers/net/ethernet/intel/idpf/idpf_main.c
index 0dd741dcfcdb2..ab3c409e587bb 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_main.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_main.c
@@ -257,7 +257,7 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_free;
 	}
 
-	err = pci_enable_ptm(pdev, NULL);
+	err = pci_enable_ptm(pdev);
 	if (err)
 		pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n");
 
diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
index a427f05814c1a..17236813965d3 100644
--- a/drivers/net/ethernet/intel/igc/igc.h
+++ b/drivers/net/ethernet/intel/igc/igc.h
@@ -781,6 +781,8 @@ int igc_ptp_hwtstamp_set(struct net_device *netdev,
 			 struct kernel_hwtstamp_config *config,
 			 struct netlink_ext_ack *extack);
 void igc_ptp_tx_hang(struct igc_adapter *adapter);
+void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter,
+				       u16 queue_id);
 void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts);
 void igc_ptp_tx_tstamp_event(struct igc_adapter *adapter);
 
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index b2e8d0c0f827f..8ac16808023cc 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -264,6 +264,13 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 	/* reset next_to_use and next_to_clean */
 	tx_ring->next_to_use = 0;
 	tx_ring->next_to_clean = 0;
+
+	/* Clear any lingering XSK TX timestamp requests */
+	if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) {
+		struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
+
+		igc_ptp_clear_xsk_tx_tstamp_queue(adapter, tx_ring->queue_index);
+	}
 }
 
 /**
@@ -1730,11 +1737,8 @@ static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
 	 * in order to meet this minimum size requirement.
 	 */
-	if (skb->len < 17) {
-		if (skb_padto(skb, 17))
-			return NETDEV_TX_OK;
-		skb->len = 17;
-	}
+	if (skb_put_padto(skb, 17))
+		return NETDEV_TX_OK;
 
 	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
 }
@@ -7137,7 +7141,7 @@ static int igc_probe(struct pci_dev *pdev,
 	if (err)
 		goto err_pci_reg;
 
-	err = pci_enable_ptm(pdev, NULL);
+	err = pci_enable_ptm(pdev);
 	if (err < 0)
 		dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
 
diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c
index 44ee193867661..3d6b2264164af 100644
--- a/drivers/net/ethernet/intel/igc/igc_ptp.c
+++ b/drivers/net/ethernet/intel/igc/igc_ptp.c
@@ -577,6 +577,39 @@ static void igc_ptp_clear_tx_tstamp(struct igc_adapter *adapter)
 	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
 }
 
+/**
+ * igc_ptp_clear_xsk_tx_tstamp_queue - Clear pending XSK TX timestamps for a queue
+ * @adapter: Board private structure
+ * @queue_id: TX queue index to clear timestamps for
+ *
+ * Iterates over all TX timestamp registers and releases any pending
+ * timestamp requests associated with the given TX queue. This is
+ * called when an XDP pool is being disabled to ensure no stale
+ * timestamp references remain.
+ */
+void igc_ptp_clear_xsk_tx_tstamp_queue(struct igc_adapter *adapter, u16 queue_id)
+{
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
+
+	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
+		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
+
+		if (tstamp->buffer_type != IGC_TX_BUFFER_TYPE_XSK)
+			continue;
+		if (tstamp->xsk_queue_index != queue_id)
+			continue;
+		if (!tstamp->xsk_tx_buffer)
+			continue;
+
+		igc_ptp_free_tx_buffer(adapter, tstamp);
+	}
+
+	spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
+}
+
 static void igc_ptp_disable_tx_timestamp(struct igc_adapter *adapter)
 {
 	struct igc_hw *hw = &adapter->hw;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index dce4936708eb4..59a1cee40b436 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -322,10 +322,11 @@ enum ixgbe_ring_state_t {
 	__IXGBE_HANG_CHECK_ARMED,
 	__IXGBE_TX_XDP_RING,
 	__IXGBE_TX_DISABLED,
+	__IXGBE_RING_STATE_NBITS, /* must be last */
 };
 
 #define ring_uses_build_skb(ring) \
-	test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state)
+	test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, (ring)->state)
 
 struct ixgbe_fwd_adapter {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
@@ -336,23 +337,23 @@ struct ixgbe_fwd_adapter {
 };
 
 #define check_for_tx_hang(ring) \
-	test_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)
+	test_bit(__IXGBE_TX_DETECT_HANG, (ring)->state)
 #define set_check_for_tx_hang(ring) \
-	set_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)
+	set_bit(__IXGBE_TX_DETECT_HANG, (ring)->state)
 #define clear_check_for_tx_hang(ring) \
-	clear_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)
+	clear_bit(__IXGBE_TX_DETECT_HANG, (ring)->state)
 #define ring_is_rsc_enabled(ring) \
-	test_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)
+	test_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state)
 #define set_ring_rsc_enabled(ring) \
-	set_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)
+	set_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state)
 #define clear_ring_rsc_enabled(ring) \
-	clear_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)
+	clear_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state)
 #define ring_is_xdp(ring) \
-	test_bit(__IXGBE_TX_XDP_RING, &(ring)->state)
+	test_bit(__IXGBE_TX_XDP_RING, (ring)->state)
 #define set_ring_xdp(ring) \
-	set_bit(__IXGBE_TX_XDP_RING, &(ring)->state)
+	set_bit(__IXGBE_TX_XDP_RING, (ring)->state)
 #define clear_ring_xdp(ring) \
-	clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state)
+	clear_bit(__IXGBE_TX_XDP_RING, (ring)->state)
 struct ixgbe_ring {
 	struct ixgbe_ring *next;	/* pointer to next ring in q_vector */
 	struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */
@@ -364,7 +365,7 @@ struct ixgbe_ring {
 		struct ixgbe_tx_buffer *tx_buffer_info;
 		struct ixgbe_rx_buffer *rx_buffer_info;
 	};
-	unsigned long state;
+	DECLARE_BITMAP(state, __IXGBE_RING_STATE_NBITS);
 	u8 __iomem *tail;
 	dma_addr_t dma;			/* phys. address of descriptor ring */
 	unsigned int size;		/* length in bytes */
@@ -453,7 +454,7 @@ struct ixgbe_ring_feature {
  */
 static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring)
 {
-	if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
+	if (test_bit(__IXGBE_RX_3K_BUFFER, ring->state))
 		return IXGBE_RXBUFFER_3K;
 #if (PAGE_SIZE < 8192)
 	if (ring_uses_build_skb(ring))
@@ -465,7 +466,7 @@ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring)
 static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring)
 {
 #if (PAGE_SIZE < 8192)
-	if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
+	if (test_bit(__IXGBE_RX_3K_BUFFER, ring->state))
 		return 1;
 #endif
 	return 0;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index 87abb36f50c13..1db4bd5cc2ba6 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -976,7 +976,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 		 * can be marked as checksum errors.
 		 */
 		if (adapter->hw.mac.type == ixgbe_mac_82599EB)
-			set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state);
+			set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, ring->state);
 
 #ifdef IXGBE_FCOE
 		if (adapter->netdev->fcoe_mtu) {
@@ -984,7 +984,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 			f = &adapter->ring_feature[RING_F_FCOE];
 			if ((rxr_idx >= f->offset) &&
 			    (rxr_idx < f->offset + f->indices))
-				set_bit(__IXGBE_RX_FCOE, &ring->state);
+				set_bit(__IXGBE_RX_FCOE, ring->state);
 		}
 
 #endif /* IXGBE_FCOE */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index e4101c59074d6..0bc806aaed905 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -968,7 +968,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
 
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		clear_bit(__IXGBE_HANG_CHECK_ARMED,
-			  &adapter->tx_ring[i]->state);
+			  adapter->tx_ring[i]->state);
 }
 
 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
@@ -1011,7 +1011,7 @@ static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
 
 		tc = tx_ring->dcb_tc;
 		if (xoff[tc])
-			clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
+			clear_bit(__IXGBE_HANG_CHECK_ARMED, tx_ring->state);
 	}
 
 	for (i = 0; i < adapter->num_xdp_queues; i++) {
@@ -1019,7 +1019,7 @@ static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
 
 		tc = xdp_ring->dcb_tc;
 		if (xoff[tc])
-			clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);
+			clear_bit(__IXGBE_HANG_CHECK_ARMED, xdp_ring->state);
 	}
 }
 
@@ -1103,11 +1103,11 @@ static bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
 	if (tx_done_old == tx_done && tx_pending)
 		/* make sure it is true for two checks in a row */
 		return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
-					&tx_ring->state);
+					tx_ring->state);
 	/* update completed stats and continue */
 	tx_ring->tx_stats.tx_done_old = tx_done;
 	/* reset the countdown */
-	clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
+	clear_bit(__IXGBE_HANG_CHECK_ARMED, tx_ring->state);
 
 	return false;
 }
@@ -1660,7 +1660,7 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
 {
 	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
 
-	return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
+	return test_bit(__IXGBE_RX_FCOE, ring->state) &&
 	       ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
 		(cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
 			     IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
@@ -1708,7 +1708,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
 		 * checksum errors.
 		 */
 		if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
-		    test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
+		    test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, ring->state))
 			return;
 
 		ring->rx_stats.csum_err++;
@@ -3526,7 +3526,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data)
 			for (i = 0; i < adapter->num_tx_queues; i++) {
 				struct ixgbe_ring *ring = adapter->tx_ring[i];
 				if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
-						       &ring->state))
+						       ring->state))
 					reinit_count++;
 			}
 			if (reinit_count) {
@@ -3952,13 +3952,13 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
 	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
 		ring->atr_sample_rate = adapter->atr_sample_rate;
 		ring->atr_count = 0;
-		set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
+		set_bit(__IXGBE_TX_FDIR_INIT_DONE, ring->state);
 	} else {
 		ring->atr_sample_rate = 0;
 	}
 
 	/* initialize XPS */
-	if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {
+	if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, ring->state)) {
 		struct ixgbe_q_vector *q_vector = ring->q_vector;
 
 		if (q_vector)
@@ -3967,7 +3967,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
 					    ring->queue_index);
 	}
 
-	clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
+	clear_bit(__IXGBE_HANG_CHECK_ARMED, ring->state);
 
 	/* reinitialize tx_buffer_info */
 	memset(ring->tx_buffer_info, 0,
@@ -4173,7 +4173,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
 			srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 		else
 			srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
-	} else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {
+	} else if (test_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state)) {
 		srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 	} else {
 		srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
@@ -4558,7 +4558,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
 		 * higher than the MTU of the PF.
 		 */
 		if (ring_uses_build_skb(ring) &&
-		    !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
+		    !test_bit(__IXGBE_RX_3K_BUFFER, ring->state))
 			rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |
 				  IXGBE_RXDCTL_RLPML_EN;
 #endif
@@ -4733,27 +4733,27 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
 		rx_ring = adapter->rx_ring[i];
 
 		clear_ring_rsc_enabled(rx_ring);
-		clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
-		clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state);
+		clear_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state);
+		clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, rx_ring->state);
 
 		if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
 			set_ring_rsc_enabled(rx_ring);
 
-		if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state))
-			set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
+		if (test_bit(__IXGBE_RX_FCOE, rx_ring->state))
+			set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state);
 
 		if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
 			continue;
 
-		set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state);
+		set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, rx_ring->state);
 
 #if (PAGE_SIZE < 8192)
 		if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
-			set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
+			set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state);
 
 		if (IXGBE_2K_TOO_SMALL_WITH_PADDING ||
 		    (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)))
-			set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
+			set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state);
 #endif
 	}
 }
@@ -7944,10 +7944,10 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
 	if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
 		for (i = 0; i < adapter->num_tx_queues; i++)
 			set_bit(__IXGBE_TX_FDIR_INIT_DONE,
-				&(adapter->tx_ring[i]->state));
+				adapter->tx_ring[i]->state);
 		for (i = 0; i < adapter->num_xdp_queues; i++)
 			set_bit(__IXGBE_TX_FDIR_INIT_DONE,
-				&adapter->xdp_ring[i]->state);
+				adapter->xdp_ring[i]->state);
 		/* re-enable flow director interrupts */
 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
 	} else {
@@ -9488,7 +9488,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
 		ixgbe_tx_csum(tx_ring, first, &ipsec_tx);
 
 	/* add the ATR filter if ATR is on */
-	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
+	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, tx_ring->state))
 		ixgbe_atr(tx_ring, first);
 
 #ifdef IXGBE_FCOE
@@ -9528,7 +9528,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
 		return NETDEV_TX_OK;
 
 	tx_ring = ring ? ring : adapter->tx_ring[skb_get_queue_mapping(skb)];
-	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &tx_ring->state)))
+	if (unlikely(test_bit(__IXGBE_TX_DISABLED, tx_ring->state)))
 		return NETDEV_TX_BUSY;
 
 	return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
@@ -11013,7 +11013,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n,
 	if (unlikely(!ring))
 		return -ENXIO;
 
-	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state)))
+	if (unlikely(test_bit(__IXGBE_TX_DISABLED, ring->state)))
 		return -ENXIO;
 
 	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
@@ -11119,7 +11119,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,
 static void ixgbe_disable_txr(struct ixgbe_adapter *adapter,
 			      struct ixgbe_ring *tx_ring)
 {
-	set_bit(__IXGBE_TX_DISABLED, &tx_ring->state);
+	set_bit(__IXGBE_TX_DISABLED, tx_ring->state);
 	ixgbe_disable_txr_hw(adapter, tx_ring);
 }
 
@@ -11273,9 +11273,9 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring)
 		ixgbe_configure_tx_ring(adapter, xdp_ring);
 	ixgbe_configure_rx_ring(adapter, rx_ring);
 
-	clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state);
+	clear_bit(__IXGBE_TX_DISABLED, tx_ring->state);
 	if (xdp_ring)
-		clear_bit(__IXGBE_TX_DISABLED, &xdp_ring->state);
+		clear_bit(__IXGBE_TX_DISABLED, xdp_ring->state);
 
 	/* Rx/Tx/XDP Tx share the same napi context. */
 	napi_enable(&rx_ring->q_vector->napi);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
index 7b941505a9d02..89f96c463f022 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
@@ -524,7 +524,7 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
 
 	ring = adapter->xdp_ring[qid];
 
-	if (test_bit(__IXGBE_TX_DISABLED, &ring->state))
+	if (test_bit(__IXGBE_TX_DISABLED, ring->state))
 		return -ENETDOWN;
 
 	if (!ring->xsk_pool)
diff --git a/drivers/net/ethernet/intel/libie/fwlog.c b/drivers/net/ethernet/intel/libie/fwlog.c
index 4d0c8370386bc..96bba57c8a5b9 100644
--- a/drivers/net/ethernet/intel/libie/fwlog.c
+++ b/drivers/net/ethernet/intel/libie/fwlog.c
@@ -433,17 +433,21 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf,
 	module = libie_find_module_by_dentry(fwlog->debugfs_modules, dentry);
 	if (module < 0) {
 		dev_info(dev, "unknown module\n");
-		return -EINVAL;
+		count = -EINVAL;
+		goto free_cmd_buf;
 	}
 
 	cnt = sscanf(cmd_buf, "%s", user_val);
-	if (cnt != 1)
-		return -EINVAL;
+	if (cnt != 1) {
+		count = -EINVAL;
+		goto free_cmd_buf;
+	}
 
 	log_level = sysfs_match_string(libie_fwlog_level_string, user_val);
 	if (log_level < 0) {
 		dev_info(dev, "unknown log level '%s'\n", user_val);
-		return -EINVAL;
+		count = -EINVAL;
+		goto free_cmd_buf;
 	}
 
 	if (module != LIBIE_AQC_FW_LOG_ID_MAX) {
@@ -458,6 +462,9 @@ libie_debugfs_module_write(struct file *filp, const char __user *buf,
 			fwlog->cfg.module_entries[i].log_level = log_level;
 	}
 
+free_cmd_buf:
+	kfree(cmd_buf);
+
 	return count;
 }
 
@@ -515,23 +522,31 @@ libie_debugfs_nr_messages_write(struct file *filp, const char __user *buf,
 		return PTR_ERR(cmd_buf);
 
 	ret = sscanf(cmd_buf, "%s", user_val);
-	if (ret != 1)
-		return -EINVAL;
+	if (ret != 1) {
+		count = -EINVAL;
+		goto free_cmd_buf;
+	}
 
 	ret = kstrtos16(user_val, 0, &nr_messages);
-	if (ret)
-		return ret;
+	if (ret) {
+		count = ret;
+		goto free_cmd_buf;
+	}
 
 	if (nr_messages < LIBIE_AQC_FW_LOG_MIN_RESOLUTION ||
 	    nr_messages > LIBIE_AQC_FW_LOG_MAX_RESOLUTION) {
 		dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n",
 			nr_messages, LIBIE_AQC_FW_LOG_MIN_RESOLUTION,
 			LIBIE_AQC_FW_LOG_MAX_RESOLUTION);
-		return -EINVAL;
+		count = -EINVAL;
+		goto free_cmd_buf;
 	}
 
 	fwlog->cfg.log_resolution = nr_messages;
 
+free_cmd_buf:
+	kfree(cmd_buf);
+
 	return count;
 }
 
@@ -588,8 +603,10 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf,
 		return PTR_ERR(cmd_buf);
 
 	ret = sscanf(cmd_buf, "%s", user_val);
-	if (ret != 1)
-		return -EINVAL;
+	if (ret != 1) {
+		ret = -EINVAL;
+		goto free_cmd_buf;
+	}
 
 	ret = kstrtobool(user_val, &enable);
 	if (ret)
@@ -624,6 +641,8 @@ libie_debugfs_enable_write(struct file *filp, const char __user *buf,
 	 */
 	if (WARN_ON(ret != (ssize_t)count && ret >= 0))
 		ret = -EIO;
+free_cmd_buf:
+	kfree(cmd_buf);
 
 	return ret;
 }
@@ -682,8 +701,10 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf,
 		return PTR_ERR(cmd_buf);
 
 	ret = sscanf(cmd_buf, "%s", user_val);
-	if (ret != 1)
-		return -EINVAL;
+	if (ret != 1) {
+		ret = -EINVAL;
+		goto free_cmd_buf;
+	}
 
 	index = sysfs_match_string(libie_fwlog_log_size, user_val);
 	if (index < 0) {
@@ -712,6 +733,8 @@ libie_debugfs_log_size_write(struct file *filp, const char __user *buf,
 	 */
 	if (WARN_ON(ret != (ssize_t)count && ret >= 0))
 		ret = -EIO;
+free_cmd_buf:
+	kfree(cmd_buf);
 
 	return ret;
 }
diff --git a/drivers/net/ethernet/litex/litex_liteeth.c b/drivers/net/ethernet/litex/litex_liteeth.c
index 829a4b828f8e2..108d0a0db2069 100644
--- a/drivers/net/ethernet/litex/litex_liteeth.c
+++ b/drivers/net/ethernet/litex/litex_liteeth.c
@@ -232,12 +232,13 @@ static void liteeth_setup_slots(struct liteeth *priv)
 
 static int liteeth_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
 	struct net_device *netdev;
 	void __iomem *buf_base;
 	struct liteeth *priv;
 	int irq, err;
 
-	netdev = devm_alloc_etherdev(&pdev->dev, sizeof(*priv));
+	netdev = devm_alloc_etherdev(dev, sizeof(*priv));
 	if (!netdev)
 		return -ENOMEM;
 
@@ -246,9 +247,9 @@ static int liteeth_probe(struct platform_device *pdev)
 
 	priv = netdev_priv(netdev);
 	priv->netdev = netdev;
-	priv->dev = &pdev->dev;
+	priv->dev = dev;
 
-	netdev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
+	netdev->tstats = devm_netdev_alloc_pcpu_stats(dev,
 						      struct pcpu_sw_netstats);
 	if (!netdev->tstats)
 		return -ENOMEM;
@@ -276,15 +277,15 @@ static int liteeth_probe(struct platform_device *pdev)
 	priv->tx_base = buf_base + priv->num_rx_slots * priv->slot_size;
 	priv->tx_slot = 0;
 
-	err = of_get_ethdev_address(pdev->dev.of_node, netdev);
+	err = of_get_ethdev_address(dev->of_node, netdev);
 	if (err)
 		eth_hw_addr_random(netdev);
 
 	netdev->netdev_ops = &liteeth_netdev_ops;
 
-	err = register_netdev(netdev);
+	err = devm_register_netdev(dev, netdev);
 	if (err) {
-		dev_err(&pdev->dev, "Failed to register netdev %d\n", err);
+		dev_err(dev, "Failed to register netdev %d\n", err);
 		return err;
 	}
 
@@ -294,13 +295,6 @@ static int liteeth_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static void liteeth_remove(struct platform_device *pdev)
-{
-	struct net_device *netdev = platform_get_drvdata(pdev);
-
-	unregister_netdev(netdev);
-}
-
 static const struct of_device_id liteeth_of_match[] = {
 	{ .compatible = "litex,liteeth" },
 	{ }
@@ -309,7 +303,6 @@ MODULE_DEVICE_TABLE(of, liteeth_of_match);
 
 static struct platform_driver liteeth_driver = {
 	.probe = liteeth_probe,
-	.remove = liteeth_remove,
 	.driver = {
 		.name = DRV_NAME,
 		.of_match_table = liteeth_of_match,
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 9ba4aef7080c0..0c061fb0ed072 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -5620,6 +5620,8 @@ static int mvneta_probe(struct platform_device *pdev)
 	}
 
 	err = of_get_ethdev_address(dn, dev);
+	if (err == -EPROBE_DEFER)
+		goto err_free_stats;
 	if (!err) {
 		mac_from = "device tree";
 	} else {
@@ -5755,6 +5757,7 @@ static int mvneta_probe(struct platform_device *pdev)
 				       1 << pp->id);
 		mvneta_bm_put(pp->bm_priv);
 	}
+err_free_stats:
 	free_percpu(pp->stats);
 err_free_ports:
 	free_percpu(pp->ports);
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index d1b8650cb4b48..f442b874bb593 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -5016,7 +5016,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu)
 	if (priv->percpu_pools)
 		numbufs = port->nrxqs * 2;
 
-	if (change_percpu)
+	if (change_percpu && priv->global_tx_fc)
 		mvpp2_bm_pool_update_priv_fc(priv, false);
 
 	for (i = 0; i < numbufs; i++)
@@ -5041,7 +5041,7 @@ static int mvpp2_bm_switch_buffers(struct mvpp2 *priv, bool percpu)
 			mvpp2_open(port->dev);
 	}
 
-	if (change_percpu)
+	if (change_percpu && priv->global_tx_fc)
 		mvpp2_bm_pool_update_priv_fc(priv, true);
 
 	return 0;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile
index 244de500963ed..91b7d6e96a614 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile
+++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile
@@ -13,4 +13,4 @@ rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \
 		  rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \
 		  rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o \
 		  rvu_rep.o cn20k/mbox_init.o cn20k/nix.o cn20k/debugfs.o \
-		  cn20k/npa.o
+		  cn20k/npa.o cn20k/npc.o
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
index 6000795823a36..4f33a816bc7a1 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
@@ -1822,7 +1822,7 @@ static int cgx_lmac_exit(struct cgx *cgx)
 			continue;
 		cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false);
 		cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true);
-		kfree(lmac->mac_to_index_bmap.bmap);
+		rvu_free_bitmap(&lmac->mac_to_index_bmap);
 		rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap);
 		rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap);
 		kfree(lmac->name);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
index 498968bf4cf59..3debf2fae1a48 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
@@ -11,7 +11,280 @@
 #include <linux/pci.h>
 
 #include "struct.h"
+#include "rvu.h"
 #include "debugfs.h"
+#include "cn20k/npc.h"
+
+static int npc_mcam_layout_show(struct seq_file *s, void *unused)
+{
+	int i, j, sbd, idx0, idx1, vidx0, vidx1;
+	struct npc_priv_t *npc_priv;
+	char buf0[32], buf1[32];
+	struct npc_subbank *sb;
+	unsigned int bw0, bw1;
+	bool v0, v1;
+	int pf1, pf2;
+	bool e0, e1;
+	void *map;
+
+	npc_priv = s->private;
+
+	sbd = npc_priv->subbank_depth;
+
+	for (i = npc_priv->num_subbanks - 1; i >= 0; i--) {
+		sb = &npc_priv->sb[i];
+		mutex_lock(&sb->lock);
+
+		if (sb->flags & NPC_SUBBANK_FLAG_FREE)
+			goto next;
+
+		bw0 = bitmap_weight(sb->b0map, npc_priv->subbank_depth);
+		if (sb->key_type == NPC_MCAM_KEY_X4) {
+			seq_printf(s, "\n\nsubbank:%u, x4, free=%u, used=%u\n",
+				   sb->idx, sb->free_cnt, bw0);
+
+			for (j = sbd - 1; j >= 0; j--) {
+				if (!test_bit(j, sb->b0map))
+					continue;
+
+				idx0 = sb->b0b + j;
+				map = xa_load(&npc_priv->xa_idx2pf_map, idx0);
+				pf1 = xa_to_value(map);
+
+				map = xa_load(&npc_priv->xa_idx2vidx_map, idx0);
+				if (map) {
+					vidx0 = xa_to_value(map);
+					snprintf(buf0, sizeof(buf0),
+						 "v:%u", vidx0);
+				}
+
+				seq_printf(s, "\t%u(%#x) %s\n", idx0, pf1,
+					   map ? buf0 : " ");
+			}
+			goto next;
+		}
+
+		bw1 = bitmap_weight(sb->b1map, npc_priv->subbank_depth);
+		seq_printf(s, "\n\nsubbank:%u, x2, free=%u, used=%u\n",
+			   sb->idx, sb->free_cnt, bw0 + bw1);
+		seq_printf(s, "bank1(%u)\t\tbank0(%u)\n", bw1, bw0);
+
+		for (j = sbd - 1; j >= 0; j--) {
+			e0 = test_bit(j, sb->b0map);
+			e1 = test_bit(j, sb->b1map);
+
+			if (!e1 && !e0)
+				continue;
+
+			if (e1 && e0) {
+				idx0 = sb->b0b + j;
+				map = xa_load(&npc_priv->xa_idx2pf_map, idx0);
+				pf1 = xa_to_value(map);
+
+				map = xa_load(&npc_priv->xa_idx2vidx_map, idx0);
+				v0 = !!map;
+				if (v0) {
+					vidx0 = xa_to_value(map);
+					snprintf(buf0, sizeof(buf0), "v:%05u",
+						 vidx0);
+				}
+
+				idx1 = sb->b1b + j;
+				map = xa_load(&npc_priv->xa_idx2pf_map, idx1);
+				pf2 = xa_to_value(map);
+
+				map = xa_load(&npc_priv->xa_idx2vidx_map, idx1);
+				v1 = !!map;
+				if (v1) {
+					vidx1 = xa_to_value(map);
+					snprintf(buf1, sizeof(buf1), "v:%05u",
+						 vidx1);
+				}
+
+				seq_printf(s, "%05u(%#x) %s\t\t%05u(%#x) %s\n",
+					   idx1, pf2, v1 ? buf1 : "       ",
+					   idx0, pf1, v0 ? buf0 : "       ");
+
+				continue;
+			}
+
+			if (e0) {
+				idx0 = sb->b0b + j;
+				map = xa_load(&npc_priv->xa_idx2pf_map, idx0);
+				pf1 = xa_to_value(map);
+
+				map = xa_load(&npc_priv->xa_idx2vidx_map, idx0);
+				if (map) {
+					vidx0 = xa_to_value(map);
+					snprintf(buf0, sizeof(buf0), "v:%05u",
+						 vidx0);
+				}
+
+				seq_printf(s, "\t\t   \t\t%05u(%#x) %s\n", idx0,
+					   pf1, map ? buf0 : " ");
+				continue;
+			}
+
+			idx1 = sb->b1b + j;
+			map = xa_load(&npc_priv->xa_idx2pf_map, idx1);
+			pf1 = xa_to_value(map);
+			map = xa_load(&npc_priv->xa_idx2vidx_map, idx1);
+			if (map) {
+				vidx1 = xa_to_value(map);
+				snprintf(buf1, sizeof(buf1), "v:%05u", vidx1);
+			}
+
+			seq_printf(s, "%05u(%#x) %s\n", idx1, pf1,
+				   map ? buf1 : " ");
+		}
+next:
+		mutex_unlock(&sb->lock);
+	}
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(npc_mcam_layout);
+
+static int npc_mcam_default_show(struct seq_file *s, void *unused)
+{
+	struct npc_priv_t *npc_priv;
+	unsigned long index;
+	u16 ptr[4], pcifunc;
+	struct rvu *rvu;
+	int rc, i;
+	void *map;
+
+	npc_priv = npc_priv_get();
+	rvu = s->private;
+
+	seq_puts(s, "\npcifunc\tBcast\tmcast\tpromisc\tucast\n");
+
+	xa_for_each(&npc_priv->xa_pf_map, index, map) {
+		pcifunc = index;
+
+		for (i = 0; i < ARRAY_SIZE(ptr); i++)
+			ptr[i] = USHRT_MAX;
+
+		rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0],
+						 &ptr[1], &ptr[2], &ptr[3]);
+		if (rc)
+			continue;
+
+		seq_printf(s, "%#x\t", pcifunc);
+		for (i = 0; i < ARRAY_SIZE(ptr); i++) {
+			if (ptr[i] != USHRT_MAX)
+				seq_printf(s, "%u\t", ptr[i]);
+			else
+				seq_puts(s, "\t");
+		}
+		seq_puts(s, "\n");
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(npc_mcam_default);
+
+static int npc_vidx2idx_map_show(struct seq_file *s, void *unused)
+{
+	struct npc_priv_t *npc_priv;
+	unsigned long index, start;
+	struct xarray *xa;
+	void *map;
+
+	npc_priv = s->private;
+	start = npc_priv->bank_depth * 2;
+	xa = &npc_priv->xa_vidx2idx_map;
+
+	seq_puts(s, "\nvidx\tmcam_idx\n");
+
+	xa_for_each_start(xa, index, map, start)
+		seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map));
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(npc_vidx2idx_map);
+
+static int npc_idx2vidx_map_show(struct seq_file *s, void *unused)
+{
+	struct npc_priv_t *npc_priv;
+	unsigned long index;
+	struct xarray *xa;
+	void *map;
+
+	npc_priv = s->private;
+	xa = &npc_priv->xa_idx2vidx_map;
+
+	seq_puts(s, "\nmidx\tvidx\n");
+
+	xa_for_each(xa, index, map)
+		seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map));
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(npc_idx2vidx_map);
+
+static int npc_defrag_show(struct seq_file *s, void *unused)
+{
+	struct npc_defrag_show_node *node;
+	struct npc_priv_t *npc_priv;
+	u16 sbd, bdm;
+
+	npc_priv = s->private;
+	bdm = npc_priv->bank_depth - 1;
+	sbd = npc_priv->subbank_depth;
+
+	seq_puts(s, "\nold(sb)   ->    new(sb)\t\tvidx\n");
+
+	mutex_lock(&npc_priv->lock);
+	list_for_each_entry(node, &npc_priv->defrag_lh, list)
+		seq_printf(s, "%u(%u)\t%u(%u)\t%u\n", node->old_midx,
+			   (node->old_midx & bdm) / sbd,
+			   node->new_midx,
+			   (node->new_midx & bdm) / sbd,
+			   node->vidx);
+	mutex_unlock(&npc_priv->lock);
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(npc_defrag);
+
+int npc_cn20k_debugfs_init(struct rvu *rvu)
+{
+	struct npc_priv_t *npc_priv = npc_priv_get();
+	struct dentry *npc_dentry;
+
+	npc_dentry = debugfs_create_file("mcam_layout", 0444, rvu->rvu_dbg.npc,
+					 npc_priv, &npc_mcam_layout_fops);
+
+	if (!npc_dentry)
+		return -EFAULT;
+
+	npc_dentry = debugfs_create_file("mcam_default", 0444, rvu->rvu_dbg.npc,
+					 rvu, &npc_mcam_default_fops);
+
+	if (!npc_dentry)
+		return -EFAULT;
+
+	npc_dentry = debugfs_create_file("vidx2idx", 0444, rvu->rvu_dbg.npc,
+					 npc_priv, &npc_vidx2idx_map_fops);
+	if (!npc_dentry)
+		return -EFAULT;
+
+	npc_dentry = debugfs_create_file("idx2vidx", 0444, rvu->rvu_dbg.npc,
+					 npc_priv, &npc_idx2vidx_map_fops);
+	if (!npc_dentry)
+		return -EFAULT;
+
+	npc_dentry = debugfs_create_file("defrag", 0444, rvu->rvu_dbg.npc,
+					 npc_priv, &npc_defrag_fops);
+	if (!npc_dentry)
+		return -EFAULT;
+
+	return 0;
+}
+
+void npc_cn20k_debugfs_deinit(struct rvu *rvu)
+{
+	debugfs_remove_recursive(rvu->rvu_dbg.npc);
+}
 
 void print_nix_cn20k_sq_ctx(struct seq_file *m,
 			    struct nix_cn20k_sq_ctx_s *sq_ctx)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h
index a2e3a2cd6edb3..0c5f058836661 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h
@@ -16,6 +16,9 @@
 #include "struct.h"
 #include "../mbox.h"
 
+int npc_cn20k_debugfs_init(struct rvu *rvu);
+void npc_cn20k_debugfs_deinit(struct rvu *rvu);
+
 void print_nix_cn20k_sq_ctx(struct seq_file *m,
 			    struct nix_cn20k_sq_ctx_s *sq_ctx);
 void print_nix_cn20k_cq_ctx(struct seq_file *m,
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c
index bd3aab7770ddf..71401dec0d776 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c
@@ -397,6 +397,12 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf,
 	if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))
 		return 0;
 
+	/* sanity check */
+	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_NIXX_CFG(0) |
+			 (RVU_AFPF << 16));
+	if (!cfg)
+		return 0;
+
 	ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3);
 	/* Alloc memory for CQINT's HW contexts */
 	cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
@@ -420,5 +426,16 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf,
 	rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf),
 		    (u64)pfvf->nix_qints_ctx->iova);
 
+	rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL, RVU_AF_BAR2_PFID);
+	rvu_write64(rvu, BLKADDR_NIX0,
+		    AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), ALTAF_RDY);
+	/* wait for ack */
+	err = rvu_poll_reg(rvu, BLKADDR_NIX0,
+			   AF_BAR2_ALIASX(0, NIX_GINT_INT), ALTAF_RDY, true);
+	if (err)
+		rvu->altaf_ready = false;
+	else
+		rvu->altaf_ready = true;
+
 	return 0;
 }
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
new file mode 100644
index 0000000000000..7291fdb89b03f
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
@@ -0,0 +1,4522 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell RVU Admin Function driver
+ *
+ * Copyright (C) 2026 Marvell.
+ *
+ */
+#include <linux/xarray.h>
+#include <linux/bitfield.h>
+
+#include "rvu.h"
+#include "npc.h"
+#include "npc_profile.h"
+#include "rvu_npc_hash.h"
+#include "rvu_npc.h"
+#include "cn20k/npc.h"
+#include "cn20k/reg.h"
+#include "rvu_npc_fs.h"
+
+static struct npc_priv_t npc_priv = {
+	.num_banks = MAX_NUM_BANKS,
+};
+
+static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = {
+	[NPC_MCAM_KEY_DYN] = "DYNAMIC",
+	[NPC_MCAM_KEY_X2] = "X2",
+	[NPC_MCAM_KEY_X4] = "X4",
+};
+
+static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = {
+	[NPC_DFT_RULE_PROMISC_ID] = "Promisc",
+	[NPC_DFT_RULE_MCAST_ID] = "Mcast",
+	[NPC_DFT_RULE_BCAST_ID] = "Bcast",
+	[NPC_DFT_RULE_UCAST_ID] = "Ucast",
+};
+
+#define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs)		\
+		     (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \
+		     ((key_ofs) & 0x3F))
+
+#define NPC_DFT_RULE_ID_MK(pcifunc, id) \
+	((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id))
+
+#define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \
+	FIELD_GET(GENMASK_ULL(15, 0), rid)
+
+#define NPC_DFT_RULE_ID_2_ID(rid) \
+	FIELD_GET(GENMASK_ULL(31, 16), rid)
+
+#define NPC_DFT_RULE_PRIO 127
+
+static const char cn20k_def_pfl_name[] = "default";
+
+static struct npc_mcam_kex_extr npc_mkex_extr_default = {
+	.mkex_sign = MKEX_CN20K_SIGN,
+	.name = "default",
+	.kpu_version = NPC_KPU_PROFILE_VER,
+	.keyx_cfg = {
+		/* nibble: LA..LE (ltype only) + Error code + Channel */
+		[NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) |
+			NPC_PARSE_NIBBLE_INTF_RX |
+				 NPC_CN20K_PARSE_NIBBLE_ERRCODE,
+
+		/* nibble: LA..LE (ltype only) */
+		[NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) |
+			NPC_CN20K_PARSE_NIBBLE_INTF_TX,
+	},
+	.intf_extr_lid = {
+	/* Default RX MCAM KEX profile */
+	[NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
+			  NPC_LID_LC, NPC_LID_LC, NPC_LID_LD },
+	[NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
+			  NPC_LID_LC, NPC_LID_LD },
+	},
+	.intf_extr_lt = {
+	/* Default RX MCAM KEX profile */
+	[NIX_INTF_RX] = {
+		[0] = {
+			/* Layer A: Ethernet: */
+			[NPC_LT_LA_ETHER] =
+				/* DMAC: 6 bytes, KW1[63:15] */
+				KEX_EXTR_CFG(0x05, 0x0, 0x1,
+					     NPC_KEXOF_DMAC + 1),
+			[NPC_LT_LA_CPT_HDR] =
+				/* DMAC: 6 bytes, KW1[63:15] */
+				KEX_EXTR_CFG(0x05, 0x0, 0x1,
+					     NPC_KEXOF_DMAC + 1),
+		},
+		[1] = {
+			/* Layer A: Ethernet: */
+			[NPC_LT_LA_ETHER] =
+				/* Ethertype: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
+			[NPC_LT_LA_CPT_HDR] =
+				/* Ethertype: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
+		},
+		[2] = {
+			/* Layer B: Single VLAN (CTAG) */
+			[NPC_LT_LB_CTAG] =
+				/* CTAG VLAN: 2 bytes, KW1[15:0] */
+				KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
+			/* Layer B: Stacked VLAN (STAG|QinQ) */
+			[NPC_LT_LB_STAG_QINQ] =
+				/* Outer VLAN: 2 bytes, KW1[15:0] */
+				KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
+			[NPC_LT_LB_FDSA] =
+				/* SWITCH PORT: 1 byte, KW1[7:0] */
+				KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8),
+		},
+		[3] = {
+			[NPC_LT_LB_CTAG] =
+				/* Ethertype: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
+			[NPC_LT_LB_STAG_QINQ] =
+				/* Ethertype: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6),
+			[NPC_LT_LB_FDSA] =
+				/* Ethertype: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
+		},
+		[4] = {
+			/* Layer C: IPv4 */
+			[NPC_LT_LC_IP] =
+				/* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */
+				KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11),
+			/* Layer C: IPv6 */
+			[NPC_LT_LC_IP6] =
+				/* Everything up to SADDR: 8 bytes, KW3[7:0],
+				 * KW2[63:8]
+				 */
+				KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11),
+		},
+		[5] = {
+			[NPC_LT_LC_IP] =
+				/* TOS: 1 byte, KW2[7:0] */
+				KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10),
+		},
+		[6] = {
+			/* Layer D:UDP */
+			[NPC_LT_LD_UDP] =
+				/* SPORT+DPORT: 4 bytes, KW3[39:8] */
+				KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
+			/* Layer D:TCP */
+			[NPC_LT_LD_TCP] =
+				/* SPORT+DPORT: 4 bytes, KW3[39:8] */
+				KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
+		},
+	},
+	/* Default TX MCAM KEX profile */
+	[NIX_INTF_TX] = {
+		[0] = {
+			/* Layer A: NIX_INST_HDR_S + Ethernet */
+			/* NIX appends 8 bytes of NIX_INST_HDR_S at the
+			 * start of each TX packet supplied to NPC.
+			 */
+			[NPC_LT_LA_IH_NIX_ETHER] =
+				/* PF_FUNC: 2B , KW0 [47:32] */
+				KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
+			/* Layer A: HiGig2: */
+			[NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
+				/* PF_FUNC: 2B , KW0 [47:32] */
+				KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
+		},
+		[1] = {
+			[NPC_LT_LA_IH_NIX_ETHER] =
+				/* SQ_ID 3 bytes, KW1[63:16] */
+				KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa),
+			[NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
+				/* VID: 2 bytes, KW1[31:16] */
+				KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa),
+		},
+		[2] = {
+			/* Layer B: Single VLAN (CTAG) */
+			[NPC_LT_LB_CTAG] =
+				/* CTAG VLAN[2..3] KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
+			/* Layer B: Stacked VLAN (STAG|QinQ) */
+			[NPC_LT_LB_STAG_QINQ] =
+				/* Outer VLAN: 2 bytes, KW0[63:48] */
+				KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
+		},
+		[3] = {
+			[NPC_LT_LB_CTAG] =
+				/* CTAG VLAN[2..3] KW1[15:0] */
+				KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8),
+			[NPC_LT_LB_STAG_QINQ] =
+				/* Outer VLAN: 2 Bytes, KW1[15:0] */
+				KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8),
+		},
+		[4] = {
+			/* Layer C: IPv4 */
+			[NPC_LT_LC_IP] =
+				/* SIP+DIP: 8 bytes, KW2[63:0] */
+				KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10),
+			/* Layer C: IPv6 */
+			[NPC_LT_LC_IP6] =
+				/* Everything up to SADDR: 8 bytes, KW2[63:0] */
+				KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10),
+		},
+		[5] = {
+			/* Layer D:UDP */
+			[NPC_LT_LD_UDP] =
+				/* SPORT+DPORT: 4 bytes, KW3[31:0] */
+				KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
+			/* Layer D:TCP */
+			[NPC_LT_LD_TCP] =
+				/* SPORT+DPORT: 4 bytes, KW3[31:0] */
+				KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
+		},
+	},
+	},
+};
+
+struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void)
+{
+	return &npc_mkex_extr_default;
+}
+
+static u16 npc_idx2vidx(u16 idx)
+{
+	unsigned long index;
+	void *map;
+	u16 vidx;
+	int val;
+
+	vidx = idx;
+	index = idx;
+
+	map = xa_load(&npc_priv.xa_idx2vidx_map, index);
+	if (!map)
+		goto done;
+
+	val = xa_to_value(map);
+	if (val == -1)
+		goto done;
+
+	vidx = val;
+
+done:
+	return vidx;
+}
+
+static bool npc_is_vidx(u16 vidx)
+{
+	return vidx >= npc_priv.bank_depth * 2;
+}
+
+static u16 npc_vidx2idx(u16 vidx)
+{
+	unsigned long sentinel = (unsigned long)-1;
+	unsigned long index;
+	void *map;
+	int val;
+	u16 idx;
+
+	idx = vidx;
+	index = vidx;
+
+	map = xa_load(&npc_priv.xa_vidx2idx_map, index);
+	if (!map)
+		goto done;
+
+	val = xa_to_value(map);
+	if (val == sentinel)
+		goto done;
+
+	idx = val;
+
+done:
+	return idx;
+}
+
+u16 npc_cn20k_vidx2idx(u16 idx)
+{
+	if (!npc_priv.init_done)
+		return idx;
+
+	if (!npc_is_vidx(idx))
+		return idx;
+
+	return npc_vidx2idx(idx);
+}
+
+u16 npc_cn20k_idx2vidx(u16 idx)
+{
+	if (!npc_priv.init_done)
+		return idx;
+
+	if (npc_is_vidx(idx))
+		return idx;
+
+	return npc_idx2vidx(idx);
+}
+
+static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx)
+{
+	u16 mcam_idx;
+	void *map;
+
+	if (!npc_is_vidx(vidx)) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) does not map to proper mcam idx\n",
+			__func__, vidx);
+		return -ESRCH;
+	}
+
+	mcam_idx = npc_vidx2idx(vidx);
+
+	map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) does not map to proper mcam idx\n",
+			__func__, vidx);
+		return -ESRCH;
+	}
+
+	map = xa_erase(&npc_priv.xa_idx2vidx_map, mcam_idx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) is not valid\n",
+			__func__, vidx);
+		return -ESRCH;
+	}
+
+	if (old_midx)
+		*old_midx = mcam_idx;
+
+	return 0;
+}
+
+static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx)
+{
+	u16 old_midx;
+	void *map;
+	int rc;
+
+	if (!npc_is_vidx(vidx)) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) does not map to proper mcam idx\n",
+			__func__, vidx);
+		return -ESRCH;
+	}
+
+	map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) could not be deleted from vidx2idx map\n",
+			__func__, vidx);
+		return -ESRCH;
+	}
+
+	old_midx = xa_to_value(map);
+
+	rc = xa_insert(&npc_priv.xa_vidx2idx_map, vidx,
+		       xa_mk_value(new_midx), GFP_KERNEL);
+	if (rc) {
+		dev_err(rvu->dev,
+			"%s: vidx(%u) cannot be added to vidx2idx map\n",
+			__func__, vidx);
+		goto fail1;
+	}
+
+	map = xa_erase(&npc_priv.xa_idx2vidx_map, old_midx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
+			__func__, old_midx, vidx);
+		rc = -ESRCH;
+		goto fail2;
+	}
+
+	rc = xa_insert(&npc_priv.xa_idx2vidx_map, new_midx,
+		       xa_mk_value(vidx), GFP_KERNEL);
+	if (rc) {
+		dev_err(rvu->dev,
+			"%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
+			__func__, new_midx, vidx);
+		goto fail3;
+	}
+
+	return 0;
+
+fail3:
+	/* Restore vidx at old_midx location */
+	if (xa_insert(&npc_priv.xa_idx2vidx_map, old_midx,
+		      xa_mk_value(vidx), GFP_KERNEL))
+		dev_err(rvu->dev,
+			"%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n",
+			__func__, old_midx, vidx);
+fail2:
+	/* Erase new_midx inserted at vidx */
+	if (!xa_erase(&npc_priv.xa_vidx2idx_map, vidx))
+		dev_err(rvu->dev,
+			"%s: Failed to roll back vidx2idx vidx=%u\n",
+			__func__, vidx);
+
+fail1:
+	/* Restore old_midx at vidx location */
+	if (xa_insert(&npc_priv.xa_vidx2idx_map, vidx,
+		      xa_mk_value(old_midx), GFP_KERNEL))
+		dev_err(rvu->dev,
+			"%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n",
+			__func__, old_midx, vidx);
+
+	return rc;
+}
+
+static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc,
+				   u16 *vidx)
+{
+	int rc, max, min;
+	u32 id;
+
+	/* Virtual index start from maximum mcam index + 1 */
+	max = npc_priv.bank_depth * 2 * 2 - 1;
+	min = npc_priv.bank_depth * 2;
+
+	rc = xa_alloc(&npc_priv.xa_vidx2idx_map, &id,
+		      xa_mk_value(mcam_idx),
+		      XA_LIMIT(min, max), GFP_KERNEL);
+	if (rc) {
+		dev_err(rvu->dev,
+			"%s: Failed to add to vidx2idx map (%u)\n",
+			__func__, mcam_idx);
+		goto fail1;
+	}
+
+	rc = xa_insert(&npc_priv.xa_idx2vidx_map, mcam_idx,
+		       xa_mk_value(id), GFP_KERNEL);
+	if (rc) {
+		dev_err(rvu->dev,
+			"%s: Failed to add to idx2vidx map (%u)\n",
+			__func__, mcam_idx);
+		goto fail2;
+	}
+
+	if (vidx)
+		*vidx = id;
+
+	return 0;
+
+fail2:
+	xa_erase(&npc_priv.xa_vidx2idx_map, id);
+fail1:
+	return rc;
+}
+
+static void npc_config_kpmcam(struct rvu *rvu, int blkaddr,
+			      const struct npc_kpu_profile_cam *kpucam,
+			      int kpm, int entry)
+{
+	struct npc_kpu_cam cam0 = {0};
+	struct npc_kpu_cam cam1 = {0};
+
+	cam1.state = kpucam->state & kpucam->state_mask;
+	cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask;
+	cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask;
+	cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask;
+
+	cam0.state = ~kpucam->state & kpucam->state_mask;
+	cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask;
+	cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask;
+	cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask;
+
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1);
+}
+
+static void
+npc_config_kpmaction(struct rvu *rvu, int blkaddr,
+		     const struct npc_kpu_profile_action *kpuaction,
+		     int kpm, int entry, bool pkind)
+{
+	struct npc_kpm_action0 action0 = {0};
+	struct npc_kpu_action1 action1 = {0};
+	u64 reg;
+
+	action1.errlev = kpuaction->errlev;
+	action1.errcode = kpuaction->errcode;
+	action1.dp0_offset = kpuaction->dp0_offset;
+	action1.dp1_offset = kpuaction->dp1_offset;
+	action1.dp2_offset = kpuaction->dp2_offset;
+
+	if (pkind)
+		reg = NPC_AF_PKINDX_ACTION1(entry);
+	else
+		reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry);
+
+	rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1);
+
+	action0.byp_count = kpuaction->bypass_count & 0x7;
+	action0.capture_ena = kpuaction->cap_ena & 1;
+	action0.parse_done = kpuaction->parse_done & 1;
+	action0.next_state = kpuaction->next_state & 0xf;
+	action0.capture_lid = kpuaction->lid & 0x7;
+
+	/* Parser functionality will work correctly even though
+	 * upper flag bits are silently discarded
+	 */
+	action0.capture_ltype = kpuaction->ltype & 0xf;
+	action0.capture_flags = kpuaction->flags & 0xf;
+	action0.ptr_advance = kpuaction->ptr_advance;
+
+	action0.var_len_offset = kpuaction->offset;
+	action0.var_len_mask = kpuaction->mask;
+	action0.var_len_right = kpuaction->right & 1;
+	action0.var_len_shift = kpuaction->shift & 1;
+
+	if (pkind)
+		reg = NPC_AF_PKINDX_ACTION0(entry);
+	else
+		reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry);
+
+	rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0);
+}
+
+static void
+npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr,
+			       int kpm, int start_entry,
+			       const struct npc_kpu_profile *profile)
+{
+	int entry, num_entries, max_entries;
+	u64 idx;
+
+	if (profile->cam_entries != profile->action_entries) {
+		dev_err(rvu->dev,
+			"kpm%d: CAM and action entries [%d != %d] not equal\n",
+			kpm, profile->cam_entries, profile->action_entries);
+
+		WARN(1, "Fatal error\n");
+		return;
+	}
+
+	max_entries = rvu->hw->npc_kpu_entries / 2;
+	entry = start_entry;
+	/* Program CAM match entries for previous kpm extracted data */
+	num_entries = min_t(int, profile->cam_entries, max_entries);
+	for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
+		npc_config_kpmcam(rvu, blkaddr, &profile->cam[idx],
+				  kpm, entry);
+
+	entry = start_entry;
+	/* Program this kpm's actions */
+	num_entries = min_t(int, profile->action_entries, max_entries);
+	for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
+		npc_config_kpmaction(rvu, blkaddr, &profile->action[idx],
+				     kpm, entry, false);
+}
+
+static void
+npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries)
+{
+	u64 entry_mask;
+
+	entry_mask = npc_enable_mask(num_entries);
+	/* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */
+	if (!rvu->kpu.custom)
+		entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask);
+	if (num_entries <= 64) {
+		/* Disable all the entries in W1, W2 and W3 */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 1),
+			    npc_enable_mask(0));
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
+			    npc_enable_mask(0));
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
+			    npc_enable_mask(0));
+		return;
+	}
+
+	num_entries = num_entries - 64;
+	entry_mask = npc_enable_mask(num_entries);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask);
+	if (num_entries <= 64) {
+		/* Disable all the entries in W2 and W3 */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
+			    npc_enable_mask(0));
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
+			    npc_enable_mask(0));
+		return;
+	}
+
+	num_entries = num_entries - 64;
+	entry_mask = npc_enable_mask(num_entries);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask);
+	if (num_entries <= 64) {
+		/* Disable all the entries in W3 */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
+			    npc_enable_mask(0));
+		return;
+	}
+
+	num_entries = num_entries - 64;
+	entry_mask = npc_enable_mask(num_entries);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask);
+}
+
+#define KPU_OFFSET	8
+static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms)
+{
+	const struct npc_kpu_profile *profile1, *profile2;
+	int idx, total_cam_entries;
+
+	for (idx = 0; idx < num_kpms; idx++) {
+		profile1 = &rvu->kpu.kpu[idx];
+		npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1);
+		profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET];
+		npc_program_single_kpm_profile(rvu, blkaddr, idx,
+					       profile1->cam_entries,
+					       profile2);
+		total_cam_entries = profile1->cam_entries +
+			profile2->cam_entries;
+		npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries);
+		rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx),
+			    profile1->cam_entries);
+		/* Enable the KPUs associated with this KPM */
+		rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01);
+		rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET),
+			    0x01);
+	}
+}
+
+void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr)
+{
+	struct rvu_hwinfo *hw = rvu->hw;
+	int num_pkinds, idx;
+
+	/* Disable all KPMs and their entries */
+	for (idx = 0; idx < hw->npc_kpms; idx++) {
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL);
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL);
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL);
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL);
+	}
+
+	for (idx = 0; idx < hw->npc_kpus; idx++)
+		rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00);
+
+	/* Load and customize KPU profile. */
+	npc_load_kpu_profile(rvu);
+
+	/* Configure KPU and KPM mapping for second pass */
+	rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210);
+
+	/* First program IKPU profile i.e PKIND configs.
+	 * Check HW max count to avoid configuring junk or
+	 * writing to unsupported CSR addresses.
+	 */
+	num_pkinds = rvu->kpu.pkinds;
+	num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds);
+
+	for (idx = 0; idx < num_pkinds; idx++)
+		npc_config_kpmaction(rvu, blkaddr, &rvu->kpu.ikpu[idx],
+				     0, idx, true);
+
+	/* Program KPM CAM and Action profiles */
+	npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms);
+}
+
+struct npc_priv_t *npc_priv_get(void)
+{
+	return &npc_priv;
+}
+
+static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr,
+				struct npc_mcam_kex_extr *mkex_extr,
+				u8 intf)
+{
+	u8 num_extr = rvu->hw->npc_kex_extr;
+	int extr, lt;
+	u64 val;
+
+	if (is_npc_intf_tx(intf))
+		return;
+
+	rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
+		    mkex_extr->keyx_cfg[NIX_INTF_RX]);
+
+	/* Program EXTRACTOR */
+	for (extr = 0; extr < num_extr; extr++)
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
+			    mkex_extr->intf_extr_lid[intf][extr]);
+
+	/* Program EXTRACTOR_LTYPE */
+	for (extr = 0; extr < num_extr; extr++) {
+		for (lt = 0; lt < NPC_MAX_LT; lt++) {
+			val = mkex_extr->intf_extr_lt[intf][extr][lt];
+			CN20K_SET_EXTR_LT(intf, extr, lt, val);
+		}
+	}
+}
+
+static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr,
+				struct npc_mcam_kex_extr *mkex_extr,
+				u8 intf)
+{
+	u8 num_extr = rvu->hw->npc_kex_extr;
+	int extr, lt;
+	u64 val;
+
+	if (is_npc_intf_rx(intf))
+		return;
+
+	rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
+		    mkex_extr->keyx_cfg[NIX_INTF_TX]);
+
+	/* Program EXTRACTOR */
+	for (extr = 0; extr < num_extr; extr++)
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
+			    mkex_extr->intf_extr_lid[intf][extr]);
+
+	/* Program EXTRACTOR_LTYPE */
+	for (extr = 0; extr < num_extr; extr++) {
+		for (lt = 0; lt < NPC_MAX_LT; lt++) {
+			val = mkex_extr->intf_extr_lt[intf][extr][lt];
+			CN20K_SET_EXTR_LT(intf, extr, lt, val);
+		}
+	}
+}
+
+static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,
+				     struct npc_mcam_kex_extr *mkex_extr)
+{
+	struct rvu_hwinfo *hw = rvu->hw;
+	u8 intf;
+
+	for (intf = 0; intf < hw->npc_intfs; intf++) {
+		npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf);
+		npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf);
+	}
+
+	/* Programme mkex hash profile */
+	npc_program_mkex_hash(rvu, blkaddr);
+}
+
+void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr,
+				 const char *mkex_profile)
+{
+	struct npc_mcam_kex_extr *mcam_kex_extr;
+	struct device *dev = &rvu->pdev->dev;
+	void __iomem *mkex_prfl_addr = NULL;
+	u64 prfl_sz;
+	int ret;
+
+	/* If user not selected mkex profile */
+	if (rvu->kpu_fwdata_sz ||
+	    !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN))
+		goto program_mkex_extr;
+
+	/* Setting up the mapping for mkex profile image */
+	ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz);
+	if (ret < 0)
+		goto program_mkex_extr;
+
+	mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr;
+
+	while (((s64)prfl_sz > 0) &&
+	       (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) {
+		/* Compare with mkex mod_param name string */
+		if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN &&
+		    !strncmp(mcam_kex_extr->name, mkex_profile,
+			     MKEX_NAME_LEN)) {
+			rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr;
+			goto program_mkex_extr;
+		}
+
+		mcam_kex_extr++;
+		prfl_sz -= sizeof(struct npc_mcam_kex_extr);
+	}
+	dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);
+	rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get();
+
+program_mkex_extr:
+	dev_info(rvu->dev, "Using %s mkex profile\n",
+		 rvu->kpu.mcam_kex_prfl.mkex_extr->name);
+	/* Program selected mkex profile */
+	npc_program_mkex_profile(rvu, blkaddr,
+				 rvu->kpu.mcam_kex_prfl.mkex_extr);
+	if (mkex_prfl_addr)
+		iounmap(mkex_prfl_addr);
+}
+
+void
+npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
+			    int index, bool enable)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	int mcam_idx = index % mcam->banksize;
+	int bank = index / mcam->banksize;
+	u64 cfg, hw_prio;
+	u8 kw_type;
+
+	npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+	if (kw_type == NPC_MCAM_KEY_X2) {
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
+								   bank));
+		hw_prio = cfg & GENMASK_ULL(30, 24);
+		cfg = enable ? 1 : 0;
+		cfg |= hw_prio;
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+			    cfg);
+		return;
+	}
+
+	/* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks
+	 * need to be programmed with the same value.
+	 */
+	for (bank = 0; bank < mcam->banks_per_entry; bank++) {
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
+								   bank));
+		hw_prio = cfg & GENMASK_ULL(30, 24);
+		cfg = enable ? 1 : 0;
+		cfg |= hw_prio;
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+			    cfg);
+	}
+}
+
+void
+npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int bank, int index)
+{
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
+		    0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
+		    0);
+
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0);
+
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0);
+
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0);
+
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0);
+
+	/* Clear corresponding stats register */
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0);
+}
+
+static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx,
+				  u64 *cam0, u64 *cam1)
+{
+	u64 kw_mask;
+
+	/* The two banks of every MCAM entry are used as a single double-wide
+	 * entry that is compared with the search key as follows:
+	 *
+	 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0]
+	 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1]
+	 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2]
+	 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3]
+	 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4]
+	 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5]
+	 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6]
+	 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7]
+	 */
+	*cam1 = entry->kw[idx];
+	kw_mask = entry->kw_mask[idx];
+	*cam1 &= kw_mask;
+	*cam0 = ~*cam1 & kw_mask;
+}
+
+/*-------------------------------------------------------------------------
+ *Kex type|  mcam	|  cam1	|cam0   | req_kwtype||<----- output >	  |
+ * in     |		|	|	|	    ||		|	  |
+ * profile|  len	|	|	|	    ||len	| type    |
+ *-------------------------------------------------------------------------
+ *X2	|  256 (X2)	|  001b	|110b	| 0	    ||X2	| X2      |
+ *------------------------------------------------------------------------|
+ *X4	|  256 (X2)	|  000b	|000b	| 0	    ||X2	| DYN     |
+ *------------------------------------------------------------------------|
+ *X4	|  512 (X4)	|  010b	|101b	| 0	    ||X4	| X4      |
+ *------------------------------------------------------------------------|
+ *DYN	|  256 (X2)	|  000b	|000b	| 0	    ||X2	| DYN     |
+ *------------------------------------------------------------------------|
+ *DYN	|  512 (X4)	|  010b	|101b	| 0	    ||X4	| X4      |
+ *------------------------------------------------------------------------|
+ *X4	|  256 (X2)	|  000b	|000b	| X2	    ||DYN	| DYN     |
+ *------------------------------------------------------------------------|
+ *DYNC	|  256 (X2)	|  000b	|000b	| X2	    ||DYN	| DYN     |
+ *------------------------------------------------------------------------|
+ * X2	|  512 (X4)	|  xxxb	|xxxb	| X4	    ||INVAL	| INVAL   |
+ *------------------------------------------------------------------------|
+ */
+static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam,
+				   int blkaddr, int index, u8 intf,
+				   struct cn20k_mcam_entry *entry,
+				   int bank, u8 kw_type, int kw,
+				   u8 req_kw_type)
+{
+	u64 intf_ext = 0, intf_ext_mask = 0;
+	u8 tx_intf_mask = ~intf & 0x3;
+	u8 tx_intf = intf, kex_type;
+	u8 kw_type_mask = ~kw_type;
+	u64 cam0, cam1, kex_cfg;
+
+	if (is_npc_intf_tx(intf)) {
+		/* Last bit must be set and rest don't care
+		 * for TX interfaces
+		 */
+		tx_intf_mask = 0x1;
+		tx_intf = intf & tx_intf_mask;
+		tx_intf_mask = ~tx_intf & tx_intf_mask;
+	}
+
+	kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
+	kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32;
+	if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) &&
+	    kw_type == NPC_MCAM_KEY_X2) {
+		kw_type = 0;
+		kw_type_mask = 0;
+	}
+
+	/* Say, we need to write x2 keyword in an x4 subbank.
+	 * req_kw_type will be x2, and kw_type will be x4.
+	 * So in the case ignore kw bits in mcam.
+	 */
+	if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) {
+		kw_type = 0;
+		kw_type_mask = 0;
+	}
+
+	intf_ext = ((u64)kw_type << 16) | tx_intf;
+	intf_ext_mask = (((u64)kw_type_mask  << 16) & GENMASK_ULL(18, 16)) |
+		tx_intf_mask;
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
+		    intf_ext);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
+		    intf_ext_mask);
+
+	/* Set the match key */
+	npc_cn20k_get_keyword(entry, kw, &cam0, &cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1),
+		    cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0),
+		    cam0);
+
+	npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1),
+		    cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0),
+		    cam0);
+
+	npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1),
+		    cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0),
+		    cam0);
+
+	npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1),
+		    cam1);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0),
+		    cam0);
+}
+
+static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam,
+				   int blkaddr, int index, u8 intf,
+				   struct cn20k_mcam_entry *entry,
+				   u8 kw_type, u8 req_kw_type)
+{
+	int kw = 0, bank;
+
+	for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4)
+		npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
+				       index, intf,
+				       entry, bank, kw_type,
+				       kw, req_kw_type);
+}
+
+static void
+npc_cn20k_set_mcam_bank_cfg(struct rvu *rvu, int blkaddr, int mcam_idx,
+			    int bank, u8 kw_type, bool enable, u8 hw_prio)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u64 bank_cfg;
+
+	bank_cfg = (u64)hw_prio << 24;
+	if (enable)
+		bank_cfg |= 0x1;
+
+	if (kw_type == NPC_MCAM_KEY_X2) {
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+			    bank_cfg);
+		return;
+	}
+
+	/* For NPC_MCAM_KEY_X4 keys, both the banks
+	 * need to be programmed with the same value.
+	 */
+	for (bank = 0; bank < mcam->banks_per_entry; bank++) {
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+			    bank_cfg);
+	}
+}
+
+void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
+				 u8 intf, struct cn20k_mcam_entry *entry,
+				 bool enable, u8 hw_prio, u8 req_kw_type)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	int mcam_idx = index % mcam->banksize;
+	int bank = index / mcam->banksize;
+	int kw = 0;
+	u8 kw_type;
+
+	/* Disable before mcam entry update */
+	npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false);
+
+	npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+	/* CAM1 takes the comparison value and
+	 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'.
+	 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0
+	 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1
+	 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare.
+	 */
+	if (kw_type == NPC_MCAM_KEY_X2) {
+		/* Clear mcam entry to avoid writes being suppressed by NPC */
+		npc_cn20k_clear_mcam_entry(rvu, blkaddr, bank, mcam_idx);
+		npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
+				       mcam_idx, intf, entry,
+				       bank, kw_type, kw, req_kw_type);
+		/* Set 'action' */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 0),
+			    entry->action);
+
+		/* Set 'action2' for inline receive */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 2),
+			    entry->action2);
+
+		/* Set TAG 'action' */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 1),
+			    entry->vtag_action);
+		goto set_cfg;
+	}
+
+	/* Clear mcam entry to avoid writes being suppressed by NPC */
+	npc_cn20k_clear_mcam_entry(rvu, blkaddr, 0, mcam_idx);
+	npc_cn20k_clear_mcam_entry(rvu, blkaddr, 1, mcam_idx);
+
+	npc_cn20k_config_kw_x4(rvu, mcam, blkaddr,
+			       mcam_idx, intf, entry,
+			       kw_type, req_kw_type);
+	for (bank = 0; bank < mcam->banks_per_entry; bank++) {
+		/* Set 'action' */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 0),
+			    entry->action);
+
+		/* Set TAG 'action' */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 1),
+			    entry->vtag_action);
+
+		/* Set 'action2' for inline receive */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+								  bank, 2),
+			    entry->action2);
+	}
+
+set_cfg:
+	/* TODO: */
+	/* PF installing VF rule */
+	npc_cn20k_set_mcam_bank_cfg(rvu, blkaddr, mcam_idx, bank,
+				    kw_type, enable, hw_prio);
+}
+
+void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u64 cfg, sreg, dreg, soff, doff;
+	u8 src_kwtype, dest_kwtype;
+	int bank, i, sb, db;
+	int dbank, sbank;
+
+	dbank = npc_get_bank(mcam, dest);
+	sbank = npc_get_bank(mcam, src);
+	npc_mcam_idx_2_key_type(rvu, src, &src_kwtype);
+	npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype);
+	if (src_kwtype != dest_kwtype)
+		return;
+
+	src &= (mcam->banksize - 1);
+	dest &= (mcam->banksize - 1);
+
+	/* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */
+	for (bank = 0; bank < mcam->banks_per_entry; bank++) {
+		sb = sbank + bank;
+		sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0);
+		db = dbank + bank;
+		dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0);
+		for (i = 0; i < 10; i++) {
+			cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8));
+			rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg);
+		}
+
+		/* Copy action */
+		for (i = 0; i < 3; i++) {
+			soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src,
+								     sb, i);
+			cfg = rvu_read64(rvu, blkaddr, soff);
+
+			doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db,
+								     i);
+			rvu_write64(rvu, blkaddr, doff, cfg);
+		}
+
+		/* Copy bank configuration */
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb));
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg);
+		if (src_kwtype == NPC_MCAM_KEY_X2)
+			break;
+	}
+}
+
+static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx,
+				     u64 cam0, u64 cam1)
+{
+	entry->kw[idx] = cam1;
+	entry->kw_mask[idx] = cam1 ^ cam0;
+}
+
+void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
+			       struct cn20k_mcam_entry *entry,
+			       u8 *intf, u8 *ena, u8 *hw_prio)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u64 cam0, cam1, bank_cfg, cfg;
+	int kw = 0, bank;
+	u8 kw_type;
+
+	npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+
+	bank = npc_get_bank(mcam, index);
+	index &= (mcam->banksize - 1);
+
+	cfg = rvu_read64(rvu, blkaddr,
+			 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index,
+								 bank, 1)) & 3;
+	*intf = cfg;
+
+	bank_cfg = rvu_read64(rvu, blkaddr,
+			      NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank));
+	*ena = bank_cfg & 0x1;
+	*hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24;
+	if (kw_type == NPC_MCAM_KEY_X2) {
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
+		goto read_action;
+	}
+
+	for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) {
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
+
+		cam1 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
+									bank,
+									1));
+		cam0 = rvu_read64(rvu, blkaddr,
+				  NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
+									bank,
+									0));
+		npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
+	}
+
+read_action:
+	/* 'action' is set to same value for both bank '0' and '1'.
+	 * Hence, reading bank '0' should be enough.
+	 */
+	cfg = rvu_read64(rvu, blkaddr,
+			 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 0));
+	entry->action = cfg;
+
+	cfg = rvu_read64(rvu, blkaddr,
+			 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 1));
+	entry->vtag_action = cfg;
+}
+
+int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu,
+						struct npc_cn20k_mcam_write_entry_req *req,
+						struct msg_rsp *rsp)
+{
+	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u16 pcifunc = req->hdr.pcifunc;
+	int blkaddr, rc;
+	u8 nix_intf;
+
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0)
+		return NPC_MCAM_INVALID_REQ;
+
+	mutex_lock(&mcam->lock);
+	rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
+	if (rc)
+		goto exit;
+
+	if (!is_npc_interface_valid(rvu, req->intf)) {
+		rc = NPC_MCAM_INVALID_REQ;
+		goto exit;
+	}
+
+	if (is_npc_intf_tx(req->intf))
+		nix_intf = pfvf->nix_tx_intf;
+	else
+		nix_intf = pfvf->nix_rx_intf;
+
+	/* For AF installed rules, the nix_intf should be set to target NIX */
+	if (is_pffunc_af(req->hdr.pcifunc))
+		nix_intf = req->intf;
+
+	npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf,
+				    &req->entry_data, req->enable_entry,
+				    req->hw_prio, req->req_kw_type);
+
+	rc = 0;
+exit:
+	mutex_unlock(&mcam->lock);
+	return rc;
+}
+
+int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu,
+					       struct npc_mcam_read_entry_req *req,
+					       struct npc_cn20k_mcam_read_entry_rsp *rsp)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u16 pcifunc = req->hdr.pcifunc;
+	int blkaddr, rc;
+
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0)
+		return NPC_MCAM_INVALID_REQ;
+
+	mutex_lock(&mcam->lock);
+	rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
+	if (!rc)
+		npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry,
+					  &rsp->entry_data, &rsp->intf,
+					  &rsp->enable, &rsp->hw_prio);
+
+	mutex_unlock(&mcam->lock);
+	return rc;
+}
+
+int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu,
+							  struct npc_cn20k_mcam_alloc_and_write_entry_req *req,
+							  struct npc_mcam_alloc_and_write_entry_rsp *rsp)
+{
+	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
+	struct npc_mcam_alloc_entry_req entry_req;
+	struct npc_mcam_alloc_entry_rsp entry_rsp;
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	u16 entry = NPC_MCAM_ENTRY_INVALID;
+	int blkaddr, rc;
+	u8 nix_intf;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0)
+		return NPC_MCAM_INVALID_REQ;
+
+	if (!is_npc_interface_valid(rvu, req->intf))
+		return NPC_MCAM_INVALID_REQ;
+
+	/* Try to allocate a MCAM entry */
+	entry_req.hdr.pcifunc = req->hdr.pcifunc;
+	entry_req.contig = true;
+	entry_req.ref_prio = req->ref_prio;
+	entry_req.ref_entry = req->ref_entry;
+	entry_req.count = 1;
+	entry_req.virt = req->virt;
+
+	rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,
+						   &entry_req, &entry_rsp);
+	if (rc)
+		return rc;
+
+	if (!entry_rsp.count)
+		return NPC_MCAM_ALLOC_FAILED;
+
+	/* entry_req.count is 1, so single entry is allocated */
+	entry = npc_cn20k_vidx2idx(entry_rsp.entry);
+
+	mutex_lock(&mcam->lock);
+
+	if (is_npc_intf_tx(req->intf))
+		nix_intf = pfvf->nix_tx_intf;
+	else
+		nix_intf = pfvf->nix_rx_intf;
+
+	npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf,
+				    &req->entry_data, req->enable_entry,
+				    req->hw_prio, req->req_kw_type);
+
+	mutex_unlock(&mcam->lock);
+
+	rsp->entry = entry_rsp.entry;
+	return 0;
+}
+
+static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc)
+{
+	if (is_lbk_vf(rvu, pcifunc))
+		return NIXLF_PROMISC_ENTRY;
+
+	return NIXLF_UCAST_ENTRY;
+}
+
+int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu,
+						    struct msg_req *req,
+						    struct npc_cn20k_mcam_read_base_rule_rsp *rsp)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	int index, blkaddr, nixlf, rc = 0;
+	u16 pcifunc = req->hdr.pcifunc;
+	u8 intf, enable, hw_prio;
+	struct rvu_pfvf *pfvf;
+	int rl_type;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0)
+		return NPC_MCAM_INVALID_REQ;
+
+	/* Return the channel number in case of PF */
+	if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
+		pfvf = rvu_get_pfvf(rvu, pcifunc);
+		rsp->entry.kw[0] = pfvf->rx_chan_base;
+		rsp->entry.kw_mask[0] = 0xFFFULL;
+		goto out;
+	}
+
+	/* Find the pkt steering rule installed by PF to this VF */
+	mutex_lock(&mcam->lock);
+	for (index = 0; index < mcam->bmap_entries; index++) {
+		if (mcam->entry2target_pffunc[index] == pcifunc)
+			goto read_entry;
+	}
+
+	rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
+	if (rc < 0) {
+		mutex_unlock(&mcam->lock);
+		goto out;
+	}
+
+	rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc);
+
+	/* Read the default ucast entry if there is no pkt steering rule */
+	index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type);
+	if (index < 0) {
+		mutex_unlock(&mcam->lock);
+		goto out;
+	}
+
+read_entry:
+	/* Read the mcam entry */
+	npc_cn20k_read_mcam_entry(rvu, blkaddr, index,
+				  &rsp->entry, &intf,
+				  &enable, &hw_prio);
+	mutex_unlock(&mcam->lock);
+out:
+	return rc;
+}
+
+static u8 npc_map2cn20k_flag(u8 flag)
+{
+	switch (flag) {
+	case NPC_F_LC_U_IP_FRAG:
+		return NPC_CN20K_F_LC_L_IP_FRAG;
+
+	case NPC_F_LC_U_IP6_FRAG:
+		return NPC_CN20K_F_LC_L_IP6_FRAG;
+
+	case NPC_F_LC_L_6TO4:
+		return NPC_CN20K_F_LC_L_6TO4;
+
+	case NPC_F_LC_L_MPLS_IN_IP:
+		return NPC_CN20K_F_LC_U_MPLS_IN_IP;
+
+	case NPC_F_LC_L_IP6_TUN_IP6:
+		return NPC_CN20K_F_LC_U_IP6_TUN_IP6;
+
+	case NPC_F_LC_L_IP6_MPLS_IN_IP:
+		return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP;
+
+	default:
+		break;
+	}
+
+	WARN(1, "%s: Invalid flag=%u\n", __func__, flag);
+	return 0xff;
+}
+
+void
+npc_cn20k_update_action_entries_n_flags(struct rvu *rvu,
+					struct npc_kpu_profile_adapter *pfl)
+{
+	struct npc_kpu_profile_action *action;
+	int entries, ltype;
+	u8 flags, val;
+
+	for (int i = 0; i < pfl->kpus; i++) {
+		action = pfl->kpu[i].action;
+		entries = pfl->kpu[i].action_entries;
+
+		for (int j = 0; j < entries; j++) {
+			if (action[j].lid != NPC_LID_LC)
+				continue;
+
+			ltype = action[j].ltype;
+
+			if (ltype != NPC_LT_LC_IP &&
+			    ltype != NPC_LT_LC_IP6 &&
+			    ltype != NPC_LT_LC_IP_OPT &&
+			    ltype != NPC_LT_LC_IP6_EXT)
+				continue;
+
+			flags = action[j].flags;
+
+			switch (flags) {
+			case NPC_F_LC_U_IP_FRAG:
+			case NPC_F_LC_U_IP6_FRAG:
+			case NPC_F_LC_L_6TO4:
+			case NPC_F_LC_L_MPLS_IN_IP:
+			case NPC_F_LC_L_IP6_TUN_IP6:
+			case NPC_F_LC_L_IP6_MPLS_IN_IP:
+				val = npc_map2cn20k_flag(flags);
+				if (val == 0xFF) {
+					dev_err(rvu->dev,
+						"%s: Error to get flag value\n",
+						__func__);
+					return;
+				}
+
+				action[j].flags = val;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+}
+
+int npc_cn20k_apply_custom_kpu(struct rvu *rvu,
+			       struct npc_kpu_profile_adapter *profile)
+{
+	size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
+	struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata;
+	struct npc_kpu_profile_action *action;
+	struct npc_kpu_profile_cam *cam;
+	struct npc_kpu_fwdata *fw_kpu;
+	size_t offset = 0;
+	u16 kpu, entry;
+	int entries;
+
+	hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
+
+	if (rvu->kpu_fwdata_sz < hdr_sz) {
+		dev_warn(rvu->dev, "Invalid KPU profile size\n");
+		return -EINVAL;
+	}
+
+	if (le64_to_cpu(fw->signature) != KPU_SIGN) {
+		dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n",
+			 fw->signature);
+		return -EINVAL;
+	}
+
+	/* Verify if the using known profile structure */
+	if (NPC_KPU_VER_MAJ(profile->version) >
+	    NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) {
+		dev_warn(rvu->dev, "Not supported Major version: %d > %d\n",
+			 NPC_KPU_VER_MAJ(profile->version),
+			 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER));
+		return -EINVAL;
+	}
+
+	/* Verify if profile is aligned with the required kernel changes */
+	if (NPC_KPU_VER_MIN(profile->version) <
+	    NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) {
+		dev_warn(rvu->dev,
+			 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n",
+			 NPC_KPU_VER_MAJ(profile->version),
+			 NPC_KPU_VER_MIN(profile->version),
+			 NPC_KPU_VER_PATCH(profile->version),
+			 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER),
+			 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER),
+			 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER));
+		return -EINVAL;
+	}
+
+	/* Verify if profile fits the HW */
+	if (fw->kpus > profile->kpus) {
+		dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus,
+			 profile->kpus);
+		return -EINVAL;
+	}
+
+	profile->mcam_kex_prfl.mkex_extr = &fw->mkex;
+	if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) {
+		dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n",
+			 profile->mcam_kex_prfl.mkex_extr->mkex_sign);
+		return -EINVAL;
+	}
+
+	profile->custom = 1;
+	profile->name = fw->name;
+	profile->version = le64_to_cpu(fw->version);
+	profile->lt_def = &fw->lt_def;
+
+	for (kpu = 0; kpu < fw->kpus; kpu++) {
+		fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset);
+		if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT)
+			dev_warn(rvu->dev,
+				 "Too many custom entries on KPU%d: %d > %d\n",
+				 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
+		entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
+		cam = (struct npc_kpu_profile_cam *)fw_kpu->data;
+		offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam);
+		action = (struct npc_kpu_profile_action *)(fw->data + offset);
+		offset += fw_kpu->entries * sizeof(*action);
+		if (rvu->kpu_fwdata_sz < hdr_sz + offset) {
+			dev_warn(rvu->dev,
+				 "Profile size mismatch on KPU%i parsing.\n",
+				 kpu + 1);
+			return -EINVAL;
+		}
+
+		for (entry = 0; entry < entries; entry++) {
+			profile->kpu[kpu].cam[entry] = cam[entry];
+			profile->kpu[kpu].action[entry] = action[entry];
+		}
+	}
+	npc_cn20k_update_action_entries_n_flags(rvu, profile);
+
+	return 0;
+}
+
+int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type)
+{
+	struct npc_subbank *sb;
+	int bank_off, sb_id;
+
+	/* mcam_idx should be less than (2 * bank depth) */
+	if (mcam_idx >= npc_priv.bank_depth * 2) {
+		dev_err(rvu->dev, "%s: bad params\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	/* find mcam offset per bank */
+	bank_off = mcam_idx & (npc_priv.bank_depth - 1);
+
+	/* Find subbank id */
+	sb_id = bank_off / npc_priv.subbank_depth;
+
+	/* Check if subbank id is more than maximum
+	 * number of subbanks available
+	 */
+	if (sb_id >= npc_priv.num_subbanks) {
+		dev_err(rvu->dev, "%s: invalid subbank %d\n",
+			__func__, sb_id);
+		return -EINVAL;
+	}
+
+	sb = &npc_priv.sb[sb_id];
+
+	*key_type = sb->key_type;
+
+	return 0;
+}
+
+static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb,
+				      u16 sub_off, u16 *mcam_idx)
+{
+	int off, bot;
+
+	/* for x4 section, maximum allowed subbank index =
+	 * subsection depth - 1
+	 */
+	if (sb->key_type == NPC_MCAM_KEY_X4 &&
+	    sub_off >= npc_priv.subbank_depth) {
+		dev_err(rvu->dev,
+			"%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u",
+			__func__, sb->idx, sub_off);
+		return -EINVAL;
+	}
+
+	/* for x2 section, maximum allowed subbank index =
+	 * 2 * subsection depth - 1
+	 */
+	if (sb->key_type == NPC_MCAM_KEY_X2 &&
+	    sub_off >= npc_priv.subbank_depth * 2) {
+		dev_err(rvu->dev,
+			"%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u",
+			__func__, sb->idx, sub_off);
+		return -EINVAL;
+	}
+
+	/* Find subbank offset from respective subbank (w.r.t bank) */
+	off = sub_off & (npc_priv.subbank_depth - 1);
+
+	/* if subsection idx is in bank1, add bank depth,
+	 * which is part of sb->b1b
+	 */
+	bot = sub_off >= npc_priv.subbank_depth ? sb->b1b : sb->b0b;
+
+	*mcam_idx = bot + off;
+	return 0;
+}
+
+static int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx,
+				      struct npc_subbank **sb,
+				      int *sb_off)
+{
+	int bank_off, sb_id;
+
+	/* mcam_idx should be less than (2 * bank depth) */
+	if (mcam_idx >= npc_priv.bank_depth * 2) {
+		dev_err(rvu->dev, "%s: Invalid mcam idx %u\n",
+			__func__, mcam_idx);
+		return -EINVAL;
+	}
+
+	/* find mcam offset per bank */
+	bank_off = mcam_idx & (npc_priv.bank_depth - 1);
+
+	/* Find subbank id */
+	sb_id = bank_off / npc_priv.subbank_depth;
+
+	/* Check if subbank id is more than maximum
+	 * number of subbanks available
+	 */
+	if (sb_id >= npc_priv.num_subbanks) {
+		dev_err(rvu->dev, "%s: invalid subbank %d\n",
+			__func__, sb_id);
+		return -EINVAL;
+	}
+
+	*sb = &npc_priv.sb[sb_id];
+
+	/* Subbank offset per bank */
+	*sb_off = bank_off % npc_priv.subbank_depth;
+
+	/* Index in a subbank should add subbank depth
+	 * if it is in bank1
+	 */
+	if (mcam_idx >= npc_priv.bank_depth)
+		*sb_off += npc_priv.subbank_depth;
+
+	return 0;
+}
+
+static int __npc_subbank_contig_alloc(struct rvu *rvu,
+				      struct npc_subbank *sb,
+				      int key_type, int sidx,
+				      int eidx, int prio,
+				      int count, int t, int b,
+				      unsigned long *bmap,
+				      u16 *save)
+{
+	int k, offset, delta = 0;
+	int cnt = 0, sbd;
+
+	sbd = npc_priv.subbank_depth;
+
+	if (sidx >= npc_priv.bank_depth)
+		delta = sbd;
+
+	switch (prio) {
+	case NPC_MCAM_LOWER_PRIO:
+	case NPC_MCAM_ANY_PRIO:
+		/* Find an area of size 'count' from sidx to eidx */
+		offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b,
+						    count, 0);
+
+		if (offset >= sbd) {
+			dev_err(rvu->dev,
+				"%s: Could not find contiguous(%d) entries\n",
+				__func__, count);
+			return -EFAULT;
+		}
+
+		dev_dbg(rvu->dev,
+			"%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n",
+			__func__, sidx, eidx, t, b, offset,
+			count, delta);
+
+		for (cnt = 0; cnt < count; cnt++)
+			save[cnt] = offset + cnt + delta;
+
+		break;
+
+	case NPC_MCAM_HIGHER_PRIO:
+		/* Find an area of 'count' from eidx to sidx */
+		for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
+			/* If an intermediate slot is not free,
+			 * reset the counter (cnt) to zero as
+			 * request is for contiguous.
+			 */
+			if (test_bit(k, bmap)) {
+				cnt = 0;
+				continue;
+			}
+
+			save[cnt++] = k + delta;
+		}
+		break;
+	}
+
+	/* Found 'count' number of free slots */
+	if (cnt == count)
+		return 0;
+
+	dev_dbg(rvu->dev,
+		"%s: Could not find contiguous(%d) entries in subbank=%u\n",
+		__func__, count, sb->idx);
+	return -EFAULT;
+}
+
+static int __npc_subbank_non_contig_alloc(struct rvu *rvu,
+					  struct npc_subbank *sb,
+					  int key_type, int sidx,
+					  int eidx, int prio,
+					  int t, int b,
+					  unsigned long *bmap,
+					  int count, u16 *save,
+					  bool max_alloc, int *alloc_cnt)
+{
+	unsigned long index;
+	int cnt = 0, delta;
+	int k, sbd;
+
+	sbd = npc_priv.subbank_depth;
+	delta = sidx >= npc_priv.bank_depth ? sbd : 0;
+
+	switch (prio) {
+		/* Find an area of size 'count' from sidx to eidx */
+	case NPC_MCAM_LOWER_PRIO:
+	case NPC_MCAM_ANY_PRIO:
+		index = find_next_zero_bit(bmap, sbd, sidx - b);
+		if (index >= sbd) {
+			dev_err(rvu->dev,
+				"%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n",
+				__func__, count,
+				bitmap_weight(bmap, sbd),
+				sb->idx);
+			break;
+		}
+
+		for (k = index; cnt < count && k <= (eidx - b); k++) {
+			/* Skip used slots */
+			if (test_bit(k, bmap))
+				continue;
+
+			save[cnt++] = k + delta;
+		}
+		break;
+
+		/* Find an area of 'count' from eidx to sidx */
+	case NPC_MCAM_HIGHER_PRIO:
+		for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
+			/* Skip used slots */
+			if (test_bit(k, bmap))
+				continue;
+
+			save[cnt++] = k + delta;
+		}
+		break;
+	}
+
+	/* Update allocated 'cnt' to alloc_cnt */
+	*alloc_cnt = cnt;
+
+	/* Successfully allocated requested count slots */
+	if (cnt == count)
+		return 0;
+
+	/* Allocation successful for cnt < count */
+	if (max_alloc && cnt > 0)
+		return 0;
+
+	dev_dbg(rvu->dev,
+		"%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n",
+		__func__, count, sb->idx, cnt, max_alloc);
+
+	return -EFAULT;
+}
+
+static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb,
+				      int sb_off, unsigned long **bmap,
+				      int *off)
+{
+	int sbd;
+
+	sbd = npc_priv.subbank_depth;
+
+	*off = sb_off & (sbd - 1);
+	*bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map;
+}
+
+/* set/clear bitmap */
+static bool __npc_subbank_mark_slot(struct rvu *rvu,
+				    struct npc_subbank *sb,
+				    int sb_off, bool set)
+{
+	unsigned long *bmap;
+	int off;
+
+	/* if sb_off >= subbank.depth, then slots are in
+	 * bank1
+	 */
+	__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
+
+	dev_dbg(rvu->dev,
+		"%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n",
+		__func__, set, sb_off, sb->idx, off);
+
+	if (set) {
+		/* Slot is already used */
+		if (test_bit(off, bmap))
+			return false;
+
+		sb->free_cnt--;
+		set_bit(off, bmap);
+		return true;
+	}
+
+	/* Slot is already free */
+	if (!test_bit(off, bmap))
+		return false;
+
+	sb->free_cnt++;
+	clear_bit(off, bmap);
+	return true;
+}
+
+static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb)
+{
+	int rc, blkaddr;
+
+	sb->flags = NPC_SUBBANK_FLAG_FREE;
+	sb->key_type = 0;
+
+	bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth);
+	bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth);
+
+	if (!xa_erase(&npc_priv.xa_sb_used, sb->arr_idx)) {
+		dev_err(rvu->dev,
+			"%s: Error to delete from xa_sb_used array\n",
+			__func__);
+		return -EFAULT;
+	}
+
+	rc = xa_insert(&npc_priv.xa_sb_free, sb->arr_idx,
+		       xa_mk_value(sb->idx), GFP_KERNEL);
+	if (rc) {
+		rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx,
+			       xa_mk_value(sb->idx), GFP_KERNEL);
+		if (rc)
+			dev_err(rvu->dev,
+				"%s: Failed to roll back sb(%u) arr_idx=%d\n",
+				__func__, sb->idx, sb->arr_idx);
+
+		dev_err(rvu->dev,
+			"%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n",
+			__func__, sb->idx, sb->arr_idx);
+		return rc;
+	}
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	rvu_write64(rvu, blkaddr,
+		    NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
+		    NPC_MCAM_KEY_X2);
+
+	return rc;
+}
+
+static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb,
+				   int key_type)
+{
+	int rc;
+
+	sb->flags = NPC_SUBBANK_FLAG_USED;
+	sb->key_type = key_type;
+	if (key_type == NPC_MCAM_KEY_X4)
+		sb->free_cnt = npc_priv.subbank_depth;
+	else
+		sb->free_cnt = 2 * npc_priv.subbank_depth;
+
+	bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth);
+	bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth);
+
+	if (!xa_erase(&npc_priv.xa_sb_free, sb->arr_idx)) {
+		dev_err(rvu->dev,
+			"%s: Error to delete from xa_sb_free array\n",
+			__func__);
+		return -EFAULT;
+	}
+
+	rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx,
+		       xa_mk_value(sb->idx), GFP_KERNEL);
+	if (rc)
+		dev_err(rvu->dev,
+			"%s: Error to add to xa_sb_used array\n", __func__);
+
+	return rc;
+}
+
+static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb,
+			       u16 sb_off)
+{
+	bool deleted = false;
+	unsigned long *bmap;
+	int rc, off;
+
+	deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false);
+	if (!deleted)
+		goto done;
+
+	__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
+
+	/* Check whether we can mark whole subbank as free */
+	if (sb->key_type == NPC_MCAM_KEY_X4) {
+		if (sb->free_cnt < npc_priv.subbank_depth)
+			goto done;
+	} else {
+		if (sb->free_cnt < 2 * npc_priv.subbank_depth)
+			goto done;
+	}
+
+	/* All slots in subbank are unused. Mark the subbank as free
+	 * and add to free pool
+	 */
+	rc = __npc_subbank_mark_free(rvu, sb);
+	if (rc)
+		dev_err(rvu->dev, "%s: Error to free subbank\n", __func__);
+
+done:
+	return deleted;
+}
+
+static int
+npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off)
+{
+	bool deleted;
+
+	mutex_lock(&sb->lock);
+	deleted = __npc_subbank_free(rvu, sb, sb_off);
+	mutex_unlock(&sb->lock);
+
+	return deleted ? 0 : -EFAULT;
+}
+
+static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
+			       int key_type, int ref, int limit, int prio,
+			       bool contig, int count, u16 *mcam_idx,
+			       int idx_sz, bool max_alloc, int *alloc_cnt)
+{
+	int cnt, t, b, i, blkaddr;
+	bool new_sub_bank = false;
+	unsigned long *bmap;
+	u16 *save = NULL;
+	int sidx, eidx;
+	bool diffbank;
+	int bw, bfree;
+	int rc = 0;
+	bool ret;
+
+	/* Check if enough space is there to return requested number of
+	 * mcam indexes in case of contiguous allocation
+	 */
+	if (!max_alloc && count > idx_sz) {
+		dev_err(rvu->dev,
+			"%s: Less space, count=%d idx_sz=%d sb_id=%d\n",
+			__func__, count, idx_sz, sb->idx);
+		return -ENOSPC;
+	}
+
+	/* Allocation on multiple subbank is not supported by this function.
+	 * it means that ref and limit should be on same subbank.
+	 *
+	 * ref and limit values should be validated w.r.t prio as below.
+	 * say ref = 100, limit = 200,
+	 * if NPC_MCAM_LOWER_PRIO, allocate index 100
+	 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error.
+	 * if NPC_MCAM_ANY_PRIO, allocate index 100
+	 *
+	 * say ref = 200, limit = 100
+	 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error.
+	 * if NPC_MCAM_HIGHER_PRIO, allocate index 200
+	 * if NPC_MCAM_ANY_PRIO, allocate index 100
+	 *
+	 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction
+	 * on "ref" and "limit" values. ie, ref > limit and limit > ref
+	 * are valid cases.
+	 */
+	if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
+	    (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
+		dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
+			__func__, ref, limit);
+		return -EINVAL;
+	}
+
+	/* x4 indexes are from 0 to bank size as it combines two x2 banks */
+	if (key_type == NPC_MCAM_KEY_X4 &&
+	    (ref >= npc_priv.bank_depth || limit >= npc_priv.bank_depth)) {
+		dev_err(rvu->dev,
+			"%s: Wrong ref_enty(%d) or limit(%d) for x4\n",
+			__func__, ref, limit);
+		return -EINVAL;
+	}
+
+	/* This function is called either bank0 or bank1 portion of a subbank.
+	 * so ref and limit should be on same bank.
+	 */
+	diffbank = !!((ref & npc_priv.bank_depth) ^
+		      (limit & npc_priv.bank_depth));
+	if (diffbank) {
+		dev_err(rvu->dev,
+			"%s: request ref and limit should be from same bank\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	sidx = min_t(int, limit, ref);
+	eidx = max_t(int, limit, ref);
+
+	/* Find total number of slots available; both used and free */
+	cnt = eidx - sidx + 1;
+	if (contig && cnt < count) {
+		dev_err(rvu->dev,
+			"%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n",
+			__func__, ref, limit, count);
+		return -EINVAL;
+	}
+
+	/* If subbank is free, check if requested number of indexes is less than
+	 * or equal to mcam entries available in the subbank if contig.
+	 */
+	if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
+		if (contig && count > npc_priv.subbank_depth) {
+			dev_err(rvu->dev, "%s: Less number of entries\n",
+				__func__);
+			return -ENOSPC;
+		}
+
+		new_sub_bank = true;
+		goto process;
+	}
+
+	/* Flag should be set for all used subbanks */
+	WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED),
+		  "Used flag is not set(%#x)\n", sb->flags);
+
+	/* If subbank key type does not match with requested key_type,
+	 * return error
+	 */
+	if (sb->key_type != key_type) {
+		dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__);
+		return -EINVAL;
+	}
+
+process:
+	/* if ref or limit >= npc_priv.bank_depth, index are in bank1.
+	 * else bank0.
+	 */
+	if (ref >= npc_priv.bank_depth) {
+		bmap = sb->b1map;
+		t = sb->b1t;
+		b = sb->b1b;
+	} else {
+		bmap = sb->b0map;
+		t = sb->b0t;
+		b = sb->b0b;
+	}
+
+	/* Calculate free slots */
+	bw = bitmap_weight(bmap, npc_priv.subbank_depth);
+	bfree = npc_priv.subbank_depth - bw;
+
+	if (!bfree) {
+		dev_dbg(rvu->dev, "%s: subbank is full\n", __func__);
+		return -ENOSPC;
+	}
+
+	/* If request is for contiguous , then max we can allocate is
+	 * equal to subbank_depth
+	 */
+	if (contig && bfree < count) {
+		dev_dbg(rvu->dev, "%s: no space for entry\n", __func__);
+		return -ENOSPC;
+	}
+
+	/* 'save' array stores available indexes temporarily before
+	 * marking it as allocated
+	 */
+	save = kcalloc(count, sizeof(u16), GFP_KERNEL);
+	if (!save) {
+		rc = -ENOMEM;
+		goto err1;
+	}
+
+	if (contig) {
+		rc =  __npc_subbank_contig_alloc(rvu, sb, key_type,
+						 sidx, eidx, prio,
+						 count, t, b,
+						 bmap, save);
+		/* contiguous allocation success means that
+		 * requested number of free slots got
+		 * allocated
+		 */
+		if (!rc)
+			*alloc_cnt = count;
+
+	} else {
+		rc =  __npc_subbank_non_contig_alloc(rvu, sb, key_type,
+						     sidx, eidx, prio,
+						     t, b, bmap,
+						     count, save,
+						     max_alloc, alloc_cnt);
+	}
+
+	if (rc)
+		goto err1;
+
+	/* Mark new subbank bank as used */
+	if (new_sub_bank) {
+		blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+		if (blkaddr < 0) {
+			dev_err(rvu->dev,
+				"%s: NPC block not implemented\n", __func__);
+			rc = -EFAULT;
+			goto err1;
+		}
+
+		rc =  __npc_subbank_mark_used(rvu, sb, key_type);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Error to mark subbank as used\n",
+				__func__);
+			goto err2;
+		}
+
+		/* Configure section type to key_type */
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
+			    key_type);
+	}
+
+	for (i = 0; i < *alloc_cnt; i++) {
+		rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i],
+						&mcam_idx[i]);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Error to find mcam idx for %u\n",
+				__func__, save[i]);
+			/* TODO: handle err case gracefully */
+			goto err3;
+		}
+
+		/* Mark all slots as used */
+		ret = __npc_subbank_mark_slot(rvu, sb, save[i], true);
+		if (!ret) {
+			dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n",
+				__func__, mcam_idx[i]);
+			rc = -EFAULT;
+			goto err3;
+		}
+	}
+	kfree(save);
+	return 0;
+
+err3:
+	for (int j = 0; j < i; j++)
+		__npc_subbank_mark_slot(rvu, sb, save[j], false);
+err2:
+	if (new_sub_bank)
+		__npc_subbank_mark_free(rvu, sb);
+err1:
+	kfree(save);
+	return rc;
+}
+
+static int
+npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
+		  int key_type, int ref, int limit, int prio,
+		  bool contig, int count, u16 *mcam_idx,
+		  int idx_sz, bool max_alloc, int *alloc_cnt)
+{
+	int rc;
+
+	mutex_lock(&sb->lock);
+	rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio,
+				 contig, count, mcam_idx, idx_sz,
+				 max_alloc, alloc_cnt);
+	mutex_unlock(&sb->lock);
+
+	return rc;
+}
+
+static int
+npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx)
+{
+	int pcifunc, idx;
+	void *map;
+
+	map = xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n",
+			__func__, mcam_idx);
+		return -EFAULT;
+	}
+
+	pcifunc = xa_to_value(map);
+	map = xa_load(&npc_priv.xa_pf_map, pcifunc);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n",
+			__func__, pcifunc, mcam_idx);
+		return -ESRCH;
+	}
+
+	idx = xa_to_value(map);
+
+	map = xa_erase(&npc_priv.xa_pf2idx_map[idx], mcam_idx);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n",
+			__func__, mcam_idx);
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static int
+npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc)
+{
+	int rc, idx;
+	void *map;
+
+	dev_dbg(rvu->dev,
+		"%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n",
+		__func__, mcam_idx, pcifunc);
+
+	rc = xa_insert(&npc_priv.xa_idx2pf_map, mcam_idx,
+		       xa_mk_value(pcifunc), GFP_KERNEL);
+
+	if (rc) {
+		map = xa_load(&npc_priv.xa_idx2pf_map, mcam_idx);
+		dev_err(rvu->dev,
+			"%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n",
+			__func__, mcam_idx, xa_to_value(map));
+		return -EFAULT;
+	}
+
+	map = xa_load(&npc_priv.xa_pf_map, pcifunc);
+	if (!map) {
+		dev_err(rvu->dev,
+			"%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n",
+			__func__, pcifunc, mcam_idx);
+		return -ESRCH;
+	}
+
+	idx = xa_to_value(map);
+
+	rc = xa_insert(&npc_priv.xa_pf2idx_map[idx], mcam_idx,
+		       xa_mk_value(pcifunc), GFP_KERNEL);
+
+	if (rc) {
+		map = xa_load(&npc_priv.xa_pf2idx_map[idx], mcam_idx);
+		xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx);
+		dev_err(rvu->dev,
+			"%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n",
+			__func__, mcam_idx, xa_to_value(map), idx);
+
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static bool
+npc_subbank_suits(struct npc_subbank *sb, int key_type)
+{
+	mutex_lock(&sb->lock);
+
+	if (!sb->key_type) {
+		mutex_unlock(&sb->lock);
+		return true;
+	}
+
+	if (sb->key_type == key_type) {
+		mutex_unlock(&sb->lock);
+		return true;
+	}
+
+	mutex_unlock(&sb->lock);
+	return false;
+}
+
+#define SB_ALIGN_UP(val)   (((val) + npc_priv.subbank_depth) & \
+			    ~((npc_priv.subbank_depth) - 1))
+#define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv.subbank_depth)
+
+static void npc_subbank_iter_down(struct rvu *rvu,
+				  int ref, int limit,
+				  int *cur_ref, int *cur_limit,
+				  bool *start, bool *stop)
+{
+	int align;
+
+	*stop = false;
+
+	/* ALIGN_DOWN the limit to current subbank boundary bottom index */
+	if (*start) {
+		*start = false;
+		*cur_ref = ref;
+		align = SB_ALIGN_DOWN(ref);
+		if (align < limit) {
+			*stop = true;
+			*cur_limit = limit;
+			return;
+		}
+		*cur_limit = align;
+		return;
+	}
+
+	*cur_ref = *cur_limit - 1;
+	align = *cur_ref - npc_priv.subbank_depth + 1;
+	if (align <= limit) {
+		*stop = true;
+		*cur_limit = limit;
+		return;
+	}
+
+	*cur_limit = align;
+}
+
+static void npc_subbank_iter_up(struct rvu *rvu,
+				int ref, int limit,
+				int *cur_ref, int *cur_limit,
+				bool *start, bool *stop)
+{
+	int align;
+
+	*stop = false;
+
+	/* ALIGN_UP the limit to current subbank boundary top index */
+	if (*start) {
+		*start = false;
+		*cur_ref = ref;
+
+		/* Find next lower prio subbank's bottom index */
+		align = SB_ALIGN_UP(ref);
+
+		/* Crosses limit ? */
+		if (align - 1 > limit) {
+			*stop = true;
+			*cur_limit = limit;
+			return;
+		}
+
+		/* Current subbank's top index */
+		*cur_limit = align - 1;
+		return;
+	}
+
+	*cur_ref = *cur_limit + 1;
+	align = *cur_ref + npc_priv.subbank_depth - 1;
+
+	if (align >= limit) {
+		*stop = true;
+		*cur_limit = limit;
+		return;
+	}
+
+	*cur_limit = align;
+}
+
+static int
+npc_subbank_iter(struct rvu *rvu, int key_type,
+		 int ref, int limit, int prio,
+		 int *cur_ref, int *cur_limit,
+		 bool *start, bool *stop)
+{
+	if (prio != NPC_MCAM_HIGHER_PRIO)
+		npc_subbank_iter_up(rvu, ref, limit,
+				    cur_ref, cur_limit,
+				    start, stop);
+	else
+		npc_subbank_iter_down(rvu, ref, limit,
+				      cur_ref, cur_limit,
+				      start, stop);
+
+	/* limit and ref should < bank_depth for x4 */
+	if (key_type == NPC_MCAM_KEY_X4) {
+		if (*cur_ref >= npc_priv.bank_depth)
+			return -EINVAL;
+
+		if (*cur_limit >= npc_priv.bank_depth)
+			return -EINVAL;
+	}
+	/* limit and ref should < 2 * bank_depth, for x2 */
+	if (*cur_ref >= 2 * npc_priv.bank_depth)
+		return -EINVAL;
+
+	if (*cur_limit >= 2 * npc_priv.bank_depth)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count,
+			bool maps_del)
+{
+	struct npc_subbank *sb;
+	u16 vidx, midx;
+	int sb_off, i;
+	bool ret;
+	int rc;
+
+	/* Check if we can dealloc indexes properly ? */
+	for (i = 0; i < count; i++) {
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]),
+						 &sb, &sb_off);
+		if (rc) {
+			dev_err(rvu->dev,
+				"Failed to free mcam idx=%u\n", mcam_idx[i]);
+			return rc;
+		}
+	}
+
+	for (i = 0; i < count; i++) {
+		if (npc_is_vidx(mcam_idx[i])) {
+			vidx = mcam_idx[i];
+			midx = npc_vidx2idx(vidx);
+		} else {
+			midx = mcam_idx[i];
+			vidx = npc_idx2vidx(midx);
+		}
+
+		if (midx >= npc_priv.bank_depth * npc_priv.num_banks) {
+			dev_err(rvu->dev,
+				"%s: Invalid mcam_idx=%u cannot be deleted\n",
+				__func__, mcam_idx[i]);
+			return -EINVAL;
+		}
+
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, midx,
+						 &sb, &sb_off);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Failed to find subbank info for vidx=%u\n",
+				__func__, vidx);
+			return rc;
+		}
+
+		ret = npc_subbank_free(rvu, sb, sb_off);
+		if (ret) {
+			dev_err(rvu->dev,
+				"%s: Failed to find subbank info for vidx=%u\n",
+				__func__, vidx);
+			return -EINVAL;
+		}
+
+		if (!maps_del)
+			continue;
+
+		rc = npc_del_from_pf_maps(rvu, midx);
+		if (rc)
+			return rc;
+
+		/* If there is no vidx mapping; continue */
+		if (vidx == midx)
+			continue;
+
+		rc = npc_vidx_maps_del_entry(rvu, vidx, NULL);
+		if (rc)
+			return rc;
+	}
+
+	return 0;
+}
+
+static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type,
+				       int ref, int limit, int prio,
+				       bool contig, int count,
+				       u16 *mcam_idx)
+{
+	struct npc_subbank *sb;
+	unsigned long *bmap;
+	int sb_off, off, rc;
+	int cnt = 0;
+	bool bitset;
+
+	if (prio != NPC_MCAM_HIGHER_PRIO) {
+		while (ref <= limit) {
+			/* Calculate subbank and subbank index */
+			rc =  npc_mcam_idx_2_subbank_idx(rvu, ref,
+							 &sb, &sb_off);
+			if (rc)
+				goto err;
+
+			/* If subbank is not suitable for requested key type
+			 * restart search from next subbank
+			 */
+			if (!npc_subbank_suits(sb, key_type)) {
+				ref = SB_ALIGN_UP(ref);
+				if (contig) {
+					rc = npc_idx_free(rvu, mcam_idx,
+							  cnt, false);
+					if (rc)
+						return rc;
+					cnt = 0;
+				}
+				continue;
+			}
+
+			mutex_lock(&sb->lock);
+
+			/* If subbank is free; mark it as used */
+			if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
+				rc =  __npc_subbank_mark_used(rvu, sb,
+							      key_type);
+				if (rc) {
+					mutex_unlock(&sb->lock);
+					dev_err(rvu->dev,
+						"%s:Error to add to use array\n",
+						__func__);
+					goto err;
+				}
+			}
+
+			/* Find correct bmap */
+			__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
+
+			/* if bit is already set, reset 'cnt' */
+			bitset = test_bit(off, bmap);
+			if (bitset) {
+				mutex_unlock(&sb->lock);
+				if (contig) {
+					rc = npc_idx_free(rvu, mcam_idx,
+							  cnt, false);
+					if (rc)
+						return rc;
+					cnt = 0;
+				}
+
+				ref++;
+				continue;
+			}
+
+			set_bit(off, bmap);
+			sb->free_cnt--;
+			mcam_idx[cnt++] = ref;
+			mutex_unlock(&sb->lock);
+
+			if (cnt == count)
+				return 0;
+			ref++;
+		}
+
+		/* Could not allocate request count slots */
+		goto err;
+	}
+	while (ref >= limit) {
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, ref,
+						 &sb, &sb_off);
+		if (rc)
+			goto err;
+
+		if (!npc_subbank_suits(sb, key_type)) {
+			ref = SB_ALIGN_DOWN(ref) - 1;
+			if (contig) {
+				rc = npc_idx_free(rvu, mcam_idx, cnt, false);
+				if (rc)
+					return rc;
+
+				cnt = 0;
+			}
+			continue;
+		}
+
+		mutex_lock(&sb->lock);
+
+		if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
+			rc =  __npc_subbank_mark_used(rvu, sb, key_type);
+			if (rc) {
+				mutex_unlock(&sb->lock);
+				dev_err(rvu->dev,
+					"%s:Error to add to use array\n",
+					__func__);
+				goto err;
+			}
+		}
+
+		__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
+		bitset = test_bit(off, bmap);
+		if (bitset) {
+			mutex_unlock(&sb->lock);
+			if (contig) {
+				rc = npc_idx_free(rvu, mcam_idx, cnt, false);
+				if (rc)
+					return rc;
+
+				cnt = 0;
+			}
+			ref--;
+			continue;
+		}
+
+		mcam_idx[cnt++] = ref;
+		sb->free_cnt--;
+		set_bit(off, bmap);
+		mutex_unlock(&sb->lock);
+
+		if (cnt == count)
+			return 0;
+		ref--;
+	}
+
+err:
+	rc = npc_idx_free(rvu, mcam_idx, cnt, false);
+	if (rc)
+		dev_err(rvu->dev,
+			"%s: Error happened while freeing cnt=%u indexes\n",
+			__func__, cnt);
+
+	return -ENOSPC;
+}
+
+static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb,
+				int key_type)
+{
+	int cnt, spd;
+
+	spd = npc_priv.subbank_depth;
+	mutex_lock(&sb->lock);
+
+	if (sb->flags & NPC_SUBBANK_FLAG_FREE)
+		cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd;
+	else
+		cnt = sb->free_cnt;
+
+	mutex_unlock(&sb->lock);
+	return cnt;
+}
+
+static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type,
+				 int ref, int limit, int prio,
+				 bool contig, int count,
+				 u16 *mcam_idx)
+{
+	struct npc_subbank *sb1, *sb2;
+	bool max_alloc, start, stop;
+	int r, l, sb_idx1, sb_idx2;
+	int tot = 0, rc;
+	int alloc_cnt;
+
+	max_alloc = !contig;
+
+	start = true;
+	stop = false;
+
+	/* Loop until we cross the ref/limit boundary */
+	while (!stop) {
+		rc = npc_subbank_iter(rvu, key_type, ref, limit, prio,
+				      &r, &l, &start, &stop);
+
+		dev_dbg(rvu->dev,
+			"%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n",
+			__func__, ref, limit, r, l,
+			start, stop, tot, count, rc);
+
+		if (rc)
+			goto err;
+
+		/* Find subbank and subbank index for ref */
+		rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1,
+						&sb_idx1);
+		if (rc)
+			goto err;
+
+		dev_dbg(rvu->dev,
+			"%s: ref subbank=%d off=%d\n",
+			__func__, sb1->idx, sb_idx1);
+
+		/* Skip subbank if it is not available for the keytype */
+		if (!npc_subbank_suits(sb1, key_type)) {
+			dev_dbg(rvu->dev,
+				"%s: not suitable sb=%d key_type=%d\n",
+				__func__, sb1->idx, key_type);
+			continue;
+		}
+
+		/* Find subbank and subbank index for limit */
+		rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2,
+						&sb_idx2);
+		if (rc)
+			goto err;
+
+		dev_dbg(rvu->dev,
+			"%s: limit subbank=%d off=%d\n",
+			__func__, sb_idx1, sb_idx2);
+
+		/* subbank of ref and limit should be same */
+		if (sb1 != sb2) {
+			dev_err(rvu->dev,
+				"%s: l(%d) and r(%d) are not in same subbank\n",
+				__func__, r, l);
+			goto err;
+		}
+
+		if (contig &&
+		    npc_subbank_free_cnt(rvu, sb1, key_type) < count) {
+			dev_dbg(rvu->dev, "%s: less count =%d\n",
+				__func__,
+				npc_subbank_free_cnt(rvu, sb1, key_type));
+			continue;
+		}
+
+		/* Try in one bank of a subbank */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb1, key_type,
+					r, l, prio, contig,
+					count - tot, mcam_idx + tot,
+					count - tot, max_alloc,
+					&alloc_cnt);
+
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n",
+			__func__, tot, alloc_cnt);
+
+		if (!rc && count == tot)
+			return 0;
+	}
+err:
+	dev_dbg(rvu->dev, "%s: Error to allocate\n",
+		__func__);
+
+	/* non contiguous allocation fails. We need to do clean up */
+	if (max_alloc) {
+		rc = npc_idx_free(rvu, mcam_idx, tot, false);
+		if (rc)
+			dev_err(rvu->dev,
+				"%s: failed to free %u indexes\n",
+				__func__, tot);
+	}
+
+	return -EFAULT;
+}
+
+/* Minimize allocation from bottom and top subbanks for noref allocations.
+ * Default allocations are ref based, and will be allocated from top
+ * subbanks (least priority subbanks). Since default allocation is at very
+ * early stage of kernel netdev probes, this subbanks will be moved to
+ * used subbanks list. This will pave a way for noref allocation from these
+ * used subbanks. Skip allocation for these top and bottom, and try free
+ * bank next. If none slot is available, come back and search in these
+ * subbanks.
+ */
+
+static int npc_subbank_restricted_idxs[2];
+static bool restrict_valid = true;
+
+static bool npc_subbank_restrict_usage(struct rvu *rvu, int index)
+{
+	int i;
+
+	if (!restrict_valid)
+		return false;
+
+	for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) {
+		if (index == npc_subbank_restricted_idxs[i])
+			return true;
+	}
+
+	return false;
+}
+
+static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig,
+				   int count, u16 *mcam_idx)
+{
+	struct npc_subbank *sb;
+	unsigned long index;
+	int tot = 0, rc;
+	bool max_alloc;
+	int alloc_cnt;
+	int idx, i;
+	void *val;
+
+	max_alloc = !contig;
+
+	/* Check used subbanks for free slots */
+	xa_for_each(&npc_priv.xa_sb_used, index, val) {
+		idx = xa_to_value(val);
+
+		/* Minimize allocation from restricted subbanks
+		 * in noref allocations.
+		 */
+		if (npc_subbank_restrict_usage(rvu, idx))
+			continue;
+
+		sb = &npc_priv.sb[idx];
+
+		/* Skip if not suitable subbank */
+		if (!npc_subbank_suits(sb, key_type))
+			continue;
+
+		if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
+			continue;
+
+		/* try in bank 0. Try passing ref and limit equal to
+		 * subbank boundaries
+		 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb, key_type,
+					sb->b0b, sb->b0t, 0,
+					contig, count - tot,
+					mcam_idx + tot,
+					count - tot,
+					max_alloc, &alloc_cnt);
+
+		/* Non contiguous allocation may allocate less than
+		 * requested 'count'.
+		 */
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		/* Successfully allocated */
+		if (!rc && count == tot)
+			return 0;
+
+		/* x4 entries can be allocated from bank 0 only */
+		if (key_type == NPC_MCAM_KEY_X4)
+			continue;
+
+		/* try in bank 1 for x2 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb, key_type,
+					sb->b1b, sb->b1t, 0,
+					contig, count - tot,
+					mcam_idx + tot,
+					count - tot, max_alloc,
+					&alloc_cnt);
+
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		if (!rc && count == tot)
+			return 0;
+	}
+
+	/* Allocate in free subbanks */
+	xa_for_each(&npc_priv.xa_sb_free, index, val) {
+		idx = xa_to_value(val);
+		sb = &npc_priv.sb[idx];
+
+		/* Minimize allocation from restricted subbanks
+		 * in noref allocations.
+		 */
+		if (npc_subbank_restrict_usage(rvu, idx))
+			continue;
+
+		if (!npc_subbank_suits(sb, key_type))
+			continue;
+
+		/* try in bank 0 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb, key_type,
+					sb->b0b, sb->b0t, 0,
+					contig, count - tot,
+					mcam_idx + tot,
+					count - tot,
+					max_alloc, &alloc_cnt);
+
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		/* Successfully allocated */
+		if (!rc && count == tot)
+			return 0;
+
+		/* x4 entries can be allocated from bank 0 only */
+		if (key_type == NPC_MCAM_KEY_X4)
+			continue;
+
+		/* try in bank 1 for x2 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb,
+					key_type, sb->b1b, sb->b1t, 0,
+					contig, count - tot,
+					mcam_idx + tot, count - tot,
+					max_alloc, &alloc_cnt);
+
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		if (!rc && count == tot)
+			return 0;
+	}
+
+	/* Allocate from restricted subbanks */
+	for (i = 0; restrict_valid &&
+	     (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) {
+		idx = npc_subbank_restricted_idxs[i];
+		sb = &npc_priv.sb[idx];
+
+		/* Skip if not suitable subbank */
+		if (!npc_subbank_suits(sb, key_type))
+			continue;
+
+		if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
+			continue;
+
+		/* try in bank 0. Try passing ref and limit equal to
+		 * subbank boundaries
+		 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb, key_type,
+					sb->b0b, sb->b0t, 0,
+					contig, count - tot,
+					mcam_idx + tot,
+					count - tot,
+					max_alloc, &alloc_cnt);
+
+		/* Non contiguous allocation may allocate less than
+		 * requested 'count'.
+		 */
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		/* Successfully allocated */
+		if (!rc && count == tot)
+			return 0;
+
+		/* x4 entries can be allocated from bank 0 only */
+		if (key_type == NPC_MCAM_KEY_X4)
+			continue;
+
+		/* try in bank 1 for x2 */
+		alloc_cnt = 0;
+		rc =  npc_subbank_alloc(rvu, sb, key_type,
+					sb->b1b, sb->b1t, 0,
+					contig, count - tot,
+					mcam_idx + tot,
+					count - tot, max_alloc,
+					&alloc_cnt);
+
+		tot += alloc_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
+			__func__, alloc_cnt, sb->idx, tot, count);
+
+		if (!rc && count == tot)
+			return 0;
+	}
+
+	/* non contiguous allocation fails. We need to do clean up */
+	if (max_alloc)
+		npc_idx_free(rvu, mcam_idx, tot, false);
+
+	dev_dbg(rvu->dev, "%s: non-contig allocation fails\n",
+		__func__);
+
+	return -EFAULT;
+}
+
+int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count)
+{
+	return npc_idx_free(rvu, mcam_idx, count, true);
+}
+
+int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type,
+			    int prio, u16 *mcam_idx, int ref, int limit,
+			    bool contig, int count, bool virt)
+{
+	bool defrag_candidate = false;
+	int i, eidx, rc, bd;
+	bool ref_valid;
+	u16 vidx;
+
+	bd = npc_priv.bank_depth;
+
+	/* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1
+	 * Here user wants to allocate 0th entry
+	 */
+	if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO &&
+	    count == 1) {
+		rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
+					   prio, contig, count, mcam_idx);
+
+		if (rc)
+			return rc;
+		goto add2map;
+	}
+
+	ref_valid = !!(limit || ref);
+	defrag_candidate = !ref_valid && !contig && virt;
+	if (!ref_valid) {
+		if (contig && count > npc_priv.subbank_depth)
+			goto try_noref_multi_subbank;
+
+		rc = npc_subbank_noref_alloc(rvu, key_type, contig,
+					     count, mcam_idx);
+		if (!rc)
+			goto add2map;
+
+try_noref_multi_subbank:
+		eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1;
+
+		if (prio == NPC_MCAM_HIGHER_PRIO)
+			rc = npc_multi_subbank_ref_alloc(rvu, key_type,
+							 eidx, 0,
+							 NPC_MCAM_HIGHER_PRIO,
+							 contig, count,
+							 mcam_idx);
+		else
+			rc = npc_multi_subbank_ref_alloc(rvu, key_type,
+							 0, eidx,
+							 NPC_MCAM_LOWER_PRIO,
+							 contig, count,
+							 mcam_idx);
+
+		if (!rc)
+			goto add2map;
+
+		return rc;
+	}
+
+	if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
+	    (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
+		dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
+			__func__, ref, limit);
+		return -EINVAL;
+	}
+
+	if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) ||
+	    (key_type == NPC_MCAM_KEY_X2 &&
+	     (ref >= 2 * bd || limit >= 2 * bd))) {
+		dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
+			__func__, ref, limit);
+		return -EINVAL;
+	}
+
+	if (contig && count > npc_priv.subbank_depth)
+		goto try_ref_multi_subbank;
+
+	rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
+				   prio, contig, count, mcam_idx);
+	if (!rc)
+		goto add2map;
+
+try_ref_multi_subbank:
+	rc = npc_multi_subbank_ref_alloc(rvu, key_type,
+					 ref, limit, prio,
+					 contig, count, mcam_idx);
+	if (!rc)
+		goto add2map;
+
+	return rc;
+
+add2map:
+	for (i = 0; i < count; i++) {
+		rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc);
+		if (rc)
+			goto err;
+
+		if (!defrag_candidate)
+			continue;
+
+		rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx);
+		if (rc) {
+			npc_del_from_pf_maps(rvu, mcam_idx[i]);
+			goto err;
+		}
+
+		/* Return vidx to caller */
+		mcam_idx[i] = vidx;
+	}
+
+	return 0;
+err:
+	for (int j = 0; j < i; j++) {
+		npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j]));
+
+		if (!defrag_candidate)
+			continue;
+
+		npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL);
+	}
+
+	return rc;
+
+}
+
+void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free,
+				 int *x4_free, int *sb_free)
+{
+	struct npc_subbank *sb;
+	int i;
+
+	/* Reset all stats to zero */
+	*x2_free = 0;
+	*x4_free = 0;
+	*sb_free = 0;
+
+	for (i = 0; i < npc_priv.num_subbanks; i++) {
+		sb = &npc_priv.sb[i];
+		mutex_lock(&sb->lock);
+
+		/* Count number of free subbanks */
+		if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
+			(*sb_free)++;
+			goto next;
+		}
+
+		/* Sumup x4 free count */
+		if (sb->key_type == NPC_MCAM_KEY_X4) {
+			(*x4_free) += sb->free_cnt;
+			goto next;
+		}
+
+		/* Sumup x2 free counts */
+		(*x2_free) += sb->free_cnt;
+next:
+		mutex_unlock(&sb->lock);
+	}
+}
+
+int
+rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu,
+				    struct msg_req *req,
+				    struct npc_cn20k_get_fcnt_rsp *rsp)
+{
+	npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2,
+				    &rsp->free_x4, &rsp->free_subbanks);
+	return 0;
+}
+
+int
+rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu,
+				       struct msg_req *req,
+				       struct npc_cn20k_get_kex_cfg_rsp *rsp)
+{
+	int extr, lt;
+
+	rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX);
+	rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX);
+
+	/* Get EXTRACTOR LID */
+	for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
+		rsp->intf_extr_lid[NIX_INTF_RX][extr] =
+			CN20K_GET_EXTR_LID(NIX_INTF_RX, extr);
+		rsp->intf_extr_lid[NIX_INTF_TX][extr] =
+			CN20K_GET_EXTR_LID(NIX_INTF_TX, extr);
+	}
+
+	/* Get EXTRACTOR LTYPE */
+	for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
+		for (lt = 0; lt < NPC_MAX_LT; lt++) {
+			rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] =
+				CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt);
+			rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] =
+				CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt);
+		}
+	}
+
+	memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN);
+	return 0;
+}
+
+static int *subbank_srch_order;
+
+static void npc_populate_restricted_idxs(int num_subbanks)
+{
+	npc_subbank_restricted_idxs[0] = num_subbanks - 1;
+	npc_subbank_restricted_idxs[1] = 0;
+}
+
+static int npc_create_srch_order(int cnt)
+{
+	int val = 0;
+
+	subbank_srch_order = kcalloc(cnt, sizeof(int),
+				     GFP_KERNEL);
+	if (!subbank_srch_order)
+		return -ENOMEM;
+
+	/* cnt(subbank depth) is always a power of 2. There is a check in
+	 * npc_priv_init() to check the same.
+	 */
+	for (int i = 0; i < cnt; i += 2) {
+		subbank_srch_order[i] = cnt / 2 - val - 1;
+		subbank_srch_order[i + 1] = cnt / 2 + 1 + val;
+		val++;
+	}
+
+	subbank_srch_order[cnt - 1] = cnt / 2;
+	return 0;
+}
+
+static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx)
+{
+	mutex_init(&sb->lock);
+
+	sb->b0b = idx * npc_priv.subbank_depth;
+	sb->b0t = sb->b0b + npc_priv.subbank_depth - 1;
+
+	sb->b1b = npc_priv.bank_depth + idx * npc_priv.subbank_depth;
+	sb->b1t = sb->b1b + npc_priv.subbank_depth - 1;
+
+	sb->flags = NPC_SUBBANK_FLAG_FREE;
+	sb->idx = idx;
+	sb->arr_idx = subbank_srch_order[idx];
+
+	dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n",
+		__func__, sb->idx, sb->arr_idx);
+
+	/* Keep first and last subbank at end of free array; so that
+	 * it will be used at last
+	 */
+	xa_store(&npc_priv.xa_sb_free, sb->arr_idx,
+		 xa_mk_value(sb->idx), GFP_KERNEL);
+}
+
+static int npc_pcifunc_map_create(struct rvu *rvu)
+{
+	int pf, vf, numvfs;
+	int cnt = 0;
+	u16 pcifunc;
+	u64 cfg;
+
+	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
+		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
+		numvfs = (cfg >> 12) & 0xFF;
+
+		/* Skip not enabled PFs */
+		if (!(cfg & BIT_ULL(20)))
+			goto chk_vfs;
+
+		/* If Admin function, check on VFs */
+		if (cfg & BIT_ULL(21))
+			goto chk_vfs;
+
+		pcifunc = pf << 9;
+
+		xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc,
+			 xa_mk_value(cnt), GFP_KERNEL);
+
+		cnt++;
+
+chk_vfs:
+		for (vf = 0; vf < numvfs; vf++) {
+			pcifunc = (pf << 9) | (vf + 1);
+
+			xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc,
+				 xa_mk_value(cnt), GFP_KERNEL);
+			cnt++;
+		}
+	}
+
+	return cnt;
+}
+
+struct npc_defrag_node {
+	u8 idx;
+	u8 key_type;
+	bool valid;
+	bool refs;
+	u16 free_cnt;
+	u16 vidx_cnt;
+	u16 *vidx;
+	struct list_head list;
+};
+
+static bool npc_defrag_skip_restricted_sb(int sb_id)
+{
+	int i;
+
+	if (!restrict_valid)
+		return false;
+
+	for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++)
+		if (sb_id == npc_subbank_restricted_idxs[i])
+			return true;
+	return false;
+}
+
+/* Find subbank with minimum number of virtual indexes */
+static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh)
+{
+	struct npc_defrag_node *node, *tnode = NULL;
+	int min = INT_MAX;
+
+	list_for_each_entry(node, lh, list) {
+		if (!node->valid)
+			continue;
+
+		/* if subbank has ref allocated mcam indexes, that subbank
+		 * is not a good candidate to move out indexes.
+		 */
+		if (node->refs)
+			continue;
+
+		if (min > node->vidx_cnt) {
+			min = node->vidx_cnt;
+			tnode = node;
+		}
+	}
+
+	return tnode;
+}
+
+/* Find subbank with maximum number of free spaces */
+static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh)
+{
+	struct npc_defrag_node *node, *tnode = NULL;
+	int max = INT_MIN;
+
+	list_for_each_entry(node, lh, list) {
+		if (!node->valid)
+			continue;
+
+		if (max < node->free_cnt) {
+			max = node->free_cnt;
+			tnode = node;
+		}
+	}
+
+	return tnode;
+}
+
+static int npc_defrag_alloc_free_slots(struct rvu *rvu,
+				       struct npc_defrag_node *f,
+				       int cnt, u16 *save)
+{
+	int alloc_cnt1, alloc_cnt2;
+	struct npc_subbank *sb;
+	int rc, sb_off, i;
+	bool deleted;
+
+	sb = &npc_priv.sb[f->idx];
+
+	alloc_cnt1 = 0;
+	alloc_cnt2 = 0;
+
+	rc = __npc_subbank_alloc(rvu, sb,
+				 NPC_MCAM_KEY_X2, sb->b0b,
+				 sb->b0t,
+				 NPC_MCAM_LOWER_PRIO,
+				 false, cnt, save, cnt, true,
+				 &alloc_cnt1);
+	if (alloc_cnt1 < cnt) {
+		rc = __npc_subbank_alloc(rvu, sb,
+					 NPC_MCAM_KEY_X2, sb->b1b,
+					 sb->b1t,
+					 NPC_MCAM_LOWER_PRIO,
+					 false, cnt - alloc_cnt1,
+					 save + alloc_cnt1,
+					 cnt - alloc_cnt1,
+					 true, &alloc_cnt2);
+	}
+
+	if (alloc_cnt1 + alloc_cnt2 != cnt) {
+		dev_err(rvu->dev,
+			"%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n",
+			__func__, cnt, alloc_cnt1, alloc_cnt2);
+		goto fail_free_alloc;
+	}
+	return 0;
+
+fail_free_alloc:
+	for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) {
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, save[i],
+						 &sb, &sb_off);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Error to find subbank for mcam idx=%u\n",
+				__func__, save[i]);
+			break;
+		}
+
+		deleted = __npc_subbank_free(rvu, sb, sb_off);
+		if (!deleted) {
+			dev_err(rvu->dev,
+				"%s: Error to free mcam idx=%u\n",
+				__func__, save[i]);
+			break;
+		}
+	}
+
+	return rc;
+}
+
+static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx,
+				      u16 new_midx, u16 vidx)
+{
+	struct npc_defrag_show_node *node;
+
+	node = kcalloc(1, sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	node->old_midx = old_midx;
+	node->new_midx = new_midx;
+	node->vidx = vidx;
+	INIT_LIST_HEAD(&node->list);
+
+	mutex_lock(&npc_priv.lock);
+	list_add_tail(&node->list, &npc_priv.defrag_lh);
+	mutex_unlock(&npc_priv.lock);
+
+	return 0;
+}
+
+static
+int npc_defrag_move_vdx_to_free(struct rvu *rvu,
+				struct npc_defrag_node *f,
+				struct npc_defrag_node *v,
+				int cnt, u16 *save)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	int i, vidx_cnt, rc, sb_off;
+	u16 new_midx, old_midx, vidx;
+	struct npc_subbank *sb;
+	bool deleted;
+	u16 pcifunc;
+	int blkaddr;
+	void *map;
+	u8 bank;
+	u16 midx;
+	u64 stats;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+
+	vidx_cnt = v->vidx_cnt;
+	for (i = 0; i < cnt; i++) {
+		vidx = v->vidx[vidx_cnt - i - 1];
+		old_midx = npc_vidx2idx(vidx);
+		new_midx = save[cnt - i - 1];
+
+		dev_dbg(rvu->dev,
+			"%s: Moving %u ---> %u  (vidx=%u)\n",
+			__func__,
+			old_midx, new_midx, vidx);
+
+		rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx);
+		if (rc)
+			dev_err(rvu->dev,
+				"%s: Error happened to add to show list vidx=%u\n",
+				__func__, vidx);
+
+		/* Modify vidx to point to new mcam idx */
+		rc = npc_vidx_maps_modify(rvu, vidx, new_midx);
+		if (rc)
+			return rc;
+
+		midx = old_midx % mcam->banksize;
+		bank = old_midx / mcam->banksize;
+		stats = rvu_read64(rvu, blkaddr,
+				   NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx,
+								      bank));
+
+		npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false);
+		npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx);
+		npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true);
+
+		midx = new_midx % mcam->banksize;
+		bank = new_midx / mcam->banksize;
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank),
+			    stats);
+
+		/* Free the old mcam idx */
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, old_midx,
+						 &sb, &sb_off);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Unable to calculate subbank off for mcamidx=%u\n",
+				__func__, old_midx);
+			return rc;
+		}
+
+		deleted = __npc_subbank_free(rvu, sb, sb_off);
+		if (!deleted) {
+			dev_err(rvu->dev,
+				"%s:  Failed to free mcamidx=%u sb=%u sb_off=%u\n",
+				__func__, old_midx, sb->idx, sb_off);
+			return -EFAULT;
+		}
+
+		/* save pcifunc */
+		map = xa_load(&npc_priv.xa_idx2pf_map, old_midx);
+		pcifunc = xa_to_value(map);
+
+		/* delete from pf maps */
+		rc =  npc_del_from_pf_maps(rvu, old_midx);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s:  Failed to delete pf maps for mcamidx=%u\n",
+				__func__, old_midx);
+			return rc;
+		}
+
+		/* add new mcam_idx to pf map */
+		rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s:  Failed to add pf maps for mcamidx=%u\n",
+				__func__, new_midx);
+			return rc;
+		}
+
+		/* Remove from mcam maps */
+		mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP;
+		mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP;
+		npc_mcam_clear_bit(mcam, old_midx);
+
+		mcam->entry2pfvf_map[new_midx] = pcifunc;
+		/* Counter is not preserved */
+		mcam->entry2cntr_map[new_midx] = new_midx;
+		npc_mcam_set_bit(mcam, new_midx);
+
+		/* Mark as invalid */
+		v->vidx[vidx_cnt - i - 1] = -1;
+		save[cnt - i - 1] = -1;
+
+		f->free_cnt--;
+		v->vidx_cnt--;
+	}
+
+	return 0;
+}
+
+static int npc_defrag_process(struct rvu *rvu, struct list_head *lh)
+{
+	struct npc_defrag_node *v = NULL;
+	struct npc_defrag_node *f = NULL;
+	int rc = 0, cnt;
+	u16 *save;
+
+	while (1) {
+		/* Find subbank with minimum vidx */
+		if (!v) {
+			v = npc_subbank_min_vidx(lh);
+			if (!v)
+				break;
+		}
+
+		/* Find subbank with maximum free slots */
+		if (!f) {
+			f = npc_subbank_max_free(lh);
+			if (!f)
+				break;
+		}
+
+		if (!v->vidx_cnt) {
+			list_del_init(&v->list);
+			v = NULL;
+			continue;
+		}
+
+		if (!f->free_cnt) {
+			list_del_init(&f->list);
+			f = NULL;
+			continue;
+		}
+
+		/* If both subbanks are same, choose vidx and
+		 * search for free list again
+		 */
+		if (f == v) {
+			list_del_init(&f->list);
+			f = NULL;
+			continue;
+		}
+
+		/* Calculate minimum free slots needs to be allocated */
+		cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt :
+			f->free_cnt;
+
+		dev_dbg(rvu->dev,
+			"%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n",
+			__func__, cnt, f->free_cnt, f->idx,
+			v->vidx_cnt, v->idx);
+
+		/* Allocate an array to store newly allocated
+		 * free slots (mcam indexes)
+		 */
+		save = kcalloc(cnt, sizeof(*save), GFP_KERNEL);
+		if (!save) {
+			rc = -ENOMEM;
+			goto err;
+		}
+
+		/* Alloc free slots for existing vidx */
+		rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save);
+		if (rc) {
+			kfree(save);
+			goto err;
+		}
+
+		/* Move vidx to free slots; update pf_map and vidx maps,
+		 * and free existing vidx mcam slots
+		 */
+		rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save);
+		if (rc) {
+			kfree(save);
+			goto err;
+		}
+
+		kfree(save);
+
+		if (!f->free_cnt) {
+			list_del_init(&f->list);
+			f = NULL;
+		}
+
+		if (!v->vidx_cnt) {
+			list_del_init(&v->list);
+			v = NULL;
+		}
+	}
+
+err:
+	/* Whole defragmentation process is done within locks. if there
+	 * is an error, it would be hard to roll back as index remove/add
+	 * can fail again if it failed before. This would mean that there
+	 * is bug in the index management algorithm.
+	 * Return from here than rolling back.
+	 */
+	return rc;
+}
+
+static void npc_defrag_list_clear(void)
+{
+	struct npc_defrag_show_node *node, *next;
+
+	mutex_lock(&npc_priv.lock);
+	list_for_each_entry_safe(node, next, &npc_priv.defrag_lh, list) {
+		list_del_init(&node->list);
+		kfree(node);
+	}
+
+	mutex_unlock(&npc_priv.lock);
+}
+
+static void npc_lock_all_subbank(void)
+{
+	int i;
+
+	for (i = 0; i < npc_priv.num_subbanks; i++)
+		mutex_lock(&npc_priv.sb[i].lock);
+}
+
+static void npc_unlock_all_subbank(void)
+{
+	int i;
+
+	for (i = npc_priv.num_subbanks - 1; i >= 0; i--)
+		mutex_unlock(&npc_priv.sb[i].lock);
+}
+
+/* Only non-ref non-contigous mcam indexes
+ * are picked for defrag process
+ */
+int npc_cn20k_defrag(struct rvu *rvu)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	struct npc_defrag_node *node, *tnode;
+	struct list_head x4lh, x2lh, *lh;
+	int rc = 0, i, sb_off, tot;
+	struct npc_subbank *sb;
+	unsigned long index;
+	void *map;
+	u16 midx;
+
+	/* Free previous show list */
+	npc_defrag_list_clear();
+
+	INIT_LIST_HEAD(&x4lh);
+	INIT_LIST_HEAD(&x2lh);
+
+	node = kcalloc(npc_priv.num_subbanks, sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	/* Lock mcam */
+	mutex_lock(&mcam->lock);
+	npc_lock_all_subbank();
+
+	/* Fill in node with subbank properties */
+	for (i = 0; i < npc_priv.num_subbanks; i++) {
+		sb = &npc_priv.sb[i];
+
+		node[i].idx = i;
+		node[i].key_type = sb->key_type;
+		node[i].free_cnt = sb->free_cnt;
+		node[i].vidx = kcalloc(npc_priv.subbank_depth * 2,
+				       sizeof(*node[i].vidx),
+				       GFP_KERNEL);
+		if (!node[i].vidx) {
+			rc = -ENOMEM;
+			goto free_vidx;
+		}
+
+		/* If subbank is empty, dont include it in defrag
+		 * process
+		 */
+		if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
+			node[i].valid = false;
+			continue;
+		}
+
+		if (npc_defrag_skip_restricted_sb(i)) {
+			node[i].valid = false;
+			continue;
+		}
+
+		node[i].valid = true;
+		INIT_LIST_HEAD(&node[i].list);
+
+		/* Add node to x2 or x4 list */
+		lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh;
+		list_add_tail(&node[i].list, lh);
+	}
+
+	/* Filling vidx[] array with all vidx in that subbank */
+	xa_for_each_start(&npc_priv.xa_vidx2idx_map, index, map,
+			  npc_priv.bank_depth * 2) {
+		midx = xa_to_value(map);
+		rc =  npc_mcam_idx_2_subbank_idx(rvu, midx,
+						 &sb, &sb_off);
+		if (rc) {
+			dev_err(rvu->dev,
+				"%s: Error to get mcam_idx for vidx=%lu\n",
+				__func__, index);
+			goto free_vidx;
+		}
+
+		tnode = &node[sb->idx];
+		tnode->vidx[tnode->vidx_cnt] = index;
+		tnode->vidx_cnt++;
+	}
+
+	/* Mark all subbank which has ref allocation */
+	for (i = 0; i < npc_priv.num_subbanks; i++) {
+		tnode = &node[i];
+
+		if (!tnode->valid)
+			continue;
+
+		tot = (tnode->key_type == NPC_MCAM_KEY_X2) ?
+			npc_priv.subbank_depth * 2 : npc_priv.subbank_depth;
+
+		if (node[i].vidx_cnt != tot - tnode->free_cnt)
+			tnode->refs = true;
+	}
+
+	rc =  npc_defrag_process(rvu, &x2lh);
+	if (rc)
+		goto free_vidx;
+
+	rc =  npc_defrag_process(rvu, &x4lh);
+	if (rc)
+		goto free_vidx;
+
+free_vidx:
+	npc_unlock_all_subbank();
+	mutex_unlock(&mcam->lock);
+	for (i = 0; i < npc_priv.num_subbanks; i++)
+		kfree(node[i].vidx);
+	kfree(node);
+	return rc;
+}
+
+int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req,
+				struct msg_rsp *rsp)
+{
+	return npc_cn20k_defrag(rvu);
+}
+
+int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
+				u16 *mcast, u16 *promisc, u16 *ucast)
+{
+	u16 *ptr[4] = {promisc, mcast, bcast, ucast};
+	unsigned long idx;
+	bool set = false;
+	void *val;
+	int i, j;
+
+	if (!npc_priv.init_done)
+		return 0;
+
+	if (is_lbk_vf(rvu, pcifunc)) {
+		if (!ptr[0])
+			return -EINVAL;
+
+		idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
+		val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx);
+		if (!val) {
+			pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
+				 __func__,
+				 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
+				 pcifunc);
+
+			*ptr[0] = USHRT_MAX;
+			return -ESRCH;
+		}
+
+		*ptr[0] = xa_to_value(val);
+		return 0;
+	}
+
+	if (is_vf(pcifunc)) {
+		if (!ptr[3])
+			return -EINVAL;
+
+		idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
+		val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx);
+		if (!val) {
+			pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
+				 __func__,
+				 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
+				 pcifunc);
+
+			*ptr[3] = USHRT_MAX;
+			return -ESRCH;
+		}
+
+		*ptr[3] = xa_to_value(val);
+		return 0;
+	}
+
+	for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++,
+	     j++) {
+		if (!ptr[j])
+			continue;
+
+		idx = NPC_DFT_RULE_ID_MK(pcifunc, i);
+		val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx);
+		if (!val) {
+			pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
+				 __func__,
+				 npc_dft_rule_name[i], pcifunc);
+
+			*ptr[j] = USHRT_MAX;
+			continue;
+		}
+
+		*ptr[j] = xa_to_value(val);
+		set = true;
+	}
+
+	return  set ? 0 : -ESRCH;
+}
+
+int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req,
+				      struct npc_get_pfl_info_rsp *rsp)
+{
+	if (!is_cn20k(rvu->pdev)) {
+		dev_err(rvu->dev, "Mbox support is only for cn20k\n");
+		return -EOPNOTSUPP;
+	}
+
+	rsp->kw_type = npc_priv.kw;
+	rsp->x4_slots = npc_priv.bank_depth;
+	return 0;
+}
+
+int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu,
+				     struct npc_get_num_kws_req *req,
+				     struct npc_get_num_kws_rsp *rsp)
+{
+	u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
+	u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
+	struct rvu_npc_mcam_rule dummy = { 0 };
+	struct mcam_entry_mdata mdata = { };
+	struct npc_install_flow_req *fl;
+	int i, cnt = 0, blkaddr;
+
+	if (!is_cn20k(rvu->pdev)) {
+		dev_err(rvu->dev, "Mbox support is only for cn20k\n");
+		return -EOPNOTSUPP;
+	}
+
+	fl = &req->fl;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0) {
+		dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
+		return NPC_MCAM_INVALID_REQ;
+	}
+
+	mdata.kw = kw;
+	mdata.kw_mask = kw_mask;
+
+	npc_update_flow(rvu, &mdata, fl->features, &fl->packet,
+			&fl->mask, &dummy, fl->intf, blkaddr);
+
+	/* Find the most significant word valid. Traverse from
+	 * MSB to LSB, check if cam0 or cam1 is set
+	 */
+	for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) {
+		if (kw[i] || kw_mask[i]) {
+			cnt = i + 1;
+			break;
+		}
+	}
+
+	rsp->kws = cnt;
+
+	return 0;
+}
+
+int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req,
+					 struct npc_get_dft_rl_idxs_rsp *rsp)
+{
+	u16 bcast, mcast, promisc, ucast;
+	u16 pcifunc;
+	int rc;
+
+	if (!is_cn20k(rvu->pdev)) {
+		dev_err(rvu->dev, "Mbox support is only for cn20k\n");
+		return -EOPNOTSUPP;
+	}
+
+	pcifunc = req->hdr.pcifunc;
+
+	rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast,
+					 &promisc, &ucast);
+	if (rc)
+		return rc;
+
+	rsp->bcast = bcast;
+	rsp->mcast = mcast;
+	rsp->promisc = promisc;
+	rsp->ucast = ucast;
+	return 0;
+}
+
+static bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
+{
+	return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) ||
+		is_lbk_vf(rvu, pcifunc);
+}
+
+void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc)
+{
+	struct npc_mcam_free_entry_req free_req = { 0 };
+	unsigned long index;
+	struct msg_rsp rsp;
+	u16 ptr[4];
+	int rc, i;
+	void *map;
+
+	if (!npc_priv.init_done)
+		return;
+
+	if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
+		dev_dbg(rvu->dev,
+			"%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
+			__func__, pcifunc);
+		return;
+	}
+
+	rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1],
+					 &ptr[2], &ptr[3]);
+	if (rc)
+		return;
+
+	/* LBK */
+	if (is_lbk_vf(rvu, pcifunc)) {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
+		map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index);
+		if (!map)
+			dev_dbg(rvu->dev,
+				"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
+				__func__,
+				npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
+				pcifunc);
+
+		goto free_rules;
+	}
+
+	/* VF */
+	if (is_vf(pcifunc)) {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
+		map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index);
+		if (!map)
+			dev_dbg(rvu->dev,
+				"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
+				__func__,
+				npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
+				pcifunc);
+
+		goto free_rules;
+	}
+
+	/* PF */
+	for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++)  {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, i);
+		map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index);
+		if (!map)
+			dev_dbg(rvu->dev,
+				"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
+				__func__, npc_dft_rule_name[i],
+				pcifunc);
+	}
+
+free_rules:
+
+	free_req.hdr.pcifunc = pcifunc;
+	free_req.all = 1;
+	rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp);
+	if (rc)
+		dev_err(rvu->dev,
+			"%s: Error deleting default entries (pcifunc=%#x\n",
+			__func__, pcifunc);
+}
+
+int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc)
+{
+	struct npc_mcam_free_entry_req free_req = { 0 };
+	u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc;
+	struct npc_mcam_alloc_entry_req req = { 0 };
+	struct npc_mcam_alloc_entry_rsp rsp = { 0 };
+	int ret, eidx, i, k, pf, cnt;
+	struct rvu_pfvf *pfvf;
+	unsigned long index;
+	struct msg_rsp free_rsp;
+	u16 b, m, p, u;
+
+	if (!npc_priv.init_done)
+		return 0;
+
+	if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
+		dev_dbg(rvu->dev,
+			"%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
+			__func__, pcifunc);
+		return 0;
+	}
+
+	/* Check if default rules are already alloced for this pcifunc */
+	ret =  npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u);
+	if (!ret) {
+		dev_dbg(rvu->dev,
+			"%s: default rules are already installed (pcifunc=%#x)\n",
+			__func__, pcifunc);
+		dev_dbg(rvu->dev,
+			"%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n",
+			__func__, b, m, p, u);
+		return 0;
+	}
+
+	/* Set ref index as lowest priority index */
+	eidx = 2 * npc_priv.bank_depth - 1;
+
+	/* Install only UCAST for VF */
+	cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx);
+
+	/* For VF pcifunc, allocate default mcam indexes by taking
+	 * ref as PF's ucast index.
+	 */
+	if (is_vf(pcifunc)) {
+		pf = rvu_get_pf(rvu->pdev, pcifunc);
+		pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT;
+
+		/* Get PF's ucast entry index */
+		ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL,
+						  NULL, NULL, &pf_ucast);
+
+		/* There is no PF rules installed; and VF installation comes
+		 * first. PF may come later.
+		 * TODO: Install PF rules before installing VF rules.
+		 */
+
+		/* Set PF's ucast as ref entry */
+		if (!ret)
+			eidx = pf_ucast;
+	}
+
+	pfvf = rvu_get_pfvf(rvu, pcifunc);
+	pfvf->hw_prio = NPC_DFT_RULE_PRIO;
+
+	req.contig = false;
+	req.ref_prio = NPC_MCAM_HIGHER_PRIO;
+	req.ref_entry = eidx;
+	req.kw_type = NPC_MCAM_KEY_X2;
+	req.count = cnt;
+	req.hdr.pcifunc = pcifunc;
+
+	ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
+
+	/* successfully allocated index */
+	if (!ret) {
+		/* Copy indexes to local array */
+		for (i = 0; i < cnt; i++)
+			mcam_idx[i] = rsp.entry_list[i];
+
+		goto chk_sanity;
+	}
+
+	/* If there is no slots available and request is for PF,
+	 * return error.
+	 */
+	if (!is_vf(pcifunc)) {
+		dev_err(rvu->dev,
+			"%s: Default index allocation failed for pcifunc=%#x\n",
+			__func__, pcifunc);
+		return ret;
+	}
+
+	/* We could not find an index with higher priority index for VF.
+	 * Find rule with lower priority index and set hardware priority
+	 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority)
+	 */
+	req.contig = false;
+	req.kw_type = NPC_MCAM_KEY_X2;
+	req.count = cnt;
+	req.hdr.pcifunc = pcifunc;
+	req.ref_prio = NPC_MCAM_LOWER_PRIO;
+	req.ref_entry = eidx + 1;
+	ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
+	if (ret) {
+		dev_err(rvu->dev,
+			"%s: Default index allocation failed for pcifunc=%#x\n",
+			__func__, pcifunc);
+		return ret;
+	}
+
+	/* Copy indexes to local array */
+	for (i = 0; i < cnt; i++)
+		mcam_idx[i] = rsp.entry_list[i];
+
+	pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1;
+
+chk_sanity:
+	/* LBK */
+	if (is_lbk_vf(rvu, pcifunc)) {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
+		ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index,
+				xa_mk_value(mcam_idx[0]), GFP_KERNEL);
+		if (ret) {
+			dev_err(rvu->dev,
+				"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
+				__func__,
+				npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
+				pcifunc);
+			goto err;
+		}
+
+		goto done;
+	}
+
+	/* VF */
+	if (is_vf(pcifunc)) {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
+		ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index,
+				xa_mk_value(mcam_idx[0]), GFP_KERNEL);
+		if (ret) {
+			dev_err(rvu->dev,
+				"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
+				__func__,
+				npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
+				pcifunc);
+			goto err;
+		}
+
+		goto done;
+	}
+
+	/* PF */
+	for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID &&
+	     k < cnt; i++, k++) {
+		index = NPC_DFT_RULE_ID_MK(pcifunc, i);
+		ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index,
+				xa_mk_value(mcam_idx[k]), GFP_KERNEL);
+		if (ret) {
+			dev_err(rvu->dev,
+				"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
+				__func__, npc_dft_rule_name[i],
+				pcifunc);
+			for (int p = NPC_DFT_RULE_START_ID; p < i; p++) {
+				index = NPC_DFT_RULE_ID_MK(pcifunc, p);
+				xa_erase(&npc_priv.xa_pf2dfl_rmap, index);
+			}
+			goto err;
+		}
+	}
+
+done:
+	return 0;
+
+err:
+	free_req.hdr.pcifunc = pcifunc;
+	free_req.all = 1;
+	ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
+	if (ret)
+		dev_err(rvu->dev,
+			"%s: Error deleting default entries (pcifunc=%#x\n",
+			__func__, pcifunc);
+
+	return -EFAULT;
+}
+
+static int npc_priv_init(struct rvu *rvu)
+{
+	struct npc_mcam *mcam = &rvu->hw->mcam;
+	int blkaddr, num_banks, bank_depth;
+	int num_subbanks, subbank_depth;
+	u64 npc_const1, npc_const2 = 0;
+	struct npc_subbank *sb;
+	u64 cfg;
+	int i;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0) {
+		dev_err(rvu->dev, "%s: NPC block not implemented\n",
+			__func__);
+		return -ENODEV;
+	}
+
+	npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1);
+	if (npc_const1 & BIT_ULL(63))
+		npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2);
+
+	num_banks = mcam->banks;
+	bank_depth = mcam->banksize;
+
+	num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2);
+	if (!num_subbanks) {
+		dev_err(rvu->dev, "Number of subbanks is zero\n");
+		return -EFAULT;
+	}
+
+	if (num_subbanks & (num_subbanks - 1)) {
+		dev_err(rvu->dev,
+			"subbanks cnt(%u) should be a power of 2\n",
+			num_subbanks);
+		return -EINVAL;
+	}
+
+	npc_priv.num_subbanks = num_subbanks;
+
+	subbank_depth =	bank_depth / num_subbanks;
+
+	npc_priv.bank_depth = bank_depth;
+	npc_priv.subbank_depth = subbank_depth;
+
+	/* Get kex configured key size */
+	cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0));
+	npc_priv.kw = FIELD_GET(GENMASK_ULL(34, 32), cfg);
+
+	dev_info(rvu->dev,
+		 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n",
+		 num_banks, bank_depth, num_subbanks, subbank_depth,
+		 npc_kw_name[npc_priv.kw]);
+
+	npc_priv.sb = kcalloc(num_subbanks, sizeof(struct npc_subbank),
+			      GFP_KERNEL);
+	if (!npc_priv.sb)
+		return -ENOMEM;
+
+	xa_init_flags(&npc_priv.xa_sb_used, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_sb_free, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_idx2pf_map, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_pf_map, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_pf2dfl_rmap, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_idx2vidx_map, XA_FLAGS_ALLOC);
+	xa_init_flags(&npc_priv.xa_vidx2idx_map, XA_FLAGS_ALLOC);
+
+	if (npc_create_srch_order(num_subbanks))
+		goto fail1;
+
+	npc_populate_restricted_idxs(num_subbanks);
+
+	/* Initialize subbanks */
+	for (i = 0, sb = npc_priv.sb; i < num_subbanks; i++, sb++)
+		npc_subbank_init(rvu, sb, i);
+
+	/* Get number of pcifuncs in the system */
+	npc_priv.pf_cnt = npc_pcifunc_map_create(rvu);
+	npc_priv.xa_pf2idx_map = kcalloc(npc_priv.pf_cnt,
+					 sizeof(struct xarray),
+					 GFP_KERNEL);
+	if (!npc_priv.xa_pf2idx_map)
+		goto fail2;
+
+	for (i = 0; i < npc_priv.pf_cnt; i++)
+		xa_init_flags(&npc_priv.xa_pf2idx_map[i], XA_FLAGS_ALLOC);
+
+	INIT_LIST_HEAD(&npc_priv.defrag_lh);
+	mutex_init(&npc_priv.lock);
+
+	return 0;
+
+fail2:
+	kfree(subbank_srch_order);
+	subbank_srch_order = NULL;
+
+fail1:
+	xa_destroy(&npc_priv.xa_sb_used);
+	xa_destroy(&npc_priv.xa_sb_free);
+	xa_destroy(&npc_priv.xa_idx2pf_map);
+	xa_destroy(&npc_priv.xa_pf_map);
+	xa_destroy(&npc_priv.xa_pf2dfl_rmap);
+	xa_destroy(&npc_priv.xa_idx2vidx_map);
+	xa_destroy(&npc_priv.xa_vidx2idx_map);
+	kfree(npc_priv.sb);
+	npc_priv.sb = NULL;
+	return -ENOMEM;
+}
+
+void npc_cn20k_deinit(struct rvu *rvu)
+{
+	int i;
+
+	xa_destroy(&npc_priv.xa_sb_used);
+	xa_destroy(&npc_priv.xa_sb_free);
+	xa_destroy(&npc_priv.xa_idx2pf_map);
+	xa_destroy(&npc_priv.xa_pf_map);
+	xa_destroy(&npc_priv.xa_pf2dfl_rmap);
+	xa_destroy(&npc_priv.xa_idx2vidx_map);
+	xa_destroy(&npc_priv.xa_vidx2idx_map);
+
+	for (i = 0; i < npc_priv.pf_cnt; i++)
+		xa_destroy(&npc_priv.xa_pf2idx_map[i]);
+
+	kfree(npc_priv.xa_pf2idx_map);
+	/* No need to destroy mutex lock as it is
+	 * part of subbank structure
+	 */
+	kfree(npc_priv.sb);
+	kfree(subbank_srch_order);
+}
+
+static int npc_setup_mcam_section(struct rvu *rvu, int key_type)
+{
+	int blkaddr, sec;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+	if (blkaddr < 0) {
+		dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
+		return -ENODEV;
+	}
+
+	for (sec = 0; sec < npc_priv.num_subbanks; sec++)
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type);
+
+	return 0;
+}
+
+int npc_cn20k_init(struct rvu *rvu)
+{
+	int err;
+
+	err = npc_priv_init(rvu);
+	if (err) {
+		dev_err(rvu->dev, "%s: Error to init\n",
+			__func__);
+		return err;
+	}
+
+	err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2);
+	if (err) {
+		dev_err(rvu->dev, "%s: mcam section configuration failure\n",
+			__func__);
+		return err;
+	}
+
+	npc_priv.init_done = true;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
new file mode 100644
index 0000000000000..815d0b257a7e1
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
@@ -0,0 +1,340 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell RVU Admin Function driver
+ *
+ * Copyright (C) 2026 Marvell.
+ *
+ */
+
+#ifndef NPC_CN20K_H
+#define NPC_CN20K_H
+
+#define MKEX_CN20K_SIGN	0x19bbfdbd160
+
+#define MAX_NUM_BANKS 2
+#define MAX_NUM_SUB_BANKS 32
+#define MAX_SUBBANK_DEPTH 256
+
+/* strtoull of "mkexprof" with base:36 */
+#define MKEX_END_SIGN  0xdeadbeef
+
+#define NPC_CN20K_BYTESM GENMASK_ULL(18, 16)
+#define NPC_CN20K_PARSE_NIBBLE GENMASK_ULL(22, 0)
+#define NPC_CN20K_TOTAL_NIBBLE 23
+
+#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg)	\
+	rvu_write64(rvu, BLKADDR_NPC,	\
+		    NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg)
+
+#define CN20K_GET_KEX_CFG(intf)	\
+	rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf))
+
+#define CN20K_GET_EXTR_LID(intf, extr)	\
+	rvu_read64(rvu, BLKADDR_NPC,	\
+		   NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr))
+
+#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg)	\
+	rvu_write64(rvu, BLKADDR_NPC,	\
+		    NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg)
+
+#define CN20K_GET_EXTR_LT(intf, extr, ltype)	\
+	rvu_read64(rvu, BLKADDR_NPC,	\
+		   NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype))
+
+/* NPC_PARSE_KEX_S nibble definitions for each field */
+#define NPC_CN20K_PARSE_NIBBLE_CHAN GENMASK_ULL(2, 0)
+#define NPC_CN20K_PARSE_NIBBLE_ERRLEV BIT_ULL(3)
+#define NPC_CN20K_PARSE_NIBBLE_ERRCODE GENMASK_ULL(5, 4)
+#define NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST BIT_ULL(6)
+#define NPC_CN20K_PARSE_NIBBLE_LA_FLAGS BIT_ULL(7)
+#define NPC_CN20K_PARSE_NIBBLE_LA_LTYPE BIT_ULL(8)
+#define NPC_CN20K_PARSE_NIBBLE_LB_FLAGS BIT_ULL(9)
+#define NPC_CN20K_PARSE_NIBBLE_LB_LTYPE BIT_ULL(10)
+#define NPC_CN20K_PARSE_NIBBLE_LC_FLAGS BIT_ULL(11)
+#define NPC_CN20K_PARSE_NIBBLE_LC_LTYPE BIT_ULL(12)
+#define NPC_CN20K_PARSE_NIBBLE_LD_FLAGS BIT_ULL(13)
+#define NPC_CN20K_PARSE_NIBBLE_LD_LTYPE BIT_ULL(14)
+#define NPC_CN20K_PARSE_NIBBLE_LE_FLAGS BIT_ULL(15)
+#define NPC_CN20K_PARSE_NIBBLE_LE_LTYPE BIT_ULL(16)
+#define NPC_CN20K_PARSE_NIBBLE_LF_FLAGS BIT_ULL(17)
+#define NPC_CN20K_PARSE_NIBBLE_LF_LTYPE BIT_ULL(18)
+#define NPC_CN20K_PARSE_NIBBLE_LG_FLAGS BIT_ULL(19)
+#define NPC_CN20K_PARSE_NIBBLE_LG_LTYPE BIT_ULL(20)
+#define NPC_CN20K_PARSE_NIBBLE_LH_FLAGS BIT_ULL(21)
+#define NPC_CN20K_PARSE_NIBBLE_LH_LTYPE BIT_ULL(22)
+
+/* Rx parse key extract nibble enable */
+#define NPC_CN20K_PARSE_NIBBLE_INTF_RX  (NPC_CN20K_PARSE_NIBBLE_CHAN | \
+					 NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST | \
+					 NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LC_FLAGS | \
+					 NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LE_LTYPE)
+
+/* Tx parse key extract nibble enable */
+#define NPC_CN20K_PARSE_NIBBLE_INTF_TX	(NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \
+					 NPC_CN20K_PARSE_NIBBLE_LE_LTYPE)
+
+/**
+ * enum npc_subbank_flag - NPC subbank status
+ *
+ * subbank flag indicates whether the subbank is free
+ * or used.
+ *
+ * @NPC_SUBBANK_FLAG_UNINIT: Subbank is not initialized.
+ * @NPC_SUBBANK_FLAG_FREE: Subbank is free.
+ * @NPC_SUBBANK_FLAG_USED: Subbank is used.
+ */
+enum npc_subbank_flag {
+	NPC_SUBBANK_FLAG_UNINIT,
+	NPC_SUBBANK_FLAG_FREE = BIT(0),
+	NPC_SUBBANK_FLAG_USED = BIT(1),
+};
+
+/**
+ * enum npc_dft_rule_id - Default rule type
+ *
+ * Mcam default rule type.
+ *
+ * @NPC_DFT_RULE_START_ID:	Not used
+ * @NPC_DFT_RULE_PROMISC_ID:	promiscuous rule
+ * @NPC_DFT_RULE_MCAST_ID:	multicast rule
+ * @NPC_DFT_RULE_BCAST_ID:	broadcast rule
+ * @NPC_DFT_RULE_UCAST_ID:	unicast rule
+ * @NPC_DFT_RULE_MAX_ID:	Maximum index.
+ */
+enum npc_dft_rule_id {
+	NPC_DFT_RULE_START_ID = 1,
+	NPC_DFT_RULE_PROMISC_ID = NPC_DFT_RULE_START_ID,
+	NPC_DFT_RULE_MCAST_ID,
+	NPC_DFT_RULE_BCAST_ID,
+	NPC_DFT_RULE_UCAST_ID,
+	NPC_DFT_RULE_MAX_ID,
+};
+
+/**
+ * struct npc_subbank - Subbank fields.
+ * @b0b:	Subbanks bottom index for bank0
+ * @b1b:	Subbanks bottom index for bank1
+ * @b0t:	Subbanks top index for bank0
+ * @b1t:	Subbanks top index for bank1
+ * @flags:	Subbank flags
+ * @lock:	Mutex lock for flags and rsrc mofiication
+ * @b0map:	Bitmap map for bank0 indexes
+ * @b1map:	Bitmap map for bank1 indexes
+ * @idx:	Subbank index
+ * @arr_idx:	Index to the free array or used array
+ * @free_cnt:	Number of free slots in the subbank.
+ * @key_type:	X4 or X2 subbank.
+ *
+ * MCAM resource is divided horizontally into multiple subbanks and
+ * Resource allocation from each subbank is managed by this data
+ * structure.
+ */
+struct npc_subbank {
+	u16 b0t, b0b, b1t, b1b;
+	enum npc_subbank_flag flags;
+	struct mutex lock;	/* Protect subbank resources */
+	DECLARE_BITMAP(b0map, MAX_SUBBANK_DEPTH);
+	DECLARE_BITMAP(b1map, MAX_SUBBANK_DEPTH);
+	u16 idx;
+	u16 arr_idx;
+	u16 free_cnt;
+	u8 key_type;
+};
+
+/**
+ * struct npc_defrag_show_node - Defragmentation show node
+ * @old_midx:	Old mcam index.
+ * @new_midx:	New mcam index.
+ * @vidx:	Virtual index
+ * @list:	Linked list of these nodes
+ *
+ * This structure holds information on last defragmentation
+ * executed on mcam resource.
+ */
+struct npc_defrag_show_node {
+	u16 old_midx;
+	u16 new_midx;
+	u16 vidx;
+	struct list_head list;
+};
+
+/**
+ * struct npc_priv_t - NPC private structure.
+ * @bank_depth:		Total entries in each bank.
+ * @num_banks:		Number of banks.
+ * @num_subbanks:	Number of subbanks.
+ * @subbank_depth:	Depth of subbank.
+ * @kw:			Kex configured key type.
+ * @sb:			Subbank array.
+ * @xa_sb_used:		Array of used subbanks.
+ * @xa_sb_free:		Array of free subbanks.
+ * @xa_pf2idx_map:	PF to mcam index map.
+ * @xa_idx2pf_map:	Mcam index to PF map.
+ * @xa_pf_map:		Pcifunc to index map.
+ * @pf_cnt:		Number of PFs.
+ * @init_done:		Indicates MCAM initialization is done.
+ * @xa_pf2dfl_rmap:	PF to default rule index map.
+ * @xa_idx2vidx_map:	Mcam index to virtual index map.
+ * @xa_vidx2idx_map:	virtual index to mcam index map.
+ * @defrag_lh:		Defrag list head.
+ * @lock:		Lock for defrag list
+ *
+ * This structure is populated during probing time by reading
+ * HW csr registers.
+ */
+struct npc_priv_t {
+	int bank_depth;
+	const int num_banks;
+	int num_subbanks;
+	int subbank_depth;
+	u8 kw;
+	struct npc_subbank *sb;
+	struct xarray xa_sb_used;
+	struct xarray xa_sb_free;
+	struct xarray *xa_pf2idx_map;
+	struct xarray xa_idx2pf_map;
+	struct xarray xa_pf_map;
+	struct xarray xa_pf2dfl_rmap;
+	struct xarray xa_idx2vidx_map;
+	struct xarray xa_vidx2idx_map;
+	struct list_head defrag_lh;
+	struct mutex lock; /* protect defrag nodes */
+	int pf_cnt;
+	bool init_done;
+};
+
+struct npc_kpm_action0 {
+#if defined(__BIG_ENDIAN_BITFIELD)
+	u64 rsvd_63_57     : 7;
+	u64 byp_count      : 3;
+	u64 capture_ena    : 1;
+	u64 parse_done     : 1;
+	u64 next_state     : 8;
+	u64 rsvd_43        : 1;
+	u64 capture_lid    : 3;
+	u64 capture_ltype  : 4;
+	u64 rsvd_32_35     : 4;
+	u64 capture_flags  : 4;
+	u64 ptr_advance    : 8;
+	u64 var_len_offset : 8;
+	u64 var_len_mask   : 8;
+	u64 var_len_right  : 1;
+	u64 var_len_shift  : 3;
+#else
+	u64 var_len_shift  : 3;
+	u64 var_len_right  : 1;
+	u64 var_len_mask   : 8;
+	u64 var_len_offset : 8;
+	u64 ptr_advance    : 8;
+	u64 capture_flags  : 4;
+	u64 rsvd_32_35     : 4;
+	u64 capture_ltype  : 4;
+	u64 capture_lid    : 3;
+	u64 rsvd_43        : 1;
+	u64 next_state     : 8;
+	u64 parse_done     : 1;
+	u64 capture_ena    : 1;
+	u64 byp_count      : 3;
+	u64 rsvd_63_57     : 7;
+#endif
+};
+
+struct npc_mcam_kex_extr {
+	/* MKEX Profle Header */
+	u64 mkex_sign; /* "mcam-kex-profile" (8 bytes/ASCII characters) */
+	u8 name[MKEX_NAME_LEN];   /* MKEX Profile name */
+	u64 cpu_model;   /* Format as profiled by CPU hardware */
+	u64 kpu_version; /* KPU firmware/profile version */
+	u64 reserved; /* Reserved for extension */
+
+	/* MKEX Profle Data */
+	u64 keyx_cfg[NPC_MAX_INTF]; /* NPC_AF_INTF(0..1)_KEX_CFG */
+#define NPC_MAX_EXTRACTOR	24
+	/* MKEX Extractor data */
+	u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR];
+	/* KEX configuration per extractor */
+	u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT];
+} __packed;
+
+struct npc_cn20k_kpu_profile_fwdata {
+#define KPU_SIGN	0x00666f727075706b
+#define KPU_NAME_LEN	32
+	/* Maximum number of custom KPU entries supported by
+	 * the built-in profile.
+	 */
+#define KPU_CN20K_MAX_CST_ENT	6
+	/* KPU Profle Header */
+	__le64	signature; /* "kpuprof\0" (8 bytes/ASCII characters) */
+	u8	name[KPU_NAME_LEN]; /* KPU Profile name */
+	__le64	version; /* KPU profile version */
+	u8	kpus;
+	u8	reserved[7];
+
+	/* Default MKEX profile to be used with this KPU profile. May be
+	 * overridden with mkex_profile module parameter.
+	 * Format is same as for the MKEX profile to streamline processing.
+	 */
+	struct npc_mcam_kex_extr	mkex;
+	/* LTYPE values for specific HW offloaded protocols. */
+	struct npc_lt_def_cfg		lt_def;
+	/* Dynamically sized data:
+	 *  Custom KPU CAM and ACTION configuration entries.
+	 * struct npc_kpu_fwdata kpu[kpus];
+	 */
+	u8	data[];
+} __packed;
+
+struct rvu;
+
+struct npc_priv_t *npc_priv_get(void);
+int npc_cn20k_init(struct rvu *rvu);
+void npc_cn20k_deinit(struct rvu *rvu);
+
+void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free,
+				 int *x4_free, int *sb_free);
+
+int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type,
+			    int prio, u16 *mcam_idx, int ref, int limit,
+			    bool contig, int count, bool virt);
+int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count);
+void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr);
+struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void);
+void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr,
+				 const char *mkex_profile);
+int npc_cn20k_apply_custom_kpu(struct rvu *rvu,
+			       struct npc_kpu_profile_adapter *profile);
+
+void
+npc_cn20k_update_action_entries_n_flags(struct rvu *rvu,
+					struct npc_kpu_profile_adapter *pfl);
+
+int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc);
+void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc);
+
+int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
+				u16 *mcast, u16 *promisc, u16 *ucast);
+
+void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
+				 u8 intf, struct cn20k_mcam_entry *entry,
+				 bool enable, u8 hw_prio, u8 req_kw_type);
+void npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
+				 int index, bool enable);
+void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr,
+			       u16 src, u16 dest);
+void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
+			       struct cn20k_mcam_entry *entry, u8 *intf,
+			       u8 *ena, u8 *hw_prio);
+void npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr,
+				int bank, int index);
+int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type);
+u16 npc_cn20k_vidx2idx(u16 index);
+u16 npc_cn20k_idx2vidx(u16 idx);
+int npc_cn20k_defrag(struct rvu *rvu);
+
+#endif /* NPC_CN20K_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h
index affb39803120e..8bfaa507ee502 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h
@@ -78,4 +78,69 @@
 #define RVU_MBOX_VF_INT_ENA_W1C			(0x38)
 
 #define RVU_MBOX_VF_VFAF_TRIGX(a)		(0x2000 | (a) << 3)
+
+#define NIX_GINT_INT                           (0x200)
+#define NIX_GINT_INT_W1S                       (0x208)
+
+#define ALTAF_FLR                              BIT_ULL(0)
+#define ALTAF_RDY                              BIT_ULL(1)
+
+/* NPC registers */
+#define NPC_AF_INTFX_EXTRACTORX_CFG(a, b) \
+	(0x20c000ull | (a) << 16 | (b) << 8)
+#define NPC_AF_INTFX_EXTRACTORX_LTX_CFG(a, b, c) \
+	(0x204000ull | (a) << 16 | (b) << 8  | (c) << 3)
+#define NPC_AF_KPMX_ENTRYX_CAMX(a, b, c) \
+	(0x20000ull | (a) << 12 | (b) << 3 | (c) << 16)
+#define NPC_AF_KPMX_ENTRYX_ACTION0(a, b) \
+	(0x40000ull | (a) << 12 | (b) << 3)
+#define NPC_AF_KPMX_ENTRYX_ACTION1(a, b) \
+	(0x50000ull | (a) << 12 | (b) << 3)
+#define NPC_AF_KPMX_ENTRY_DISX(a, b)	(0x60000ull | (a) << 12 | (b) << 3)
+#define NPC_AF_KPM_PASS2_CFG	0x10210
+#define NPC_AF_KPMX_PASS2_OFFSET(a)	(0x60040ull | (a) << 12)
+#define NPC_AF_MCAM_SECTIONX_CFG_EXT(a)	(0xf000000ull | (a) << 3)
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0x8000000ull | (a) << 4 | (b) << 20 | (c) << 3);	\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0x9000000ull | (a) << 4 | (b) << 20 | (c) << 3);	\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0x9400000ull | (a) << 4 | (b) << 20 | (c) << 3);	\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0x9800000ull | (a) << 4 | (b) << 20 | (c) << 3);	\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0x9c00000ull | (a) << 4 | (b) << 20 | (c) << 3);	\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(a, b) ({		\
+	u64 offset;						\
+	offset = (0xa000000ull | (a) << 4 | (b) << 20);		\
+	offset; })
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(a, b, c) ({		\
+	u64 offset;							\
+	offset = (0xc000000ull | (a) << 4 | (b) << 20 | (c) << 22);	\
+	offset; })
+
+#define NPC_AF_INTFX_MISS_ACTX(a, b)	(0xf003000 | (a) << 6 | (b) << 4)
+
+#define NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(a, b) ({		\
+	u64 offset;						\
+	offset = (0xb000000ull | (a) << 4 | (b) << 20);		\
+	offset; })
+
 #endif /* RVU_MBOX_REG_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h
index 8a08bebf08c2c..779413a383b74 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h
@@ -177,10 +177,6 @@ enum nix_scheduler {
 #define NIX_TX_ACTIONOP_MCAST		(0x3ull)
 #define NIX_TX_ACTIONOP_DROP_VIOL	(0x5ull)
 
-#define NPC_MCAM_KEY_X1			0
-#define NPC_MCAM_KEY_X2			1
-#define NPC_MCAM_KEY_X4			2
-
 #define NIX_INTFX_RX(a)			(0x0ull | (a) << 1)
 #define NIX_INTFX_TX(a)			(0x1ull | (a) << 1)
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
index a3e273126e4e1..dc42c81c09425 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
@@ -52,6 +52,14 @@
 #define MBOX_DIR_PFVF_UP	6  /* PF sends messages to VF */
 #define MBOX_DIR_VFPF_UP	7  /* VF replies to PF */
 
+enum {
+	NPC_MCAM_KEY_X1 = 0,
+	NPC_MCAM_KEY_DYN = NPC_MCAM_KEY_X1,
+	NPC_MCAM_KEY_X2,
+	NPC_MCAM_KEY_X4,
+	NPC_MCAM_KEY_MAX,
+};
+
 enum {
 	TYPE_AFVF,
 	TYPE_AFPF,
@@ -275,6 +283,33 @@ M(NPC_GET_FIELD_HASH_INFO, 0x6013, npc_get_field_hash_info,
 M(NPC_GET_FIELD_STATUS, 0x6014, npc_get_field_status,                     \
 				   npc_get_field_status_req,              \
 				   npc_get_field_status_rsp)              \
+M(NPC_CN20K_MCAM_GET_FREE_COUNT, 0x6015, npc_cn20k_get_fcnt,		\
+				 msg_req, npc_cn20k_get_fcnt_rsp)	\
+M(NPC_CN20K_GET_KEX_CFG, 0x6016, npc_cn20k_get_kex_cfg,			\
+				   msg_req, npc_cn20k_get_kex_cfg_rsp)	\
+M(NPC_CN20K_MCAM_WRITE_ENTRY,	0x6017, npc_cn20k_mcam_write_entry,	\
+				 npc_cn20k_mcam_write_entry_req, msg_rsp)  \
+M(NPC_CN20K_MCAM_ALLOC_AND_WRITE_ENTRY, 0x6018,				   \
+npc_cn20k_mcam_alloc_and_write_entry,					   \
+				npc_cn20k_mcam_alloc_and_write_entry_req,  \
+				npc_mcam_alloc_and_write_entry_rsp)  \
+M(NPC_CN20K_MCAM_READ_ENTRY,	0x6019, npc_cn20k_mcam_read_entry,	\
+				  npc_mcam_read_entry_req,		\
+				  npc_cn20k_mcam_read_entry_rsp)	\
+M(NPC_CN20K_MCAM_READ_BASE_RULE, 0x601a, npc_cn20k_read_base_steer_rule,       \
+				   msg_req, npc_cn20k_mcam_read_base_rule_rsp) \
+M(NPC_MCAM_DEFRAG,	     0x601b,	npc_defrag,			\
+					msg_req,			\
+					msg_rsp)			\
+M(NPC_MCAM_GET_NUM_KWS, 0x601c, npc_get_num_kws,		\
+				npc_get_num_kws_req,		\
+				npc_get_num_kws_rsp)		\
+M(NPC_MCAM_GET_DFT_RL_IDXS, 0x601d, npc_get_dft_rl_idxs,	\
+					msg_req,		\
+					npc_get_dft_rl_idxs_rsp)\
+M(NPC_MCAM_GET_NPC_PFL_INFO, 0x601e, npc_get_pfl_info,		\
+					msg_req,		\
+					npc_get_pfl_info_rsp)	\
 /* NIX mbox IDs (range 0x8000 - 0xFFFF) */				\
 M(NIX_LF_ALLOC,		0x8000, nix_lf_alloc,				\
 				 nix_lf_alloc_req, nix_lf_alloc_rsp)	\
@@ -1527,9 +1562,11 @@ struct npc_mcam_alloc_entry_req {
 #define NPC_MCAM_ANY_PRIO		0
 #define NPC_MCAM_LOWER_PRIO		1
 #define NPC_MCAM_HIGHER_PRIO		2
-	u8  priority; /* Lower or higher w.r.t ref_entry */
+	u8  ref_prio; /* Lower or higher w.r.t ref_entry */
 	u16 ref_entry;
 	u16 count;    /* Number of entries requested */
+	u8 kw_type; /* entry key type, valid for cn20k */
+	u8 virt;    /* Request virtual index */
 };
 
 struct npc_mcam_alloc_entry_rsp {
@@ -1548,14 +1585,47 @@ struct npc_mcam_free_entry_req {
 	u8  all;   /* If all entries allocated to this PFVF to be freed */
 };
 
+struct mcam_entry_mdata {
+	u64 *kw;
+	u64 *kw_mask;
+	u64 *action;
+	u64 *vtag_action;
+	u8 max_kw;
+};
+
+enum npc_kws_in_key_sz {
+	NPC_KWS_IN_KEY_SZ_7 = 7,
+	NPC_KWS_IN_KEY_SZ_8 = 8,
+	NPC_KWS_IN_KEY_SZ_MAX,
+};
+
 struct mcam_entry {
-#define NPC_MAX_KWS_IN_KEY	7 /* Number of keywords in max keywidth */
-	u64	kw[NPC_MAX_KWS_IN_KEY];
-	u64	kw_mask[NPC_MAX_KWS_IN_KEY];
+	u64	kw[NPC_KWS_IN_KEY_SZ_7];
+	u64	kw_mask[NPC_KWS_IN_KEY_SZ_7];
 	u64	action;
 	u64	vtag_action;
 };
 
+struct cn20k_mcam_entry {
+	u64	kw[NPC_KWS_IN_KEY_SZ_8];
+	u64	kw_mask[NPC_KWS_IN_KEY_SZ_8];
+	u64	action;
+	u64	vtag_action;
+	u64	action2;
+};
+
+struct npc_cn20k_mcam_write_entry_req {
+	struct mbox_msghdr hdr;
+	struct cn20k_mcam_entry entry_data;
+	u16 entry;	 /* MCAM entry to write this match key */
+	u16 cntr;	 /* Counter for this MCAM entry */
+	u8  intf;	 /* Rx or Tx interface */
+	u8  enable_entry;/* Enable this MCAM entry ? */
+	u8  hw_prio;	 /* hardware priority, valid for cn20k */
+	u8  req_kw_type; /* Type of kw which should be written */
+	u64 reserved;	 /* reserved for future use */
+};
+
 struct npc_mcam_write_entry_req {
 	struct mbox_msghdr hdr;
 	struct mcam_entry entry_data;
@@ -1564,6 +1634,8 @@ struct npc_mcam_write_entry_req {
 	u8  intf;	 /* Rx or Tx interface */
 	u8  enable_entry;/* Enable this MCAM entry ? */
 	u8  set_cntr;    /* Set counter for this entry ? */
+	u8  hw_prio;	 /* hardware priority, valid for cn20k */
+	u64 reserved;	 /* reserved for future use */
 };
 
 /* Enable/Disable a given entry */
@@ -1622,12 +1694,38 @@ struct npc_mcam_alloc_and_write_entry_req {
 	struct mbox_msghdr hdr;
 	struct mcam_entry entry_data;
 	u16 ref_entry;
-	u8  priority;    /* Lower or higher w.r.t ref_entry */
+	u8  ref_prio;    /* Lower or higher w.r.t ref_entry */
 	u8  intf;	 /* Rx or Tx interface */
 	u8  enable_entry;/* Enable this MCAM entry ? */
 	u8  alloc_cntr;  /* Allocate counter and map ? */
 };
 
+struct npc_cn20k_mcam_alloc_and_write_entry_req {
+	struct mbox_msghdr hdr;
+	struct cn20k_mcam_entry entry_data;
+	u16 ref_entry;
+	u8  ref_prio;    /* Lower or higher w.r.t ref_entry */
+	u8  intf;	 /* Rx or Tx interface */
+	u8  enable_entry;/* Enable this MCAM entry ? */
+	u8  hw_prio;	 /* hardware priority, valid for cn20k */
+	u8  virt;	 /* Allocate virtual index */
+	u8  req_kw_type; /* Key type to be written */
+	u16 reserved[4]; /* reserved for future use */
+};
+
+struct npc_cn20k_mcam_read_entry_rsp {
+	struct mbox_msghdr hdr;
+	struct cn20k_mcam_entry entry_data;
+	u8 intf;
+	u8 enable;
+	u8 hw_prio; /* valid for cn20k */
+};
+
+struct npc_cn20k_mcam_read_base_rule_rsp {
+	struct mbox_msghdr hdr;
+	struct cn20k_mcam_entry entry;
+};
+
 struct npc_mcam_alloc_and_write_entry_rsp {
 	struct mbox_msghdr hdr;
 	u16 entry;
@@ -1653,6 +1751,19 @@ struct npc_get_kex_cfg_rsp {
 	u8 mkex_pfl_name[MKEX_NAME_LEN];
 };
 
+struct npc_cn20k_get_kex_cfg_rsp {
+	struct mbox_msghdr hdr;
+	u64 rx_keyx_cfg;   /* NPC_AF_INTF(0)_KEX_CFG */
+	u64 tx_keyx_cfg;   /* NPC_AF_INTF(1)_KEX_CFG */
+#define NPC_MAX_EXTRACTOR 24
+	/* MKEX Extractor data */
+	u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR];
+	/* KEX configuration per extractor */
+	u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT];
+#define MKEX_NAME_LEN 128
+	u8 mkex_pfl_name[MKEX_NAME_LEN];
+};
+
 struct ptp_get_cap_rsp {
 	struct mbox_msghdr hdr;
 #define        PTP_CAP_HW_ATOMIC_UPDATE BIT_ULL(0)
@@ -1765,11 +1876,53 @@ struct npc_install_flow_req {
 	u8  vtag1_op;
 	/* old counter value */
 	u16 cntr_val;
+	u8 hw_prio;
+	u8  req_kw_type; /* Key type to be written */
+	u8 alloc_entry;	/* only for cn20k */
+/* For now use any priority, once AF driver is changed to
+ * allocate least priority entry instead of mid zone then make
+ * NPC_MCAM_LEAST_PRIO as 3
+ */
+#define NPC_MCAM_LEAST_PRIO	NPC_MCAM_ANY_PRIO
+	u16 ref_prio;
+	u16 ref_entry;
 };
 
 struct npc_install_flow_rsp {
 	struct mbox_msghdr hdr;
 	int counter; /* negative if no counter else counter number */
+	u16 entry;
+	u8 kw_type;
+};
+
+struct npc_get_num_kws_req {
+	struct mbox_msghdr hdr;
+	struct npc_install_flow_req fl;
+	u32 rsvd[4];
+};
+
+struct npc_get_num_kws_rsp {
+	struct mbox_msghdr hdr;
+	int kws;
+	u32 rsvd[4];
+};
+
+struct npc_get_dft_rl_idxs_rsp {
+	struct mbox_msghdr hdr;
+	u16 bcast;
+	u16 mcast;
+	u16 promisc;
+	u16 ucast;
+	u16 vf_ucast;
+	u16 rsvd[7];
+};
+
+struct npc_get_pfl_info_rsp {
+	struct mbox_msghdr hdr;
+	u16 x4_slots;
+	u8 kw_type;
+	u8 rsvd1[3];
+	u32 rsvd2[4];
 };
 
 struct npc_delete_flow_req {
@@ -1795,6 +1948,15 @@ struct npc_mcam_read_entry_rsp {
 	struct mcam_entry entry_data;
 	u8 intf;
 	u8 enable;
+	u8 hw_prio; /* valid for cn20k */
+};
+
+/* Available entries to use */
+struct npc_cn20k_get_fcnt_rsp {
+	struct mbox_msghdr hdr;
+	int free_x2;
+	int free_x4;
+	int free_subbanks;
 };
 
 struct npc_mcam_read_base_rule_rsp {
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
index 6c3aca6f278db..cefc5d70f3e49 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
@@ -429,6 +429,7 @@ struct nix_rx_action {
 
 /* NPC_AF_INTFX_KEX_CFG field masks */
 #define NPC_PARSE_NIBBLE		GENMASK_ULL(30, 0)
+#define NPC_TOTAL_NIBBLE		31
 
 /* NPC_PARSE_KEX_S nibble definitions for each field */
 #define NPC_PARSE_NIBBLE_CHAN		GENMASK_ULL(2, 0)
@@ -643,6 +644,7 @@ struct rvu_npc_mcam_rule {
 	u16 chan;
 	u16 chan_mask;
 	u8 lxmb;
+	u8 hw_prio;
 };
 
 #endif /* NPC_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
index 41de72c8607f0..db74f7fdf0288 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h
@@ -321,6 +321,18 @@ enum npc_kpu_lb_lflag {
 	NPC_F_LB_L_FDSA,
 };
 
+enum npc_cn20k_kpu_lc_uflag {
+	NPC_CN20K_F_LC_U_MPLS_IN_IP = 0x20,
+	NPC_CN20K_F_LC_U_IP6_TUN_IP6 = 0x40,
+	NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP = 0x80,
+};
+
+enum npc_cn20k_kpu_lc_lflag {
+	NPC_CN20K_F_LC_L_IP_FRAG = 2,
+	NPC_CN20K_F_LC_L_IP6_FRAG,
+	NPC_CN20K_F_LC_L_6TO4,
+};
+
 enum npc_kpu_lc_uflag {
 	NPC_F_LC_U_UNK_PROTO = 0x10,
 	NPC_F_LC_U_IP_FRAG = 0x20,
@@ -489,7 +501,7 @@ enum NPC_ERRLEV_E {
 		0, 0, 0, 0,			\
 	}
 
-static struct npc_kpu_profile_action ikpu_action_entries[] = {
+static struct npc_kpu_profile_action ikpu_action_entries[] __maybe_unused = {
 	{
 		NPC_ERRLEV_RE, NPC_EC_NOERR,
 		12, 16, 20, 0, 0,
@@ -1068,7 +1080,7 @@ static struct npc_kpu_profile_action ikpu_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu1_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -1878,7 +1890,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu2_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu2_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -2823,7 +2835,7 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu3_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu3_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -3804,7 +3816,7 @@ static struct npc_kpu_profile_cam kpu3_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu4_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu4_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -4119,7 +4131,7 @@ static struct npc_kpu_profile_cam kpu4_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu5_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu5_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -5172,7 +5184,7 @@ static struct npc_kpu_profile_cam kpu5_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu6_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu6_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -5901,7 +5913,7 @@ static struct npc_kpu_profile_cam kpu6_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu7_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu7_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -6252,7 +6264,7 @@ static struct npc_kpu_profile_cam kpu7_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu8_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu8_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -7089,7 +7101,7 @@ static struct npc_kpu_profile_cam kpu8_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu9_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu9_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -7575,7 +7587,7 @@ static struct npc_kpu_profile_cam kpu9_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu10_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu10_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -7746,7 +7758,7 @@ static struct npc_kpu_profile_cam kpu10_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu11_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu11_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8061,7 +8073,7 @@ static struct npc_kpu_profile_cam kpu11_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu12_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu12_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8322,7 +8334,7 @@ static struct npc_kpu_profile_cam kpu12_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu13_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu13_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8340,7 +8352,7 @@ static struct npc_kpu_profile_cam kpu13_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu14_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu14_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8358,7 +8370,7 @@ static struct npc_kpu_profile_cam kpu14_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu15_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu15_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8565,7 +8577,7 @@ static struct npc_kpu_profile_cam kpu15_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_cam kpu16_cam_entries[] = {
+static struct npc_kpu_profile_cam kpu16_cam_entries[] __maybe_unused = {
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
 	NPC_KPU_NOP_CAM,
@@ -8628,7 +8640,7 @@ static struct npc_kpu_profile_cam kpu16_cam_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu1_action_entries[] = {
+static struct npc_kpu_profile_action kpu1_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -9368,7 +9380,7 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu2_action_entries[] = {
+static struct npc_kpu_profile_action kpu2_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -10209,7 +10221,7 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu3_action_entries[] = {
+static struct npc_kpu_profile_action kpu3_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -11082,7 +11094,7 @@ static struct npc_kpu_profile_action kpu3_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu4_action_entries[] = {
+static struct npc_kpu_profile_action kpu4_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -11363,7 +11375,7 @@ static struct npc_kpu_profile_action kpu4_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu5_action_entries[] = {
+static struct npc_kpu_profile_action kpu5_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -12300,7 +12312,7 @@ static struct npc_kpu_profile_action kpu5_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu6_action_entries[] = {
+static struct npc_kpu_profile_action kpu6_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -12949,7 +12961,7 @@ static struct npc_kpu_profile_action kpu6_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu7_action_entries[] = {
+static struct npc_kpu_profile_action kpu7_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -13262,7 +13274,7 @@ static struct npc_kpu_profile_action kpu7_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu8_action_entries[] = {
+static struct npc_kpu_profile_action kpu8_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -14007,7 +14019,7 @@ static struct npc_kpu_profile_action kpu8_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu9_action_entries[] = {
+static struct npc_kpu_profile_action kpu9_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -14440,7 +14452,7 @@ static struct npc_kpu_profile_action kpu9_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu10_action_entries[] = {
+static struct npc_kpu_profile_action kpu10_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -14593,7 +14605,7 @@ static struct npc_kpu_profile_action kpu10_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu11_action_entries[] = {
+static struct npc_kpu_profile_action kpu11_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -14874,7 +14886,7 @@ static struct npc_kpu_profile_action kpu11_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu12_action_entries[] = {
+static struct npc_kpu_profile_action kpu12_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -15107,7 +15119,7 @@ static struct npc_kpu_profile_action kpu12_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu13_action_entries[] = {
+static struct npc_kpu_profile_action kpu13_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -15124,7 +15136,7 @@ static struct npc_kpu_profile_action kpu13_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu14_action_entries[] = {
+static struct npc_kpu_profile_action kpu14_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -15141,7 +15153,7 @@ static struct npc_kpu_profile_action kpu14_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu15_action_entries[] = {
+static struct npc_kpu_profile_action kpu15_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -15326,7 +15338,7 @@ static struct npc_kpu_profile_action kpu15_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile_action kpu16_action_entries[] = {
+static struct npc_kpu_profile_action kpu16_action_entries[] __maybe_unused = {
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
 	NPC_KPU_NOP_ACTION,
@@ -15383,7 +15395,7 @@ static struct npc_kpu_profile_action kpu16_action_entries[] = {
 	},
 };
 
-static struct npc_kpu_profile npc_kpu_profiles[] = {
+static struct npc_kpu_profile npc_kpu_profiles[] __maybe_unused = {
 	{
 		ARRAY_SIZE(kpu1_cam_entries),
 		ARRAY_SIZE(kpu1_action_entries),
@@ -15482,7 +15494,7 @@ static struct npc_kpu_profile npc_kpu_profiles[] = {
 	},
 };
 
-static struct npc_lt_def_cfg npc_lt_defaults = {
+static struct npc_lt_def_cfg npc_lt_defaults __maybe_unused = {
 	.rx_ol2 = {
 		.lid = NPC_LID_LA,
 		.ltype_match = NPC_LT_LA_ETHER,
@@ -15604,7 +15616,7 @@ static struct npc_lt_def_cfg npc_lt_defaults = {
 	},
 };
 
-static struct npc_mcam_kex npc_mkex_default = {
+static struct npc_mcam_kex npc_mkex_default __maybe_unused = {
 	.mkex_sign = MKEX_SIGN,
 	.name = "default",
 	.kpu_version = NPC_KPU_PROFILE_VER,
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
index 5b1129558e8b3..1c8f657719b2f 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
@@ -22,6 +22,7 @@
 #include "rvu_npc_hash.h"
 #include "cn20k/reg.h"
 #include "cn20k/api.h"
+#include "cn20k/npc.h"
 
 #define DRV_NAME	"rvu_af"
 #define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
@@ -599,6 +600,7 @@ static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
 
 static int rvu_setup_msix_resources(struct rvu *rvu)
 {
+	struct altaf_intr_notify *altaf_intr_data;
 	struct rvu_hwinfo *hw = rvu->hw;
 	int pf, vf, numvfs, hwvf, err;
 	int nvecs, offset, max_msix;
@@ -705,7 +707,30 @@ static int rvu_setup_msix_resources(struct rvu *rvu)
 	rvu->msix_base_iova = iova;
 	rvu->msixtr_base_phy = phy_addr;
 
+	if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))
+		return 0;
+
+	if (!rvu->fwdata)
+		goto fail;
+
+	altaf_intr_data = &rvu->fwdata->altaf_intr_info;
+	if (altaf_intr_data->gint_paddr) {
+		iova = dma_map_resource(rvu->dev, altaf_intr_data->gint_paddr,
+					PCI_MSIX_ENTRY_SIZE,
+					DMA_BIDIRECTIONAL, 0);
+
+		if (dma_mapping_error(rvu->dev, iova))
+			goto fail;
+
+		altaf_intr_data->gint_iova_addr = iova;
+	}
+
 	return 0;
+
+fail:
+	dma_unmap_resource(rvu->dev, phy_addr, max_msix * PCI_MSIX_ENTRY_SIZE,
+			   DMA_BIDIRECTIONAL, 0);
+	return -EFAULT;
 }
 
 static void rvu_reset_msix(struct rvu *rvu)
@@ -1396,7 +1421,6 @@ static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
 	if (blkaddr < 0)
 		return;
 
-
 	block = &hw->block[blkaddr];
 
 	num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
@@ -1467,6 +1491,13 @@ static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
 			else if ((blkid == BLKADDR_CPT1) && !detach->cptlfs)
 				continue;
 		}
+
+		if (detach_all ||
+		    (detach && (blkid == BLKADDR_NIX0 ||
+				blkid == BLKADDR_NIX1) &&
+		     detach->nixlf))
+			npc_cn20k_dft_rules_free(rvu, pcifunc);
+
 		rvu_detach_block(rvu, pcifunc, block->type);
 	}
 
@@ -1750,6 +1781,12 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu,
 		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach);
 		if (err)
 			goto fail2;
+
+		if (is_cn20k(rvu->pdev)) {
+			err = npc_cn20k_dft_rules_alloc(rvu, pcifunc);
+			if (err)
+				goto fail3;
+		}
 	}
 
 	if (attach->sso) {
@@ -1763,7 +1800,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu,
 		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO,
 				       attach->sso, attach);
 		if (err)
-			goto fail3;
+			goto fail4;
 	}
 
 	if (attach->ssow) {
@@ -1772,7 +1809,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu,
 		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW,
 				       attach->ssow, attach);
 		if (err)
-			goto fail4;
+			goto fail5;
 	}
 
 	if (attach->timlfs) {
@@ -1781,7 +1818,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu,
 		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM,
 				       attach->timlfs, attach);
 		if (err)
-			goto fail5;
+			goto fail6;
 	}
 
 	if (attach->cptlfs) {
@@ -1791,24 +1828,28 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu,
 		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT,
 				       attach->cptlfs, attach);
 		if (err)
-			goto fail6;
+			goto fail7;
 	}
 
 	mutex_unlock(&rvu->rsrc_lock);
 	return 0;
 
-fail6:
+fail7:
 	if (attach->timlfs)
 		rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
 
-fail5:
+fail6:
 	if (attach->ssow)
 		rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
 
-fail4:
+fail5:
 	if (attach->sso)
 		rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
 
+fail4:
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_dft_rules_free(rvu, pcifunc);
+
 fail3:
 	if (attach->nixlf)
 		rvu_detach_block(rvu, pcifunc, BLKTYPE_NIX);
@@ -2185,6 +2226,33 @@ int rvu_mbox_handler_ndc_sync_op(struct rvu *rvu,
 	return 0;
 }
 
+static void rvu_notify_altaf(struct rvu *rvu, u16 pcifunc, u64 op)
+{
+	int pf, vf;
+
+	if (!rvu->fwdata)
+		return;
+
+	if (op == ALTAF_FLR) {
+		pf = rvu_get_pf(rvu->pdev, pcifunc);
+		set_bit(pf, rvu->fwdata->altaf_intr_info.flr_pf_bmap);
+		if (pcifunc & RVU_PFVF_FUNC_MASK) {
+			vf = pcifunc & RVU_PFVF_FUNC_MASK;
+			if (vf >= 128) {
+				WARN(1,
+				     "flr_vf_bmap size is 128 bits, vf=%u\n",
+				     vf);
+				return;
+			}
+
+			set_bit(vf, rvu->fwdata->altaf_intr_info.flr_vf_bmap);
+		}
+	}
+
+	rvu_write64(rvu, BLKADDR_NIX0, AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), op);
+	usleep_range(5000, 6000);
+}
+
 static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
 				struct mbox_msghdr *req)
 {
@@ -2268,7 +2336,8 @@ static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll)
 
 	offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
 
-	if (req_hdr->sig && !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) {
+	if (req_hdr->sig && rvu->altaf_ready &&
+	    !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) {
 		req_hdr->opt_msg = mw->mbox_wrk[devid].num_msgs;
 		rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL,
 			    RVU_AF_BAR2_PFID);
@@ -2777,6 +2846,16 @@ static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
 	block = &rvu->hw->block[blkaddr];
 	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
 					block->addr);
+
+	if (block->addr == BLKADDR_TIM && rvu->altaf_ready) {
+		rvu_notify_altaf(rvu, pcifunc, ALTAF_FLR);
+		return;
+	}
+
+	if ((block->addr == BLKADDR_SSO || block->addr == BLKADDR_SSOW) &&
+	    rvu->altaf_ready)
+		return;
+
 	if (!num_lfs)
 		return;
 	for (slot = 0; slot < num_lfs; slot++) {
@@ -3060,12 +3139,12 @@ static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
 
 static int rvu_register_interrupts(struct rvu *rvu)
 {
-	int ret, offset, pf_vec_start;
+	int i, ret, offset, pf_vec_start;
 
 	rvu->num_vec = pci_msix_vec_count(rvu->pdev);
 
-	rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
-					   NAME_SIZE, GFP_KERNEL);
+	rvu->irq_name = devm_kcalloc(rvu->dev, rvu->num_vec,
+				     NAME_SIZE, GFP_KERNEL);
 	if (!rvu->irq_name)
 		return -ENOMEM;
 
@@ -3251,6 +3330,13 @@ static int rvu_register_interrupts(struct rvu *rvu)
 	if (ret)
 		goto fail;
 
+	for (i = 0; i < rvu->num_vec; i++) {
+		if (strstr(&rvu->irq_name[i * NAME_SIZE], "Mbox") ||
+		    strstr(&rvu->irq_name[i * NAME_SIZE], "FLR"))
+			irq_set_affinity(pci_irq_vector(rvu->pdev, i),
+					 cpumask_of(0));
+	}
+
 	return 0;
 
 fail:
@@ -3279,8 +3365,8 @@ static int rvu_flr_init(struct rvu *rvu)
 			    cfg | BIT_ULL(22));
 	}
 
-	rvu->flr_wq = alloc_ordered_workqueue("rvu_afpf_flr",
-					      WQ_HIGHPRI | WQ_MEM_RECLAIM);
+	rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
+				      WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
 	if (!rvu->flr_wq)
 		return -ENOMEM;
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index e85dac2c806d9..a466181cf9082 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -197,7 +197,7 @@ struct npc_key_field {
 	/* Masks where all set bits indicate position
 	 * of a field in the key
 	 */
-	u64 kw_mask[NPC_MAX_KWS_IN_KEY];
+	u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX];
 	/* Number of words in the key a field spans. If a field is
 	 * of 16 bytes and key offset is 4 then the field will use
 	 * 4 bytes in KW0, 8 bytes in KW1 and 4 bytes in KW2 and
@@ -308,6 +308,7 @@ struct rvu_pfvf {
 	u64     lmt_map_ent_w1; /* Preseving the word1 of lmtst map table entry*/
 	unsigned long flags;
 	struct  sdp_node_info *sdp_info;
+	u8	hw_prio;   /* Hw priority of default rules */
 };
 
 enum rvu_pfvf_flags {
@@ -447,9 +448,11 @@ struct rvu_hwinfo {
 	u8	sdp_links;
 	u8	cpt_links;	/* Number of CPT links */
 	u8	npc_kpus;          /* No of parser units */
+	u8	npc_kpms;	/* Number of enhanced parser units */
+	u8	npc_kex_extr;	/* Number of LDATA extractors per KEX */
 	u8	npc_pkinds;        /* No of port kinds */
 	u8	npc_intfs;         /* No of interfaces */
-	u8	npc_kpu_entries;   /* No of KPU entries */
+	u16	npc_kpu_entries;   /* No of KPU entries */
 	u16	npc_counters;	   /* No of match stats counters */
 	u32	lbk_bufsize;	   /* FIFO size supported by LBK */
 	bool	npc_ext_set;	   /* Extended register set */
@@ -552,7 +555,11 @@ struct npc_kpu_profile_adapter {
 	const struct npc_lt_def_cfg	*lt_def;
 	const struct npc_kpu_profile_action	*ikpu; /* array[pkinds] */
 	const struct npc_kpu_profile	*kpu; /* array[kpus] */
-	struct npc_mcam_kex		*mkex;
+	union npc_mcam_key_prfl {
+		struct npc_mcam_kex		*mkex;
+					/* used for cn9k and cn10k */
+		struct npc_mcam_kex_extr	*mkex_extr; /* used for cn20k */
+	} mcam_kex_prfl;
 	struct npc_mcam_kex_hash	*mkex_hash;
 	bool				custom;
 	size_t				pkinds;
@@ -1117,8 +1124,8 @@ int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause
 void rvu_mac_reset(struct rvu *rvu, u16 pcifunc);
 u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac);
 void cgx_start_linkup(struct rvu *rvu);
-int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf,
-			     int type);
+int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
+			     u16 pcifunc, int nixlf, int type);
 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr,
 			   int index);
 int rvu_npc_init(struct rvu *rvu);
@@ -1184,4 +1191,5 @@ int rvu_rep_pf_init(struct rvu *rvu);
 int rvu_rep_install_mcam_rules(struct rvu *rvu);
 void rvu_rep_update_rules(struct rvu *rvu, u16 pcifunc, bool ena);
 int rvu_rep_notify_pfvf_state(struct rvu *rvu, u16 pcifunc, bool enable);
+int npc_mcam_verify_entry(struct npc_mcam *mcam, u16 pcifunc, int entry);
 #endif /* RVU_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
index 15d3cb0b9da64..413f9fa40b330 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
@@ -21,7 +21,9 @@
 #include "rvu_npc_hash.h"
 #include "mcs.h"
 
+#include "cn20k/reg.h"
 #include "cn20k/debugfs.h"
+#include "cn20k/npc.h"
 
 #define DEBUGFS_DIR_NAME "octeontx2"
 
@@ -3196,7 +3198,9 @@ static void rvu_print_npc_mcam_info(struct seq_file *s,
 static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued)
 {
 	struct rvu *rvu = filp->private;
+	int x4_free, x2_free, sb_free;
 	int pf, vf, numvfs, blkaddr;
+	struct npc_priv_t *npc_priv;
 	struct npc_mcam *mcam;
 	u16 pcifunc, counters;
 	u64 cfg;
@@ -3210,16 +3214,34 @@ static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued)
 
 	seq_puts(filp, "\nNPC MCAM info:\n");
 	/* MCAM keywidth on receive and transmit sides */
-	cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX));
-	cfg = (cfg >> 32) & 0x07;
-	seq_printf(filp, "\t\t RX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ?
-		   "112bits" : ((cfg == NPC_MCAM_KEY_X2) ?
-		   "224bits" : "448bits"));
-	cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX));
-	cfg = (cfg >> 32) & 0x07;
-	seq_printf(filp, "\t\t TX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ?
-		   "112bits" : ((cfg == NPC_MCAM_KEY_X2) ?
-		   "224bits" : "448bits"));
+	if (is_cn20k(rvu->pdev)) {
+		npc_priv = npc_priv_get();
+		seq_printf(filp, "\t\t RX keywidth \t: %s\n",
+			   (npc_priv->kw == NPC_MCAM_KEY_X1) ?
+			   "256bits" : "512bits");
+
+		npc_cn20k_subbank_calc_free(rvu, &x2_free, &x4_free, &sb_free);
+		seq_printf(filp, "\t\t free x4 slots\t: %d\n", x4_free);
+
+		seq_printf(filp, "\t\t free x2 slots\t: %d\n", x2_free);
+
+		seq_printf(filp, "\t\t free subbanks\t: %d\n", sb_free);
+	} else {
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX));
+		cfg = (cfg >> 32) & 0x07;
+		seq_printf(filp, "\t\t RX keywidth \t: %s\n",
+			   (cfg == NPC_MCAM_KEY_X1) ?
+			   "112bits" : ((cfg == NPC_MCAM_KEY_X2) ?
+					"224bits" : "448bits"));
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX));
+		cfg = (cfg >> 32) & 0x07;
+		seq_printf(filp, "\t\t TX keywidth \t: %s\n",
+			   (cfg == NPC_MCAM_KEY_X1) ?
+			   "112bits" : ((cfg == NPC_MCAM_KEY_X2) ?
+					"224bits" : "448bits"));
+	}
 
 	mutex_lock(&mcam->lock);
 	/* MCAM entries */
@@ -3506,11 +3528,11 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused)
 	struct rvu_npc_mcam_rule *iter;
 	struct rvu *rvu = s->private;
 	struct npc_mcam *mcam;
-	int pf, vf = -1;
+	int pf, vf = -1, bank;
+	u16 target, index;
 	bool enabled;
+	u64 hits, off;
 	int blkaddr;
-	u16 target;
-	u64 hits;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
@@ -3554,6 +3576,15 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused)
 
 		enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, iter->entry);
 		seq_printf(s, "\tenabled: %s\n", enabled ? "yes" : "no");
+		if (is_cn20k(rvu->pdev)) {
+			seq_printf(s, "\tpriority: %u\n", iter->hw_prio);
+			index = iter->entry & (mcam->banksize - 1);
+			bank = npc_get_bank(mcam, iter->entry);
+			off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank);
+			hits = rvu_read64(rvu, blkaddr, off);
+			seq_printf(s, "\thits: %lld\n", hits);
+			continue;
+		}
 
 		if (!iter->has_cntr)
 			continue;
@@ -3698,9 +3729,9 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused)
 	struct npc_exact_table *table;
 	struct rvu *rvu = s->private;
 	struct npc_key_field *field;
+	u64 cfg, cam1, off;
 	u16 chan, pcifunc;
 	int blkaddr, i;
-	u64 cfg, cam1;
 	char *str;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
@@ -3721,11 +3752,17 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused)
 		chan = field->kw_mask[0] & cam1;
 
 		str = (cfg & 1) ? "enabled" : "disabled";
+		if (is_cn20k(rvu->pdev)) {
+			off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(i, 0);
+		seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc,
+			   i, rvu_read64(rvu, blkaddr, off), chan, str);
+		} else {
+			off = NPC_AF_MATCH_STATX(table->counter_idx[i]);
+			seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc,
+				   i, rvu_read64(rvu, blkaddr, off),
+				   chan, str);
+		}
 
-		seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, i,
-			   rvu_read64(rvu, blkaddr,
-				      NPC_AF_MATCH_STATX(table->counter_idx[i])),
-			   chan, str);
 	}
 
 	return 0;
@@ -3745,6 +3782,9 @@ static void rvu_dbg_npc_init(struct rvu *rvu)
 	debugfs_create_file("rx_miss_act_stats", 0444, rvu->rvu_dbg.npc, rvu,
 			    &rvu_dbg_npc_rx_miss_act_fops);
 
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_debugfs_init(rvu);
+
 	if (!rvu->hw->cap.npc_exact_match_enabled)
 		return;
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c
index a29f1ea04c7d8..6494a9ee2f0dd 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c
@@ -11,6 +11,7 @@
 #include "rvu_reg.h"
 #include "rvu_struct.h"
 #include "rvu_npc_hash.h"
+#include "cn20k/npc.h"
 
 #define DRV_NAME "octeontx2-af"
 
@@ -1256,9 +1257,71 @@ enum rvu_af_dl_param_id {
 	RVU_AF_DEVLINK_PARAM_ID_NPC_MCAM_ZONE_PERCENT,
 	RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE,
 	RVU_AF_DEVLINK_PARAM_ID_NPC_DEF_RULE_CNTR_ENABLE,
+	RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG,
 	RVU_AF_DEVLINK_PARAM_ID_NIX_MAXLF,
 };
 
+static int rvu_af_npc_defrag_feature_get(struct devlink *devlink, u32 id,
+					 struct devlink_param_gset_ctx *ctx,
+					 struct netlink_ext_ack *extack)
+{
+	struct rvu_devlink *rvu_dl = devlink_priv(devlink);
+	struct rvu *rvu = rvu_dl->rvu;
+	bool enabled;
+
+	enabled = is_cn20k(rvu->pdev);
+
+	snprintf(ctx->val.vstr, sizeof(ctx->val.vstr), "%s",
+		 enabled ? "enabled" : "disabled");
+
+	return 0;
+}
+
+static int rvu_af_npc_defrag(struct devlink *devlink, u32 id,
+			     struct devlink_param_gset_ctx *ctx,
+			     struct netlink_ext_ack *extack)
+{
+	struct rvu_devlink *rvu_dl = devlink_priv(devlink);
+	struct rvu *rvu = rvu_dl->rvu;
+
+	/* It is hard to roll back if defrag process fails.
+	 * print a error message and return fault.
+	 */
+	if (npc_cn20k_defrag(rvu)) {
+		dev_err(rvu->dev, "Defrag process failed\n");
+		return -EFAULT;
+	}
+	return 0;
+}
+
+static int rvu_af_npc_defrag_feature_validate(struct devlink *devlink, u32 id,
+					      union devlink_param_value val,
+					      struct netlink_ext_ack *extack)
+{
+	struct rvu_devlink *rvu_dl = devlink_priv(devlink);
+	struct rvu *rvu = rvu_dl->rvu;
+	u64 enable;
+
+	if (kstrtoull(val.vstr, 10, &enable)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Only 1 value is supported");
+		return -EINVAL;
+	}
+
+	if (enable != 1) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Only initiating defrag is supported");
+		return -EINVAL;
+	}
+
+	if (is_cn20k(rvu->pdev))
+		return 0;
+
+	NL_SET_ERR_MSG_MOD(extack,
+			   "Can defrag NPC only in cn20k silicon");
+	return -EFAULT;
+}
+
 static int rvu_af_npc_exact_feature_get(struct devlink *devlink, u32 id,
 					struct devlink_param_gset_ctx *ctx,
 					struct netlink_ext_ack *extack)
@@ -1561,6 +1624,15 @@ static const struct devlink_ops rvu_devlink_ops = {
 	.eswitch_mode_set = rvu_devlink_eswitch_mode_set,
 };
 
+static const struct devlink_param rvu_af_dl_param_defrag[] = {
+	DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG,
+			     "npc_defrag", DEVLINK_PARAM_TYPE_STRING,
+			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
+			     rvu_af_npc_defrag_feature_get,
+			     rvu_af_npc_defrag,
+			     rvu_af_npc_defrag_feature_validate),
+};
+
 int rvu_register_dl(struct rvu *rvu)
 {
 	struct rvu_devlink *rvu_dl;
@@ -1593,6 +1665,17 @@ int rvu_register_dl(struct rvu *rvu)
 		goto err_dl_health;
 	}
 
+	if (is_cn20k(rvu->pdev)) {
+		err = devlink_params_register(dl, rvu_af_dl_param_defrag,
+					      ARRAY_SIZE(rvu_af_dl_param_defrag));
+		if (err) {
+			dev_err(rvu->dev,
+				"devlink defrag params register failed with error %d",
+				err);
+			goto err_dl_defrag;
+		}
+	}
+
 	/* Register exact match devlink only for CN10K-B */
 	if (!rvu_npc_exact_has_match_table(rvu))
 		goto done;
@@ -1601,7 +1684,8 @@ int rvu_register_dl(struct rvu *rvu)
 				      ARRAY_SIZE(rvu_af_dl_param_exact_match));
 	if (err) {
 		dev_err(rvu->dev,
-			"devlink exact match params register failed with error %d", err);
+			"devlink exact match params register failed with error %d",
+			err);
 		goto err_dl_exact_match;
 	}
 
@@ -1610,6 +1694,11 @@ int rvu_register_dl(struct rvu *rvu)
 	return 0;
 
 err_dl_exact_match:
+	if (is_cn20k(rvu->pdev))
+		devlink_params_unregister(dl, rvu_af_dl_param_defrag,
+					  ARRAY_SIZE(rvu_af_dl_param_defrag));
+
+err_dl_defrag:
 	devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params));
 
 err_dl_health:
@@ -1627,6 +1716,10 @@ void rvu_unregister_dl(struct rvu *rvu)
 
 	devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params));
 
+	if (is_cn20k(rvu->pdev))
+		devlink_params_unregister(dl, rvu_af_dl_param_defrag,
+					  ARRAY_SIZE(rvu_af_dl_param_defrag));
+
 	/* Unregister exact match devlink only for CN10K-B */
 	if (rvu_npc_exact_has_match_table(rvu))
 		devlink_params_unregister(dl, rvu_af_dl_param_exact_match,
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index 98dd68137a098..ef5b081162ebf 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -2391,8 +2391,8 @@ static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr,
 			continue;
 		/* skip if PF_FUNC doesn't match */
 		if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) !=
-		    (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq] &
-				    ~RVU_PFVF_FUNC_MASK)))
+		    (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq]) &
+				    ~RVU_PFVF_FUNC_MASK))
 			continue;
 		/* enable/disable XOFF */
 		regoff = NIX_AF_TL2X_SW_XOFF(tl2);
@@ -5289,7 +5289,6 @@ int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req,
 	/* Disable the interface if it is in any multicast list */
 	nix_mcast_update_mce_entry(rvu, pcifunc, 0);
 
-
 	pfvf = rvu_get_pfvf(rvu, pcifunc);
 	clear_bit(NIXLF_INITIALIZED, &pfvf->flags);
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
index 8658cb2143dfc..c2ca5ed1d028a 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
@@ -16,6 +16,9 @@
 #include "cgx.h"
 #include "npc_profile.h"
 #include "rvu_npc_hash.h"
+#include "cn20k/npc.h"
+#include "rvu_npc.h"
+#include "cn20k/reg.h"
 
 #define RSVD_MCAM_ENTRIES_PER_PF	3 /* Broadcast, Promisc and AllMulticast */
 #define RSVD_MCAM_ENTRIES_PER_NIXLF	1 /* Ucast for LFs */
@@ -149,10 +152,33 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
 {
 	struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam);
 	struct rvu *rvu = hw->rvu;
-	int pf = rvu_get_pf(rvu->pdev, pcifunc);
+	u16 bcast, mcast, promisc, ucast;
 	int index;
+	int rc;
+	int pf;
+
+	if (is_cn20k(rvu->pdev)) {
+		rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast,
+						 &promisc, &ucast);
+		if (rc)
+			return -EFAULT;
+
+		switch (type) {
+		case NIXLF_BCAST_ENTRY:
+			return bcast;
+		case NIXLF_ALLMULTI_ENTRY:
+			return mcast;
+		case NIXLF_PROMISC_ENTRY:
+			return promisc;
+		case NIXLF_UCAST_ENTRY:
+			return ucast;
+		default:
+			return -EINVAL;
+		}
+	}
 
 	/* Check if this is for a PF */
+	pf = rvu_get_pf(rvu->pdev, pcifunc);
 	if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) {
 		/* Reserved entries exclude PF0 */
 		pf--;
@@ -173,7 +199,12 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
 
 int npc_get_bank(struct npc_mcam *mcam, int index)
 {
+	struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam);
 	int bank = index / mcam->banksize;
+	struct rvu *rvu = hw->rvu;
+
+	if (is_cn20k(rvu->pdev))
+		return bank;
 
 	/* 0,1 & 2,3 banks are combined for this keysize */
 	if (mcam->keysize == NPC_MCAM_KEY_X2)
@@ -189,7 +220,14 @@ bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam,
 	u64 cfg;
 
 	index &= (mcam->banksize - 1);
-	cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank));
+	if (is_cn20k(rvu->pdev))
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index,
+								   bank));
+	else
+		cfg = rvu_read64(rvu, blkaddr,
+				 NPC_AF_MCAMEX_BANKX_CFG(index, bank));
+
 	return (cfg & 1);
 }
 
@@ -199,6 +237,13 @@ void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
 	int bank = npc_get_bank(mcam, index);
 	int actbank = bank;
 
+	if (is_cn20k(rvu->pdev)) {
+		if (index < 0 || index >= mcam->banksize * mcam->banks)
+			return;
+
+		return npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable);
+	}
+
 	index &= (mcam->banksize - 1);
 	for (; bank < (actbank + mcam->banks_per_entry); bank++) {
 		rvu_write64(rvu, blkaddr,
@@ -367,6 +412,7 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam,
 					int blkaddr, u16 pf_func)
 {
 	int bank, nixlf, index;
+	u64 reg;
 
 	/* get ucast entry rule entry index */
 	if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) {
@@ -381,8 +427,12 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam,
 	bank = npc_get_bank(mcam, index);
 	index &= (mcam->banksize - 1);
 
-	return rvu_read64(rvu, blkaddr,
-			  NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
+	if (is_cn20k(rvu->pdev))
+		reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0);
+	else
+		reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank);
+
+	return rvu_read64(rvu, blkaddr, reg);
 }
 
 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam,
@@ -547,6 +597,9 @@ static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
 	u64 cfg, sreg, dreg;
 	int bank, i;
 
+	if (is_cn20k(rvu->pdev))
+		return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest);
+
 	src &= (mcam->banksize - 1);
 	dest &= (mcam->banksize - 1);
 
@@ -583,20 +636,31 @@ u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
 			int blkaddr, int index)
 {
 	int bank = npc_get_bank(mcam, index);
+	u64 reg;
 
 	index &= (mcam->banksize - 1);
-	return rvu_read64(rvu, blkaddr,
-			  NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
+
+	if (is_cn20k(rvu->pdev))
+		reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0);
+	else
+		reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank);
+	return rvu_read64(rvu, blkaddr, reg);
 }
 
 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
 			 int blkaddr, int index, u64 cfg)
 {
 	int bank = npc_get_bank(mcam, index);
+	u64 reg;
 
 	index &= (mcam->banksize - 1);
-	return rvu_write64(rvu, blkaddr,
-			   NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg);
+
+	if (is_cn20k(rvu->pdev))
+		reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0);
+	else
+		reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank);
+
+	return rvu_write64(rvu, blkaddr, reg, cfg);
 }
 
 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
@@ -651,6 +715,9 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
 	req.match_id = action.match_id;
 	req.flow_key_alg = action.flow_key_alg;
 
+	if (is_cn20k(rvu->pdev))
+		req.hw_prio = pfvf->hw_prio;
+
 	rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
 }
 
@@ -739,6 +806,9 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
 	req.match_id = action.match_id;
 	req.flow_key_alg = flow_key_alg;
 
+	if (is_cn20k(rvu->pdev))
+		req.hw_prio = pfvf->hw_prio;
+
 	rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
 }
 
@@ -819,6 +889,9 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc,
 	req.hdr.pcifunc = 0; /* AF is requester */
 	req.vf = pcifunc;
 
+	if (is_cn20k(rvu->pdev))
+		req.hw_prio = pfvf->hw_prio;
+
 	rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
 }
 
@@ -907,6 +980,9 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
 	req.match_id = action.match_id;
 	req.flow_key_alg = flow_key_alg;
 
+	if (is_cn20k(rvu->pdev))
+		req.hw_prio = pfvf->hw_prio;
+
 	rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
 }
 
@@ -940,33 +1016,42 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam,
 
 	mutex_lock(&mcam->lock);
 	for (index = 0; index < mcam->bmap_entries; index++) {
-		if (mcam->entry2target_pffunc[index] == pcifunc) {
-			update = true;
-			/* update not needed for the rules added via ntuple filters */
-			list_for_each_entry(rule, &mcam->mcam_rules, list) {
-				if (rule->entry == index)
-					update = false;
-			}
-			if (!update)
-				continue;
-			bank = npc_get_bank(mcam, index);
-			actindex = index;
-			entry = index & (mcam->banksize - 1);
-
-			/* read vf flow entry enable status */
-			enable = is_mcam_entry_enabled(rvu, mcam, blkaddr,
-						       actindex);
-			/* disable before mcam entry update */
-			npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex,
-					      false);
-			/* update 'action' */
+		if (mcam->entry2target_pffunc[index] != pcifunc)
+			continue;
+		update = true;
+		/* update not needed for the rules added via ntuple filters */
+		list_for_each_entry(rule, &mcam->mcam_rules, list) {
+			if (rule->entry == index)
+				update = false;
+		}
+		if (!update)
+			continue;
+		bank = npc_get_bank(mcam, index);
+		actindex = index;
+		entry = index & (mcam->banksize - 1);
+
+		/* read vf flow entry enable status */
+		enable = is_mcam_entry_enabled(rvu, mcam, blkaddr,
+					       actindex);
+		/* disable before mcam entry update */
+		npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex,
+				      false);
+
+		/* update 'action' */
+		if (is_cn20k(rvu->pdev))
+			rvu_write64(rvu, blkaddr,
+				    NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry,
+									  bank,
+									  0),
+				    rx_action);
+		else
 			rvu_write64(rvu, blkaddr,
 				    NPC_AF_MCAMEX_BANKX_ACTION(entry, bank),
 				    rx_action);
-			if (enable)
-				npc_enable_mcam_entry(rvu, mcam, blkaddr,
-						      actindex, true);
-		}
+
+		if (enable)
+			npc_enable_mcam_entry(rvu, mcam, blkaddr,
+					      actindex, true);
 	}
 	mutex_unlock(&mcam->lock);
 }
@@ -979,6 +1064,7 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act
 	struct npc_mcam *mcam = &rvu->hw->mcam;
 	struct rvu_hwinfo *hw = rvu->hw;
 	int bank, op_rss;
+	u64 reg;
 
 	if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index))
 		return;
@@ -988,15 +1074,19 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act
 	bank = npc_get_bank(mcam, mcam_index);
 	mcam_index &= (mcam->banksize - 1);
 
+	if (is_cn20k(rvu->pdev))
+		reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index,
+							    bank, 0);
+	else
+		reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank);
+
 	/* If Rx action is MCAST update only RSS algorithm index */
 	if (!op_rss) {
-		*(u64 *)&action = rvu_read64(rvu, blkaddr,
-				NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank));
+		*(u64 *)&action = rvu_read64(rvu, blkaddr, reg);
 
 		action.flow_key_alg = alg_idx;
 	}
-	rvu_write64(rvu, blkaddr,
-		    NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action);
+	rvu_write64(rvu, blkaddr, reg, *(u64 *)&action);
 }
 
 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
@@ -1006,6 +1096,7 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
 	struct nix_rx_action action;
 	int blkaddr, index, bank;
 	struct rvu_pfvf *pfvf;
+	u64 reg;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
@@ -1028,8 +1119,12 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
 	bank = npc_get_bank(mcam, index);
 	index &= (mcam->banksize - 1);
 
-	*(u64 *)&action = rvu_read64(rvu, blkaddr,
-				     NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
+	if (is_cn20k(rvu->pdev))
+		reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0);
+	else
+		reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank);
+
+	*(u64 *)&action = rvu_read64(rvu, blkaddr, reg);
 	/* Ignore if no action was set earlier */
 	if (!*(u64 *)&action)
 		return;
@@ -1039,8 +1134,11 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
 	action.index = group;
 	action.flow_key_alg = alg_idx;
 
-	rvu_write64(rvu, blkaddr,
-		    NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action);
+	rvu_write64(rvu, blkaddr, reg, *(u64 *)&action);
+	/* update the VF flow rule action with the VF default entry action */
+	if (mcam_index < 0)
+		npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc,
+					 *(u64 *)&action);
 
 	/* update the action change in default rule */
 	pfvf = rvu_get_pfvf(rvu, pcifunc);
@@ -1338,8 +1436,8 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,
 	npc_program_mkex_hash(rvu, blkaddr);
 }
 
-static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr,
-				 u64 *size)
+int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr,
+			  u64 *size)
 {
 	u64 prfl_addr, prfl_sz;
 
@@ -1395,7 +1493,7 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr,
 			 */
 			if (!is_rvu_96xx_B0(rvu) ||
 			    mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX])
-				rvu->kpu.mkex = mcam_kex;
+				rvu->kpu.mcam_kex_prfl.mkex = mcam_kex;
 			goto program_mkex;
 		}
 
@@ -1405,16 +1503,17 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr,
 	dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);
 
 program_mkex:
-	dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name);
+	dev_info(rvu->dev, "Using %s mkex profile\n",
+		 rvu->kpu.mcam_kex_prfl.mkex->name);
 	/* Program selected mkex profile */
-	npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex);
+	npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex);
 	if (mkex_prfl_addr)
 		iounmap(mkex_prfl_addr);
 }
 
-static void npc_config_kpuaction(struct rvu *rvu, int blkaddr,
-				 const struct npc_kpu_profile_action *kpuaction,
-				 int kpu, int entry, bool pkind)
+void npc_config_kpuaction(struct rvu *rvu, int blkaddr,
+			  const struct npc_kpu_profile_action *kpuaction,
+			  int kpu, int entry, bool pkind)
 {
 	struct npc_kpu_action0 action0 = {0};
 	struct npc_kpu_action1 action1 = {0};
@@ -1477,7 +1576,7 @@ static void npc_config_kpucam(struct rvu *rvu, int blkaddr,
 		    NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1);
 }
 
-static inline u64 enable_mask(int count)
+u64 npc_enable_mask(int count)
 {
 	return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL));
 }
@@ -1510,7 +1609,7 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu,
 
 	/* Enable all programmed entries */
 	num_entries = min_t(int, profile->action_entries, profile->cam_entries);
-	entry_mask = enable_mask(num_entries);
+	entry_mask = npc_enable_mask(num_entries);
 	/* Disable first KPU_MAX_CST_ENT entries for built-in profile */
 	if (!rvu->kpu.custom)
 		entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0);
@@ -1519,14 +1618,15 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu,
 	if (num_entries > 64) {
 		rvu_write64(rvu, blkaddr,
 			    NPC_AF_KPUX_ENTRY_DISX(kpu, 1),
-			    enable_mask(num_entries - 64));
+			    npc_enable_mask(num_entries - 64));
 	}
 
 	/* Enable this KPU */
 	rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01);
 }
 
-static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile)
+static void npc_prepare_default_kpu(struct rvu *rvu,
+				    struct npc_kpu_profile_adapter *profile)
 {
 	profile->custom = 0;
 	profile->name = def_pfl_name;
@@ -1536,23 +1636,38 @@ static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile)
 	profile->kpu = npc_kpu_profiles;
 	profile->kpus = ARRAY_SIZE(npc_kpu_profiles);
 	profile->lt_def = &npc_lt_defaults;
-	profile->mkex = &npc_mkex_default;
 	profile->mkex_hash = &npc_mkex_hash_default;
 
-	return 0;
+	if (!is_cn20k(rvu->pdev)) {
+		profile->mcam_kex_prfl.mkex = &npc_mkex_default;
+		return;
+	}
+
+	profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get();
+	ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6;
+	ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0;
+	ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5;
+	ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1;
+
+	npc_cn20k_update_action_entries_n_flags(rvu, profile);
 }
 
 static int npc_apply_custom_kpu(struct rvu *rvu,
 				struct npc_kpu_profile_adapter *profile)
 {
 	size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0;
-	struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata;
 	struct npc_kpu_profile_action *action;
+	struct npc_kpu_profile_fwdata *fw;
 	struct npc_kpu_profile_cam *cam;
 	struct npc_kpu_fwdata *fw_kpu;
 	int entries;
 	u16 kpu, entry;
 
+	if (is_cn20k(rvu->pdev))
+		return npc_cn20k_apply_custom_kpu(rvu, profile);
+
+	fw = rvu->kpu_fwdata;
+
 	if (rvu->kpu_fwdata_sz < hdr_sz) {
 		dev_warn(rvu->dev, "Invalid KPU profile size\n");
 		return -EINVAL;
@@ -1593,7 +1708,7 @@ static int npc_apply_custom_kpu(struct rvu *rvu,
 	profile->custom = 1;
 	profile->name = fw->name;
 	profile->version = le64_to_cpu(fw->version);
-	profile->mkex = &fw->mkex;
+	profile->mcam_kex_prfl.mkex = &fw->mkex;
 	profile->lt_def = &fw->lt_def;
 
 	for (kpu = 0; kpu < fw->kpus; kpu++) {
@@ -1707,7 +1822,7 @@ static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile)
 	return ret;
 }
 
-static void npc_load_kpu_profile(struct rvu *rvu)
+void npc_load_kpu_profile(struct rvu *rvu)
 {
 	struct npc_kpu_profile_adapter *profile = &rvu->kpu;
 	const char *kpu_profile = rvu->kpu_pfl_name;
@@ -1718,7 +1833,7 @@ static void npc_load_kpu_profile(struct rvu *rvu)
 	if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN))
 		goto revert_to_default;
 	/* First prepare default KPU, then we'll customize top entries. */
-	npc_prepare_default_kpu(profile);
+	npc_prepare_default_kpu(rvu, profile);
 
 	/* Order of preceedence for load loading NPC profile (high to low)
 	 * Firmware binary in filesystem.
@@ -1781,7 +1896,7 @@ static void npc_load_kpu_profile(struct rvu *rvu)
 	return;
 
 revert_to_default:
-	npc_prepare_default_kpu(profile);
+	npc_prepare_default_kpu(rvu, profile);
 }
 
 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr)
@@ -1842,39 +1957,56 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr)
 	int cntr;
 	u64 cfg;
 
-	/* Actual number of MCAM entries vary by entry size */
 	cfg = (rvu_read64(rvu, blkaddr,
 			  NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07;
-	mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize;
 	mcam->keysize = cfg;
 
 	/* Number of banks combined per MCAM entry */
-	if (cfg == NPC_MCAM_KEY_X4)
-		mcam->banks_per_entry = 4;
-	else if (cfg == NPC_MCAM_KEY_X2)
-		mcam->banks_per_entry = 2;
-	else
-		mcam->banks_per_entry = 1;
-
-	/* Reserve one MCAM entry for each of the NIX LF to
-	 * guarantee space to install default matching DMAC rule.
-	 * Also reserve 2 MCAM entries for each PF for default
-	 * channel based matching or 'bcast & promisc' matching to
-	 * support BCAST and PROMISC modes of operation for PFs.
-	 * PF0 is excluded.
-	 */
-	rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) +
-		((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF);
-	if (mcam->total_entries <= rsvd) {
-		dev_warn(rvu->dev,
-			 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n",
-			 mcam->total_entries);
-		return -ENOMEM;
+	if (is_cn20k(rvu->pdev)) {
+		/* In cn20k, x2 entries is allowed for x4 profile.
+		 * set total_entries as 8192 * 2 and key size as x2.
+		 */
+		mcam->total_entries = mcam->banks * mcam->banksize;
+		if (cfg == NPC_MCAM_KEY_X2)
+			mcam->banks_per_entry = 1;
+		else
+			mcam->banks_per_entry = 2;
+
+		rsvd = 0;
+	} else {
+		mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) *
+						mcam->banksize;
+
+		if (cfg == NPC_MCAM_KEY_X4)
+			mcam->banks_per_entry = 4;
+		else if (cfg == NPC_MCAM_KEY_X2)
+			mcam->banks_per_entry = 2;
+		else
+			mcam->banks_per_entry = 1;
+
+		/* Reserve one MCAM entry for each of the NIX LF to
+		 * guarantee space to install default matching DMAC rule.
+		 * Also reserve 2 MCAM entries for each PF for default
+		 * channel based matching or 'bcast & promisc' matching to
+		 * support BCAST and PROMISC modes of operation for PFs.
+		 * PF0 is excluded.
+		 */
+		rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) +
+			((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF);
+		if (mcam->total_entries <= rsvd) {
+			dev_warn(rvu->dev,
+				 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n",
+				 mcam->total_entries);
+			return -ENOMEM;
+		}
 	}
 
 	mcam->bmap_entries = mcam->total_entries - rsvd;
-	mcam->nixlf_offset = mcam->bmap_entries;
-	mcam->pf_offset = mcam->nixlf_offset + nixlf_count;
+	/* cn20k does not need offsets to alloc mcam entries */
+	if (!is_cn20k(rvu->pdev)) {
+		mcam->nixlf_offset = mcam->bmap_entries;
+		mcam->pf_offset = mcam->nixlf_offset + nixlf_count;
+	}
 
 	/* Allocate bitmaps for managing MCAM entries */
 	mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL);
@@ -1898,13 +2030,15 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr)
 	 * allocations and another 1/8th at the top for high priority
 	 * allocations.
 	 */
-	mcam->lprio_count = mcam->bmap_entries / 8;
-	if (mcam->lprio_count > BITS_PER_LONG)
-		mcam->lprio_count = round_down(mcam->lprio_count,
-					       BITS_PER_LONG);
-	mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count;
-	mcam->hprio_count = mcam->lprio_count;
-	mcam->hprio_end = mcam->hprio_count;
+	if (!is_cn20k(rvu->pdev)) {
+		mcam->lprio_count = mcam->bmap_entries / 8;
+		if (mcam->lprio_count > BITS_PER_LONG)
+			mcam->lprio_count = round_down(mcam->lprio_count,
+						       BITS_PER_LONG);
+		mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count;
+		mcam->hprio_count = mcam->lprio_count;
+		mcam->hprio_end = mcam->hprio_count;
+	}
 
 	/* Allocate bitmap for managing MCAM counters and memory
 	 * for saving counter to RVU PFFUNC allocation mapping.
@@ -1984,6 +2118,19 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr)
 	hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL;
 	hw->npc_kpu_entries = npc_const1 & 0xFFFULL;
 	hw->npc_kpus = (npc_const >> 8) & 0x1FULL;
+	/* For Cn20k silicon, check if enhanced parser
+	 * is present, then set the NUM_KPMS = NUM_KPUS / 2 and
+	 * number of LDATA extractors per KEX.
+	 */
+	if (is_cn20k(rvu->pdev)) {
+		if (!(npc_const1 & BIT_ULL(62))) {
+			WARN(1, "Enhanced parser is not supported\n");
+			return;
+		}
+		hw->npc_kpms = hw->npc_kpus / 2;
+		hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL;
+	}
+
 	hw->npc_intfs = npc_const & 0xFULL;
 	hw->npc_counters = (npc_const >> 48) & 0xFFFFULL;
 
@@ -2014,28 +2161,36 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr)
 
 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr)
 {
-	struct npc_mcam_kex *mkex = rvu->kpu.mkex;
+	struct npc_mcam_kex_extr *mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr;
+	struct npc_mcam_kex *mkex = rvu->kpu.mcam_kex_prfl.mkex;
 	struct npc_mcam *mcam = &rvu->hw->mcam;
 	struct rvu_hwinfo *hw = rvu->hw;
 	u64 nibble_ena, rx_kex, tx_kex;
+	u64 *keyx_cfg, reg;
 	u8 intf;
 
-	/* Reserve last counter for MCAM RX miss action which is set to
-	 * drop packet. This way we will know how many pkts didn't match
-	 * any MCAM entry.
-	 */
-	mcam->counters.max--;
-	mcam->rx_miss_act_cntr = mcam->counters.max;
+	if (is_cn20k(rvu->pdev)) {
+		keyx_cfg = mkex_extr->keyx_cfg;
+	} else {
+		keyx_cfg = mkex->keyx_cfg;
+		/* Reserve last counter for MCAM RX miss action which is set to
+		 * drop packet. This way we will know how many pkts didn't
+		 * match any MCAM entry.
+		 */
+		mcam->counters.max--;
+		mcam->rx_miss_act_cntr = mcam->counters.max;
+	}
+
+	rx_kex = keyx_cfg[NIX_INTF_RX];
+	tx_kex = keyx_cfg[NIX_INTF_TX];
 
-	rx_kex = mkex->keyx_cfg[NIX_INTF_RX];
-	tx_kex = mkex->keyx_cfg[NIX_INTF_TX];
 	nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex);
 
 	nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena);
 	if (nibble_ena) {
 		tx_kex &= ~NPC_PARSE_NIBBLE;
 		tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena);
-		mkex->keyx_cfg[NIX_INTF_TX] = tx_kex;
+		keyx_cfg[NIX_INTF_TX] = tx_kex;
 	}
 
 	/* Configure RX interfaces */
@@ -2047,12 +2202,19 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr)
 		rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
 			    rx_kex);
 
+		if (is_cn20k(rvu->pdev))
+			reg = NPC_AF_INTFX_MISS_ACTX(intf, 0);
+		else
+			reg = NPC_AF_INTFX_MISS_ACT(intf);
+
 		/* If MCAM lookup doesn't result in a match, drop the received
 		 * packet. And map this action to a counter to count dropped
 		 * packets.
 		 */
-		rvu_write64(rvu, blkaddr,
-			    NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP);
+		rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP);
+
+		if (is_cn20k(rvu->pdev))
+			continue;
 
 		/* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9]
 		 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0]
@@ -2072,12 +2234,15 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr)
 		rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
 			    tx_kex);
 
+		if (is_cn20k(rvu->pdev))
+			reg = NPC_AF_INTFX_MISS_ACTX(intf, 0);
+		else
+			reg = NPC_AF_INTFX_MISS_ACT(intf);
+
 		/* Set TX miss action to UCAST_DEFAULT i.e
 		 * transmit the packet on NIX LF SQ's default channel.
 		 */
-		rvu_write64(rvu, blkaddr,
-			    NPC_AF_INTFX_MISS_ACT(intf),
-			    NIX_TX_ACTIONOP_UCAST_DEFAULT);
+		rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT);
 	}
 }
 
@@ -2118,7 +2283,10 @@ int rvu_npc_init(struct rvu *rvu)
 		return -ENOMEM;
 
 	/* Configure KPU profile */
-	npc_parser_profile_init(rvu, blkaddr);
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_parser_profile_init(rvu, blkaddr);
+	else
+		npc_parser_profile_init(rvu, blkaddr);
 
 	/* Config Outer L2, IPv4's NPC layer info */
 	rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2,
@@ -2149,7 +2317,10 @@ int rvu_npc_init(struct rvu *rvu)
 
 	npc_config_secret_key(rvu, blkaddr);
 	/* Configure MKEX profile */
-	npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name);
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name);
+	else
+		npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name);
 
 	err = npc_mcam_rsrcs_init(rvu, blkaddr);
 	if (err)
@@ -2159,9 +2330,15 @@ int rvu_npc_init(struct rvu *rvu)
 	if (err) {
 		dev_err(rvu->dev,
 			"Incorrect mkex profile loaded using default mkex\n");
-		npc_load_mkex_profile(rvu, blkaddr, def_pfl_name);
+		if (is_cn20k(rvu->pdev))
+			npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name);
+		else
+			npc_load_mkex_profile(rvu, blkaddr, def_pfl_name);
 	}
 
+	if (is_cn20k(rvu->pdev))
+		return npc_cn20k_init(rvu);
+
 	return 0;
 }
 
@@ -2177,6 +2354,9 @@ void rvu_npc_freemem(struct rvu *rvu)
 	else
 		kfree(rvu->kpu_fwdata);
 	mutex_destroy(&mcam->lock);
+
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_deinit(rvu);
 }
 
 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc,
@@ -2217,8 +2397,8 @@ void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc,
 	}
 }
 
-static int npc_mcam_verify_entry(struct npc_mcam *mcam,
-				 u16 pcifunc, int entry)
+int npc_mcam_verify_entry(struct npc_mcam *mcam,
+			  u16 pcifunc, int entry)
 {
 	/* verify AF installed entries */
 	if (is_pffunc_af(pcifunc))
@@ -2260,6 +2440,10 @@ static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam,
 	/* Set mapping and increment counter's refcnt */
 	mcam->entry2cntr_map[entry] = cntr;
 	mcam->cntr_refcnt[cntr]++;
+
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	/* Enable stats */
 	rvu_write64(rvu, blkaddr,
 		    NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank),
@@ -2276,6 +2460,10 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu,
 	/* Remove mapping and reduce counter's refcnt */
 	mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP;
 	mcam->cntr_refcnt[cntr]--;
+
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	/* Disable stats */
 	rvu_write64(rvu, blkaddr,
 		    NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00);
@@ -2285,7 +2473,7 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu,
  * reverse bitmap too. Should be called with
  * 'mcam->lock' held.
  */
-static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index)
+void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index)
 {
 	u16 entry, rentry;
 
@@ -2301,7 +2489,7 @@ static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index)
  * reverse bitmap too. Should be called with
  * 'mcam->lock' held.
  */
-static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index)
+void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index)
 {
 	u16 entry, rentry;
 
@@ -2317,6 +2505,7 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,
 				      int blkaddr, u16 pcifunc)
 {
 	u16 index, cntr;
+	int rc;
 
 	/* Scan all MCAM entries and free the ones mapped to 'pcifunc' */
 	for (index = 0; index < mcam->bmap_entries; index++) {
@@ -2334,6 +2523,13 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,
 							      blkaddr, index,
 							      cntr);
 			mcam->entry2target_pffunc[index] = 0x0;
+			if (is_cn20k(rvu->pdev)) {
+				rc = npc_cn20k_idx_free(rvu, &index, 1);
+				if (rc)
+					dev_err(rvu->dev,
+						"Failed to free mcam idx=%u pcifunc=%#x\n",
+						index, pcifunc);
+			}
 		}
 	}
 }
@@ -2423,7 +2619,7 @@ npc_get_mcam_search_range_priority(struct npc_mcam *mcam,
 {
 	u16 fcnt;
 
-	if (req->priority == NPC_MCAM_HIGHER_PRIO)
+	if (req->ref_prio == NPC_MCAM_HIGHER_PRIO)
 		goto hprio;
 
 	/* For a low priority entry allocation
@@ -2480,14 +2676,80 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
 				  struct npc_mcam_alloc_entry_req *req,
 				  struct npc_mcam_alloc_entry_rsp *rsp)
 {
+	struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam);
 	u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES];
 	u16 fcnt, hp_fcnt, lp_fcnt;
+	struct rvu *rvu = hw->rvu;
 	u16 start, end, index;
 	int entry, next_start;
 	bool reverse = false;
 	unsigned long *bmap;
+	int ret, limit = 0;
 	u16 max_contig;
 
+	if (!is_cn20k(rvu->pdev))
+		goto not_cn20k;
+
+	/* Only x2 or x4 key types are accepted */
+	if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4)
+		return NPC_MCAM_INVALID_REQ;
+
+	/* The below table is being followed during allocation,
+	 *
+	 * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to
+	 *						    allocate 0th index
+	 * 2. ref_entry == 0 && prio == HIGH && count > 1  : Invalid request
+	 * 3. ref_entry == 0 && prio == LOW && count >= 1  : limit = 0
+	 * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0
+	 * 5. ref_entry != 0 && prio == LOW && count >=1   : limit = Max
+	 *						(X2 2*8192, X4 8192)
+	 */
+	if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) {
+		if (req->kw_type == NPC_MCAM_KEY_X2)
+			limit = 2 * mcam->bmap_entries;
+		else
+			limit = mcam->bmap_entries;
+	}
+
+	ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type,
+				      req->ref_prio, rsp->entry_list,
+				      req->ref_entry, limit,
+				      req->contig, req->count, !!req->virt);
+
+	if (ret) {
+		rsp->count = 0;
+		return NPC_MCAM_ALLOC_FAILED;
+	}
+
+	rsp->count = req->count;
+	if (req->contig)
+		rsp->entry = rsp->entry_list[0];
+
+	/* cn20k, entries allocation algorithm is different.
+	 * This common API updates some bitmap on usage etc, which
+	 * will be used by other functions. So update those for
+	 * cn20k as well.
+	 */
+
+	mutex_lock(&mcam->lock);
+	/* Mark the allocated entries as used and set nixlf mapping */
+	for (entry = 0; entry < rsp->count; entry++) {
+		index = npc_cn20k_vidx2idx(rsp->entry_list[entry]);
+		npc_mcam_set_bit(mcam, index);
+		mcam->entry2pfvf_map[index] = pcifunc;
+		mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP;
+	}
+
+	/* cn20k, free count is provided thru different mbox message.
+	 * one counter to indicate free x2 slots and free x4 slots
+	 * does not provide any useful information to the user.
+	 */
+	rsp->free_count = -1;
+	mutex_unlock(&mcam->lock);
+
+	return 0;
+
+not_cn20k:
 	mutex_lock(&mcam->lock);
 
 	/* Check if there are any free entries */
@@ -2523,7 +2785,7 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
 		goto lprio_alloc;
 
 	/* Get the search range for priority allocation request */
-	if (req->priority) {
+	if (req->ref_prio) {
 		npc_get_mcam_search_range_priority(mcam, req,
 						   &start, &end, &reverse);
 		goto alloc;
@@ -2564,11 +2826,11 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
 		 * and not in mid zone.
 		 */
 		if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&
-		    req->priority == NPC_MCAM_HIGHER_PRIO)
+		    req->ref_prio == NPC_MCAM_HIGHER_PRIO)
 			end = req->ref_entry;
 
 		if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&
-		    req->priority == NPC_MCAM_LOWER_PRIO)
+		    req->ref_prio == NPC_MCAM_LOWER_PRIO)
 			start = req->ref_entry;
 	}
 
@@ -2617,7 +2879,7 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
 	/* If allocating requested no of entries is unsucessful,
 	 * expand the search range to full bitmap length and retry.
 	 */
-	if (!req->priority && (rsp->count < req->count) &&
+	if (!req->ref_prio && rsp->count < req->count &&
 	    ((end - start) != mcam->bmap_entries)) {
 		reverse = true;
 		start = 0;
@@ -2628,14 +2890,14 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
 	/* For priority entry allocation requests, if allocation is
 	 * failed then expand search to max possible range and retry.
 	 */
-	if (req->priority && rsp->count < req->count) {
-		if (req->priority == NPC_MCAM_LOWER_PRIO &&
+	if (req->ref_prio && rsp->count < req->count) {
+		if (req->ref_prio == NPC_MCAM_LOWER_PRIO &&
 		    (start != (req->ref_entry + 1))) {
 			start = req->ref_entry + 1;
 			end = mcam->bmap_entries;
 			reverse = false;
 			goto alloc;
-		} else if ((req->priority == NPC_MCAM_HIGHER_PRIO) &&
+		} else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) &&
 			   ((end - start) != req->ref_entry)) {
 			start = 0;
 			end = req->ref_entry;
@@ -2689,6 +2951,10 @@ int npc_config_cntr_default_entries(struct rvu *rvu, bool enable)
 	struct rvu_npc_mcam_rule *rule;
 	int blkaddr;
 
+	/* Counter is set for each rule by default */
+	if (is_cn20k(rvu->pdev))
+		return -EINVAL;
+
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return -EINVAL;
@@ -2749,9 +3015,9 @@ int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu,
 	/* ref_entry can't be '0' if requested priority is high.
 	 * Can't be last entry if requested priority is low.
 	 */
-	if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) ||
-	    ((req->ref_entry == mcam->bmap_entries) &&
-	     req->priority == NPC_MCAM_LOWER_PRIO))
+	if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) ||
+	    (req->ref_entry == mcam->bmap_entries &&
+	     req->ref_prio == NPC_MCAM_LOWER_PRIO))
 		return NPC_MCAM_INVALID_REQ;
 
 	/* Since list of allocated indices needs to be sent to requester,
@@ -2780,6 +3046,8 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu,
 	int blkaddr, rc = 0;
 	u16 cntr;
 
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
@@ -2808,6 +3076,14 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu,
 		npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
 					      req->entry, cntr);
 
+	if (is_cn20k(rvu->pdev)) {
+		rc = npc_cn20k_idx_free(rvu, &req->entry, 1);
+		if (rc)
+			dev_err(rvu->dev,
+				"Failed to free index=%u\n",
+				req->entry);
+	}
+
 	goto exit;
 
 free_all:
@@ -2861,10 +3137,13 @@ int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu,
 	if (rc)
 		goto exit;
 
-	if (req->set_cntr &&
-	    npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) {
-		rc = NPC_MCAM_INVALID_REQ;
-		goto exit;
+	if (!is_cn20k(rvu->pdev)) {
+		/* Verify counter in SoCs other than cn20k */
+		if (req->set_cntr &&
+		    npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) {
+			rc = NPC_MCAM_INVALID_REQ;
+			goto exit;
+		}
 	}
 
 	if (!is_npc_interface_valid(rvu, req->intf)) {
@@ -2902,6 +3181,8 @@ int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu,
 	u16 pcifunc = req->hdr.pcifunc;
 	int blkaddr, rc;
 
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
@@ -2925,6 +3206,8 @@ int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu,
 	u16 pcifunc = req->hdr.pcifunc;
 	int blkaddr, rc;
 
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
@@ -2959,8 +3242,8 @@ int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu,
 
 	mutex_lock(&mcam->lock);
 	for (index = 0; index < req->shift_count; index++) {
-		old_entry = req->curr_entry[index];
-		new_entry = req->new_entry[index];
+		old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]);
+		new_entry = npc_cn20k_vidx2idx(req->new_entry[index]);
 
 		/* Check if both old and new entries are valid and
 		 * does belong to this PFFUNC or not.
@@ -3032,7 +3315,6 @@ static int __npc_mcam_alloc_counter(struct rvu *rvu,
 	if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS)
 		return NPC_MCAM_INVALID_REQ;
 
-
 	/* Check if unused counters are available or not */
 	if (!rvu_rsrc_free_count(&mcam->counters)) {
 		return NPC_MCAM_ALLOC_FAILED;
@@ -3077,6 +3359,10 @@ int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu,
 	struct npc_mcam *mcam = &rvu->hw->mcam;
 	int err;
 
+	/* Counter is not supported for CN20K */
+	if (is_cn20k(rvu->pdev))
+		return NPC_MCAM_INVALID_REQ;
+
 	mutex_lock(&mcam->lock);
 
 	err = __npc_mcam_alloc_counter(rvu, req, rsp);
@@ -3131,6 +3417,10 @@ int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu,
 	struct npc_mcam *mcam = &rvu->hw->mcam;
 	int err;
 
+	/* Counter is not supported for CN20K */
+	if (is_cn20k(rvu->pdev))
+		return NPC_MCAM_INVALID_REQ;
+
 	mutex_lock(&mcam->lock);
 
 	err = __npc_mcam_free_counter(rvu, req, rsp);
@@ -3189,6 +3479,10 @@ int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu,
 	u16 index, entry = 0;
 	int blkaddr, rc;
 
+	/* Counter is not supported for CN20K */
+	if (is_cn20k(rvu->pdev))
+		return NPC_MCAM_INVALID_REQ;
+
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
@@ -3233,12 +3527,23 @@ int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu,
 		struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp)
 {
 	struct npc_mcam *mcam = &rvu->hw->mcam;
-	int blkaddr, err;
+	int blkaddr, err, index, bank;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
 
+	/* For cn20k, npc mcam index is passed as cntr, as each
+	 * mcam entry has corresponding counter.
+	 */
+	if (is_cn20k(rvu->pdev)) {
+		index = req->cntr & (mcam->banksize - 1);
+		bank = npc_get_bank(mcam, req->cntr);
+		rvu_write64(rvu, blkaddr,
+			    NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0);
+		return 0;
+	}
+
 	mutex_lock(&mcam->lock);
 	err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
 	mutex_unlock(&mcam->lock);
@@ -3255,12 +3560,26 @@ int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu,
 			struct npc_mcam_oper_counter_rsp *rsp)
 {
 	struct npc_mcam *mcam = &rvu->hw->mcam;
-	int blkaddr, err;
+	int blkaddr, err, index, bank;
+	u64 regval;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
 
+	/* In CN20K, mcam index is passed cntr. Each cn20k mcam entry
+	 * has its own counter. No need to verify the counter index.
+	 */
+	if (is_cn20k(rvu->pdev)) {
+		index = req->cntr & (mcam->banksize - 1);
+		bank = npc_get_bank(mcam, req->cntr);
+		regval = rvu_read64(rvu, blkaddr,
+				    NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index,
+								       bank));
+		rsp->stat = regval;
+		return 0;
+	}
+
 	mutex_lock(&mcam->lock);
 	err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
 	mutex_unlock(&mcam->lock);
@@ -3298,7 +3617,7 @@ int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu,
 	/* Try to allocate a MCAM entry */
 	entry_req.hdr.pcifunc = req->hdr.pcifunc;
 	entry_req.contig = true;
-	entry_req.priority = req->priority;
+	entry_req.ref_prio = req->ref_prio;
 	entry_req.ref_entry = req->ref_entry;
 	entry_req.count = 1;
 
@@ -3555,13 +3874,26 @@ int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu,
 	if (blkaddr < 0)
 		return NPC_MCAM_INVALID_REQ;
 
-	mutex_lock(&mcam->lock);
+	req->entry = npc_cn20k_vidx2idx(req->entry);
 
 	index = req->entry & (mcam->banksize - 1);
 	bank = npc_get_bank(mcam, req->entry);
 
+	mutex_lock(&mcam->lock);
+
+	if (is_cn20k(rvu->pdev)) {
+		regval = rvu_read64(rvu, blkaddr,
+				    NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index,
+								       bank));
+		rsp->stat_ena = 1;
+		rsp->stat = regval;
+		mutex_unlock(&mcam->lock);
+		return 0;
+	}
+
 	/* read MCAM entry STAT_ACT register */
-	regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank));
+	regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index,
+								       bank));
 
 	if (!(regval & rvu->hw->npc_stat_ena)) {
 		rsp->stat_ena = 0;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h
new file mode 100644
index 0000000000000..83c5e32e2afca
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell RVU Admin Function driver
+ *
+ * Copyright (C) 2026 Marvell.
+ *
+ */
+
+#ifndef RVU_NPC_H
+#define RVU_NPC_H
+
+u64 npc_enable_mask(int count);
+void npc_load_kpu_profile(struct rvu *rvu);
+void npc_config_kpuaction(struct rvu *rvu, int blkaddr,
+			  const struct npc_kpu_profile_action *kpuaction,
+			  int kpu, int entry, bool pkind);
+int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr,
+			  u64 *size);
+
+void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index);
+void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index);
+#endif /* RVU_NPC_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
index 1930b54e72f21..b45798d9fdabd 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
@@ -12,6 +12,8 @@
 #include "npc.h"
 #include "rvu_npc_fs.h"
 #include "rvu_npc_hash.h"
+#include "cn20k/reg.h"
+#include "cn20k/npc.h"
 
 static const char * const npc_flow_names[] = {
 	[NPC_DMAC]	= "dmac",
@@ -81,19 +83,26 @@ const char *npc_get_field_name(u8 hdr)
 /* Compute keyword masks and figure out the number of keywords a field
  * spans in the key.
  */
-static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type,
+static void npc_set_kw_masks(struct rvu *rvu, struct npc_mcam *mcam, u8 type,
 			     u8 nr_bits, int start_kwi, int offset, u8 intf)
 {
 	struct npc_key_field *field = &mcam->rx_key_fields[type];
 	u8 bits_in_kw;
 	int max_kwi;
 
-	if (mcam->banks_per_entry == 1)
-		max_kwi = 1; /* NPC_MCAM_KEY_X1 */
-	else if (mcam->banks_per_entry == 2)
-		max_kwi = 3; /* NPC_MCAM_KEY_X2 */
-	else
-		max_kwi = 6; /* NPC_MCAM_KEY_X4 */
+	if (is_cn20k(rvu->pdev)) {
+		if (mcam->banks_per_entry == 1)
+			max_kwi = 3; /* NPC_MCAM_KEY_X2 */
+		else
+			max_kwi = 7; /* NPC_MCAM_KEY_X4 */
+	} else {
+		if (mcam->banks_per_entry == 1)
+			max_kwi = 1; /* NPC_MCAM_KEY_X1 */
+		else if (mcam->banks_per_entry == 2)
+			max_kwi = 3; /* NPC_MCAM_KEY_X2 */
+		else
+			max_kwi = 6; /* NPC_MCAM_KEY_X4 */
+	}
 
 	if (is_npc_intf_tx(intf))
 		field = &mcam->tx_key_fields[type];
@@ -155,7 +164,8 @@ static bool npc_is_same(struct npc_key_field *input,
 		     sizeof(struct npc_layer_mdata)) == 0;
 }
 
-static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
+static void npc_set_layer_mdata(struct rvu *rvu,
+				struct npc_mcam *mcam, enum key_fields type,
 				u64 cfg, u8 lid, u8 lt, u8 intf)
 {
 	struct npc_key_field *input = &mcam->rx_key_fields[type];
@@ -165,13 +175,17 @@ static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
 
 	input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 	input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg);
-	input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
+	if (is_cn20k(rvu->pdev))
+		input->layer_mdata.len = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1;
+	else
+		input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
 	input->layer_mdata.ltype = lt;
 	input->layer_mdata.lid = lid;
 }
 
 static bool npc_check_overlap_fields(struct npc_key_field *input1,
-				     struct npc_key_field *input2)
+				     struct npc_key_field *input2,
+				     int max_kw)
 {
 	int kwi;
 
@@ -182,7 +196,7 @@ static bool npc_check_overlap_fields(struct npc_key_field *input1,
 	    input1->layer_mdata.ltype != input2->layer_mdata.ltype)
 		return false;
 
-	for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) {
+	for (kwi = 0; kwi < max_kw; kwi++) {
 		if (input1->kw_mask[kwi] & input2->kw_mask[kwi])
 			return true;
 	}
@@ -202,6 +216,7 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 	struct npc_key_field *dummy, *input;
 	int start_kwi, offset;
 	u8 nr_bits, lid, lt, ld;
+	int extr, kws;
 	u64 cfg;
 
 	dummy = &mcam->rx_key_fields[NPC_UNKNOWN];
@@ -212,6 +227,11 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 		input = &mcam->tx_key_fields[type];
 	}
 
+	kws = NPC_KWS_IN_KEY_SZ_7;
+
+	if (is_cn20k(rvu->pdev))
+		goto skip_cn10k_config;
+
 	for (lid = start_lid; lid < NPC_MAX_LID; lid++) {
 		for (lt = 0; lt < NPC_MAX_LT; lt++) {
 			for (ld = 0; ld < NPC_MAX_LD; ld++) {
@@ -221,8 +241,8 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 				if (!FIELD_GET(NPC_LDATA_EN, cfg))
 					continue;
 				memset(dummy, 0, sizeof(struct npc_key_field));
-				npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg,
-						    lid, lt, intf);
+				npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN,
+						    cfg, lid, lt, intf);
 				/* exclude input */
 				if (npc_is_same(input, dummy))
 					continue;
@@ -230,16 +250,49 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 				offset = (dummy->layer_mdata.key * 8) % 64;
 				nr_bits = dummy->layer_mdata.len * 8;
 				/* form KW masks */
-				npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits,
-						 start_kwi, offset, intf);
+				npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN,
+						 nr_bits, start_kwi,
+						 offset, intf);
 				/* check any input field bits falls in any
 				 * other field bits.
 				 */
-				if (npc_check_overlap_fields(dummy, input))
+				if (npc_check_overlap_fields(dummy, input, kws))
 					return true;
 			}
 		}
 	}
+	return false;
+
+skip_cn10k_config:
+	for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) {
+		lid = CN20K_GET_EXTR_LID(intf, extr);
+		if (lid < start_lid)
+			continue;
+		for (lt = 0; lt < NPC_MAX_LT; lt++) {
+			cfg = CN20K_GET_EXTR_LT(intf, extr, lt);
+			if (!FIELD_GET(NPC_LDATA_EN, cfg))
+				continue;
+
+			memset(dummy, 0, sizeof(struct npc_key_field));
+			npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN, cfg,
+					    lid, lt, intf);
+			/* exclude input */
+			if (npc_is_same(input, dummy))
+				continue;
+			start_kwi = dummy->layer_mdata.key / 8;
+			offset = (dummy->layer_mdata.key * 8) % 64;
+			nr_bits = dummy->layer_mdata.len * 8;
+			/* form KW masks */
+			npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN, nr_bits,
+					 start_kwi, offset, intf);
+			/* check any input field bits falls in any other
+			 * field bits
+			 */
+			if (npc_check_overlap_fields(dummy, input,
+						     NPC_KWS_IN_KEY_SZ_8))
+				return true;
+		}
+	}
 
 	return false;
 }
@@ -253,7 +306,8 @@ static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type,
 	return true;
 }
 
-static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number,
+static void npc_scan_exact_result(struct rvu *rvu,
+				  struct npc_mcam *mcam, u8 bit_number,
 				  u8 key_nibble, u8 intf)
 {
 	u8 offset = (key_nibble * 4) % 64; /* offset within key word */
@@ -269,10 +323,63 @@ static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number,
 	default:
 		return;
 	}
-	npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
+	npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf);
+}
+
+static void npc_cn20k_scan_parse_result(struct rvu *rvu, struct npc_mcam *mcam,
+					u8 bit_number, u8 key_nibble, u8 intf)
+{
+	u8 offset = (key_nibble * 4) % 64; /* offset within key word */
+	u8 kwi = (key_nibble * 4) / 64; /* which word in key */
+	u8 nr_bits = 4; /* bits in a nibble */
+	u8 type;
+
+	switch (bit_number) {
+	case 0 ... 2:
+		type = NPC_CHAN;
+		break;
+	case 3:
+		type = NPC_ERRLEV;
+		break;
+	case 4 ... 5:
+		type = NPC_ERRCODE;
+		break;
+	case 6:
+		type = NPC_LXMB;
+		break;
+	case 8:
+		type = NPC_LA;
+		break;
+	case 10:
+		type = NPC_LB;
+		break;
+	case 12:
+		type = NPC_LC;
+		break;
+	case 14:
+		type = NPC_LD;
+		break;
+	case 16:
+		type = NPC_LE;
+		break;
+	case 18:
+		type = NPC_LF;
+		break;
+	case 20:
+		type = NPC_LG;
+		break;
+	case 22:
+		type = NPC_LH;
+		break;
+	default:
+		return;
+	}
+
+	npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf);
 }
 
-static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
+static void npc_scan_parse_result(struct rvu *rvu,
+				  struct npc_mcam *mcam, u8 bit_number,
 				  u8 key_nibble, u8 intf)
 {
 	u8 offset = (key_nibble * 4) % 64; /* offset within key word */
@@ -280,6 +387,12 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
 	u8 nr_bits = 4; /* bits in a nibble */
 	u8 type;
 
+	if (is_cn20k(rvu->pdev)) {
+		npc_cn20k_scan_parse_result(rvu, mcam, bit_number,
+					    key_nibble, intf);
+		return;
+	}
+
 	switch (bit_number) {
 	case 0 ... 2:
 		type = NPC_CHAN;
@@ -322,7 +435,7 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
 		return;
 	}
 
-	npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
+	npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf);
 }
 
 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
@@ -343,8 +456,13 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 	/* Inner VLAN TCI for double tagged frames */
 	struct npc_key_field *vlan_tag3;
 	u64 *features;
+	int i, max_kw;
 	u8 start_lid;
-	int i;
+
+	if (is_cn20k(rvu->pdev))
+		max_kw = NPC_KWS_IN_KEY_SZ_8;
+	else
+		max_kw = NPC_KWS_IN_KEY_SZ_7;
 
 	key_fields = mcam->rx_key_fields;
 	features = &mcam->rx_features;
@@ -382,7 +500,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 
 	/* if key profile programmed extracts Ethertype from multiple layers */
 	if (etype_ether->nr_kws && etype_tag1->nr_kws) {
-		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+		for (i = 0; i < max_kw; i++) {
 			if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) {
 				dev_err(rvu->dev, "mkex: Etype pos is different for untagged and tagged pkts.\n");
 				goto vlan_tci;
@@ -391,7 +509,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 		key_fields[NPC_ETYPE] = *etype_tag1;
 	}
 	if (etype_ether->nr_kws && etype_tag2->nr_kws) {
-		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+		for (i = 0; i < max_kw; i++) {
 			if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) {
 				dev_err(rvu->dev, "mkex: Etype pos is different for untagged and double tagged pkts.\n");
 				goto vlan_tci;
@@ -400,7 +518,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 		key_fields[NPC_ETYPE] = *etype_tag2;
 	}
 	if (etype_tag1->nr_kws && etype_tag2->nr_kws) {
-		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+		for (i = 0; i < max_kw; i++) {
 			if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) {
 				dev_err(rvu->dev, "mkex: Etype pos is different for tagged and double tagged pkts.\n");
 				goto vlan_tci;
@@ -431,7 +549,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 
 	/* if key profile extracts outer vlan tci from multiple layers */
 	if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) {
-		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+		for (i = 0; i < max_kw; i++) {
 			if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) {
 				dev_err(rvu->dev, "mkex: Out vlan tci pos is different for tagged and double tagged pkts.\n");
 				goto done;
@@ -466,7 +584,11 @@ static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid,
 	/* starting KW index and starting bit position */
 	int start_kwi, offset;
 
-	nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
+	if (is_cn20k(rvu->pdev))
+		nr_bytes = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1;
+	else
+		nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
+
 	hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 	key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 
@@ -489,11 +611,12 @@ do {									       \
 		if ((hstart) >= hdr &&					       \
 		    ((hstart) + (hlen)) <= (hdr + nr_bytes)) {	               \
 			bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \
-			npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \
+			npc_set_layer_mdata(rvu, mcam, (name), cfg, lid, lt,   \
+									intf); \
 			offset += bit_offset;				       \
 			start_kwi += offset / 64;			       \
 			offset %= 64;					       \
-			npc_set_kw_masks(mcam, (name), (hlen) * 8,	       \
+			npc_set_kw_masks(rvu, mcam, (name), (hlen) * 8,	       \
 					 start_kwi, offset, intf);	       \
 		}							       \
 	}								       \
@@ -636,6 +759,7 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 	u8 lid, lt, ld, bitnr;
 	u64 cfg, masked_cfg;
 	u8 key_nibble = 0;
+	int extr;
 
 	/* Scan and note how parse result is going to be in key.
 	 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from
@@ -643,10 +767,22 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 	 * will be concatenated in key.
 	 */
 	cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
-	masked_cfg = cfg & NPC_PARSE_NIBBLE;
-	for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) {
-		npc_scan_parse_result(mcam, bitnr, key_nibble, intf);
-		key_nibble++;
+	if (is_cn20k(rvu->pdev)) {
+		masked_cfg = cfg & NPC_CN20K_PARSE_NIBBLE;
+		for_each_set_bit(bitnr, (unsigned long *)&masked_cfg,
+				 NPC_CN20K_TOTAL_NIBBLE) {
+			npc_scan_parse_result(rvu, mcam, bitnr,
+					      key_nibble, intf);
+			key_nibble++;
+		}
+	} else {
+		masked_cfg = cfg & NPC_PARSE_NIBBLE;
+		for_each_set_bit(bitnr, (unsigned long *)&masked_cfg,
+				 NPC_TOTAL_NIBBLE) {
+			npc_scan_parse_result(rvu, mcam, bitnr,
+					      key_nibble, intf);
+			key_nibble++;
+		}
 	}
 
 	/* Ignore exact match bits for mcam entries except the first rule
@@ -656,10 +792,13 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 	masked_cfg = cfg & NPC_EXACT_NIBBLE;
 	bitnr = NPC_EXACT_NIBBLE_START;
 	for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, NPC_EXACT_NIBBLE_END + 1) {
-		npc_scan_exact_result(mcam, bitnr, key_nibble, intf);
+		npc_scan_exact_result(rvu, mcam, bitnr, key_nibble, intf);
 		key_nibble++;
 	}
 
+	if (is_cn20k(rvu->pdev))
+		goto skip_cn10k_config;
+
 	/* Scan and note how layer data is going to be in key */
 	for (lid = 0; lid < NPC_MAX_LID; lid++) {
 		for (lt = 0; lt < NPC_MAX_LT; lt++) {
@@ -676,6 +815,19 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 	}
 
 	return 0;
+
+skip_cn10k_config:
+	for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) {
+		lid = CN20K_GET_EXTR_LID(intf, extr);
+		for (lt = 0; lt < NPC_MAX_LT; lt++) {
+			cfg = CN20K_GET_EXTR_LT(intf, extr, lt);
+			if (!FIELD_GET(NPC_LDATA_EN, cfg))
+				continue;
+			npc_scan_ldata(rvu, blkaddr, lid, lt, cfg,
+				       intf);
+		}
+	}
+	return 0;
 }
 
 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr)
@@ -751,15 +903,17 @@ static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf)
  * dont care.
  */
 void npc_update_entry(struct rvu *rvu, enum key_fields type,
-		      struct mcam_entry *entry, u64 val_lo,
+		      struct mcam_entry_mdata *mdata, u64 val_lo,
 		      u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf)
 {
+	u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
+	u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
 	struct npc_mcam *mcam = &rvu->hw->mcam;
-	struct mcam_entry dummy = { {0} };
 	struct npc_key_field *field;
 	u64 kw1, kw2, kw3;
+	u64 *val, *mask;
+	int i, max_kw;
 	u8 shift;
-	int i;
 
 	field = &mcam->rx_key_fields[type];
 	if (is_npc_intf_tx(intf))
@@ -768,17 +922,22 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type,
 	if (!field->nr_kws)
 		return;
 
-	for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+	if (is_cn20k(rvu->pdev))
+		max_kw = NPC_KWS_IN_KEY_SZ_8;
+	else
+		max_kw = NPC_KWS_IN_KEY_SZ_7;
+
+	for (i = 0; i < max_kw; i++) {
 		if (!field->kw_mask[i])
 			continue;
 		/* place key value in kw[x] */
 		shift = __ffs64(field->kw_mask[i]);
 		/* update entry value */
 		kw1 = (val_lo << shift) & field->kw_mask[i];
-		dummy.kw[i] = kw1;
+		kw[i] = kw1;
 		/* update entry mask */
 		kw1 = (mask_lo << shift) & field->kw_mask[i];
-		dummy.kw_mask[i] = kw1;
+		kw_mask[i] = kw1;
 
 		if (field->nr_kws == 1)
 			break;
@@ -788,12 +947,12 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type,
 			kw2 = shift ? val_lo >> (64 - shift) : 0;
 			kw2 |= (val_hi << shift);
 			kw2 &= field->kw_mask[i + 1];
-			dummy.kw[i + 1] = kw2;
+			kw[i + 1] = kw2;
 			/* update entry mask */
 			kw2 = shift ? mask_lo >> (64 - shift) : 0;
 			kw2 |= (mask_hi << shift);
 			kw2 &= field->kw_mask[i + 1];
-			dummy.kw_mask[i + 1] = kw2;
+			kw_mask[i + 1] = kw2;
 			break;
 		}
 		/* place remaining bits of key value in kw[x + 1], kw[x + 2] */
@@ -804,34 +963,40 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type,
 			kw2 &= field->kw_mask[i + 1];
 			kw3 = shift ? val_hi >> (64 - shift) : 0;
 			kw3 &= field->kw_mask[i + 2];
-			dummy.kw[i + 1] = kw2;
-			dummy.kw[i + 2] = kw3;
+			kw[i + 1] = kw2;
+			kw[i + 2] = kw3;
 			/* update entry mask */
 			kw2 = shift ? mask_lo >> (64 - shift) : 0;
 			kw2 |= (mask_hi << shift);
 			kw2 &= field->kw_mask[i + 1];
 			kw3 = shift ? mask_hi >> (64 - shift) : 0;
 			kw3 &= field->kw_mask[i + 2];
-			dummy.kw_mask[i + 1] = kw2;
-			dummy.kw_mask[i + 2] = kw3;
+			kw_mask[i + 1] = kw2;
+			kw_mask[i + 2] = kw3;
 			break;
 		}
 	}
 	/* dummy is ready with values and masks for given key
 	 * field now clear and update input entry with those
 	 */
-	for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
+
+	val = mdata->kw;
+	mask = mdata->kw_mask;
+
+	for (i = 0; i < max_kw; i++, val++, mask++) {
 		if (!field->kw_mask[i])
 			continue;
-		entry->kw[i] &= ~field->kw_mask[i];
-		entry->kw_mask[i] &= ~field->kw_mask[i];
 
-		entry->kw[i] |= dummy.kw[i];
-		entry->kw_mask[i] |= dummy.kw_mask[i];
+		*val &= ~field->kw_mask[i];
+		*mask &= ~field->kw_mask[i];
+
+		*val |= kw[i];
+		*mask |= kw_mask[i];
 	}
 }
 
-static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
+static void npc_update_ipv6_flow(struct rvu *rvu,
+				 struct mcam_entry_mdata *mdata,
 				 u64 features, struct flow_msg *pkt,
 				 struct flow_msg *mask,
 				 struct rvu_npc_mcam_rule *output, u8 intf)
@@ -857,7 +1022,7 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
 		val_hi = (u64)src_ip[0] << 32 | src_ip[1];
 		val_lo = (u64)src_ip[2] << 32 | src_ip[3];
 
-		npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi,
+		npc_update_entry(rvu, NPC_SIP_IPV6, mdata, val_lo, val_hi,
 				 mask_lo, mask_hi, intf);
 		memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src));
 		memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src));
@@ -871,14 +1036,15 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
 		val_hi = (u64)dst_ip[0] << 32 | dst_ip[1];
 		val_lo = (u64)dst_ip[2] << 32 | dst_ip[3];
 
-		npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi,
+		npc_update_entry(rvu, NPC_DIP_IPV6, mdata, val_lo, val_hi,
 				 mask_lo, mask_hi, intf);
 		memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst));
 		memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst));
 	}
 }
 
-static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry,
+static void npc_update_vlan_features(struct rvu *rvu,
+				     struct mcam_entry_mdata *mdata,
 				     u64 features, u8 intf)
 {
 	bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG));
@@ -887,24 +1053,24 @@ static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry,
 
 	/* If only VLAN id is given then always match outer VLAN id */
 	if (vid && !ctag && !stag) {
-		npc_update_entry(rvu, NPC_LB, entry,
+		npc_update_entry(rvu, NPC_LB, mdata,
 				 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0,
 				 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf);
 		return;
 	}
 	if (ctag)
-		npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0,
+		npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_CTAG, 0,
 				 ~0ULL, 0, intf);
 	if (stag)
-		npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0,
+		npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_STAG_QINQ, 0,
 				 ~0ULL, 0, intf);
 }
 
-static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
-			    u64 features, struct flow_msg *pkt,
-			    struct flow_msg *mask,
-			    struct rvu_npc_mcam_rule *output, u8 intf,
-			    int blkaddr)
+void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata,
+		     u64 features, struct flow_msg *pkt,
+		     struct flow_msg *mask,
+		     struct rvu_npc_mcam_rule *output, u8 intf,
+		     int blkaddr)
 {
 	u64 dmac_mask = ether_addr_to_u64(mask->dmac);
 	u64 smac_mask = ether_addr_to_u64(mask->smac);
@@ -918,39 +1084,39 @@ static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
 
 	/* For tcp/udp/sctp LTYPE should be present in entry */
 	if (features & BIT_ULL(NPC_IPPROTO_TCP))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_TCP,
 				 0, ~0ULL, 0, intf);
 	if (features & BIT_ULL(NPC_IPPROTO_UDP))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_UDP,
 				 0, ~0ULL, 0, intf);
 	if (features & BIT_ULL(NPC_IPPROTO_SCTP))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_SCTP,
 				 0, ~0ULL, 0, intf);
 	if (features & BIT_ULL(NPC_IPPROTO_ICMP))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP,
 				 0, ~0ULL, 0, intf);
 	if (features & BIT_ULL(NPC_IPPROTO_ICMP6))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP6,
 				 0, ~0ULL, 0, intf);
 
 	/* For AH, LTYPE should be present in entry */
 	if (features & BIT_ULL(NPC_IPPROTO_AH))
-		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH,
+		npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_AH,
 				 0, ~0ULL, 0, intf);
 	/* For ESP, LTYPE should be present in entry */
 	if (features & BIT_ULL(NPC_IPPROTO_ESP))
-		npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP,
+		npc_update_entry(rvu, NPC_LE, mdata, NPC_LT_LE_ESP,
 				 0, ~0ULL, 0, intf);
 
 	if (features & BIT_ULL(NPC_LXMB)) {
 		output->lxmb = is_broadcast_ether_addr(pkt->dmac) ? 2 : 1;
-		npc_update_entry(rvu, NPC_LXMB, entry, output->lxmb, 0,
+		npc_update_entry(rvu, NPC_LXMB, mdata, output->lxmb, 0,
 				 output->lxmb, 0, intf);
 	}
 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi)	      \
 do {									      \
 	if (features & BIT_ULL((field))) {				      \
-		npc_update_entry(rvu, (field), entry, (val_lo), (val_hi),     \
+		npc_update_entry(rvu, (field), mdata, (val_lo), (val_hi),     \
 				 (mask_lo), (mask_hi), intf);		      \
 		memcpy(&opkt->member, &pkt->member, sizeof(pkt->member));     \
 		memcpy(&omask->member, &mask->member, sizeof(mask->member));  \
@@ -1038,10 +1204,10 @@ do {									      \
 
 	NPC_WRITE_FLOW(NPC_IPFRAG_IPV6, next_header, pkt->next_header, 0,
 		       mask->next_header, 0);
-	npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf);
-	npc_update_vlan_features(rvu, entry, features, intf);
+	npc_update_ipv6_flow(rvu, mdata, features, pkt, mask, output, intf);
+	npc_update_vlan_features(rvu, mdata, features, intf);
 
-	npc_update_field_hash(rvu, intf, entry, blkaddr, features,
+	npc_update_field_hash(rvu, intf, mdata, blkaddr, features,
 			      pkt, mask, opkt, omask);
 }
 
@@ -1083,6 +1249,10 @@ static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
 {
 	struct npc_mcam *mcam = &rvu->hw->mcam;
 
+	/* There is no counter allotted for cn20k */
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	mutex_lock(&mcam->lock);
 
 	__rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule);
@@ -1129,8 +1299,29 @@ static int npc_mcast_update_action_index(struct rvu *rvu, struct npc_install_flo
 	return 0;
 }
 
+void
+npc_populate_mcam_mdata(struct rvu *rvu,
+			struct mcam_entry_mdata *mdata,
+			struct cn20k_mcam_entry *cn20k_entry,
+			struct mcam_entry *entry)
+{
+	if (is_cn20k(rvu->pdev)) {
+		mdata->kw = cn20k_entry->kw;
+		mdata->kw_mask = cn20k_entry->kw_mask;
+		mdata->action = &cn20k_entry->action;
+		mdata->vtag_action = &cn20k_entry->vtag_action;
+		mdata->max_kw = NPC_KWS_IN_KEY_SZ_8;
+		return;
+	}
+	mdata->kw = entry->kw;
+	mdata->kw_mask = entry->kw_mask;
+	mdata->action = &entry->action;
+	mdata->vtag_action = &entry->vtag_action;
+	mdata->max_kw = NPC_KWS_IN_KEY_SZ_7;
+}
+
 static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
-			       struct mcam_entry *entry,
+			       struct mcam_entry_mdata *mdata,
 			       struct npc_install_flow_req *req,
 			       u16 target, bool pf_set_vfs_mac)
 {
@@ -1141,7 +1332,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 	if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac)
 		req->chan_mask = 0x0; /* Do not care channel */
 
-	npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask,
+	npc_update_entry(rvu, NPC_CHAN, mdata, req->channel, 0, req->chan_mask,
 			 0, NIX_INTF_RX);
 
 	*(u64 *)&action = 0x00;
@@ -1173,12 +1364,12 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 			action.match_id = req->match_id;
 	}
 
-	entry->action = *(u64 *)&action;
+	*mdata->action = *(u64 *)&action;
 
 	/* VTAG0 starts at 0th byte of LID_B.
 	 * VTAG1 starts at 4th byte of LID_B.
 	 */
-	entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
+	*mdata->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
 			     FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) |
 			     FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) |
 			     FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) |
@@ -1191,7 +1382,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 }
 
 static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
-			       struct mcam_entry *entry,
+			       struct mcam_entry_mdata *mdata,
 			       struct npc_install_flow_req *req, u16 target)
 {
 	struct nix_tx_action action;
@@ -1204,7 +1395,7 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 	if (is_pffunc_af(req->hdr.pcifunc))
 		mask = 0;
 
-	npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target),
+	npc_update_entry(rvu, NPC_PF_FUNC, mdata, (__force u16)htons(target),
 			 0, mask, 0, NIX_INTF_TX);
 
 	*(u64 *)&action = 0x00;
@@ -1217,12 +1408,12 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 
 	action.match_id = req->match_id;
 
-	entry->action = *(u64 *)&action;
+	*mdata->action = *(u64 *)&action;
 
 	/* VTAG0 starts at 0th byte of LID_B.
 	 * VTAG1 starts at 4th byte of LID_B.
 	 */
-	entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
+	*mdata->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
 			     FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) |
 			     FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) |
 			     FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) |
@@ -1241,9 +1432,12 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 			    bool pf_set_vfs_mac)
 {
 	struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule;
+	struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 };
 	u64 features, installed_features, missing_features = 0;
 	struct npc_mcam_write_entry_req write_req = { 0 };
 	struct npc_mcam *mcam = &rvu->hw->mcam;
+	struct cn20k_mcam_entry *cn20k_entry;
+	struct mcam_entry_mdata mdata = { };
 	struct rvu_npc_mcam_rule dummy = { 0 };
 	struct rvu_npc_mcam_rule *rule;
 	u16 owner = req->hdr.pcifunc;
@@ -1255,18 +1449,23 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 
 	installed_features = req->features;
 	features = req->features;
-	entry = &write_req.entry_data;
 	entry_index = req->entry;
 
-	npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy,
+	cn20k_entry = &cn20k_wreq.entry_data;
+	entry = &write_req.entry_data;
+
+	npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry);
+
+	npc_update_flow(rvu, &mdata, features, &req->packet, &req->mask, &dummy,
 			req->intf, blkaddr);
 
 	if (is_npc_intf_rx(req->intf)) {
-		err = npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac);
+		err = npc_update_rx_entry(rvu, pfvf, &mdata, req, target,
+					  pf_set_vfs_mac);
 		if (err)
 			return err;
 	} else {
-		err = npc_update_tx_entry(rvu, pfvf, entry, req, target);
+		err = npc_update_tx_entry(rvu, pfvf, &mdata, req, target);
 		if (err)
 			return err;
 	}
@@ -1286,7 +1485,7 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 		missing_features = (def_ucast_rule->features ^ features) &
 					def_ucast_rule->features;
 		if (missing_features)
-			npc_update_flow(rvu, entry, missing_features,
+			npc_update_flow(rvu, &mdata, missing_features,
 					&def_ucast_rule->packet,
 					&def_ucast_rule->mask,
 					&dummy, req->intf,
@@ -1303,51 +1502,91 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 		new = true;
 	}
 
-	/* allocate new counter if rule has no counter */
-	if (!req->default_rule && req->set_cntr && !rule->has_cntr)
-		rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
-
-	/* if user wants to delete an existing counter for a rule then
-	 * free the counter
-	 */
-	if (!req->set_cntr && rule->has_cntr)
-		rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
+	if (!is_cn20k(rvu->pdev)) {
+		write_req.hdr.pcifunc = owner;
+
+		/* allocate new counter if rule has no counter */
+		if (!req->default_rule && req->set_cntr && !rule->has_cntr)
+			rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
+
+		/* if user wants to delete an existing counter for a rule then
+		 * free the counter
+		 */
+		if (!req->set_cntr && rule->has_cntr)
+			rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
+
+		/* AF owns the default rules so change the owner just to relax
+		 * the checks in rvu_mbox_handler_npc_mcam_write_entry
+		 */
+		if (req->default_rule)
+			write_req.hdr.pcifunc = 0;
+
+		write_req.entry = entry_index;
+		write_req.intf = req->intf;
+		write_req.enable_entry = (u8)enable;
+		/* if counter is available then clear and use it */
+		if (req->set_cntr && rule->has_cntr) {
+			rvu_write64(rvu, blkaddr,
+				    NPC_AF_MATCH_STATX(rule->cntr),
+				    req->cntr_val);
+			write_req.set_cntr = 1;
+			write_req.cntr = rule->cntr;
+		}
+		goto update_rule;
+	}
 
-	write_req.hdr.pcifunc = owner;
+	cn20k_wreq.hdr.pcifunc = owner;
 
-	/* AF owns the default rules so change the owner just to relax
-	 * the checks in rvu_mbox_handler_npc_mcam_write_entry
-	 */
 	if (req->default_rule)
-		write_req.hdr.pcifunc = 0;
+		cn20k_wreq.hdr.pcifunc = 0;
 
-	write_req.entry = entry_index;
-	write_req.intf = req->intf;
-	write_req.enable_entry = (u8)enable;
-	/* if counter is available then clear and use it */
-	if (req->set_cntr && rule->has_cntr) {
-		rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), req->cntr_val);
-		write_req.set_cntr = 1;
-		write_req.cntr = rule->cntr;
-	}
+	cn20k_wreq.entry = entry_index;
+	cn20k_wreq.intf = req->intf;
+	cn20k_wreq.enable_entry = (u8)enable;
+	cn20k_wreq.hw_prio = req->hw_prio;
+	cn20k_wreq.req_kw_type = req->req_kw_type;
+
+update_rule:
 
 	/* update rule */
 	memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
 	memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
 	rule->entry = entry_index;
-	memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action));
-	if (is_npc_intf_tx(req->intf))
-		memcpy(&rule->tx_action, &entry->action,
-		       sizeof(struct nix_tx_action));
-	rule->vtag_action = entry->vtag_action;
+	if (is_cn20k(rvu->pdev)) {
+		memcpy(&rule->rx_action, &cn20k_entry->action,
+		       sizeof(struct nix_rx_action));
+		if (is_npc_intf_tx(req->intf))
+			memcpy(&rule->tx_action, &cn20k_entry->action,
+			       sizeof(struct nix_tx_action));
+		rule->vtag_action = cn20k_entry->vtag_action;
+	} else {
+		memcpy(&rule->rx_action, &entry->action,
+		       sizeof(struct nix_rx_action));
+		if (is_npc_intf_tx(req->intf))
+			memcpy(&rule->tx_action, &entry->action,
+			       sizeof(struct nix_tx_action));
+		rule->vtag_action = entry->vtag_action;
+	}
+
 	rule->features = installed_features;
 	rule->default_rule = req->default_rule;
 	rule->owner = owner;
 	rule->enable = enable;
-	rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK;
-	rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK;
+
+	if (is_cn20k(rvu->pdev)) {
+		rule->chan_mask = cn20k_wreq.entry_data.kw_mask[0] &
+						NPC_KEX_CHAN_MASK;
+		rule->chan = cn20k_wreq.entry_data.kw[0] &
+					NPC_KEX_CHAN_MASK;
+	} else {
+		rule->chan_mask = write_req.entry_data.kw_mask[0] &
+						NPC_KEX_CHAN_MASK;
+		rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK;
+	}
+
 	rule->chan &= rule->chan_mask;
 	rule->lxmb = dummy.lxmb;
+	rule->hw_prio = req->hw_prio;
 	if (is_npc_intf_tx(req->intf))
 		rule->intf = pfvf->nix_tx_intf;
 	else
@@ -1359,8 +1598,14 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 		pfvf->def_ucast_rule = rule;
 
 	/* write to mcam entry registers */
-	err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
-						    &write_rsp);
+	if (is_cn20k(rvu->pdev))
+		err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu,
+								  &cn20k_wreq,
+								  &write_rsp);
+	else
+		err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
+							    &write_rsp);
+
 	if (err) {
 		rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
 		if (new) {
@@ -1393,6 +1638,79 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 	return 0;
 }
 
+static int
+rvu_npc_free_entry_for_flow_install(struct rvu *rvu, u16 pcifunc,
+				    bool free_entry, int mcam_idx)
+{
+	struct npc_mcam_free_entry_req free_req = { 0 };
+	struct msg_rsp rsp;
+	int rc;
+
+	if (!free_entry)
+		return 0;
+
+	free_req.hdr.pcifunc = pcifunc;
+	free_req.entry = mcam_idx;
+	rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp);
+	return rc;
+}
+
+static int
+rvu_npc_alloc_entry_for_flow_install(struct rvu *rvu,
+				     struct npc_install_flow_req *fl_req,
+				     u16 *mcam_idx, u8 *kw_type,
+				     bool *allocated)
+{
+	struct npc_mcam_alloc_entry_req entry_req;
+	struct npc_mcam_alloc_entry_rsp entry_rsp;
+	struct npc_get_num_kws_req kws_req;
+	struct npc_get_num_kws_rsp kws_rsp;
+	int off, kw_bits, rc;
+	u8 *src, *dst;
+
+	if (!is_cn20k(rvu->pdev)) {
+		*kw_type = -1;
+		return 0;
+	}
+
+	if (!fl_req->alloc_entry) {
+		*kw_type = -1;
+		return 0;
+	}
+
+	off = offsetof(struct npc_install_flow_req, packet);
+	dst = (u8 *)&kws_req.fl + off;
+	src = (u8 *)fl_req + off;
+	memcpy(dst, src, sizeof(struct npc_install_flow_req) - off);
+	rc = rvu_mbox_handler_npc_get_num_kws(rvu, &kws_req, &kws_rsp);
+	if (rc)
+		return rc;
+
+	kw_bits = kws_rsp.kws * 64;
+
+	*kw_type = NPC_MCAM_KEY_X2;
+	if (kw_bits > 256)
+		*kw_type = NPC_MCAM_KEY_X4;
+
+	memset(&entry_req, 0, sizeof(entry_req));
+	memset(&entry_rsp, 0, sizeof(entry_rsp));
+
+	entry_req.hdr.pcifunc = fl_req->hdr.pcifunc;
+	entry_req.ref_prio = fl_req->ref_prio;
+	entry_req.ref_entry = fl_req->ref_entry;
+	entry_req.kw_type = *kw_type;
+	entry_req.count = 1;
+	rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,
+						   &entry_req,
+						   &entry_rsp);
+	if (rc)
+		return rc;
+
+	*mcam_idx = entry_rsp.entry_list[0];
+	*allocated = true;
+	return 0;
+}
+
 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 				      struct npc_install_flow_req *req,
 				      struct npc_install_flow_rsp *rsp)
@@ -1403,7 +1721,9 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 	int blkaddr, nixlf, err;
 	struct rvu_pfvf *pfvf;
 	bool pf_set_vfs_mac = false;
+	bool allocated = false;
 	bool enable = true;
+	u8 kw_type;
 	u16 target;
 
 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
@@ -1415,6 +1735,17 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 	if (!is_npc_interface_valid(rvu, req->intf))
 		return NPC_FLOW_INTF_INVALID;
 
+	err = rvu_npc_alloc_entry_for_flow_install(rvu, req, &req->entry,
+						   &kw_type, &allocated);
+	if (err) {
+		dev_err(rvu->dev,
+			"%s: Error to alloc mcam entry for pcifunc=%#x\n",
+			__func__, req->hdr.pcifunc);
+		return err;
+	}
+
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+
 	/* If DMAC is not extracted in MKEX, rules installed by AF
 	 * can rely on L2MB bit set by hardware protocol checker for
 	 * broadcast and multicast addresses.
@@ -1428,6 +1759,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 			dev_warn(rvu->dev,
 				 "%s: mkex profile does not support ucast flow\n",
 				 __func__);
+			rvu_npc_free_entry_for_flow_install(rvu,
+							    req->hdr.pcifunc,
+							    allocated,
+							    req->entry);
 			return NPC_FLOW_NOT_SUPPORTED;
 		}
 
@@ -1435,6 +1770,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 			dev_warn(rvu->dev,
 				 "%s: mkex profile does not support bcast/mcast flow",
 				 __func__);
+			rvu_npc_free_entry_for_flow_install(rvu,
+							    req->hdr.pcifunc,
+							    allocated,
+							    req->entry);
 			return NPC_FLOW_NOT_SUPPORTED;
 		}
 
@@ -1444,8 +1783,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 	}
 
 process_flow:
-	if (from_vf && req->default_rule)
+	if (from_vf && req->default_rule) {
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
 		return NPC_FLOW_VF_PERM_DENIED;
+	}
 
 	/* Each PF/VF info is maintained in struct rvu_pfvf.
 	 * rvu_pfvf for the target PF/VF needs to be retrieved
@@ -1473,8 +1815,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 		req->chan_mask = 0xFFF;
 
 	err = npc_check_unsupported_flows(rvu, req->features, req->intf);
-	if (err)
+	if (err) {
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
 		return NPC_FLOW_NOT_SUPPORTED;
+	}
 
 	pfvf = rvu_get_pfvf(rvu, target);
 
@@ -1493,8 +1838,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 
 	/* Proceed if NIXLF is attached or not for TX rules */
 	err = nix_get_nixlf(rvu, target, &nixlf, NULL);
-	if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac)
+	if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) {
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
 		return NPC_FLOW_NO_NIXLF;
+	}
 
 	/* don't enable rule when nixlf not attached or initialized */
 	if (!(is_nixlf_attached(rvu, target) &&
@@ -1509,20 +1857,31 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
 		enable = true;
 
 	/* Do not allow requests from uninitialized VFs */
-	if (from_vf && !enable)
+	if (from_vf && !enable) {
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
 		return NPC_FLOW_VF_NOT_INIT;
+	}
 
 	/* PF sets VF mac & VF NIXLF is not attached, update the mac addr */
 	if (pf_set_vfs_mac && !enable) {
 		ether_addr_copy(pfvf->default_mac, req->packet.dmac);
 		ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
 		set_bit(PF_SET_VF_MAC, &pfvf->flags);
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
 		return 0;
 	}
 
 	mutex_lock(&rswitch->switch_lock);
 	err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf,
 			       req, rsp, enable, pf_set_vfs_mac);
+	if (err)
+		rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc,
+						    allocated, req->entry);
+
+	rsp->kw_type = kw_type;
+	rsp->entry = req->entry;
 	mutex_unlock(&rswitch->switch_lock);
 
 	return err;
@@ -1559,6 +1918,10 @@ int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu,
 	struct list_head del_list;
 	int blkaddr;
 
+	req->entry = npc_cn20k_vidx2idx(req->entry);
+	req->start = npc_cn20k_vidx2idx(req->start);
+	req->end = npc_cn20k_vidx2idx(req->end);
+
 	INIT_LIST_HEAD(&del_list);
 
 	mutex_lock(&mcam->lock);
@@ -1601,28 +1964,49 @@ static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
 				 struct rvu_npc_mcam_rule *rule,
 				 struct rvu_pfvf *pfvf)
 {
+	struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 };
 	struct npc_mcam_write_entry_req write_req = { 0 };
-	struct mcam_entry *entry = &write_req.entry_data;
+	struct mcam_entry_mdata mdata = { };
 	struct npc_mcam *mcam = &rvu->hw->mcam;
+	struct cn20k_mcam_entry *cn20k_entry;
+	struct mcam_entry *entry;
+	u8 intf, enable, hw_prio;
 	struct msg_rsp rsp;
-	u8 intf, enable;
 	int err;
 
+	cn20k_entry = &cn20k_wreq.entry_data;
+	entry = &write_req.entry_data;
+	npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry);
+
 	ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
 
-	npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
-			    entry, &intf,  &enable);
+	if (is_cn20k(rvu->pdev))
+		npc_cn20k_read_mcam_entry(rvu, npcblkaddr, rule->entry,
+					  cn20k_entry, &intf,
+					  &enable, &hw_prio);
+	else
+		npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
+				    entry, &intf, &enable);
 
-	npc_update_entry(rvu, NPC_DMAC, entry,
+	npc_update_entry(rvu, NPC_DMAC, &mdata,
 			 ether_addr_to_u64(pfvf->mac_addr), 0,
 			 0xffffffffffffull, 0, intf);
 
-	write_req.hdr.pcifunc = rule->owner;
-	write_req.entry = rule->entry;
-	write_req.intf = pfvf->nix_rx_intf;
-
 	mutex_unlock(&mcam->lock);
-	err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp);
+	if (is_cn20k(rvu->pdev)) {
+		cn20k_wreq.hdr.pcifunc = rule->owner;
+		cn20k_wreq.entry = rule->entry;
+		cn20k_wreq.intf = pfvf->nix_rx_intf;
+		err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu,
+								  &cn20k_wreq,
+								  &rsp);
+	} else {
+		write_req.hdr.pcifunc = rule->owner;
+		write_req.entry = rule->entry;
+		write_req.intf = pfvf->nix_rx_intf;
+		err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
+							    &rsp);
+	}
 	mutex_lock(&mcam->lock);
 
 	return err;
@@ -1710,10 +2094,12 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx,
 			       u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask,
 			       u64 bcast_mcast_val, u64 bcast_mcast_mask)
 {
+	struct npc_cn20k_mcam_write_entry_req cn20k_req = { 0 };
 	struct npc_mcam_alloc_counter_req cntr_req = { 0 };
 	struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
 	struct npc_mcam_write_entry_req req = { 0 };
 	struct npc_mcam *mcam = &rvu->hw->mcam;
+	struct mcam_entry_mdata mdata = { };
 	struct rvu_npc_mcam_rule *rule;
 	struct msg_rsp rsp;
 	bool enabled;
@@ -1757,26 +2143,51 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx,
 	/* Reserve slot 0 */
 	npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx);
 
-	/* Allocate counter for this single drop on non hit rule */
-	cntr_req.hdr.pcifunc = 0; /* AF request */
-	cntr_req.contig = true;
-	cntr_req.count = 1;
-	err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp);
-	if (err) {
-		dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n",
-			__func__, err);
-		return	-EFAULT;
+	if (!is_cn20k(rvu->pdev)) {
+		/* Allocate counter for this single drop on non hit rule */
+		cntr_req.hdr.pcifunc = 0; /* AF request */
+		cntr_req.contig = true;
+		cntr_req.count = 1;
+		err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req,
+							      &cntr_rsp);
+		if (err) {
+			dev_err(rvu->dev,
+				"%s: Err to allocate cntr for drop rule (err=%d)\n",
+				__func__, err);
+			return	-EFAULT;
+		}
+		*counter_idx = cntr_rsp.cntr;
 	}
-	*counter_idx = cntr_rsp.cntr;
+
+	npc_populate_mcam_mdata(rvu, &mdata,
+				&cn20k_req.entry_data,
+				&req.entry_data);
 
 	/* Fill in fields for this mcam entry */
-	npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0,
+	npc_update_entry(rvu, NPC_EXACT_RESULT, &mdata, exact_val, 0,
 			 exact_mask, 0, NIX_INTF_RX);
-	npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0,
+	npc_update_entry(rvu, NPC_CHAN, &mdata, chan_val, 0,
 			 chan_mask, 0, NIX_INTF_RX);
-	npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0,
+	npc_update_entry(rvu, NPC_LXMB, &mdata, bcast_mcast_val, 0,
 			 bcast_mcast_mask, 0, NIX_INTF_RX);
 
+	if (is_cn20k(rvu->pdev)) {
+		cn20k_req.intf = NIX_INTF_RX;
+		cn20k_req.entry = mcam_idx;
+
+		err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu,
+								  &cn20k_req,
+								  &rsp);
+		if (err) {
+			dev_err(rvu->dev,
+				"%s: Installation of single drop on non hit rule at %d failed\n",
+				__func__, mcam_idx);
+			return err;
+		}
+
+		goto enable_entry;
+	}
+
 	req.intf = NIX_INTF_RX;
 	req.set_cntr = true;
 	req.cntr = cntr_rsp.cntr;
@@ -1784,14 +2195,17 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx,
 
 	err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp);
 	if (err) {
-		dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n",
+		dev_err(rvu->dev,
+			"%s: Installation of single drop on non hit rule at %d failed\n",
 			__func__, mcam_idx);
 		return err;
 	}
 
-	dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n",
+	dev_err(rvu->dev,
+		"%s: Installed single drop on non hit rule at %d, cntr=%d\n",
 		__func__, mcam_idx, req.cntr);
 
+enable_entry:
 	/* disable entry at Bank 0, index 0 */
 	npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false);
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h
index 3f5c9042d10e7..d3ba86c23959f 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h
@@ -15,7 +15,19 @@
 #define NPC_LDATA_EN	BIT_ULL(7)
 
 void npc_update_entry(struct rvu *rvu, enum key_fields type,
-		      struct mcam_entry *entry, u64 val_lo,
+		      struct mcam_entry_mdata *mdata, u64 val_lo,
 		      u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf);
 
+void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata,
+		     u64 features, struct flow_msg *pkt,
+		     struct flow_msg *mask,
+		     struct rvu_npc_mcam_rule *output, u8 intf,
+		     int blkaddr);
+
+void
+npc_populate_mcam_mdata(struct rvu *rvu,
+			struct mcam_entry_mdata *mdata,
+			struct cn20k_mcam_entry *cn20k_entry,
+			struct mcam_entry *entry);
+
 #endif /* RVU_NPC_FS_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c
index 906d712cef198..03bb485a1aca1 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c
@@ -125,6 +125,9 @@ static void npc_program_mkex_hash_rx(struct rvu *rvu, int blkaddr,
 	struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash;
 	int lid, lt, ld, hash_cnt = 0;
 
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	if (is_npc_intf_tx(intf))
 		return;
 
@@ -165,6 +168,9 @@ static void npc_program_mkex_hash_tx(struct rvu *rvu, int blkaddr,
 	struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash;
 	int lid, lt, ld, hash_cnt = 0;
 
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	if (is_npc_intf_rx(intf))
 		return;
 
@@ -224,6 +230,9 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr)
 	struct rvu_hwinfo *hw = rvu->hw;
 	u64 cfg;
 
+	if (is_cn20k(rvu->pdev))
+		return;
+
 	/* Check if hardware supports hash extraction */
 	if (!hwcap->npc_hash_extract)
 		return;
@@ -273,7 +282,7 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr)
 }
 
 void npc_update_field_hash(struct rvu *rvu, u8 intf,
-			   struct mcam_entry *entry,
+			   struct mcam_entry_mdata *mdata,
 			   int blkaddr,
 			   u64 features,
 			   struct flow_msg *pkt,
@@ -284,9 +293,13 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf,
 	struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash;
 	struct npc_get_field_hash_info_req req;
 	struct npc_get_field_hash_info_rsp rsp;
+	u8 hash_idx, lid, ltype, ltype_mask;
 	u64 ldata[2], cfg;
 	u32 field_hash;
-	u8 hash_idx;
+	bool en;
+
+	if (is_cn20k(rvu->pdev))
+		return;
 
 	if (!rvu->hw->cap.npc_hash_extract) {
 		dev_dbg(rvu->dev, "%s: Field hash extract feature is not supported\n", __func__);
@@ -298,60 +311,60 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf,
 
 	for (hash_idx = 0; hash_idx < NPC_MAX_HASH; hash_idx++) {
 		cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_HASHX_CFG(intf, hash_idx));
-		if ((cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12))) {
-			u8 lid = (cfg & GENMASK_ULL(10, 8)) >> 8;
-			u8 ltype = (cfg & GENMASK_ULL(7, 4)) >> 4;
-			u8 ltype_mask = cfg & GENMASK_ULL(3, 0);
-
-			if (mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) {
-				switch (ltype & ltype_mask) {
-				/* If hash extract enabled is supported for IPv6 then
-				 * 128 bit IPv6 source and destination addressed
-				 * is hashed to 32 bit value.
-				 */
-				case NPC_LT_LC_IP6:
-					/* ld[0] == hash_idx[0] == Source IPv6
-					 * ld[1] == hash_idx[1] == Destination IPv6
-					 */
-					if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) {
-						u32 src_ip[IPV6_WORDS];
-
-						be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
-						ldata[1] = (u64)src_ip[0] << 32 | src_ip[1];
-						ldata[0] = (u64)src_ip[2] << 32 | src_ip[3];
-						field_hash = npc_field_hash_calc(ldata,
-										 rsp,
-										 intf,
-										 hash_idx);
-						npc_update_entry(rvu, NPC_SIP_IPV6, entry,
-								 field_hash, 0,
-								 GENMASK(31, 0), 0, intf);
-						memcpy(&opkt->ip6src, &pkt->ip6src,
-						       sizeof(pkt->ip6src));
-						memcpy(&omask->ip6src, &mask->ip6src,
-						       sizeof(mask->ip6src));
-					} else if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) {
-						u32 dst_ip[IPV6_WORDS];
-
-						be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
-						ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1];
-						ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3];
-						field_hash = npc_field_hash_calc(ldata,
-										 rsp,
-										 intf,
-										 hash_idx);
-						npc_update_entry(rvu, NPC_DIP_IPV6, entry,
-								 field_hash, 0,
-								 GENMASK(31, 0), 0, intf);
-						memcpy(&opkt->ip6dst, &pkt->ip6dst,
-						       sizeof(pkt->ip6dst));
-						memcpy(&omask->ip6dst, &mask->ip6dst,
-						       sizeof(mask->ip6dst));
-					}
-
-					break;
-				}
-			}
+		en = !!(cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12));
+		if (!en)
+			continue;
+
+		lid = (cfg & GENMASK_ULL(10, 8)) >> 8;
+		ltype = (cfg & GENMASK_ULL(7, 4)) >> 4;
+		ltype_mask = cfg & GENMASK_ULL(3, 0);
+
+		if (!mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx])
+			continue;
+
+		/* If hash extract enabled is supported for IPv6 then
+		 * 128 bit IPv6 source and destination addressed
+		 * is hashed to 32 bit value.
+		 */
+		if ((ltype & ltype_mask) != NPC_LT_LC_IP6)
+			continue;
+
+		/* ld[0] == hash_idx[0] == Source IPv6
+		 * ld[1] == hash_idx[1] == Destination IPv6
+		 */
+		if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) {
+			u32 src_ip[IPV6_WORDS];
+
+			be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
+			ldata[1] = (u64)src_ip[0] << 32 | src_ip[1];
+			ldata[0] = (u64)src_ip[2] << 32 | src_ip[3];
+			field_hash = npc_field_hash_calc(ldata, rsp, intf,
+							 hash_idx);
+			npc_update_entry(rvu, NPC_SIP_IPV6, mdata, field_hash,
+					 0, GENMASK(31, 0), 0, intf);
+			memcpy(&opkt->ip6src, &pkt->ip6src,
+			       sizeof(pkt->ip6src));
+			memcpy(&omask->ip6src, &mask->ip6src,
+			       sizeof(mask->ip6src));
+			continue;
+		}
+
+		if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) {
+			u32 dst_ip[IPV6_WORDS];
+
+			be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
+			ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1];
+			ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3];
+			field_hash = npc_field_hash_calc(ldata, rsp, intf,
+							 hash_idx);
+			npc_update_entry(rvu, NPC_DIP_IPV6, mdata,
+					 field_hash, 0, GENMASK(31, 0),
+					 0, intf);
+			memcpy(&opkt->ip6dst, &pkt->ip6dst,
+			       sizeof(pkt->ip6dst));
+			memcpy(&omask->ip6dst, &mask->ip6dst,
+			       sizeof(mask->ip6dst));
+			continue;
 		}
 	}
 }
@@ -1874,6 +1887,9 @@ int rvu_npc_exact_init(struct rvu *rvu)
 	u64 cfg;
 	bool rc;
 
+	if (is_cn20k(rvu->pdev))
+		return 0;
+
 	/* Read NPC_AF_CONST3 and check for have exact
 	 * match functionality is present
 	 */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h
index cb25cf478f1fd..4cbcae69b6d32 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h
@@ -53,7 +53,7 @@ struct npc_mcam_kex_hash {
 } __packed;
 
 void npc_update_field_hash(struct rvu *rvu, u8 intf,
-			   struct mcam_entry *entry,
+			   struct mcam_entry_mdata *mdata,
 			   int blkaddr,
 			   u64 features,
 			   struct flow_msg *pkt,
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
index a60f8cf53febb..a5a8f45587174 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
@@ -251,6 +251,272 @@ static u8 cn20k_aura_bpid_idx(struct otx2_nic *pfvf, int aura_id)
 #endif
 }
 
+static int cn20k_tc_get_entry_index(struct otx2_flow_config *flow_cfg,
+				    struct otx2_tc_flow *node)
+{
+	struct otx2_tc_flow *tmp;
+	int index = 0;
+
+	list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) {
+		if (tmp == node)
+			return index;
+
+		index++;
+	}
+
+	return -1;
+}
+
+int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry)
+{
+	struct npc_mcam_free_entry_req *req;
+	int err;
+
+	mutex_lock(&nic->mbox.lock);
+	req = otx2_mbox_alloc_msg_npc_mcam_free_entry(&nic->mbox);
+	if (!req) {
+		mutex_unlock(&nic->mbox.lock);
+		return -ENOMEM;
+	}
+
+	req->entry = entry;
+	/* Send message to AF to free MCAM entries */
+	err = otx2_sync_mbox_msg(&nic->mbox);
+	if (err) {
+		mutex_unlock(&nic->mbox.lock);
+		return err;
+	}
+
+	mutex_unlock(&nic->mbox.lock);
+
+	return 0;
+}
+
+static bool cn20k_tc_check_entry_shiftable(struct otx2_nic *nic,
+					   struct otx2_flow_config *flow_cfg,
+					   struct otx2_tc_flow *node, int index,
+					   bool error)
+{
+	struct otx2_tc_flow *first, *tmp, *n;
+	u32 prio = 0;
+	int i = 0;
+	u8 type;
+
+	first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow,
+				 list);
+	type = first->kw_type;
+
+	/* Check all the nodes from start to given index (including index) has
+	 * same type i.e, either X2 or X4
+	 */
+	list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) {
+		if (i > index)
+			break;
+
+		if (type != tmp->kw_type) {
+			/* List has both X2 and X4 entries so entries cannot be
+			 * shifted to save MCAM space.
+			 */
+			if (error)
+				dev_err(nic->dev, "Rule %d cannot be shifted to %d\n",
+					tmp->prio, prio);
+			return false;
+		}
+
+		type = tmp->kw_type;
+		prio = tmp->prio;
+		i++;
+	}
+
+	return true;
+}
+
+void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic,
+					struct otx2_flow_config *flow_cfg,
+					struct otx2_tc_flow *node)
+{
+	struct otx2_tc_flow *first, *tmp, *n;
+	int i = 0, index;
+	u16 cntr_val = 0;
+	u16 entry;
+
+	index = cn20k_tc_get_entry_index(flow_cfg, node);
+	if (index < 0) {
+		netdev_dbg(nic->netdev, "Could not find node\n");
+		return;
+	}
+
+	first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow,
+				 list);
+	entry = first->entry;
+
+	/* If entries cannot be shifted then delete given entry
+	 * and free it to AF too.
+	 */
+	if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node,
+					    index, false)) {
+		list_del(&node->list);
+		entry = node->entry;
+		goto free_mcam_entry;
+	}
+
+	/* Find and delete the entry from the list and re-install
+	 * all the entries from beginning to the index of the
+	 * deleted entry to higher mcam indexes.
+	 */
+	list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) {
+		if (node == tmp) {
+			list_del(&tmp->list);
+			break;
+		}
+
+		otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val);
+		tmp->entry = (list_next_entry(tmp, list))->entry;
+		tmp->req.entry = tmp->entry;
+		tmp->req.cntr_val = cntr_val;
+	}
+
+	list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) {
+		if (i == index)
+			break;
+
+		otx2_add_mcam_flow_entry(nic, &tmp->req);
+		i++;
+	}
+
+free_mcam_entry:
+	if (cn20k_tc_free_mcam_entry(nic, entry))
+		netdev_err(nic->netdev, "Freeing entry %d to AF failed\n",
+			   entry);
+}
+
+int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic,
+				       struct otx2_flow_config *flow_cfg,
+				       struct otx2_tc_flow *node)
+{
+	struct otx2_tc_flow *tmp;
+	u16 cntr_val = 0;
+	int list_idx, i;
+	int entry, prev;
+
+	/* Find the index of the entry(list_idx) whose priority
+	 * is greater than the new entry and re-install all
+	 * the entries from beginning to list_idx to higher
+	 * mcam indexes.
+	 */
+	list_idx = otx2_tc_add_to_flow_list(flow_cfg, node);
+	entry = node->entry;
+	if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node,
+					    list_idx, true))
+		/* Due to mix of X2 and X4, entries cannot be shifted.
+		 * In this case free the entry allocated for this rule.
+		 */
+		return -EINVAL;
+
+	for (i = 0; i < list_idx; i++) {
+		tmp = otx2_tc_get_entry_by_index(flow_cfg, i);
+		if (!tmp)
+			return -ENOMEM;
+
+		otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val);
+		prev = tmp->entry;
+		tmp->entry = entry;
+		tmp->req.entry = tmp->entry;
+		tmp->req.cntr_val = cntr_val;
+		otx2_add_mcam_flow_entry(nic, &tmp->req);
+		entry = prev;
+	}
+
+	return entry;
+}
+
+#define MAX_TC_HW_PRIORITY		125
+#define MAX_TC_VF_PRIORITY		126
+#define MAX_TC_PF_PRIORITY		127
+
+static int __cn20k_tc_alloc_entry(struct otx2_nic *nic,
+				  struct npc_install_flow_req *flow_req,
+				  u16 *entry, u8 *type,
+				  u32 tc_priority, bool hw_priority)
+{
+	struct otx2_flow_config *flow_cfg = nic->flow_cfg;
+	struct npc_install_flow_req *req;
+	struct npc_install_flow_rsp *rsp;
+	struct otx2_tc_flow *tmp;
+	int ret = 0;
+
+	req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
+	if (!req)
+		return -ENOMEM;
+
+	memcpy(&flow_req->hdr, &req->hdr, sizeof(struct mbox_msghdr));
+	memcpy(req, flow_req, sizeof(struct npc_install_flow_req));
+	req->alloc_entry = 1;
+
+	/* Allocate very least priority for first rule */
+	if (hw_priority || list_empty(&flow_cfg->flow_list_tc)) {
+		req->ref_prio = NPC_MCAM_LEAST_PRIO;
+	} else {
+		req->ref_prio = NPC_MCAM_HIGHER_PRIO;
+		tmp = list_first_entry(&flow_cfg->flow_list_tc,
+				       struct otx2_tc_flow, list);
+		req->ref_entry = tmp->entry;
+	}
+
+	ret = otx2_sync_mbox_msg(&nic->mbox);
+	if (ret)
+		return ret;
+
+	rsp = (struct npc_install_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox,
+							       0, &req->hdr);
+	if (IS_ERR(rsp))
+		return -EFAULT;
+
+	if (entry)
+		*entry = rsp->entry;
+	if (type)
+		*type = rsp->kw_type;
+
+	return ret;
+}
+
+int cn20k_tc_alloc_entry(struct otx2_nic *nic,
+			 struct flow_cls_offload *tc_flow_cmd,
+			 struct otx2_tc_flow *new_node,
+			 struct npc_install_flow_req *flow_req)
+{
+	bool hw_priority = false;
+	u16 entry_from_af;
+	u8 entry_type;
+	int ret;
+
+	if (is_otx2_vf(nic->pcifunc))
+		flow_req->hw_prio = MAX_TC_VF_PRIORITY;
+	else
+		flow_req->hw_prio = MAX_TC_PF_PRIORITY;
+
+	if (new_node->prio <= MAX_TC_HW_PRIORITY) {
+		flow_req->hw_prio = new_node->prio;
+		hw_priority = true;
+	}
+
+	mutex_lock(&nic->mbox.lock);
+
+	ret = __cn20k_tc_alloc_entry(nic, flow_req, &entry_from_af, &entry_type,
+				     new_node->prio, hw_priority);
+	if (ret) {
+		mutex_unlock(&nic->mbox.lock);
+		return ret;
+	}
+
+	new_node->kw_type = entry_type;
+	new_node->entry = entry_from_af;
+
+	mutex_unlock(&nic->mbox.lock);
+
+	return 0;
+}
+
 static int cn20k_aura_aq_init(struct otx2_nic *pfvf, int aura_id,
 			      int pool_id, int numptrs)
 {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h
index 832adaf8c57f6..b5e527f6d7eba 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h
@@ -10,8 +10,22 @@
 
 #include "otx2_common.h"
 
+struct otx2_flow_config;
+struct otx2_tc_flow;
+
 void cn20k_init(struct otx2_nic *pfvf);
 int cn20k_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs);
 void cn20k_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs);
 void cn20k_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs);
+void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic,
+					struct otx2_flow_config *flow_cfg,
+					struct otx2_tc_flow *node);
+int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic,
+				       struct otx2_flow_config *flow_cfg,
+				       struct otx2_tc_flow *node);
+int cn20k_tc_alloc_entry(struct otx2_nic *nic,
+			 struct flow_cls_offload *tc_flow_cmd,
+			 struct otx2_tc_flow *new_node,
+			 struct npc_install_flow_req *dummy);
+int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry);
 #endif /* CN20K_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 255c7e2633bbb..eecee612b7b2c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -366,6 +366,31 @@ struct otx2_flow_config {
 	u16			ntuple_cnt;
 };
 
+struct otx2_tc_flow_stats {
+	u64 bytes;
+	u64 pkts;
+	u64 used;
+};
+
+struct otx2_tc_flow {
+	struct list_head		list;
+	unsigned long			cookie;
+	struct rcu_head			rcu;
+	struct otx2_tc_flow_stats	stats;
+	spinlock_t			lock; /* lock for stats */
+	u16				rq;
+	u16				entry;
+	u16				leaf_profile;
+	bool				is_act_police;
+	u32				prio;
+	struct npc_install_flow_req	req;
+	u64				rate;
+	u32				burst;
+	u32				mcast_grp_idx;
+	bool				is_pps;
+	u8				kw_type; /* X2/X4 */
+};
+
 struct dev_hw_ops {
 	int	(*sq_aq_init)(void *dev, u16 qidx, u8 chan_offset,
 			      u16 sqb_aura);
@@ -1223,4 +1248,14 @@ void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);
 int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx);
 void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
 			int first, int mdevs, u64 intr);
+int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry,
+			     u16 *cntr_val);
+int otx2_add_mcam_flow_entry(struct otx2_nic *nic,
+			     struct npc_install_flow_req *req);
+int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg,
+			     struct otx2_tc_flow *node);
+
+struct otx2_tc_flow *
+otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg,
+			   int index);
 #endif /* OTX2_COMMON_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
index 12c001ee34e20..38cc539d724d8 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
@@ -37,6 +37,98 @@ static void otx2_clear_ntuple_flow_info(struct otx2_nic *pfvf, struct otx2_flow_
 	flow_cfg->max_flows = 0;
 }
 
+static int otx2_mcam_pfl_info_get(struct otx2_nic *pfvf, bool *is_x2,
+				  u16 *x4_slots)
+{
+	struct npc_get_pfl_info_rsp *rsp;
+	struct msg_req *req;
+	static struct {
+		bool is_set;
+		bool is_x2;
+		u16 x4_slots;
+	} pfl_info;
+
+	/* Avoid sending mboxes for constant information
+	 * like x4_slots
+	 */
+	mutex_lock(&pfvf->mbox.lock);
+	if (pfl_info.is_set) {
+		*is_x2 = pfl_info.is_x2;
+		*x4_slots = pfl_info.x4_slots;
+		mutex_unlock(&pfvf->mbox.lock);
+		return 0;
+	}
+
+	req = otx2_mbox_alloc_msg_npc_get_pfl_info(&pfvf->mbox);
+	if (!req) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -ENOMEM;
+	}
+
+	/* Send message to AF */
+	if (otx2_sync_mbox_msg(&pfvf->mbox)) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -EFAULT;
+	}
+
+	rsp = (struct npc_get_pfl_info_rsp *)otx2_mbox_get_rsp
+		(&pfvf->mbox.mbox, 0, &req->hdr);
+
+	if (IS_ERR(rsp)) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -EFAULT;
+	}
+
+	*is_x2 = (rsp->kw_type == NPC_MCAM_KEY_X2);
+	if (*is_x2)
+		*x4_slots = 0;
+	else
+		*x4_slots = rsp->x4_slots;
+
+	pfl_info.is_x2 = *is_x2;
+	pfl_info.x4_slots = *x4_slots;
+	pfl_info.is_set = true;
+
+	mutex_unlock(&pfvf->mbox.lock);
+	return 0;
+}
+
+static int otx2_get_dft_rl_idx(struct otx2_nic *pfvf, u16 *mcam_idx)
+{
+	struct npc_get_dft_rl_idxs_rsp *rsp;
+	struct msg_req *req;
+
+	mutex_lock(&pfvf->mbox.lock);
+
+	req = otx2_mbox_alloc_msg_npc_get_dft_rl_idxs(&pfvf->mbox);
+	if (!req) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -ENOMEM;
+	}
+
+	/* Send message to AF */
+	if (otx2_sync_mbox_msg(&pfvf->mbox)) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -EINVAL;
+	}
+
+	rsp = (struct npc_get_dft_rl_idxs_rsp *)otx2_mbox_get_rsp
+		(&pfvf->mbox.mbox, 0, &req->hdr);
+
+	if (IS_ERR(rsp)) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -EFAULT;
+	}
+
+	if (is_otx2_lbkvf(pfvf->pdev))
+		*mcam_idx = rsp->promisc;
+	else
+		*mcam_idx = rsp->ucast;
+
+	mutex_unlock(&pfvf->mbox.lock);
+	return 0;
+}
+
 static int otx2_free_ntuple_mcam_entries(struct otx2_nic *pfvf)
 {
 	struct otx2_flow_config *flow_cfg = pfvf->flow_cfg;
@@ -69,7 +161,10 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count)
 	struct otx2_flow_config *flow_cfg = pfvf->flow_cfg;
 	struct npc_mcam_alloc_entry_req *req;
 	struct npc_mcam_alloc_entry_rsp *rsp;
-	int ent, allocated = 0;
+	u16 dft_idx = 0, x4_slots = 0;
+	int ent, allocated = 0, ref;
+	bool is_x2 = false;
+	int rc;
 
 	/* Free current ones and allocate new ones with requested count */
 	otx2_free_ntuple_mcam_entries(pfvf);
@@ -86,6 +181,22 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count)
 		return -ENOMEM;
 	}
 
+	if (is_cn20k(pfvf->pdev)) {
+		rc = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots);
+		if (rc) {
+			netdev_err(pfvf->netdev, "Error to retrieve profile info\n");
+			return rc;
+		}
+
+		rc = otx2_get_dft_rl_idx(pfvf, &dft_idx);
+		if (rc) {
+			netdev_err(pfvf->netdev,
+				   "Error to retrieve ucast mcam idx for pcifunc %#x\n",
+				   pfvf->pcifunc);
+			return rc;
+		}
+	}
+
 	mutex_lock(&pfvf->mbox.lock);
 
 	/* In a single request a max of NPC_MAX_NONCONTIG_ENTRIES MCAM entries
@@ -96,18 +207,31 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count)
 		if (!req)
 			goto exit;
 
+		req->kw_type = is_x2 ? NPC_MCAM_KEY_X2 : NPC_MCAM_KEY_X4;
 		req->contig = false;
 		req->count = (count - allocated) > NPC_MAX_NONCONTIG_ENTRIES ?
 				NPC_MAX_NONCONTIG_ENTRIES : count - allocated;
 
+		ref = 0;
+
+		if (is_cn20k(pfvf->pdev)) {
+			req->ref_prio = NPC_MCAM_HIGHER_PRIO;
+			ref = dft_idx;
+		}
+
 		/* Allocate higher priority entries for PFs, so that VF's entries
 		 * will be on top of PF.
 		 */
 		if (!is_otx2_vf(pfvf->pcifunc)) {
-			req->priority = NPC_MCAM_HIGHER_PRIO;
-			req->ref_entry = flow_cfg->def_ent[0];
+			req->ref_prio = NPC_MCAM_HIGHER_PRIO;
+			ref = flow_cfg->def_ent[0];
 		}
 
+		if (is_cn20k(pfvf->pdev))
+			ref = is_x2 ? ref : ref & (x4_slots - 1);
+
+		req->ref_entry = ref;
+
 		/* Send message to AF */
 		if (otx2_sync_mbox_msg(&pfvf->mbox))
 			goto exit;
@@ -163,8 +287,24 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf)
 	struct npc_get_field_status_rsp *frsp;
 	struct npc_mcam_alloc_entry_req *req;
 	struct npc_mcam_alloc_entry_rsp *rsp;
-	int vf_vlan_max_flows;
-	int ent, count;
+	int vf_vlan_max_flows, count;
+	int rc, ref, prio, ent;
+	u16 dft_idx;
+
+	ref = 0;
+	prio = 0;
+	if (is_cn20k(pfvf->pdev)) {
+		rc = otx2_get_dft_rl_idx(pfvf, &dft_idx);
+		if (rc) {
+			netdev_err(pfvf->netdev,
+				   "Error to retrieve ucast mcam idx for pcifunc %#x\n",
+				   pfvf->pcifunc);
+			return rc;
+		}
+
+		ref = dft_idx;
+		prio = NPC_MCAM_HIGHER_PRIO;
+	}
 
 	vf_vlan_max_flows = pfvf->total_vfs * OTX2_PER_VF_VLAN_FLOWS;
 	count = flow_cfg->ucast_flt_cnt +
@@ -183,8 +323,11 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf)
 		return -ENOMEM;
 	}
 
+	req->kw_type = NPC_MCAM_KEY_X2;
 	req->contig = false;
 	req->count = count;
+	req->ref_prio = prio;
+	req->ref_entry = ref;
 
 	/* Send message to AF */
 	if (otx2_sync_mbox_msg(&pfvf->mbox)) {
@@ -819,7 +962,7 @@ static int otx2_prepare_ipv6_flow(struct ethtool_rx_flow_spec *fsp,
 }
 
 static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp,
-			      struct npc_install_flow_req *req)
+				     struct npc_install_flow_req *req)
 {
 	struct ethhdr *eth_mask = &fsp->m_u.ether_spec;
 	struct ethhdr *eth_hdr = &fsp->h_u.ether_spec;
@@ -945,6 +1088,58 @@ static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp,
 	return 0;
 }
 
+static int otx2_get_kw_type(struct otx2_nic *pfvf,
+			    struct npc_install_flow_req *fl_req,
+			    u8 *kw_type)
+{
+	struct npc_get_num_kws_req *req;
+	struct npc_get_num_kws_rsp *rsp;
+	u8 *src, *dst;
+	int off, err;
+	int kw_bits;
+
+	off = offsetof(struct npc_install_flow_req, packet);
+
+	mutex_lock(&pfvf->mbox.lock);
+
+	req = otx2_mbox_alloc_msg_npc_get_num_kws(&pfvf->mbox);
+	if (!req) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -ENOMEM;
+	}
+
+	dst = (u8 *)&req->fl + off;
+	src = (u8 *)fl_req + off;
+
+	memcpy(dst, src, sizeof(struct npc_install_flow_req) - off);
+
+	err = otx2_sync_mbox_msg(&pfvf->mbox);
+	if (err)  {
+		mutex_unlock(&pfvf->mbox.lock);
+		netdev_err(pfvf->netdev,
+			   "Error to get default number of keywords\n");
+		return err;
+	}
+
+	rsp = (struct npc_get_num_kws_rsp *)otx2_mbox_get_rsp
+		(&pfvf->mbox.mbox, 0, &req->hdr);
+	if (IS_ERR(rsp)) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -EFAULT;
+	}
+
+	kw_bits = rsp->kws * 64;
+
+	if (kw_bits <= 256)
+		*kw_type = NPC_MCAM_KEY_X2;
+	else
+		*kw_type = NPC_MCAM_KEY_X4;
+
+	mutex_unlock(&pfvf->mbox.lock);
+
+	return 0;
+}
+
 static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf,
 					struct ethtool_rx_flow_spec *fsp)
 {
@@ -973,12 +1168,41 @@ static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf,
 
 static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow)
 {
+	struct otx2_flow_config *flow_cfg = pfvf->flow_cfg;
+	struct npc_install_flow_req *req, treq = { 0 };
 	u64 ring_cookie = flow->flow_spec.ring_cookie;
 #ifdef CONFIG_DCB
 	int vlan_prio, qidx, pfc_rule = 0;
 #endif
-	struct npc_install_flow_req *req;
-	int err, vf = 0;
+	int err, vf = 0, off, sz;
+	bool modify = false;
+	u8 kw_type = 0;
+	u8 *src, *dst;
+	u16 x4_slots;
+	bool is_x2;
+
+	if (is_cn20k(pfvf->pdev)) {
+		err = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots);
+		if (err) {
+			netdev_err(pfvf->netdev,
+				   "Error to retrieve NPC profile info, pcifunc=%#x\n",
+				   pfvf->pcifunc);
+			return -EFAULT;
+		}
+
+		if (!is_x2) {
+			err = otx2_prepare_flow_request(&flow->flow_spec,
+							&treq);
+			if (err)
+				return err;
+
+			err = otx2_get_kw_type(pfvf, &treq, &kw_type);
+			if (err)
+				return err;
+
+			modify = true;
+		}
+	}
 
 	mutex_lock(&pfvf->mbox.lock);
 	req = otx2_mbox_alloc_msg_npc_install_flow(&pfvf->mbox);
@@ -987,14 +1211,29 @@ static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow)
 		return -ENOMEM;
 	}
 
-	err = otx2_prepare_flow_request(&flow->flow_spec, req);
-	if (err) {
-		/* free the allocated msg above */
-		otx2_mbox_reset(&pfvf->mbox.mbox, 0);
-		mutex_unlock(&pfvf->mbox.lock);
-		return err;
+	if (modify) {
+		off = offsetof(struct npc_install_flow_req, packet);
+		sz = sizeof(struct npc_install_flow_req) - off;
+		dst = (u8 *)req + off;
+		src = (u8 *)&treq + off;
+
+		memcpy(dst, src, sz);
+		req->req_kw_type = kw_type;
+	} else {
+		err = otx2_prepare_flow_request(&flow->flow_spec, req);
+		if (err) {
+			/* free the allocated msg above */
+			otx2_mbox_reset(&pfvf->mbox.mbox, 0);
+			mutex_unlock(&pfvf->mbox.lock);
+			return err;
+		}
 	}
 
+	netdev_dbg(pfvf->netdev,
+		   "flow entry (%u) installed at loc:%u kw_type=%u\n",
+		   flow_cfg->flow_ent[flow->location],
+		   flow->location, kw_type);
+
 	req->entry = flow->entry;
 	req->intf = NIX_INTF_RX;
 	req->set_cntr = 1;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
index 04fcfbdcf69b5..40162b08014dd 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
@@ -31,30 +31,6 @@
 
 #define MCAST_INVALID_GRP		(-1U)
 
-struct otx2_tc_flow_stats {
-	u64 bytes;
-	u64 pkts;
-	u64 used;
-};
-
-struct otx2_tc_flow {
-	struct list_head		list;
-	unsigned long			cookie;
-	struct rcu_head			rcu;
-	struct otx2_tc_flow_stats	stats;
-	spinlock_t			lock; /* lock for stats */
-	u16				rq;
-	u16				entry;
-	u16				leaf_profile;
-	bool				is_act_police;
-	u32				prio;
-	struct npc_install_flow_req	req;
-	u32				mcast_grp_idx;
-	u64				rate;
-	u32				burst;
-	bool				is_pps;
-};
-
 static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst,
 				      u32 *burst_exp, u32 *burst_mantissa)
 {
@@ -971,8 +947,9 @@ static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf)
 	}
 }
 
-static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg,
-							unsigned long cookie)
+static struct otx2_tc_flow *
+otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg,
+			    unsigned long cookie)
 {
 	struct otx2_tc_flow *tmp;
 
@@ -984,8 +961,8 @@ static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config
 	return NULL;
 }
 
-static struct otx2_tc_flow *otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg,
-						       int index)
+struct otx2_tc_flow *
+otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, int index)
 {
 	struct otx2_tc_flow *tmp;
 	int i = 0;
@@ -1014,8 +991,8 @@ static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg,
 	}
 }
 
-static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg,
-				    struct otx2_tc_flow *node)
+int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg,
+			     struct otx2_tc_flow *node)
 {
 	struct list_head *pos, *n;
 	struct otx2_tc_flow *tmp;
@@ -1038,7 +1015,8 @@ static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg,
 	return index;
 }
 
-static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flow_req *req)
+int otx2_add_mcam_flow_entry(struct otx2_nic *nic,
+			     struct npc_install_flow_req *req)
 {
 	struct npc_install_flow_req *tmp_req;
 	int err;
@@ -1064,7 +1042,7 @@ static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flo
 	return 0;
 }
 
-static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val)
+int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val)
 {
 	struct npc_delete_flow_rsp *rsp;
 	struct npc_delete_flow_req *req;
@@ -1114,6 +1092,11 @@ static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic,
 	int i = 0, index = 0;
 	u16 cntr_val = 0;
 
+	if (is_cn20k(nic->pdev)) {
+		cn20k_tc_update_mcam_table_del_req(nic, flow_cfg, node);
+		return 0;
+	}
+
 	/* Find and delete the entry from the list and re-install
 	 * all the entries from beginning to the index of the
 	 * deleted entry to higher mcam indexes.
@@ -1153,6 +1136,9 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic,
 	int list_idx, i;
 	u16 cntr_val = 0;
 
+	if (is_cn20k(nic->pdev))
+		return cn20k_tc_update_mcam_table_add_req(nic, flow_cfg, node);
+
 	/* Find the index of the entry(list_idx) whose priority
 	 * is greater than the new entry and re-install all
 	 * the entries from beginning to list_idx to higher
@@ -1172,7 +1158,7 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic,
 		mcam_idx++;
 	}
 
-	return mcam_idx;
+	return flow_cfg->flow_ent[mcam_idx];
 }
 
 static int otx2_tc_update_mcam_table(struct otx2_nic *nic,
@@ -1238,7 +1224,6 @@ static int otx2_tc_del_flow(struct otx2_nic *nic,
 		mutex_unlock(&nic->mbox.lock);
 	}
 
-
 free_mcam_flow:
 	otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL);
 	otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false);
@@ -1254,7 +1239,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic,
 	struct otx2_flow_config *flow_cfg = nic->flow_cfg;
 	struct otx2_tc_flow *new_node, *old_node;
 	struct npc_install_flow_req *req, dummy;
-	int rc, err, mcam_idx;
+	int rc, err, entry;
 
 	if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT))
 		return -ENOMEM;
@@ -1264,7 +1249,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic,
 		return -EINVAL;
 	}
 
-	if (flow_cfg->nr_flows == flow_cfg->max_flows) {
+	if (!is_cn20k(nic->pdev) && flow_cfg->nr_flows == flow_cfg->max_flows) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "Free MCAM entry not available to add the flow");
 		return -ENOMEM;
@@ -1292,7 +1277,23 @@ static int otx2_tc_add_flow(struct otx2_nic *nic,
 	if (old_node)
 		otx2_tc_del_flow(nic, tc_flow_cmd);
 
-	mcam_idx = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true);
+	if (is_cn20k(nic->pdev)) {
+		rc = cn20k_tc_alloc_entry(nic, tc_flow_cmd, new_node, &dummy);
+		if (rc) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "MCAM rule allocation failed");
+			kfree_rcu(new_node, rcu);
+			return rc;
+		}
+	}
+
+	entry = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true);
+	if (entry < 0) {
+		NL_SET_ERR_MSG_MOD(extack, "Adding rule failed");
+		rc = entry;
+		goto free_leaf;
+	}
+
 	mutex_lock(&nic->mbox.lock);
 	req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
 	if (!req) {
@@ -1304,7 +1305,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic,
 	memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr));
 	memcpy(req, &dummy, sizeof(struct npc_install_flow_req));
 	req->channel = nic->hw.rx_chan_base;
-	req->entry = flow_cfg->flow_ent[mcam_idx];
+	req->entry = (u16)entry;
 	req->intf = NIX_INTF_RX;
 	req->vf = nic->pcifunc;
 	req->set_cntr = 1;
@@ -1325,6 +1326,8 @@ static int otx2_tc_add_flow(struct otx2_nic *nic,
 	return 0;
 
 free_leaf:
+	if (is_cn20k(nic->pdev))
+		cn20k_tc_free_mcam_entry(nic, new_node->entry);
 	otx2_tc_del_from_flow_list(flow_cfg, new_node);
 	if (new_node->is_act_police) {
 		mutex_lock(&nic->mbox.lock);
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index ddc321a02fdae..bb8ced22ca3be 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -562,9 +562,7 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
 	int val, ge_mode, err = 0;
 	u32 i;
 
-	/* MT76x8 has no hardware settings between for the MAC */
-	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
-	    mac->interface != state->interface) {
+	if (mac->interface != state->interface) {
 		/* Setup soc pin functions */
 		switch (state->interface) {
 		case PHY_INTERFACE_MODE_TRGMII:
@@ -956,6 +954,30 @@ static const struct phylink_mac_ops mtk_phylink_ops = {
 	.mac_enable_tx_lpi = mtk_mac_enable_tx_lpi,
 };
 
+static void rt5350_mac_config(struct phylink_config *config, unsigned int mode,
+				const struct phylink_link_state *state)
+{
+}
+
+static void rt5350_mac_link_down(struct phylink_config *config, unsigned int mode,
+				phy_interface_t interface)
+{
+}
+
+static void rt5350_mac_link_up(struct phylink_config *config,
+			    struct phy_device *phy,
+			    unsigned int mode, phy_interface_t interface,
+			    int speed, int duplex, bool tx_pause, bool rx_pause)
+{
+}
+
+/* MT76x8 (rt5350-eth) does not expose any MAC control registers */
+static const struct phylink_mac_ops rt5350_phylink_ops = {
+	.mac_config = rt5350_mac_config,
+	.mac_link_down = rt5350_mac_link_down,
+	.mac_link_up = rt5350_mac_link_up,
+};
+
 static void mtk_mdio_config(struct mtk_eth *eth)
 {
 	u32 val;
@@ -4780,6 +4802,7 @@ static const struct net_device_ops mtk_netdev_ops = {
 
 static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 {
+	const struct phylink_mac_ops *mac_ops = &mtk_phylink_ops;
 	const __be32 *_id = of_get_property(np, "reg", NULL);
 	phy_interface_t phy_mode;
 	struct phylink *phylink;
@@ -4914,9 +4937,12 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 			  mac->phylink_config.supported_interfaces);
 	}
 
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
+		mac_ops = &rt5350_phylink_ops;
+
 	phylink = phylink_create(&mac->phylink_config,
 				 of_fwnode_handle(mac->of_node),
-				 phy_mode, &mtk_phylink_ops);
+				 phy_mode, mac_ops);
 	if (IS_ERR(phylink)) {
 		err = PTR_ERR(phylink);
 		goto free_netdev;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 8ffa286a18f5d..d39fe9c4a87cd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -16,8 +16,9 @@ mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
 		fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \
 		lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \
-		diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o diag/reporter_vnic.o \
-		fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o lib/nv_param.o
+		diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \
+		diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \
+		lib/nv_param.o
 
 #
 # Netdev basic
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
index 6d73127b7217f..2cf1d3825defa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
@@ -282,6 +282,9 @@ const char *parse_fs_dst(struct trace_seq *p,
 	case MLX5_FLOW_DESTINATION_TYPE_NONE:
 		trace_seq_printf(p, "none\n");
 		break;
+	case MLX5_FLOW_DESTINATION_TYPE_VHCA_RX:
+		trace_seq_printf(p, "rx_vhca_id=%u\n", dst->vhca.id);
+		break;
 	}
 
 	trace_seq_putc(p, 0);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index ea2cd1f5d1d07..c7ac6ebe82906 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -80,6 +80,7 @@ struct page_pool;
 #define MLX5_SKB_FRAG_SZ(len)	(SKB_DATA_ALIGN(len) +	\
 				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 
+#define MLX5E_PAGECNT_BIAS_MAX U16_MAX
 #define MLX5E_RX_MAX_HEAD (256)
 #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8)
 #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \
@@ -1060,13 +1061,15 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv);
 struct mlx5e_xsk_param;
 
 struct mlx5e_rq_param;
-int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
-		  struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
+struct mlx5e_rq_opt_param;
+int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
+		  struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
 		  struct mlx5e_rq *rq);
 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
 void mlx5e_close_rq(struct mlx5e_rq *rq);
-int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter);
+int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
+		    u16 q_counter);
 void mlx5e_destroy_rq(struct mlx5e_rq *rq);
 
 bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index 8e99d07586fa5..26bb31c56e457 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -10,6 +10,7 @@
 #include <linux/dim.h>
 #include <net/page_pool/types.h>
 #include <net/xdp_sock_drv.h>
+#include <net/netdev_queues.h>
 
 #define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18
 #define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17
@@ -21,10 +22,20 @@ static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev)
 	return min_page_shift ? : 12;
 }
 
-u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
+u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev,
+			  struct mlx5e_rq_opt_param *rqo)
 {
-	u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT;
+	struct netdev_queue_config *qcfg = rqo ? rqo->qcfg : NULL;
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
 	u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev);
+	u8 req_page_shift;
+
+	if (xsk)
+		req_page_shift = order_base_2(xsk->chunk_size);
+	else if (qcfg && qcfg->rx_page_size)
+		req_page_shift = order_base_2(qcfg->rx_page_size);
+	else
+		req_page_shift = PAGE_SHIFT;
 
 	/* Regular RQ uses order-0 pages, the NIC must be able to map them. */
 	if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift))
@@ -34,7 +45,8 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs
 }
 
 enum mlx5e_mpwrq_umr_mode
-mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
+mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev,
+		     struct mlx5e_rq_opt_param *rqo)
 {
 	/* Different memory management schemes use different mechanisms to map
 	 * user-mode memory. The stricter guarantees we have, the faster
@@ -45,7 +57,8 @@ mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
 	 * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and
 	 *    mappings can have different sizes.
 	 */
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	bool unaligned = xsk ? xsk->unaligned : false;
 	bool oversized = false;
 
@@ -225,12 +238,12 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
 }
 
 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
-				 struct mlx5e_xsk_param *xsk)
+				 struct mlx5e_rq_opt_param *rqo)
 {
 	u16 headroom;
 
-	if (xsk)
-		return xsk->headroom;
+	if (mlx5e_rqo_xsk_param(rqo))
+		return rqo->xsk->headroom;
 
 	headroom = NET_IP_ALIGN;
 	if (params->xdp_prog)
@@ -263,19 +276,23 @@ static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_
 
 static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
 					 struct mlx5e_params *params,
-					 struct mlx5e_xsk_param *xsk,
+					 struct mlx5e_rq_opt_param *rqo,
 					 bool mpwqe)
 {
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
 	bool no_head_tail_room;
 	u32 sz;
 
 	/* XSK frames are mapped as individual pages, because frames may come in
 	 * an arbitrary order from random locations in the UMEM.
 	 */
-	if (xsk)
-		return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
+	if (xsk) {
+		return mpwqe ?
+			BIT(mlx5e_mpwrq_page_shift(mdev, rqo)) : PAGE_SIZE;
+	}
 
-	no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk);
+	no_head_tail_room = params->xdp_prog && mpwqe &&
+			    !mlx5e_rx_is_linear_skb(mdev, params, rqo);
 
 	/* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations.
 	 * no_head_tail_room should be set in the case of XDP with Striding RQ
@@ -291,11 +308,12 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
 
 static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
 				       struct mlx5e_params *params,
-				       struct mlx5e_xsk_param *xsk)
+				       struct mlx5e_rq_opt_param *rqo)
 {
-	u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	u32 linear_stride_sz =
+		mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 
 	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
 		order_base_2(linear_stride_sz);
@@ -303,8 +321,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
 
 bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
 			    struct mlx5e_params *params,
-			    struct mlx5e_xsk_param *xsk)
+			    struct mlx5e_rq_opt_param *rqo)
 {
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
+
 	if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE)
 		return false;
 
@@ -315,7 +335,7 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
 	 * Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data
 	 * must fit into a CPU page.
 	 */
-	if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE)
+	if (mlx5e_rx_get_linear_sz_skb(params, !!xsk) > PAGE_SIZE)
 		return false;
 
 	/* XSK frames must be big enough to hold the packet data. */
@@ -349,12 +369,14 @@ static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
 
 bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
 					  struct mlx5e_params *params,
-					  struct mlx5e_xsk_param *xsk)
+					  struct mlx5e_rq_opt_param *rqo)
 {
-	u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
-	u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	u8 log_wqe_num_of_strides =
+		mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo);
+	u8 log_wqe_stride_size =
+		mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 
 	return mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
 					     log_wqe_num_of_strides,
@@ -363,18 +385,20 @@ bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
 
 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 				  struct mlx5e_params *params,
-				  struct mlx5e_xsk_param *xsk)
+				  struct mlx5e_rq_opt_param *rqo)
 {
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	u32 linear_stride_sz =
+		mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	u8 log_num_strides;
 	u8 log_stride_sz;
 	u8 log_wqe_sz;
 
-	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk))
+	if (!mlx5e_rx_is_linear_skb(mdev, params, rqo))
 		return false;
 
-	log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
+	log_stride_sz = order_base_2(linear_stride_sz);
 	log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
 
 	if (log_wqe_sz < log_stride_sz)
@@ -389,13 +413,13 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 
 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
 			       struct mlx5e_params *params,
-			       struct mlx5e_xsk_param *xsk)
+			       struct mlx5e_rq_opt_param *rqo)
 {
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
 	u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
 
-	log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
-	page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, rqo);
+	page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);
 
 	/* Numbers are unsigned, don't subtract to avoid underflow. */
@@ -423,10 +447,11 @@ static u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5e_params *params)
 
 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 				   struct mlx5e_params *params,
-				   struct mlx5e_xsk_param *xsk)
+				   struct mlx5e_rq_opt_param *rqo)
 {
-	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
-		return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
+	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo))
+		return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params,
+								  rqo, true));
 
 	/* XDP in mlx5e doesn't support multiple packets per page. */
 	if (params->xdp_prog)
@@ -437,17 +462,18 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 
 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
 				   struct mlx5e_params *params,
-				   struct mlx5e_xsk_param *xsk)
+				   struct mlx5e_rq_opt_param *rqo)
 {
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	u8 log_wqe_size, log_stride_size;
 
 	log_wqe_size = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
-	log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
+	log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
 	WARN(log_wqe_size < log_stride_size,
 	     "Log WQE size %u < log stride size %u (page shift %u, umr mode %d, xsk on? %d)\n",
-	     log_wqe_size, log_stride_size, page_shift, umr_mode, !!xsk);
+	     log_wqe_size, log_stride_size, page_shift, umr_mode,
+	     rqo && rqo->xsk);
 	return log_wqe_size - log_stride_size;
 }
 
@@ -459,14 +485,14 @@ u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz)
 
 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
 			  struct mlx5e_params *params,
-			  struct mlx5e_xsk_param *xsk)
+			  struct mlx5e_rq_opt_param *rqo)
 {
-	u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
+	u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, rqo);
 
 	if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
 		return linear_headroom;
 
-	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
+	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo))
 		return linear_headroom;
 
 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
@@ -475,6 +501,18 @@ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
 	return 0;
 }
 
+u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev)
+{
+	if (mlx5_core_is_ecpf(mdev))
+		return PAGE_SIZE;
+
+	/* Two MTTs are needed to form an octword. Driver is using a
+	 * single page per MTT for simplicity. Hence the limit of having
+	 * at least 2 pages per WQE.
+	 */
+	return BIT(MLX5_MPWRQ_MAX_LOG_WQE_SZ - 1);
+}
+
 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 {
 	bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
@@ -535,10 +573,11 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params
 }
 
 int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
-			     struct mlx5e_xsk_param *xsk)
+			     struct mlx5e_rq_opt_param *rqo)
 {
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	u16 max_mtu_pkts;
 
 	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) {
@@ -547,7 +586,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
 		return -EOPNOTSUPP;
 	}
 
-	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) {
+	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) {
 		mlx5_core_err(mdev, "Striding RQ linear mode for XSK can't be activated with current params\n");
 		return -EINVAL;
 	}
@@ -559,7 +598,8 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
 			     mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, xsk->unaligned));
 	if (params->log_rq_mtu_frames > max_mtu_pkts) {
 		mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
-			      1 << params->log_rq_mtu_frames, xsk->chunk_size);
+			      1 << params->log_rq_mtu_frames,
+			      xsk->chunk_size);
 		return -EINVAL;
 	}
 
@@ -672,7 +712,7 @@ static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params,
 
 static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 				     struct mlx5e_params *params,
-				     struct mlx5e_xsk_param *xsk,
+				     struct mlx5e_rq_opt_param *rqo,
 				     struct mlx5e_rq_frags_info *info,
 				     u32 *xdp_frag_size)
 {
@@ -684,10 +724,11 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 	int max_mtu;
 	int i;
 
-	if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
+	if (mlx5e_rx_is_linear_skb(mdev, params, rqo)) {
 		int frag_stride;
 
-		frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false);
+		frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, rqo,
+							    false);
 
 		info->arr[0].frag_size = byte_count;
 		info->arr[0].frag_stride = frag_stride;
@@ -703,7 +744,7 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 		goto out;
 	}
 
-	headroom = mlx5e_get_linear_rq_headroom(params, xsk);
+	headroom = mlx5e_get_linear_rq_headroom(params, rqo);
 	first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
 
 	max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max,
@@ -819,12 +860,13 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
 
 static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
 					struct mlx5e_params *params,
-					struct mlx5e_xsk_param *xsk)
+					struct mlx5e_rq_opt_param *rqo)
 {
-	u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
-	u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
+	u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
+	u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params,
+							      rqo));
 	int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params));
-	int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
+	int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo));
 	int wqe_size = BIT(log_stride_sz) * num_strides;
 	int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE;
 
@@ -836,7 +878,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
 
 static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
 				    struct mlx5e_params *params,
-				    struct mlx5e_xsk_param *xsk,
+				    struct mlx5e_rq_opt_param *rqo,
 				    struct mlx5e_cq_param *param)
 {
 	bool hw_stridx = false;
@@ -847,10 +889,13 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 		hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
 		if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
-			log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk);
+			log_cq_size =
+				mlx5e_shampo_get_log_cq_size(mdev, params, rqo);
 		else
-			log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) +
-				mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
+			log_cq_size =
+				mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo) +
+				mlx5e_mpwqe_get_log_num_strides(mdev, params,
+								rqo);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
 		log_cq_size = params->log_rq_mtu_frames;
@@ -880,69 +925,90 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param
 		MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
 }
 
+static int mlx5e_mpwqe_build_rq_param(struct mlx5_core_dev *mdev,
+				      struct mlx5e_params *params,
+				      struct mlx5e_rq_opt_param *rqo,
+				      struct mlx5e_rq_param *rq_param)
+{
+	u8 log_rq_sz = mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
+	u8 log_wqe_num_of_strides, log_wqe_stride_size;
+	enum mlx5e_mpwrq_umr_mode umr_mode;
+	void *rqc = rq_param->rqc;
+	u32 lro_timeout;
+	void *wq;
+
+	log_wqe_num_of_strides =
+		mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo);
+	log_wqe_stride_size =
+		mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
+	umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+
+	wq = MLX5_ADDR_OF(rqc, rqc, wq);
+	if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
+					   log_wqe_num_of_strides,
+					   page_shift, umr_mode)) {
+		mlx5_core_err(mdev,
+			      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
+			      log_wqe_stride_size, log_wqe_num_of_strides,
+			      umr_mode);
+		return -EINVAL;
+	}
+
+	MLX5_SET(wq, wq, log_wqe_num_of_strides,
+		 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
+	MLX5_SET(wq, wq, log_wqe_stride_size,
+		 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
+	MLX5_SET(wq, wq, log_wq_sz, log_rq_sz);
+	if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO)
+		return 0;
+
+	MLX5_SET(wq, wq, shampo_enable, true);
+	MLX5_SET(wq, wq, log_reservation_size,
+		 MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE -
+		 MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT);
+	MLX5_SET(wq, wq, log_max_num_of_packets_per_reservation,
+		 mlx5e_shampo_get_log_pkt_per_rsrv(params));
+	MLX5_SET(wq, wq, log_headers_entry_size,
+		 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE -
+		 MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT);
+	lro_timeout = mlx5e_choose_lro_timeout(mdev,
+					       MLX5E_DEFAULT_SHAMPO_TIMEOUT);
+	MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout);
+	MLX5_SET(rqc, rqc, shampo_match_criteria_type,
+		 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED);
+	MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
+		 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE);
+
+	return 0;
+}
+
 int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 			 struct mlx5e_params *params,
-			 struct mlx5e_xsk_param *xsk,
-			 struct mlx5e_rq_param *param)
+			 struct mlx5e_rq_opt_param *rqo,
+			 struct mlx5e_rq_param *rq_param)
 {
-	void *rqc = param->rqc;
-	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
-	u32 lro_timeout;
+	void *rqc = rq_param->rqc;
 	int ndsegs = 1;
+	void *wq;
 	int err;
 
-	switch (params->rq_wq_type) {
-	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
-		u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
-		u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
-		enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-		u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
-
-		if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
-						   log_wqe_num_of_strides,
-						   page_shift, umr_mode)) {
-			mlx5_core_err(mdev,
-				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
-				      log_wqe_stride_size, log_wqe_num_of_strides,
-				      umr_mode);
-			return -EINVAL;
-		}
-
-		MLX5_SET(wq, wq, log_wqe_num_of_strides,
-			 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
-		MLX5_SET(wq, wq, log_wqe_stride_size,
-			 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
-		MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
-		if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO)
-			break;
+	wq = MLX5_ADDR_OF(rqc, rqc, wq);
 
-		MLX5_SET(wq, wq, shampo_enable, true);
-		MLX5_SET(wq, wq, log_reservation_size,
-			 MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE -
-			 MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT);
-		MLX5_SET(wq, wq,
-			 log_max_num_of_packets_per_reservation,
-			 mlx5e_shampo_get_log_pkt_per_rsrv(params));
-		MLX5_SET(wq, wq, log_headers_entry_size,
-			 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE -
-			 MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT);
-		lro_timeout =
-			mlx5e_choose_lro_timeout(mdev,
-						 MLX5E_DEFAULT_SHAMPO_TIMEOUT);
-		MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout);
-		MLX5_SET(rqc, rqc, shampo_match_criteria_type,
-			 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED);
-		MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
-			 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE);
+	switch (params->rq_wq_type) {
+	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
+		err = mlx5e_mpwqe_build_rq_param(mdev, params, rqo, rq_param);
+		if (err)
+			return err;
 		break;
-	}
 	default: /* MLX5_WQ_TYPE_CYCLIC */
 		MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
-		err = mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info,
-						&param->xdp_frag_size);
+		err = mlx5e_build_rq_frags_info(mdev, params, rqo,
+						&rq_param->frags_info,
+						&rq_param->xdp_frag_size);
 		if (err)
 			return err;
-		ndsegs = param->frags_info.num_frags;
+		ndsegs = rq_param->frags_info.num_frags;
 	}
 
 	MLX5_SET(wq, wq, wq_type,          params->rq_wq_type);
@@ -953,23 +1019,23 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 	MLX5_SET(rqc, rqc, vsd,            params->vlan_strip_disable);
 	MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
 
-	param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
-	mlx5e_build_rx_cq_param(mdev, params, xsk, &param->cqp);
+	rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
+	mlx5e_build_rx_cq_param(mdev, params, rqo, &rq_param->cqp);
 
 	return 0;
 }
 
 void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
-			       struct mlx5e_rq_param *param)
+			       struct mlx5e_rq_param *rq_param)
 {
-	void *rqc = param->rqc;
+	void *rqc = rq_param->rqc;
 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
 
 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
 	MLX5_SET(wq, wq, log_wq_stride,
 		 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
 
-	param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
+	rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
 }
 
 void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
@@ -1084,29 +1150,30 @@ u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
 
 static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
 					struct mlx5e_params *params,
-					struct mlx5e_xsk_param *xsk)
+					struct mlx5e_rq_opt_param *rqo)
 {
-	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
-	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
 	u8 umr_wqebbs;
 
 	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
 
-	return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
+	return umr_wqebbs *
+	       (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo));
 }
 
-static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
-				      struct mlx5e_params *params,
-				      struct mlx5e_rq_param *rqp)
+static u32 mlx5e_max_xsk_wqebbs(struct mlx5_core_dev *mdev,
+				struct mlx5e_params *params)
 {
-	u32 wqebbs, total_pages, useful_space;
+	struct mlx5e_rq_opt_param rqo = {0};
+	struct mlx5e_xsk_param xsk = {0};
+	u32 max_xsk_wqebbs = 0;
+	u8 frame_shift;
 
-	/* MLX5_WQ_TYPE_CYCLIC */
-	if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
-		return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
+	if (!params->xdp_prog)
+		return 0;
 
-	/* UMR WQEs for the regular RQ. */
-	wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
+	rqo.xsk = &xsk;
 
 	/* If XDP program is attached, XSK may be turned on at any time without
 	 * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
@@ -1118,41 +1185,54 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
 	 * from capabilities. Hence, we have to try all valid values of XSK
 	 * frame size (and page_shift) to find the maximum.
 	 */
-	if (params->xdp_prog) {
-		u32 max_xsk_wqebbs = 0;
-		u8 frame_shift;
-
-		for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
-		     frame_shift <= PAGE_SHIFT; frame_shift++) {
-			/* The headroom doesn't affect the calculation. */
-			struct mlx5e_xsk_param xsk = {
-				.chunk_size = 1 << frame_shift,
-				.unaligned = false,
-			};
-
-			/* XSK aligned mode. */
-			max_xsk_wqebbs = max(max_xsk_wqebbs,
-				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
-
-			/* XSK unaligned mode, frame size is a power of two. */
-			xsk.unaligned = true;
-			max_xsk_wqebbs = max(max_xsk_wqebbs,
-				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
-
-			/* XSK unaligned mode, frame size is not equal to stride size. */
-			xsk.chunk_size -= 1;
-			max_xsk_wqebbs = max(max_xsk_wqebbs,
-				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
-
-			/* XSK unaligned mode, frame size is a triple power of two. */
-			xsk.chunk_size = (1 << frame_shift) / 4 * 3;
-			max_xsk_wqebbs = max(max_xsk_wqebbs,
-				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
-		}
+	for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
+	     frame_shift <= PAGE_SHIFT; frame_shift++) {
+		u32 total_wqebbs;
+
+		/* The headroom doesn't affect the calculations below. */
+
+		/* XSK aligned mode. */
+		xsk.chunk_size = 1 << frame_shift;
+		xsk.unaligned = false;
+		total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
+		max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
 
-		wqebbs += max_xsk_wqebbs;
+		/* XSK unaligned mode, frame size is a power of two. */
+		xsk.unaligned = true;
+		total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
+		max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
+
+		/* XSK unaligned mode, frame size is not equal to stride
+		 * size.
+		 */
+		xsk.chunk_size -= 1;
+		total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
+		max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
+
+		/* XSK unaligned mode, frame size is a triple power of two. */
+		xsk.chunk_size = (1 << frame_shift) / 4 * 3;
+		total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
+		max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
 	}
 
+	return max_xsk_wqebbs;
+}
+
+static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
+				      struct mlx5e_params *params,
+				      struct mlx5e_rq_param *rq_param)
+{
+	u32 wqebbs, total_pages, useful_space;
+
+	/* MLX5_WQ_TYPE_CYCLIC */
+	if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
+		return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
+
+	/* UMR WQEs for the regular RQ. */
+	wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
+
+	wqebbs += mlx5e_max_xsk_wqebbs(mdev, params);
+
 	/* UMR WQEs don't cross the page boundary, they are padded with NOPs.
 	 * This padding is always smaller than the max WQE size. That gives us
 	 * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes
@@ -1222,12 +1302,15 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
 
 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
 			      struct mlx5e_params *params,
+			      struct netdev_queue_config *qcfg,
 			      struct mlx5e_channel_param *cparam)
 {
 	u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
 	int err;
 
-	err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq);
+	cparam->rq_opt.qcfg = qcfg;
+
+	err = mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq);
 	if (err)
 		return err;
 
@@ -1241,3 +1324,13 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
 
 	return 0;
 }
+
+void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev,
+				   struct mlx5e_params *params,
+				   struct mlx5e_xsk_param *xsk,
+				   struct mlx5e_channel_param *cparam)
+{
+	cparam->rq_opt.xsk = xsk;
+	mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq);
+	mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
index 00617c65fe3cd..9b1a2aed17c3f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
@@ -12,6 +12,11 @@ struct mlx5e_xsk_param {
 	bool unaligned;
 };
 
+struct mlx5e_rq_opt_param {
+	struct mlx5e_xsk_param *xsk;
+	struct netdev_queue_config *qcfg;
+};
+
 struct mlx5e_cq_param {
 	u32                        cqc[MLX5_ST_SZ_DW(cqc)];
 	struct mlx5_wq_param       wq;
@@ -38,6 +43,7 @@ struct mlx5e_sq_param {
 
 struct mlx5e_channel_param {
 	struct mlx5e_rq_param      rq;
+	struct mlx5e_rq_opt_param  rq_opt;
 	struct mlx5e_sq_param      txq_sq;
 	struct mlx5e_sq_param      xdp_sq;
 	struct mlx5e_sq_param      icosq;
@@ -56,9 +62,11 @@ struct mlx5e_create_sq_param {
 
 /* Striding RQ dynamic parameters */
 
-u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
+u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev,
+			  struct mlx5e_rq_opt_param *rqo);
 enum mlx5e_mpwrq_umr_mode
-mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
+mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev,
+		     struct mlx5e_rq_opt_param *rqo);
 u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode);
 u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
 			  enum mlx5e_mpwrq_umr_mode umr_mode);
@@ -80,22 +88,22 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
 bool slow_pci_heuristic(struct mlx5_core_dev *mdev);
 int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
-			     struct mlx5e_xsk_param *xsk);
+			     struct mlx5e_rq_opt_param *rqo);
 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 
 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
-				 struct mlx5e_xsk_param *xsk);
+				 struct mlx5e_rq_opt_param *rqo);
 bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
 			    struct mlx5e_params *params,
-			    struct mlx5e_xsk_param *xsk);
+			    struct mlx5e_rq_opt_param *rqo);
 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 				  struct mlx5e_params *params,
-				  struct mlx5e_xsk_param *xsk);
+				  struct mlx5e_rq_opt_param *rqo);
 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
 			       struct mlx5e_params *params,
-			       struct mlx5e_xsk_param *xsk);
+			       struct mlx5e_rq_opt_param *rqo);
 u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev,
 			    struct mlx5e_params *params,
 			    struct mlx5e_rq_param *rq_param);
@@ -105,21 +113,22 @@ u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev,
 u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout);
 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 				   struct mlx5e_params *params,
-				   struct mlx5e_xsk_param *xsk);
+				   struct mlx5e_rq_opt_param *rqo);
 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
 				   struct mlx5e_params *params,
-				   struct mlx5e_xsk_param *xsk);
+				   struct mlx5e_rq_opt_param *rqo);
 u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz);
 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
 			  struct mlx5e_params *params,
-			  struct mlx5e_xsk_param *xsk);
+			  struct mlx5e_rq_opt_param *rqo);
+u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev);
 
 /* Build queue parameters */
 
 void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c);
 int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 			 struct mlx5e_params *params,
-			 struct mlx5e_xsk_param *xsk,
+			 struct mlx5e_rq_opt_param *rqo,
 			 struct mlx5e_rq_param *param);
 void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
 			       struct mlx5e_rq_param *param);
@@ -136,13 +145,19 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
 			     struct mlx5e_sq_param *param);
 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
 			      struct mlx5e_params *params,
+			      struct netdev_queue_config *qcfg,
 			      struct mlx5e_channel_param *cparam);
 
+void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev,
+				   struct mlx5e_params *params,
+				   struct mlx5e_xsk_param *xsk,
+				   struct mlx5e_channel_param *cparam);
+
 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
 					  struct mlx5e_params *params,
-					  struct mlx5e_xsk_param *xsk);
+					  struct mlx5e_rq_opt_param *rqo);
 
 static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
 					   struct mlx5e_params *params)
@@ -158,4 +173,10 @@ static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
 				       "enhanced" : "basic");
 };
 
+static inline struct mlx5e_xsk_param *
+mlx5e_rqo_xsk_param(struct mlx5e_rq_opt_param *rqo)
+{
+	return rqo ? rqo->xsk : NULL;
+}
+
 #endif /* __MLX5_EN_PARAMS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 1b76647f31948..723f66a6bd634 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -660,13 +660,13 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev,
 				     struct net_device *netdev,
 				     struct mlx5e_ptp_params *ptp_params)
 {
-	struct mlx5e_rq_param *rq_params = &ptp_params->rq_param;
+	struct mlx5e_rq_param *rq_param = &ptp_params->rq_param;
 	struct mlx5e_params *params = &ptp_params->params;
 
 	params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC;
 	mlx5e_init_rq_type_params(mdev, params);
 	params->sw_mtu = netdev->max_mtu;
-	mlx5e_build_rq_param(mdev, params, NULL, rq_params);
+	mlx5e_build_rq_param(mdev, params, NULL, rq_param);
 }
 
 static void mlx5e_ptp_build_params(struct mlx5e_ptp *c,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 80f9fc10877ad..c539506bf40f0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -37,9 +37,10 @@
 #include <linux/bitfield.h>
 #include <net/page_pool/helpers.h>
 
-int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
+int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
+		      struct mlx5e_rq_opt_param *rqo)
 {
-	int hr = mlx5e_get_linear_rq_headroom(params, xsk);
+	int hr = mlx5e_get_linear_rq_headroom(params, rqo);
 
 	/* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)).
 	 * The condition checked in mlx5e_rx_is_linear_skb is:
@@ -707,7 +708,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
 				xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
 				page = xdpi.page.page;
 
-				/* No need to check page_pool_page_is_pp() as we
+				/* No need to check PageNetpp() as we
 				 * know this is a page_pool page.
 				 */
 				page_pool_recycle_direct(pp_page_to_nmdesc(page)->pp,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index 46ab0a9e8cddb..3c54f8962664f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -96,7 +96,8 @@ union mlx5e_xdp_info {
 };
 
 struct mlx5e_xsk_param;
-int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
+int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
+		      struct mlx5e_rq_opt_param *rqo);
 bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
 		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
 void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
index c5ba22eed8f19..96a259914cbde 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
@@ -78,6 +78,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 				   struct xsk_buff_pool *pool, u16 ix)
 {
 	struct mlx5e_params *params = &priv->channels.params;
+	struct mlx5e_channel_param *cparam;
+	enum mlx5e_mpwrq_umr_mode umr_mode;
 	struct mlx5e_xsk_param xsk;
 	struct mlx5e_channel *c;
 	int err;
@@ -88,18 +90,24 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 	if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
 		return -EINVAL;
 
+	cparam = kvzalloc_obj(*cparam);
+	if (!cparam)
+		return -ENOMEM;
+
 	err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool);
 	if (unlikely(err))
-		return err;
+		goto err_free_cparam;
 
 	err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
 	if (unlikely(err))
 		goto err_unmap_pool;
 
 	mlx5e_build_xsk_param(pool, &xsk);
+	mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
 
+	umr_mode = mlx5e_mpwrq_umr_mode(priv->mdev, &cparam->rq_opt);
 	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
-	    mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
+	    umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
 		const char *recommendation = is_power_of_2(xsk.chunk_size) ?
 			"Upgrade firmware" : "Disable striding RQ";
 
@@ -121,7 +129,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 
 	c = priv->channels.c[ix];
 
-	err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
+	err = mlx5e_open_xsk(priv, params, cparam, pool, c);
 	if (unlikely(err))
 		goto err_remove_pool;
 
@@ -137,6 +145,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 	mlx5e_deactivate_rq(&c->rq);
 	mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
 
+	kvfree(cparam);
+
 	return 0;
 
 err_remove_pool:
@@ -145,17 +155,22 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 err_unmap_pool:
 	mlx5e_xsk_unmap_pool(priv, pool);
 
+err_free_cparam:
+	kvfree(cparam);
+
 	return err;
 
 validate_closed:
 	/* Check the configuration in advance, rather than fail at a later stage
 	 * (in mlx5e_xdp_set or on open) and end up with no channels.
 	 */
-	if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
+	if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) {
 		err = -EINVAL;
 		goto err_remove_pool;
 	}
 
+	kvfree(cparam);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index d39f778ecd85b..11500fd213a55 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -9,9 +9,9 @@
 
 static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
 					struct mlx5e_params *params,
-					struct mlx5e_xsk_param *xsk)
+					struct mlx5e_rq_opt_param *rqo)
 {
-	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
+	if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) {
 		mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
 		return -EINVAL;
 	}
@@ -25,9 +25,14 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
 #define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
 
 bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
-			      struct mlx5e_xsk_param *xsk,
+			      struct mlx5e_rq_opt_param *rqo,
 			      struct mlx5_core_dev *mdev)
 {
+	struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
+
+	if (WARN_ON(!xsk))
+		return false;
+
 	/* AF_XDP doesn't support frames larger than PAGE_SIZE,
 	 * and xsk->chunk_size is limited to 65535 bytes.
 	 */
@@ -42,21 +47,12 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
 	 */
 	switch (params->rq_wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
+		return !mlx5e_mpwrq_validate_xsk(mdev, params, rqo);
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
+		return !mlx5e_legacy_rq_validate_xsk(mdev, params, rqo);
 	}
 }
 
-static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
-				   struct mlx5e_params *params,
-				   struct mlx5e_xsk_param *xsk,
-				   struct mlx5e_channel_param *cparam)
-{
-	mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
-	mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
-}
-
 static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
 			     struct mlx5e_params *params,
 			     struct xsk_buff_pool *pool,
@@ -90,19 +86,23 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
 	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
 }
 
-static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
-			     struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
-			     struct mlx5e_xsk_param *xsk)
+static int mlx5e_open_xsk_rq(struct mlx5e_channel *c,
+			     struct mlx5e_params *params,
+			     struct mlx5e_channel_param *cparam,
+			     struct xsk_buff_pool *pool)
 {
+	struct mlx5e_rq_param *rq_param = &cparam->rq;
+	struct mlx5e_rq_opt_param *rqo = &cparam->rq_opt;
 	u16 q_counter = c->priv->q_counter[c->sd_ix];
 	struct mlx5e_rq *xskrq = &c->xskrq;
 	int err;
 
-	err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
+	err = mlx5e_init_xsk_rq(c, params, pool, rqo->xsk, xskrq);
 	if (err)
 		return err;
 
-	err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
+	err = mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
+			    q_counter, xskrq);
 	if (err)
 		return err;
 
@@ -111,30 +111,24 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param
 }
 
 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
-		   struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
+		   struct mlx5e_channel_param *cparam,
+		   struct xsk_buff_pool *pool,
 		   struct mlx5e_channel *c)
 {
-	struct mlx5e_channel_param *cparam;
 	struct mlx5e_create_cq_param ccp;
 	int err;
 
 	mlx5e_build_create_cq_param(&ccp, c);
 
-	if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
+	if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev))
 		return -EINVAL;
 
-	cparam = kvzalloc_obj(*cparam);
-	if (!cparam)
-		return -ENOMEM;
-
-	mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
-
 	err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
 			    &c->xskrq.cq);
 	if (unlikely(err))
-		goto err_free_cparam;
+		return err;
 
-	err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
+	err = mlx5e_open_xsk_rq(c, params, cparam, pool);
 	if (unlikely(err))
 		goto err_close_rx_cq;
 
@@ -153,8 +147,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
 	if (unlikely(err))
 		goto err_close_tx_cq;
 
-	kvfree(cparam);
-
 	set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
 
 	return 0;
@@ -168,9 +160,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
 err_close_rx_cq:
 	mlx5e_close_cq(&c->xskrq.cq);
 
-err_free_cparam:
-	kvfree(cparam);
-
 	return err;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
index 50e111b85efdc..664ec78192c33 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
@@ -9,10 +9,12 @@
 struct mlx5e_xsk_param;
 
 bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
-			      struct mlx5e_xsk_param *xsk,
+			      struct mlx5e_rq_opt_param *rqo,
 			      struct mlx5_core_dev *mdev);
+struct mlx5e_channel_param;
 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
-		   struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
+		   struct mlx5e_channel_param *cparam,
+		   struct xsk_buff_pool *pool,
 		   struct mlx5e_channel *c);
 void mlx5e_close_xsk(struct mlx5e_channel *c);
 void mlx5e_activate_xsk(struct mlx5e_channel *c);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
index f8eaaf37963b1..abcbd38db9dbb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -287,6 +287,7 @@ struct mlx5e_ipsec_sa_entry {
 	struct mlx5e_ipsec_dwork *dwork;
 	struct mlx5e_ipsec_limits limits;
 	u32 rx_mapped_id;
+	u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)];
 };
 
 struct mlx5_accel_pol_xfrm_attrs {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
index 33344e00719b8..05faad5083d9d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
@@ -310,10 +310,11 @@ static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry,
 	mlx5e_ipsec_aso_query(sa_entry, data);
 }
 
-static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
-					 u32 mode_param)
+static void
+mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
+			     u32 mode_param,
+			     struct mlx5_accel_esp_xfrm_attrs *attrs)
 {
-	struct mlx5_accel_esp_xfrm_attrs attrs = {};
 	struct mlx5_wqe_aso_ctrl_seg data = {};
 
 	if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
@@ -323,18 +324,7 @@ static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
 		sa_entry->esn_state.overlap = 1;
 	}
 
-	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
-
-	/* It is safe to execute the modify below unlocked since the only flows
-	 * that could affect this HW object, are create, destroy and this work.
-	 *
-	 * Creation flow can't co-exist with this modify work, the destruction
-	 * flow would cancel this work, and this work is a single entity that
-	 * can't conflict with it self.
-	 */
-	spin_unlock_bh(&sa_entry->x->lock);
-	mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
-	spin_lock_bh(&sa_entry->x->lock);
+	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, attrs);
 
 	data.data_offset_condition_operand =
 		MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
@@ -370,20 +360,18 @@ static void mlx5e_ipsec_aso_update_soft(struct mlx5e_ipsec_sa_entry *sa_entry,
 static void mlx5e_ipsec_handle_limits(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
 	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
-	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-	struct mlx5e_ipsec_aso *aso = ipsec->aso;
 	bool soft_arm, hard_arm;
 	u64 hard_cnt;
 
 	lockdep_assert_held(&sa_entry->x->lock);
 
-	soft_arm = !MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm);
-	hard_arm = !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm);
+	soft_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, soft_lft_arm);
+	hard_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, hard_lft_arm);
 	if (!soft_arm && !hard_arm)
 		/* It is not lifetime event */
 		return;
 
-	hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
+	hard_cnt = MLX5_GET(ipsec_aso, sa_entry->ctx, remove_flow_pkt_cnt);
 	if (!hard_cnt || hard_arm) {
 		/* It is possible to see packet counter equal to zero without
 		 * hard limit event armed. Such situation can be if packet
@@ -453,11 +441,11 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work)
 	struct mlx5e_ipsec_work *work =
 		container_of(_work, struct mlx5e_ipsec_work, work);
 	struct mlx5e_ipsec_sa_entry *sa_entry = work->data;
+	struct mlx5_accel_esp_xfrm_attrs tmp = {};
 	struct mlx5_accel_esp_xfrm_attrs *attrs;
-	struct mlx5e_ipsec_aso *aso;
+	bool need_modify = false;
 	int ret;
 
-	aso = sa_entry->ipsec->aso;
 	attrs = &sa_entry->attrs;
 
 	spin_lock_bh(&sa_entry->x->lock);
@@ -465,18 +453,22 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work)
 	if (ret)
 		goto unlock;
 
+	if (attrs->lft.soft_packet_limit != XFRM_INF)
+		mlx5e_ipsec_handle_limits(sa_entry);
+
 	if (attrs->replay_esn.trigger &&
-	    !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
-		u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
+	    !MLX5_GET(ipsec_aso, sa_entry->ctx, esn_event_arm)) {
+		u32 mode_param = MLX5_GET(ipsec_aso, sa_entry->ctx,
+					  mode_parameter);
 
-		mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
+		mlx5e_ipsec_update_esn_state(sa_entry, mode_param, &tmp);
+		need_modify = true;
 	}
 
-	if (attrs->lft.soft_packet_limit != XFRM_INF)
-		mlx5e_ipsec_handle_limits(sa_entry);
-
 unlock:
 	spin_unlock_bh(&sa_entry->x->lock);
+	if (need_modify)
+		mlx5_accel_esp_modify_xfrm(sa_entry, &tmp);
 	kfree(work);
 }
 
@@ -629,6 +621,8 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
 			/* We are in atomic context */
 			udelay(10);
 	} while (ret && time_is_after_jiffies(expires));
+	if (!ret)
+		memcpy(sa_entry->ctx, aso->ctx, MLX5_ST_SZ_BYTES(ipsec_aso));
 	spin_unlock_bh(&aso->lock);
 	return ret;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 4a8dc85d5924c..bb61e21790788 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -371,6 +371,9 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
 	param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
 	param->rx_pending     = 1 << priv->channels.params.log_rq_mtu_frames;
 	param->tx_pending     = 1 << priv->channels.params.log_sq_size;
+
+	kernel_param->hds_thresh = 0;
+	kernel_param->hds_thresh_max = 0;
 }
 
 static void mlx5e_get_ringparam(struct net_device *dev,
@@ -2735,7 +2738,8 @@ const struct ethtool_ops mlx5e_ethtool_ops = {
 				     ETHTOOL_COALESCE_USE_ADAPTIVE |
 				     ETHTOOL_COALESCE_USE_CQE,
 	.supported_input_xfrm = RXH_XFRM_SYM_OR_XOR,
-	.supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT,
+	.supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT |
+				 ETHTOOL_RING_USE_HDS_THRS,
 	.get_drvinfo       = mlx5e_get_drvinfo,
 	.get_link          = ethtool_op_get_link,
 	.get_link_ext_state  = mlx5e_get_link_ext_state,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index 9352e21833124..55255fe6e415f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -47,7 +47,6 @@ struct mlx5e_flow_steering {
 	bool				state_destroy;
 	bool				vlan_strip_disable;
 	struct mlx5_core_dev		*mdev;
-	struct net_device		*netdev;
 	struct mlx5_flow_namespace      *ns;
 	struct mlx5_flow_namespace      *egress_ns;
 #ifdef CONFIG_MLX5_EN_RXNFC
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index b6c12460b54a9..f7009da94f0b4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -636,14 +636,18 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work)
 
 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 {
-	rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
+	/* xsk can have page_shift < PAGE_SHIFT */
+	u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
+	u32 page_size = BIT(PAGE_SHIFT + page_order);
+
+	rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order);
 	if (!rq->wqe_overflow.page)
 		return -ENOMEM;
 
 	rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
-					     PAGE_SIZE, rq->buff.map_dir);
+					     page_size, rq->buff.map_dir);
 	if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
-		__free_page(rq->wqe_overflow.page);
+		__free_pages(rq->wqe_overflow.page, page_order);
 		return -ENOMEM;
 	}
 	return 0;
@@ -651,9 +655,12 @@ static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 
 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 {
-	 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
-			rq->buff.map_dir);
-	 __free_page(rq->wqe_overflow.page);
+	u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
+	u32 page_size = BIT(PAGE_SHIFT + page_order);
+
+	dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size,
+		       rq->buff.map_dir);
+	 __free_pages(rq->wqe_overflow.page, page_order);
 }
 
 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
@@ -780,7 +787,7 @@ static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev,
 
 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
 				struct mlx5e_params *params,
-				struct mlx5e_rq_param *rqp,
+				struct mlx5e_rq_param *rq_param,
 				struct mlx5e_rq *rq,
 				int node)
 {
@@ -791,7 +798,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
 	if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 		return 0;
 
-	hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp);
+	hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param);
 	hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE);
 	nentries = hd_buf_size / PAGE_SIZE;
 	if (!nentries) {
@@ -851,19 +858,19 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
 }
 
 static int mlx5e_alloc_rq(struct mlx5e_params *params,
-			  struct mlx5e_xsk_param *xsk,
-			  struct mlx5e_rq_param *rqp,
+			  struct mlx5e_rq_param *rq_param,
+			  struct mlx5e_rq_opt_param *rqo,
 			  int node, struct mlx5e_rq *rq)
 {
+	void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
 	struct mlx5_core_dev *mdev = rq->mdev;
-	void *rqc = rqp->rqc;
-	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
+	u32 pool_order = 0;
 	u32 pool_size;
 	int wq_sz;
 	int err;
 	int i;
 
-	rqp->wq.db_numa_node = node;
+	rq_param->wq.db_numa_node = node;
 	INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
 	INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work);
 
@@ -872,28 +879,28 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 	RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
 
 	rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
-	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
+	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo);
 	pool_size = 1 << params->log_rq_mtu_frames;
 
 	rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
-					&rq->wq_ctrl);
+		err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq,
+					&rq->mpwqe.wq, &rq->wq_ctrl);
 		if (err)
 			goto err_rq_xdp_prog;
 
-		err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
-		if (err)
-			goto err_rq_wq_destroy;
-
 		rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
 
 		wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 
-		rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
-		rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+		rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
+		err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
+		if (err)
+			goto err_rq_wq_destroy;
+
+		rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
 		rq->mpwqe.pages_per_wqe =
 			mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
 						  rq->mpwqe.umr_mode);
@@ -905,14 +912,18 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 						 rq->mpwqe.umr_mode);
 
 		pool_size = rq->mpwqe.pages_per_wqe <<
-			mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
+			mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo);
+		pool_order = rq->mpwqe.page_shift - PAGE_SHIFT;
 
-		if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
+		if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) &&
+		    params->xdp_prog)
 			pool_size *= 2; /* additional page per packet for the linear part */
 
-		rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
+		rq->mpwqe.log_stride_sz =
+				mlx5e_mpwqe_get_log_stride_size(mdev, params,
+								rqo);
 		rq->mpwqe.num_strides =
-			BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
+			BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo));
 		rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
 
 		rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
@@ -925,14 +936,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		if (err)
 			goto err_rq_mkey;
 
-		err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node);
+		err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node);
 		if (err)
 			goto err_free_mpwqe_info;
 
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
-					 &rq->wq_ctrl);
+		err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq,
+					 &rq->wqe.wq, &rq->wq_ctrl);
 		if (err)
 			goto err_rq_xdp_prog;
 
@@ -940,7 +951,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 
 		wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 
-		rq->wqe.info = rqp->frags_info;
+		rq->wqe.info = rq_param->frags_info;
 		rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 
 		err = mlx5e_init_wqe_alloc_info(rq, node);
@@ -948,7 +959,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 			goto err_rq_wq_destroy;
 	}
 
-	if (xsk) {
+	if (mlx5e_rqo_xsk_param(rqo)) {
 		err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 						 MEM_TYPE_XSK_BUFF_POOL, NULL);
 		if (err)
@@ -958,7 +969,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		/* Create a page_pool and register it with rxq */
 		struct page_pool_params pp_params = { 0 };
 
-		pp_params.order     = 0;
+		if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 >
+			    MLX5E_PAGECNT_BIAS_MAX)) {
+			err = -E2BIG;
+			goto err_free_by_rq_type;
+		}
+
+		pp_params.order     = pool_order;
 		pp_params.flags     = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
 		pp_params.pool_size = pool_size;
 		pp_params.nid       = node;
@@ -966,7 +983,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		pp_params.napi      = rq->cq.napi;
 		pp_params.netdev    = rq->netdev;
 		pp_params.dma_dir   = rq->buff.map_dir;
-		pp_params.max_len   = PAGE_SIZE;
+		pp_params.max_len   = BIT(PAGE_SHIFT + pool_order);
 		pp_params.queue_idx = rq->ix;
 
 		/* Shampo header data split allow for unreadable netmem */
@@ -1085,7 +1102,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
 	xdp_rxq_info_unreg(&rq->xdp_rxq);
 }
 
-int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter)
+int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
+		    u16 q_counter)
 {
 	struct mlx5_core_dev *mdev = rq->mdev;
 	u8 ts_format;
@@ -1107,7 +1125,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou
 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 	wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 
-	memcpy(rqc, param->rqc, sizeof(param->rqc));
+	memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc));
 
 	MLX5_SET(rqc,  rqc, cqn,		rq->cq.mcq.cqn);
 	MLX5_SET(rqc,  rqc, state,		MLX5_RQC_STATE_RST);
@@ -1323,8 +1341,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 
 }
 
-int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
-		  struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
+int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
+		  struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
 		  struct mlx5e_rq *rq)
 {
 	struct mlx5_core_dev *mdev = rq->mdev;
@@ -1333,11 +1351,11 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
 		__set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
 
-	err = mlx5e_alloc_rq(params, xsk, param, node, rq);
+	err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq);
 	if (err)
 		return err;
 
-	err = mlx5e_create_rq(rq, param, q_counter);
+	err = mlx5e_create_rq(rq, rq_param, q_counter);
 	if (err)
 		goto err_free_rq;
 
@@ -2506,17 +2524,20 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
 	return err;
 }
 
-static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
-			     struct mlx5e_rq_param *rq_params)
+static int mlx5e_open_rxq_rq(struct mlx5e_channel *c,
+			     struct mlx5e_params *params,
+			     struct mlx5e_rq_param *rq_param,
+			     struct mlx5e_rq_opt_param *rqo)
 {
 	u16 q_counter = c->priv->q_counter[c->sd_ix];
 	int err;
 
-	err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq);
+	err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq);
 	if (err)
 		return err;
 
-	return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq);
+	return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
+			     q_counter, &c->rq);
 }
 
 static struct mlx5e_icosq *
@@ -2619,7 +2640,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
 	if (err)
 		goto err_close_icosq;
 
-	err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
+	err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt);
 	if (err)
 		goto err_close_sqs;
 
@@ -2764,6 +2785,7 @@ static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c)
 
 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 			      struct mlx5e_params *params,
+			      struct netdev_queue_config *qcfg,
 			      struct xsk_buff_pool *xsk_pool,
 			      struct mlx5e_channel **cp)
 {
@@ -2797,7 +2819,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 		goto err_free;
 	}
 
-	err = mlx5e_build_channel_param(mdev, params, cparam);
+	err = mlx5e_build_channel_param(mdev, params, qcfg, cparam);
 	if (err)
 		goto err_free;
 
@@ -2828,7 +2850,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 
 	if (xsk_pool) {
 		mlx5e_build_xsk_param(xsk_pool, &xsk);
-		err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
+		mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
+		err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c);
 		if (unlikely(err))
 			goto err_close_queues;
 	}
@@ -2921,7 +2944,8 @@ int mlx5e_open_channels(struct mlx5e_priv *priv,
 		if (chs->params.xdp_prog)
 			xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
 
-		err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]);
+		err = mlx5e_open_channel(priv, i, &chs->params, NULL,
+					 xsk_pool, &chs->c[i]);
 		if (err)
 			goto err_close_channels;
 	}
@@ -3577,15 +3601,14 @@ static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
 
 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
 			       struct mlx5e_rq *rq,
-			       struct mlx5e_rq_param *param)
+			       struct mlx5e_rq_param *rq_param)
 {
-	void *rqc = param->rqc;
-	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
+	void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
 	int err;
 
-	param->wq.db_numa_node = param->wq.buf_numa_node;
+	rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node;
 
-	err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
+	err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq,
 				 &rq->wq_ctrl);
 	if (err)
 		return err;
@@ -4586,6 +4609,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
 	for (ix = 0; ix < chs->params.num_channels; ix++) {
 		struct xsk_buff_pool *xsk_pool =
 			mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
+		struct mlx5e_rq_opt_param rqo = {0};
 		struct mlx5e_xsk_param xsk;
 		int max_xdp_mtu;
 
@@ -4593,12 +4617,13 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
 			continue;
 
 		mlx5e_build_xsk_param(xsk_pool, &xsk);
-		max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk);
+		rqo.xsk = &xsk;
+		max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo);
 
 		/* Validate XSK params and XDP MTU in advance */
-		if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) ||
+		if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) ||
 		    new_params->sw_mtu > max_xdp_mtu) {
-			u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
+			u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo);
 			int max_mtu_frame, max_mtu_page, max_mtu;
 
 			/* Two criteria must be met:
@@ -5549,9 +5574,52 @@ static const struct netdev_stat_ops mlx5e_stat_ops = {
 
 struct mlx5_qmgmt_data {
 	struct mlx5e_channel *c;
-	struct mlx5e_channel_param cparam;
 };
 
+static void mlx5e_queue_default_qcfg(struct net_device *dev,
+				     struct netdev_queue_config *qcfg)
+{
+	qcfg->rx_page_size = PAGE_SIZE;
+}
+
+static int mlx5e_queue_validate_qcfg(struct net_device *dev,
+				     struct netdev_queue_config *qcfg,
+				     struct netlink_ext_ack *extack)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 max;
+
+	if (!is_power_of_2(qcfg->rx_page_size)) {
+		netdev_err(priv->netdev, "rx_page_size not power of 2: %u",
+			   qcfg->rx_page_size);
+		return -EINVAL;
+	}
+
+	max = mlx5e_mpwrq_max_page_size(mdev);
+	if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) {
+		netdev_err(priv->netdev,
+			   "Selected rx_page_size %u not in supported range [%lu, %u]\n",
+			   qcfg->rx_page_size, PAGE_SIZE, max);
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static bool mlx5e_queue_validate_page_size(struct net_device *dev,
+					   struct netdev_queue_config *qcfg,
+					   int queue_index)
+{
+	if (qcfg->rx_page_size == PAGE_SIZE)
+		return true;
+
+	if (!netif_rxq_has_unreadable_mp(dev, queue_index))
+		return false;
+
+	return true;
+}
+
 static int mlx5e_queue_mem_alloc(struct net_device *dev,
 				 struct netdev_queue_config *qcfg,
 				 void *newq, int queue_index)
@@ -5560,7 +5628,6 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_channels *chs = &priv->channels;
 	struct mlx5e_params params = chs->params;
-	struct mlx5_core_dev *mdev;
 	int err;
 
 	mutex_lock(&priv->state_lock);
@@ -5584,12 +5651,14 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
 		goto unlock;
 	}
 
-	mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index);
-	err = mlx5e_build_channel_param(mdev, &params, &new->cparam);
-	if (err)
+	if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) {
+		netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n");
+		err = -EINVAL;
 		goto unlock;
+	}
 
-	err = mlx5e_open_channel(priv, queue_index, &params, NULL, &new->c);
+	err = mlx5e_open_channel(priv, queue_index, &params, qcfg, NULL,
+				 &new->c);
 unlock:
 	mutex_unlock(&priv->state_lock);
 	return err;
@@ -5668,6 +5737,9 @@ static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = {
 	.ndo_queue_start	=	mlx5e_queue_start,
 	.ndo_queue_stop		=	mlx5e_queue_stop,
 	.ndo_queue_get_dma_dev	=	mlx5e_queue_get_dma_dev,
+	.ndo_default_qcfg       =	mlx5e_queue_default_qcfg,
+	.ndo_validate_qcfg	=	mlx5e_queue_validate_qcfg,
+	.supported_params       =	QCFG_RX_PAGE_SIZE,
 };
 
 static void mlx5e_build_nic_netdev(struct net_device *netdev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 268e208847577..f5c0e2a0ada9a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -272,8 +272,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
 	return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem);
 }
 
-#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64)
-
 static int mlx5e_page_alloc_fragmented(struct page_pool *pp,
 				       struct mlx5e_frag_page *frag_page)
 {
@@ -1844,11 +1842,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
 			   struct mlx5e_frag_page *frag_page,
 			   u32 data_bcnt, u32 data_offset)
 {
+	u32 page_size = BIT(rq->mpwqe.page_shift);
+
 	net_prefetchw(skb->data);
 
 	do {
 		/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
-		u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt);
+		u32 pg_consumed_bytes = min_t(u32, page_size - data_offset,
+					      data_bcnt);
 		unsigned int truesize = pg_consumed_bytes;
 
 		mlx5e_add_skb_frag(rq, skb, frag_page, data_offset,
@@ -1869,6 +1870,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 	u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
 	struct mlx5e_frag_page *head_page = frag_page;
 	struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf;
+	u32 page_size = BIT(rq->mpwqe.page_shift);
 	u32 frag_offset    = head_offset;
 	u32 byte_cnt       = cqe_bcnt;
 	struct skb_shared_info *sinfo;
@@ -1923,9 +1925,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 		linear_hr = skb_headroom(skb);
 		linear_data_len = headlen;
 		linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb));
-		if (unlikely(frag_offset >= PAGE_SIZE)) {
+		if (unlikely(frag_offset >= page_size)) {
 			frag_page++;
-			frag_offset -= PAGE_SIZE;
+			frag_offset -= page_size;
 		}
 	}
 
@@ -1937,7 +1939,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 	while (byte_cnt) {
 		/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
 		pg_consumed_bytes =
-			min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
+			min_t(u32, page_size - frag_offset, byte_cnt);
 
 		if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
 			truesize += pg_consumed_bytes;
@@ -1974,7 +1976,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 		new_nr_frags = sinfo->nr_frags;
 		nr_frags_free = old_nr_frags - new_nr_frags;
 		if (unlikely(nr_frags_free))
-			truesize -= (nr_frags_free - 1) * PAGE_SIZE +
+			truesize -= (nr_frags_free - 1) * page_size +
 				ALIGN(pg_consumed_bytes,
 				      BIT(rq->mpwqe.log_stride_sz));
 
@@ -2161,15 +2163,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
 	rq->hw_gro_data->skb = NULL;
 }
 
-static bool
-mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
+static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb,
+					      u16 data_bcnt,
+					      u32 page_size)
 {
 	int nr_frags = skb_shinfo(skb)->nr_frags;
 
-	if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE)
+	if (page_size >= GRO_LEGACY_MAX_SIZE)
 		return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE;
 	else
-		return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
+		return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
 }
 
 static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
@@ -2178,18 +2181,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 	u16 header_index	= mlx5e_shampo_get_cqe_header_index(rq, cqe);
 	u32 wqe_offset		= be32_to_cpu(cqe->shampo.data_offset);
 	u16 cstrides		= mpwrq_get_cqe_consumed_strides(cqe);
-	u32 data_offset		= wqe_offset & (PAGE_SIZE - 1);
 	u32 cqe_bcnt		= mpwrq_get_cqe_byte_cnt(cqe);
 	u16 wqe_id		= be16_to_cpu(cqe->wqe_id);
-	u32 page_idx		= wqe_offset >> PAGE_SHIFT;
 	u16 head_size		= cqe->shampo.header_size;
 	struct sk_buff **skb	= &rq->hw_gro_data->skb;
 	bool flush		= cqe->shampo.flush;
 	bool match		= cqe->shampo.match;
+	u32 page_size = BIT(rq->mpwqe.page_shift);
 	struct mlx5e_rq_stats *stats = rq->stats;
 	struct mlx5e_rx_wqe_ll *wqe;
 	struct mlx5e_mpw_info *wi;
 	struct mlx5_wq_ll *wq;
+	u32 data_offset;
+	u32 page_idx;
 
 	wi = mlx5e_get_mpw_info(rq, wqe_id);
 	wi->consumed_strides += cstrides;
@@ -2205,7 +2209,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 		goto mpwrq_cqe_out;
 	}
 
-	if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) {
+	data_offset = wqe_offset & (page_size - 1);
+	page_idx = wqe_offset >> rq->mpwqe.page_shift;
+	if (*skb &&
+	    !(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt,
+							 page_size))) {
 		match = false;
 		mlx5e_shampo_flush_skb(rq, cqe, match);
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index a8af84fc97638..1a3ecf073913c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -916,11 +916,30 @@ static int mlx5e_stats_get_ieee(struct mlx5_core_dev *mdev,
 				    sz, MLX5_REG_PPCNT, 0, 0);
 }
 
+static int mlx5e_stats_get_per_prio(struct mlx5_core_dev *mdev,
+				    u32 *ppcnt_per_prio, int prio)
+{
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+
+	if (!(MLX5_CAP_PCAM_FEATURE(mdev, pfcc_mask) &&
+	      MLX5_CAP_DEBUG(mdev, stall_detect)))
+		return -EOPNOTSUPP;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
+	MLX5_SET(ppcnt_reg, in, prio_tc, prio);
+	return mlx5_core_access_reg(mdev, in, sz, ppcnt_per_prio, sz,
+				    MLX5_REG_PPCNT, 0, 0);
+}
+
 void mlx5e_stats_pause_get(struct mlx5e_priv *priv,
 			   struct ethtool_pause_stats *pause_stats)
 {
 	u32 ppcnt_ieee_802_3[MLX5_ST_SZ_DW(ppcnt_reg)];
 	struct mlx5_core_dev *mdev = priv->mdev;
+	u64 ps_stats = 0;
+	int prio;
 
 	if (mlx5e_stats_get_ieee(mdev, ppcnt_ieee_802_3))
 		return;
@@ -933,6 +952,17 @@ void mlx5e_stats_pause_get(struct mlx5e_priv *priv,
 		MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3,
 				      eth_802_3_cntrs_grp_data_layout,
 				      a_pause_mac_ctrl_frames_received);
+
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
+		if (mlx5e_stats_get_per_prio(mdev, ppcnt_ieee_802_3, prio))
+			return;
+
+		ps_stats += MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3,
+						  eth_per_prio_grp_data_layout,
+						  device_stall_critical_watermark_cnt);
+	}
+
+	pause_stats->tx_pause_storm_events = ps_stats;
 }
 
 void mlx5e_stats_eth_phy_get(struct mlx5e_priv *priv,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 1434b65d47465..397a93584fd66 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -35,6 +35,7 @@
 #include <net/sch_generic.h>
 #include <net/pkt_cls.h>
 #include <linux/mlx5/fs.h>
+#include <linux/mlx5/lag.h>
 #include <linux/mlx5/device.h>
 #include <linux/rhashtable.h>
 #include <linux/refcount.h>
@@ -2131,7 +2132,7 @@ static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *flow,
 	mutex_unlock(&esw->offloads.peer_mutex);
 
 	list_for_each_entry_safe(peer_flow, tmp, &flow->peer_flows, peer_flows) {
-		if (peer_index != mlx5_get_dev_index(peer_flow->priv->mdev))
+		if (peer_index != mlx5_lag_get_dev_seq(peer_flow->priv->mdev))
 			continue;
 
 		list_del(&peer_flow->peer_flows);
@@ -2154,7 +2155,7 @@ static void mlx5e_tc_del_fdb_peers_flow(struct mlx5e_tc_flow *flow)
 
 	devcom = flow->priv->mdev->priv.eswitch->devcom;
 	mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) {
-		i = mlx5_get_dev_index(peer_esw->dev);
+		i = mlx5_lag_get_dev_seq(peer_esw->dev);
 		mlx5e_tc_del_fdb_peer_flow(flow, i);
 	}
 }
@@ -4584,7 +4585,7 @@ static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f,
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr;
 	struct mlx5e_tc_flow_parse_attr *parse_attr;
-	int i = mlx5_get_dev_index(peer_esw->dev);
+	int i = mlx5_lag_get_dev_seq(peer_esw->dev);
 	struct mlx5e_rep_priv *peer_urpriv;
 	struct mlx5e_tc_flow *peer_flow;
 	struct mlx5_core_dev *in_mdev;
@@ -5525,7 +5526,7 @@ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw)
 	devcom = esw->devcom;
 
 	mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) {
-		i = mlx5_get_dev_index(peer_esw->dev);
+		i = mlx5_lag_get_dev_seq(peer_esw->dev);
 		list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i])
 			mlx5e_tc_del_fdb_peers_flow(flow);
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
index 26178d0bac925..faccc60fc93a8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
@@ -1489,24 +1489,24 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node,
 	return err;
 }
 
-static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev)
+static u32 mlx5_esw_qos_lag_link_speed_get(struct mlx5_core_dev *mdev,
+					   bool take_rtnl)
 {
 	struct ethtool_link_ksettings lksettings;
 	struct net_device *slave, *master;
 	u32 speed = SPEED_UNKNOWN;
 
-	/* Lock ensures a stable reference to master and slave netdevice
-	 * while port speed of master is queried.
-	 */
-	ASSERT_RTNL();
-
 	slave = mlx5_uplink_netdev_get(mdev);
 	if (!slave)
 		goto out;
 
+	if (take_rtnl)
+		rtnl_lock();
 	master = netdev_master_upper_dev_get(slave);
 	if (master && !__ethtool_get_link_ksettings(master, &lksettings))
 		speed = lksettings.base.speed;
+	if (take_rtnl)
+		rtnl_unlock();
 
 out:
 	mlx5_uplink_netdev_put(mdev, slave);
@@ -1514,20 +1514,15 @@ static u32 mlx5_esw_qos_lag_link_speed_get_locked(struct mlx5_core_dev *mdev)
 }
 
 static int mlx5_esw_qos_max_link_speed_get(struct mlx5_core_dev *mdev, u32 *link_speed_max,
-					   bool hold_rtnl_lock, struct netlink_ext_ack *extack)
+					   bool take_rtnl,
+					   struct netlink_ext_ack *extack)
 {
 	int err;
 
 	if (!mlx5_lag_is_active(mdev))
 		goto skip_lag;
 
-	if (hold_rtnl_lock)
-		rtnl_lock();
-
-	*link_speed_max = mlx5_esw_qos_lag_link_speed_get_locked(mdev);
-
-	if (hold_rtnl_lock)
-		rtnl_unlock();
+	*link_speed_max = mlx5_esw_qos_lag_link_speed_get(mdev, take_rtnl);
 
 	if (*link_speed_max != (u32)SPEED_UNKNOWN)
 		return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index c2563bee74dfe..5128f5020daec 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -273,7 +273,7 @@ struct mlx5_eswitch_fdb {
 			struct mlx5_flow_group *send_to_vport_grp;
 			struct mlx5_flow_group *send_to_vport_meta_grp;
 			struct mlx5_flow_group *peer_miss_grp;
-			struct mlx5_flow_handle **peer_miss_rules[MLX5_MAX_PORTS];
+			struct xarray peer_miss_rules;
 			struct mlx5_flow_group *miss_grp;
 			struct mlx5_flow_handle **send_to_vport_meta_rules;
 			struct mlx5_flow_handle *miss_rule_uni;
@@ -942,6 +942,12 @@ int mlx5_esw_ipsec_vf_packet_offload_supported(struct mlx5_core_dev *dev,
 					       u16 vport_num);
 bool mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev);
 void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev);
+struct mlx5_flow_group *
+mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw,
+			     struct mlx5_flow_table *ft);
+struct mlx5_flow_handle *
+mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num,
+			       struct mlx5_flow_table *lag_ft);
 #else  /* CONFIG_MLX5_ESWITCH */
 /* eswitch API stubs */
 static inline int  mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; }
@@ -1027,6 +1033,12 @@ mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id)
 
 static inline void
 mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev) {}
+static inline struct mlx5_flow_handle *
+mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num,
+			       struct mlx5_flow_table *lag_ft)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
 
 #endif /* CONFIG_MLX5_ESWITCH */
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 7a9ee36b8dcaa..bcde267df8eb5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -1190,7 +1190,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 	struct mlx5_flow_handle *flow;
 	struct mlx5_vport *peer_vport;
 	struct mlx5_flow_spec *spec;
-	int err, pfindex;
+	int err;
 	unsigned long i;
 	void *misc;
 
@@ -1270,14 +1270,10 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 		}
 	}
 
-	pfindex = mlx5_get_dev_index(peer_dev);
-	if (pfindex >= MLX5_MAX_PORTS) {
-		esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
-			 pfindex, MLX5_MAX_PORTS);
-		err = -EINVAL;
+	err = xa_insert(&esw->fdb_table.offloads.peer_miss_rules,
+			MLX5_CAP_GEN(peer_dev, vhca_id), flows, GFP_KERNEL);
+	if (err)
 		goto add_ec_vf_flow_err;
-	}
-	esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
 
 	kvfree(spec);
 	return 0;
@@ -1319,12 +1315,13 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 					struct mlx5_core_dev *peer_dev)
 {
 	struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch;
-	u16 peer_index = mlx5_get_dev_index(peer_dev);
+	u16 peer_vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
 	struct mlx5_flow_handle **flows;
 	struct mlx5_vport *peer_vport;
 	unsigned long i;
 
-	flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
+	flows = xa_erase(&esw->fdb_table.offloads.peer_miss_rules,
+			 peer_vhca_id);
 	if (!flows)
 		return;
 
@@ -1350,7 +1347,6 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 	}
 
 	kvfree(flows);
-	esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
 }
 
 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
@@ -1460,6 +1456,83 @@ esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
 	return flow_rule;
 }
 
+struct mlx5_flow_group *
+mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw,
+			     struct mlx5_flow_table *ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *fg;
+	void *match_criteria;
+	void *flow_group_in;
+
+	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (IS_ERR(ft))
+		return ERR_CAST(ft);
+
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!flow_group_in)
+		return ERR_PTR(-ENOMEM);
+
+	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
+				      match_criteria);
+	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
+		 MLX5_MATCH_MISC_PARAMETERS_2);
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
+		 ft->max_fte - 1);
+
+	MLX5_SET(fte_match_param, match_criteria,
+		 misc_parameters_2.metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_mask());
+
+	fg = mlx5_create_flow_group(ft, flow_group_in);
+	kvfree(flow_group_in);
+	if (IS_ERR(fg))
+		esw_warn(esw->dev, "Can't create LAG demux flow group\n");
+
+	return fg;
+}
+
+struct mlx5_flow_handle *
+mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num,
+			       struct mlx5_flow_table *lag_ft)
+{
+	struct mlx5_flow_spec *spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *ret;
+	void *misc;
+
+	if (!spec)
+		return ERR_PTR(-ENOMEM);
+
+	if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
+		kvfree(spec);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+			    misc_parameters_2);
+	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_mask());
+	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
+
+	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+			    misc_parameters_2);
+	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_VHCA_RX;
+	dest.vhca.id = MLX5_CAP_GEN(esw->dev, vhca_id);
+
+	ret = mlx5_add_flow_rules(lag_ft, spec, &flow_act, &dest, 1);
+	kvfree(spec);
+	return ret;
+}
+
 #define MAX_PF_SQ 256
 #define MAX_SQ_NVPORTS 32
 
@@ -2048,7 +2121,8 @@ static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
 
 	if (IS_ERR(g)) {
 		err = PTR_ERR(g);
-		mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
+		esw_warn(esw->dev, "Failed to create vport rx group err %d\n",
+			 err);
 		goto out;
 	}
 
@@ -2093,7 +2167,8 @@ static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
 
 	if (IS_ERR(g)) {
 		err = PTR_ERR(g);
-		mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
+		esw_warn(esw->dev,
+			 "Failed to create vport rx drop group err %d\n", err);
 		goto out;
 	}
 
@@ -3247,6 +3322,7 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw,
 		return;
 
 	xa_init(&esw->paired);
+	xa_init(&esw->fdb_table.offloads.peer_miss_rules);
 	esw->num_peers = 0;
 	esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc,
 						     MLX5_DEVCOM_ESW_OFFLOADS,
@@ -3274,6 +3350,7 @@ void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
 
 	mlx5_devcom_unregister_component(esw->devcom);
 	xa_destroy(&esw->paired);
+	xa_destroy(&esw->fdb_table.offloads.peer_miss_rules);
 	esw->devcom = NULL;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index c348ee62cd3af..1cd4cd898ec2f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -716,6 +716,10 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 				id = dst->dest_attr.ft->id;
 				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE;
 				break;
+			case MLX5_FLOW_DESTINATION_TYPE_VHCA_RX:
+				id = dst->dest_attr.vhca.id;
+				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VHCA_RX;
+				break;
 			default:
 				id = dst->dest_attr.tir_num;
 				ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR;
@@ -1183,7 +1187,7 @@ int mlx5_fs_cmd_set_l2table_entry_silent(struct mlx5_core_dev *dev, u8 silent_mo
 {
 	u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)] = {};
 
-	if (silent_mode && !MLX5_CAP_GEN(dev, silent_mode))
+	if (silent_mode && !MLX5_CAP_GEN(dev, silent_mode_set))
 		return -EOPNOTSUPP;
 
 	MLX5_SET(set_l2_table_entry_in, in, opcode, MLX5_CMD_OP_SET_L2_TABLE_ENTRY);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index 2c3544880a306..61a6ba1e49dda 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -503,7 +503,8 @@ static bool is_fwd_dest_type(enum mlx5_flow_destination_type type)
 		type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER ||
 		type == MLX5_FLOW_DESTINATION_TYPE_TIR ||
 		type == MLX5_FLOW_DESTINATION_TYPE_RANGE ||
-		type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+		type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE ||
+		type == MLX5_FLOW_DESTINATION_TYPE_VHCA_RX;
 }
 
 static bool check_valid_spec(const struct mlx5_flow_spec *spec)
@@ -1437,15 +1438,9 @@ mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
 
 struct mlx5_flow_table*
 mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
-				 int prio, u32 level)
+				 struct mlx5_flow_table_attr *ft_attr)
 {
-	struct mlx5_flow_table_attr ft_attr = {};
-
-	ft_attr.level = level;
-	ft_attr.prio  = prio;
-	ft_attr.max_fte = 1;
-
-	return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
+	return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
 }
 EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
 
@@ -1890,7 +1885,9 @@ static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
 		     d1->range.hit_ft == d2->range.hit_ft &&
 		     d1->range.miss_ft == d2->range.miss_ft &&
 		     d1->range.min == d2->range.min &&
-		     d1->range.max == d2->range.max))
+		     d1->range.max == d2->range.max) ||
+		    (d1->type == MLX5_FLOW_DESTINATION_TYPE_VHCA_RX &&
+		     d1->vhca.id == d2->vhca.id))
 			return true;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index eeb4437975f20..55249f4058416 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -255,6 +255,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 			return err;
 	}
 
+	if (MLX5_CAP_GEN(dev, tlp_device_emulation_manager)) {
+		err = mlx5_core_get_caps_mode(dev, MLX5_CAP_TLP_EMULATION, HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+	}
+
 	if (MLX5_CAP_GEN(dev, ipsec_offload)) {
 		err = mlx5_core_get_caps_mode(dev, MLX5_CAP_IPSEC, HCA_CAP_OPMOD_GET_CUR);
 		if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
index 62b6faa4276aa..37de4be0e6204 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
@@ -145,7 +145,8 @@ static int members_show(struct seq_file *file, void *priv)
 	ldev = mlx5_lag_dev(dev);
 	mutex_lock(&ldev->lock);
 	mlx5_ldev_for_each(i, 0, ldev)
-		seq_printf(file, "%s\n", dev_name(ldev->pf[i].dev->device));
+		seq_printf(file, "%s\n",
+			   dev_name(mlx5_lag_pf(ldev, i)->dev->device));
 	mutex_unlock(&ldev->lock);
 
 	return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 044adfdf9aa26..449e4bd86c064 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -35,6 +35,7 @@
 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/eswitch.h>
 #include <linux/mlx5/vport.h>
+#include <linux/mlx5/lag.h>
 #include "lib/mlx5.h"
 #include "lib/devcom.h"
 #include "mlx5_core.h"
@@ -232,6 +233,7 @@ static void mlx5_do_bond_work(struct work_struct *work);
 static void mlx5_ldev_free(struct kref *ref)
 {
 	struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref);
+	struct lag_func *pf;
 	struct net *net;
 	int i;
 
@@ -241,13 +243,16 @@ static void mlx5_ldev_free(struct kref *ref)
 	}
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (ldev->pf[i].dev &&
-		    ldev->pf[i].port_change_nb.nb.notifier_call) {
-			struct mlx5_nb *nb = &ldev->pf[i].port_change_nb;
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->port_change_nb.nb.notifier_call) {
+			struct mlx5_nb *nb = &pf->port_change_nb;
 
-			mlx5_eq_notifier_unregister(ldev->pf[i].dev, nb);
+			mlx5_eq_notifier_unregister(pf->dev, nb);
 		}
+		xa_erase(&ldev->pfs, i);
+		kfree(pf);
 	}
+	xa_destroy(&ldev->pfs);
 
 	mlx5_lag_mp_cleanup(ldev);
 	cancel_delayed_work_sync(&ldev->bond_work);
@@ -284,6 +289,7 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
 
 	kref_init(&ldev->ref);
 	mutex_init(&ldev->lock);
+	xa_init_flags(&ldev->pfs, XA_FLAGS_ALLOC);
 	INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
 	INIT_WORK(&ldev->speed_update_work, mlx5_mpesw_speed_update_work);
 
@@ -309,23 +315,54 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
 int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
 				struct net_device *ndev)
 {
+	struct lag_func *pf;
 	int i;
 
-	mlx5_ldev_for_each(i, 0, ldev)
-		if (ldev->pf[i].netdev == ndev)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->netdev == ndev)
 			return i;
+	}
 
 	return -ENOENT;
 }
 
+static int mlx5_lag_get_master_idx(struct mlx5_lag *ldev)
+{
+	unsigned long idx = 0;
+	void *entry;
+
+	if (!ldev)
+		return -ENOENT;
+
+	entry = xa_find(&ldev->pfs, &idx, U8_MAX, MLX5_LAG_XA_MARK_MASTER);
+	if (!entry)
+		return -ENOENT;
+
+	return (int)idx;
+}
+
 int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq)
 {
-	int i, num = 0;
+	int master_idx, i, num = 0;
 
 	if (!ldev)
 		return -ENOENT;
 
+	master_idx = mlx5_lag_get_master_idx(ldev);
+
+	/* If seq 0 is requested and there's a primary PF, return it */
+	if (master_idx >= 0) {
+		if (seq == 0)
+			return master_idx;
+		num++;
+	}
+
 	mlx5_ldev_for_each(i, 0, ldev) {
+		/* Skip the primary PF in the loop */
+		if (i == master_idx)
+			continue;
+
 		if (num == seq)
 			return i;
 		num++;
@@ -333,6 +370,108 @@ int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq)
 	return -ENOENT;
 }
 
+/* Reverse of mlx5_lag_get_dev_index_by_seq: given a device, return its
+ * sequence number in the LAG. Master is always 0, others numbered
+ * sequentially starting from 1.
+ */
+int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev)
+{
+	struct mlx5_lag *ldev = mlx5_lag_dev(dev);
+	int master_idx, i, num = 1;
+	struct lag_func *pf;
+
+	if (!ldev)
+		return -ENOENT;
+
+	master_idx = mlx5_lag_get_master_idx(ldev);
+	if (master_idx < 0)
+		return -ENOENT;
+
+	pf = mlx5_lag_pf(ldev, master_idx);
+	if (pf && pf->dev == dev)
+		return 0;
+
+	mlx5_ldev_for_each(i, 0, ldev) {
+		if (i == master_idx)
+			continue;
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->dev == dev)
+			return num;
+		num++;
+	}
+	return -ENOENT;
+}
+EXPORT_SYMBOL(mlx5_lag_get_dev_seq);
+
+/* Devcom events for LAG master marking */
+#define LAG_DEVCOM_PAIR		(0)
+#define LAG_DEVCOM_UNPAIR	(1)
+
+static void mlx5_lag_mark_master(struct mlx5_lag *ldev)
+{
+	int lowest_dev_idx = INT_MAX;
+	struct lag_func *pf;
+	int master_xa_idx = -1;
+	int dev_idx;
+	int i;
+
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		dev_idx = mlx5_get_dev_index(pf->dev);
+		if (dev_idx < lowest_dev_idx) {
+			lowest_dev_idx = dev_idx;
+			master_xa_idx = i;
+		}
+	}
+
+	if (master_xa_idx >= 0)
+		xa_set_mark(&ldev->pfs, master_xa_idx, MLX5_LAG_XA_MARK_MASTER);
+}
+
+static void mlx5_lag_clear_master(struct mlx5_lag *ldev)
+{
+	unsigned long idx = 0;
+	void *entry;
+
+	entry = xa_find(&ldev->pfs, &idx, U8_MAX, MLX5_LAG_XA_MARK_MASTER);
+	if (!entry)
+		return;
+
+	xa_clear_mark(&ldev->pfs, idx, MLX5_LAG_XA_MARK_MASTER);
+}
+
+/* Devcom event handler to manage LAG master marking */
+static int mlx5_lag_devcom_event(int event, void *my_data, void *event_data)
+{
+	struct mlx5_core_dev *dev = my_data;
+	struct mlx5_lag *ldev;
+	int idx;
+
+	ldev = mlx5_lag_dev(dev);
+	if (!ldev)
+		return 0;
+
+	mutex_lock(&ldev->lock);
+	switch (event) {
+	case LAG_DEVCOM_PAIR:
+		/* No need to mark more than once */
+		idx = mlx5_lag_get_master_idx(ldev);
+		if (idx >= 0)
+			break;
+		/* Check if all LAG ports are now registered */
+		if (mlx5_lag_num_devs(ldev) == ldev->ports)
+			mlx5_lag_mark_master(ldev);
+		break;
+
+	case LAG_DEVCOM_UNPAIR:
+		/* Clear master mark when a device is removed */
+		mlx5_lag_clear_master(ldev);
+		break;
+	}
+	mutex_unlock(&ldev->lock);
+	return 0;
+}
+
 int mlx5_lag_num_devs(struct mlx5_lag *ldev)
 {
 	int i, num = 0;
@@ -349,14 +488,17 @@ int mlx5_lag_num_devs(struct mlx5_lag *ldev)
 
 int mlx5_lag_num_netdevs(struct mlx5_lag *ldev)
 {
+	struct lag_func *pf;
 	int i, num = 0;
 
 	if (!ldev)
 		return 0;
 
-	mlx5_ldev_for_each(i, 0, ldev)
-		if (ldev->pf[i].netdev)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->netdev)
 			num++;
+	}
 	return num;
 }
 
@@ -400,11 +542,12 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
 
 	/* Use native mapping by default where each port's buckets
 	 * point the native port: 1 1 1 .. 1 2 2 2 ... 2 3 3 3 ... 3 etc
+	 * ports[] values are 1-indexed device indices for FW.
 	 */
 	mlx5_ldev_for_each(i, 0, ldev) {
 		for (j = 0; j < buckets; j++) {
 			idx = i * buckets + j;
-			ports[idx] = i + 1;
+			ports[idx] = mlx5_lag_xa_to_dev_idx(ldev, i) + 1;
 		}
 	}
 
@@ -416,33 +559,42 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
 	/* Go over the disabled ports and for each assign a random active port */
 	for (i = 0; i < disabled_ports_num; i++) {
 		for (j = 0; j < buckets; j++) {
+			int rand_xa_idx;
+
 			get_random_bytes(&rand, 4);
-			ports[disabled[i] * buckets + j] = enabled[rand % enabled_ports_num] + 1;
+			rand_xa_idx = enabled[rand % enabled_ports_num];
+			ports[disabled[i] * buckets + j] =
+				mlx5_lag_xa_to_dev_idx(ldev, rand_xa_idx) + 1;
 		}
 	}
 }
 
 static bool mlx5_lag_has_drop_rule(struct mlx5_lag *ldev)
 {
+	struct lag_func *pf;
 	int i;
 
-	mlx5_ldev_for_each(i, 0, ldev)
-		if (ldev->pf[i].has_drop)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->has_drop)
 			return true;
+	}
 	return false;
 }
 
 static void mlx5_lag_drop_rule_cleanup(struct mlx5_lag *ldev)
 {
+	struct lag_func *pf;
 	int i;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (!ldev->pf[i].has_drop)
+		pf = mlx5_lag_pf(ldev, i);
+		if (!pf->has_drop)
 			continue;
 
-		mlx5_esw_acl_ingress_vport_drop_rule_destroy(ldev->pf[i].dev->priv.eswitch,
+		mlx5_esw_acl_ingress_vport_drop_rule_destroy(pf->dev->priv.eswitch,
 							     MLX5_VPORT_UPLINK);
-		ldev->pf[i].has_drop = false;
+		pf->has_drop = false;
 	}
 }
 
@@ -451,6 +603,7 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 {
 	u8 disabled_ports[MLX5_MAX_PORTS] = {};
 	struct mlx5_core_dev *dev;
+	struct lag_func *pf;
 	int disabled_index;
 	int num_disabled;
 	int err;
@@ -468,11 +621,12 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 
 	for (i = 0; i < num_disabled; i++) {
 		disabled_index = disabled_ports[i];
-		dev = ldev->pf[disabled_index].dev;
+		pf = mlx5_lag_pf(ldev, disabled_index);
+		dev = pf->dev;
 		err = mlx5_esw_acl_ingress_vport_drop_rule_create(dev->priv.eswitch,
 								  MLX5_VPORT_UPLINK);
 		if (!err)
-			ldev->pf[disabled_index].has_drop = true;
+			pf->has_drop = true;
 		else
 			mlx5_core_err(dev,
 				      "Failed to create lag drop rule, error: %d", err);
@@ -504,7 +658,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports)
 	if (idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[idx].dev;
+	dev0 = mlx5_lag_pf(ldev, idx)->dev;
 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) {
 		ret = mlx5_lag_port_sel_modify(ldev, ports);
 		if (ret ||
@@ -521,6 +675,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports)
 static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev *dev)
 {
 	struct net_device *ndev = NULL;
+	struct lag_func *pf;
 	struct mlx5_lag *ldev;
 	unsigned long flags;
 	int i, last_idx;
@@ -531,14 +686,17 @@ static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev
 	if (!ldev)
 		goto unlock;
 
-	mlx5_ldev_for_each(i, 0, ldev)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
 		if (ldev->tracker.netdev_state[i].tx_enabled)
-			ndev = ldev->pf[i].netdev;
+			ndev = pf->netdev;
+	}
 	if (!ndev) {
 		last_idx = mlx5_lag_get_dev_index_by_seq(ldev, ldev->ports - 1);
 		if (last_idx < 0)
 			goto unlock;
-		ndev = ldev->pf[last_idx].netdev;
+		pf = mlx5_lag_pf(ldev, last_idx);
+		ndev = pf->netdev;
 	}
 
 	dev_hold(ndev);
@@ -563,7 +721,7 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 	if (first_idx < 0)
 		return;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, first_idx)->dev;
 	mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ports);
 
 	mlx5_ldev_for_each(i, 0, ldev) {
@@ -615,7 +773,7 @@ static int mlx5_lag_set_port_sel_mode(struct mlx5_lag *ldev,
 	    mode == MLX5_LAG_MODE_MULTIPATH)
 		return 0;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, first_idx)->dev;
 
 	if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) {
 		if (ldev->ports > 2)
@@ -661,19 +819,24 @@ char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
 
 static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev)
 {
-	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	struct mlx5_eswitch *master_esw;
 	struct mlx5_core_dev *dev0;
 	int i, j;
 	int err;
 
-	if (first_idx < 0)
+	if (master_idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, master_idx)->dev;
 	master_esw = dev0->priv.eswitch;
-	mlx5_ldev_for_each(i, first_idx + 1, ldev) {
-		struct mlx5_eswitch *slave_esw = ldev->pf[i].dev->priv.eswitch;
+	mlx5_ldev_for_each(i, 0, ldev) {
+		struct mlx5_eswitch *slave_esw;
+
+		if (i == master_idx)
+			continue;
+
+		slave_esw = mlx5_lag_pf(ldev, i)->dev->priv.eswitch;
 
 		err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw,
 							       slave_esw, ldev->ports);
@@ -682,9 +845,12 @@ static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev)
 	}
 	return 0;
 err:
-	mlx5_ldev_for_each_reverse(j, i, first_idx + 1, ldev)
+	mlx5_ldev_for_each_reverse(j, i, 0, ldev) {
+		if (j == master_idx)
+			continue;
 		mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
-							 ldev->pf[j].dev->priv.eswitch);
+							 mlx5_lag_pf(ldev, j)->dev->priv.eswitch);
+	}
 	return err;
 }
 
@@ -693,8 +859,8 @@ static int mlx5_create_lag(struct mlx5_lag *ldev,
 			   enum mlx5_lag_mode mode,
 			   unsigned long flags)
 {
-	bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
 	u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
 	struct mlx5_core_dev *dev0;
 	int err;
@@ -702,7 +868,7 @@ static int mlx5_create_lag(struct mlx5_lag *ldev,
 	if (first_idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, first_idx)->dev;
 	if (tracker)
 		mlx5_lag_print_mapping(dev0, ldev, tracker, flags);
 	mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n",
@@ -740,16 +906,17 @@ int mlx5_activate_lag(struct mlx5_lag *ldev,
 		      enum mlx5_lag_mode mode,
 		      bool shared_fdb)
 {
-	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	bool roce_lag = mode == MLX5_LAG_MODE_ROCE;
 	struct mlx5_core_dev *dev0;
 	unsigned long flags = 0;
+	int master_idx;
 	int err;
 
-	if (first_idx < 0)
+	master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	if (master_idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, master_idx)->dev;
 	err = mlx5_lag_set_flags(ldev, mode, tracker, shared_fdb, &flags);
 	if (err)
 		return err;
@@ -793,7 +960,7 @@ int mlx5_activate_lag(struct mlx5_lag *ldev,
 
 int mlx5_deactivate_lag(struct mlx5_lag *ldev)
 {
-	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {};
 	bool roce_lag = __mlx5_lag_is_roce(ldev);
 	unsigned long flags = ldev->mode_flags;
@@ -802,19 +969,22 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev)
 	int err;
 	int i;
 
-	if (first_idx < 0)
+	if (master_idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[first_idx].dev;
+	dev0 = mlx5_lag_pf(ldev, master_idx)->dev;
 	master_esw = dev0->priv.eswitch;
 	ldev->mode = MLX5_LAG_MODE_NONE;
 	ldev->mode_flags = 0;
 	mlx5_lag_mp_reset(ldev);
 
 	if (test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags)) {
-		mlx5_ldev_for_each(i, first_idx + 1, ldev)
+		mlx5_ldev_for_each(i, 0, ldev) {
+			if (i == master_idx)
+				continue;
 			mlx5_eswitch_offloads_single_fdb_del_one(master_esw,
-								 ldev->pf[i].dev->priv.eswitch);
+								 mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
+		}
 		clear_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags);
 	}
 
@@ -844,68 +1014,82 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev)
 
 bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
 {
-	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 #ifdef CONFIG_MLX5_ESWITCH
 	struct mlx5_core_dev *dev;
 	u8 mode;
 #endif
+	struct lag_func *pf;
 	bool roce_support;
 	int i;
 
-	if (first_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports)
+	if (master_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports)
 		return false;
 
 #ifdef CONFIG_MLX5_ESWITCH
 	mlx5_ldev_for_each(i, 0, ldev) {
-		dev = ldev->pf[i].dev;
+		pf = mlx5_lag_pf(ldev, i);
+		dev = pf->dev;
 		if (mlx5_eswitch_num_vfs(dev->priv.eswitch) && !is_mdev_switchdev_mode(dev))
 			return false;
 	}
 
-	dev = ldev->pf[first_idx].dev;
+	pf = mlx5_lag_pf(ldev, master_idx);
+	dev = pf->dev;
 	mode = mlx5_eswitch_mode(dev);
-	mlx5_ldev_for_each(i, 0, ldev)
-		if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (mlx5_eswitch_mode(pf->dev) != mode)
 			return false;
+	}
 
 #else
-	mlx5_ldev_for_each(i, 0, ldev)
-		if (mlx5_sriov_is_enabled(ldev->pf[i].dev))
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (mlx5_sriov_is_enabled(pf->dev))
 			return false;
+	}
 #endif
-	roce_support = mlx5_get_roce_state(ldev->pf[first_idx].dev);
-	mlx5_ldev_for_each(i, first_idx + 1, ldev)
-		if (mlx5_get_roce_state(ldev->pf[i].dev) != roce_support)
+	pf = mlx5_lag_pf(ldev, master_idx);
+	roce_support = mlx5_get_roce_state(pf->dev);
+	mlx5_ldev_for_each(i, 0, ldev) {
+		if (i == master_idx)
+			continue;
+		pf = mlx5_lag_pf(ldev, i);
+		if (mlx5_get_roce_state(pf->dev) != roce_support)
 			return false;
+	}
 
 	return true;
 }
 
 void mlx5_lag_add_devices(struct mlx5_lag *ldev)
 {
+	struct lag_func *pf;
 	int i;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (ldev->pf[i].dev->priv.flags &
-		    MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
 			continue;
 
-		ldev->pf[i].dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-		mlx5_rescan_drivers_locked(ldev->pf[i].dev);
+		pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+		mlx5_rescan_drivers_locked(pf->dev);
 	}
 }
 
 void mlx5_lag_remove_devices(struct mlx5_lag *ldev)
 {
+	struct lag_func *pf;
 	int i;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (ldev->pf[i].dev->priv.flags &
-		    MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
 			continue;
 
-		ldev->pf[i].dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-		mlx5_rescan_drivers_locked(ldev->pf[i].dev);
+		pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+		mlx5_rescan_drivers_locked(pf->dev);
 	}
 }
 
@@ -921,7 +1105,7 @@ void mlx5_disable_lag(struct mlx5_lag *ldev)
 	if (idx < 0)
 		return;
 
-	dev0 = ldev->pf[idx].dev;
+	dev0 = mlx5_lag_pf(ldev, idx)->dev;
 	roce_lag = __mlx5_lag_is_roce(ldev);
 
 	if (shared_fdb) {
@@ -931,8 +1115,11 @@ void mlx5_disable_lag(struct mlx5_lag *ldev)
 			dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
 			mlx5_rescan_drivers_locked(dev0);
 		}
-		mlx5_ldev_for_each(i, idx + 1, ldev)
-			mlx5_nic_vport_disable_roce(ldev->pf[i].dev);
+		mlx5_ldev_for_each(i, 0, ldev) {
+			if (i == idx)
+				continue;
+			mlx5_nic_vport_disable_roce(mlx5_lag_pf(ldev, i)->dev);
+		}
 	}
 
 	err = mlx5_deactivate_lag(ldev);
@@ -944,21 +1131,25 @@ void mlx5_disable_lag(struct mlx5_lag *ldev)
 
 	if (shared_fdb)
 		mlx5_ldev_for_each(i, 0, ldev)
-			if (!(ldev->pf[i].dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV))
-				mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+			if (!(mlx5_lag_pf(ldev, i)->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV))
+				mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 }
 
 bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
 {
-	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	struct mlx5_core_dev *dev;
+	bool ret = false;
+	int idx;
 	int i;
 
+	idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	if (idx < 0)
 		return false;
 
-	mlx5_ldev_for_each(i, idx + 1, ldev) {
-		dev = ldev->pf[i].dev;
+	mlx5_ldev_for_each(i, 0, ldev) {
+		if (i == idx)
+			continue;
+		dev = mlx5_lag_pf(ldev, i)->dev;
 		if (is_mdev_switchdev_mode(dev) &&
 		    mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
 		    MLX5_CAP_GEN(dev, lag_native_fdb_selection) &&
@@ -969,28 +1160,33 @@ bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
 		return false;
 	}
 
-	dev = ldev->pf[idx].dev;
+	dev = mlx5_lag_pf(ldev, idx)->dev;
 	if (is_mdev_switchdev_mode(dev) &&
 	    mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) &&
 	    mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) &&
 	    MLX5_CAP_ESW(dev, esw_shared_ingress_acl) &&
 	    mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1)
-		return true;
+		ret = true;
 
-	return false;
+	return ret;
 }
 
 static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev)
 {
 	bool roce_lag = true;
+	struct lag_func *pf;
 	int i;
 
-	mlx5_ldev_for_each(i, 0, ldev)
-		roce_lag = roce_lag && !mlx5_sriov_is_enabled(ldev->pf[i].dev);
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		roce_lag = roce_lag && !mlx5_sriov_is_enabled(pf->dev);
+	}
 
 #ifdef CONFIG_MLX5_ESWITCH
-	mlx5_ldev_for_each(i, 0, ldev)
-		roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev);
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		roce_lag = roce_lag && is_mdev_legacy_mode(pf->dev);
+	}
 #endif
 
 	return roce_lag;
@@ -1014,13 +1210,17 @@ mlx5_lag_sum_devices_speed(struct mlx5_lag *ldev, u32 *sum_speed,
 			   int (*get_speed)(struct mlx5_core_dev *, u32 *))
 {
 	struct mlx5_core_dev *pf_mdev;
+	struct lag_func *pf;
 	int pf_idx;
 	u32 speed;
 	int ret;
 
 	*sum_speed = 0;
 	mlx5_ldev_for_each(pf_idx, 0, ldev) {
-		pf_mdev = ldev->pf[pf_idx].dev;
+		pf = mlx5_lag_pf(ldev, pf_idx);
+		if (!pf)
+			continue;
+		pf_mdev = pf->dev;
 		if (!pf_mdev)
 			continue;
 
@@ -1086,6 +1286,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev,
 void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev)
 {
 	struct mlx5_core_dev *mdev;
+	struct lag_func *pf;
 	u32 speed;
 	int pf_idx;
 
@@ -1105,7 +1306,10 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev)
 	speed = speed / MLX5_MAX_TX_SPEED_UNIT;
 
 	mlx5_ldev_for_each(pf_idx, 0, ldev) {
-		mdev = ldev->pf[pf_idx].dev;
+		pf = mlx5_lag_pf(ldev, pf_idx);
+		if (!pf)
+			continue;
+		mdev = pf->dev;
 		if (!mdev)
 			continue;
 
@@ -1116,12 +1320,16 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev)
 void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev)
 {
 	struct mlx5_core_dev *mdev;
+	struct lag_func *pf;
 	u32 speed;
 	int pf_idx;
 	int ret;
 
 	mlx5_ldev_for_each(pf_idx, 0, ldev) {
-		mdev = ldev->pf[pf_idx].dev;
+		pf = mlx5_lag_pf(ldev, pf_idx);
+		if (!pf)
+			continue;
+		mdev = pf->dev;
 		if (!mdev)
 			continue;
 
@@ -1152,7 +1360,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 	if (idx < 0)
 		return;
 
-	dev0 = ldev->pf[idx].dev;
+	dev0 = mlx5_lag_pf(ldev, idx)->dev;
 	if (!mlx5_lag_is_ready(ldev)) {
 		do_bond = false;
 	} else {
@@ -1182,16 +1390,23 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 				mlx5_lag_add_devices(ldev);
 			if (shared_fdb) {
 				mlx5_ldev_for_each(i, 0, ldev)
-					mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+					mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 			}
 
 			return;
-		} else if (roce_lag) {
+		}
+
+		if (roce_lag) {
+			struct mlx5_core_dev *dev;
+
 			dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
 			mlx5_rescan_drivers_locked(dev0);
-			mlx5_ldev_for_each(i, idx + 1, ldev) {
-				if (mlx5_get_roce_state(ldev->pf[i].dev))
-					mlx5_nic_vport_enable_roce(ldev->pf[i].dev);
+			mlx5_ldev_for_each(i, 0, ldev) {
+				if (i == idx)
+					continue;
+				dev = mlx5_lag_pf(ldev, i)->dev;
+				if (mlx5_get_roce_state(dev))
+					mlx5_nic_vport_enable_roce(dev);
 			}
 		} else if (shared_fdb) {
 			int i;
@@ -1200,7 +1415,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 			mlx5_rescan_drivers_locked(dev0);
 
 			mlx5_ldev_for_each(i, 0, ldev) {
-				err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+				err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 				if (err)
 					break;
 			}
@@ -1211,7 +1426,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 				mlx5_deactivate_lag(ldev);
 				mlx5_lag_add_devices(ldev);
 				mlx5_ldev_for_each(i, 0, ldev)
-					mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+					mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 				mlx5_core_err(dev0, "Failed to enable lag\n");
 				return;
 			}
@@ -1243,16 +1458,171 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev)
 {
 	struct mlx5_devcom_comp_dev *devcom = NULL;
+	struct lag_func *pf;
 	int i;
 
 	mutex_lock(&ldev->lock);
 	i = mlx5_get_next_ldev_func(ldev, 0);
-	if (i < MLX5_MAX_PORTS)
-		devcom = ldev->pf[i].dev->priv.hca_devcom_comp;
+	if (i < MLX5_MAX_PORTS) {
+		pf = mlx5_lag_pf(ldev, i);
+		devcom = pf->dev->priv.hca_devcom_comp;
+	}
 	mutex_unlock(&ldev->lock);
 	return devcom;
 }
 
+static int mlx5_lag_demux_ft_fg_init(struct mlx5_core_dev *dev,
+				     struct mlx5_flow_table_attr *ft_attr,
+				     struct mlx5_lag *ldev)
+{
+#ifdef CONFIG_MLX5_ESWITCH
+	struct mlx5_flow_namespace *ns;
+	struct mlx5_flow_group *fg;
+	int err;
+
+	ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_LAG);
+	if (!ns)
+		return 0;
+
+	ldev->lag_demux_ft = mlx5_create_flow_table(ns, ft_attr);
+	if (IS_ERR(ldev->lag_demux_ft))
+		return PTR_ERR(ldev->lag_demux_ft);
+
+	fg = mlx5_esw_lag_demux_fg_create(dev->priv.eswitch,
+					  ldev->lag_demux_ft);
+	if (IS_ERR(fg)) {
+		err = PTR_ERR(fg);
+		mlx5_destroy_flow_table(ldev->lag_demux_ft);
+		ldev->lag_demux_ft = NULL;
+		return err;
+	}
+
+	ldev->lag_demux_fg = fg;
+	return 0;
+#else
+	return -EOPNOTSUPP;
+#endif
+}
+
+static int mlx5_lag_demux_fw_init(struct mlx5_core_dev *dev,
+				  struct mlx5_flow_table_attr *ft_attr,
+				  struct mlx5_lag *ldev)
+{
+	struct mlx5_flow_namespace *ns;
+	int err;
+
+	ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_LAG);
+	if (!ns)
+		return 0;
+
+	ldev->lag_demux_fg = NULL;
+	ft_attr->max_fte = 1;
+	ldev->lag_demux_ft = mlx5_create_lag_demux_flow_table(ns, ft_attr);
+	if (IS_ERR(ldev->lag_demux_ft)) {
+		err = PTR_ERR(ldev->lag_demux_ft);
+		ldev->lag_demux_ft = NULL;
+		return err;
+	}
+
+	return 0;
+}
+
+int mlx5_lag_demux_init(struct mlx5_core_dev *dev,
+			struct mlx5_flow_table_attr *ft_attr)
+{
+	struct mlx5_lag *ldev;
+
+	if (!ft_attr)
+		return -EINVAL;
+
+	ldev = mlx5_lag_dev(dev);
+	if (!ldev)
+		return -ENODEV;
+
+	xa_init(&ldev->lag_demux_rules);
+
+	if (mlx5_get_sd(dev))
+		return mlx5_lag_demux_ft_fg_init(dev, ft_attr, ldev);
+
+	return mlx5_lag_demux_fw_init(dev, ft_attr, ldev);
+}
+EXPORT_SYMBOL(mlx5_lag_demux_init);
+
+void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_handle *rule;
+	struct mlx5_lag *ldev;
+	unsigned long vport_num;
+
+	ldev = mlx5_lag_dev(dev);
+	if (!ldev)
+		return;
+
+	xa_for_each(&ldev->lag_demux_rules, vport_num, rule)
+		mlx5_del_flow_rules(rule);
+	xa_destroy(&ldev->lag_demux_rules);
+
+	if (ldev->lag_demux_fg)
+		mlx5_destroy_flow_group(ldev->lag_demux_fg);
+	if (ldev->lag_demux_ft)
+		mlx5_destroy_flow_table(ldev->lag_demux_ft);
+	ldev->lag_demux_fg = NULL;
+	ldev->lag_demux_ft = NULL;
+}
+EXPORT_SYMBOL(mlx5_lag_demux_cleanup);
+
+int mlx5_lag_demux_rule_add(struct mlx5_core_dev *vport_dev, u16 vport_num,
+			    int index)
+{
+	struct mlx5_flow_handle *rule;
+	struct mlx5_lag *ldev;
+	int err;
+
+	ldev = mlx5_lag_dev(vport_dev);
+	if (!ldev || !ldev->lag_demux_fg)
+		return 0;
+
+	if (xa_load(&ldev->lag_demux_rules, index))
+		return 0;
+
+	rule = mlx5_esw_lag_demux_rule_create(vport_dev->priv.eswitch,
+					      vport_num, ldev->lag_demux_ft);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		mlx5_core_warn(vport_dev,
+			       "Failed to create LAG demux rule for vport %u, err %d\n",
+			       vport_num, err);
+		return err;
+	}
+
+	err = xa_err(xa_store(&ldev->lag_demux_rules, index, rule,
+			      GFP_KERNEL));
+	if (err) {
+		mlx5_del_flow_rules(rule);
+		mlx5_core_warn(vport_dev,
+			       "Failed to store LAG demux rule for vport %u, err %d\n",
+			       vport_num, err);
+	}
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_lag_demux_rule_add);
+
+void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int index)
+{
+	struct mlx5_flow_handle *rule;
+	struct mlx5_lag *ldev;
+
+	ldev = mlx5_lag_dev(dev);
+	if (!ldev || !ldev->lag_demux_fg)
+		return;
+
+	rule = xa_erase(&ldev->lag_demux_rules, index);
+	if (rule)
+		mlx5_del_flow_rules(rule);
+}
+EXPORT_SYMBOL(mlx5_lag_demux_rule_del);
+
 static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
 {
 	queue_delayed_work(ldev->wq, &ldev->bond_work, delay);
@@ -1297,6 +1667,7 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
 	struct netdev_lag_upper_info *lag_upper_info = NULL;
 	bool is_bonded, is_in_lag, mode_supported;
 	bool has_inactive = 0;
+	struct lag_func *pf;
 	struct slave *slave;
 	u8 bond_status = 0;
 	int num_slaves = 0;
@@ -1317,7 +1688,8 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
 	rcu_read_lock();
 	for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
 		mlx5_ldev_for_each(i, 0, ldev) {
-			if (ldev->pf[i].netdev == ndev_tmp) {
+			pf = mlx5_lag_pf(ldev, i);
+			if (pf->netdev == ndev_tmp) {
 				idx++;
 				break;
 			}
@@ -1537,59 +1909,92 @@ static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev,
 				struct mlx5_core_dev *dev,
 				struct net_device *netdev)
 {
-	unsigned int fn = mlx5_get_dev_index(dev);
+	struct lag_func *pf;
 	unsigned long flags;
+	int i;
 
 	spin_lock_irqsave(&lag_lock, flags);
-	ldev->pf[fn].netdev = netdev;
-	ldev->tracker.netdev_state[fn].link_up = 0;
-	ldev->tracker.netdev_state[fn].tx_enabled = 0;
+	/* Find pf entry by matching dev pointer */
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->dev == dev) {
+			pf->netdev = netdev;
+			ldev->tracker.netdev_state[i].link_up = 0;
+			ldev->tracker.netdev_state[i].tx_enabled = 0;
+			break;
+		}
+	}
 	spin_unlock_irqrestore(&lag_lock, flags);
 }
 
 static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev,
 				    struct net_device *netdev)
 {
+	struct lag_func *pf;
 	unsigned long flags;
 	int i;
 
 	spin_lock_irqsave(&lag_lock, flags);
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (ldev->pf[i].netdev == netdev) {
-			ldev->pf[i].netdev = NULL;
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->netdev == netdev) {
+			pf->netdev = NULL;
 			break;
 		}
 	}
 	spin_unlock_irqrestore(&lag_lock, flags);
 }
 
-static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev,
+static int mlx5_ldev_add_mdev(struct mlx5_lag *ldev,
 			      struct mlx5_core_dev *dev)
 {
-	unsigned int fn = mlx5_get_dev_index(dev);
+	struct lag_func *pf;
+	u32 idx;
+	int err;
+
+	pf = kzalloc_obj(*pf);
+	if (!pf)
+		return -ENOMEM;
+
+	err = xa_alloc(&ldev->pfs, &idx, pf, XA_LIMIT(0, MLX5_MAX_PORTS - 1),
+		       GFP_KERNEL);
+	if (err) {
+		kfree(pf);
+		return err;
+	}
 
-	ldev->pf[fn].dev = dev;
+	pf->idx = idx;
+	pf->dev = dev;
 	dev->priv.lag = ldev;
 
-	MLX5_NB_INIT(&ldev->pf[fn].port_change_nb,
+	MLX5_NB_INIT(&pf->port_change_nb,
 		     mlx5_lag_mpesw_port_change_event, PORT_CHANGE);
-	mlx5_eq_notifier_register(dev, &ldev->pf[fn].port_change_nb);
+	mlx5_eq_notifier_register(dev, &pf->port_change_nb);
+
+	return 0;
 }
 
 static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev,
 				  struct mlx5_core_dev *dev)
 {
-	int fn;
+	struct lag_func *pf;
+	int i;
 
-	fn = mlx5_get_dev_index(dev);
-	if (ldev->pf[fn].dev != dev)
+	mlx5_ldev_for_each(i, 0, ldev) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->dev == dev)
+			break;
+	}
+	if (i >= MLX5_MAX_PORTS)
 		return;
 
-	if (ldev->pf[fn].port_change_nb.nb.notifier_call)
-		mlx5_eq_notifier_unregister(dev, &ldev->pf[fn].port_change_nb);
+	if (pf->port_change_nb.nb.notifier_call)
+		mlx5_eq_notifier_unregister(dev, &pf->port_change_nb);
 
-	ldev->pf[fn].dev = NULL;
+	pf->dev = NULL;
 	dev->priv.lag = NULL;
+	xa_erase(&ldev->pfs, pf->idx);
+	kfree(pf);
 }
 
 /* Must be called with HCA devcom component lock held */
@@ -1598,6 +2003,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
 	struct mlx5_devcom_comp_dev *pos = NULL;
 	struct mlx5_lag *ldev = NULL;
 	struct mlx5_core_dev *tmp_dev;
+	int err;
 
 	tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos);
 	if (tmp_dev)
@@ -1609,7 +2015,12 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
 			mlx5_core_err(dev, "Failed to alloc lag dev\n");
 			return 0;
 		}
-		mlx5_ldev_add_mdev(ldev, dev);
+		err = mlx5_ldev_add_mdev(ldev, dev);
+		if (err) {
+			mlx5_core_err(dev, "Failed to add mdev to lag dev\n");
+			mlx5_ldev_put(ldev);
+			return 0;
+		}
 		return 0;
 	}
 
@@ -1619,7 +2030,12 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
 		return -EAGAIN;
 	}
 	mlx5_ldev_get(ldev);
-	mlx5_ldev_add_mdev(ldev, dev);
+	err = mlx5_ldev_add_mdev(ldev, dev);
+	if (err) {
+		mlx5_ldev_put(ldev);
+		mutex_unlock(&ldev->lock);
+		return err;
+	}
 	mutex_unlock(&ldev->lock);
 
 	return 0;
@@ -1647,7 +2063,8 @@ static int mlx5_lag_register_hca_devcom_comp(struct mlx5_core_dev *dev)
 	dev->priv.hca_devcom_comp =
 		mlx5_devcom_register_component(dev->priv.devc,
 					       MLX5_DEVCOM_HCA_PORTS,
-					       &attr, NULL, dev);
+					       &attr, mlx5_lag_devcom_event,
+					       dev);
 	if (!dev->priv.hca_devcom_comp) {
 		mlx5_core_err(dev,
 			      "Failed to register devcom HCA component.");
@@ -1678,6 +2095,9 @@ void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev)
 	}
 	mlx5_ldev_remove_mdev(ldev, dev);
 	mutex_unlock(&ldev->lock);
+	/* Send devcom event to notify peers that a device is being removed */
+	mlx5_devcom_send_event(dev->priv.hca_devcom_comp,
+			       LAG_DEVCOM_UNPAIR, LAG_DEVCOM_UNPAIR, dev);
 	mlx5_lag_unregister_hca_devcom_comp(dev);
 	mlx5_ldev_put(ldev);
 }
@@ -1701,6 +2121,9 @@ void mlx5_lag_add_mdev(struct mlx5_core_dev *dev)
 		msleep(100);
 		goto recheck;
 	}
+	/* Send devcom event to notify peers that a device was added */
+	mlx5_devcom_send_event(dev->priv.hca_devcom_comp,
+			       LAG_DEVCOM_PAIR, LAG_DEVCOM_UNPAIR, dev);
 	mlx5_ldev_add_debugfs(dev);
 }
 
@@ -1746,21 +2169,25 @@ void mlx5_lag_add_netdev(struct mlx5_core_dev *dev,
 
 int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx)
 {
+	struct lag_func *pf;
 	int i;
 
-	for (i = start_idx; i >= end_idx; i--)
-		if (ldev->pf[i].dev)
+	for (i = start_idx; i >= end_idx; i--) {
+		pf = xa_load(&ldev->pfs, i);
+		if (pf && pf->dev)
 			return i;
+	}
 	return -1;
 }
 
 int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx)
 {
-	int i;
+	struct lag_func *pf;
+	unsigned long idx;
 
-	for (i = start_idx; i < MLX5_MAX_PORTS; i++)
-		if (ldev->pf[i].dev)
-			return i;
+	xa_for_each_start(&ldev->pfs, idx, pf, start_idx)
+		if (pf->dev)
+			return idx;
 	return MLX5_MAX_PORTS;
 }
 
@@ -1814,13 +2241,17 @@ bool mlx5_lag_is_master(struct mlx5_core_dev *dev)
 {
 	struct mlx5_lag *ldev;
 	unsigned long flags;
+	struct lag_func *pf;
 	bool res = false;
 	int idx;
 
 	spin_lock_irqsave(&lag_lock, flags);
 	ldev = mlx5_lag_dev(dev);
 	idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
-	res = ldev && __mlx5_lag_is_active(ldev) && idx >= 0 && dev == ldev->pf[idx].dev;
+	if (ldev && __mlx5_lag_is_active(ldev) && idx >= 0) {
+		pf = mlx5_lag_pf(ldev, idx);
+		res = pf && dev == pf->dev;
+	}
 	spin_unlock_irqrestore(&lag_lock, flags);
 
 	return res;
@@ -1899,6 +2330,7 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev,
 {
 	struct mlx5_lag *ldev;
 	unsigned long flags;
+	struct lag_func *pf;
 	u8 port = 0;
 	int i;
 
@@ -1908,7 +2340,8 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev,
 		goto unlock;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		if (ldev->pf[i].netdev == slave) {
+		pf = mlx5_lag_pf(ldev, i);
+		if (pf->netdev == slave) {
 			port = i;
 			break;
 		}
@@ -1939,6 +2372,7 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int
 	struct mlx5_core_dev *peer_dev = NULL;
 	struct mlx5_lag *ldev;
 	unsigned long flags;
+	struct lag_func *pf;
 	int idx;
 
 	spin_lock_irqsave(&lag_lock, flags);
@@ -1948,9 +2382,11 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int
 
 	if (*i == MLX5_MAX_PORTS)
 		goto unlock;
-	mlx5_ldev_for_each(idx, *i, ldev)
-		if (ldev->pf[idx].dev != dev)
+	mlx5_ldev_for_each(idx, *i, ldev) {
+		pf = mlx5_lag_pf(ldev, idx);
+		if (pf->dev != dev)
 			break;
+	}
 
 	if (idx == MLX5_MAX_PORTS) {
 		*i = idx;
@@ -1958,7 +2394,8 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int
 	}
 	*i = idx + 1;
 
-	peer_dev = ldev->pf[idx].dev;
+	pf = mlx5_lag_pf(ldev, idx);
+	peer_dev = pf->dev;
 
 unlock:
 	spin_unlock_irqrestore(&lag_lock, flags);
@@ -1976,6 +2413,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
 	int ret = 0, i, j, idx = 0;
 	struct mlx5_lag *ldev;
 	unsigned long flags;
+	struct lag_func *pf;
 	int num_ports;
 	void *out;
 
@@ -1995,8 +2433,10 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
 	ldev = mlx5_lag_dev(dev);
 	if (ldev && __mlx5_lag_is_active(ldev)) {
 		num_ports = ldev->ports;
-		mlx5_ldev_for_each(i, 0, ldev)
-			mdev[idx++] = ldev->pf[i].dev;
+		mlx5_ldev_for_each(i, 0, ldev) {
+			pf = mlx5_lag_pf(ldev, i);
+			mdev[idx++] = pf->dev;
+		}
 	} else {
 		num_ports = 1;
 		mdev[MLX5_LAG_P1] = dev;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
index be1afece5fdc1..6c911374f409b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
@@ -5,8 +5,17 @@
 #define __MLX5_LAG_H__
 
 #include <linux/debugfs.h>
+#include <linux/errno.h>
+#include <linux/xarray.h>
+#include <linux/mlx5/fs.h>
 
 #define MLX5_LAG_MAX_HASH_BUCKETS 16
+/* XArray mark for the LAG master device
+ * (device with lowest mlx5_get_dev_index).
+ * Note: XA_MARK_0 is reserved by XA_FLAGS_ALLOC for free-slot tracking.
+ */
+#define MLX5_LAG_XA_MARK_MASTER XA_MARK_1
+
 #include "mlx5_core.h"
 #include "mp.h"
 #include "port_sel.h"
@@ -39,6 +48,7 @@ struct lag_func {
 	struct mlx5_core_dev *dev;
 	struct net_device    *netdev;
 	bool has_drop;
+	unsigned int idx; /* xarray index assigned by LAG */
 	struct mlx5_nb port_change_nb;
 };
 
@@ -64,7 +74,7 @@ struct mlx5_lag {
 	int			  mode_changes_in_progress;
 	u8			  v2p_map[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS];
 	struct kref               ref;
-	struct lag_func           pf[MLX5_MAX_PORTS];
+	struct xarray             pfs;
 	struct lag_tracker        tracker;
 	struct workqueue_struct   *wq;
 	struct delayed_work       bond_work;
@@ -76,6 +86,9 @@ struct mlx5_lag {
 	/* Protect lag fields/state changes */
 	struct mutex		  lock;
 	struct lag_mpesw	  lag_mpesw;
+	struct mlx5_flow_table   *lag_demux_ft;
+	struct mlx5_flow_group   *lag_demux_fg;
+	struct xarray		  lag_demux_rules;
 };
 
 static inline struct mlx5_lag *
@@ -84,6 +97,34 @@ mlx5_lag_dev(struct mlx5_core_dev *dev)
 	return dev->priv.lag;
 }
 
+static inline struct lag_func *
+mlx5_lag_pf(struct mlx5_lag *ldev, unsigned int idx)
+{
+	return xa_load(&ldev->pfs, idx);
+}
+
+/* Get device index (mlx5_get_dev_index) from xarray index */
+static inline int mlx5_lag_xa_to_dev_idx(struct mlx5_lag *ldev, int xa_idx)
+{
+	struct lag_func *pf = mlx5_lag_pf(ldev, xa_idx);
+
+	return pf ? mlx5_get_dev_index(pf->dev) : -ENOENT;
+}
+
+/* Find lag_func by device index (reverse lookup from mlx5_get_dev_index) */
+static inline struct lag_func *
+mlx5_lag_pf_by_dev_idx(struct mlx5_lag *ldev, int dev_idx)
+{
+	struct lag_func *pf;
+	unsigned long idx;
+
+	xa_for_each(&ldev->pfs, idx, pf) {
+		if (mlx5_get_dev_index(pf->dev) == dev_idx)
+			return pf;
+	}
+	return NULL;
+}
+
 static inline bool
 __mlx5_lag_is_active(struct mlx5_lag *ldev)
 {
@@ -98,6 +139,12 @@ mlx5_lag_is_ready(struct mlx5_lag *ldev)
 
 bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev);
 bool mlx5_lag_check_prereq(struct mlx5_lag *ldev);
+int mlx5_lag_demux_init(struct mlx5_core_dev *dev,
+			struct mlx5_flow_table_attr *ft_attr);
+void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev);
+int mlx5_lag_demux_rule_add(struct mlx5_core_dev *dev, u16 vport_num,
+			    int vport_index);
+void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int vport_index);
 void mlx5_modify_lag(struct mlx5_lag *ldev,
 		     struct lag_tracker *tracker);
 int mlx5_activate_lag(struct mlx5_lag *ldev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c
index c4c2bf33ef351..f42e051fa7e79 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c
@@ -29,8 +29,8 @@ static bool mlx5_lag_multipath_check_prereq(struct mlx5_lag *ldev)
 	if (ldev->ports > MLX5_LAG_MULTIPATH_OFFLOADS_SUPPORTED_PORTS)
 		return false;
 
-	return mlx5_esw_multipath_prereq(ldev->pf[idx0].dev,
-					 ldev->pf[idx1].dev);
+	return mlx5_esw_multipath_prereq(mlx5_lag_pf(ldev, idx0)->dev,
+					 mlx5_lag_pf(ldev, idx1)->dev);
 }
 
 bool mlx5_lag_is_multipath(struct mlx5_core_dev *dev)
@@ -80,18 +80,18 @@ static void mlx5_lag_set_port_affinity(struct mlx5_lag *ldev,
 		tracker.netdev_state[idx1].link_up = true;
 		break;
 	default:
-		mlx5_core_warn(ldev->pf[idx0].dev,
+		mlx5_core_warn(mlx5_lag_pf(ldev, idx0)->dev,
 			       "Invalid affinity port %d", port);
 		return;
 	}
 
 	if (tracker.netdev_state[idx0].tx_enabled)
-		mlx5_notifier_call_chain(ldev->pf[idx0].dev->priv.events,
+		mlx5_notifier_call_chain(mlx5_lag_pf(ldev, idx0)->dev->priv.events,
 					 MLX5_DEV_EVENT_PORT_AFFINITY,
 					 (void *)0);
 
 	if (tracker.netdev_state[idx1].tx_enabled)
-		mlx5_notifier_call_chain(ldev->pf[idx1].dev->priv.events,
+		mlx5_notifier_call_chain(mlx5_lag_pf(ldev, idx1)->dev->priv.events,
 					 MLX5_DEV_EVENT_PORT_AFFINITY,
 					 (void *)0);
 
@@ -146,7 +146,7 @@ mlx5_lag_get_next_fib_dev(struct mlx5_lag *ldev,
 		fib_dev = fib_info_nh(fi, i)->fib_nh_dev;
 		ldev_idx = mlx5_lag_dev_get_netdev_idx(ldev, fib_dev);
 		if (ldev_idx >= 0)
-			return ldev->pf[ldev_idx].netdev;
+			return mlx5_lag_pf(ldev, ldev_idx)->netdev;
 	}
 
 	return NULL;
@@ -178,7 +178,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event,
 	    mp->fib.dst_len <= fen_info->dst_len &&
 	    !(mp->fib.dst_len == fen_info->dst_len &&
 	      fi->fib_priority < mp->fib.priority)) {
-		mlx5_core_dbg(ldev->pf[idx].dev,
+		mlx5_core_dbg(mlx5_lag_pf(ldev, idx)->dev,
 			      "Multipath entry with lower priority was rejected\n");
 		return;
 	}
@@ -194,7 +194,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event,
 	}
 
 	if (nh_dev0 == nh_dev1) {
-		mlx5_core_warn(ldev->pf[idx].dev,
+		mlx5_core_warn(mlx5_lag_pf(ldev, idx)->dev,
 			       "Multipath offload doesn't support routes with multiple nexthops of the same device");
 		return;
 	}
@@ -203,7 +203,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event,
 		if (__mlx5_lag_is_active(ldev)) {
 			mlx5_ldev_for_each(i, 0, ldev) {
 				dev_idx++;
-				if (ldev->pf[i].netdev == nh_dev0)
+				if (mlx5_lag_pf(ldev, i)->netdev == nh_dev0)
 					break;
 			}
 			mlx5_lag_set_port_affinity(ldev, dev_idx);
@@ -240,7 +240,7 @@ static void mlx5_lag_fib_nexthop_event(struct mlx5_lag *ldev,
 	/* nh added/removed */
 	if (event == FIB_EVENT_NH_DEL) {
 		mlx5_ldev_for_each(i, 0, ldev) {
-			if (ldev->pf[i].netdev == fib_nh->fib_nh_dev)
+			if (mlx5_lag_pf(ldev, i)->netdev == fib_nh->fib_nh_dev)
 				break;
 			dev_idx++;
 		}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
index 74d5c2ed14ff3..5eea12a6887ac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
@@ -16,7 +16,7 @@ static void mlx5_mpesw_metadata_cleanup(struct mlx5_lag *ldev)
 	int i;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		dev = ldev->pf[i].dev;
+		dev = mlx5_lag_pf(ldev, i)->dev;
 		esw = dev->priv.eswitch;
 		pf_metadata = ldev->lag_mpesw.pf_metadata[i];
 		if (!pf_metadata)
@@ -37,7 +37,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev)
 	int i, err;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		dev = ldev->pf[i].dev;
+		dev = mlx5_lag_pf(ldev, i)->dev;
 		esw = dev->priv.eswitch;
 		pf_metadata = mlx5_esw_match_metadata_alloc(esw);
 		if (!pf_metadata) {
@@ -53,7 +53,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev)
 	}
 
 	mlx5_ldev_for_each(i, 0, ldev) {
-		dev = ldev->pf[i].dev;
+		dev = mlx5_lag_pf(ldev, i)->dev;
 		mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_MULTIPORT_ESW,
 					 (void *)0);
 	}
@@ -67,9 +67,9 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev)
 
 static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 {
+	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	struct mlx5_core_dev *dev0;
 	int err;
-	int idx;
 	int i;
 
 	if (ldev->mode == MLX5_LAG_MODE_MPESW)
@@ -78,11 +78,10 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 	if (ldev->mode != MLX5_LAG_MODE_NONE)
 		return -EINVAL;
 
-	idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	if (idx < 0)
 		return -EINVAL;
 
-	dev0 = ldev->pf[idx].dev;
+	dev0 = mlx5_lag_pf(ldev, idx)->dev;
 	if (mlx5_eswitch_mode(dev0) != MLX5_ESWITCH_OFFLOADS ||
 	    !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table) ||
 	    !MLX5_CAP_GEN(dev0, create_lag_when_not_master_up) ||
@@ -105,7 +104,7 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 	dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
 	mlx5_rescan_drivers_locked(dev0);
 	mlx5_ldev_for_each(i, 0, ldev) {
-		err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+		err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 		if (err)
 			goto err_rescan_drivers;
 	}
@@ -121,7 +120,7 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 err_add_devices:
 	mlx5_lag_add_devices(ldev);
 	mlx5_ldev_for_each(i, 0, ldev)
-		mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch);
+		mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
 	mlx5_mpesw_metadata_cleanup(ldev);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c
index 16c7d16215c46..2a034b2a3eee3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c
@@ -50,7 +50,7 @@ static int mlx5_lag_create_port_sel_table(struct mlx5_lag *ldev,
 	if (first_idx < 0)
 		return -EINVAL;
 
-	dev = ldev->pf[first_idx].dev;
+	dev = mlx5_lag_pf(ldev, first_idx)->dev;
 	ft_attr.max_fte = ldev->ports * ldev->buckets;
 	ft_attr.level = MLX5_LAG_FT_LEVEL_DEFINER;
 
@@ -84,8 +84,12 @@ static int mlx5_lag_create_port_sel_table(struct mlx5_lag *ldev,
 			idx = i * ldev->buckets + j;
 			affinity = ports[idx];
 
-			dest.vport.vhca_id = MLX5_CAP_GEN(ldev->pf[affinity - 1].dev,
-							  vhca_id);
+			/* affinity is 1-indexed device index,
+			 * use reverse lookup.
+			 */
+			dest.vport.vhca_id =
+				MLX5_CAP_GEN(mlx5_lag_pf_by_dev_idx(ldev, affinity - 1)->dev,
+					     vhca_id);
 			lag_definer->rules[idx] = mlx5_add_flow_rules(lag_definer->ft,
 								      NULL, &flow_act,
 								      &dest, 1);
@@ -307,7 +311,7 @@ mlx5_lag_create_definer(struct mlx5_lag *ldev, enum netdev_lag_hash hash,
 	if (first_idx < 0)
 		return ERR_PTR(-EINVAL);
 
-	dev = ldev->pf[first_idx].dev;
+	dev = mlx5_lag_pf(ldev, first_idx)->dev;
 	lag_definer = kzalloc_obj(*lag_definer);
 	if (!lag_definer)
 		return ERR_PTR(-ENOMEM);
@@ -356,8 +360,8 @@ static void mlx5_lag_destroy_definer(struct mlx5_lag *ldev,
 	if (first_idx < 0)
 		return;
 
-	dev = ldev->pf[first_idx].dev;
-	mlx5_ldev_for_each(i, first_idx, ldev) {
+	dev = mlx5_lag_pf(ldev, first_idx)->dev;
+	mlx5_ldev_for_each(i, 0, ldev) {
 		for (j = 0; j < ldev->buckets; j++) {
 			idx = i * ldev->buckets + j;
 			mlx5_del_flow_rules(lag_definer->rules[idx]);
@@ -520,7 +524,7 @@ static int mlx5_lag_create_ttc_table(struct mlx5_lag *ldev)
 	if (first_idx < 0)
 		return -EINVAL;
 
-	dev = ldev->pf[first_idx].dev;
+	dev = mlx5_lag_pf(ldev, first_idx)->dev;
 	mlx5_lag_set_outer_ttc_params(ldev, &ttc_params);
 	port_sel->outer.ttc = mlx5_create_ttc_table(dev, &ttc_params);
 	return PTR_ERR_OR_ZERO(port_sel->outer.ttc);
@@ -536,7 +540,7 @@ static int mlx5_lag_create_inner_ttc_table(struct mlx5_lag *ldev)
 	if (first_idx < 0)
 		return -EINVAL;
 
-	dev = ldev->pf[first_idx].dev;
+	dev = mlx5_lag_pf(ldev, first_idx)->dev;
 	mlx5_lag_set_inner_ttc_params(ldev, &ttc_params);
 	port_sel->inner.ttc = mlx5_create_inner_ttc_table(dev, &ttc_params);
 	return PTR_ERR_OR_ZERO(port_sel->inner.ttc);
@@ -594,8 +598,12 @@ static int __mlx5_lag_modify_definers_destinations(struct mlx5_lag *ldev,
 			if (ldev->v2p_map[idx] == ports[idx])
 				continue;
 
-			dest.vport.vhca_id = MLX5_CAP_GEN(ldev->pf[ports[idx] - 1].dev,
-							  vhca_id);
+			/* ports[] contains 1-indexed device indices,
+			 * use reverse lookup.
+			 */
+			dest.vport.vhca_id =
+				MLX5_CAP_GEN(mlx5_lag_pf_by_dev_idx(ldev, ports[idx] - 1)->dev,
+					     vhca_id);
 			err = mlx5_modify_rule_destination(def->rules[idx], &dest, NULL);
 			if (err)
 				return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index bd4e042077afd..d785f1b4f2e1c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -38,10 +38,10 @@
 #include "lib/eq.h"
 #include "en.h"
 #include "clock.h"
-#ifdef CONFIG_X86
+#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER)
 #include <linux/timekeeping.h>
 #include <linux/cpufeature.h>
-#endif /* CONFIG_X86 */
+#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */
 
 #define MLX5_RT_CLOCK_IDENTITY_SIZE MLX5_FLD_SZ_BYTES(mrtcq_reg, rt_clock_identity)
 
@@ -229,7 +229,7 @@ static int mlx5_set_mtutc(struct mlx5_core_dev *dev, u32 *mtutc, u32 size)
 				    MLX5_REG_MTUTC, 0, 1);
 }
 
-#ifdef CONFIG_X86
+#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER)
 static bool mlx5_is_ptm_source_time_available(struct mlx5_core_dev *dev)
 {
 	u32 out[MLX5_ST_SZ_DW(mtptm_reg)] = {0};
@@ -275,7 +275,8 @@ static int mlx5_mtctr_read(struct mlx5_core_dev *mdev,
 	host = MLX5_GET64(mtctr_reg, out, first_clock_timestamp);
 	*sys_counterval = (struct system_counterval_t) {
 			.cycles = host,
-			.cs_id = CSID_X86_ART,
+			.cs_id = IS_ENABLED(CONFIG_X86) ? CSID_X86_ART :
+							  CSID_ARM_ARCH_COUNTER,
 			.use_nsecs = true,
 	};
 	*device = MLX5_GET64(mtctr_reg, out, second_clock_timestamp);
@@ -373,7 +374,7 @@ static int mlx5_ptp_getcrosscycles(struct ptp_clock_info *ptp,
 	mlx5_clock_unlock(clock);
 	return err;
 }
-#endif /* CONFIG_X86 */
+#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */
 
 static u64 mlx5_read_time(struct mlx5_core_dev *dev,
 			  struct ptp_system_timestamp *sts,
@@ -1301,6 +1302,25 @@ static void mlx5_init_timer_max_freq_adjustment(struct mlx5_core_dev *mdev)
 			min(S32_MAX, 1 << log_max_freq_adjustment);
 }
 
+static void mlx5_init_crosststamp(struct mlx5_core_dev *mdev,
+				  bool expose_cycles, struct mlx5_clock *clock)
+{
+#if defined(CONFIG_X86)
+	if (!boot_cpu_has(X86_FEATURE_ART))
+		return;
+#endif /* CONFIG_X86 */
+#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER)
+	if (!MLX5_CAP_MCAM_REG3(mdev, mtptm) ||
+	    !MLX5_CAP_MCAM_REG3(mdev, mtctr))
+		return;
+
+	clock->ptp_info.getcrosststamp = mlx5_ptp_getcrosststamp;
+	if (expose_cycles)
+		clock->ptp_info.getcrosscycles = mlx5_ptp_getcrosscycles;
+
+#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */
+}
+
 static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_clock *clock = mdev->clock;
@@ -1315,15 +1335,7 @@ static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev)
 	expose_cycles = !MLX5_CAP_GEN(mdev, disciplined_fr_counter) ||
 			!mlx5_real_time_mode(mdev);
 
-#ifdef CONFIG_X86
-	if (MLX5_CAP_MCAM_REG3(mdev, mtptm) &&
-	    MLX5_CAP_MCAM_REG3(mdev, mtctr) && boot_cpu_has(X86_FEATURE_ART)) {
-		clock->ptp_info.getcrosststamp = mlx5_ptp_getcrosststamp;
-		if (expose_cycles)
-			clock->ptp_info.getcrosscycles =
-				mlx5_ptp_getcrosscycles;
-	}
-#endif /* CONFIG_X86 */
+	mlx5_init_crosststamp(mdev, expose_cycles, clock);
 
 	if (expose_cycles)
 		clock->ptp_info.getcyclesx64 = mlx5_ptp_getcyclesx;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
index 954942ad93c55..762c783156b41 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
@@ -107,7 +107,7 @@ static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses)
 	/* Disconnect secondaries from the network */
 	if (!MLX5_CAP_GEN(dev, eswitch_manager))
 		return false;
-	if (!MLX5_CAP_GEN(dev, silent_mode))
+	if (!MLX5_CAP_GEN(dev, silent_mode_set))
 		return false;
 
 	/* RX steering from primary to secondaries */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index fdc3ba20912e4..2f8a8a889f80f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -74,6 +74,7 @@
 #include "mlx5_irq.h"
 #include "hwmon.h"
 #include "lag/lag.h"
+#include "sh_devlink.h"
 
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
@@ -110,74 +111,9 @@ static struct mlx5_profile profile[] = {
 
 	},
 	[2] = {
-		.mask		= MLX5_PROF_MASK_QP_SIZE |
-				  MLX5_PROF_MASK_MR_CACHE,
+		.mask		= MLX5_PROF_MASK_QP_SIZE,
 		.log_max_qp	= LOG_MAX_SUPPORTED_QPS,
 		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
-		.mr_cache[0]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[1]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[2]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[3]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[4]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[5]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[6]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[7]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[8]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[9]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[10]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[11]	= {
-			.size	= 500,
-			.limit	= 250
-		},
-		.mr_cache[12]	= {
-			.size	= 64,
-			.limit	= 32
-		},
-		.mr_cache[13]	= {
-			.size	= 32,
-			.limit	= 16
-		},
-		.mr_cache[14]	= {
-			.size	= 16,
-			.limit	= 8
-		},
-		.mr_cache[15]	= {
-			.size	= 8,
-			.limit	= 4
-		},
 	},
 	[3] = {
 		.mask		= MLX5_PROF_MASK_QP_SIZE,
@@ -960,7 +896,7 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev,
 
 	mlx5_pci_vsc_init(dev);
 
-	pci_enable_ptm(pdev, NULL);
+	pci_enable_ptm(pdev);
 
 	return 0;
 
@@ -1520,10 +1456,16 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
 	int err;
 
 	devl_lock(devlink);
+	if (dev->shd) {
+		err = devl_nested_devlink_set(dev->shd, devlink);
+		if (err)
+			goto unlock;
+	}
 	devl_register(devlink);
 	err = mlx5_init_one_devl_locked(dev);
 	if (err)
 		devl_unregister(devlink);
+unlock:
 	devl_unlock(devlink);
 	return err;
 }
@@ -1772,6 +1714,7 @@ static const int types[] = {
 	MLX5_CAP_CRYPTO,
 	MLX5_CAP_SHAMPO,
 	MLX5_CAP_ADV_RDMA,
+	MLX5_CAP_TLP_EMULATION,
 };
 
 static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
@@ -2005,6 +1948,13 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto pci_init_err;
 	}
 
+	err = mlx5_shd_init(dev);
+	if (err) {
+		mlx5_core_err(dev, "mlx5_shd_init failed with error code %d\n",
+			      err);
+		goto shd_init_err;
+	}
+
 	err = mlx5_init_one(dev);
 	if (err) {
 		mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n",
@@ -2018,6 +1968,8 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err_init_one:
+	mlx5_shd_uninit(dev);
+shd_init_err:
 	mlx5_pci_close(dev);
 pci_init_err:
 	mlx5_mdev_uninit(dev);
@@ -2039,6 +1991,7 @@ static void remove_one(struct pci_dev *pdev)
 	mlx5_drain_health_wq(dev);
 	mlx5_sriov_disable(pdev, false);
 	mlx5_uninit_one(dev);
+	mlx5_shd_uninit(dev);
 	mlx5_pci_close(dev);
 	mlx5_mdev_uninit(dev);
 	mlx5_adev_idx_free(dev->priv.adev_idx);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
new file mode 100644
index 0000000000000..bc33f95302df6
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <linux/mlx5/driver.h>
+#include <net/devlink.h>
+
+#include "sh_devlink.h"
+
+static const struct devlink_ops mlx5_shd_ops = {
+};
+
+int mlx5_shd_init(struct mlx5_core_dev *dev)
+{
+	u8 *vpd_data __free(kfree) = NULL;
+	struct pci_dev *pdev = dev->pdev;
+	unsigned int vpd_size, kw_len;
+	struct devlink *devlink;
+	char *sn, *end;
+	int start;
+	int err;
+
+	if (!mlx5_core_is_pf(dev))
+		return 0;
+
+	vpd_data = pci_vpd_alloc(pdev, &vpd_size);
+	if (IS_ERR(vpd_data)) {
+		err = PTR_ERR(vpd_data);
+		return err == -ENODEV ? 0 : err;
+	}
+	start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "V3", &kw_len);
+	if (start < 0) {
+		/* Fall-back to SN for older devices. */
+		start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
+						     PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len);
+		if (start < 0)
+			return -ENOENT;
+	}
+	sn = kstrndup(vpd_data + start, kw_len, GFP_KERNEL);
+	if (!sn)
+		return -ENOMEM;
+	/* Firmware may return spaces at the end of the string, strip it. */
+	end = strchrnul(sn, ' ');
+	*end = '\0';
+
+	/* Get or create shared devlink instance */
+	devlink = devlink_shd_get(sn, &mlx5_shd_ops, 0, pdev->dev.driver);
+	kfree(sn);
+	if (!devlink)
+		return -ENOMEM;
+
+	dev->shd = devlink;
+	return 0;
+}
+
+void mlx5_shd_uninit(struct mlx5_core_dev *dev)
+{
+	if (!dev->shd)
+		return;
+
+	devlink_shd_put(dev->shd);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h
new file mode 100644
index 0000000000000..8ab8d69402274
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_SH_DEVLINK_H__
+#define __MLX5_SH_DEVLINK_H__
+
+#include <linux/mlx5/driver.h>
+
+int mlx5_shd_init(struct mlx5_core_dev *dev);
+void mlx5_shd_uninit(struct mlx5_core_dev *dev);
+
+#endif /* __MLX5_SH_DEVLINK_H__ */
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h
index 779a083b92159..f7df5302e91a3 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic.h
@@ -98,6 +98,9 @@ struct fbnic_dev {
 
 	/* MDIO bus for PHYs */
 	struct mii_bus *mdio_bus;
+
+	/* In units of ms since API supports values in ms */
+	u16 ps_timeout;
 };
 
 /* Reserve entry 0 in the MSI-X "others" array until we have filled all
@@ -194,6 +197,38 @@ void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr);
 int fbnic_request_irq(struct fbnic_dev *dev, int nr, irq_handler_t handler,
 		      unsigned long flags, const char *name, void *data);
 void fbnic_free_irq(struct fbnic_dev *dev, int nr, void *data);
+
+/**
+ * enum fbnic_msix_self_test_codes - return codes from self test routines
+ *
+ * These are the codes returned from the self test routines and
+ * stored in the test result array indexed by the specific
+ * test name.
+ *
+ * @FBNIC_TEST_MSIX_SUCCESS: no errors
+ * @FBNIC_TEST_MSIX_NOMEM: allocation failure
+ * @FBNIC_TEST_MSIX_IRQ_REQ_FAIL: IRQ request failure
+ * @FBNIC_TEST_MSIX_MASK: masking failed to prevent IRQ
+ * @FBNIC_TEST_MSIX_UNMASK: unmasking failure w/ sw status set
+ * @FBNIC_TEST_MSIX_IRQ_CLEAR: interrupt when clearing mask
+ * @FBNIC_TEST_MSIX_NO_INTERRUPT: no interrupt when not masked
+ * @FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK: status not cleared, or mask not set
+ * @FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST: Bits are set after test
+ */
+enum fbnic_msix_self_test_codes {
+	FBNIC_TEST_MSIX_SUCCESS = 0,
+	FBNIC_TEST_MSIX_NOMEM = 5,
+	FBNIC_TEST_MSIX_IRQ_REQ_FAIL = 10,
+	FBNIC_TEST_MSIX_MASK = 20,
+	FBNIC_TEST_MSIX_UNMASK = 30,
+	FBNIC_TEST_MSIX_IRQ_CLEAR = 40,
+	FBNIC_TEST_MSIX_NO_INTERRUPT = 50,
+	FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK = 60,
+	FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST = 70,
+};
+
+enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd);
+
 void fbnic_free_irqs(struct fbnic_dev *fbd);
 int fbnic_alloc_irqs(struct fbnic_dev *fbd);
 
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c
index d9c0dc1c2af97..dc62d623e37c6 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c
@@ -147,3 +147,131 @@ int fbnic_csr_regs_len(struct fbnic_dev *fbd)
 
 	return len;
 }
+
+/* CSR register test data
+ *
+ * The register test will be used to verify hardware is behaving as expected.
+ *
+ * The test itself will have us writing to registers that should have no
+ * side effects due to us resetting after the test has been completed.
+ * While the test is being run the interface should be offline.
+ */
+struct fbnic_csr_reg_test_data {
+	int	reg;
+	u16	reg_offset;
+	u8	array_len;
+	u32	read;
+	u32	write;
+};
+
+#define FBNIC_QUEUE_REG_TEST(_name, _read, _write) { \
+	.reg = FBNIC_QUEUE(0) + FBNIC_QUEUE_##_name, \
+	.reg_offset = FBNIC_QUEUE_STRIDE, \
+	.array_len = 64, \
+	.read = _read, \
+	.write = _write \
+}
+
+static const struct fbnic_csr_reg_test_data pattern_test[] = {
+	FBNIC_QUEUE_REG_TEST(TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET,
+			     FBNIC_QUEUE_TWQ_CTL_RESET),
+	FBNIC_QUEUE_REG_TEST(TWQ0_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ0_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ0_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ0_BAH, ~0, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET,
+			     FBNIC_QUEUE_TWQ_CTL_RESET),
+	FBNIC_QUEUE_REG_TEST(TWQ1_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ1_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ1_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TWQ1_BAH, ~0, ~0),
+	FBNIC_QUEUE_REG_TEST(TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET,
+			     FBNIC_QUEUE_TCQ_CTL_RESET),
+	FBNIC_QUEUE_REG_TEST(TCQ_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(TCQ_SIZE, FBNIC_QUEUE_TCQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(TCQ_BAH, ~0, ~0),
+	FBNIC_QUEUE_REG_TEST(RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET,
+			     FBNIC_QUEUE_RCQ_CTL_RESET),
+	FBNIC_QUEUE_REG_TEST(RCQ_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(RCQ_SIZE, FBNIC_QUEUE_RCQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(RCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(RCQ_BAH, ~0, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET,
+			     FBNIC_QUEUE_BDQ_CTL_RESET),
+	FBNIC_QUEUE_REG_TEST(BDQ_HPQ_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_HPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAH, ~0, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_PPQ_PTRS, 0, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_PPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0),
+	FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAH, ~0, ~0),
+};
+
+static enum fbnic_reg_self_test_codes
+fbnic_csr_reg_pattern_test(struct fbnic_dev *fbd, int index,
+			   const struct fbnic_csr_reg_test_data *test_data)
+{
+	static const u32 pattern[] = { ~0, 0x5A5A5A5A, 0xA5A5A5A5, 0};
+	enum fbnic_reg_self_test_codes reg;
+	int i;
+
+	reg = test_data->reg + test_data->reg_offset * index;
+	for (i = 0; i < ARRAY_SIZE(pattern); i++) {
+		u32 val = pattern[i] & test_data->write;
+		u32 result;
+
+		wr32(fbd, reg, val);
+		result = rd32(fbd, reg);
+		val &= test_data->read;
+
+		if (result == val)
+			continue;
+
+		dev_err(fbd->dev,
+			"%s: reg 0x%06X failed, expected 0x%08X received 0x%08X\n",
+			__func__, reg, val, result);
+
+		/* Note that FBNIC_INTR_STATUS(0) could be tested and fail
+		 * and the result would not be reported since the register
+		 * offset is 0. However as that register isn't included in
+		 * the register test that isn't an issue.
+		 */
+		return reg;
+	}
+
+	return FBNIC_REG_TEST_SUCCESS;
+}
+
+/**
+ * fbnic_csr_regs_test() - Verify behavior of NIC registers
+ * @fbd: device to test
+ *
+ * This function is meant to test the bit values of various registers in
+ * the NIC device. Specifically this test will verify which bits are
+ * writable and which ones are not. It will write varying patterns of bits
+ * to the registers testing for sticky bits, or bits that are writable but
+ * should not be.
+ *
+ * Return: FBNIC_REG_TEST_SUCCESS on success, register number on failure
+ **/
+enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd)
+{
+	const struct fbnic_csr_reg_test_data *test_data;
+
+	for (test_data = pattern_test;
+	     test_data < pattern_test + ARRAY_SIZE(pattern_test); test_data++) {
+		u32 i;
+
+		for (i = 0; i < test_data->array_len; i++) {
+			enum fbnic_reg_self_test_codes reg =
+				fbnic_csr_reg_pattern_test(fbd, i, test_data);
+
+			if (reg)
+				return reg;
+		}
+	}
+
+	return FBNIC_REG_TEST_SUCCESS;
+}
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
index b717db879cd31..43de522af1727 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
@@ -6,6 +6,8 @@
 
 #include <linux/bitops.h>
 
+struct fbnic_dev;
+
 #define CSR_BIT(nr)		(1u << (nr))
 #define CSR_GENMASK(h, l)	GENMASK(h, l)
 
@@ -230,6 +232,7 @@ enum {
 #define FBNIC_INTR_MSIX_CTRL_VECTOR_MASK	CSR_GENMASK(7, 0)
 #define FBNIC_INTR_MSIX_CTRL_ENABLE		CSR_BIT(31)
 enum {
+	FBNIC_INTR_MSIX_CTRL_RXB_IDX	= 7,
 	FBNIC_INTR_MSIX_CTRL_PCS_IDX	= 34,
 };
 
@@ -560,6 +563,11 @@ enum {
 #define FBNIC_RXB_DROP_THLD_CNT			8
 #define FBNIC_RXB_DROP_THLD_ON			CSR_GENMASK(12, 0)
 #define FBNIC_RXB_DROP_THLD_OFF			CSR_GENMASK(25, 13)
+#define FBNIC_RXB_PAUSE_STORM(n)	(0x08019 + (n)) /* 0x20064 + 4*n */
+#define FBNIC_RXB_PAUSE_STORM_CNT		4
+#define FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL	CSR_BIT(20)
+#define FBNIC_RXB_PAUSE_STORM_THLD_TIME		CSR_GENMASK(19, 0)
+#define FBNIC_RXB_PAUSE_STORM_UNIT_WR	0x0801d		/* 0x20074 */
 #define FBNIC_RXB_ECN_THLD(n)		(0x0801e + (n)) /* 0x20078 + 4*n */
 #define FBNIC_RXB_ECN_THLD_CNT			8
 #define FBNIC_RXB_ECN_THLD_ON			CSR_GENMASK(12, 0)
@@ -596,6 +604,9 @@ enum {
 #define FBNIC_RXB_INTF_CREDIT_MASK2		CSR_GENMASK(11, 8)
 #define FBNIC_RXB_INTF_CREDIT_MASK3		CSR_GENMASK(15, 12)
 
+#define FBNIC_RXB_ERR_INTR_STS		0x08050		/* 0x20140 */
+#define FBNIC_RXB_ERR_INTR_STS_PS		CSR_GENMASK(15, 12)
+#define FBNIC_RXB_ERR_INTR_MASK		0x08052		/* 0x20148 */
 #define FBNIC_RXB_PAUSE_EVENT_CNT(n)	(0x08053 + (n))	/* 0x2014c + 4*n */
 #define FBNIC_RXB_DROP_FRMS_STS(n)	(0x08057 + (n))	/* 0x2015c + 4*n */
 #define FBNIC_RXB_DROP_BYTES_STS_L(n) \
@@ -618,6 +629,7 @@ enum {
 	FBNIC_RXB_ENQUEUE_INDICES	= 4
 };
 
+#define FBNIC_RXB_INTR_PS_COUNT(n)	(0x080e9 + (n))	/* 0x203a4 + 4*n */
 #define FBNIC_RXB_DRBO_FRM_CNT_SRC(n)	(0x080f9 + (n))	/* 0x203e4 + 4*n */
 #define FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(n) \
 					(0x080fd + (n))	/* 0x203f4 + 4*n */
@@ -636,6 +648,7 @@ enum {
 
 #define FBNIC_RXB_PBUF_FIFO_LEVEL(n)	(0x0811d + (n)) /* 0x20474 + 4*n */
 
+#define FBNIC_RXB_PAUSE_STORM_UNIT_RD	0x08125		/* 0x20494 */
 #define FBNIC_RXB_INTEGRITY_ERR(n)	(0x0812f + (n))	/* 0x204bc + 4*n */
 #define FBNIC_RXB_MAC_ERR(n)		(0x08133 + (n))	/* 0x204cc + 4*n */
 #define FBNIC_RXB_PARSER_ERR(n)		(0x08137 + (n))	/* 0x204dc + 4*n */
@@ -1210,4 +1223,21 @@ enum{
 	FBNIC_CSR_VERSION_V1_0_ASIC = 1,
 };
 
+/**
+ * enum fbnic_reg_self_test_codes - return codes from self test routines
+ *
+ * This is the code that is returned from the register self test
+ * routines.
+ *
+ * The test either returns success or the register number
+ * that failed during the test.
+ *
+ * @FBNIC_REG_TEST_SUCCESS: no errors
+ */
+enum fbnic_reg_self_test_codes {
+	FBNIC_REG_TEST_SUCCESS = 0,
+};
+
+enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd);
+
 #endif /* _FBNIC_CSR_H_ */
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
index 401c2196b9ff3..f14de23668544 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
@@ -126,6 +126,20 @@ static const struct fbnic_stat fbnic_gstrings_xdp_stats[] = {
 #define FBNIC_STATS_LEN \
 	(FBNIC_HW_STATS_LEN + FBNIC_XDP_STATS_LEN * FBNIC_MAX_XDPQS)
 
+enum fbnic_self_test_results {
+	TEST_REG = 0,
+	TEST_MSIX,
+	TEST_MBX,
+};
+
+static const char fbnic_gstrings_self_test[][ETH_GSTRING_LEN] = {
+	[TEST_REG]	= "Register test (offline)",
+	[TEST_MSIX]	= "MSI-X Interrupt test (offline)",
+	[TEST_MBX]      = "FW mailbox test (on/offline)",
+};
+
+#define FBNIC_TEST_LEN ARRAY_SIZE(fbnic_gstrings_self_test)
+
 static void
 fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 {
@@ -475,6 +489,10 @@ static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data)
 		for (i = 0; i < FBNIC_MAX_XDPQS; i++)
 			fbnic_get_xdp_queue_strings(&data, i);
 		break;
+	case ETH_SS_TEST:
+		memcpy(data, fbnic_gstrings_self_test,
+		       sizeof(fbnic_gstrings_self_test));
+		break;
 	}
 }
 
@@ -566,6 +584,8 @@ static int fbnic_get_sset_count(struct net_device *dev, int sset)
 	switch (sset) {
 	case ETH_SS_STATS:
 		return FBNIC_STATS_LEN;
+	case ETH_SS_TEST:
+		return FBNIC_TEST_LEN;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -1478,6 +1498,78 @@ fbnic_remove_rxfh_context(struct net_device *netdev,
 	return 0;
 }
 
+static int fbnic_ethtool_regs_test(struct net_device *netdev, u64 *data)
+{
+	struct fbnic_net *fbn = netdev_priv(netdev);
+	struct fbnic_dev *fbd = fbn->fbd;
+
+	*data = fbnic_csr_regs_test(fbd);
+
+	return !!*data;
+}
+
+/**
+ * fbnic_ethtool_msix_test - Verify behavior of NIC interrupts
+ * @netdev: netdev device to test
+ * @data: Pointer to results storage
+ *
+ * This function is meant to test the global interrupt registers and the
+ * PCIe IP MSI-X functionality. It essentially goes through and tests
+ * test various combinations of the set, clear, and mask bits in order to
+ * verify the behavior is as we expect it to be from the driver.
+ *
+ * Return: non-zero on failure.
+ **/
+static int fbnic_ethtool_msix_test(struct net_device *netdev, u64 *data)
+{
+	struct fbnic_net *fbn = netdev_priv(netdev);
+	struct fbnic_dev *fbd = fbn->fbd;
+
+	*data = fbnic_msix_test(fbd);
+
+	return !!*data;
+}
+
+static int fbnic_ethtool_mbx_self_test(struct net_device *netdev, u64 *data)
+{
+	struct fbnic_net *fbn = netdev_priv(netdev);
+	struct fbnic_dev *fbd = fbn->fbd;
+
+	*data = fbnic_fw_mbx_self_test(fbd);
+
+	return !!*data;
+}
+
+static void fbnic_self_test(struct net_device *netdev,
+			    struct ethtool_test *eth_test, u64 *data)
+{
+	bool if_running = netif_running(netdev);
+
+	if (fbnic_ethtool_mbx_self_test(netdev, &data[TEST_MBX]))
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+
+	if (!(eth_test->flags & ETH_TEST_FL_OFFLINE)) {
+		data[TEST_REG] = 0;
+		data[TEST_MSIX] = 0;
+		return;
+	}
+
+	if (if_running)
+		netif_close(netdev);
+
+	if (fbnic_ethtool_regs_test(netdev, &data[TEST_REG]))
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+
+	if (fbnic_ethtool_msix_test(netdev, &data[TEST_MSIX]))
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+
+	if (if_running && netif_open(netdev, NULL)) {
+		netdev_err(netdev,
+			   "Failed to re-initialize hardware following test\n");
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+	}
+}
+
 static void fbnic_get_channels(struct net_device *netdev,
 			       struct ethtool_channels *ch)
 {
@@ -1641,6 +1733,47 @@ static void fbnic_get_ts_stats(struct net_device *netdev,
 	}
 }
 
+static int fbnic_get_tunable(struct net_device *netdev,
+			     const struct ethtool_tunable *tun,
+			     void *data)
+{
+	struct fbnic_net *fbn = netdev_priv(netdev);
+	int err = 0;
+
+	switch (tun->id) {
+	case ETHTOOL_PFC_PREVENTION_TOUT:
+		*(u16 *)data = fbn->fbd->ps_timeout;
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
+
+	return err;
+}
+
+static int fbnic_set_tunable(struct net_device *netdev,
+			     const struct ethtool_tunable *tun,
+			     const void *data)
+{
+	struct fbnic_net *fbn = netdev_priv(netdev);
+	int err;
+
+	switch (tun->id) {
+	case ETHTOOL_PFC_PREVENTION_TOUT: {
+		u16 ps_timeout = *(u16 *)data;
+
+		err = fbnic_mac_ps_protect_to_config(fbn->fbd, ps_timeout);
+		break;
+	}
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
+
+	return err;
+}
+
 static int
 fbnic_get_module_eeprom_by_page(struct net_device *netdev,
 				const struct ethtool_module_eeprom *page_data,
@@ -1713,6 +1846,7 @@ fbnic_get_pause_stats(struct net_device *netdev,
 	struct fbnic_net *fbn = netdev_priv(netdev);
 	struct fbnic_mac_stats *mac_stats;
 	struct fbnic_dev *fbd = fbn->fbd;
+	u64 tx_ps_events;
 
 	mac_stats = &fbd->hw_stats.mac;
 
@@ -1720,6 +1854,8 @@ fbnic_get_pause_stats(struct net_device *netdev,
 
 	pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value;
 	pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value;
+	tx_ps_events = mac_stats->pause.tx_pause_storm_events.value;
+	pause_stats->tx_pause_storm_events = tx_ps_events;
 }
 
 static void
@@ -1896,6 +2032,7 @@ static const struct ethtool_ops fbnic_ethtool_ops = {
 	.get_pause_stats		= fbnic_get_pause_stats,
 	.get_pauseparam			= fbnic_phylink_get_pauseparam,
 	.set_pauseparam			= fbnic_phylink_set_pauseparam,
+	.self_test			= fbnic_self_test,
 	.get_strings			= fbnic_get_strings,
 	.get_ethtool_stats		= fbnic_get_ethtool_stats,
 	.get_sset_count			= fbnic_get_sset_count,
@@ -1915,6 +2052,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = {
 	.set_channels			= fbnic_set_channels,
 	.get_ts_info			= fbnic_get_ts_info,
 	.get_ts_stats			= fbnic_get_ts_stats,
+	.get_tunable			= fbnic_get_tunable,
+	.set_tunable			= fbnic_set_tunable,
 	.get_link_ksettings		= fbnic_phylink_ethtool_ksettings_get,
 	.get_fec_stats			= fbnic_get_fec_stats,
 	.get_fecparam			= fbnic_phylink_get_fecparam,
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
index 1f0b6350bef44..c2bad51bdde60 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
@@ -378,6 +378,37 @@ fbnic_fw_get_cmpl_by_type(struct fbnic_dev *fbd, u32 msg_type)
 	return cmpl_data;
 }
 
+/**
+ * fbnic_fw_xmit_test_msg - Create and transmit a test message to FW mailbox
+ * @fbd: FBNIC device structure
+ * @cmpl: fw completion struct
+ *
+ * Return: zero on success, negative value on failure
+ *
+ * Generates a single page mailbox test message and places it in the Tx
+ * mailbox queue. Expectation is that the FW will validate that the nested
+ * value matches the external values, and then will echo them back to us.
+ *
+ * Also sets a completion slot for use in the completion wait calls when
+ * the cmpl arg is non-NULL.
+ */
+int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd,
+			   struct fbnic_fw_completion *cmpl)
+{
+	struct fbnic_tlv_msg *test_msg;
+	int err;
+
+	test_msg = fbnic_tlv_test_create(fbd);
+	if (!test_msg)
+		return -ENOMEM;
+
+	err = fbnic_mbx_map_req_w_cmpl(fbd, test_msg, cmpl);
+	if (err)
+		free_page((unsigned long)test_msg);
+
+	return err;
+}
+
 /**
  * fbnic_fw_xmit_simple_msg - Transmit a simple single TLV message w/o data
  * @fbd: FBNIC device structure
@@ -1556,7 +1587,29 @@ int fbnic_fw_xmit_send_logs(struct fbnic_dev *fbd, bool enable,
 	return err;
 }
 
+static int
+fbnic_fw_parser_test(void *opaque, struct fbnic_tlv_msg **results)
+{
+	struct fbnic_fw_completion *cmpl;
+	struct fbnic_dev *fbd = opaque;
+	int err;
+
+	/* find cmpl */
+	cmpl = fbnic_fw_get_cmpl_by_type(fbd, FBNIC_TLV_MSG_ID_TEST);
+	if (!cmpl)
+		return -ENOSPC;
+
+	err = fbnic_tlv_parser_test(opaque, results);
+
+	cmpl->result = err;
+	complete(&cmpl->done);
+	fbnic_fw_put_cmpl(cmpl);
+
+	return err;
+}
+
 static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = {
+	FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, fbnic_fw_parser_test),
 	FBNIC_TLV_PARSER(FW_CAP_RESP, fbnic_fw_cap_resp_index,
 			 fbnic_fw_parse_cap_resp),
 	FBNIC_TLV_PARSER(OWNERSHIP_RESP, fbnic_ownership_resp_index,
@@ -1787,6 +1840,53 @@ void fbnic_mbx_flush_tx(struct fbnic_dev *fbd)
 	} while (time_is_after_jiffies(timeout));
 }
 
+/**
+ * fbnic_fw_mbx_self_test() - verify firmware interface
+ * @fbd: device to test
+ *
+ * This function tests the interfaces to/from the firmware.
+ *
+ * Return: See enum fbnic_fw_self_test_codes
+ **/
+enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd)
+{
+	enum fbnic_fw_self_test_codes err;
+	struct fbnic_fw_completion *cmpl;
+
+	/* Skip test if FW interface is not present */
+	if (!fbnic_fw_present(fbd))
+		return FBNIC_TEST_FW_NO_FIRMWARE;
+
+	cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_TEST);
+	if (!cmpl)
+		return FBNIC_TEST_FW_NO_CMPL;
+
+	/* Load a test message onto the FW mailbox interface
+	 * and arm the completion.
+	 */
+	err = fbnic_fw_xmit_test_msg(fbd, cmpl);
+	if (err) {
+		err = FBNIC_TEST_FW_NO_XMIT;
+		goto exit_free;
+	}
+
+	/* Verify we received a message back */
+	if (!fbnic_mbx_wait_for_cmpl(cmpl)) {
+		err = FBNIC_TEST_FW_NO_MSG;
+		goto exit_cleanup;
+	}
+
+	/* Verify there were no parsing errors */
+	if (cmpl->result)
+		err = FBNIC_TEST_FW_PARSE;
+exit_cleanup:
+	fbnic_mbx_clear_cmpl(fbd, cmpl);
+exit_free:
+	fbnic_fw_put_cmpl(cmpl);
+
+	return err;
+}
+
 int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd)
 {
 	struct fbnic_tlv_msg *mac_array;
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
index 8f7218900562e..d84723e4cfa36 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
@@ -104,6 +104,33 @@ void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd,
 void fbnic_mbx_poll(struct fbnic_dev *fbd);
 int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd);
 void fbnic_mbx_flush_tx(struct fbnic_dev *fbd);
+
+/**
+ * enum fbnic_fw_self_test_codes - return codes from self test routines
+ *
+ * These are the codes returned from the self test routines and
+ * stored in the test result array indexed by the specific
+ * test name.
+ *
+ * @FBNIC_TEST_FW_SUCCESS: test success
+ * @FBNIC_TEST_FW_NO_FIRMWARE: FW interface not present
+ * @FBNIC_TEST_FW_NO_CMPL: No completion available
+ * @FBNIC_TEST_FW_NO_XMIT: Could not xmit message
+ * @FBNIC_TEST_FW_NO_MSG: no message returned
+ * @FBNIC_TEST_FW_PARSE: returned message had parsing error
+ */
+enum fbnic_fw_self_test_codes {
+	FBNIC_TEST_FW_SUCCESS = 0,
+	FBNIC_TEST_FW_NO_FIRMWARE = 10,
+	FBNIC_TEST_FW_NO_CMPL = 20,
+	FBNIC_TEST_FW_NO_XMIT = 30,
+	FBNIC_TEST_FW_NO_MSG = 40,
+	FBNIC_TEST_FW_PARSE = 50,
+};
+
+enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd);
+int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd,
+			   struct fbnic_fw_completion *c);
 int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership);
 int fbnic_fw_init_heartbeat(struct fbnic_dev *fbd, bool poll);
 void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd);
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h
index aa3f429a9aeda..caea4be46762a 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h
@@ -54,6 +54,7 @@ struct fbnic_rmon_stats {
 struct fbnic_pause_stats {
 	struct fbnic_stat_counter tx_pause_frames;
 	struct fbnic_stat_counter rx_pause_frames;
+	struct fbnic_stat_counter tx_pause_storm_events;
 };
 
 struct fbnic_eth_mac_stats {
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c
index 02e8b0b257fed..5e383d40abc7e 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c
@@ -170,6 +170,8 @@ int fbnic_mac_request_irq(struct fbnic_dev *fbd)
 	fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX),
 		   FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE);
 
+	fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0);
+
 	fbd->mac_msix_vector = vector;
 
 	return 0;
@@ -238,6 +240,160 @@ void fbnic_free_irq(struct fbnic_dev *fbd, int nr, void *data)
 	free_irq(irq, data);
 }
 
+struct fbnic_msix_test_data {
+	struct fbnic_dev *fbd;
+	unsigned long test_msix_status[BITS_TO_LONGS(FBNIC_MAX_MSIX_VECS)];
+	int irq_vector[FBNIC_MAX_MSIX_VECS];
+};
+
+static irqreturn_t fbnic_irq_test(int irq, void *data)
+{
+	struct fbnic_msix_test_data *test_data = data;
+	struct fbnic_dev *fbd = test_data->fbd;
+	int i;
+
+	for (i = fbd->num_irqs; i--;) {
+		if (test_data->irq_vector[i] == irq) {
+			set_bit(i, test_data->test_msix_status);
+			break;
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * fbnic_msix_test - Verify behavior of NIC interrupts
+ * @fbd: device to test
+ *
+ * This function is meant to test the global interrupt registers and the
+ * PCIe IP MSI-X functionality. It essentially goes through and tests
+ * various combinations of the set, clear, and mask bits in order to
+ * verify the behavior is as we expect it to be from the driver.
+ *
+ * Return: See enum fbnic_msix_self_test_codes
+ **/
+enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd)
+{
+	enum fbnic_msix_self_test_codes result = FBNIC_TEST_MSIX_SUCCESS;
+	struct pci_dev *pdev = to_pci_dev(fbd->dev);
+	struct fbnic_msix_test_data *test_data;
+	u32 mask = 0;
+	int i;
+
+	/* Allocate bitmap and IRQ vector table */
+	test_data = kzalloc_obj(*test_data, GFP_KERNEL);
+
+	/* memory allocation failure */
+	if (!test_data)
+		return FBNIC_TEST_MSIX_NOMEM;
+
+	/* Initialize test data */
+	test_data->fbd = fbd;
+
+	for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
+		/* Add IRQ to vector table so it can be found */
+		test_data->irq_vector[i] = pci_irq_vector(pdev, i);
+
+		/* Enable the interrupt */
+		if (!fbnic_request_irq(fbd, i, fbnic_irq_test, 0,
+				       fbd->netdev->name, test_data))
+			continue;
+
+		while (i-- > FBNIC_NON_NAPI_VECTORS)
+			fbnic_free_irq(fbd, i, test_data);
+		kfree(test_data);
+
+		/* IRQ request failure */
+		return FBNIC_TEST_MSIX_IRQ_REQ_FAIL;
+	}
+
+	/* Test each bit individually */
+	for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
+		mask = 1U << (i % 32);
+
+		/* Start with mask set and interrupt cleared */
+		fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask);
+		fbnic_wrfl(fbd);
+		fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask);
+		fbnic_wrfl(fbd);
+
+		/* masking failure to prevent interrupt */
+		result = FBNIC_TEST_MSIX_MASK;
+
+		fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask);
+		fbnic_wrfl(fbd);
+		usleep_range(10000, 11000);
+
+		if (test_bit(i, test_data->test_msix_status))
+			break;
+
+		/* unmasking failure w/ sw status set */
+		result = FBNIC_TEST_MSIX_UNMASK;
+
+		fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask);
+		fbnic_wrfl(fbd);
+		usleep_range(10000, 11000);
+
+		if (!test_bit(i, test_data->test_msix_status))
+			break;
+
+		/* interrupt when clearing mask */
+		result = FBNIC_TEST_MSIX_IRQ_CLEAR;
+
+		clear_bit(i, test_data->test_msix_status);
+		fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask);
+		fbnic_wrfl(fbd);
+		usleep_range(10000, 11000);
+
+		if (test_bit(i, test_data->test_msix_status))
+			break;
+
+		/* interrupt not triggering when not masked */
+		result = FBNIC_TEST_MSIX_NO_INTERRUPT;
+
+		fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask);
+		fbnic_wrfl(fbd);
+		usleep_range(10000, 11000);
+
+		if (!test_bit(i, test_data->test_msix_status))
+			break;
+
+		/* status not cleared, or mask not set */
+		result = FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK;
+		if (mask & fbnic_rd32(fbd, FBNIC_INTR_STATUS(i / 32)))
+			break;
+		if (!(mask & fbnic_rd32(fbd, FBNIC_INTR_MASK(i / 32))))
+			break;
+
+		/* Result = 0 - Success */
+		result = FBNIC_TEST_MSIX_SUCCESS;
+
+		clear_bit(i, test_data->test_msix_status);
+	}
+
+	if (i < fbd->num_irqs) {
+		fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask);
+		fbnic_wrfl(fbd);
+		fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask);
+		fbnic_wrfl(fbd);
+		clear_bit(i, test_data->test_msix_status);
+	}
+
+	for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
+		/* Test for bits set after testing */
+		if (test_bit(i, test_data->test_msix_status))
+			result = FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST;
+
+		/* Free IRQ */
+		fbnic_free_irq(fbd, i, test_data);
+	}
+
+	kfree(test_data);
+
+	return result;
+}
+
 void fbnic_napi_name_irqs(struct fbnic_dev *fbd)
 {
 	unsigned int i;
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
index 9d0e4b2cc9ac4..53b7a938b4c27 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
@@ -143,6 +143,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd)
 #define FBNIC_DROP_EN_MASK	0x7d
 #define FBNIC_PAUSE_EN_MASK	0x14
 #define FBNIC_ECN_EN_MASK	0x10
+#define FBNIC_PS_EN_MASK	0x01
 
 struct fbnic_fifo_config {
 	unsigned int addr;
@@ -417,9 +418,29 @@ static void __fbnic_mac_stat_rd64(struct fbnic_dev *fbd, bool reset, u32 reg,
 	stat->reported = true;
 }
 
+static void fbnic_mac_stat_rd32(struct fbnic_dev *fbd, bool reset, u32 reg,
+				struct fbnic_stat_counter *stat)
+{
+	u32 new_reg_value;
+
+	new_reg_value = rd32(fbd, reg);
+	if (!reset)
+		stat->value += new_reg_value - stat->u.old_reg_value_32;
+	stat->u.old_reg_value_32 = new_reg_value;
+	stat->reported = true;
+}
+
 #define fbnic_mac_stat_rd64(fbd, reset, __stat, __CSR) \
 	__fbnic_mac_stat_rd64(fbd, reset, FBNIC_##__CSR##_L, &(__stat))
 
+bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd)
+{
+	u32 command_config;
+
+	command_config = rd32(fbd, FBNIC_MAC_COMMAND_CONFIG);
+	return !(command_config & FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS);
+}
+
 static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause)
 {
 	u32 rxb_pause_ctrl;
@@ -434,6 +455,49 @@ static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause)
 	wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, rxb_pause_ctrl);
 }
 
+static void
+fbnic_mac_ps_protect_to_reset(struct fbnic_dev *fbd, u16 timeout_ms)
+{
+	wr32(fbd, FBNIC_RXB_PAUSE_STORM_UNIT_WR, FBNIC_RXB_PS_CLK_DIV);
+
+	wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET),
+	     FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME,
+			FBNIC_MAC_RXB_PS_TO(timeout_ms)) |
+			FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL);
+	wrfl(fbd);
+	wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET),
+	     FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME,
+			FBNIC_MAC_RXB_PS_TO(timeout_ms)));
+}
+
+static void
+fbnic_mac_ps_protect_config(struct fbnic_dev *fbd, bool ps_protect)
+{
+	u16 timeout;
+	u32 reg;
+
+	ps_protect = ps_protect && fbd->ps_timeout;
+	timeout = ps_protect ? fbd->ps_timeout : FBNIC_MAC_PS_TO_DEFAULT_MS;
+
+	fbnic_mac_ps_protect_to_reset(fbd, timeout);
+
+	reg = rd32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL);
+	reg &= ~FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE;
+	reg |= FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE, ps_protect);
+	wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, reg);
+
+	/* Clear any pending interrupt status first */
+	wr32(fbd, FBNIC_RXB_ERR_INTR_STS,
+	     FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK));
+
+	/* Unmask the Network to Host PS interrupt if tx_pause is on */
+	reg = rd32(fbd, FBNIC_RXB_ERR_INTR_MASK);
+	reg |= FBNIC_RXB_ERR_INTR_STS_PS;
+	if (ps_protect)
+		reg &= ~FBNIC_RXB_ERR_INTR_STS_PS;
+	wr32(fbd, FBNIC_RXB_ERR_INTR_MASK, reg);
+}
+
 static int fbnic_mac_get_link_event(struct fbnic_dev *fbd)
 {
 	u32 intr_mask = rd32(fbd, FBNIC_SIG_PCS_INTR_STS);
@@ -658,6 +722,7 @@ static void fbnic_mac_link_up_asic(struct fbnic_dev *fbd,
 	u32 cmd_cfg, mac_ctrl;
 
 	fbnic_mac_tx_pause_config(fbd, tx_pause);
+	fbnic_mac_ps_protect_config(fbd, tx_pause);
 
 	cmd_cfg = __fbnic_mac_cmd_config_asic(fbd, tx_pause, rx_pause);
 	mac_ctrl = rd32(fbd, FBNIC_SIG_MAC_IN0);
@@ -759,6 +824,9 @@ fbnic_mac_get_pause_stats(struct fbnic_dev *fbd, bool reset,
 			    MAC_STAT_TX_XOFF_STB);
 	fbnic_mac_stat_rd64(fbd, reset, pause_stats->rx_pause_frames,
 			    MAC_STAT_RX_XOFF_STB);
+	fbnic_mac_stat_rd32(fbd, reset,
+			    FBNIC_RXB_INTR_PS_COUNT(FBNIC_RXB_INTF_NET),
+			    &pause_stats->tx_pause_storm_events);
 }
 
 static void
@@ -918,3 +986,46 @@ int fbnic_mac_init(struct fbnic_dev *fbd)
 
 	return 0;
 }
+
+int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout_ms)
+{
+	u16 old_timeout_ms = fbd->ps_timeout;
+
+	if (timeout_ms == old_timeout_ms)
+		return 0;
+
+	if (timeout_ms == PFC_STORM_PREVENTION_AUTO)
+		timeout_ms = FBNIC_MAC_PS_TO_DEFAULT_MS;
+
+	if (timeout_ms > FBNIC_MAC_PS_TO_MAX_MS)
+		return -EINVAL;
+
+	fbd->ps_timeout = timeout_ms;
+
+	if (!fbnic_mac_check_tx_pause(fbd))
+		return 0;
+
+	if (timeout_ms == 0)
+		fbnic_mac_ps_protect_config(fbd, false);
+	else if (old_timeout_ms == 0)
+		fbnic_mac_ps_protect_config(fbd, true);
+	else
+		fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout);
+
+	return 0;
+}
+
+void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd)
+{
+	u32 rxb_err_sts = rd32(fbd, FBNIC_RXB_ERR_INTR_STS);
+
+	/* Check if pause storm interrupt for network was triggered */
+	if (rxb_err_sts & FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS,
+				     FBNIC_PS_EN_MASK)) {
+		/* Write 1 to clear the interrupt status first */
+		wr32(fbd, FBNIC_RXB_ERR_INTR_STS,
+		     FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK));
+
+		fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout);
+	}
+}
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h
index f08fe8b7c497a..10f30e0e8f691 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h
@@ -8,6 +8,30 @@
 
 struct fbnic_dev;
 
+/* The RXB clock runs at 600 MHZ in the ASIC and the PAUSE_STORM_UNIT_WR
+ * is 10us granularity, so set the clock to 6000 (0x1770)
+ */
+#define FBNIC_RXB_PS_CLK_DIV		0x1770
+
+/* Convert milliseconds to pause storm timeout units (10us granularity) */
+#define FBNIC_MAC_RXB_PS_TO(ms)		((ms) * 100)
+
+/* Convert pause storm timeout units (10us granularity) to milliseconds */
+#define FBNIC_MAC_RXB_PS_TO_MS(ps)	((ps) / 100)
+
+/* Set the default timer to 500ms, which should be longer than any
+ * reasonable period of continuous pausing. The service task, which runs
+ * once per second, periodically resets the pause storm trigger.
+ *
+ * As a result, on a functioning system, if pause continues, we enforce
+ * a duty cycle determined by the configured pause storm timeout (50%
+ * default). A crashed system will not have the service task and therefore
+ * pause will remain disabled until reboot recovery.
+ */
+#define FBNIC_MAC_PS_TO_DEFAULT_MS	500
+#define FBNIC_MAC_PS_TO_MAX_MS	\
+	FBNIC_MAC_RXB_PS_TO_MS(FIELD_MAX(FBNIC_RXB_PAUSE_STORM_THLD_TIME))
+
 #define FBNIC_MAX_JUMBO_FRAME_SIZE	9742
 
 /* States loosely based on section 136.8.11.7.5 of IEEE 802.3-2022 Ethernet
@@ -119,4 +143,7 @@ struct fbnic_mac {
 
 int fbnic_mac_init(struct fbnic_dev *fbd);
 void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec);
+int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout);
+void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd);
+bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd);
 #endif /* _FBNIC_MAC_H_ */
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
index 3fa9d1910daa1..e3aebbe3656d5 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
@@ -220,6 +220,9 @@ static void fbnic_service_task(struct work_struct *work)
 
 	fbnic_get_hw_stats32(fbd);
 
+	if (fbd->ps_timeout && fbnic_mac_check_tx_pause(fbd))
+		fbnic_mac_ps_protect_handler(fbd);
+
 	fbnic_fw_check_heartbeat(fbd);
 
 	fbnic_health_check(fbd);
@@ -296,6 +299,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Populate driver with hardware-specific info and handlers */
 	fbd->max_num_queues = info->max_num_queues;
 
+	fbd->ps_timeout = FBNIC_MAC_PS_TO_DEFAULT_MS;
+
 	pci_set_master(pdev);
 	pci_save_state(pdev);
 
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
index 517ed8b2f1cb7..c55d4f76a5fc0 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
@@ -551,6 +551,172 @@ int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results)
 	return -EBADMSG;
 }
 
+#define FBNIC_TLV_TEST_STRING_LEN	32
+
+struct fbnic_tlv_test {
+	u64	test_u64;
+	s64	test_s64;
+	u32	test_u32;
+	s32	test_s32;
+	u16	test_u16;
+	s16	test_s16;
+	u8	test_mac[ETH_ALEN];
+	u8	test_mac_array[4][ETH_ALEN];
+	u8	test_true;
+	u8	test_false;
+	char	test_string[FBNIC_TLV_TEST_STRING_LEN];
+};
+
+static struct fbnic_tlv_test test_struct;
+
+const struct fbnic_tlv_index fbnic_tlv_test_index[] = {
+	FBNIC_TLV_ATTR_U64(FBNIC_TLV_TEST_MSG_U64),
+	FBNIC_TLV_ATTR_S64(FBNIC_TLV_TEST_MSG_S64),
+	FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U32),
+	FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S32),
+	FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U16),
+	FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S16),
+	FBNIC_TLV_ATTR_MAC_ADDR(FBNIC_TLV_TEST_MSG_MAC_ADDR),
+	FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_TRUE),
+	FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_FALSE),
+	FBNIC_TLV_ATTR_STRING(FBNIC_TLV_TEST_MSG_STRING,
+			      FBNIC_TLV_TEST_STRING_LEN),
+	FBNIC_TLV_ATTR_ARRAY(FBNIC_TLV_TEST_MSG_ARRAY),
+	FBNIC_TLV_ATTR_NESTED(FBNIC_TLV_TEST_MSG_NESTED),
+	FBNIC_TLV_ATTR_LAST
+};
+
+static void fbnic_tlv_test_struct_init(void)
+{
+	int i = FBNIC_TLV_TEST_STRING_LEN - 1;
+
+	/* Populate the struct with random data */
+	get_random_once(&test_struct,
+			offsetof(struct fbnic_tlv_test, test_string) + i);
+
+	/* Force true/false to their expected values */
+	test_struct.test_false = false;
+	test_struct.test_true = true;
+
+	/* Convert test_string to a true ASCII string */
+	test_struct.test_string[i] = '\0';
+	while (i--) {
+		/* Force characters into displayable range */
+		if (test_struct.test_string[i] < 64 ||
+		    test_struct.test_string[i] >= 96) {
+			test_struct.test_string[i] %= 32;
+			test_struct.test_string[i] += 64;
+		}
+	}
+}
+
+static int fbnic_tlv_test_attr_data(struct fbnic_tlv_msg *msg)
+{
+	struct fbnic_tlv_msg *array;
+	int err, i;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U64,
+				     test_struct.test_u64);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S64,
+				     test_struct.test_s64);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U32,
+				     test_struct.test_u32);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S32,
+				     test_struct.test_s32);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U16,
+				     test_struct.test_u16);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S16,
+				     test_struct.test_s16);
+	if (err)
+		return err;
+
+	err = fbnic_tlv_attr_put_value(msg, FBNIC_TLV_TEST_MSG_MAC_ADDR,
+				       test_struct.test_mac, ETH_ALEN);
+	if (err)
+		return err;
+
+	/* Start MAC address array */
+	array = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_ARRAY);
+	if (!array)
+		return -ENOSPC;
+
+	for (i = 0; i < 4; i++) {
+		err = fbnic_tlv_attr_put_value(array,
+					       FBNIC_TLV_TEST_MSG_MAC_ADDR,
+					       test_struct.test_mac_array[i],
+					       ETH_ALEN);
+		if (err)
+			return err;
+	}
+
+	/* Close array */
+	fbnic_tlv_attr_nest_stop(msg);
+
+	err = fbnic_tlv_attr_put_flag(msg, FBNIC_TLV_TEST_MSG_FLAG_TRUE);
+	if (err)
+		return err;
+
+	return fbnic_tlv_attr_put_string(msg, FBNIC_TLV_TEST_MSG_STRING,
+					 test_struct.test_string);
+}
+
+/**
+ * fbnic_tlv_test_create - Allocate a test message and fill it w/ data
+ * @fbd: FBNIC device structure
+ *
+ * Return: NULL on failure to allocate or pointer to new TLV test message.
+ **/
+struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd)
+{
+	struct fbnic_tlv_msg *msg, *nest;
+	int err;
+
+	msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_TEST);
+	if (!msg)
+		return NULL;
+
+	/* Randomize struct data */
+	fbnic_tlv_test_struct_init();
+
+	/* Add first level of data to message */
+	err = fbnic_tlv_test_attr_data(msg);
+	if (err)
+		goto free_message;
+
+	/* Start second level nested */
+	nest = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_NESTED);
+	if (!nest)
+		goto free_message;
+
+	/* Add nested data */
+	err = fbnic_tlv_test_attr_data(nest);
+	if (err)
+		goto free_message;
+
+	/* Close nest and report full message */
+	fbnic_tlv_attr_nest_stop(msg);
+
+	return msg;
+free_message:
+	free_page((unsigned long)msg);
+	return NULL;
+}
+
 void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src)
 {
 	u8 *mac_addr;
@@ -558,3 +724,113 @@ void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src)
 	mac_addr = fbnic_tlv_attr_get_value_ptr(src);
 	memcpy(dest, mac_addr, ETH_ALEN);
 }
+
+/**
+ * fbnic_tlv_parser_test_attr - Function loading test attributes into structure
+ * @str: Test structure to load
+ * @results: Pointer to results array
+ *
+ * Copies attributes into structure. Any attribute that doesn't exist in the
+ * results array is not populated.
+ **/
+static void fbnic_tlv_parser_test_attr(struct fbnic_tlv_test *str,
+				       struct fbnic_tlv_msg **results)
+{
+	struct fbnic_tlv_msg *array_results[4];
+	struct fbnic_tlv_msg *attr;
+	char *string = NULL;
+	int i, err;
+
+	str->test_u64 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U64);
+	str->test_u32 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U32);
+	str->test_u16 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U16);
+
+	str->test_s64 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S64);
+	str->test_s32 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S32);
+	str->test_s16 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S16);
+
+	attr = results[FBNIC_TLV_TEST_MSG_MAC_ADDR];
+	if (attr)
+		fbnic_tlv_attr_addr_copy(str->test_mac, attr);
+
+	attr = results[FBNIC_TLV_TEST_MSG_ARRAY];
+	if (attr) {
+		int len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1;
+
+		err = fbnic_tlv_attr_parse_array(&attr[1], len,
+						 array_results,
+						 fbnic_tlv_test_index,
+						 FBNIC_TLV_TEST_MSG_MAC_ADDR,
+						 4);
+		if (!err) {
+			for (i = 0; i < 4 && array_results[i]; i++)
+				fbnic_tlv_attr_addr_copy(str->test_mac_array[i],
+							 array_results[i]);
+		}
+	}
+
+	str->test_true = !!results[FBNIC_TLV_TEST_MSG_FLAG_TRUE];
+	str->test_false = !!results[FBNIC_TLV_TEST_MSG_FLAG_FALSE];
+
+	attr = results[FBNIC_TLV_TEST_MSG_STRING];
+	if (attr) {
+		string = fbnic_tlv_attr_get_value_ptr(attr);
+		strscpy(str->test_string, string, FBNIC_TLV_TEST_STRING_LEN);
+	}
+}
+
+static void fbnic_tlv_test_dump(struct fbnic_tlv_test *value, char *prefix)
+{
+	print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_OFFSET, 16, 1,
+		       value, sizeof(*value), true);
+}
+
+/**
+ * fbnic_tlv_parser_test - Function for parsing and testing test message
+ * @opaque: Unused value
+ * @results: Results of parser output
+ *
+ * Return: negative value on error, or 0 on success.
+ *
+ * Parses attributes to structures and compares the structure to the
+ * expected test value that should have been used to populate the message.
+ *
+ * Used to verify message generation and parser are working correctly.
+ **/
+int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results)
+{
+	struct fbnic_tlv_msg *nest_results[FBNIC_TLV_RESULTS_MAX] = { 0 };
+	struct fbnic_tlv_test result_struct;
+	struct fbnic_tlv_msg *attr;
+	int err;
+
+	memset(&result_struct, 0, sizeof(result_struct));
+	fbnic_tlv_parser_test_attr(&result_struct, results);
+
+	if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) {
+		fbnic_tlv_test_dump(&result_struct, "fbnic: found - ");
+		fbnic_tlv_test_dump(&test_struct, "fbnic: expected - ");
+		return -EINVAL;
+	}
+
+	attr = results[FBNIC_TLV_TEST_MSG_NESTED];
+	if (!attr)
+		return -EINVAL;
+
+	err = fbnic_tlv_attr_parse(&attr[1],
+				   le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1,
+				   nest_results, fbnic_tlv_test_index);
+	if (err)
+		return err;
+
+	memset(&result_struct, 0, sizeof(result_struct));
+	fbnic_tlv_parser_test_attr(&result_struct, nest_results);
+
+	if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) {
+		fbnic_tlv_test_dump(&result_struct, "fbnic: found - ");
+		fbnic_tlv_test_dump(&test_struct, "fbnic: expected - ");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h
index 3508b46ebdd00..9c4e4759394a3 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h
@@ -9,6 +9,8 @@
 #include <linux/const.h>
 #include <linux/types.h>
 
+struct fbnic_dev;
+
 #define FBNIC_TLV_MSG_ALIGN(len)	ALIGN(len, sizeof(u32))
 #define FBNIC_TLV_MSG_SIZE(len)		\
 		(FBNIC_TLV_MSG_ALIGN(len) / sizeof(u32))
@@ -153,6 +155,31 @@ int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results);
 #define fta_get_str(_results, _id, _dst, _dstsize) \
 	fbnic_tlv_attr_get_string(_results[_id], _dst, _dstsize)
 
+#define FBNIC_TLV_MSG_ID_TEST	0
+
+enum fbnic_tlv_test_attr_id {
+	FBNIC_TLV_TEST_MSG_U64,
+	FBNIC_TLV_TEST_MSG_S64,
+	FBNIC_TLV_TEST_MSG_U32,
+	FBNIC_TLV_TEST_MSG_S32,
+	FBNIC_TLV_TEST_MSG_U16,
+	FBNIC_TLV_TEST_MSG_S16,
+	FBNIC_TLV_TEST_MSG_MAC_ADDR,
+	FBNIC_TLV_TEST_MSG_FLAG_TRUE,
+	FBNIC_TLV_TEST_MSG_FLAG_FALSE,
+	FBNIC_TLV_TEST_MSG_STRING,
+	FBNIC_TLV_TEST_MSG_NESTED,
+	FBNIC_TLV_TEST_MSG_ARRAY,
+	FBNIC_TLV_TEST_MSG_MAX
+};
+
+extern const struct fbnic_tlv_index fbnic_tlv_test_index[];
+struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd);
+int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results);
+
+#define FBNIC_TLV_MSG_TEST \
+	FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, \
+			 fbnic_tlv_parser_test)
 #define FBNIC_TLV_MSG_ERROR \
 	FBNIC_TLV_PARSER(UNKNOWN, NULL, fbnic_tlv_parser_error)
 #endif /* _FBNIC_TLV_H_ */
diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c
index 8a3c1ecc7866c..9195419ecee02 100644
--- a/drivers/net/ethernet/microchip/lan743x_ethtool.c
+++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c
@@ -1079,6 +1079,13 @@ static int lan743x_ethtool_set_eee(struct net_device *netdev,
 	return phylink_ethtool_set_eee(adapter->phylink, eee);
 }
 
+static int lan743x_ethtool_nway_reset(struct net_device *netdev)
+{
+	struct lan743x_adapter *adapter = netdev_priv(netdev);
+
+	return phylink_ethtool_nway_reset(adapter->phylink);
+}
+
 static int
 lan743x_ethtool_set_link_ksettings(struct net_device *netdev,
 				   const struct ethtool_link_ksettings *cmd)
@@ -1369,6 +1376,7 @@ const struct ethtool_ops lan743x_ethtool_ops = {
 	.set_rxfh = lan743x_ethtool_set_rxfh,
 	.get_rxfh_fields = lan743x_ethtool_get_rxfh_fields,
 	.get_ts_info = lan743x_ethtool_get_ts_info,
+	.nway_reset = lan743x_ethtool_nway_reset,
 	.get_eee = lan743x_ethtool_get_eee,
 	.set_eee = lan743x_ethtool_set_eee,
 	.get_link_ksettings = lan743x_ethtool_get_link_ksettings,
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c
index 8c67ff1d411ba..35815fb58080f 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c
@@ -151,7 +151,7 @@ enum sparx5_cal_bw sparx5_get_port_cal_speed(struct sparx5 *sparx5, u32 portno)
 }
 
 /* Auto configure the QSYS calendar based on port configuration */
-int sparx5_config_auto_calendar(struct sparx5 *sparx5)
+static int sparx5_config_auto_calendar(struct sparx5 *sparx5)
 {
 	const struct sparx5_consts *consts = sparx5->data->consts;
 	u32 cal[7], value, idx, portno;
@@ -578,7 +578,7 @@ static int sparx5_dsm_calendar_update(struct sparx5 *sparx5, u32 taxi,
 }
 
 /* Configure the DSM calendar based on port configuration */
-int sparx5_config_dsm_calendar(struct sparx5 *sparx5)
+static int sparx5_config_dsm_calendar(struct sparx5 *sparx5)
 {
 	const struct sparx5_ops *ops = sparx5->data->ops;
 	int taxi;
@@ -610,3 +610,14 @@ int sparx5_config_dsm_calendar(struct sparx5 *sparx5)
 	kfree(data);
 	return err;
 }
+
+int sparx5_calendar_init(struct sparx5 *sparx5)
+{
+	int err;
+
+	err = sparx5_config_auto_calendar(sparx5);
+	if (err)
+		return err;
+
+	return sparx5_config_dsm_calendar(sparx5);
+}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
index 049541eeaae0d..d42c57bead89d 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c
@@ -1244,7 +1244,7 @@ const struct ethtool_ops sparx5_ethtool_ops = {
 	.set_pauseparam         = sparx5_set_pauseparam,
 };
 
-int sparx_stats_init(struct sparx5 *sparx5)
+int sparx5_stats_init(struct sparx5 *sparx5)
 {
 	const struct sparx5_consts *consts = sparx5->data->consts;
 	char queue_name[32];
@@ -1278,3 +1278,10 @@ int sparx_stats_init(struct sparx5 *sparx5)
 
 	return 0;
 }
+
+void sparx5_stats_deinit(struct sparx5 *sparx5)
+{
+	cancel_delayed_work_sync(&sparx5->stats_work);
+	destroy_workqueue(sparx5->stats_queue);
+	mutex_destroy(&sparx5->queue_stats_lock);
+}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
index f5584244612cd..2bf9c5f64151c 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
@@ -419,7 +419,7 @@ static void sparx5_mact_handle_entry(struct sparx5 *sparx5,
 				  true);
 }
 
-void sparx5_mact_pull_work(struct work_struct *work)
+static void sparx5_mact_pull_work(struct work_struct *work)
 {
 	struct delayed_work *del_work = to_delayed_work(work);
 	struct sparx5 *sparx5 = container_of(del_work, struct sparx5,
@@ -489,8 +489,10 @@ void sparx5_set_ageing(struct sparx5 *sparx5, int msecs)
 		 LRN_AUTOAGE_CFG(0));
 }
 
-void sparx5_mact_init(struct sparx5 *sparx5)
+int sparx5_mact_init(struct sparx5 *sparx5)
 {
+	char queue_name[32];
+
 	mutex_init(&sparx5->lock);
 
 	/*  Flush MAC table */
@@ -502,4 +504,32 @@ void sparx5_mact_init(struct sparx5 *sparx5)
 		dev_warn(sparx5->dev, "MAC flush error\n");
 
 	sparx5_set_ageing(sparx5, BR_DEFAULT_AGEING_TIME / HZ * 1000);
+
+	/* Add host mode BC address (points only to CPU) */
+	sparx5_mact_learn(sparx5, sparx5_get_pgid(sparx5, PGID_CPU),
+			  (unsigned char[]){0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+			  NULL_VID);
+
+	mutex_init(&sparx5->mdb_lock);
+	INIT_LIST_HEAD(&sparx5->mdb_entries);
+	mutex_init(&sparx5->mact_lock);
+	INIT_LIST_HEAD(&sparx5->mact_entries);
+	snprintf(queue_name, sizeof(queue_name), "%s-mact",
+		 dev_name(sparx5->dev));
+	sparx5->mact_queue = create_singlethread_workqueue(queue_name);
+	if (!sparx5->mact_queue)
+		return -ENOMEM;
+
+	INIT_DELAYED_WORK(&sparx5->mact_work, sparx5_mact_pull_work);
+	queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work,
+			   SPX5_MACT_PULL_DELAY);
+
+	return 0;
+}
+
+void sparx5_mact_deinit(struct sparx5 *sparx5)
+{
+	cancel_delayed_work_sync(&sparx5->mact_work);
+	destroy_workqueue(sparx5->mact_queue);
+	mutex_destroy(&sparx5->mact_lock);
 }
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
index 9befd0719f74e..dad713e9ddd5b 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
@@ -658,6 +658,58 @@ static int sparx5_qlim_set(struct sparx5 *sparx5)
 	return 0;
 }
 
+static int sparx5_frame_io_init(struct sparx5 *sparx5)
+{
+	const struct sparx5_ops *ops = sparx5->data->ops;
+	int err = -ENXIO;
+
+	/* Start Frame DMA with fallback to register based INJ/XTR */
+	if (sparx5->fdma_irq >= 0) {
+		if (GCB_CHIP_ID_REV_ID_GET(sparx5->chip_id) > 0 ||
+		    !is_sparx5(sparx5))
+			err = devm_request_irq(sparx5->dev,
+					       sparx5->fdma_irq,
+					       sparx5_fdma_handler,
+					       0,
+					       "sparx5-fdma", sparx5);
+		if (!err) {
+			err = ops->fdma_init(sparx5);
+			if (!err)
+				sparx5_fdma_start(sparx5);
+		}
+		if (err)
+			sparx5->fdma_irq = -ENXIO;
+	} else {
+		sparx5->fdma_irq = -ENXIO;
+	}
+	if (err && sparx5->xtr_irq >= 0) {
+		err = devm_request_irq(sparx5->dev, sparx5->xtr_irq,
+				       sparx5_xtr_handler, IRQF_SHARED,
+				       "sparx5-xtr", sparx5);
+		if (!err)
+			err = sparx5_manual_injection_mode(sparx5);
+		if (err)
+			sparx5->xtr_irq = -ENXIO;
+	} else {
+		sparx5->xtr_irq = -ENXIO;
+	}
+
+	return err;
+}
+
+static void sparx5_frame_io_deinit(struct sparx5 *sparx5)
+{
+	if (sparx5->xtr_irq >= 0) {
+		disable_irq(sparx5->xtr_irq);
+		sparx5->xtr_irq = -ENXIO;
+	}
+	if (sparx5->fdma_irq >= 0) {
+		disable_irq(sparx5->fdma_irq);
+		sparx5->data->ops->fdma_deinit(sparx5);
+		sparx5->fdma_irq = -ENXIO;
+	}
+}
+
 /* Some boards needs to map the SGPIO for signal detect explicitly to the
  * port module
  */
@@ -683,14 +735,10 @@ static void sparx5_board_init(struct sparx5 *sparx5)
 					GCB_HW_SGPIO_TO_SD_MAP_CFG(idx));
 }
 
-static int sparx5_start(struct sparx5 *sparx5)
+static void sparx5_forwarding_init(struct sparx5 *sparx5)
 {
-	u8 broadcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 	const struct sparx5_consts *consts = sparx5->data->consts;
-	const struct sparx5_ops *ops = sparx5->data->ops;
-	char queue_name[32];
 	u32 idx;
-	int err;
 
 	/* Setup own UPSIDs */
 	for (idx = 0; idx < consts->n_own_upsids; idx++) {
@@ -728,117 +776,9 @@ static int sparx5_start(struct sparx5 *sparx5)
 			 ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA,
 			 sparx5, ANA_CL_FILTER_CTRL(idx));
 
-	/* Init MAC table, ageing */
-	sparx5_mact_init(sparx5);
-
-	/* Init PGID table arbitrator */
-	sparx5_pgid_init(sparx5);
-
-	/* Setup VLANs */
-	sparx5_vlan_init(sparx5);
-
-	/* Add host mode BC address (points only to CPU) */
-	sparx5_mact_learn(sparx5, sparx5_get_pgid(sparx5, PGID_CPU), broadcast,
-			  NULL_VID);
-
 	/* Enable queue limitation watermarks */
 	sparx5_qlim_set(sparx5);
 
-	err = sparx5_config_auto_calendar(sparx5);
-	if (err)
-		return err;
-
-	err = sparx5_config_dsm_calendar(sparx5);
-	if (err)
-		return err;
-
-	/* Init stats */
-	err = sparx_stats_init(sparx5);
-	if (err)
-		return err;
-
-	/* Init mact_sw struct */
-	mutex_init(&sparx5->mact_lock);
-	INIT_LIST_HEAD(&sparx5->mact_entries);
-	snprintf(queue_name, sizeof(queue_name), "%s-mact",
-		 dev_name(sparx5->dev));
-	sparx5->mact_queue = create_singlethread_workqueue(queue_name);
-	if (!sparx5->mact_queue)
-		return -ENOMEM;
-
-	INIT_DELAYED_WORK(&sparx5->mact_work, sparx5_mact_pull_work);
-	queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work,
-			   SPX5_MACT_PULL_DELAY);
-
-	mutex_init(&sparx5->mdb_lock);
-	INIT_LIST_HEAD(&sparx5->mdb_entries);
-
-	err = sparx5_register_netdevs(sparx5);
-	if (err)
-		return err;
-
-	sparx5_board_init(sparx5);
-	err = sparx5_register_notifier_blocks(sparx5);
-	if (err)
-		return err;
-
-	err = sparx5_vcap_init(sparx5);
-	if (err) {
-		sparx5_unregister_notifier_blocks(sparx5);
-		return err;
-	}
-
-	/* Start Frame DMA with fallback to register based INJ/XTR */
-	err = -ENXIO;
-	if (sparx5->fdma_irq >= 0) {
-		if (GCB_CHIP_ID_REV_ID_GET(sparx5->chip_id) > 0 ||
-		    !is_sparx5(sparx5))
-			err = devm_request_irq(sparx5->dev,
-					       sparx5->fdma_irq,
-					       sparx5_fdma_handler,
-					       0,
-					       "sparx5-fdma", sparx5);
-		if (!err) {
-			err = ops->fdma_init(sparx5);
-			if (!err)
-				sparx5_fdma_start(sparx5);
-		}
-		if (err)
-			sparx5->fdma_irq = -ENXIO;
-	} else {
-		sparx5->fdma_irq = -ENXIO;
-	}
-	if (err && sparx5->xtr_irq >= 0) {
-		err = devm_request_irq(sparx5->dev, sparx5->xtr_irq,
-				       sparx5_xtr_handler, IRQF_SHARED,
-				       "sparx5-xtr", sparx5);
-		if (!err)
-			err = sparx5_manual_injection_mode(sparx5);
-		if (err)
-			sparx5->xtr_irq = -ENXIO;
-	} else {
-		sparx5->xtr_irq = -ENXIO;
-	}
-
-	if (sparx5->ptp_irq >= 0 &&
-	    sparx5_has_feature(sparx5, SPX5_FEATURE_PTP)) {
-		err = devm_request_threaded_irq(sparx5->dev, sparx5->ptp_irq,
-						NULL, ops->ptp_irq_handler,
-						IRQF_ONESHOT, "sparx5-ptp",
-						sparx5);
-		if (err)
-			sparx5->ptp_irq = -ENXIO;
-
-		sparx5->ptp = 1;
-	}
-
-	return err;
-}
-
-static void sparx5_cleanup_ports(struct sparx5 *sparx5)
-{
-	sparx5_unregister_netdevs(sparx5);
-	sparx5_destroy_netdevs(sparx5);
 }
 
 static int mchp_sparx5_probe(struct platform_device *pdev)
@@ -999,9 +939,14 @@ static int mchp_sparx5_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = sparx5_start(sparx5);
+	sparx5_pgid_init(sparx5);
+	sparx5_vlan_init(sparx5);
+	sparx5_board_init(sparx5);
+	sparx5_forwarding_init(sparx5);
+
+	err = sparx5_calendar_init(sparx5);
 	if (err) {
-		dev_err(sparx5->dev, "Start failed\n");
+		dev_err(sparx5->dev, "Failed to initialize calendar\n");
 		goto cleanup_ports;
 	}
 
@@ -1011,20 +956,66 @@ static int mchp_sparx5_probe(struct platform_device *pdev)
 		goto cleanup_ports;
 	}
 
-	err = sparx5_ptp_init(sparx5);
+	err = sparx5_vcap_init(sparx5);
 	if (err) {
-		dev_err(sparx5->dev, "PTP failed\n");
+		dev_err(sparx5->dev, "Failed to initialize VCAP\n");
 		goto cleanup_ports;
 	}
 
+	err = sparx5_mact_init(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to initialize MAC table\n");
+		goto cleanup_vcap;
+	}
+
+	err = sparx5_stats_init(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to initialize stats\n");
+		goto cleanup_mact;
+	}
+
 	INIT_LIST_HEAD(&sparx5->mall_entries);
 
+	err = sparx5_frame_io_init(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to initialize frame I/O\n");
+		goto cleanup_stats;
+	}
+
+	err = sparx5_ptp_init(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to initialize PTP\n");
+		goto cleanup_frame_io;
+	}
+
+	err = sparx5_register_netdevs(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to register net devices\n");
+		goto cleanup_ptp;
+	}
+
+	err = sparx5_register_notifier_blocks(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to register notifier blocks\n");
+		goto cleanup_netdevs;
+	}
+
 	goto cleanup_config;
 
+cleanup_netdevs:
+	sparx5_unregister_netdevs(sparx5);
+cleanup_ptp:
+	sparx5_ptp_deinit(sparx5);
+cleanup_frame_io:
+	sparx5_frame_io_deinit(sparx5);
+cleanup_stats:
+	sparx5_stats_deinit(sparx5);
+cleanup_mact:
+	sparx5_mact_deinit(sparx5);
+cleanup_vcap:
+	sparx5_vcap_deinit(sparx5);
 cleanup_ports:
-	sparx5_cleanup_ports(sparx5);
-	if (sparx5->mact_queue)
-		destroy_workqueue(sparx5->mact_queue);
+	sparx5_destroy_netdevs(sparx5);
 cleanup_config:
 	kfree(configs);
 cleanup_pnode:
@@ -1035,24 +1026,16 @@ static int mchp_sparx5_probe(struct platform_device *pdev)
 static void mchp_sparx5_remove(struct platform_device *pdev)
 {
 	struct sparx5 *sparx5 = platform_get_drvdata(pdev);
-	const struct sparx5_ops *ops = sparx5->data->ops;
 
 	debugfs_remove_recursive(sparx5->debugfs_root);
-	if (sparx5->xtr_irq) {
-		disable_irq(sparx5->xtr_irq);
-		sparx5->xtr_irq = -ENXIO;
-	}
-	if (sparx5->fdma_irq) {
-		disable_irq(sparx5->fdma_irq);
-		sparx5->fdma_irq = -ENXIO;
-	}
-	sparx5_ptp_deinit(sparx5);
-	ops->fdma_deinit(sparx5);
-	sparx5_cleanup_ports(sparx5);
-	sparx5_vcap_destroy(sparx5);
-	/* Unregister netdevs */
 	sparx5_unregister_notifier_blocks(sparx5);
-	destroy_workqueue(sparx5->mact_queue);
+	sparx5_unregister_netdevs(sparx5);
+	sparx5_ptp_deinit(sparx5);
+	sparx5_frame_io_deinit(sparx5);
+	sparx5_stats_deinit(sparx5);
+	sparx5_mact_deinit(sparx5);
+	sparx5_vcap_deinit(sparx5);
+	sparx5_destroy_netdevs(sparx5);
 }
 
 static const struct sparx5_regs sparx5_regs = {
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
index fe7d8bcc0cd99..6a745bb71b5c5 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
@@ -470,7 +470,6 @@ void sparx5_fdma_reload(struct sparx5 *sparx5, struct fdma *fdma);
 void sparx5_fdma_injection_mode(struct sparx5 *sparx5);
 
 /* sparx5_mactable.c */
-void sparx5_mact_pull_work(struct work_struct *work);
 int sparx5_mact_learn(struct sparx5 *sparx5, int port,
 		      const unsigned char mac[ETH_ALEN], u16 vid);
 bool sparx5_mact_getnext(struct sparx5 *sparx5,
@@ -489,7 +488,8 @@ int sparx5_del_mact_entry(struct sparx5 *sparx5,
 int sparx5_mc_sync(struct net_device *dev, const unsigned char *addr);
 int sparx5_mc_unsync(struct net_device *dev, const unsigned char *addr);
 void sparx5_set_ageing(struct sparx5 *sparx5, int msecs);
-void sparx5_mact_init(struct sparx5 *sparx5);
+int sparx5_mact_init(struct sparx5 *sparx5);
+void sparx5_mact_deinit(struct sparx5 *sparx5);
 
 /* sparx5_vlan.c */
 void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable);
@@ -504,8 +504,7 @@ int sparx5_vlan_vid_del(struct sparx5_port *port, u16 vid);
 void sparx5_vlan_port_apply(struct sparx5 *sparx5, struct sparx5_port *port);
 
 /* sparx5_calendar.c */
-int sparx5_config_auto_calendar(struct sparx5 *sparx5);
-int sparx5_config_dsm_calendar(struct sparx5 *sparx5);
+int sparx5_calendar_init(struct sparx5 *sparx5);
 int sparx5_dsm_calendar_calc(struct sparx5 *sparx5, u32 taxi,
 			     struct sparx5_calendar_data *data);
 u32 sparx5_cal_speed_to_value(enum sparx5_cal_bw speed);
@@ -514,7 +513,8 @@ enum sparx5_cal_bw sparx5_get_port_cal_speed(struct sparx5 *sparx5, u32 portno);
 
 /* sparx5_ethtool.c */
 void sparx5_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *stats);
-int sparx_stats_init(struct sparx5 *sparx5);
+int sparx5_stats_init(struct sparx5 *sparx5);
+void sparx5_stats_deinit(struct sparx5 *sparx5);
 
 /* sparx5_dcb.c */
 #ifdef CONFIG_SPARX5_DCB
@@ -563,7 +563,7 @@ void sparx5_get_hwtimestamp(struct sparx5 *sparx5,
 
 /* sparx5_vcap_impl.c */
 int sparx5_vcap_init(struct sparx5 *sparx5);
-void sparx5_vcap_destroy(struct sparx5 *sparx5);
+void sparx5_vcap_deinit(struct sparx5 *sparx5);
 
 /* sparx5_pgid.c */
 enum sparx5_pgid_type {
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
index 8b2e07821a950..a16ec8136d6d6 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
@@ -606,9 +606,22 @@ static int sparx5_ptp_phc_init(struct sparx5 *sparx5,
 int sparx5_ptp_init(struct sparx5 *sparx5)
 {
 	u64 tod_adj = sparx5_ptp_get_nominal_value(sparx5);
+	const struct sparx5_ops *ops = sparx5->data->ops;
 	struct sparx5_port *port;
 	int err, i;
 
+	if (sparx5->ptp_irq >= 0 &&
+	    sparx5_has_feature(sparx5, SPX5_FEATURE_PTP)) {
+		err = devm_request_threaded_irq(sparx5->dev, sparx5->ptp_irq,
+						NULL, ops->ptp_irq_handler,
+						IRQF_ONESHOT, "sparx5-ptp",
+						sparx5);
+		if (err)
+			sparx5->ptp_irq = -ENXIO;
+
+		sparx5->ptp = 1;
+	}
+
 	if (!sparx5->ptp)
 		return 0;
 
@@ -660,6 +673,11 @@ void sparx5_ptp_deinit(struct sparx5 *sparx5)
 	struct sparx5_port *port;
 	int i;
 
+	if (sparx5->ptp_irq >= 0) {
+		disable_irq(sparx5->ptp_irq);
+		sparx5->ptp_irq = -ENXIO;
+	}
+
 	for (i = 0; i < sparx5->data->consts->n_ports; i++) {
 		port = sparx5->ports[i];
 		if (!port)
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
index 3d64a0448d43a..95b93e46a41dc 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c
@@ -2083,7 +2083,7 @@ int sparx5_vcap_init(struct sparx5 *sparx5)
 	return err;
 }
 
-void sparx5_vcap_destroy(struct sparx5 *sparx5)
+void sparx5_vcap_deinit(struct sparx5 *sparx5)
 {
 	struct vcap_control *ctrl = sparx5->vcap_ctrl;
 	struct vcap_admin *admin, *admin_next;
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 786186c9a115f..2ba1fa3336f90 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -39,49 +39,66 @@ static u64 mana_gd_r64(struct gdma_context *g, u64 offset)
 	return readq(g->bar0_va + offset);
 }
 
-static void mana_gd_init_pf_regs(struct pci_dev *pdev)
+static int mana_gd_init_pf_regs(struct pci_dev *pdev)
 {
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 	void __iomem *sriov_base_va;
 	u64 sriov_base_off;
 
 	gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;
-	gc->db_page_base = gc->bar0_va +
-				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);
+	gc->db_page_off = mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);
 
-	gc->phys_db_page_base = gc->bar0_pa +
-				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);
+	/* Validate doorbell offset is within BAR0 */
+	if (gc->db_page_off >= gc->bar0_size) {
+		dev_err(gc->dev,
+			"Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n",
+			gc->db_page_off, (u64)gc->bar0_size);
+		return -EPROTO;
+	}
+
+	gc->db_page_base = gc->bar0_va + gc->db_page_off;
+	gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off;
 
 	sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);
 
 	sriov_base_va = gc->bar0_va + sriov_base_off;
 	gc->shm_base = sriov_base_va +
 			mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);
+
+	return 0;
 }
 
-static void mana_gd_init_vf_regs(struct pci_dev *pdev)
+static int mana_gd_init_vf_regs(struct pci_dev *pdev)
 {
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 
 	gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;
+	gc->db_page_off = mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);
 
-	gc->db_page_base = gc->bar0_va +
-				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);
+	/* Validate doorbell offset is within BAR0 */
+	if (gc->db_page_off >= gc->bar0_size) {
+		dev_err(gc->dev,
+			"Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n",
+			gc->db_page_off, (u64)gc->bar0_size);
+		return -EPROTO;
+	}
 
-	gc->phys_db_page_base = gc->bar0_pa +
-				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);
+	gc->db_page_base = gc->bar0_va + gc->db_page_off;
+	gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off;
 
 	gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);
+
+	return 0;
 }
 
-static void mana_gd_init_registers(struct pci_dev *pdev)
+static int mana_gd_init_registers(struct pci_dev *pdev)
 {
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 
 	if (gc->is_pf)
-		mana_gd_init_pf_regs(pdev);
+		return mana_gd_init_pf_regs(pdev);
 	else
-		mana_gd_init_vf_regs(pdev);
+		return mana_gd_init_vf_regs(pdev);
 }
 
 /* Suppress logging when we set timeout to zero */
@@ -490,15 +507,9 @@ static void mana_serv_reset(struct pci_dev *pdev)
 		dev_info(&pdev->dev, "MANA reset cycle completed\n");
 
 out:
-	gc->in_service = false;
+	clear_bit(GC_IN_SERVICE, &gc->flags);
 }
 
-struct mana_serv_work {
-	struct work_struct serv_work;
-	struct pci_dev *pdev;
-	enum gdma_eqe_type type;
-};
-
 static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev)
 {
 	switch (type) {
@@ -558,12 +569,42 @@ static void mana_serv_func(struct work_struct *w)
 	module_put(THIS_MODULE);
 }
 
+int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type)
+{
+	struct mana_serv_work *mns_wk;
+
+	if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) {
+		dev_info(gc->dev, "Already in service\n");
+		return -EBUSY;
+	}
+
+	if (!try_module_get(THIS_MODULE)) {
+		dev_info(gc->dev, "Module is unloading\n");
+		clear_bit(GC_IN_SERVICE, &gc->flags);
+		return -ENODEV;
+	}
+
+	mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC);
+	if (!mns_wk) {
+		module_put(THIS_MODULE);
+		clear_bit(GC_IN_SERVICE, &gc->flags);
+		return -ENOMEM;
+	}
+
+	dev_info(gc->dev, "Start MANA service type:%d\n", type);
+	mns_wk->pdev = to_pci_dev(gc->dev);
+	mns_wk->type = type;
+	pci_dev_get(mns_wk->pdev);
+	INIT_WORK(&mns_wk->serv_work, mana_serv_func);
+	schedule_work(&mns_wk->serv_work);
+	return 0;
+}
+
 static void mana_gd_process_eqe(struct gdma_queue *eq)
 {
 	u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE);
 	struct gdma_context *gc = eq->gdma_dev->gdma_context;
 	struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr;
-	struct mana_serv_work *mns_wk;
 	union gdma_eqe_info eqe_info;
 	enum gdma_eqe_type type;
 	struct gdma_event event;
@@ -623,30 +664,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)
 				 "Service is to be processed in probe\n");
 			break;
 		}
-
-		if (gc->in_service) {
-			dev_info(gc->dev, "Already in service\n");
-			break;
-		}
-
-		if (!try_module_get(THIS_MODULE)) {
-			dev_info(gc->dev, "Module is unloading\n");
-			break;
-		}
-
-		mns_wk = kzalloc_obj(*mns_wk, GFP_ATOMIC);
-		if (!mns_wk) {
-			module_put(THIS_MODULE);
-			break;
-		}
-
-		dev_info(gc->dev, "Start MANA service type:%d\n", type);
-		gc->in_service = true;
-		mns_wk->pdev = to_pci_dev(gc->dev);
-		mns_wk->type = type;
-		pci_dev_get(mns_wk->pdev);
-		INIT_WORK(&mns_wk->serv_work, mana_serv_func);
-		schedule_work(&mns_wk->serv_work);
+		mana_schedule_serv_work(gc, type);
 		break;
 
 	default:
@@ -1255,6 +1273,17 @@ int mana_gd_register_device(struct gdma_dev *gd)
 		return err ? err : -EPROTO;
 	}
 
+	/* Validate that doorbell page for db_id is within the BAR0 region.
+	 * In mana_gd_ring_doorbell(), the address is calculated as:
+	 *   addr = db_page_base + db_page_size * db_id
+	 *        = (bar0_va + db_page_off) + (db_page_size * db_id)
+	 * So we need: db_page_off + db_page_size * (db_id + 1) <= bar0_size
+	 */
+	if (gc->db_page_off + gc->db_page_size * ((u64)resp.db_id + 1) > gc->bar0_size) {
+		dev_err(gc->dev, "Doorbell ID %u out of range\n", resp.db_id);
+		return -EPROTO;
+	}
+
 	gd->pdid = resp.pdid;
 	gd->gpa_mkey = resp.gpa_mkey;
 	gd->doorbell = resp.db_id;
@@ -1889,7 +1918,10 @@ static int mana_gd_setup(struct pci_dev *pdev)
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 	int err;
 
-	mana_gd_init_registers(pdev);
+	err = mana_gd_init_registers(pdev);
+	if (err)
+		return err;
+
 	mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base);
 
 	gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0);
@@ -1996,6 +2028,7 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	mutex_init(&gc->eq_test_event_mutex);
 	pci_set_drvdata(pdev, gc);
 	gc->bar0_pa = pci_resource_start(pdev, 0);
+	gc->bar0_size = pci_resource_len(pdev, 0);
 
 	bar0_va = pci_iomap(pdev, bar, 0);
 	if (!bar0_va)
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index ba3467f1e2eae..dbbde0fa57e71 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -814,9 +814,6 @@ void mana_hwc_destroy_channel(struct gdma_context *gc)
 		gc->max_num_cqs = 0;
 	}
 
-	kfree(hwc->caller_ctx);
-	hwc->caller_ctx = NULL;
-
 	if (hwc->txq)
 		mana_hwc_destroy_wq(hwc, hwc->txq);
 
@@ -826,6 +823,9 @@ void mana_hwc_destroy_channel(struct gdma_context *gc)
 	if (hwc->cq)
 		mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
 
+	kfree(hwc->caller_ctx);
+	hwc->caller_ctx = NULL;
+
 	mana_gd_free_res_map(&hwc->inflight_msg_res);
 
 	hwc->num_inflight_msg = 0;
@@ -853,6 +853,7 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	struct hwc_caller_ctx *ctx;
 	u32 dest_vrcq = 0;
 	u32 dest_vrq = 0;
+	u32 command;
 	u16 msg_id;
 	int err;
 
@@ -878,6 +879,7 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	req_msg->req.hwc_msg_id = msg_id;
 
 	tx_wr->msg_size = req_len;
+	command = req_msg->req.msg_type;
 
 	if (gc->is_pf) {
 		dest_vrq = hwc->pf_dest_vrq_id;
@@ -893,8 +895,8 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	if (!wait_for_completion_timeout(&ctx->comp_event,
 					 (msecs_to_jiffies(hwc->hwc_timeout)))) {
 		if (hwc->hwc_timeout != 0)
-			dev_err(hwc->dev, "HWC: Request timed out: %u ms\n",
-				hwc->hwc_timeout);
+			dev_err(hwc->dev, "Command 0x%x timed out: %u ms\n",
+				command, hwc->hwc_timeout);
 
 		/* Reduce further waiting if HWC no response */
 		if (hwc->hwc_timeout > 1)
@@ -914,9 +916,9 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 			err = -EOPNOTSUPP;
 			goto out;
 		}
-		if (req_msg->req.msg_type != MANA_QUERY_PHY_STAT)
-			dev_err(hwc->dev, "HWC: Failed hw_channel req: 0x%x\n",
-				ctx->status_code);
+		if (command != MANA_QUERY_PHY_STAT)
+			dev_err(hwc->dev, "Command 0x%x failed with status: 0x%x\n",
+				command, ctx->status_code);
 		err = -EPROTO;
 		goto out;
 	}
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 9017e806ecdaa..49c65cc1697c6 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -875,7 +875,7 @@ static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
 
 	/* Already in service, hence tx queue reset is not required.*/
-	if (gc->in_service)
+	if (test_bit(GC_IN_SERVICE, &gc->flags))
 		return;
 
 	/* Note: If there are pending queue reset work for this port(apc),
@@ -1021,8 +1021,8 @@ static int mana_send_request(struct mana_context *ac, void *in_buf,
 
 		if (req->req.msg_type != MANA_QUERY_PHY_STAT &&
 		    mana_need_log(gc, err))
-			dev_err(dev, "Failed to send mana message: %d, 0x%x\n",
-				err, resp->status);
+			dev_err(dev, "Command 0x%x failed with status: 0x%x, err: %d\n",
+				req->req.msg_type, resp->status, err);
 		return err ? err : -EPROTO;
 	}
 
@@ -1335,8 +1335,8 @@ int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
 	apc->tx_shortform_allowed = resp.short_form_allowed;
 	apc->tx_vp_offset = resp.tx_vport_offset;
 
-	netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n",
-		    apc->port_handle, protection_dom_id, doorbell_pg_id);
+	netdev_info(apc->ndev, "Enabled vPort %llu PD %u DB %u MAC %pM\n",
+		    apc->port_handle, protection_dom_id, doorbell_pg_id, apc->mac_addr);
 out:
 	if (err)
 		mana_uncfg_vport(apc);
@@ -1365,6 +1365,7 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 			     sizeof(resp));
 
 	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
+	req->hdr.resp.msg_version = GDMA_MESSAGE_V2;
 
 	req->vport = apc->port_handle;
 	req->num_indir_entries = apc->indir_table_sz;
@@ -1376,7 +1377,9 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 	req->update_hashkey = update_key;
 	req->update_indir_tab = update_tab;
 	req->default_rxobj = apc->default_rxobj;
-	req->cqe_coalescing_enable = 0;
+
+	if (rx != TRI_STATE_FALSE)
+		req->cqe_coalescing_enable = apc->cqe_coalescing_enable;
 
 	if (update_key)
 		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
@@ -1405,8 +1408,13 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
 			   resp.hdr.status);
 		err = -EPROTO;
+		goto out;
 	}
 
+	if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2)
+		apc->cqe_coalescing_timeout_ns =
+			resp.cqe_coalescing_timeout_ns;
+
 	netdev_info(ndev, "Configured steering vPort %llu entries %u\n",
 		    apc->port_handle, apc->indir_table_sz);
 out:
@@ -1915,11 +1923,12 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
 }
 
 static void mana_rx_skb(void *buf_va, bool from_pool,
-			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)
+			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq,
+			int i)
 {
 	struct mana_stats_rx *rx_stats = &rxq->stats;
 	struct net_device *ndev = rxq->ndev;
-	uint pkt_len = cqe->ppi[0].pkt_len;
+	uint pkt_len = cqe->ppi[i].pkt_len;
 	u16 rxq_idx = rxq->rxq_idx;
 	struct napi_struct *napi;
 	struct xdp_buff xdp = {};
@@ -1963,7 +1972,7 @@ static void mana_rx_skb(void *buf_va, bool from_pool,
 	}
 
 	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
-		hash_value = cqe->ppi[0].pkt_hash;
+		hash_value = cqe->ppi[i].pkt_hash;
 
 		if (cqe->rx_hashtype & MANA_HASH_L4)
 			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
@@ -2098,9 +2107,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	struct mana_recv_buf_oob *rxbuf_oob;
 	struct mana_port_context *apc;
 	struct device *dev = gc->dev;
+	bool coalesced = false;
 	void *old_buf = NULL;
 	u32 curr, pktlen;
 	bool old_fp;
+	int i;
 
 	apc = netdev_priv(ndev);
 
@@ -2112,13 +2123,16 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		++ndev->stats.rx_dropped;
 		rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
 		netdev_warn_once(ndev, "Dropped a truncated packet\n");
-		goto drop;
 
-	case CQE_RX_COALESCED_4:
-		netdev_err(ndev, "RX coalescing is unsupported\n");
-		apc->eth_stats.rx_coalesced_err++;
+		mana_move_wq_tail(rxq->gdma_rq,
+				  rxbuf_oob->wqe_inf.wqe_size_in_bu);
+		mana_post_pkt_rxq(rxq);
 		return;
 
+	case CQE_RX_COALESCED_4:
+		coalesced = true;
+		break;
+
 	case CQE_RX_OBJECT_FENCE:
 		complete(&rxq->fence_event);
 		return;
@@ -2130,30 +2144,47 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		return;
 	}
 
-	pktlen = oob->ppi[0].pkt_len;
+	for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) {
+		old_buf = NULL;
+		pktlen = oob->ppi[i].pkt_len;
+		if (pktlen == 0)
+			break;
 
-	if (pktlen == 0) {
-		/* data packets should never have packetlength of zero */
-		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
-			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
-		return;
-	}
+		curr = rxq->buf_index;
+		rxbuf_oob = &rxq->rx_oobs[curr];
+		WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
 
-	curr = rxq->buf_index;
-	rxbuf_oob = &rxq->rx_oobs[curr];
-	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
+		mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
 
-	mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
+		/* Unsuccessful refill will have old_buf == NULL.
+		 * In this case, mana_rx_skb() will drop the packet.
+		 */
+		mana_rx_skb(old_buf, old_fp, oob, rxq, i);
 
-	/* Unsuccessful refill will have old_buf == NULL.
-	 * In this case, mana_rx_skb() will drop the packet.
-	 */
-	mana_rx_skb(old_buf, old_fp, oob, rxq);
+		mana_move_wq_tail(rxq->gdma_rq,
+				  rxbuf_oob->wqe_inf.wqe_size_in_bu);
 
-drop:
-	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
+		mana_post_pkt_rxq(rxq);
 
-	mana_post_pkt_rxq(rxq);
+		if (!coalesced)
+			break;
+	}
+
+	/* Collect coalesced CQE count based on packets processed.
+	 * Coalesced CQEs have at least 2 packets, so index is i - 2.
+	 */
+	if (i > 1) {
+		u64_stats_update_begin(&rxq->stats.syncp);
+		rxq->stats.coalesced_cqe[i - 2]++;
+		u64_stats_update_end(&rxq->stats.syncp);
+	} else if (!i && !pktlen) {
+		u64_stats_update_begin(&rxq->stats.syncp);
+		rxq->stats.pkt_len0_err++;
+		u64_stats_update_end(&rxq->stats.syncp);
+		netdev_err_once(ndev,
+				"RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
+				rxq->gdma_id, cq->gdma_id, rxq->rxobj);
+	}
 }
 
 static void mana_poll_rx_cq(struct mana_cq *cq)
@@ -3332,6 +3363,7 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 	apc->port_handle = INVALID_MANA_HANDLE;
 	apc->pf_filter_handle = INVALID_MANA_HANDLE;
 	apc->port_idx = port_idx;
+	apc->cqe_coalescing_enable = 0;
 
 	mutex_init(&apc->vport_mutex);
 	apc->vport_use_count = 0;
@@ -3538,6 +3570,7 @@ static void mana_gf_stats_work_handler(struct work_struct *work)
 {
 	struct mana_context *ac =
 		container_of(to_delayed_work(work), struct mana_context, gf_stats_work);
+	struct gdma_context *gc = ac->gdma_dev->gdma_context;
 	int err;
 
 	err = mana_query_gf_stats(ac);
@@ -3545,6 +3578,12 @@ static void mana_gf_stats_work_handler(struct work_struct *work)
 		/* HWC timeout detected - reset stats and stop rescheduling */
 		ac->hwc_timeout_occurred = true;
 		memset(&ac->hc_stats, 0, sizeof(ac->hc_stats));
+		dev_warn(gc->dev,
+			 "Gf stats wk handler: gf stats query timed out.\n");
+		/* As HWC timed out, indicating a faulty HW state and needs a
+		 * reset.
+		 */
+		mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST);
 		return;
 	}
 	schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD);
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index f2d220b371b5d..6a4b42fe09445 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -20,8 +20,6 @@ static const struct mana_stats_desc mana_eth_stats[] = {
 					tx_cqe_unknown_type)},
 	{"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats,
 				       tx_linear_pkt_cnt)},
-	{"rx_coalesced_err", offsetof(struct mana_ethtool_stats,
-					rx_coalesced_err)},
 	{"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
 					rx_cqe_unknown_type)},
 };
@@ -151,7 +149,7 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 {
 	struct mana_port_context *apc = netdev_priv(ndev);
 	unsigned int num_queues = apc->num_queues;
-	int i;
+	int i, j;
 
 	if (stringset != ETH_SS_STATS)
 		return;
@@ -170,6 +168,9 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 		ethtool_sprintf(&data, "rx_%d_xdp_drop", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
+		ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2);
 	}
 
 	for (i = 0; i < num_queues; i++) {
@@ -203,6 +204,8 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 xdp_xmit;
 	u64 xdp_drop;
 	u64 xdp_tx;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	u64 tso_packets;
 	u64 tso_bytes;
 	u64 tso_inner_packets;
@@ -211,7 +214,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 short_pkt_fmt;
 	u64 csum_partial;
 	u64 mana_map_err;
-	int q, i = 0;
+	int q, i = 0, j;
 
 	if (!apc->port_is_up)
 		return;
@@ -241,6 +244,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 			xdp_drop = rx_stats->xdp_drop;
 			xdp_tx = rx_stats->xdp_tx;
 			xdp_redirect = rx_stats->xdp_redirect;
+			pkt_len0_err = rx_stats->pkt_len0_err;
+			for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+				coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
 		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
 
 		data[i++] = packets;
@@ -248,6 +254,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 		data[i++] = xdp_drop;
 		data[i++] = xdp_tx;
 		data[i++] = xdp_redirect;
+		data[i++] = pkt_len0_err;
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			data[i++] = coalesced_cqe[j];
 	}
 
 	for (q = 0; q < num_queues; q++) {
@@ -390,6 +399,61 @@ static void mana_get_channels(struct net_device *ndev,
 	channel->combined_count = apc->num_queues;
 }
 
+#define MANA_RX_CQE_NSEC_DEF 2048
+static int mana_get_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack)
+{
+	struct mana_port_context *apc = netdev_priv(ndev);
+
+	kernel_coal->rx_cqe_frames =
+		apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1;
+
+	kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns;
+
+	/* Return the default timeout value for old FW not providing
+	 * this value.
+	 */
+	if (apc->port_is_up && apc->cqe_coalescing_enable &&
+	    !kernel_coal->rx_cqe_nsecs)
+		kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF;
+
+	return 0;
+}
+
+static int mana_set_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack)
+{
+	struct mana_port_context *apc = netdev_priv(ndev);
+	u8 saved_cqe_coalescing_enable;
+	int err;
+
+	if (kernel_coal->rx_cqe_frames != 1 &&
+	    kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "rx-frames must be 1 or %u, got %u",
+				   MANA_RXCOMP_OOB_NUM_PPI,
+				   kernel_coal->rx_cqe_frames);
+		return -EINVAL;
+	}
+
+	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	apc->cqe_coalescing_enable =
+		kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;
+
+	if (!apc->port_is_up)
+		return 0;
+
+	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+	if (err)
+		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+
+	return err;
+}
+
 static int mana_set_channels(struct net_device *ndev,
 			     struct ethtool_channels *channels)
 {
@@ -510,6 +574,7 @@ static int mana_get_link_ksettings(struct net_device *ndev,
 }
 
 const struct ethtool_ops mana_ethtool_ops = {
+	.supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES,
 	.get_ethtool_stats	= mana_get_ethtool_stats,
 	.get_sset_count		= mana_get_sset_count,
 	.get_strings		= mana_get_strings,
@@ -520,6 +585,8 @@ const struct ethtool_ops mana_ethtool_ops = {
 	.set_rxfh		= mana_set_rxfh,
 	.get_channels		= mana_get_channels,
 	.set_channels		= mana_set_channels,
+	.get_coalesce		= mana_get_coalesce,
+	.set_coalesce		= mana_set_coalesce,
 	.get_ringparam          = mana_get_ringparam,
 	.set_ringparam          = mana_set_ringparam,
 	.get_link_ksettings	= mana_get_link_ksettings,
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index 3a02eef58cc6d..c9ab55abfd4c5 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -1731,7 +1731,7 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 	/* We only have to reload LM0 if the key is not at start of stack */
 	lm_off = nfp_prog->stack_frame_depth;
-	lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off;
+	lm_off += meta->arg2.reg.var_off.value;
 	load_lm_ptr = meta->arg2.var_off || lm_off;
 
 	/* Set LM0 to start of key */
@@ -2874,8 +2874,7 @@ mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	}
 
 	if (meta->ptr.type == PTR_TO_STACK)
-		return mem_ldx_stack(nfp_prog, meta, size,
-				     meta->ptr.off + meta->ptr.var_off.value);
+		return mem_ldx_stack(nfp_prog, meta, size, meta->ptr.var_off.value);
 
 	if (meta->ptr.type == PTR_TO_MAP_VALUE)
 		return mem_ldx_emem(nfp_prog, meta, size);
@@ -2985,8 +2984,7 @@ mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 		return mem_stx_data(nfp_prog, meta, size);
 
 	if (meta->ptr.type == PTR_TO_STACK)
-		return mem_stx_stack(nfp_prog, meta, size,
-				     meta->ptr.off + meta->ptr.var_off.value);
+		return mem_stx_stack(nfp_prog, meta, size, meta->ptr.var_off.value);
 
 	return -EOPNOTSUPP;
 }
@@ -4153,9 +4151,9 @@ cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta,
 	/* Canonicalize the offsets. Turn all of them against the original
 	 * base register.
 	 */
-	head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off;
-	head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off;
-	ld_off = ld->off + head_ld_meta->ptr.off;
+	head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.var_off.value;
+	head_st_off = head_st_meta->insn.off + head_st_meta->ptr.var_off.value;
+	ld_off = ld->off + head_ld_meta->ptr.var_off.value;
 
 	/* Ascending order cross. */
 	if (ld_off > head_ld_off &&
@@ -4326,7 +4324,7 @@ static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
 		 * support this.
 		 */
 		if (meta->ptr.id == range_ptr_id &&
-		    meta->ptr.off == range_ptr_off) {
+		    meta->ptr.var_off.value == range_ptr_off) {
 			s16 new_start = range_start;
 			s16 end, off = insn->off;
 			s16 new_end = range_end;
@@ -4361,7 +4359,7 @@ static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
 		range_node = meta;
 		range_node->pkt_cache.do_init = true;
 		range_ptr_id = range_node->ptr.id;
-		range_ptr_off = range_node->ptr.off;
+		range_ptr_off = range_node->ptr.var_off.value;
 		range_start = insn->off;
 		range_end = insn->off + BPF_LDST_BYTES(insn);
 	}
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index bc594860e2b5a..70368fe7c5106 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -98,7 +98,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env)
 
 	offmap = map_to_offmap(reg1->map_ptr);
 	nfp_map = offmap->dev_priv;
-	off = reg3->off + reg3->var_off.value;
+	off = reg3->var_off.value;
 
 	for (i = 0; i < offmap->map.value_size; i++) {
 		struct bpf_stack_state *stack_entry;
@@ -137,7 +137,7 @@ nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,
 		return false;
 	}
 
-	off = reg->var_off.value + reg->off;
+	off = reg->var_off.value;
 	if (-off % 4) {
 		pr_vlog(env, "%s: unaligned stack pointer %lld\n", fname, -off);
 		return false;
@@ -147,7 +147,7 @@ nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,
 	if (!old_arg)
 		return true;
 
-	old_off = old_arg->reg.var_off.value + old_arg->reg.off;
+	old_off = old_arg->reg.var_off.value;
 	old_arg->var_off |= off != old_off;
 
 	return true;
@@ -358,8 +358,8 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
 	if (meta->ptr.type == NOT_INIT)
 		return 0;
 
-	old_off = meta->ptr.off + meta->ptr.var_off.value;
-	new_off = reg->off + reg->var_off.value;
+	old_off = meta->ptr.var_off.value;
+	new_off = reg->var_off.value;
 
 	meta->ptr_not_const |= old_off != new_off;
 
@@ -428,7 +428,7 @@ nfp_bpf_map_mark_used(struct bpf_verifier_env *env, struct nfp_insn_meta *meta,
 		return -EOPNOTSUPP;
 	}
 
-	off = reg->var_off.value + meta->insn.off + reg->off;
+	off = reg->var_off.value + meta->insn.off;
 	size = BPF_LDST_BYTES(&meta->insn);
 	offmap = map_to_offmap(reg->map_ptr);
 	nfp_map = offmap->dev_priv;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
index 1514c1019f28a..78a802eb159f0 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
@@ -188,10 +188,9 @@ static int ionic_get_link_ksettings(struct net_device *netdev,
 	case IONIC_XCVR_PID_QSFP_100G_CWDM4:
 	case IONIC_XCVR_PID_QSFP_100G_PSM4:
 	case IONIC_XCVR_PID_QSFP_100G_LR4:
-		ethtool_link_ksettings_add_link_mode(ks, supported,
-						     100000baseLR4_ER4_Full);
-		break;
 	case IONIC_XCVR_PID_QSFP_100G_ER4:
+	case IONIC_XCVR_PID_QSFP_100G_FR4:
+	case IONIC_XCVR_PID_QSFP_100G_DR4:
 		ethtool_link_ksettings_add_link_mode(ks, supported,
 						     100000baseLR4_ER4_Full);
 		break;
@@ -212,6 +211,7 @@ static int ionic_get_link_ksettings(struct net_device *netdev,
 		break;
 	case IONIC_XCVR_PID_QSFP_200G_AOC:
 	case IONIC_XCVR_PID_QSFP_200G_SR4:
+	case IONIC_XCVR_PID_QSFP_200G_AEC:
 		ethtool_link_ksettings_add_link_mode(ks, supported,
 						     200000baseSR4_Full);
 		break;
@@ -232,6 +232,9 @@ static int ionic_get_link_ksettings(struct net_device *netdev,
 						     400000baseDR4_Full);
 		break;
 	case IONIC_XCVR_PID_QSFP_400G_SR4:
+	case IONIC_XCVR_PID_QSFP_400G_AOC:
+	case IONIC_XCVR_PID_QSFP_400G_AEC:
+	case IONIC_XCVR_PID_QSFP_400G_LPO:
 		ethtool_link_ksettings_add_link_mode(ks, supported,
 						     400000baseSR4_Full);
 		break;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h
index 47559c909c8b4..23d6e2b4791e0 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_if.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h
@@ -1341,6 +1341,12 @@ enum ionic_xcvr_pid {
 	IONIC_XCVR_PID_QSFP_400G_DR4    = 80,
 	IONIC_XCVR_PID_QSFP_400G_SR4    = 81,
 	IONIC_XCVR_PID_QSFP_400G_VR4    = 82,
+	IONIC_XCVR_PID_QSFP_400G_AOC    = 83,
+	IONIC_XCVR_PID_QSFP_400G_AEC    = 84,
+	IONIC_XCVR_PID_QSFP_200G_AEC    = 85,
+	IONIC_XCVR_PID_QSFP_400G_LPO    = 86,
+	IONIC_XCVR_PID_QSFP_100G_FR4    = 87,
+	IONIC_XCVR_PID_QSFP_100G_DR4    = 88,
 };
 
 /**
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 8d040e611d5aa..637e635bbf03a 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -1719,13 +1719,18 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa)
 	if (ether_addr_equal(netdev->dev_addr, mac))
 		return 0;
 
-	err = ionic_program_mac(lif, mac);
-	if (err < 0)
-		return err;
+	/* Only program macs for virtual functions to avoid losing the permanent
+	 * Mac across warm reset/reboot.
+	 */
+	if (lif->ionic->pdev->is_virtfn) {
+		err = ionic_program_mac(lif, mac);
+		if (err < 0)
+			return err;
 
-	if (err > 0)
-		netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
-			   __func__);
+		if (err > 0)
+			netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
+				   __func__);
+	}
 
 	err = eth_prepare_mac_addr_change(netdev, addr);
 	if (err)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index 0f6905e33df62..2d873a5962618 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -7,6 +7,7 @@
 #include <linux/types.h>
 #include <asm/byteorder.h>
 #include <asm/param.h>
+#include <linux/crc32.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/etherdevice.h>
@@ -39,7 +40,6 @@
 #include "qed_sriov.h"
 
 #define QED_MAX_SGES_NUM 16
-#define CRC32_POLY 0x1edc6f41
 
 struct qed_l2_info {
 	u32 queues;
@@ -1412,50 +1412,19 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
-/*******************************************************************************
- * Description:
- *         Calculates crc 32 on a buffer
- *         Note: crc32_length MUST be aligned to 8
- * Return:
- ******************************************************************************/
-static u32 qed_calc_crc32c(u8 *crc32_packet,
-			   u32 crc32_length, u32 crc32_seed, u8 complement)
-{
-	u32 byte = 0, bit = 0, crc32_result = crc32_seed;
-	u8 msb = 0, current_byte = 0;
-
-	if ((!crc32_packet) ||
-	    (crc32_length == 0) ||
-	    ((crc32_length % 8) != 0))
-		return crc32_result;
-	for (byte = 0; byte < crc32_length; byte++) {
-		current_byte = crc32_packet[byte];
-		for (bit = 0; bit < 8; bit++) {
-			msb = (u8)(crc32_result >> 31);
-			crc32_result = crc32_result << 1;
-			if (msb != (0x1 & (current_byte >> bit))) {
-				crc32_result = crc32_result ^ CRC32_POLY;
-				crc32_result |= 1; /*crc32_result[0] = 1;*/
-			}
-		}
-	}
-	return crc32_result;
-}
-
-static u32 qed_crc32c_le(u32 seed, u8 *mac, u32 len)
-{
-	u32 packet_buf[2] = { 0 };
-
-	memcpy((u8 *)(&packet_buf[0]), &mac[0], 6);
-	return qed_calc_crc32c((u8 *)packet_buf, 8, seed, 0);
-}
-
 u8 qed_mcast_bin_from_mac(u8 *mac)
 {
-	u32 crc = qed_crc32c_le(ETH_MULTICAST_BIN_FROM_MAC_SEED,
-				mac, ETH_ALEN);
+	u8 padded_mac[8] = {};
+	u32 crc;
 
-	return crc & 0xff;
+	memcpy(padded_mac, mac, ETH_ALEN);
+	/*
+	 * This uses the standard CRC-32C, but with the input and output CRCs
+	 * bit-reflected and the bit-reflected output CRC truncated to 8 bits.
+	 */
+	crc = crc32c(bitrev32(ETH_MULTICAST_BIN_FROM_MAC_SEED), padded_mac,
+		     sizeof(padded_mac));
+	return bitrev8(crc >> 24);
 }
 
 static int
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
index 3d0b5cd978cbb..d7c8fadb49de1 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
@@ -354,7 +354,7 @@ struct qlcnic_skb_frag {
 
 /*
  * There will be one qlcnic_buffer per skb packet.    These will be
- * used to save the dma info for pci_unmap_page()
+ * used to save the dma info for dma_unmap_page()
  */
 struct qlcnic_cmd_buffer {
 	struct sk_buff *skb;
diff --git a/drivers/net/ethernet/realtek/r8169.h b/drivers/net/ethernet/realtek/r8169.h
index aed4cf852091c..0b9c1d4eb48ba 100644
--- a/drivers/net/ethernet/realtek/r8169.h
+++ b/drivers/net/ethernet/realtek/r8169.h
@@ -68,6 +68,7 @@ enum mac_version {
 	RTL_GIGA_MAC_VER_61,
 	RTL_GIGA_MAC_VER_63,
 	RTL_GIGA_MAC_VER_64,
+	RTL_GIGA_MAC_VER_65,
 	RTL_GIGA_MAC_VER_66,
 	RTL_GIGA_MAC_VER_70,
 	RTL_GIGA_MAC_VER_80,
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 58788d196c57d..791277e750bad 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -60,6 +60,7 @@
 #define FIRMWARE_8125D_2	"rtl_nic/rtl8125d-2.fw"
 #define FIRMWARE_8125K_1	"rtl_nic/rtl8125k-1.fw"
 #define FIRMWARE_8125BP_2	"rtl_nic/rtl8125bp-2.fw"
+#define FIRMWARE_8125CP_1	"rtl_nic/rtl8125cp-1.fw"
 #define FIRMWARE_9151A_1	"rtl_nic/rtl9151a-1.fw"
 #define FIRMWARE_8126A_2	"rtl_nic/rtl8126a-2.fw"
 #define FIRMWARE_8126A_3	"rtl_nic/rtl8126a-3.fw"
@@ -112,6 +113,9 @@ static const struct rtl_chip_info {
 	/* 8125BP family. */
 	{ 0x7cf, 0x681,	RTL_GIGA_MAC_VER_66, "RTL8125BP", FIRMWARE_8125BP_2 },
 
+	/* 8125CP family*/
+	{ 0x7cf, 0x708, RTL_GIGA_MAC_VER_65, "RTL8125CP", FIRMWARE_8125CP_1 },
+
 	/* 8125D family. */
 	{ 0x7cf, 0x68b, RTL_GIGA_MAC_VER_64, "RTL9151A", FIRMWARE_9151A_1 },
 	{ 0x7cf, 0x68a, RTL_GIGA_MAC_VER_64, "RTL8125K", FIRMWARE_8125K_1 },
@@ -802,6 +806,7 @@ MODULE_FIRMWARE(FIRMWARE_8125D_1);
 MODULE_FIRMWARE(FIRMWARE_8125D_2);
 MODULE_FIRMWARE(FIRMWARE_8125K_1);
 MODULE_FIRMWARE(FIRMWARE_8125BP_2);
+MODULE_FIRMWARE(FIRMWARE_8125CP_1);
 MODULE_FIRMWARE(FIRMWARE_9151A_1);
 MODULE_FIRMWARE(FIRMWARE_8126A_2);
 MODULE_FIRMWARE(FIRMWARE_8126A_3);
@@ -4021,6 +4026,7 @@ static void rtl_hw_config(struct rtl8169_private *tp)
 		[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
 		[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
 		[RTL_GIGA_MAC_VER_64] = rtl_hw_start_8125d,
+		[RTL_GIGA_MAC_VER_65] = rtl_hw_start_8125d,
 		[RTL_GIGA_MAC_VER_66] = rtl_hw_start_8125d,
 		[RTL_GIGA_MAC_VER_70] = rtl_hw_start_8126a,
 		[RTL_GIGA_MAC_VER_80] = rtl_hw_start_8127a,
@@ -4040,6 +4046,7 @@ static void rtl_hw_start_8125(struct rtl8169_private *tp)
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_61:
 	case RTL_GIGA_MAC_VER_64:
+	case RTL_GIGA_MAC_VER_65:
 	case RTL_GIGA_MAC_VER_66:
 	case RTL_GIGA_MAC_VER_80:
 		for (i = 0xa00; i < 0xb00; i += 4)
diff --git a/drivers/net/ethernet/realtek/r8169_phy_config.c b/drivers/net/ethernet/realtek/r8169_phy_config.c
index 032d9d2cfa2a0..4f1d447136ec1 100644
--- a/drivers/net/ethernet/realtek/r8169_phy_config.c
+++ b/drivers/net/ethernet/realtek/r8169_phy_config.c
@@ -14,6 +14,9 @@
 
 #include "r8169.h"
 
+#define LINK_SPEED_10M_PLL_OFF		BIT(0)
+#define ALDPS_PLL_OFF			BIT(1)
+
 typedef void (*rtl_phy_cfg_fct)(struct rtl8169_private *tp,
 				struct phy_device *phydev);
 
@@ -1102,6 +1105,28 @@ static void rtl8125d_hw_phy_config(struct rtl8169_private *tp,
 	rtl8125_config_eee_phy(phydev);
 }
 
+static void rtl8125cp_hw_phy_config(struct rtl8169_private *tp,
+				    struct phy_device *phydev)
+{
+	r8169_apply_firmware(tp);
+	rtl8168g_enable_gphy_10m(phydev);
+
+	phy_modify_paged(phydev, 0xad0, 0x17, 0x007f, 0x000b);
+	phy_modify_paged(phydev, 0xad7, 0x14, 0x0000, BIT(4));
+	rtl8125_phy_param(phydev, 0x807f, 0xff00, 0x5300);
+	r8168g_phy_param(phydev, 0x81b8, 0xffff, 0x00b4);
+	r8168g_phy_param(phydev, 0x81ba, 0xffff, 0x00e4);
+	r8168g_phy_param(phydev, 0x81c5, 0xffff, 0x0104);
+	r8168g_phy_param(phydev, 0x81d0, 0xffff, 0x054d);
+	phy_modify_paged(phydev, 0x0a43, 0x10, 0x0000,
+			 LINK_SPEED_10M_PLL_OFF | ALDPS_PLL_OFF);
+	phy_modify_paged(phydev, 0x0a44, 0x11, 0x0000, BIT(7));
+
+	rtl8125_legacy_force_mode(phydev);
+	rtl8168g_disable_aldps(phydev);
+	rtl8125_config_eee_phy(phydev);
+}
+
 static void rtl8125bp_hw_phy_config(struct rtl8169_private *tp,
 				    struct phy_device *phydev)
 {
@@ -1344,6 +1369,7 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev,
 		[RTL_GIGA_MAC_VER_61] = rtl8125a_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_63] = rtl8125b_hw_phy_config,
 		[RTL_GIGA_MAC_VER_64] = rtl8125d_hw_phy_config,
+		[RTL_GIGA_MAC_VER_65] = rtl8125cp_hw_phy_config,
 		[RTL_GIGA_MAC_VER_66] = rtl8125bp_hw_phy_config,
 		[RTL_GIGA_MAC_VER_70] = rtl8126a_hw_phy_config,
 		[RTL_GIGA_MAC_VER_80] = rtl8127a_1_hw_phy_config,
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 84b657fc2e158..2c725824b3488 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -2367,6 +2367,7 @@ static int ravb_close(struct net_device *ndev)
 	ravb_write(ndev, 0, RIC0);
 	ravb_write(ndev, 0, RIC2);
 	ravb_write(ndev, 0, TIC);
+	ravb_write(ndev, 0, ECSIPR);
 
 	/* PHY disconnect */
 	if (ndev->phydev) {
diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
index 50ea5f9ef63ae..66a8ae67c3ea8 100644
--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
+++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
@@ -104,7 +104,6 @@ struct ofdpa_group_tbl_entry {
 	u32 cmd;
 	u32 group_id; /* key */
 	u16 group_count;
-	u32 *group_ids;
 	union {
 		struct {
 			u8 pop_vlan;
@@ -123,6 +122,8 @@ struct ofdpa_group_tbl_entry {
 			u32 group_id;
 		} l3_unicast;
 	};
+
+	u32 group_ids[] __counted_by(group_count);
 };
 
 struct ofdpa_fdb_tbl_entry {
@@ -1059,19 +1060,6 @@ ofdpa_group_tbl_find(const struct ofdpa *ofdpa,
 	return NULL;
 }
 
-static void ofdpa_group_tbl_entry_free(struct ofdpa_group_tbl_entry *entry)
-{
-	switch (ROCKER_GROUP_TYPE_GET(entry->group_id)) {
-	case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
-	case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
-		kfree(entry->group_ids);
-		break;
-	default:
-		break;
-	}
-	kfree(entry);
-}
-
 static int ofdpa_group_tbl_add(struct ofdpa_port *ofdpa_port, int flags,
 			       struct ofdpa_group_tbl_entry *match)
 {
@@ -1085,7 +1073,7 @@ static int ofdpa_group_tbl_add(struct ofdpa_port *ofdpa_port, int flags,
 
 	if (found) {
 		hash_del(&found->entry);
-		ofdpa_group_tbl_entry_free(found);
+		kfree(found);
 		found = match;
 		found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD;
 	} else {
@@ -1122,14 +1110,14 @@ static int ofdpa_group_tbl_del(struct ofdpa_port *ofdpa_port, int flags,
 
 	spin_unlock_irqrestore(&ofdpa->group_tbl_lock, lock_flags);
 
-	ofdpa_group_tbl_entry_free(match);
+	kfree(match);
 
 	if (found) {
 		err = rocker_cmd_exec(ofdpa_port->rocker_port,
 				      ofdpa_flags_nowait(flags),
 				      ofdpa_cmd_group_tbl_del,
 				      found, NULL, NULL);
-		ofdpa_group_tbl_entry_free(found);
+		kfree(found);
 	}
 
 	return err;
@@ -1166,18 +1154,13 @@ static int ofdpa_group_l2_fan_out(struct ofdpa_port *ofdpa_port,
 {
 	struct ofdpa_group_tbl_entry *entry;
 
-	entry = kzalloc_obj(*entry);
+	entry = kzalloc_flex(*entry, group_ids, group_count);
 	if (!entry)
 		return -ENOMEM;
 
-	entry->group_id = group_id;
 	entry->group_count = group_count;
+	entry->group_id = group_id;
 
-	entry->group_ids = kcalloc(group_count, sizeof(u32), GFP_KERNEL);
-	if (!entry->group_ids) {
-		kfree(entry);
-		return -ENOMEM;
-	}
 	memcpy(entry->group_ids, group_ids, group_count * sizeof(u32));
 
 	return ofdpa_group_tbl_do(ofdpa_port, flags, entry);
diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c
index 15d43e4a748bb..f7f16397a2c2a 100644
--- a/drivers/net/ethernet/spacemit/k1_emac.c
+++ b/drivers/net/ethernet/spacemit/k1_emac.c
@@ -57,7 +57,6 @@
 
 struct desc_buf {
 	u64 dma_addr;
-	void *buff_addr;
 	u16 dma_len;
 	u8 map_as_page;
 };
@@ -70,7 +69,6 @@ struct emac_tx_desc_buffer {
 struct emac_rx_desc_buffer {
 	struct sk_buff *skb;
 	u64 dma_addr;
-	void *buff_addr;
 	u16 dma_len;
 	u8 map_as_page;
 };
@@ -340,7 +338,6 @@ static void emac_free_tx_buf(struct emac_priv *priv, int i)
 
 		buf->dma_addr = 0;
 		buf->map_as_page = false;
-		buf->buff_addr = NULL;
 	}
 
 	if (tx_buf->skb) {
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 07088d03dbab5..d3a6ab7383fc1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -216,6 +216,18 @@ config DWMAC_SOPHGO
 	  for the stmmac device driver. This driver is used for the
 	  ethernet controllers on various Sophgo SoCs.
 
+config DWMAC_SPACEMIT
+	tristate "Spacemit dwmac support"
+	depends on OF && (ARCH_SPACEMIT || COMPILE_TEST)
+	select MFD_SYSCON
+	default m if ARCH_SPACEMIT
+	help
+	  Support for ethernet controllers on Spacemit RISC-V SoCs
+
+	  This selects the Spacemit platform specific glue layer support
+	  for the stmmac device driver. This driver is used for the
+	  Spacemit K3 ethernet controllers.
+
 config DWMAC_STARFIVE
 	tristate "StarFive dwmac support"
 	depends on OF && (ARCH_STARFIVE || COMPILE_TEST)
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index c9263987ef8d2..945c5354ecedc 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_DWMAC_RZN1)	+= dwmac-rzn1.o
 obj-$(CONFIG_DWMAC_S32)		+= dwmac-s32.o
 obj-$(CONFIG_DWMAC_SOCFPGA)	+= dwmac-altr-socfpga.o
 obj-$(CONFIG_DWMAC_SOPHGO)	+= dwmac-sophgo.o
+obj-$(CONFIG_DWMAC_SPACEMIT)	+= dwmac-spacemit.o
 obj-$(CONFIG_DWMAC_STARFIVE)	+= dwmac-starfive.o
 obj-$(CONFIG_DWMAC_STI)		+= dwmac-sti.o
 obj-$(CONFIG_DWMAC_STM32)	+= dwmac-stm32.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
index 120a009c99929..dc7df4208c538 100644
--- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c
@@ -46,7 +46,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
 
 	while (len != 0) {
 		tx_q->tx_skbuff[entry] = NULL;
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 		desc = tx_q->dma_tx + entry;
 
 		if (len > bmax) {
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index d26e8a0630224..8166389c853f0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -63,7 +63,7 @@ static inline bool dwmac_is_xmac(enum dwmac_core_type core_type)
 #define DMA_MIN_RX_SIZE		64
 #define DMA_MAX_RX_SIZE		1024
 #define DMA_DEFAULT_RX_SIZE	512
-#define STMMAC_GET_ENTRY(x, size)	((x + 1) & (size - 1))
+#define STMMAC_NEXT_ENTRY(x, size)	((x + 1) & (size - 1))
 
 #undef FRAME_FILTER_DEBUG
 /* #define FRAME_FILTER_DEBUG */
@@ -257,6 +257,7 @@ struct stmmac_safety_stats {
 	(sizeof(struct stmmac_safety_stats) / sizeof(unsigned long))
 
 /* CSR Frequency Access Defines*/
+#define CSR_F_20M	20000000
 #define CSR_F_35M	35000000
 #define CSR_F_60M	60000000
 #define CSR_F_100M	100000000
@@ -445,8 +446,8 @@ struct dma_features {
 	unsigned int number_rx_channel;
 	unsigned int number_tx_channel;
 	/* TX and RX number of queues */
-	unsigned int number_rx_queues;
-	unsigned int number_tx_queues;
+	u8 number_rx_queues;
+	u8 number_tx_queues;
 	/* PPS output */
 	unsigned int pps_out_num;
 	/* Number of Traffic Classes */
@@ -607,12 +608,9 @@ struct mac_link {
 struct mii_regs {
 	unsigned int addr;	/* MII Address */
 	unsigned int data;	/* MII Data */
-	unsigned int addr_shift;	/* MII address shift */
-	unsigned int reg_shift;		/* MII reg shift */
-	unsigned int addr_mask;		/* MII address mask */
-	unsigned int reg_mask;		/* MII reg mask */
-	unsigned int clk_csr_shift;
-	unsigned int clk_csr_mask;
+	u32 addr_mask;		/* MII address mask */
+	u32 reg_mask;		/* MII reg mask */
+	u32 clk_csr_mask;
 };
 
 struct mac_device_info {
@@ -635,7 +633,6 @@ struct mac_device_info {
 	unsigned int mcast_bits_log2;
 	unsigned int rx_csum;
 	unsigned int pcs;
-	unsigned int xlgmac;
 	unsigned int num_vlan;
 	u32 vlan_filter[32];
 	bool vlan_fail_q_en;
diff --git a/drivers/net/ethernet/stmicro/stmmac/descs.h b/drivers/net/ethernet/stmicro/stmmac/descs.h
index e62e2ebcf2733..846007cc245f0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/descs.h
+++ b/drivers/net/ethernet/stmicro/stmmac/descs.h
@@ -173,6 +173,8 @@ struct dma_edesc {
 	struct dma_desc basic;
 };
 
+#define dma_desc_to_edesc(x) container_of(x, struct dma_edesc, basic)
+
 /* Transmit checksum insertion control */
 #define	TX_CIC_FULL	3	/* Include IP header and pseudoheader */
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
index d043bad4a8629..b0c5d1ecabcec 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
@@ -88,7 +88,7 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev,
 	plat_dat->core_type = DWMAC_CORE_GMAC4;
 	plat_dat->dma_cfg->aal = 1;
 	plat_dat->flags |= STMMAC_FLAG_TSO_EN;
-	plat_dat->pmt = 1;
+	plat_dat->pmt = true;
 
 	return 0;
 }
@@ -112,7 +112,8 @@ static int dwc_qos_probe(struct platform_device *pdev,
 #define AUTO_CAL_STATUS 0x880c
 #define  AUTO_CAL_STATUS_ACTIVE BIT(31)
 
-static void tegra_eqos_fix_speed(void *bsp_priv, int speed, unsigned int mode)
+static void tegra_eqos_fix_speed(void *bsp_priv, phy_interface_t interface,
+				 int speed, unsigned int mode)
 {
 	struct tegra_eqos *eqos = bsp_priv;
 	bool needs_calibration = false;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
index c4e85197629dd..9d1bd72ffb730 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
@@ -42,13 +42,14 @@
 struct imx_priv_data;
 
 struct imx_dwmac_ops {
-	u32 addr_width;
 	u32 flags;
+	u8 addr_width;
 	bool mac_rgmii_txclk_auto_adj;
 
 	int (*fix_soc_reset)(struct stmmac_priv *priv);
 	int (*set_intf_mode)(struct imx_priv_data *dwmac, u8 phy_intf_sel);
-	void (*fix_mac_speed)(void *priv, int speed, unsigned int mode);
+	void (*fix_mac_speed)(void *priv, phy_interface_t interface,
+			      int speed, unsigned int mode);
 };
 
 struct imx_priv_data {
@@ -160,7 +161,8 @@ static int imx_dwmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
 	return stmmac_set_clk_tx_rate(bsp_priv, clk_tx_i, interface, speed);
 }
 
-static void imx_dwmac_fix_speed(void *priv, int speed, unsigned int mode)
+static void imx_dwmac_fix_speed(void *priv, phy_interface_t interface,
+				int speed, unsigned int mode)
 {
 	struct plat_stmmacenet_data *plat_dat;
 	struct imx_priv_data *dwmac = priv;
@@ -185,13 +187,14 @@ static void imx_dwmac_fix_speed(void *priv, int speed, unsigned int mode)
 		dev_err(dwmac->dev, "failed to set tx rate %lu\n", rate);
 }
 
-static void imx93_dwmac_fix_speed(void *priv, int speed, unsigned int mode)
+static void imx93_dwmac_fix_speed(void *priv, phy_interface_t interface,
+				  int speed, unsigned int mode)
 {
 	struct imx_priv_data *dwmac = priv;
 	unsigned int iface;
 	int ctrl, old_ctrl;
 
-	imx_dwmac_fix_speed(priv, speed, mode);
+	imx_dwmac_fix_speed(priv, interface, speed, mode);
 
 	if (!dwmac || mode != MLO_AN_FIXED)
 		return;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
index 92d77b0c2f54b..f621077c30a40 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
@@ -566,7 +566,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat)
 	/* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */
 	plat->clk_csr = STMMAC_CSR_20_35M;
 	plat->core_type = DWMAC_CORE_GMAC;
-	plat->force_sf_dma_mode = 1;
+	plat->force_sf_dma_mode = true;
 
 	plat->mdio_bus_data->needs_reset = true;
 }
@@ -589,7 +589,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev,
 	int ret;
 	int i;
 
-	plat->pdev = pdev;
+	plat->provide_bus_info = true;
 	plat->phy_addr = -1;
 	plat->clk_csr = STMMAC_CSR_250_300M;
 	plat->core_type = DWMAC_CORE_GMAC4;
@@ -636,8 +636,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev,
 
 	plat->dma_cfg->pbl = 32;
 	plat->dma_cfg->pblx8 = true;
-	plat->dma_cfg->fixed_burst = 0;
-	plat->dma_cfg->mixed_burst = 0;
 	plat->dma_cfg->aal = 0;
 	plat->dma_cfg->dche = true;
 
@@ -701,7 +699,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev,
 	/* Intel mgbe SGMII interface uses pcs-xcps */
 	if (plat->phy_interface == PHY_INTERFACE_MODE_SGMII ||
 	    plat->phy_interface == PHY_INTERFACE_MODE_1000BASEX) {
-		plat->mdio_bus_data->pcs_mask = BIT(INTEL_MGBE_XPCS_ADDR);
+		plat->mdio_bus_data->pcs_mask = BIT_U32(INTEL_MGBE_XPCS_ADDR);
 		plat->mdio_bus_data->default_an_inband = true;
 		plat->select_pcs = intel_mgbe_select_pcs;
 	}
@@ -1106,7 +1104,7 @@ static int quark_default_data(struct pci_dev *pdev,
 
 	plat->dma_cfg->pbl = 16;
 	plat->dma_cfg->pblx8 = true;
-	plat->dma_cfg->fixed_burst = 1;
+	plat->dma_cfg->fixed_burst = true;
 	/* AXI (TODO) */
 
 	return 0;
@@ -1253,11 +1251,6 @@ static int intel_eth_pci_probe(struct pci_dev *pdev,
 	if (!plat->mdio_bus_data)
 		return -ENOMEM;
 
-	plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg),
-				     GFP_KERNEL);
-	if (!plat->dma_cfg)
-		return -ENOMEM;
-
 	plat->safety_feat_cfg = devm_kzalloc(&pdev->dev,
 					     sizeof(*plat->safety_feat_cfg),
 					     GFP_KERNEL);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c
index 8152132235834..eb14c197d6aef 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c
@@ -94,7 +94,7 @@ static void loongson_default_data(struct pci_dev *pdev,
 	/* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */
 	plat->clk_csr = STMMAC_CSR_100_150M;
 	plat->core_type = DWMAC_CORE_GMAC;
-	plat->force_sf_dma_mode = 1;
+	plat->force_sf_dma_mode = true;
 
 	/* Increase the default value for multicast hash bins */
 	plat->multicast_filter_bins = 256;
@@ -143,7 +143,8 @@ static struct stmmac_pci_info loongson_gmac_pci_info = {
 	.setup = loongson_gmac_data,
 };
 
-static void loongson_gnet_fix_speed(void *priv, int speed, unsigned int mode)
+static void loongson_gnet_fix_speed(void *priv, phy_interface_t interface,
+				    int speed, unsigned int mode)
 {
 	struct loongson_data *ld = (struct loongson_data *)priv;
 	struct net_device *ndev = dev_get_drvdata(ld->dev);
@@ -167,7 +168,7 @@ static int loongson_gnet_data(struct pci_dev *pdev,
 	loongson_default_data(pdev, plat);
 
 	plat->phy_interface = PHY_INTERFACE_MODE_GMII;
-	plat->mdio_bus_data->phy_mask = ~(u32)BIT(2);
+	plat->mdio_bus_data->phy_mask = ~BIT_U32(2);
 	plat->fix_mac_speed = loongson_gnet_fix_speed;
 
 	return 0;
@@ -366,12 +367,9 @@ static int loongson_dwmac_setup(void *apriv, struct mac_device_info *mac)
 	mac->link.speed_mask = GMAC_CONTROL_PS | GMAC_CONTROL_FES;
 	mac->mii.addr = GMAC_MII_ADDR;
 	mac->mii.data = GMAC_MII_DATA;
-	mac->mii.addr_shift = 11;
-	mac->mii.addr_mask = 0x0000F800;
-	mac->mii.reg_shift = 6;
-	mac->mii.reg_mask = 0x000007C0;
-	mac->mii.clk_csr_shift = 2;
-	mac->mii.clk_csr_mask = GENMASK(5, 2);
+	mac->mii.addr_mask = GENMASK_U32(15, 11);
+	mac->mii.reg_mask = GENMASK_U32(10, 6);
+	mac->mii.clk_csr_mask = GENMASK_U32(5, 2);
 
 	return 0;
 }
@@ -515,10 +513,6 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id
 	if (!plat->mdio_bus_data)
 		return -ENOMEM;
 
-	plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL);
-	if (!plat->dma_cfg)
-		return -ENOMEM;
-
 	ld = devm_kzalloc(&pdev->dev, sizeof(*ld), GFP_KERNEL);
 	if (!ld)
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c
index 1f2d7d19ca56e..30ae0dba7fffd 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c
@@ -93,9 +93,9 @@ struct mediatek_dwmac_variant {
 	const char * const *clk_list;
 	int num_clks;
 
-	u32 dma_bit_mask;
 	u32 rx_delay_max;
 	u32 tx_delay_max;
+	u8 dma_bit_mask;
 };
 
 /* list of clocks required for mac */
@@ -268,9 +268,9 @@ static const struct mediatek_dwmac_variant mt2712_gmac_variant = {
 		.dwmac_set_delay = mt2712_set_delay,
 		.clk_list = mt2712_dwmac_clk_l,
 		.num_clks = ARRAY_SIZE(mt2712_dwmac_clk_l),
-		.dma_bit_mask = 33,
 		.rx_delay_max = 17600,
 		.tx_delay_max = 17600,
+		.dma_bit_mask = 33,
 };
 
 static int mt8195_set_interface(struct mediatek_dwmac_plat_data *plat,
@@ -418,9 +418,9 @@ static const struct mediatek_dwmac_variant mt8195_gmac_variant = {
 	.dwmac_set_delay = mt8195_set_delay,
 	.clk_list = mt8195_dwmac_clk_l,
 	.num_clks = ARRAY_SIZE(mt8195_dwmac_clk_l),
-	.dma_bit_mask = 35,
 	.rx_delay_max = 9280,
 	.tx_delay_max = 9280,
+	.dma_bit_mask = 35,
 };
 
 static int mediatek_dwmac_config_dt(struct mediatek_dwmac_plat_data *plat)
@@ -564,7 +564,7 @@ static int mediatek_dwmac_common_data(struct platform_device *pdev,
 		plat->flags &= ~STMMAC_FLAG_USE_PHY_WOL;
 	else
 		plat->flags |= STMMAC_FLAG_USE_PHY_WOL;
-	plat->riwt_off = 1;
+	plat->riwt_off = true;
 	plat->maxmtu = ETH_DATA_LEN;
 	plat->host_dma_width = priv_plat->variant->dma_bit_mask;
 	plat->bsp_priv = priv_plat;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c
index 8b45b9cf7202f..d245546b90db6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c
@@ -218,10 +218,6 @@ motorcomm_default_plat_data(struct pci_dev *pdev)
 	if (!plat->mdio_bus_data)
 		return NULL;
 
-	plat->dma_cfg = devm_kzalloc(dev, sizeof(*plat->dma_cfg), GFP_KERNEL);
-	if (!plat->dma_cfg)
-		return NULL;
-
 	plat->axi = devm_kzalloc(dev, sizeof(*plat->axi), GFP_KERNEL);
 	if (!plat->axi)
 		return NULL;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
index af8204c0e188c..3ccf20fdf52a8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
@@ -87,12 +87,12 @@ struct ethqos_emac_por {
 };
 
 struct ethqos_emac_driver_data {
-	const struct ethqos_emac_por *por;
-	unsigned int num_por;
+	const struct ethqos_emac_por *rgmii_por;
+	unsigned int num_rgmii_por;
 	bool rgmii_config_loopback_en;
 	bool has_emac_ge_3;
+	u8 dma_addr_width;
 	const char *link_clk_name;
-	u32 dma_addr_width;
 	struct dwmac4_addrs dwmac4_addrs;
 	bool needs_sgmii_loopback;
 };
@@ -100,16 +100,15 @@ struct ethqos_emac_driver_data {
 struct qcom_ethqos {
 	struct platform_device *pdev;
 	void __iomem *rgmii_base;
-	int (*configure_func)(struct qcom_ethqos *ethqos, int speed);
+	void (*configure_func)(struct qcom_ethqos *ethqos,
+			       phy_interface_t interface, int speed);
 
-	unsigned int link_clk_rate;
 	struct clk *link_clk;
 	struct phy *serdes_phy;
-	int serdes_speed;
 	phy_interface_t phy_mode;
 
-	const struct ethqos_emac_por *por;
-	unsigned int num_por;
+	const struct ethqos_emac_por *rgmii_por;
+	unsigned int num_rgmii_por;
 	bool rgmii_config_loopback_en;
 	bool has_emac_ge_3;
 	bool needs_sgmii_loopback;
@@ -174,19 +173,20 @@ static void rgmii_dump(void *priv)
 		rgmii_readl(ethqos, EMAC_SYSTEM_LOW_POWER_DEBUG));
 }
 
-static void
-ethqos_update_link_clk(struct qcom_ethqos *ethqos, int speed)
+static int ethqos_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
+				  phy_interface_t interface, int speed)
 {
+	struct qcom_ethqos *ethqos = bsp_priv;
 	long rate;
 
-	if (!phy_interface_mode_is_rgmii(ethqos->phy_mode))
-		return;
+	if (!phy_interface_mode_is_rgmii(interface))
+		return 0;
 
 	rate = rgmii_clock(speed);
-	if (rate > 0)
-		ethqos->link_clk_rate = rate * 2;
+	if (rate < 0)
+		return rate;
 
-	clk_set_rate(ethqos->link_clk, ethqos->link_clk_rate);
+	return clk_set_rate(ethqos->link_clk, rate * 2);
 }
 
 static void
@@ -204,7 +204,6 @@ qcom_ethqos_set_sgmii_loopback(struct qcom_ethqos *ethqos, bool enable)
 
 static void ethqos_set_func_clk_en(struct qcom_ethqos *ethqos)
 {
-	qcom_ethqos_set_sgmii_loopback(ethqos, true);
 	rgmii_setmask(ethqos, RGMII_CONFIG_FUNC_CLK_EN, RGMII_IO_MACRO_CONFIG);
 }
 
@@ -218,8 +217,8 @@ static const struct ethqos_emac_por emac_v2_3_0_por[] = {
 };
 
 static const struct ethqos_emac_driver_data emac_v2_3_0_data = {
-	.por = emac_v2_3_0_por,
-	.num_por = ARRAY_SIZE(emac_v2_3_0_por),
+	.rgmii_por = emac_v2_3_0_por,
+	.num_rgmii_por = ARRAY_SIZE(emac_v2_3_0_por),
 	.rgmii_config_loopback_en = true,
 	.has_emac_ge_3 = false,
 };
@@ -234,8 +233,8 @@ static const struct ethqos_emac_por emac_v2_1_0_por[] = {
 };
 
 static const struct ethqos_emac_driver_data emac_v2_1_0_data = {
-	.por = emac_v2_1_0_por,
-	.num_por = ARRAY_SIZE(emac_v2_1_0_por),
+	.rgmii_por = emac_v2_1_0_por,
+	.num_rgmii_por = ARRAY_SIZE(emac_v2_1_0_por),
 	.rgmii_config_loopback_en = false,
 	.has_emac_ge_3 = false,
 };
@@ -250,8 +249,8 @@ static const struct ethqos_emac_por emac_v3_0_0_por[] = {
 };
 
 static const struct ethqos_emac_driver_data emac_v3_0_0_data = {
-	.por = emac_v3_0_0_por,
-	.num_por = ARRAY_SIZE(emac_v3_0_0_por),
+	.rgmii_por = emac_v3_0_0_por,
+	.num_rgmii_por = ARRAY_SIZE(emac_v3_0_0_por),
 	.rgmii_config_loopback_en = false,
 	.has_emac_ge_3 = true,
 	.dwmac4_addrs = {
@@ -282,8 +281,8 @@ static const struct ethqos_emac_por emac_v4_0_0_por[] = {
 };
 
 static const struct ethqos_emac_driver_data emac_v4_0_0_data = {
-	.por = emac_v4_0_0_por,
-	.num_por = ARRAY_SIZE(emac_v4_0_0_por),
+	.rgmii_por = emac_v4_0_0_por,
+	.num_rgmii_por = ARRAY_SIZE(emac_v4_0_0_por),
 	.rgmii_config_loopback_en = false,
 	.has_emac_ge_3 = true,
 	.link_clk_name = "phyaux",
@@ -361,10 +360,12 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos)
 			      SDCC_HC_REG_DLL_CONFIG2);
 
 		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_MCLK_FREQ_CALC,
-			      0x1A << 10, SDCC_HC_REG_DLL_CONFIG2);
+			      FIELD_PREP(SDCC_DLL_CONFIG2_MCLK_FREQ_CALC, 26),
+			      SDCC_HC_REG_DLL_CONFIG2);
 
 		rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL,
-			      BIT(2), SDCC_HC_REG_DLL_CONFIG2);
+			      FIELD_PREP(SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL,
+					 1), SDCC_HC_REG_DLL_CONFIG2);
 
 		rgmii_setmask(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW,
 			      SDCC_HC_REG_DLL_CONFIG2);
@@ -425,11 +426,13 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 		if (ethqos->has_emac_ge_3) {
 			/* 0.9 ns */
 			rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY,
-				      115, SDCC_HC_REG_DDR_CONFIG);
+				      FIELD_PREP(SDCC_DDR_CONFIG_PRG_RCLK_DLY,
+						 115), SDCC_HC_REG_DDR_CONFIG);
 		} else {
 			/* 1.8 ns */
 			rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY,
-				      57, SDCC_HC_REG_DDR_CONFIG);
+				      FIELD_PREP(SDCC_DDR_CONFIG_PRG_RCLK_DLY,
+						 57), SDCC_HC_REG_DDR_CONFIG);
 		}
 		rgmii_setmask(ethqos, SDCC_DDR_CONFIG_PRG_DLY_EN,
 			      SDCC_HC_REG_DDR_CONFIG);
@@ -451,7 +454,8 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 		rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
 			      phase_shift, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_2,
-			      BIT(6), RGMII_IO_MACRO_CONFIG);
+			      FIELD_PREP(RGMII_CONFIG_MAX_SPD_PRG_2, 1),
+			      RGMII_IO_MACRO_CONFIG);
 		rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15,
 			      RGMII_IO_MACRO_CONFIG2);
 
@@ -464,7 +468,8 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 
 		/* Write 0x5 to PRG_RCLK_DLY_CODE */
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
-			      (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG);
+			      FIELD_PREP(SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
+					 5), SDCC_HC_REG_DDR_CONFIG);
 		rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY,
 			      SDCC_HC_REG_DDR_CONFIG);
 		rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN,
@@ -487,7 +492,7 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 		rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN,
 			      phase_shift, RGMII_IO_MACRO_CONFIG2);
 		rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_9,
-			      BIT(12) | GENMASK(9, 8),
+			      FIELD_PREP(RGMII_CONFIG_MAX_SPD_PRG_9, 19),
 			      RGMII_IO_MACRO_CONFIG);
 		rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15,
 			      RGMII_IO_MACRO_CONFIG2);
@@ -499,7 +504,8 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 				      RGMII_IO_MACRO_CONFIG2);
 		/* Write 0x5 to PRG_RCLK_DLY_CODE */
 		rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
-			      (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG);
+			      FIELD_PREP(SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE,
+					 5), SDCC_HC_REG_DDR_CONFIG);
 		rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY,
 			      SDCC_HC_REG_DDR_CONFIG);
 		rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN,
@@ -515,16 +521,18 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed)
 	return 0;
 }
 
-static int ethqos_configure_rgmii(struct qcom_ethqos *ethqos, int speed)
+static void ethqos_configure_rgmii(struct qcom_ethqos *ethqos,
+				   phy_interface_t interface, int speed)
 {
 	struct device *dev = &ethqos->pdev->dev;
 	unsigned int i;
 	u32 val;
 
 	/* Reset to POR values and enable clk */
-	for (i = 0; i < ethqos->num_por; i++)
-		rgmii_writel(ethqos, ethqos->por[i].value,
-			     ethqos->por[i].offset);
+	for (i = 0; i < ethqos->num_rgmii_por; i++)
+		rgmii_writel(ethqos, ethqos->rgmii_por[i].value,
+			     ethqos->rgmii_por[i].offset);
+
 	ethqos_set_func_clk_en(ethqos);
 
 	/* Initialize the DLL first */
@@ -580,16 +588,6 @@ static int ethqos_configure_rgmii(struct qcom_ethqos *ethqos, int speed)
 		ethqos_dll_configure(ethqos);
 
 	ethqos_rgmii_macro_init(ethqos, speed);
-
-	return 0;
-}
-
-static void ethqos_set_serdes_speed(struct qcom_ethqos *ethqos, int speed)
-{
-	if (ethqos->serdes_speed != speed) {
-		phy_set_speed(ethqos->serdes_phy, speed);
-		ethqos->serdes_speed = speed;
-	}
 }
 
 static void ethqos_pcs_set_inband(struct stmmac_priv *priv, bool enable)
@@ -600,53 +598,43 @@ static void ethqos_pcs_set_inband(struct stmmac_priv *priv, bool enable)
 /* On interface toggle MAC registers gets reset.
  * Configure MAC block for SGMII on ethernet phy link up
  */
-static int ethqos_configure_sgmii(struct qcom_ethqos *ethqos, int speed)
+static void ethqos_configure_sgmii(struct qcom_ethqos *ethqos,
+				   phy_interface_t interface, int speed)
 {
 	struct net_device *dev = platform_get_drvdata(ethqos->pdev);
 	struct stmmac_priv *priv = netdev_priv(dev);
 
 	switch (speed) {
 	case SPEED_2500:
-		rgmii_setmask(ethqos, RGMII_CONFIG2_RGMII_CLK_SEL_CFG,
-			      RGMII_IO_MACRO_CONFIG2);
-		ethqos_set_serdes_speed(ethqos, SPEED_2500);
-		ethqos_pcs_set_inband(priv, false);
-		break;
 	case SPEED_1000:
 		rgmii_setmask(ethqos, RGMII_CONFIG2_RGMII_CLK_SEL_CFG,
 			      RGMII_IO_MACRO_CONFIG2);
-		ethqos_set_serdes_speed(ethqos, SPEED_1000);
-		ethqos_pcs_set_inband(priv, true);
 		break;
 	case SPEED_100:
-		ethqos_set_serdes_speed(ethqos, SPEED_1000);
-		ethqos_pcs_set_inband(priv, true);
 		break;
 	case SPEED_10:
 		rgmii_updatel(ethqos, RGMII_CONFIG_SGMII_CLK_DVDR,
 			      FIELD_PREP(RGMII_CONFIG_SGMII_CLK_DVDR,
 					 SGMII_10M_RX_CLK_DVDR),
 			      RGMII_IO_MACRO_CONFIG);
-		ethqos_set_serdes_speed(ethqos, SPEED_1000);
-		ethqos_pcs_set_inband(priv, true);
 		break;
 	}
 
-	return 0;
+	ethqos_pcs_set_inband(priv, interface == PHY_INTERFACE_MODE_SGMII);
 }
 
-static int ethqos_configure(struct qcom_ethqos *ethqos, int speed)
+static void ethqos_configure(struct qcom_ethqos *ethqos,
+			     phy_interface_t interface, int speed)
 {
-	return ethqos->configure_func(ethqos, speed);
+	return ethqos->configure_func(ethqos, interface, speed);
 }
 
-static void ethqos_fix_mac_speed(void *priv, int speed, unsigned int mode)
+static void ethqos_fix_mac_speed(void *priv, phy_interface_t interface,
+				 int speed, unsigned int mode)
 {
 	struct qcom_ethqos *ethqos = priv;
 
-	qcom_ethqos_set_sgmii_loopback(ethqos, false);
-	ethqos_update_link_clk(ethqos, speed);
-	ethqos_configure(ethqos, speed);
+	ethqos_configure(ethqos, interface, speed);
 }
 
 static int qcom_ethqos_serdes_powerup(struct net_device *ndev, void *priv)
@@ -659,16 +647,8 @@ static int qcom_ethqos_serdes_powerup(struct net_device *ndev, void *priv)
 		return ret;
 
 	ret = phy_power_on(ethqos->serdes_phy);
-	if (ret) {
-		phy_exit(ethqos->serdes_phy);
-		return ret;
-	}
-
-	ret = phy_set_speed(ethqos->serdes_phy, ethqos->serdes_speed);
-	if (ret) {
-		phy_power_off(ethqos->serdes_phy);
+	if (ret)
 		phy_exit(ethqos->serdes_phy);
-	}
 
 	return ret;
 }
@@ -681,6 +661,23 @@ static void qcom_ethqos_serdes_powerdown(struct net_device *ndev, void *priv)
 	phy_exit(ethqos->serdes_phy);
 }
 
+static int ethqos_mac_finish_serdes(struct net_device *ndev, void *priv,
+				    unsigned int mode,
+				    phy_interface_t interface)
+{
+	struct qcom_ethqos *ethqos = priv;
+	int ret = 0;
+
+	qcom_ethqos_set_sgmii_loopback(ethqos, false);
+
+	if (interface == PHY_INTERFACE_MODE_SGMII ||
+	    interface == PHY_INTERFACE_MODE_2500BASEX)
+		ret = phy_set_mode_ext(ethqos->serdes_phy, PHY_MODE_ETHERNET,
+				       interface);
+
+	return ret;
+}
+
 static int ethqos_clks_config(void *priv, bool enabled)
 {
 	struct qcom_ethqos *ethqos = priv;
@@ -698,6 +695,7 @@ static int ethqos_clks_config(void *priv, bool enabled)
 		 * cycled. The actual configuration will be adjusted once
 		 * ethqos_fix_mac_speed() is invoked.
 		 */
+		qcom_ethqos_set_sgmii_loopback(ethqos, true);
 		ethqos_set_func_clk_en(ethqos);
 	} else {
 		clk_disable_unprepare(ethqos->link_clk);
@@ -766,6 +764,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
 	case PHY_INTERFACE_MODE_2500BASEX:
 	case PHY_INTERFACE_MODE_SGMII:
 		ethqos->configure_func = ethqos_configure_sgmii;
+		plat_dat->mac_finish = ethqos_mac_finish_serdes;
 		break;
 	default:
 		dev_err(dev, "Unsupported phy mode %s\n",
@@ -780,8 +779,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
 				     "Failed to map rgmii resource\n");
 
 	data = of_device_get_match_data(dev);
-	ethqos->por = data->por;
-	ethqos->num_por = data->num_por;
+	ethqos->rgmii_por = data->rgmii_por;
+	ethqos->num_rgmii_por = data->num_rgmii_por;
 	ethqos->rgmii_config_loopback_en = data->rgmii_config_loopback_en;
 	ethqos->has_emac_ge_3 = data->has_emac_ge_3;
 	ethqos->needs_sgmii_loopback = data->needs_sgmii_loopback;
@@ -804,18 +803,21 @@ static int qcom_ethqos_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(ethqos->serdes_phy),
 				     "Failed to get serdes phy\n");
 
-	ethqos->serdes_speed = SPEED_1000;
-	ethqos_update_link_clk(ethqos, SPEED_1000);
+	ethqos_set_clk_tx_rate(ethqos, NULL, plat_dat->phy_interface,
+			       SPEED_1000);
+
+	qcom_ethqos_set_sgmii_loopback(ethqos, true);
 	ethqos_set_func_clk_en(ethqos);
 
 	plat_dat->bsp_priv = ethqos;
+	plat_dat->set_clk_tx_rate = ethqos_set_clk_tx_rate;
 	plat_dat->fix_mac_speed = ethqos_fix_mac_speed;
 	plat_dat->dump_debug_regs = rgmii_dump;
 	plat_dat->ptp_clk_freq_config = ethqos_ptp_clk_freq_config;
 	plat_dat->core_type = DWMAC_CORE_GMAC4;
 	if (ethqos->has_emac_ge_3)
 		plat_dat->dwmac4_addrs = &data->dwmac4_addrs;
-	plat_dat->pmt = 1;
+	plat_dat->pmt = true;
 	if (of_property_read_bool(np, "snps,tso"))
 		plat_dat->flags |= STMMAC_FLAG_TSO_EN;
 	if (of_device_is_compatible(np, "qcom,qcs404-ethqos"))
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c
index af594a0966765..024d8e10e9188 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c
@@ -2,7 +2,7 @@
 /*
  * NXP S32G/R GMAC glue layer
  *
- * Copyright 2019-2024 NXP
+ * Copyright 2019-2026 NXP
  *
  */
 
@@ -110,6 +110,37 @@ static void s32_gmac_exit(struct device *dev, void *priv)
 	clk_disable_unprepare(gmac->rx_clk);
 }
 
+static void s32_gmac_setup_multi_irq(struct device *dev,
+				     struct plat_stmmacenet_data *plat,
+				     struct stmmac_resources *res)
+{
+	int i;
+
+	/* RX IRQs */
+	for (i = 0; i < plat->rx_queues_to_use; i++) {
+		if (res->rx_irq[i] <= 0) {
+			dev_dbg(dev, "Missing RX queue %d interrupt\n", i);
+			goto mac_irq_mode;
+		}
+	}
+
+	/* TX IRQs */
+	for (i = 0; i < plat->tx_queues_to_use; i++) {
+		if (res->tx_irq[i] <= 0) {
+			dev_dbg(dev, "Missing TX queue %d interrupt\n", i);
+			goto mac_irq_mode;
+		}
+	}
+
+	plat->flags |= STMMAC_FLAG_MULTI_MSI_EN;
+	dev_info(dev, "Multi-IRQ mode (per queue IRQs) selected\n");
+	return;
+
+mac_irq_mode:
+	plat->flags &= ~STMMAC_FLAG_MULTI_MSI_EN;
+	dev_info(dev, "MAC IRQ mode selected\n");
+}
+
 static int s32_dwmac_probe(struct platform_device *pdev)
 {
 	struct plat_stmmacenet_data *plat;
@@ -163,8 +194,11 @@ static int s32_dwmac_probe(struct platform_device *pdev)
 
 	/* S32CC core feature set */
 	plat->core_type = DWMAC_CORE_GMAC4;
-	plat->pmt = 1;
+	plat->pmt = true;
 	plat->flags |= STMMAC_FLAG_SPH_DISABLE;
+
+	s32_gmac_setup_multi_irq(dev, plat, &res);
+
 	plat->rx_fifo_size = 20480;
 	plat->tx_fifo_size = 20480;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
index 4c8991f3b38d7..5f89fd968ae94 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
@@ -72,7 +72,8 @@ struct socfpga_dwmac {
 	const struct socfpga_dwmac_ops *ops;
 };
 
-static void socfpga_dwmac_fix_mac_speed(void *bsp_priv, int speed,
+static void socfpga_dwmac_fix_mac_speed(void *bsp_priv,
+					phy_interface_t interface, int speed,
 					unsigned int mode)
 {
 	struct socfpga_dwmac *dwmac = (struct socfpga_dwmac *)bsp_priv;
@@ -564,7 +565,7 @@ static void socfpga_gen5_setup_plat_dat(struct socfpga_dwmac *dwmac)
 	plat_dat->core_type = DWMAC_CORE_GMAC;
 
 	/* Rx watchdog timer in dwmac is buggy in this hw */
-	plat_dat->riwt_off = 1;
+	plat_dat->riwt_off = true;
 }
 
 static void socfpga_agilex5_setup_plat_dat(struct socfpga_dwmac *dwmac)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c
new file mode 100644
index 0000000000000..223754cc5c79b
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Spacemit DWMAC platform driver
+ *
+ * Copyright (C) 2026 Inochi Amaoto <inochiama@gmail.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/math.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#include "stmmac_platform.h"
+
+/* ctrl register bits */
+#define CTRL_PHY_INTF_RGMII		BIT(3)
+#define CTRL_PHY_INTF_MII		BIT(4)
+#define CTRL_WAKE_IRQ_EN		BIT(9)
+#define CTRL_PHY_IRQ_EN			BIT(12)
+
+/* dline register bits */
+#define RGMII_RX_DLINE_EN		BIT(0)
+#define RGMII_RX_DLINE_STEP		GENMASK(5, 4)
+#define RGMII_RX_DLINE_CODE		GENMASK(15, 8)
+#define RGMII_TX_DLINE_EN		BIT(16)
+#define RGMII_TX_DLINE_STEP		GENMASK(21, 20)
+#define RGMII_TX_DLINE_CODE		GENMASK(31, 24)
+
+#define MAX_DLINE_DELAY_CODE		0xff
+#define MAX_WORKED_DELAY		2800
+/* Note: the delay step value is at 0.1ps */
+#define K3_DELAY_STEP			367
+
+struct spacmit_dwmac {
+	struct regmap *apmu;
+	unsigned int ctrl_offset;
+	unsigned int dline_offset;
+};
+
+static int spacemit_dwmac_set_delay(struct spacmit_dwmac *dwmac,
+				    unsigned int tx_code, unsigned int rx_code)
+{
+	unsigned int mask, val;
+
+	mask = RGMII_TX_DLINE_STEP | RGMII_TX_DLINE_CODE | RGMII_TX_DLINE_EN |
+	       RGMII_RX_DLINE_STEP | RGMII_RX_DLINE_CODE | RGMII_RX_DLINE_EN;
+
+	/*
+	 * Since the delay step provided by config 0 is small enough, and
+	 * it can cover the range of the valid delay, so there is no needed
+	 * to use other step config.
+	 */
+	val = FIELD_PREP(RGMII_TX_DLINE_STEP, 0) |
+	      FIELD_PREP(RGMII_TX_DLINE_CODE, tx_code) | RGMII_TX_DLINE_EN |
+	      FIELD_PREP(RGMII_RX_DLINE_STEP, 0) |
+	      FIELD_PREP(RGMII_RX_DLINE_CODE, rx_code) | RGMII_RX_DLINE_EN;
+
+	return regmap_update_bits(dwmac->apmu, dwmac->dline_offset,
+				  mask, val);
+}
+
+static int spacemit_dwmac_detected_delay_value(unsigned int delay)
+{
+	if (delay == 0)
+		return 0;
+
+	if (delay > MAX_WORKED_DELAY)
+		return -EINVAL;
+
+	/*
+	 * Note K3 require a specific factor for calculate
+	 * the delay, in this scenario it is 0.9. So the
+	 * formula is code * step / 10 * 0.9
+	 */
+	return DIV_ROUND_CLOSEST(delay * 10 * 10, K3_DELAY_STEP * 9);
+}
+
+static int spacemit_dwmac_fix_delay(struct spacmit_dwmac *dwmac,
+				    unsigned int tx_delay,
+				    unsigned int rx_delay)
+{
+	int rx_code;
+	int tx_code;
+
+	rx_code = spacemit_dwmac_detected_delay_value(rx_delay);
+	if (rx_code < 0)
+		return rx_code;
+
+	tx_code = spacemit_dwmac_detected_delay_value(tx_delay);
+	if (tx_code < 0)
+		return tx_code;
+
+	return spacemit_dwmac_set_delay(dwmac, tx_code, rx_code);
+}
+
+static int spacemit_dwmac_update_irq_config(struct spacmit_dwmac *dwmac,
+					    struct stmmac_resources *stmmac_res)
+{
+	unsigned int val = stmmac_res->wol_irq >= 0 ? CTRL_WAKE_IRQ_EN : 0;
+	unsigned int mask = CTRL_WAKE_IRQ_EN;
+
+	return regmap_update_bits(dwmac->apmu, dwmac->ctrl_offset,
+				  mask, val);
+}
+
+static void spacemit_get_interfaces(struct stmmac_priv *priv, void *bsp_priv,
+				    unsigned long *interfaces)
+{
+	__set_bit(PHY_INTERFACE_MODE_MII, interfaces);
+	__set_bit(PHY_INTERFACE_MODE_RMII, interfaces);
+	phy_interface_set_rgmii(interfaces);
+}
+
+static int spacemit_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel)
+{
+	unsigned int mask = CTRL_PHY_INTF_MII | CTRL_PHY_INTF_RGMII;
+	struct spacmit_dwmac *dwmac = bsp_priv;
+	unsigned int val = 0;
+
+	switch (phy_intf_sel) {
+	case PHY_INTF_SEL_GMII_MII:
+		val = CTRL_PHY_INTF_MII;
+		break;
+
+	case PHY_INTF_SEL_RMII:
+		break;
+
+	case PHY_INTF_SEL_RGMII:
+		val = CTRL_PHY_INTF_RGMII;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(dwmac->apmu, dwmac->ctrl_offset,
+				  mask, val);
+}
+
+static int spacemit_dwmac_probe(struct platform_device *pdev)
+{
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct device *dev = &pdev->dev;
+	struct spacmit_dwmac *dwmac;
+	unsigned int offset[2];
+	struct regmap *apmu;
+	struct clk *clk_tx;
+	u32 rx_delay = 0;
+	u32 tx_delay = 0;
+	int ret;
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to get platform resources\n");
+
+	dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL);
+	if (!dwmac)
+		return -ENOMEM;
+
+	plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return dev_err_probe(dev, PTR_ERR(plat_dat),
+				     "failed to parse DT parameters\n");
+
+	clk_tx = devm_clk_get_enabled(&pdev->dev, "tx");
+	if (IS_ERR(clk_tx))
+		return dev_err_probe(&pdev->dev, PTR_ERR(clk_tx),
+				     "failed to get tx clock\n");
+
+	apmu = syscon_regmap_lookup_by_phandle_args(pdev->dev.of_node,
+						    "spacemit,apmu", 2,
+						    offset);
+	if (IS_ERR(apmu))
+		return dev_err_probe(dev, PTR_ERR(apmu),
+				     "Failed to get apmu regmap\n");
+
+	dwmac->apmu = apmu;
+	dwmac->ctrl_offset = offset[0];
+	dwmac->dline_offset = offset[1];
+
+	ret = spacemit_dwmac_update_irq_config(dwmac, &stmmac_res);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to configure irq config\n");
+
+	of_property_read_u32(pdev->dev.of_node, "tx-internal-delay-ps",
+			     &tx_delay);
+	of_property_read_u32(pdev->dev.of_node, "rx-internal-delay-ps",
+			     &rx_delay);
+
+	plat_dat->get_interfaces = spacemit_get_interfaces;
+	plat_dat->set_phy_intf_sel = spacemit_set_phy_intf_sel;
+	plat_dat->bsp_priv = dwmac;
+
+	ret = spacemit_dwmac_fix_delay(dwmac, tx_delay, rx_delay);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to configure delay\n");
+
+	return stmmac_dvr_probe(dev, plat_dat, &stmmac_res);
+}
+
+static const struct of_device_id spacemit_dwmac_match[] = {
+	{ .compatible = "spacemit,k3-dwmac" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, spacemit_dwmac_match);
+
+static struct platform_driver spacemit_dwmac_driver = {
+	.probe  = spacemit_dwmac_probe,
+	.remove = stmmac_pltfr_remove,
+	.driver = {
+		.name = "spacemit-dwmac",
+		.pm = &stmmac_pltfr_pm_ops,
+		.of_match_table = spacemit_dwmac_match,
+	},
+};
+module_platform_driver(spacemit_dwmac_driver);
+
+MODULE_AUTHOR("Inochi Amaoto <inochiama@gmail.com>");
+MODULE_DESCRIPTION("Spacemit DWMAC platform driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
index f50547b67fbc8..6ebbf95d158f2 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c
@@ -91,11 +91,13 @@ struct sti_dwmac {
 	struct regmap *regmap;
 	bool gmac_en;
 	int speed;
-	void (*fix_retime_src)(void *priv, int speed, unsigned int mode);
+	void (*fix_retime_src)(void *priv, phy_interface_t interface,
+			       int speed, unsigned int mode);
 };
 
 struct sti_dwmac_of_data {
-	void (*fix_retime_src)(void *priv, int speed, unsigned int mode);
+	void (*fix_retime_src)(void *priv, phy_interface_t interface,
+			       int speed, unsigned int mode);
 };
 
 enum {
@@ -114,7 +116,8 @@ static u32 stih4xx_tx_retime_val[] = {
 				 | STIH4XX_ETH_SEL_INTERNAL_NOTEXT_PHYCLK,
 };
 
-static void stih4xx_fix_retime_src(void *priv, int spd, unsigned int mode)
+static void stih4xx_fix_retime_src(void *priv, phy_interface_t interface,
+				   int spd, unsigned int mode)
 {
 	struct sti_dwmac *dwmac = priv;
 	u32 src = dwmac->tx_retime_src;
@@ -170,7 +173,7 @@ static int sti_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel)
 	val = (dwmac->interface == PHY_INTERFACE_MODE_REVMII) ? 0 : ENMII;
 	regmap_update_bits(regmap, reg, ENMII_MASK, val);
 
-	dwmac->fix_retime_src(dwmac, dwmac->speed, 0);
+	dwmac->fix_retime_src(dwmac, dwmac->interface, dwmac->speed, 0);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index c01b86fd64da1..48c52eb962335 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -718,7 +718,7 @@ static void sun8i_dwmac_set_filter(struct mac_device_info *hw,
 
 static void sun8i_dwmac_flow_ctrl(struct mac_device_info *hw,
 				  unsigned int duplex, unsigned int fc,
-				  unsigned int pause_time, u32 tx_cnt)
+				  unsigned int pause_time, u8 tx_cnt)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	u32 v;
@@ -1063,12 +1063,9 @@ static int sun8i_dwmac_setup(void *ppriv, struct mac_device_info *mac)
 	mac->link.duplex = EMAC_DUPLEX_FULL;
 	mac->mii.addr = EMAC_MDIO_CMD;
 	mac->mii.data = EMAC_MDIO_DATA;
-	mac->mii.reg_shift = 4;
-	mac->mii.reg_mask = GENMASK(8, 4);
-	mac->mii.addr_shift = 12;
-	mac->mii.addr_mask = GENMASK(16, 12);
-	mac->mii.clk_csr_shift = 20;
-	mac->mii.clk_csr_mask = GENMASK(22, 20);
+	mac->mii.reg_mask = GENMASK_U32(8, 4);
+	mac->mii.addr_mask = GENMASK_U32(16, 12);
+	mac->mii.clk_csr_mask = GENMASK_U32(22, 20);
 	mac->unicast_filter_entries = 8;
 
 	/* Synopsys Id is not available */
@@ -1182,7 +1179,7 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
 	 * hardware features were copied from Allwinner drivers.
 	 */
 	plat_dat->rx_coe = STMMAC_RX_COE_TYPE2;
-	plat_dat->tx_coe = 1;
+	plat_dat->tx_coe = true;
 	plat_dat->flags |= STMMAC_FLAG_HAS_SUN8I;
 	plat_dat->bsp_priv = gmac;
 	plat_dat->init = sun8i_dwmac_init;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c
index 52593ba3a3a31..74bd996d93c90 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c
@@ -135,7 +135,7 @@ static int sun7i_gmac_probe(struct platform_device *pdev)
 
 	/* platform data specifying hardware features and callbacks.
 	 * hardware features were copied from Allwinner drivers. */
-	plat_dat->tx_coe = 1;
+	plat_dat->tx_coe = true;
 	plat_dat->core_type = DWMAC_CORE_GMAC;
 	plat_dat->bsp_priv = gmac;
 	plat_dat->init = sun7i_gmac_init;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c
index d765acbe37548..b4b39e6a169e4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c
@@ -310,7 +310,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev)
 
 	plat->core_type = DWMAC_CORE_XGMAC;
 	plat->flags |= STMMAC_FLAG_TSO_EN;
-	plat->pmt = 1;
+	plat->pmt = true;
 	plat->bsp_priv = mgbe;
 
 	if (!plat->mdio_node)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index af566636fad90..01f8353eb6ef2 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -222,7 +222,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw,
 
 static void dwmac1000_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
 				unsigned int fc, unsigned int pause_time,
-				u32 tx_cnt)
+				u8 tx_cnt)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	/* Set flow such that DZPQ in Mac Register 6 is 0,
@@ -496,12 +496,9 @@ int dwmac1000_setup(struct stmmac_priv *priv)
 	mac->link.speed_mask = GMAC_CONTROL_PS | GMAC_CONTROL_FES;
 	mac->mii.addr = GMAC_MII_ADDR;
 	mac->mii.data = GMAC_MII_DATA;
-	mac->mii.addr_shift = 11;
-	mac->mii.addr_mask = 0x0000F800;
-	mac->mii.reg_shift = 6;
-	mac->mii.reg_mask = 0x000007C0;
-	mac->mii.clk_csr_shift = 2;
-	mac->mii.clk_csr_mask = GENMASK(5, 2);
+	mac->mii.addr_mask = GENMASK_U32(15, 11);
+	mac->mii.reg_mask = GENMASK_U32(10, 6);
+	mac->mii.clk_csr_mask = GENMASK_U32(5, 2);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
index db4fbe64a38af..94d24d355d957 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
@@ -126,7 +126,7 @@ static void dwmac100_set_filter(struct mac_device_info *hw,
 
 static void dwmac100_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
 			       unsigned int fc, unsigned int pause_time,
-			       u32 tx_cnt)
+			       u8 tx_cnt)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	unsigned int flow = MAC_FLOW_CTRL_ENABLE;
@@ -184,12 +184,9 @@ int dwmac100_setup(struct stmmac_priv *priv)
 	mac->link.speed_mask = MAC_CONTROL_PS;
 	mac->mii.addr = MAC_MII_ADDR;
 	mac->mii.data = MAC_MII_DATA;
-	mac->mii.addr_shift = 11;
-	mac->mii.addr_mask = 0x0000F800;
-	mac->mii.reg_shift = 6;
-	mac->mii.reg_mask = 0x000007C0;
-	mac->mii.clk_csr_shift = 2;
-	mac->mii.clk_csr_mask = GENMASK(5, 2);
+	mac->mii.addr_mask = GENMASK_U32(15, 11);
+	mac->mii.reg_mask = GENMASK_U32(10, 6);
+	mac->mii.clk_csr_mask = GENMASK_U32(5, 2);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 623868afe93d2..4c6fed3ecbcf5 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -373,8 +373,8 @@ static void dwmac4_get_umac_addr(struct mac_device_info *hw,
 {
 	void __iomem *ioaddr = hw->pcsr;
 
-	stmmac_dwmac4_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
-				   GMAC_ADDR_LOW(reg_n));
+	stmmac_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
+			    GMAC_ADDR_LOW(reg_n));
 }
 
 static int dwmac4_set_lpi_mode(struct mac_device_info *hw,
@@ -547,11 +547,11 @@ static void dwmac4_set_filter(struct mac_device_info *hw,
 
 static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
 			     unsigned int fc, unsigned int pause_time,
-			     u32 tx_cnt)
+			     u8 tx_cnt)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	unsigned int flow = 0;
-	u32 queue = 0;
+	u8 queue;
 
 	pr_debug("GMAC Flow-Control:\n");
 	if (fc & FLOW_RX) {
@@ -1030,12 +1030,9 @@ int dwmac4_setup(struct stmmac_priv *priv)
 	mac->link.speed_mask = GMAC_CONFIG_FES | GMAC_CONFIG_PS;
 	mac->mii.addr = GMAC_MDIO_ADDR;
 	mac->mii.data = GMAC_MDIO_DATA;
-	mac->mii.addr_shift = 21;
-	mac->mii.addr_mask = GENMASK(25, 21);
-	mac->mii.reg_shift = 16;
-	mac->mii.reg_mask = GENMASK(20, 16);
-	mac->mii.clk_csr_shift = 8;
-	mac->mii.clk_csr_mask = GENMASK(11, 8);
+	mac->mii.addr_mask = GENMASK_U32(25, 21);
+	mac->mii.reg_mask = GENMASK_U32(20, 16);
+	mac->mii.clk_csr_mask = GENMASK_U32(11, 8);
 	mac->num_vlan = stmmac_get_num_vlan(priv->ioaddr);
 
 	return 0;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index e226dc6a1b17d..2994df41ec2c4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -171,16 +171,6 @@ static int dwmac4_wrback_get_rx_status(struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static int dwmac4_rd_get_tx_len(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des2) & TDES2_BUFFER1_SIZE_MASK);
-}
-
-static int dwmac4_get_tx_owner(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des3) & TDES3_OWN) >> TDES3_OWN_SHIFT;
-}
-
 static void dwmac4_set_tx_owner(struct dma_desc *p)
 {
 	p->des3 |= cpu_to_le32(TDES3_OWN);
@@ -196,12 +186,6 @@ static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 	p->des3 |= cpu_to_le32(flags);
 }
 
-static int dwmac4_get_tx_ls(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des3) & TDES3_LAST_DESCRIPTOR)
-		>> TDES3_LAST_DESCRIPTOR_SHIFT;
-}
-
 static u16 dwmac4_wrback_get_rx_vlan_tci(struct dma_desc *p)
 {
 	return (le32_to_cpu(p->des0) & RDES0_VLAN_TAG_MASK);
@@ -305,12 +289,13 @@ static int dwmac4_wrback_get_rx_timestamp_status(void *desc, void *next_desc,
 }
 
 static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
-				   int mode, int end, int bfsize)
+				   u8 descriptor_mode, int end, int bfsize)
 {
 	dwmac4_set_rx_owner(p, disable_rx_ic);
 }
 
-static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
+static void dwmac4_rd_init_tx_desc(struct dma_desc *p, u8 descriptor_mode,
+				   int end)
 {
 	p->des0 = 0;
 	p->des1 = 0;
@@ -319,8 +304,9 @@ static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
 }
 
 static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
-				      bool csum_flag, int mode, bool tx_own,
-				      bool ls, unsigned int tot_pkt_len)
+				      bool csum_flag, u8 descriptor_mode,
+				      bool tx_own, bool ls,
+				      unsigned int tot_pkt_len)
 {
 	u32 tdes3 = le32_to_cpu(p->des3);
 
@@ -397,7 +383,7 @@ static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
 	p->des3 = cpu_to_le32(tdes3);
 }
 
-static void dwmac4_release_tx_desc(struct dma_desc *p, int mode)
+static void dwmac4_release_tx_desc(struct dma_desc *p, u8 descriptor_mode)
 {
 	p->des0 = 0;
 	p->des1 = 0;
@@ -443,7 +429,7 @@ static void dwmac4_display_ring(void *head, unsigned int size, bool rx,
 			extp++;
 		}
 	} else if (desc_size == sizeof(struct dma_edesc)) {
-		struct dma_edesc *ep = (struct dma_edesc *)head;
+		struct dma_edesc *ep = dma_desc_to_edesc(head);
 
 		for (i = 0; i < size; i++) {
 			dma_addr = dma_rx_phy + i * sizeof(*ep);
@@ -551,11 +537,8 @@ static void dwmac4_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec)
 const struct stmmac_desc_ops dwmac4_desc_ops = {
 	.tx_status = dwmac4_wrback_get_tx_status,
 	.rx_status = dwmac4_wrback_get_rx_status,
-	.get_tx_len = dwmac4_rd_get_tx_len,
-	.get_tx_owner = dwmac4_get_tx_owner,
 	.set_tx_owner = dwmac4_set_tx_owner,
 	.set_rx_owner = dwmac4_set_rx_owner,
-	.get_tx_ls = dwmac4_get_tx_ls,
 	.get_rx_vlan_tci = dwmac4_wrback_get_rx_vlan_tci,
 	.get_rx_vlan_valid = dwmac4_wrback_get_rx_vlan_valid,
 	.get_rx_frame_len = dwmac4_wrback_get_rx_frame_len,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
index 60b880cdd9da2..28728271fbc9e 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
@@ -582,7 +582,7 @@ const struct stmmac_dma_ops dwmac410_dma_ops = {
 	.dump_regs = dwmac4_dump_dma_regs,
 	.dma_rx_mode = dwmac4_dma_rx_chan_op_mode,
 	.dma_tx_mode = dwmac4_dma_tx_chan_op_mode,
-	.enable_dma_irq = dwmac410_enable_dma_irq,
+	.enable_dma_irq = dwmac4_enable_dma_irq,
 	.disable_dma_irq = dwmac4_disable_dma_irq,
 	.start_tx = dwmac4_dma_start_tx,
 	.stop_tx = dwmac4_dma_stop_tx,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
index 9d9077a4ac9f1..af6580332d49f 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h
@@ -111,8 +111,6 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs,
 /* DMA default interrupt mask for 4.00 */
 #define DMA_CHAN_INTR_DEFAULT_MASK	(DMA_CHAN_INTR_NORMAL | \
 					 DMA_CHAN_INTR_ABNORMAL)
-#define DMA_CHAN_INTR_DEFAULT_RX	(DMA_CHAN_INTR_ENA_RIE)
-#define DMA_CHAN_INTR_DEFAULT_TX	(DMA_CHAN_INTR_ENA_TIE)
 
 #define DMA_CHAN_INTR_NORMAL_4_10	(DMA_CHAN_INTR_ENA_NIE_4_10 | \
 					 DMA_CHAN_INTR_ENA_RIE | \
@@ -123,8 +121,6 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs,
 /* DMA default interrupt mask for 4.10a */
 #define DMA_CHAN_INTR_DEFAULT_MASK_4_10	(DMA_CHAN_INTR_NORMAL_4_10 | \
 					 DMA_CHAN_INTR_ABNORMAL_4_10)
-#define DMA_CHAN_INTR_DEFAULT_RX_4_10	(DMA_CHAN_INTR_ENA_RIE)
-#define DMA_CHAN_INTR_DEFAULT_TX_4_10	(DMA_CHAN_INTR_ENA_TIE)
 
 #define DMA_CHAN_RX_WATCHDOG(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x38)
 #define DMA_CHAN_SLOT_CTRL_STATUS(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x3c)
@@ -174,12 +170,8 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs,
 int dwmac4_dma_reset(void __iomem *ioaddr);
 void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
 			   u32 chan, bool rx, bool tx);
-void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
-			     u32 chan, bool rx, bool tx);
 void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
 			    u32 chan, bool rx, bool tx);
-void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
-			      u32 chan, bool rx, bool tx);
 void dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
 			 u32 chan);
 void dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
index c098047a3bff8..a0249715fafaf 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c
@@ -116,23 +116,9 @@ void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
 	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
-		value |= DMA_CHAN_INTR_DEFAULT_RX;
+		value |= DMA_CHAN_INTR_ENA_RIE;
 	if (tx)
-		value |= DMA_CHAN_INTR_DEFAULT_TX;
-
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
-}
-
-void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
-			     u32 chan, bool rx, bool tx)
-{
-	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
-
-	if (rx)
-		value |= DMA_CHAN_INTR_DEFAULT_RX_4_10;
-	if (tx)
-		value |= DMA_CHAN_INTR_DEFAULT_TX_4_10;
+		value |= DMA_CHAN_INTR_ENA_TIE;
 
 	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
@@ -144,23 +130,9 @@ void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
 	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 
 	if (rx)
-		value &= ~DMA_CHAN_INTR_DEFAULT_RX;
-	if (tx)
-		value &= ~DMA_CHAN_INTR_DEFAULT_TX;
-
-	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
-}
-
-void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
-			      u32 chan, bool rx, bool tx)
-{
-	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
-	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
-
-	if (rx)
-		value &= ~DMA_CHAN_INTR_DEFAULT_RX_4_10;
+		value &= ~DMA_CHAN_INTR_ENA_RIE;
 	if (tx)
-		value &= ~DMA_CHAN_INTR_DEFAULT_TX_4_10;
+		value &= ~DMA_CHAN_INTR_ENA_TIE;
 
 	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
 }
@@ -254,21 +226,3 @@ void stmmac_dwmac4_set_mac(void __iomem *ioaddr, bool enable)
 	if (value != old_val)
 		writel(value, ioaddr + GMAC_CONFIG);
 }
-
-void stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
-				unsigned int high, unsigned int low)
-{
-	unsigned int hi_addr, lo_addr;
-
-	/* Read the MAC address from the hardware */
-	hi_addr = readl(ioaddr + high);
-	lo_addr = readl(ioaddr + low);
-
-	/* Extract the MAC address from the high and low words */
-	addr[0] = lo_addr & 0xff;
-	addr[1] = (lo_addr >> 8) & 0xff;
-	addr[2] = (lo_addr >> 16) & 0xff;
-	addr[3] = (lo_addr >> 24) & 0xff;
-	addr[4] = hi_addr & 0xff;
-	addr[5] = (hi_addr >> 8) & 0xff;
-}
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
index 1c431b918719e..f1bb981cab7cd 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c
@@ -525,8 +525,6 @@ int dwmac5_flex_pps_config(void __iomem *ioaddr, int index,
 	u32 val = readl(ioaddr + MAC_PPS_CONTROL);
 	u64 period;
 
-	if (!cfg->available)
-		return -EINVAL;
 	if (tnsec & TRGTBUSY0)
 		return -EBUSY;
 	if (!sub_second_inc || !systime_flags)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
index 49893b9fb88c4..f02b434bbd505 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
@@ -355,10 +355,10 @@ static int dwxgmac2_host_mtl_irq_status(struct stmmac_priv *priv,
 
 static void dwxgmac2_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
 			       unsigned int fc, unsigned int pause_time,
-			       u32 tx_cnt)
+			       u8 tx_cnt)
 {
 	void __iomem *ioaddr = hw->pcsr;
-	u32 i;
+	u8 i;
 
 	if (fc & FLOW_RX)
 		writel(XGMAC_RFE, ioaddr + XGMAC_RX_FLOW_CTRL);
@@ -1162,8 +1162,6 @@ static int dwxgmac2_flex_pps_config(void __iomem *ioaddr, int index,
 	u32 val = readl(ioaddr + XGMAC_PPS_CONTROL);
 	u64 period;
 
-	if (!cfg->available)
-		return -EINVAL;
 	if (tnsec & XGMAC_TRGTBUSY0)
 		return -EBUSY;
 	if (!sub_second_inc || !systime_flags)
@@ -1551,12 +1549,9 @@ int dwxgmac2_setup(struct stmmac_priv *priv)
 
 	mac->mii.addr = XGMAC_MDIO_ADDR;
 	mac->mii.data = XGMAC_MDIO_DATA;
-	mac->mii.addr_shift = 16;
-	mac->mii.addr_mask = GENMASK(20, 16);
-	mac->mii.reg_shift = 0;
-	mac->mii.reg_mask = GENMASK(15, 0);
-	mac->mii.clk_csr_shift = 19;
-	mac->mii.clk_csr_mask = GENMASK(21, 19);
+	mac->mii.addr_mask = GENMASK_U32(20, 16);
+	mac->mii.reg_mask = GENMASK_U32(15, 0);
+	mac->mii.clk_csr_mask = GENMASK_U32(21, 19);
 	mac->num_vlan = stmmac_get_num_vlan(priv->ioaddr);
 
 	return 0;
@@ -1594,12 +1589,9 @@ int dwxlgmac2_setup(struct stmmac_priv *priv)
 
 	mac->mii.addr = XGMAC_MDIO_ADDR;
 	mac->mii.data = XGMAC_MDIO_DATA;
-	mac->mii.addr_shift = 16;
-	mac->mii.addr_mask = GENMASK(20, 16);
-	mac->mii.reg_shift = 0;
-	mac->mii.reg_mask = GENMASK(15, 0);
-	mac->mii.clk_csr_shift = 19;
-	mac->mii.clk_csr_mask = GENMASK(21, 19);
+	mac->mii.addr_mask = GENMASK_U32(20, 16);
+	mac->mii.reg_mask = GENMASK_U32(15, 0);
+	mac->mii.clk_csr_mask = GENMASK_U32(21, 19);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
index 41e5b420a2151..b5f200a874840 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c
@@ -40,16 +40,6 @@ static int dwxgmac2_get_rx_status(struct stmmac_extra_stats *x,
 	return good_frame;
 }
 
-static int dwxgmac2_get_tx_len(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des2) & XGMAC_TDES2_B1L);
-}
-
-static int dwxgmac2_get_tx_owner(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des3) & XGMAC_TDES3_OWN) > 0;
-}
-
 static void dwxgmac2_set_tx_owner(struct dma_desc *p)
 {
 	p->des3 |= cpu_to_le32(XGMAC_TDES3_OWN);
@@ -65,11 +55,6 @@ static void dwxgmac2_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 	p->des3 |= cpu_to_le32(flags);
 }
 
-static int dwxgmac2_get_tx_ls(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des3) & XGMAC_RDES3_LD) > 0;
-}
-
 static u16 dwxgmac2_wrback_get_rx_vlan_tci(struct dma_desc *p)
 {
 	return le32_to_cpu(p->des0) & XGMAC_RDES0_VLAN_TAG_MASK;
@@ -145,12 +130,13 @@ static int dwxgmac2_get_rx_timestamp_status(void *desc, void *next_desc,
 }
 
 static void dwxgmac2_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
-				  int mode, int end, int bfsize)
+				  u8 descriptor_mode, int end, int bfsize)
 {
 	dwxgmac2_set_rx_owner(p, disable_rx_ic);
 }
 
-static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end)
+static void dwxgmac2_init_tx_desc(struct dma_desc *p, u8 descriptor_mode,
+				  int end)
 {
 	p->des0 = 0;
 	p->des1 = 0;
@@ -159,8 +145,9 @@ static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end)
 }
 
 static void dwxgmac2_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
-				     bool csum_flag, int mode, bool tx_own,
-				     bool ls, unsigned int tot_pkt_len)
+				     bool csum_flag, u8 descriptor_mode,
+				     bool tx_own, bool ls,
+				     unsigned int tot_pkt_len)
 {
 	u32 tdes3 = le32_to_cpu(p->des3);
 
@@ -234,7 +221,7 @@ static void dwxgmac2_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
 	p->des3 = cpu_to_le32(tdes3);
 }
 
-static void dwxgmac2_release_tx_desc(struct dma_desc *p, int mode)
+static void dwxgmac2_release_tx_desc(struct dma_desc *p, u8 descriptor_mode)
 {
 	p->des0 = 0;
 	p->des1 = 0;
@@ -355,11 +342,8 @@ static void dwxgmac2_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec)
 const struct stmmac_desc_ops dwxgmac210_desc_ops = {
 	.tx_status = dwxgmac2_get_tx_status,
 	.rx_status = dwxgmac2_get_rx_status,
-	.get_tx_len = dwxgmac2_get_tx_len,
-	.get_tx_owner = dwxgmac2_get_tx_owner,
 	.set_tx_owner = dwxgmac2_set_tx_owner,
 	.set_rx_owner = dwxgmac2_set_rx_owner,
-	.get_tx_ls = dwxgmac2_get_tx_ls,
 	.get_rx_vlan_tci = dwxgmac2_wrback_get_rx_vlan_tci,
 	.get_rx_vlan_valid = dwxgmac2_wrback_get_rx_vlan_valid,
 	.get_rx_frame_len = dwxgmac2_get_rx_frame_len,
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 8f6993c8bcae5..0512536012251 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -76,11 +76,6 @@ static int enh_desc_get_tx_status(struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static int enh_desc_get_tx_len(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des1) & ETDES1_BUFFER1_SIZE_MASK);
-}
-
 static int enh_desc_coe_rdes0(int ipc_err, int type, int payload_err)
 {
 	int ret = good_frame;
@@ -250,7 +245,7 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x,
 }
 
 static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
-				  int mode, int end, int bfsize)
+				  u8 descriptor_mode, int end, int bfsize)
 {
 	int bfsize1;
 
@@ -259,7 +254,7 @@ static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
 	bfsize1 = min(bfsize, BUF_SIZE_8KiB);
 	p->des1 |= cpu_to_le32(bfsize1 & ERDES1_BUFFER1_SIZE_MASK);
 
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		ehn_desc_rx_set_on_chain(p);
 	else
 		ehn_desc_rx_set_on_ring(p, end, bfsize);
@@ -268,20 +263,16 @@ static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
 		p->des1 |= cpu_to_le32(ERDES1_DISABLE_IC);
 }
 
-static void enh_desc_init_tx_desc(struct dma_desc *p, int mode, int end)
+static void enh_desc_init_tx_desc(struct dma_desc *p, u8 descriptor_mode,
+				  int end)
 {
 	p->des0 &= cpu_to_le32(~ETDES0_OWN);
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		enh_desc_end_tx_desc_on_chain(p);
 	else
 		enh_desc_end_tx_desc_on_ring(p, end);
 }
 
-static int enh_desc_get_tx_owner(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des0) & ETDES0_OWN) >> 31;
-}
-
 static void enh_desc_set_tx_owner(struct dma_desc *p)
 {
 	p->des0 |= cpu_to_le32(ETDES0_OWN);
@@ -292,29 +283,25 @@ static void enh_desc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 	p->des0 |= cpu_to_le32(RDES0_OWN);
 }
 
-static int enh_desc_get_tx_ls(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des0) & ETDES0_LAST_SEGMENT) >> 29;
-}
-
-static void enh_desc_release_tx_desc(struct dma_desc *p, int mode)
+static void enh_desc_release_tx_desc(struct dma_desc *p, u8 descriptor_mode)
 {
 	int ter = (le32_to_cpu(p->des0) & ETDES0_END_RING) >> 21;
 
 	memset(p, 0, offsetof(struct dma_desc, des2));
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		enh_desc_end_tx_desc_on_chain(p);
 	else
 		enh_desc_end_tx_desc_on_ring(p, ter);
 }
 
 static void enh_desc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
-				     bool csum_flag, int mode, bool tx_own,
-				     bool ls, unsigned int tot_pkt_len)
+				     bool csum_flag, u8 descriptor_mode,
+				     bool tx_own,  bool ls,
+				     unsigned int tot_pkt_len)
 {
 	u32 tdes0 = le32_to_cpu(p->des0);
 
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		enh_set_tx_desc_len_on_chain(p, len);
 	else
 		enh_set_tx_desc_len_on_ring(p, len);
@@ -445,14 +432,11 @@ static void enh_desc_clear(struct dma_desc *p)
 const struct stmmac_desc_ops enh_desc_ops = {
 	.tx_status = enh_desc_get_tx_status,
 	.rx_status = enh_desc_get_rx_status,
-	.get_tx_len = enh_desc_get_tx_len,
 	.init_rx_desc = enh_desc_init_rx_desc,
 	.init_tx_desc = enh_desc_init_tx_desc,
-	.get_tx_owner = enh_desc_get_tx_owner,
 	.release_tx_desc = enh_desc_release_tx_desc,
 	.prepare_tx_desc = enh_desc_prepare_tx_desc,
 	.set_tx_ic = enh_desc_set_tx_ic,
-	.get_tx_ls = enh_desc_get_tx_ls,
 	.set_tx_owner = enh_desc_set_tx_owner,
 	.set_rx_owner = enh_desc_set_rx_owner,
 	.get_rx_frame_len = enh_desc_get_rx_frame_len,
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c
index 7e69ff4b9a98d..511b0fd5e8343 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.c
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c
@@ -57,11 +57,11 @@ static void stmmac_dwmac_mode_quirk(struct stmmac_priv *priv)
 
 	if (priv->chain_mode) {
 		dev_info(priv->device, "Chain mode enabled\n");
-		priv->mode = STMMAC_CHAIN_MODE;
+		priv->descriptor_mode = STMMAC_CHAIN_MODE;
 		mac->mode = &chain_mode_ops;
 	} else {
 		dev_info(priv->device, "Ring mode enabled\n");
-		priv->mode = STMMAC_RING_MODE;
+		priv->descriptor_mode = STMMAC_RING_MODE;
 		mac->mode = &ring_mode_ops;
 	}
 }
@@ -76,7 +76,7 @@ static int stmmac_dwmac1_quirks(struct stmmac_priv *priv)
 		/* GMAC older than 3.50 has no extended descriptors */
 		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
 			dev_info(priv->device, "Enabled extended descriptors\n");
-			priv->extend_desc = 1;
+			priv->extend_desc = true;
 		} else {
 			dev_warn(priv->device, "Extended descriptors not supported\n");
 		}
@@ -97,12 +97,6 @@ static int stmmac_dwmac4_quirks(struct stmmac_priv *priv)
 	return 0;
 }
 
-static int stmmac_dwxlgmac_quirks(struct stmmac_priv *priv)
-{
-	priv->hw->xlgmac = true;
-	return 0;
-}
-
 int stmmac_reset(struct stmmac_priv *priv)
 {
 	struct plat_stmmacenet_data *plat = priv->plat;
@@ -293,7 +287,6 @@ static const struct stmmac_hwif_entry {
 		.mmc = &dwxgmac_mmc_ops,
 		.est = &dwmac510_est_ops,
 		.setup = dwxlgmac2_setup,
-		.quirks = stmmac_dwxlgmac_quirks,
 	},
 };
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
index 0db96a3872592..e6317b94fff7d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.h
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -38,27 +38,24 @@ struct dma_edesc;
 /* Descriptors helpers */
 struct stmmac_desc_ops {
 	/* DMA RX descriptor ring initialization */
-	void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic, int mode,
-			int end, int bfsize);
+	void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic,
+			     u8 descriptor_mode, int end, int bfsize);
 	/* DMA TX descriptor ring initialization */
-	void (*init_tx_desc)(struct dma_desc *p, int mode, int end);
+	void (*init_tx_desc)(struct dma_desc *p, u8 descriptor_mode, int end);
 	/* Invoked by the xmit function to prepare the tx descriptor */
 	void (*prepare_tx_desc)(struct dma_desc *p, int is_fs, int len,
-			bool csum_flag, int mode, bool tx_own, bool ls,
-			unsigned int tot_pkt_len);
+			bool csum_flag, u8 descriptor_mode, bool tx_own,
+			bool ls, unsigned int tot_pkt_len);
 	void (*prepare_tso_tx_desc)(struct dma_desc *p, int is_fs, int len1,
 			int len2, bool tx_own, bool ls, unsigned int tcphdrlen,
 			unsigned int tcppayloadlen);
 	/* Set/get the owner of the descriptor */
 	void (*set_tx_owner)(struct dma_desc *p);
-	int (*get_tx_owner)(struct dma_desc *p);
 	/* Clean the tx descriptor as soon as the tx irq is received */
-	void (*release_tx_desc)(struct dma_desc *p, int mode);
+	void (*release_tx_desc)(struct dma_desc *p, u8 descriptor_mode);
 	/* Clear interrupt on tx frame completion. When this bit is
 	 * set an interrupt happens as soon as the frame is transmitted */
 	void (*set_tx_ic)(struct dma_desc *p);
-	/* Last tx segment reports the transmit status */
-	int (*get_tx_ls)(struct dma_desc *p);
 	/* Get the tag of the descriptor */
 	u16 (*get_rx_vlan_tci)(struct dma_desc *p);
 	/* Get the valid status of descriptor */
@@ -66,8 +63,6 @@ struct stmmac_desc_ops {
 	/* Return the transmit status looking at the TDES1 */
 	int (*tx_status)(struct stmmac_extra_stats *x,
 			 struct dma_desc *p, void __iomem *ioaddr);
-	/* Get the buffer size from the descriptor */
-	int (*get_tx_len)(struct dma_desc *p);
 	/* Handle extra events on specific interrupts hw dependent */
 	void (*set_rx_owner)(struct dma_desc *p, int disable_rx_ic);
 	/* Get the receive frame size */
@@ -116,22 +111,16 @@ struct stmmac_desc_ops {
 	stmmac_do_void_callback(__priv, desc, prepare_tso_tx_desc, __args)
 #define stmmac_set_tx_owner(__priv, __args...) \
 	stmmac_do_void_callback(__priv, desc, set_tx_owner, __args)
-#define stmmac_get_tx_owner(__priv, __args...) \
-	stmmac_do_callback(__priv, desc, get_tx_owner, __args)
 #define stmmac_release_tx_desc(__priv, __args...) \
 	stmmac_do_void_callback(__priv, desc, release_tx_desc, __args)
 #define stmmac_set_tx_ic(__priv, __args...) \
 	stmmac_do_void_callback(__priv, desc, set_tx_ic, __args)
-#define stmmac_get_tx_ls(__priv, __args...) \
-	stmmac_do_callback(__priv, desc, get_tx_ls, __args)
 #define stmmac_get_rx_vlan_tci(__priv, __args...) \
 	stmmac_do_callback(__priv, desc, get_rx_vlan_tci, __args)
 #define stmmac_get_rx_vlan_valid(__priv, __args...) \
 	stmmac_do_callback(__priv, desc, get_rx_vlan_valid, __args)
 #define stmmac_tx_status(__priv, __args...) \
 	stmmac_do_callback(__priv, desc, tx_status, __args)
-#define stmmac_get_tx_len(__priv, __args...) \
-	stmmac_do_callback(__priv, desc, get_tx_len, __args)
 #define stmmac_set_rx_owner(__priv, __args...) \
 	stmmac_do_void_callback(__priv, desc, set_rx_owner, __args)
 #define stmmac_get_rx_frame_len(__priv, __args...) \
@@ -363,7 +352,7 @@ struct stmmac_ops {
 	void (*set_filter)(struct mac_device_info *hw, struct net_device *dev);
 	/* Flow control setting */
 	void (*flow_ctrl)(struct mac_device_info *hw, unsigned int duplex,
-			  unsigned int fc, unsigned int pause_time, u32 tx_cnt);
+			  unsigned int fc, unsigned int pause_time, u8 tx_cnt);
 	/* Set power management mode (e.g. magic frame) */
 	void (*pmt)(struct mac_device_info *hw, unsigned long mode);
 	/* Set/Get Unicast MAC addresses */
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index 859cb9242a523..c4b613564f87c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -55,11 +55,6 @@ static int ndesc_get_tx_status(struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static int ndesc_get_tx_len(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des1) & RDES1_BUFFER1_SIZE_MASK);
-}
-
 /* This function verifies if each incoming frame has some errors
  * and, if required, updates the multicast statistics.
  * In case of success, it returns good_frame because the GMAC device
@@ -113,8 +108,8 @@ static int ndesc_get_rx_status(struct stmmac_extra_stats *x,
 	return ret;
 }
 
-static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode,
-			       int end, int bfsize)
+static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
+			       u8 descriptor_mode, int end, int bfsize)
 {
 	int bfsize1;
 
@@ -123,7 +118,7 @@ static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode,
 	bfsize1 = min(bfsize, BUF_SIZE_2KiB - 1);
 	p->des1 |= cpu_to_le32(bfsize1 & RDES1_BUFFER1_SIZE_MASK);
 
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		ndesc_rx_set_on_chain(p, end);
 	else
 		ndesc_rx_set_on_ring(p, end, bfsize);
@@ -132,20 +127,15 @@ static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode,
 		p->des1 |= cpu_to_le32(RDES1_DISABLE_IC);
 }
 
-static void ndesc_init_tx_desc(struct dma_desc *p, int mode, int end)
+static void ndesc_init_tx_desc(struct dma_desc *p, u8 descriptor_mode, int end)
 {
 	p->des0 &= cpu_to_le32(~TDES0_OWN);
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		ndesc_tx_set_on_chain(p);
 	else
 		ndesc_end_tx_desc_on_ring(p, end);
 }
 
-static int ndesc_get_tx_owner(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des0) & TDES0_OWN) >> 31;
-}
-
 static void ndesc_set_tx_owner(struct dma_desc *p)
 {
 	p->des0 |= cpu_to_le32(TDES0_OWN);
@@ -156,25 +146,21 @@ static void ndesc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
 	p->des0 |= cpu_to_le32(RDES0_OWN);
 }
 
-static int ndesc_get_tx_ls(struct dma_desc *p)
-{
-	return (le32_to_cpu(p->des1) & TDES1_LAST_SEGMENT) >> 30;
-}
-
-static void ndesc_release_tx_desc(struct dma_desc *p, int mode)
+static void ndesc_release_tx_desc(struct dma_desc *p, u8 descriptor_mode)
 {
 	int ter = (le32_to_cpu(p->des1) & TDES1_END_RING) >> 25;
 
 	memset(p, 0, offsetof(struct dma_desc, des2));
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		ndesc_tx_set_on_chain(p);
 	else
 		ndesc_end_tx_desc_on_ring(p, ter);
 }
 
 static void ndesc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
-				  bool csum_flag, int mode, bool tx_own,
-				  bool ls, unsigned int tot_pkt_len)
+				  bool csum_flag, u8 descriptor_mode,
+				  bool tx_own, bool ls,
+				  unsigned int tot_pkt_len)
 {
 	u32 tdes1 = le32_to_cpu(p->des1);
 
@@ -191,7 +177,7 @@ static void ndesc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
 
 	p->des1 = cpu_to_le32(tdes1);
 
-	if (mode == STMMAC_CHAIN_MODE)
+	if (descriptor_mode == STMMAC_CHAIN_MODE)
 		norm_set_tx_desc_len_on_chain(p, len);
 	else
 		norm_set_tx_desc_len_on_ring(p, len);
@@ -291,14 +277,11 @@ static void ndesc_clear(struct dma_desc *p)
 const struct stmmac_desc_ops ndesc_ops = {
 	.tx_status = ndesc_get_tx_status,
 	.rx_status = ndesc_get_rx_status,
-	.get_tx_len = ndesc_get_tx_len,
 	.init_rx_desc = ndesc_init_rx_desc,
 	.init_tx_desc = ndesc_init_tx_desc,
-	.get_tx_owner = ndesc_get_tx_owner,
 	.release_tx_desc = ndesc_release_tx_desc,
 	.prepare_tx_desc = ndesc_prepare_tx_desc,
 	.set_tx_ic = ndesc_set_tx_ic,
-	.get_tx_ls = ndesc_get_tx_ls,
 	.set_tx_owner = ndesc_set_tx_owner,
 	.set_rx_owner = ndesc_set_rx_owner,
 	.get_rx_frame_len = ndesc_get_rx_frame_len,
diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
index 382d94a3b9720..78fc6aa5bbe95 100644
--- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
@@ -51,7 +51,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
 		stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum,
 				STMMAC_RING_MODE, 0, false, skb->len);
 		tx_q->tx_skbuff[entry] = NULL;
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 
 		if (priv->extend_desc)
 			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index 33667a26708c8..919a93a523903 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -47,13 +47,13 @@ enum stmmac_txbuf_type {
 };
 
 struct stmmac_tx_info {
+	struct xsk_tx_metadata_compl xsk_meta;
 	dma_addr_t buf;
-	bool map_as_page;
 	unsigned len;
+	enum stmmac_txbuf_type buf_type;
+	bool map_as_page;
 	bool last_segment;
 	bool is_jumbo;
-	enum stmmac_txbuf_type buf_type;
-	struct xsk_tx_metadata_compl xsk_meta;
 };
 
 #define STMMAC_TBS_AVAIL	BIT(0)
@@ -79,7 +79,6 @@ struct stmmac_tx_queue {
 	unsigned int cur_tx;
 	unsigned int dirty_tx;
 	dma_addr_t dma_tx_phy;
-	dma_addr_t tx_tail_addr;
 	u32 mss;
 };
 
@@ -131,7 +130,6 @@ struct stmmac_rx_queue {
 	unsigned int buf_alloc_num;
 	unsigned int napi_skb_frag_size;
 	dma_addr_t dma_rx_phy;
-	u32 rx_tail_addr;
 	unsigned int state_saved;
 	struct {
 		struct sk_buff *skb;
@@ -181,7 +179,6 @@ struct stmmac_tc_entry {
 
 #define STMMAC_PPS_MAX		4
 struct stmmac_pps_cfg {
-	bool available;
 	struct timespec64 start;
 	struct timespec64 period;
 };
@@ -244,6 +241,23 @@ struct stmmac_est {
 	u32 max_sdu[MTL_MAX_TX_QUEUES];
 };
 
+struct stmmac_msi {
+	int sfty_ce_irq;
+	int sfty_ue_irq;
+	int rx_irq[MTL_MAX_RX_QUEUES];
+	int tx_irq[MTL_MAX_TX_QUEUES];
+
+	/*irq name */
+	char int_name_mac[IFNAMSIZ + 9];
+	char int_name_wol[IFNAMSIZ + 9];
+	char int_name_lpi[IFNAMSIZ + 9];
+	char int_name_sfty[IFNAMSIZ + 10];
+	char int_name_sfty_ce[IFNAMSIZ + 10];
+	char int_name_sfty_ue[IFNAMSIZ + 10];
+	char int_name_rx_irq[MTL_MAX_RX_QUEUES][IFNAMSIZ + 14];
+	char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18];
+};
+
 struct stmmac_priv {
 	/* Frequently used values are kept adjacent for cache effect */
 	u32 tx_coal_frames[MTL_MAX_TX_QUEUES];
@@ -302,9 +316,17 @@ struct stmmac_priv {
 	bool eee_active;
 	bool eee_sw_timer_en;
 	bool legacy_serdes_is_powered;
-	unsigned int mode;
-	unsigned int chain_mode;
-	int extend_desc;
+	/* descriptor format:
+	 *  when clear: struct dma_desc or for tx TBS struct dma_edesc
+	 *  when set, struct dma_extended_desc
+	 */
+	bool extend_desc;
+	/* chain_mode: requested descriptor mode */
+	bool chain_mode;
+	/* descriptor_mode: actual descriptor mode,
+	 * see STMMAC_CHAIN_MODE or STMMAC_RING_MODE
+	 */
+	u8 descriptor_mode;
 	struct kernel_hwtstamp_config tstamp_config;
 	struct ptp_clock *ptp_clock;
 	struct ptp_clock_info ptp_clock_ops;
@@ -325,19 +347,7 @@ struct stmmac_priv {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
 	unsigned int num_double_vlans;
 	int sfty_irq;
-	int sfty_ce_irq;
-	int sfty_ue_irq;
-	int rx_irq[MTL_MAX_RX_QUEUES];
-	int tx_irq[MTL_MAX_TX_QUEUES];
-	/*irq name */
-	char int_name_mac[IFNAMSIZ + 9];
-	char int_name_wol[IFNAMSIZ + 9];
-	char int_name_lpi[IFNAMSIZ + 9];
-	char int_name_sfty[IFNAMSIZ + 10];
-	char int_name_sfty_ce[IFNAMSIZ + 10];
-	char int_name_sfty_ue[IFNAMSIZ + 10];
-	char int_name_rx_irq[MTL_MAX_RX_QUEUES][IFNAMSIZ + 14];
-	char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18];
+	struct stmmac_msi *msi;
 
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dbgfs_dir;
@@ -403,7 +413,7 @@ void stmmac_dvr_remove(struct device *dev);
 int stmmac_dvr_probe(struct device *device,
 		     struct plat_stmmacenet_data *plat_dat,
 		     struct stmmac_resources *res);
-int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt);
+int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt);
 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size);
 int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
 			   phy_interface_t interface, int speed);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index c1e26965d9b5d..92585d27ab883 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -312,10 +312,9 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev,
 		strscpy(info->driver, MAC100_ETHTOOL_NAME,
 			sizeof(info->driver));
 
-	if (priv->plat->pdev) {
-		strscpy(info->bus_info, pci_name(priv->plat->pdev),
+	if (priv->plat->provide_bus_info)
+		strscpy(info->bus_info, dev_name(priv->device),
 			sizeof(info->bus_info));
-	}
 }
 
 static int stmmac_ethtool_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 6827c99bde8c2..5062537f79e9c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -14,6 +14,7 @@
 	https://bugzilla.stlinux.com/
 *******************************************************************************/
 
+#include <linux/circ_buf.h>
 #include <linux/clk.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
@@ -263,10 +264,10 @@ static void stmmac_verify_args(void)
 
 static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
 {
-	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
-	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
-	u32 queue;
+	u8 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_queues_cnt = priv->plat->tx_queues_to_use;
+	u8 maxq = max(rx_queues_cnt, tx_queues_cnt);
+	u8 queue;
 
 	for (queue = 0; queue < maxq; queue++) {
 		struct stmmac_channel *ch = &priv->channel[queue];
@@ -290,9 +291,9 @@ static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
  */
 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 {
-	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	u8 rx_queues_cnt = priv->plat->rx_queues_to_use;
 	struct stmmac_rx_queue *rx_q;
-	u32 queue;
+	u8 queue;
 
 	/* synchronize_rcu() needed for pending XDP buffers to drain */
 	for (queue = 0; queue < rx_queues_cnt; queue++) {
@@ -312,10 +313,10 @@ static void stmmac_disable_all_queues(struct stmmac_priv *priv)
  */
 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 {
-	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
-	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
-	u32 queue;
+	u8 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_queues_cnt = priv->plat->tx_queues_to_use;
+	u8 maxq = max(rx_queues_cnt, tx_queues_cnt);
+	u8 queue;
 
 	for (queue = 0; queue < maxq; queue++) {
 		struct stmmac_channel *ch = &priv->channel[queue];
@@ -356,14 +357,90 @@ static void print_pkt(unsigned char *buf, int len)
 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 {
 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
-	u32 avail;
 
-	if (tx_q->dirty_tx > tx_q->cur_tx)
-		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
+	return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx,
+			  priv->dma_conf.dma_tx_size);
+}
+
+static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv,
+				      struct stmmac_tx_queue *tx_q)
+{
+	if (priv->extend_desc)
+		return sizeof(struct dma_extended_desc);
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		return sizeof(struct dma_edesc);
+	else
+		return sizeof(struct dma_desc);
+}
+
+static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv,
+					   struct stmmac_tx_queue *tx_q,
+					   unsigned int index)
+{
+	if (priv->extend_desc)
+		return &tx_q->dma_etx[index].basic;
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		return &tx_q->dma_entx[index].basic;
+	else
+		return &tx_q->dma_tx[index];
+}
+
+static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv,
+					 struct stmmac_tx_queue *tx_q,
+					 unsigned int chan, unsigned int index)
+{
+	size_t desc_size;
+	u32 tx_tail_addr;
+
+	desc_size = stmmac_get_tx_desc_size(priv, tx_q);
+
+	tx_tail_addr = tx_q->dma_tx_phy + index * desc_size;
+	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan);
+}
+
+static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv)
+{
+	if (priv->extend_desc)
+		return sizeof(struct dma_extended_desc);
+	else
+		return sizeof(struct dma_desc);
+}
+
+static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv,
+					   struct stmmac_rx_queue *rx_q,
+					   unsigned int index)
+{
+	if (priv->extend_desc)
+		return &rx_q->dma_erx[index].basic;
 	else
-		avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
+		return &rx_q->dma_rx[index];
+}
+
+static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv,
+					 struct stmmac_rx_queue *rx_q,
+					 unsigned int chan, unsigned int index)
+{
+	/* This only needs to deal with normal descriptors as enhanced
+	 * descriptiors are only supported with dwmac1000 (<v4.0) which
+	 * does not implement .set_rx_tail_ptr
+	 */
+	u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc);
 
-	return avail;
+	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan);
+}
+
+static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv,
+					 struct stmmac_rx_queue *rx_q,
+					 unsigned int chan)
+{
+	u32 buf_size;
+
+	if (rx_q->xsk_pool && rx_q->buf_alloc_num)
+		buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
+	else
+		buf_size = priv->dma_conf.dma_buf_sz;
+
+	stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan);
 }
 
 /**
@@ -374,20 +451,15 @@ static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 {
 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
-	u32 dirty;
-
-	if (rx_q->dirty_rx <= rx_q->cur_rx)
-		dirty = rx_q->cur_rx - rx_q->dirty_rx;
-	else
-		dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
 
-	return dirty;
+	return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx,
+			priv->dma_conf.dma_rx_size);
 }
 
 static bool stmmac_eee_tx_busy(struct stmmac_priv *priv)
 {
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	/* check if all TX queues have the work finished */
 	for (queue = 0; queue < tx_cnt; queue++) {
@@ -918,7 +990,7 @@ static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv)
 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex,
 				 unsigned int flow_ctrl)
 {
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
 
 	stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time,
 			 tx_cnt);
@@ -978,7 +1050,8 @@ static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode,
 	struct stmmac_priv *priv = netdev_priv(ndev);
 
 	if (priv->plat->mac_finish)
-		priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface);
+		priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode,
+				       interface);
 
 	return 0;
 }
@@ -1073,7 +1146,8 @@ static void stmmac_mac_link_up(struct phylink_config *config,
 	}
 
 	if (priv->plat->fix_mac_speed)
-		priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode);
+		priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface,
+					  speed, mode);
 
 	if (!duplex)
 		ctrl &= ~priv->hw->link.duplex;
@@ -1260,10 +1334,7 @@ static int stmmac_init_phy(struct net_device *dev)
 	    xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73)
 		return 0;
 
-	fwnode = priv->plat->port_node;
-	if (!fwnode)
-		fwnode = dev_fwnode(priv->device);
-
+	fwnode = dev_fwnode(priv->device);
 	if (fwnode)
 		phy_fwnode = fwnode_get_phy_node(fwnode);
 	else
@@ -1321,7 +1392,6 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv)
 {
 	struct stmmac_mdio_bus_data *mdio_bus_data;
 	struct phylink_config *config;
-	struct fwnode_handle *fwnode;
 	struct phylink_pcs *pcs;
 	struct phylink *phylink;
 
@@ -1376,9 +1446,21 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv)
 				 pcs->supported_interfaces);
 
 	if (priv->dma_cap.eee) {
-		/* Assume all supported interfaces also support LPI */
-		memcpy(config->lpi_interfaces, config->supported_interfaces,
-		       sizeof(config->lpi_interfaces));
+		/* The GMAC 3.74a databook states that EEE is only supported
+		 * in MII, GMII, and RGMII interfaces.
+		 */
+		__set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces);
+		phy_interface_set_rgmii(config->lpi_interfaces);
+
+		/* If we have a non-integrated PCS, assume that it is connected
+		 * to the GMAC using GMII or another EEE compatible interface,
+		 * and thus all PCS-supported interfaces support LPI.
+		 */
+		if (pcs)
+			phy_interface_or(config->lpi_interfaces,
+					 config->lpi_interfaces,
+					 pcs->supported_interfaces);
 
 		/* All full duplex speeds above 100Mbps are supported */
 		config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD;
@@ -1396,11 +1478,8 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv)
 			config->wol_mac_support |= WAKE_MAGIC;
 	}
 
-	fwnode = priv->plat->port_node;
-	if (!fwnode)
-		fwnode = dev_fwnode(priv->device);
-
-	phylink = phylink_create(config, fwnode, priv->plat->phy_interface,
+	phylink = phylink_create(config, dev_fwnode(priv->device),
+				 priv->plat->phy_interface,
 				 &stmmac_phylink_mac_ops);
 	if (IS_ERR(phylink))
 		return PTR_ERR(phylink);
@@ -1412,10 +1491,10 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv)
 static void stmmac_display_rx_rings(struct stmmac_priv *priv,
 				    struct stmmac_dma_conf *dma_conf)
 {
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
 	unsigned int desc_size;
 	void *head_rx;
-	u32 queue;
+	u8 queue;
 
 	/* Display RX rings */
 	for (queue = 0; queue < rx_cnt; queue++) {
@@ -1423,13 +1502,8 @@ static void stmmac_display_rx_rings(struct stmmac_priv *priv,
 
 		pr_info("\tRX Queue %u rings\n", queue);
 
-		if (priv->extend_desc) {
-			head_rx = (void *)rx_q->dma_erx;
-			desc_size = sizeof(struct dma_extended_desc);
-		} else {
-			head_rx = (void *)rx_q->dma_rx;
-			desc_size = sizeof(struct dma_desc);
-		}
+		head_rx = stmmac_get_rx_desc(priv, rx_q, 0);
+		desc_size = stmmac_get_rx_desc_size(priv);
 
 		/* Display RX ring */
 		stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true,
@@ -1440,10 +1514,10 @@ static void stmmac_display_rx_rings(struct stmmac_priv *priv,
 static void stmmac_display_tx_rings(struct stmmac_priv *priv,
 				    struct stmmac_dma_conf *dma_conf)
 {
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
 	unsigned int desc_size;
 	void *head_tx;
-	u32 queue;
+	u8 queue;
 
 	/* Display TX rings */
 	for (queue = 0; queue < tx_cnt; queue++) {
@@ -1451,16 +1525,8 @@ static void stmmac_display_tx_rings(struct stmmac_priv *priv,
 
 		pr_info("\tTX Queue %d rings\n", queue);
 
-		if (priv->extend_desc) {
-			head_tx = (void *)tx_q->dma_etx;
-			desc_size = sizeof(struct dma_extended_desc);
-		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
-			head_tx = (void *)tx_q->dma_entx;
-			desc_size = sizeof(struct dma_edesc);
-		} else {
-			head_tx = (void *)tx_q->dma_tx;
-			desc_size = sizeof(struct dma_desc);
-		}
+		head_tx = stmmac_get_tx_desc(priv, tx_q, 0);
+		desc_size = stmmac_get_tx_desc_size(priv, tx_q);
 
 		stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false,
 				    tx_q->dma_tx_phy, desc_size);
@@ -1516,20 +1582,18 @@ static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,
 					u32 queue)
 {
 	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	struct dma_desc *desc;
 	int i;
 
 	/* Clear the RX descriptors */
-	for (i = 0; i < dma_conf->dma_rx_size; i++)
-		if (priv->extend_desc)
-			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
-					priv->use_riwt, priv->mode,
-					(i == dma_conf->dma_rx_size - 1),
-					dma_conf->dma_buf_sz);
-		else
-			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
-					priv->use_riwt, priv->mode,
-					(i == dma_conf->dma_rx_size - 1),
-					dma_conf->dma_buf_sz);
+	for (i = 0; i < dma_conf->dma_rx_size; i++) {
+		desc = stmmac_get_rx_desc(priv, rx_q, i);
+
+		stmmac_init_rx_desc(priv, desc, priv->use_riwt,
+				    priv->descriptor_mode,
+				    (i == dma_conf->dma_rx_size - 1),
+				    dma_conf->dma_buf_sz);
+	}
 }
 
 /**
@@ -1552,14 +1616,8 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,
 		int last = (i == (dma_conf->dma_tx_size - 1));
 		struct dma_desc *p;
 
-		if (priv->extend_desc)
-			p = &tx_q->dma_etx[i].basic;
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			p = &tx_q->dma_entx[i].basic;
-		else
-			p = &tx_q->dma_tx[i];
-
-		stmmac_init_tx_desc(priv, p, priv->mode, last);
+		p = stmmac_get_tx_desc(priv, tx_q, i);
+		stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last);
 	}
 }
 
@@ -1573,9 +1631,9 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,
 static void stmmac_clear_descriptors(struct stmmac_priv *priv,
 				     struct stmmac_dma_conf *dma_conf)
 {
-	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 rx_queue_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_queue_cnt = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	/* Clear the RX descriptors */
 	for (queue = 0; queue < rx_queue_cnt; queue++)
@@ -1733,10 +1791,7 @@ static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,
 		struct dma_desc *p;
 		int ret;
 
-		if (priv->extend_desc)
-			p = &((rx_q->dma_erx + i)->basic);
-		else
-			p = rx_q->dma_rx + i;
+		p = stmmac_get_rx_desc(priv, rx_q, i);
 
 		ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags,
 					     queue);
@@ -1791,10 +1846,7 @@ static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,
 		dma_addr_t dma_addr;
 		struct dma_desc *p;
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(rx_q->dma_erx + i);
-		else
-			p = rx_q->dma_rx + i;
+		p = stmmac_get_rx_desc(priv, rx_q, i);
 
 		buf = &rx_q->buf_pool[i];
 
@@ -1833,6 +1885,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
 				    u32 queue, gfp_t flags)
 {
 	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	void *des;
 	int ret;
 
 	netif_dbg(priv, probe, priv->dev,
@@ -1851,7 +1904,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
 						   NULL));
 		netdev_info(priv->dev,
 			    "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n",
-			    rx_q->queue_index);
+			    queue);
 		xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq);
 	} else {
 		WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
@@ -1859,7 +1912,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
 						   rx_q->page_pool));
 		netdev_info(priv->dev,
 			    "Register MEM_TYPE_PAGE_POOL RxQ-%d\n",
-			    rx_q->queue_index);
+			    queue);
 	}
 
 	if (rx_q->xsk_pool) {
@@ -1874,15 +1927,14 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
 	}
 
 	/* Setup the chained descriptor addresses */
-	if (priv->mode == STMMAC_CHAIN_MODE) {
+	if (priv->descriptor_mode == STMMAC_CHAIN_MODE) {
 		if (priv->extend_desc)
-			stmmac_mode_init(priv, rx_q->dma_erx,
-					 rx_q->dma_rx_phy,
-					 dma_conf->dma_rx_size, 1);
+			des = rx_q->dma_erx;
 		else
-			stmmac_mode_init(priv, rx_q->dma_rx,
-					 rx_q->dma_rx_phy,
-					 dma_conf->dma_rx_size, 0);
+			des = rx_q->dma_rx;
+
+		stmmac_mode_init(priv, des, rx_q->dma_rx_phy,
+				 dma_conf->dma_rx_size, priv->extend_desc);
 	}
 
 	return 0;
@@ -1893,7 +1945,7 @@ static int init_dma_rx_desc_rings(struct net_device *dev,
 				  gfp_t flags)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 rx_count = priv->plat->rx_queues_to_use;
+	u8 rx_count = priv->plat->rx_queues_to_use;
 	int queue;
 	int ret;
 
@@ -1927,6 +1979,34 @@ static int init_dma_rx_desc_rings(struct net_device *dev,
 	return ret;
 }
 
+static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q,
+				    unsigned int entry,
+				    enum stmmac_txbuf_type type,
+				    dma_addr_t addr, size_t len,
+				    bool map_as_page)
+{
+	tx_q->tx_skbuff_dma[entry].buf = addr;
+	tx_q->tx_skbuff_dma[entry].len = len;
+	tx_q->tx_skbuff_dma[entry].buf_type = type;
+	tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page;
+	tx_q->tx_skbuff_dma[entry].last_segment = false;
+	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+}
+
+static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q,
+					unsigned int entry, dma_addr_t addr,
+					size_t len, bool map_as_page)
+{
+	stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len,
+				map_as_page);
+}
+
+static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q,
+					   unsigned int entry)
+{
+	tx_q->tx_skbuff_dma[entry].last_segment = true;
+}
+
 /**
  * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
  * @priv: driver private structure
@@ -1948,7 +2028,7 @@ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,
 		  (u32)tx_q->dma_tx_phy);
 
 	/* Setup the chained descriptor addresses */
-	if (priv->mode == STMMAC_CHAIN_MODE) {
+	if (priv->descriptor_mode == STMMAC_CHAIN_MODE) {
 		if (priv->extend_desc)
 			stmmac_mode_init(priv, tx_q->dma_etx,
 					 tx_q->dma_tx_phy,
@@ -1964,19 +2044,10 @@ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,
 	for (i = 0; i < dma_conf->dma_tx_size; i++) {
 		struct dma_desc *p;
 
-		if (priv->extend_desc)
-			p = &((tx_q->dma_etx + i)->basic);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			p = &((tx_q->dma_entx + i)->basic);
-		else
-			p = tx_q->dma_tx + i;
-
+		p = stmmac_get_tx_desc(priv, tx_q, i);
 		stmmac_clear_desc(priv, p);
+		stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false);
 
-		tx_q->tx_skbuff_dma[i].buf = 0;
-		tx_q->tx_skbuff_dma[i].map_as_page = false;
-		tx_q->tx_skbuff_dma[i].len = 0;
-		tx_q->tx_skbuff_dma[i].last_segment = false;
 		tx_q->tx_skbuff[i] = NULL;
 	}
 
@@ -1987,8 +2058,8 @@ static int init_dma_tx_desc_rings(struct net_device *dev,
 				  struct stmmac_dma_conf *dma_conf)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 tx_queue_cnt;
-	u32 queue;
+	u8 tx_queue_cnt;
+	u8 queue;
 
 	tx_queue_cnt = priv->plat->tx_queues_to_use;
 
@@ -2059,8 +2130,8 @@ static void dma_free_tx_skbufs(struct stmmac_priv *priv,
  */
 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
 {
-	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 tx_queue_cnt = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	for (queue = 0; queue < tx_queue_cnt; queue++)
 		dma_free_tx_skbufs(priv, &priv->dma_conf, queue);
@@ -2077,6 +2148,8 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
 					 u32 queue)
 {
 	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	size_t size;
+	void *addr;
 
 	/* Release the DMA RX socket buffers */
 	if (rx_q->xsk_pool)
@@ -2088,14 +2161,14 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
 	rx_q->xsk_pool = NULL;
 
 	/* Free DMA regions of consistent memory previously allocated */
-	if (!priv->extend_desc)
-		dma_free_coherent(priv->device, dma_conf->dma_rx_size *
-				  sizeof(struct dma_desc),
-				  rx_q->dma_rx, rx_q->dma_rx_phy);
+	if (priv->extend_desc)
+		addr = rx_q->dma_erx;
 	else
-		dma_free_coherent(priv->device, dma_conf->dma_rx_size *
-				  sizeof(struct dma_extended_desc),
-				  rx_q->dma_erx, rx_q->dma_rx_phy);
+		addr = rx_q->dma_rx;
+
+	size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size;
+
+	dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy);
 
 	if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq))
 		xdp_rxq_info_unreg(&rx_q->xdp_rxq);
@@ -2108,8 +2181,8 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
 static void free_dma_rx_desc_resources(struct stmmac_priv *priv,
 				       struct stmmac_dma_conf *dma_conf)
 {
-	u32 rx_count = priv->plat->rx_queues_to_use;
-	u32 queue;
+	u8 rx_count = priv->plat->rx_queues_to_use;
+	u8 queue;
 
 	/* Free RX queue resources */
 	for (queue = 0; queue < rx_count; queue++)
@@ -2134,17 +2207,14 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
 	dma_free_tx_skbufs(priv, dma_conf, queue);
 
 	if (priv->extend_desc) {
-		size = sizeof(struct dma_extended_desc);
 		addr = tx_q->dma_etx;
 	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
-		size = sizeof(struct dma_edesc);
 		addr = tx_q->dma_entx;
 	} else {
-		size = sizeof(struct dma_desc);
 		addr = tx_q->dma_tx;
 	}
 
-	size *= dma_conf->dma_tx_size;
+	size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size;
 
 	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
 
@@ -2155,8 +2225,8 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
 static void free_dma_tx_desc_resources(struct stmmac_priv *priv,
 				       struct stmmac_dma_conf *dma_conf)
 {
-	u32 tx_count = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 tx_count = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	/* Free TX queue resources */
 	for (queue = 0; queue < tx_count; queue++)
@@ -2183,6 +2253,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 	struct page_pool_params pp_params = { 0 };
 	unsigned int dma_buf_sz_pad, num_pages;
 	unsigned int napi_id;
+	size_t size;
+	void *addr;
 	int ret;
 
 	dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz +
@@ -2218,24 +2290,17 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 	if (!rx_q->buf_pool)
 		return -ENOMEM;
 
-	if (priv->extend_desc) {
-		rx_q->dma_erx = dma_alloc_coherent(priv->device,
-						   dma_conf->dma_rx_size *
-						   sizeof(struct dma_extended_desc),
-						   &rx_q->dma_rx_phy,
-						   GFP_KERNEL);
-		if (!rx_q->dma_erx)
-			return -ENOMEM;
+	size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size;
 
-	} else {
-		rx_q->dma_rx = dma_alloc_coherent(priv->device,
-						  dma_conf->dma_rx_size *
-						  sizeof(struct dma_desc),
-						  &rx_q->dma_rx_phy,
-						  GFP_KERNEL);
-		if (!rx_q->dma_rx)
-			return -ENOMEM;
-	}
+	addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy,
+				  GFP_KERNEL);
+	if (!addr)
+		return -ENOMEM;
+
+	if (priv->extend_desc)
+		rx_q->dma_erx = addr;
+	else
+		rx_q->dma_rx = addr;
 
 	if (stmmac_xdp_is_enabled(priv) &&
 	    test_bit(queue, priv->af_xdp_zc_qps))
@@ -2243,9 +2308,7 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 	else
 		napi_id = ch->rx_napi.napi_id;
 
-	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,
-			       rx_q->queue_index,
-			       napi_id);
+	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id);
 	if (ret) {
 		netdev_err(priv->dev, "Failed to register xdp rxq info\n");
 		return -EINVAL;
@@ -2257,8 +2320,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 				       struct stmmac_dma_conf *dma_conf)
 {
-	u32 rx_count = priv->plat->rx_queues_to_use;
-	u32 queue;
+	u8 rx_count = priv->plat->rx_queues_to_use;
+	u8 queue;
 	int ret;
 
 	/* RX queues buffers and DMA */
@@ -2306,14 +2369,7 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
 	if (!tx_q->tx_skbuff)
 		return -ENOMEM;
 
-	if (priv->extend_desc)
-		size = sizeof(struct dma_extended_desc);
-	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-		size = sizeof(struct dma_edesc);
-	else
-		size = sizeof(struct dma_desc);
-
-	size *= dma_conf->dma_tx_size;
+	size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size;
 
 	addr = dma_alloc_coherent(priv->device, size,
 				  &tx_q->dma_tx_phy, GFP_KERNEL);
@@ -2333,8 +2389,8 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
 				       struct stmmac_dma_conf *dma_conf)
 {
-	u32 tx_count = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 tx_count = priv->plat->tx_queues_to_use;
+	u8 queue;
 	int ret;
 
 	/* TX queues buffers and DMA */
@@ -2398,8 +2454,8 @@ static void free_dma_desc_resources(struct stmmac_priv *priv,
  */
 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
 {
-	u32 rx_queues_count = priv->plat->rx_queues_to_use;
-	int queue;
+	u8 rx_queues_count = priv->plat->rx_queues_to_use;
+	u8 queue;
 	u8 mode;
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
@@ -2462,10 +2518,10 @@ static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
 
 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
-	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
-	u32 chan;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 dma_csr_ch = max(rx_channels_count, tx_channels_count);
+	u8 chan;
 
 	for (chan = 0; chan < dma_csr_ch; chan++) {
 		struct stmmac_channel *ch = &priv->channel[chan];
@@ -2485,9 +2541,9 @@ static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
  */
 static void stmmac_start_all_dma(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
-	u32 chan = 0;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 chan;
 
 	for (chan = 0; chan < rx_channels_count; chan++)
 		stmmac_start_rx_dma(priv, chan);
@@ -2504,9 +2560,9 @@ static void stmmac_start_all_dma(struct stmmac_priv *priv)
  */
 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
-	u32 chan = 0;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 chan;
 
 	for (chan = 0; chan < rx_channels_count; chan++)
 		stmmac_stop_rx_dma(priv, chan);
@@ -2523,14 +2579,14 @@ static void stmmac_stop_all_dma(struct stmmac_priv *priv)
  */
 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
 	int rxfifosz = priv->plat->rx_fifo_size;
 	int txfifosz = priv->plat->tx_fifo_size;
 	u32 txmode = 0;
 	u32 rxmode = 0;
-	u32 chan = 0;
 	u8 qmode = 0;
+	u8 chan;
 
 	if (rxfifosz == 0)
 		rxfifosz = priv->dma_cap.rx_fifo_size;
@@ -2565,23 +2621,13 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
 	/* configure all channels */
 	for (chan = 0; chan < rx_channels_count; chan++) {
 		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];
-		u32 buf_size;
 
 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
 
 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
 				rxfifosz, qmode);
 
-		if (rx_q->xsk_pool) {
-			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
-			stmmac_set_dma_bfsize(priv, priv->ioaddr,
-					      buf_size,
-					      chan);
-		} else {
-			stmmac_set_dma_bfsize(priv, priv->ioaddr,
-					      priv->dma_conf.dma_buf_sz,
-					      chan);
-		}
+		stmmac_set_queue_rx_buf_size(priv, rx_q, chan);
 	}
 
 	for (chan = 0; chan < tx_channels_count; chan++) {
@@ -2686,30 +2732,20 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 			continue;
 		}
 
-		if (likely(priv->extend_desc))
-			tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			tx_desc = &tx_q->dma_entx[entry].basic;
-		else
-			tx_desc = tx_q->dma_tx + entry;
-
+		tx_desc = stmmac_get_tx_desc(priv, tx_q, entry);
 		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
 		meta = xsk_buff_get_metadata(pool, xdp_desc.addr);
 		xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len);
 
-		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
-
 		/* To return XDP buffer to XSK pool, we simple call
 		 * xsk_tx_completed(), so we don't need to fill up
 		 * 'buf' and 'xdpf'.
 		 */
-		tx_q->tx_skbuff_dma[entry].buf = 0;
-		tx_q->xdpf[entry] = NULL;
+		stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX,
+					0, xdp_desc.len, false);
+		stmmac_set_tx_dma_last_segment(tx_q, entry);
 
-		tx_q->tx_skbuff_dma[entry].map_as_page = false;
-		tx_q->tx_skbuff_dma[entry].len = xdp_desc.len;
-		tx_q->tx_skbuff_dma[entry].last_segment = true;
-		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+		tx_q->xdpf[entry] = NULL;
 
 		stmmac_set_desc_addr(priv, tx_desc, dma_addr);
 
@@ -2736,7 +2772,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 		}
 
 		stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,
-				       csum, priv->mode, true, true,
+				       csum, priv->descriptor_mode, true, true,
 				       xdp_desc.len);
 
 		stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
@@ -2744,7 +2780,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 		xsk_tx_metadata_to_compl(meta,
 					 &tx_q->tx_skbuff_dma[entry].xsk_meta);
 
-		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+		tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
 		entry = tx_q->cur_tx;
 	}
 	u64_stats_update_begin(&txq_stats->napi_syncp);
@@ -2823,13 +2859,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			skb = NULL;
 		}
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(tx_q->dma_etx + entry);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			p = &tx_q->dma_entx[entry].basic;
-		else
-			p = tx_q->dma_tx + entry;
-
+		p = stmmac_get_tx_desc(priv, tx_q, entry);
 		status = stmmac_tx_status(priv,	&priv->xstats, p, priv->ioaddr);
 		/* Check if the descriptor is owned by the DMA */
 		if (unlikely(status & tx_dma_own))
@@ -2884,6 +2914,9 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
 		}
 
+		/* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo
+		 * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment
+		 */
 		stmmac_clean_desc3(priv, tx_q, p);
 
 		tx_q->tx_skbuff_dma[entry].last_segment = false;
@@ -2913,9 +2946,9 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
 			}
 		}
 
-		stmmac_release_tx_desc(priv, p, priv->mode);
+		stmmac_release_tx_desc(priv, p, priv->descriptor_mode);
 
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 	}
 	tx_q->dirty_tx = entry;
 
@@ -3014,8 +3047,8 @@ static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
 {
 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
 	int rxfifosz = priv->plat->rx_fifo_size;
 	int txfifosz = priv->plat->tx_fifo_size;
 
@@ -3090,12 +3123,12 @@ static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)
  */
 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 {
-	u32 tx_channel_count = priv->plat->tx_queues_to_use;
-	u32 rx_channel_count = priv->plat->rx_queues_to_use;
-	u32 channels_to_check = tx_channel_count > rx_channel_count ?
-				tx_channel_count : rx_channel_count;
-	u32 chan;
+	u8 tx_channel_count = priv->plat->tx_queues_to_use;
+	u8 rx_channel_count = priv->plat->rx_queues_to_use;
+	u8 channels_to_check = tx_channel_count > rx_channel_count ?
+			       tx_channel_count : rx_channel_count;
 	int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
+	u8 chan;
 
 	/* Make sure we never check beyond our status buffer. */
 	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
@@ -3239,21 +3272,13 @@ static int stmmac_prereset_configure(struct stmmac_priv *priv)
  */
 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
-	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 dma_csr_ch = max(rx_channels_count, tx_channels_count);
 	struct stmmac_rx_queue *rx_q;
 	struct stmmac_tx_queue *tx_q;
-	u32 chan = 0;
 	int ret = 0;
-
-	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
-		netdev_err(priv->dev, "Invalid DMA configuration\n");
-		return -EINVAL;
-	}
-
-	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
-		priv->plat->dma_cfg->atds = 1;
+	u8 chan;
 
 	ret = stmmac_prereset_configure(priv);
 	if (ret)
@@ -3284,11 +3309,8 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv)
 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
 				    rx_q->dma_rx_phy, chan);
 
-		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
-				     (rx_q->buf_alloc_num *
-				      sizeof(struct dma_desc));
-		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
-				       rx_q->rx_tail_addr, chan);
+		stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan,
+					     rx_q->buf_alloc_num);
 	}
 
 	/* DMA TX Channel Configuration */
@@ -3298,9 +3320,7 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv)
 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
 				    tx_q->dma_tx_phy, chan);
 
-		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
-		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
-				       tx_q->tx_tail_addr, chan);
+		stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0);
 	}
 
 	return ret;
@@ -3316,7 +3336,7 @@ static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
 	if (!tx_coal_timer)
 		return;
 
-	ch = &priv->channel[tx_q->queue_index];
+	ch = &priv->channel[queue];
 	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
 
 	/* Arm timer only if napi is not already scheduled.
@@ -3369,9 +3389,9 @@ static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t)
  */
 static void stmmac_init_coalesce(struct stmmac_priv *priv)
 {
-	u32 tx_channel_count = priv->plat->tx_queues_to_use;
-	u32 rx_channel_count = priv->plat->rx_queues_to_use;
-	u32 chan;
+	u8 tx_channel_count = priv->plat->tx_queues_to_use;
+	u8 rx_channel_count = priv->plat->rx_queues_to_use;
+	u8 chan;
 
 	for (chan = 0; chan < tx_channel_count; chan++) {
 		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
@@ -3388,9 +3408,9 @@ static void stmmac_init_coalesce(struct stmmac_priv *priv)
 
 static void stmmac_set_rings_length(struct stmmac_priv *priv)
 {
-	u32 rx_channels_count = priv->plat->rx_queues_to_use;
-	u32 tx_channels_count = priv->plat->tx_queues_to_use;
-	u32 chan;
+	u8 rx_channels_count = priv->plat->rx_queues_to_use;
+	u8 tx_channels_count = priv->plat->tx_queues_to_use;
+	u8 chan;
 
 	/* set TX ring length */
 	for (chan = 0; chan < tx_channels_count; chan++)
@@ -3410,9 +3430,9 @@ static void stmmac_set_rings_length(struct stmmac_priv *priv)
  */
 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
 {
-	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u8 tx_queues_count = priv->plat->tx_queues_to_use;
 	u32 weight;
-	u32 queue;
+	u8 queue;
 
 	for (queue = 0; queue < tx_queues_count; queue++) {
 		weight = priv->plat->tx_queues_cfg[queue].weight;
@@ -3427,9 +3447,9 @@ static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
  */
 static void stmmac_configure_cbs(struct stmmac_priv *priv)
 {
-	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u8 tx_queues_count = priv->plat->tx_queues_to_use;
 	u32 mode_to_use;
-	u32 queue;
+	u8 queue;
 
 	/* queue 0 is reserved for legacy traffic */
 	for (queue = 1; queue < tx_queues_count; queue++) {
@@ -3453,8 +3473,8 @@ static void stmmac_configure_cbs(struct stmmac_priv *priv)
  */
 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
 {
-	u32 rx_queues_count = priv->plat->rx_queues_to_use;
-	u32 queue;
+	u8 rx_queues_count = priv->plat->rx_queues_to_use;
+	u8 queue;
 	u32 chan;
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
@@ -3470,8 +3490,8 @@ static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
  */
 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
 {
-	u32 rx_queues_count = priv->plat->rx_queues_to_use;
-	u32 queue;
+	u8 rx_queues_count = priv->plat->rx_queues_to_use;
+	u8 queue;
 	u32 prio;
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
@@ -3490,8 +3510,8 @@ static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
  */
 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
 {
-	u32 tx_queues_count = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 tx_queues_count = priv->plat->tx_queues_to_use;
+	u8 queue;
 	u32 prio;
 
 	for (queue = 0; queue < tx_queues_count; queue++) {
@@ -3510,9 +3530,9 @@ static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
  */
 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
 {
-	u32 rx_queues_count = priv->plat->rx_queues_to_use;
-	u32 queue;
+	u8 rx_queues_count = priv->plat->rx_queues_to_use;
 	u8 packet;
+	u8 queue;
 
 	for (queue = 0; queue < rx_queues_count; queue++) {
 		/* no specific packet type routing specified for the queue */
@@ -3547,8 +3567,8 @@ static void stmmac_mac_config_rss(struct stmmac_priv *priv)
  */
 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
 {
-	u32 rx_queues_count = priv->plat->rx_queues_to_use;
-	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u8 rx_queues_count = priv->plat->rx_queues_to_use;
+	u8 tx_queues_count = priv->plat->tx_queues_to_use;
 
 	if (tx_queues_count > 1)
 		stmmac_set_tx_queue_weight(priv);
@@ -3616,10 +3636,10 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
 static int stmmac_hw_setup(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
 	bool sph_en;
-	u32 chan;
+	u8 chan;
 	int ret;
 
 	/* Make sure RX clock is enabled */
@@ -3735,6 +3755,7 @@ static void stmmac_free_irq(struct net_device *dev,
 			    enum request_irq_err irq_err, int irq_idx)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
+	struct stmmac_msi *msi = priv->msi;
 	int j;
 
 	switch (irq_err) {
@@ -3742,28 +3763,30 @@ static void stmmac_free_irq(struct net_device *dev,
 		irq_idx = priv->plat->tx_queues_to_use;
 		fallthrough;
 	case REQ_IRQ_ERR_TX:
-		for (j = irq_idx - 1; j >= 0; j--) {
-			if (priv->tx_irq[j] > 0) {
-				irq_set_affinity_hint(priv->tx_irq[j], NULL);
-				free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]);
+		for (j = irq_idx - 1; msi && j >= 0; j--) {
+			if (msi->tx_irq[j] > 0) {
+				irq_set_affinity_hint(msi->tx_irq[j], NULL);
+				free_irq(msi->tx_irq[j],
+					 &priv->dma_conf.tx_queue[j]);
 			}
 		}
 		irq_idx = priv->plat->rx_queues_to_use;
 		fallthrough;
 	case REQ_IRQ_ERR_RX:
-		for (j = irq_idx - 1; j >= 0; j--) {
-			if (priv->rx_irq[j] > 0) {
-				irq_set_affinity_hint(priv->rx_irq[j], NULL);
-				free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]);
+		for (j = irq_idx - 1; msi && j >= 0; j--) {
+			if (msi->rx_irq[j] > 0) {
+				irq_set_affinity_hint(msi->rx_irq[j], NULL);
+				free_irq(msi->rx_irq[j],
+					 &priv->dma_conf.rx_queue[j]);
 			}
 		}
 
-		if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)
-			free_irq(priv->sfty_ue_irq, dev);
+		if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq)
+			free_irq(msi->sfty_ue_irq, dev);
 		fallthrough;
 	case REQ_IRQ_ERR_SFTY_UE:
-		if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)
-			free_irq(priv->sfty_ce_irq, dev);
+		if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq)
+			free_irq(msi->sfty_ce_irq, dev);
 		fallthrough;
 	case REQ_IRQ_ERR_SFTY_CE:
 		if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)
@@ -3783,9 +3806,30 @@ static void stmmac_free_irq(struct net_device *dev,
 	}
 }
 
+static int stmmac_msi_init(struct stmmac_priv *priv,
+			   struct stmmac_resources *res)
+{
+	int i;
+
+	priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL);
+	if (!priv->msi)
+		return -ENOMEM;
+
+	priv->msi->sfty_ce_irq = res->sfty_ce_irq;
+	priv->msi->sfty_ue_irq = res->sfty_ue_irq;
+
+	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
+		priv->msi->rx_irq[i] = res->rx_irq[i];
+	for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
+		priv->msi->tx_irq[i] = res->tx_irq[i];
+
+	return 0;
+}
+
 static int stmmac_request_irq_multi_msi(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
+	struct stmmac_msi *msi = priv->msi;
 	enum request_irq_err irq_err;
 	int irq_idx = 0;
 	char *int_name;
@@ -3793,7 +3837,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	int i;
 
 	/* For common interrupt */
-	int_name = priv->int_name_mac;
+	int_name = msi->int_name_mac;
 	sprintf(int_name, "%s:%s", dev->name, "mac");
 	ret = request_irq(dev->irq, stmmac_mac_interrupt,
 			  0, int_name, dev);
@@ -3809,7 +3853,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	 * is used for WoL
 	 */
 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
-		int_name = priv->int_name_wol;
+		int_name = msi->int_name_wol;
 		sprintf(int_name, "%s:%s", dev->name, "wol");
 		ret = request_irq(priv->wol_irq,
 				  stmmac_mac_interrupt,
@@ -3827,7 +3871,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	 * Error line in case of another line is used
 	 */
 	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {
-		int_name = priv->int_name_sfty;
+		int_name = msi->int_name_sfty;
 		sprintf(int_name, "%s:%s", dev->name, "safety");
 		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,
 				  0, int_name, dev);
@@ -3843,16 +3887,16 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	/* Request the Safety Feature Correctible Error line in
 	 * case of another line is used
 	 */
-	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {
-		int_name = priv->int_name_sfty_ce;
+	if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) {
+		int_name = msi->int_name_sfty_ce;
 		sprintf(int_name, "%s:%s", dev->name, "safety-ce");
-		ret = request_irq(priv->sfty_ce_irq,
+		ret = request_irq(msi->sfty_ce_irq,
 				  stmmac_safety_interrupt,
 				  0, int_name, dev);
 		if (unlikely(ret < 0)) {
 			netdev_err(priv->dev,
 				   "%s: alloc sfty ce MSI %d (error: %d)\n",
-				   __func__, priv->sfty_ce_irq, ret);
+				   __func__, msi->sfty_ce_irq, ret);
 			irq_err = REQ_IRQ_ERR_SFTY_CE;
 			goto irq_error;
 		}
@@ -3861,16 +3905,16 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	/* Request the Safety Feature Uncorrectible Error line in
 	 * case of another line is used
 	 */
-	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {
-		int_name = priv->int_name_sfty_ue;
+	if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) {
+		int_name = msi->int_name_sfty_ue;
 		sprintf(int_name, "%s:%s", dev->name, "safety-ue");
-		ret = request_irq(priv->sfty_ue_irq,
+		ret = request_irq(msi->sfty_ue_irq,
 				  stmmac_safety_interrupt,
 				  0, int_name, dev);
 		if (unlikely(ret < 0)) {
 			netdev_err(priv->dev,
 				   "%s: alloc sfty ue MSI %d (error: %d)\n",
-				   __func__, priv->sfty_ue_irq, ret);
+				   __func__, msi->sfty_ue_irq, ret);
 			irq_err = REQ_IRQ_ERR_SFTY_UE;
 			goto irq_error;
 		}
@@ -3880,23 +3924,23 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {
 		if (i >= MTL_MAX_RX_QUEUES)
 			break;
-		if (priv->rx_irq[i] == 0)
+		if (msi->rx_irq[i] == 0)
 			continue;
 
-		int_name = priv->int_name_rx_irq[i];
+		int_name = msi->int_name_rx_irq[i];
 		sprintf(int_name, "%s:%s-%d", dev->name, "rx", i);
-		ret = request_irq(priv->rx_irq[i],
+		ret = request_irq(msi->rx_irq[i],
 				  stmmac_msi_intr_rx,
 				  0, int_name, &priv->dma_conf.rx_queue[i]);
 		if (unlikely(ret < 0)) {
 			netdev_err(priv->dev,
 				   "%s: alloc rx-%d  MSI %d (error: %d)\n",
-				   __func__, i, priv->rx_irq[i], ret);
+				   __func__, i, msi->rx_irq[i], ret);
 			irq_err = REQ_IRQ_ERR_RX;
 			irq_idx = i;
 			goto irq_error;
 		}
-		irq_set_affinity_hint(priv->rx_irq[i],
+		irq_set_affinity_hint(msi->rx_irq[i],
 				      cpumask_of(i % num_online_cpus()));
 	}
 
@@ -3904,23 +3948,23 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev)
 	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
 		if (i >= MTL_MAX_TX_QUEUES)
 			break;
-		if (priv->tx_irq[i] == 0)
+		if (msi->tx_irq[i] == 0)
 			continue;
 
-		int_name = priv->int_name_tx_irq[i];
+		int_name = msi->int_name_tx_irq[i];
 		sprintf(int_name, "%s:%s-%d", dev->name, "tx", i);
-		ret = request_irq(priv->tx_irq[i],
+		ret = request_irq(msi->tx_irq[i],
 				  stmmac_msi_intr_tx,
 				  0, int_name, &priv->dma_conf.tx_queue[i]);
 		if (unlikely(ret < 0)) {
 			netdev_err(priv->dev,
 				   "%s: alloc tx-%d  MSI %d (error: %d)\n",
-				   __func__, i, priv->tx_irq[i], ret);
+				   __func__, i, msi->tx_irq[i], ret);
 			irq_err = REQ_IRQ_ERR_TX;
 			irq_idx = i;
 			goto irq_error;
 		}
-		irq_set_affinity_hint(priv->tx_irq[i],
+		irq_set_affinity_hint(msi->tx_irq[i],
 				      cpumask_of(i % num_online_cpus()));
 	}
 
@@ -4011,7 +4055,8 @@ static struct stmmac_dma_conf *
 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)
 {
 	struct stmmac_dma_conf *dma_conf;
-	int chan, bfsize, ret;
+	int bfsize, ret;
+	u8 chan;
 
 	dma_conf = kzalloc_obj(*dma_conf);
 	if (!dma_conf) {
@@ -4086,7 +4131,7 @@ static int __stmmac_open(struct net_device *dev,
 			 struct stmmac_dma_conf *dma_conf)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 chan;
+	u8 chan;
 	int ret;
 
 	for (int i = 0; i < MTL_MAX_TX_QUEUES; i++)
@@ -4185,7 +4230,7 @@ static int stmmac_open(struct net_device *dev)
 static void __stmmac_release(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 chan;
+	u8 chan;
 
 	/* Stop and disconnect the PHY */
 	phylink_stop(priv->phylink);
@@ -4258,7 +4303,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
 		return false;
 
 	stmmac_set_tx_owner(priv, p);
-	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+	tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
 	return true;
 }
 
@@ -4286,7 +4331,7 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
 	while (tmp_len > 0) {
 		dma_addr_t curr_addr;
 
-		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
+		tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx,
 						priv->dma_conf.dma_tx_size);
 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
 
@@ -4312,14 +4357,6 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
 {
 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
-	int desc_size;
-
-	if (likely(priv->extend_desc))
-		desc_size = sizeof(struct dma_extended_desc);
-	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-		desc_size = sizeof(struct dma_edesc);
-	else
-		desc_size = sizeof(struct dma_desc);
 
 	/* The own bit must be the latest setting done when prepare the
 	 * descriptor and then barrier is needed to make sure that
@@ -4327,8 +4364,7 @@ static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
 	 */
 	wmb();
 
-	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
-	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
+	stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx);
 }
 
 /**
@@ -4437,7 +4473,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 
 		stmmac_set_mss(priv, mss_desc, mss);
 		tx_q->mss = mss;
-		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
+		tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx,
 						priv->dma_conf.dma_tx_size);
 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
 	}
@@ -4480,10 +4516,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * this DMA buffer right after the DMA engine completely finishes the
 	 * full buffer transmission.
 	 */
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb);
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false;
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
+	stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, skb_headlen(skb),
+				    false);
 
 	/* Prepare fragments */
 	for (i = 0; i < nfrags; i++) {
@@ -4498,20 +4532,18 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
 				     (i == nfrags - 1), queue);
 
-		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
-		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
-		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
-		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
+		stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des,
+					    skb_frag_size(frag), true);
 	}
 
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
+	stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx);
 
 	/* Only the last descriptor gets to point to the skb. */
 	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
-	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
 
 	/* Manage tx mitigation */
-	tx_packets = (tx_q->cur_tx + 1) - first_tx;
+	tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx,
+			      priv->dma_conf.dma_tx_size);
 	tx_q->tx_count_frames += tx_packets;
 
 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
@@ -4541,7 +4573,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * ndo_start_xmit will fill this descriptor the next time it's
 	 * called and stmmac_tx_clean may clean up to this descriptor.
 	 */
-	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+	tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
 
 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
@@ -4570,8 +4602,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
 	/* If we only have one entry used, then the first entry is the last
 	 * segment.
 	 */
-	is_last_segment = ((tx_q->cur_tx - first_entry) &
-			   (priv->dma_conf.dma_tx_size - 1)) == 1;
+	is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry,
+				   priv->dma_conf.dma_tx_size) == 1;
 
 	/* Complete the first descriptor before granting the DMA */
 	stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1,
@@ -4652,12 +4684,12 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	unsigned int first_entry, tx_packets;
 	int gso = skb_shinfo(skb)->gso_type;
 	struct stmmac_txq_stats *txq_stats;
+	struct dma_desc *desc, *first_desc;
 	struct dma_edesc *tbs_desc = NULL;
-	struct dma_desc *desc, *first;
 	struct stmmac_tx_queue *tx_q;
 	int i, csum_insertion = 0;
 	int entry, first_tx;
-	dma_addr_t des;
+	dma_addr_t dma_addr;
 	u32 sdu_len;
 
 	tx_q = &priv->dma_conf.tx_queue[queue];
@@ -4723,17 +4755,11 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 		csum_insertion = !csum_insertion;
 	}
 
-	if (likely(priv->extend_desc))
-		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
-	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-		desc = &tx_q->dma_entx[entry].basic;
-	else
-		desc = tx_q->dma_tx + entry;
-
-	first = desc;
+	desc = stmmac_get_tx_desc(priv, tx_q, entry);
+	first_desc = desc;
 
 	if (has_vlan)
-		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
+		stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT);
 
 	enh_desc = priv->plat->enh_desc;
 	/* To program the descriptors according to the size of the frame */
@@ -4748,48 +4774,40 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	for (i = 0; i < nfrags; i++) {
 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-		int len = skb_frag_size(frag);
+		unsigned int frag_size = skb_frag_size(frag);
 		bool last_segment = (i == (nfrags - 1));
 
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 		WARN_ON(tx_q->tx_skbuff[entry]);
 
-		if (likely(priv->extend_desc))
-			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			desc = &tx_q->dma_entx[entry].basic;
-		else
-			desc = tx_q->dma_tx + entry;
+		desc = stmmac_get_tx_desc(priv, tx_q, entry);
 
-		des = skb_frag_dma_map(priv->device, frag, 0, len,
-				       DMA_TO_DEVICE);
-		if (dma_mapping_error(priv->device, des))
+		dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size,
+					    DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, dma_addr))
 			goto dma_map_err; /* should reuse desc w/o issues */
 
-		tx_q->tx_skbuff_dma[entry].buf = des;
-
-		stmmac_set_desc_addr(priv, desc, des);
-
-		tx_q->tx_skbuff_dma[entry].map_as_page = true;
-		tx_q->tx_skbuff_dma[entry].len = len;
-		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
-		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
+		stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size,
+					    true);
+		stmmac_set_desc_addr(priv, desc, dma_addr);
 
 		/* Prepare the descriptor and set the own bit too */
-		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
-				priv->mode, 1, last_segment, skb->len);
+		stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion,
+				       priv->descriptor_mode, 1, last_segment,
+				       skb->len);
 	}
 
+	stmmac_set_tx_dma_last_segment(tx_q, entry);
+
 	/* Only the last descriptor gets to point to the skb. */
 	tx_q->tx_skbuff[entry] = skb;
-	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
 
 	/* According to the coalesce parameter the IC bit for the latest
 	 * segment is reset and the timer re-started to clean the tx status.
 	 * This approach takes care about the fragments: desc is the first
 	 * element in case of no SG.
 	 */
-	tx_packets = (entry + 1) - first_tx;
+	tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size);
 	tx_q->tx_count_frames += tx_packets;
 
 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
@@ -4805,13 +4823,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 		set_ic = false;
 
 	if (set_ic) {
-		if (likely(priv->extend_desc))
-			desc = &tx_q->dma_etx[entry].basic;
-		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-			desc = &tx_q->dma_entx[entry].basic;
-		else
-			desc = &tx_q->dma_tx[entry];
-
+		desc = stmmac_get_tx_desc(priv, tx_q, entry);
 		tx_q->tx_count_frames = 0;
 		stmmac_set_tx_ic(priv, desc);
 	}
@@ -4821,14 +4833,14 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * ndo_start_xmit will fill this descriptor the next time it's
 	 * called and stmmac_tx_clean may clean up to this descriptor.
 	 */
-	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+	entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 	tx_q->cur_tx = entry;
 
 	if (netif_msg_pktdata(priv)) {
 		netdev_dbg(priv->dev,
 			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
 			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
-			   entry, first, nfrags);
+			   entry, first_desc, nfrags);
 
 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
 		print_pkt(skb->data, skb->len);
@@ -4847,7 +4859,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	u64_stats_update_end(&txq_stats->q_syncp);
 
 	if (priv->sarc_type)
-		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
+		stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type);
 
 	/* Ready to fill the first descriptor and set the OWN bit w/o any
 	 * problems because all the descriptors are actually ready to be
@@ -4856,31 +4868,30 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (likely(!is_jumbo)) {
 		bool last_segment = (nfrags == 0);
 
-		des = dma_map_single(priv->device, skb->data,
-				     nopaged_len, DMA_TO_DEVICE);
-		if (dma_mapping_error(priv->device, des))
+		dma_addr = dma_map_single(priv->device, skb->data,
+					  nopaged_len, DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, dma_addr))
 			goto dma_map_err;
 
-		tx_q->tx_skbuff_dma[first_entry].buf = des;
-		tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
-		tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
+		stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr,
+					    nopaged_len, false);
 
-		stmmac_set_desc_addr(priv, first, des);
+		stmmac_set_desc_addr(priv, first_desc, dma_addr);
 
-		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
-		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
+		if (last_segment)
+			stmmac_set_tx_dma_last_segment(tx_q, first_entry);
 
 		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 			     priv->hwts_tx_en)) {
 			/* declare that device is doing timestamping */
 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
-			stmmac_enable_tx_timestamp(priv, first);
+			stmmac_enable_tx_timestamp(priv, first_desc);
 		}
 
 		/* Prepare the first descriptor setting the OWN bit too */
-		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
-				csum_insertion, priv->mode, 0, last_segment,
-				skb->len);
+		stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len,
+				       csum_insertion, priv->descriptor_mode,
+				       0, last_segment, skb->len);
 	}
 
 	if (tx_q->tbs & STMMAC_TBS_EN) {
@@ -4890,7 +4901,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
 		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
 	}
 
-	stmmac_set_tx_owner(priv, first);
+	stmmac_set_tx_owner(priv, first_desc);
 
 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
 
@@ -4949,10 +4960,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 		struct dma_desc *p;
 		bool use_rx_wd;
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(rx_q->dma_erx + entry);
-		else
-			p = rx_q->dma_rx + entry;
+		p = stmmac_get_rx_desc(priv, rx_q, entry);
 
 		if (!buf->page) {
 			buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
@@ -4990,12 +4998,10 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 		dma_wmb();
 		stmmac_set_rx_owner(priv, p, use_rx_wd);
 
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size);
 	}
 	rx_q->dirty_rx = entry;
-	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
-			    (rx_q->dirty_rx * sizeof(struct dma_desc));
-	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
+	stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx);
 	/* Wake up Rx DMA from the suspend state if required */
 	stmmac_enable_dma_reception(priv, priv->ioaddr, queue);
 }
@@ -5070,6 +5076,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
 	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
 	bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported;
 	unsigned int entry = tx_q->cur_tx;
+	enum stmmac_txbuf_type buf_type;
 	struct dma_desc *tx_desc;
 	dma_addr_t dma_addr;
 	bool set_ic;
@@ -5084,20 +5091,14 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
 		return STMMAC_XDP_CONSUMED;
 	}
 
-	if (likely(priv->extend_desc))
-		tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
-	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
-		tx_desc = &tx_q->dma_entx[entry].basic;
-	else
-		tx_desc = tx_q->dma_tx + entry;
-
+	tx_desc = stmmac_get_tx_desc(priv, tx_q, entry);
 	if (dma_map) {
 		dma_addr = dma_map_single(priv->device, xdpf->data,
 					  xdpf->len, DMA_TO_DEVICE);
 		if (dma_mapping_error(priv->device, dma_addr))
 			return STMMAC_XDP_CONSUMED;
 
-		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO;
+		buf_type = STMMAC_TXBUF_T_XDP_NDO;
 	} else {
 		struct page *page = virt_to_page(xdpf->data);
 
@@ -5106,21 +5107,19 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
 		dma_sync_single_for_device(priv->device, dma_addr,
 					   xdpf->len, DMA_BIDIRECTIONAL);
 
-		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX;
+		buf_type = STMMAC_TXBUF_T_XDP_TX;
 	}
 
-	tx_q->tx_skbuff_dma[entry].buf = dma_addr;
-	tx_q->tx_skbuff_dma[entry].map_as_page = false;
-	tx_q->tx_skbuff_dma[entry].len = xdpf->len;
-	tx_q->tx_skbuff_dma[entry].last_segment = true;
-	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+	stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len,
+				false);
+	stmmac_set_tx_dma_last_segment(tx_q, entry);
 
 	tx_q->xdpf[entry] = xdpf;
 
 	stmmac_set_desc_addr(priv, tx_desc, dma_addr);
 
 	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,
-			       csum, priv->mode, true, true,
+			       csum, priv->descriptor_mode, true, true,
 			       xdpf->len);
 
 	tx_q->tx_count_frames++;
@@ -5140,7 +5139,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
 
 	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
 
-	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+	entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size);
 	tx_q->cur_tx = entry;
 
 	return STMMAC_XDP_TX;
@@ -5351,10 +5350,7 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 			}
 		}
 
-		if (priv->extend_desc)
-			rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry);
-		else
-			rx_desc = rx_q->dma_rx + entry;
+		rx_desc = stmmac_get_rx_desc(priv, rx_q, entry);
 
 		dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
 		stmmac_set_desc_addr(priv, rx_desc, dma_addr);
@@ -5374,14 +5370,12 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 		dma_wmb();
 		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
 
-		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
+		entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size);
 	}
 
 	if (rx_desc) {
 		rx_q->dirty_rx = entry;
-		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
-				     (rx_q->dirty_rx * sizeof(struct dma_desc));
-		stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
+		stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx);
 	}
 
 	return ret;
@@ -5412,16 +5406,10 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
 	int status = 0;
 
 	if (netif_msg_rx_status(priv)) {
-		void *rx_head;
+		void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0);
 
 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
-		if (priv->extend_desc) {
-			rx_head = (void *)rx_q->dma_erx;
-			desc_size = sizeof(struct dma_extended_desc);
-		} else {
-			rx_head = (void *)rx_q->dma_rx;
-			desc_size = sizeof(struct dma_desc);
-		}
+		desc_size = stmmac_get_rx_desc_size(priv);
 
 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
 				    rx_q->dma_rx_phy, desc_size);
@@ -5457,10 +5445,7 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
 			dirty = 0;
 		}
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(rx_q->dma_erx + entry);
-		else
-			p = rx_q->dma_rx + entry;
+		p = stmmac_get_rx_desc(priv, rx_q, entry);
 
 		/* read the status of the incoming frame */
 		status = stmmac_rx_status(priv, &priv->xstats, p);
@@ -5469,14 +5454,11 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
 			break;
 
 		/* Prefetch the next RX descriptor */
-		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
+		rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
 						priv->dma_conf.dma_rx_size);
 		next_entry = rx_q->cur_rx;
 
-		if (priv->extend_desc)
-			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
-		else
-			np = rx_q->dma_rx + next_entry;
+		np = stmmac_get_rx_desc(priv, rx_q, next_entry);
 
 		prefetch(np);
 
@@ -5612,16 +5594,10 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 	limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);
 
 	if (netif_msg_rx_status(priv)) {
-		void *rx_head;
+		void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0);
 
 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
-		if (priv->extend_desc) {
-			rx_head = (void *)rx_q->dma_erx;
-			desc_size = sizeof(struct dma_extended_desc);
-		} else {
-			rx_head = (void *)rx_q->dma_rx;
-			desc_size = sizeof(struct dma_desc);
-		}
+		desc_size = stmmac_get_rx_desc_size(priv);
 
 		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
 				    rx_q->dma_rx_phy, desc_size);
@@ -5654,10 +5630,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 		entry = next_entry;
 		buf = &rx_q->buf_pool[entry];
 
-		if (priv->extend_desc)
-			p = (struct dma_desc *)(rx_q->dma_erx + entry);
-		else
-			p = rx_q->dma_rx + entry;
+		p = stmmac_get_rx_desc(priv, rx_q, entry);
 
 		/* read the status of the incoming frame */
 		status = stmmac_rx_status(priv, &priv->xstats, p);
@@ -5665,14 +5638,11 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 		if (unlikely(status & dma_own))
 			break;
 
-		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
+		rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
 						priv->dma_conf.dma_rx_size);
 		next_entry = rx_q->cur_rx;
 
-		if (priv->extend_desc)
-			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
-		else
-			np = rx_q->dma_rx + next_entry;
+		np = stmmac_get_rx_desc(priv, rx_q, next_entry);
 
 		prefetch(np);
 
@@ -6132,7 +6102,7 @@ static int stmmac_set_features(struct net_device *netdev,
 
 	if (priv->sph_capable) {
 		bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active;
-		u32 chan;
+		u8 chan;
 
 		for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
 			stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
@@ -6152,11 +6122,11 @@ static int stmmac_set_features(struct net_device *netdev,
 
 static void stmmac_common_interrupt(struct stmmac_priv *priv)
 {
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
-	u32 queues_count;
-	u32 queue;
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 queues_count;
 	bool xmac;
+	u8 queue;
 
 	xmac = dwmac_is_xmac(priv->plat->core_type);
 	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
@@ -6454,9 +6424,9 @@ static int stmmac_rings_status_show(struct seq_file *seq, void *v)
 {
 	struct net_device *dev = seq->private;
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 rx_count = priv->plat->rx_queues_to_use;
-	u32 tx_count = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 rx_count = priv->plat->rx_queues_to_use;
+	u8 tx_count = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	if ((dev->flags & IFF_UP) == 0)
 		return 0;
@@ -6581,9 +6551,9 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
 		   priv->dma_cap.number_rx_channel);
 	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
 		   priv->dma_cap.number_tx_channel);
-	seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
+	seq_printf(seq, "\tNumber of Additional RX queues: %u\n",
 		   priv->dma_cap.number_rx_queues);
-	seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
+	seq_printf(seq, "\tNumber of Additional TX queues: %u\n",
 		   priv->dma_cap.number_tx_queues);
 	seq_printf(seq, "\tEnhanced descriptors: %s\n",
 		   (priv->dma_cap.enh_desc) ? "Y" : "N");
@@ -6951,7 +6921,6 @@ void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
 	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
 	struct stmmac_channel *ch = &priv->channel[queue];
 	unsigned long flags;
-	u32 buf_size;
 	int ret;
 
 	ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue);
@@ -6971,23 +6940,11 @@ void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
 	stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue);
 
 	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
-			    rx_q->dma_rx_phy, rx_q->queue_index);
+			    rx_q->dma_rx_phy, queue);
 
-	rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num *
-			     sizeof(struct dma_desc));
-	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
-			       rx_q->rx_tail_addr, rx_q->queue_index);
+	stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num);
 
-	if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
-		buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
-		stmmac_set_dma_bfsize(priv, priv->ioaddr,
-				      buf_size,
-				      rx_q->queue_index);
-	} else {
-		stmmac_set_dma_bfsize(priv, priv->ioaddr,
-				      priv->dma_conf.dma_buf_sz,
-				      rx_q->queue_index);
-	}
+	stmmac_set_queue_rx_buf_size(priv, rx_q, queue);
 
 	stmmac_start_rx_dma(priv, queue);
 
@@ -7033,14 +6990,12 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
 	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue);
 
 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
-			    tx_q->dma_tx_phy, tx_q->queue_index);
+			    tx_q->dma_tx_phy, queue);
 
 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
-		stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);
+		stmmac_enable_tbs(priv, priv->ioaddr, 1, queue);
 
-	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
-	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
-			       tx_q->tx_tail_addr, tx_q->queue_index);
+	stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0);
 
 	stmmac_start_tx_dma(priv, queue);
 
@@ -7052,7 +7007,7 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
 void stmmac_xdp_release(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 chan;
+	u8 chan;
 
 	/* Ensure tx function is not running */
 	netif_tx_disable(dev);
@@ -7085,14 +7040,13 @@ void stmmac_xdp_release(struct net_device *dev)
 int stmmac_xdp_open(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
-	u32 dma_csr_ch = max(rx_cnt, tx_cnt);
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 dma_csr_ch = max(rx_cnt, tx_cnt);
 	struct stmmac_rx_queue *rx_q;
 	struct stmmac_tx_queue *tx_q;
-	u32 buf_size;
 	bool sph_en;
-	u32 chan;
+	u8 chan;
 	int ret;
 
 	ret = alloc_dma_desc_resources(priv, &priv->dma_conf);
@@ -7127,22 +7081,10 @@ int stmmac_xdp_open(struct net_device *dev)
 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
 				    rx_q->dma_rx_phy, chan);
 
-		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
-				     (rx_q->buf_alloc_num *
-				      sizeof(struct dma_desc));
-		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
-				       rx_q->rx_tail_addr, chan);
-
-		if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
-			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
-			stmmac_set_dma_bfsize(priv, priv->ioaddr,
-					      buf_size,
-					      rx_q->queue_index);
-		} else {
-			stmmac_set_dma_bfsize(priv, priv->ioaddr,
-					      priv->dma_conf.dma_buf_sz,
-					      rx_q->queue_index);
-		}
+		stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan,
+					     rx_q->buf_alloc_num);
+
+		stmmac_set_queue_rx_buf_size(priv, rx_q, chan);
 
 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
 	}
@@ -7154,9 +7096,7 @@ int stmmac_xdp_open(struct net_device *dev)
 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
 				    tx_q->dma_tx_phy, chan);
 
-		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
-		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
-				       tx_q->tx_tail_addr, chan);
+		stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0);
 
 		hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	}
@@ -7228,10 +7168,10 @@ int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
 	unsigned int start;
-	int q;
+	u8 q;
 
 	for (q = 0; q < tx_cnt; q++) {
 		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];
@@ -7381,7 +7321,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 	/* dwmac-sun8i only work in chain mode */
 	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)
 		chain_mode = 1;
-	priv->chain_mode = chain_mode;
+	priv->chain_mode = !!chain_mode;
 
 	/* Initialize HW Interface */
 	ret = stmmac_hwif_init(priv);
@@ -7410,7 +7350,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 
 		/* TXCOE doesn't work in thresh DMA mode */
 		if (priv->plat->force_thresh_dma_mode)
-			priv->plat->tx_coe = 0;
+			priv->plat->tx_coe = false;
 		else
 			priv->plat->tx_coe = priv->dma_cap.tx_coe;
 
@@ -7486,6 +7426,14 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 			return ret;
 	}
 
+	/* Set alternate descriptor size (which tells the hardware that
+	 * descriptors are 8 32-bit words) when using extended descriptors
+	 * with ring mode. Only applicable for pre-v4.0 cores. Platform glue
+	 * is not expected to change this.
+	 */
+	priv->plat->dma_cfg->atds = priv->extend_desc &&
+				    priv->descriptor_mode == STMMAC_RING_MODE;
+
 	/* Rx Watchdog is available in the COREs newer than the 3.40.
 	 * In some case, for example on bugged HW this feature
 	 * has to be disable and this can be done by passing the
@@ -7512,7 +7460,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 static void stmmac_napi_add(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 queue, maxq;
+	u8 queue, maxq;
 
 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
 
@@ -7541,7 +7489,7 @@ static void stmmac_napi_add(struct net_device *dev)
 static void stmmac_napi_del(struct net_device *dev)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
-	u32 queue, maxq;
+	u8 queue, maxq;
 
 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
 
@@ -7559,7 +7507,7 @@ static void stmmac_napi_del(struct net_device *dev)
 	}
 }
 
-int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
+int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt)
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 	int ret = 0, i;
@@ -7731,6 +7679,8 @@ struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev)
 	if (!plat_dat)
 		return NULL;
 
+	plat_dat->dma_cfg = &plat_dat->__dma_cfg;
+
 	/* Set the defaults:
 	 * - phy autodetection
 	 * - determine GMII_Address CR field from CSR clock
@@ -7762,8 +7712,13 @@ static int __stmmac_dvr_probe(struct device *device,
 {
 	struct net_device *ndev = NULL;
 	struct stmmac_priv *priv;
-	u32 rxq;
 	int i, ret = 0;
+	u8 rxq;
+
+	if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) {
+		dev_err(device, "invalid DMA configuration\n");
+		return -EINVAL;
+	}
 
 	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
 				       MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
@@ -7799,12 +7754,12 @@ static int __stmmac_dvr_probe(struct device *device,
 	priv->dev->irq = res->irq;
 	priv->wol_irq = res->wol_irq;
 	priv->sfty_irq = res->sfty_irq;
-	priv->sfty_ce_irq = res->sfty_ce_irq;
-	priv->sfty_ue_irq = res->sfty_ue_irq;
-	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
-		priv->rx_irq[i] = res->rx_irq[i];
-	for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
-		priv->tx_irq[i] = res->tx_irq[i];
+
+	if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) {
+		ret = stmmac_msi_init(priv, res);
+		if (ret)
+			return ret;
+	}
 
 	if (!is_zero_ether_addr(res->mac))
 		eth_hw_addr_set(priv->dev, res->mac);
@@ -8141,7 +8096,7 @@ int stmmac_suspend(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct stmmac_priv *priv = netdev_priv(ndev);
-	u32 chan;
+	u8 chan;
 
 	if (!ndev || !netif_running(ndev))
 		goto suspend_bsp;
@@ -8216,9 +8171,9 @@ static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)
  */
 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
 {
-	u32 rx_cnt = priv->plat->rx_queues_to_use;
-	u32 tx_cnt = priv->plat->tx_queues_to_use;
-	u32 queue;
+	u8 rx_cnt = priv->plat->rx_queues_to_use;
+	u8 tx_cnt = priv->plat->tx_queues_to_use;
+	u8 queue;
 
 	for (queue = 0; queue < rx_cnt; queue++)
 		stmmac_reset_rx_queue(priv, queue);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
index a7c2496b39f28..afe98ff5bdcb0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
@@ -234,8 +234,8 @@ static u32 stmmac_mdio_format_addr(struct stmmac_priv *priv,
 {
 	const struct mii_regs *mii_regs = &priv->hw->mii;
 
-	return ((pa << mii_regs->addr_shift) & mii_regs->addr_mask) |
-	       ((gr << mii_regs->reg_shift) & mii_regs->reg_mask) |
+	return field_prep(mii_regs->addr_mask, pa) |
+	       field_prep(mii_regs->reg_mask, gr) |
 	       priv->gmii_address_bus_config |
 	       MII_ADDR_GBUSY;
 }
@@ -430,7 +430,7 @@ int stmmac_pcs_setup(struct net_device *ndev)
 	struct dw_xpcs *xpcs = NULL;
 	int addr, ret;
 
-	devnode = priv->plat->port_node;
+	devnode = dev_fwnode(priv->device);
 
 	if (priv->plat->pcs_init) {
 		ret = priv->plat->pcs_init(priv);
@@ -473,6 +473,53 @@ void stmmac_pcs_clean(struct net_device *ndev)
 	priv->hw->xpcs = NULL;
 }
 
+struct stmmac_clk_rate {
+	unsigned long rate;
+	u8 cr;
+};
+
+/* The standard clk_csr_i to GMII_Address CR field mapping. The rate provided
+ * in this table is the exclusive maximum frequency for the divisor. The
+ * comments for each entry give the divisor and the resulting range of MDC
+ * clock frequencies.
+ */
+static const struct stmmac_clk_rate stmmac_std_csr_to_mdc[] = {
+	{ CSR_F_800M, ~0 },
+	{ CSR_F_500M, STMMAC_CSR_500_800M },
+	{ CSR_F_300M, STMMAC_CSR_300_500M },
+	{ CSR_F_250M, STMMAC_CSR_250_300M },
+	{ CSR_F_150M, STMMAC_CSR_150_250M },
+	{ CSR_F_100M, STMMAC_CSR_100_150M },
+	{ CSR_F_60M,  STMMAC_CSR_60_100M },
+	{ CSR_F_35M,  STMMAC_CSR_35_60M },
+	{ CSR_F_20M,  STMMAC_CSR_20_35M },
+	{ 0, ~0 },
+};
+
+/* The sun8i clk_csr_i to GMII_Address CR field mapping uses rate as the
+ * exclusive minimum frequency for the divisor. Note that the last entry
+ * is valid and also acts as the sentinel.
+ */
+static const struct stmmac_clk_rate stmmac_sun8i_csr_to_mdc[] = {
+	{ 160000000, 3 },
+	{ 80000000, 2 },
+	{ 40000000, 1 },
+	{ 0, 0 },
+};
+
+/* The xgmac clk_csr_i to GMII_Address CR field mapping similarly uses rate
+ * as the exclusive minimum frequency for the divisor, and again the last
+ * entry is valid and also the sentinel.
+ */
+static const struct stmmac_clk_rate stmmac_xgmac_csr_to_mdc[] = {
+	{ 400000000, 5 },
+	{ 350000000, 4 },
+	{ 300000000, 3 },
+	{ 250000000, 2 },
+	{ 150000000, 1 },
+	{ 0, 0 },
+};
+
 /**
  * stmmac_clk_csr_set - dynamically set the MDC clock
  * @priv: driver private structure
@@ -488,8 +535,10 @@ void stmmac_pcs_clean(struct net_device *ndev)
  */
 static u32 stmmac_clk_csr_set(struct stmmac_priv *priv)
 {
+	const struct stmmac_clk_rate *rates;
 	unsigned long clk_rate;
 	u32 value = ~0;
+	int i;
 
 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 
@@ -500,48 +549,17 @@ static u32 stmmac_clk_csr_set(struct stmmac_priv *priv)
 	 * the frequency of clk_csr_i. So we do not change the default
 	 * divider.
 	 */
-	if (clk_rate < CSR_F_35M)
-		value = STMMAC_CSR_20_35M;
-	else if (clk_rate < CSR_F_60M)
-		value = STMMAC_CSR_35_60M;
-	else if (clk_rate < CSR_F_100M)
-		value = STMMAC_CSR_60_100M;
-	else if (clk_rate < CSR_F_150M)
-		value = STMMAC_CSR_100_150M;
-	else if (clk_rate < CSR_F_250M)
-		value = STMMAC_CSR_150_250M;
-	else if (clk_rate <= CSR_F_300M)
-		value = STMMAC_CSR_250_300M;
-	else if (clk_rate < CSR_F_500M)
-		value = STMMAC_CSR_300_500M;
-	else if (clk_rate < CSR_F_800M)
-		value = STMMAC_CSR_500_800M;
-
-	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) {
-		if (clk_rate > 160000000)
-			value = 0x03;
-		else if (clk_rate > 80000000)
-			value = 0x02;
-		else if (clk_rate > 40000000)
-			value = 0x01;
-		else
-			value = 0;
-	}
+	rates = stmmac_std_csr_to_mdc;
+	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)
+		rates = stmmac_sun8i_csr_to_mdc;
+	if (priv->plat->core_type == DWMAC_CORE_XGMAC)
+		rates = stmmac_xgmac_csr_to_mdc;
 
-	if (priv->plat->core_type == DWMAC_CORE_XGMAC) {
-		if (clk_rate > 400000000)
-			value = 0x5;
-		else if (clk_rate > 350000000)
-			value = 0x4;
-		else if (clk_rate > 300000000)
-			value = 0x3;
-		else if (clk_rate > 250000000)
-			value = 0x2;
-		else if (clk_rate > 150000000)
-			value = 0x1;
-		else
-			value = 0x0;
-	}
+	for (i = 0; rates[i].rate; i++)
+		if (clk_rate > rates[i].rate)
+			break;
+	if (rates[i].cr != (u8)~0)
+		value = rates[i].cr;
 
 	return value;
 }
@@ -559,7 +577,7 @@ static void stmmac_mdio_bus_config(struct stmmac_priv *priv)
 	else
 		value = stmmac_clk_csr_set(priv);
 
-	value <<= priv->hw->mii.clk_csr_shift;
+	value <<= __ffs(priv->hw->mii.clk_csr_mask);
 
 	if (value & ~priv->hw->mii.clk_csr_mask)
 		dev_warn(priv->device,
@@ -649,10 +667,7 @@ int stmmac_mdio_register(struct net_device *ndev)
 		stmmac_xgmac2_mdio_read_c45(new_bus, 0, 0, 0);
 
 	/* If fixed-link is set, skip PHY scanning */
-	fwnode = priv->plat->port_node;
-	if (!fwnode)
-		fwnode = dev_fwnode(priv->device);
-
+	fwnode = dev_fwnode(priv->device);
 	if (fwnode) {
 		fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link");
 		if (fixed_node) {
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
index 270ad066ced31..d584fd2daa6f1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c
@@ -25,7 +25,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat)
 	/* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */
 	plat->clk_csr = STMMAC_CSR_20_35M;
 	plat->core_type = DWMAC_CORE_GMAC;
-	plat->force_sf_dma_mode = 1;
+	plat->force_sf_dma_mode = true;
 
 	plat->mdio_bus_data->needs_reset = true;
 }
@@ -58,9 +58,9 @@ static int snps_gmac5_default_data(struct pci_dev *pdev,
 
 	plat->clk_csr = STMMAC_CSR_250_300M;
 	plat->core_type = DWMAC_CORE_GMAC4;
-	plat->force_sf_dma_mode = 1;
+	plat->force_sf_dma_mode = true;
 	plat->flags |= STMMAC_FLAG_TSO_EN;
-	plat->pmt = 1;
+	plat->pmt = true;
 
 	/* Set default number of RX and TX queues to use */
 	plat->tx_queues_to_use = 4;
@@ -134,11 +134,6 @@ static int stmmac_pci_probe(struct pci_dev *pdev,
 	if (!plat->mdio_bus_data)
 		return -ENOMEM;
 
-	plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg),
-				     GFP_KERNEL);
-	if (!plat->dma_cfg)
-		return -ENOMEM;
-
 	plat->safety_feat_cfg = devm_kzalloc(&pdev->dev,
 					     sizeof(*plat->safety_feat_cfg),
 					     GFP_KERNEL);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 5c9fd91a1db9d..545b8a3425eb4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -138,6 +138,7 @@ static int stmmac_mtl_setup(struct platform_device *pdev,
 	struct device_node *tx_node;
 	u8 queue = 0;
 	int ret = 0;
+	u32 value;
 
 	/* First Queue must always be in DCB mode. As MTL_QUEUE_DCB = 1 we need
 	 * to always set this, otherwise Queue will be classified as AVB
@@ -157,8 +158,11 @@ static int stmmac_mtl_setup(struct platform_device *pdev,
 	}
 
 	/* Processing RX queues common config */
-	of_property_read_u32(rx_node, "snps,rx-queues-to-use",
-			     &plat->rx_queues_to_use);
+	if (!of_property_read_u32(rx_node, "snps,rx-queues-to-use", &value)) {
+		if (value > U8_MAX)
+			value = U8_MAX;
+		plat->rx_queues_to_use = value;
+	}
 
 	if (of_property_read_bool(rx_node, "snps,rx-sched-sp"))
 		plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP;
@@ -208,8 +212,11 @@ static int stmmac_mtl_setup(struct platform_device *pdev,
 	}
 
 	/* Processing TX queues common config */
-	of_property_read_u32(tx_node, "snps,tx-queues-to-use",
-			     &plat->tx_queues_to_use);
+	if (!of_property_read_u32(tx_node, "snps,tx-queues-to-use", &value)) {
+		if (value > U8_MAX)
+			value = U8_MAX;
+		plat->tx_queues_to_use = value;
+	}
 
 	if (of_property_read_bool(tx_node, "snps,tx-sched-wrr"))
 		plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR;
@@ -396,6 +403,7 @@ static const char * const stmmac_gmac4_compats[] = {
 	"snps,dwmac-5.10a",
 	"snps,dwmac-5.20",
 	"snps,dwmac-5.30a",
+	"snps,dwmac-5.40a",
 	NULL
 };
 
@@ -446,9 +454,6 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
 	 * they are not converted to phylink. */
 	plat->phy_node = of_parse_phandle(np, "phy-handle", 0);
 
-	/* PHYLINK automatically parses the phy-handle property */
-	plat->port_node = of_fwnode_handle(np);
-
 	/* Get max speed of operation from device tree */
 	of_property_read_u32(np, "max-speed", &plat->max_speed);
 
@@ -517,47 +522,41 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
 		plat->multicast_filter_bins = dwmac1000_validate_mcast_bins(
 				&pdev->dev, plat->multicast_filter_bins);
 		plat->core_type = DWMAC_CORE_GMAC;
-		plat->pmt = 1;
+		plat->pmt = true;
 	}
 
 	if (of_device_is_compatible(np, "snps,dwmac-3.40a")) {
 		plat->core_type = DWMAC_CORE_GMAC;
-		plat->enh_desc = 1;
-		plat->tx_coe = 1;
-		plat->bugged_jumbo = 1;
-		plat->pmt = 1;
+		plat->enh_desc = true;
+		plat->tx_coe = true;
+		plat->bugged_jumbo = true;
+		plat->pmt = true;
 	}
 
 	if (of_device_compatible_match(np, stmmac_gmac4_compats)) {
 		plat->core_type = DWMAC_CORE_GMAC4;
-		plat->pmt = 1;
+		plat->pmt = true;
 		if (of_property_read_bool(np, "snps,tso"))
 			plat->flags |= STMMAC_FLAG_TSO_EN;
 	}
 
 	if (of_device_is_compatible(np, "snps,dwmac-3.610") ||
 		of_device_is_compatible(np, "snps,dwmac-3.710")) {
-		plat->enh_desc = 1;
-		plat->bugged_jumbo = 1;
-		plat->force_sf_dma_mode = 1;
+		plat->enh_desc = true;
+		plat->bugged_jumbo = true;
+		plat->force_sf_dma_mode = true;
 	}
 
 	if (of_device_is_compatible(np, "snps,dwxgmac")) {
 		plat->core_type = DWMAC_CORE_XGMAC;
-		plat->pmt = 1;
+		plat->pmt = true;
 		if (of_property_read_bool(np, "snps,tso"))
 			plat->flags |= STMMAC_FLAG_TSO_EN;
 		of_property_read_u32(np, "snps,multicast-filter-bins",
 				     &plat->multicast_filter_bins);
 	}
 
-	dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*dma_cfg),
-			       GFP_KERNEL);
-	if (!dma_cfg) {
-		ret = ERR_PTR(-ENOMEM);
-		goto error_put_mdio;
-	}
-	plat->dma_cfg = dma_cfg;
+	dma_cfg = plat->dma_cfg;
 
 	of_property_read_u32(np, "snps,pbl", &dma_cfg->pbl);
 	if (!dma_cfg->pbl)
@@ -697,9 +696,47 @@ struct clk *stmmac_pltfr_find_clk(struct plat_stmmacenet_data *plat_dat,
 }
 EXPORT_SYMBOL_GPL(stmmac_pltfr_find_clk);
 
+/**
+ * stmmac_pltfr_get_irq_array - Read per-channel IRQs from platform device
+ * @pdev: platform device
+ * @fmt: IRQ name format string (e.g., "tx-queue-%d")
+ * @irqs: array to store IRQ numbers
+ * @num: maximum number of IRQs to read
+ *
+ * Return: 0 on success, -EPROBE_DEFER if IRQ is deferred, -EINVAL on error.
+ * Missing IRQs are set to 0 and iteration stops at first missing IRQ.
+ */
+static int stmmac_pltfr_get_irq_array(struct platform_device *pdev,
+				      const char *fmt, int *irqs, size_t num)
+{
+	char name[16];
+	int i;
+
+	for (i = 0; i < num; i++) {
+		if (snprintf(name, sizeof(name), fmt, i) >= sizeof(name))
+			return -EINVAL;
+
+		irqs[i] = platform_get_irq_byname_optional(pdev, name);
+		if (irqs[i] == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		if (irqs[i] <= 0) {
+			dev_dbg(&pdev->dev, "IRQ %s not found\n", name);
+
+			/* Stop silently on first unset irq */
+			irqs[i] = 0;
+			break;
+		}
+	}
+
+	return 0;
+}
+
 int stmmac_get_platform_resources(struct platform_device *pdev,
 				  struct stmmac_resources *stmmac_res)
 {
+	int ret;
+
 	memset(stmmac_res, 0, sizeof(*stmmac_res));
 
 	/* Get IRQ information early to have an ability to ask for deferred
@@ -735,7 +772,24 @@ int stmmac_get_platform_resources(struct platform_device *pdev,
 
 	stmmac_res->addr = devm_platform_ioremap_resource(pdev, 0);
 
-	return PTR_ERR_OR_ZERO(stmmac_res->addr);
+	if (IS_ERR(stmmac_res->addr))
+		return PTR_ERR(stmmac_res->addr);
+
+	/* TX channels irq */
+	ret = stmmac_pltfr_get_irq_array(pdev, "tx-queue-%d",
+					 stmmac_res->tx_irq,
+					 MTL_MAX_TX_QUEUES);
+	if (ret)
+		return ret;
+
+	/* RX channels irq */
+	ret = stmmac_pltfr_get_irq_array(pdev, "rx-queue-%d",
+					 stmmac_res->rx_irq,
+					 MTL_MAX_RX_QUEUES);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(stmmac_get_platform_resources);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
index 3e30172fa1294..960249960004c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
@@ -334,12 +334,14 @@ const struct ptp_clock_info dwmac1000_ptp_clock_ops = {
  */
 void stmmac_ptp_register(struct stmmac_priv *priv)
 {
-	int i;
-
-	for (i = 0; i < priv->dma_cap.pps_out_num; i++) {
-		if (i >= STMMAC_PPS_MAX)
-			break;
-		priv->pps[i].available = true;
+	unsigned int pps_out_num = priv->dma_cap.pps_out_num;
+	unsigned int n_ext_ts;
+
+	if (pps_out_num > STMMAC_PPS_MAX) {
+		dev_warn(priv->device,
+			 "pps outputs (%u) exceeds driver maximum, limiting to %u\n",
+			 pps_out_num, STMMAC_PPS_MAX);
+		pps_out_num = STMMAC_PPS_MAX;
 	}
 
 	/* Calculate the clock domain crossing (CDC) error if necessary */
@@ -350,11 +352,12 @@ void stmmac_ptp_register(struct stmmac_priv *priv)
 	/* Update the ptp clock parameters based on feature discovery, when
 	 * available
 	 */
-	if (priv->dma_cap.pps_out_num)
-		priv->ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num;
+	if (pps_out_num)
+		priv->ptp_clock_ops.n_per_out = pps_out_num;
 
-	if (priv->dma_cap.aux_snapshot_n)
-		priv->ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n;
+	n_ext_ts = priv->dma_cap.aux_snapshot_n;
+	if (n_ext_ts)
+		priv->ptp_clock_ops.n_ext_ts = n_ext_ts;
 
 	if (priv->plat->ptp_max_adj)
 		priv->ptp_clock_ops.max_adj = priv->plat->ptp_max_adj;
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index 265ce5479915f..d9400599e80a4 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -1352,7 +1352,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow,
 	am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
 	skb_put(skb, pkt_len);
 	if (port->rx_ts_filter)
-		am65_cpts_rx_timestamp(common->cpts, skb);
+		am65_cpts_rx_timestamp(common->cpts, port_id, skb);
 	skb_mark_for_recycle(skb);
 	skb->protocol = eth_type_trans(skb, ndev);
 	am65_cpsw_nuss_rx_csum(skb, csum_info);
@@ -1607,7 +1607,7 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
 
 	/* SKB TX timestamp */
 	if (port->tx_ts_enabled)
-		am65_cpts_prep_tx_timestamp(common->cpts, skb);
+		am65_cpts_prep_tx_timestamp(common->cpts, port->port_id, skb);
 
 	q_idx = skb_get_queue_mapping(skb);
 	dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c
index 8ffbfaa3ab18c..efbf2ef976a32 100644
--- a/drivers/net/ethernet/ti/am65-cpts.c
+++ b/drivers/net/ethernet/ti/am65-cpts.c
@@ -796,11 +796,7 @@ static bool am65_cpts_match_tx_ts(struct am65_cpts *cpts,
 	bool found = false;
 	u32 mtype_seqid;
 
-	mtype_seqid = event->event1 &
-		      (AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK |
-		       AM65_CPTS_EVENT_1_EVENT_TYPE_MASK |
-		       AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK);
-
+	mtype_seqid = event->event1;
 	__skb_queue_head_init(&txq_list);
 
 	spin_lock_irqsave(&cpts->txq.lock, flags);
@@ -922,7 +918,6 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid)
 	struct list_head *this, *next;
 	struct am65_cpts_event *event;
 	unsigned long flags;
-	u32 mtype_seqid;
 	u64 ns = 0;
 
 	spin_lock_irqsave(&cpts->lock, flags);
@@ -934,12 +929,7 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid)
 			continue;
 		}
 
-		mtype_seqid = event->event1 &
-			      (AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK |
-			       AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK |
-			       AM65_CPTS_EVENT_1_EVENT_TYPE_MASK);
-
-		if (mtype_seqid == skb_mtype_seqid) {
+		if (event->event1 == skb_mtype_seqid) {
 			ns = event->timestamp;
 			list_move(&event->list, &cpts->pool);
 			break;
@@ -950,7 +940,8 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid)
 	return ns;
 }
 
-void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
+void am65_cpts_rx_timestamp(struct am65_cpts *cpts, unsigned int port_id,
+			    struct sk_buff *skb)
 {
 	struct am65_cpts_skb_cb_data *skb_cb = (struct am65_cpts_skb_cb_data *)skb->cb;
 	struct skb_shared_hwtstamps *ssh;
@@ -966,6 +957,7 @@ void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
 	if (!ret)
 		return; /* if not PTP class packet */
 
+	skb_cb->skb_mtype_seqid |= port_id << AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT;
 	skb_cb->skb_mtype_seqid |= (AM65_CPTS_EV_RX << AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT);
 
 	dev_dbg(cpts->dev, "%s mtype seqid %08x\n", __func__, skb_cb->skb_mtype_seqid);
@@ -1009,13 +1001,15 @@ EXPORT_SYMBOL_GPL(am65_cpts_tx_timestamp);
 /**
  * am65_cpts_prep_tx_timestamp - check and prepare tx packet for timestamping
  * @cpts: cpts handle
+ * @port_id: The port on which the skb will be sent
  * @skb: packet
  *
  * This functions should be called from .xmit().
  * It checks if packet can be timestamped, fills internal cpts data
  * in skb-cb and marks packet as SKBTX_IN_PROGRESS.
  */
-void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
+void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, unsigned int port_id,
+				 struct sk_buff *skb)
 {
 	struct am65_cpts_skb_cb_data *skb_cb = (void *)skb->cb;
 	int ret;
@@ -1026,6 +1020,7 @@ void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb)
 	ret = am65_skb_get_mtype_seqid(skb, &skb_cb->skb_mtype_seqid);
 	if (!ret)
 		return;
+	skb_cb->skb_mtype_seqid |= port_id << AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT;
 	skb_cb->skb_mtype_seqid |= (AM65_CPTS_EV_TX <<
 				   AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT);
 
diff --git a/drivers/net/ethernet/ti/am65-cpts.h b/drivers/net/ethernet/ti/am65-cpts.h
index 6099d772799da..5fa77b0fc8372 100644
--- a/drivers/net/ethernet/ti/am65-cpts.h
+++ b/drivers/net/ethernet/ti/am65-cpts.h
@@ -22,9 +22,11 @@ void am65_cpts_release(struct am65_cpts *cpts);
 struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs,
 				   struct device_node *node);
 int am65_cpts_phc_index(struct am65_cpts *cpts);
-void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb);
+void am65_cpts_rx_timestamp(struct am65_cpts *cpts, unsigned int port_id,
+			    struct sk_buff *skb);
 void am65_cpts_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb);
-void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb);
+void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, unsigned int port_id,
+				 struct sk_buff *skb);
 u64 am65_cpts_ns_gettime(struct am65_cpts *cpts);
 int am65_cpts_estf_enable(struct am65_cpts *cpts, int idx,
 			  struct am65_cpts_estf_cfg *cfg);
@@ -49,6 +51,7 @@ static inline int am65_cpts_phc_index(struct am65_cpts *cpts)
 }
 
 static inline void am65_cpts_rx_timestamp(struct am65_cpts *cpts,
+					  unsigned int port_id,
 					  struct sk_buff *skb)
 {
 }
@@ -59,6 +62,7 @@ static inline void am65_cpts_tx_timestamp(struct am65_cpts *cpts,
 }
 
 static inline void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts,
+					       unsigned int port_id,
 					       struct sk_buff *skb)
 {
 }
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index ed8116fb05e9b..3802122d0f1b2 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -1389,15 +1389,6 @@ static int emac_devioctl(struct net_device *ndev, struct ifreq *ifrq, int cmd)
 		return -EOPNOTSUPP;
 }
 
-static int match_first_device(struct device *dev, const void *data)
-{
-	if (dev->parent && dev->parent->of_node)
-		return of_device_is_compatible(dev->parent->of_node,
-					       "ti,davinci_mdio");
-
-	return !strncmp(dev_name(dev), "davinci_mdio", 12);
-}
-
 /**
  * emac_dev_open - EMAC device open
  * @ndev: The DaVinci EMAC network adapter
@@ -1417,7 +1408,6 @@ static int emac_dev_open(struct net_device *ndev)
 	int i = 0;
 	struct emac_priv *priv = netdev_priv(ndev);
 	struct phy_device *phydev = NULL;
-	struct device *phy = NULL;
 
 	ret = pm_runtime_resume_and_get(&priv->pdev->dev);
 	if (ret < 0) {
@@ -1502,20 +1492,22 @@ static int emac_dev_open(struct net_device *ndev)
 		}
 	}
 
-	/* use the first phy on the bus if pdata did not give us a phy id */
+	/* if no phy-handle and no fixed link, use the first phy on the bus */
 	if (!phydev && !priv->phy_id) {
-		/* NOTE: we can't use bus_find_device_by_name() here because
-		 * the device name is not guaranteed to be 'davinci_mdio'. On
-		 * some systems it can be 'davinci_mdio.0' so we need to use
-		 * strncmp() against the first part of the string to correctly
-		 * match it.
-		 */
-		phy = bus_find_device(&mdio_bus_type, NULL, NULL,
-				      match_first_device);
-		if (phy) {
-			priv->phy_id = dev_name(phy);
-			if (!priv->phy_id || !*priv->phy_id)
-				put_device(phy);
+		struct device_node *np;
+
+		np = of_find_compatible_node(NULL, NULL, "ti,davinci_mdio");
+		if (np) {
+			struct mii_bus *bus = of_mdio_find_bus(np);
+
+			if (bus) {
+				struct phy_device *phy = phy_find_first(bus);
+
+				if (phy)
+					priv->phy_id = phydev_name(phy);
+				put_device(&bus->dev); /* of_mdio_find_bus */
+			}
+			of_node_put(np); /* of_find_compatible_node */
 		}
 	}
 
@@ -1523,7 +1515,6 @@ static int emac_dev_open(struct net_device *ndev)
 		phydev = phy_connect(ndev, priv->phy_id,
 				     &emac_adjust_link,
 				     PHY_INTERFACE_MODE_MII);
-		put_device(phy);	/* reference taken by bus_find_device */
 		if (IS_ERR(phydev)) {
 			dev_err(emac_dev, "could not connect to phy %s\n",
 				priv->phy_id);
diff --git a/drivers/net/ethernet/ti/icssg/icssg_classifier.c b/drivers/net/ethernet/ti/icssg/icssg_classifier.c
index 833ca86d0b717..87a8577093dbb 100644
--- a/drivers/net/ethernet/ti/icssg/icssg_classifier.c
+++ b/drivers/net/ethernet/ti/icssg/icssg_classifier.c
@@ -27,7 +27,7 @@
 #define FT1_DA0_MASK	0x8
 #define FT1_DA1_MASK	0xc
 
-#define FT1_N_REG(slize, n, reg)	\
+#define FT1_N_REG(slice, n, reg)	\
 	(offs[slice].ft1_slot_base + FT1_SLOT_SIZE * (n) + (reg))
 
 #define FT1_LEN_MASK		GENMASK(19, 16)
@@ -62,7 +62,7 @@ enum ft1_cfg_type {
 #define FT3_T			0x18
 #define FT3_T_MASK		0x1c
 
-#define FT3_N_REG(slize, n, reg)	\
+#define FT3_N_REG(slice, n, reg)	\
 	(offs[slice].ft3_slot_base + FT3_SLOT_SIZE * (n) + (reg))
 
 /* offsets from rx_class n's base */
@@ -74,6 +74,9 @@ enum ft1_cfg_type {
 #define RX_CLASS_N_REG(slice, n, reg)	\
 	(offs[slice].rx_class_base + RX_CLASS_EN_SIZE * (n) + (reg))
 
+#define RX_CLASS_OR_REG(slice, n, reg)	\
+	(offs[slice].rx_class_or_base + RX_CLASS_EN_SIZE * (n) + (reg))
+
 /* RX Class Gates */
 #define RX_CLASS_GATES_SIZE	0x4	/* bytes */
 
@@ -102,6 +105,22 @@ enum ft1_cfg_type {
 	((BIT(slot) << RX_CLASS_FT_FT1_MATCH_SHIFT) & \
 	RX_CLASS_FT_FT1_MATCH_MASK)
 
+/* HSR/PRP protocol frame filtering */
+#define HSR_PTP_ETHERTYPE_OFFSET	0x12
+#define PRP_PTP_ETHERTYPE_OFFSET	0x0C
+#define FT3_PTP_PATTERN			0xF788
+#define FT3_ETHERTYPE_MASK		0xFFFF0000U
+#define FT3_VLAN_MODE_BOTH		0x1
+#define RX_CLASS_OR_DUP_PTP		0x4200
+#define RX_CLASS_OR_HSR_TAG		0x4000
+#define RX_CLASS_GATE_PTP		0x50
+#define RX_CLASS_DISABLE		0x0
+
+/* HSR/PRP classifier indices */
+#define CLASSIFIER_PTP_DUP		10
+#define CLASSIFIER_HSR_TAG		11
+#define FT3_PTP_SLOT			14
+
 /* RX class type */
 enum rx_class_sel_type {
 	RX_CLASS_SEL_TYPE_OR = 0,
@@ -133,6 +152,7 @@ struct miig_rt_offsets {
 	u32 ft3_p_base;
 	u32 ft_rx_ptr;
 	u32 rx_class_base;
+	u32 rx_class_or_base;
 	u32 rx_class_cfg1;
 	u32 rx_class_cfg2;
 	u32 rx_class_gates_base;
@@ -161,6 +181,7 @@ static const struct miig_rt_offsets offs[] = {
 		0x308,
 		0x408,
 		0x40c,
+		0x410,
 		0x48c,
 		0x490,
 		0x494,
@@ -186,6 +207,7 @@ static const struct miig_rt_offsets offs[] = {
 		0x8d4,
 		0x9d4,
 		0x9d8,
+		0x9dc,
 		0xa58,
 		0xa5c,
 		0xa60,
@@ -467,3 +489,46 @@ void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr)
 	rx_class_ft1_cfg_set_type(miig_rt, slice, 0, FT1_CFG_TYPE_EQ);
 }
 EXPORT_SYMBOL_GPL(icssg_ft1_set_mac_addr);
+
+/**
+ * icssg_ft3_hsr_configurations - Configure filter table for HSR/PRP protocol frames
+ * @miig_rt: Pointer to the MII-G register map
+ * @slice: ICSSG slice number (0 or 1)
+ * @prueth: Pointer to prueth structure to determine HSR/PRP mode
+ *
+ * Configures FT3 to detect PTP frames (EtherType 0x88F7) in HSR/PRP tagged packets.
+ * HSR frames have a 6-byte tag, while PRP has no tag offset for EtherType detection.
+ */
+void icssg_ft3_hsr_configurations(struct regmap *miig_rt, int slice,
+				  struct prueth *prueth)
+{
+	u8 offset = (prueth->hsr_prp_version == PRP_V1) ?
+		    PRP_PTP_ETHERTYPE_OFFSET : HSR_PTP_ETHERTYPE_OFFSET;
+
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START), offset);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START_AUTO), 0);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START_OFFSET), 0);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_JUMP_OFFSET), 0);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_LEN), 0);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_CFG), FT3_VLAN_MODE_BOTH);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_T), FT3_PTP_PATTERN);
+	regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_T_MASK),
+		     FT3_ETHERTYPE_MASK);
+
+	regmap_write(miig_rt, RX_CLASS_N_REG(slice, CLASSIFIER_PTP_DUP, RX_CLASS_AND_EN),
+		     RX_CLASS_DISABLE);
+	regmap_write(miig_rt, RX_CLASS_OR_REG(slice, CLASSIFIER_PTP_DUP, RX_CLASS_AND_EN),
+		     RX_CLASS_OR_DUP_PTP);
+	regmap_write(miig_rt, RX_CLASS_GATES_N_REG(slice, CLASSIFIER_PTP_DUP),
+		     RX_CLASS_GATE_PTP);
+
+	if (prueth->hsr_prp_version != PRP_V1) {
+		regmap_write(miig_rt, RX_CLASS_N_REG(slice, CLASSIFIER_HSR_TAG, RX_CLASS_AND_EN),
+			     RX_CLASS_DISABLE);
+		regmap_write(miig_rt, RX_CLASS_OR_REG(slice, CLASSIFIER_HSR_TAG, RX_CLASS_AND_EN),
+			     RX_CLASS_OR_HSR_TAG);
+		regmap_write(miig_rt, RX_CLASS_GATES_N_REG(slice, CLASSIFIER_HSR_TAG),
+			     RX_CLASS_GATE_PTP);
+	}
+}
+EXPORT_SYMBOL_GPL(icssg_ft3_hsr_configurations);
diff --git a/drivers/net/ethernet/ti/icssg/icssg_common.c b/drivers/net/ethernet/ti/icssg/icssg_common.c
index 0cf9dfe0fa362..0a3cf2f848a57 100644
--- a/drivers/net/ethernet/ti/icssg/icssg_common.c
+++ b/drivers/net/ethernet/ti/icssg/icssg_common.c
@@ -1075,6 +1075,11 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state)
 		xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false);
 
 		*xdp_state = emac_run_xdp(emac, &xdp, &pkt_len);
+		if (*xdp_state == ICSSG_XDP_CONSUMED) {
+			page_pool_recycle_direct(pool, page);
+			goto requeue;
+		}
+
 		if (*xdp_state != ICSSG_XDP_PASS)
 			goto requeue;
 		headroom = xdp.data - xdp.data_hard_start;
diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c
index 42a881bee1092..591be5c8056b4 100644
--- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c
+++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c
@@ -260,6 +260,12 @@ static int prueth_emac_common_start(struct prueth *prueth)
 	icssg_class_default(prueth->miig_rt, ICSS_SLICE0, 0, false);
 	icssg_class_default(prueth->miig_rt, ICSS_SLICE1, 0, false);
 
+	/* Configure HSR/PRP protocol filtering if in HSR offload mode */
+	if (prueth->is_hsr_offload_mode) {
+		icssg_ft3_hsr_configurations(prueth->miig_rt, ICSS_SLICE0, prueth);
+		icssg_ft3_hsr_configurations(prueth->miig_rt, ICSS_SLICE1, prueth);
+	}
+
 	if (prueth->is_switch_mode || prueth->is_hsr_offload_mode)
 		icssg_init_fw_offload_mode(prueth);
 	else
@@ -663,28 +669,133 @@ static int icssg_prueth_del_mcast(struct net_device *ndev, const u8 *addr)
 	return 0;
 }
 
+/**
+ * icssg_is_addr_synced - Check if address is synced from HSR master
+ * @ndev: network device
+ * @addr: multicast MAC address
+ *
+ * Checks if the address is synced from HSR master (hsr0) via
+ * dev_mc_sync_multiple() or added directly to the slave port.
+ *
+ * Return: true if synced from HSR master, false if added directly
+ */
+static bool icssg_is_addr_synced(struct net_device *ndev, const u8 *addr)
+{
+	struct netdev_hw_addr *ha;
+	bool is_synced = false;
+
+	netif_addr_lock_bh(ndev);
+	netdev_for_each_mc_addr(ha, ndev) {
+		if (ether_addr_equal(ha->addr, addr)) {
+			is_synced = ha->synced;
+			break;
+		}
+	}
+	netif_addr_unlock_bh(ndev);
+
+	return is_synced;
+}
+
+/**
+ * icssg_hsr_fdb_update - Update FDB and VLAN table for HSR multicast
+ * @emac: PRUETH EMAC private data
+ * @addr: multicast MAC address
+ * @vid: VLAN ID
+ * @port_membership: port membership bitmap (P0=0x1, P1=0x2, P2=0x4)
+ * @add: true to add entry, false to delete
+ *
+ * Updates both FDB and VLAN table entries for the given address.
+ */
+static void icssg_hsr_fdb_update(struct prueth_emac *emac, const u8 *addr,
+				 u8 vid, u8 port_membership, bool add)
+{
+	icssg_fdb_add_del(emac, addr, vid, port_membership, add);
+	icssg_vtbl_modify(emac, vid, BIT(emac->port_id),
+			  BIT(emac->port_id), add);
+}
+
+/**
+ * icssg_prueth_hsr_fdb_add_del - Manage FDB port membership for HSR multicast
+ * @emac: PRUETH EMAC private data
+ * @addr: multicast MAC address
+ * @vid: VLAN ID
+ * @is_synced: true if synced from HSR master, false if added directly
+ * @is_vlan_path: true if called from VLAN interface path, false for direct path
+ * @add: true to add/update, false to remove
+ *
+ * Manages FDB port membership bits (P0/P1/P2) for multicast addresses.
+ * On add: accumulates membership with existing ports.
+ * On delete: removes only the specific port, cleans up orphaned P0 if needed.
+ */
 static void icssg_prueth_hsr_fdb_add_del(struct prueth_emac *emac,
-					 const u8 *addr, u8 vid, bool add)
+					 const u8 *addr, u8 vid,
+					 bool is_synced, bool is_vlan_path,
+					 bool add)
 {
-	icssg_fdb_add_del(emac, addr, vid,
-			  ICSSG_FDB_ENTRY_P0_MEMBERSHIP |
-			  ICSSG_FDB_ENTRY_P1_MEMBERSHIP |
-			  ICSSG_FDB_ENTRY_P2_MEMBERSHIP |
-			  ICSSG_FDB_ENTRY_BLOCK, add);
-
-	if (add)
-		icssg_vtbl_modify(emac, vid, BIT(emac->port_id),
-				  BIT(emac->port_id), add);
+	int existing_membership, other_port_membership;
+	u8 port_membership;
+
+	/* Set P0 (HSR master) bit when:
+	 * - Address is synced from HSR master (is_synced=true), OR
+	 * - In HSR offload mode AND it's a VLAN interface (is_vlan_path=true)
+	 * This ensures VLAN interfaces on HSR master (hsr0.X) also get P0 set.
+	 */
+	if (is_synced || (emac->prueth->is_hsr_offload_mode && is_vlan_path))
+		port_membership = ICSSG_FDB_ENTRY_P0_MEMBERSHIP | BIT(emac->port_id);
+	else
+		port_membership = BIT(emac->port_id);
+	existing_membership = icssg_fdb_lookup(emac, addr, vid);
+	if (existing_membership < 0) {
+		netdev_dbg(emac->ndev,
+			   "FDB lookup failed for %pM: %d, assuming no existing entry\n",
+			   addr, existing_membership);
+		existing_membership = 0;
+	}
+
+	if (add) {
+		/* Accumulate with existing membership */
+		port_membership |= existing_membership;
+
+		netdev_dbg(emac->ndev,
+			   "FDB add %pM: P%d%s -> membership 0x%x\n",
+			   addr, emac->port_id, is_synced ? "+P0" : "",
+			   port_membership);
+
+		icssg_hsr_fdb_update(emac, addr, vid, port_membership, true);
+	} else {
+		other_port_membership = existing_membership & ~port_membership;
+
+		/* Clean up orphaned P0 if neither HSR synced nor VLAN path */
+		if (!is_synced && !is_vlan_path &&
+		    (existing_membership & ICSSG_FDB_ENTRY_P0_MEMBERSHIP))
+			other_port_membership &=
+				~ICSSG_FDB_ENTRY_P0_MEMBERSHIP;
+
+		netdev_dbg(emac->ndev,
+			   "FDB del %pM: 0x%x -> 0x%x\n",
+			   addr, existing_membership, other_port_membership);
+
+		icssg_hsr_fdb_update(emac, addr, vid, port_membership, false);
+
+		if (other_port_membership)
+			icssg_hsr_fdb_update(emac, addr, vid,
+					     other_port_membership, true);
+	}
 }
 
 static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr)
 {
 	struct net_device *real_dev, *port_dev;
+	bool is_synced, is_vlan_path;
 	struct prueth_emac *emac;
 	u8 vlan_id, i;
 
-	vlan_id = is_vlan_dev(ndev) ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR;
-	real_dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev;
+	is_vlan_path = is_vlan_dev(ndev);
+	vlan_id = is_vlan_path ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR;
+	real_dev = is_vlan_path ? vlan_dev_real_dev(ndev) : ndev;
+
+	/* Check if this address is synced from HSR master */
+	is_synced = icssg_is_addr_synced(ndev, addr);
 
 	if (is_hsr_master(real_dev)) {
 		for (i = HSR_PT_SLAVE_A; i < HSR_PT_INTERLINK; i++) {
@@ -695,12 +806,14 @@ static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr)
 				return -EINVAL;
 			}
 			icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id,
+						     is_synced, is_vlan_path,
 						     true);
 			dev_put(port_dev);
 		}
 	} else {
 		emac = netdev_priv(real_dev);
-		icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, true);
+		icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id,
+					     is_synced, is_vlan_path, true);
 	}
 
 	return 0;
@@ -709,11 +822,16 @@ static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr)
 static int icssg_prueth_hsr_del_mcast(struct net_device *ndev, const u8 *addr)
 {
 	struct net_device *real_dev, *port_dev;
+	bool is_synced, is_vlan_path;
 	struct prueth_emac *emac;
 	u8 vlan_id, i;
 
-	vlan_id = is_vlan_dev(ndev) ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR;
-	real_dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev;
+	is_vlan_path = is_vlan_dev(ndev);
+	vlan_id = is_vlan_path ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR;
+	real_dev = is_vlan_path ? vlan_dev_real_dev(ndev) : ndev;
+
+	/* Check if this address was synced from HSR master */
+	is_synced = icssg_is_addr_synced(ndev, addr);
 
 	if (is_hsr_master(real_dev)) {
 		for (i = HSR_PT_SLAVE_A; i < HSR_PT_INTERLINK; i++) {
@@ -724,12 +842,14 @@ static int icssg_prueth_hsr_del_mcast(struct net_device *ndev, const u8 *addr)
 				return -EINVAL;
 			}
 			icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id,
+						     is_synced, is_vlan_path,
 						     false);
 			dev_put(port_dev);
 		}
 	} else {
 		emac = netdev_priv(real_dev);
-		icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, false);
+		icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id,
+					     is_synced, is_vlan_path, false);
 	}
 
 	return 0;
@@ -1053,6 +1173,104 @@ static int emac_ndo_stop(struct net_device *ndev)
 	return 0;
 }
 
+/**
+ * icssg_hsr_handle_multicast_sync() - Handle HSR multicast overlapping memberships
+ * @emac: PRUETH EMAC private structure
+ *
+ * Post-process multicast addresses to handle overlapping memberships where
+ * the same address is added from multiple paths (hsr0 + eth1). The kernel
+ * increments refcount without triggering callbacks, so manually check refcount
+ * to detect and update FDB port membership accordingly.
+ */
+static void icssg_hsr_handle_multicast_sync(struct prueth_emac *emac)
+{
+	struct hsr_mcast_update {
+		struct list_head list;
+		u8 addr[ETH_ALEN];
+		int refcount;
+		bool synced;
+	};
+	struct hsr_mcast_update *update, *tmp;
+	struct net_device *ndev = emac->ndev;
+	u8 vlan_id = PRUETH_DFLT_VLAN_HSR;
+	struct netdev_hw_addr *ha;
+	int existing_membership;
+	LIST_HEAD(update_list);
+	u8 port_membership;
+
+	/* Handle overlapping memberships: When the same address
+	 * is added from multiple paths (hsr0 + eth1), kernel
+	 * increments refcount without triggering callbacks.
+	 * Manually check refcount to detect:
+	 * - refcount=2 + synced: HSR only, membership = P0
+	 * - refcount>=3 + synced: HSR + direct, membership = P0|Px
+	 * - !synced but P0 set: HSR removed, clean up orphaned P0
+	 *
+	 * Collect addresses to update while holding the lock, then
+	 * process them after releasing the lock to avoid sleeping
+	 * while atomic (icssg_fdb_lookup/update can sleep).
+	 */
+	netif_addr_lock_bh(ndev);
+	netdev_for_each_mc_addr(ha, ndev) {
+		/* Queue this address for processing.
+		 * We need to check existing_membership via FDB lookup
+		 * (which can sleep), so defer that until after unlock.
+		 * Save synced/refcount to reproduce original logic.
+		 */
+		update = kmalloc_obj(*update, GFP_ATOMIC);
+		if (!update)
+			continue;
+
+		ether_addr_copy(update->addr, ha->addr);
+		update->synced = ha->synced;
+		update->refcount = ha->refcount;
+		list_add_tail(&update->list, &update_list);
+	}
+	netif_addr_unlock_bh(ndev);
+
+	/* Process collected addresses outside spinlock */
+	list_for_each_entry_safe(update, tmp, &update_list, list) {
+		existing_membership = icssg_fdb_lookup(emac,
+						       update->addr,
+						       vlan_id);
+		if (existing_membership < 0) {
+			netdev_dbg(ndev,
+				   "FDB lookup failed for %pM: %d, skipping\n",
+				   update->addr, existing_membership);
+			list_del(&update->list);
+			kfree(update);
+			continue;
+		}
+
+		port_membership = existing_membership;
+		if (update->synced) {
+			port_membership |=
+				ICSSG_FDB_ENTRY_P0_MEMBERSHIP;
+			if (update->refcount >= 3)
+				port_membership |= BIT(emac->port_id);
+			else
+				port_membership &= ~BIT(emac->port_id);
+		} else if (existing_membership &
+			   ICSSG_FDB_ENTRY_P0_MEMBERSHIP) {
+			port_membership &=
+				~ICSSG_FDB_ENTRY_P0_MEMBERSHIP;
+		}
+
+		if (existing_membership != port_membership) {
+			netdev_dbg(ndev, "Update %pM: 0x%x -> 0x%x\n",
+				   update->addr, existing_membership,
+				   port_membership);
+
+			icssg_hsr_fdb_update(emac, update->addr,
+					     vlan_id, port_membership,
+					     true);
+		}
+
+		list_del(&update->list);
+		kfree(update);
+	}
+}
+
 static void emac_ndo_set_rx_mode_work(struct work_struct *work)
 {
 	struct prueth_emac *emac = container_of(work, struct prueth_emac, rx_mode_work);
@@ -1079,8 +1297,13 @@ static void emac_ndo_set_rx_mode_work(struct work_struct *work)
 	}
 
 	if (emac->prueth->is_hsr_offload_mode) {
+		/* Track basic add/delete via callbacks */
 		__dev_mc_sync(ndev, icssg_prueth_hsr_add_mcast,
 			      icssg_prueth_hsr_del_mcast);
+
+		/* Handle overlapping memberships */
+		icssg_hsr_handle_multicast_sync(emac);
+
 		if (rtnl_trylock()) {
 			vlan_for_each(emac->prueth->hsr_dev,
 				      icssg_update_vlan_mcast, emac);
diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.h b/drivers/net/ethernet/ti/icssg/icssg_prueth.h
index 3d94fa5a7ac10..df93d15c5b786 100644
--- a/drivers/net/ethernet/ti/icssg/icssg_prueth.h
+++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.h
@@ -423,6 +423,8 @@ void icssg_class_promiscuous_sr1(struct regmap *miig_rt, int slice);
 void icssg_class_add_mcast_sr1(struct regmap *miig_rt, int slice,
 			       struct net_device *ndev);
 void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr);
+void icssg_ft3_hsr_configurations(struct regmap *miig_rt, int slice,
+				  struct prueth *prueth);
 
 /* config helpers */
 void icssg_config_ipg(struct prueth_emac *emac);
diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c
index b63965d9a1bab..1f0f389805498 100644
--- a/drivers/net/fjes/fjes_main.c
+++ b/drivers/net/fjes/fjes_main.c
@@ -111,56 +111,6 @@ fjes_get_acpi_resource(struct acpi_resource *acpi_res, void *data)
 	return AE_OK;
 }
 
-static struct resource fjes_resource[] = {
-	DEFINE_RES_MEM(0, 1),
-	DEFINE_RES_IRQ(0)
-};
-
-static int fjes_acpi_add(struct acpi_device *device)
-{
-	struct platform_device *plat_dev;
-	acpi_status status;
-
-	if (!is_extended_socket_device(device))
-		return -ENODEV;
-
-	if (acpi_check_extended_socket_status(device))
-		return -ENODEV;
-
-	status = acpi_walk_resources(device->handle, METHOD_NAME__CRS,
-				     fjes_get_acpi_resource, fjes_resource);
-	if (ACPI_FAILURE(status))
-		return -ENODEV;
-
-	/* create platform_device */
-	plat_dev = platform_device_register_simple(DRV_NAME, 0, fjes_resource,
-						   ARRAY_SIZE(fjes_resource));
-	if (IS_ERR(plat_dev))
-		return PTR_ERR(plat_dev);
-
-	device->driver_data = plat_dev;
-
-	return 0;
-}
-
-static void fjes_acpi_remove(struct acpi_device *device)
-{
-	struct platform_device *plat_dev;
-
-	plat_dev = (struct platform_device *)acpi_driver_data(device);
-	platform_device_unregister(plat_dev);
-}
-
-static struct acpi_driver fjes_acpi_driver = {
-	.name = DRV_NAME,
-	.class = DRV_NAME,
-	.ids = fjes_acpi_ids,
-	.ops = {
-		.add = fjes_acpi_add,
-		.remove = fjes_acpi_remove,
-	},
-};
-
 static int fjes_setup_resources(struct fjes_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
@@ -1470,43 +1420,81 @@ static struct platform_driver fjes_driver = {
 	.remove = fjes_remove,
 };
 
+struct fjes_acpi_walk_context {
+	struct acpi_device *adev;
+	struct resource resources[2];
+};
+
 static acpi_status
 acpi_find_extended_socket_device(acpi_handle obj_handle, u32 level,
 				 void *context, void **return_value)
 {
+	struct fjes_acpi_walk_context *fjes_context = context;
 	struct acpi_device *device;
-	bool *found = context;
+	acpi_status status;
 
-	device = acpi_fetch_acpi_dev(obj_handle);
+	device = acpi_get_acpi_dev(obj_handle);
 	if (!device)
 		return AE_OK;
 
 	if (strcmp(acpi_device_hid(device), ACPI_MOTHERBOARD_RESOURCE_HID))
-		return AE_OK;
+		goto skip;
 
 	if (!is_extended_socket_device(device))
-		return AE_OK;
+		goto skip;
 
 	if (acpi_check_extended_socket_status(device))
-		return AE_OK;
+		goto skip;
+
+	status = acpi_walk_resources(obj_handle, METHOD_NAME__CRS,
+				     fjes_get_acpi_resource, fjes_context->resources);
+	if (ACPI_FAILURE(status))
+		goto skip;
+
+	fjes_context->adev = device;
 
-	*found = true;
 	return AE_CTRL_TERMINATE;
+
+skip:
+	acpi_dev_put(device);
+	return AE_OK;
 }
 
+static struct platform_device *fjes_plat_dev;
+
 /* fjes_init_module - Driver Registration Routine */
 static int __init fjes_init_module(void)
 {
-	bool found = false;
+	struct fjes_acpi_walk_context fjes_context = {
+		.adev = NULL,
+		.resources = {
+			DEFINE_RES_MEM(0, 1),
+			DEFINE_RES_IRQ(0)
+		}
+	};
+	struct platform_device_info pdevinfo;
 	int result;
 
 	acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT, ACPI_UINT32_MAX,
-			    acpi_find_extended_socket_device, NULL, &found,
+			    acpi_find_extended_socket_device, NULL, &fjes_context,
 			    NULL);
-
-	if (!found)
+	if (!fjes_context.adev)
 		return -ENODEV;
 
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+
+	pdevinfo.name = DRV_NAME;
+	pdevinfo.res = fjes_context.resources;
+	pdevinfo.num_res = ARRAY_SIZE(fjes_context.resources);
+	pdevinfo.fwnode = acpi_fwnode_handle(fjes_context.adev);
+
+	fjes_plat_dev = platform_device_register_full(&pdevinfo);
+
+	acpi_dev_put(fjes_context.adev);
+
+	if (IS_ERR(fjes_plat_dev))
+		return PTR_ERR(fjes_plat_dev);
+
 	pr_info("%s - version %s - %s\n",
 		fjes_driver_string, fjes_driver_version, fjes_copyright);
 
@@ -1515,19 +1503,11 @@ static int __init fjes_init_module(void)
 	result = platform_driver_register(&fjes_driver);
 	if (result < 0) {
 		fjes_dbg_exit();
+		platform_device_unregister(fjes_plat_dev);
 		return result;
 	}
 
-	result = acpi_bus_register_driver(&fjes_acpi_driver);
-	if (result < 0)
-		goto fail_acpi_driver;
-
 	return 0;
-
-fail_acpi_driver:
-	platform_driver_unregister(&fjes_driver);
-	fjes_dbg_exit();
-	return result;
 }
 
 module_init(fjes_init_module);
@@ -1535,9 +1515,9 @@ module_init(fjes_init_module);
 /* fjes_exit_module - Driver Exit Cleanup Routine */
 static void __exit fjes_exit_module(void)
 {
-	acpi_bus_unregister_driver(&fjes_acpi_driver);
 	platform_driver_unregister(&fjes_driver);
 	fjes_dbg_exit();
+	platform_device_unregister(fjes_plat_dev);
 }
 
 module_exit(fjes_exit_module);
diff --git a/drivers/net/ipa/ipa_data.h b/drivers/net/ipa/ipa_data.h
index 2fd03f0799b20..f3bdc64cef05d 100644
--- a/drivers/net/ipa/ipa_data.h
+++ b/drivers/net/ipa/ipa_data.h
@@ -185,8 +185,13 @@ struct ipa_resource_data {
 struct ipa_mem_data {
 	u32 local_count;
 	const struct ipa_mem *local;
-	u32 imem_addr;
-	u32 imem_size;
+
+	/* These values are now passed via DT, but to support
+	 * older systems we must allow this to be specified here.
+	 */
+	u32 imem_addr; /* DEPRECATED */
+	u32 imem_size; /* DEPRECATED */
+
 	u32 smem_size;
 };
 
diff --git a/drivers/net/ipa/ipa_mem.c b/drivers/net/ipa/ipa_mem.c
index 835a3c9c1fd47..078d32a18dbfd 100644
--- a/drivers/net/ipa/ipa_mem.c
+++ b/drivers/net/ipa/ipa_mem.c
@@ -7,6 +7,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/iommu.h>
+#include <linux/of_address.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
 
@@ -617,7 +618,9 @@ static void ipa_smem_exit(struct ipa *ipa)
 int ipa_mem_init(struct ipa *ipa, struct platform_device *pdev,
 		 const struct ipa_mem_data *mem_data)
 {
+	struct device_node *ipa_slice_np;
 	struct device *dev = &pdev->dev;
+	u32 imem_base, imem_size;
 	struct resource *res;
 	int ret;
 
@@ -656,7 +659,26 @@ int ipa_mem_init(struct ipa *ipa, struct platform_device *pdev,
 	ipa->mem_addr = res->start;
 	ipa->mem_size = resource_size(res);
 
-	ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
+	ipa_slice_np = of_parse_phandle(dev->of_node, "sram", 0);
+	if (ipa_slice_np) {
+		struct resource sram_res;
+
+		ret = of_address_to_resource(ipa_slice_np, 0, &sram_res);
+		of_node_put(ipa_slice_np);
+		if (ret)
+			goto err_unmap;
+
+		imem_base = sram_res.start;
+		imem_size = resource_size(&sram_res);
+	} else {
+		/* Backwards compatibility for DTs lacking
+		 * an explicit reference
+		 */
+		imem_base = mem_data->imem_addr;
+		imem_size = mem_data->imem_size;
+	}
+
+	ret = ipa_imem_init(ipa, imem_base, imem_size);
 	if (ret)
 		goto err_unmap;
 
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index a71f058eceef0..ea22909cb09de 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -313,11 +313,15 @@ static void macvlan_multicast_rx(const struct macvlan_port *port,
 				  MACVLAN_MODE_BRIDGE);
 	else
 		/*
-		 * flood only to VEPA ports, bridge ports
-		 * already saw the frame on the way out.
+		 * Flood to VEPA and bridge ports. We cannot distinguish
+		 * a looped-back locally-originated multicast from one
+		 * sent by an external source sharing the same source MAC
+		 * (e.g., VRRP virtual MAC), so deliver to bridge ports
+		 * as well to ensure correct reception in all cases.
 		 */
-		macvlan_broadcast(skb, port, src->dev,
-				  MACVLAN_MODE_VEPA);
+		macvlan_broadcast(skb, port, NULL,
+				  MACVLAN_MODE_VEPA |
+				  MACVLAN_MODE_BRIDGE);
 }
 
 static void macvlan_process_broadcast(struct work_struct *w)
diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig
index 44380378911b7..53e2c047d8042 100644
--- a/drivers/net/mdio/Kconfig
+++ b/drivers/net/mdio/Kconfig
@@ -7,7 +7,6 @@ if PHYLIB
 
 config FWNODE_MDIO
 	def_tristate (ACPI || OF) || COMPILE_TEST
-	select FIXED_PHY
 	help
 	  FWNODE MDIO bus (Ethernet PHY) accessors
 
diff --git a/drivers/net/mdio/mdio-gpio.c b/drivers/net/mdio/mdio-gpio.c
index 1cfd538b5105d..958d1c6608ab7 100644
--- a/drivers/net/mdio/mdio-gpio.c
+++ b/drivers/net/mdio/mdio-gpio.c
@@ -20,13 +20,15 @@
 #include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/mdio-bitbang.h>
-#include <linux/mdio-gpio.h>
 #include <linux/module.h>
 #include <linux/of_mdio.h>
-#include <linux/platform_data/mdio-gpio.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
+#define MDIO_GPIO_MDC	0
+#define MDIO_GPIO_MDIO	1
+#define MDIO_GPIO_MDO	2
+
 struct mdio_gpio_info {
 	struct mdiobb_ctrl ctrl;
 	struct gpio_desc *mdc, *mdio, *mdo;
@@ -110,7 +112,6 @@ static struct mii_bus *mdio_gpio_bus_init(struct device *dev,
 					  struct mdio_gpio_info *bitbang,
 					  int bus_id)
 {
-	struct mdio_gpio_platform_data *pdata = dev_get_platdata(dev);
 	struct mii_bus *new_bus;
 
 	bitbang->ctrl.ops = &mdio_gpio_ops;
@@ -127,11 +128,6 @@ static struct mii_bus *mdio_gpio_bus_init(struct device *dev,
 	else
 		strscpy(new_bus->id, "gpio", sizeof(new_bus->id));
 
-	if (pdata) {
-		new_bus->phy_mask = pdata->phy_mask;
-		new_bus->phy_ignore_ta_mask = pdata->phy_ignore_ta_mask;
-	}
-
 	if (device_is_compatible(dev, "microchip,mdio-smi0")) {
 		bitbang->ctrl.op_c22_read = 0;
 		bitbang->ctrl.op_c22_write = 0;
diff --git a/drivers/net/mdio/mdio-mvusb.c b/drivers/net/mdio/mdio-mvusb.c
index 554837c21e73c..86786ecaa4b1c 100644
--- a/drivers/net/mdio/mdio-mvusb.c
+++ b/drivers/net/mdio/mdio-mvusb.c
@@ -67,7 +67,6 @@ static int mvusb_mdio_probe(struct usb_interface *interface,
 	struct device *dev = &interface->dev;
 	struct mvusb_mdio *mvusb;
 	struct mii_bus *mdio;
-	int ret;
 
 	mdio = devm_mdiobus_alloc_size(dev, sizeof(*mvusb));
 	if (!mdio)
@@ -75,7 +74,7 @@ static int mvusb_mdio_probe(struct usb_interface *interface,
 
 	mvusb = mdio->priv;
 	mvusb->mdio = mdio;
-	mvusb->udev = usb_get_dev(interface_to_usbdev(interface));
+	mvusb->udev = interface_to_usbdev(interface);
 
 	/* Reversed from USB PCAPs, no idea what these mean. */
 	mvusb->buf[MVUSB_CMD_PREAMBLE0] = cpu_to_le16(0xe800);
@@ -88,25 +87,16 @@ static int mvusb_mdio_probe(struct usb_interface *interface,
 	mdio->write = mvusb_mdio_write;
 
 	usb_set_intfdata(interface, mvusb);
-	ret = of_mdiobus_register(mdio, dev->of_node);
-	if (ret)
-		goto put_dev;
 
-	return 0;
-
-put_dev:
-	usb_put_dev(mvusb->udev);
-	return ret;
+	return of_mdiobus_register(mdio, dev->of_node);
 }
 
 static void mvusb_mdio_disconnect(struct usb_interface *interface)
 {
 	struct mvusb_mdio *mvusb = usb_get_intfdata(interface);
-	struct usb_device *udev = mvusb->udev;
 
 	mdiobus_unregister(mvusb->mdio);
 	usb_set_intfdata(interface, NULL);
-	usb_put_dev(udev);
 }
 
 static struct usb_driver mvusb_mdio_driver = {
diff --git a/drivers/net/mdio/mdio-xgene.c b/drivers/net/mdio/mdio-xgene.c
index a8f91a4b7fed0..ede6b9ddc4261 100644
--- a/drivers/net/mdio/mdio-xgene.c
+++ b/drivers/net/mdio/mdio-xgene.c
@@ -250,7 +250,7 @@ static int xgene_xfi_mdio_read(struct mii_bus *bus, int phy_id, int reg)
 	} while ((status & BUSY_MASK) && timeout--);
 
 	if (status & BUSY_MASK) {
-		pr_err("XGENET_MII_MGMT write failed\n");
+		pr_err("XGENET_MII_MGMT read failed\n");
 		return -EBUSY;
 	}
 
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
index 5ec028a00c620..3645ebde049a0 100644
--- a/drivers/net/netdevsim/netdev.c
+++ b/drivers/net/netdevsim/netdev.c
@@ -109,8 +109,11 @@ static int nsim_forward_skb(struct net_device *tx_dev,
 	int ret;
 
 	ret = __dev_forward_skb(rx_dev, skb);
-	if (ret)
+	if (ret) {
+		if (psp_ext)
+			__skb_ext_put(psp_ext);
 		return ret;
+	}
 
 	nsim_psp_handle_ext(skb, psp_ext);
 
diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c
index fbeae05817e9f..c3a6ba96fc8a0 100644
--- a/drivers/net/ntb_netdev.c
+++ b/drivers/net/ntb_netdev.c
@@ -1,51 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
 /*
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- *   redistributing this file, you may do so under either license.
- *
- *   GPL LICENSE SUMMARY
- *
- *   Copyright(c) 2012 Intel Corporation. All rights reserved.
- *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of version 2 of the GNU General Public License as
- *   published by the Free Software Foundation.
- *
- *   BSD LICENSE
- *
- *   Copyright(c) 2012 Intel Corporation. All rights reserved.
- *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copy
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
  * PCIe NTB Network Linux driver
- *
- * Contact Information:
- * Jon Mason <jon.mason@intel.com>
  */
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
@@ -53,6 +8,7 @@
 #include <linux/pci.h>
 #include <linux/ntb.h>
 #include <linux/ntb_transport.h>
+#include <linux/slab.h>
 
 #define NTB_NETDEV_VER	"0.7"
 
@@ -70,39 +26,112 @@ static unsigned int tx_start = 10;
 /* Number of descriptors still available before stop upper layer tx */
 static unsigned int tx_stop = 5;
 
+#define NTB_NETDEV_MAX_QUEUES		64
+#define NTB_NETDEV_DEFAULT_QUEUES	1
+
+struct ntb_netdev;
+
+struct ntb_netdev_queue {
+	struct ntb_netdev *ntdev;
+	struct ntb_transport_qp *qp;
+	struct timer_list tx_timer;
+	u16 qid;
+};
+
 struct ntb_netdev {
 	struct pci_dev *pdev;
+	struct device *client_dev;
 	struct net_device *ndev;
-	struct ntb_transport_qp *qp;
-	struct timer_list tx_timer;
+	unsigned int num_queues;
+	struct ntb_netdev_queue *queues;
 };
 
 #define	NTB_TX_TIMEOUT_MS	1000
 #define	NTB_RXQ_SIZE		100
 
-static void ntb_netdev_event_handler(void *data, int link_is_up)
+static void ntb_netdev_update_carrier(struct ntb_netdev *dev)
 {
-	struct net_device *ndev = data;
-	struct ntb_netdev *dev = netdev_priv(ndev);
+	struct net_device *ndev;
+	bool any_up = false;
+	unsigned int i;
 
-	netdev_dbg(ndev, "Event %x, Link %x\n", link_is_up,
-		   ntb_transport_link_query(dev->qp));
+	ndev = dev->ndev;
 
-	if (link_is_up) {
-		if (ntb_transport_link_query(dev->qp))
-			netif_carrier_on(ndev);
-	} else {
+	for (i = 0; i < dev->num_queues; i++) {
+		if (ntb_transport_link_query(dev->queues[i].qp)) {
+			any_up = true;
+			break;
+		}
+	}
+
+	if (any_up)
+		netif_carrier_on(ndev);
+	else
 		netif_carrier_off(ndev);
+}
+
+static void ntb_netdev_queue_rx_drain(struct ntb_netdev_queue *queue)
+{
+	struct sk_buff *skb;
+	int len;
+
+	while ((skb = ntb_transport_rx_remove(queue->qp, &len)))
+		dev_kfree_skb(skb);
+}
+
+static int ntb_netdev_queue_rx_fill(struct net_device *ndev,
+				    struct ntb_netdev_queue *queue)
+{
+	struct sk_buff *skb;
+	int rc, i;
+
+	for (i = 0; i < NTB_RXQ_SIZE; i++) {
+		skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN);
+		if (!skb)
+			return -ENOMEM;
+
+		rc = ntb_transport_rx_enqueue(queue->qp, skb, skb->data,
+					      ndev->mtu + ETH_HLEN);
+		if (rc) {
+			dev_kfree_skb(skb);
+			return rc;
+		}
 	}
+
+	return 0;
+}
+
+static void ntb_netdev_event_handler(void *data, int link_is_up)
+{
+	struct ntb_netdev_queue *q = data;
+	struct ntb_netdev *dev = q->ntdev;
+	struct net_device *ndev;
+
+	ndev = dev->ndev;
+
+	netdev_dbg(ndev, "Event %x, Link %x, qp %u\n", link_is_up,
+		   ntb_transport_link_query(q->qp), q->qid);
+
+	if (netif_running(ndev)) {
+		if (link_is_up)
+			netif_wake_subqueue(ndev, q->qid);
+		else
+			netif_stop_subqueue(ndev, q->qid);
+	}
+
+	ntb_netdev_update_carrier(dev);
 }
 
 static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
 				  void *data, int len)
 {
-	struct net_device *ndev = qp_data;
+	struct ntb_netdev_queue *q = qp_data;
+	struct ntb_netdev *dev = q->ntdev;
+	struct net_device *ndev;
 	struct sk_buff *skb;
 	int rc;
 
+	ndev = dev->ndev;
 	skb = data;
 	if (!skb)
 		return;
@@ -118,6 +147,7 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
 	skb_put(skb, len);
 	skb->protocol = eth_type_trans(skb, ndev);
 	skb->ip_summed = CHECKSUM_NONE;
+	skb_record_rx_queue(skb, q->qid);
 
 	if (netif_rx(skb) == NET_RX_DROP) {
 		ndev->stats.rx_errors++;
@@ -144,42 +174,46 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
 }
 
 static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev,
-				      struct ntb_transport_qp *qp, int size)
+				      struct ntb_netdev_queue *q, int size)
 {
-	struct ntb_netdev *dev = netdev_priv(netdev);
+	netif_stop_subqueue(netdev, q->qid);
 
-	netif_stop_queue(netdev);
 	/* Make sure to see the latest value of ntb_transport_tx_free_entry()
 	 * since the queue was last started.
 	 */
 	smp_mb();
 
-	if (likely(ntb_transport_tx_free_entry(qp) < size)) {
-		mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time));
+	if (likely(ntb_transport_tx_free_entry(q->qp) < size)) {
+		mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time));
 		return -EBUSY;
 	}
 
-	netif_start_queue(netdev);
+	/* The subqueue must be kept stopped if the link is down */
+	if (ntb_transport_link_query(q->qp))
+		netif_start_subqueue(netdev, q->qid);
+
 	return 0;
 }
 
 static int ntb_netdev_maybe_stop_tx(struct net_device *ndev,
-				    struct ntb_transport_qp *qp, int size)
+				    struct ntb_netdev_queue *q, int size)
 {
-	if (netif_queue_stopped(ndev) ||
-	    (ntb_transport_tx_free_entry(qp) >= size))
+	if (__netif_subqueue_stopped(ndev, q->qid) ||
+	    (ntb_transport_tx_free_entry(q->qp) >= size))
 		return 0;
 
-	return __ntb_netdev_maybe_stop_tx(ndev, qp, size);
+	return __ntb_netdev_maybe_stop_tx(ndev, q, size);
 }
 
 static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
 				  void *data, int len)
 {
-	struct net_device *ndev = qp_data;
+	struct ntb_netdev_queue *q = qp_data;
+	struct ntb_netdev *dev = q->ntdev;
+	struct net_device *ndev;
 	struct sk_buff *skb;
-	struct ntb_netdev *dev = netdev_priv(ndev);
 
+	ndev = dev->ndev;
 	skb = data;
 	if (!skb || !ndev)
 		return;
@@ -194,30 +228,41 @@ static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
 
 	dev_kfree_skb_any(skb);
 
-	if (ntb_transport_tx_free_entry(dev->qp) >= tx_start) {
+	if (ntb_transport_tx_free_entry(qp) >= tx_start) {
 		/* Make sure anybody stopping the queue after this sees the new
 		 * value of ntb_transport_tx_free_entry()
 		 */
 		smp_mb();
-		if (netif_queue_stopped(ndev))
-			netif_wake_queue(ndev);
+		if (__netif_subqueue_stopped(ndev, q->qid) &&
+		    ntb_transport_link_query(q->qp))
+			netif_wake_subqueue(ndev, q->qid);
 	}
 }
 
+static const struct ntb_queue_handlers ntb_netdev_handlers = {
+	.tx_handler = ntb_netdev_tx_handler,
+	.rx_handler = ntb_netdev_rx_handler,
+	.event_handler = ntb_netdev_event_handler,
+};
+
 static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb,
 					 struct net_device *ndev)
 {
 	struct ntb_netdev *dev = netdev_priv(ndev);
+	u16 qid = skb_get_queue_mapping(skb);
+	struct ntb_netdev_queue *q;
 	int rc;
 
-	ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop);
+	q = &dev->queues[qid];
 
-	rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len);
+	ntb_netdev_maybe_stop_tx(ndev, q, tx_stop);
+
+	rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len);
 	if (rc)
 		goto err;
 
 	/* check for next submit */
-	ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop);
+	ntb_netdev_maybe_stop_tx(ndev, q, tx_stop);
 
 	return NETDEV_TX_OK;
 
@@ -229,69 +274,77 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb,
 
 static void ntb_netdev_tx_timer(struct timer_list *t)
 {
-	struct ntb_netdev *dev = timer_container_of(dev, t, tx_timer);
-	struct net_device *ndev = dev->ndev;
+	struct ntb_netdev_queue *q = timer_container_of(q, t, tx_timer);
+	struct ntb_netdev *dev = q->ntdev;
+	struct net_device *ndev;
 
-	if (ntb_transport_tx_free_entry(dev->qp) < tx_stop) {
-		mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time));
+	ndev = dev->ndev;
+
+	if (ntb_transport_tx_free_entry(q->qp) < tx_stop) {
+		mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time));
 	} else {
 		/* Make sure anybody stopping the queue after this sees the new
 		 * value of ntb_transport_tx_free_entry()
 		 */
 		smp_mb();
-		if (netif_queue_stopped(ndev))
-			netif_wake_queue(ndev);
+
+		/* The subqueue must be kept stopped if the link is down */
+		if (__netif_subqueue_stopped(ndev, q->qid) &&
+		    ntb_transport_link_query(q->qp))
+			netif_wake_subqueue(ndev, q->qid);
 	}
 }
 
 static int ntb_netdev_open(struct net_device *ndev)
 {
 	struct ntb_netdev *dev = netdev_priv(ndev);
-	struct sk_buff *skb;
-	int rc, i, len;
+	struct ntb_netdev_queue *queue;
+	unsigned int q;
+	int rc = 0;
 
-	/* Add some empty rx bufs */
-	for (i = 0; i < NTB_RXQ_SIZE; i++) {
-		skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN);
-		if (!skb) {
-			rc = -ENOMEM;
-			goto err;
-		}
+	/* Add some empty rx bufs for each queue */
+	for (q = 0; q < dev->num_queues; q++) {
+		queue = &dev->queues[q];
 
-		rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data,
-					      ndev->mtu + ETH_HLEN);
-		if (rc) {
-			dev_kfree_skb(skb);
+		rc = ntb_netdev_queue_rx_fill(ndev, queue);
+		if (rc)
 			goto err;
-		}
-	}
 
-	timer_setup(&dev->tx_timer, ntb_netdev_tx_timer, 0);
+		timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0);
+	}
 
 	netif_carrier_off(ndev);
-	ntb_transport_link_up(dev->qp);
-	netif_start_queue(ndev);
+	netif_tx_stop_all_queues(ndev);
+
+	for (q = 0; q < dev->num_queues; q++)
+		ntb_transport_link_up(dev->queues[q].qp);
 
 	return 0;
 
 err:
-	while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
-		dev_kfree_skb(skb);
+	for (q = 0; q < dev->num_queues; q++) {
+		queue = &dev->queues[q];
+		ntb_netdev_queue_rx_drain(queue);
+	}
 	return rc;
 }
 
 static int ntb_netdev_close(struct net_device *ndev)
 {
 	struct ntb_netdev *dev = netdev_priv(ndev);
-	struct sk_buff *skb;
-	int len;
+	struct ntb_netdev_queue *queue;
+	unsigned int q;
 
-	ntb_transport_link_down(dev->qp);
+	netif_tx_stop_all_queues(ndev);
+	netif_carrier_off(ndev);
 
-	while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
-		dev_kfree_skb(skb);
+	for (q = 0; q < dev->num_queues; q++) {
+		queue = &dev->queues[q];
 
-	timer_delete_sync(&dev->tx_timer);
+		ntb_transport_link_down(queue->qp);
+		ntb_netdev_queue_rx_drain(queue);
+		timer_delete_sync(&queue->tx_timer);
+	}
 
 	return 0;
 }
@@ -299,10 +352,12 @@ static int ntb_netdev_close(struct net_device *ndev)
 static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu)
 {
 	struct ntb_netdev *dev = netdev_priv(ndev);
+	struct ntb_netdev_queue *queue;
 	struct sk_buff *skb;
-	int len, rc;
+	unsigned int q, i;
+	int len, rc = 0;
 
-	if (new_mtu > ntb_transport_max_size(dev->qp) - ETH_HLEN)
+	if (new_mtu > ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN)
 		return -EINVAL;
 
 	if (!netif_running(ndev)) {
@@ -311,41 +366,53 @@ static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu)
 	}
 
 	/* Bring down the link and dispose of posted rx entries */
-	ntb_transport_link_down(dev->qp);
+	for (q = 0; q < dev->num_queues; q++)
+		ntb_transport_link_down(dev->queues[q].qp);
 
 	if (ndev->mtu < new_mtu) {
-		int i;
-
-		for (i = 0; (skb = ntb_transport_rx_remove(dev->qp, &len)); i++)
-			dev_kfree_skb(skb);
-
-		for (; i; i--) {
-			skb = netdev_alloc_skb(ndev, new_mtu + ETH_HLEN);
-			if (!skb) {
-				rc = -ENOMEM;
-				goto err;
-			}
+		for (q = 0; q < dev->num_queues; q++) {
+			queue = &dev->queues[q];
 
-			rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data,
-						      new_mtu + ETH_HLEN);
-			if (rc) {
+			for (i = 0;
+			     (skb = ntb_transport_rx_remove(queue->qp, &len));
+			     i++)
 				dev_kfree_skb(skb);
-				goto err;
+
+			for (; i; i--) {
+				skb = netdev_alloc_skb(ndev,
+						       new_mtu + ETH_HLEN);
+				if (!skb) {
+					rc = -ENOMEM;
+					goto err;
+				}
+
+				rc = ntb_transport_rx_enqueue(queue->qp, skb,
+							      skb->data,
+							      new_mtu +
+							      ETH_HLEN);
+				if (rc) {
+					dev_kfree_skb(skb);
+					goto err;
+				}
 			}
 		}
 	}
 
 	WRITE_ONCE(ndev->mtu, new_mtu);
 
-	ntb_transport_link_up(dev->qp);
+	for (q = 0; q < dev->num_queues; q++)
+		ntb_transport_link_up(dev->queues[q].qp);
 
 	return 0;
 
 err:
-	ntb_transport_link_down(dev->qp);
+	for (q = 0; q < dev->num_queues; q++) {
+		struct ntb_netdev_queue *queue = &dev->queues[q];
 
-	while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
-		dev_kfree_skb(skb);
+		ntb_transport_link_down(queue->qp);
+
+		ntb_netdev_queue_rx_drain(queue);
+	}
 
 	netdev_err(ndev, "Error changing MTU, device inoperable\n");
 	return rc;
@@ -386,16 +453,155 @@ static int ntb_get_link_ksettings(struct net_device *dev,
 	return 0;
 }
 
+static void ntb_get_channels(struct net_device *ndev,
+			     struct ethtool_channels *channels)
+{
+	struct ntb_netdev *dev = netdev_priv(ndev);
+
+	channels->combined_count = dev->num_queues;
+	channels->max_combined = ndev->num_tx_queues;
+}
+
+static int ntb_inc_channels(struct net_device *ndev,
+			    unsigned int old, unsigned int new)
+{
+	struct ntb_netdev *dev = netdev_priv(ndev);
+	bool running = netif_running(ndev);
+	struct ntb_netdev_queue *queue;
+	unsigned int q, created;
+	int rc;
+
+	created = old;
+	for (q = old; q < new; q++) {
+		queue = &dev->queues[q];
+
+		queue->ntdev = dev;
+		queue->qid = q;
+		queue->qp = ntb_transport_create_queue(queue, dev->client_dev,
+						       &ntb_netdev_handlers);
+		if (!queue->qp) {
+			rc = -ENOSPC;
+			goto err_new;
+		}
+		created++;
+
+		if (!running)
+			continue;
+
+		timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0);
+
+		rc = ntb_netdev_queue_rx_fill(ndev, queue);
+		if (rc)
+			goto err_new;
+
+		/*
+		 * Carrier may already be on due to other QPs. Keep the new
+		 * subqueue stopped until we get a Link Up event for this QP.
+		 */
+		netif_stop_subqueue(ndev, q);
+	}
+
+	rc = netif_set_real_num_queues(ndev, new, new);
+	if (rc)
+		goto err_new;
+
+	dev->num_queues = new;
+
+	if (running)
+		for (q = old; q < new; q++)
+			ntb_transport_link_up(dev->queues[q].qp);
+
+	return 0;
+
+err_new:
+	if (running) {
+		unsigned int rollback = created;
+
+		while (rollback-- > old) {
+			queue = &dev->queues[rollback];
+			ntb_transport_link_down(queue->qp);
+			ntb_netdev_queue_rx_drain(queue);
+			timer_delete_sync(&queue->tx_timer);
+		}
+	}
+	while (created-- > old) {
+		queue = &dev->queues[created];
+		ntb_transport_free_queue(queue->qp);
+		queue->qp = NULL;
+	}
+	return rc;
+}
+
+static int ntb_dec_channels(struct net_device *ndev,
+			    unsigned int old, unsigned int new)
+{
+	struct ntb_netdev *dev = netdev_priv(ndev);
+	bool running = netif_running(ndev);
+	struct ntb_netdev_queue *queue;
+	unsigned int q;
+	int rc;
+
+	if (running)
+		for (q = new; q < old; q++)
+			netif_stop_subqueue(ndev, q);
+
+	rc = netif_set_real_num_queues(ndev, new, new);
+	if (rc)
+		goto err;
+
+	/* Publish new queue count before invalidating QP pointers */
+	dev->num_queues = new;
+
+	for (q = new; q < old; q++) {
+		queue = &dev->queues[q];
+
+		if (running) {
+			ntb_transport_link_down(queue->qp);
+			ntb_netdev_queue_rx_drain(queue);
+			timer_delete_sync(&queue->tx_timer);
+		}
+
+		ntb_transport_free_queue(queue->qp);
+		queue->qp = NULL;
+	}
+
+	/*
+	 * It might be the case that the removed queues are the only queues that
+	 * were up, so see if the global carrier needs to change.
+	 */
+	ntb_netdev_update_carrier(dev);
+	return 0;
+
+err:
+	if (running) {
+		for (q = new; q < old; q++)
+			netif_wake_subqueue(ndev, q);
+	}
+	return rc;
+}
+
+static int ntb_set_channels(struct net_device *ndev,
+			    struct ethtool_channels *channels)
+{
+	struct ntb_netdev *dev = netdev_priv(ndev);
+	unsigned int new = channels->combined_count;
+	unsigned int old = dev->num_queues;
+
+	if (new == old)
+		return 0;
+
+	if (new < old)
+		return ntb_dec_channels(ndev, old, new);
+	else
+		return ntb_inc_channels(ndev, old, new);
+}
+
 static const struct ethtool_ops ntb_ethtool_ops = {
 	.get_drvinfo = ntb_get_drvinfo,
 	.get_link = ethtool_op_get_link,
 	.get_link_ksettings = ntb_get_link_ksettings,
-};
-
-static const struct ntb_queue_handlers ntb_netdev_handlers = {
-	.tx_handler = ntb_netdev_tx_handler,
-	.rx_handler = ntb_netdev_rx_handler,
-	.event_handler = ntb_netdev_event_handler,
+	.get_channels = ntb_get_channels,
+	.set_channels = ntb_set_channels,
 };
 
 static int ntb_netdev_probe(struct device *client_dev)
@@ -404,6 +610,7 @@ static int ntb_netdev_probe(struct device *client_dev)
 	struct net_device *ndev;
 	struct pci_dev *pdev;
 	struct ntb_netdev *dev;
+	unsigned int q;
 	int rc;
 
 	ntb = dev_ntb(client_dev->parent);
@@ -411,7 +618,7 @@ static int ntb_netdev_probe(struct device *client_dev)
 	if (!pdev)
 		return -ENODEV;
 
-	ndev = alloc_etherdev(sizeof(*dev));
+	ndev = alloc_etherdev_mq(sizeof(*dev), NTB_NETDEV_MAX_QUEUES);
 	if (!ndev)
 		return -ENOMEM;
 
@@ -420,6 +627,16 @@ static int ntb_netdev_probe(struct device *client_dev)
 	dev = netdev_priv(ndev);
 	dev->ndev = ndev;
 	dev->pdev = pdev;
+	dev->client_dev = client_dev;
+	dev->num_queues = 0;
+
+	dev->queues = kzalloc_objs(*dev->queues, NTB_NETDEV_MAX_QUEUES,
+				   GFP_KERNEL);
+	if (!dev->queues) {
+		rc = -ENOMEM;
+		goto err_free_netdev;
+	}
+
 	ndev->features = NETIF_F_HIGHDMA;
 
 	ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
@@ -436,26 +653,47 @@ static int ntb_netdev_probe(struct device *client_dev)
 	ndev->min_mtu = 0;
 	ndev->max_mtu = ETH_MAX_MTU;
 
-	dev->qp = ntb_transport_create_queue(ndev, client_dev,
-					     &ntb_netdev_handlers);
-	if (!dev->qp) {
+	for (q = 0; q < NTB_NETDEV_DEFAULT_QUEUES; q++) {
+		struct ntb_netdev_queue *queue = &dev->queues[q];
+
+		queue->ntdev = dev;
+		queue->qid = q;
+		queue->qp = ntb_transport_create_queue(queue, client_dev,
+						       &ntb_netdev_handlers);
+		if (!queue->qp)
+			break;
+
+		dev->num_queues++;
+	}
+
+	if (!dev->num_queues) {
 		rc = -EIO;
-		goto err;
+		goto err_free_queues;
 	}
 
-	ndev->mtu = ntb_transport_max_size(dev->qp) - ETH_HLEN;
+	rc = netif_set_real_num_queues(ndev, dev->num_queues, dev->num_queues);
+	if (rc)
+		goto err_free_qps;
+
+	ndev->mtu = ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN;
 
 	rc = register_netdev(ndev);
 	if (rc)
-		goto err1;
+		goto err_free_qps;
 
 	dev_set_drvdata(client_dev, ndev);
-	dev_info(&pdev->dev, "%s created\n", ndev->name);
+	dev_info(&pdev->dev, "%s created with %u queue pairs\n",
+		 ndev->name, dev->num_queues);
 	return 0;
 
-err1:
-	ntb_transport_free_queue(dev->qp);
-err:
+err_free_qps:
+	for (q = 0; q < dev->num_queues; q++)
+		ntb_transport_free_queue(dev->queues[q].qp);
+
+err_free_queues:
+	kfree(dev->queues);
+
+err_free_netdev:
 	free_netdev(ndev);
 	return rc;
 }
@@ -464,9 +702,13 @@ static void ntb_netdev_remove(struct device *client_dev)
 {
 	struct net_device *ndev = dev_get_drvdata(client_dev);
 	struct ntb_netdev *dev = netdev_priv(ndev);
+	unsigned int q;
 
 	unregister_netdev(ndev);
-	ntb_transport_free_queue(dev->qp);
+	for (q = 0; q < dev->num_queues; q++)
+		ntb_transport_free_queue(dev->queues[q].qp);
+
+	kfree(dev->queues);
 	free_netdev(ndev);
 }
 
diff --git a/drivers/net/ovpn/crypto_aead.c b/drivers/net/ovpn/crypto_aead.c
index 77be0942a2691..8f07c418622b2 100644
--- a/drivers/net/ovpn/crypto_aead.c
+++ b/drivers/net/ovpn/crypto_aead.c
@@ -36,6 +36,104 @@ static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks)
 		crypto_aead_authsize(ks->encrypt);	/* Auth Tag */
 }
 
+/**
+ * ovpn_aead_crypto_tmp_size - compute the size of a temporary object containing
+ *			       an AEAD request structure with extra space for SG
+ *			       and IV.
+ * @tfm: the AEAD cipher handle
+ * @nfrags: the number of fragments in the skb
+ *
+ * This function calculates the size of a contiguous memory block that includes
+ * the initialization vector (IV), the AEAD request, and an array of scatterlist
+ * entries. For alignment considerations, the IV is placed first, followed by
+ * the request, and then the scatterlist.
+ * Additional alignment is applied according to the requirements of the
+ * underlying structures.
+ *
+ * Return: the size of the temporary memory that needs to be allocated
+ */
+static unsigned int ovpn_aead_crypto_tmp_size(struct crypto_aead *tfm,
+					      const unsigned int nfrags)
+{
+	unsigned int len = OVPN_NONCE_SIZE;
+
+	DEBUG_NET_WARN_ON_ONCE(crypto_aead_ivsize(tfm) != OVPN_NONCE_SIZE);
+
+	/* min size for a buffer of ivsize, aligned to alignmask */
+	len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1);
+	/* round up to the next multiple of the crypto ctx alignment */
+	len = ALIGN(len, crypto_tfm_ctx_alignment());
+
+	/* reserve space for the AEAD request */
+	len += sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
+	/* round up to the next multiple of the scatterlist alignment */
+	len = ALIGN(len, __alignof__(struct scatterlist));
+
+	/* add enough space for nfrags + 2 scatterlist entries */
+	len += array_size(sizeof(struct scatterlist), nfrags + 2);
+	return len;
+}
+
+/**
+ * ovpn_aead_crypto_tmp_iv - retrieve the pointer to the IV within a temporary
+ *			     buffer allocated using ovpn_aead_crypto_tmp_size
+ * @aead: the AEAD cipher handle
+ * @tmp: a pointer to the beginning of the temporary buffer
+ *
+ * This function retrieves a pointer to the initialization vector (IV) in the
+ * temporary buffer. If the AEAD cipher specifies an IV size, the pointer is
+ * adjusted using the AEAD's alignment mask to ensure proper alignment.
+ *
+ * Returns: a pointer to the IV within the temporary buffer
+ */
+static u8 *ovpn_aead_crypto_tmp_iv(struct crypto_aead *aead, void *tmp)
+{
+	return likely(crypto_aead_ivsize(aead)) ?
+		      PTR_ALIGN((u8 *)tmp, crypto_aead_alignmask(aead) + 1) :
+		      tmp;
+}
+
+/**
+ * ovpn_aead_crypto_tmp_req - retrieve the pointer to the AEAD request structure
+ *			      within a temporary buffer allocated using
+ *			      ovpn_aead_crypto_tmp_size
+ * @aead: the AEAD cipher handle
+ * @iv: a pointer to the initialization vector in the temporary buffer
+ *
+ * This function computes the location of the AEAD request structure that
+ * immediately follows the IV in the temporary buffer and it ensures the request
+ * is aligned to the crypto transform context alignment.
+ *
+ * Returns: a pointer to the AEAD request structure
+ */
+static struct aead_request *ovpn_aead_crypto_tmp_req(struct crypto_aead *aead,
+						     const u8 *iv)
+{
+	return (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
+				 crypto_tfm_ctx_alignment());
+}
+
+/**
+ * ovpn_aead_crypto_req_sg - locate the scatterlist following the AEAD request
+ *			     within a temporary buffer allocated using
+ *			     ovpn_aead_crypto_tmp_size
+ * @aead: the AEAD cipher handle
+ * @req: a pointer to the AEAD request structure in the temporary buffer
+ *
+ * This function computes the starting address of the scatterlist that is
+ * allocated immediately after the AEAD request structure. It aligns the pointer
+ * based on the alignment requirements of the scatterlist structure.
+ *
+ * Returns: a pointer to the scatterlist
+ */
+static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
+						   struct aead_request *req)
+{
+	return (void *)ALIGN((unsigned long)(req + 1) +
+			     crypto_aead_reqsize(aead),
+			     __alignof__(struct scatterlist));
+}
+
 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 		      struct sk_buff *skb)
 {
@@ -45,6 +143,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	struct scatterlist *sg;
 	int nfrags, ret;
 	u32 pktid, op;
+	void *tmp;
 	u8 *iv;
 
 	ovpn_skb_cb(skb)->peer = peer;
@@ -71,13 +170,17 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2)))
 		return -ENOSPC;
 
-	/* sg may be required by async crypto */
-	ovpn_skb_cb(skb)->sg = kmalloc(sizeof(*ovpn_skb_cb(skb)->sg) *
-				       (nfrags + 2), GFP_ATOMIC);
-	if (unlikely(!ovpn_skb_cb(skb)->sg))
+	/* allocate temporary memory for iv, sg and req */
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->encrypt, nfrags),
+		      GFP_ATOMIC);
+	if (unlikely(!tmp))
 		return -ENOMEM;
 
-	sg = ovpn_skb_cb(skb)->sg;
+	ovpn_skb_cb(skb)->crypto_tmp = tmp;
+
+	iv = ovpn_aead_crypto_tmp_iv(ks->encrypt, tmp);
+	req = ovpn_aead_crypto_tmp_req(ks->encrypt, iv);
+	sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);
 
 	/* sg table:
 	 * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+OVPN_NONCE_WIRE_SIZE),
@@ -105,13 +208,6 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	if (unlikely(ret < 0))
 		return ret;
 
-	/* iv may be required by async crypto */
-	ovpn_skb_cb(skb)->iv = kmalloc(OVPN_NONCE_SIZE, GFP_ATOMIC);
-	if (unlikely(!ovpn_skb_cb(skb)->iv))
-		return -ENOMEM;
-
-	iv = ovpn_skb_cb(skb)->iv;
-
 	/* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes
 	 * nonce
 	 */
@@ -122,7 +218,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	memcpy(skb->data, iv, OVPN_NONCE_WIRE_SIZE);
 
 	/* add packet op as head of additional data */
-	op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->id);
+	op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id);
 	__skb_push(skb, OVPN_OPCODE_SIZE);
 	BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
 	*((__force __be32 *)skb->data) = htonl(op);
@@ -130,12 +226,6 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	/* AEAD Additional data */
 	sg_set_buf(sg, skb->data, OVPN_AAD_SIZE);
 
-	req = aead_request_alloc(ks->encrypt, GFP_ATOMIC);
-	if (unlikely(!req))
-		return -ENOMEM;
-
-	ovpn_skb_cb(skb)->req = req;
-
 	/* setup async crypto operation */
 	aead_request_set_tfm(req, ks->encrypt);
 	aead_request_set_callback(req, 0, ovpn_encrypt_post, skb);
@@ -156,6 +246,7 @@ int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	struct aead_request *req;
 	struct sk_buff *trailer;
 	struct scatterlist *sg;
+	void *tmp;
 	u8 *iv;
 
 	payload_offset = OVPN_AAD_SIZE + tag_size;
@@ -184,13 +275,17 @@ int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2)))
 		return -ENOSPC;
 
-	/* sg may be required by async crypto */
-	ovpn_skb_cb(skb)->sg = kmalloc(sizeof(*ovpn_skb_cb(skb)->sg) *
-				       (nfrags + 2), GFP_ATOMIC);
-	if (unlikely(!ovpn_skb_cb(skb)->sg))
+	/* allocate temporary memory for iv, sg and req */
+	tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->decrypt, nfrags),
+		      GFP_ATOMIC);
+	if (unlikely(!tmp))
 		return -ENOMEM;
 
-	sg = ovpn_skb_cb(skb)->sg;
+	ovpn_skb_cb(skb)->crypto_tmp = tmp;
+
+	iv = ovpn_aead_crypto_tmp_iv(ks->decrypt, tmp);
+	req = ovpn_aead_crypto_tmp_req(ks->decrypt, iv);
+	sg = ovpn_aead_crypto_req_sg(ks->decrypt, req);
 
 	/* sg table:
 	 * 0: op, wire nonce (AD, len=OVPN_OPCODE_SIZE+OVPN_NONCE_WIRE_SIZE),
@@ -213,24 +308,11 @@ int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
 	/* append auth_tag onto scatterlist */
 	sg_set_buf(sg + ret + 1, skb->data + OVPN_AAD_SIZE, tag_size);
 
-	/* iv may be required by async crypto */
-	ovpn_skb_cb(skb)->iv = kmalloc(OVPN_NONCE_SIZE, GFP_ATOMIC);
-	if (unlikely(!ovpn_skb_cb(skb)->iv))
-		return -ENOMEM;
-
-	iv = ovpn_skb_cb(skb)->iv;
-
 	/* copy nonce into IV buffer */
 	memcpy(iv, skb->data + OVPN_OPCODE_SIZE, OVPN_NONCE_WIRE_SIZE);
 	memcpy(iv + OVPN_NONCE_WIRE_SIZE, ks->nonce_tail_recv,
 	       OVPN_NONCE_TAIL_SIZE);
 
-	req = aead_request_alloc(ks->decrypt, GFP_ATOMIC);
-	if (unlikely(!req))
-		return -ENOMEM;
-
-	ovpn_skb_cb(skb)->req = req;
-
 	/* setup async crypto operation */
 	aead_request_set_tfm(req, ks->decrypt);
 	aead_request_set_callback(req, 0, ovpn_decrypt_post, skb);
@@ -273,7 +355,11 @@ static struct crypto_aead *ovpn_aead_init(const char *title,
 		goto error;
 	}
 
-	/* basic AEAD assumption */
+	/* basic AEAD assumption
+	 * all current algorithms use OVPN_NONCE_SIZE.
+	 * ovpn_aead_crypto_tmp_size and ovpn_aead_encrypt/decrypt
+	 * expect this.
+	 */
 	if (crypto_aead_ivsize(aead) != OVPN_NONCE_SIZE) {
 		pr_err("%s IV size must be %d\n", title, OVPN_NONCE_SIZE);
 		ret = -EINVAL;
diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c
index 955c9a37e1f8d..db43a1f8a07a2 100644
--- a/drivers/net/ovpn/io.c
+++ b/drivers/net/ovpn/io.c
@@ -119,9 +119,7 @@ void ovpn_decrypt_post(void *data, int ret)
 	peer = ovpn_skb_cb(skb)->peer;
 
 	/* crypto is done, cleanup skb CB and its members */
-	kfree(ovpn_skb_cb(skb)->iv);
-	kfree(ovpn_skb_cb(skb)->sg);
-	aead_request_free(ovpn_skb_cb(skb)->req);
+	kfree(ovpn_skb_cb(skb)->crypto_tmp);
 
 	if (unlikely(ret < 0))
 		goto drop;
@@ -248,9 +246,7 @@ void ovpn_encrypt_post(void *data, int ret)
 	peer = ovpn_skb_cb(skb)->peer;
 
 	/* crypto is done, cleanup skb CB and its members */
-	kfree(ovpn_skb_cb(skb)->iv);
-	kfree(ovpn_skb_cb(skb)->sg);
-	aead_request_free(ovpn_skb_cb(skb)->req);
+	kfree(ovpn_skb_cb(skb)->crypto_tmp);
 
 	if (unlikely(ret == -ERANGE)) {
 		/* we ran out of IVs and we must kill the key as it can't be
diff --git a/drivers/net/ovpn/netlink-gen.c b/drivers/net/ovpn/netlink-gen.c
index ecbe9dcf4f7de..2147cec7c2c52 100644
--- a/drivers/net/ovpn/netlink-gen.c
+++ b/drivers/net/ovpn/netlink-gen.c
@@ -16,6 +16,10 @@ static const struct netlink_range_validation ovpn_a_peer_id_range = {
 	.max	= 16777215ULL,
 };
 
+static const struct netlink_range_validation ovpn_a_peer_tx_id_range = {
+	.max	= 16777215ULL,
+};
+
 static const struct netlink_range_validation ovpn_a_keyconf_peer_id_range = {
 	.max	= 16777215ULL,
 };
@@ -51,7 +55,7 @@ const struct nla_policy ovpn_keydir_nl_policy[OVPN_A_KEYDIR_NONCE_TAIL + 1] = {
 	[OVPN_A_KEYDIR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(OVPN_NONCE_TAIL_SIZE),
 };
 
-const struct nla_policy ovpn_peer_nl_policy[OVPN_A_PEER_LINK_TX_PACKETS + 1] = {
+const struct nla_policy ovpn_peer_nl_policy[OVPN_A_PEER_TX_ID + 1] = {
 	[OVPN_A_PEER_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_id_range),
 	[OVPN_A_PEER_REMOTE_IPV4] = { .type = NLA_BE32, },
 	[OVPN_A_PEER_REMOTE_IPV6] = NLA_POLICY_EXACT_LEN(16),
@@ -75,13 +79,14 @@ const struct nla_policy ovpn_peer_nl_policy[OVPN_A_PEER_LINK_TX_PACKETS + 1] = {
 	[OVPN_A_PEER_LINK_TX_BYTES] = { .type = NLA_UINT, },
 	[OVPN_A_PEER_LINK_RX_PACKETS] = { .type = NLA_UINT, },
 	[OVPN_A_PEER_LINK_TX_PACKETS] = { .type = NLA_UINT, },
+	[OVPN_A_PEER_TX_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_tx_id_range),
 };
 
 const struct nla_policy ovpn_peer_del_input_nl_policy[OVPN_A_PEER_ID + 1] = {
 	[OVPN_A_PEER_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_id_range),
 };
 
-const struct nla_policy ovpn_peer_new_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIMEOUT + 1] = {
+const struct nla_policy ovpn_peer_new_input_nl_policy[OVPN_A_PEER_TX_ID + 1] = {
 	[OVPN_A_PEER_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_id_range),
 	[OVPN_A_PEER_REMOTE_IPV4] = { .type = NLA_BE32, },
 	[OVPN_A_PEER_REMOTE_IPV6] = NLA_POLICY_EXACT_LEN(16),
@@ -94,9 +99,10 @@ const struct nla_policy ovpn_peer_new_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIME
 	[OVPN_A_PEER_LOCAL_IPV6] = NLA_POLICY_EXACT_LEN(16),
 	[OVPN_A_PEER_KEEPALIVE_INTERVAL] = { .type = NLA_U32, },
 	[OVPN_A_PEER_KEEPALIVE_TIMEOUT] = { .type = NLA_U32, },
+	[OVPN_A_PEER_TX_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_tx_id_range),
 };
 
-const struct nla_policy ovpn_peer_set_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIMEOUT + 1] = {
+const struct nla_policy ovpn_peer_set_input_nl_policy[OVPN_A_PEER_TX_ID + 1] = {
 	[OVPN_A_PEER_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_id_range),
 	[OVPN_A_PEER_REMOTE_IPV4] = { .type = NLA_BE32, },
 	[OVPN_A_PEER_REMOTE_IPV6] = NLA_POLICY_EXACT_LEN(16),
@@ -108,6 +114,7 @@ const struct nla_policy ovpn_peer_set_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIME
 	[OVPN_A_PEER_LOCAL_IPV6] = NLA_POLICY_EXACT_LEN(16),
 	[OVPN_A_PEER_KEEPALIVE_INTERVAL] = { .type = NLA_U32, },
 	[OVPN_A_PEER_KEEPALIVE_TIMEOUT] = { .type = NLA_U32, },
+	[OVPN_A_PEER_TX_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &ovpn_a_peer_tx_id_range),
 };
 
 /* OVPN_CMD_PEER_NEW - do */
diff --git a/drivers/net/ovpn/netlink-gen.h b/drivers/net/ovpn/netlink-gen.h
index b2301580770fd..67cd85f86173f 100644
--- a/drivers/net/ovpn/netlink-gen.h
+++ b/drivers/net/ovpn/netlink-gen.h
@@ -18,10 +18,10 @@ extern const struct nla_policy ovpn_keyconf_del_input_nl_policy[OVPN_A_KEYCONF_S
 extern const struct nla_policy ovpn_keyconf_get_nl_policy[OVPN_A_KEYCONF_CIPHER_ALG + 1];
 extern const struct nla_policy ovpn_keyconf_swap_input_nl_policy[OVPN_A_KEYCONF_PEER_ID + 1];
 extern const struct nla_policy ovpn_keydir_nl_policy[OVPN_A_KEYDIR_NONCE_TAIL + 1];
-extern const struct nla_policy ovpn_peer_nl_policy[OVPN_A_PEER_LINK_TX_PACKETS + 1];
+extern const struct nla_policy ovpn_peer_nl_policy[OVPN_A_PEER_TX_ID + 1];
 extern const struct nla_policy ovpn_peer_del_input_nl_policy[OVPN_A_PEER_ID + 1];
-extern const struct nla_policy ovpn_peer_new_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIMEOUT + 1];
-extern const struct nla_policy ovpn_peer_set_input_nl_policy[OVPN_A_PEER_KEEPALIVE_TIMEOUT + 1];
+extern const struct nla_policy ovpn_peer_new_input_nl_policy[OVPN_A_PEER_TX_ID + 1];
+extern const struct nla_policy ovpn_peer_set_input_nl_policy[OVPN_A_PEER_TX_ID + 1];
 
 int ovpn_nl_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
 		     struct genl_info *info);
diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c
index c7f3824376302..291e2e5bb450d 100644
--- a/drivers/net/ovpn/netlink.c
+++ b/drivers/net/ovpn/netlink.c
@@ -305,6 +305,12 @@ static int ovpn_nl_peer_modify(struct ovpn_peer *peer, struct genl_info *info,
 		dst_cache_reset(&peer->dst_cache);
 	}
 
+	/* In a multipeer-to-multipeer setup we may have asymmetric peer IDs,
+	 * that is peer->id might be different from peer->tx_id.
+	 */
+	if (attrs[OVPN_A_PEER_TX_ID])
+		peer->tx_id = nla_get_u32(attrs[OVPN_A_PEER_TX_ID]);
+
 	if (attrs[OVPN_A_PEER_VPN_IPV4]) {
 		rehash = true;
 		peer->vpn_addrs.ipv4.s_addr =
@@ -326,8 +332,8 @@ static int ovpn_nl_peer_modify(struct ovpn_peer *peer, struct genl_info *info,
 	}
 
 	netdev_dbg(peer->ovpn->dev,
-		   "modify peer id=%u endpoint=%pIScp VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n",
-		   peer->id, &ss,
+		   "modify peer id=%u tx_id=%u endpoint=%pIScp VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n",
+		   peer->id, peer->tx_id, &ss,
 		   &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6);
 
 	spin_unlock_bh(&peer->lock);
@@ -373,6 +379,7 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	peer_id = nla_get_u32(attrs[OVPN_A_PEER_ID]);
+
 	peer = ovpn_peer_new(ovpn, peer_id);
 	if (IS_ERR(peer)) {
 		NL_SET_ERR_MSG_FMT_MOD(info->extack,
@@ -572,6 +579,9 @@ static int ovpn_nl_send_peer(struct sk_buff *skb, const struct genl_info *info,
 	if (nla_put_u32(skb, OVPN_A_PEER_ID, peer->id))
 		goto err;
 
+	if (nla_put_u32(skb, OVPN_A_PEER_TX_ID, peer->tx_id))
+		goto err;
+
 	if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY))
 		if (nla_put_in_addr(skb, OVPN_A_PEER_VPN_IPV4,
 				    peer->vpn_addrs.ipv4.s_addr))
@@ -1061,8 +1071,8 @@ int ovpn_nl_key_get_doit(struct sk_buff *skb, struct genl_info *info)
 
 int ovpn_nl_key_swap_doit(struct sk_buff *skb, struct genl_info *info)
 {
+	struct nlattr *attrs[OVPN_A_KEYCONF_MAX + 1];
 	struct ovpn_priv *ovpn = info->user_ptr[0];
-	struct nlattr *attrs[OVPN_A_PEER_MAX + 1];
 	struct ovpn_peer *peer;
 	u32 peer_id;
 	int ret;
@@ -1203,6 +1213,88 @@ int ovpn_nl_peer_del_notify(struct ovpn_peer *peer)
 	return ret;
 }
 
+/**
+ * ovpn_nl_peer_float_notify - notify userspace about peer floating
+ * @peer: the floated peer
+ * @ss: sockaddr representing the new remote endpoint
+ *
+ * Return: 0 on success or a negative error code otherwise
+ */
+int ovpn_nl_peer_float_notify(struct ovpn_peer *peer,
+			      const struct sockaddr_storage *ss)
+{
+	struct ovpn_socket *sock;
+	struct sockaddr_in6 *sa6;
+	struct sockaddr_in *sa;
+	struct sk_buff *msg;
+	struct nlattr *attr;
+	int ret = -EMSGSIZE;
+	void *hdr;
+
+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put(msg, 0, 0, &ovpn_nl_family, 0,
+			  OVPN_CMD_PEER_FLOAT_NTF);
+	if (!hdr) {
+		ret = -ENOBUFS;
+		goto err_free_msg;
+	}
+
+	if (nla_put_u32(msg, OVPN_A_IFINDEX, peer->ovpn->dev->ifindex))
+		goto err_cancel_msg;
+
+	attr = nla_nest_start(msg, OVPN_A_PEER);
+	if (!attr)
+		goto err_cancel_msg;
+
+	if (nla_put_u32(msg, OVPN_A_PEER_ID, peer->id))
+		goto err_cancel_msg;
+
+	if (ss->ss_family == AF_INET) {
+		sa = (struct sockaddr_in *)ss;
+		if (nla_put_in_addr(msg, OVPN_A_PEER_REMOTE_IPV4,
+				    sa->sin_addr.s_addr) ||
+		    nla_put_net16(msg, OVPN_A_PEER_REMOTE_PORT, sa->sin_port))
+			goto err_cancel_msg;
+	} else if (ss->ss_family == AF_INET6) {
+		sa6 = (struct sockaddr_in6 *)ss;
+		if (nla_put_in6_addr(msg, OVPN_A_PEER_REMOTE_IPV6,
+				     &sa6->sin6_addr) ||
+		    nla_put_u32(msg, OVPN_A_PEER_REMOTE_IPV6_SCOPE_ID,
+				sa6->sin6_scope_id) ||
+		    nla_put_net16(msg, OVPN_A_PEER_REMOTE_PORT, sa6->sin6_port))
+			goto err_cancel_msg;
+	} else {
+		ret = -EAFNOSUPPORT;
+		goto err_cancel_msg;
+	}
+
+	nla_nest_end(msg, attr);
+	genlmsg_end(msg, hdr);
+
+	rcu_read_lock();
+	sock = rcu_dereference(peer->sock);
+	if (!sock) {
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+	genlmsg_multicast_netns(&ovpn_nl_family, sock_net(sock->sk), msg,
+				0, OVPN_NLGRP_PEERS, GFP_ATOMIC);
+	rcu_read_unlock();
+
+	return 0;
+
+err_unlock:
+	rcu_read_unlock();
+err_cancel_msg:
+	genlmsg_cancel(msg, hdr);
+err_free_msg:
+	nlmsg_free(msg);
+	return ret;
+}
+
 /**
  * ovpn_nl_key_swap_notify - notify userspace peer's key must be renewed
  * @peer: the peer whose key needs to be renewed
diff --git a/drivers/net/ovpn/netlink.h b/drivers/net/ovpn/netlink.h
index 8615dfc3c4720..11ee7c681885d 100644
--- a/drivers/net/ovpn/netlink.h
+++ b/drivers/net/ovpn/netlink.h
@@ -13,6 +13,8 @@ int ovpn_nl_register(void);
 void ovpn_nl_unregister(void);
 
 int ovpn_nl_peer_del_notify(struct ovpn_peer *peer);
+int ovpn_nl_peer_float_notify(struct ovpn_peer *peer,
+			      const struct sockaddr_storage *ss);
 int ovpn_nl_key_swap_notify(struct ovpn_peer *peer, u8 key_id);
 
 #endif /* _NET_OVPN_NETLINK_H_ */
diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c
index 3716a1d828015..26b55d813f0ee 100644
--- a/drivers/net/ovpn/peer.c
+++ b/drivers/net/ovpn/peer.c
@@ -99,7 +99,11 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
 	if (!peer)
 		return ERR_PTR(-ENOMEM);
 
+	/* in the default case TX and RX IDs are the same.
+	 * the user may set a different TX ID via netlink
+	 */
 	peer->id = id;
+	peer->tx_id = id;
 	peer->ovpn = ovpn;
 
 	peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY);
@@ -287,6 +291,8 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 
 	spin_unlock_bh(&peer->lock);
 
+	ovpn_nl_peer_float_notify(peer, &ss);
+
 	/* rehashing is required only in MP mode as P2P has one peer
 	 * only and thus there is no hashtable
 	 */
diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h
index a1423f2b09e06..328401570cba8 100644
--- a/drivers/net/ovpn/peer.h
+++ b/drivers/net/ovpn/peer.h
@@ -21,7 +21,8 @@
  * struct ovpn_peer - the main remote peer object
  * @ovpn: main openvpn instance this peer belongs to
  * @dev_tracker: reference tracker for associated dev
- * @id: unique identifier
+ * @id: unique identifier, used to match incoming packets
+ * @tx_id: identifier to be used in TX packets
  * @vpn_addrs: IP addresses assigned over the tunnel
  * @vpn_addrs.ipv4: IPv4 assigned to peer on the tunnel
  * @vpn_addrs.ipv6: IPv6 assigned to peer on the tunnel
@@ -64,6 +65,7 @@ struct ovpn_peer {
 	struct ovpn_priv *ovpn;
 	netdevice_tracker dev_tracker;
 	u32 id;
+	u32 tx_id;
 	struct {
 		struct in_addr ipv4;
 		struct in6_addr ipv6;
diff --git a/drivers/net/ovpn/pktid.c b/drivers/net/ovpn/pktid.c
index 2f29049897e39..f1c243b844635 100644
--- a/drivers/net/ovpn/pktid.c
+++ b/drivers/net/ovpn/pktid.c
@@ -65,7 +65,7 @@ int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time)
 	if (likely(pkt_id == pr->id + 1)) {
 		/* well-formed ID sequence (incremented by 1) */
 		pr->base = REPLAY_INDEX(pr->base, -1);
-		pr->history[pr->base / 8] |= (1 << (pr->base % 8));
+		__set_bit(pr->base, pr->history);
 		if (pr->extent < REPLAY_WINDOW_SIZE)
 			++pr->extent;
 		pr->id = pkt_id;
@@ -77,14 +77,14 @@ int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time)
 			unsigned int i;
 
 			pr->base = REPLAY_INDEX(pr->base, -delta);
-			pr->history[pr->base / 8] |= (1 << (pr->base % 8));
+			__set_bit(pr->base, pr->history);
 			pr->extent += delta;
 			if (pr->extent > REPLAY_WINDOW_SIZE)
 				pr->extent = REPLAY_WINDOW_SIZE;
 			for (i = 1; i < delta; ++i) {
 				unsigned int newb = REPLAY_INDEX(pr->base, i);
 
-				pr->history[newb / 8] &= ~BIT(newb % 8);
+				__clear_bit(newb, pr->history);
 			}
 		} else {
 			pr->base = 0;
@@ -103,14 +103,11 @@ int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time)
 			if (pkt_id > pr->id_floor) {
 				const unsigned int ri = REPLAY_INDEX(pr->base,
 								     delta);
-				u8 *p = &pr->history[ri / 8];
-				const u8 mask = (1 << (ri % 8));
 
-				if (*p & mask) {
+				if (__test_and_set_bit(ri, pr->history)) {
 					ret = -EINVAL;
 					goto out;
 				}
-				*p |= mask;
 			} else {
 				ret = -EINVAL;
 				goto out;
diff --git a/drivers/net/ovpn/pktid.h b/drivers/net/ovpn/pktid.h
index 0262d026d15e2..21845f353bc8c 100644
--- a/drivers/net/ovpn/pktid.h
+++ b/drivers/net/ovpn/pktid.h
@@ -34,7 +34,7 @@ struct ovpn_pktid_xmit {
  */
 struct ovpn_pktid_recv {
 	/* "sliding window" bitmask of recent packet IDs received */
-	u8 history[REPLAY_WINDOW_BYTES];
+	DECLARE_BITMAP(history, REPLAY_WINDOW_SIZE);
 	/* bit position of deque base in history */
 	unsigned int base;
 	/* extent (in bits) of deque in history */
diff --git a/drivers/net/ovpn/skb.h b/drivers/net/ovpn/skb.h
index 64430880f1dae..4fb7ea0254263 100644
--- a/drivers/net/ovpn/skb.h
+++ b/drivers/net/ovpn/skb.h
@@ -18,12 +18,19 @@
 #include <linux/socket.h>
 #include <linux/types.h>
 
+/**
+ * struct ovpn_cb - ovpn skb control block
+ * @peer: the peer this skb was received from/sent to
+ * @ks: the crypto key slot used to encrypt/decrypt this skb
+ * @crypto_tmp: pointer to temporary memory used for crypto operations
+ *		containing the IV, the scatter gather list and the aead request
+ * @payload_offset: offset in the skb where the payload starts
+ * @nosignal: whether this skb should be sent with the MSG_NOSIGNAL flag (TCP)
+ */
 struct ovpn_cb {
 	struct ovpn_peer *peer;
 	struct ovpn_crypto_key_slot *ks;
-	struct aead_request *req;
-	struct scatterlist *sg;
-	u8 *iv;
+	void *crypto_tmp;
 	unsigned int payload_offset;
 	bool nosignal;
 };
diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c
index 5499c1572f3e2..65054cc84be55 100644
--- a/drivers/net/ovpn/tcp.c
+++ b/drivers/net/ovpn/tcp.c
@@ -158,7 +158,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb)
 }
 
 static int ovpn_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-			    int flags, int *addr_len)
+			    int flags)
 {
 	int err = 0, off, copied = 0, ret;
 	struct ovpn_socket *sock;
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 7b73332a13d95..b5ee338b620d5 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -3,11 +3,6 @@
 # PHY Layer Configuration
 #
 
-config MDIO_BUS
-	tristate "MDIO bus consumer layer"
-	help
-	  MDIO bus consumer layer
-
 config PHYLINK
 	tristate
 	select PHYLIB
@@ -19,7 +14,6 @@ config PHYLINK
 
 menuconfig PHYLIB
 	tristate "PHY Device support and infrastructure"
-	select MDIO_BUS
 	help
 	  Ethernet controllers are usually attached to PHY
 	  devices.  This option provides infrastructure for
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index 3a34917adea72..05e4878af27ab 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -3,8 +3,8 @@
 
 libphy-y			:= phy.o phy-c45.o phy-core.o phy_device.o \
 				   linkmode.o phy_link_topology.o \
-				   phy_caps.o mdio_bus_provider.o phy_port.o
-mdio-bus-y			+= mdio_bus.o mdio_device.o
+				   phy_caps.o mdio_bus_provider.o phy_port.o \
+				   mdio_device.o
 
 ifdef CONFIG_PHYLIB
 # built-in whenever PHYLIB is built-in or module
@@ -15,7 +15,7 @@ libphy-$(CONFIG_SWPHY)		+= swphy.o
 libphy-$(CONFIG_LED_TRIGGER_PHY)	+= phy_led_triggers.o
 libphy-$(CONFIG_OPEN_ALLIANCE_HELPERS) += open_alliance_helpers.o
 
-obj-$(CONFIG_MDIO_BUS)		+= mdio-bus.o
+obj-$(CONFIG_PHYLIB)		+= mdio_bus.o
 obj-$(CONFIG_PHYLINK)		+= phylink.o
 obj-$(CONFIG_PHYLIB)		+= libphy.o
 obj-$(CONFIG_PHYLIB)		+= mdio_devres.o
diff --git a/drivers/net/phy/mdio-private.h b/drivers/net/phy/mdio-private.h
deleted file mode 100644
index 8bc6d9088af1d..0000000000000
--- a/drivers/net/phy/mdio-private.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef __MDIO_PRIVATE_H
-#define __MDIO_PRIVATE_H
-
-/* MDIO internal helpers
- */
-
-int mdio_device_register_reset(struct mdio_device *mdiodev);
-void mdio_device_unregister_reset(struct mdio_device *mdiodev);
-
-#endif /* __MDIO_PRIVATE_H */
diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
index afdf1ad6c0e61..00d0e4159e9bb 100644
--- a/drivers/net/phy/mdio_bus.c
+++ b/drivers/net/phy/mdio_bus.c
@@ -10,65 +10,23 @@
 
 #include <linux/device.h>
 #include <linux/errno.h>
-#include <linux/etherdevice.h>
 #include <linux/ethtool.h>
-#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/mii.h>
 #include <linux/mm.h>
 #include <linux/module.h>
-#include <linux/netdevice.h>
-#include <linux/of_device.h>
-#include <linux/of_mdio.h>
 #include <linux/phy.h>
-#include <linux/reset.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/uaccess.h>
 #include <linux/unistd.h>
-#include "mdio-private.h"
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/mdio.h>
 
-int mdiobus_register_device(struct mdio_device *mdiodev)
-{
-	int err;
-
-	if (mdiodev->bus->mdio_map[mdiodev->addr])
-		return -EBUSY;
-
-	if (mdiodev->flags & MDIO_DEVICE_FLAG_PHY) {
-		err = mdio_device_register_reset(mdiodev);
-		if (err)
-			return err;
-
-		/* Assert the reset signal */
-		mdio_device_reset(mdiodev, 1);
-	}
-
-	mdiodev->bus->mdio_map[mdiodev->addr] = mdiodev;
-
-	return 0;
-}
-EXPORT_SYMBOL(mdiobus_register_device);
-
-int mdiobus_unregister_device(struct mdio_device *mdiodev)
-{
-	if (mdiodev->bus->mdio_map[mdiodev->addr] != mdiodev)
-		return -EINVAL;
-
-	mdio_device_unregister_reset(mdiodev);
-
-	mdiodev->bus->mdio_map[mdiodev->addr] = NULL;
-
-	return 0;
-}
-EXPORT_SYMBOL(mdiobus_unregister_device);
-
 static struct mdio_device *mdiobus_find_device(struct mii_bus *bus, int addr)
 {
 	bool addr_valid = addr >= 0 && addr < ARRAY_SIZE(bus->mdio_map);
@@ -100,285 +58,6 @@ bool mdiobus_is_registered_device(struct mii_bus *bus, int addr)
 }
 EXPORT_SYMBOL(mdiobus_is_registered_device);
 
-/**
- * mdiobus_release - mii_bus device release callback
- * @d: the target struct device that contains the mii_bus
- *
- * Description: called when the last reference to an mii_bus is
- * dropped, to free the underlying memory.
- */
-static void mdiobus_release(struct device *d)
-{
-	struct mii_bus *bus = to_mii_bus(d);
-
-	WARN(bus->state != MDIOBUS_RELEASED &&
-	     /* for compatibility with error handling in drivers */
-	     bus->state != MDIOBUS_ALLOCATED,
-	     "%s: not in RELEASED or ALLOCATED state\n",
-	     bus->id);
-
-	if (bus->state == MDIOBUS_RELEASED)
-		fwnode_handle_put(dev_fwnode(d));
-
-	kfree(bus);
-}
-
-struct mdio_bus_stat_attr {
-	int addr;
-	unsigned int field_offset;
-};
-
-static u64 mdio_bus_get_stat(struct mdio_bus_stats *s, unsigned int offset)
-{
-	const char *p = (const char *)s + offset;
-	unsigned int start;
-	u64 val = 0;
-
-	do {
-		start = u64_stats_fetch_begin(&s->syncp);
-		val = u64_stats_read((const u64_stats_t *)p);
-	} while (u64_stats_fetch_retry(&s->syncp, start));
-
-	return val;
-}
-
-static u64 mdio_bus_get_global_stat(struct mii_bus *bus, unsigned int offset)
-{
-	unsigned int i;
-	u64 val = 0;
-
-	for (i = 0; i < PHY_MAX_ADDR; i++)
-		val += mdio_bus_get_stat(&bus->stats[i], offset);
-
-	return val;
-}
-
-static ssize_t mdio_bus_stat_field_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
-{
-	struct mii_bus *bus = to_mii_bus(dev);
-	struct mdio_bus_stat_attr *sattr;
-	struct dev_ext_attribute *eattr;
-	u64 val;
-
-	eattr = container_of(attr, struct dev_ext_attribute, attr);
-	sattr = eattr->var;
-
-	if (sattr->addr < 0)
-		val = mdio_bus_get_global_stat(bus, sattr->field_offset);
-	else
-		val = mdio_bus_get_stat(&bus->stats[sattr->addr],
-					sattr->field_offset);
-
-	return sysfs_emit(buf, "%llu\n", val);
-}
-
-static ssize_t mdio_bus_device_stat_field_show(struct device *dev,
-					       struct device_attribute *attr,
-					       char *buf)
-{
-	struct mdio_device *mdiodev = to_mdio_device(dev);
-	struct mii_bus *bus = mdiodev->bus;
-	struct mdio_bus_stat_attr *sattr;
-	struct dev_ext_attribute *eattr;
-	int addr = mdiodev->addr;
-	u64 val;
-
-	eattr = container_of(attr, struct dev_ext_attribute, attr);
-	sattr = eattr->var;
-
-	val = mdio_bus_get_stat(&bus->stats[addr], sattr->field_offset);
-
-	return sysfs_emit(buf, "%llu\n", val);
-}
-
-#define MDIO_BUS_STATS_ATTR_DECL(field, file)				\
-static struct dev_ext_attribute dev_attr_mdio_bus_##field = {		\
-	.attr = { .attr = { .name = file, .mode = 0444 },		\
-		     .show = mdio_bus_stat_field_show,			\
-	},								\
-	.var = &((struct mdio_bus_stat_attr) {				\
-		-1, offsetof(struct mdio_bus_stats, field)		\
-	}),								\
-};									\
-static struct dev_ext_attribute dev_attr_mdio_bus_device_##field = {	\
-	.attr = { .attr = { .name = file, .mode = 0444 },		\
-		     .show = mdio_bus_device_stat_field_show,		\
-	},								\
-	.var = &((struct mdio_bus_stat_attr) {				\
-		-1, offsetof(struct mdio_bus_stats, field)		\
-	}),								\
-};
-
-#define MDIO_BUS_STATS_ATTR(field)					\
-	MDIO_BUS_STATS_ATTR_DECL(field, __stringify(field))
-
-MDIO_BUS_STATS_ATTR(transfers);
-MDIO_BUS_STATS_ATTR(errors);
-MDIO_BUS_STATS_ATTR(writes);
-MDIO_BUS_STATS_ATTR(reads);
-
-#define MDIO_BUS_STATS_ADDR_ATTR_DECL(field, addr, file)		\
-static struct dev_ext_attribute dev_attr_mdio_bus_addr_##field##_##addr = { \
-	.attr = { .attr = { .name = file, .mode = 0444 },		\
-		     .show = mdio_bus_stat_field_show,			\
-	},								\
-	.var = &((struct mdio_bus_stat_attr) {				\
-		addr, offsetof(struct mdio_bus_stats, field)		\
-	}),								\
-}
-
-#define MDIO_BUS_STATS_ADDR_ATTR(field, addr)				\
-	MDIO_BUS_STATS_ADDR_ATTR_DECL(field, addr,			\
-				 __stringify(field) "_" __stringify(addr))
-
-#define MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(addr)			\
-	MDIO_BUS_STATS_ADDR_ATTR(transfers, addr);			\
-	MDIO_BUS_STATS_ADDR_ATTR(errors, addr);				\
-	MDIO_BUS_STATS_ADDR_ATTR(writes, addr);				\
-	MDIO_BUS_STATS_ADDR_ATTR(reads, addr)				\
-
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(0);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(1);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(2);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(3);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(4);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(5);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(6);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(7);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(8);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(9);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(10);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(11);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(12);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(13);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(14);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(15);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(16);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(17);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(18);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(19);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(20);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(21);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(22);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(23);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(24);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(25);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(26);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(27);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(28);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(29);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(30);
-MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(31);
-
-#define MDIO_BUS_STATS_ADDR_ATTR_GROUP(addr)				\
-	&dev_attr_mdio_bus_addr_transfers_##addr.attr.attr,		\
-	&dev_attr_mdio_bus_addr_errors_##addr.attr.attr,		\
-	&dev_attr_mdio_bus_addr_writes_##addr.attr.attr,		\
-	&dev_attr_mdio_bus_addr_reads_##addr.attr.attr			\
-
-static struct attribute *mdio_bus_statistics_attrs[] = {
-	&dev_attr_mdio_bus_transfers.attr.attr,
-	&dev_attr_mdio_bus_errors.attr.attr,
-	&dev_attr_mdio_bus_writes.attr.attr,
-	&dev_attr_mdio_bus_reads.attr.attr,
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(0),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(1),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(2),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(3),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(4),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(5),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(6),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(7),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(8),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(9),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(10),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(11),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(12),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(13),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(14),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(15),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(16),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(17),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(18),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(19),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(20),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(21),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(22),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(23),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(24),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(25),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(26),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(27),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(28),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(29),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(30),
-	MDIO_BUS_STATS_ADDR_ATTR_GROUP(31),
-	NULL,
-};
-
-static const struct attribute_group mdio_bus_statistics_group = {
-	.name	= "statistics",
-	.attrs	= mdio_bus_statistics_attrs,
-};
-
-static const struct attribute_group *mdio_bus_groups[] = {
-	&mdio_bus_statistics_group,
-	NULL,
-};
-
-const struct class mdio_bus_class = {
-	.name		= "mdio_bus",
-	.dev_release	= mdiobus_release,
-	.dev_groups	= mdio_bus_groups,
-};
-EXPORT_SYMBOL_GPL(mdio_bus_class);
-
-/**
- * mdio_find_bus - Given the name of a mdiobus, find the mii_bus.
- * @mdio_name: The name of a mdiobus.
- *
- * Return: a reference to the mii_bus, or NULL if none found. The
- * embedded struct device will have its reference count incremented,
- * and this must be put_deviced'ed once the bus is finished with.
- */
-struct mii_bus *mdio_find_bus(const char *mdio_name)
-{
-	struct device *d;
-
-	d = class_find_device_by_name(&mdio_bus_class, mdio_name);
-	return d ? to_mii_bus(d) : NULL;
-}
-EXPORT_SYMBOL(mdio_find_bus);
-
-#if IS_ENABLED(CONFIG_OF_MDIO)
-/**
- * of_mdio_find_bus - Given an mii_bus node, find the mii_bus.
- * @mdio_bus_np: Pointer to the mii_bus.
- *
- * Return: a reference to the mii_bus, or NULL if none found. The
- * embedded struct device will have its reference count incremented,
- * and this must be put once the bus is finished with.
- *
- * Because the association of a device_node and mii_bus is made via
- * of_mdiobus_register(), the mii_bus cannot be found before it is
- * registered with of_mdiobus_register().
- *
- */
-struct mii_bus *of_mdio_find_bus(struct device_node *mdio_bus_np)
-{
-	struct device *d;
-
-	if (!mdio_bus_np)
-		return NULL;
-
-	d = class_find_device_by_of_node(&mdio_bus_class, mdio_bus_np);
-	return d ? to_mii_bus(d) : NULL;
-}
-EXPORT_SYMBOL(of_mdio_find_bus);
-#endif
-
 static void mdiobus_stats_acct(struct mdio_bus_stats *stats, bool op, int ret)
 {
 	preempt_disable();
@@ -944,96 +623,5 @@ int mdiobus_c45_modify_changed(struct mii_bus *bus, int addr, int devad,
 }
 EXPORT_SYMBOL_GPL(mdiobus_c45_modify_changed);
 
-/**
- * mdio_bus_match - determine if given MDIO driver supports the given
- *		    MDIO device
- * @dev: target MDIO device
- * @drv: given MDIO driver
- *
- * Return: 1 if the driver supports the device, 0 otherwise
- *
- * Description: This may require calling the devices own match function,
- *   since different classes of MDIO devices have different match criteria.
- */
-static int mdio_bus_match(struct device *dev, const struct device_driver *drv)
-{
-	const struct mdio_driver *mdiodrv = to_mdio_driver(drv);
-	struct mdio_device *mdio = to_mdio_device(dev);
-
-	/* Both the driver and device must type-match */
-	if (!(mdiodrv->mdiodrv.flags & MDIO_DEVICE_IS_PHY) !=
-	    !(mdio->flags & MDIO_DEVICE_FLAG_PHY))
-		return 0;
-
-	if (of_driver_match_device(dev, drv))
-		return 1;
-
-	if (mdio->bus_match)
-		return mdio->bus_match(dev, drv);
-
-	return 0;
-}
-
-static int mdio_uevent(const struct device *dev, struct kobj_uevent_env *env)
-{
-	int rc;
-
-	/* Some devices have extra OF data and an OF-style MODALIAS */
-	rc = of_device_uevent_modalias(dev, env);
-	if (rc != -ENODEV)
-		return rc;
-
-	return 0;
-}
-
-static struct attribute *mdio_bus_device_statistics_attrs[] = {
-	&dev_attr_mdio_bus_device_transfers.attr.attr,
-	&dev_attr_mdio_bus_device_errors.attr.attr,
-	&dev_attr_mdio_bus_device_writes.attr.attr,
-	&dev_attr_mdio_bus_device_reads.attr.attr,
-	NULL,
-};
-
-static const struct attribute_group mdio_bus_device_statistics_group = {
-	.name	= "statistics",
-	.attrs	= mdio_bus_device_statistics_attrs,
-};
-
-static const struct attribute_group *mdio_bus_dev_groups[] = {
-	&mdio_bus_device_statistics_group,
-	NULL,
-};
-
-const struct bus_type mdio_bus_type = {
-	.name		= "mdio_bus",
-	.dev_groups	= mdio_bus_dev_groups,
-	.match		= mdio_bus_match,
-	.uevent		= mdio_uevent,
-};
-EXPORT_SYMBOL(mdio_bus_type);
-
-static int __init mdio_bus_init(void)
-{
-	int ret;
-
-	ret = class_register(&mdio_bus_class);
-	if (!ret) {
-		ret = bus_register(&mdio_bus_type);
-		if (ret)
-			class_unregister(&mdio_bus_class);
-	}
-
-	return ret;
-}
-
-static void __exit mdio_bus_exit(void)
-{
-	class_unregister(&mdio_bus_class);
-	bus_unregister(&mdio_bus_type);
-}
-
-subsys_initcall(mdio_bus_init);
-module_exit(mdio_bus_exit);
-
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("MDIO bus/device layer");
diff --git a/drivers/net/phy/mdio_bus_provider.c b/drivers/net/phy/mdio_bus_provider.c
index 4b0637405740e..041576eba47af 100644
--- a/drivers/net/phy/mdio_bus_provider.c
+++ b/drivers/net/phy/mdio_bus_provider.c
@@ -28,6 +28,281 @@
 #include <linux/string.h>
 #include <linux/uaccess.h>
 #include <linux/unistd.h>
+#include "phylib-internal.h"
+
+/**
+ * mdiobus_release - mii_bus device release callback
+ * @d: the target struct device that contains the mii_bus
+ *
+ * Description: called when the last reference to an mii_bus is
+ * dropped, to free the underlying memory.
+ */
+static void mdiobus_release(struct device *d)
+{
+	struct mii_bus *bus = to_mii_bus(d);
+
+	WARN(bus->state != MDIOBUS_RELEASED &&
+	     /* for compatibility with error handling in drivers */
+	     bus->state != MDIOBUS_ALLOCATED,
+	     "%s: not in RELEASED or ALLOCATED state\n",
+	     bus->id);
+
+	if (bus->state == MDIOBUS_RELEASED)
+		fwnode_handle_put(dev_fwnode(d));
+
+	kfree(bus);
+}
+
+struct mdio_bus_stat_attr {
+	struct device_attribute attr;
+	int address;
+	unsigned int field_offset;
+};
+
+static struct mdio_bus_stat_attr *to_sattr(struct device_attribute *attr)
+{
+	return container_of(attr, struct mdio_bus_stat_attr, attr);
+}
+
+static u64 mdio_bus_get_stat(struct mdio_bus_stats *s, unsigned int offset)
+{
+	const u64_stats_t *stats = (const void *)s + offset;
+	unsigned int start;
+	u64 val = 0;
+
+	do {
+		start = u64_stats_fetch_begin(&s->syncp);
+		val = u64_stats_read(stats);
+	} while (u64_stats_fetch_retry(&s->syncp, start));
+
+	return val;
+}
+
+static ssize_t mdio_bus_stat_field_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct mdio_bus_stat_attr *sattr = to_sattr(attr);
+	struct mii_bus *bus = to_mii_bus(dev);
+	u64 val = 0;
+
+	if (sattr->address < 0) {
+		/* get global stats */
+		for (int i = 0; i < PHY_MAX_ADDR; i++)
+			val += mdio_bus_get_stat(&bus->stats[i],
+						 sattr->field_offset);
+	} else {
+		val = mdio_bus_get_stat(&bus->stats[sattr->address],
+					sattr->field_offset);
+	}
+
+	return sysfs_emit(buf, "%llu\n", val);
+}
+
+static ssize_t mdio_bus_device_stat_field_show(struct device *dev,
+					       struct device_attribute *attr,
+					       char *buf)
+{
+	struct mdio_bus_stat_attr *sattr = to_sattr(attr);
+	struct mdio_device *mdiodev = to_mdio_device(dev);
+	struct mii_bus *bus = mdiodev->bus;
+	int addr = mdiodev->addr;
+	u64 val;
+
+	val = mdio_bus_get_stat(&bus->stats[addr], sattr->field_offset);
+
+	return sysfs_emit(buf, "%llu\n", val);
+}
+
+#define MDIO_BUS_STATS_ATTR(field)					\
+static const struct mdio_bus_stat_attr dev_attr_mdio_bus_##field = {	\
+	.attr = __ATTR(field, 0444, mdio_bus_stat_field_show, NULL),	\
+	.address = -1,							\
+	.field_offset = offsetof(struct mdio_bus_stats, field),		\
+};									\
+static const struct mdio_bus_stat_attr dev_attr_mdio_bus_device_##field = { \
+	.attr = __ATTR(field, 0444, mdio_bus_device_stat_field_show, NULL), \
+	.field_offset = offsetof(struct mdio_bus_stats, field),		\
+}
+
+MDIO_BUS_STATS_ATTR(transfers);
+MDIO_BUS_STATS_ATTR(errors);
+MDIO_BUS_STATS_ATTR(writes);
+MDIO_BUS_STATS_ATTR(reads);
+
+#define MDIO_BUS_STATS_ADDR_ATTR_DECL(field, addr, file)		\
+static const struct mdio_bus_stat_attr					\
+dev_attr_mdio_bus_addr_##field##_##addr = {				\
+	.attr = { .attr = { .name = file, .mode = 0444 },		\
+		     .show = mdio_bus_stat_field_show,			\
+	},								\
+	.address = addr,						\
+	.field_offset = offsetof(struct mdio_bus_stats, field),		\
+}
+
+#define MDIO_BUS_STATS_ADDR_ATTR(field, addr)				\
+	MDIO_BUS_STATS_ADDR_ATTR_DECL(field, addr,			\
+				 __stringify(field) "_" __stringify(addr))
+
+#define MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(addr)			\
+	MDIO_BUS_STATS_ADDR_ATTR(transfers, addr);			\
+	MDIO_BUS_STATS_ADDR_ATTR(errors, addr);				\
+	MDIO_BUS_STATS_ADDR_ATTR(writes, addr);				\
+	MDIO_BUS_STATS_ADDR_ATTR(reads, addr)				\
+
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(0);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(1);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(2);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(3);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(4);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(5);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(6);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(7);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(8);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(9);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(10);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(11);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(12);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(13);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(14);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(15);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(16);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(17);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(18);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(19);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(20);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(21);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(22);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(23);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(24);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(25);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(26);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(27);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(28);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(29);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(30);
+MDIO_BUS_STATS_ADDR_ATTR_GROUP_DECL(31);
+
+#define MDIO_BUS_STATS_ADDR_ATTR_GROUP(addr)				\
+	&(dev_attr_mdio_bus_addr_transfers_##addr).attr.attr,		\
+	&(dev_attr_mdio_bus_addr_errors_##addr).attr.attr,		\
+	&(dev_attr_mdio_bus_addr_writes_##addr).attr.attr,		\
+	&(dev_attr_mdio_bus_addr_reads_##addr).attr.attr			\
+
+static const struct attribute *const mdio_bus_statistics_attrs[] = {
+	&dev_attr_mdio_bus_transfers.attr.attr,
+	&dev_attr_mdio_bus_errors.attr.attr,
+	&dev_attr_mdio_bus_writes.attr.attr,
+	&dev_attr_mdio_bus_reads.attr.attr,
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(0),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(1),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(2),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(3),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(4),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(5),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(6),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(7),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(8),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(9),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(10),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(11),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(12),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(13),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(14),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(15),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(16),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(17),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(18),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(19),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(20),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(21),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(22),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(23),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(24),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(25),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(26),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(27),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(28),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(29),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(30),
+	MDIO_BUS_STATS_ADDR_ATTR_GROUP(31),
+	NULL,
+};
+
+static const struct attribute_group mdio_bus_statistics_group = {
+	.name		= "statistics",
+	.attrs_const	= mdio_bus_statistics_attrs,
+};
+__ATTRIBUTE_GROUPS(mdio_bus_statistics);
+
+const struct class mdio_bus_class = {
+	.name		= "mdio_bus",
+	.dev_release	= mdiobus_release,
+	.dev_groups	= mdio_bus_statistics_groups,
+};
+
+/**
+ * mdio_bus_match - determine if given MDIO driver supports the given
+ *		    MDIO device
+ * @dev: target MDIO device
+ * @drv: given MDIO driver
+ *
+ * Return: 1 if the driver supports the device, 0 otherwise
+ *
+ * Description: This may require calling the devices own match function,
+ *   since different classes of MDIO devices have different match criteria.
+ */
+static int mdio_bus_match(struct device *dev, const struct device_driver *drv)
+{
+	const struct mdio_driver *mdiodrv = to_mdio_driver(drv);
+	struct mdio_device *mdio = to_mdio_device(dev);
+
+	/* Both the driver and device must type-match */
+	if (!(mdiodrv->mdiodrv.flags & MDIO_DEVICE_IS_PHY) !=
+	    !(mdio->flags & MDIO_DEVICE_FLAG_PHY))
+		return 0;
+
+	if (of_driver_match_device(dev, drv))
+		return 1;
+
+	if (mdio->bus_match)
+		return mdio->bus_match(dev, drv);
+
+	return 0;
+}
+
+static int mdio_uevent(const struct device *dev, struct kobj_uevent_env *env)
+{
+	int rc;
+
+	/* Some devices have extra OF data and an OF-style MODALIAS */
+	rc = of_device_uevent_modalias(dev, env);
+	if (rc != -ENODEV)
+		return rc;
+
+	return 0;
+}
+
+static const struct attribute *const mdio_bus_device_statistics_attrs[] = {
+	&dev_attr_mdio_bus_device_transfers.attr.attr,
+	&dev_attr_mdio_bus_device_errors.attr.attr,
+	&dev_attr_mdio_bus_device_writes.attr.attr,
+	&dev_attr_mdio_bus_device_reads.attr.attr,
+	NULL,
+};
+
+static const struct attribute_group mdio_bus_device_statistics_group = {
+	.name		= "statistics",
+	.attrs_const	= mdio_bus_device_statistics_attrs,
+};
+__ATTRIBUTE_GROUPS(mdio_bus_device_statistics);
+
+const struct bus_type mdio_bus_type = {
+	.name		= "mdio_bus",
+	.dev_groups	= mdio_bus_device_statistics_groups,
+	.match		= mdio_bus_match,
+	.uevent		= mdio_uevent,
+};
 
 /**
  * mdiobus_alloc_size - allocate a mii_bus structure
@@ -440,3 +715,47 @@ void mdiobus_free(struct mii_bus *bus)
 	put_device(&bus->dev);
 }
 EXPORT_SYMBOL(mdiobus_free);
+
+/**
+ * mdio_find_bus - Given the name of a mdiobus, find the mii_bus.
+ * @mdio_name: The name of a mdiobus.
+ *
+ * Return: a reference to the mii_bus, or NULL if none found. The
+ * embedded struct device will have its reference count incremented,
+ * and this must be put_deviced'ed once the bus is finished with.
+ */
+struct mii_bus *mdio_find_bus(const char *mdio_name)
+{
+	struct device *d;
+
+	d = class_find_device_by_name(&mdio_bus_class, mdio_name);
+	return d ? to_mii_bus(d) : NULL;
+}
+EXPORT_SYMBOL(mdio_find_bus);
+
+#if IS_ENABLED(CONFIG_OF_MDIO)
+/**
+ * of_mdio_find_bus - Given an mii_bus node, find the mii_bus.
+ * @mdio_bus_np: Pointer to the mii_bus.
+ *
+ * Return: a reference to the mii_bus, or NULL if none found. The
+ * embedded struct device will have its reference count incremented,
+ * and this must be put once the bus is finished with.
+ *
+ * Because the association of a device_node and mii_bus is made via
+ * of_mdiobus_register(), the mii_bus cannot be found before it is
+ * registered with of_mdiobus_register().
+ *
+ */
+struct mii_bus *of_mdio_find_bus(struct device_node *mdio_bus_np)
+{
+	struct device *d;
+
+	if (!mdio_bus_np)
+		return NULL;
+
+	d = class_find_device_by_of_node(&mdio_bus_class, mdio_bus_np);
+	return d ? to_mii_bus(d) : NULL;
+}
+EXPORT_SYMBOL(of_mdio_find_bus);
+#endif
diff --git a/drivers/net/phy/mdio_device.c b/drivers/net/phy/mdio_device.c
index b8a5a18381965..56080d3d2d25e 100644
--- a/drivers/net/phy/mdio_device.c
+++ b/drivers/net/phy/mdio_device.c
@@ -22,7 +22,88 @@
 #include <linux/string.h>
 #include <linux/unistd.h>
 #include <linux/property.h>
-#include "mdio-private.h"
+#include "phylib-internal.h"
+
+/**
+ * mdio_device_register_reset - Read and initialize the reset properties of
+ *				an mdio device
+ * @mdiodev: mdio_device structure
+ *
+ * Return: Zero if successful, negative error code on failure
+ */
+static int mdio_device_register_reset(struct mdio_device *mdiodev)
+{
+	struct reset_control *reset;
+
+	/* Deassert the optional reset signal */
+	mdiodev->reset_gpio = gpiod_get_optional(&mdiodev->dev,
+						 "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(mdiodev->reset_gpio))
+		return PTR_ERR(mdiodev->reset_gpio);
+
+	if (mdiodev->reset_gpio)
+		gpiod_set_consumer_name(mdiodev->reset_gpio, "PHY reset");
+
+	reset = reset_control_get_optional_exclusive(&mdiodev->dev, "phy");
+	if (IS_ERR(reset)) {
+		gpiod_put(mdiodev->reset_gpio);
+		mdiodev->reset_gpio = NULL;
+		return PTR_ERR(reset);
+	}
+
+	mdiodev->reset_ctrl = reset;
+
+	/* Read optional firmware properties */
+	device_property_read_u32(&mdiodev->dev, "reset-assert-us",
+				 &mdiodev->reset_assert_delay);
+	device_property_read_u32(&mdiodev->dev, "reset-deassert-us",
+				 &mdiodev->reset_deassert_delay);
+
+	return 0;
+}
+
+/**
+ * mdio_device_unregister_reset - uninitialize the reset properties of
+ *				  an mdio device
+ * @mdiodev: mdio_device structure
+ */
+static void mdio_device_unregister_reset(struct mdio_device *mdiodev)
+{
+	gpiod_put(mdiodev->reset_gpio);
+	mdiodev->reset_gpio = NULL;
+	reset_control_put(mdiodev->reset_ctrl);
+	mdiodev->reset_ctrl = NULL;
+	mdiodev->reset_assert_delay = 0;
+	mdiodev->reset_deassert_delay = 0;
+}
+
+void mdio_device_reset(struct mdio_device *mdiodev, int value)
+{
+	unsigned int d;
+
+	if (!mdiodev->reset_gpio && !mdiodev->reset_ctrl)
+		return;
+
+	if (mdiodev->reset_state == value)
+		return;
+
+	if (mdiodev->reset_gpio)
+		gpiod_set_value_cansleep(mdiodev->reset_gpio, value);
+
+	if (mdiodev->reset_ctrl) {
+		if (value)
+			reset_control_assert(mdiodev->reset_ctrl);
+		else
+			reset_control_deassert(mdiodev->reset_ctrl);
+	}
+
+	d = value ? mdiodev->reset_assert_delay : mdiodev->reset_deassert_delay;
+	if (d)
+		fsleep(d);
+
+	mdiodev->reset_state = value;
+}
+EXPORT_SYMBOL(mdio_device_reset);
 
 void mdio_device_free(struct mdio_device *mdiodev)
 {
@@ -108,86 +189,38 @@ void mdio_device_remove(struct mdio_device *mdiodev)
 }
 EXPORT_SYMBOL(mdio_device_remove);
 
-/**
- * mdio_device_register_reset - Read and initialize the reset properties of
- *				an mdio device
- * @mdiodev: mdio_device structure
- *
- * Return: Zero if successful, negative error code on failure
- */
-int mdio_device_register_reset(struct mdio_device *mdiodev)
+int mdiobus_register_device(struct mdio_device *mdiodev)
 {
-	struct reset_control *reset;
+	int err;
 
-	/* Deassert the optional reset signal */
-	mdiodev->reset_gpio = gpiod_get_optional(&mdiodev->dev,
-						 "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(mdiodev->reset_gpio))
-		return PTR_ERR(mdiodev->reset_gpio);
+	if (mdiodev->bus->mdio_map[mdiodev->addr])
+		return -EBUSY;
 
-	if (mdiodev->reset_gpio)
-		gpiod_set_consumer_name(mdiodev->reset_gpio, "PHY reset");
+	if (mdiodev->flags & MDIO_DEVICE_FLAG_PHY) {
+		err = mdio_device_register_reset(mdiodev);
+		if (err)
+			return err;
 
-	reset = reset_control_get_optional_exclusive(&mdiodev->dev, "phy");
-	if (IS_ERR(reset)) {
-		gpiod_put(mdiodev->reset_gpio);
-		mdiodev->reset_gpio = NULL;
-		return PTR_ERR(reset);
+		/* Assert the reset signal */
+		mdio_device_reset(mdiodev, 1);
 	}
 
-	mdiodev->reset_ctrl = reset;
-
-	/* Read optional firmware properties */
-	device_property_read_u32(&mdiodev->dev, "reset-assert-us",
-				 &mdiodev->reset_assert_delay);
-	device_property_read_u32(&mdiodev->dev, "reset-deassert-us",
-				 &mdiodev->reset_deassert_delay);
+	mdiodev->bus->mdio_map[mdiodev->addr] = mdiodev;
 
 	return 0;
 }
 
-/**
- * mdio_device_unregister_reset - uninitialize the reset properties of
- *				  an mdio device
- * @mdiodev: mdio_device structure
- */
-void mdio_device_unregister_reset(struct mdio_device *mdiodev)
-{
-	gpiod_put(mdiodev->reset_gpio);
-	mdiodev->reset_gpio = NULL;
-	reset_control_put(mdiodev->reset_ctrl);
-	mdiodev->reset_ctrl = NULL;
-	mdiodev->reset_assert_delay = 0;
-	mdiodev->reset_deassert_delay = 0;
-}
-
-void mdio_device_reset(struct mdio_device *mdiodev, int value)
+int mdiobus_unregister_device(struct mdio_device *mdiodev)
 {
-	unsigned int d;
-
-	if (!mdiodev->reset_gpio && !mdiodev->reset_ctrl)
-		return;
+	if (mdiodev->bus->mdio_map[mdiodev->addr] != mdiodev)
+		return -EINVAL;
 
-	if (mdiodev->reset_state == value)
-		return;
-
-	if (mdiodev->reset_gpio)
-		gpiod_set_value_cansleep(mdiodev->reset_gpio, value);
-
-	if (mdiodev->reset_ctrl) {
-		if (value)
-			reset_control_assert(mdiodev->reset_ctrl);
-		else
-			reset_control_deassert(mdiodev->reset_ctrl);
-	}
+	mdio_device_unregister_reset(mdiodev);
 
-	d = value ? mdiodev->reset_assert_delay : mdiodev->reset_deassert_delay;
-	if (d)
-		fsleep(d);
+	mdiodev->bus->mdio_map[mdiodev->addr] = NULL;
 
-	mdiodev->reset_state = value;
+	return 0;
 }
-EXPORT_SYMBOL(mdio_device_reset);
 
 /**
  * mdio_probe - probe an MDIO device
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index c6b011a9d6369..2aa1dedd21b8e 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -6523,6 +6523,142 @@ static void lan8842_get_phy_stats(struct phy_device *phydev,
 	stats->tx_errors = priv->phy_stats.tx_errors;
 }
 
+#define LAN9645X_CTRL_REG			0x1f
+#define LAN9645X_CTRL_REG_SW_SOFT_RST		BIT(1)
+
+#define LAN9645X_DAC_ICAS_AMP_POWER_DOWN	0x47
+#define LAN9645X_BTRX_QBIAS_POWER_DOWN		0x46
+#define LAN9645X_TX_LOW_I_CH_CD_POWER_MGMT	0x45
+#define LAN9645X_TX_LOW_I_CH_B_POWER_MGMT	0x44
+#define LAN9645X_TX_LOW_I_CH_A_POWER_MGMT	0x43
+
+static const struct lanphy_reg_data force_dac_tx_errata[] = {
+	/* Force channel A/B/C/D TX on */
+	{ LAN8814_PAGE_POWER_REGS,
+	  LAN9645X_DAC_ICAS_AMP_POWER_DOWN,
+	  0 },
+	/* Force channel A/B/C/D QBias on */
+	{ LAN8814_PAGE_POWER_REGS,
+	  LAN9645X_BTRX_QBIAS_POWER_DOWN,
+	  0xaa },
+	/* Tx low I on channel C/D overwrite */
+	{ LAN8814_PAGE_POWER_REGS,
+	  LAN9645X_TX_LOW_I_CH_CD_POWER_MGMT,
+	  0xbfff },
+	/* Channel B low I overwrite */
+	{ LAN8814_PAGE_POWER_REGS,
+	  LAN9645X_TX_LOW_I_CH_B_POWER_MGMT,
+	  0xabbf },
+	/* Channel A low I overwrite */
+	{ LAN8814_PAGE_POWER_REGS,
+	  LAN9645X_TX_LOW_I_CH_A_POWER_MGMT,
+	  0xbd3f },
+};
+
+static int lan9645x_config_init(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Apply erratas from previous generations.  */
+	ret = lan8842_erratas(phydev);
+	if (ret < 0)
+		return ret;
+
+	/* Apply errata for an issue where bringing a port down, can cause a few
+	 * CRC errors for traffic flowing through adjacent ports.
+	 */
+	return lanphy_write_reg_data(phydev, force_dac_tx_errata,
+				     ARRAY_SIZE(force_dac_tx_errata));
+}
+
+static int lan9645x_suspend(struct phy_device *phydev)
+{
+	int ret, val;
+
+	/* Force link down before software power down (SPD), by doing software
+	 * soft reset. This resets the PHY, but keeps all register configuration
+	 * intact. The bit self clears.
+	 *
+	 * This is needed as a workaround for an issue where performing SPD on a
+	 * port can bring adjacent ports down, when there is traffic flowing
+	 * through the ports.
+	 */
+	ret = phy_set_bits(phydev, LAN9645X_CTRL_REG,
+			   LAN9645X_CTRL_REG_SW_SOFT_RST);
+	if (ret)
+		return ret;
+
+	ret = phy_read_poll_timeout(phydev, LAN9645X_CTRL_REG, val,
+				    !(val & LAN9645X_CTRL_REG_SW_SOFT_RST),
+				    3000, 100000, true);
+	if (ret)
+		return ret;
+
+	return genphy_suspend(phydev);
+}
+
+static int lan9645x_config_intr(struct phy_device *phydev)
+{
+	int err;
+
+	/* enable / disable interrupts */
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		/* This is an internal PHY of lan9645x and is not possible to
+		 * change the polarity of irq sources in the OIC (CPU_INTR)
+		 * found in lan9645x. Therefore change the polarity of the
+		 * interrupt in the PHY from being active low instead of active
+		 * high.
+		 */
+		err = phy_write(phydev, LAN8804_CONTROL,
+				LAN8804_CONTROL_INTR_POLARITY);
+		if (err)
+			return err;
+
+		/* By default interrupt buffer is open-drain in which case the
+		 * interrupt can be active only low. Therefore change the
+		 * interrupt buffer to be push-pull to be able to change
+		 * interrupt polarity.
+		 */
+		err = phy_write(phydev, LAN8804_OUTPUT_CONTROL,
+				LAN8804_OUTPUT_CONTROL_INTR_BUFFER);
+		if (err)
+			return err;
+
+		err = lan8814_ack_interrupt(phydev);
+		if (err)
+			return err;
+
+		err = phy_write(phydev, LAN8814_INTC,
+				LAN8814_INT_LINK | LAN8814_INT_FLF);
+	} else {
+		err = phy_write(phydev, LAN8814_INTC, 0);
+		if (err)
+			return err;
+
+		err = lan8814_ack_interrupt(phydev);
+	}
+
+	return err;
+}
+
+static irqreturn_t lan9645x_handle_interrupt(struct phy_device *phydev)
+{
+	int status;
+
+	status = phy_read(phydev, LAN8814_INTS);
+	if (status < 0) {
+		phy_error(phydev);
+		return IRQ_NONE;
+	}
+
+	if (status & (LAN8814_INT_LINK | LAN8814_INT_FLF)) {
+		phy_trigger_machine(phydev);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
 static struct phy_driver ksphy_driver[] = {
 {
 	PHY_ID_MATCH_MODEL(PHY_ID_KS8737),
@@ -6761,6 +6897,21 @@ static struct phy_driver ksphy_driver[] = {
 	.set_tunable	= lan8842_set_tunable,
 	.cable_test_start	= lan8814_cable_test_start,
 	.cable_test_get_status	= ksz886x_cable_test_get_status,
+}, {
+	PHY_ID_MATCH_MODEL(PHY_ID_LAN9645X),
+	.name		= "Microchip LAN9645X Gigabit PHY",
+	.config_init	= lan9645x_config_init,
+	.driver_data	= &ksz9021_type,
+	.probe		= kszphy_probe,
+	.soft_reset	= genphy_soft_reset,
+	.suspend	= lan9645x_suspend,
+	.resume		= genphy_resume,
+	.config_intr	= lan9645x_config_intr,
+	.handle_interrupt = lan9645x_handle_interrupt,
+	.get_tunable	= lan8842_get_tunable,
+	.set_tunable	= lan8842_set_tunable,
+	.get_phy_stats	= lan8842_get_phy_stats,
+	.update_stats	= lan8842_update_stats,
 }, {
 	PHY_ID_MATCH_MODEL(PHY_ID_KSZ9131),
 	.name		= "Microchip KSZ9131 Gigabit PHY",
@@ -6859,6 +7010,7 @@ static const struct mdio_device_id __maybe_unused micrel_tbl[] = {
 	{ PHY_ID_MATCH_MODEL(PHY_ID_LAN8804) },
 	{ PHY_ID_MATCH_MODEL(PHY_ID_LAN8841) },
 	{ PHY_ID_MATCH_MODEL(PHY_ID_LAN8842) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_LAN9645X) },
 	{ }
 };
 
diff --git a/drivers/net/phy/mxl-gpy.c b/drivers/net/phy/mxl-gpy.c
index 5f99766fb64c3..0da2d4e9d8548 100644
--- a/drivers/net/phy/mxl-gpy.c
+++ b/drivers/net/phy/mxl-gpy.c
@@ -38,12 +38,17 @@
 #define PHY_CTL1_MDICD		BIT(3)
 #define PHY_CTL1_MDIAB		BIT(2)
 #define PHY_CTL1_AMDIX		BIT(0)
+#define PHY_ERRCNT		0x15	/* Error counter */
 #define PHY_MIISTAT		0x18	/* MII state */
 #define PHY_IMASK		0x19	/* interrupt mask */
 #define PHY_ISTAT		0x1A	/* interrupt status */
 #define PHY_LED			0x1B	/* LEDs */
 #define PHY_FWV			0x1E	/* firmware version */
 
+#define PHY_ERRCNT_SEL		GENMASK(11, 8)
+#define PHY_ERRCNT_COUNT	GENMASK(7, 0)
+#define PHY_ERRCNT_SEL_RXERR	0
+
 #define PHY_MIISTAT_SPD_MASK	GENMASK(2, 0)
 #define PHY_MIISTAT_DPX		BIT(3)
 #define PHY_MIISTAT_LS		BIT(10)
@@ -134,6 +139,7 @@ struct gpy_priv {
 	u8 fw_major;
 	u8 fw_minor;
 	u32 wolopts;
+	u64 rx_errors;
 
 	/* It takes 3 seconds to fully switch out of loopback mode before
 	 * it can safely re-enter loopback mode. Record the time when
@@ -331,8 +337,9 @@ static int gpy_mbox_read(struct phy_device *phydev, u32 addr)
 
 static int gpy_config_init(struct phy_device *phydev)
 {
-	/* Nothing to configure. Configuration Requirement Placeholder */
-	return 0;
+	/* Count MDI RX errors (SymbolErrorDuringCarrier) */
+	return phy_write(phydev, PHY_ERRCNT,
+			 FIELD_PREP(PHY_ERRCNT_SEL, PHY_ERRCNT_SEL_RXERR));
 }
 
 static int gpy21x_config_init(struct phy_device *phydev)
@@ -1067,6 +1074,31 @@ static int gpy_config_inband(struct phy_device *phydev, unsigned int modes)
 			      VSPEC1_SGMII_ANEN_ANRS);
 }
 
+static int gpy_update_stats(struct phy_device *phydev)
+{
+	struct gpy_priv *priv = phydev->priv;
+	int ret;
+
+	/* PHY_ERRCNT: 8-bit read-clear counter, SEL set to RXERR */
+	ret = phy_read(phydev, PHY_ERRCNT);
+	if (ret < 0)
+		return ret;
+
+	priv->rx_errors += FIELD_GET(PHY_ERRCNT_COUNT, ret);
+
+	return 0;
+}
+
+static void gpy_get_phy_stats(struct phy_device *phydev,
+			      struct ethtool_eth_phy_stats *eth_stats,
+			      struct ethtool_phy_stats *stats)
+{
+	struct gpy_priv *priv = phydev->priv;
+
+	eth_stats->SymbolErrorDuringCarrier = priv->rx_errors;
+	stats->rx_errors = priv->rx_errors;
+}
+
 static struct phy_driver gpy_drivers[] = {
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY2xx),
@@ -1091,6 +1123,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		.phy_id		= PHY_ID_GPY115B,
@@ -1116,6 +1150,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY115C),
@@ -1140,6 +1176,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		.phy_id		= PHY_ID_GPY211B,
@@ -1165,6 +1203,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY211C),
@@ -1189,6 +1229,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		.phy_id		= PHY_ID_GPY212B,
@@ -1214,6 +1256,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY212C),
@@ -1238,6 +1282,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		.phy_id		= PHY_ID_GPY215B,
@@ -1263,6 +1309,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY215C),
@@ -1287,6 +1335,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY241B),
@@ -1306,6 +1356,8 @@ static struct phy_driver gpy_drivers[] = {
 		.set_wol	= gpy_set_wol,
 		.get_wol	= gpy_get_wol,
 		.set_loopback	= gpy_loopback,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY241BM),
@@ -1325,6 +1377,8 @@ static struct phy_driver gpy_drivers[] = {
 		.set_wol	= gpy_set_wol,
 		.get_wol	= gpy_get_wol,
 		.set_loopback	= gpy_loopback,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_GPY245B),
@@ -1344,6 +1398,8 @@ static struct phy_driver gpy_drivers[] = {
 		.set_wol	= gpy_set_wol,
 		.get_wol	= gpy_get_wol,
 		.set_loopback	= gpy_loopback,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_MXL86211C),
@@ -1368,6 +1424,8 @@ static struct phy_driver gpy_drivers[] = {
 		.led_hw_control_get = gpy_led_hw_control_get,
 		.led_hw_control_set = gpy_led_hw_control_set,
 		.led_polarity_set = gpy_led_polarity_set,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 	},
 	{
 		PHY_ID_MATCH_MODEL(PHY_ID_MXL86252),
@@ -1385,6 +1443,8 @@ static struct phy_driver gpy_drivers[] = {
 		.set_wol	= gpy_set_wol,
 		.get_wol	= gpy_get_wol,
 		.set_loopback	= gpy_loopback,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 		.led_brightness_set = gpy_led_brightness_set,
 		.led_hw_is_supported = gpy_led_hw_is_supported,
 		.led_hw_control_get = gpy_led_hw_control_get,
@@ -1407,6 +1467,8 @@ static struct phy_driver gpy_drivers[] = {
 		.set_wol	= gpy_set_wol,
 		.get_wol	= gpy_get_wol,
 		.set_loopback	= gpy_loopback,
+		.update_stats	= gpy_update_stats,
+		.get_phy_stats	= gpy_get_phy_stats,
 		.led_brightness_set = gpy_led_brightness_set,
 		.led_hw_is_supported = gpy_led_hw_is_supported,
 		.led_hw_control_get = gpy_led_hw_control_get,
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 3bd415710bf3f..0edff47478c20 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -1895,44 +1895,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 }
 EXPORT_SYMBOL(phy_attach_direct);
 
-/**
- * phy_attach - attach a network device to a particular PHY device
- * @dev: network device to attach
- * @bus_id: Bus ID of PHY device to attach
- * @interface: PHY device's interface
- *
- * Description: Same as phy_attach_direct() except that a PHY bus_id
- *     string is passed instead of a pointer to a struct phy_device.
- */
-struct phy_device *phy_attach(struct net_device *dev, const char *bus_id,
-			      phy_interface_t interface)
-{
-	struct phy_device *phydev;
-	struct device *d;
-	int rc;
-
-	if (!dev)
-		return ERR_PTR(-EINVAL);
-
-	/* Search the list of PHY devices on the mdio bus for the
-	 * PHY with the requested name
-	 */
-	d = bus_find_device_by_name(&mdio_bus_type, NULL, bus_id);
-	if (!d) {
-		pr_err("PHY %s not found\n", bus_id);
-		return ERR_PTR(-ENODEV);
-	}
-	phydev = to_phy_device(d);
-
-	rc = phy_attach_direct(dev, phydev, phydev->dev_flags, interface);
-	put_device(d);
-	if (rc)
-		return ERR_PTR(rc);
-
-	return phydev;
-}
-EXPORT_SYMBOL(phy_attach);
-
 /**
  * phy_detach - detach a PHY device from its network device
  * @phydev: target phy_device struct
@@ -3951,6 +3913,14 @@ static int __init phy_init(void)
 {
 	int rc;
 
+	rc = class_register(&mdio_bus_class);
+	if (rc)
+		return rc;
+
+	rc = bus_register(&mdio_bus_type);
+	if (rc)
+		goto err_class;
+
 	rtnl_lock();
 	ethtool_set_ethtool_phy_ops(&phy_ethtool_phy_ops);
 	phylib_register_stubs();
@@ -3979,6 +3949,9 @@ static int __init phy_init(void)
 	phylib_unregister_stubs();
 	ethtool_set_ethtool_phy_ops(NULL);
 	rtnl_unlock();
+	bus_unregister(&mdio_bus_type);
+err_class:
+	class_unregister(&mdio_bus_class);
 
 	return rc;
 }
@@ -3991,6 +3964,8 @@ static void __exit phy_exit(void)
 	phylib_unregister_stubs();
 	ethtool_set_ethtool_phy_ops(NULL);
 	rtnl_unlock();
+	bus_unregister(&mdio_bus_type);
+	class_unregister(&mdio_bus_class);
 }
 
 subsys_initcall(phy_init);
diff --git a/drivers/net/phy/phylib-internal.h b/drivers/net/phy/phylib-internal.h
index dc9592c6bb8e7..664ed7faa5184 100644
--- a/drivers/net/phy/phylib-internal.h
+++ b/drivers/net/phy/phylib-internal.h
@@ -6,8 +6,12 @@
 #ifndef __PHYLIB_INTERNAL_H
 #define __PHYLIB_INTERNAL_H
 
+struct mdio_device;
 struct phy_device;
 
+extern const struct bus_type mdio_bus_type;
+extern const struct class mdio_bus_class;
+
 /*
  * phy_supported_speeds - return all speeds currently supported by a PHY device
  */
@@ -20,6 +24,9 @@ void of_set_phy_timing_role(struct phy_device *phydev);
 int phy_speed_down_core(struct phy_device *phydev);
 void phy_check_downshift(struct phy_device *phydev);
 
+int mdiobus_register_device(struct mdio_device *mdiodev);
+int mdiobus_unregister_device(struct mdio_device *mdiodev);
+
 int genphy_c45_read_eee_adv(struct phy_device *phydev, unsigned long *adv);
 
 #endif /* __PHYLIB_INTERNAL_H */
diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c
index 75565fbdbf6d4..530b4e26d16ef 100644
--- a/drivers/net/phy/realtek/realtek_main.c
+++ b/drivers/net/phy/realtek/realtek_main.c
@@ -2243,6 +2243,9 @@ static struct phy_driver realtek_drvs[] = {
 		.read_page	= rtl821x_read_page,
 		.write_page	= rtl821x_write_page,
 		.flags		= PHY_ALWAYS_CALL_SUSPEND,
+		.led_hw_is_supported = rtl8211x_led_hw_is_supported,
+		.led_hw_control_get = rtl8211f_led_hw_control_get,
+		.led_hw_control_set = rtl8211f_led_hw_control_set,
 	}, {
 		.name		= "Generic FE-GE Realtek PHY",
 		.match_phy_device = rtlgen_match_phy_device,
diff --git a/drivers/net/phy/vitesse.c b/drivers/net/phy/vitesse.c
index b1b7bbba284e6..1a430e832f66e 100644
--- a/drivers/net/phy/vitesse.c
+++ b/drivers/net/phy/vitesse.c
@@ -62,6 +62,13 @@
 /* Vitesse Extended Page Access Register */
 #define MII_VSC82X4_EXT_PAGE_ACCESS	0x1f
 
+/* Vitesse VSC8662 extended control register */
+#define VSC8662_EXT_CON1		0x17
+#define VSC8662_EXT_CON_MAC_AN		BIT(13)
+
+#define VSC8662_MAC_AN			0x1b
+#define VSC8662_MAC_AN_BYPASS		BIT(13)
+
 /* Vitesse VSC73XX Extended Control Register */
 #define MII_VSC73XX_PHY_CTRL_EXT3		0x14
 
@@ -140,6 +147,38 @@ static int vsc824x_config_init(struct phy_device *phydev)
 	return err;
 }
 
+static unsigned int vsc8662_inband_caps(struct phy_device *phydev,
+					phy_interface_t interface)
+{
+	if (interface == PHY_INTERFACE_MODE_SGMII)
+		return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE |
+		       LINK_INBAND_BYPASS;
+
+	return 0;
+}
+
+static int vsc8662_config_inband(struct phy_device *phydev, unsigned int modes)
+{
+	u16 mask, set;
+	int ret;
+
+	mask = VSC8662_MAC_AN_BYPASS;
+	set = modes & LINK_INBAND_BYPASS ? mask : 0;
+	ret = phy_modify(phydev, VSC8662_MAC_AN, mask, set);
+	if (ret < 0)
+		return ret;
+
+	mask = VSC8662_EXT_CON_MAC_AN;
+	set = modes & (LINK_INBAND_ENABLE | LINK_INBAND_BYPASS) ? mask : 0;
+
+	ret = phy_modify_changed(phydev, VSC8662_EXT_CON1, mask, set);
+	if (ret <= 0)
+		return ret;
+
+	/* We need to soft-reset the PHY when changing VSC8662_EXT_CON_MAC_AN */
+	return genphy_soft_reset(phydev);
+}
+
 #define VSC73XX_EXT_PAGE_ACCESS 0x1f
 
 static int vsc73xx_read_page(struct phy_device *phydev)
@@ -649,6 +688,8 @@ static struct phy_driver vsc82xx_driver[] = {
 	.phy_id_mask    = 0x000ffff0,
 	/* PHY_GBIT_FEATURES */
 	.config_init    = &vsc824x_config_init,
+	.inband_caps	= vsc8662_inband_caps,
+	.config_inband	= vsc8662_config_inband,
 	.config_aneg    = &vsc82x4_config_aneg,
 	.config_intr    = &vsc82xx_config_intr,
 	.handle_interrupt = &vsc82xx_handle_interrupt,
diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c
index b4cf2d09c6bd1..93a7b0f6c4e7e 100644
--- a/drivers/net/ppp/ppp_async.c
+++ b/drivers/net/ppp/ppp_async.c
@@ -491,7 +491,7 @@ static void ppp_async_process(struct tasklet_struct *t)
 	/* process received packets */
 	while ((skb = skb_dequeue(&ap->rqueue)) != NULL) {
 		if (skb->cb[0])
-			ppp_input_error(&ap->chan, 0);
+			ppp_input_error(&ap->chan);
 		ppp_input(&ap->chan, skb);
 	}
 
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index e9b41777be809..cb29a6968c633 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -134,7 +134,6 @@ struct ppp {
 	int		debug;		/* debug flags 70 */
 	struct slcompress *vj;		/* state for VJ header compression */
 	enum NPmode	npmode[NUM_NP];	/* what to do with each net proto 78 */
-	struct sk_buff	*xmit_pending;	/* a packet ready to go out 88 */
 	struct compressor *xcomp;	/* transmit packet compressor 8c */
 	void		*xc_state;	/* its internal state 90 */
 	struct compressor *rcomp;	/* receive decompressor 94 */
@@ -264,8 +263,8 @@ struct ppp_net {
 static int ppp_unattached_ioctl(struct net *net, struct ppp_file *pf,
 			struct file *file, unsigned int cmd, unsigned long arg);
 static void ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb);
-static void ppp_send_frame(struct ppp *ppp, struct sk_buff *skb);
-static void ppp_push(struct ppp *ppp);
+static int ppp_prepare_tx_skb(struct ppp *ppp, struct sk_buff **pskb);
+static int ppp_push(struct ppp *ppp, struct sk_buff *skb);
 static void ppp_channel_push(struct channel *pch);
 static void ppp_receive_frame(struct ppp *ppp, struct sk_buff *skb,
 			      struct channel *pch);
@@ -1603,10 +1602,7 @@ static int ppp_fill_forward_path(struct net_device_path_ctx *ctx,
 	if (!pch)
 		return -ENODEV;
 
-	chan = READ_ONCE(pch->chan);
-	if (!chan)
-		return -ENODEV;
-
+	chan = pch->chan;
 	if (!chan->ops->fill_forward_path)
 		return -EOPNOTSUPP;
 
@@ -1651,26 +1647,44 @@ static void ppp_setup(struct net_device *dev)
  */
 
 /* Called to do any work queued up on the transmit side that can now be done */
+static void ppp_xmit_flush(struct ppp *ppp)
+{
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&ppp->file.xq))) {
+		if (unlikely(!ppp_push(ppp, skb))) {
+			skb_queue_head(&ppp->file.xq, skb);
+			return;
+		}
+	}
+	/* If there's no work left to do, tell the core net code that we can
+	 * accept some more.
+	 */
+	netif_wake_queue(ppp->dev);
+}
+
 static void __ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb)
 {
 	ppp_xmit_lock(ppp);
-	if (!ppp->closing) {
-		ppp_push(ppp);
-
-		if (skb)
+	if (unlikely(ppp->closing)) {
+		kfree_skb(skb);
+		goto out;
+	}
+	if (unlikely(ppp_prepare_tx_skb(ppp, &skb)))
+		goto out;
+	/* Fastpath: No backlog, just send the new skb. */
+	if (likely(skb_queue_empty(&ppp->file.xq))) {
+		if (unlikely(!ppp_push(ppp, skb))) {
 			skb_queue_tail(&ppp->file.xq, skb);
-		while (!ppp->xmit_pending &&
-		       (skb = skb_dequeue(&ppp->file.xq)))
-			ppp_send_frame(ppp, skb);
-		/* If there's no work left to do, tell the core net
-		   code that we can accept some more. */
-		if (!ppp->xmit_pending && !skb_peek(&ppp->file.xq))
-			netif_wake_queue(ppp->dev);
-		else
 			netif_stop_queue(ppp->dev);
-	} else {
-		kfree_skb(skb);
+		}
+		goto out;
 	}
+
+	/* Slowpath: Enqueue the new skb and process backlog */
+	skb_queue_tail(&ppp->file.xq, skb);
+	ppp_xmit_flush(ppp);
+out:
 	ppp_xmit_unlock(ppp);
 }
 
@@ -1757,13 +1771,15 @@ pad_compress_skb(struct ppp *ppp, struct sk_buff *skb)
 }
 
 /*
- * Compress and send a frame.
- * The caller should have locked the xmit path,
- * and xmit_pending should be 0.
+ * Compress and prepare to send a frame.
+ * The caller should have locked the xmit path.
+ * Returns 1 if the skb was consumed, 0 if it can be passed to ppp_push().
+ * @pskb is updated if a compressor is in use.
  */
-static void
-ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
+static int
+ppp_prepare_tx_skb(struct ppp *ppp, struct sk_buff **pskb)
 {
+	struct sk_buff *skb = *pskb;
 	int proto = PPP_PROTO(skb);
 	struct sk_buff *new_skb;
 	int len;
@@ -1784,7 +1800,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
 					      "PPP: outbound frame "
 					      "not passed\n");
 			kfree_skb(skb);
-			return;
+			return 1;
 		}
 		/* if this packet passes the active filter, record the time */
 		if (!(ppp->active_filter &&
@@ -1832,6 +1848,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
 			}
 			consume_skb(skb);
 			skb = new_skb;
+			*pskb = skb;
 			cp = skb_put(skb, len + 2);
 			cp[0] = 0;
 			cp[1] = proto;
@@ -1858,6 +1875,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
 		if (!new_skb)
 			goto drop;
 		skb = new_skb;
+		*pskb = skb;
 	}
 
 	/*
@@ -1869,74 +1887,69 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb)
 			goto drop;
 		skb_queue_tail(&ppp->file.rq, skb);
 		wake_up_interruptible(&ppp->file.rwait);
-		return;
+		return 1;
 	}
 
-	ppp->xmit_pending = skb;
-	ppp_push(ppp);
-	return;
+	return 0;
 
  drop:
 	kfree_skb(skb);
 	++ppp->dev->stats.tx_errors;
+	return 1;
 }
 
 /*
- * Try to send the frame in xmit_pending.
+ * Try to send the frame.
  * The caller should have the xmit path locked.
+ * Returns 1 if the skb was consumed, 0 if not.
  */
-static void
-ppp_push(struct ppp *ppp)
+static int
+ppp_push(struct ppp *ppp, struct sk_buff *skb)
 {
 	struct list_head *list;
 	struct channel *pch;
-	struct sk_buff *skb = ppp->xmit_pending;
-
-	if (!skb)
-		return;
 
 	list = &ppp->channels;
 	if (list_empty(list)) {
 		/* nowhere to send the packet, just drop it */
-		ppp->xmit_pending = NULL;
 		kfree_skb(skb);
-		return;
+		return 1;
 	}
 
 	if ((ppp->flags & SC_MULTILINK) == 0) {
 		struct ppp_channel *chan;
+		int ret;
 		/* not doing multilink: send it down the first channel */
 		list = list->next;
 		pch = list_entry(list, struct channel, clist);
 
 		spin_lock(&pch->downl);
 		chan = pch->chan;
-		if (unlikely(!chan || (!chan->direct_xmit && skb_linearize(skb)))) {
-			/* channel got unregistered, or it requires a linear
-			 * skb but linearization failed
+		if (unlikely(!chan->direct_xmit && skb_linearize(skb))) {
+			/* channel requires a linear skb but linearization
+			 * failed
 			 */
 			kfree_skb(skb);
-			ppp->xmit_pending = NULL;
+			ret = 1;
 			goto out;
 		}
 
-		if (chan->ops->start_xmit(chan, skb))
-			ppp->xmit_pending = NULL;
+		ret = chan->ops->start_xmit(chan, skb);
 
 out:
 		spin_unlock(&pch->downl);
-		return;
+		return ret;
 	}
 
 #ifdef CONFIG_PPP_MULTILINK
 	/* Multilink: fragment the packet over as many links
 	   as can take the packet at the moment. */
 	if (!ppp_mp_explode(ppp, skb))
-		return;
+		return 0;
 #endif /* CONFIG_PPP_MULTILINK */
 
-	ppp->xmit_pending = NULL;
 	kfree_skb(skb);
+	return 1;
 }
 
 #ifdef CONFIG_PPP_MULTILINK
@@ -1975,28 +1988,23 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
 	hdrlen = (ppp->flags & SC_MP_XSHORTSEQ)? MPHDRLEN_SSN: MPHDRLEN;
 	i = 0;
 	list_for_each_entry(pch, &ppp->channels, clist) {
-		if (pch->chan) {
-			pch->avail = 1;
-			navail++;
-			pch->speed = pch->chan->speed;
-		} else {
-			pch->avail = 0;
-		}
-		if (pch->avail) {
-			if (skb_queue_empty(&pch->file.xq) ||
-				!pch->had_frag) {
-					if (pch->speed == 0)
-						nzero++;
-					else
-						totspeed += pch->speed;
-
-					pch->avail = 2;
-					++nfree;
-					++totfree;
-				}
-			if (!pch->had_frag && i < ppp->nxchan)
-				ppp->nxchan = i;
+		pch->avail = 1;
+		navail++;
+		pch->speed = pch->chan->speed;
+
+		if (skb_queue_empty(&pch->file.xq) || !pch->had_frag) {
+			if (pch->speed == 0)
+				nzero++;
+			else
+				totspeed += pch->speed;
+
+			pch->avail = 2;
+			++nfree;
+			++totfree;
 		}
+		if (!pch->had_frag && i < ppp->nxchan)
+			ppp->nxchan = i;
+
 		++i;
 	}
 	/*
@@ -2005,7 +2013,7 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
 	 * performance if we have a lot of channels.
 	 */
 	if (nfree == 0 || nfree < navail / 2)
-		return 0; /* can't take now, leave it in xmit_pending */
+		return 0; /* can't take now, leave it in transmit queue */
 
 	/* Do protocol field compression */
 	if (skb_linearize(skb))
@@ -2055,25 +2063,7 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
 			pch->avail = 1;
 		}
 
-		/* check the channel's mtu and whether it is still attached. */
 		spin_lock(&pch->downl);
-		if (pch->chan == NULL) {
-			/* can't use this channel, it's being deregistered */
-			if (pch->speed == 0)
-				nzero--;
-			else
-				totspeed -= pch->speed;
-
-			spin_unlock(&pch->downl);
-			pch->avail = 0;
-			totlen = len;
-			totfree--;
-			nfree--;
-			if (--navail == 0)
-				break;
-			continue;
-		}
-
 		/*
 		*if the channel speed is not set divide
 		*the packet evenly among the free channels;
@@ -2199,8 +2189,12 @@ static void __ppp_channel_push(struct channel *pch, struct ppp *ppp)
 	spin_unlock(&pch->downl);
 	/* see if there is anything from the attached unit to be sent */
 	if (skb_queue_empty(&pch->file.xq)) {
-		if (ppp)
-			__ppp_xmit_process(ppp, NULL);
+		if (ppp) {
+			ppp_xmit_lock(ppp);
+			if (!ppp->closing)
+				ppp_xmit_flush(ppp);
+			ppp_xmit_unlock(ppp);
+		}
 	}
 }
 
@@ -2363,12 +2357,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb)
 	rcu_read_unlock_bh();
 }
 
-/* Put a 0-length skb in the receive queue as an error indication */
 void
-ppp_input_error(struct ppp_channel *chan, int code)
+ppp_input_error(struct ppp_channel *chan)
 {
 	struct channel *pch = chan->ppp;
-	struct sk_buff *skb;
 	struct ppp *ppp;
 
 	if (!pch)
@@ -2377,12 +2369,9 @@ ppp_input_error(struct ppp_channel *chan, int code)
 	rcu_read_lock_bh();
 	ppp = rcu_dereference_bh(pch->ppp);
 	if (ppp) {
-		skb = alloc_skb(0, GFP_ATOMIC);
-		if (skb) {
-			skb->len = 0;		/* probably unnecessary */
-			skb->cb[0] = code;
-			ppp_do_recv(ppp, skb, pch);
-		}
+		ppp_recv_lock(ppp);
+		ppp_receive_error(ppp);
+		ppp_recv_unlock(ppp);
 	}
 	rcu_read_unlock_bh();
 }
@@ -2394,20 +2383,14 @@ ppp_input_error(struct ppp_channel *chan, int code)
 static void
 ppp_receive_frame(struct ppp *ppp, struct sk_buff *skb, struct channel *pch)
 {
-	/* note: a 0-length skb is used as an error indication */
-	if (skb->len > 0) {
-		skb_checksum_complete_unset(skb);
+	skb_checksum_complete_unset(skb);
 #ifdef CONFIG_PPP_MULTILINK
-		/* XXX do channel-level decompression here */
-		if (PPP_PROTO(skb) == PPP_MP)
-			ppp_receive_mp_frame(ppp, skb, pch);
-		else
+	/* XXX do channel-level decompression here */
+	if (PPP_PROTO(skb) == PPP_MP)
+		ppp_receive_mp_frame(ppp, skb, pch);
+	else
 #endif /* CONFIG_PPP_MULTILINK */
-			ppp_receive_nonmp_frame(ppp, skb);
-	} else {
-		kfree_skb(skb);
-		ppp_receive_error(ppp);
-	}
+		ppp_receive_nonmp_frame(ppp, skb);
 }
 
 static void
@@ -2986,6 +2969,7 @@ int ppp_unit_number(struct ppp_channel *chan)
 
 /*
  * Return the PPP device interface name of a channel.
+ * Caller must hold RCU read lock.
  */
 char *ppp_dev_name(struct ppp_channel *chan)
 {
@@ -2994,11 +2978,9 @@ char *ppp_dev_name(struct ppp_channel *chan)
 	struct ppp *ppp;
 
 	if (pch) {
-		rcu_read_lock();
 		ppp = rcu_dereference(pch->ppp);
 		if (ppp && ppp->dev)
 			name = ppp->dev->name;
-		rcu_read_unlock();
 	}
 	return name;
 }
@@ -3023,12 +3005,12 @@ ppp_unregister_channel(struct ppp_channel *chan)
 	 * This ensures that we have returned from any calls into
 	 * the channel's start_xmit or ioctl routine before we proceed.
 	 */
+	ppp_disconnect_channel(pch);
 	down_write(&pch->chan_sem);
 	spin_lock_bh(&pch->downl);
-	WRITE_ONCE(pch->chan, NULL);
+	pch->chan = NULL;
 	spin_unlock_bh(&pch->downl);
 	up_write(&pch->chan_sem);
-	ppp_disconnect_channel(pch);
 
 	pn = ppp_pernet(pch->chan_net);
 	spin_lock_bh(&pn->all_channels_lock);
@@ -3460,7 +3442,6 @@ static void ppp_destroy_interface(struct ppp *ppp)
 	}
 #endif /* CONFIG_PPP_FILTER */
 
-	kfree_skb(ppp->xmit_pending);
 	free_percpu(ppp->xmit_recursion);
 
 	free_netdev(ppp->dev);
diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c
index c2063961f3950..b7f243b416f82 100644
--- a/drivers/net/ppp/ppp_synctty.c
+++ b/drivers/net/ppp/ppp_synctty.c
@@ -483,7 +483,7 @@ static void ppp_sync_process(struct tasklet_struct *t)
 	while ((skb = skb_dequeue(&ap->rqueue)) != NULL) {
 		if (skb->len == 0) {
 			/* zero length buffers indicate error */
-			ppp_input_error(&ap->chan, 0);
+			ppp_input_error(&ap->chan);
 			kfree_skb(skb);
 		}
 		else
diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
index 4275b393a4544..1ac61c273b283 100644
--- a/drivers/net/ppp/pppoe.c
+++ b/drivers/net/ppp/pppoe.c
@@ -237,25 +237,6 @@ static inline struct pppox_sock *get_item(struct pppoe_net *pn, __be16 sid,
 	return po;
 }
 
-static inline struct pppox_sock *__get_item_by_addr(struct net *net,
-						    struct sockaddr_pppox *sp)
-{
-	struct net_device *dev;
-	struct pppoe_net *pn;
-	struct pppox_sock *pppox_sock = NULL;
-
-	int ifindex;
-
-	dev = dev_get_by_name_rcu(net, sp->sa_addr.pppoe.dev);
-	if (dev) {
-		ifindex = dev->ifindex;
-		pn = pppoe_pernet(net);
-		pppox_sock = __get_item(pn, sp->sa_addr.pppoe.sid,
-					sp->sa_addr.pppoe.remote, ifindex);
-	}
-	return pppox_sock;
-}
-
 static inline void delete_item(struct pppoe_net *pn, __be16 sid,
 					char *addr, int ifindex)
 {
@@ -369,7 +350,6 @@ static struct notifier_block pppoe_notifier = {
 static int pppoe_rcv_core(struct sock *sk, struct sk_buff *skb)
 {
 	struct pppox_sock *po = pppox_sk(sk);
-	struct pppox_sock *relay_po;
 
 	/* Backlog receive. Semantics of backlog rcv preclude any code from
 	 * executing in lock_sock()/release_sock() bounds; meaning sk->sk_state
@@ -378,17 +358,6 @@ static int pppoe_rcv_core(struct sock *sk, struct sk_buff *skb)
 
 	if (sk->sk_state & PPPOX_BOUND) {
 		ppp_input(&po->chan, skb);
-	} else if (sk->sk_state & PPPOX_RELAY) {
-		relay_po = __get_item_by_addr(sock_net(sk),
-					      &po->pppoe_relay);
-		if (relay_po == NULL)
-			goto abort_kfree;
-
-		if ((sk_pppox(relay_po)->sk_state & PPPOX_CONNECTED) == 0)
-			goto abort_kfree;
-
-		if (!__pppoe_xmit(sk_pppox(relay_po), skb))
-			goto abort_kfree;
 	} else {
 		if (sock_queue_rcv_skb(sk, skb))
 			goto abort_kfree;
@@ -656,7 +625,6 @@ static int pppoe_connect(struct socket *sock, struct sockaddr_unsized *uservaddr
 
 		po->pppoe_ifindex = 0;
 		memset(&po->pppoe_pa, 0, sizeof(po->pppoe_pa));
-		memset(&po->pppoe_relay, 0, sizeof(po->pppoe_relay));
 		memset(&po->chan, 0, sizeof(po->chan));
 		po->next = NULL;
 		po->num = 0;
@@ -783,53 +751,6 @@ static int pppoe_ioctl(struct socket *sock, unsigned int cmd,
 		err = 0;
 		break;
 
-	case PPPOEIOCSFWD:
-	{
-		struct pppox_sock *relay_po;
-
-		err = -EBUSY;
-		if (sk->sk_state & (PPPOX_BOUND | PPPOX_DEAD))
-			break;
-
-		err = -ENOTCONN;
-		if (!(sk->sk_state & PPPOX_CONNECTED))
-			break;
-
-		/* PPPoE address from the user specifies an outbound
-		   PPPoE address which frames are forwarded to */
-		err = -EFAULT;
-		if (copy_from_user(&po->pppoe_relay,
-				   (void __user *)arg,
-				   sizeof(struct sockaddr_pppox)))
-			break;
-
-		err = -EINVAL;
-		if (po->pppoe_relay.sa_family != AF_PPPOX ||
-		    po->pppoe_relay.sa_protocol != PX_PROTO_OE)
-			break;
-
-		/* Check that the socket referenced by the address
-		   actually exists. */
-		rcu_read_lock();
-		relay_po = __get_item_by_addr(sock_net(sk), &po->pppoe_relay);
-		rcu_read_unlock();
-		if (!relay_po)
-			break;
-
-		sk->sk_state |= PPPOX_RELAY;
-		err = 0;
-		break;
-	}
-
-	case PPPOEIOCDFWD:
-		err = -EALREADY;
-		if (!(sk->sk_state & PPPOX_RELAY))
-			break;
-
-		sk->sk_state &= ~PPPOX_RELAY;
-		err = 0;
-		break;
-
 	default:
 		err = -ENOTTY;
 	}
@@ -885,7 +806,7 @@ static int pppoe_sendmsg(struct socket *sock, struct msghdr *m,
 	skb->protocol = cpu_to_be16(ETH_P_PPP_SES);
 
 	ph = skb_put(skb, total_len + sizeof(struct pppoe_hdr));
-	start = (char *)&ph->tag[0];
+	start = (char *)ph + sizeof(*ph);
 
 	error = memcpy_from_msg(start, m, total_len);
 	if (error < 0) {
diff --git a/drivers/net/ppp/pppox.c b/drivers/net/ppp/pppox.c
index 08364f10a43fa..5861a2f6ce3e6 100644
--- a/drivers/net/ppp/pppox.c
+++ b/drivers/net/ppp/pppox.c
@@ -102,9 +102,6 @@ EXPORT_SYMBOL(pppox_ioctl);
 #ifdef CONFIG_COMPAT
 int pppox_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
 {
-	if (cmd == PPPOEIOCSFWD32)
-		cmd = PPPOEIOCSFWD;
-
 	return pppox_ioctl(sock, cmd, (unsigned long)compat_ptr(arg));
 }
 
diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c
index cbffa9ae1bb6b..dd53f413c38f6 100644
--- a/drivers/net/usb/aqc111.c
+++ b/drivers/net/usb/aqc111.c
@@ -1395,14 +1395,14 @@ static int aqc111_suspend(struct usb_interface *intf, pm_message_t message)
 		aqc111_write16_cmd_nopm(dev, AQ_ACCESS_MAC,
 					SFR_MEDIUM_STATUS_MODE, 2, &reg16);
 
-		aqc111_write_cmd(dev, AQ_WOL_CFG, 0, 0,
-				 WOL_CFG_SIZE, &wol_cfg);
-		aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-				   &aqc111_data->phy_cfg);
+		aqc111_write_cmd_nopm(dev, AQ_WOL_CFG, 0, 0,
+				      WOL_CFG_SIZE, &wol_cfg);
+		aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0,
+					&aqc111_data->phy_cfg);
 	} else {
 		aqc111_data->phy_cfg |= AQ_LOW_POWER;
-		aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-				   &aqc111_data->phy_cfg);
+		aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0,
+					&aqc111_data->phy_cfg);
 
 		/* Disable RX path */
 		aqc111_read16_cmd_nopm(dev, AQ_ACCESS_MAC,
diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c
index a032c1ded4063..a0a5740590b94 100644
--- a/drivers/net/usb/cdc_ether.c
+++ b/drivers/net/usb/cdc_ether.c
@@ -115,7 +115,7 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 	int				len = intf->cur_altsetting->extralen;
 	struct usb_interface_descriptor	*d;
 	struct cdc_state		*info = (void *) &dev->data;
-	int				status;
+	int				status = -ENODEV;
 	int				rndis;
 	bool				android_rndis_quirk = false;
 	struct usb_driver		*driver = driver_of(intf);
@@ -169,10 +169,13 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 	info->header = header.usb_cdc_header_desc;
 	info->ether = header.usb_cdc_ether_desc;
 	if (!info->u) {
-		if (rndis)
+		if (rndis) {
 			goto skip;
-		else /* in that case a quirk is mandatory */
+		} else {
+			/* in that case a quirk is mandatory */
+			dev_err(&dev->udev->dev, "No union descriptors\n");
 			goto bad_desc;
+		}
 	}
 	/* we need a master/control interface (what we're
 	 * probed with) and a slave/data interface; union
@@ -192,18 +195,20 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 			android_rndis_quirk = true;
 			goto skip;
 		}
+		dev_err(&intf->dev, "bad CDC descriptors\n");
 		goto bad_desc;
 	}
 	if (info->control != intf) {
-		dev_dbg(&intf->dev, "bogus CDC Union\n");
 		/* Ambit USB Cable Modem (and maybe others)
 		 * interchanges master and slave interface.
 		 */
 		if (info->data == intf) {
 			info->data = info->control;
 			info->control = intf;
-		} else
+		} else {
+			dev_err(&intf->dev, "bogus CDC Union\n");
 			goto bad_desc;
+		}
 	}
 
 	/* some devices merge these - skip class check */
@@ -213,7 +218,7 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 	/* a data interface altsetting does the real i/o */
 	d = &info->data->cur_altsetting->desc;
 	if (d->bInterfaceClass != USB_CLASS_CDC_DATA) {
-		dev_dbg(&intf->dev, "slave class %u\n", d->bInterfaceClass);
+		dev_err(&intf->dev, "slave class %u\n", d->bInterfaceClass);
 		goto bad_desc;
 	}
 skip:
@@ -227,7 +232,7 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 	if (rndis && is_rndis(&intf->cur_altsetting->desc) &&
 	    header.usb_cdc_acm_descriptor &&
 	    header.usb_cdc_acm_descriptor->bmCapabilities) {
-		dev_dbg(&intf->dev,
+		dev_err(&intf->dev,
 			"ACM capabilities %02x, not really RNDIS?\n",
 			header.usb_cdc_acm_descriptor->bmCapabilities);
 		goto bad_desc;
@@ -242,14 +247,14 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 
 	if (header.usb_cdc_mdlm_desc &&
 	    memcmp(header.usb_cdc_mdlm_desc->bGUID, mbm_guid, 16)) {
-		dev_dbg(&intf->dev, "GUID doesn't match\n");
+		dev_err(&intf->dev, "GUID doesn't match\n");
 		goto bad_desc;
 	}
 
 	if (header.usb_cdc_mdlm_detail_desc &&
 		header.usb_cdc_mdlm_detail_desc->bLength <
 			(sizeof(struct usb_cdc_mdlm_detail_desc) + 1)) {
-		dev_dbg(&intf->dev, "Descriptor too short\n");
+		dev_err(&intf->dev, "Descriptor too short\n");
 		goto bad_desc;
 	}
 
@@ -267,7 +272,7 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 		info->control = usb_ifnum_to_if(dev->udev, 0);
 		info->data = usb_ifnum_to_if(dev->udev, 1);
 		if (!info->control || !info->data || info->control != intf) {
-			dev_dbg(&intf->dev,
+			dev_err(&intf->dev,
 				"rndis: master #0/%p slave #1/%p\n",
 				info->control,
 				info->data);
@@ -275,7 +280,7 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 		}
 
 	} else if (!info->header || (!rndis && !info->ether)) {
-		dev_dbg(&intf->dev, "missing cdc %s%s%sdescriptor\n",
+		dev_err(&intf->dev, "missing cdc %s%s%sdescriptor\n",
 			info->header ? "" : "header ",
 			info->u ? "" : "union ",
 			info->ether ? "" : "ether ");
@@ -287,16 +292,15 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 	 */
 	if (info->data != info->control) {
 		status = usb_driver_claim_interface(driver, info->data, dev);
-		if (status < 0)
-			return status;
+		if (status < 0) {
+			dev_err(&intf->dev, "Second interface unclaimable\n");
+			goto bad_desc;
+		}
 	}
 	status = usbnet_get_endpoints(dev, info->data);
 	if (status < 0) {
-		/* ensure immediate exit from usbnet_disconnect */
-		usb_set_intfdata(info->data, NULL);
-		if (info->data != info->control)
-			usb_driver_release_interface(driver, info->data);
-		return status;
+		dev_dbg(&intf->dev, "Mandatory endpoints missing\n");
+		goto bail_out_and_release;
 	}
 
 	/* status endpoint: optional for CDC Ethernet, not RNDIS (or ACM) */
@@ -316,10 +320,9 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 		}
 	}
 	if (rndis && !dev->status) {
-		dev_dbg(&intf->dev, "missing RNDIS status endpoint\n");
-		usb_set_intfdata(info->data, NULL);
-		usb_driver_release_interface(driver, info->data);
-		return -ENODEV;
+		dev_err(&intf->dev, "missing RNDIS status endpoint\n");
+		status = -ENODEV;
+		goto bail_out_and_release;
 	}
 
 	/* override ethtool_ops */
@@ -327,9 +330,12 @@ int usbnet_generic_cdc_bind(struct usbnet *dev, struct usb_interface *intf)
 
 	return 0;
 
+bail_out_and_release:
+	usb_set_intfdata(info->data, NULL);
+	if (info->data != info->control)
+		usb_driver_release_interface(driver, info->data);
 bad_desc:
-	dev_info(&dev->udev->dev, "bad CDC descriptors\n");
-	return -ENODEV;
+	return status;
 }
 EXPORT_SYMBOL_GPL(usbnet_generic_cdc_bind);
 
diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c
index 7057c6c0cfc6b..bb9929727eb93 100644
--- a/drivers/net/usb/cdc_ncm.c
+++ b/drivers/net/usb/cdc_ncm.c
@@ -1656,6 +1656,7 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset)
 	struct usbnet *dev = netdev_priv(skb_in->dev);
 	struct usb_cdc_ncm_ndp16 *ndp16;
 	int ret = -EINVAL;
+	size_t ndp_len;
 
 	if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp16)) > skb_in->len) {
 		netif_dbg(dev, rx_err, dev->net, "invalid NDP offset  <%u>\n",
@@ -1675,8 +1676,8 @@ int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset)
 					sizeof(struct usb_cdc_ncm_dpe16));
 	ret--; /* we process NDP entries except for the last one */
 
-	if ((sizeof(struct usb_cdc_ncm_ndp16) +
-	     ret * (sizeof(struct usb_cdc_ncm_dpe16))) > skb_in->len) {
+	ndp_len = struct_size_t(struct usb_cdc_ncm_ndp16, dpe16, ret);
+	if (ndpoffset + ndp_len > skb_in->len) {
 		netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret);
 		ret = -EINVAL;
 	}
@@ -1692,6 +1693,7 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset)
 	struct usbnet *dev = netdev_priv(skb_in->dev);
 	struct usb_cdc_ncm_ndp32 *ndp32;
 	int ret = -EINVAL;
+	size_t ndp_len;
 
 	if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp32)) > skb_in->len) {
 		netif_dbg(dev, rx_err, dev->net, "invalid NDP offset  <%u>\n",
@@ -1711,8 +1713,8 @@ int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset)
 					sizeof(struct usb_cdc_ncm_dpe32));
 	ret--; /* we process NDP entries except for the last one */
 
-	if ((sizeof(struct usb_cdc_ncm_ndp32) +
-	     ret * (sizeof(struct usb_cdc_ncm_dpe32))) > skb_in->len) {
+	ndp_len = struct_size_t(struct usb_cdc_ncm_ndp32, dpe32, ret);
+	if (ndpoffset + ndp_len > skb_in->len) {
 		netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret);
 		ret = -EINVAL;
 	}
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index 19cdf69fa589c..bcf293ea1bd38 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -4536,7 +4536,6 @@ static void intr_complete(struct urb *urb)
 static void lan78xx_disconnect(struct usb_interface *intf)
 {
 	struct lan78xx_net *dev;
-	struct usb_device *udev;
 	struct net_device *net;
 
 	dev = usb_get_intfdata(intf);
@@ -4544,7 +4543,6 @@ static void lan78xx_disconnect(struct usb_interface *intf)
 	if (!dev)
 		return;
 
-	udev = interface_to_usbdev(intf);
 	net = dev->net;
 
 	rtnl_lock();
@@ -4571,7 +4569,6 @@ static void lan78xx_disconnect(struct usb_interface *intf)
 	usb_free_urb(dev->urb_intr);
 
 	free_netdev(net);
-	usb_put_dev(udev);
 }
 
 static void lan78xx_tx_timeout(struct net_device *net, unsigned int txqueue)
@@ -4633,13 +4630,11 @@ static int lan78xx_probe(struct usb_interface *intf,
 	u8 *buf = NULL;
 
 	udev = interface_to_usbdev(intf);
-	udev = usb_get_dev(udev);
 
 	netdev = alloc_etherdev(sizeof(struct lan78xx_net));
 	if (!netdev) {
 		dev_err(&intf->dev, "Error: OOM\n");
-		ret = -ENOMEM;
-		goto out1;
+		return -ENOMEM;
 	}
 
 	SET_NETDEV_DEV(netdev, &intf->dev);
@@ -4788,8 +4783,6 @@ static int lan78xx_probe(struct usb_interface *intf,
 	lan78xx_free_tx_resources(dev);
 out2:
 	free_netdev(netdev);
-out1:
-	usb_put_dev(udev);
 
 	return ret;
 }
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 0c83bbbea2e7c..3b6d4252d34c6 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -207,6 +207,7 @@
 #define OCP_EEE_LPABLE		0xa5d2
 #define OCP_10GBT_CTRL		0xa5d4
 #define OCP_10GBT_STAT		0xa5d6
+#define OCP_EEE_LPABLE2		0xa6d0
 #define OCP_EEE_ADV2		0xa6d4
 #define OCP_PHY_STATE		0xa708		/* nway state for 8153 */
 #define OCP_PHY_PATCH_STAT	0xb800
@@ -948,6 +949,7 @@ struct r8152 {
 	u16 ocp_base;
 	u16 speed;
 	u16 eee_adv;
+	u16 eee_adv2;
 	u8 *intr_buff;
 	u8 version;
 	u8 duplex;
@@ -5393,7 +5395,7 @@ static void r8156_eee_en(struct r8152 *tp, bool enable)
 
 	config = ocp_reg_read(tp, OCP_EEE_ADV2);
 
-	if (enable)
+	if (enable && (tp->eee_adv2 & MDIO_EEE_2_5GT))
 		config |= MDIO_EEE_2_5GT;
 	else
 		config &= ~MDIO_EEE_2_5GT;
@@ -8922,7 +8924,8 @@ static void rtl8152_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 
 static int r8152_get_eee(struct r8152 *tp, struct ethtool_keee *eee)
 {
-	__ETHTOOL_DECLARE_LINK_MODE_MASK(common);
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(common) = {};
+	u16 speed = rtl8152_get_speed(tp);
 	u16 val;
 
 	val = r8152_mmd_read(tp, MDIO_MMD_PCS, MDIO_PCS_EEE_ABLE);
@@ -8936,8 +8939,14 @@ static int r8152_get_eee(struct r8152 *tp, struct ethtool_keee *eee)
 
 	eee->eee_enabled = tp->eee_en;
 
-	linkmode_and(common, eee->advertised, eee->lp_advertised);
-	eee->eee_active = phy_check_valid(tp->speed, tp->duplex, common);
+	if (speed & _1000bps)
+		linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, common);
+	if (speed & _100bps)
+		linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, common);
+
+	linkmode_and(common, common, eee->advertised);
+	linkmode_and(common, common, eee->lp_advertised);
+	eee->eee_active = !linkmode_empty(common);
 
 	return 0;
 }
@@ -8948,7 +8957,10 @@ static int r8152_set_eee(struct r8152 *tp, struct ethtool_keee *eee)
 
 	tp->eee_en = eee->eee_enabled;
 	tp->eee_adv = val;
-
+	if (tp->support_2500full) {
+		val = linkmode_to_mii_eee_cap2_t(eee->advertised);
+		tp->eee_adv2 = val;
+	}
 	rtl_eee_enable(tp, tp->eee_en);
 
 	return 0;
@@ -8956,7 +8968,8 @@ static int r8152_set_eee(struct r8152 *tp, struct ethtool_keee *eee)
 
 static int r8153_get_eee(struct r8152 *tp, struct ethtool_keee *eee)
 {
-	__ETHTOOL_DECLARE_LINK_MODE_MASK(common);
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(common) = {};
+	u16 speed = rtl8152_get_speed(tp);
 	u16 val;
 
 	val = ocp_reg_read(tp, OCP_EEE_ABLE);
@@ -8968,10 +8981,29 @@ static int r8153_get_eee(struct r8152 *tp, struct ethtool_keee *eee)
 	val = ocp_reg_read(tp, OCP_EEE_LPABLE);
 	mii_eee_cap1_mod_linkmode_t(eee->lp_advertised, val);
 
+	if (tp->support_2500full) {
+		linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->supported);
+
+		val = ocp_reg_read(tp, OCP_EEE_ADV2);
+		mii_eee_cap2_mod_linkmode_adv_t(eee->advertised, val);
+
+		val = ocp_reg_read(tp, OCP_EEE_LPABLE2);
+		mii_eee_cap2_mod_linkmode_adv_t(eee->lp_advertised, val);
+
+		if (speed & _2500bps)
+			linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, common);
+	}
+
 	eee->eee_enabled = tp->eee_en;
 
-	linkmode_and(common, eee->advertised, eee->lp_advertised);
-	eee->eee_active = phy_check_valid(tp->speed, tp->duplex, common);
+	if (speed & _1000bps)
+		linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, common);
+	if (speed & _100bps)
+		linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, common);
+
+	linkmode_and(common, common, eee->advertised);
+	linkmode_and(common, common, eee->lp_advertised);
+	eee->eee_active = !linkmode_empty(common);
 
 	return 0;
 }
@@ -9512,6 +9544,7 @@ static int rtl_ops_init(struct r8152 *tp)
 	case RTL_VER_11:
 		tp->eee_en		= true;
 		tp->eee_adv		= MDIO_EEE_1000T | MDIO_EEE_100TX;
+		tp->eee_adv2		= MDIO_EEE_2_5GT;
 		fallthrough;
 	case RTL_VER_10:
 		ops->init		= r8156_init;
@@ -9537,6 +9570,7 @@ static int rtl_ops_init(struct r8152 *tp)
 	case RTL_VER_15:
 		tp->eee_en		= true;
 		tp->eee_adv		= MDIO_EEE_1000T | MDIO_EEE_100TX;
+		tp->eee_adv2		= MDIO_EEE_2_5GT;
 		ops->init		= r8156b_init;
 		ops->enable		= rtl8156b_enable;
 		ops->disable		= rtl8153_disable;
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 72d6a9c6a5a24..022f60728721a 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -26,6 +26,7 @@
 #include <net/netdev_rx_queue.h>
 #include <net/netdev_queues.h>
 #include <net/xdp_sock_drv.h>
+#include <net/page_pool/helpers.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -290,14 +291,6 @@ struct virtnet_interrupt_coalesce {
 	u32 max_usecs;
 };
 
-/* The dma information of pages allocated at a time. */
-struct virtnet_rq_dma {
-	dma_addr_t addr;
-	u32 ref;
-	u16 len;
-	u16 need_sync;
-};
-
 /* Internal representation of a send virtqueue */
 struct send_queue {
 	/* Virtqueue associated with this send _queue */
@@ -356,8 +349,10 @@ struct receive_queue {
 	/* Average packet length for mergeable receive buffers. */
 	struct ewma_pkt_len mrg_avg_pkt_len;
 
-	/* Page frag for packet buffer allocation. */
-	struct page_frag alloc_frag;
+	struct page_pool *page_pool;
+
+	/* True if page_pool handles DMA mapping via PP_FLAG_DMA_MAP */
+	bool use_page_pool_dma;
 
 	/* RX: fragments + linear part + virtio header */
 	struct scatterlist sg[MAX_SKB_FRAGS + 2];
@@ -370,9 +365,6 @@ struct receive_queue {
 
 	struct xdp_rxq_info xdp_rxq;
 
-	/* Record the last dma info to free after new pages is allocated. */
-	struct virtnet_rq_dma *last_dma;
-
 	struct xsk_buff_pool *xsk_pool;
 
 	/* xdp rxq used by xsk */
@@ -521,11 +513,14 @@ static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
 			       struct virtnet_rq_stats *stats);
 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq,
 				 struct sk_buff *skb, u8 flags);
-static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
+static struct sk_buff *virtnet_skb_append_frag(struct receive_queue *rq,
+					       struct sk_buff *head_skb,
 					       struct sk_buff *curr_skb,
 					       struct page *page, void *buf,
 					       int len, int truesize);
 static void virtnet_xsk_completed(struct send_queue *sq, int num);
+static void free_unused_bufs(struct virtnet_info *vi);
+static void virtnet_del_vqs(struct virtnet_info *vi);
 
 enum virtnet_xmit_type {
 	VIRTNET_XMIT_TYPE_SKB,
@@ -709,12 +704,10 @@ static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 static void virtnet_rq_free_buf(struct virtnet_info *vi,
 				struct receive_queue *rq, void *buf)
 {
-	if (vi->mergeable_rx_bufs)
-		put_page(virt_to_head_page(buf));
-	else if (vi->big_packets)
+	if (!rq->page_pool)
 		give_pages(rq, buf);
 	else
-		put_page(virt_to_head_page(buf));
+		page_pool_put_page(rq->page_pool, virt_to_head_page(buf), -1, false);
 }
 
 static void enable_rx_mode_work(struct virtnet_info *vi)
@@ -876,10 +869,16 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 		skb = virtnet_build_skb(buf, truesize, p - buf, len);
 		if (unlikely(!skb))
 			return NULL;
+		/* Big packets mode chains pages via page->private, which is
+		 * incompatible with the way page_pool uses page->private.
+		 * Currently, big packets mode doesn't use page pools.
+		 */
+		if (!rq->page_pool) {
+			page = (struct page *)page->private;
+			if (page)
+				give_pages(rq, page);
+		}
 
-		page = (struct page *)page->private;
-		if (page)
-			give_pages(rq, page);
 		goto ok;
 	}
 
@@ -925,133 +924,16 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 	hdr = skb_vnet_common_hdr(skb);
 	memcpy(hdr, hdr_p, hdr_len);
 	if (page_to_free)
-		put_page(page_to_free);
+		page_pool_put_page(rq->page_pool, page_to_free, -1, true);
 
 	return skb;
 }
 
-static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
-{
-	struct virtnet_info *vi = rq->vq->vdev->priv;
-	struct page *page = virt_to_head_page(buf);
-	struct virtnet_rq_dma *dma;
-	void *head;
-	int offset;
-
-	BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs);
-
-	head = page_address(page);
-
-	dma = head;
-
-	--dma->ref;
-
-	if (dma->need_sync && len) {
-		offset = buf - (head + sizeof(*dma));
-
-		virtqueue_map_sync_single_range_for_cpu(rq->vq, dma->addr,
-							offset, len,
-							DMA_FROM_DEVICE);
-	}
-
-	if (dma->ref)
-		return;
-
-	virtqueue_unmap_single_attrs(rq->vq, dma->addr, dma->len,
-				     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
-	put_page(page);
-}
-
 static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
 {
-	struct virtnet_info *vi = rq->vq->vdev->priv;
-	void *buf;
-
-	BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs);
-
-	buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
-	if (buf)
-		virtnet_rq_unmap(rq, buf, *len);
-
-	return buf;
-}
-
-static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
-{
-	struct virtnet_info *vi = rq->vq->vdev->priv;
-	struct virtnet_rq_dma *dma;
-	dma_addr_t addr;
-	u32 offset;
-	void *head;
-
-	BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs);
-
-	head = page_address(rq->alloc_frag.page);
-
-	offset = buf - head;
-
-	dma = head;
-
-	addr = dma->addr - sizeof(*dma) + offset;
-
-	sg_init_table(rq->sg, 1);
-	sg_fill_dma(rq->sg, addr, len);
-}
-
-static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
-{
-	struct page_frag *alloc_frag = &rq->alloc_frag;
-	struct virtnet_info *vi = rq->vq->vdev->priv;
-	struct virtnet_rq_dma *dma;
-	void *buf, *head;
-	dma_addr_t addr;
+	BUG_ON(!rq->page_pool);
 
-	BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs);
-
-	head = page_address(alloc_frag->page);
-
-	dma = head;
-
-	/* new pages */
-	if (!alloc_frag->offset) {
-		if (rq->last_dma) {
-			/* Now, the new page is allocated, the last dma
-			 * will not be used. So the dma can be unmapped
-			 * if the ref is 0.
-			 */
-			virtnet_rq_unmap(rq, rq->last_dma, 0);
-			rq->last_dma = NULL;
-		}
-
-		dma->len = alloc_frag->size - sizeof(*dma);
-
-		addr = virtqueue_map_single_attrs(rq->vq, dma + 1,
-						  dma->len, DMA_FROM_DEVICE, 0);
-		if (virtqueue_map_mapping_error(rq->vq, addr))
-			return NULL;
-
-		dma->addr = addr;
-		dma->need_sync = virtqueue_map_need_sync(rq->vq, addr);
-
-		/* Add a reference to dma to prevent the entire dma from
-		 * being released during error handling. This reference
-		 * will be freed after the pages are no longer used.
-		 */
-		get_page(alloc_frag->page);
-		dma->ref = 1;
-		alloc_frag->offset = sizeof(*dma);
-
-		rq->last_dma = dma;
-	}
-
-	++dma->ref;
-
-	buf = head + alloc_frag->offset;
-
-	get_page(alloc_frag->page);
-	alloc_frag->offset += size;
-
-	return buf;
+	return virtqueue_get_buf_ctx(rq->vq, len, ctx);
 }
 
 static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf)
@@ -1067,9 +949,6 @@ static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf)
 		return;
 	}
 
-	if (!vi->big_packets || vi->mergeable_rx_bufs)
-		virtnet_rq_unmap(rq, buf, 0);
-
 	virtnet_rq_free_buf(vi, rq, buf);
 }
 
@@ -1335,7 +1214,7 @@ static int xsk_append_merge_buffer(struct virtnet_info *vi,
 
 		truesize = len;
 
-		curr_skb  = virtnet_skb_append_frag(head_skb, curr_skb, page,
+		curr_skb  = virtnet_skb_append_frag(rq, head_skb, curr_skb, page,
 						    buf, len, truesize);
 		if (!curr_skb) {
 			put_page(page);
@@ -1454,8 +1333,19 @@ static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue
 	xsk_buffs = rq->xsk_buffs;
 
 	num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
-	if (!num)
+	if (!num) {
+		if (xsk_uses_need_wakeup(pool)) {
+			xsk_set_rx_need_wakeup(pool);
+			/* Return 0 instead of -ENOMEM so that NAPI is
+			 * descheduled.
+			 */
+			return 0;
+		}
+
 		return -ENOMEM;
+	} else {
+		xsk_clear_rx_need_wakeup(pool);
+	}
 
 	len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
 
@@ -1588,20 +1478,19 @@ static bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
 	return sent;
 }
 
-static void xsk_wakeup(struct send_queue *sq)
+static void xsk_wakeup(struct napi_struct *napi, struct virtqueue *vq)
 {
-	if (napi_if_scheduled_mark_missed(&sq->napi))
+	if (napi_if_scheduled_mark_missed(napi))
 		return;
 
 	local_bh_disable();
-	virtqueue_napi_schedule(&sq->napi, sq->vq);
+	virtqueue_napi_schedule(napi, vq);
 	local_bh_enable();
 }
 
 static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
-	struct send_queue *sq;
 
 	if (!netif_running(dev))
 		return -ENETDOWN;
@@ -1609,9 +1498,18 @@ static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)
 	if (qid >= vi->curr_queue_pairs)
 		return -EINVAL;
 
-	sq = &vi->sq[qid];
+	if (flag & XDP_WAKEUP_TX) {
+		struct send_queue *sq = &vi->sq[qid];
+
+		xsk_wakeup(&sq->napi, sq->vq);
+	}
+
+	if (flag & XDP_WAKEUP_RX) {
+		struct receive_queue *rq = &vi->rq[qid];
+
+		xsk_wakeup(&rq->napi, rq->vq);
+	}
 
-	xsk_wakeup(sq);
 	return 0;
 }
 
@@ -1623,7 +1521,7 @@ static void virtnet_xsk_completed(struct send_queue *sq, int num)
 	 * wakeup the tx napi to consume the xsk tx queue, because the tx
 	 * interrupt may not be triggered.
 	 */
-	xsk_wakeup(sq);
+	xsk_wakeup(&sq->napi, sq->vq);
 }
 
 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
@@ -1771,7 +1669,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
 	return ret;
 }
 
-static void put_xdp_frags(struct xdp_buff *xdp)
+static void put_xdp_frags(struct receive_queue *rq, struct xdp_buff *xdp)
 {
 	struct skb_shared_info *shinfo;
 	struct page *xdp_page;
@@ -1781,7 +1679,7 @@ static void put_xdp_frags(struct xdp_buff *xdp)
 		shinfo = xdp_get_shared_info_from_buff(xdp);
 		for (i = 0; i < shinfo->nr_frags; i++) {
 			xdp_page = skb_frag_page(&shinfo->frags[i]);
-			put_page(xdp_page);
+			page_pool_put_page(rq->page_pool, xdp_page, -1, true);
 		}
 	}
 }
@@ -1873,7 +1771,7 @@ static struct page *xdp_linearize_page(struct net_device *dev,
 	if (page_off + *len + tailroom > PAGE_SIZE)
 		return NULL;
 
-	page = alloc_page(GFP_ATOMIC);
+	page = page_pool_alloc_pages(rq->page_pool, GFP_ATOMIC);
 	if (!page)
 		return NULL;
 
@@ -1896,8 +1794,12 @@ static struct page *xdp_linearize_page(struct net_device *dev,
 		p = virt_to_head_page(buf);
 		off = buf - page_address(p);
 
+		if (rq->use_page_pool_dma)
+			page_pool_dma_sync_for_cpu(rq->page_pool, p,
+						   off, buflen);
+
 		if (check_mergeable_len(dev, ctx, buflen)) {
-			put_page(p);
+			page_pool_put_page(rq->page_pool, p, -1, true);
 			goto err_buf;
 		}
 
@@ -1905,38 +1807,36 @@ static struct page *xdp_linearize_page(struct net_device *dev,
 		 * is sending packet larger than the MTU.
 		 */
 		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
-			put_page(p);
+			page_pool_put_page(rq->page_pool, p, -1, true);
 			goto err_buf;
 		}
 
 		memcpy(page_address(page) + page_off,
 		       page_address(p) + off, buflen);
 		page_off += buflen;
-		put_page(p);
+		page_pool_put_page(rq->page_pool, p, -1, true);
 	}
 
 	/* Headroom does not contribute to packet length */
 	*len = page_off - XDP_PACKET_HEADROOM;
 	return page;
 err_buf:
-	__free_pages(page, 0);
+	page_pool_put_page(rq->page_pool, page, -1, true);
 	return NULL;
 }
 
 static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
 					       unsigned int xdp_headroom,
 					       void *buf,
-					       unsigned int len)
+					       unsigned int len,
+					       unsigned int buflen)
 {
 	unsigned int header_offset;
 	unsigned int headroom;
-	unsigned int buflen;
 	struct sk_buff *skb;
 
 	header_offset = VIRTNET_RX_PAD + xdp_headroom;
 	headroom = vi->hdr_len + header_offset;
-	buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
-		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
 	skb = virtnet_build_skb(buf, buflen, headroom, len);
 	if (unlikely(!skb))
@@ -1955,6 +1855,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 					 void *buf,
 					 unsigned int xdp_headroom,
 					 unsigned int len,
+					 unsigned int buflen,
 					 unsigned int *xdp_xmit,
 					 struct virtnet_rq_stats *stats)
 {
@@ -1963,7 +1864,6 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 	struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
 	struct page *page = virt_to_head_page(buf);
 	struct page *xdp_page;
-	unsigned int buflen;
 	struct xdp_buff xdp;
 	struct sk_buff *skb;
 	unsigned int metasize = 0;
@@ -1976,9 +1876,6 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 	if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM))
 		goto err_xdp;
 
-	buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
-		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-
 	if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
 		int offset = buf - page_address(page) + header_offset;
 		unsigned int tlen = len + vi->hdr_len;
@@ -1996,7 +1893,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 			goto err_xdp;
 
 		buf = page_address(xdp_page);
-		put_page(page);
+		page_pool_put_page(rq->page_pool, page, -1, true);
 		page = xdp_page;
 	}
 
@@ -2028,13 +1925,15 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 	if (metasize)
 		skb_metadata_set(skb, metasize);
 
+	skb_mark_for_recycle(skb);
+
 	return skb;
 
 err_xdp:
 	u64_stats_inc(&stats->xdp_drops);
 err:
 	u64_stats_inc(&stats->drops);
-	put_page(page);
+	page_pool_put_page(rq->page_pool, page, -1, true);
 xdp_xmit:
 	return NULL;
 }
@@ -2047,7 +1946,8 @@ static struct sk_buff *receive_small(struct net_device *dev,
 				     unsigned int *xdp_xmit,
 				     struct virtnet_rq_stats *stats)
 {
-	unsigned int xdp_headroom = (unsigned long)ctx;
+	unsigned int xdp_headroom = mergeable_ctx_to_headroom(ctx);
+	unsigned int buflen = mergeable_ctx_to_truesize(ctx);
 	struct page *page = virt_to_head_page(buf);
 	struct sk_buff *skb;
 
@@ -2056,6 +1956,13 @@ static struct sk_buff *receive_small(struct net_device *dev,
 	 */
 	buf -= VIRTNET_RX_PAD + xdp_headroom;
 
+	if (rq->use_page_pool_dma) {
+		int offset = buf - page_address(page) +
+			     VIRTNET_RX_PAD + xdp_headroom;
+
+		page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, len);
+	}
+
 	len -= vi->hdr_len;
 	u64_stats_add(&stats->bytes, len);
 
@@ -2073,21 +1980,23 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		xdp_prog = rcu_dereference(rq->xdp_prog);
 		if (xdp_prog) {
 			skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf,
-						xdp_headroom, len, xdp_xmit,
-						stats);
+						xdp_headroom, len, buflen,
+						xdp_xmit, stats);
 			rcu_read_unlock();
 			return skb;
 		}
 		rcu_read_unlock();
 	}
 
-	skb = receive_small_build_skb(vi, xdp_headroom, buf, len);
-	if (likely(skb))
+	skb = receive_small_build_skb(vi, xdp_headroom, buf, len, buflen);
+	if (likely(skb)) {
+		skb_mark_for_recycle(skb);
 		return skb;
+	}
 
 err:
 	u64_stats_inc(&stats->drops);
-	put_page(page);
+	page_pool_put_page(rq->page_pool, page, -1, true);
 	return NULL;
 }
 
@@ -2142,7 +2051,7 @@ static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
 		}
 		u64_stats_add(&stats->bytes, len);
 		page = virt_to_head_page(buf);
-		put_page(page);
+		page_pool_put_page(rq->page_pool, page, -1, true);
 	}
 }
 
@@ -2252,8 +2161,12 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
 		page = virt_to_head_page(buf);
 		offset = buf - page_address(page);
 
+		if (rq->use_page_pool_dma)
+			page_pool_dma_sync_for_cpu(rq->page_pool, page,
+						   offset, len);
+
 		if (check_mergeable_len(dev, ctx, len)) {
-			put_page(page);
+			page_pool_put_page(rq->page_pool, page, -1, true);
 			goto err;
 		}
 
@@ -2272,7 +2185,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
 	return 0;
 
 err:
-	put_xdp_frags(xdp);
+	put_xdp_frags(rq, xdp);
 	return -EINVAL;
 }
 
@@ -2337,7 +2250,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
 		if (*len + xdp_room > PAGE_SIZE)
 			return NULL;
 
-		xdp_page = alloc_page(GFP_ATOMIC);
+		xdp_page = page_pool_alloc_pages(rq->page_pool, GFP_ATOMIC);
 		if (!xdp_page)
 			return NULL;
 
@@ -2347,7 +2260,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
 
 	*frame_sz = PAGE_SIZE;
 
-	put_page(*page);
+	page_pool_put_page(rq->page_pool, *page, -1, true);
 
 	*page = xdp_page;
 
@@ -2393,6 +2306,8 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
 		head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
 		if (unlikely(!head_skb))
 			break;
+
+		skb_mark_for_recycle(head_skb);
 		return head_skb;
 
 	case XDP_TX:
@@ -2403,10 +2318,10 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
 		break;
 	}
 
-	put_xdp_frags(&xdp);
+	put_xdp_frags(rq, &xdp);
 
 err_xdp:
-	put_page(page);
+	page_pool_put_page(rq->page_pool, page, -1, true);
 	mergeable_buf_free(rq, num_buf, dev, stats);
 
 	u64_stats_inc(&stats->xdp_drops);
@@ -2414,7 +2329,8 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
 	return NULL;
 }
 
-static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
+static struct sk_buff *virtnet_skb_append_frag(struct receive_queue *rq,
+					       struct sk_buff *head_skb,
 					       struct sk_buff *curr_skb,
 					       struct page *page, void *buf,
 					       int len, int truesize)
@@ -2429,6 +2345,9 @@ static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
 		if (unlikely(!nskb))
 			return NULL;
 
+		if (head_skb->pp_recycle)
+			skb_mark_for_recycle(nskb);
+
 		if (curr_skb == head_skb)
 			skb_shinfo(curr_skb)->frag_list = nskb;
 		else
@@ -2446,7 +2365,10 @@ static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb,
 
 	offset = buf - page_address(page);
 	if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
-		put_page(page);
+		if (head_skb->pp_recycle)
+			page_pool_put_page(rq->page_pool, page, -1, true);
+		else
+			put_page(page);
 		skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
 				     len, truesize);
 	} else {
@@ -2475,6 +2397,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 
 	head_skb = NULL;
+
+	if (rq->use_page_pool_dma)
+		page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, len);
+
 	u64_stats_add(&stats->bytes, len - vi->hdr_len);
 
 	if (check_mergeable_len(dev, ctx, len))
@@ -2499,6 +2425,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 
 	if (unlikely(!curr_skb))
 		goto err_skb;
+
+	skb_mark_for_recycle(head_skb);
 	while (--num_buf) {
 		buf = virtnet_rq_get_buf(rq, &len, &ctx);
 		if (unlikely(!buf)) {
@@ -2513,11 +2441,17 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		u64_stats_add(&stats->bytes, len);
 		page = virt_to_head_page(buf);
 
+		if (rq->use_page_pool_dma) {
+			offset = buf - page_address(page);
+			page_pool_dma_sync_for_cpu(rq->page_pool, page,
+						   offset, len);
+		}
+
 		if (check_mergeable_len(dev, ctx, len))
 			goto err_skb;
 
 		truesize = mergeable_ctx_to_truesize(ctx);
-		curr_skb  = virtnet_skb_append_frag(head_skb, curr_skb, page,
+		curr_skb  = virtnet_skb_append_frag(rq, head_skb, curr_skb, page,
 						    buf, len, truesize);
 		if (!curr_skb)
 			goto err_skb;
@@ -2527,7 +2461,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	return head_skb;
 
 err_skb:
-	put_page(page);
+	page_pool_put_page(rq->page_pool, page, -1, true);
 	mergeable_buf_free(rq, num_buf, dev, stats);
 
 err_buf:
@@ -2658,40 +2592,54 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
 	virtnet_receive_done(vi, rq, skb, flags);
 }
 
-/* Unlike mergeable buffers, all buffers are allocated to the
- * same size, except for the headroom. For this reason we do
- * not need to use  mergeable_len_to_ctx here - it is enough
- * to store the headroom as the context ignoring the truesize.
+static int virtnet_rq_submit(struct receive_queue *rq, char *buf,
+			     int len, void *ctx, gfp_t gfp)
+{
+	if (rq->use_page_pool_dma) {
+		struct page *page = virt_to_head_page(buf);
+		dma_addr_t addr = page_pool_get_dma_addr(page) +
+				  (buf - (char *)page_address(page));
+
+		sg_init_table(rq->sg, 1);
+		sg_fill_dma(rq->sg, addr, len);
+		return virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1,
+						     buf, ctx, gfp);
+	}
+
+	sg_init_one(rq->sg, buf, len);
+	return virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
+}
+
+/* With page_pool, the actual allocation may exceed the requested size
+ * when the remaining page fragment can't fit another buffer. Encode
+ * the actual allocation size in ctx so build_skb() gets the correct
+ * buflen for truesize accounting.
  */
 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
 			     gfp_t gfp)
 {
-	char *buf;
 	unsigned int xdp_headroom = virtnet_get_headroom(vi);
-	void *ctx = (void *)(unsigned long)xdp_headroom;
-	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
+	unsigned int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
+	unsigned int alloc_len;
+	char *buf;
+	void *ctx;
 	int err;
 
 	len = SKB_DATA_ALIGN(len) +
 	      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
-	if (unlikely(!skb_page_frag_refill(len, &rq->alloc_frag, gfp)))
-		return -ENOMEM;
-
-	buf = virtnet_rq_alloc(rq, len, gfp);
+	alloc_len = len;
+	buf = page_pool_alloc_va(rq->page_pool, &alloc_len, gfp);
 	if (unlikely(!buf))
 		return -ENOMEM;
 
 	buf += VIRTNET_RX_PAD + xdp_headroom;
 
-	virtnet_rq_init_one_sg(rq, buf, vi->hdr_len + GOOD_PACKET_LEN);
-
-	err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp);
-	if (err < 0) {
-		virtnet_rq_unmap(rq, buf, 0);
-		put_page(virt_to_head_page(buf));
-	}
+	ctx = mergeable_len_to_ctx(alloc_len, xdp_headroom);
+	err = virtnet_rq_submit(rq, buf, vi->hdr_len + GOOD_PACKET_LEN, ctx, gfp);
 
+	if (err < 0)
+		page_pool_put_page(rq->page_pool, virt_to_head_page(buf), -1, false);
 	return err;
 }
 
@@ -2764,13 +2712,12 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
 static int add_recvbuf_mergeable(struct virtnet_info *vi,
 				 struct receive_queue *rq, gfp_t gfp)
 {
-	struct page_frag *alloc_frag = &rq->alloc_frag;
 	unsigned int headroom = virtnet_get_headroom(vi);
 	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
 	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
-	unsigned int len, hole;
-	void *ctx;
+	unsigned int len, alloc_len;
 	char *buf;
+	void *ctx;
 	int err;
 
 	/* Extra tailroom is needed to satisfy XDP's assumption. This
@@ -2779,44 +2726,29 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
 	 */
 	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
 
-	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
-		return -ENOMEM;
-
-	if (!alloc_frag->offset && len + room + sizeof(struct virtnet_rq_dma) > alloc_frag->size)
-		len -= sizeof(struct virtnet_rq_dma);
-
-	buf = virtnet_rq_alloc(rq, len + room, gfp);
+	alloc_len = len + room;
+	buf = page_pool_alloc_va(rq->page_pool, &alloc_len, gfp);
 	if (unlikely(!buf))
 		return -ENOMEM;
 
 	buf += headroom; /* advance address leaving hole at front of pkt */
-	hole = alloc_frag->size - alloc_frag->offset;
-	if (hole < len + room) {
-		/* To avoid internal fragmentation, if there is very likely not
-		 * enough space for another buffer, add the remaining space to
-		 * the current buffer.
-		 * XDP core assumes that frame_size of xdp_buff and the length
-		 * of the frag are PAGE_SIZE, so we disable the hole mechanism.
-		 */
-		if (!headroom)
-			len += hole;
-		alloc_frag->offset += hole;
-	}
 
-	virtnet_rq_init_one_sg(rq, buf, len);
+	if (!headroom)
+		len = alloc_len - room;
 
 	ctx = mergeable_len_to_ctx(len + room, headroom);
-	err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp);
-	if (err < 0) {
-		virtnet_rq_unmap(rq, buf, 0);
-		put_page(virt_to_head_page(buf));
-	}
 
+	err = virtnet_rq_submit(rq, buf, len, ctx, gfp);
+
+	if (err < 0)
+		page_pool_put_page(rq->page_pool, virt_to_head_page(buf), -1, false);
 	return err;
 }
 
 /*
- * Returns false if we couldn't fill entirely (OOM).
+ * Returns false if we couldn't fill entirely (OOM) and need to retry.
+ * In XSK mode, it's when the receive buffer is not allocated and
+ * xsk_use_need_wakeup is not set.
  *
  * Normally run in the receive path, but can also be run from ndo_open
  * before we're receiving packets, or from refill_work which is
@@ -2963,7 +2895,7 @@ static int virtnet_receive_packets(struct virtnet_info *vi,
 	int packets = 0;
 	void *buf;
 
-	if (!vi->big_packets || vi->mergeable_rx_bufs) {
+	if (rq->page_pool) {
 		void *ctx;
 		while (packets < budget &&
 		       (buf = virtnet_rq_get_buf(rq, &len, &ctx))) {
@@ -3128,7 +3060,10 @@ static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index)
 		return err;
 
 	err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq,
-					 MEM_TYPE_PAGE_SHARED, NULL);
+					 vi->rq[qp_index].page_pool ?
+						MEM_TYPE_PAGE_POOL :
+						MEM_TYPE_PAGE_SHARED,
+					 vi->rq[qp_index].page_pool);
 	if (err < 0)
 		goto err_xdp_reg_mem_model;
 
@@ -3168,6 +3103,82 @@ static void virtnet_update_settings(struct virtnet_info *vi)
 		vi->duplex = duplex;
 }
 
+static int virtnet_create_page_pools(struct virtnet_info *vi)
+{
+	int i, err;
+
+	if (vi->big_packets && !vi->mergeable_rx_bufs)
+		return 0;
+
+	for (i = 0; i < vi->max_queue_pairs; i++) {
+		struct receive_queue *rq = &vi->rq[i];
+		struct page_pool_params pp_params = { 0 };
+		struct device *dma_dev;
+
+		if (rq->page_pool)
+			continue;
+
+		if (rq->xsk_pool)
+			continue;
+
+		pp_params.order = 0;
+		pp_params.pool_size = virtqueue_get_vring_size(rq->vq);
+		pp_params.nid = dev_to_node(vi->vdev->dev.parent);
+		pp_params.netdev = vi->dev;
+		pp_params.napi = &rq->napi;
+
+		/* Use page_pool DMA mapping if backend supports DMA API.
+		 * DMA_SYNC_DEV is needed for non-coherent archs on recycle.
+		 */
+		dma_dev = virtqueue_dma_dev(rq->vq);
+		if (dma_dev) {
+			pp_params.dev = dma_dev;
+			pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+			pp_params.dma_dir = DMA_FROM_DEVICE;
+			pp_params.max_len = PAGE_SIZE;
+			pp_params.offset = 0;
+			rq->use_page_pool_dma = true;
+		} else {
+			/* No DMA API (e.g., VDUSE): page_pool for allocation only. */
+			pp_params.flags = 0;
+			rq->use_page_pool_dma = false;
+		}
+
+		rq->page_pool = page_pool_create(&pp_params);
+		if (IS_ERR(rq->page_pool)) {
+			err = PTR_ERR(rq->page_pool);
+			rq->page_pool = NULL;
+			goto err_cleanup;
+		}
+	}
+	return 0;
+
+err_cleanup:
+	while (--i >= 0) {
+		struct receive_queue *rq = &vi->rq[i];
+
+		if (rq->page_pool) {
+			page_pool_destroy(rq->page_pool);
+			rq->page_pool = NULL;
+		}
+	}
+	return err;
+}
+
+static void virtnet_destroy_page_pools(struct virtnet_info *vi)
+{
+	int i;
+
+	for (i = 0; i < vi->max_queue_pairs; i++) {
+		struct receive_queue *rq = &vi->rq[i];
+
+		if (rq->page_pool) {
+			page_pool_destroy(rq->page_pool);
+			rq->page_pool = NULL;
+		}
+	}
+}
+
 static int virtnet_open(struct net_device *dev)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
@@ -5715,6 +5726,10 @@ static int virtnet_restore_up(struct virtio_device *vdev)
 	if (err)
 		return err;
 
+	err = virtnet_create_page_pools(vi);
+	if (err)
+		goto err_del_vqs;
+
 	virtio_device_ready(vdev);
 
 	enable_rx_mode_work(vi);
@@ -5724,12 +5739,24 @@ static int virtnet_restore_up(struct virtio_device *vdev)
 		err = virtnet_open(vi->dev);
 		rtnl_unlock();
 		if (err)
-			return err;
+			goto err_destroy_pools;
 	}
 
 	netif_tx_lock_bh(vi->dev);
 	netif_device_attach(vi->dev);
 	netif_tx_unlock_bh(vi->dev);
+	return 0;
+
+err_destroy_pools:
+	virtio_reset_device(vdev);
+	free_unused_bufs(vi);
+	virtnet_destroy_page_pools(vi);
+	virtnet_del_vqs(vi);
+	return err;
+
+err_del_vqs:
+	virtio_reset_device(vdev);
+	virtnet_del_vqs(vi);
 	return err;
 }
 
@@ -5857,7 +5884,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
 	/* In big_packets mode, xdp cannot work, so there is no need to
 	 * initialize xsk of rq.
 	 */
-	if (vi->big_packets && !vi->mergeable_rx_bufs)
+	if (!vi->rq[qid].page_pool)
 		return -ENOENT;
 
 	if (qid >= vi->curr_queue_pairs)
@@ -6287,17 +6314,6 @@ static void free_receive_bufs(struct virtnet_info *vi)
 	rtnl_unlock();
 }
 
-static void free_receive_page_frags(struct virtnet_info *vi)
-{
-	int i;
-	for (i = 0; i < vi->max_queue_pairs; i++)
-		if (vi->rq[i].alloc_frag.page) {
-			if (vi->rq[i].last_dma)
-				virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0);
-			put_page(vi->rq[i].alloc_frag.page);
-		}
-}
-
 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
 {
 	struct virtnet_info *vi = vq->vdev->priv;
@@ -6401,7 +6417,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
 	vqs_info = kzalloc_objs(*vqs_info, total_vqs);
 	if (!vqs_info)
 		goto err_vqs_info;
-	if (!vi->big_packets || vi->mergeable_rx_bufs) {
+	if (vi->mergeable_rx_bufs || !vi->big_packets) {
 		ctx = kzalloc_objs(*ctx, total_vqs);
 		if (!ctx)
 			goto err_ctx;
@@ -6441,10 +6457,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
 		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
 		vi->sq[i].vq = vqs[txq2vq(i)];
 	}
-
 	/* run here: ret == 0. */
 
-
 err_find:
 	kfree(ctx);
 err_ctx:
@@ -6945,6 +6959,14 @@ static int virtnet_probe(struct virtio_device *vdev)
 			goto free;
 	}
 
+	/* Create page pools for receive queues.
+	 * Page pools are created at probe time so they can be used
+	 * with premapped DMA addresses throughout the device lifetime.
+	 */
+	err = virtnet_create_page_pools(vi);
+	if (err)
+		goto free_irq_moder;
+
 #ifdef CONFIG_SYSFS
 	if (vi->mergeable_rx_bufs)
 		dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
@@ -6958,7 +6980,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 		vi->failover = net_failover_create(vi->dev);
 		if (IS_ERR(vi->failover)) {
 			err = PTR_ERR(vi->failover);
-			goto free_vqs;
+			goto free_page_pools;
 		}
 	}
 
@@ -7075,9 +7097,11 @@ static int virtnet_probe(struct virtio_device *vdev)
 	unregister_netdev(dev);
 free_failover:
 	net_failover_destroy(vi->failover);
-free_vqs:
+free_page_pools:
+	virtnet_destroy_page_pools(vi);
+free_irq_moder:
+	virtnet_free_irq_moder(vi);
 	virtio_reset_device(vdev);
-	free_receive_page_frags(vi);
 	virtnet_del_vqs(vi);
 free:
 	free_netdev(dev);
@@ -7102,7 +7126,7 @@ static void remove_vq_common(struct virtnet_info *vi)
 
 	free_receive_bufs(vi);
 
-	free_receive_page_frags(vi);
+	virtnet_destroy_page_pools(vi);
 
 	virtnet_del_vqs(vi);
 }
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 0572f6a9bdb62..40522afc05320 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -2268,10 +2268,10 @@ vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
 		rq->data_ring.base =
 			dma_alloc_coherent(&adapter->pdev->dev, sz,
 					   &rq->data_ring.basePA,
-					   GFP_KERNEL);
+					   GFP_KERNEL | __GFP_NOWARN);
 		if (!rq->data_ring.base) {
 			netdev_err(adapter->netdev,
-				   "rx data ring will be disabled\n");
+				   "failed to allocate %zu bytes, rx data ring will be disabled\n", sz);
 			adapter->rxdataring_enabled = false;
 		}
 	} else {
diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c
index 6661fff326e08..987d57a01ddfa 100644
--- a/drivers/net/wireless/ath/ath10k/usb.c
+++ b/drivers/net/wireless/ath/ath10k/usb.c
@@ -1016,7 +1016,6 @@ static int ath10k_usb_probe(struct usb_interface *interface,
 
 	netif_napi_add(ar->napi_dev, &ar->napi, ath10k_usb_napi_poll);
 
-	usb_get_dev(dev);
 	vendor_id = le16_to_cpu(dev->descriptor.idVendor);
 	product_id = le16_to_cpu(dev->descriptor.idProduct);
 
@@ -1055,12 +1054,10 @@ static int ath10k_usb_probe(struct usb_interface *interface,
 err:
 	ath10k_core_destroy(ar);
 
-	usb_put_dev(dev);
-
 	return ret;
 }
 
-static void ath10k_usb_remove(struct usb_interface *interface)
+static void ath10k_usb_disconnect(struct usb_interface *interface)
 {
 	struct ath10k_usb *ar_usb;
 
@@ -1071,7 +1068,6 @@ static void ath10k_usb_remove(struct usb_interface *interface)
 	ath10k_core_unregister(ar_usb->ar);
 	netif_napi_del(&ar_usb->ar->napi);
 	ath10k_usb_destroy(ar_usb->ar);
-	usb_put_dev(interface_to_usbdev(interface));
 	ath10k_core_destroy(ar_usb->ar);
 }
 
@@ -1117,7 +1113,7 @@ static struct usb_driver ath10k_usb_driver = {
 	.probe = ath10k_usb_probe,
 	.suspend = ath10k_usb_pm_suspend,
 	.resume = ath10k_usb_pm_resume,
-	.disconnect = ath10k_usb_remove,
+	.disconnect = ath10k_usb_disconnect,
 	.id_table = ath10k_usb_ids,
 	.supports_autosuspend = true,
 	.disable_hub_initiated_lpm = 1,
diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
index 8dfe9b40c1262..08d3a0c8f105b 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.c
+++ b/drivers/net/wireless/ath/ath11k/ahb.c
@@ -807,10 +807,8 @@ static int ath11k_core_get_rproc(struct ath11k_base *ab)
 	}
 
 	prproc = rproc_get_by_phandle(rproc_phandle);
-	if (!prproc) {
-		ath11k_dbg(ab, ATH11K_DBG_AHB, "failed to get rproc, deferring\n");
-		return -EPROBE_DEFER;
-	}
+	if (!prproc)
+		return dev_err_probe(&ab->pdev->dev, -EPROBE_DEFER, "failed to get rproc\n");
 	ab_ahb->tgt_rproc = prproc;
 
 	return 0;
@@ -1190,10 +1188,8 @@ static int ath11k_ahb_probe(struct platform_device *pdev)
 	ath11k_ahb_init_qmi_ce_config(ab);
 
 	ret = ath11k_core_get_rproc(ab);
-	if (ret) {
-		ath11k_err(ab, "failed to get rproc: %d\n", ret);
+	if (ret)
 		goto err_ce_free;
-	}
 
 	ret = ath11k_core_init(ab);
 	if (ret) {
diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c
index e4ee2ba1f669e..a48b6bf1f29a8 100644
--- a/drivers/net/wireless/ath/ath11k/mac.c
+++ b/drivers/net/wireless/ath/ath11k/mac.c
@@ -1557,12 +1557,15 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif,
 	if (!beacons || !beacons->cnt) {
 		ath11k_warn(arvif->ar->ab,
 			    "failed to get ema beacon templates from mac80211\n");
-		return -EPERM;
+		ret = -EPERM;
+		goto free;
 	}
 
 	if (tx_arvif == arvif) {
-		if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb))
-			return -EINVAL;
+		if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb)) {
+			ret = -EINVAL;
+			goto free;
+		}
 	} else {
 		arvif->wpaie_present = tx_arvif->wpaie_present;
 	}
@@ -1589,11 +1592,11 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif,
 		}
 	}
 
-	ieee80211_beacon_free_ema_list(beacons);
-
 	if (tx_arvif != arvif && !nontx_vif_params_set)
-		return -EINVAL; /* Profile not found in the beacons */
+		ret = -EINVAL; /* Profile not found in the beacons */
 
+free:
+	ieee80211_beacon_free_ema_list(beacons);
 	return ret;
 }
 
@@ -1622,19 +1625,22 @@ static int ath11k_mac_setup_bcn_tmpl_mbssid(struct ath11k_vif *arvif,
 	}
 
 	if (tx_arvif == arvif) {
-		if (ath11k_mac_set_vif_params(tx_arvif, bcn))
-			return -EINVAL;
+		if (ath11k_mac_set_vif_params(tx_arvif, bcn)) {
+			ret = -EINVAL;
+			goto free;
+		}
 	} else if (!ath11k_mac_set_nontx_vif_params(tx_arvif, arvif, bcn)) {
-		return -EINVAL;
+		ret = -EINVAL;
+		goto free;
 	}
 
 	ret = ath11k_wmi_bcn_tmpl(ar, arvif->vdev_id, &offs, bcn, 0);
-	kfree_skb(bcn);
-
 	if (ret)
 		ath11k_warn(ab, "failed to submit beacon template command: %d\n",
 			    ret);
 
+free:
+	kfree_skb(bcn);
 	return ret;
 }
 
@@ -3305,7 +3311,7 @@ static int ath11k_mac_fils_discovery(struct ath11k_vif *arvif,
 	if (info->fils_discovery.max_interval) {
 		interval = info->fils_discovery.max_interval;
 
-		tmpl = ieee80211_get_fils_discovery_tmpl(ar->hw, arvif->vif);
+		tmpl = ieee80211_get_fils_discovery_tmpl(ar->hw, arvif->vif, 0);
 		if (tmpl)
 			ret = ath11k_wmi_fils_discovery_tmpl(ar, arvif->vdev_id,
 							     tmpl);
@@ -3314,7 +3320,7 @@ static int ath11k_mac_fils_discovery(struct ath11k_vif *arvif,
 		interval = info->unsol_bcast_probe_resp_interval;
 
 		tmpl = ieee80211_get_unsol_bcast_probe_resp_tmpl(ar->hw,
-								 arvif->vif);
+								 arvif->vif, 0);
 		if (tmpl)
 			ret = ath11k_wmi_probe_resp_tmpl(ar, arvif->vdev_id,
 							 tmpl);
@@ -6282,10 +6288,10 @@ static int ath11k_mac_mgmt_action_frame_fill_elem_data(struct ath11k_vif *arvif,
 	lockdep_assert_held(&ar->conf_mutex);
 
 	/* make sure category field is present */
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE)
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
 		return -EINVAL;
 
-	remaining_len = skb->len - IEEE80211_MIN_ACTION_SIZE;
+	remaining_len = skb->len - IEEE80211_MIN_ACTION_SIZE(category);
 	has_protected = ieee80211_has_protected(hdr->frame_control);
 
 	/* In case of SW crypto and hdr protected (PMF), packet will already be encrypted,
diff --git a/drivers/net/wireless/ath/ath12k/Makefile b/drivers/net/wireless/ath/ath12k/Makefile
index 3ba1236956cc8..3b39b2c333076 100644
--- a/drivers/net/wireless/ath/ath12k/Makefile
+++ b/drivers/net/wireless/ath/ath12k/Makefile
@@ -32,6 +32,7 @@ ath12k-$(CONFIG_ATH12K_TRACING) += trace.o
 ath12k-$(CONFIG_PM) += wow.o
 ath12k-$(CONFIG_ATH12K_COREDUMP) += coredump.o
 ath12k-$(CONFIG_NL80211_TESTMODE) += testmode.o
+ath12k-$(CONFIG_THERMAL) += thermal.o
 
 # for tracing framework to find trace.h
 CFLAGS_trace.o := -I$(src)
diff --git a/drivers/net/wireless/ath/ath12k/ahb.h b/drivers/net/wireless/ath/ath12k/ahb.h
index 8a040d03d27a3..be9e31b3682d6 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.h
+++ b/drivers/net/wireless/ath/ath12k/ahb.h
@@ -21,8 +21,8 @@
 #define ATH12K_ROOTPD_READY_TIMEOUT		(5 * HZ)
 #define ATH12K_RPROC_AFTER_POWERUP		QCOM_SSR_AFTER_POWERUP
 #define ATH12K_AHB_FW_PREFIX			"q6_fw"
-#define ATH12K_AHB_FW_SUFFIX			".mdt"
-#define ATH12K_AHB_FW2				"iu_fw.mdt"
+#define ATH12K_AHB_FW_SUFFIX			".mbn"
+#define ATH12K_AHB_FW2				"iu_fw.mbn"
 #define ATH12K_AHB_UPD_SWID			0x12
 #define ATH12K_USERPD_SPAWN_TIMEOUT		(5 * HZ)
 #define ATH12K_USERPD_READY_TIMEOUT		(10 * HZ)
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index 4ed608ba3c304..c31c47fb5a73b 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -863,11 +863,22 @@ static int ath12k_core_pdev_create(struct ath12k_base *ab)
 		return ret;
 	}
 
+	ret = ath12k_thermal_register(ab);
+	if (ret) {
+		ath12k_err(ab, "could not register thermal device: %d\n", ret);
+		goto err_dp_pdev_free;
+	}
+
 	return 0;
+
+err_dp_pdev_free:
+	ath12k_dp_pdev_free(ab);
+	return ret;
 }
 
 static void ath12k_core_pdev_destroy(struct ath12k_base *ab)
 {
+	ath12k_thermal_unregister(ab);
 	ath12k_dp_pdev_free(ab);
 }
 
@@ -1006,6 +1017,8 @@ static void ath12k_core_hw_group_stop(struct ath12k_hw_group *ag)
 
 	ath12k_mac_unregister(ag);
 
+	ath12k_mac_mlo_teardown(ag);
+
 	for (i = ag->num_devices - 1; i >= 0; i--) {
 		ab = ag->ab[i];
 		if (!ab)
@@ -1123,8 +1136,14 @@ static int ath12k_core_hw_group_start(struct ath12k_hw_group *ag)
 
 	lockdep_assert_held(&ag->mutex);
 
-	if (test_bit(ATH12K_GROUP_FLAG_REGISTERED, &ag->flags))
+	if (test_bit(ATH12K_GROUP_FLAG_REGISTERED, &ag->flags)) {
+		ret = ath12k_core_mlo_setup(ag);
+		if (WARN_ON(ret)) {
+			ath12k_mac_unregister(ag);
+			goto err_mac_destroy;
+		}
 		goto core_pdev_create;
+	}
 
 	ret = ath12k_mac_allocate(ag);
 	if (WARN_ON(ret))
@@ -1361,6 +1380,7 @@ static int ath12k_core_reconfigure_on_crash(struct ath12k_base *ab)
 
 	mutex_lock(&ab->core_lock);
 	ath12k_link_sta_rhash_tbl_destroy(ab);
+	ath12k_thermal_unregister(ab);
 	ath12k_dp_pdev_free(ab);
 	ath12k_ce_cleanup_pipes(ab);
 	ath12k_wmi_detach(ab);
@@ -1502,6 +1522,7 @@ static void ath12k_core_pre_reconfigure_recovery(struct ath12k_base *ab)
 			complete(&ar->vdev_delete_done);
 			complete(&ar->bss_survey_done);
 			complete_all(&ar->regd_update_completed);
+			complete_all(&ar->thermal.wmi_sync);
 
 			wake_up(&ar->dp.tx_empty_waitq);
 			idr_for_each(&ar->txmgmt_idr,
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index 990934ec92fca..59c193b247646 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -36,6 +36,7 @@
 #include "coredump.h"
 #include "cmn_defs.h"
 #include "dp_cmn.h"
+#include "thermal.h"
 
 #define SM(_v, _f) (((_v) << _f##_LSB) & _f##_MASK)
 
@@ -522,7 +523,7 @@ struct ath12k_sta {
 	u16 links_map;
 	u8 assoc_link_id;
 	u16 ml_peer_id;
-	u8 num_peer;
+	u16 free_logical_link_idx_map;
 
 	enum ieee80211_sta_state state;
 };
@@ -757,6 +758,8 @@ struct ath12k {
 
 	s8 max_allowed_tx_power;
 	struct ath12k_pdev_rssi_offsets rssi_info;
+
+	struct ath12k_thermal thermal;
 };
 
 struct ath12k_hw {
diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.c b/drivers/net/wireless/ath/ath12k/dp_htt.c
index e71bb71a6020e..9c19d9707abfb 100644
--- a/drivers/net/wireless/ath/ath12k/dp_htt.c
+++ b/drivers/net/wireless/ath/ath12k/dp_htt.c
@@ -205,16 +205,9 @@ ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev,
 	if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE)))
 		return;
 
-	if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) {
+	if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON))
 		is_ampdu =
 			HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags);
-		tx_retry_failed =
-			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) -
-			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success);
-		tx_retry_count =
-			HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
-			HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
-	}
 
 	if (usr_stats->tlv_flags &
 	    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) {
@@ -223,10 +216,19 @@ ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev,
 					  HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M);
 		tid = le32_get_bits(usr_stats->ack_ba.info,
 				    HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM);
-	}
 
-	if (common->fes_duration_us)
-		tx_duration = le32_to_cpu(common->fes_duration_us);
+		if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) {
+			tx_retry_failed =
+				__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) -
+				__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success);
+			tx_retry_count =
+				HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
+				HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
+		}
+
+		if (common->fes_duration_us)
+			tx_duration = le32_to_cpu(common->fes_duration_us);
+	}
 
 	user_rate = &usr_stats->rate;
 	flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags);
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index 5a82ede65dd3d..f0b4f607b8452 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -1117,7 +1117,6 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k_pdev_dp *dp_pdev,
 }
 
 void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
-			    struct hal_rx_desc *rx_desc,
 			    enum hal_encrypt_type enctype,
 			    bool decrypted,
 			    struct hal_rx_desc_data *rx_info)
@@ -1393,7 +1392,6 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 EXPORT_SYMBOL(ath12k_dp_rx_deliver_msdu);
 
 bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp,
-					    struct hal_rx_desc *rx_desc,
 					    struct sk_buff *msdu,
 					    struct hal_rx_desc_data *rx_info)
 {
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.h b/drivers/net/wireless/ath/ath12k/dp_rx.h
index 1ec5382f59955..bd62af0c80d46 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.h
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.h
@@ -189,7 +189,6 @@ void ath12k_dp_extract_rx_desc_data(struct ath12k_hal *hal,
 }
 
 void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
-			    struct hal_rx_desc *rx_desc,
 			    enum hal_encrypt_type enctype,
 			    bool decrypted,
 			    struct hal_rx_desc_data *rx_info);
@@ -197,7 +196,6 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 			       struct sk_buff *msdu,
 			       struct hal_rx_desc_data *rx_info);
 bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp,
-					    struct hal_rx_desc *rx_desc,
 					    struct sk_buff *msdu,
 					    struct hal_rx_desc_data *rx_info);
 u64 ath12k_dp_rx_h_get_pn(struct ath12k_dp *dp, struct sk_buff *skb);
diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h
index 43e3880f82579..bf4f7dbae8669 100644
--- a/drivers/net/wireless/ath/ath12k/hal.h
+++ b/drivers/net/wireless/ath/ath12k/hal.h
@@ -268,21 +268,28 @@ enum hal_rx_reception_type {
 };
 
 enum hal_rx_legacy_rate {
-	HAL_RX_LEGACY_RATE_1_MBPS,
-	HAL_RX_LEGACY_RATE_2_MBPS,
-	HAL_RX_LEGACY_RATE_5_5_MBPS,
-	HAL_RX_LEGACY_RATE_6_MBPS,
-	HAL_RX_LEGACY_RATE_9_MBPS,
-	HAL_RX_LEGACY_RATE_11_MBPS,
-	HAL_RX_LEGACY_RATE_12_MBPS,
-	HAL_RX_LEGACY_RATE_18_MBPS,
-	HAL_RX_LEGACY_RATE_24_MBPS,
-	HAL_RX_LEGACY_RATE_36_MBPS,
-	HAL_RX_LEGACY_RATE_48_MBPS,
-	HAL_RX_LEGACY_RATE_54_MBPS,
+	HAL_RX_LEGACY_RATE_LP_1_MBPS,
+	HAL_RX_LEGACY_RATE_LP_2_MBPS,
+	HAL_RX_LEGACY_RATE_LP_5_5_MBPS,
+	HAL_RX_LEGACY_RATE_LP_11_MBPS,
+	HAL_RX_LEGACY_RATE_SP_2_MBPS,
+	HAL_RX_LEGACY_RATE_SP_5_5_MBPS,
+	HAL_RX_LEGACY_RATE_SP_11_MBPS,
 	HAL_RX_LEGACY_RATE_INVALID,
 };
 
+enum hal_rx_legacy_rates_ofdm {
+	HAL_RX_LEGACY_RATE_OFDM_48_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_24_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_12_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_6_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_54_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_36_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_18_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_9_MBPS,
+	HAL_RX_LEGACY_RATE_OFDM_INVALID,
+};
+
 enum hal_ring_type {
 	HAL_REO_DST,
 	HAL_REO_EXCEPTION,
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index b253d1e3f4052..553ec28b6aaa0 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -164,30 +164,31 @@ static const struct ieee80211_channel ath12k_6ghz_channels[] = {
 	CHAN6G(233, 7115, 0),
 };
 
+#define ATH12K_MAC_RATE_A_M(bps, code) \
+	{ .bitrate = (bps), .hw_value = (code),\
+	  .flags = IEEE80211_RATE_MANDATORY_A }
+
+#define ATH12K_MAC_RATE_B(bps, code, code_short) \
+	{ .bitrate = (bps), .hw_value = (code), .hw_value_short = (code_short),\
+	  .flags = IEEE80211_RATE_SHORT_PREAMBLE }
+
 static struct ieee80211_rate ath12k_legacy_rates[] = {
 	{ .bitrate = 10,
 	  .hw_value = ATH12K_HW_RATE_CCK_LP_1M },
-	{ .bitrate = 20,
-	  .hw_value = ATH12K_HW_RATE_CCK_LP_2M,
-	  .hw_value_short = ATH12K_HW_RATE_CCK_SP_2M,
-	  .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 55,
-	  .hw_value = ATH12K_HW_RATE_CCK_LP_5_5M,
-	  .hw_value_short = ATH12K_HW_RATE_CCK_SP_5_5M,
-	  .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 110,
-	  .hw_value = ATH12K_HW_RATE_CCK_LP_11M,
-	  .hw_value_short = ATH12K_HW_RATE_CCK_SP_11M,
-	  .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-
-	{ .bitrate = 60, .hw_value = ATH12K_HW_RATE_OFDM_6M },
-	{ .bitrate = 90, .hw_value = ATH12K_HW_RATE_OFDM_9M },
-	{ .bitrate = 120, .hw_value = ATH12K_HW_RATE_OFDM_12M },
-	{ .bitrate = 180, .hw_value = ATH12K_HW_RATE_OFDM_18M },
-	{ .bitrate = 240, .hw_value = ATH12K_HW_RATE_OFDM_24M },
-	{ .bitrate = 360, .hw_value = ATH12K_HW_RATE_OFDM_36M },
-	{ .bitrate = 480, .hw_value = ATH12K_HW_RATE_OFDM_48M },
-	{ .bitrate = 540, .hw_value = ATH12K_HW_RATE_OFDM_54M },
+	ATH12K_MAC_RATE_B(20, ATH12K_HW_RATE_CCK_LP_2M,
+			  ATH12K_HW_RATE_CCK_SP_2M),
+	ATH12K_MAC_RATE_B(55, ATH12K_HW_RATE_CCK_LP_5_5M,
+			  ATH12K_HW_RATE_CCK_SP_5_5M),
+	ATH12K_MAC_RATE_B(110, ATH12K_HW_RATE_CCK_LP_11M,
+			  ATH12K_HW_RATE_CCK_SP_11M),
+	ATH12K_MAC_RATE_A_M(60, ATH12K_HW_RATE_OFDM_6M),
+	ATH12K_MAC_RATE_A_M(90, ATH12K_HW_RATE_OFDM_9M),
+	ATH12K_MAC_RATE_A_M(120, ATH12K_HW_RATE_OFDM_12M),
+	ATH12K_MAC_RATE_A_M(180, ATH12K_HW_RATE_OFDM_18M),
+	ATH12K_MAC_RATE_A_M(240, ATH12K_HW_RATE_OFDM_24M),
+	ATH12K_MAC_RATE_A_M(360, ATH12K_HW_RATE_OFDM_36M),
+	ATH12K_MAC_RATE_A_M(480, ATH12K_HW_RATE_OFDM_48M),
+	ATH12K_MAC_RATE_A_M(540, ATH12K_HW_RATE_OFDM_54M),
 };
 
 static const int
@@ -732,11 +733,17 @@ u8 ath12k_mac_hw_rate_to_idx(const struct ieee80211_supported_band *sband,
 		if (ath12k_mac_bitrate_is_cck(rate->bitrate) != cck)
 			continue;
 
-		if (rate->hw_value == hw_rate)
+		/* To handle 802.11a PPDU type */
+		if ((!cck) && (rate->hw_value == hw_rate) &&
+		    (rate->flags & IEEE80211_RATE_MANDATORY_A))
 			return i;
+		/* To handle 802.11b short PPDU type */
 		else if (rate->flags & IEEE80211_RATE_SHORT_PREAMBLE &&
 			 rate->hw_value_short == hw_rate)
 			return i;
+		/* To handle 802.11b long PPDU type */
+		else if (rate->hw_value == hw_rate)
+			return i;
 	}
 
 	return 0;
@@ -4311,7 +4318,8 @@ static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif,
 	if (info->fils_discovery.max_interval) {
 		interval = info->fils_discovery.max_interval;
 
-		tmpl = ieee80211_get_fils_discovery_tmpl(hw, vif);
+		tmpl = ieee80211_get_fils_discovery_tmpl(hw, vif,
+							 info->link_id);
 		if (tmpl)
 			ret = ath12k_wmi_fils_discovery_tmpl(ar, arvif->vdev_id,
 							     tmpl);
@@ -4319,7 +4327,8 @@ static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif,
 		unsol_bcast_probe_resp_enabled = 1;
 		interval = info->unsol_bcast_probe_resp_interval;
 
-		tmpl = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif);
+		tmpl = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif,
+								 info->link_id);
 		if (tmpl)
 			ret = ath12k_wmi_probe_resp_tmpl(ar, arvif->vdev_id,
 							 tmpl);
@@ -6784,6 +6793,8 @@ static void ath12k_mac_free_unassign_link_sta(struct ath12k_hw *ah,
 		return;
 
 	ahsta->links_map &= ~BIT(link_id);
+	ahsta->free_logical_link_idx_map |= BIT(arsta->link_idx);
+
 	rcu_assign_pointer(ahsta->link[link_id], NULL);
 	synchronize_rcu();
 
@@ -7102,6 +7113,7 @@ static int ath12k_mac_assign_link_sta(struct ath12k_hw *ah,
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta);
 	struct ieee80211_link_sta *link_sta;
 	struct ath12k_link_vif *arvif;
+	int link_idx;
 
 	lockdep_assert_wiphy(ah->hw->wiphy);
 
@@ -7120,8 +7132,16 @@ static int ath12k_mac_assign_link_sta(struct ath12k_hw *ah,
 
 	ether_addr_copy(arsta->addr, link_sta->addr);
 
-	/* logical index of the link sta in order of creation */
-	arsta->link_idx = ahsta->num_peer++;
+	if (!ahsta->free_logical_link_idx_map)
+		return -ENOSPC;
+
+	/*
+	 * Allocate a logical link index by selecting the first available bit
+	 * from the free logical index map
+	 */
+	link_idx = __ffs(ahsta->free_logical_link_idx_map);
+	ahsta->free_logical_link_idx_map &= ~BIT(link_idx);
+	arsta->link_idx = link_idx;
 
 	arsta->link_id = link_id;
 	ahsta->links_map |= BIT(arsta->link_id);
@@ -7630,6 +7650,7 @@ int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 	if (old_state == IEEE80211_STA_NOTEXIST &&
 	    new_state == IEEE80211_STA_NONE) {
 		memset(ahsta, 0, sizeof(*ahsta));
+		ahsta->free_logical_link_idx_map = U16_MAX;
 
 		arsta = &ahsta->deflink;
 
@@ -9117,10 +9138,10 @@ static int ath12k_mac_mgmt_action_frame_fill_elem_data(struct ath12k_link_vif *a
 	lockdep_assert_wiphy(wiphy);
 
 	/* make sure category field is present */
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE)
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
 		return -EINVAL;
 
-	remaining_len = skb->len - IEEE80211_MIN_ACTION_SIZE;
+	remaining_len = skb->len - IEEE80211_MIN_ACTION_SIZE(category);
 	has_protected = ieee80211_has_protected(hdr->frame_control);
 
 	/* In case of SW crypto and hdr protected (PMF), packet will already be encrypted,
@@ -14791,6 +14812,10 @@ static void ath12k_mac_setup(struct ath12k *ar)
 	init_completion(&ar->mlo_setup_done);
 	init_completion(&ar->completed_11d_scan);
 	init_completion(&ar->regd_update_completed);
+	init_completion(&ar->thermal.wmi_sync);
+
+	ar->thermal.temperature = 0;
+	ar->thermal.hwmon_dev = NULL;
 
 	INIT_DELAYED_WORK(&ar->scan.timeout, ath12k_scan_timeout_work);
 	wiphy_work_init(&ar->scan.vdev_clean_wk, ath12k_scan_vdev_clean_work);
diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c
index c11b84b56f8fc..8c5dacf227da5 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.c
+++ b/drivers/net/wireless/ath/ath12k/qmi.c
@@ -3339,7 +3339,7 @@ static int ath12k_qmi_aux_uc_load(struct ath12k_base *ab)
 		goto out;
 	}
 
-	 aux_uc_mem->total_size = aux_uc_len;
+	aux_uc_mem->total_size = aux_uc_len;
 
 copy:
 	memcpy(aux_uc_mem->vaddr, aux_uc_data, aux_uc_len);
diff --git a/drivers/net/wireless/ath/ath12k/thermal.c b/drivers/net/wireless/ath/ath12k/thermal.c
new file mode 100644
index 0000000000000..a764d2112a3cf
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/thermal.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2020 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/device.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/sysfs.h>
+#include <linux/thermal.h>
+#include "core.h"
+#include "debug.h"
+
+static ssize_t ath12k_thermal_temp_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct ath12k *ar = dev_get_drvdata(dev);
+	unsigned long time_left;
+	int ret, temperature;
+
+	guard(wiphy)(ath12k_ar_to_hw(ar)->wiphy);
+
+	if (ar->ah->state != ATH12K_HW_STATE_ON)
+		return -ENETDOWN;
+
+	reinit_completion(&ar->thermal.wmi_sync);
+	ret = ath12k_wmi_send_pdev_temperature_cmd(ar);
+	if (ret) {
+		ath12k_warn(ar->ab, "failed to read temperature %d\n", ret);
+		return ret;
+	}
+
+	if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags))
+		return -ESHUTDOWN;
+
+	time_left = wait_for_completion_timeout(&ar->thermal.wmi_sync,
+						ATH12K_THERMAL_SYNC_TIMEOUT_HZ);
+	if (!time_left) {
+		ath12k_warn(ar->ab, "failed to synchronize thermal read\n");
+		return -ETIMEDOUT;
+	}
+
+	spin_lock_bh(&ar->data_lock);
+	temperature = ar->thermal.temperature;
+	spin_unlock_bh(&ar->data_lock);
+
+	/* display in millidegree celsius */
+	return sysfs_emit(buf, "%d\n", temperature * 1000);
+}
+
+void ath12k_thermal_event_temperature(struct ath12k *ar, int temperature)
+{
+	spin_lock_bh(&ar->data_lock);
+	ar->thermal.temperature = temperature;
+	spin_unlock_bh(&ar->data_lock);
+	complete_all(&ar->thermal.wmi_sync);
+}
+
+static SENSOR_DEVICE_ATTR_RO(temp1_input, ath12k_thermal_temp, 0);
+
+static struct attribute *ath12k_hwmon_attrs[] = {
+	&sensor_dev_attr_temp1_input.dev_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(ath12k_hwmon);
+
+int ath12k_thermal_register(struct ath12k_base *ab)
+{
+	struct ath12k *ar;
+	int i, j, ret;
+
+	if (!IS_REACHABLE(CONFIG_HWMON))
+		return 0;
+
+	for (i = 0; i < ab->num_radios; i++) {
+		ar = ab->pdevs[i].ar;
+		if (!ar)
+			continue;
+
+		ar->thermal.hwmon_dev =
+			hwmon_device_register_with_groups(&ar->ah->hw->wiphy->dev,
+							  "ath12k_hwmon", ar,
+							  ath12k_hwmon_groups);
+		if (IS_ERR(ar->thermal.hwmon_dev)) {
+			ret = PTR_ERR(ar->thermal.hwmon_dev);
+			ar->thermal.hwmon_dev = NULL;
+			ath12k_err(ar->ab, "failed to register hwmon device: %d\n",
+				   ret);
+			for (j = i - 1; j >= 0; j--) {
+				ar = ab->pdevs[j].ar;
+				if (!ar)
+					continue;
+
+				hwmon_device_unregister(ar->thermal.hwmon_dev);
+				ar->thermal.hwmon_dev = NULL;
+			}
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+void ath12k_thermal_unregister(struct ath12k_base *ab)
+{
+	struct ath12k *ar;
+	int i;
+
+	if (!IS_REACHABLE(CONFIG_HWMON))
+		return;
+
+	for (i = 0; i < ab->num_radios; i++) {
+		ar = ab->pdevs[i].ar;
+		if (!ar)
+			continue;
+
+		if (ar->thermal.hwmon_dev) {
+			hwmon_device_unregister(ar->thermal.hwmon_dev);
+			ar->thermal.hwmon_dev = NULL;
+		}
+	}
+}
diff --git a/drivers/net/wireless/ath/ath12k/thermal.h b/drivers/net/wireless/ath/ath12k/thermal.h
new file mode 100644
index 0000000000000..9d84056188e1e
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/thermal.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2020 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _ATH12K_THERMAL_
+#define _ATH12K_THERMAL_
+
+#define ATH12K_THERMAL_SYNC_TIMEOUT_HZ (5 * HZ)
+
+struct ath12k_thermal {
+	struct completion wmi_sync;
+
+	/* temperature value in Celsius degree protected by data_lock. */
+	int temperature;
+	struct device *hwmon_dev;
+};
+
+#if IS_REACHABLE(CONFIG_THERMAL)
+int ath12k_thermal_register(struct ath12k_base *ab);
+void ath12k_thermal_unregister(struct ath12k_base *ab);
+void ath12k_thermal_event_temperature(struct ath12k *ar, int temperature);
+#else
+static inline int ath12k_thermal_register(struct ath12k_base *ab)
+{
+	return 0;
+}
+
+static inline void ath12k_thermal_unregister(struct ath12k_base *ab)
+{
+}
+
+static inline void ath12k_thermal_event_temperature(struct ath12k *ar,
+						    int temperature)
+{
+}
+
+#endif
+#endif /* _ATH12K_THERMAL_ */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
index c9cea597a92eb..77f5d23be78d6 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
@@ -405,6 +405,42 @@ ath12k_wifi7_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *r
 	}
 }
 
+static __always_inline u8
+ath12k_wifi7_hal_mon_map_legacy_rate_to_hw_rate(u8 rate)
+{
+	u8 ath12k_rate;
+
+	/* Map hal_rx_legacy_rate to ath12k_hw_rate_cck */
+	switch (rate) {
+	case HAL_RX_LEGACY_RATE_LP_1_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_LP_1M;
+		break;
+	case HAL_RX_LEGACY_RATE_LP_2_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_LP_2M;
+		break;
+	case HAL_RX_LEGACY_RATE_LP_5_5_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_LP_5_5M;
+		break;
+	case HAL_RX_LEGACY_RATE_LP_11_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_LP_11M;
+		break;
+	case HAL_RX_LEGACY_RATE_SP_2_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_SP_2M;
+		break;
+	case HAL_RX_LEGACY_RATE_SP_5_5_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_SP_5_5M;
+		break;
+	case HAL_RX_LEGACY_RATE_SP_11_MBPS:
+		ath12k_rate = ATH12K_HW_RATE_CCK_SP_11M;
+		break;
+	default:
+		ath12k_rate = rate;
+		break;
+	}
+
+	return ath12k_rate;
+}
+
 static void
 ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb,
 				  struct hal_rx_mon_ppdu_info *ppdu_info)
@@ -415,25 +451,32 @@ ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb,
 	rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE);
 	switch (rate) {
 	case 1:
-		rate = HAL_RX_LEGACY_RATE_1_MBPS;
+		rate = HAL_RX_LEGACY_RATE_LP_1_MBPS;
 		break;
 	case 2:
-	case 5:
-		rate = HAL_RX_LEGACY_RATE_2_MBPS;
+		rate = HAL_RX_LEGACY_RATE_LP_2_MBPS;
 		break;
 	case 3:
-	case 6:
-		rate = HAL_RX_LEGACY_RATE_5_5_MBPS;
+		rate = HAL_RX_LEGACY_RATE_LP_5_5_MBPS;
 		break;
 	case 4:
+		rate = HAL_RX_LEGACY_RATE_LP_11_MBPS;
+		break;
+	case 5:
+		rate = HAL_RX_LEGACY_RATE_SP_2_MBPS;
+		break;
+	case 6:
+		rate = HAL_RX_LEGACY_RATE_SP_5_5_MBPS;
+		break;
 	case 7:
-		rate = HAL_RX_LEGACY_RATE_11_MBPS;
+		rate = HAL_RX_LEGACY_RATE_SP_11_MBPS;
 		break;
 	default:
 		rate = HAL_RX_LEGACY_RATE_INVALID;
+		break;
 	}
 
-	ppdu_info->rate = rate;
+	ppdu_info->rate = ath12k_wifi7_hal_mon_map_legacy_rate_to_hw_rate(rate);
 	ppdu_info->cck_flag = 1;
 }
 
@@ -447,31 +490,32 @@ ath12k_wifi7_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga,
 	rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE);
 	switch (rate) {
 	case 8:
-		rate = HAL_RX_LEGACY_RATE_48_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_48_MBPS;
 		break;
 	case 9:
-		rate = HAL_RX_LEGACY_RATE_24_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_24_MBPS;
 		break;
 	case 10:
-		rate = HAL_RX_LEGACY_RATE_12_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_12_MBPS;
 		break;
 	case 11:
-		rate = HAL_RX_LEGACY_RATE_6_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_6_MBPS;
 		break;
 	case 12:
-		rate = HAL_RX_LEGACY_RATE_54_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_54_MBPS;
 		break;
 	case 13:
-		rate = HAL_RX_LEGACY_RATE_36_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_36_MBPS;
 		break;
 	case 14:
-		rate = HAL_RX_LEGACY_RATE_18_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_18_MBPS;
 		break;
 	case 15:
-		rate = HAL_RX_LEGACY_RATE_9_MBPS;
+		rate = HAL_RX_LEGACY_RATE_OFDM_9_MBPS;
 		break;
 	default:
-		rate = HAL_RX_LEGACY_RATE_INVALID;
+		rate = HAL_RX_LEGACY_RATE_OFDM_INVALID;
+		break;
 	}
 
 	ppdu_info->rate = rate;
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
index 038e339e8fb1c..e6a934d74e85d 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
@@ -325,7 +325,6 @@ static void ath12k_wifi7_dp_rx_h_csum_offload(struct sk_buff *msdu,
 
 static void ath12k_wifi7_dp_rx_h_mpdu(struct ath12k_pdev_dp *dp_pdev,
 				      struct sk_buff *msdu,
-				      struct hal_rx_desc *rx_desc,
 				      struct hal_rx_desc_data *rx_info)
 {
 	struct ath12k_skb_rxcb *rxcb;
@@ -388,8 +387,7 @@ static void ath12k_wifi7_dp_rx_h_mpdu(struct ath12k_pdev_dp *dp_pdev,
 	}
 
 	ath12k_wifi7_dp_rx_h_csum_offload(msdu, rx_info);
-	ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc,
-			       enctype, is_decrypted, rx_info);
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, enctype, is_decrypted, rx_info);
 
 	if (!is_decrypted || rx_info->is_mcbc)
 		return;
@@ -549,14 +547,14 @@ static int ath12k_wifi7_dp_rx_process_msdu(struct ath12k_pdev_dp *dp_pdev,
 		}
 	}
 
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu,
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu,
 							     rx_info))) {
 		ret = -EINVAL;
 		goto free_out;
 	}
 
 	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
-	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_desc, rx_info);
+	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_info);
 
 	rx_info->rx_status->flag |= RX_FLAG_SKIP_MONITOR | RX_FLAG_DUP_VALIDATED;
 
@@ -1030,13 +1028,13 @@ static int ath12k_wifi7_dp_rx_h_verify_tkip_mic(struct ath12k_pdev_dp *dp_pdev,
 		    RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED;
 	skb_pull(msdu, hal_rx_desc_sz);
 
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu,
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu,
 							     rx_info)))
 		return -EINVAL;
 
 	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
-	ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc,
-			       HAL_ENCRYPT_TYPE_TKIP_MIC, true, rx_info);
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, HAL_ENCRYPT_TYPE_TKIP_MIC, true,
+			       rx_info);
 	ieee80211_rx(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
 	return -EINVAL;
 }
@@ -1588,7 +1586,6 @@ static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev,
 	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ath12k_base *ab = dp->ab;
 	u16 msdu_len = rx_info->msdu_len;
-	struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data;
 	u8 l3pad_bytes = rx_info->l3_pad_bytes;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz;
@@ -1632,11 +1629,11 @@ static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev,
 		skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
 		skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
 	}
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info)))
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu, rx_info)))
 		return -EINVAL;
 
 	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
-	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, desc, rx_info);
+	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_info);
 
 	rxcb->tid = rx_info->tid;
 
@@ -1673,7 +1670,7 @@ static bool ath12k_wifi7_dp_rx_h_tkip_mic_err(struct ath12k_pdev_dp *dp_pdev,
 	skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
 	skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
 
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info)))
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu, rx_info)))
 		return true;
 
 	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
@@ -1681,8 +1678,8 @@ static bool ath12k_wifi7_dp_rx_h_tkip_mic_err(struct ath12k_pdev_dp *dp_pdev,
 	rx_info->rx_status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR |
 				     RX_FLAG_DECRYPTED);
 
-	ath12k_dp_rx_h_undecap(dp_pdev, msdu, desc,
-			       HAL_ENCRYPT_TYPE_TKIP_MIC, false, rx_info);
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, HAL_ENCRYPT_TYPE_TKIP_MIC, false,
+			       rx_info);
 	return false;
 }
 
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
index df045ddf42da9..ec6dba96640bb 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hw.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
@@ -104,7 +104,7 @@ static bool ath12k_is_addba_resp_action_code(struct ieee80211_mgmt *mgmt)
 	if (mgmt->u.action.category != WLAN_CATEGORY_BACK)
 		return false;
 
-	if (mgmt->u.action.u.addba_resp.action_code != WLAN_ACTION_ADDBA_RESP)
+	if (mgmt->u.action.action_code != WLAN_ACTION_ADDBA_RESP)
 		return false;
 
 	return true;
@@ -617,7 +617,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 			ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332,
 		.svc_to_ce_map_len = 18,
 
-		.rxdma1_enable = false,
+		.rxdma1_enable = true,
 		.num_rxdma_per_pdev = 1,
 		.num_rxdma_dst_ring = 0,
 		.rx_mac_buf_ring = false,
@@ -626,7 +626,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
 				   BIT(NL80211_IFTYPE_AP) |
 				   BIT(NL80211_IFTYPE_MESH_POINT),
-		.supports_monitor = false,
+		.supports_monitor = true,
 
 		.idle_ps = false,
 		.download_calib = true,
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index 48fee9346de8c..65a05a9520ffc 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -6778,31 +6778,6 @@ static int ath12k_pull_peer_assoc_conf_ev(struct ath12k_base *ab, struct sk_buff
 	return 0;
 }
 
-static int
-ath12k_pull_pdev_temp_ev(struct ath12k_base *ab, struct sk_buff *skb,
-			 const struct wmi_pdev_temperature_event *ev)
-{
-	const void **tb;
-	int ret;
-
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
-	if (IS_ERR(tb)) {
-		ret = PTR_ERR(tb);
-		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
-		return ret;
-	}
-
-	ev = tb[WMI_TAG_PDEV_TEMPERATURE_EVENT];
-	if (!ev) {
-		ath12k_warn(ab, "failed to fetch pdev temp ev");
-		kfree(tb);
-		return -EPROTO;
-	}
-
-	kfree(tb);
-	return 0;
-}
-
 static void ath12k_wmi_op_ep_tx_credits(struct ath12k_base *ab)
 {
 	/* try to send pending beacons first. they take priority */
@@ -8801,25 +8776,45 @@ static void
 ath12k_wmi_pdev_temperature_event(struct ath12k_base *ab,
 				  struct sk_buff *skb)
 {
+	const struct wmi_pdev_temperature_event *ev;
 	struct ath12k *ar;
-	struct wmi_pdev_temperature_event ev = {};
+	const void **tb;
+	int temp;
+	u32 pdev_id;
+
+	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	if (IS_ERR(tb)) {
+		ath12k_warn(ab, "failed to parse tlv: %ld\n", PTR_ERR(tb));
+		return;
+	}
 
-	if (ath12k_pull_pdev_temp_ev(ab, skb, &ev) != 0) {
-		ath12k_warn(ab, "failed to extract pdev temperature event");
+	ev = tb[WMI_TAG_PDEV_TEMPERATURE_EVENT];
+	if (!ev) {
+		ath12k_warn(ab, "failed to fetch pdev temp ev\n");
+		kfree(tb);
 		return;
 	}
 
+	temp = a_sle32_to_cpu(ev->temp);
+	pdev_id = le32_to_cpu(ev->pdev_id);
+
+	kfree(tb);
+
 	ath12k_dbg(ab, ATH12K_DBG_WMI,
-		   "pdev temperature ev temp %d pdev_id %d\n", ev.temp, ev.pdev_id);
+		   "pdev temperature ev temp %d pdev_id %u\n",
+		   temp, pdev_id);
 
 	rcu_read_lock();
 
-	ar = ath12k_mac_get_ar_by_pdev_id(ab, le32_to_cpu(ev.pdev_id));
+	ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id);
 	if (!ar) {
-		ath12k_warn(ab, "invalid pdev id in pdev temperature ev %d", ev.pdev_id);
+		ath12k_warn(ab, "invalid pdev id %u in pdev temperature ev\n",
+			    pdev_id);
 		goto exit;
 	}
 
+	ath12k_thermal_event_temperature(ar, temp);
+
 exit:
 	rcu_read_unlock();
 }
@@ -10022,50 +10017,46 @@ static int ath12k_connect_pdev_htc_service(struct ath12k_base *ab,
 
 static int
 ath12k_wmi_send_unit_test_cmd(struct ath12k *ar,
-			      struct wmi_unit_test_cmd ut_cmd,
-			      u32 *test_args)
+			      const struct wmi_unit_test_arg *ut)
 {
 	struct ath12k_wmi_pdev *wmi = ar->wmi;
 	struct wmi_unit_test_cmd *cmd;
+	int buf_len, arg_len;
 	struct sk_buff *skb;
 	struct wmi_tlv *tlv;
+	__le32 *ut_cmd_args;
 	void *ptr;
-	u32 *ut_cmd_args;
-	int buf_len, arg_len;
 	int ret;
 	int i;
 
-	arg_len = sizeof(u32) * le32_to_cpu(ut_cmd.num_args);
-	buf_len = sizeof(ut_cmd) + arg_len + TLV_HDR_SIZE;
+	arg_len = sizeof(*ut_cmd_args) * ut->num_args;
+	buf_len = sizeof(*cmd) + arg_len + TLV_HDR_SIZE;
 
 	skb = ath12k_wmi_alloc_skb(wmi->wmi_ab, buf_len);
 	if (!skb)
 		return -ENOMEM;
 
-	cmd = (struct wmi_unit_test_cmd *)skb->data;
+	ptr = skb->data;
+	cmd = ptr;
 	cmd->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_UNIT_TEST_CMD,
-						 sizeof(ut_cmd));
-
-	cmd->vdev_id = ut_cmd.vdev_id;
-	cmd->module_id = ut_cmd.module_id;
-	cmd->num_args = ut_cmd.num_args;
-	cmd->diag_token = ut_cmd.diag_token;
-
-	ptr = skb->data + sizeof(ut_cmd);
+						 sizeof(*cmd));
+	cmd->vdev_id = cpu_to_le32(ut->vdev_id);
+	cmd->module_id = cpu_to_le32(ut->module_id);
+	cmd->num_args = cpu_to_le32(ut->num_args);
+	cmd->diag_token = cpu_to_le32(ut->diag_token);
 
+	ptr += sizeof(*cmd);
 	tlv = ptr;
 	tlv->header = ath12k_wmi_tlv_hdr(WMI_TAG_ARRAY_UINT32, arg_len);
 
 	ptr += TLV_HDR_SIZE;
-
 	ut_cmd_args = ptr;
-	for (i = 0; i < le32_to_cpu(ut_cmd.num_args); i++)
-		ut_cmd_args[i] = test_args[i];
+	for (i = 0; i < ut->num_args; i++)
+		ut_cmd_args[i] = cpu_to_le32(ut->args[i]);
 
 	ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
 		   "WMI unit test : module %d vdev %d n_args %d token %d\n",
-		   cmd->module_id, cmd->vdev_id, cmd->num_args,
-		   cmd->diag_token);
+		   ut->module_id, ut->vdev_id, ut->num_args, ut->diag_token);
 
 	ret = ath12k_wmi_cmd_send(wmi, skb, WMI_UNIT_TEST_CMDID);
 
@@ -10081,8 +10072,7 @@ ath12k_wmi_send_unit_test_cmd(struct ath12k *ar,
 int ath12k_wmi_simulate_radar(struct ath12k *ar)
 {
 	struct ath12k_link_vif *arvif;
-	u32 dfs_args[DFS_MAX_TEST_ARGS];
-	struct wmi_unit_test_cmd wmi_ut;
+	struct wmi_unit_test_arg wmi_ut = {};
 	bool arvif_found = false;
 
 	list_for_each_entry(arvif, &ar->arvifs, list) {
@@ -10095,22 +10085,23 @@ int ath12k_wmi_simulate_radar(struct ath12k *ar)
 	if (!arvif_found)
 		return -EINVAL;
 
-	dfs_args[DFS_TEST_CMDID] = 0;
-	dfs_args[DFS_TEST_PDEV_ID] = ar->pdev->pdev_id;
-	/* Currently we could pass segment_id(b0 - b1), chirp(b2)
+	wmi_ut.args[DFS_TEST_CMDID] = 0;
+	wmi_ut.args[DFS_TEST_PDEV_ID] = ar->pdev->pdev_id;
+	/*
+	 * Currently we could pass segment_id(b0 - b1), chirp(b2)
 	 * freq offset (b3 - b10) to unit test. For simulation
 	 * purpose this can be set to 0 which is valid.
 	 */
-	dfs_args[DFS_TEST_RADAR_PARAM] = 0;
+	wmi_ut.args[DFS_TEST_RADAR_PARAM] = 0;
 
-	wmi_ut.vdev_id = cpu_to_le32(arvif->vdev_id);
-	wmi_ut.module_id = cpu_to_le32(DFS_UNIT_TEST_MODULE);
-	wmi_ut.num_args = cpu_to_le32(DFS_MAX_TEST_ARGS);
-	wmi_ut.diag_token = cpu_to_le32(DFS_UNIT_TEST_TOKEN);
+	wmi_ut.vdev_id = arvif->vdev_id;
+	wmi_ut.module_id = DFS_UNIT_TEST_MODULE;
+	wmi_ut.num_args = DFS_MAX_TEST_ARGS;
+	wmi_ut.diag_token = DFS_UNIT_TEST_TOKEN;
 
 	ath12k_dbg(ar->ab, ATH12K_DBG_REG, "Triggering Radar Simulation\n");
 
-	return ath12k_wmi_send_unit_test_cmd(ar, wmi_ut, dfs_args);
+	return ath12k_wmi_send_unit_test_cmd(ar, &wmi_ut);
 }
 
 int ath12k_wmi_send_tpc_stats_request(struct ath12k *ar,
diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h
index 0bf0a7941cd3c..5ba9b7d3a8888 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.h
+++ b/drivers/net/wireless/ath/ath12k/wmi.h
@@ -4193,7 +4193,6 @@ struct wmi_addba_clear_resp_cmd {
 	struct ath12k_wmi_mac_addr_params peer_macaddr;
 } __packed;
 
-#define DFS_PHYERR_UNIT_TEST_CMD 0
 #define DFS_UNIT_TEST_MODULE	0x2b
 #define DFS_UNIT_TEST_TOKEN	0xAA
 
@@ -4204,10 +4203,15 @@ enum dfs_test_args_idx {
 	DFS_MAX_TEST_ARGS,
 };
 
-struct wmi_dfs_unit_test_arg {
-	u32 cmd_id;
-	u32 pdev_id;
-	u32 radar_param;
+/* update if another test command requires more */
+#define WMI_UNIT_TEST_ARGS_MAX DFS_MAX_TEST_ARGS
+
+struct wmi_unit_test_arg {
+	u32 vdev_id;
+	u32 module_id;
+	u32 diag_token;
+	u32 num_args;
+	u32 args[WMI_UNIT_TEST_ARGS_MAX];
 };
 
 struct wmi_unit_test_cmd {
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index 88f0197fc041a..739a24a6ad676 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1123,13 +1123,13 @@ void ath6kl_cfg80211_ch_switch_notify(struct ath6kl_vif *vif, int freq,
 	wiphy_unlock(vif->ar->wiphy);
 }
 
-static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
+static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr,
 				   struct key_params *params)
 {
-	struct ath6kl *ar = ath6kl_priv(ndev);
-	struct ath6kl_vif *vif = netdev_priv(ndev);
+	struct ath6kl *ar = ath6kl_priv(wdev->netdev);
+	struct ath6kl_vif *vif = netdev_priv(wdev->netdev);
 	struct ath6kl_key *key = NULL;
 	int seq_len;
 	u8 key_usage;
@@ -1248,12 +1248,12 @@ static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
 				     (u8 *) mac_addr, SYNC_BOTH_WMIFLAG);
 }
 
-static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
+static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr)
 {
-	struct ath6kl *ar = ath6kl_priv(ndev);
-	struct ath6kl_vif *vif = netdev_priv(ndev);
+	struct ath6kl *ar = ath6kl_priv(wdev->netdev);
+	struct ath6kl_vif *vif = netdev_priv(wdev->netdev);
 
 	ath6kl_dbg(ATH6KL_DBG_WLAN_CFG, "%s: index %d\n", __func__, key_index);
 
@@ -1278,13 +1278,13 @@ static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
 	return ath6kl_wmi_deletekey_cmd(ar->wmi, vif->fw_vif_idx, key_index);
 }
 
-static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
+static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr, void *cookie,
 				   void (*callback) (void *cookie,
 						     struct key_params *))
 {
-	struct ath6kl_vif *vif = netdev_priv(ndev);
+	struct ath6kl_vif *vif = netdev_priv(wdev->netdev);
 	struct ath6kl_key *key = NULL;
 	struct key_params params;
 
@@ -1775,9 +1775,10 @@ static bool is_rate_ht40(s32 rate, u8 *mcs, bool *sgi)
 	return false;
 }
 
-static int ath6kl_get_station(struct wiphy *wiphy, struct net_device *dev,
+static int ath6kl_get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			      const u8 *mac, struct station_info *sinfo)
 {
+	struct net_device *dev = wdev->netdev;
 	struct ath6kl *ar = ath6kl_priv(dev);
 	struct ath6kl_vif *vif = netdev_priv(dev);
 	long left;
@@ -2992,9 +2993,10 @@ static int ath6kl_stop_ap(struct wiphy *wiphy, struct net_device *dev,
 
 static const u8 bcast_addr[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
-static int ath6kl_del_station(struct wiphy *wiphy, struct net_device *dev,
+static int ath6kl_del_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			      struct station_del_parameters *params)
 {
+	struct net_device *dev = wdev->netdev;
 	struct ath6kl *ar = ath6kl_priv(dev);
 	struct ath6kl_vif *vif = netdev_priv(dev);
 	const u8 *addr = params->mac ? params->mac : bcast_addr;
@@ -3003,10 +3005,11 @@ static int ath6kl_del_station(struct wiphy *wiphy, struct net_device *dev,
 				      addr, WLAN_REASON_PREV_AUTH_NOT_VALID);
 }
 
-static int ath6kl_change_station(struct wiphy *wiphy, struct net_device *dev,
+static int ath6kl_change_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 				 const u8 *mac,
 				 struct station_parameters *params)
 {
+	struct net_device *dev = wdev->netdev;
 	struct ath6kl *ar = ath6kl_priv(dev);
 	struct ath6kl_vif *vif = netdev_priv(dev);
 	int err;
diff --git a/drivers/net/wireless/ath/ath6kl/main.c b/drivers/net/wireless/ath/ath6kl/main.c
index 85d6ad53cf94b..8afc6589fc51e 100644
--- a/drivers/net/wireless/ath/ath6kl/main.c
+++ b/drivers/net/wireless/ath/ath6kl/main.c
@@ -494,7 +494,7 @@ void ath6kl_connect_ap_mode_sta(struct ath6kl_vif *vif, u16 aid, u8 *mac_addr,
 	sinfo->assoc_req_ies = ies;
 	sinfo->assoc_req_ies_len = ies_len;
 
-	cfg80211_new_sta(vif->ndev, mac_addr, sinfo, GFP_KERNEL);
+	cfg80211_new_sta(&vif->wdev, mac_addr, sinfo, GFP_KERNEL);
 
 	netif_wake_queue(vif->ndev);
 
@@ -1011,7 +1011,7 @@ void ath6kl_disconnect_event(struct ath6kl_vif *vif, u8 reason, u8 *bssid,
 
 		if (!is_broadcast_ether_addr(bssid)) {
 			/* send event to application */
-			cfg80211_del_sta(vif->ndev, bssid, GFP_KERNEL);
+			cfg80211_del_sta(&vif->wdev, bssid, GFP_KERNEL);
 		}
 
 		if (memcmp(vif->ndev->dev_addr, bssid, ETH_ALEN) == 0) {
diff --git a/drivers/net/wireless/ath/ath6kl/usb.c b/drivers/net/wireless/ath/ath6kl/usb.c
index 852e77e41bde5..79c18f5ee02bc 100644
--- a/drivers/net/wireless/ath/ath6kl/usb.c
+++ b/drivers/net/wireless/ath/ath6kl/usb.c
@@ -1124,8 +1124,6 @@ static int ath6kl_usb_probe(struct usb_interface *interface,
 	int vendor_id, product_id;
 	int ret = 0;
 
-	usb_get_dev(dev);
-
 	vendor_id = le16_to_cpu(dev->descriptor.idVendor);
 	product_id = le16_to_cpu(dev->descriptor.idProduct);
 
@@ -1143,11 +1141,8 @@ static int ath6kl_usb_probe(struct usb_interface *interface,
 		ath6kl_dbg(ATH6KL_DBG_USB, "USB 1.1 Host\n");
 
 	ar_usb = ath6kl_usb_create(interface);
-
-	if (ar_usb == NULL) {
-		ret = -ENOMEM;
-		goto err_usb_put;
-	}
+	if (ar_usb == NULL)
+		return -ENOMEM;
 
 	ar = ath6kl_core_create(&ar_usb->udev->dev);
 	if (ar == NULL) {
@@ -1176,15 +1171,12 @@ static int ath6kl_usb_probe(struct usb_interface *interface,
 	ath6kl_core_destroy(ar);
 err_usb_destroy:
 	ath6kl_usb_destroy(ar_usb);
-err_usb_put:
-	usb_put_dev(dev);
 
 	return ret;
 }
 
-static void ath6kl_usb_remove(struct usb_interface *interface)
+static void ath6kl_usb_disconnect(struct usb_interface *interface)
 {
-	usb_put_dev(interface_to_usbdev(interface));
 	ath6kl_usb_device_detached(interface);
 }
 
@@ -1235,7 +1227,7 @@ static struct usb_driver ath6kl_usb_driver = {
 	.probe = ath6kl_usb_probe,
 	.suspend = ath6kl_usb_pm_suspend,
 	.resume = ath6kl_usb_pm_resume,
-	.disconnect = ath6kl_usb_remove,
+	.disconnect = ath6kl_usb_disconnect,
 	.id_table = ath6kl_usb_ids,
 	.supports_autosuspend = true,
 	.disable_hub_initiated_lpm = 1,
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.h b/drivers/net/wireless/ath/ath6kl/wmi.h
index 3080d82e25cc2..8fbece3fdad9f 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.h
+++ b/drivers/net/wireless/ath/ath6kl/wmi.h
@@ -1630,16 +1630,6 @@ enum wmi_roam_mode {
 	WMI_LOCK_BSS_MODE = 3, /* Lock to the current BSS */
 };
 
-struct bss_bias {
-	u8 bssid[ETH_ALEN];
-	s8 bias;
-} __packed;
-
-struct bss_bias_info {
-	u8 num_bss;
-	struct bss_bias bss_bias[];
-} __packed;
-
 struct low_rssi_scan_params {
 	__le16 lrssi_scan_period;
 	a_sle16 lrssi_scan_threshold;
@@ -1652,7 +1642,6 @@ struct roam_ctrl_cmd {
 	union {
 		u8 bssid[ETH_ALEN]; /* WMI_FORCE_ROAM */
 		u8 roam_mode; /* WMI_SET_ROAM_MODE */
-		struct bss_bias_info bss; /* WMI_SET_HOST_BIAS */
 		struct low_rssi_scan_params params; /* WMI_SET_LRSSI_SCAN_PARAMS
 						     */
 	} __packed info;
diff --git a/drivers/net/wireless/ath/ath9k/ath9k.h b/drivers/net/wireless/ath/ath9k/ath9k.h
index 6e38aa7351e33..e8635bf81f9d0 100644
--- a/drivers/net/wireless/ath/ath9k/ath9k.h
+++ b/drivers/net/wireless/ath/ath9k/ath9k.h
@@ -443,7 +443,7 @@ ath_node_to_tid(struct ath_node *an, u8 tidno)
 
 #define case_rtn_string(val) case val: return #val
 
-#define ath_for_each_chanctx(_sc, _ctx)                             \
+#define ath_for_each_chanctx(sc, ctx)                               \
 	for (ctx = &sc->chanctx[0];                                 \
 	     ctx <= &sc->chanctx[ARRAY_SIZE(sc->chanctx) - 1];      \
 	     ctx++)
diff --git a/drivers/net/wireless/ath/ath9k/channel.c b/drivers/net/wireless/ath/ath9k/channel.c
index 121e51ce1bc0e..8b27d8cc086ab 100644
--- a/drivers/net/wireless/ath/ath9k/channel.c
+++ b/drivers/net/wireless/ath/ath9k/channel.c
@@ -1006,7 +1006,7 @@ static void ath_scan_send_probe(struct ath_softc *sc,
 	skb_set_queue_mapping(skb, IEEE80211_AC_VO);
 
 	if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, NULL))
-		goto error;
+		return;
 
 	txctl.txq = sc->tx.txq_map[IEEE80211_AC_VO];
 	if (ath_tx_start(sc->hw, skb, &txctl))
@@ -1119,10 +1119,8 @@ ath_chanctx_send_vif_ps_frame(struct ath_softc *sc, struct ath_vif *avp,
 
 		skb->priority = 7;
 		skb_set_queue_mapping(skb, IEEE80211_AC_VO);
-		if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta)) {
-			dev_kfree_skb_any(skb);
+		if (!ieee80211_tx_prepare_skb(sc->hw, vif, skb, band, &sta))
 			return false;
-		}
 		break;
 	default:
 		return false;
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index 8533b88974b20..821909b81ea91 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -1382,8 +1382,6 @@ static int ath9k_hif_usb_probe(struct usb_interface *interface,
 		goto err_alloc;
 	}
 
-	usb_get_dev(udev);
-
 	hif_dev->udev = udev;
 	hif_dev->interface = interface;
 	hif_dev->usb_device_id = id;
@@ -1403,7 +1401,6 @@ static int ath9k_hif_usb_probe(struct usb_interface *interface,
 err_fw_req:
 	usb_set_intfdata(interface, NULL);
 	kfree(hif_dev);
-	usb_put_dev(udev);
 err_alloc:
 	return ret;
 }
@@ -1451,7 +1448,6 @@ static void ath9k_hif_usb_disconnect(struct usb_interface *interface)
 
 	kfree(hif_dev);
 	dev_info(&udev->dev, "ath9k_htc: USB layer deinitialized\n");
-	usb_put_dev(udev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index c74f5e66166df..3d6e5aad48b1a 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -533,11 +533,11 @@ int wil_cid_fill_sinfo(struct wil6210_vif *vif, int cid,
 }
 
 static int wil_cfg80211_get_station(struct wiphy *wiphy,
-				    struct net_device *ndev,
+				    struct wireless_dev *wdev,
 				    const u8 *mac, struct station_info *sinfo)
 {
-	struct wil6210_vif *vif = ndev_to_vif(ndev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	int rc;
 
 	int cid = wil_find_cid(wil, vif->mid, mac);
@@ -573,11 +573,11 @@ int wil_find_cid_by_idx(struct wil6210_priv *wil, u8 mid, int idx)
 }
 
 static int wil_cfg80211_dump_station(struct wiphy *wiphy,
-				     struct net_device *dev, int idx,
+				     struct wireless_dev *wdev, int idx,
 				     u8 *mac, struct station_info *sinfo)
 {
-	struct wil6210_vif *vif = ndev_to_vif(dev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	int rc;
 	int cid = wil_find_cid_by_idx(wil, vif->mid, idx);
 
@@ -1619,15 +1619,14 @@ static void wil_del_rx_key(u8 key_index, enum wmi_key_usage key_usage,
 }
 
 static int wil_cfg80211_add_key(struct wiphy *wiphy,
-				struct net_device *ndev, int link_id,
+				struct wireless_dev *wdev, int link_id,
 				u8 key_index, bool pairwise,
 				const u8 *mac_addr,
 				struct key_params *params)
 {
 	int rc;
-	struct wil6210_vif *vif = ndev_to_vif(ndev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-	struct wireless_dev *wdev = vif_to_wdev(vif);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	enum wmi_key_usage key_usage = wil_detect_key_usage(wdev, pairwise);
 	struct wil_sta_info *cs = wil_find_sta_by_key_usage(wil, vif->mid,
 							    key_usage,
@@ -1695,13 +1694,12 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy,
 }
 
 static int wil_cfg80211_del_key(struct wiphy *wiphy,
-				struct net_device *ndev, int link_id,
+				struct wireless_dev *wdev, int link_id,
 				u8 key_index, bool pairwise,
 				const u8 *mac_addr)
 {
-	struct wil6210_vif *vif = ndev_to_vif(ndev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-	struct wireless_dev *wdev = vif_to_wdev(vif);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	enum wmi_key_usage key_usage = wil_detect_key_usage(wdev, pairwise);
 	struct wil_sta_info *cs = wil_find_sta_by_key_usage(wil, vif->mid,
 							    key_usage,
@@ -2071,7 +2069,8 @@ void wil_cfg80211_ap_recovery(struct wil6210_priv *wil)
 		key_params.key = vif->gtk;
 		key_params.key_len = vif->gtk_len;
 		key_params.seq_len = IEEE80211_GCMP_PN_LEN;
-		rc = wil_cfg80211_add_key(wiphy, ndev, -1, vif->gtk_index,
+		rc = wil_cfg80211_add_key(wiphy, vif_to_wdev(vif), -1,
+					  vif->gtk_index,
 					  false, NULL, &key_params);
 		if (rc)
 			wil_err(wil, "vif %d recovery add key failed (%d)\n",
@@ -2225,12 +2224,12 @@ static int wil_cfg80211_stop_ap(struct wiphy *wiphy,
 }
 
 static int wil_cfg80211_add_station(struct wiphy *wiphy,
-				    struct net_device *dev,
+				    struct wireless_dev *wdev,
 				    const u8 *mac,
 				    struct station_parameters *params)
 {
-	struct wil6210_vif *vif = ndev_to_vif(dev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 
 	wil_dbg_misc(wil, "add station %pM aid %d mid %d mask 0x%x set 0x%x\n",
 		     mac, params->aid, vif->mid,
@@ -2250,11 +2249,11 @@ static int wil_cfg80211_add_station(struct wiphy *wiphy,
 }
 
 static int wil_cfg80211_del_station(struct wiphy *wiphy,
-				    struct net_device *dev,
+				    struct wireless_dev *wdev,
 				    struct station_del_parameters *params)
 {
-	struct wil6210_vif *vif = ndev_to_vif(dev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 
 	wil_dbg_misc(wil, "del_station: %pM, reason=%d mid=%d\n",
 		     params->mac, params->reason_code, vif->mid);
@@ -2267,12 +2266,12 @@ static int wil_cfg80211_del_station(struct wiphy *wiphy,
 }
 
 static int wil_cfg80211_change_station(struct wiphy *wiphy,
-				       struct net_device *dev,
+				       struct wireless_dev *wdev,
 				       const u8 *mac,
 				       struct station_parameters *params)
 {
-	struct wil6210_vif *vif = ndev_to_vif(dev);
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	int authorize;
 	int cid, i;
 	struct wil_ring_tx_data *txdata = NULL;
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index 44c24c6c8360b..d5aec72ecdce0 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -245,7 +245,6 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock)
 {
 	uint i;
 	struct wil6210_priv *wil = vif_to_wil(vif);
-	struct net_device *ndev = vif_to_ndev(vif);
 	struct wireless_dev *wdev = vif_to_wdev(vif);
 	struct wil_sta_info *sta = &wil->sta[cid];
 	int min_ring_id = wil_get_min_tx_ring_id(wil);
@@ -265,7 +264,7 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock)
 		case NL80211_IFTYPE_AP:
 		case NL80211_IFTYPE_P2P_GO:
 			/* AP-like interface */
-			cfg80211_del_sta(ndev, sta->addr, GFP_KERNEL);
+			cfg80211_del_sta(wdev, sta->addr, GFP_KERNEL);
 			break;
 		default:
 			break;
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index 05b040c684e8e..479b2418ca340 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -1076,7 +1076,7 @@ static void wmi_evt_connect(struct wil6210_vif *vif, int id, void *d, int len)
 		if (rc) {
 			if (disable_ap_sme)
 				/* notify new_sta has failed */
-				cfg80211_del_sta(ndev, evt->bssid, GFP_KERNEL);
+				cfg80211_del_sta(wdev, evt->bssid, GFP_KERNEL);
 			goto out;
 		}
 
@@ -1093,7 +1093,8 @@ static void wmi_evt_connect(struct wil6210_vif *vif, int id, void *d, int len)
 			sinfo->assoc_req_ies_len = assoc_req_ielen;
 		}
 
-		cfg80211_new_sta(ndev, evt->bssid, sinfo, GFP_KERNEL);
+		cfg80211_new_sta(ndev->ieee80211_ptr, evt->bssid, sinfo,
+				 GFP_KERNEL);
 
 		kfree(sinfo);
 	} else {
diff --git a/drivers/net/wireless/atmel/at76c50x-usb.c b/drivers/net/wireless/atmel/at76c50x-usb.c
index 6445332801a4c..44b04ea3cc8b0 100644
--- a/drivers/net/wireless/atmel/at76c50x-usb.c
+++ b/drivers/net/wireless/atmel/at76c50x-usb.c
@@ -2440,13 +2440,11 @@ static int at76_probe(struct usb_interface *interface,
 	struct mib_fw_version *fwv;
 	int board_type = (int)id->driver_info;
 
-	udev = usb_get_dev(interface_to_usbdev(interface));
+	udev = interface_to_usbdev(interface);
 
 	fwv = kmalloc_obj(*fwv);
-	if (!fwv) {
-		ret = -ENOMEM;
-		goto exit;
-	}
+	if (!fwv)
+		return -ENOMEM;
 
 	/* Load firmware into kernel memory */
 	fwe = at76_load_firmware(udev, board_type);
@@ -2534,8 +2532,7 @@ static int at76_probe(struct usb_interface *interface,
 
 exit:
 	kfree(fwv);
-	if (ret < 0)
-		usb_put_dev(udev);
+
 	return ret;
 }
 
@@ -2552,7 +2549,6 @@ static void at76_disconnect(struct usb_interface *interface)
 
 	wiphy_info(priv->hw->wiphy, "disconnecting\n");
 	at76_delete_device(priv);
-	usb_put_dev(interface_to_usbdev(interface));
 	dev_info(&interface->dev, "disconnected\n");
 }
 
diff --git a/drivers/net/wireless/broadcom/b43/phy_n.c b/drivers/net/wireless/broadcom/b43/phy_n.c
index bbc30cbad0bb8..22359bd9db47d 100644
--- a/drivers/net/wireless/broadcom/b43/phy_n.c
+++ b/drivers/net/wireless/broadcom/b43/phy_n.c
@@ -6566,19 +6566,19 @@ static void b43_nphy_op_software_rfkill(struct b43_wldev *dev,
 
 			b43_radio_mask(dev, 0x09, ~0x2);
 
-			b43_radio_write(dev, 0x204D, 0);
-			b43_radio_write(dev, 0x2053, 0);
-			b43_radio_write(dev, 0x2058, 0);
-			b43_radio_write(dev, 0x205E, 0);
-			b43_radio_mask(dev, 0x2062, ~0xF0);
-			b43_radio_write(dev, 0x2064, 0);
-
-			b43_radio_write(dev, 0x304D, 0);
-			b43_radio_write(dev, 0x3053, 0);
-			b43_radio_write(dev, 0x3058, 0);
-			b43_radio_write(dev, 0x305E, 0);
-			b43_radio_mask(dev, 0x3062, ~0xF0);
-			b43_radio_write(dev, 0x3064, 0);
+			b43_radio_write(dev, B2056_TX0 | B2056_TX_PADA_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX0 | B2056_TX_PADG_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX0 | B2056_TX_PGAA_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX0 | B2056_TX_PGAG_BOOST_TUNE, 0);
+			b43_radio_mask(dev, B2056_TX0 | B2056_TX_MIXA_BOOST_TUNE, ~0xF0);
+			b43_radio_write(dev, B2056_TX0 | B2056_TX_MIXG_BOOST_TUNE, 0);
+
+			b43_radio_write(dev, B2056_TX1 | B2056_TX_PADA_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX1 | B2056_TX_PADG_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX1 | B2056_TX_PGAA_BOOST_TUNE, 0);
+			b43_radio_write(dev, B2056_TX1 | B2056_TX_PGAG_BOOST_TUNE, 0);
+			b43_radio_mask(dev, B2056_TX1 | B2056_TX_MIXA_BOOST_TUNE, ~0xF0);
+			b43_radio_write(dev, B2056_TX1 | B2056_TX_MIXG_BOOST_TUNE, 0);
 		}
 	} else {
 		if (phy->rev >= 19) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index cea02b33b7985..0b55d445895f2 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -2758,11 +2758,11 @@ brcmf_cfg80211_config_default_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static s32
-brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       int link_id, u8 key_idx, bool pairwise,
 		       const u8 *mac_addr)
 {
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	struct brcmf_wsec_key *key;
 	s32 err;
 
@@ -2796,12 +2796,12 @@ brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static s32
-brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       int link_id, u8 key_idx, bool pairwise,
 		       const u8 *mac_addr, struct key_params *params)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	struct brcmf_pub *drvr = cfg->pub;
 	struct brcmf_wsec_key *key;
 	s32 val;
@@ -2822,7 +2822,7 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
 	}
 
 	if (params->key_len == 0)
-		return brcmf_cfg80211_del_key(wiphy, ndev, -1, key_idx,
+		return brcmf_cfg80211_del_key(wiphy, wdev, -1, key_idx,
 					      pairwise, mac_addr);
 
 	if (params->key_len > sizeof(key->data)) {
@@ -2918,7 +2918,7 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static s32
-brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       int link_id, u8 key_idx, bool pairwise,
 		       const u8 *mac_addr, void *cookie,
 		       void (*callback)(void *cookie,
@@ -2926,7 +2926,7 @@ brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct key_params params;
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	struct brcmf_cfg80211_profile *profile = &ifp->vif->profile;
 	struct brcmf_pub *drvr = cfg->pub;
 	struct brcmf_cfg80211_security *sec;
@@ -2976,10 +2976,10 @@ brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
 
 static s32
 brcmf_cfg80211_config_default_mgmt_key(struct wiphy *wiphy,
-				       struct net_device *ndev, int link_id,
+				       struct wireless_dev *wdev, int link_id,
 				       u8 key_idx)
 {
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 
 	brcmf_dbg(TRACE, "Enter key_idx %d\n", key_idx);
 
@@ -3132,11 +3132,11 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp,
 }
 
 static s32
-brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   const u8 *mac, struct station_info *sinfo)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	struct brcmf_pub *drvr = cfg->pub;
 	struct brcmf_scb_val_le scb_val;
 	s32 err = 0;
@@ -3255,11 +3255,11 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int
-brcmf_cfg80211_dump_station(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			    int idx, u8 *mac, struct station_info *sinfo)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	struct brcmf_pub *drvr = cfg->pub;
 	s32 err;
 
@@ -3284,7 +3284,8 @@ brcmf_cfg80211_dump_station(struct wiphy *wiphy, struct net_device *ndev,
 	}
 	if (idx < le32_to_cpu(cfg->assoclist.count)) {
 		memcpy(mac, cfg->assoclist.mac[idx], ETH_ALEN);
-		return brcmf_cfg80211_get_station(wiphy, ndev, mac, sinfo);
+		return brcmf_cfg80211_get_station(wiphy, wdev,
+						  mac, sinfo);
 	}
 	return -ENOENT;
 }
@@ -5452,12 +5453,13 @@ brcmf_cfg80211_change_beacon(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int
-brcmf_cfg80211_del_station(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_del_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   struct station_del_parameters *params)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_pub *drvr = cfg->pub;
 	struct brcmf_scb_val_le scbval;
+	struct net_device *ndev = wdev->netdev;
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	s32 err;
 
@@ -5484,12 +5486,12 @@ brcmf_cfg80211_del_station(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int
-brcmf_cfg80211_change_station(struct wiphy *wiphy, struct net_device *ndev,
+brcmf_cfg80211_change_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			      const u8 *mac, struct station_parameters *params)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_pub *drvr = cfg->pub;
-	struct brcmf_if *ifp = netdev_priv(ndev);
+	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
 	s32 err;
 
 	brcmf_dbg(TRACE, "Enter, MAC %pM, mask 0x%04x set 0x%04x\n", mac,
@@ -6548,13 +6550,14 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg,
 		sinfo->assoc_req_ies_len = e->datalen;
 		generation++;
 		sinfo->generation = generation;
-		cfg80211_new_sta(ndev, e->addr, sinfo, GFP_KERNEL);
+		cfg80211_new_sta(ndev->ieee80211_ptr, e->addr, sinfo,
+				 GFP_KERNEL);
 
 		kfree(sinfo);
 	} else if ((event == BRCMF_E_DISASSOC_IND) ||
 		   (event == BRCMF_E_DEAUTH_IND) ||
 		   (event == BRCMF_E_DEAUTH)) {
-		cfg80211_del_sta(ndev, e->addr, GFP_KERNEL);
+		cfg80211_del_sta(ndev->ieee80211_ptr, e->addr, GFP_KERNEL);
 	}
 	return 0;
 }
diff --git a/drivers/net/wireless/intel/iwlegacy/3945.h b/drivers/net/wireless/intel/iwlegacy/3945.h
index fb1e33c89d0e8..ed63b31fee9ad 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945.h
+++ b/drivers/net/wireless/intel/iwlegacy/3945.h
@@ -123,13 +123,15 @@ enum il3945_antenna {
 #define IEEE80211_FRAME_LEN             (IEEE80211_DATA_LEN + IEEE80211_HLEN)
 
 struct il3945_frame {
+	struct list_head list;
+
+	/* Must be last as it ends in a flexible-array member. */
 	union {
 		struct ieee80211_hdr frame;
 		struct il3945_tx_beacon_cmd beacon;
 		u8 raw[IEEE80211_FRAME_LEN];
 		u8 cmd[360];
 	} u;
-	struct list_head list;
 };
 
 #define SUP_RATE_11A_MAX_NUM_CHANNELS  8
diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
index 7921bc45e21db..18bb556826431 100644
--- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
@@ -4076,7 +4076,7 @@ il4965_hdl_beacon(struct il_priv *il, struct il_rx_buf *rxb)
 	u8 rate = il4965_hw_get_rate(beacon->beacon_notify_hdr.rate_n_flags);
 
 	D_RX("beacon status %x retries %d iss %d tsf:0x%.8x%.8x rate %d\n",
-	     le32_to_cpu(beacon->beacon_notify_hdr.u.status) & TX_STATUS_MSK,
+	     le32_to_cpu(beacon->beacon_tx_status) & TX_STATUS_MSK,
 	     beacon->beacon_notify_hdr.failure_frame,
 	     le32_to_cpu(beacon->ibss_mgr_status),
 	     le32_to_cpu(beacon->high_tsf), le32_to_cpu(beacon->low_tsf), rate);
diff --git a/drivers/net/wireless/intel/iwlegacy/commands.h b/drivers/net/wireless/intel/iwlegacy/commands.h
index b61b8f377702f..7aba84ac88a16 100644
--- a/drivers/net/wireless/intel/iwlegacy/commands.h
+++ b/drivers/net/wireless/intel/iwlegacy/commands.h
@@ -1690,7 +1690,7 @@ struct agg_tx_status {
 	__le16 sequence;
 } __packed;
 
-struct il4965_tx_resp {
+struct il4965_tx_resp_hdr {
 	u8 frame_count;		/* 1 no aggregation, >1 aggregation */
 	u8 bt_kill_count;	/* # blocked by bluetooth (unused for agg) */
 	u8 failure_rts;		/* # failures due to unsuccessful RTS */
@@ -1707,6 +1707,10 @@ struct il4965_tx_resp {
 	__le16 reserved;
 	__le32 pa_power1;	/* RF power amplifier measurement (not used) */
 	__le32 pa_power2;
+} __packed;
+
+struct il4965_tx_resp {
+	struct il4965_tx_resp_hdr;
 
 	/*
 	 * For non-agg:  frame status TX_STATUS_*
@@ -2664,7 +2668,8 @@ struct il3945_beacon_notif {
 } __packed;
 
 struct il4965_beacon_notif {
-	struct il4965_tx_resp beacon_notify_hdr;
+	struct il4965_tx_resp_hdr beacon_notify_hdr;
+	__le32 beacon_tx_status;
 	__le32 low_tsf;
 	__le32 high_tsf;
 	__le32 ibss_mgr_status;
diff --git a/drivers/net/wireless/intel/iwlegacy/common.h b/drivers/net/wireless/intel/iwlegacy/common.h
index 4c9836ab11dd4..21f1c7702addd 100644
--- a/drivers/net/wireless/intel/iwlegacy/common.h
+++ b/drivers/net/wireless/intel/iwlegacy/common.h
@@ -518,13 +518,15 @@ struct il_channel_info {
 #define IEEE80211_FRAME_LEN             (IEEE80211_DATA_LEN + IEEE80211_HLEN)
 
 struct il_frame {
+	struct list_head list;
+
+	/* Must be last as it ends in a flexible-array member. */
 	union {
 		struct ieee80211_hdr frame;
 		struct il_tx_beacon_cmd beacon;
 		u8 raw[IEEE80211_FRAME_LEN];
 		u8 cmd[360];
 	} u;
-	struct list_head list;
 };
 
 enum {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
index de9aef0d924c9..bf0f851a90755 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
@@ -504,7 +504,8 @@ iwl_acpi_parse_chains_table(union acpi_object *table,
 			    u8 num_chains, u8 num_sub_bands)
 {
 	for (u8 chain = 0; chain < num_chains; chain++) {
-		for (u8 subband = 0; subband < BIOS_SAR_MAX_SUB_BANDS_NUM;
+		for (u8 subband = 0;
+		     subband < ARRAY_SIZE(chains[chain].subbands);
 		     subband++) {
 			/* if we don't have the values, use the default */
 			if (subband >= num_sub_bands) {
@@ -534,7 +535,23 @@ int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	/* start by trying to read revision 2 */
+	/* start by trying to read revision 3 */
+	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+					 ACPI_WRDS_WIFI_DATA_SIZE_REV3,
+					 &tbl_rev);
+	if (!IS_ERR(wifi_pkg)) {
+		if (tbl_rev != 3) {
+			ret = -EINVAL;
+			goto out_free;
+		}
+
+		num_chains = ACPI_SAR_NUM_CHAINS_REV2;
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV3;
+
+		goto read_table;
+	}
+
+	/* then try revision 2 */
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 					 ACPI_WRDS_WIFI_DATA_SIZE_REV2,
 					 &tbl_rev);
@@ -591,6 +608,13 @@ int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 		goto out_free;
 	}
 
+	if (WARN_ON(num_chains * num_sub_bands >
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains) *
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands))) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+
 	IWL_DEBUG_RADIO(fwrt, "Reading WRDS tbl_rev=%d\n", tbl_rev);
 
 	flags = wifi_pkg->package.elements[1].integer.value;
@@ -624,7 +648,22 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	/* start by trying to read revision 2 */
+	/* start by trying to read revision 3 */
+	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+					 ACPI_EWRD_WIFI_DATA_SIZE_REV3,
+					 &tbl_rev);
+	if (!IS_ERR(wifi_pkg)) {
+		if (tbl_rev != 3) {
+			ret = -EINVAL;
+			goto out_free;
+		}
+
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV3;
+
+		goto read_table;
+	}
+
+	/* then try revision 2 */
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 					 ACPI_EWRD_WIFI_DATA_SIZE_REV2,
 					 &tbl_rev);
@@ -679,6 +718,13 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 		goto out_free;
 	}
 
+	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands >
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains) *
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands))) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+
 	enabled = !!(wifi_pkg->package.elements[1].integer.value);
 	n_profiles = wifi_pkg->package.elements[2].integer.value;
 
@@ -721,6 +767,13 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	if (tbl_rev < 2)
 		goto set_enabled;
 
+	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * 2 * num_sub_bands >
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains) *
+		    ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands))) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+
 	/* parse cdb chains for all profiles */
 	for (i = 0; i < n_profiles; i++) {
 		struct iwl_sar_profile_chain *chains;
@@ -759,6 +812,12 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 		u8 profiles;
 		u8 min_profiles;
 	} rev_data[] = {
+		{
+			.revisions = BIT(4),
+			.bands = ACPI_GEO_NUM_BANDS_REV4,
+			.profiles = ACPI_NUM_GEO_PROFILES_REV3,
+			.min_profiles = BIOS_GEO_MIN_PROFILE_NUM,
+		},
 		{
 			.revisions = BIT(3),
 			.bands = ACPI_GEO_NUM_BANDS_REV2,
@@ -812,6 +871,18 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 			num_bands = rev_data[idx].bands;
 			num_profiles = rev_data[idx].profiles;
 
+			if (WARN_ON(num_profiles >
+				    ARRAY_SIZE(fwrt->geo_profiles))) {
+				ret = -EINVAL;
+				goto out_free;
+			}
+
+			if (WARN_ON(num_bands >
+				    ARRAY_SIZE(fwrt->geo_profiles[0].bands))) {
+				ret = -EINVAL;
+				goto out_free;
+			}
+
 			if (rev_data[idx].min_profiles) {
 				/* read header that says # of profiles */
 				union acpi_object *entry;
@@ -851,18 +922,20 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 
 read_table:
 	fwrt->geo_rev = tbl_rev;
+
 	for (i = 0; i < num_profiles; i++) {
-		for (j = 0; j < BIOS_GEO_MAX_NUM_BANDS; j++) {
+		struct iwl_geo_profile *prof = &fwrt->geo_profiles[i];
+
+		for (j = 0; j < ARRAY_SIZE(prof->bands); j++) {
 			union acpi_object *entry;
 
 			/*
-			 * num_bands is either 2 or 3, if it's only 2 then
-			 * fill the third band (6 GHz) with the values from
-			 * 5 GHz (second band)
+			 * num_bands is either 2 or 3 or 4, if it's lower
+			 * than 4, fill the third band (6 GHz) with the values
+			 * from 5 GHz (second band)
 			 */
 			if (j >= num_bands) {
-				fwrt->geo_profiles[i].bands[j].max =
-					fwrt->geo_profiles[i].bands[1].max;
+				prof->bands[j].max = prof->bands[1].max;
 			} else {
 				entry = &wifi_pkg->package.elements[entry_idx];
 				entry_idx++;
@@ -872,15 +945,17 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 					goto out_free;
 				}
 
-				fwrt->geo_profiles[i].bands[j].max =
+				prof->bands[j].max =
 					entry->integer.value;
 			}
 
-			for (k = 0; k < BIOS_GEO_NUM_CHAINS; k++) {
+			for (k = 0;
+			     k < ARRAY_SIZE(prof->bands[0].chains);
+			     k++) {
 				/* same here as above */
 				if (j >= num_bands) {
-					fwrt->geo_profiles[i].bands[j].chains[k] =
-						fwrt->geo_profiles[i].bands[1].chains[k];
+					prof->bands[j].chains[k] =
+						prof->bands[1].chains[k];
 				} else {
 					entry = &wifi_pkg->package.elements[entry_idx];
 					entry_idx++;
@@ -890,7 +965,7 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 						goto out_free;
 					}
 
-					fwrt->geo_profiles[i].bands[j].chains[k] =
+					prof->bands[j].chains[k] =
 						entry->integer.value;
 				}
 			}
@@ -898,6 +973,7 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 	}
 
 	fwrt->geo_num_profiles = num_profiles;
+	fwrt->geo_bios_source = BIOS_SOURCE_ACPI;
 	fwrt->geo_enabled = true;
 	ret = 0;
 out_free:
@@ -915,6 +991,22 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
+	/* try to read ppag table rev 5 */
+	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+					 ACPI_PPAG_WIFI_DATA_SIZE_V3, &tbl_rev);
+	if (!IS_ERR(wifi_pkg)) {
+		if (tbl_rev == 5) {
+			num_sub_bands = IWL_NUM_SUB_BANDS_V3;
+			IWL_DEBUG_RADIO(fwrt,
+					"Reading PPAG table (tbl_rev=%d)\n",
+					tbl_rev);
+			goto read_table;
+		} else {
+			ret = -EINVAL;
+			goto out_free;
+		}
+	}
+
 	/* try to read ppag table rev 1 to 4 (all have the same data size) */
 	wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
 				ACPI_PPAG_WIFI_DATA_SIZE_V2, &tbl_rev);
@@ -950,6 +1042,15 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	goto out_free;
 
 read_table:
+	if (WARN_ON_ONCE(num_sub_bands >
+			 ARRAY_SIZE(fwrt->ppag_chains[0].subbands))) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+
+	BUILD_BUG_ON(ACPI_PPAG_NUM_CHAINS >
+		     ARRAY_SIZE(fwrt->ppag_chains));
+
 	fwrt->ppag_bios_rev = tbl_rev;
 	flags = &wifi_pkg->package.elements[1];
 
@@ -966,7 +1067,7 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	 * first sub-band (j=0) corresponds to Low-Band (2.4GHz), and the
 	 * following sub-bands to High-Band (5GHz).
 	 */
-	for (i = 0; i < IWL_NUM_CHAIN_LIMITS; i++) {
+	for (i = 0; i < ACPI_PPAG_NUM_CHAINS; i++) {
 		for (j = 0; j < num_sub_bands; j++) {
 			union acpi_object *ent;
 
@@ -980,6 +1081,7 @@ int iwl_acpi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 		}
 	}
 
+	iwl_bios_print_ppag(fwrt, num_sub_bands);
 	fwrt->ppag_bios_source = BIOS_SOURCE_ACPI;
 	ret = 0;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
index 06cece4ea6d95..51a57e57de7a6 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
@@ -8,11 +8,6 @@
 
 #include <linux/acpi.h>
 #include "fw/regulatory.h"
-#include "fw/api/commands.h"
-#include "fw/api/power.h"
-#include "fw/api/phy.h"
-#include "fw/api/nvm-reg.h"
-#include "fw/api/config.h"
 #include "fw/img.h"
 #include "iwl-trans.h"
 
@@ -44,6 +39,7 @@
 #define ACPI_SAR_NUM_SUB_BANDS_REV0	5
 #define ACPI_SAR_NUM_SUB_BANDS_REV1	11
 #define ACPI_SAR_NUM_SUB_BANDS_REV2	11
+#define ACPI_SAR_NUM_SUB_BANDS_REV3	12
 
 #define ACPI_WRDS_WIFI_DATA_SIZE_REV0	(ACPI_SAR_NUM_CHAINS_REV0 * \
 					 ACPI_SAR_NUM_SUB_BANDS_REV0 + 2)
@@ -51,6 +47,8 @@
 					 ACPI_SAR_NUM_SUB_BANDS_REV1 + 2)
 #define ACPI_WRDS_WIFI_DATA_SIZE_REV2	(ACPI_SAR_NUM_CHAINS_REV2 * \
 					 ACPI_SAR_NUM_SUB_BANDS_REV2 + 2)
+#define ACPI_WRDS_WIFI_DATA_SIZE_REV3	(ACPI_SAR_NUM_CHAINS_REV2 * \
+					 ACPI_SAR_NUM_SUB_BANDS_REV3 + 2)
 #define ACPI_EWRD_WIFI_DATA_SIZE_REV0	((ACPI_SAR_PROFILE_NUM - 1) * \
 					 ACPI_SAR_NUM_CHAINS_REV0 * \
 					 ACPI_SAR_NUM_SUB_BANDS_REV0 + 3)
@@ -60,11 +58,15 @@
 #define ACPI_EWRD_WIFI_DATA_SIZE_REV2	((ACPI_SAR_PROFILE_NUM - 1) * \
 					 ACPI_SAR_NUM_CHAINS_REV2 * \
 					 ACPI_SAR_NUM_SUB_BANDS_REV2 + 3)
+#define ACPI_EWRD_WIFI_DATA_SIZE_REV3	((ACPI_SAR_PROFILE_NUM - 1) * \
+					 ACPI_SAR_NUM_CHAINS_REV2 * \
+					 ACPI_SAR_NUM_SUB_BANDS_REV3 + 3)
 #define ACPI_WPFC_WIFI_DATA_SIZE	5 /* domain and 4 filter config words */
 
 /* revision 0 and 1 are identical, except for the semantics in the FW */
 #define ACPI_GEO_NUM_BANDS_REV0		2
 #define ACPI_GEO_NUM_BANDS_REV2		3
+#define ACPI_GEO_NUM_BANDS_REV4		4
 
 #define ACPI_WRDD_WIFI_DATA_SIZE	2
 #define ACPI_SPLC_WIFI_DATA_SIZE	2
@@ -96,10 +98,18 @@
  */
 #define ACPI_WTAS_WIFI_DATA_SIZE	(3 + IWL_WTAS_BLACK_LIST_MAX)
 
-#define ACPI_PPAG_WIFI_DATA_SIZE_V1	((IWL_NUM_CHAIN_LIMITS * \
-					  IWL_NUM_SUB_BANDS_V1) + 2)
-#define ACPI_PPAG_WIFI_DATA_SIZE_V2	((IWL_NUM_CHAIN_LIMITS * \
-					  IWL_NUM_SUB_BANDS_V2) + 2)
+#define ACPI_PPAG_NUM_CHAINS		2
+#define ACPI_PPAG_NUM_BANDS_V1		5
+#define ACPI_PPAG_NUM_BANDS_V2		11
+#define ACPI_PPAG_NUM_BANDS_V3		12
+#define ACPI_PPAG_WIFI_DATA_SIZE_V1	((ACPI_PPAG_NUM_CHAINS * \
+					  ACPI_PPAG_NUM_BANDS_V1) + 2)
+#define ACPI_PPAG_WIFI_DATA_SIZE_V2	((ACPI_PPAG_NUM_CHAINS * \
+					  ACPI_PPAG_NUM_BANDS_V2) + 2)
+
+/* used for ACPI PPAG table rev 5 */
+#define ACPI_PPAG_WIFI_DATA_SIZE_V3	((ACPI_PPAG_NUM_CHAINS * \
+					  ACPI_PPAG_NUM_BANDS_V3) + 2)
 
 #define IWL_SAR_ENABLE_MSK		BIT(0)
 #define IWL_REDUCE_POWER_FLAGS_POS	1
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index c7a833f8041a0..444d60a05a98b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -42,7 +42,8 @@ enum iwl_mac_conf_subcmd_ids {
 	 */
 	LINK_CONFIG_CMD = 0x9,
 	/**
-	 * @STA_CONFIG_CMD: &struct iwl_sta_cfg_cmd
+	 * @STA_CONFIG_CMD: &struct iwl_sta_cfg_cmd_v1,
+	 *	&struct iwl_sta_cfg_cmd_v2, or struct iwl_sta_cfg_cmd
 	 */
 	STA_CONFIG_CMD = 0xA,
 	/**
@@ -664,13 +665,21 @@ struct iwl_link_config_cmd {
  *	power save state and the DTIM timing
  * @STATION_TYPE_AUX: aux sta. In the FW there is no need for a special type
  *	for the aux sta, so this type is only for driver - internal use.
+ * @STATION_TYPE_NAN_PEER_NMI: NAN management peer station type. A station
+ *	of this type can have any number of links (even none) set in the
+ *	link_mask. (Supported since version 3.)
+ * @STATION_TYPE_NAN_PEER_NDI: NAN data peer station type. A station
+ *	of this type can have any number of links (even none) set in the
+ *	link_mask. (Supported since version 3.)
  */
 enum iwl_fw_sta_type {
 	STATION_TYPE_PEER,
 	STATION_TYPE_BCAST_MGMT,
 	STATION_TYPE_MCAST,
 	STATION_TYPE_AUX,
-}; /* STATION_TYPE_E_VER_1 */
+	STATION_TYPE_NAN_PEER_NMI,
+	STATION_TYPE_NAN_PEER_NDI,
+}; /* STATION_TYPE_E_VER_1, _VER_2 */
 
 /**
  * struct iwl_sta_cfg_cmd_v1 - cmd structure to add a peer sta to the uCode's
@@ -729,7 +738,7 @@ struct iwl_sta_cfg_cmd_v1 {
 } __packed; /* STA_CMD_API_S_VER_1 */
 
 /**
- * struct iwl_sta_cfg_cmd - cmd structure to add a peer sta to the uCode's
+ * struct iwl_sta_cfg_cmd_v2 - cmd structure to add a peer sta to the uCode's
  *	station table
  * ( STA_CONFIG_CMD = 0xA )
  *
@@ -769,7 +778,7 @@ struct iwl_sta_cfg_cmd_v1 {
  * @mic_compute_pad_delay: MIC compute time padding
  * @reserved: Reserved for alignment
  */
-struct iwl_sta_cfg_cmd {
+struct iwl_sta_cfg_cmd_v2 {
 	__le32 sta_id;
 	__le32 link_id;
 	u8 peer_mld_address[ETH_ALEN];
@@ -799,6 +808,83 @@ struct iwl_sta_cfg_cmd {
 	u8 reserved[2];
 } __packed; /* STA_CMD_API_S_VER_2 */
 
+/**
+ * struct iwl_sta_cfg_cmd - cmd structure to add a peer sta to the uCode's
+ *	station table
+ * ( STA_CONFIG_CMD = 0xA )
+ *
+ * @sta_id: index of station in uCode's station table
+ * @link_mask: bitmap of link FW IDs used with this STA
+ * @peer_mld_address: the peers mld address
+ * @reserved_for_peer_mld_address: reserved
+ * @peer_link_address: the address of the link that is used to communicate
+ *	with this sta
+ * @reserved_for_peer_link_address: reserved
+ * @station_type: type of this station. See &enum iwl_fw_sta_type
+ * @assoc_id: for GO only
+ * @beamform_flags: beam forming controls
+ * @mfp: indicates whether the STA uses management frame protection or not.
+ * @mimo: indicates whether the sta uses mimo or not
+ * @mimo_protection: indicates whether the sta uses mimo protection or not
+ * @ack_enabled: indicates that the AP supports receiving ACK-
+ *	enabled AGG, i.e. both BACK and non-BACK frames in a single AGG
+ * @trig_rnd_alloc: indicates that trigger based random allocation
+ *	is enabled according to UORA element existence
+ * @tx_ampdu_spacing: minimum A-MPDU spacing:
+ *	4 - 2us density, 5 - 4us density, 6 - 8us density, 7 - 16us density
+ * @tx_ampdu_max_size: maximum A-MPDU length: 0 - 8K, 1 - 16K, 2 - 32K,
+ *	3 - 64K, 4 - 128K, 5 - 256K, 6 - 512K, 7 - 1024K.
+ * @sp_length: the size of the SP in actual number of frames
+ * @uapsd_acs:  4 LS bits are trigger enabled ACs, 4 MS bits are the deliver
+ *	enabled ACs.
+ * @pkt_ext: optional, exists according to PPE-present bit in the HE/EHT-PHY
+ *	capa
+ * @htc_flags: which features are supported in HTC
+ * @use_ldpc_x2_cw: Indicates whether to use LDPC with double CW
+ * @use_icf: Indicates whether to use ICF instead of RTS
+ * @dps_pad_time: DPS (Dynamic Power Save) padding delay resolution to ensure
+ *	proper timing alignment
+ * @dps_trans_delay: DPS minimal time that takes the peer to return to low power
+ * @dps_enabled: flag indicating whether or not DPS is enabled
+ * @mic_prep_pad_delay: MIC prep time padding
+ * @mic_compute_pad_delay: MIC compute time padding
+ * @nmi_sta_id: for an NDI peer STA, the NMI peer STA ID it relates to
+ * @ndi_local_addr: for an NDI peer STA, the local NDI interface MAC address
+ * @reserved: Reserved for alignment
+ */
+struct iwl_sta_cfg_cmd {
+	__le32 sta_id;
+	__le32 link_mask;
+	u8 peer_mld_address[ETH_ALEN];
+	__le16 reserved_for_peer_mld_address;
+	u8 peer_link_address[ETH_ALEN];
+	__le16 reserved_for_peer_link_address;
+	__le32 station_type;
+	__le32 assoc_id;
+	__le32 beamform_flags;
+	__le32 mfp;
+	__le32 mimo;
+	__le32 mimo_protection;
+	__le32 ack_enabled;
+	__le32 trig_rnd_alloc;
+	__le32 tx_ampdu_spacing;
+	__le32 tx_ampdu_max_size;
+	__le32 sp_length;
+	__le32 uapsd_acs;
+	struct iwl_he_pkt_ext_v2 pkt_ext;
+	__le32 htc_flags;
+	u8 use_ldpc_x2_cw;
+	u8 use_icf;
+	u8 dps_pad_time;
+	u8 dps_trans_delay;
+	u8 dps_enabled;
+	u8 mic_prep_pad_delay;
+	u8 mic_compute_pad_delay;
+	u8 nmi_sta_id;
+	u8 ndi_local_addr[ETH_ALEN];
+	u8 reserved[2];
+} __packed; /* STA_CMD_API_S_VER_3 */
+
 /**
  * struct iwl_aux_sta_cmd - command for AUX STA configuration
  * ( AUX_STA_CMD = 0xB )
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
index bd6bf931866f8..25c860a05b0e9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
@@ -701,13 +701,23 @@ struct iwl_pnvm_init_complete_ntfy {
 #define UATS_TABLE_COL_SIZE	13
 
 /**
- * struct iwl_mcc_allowed_ap_type_cmd - struct for MCC_ALLOWED_AP_TYPE_CMD
+ * struct iwl_mcc_allowed_ap_type_cmd_v1 - struct for MCC_ALLOWED_AP_TYPE_CMD
  * @mcc_to_ap_type_map: mapping an MCC to 6 GHz AP type support (UATS)
  * @reserved: reserved
  */
-struct iwl_mcc_allowed_ap_type_cmd {
+struct iwl_mcc_allowed_ap_type_cmd_v1 {
 	u8 mcc_to_ap_type_map[UATS_TABLE_ROW_SIZE][UATS_TABLE_COL_SIZE];
 	__le16 reserved;
 } __packed; /* MCC_ALLOWED_AP_TYPE_CMD_API_S_VER_1 */
 
+/**
+ * struct iwl_mcc_allowed_ap_type_cmd - struct for MCC_ALLOWED_AP_TYPE_CMD
+ * @mcc_to_ap_type_map: mapping an MCC to 6 GHz AP type support (UATS)
+ * @mcc_to_ap_type_unii9_map: mapping an MCC to UNII-9 AP type support allowed
+ */
+struct iwl_mcc_allowed_ap_type_cmd {
+	u8 mcc_to_ap_type_map[UATS_TABLE_ROW_SIZE][UATS_TABLE_COL_SIZE];
+	u8 mcc_to_ap_type_unii9_map[UATS_TABLE_ROW_SIZE][UATS_TABLE_COL_SIZE];
+} __packed; /* MCC_ALLOWED_AP_TYPE_CMD_API_S_VER_2 */
+
 #endif /* __iwl_fw_api_nvm_reg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
index 0cd8a12e0f7c0..a3f916630df2b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
@@ -269,6 +269,7 @@ enum iwl_dev_tx_power_cmd_mode {
 #define IWL_NUM_CHAIN_LIMITS	2
 #define IWL_NUM_SUB_BANDS_V1	5
 #define IWL_NUM_SUB_BANDS_V2	11
+#define IWL_NUM_SUB_BANDS_V3	12
 
 /**
  * struct iwl_dev_tx_power_common - Common part of the TX power reduction cmd
@@ -425,24 +426,38 @@ struct iwl_dev_tx_power_cmd_v10 {
 	__le32 flags;
 } __packed; /* TX_REDUCED_POWER_API_S_VER_10 */
 
+struct iwl_dev_tx_power_cmd_v11 {
+	__le16 per_chain[IWL_NUM_CHAIN_TABLES_V2][IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V3];
+	u8 per_chain_restriction_changed;
+	u8 reserved;
+	__le32 timer_period;
+	__le32 flags;
+} __packed; /* TX_REDUCED_POWER_API_S_VER_11 */
+
 /*
  * struct iwl_dev_tx_power_cmd - TX power reduction command (multiversion)
  * @common: common part of the command
  * @v9: version 9 part of the command
  * @v10: version 10 part of the command
+ * @v11: version 11 part of the command
  */
 struct iwl_dev_tx_power_cmd {
 	struct iwl_dev_tx_power_common common;
 	union {
 		struct iwl_dev_tx_power_cmd_v9 v9;
 		struct iwl_dev_tx_power_cmd_v10 v10;
+		struct iwl_dev_tx_power_cmd_v11 v11;
 	};
-} __packed; /* TX_REDUCED_POWER_API_S_VER_9_VER10 */
+} __packed; /* TX_REDUCED_POWER_API_S_VER_9
+	     * TX_REDUCED_POWER_API_S_VER_10
+	     * TX_REDUCED_POWER_API_S_VER_11
+	     */
 
 #define IWL_NUM_GEO_PROFILES		3
 #define IWL_NUM_GEO_PROFILES_V3		8
 #define IWL_NUM_BANDS_PER_CHAIN_V1	2
 #define IWL_NUM_BANDS_PER_CHAIN_V2	3
+#define IWL_NUM_BANDS_PER_CHAIN_V6	4
 
 /**
  * enum iwl_geo_per_chain_offset_operation - type of operation
@@ -524,12 +539,25 @@ struct iwl_geo_tx_power_profiles_cmd_v5 {
 	__le32 table_revision;
 } __packed; /* PER_CHAIN_LIMIT_OFFSET_CMD_VER_5 */
 
+/**
+ * struct iwl_geo_tx_power_profiles_cmd_v6 - struct for PER_CHAIN_LIMIT_OFFSET_CMD cmd.
+ * @ops: operations, value from &enum iwl_geo_per_chain_offset_operation
+ * @table: offset profile per band.
+ * @bios_hdr: describes the revision and the source of the BIOS
+ */
+struct iwl_geo_tx_power_profiles_cmd_v6 {
+	__le32 ops;
+	struct iwl_per_chain_offset table[IWL_NUM_GEO_PROFILES_V3][IWL_NUM_BANDS_PER_CHAIN_V6];
+	struct iwl_bios_config_hdr bios_hdr;
+} __packed; /* PER_CHAIN_LIMIT_OFFSET_CMD_VER_6 */
+
 union iwl_geo_tx_power_profiles_cmd {
 	struct iwl_geo_tx_power_profiles_cmd_v1 v1;
 	struct iwl_geo_tx_power_profiles_cmd_v2 v2;
 	struct iwl_geo_tx_power_profiles_cmd_v3 v3;
 	struct iwl_geo_tx_power_profiles_cmd_v4 v4;
 	struct iwl_geo_tx_power_profiles_cmd_v5 v5;
+	struct iwl_geo_tx_power_profiles_cmd_v6 v6;
 };
 
 /**
@@ -573,6 +601,7 @@ enum iwl_ppag_flags {
  * @v1: command version 1 structure.
  * @v5: command version 5 structure.
  * @v7: command version 7 structure.
+ * @v8: command version 8 structure.
  * @v1.flags: values from &enum iwl_ppag_flags
  * @v1.gain: table of antenna gain values per chain and sub-band
  * @v1.reserved: reserved
@@ -581,6 +610,8 @@ enum iwl_ppag_flags {
  * @v7.ppag_config_info: see @struct bios_value_u32
  * @v7.gain: table of antenna gain values per chain and sub-band
  * @v7.reserved: reserved
+ * @v8.ppag_config_info: see @struct bios_value_u32
+ * @v8.gain: table of antenna gain values per chain and sub-band
  */
 union iwl_ppag_table_cmd {
 	struct {
@@ -598,6 +629,10 @@ union iwl_ppag_table_cmd {
 		s8 gain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V2];
 		s8 reserved[2];
 	} __packed v7; /* PER_PLAT_ANTENNA_GAIN_CMD_API_S_VER_7 */
+	struct {
+		struct bios_value_u32 ppag_config_info;
+		s8 gain[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V3];
+	} __packed v8; /* PER_PLAT_ANTENNA_GAIN_CMD_API_S_VER_8 */
 } __packed;
 
 #define IWL_PPAG_CMD_V1_MASK (IWL_PPAG_ETSI_MASK | IWL_PPAG_CHINA_MASK)
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index 60f0a4924ddfb..c2bb400c834cf 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -985,6 +985,7 @@ struct iwl_scan_probe_params_v4 {
 } __packed; /* SCAN_PROBE_PARAMS_API_S_VER_4 */
 
 #define SCAN_MAX_NUM_CHANS_V3 67
+#define SCAN_MAX_NUM_CHANS_V4 68
 
 /**
  * struct iwl_scan_channel_params_v4 - channel params
@@ -1027,6 +1028,24 @@ struct iwl_scan_channel_params_v7 {
 	struct iwl_scan_channel_cfg_umac channel_config[SCAN_MAX_NUM_CHANS_V3];
 } __packed; /* SCAN_CHANNEL_PARAMS_API_S_VER_6 */
 
+/**
+ * struct iwl_scan_channel_params_v8 - channel params
+ * @flags: channel flags &enum iwl_scan_channel_flags
+ * @count: num of channels in scan request
+ * @n_aps_override: override the number of APs the FW uses to calculate dwell
+ *	time when adaptive dwell is used.
+ *	Channel k will use n_aps_override[i] when BIT(20 + i) is set in
+ *	channel_config[k].flags
+ * @channel_config: array of explicit channel configurations
+ *      for 2.4Ghz and 5.2Ghz bands
+ */
+struct iwl_scan_channel_params_v8 {
+	u8 flags;
+	u8 count;
+	u8 n_aps_override[2];
+	struct iwl_scan_channel_cfg_umac channel_config[SCAN_MAX_NUM_CHANS_V4];
+} __packed; /* SCAN_CHANNEL_PARAMS_API_S_VER_8 */
+
 /**
  * struct iwl_scan_general_params_v11 - channel params
  * @flags: &enum iwl_umac_scan_general_flags_v2
@@ -1109,6 +1128,20 @@ struct iwl_scan_req_params_v17 {
 	struct iwl_scan_probe_params_v4 probe_params;
 } __packed; /* SCAN_REQUEST_PARAMS_API_S_VER_17 - 14 */
 
+/**
+ * struct iwl_scan_req_params_v18 - scan request parameters (v18)
+ * @general_params: &struct iwl_scan_general_params_v11
+ * @channel_params: &struct iwl_scan_channel_params_v8
+ * @periodic_params: &struct iwl_scan_periodic_parms_v1
+ * @probe_params: &struct iwl_scan_probe_params_v4
+ */
+struct iwl_scan_req_params_v18 {
+	struct iwl_scan_general_params_v11 general_params;
+	struct iwl_scan_channel_params_v8 channel_params;
+	struct iwl_scan_periodic_parms_v1 periodic_params;
+	struct iwl_scan_probe_params_v4 probe_params;
+} __packed; /* SCAN_REQUEST_PARAMS_API_S_VER_18 */
+
 /**
  * struct iwl_scan_req_umac_v12 - scan request command (v12)
  * @uid: scan id, &enum iwl_umac_scan_uid_offsets
@@ -1133,6 +1166,18 @@ struct iwl_scan_req_umac_v17 {
 	struct iwl_scan_req_params_v17 scan_params;
 } __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_17 - 14 */
 
+/**
+ * struct iwl_scan_req_umac_v18 - scan request command (v18)
+ * @uid: scan id, &enum iwl_umac_scan_uid_offsets
+ * @ooc_priority: out of channel priority - &enum iwl_scan_priority
+ * @scan_params: scan parameters
+ */
+struct iwl_scan_req_umac_v18 {
+	__le32 uid;
+	__le32 ooc_priority;
+	struct iwl_scan_req_params_v18 scan_params;
+} __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_18 */
+
 /**
  * struct iwl_umac_scan_abort - scan abort command
  * @uid: scan id, &enum iwl_umac_scan_uid_offsets
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index 1f26d89fc9080..0cffa5493704d 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -2933,7 +2933,7 @@ int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
 	IWL_WARN(fwrt, "Collecting data: trigger %d fired.\n",
 		 le32_to_cpu(desc->trig_desc.type));
 
-	queue_delayed_work(system_unbound_wq, &wk_data->wk,
+	queue_delayed_work(system_dfl_wq, &wk_data->wk,
 			   usecs_to_jiffies(delay));
 
 	return 0;
@@ -3236,7 +3236,7 @@ int iwl_fw_dbg_ini_collect(struct iwl_fw_runtime *fwrt,
 	if (sync)
 		iwl_fw_dbg_collect_sync(fwrt, idx);
 	else
-		queue_delayed_work(system_unbound_wq,
+		queue_delayed_work(system_dfl_wq,
 				   &fwrt->dump.wks[idx].wk,
 				   usecs_to_jiffies(delay));
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
index 958e71a3c958b..55128caac7ed9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
@@ -241,6 +241,10 @@ static int iwl_sar_fill_table(struct iwl_fw_runtime *fwrt,
 	int profs[BIOS_SAR_NUM_CHAINS] = { prof_a, prof_b };
 	int i, j;
 
+	if (WARN_ON_ONCE(n_subbands >
+			 ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands)))
+		return -EINVAL;
+
 	for (i = 0; i < BIOS_SAR_NUM_CHAINS; i++) {
 		struct iwl_sar_profile *prof;
 
@@ -300,132 +304,6 @@ int iwl_sar_fill_profile(struct iwl_fw_runtime *fwrt,
 }
 IWL_EXPORT_SYMBOL(iwl_sar_fill_profile);
 
-static bool iwl_ppag_value_valid(struct iwl_fw_runtime *fwrt, int chain,
-				 int subband)
-{
-	s8 ppag_val = fwrt->ppag_chains[chain].subbands[subband];
-
-	if ((subband == 0 &&
-	     (ppag_val > IWL_PPAG_MAX_LB || ppag_val < IWL_PPAG_MIN_LB)) ||
-	    (subband != 0 &&
-	     (ppag_val > IWL_PPAG_MAX_HB || ppag_val < IWL_PPAG_MIN_HB))) {
-		IWL_DEBUG_RADIO(fwrt, "Invalid PPAG value: %d\n", ppag_val);
-		return false;
-	}
-	return true;
-}
-
-/* Utility function for iwlmvm and iwlxvt */
-int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
-			union iwl_ppag_table_cmd *cmd, int *cmd_size)
-{
-	u8 cmd_ver;
-	int i, j, num_sub_bands;
-	s8 *gain;
-	bool send_ppag_always;
-
-	/* many firmware images for JF lie about this */
-	if (CSR_HW_RFID_TYPE(fwrt->trans->info.hw_rf_id) ==
-	    CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_JF))
-		return -EOPNOTSUPP;
-
-	if (!fw_has_capa(&fwrt->fw->ucode_capa, IWL_UCODE_TLV_CAPA_SET_PPAG)) {
-		IWL_DEBUG_RADIO(fwrt,
-				"PPAG capability not supported by FW, command not sent.\n");
-		return -EINVAL;
-	}
-
-	cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
-					WIDE_ID(PHY_OPS_GROUP,
-						PER_PLATFORM_ANT_GAIN_CMD), 1);
-	/*
-	 * Starting from ver 4, driver needs to send the PPAG CMD regardless
-	 * if PPAG is enabled/disabled or valid/invalid.
-	 */
-	send_ppag_always = cmd_ver > 3;
-
-	/* Don't send PPAG if it is disabled */
-	if (!send_ppag_always && !fwrt->ppag_flags) {
-		IWL_DEBUG_RADIO(fwrt, "PPAG not enabled, command not sent.\n");
-		return -EINVAL;
-	}
-
-	IWL_DEBUG_RADIO(fwrt, "PPAG cmd ver is %d\n", cmd_ver);
-	if (cmd_ver == 1) {
-		num_sub_bands = IWL_NUM_SUB_BANDS_V1;
-		gain = cmd->v1.gain[0];
-		*cmd_size = sizeof(cmd->v1);
-		cmd->v1.flags = cpu_to_le32(fwrt->ppag_flags & IWL_PPAG_CMD_V1_MASK);
-		if (fwrt->ppag_bios_rev >= 1) {
-			/* in this case FW supports revision 0 */
-			IWL_DEBUG_RADIO(fwrt,
-					"PPAG table rev is %d, send truncated table\n",
-					fwrt->ppag_bios_rev);
-		}
-	} else if (cmd_ver == 5) {
-		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
-		gain = cmd->v5.gain[0];
-		*cmd_size = sizeof(cmd->v5);
-		cmd->v5.flags = cpu_to_le32(fwrt->ppag_flags & IWL_PPAG_CMD_V5_MASK);
-		if (fwrt->ppag_bios_rev == 0) {
-			/* in this case FW supports revisions 1,2 or 3 */
-			IWL_DEBUG_RADIO(fwrt,
-					"PPAG table rev is 0, send padded table\n");
-		}
-	} else if (cmd_ver == 7) {
-		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
-		gain = cmd->v7.gain[0];
-		*cmd_size = sizeof(cmd->v7);
-		cmd->v7.ppag_config_info.hdr.table_source =
-			fwrt->ppag_bios_source;
-		cmd->v7.ppag_config_info.hdr.table_revision =
-			fwrt->ppag_bios_rev;
-		cmd->v7.ppag_config_info.value = cpu_to_le32(fwrt->ppag_flags);
-	} else {
-		IWL_DEBUG_RADIO(fwrt, "Unsupported PPAG command version\n");
-		return -EINVAL;
-	}
-
-	/* ppag mode */
-	IWL_DEBUG_RADIO(fwrt,
-			"PPAG MODE bits were read from bios: %d\n",
-			fwrt->ppag_flags);
-
-	if (cmd_ver == 1 &&
-	    !fw_has_capa(&fwrt->fw->ucode_capa,
-			 IWL_UCODE_TLV_CAPA_PPAG_CHINA_BIOS_SUPPORT)) {
-		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_ETSI_MASK);
-		IWL_DEBUG_RADIO(fwrt, "masking ppag China bit\n");
-	} else {
-		IWL_DEBUG_RADIO(fwrt, "isn't masking ppag China bit\n");
-	}
-
-	/* The 'flags' field is the same in v1 and v5 so we can just
-	 * use v1 to access it.
-	 */
-	IWL_DEBUG_RADIO(fwrt,
-			"PPAG MODE bits going to be sent: %d\n",
-			(cmd_ver < 7) ? le32_to_cpu(cmd->v1.flags) :
-					le32_to_cpu(cmd->v7.ppag_config_info.value));
-
-	for (i = 0; i < IWL_NUM_CHAIN_LIMITS; i++) {
-		for (j = 0; j < num_sub_bands; j++) {
-			if (!send_ppag_always &&
-			    !iwl_ppag_value_valid(fwrt, i, j))
-				return -EINVAL;
-
-			gain[i * num_sub_bands + j] =
-				fwrt->ppag_chains[i].subbands[j];
-			IWL_DEBUG_RADIO(fwrt,
-					"PPAG table: chain[%d] band[%d]: gain = %d\n",
-					i, j, gain[i * num_sub_bands + j]);
-		}
-	}
-
-	return 0;
-}
-IWL_EXPORT_SYMBOL(iwl_fill_ppag_table);
-
 bool iwl_is_ppag_approved(struct iwl_fw_runtime *fwrt)
 {
 	if (!dmi_check_system(dmi_ppag_approved_list)) {
@@ -440,6 +318,27 @@ bool iwl_is_ppag_approved(struct iwl_fw_runtime *fwrt)
 }
 IWL_EXPORT_SYMBOL(iwl_is_ppag_approved);
 
+/* Print the PPAG table as read from BIOS */
+void iwl_bios_print_ppag(struct iwl_fw_runtime *fwrt, int n_subbands)
+{
+	int i, j;
+
+	IWL_DEBUG_RADIO(fwrt, "PPAG table as read from BIOS:\n");
+	IWL_DEBUG_RADIO(fwrt, "PPAG revision = %d\n", fwrt->ppag_bios_rev);
+	IWL_DEBUG_RADIO(fwrt, "PPAG flags = 0x%x\n", fwrt->ppag_flags);
+
+	if (WARN_ON_ONCE(n_subbands >
+			 ARRAY_SIZE(fwrt->ppag_chains[0].subbands)))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(fwrt->ppag_chains); i++)
+		for (j = 0; j < n_subbands; j++)
+			IWL_DEBUG_RADIO(fwrt,
+					"ppag_chains[%d].subbands[%d] = %d\n",
+					i, j,
+					fwrt->ppag_chains[i].subbands[j]);
+}
+
 bool iwl_is_tas_approved(void)
 {
 	return dmi_check_system(dmi_tas_approved_list);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
index 1489031687b78..6fffc032efd31 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
@@ -21,10 +21,11 @@
  */
 #define BIOS_SAR_MAX_CHAINS_PER_PROFILE 4
 #define BIOS_SAR_NUM_CHAINS             2
-#define BIOS_SAR_MAX_SUB_BANDS_NUM      11
+#define BIOS_SAR_MAX_SUB_BANDS_NUM      12
+#define BIOS_PPAG_MAX_SUB_BANDS_NUM     12
 
 #define BIOS_GEO_NUM_CHAINS		2
-#define BIOS_GEO_MAX_NUM_BANDS		3
+#define BIOS_GEO_MAX_NUM_BANDS		4
 #define BIOS_GEO_MAX_PROFILE_NUM	8
 #define BIOS_GEO_MIN_PROFILE_NUM	3
 
@@ -100,7 +101,7 @@ struct iwl_geo_profile {
 
 /* Same thing as with SAR, all revisions fit in revision 2 */
 struct iwl_ppag_chain {
-	s8 subbands[BIOS_SAR_MAX_SUB_BANDS_NUM];
+	s8 subbands[BIOS_PPAG_MAX_SUB_BANDS_NUM];
 };
 
 struct iwl_tas_data {
@@ -180,6 +181,9 @@ enum iwl_dsm_masks_reg {
 
 struct iwl_fw_runtime;
 
+/* Print the PPAG table as read from BIOS */
+void iwl_bios_print_ppag(struct iwl_fw_runtime *fwrt, int n_subbands);
+
 bool iwl_sar_geo_support(struct iwl_fw_runtime *fwrt);
 
 int iwl_sar_geo_fill_table(struct iwl_fw_runtime *fwrt,
@@ -190,10 +194,6 @@ int iwl_sar_fill_profile(struct iwl_fw_runtime *fwrt,
 			 __le16 *per_chain, u32 n_tables, u32 n_subbands,
 			 int prof_a, int prof_b);
 
-int iwl_fill_ppag_table(struct iwl_fw_runtime *fwrt,
-			union iwl_ppag_table_cmd *cmd,
-			int *cmd_size);
-
 bool iwl_is_ppag_approved(struct iwl_fw_runtime *fwrt);
 
 bool iwl_is_tas_approved(void);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index ff186fb2e0da1..d80ae610e56c3 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -106,13 +106,14 @@ struct iwl_txf_iter_data {
  * @cur_fw_img: current firmware image, must be maintained by
  *	the driver by calling &iwl_fw_set_current_image()
  * @dump: debug dump data
- * @uats_table: AP type table
- * @uats_valid: is AP type table valid
+ * @ap_type_cmd: AP type tables (for enablement on 6 GHz)
+ * @ap_type_cmd_valid: if &ap_type_cmd is valid
  * @uefi_tables_lock_status: The status of the WIFI GUID UEFI variables lock:
  *	0: Unlocked, 1 and 2: Locked.
  *	Only read the UEFI variables if locked.
  * @sar_profiles: sar profiles as read from WRDS/EWRD BIOS tables
  * @geo_profiles: geographic profiles as read from WGDS BIOS table
+ * @geo_bios_source: see &enum bios_source
  * @phy_filters: specific phy filters as read from WPFC BIOS table
  * @ppag_bios_rev: PPAG BIOS revision
  * @ppag_bios_source: see &enum bios_source
@@ -204,6 +205,7 @@ struct iwl_fw_runtime {
 	u8 sar_chain_b_profile;
 	u8 reduced_power_flags;
 	struct iwl_geo_profile geo_profiles[BIOS_GEO_MAX_PROFILE_NUM];
+	enum bios_source geo_bios_source;
 	u32 geo_rev;
 	u32 geo_num_profiles;
 	bool geo_enabled;
@@ -213,8 +215,8 @@ struct iwl_fw_runtime {
 	u8 ppag_bios_source;
 	struct iwl_sar_offset_mapping_cmd sgom_table;
 	bool sgom_enabled;
-	struct iwl_mcc_allowed_ap_type_cmd uats_table;
-	bool uats_valid;
+	struct iwl_mcc_allowed_ap_type_cmd ap_type_cmd;
+	bool ap_type_cmd_valid;
 	u8 uefi_tables_lock_status;
 	struct iwl_phy_specific_cfg phy_filters;
 	enum bios_source dsm_source;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
index a7ba86e06c093..2ef0a7a920adb 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
@@ -402,11 +402,11 @@ static int iwl_uefi_uats_parse(struct uefi_cnv_wlan_uats_data *uats_data,
 	if (uats_data->revision != 1)
 		return -EINVAL;
 
-	memcpy(fwrt->uats_table.mcc_to_ap_type_map,
+	memcpy(fwrt->ap_type_cmd.mcc_to_ap_type_map,
 	       uats_data->mcc_to_ap_type_map,
-	       sizeof(fwrt->uats_table.mcc_to_ap_type_map));
+	       sizeof(fwrt->ap_type_cmd.mcc_to_ap_type_map));
 
-	fwrt->uats_valid = true;
+	fwrt->ap_type_cmd_valid = true;
 
 	return 0;
 }
@@ -429,12 +429,61 @@ void iwl_uefi_get_uats_table(struct iwl_trans *trans,
 }
 IWL_EXPORT_SYMBOL(iwl_uefi_get_uats_table);
 
+void iwl_uefi_get_uneb_table(struct iwl_trans *trans,
+			     struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_wlan_uneb_data *data;
+
+	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_UNEB_NAME,
+					      "UNEB", sizeof(*data), NULL);
+	if (IS_ERR(data))
+		return;
+
+	if (data->revision != 1) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Cannot read UNEB table. rev is invalid\n");
+		goto out;
+	}
+
+	BUILD_BUG_ON(sizeof(data->mcc_to_ap_type_map) !=
+		     sizeof(fwrt->ap_type_cmd.mcc_to_ap_type_unii9_map));
+
+	memcpy(fwrt->ap_type_cmd.mcc_to_ap_type_unii9_map,
+	       data->mcc_to_ap_type_map,
+	       sizeof(fwrt->ap_type_cmd.mcc_to_ap_type_unii9_map));
+
+	fwrt->ap_type_cmd_valid = true;
+
+out:
+	kfree(data);
+}
+IWL_EXPORT_SYMBOL(iwl_uefi_get_uneb_table);
+
 static void iwl_uefi_set_sar_profile(struct iwl_fw_runtime *fwrt,
-				     struct uefi_sar_profile *uefi_sar_prof,
-				     u8 prof_index, bool enabled)
+				     const u8 *vals, u8 prof_index,
+				     u8 num_subbands, bool enabled)
 {
-	memcpy(&fwrt->sar_profiles[prof_index].chains, uefi_sar_prof,
-	       sizeof(struct uefi_sar_profile));
+	struct iwl_sar_profile *sar_prof = &fwrt->sar_profiles[prof_index];
+
+	/*
+	 * Make sure fwrt has enough room to hold the data
+	 * coming from the UEFI table
+	 */
+	if (WARN_ON(ARRAY_SIZE(sar_prof->chains) *
+		    ARRAY_SIZE(sar_prof->chains[0].subbands)  <
+		    UEFI_SAR_MAX_CHAINS_PER_PROFILE * num_subbands))
+		return;
+
+	BUILD_BUG_ON(ARRAY_SIZE(sar_prof->chains) !=
+		     UEFI_SAR_MAX_CHAINS_PER_PROFILE);
+
+	for (int chain = 0;
+	     chain < UEFI_SAR_MAX_CHAINS_PER_PROFILE;
+	     chain++) {
+		for (int subband = 0; subband < num_subbands; subband++)
+			sar_prof->chains[chain].subbands[subband] =
+				vals[chain * num_subbands + subband];
+	}
 
 	fwrt->sar_profiles[prof_index].enabled = enabled & IWL_SAR_ENABLE_MSK;
 }
@@ -442,24 +491,46 @@ static void iwl_uefi_set_sar_profile(struct iwl_fw_runtime *fwrt,
 int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_wrds *data;
+	unsigned long size;
+	unsigned long expected_size;
+	int num_subbands;
 	int ret = 0;
 
 	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WRDS_NAME,
-					      "WRDS", sizeof(*data), NULL);
+					      "WRDS",
+					      UEFI_SAR_WRDS_TABLE_SIZE_REV2,
+					      &size);
+
 	if (IS_ERR(data))
 		return -EINVAL;
 
-	if (data->revision != IWL_UEFI_WRDS_REVISION) {
-		ret = -EINVAL;
-		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI WRDS revision:%d\n",
+	switch (data->revision) {
+	case 2:
+		expected_size = UEFI_SAR_WRDS_TABLE_SIZE_REV2;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV2;
+		break;
+	case 3:
+		expected_size = UEFI_SAR_WRDS_TABLE_SIZE_REV3;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV3;
+		break;
+	default:
+		IWL_DEBUG_RADIO(fwrt,
+				"Unsupported UEFI WRDS revision:%d\n",
 				data->revision);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (size != expected_size) {
+		ret = -EINVAL;
 		goto out;
 	}
 
 	/* The profile from WRDS is officially profile 1, but goes
 	 * into sar_profiles[0] (because we don't have a profile 0).
 	 */
-	iwl_uefi_set_sar_profile(fwrt, &data->sar_profile, 0, data->mode);
+	iwl_uefi_set_sar_profile(fwrt, data->vals, 0,
+				 num_subbands, data->mode);
 out:
 	kfree(data);
 	return ret;
@@ -468,21 +539,40 @@ int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_ewrd *data;
+	unsigned long expected_size;
 	int i, ret = 0;
+	unsigned long size;
+	int num_subbands;
+	int profile_size;
 
 	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_EWRD_NAME,
-					      "EWRD", sizeof(*data), NULL);
+					      "EWRD",
+					      UEFI_SAR_EWRD_TABLE_SIZE_REV2,
+					      &size);
 	if (IS_ERR(data))
 		return -EINVAL;
 
-	if (data->revision != IWL_UEFI_EWRD_REVISION) {
-		ret = -EINVAL;
-		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI EWRD revision:%d\n",
+	switch (data->revision) {
+	case 2:
+		expected_size = UEFI_SAR_EWRD_TABLE_SIZE_REV2;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV2;
+		profile_size = UEFI_SAR_PROFILE_SIZE_REV2;
+		break;
+	case 3:
+		expected_size = UEFI_SAR_EWRD_TABLE_SIZE_REV3;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV3;
+		profile_size = UEFI_SAR_PROFILE_SIZE_REV3;
+		break;
+	default:
+		IWL_DEBUG_RADIO(fwrt,
+				"Unsupported UEFI EWRD revision:%d\n",
 				data->revision);
+		ret = -EINVAL;
 		goto out;
 	}
 
-	if (data->num_profiles >= BIOS_SAR_MAX_PROFILE_NUM) {
+	if (size != expected_size ||
+	    data->num_profiles >= BIOS_SAR_MAX_PROFILE_NUM) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -492,8 +582,8 @@ int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 		 * save them in sar_profiles[1-3] (because we don't
 		 * have profile 0).  So in the array we start from 1.
 		 */
-		iwl_uefi_set_sar_profile(fwrt, &data->sar_profiles[i], i + 1,
-					 data->mode);
+		iwl_uefi_set_sar_profile(fwrt, &data->vals[i * profile_size],
+					 i + 1, num_subbands, data->mode);
 
 out:
 	kfree(data);
@@ -503,20 +593,39 @@ int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_wgds *data;
-	int i, ret = 0;
+	unsigned long expected_size;
+	unsigned long size;
+	int profile_size;
+	int n_subbands;
+	int ret = 0;
 
 	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WGDS_NAME,
-					      "WGDS", sizeof(*data), NULL);
+					      "WGDS", UEFI_WGDS_TABLE_SIZE_REV3,
+					      &size);
 	if (IS_ERR(data))
 		return -EINVAL;
 
-	if (data->revision != IWL_UEFI_WGDS_REVISION) {
+	switch (data->revision) {
+	case 3:
+		expected_size = UEFI_WGDS_TABLE_SIZE_REV3;
+		n_subbands = UEFI_GEO_NUM_BANDS_REV3;
+		break;
+	case 4:
+		expected_size = UEFI_WGDS_TABLE_SIZE_REV4;
+		n_subbands = UEFI_GEO_NUM_BANDS_REV4;
+		break;
+	default:
 		ret = -EINVAL;
 		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI WGDS revision:%d\n",
 				data->revision);
 		goto out;
 	}
 
+	if (size != expected_size) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	if (data->num_profiles < BIOS_GEO_MIN_PROFILE_NUM ||
 	    data->num_profiles > BIOS_GEO_MAX_PROFILE_NUM) {
 		ret = -EINVAL;
@@ -525,10 +634,31 @@ int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 		goto out;
 	}
 
+	if (WARN_ON(BIOS_GEO_MAX_PROFILE_NUM >
+		    ARRAY_SIZE(fwrt->geo_profiles) ||
+		    n_subbands > ARRAY_SIZE(fwrt->geo_profiles[0].bands) ||
+		    BIOS_GEO_NUM_CHAINS >
+		    ARRAY_SIZE(fwrt->geo_profiles[0].bands[0].chains))) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	fwrt->geo_rev = data->revision;
-	for (i = 0; i < data->num_profiles; i++)
-		memcpy(&fwrt->geo_profiles[i], &data->geo_profiles[i],
-		       sizeof(struct iwl_geo_profile));
+	fwrt->geo_bios_source = BIOS_SOURCE_UEFI;
+	profile_size = 3 * n_subbands;
+	for (int prof = 0; prof < data->num_profiles; prof++) {
+		const u8 *val = &data->vals[profile_size * prof];
+		struct iwl_geo_profile *geo_prof = &fwrt->geo_profiles[prof];
+
+		for (int subband = 0; subband < n_subbands; subband++) {
+			geo_prof->bands[subband].max = *val++;
+
+			for (int chain = 0;
+			     chain < BIOS_GEO_NUM_CHAINS;
+			     chain++)
+				geo_prof->bands[subband].chains[chain] = *val++;
+		}
+	}
 
 	fwrt->geo_num_profiles = data->num_profiles;
 	fwrt->geo_enabled = true;
@@ -540,28 +670,66 @@ int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 int iwl_uefi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_ppag *data;
+	int n_subbands;
+	u32 valid_rev;
 	int ret = 0;
 
 	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_PPAG_NAME,
-					      "PPAG", sizeof(*data), NULL);
-	if (IS_ERR(data))
-		return -EINVAL;
+					      "PPAG", UEFI_PPAG_DATA_SIZE_V5,
+					      NULL);
+	if (!IS_ERR(data)) {
+		n_subbands = UEFI_PPAG_SUB_BANDS_NUM_REV5;
+		valid_rev = BIT(5);
+
+		goto parse_table;
+	}
+
+	data = iwl_uefi_get_verified_variable(fwrt->trans,
+					      IWL_UEFI_PPAG_NAME,
+					      "PPAG",
+					      UEFI_PPAG_DATA_SIZE_V4,
+					      NULL);
+	if (!IS_ERR(data)) {
+		n_subbands = UEFI_PPAG_SUB_BANDS_NUM_REV4;
+		/* revisions 1-4 have all the same size */
+		valid_rev = BIT(1) | BIT(2) | BIT(3) | BIT(4);
+
+		goto parse_table;
+	}
 
-	if (data->revision < IWL_UEFI_MIN_PPAG_REV ||
-	    data->revision > IWL_UEFI_MAX_PPAG_REV) {
+	return -EINVAL;
+
+parse_table:
+	if (!(BIT(data->revision) & valid_rev)) {
 		ret = -EINVAL;
-		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI PPAG revision:%d\n",
+		IWL_DEBUG_RADIO(fwrt,
+				"Unsupported UEFI PPAG revision:%d\n",
 				data->revision);
 		goto out;
 	}
 
+	/*
+	 * Make sure fwrt has enough room to hold
+	 * data coming from the UEFI table
+	 */
+	if (WARN_ON(ARRAY_SIZE(fwrt->ppag_chains) *
+		    ARRAY_SIZE(fwrt->ppag_chains[0].subbands)  <
+		    UEFI_PPAG_NUM_CHAINS * n_subbands)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	fwrt->ppag_bios_rev = data->revision;
 	fwrt->ppag_flags = iwl_bios_get_ppag_flags(data->ppag_modes,
 						   fwrt->ppag_bios_rev);
 
-	BUILD_BUG_ON(sizeof(fwrt->ppag_chains) != sizeof(data->ppag_chains));
-	memcpy(&fwrt->ppag_chains, &data->ppag_chains,
-	       sizeof(data->ppag_chains));
+	for (int chain = 0; chain < UEFI_PPAG_NUM_CHAINS; chain++) {
+		for (int subband = 0; subband < n_subbands; subband++)
+			fwrt->ppag_chains[chain].subbands[subband] =
+				data->vals[chain * n_subbands + subband];
+	}
+
+	iwl_bios_print_ppag(fwrt, n_subbands);
 	fwrt->ppag_bios_source = BIOS_SOURCE_UEFI;
 out:
 	kfree(data);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
index 349ac1505ad74..474f06db4d430 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
@@ -25,16 +25,12 @@
 #define IWL_UEFI_PUNCTURING_NAME	L"UefiCnvWlanPuncturing"
 #define IWL_UEFI_DSBR_NAME		L"UefiCnvCommonDSBR"
 #define IWL_UEFI_WPFC_NAME		L"WPFC"
+#define IWL_UEFI_UNEB_NAME		L"CnvUefiWlanUNEB"
 
 
 #define IWL_SGOM_MAP_SIZE		339
 #define IWL_UATS_MAP_SIZE		339
 
-#define IWL_UEFI_WRDS_REVISION		2
-#define IWL_UEFI_EWRD_REVISION		2
-#define IWL_UEFI_WGDS_REVISION		3
-#define IWL_UEFI_MIN_PPAG_REV		1
-#define IWL_UEFI_MAX_PPAG_REV		4
 #define IWL_UEFI_MIN_WTAS_REVISION	1
 #define IWL_UEFI_MAX_WTAS_REVISION	2
 #define IWL_UEFI_SPLC_REVISION		0
@@ -63,6 +59,9 @@ struct uefi_cnv_wlan_uats_data {
 	u8 mcc_to_ap_type_map[IWL_UATS_MAP_SIZE - 1];
 } __packed;
 
+/* UNEB's layout is identical to UATS's */
+#define uefi_cnv_wlan_uneb_data uefi_cnv_wlan_uats_data
+
 struct uefi_cnv_common_step_data {
 	u8 revision;
 	u8 step_mode;
@@ -72,68 +71,135 @@ struct uefi_cnv_common_step_data {
 	u8 radio2;
 } __packed;
 
-/*
- * struct uefi_sar_profile - a SAR profile as defined in UEFI
- *
- * @chains: a per-chain table of SAR values
- */
-struct uefi_sar_profile {
-	struct iwl_sar_profile_chain chains[BIOS_SAR_MAX_CHAINS_PER_PROFILE];
-} __packed;
+#define UEFI_PPAG_SUB_BANDS_NUM_REV4	11
+#define UEFI_PPAG_SUB_BANDS_NUM_REV5	12
+#define UEFI_PPAG_NUM_CHAINS		2
 
-/*
+#define UEFI_SAR_SUB_BANDS_NUM_REV2	11
+#define UEFI_SAR_SUB_BANDS_NUM_REV3	12
+
+#define UEFI_SAR_MAX_CHAINS_PER_PROFILE	4
+
+#define UEFI_GEO_NUM_BANDS_REV3		3
+#define UEFI_GEO_NUM_BANDS_REV4		4
+
+/**
  * struct uefi_cnv_var_wrds - WRDS table as defined in UEFI
  *
  * @revision: the revision of the table
  * @mode: is WRDS enbaled/disabled
- * @sar_profile: sar profile #1
+ * @vals: values for sar profile #1 as an array:
+ *	vals[chain * num_of_subbands + subband] will return the right value.
+ *	num_of_subbands depends on the revision. For revision 3, it is
+ *	%UEFI_SAR_SUB_BANDS_NUM_REV3, for earlier revision, it is
+ *	%UEFI_SAR_SUB_BANDS_NUM_REV2.
+ *	The max number of chains is currently 2
  */
 struct uefi_cnv_var_wrds {
 	u8 revision;
 	u32 mode;
-	struct uefi_sar_profile sar_profile;
+	u8 vals[];
 } __packed;
 
-/*
+#define UEFI_SAR_PROFILE_SIZE_REV2			\
+	(sizeof(u8) * UEFI_SAR_MAX_CHAINS_PER_PROFILE *	\
+	 UEFI_SAR_SUB_BANDS_NUM_REV2)
+
+#define UEFI_SAR_PROFILE_SIZE_REV3			\
+	(sizeof(u8) * UEFI_SAR_MAX_CHAINS_PER_PROFILE *	\
+	 UEFI_SAR_SUB_BANDS_NUM_REV3)
+
+#define UEFI_SAR_WRDS_TABLE_SIZE_REV2			\
+	(offsetof(struct uefi_cnv_var_wrds, vals) +	\
+	 UEFI_SAR_PROFILE_SIZE_REV2)
+
+#define UEFI_SAR_WRDS_TABLE_SIZE_REV3			\
+	(offsetof(struct uefi_cnv_var_wrds, vals) +	\
+	 UEFI_SAR_PROFILE_SIZE_REV3)
+
+/**
  * struct uefi_cnv_var_ewrd - EWRD table as defined in UEFI
  * @revision: the revision of the table
  * @mode: is WRDS enbaled/disabled
  * @num_profiles: how many additional profiles we have in this table (0-3)
- * @sar_profiles: the additional SAR profiles (#2-#4)
+ * @vals: the additional SAR profiles (#2-#4) as an array of SAR profiles.
+ *	A SAR profile is defined the &struct uefi_cnv_var_wrds::vals. The size
+ *	of each profile depends on the number of subbands which depends on the
+ *	revision. This is explained in &struct uefi_cnv_var_wrds.
  */
 struct uefi_cnv_var_ewrd {
 	u8 revision;
 	u32 mode;
 	u32 num_profiles;
-	struct uefi_sar_profile sar_profiles[BIOS_SAR_MAX_PROFILE_NUM - 1];
+	u8 vals[];
 } __packed;
 
-/*
+#define UEFI_SAR_EWRD_TABLE_SIZE_REV2				\
+	(offsetof(struct uefi_cnv_var_ewrd, vals) +		\
+	 UEFI_SAR_PROFILE_SIZE_REV2 * (BIOS_SAR_MAX_PROFILE_NUM - 1))
+
+#define UEFI_SAR_EWRD_TABLE_SIZE_REV3				\
+	(offsetof(struct uefi_cnv_var_ewrd, vals) +		\
+	 UEFI_SAR_PROFILE_SIZE_REV3 * (BIOS_SAR_MAX_PROFILE_NUM - 1))
+
+/**
  * struct uefi_cnv_var_wgds - WGDS table as defined in UEFI
  * @revision: the revision of the table
  * @num_profiles: the number of geo profiles we have in the table.
  *	The first 3 are mandatory, and can have up to 8.
- * @geo_profiles: a per-profile table of the offsets to add to SAR values.
+ * @vals: a per-profile table of the offsets to add to SAR values. This is an
+ *	array of profiles, each profile is an array of
+ *	&struct iwl_geo_profile_band, one for each subband.
+ *	There are %UEFI_GEO_NUM_BANDS_REV3 or %UEFI_GEO_NUM_BANDS_REV4 subbands
+ *	depending on the revision.
  */
 struct uefi_cnv_var_wgds {
 	u8 revision;
 	u8 num_profiles;
-	struct iwl_geo_profile geo_profiles[BIOS_GEO_MAX_PROFILE_NUM];
+	u8 vals[];
 } __packed;
 
-/*
+/* struct iwl_geo_profile_band is 3 bytes-long, but since it is not packed,
+ * we can't use sizeof()
+ */
+#define UEFI_WGDS_PROFILE_SIZE_REV3 (sizeof(u8) * 3 * UEFI_GEO_NUM_BANDS_REV3)
+
+#define UEFI_WGDS_PROFILE_SIZE_REV4 (sizeof(u8) * 3 * UEFI_GEO_NUM_BANDS_REV4)
+
+#define UEFI_WGDS_TABLE_SIZE_REV3				\
+	(offsetof(struct uefi_cnv_var_wgds, vals) +		\
+	 UEFI_WGDS_PROFILE_SIZE_REV3 * BIOS_GEO_MAX_PROFILE_NUM)
+
+#define UEFI_WGDS_TABLE_SIZE_REV4				\
+	(offsetof(struct uefi_cnv_var_wgds, vals) +		\
+	 UEFI_WGDS_PROFILE_SIZE_REV4 * BIOS_GEO_MAX_PROFILE_NUM)
+
+/**
  * struct uefi_cnv_var_ppag - PPAG table as defined in UEFI
  * @revision: the revision of the table
  * @ppag_modes: values from &enum iwl_ppag_flags
- * @ppag_chains: the PPAG values per chain and band
+ * @vals: the PPAG values per chain and band as an array.
+ *	vals[chain * num_of_subbands + subband] will return the right value.
+ *	num_of_subbands depends on the revision. For revision 5, it is
+ *	%UEFI_PPAG_SUB_BANDS_NUM_REV5, for earlier revision it is
+ *	%UEFI_PPAG_SUB_BANDS_NUM_REV4.
+ *	the max number of chains is currently 2
  */
 struct uefi_cnv_var_ppag {
 	u8 revision;
 	u32 ppag_modes;
-	struct iwl_ppag_chain ppag_chains[IWL_NUM_CHAIN_LIMITS];
+	s8 vals[];
 } __packed;
 
-/* struct uefi_cnv_var_wtas - WTAS tabled as defined in UEFI
+#define UEFI_PPAG_DATA_SIZE_V4				\
+	(offsetof(struct uefi_cnv_var_ppag, vals) +	\
+	sizeof(s8) * UEFI_PPAG_NUM_CHAINS * UEFI_PPAG_SUB_BANDS_NUM_REV4)
+#define UEFI_PPAG_DATA_SIZE_V5				\
+	(offsetof(struct uefi_cnv_var_ppag, vals) +	\
+	sizeof(s8) * UEFI_PPAG_NUM_CHAINS * UEFI_PPAG_SUB_BANDS_NUM_REV5)
+
+/**
+ * struct uefi_cnv_var_wtas - WTAS tabled as defined in UEFI
  * @revision: the revision of the table
  * @tas_selection: different options of TAS enablement.
  * @black_list_size: the number of defined entried in the black list
@@ -146,7 +212,8 @@ struct uefi_cnv_var_wtas {
 	u16 black_list[IWL_WTAS_BLACK_LIST_MAX];
 } __packed;
 
-/* struct uefi_cnv_var_splc - SPLC tabled as defined in UEFI
+/**
+ * struct uefi_cnv_var_splc - SPLC tabled as defined in UEFI
  * @revision: the revision of the table
  * @default_pwr_limit: The default maximum power per device
  */
@@ -155,7 +222,8 @@ struct uefi_cnv_var_splc {
 	u32 default_pwr_limit;
 } __packed;
 
-/* struct uefi_cnv_var_wrdd - WRDD table as defined in UEFI
+/**
+ * struct uefi_cnv_var_wrdd - WRDD table as defined in UEFI
  * @revision: the revision of the table
  * @mcc: country identifier as defined in ISO/IEC 3166-1 Alpha 2 code
  */
@@ -164,7 +232,8 @@ struct uefi_cnv_var_wrdd {
 	u32 mcc;
 } __packed;
 
-/* struct uefi_cnv_var_eckv - ECKV table as defined in UEFI
+/**
+ * struct uefi_cnv_var_eckv - ECKV table as defined in UEFI
  * @revision: the revision of the table
  * @ext_clock_valid: indicates if external 32KHz clock is valid
  */
@@ -175,7 +244,8 @@ struct uefi_cnv_var_eckv {
 
 #define UEFI_MAX_DSM_FUNCS 32
 
-/* struct uefi_cnv_var_general_cfg - DSM-like table as defined in UEFI
+/**
+ * struct uefi_cnv_var_general_cfg - DSM-like table as defined in UEFI
  * @revision: the revision of the table
  * @functions: payload of the different DSM functions
  */
@@ -185,7 +255,9 @@ struct uefi_cnv_var_general_cfg {
 } __packed;
 
 #define IWL_UEFI_WBEM_REV0_MASK (BIT(0) | BIT(1))
-/* struct uefi_cnv_wlan_wbem_data - Bandwidth enablement per MCC as defined
+
+/**
+ * struct uefi_cnv_wlan_wbem_data - Bandwidth enablement per MCC as defined
  *	in UEFI
  * @revision: the revision of the table
  * @wbem_320mhz_per_mcc: enablement of 320MHz bandwidth per MCC
@@ -274,6 +346,8 @@ int iwl_uefi_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 void iwl_uefi_get_sgom_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwrt);
 void iwl_uefi_get_uats_table(struct iwl_trans *trans,
 			     struct iwl_fw_runtime *fwrt);
+void iwl_uefi_get_uneb_table(struct iwl_trans *trans,
+			     struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
 int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt);
@@ -373,6 +447,11 @@ iwl_uefi_get_uats_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwrt)
 {
 }
 
+static inline void
+iwl_uefi_get_uneb_table(struct iwl_trans *trans, struct iwl_fw_runtime *fwrt)
+{
+}
+
 static inline
 int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index 45cf2bc68e41d..5f40cd15e27f7 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -85,7 +85,6 @@ enum iwl_nvm_type {
 #define IWL_WATCHDOG_DISABLED	0
 #define IWL_DEF_WD_TIMEOUT	2500
 #define IWL_LONG_WD_TIMEOUT	10000
-#define IWL_MAX_WD_TIMEOUT	120000
 
 #define IWL_DEFAULT_MAX_TX_POWER 22
 #define IWL_TX_CSUM_NETIF_FLAGS (NETIF_F_IPV6_CSUM | NETIF_F_IP_CSUM |\
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
index 89901786fd687..16b2c313e72b2 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
@@ -138,7 +138,7 @@ iwl_trans_determine_restart_mode(struct iwl_trans *trans)
 		IWL_RESET_MODE_FUNC_RESET,
 		IWL_RESET_MODE_PROD_RESET,
 	};
-	static const enum iwl_reset_mode escalation_list_sc[] = {
+	static const enum iwl_reset_mode escalation_list_top[] = {
 		IWL_RESET_MODE_SW_RESET,
 		IWL_RESET_MODE_REPROBE,
 		IWL_RESET_MODE_REPROBE,
@@ -159,14 +159,14 @@ iwl_trans_determine_restart_mode(struct iwl_trans *trans)
 
 	if (trans->request_top_reset) {
 		trans->request_top_reset = 0;
-		if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC)
+		if (iwl_trans_is_top_reset_supported(trans))
 			return IWL_RESET_MODE_TOP_RESET;
 		return IWL_RESET_MODE_PROD_RESET;
 	}
 
-	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_SC) {
-		escalation_list = escalation_list_sc;
-		escalation_list_size = ARRAY_SIZE(escalation_list_sc);
+	if (iwl_trans_is_top_reset_supported(trans)) {
+		escalation_list = escalation_list_top;
+		escalation_list_size = ARRAY_SIZE(escalation_list_top);
 	} else {
 		escalation_list = escalation_list_old;
 		escalation_list_size = ARRAY_SIZE(escalation_list_old);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index 688f9fee28210..61e4f4776dcb3 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -1088,7 +1088,7 @@ static inline void iwl_trans_schedule_reset(struct iwl_trans *trans,
 	 */
 	trans->restart.during_reset = test_bit(STATUS_IN_SW_RESET,
 					       &trans->status);
-	queue_delayed_work(system_unbound_wq, &trans->restart.wk, 0);
+	queue_delayed_work(system_dfl_wq, &trans->restart.wk, 0);
 }
 
 static inline void iwl_trans_fw_error(struct iwl_trans *trans,
@@ -1258,4 +1258,22 @@ bool iwl_trans_is_pm_supported(struct iwl_trans *trans);
 
 bool iwl_trans_is_ltr_enabled(struct iwl_trans *trans);
 
+static inline bool iwl_trans_is_top_reset_supported(struct iwl_trans *trans)
+{
+	/* not supported before Sc family */
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC)
+		return false;
+
+	/* for Sc family only supported for Sc2/Sc2f */
+	if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_SC &&
+	    CSR_HW_REV_TYPE(trans->info.hw_rev) == IWL_CFG_MAC_TYPE_SC)
+		return false;
+
+	/* so far these numbers are increasing - not before Pe */
+	if (CSR_HW_RFID_TYPE(trans->info.hw_rf_id) < IWL_CFG_RF_TYPE_PE)
+		return false;
+
+	return true;
+}
+
 #endif /* __iwl_trans_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/constants.h b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
index 5d23a618ae3ca..e2a5eecc18c34 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/constants.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
@@ -36,7 +36,6 @@
 #define IWL_MLD_PS_HEAVY_RX_THLD_PACKETS	8
 
 #define IWL_MLD_TRIGGER_LINK_SEL_TIME_SEC	30
-#define IWL_MLD_SCAN_EXPIRE_TIME_SEC		20
 
 #define IWL_MLD_TPT_COUNT_WINDOW (5 * HZ)
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/fw.c b/drivers/net/wireless/intel/iwlwifi/mld/fw.c
index 19da521a4babc..7b1fb84a641c7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/fw.c
@@ -513,7 +513,7 @@ static int iwl_mld_config_fw(struct iwl_mld *mld)
 		return ret;
 
 	iwl_mld_init_tas(mld);
-	iwl_mld_init_uats(mld);
+	iwl_mld_init_ap_type_tables(mld);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
index 29df747c89380..9215fc7e2eca7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
@@ -111,14 +111,75 @@ static bool iwl_mld_is_nic_ack_enabled(struct iwl_mld *mld,
 			       IEEE80211_HE_MAC_CAP2_ACK_EN);
 }
 
-static void iwl_mld_set_he_support(struct iwl_mld *mld,
-				   struct ieee80211_vif *vif,
-				   struct iwl_mac_config_cmd *cmd)
+struct iwl_mld_mac_wifi_gen_sta_iter_data {
+	struct ieee80211_vif *vif;
+	struct iwl_mac_wifi_gen_support *support;
+};
+
+static void iwl_mld_mac_wifi_gen_sta_iter(void *_data,
+					  struct ieee80211_sta *sta)
 {
-	if (vif->type == NL80211_IFTYPE_AP)
-		cmd->wifi_gen.he_ap_support = 1;
-	else
-		cmd->wifi_gen.he_support = 1;
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct iwl_mld_mac_wifi_gen_sta_iter_data *data = _data;
+	struct ieee80211_link_sta *link_sta;
+	unsigned int link_id;
+
+	if (mld_sta->vif != data->vif)
+		return;
+
+	for_each_sta_active_link(data->vif, sta, link_sta, link_id) {
+		if (link_sta->he_cap.has_he)
+			data->support->he_support = 1;
+		if (link_sta->eht_cap.has_eht)
+			data->support->eht_support = 1;
+	}
+}
+
+static void iwl_mld_set_wifi_gen(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 struct iwl_mac_wifi_gen_support *support)
+{
+	struct iwl_mld_mac_wifi_gen_sta_iter_data sta_iter_data = {
+		.vif = vif,
+		.support = support,
+	};
+	struct ieee80211_bss_conf *link_conf;
+	unsigned int link_id;
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_MONITOR:
+		/* for sniffer, set to HW capabilities */
+		support->he_support = 1;
+		support->eht_support = mld->trans->cfg->eht_supported;
+		break;
+	case NL80211_IFTYPE_AP:
+		/* for AP set according to the link configs */
+		for_each_vif_active_link(vif, link_conf, link_id) {
+			support->he_ap_support |= link_conf->he_support;
+			support->eht_support |= link_conf->eht_support;
+		}
+		break;
+	default:
+		/*
+		 * If we have MLO enabled, then the firmware needs to enable
+		 * address translation for the station(s) we add. That depends
+		 * on having EHT enabled in firmware, which in turn depends on
+		 * mac80211 in the iteration below.
+		 * However, mac80211 doesn't enable capabilities on the AP STA
+		 * until it has parsed the association response successfully,
+		 * so set EHT (and HE as a pre-requisite for EHT) when the vif
+		 * is an MLD.
+		 */
+		if (ieee80211_vif_is_mld(vif)) {
+			support->he_support = 1;
+			support->eht_support = 1;
+		}
+
+		ieee80211_iterate_stations_mtx(mld->hw,
+					       iwl_mld_mac_wifi_gen_sta_iter,
+					       &sta_iter_data);
+		break;
+	}
 }
 
 /* fill the common part for all interface types */
@@ -128,8 +189,6 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld,
 					u32 action)
 {
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-	struct ieee80211_bss_conf *link_conf;
-	unsigned int link_id;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
@@ -147,29 +206,7 @@ static void iwl_mld_mac_cmd_fill_common(struct iwl_mld *mld,
 	cmd->nic_not_ack_enabled =
 		cpu_to_le32(!iwl_mld_is_nic_ack_enabled(mld, vif));
 
-	/* If we have MLO enabled, then the firmware needs to enable
-	 * address translation for the station(s) we add. That depends
-	 * on having EHT enabled in firmware, which in turn depends on
-	 * mac80211 in the code below.
-	 * However, mac80211 doesn't enable HE/EHT until it has parsed
-	 * the association response successfully, so just skip all that
-	 * and enable both when we have MLO.
-	 */
-	if (ieee80211_vif_is_mld(vif)) {
-		iwl_mld_set_he_support(mld, vif, cmd);
-		cmd->wifi_gen.eht_support = 1;
-		return;
-	}
-
-	for_each_vif_active_link(vif, link_conf, link_id) {
-		if (!link_conf->he_support)
-			continue;
-
-		iwl_mld_set_he_support(mld, vif, cmd);
-
-		/* EHT, if supported, was already set above */
-		break;
-	}
+	iwl_mld_set_wifi_gen(mld, vif, &cmd->wifi_gen);
 }
 
 static void iwl_mld_fill_mac_cmd_sta(struct iwl_mld *mld,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index 0c53d6bd96514..71a9a72c9ac0e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -1761,6 +1761,16 @@ static int iwl_mld_move_sta_state_up(struct iwl_mld *mld,
 
 		if (vif->type == NL80211_IFTYPE_STATION)
 			iwl_mld_link_set_2mhz_block(mld, vif, sta);
+
+		if (sta->tdls) {
+			/*
+			 * update MAC since wifi generation flags may change,
+			 * we also update MAC on association to the AP via the
+			 * vif assoc change
+			 */
+			iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY);
+		}
+
 		/* Now the link_sta's capabilities are set, update the FW */
 		iwl_mld_config_tlc(mld, vif, sta);
 
@@ -1873,6 +1883,15 @@ static int iwl_mld_move_sta_state_down(struct iwl_mld *mld,
 			/* just removed last TDLS STA, so enable PM */
 			iwl_mld_update_mac_power(mld, vif, false);
 		}
+
+		if (sta->tdls) {
+			/*
+			 * update MAC since wifi generation flags may change,
+			 * we also update MAC on disassociation to the AP via
+			 * the vif assoc change
+			 */
+			iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY);
+		}
 	} else {
 		return -EINVAL;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
index f842f5183223b..f693f92e42b4c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
@@ -110,7 +110,6 @@ void iwl_mld_emlsr_tmp_non_bss_done_wk(struct wiphy *wiphy,
 }
 
 #define IWL_MLD_TRIGGER_LINK_SEL_TIME	(HZ * IWL_MLD_TRIGGER_LINK_SEL_TIME_SEC)
-#define IWL_MLD_SCAN_EXPIRE_TIME	(HZ * IWL_MLD_SCAN_EXPIRE_TIME_SEC)
 
 /* Exit reasons that can cause longer EMLSR prevention */
 #define IWL_MLD_PREVENT_EMLSR_REASONS	(IWL_MLD_EMLSR_EXIT_MISSED_BEACON	| \
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.c b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
index 2dbd3d58b0c62..4d8e85f2bd7c2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/nan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
@@ -54,9 +54,8 @@ static int iwl_mld_nan_config(struct iwl_mld *mld,
 	ether_addr_copy(cmd.nmi_addr, vif->addr);
 	cmd.master_pref = conf->master_pref;
 
-	if (conf->cluster_id)
-		memcpy(cmd.cluster_id, conf->cluster_id + 4,
-		       sizeof(cmd.cluster_id));
+	memcpy(cmd.cluster_id, conf->cluster_id + 4,
+	       sizeof(cmd.cluster_id));
 
 	cmd.scan_period = conf->scan_period < 255 ? conf->scan_period : 255;
 	cmd.dwell_time =
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c
index c3318e84f4a27..49b0d9f8f865d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c
@@ -405,7 +405,10 @@ int iwl_mld_set_tx_power(struct iwl_mld *mld,
 		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_LINK),
 		.common.pwr_restriction = cpu_to_le16(u_tx_power),
 	};
-	int len = sizeof(cmd.common) + sizeof(cmd.v10);
+	int len = sizeof(cmd.common) + sizeof(cmd.v11);
+
+	if (iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 10) == 10)
+		len = sizeof(cmd.common) + sizeof(cmd.v10);
 
 	if (WARN_ON(!mld_link))
 		return -ENODEV;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
index 6ab5a34103532..2486d78d6fc38 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
@@ -64,6 +64,7 @@ void iwl_mld_get_bios_tables(struct iwl_mld *mld)
 	}
 
 	iwl_uefi_get_uats_table(mld->trans, &mld->fwrt);
+	iwl_uefi_get_uneb_table(mld->trans, &mld->fwrt);
 
 	iwl_bios_get_phy_filters(&mld->fwrt);
 }
@@ -72,16 +73,36 @@ static int iwl_mld_geo_sar_init(struct iwl_mld *mld)
 {
 	u32 cmd_id = WIDE_ID(PHY_OPS_GROUP, PER_CHAIN_LIMIT_OFFSET_CMD);
 	/* Only set to South Korea if the table revision is 1 */
-	__le32 sk = cpu_to_le32(mld->fwrt.geo_rev == 1 ? 1 : 0);
+	u8 sk = mld->fwrt.geo_rev == 1 ? 1 : 0;
 	union iwl_geo_tx_power_profiles_cmd cmd = {
 		.v5.ops = cpu_to_le32(IWL_PER_CHAIN_OFFSET_SET_TABLES),
-		.v5.table_revision = sk,
 	};
+	u32 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0);
+	int n_subbands;
+	int cmd_size;
 	int ret;
 
-	ret = iwl_sar_geo_fill_table(&mld->fwrt, &cmd.v5.table[0][0],
-				     ARRAY_SIZE(cmd.v5.table[0]),
-				     BIOS_GEO_MAX_PROFILE_NUM);
+	switch (cmd_ver) {
+	case 5:
+		n_subbands = ARRAY_SIZE(cmd.v5.table[0]);
+		cmd.v5.table_revision = cpu_to_le32(sk);
+		cmd_size = sizeof(cmd.v5);
+		break;
+	case 6:
+		n_subbands = ARRAY_SIZE(cmd.v6.table[0]);
+		cmd.v6.bios_hdr.table_revision = mld->fwrt.geo_rev;
+		cmd.v6.bios_hdr.table_source = mld->fwrt.geo_bios_source;
+		cmd_size = sizeof(cmd.v6);
+		break;
+	default:
+		WARN(false, "unsupported version: %d", cmd_ver);
+		return -EINVAL;
+	}
+
+	BUILD_BUG_ON(offsetof(typeof(cmd), v6.table) !=
+		     offsetof(typeof(cmd), v5.table));
+	ret = iwl_sar_geo_fill_table(&mld->fwrt, &cmd.v6.table[0][0],
+				     n_subbands, BIOS_GEO_MAX_PROFILE_NUM);
 
 	/* It is a valid scenario to not support SAR, or miss wgds table,
 	 * but in that case there is no need to send the command.
@@ -89,28 +110,48 @@ static int iwl_mld_geo_sar_init(struct iwl_mld *mld)
 	if (ret)
 		return 0;
 
-	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, sizeof(cmd.v5));
+	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, cmd_size);
 }
 
 int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
 {
-	u32 cmd_id = REDUCE_TX_POWER_CMD;
 	struct iwl_dev_tx_power_cmd cmd = {
 		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
-		.v10.flags = cpu_to_le32(mld->fwrt.reduced_power_flags),
 	};
+	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, REDUCE_TX_POWER_CMD, 10);
+	int num_subbands;
+	int cmd_size;
 	int ret;
 
+	switch (cmd_ver) {
+	case 10:
+		cmd.v10.flags = cpu_to_le32(mld->fwrt.reduced_power_flags);
+		cmd_size = sizeof(cmd.common) + sizeof(cmd.v10);
+		num_subbands = IWL_NUM_SUB_BANDS_V2;
+		break;
+	case 11:
+		cmd.v11.flags = cpu_to_le32(mld->fwrt.reduced_power_flags);
+		cmd_size = sizeof(cmd.common) + sizeof(cmd.v11);
+		num_subbands = IWL_NUM_SUB_BANDS_V3;
+		break;
+	default:
+		WARN_ONCE(1, "Bad version for REDUCE_TX_POWER_CMD: %d\n",
+			  cmd_ver);
+		return -EOPNOTSUPP;
+	}
+
 	/* TODO: CDB - support IWL_NUM_CHAIN_TABLES_V2 */
-	ret = iwl_sar_fill_profile(&mld->fwrt, &cmd.v10.per_chain[0][0][0],
-				   IWL_NUM_CHAIN_TABLES, IWL_NUM_SUB_BANDS_V2,
+	/* v10 and v11 have the same position for per_chain */
+	BUILD_BUG_ON(offsetof(typeof(cmd), v11.per_chain) !=
+		     offsetof(typeof(cmd), v10.per_chain));
+	ret = iwl_sar_fill_profile(&mld->fwrt, &cmd.v11.per_chain[0][0][0],
+				   IWL_NUM_CHAIN_TABLES, num_subbands,
 				   prof_a, prof_b);
 	/* return on error or if the profile is disabled (positive number) */
 	if (ret)
 		return ret;
 
-	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd,
-				    sizeof(cmd.common) + sizeof(cmd.v10));
+	return iwl_mld_send_cmd_pdu(mld, REDUCE_TX_POWER_CMD, &cmd, cmd_size);
 }
 
 int iwl_mld_init_sar(struct iwl_mld *mld)
@@ -165,30 +206,74 @@ static int iwl_mld_ppag_send_cmd(struct iwl_mld *mld)
 {
 	struct iwl_fw_runtime *fwrt = &mld->fwrt;
 	union iwl_ppag_table_cmd cmd = {
-		.v7.ppag_config_info.hdr.table_source = fwrt->ppag_bios_source,
-		.v7.ppag_config_info.hdr.table_revision = fwrt->ppag_bios_rev,
-		.v7.ppag_config_info.value = cpu_to_le32(fwrt->ppag_flags),
+		/* v7 and v8 have the same layout for the ppag_config_info */
+		.v8.ppag_config_info.hdr.table_source = fwrt->ppag_bios_source,
+		.v8.ppag_config_info.hdr.table_revision = fwrt->ppag_bios_rev,
+		.v8.ppag_config_info.value = cpu_to_le32(fwrt->ppag_flags),
 	};
+	int cmd_ver =
+		iwl_fw_lookup_cmd_ver(mld->fw,
+				      WIDE_ID(PHY_OPS_GROUP,
+					      PER_PLATFORM_ANT_GAIN_CMD), 1);
+	int cmd_len = sizeof(cmd.v8);
 	int ret;
 
+	BUILD_BUG_ON(offsetof(typeof(cmd), v8.ppag_config_info.hdr) !=
+		     offsetof(typeof(cmd), v7.ppag_config_info.hdr));
+	BUILD_BUG_ON(offsetof(typeof(cmd), v8.gain) !=
+		     offsetof(typeof(cmd), v7.gain));
+
+	BUILD_BUG_ON(ARRAY_SIZE(cmd.v7.gain) > ARRAY_SIZE(fwrt->ppag_chains));
+	BUILD_BUG_ON(ARRAY_SIZE(cmd.v7.gain[0]) >
+		     ARRAY_SIZE(fwrt->ppag_chains[0].subbands));
+	BUILD_BUG_ON(ARRAY_SIZE(cmd.v8.gain) > ARRAY_SIZE(fwrt->ppag_chains));
+	BUILD_BUG_ON(ARRAY_SIZE(cmd.v8.gain[0]) >
+		     ARRAY_SIZE(fwrt->ppag_chains[0].subbands));
+
 	IWL_DEBUG_RADIO(fwrt,
 			"PPAG MODE bits going to be sent: %d\n",
 			fwrt->ppag_flags);
 
-	for (int chain = 0; chain < IWL_NUM_CHAIN_LIMITS; chain++) {
-		for (int subband = 0; subband < IWL_NUM_SUB_BANDS_V2; subband++) {
-			cmd.v7.gain[chain][subband] =
-				fwrt->ppag_chains[chain].subbands[subband];
-			IWL_DEBUG_RADIO(fwrt,
-					"PPAG table: chain[%d] band[%d]: gain = %d\n",
-					chain, subband, cmd.v7.gain[chain][subband]);
+	/* Since ver 7 will be deprecated at some point, don't bother making
+	 * this code generic for both ver 7 and ver 8: duplicate the code.
+	 */
+	if (cmd_ver == 7) {
+		for (int chain = 0; chain < ARRAY_SIZE(cmd.v7.gain); chain++) {
+			for (int subband = 0;
+			     subband < ARRAY_SIZE(cmd.v7.gain[0]);
+			     subband++) {
+				cmd.v7.gain[chain][subband] =
+					fwrt->ppag_chains[chain].subbands[subband];
+				IWL_DEBUG_RADIO(fwrt,
+						"PPAG table: chain[%d] band[%d]: gain = %d\n",
+						chain, subband,
+						cmd.v7.gain[chain][subband]);
+			}
 		}
+		cmd_len = sizeof(cmd.v7);
+	} else if (cmd_ver == 8) {
+		for (int chain = 0; chain < ARRAY_SIZE(cmd.v8.gain); chain++) {
+			for (int subband = 0;
+			     subband < ARRAY_SIZE(cmd.v8.gain[0]);
+			     subband++) {
+				cmd.v8.gain[chain][subband] =
+					fwrt->ppag_chains[chain].subbands[subband];
+				IWL_DEBUG_RADIO(fwrt,
+						"PPAG table: chain[%d] band[%d]: gain = %d\n",
+						chain, subband,
+						cmd.v8.gain[chain][subband]);
+			}
+		}
+	} else {
+		WARN(1, "Bad version for PER_PLATFORM_ANT_GAIN_CMD %d\n",
+		     cmd_ver);
+		return -EINVAL;
 	}
 
 	IWL_DEBUG_RADIO(mld, "Sending PER_PLATFORM_ANT_GAIN_CMD\n");
 	ret = iwl_mld_send_cmd_pdu(mld, WIDE_ID(PHY_OPS_GROUP,
 						PER_PLATFORM_ANT_GAIN_CMD),
-				   &cmd, sizeof(cmd.v7));
+				   &cmd, cmd_len);
 	if (ret < 0)
 		IWL_ERR(mld, "failed to send PER_PLATFORM_ANT_GAIN_CMD (%d)\n",
 			ret);
@@ -352,21 +437,42 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 				ret);
 }
 
-void iwl_mld_init_uats(struct iwl_mld *mld)
+void iwl_mld_init_ap_type_tables(struct iwl_mld *mld)
 {
 	int ret;
 	struct iwl_host_cmd cmd = {
 		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
 			      MCC_ALLOWED_AP_TYPE_CMD),
-		.data[0] = &mld->fwrt.uats_table,
-		.len[0] =  sizeof(mld->fwrt.uats_table),
+		.data[0] = &mld->fwrt.ap_type_cmd,
+		.len[0] =  sizeof(mld->fwrt.ap_type_cmd),
 		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
 	};
 
-	if (!mld->fwrt.uats_valid)
+	if (!mld->fwrt.ap_type_cmd_valid)
 		return;
 
-	ret = iwl_mld_send_cmd(mld, &cmd);
+	if (iwl_fw_lookup_cmd_ver(mld->fw, cmd.id, 1) == 1) {
+		struct iwl_mcc_allowed_ap_type_cmd_v1 *cmd_v1 =
+			kzalloc(sizeof(*cmd_v1), GFP_KERNEL);
+
+		if (!cmd_v1)
+			return;
+
+		BUILD_BUG_ON(sizeof(mld->fwrt.ap_type_cmd.mcc_to_ap_type_map) !=
+			     sizeof(cmd_v1->mcc_to_ap_type_map));
+
+		memcpy(cmd_v1->mcc_to_ap_type_map,
+		       mld->fwrt.ap_type_cmd.mcc_to_ap_type_map,
+		       sizeof(mld->fwrt.ap_type_cmd.mcc_to_ap_type_map));
+
+		cmd.data[0] = cmd_v1;
+		cmd.len[0] = sizeof(*cmd_v1);
+		ret = iwl_mld_send_cmd(mld, &cmd);
+		kfree(cmd_v1);
+	} else {
+		ret = iwl_mld_send_cmd(mld, &cmd);
+	}
+
 	if (ret)
 		IWL_ERR(mld, "failed to send MCC_ALLOWED_AP_TYPE_CMD (%d)\n",
 			ret);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
index 3b01c645adda3..5498c19789f4e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
@@ -9,7 +9,7 @@
 
 void iwl_mld_get_bios_tables(struct iwl_mld *mld);
 void iwl_mld_configure_lari(struct iwl_mld *mld);
-void iwl_mld_init_uats(struct iwl_mld *mld);
+void iwl_mld_init_ap_type_tables(struct iwl_mld *mld);
 void iwl_mld_init_tas(struct iwl_mld *mld);
 
 int iwl_mld_init_ppag(struct iwl_mld *mld);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.c b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
index a1a4cf3ab3d31..17e0b13b5ce8f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
@@ -47,8 +47,6 @@
 /* adaptive dwell number of APs override mask for social channels */
 #define IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS_BIT BIT(21)
 
-#define SCAN_TIMEOUT_MSEC (30000 * HZ)
-
 /* minimal number of 2GHz and 5GHz channels in the regular scan request */
 #define IWL_MLD_6GHZ_PASSIVE_SCAN_MIN_CHANS 4
 
@@ -116,6 +114,13 @@ struct iwl_mld_scan_params {
 	u8 bssid[ETH_ALEN] __aligned(2);
 };
 
+struct iwl_scan_req_params_ptrs {
+	struct iwl_scan_general_params_v11 *general_params;
+	struct iwl_scan_channel_params_v8 *channel_params;
+	struct iwl_scan_periodic_parms_v1 *periodic_params;
+	struct iwl_scan_probe_params_v4 *probe_params;
+};
+
 struct iwl_mld_scan_respect_p2p_go_iter_data {
 	struct ieee80211_vif *current_vif;
 	bool p2p_go;
@@ -512,9 +517,10 @@ iwl_mld_scan_get_cmd_gen_flags2(struct iwl_mld *mld,
 
 static void
 iwl_mld_scan_cmd_set_dwell(struct iwl_mld *mld,
-			   struct iwl_scan_general_params_v11 *gp,
-			   struct iwl_mld_scan_params *params)
+			   struct iwl_mld_scan_params *params,
+			   struct iwl_scan_req_params_ptrs *scan_ptrs)
 {
+	struct iwl_scan_general_params_v11 *gp = scan_ptrs->general_params;
 	const struct iwl_mld_scan_timing_params *timing =
 		&scan_timing[params->type];
 
@@ -551,9 +557,10 @@ static void
 iwl_mld_scan_cmd_set_gen_params(struct iwl_mld *mld,
 				struct iwl_mld_scan_params *params,
 				struct ieee80211_vif *vif,
-				struct iwl_scan_general_params_v11 *gp,
+				struct iwl_scan_req_params_ptrs *scan_ptrs,
 				enum iwl_mld_scan_status scan_status)
 {
+	struct iwl_scan_general_params_v11 *gp = scan_ptrs->general_params;
 	u16 gen_flags = iwl_mld_scan_get_cmd_gen_flags(mld, params, vif,
 						       scan_status);
 	u8 gen_flags2 = iwl_mld_scan_get_cmd_gen_flags2(mld, params, vif,
@@ -566,7 +573,7 @@ iwl_mld_scan_cmd_set_gen_params(struct iwl_mld *mld,
 	gp->flags = cpu_to_le16(gen_flags);
 	gp->flags2 = gen_flags2;
 
-	iwl_mld_scan_cmd_set_dwell(mld, gp, params);
+	iwl_mld_scan_cmd_set_dwell(mld, params, scan_ptrs);
 
 	if (gen_flags & IWL_UMAC_SCAN_GEN_FLAGS_V2_FRAGMENTED_LMAC1)
 		gp->num_of_fragments[SCAN_LB_LMAC_IDX] = IWL_SCAN_NUM_OF_FRAGS;
@@ -577,9 +584,12 @@ iwl_mld_scan_cmd_set_gen_params(struct iwl_mld *mld,
 
 static int
 iwl_mld_scan_cmd_set_sched_params(struct iwl_mld_scan_params *params,
-				  struct iwl_scan_umac_schedule *schedule,
-				  __le16 *delay)
+				  struct iwl_scan_req_params_ptrs *scan_ptrs)
 {
+	struct iwl_scan_umac_schedule *schedule =
+		scan_ptrs->periodic_params->schedule;
+	__le16 *delay = &scan_ptrs->periodic_params->delay;
+
 	if (WARN_ON(!params->n_scan_plans ||
 		    params->n_scan_plans > IWL_MAX_SCHED_SCAN_PLANS))
 		return -EINVAL;
@@ -657,11 +667,12 @@ iwl_mld_scan_cmd_build_ssids(struct iwl_mld_scan_params *params,
 
 static void
 iwl_mld_scan_fill_6g_chan_list(struct iwl_mld_scan_params *params,
-			       struct iwl_scan_probe_params_v4 *pp)
+			       struct iwl_scan_req_params_ptrs *scan_ptrs)
 {
 	int j, idex_s = 0, idex_b = 0;
 	struct cfg80211_scan_6ghz_params *scan_6ghz_params =
 		params->scan_6ghz_params;
+	struct iwl_scan_probe_params_v4 *pp = scan_ptrs->probe_params;
 
 	for (j = 0;
 	     j < params->n_ssids && idex_s < SCAN_SHORT_SSID_MAX_SIZE;
@@ -725,13 +736,15 @@ iwl_mld_scan_fill_6g_chan_list(struct iwl_mld_scan_params *params,
 
 static void
 iwl_mld_scan_cmd_set_probe_params(struct iwl_mld_scan_params *params,
-				  struct iwl_scan_probe_params_v4 *pp,
+				  struct iwl_scan_req_params_ptrs *scan_ptrs,
 				  u32 *bitmap_ssid)
 {
+	struct iwl_scan_probe_params_v4 *pp = scan_ptrs->probe_params;
+
 	pp->preq = params->preq;
 
 	if (params->scan_6ghz) {
-		iwl_mld_scan_fill_6g_chan_list(params, pp);
+		iwl_mld_scan_fill_6g_chan_list(params, scan_ptrs);
 		return;
 	}
 
@@ -821,10 +834,12 @@ static u32 iwl_mld_scan_ch_n_aps_flag(enum nl80211_iftype vif_type, u8 ch_id)
 static void
 iwl_mld_scan_cmd_set_channels(struct iwl_mld *mld,
 			      struct ieee80211_channel **channels,
-			      struct iwl_scan_channel_params_v7 *cp,
+			      struct iwl_scan_req_params_ptrs *scan_ptrs,
 			      int n_channels, u32 flags,
 			      enum nl80211_iftype vif_type)
 {
+	struct iwl_scan_channel_params_v8 *cp = scan_ptrs->channel_params;
+
 	for (int i = 0; i < n_channels; i++) {
 		enum nl80211_band band = channels[i]->band;
 		struct iwl_scan_channel_cfg_umac *cfg = &cp->channel_config[i];
@@ -862,10 +877,11 @@ static u8
 iwl_mld_scan_cfg_channels_6g(struct iwl_mld *mld,
 			     struct iwl_mld_scan_params *params,
 			     u32 n_channels,
-			     struct iwl_scan_probe_params_v4 *pp,
-			     struct iwl_scan_channel_params_v7 *cp,
+			     struct iwl_scan_req_params_ptrs *scan_ptrs,
 			     enum nl80211_iftype vif_type)
 {
+	struct iwl_scan_probe_params_v4 *pp = scan_ptrs->probe_params;
+	struct iwl_scan_channel_params_v8 *cp = scan_ptrs->channel_params;
 	struct cfg80211_scan_6ghz_params *scan_6ghz_params =
 		params->scan_6ghz_params;
 	u32 i;
@@ -1063,25 +1079,23 @@ static int
 iwl_mld_scan_cmd_set_6ghz_chan_params(struct iwl_mld *mld,
 				      struct iwl_mld_scan_params *params,
 				      struct ieee80211_vif *vif,
-				      struct iwl_scan_req_params_v17 *scan_p)
+				      struct iwl_scan_req_params_ptrs *scan_ptrs)
 {
-	struct iwl_scan_channel_params_v7 *chan_p = &scan_p->channel_params;
-	struct iwl_scan_probe_params_v4 *probe_p = &scan_p->probe_params;
+	struct iwl_scan_channel_params_v8 *cp = scan_ptrs->channel_params;
 
 	/* Explicitly clear the flags since most of them are not
 	 * relevant for 6 GHz scan.
 	 */
-	chan_p->flags = 0;
-	chan_p->count = iwl_mld_scan_cfg_channels_6g(mld, params,
-						     params->n_channels,
-						     probe_p, chan_p,
-						     vif->type);
-	if (!chan_p->count)
+	cp->flags = 0;
+	cp->count = iwl_mld_scan_cfg_channels_6g(mld, params,
+						 params->n_channels,
+						 scan_ptrs, vif->type);
+	if (!cp->count)
 		return -EINVAL;
 
 	if (!params->n_ssids ||
 	    (params->n_ssids == 1 && !params->ssids[0].ssid_len))
-		chan_p->flags |= IWL_SCAN_CHANNEL_FLAG_6G_PSC_NO_FILTER;
+		cp->flags |= IWL_SCAN_CHANNEL_FLAG_6G_PSC_NO_FILTER;
 
 	return 0;
 }
@@ -1090,12 +1104,12 @@ static int
 iwl_mld_scan_cmd_set_chan_params(struct iwl_mld *mld,
 				 struct iwl_mld_scan_params *params,
 				 struct ieee80211_vif *vif,
-				 struct iwl_scan_req_params_v17 *scan_p,
+				 struct iwl_scan_req_params_ptrs *scan_ptrs,
 				 bool low_latency,
 				 enum iwl_mld_scan_status scan_status,
 				 u32 channel_cfg_flags)
 {
-	struct iwl_scan_channel_params_v7 *cp = &scan_p->channel_params;
+	struct iwl_scan_channel_params_v8 *cp = scan_ptrs->channel_params;
 	struct ieee80211_supported_band *sband =
 		&mld->nvm_data->bands[NL80211_BAND_6GHZ];
 
@@ -1107,14 +1121,14 @@ iwl_mld_scan_cmd_set_chan_params(struct iwl_mld *mld,
 
 	if (params->scan_6ghz)
 		return iwl_mld_scan_cmd_set_6ghz_chan_params(mld, params,
-							     vif, scan_p);
+							     vif, scan_ptrs);
 
 	/* relevant only for 2.4 GHz/5 GHz scan */
 	cp->flags = iwl_mld_scan_cmd_set_chan_flags(mld, params, vif,
 						    low_latency);
 	cp->count = params->n_channels;
 
-	iwl_mld_scan_cmd_set_channels(mld, params->channels, cp,
+	iwl_mld_scan_cmd_set_channels(mld, params->channels, scan_ptrs,
 				      params->n_channels, channel_cfg_flags,
 				      vif->type);
 
@@ -1144,49 +1158,148 @@ iwl_mld_scan_cmd_set_chan_params(struct iwl_mld *mld,
 	return 0;
 }
 
-static int
-iwl_mld_scan_build_cmd(struct iwl_mld *mld, struct ieee80211_vif *vif,
+struct iwl_scan_umac_handler {
+	u8 version;
+	int (*handler)(struct iwl_mld *mld, struct ieee80211_vif *vif,
 		       struct iwl_mld_scan_params *params,
 		       enum iwl_mld_scan_status scan_status,
-		       bool low_latency)
+		       int uid, u32 ooc_priority, bool low_latency);
+};
+
+#define IWL_SCAN_UMAC_HANDLER(_ver) {		\
+	.version = _ver,			\
+	.handler = iwl_mld_scan_umac_v##_ver,	\
+}
+
+static int iwl_mld_scan_umac_common(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct iwl_mld_scan_params *params,
+				    struct iwl_scan_req_params_ptrs *scan_ptrs,
+				    enum iwl_mld_scan_status scan_status,
+				    bool low_latency)
 {
-	struct iwl_scan_req_umac_v17 *cmd = mld->scan.cmd;
-	struct iwl_scan_req_params_v17 *scan_p = &cmd->scan_params;
 	u32 bitmap_ssid = 0;
-	int uid, ret;
+	int ret;
 
-	memset(mld->scan.cmd, 0, mld->scan.cmd_size);
+	iwl_mld_scan_cmd_set_gen_params(mld, params, vif, scan_ptrs,
+					scan_status);
 
-	/* find a free UID entry */
-	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_NONE);
-	if (uid < 0)
-		return uid;
+	ret = iwl_mld_scan_cmd_set_sched_params(params, scan_ptrs);
+	if (ret)
+		return ret;
 
-	cmd->uid = cpu_to_le32(uid);
-	cmd->ooc_priority =
-		cpu_to_le32(iwl_mld_scan_ooc_priority(scan_status));
+	iwl_mld_scan_cmd_set_probe_params(params, scan_ptrs, &bitmap_ssid);
+
+	return iwl_mld_scan_cmd_set_chan_params(mld, params, vif, scan_ptrs,
+						low_latency, scan_status,
+						bitmap_ssid);
+}
+
+static int iwl_mld_scan_umac_v18(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				 struct iwl_mld_scan_params *params,
+				 enum iwl_mld_scan_status scan_status,
+				 int uid, u32 ooc_priority, bool low_latency)
+{
+	struct iwl_scan_req_umac_v18 *cmd = mld->scan.cmd;
+	struct iwl_scan_req_params_ptrs scan_ptrs = {
+		.general_params = &cmd->scan_params.general_params,
+		.probe_params = &cmd->scan_params.probe_params,
+		.channel_params = &cmd->scan_params.channel_params,
+		.periodic_params = &cmd->scan_params.periodic_params
+	};
+	int ret;
 
-	iwl_mld_scan_cmd_set_gen_params(mld, params, vif,
-					&scan_p->general_params, scan_status);
+	if (WARN_ON(params->n_channels > SCAN_MAX_NUM_CHANS_V4))
+		return -EINVAL;
 
-	ret = iwl_mld_scan_cmd_set_sched_params(params,
-						scan_p->periodic_params.schedule,
-						&scan_p->periodic_params.delay);
+	cmd->uid = cpu_to_le32(uid);
+	cmd->ooc_priority = cpu_to_le32(ooc_priority);
+
+	ret = iwl_mld_scan_umac_common(mld, vif, params, &scan_ptrs,
+				       scan_status, low_latency);
 	if (ret)
 		return ret;
 
-	iwl_mld_scan_cmd_set_probe_params(params, &scan_p->probe_params,
-					  &bitmap_ssid);
+	return uid;
+}
+
+static int iwl_mld_scan_umac_v17(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				 struct iwl_mld_scan_params *params,
+				 enum iwl_mld_scan_status scan_status,
+				 int uid, u32 ooc_priority, bool low_latency)
+{
+	struct iwl_scan_req_umac_v17 *cmd = mld->scan.cmd;
+	struct iwl_scan_req_params_ptrs scan_ptrs = {
+		.general_params = &cmd->scan_params.general_params,
+		.probe_params = &cmd->scan_params.probe_params,
+
+		/* struct iwl_scan_channel_params_v8 and struct
+		 * iwl_scan_channel_params_v7 are almost identical. The only
+		 * difference is that the newer version allows configuration of
+		 * more channels. So casting here is ok as long as we ensure
+		 * that we don't exceed the max number of channels supported by
+		 * the older version (see the WARN_ON below).
+		 */
+		.channel_params = (struct iwl_scan_channel_params_v8 *)
+			&cmd->scan_params.channel_params,
+		.periodic_params = &cmd->scan_params.periodic_params
+	};
+	int ret;
+
+	if (WARN_ON(params->n_channels > SCAN_MAX_NUM_CHANS_V3))
+		return -EINVAL;
+
+	cmd->uid = cpu_to_le32(uid);
+	cmd->ooc_priority = cpu_to_le32(ooc_priority);
 
-	ret = iwl_mld_scan_cmd_set_chan_params(mld, params, vif, scan_p,
-					       low_latency, scan_status,
-					       bitmap_ssid);
+	ret = iwl_mld_scan_umac_common(mld, vif, params, &scan_ptrs,
+				       scan_status, low_latency);
 	if (ret)
 		return ret;
 
 	return uid;
 }
 
+static const struct iwl_scan_umac_handler iwl_scan_umac_handlers[] = {
+	/* set the newest version first to shorten the list traverse time */
+	IWL_SCAN_UMAC_HANDLER(18),
+	IWL_SCAN_UMAC_HANDLER(17),
+};
+
+static int
+iwl_mld_scan_build_cmd(struct iwl_mld *mld, struct ieee80211_vif *vif,
+		       struct iwl_mld_scan_params *params,
+		       enum iwl_mld_scan_status scan_status,
+		       bool low_latency)
+{
+	int uid, err;
+	u32 ooc_priority;
+
+	memset(mld->scan.cmd, 0, mld->scan.cmd_size);
+	uid = iwl_mld_scan_uid_by_status(mld, IWL_MLD_SCAN_NONE);
+	if (uid < 0)
+		return uid;
+
+	ooc_priority = iwl_mld_scan_ooc_priority(scan_status);
+
+	for (size_t i = 0; i < ARRAY_SIZE(iwl_scan_umac_handlers); i++) {
+		const struct iwl_scan_umac_handler *ver_handler =
+			&iwl_scan_umac_handlers[i];
+
+		if (ver_handler->version != mld->scan.cmd_ver)
+			continue;
+
+		err = ver_handler->handler(mld, vif, params, scan_status,
+					   uid, ooc_priority, low_latency);
+		return err ? : uid;
+	}
+
+	IWL_ERR(mld, "No handler for UMAC scan cmd version %d\n",
+		mld->scan.cmd_ver);
+
+	return -EINVAL;
+}
+
 static bool
 iwl_mld_scan_pass_all(struct iwl_mld *mld,
 		      struct cfg80211_sched_scan_request *req)
@@ -2031,6 +2144,8 @@ int iwl_mld_alloc_scan_cmd(struct iwl_mld *mld)
 
 	if (scan_cmd_ver == 17) {
 		scan_cmd_size = sizeof(struct iwl_scan_req_umac_v17);
+	} else if (scan_cmd_ver == 18) {
+		scan_cmd_size = sizeof(struct iwl_scan_req_umac_v18);
 	} else {
 		IWL_ERR(mld, "Unexpected scan cmd version %d\n", scan_cmd_ver);
 		return -EINVAL;
@@ -2041,6 +2156,7 @@ int iwl_mld_alloc_scan_cmd(struct iwl_mld *mld)
 		return -ENOMEM;
 
 	mld->scan.cmd_size = scan_cmd_size;
+	mld->scan.cmd_ver = scan_cmd_ver;
 
 	return 0;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.h b/drivers/net/wireless/intel/iwlwifi/mld/scan.h
index 69110f0cfc8e2..772b3a02c4c4d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.h
@@ -109,6 +109,7 @@ enum iwl_mld_traffic_load {
  * @traffic_load.status: The current traffic load status, see
  *	&enum iwl_mld_traffic_load
  * @cmd_size: size of %cmd.
+ * @cmd_ver: version of the scan command format.
  * @cmd: pointer to scan cmd buffer (allocated once in op mode start).
  * @last_6ghz_passive_jiffies: stores the last 6GHz passive scan time
  *	in jiffies.
@@ -134,6 +135,7 @@ struct iwl_mld_scan {
 	/* And here fields that survive a fw restart */
 	size_t cmd_size;
 	void *cmd;
+	u8 cmd_ver;
 	unsigned long last_6ghz_passive_jiffies;
 	unsigned long last_start_time_jiffies;
 	u64 last_mlo_scan_time;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/sta.c b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
index 6b7a89e050e67..f40c49377466a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
@@ -398,12 +398,42 @@ static u32 iwl_mld_get_htc_flags(struct ieee80211_link_sta *link_sta)
 	return htc_flags;
 }
 
+/* Note: modifies the command depending on FW command version */
 static int iwl_mld_send_sta_cmd(struct iwl_mld *mld,
-				const struct iwl_sta_cfg_cmd *cmd)
+				struct iwl_sta_cfg_cmd *cmd)
 {
-	int ret = iwl_mld_send_cmd_pdu(mld,
-				       WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD),
-				       cmd);
+	int cmd_id = WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD);
+	int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0);
+	int len = sizeof(*cmd);
+	int ret;
+
+	if (cmd_ver < 2) {
+		IWL_ERR(mld, "Unsupported STA_CONFIG_CMD version %d\n",
+			cmd_ver);
+		return -EINVAL;
+	} else if (cmd_ver == 2) {
+		struct iwl_sta_cfg_cmd_v2 *cmd_v2 = (void *)cmd;
+
+		if (WARN_ON(cmd->station_type == cpu_to_le32(STATION_TYPE_NAN_PEER_NMI) ||
+			    cmd->station_type == cpu_to_le32(STATION_TYPE_NAN_PEER_NDI) ||
+			    hweight32(le32_to_cpu(cmd->link_mask)) != 1))
+			return -EINVAL;
+		/*
+		 * These fields are located in a different place in the struct of v2.
+		 * The assumption is that UHR won't be used with FW that has v2.
+		 */
+		if (WARN_ON(cmd->mic_prep_pad_delay || cmd->mic_compute_pad_delay))
+			return -EINVAL;
+
+		len = sizeof(struct iwl_sta_cfg_cmd_v2);
+		cmd_v2->link_id = cpu_to_le32(__ffs(le32_to_cpu(cmd->link_mask)));
+	} else if (WARN_ON(cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_PEER_NMI) &&
+			   cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_PEER_NDI) &&
+			   hweight32(le32_to_cpu(cmd->link_mask)) != 1)) {
+		return -EINVAL;
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld, cmd_id, cmd, len);
 	if (ret)
 		IWL_ERR(mld, "STA_CONFIG_CMD send failed, ret=0x%x\n", ret);
 	return ret;
@@ -431,8 +461,8 @@ iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
 		return -EINVAL;
 
 	cmd.sta_id = cpu_to_le32(fw_id);
+	cmd.link_mask = cpu_to_le32(BIT(mld_link->fw_id));
 	cmd.station_type = cpu_to_le32(mld_sta->sta_type);
-	cmd.link_id = cpu_to_le32(mld_link->fw_id);
 
 	memcpy(&cmd.peer_mld_address, sta->addr, ETH_ALEN);
 	memcpy(&cmd.peer_link_address, link_sta->addr, ETH_ALEN);
@@ -982,7 +1012,7 @@ iwl_mld_add_internal_sta_to_fw(struct iwl_mld *mld,
 		return iwl_mld_send_aux_sta_cmd(mld, internal_sta);
 
 	cmd.sta_id = cpu_to_le32((u8)internal_sta->sta_id);
-	cmd.link_id = cpu_to_le32(fw_link_id);
+	cmd.link_mask = cpu_to_le32(BIT(fw_link_id));
 	cmd.station_type = cpu_to_le32(internal_sta->sta_type);
 
 	/* FW doesn't allow to add a IGTK/BIGTK if the sta isn't marked as MFP.
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/time_sync.c b/drivers/net/wireless/intel/iwlwifi/mld/time_sync.c
index 897ab65b71aaa..474dd555e70bd 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/time_sync.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/time_sync.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2025-2026 Intel Corporation
  */
 
 #include "mld.h"
@@ -116,9 +116,9 @@ static bool iwl_mld_is_skb_match(struct sk_buff *skb, u8 *addr, u8 dialog_token)
 	u8 skb_dialog_token;
 
 	if (ieee80211_is_timing_measurement(skb))
-		skb_dialog_token = mgmt->u.action.u.wnm_timing_msr.dialog_token;
+		skb_dialog_token = mgmt->u.action.wnm_timing_msr.dialog_token;
 	else
-		skb_dialog_token = mgmt->u.action.u.ftm.dialog_token;
+		skb_dialog_token = mgmt->u.action.ftm.dialog_token;
 
 	if ((ether_addr_equal(mgmt->sa, addr) ||
 	     ether_addr_equal(mgmt->da, addr)) &&
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
index ebc569e94f55f..1b67836b1fac5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/etherdevice.h>
 #include <linux/math64.h>
@@ -1409,8 +1409,7 @@ void iwl_mvm_ftm_lc_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	struct iwl_mvm_loc_entry *entry;
 	const u8 *ies, *lci, *civic, *msr_ie;
 	size_t ies_len, lci_len = 0, civic_len = 0;
-	size_t baselen = IEEE80211_MIN_ACTION_SIZE +
-			 sizeof(mgmt->u.action.u.ftm);
+	size_t baselen = IEEE80211_MIN_ACTION_SIZE(ftm);
 	static const u8 rprt_type_lci = IEEE80211_SPCT_MSR_RPRT_TYPE_LCI;
 	static const u8 rprt_type_civic = IEEE80211_SPCT_MSR_RPRT_TYPE_CIVIC;
 
@@ -1419,7 +1418,7 @@ void iwl_mvm_ftm_lc_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	ies = mgmt->u.action.u.ftm.variable;
+	ies = mgmt->u.action.ftm.variable;
 	ies_len = len - baselen;
 
 	msr_ie = cfg80211_find_ie_match(WLAN_EID_MEASURE_REPORT, ies, ies_len,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 43cf94c9a36bd..0c643f0b71054 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -459,23 +459,18 @@ static void iwl_mvm_phy_filter_init(struct iwl_mvm *mvm,
 
 static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
 {
+	int cmd_id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
+			     MCC_ALLOWED_AP_TYPE_CMD);
+	struct iwl_mcc_allowed_ap_type_cmd_v1 cmd = {};
 	u8 cmd_ver;
 	int ret;
-	struct iwl_host_cmd cmd = {
-		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
-			      MCC_ALLOWED_AP_TYPE_CMD),
-		.flags = 0,
-		.data[0] = &mvm->fwrt.uats_table,
-		.len[0] =  sizeof(mvm->fwrt.uats_table),
-		.dataflags[0] = IWL_HCMD_DFL_NOCOPY,
-	};
 
 	if (mvm->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
 		IWL_DEBUG_RADIO(mvm, "UATS feature is not supported\n");
 		return;
 	}
 
-	cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd.id,
+	cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id,
 					IWL_FW_CMD_VER_UNKNOWN);
 	if (cmd_ver != 1) {
 		IWL_DEBUG_RADIO(mvm,
@@ -486,10 +481,17 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
 
 	iwl_uefi_get_uats_table(mvm->trans, &mvm->fwrt);
 
-	if (!mvm->fwrt.uats_valid)
+	if (!mvm->fwrt.ap_type_cmd_valid)
 		return;
 
-	ret = iwl_mvm_send_cmd(mvm, &cmd);
+	BUILD_BUG_ON(sizeof(mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map) !=
+		     sizeof(cmd.mcc_to_ap_type_map));
+
+	memcpy(cmd.mcc_to_ap_type_map,
+	       mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map,
+	       sizeof(mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map));
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
 	if (ret < 0)
 		IWL_ERR(mvm, "failed to send MCC_ALLOWED_AP_TYPE_CMD (%d)\n",
 			ret);
@@ -906,7 +908,7 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 
 int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
 {
-	union iwl_geo_tx_power_profiles_cmd geo_tx_cmd;
+	union iwl_geo_tx_power_profiles_cmd geo_tx_cmd = {};
 	struct iwl_geo_tx_power_profiles_resp *resp;
 	u16 len;
 	int ret;
@@ -958,7 +960,7 @@ int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
 static int iwl_mvm_sar_geo_init(struct iwl_mvm *mvm)
 {
 	u32 cmd_id = WIDE_ID(PHY_OPS_GROUP, PER_CHAIN_LIMIT_OFFSET_CMD);
-	union iwl_geo_tx_power_profiles_cmd cmd;
+	union iwl_geo_tx_power_profiles_cmd cmd = {};
 	u16 len;
 	u32 n_bands;
 	u32 n_profiles;
@@ -1032,12 +1034,139 @@ static int iwl_mvm_sar_geo_init(struct iwl_mvm *mvm)
 	return iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, len, &cmd);
 }
 
+static bool iwl_mvm_ppag_value_valid(struct iwl_fw_runtime *fwrt, int chain,
+				     int subband)
+{
+	s8 ppag_val = fwrt->ppag_chains[chain].subbands[subband];
+
+	if ((subband == 0 &&
+	     (ppag_val > IWL_PPAG_MAX_LB || ppag_val < IWL_PPAG_MIN_LB)) ||
+	    (subband != 0 &&
+	     (ppag_val > IWL_PPAG_MAX_HB || ppag_val < IWL_PPAG_MIN_HB))) {
+		IWL_DEBUG_RADIO(fwrt, "Invalid PPAG value: %d\n", ppag_val);
+		return false;
+	}
+	return true;
+}
+
+static int iwl_mvm_fill_ppag_table(struct iwl_fw_runtime *fwrt,
+				   union iwl_ppag_table_cmd *cmd,
+				   int *cmd_size)
+{
+	u8 cmd_ver;
+	int i, j, num_sub_bands;
+	s8 *gain;
+	bool send_ppag_always;
+
+	/* many firmware images for JF lie about this */
+	if (CSR_HW_RFID_TYPE(fwrt->trans->info.hw_rf_id) ==
+	    CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_JF))
+		return -EOPNOTSUPP;
+
+	if (!fw_has_capa(&fwrt->fw->ucode_capa, IWL_UCODE_TLV_CAPA_SET_PPAG)) {
+		IWL_DEBUG_RADIO(fwrt,
+				"PPAG capability not supported by FW, command not sent.\n");
+		return -EINVAL;
+	}
+
+	cmd_ver = iwl_fw_lookup_cmd_ver(fwrt->fw,
+					WIDE_ID(PHY_OPS_GROUP,
+						PER_PLATFORM_ANT_GAIN_CMD), 1);
+	/*
+	 * Starting from ver 4, driver needs to send the PPAG CMD regardless
+	 * if PPAG is enabled/disabled or valid/invalid.
+	 */
+	send_ppag_always = cmd_ver > 3;
+
+	/* Don't send PPAG if it is disabled */
+	if (!send_ppag_always && !fwrt->ppag_flags) {
+		IWL_DEBUG_RADIO(fwrt, "PPAG not enabled, command not sent.\n");
+		return -EINVAL;
+	}
+
+	IWL_DEBUG_RADIO(fwrt, "PPAG cmd ver is %d\n", cmd_ver);
+	if (cmd_ver == 1) {
+		num_sub_bands = IWL_NUM_SUB_BANDS_V1;
+		gain = cmd->v1.gain[0];
+		*cmd_size = sizeof(cmd->v1);
+		cmd->v1.flags =
+			cpu_to_le32(fwrt->ppag_flags & IWL_PPAG_CMD_V1_MASK);
+		if (fwrt->ppag_bios_rev >= 1) {
+			/* in this case FW supports revision 0 */
+			IWL_DEBUG_RADIO(fwrt,
+					"PPAG table rev is %d, send truncated table\n",
+					fwrt->ppag_bios_rev);
+		}
+	} else if (cmd_ver == 5) {
+		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
+		gain = cmd->v5.gain[0];
+		*cmd_size = sizeof(cmd->v5);
+		cmd->v5.flags =
+			cpu_to_le32(fwrt->ppag_flags & IWL_PPAG_CMD_V5_MASK);
+		if (fwrt->ppag_bios_rev == 0) {
+			/* in this case FW supports revisions 1,2 or 3 */
+			IWL_DEBUG_RADIO(fwrt,
+					"PPAG table rev is 0, send padded table\n");
+		}
+	} else if (cmd_ver == 7) {
+		num_sub_bands = IWL_NUM_SUB_BANDS_V2;
+		gain = cmd->v7.gain[0];
+		*cmd_size = sizeof(cmd->v7);
+		cmd->v7.ppag_config_info.hdr.table_source =
+			fwrt->ppag_bios_source;
+		cmd->v7.ppag_config_info.hdr.table_revision =
+			fwrt->ppag_bios_rev;
+		cmd->v7.ppag_config_info.value = cpu_to_le32(fwrt->ppag_flags);
+	} else {
+		IWL_DEBUG_RADIO(fwrt, "Unsupported PPAG command version\n");
+		return -EINVAL;
+	}
+
+	/* ppag mode */
+	IWL_DEBUG_RADIO(fwrt,
+			"PPAG MODE bits were read from bios: %d\n",
+			fwrt->ppag_flags);
+
+	if (cmd_ver == 1 &&
+	    !fw_has_capa(&fwrt->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_PPAG_CHINA_BIOS_SUPPORT)) {
+		cmd->v1.flags &= cpu_to_le32(IWL_PPAG_ETSI_MASK);
+		IWL_DEBUG_RADIO(fwrt, "masking ppag China bit\n");
+	} else {
+		IWL_DEBUG_RADIO(fwrt, "isn't masking ppag China bit\n");
+	}
+
+	/* The 'flags' field is the same in v1 and v5 so we can just
+	 * use v1 to access it.
+	 */
+	IWL_DEBUG_RADIO(fwrt,
+			"PPAG MODE bits going to be sent: %d\n",
+			(cmd_ver < 7) ? le32_to_cpu(cmd->v1.flags) :
+					le32_to_cpu(cmd->v7.ppag_config_info.value));
+
+	for (i = 0; i < IWL_NUM_CHAIN_LIMITS; i++) {
+		for (j = 0; j < num_sub_bands; j++) {
+			if (!send_ppag_always &&
+			    !iwl_mvm_ppag_value_valid(fwrt, i, j))
+				return -EINVAL;
+
+			gain[i * num_sub_bands + j] =
+				fwrt->ppag_chains[i].subbands[j];
+			IWL_DEBUG_RADIO(fwrt,
+					"PPAG table: chain[%d] band[%d]: gain = %d\n",
+					i, j, gain[i * num_sub_bands + j]);
+		}
+	}
+
+	return 0;
+}
+
 int iwl_mvm_ppag_send_cmd(struct iwl_mvm *mvm)
 {
 	union iwl_ppag_table_cmd cmd;
 	int ret, cmd_size;
 
-	ret = iwl_fill_ppag_table(&mvm->fwrt, &cmd, &cmd_size);
+	ret = iwl_mvm_fill_ppag_table(&mvm->fwrt, &cmd, &cmd_size);
 	/* Not supporting PPAG table is a valid scenario */
 	if (ret < 0)
 		return 0;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
index 9bb253dcf4a76..4869a5fa8abca 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-key.c
@@ -121,52 +121,6 @@ struct iwl_mvm_sta_key_update_data {
 	int err;
 };
 
-static void iwl_mvm_mld_update_sta_key(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_sta *sta,
-				       struct ieee80211_key_conf *key,
-				       void *_data)
-{
-	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD);
-	struct iwl_mvm_sta_key_update_data *data = _data;
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
-	struct iwl_sec_key_cmd cmd = {
-		.action = cpu_to_le32(FW_CTXT_ACTION_MODIFY),
-		.u.modify.old_sta_mask = cpu_to_le32(data->old_sta_mask),
-		.u.modify.new_sta_mask = cpu_to_le32(data->new_sta_mask),
-		.u.modify.key_id = cpu_to_le32(key->keyidx),
-		.u.modify.key_flags =
-			cpu_to_le32(iwl_mvm_get_sec_flags(mvm, vif, sta, key)),
-	};
-	int err;
-
-	/* only need to do this for pairwise keys (link_id == -1) */
-	if (sta != data->sta || key->link_id >= 0)
-		return;
-
-	err = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
-
-	if (err)
-		data->err = err;
-}
-
-int iwl_mvm_mld_update_sta_keys(struct iwl_mvm *mvm,
-				struct ieee80211_vif *vif,
-				struct ieee80211_sta *sta,
-				u32 old_sta_mask,
-				u32 new_sta_mask)
-{
-	struct iwl_mvm_sta_key_update_data data = {
-		.sta = sta,
-		.old_sta_mask = old_sta_mask,
-		.new_sta_mask = new_sta_mask,
-	};
-
-	ieee80211_iter_keys(mvm->hw, vif, iwl_mvm_mld_update_sta_key,
-			    &data);
-	return data.err;
-}
-
 static int __iwl_mvm_sec_key_del(struct iwl_mvm *mvm, u32 sta_mask,
 				 u32 key_flags, u32 keyidx, u32 flags)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
index 896ed98230214..f1dbfeae20bc0 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
@@ -886,133 +886,6 @@ static int iwl_mvm_mld_roc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	return iwl_mvm_roc_common(hw, vif, channel, duration, type, &ops);
 }
 
-static int
-iwl_mvm_mld_change_vif_links(struct ieee80211_hw *hw,
-			     struct ieee80211_vif *vif,
-			     u16 old_links, u16 new_links,
-			     struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
-{
-	struct iwl_mvm_vif_link_info *new_link[IEEE80211_MLD_MAX_NUM_LINKS] = {};
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
-	u16 removed = old_links & ~new_links;
-	u16 added = new_links & ~old_links;
-	int err, i;
-
-	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-		if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-			break;
-
-		if (!(added & BIT(i)))
-			continue;
-		new_link[i] = kzalloc_obj(*new_link[i]);
-		if (!new_link[i]) {
-			err = -ENOMEM;
-			goto free;
-		}
-
-		new_link[i]->fw_link_id = IWL_MVM_FW_LINK_ID_INVALID;
-		iwl_mvm_init_link(new_link[i]);
-	}
-
-	mutex_lock(&mvm->mutex);
-
-	/* If we're in RESTART flow, the default link wasn't added in
-         * drv_add_interface(), and link[0] doesn't point to it.
-	 */
-	if (old_links == 0 && !test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
-					&mvm->status)) {
-		err = iwl_mvm_disable_link(mvm, vif, &vif->bss_conf);
-		if (err)
-			goto out_err;
-		mvmvif->link[0] = NULL;
-	}
-
-	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-		if (removed & BIT(i)) {
-			struct ieee80211_bss_conf *link_conf = old[i];
-
-			err = iwl_mvm_disable_link(mvm, vif, link_conf);
-			if (err)
-				goto out_err;
-			kfree(mvmvif->link[i]);
-			mvmvif->link[i] = NULL;
-		} else if (added & BIT(i)) {
-			struct ieee80211_bss_conf *link_conf;
-
-			link_conf = link_conf_dereference_protected(vif, i);
-			if (WARN_ON(!link_conf))
-				continue;
-
-			if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART,
-				      &mvm->status))
-				mvmvif->link[i] = new_link[i];
-			new_link[i] = NULL;
-			err = iwl_mvm_add_link(mvm, vif, link_conf);
-			if (err)
-				goto out_err;
-		}
-	}
-
-	err = 0;
-	if (new_links == 0) {
-		mvmvif->link[0] = &mvmvif->deflink;
-		err = iwl_mvm_add_link(mvm, vif, &vif->bss_conf);
-	}
-
-out_err:
-	/* we really don't have a good way to roll back here ... */
-	mutex_unlock(&mvm->mutex);
-
-free:
-	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++)
-		kfree(new_link[i]);
-	return err;
-}
-
-static int
-iwl_mvm_mld_change_sta_links(struct ieee80211_hw *hw,
-			     struct ieee80211_vif *vif,
-			     struct ieee80211_sta *sta,
-			     u16 old_links, u16 new_links)
-{
-	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
-
-	guard(mvm)(mvm);
-	return iwl_mvm_mld_update_sta_links(mvm, vif, sta, old_links, new_links);
-}
-
-static bool iwl_mvm_mld_can_activate_links(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif,
-					   u16 desired_links)
-{
-	int n_links = hweight16(desired_links);
-
-	if (n_links <= 1)
-		return true;
-
-	WARN_ON(1);
-	return false;
-}
-
-static enum ieee80211_neg_ttlm_res
-iwl_mvm_mld_can_neg_ttlm(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			 struct ieee80211_neg_ttlm *neg_ttlm)
-{
-	u16 map;
-	u8 i;
-
-	/* Verify all TIDs are mapped to the same links set */
-	map = neg_ttlm->downlink[0];
-	for (i = 0; i < IEEE80211_TTLM_NUM_TIDS; i++) {
-		if (neg_ttlm->downlink[i] != neg_ttlm->uplink[i] ||
-		    neg_ttlm->uplink[i] != map)
-			return NEG_TTLM_RES_REJECT;
-	}
-
-	return NEG_TTLM_RES_ACCEPT;
-}
-
 const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 	.tx = iwl_mvm_mac_tx,
 	.wake_tx_queue = iwl_mvm_mac_wake_tx_queue,
@@ -1102,9 +975,4 @@ const struct ieee80211_ops iwl_mvm_mld_hw_ops = {
 	.link_sta_add_debugfs = iwl_mvm_link_sta_add_debugfs,
 #endif
 	.set_hw_timestamp = iwl_mvm_set_hw_timestamp,
-
-	.change_vif_links = iwl_mvm_mld_change_vif_links,
-	.change_sta_links = iwl_mvm_mld_change_sta_links,
-	.can_activate_links = iwl_mvm_mld_can_activate_links,
-	.can_neg_ttlm = iwl_mvm_mld_can_neg_ttlm,
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c
index 3359e02e151ff..da7ed4639a939 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-sta.c
@@ -20,7 +20,7 @@ u32 iwl_mvm_sta_fw_id_mask(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 }
 
 static int iwl_mvm_mld_send_sta_cmd(struct iwl_mvm *mvm,
-				    struct iwl_sta_cfg_cmd *cmd)
+				    struct iwl_sta_cfg_cmd_v2 *cmd)
 {
 	u32 cmd_id = WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD);
 	int cmd_len = iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 0) > 1 ?
@@ -41,7 +41,7 @@ static int iwl_mvm_mld_add_int_sta_to_fw(struct iwl_mvm *mvm,
 					 struct iwl_mvm_int_sta *sta,
 					 const u8 *addr, int link_id)
 {
-	struct iwl_sta_cfg_cmd cmd;
+	struct iwl_sta_cfg_cmd_v2 cmd;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -416,7 +416,7 @@ static int iwl_mvm_mld_cfg_sta(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_vif_link_info *link_info =
 					mvm_vif->link[link_conf->link_id];
-	struct iwl_sta_cfg_cmd cmd = {
+	struct iwl_sta_cfg_cmd_v2 cmd = {
 		.sta_id = cpu_to_le32(mvm_link_sta->sta_id),
 		.station_type = cpu_to_le32(mvm_sta->sta_type),
 	};
@@ -913,288 +913,3 @@ void iwl_mvm_mld_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 
 	rcu_read_unlock();
 }
-
-static int iwl_mvm_mld_update_sta_queues(struct iwl_mvm *mvm,
-					 struct ieee80211_sta *sta,
-					 u32 old_sta_mask,
-					 u32 new_sta_mask)
-{
-	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_scd_queue_cfg_cmd cmd = {
-		.operation = cpu_to_le32(IWL_SCD_QUEUE_MODIFY),
-		.u.modify.old_sta_mask = cpu_to_le32(old_sta_mask),
-		.u.modify.new_sta_mask = cpu_to_le32(new_sta_mask),
-	};
-	struct iwl_host_cmd hcmd = {
-		.id = WIDE_ID(DATA_PATH_GROUP, SCD_QUEUE_CONFIG_CMD),
-		.len[0] = sizeof(cmd),
-		.data[0] = &cmd
-	};
-	int tid;
-	int ret;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	for (tid = 0; tid <= IWL_MAX_TID_COUNT; tid++) {
-		struct iwl_mvm_tid_data *tid_data = &mvm_sta->tid_data[tid];
-		int txq_id = tid_data->txq_id;
-
-		if (txq_id == IWL_MVM_INVALID_QUEUE)
-			continue;
-
-		if (tid == IWL_MAX_TID_COUNT)
-			cmd.u.modify.tid = cpu_to_le32(IWL_MGMT_TID);
-		else
-			cmd.u.modify.tid = cpu_to_le32(tid);
-
-		ret = iwl_mvm_send_cmd(mvm, &hcmd);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int iwl_mvm_mld_update_sta_baids(struct iwl_mvm *mvm,
-					u32 old_sta_mask,
-					u32 new_sta_mask)
-{
-	struct iwl_rx_baid_cfg_cmd cmd = {
-		.action = cpu_to_le32(IWL_RX_BAID_ACTION_MODIFY),
-		.modify.old_sta_id_mask = cpu_to_le32(old_sta_mask),
-		.modify.new_sta_id_mask = cpu_to_le32(new_sta_mask),
-	};
-	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, RX_BAID_ALLOCATION_CONFIG_CMD);
-	int baid;
-
-	/* mac80211 will remove sessions later, but we ignore all that */
-	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
-		return 0;
-
-	BUILD_BUG_ON(sizeof(struct iwl_rx_baid_cfg_resp) != sizeof(baid));
-
-	for (baid = 0; baid < ARRAY_SIZE(mvm->baid_map); baid++) {
-		struct iwl_mvm_baid_data *data;
-		int ret;
-
-		data = rcu_dereference_protected(mvm->baid_map[baid],
-						 lockdep_is_held(&mvm->mutex));
-		if (!data)
-			continue;
-
-		if (!(data->sta_mask & old_sta_mask))
-			continue;
-
-		WARN_ONCE(data->sta_mask != old_sta_mask,
-			  "BAID data for %d corrupted - expected 0x%x found 0x%x\n",
-			  baid, old_sta_mask, data->sta_mask);
-
-		cmd.modify.tid = cpu_to_le32(data->tid);
-
-		ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, CMD_SEND_IN_RFKILL,
-					   sizeof(cmd), &cmd);
-		data->sta_mask = new_sta_mask;
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int iwl_mvm_mld_update_sta_resources(struct iwl_mvm *mvm,
-					    struct ieee80211_vif *vif,
-					    struct ieee80211_sta *sta,
-					    u32 old_sta_mask,
-					    u32 new_sta_mask)
-{
-	int ret;
-
-	ret = iwl_mvm_mld_update_sta_queues(mvm, sta,
-					    old_sta_mask,
-					    new_sta_mask);
-	if (ret)
-		return ret;
-
-	ret = iwl_mvm_mld_update_sta_keys(mvm, vif, sta,
-					  old_sta_mask,
-					  new_sta_mask);
-	if (ret)
-		return ret;
-
-	return iwl_mvm_mld_update_sta_baids(mvm, old_sta_mask, new_sta_mask);
-}
-
-int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
-				 struct ieee80211_vif *vif,
-				 struct ieee80211_sta *sta,
-				 u16 old_links, u16 new_links)
-{
-	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_mvm_vif *mvm_vif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mvm_link_sta *mvm_sta_link;
-	struct iwl_mvm_vif_link_info *mvm_vif_link;
-	unsigned long links_to_add = ~old_links & new_links;
-	unsigned long links_to_rem = old_links & ~new_links;
-	unsigned long old_links_long = old_links;
-	u32 current_sta_mask = 0, sta_mask_added = 0, sta_mask_to_rem = 0;
-	unsigned long link_sta_added_to_fw = 0, link_sta_allocated = 0;
-	unsigned int link_id;
-	int ret;
-
-	lockdep_assert_wiphy(mvm->hw->wiphy);
-	lockdep_assert_held(&mvm->mutex);
-
-	for_each_set_bit(link_id, &old_links_long,
-			 IEEE80211_MLD_MAX_NUM_LINKS) {
-		mvm_sta_link =
-			rcu_dereference_protected(mvm_sta->link[link_id],
-						  lockdep_is_held(&mvm->mutex));
-
-		if (WARN_ON(!mvm_sta_link)) {
-			ret = -EINVAL;
-			goto err;
-		}
-
-		current_sta_mask |= BIT(mvm_sta_link->sta_id);
-		if (links_to_rem & BIT(link_id))
-			sta_mask_to_rem |= BIT(mvm_sta_link->sta_id);
-	}
-
-	if (sta_mask_to_rem) {
-		ret = iwl_mvm_mld_update_sta_resources(mvm, vif, sta,
-						       current_sta_mask,
-						       current_sta_mask &
-							~sta_mask_to_rem);
-		if (WARN_ON(ret))
-			goto err;
-
-		current_sta_mask &= ~sta_mask_to_rem;
-	}
-
-	for_each_set_bit(link_id, &links_to_rem, IEEE80211_MLD_MAX_NUM_LINKS) {
-		mvm_sta_link =
-			rcu_dereference_protected(mvm_sta->link[link_id],
-						  lockdep_is_held(&mvm->mutex));
-		mvm_vif_link = mvm_vif->link[link_id];
-
-		if (WARN_ON(!mvm_sta_link || !mvm_vif_link)) {
-			ret = -EINVAL;
-			goto err;
-		}
-
-		ret = iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_sta_link->sta_id);
-		if (WARN_ON(ret))
-			goto err;
-
-		if (vif->type == NL80211_IFTYPE_STATION)
-			mvm_vif_link->ap_sta_id = IWL_INVALID_STA;
-
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id);
-	}
-
-	for_each_set_bit(link_id, &links_to_add, IEEE80211_MLD_MAX_NUM_LINKS) {
-		struct ieee80211_bss_conf *link_conf =
-			link_conf_dereference_protected(vif, link_id);
-		struct ieee80211_link_sta *link_sta =
-			link_sta_dereference_protected(sta, link_id);
-		mvm_vif_link = mvm_vif->link[link_id];
-
-		if (WARN_ON(!mvm_vif_link || !link_conf || !link_sta)) {
-			ret = -EINVAL;
-			goto err;
-		}
-
-		if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
-			struct iwl_mvm_link_sta *mvm_link_sta =
-				rcu_dereference_protected(mvm_sta->link[link_id],
-							  lockdep_is_held(&mvm->mutex));
-			u32 sta_id;
-
-			if (WARN_ON(!mvm_link_sta)) {
-				ret = -EINVAL;
-				goto err;
-			}
-
-			sta_id = mvm_link_sta->sta_id;
-
-			rcu_assign_pointer(mvm->fw_id_to_mac_id[sta_id], sta);
-			rcu_assign_pointer(mvm->fw_id_to_link_sta[sta_id],
-					   link_sta);
-		} else {
-			if (WARN_ON(mvm_sta->link[link_id])) {
-				ret = -EINVAL;
-				goto err;
-			}
-			ret = iwl_mvm_mld_alloc_sta_link(mvm, vif, sta,
-							 link_id);
-			if (WARN_ON(ret))
-				goto err;
-		}
-
-		link_sta->agg.max_rc_amsdu_len = 1;
-		ieee80211_sta_recalc_aggregates(sta);
-
-		mvm_sta_link =
-			rcu_dereference_protected(mvm_sta->link[link_id],
-						  lockdep_is_held(&mvm->mutex));
-
-		if (WARN_ON(!mvm_sta_link)) {
-			ret = -EINVAL;
-			goto err;
-		}
-
-		if (vif->type == NL80211_IFTYPE_STATION)
-			iwl_mvm_mld_set_ap_sta_id(sta, mvm_vif_link,
-						  mvm_sta_link);
-
-		link_sta_allocated |= BIT(link_id);
-
-		sta_mask_added |= BIT(mvm_sta_link->sta_id);
-
-		ret = iwl_mvm_mld_cfg_sta(mvm, sta, vif, link_sta, link_conf,
-					  mvm_sta_link);
-		if (WARN_ON(ret))
-			goto err;
-
-		link_sta_added_to_fw |= BIT(link_id);
-
-		iwl_mvm_rs_add_sta_link(mvm, mvm_sta_link);
-
-		iwl_mvm_rs_rate_init(mvm, vif, sta, link_conf, link_sta,
-				     link_conf->chanreq.oper.chan->band);
-	}
-
-	if (sta_mask_added) {
-		ret = iwl_mvm_mld_update_sta_resources(mvm, vif, sta,
-						       current_sta_mask,
-						       current_sta_mask |
-							sta_mask_added);
-		if (WARN_ON(ret))
-			goto err;
-	}
-
-	return 0;
-
-err:
-	/* remove all already allocated stations in FW */
-	for_each_set_bit(link_id, &link_sta_added_to_fw,
-			 IEEE80211_MLD_MAX_NUM_LINKS) {
-		mvm_sta_link =
-			rcu_dereference_protected(mvm_sta->link[link_id],
-						  lockdep_is_held(&mvm->mutex));
-
-		iwl_mvm_mld_rm_sta_from_fw(mvm, mvm_sta_link->sta_id);
-	}
-
-	/* remove all already allocated station links in driver */
-	for_each_set_bit(link_id, &link_sta_allocated,
-			 IEEE80211_MLD_MAX_NUM_LINKS) {
-		mvm_sta_link =
-			rcu_dereference_protected(mvm_sta->link[link_id],
-						  lockdep_is_held(&mvm->mutex));
-
-		iwl_mvm_mld_free_sta_link(mvm, mvm_sta, mvm_sta_link, link_id);
-	}
-
-	return ret;
-}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index 46a9dfa58a539..402ba5dee8b2c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -2450,11 +2450,6 @@ void iwl_mvm_sec_key_remove_ap(struct iwl_mvm *mvm,
 			       struct ieee80211_vif *vif,
 			       struct iwl_mvm_vif_link_info *link,
 			       unsigned int link_id);
-int iwl_mvm_mld_update_sta_keys(struct iwl_mvm *mvm,
-				struct ieee80211_vif *vif,
-				struct ieee80211_sta *sta,
-				u32 old_sta_mask,
-				u32 new_sta_mask);
 int iwl_mvm_mld_send_key(struct iwl_mvm *mvm, u32 sta_mask, u32 key_flags,
 			 struct ieee80211_key_conf *keyconf);
 u32 iwl_mvm_get_sec_flags(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index c25edc7c18133..ff099aec78867 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -637,10 +637,6 @@ void iwl_mvm_mld_free_sta_link(struct iwl_mvm *mvm,
 			       struct iwl_mvm_link_sta *mvm_sta_link,
 			       unsigned int link_id);
 int iwl_mvm_mld_rm_sta_id(struct iwl_mvm *mvm, u8 sta_id);
-int iwl_mvm_mld_update_sta_links(struct iwl_mvm *mvm,
-				 struct ieee80211_vif *vif,
-				 struct ieee80211_sta *sta,
-				 u16 old_links, u16 new_links);
 u32 iwl_mvm_sta_fw_id_mask(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			   int filter_link_id);
 int iwl_mvm_mld_add_int_sta_with_queue(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
index 4945ebf19f6bb..a7cd2e4ba1ae9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tdls.c
@@ -234,7 +234,7 @@ void iwl_mvm_rx_tdls_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	 * Also convert TU to msec.
 	 */
 	delay = TU_TO_MS(vif->bss_conf.dtim_period * vif->bss_conf.beacon_int);
-	mod_delayed_work(system_wq, &mvm->tdls_cs.dwork,
+	mod_delayed_work(system_percpu_wq, &mvm->tdls_cs.dwork,
 			 msecs_to_jiffies(delay));
 
 	iwl_mvm_tdls_update_cs_state(mvm, IWL_MVM_TDLS_SW_ACTIVE);
@@ -548,7 +548,7 @@ iwl_mvm_tdls_channel_switch(struct ieee80211_hw *hw,
 	 */
 	delay = 2 * TU_TO_MS(vif->bss_conf.dtim_period *
 			     vif->bss_conf.beacon_int);
-	mod_delayed_work(system_wq, &mvm->tdls_cs.dwork,
+	mod_delayed_work(system_percpu_wq, &mvm->tdls_cs.dwork,
 			 msecs_to_jiffies(delay));
 	return 0;
 }
@@ -659,6 +659,6 @@ iwl_mvm_tdls_recv_channel_switch(struct ieee80211_hw *hw,
 	/* register a timeout in case we don't succeed in switching */
 	delay = vif->bss_conf.dtim_period * vif->bss_conf.beacon_int *
 		1024 / 1000;
-	mod_delayed_work(system_wq, &mvm->tdls_cs.dwork,
+	mod_delayed_work(system_percpu_wq, &mvm->tdls_cs.dwork,
 			 msecs_to_jiffies(delay));
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c
index edae3e24192b1..039b4daac73fb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-sync.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2022 Intel Corporation
+ * Copyright (C) 2022, 2026 Intel Corporation
  */
 
 #include "mvm.h"
@@ -18,9 +18,9 @@ static bool iwl_mvm_is_skb_match(struct sk_buff *skb, u8 *addr, u8 dialog_token)
 	u8 skb_dialog_token;
 
 	if (ieee80211_is_timing_measurement(skb))
-		skb_dialog_token = mgmt->u.action.u.wnm_timing_msr.dialog_token;
+		skb_dialog_token = mgmt->u.action.wnm_timing_msr.dialog_token;
 	else
-		skb_dialog_token = mgmt->u.action.u.ftm.dialog_token;
+		skb_dialog_token = mgmt->u.action.ftm.dialog_token;
 
 	if ((ether_addr_equal(mgmt->sa, addr) ||
 	     ether_addr_equal(mgmt->da, addr)) &&
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
index 4560d92d76fe0..a05f60f9224b4 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
@@ -3197,7 +3197,7 @@ static ssize_t iwl_dbgfs_reset_write(struct file *file,
 		if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
 			return -EINVAL;
 		if (mode == IWL_RESET_MODE_TOP_RESET) {
-			if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC)
+			if (!iwl_trans_is_top_reset_supported(trans))
 				return -EINVAL;
 			trans->request_top_reset = 1;
 		}
diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c
index 98517888dba7f..72c92f72469de 100644
--- a/drivers/net/wireless/marvell/libertas/cfg.c
+++ b/drivers/net/wireless/marvell/libertas/cfg.c
@@ -1507,7 +1507,7 @@ static int lbs_cfg_set_default_key(struct wiphy *wiphy,
 }
 
 
-static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev,
+static int lbs_cfg_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   int link_id, u8 idx, bool pairwise,
 			   const u8 *mac_addr, struct key_params *params)
 {
@@ -1516,7 +1516,7 @@ static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev,
 	u16 key_type;
 	int ret = 0;
 
-	if (netdev == priv->mesh_dev)
+	if (wdev->netdev == priv->mesh_dev)
 		return -EOPNOTSUPP;
 
 	lbs_deb_assoc("add_key: cipher 0x%x, mac_addr %pM\n",
@@ -1568,7 +1568,7 @@ static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev,
 }
 
 
-static int lbs_cfg_del_key(struct wiphy *wiphy, struct net_device *netdev,
+static int lbs_cfg_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   int link_id, u8 key_index, bool pairwise,
 			   const u8 *mac_addr)
 {
@@ -1607,7 +1607,7 @@ static int lbs_cfg_del_key(struct wiphy *wiphy, struct net_device *netdev,
  * Get station
  */
 
-static int lbs_cfg_get_station(struct wiphy *wiphy, struct net_device *dev,
+static int lbs_cfg_get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       const u8 *mac, struct station_info *sinfo)
 {
 	struct lbs_private *priv = wiphy_priv(wiphy);
diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c
index 8a6bf1365cfab..176f2106bab6f 100644
--- a/drivers/net/wireless/marvell/libertas/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas/if_usb.c
@@ -114,8 +114,8 @@ static void if_usb_write_bulk_callback(struct urb *urb)
 static void if_usb_free(struct if_usb_card *cardp)
 {
 	/* Unlink tx & rx urb */
-	usb_kill_urb(cardp->tx_urb);
-	usb_kill_urb(cardp->rx_urb);
+	usb_kill_anchored_urbs(&cardp->tx_submitted);
+	usb_kill_anchored_urbs(&cardp->rx_submitted);
 
 	usb_free_urb(cardp->tx_urb);
 	cardp->tx_urb = NULL;
@@ -221,6 +221,9 @@ static int if_usb_probe(struct usb_interface *intf,
 		     udev->descriptor.bDeviceSubClass,
 		     udev->descriptor.bDeviceProtocol);
 
+	init_usb_anchor(&cardp->rx_submitted);
+	init_usb_anchor(&cardp->tx_submitted);
+
 	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
 		endpoint = &iface_desc->endpoint[i].desc;
 		if (usb_endpoint_is_bulk_in(endpoint)) {
@@ -276,7 +279,6 @@ static int if_usb_probe(struct usb_interface *intf,
 
 	cardp->boot2_version = udev->descriptor.bcdDevice;
 
-	usb_get_dev(udev);
 	usb_set_intfdata(intf, cardp);
 
 	r = lbs_get_firmware_async(priv, &udev->dev, cardp->model,
@@ -287,7 +289,6 @@ static int if_usb_probe(struct usb_interface *intf,
 	return 0;
 
 err_get_fw:
-	usb_put_dev(udev);
 	lbs_remove_card(priv);
 err_add_card:
 	if_usb_reset_device(cardp);
@@ -321,7 +322,6 @@ static void if_usb_disconnect(struct usb_interface *intf)
 	kfree(cardp);
 
 	usb_set_intfdata(intf, NULL);
-	usb_put_dev(interface_to_usbdev(intf));
 }
 
 /**
@@ -426,7 +426,12 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb
 		goto tx_ret;
 	}
 
-	usb_kill_urb(cardp->tx_urb);
+	/* check if there are pending URBs */
+	if (!usb_anchor_empty(&cardp->tx_submitted)) {
+		lbs_deb_usbd(&cardp->udev->dev, "%s failed: pending URB\n", __func__);
+		ret = -EBUSY;
+		goto tx_ret;
+	}
 
 	usb_fill_bulk_urb(cardp->tx_urb, cardp->udev,
 			  usb_sndbulkpipe(cardp->udev,
@@ -435,8 +440,10 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb
 
 	cardp->tx_urb->transfer_flags |= URB_ZERO_PACKET;
 
+	usb_anchor_urb(cardp->tx_urb, &cardp->tx_submitted);
 	if ((ret = usb_submit_urb(cardp->tx_urb, GFP_ATOMIC))) {
 		lbs_deb_usbd(&cardp->udev->dev, "usb_submit_urb failed: %d\n", ret);
+		usb_unanchor_urb(cardp->tx_urb);
 	} else {
 		lbs_deb_usb2(&cardp->udev->dev, "usb_submit_urb success\n");
 		ret = 0;
@@ -467,8 +474,10 @@ static int __if_usb_submit_rx_urb(struct if_usb_card *cardp,
 			  cardp);
 
 	lbs_deb_usb2(&cardp->udev->dev, "Pointer for rx_urb %p\n", cardp->rx_urb);
+	usb_anchor_urb(cardp->rx_urb, &cardp->rx_submitted);
 	if ((ret = usb_submit_urb(cardp->rx_urb, GFP_ATOMIC))) {
 		lbs_deb_usbd(&cardp->udev->dev, "Submit Rx URB failed: %d\n", ret);
+		usb_unanchor_urb(cardp->rx_urb);
 		kfree_skb(skb);
 		cardp->rx_skb = NULL;
 		ret = -1;
@@ -838,8 +847,8 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret,
 	}
 
 	/* Cancel any pending usb business */
-	usb_kill_urb(cardp->rx_urb);
-	usb_kill_urb(cardp->tx_urb);
+	usb_kill_anchored_urbs(&cardp->rx_submitted);
+	usb_kill_anchored_urbs(&cardp->tx_submitted);
 
 	cardp->fwlastblksent = 0;
 	cardp->fwdnldover = 0;
@@ -869,8 +878,8 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret,
 	if (cardp->bootcmdresp == BOOT_CMD_RESP_NOT_SUPPORTED) {
 		/* Return to normal operation */
 		ret = -EOPNOTSUPP;
-		usb_kill_urb(cardp->rx_urb);
-		usb_kill_urb(cardp->tx_urb);
+		usb_kill_anchored_urbs(&cardp->rx_submitted);
+		usb_kill_anchored_urbs(&cardp->tx_submitted);
 		if (if_usb_submit_rx_urb(cardp) < 0)
 			ret = -EIO;
 		goto done;
@@ -900,7 +909,7 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret,
 	wait_event_interruptible(cardp->fw_wq, cardp->surprise_removed || cardp->fwdnldover);
 
 	timer_delete_sync(&cardp->fw_timeout);
-	usb_kill_urb(cardp->rx_urb);
+	usb_kill_anchored_urbs(&cardp->rx_submitted);
 
 	if (!cardp->fwdnldover) {
 		pr_info("failed to load fw, resetting device!\n");
@@ -960,8 +969,8 @@ static int if_usb_suspend(struct usb_interface *intf, pm_message_t message)
 		goto out;
 
 	/* Unlink tx & rx urb */
-	usb_kill_urb(cardp->tx_urb);
-	usb_kill_urb(cardp->rx_urb);
+	usb_kill_anchored_urbs(&cardp->tx_submitted);
+	usb_kill_anchored_urbs(&cardp->rx_submitted);
 
  out:
 	return ret;
diff --git a/drivers/net/wireless/marvell/libertas/if_usb.h b/drivers/net/wireless/marvell/libertas/if_usb.h
index 7d0daeb33c3f7..a0cd36197c2b0 100644
--- a/drivers/net/wireless/marvell/libertas/if_usb.h
+++ b/drivers/net/wireless/marvell/libertas/if_usb.h
@@ -48,6 +48,9 @@ struct if_usb_card {
 	struct urb *rx_urb, *tx_urb;
 	struct lbs_private *priv;
 
+	struct usb_anchor rx_submitted;
+	struct usb_anchor tx_submitted;
+
 	struct sk_buff *rx_skb;
 
 	uint8_t ep_in;
diff --git a/drivers/net/wireless/marvell/libertas_tf/if_usb.c b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
index f49151c18b793..07b38f2b8f582 100644
--- a/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
@@ -223,7 +223,6 @@ static int if_usb_probe(struct usb_interface *intf,
 	if (!priv)
 		goto dealloc;
 
-	usb_get_dev(udev);
 	usb_set_intfdata(intf, cardp);
 
 	return 0;
@@ -258,7 +257,6 @@ static void if_usb_disconnect(struct usb_interface *intf)
 	kfree(cardp);
 
 	usb_set_intfdata(intf, NULL);
-	usb_put_dev(interface_to_usbdev(intf));
 
 	lbtf_deb_leave(LBTF_DEB_MAIN);
 }
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
index 34b4b34276d6d..042b1fe5f0d67 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
@@ -203,6 +203,7 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv,
 
 		if (!mwifiex_is_ralist_valid(priv, pra_list, ptrindex)) {
 			spin_unlock_bh(&priv->wmm.ra_list_spinlock);
+			mwifiex_write_data_complete(adapter, skb_aggr, 1, -1);
 			return -1;
 		}
 
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index eb28fe718e718..c9a651bdf8822 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -141,11 +141,11 @@ static void *mwifiex_cfg80211_get_adapter(struct wiphy *wiphy)
  * CFG802.11 operation handler to delete a network key.
  */
 static int
-mwifiex_cfg80211_del_key(struct wiphy *wiphy, struct net_device *netdev,
+mwifiex_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			 int link_id, u8 key_index, bool pairwise,
 			 const u8 *mac_addr)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 	const u8 *peer_mac = pairwise ? mac_addr : bc_mac;
 
@@ -480,11 +480,11 @@ mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
  * CFG802.11 operation handler to add a network key.
  */
 static int
-mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev,
+mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			 int link_id, u8 key_index, bool pairwise,
 			 const u8 *mac_addr, struct key_params *params)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	struct mwifiex_wep_key *wep_key;
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 	const u8 *peer_mac = pairwise ? mac_addr : bc_mac;
@@ -518,11 +518,11 @@ mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev,
  */
 static int
 mwifiex_cfg80211_set_default_mgmt_key(struct wiphy *wiphy,
-				      struct net_device *netdev,
+				      struct wireless_dev *wdev,
 				      int link_id,
 				      u8 key_index)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	struct mwifiex_ds_encrypt_key encrypt_key;
 
 	wiphy_dbg(wiphy, "set default mgmt key, key index=%d\n", key_index);
@@ -1554,10 +1554,10 @@ mwifiex_dump_station_info(struct mwifiex_private *priv,
  * requested station information, if available.
  */
 static int
-mwifiex_cfg80211_get_station(struct wiphy *wiphy, struct net_device *dev,
+mwifiex_cfg80211_get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			     const u8 *mac, struct station_info *sinfo)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 
 	if (!priv->media_connected)
 		return -ENOENT;
@@ -1571,10 +1571,10 @@ mwifiex_cfg80211_get_station(struct wiphy *wiphy, struct net_device *dev,
  * CFG802.11 operation handler to dump station information.
  */
 static int
-mwifiex_cfg80211_dump_station(struct wiphy *wiphy, struct net_device *dev,
+mwifiex_cfg80211_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			      int idx, u8 *mac, struct station_info *sinfo)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	struct mwifiex_sta_node *node;
 	int i;
 
@@ -1901,10 +1901,11 @@ static int mwifiex_cfg80211_change_beacon(struct wiphy *wiphy,
  * associated stations list, no action is taken.
  */
 static int
-mwifiex_cfg80211_del_station(struct wiphy *wiphy, struct net_device *dev,
+mwifiex_cfg80211_del_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			     struct station_del_parameters *params)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct mwifiex_private *priv =
+		mwifiex_netdev_get_priv(wdev->netdev);
 	struct mwifiex_sta_node *sta_node;
 	u8 deauth_mac[ETH_ALEN];
 
@@ -3993,7 +3994,8 @@ mwifiex_cfg80211_uap_add_station(struct mwifiex_private *priv, const u8 *mac,
 		if (!sinfo)
 			return -ENOMEM;
 
-		cfg80211_new_sta(priv->netdev, mac, sinfo, GFP_KERNEL);
+		cfg80211_new_sta(priv->netdev->ieee80211_ptr, mac, sinfo,
+				 GFP_KERNEL);
 		kfree(sinfo);
 	}
 
@@ -4001,10 +4003,10 @@ mwifiex_cfg80211_uap_add_station(struct mwifiex_private *priv, const u8 *mac,
 }
 
 static int
-mwifiex_cfg80211_add_station(struct wiphy *wiphy, struct net_device *dev,
+mwifiex_cfg80211_add_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			     const u8 *mac, struct station_parameters *params)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 
 	if (priv->adapter->host_mlme_enabled &&
 	    (GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_UAP))
@@ -4240,12 +4242,12 @@ mwifiex_cfg80211_start_radar_detection(struct wiphy *wiphy,
 }
 
 static int
-mwifiex_cfg80211_change_station(struct wiphy *wiphy, struct net_device *dev,
+mwifiex_cfg80211_change_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 				const u8 *mac,
 				struct station_parameters *params)
 {
 	int ret;
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 
 	if (priv->adapter->host_mlme_enabled &&
 	    (GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_UAP))
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index 5c9a46e64d23c..0c89250137248 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -391,7 +391,7 @@ static void mwifiex_invalidate_lists(struct mwifiex_adapter *adapter)
 static void
 mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter)
 {
-	timer_delete(&adapter->wakeup_timer);
+	timer_delete_sync(&adapter->wakeup_timer);
 	cancel_delayed_work_sync(&adapter->devdump_work);
 	mwifiex_cancel_all_pending_cmd(adapter);
 	wake_up_interruptible(&adapter->cmd_wait_q.wait);
diff --git a/drivers/net/wireless/marvell/mwifiex/tdls.c b/drivers/net/wireless/marvell/mwifiex/tdls.c
index a4cf323e704be..845f2a22e0714 100644
--- a/drivers/net/wireless/marvell/mwifiex/tdls.c
+++ b/drivers/net/wireless/marvell/mwifiex/tdls.c
@@ -755,16 +755,12 @@ mwifiex_construct_tdls_action_frame(struct mwifiex_private *priv,
 	switch (action_code) {
 	case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
 		/* See the layout of 'struct ieee80211_mgmt'. */
-		extra = sizeof(mgmt->u.action.u.tdls_discover_resp) +
-			sizeof(mgmt->u.action.category);
+		extra = IEEE80211_MIN_ACTION_SIZE(tdls_discover_resp) - 24;
 		skb_put(skb, extra);
 		mgmt->u.action.category = WLAN_CATEGORY_PUBLIC;
-		mgmt->u.action.u.tdls_discover_resp.action_code =
-					      WLAN_PUB_ACTION_TDLS_DISCOVER_RES;
-		mgmt->u.action.u.tdls_discover_resp.dialog_token =
-								   dialog_token;
-		mgmt->u.action.u.tdls_discover_resp.capability =
-							     cpu_to_le16(capab);
+		mgmt->u.action.action_code = WLAN_PUB_ACTION_TDLS_DISCOVER_RES;
+		mgmt->u.action.tdls_discover_resp.dialog_token = dialog_token;
+		mgmt->u.action.tdls_discover_resp.capability = cpu_to_le16(capab);
 		/* move back for addr4 */
 		memmove(pos + ETH_ALEN, &mgmt->u.action, extra);
 		/* init address 4 */
diff --git a/drivers/net/wireless/marvell/mwifiex/uap_event.c b/drivers/net/wireless/marvell/mwifiex/uap_event.c
index 703104fd1fbe6..679fdae0f001a 100644
--- a/drivers/net/wireless/marvell/mwifiex/uap_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/uap_event.c
@@ -130,8 +130,8 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 					le16_to_cpu(event->len) - (u16)len;
 			}
 		}
-		cfg80211_new_sta(priv->netdev, event->sta_addr, sinfo,
-				 GFP_KERNEL);
+		cfg80211_new_sta(priv->netdev->ieee80211_ptr, event->sta_addr,
+				 sinfo, GFP_KERNEL);
 
 		node = mwifiex_add_sta_entry(priv, event->sta_addr);
 		if (!node) {
@@ -162,7 +162,8 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 	case EVENT_UAP_STA_DEAUTH:
 		deauth_mac = adapter->event_body +
 			     MWIFIEX_UAP_EVENT_EXTRA_HEADER;
-		cfg80211_del_sta(priv->netdev, deauth_mac, GFP_KERNEL);
+		cfg80211_del_sta(priv->netdev->ieee80211_ptr, deauth_mac,
+				 GFP_KERNEL);
 
 		if (priv->ap_11n_enabled) {
 			mwifiex_11n_del_rx_reorder_tbl_by_ta(priv, deauth_mac);
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
index 947ecb0a7b405..f4b94a1054f65 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.c
+++ b/drivers/net/wireless/marvell/mwifiex/usb.c
@@ -520,8 +520,6 @@ static int mwifiex_usb_probe(struct usb_interface *intf,
 		return ret;
 	}
 
-	usb_get_dev(udev);
-
 	return 0;
 }
 
@@ -666,8 +664,6 @@ static void mwifiex_usb_disconnect(struct usb_interface *intf)
 	mwifiex_dbg(adapter, FATAL,
 		    "%s: removing card\n", __func__);
 	mwifiex_remove_card(adapter);
-
-	usb_put_dev(interface_to_usbdev(intf));
 }
 
 static void mwifiex_usb_coredump(struct device *dev)
diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c
index 99321d180f34a..b1af021803415 100644
--- a/drivers/net/wireless/marvell/mwl8k.c
+++ b/drivers/net/wireless/marvell/mwl8k.c
@@ -1985,9 +1985,9 @@ mwl8k_txq_xmit(struct ieee80211_hw *hw,
 	 */
 	if (unlikely(ieee80211_is_action(wh->frame_control) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ &&
+	    mgmt->u.action.action_code == WLAN_ACTION_ADDBA_REQ &&
 	    priv->ap_fw)) {
-		u16 capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab);
+		u16 capab = le16_to_cpu(mgmt->u.action.addba_req.capab);
 		tid = (capab & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
 		index = mwl8k_tid_queue_mapping(tid);
 	}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
index b41ca1410da92..f946ddc20a479 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
@@ -413,10 +413,10 @@ mt76_connac2_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi,
 	u32 val;
 
 	if (ieee80211_is_action(fc) &&
-	    skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 + 1 + 2 &&
+	    skb->len >= IEEE80211_MIN_ACTION_SIZE(addba_req.capab) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) {
-		u16 capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab);
+	    mgmt->u.action.action_code == WLAN_ACTION_ADDBA_REQ) {
+		u16 capab = le16_to_cpu(mgmt->u.action.addba_req.capab);
 
 		txwi[5] |= cpu_to_le32(MT_TXD5_ADD_BA);
 		tid = (capab >> 2) & IEEE80211_QOS_CTL_TID_MASK;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
index 00bff4d3aab80..2d2f34aa465d1 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
@@ -1977,11 +1977,11 @@ mt7915_mcu_add_inband_discov(struct mt7915_dev *dev, struct ieee80211_vif *vif,
 
 	if (changed & BSS_CHANGED_FILS_DISCOVERY) {
 		interval = vif->bss_conf.fils_discovery.max_interval;
-		skb = ieee80211_get_fils_discovery_tmpl(hw, vif);
+		skb = ieee80211_get_fils_discovery_tmpl(hw, vif, 0);
 	} else if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP &&
 		   vif->bss_conf.unsol_bcast_probe_resp_interval) {
 		interval = vif->bss_conf.unsol_bcast_probe_resp_interval;
-		skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif);
+		skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif, 0);
 	}
 
 	if (!skb) {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
index 0d94359004233..caaf71c314802 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c
@@ -668,9 +668,9 @@ mt7925_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi,
 	u32 val;
 
 	if (ieee80211_is_action(fc) &&
-	    skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 &&
+	    skb->len >= IEEE80211_MIN_ACTION_SIZE(action_code) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ)
+	    mgmt->u.action.action_code == WLAN_ACTION_ADDBA_REQ)
 		tid = MT_TX_ADDBA;
 	else if (ieee80211_is_mgmt(hdr->frame_control))
 		tid = MT_TX_NORMAL;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
index d4f3ee943b472..84cbf36b493c5 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c
@@ -800,9 +800,9 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi,
 	u32 val;
 
 	if (ieee80211_is_action(fc) &&
-	    skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 &&
+	    skb->len >= IEEE80211_MIN_ACTION_SIZE(action_code) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) {
+	    mgmt->u.action.action_code == WLAN_ACTION_ADDBA_REQ) {
 		if (is_mt7990(&dev->mt76))
 			txwi[6] |= cpu_to_le32(FIELD_PREP(MT_TXD6_TID_ADDBA, tid));
 		else
diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
index c0c042de477b8..b4422a4754cdb 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c
@@ -2863,11 +2863,13 @@ int mt7996_mcu_beacon_inband_discov(struct mt7996_dev *dev,
 	if (changed & BSS_CHANGED_FILS_DISCOVERY &&
 	    link_conf->fils_discovery.max_interval) {
 		interval = link_conf->fils_discovery.max_interval;
-		skb = ieee80211_get_fils_discovery_tmpl(hw, vif);
+		skb = ieee80211_get_fils_discovery_tmpl(hw, vif,
+							link_conf->link_id);
 	} else if (changed & BSS_CHANGED_UNSOL_BCAST_PROBE_RESP &&
 		   link_conf->unsol_bcast_probe_resp_interval) {
 		interval = link_conf->unsol_bcast_probe_resp_interval;
-		skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif);
+		skb = ieee80211_get_unsol_bcast_probe_resp_tmpl(hw, vif,
+								link_conf->link_id);
 	}
 
 	if (!skb) {
diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c
index ff9176cdee3de..63b0447e55c15 100644
--- a/drivers/net/wireless/mediatek/mt76/scan.c
+++ b/drivers/net/wireless/mediatek/mt76/scan.c
@@ -63,10 +63,8 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid)
 
 	rcu_read_lock();
 
-	if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL)) {
-		ieee80211_free_txskb(phy->hw, skb);
+	if (!ieee80211_tx_prepare_skb(phy->hw, vif, skb, band, NULL))
 		goto out;
-	}
 
 	info = IEEE80211_SKB_CB(skb);
 	if (req->no_cck)
diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
index 1561a601c7f2c..3a774cc44b263 100644
--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
@@ -534,7 +534,7 @@ static int wilc_wfi_cfg_copy_wpa_info(struct wilc_wfi_key *key_info,
 	return 0;
 }
 
-static int add_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+static int add_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 		   u8 key_index, bool pairwise, const u8 *mac_addr,
 		   struct key_params *params)
 
@@ -544,7 +544,7 @@ static int add_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
 	const u8 *tx_mic = NULL;
 	u8 mode = WILC_FW_SEC_NO;
 	u8 op_mode;
-	struct wilc_vif *vif = netdev_priv(netdev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 	struct wilc_wfi_key *key;
 
@@ -632,19 +632,19 @@ static int add_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
 		break;
 
 	default:
-		netdev_err(netdev, "%s: Unsupported cipher\n", __func__);
+		netdev_err(wdev->netdev, "%s: Unsupported cipher\n", __func__);
 		ret = -ENOTSUPP;
 	}
 
 	return ret;
 }
 
-static int del_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+static int del_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 		   u8 key_index,
 		   bool pairwise,
 		   const u8 *mac_addr)
 {
-	struct wilc_vif *vif = netdev_priv(netdev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 
 	if (!pairwise && (key_index == 4 || key_index == 5)) {
@@ -680,12 +680,12 @@ static int del_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
 	return 0;
 }
 
-static int get_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+static int get_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 		   u8 key_index, bool pairwise, const u8 *mac_addr,
 		   void *cookie,
 		   void (*callback)(void *cookie, struct key_params *))
 {
-	struct wilc_vif *vif = netdev_priv(netdev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 	struct  key_params key_params;
 
@@ -725,17 +725,18 @@ static int set_default_key(struct wiphy *wiphy, struct net_device *netdev,
 	return 0;
 }
 
-static int set_default_mgmt_key(struct wiphy *wiphy, struct net_device *netdev,
+static int set_default_mgmt_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				int link_id, u8 key_index)
 {
-	struct wilc_vif *vif = netdev_priv(netdev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 
 	return wilc_set_default_mgmt_key_index(vif, key_index);
 }
 
-static int get_station(struct wiphy *wiphy, struct net_device *dev,
+static int get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       const u8 *mac, struct station_info *sinfo)
 {
+	struct net_device *dev = wdev->netdev;
 	struct wilc_vif *vif = netdev_priv(dev);
 	struct wilc_priv *priv = &vif->priv;
 	struct wilc *wilc = vif->wilc;
@@ -1312,10 +1313,10 @@ static int set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *dev,
 	return 0;
 }
 
-static int dump_station(struct wiphy *wiphy, struct net_device *dev,
+static int dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			int idx, u8 *mac, struct station_info *sinfo)
 {
-	struct wilc_vif *vif = netdev_priv(dev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	int ret;
 
 	if (idx != 0)
@@ -1450,11 +1451,11 @@ static int stop_ap(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
-static int add_station(struct wiphy *wiphy, struct net_device *dev,
+static int add_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       const u8 *mac, struct station_parameters *params)
 {
 	int ret = 0;
-	struct wilc_vif *vif = netdev_priv(dev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 
 	if (vif->iftype == WILC_AP_MODE || vif->iftype == WILC_GO_MODE) {
@@ -1463,18 +1464,18 @@ static int add_station(struct wiphy *wiphy, struct net_device *dev,
 
 		ret = wilc_add_station(vif, mac, params);
 		if (ret)
-			netdev_err(dev, "Host add station fail\n");
+			netdev_err(wdev->netdev, "Host add station fail\n");
 	}
 
 	return ret;
 }
 
-static int del_station(struct wiphy *wiphy, struct net_device *dev,
+static int del_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		       struct station_del_parameters *params)
 {
 	const u8 *mac = params->mac;
 	int ret = 0;
-	struct wilc_vif *vif = netdev_priv(dev);
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 	struct sta_info *info;
 
@@ -1488,20 +1489,19 @@ static int del_station(struct wiphy *wiphy, struct net_device *dev,
 
 	ret = wilc_del_station(vif, mac);
 	if (ret)
-		netdev_err(dev, "Host delete station fail\n");
+		netdev_err(wdev->netdev, "Host delete station fail\n");
 	return ret;
 }
 
-static int change_station(struct wiphy *wiphy, struct net_device *dev,
+static int change_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 			  const u8 *mac, struct station_parameters *params)
 {
 	int ret = 0;
-	struct wilc_vif *vif = netdev_priv(dev);
-
+	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	if (vif->iftype == WILC_AP_MODE || vif->iftype == WILC_GO_MODE) {
 		ret = wilc_edit_station(vif, mac, params);
 		if (ret)
-			netdev_err(dev, "Host edit station fail\n");
+			netdev_err(wdev->netdev, "Host edit station fail\n");
 	}
 	return ret;
 }
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index f1188368e66bb..9e44c85d2051c 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -483,26 +483,26 @@ qtnf_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int
-qtnf_get_station(struct wiphy *wiphy, struct net_device *dev,
+qtnf_get_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		 const u8 *mac, struct station_info *sinfo)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 
 	sinfo->generation = vif->generation;
 	return qtnf_cmd_get_sta_info(vif, mac, sinfo);
 }
 
 static int
-qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev,
+qtnf_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		  int idx, u8 *mac, struct station_info *sinfo)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	const struct qtnf_sta_node *sta_node;
 	int ret;
 
-	switch (vif->wdev.iftype) {
+	switch (wdev->iftype) {
 	case NL80211_IFTYPE_STATION:
-		if (idx != 0 || !vif->wdev.connected)
+		if (idx != 0 || !wdev->connected)
 			return -ENOENT;
 
 		ether_addr_copy(mac, vif->bssid);
@@ -520,9 +520,9 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev,
 
 	ret = qtnf_cmd_get_sta_info(vif, mac, sinfo);
 
-	if (vif->wdev.iftype == NL80211_IFTYPE_AP) {
+	if (wdev->iftype == NL80211_IFTYPE_AP) {
 		if (ret == -ENOENT) {
-			cfg80211_del_sta(vif->netdev, mac, GFP_KERNEL);
+			cfg80211_del_sta(&vif->wdev, mac, GFP_KERNEL);
 			sinfo->filled = 0;
 		}
 	}
@@ -532,11 +532,11 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
-static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev,
+static int qtnf_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			int link_id, u8 key_index, bool pairwise,
 			const u8 *mac_addr, struct key_params *params)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	int ret;
 
 	ret = qtnf_cmd_send_add_key(vif, key_index, pairwise, mac_addr, params);
@@ -548,11 +548,11 @@ static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
-static int qtnf_del_key(struct wiphy *wiphy, struct net_device *dev,
+static int qtnf_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			int link_id, u8 key_index, bool pairwise,
 			const u8 *mac_addr)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	int ret;
 
 	ret = qtnf_cmd_send_del_key(vif, key_index, pairwise, mac_addr);
@@ -587,10 +587,10 @@ static int qtnf_set_default_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int
-qtnf_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *dev,
+qtnf_set_default_mgmt_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 			  int link_id, u8 key_index)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	int ret;
 
 	ret = qtnf_cmd_send_set_default_mgmt_key(vif, key_index);
@@ -602,10 +602,10 @@ qtnf_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int
-qtnf_change_station(struct wiphy *wiphy, struct net_device *dev,
+qtnf_change_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		    const u8 *mac, struct station_parameters *params)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	int ret;
 
 	ret = qtnf_cmd_send_change_sta(vif, mac, params);
@@ -617,14 +617,14 @@ qtnf_change_station(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int
-qtnf_del_station(struct wiphy *wiphy, struct net_device *dev,
+qtnf_del_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 		 struct station_del_parameters *params)
 {
-	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
 	int ret;
 
 	if (params->mac &&
-	    (vif->wdev.iftype == NL80211_IFTYPE_AP) &&
+	    (wdev->iftype == NL80211_IFTYPE_AP) &&
 	    !is_broadcast_ether_addr(params->mac) &&
 	    !qtnf_sta_list_lookup(&vif->sta_list, params->mac))
 		return 0;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.c b/drivers/net/wireless/quantenna/qtnfmac/core.c
index 0c106709ae292..0b5b60815c7fd 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.c
@@ -452,8 +452,8 @@ int qtnf_core_net_attach(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 	void *qdev_vif;
 	int ret;
 
-	dev = alloc_netdev_mqs(sizeof(struct qtnf_vif *), name,
-			       name_assign_type, ether_setup, 1, 1);
+	dev = alloc_netdev(sizeof(struct qtnf_vif *), name,
+			   name_assign_type, ether_setup);
 	if (!dev)
 		return -ENOMEM;
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c
index 2551d74ed56ec..fb5a56b6dd05a 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/event.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/event.c
@@ -90,8 +90,8 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 		goto out;
 	}
 
-	cfg80211_new_sta(vif->netdev, sta_assoc->sta_addr, sinfo,
-			 GFP_KERNEL);
+	cfg80211_new_sta(vif->netdev->ieee80211_ptr, sta_assoc->sta_addr,
+			 sinfo, GFP_KERNEL);
 
 out:
 	kfree(sinfo);
@@ -126,7 +126,7 @@ qtnf_event_handle_sta_deauth(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 		 sta_addr, reason);
 
 	if (qtnf_sta_list_del(vif, sta_addr))
-		cfg80211_del_sta(vif->netdev, sta_deauth->sta_addr,
+		cfg80211_del_sta(&vif->wdev, sta_deauth->sta_addr,
 				 GFP_KERNEL);
 
 	return 0;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index ca18a4c7e14ad..2e2e382b62475 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -10965,13 +10965,13 @@ EXPORT_SYMBOL_GPL(rt2800_read_eeprom_efuse);
 
 int rt2800_read_eeprom_nvmem(struct rt2x00_dev *rt2x00dev)
 {
-	struct device_node *np = rt2x00dev->dev->of_node;
+	struct device *dev = rt2x00dev->dev;
 	unsigned int len = rt2x00dev->ops->eeprom_size;
 	struct nvmem_cell *cell;
 	const void *data;
 	size_t retlen;
 
-	cell = of_nvmem_cell_get(np, "eeprom");
+	cell = nvmem_cell_get(dev, "eeprom");
 	if (IS_ERR(cell))
 		return PTR_ERR(cell);
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
index 54599cad78f98..83d00b6baf642 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
@@ -802,14 +802,12 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 	struct rt2x00_dev *rt2x00dev;
 	int retval;
 
-	usb_dev = usb_get_dev(usb_dev);
 	usb_reset_device(usb_dev);
 
 	hw = ieee80211_alloc_hw(sizeof(struct rt2x00_dev), ops->hw);
 	if (!hw) {
 		rt2x00_probe_err("Failed to allocate hardware\n");
-		retval = -ENOMEM;
-		goto exit_put_device;
+		return -ENOMEM;
 	}
 
 	usb_set_intfdata(usb_intf, hw);
@@ -851,10 +849,6 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 
 exit_free_device:
 	ieee80211_free_hw(hw);
-
-exit_put_device:
-	usb_put_dev(usb_dev);
-
 	usb_set_intfdata(usb_intf, NULL);
 
 	return retval;
@@ -873,11 +867,7 @@ void rt2x00usb_disconnect(struct usb_interface *usb_intf)
 	rt2x00usb_free_reg(rt2x00dev);
 	ieee80211_free_hw(hw);
 
-	/*
-	 * Free the USB device data.
-	 */
 	usb_set_intfdata(usb_intf, NULL);
-	usb_put_dev(interface_to_usbdev(usb_intf));
 }
 EXPORT_SYMBOL_GPL(rt2x00usb_disconnect);
 
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c
index 794187d28caa0..804fc604e5f85 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/core.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c
@@ -5146,10 +5146,10 @@ static void rtl8xxxu_dump_action(struct device *dev,
 	if (!(rtl8xxxu_debug & RTL8XXXU_DEBUG_ACTION))
 		return;
 
-	switch (mgmt->u.action.u.addba_resp.action_code) {
+	switch (mgmt->u.action.action_code) {
 	case WLAN_ACTION_ADDBA_RESP:
-		cap = le16_to_cpu(mgmt->u.action.u.addba_resp.capab);
-		timeout = le16_to_cpu(mgmt->u.action.u.addba_resp.timeout);
+		cap = le16_to_cpu(mgmt->u.action.addba_resp.capab);
+		timeout = le16_to_cpu(mgmt->u.action.addba_resp.timeout);
 		dev_info(dev, "WLAN_ACTION_ADDBA_RESP: "
 			 "timeout %i, tid %02x, buf_size %02x, policy %02x, "
 			 "status %02x\n",
@@ -5157,11 +5157,11 @@ static void rtl8xxxu_dump_action(struct device *dev,
 			 (cap & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2,
 			 (cap & IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK) >> 6,
 			 (cap >> 1) & 0x1,
-			 le16_to_cpu(mgmt->u.action.u.addba_resp.status));
+			 le16_to_cpu(mgmt->u.action.addba_resp.status));
 		break;
 	case WLAN_ACTION_ADDBA_REQ:
-		cap = le16_to_cpu(mgmt->u.action.u.addba_req.capab);
-		timeout = le16_to_cpu(mgmt->u.action.u.addba_req.timeout);
+		cap = le16_to_cpu(mgmt->u.action.addba_req.capab);
+		timeout = le16_to_cpu(mgmt->u.action.addba_req.timeout);
 		dev_info(dev, "WLAN_ACTION_ADDBA_REQ: "
 			 "timeout %i, tid %02x, buf_size %02x, policy %02x\n",
 			 timeout,
@@ -5171,7 +5171,7 @@ static void rtl8xxxu_dump_action(struct device *dev,
 		break;
 	default:
 		dev_info(dev, "action frame %02x\n",
-			 mgmt->u.action.u.addba_resp.action_code);
+			 mgmt->u.action.action_code);
 		break;
 	}
 }
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c
index 0ac9cf0937aa5..aad377864e73b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.c
+++ b/drivers/net/wireless/realtek/rtlwifi/base.c
@@ -1409,7 +1409,7 @@ bool rtl_action_proc(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx)
 				sta_entry =
 					(struct rtl_sta_info *)sta->drv_priv;
 				capab =
-				  le16_to_cpu(mgmt->u.action.u.addba_req.capab);
+				  le16_to_cpu(mgmt->u.action.addba_req.capab);
 				tid = (capab &
 				       IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
 				if (tid >= MAX_TID_COUNT) {
@@ -2392,35 +2392,35 @@ static struct sk_buff *rtl_make_smps_action(struct ieee80211_hw *hw,
 	struct sk_buff *skb;
 	struct ieee80211_mgmt *action_frame;
 
-	/* 27 = header + category + action + smps mode */
-	skb = dev_alloc_skb(27 + hw->extra_tx_headroom);
+	skb = dev_alloc_skb(IEEE80211_MIN_ACTION_SIZE(ht_smps) +
+			    hw->extra_tx_headroom);
 	if (!skb)
 		return NULL;
 
 	skb_reserve(skb, hw->extra_tx_headroom);
-	action_frame = skb_put_zero(skb, 27);
+	action_frame = skb_put_zero(skb, IEEE80211_MIN_ACTION_SIZE(ht_smps));
 	memcpy(action_frame->da, da, ETH_ALEN);
 	memcpy(action_frame->sa, rtlefuse->dev_addr, ETH_ALEN);
 	memcpy(action_frame->bssid, bssid, ETH_ALEN);
 	action_frame->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
 						  IEEE80211_STYPE_ACTION);
 	action_frame->u.action.category = WLAN_CATEGORY_HT;
-	action_frame->u.action.u.ht_smps.action = WLAN_HT_ACTION_SMPS;
+	action_frame->u.action.action_code = WLAN_HT_ACTION_SMPS;
 	switch (smps) {
 	case IEEE80211_SMPS_AUTOMATIC:/* 0 */
 	case IEEE80211_SMPS_NUM_MODES:/* 4 */
 		WARN_ON(1);
 		fallthrough;
 	case IEEE80211_SMPS_OFF:/* 1 */ /*MIMO_PS_NOLIMIT*/
-		action_frame->u.action.u.ht_smps.smps_control =
+		action_frame->u.action.ht_smps.smps_control =
 				WLAN_HT_SMPS_CONTROL_DISABLED;/* 0 */
 		break;
 	case IEEE80211_SMPS_STATIC:/* 2 */ /*MIMO_PS_STATIC*/
-		action_frame->u.action.u.ht_smps.smps_control =
+		action_frame->u.action.ht_smps.smps_control =
 				WLAN_HT_SMPS_CONTROL_STATIC;/* 1 */
 		break;
 	case IEEE80211_SMPS_DYNAMIC:/* 3 */ /*MIMO_PS_DYNAMIC*/
-		action_frame->u.action.u.ht_smps.smps_control =
+		action_frame->u.action.ht_smps.smps_control =
 				WLAN_HT_SMPS_CONTROL_DYNAMIC;/* 3 */
 		break;
 	}
@@ -2519,25 +2519,25 @@ struct sk_buff *rtl_make_del_ba(struct ieee80211_hw *hw,
 	struct ieee80211_mgmt *action_frame;
 	u16 params;
 
-	/* 27 = header + category + action + smps mode */
-	skb = dev_alloc_skb(34 + hw->extra_tx_headroom);
+	skb = dev_alloc_skb(IEEE80211_MIN_ACTION_SIZE(delba) +
+			    hw->extra_tx_headroom);
 	if (!skb)
 		return NULL;
 
 	skb_reserve(skb, hw->extra_tx_headroom);
-	action_frame = skb_put_zero(skb, 34);
+	action_frame = skb_put_zero(skb, IEEE80211_MIN_ACTION_SIZE(delba));
 	memcpy(action_frame->sa, sa, ETH_ALEN);
 	memcpy(action_frame->da, rtlefuse->dev_addr, ETH_ALEN);
 	memcpy(action_frame->bssid, bssid, ETH_ALEN);
 	action_frame->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
 						  IEEE80211_STYPE_ACTION);
 	action_frame->u.action.category = WLAN_CATEGORY_BACK;
-	action_frame->u.action.u.delba.action_code = WLAN_ACTION_DELBA;
+	action_frame->u.action.action_code = WLAN_ACTION_DELBA;
 	params = (u16)(1 << 11);	/* bit 11 initiator */
 	params |= (u16)(tid << 12);	/* bit 15:12 TID number */
 
-	action_frame->u.action.u.delba.params = cpu_to_le16(params);
-	action_frame->u.action.u.delba.reason_code =
+	action_frame->u.action.delba.params = cpu_to_le16(params);
+	action_frame->u.action.delba.reason_code =
 		cpu_to_le16(WLAN_REASON_QSTA_TIMEOUT);
 
 	return skb;
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c
index d080469264cf8..19e2ff62d9f18 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.c
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.c
@@ -507,7 +507,7 @@ static void _rtl_pci_tx_isr(struct ieee80211_hw *hw, int prio)
 		if (ieee80211_is_action(fc)) {
 			struct ieee80211_mgmt *action_frame =
 				(struct ieee80211_mgmt *)skb->data;
-			if (action_frame->u.action.u.ht_smps.action ==
+			if (action_frame->u.action.action_code ==
 			    WLAN_HT_ACTION_SMPS) {
 				dev_kfree_skb(skb);
 				goto tx_status_ok;
diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
index c7ae8031436ae..3faf2235728be 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
@@ -325,6 +325,22 @@ void rsi_mac80211_detach(struct rsi_hw *adapter)
 }
 EXPORT_SYMBOL_GPL(rsi_mac80211_detach);
 
+/**
+ * rsi_mac80211_rfkill_exit() - This function is used to stop rfkill polling
+ *                              when the device is removed.
+ * @adapter: Pointer to the adapter structure.
+ *
+ * Return: None.
+ */
+void rsi_mac80211_rfkill_exit(struct rsi_hw *adapter)
+{
+	struct ieee80211_hw *hw = adapter->hw;
+
+	if (hw)
+		wiphy_rfkill_stop_polling(hw->wiphy);
+}
+EXPORT_SYMBOL_GPL(rsi_mac80211_rfkill_exit);
+
 /**
  * rsi_indicate_tx_status() - This function indicates the transmit status.
  * @adapter: Pointer to the adapter structure.
@@ -422,7 +438,6 @@ static void rsi_mac80211_stop(struct ieee80211_hw *hw, bool suspend)
 	rsi_dbg(ERR_ZONE, "===> Interface DOWN <===\n");
 	mutex_lock(&common->mutex);
 	common->iface_down = true;
-	wiphy_rfkill_stop_polling(hw->wiphy);
 
 	/* Block all rx frames */
 	rsi_send_rx_filter_frame(common, 0xffff);
diff --git a/drivers/net/wireless/rsi/rsi_91x_usb.c b/drivers/net/wireless/rsi/rsi_91x_usb.c
index d83204701e27e..8765cac6f875b 100644
--- a/drivers/net/wireless/rsi/rsi_91x_usb.c
+++ b/drivers/net/wireless/rsi/rsi_91x_usb.c
@@ -877,6 +877,8 @@ static void rsi_disconnect(struct usb_interface *pfunction)
 	if (!adapter)
 		return;
 
+	rsi_mac80211_rfkill_exit(adapter);
+
 	rsi_mac80211_detach(adapter);
 
 	if (IS_ENABLED(CONFIG_RSI_COEX) && adapter->priv->coex_mode > 1 &&
diff --git a/drivers/net/wireless/rsi/rsi_common.h b/drivers/net/wireless/rsi/rsi_common.h
index 7aa5124575cfe..591602beeec68 100644
--- a/drivers/net/wireless/rsi/rsi_common.h
+++ b/drivers/net/wireless/rsi/rsi_common.h
@@ -79,6 +79,7 @@ static inline int rsi_kill_thread(struct rsi_thread *handle)
 }
 
 void rsi_mac80211_detach(struct rsi_hw *hw);
+void rsi_mac80211_rfkill_exit(struct rsi_hw *hw);
 u16 rsi_get_connected_channel(struct ieee80211_vif *vif);
 struct rsi_hw *rsi_91x_init(u16 oper_mode);
 void rsi_91x_deinit(struct rsi_hw *adapter);
diff --git a/drivers/net/wireless/silabs/wfx/data_rx.c b/drivers/net/wireless/silabs/wfx/data_rx.c
index e099a9e65bae6..15c06b2b8633a 100644
--- a/drivers/net/wireless/silabs/wfx/data_rx.c
+++ b/drivers/net/wireless/silabs/wfx/data_rx.c
@@ -21,14 +21,14 @@ static void wfx_rx_handle_ba(struct wfx_vif *wvif, struct ieee80211_mgmt *mgmt)
 	if (wfx_api_older_than(wvif->wdev, 3, 6))
 		return;
 
-	switch (mgmt->u.action.u.addba_req.action_code) {
+	switch (mgmt->u.action.action_code) {
 	case WLAN_ACTION_ADDBA_REQ:
-		params = le16_to_cpu(mgmt->u.action.u.addba_req.capab);
+		params = le16_to_cpu(mgmt->u.action.addba_req.capab);
 		tid = (params & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
 		ieee80211_start_rx_ba_session_offl(vif, mgmt->sa, tid);
 		break;
 	case WLAN_ACTION_DELBA:
-		params = le16_to_cpu(mgmt->u.action.u.delba.params);
+		params = le16_to_cpu(mgmt->u.action.delba.params);
 		tid = (params &  IEEE80211_DELBA_PARAM_TID_MASK) >> 12;
 		ieee80211_stop_rx_ba_session_offl(vif, mgmt->sa, tid);
 		break;
@@ -80,7 +80,7 @@ void wfx_rx_cb(struct wfx_vif *wvif, const struct wfx_hif_ind_rx *arg, struct sk
 	 */
 	if (ieee80211_is_action(frame->frame_control) &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK &&
-	    skb->len > IEEE80211_MIN_ACTION_SIZE) {
+	    skb->len > IEEE80211_MIN_ACTION_SIZE(action_code)) {
 		wfx_rx_handle_ba(wvif, mgmt);
 		goto drop;
 	}
diff --git a/drivers/net/wireless/ti/wlcore/tx.c b/drivers/net/wireless/ti/wlcore/tx.c
index 6241866d39df6..75cfbcfb7626d 100644
--- a/drivers/net/wireless/ti/wlcore/tx.c
+++ b/drivers/net/wireless/ti/wlcore/tx.c
@@ -210,7 +210,7 @@ static int wl1271_tx_allocate(struct wl1271 *wl, struct wl12xx_vif *wlvif,
 		if (skb_headroom(skb) < (total_len - skb->len) &&
 		    pskb_expand_head(skb, (total_len - skb->len), 0, GFP_ATOMIC)) {
 			wl1271_free_tx_id(wl, id);
-			return -EAGAIN;
+			return -ENOMEM;
 		}
 		desc = skb_push(skb, total_len - skb->len);
 
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c
index e89173f916377..4a1eee1671b91 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim.c
+++ b/drivers/net/wireless/virtual/mac80211_hwsim.c
@@ -36,6 +36,8 @@
 #include <linux/virtio.h>
 #include <linux/virtio_ids.h>
 #include <linux/virtio_config.h>
+#include <linux/uaccess.h>
+#include <linux/string.h>
 #include "mac80211_hwsim.h"
 
 #define WARN_QUEUE 100
@@ -337,7 +339,9 @@ static inline void hwsim_net_set_wmediumd(struct net *net, u32 portid)
 	hwsim_net->wmediumd = portid;
 }
 
-static struct class *hwsim_class;
+static const struct class hwsim_class = {
+	.name	= "mac80211_hwsim"
+};
 
 static struct net_device *hwsim_mon; /* global monitor netdev */
 
@@ -715,6 +719,7 @@ struct mac80211_hwsim_data {
 	} ps;
 	bool ps_poll_pending;
 	struct dentry *debugfs;
+	struct cfg80211_chan_def radar_background_chandef;
 
 	atomic_t pending_cookie;
 	struct sk_buff_head pending;	/* packets pending */
@@ -936,6 +941,7 @@ static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
 	[HWSIM_ATTR_PMSR_RESULT] = NLA_POLICY_NESTED(hwsim_pmsr_peers_result_policy),
 	[HWSIM_ATTR_MULTI_RADIO] = { .type = NLA_FLAG },
 	[HWSIM_ATTR_SUPPORT_NAN_DEVICE] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR] = { .type = NLA_FLAG },
 };
 
 #if IS_REACHABLE(CONFIG_VIRTIO)
@@ -1164,6 +1170,100 @@ static int hwsim_write_simulate_radar(void *dat, u64 val)
 DEFINE_DEBUGFS_ATTRIBUTE(hwsim_simulate_radar, NULL,
 			 hwsim_write_simulate_radar, "%llu\n");
 
+static ssize_t hwsim_background_cac_write(struct file *file,
+					  const char __user *user_buf,
+					  size_t count, loff_t *ppos)
+{
+	struct mac80211_hwsim_data *data = file->private_data;
+	char buf[8] = {};
+
+	if (count >= sizeof(buf))
+		return -EINVAL;
+
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+
+	/* Check if background radar channel is configured */
+	if (!data->radar_background_chandef.chan)
+		return -ENOENT;
+
+	if (sysfs_streq(buf, "radar"))
+		cfg80211_background_radar_event(data->hw->wiphy,
+						&data->radar_background_chandef,
+						GFP_KERNEL);
+	else if (sysfs_streq(buf, "cancel"))
+		cfg80211_background_cac_abort(data->hw->wiphy);
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static const struct file_operations hwsim_background_cac_ops = {
+	.write = hwsim_background_cac_write,
+	.open = simple_open,
+	.llseek = default_llseek,
+};
+
+struct hwsim_chanctx_iter_arg {
+	struct ieee80211_chanctx_conf *conf;
+	u32 freq_mhz;
+};
+
+static void hwsim_6ghz_chanctx_iter(struct ieee80211_hw *hw,
+				    struct ieee80211_chanctx_conf *conf,
+				    void *data)
+{
+	struct hwsim_chanctx_iter_arg *arg = data;
+
+	if (conf->def.chan &&
+	    conf->def.chan->band == NL80211_BAND_6GHZ &&
+	    conf->def.chan->center_freq == arg->freq_mhz)
+		arg->conf = conf;
+}
+
+static ssize_t hwsim_simulate_incumbent_signal_write(struct file *file,
+						     const char __user *ubuf,
+						     size_t len, loff_t *ppos)
+{
+	struct mac80211_hwsim_data *data = file->private_data;
+	struct hwsim_chanctx_iter_arg arg = {};
+	u32 bitmap;
+	char buf[64];
+
+	if (!len || len > sizeof(buf) - 1)
+		return -EINVAL;
+
+	if (copy_from_user(buf, ubuf, len))
+		return -EFAULT;
+	buf[len] = '\0';
+
+	if (sscanf(buf, "%u %i", &arg.freq_mhz, &bitmap) != 2)
+		return -EINVAL;
+
+	if (!arg.freq_mhz)
+		return -EINVAL;
+
+	ieee80211_iter_chan_contexts_atomic(data->hw,
+					    hwsim_6ghz_chanctx_iter,
+					    &arg);
+
+	if (!arg.conf)
+		return -EINVAL;
+
+	cfg80211_incumbent_signal_notify(data->hw->wiphy,
+					 &arg.conf->def,
+					 bitmap,
+					 GFP_KERNEL);
+
+	return len;
+}
+
+static const struct file_operations hwsim_simulate_incumbent_signal_fops = {
+	.open	= simple_open,
+	.write	= hwsim_simulate_incumbent_signal_write,
+};
+
 static int hwsim_fops_group_read(void *dat, u64 *val)
 {
 	struct mac80211_hwsim_data *data = dat;
@@ -1955,6 +2055,25 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
 	return NULL;
 }
 
+static int mac80211_hwsim_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct ieee80211_key_conf *key)
+{
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_CCMP_256:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		break;
+	default:
+		return 1;
+	}
+
+	key->flags |= IEEE80211_KEY_FLAG_RESERVE_TAILROOM;
+	return 0;
+}
+
 static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 			      struct ieee80211_tx_control *control,
 			      struct sk_buff *skb)
@@ -1965,7 +2084,7 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 	struct ieee80211_chanctx_conf *chanctx_conf;
 	struct ieee80211_channel *channel;
 	struct ieee80211_vif *vif = txi->control.vif;
-	bool ack;
+	bool ack, unicast_data;
 	enum nl80211_chan_width confbw = NL80211_CHAN_WIDTH_20_NOHT;
 	u32 _portid, i;
 
@@ -1975,6 +2094,16 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 		return;
 	}
 
+	unicast_data = is_unicast_ether_addr(hdr->addr1) &&
+		       ieee80211_is_data(hdr->frame_control);
+
+	if (unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+	/* re-assign hdr since skb data may have shifted after encryption */
+	hdr = (void *)skb->data;
+
 	if (vif && vif->type == NL80211_IFTYPE_NAN && !data->tmp_chan) {
 		/* For NAN Device simulation purposes, assume that NAN is always
 		 * on channel 6 or channel 149, unless a ROC is in progress (for
@@ -2060,6 +2189,13 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 		}
 	}
 
+	if (!unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+	/* re-assign hdr since skb data may have shifted after encryption */
+	hdr = (void *)skb->data;
+
 	if (WARN(!channel, "TX w/o channel - queue = %d\n", txi->hw_queue)) {
 		ieee80211_free_txskb(hw, skb);
 		return;
@@ -3021,7 +3157,6 @@ static void hw_scan_work(struct work_struct *work)
 						      hwsim->tmp_chan->band,
 						      NULL)) {
 				rcu_read_unlock();
-				kfree_skb(probe);
 				continue;
 			}
 
@@ -4078,7 +4213,7 @@ static int mac80211_hwsim_start_nan(struct ieee80211_hw *hw,
 		      ns_to_ktime(until_dw * NSEC_PER_USEC),
 		      HRTIMER_MODE_REL_SOFT);
 
-	if (conf->cluster_id && !is_zero_ether_addr(conf->cluster_id) &&
+	if (!is_zero_ether_addr(conf->cluster_id) &&
 	    is_zero_ether_addr(hwsim_nan_cluster_id)) {
 		memcpy(hwsim_nan_cluster_id, conf->cluster_id, ETH_ALEN);
 	} else if (is_zero_ether_addr(hwsim_nan_cluster_id)) {
@@ -4154,6 +4289,24 @@ static int mac80211_hwsim_change_nan_config(struct ieee80211_hw *hw,
 	return 0;
 }
 
+static int mac80211_hwsim_set_radar_background(struct ieee80211_hw *hw,
+					       struct cfg80211_chan_def *chan)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	if (!wiphy_ext_feature_isset(hw->wiphy,
+				     NL80211_EXT_FEATURE_RADAR_BACKGROUND))
+		return -EOPNOTSUPP;
+
+	if (chan)
+		data->radar_background_chandef = *chan;
+	else
+		memset(&data->radar_background_chandef, 0,
+		       sizeof(data->radar_background_chandef));
+
+	return 0;
+}
+
 #ifdef CONFIG_MAC80211_DEBUGFS
 #define HWSIM_DEBUGFS_OPS					\
 	.link_add_debugfs = mac80211_hwsim_link_add_debugfs,
@@ -4189,6 +4342,8 @@ static int mac80211_hwsim_change_nan_config(struct ieee80211_hw *hw,
 	.start_nan = mac80211_hwsim_start_nan,                  \
 	.stop_nan = mac80211_hwsim_stop_nan,                    \
 	.nan_change_conf = mac80211_hwsim_change_nan_config,    \
+	.set_radar_background = mac80211_hwsim_set_radar_background, \
+	.set_key = mac80211_hwsim_set_key,			\
 	HWSIM_DEBUGFS_OPS
 
 #define HWSIM_NON_MLO_OPS					\
@@ -4255,6 +4410,7 @@ struct hwsim_new_radio_params {
 	bool mlo;
 	const struct cfg80211_pmsr_capabilities *pmsr_capa;
 	bool nan_device;
+	bool background_radar;
 };
 
 static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb,
@@ -4340,6 +4496,12 @@ static int append_radio_msg(struct sk_buff *skb, int id,
 		if (ret < 0)
 			return ret;
 	}
+
+	if (param->background_radar) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR);
+		if (ret < 0)
+			return ret;
+	}
 	return 0;
 }
 
@@ -5424,7 +5586,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 	data = hw->priv;
 	data->hw = hw;
 
-	data->dev = device_create(hwsim_class, NULL, 0, hw, "hwsim%d", idx);
+	data->dev = device_create(&hwsim_class, NULL, 0, hw, "hwsim%d", idx);
 	if (IS_ERR(data->dev)) {
 		printk(KERN_DEBUG
 		       "mac80211_hwsim: device_create failed (%ld)\n",
@@ -5621,6 +5783,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 			    WIPHY_FLAG_AP_UAPSD |
 			    WIPHY_FLAG_SUPPORTS_5_10_MHZ |
 			    WIPHY_FLAG_HAS_CHANNEL_SWITCH;
+	hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN;
 	hw->wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR |
 			       NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
 			       NL80211_FEATURE_STATIC_SMPS |
@@ -5639,6 +5802,14 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 			      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
 	wiphy_ext_feature_set(hw->wiphy,
 			      NL80211_EXT_FEATURE_BSS_COLOR);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_EXT_KEY_ID);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION);
 
 	hw->wiphy->interface_modes = param->iftypes;
 
@@ -5794,6 +5965,9 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 
 	wiphy_ext_feature_set(hw->wiphy,
 			      NL80211_EXT_FEATURE_DFS_CONCURRENT);
+	if (param->background_radar)
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_RADAR_BACKGROUND);
 
 	if (param->no_vif)
 		ieee80211_hw_set(hw, NO_AUTO_VIF);
@@ -5832,6 +6006,13 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 		debugfs_create_file("dfs_simulate_radar", 0222,
 				    data->debugfs,
 				    data, &hwsim_simulate_radar);
+	if (param->background_radar)
+		debugfs_create_file("dfs_background_cac", 0200,
+				    data->debugfs,
+				    data, &hwsim_background_cac_ops);
+	debugfs_create_file("simulate_incumbent_signal_interference", 0200,
+			    data->debugfs,
+			    data, &hwsim_simulate_incumbent_signal_fops);
 
 	if (param->pmsr_capa) {
 		data->pmsr_capa = *param->pmsr_capa;
@@ -5950,6 +6131,9 @@ static int mac80211_hwsim_get_radio(struct sk_buff *skb,
 	param.channels = data->channels;
 	param.hwname = wiphy_name(data->hw->wiphy);
 	param.pmsr_capa = &data->pmsr_capa;
+	param.background_radar =
+		wiphy_ext_feature_isset(data->hw->wiphy,
+					NL80211_EXT_FEATURE_RADAR_BACKGROUND);
 
 	res = append_radio_msg(skb, data->idx, &param);
 	if (res < 0)
@@ -5978,7 +6162,7 @@ static void mac80211_hwsim_free(void)
 		spin_lock_bh(&hwsim_radio_lock);
 	}
 	spin_unlock_bh(&hwsim_radio_lock);
-	class_destroy(hwsim_class);
+	class_unregister(&hwsim_class);
 }
 
 static const struct net_device_ops hwsim_netdev_ops = {
@@ -6387,6 +6571,9 @@ static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
 	if (info->attrs[HWSIM_ATTR_MULTI_RADIO])
 		param.multi_radio = true;
 
+	if (info->attrs[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR])
+		param.background_radar = true;
+
 	if (info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2])
 		param.reg_alpha2 =
 			nla_data(info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2]);
@@ -6489,7 +6676,7 @@ static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
 	if (info->attrs[HWSIM_ATTR_PMSR_SUPPORT]) {
 		struct cfg80211_pmsr_capabilities *pmsr_capa;
 
-		pmsr_capa = kmalloc_obj(*pmsr_capa);
+		pmsr_capa = kzalloc_obj(*pmsr_capa);
 		if (!pmsr_capa) {
 			ret = -ENOMEM;
 			goto out_free;
@@ -7083,11 +7270,9 @@ static int __init init_mac80211_hwsim(void)
 	if (err)
 		goto out_exit_netlink;
 
-	hwsim_class = class_create("mac80211_hwsim");
-	if (IS_ERR(hwsim_class)) {
-		err = PTR_ERR(hwsim_class);
+	err = class_register(&hwsim_class);
+	if (err)
 		goto out_exit_virtio;
-	}
 
 	hwsim_init_s1g_channels(hwsim_channels_s1g);
 
@@ -7165,6 +7350,7 @@ static int __init init_mac80211_hwsim(void)
 		param.p2p_device = support_p2p_device;
 		param.mlo = mlo;
 		param.multi_radio = multi_radio;
+		param.background_radar = true;
 		param.use_chanctx = channels > 1 || mlo || multi_radio;
 		param.iftypes = HWSIM_IFTYPE_SUPPORT_MASK;
 		if (param.p2p_device)
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.h b/drivers/net/wireless/virtual/mac80211_hwsim.h
index c2d06cf852a56..a022cd5c0f1c7 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim.h
+++ b/drivers/net/wireless/virtual/mac80211_hwsim.h
@@ -161,6 +161,7 @@ enum hwsim_commands {
  *	Adds one radio for each band. Number of supported channels will be set for
  *	each radio instead of for the wiphy.
  * @HWSIM_ATTR_SUPPORT_NAN_DEVICE: support NAN Device virtual interface (flag)
+ * @HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR: background radar/CAC support (flag)
  * @__HWSIM_ATTR_MAX: enum limit
  */
 enum hwsim_attrs {
@@ -195,6 +196,7 @@ enum hwsim_attrs {
 	HWSIM_ATTR_PMSR_RESULT,
 	HWSIM_ATTR_MULTI_RADIO,
 	HWSIM_ATTR_SUPPORT_NAN_DEVICE,
+	HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR,
 	__HWSIM_ATTR_MAX,
 };
 #define HWSIM_ATTR_MAX (__HWSIM_ATTR_MAX - 1)
diff --git a/drivers/net/wireless/virtual/virt_wifi.c b/drivers/net/wireless/virtual/virt_wifi.c
index 885dc7243e8dc..bc349c7635780 100644
--- a/drivers/net/wireless/virtual/virt_wifi.c
+++ b/drivers/net/wireless/virtual/virt_wifi.c
@@ -320,9 +320,11 @@ static int virt_wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev,
 }
 
 /* Called with the rtnl lock held. */
-static int virt_wifi_get_station(struct wiphy *wiphy, struct net_device *dev,
-				 const u8 *mac, struct station_info *sinfo)
+static int virt_wifi_get_station(struct wiphy *wiphy,
+				 struct wireless_dev *wdev, const u8 *mac,
+				 struct station_info *sinfo)
 {
+	struct net_device *dev = wdev->netdev;
 	struct virt_wifi_netdev_priv *priv = netdev_priv(dev);
 
 	wiphy_debug(wiphy, "get_station\n");
@@ -345,10 +347,10 @@ static int virt_wifi_get_station(struct wiphy *wiphy, struct net_device *dev,
 }
 
 /* Called with the rtnl lock held. */
-static int virt_wifi_dump_station(struct wiphy *wiphy, struct net_device *dev,
+static int virt_wifi_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 				  int idx, u8 *mac, struct station_info *sinfo)
 {
-	struct virt_wifi_netdev_priv *priv = netdev_priv(dev);
+	struct virt_wifi_netdev_priv *priv = netdev_priv(wdev->netdev);
 
 	wiphy_debug(wiphy, "dump_station\n");
 
@@ -356,7 +358,7 @@ static int virt_wifi_dump_station(struct wiphy *wiphy, struct net_device *dev,
 		return -ENOENT;
 
 	ether_addr_copy(mac, fake_router_bssid);
-	return virt_wifi_get_station(wiphy, dev, fake_router_bssid, sinfo);
+	return virt_wifi_get_station(wiphy, wdev, fake_router_bssid, sinfo);
 }
 
 static const struct cfg80211_ops virt_wifi_cfg80211_ops = {
diff --git a/drivers/nfc/nfcmrvl/fw_dnld.c b/drivers/nfc/nfcmrvl/fw_dnld.c
index a9b03dcc41006..2b8f401d8fd7a 100644
--- a/drivers/nfc/nfcmrvl/fw_dnld.c
+++ b/drivers/nfc/nfcmrvl/fw_dnld.c
@@ -492,7 +492,7 @@ int nfcmrvl_fw_dnld_start(struct nci_dev *ndev, const char *firmware_name)
 	if (!firmware_name || !firmware_name[0])
 		return -EINVAL;
 
-	strcpy(fw_dnld->name, firmware_name);
+	strscpy(fw_dnld->name, firmware_name);
 
 	/*
 	 * Retrieve FW binary file and parse it to initialize FW download
diff --git a/drivers/nfc/nfcmrvl/main.c b/drivers/nfc/nfcmrvl/main.c
index c51d22e4579c1..6efa832191172 100644
--- a/drivers/nfc/nfcmrvl/main.c
+++ b/drivers/nfc/nfcmrvl/main.c
@@ -6,9 +6,9 @@
  */
 
 #include <linux/module.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/delay.h>
-#include <linux/of_gpio.h>
+#include <linux/of.h>
 #include <linux/nfc.h>
 #include <net/nfc/nci.h>
 #include <net/nfc/nci_core.h>
@@ -112,13 +112,12 @@ struct nfcmrvl_private *nfcmrvl_nci_register_dev(enum nfcmrvl_phy phy,
 
 	memcpy(&priv->config, pdata, sizeof(*pdata));
 
-	if (gpio_is_valid(priv->config.reset_n_io)) {
-		rc = gpio_request_one(priv->config.reset_n_io,
-				      GPIOF_OUT_INIT_LOW,
-				      "nfcmrvl_reset_n");
-		if (rc < 0) {
-			priv->config.reset_n_io = -EINVAL;
-			nfc_err(dev, "failed to request reset_n io\n");
+	if (!priv->config.reset_gpio) {
+		priv->config.reset_gpio =
+			devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+		if (IS_ERR(priv->config.reset_gpio)) {
+			priv->config.reset_gpio = NULL;
+			nfc_err(dev, "failed to get reset gpio\n");
 		}
 	}
 
@@ -144,7 +143,7 @@ struct nfcmrvl_private *nfcmrvl_nci_register_dev(enum nfcmrvl_phy phy,
 	if (!priv->ndev) {
 		nfc_err(dev, "nci_allocate_device failed\n");
 		rc = -ENOMEM;
-		goto error_free_gpio;
+		goto error_free;
 	}
 
 	rc = nfcmrvl_fw_dnld_init(priv);
@@ -171,9 +170,7 @@ struct nfcmrvl_private *nfcmrvl_nci_register_dev(enum nfcmrvl_phy phy,
 	nfcmrvl_fw_dnld_deinit(priv);
 error_free_dev:
 	nci_free_device(priv->ndev);
-error_free_gpio:
-	if (gpio_is_valid(priv->config.reset_n_io))
-		gpio_free(priv->config.reset_n_io);
+error_free:
 	kfree(priv);
 	return ERR_PTR(rc);
 }
@@ -189,9 +186,6 @@ void nfcmrvl_nci_unregister_dev(struct nfcmrvl_private *priv)
 
 	nfcmrvl_fw_dnld_deinit(priv);
 
-	if (gpio_is_valid(priv->config.reset_n_io))
-		gpio_free(priv->config.reset_n_io);
-
 	nci_free_device(ndev);
 	kfree(priv);
 }
@@ -233,34 +227,25 @@ void nfcmrvl_chip_reset(struct nfcmrvl_private *priv)
 	/* Reset possible fault of previous session */
 	clear_bit(NFCMRVL_PHY_ERROR, &priv->flags);
 
-	if (gpio_is_valid(priv->config.reset_n_io)) {
+	if (priv->config.reset_gpio) {
 		nfc_info(priv->dev, "reset the chip\n");
-		gpio_set_value(priv->config.reset_n_io, 0);
+		gpiod_set_value(priv->config.reset_gpio, 1);
 		usleep_range(5000, 10000);
-		gpio_set_value(priv->config.reset_n_io, 1);
+		gpiod_set_value(priv->config.reset_gpio, 0);
 	} else
 		nfc_info(priv->dev, "no reset available on this interface\n");
 }
 
 void nfcmrvl_chip_halt(struct nfcmrvl_private *priv)
 {
-	if (gpio_is_valid(priv->config.reset_n_io))
-		gpio_set_value(priv->config.reset_n_io, 0);
+	if (priv->config.reset_gpio)
+		gpiod_set_value(priv->config.reset_gpio, 1);
 }
 
 int nfcmrvl_parse_dt(struct device_node *node,
 		     struct nfcmrvl_platform_data *pdata)
 {
-	int reset_n_io;
-
-	reset_n_io = of_get_named_gpio(node, "reset-n-io", 0);
-	if (reset_n_io < 0) {
-		pr_info("no reset-n-io config\n");
-	} else if (!gpio_is_valid(reset_n_io)) {
-		pr_err("invalid reset-n-io GPIO\n");
-		return reset_n_io;
-	}
-	pdata->reset_n_io = reset_n_io;
+	pdata->reset_gpio = NULL;
 	pdata->hci_muxed = of_property_read_bool(node, "hci-muxed");
 
 	return 0;
diff --git a/drivers/nfc/nfcmrvl/nfcmrvl.h b/drivers/nfc/nfcmrvl/nfcmrvl.h
index f61a99e553db0..62fa77d587b2b 100644
--- a/drivers/nfc/nfcmrvl/nfcmrvl.h
+++ b/drivers/nfc/nfcmrvl/nfcmrvl.h
@@ -10,6 +10,8 @@
 
 #include "fw_dnld.h"
 
+struct gpio_desc;
+
 /* Define private flags: */
 #define NFCMRVL_NCI_RUNNING			1
 #define NFCMRVL_PHY_ERROR			2
@@ -54,7 +56,7 @@ struct nfcmrvl_platform_data {
 	 */
 
 	/* GPIO that is wired to RESET_N signal */
-	int reset_n_io;
+	struct gpio_desc *reset_gpio;
 	/* Tell if transport is muxed in HCI one */
 	bool hci_muxed;
 
diff --git a/drivers/nfc/nfcmrvl/uart.c b/drivers/nfc/nfcmrvl/uart.c
index 2037cd6d4f4f4..9aedd168759c7 100644
--- a/drivers/nfc/nfcmrvl/uart.c
+++ b/drivers/nfc/nfcmrvl/uart.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/printk.h>
@@ -20,7 +21,6 @@
 static unsigned int hci_muxed;
 static unsigned int flow_control;
 static unsigned int break_control;
-static int reset_n_io = -EINVAL;
 
 /*
  * NFCMRVL NCI OPS
@@ -62,9 +62,11 @@ static const struct nfcmrvl_if_ops uart_ops = {
 };
 
 static int nfcmrvl_uart_parse_dt(struct device_node *node,
-				 struct nfcmrvl_platform_data *pdata)
+				 struct nfcmrvl_platform_data *pdata,
+				 struct device *dev)
 {
 	struct device_node *matched_node;
+	struct gpio_desc *reset_gpio;
 	int ret;
 
 	matched_node = of_get_compatible_child(node, "marvell,nfc-uart");
@@ -84,6 +86,16 @@ static int nfcmrvl_uart_parse_dt(struct device_node *node,
 	pdata->flow_control = of_property_read_bool(matched_node, "flow-control");
 	pdata->break_control = of_property_read_bool(matched_node, "break-control");
 
+	reset_gpio = devm_fwnode_gpiod_get_optional(dev,
+						    of_fwnode_handle(matched_node),
+						    "reset", GPIOD_OUT_HIGH,
+						    "nfcmrvl_reset_n");
+	if (IS_ERR(reset_gpio)) {
+		of_node_put(matched_node);
+		return PTR_ERR(reset_gpio);
+	}
+	pdata->reset_gpio = reset_gpio;
+
 	of_node_put(matched_node);
 
 	return 0;
@@ -107,13 +119,13 @@ static int nfcmrvl_nci_uart_open(struct nci_uart *nu)
 	 */
 
 	if (dev && dev->parent && dev->parent->of_node)
-		if (nfcmrvl_uart_parse_dt(dev->parent->of_node, &config) == 0)
+		if (nfcmrvl_uart_parse_dt(dev->parent->of_node, &config, dev) == 0)
 			pdata = &config;
 
 	if (!pdata) {
 		pr_info("No platform data / DT -> fallback to module params\n");
 		config.hci_muxed = hci_muxed;
-		config.reset_n_io = reset_n_io;
+		config.reset_gpio = NULL;
 		config.flow_control = flow_control;
 		config.break_control = break_control;
 		pdata = &config;
@@ -201,6 +213,3 @@ MODULE_PARM_DESC(break_control, "Tell if UART driver must drive break signal.");
 
 module_param(hci_muxed, uint, 0);
 MODULE_PARM_DESC(hci_muxed, "Tell if transport is muxed in HCI one.");
-
-module_param(reset_n_io, int, 0);
-MODULE_PARM_DESC(reset_n_io, "GPIO that is wired to RESET_N signal.");
diff --git a/drivers/nfc/nfcmrvl/usb.c b/drivers/nfc/nfcmrvl/usb.c
index ea73094530968..ac62358445bf8 100644
--- a/drivers/nfc/nfcmrvl/usb.c
+++ b/drivers/nfc/nfcmrvl/usb.c
@@ -294,7 +294,7 @@ static int nfcmrvl_probe(struct usb_interface *intf,
 
 	/* No configuration for USB */
 	memset(&config, 0, sizeof(config));
-	config.reset_n_io = -EINVAL;
+	config.reset_gpio = NULL;
 
 	nfc_info(&udev->dev, "intf %p id %p\n", intf, id);
 
diff --git a/drivers/nfc/nxp-nci/firmware.c b/drivers/nfc/nxp-nci/firmware.c
index 381b5bb754779..a9533977aff8e 100644
--- a/drivers/nfc/nxp-nci/firmware.c
+++ b/drivers/nfc/nxp-nci/firmware.c
@@ -211,7 +211,7 @@ int nxp_nci_fw_download(struct nci_dev *ndev, const char *firmware_name)
 		goto fw_download_exit;
 	}
 
-	strcpy(fw_info->name, firmware_name);
+	strscpy(fw_info->name, firmware_name);
 
 	r = request_firmware(&fw_info->fw, firmware_name,
 			     ndev->nfc_dev->dev.parent);
diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c
index 6a5ce8ff91f0b..b3d34433bd14a 100644
--- a/drivers/nfc/nxp-nci/i2c.c
+++ b/drivers/nfc/nxp-nci/i2c.c
@@ -47,8 +47,8 @@ static int nxp_nci_i2c_set_mode(void *phy_id,
 {
 	struct nxp_nci_i2c_phy *phy = (struct nxp_nci_i2c_phy *) phy_id;
 
-	gpiod_set_value(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0);
-	gpiod_set_value(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0);
+	gpiod_set_value_cansleep(phy->gpiod_fw, (mode == NXP_NCI_MODE_FW) ? 1 : 0);
+	gpiod_set_value_cansleep(phy->gpiod_en, (mode != NXP_NCI_MODE_COLD) ? 1 : 0);
 	usleep_range(10000, 15000);
 
 	if (mode == NXP_NCI_MODE_COLD)
diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c
index 0f12f86ebb023..0ff2f0d7caf4d 100644
--- a/drivers/nfc/pn533/usb.c
+++ b/drivers/nfc/pn533/usb.c
@@ -500,7 +500,7 @@ static int pn533_usb_probe(struct usb_interface *interface,
 	if (!in_buf)
 		return -ENOMEM;
 
-	phy->udev = usb_get_dev(interface_to_usbdev(interface));
+	phy->udev = interface_to_usbdev(interface);
 	phy->interface = interface;
 
 	iface_desc = interface->cur_altsetting;
@@ -600,7 +600,6 @@ static int pn533_usb_probe(struct usb_interface *interface,
 	usb_free_urb(phy->in_urb);
 	usb_free_urb(phy->out_urb);
 	usb_free_urb(phy->ack_urb);
-	usb_put_dev(phy->udev);
 	kfree(in_buf);
 	kfree(phy->ack_buffer);
 
@@ -628,7 +627,6 @@ static void pn533_usb_disconnect(struct usb_interface *interface)
 	usb_free_urb(phy->out_urb);
 	usb_free_urb(phy->ack_urb);
 	kfree(phy->ack_buffer);
-	usb_put_dev(phy->udev);
 
 	nfc_info(&interface->dev, "NXP PN533 NFC device disconnected\n");
 }
diff --git a/drivers/nfc/pn544/i2c.c b/drivers/nfc/pn544/i2c.c
index a0dfb3f98d5a9..b31b5bef71877 100644
--- a/drivers/nfc/pn544/i2c.c
+++ b/drivers/nfc/pn544/i2c.c
@@ -526,7 +526,7 @@ static int pn544_hci_i2c_fw_download(void *phy_id, const char *firmware_name,
 
 	pr_info("Starting Firmware Download (%s)\n", firmware_name);
 
-	strcpy(phy->firmware_name, firmware_name);
+	strscpy(phy->firmware_name, firmware_name);
 
 	phy->hw_variant = hw_variant;
 	phy->fw_work_state = FW_WORK_STATE_START;
diff --git a/drivers/nfc/port100.c b/drivers/nfc/port100.c
index a922569d7b48b..09dcc6f4c4f38 100644
--- a/drivers/nfc/port100.c
+++ b/drivers/nfc/port100.c
@@ -1504,7 +1504,7 @@ static int port100_probe(struct usb_interface *interface,
 		return -ENOMEM;
 
 	mutex_init(&dev->out_urb_lock);
-	dev->udev = usb_get_dev(interface_to_usbdev(interface));
+	dev->udev = interface_to_usbdev(interface);
 	dev->interface = interface;
 	usb_set_intfdata(interface, dev);
 
@@ -1616,7 +1616,6 @@ static int port100_probe(struct usb_interface *interface,
 	usb_free_urb(dev->in_urb);
 	usb_kill_urb(dev->out_urb);
 	usb_free_urb(dev->out_urb);
-	usb_put_dev(dev->udev);
 
 	return rc;
 }
@@ -1636,7 +1635,6 @@ static void port100_disconnect(struct usb_interface *interface)
 
 	usb_free_urb(dev->in_urb);
 	usb_free_urb(dev->out_urb);
-	usb_put_dev(dev->udev);
 
 	kfree(dev->cmd);
 
diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c
index 64d61b2a715ae..9145deec7f6c1 100644
--- a/drivers/nfc/s3fwrn5/firmware.c
+++ b/drivers/nfc/s3fwrn5/firmware.c
@@ -454,7 +454,7 @@ void s3fwrn5_fw_init(struct s3fwrn5_fw_info *fw_info, const char *fw_name)
 	fw_info->parity = 0x00;
 	fw_info->rsp = NULL;
 	fw_info->fw.fw = NULL;
-	strcpy(fw_info->fw_name, fw_name);
+	strscpy(fw_info->fw_name, fw_name);
 	init_completion(&fw_info->completion);
 }
 
diff --git a/drivers/nfc/s3fwrn5/i2c.c b/drivers/nfc/s3fwrn5/i2c.c
index 110d086cfe5ba..91b8d1445efd4 100644
--- a/drivers/nfc/s3fwrn5/i2c.c
+++ b/drivers/nfc/s3fwrn5/i2c.c
@@ -8,10 +8,8 @@
 
 #include <linux/clk.h>
 #include <linux/i2c.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/delay.h>
-#include <linux/of_gpio.h>
-#include <linux/of_irq.h>
 #include <linux/module.h>
 
 #include <net/nfc/nfc.h>
@@ -146,37 +144,6 @@ static irqreturn_t s3fwrn5_i2c_irq_thread_fn(int irq, void *phy_id)
 	return IRQ_HANDLED;
 }
 
-static int s3fwrn5_i2c_parse_dt(struct i2c_client *client)
-{
-	struct s3fwrn5_i2c_phy *phy = i2c_get_clientdata(client);
-	struct device_node *np = client->dev.of_node;
-
-	if (!np)
-		return -ENODEV;
-
-	phy->common.gpio_en = of_get_named_gpio(np, "en-gpios", 0);
-	if (!gpio_is_valid(phy->common.gpio_en)) {
-		/* Support also deprecated property */
-		phy->common.gpio_en = of_get_named_gpio(np,
-							"s3fwrn5,en-gpios",
-							0);
-		if (!gpio_is_valid(phy->common.gpio_en))
-			return -ENODEV;
-	}
-
-	phy->common.gpio_fw_wake = of_get_named_gpio(np, "wake-gpios", 0);
-	if (!gpio_is_valid(phy->common.gpio_fw_wake)) {
-		/* Support also deprecated property */
-		phy->common.gpio_fw_wake = of_get_named_gpio(np,
-							     "s3fwrn5,fw-gpios",
-							     0);
-		if (!gpio_is_valid(phy->common.gpio_fw_wake))
-			return -ENODEV;
-	}
-
-	return 0;
-}
-
 static int s3fwrn5_i2c_probe(struct i2c_client *client)
 {
 	struct s3fwrn5_i2c_phy *phy;
@@ -193,20 +160,13 @@ static int s3fwrn5_i2c_probe(struct i2c_client *client)
 	phy->i2c_dev = client;
 	i2c_set_clientdata(client, phy);
 
-	ret = s3fwrn5_i2c_parse_dt(client);
-	if (ret < 0)
-		return ret;
-
-	ret = devm_gpio_request_one(&phy->i2c_dev->dev, phy->common.gpio_en,
-				    GPIOF_OUT_INIT_HIGH, "s3fwrn5_en");
-	if (ret < 0)
-		return ret;
+	phy->common.gpio_en = devm_gpiod_get(&client->dev, "en", GPIOD_OUT_HIGH);
+	if (IS_ERR(phy->common.gpio_en))
+		return PTR_ERR(phy->common.gpio_en);
 
-	ret = devm_gpio_request_one(&phy->i2c_dev->dev,
-				    phy->common.gpio_fw_wake,
-				    GPIOF_OUT_INIT_LOW, "s3fwrn5_fw_wake");
-	if (ret < 0)
-		return ret;
+	phy->common.gpio_fw_wake = devm_gpiod_get(&client->dev, "wake", GPIOD_OUT_LOW);
+	if (IS_ERR(phy->common.gpio_fw_wake))
+		return PTR_ERR(phy->common.gpio_fw_wake);
 
 	/*
 	 * S3FWRN5 depends on a clock input ("XI" pin) to function properly.
diff --git a/drivers/nfc/s3fwrn5/phy_common.c b/drivers/nfc/s3fwrn5/phy_common.c
index deb2c039f0fd6..39a60e34136c5 100644
--- a/drivers/nfc/s3fwrn5/phy_common.c
+++ b/drivers/nfc/s3fwrn5/phy_common.c
@@ -8,7 +8,6 @@
  * Bongsu Jeon <bongsu.jeon@samsung.com>
  */
 
-#include <linux/gpio.h>
 #include <linux/delay.h>
 #include <linux/module.h>
 
@@ -19,7 +18,7 @@ void s3fwrn5_phy_set_wake(void *phy_id, bool wake)
 	struct phy_common *phy = phy_id;
 
 	mutex_lock(&phy->mutex);
-	gpio_set_value(phy->gpio_fw_wake, wake);
+	gpiod_set_value(phy->gpio_fw_wake, wake);
 	if (wake)
 		msleep(S3FWRN5_EN_WAIT_TIME);
 	mutex_unlock(&phy->mutex);
@@ -33,14 +32,14 @@ bool s3fwrn5_phy_power_ctrl(struct phy_common *phy, enum s3fwrn5_mode mode)
 
 	phy->mode = mode;
 
-	gpio_set_value(phy->gpio_en, 1);
-	gpio_set_value(phy->gpio_fw_wake, 0);
+	gpiod_set_value(phy->gpio_en, 1);
+	gpiod_set_value(phy->gpio_fw_wake, 0);
 	if (mode == S3FWRN5_MODE_FW)
-		gpio_set_value(phy->gpio_fw_wake, 1);
+		gpiod_set_value(phy->gpio_fw_wake, 1);
 
 	if (mode != S3FWRN5_MODE_COLD) {
 		msleep(S3FWRN5_EN_WAIT_TIME);
-		gpio_set_value(phy->gpio_en, 0);
+		gpiod_set_value(phy->gpio_en, 0);
 		msleep(S3FWRN5_EN_WAIT_TIME);
 	}
 
diff --git a/drivers/nfc/s3fwrn5/phy_common.h b/drivers/nfc/s3fwrn5/phy_common.h
index 9cef25436bf90..871bec53dd0a9 100644
--- a/drivers/nfc/s3fwrn5/phy_common.h
+++ b/drivers/nfc/s3fwrn5/phy_common.h
@@ -12,6 +12,7 @@
 #define __NFC_S3FWRN5_PHY_COMMON_H
 
 #include <linux/mutex.h>
+#include <linux/gpio/consumer.h>
 #include <net/nfc/nci_core.h>
 
 #include "s3fwrn5.h"
@@ -21,8 +22,8 @@
 struct phy_common {
 	struct nci_dev *ndev;
 
-	int gpio_en;
-	int gpio_fw_wake;
+	struct gpio_desc *gpio_en;
+	struct gpio_desc *gpio_fw_wake;
 
 	struct mutex mutex;
 
diff --git a/drivers/nfc/s3fwrn5/uart.c b/drivers/nfc/s3fwrn5/uart.c
index 9c09c10c2a464..ccfcee06ee774 100644
--- a/drivers/nfc/s3fwrn5/uart.c
+++ b/drivers/nfc/s3fwrn5/uart.c
@@ -10,13 +10,12 @@
 
 #include <linux/device.h>
 #include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/nfc.h>
 #include <linux/netdevice.h>
-#include <linux/of.h>
 #include <linux/serdev.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 
 #include "phy_common.h"
 
@@ -88,25 +87,6 @@ static const struct of_device_id s3fwrn82_uart_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, s3fwrn82_uart_of_match);
 
-static int s3fwrn82_uart_parse_dt(struct serdev_device *serdev)
-{
-	struct s3fwrn82_uart_phy *phy = serdev_device_get_drvdata(serdev);
-	struct device_node *np = serdev->dev.of_node;
-
-	if (!np)
-		return -ENODEV;
-
-	phy->common.gpio_en = of_get_named_gpio(np, "en-gpios", 0);
-	if (!gpio_is_valid(phy->common.gpio_en))
-		return -ENODEV;
-
-	phy->common.gpio_fw_wake = of_get_named_gpio(np, "wake-gpios", 0);
-	if (!gpio_is_valid(phy->common.gpio_fw_wake))
-		return -ENODEV;
-
-	return 0;
-}
-
 static int s3fwrn82_uart_probe(struct serdev_device *serdev)
 {
 	struct s3fwrn82_uart_phy *phy;
@@ -140,20 +120,17 @@ static int s3fwrn82_uart_probe(struct serdev_device *serdev)
 
 	serdev_device_set_flow_control(serdev, false);
 
-	ret = s3fwrn82_uart_parse_dt(serdev);
-	if (ret < 0)
-		goto err_serdev;
-
-	ret = devm_gpio_request_one(&phy->ser_dev->dev, phy->common.gpio_en,
-				    GPIOF_OUT_INIT_HIGH, "s3fwrn82_en");
-	if (ret < 0)
+	phy->common.gpio_en = devm_gpiod_get(&serdev->dev, "en", GPIOD_OUT_HIGH);
+	if (IS_ERR(phy->common.gpio_en)) {
+		ret = PTR_ERR(phy->common.gpio_en);
 		goto err_serdev;
+	}
 
-	ret = devm_gpio_request_one(&phy->ser_dev->dev,
-				    phy->common.gpio_fw_wake,
-				    GPIOF_OUT_INIT_LOW, "s3fwrn82_fw_wake");
-	if (ret < 0)
+	phy->common.gpio_fw_wake = devm_gpiod_get(&serdev->dev, "wake", GPIOD_OUT_LOW);
+	if (IS_ERR(phy->common.gpio_fw_wake)) {
+		ret = PTR_ERR(phy->common.gpio_fw_wake);
 		goto err_serdev;
+	}
 
 	ret = s3fwrn5_probe(&phy->common.ndev, phy, &phy->ser_dev->dev,
 			    &uart_phy_ops);
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 1a163596ddf54..df9150669ad3a 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -376,7 +376,7 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb)
 
 	if (ndev->ntb.topo == NTB_TOPO_SEC)
 		return -EINVAL;
-	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
+	dev_dbg(&ntb->pdev->dev, "Disabling Link.\n");
 
 	return 0;
 }
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index b6bef092f8b80..fdcb080a4314f 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -1435,14 +1435,16 @@ static void btt_submit_bio(struct bio *bio)
 {
 	struct bio_integrity_payload *bip = bio_integrity(bio);
 	struct btt *btt = bio->bi_bdev->bd_disk->private_data;
+	unsigned int integrity_action;
 	struct bvec_iter iter;
 	unsigned long start;
 	struct bio_vec bvec;
 	int err = 0;
 	bool do_acct;
 
-	if (!bio_integrity_prep(bio))
-		return;
+	integrity_action = bio_integrity_action(bio);
+	if (integrity_action)
+		bio_integrity_prep(bio, integrity_action);
 
 	do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
 	if (do_acct)
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index bd9621d3f73cf..45b7d756e39a0 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -486,14 +486,15 @@ EXPORT_SYMBOL_GPL(nd_synchronize);
 static void nd_async_device_register(void *d, async_cookie_t cookie)
 {
 	struct device *dev = d;
+	struct device *parent = dev->parent;
 
 	if (device_add(dev) != 0) {
 		dev_err(dev, "%s: failed\n", __func__);
 		put_device(dev);
 	}
 	put_device(dev);
-	if (dev->parent)
-		put_device(dev->parent);
+	if (parent)
+		put_device(parent);
 }
 
 static void nd_async_device_unregister(void *d, async_cookie_t cookie)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 766e9cc4ffca5..4598f4c5262c7 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1875,6 +1875,7 @@ static bool nvme_init_integrity(struct nvme_ns_head *head,
 		break;
 	}
 
+	bi->flags |= BLK_SPLIT_INTERVAL_CAPABLE;
 	bi->metadata_size = head->ms;
 	if (bi->csum_type) {
 		bi->pi_tuple_size = head->pi_size;
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 8844bbd395159..9597a87cf05dc 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -786,10 +786,6 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
 	struct nvme_ctrl *ctrl = ioucmd->file->private_data;
 	int ret;
 
-	/* IOPOLL not supported yet */
-	if (issue_flags & IO_URING_F_IOPOLL)
-		return -EOPNOTSUPP;
-
 	ret = nvme_uring_cmd_checks(issue_flags);
 	if (ret)
 		return ret;
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index fc6800a9f7f94..ba00f0b72b858 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -154,21 +154,8 @@ void nvme_failover_req(struct request *req)
 	}
 
 	spin_lock_irqsave(&ns->head->requeue_lock, flags);
-	for (bio = req->bio; bio; bio = bio->bi_next) {
+	for (bio = req->bio; bio; bio = bio->bi_next)
 		bio_set_dev(bio, ns->head->disk->part0);
-		if (bio->bi_opf & REQ_POLLED) {
-			bio->bi_opf &= ~REQ_POLLED;
-			bio->bi_cookie = BLK_QC_T_NONE;
-		}
-		/*
-		 * The alternate request queue that we may end up submitting
-		 * the bio to may be frozen temporarily, in this case REQ_NOWAIT
-		 * will fail the I/O immediately with EAGAIN to the issuer.
-		 * We are not in the issuer context which cannot block. Clear
-		 * the flag to avoid spurious EAGAIN I/O failures.
-		 */
-		bio->bi_opf &= ~REQ_NOWAIT;
-	}
 	blk_steal_bios(&ns->head->requeue_list, req);
 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
 
diff --git a/drivers/nvmem/imx-ocotp-ele.c b/drivers/nvmem/imx-ocotp-ele.c
index 7cf7e809a8f51..a0d2985c6d030 100644
--- a/drivers/nvmem/imx-ocotp-ele.c
+++ b/drivers/nvmem/imx-ocotp-ele.c
@@ -131,6 +131,7 @@ static int imx_ocotp_cell_pp(void *context, const char *id, int index,
 static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem,
 					 struct nvmem_cell_info *cell)
 {
+	cell->raw_len = round_up(cell->bytes, 4);
 	cell->read_post_process = imx_ocotp_cell_pp;
 }
 
diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c
index 7bf7656d4f963..108d78d7f6cba 100644
--- a/drivers/nvmem/imx-ocotp.c
+++ b/drivers/nvmem/imx-ocotp.c
@@ -589,6 +589,7 @@ MODULE_DEVICE_TABLE(of, imx_ocotp_dt_ids);
 static void imx_ocotp_fixup_dt_cell_info(struct nvmem_device *nvmem,
 					 struct nvmem_cell_info *cell)
 {
+	cell->raw_len = round_up(cell->bytes, 4);
 	cell->read_post_process = imx_ocotp_cell_pp;
 }
 
diff --git a/drivers/nvmem/qnap-mcu-eeprom.c b/drivers/nvmem/qnap-mcu-eeprom.c
index 0b919895b3b2b..07bdaa2a33fa2 100644
--- a/drivers/nvmem/qnap-mcu-eeprom.c
+++ b/drivers/nvmem/qnap-mcu-eeprom.c
@@ -86,10 +86,10 @@ static int qnap_mcu_eeprom_probe(struct platform_device *pdev)
 	nvcfg.read_only = true;
 	nvcfg.root_only = false;
 	nvcfg.reg_read = qnap_mcu_eeprom_read;
-	nvcfg.size = QNAP_MCU_EEPROM_SIZE,
-	nvcfg.word_size = 1,
-	nvcfg.stride = 1,
-	nvcfg.priv = mcu,
+	nvcfg.size = QNAP_MCU_EEPROM_SIZE;
+	nvcfg.word_size = 1;
+	nvcfg.stride = 1;
+	nvcfg.priv = mcu;
 
 	ndev = devm_nvmem_register(&pdev->dev, &nvcfg);
 	if (IS_ERR(ndev))
diff --git a/drivers/nvmem/rockchip-otp.c b/drivers/nvmem/rockchip-otp.c
index d88f12c532426..0ec78b5e19e7d 100644
--- a/drivers/nvmem/rockchip-otp.c
+++ b/drivers/nvmem/rockchip-otp.c
@@ -27,6 +27,7 @@
 #define OTPC_USER_CTRL			0x0100
 #define OTPC_USER_ADDR			0x0104
 #define OTPC_USER_ENABLE		0x0108
+#define OTPC_USER_QP			0x0120
 #define OTPC_USER_Q			0x0124
 #define OTPC_INT_STATUS			0x0304
 #define OTPC_SBPI_CMD0_OFFSET		0x1000
@@ -59,7 +60,6 @@
 #define RK3588_OTPC_AUTO_EN		0x08
 #define RK3588_OTPC_INT_ST		0x84
 #define RK3588_OTPC_DOUT0		0x20
-#define RK3588_NBYTES			4
 #define RK3588_BURST_NUM		1
 #define RK3588_BURST_SHIFT		8
 #define RK3588_ADDR_SHIFT		16
@@ -69,6 +69,7 @@
 struct rockchip_data {
 	int size;
 	int read_offset;
+	int word_size;
 	const char * const *clks;
 	int num_clks;
 	nvmem_reg_read_t reg_read;
@@ -184,49 +185,81 @@ static int px30_otp_read(void *context, unsigned int offset,
 	return ret;
 }
 
+static int rk3568_otp_read(void *context, unsigned int offset, void *val,
+			   size_t count)
+{
+	struct rockchip_otp *otp = context;
+	u16 *buf = val;
+	u32 otp_qp;
+	int ret;
+
+	ret = rockchip_otp_reset(otp);
+	if (ret) {
+		dev_err(otp->dev, "failed to reset otp phy\n");
+		return ret;
+	}
+
+	ret = rockchip_otp_ecc_enable(otp, true);
+	if (ret) {
+		dev_err(otp->dev, "rockchip_otp_ecc_enable err\n");
+		return ret;
+	}
+
+	writel(OTPC_USE_USER | OTPC_USE_USER_MASK, otp->base + OTPC_USER_CTRL);
+	udelay(5);
+
+	while (count--) {
+		writel(offset++ | OTPC_USER_ADDR_MASK,
+		       otp->base + OTPC_USER_ADDR);
+		writel(OTPC_USER_FSM_ENABLE | OTPC_USER_FSM_ENABLE_MASK,
+		       otp->base + OTPC_USER_ENABLE);
+
+		ret = rockchip_otp_wait_status(otp, OTPC_INT_STATUS,
+					       OTPC_USER_DONE);
+		if (ret) {
+			dev_err(otp->dev, "timeout during read setup\n");
+			goto read_end;
+		}
+
+		otp_qp = readl(otp->base + OTPC_USER_QP);
+		if (((otp_qp & 0xc0) == 0xc0) || (otp_qp & 0x20)) {
+			ret = -EIO;
+			dev_err(otp->dev, "ecc check error during read setup\n");
+			goto read_end;
+		}
+
+		*buf++ = readl(otp->base + OTPC_USER_Q);
+	}
+
+read_end:
+	writel(0x0 | OTPC_USE_USER_MASK, otp->base + OTPC_USER_CTRL);
+
+	return ret;
+}
+
 static int rk3588_otp_read(void *context, unsigned int offset,
-			   void *val, size_t bytes)
+			   void *val, size_t count)
 {
 	struct rockchip_otp *otp = context;
-	unsigned int addr_start, addr_end, addr_len;
-	int ret, i = 0;
-	u32 data;
-	u8 *buf;
-
-	addr_start = round_down(offset, RK3588_NBYTES) / RK3588_NBYTES;
-	addr_end = round_up(offset + bytes, RK3588_NBYTES) / RK3588_NBYTES;
-	addr_len = addr_end - addr_start;
-	addr_start += otp->data->read_offset / RK3588_NBYTES;
-
-	buf = kzalloc(array_size(addr_len, RK3588_NBYTES), GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
+	u32 *buf = val;
+	int ret;
 
-	while (addr_len--) {
-		writel((addr_start << RK3588_ADDR_SHIFT) |
+	while (count--) {
+		writel((offset++ << RK3588_ADDR_SHIFT) |
 		       (RK3588_BURST_NUM << RK3588_BURST_SHIFT),
 		       otp->base + RK3588_OTPC_AUTO_CTRL);
 		writel(RK3588_AUTO_EN, otp->base + RK3588_OTPC_AUTO_EN);
 
 		ret = rockchip_otp_wait_status(otp, RK3588_OTPC_INT_ST,
 					       RK3588_RD_DONE);
-		if (ret < 0) {
+		if (ret) {
 			dev_err(otp->dev, "timeout during read setup\n");
-			goto read_end;
+			return ret;
 		}
 
-		data = readl(otp->base + RK3588_OTPC_DOUT0);
-		memcpy(&buf[i], &data, RK3588_NBYTES);
-
-		i += RK3588_NBYTES;
-		addr_start++;
+		*buf++ = readl(otp->base + RK3588_OTPC_DOUT0);
 	}
 
-	memcpy(val, buf + offset % RK3588_NBYTES, bytes);
-
-read_end:
-	kfree(buf);
-
 	return ret;
 }
 
@@ -234,7 +267,7 @@ static int rockchip_otp_read(void *context, unsigned int offset,
 			     void *val, size_t bytes)
 {
 	struct rockchip_otp *otp = context;
-	int ret;
+	int ret, word_size;
 
 	if (!otp->data || !otp->data->reg_read)
 		return -EINVAL;
@@ -245,8 +278,34 @@ static int rockchip_otp_read(void *context, unsigned int offset,
 		return ret;
 	}
 
-	ret = otp->data->reg_read(context, offset, val, bytes);
+	offset += otp->data->read_offset;
+	word_size = otp->data->word_size;
+
+	if (word_size > 1) {
+		unsigned int addr_start, addr_end;
+		size_t count;
+		u8 *buf;
+
+		addr_start = offset / word_size;
+		addr_end = DIV_ROUND_UP(offset + bytes, word_size);
+		count = addr_end - addr_start;
+
+		buf = kzalloc(array_size(count, word_size), GFP_KERNEL);
+		if (!buf) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		ret = otp->data->reg_read(context, addr_start, buf, count);
+		if (!ret)
+			memcpy(val, buf + (offset % word_size), bytes);
+
+		kfree(buf);
+	} else {
+		ret = otp->data->reg_read(context, offset, val, bytes);
+	}
 
+err:
 	clk_bulk_disable_unprepare(otp->data->num_clks, otp->clks);
 
 	return ret;
@@ -259,7 +318,7 @@ static struct nvmem_config otp_config = {
 	.type = NVMEM_TYPE_OTP,
 	.read_only = true,
 	.stride = 1,
-	.word_size = 1,
+	.word_size = sizeof(u8),
 	.reg_read = rockchip_otp_read,
 };
 
@@ -274,9 +333,34 @@ static const struct rockchip_data px30_data = {
 	.reg_read = px30_otp_read,
 };
 
+static const char * const rk3528_otp_clocks[] = {
+	"otp", "apb_pclk", "sbpi",
+};
+
+static const struct rockchip_data rk3528_data = {
+	.size = 0x80,
+	.word_size = sizeof(u16),
+	.clks = rk3528_otp_clocks,
+	.num_clks = ARRAY_SIZE(rk3528_otp_clocks),
+	.reg_read = rk3568_otp_read,
+};
+
+static const char * const rk3568_otp_clocks[] = {
+	"otp", "apb_pclk", "phy", "sbpi",
+};
+
+static const struct rockchip_data rk3568_data = {
+	.size = 0x80,
+	.word_size = sizeof(u16),
+	.clks = rk3568_otp_clocks,
+	.num_clks = ARRAY_SIZE(rk3568_otp_clocks),
+	.reg_read = rk3568_otp_read,
+};
+
 static const struct rockchip_data rk3576_data = {
 	.size = 0x100,
 	.read_offset = 0x700,
+	.word_size = sizeof(u32),
 	.clks = px30_otp_clocks,
 	.num_clks = ARRAY_SIZE(px30_otp_clocks),
 	.reg_read = rk3588_otp_read,
@@ -289,6 +373,7 @@ static const char * const rk3588_otp_clocks[] = {
 static const struct rockchip_data rk3588_data = {
 	.size = 0x400,
 	.read_offset = 0xc00,
+	.word_size = sizeof(u32),
 	.clks = rk3588_otp_clocks,
 	.num_clks = ARRAY_SIZE(rk3588_otp_clocks),
 	.reg_read = rk3588_otp_read,
@@ -303,6 +388,18 @@ static const struct of_device_id rockchip_otp_match[] = {
 		.compatible = "rockchip,rk3308-otp",
 		.data = &px30_data,
 	},
+	{
+		.compatible = "rockchip,rk3528-otp",
+		.data = &rk3528_data,
+	},
+	{
+		.compatible = "rockchip,rk3562-otp",
+		.data = &rk3568_data,
+	},
+	{
+		.compatible = "rockchip,rk3568-otp",
+		.data = &rk3568_data,
+	},
 	{
 		.compatible = "rockchip,rk3576-otp",
 		.data = &rk3576_data,
diff --git a/drivers/nvmem/zynqmp_nvmem.c b/drivers/nvmem/zynqmp_nvmem.c
index 7da717d6c7faf..d297ff150dc02 100644
--- a/drivers/nvmem/zynqmp_nvmem.c
+++ b/drivers/nvmem/zynqmp_nvmem.c
@@ -66,7 +66,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset,
 	dma_addr_t dma_buf;
 	size_t words = bytes / WORD_INBYTES;
 	int ret;
-	int value;
+	unsigned int value;
 	char *data;
 
 	if (bytes % WORD_INBYTES != 0) {
@@ -80,7 +80,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset,
 	}
 
 	if (pufflag == 1 && flag == EFUSE_WRITE) {
-		memcpy(&value, val, bytes);
+		memcpy(&value, val, sizeof(value));
 		if ((offset == EFUSE_PUF_START_OFFSET ||
 		     offset == EFUSE_PUF_MID_OFFSET) &&
 		    value & P_USER_0_64_UPPER_MASK) {
@@ -100,7 +100,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset,
 	if (!efuse)
 		return -ENOMEM;
 
-	data = dma_alloc_coherent(dev, sizeof(bytes),
+	data = dma_alloc_coherent(dev, bytes,
 				  &dma_buf, GFP_KERNEL);
 	if (!data) {
 		ret = -ENOMEM;
@@ -134,7 +134,7 @@ static int zynqmp_efuse_access(void *context, unsigned int offset,
 	if (flag == EFUSE_READ)
 		memcpy(val, data, bytes);
 efuse_access_err:
-	dma_free_coherent(dev, sizeof(bytes),
+	dma_free_coherent(dev, bytes,
 			  data, dma_buf);
 efuse_data_fail:
 	dma_free_coherent(dev, sizeof(struct xilinx_efuse),
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 57420806c1a2b..49816bae936d6 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -435,13 +435,40 @@ bool of_machine_compatible_match(const char *const *compats)
 EXPORT_SYMBOL(of_machine_compatible_match);
 
 /**
- * of_machine_device_match - Test root of device tree against a of_device_id array
+ * of_machine_read_compatible - Get the compatible string of this machine
+ * @compatible: address at which the address of the compatible string will be
+ *              stored
+ * @index: index of the compatible entry in the list
+ *
+ * Returns:
+ * 0 on success, negative error number on failure.
+ */
+int of_machine_read_compatible(const char **compatible, unsigned int index)
+{
+	return of_property_read_string_index(of_root, "compatible", index, compatible);
+}
+EXPORT_SYMBOL_GPL(of_machine_read_compatible);
+
+/**
+ * of_machine_read_model - Get the model string of this machine
+ * @model: address at which the address of the model string will be stored
+ *
+ * Returns:
+ * 0 on success, negative error number on failure.
+ */
+int of_machine_read_model(const char **model)
+{
+	return of_property_read_string(of_root, "model", model);
+}
+EXPORT_SYMBOL_GPL(of_machine_read_model);
+
+/**
+ * of_machine_get_match - Test root of device tree against an of_device_id array
  * @matches:	NULL terminated array of of_device_id match structures to search in
  *
- * Returns true if the root node has any of the given compatible values in its
- * compatible property.
+ * Returns matched entry or NULL
  */
-bool of_machine_device_match(const struct of_device_id *matches)
+const struct of_device_id *of_machine_get_match(const struct of_device_id *matches)
 {
 	struct device_node *root;
 	const struct of_device_id *match = NULL;
@@ -452,9 +479,9 @@ bool of_machine_device_match(const struct of_device_id *matches)
 		of_node_put(root);
 	}
 
-	return match != NULL;
+	return match;
 }
-EXPORT_SYMBOL(of_machine_device_match);
+EXPORT_SYMBOL(of_machine_get_match);
 
 /**
  * of_machine_get_match_data - Tell if root of device tree has a matching of_match structure
@@ -465,15 +492,8 @@ EXPORT_SYMBOL(of_machine_device_match);
 const void *of_machine_get_match_data(const struct of_device_id *matches)
 {
 	const struct of_device_id *match;
-	struct device_node *root;
-
-	root = of_find_node_by_path("/");
-	if (!root)
-		return NULL;
-
-	match = of_match_node(matches, root);
-	of_node_put(root);
 
+	match = of_machine_get_match(matches);
 	if (!match)
 		return NULL;
 
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index 331646d667b9b..2967e4aff8070 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -866,6 +866,26 @@ static void __init early_init_dt_check_for_elfcorehdr(unsigned long node)
 		 elfcorehdr_addr, elfcorehdr_size);
 }
 
+static void __init early_init_dt_check_for_dmcryptkeys(unsigned long node)
+{
+	const char *prop_name = "linux,dmcryptkeys";
+	const __be32 *prop;
+
+	if (!IS_ENABLED(CONFIG_CRASH_DM_CRYPT))
+		return;
+
+	pr_debug("Looking for dmcryptkeys property... ");
+
+	prop = of_get_flat_dt_prop(node, prop_name, NULL);
+	if (!prop)
+		return;
+
+	dm_crypt_keys_addr = dt_mem_next_cell(dt_root_addr_cells, &prop);
+
+	/* Property only accessible to crash dump kernel */
+	fdt_delprop(initial_boot_params, node, prop_name);
+}
+
 static unsigned long chosen_node_offset = -FDT_ERR_NOTFOUND;
 
 /*
@@ -1097,6 +1117,7 @@ int __init early_init_dt_scan_chosen(char *cmdline)
 
 	early_init_dt_check_for_initrd(node);
 	early_init_dt_check_for_elfcorehdr(node);
+	early_init_dt_check_for_dmcryptkeys(node);
 
 	rng_seed = of_get_flat_dt_prop(node, "rng-seed", &l);
 	if (rng_seed && l > 0) {
diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c
index c4cf3552c0183..fbd253f0d3c5d 100644
--- a/drivers/of/kexec.c
+++ b/drivers/of/kexec.c
@@ -423,6 +423,25 @@ void *of_kexec_alloc_and_setup_fdt(const struct kimage *image,
 		if (ret)
 			goto out;
 
+		if (image->dm_crypt_keys_addr != 0) {
+			ret = fdt_appendprop_addrrange(fdt, 0, chosen_node,
+						       "linux,dmcryptkeys",
+						       image->dm_crypt_keys_addr,
+						       image->dm_crypt_keys_sz);
+
+			if (ret)
+				goto out;
+
+			/*
+			 * Avoid dmcryptkeys from being stomped on in kdump kernel by
+			 * setting up memory reserve map.
+			 */
+			ret = fdt_add_mem_rsv(fdt, image->dm_crypt_keys_addr,
+					      image->dm_crypt_keys_sz);
+			if (ret)
+				goto out;
+		}
+
 #ifdef CONFIG_CRASH_DUMP
 		/* add linux,usable-memory-range */
 		ret = fdt_appendprop_addrrange(fdt, 0, chosen_node,
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index ba591fbceb569..2a7111e8354d7 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -500,7 +500,7 @@ static const struct of_device_id reserved_mem_matches[] = {
 
 static int __init of_platform_default_populate_init(void)
 {
-	struct device_node *node;
+	struct device_node *node, *reserved;
 
 	device_links_supplier_sync_state_pause();
 
@@ -563,8 +563,14 @@ static int __init of_platform_default_populate_init(void)
 		 * platform_devices for every node in /reserved-memory with a
 		 * "compatible",
 		 */
-		for_each_matching_node(node, reserved_mem_matches)
-			of_platform_device_create(node, NULL, NULL);
+		reserved = of_find_node_by_path("/reserved-memory");
+		if (reserved) {
+			for_each_child_of_node(reserved, node) {
+				if (of_match_node(reserved_mem_matches, node))
+					of_platform_device_create(node, NULL, NULL);
+			}
+			of_node_put(reserved);
+		}
 
 		node = of_find_node_by_path("/firmware");
 		if (node) {
diff --git a/drivers/of/property.c b/drivers/of/property.c
index 50d95d512bf55..457e628ff9db4 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -88,7 +88,7 @@ EXPORT_SYMBOL(of_graph_is_present);
  * Search for a property in a device node and count the number of elements of
  * size elem_size in it.
  *
- * Return: The number of elements on sucess, -EINVAL if the property does not
+ * Return: The number of elements on success, -EINVAL if the property does not
  * exist or its length does not match a multiple of elem_size and -ENODATA if
  * the property does not have a value.
  */
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index 866641666e410..da3f5eba43419 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -2742,8 +2742,8 @@ struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table,
 					break;
 				}
 			}
-			break;
 		}
+		break;
 	}
 
 	if (IS_ERR(dest_opp)) {
diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c
index 8fc6238b17284..61506d30d5ff0 100644
--- a/drivers/opp/debugfs.c
+++ b/drivers/opp/debugfs.c
@@ -130,22 +130,24 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
 {
 	struct dentry *pdentry = opp_table->dentry;
 	struct dentry *d;
-	unsigned long id;
-	char name[25];	/* 20 chars for 64 bit value + 5 (opp:\0) */
+	char name[36];	/* "opp:"(4) + u64(20) + "-" (1) + u32(10) + NULL(1) */
 
 	/*
 	 * Get directory name for OPP.
 	 *
-	 * - Normally rate is unique to each OPP, use it to get unique opp-name.
+	 * - Normally rate is unique to each OPP, use it to get unique opp-name,
+	 *   together with performance level if available.
 	 * - For some devices rate isn't available or there are multiple, use
 	 *   index instead for them.
 	 */
-	if (likely(opp_table->clk_count == 1 && opp->rates[0]))
-		id = opp->rates[0];
-	else
-		id = _get_opp_count(opp_table);
-
-	snprintf(name, sizeof(name), "opp:%lu", id);
+	if (likely(opp_table->clk_count == 1 && opp->rates[0])) {
+		if (opp->level == OPP_LEVEL_UNSET)
+			snprintf(name, sizeof(name), "opp:%lu", opp->rates[0]);
+		else
+			snprintf(name, sizeof(name), "opp:%lu-%u", opp->rates[0], opp->level);
+	} else {
+		snprintf(name, sizeof(name), "opp:%u", _get_opp_count(opp_table));
+	}
 
 	/* Create per-opp directory */
 	d = debugfs_create_dir(name, pdentry);
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 5aaed8ac6e443..e72ac69343794 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -222,6 +222,7 @@ config PCIE_MEDIATEK_GEN3
 	depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST
 	depends on PCI_MSI
 	select IRQ_MSI_LIB
+	select PCI_PWRCTRL_SLOT
 	help
 	  Adds support for PCIe Gen3 MAC controller for MediaTek SoCs.
 	  This PCIe controller is compatible with Gen3, Gen2 and Gen1 speed,
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index d0aa031397fab..f2fde13107f2e 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -61,6 +61,17 @@ config PCI_MESON
 	  and therefore the driver re-uses the DesignWare core functions to
 	  implement the driver.
 
+config PCIE_ANDES_QILAI
+	tristate "Andes QiLai PCIe controller"
+	depends on ARCH_ANDES || COMPILE_TEST
+	depends on PCI_MSI
+	select PCIE_DW_HOST
+	help
+	  Say Y here to enable PCIe controller support on Andes QiLai SoCs,
+	  which operate in Root Complex mode. The Andes QiLai SoC PCIe
+	  controller is based on DesignWare IP and therefore the driver
+	  re-uses the DesignWare core functions to implement the driver.
+
 config PCIE_ARTPEC6
 	bool
 
@@ -84,14 +95,15 @@ config PCIE_ARTPEC6_EP
 	  Enables support for the PCIe controller in the ARTPEC-6 SoC to work in
 	  endpoint mode. This uses the DesignWare core.
 
-config PCIE_BT1
-	tristate "Baikal-T1 PCIe controller"
-	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+config PCIE_ESWIN
+	tristate "ESWIN PCIe controller"
+	depends on ARCH_ESWIN || COMPILE_TEST
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	help
-	  Enables support for the PCIe controller in the Baikal-T1 SoC to work
-	  in host mode. It's based on the Synopsys DWC PCIe v4.60a IP-core.
+	  Say Y here if you want PCIe controller support for the ESWIN SoCs.
+	  The PCIe controller in ESWIN SoCs is based on DesignWare hardware, and
+	  works only in host mode.
 
 config PCI_IMX6
 	bool
@@ -121,7 +133,7 @@ config PCI_IMX6_EP
 	  DesignWare core functions to implement the driver.
 
 config PCI_LAYERSCAPE
-	bool "Freescale Layerscape PCIe controller (host mode)"
+	tristate "Freescale Layerscape PCIe controller (host mode)"
 	depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST)
 	depends on PCI_MSI
 	select PCIE_DW_HOST
@@ -309,7 +321,7 @@ config PCIE_QCOM
 	select CRC8
 	select PCIE_QCOM_COMMON
 	select PCI_HOST_COMMON
-	select PCI_PWRCTRL_SLOT
+	select PCI_PWRCTRL_GENERIC
 	help
 	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
 	  PCIe controller uses the DesignWare core plus Qualcomm-specific
@@ -431,7 +443,7 @@ config PCIE_SPACEMIT_K1
 	depends on ARCH_SPACEMIT || COMPILE_TEST
 	depends on HAS_IOMEM
 	select PCIE_DW_HOST
-	select PCI_PWRCTRL_SLOT
+	select PCI_PWRCTRL_GENERIC
 	default ARCH_SPACEMIT
 	help
 	  Enables support for the DesignWare based PCIe controller in
diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile
index 67ba59c02038c..7177451db8aa1 100644
--- a/drivers/pci/controller/dwc/Makefile
+++ b/drivers/pci/controller/dwc/Makefile
@@ -5,7 +5,8 @@ obj-$(CONFIG_PCIE_DW_HOST) += pcie-designware-host.o
 obj-$(CONFIG_PCIE_DW_EP) += pcie-designware-ep.o
 obj-$(CONFIG_PCIE_DW_PLAT) += pcie-designware-plat.o
 obj-$(CONFIG_PCIE_AMD_MDB) += pcie-amd-mdb.o
-obj-$(CONFIG_PCIE_BT1) += pcie-bt1.o
+obj-$(CONFIG_PCIE_ANDES_QILAI) += pcie-andes-qilai.o
+obj-$(CONFIG_PCIE_ESWIN) += pcie-eswin.o
 obj-$(CONFIG_PCI_DRA7XX) += pci-dra7xx.o
 obj-$(CONFIG_PCI_EXYNOS) += pci-exynos.o
 obj-$(CONFIG_PCIE_FU740) += pcie-fu740.o
diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index d5d26229063f2..cd904659c3216 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -378,10 +378,6 @@ static void dra7xx_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);
-	enum pci_barno bar;
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
 
 	dra7xx_pcie_enable_wrapper_interrupts(dra7xx);
 }
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index a5b8d0b71677e..20dafd2710a39 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -1401,15 +1401,6 @@ static const struct dw_pcie_ops dw_pcie_ops = {
 	.stop_link = imx_pcie_stop_link,
 };
 
-static void imx_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	enum pci_barno bar;
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-
-	for (bar = BAR_0; bar <= BAR_5; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
 				  unsigned int type, u16 interrupt_num)
 {
@@ -1433,19 +1424,19 @@ static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
 static const struct pci_epc_features imx8m_pcie_epc_features = {
 	DWC_EPC_COMMON_FEATURES,
 	.msi_capable = true,
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
+	.bar[BAR_1] = { .type = BAR_DISABLED, },
+	.bar[BAR_3] = { .type = BAR_DISABLED, },
 	.bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = SZ_256, },
-	.bar[BAR_5] = { .type = BAR_RESERVED, },
+	.bar[BAR_5] = { .type = BAR_DISABLED, },
 	.align = SZ_64K,
 };
 
 static const struct pci_epc_features imx8q_pcie_epc_features = {
 	DWC_EPC_COMMON_FEATURES,
 	.msi_capable = true,
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
-	.bar[BAR_5] = { .type = BAR_RESERVED, },
+	.bar[BAR_1] = { .type = BAR_DISABLED, },
+	.bar[BAR_3] = { .type = BAR_DISABLED, },
+	.bar[BAR_5] = { .type = BAR_DISABLED, },
 	.align = SZ_64K,
 };
 
@@ -1478,7 +1469,6 @@ imx_pcie_ep_get_features(struct dw_pcie_ep *ep)
 }
 
 static const struct dw_pcie_ep_ops pcie_ep_ops = {
-	.init = imx_pcie_ep_init,
 	.raise_irq = imx_pcie_ep_raise_irq,
 	.get_features = imx_pcie_ep_get_features,
 };
@@ -1647,7 +1637,6 @@ static int imx_pcie_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct imx_pcie *imx_pcie;
-	struct device_node *np;
 	struct device_node *node = dev->of_node;
 	int i, ret, domain;
 	u16 val;
@@ -1674,7 +1663,8 @@ static int imx_pcie_probe(struct platform_device *pdev)
 		pci->pp.ops = &imx_pcie_host_dw_pme_ops;
 
 	/* Find the PHY if one is defined, only imx7d uses it */
-	np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
+	struct device_node *np __free(device_node) =
+		of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
 	if (np) {
 		struct resource res;
 
diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c
index 20fa4dadb82af..278d2dba1db08 100644
--- a/drivers/pci/controller/dwc/pci-keystone.c
+++ b/drivers/pci/controller/dwc/pci-keystone.c
@@ -933,6 +933,18 @@ static const struct pci_epc_features ks_pcie_am654_epc_features = {
 	DWC_EPC_COMMON_FEATURES,
 	.msi_capable = true,
 	.msix_capable = true,
+	/*
+	 * TODO: This driver is the only DWC glue driver that had BAR_RESERVED
+	 * BARs, but did not call dw_pcie_ep_reset_bar() for the reserved BARs.
+	 *
+	 * To not change the existing behavior, these BARs were not migrated to
+	 * BAR_DISABLED. If this driver wants the BAR_RESERVED BARs to be
+	 * disabled, it should migrate them to BAR_DISABLED.
+	 *
+	 * If they actually should be enabled, then the driver must also define
+	 * what is behind these reserved BARs, see the definition of struct
+	 * pci_epc_bar_rsvd_region.
+	 */
 	.bar[BAR_0] = { .type = BAR_RESERVED, },
 	.bar[BAR_1] = { .type = BAR_RESERVED, },
 	.bar[BAR_2] = { .type = BAR_RESIZABLE, },
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index a4a800699f898..8936975ff104e 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -152,15 +152,11 @@ static void ls_pcie_ep_init(struct dw_pcie_ep *ep)
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	struct dw_pcie_ep_func *ep_func;
-	enum pci_barno bar;
 
 	ep_func = dw_pcie_ep_get_func_from_ep(ep, 0);
 	if (!ep_func)
 		return;
 
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-
 	pcie->ls_epc->msi_capable = ep_func->msi_cap ? true : false;
 	pcie->ls_epc->msix_capable = ep_func->msix_cap ? true : false;
 }
@@ -251,9 +247,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	pci->ops = pcie->drvdata->dw_pcie_ops;
 
 	ls_epc->bar[BAR_2].only_64bit = true;
-	ls_epc->bar[BAR_3].type = BAR_RESERVED;
 	ls_epc->bar[BAR_4].only_64bit = true;
-	ls_epc->bar[BAR_5].type = BAR_RESERVED;
 	ls_epc->linkup_notifier = true;
 
 	pcie->pci = pci;
diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
index a44b5c256d6e2..14d6ac4fc53fd 100644
--- a/drivers/pci/controller/dwc/pci-layerscape.c
+++ b/drivers/pci/controller/dwc/pci-layerscape.c
@@ -13,6 +13,7 @@
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/iopoll.h>
+#include <linux/module.h>
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
 #include <linux/of_address.h>
@@ -403,8 +404,16 @@ static const struct dev_pm_ops ls_pcie_pm_ops = {
 	NOIRQ_SYSTEM_SLEEP_PM_OPS(ls_pcie_suspend_noirq, ls_pcie_resume_noirq)
 };
 
+static void ls_pcie_remove(struct platform_device *pdev)
+{
+	struct ls_pcie *pcie = platform_get_drvdata(pdev);
+
+	dw_pcie_host_deinit(&pcie->pci->pp);
+}
+
 static struct platform_driver ls_pcie_driver = {
 	.probe = ls_pcie_probe,
+	.remove = ls_pcie_remove,
 	.driver = {
 		.name = "layerscape-pcie",
 		.of_match_table = ls_pcie_of_match,
@@ -412,4 +421,9 @@ static struct platform_driver ls_pcie_driver = {
 		.pm = &ls_pcie_pm_ops,
 	},
 };
-builtin_platform_driver(ls_pcie_driver);
+module_platform_driver(ls_pcie_driver);
+
+MODULE_AUTHOR("Minghuan Lian <Minghuan.Lian@freescale.com>");
+MODULE_DESCRIPTION("Layerscape PCIe host controller driver");
+MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(of, ls_pcie_of_match);
diff --git a/drivers/pci/controller/dwc/pcie-amd-mdb.c b/drivers/pci/controller/dwc/pcie-amd-mdb.c
index 3c6e837465bb7..7e50e11fbffda 100644
--- a/drivers/pci/controller/dwc/pcie-amd-mdb.c
+++ b/drivers/pci/controller/dwc/pcie-amd-mdb.c
@@ -389,7 +389,7 @@ static int amd_mdb_setup_irq(struct amd_mdb_pcie *pcie,
 			       IRQF_NO_THREAD, NULL, pcie);
 	if (err) {
 		dev_err(dev, "Failed to request INTx IRQ %d, err=%d\n",
-			irq, err);
+			pcie->intx_irq, err);
 		return err;
 	}
 
diff --git a/drivers/pci/controller/dwc/pcie-andes-qilai.c b/drivers/pci/controller/dwc/pcie-andes-qilai.c
new file mode 100644
index 0000000000000..bd1588be44e06
--- /dev/null
+++ b/drivers/pci/controller/dwc/pcie-andes-qilai.c
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for the PCIe Controller in QiLai from Andes
+ *
+ * Copyright (C) 2026 Andes Technology Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+
+#include "pcie-designware.h"
+
+#define PCIE_INTR_CONTROL1			0x15c
+#define PCIE_MSI_CTRL_INT_EN			BIT(28)
+
+#define PCIE_LOGIC_COHERENCY_CONTROL3		0x8e8
+
+/*
+ * Refer to Table A4-5 (Memory type encoding) in the
+ * AMBA AXI and ACE Protocol Specification.
+ *
+ * The selected value corresponds to the Memory type field:
+ * "Write-back, Read and Write-allocate".
+ *
+ * The last three rows in the table A4-5 in
+ * AMBA AXI and ACE Protocol Specification:
+ * ARCACHE        AWCACHE        Memory type
+ * ------------------------------------------------------------------
+ * 1111 (0111)    0111           Write-back Read-allocate
+ * 1011           1111 (1011)    Write-back Write-allocate
+ * 1111           1111           Write-back Read and Write-allocate (selected)
+ */
+#define IOCP_ARCACHE				0b1111
+#define IOCP_AWCACHE				0b1111
+
+#define PCIE_CFG_MSTR_ARCACHE_MODE		GENMASK(6, 3)
+#define PCIE_CFG_MSTR_AWCACHE_MODE		GENMASK(14, 11)
+#define PCIE_CFG_MSTR_ARCACHE_VALUE		GENMASK(22, 19)
+#define PCIE_CFG_MSTR_AWCACHE_VALUE		GENMASK(30, 27)
+
+#define PCIE_GEN_CONTROL2			0x54
+#define PCIE_CFG_LTSSM_EN			BIT(0)
+
+#define PCIE_REGS_PCIE_SII_PM_STATE		0xc0
+#define SMLH_LINK_UP				BIT(6)
+#define RDLH_LINK_UP				BIT(7)
+
+struct qilai_pcie {
+	struct dw_pcie pci;
+	void __iomem *apb_base;
+};
+
+#define to_qilai_pcie(_pci) container_of(_pci, struct qilai_pcie, pci)
+
+static bool qilai_pcie_link_up(struct dw_pcie *pci)
+{
+	struct qilai_pcie *pcie = to_qilai_pcie(pci);
+	u32 val;
+
+	val = readl(pcie->apb_base + PCIE_REGS_PCIE_SII_PM_STATE);
+
+	return FIELD_GET(SMLH_LINK_UP, val) && FIELD_GET(RDLH_LINK_UP, val);
+}
+
+static int qilai_pcie_start_link(struct dw_pcie *pci)
+{
+	struct qilai_pcie *pcie = to_qilai_pcie(pci);
+	u32 val;
+
+	val = readl(pcie->apb_base + PCIE_GEN_CONTROL2);
+	val |= PCIE_CFG_LTSSM_EN;
+	writel(val, pcie->apb_base + PCIE_GEN_CONTROL2);
+
+	return 0;
+}
+
+static const struct dw_pcie_ops qilai_pcie_ops = {
+	.link_up = qilai_pcie_link_up,
+	.start_link = qilai_pcie_start_link,
+};
+
+/*
+ * Set up the QiLai PCIe IOCP (IO Coherence Port) Read/Write Behaviors to the
+ * Write-Back, Read and Write Allocate mode.
+ *
+ * The IOCP HW target is SoC last-level cache (L2 Cache), which serves as the
+ * system cache. The IOCP HW helps maintain cache monitoring, ensuring that
+ * the device can snoop data from/to the cache.
+ */
+static void qilai_pcie_iocp_cache_setup(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	u32 val;
+
+	dw_pcie_dbi_ro_wr_en(pci);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_LOGIC_COHERENCY_CONTROL3);
+	FIELD_MODIFY(PCIE_CFG_MSTR_ARCACHE_MODE, &val, IOCP_ARCACHE);
+	FIELD_MODIFY(PCIE_CFG_MSTR_AWCACHE_MODE, &val, IOCP_AWCACHE);
+	FIELD_MODIFY(PCIE_CFG_MSTR_ARCACHE_VALUE, &val, IOCP_ARCACHE);
+	FIELD_MODIFY(PCIE_CFG_MSTR_AWCACHE_VALUE, &val, IOCP_AWCACHE);
+	dw_pcie_writel_dbi(pci, PCIE_LOGIC_COHERENCY_CONTROL3, val);
+
+	dw_pcie_dbi_ro_wr_dis(pci);
+}
+
+static void qilai_pcie_enable_msi(struct qilai_pcie *pcie)
+{
+	u32 val;
+
+	val = readl(pcie->apb_base + PCIE_INTR_CONTROL1);
+	val |= PCIE_MSI_CTRL_INT_EN;
+	writel(val, pcie->apb_base + PCIE_INTR_CONTROL1);
+}
+
+static int qilai_pcie_host_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct qilai_pcie *pcie = to_qilai_pcie(pci);
+
+	qilai_pcie_enable_msi(pcie);
+
+	return 0;
+}
+
+static void qilai_pcie_host_post_init(struct dw_pcie_rp *pp)
+{
+	qilai_pcie_iocp_cache_setup(pp);
+}
+
+static const struct dw_pcie_host_ops qilai_pcie_host_ops = {
+	.init = qilai_pcie_host_init,
+	.post_init = qilai_pcie_host_post_init,
+};
+
+static int qilai_pcie_probe(struct platform_device *pdev)
+{
+	struct qilai_pcie *pcie;
+	struct dw_pcie *pci;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, pcie);
+
+	pci = &pcie->pci;
+	pcie->pci.dev = dev;
+	pcie->pci.ops = &qilai_pcie_ops;
+	pcie->pci.pp.ops = &qilai_pcie_host_ops;
+	pci->use_parent_dt_ranges = true;
+
+	dw_pcie_cap_set(&pcie->pci, REQ_RES);
+
+	pcie->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb");
+	if (IS_ERR(pcie->apb_base))
+		return PTR_ERR(pcie->apb_base);
+
+	pm_runtime_set_active(dev);
+	pm_runtime_no_callbacks(dev);
+	devm_pm_runtime_enable(dev);
+
+	ret = dw_pcie_host_init(&pcie->pci.pp);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to initialize PCIe host\n");
+
+	return 0;
+}
+
+static const struct of_device_id qilai_pcie_of_match[] = {
+	{ .compatible = "andestech,qilai-pcie" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, qilai_pcie_of_match);
+
+static struct platform_driver qilai_pcie_driver = {
+	.probe = qilai_pcie_probe,
+	.driver = {
+		.name	= "qilai-pcie",
+		.of_match_table = qilai_pcie_of_match,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+};
+
+builtin_platform_driver(qilai_pcie_driver);
+
+MODULE_AUTHOR("Randolph Lin <randolph@andestech.com>");
+MODULE_DESCRIPTION("Andes QiLai PCIe driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c
index e994b75986c34..55cb957ae1f3b 100644
--- a/drivers/pci/controller/dwc/pcie-artpec6.c
+++ b/drivers/pci/controller/dwc/pcie-artpec6.c
@@ -340,15 +340,11 @@ static void artpec6_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci);
-	enum pci_barno bar;
 
 	artpec6_pcie_assert_core_reset(artpec6_pcie);
 	artpec6_pcie_init_phy(artpec6_pcie);
 	artpec6_pcie_deassert_core_reset(artpec6_pcie);
 	artpec6_pcie_wait_for_phy(artpec6_pcie);
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
 }
 
 static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
diff --git a/drivers/pci/controller/dwc/pcie-bt1.c b/drivers/pci/controller/dwc/pcie-bt1.c
deleted file mode 100644
index 1340edc18d128..0000000000000
--- a/drivers/pci/controller/dwc/pcie-bt1.c
+++ /dev/null
@@ -1,645 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2021 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Vadim Vlasov <Vadim.Vlasov@baikalelectronics.ru>
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- * Baikal-T1 PCIe controller driver
- */
-
-#include <linux/bitfield.h>
-#include <linux/bits.h>
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
-#include <linux/kernel.h>
-#include <linux/mfd/syscon.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/platform_device.h>
-#include <linux/regmap.h>
-#include <linux/reset.h>
-#include <linux/types.h>
-
-#include "pcie-designware.h"
-
-/* Baikal-T1 System CCU control registers */
-#define BT1_CCU_PCIE_CLKC			0x140
-#define BT1_CCU_PCIE_REQ_PCS_CLK		BIT(16)
-#define BT1_CCU_PCIE_REQ_MAC_CLK		BIT(17)
-#define BT1_CCU_PCIE_REQ_PIPE_CLK		BIT(18)
-
-#define BT1_CCU_PCIE_RSTC			0x144
-#define BT1_CCU_PCIE_REQ_LINK_RST		BIT(13)
-#define BT1_CCU_PCIE_REQ_SMLH_RST		BIT(14)
-#define BT1_CCU_PCIE_REQ_PHY_RST		BIT(16)
-#define BT1_CCU_PCIE_REQ_CORE_RST		BIT(24)
-#define BT1_CCU_PCIE_REQ_STICKY_RST		BIT(26)
-#define BT1_CCU_PCIE_REQ_NSTICKY_RST		BIT(27)
-
-#define BT1_CCU_PCIE_PMSC			0x148
-#define BT1_CCU_PCIE_LTSSM_STATE_MASK		GENMASK(5, 0)
-#define BT1_CCU_PCIE_LTSSM_DET_QUIET		0x00
-#define BT1_CCU_PCIE_LTSSM_DET_ACT		0x01
-#define BT1_CCU_PCIE_LTSSM_POLL_ACT		0x02
-#define BT1_CCU_PCIE_LTSSM_POLL_COMP		0x03
-#define BT1_CCU_PCIE_LTSSM_POLL_CONF		0x04
-#define BT1_CCU_PCIE_LTSSM_PRE_DET_QUIET	0x05
-#define BT1_CCU_PCIE_LTSSM_DET_WAIT		0x06
-#define BT1_CCU_PCIE_LTSSM_CFG_LNKWD_START	0x07
-#define BT1_CCU_PCIE_LTSSM_CFG_LNKWD_ACEPT	0x08
-#define BT1_CCU_PCIE_LTSSM_CFG_LNNUM_WAIT	0x09
-#define BT1_CCU_PCIE_LTSSM_CFG_LNNUM_ACEPT	0x0a
-#define BT1_CCU_PCIE_LTSSM_CFG_COMPLETE		0x0b
-#define BT1_CCU_PCIE_LTSSM_CFG_IDLE		0x0c
-#define BT1_CCU_PCIE_LTSSM_RCVR_LOCK		0x0d
-#define BT1_CCU_PCIE_LTSSM_RCVR_SPEED		0x0e
-#define BT1_CCU_PCIE_LTSSM_RCVR_RCVRCFG		0x0f
-#define BT1_CCU_PCIE_LTSSM_RCVR_IDLE		0x10
-#define BT1_CCU_PCIE_LTSSM_L0			0x11
-#define BT1_CCU_PCIE_LTSSM_L0S			0x12
-#define BT1_CCU_PCIE_LTSSM_L123_SEND_IDLE	0x13
-#define BT1_CCU_PCIE_LTSSM_L1_IDLE		0x14
-#define BT1_CCU_PCIE_LTSSM_L2_IDLE		0x15
-#define BT1_CCU_PCIE_LTSSM_L2_WAKE		0x16
-#define BT1_CCU_PCIE_LTSSM_DIS_ENTRY		0x17
-#define BT1_CCU_PCIE_LTSSM_DIS_IDLE		0x18
-#define BT1_CCU_PCIE_LTSSM_DISABLE		0x19
-#define BT1_CCU_PCIE_LTSSM_LPBK_ENTRY		0x1a
-#define BT1_CCU_PCIE_LTSSM_LPBK_ACTIVE		0x1b
-#define BT1_CCU_PCIE_LTSSM_LPBK_EXIT		0x1c
-#define BT1_CCU_PCIE_LTSSM_LPBK_EXIT_TOUT	0x1d
-#define BT1_CCU_PCIE_LTSSM_HOT_RST_ENTRY	0x1e
-#define BT1_CCU_PCIE_LTSSM_HOT_RST		0x1f
-#define BT1_CCU_PCIE_LTSSM_RCVR_EQ0		0x20
-#define BT1_CCU_PCIE_LTSSM_RCVR_EQ1		0x21
-#define BT1_CCU_PCIE_LTSSM_RCVR_EQ2		0x22
-#define BT1_CCU_PCIE_LTSSM_RCVR_EQ3		0x23
-#define BT1_CCU_PCIE_SMLH_LINKUP		BIT(6)
-#define BT1_CCU_PCIE_RDLH_LINKUP		BIT(7)
-#define BT1_CCU_PCIE_PM_LINKSTATE_L0S		BIT(8)
-#define BT1_CCU_PCIE_PM_LINKSTATE_L1		BIT(9)
-#define BT1_CCU_PCIE_PM_LINKSTATE_L2		BIT(10)
-#define BT1_CCU_PCIE_L1_PENDING			BIT(12)
-#define BT1_CCU_PCIE_REQ_EXIT_L1		BIT(14)
-#define BT1_CCU_PCIE_LTSSM_RCVR_EQ		BIT(15)
-#define BT1_CCU_PCIE_PM_DSTAT_MASK		GENMASK(18, 16)
-#define BT1_CCU_PCIE_PM_PME_EN			BIT(20)
-#define BT1_CCU_PCIE_PM_PME_STATUS		BIT(21)
-#define BT1_CCU_PCIE_AUX_PM_EN			BIT(22)
-#define BT1_CCU_PCIE_AUX_PWR_DET		BIT(23)
-#define BT1_CCU_PCIE_WAKE_DET			BIT(24)
-#define BT1_CCU_PCIE_TURNOFF_REQ		BIT(30)
-#define BT1_CCU_PCIE_TURNOFF_ACK		BIT(31)
-
-#define BT1_CCU_PCIE_GENC			0x14c
-#define BT1_CCU_PCIE_LTSSM_EN			BIT(1)
-#define BT1_CCU_PCIE_DBI2_MODE			BIT(2)
-#define BT1_CCU_PCIE_MGMT_EN			BIT(3)
-#define BT1_CCU_PCIE_RXLANE_FLIP_EN		BIT(16)
-#define BT1_CCU_PCIE_TXLANE_FLIP_EN		BIT(17)
-#define BT1_CCU_PCIE_SLV_XFER_PEND		BIT(24)
-#define BT1_CCU_PCIE_RCV_XFER_PEND		BIT(25)
-#define BT1_CCU_PCIE_DBI_XFER_PEND		BIT(26)
-#define BT1_CCU_PCIE_DMA_XFER_PEND		BIT(27)
-
-#define BT1_CCU_PCIE_LTSSM_LINKUP(_pmsc) \
-({ \
-	int __state = FIELD_GET(BT1_CCU_PCIE_LTSSM_STATE_MASK, _pmsc); \
-	__state >= BT1_CCU_PCIE_LTSSM_L0 && __state <= BT1_CCU_PCIE_LTSSM_L2_WAKE; \
-})
-
-/* Baikal-T1 PCIe specific control registers */
-#define BT1_PCIE_AXI2MGM_LANENUM		0xd04
-#define BT1_PCIE_AXI2MGM_LANESEL_MASK		GENMASK(3, 0)
-
-#define BT1_PCIE_AXI2MGM_ADDRCTL		0xd08
-#define BT1_PCIE_AXI2MGM_PHYREG_ADDR_MASK	GENMASK(20, 0)
-#define BT1_PCIE_AXI2MGM_READ_FLAG		BIT(29)
-#define BT1_PCIE_AXI2MGM_DONE			BIT(30)
-#define BT1_PCIE_AXI2MGM_BUSY			BIT(31)
-
-#define BT1_PCIE_AXI2MGM_WRITEDATA		0xd0c
-#define BT1_PCIE_AXI2MGM_WDATA			GENMASK(15, 0)
-
-#define BT1_PCIE_AXI2MGM_READDATA		0xd10
-#define BT1_PCIE_AXI2MGM_RDATA			GENMASK(15, 0)
-
-/* Generic Baikal-T1 PCIe interface resources */
-#define BT1_PCIE_NUM_APP_CLKS			ARRAY_SIZE(bt1_pcie_app_clks)
-#define BT1_PCIE_NUM_CORE_CLKS			ARRAY_SIZE(bt1_pcie_core_clks)
-#define BT1_PCIE_NUM_APP_RSTS			ARRAY_SIZE(bt1_pcie_app_rsts)
-#define BT1_PCIE_NUM_CORE_RSTS			ARRAY_SIZE(bt1_pcie_core_rsts)
-
-/* PCIe bus setup delays and timeouts */
-#define BT1_PCIE_RST_DELAY_MS			100
-#define BT1_PCIE_RUN_DELAY_US			100
-#define BT1_PCIE_REQ_DELAY_US			1
-#define BT1_PCIE_REQ_TIMEOUT_US			1000
-#define BT1_PCIE_LNK_DELAY_US			1000
-#define BT1_PCIE_LNK_TIMEOUT_US			1000000
-
-static const enum dw_pcie_app_clk bt1_pcie_app_clks[] = {
-	DW_PCIE_DBI_CLK, DW_PCIE_MSTR_CLK, DW_PCIE_SLV_CLK,
-};
-
-static const enum dw_pcie_core_clk bt1_pcie_core_clks[] = {
-	DW_PCIE_REF_CLK,
-};
-
-static const enum dw_pcie_app_rst bt1_pcie_app_rsts[] = {
-	DW_PCIE_MSTR_RST, DW_PCIE_SLV_RST,
-};
-
-static const enum dw_pcie_core_rst bt1_pcie_core_rsts[] = {
-	DW_PCIE_NON_STICKY_RST, DW_PCIE_STICKY_RST, DW_PCIE_CORE_RST,
-	DW_PCIE_PIPE_RST, DW_PCIE_PHY_RST, DW_PCIE_HOT_RST, DW_PCIE_PWR_RST,
-};
-
-struct bt1_pcie {
-	struct dw_pcie dw;
-	struct platform_device *pdev;
-	struct regmap *sys_regs;
-};
-#define to_bt1_pcie(_dw) container_of(_dw, struct bt1_pcie, dw)
-
-/*
- * Baikal-T1 MMIO space must be read/written by the dword-aligned
- * instructions. Note the methods are optimized to have the dword operations
- * performed with minimum overhead as the most frequently used ones.
- */
-static int bt1_pcie_read_mmio(void __iomem *addr, int size, u32 *val)
-{
-	unsigned int ofs = (uintptr_t)addr & 0x3;
-
-	if (!IS_ALIGNED((uintptr_t)addr, size))
-		return -EINVAL;
-
-	*val = readl(addr - ofs) >> ofs * BITS_PER_BYTE;
-	if (size == 4) {
-		return 0;
-	} else if (size == 2) {
-		*val &= 0xffff;
-		return 0;
-	} else if (size == 1) {
-		*val &= 0xff;
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static int bt1_pcie_write_mmio(void __iomem *addr, int size, u32 val)
-{
-	unsigned int ofs = (uintptr_t)addr & 0x3;
-	u32 tmp, mask;
-
-	if (!IS_ALIGNED((uintptr_t)addr, size))
-		return -EINVAL;
-
-	if (size == 4) {
-		writel(val, addr);
-		return 0;
-	} else if (size == 2 || size == 1) {
-		mask = GENMASK(size * BITS_PER_BYTE - 1, 0);
-		tmp = readl(addr - ofs) & ~(mask << ofs * BITS_PER_BYTE);
-		tmp |= (val & mask) << ofs * BITS_PER_BYTE;
-		writel(tmp, addr - ofs);
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static u32 bt1_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
-			     size_t size)
-{
-	int ret;
-	u32 val;
-
-	ret = bt1_pcie_read_mmio(base + reg, size, &val);
-	if (ret) {
-		dev_err(pci->dev, "Read DBI address failed\n");
-		return ~0U;
-	}
-
-	return val;
-}
-
-static void bt1_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
-			       size_t size, u32 val)
-{
-	int ret;
-
-	ret = bt1_pcie_write_mmio(base + reg, size, val);
-	if (ret)
-		dev_err(pci->dev, "Write DBI address failed\n");
-}
-
-static void bt1_pcie_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
-				size_t size, u32 val)
-{
-	struct bt1_pcie *btpci = to_bt1_pcie(pci);
-	int ret;
-
-	regmap_update_bits(btpci->sys_regs, BT1_CCU_PCIE_GENC,
-			   BT1_CCU_PCIE_DBI2_MODE, BT1_CCU_PCIE_DBI2_MODE);
-
-	ret = bt1_pcie_write_mmio(base + reg, size, val);
-	if (ret)
-		dev_err(pci->dev, "Write DBI2 address failed\n");
-
-	regmap_update_bits(btpci->sys_regs, BT1_CCU_PCIE_GENC,
-			   BT1_CCU_PCIE_DBI2_MODE, 0);
-}
-
-static int bt1_pcie_start_link(struct dw_pcie *pci)
-{
-	struct bt1_pcie *btpci = to_bt1_pcie(pci);
-	u32 val;
-	int ret;
-
-	/*
-	 * Enable LTSSM and make sure it was able to establish both PHY and
-	 * data links. This procedure shall work fine to reach 2.5 GT/s speed.
-	 */
-	regmap_update_bits(btpci->sys_regs, BT1_CCU_PCIE_GENC,
-			   BT1_CCU_PCIE_LTSSM_EN, BT1_CCU_PCIE_LTSSM_EN);
-
-	ret = regmap_read_poll_timeout(btpci->sys_regs, BT1_CCU_PCIE_PMSC, val,
-				       (val & BT1_CCU_PCIE_SMLH_LINKUP),
-				       BT1_PCIE_LNK_DELAY_US, BT1_PCIE_LNK_TIMEOUT_US);
-	if (ret) {
-		dev_err(pci->dev, "LTSSM failed to set PHY link up\n");
-		return ret;
-	}
-
-	ret = regmap_read_poll_timeout(btpci->sys_regs, BT1_CCU_PCIE_PMSC, val,
-				       (val & BT1_CCU_PCIE_RDLH_LINKUP),
-				       BT1_PCIE_LNK_DELAY_US, BT1_PCIE_LNK_TIMEOUT_US);
-	if (ret) {
-		dev_err(pci->dev, "LTSSM failed to set data link up\n");
-		return ret;
-	}
-
-	/*
-	 * Activate direct speed change after the link is established in an
-	 * attempt to reach a higher bus performance (up to Gen.3 - 8.0 GT/s).
-	 * This is required at least to get 8.0 GT/s speed.
-	 */
-	val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
-	val |= PORT_LOGIC_SPEED_CHANGE;
-	dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
-
-	ret = regmap_read_poll_timeout(btpci->sys_regs, BT1_CCU_PCIE_PMSC, val,
-				       BT1_CCU_PCIE_LTSSM_LINKUP(val),
-				       BT1_PCIE_LNK_DELAY_US, BT1_PCIE_LNK_TIMEOUT_US);
-	if (ret)
-		dev_err(pci->dev, "LTSSM failed to get into L0 state\n");
-
-	return ret;
-}
-
-static void bt1_pcie_stop_link(struct dw_pcie *pci)
-{
-	struct bt1_pcie *btpci = to_bt1_pcie(pci);
-
-	regmap_update_bits(btpci->sys_regs, BT1_CCU_PCIE_GENC,
-			   BT1_CCU_PCIE_LTSSM_EN, 0);
-}
-
-static const struct dw_pcie_ops bt1_pcie_ops = {
-	.read_dbi = bt1_pcie_read_dbi,
-	.write_dbi = bt1_pcie_write_dbi,
-	.write_dbi2 = bt1_pcie_write_dbi2,
-	.start_link = bt1_pcie_start_link,
-	.stop_link = bt1_pcie_stop_link,
-};
-
-static struct pci_ops bt1_pci_ops = {
-	.map_bus = dw_pcie_own_conf_map_bus,
-	.read = pci_generic_config_read32,
-	.write = pci_generic_config_write32,
-};
-
-static int bt1_pcie_get_resources(struct bt1_pcie *btpci)
-{
-	struct device *dev = btpci->dw.dev;
-	int i;
-
-	/* DBI access is supposed to be performed by the dword-aligned IOs */
-	btpci->dw.pp.bridge->ops = &bt1_pci_ops;
-
-	/* These CSRs are in MMIO so we won't check the regmap-methods status */
-	btpci->sys_regs =
-		syscon_regmap_lookup_by_phandle(dev->of_node, "baikal,bt1-syscon");
-	if (IS_ERR(btpci->sys_regs))
-		return dev_err_probe(dev, PTR_ERR(btpci->sys_regs),
-				     "Failed to get syscon\n");
-
-	/* Make sure all the required resources have been specified */
-	for (i = 0; i < BT1_PCIE_NUM_APP_CLKS; i++) {
-		if (!btpci->dw.app_clks[bt1_pcie_app_clks[i]].clk) {
-			dev_err(dev, "App clocks set is incomplete\n");
-			return -ENOENT;
-		}
-	}
-
-	for (i = 0; i < BT1_PCIE_NUM_CORE_CLKS; i++) {
-		if (!btpci->dw.core_clks[bt1_pcie_core_clks[i]].clk) {
-			dev_err(dev, "Core clocks set is incomplete\n");
-			return -ENOENT;
-		}
-	}
-
-	for (i = 0; i < BT1_PCIE_NUM_APP_RSTS; i++) {
-		if (!btpci->dw.app_rsts[bt1_pcie_app_rsts[i]].rstc) {
-			dev_err(dev, "App resets set is incomplete\n");
-			return -ENOENT;
-		}
-	}
-
-	for (i = 0; i < BT1_PCIE_NUM_CORE_RSTS; i++) {
-		if (!btpci->dw.core_rsts[bt1_pcie_core_rsts[i]].rstc) {
-			dev_err(dev, "Core resets set is incomplete\n");
-			return -ENOENT;
-		}
-	}
-
-	return 0;
-}
-
-static void bt1_pcie_full_stop_bus(struct bt1_pcie *btpci, bool init)
-{
-	struct device *dev = btpci->dw.dev;
-	struct dw_pcie *pci = &btpci->dw;
-	int ret;
-
-	/* Disable LTSSM for sure */
-	regmap_update_bits(btpci->sys_regs, BT1_CCU_PCIE_GENC,
-			   BT1_CCU_PCIE_LTSSM_EN, 0);
-
-	/*
-	 * Application reset controls are trigger-based so assert the core
-	 * resets only.
-	 */
-	ret = reset_control_bulk_assert(DW_PCIE_NUM_CORE_RSTS, pci->core_rsts);
-	if (ret)
-		dev_err(dev, "Failed to assert core resets\n");
-
-	/*
-	 * Clocks are disabled by default at least in accordance with the clk
-	 * enable counter value on init stage.
-	 */
-	if (!init) {
-		clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, pci->core_clks);
-
-		clk_bulk_disable_unprepare(DW_PCIE_NUM_APP_CLKS, pci->app_clks);
-	}
-
-	/* The peripheral devices are unavailable anyway so reset them too */
-	gpiod_set_value_cansleep(pci->pe_rst, 1);
-
-	/* Make sure all the resets are settled */
-	msleep(BT1_PCIE_RST_DELAY_MS);
-}
-
-/*
- * Implements the cold reset procedure in accordance with the reference manual
- * and available PM signals.
- */
-static int bt1_pcie_cold_start_bus(struct bt1_pcie *btpci)
-{
-	struct device *dev = btpci->dw.dev;
-	struct dw_pcie *pci = &btpci->dw;
-	u32 val;
-	int ret;
-
-	/* First get out of the Power/Hot reset state */
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_PWR_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert PHY reset\n");
-		return ret;
-	}
-
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_HOT_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert hot reset\n");
-		goto err_assert_pwr_rst;
-	}
-
-	/* Wait for the PM-core to stop requesting the PHY reset */
-	ret = regmap_read_poll_timeout(btpci->sys_regs, BT1_CCU_PCIE_RSTC, val,
-				       !(val & BT1_CCU_PCIE_REQ_PHY_RST),
-				       BT1_PCIE_REQ_DELAY_US, BT1_PCIE_REQ_TIMEOUT_US);
-	if (ret) {
-		dev_err(dev, "Timed out waiting for PM to stop PHY resetting\n");
-		goto err_assert_hot_rst;
-	}
-
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_PHY_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert PHY reset\n");
-		goto err_assert_hot_rst;
-	}
-
-	/* Clocks can be now enabled, but the ref one is crucial at this stage */
-	ret = clk_bulk_prepare_enable(DW_PCIE_NUM_APP_CLKS, pci->app_clks);
-	if (ret) {
-		dev_err(dev, "Failed to enable app clocks\n");
-		goto err_assert_phy_rst;
-	}
-
-	ret = clk_bulk_prepare_enable(DW_PCIE_NUM_CORE_CLKS, pci->core_clks);
-	if (ret) {
-		dev_err(dev, "Failed to enable ref clocks\n");
-		goto err_disable_app_clk;
-	}
-
-	/* Wait for the PM to stop requesting the controller core reset */
-	ret = regmap_read_poll_timeout(btpci->sys_regs, BT1_CCU_PCIE_RSTC, val,
-				       !(val & BT1_CCU_PCIE_REQ_CORE_RST),
-				       BT1_PCIE_REQ_DELAY_US, BT1_PCIE_REQ_TIMEOUT_US);
-	if (ret) {
-		dev_err(dev, "Timed out waiting for PM to stop core resetting\n");
-		goto err_disable_core_clk;
-	}
-
-	/* PCS-PIPE interface and controller core can be now activated */
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_PIPE_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert PIPE reset\n");
-		goto err_disable_core_clk;
-	}
-
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_CORE_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert core reset\n");
-		goto err_assert_pipe_rst;
-	}
-
-	/* It's recommended to reset the core and application logic together */
-	ret = reset_control_bulk_reset(DW_PCIE_NUM_APP_RSTS, pci->app_rsts);
-	if (ret) {
-		dev_err(dev, "Failed to reset app domain\n");
-		goto err_assert_core_rst;
-	}
-
-	/* Sticky/Non-sticky CSR flags can be now unreset too */
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_STICKY_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert sticky reset\n");
-		goto err_assert_core_rst;
-	}
-
-	ret = reset_control_deassert(pci->core_rsts[DW_PCIE_NON_STICKY_RST].rstc);
-	if (ret) {
-		dev_err(dev, "Failed to deassert non-sticky reset\n");
-		goto err_assert_sticky_rst;
-	}
-
-	/* Activate the PCIe bus peripheral devices */
-	gpiod_set_value_cansleep(pci->pe_rst, 0);
-
-	/* Make sure the state is settled (LTSSM is still disabled though) */
-	usleep_range(BT1_PCIE_RUN_DELAY_US, BT1_PCIE_RUN_DELAY_US + 100);
-
-	return 0;
-
-err_assert_sticky_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_STICKY_RST].rstc);
-
-err_assert_core_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_CORE_RST].rstc);
-
-err_assert_pipe_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_PIPE_RST].rstc);
-
-err_disable_core_clk:
-	clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, pci->core_clks);
-
-err_disable_app_clk:
-	clk_bulk_disable_unprepare(DW_PCIE_NUM_APP_CLKS, pci->app_clks);
-
-err_assert_phy_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_PHY_RST].rstc);
-
-err_assert_hot_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_HOT_RST].rstc);
-
-err_assert_pwr_rst:
-	reset_control_assert(pci->core_rsts[DW_PCIE_PWR_RST].rstc);
-
-	return ret;
-}
-
-static int bt1_pcie_host_init(struct dw_pcie_rp *pp)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct bt1_pcie *btpci = to_bt1_pcie(pci);
-	int ret;
-
-	ret = bt1_pcie_get_resources(btpci);
-	if (ret)
-		return ret;
-
-	bt1_pcie_full_stop_bus(btpci, true);
-
-	return bt1_pcie_cold_start_bus(btpci);
-}
-
-static void bt1_pcie_host_deinit(struct dw_pcie_rp *pp)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct bt1_pcie *btpci = to_bt1_pcie(pci);
-
-	bt1_pcie_full_stop_bus(btpci, false);
-}
-
-static const struct dw_pcie_host_ops bt1_pcie_host_ops = {
-	.init = bt1_pcie_host_init,
-	.deinit = bt1_pcie_host_deinit,
-};
-
-static struct bt1_pcie *bt1_pcie_create_data(struct platform_device *pdev)
-{
-	struct bt1_pcie *btpci;
-
-	btpci = devm_kzalloc(&pdev->dev, sizeof(*btpci), GFP_KERNEL);
-	if (!btpci)
-		return ERR_PTR(-ENOMEM);
-
-	btpci->pdev = pdev;
-
-	platform_set_drvdata(pdev, btpci);
-
-	return btpci;
-}
-
-static int bt1_pcie_add_port(struct bt1_pcie *btpci)
-{
-	struct device *dev = &btpci->pdev->dev;
-	int ret;
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
-	if (ret)
-		return ret;
-
-	btpci->dw.version = DW_PCIE_VER_460A;
-	btpci->dw.dev = dev;
-	btpci->dw.ops = &bt1_pcie_ops;
-
-	btpci->dw.pp.num_vectors = MAX_MSI_IRQS;
-	btpci->dw.pp.ops = &bt1_pcie_host_ops;
-
-	dw_pcie_cap_set(&btpci->dw, REQ_RES);
-
-	ret = dw_pcie_host_init(&btpci->dw.pp);
-
-	return dev_err_probe(dev, ret, "Failed to initialize DWC PCIe host\n");
-}
-
-static void bt1_pcie_del_port(struct bt1_pcie *btpci)
-{
-	dw_pcie_host_deinit(&btpci->dw.pp);
-}
-
-static int bt1_pcie_probe(struct platform_device *pdev)
-{
-	struct bt1_pcie *btpci;
-
-	btpci = bt1_pcie_create_data(pdev);
-	if (IS_ERR(btpci))
-		return PTR_ERR(btpci);
-
-	return bt1_pcie_add_port(btpci);
-}
-
-static void bt1_pcie_remove(struct platform_device *pdev)
-{
-	struct bt1_pcie *btpci = platform_get_drvdata(pdev);
-
-	bt1_pcie_del_port(btpci);
-}
-
-static const struct of_device_id bt1_pcie_of_match[] = {
-	{ .compatible = "baikal,bt1-pcie" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, bt1_pcie_of_match);
-
-static struct platform_driver bt1_pcie_driver = {
-	.probe = bt1_pcie_probe,
-	.remove = bt1_pcie_remove,
-	.driver = {
-		.name	= "bt1-pcie",
-		.of_match_table = bt1_pcie_of_match,
-	},
-};
-module_platform_driver(bt1_pcie_driver);
-
-MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
-MODULE_DESCRIPTION("Baikal-T1 PCIe driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index c57ae4d6c5c0e..d4dc3b24da607 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -754,7 +754,7 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 	val = dw_pcie_ep_readw_dbi(ep, func_no, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= nr_irqs - 1; /* encoded as N-1 */
-	dw_pcie_writew_dbi(pci, reg, val);
+	dw_pcie_ep_writew_dbi(ep, func_no, reg, val);
 
 	reg = ep_func->msix_cap + PCI_MSIX_TABLE;
 	val = offset | bir;
@@ -1110,7 +1110,8 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
 {
 	struct dw_pcie_ep *ep = &pci->ep;
 	u8 funcs = ep->epc->max_functions;
-	u8 func_no;
+	u32 func0_lnkcap, lnkcap;
+	u8 func_no, offset;
 
 	dw_pcie_dbi_ro_wr_en(pci);
 
@@ -1118,9 +1119,57 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
 		dw_pcie_ep_init_rebar_registers(ep, func_no);
 
 	dw_pcie_setup(pci);
+
+	/*
+	 * PCIe r7.0, section 7.5.3.6 states that for multi-function
+	 * endpoints, max link width and speed fields must report same
+	 * values for all functions. However, dw_pcie_setup() programs
+	 * these fields only for function 0. Hence, mirror these fields
+	 * to all other functions as well.
+	 */
+	if (funcs > 1) {
+		offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+		func0_lnkcap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
+		func0_lnkcap = FIELD_GET(PCI_EXP_LNKCAP_MLW |
+					 PCI_EXP_LNKCAP_SLS, func0_lnkcap);
+
+		for (func_no = 1; func_no < funcs; func_no++) {
+			offset = dw_pcie_ep_find_capability(ep, func_no,
+							    PCI_CAP_ID_EXP);
+			lnkcap = dw_pcie_ep_readl_dbi(ep, func_no,
+						      offset + PCI_EXP_LNKCAP);
+			FIELD_MODIFY(PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS,
+				     &lnkcap, func0_lnkcap);
+			dw_pcie_ep_writel_dbi(ep, func_no,
+					      offset + PCI_EXP_LNKCAP, lnkcap);
+		}
+	}
+
 	dw_pcie_dbi_ro_wr_dis(pci);
 }
 
+static void dw_pcie_ep_disable_bars(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	enum pci_epc_bar_type bar_type;
+	enum pci_barno bar;
+
+	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
+		bar_type = dw_pcie_ep_get_bar_type(ep, bar);
+
+		/*
+		 * Reserved BARs should not get disabled by default. All other
+		 * BAR types are disabled by default.
+		 *
+		 * This is in line with the current EPC core design, where all
+		 * BARs are disabled by default, and then the EPF driver enables
+		 * the BARs it wishes to use.
+		 */
+		if (bar_type != BAR_RESERVED)
+			dw_pcie_ep_reset_bar(pci, bar);
+	}
+}
+
 /**
  * dw_pcie_ep_init_registers - Initialize DWC EP specific registers
  * @ep: DWC EP device
@@ -1203,6 +1252,8 @@ int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep)
 	if (ep->ops->init)
 		ep->ops->init(ep);
 
+	dw_pcie_ep_disable_bars(ep);
+
 	/*
 	 * PCIe r6.0, section 7.9.15 states that for endpoints that support
 	 * PTM, this capability structure is required in exactly one
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 6ae6189e9b8a9..a74339982c246 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -1256,9 +1256,13 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci)
 				PCIE_PME_TO_L2_TIMEOUT_US/10,
 				PCIE_PME_TO_L2_TIMEOUT_US, false, pci);
 	if (ret) {
-		/* Only log message when LTSSM isn't in DETECT or POLL */
-		dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val);
-		return ret;
+		/*
+		 * Failure is non-fatal since spec r7.0, sec 5.3.3.2.1,
+		 * recommends proceeding with L2/L3 sequence even if one or more
+		 * devices do not respond with PME_TO_Ack after 10ms timeout.
+		 */
+		dev_warn(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val);
+		ret = 0;
 	}
 
 	/*
@@ -1300,15 +1304,24 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci)
 
 	ret = dw_pcie_start_link(pci);
 	if (ret)
-		return ret;
+		goto err_deinit;
 
 	ret = dw_pcie_wait_for_link(pci);
-	if (ret)
-		return ret;
+	if (ret == -ETIMEDOUT)
+		goto err_stop_link;
 
 	if (pci->pp.ops->post_init)
 		pci->pp.ops->post_init(&pci->pp);
 
+	return 0;
+
+err_stop_link:
+	dw_pcie_stop_link(pci);
+
+err_deinit:
+	if (pci->pp.ops->deinit)
+		pci->pp.ops->deinit(&pci->pp);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c
index 8530746ec5cbb..d103ab759c4ee 100644
--- a/drivers/pci/controller/dwc/pcie-designware-plat.c
+++ b/drivers/pci/controller/dwc/pcie-designware-plat.c
@@ -32,15 +32,6 @@ struct dw_plat_pcie_of_data {
 static const struct dw_pcie_host_ops dw_plat_pcie_host_ops = {
 };
 
-static void dw_plat_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
 				     unsigned int type, u16 interrupt_num)
 {
@@ -73,7 +64,6 @@ dw_plat_pcie_get_features(struct dw_pcie_ep *ep)
 }
 
 static const struct dw_pcie_ep_ops pcie_ep_ops = {
-	.init = dw_plat_pcie_ep_init,
 	.raise_irq = dw_plat_pcie_ep_raise_irq,
 	.get_features = dw_plat_pcie_get_features,
 };
diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c
index 5b17da63151d5..bb5d1a35e89df 100644
--- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c
+++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c
@@ -361,13 +361,9 @@ static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep)
 static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
 
 	rockchip_pcie_enable_l0s(pci);
 	rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep);
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
 };
 
 static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
@@ -403,12 +399,19 @@ static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = {
 	.bar[BAR_5] = { .type = BAR_RESIZABLE, },
 };
 
+static const struct pci_epc_bar_rsvd_region rk3588_bar4_rsvd[] = {
+	{
+		/* DMA_CAP (BAR4: DMA Port Logic Structure) */
+		.type = PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO,
+		.offset = 0x0,
+		.size = 0x2000,
+	},
+};
+
 /*
  * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of
  * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver,
- * so mark it as RESERVED. (rockchip_pcie_ep_init() will disable all BARs by
- * default.) If the host could write to BAR4, the iATU settings (for all other
- * BARs) would be overwritten, resulting in (all other BARs) no longer working.
+ * so mark it as RESERVED.
  */
 static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = {
 	DWC_EPC_COMMON_FEATURES,
@@ -420,7 +423,11 @@ static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = {
 	.bar[BAR_1] = { .type = BAR_RESIZABLE, },
 	.bar[BAR_2] = { .type = BAR_RESIZABLE, },
 	.bar[BAR_3] = { .type = BAR_RESIZABLE, },
-	.bar[BAR_4] = { .type = BAR_RESERVED, },
+	.bar[BAR_4] = {
+		.type = BAR_RESERVED,
+		.nr_rsvd_regions = ARRAY_SIZE(rk3588_bar4_rsvd),
+		.rsvd_regions = rk3588_bar4_rsvd,
+	},
 	.bar[BAR_5] = { .type = BAR_RESIZABLE, },
 };
 
@@ -746,6 +753,7 @@ static struct platform_driver rockchip_pcie_driver = {
 		.name	= "rockchip-dw-pcie",
 		.of_match_table = rockchip_pcie_of_match,
 		.suppress_bind_attrs = true,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.probe = rockchip_pcie_probe,
 };
diff --git a/drivers/pci/controller/dwc/pcie-eswin.c b/drivers/pci/controller/dwc/pcie-eswin.c
new file mode 100644
index 0000000000000..2845832b38243
--- /dev/null
+++ b/drivers/pci/controller/dwc/pcie-eswin.c
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN PCIe Root Complex driver
+ *
+ * Copyright 2026, Beijing ESWIN Computing Technology Co., Ltd.
+ *
+ * Authors: Yu Ning <ningyu@eswincomputing.com>
+ *          Senchuan Zhang <zhangsenchuan@eswincomputing.com>
+ *          Yanghui Ou <ouyanghui@eswincomputing.com>
+ */
+
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/resource.h>
+#include <linux/reset.h>
+#include <linux/types.h>
+
+#include "pcie-designware.h"
+
+/* ELBI registers */
+#define PCIEELBI_CTRL0_OFFSET		0x0
+#define PCIEELBI_STATUS0_OFFSET		0x100
+
+/* LTSSM register fields */
+#define PCIEELBI_APP_LTSSM_ENABLE	BIT(5)
+
+/* APP_HOLD_PHY_RST register fields */
+#define PCIEELBI_APP_HOLD_PHY_RST	BIT(6)
+
+/* PM_SEL_AUX_CLK register fields */
+#define PCIEELBI_PM_SEL_AUX_CLK		BIT(16)
+
+/* DEV_TYPE register fields */
+#define PCIEELBI_CTRL0_DEV_TYPE		GENMASK(3, 0)
+
+/* Vendor and device ID value */
+#define PCI_VENDOR_ID_ESWIN		0x1fe1
+#define PCI_DEVICE_ID_ESWIN_EIC7700	0x2030
+
+#define ESWIN_NUM_RSTS			ARRAY_SIZE(eswin_pcie_rsts)
+
+static const char * const eswin_pcie_rsts[] = {
+	"pwr",
+	"dbi",
+};
+
+struct eswin_pcie_data {
+	bool skip_l23;
+};
+
+struct eswin_pcie_port {
+	struct list_head list;
+	struct reset_control *perst;
+	int num_lanes;
+};
+
+struct eswin_pcie {
+	struct dw_pcie pci;
+	struct clk_bulk_data *clks;
+	struct reset_control_bulk_data resets[ESWIN_NUM_RSTS];
+	struct list_head ports;
+	const struct eswin_pcie_data *data;
+	int num_clks;
+};
+
+#define to_eswin_pcie(x) dev_get_drvdata((x)->dev)
+
+static int eswin_pcie_start_link(struct dw_pcie *pci)
+{
+	u32 val;
+
+	/* Enable LTSSM */
+	val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+	val |= PCIEELBI_APP_LTSSM_ENABLE;
+	writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+
+	return 0;
+}
+
+static bool eswin_pcie_link_up(struct dw_pcie *pci)
+{
+	u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+	u16 val = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA);
+
+	return val & PCI_EXP_LNKSTA_DLLLA;
+}
+
+static int eswin_pcie_perst_reset(struct eswin_pcie_port *port,
+				  struct eswin_pcie *pcie)
+{
+	int ret;
+
+	ret = reset_control_assert(port->perst);
+	if (ret) {
+		dev_err(pcie->pci.dev, "Failed to assert PERST#\n");
+		return ret;
+	}
+
+	/* Ensure that PERST# has been asserted for at least 100 ms */
+	msleep(PCIE_T_PVPERL_MS);
+
+	ret = reset_control_deassert(port->perst);
+	if (ret) {
+		dev_err(pcie->pci.dev, "Failed to deassert PERST#\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void eswin_pcie_assert(struct eswin_pcie *pcie)
+{
+	struct eswin_pcie_port *port;
+
+	list_for_each_entry(port, &pcie->ports, list)
+		reset_control_assert(port->perst);
+	reset_control_bulk_assert(ESWIN_NUM_RSTS, pcie->resets);
+}
+
+static int eswin_pcie_parse_port(struct eswin_pcie *pcie,
+				 struct device_node *node)
+{
+	struct device *dev = pcie->pci.dev;
+	struct eswin_pcie_port *port;
+
+	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	port->perst = of_reset_control_get_exclusive(node, "perst");
+	if (IS_ERR(port->perst)) {
+		dev_err(dev, "Failed to get PERST# reset\n");
+		return PTR_ERR(port->perst);
+	}
+
+	/*
+	 * TODO: Since the Root Port node is separated out by pcie devicetree,
+	 * the DWC core initialization code can't parse the num-lanes attribute
+	 * in the Root Port. Before entering the DWC core initialization code,
+	 * the platform driver code parses the Root Port node. The ESWIN only
+	 * supports one Root Port node, and the num-lanes attribute is suitable
+	 * for the case of one Root Port.
+	 */
+	if (!of_property_read_u32(node, "num-lanes", &port->num_lanes))
+		pcie->pci.num_lanes = port->num_lanes;
+
+	INIT_LIST_HEAD(&port->list);
+	list_add_tail(&port->list, &pcie->ports);
+
+	return 0;
+}
+
+static int eswin_pcie_parse_ports(struct eswin_pcie *pcie)
+{
+	struct eswin_pcie_port *port, *tmp;
+	struct device *dev = pcie->pci.dev;
+	int ret;
+
+	for_each_available_child_of_node_scoped(dev->of_node, of_port) {
+		ret = eswin_pcie_parse_port(pcie, of_port);
+		if (ret)
+			goto err_port;
+	}
+
+	return 0;
+
+err_port:
+	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+		reset_control_put(port->perst);
+		list_del(&port->list);
+	}
+
+	return ret;
+}
+
+static int eswin_pcie_host_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct eswin_pcie *pcie = to_eswin_pcie(pci);
+	struct eswin_pcie_port *port, *tmp;
+	u32 val;
+	int ret;
+
+	ret = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks);
+	if (ret)
+		return ret;
+
+	/*
+	 * The PWR and DBI reset signals are respectively used to reset the
+	 * PCIe controller and the DBI register.
+	 *
+	 * The PERST# signal is a reset signal that simultaneously controls the
+	 * PCIe controller, PHY, and Endpoint. Before configuring the PHY, the
+	 * PERST# signal must first be deasserted.
+	 *
+	 * The external reference clock is supplied simultaneously to the PHY
+	 * and EP. When the PHY is configurable, the entire chip already has
+	 * stable power and reference clock. The PHY will be ready within 20ms
+	 * after writing app_hold_phy_rst register bit of ELBI register space.
+	 */
+	ret = reset_control_bulk_deassert(ESWIN_NUM_RSTS, pcie->resets);
+	if (ret) {
+		dev_err(pcie->pci.dev, "Failed to deassert resets\n");
+		goto err_deassert;
+	}
+
+	/* Configure Root Port type */
+	val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+	val &= ~PCIEELBI_CTRL0_DEV_TYPE;
+	val |= FIELD_PREP(PCIEELBI_CTRL0_DEV_TYPE, PCI_EXP_TYPE_ROOT_PORT);
+	writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+
+	list_for_each_entry(port, &pcie->ports, list) {
+		ret = eswin_pcie_perst_reset(port, pcie);
+		if (ret)
+			goto err_perst;
+	}
+
+	/* Configure app_hold_phy_rst */
+	val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+	val &= ~PCIEELBI_APP_HOLD_PHY_RST;
+	writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
+
+	/* The maximum waiting time for the clock switch lock is 20ms */
+	ret = readl_poll_timeout(pci->elbi_base + PCIEELBI_STATUS0_OFFSET, val,
+				 !(val & PCIEELBI_PM_SEL_AUX_CLK), 1000,
+				 20000);
+	if (ret) {
+		dev_err(pci->dev, "Timeout waiting for PM_SEL_AUX_CLK ready\n");
+		goto err_phy_init;
+	}
+
+	/*
+	 * Configure ESWIN VID:DID for Root Port as the default values are
+	 * invalid.
+	 */
+	dw_pcie_dbi_ro_wr_en(pci);
+	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, PCI_VENDOR_ID_ESWIN);
+	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, PCI_DEVICE_ID_ESWIN_EIC7700);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
+	return 0;
+
+err_phy_init:
+	list_for_each_entry(port, &pcie->ports, list)
+		reset_control_assert(port->perst);
+err_perst:
+	reset_control_bulk_assert(ESWIN_NUM_RSTS, pcie->resets);
+err_deassert:
+	clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks);
+	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+		reset_control_put(port->perst);
+		list_del(&port->list);
+	}
+
+	return ret;
+}
+
+static void eswin_pcie_host_deinit(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct eswin_pcie *pcie = to_eswin_pcie(pci);
+
+	eswin_pcie_assert(pcie);
+	clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks);
+}
+
+static void eswin_pcie_pme_turn_off(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct eswin_pcie *pcie = to_eswin_pcie(pci);
+
+	/*
+	 * The ESWIN EIC7700 SoC lacks hardware support for the L2/L3 low-power
+	 * link states. It cannot enter the L2/L3 Ready state through the
+	 * PME_Turn_Off/PME_To_Ack handshake protocol. To avoid this problem,
+	 * the skip_l23_ready has been set.
+	 */
+	pp->skip_l23_ready = pcie->data->skip_l23;
+}
+
+static const struct dw_pcie_host_ops eswin_pcie_host_ops = {
+	.init = eswin_pcie_host_init,
+	.deinit = eswin_pcie_host_deinit,
+	.pme_turn_off = eswin_pcie_pme_turn_off,
+};
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.start_link = eswin_pcie_start_link,
+	.link_up = eswin_pcie_link_up,
+};
+
+static int eswin_pcie_probe(struct platform_device *pdev)
+{
+	const struct eswin_pcie_data *data;
+	struct eswin_pcie_port *port, *tmp;
+	struct device *dev = &pdev->dev;
+	struct eswin_pcie *pcie;
+	struct dw_pcie *pci;
+	int ret, i;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return dev_err_probe(dev, -ENODATA, "No platform data\n");
+
+	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&pcie->ports);
+
+	pci = &pcie->pci;
+	pci->dev = dev;
+	pci->ops = &dw_pcie_ops;
+	pci->pp.ops = &eswin_pcie_host_ops;
+	pcie->data = data;
+
+	pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks);
+	if (pcie->num_clks < 0)
+		return dev_err_probe(dev, pcie->num_clks,
+				     "Failed to get pcie clocks\n");
+
+	for (i = 0; i < ESWIN_NUM_RSTS; i++)
+		pcie->resets[i].id = eswin_pcie_rsts[i];
+
+	ret = devm_reset_control_bulk_get_exclusive(dev, ESWIN_NUM_RSTS,
+						    pcie->resets);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get resets\n");
+
+	ret = eswin_pcie_parse_ports(pcie);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to parse Root Port\n");
+
+	platform_set_drvdata(pdev, pcie);
+
+	pm_runtime_no_callbacks(dev);
+	devm_pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0)
+		goto err_pm_runtime_put;
+
+	ret = dw_pcie_host_init(&pci->pp);
+	if (ret) {
+		dev_err(dev, "Failed to init host\n");
+		goto err_init;
+	}
+
+	return 0;
+
+err_pm_runtime_put:
+	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+		reset_control_put(port->perst);
+		list_del(&port->list);
+	}
+err_init:
+	pm_runtime_put(dev);
+
+	return ret;
+}
+
+static int eswin_pcie_suspend_noirq(struct device *dev)
+{
+	struct eswin_pcie *pcie = dev_get_drvdata(dev);
+
+	return dw_pcie_suspend_noirq(&pcie->pci);
+}
+
+static int eswin_pcie_resume_noirq(struct device *dev)
+{
+	struct eswin_pcie *pcie = dev_get_drvdata(dev);
+
+	return dw_pcie_resume_noirq(&pcie->pci);
+}
+
+static DEFINE_NOIRQ_DEV_PM_OPS(eswin_pcie_pm, eswin_pcie_suspend_noirq,
+				eswin_pcie_resume_noirq);
+
+static const struct eswin_pcie_data eswin_eic7700_data = {
+	.skip_l23 = true,
+};
+
+static const struct of_device_id eswin_pcie_of_match[] = {
+	{ .compatible = "eswin,eic7700-pcie", .data = &eswin_eic7700_data },
+	{}
+};
+
+static struct platform_driver eswin_pcie_driver = {
+	.probe = eswin_pcie_probe,
+	.driver = {
+		.name = "eswin-pcie",
+		.of_match_table = eswin_pcie_of_match,
+		.suppress_bind_attrs = true,
+		.pm = &eswin_pcie_pm,
+	},
+};
+builtin_platform_driver(eswin_pcie_driver);
+
+MODULE_DESCRIPTION("ESWIN PCIe Root Complex driver");
+MODULE_AUTHOR("Yu Ning <ningyu@eswincomputing.com>");
+MODULE_AUTHOR("Senchuan Zhang <zhangsenchuan@eswincomputing.com>");
+MODULE_AUTHOR("Yanghui Ou <ouyanghui@eswincomputing.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
index 2666a9c3d67e7..7cf2c312ecec2 100644
--- a/drivers/pci/controller/dwc/pcie-keembay.c
+++ b/drivers/pci/controller/dwc/pcie-keembay.c
@@ -313,11 +313,8 @@ static const struct pci_epc_features keembay_pcie_epc_features = {
 	.msi_capable		= true,
 	.msix_capable		= true,
 	.bar[BAR_0]		= { .only_64bit = true, },
-	.bar[BAR_1]		= { .type = BAR_RESERVED, },
 	.bar[BAR_2]		= { .only_64bit = true, },
-	.bar[BAR_3]		= { .type = BAR_RESERVED, },
 	.bar[BAR_4]		= { .only_64bit = true, },
-	.bar[BAR_5]		= { .type = BAR_RESERVED, },
 	.align			= SZ_16K,
 };
 
diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
index 18460f01b2c67..8e8c58a42bc3b 100644
--- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
+++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
@@ -850,9 +850,7 @@ static const struct pci_epc_features qcom_pcie_epc_features = {
 	.msi_capable = true,
 	.align = SZ_4K,
 	.bar[BAR_0] = { .only_64bit = true, },
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
 	.bar[BAR_2] = { .only_64bit = true, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
 };
 
 static const struct pci_epc_features *
@@ -861,17 +859,7 @@ qcom_pcie_epc_get_features(struct dw_pcie_ep *pci_ep)
 	return &qcom_pcie_epc_features;
 }
 
-static void qcom_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = BAR_0; bar <= BAR_5; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static const struct dw_pcie_ep_ops pci_ep_ops = {
-	.init = qcom_pcie_ep_init,
 	.raise_irq = qcom_pcie_ep_raise_irq,
 	.get_features = qcom_pcie_epc_get_features,
 };
diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
index a6912e85e4ddc..79a8f6e85e45a 100644
--- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c
+++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
@@ -386,15 +386,6 @@ static void rcar_gen4_pcie_ep_pre_init(struct dw_pcie_ep *ep)
 	writel(PCIEDMAINTSTSEN_INIT, rcar->base + PCIEDMAINTSTSEN);
 }
 
-static void rcar_gen4_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static void rcar_gen4_pcie_ep_deinit(struct rcar_gen4_pcie *rcar)
 {
 	writel(0, rcar->base + PCIEDMAINTSTSEN);
@@ -422,10 +413,12 @@ static int rcar_gen4_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
 static const struct pci_epc_features rcar_gen4_pcie_epc_features = {
 	DWC_EPC_COMMON_FEATURES,
 	.msi_capable = true,
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
+	.bar[BAR_0] = { .type = BAR_RESIZABLE, },
+	.bar[BAR_1] = { .type = BAR_DISABLED, },
+	.bar[BAR_2] = { .type = BAR_RESIZABLE, },
+	.bar[BAR_3] = { .type = BAR_DISABLED, },
 	.bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256 },
-	.bar[BAR_5] = { .type = BAR_RESERVED, },
+	.bar[BAR_5] = { .type = BAR_DISABLED, },
 	.align = SZ_1M,
 };
 
@@ -449,7 +442,6 @@ static unsigned int rcar_gen4_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep,
 
 static const struct dw_pcie_ep_ops pcie_ep_ops = {
 	.pre_init = rcar_gen4_pcie_ep_pre_init,
-	.init = rcar_gen4_pcie_ep_init,
 	.raise_irq = rcar_gen4_pcie_ep_raise_irq,
 	.get_features = rcar_gen4_pcie_ep_get_features,
 	.get_dbi_offset = rcar_gen4_pcie_ep_get_dbi_offset,
diff --git a/drivers/pci/controller/dwc/pcie-stm32-ep.c b/drivers/pci/controller/dwc/pcie-stm32-ep.c
index c1944b40ce02f..a7988dff10453 100644
--- a/drivers/pci/controller/dwc/pcie-stm32-ep.c
+++ b/drivers/pci/controller/dwc/pcie-stm32-ep.c
@@ -28,15 +28,6 @@ struct stm32_pcie {
 	unsigned int perst_irq;
 };
 
-static void stm32_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static int stm32_pcie_start_link(struct dw_pcie *pci)
 {
 	struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci);
@@ -82,7 +73,6 @@ stm32_pcie_get_features(struct dw_pcie_ep *ep)
 }
 
 static const struct dw_pcie_ep_ops stm32_pcie_ep_ops = {
-	.init = stm32_pcie_ep_init,
 	.raise_irq = stm32_pcie_raise_irq,
 	.get_features = stm32_pcie_get_features,
 };
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
index 06571d806ab31..6881f0b94c739 100644
--- a/drivers/pci/controller/dwc/pcie-tegra194.c
+++ b/drivers/pci/controller/dwc/pcie-tegra194.c
@@ -1923,15 +1923,6 @@ static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-static void tegra_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-};
-
 static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)
 {
 	/* Tegra194 supports only INTA */
@@ -1993,11 +1984,10 @@ static const struct pci_epc_features tegra_pcie_epc_features = {
 	.msi_capable = true,
 	.bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M,
 			.only_64bit = true, },
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
-	.bar[BAR_2] = { .type = BAR_RESERVED, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
-	.bar[BAR_4] = { .type = BAR_RESERVED, },
-	.bar[BAR_5] = { .type = BAR_RESERVED, },
+	.bar[BAR_2] = { .type = BAR_DISABLED, },
+	.bar[BAR_3] = { .type = BAR_DISABLED, },
+	.bar[BAR_4] = { .type = BAR_DISABLED, },
+	.bar[BAR_5] = { .type = BAR_DISABLED, },
 	.align = SZ_64K,
 };
 
@@ -2008,7 +1998,6 @@ tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
 }
 
 static const struct dw_pcie_ep_ops pcie_ep_ops = {
-	.init = tegra_pcie_ep_init,
 	.raise_irq = tegra_pcie_ep_raise_irq,
 	.get_features = tegra_pcie_ep_get_features,
 };
diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c
index d52753060970f..89fb782002223 100644
--- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c
+++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c
@@ -203,15 +203,6 @@ static void uniphier_pcie_stop_link(struct dw_pcie *pci)
 	uniphier_pcie_ltssm_enable(priv, false);
 }
 
-static void uniphier_pcie_ep_init(struct dw_pcie_ep *ep)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
-	enum pci_barno bar;
-
-	for (bar = BAR_0; bar <= BAR_5; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-}
-
 static int uniphier_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
@@ -283,7 +274,6 @@ uniphier_pcie_get_features(struct dw_pcie_ep *ep)
 }
 
 static const struct dw_pcie_ep_ops uniphier_pcie_ep_ops = {
-	.init = uniphier_pcie_ep_init,
 	.raise_irq = uniphier_pcie_ep_raise_irq,
 	.get_features = uniphier_pcie_get_features,
 };
@@ -426,11 +416,9 @@ static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = {
 		.msix_capable = false,
 		.align = 1 << 16,
 		.bar[BAR_0] = { .only_64bit = true, },
-		.bar[BAR_1] = { .type = BAR_RESERVED, },
 		.bar[BAR_2] = { .only_64bit = true, },
-		.bar[BAR_3] = { .type = BAR_RESERVED, },
-		.bar[BAR_4] = { .type = BAR_RESERVED, },
-		.bar[BAR_5] = { .type = BAR_RESERVED, },
+		.bar[BAR_4] = { .type = BAR_DISABLED, },
+		.bar[BAR_5] = { .type = BAR_DISABLED, },
 	},
 };
 
@@ -445,11 +433,8 @@ static const struct uniphier_pcie_ep_soc_data uniphier_nx1_data = {
 		.msix_capable = false,
 		.align = 1 << 12,
 		.bar[BAR_0] = { .only_64bit = true, },
-		.bar[BAR_1] = { .type = BAR_RESERVED, },
 		.bar[BAR_2] = { .only_64bit = true, },
-		.bar[BAR_3] = { .type = BAR_RESERVED, },
 		.bar[BAR_4] = { .only_64bit = true, },
-		.bar[BAR_5] = { .type = BAR_RESERVED, },
 	},
 };
 
diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c
index 75ddb8bee168f..b630d883e17ae 100644
--- a/drivers/pci/controller/pcie-mediatek-gen3.c
+++ b/drivers/pci/controller/pcie-mediatek-gen3.c
@@ -22,6 +22,7 @@
 #include <linux/of_device.h>
 #include <linux/of_pci.h>
 #include <linux/pci.h>
+#include <linux/pci-pwrctrl.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
@@ -403,6 +404,64 @@ static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie)
 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);
 }
 
+static int mtk_pcie_device_power_up(struct mtk_gen3_pcie *pcie)
+{
+	int err;
+	u32 val;
+
+	/*
+	 * Airoha EN7581 has a hw bug asserting/releasing PCIE_PE_RSTB signal
+	 * causing occasional PCIe link down. In order to overcome the issue,
+	 * PCIE_RSTB signals are not asserted/released at this stage and the
+	 * PCIe block is reset using en7523_reset_assert() and
+	 * en7581_pci_enable().
+	 */
+	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
+		/* Assert all reset signals */
+		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
+		val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
+		       PCIE_PE_RSTB;
+		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
+	}
+
+	err = pci_pwrctrl_power_on_devices(pcie->dev);
+	if (err) {
+		dev_err(pcie->dev, "Failed to power on devices: %d\n", err);
+		return err;
+	}
+
+	/*
+	 * Described in PCIe CEM specification revision 6.0.
+	 *
+	 * The deassertion of PERST# should be delayed 100ms (TPVPERL)
+	 * for the power and clock to become stable.
+	 */
+	msleep(PCIE_T_PVPERL_MS);
+
+	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
+		/* De-assert reset signals */
+		val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
+			 PCIE_PE_RSTB);
+		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
+	}
+
+	return 0;
+}
+
+static void mtk_pcie_device_power_down(struct mtk_gen3_pcie *pcie)
+{
+	u32 val;
+
+	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
+		/* Assert the PERST# pin */
+		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
+		val |= PCIE_PE_RSTB;
+		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
+	}
+
+	pci_pwrctrl_power_off_devices(pcie->dev);
+}
+
 static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 {
 	struct resource_entry *entry;
@@ -464,33 +523,9 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 	val |= PCIE_DISABLE_DVFSRC_VLT_REQ;
 	writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG);
 
-	/*
-	 * Airoha EN7581 has a hw bug asserting/releasing PCIE_PE_RSTB signal
-	 * causing occasional PCIe link down. In order to overcome the issue,
-	 * PCIE_RSTB signals are not asserted/released at this stage and the
-	 * PCIe block is reset using en7523_reset_assert() and
-	 * en7581_pci_enable().
-	 */
-	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
-		/* Assert all reset signals */
-		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
-		val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
-		       PCIE_PE_RSTB;
-		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
-
-		/*
-		 * Described in PCIe CEM specification revision 6.0.
-		 *
-		 * The deassertion of PERST# should be delayed 100ms (TPVPERL)
-		 * for the power and clock to become stable.
-		 */
-		msleep(PCIE_T_PVPERL_MS);
-
-		/* De-assert reset signals */
-		val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
-			 PCIE_PE_RSTB);
-		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
-	}
+	err = mtk_pcie_device_power_up(pcie);
+	if (err)
+		return err;
 
 	/* Check if the link is up or not */
 	err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val,
@@ -507,7 +542,7 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 		dev_err(pcie->dev,
 			"PCIe link down, current LTSSM state: %s (%#x)\n",
 			ltssm_state, val);
-		return err;
+		goto err_power_down_device;
 	}
 
 	mtk_pcie_enable_msi(pcie);
@@ -532,10 +567,14 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 		err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size,
 					       type, &table_index);
 		if (err)
-			return err;
+			goto err_power_down_device;
 	}
 
 	return 0;
+
+err_power_down_device:
+	mtk_pcie_device_power_down(pcie);
+	return err;
 }
 
 #define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
@@ -876,10 +915,8 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie)
 	if (!regs)
 		return -EINVAL;
 	pcie->base = devm_ioremap_resource(dev, regs);
-	if (IS_ERR(pcie->base)) {
-		dev_err(dev, "failed to map register base\n");
-		return PTR_ERR(pcie->base);
-	}
+	if (IS_ERR(pcie->base))
+		return dev_err_probe(dev, PTR_ERR(pcie->base), "failed to map register base\n");
 
 	pcie->reg_base = regs->start;
 
@@ -888,34 +925,20 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie)
 
 	ret = devm_reset_control_bulk_get_optional_shared(dev, num_resets,
 							  pcie->phy_resets);
-	if (ret) {
-		dev_err(dev, "failed to get PHY bulk reset\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get PHY bulk reset\n");
 
 	pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac");
-	if (IS_ERR(pcie->mac_reset)) {
-		ret = PTR_ERR(pcie->mac_reset);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "failed to get MAC reset\n");
-
-		return ret;
-	}
+	if (IS_ERR(pcie->mac_reset))
+		return dev_err_probe(dev, PTR_ERR(pcie->mac_reset), "failed to get MAC reset\n");
 
 	pcie->phy = devm_phy_optional_get(dev, "pcie-phy");
-	if (IS_ERR(pcie->phy)) {
-		ret = PTR_ERR(pcie->phy);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "failed to get PHY\n");
-
-		return ret;
-	}
+	if (IS_ERR(pcie->phy))
+		return dev_err_probe(dev, PTR_ERR(pcie->phy), "failed to get PHY\n");
 
 	pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks);
-	if (pcie->num_clks < 0) {
-		dev_err(dev, "failed to get clocks\n");
-		return pcie->num_clks;
-	}
+	if (pcie->num_clks < 0)
+		return dev_err_probe(dev, pcie->num_clks, "failed to get clocks\n");
 
 	ret = of_property_read_u32(dev->of_node, "num-lanes", &num_lanes);
 	if (ret == 0) {
@@ -1166,15 +1189,17 @@ static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie)
 	/* Try link up */
 	err = mtk_pcie_startup_port(pcie);
 	if (err)
-		goto err_setup;
+		goto err_power_down;
 
 	err = mtk_pcie_setup_irq(pcie);
 	if (err)
-		goto err_setup;
+		goto err_device_power_off;
 
 	return 0;
 
-err_setup:
+err_device_power_off:
+	mtk_pcie_device_power_down(pcie);
+err_power_down:
 	mtk_pcie_power_down(pcie);
 
 	return err;
@@ -1197,21 +1222,32 @@ static int mtk_pcie_probe(struct platform_device *pdev)
 	pcie->soc = device_get_match_data(dev);
 	platform_set_drvdata(pdev, pcie);
 
+	err = pci_pwrctrl_create_devices(pcie->dev);
+	if (err)
+		return dev_err_probe(dev, err, "failed to create pwrctrl devices\n");
+
 	err = mtk_pcie_setup(pcie);
 	if (err)
-		return err;
+		goto err_destroy_pwrctrl;
 
 	host->ops = &mtk_pcie_ops;
 	host->sysdata = pcie;
 
 	err = pci_host_probe(host);
-	if (err) {
-		mtk_pcie_irq_teardown(pcie);
-		mtk_pcie_power_down(pcie);
-		return err;
-	}
+	if (err)
+		goto err_teardown_irq_and_power_down;
 
 	return 0;
+
+err_teardown_irq_and_power_down:
+	mtk_pcie_irq_teardown(pcie);
+	mtk_pcie_device_power_down(pcie);
+	mtk_pcie_power_down(pcie);
+err_destroy_pwrctrl:
+	if (err != -EPROBE_DEFER)
+		pci_pwrctrl_destroy_devices(pcie->dev);
+
+	return err;
 }
 
 static void mtk_pcie_remove(struct platform_device *pdev)
@@ -1225,7 +1261,9 @@ static void mtk_pcie_remove(struct platform_device *pdev)
 	pci_unlock_rescan_remove();
 
 	mtk_pcie_irq_teardown(pcie);
+	pci_pwrctrl_power_off_devices(pcie->dev);
 	mtk_pcie_power_down(pcie);
+	pci_pwrctrl_destroy_devices(pcie->dev);
 }
 
 static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie)
@@ -1283,7 +1321,6 @@ static int mtk_pcie_suspend_noirq(struct device *dev)
 {
 	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);
 	int err;
-	u32 val;
 
 	/* Trigger link to L2 state */
 	err = mtk_pcie_turn_off_link(pcie);
@@ -1292,13 +1329,7 @@ static int mtk_pcie_suspend_noirq(struct device *dev)
 		return err;
 	}
 
-	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
-		/* Assert the PERST# pin */
-		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
-		val |= PCIE_PE_RSTB;
-		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
-	}
-
+	mtk_pcie_device_power_down(pcie);
 	dev_dbg(pcie->dev, "entered L2 states successfully");
 
 	mtk_pcie_irq_save(pcie);
@@ -1317,14 +1348,16 @@ static int mtk_pcie_resume_noirq(struct device *dev)
 		return err;
 
 	err = mtk_pcie_startup_port(pcie);
-	if (err) {
-		mtk_pcie_power_down(pcie);
-		return err;
-	}
+	if (err)
+		goto err_power_down;
 
 	mtk_pcie_irq_restore(pcie);
 
 	return 0;
+
+err_power_down:
+	mtk_pcie_power_down(pcie);
+	return err;
 }
 
 static const struct dev_pm_ops mtk_pcie_pm_ops = {
diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c
index 657875ef4657f..c2da8ac1f2e85 100644
--- a/drivers/pci/controller/pcie-rcar-ep.c
+++ b/drivers/pci/controller/pcie-rcar-ep.c
@@ -440,13 +440,10 @@ static const struct pci_epc_features rcar_pcie_epc_features = {
 	/* use 64-bit BARs so mark BAR[1,3,5] as reserved */
 	.bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = 128,
 			.only_64bit = true, },
-	.bar[BAR_1] = { .type = BAR_RESERVED, },
 	.bar[BAR_2] = { .type = BAR_FIXED, .fixed_size = 256,
 			.only_64bit = true, },
-	.bar[BAR_3] = { .type = BAR_RESERVED, },
 	.bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256,
 			.only_64bit = true, },
-	.bar[BAR_5] = { .type = BAR_RESERVED, },
 };
 
 static const struct pci_epc_features*
diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c
index 2809112e63171..bfc210e696ed2 100644
--- a/drivers/pci/controller/pcie-rzg3s-host.c
+++ b/drivers/pci/controller/pcie-rzg3s-host.c
@@ -111,6 +111,15 @@
 #define RZG3S_PCI_PERM_CFG_HWINIT_EN		BIT(2)
 #define RZG3S_PCI_PERM_PIPE_PHY_REG_EN		BIT(1)
 
+#define RZG3S_PCI_RESET				0x310
+#define RZG3S_PCI_RESET_RST_OUT_B		BIT(6)
+#define RZG3S_PCI_RESET_RST_PS_B		BIT(5)
+#define RZG3S_PCI_RESET_RST_LOAD_B		BIT(4)
+#define RZG3S_PCI_RESET_RST_CFG_B		BIT(3)
+#define RZG3S_PCI_RESET_RST_RSM_B		BIT(2)
+#define RZG3S_PCI_RESET_RST_GP_B		BIT(1)
+#define RZG3S_PCI_RESET_RST_B			BIT(0)
+
 #define RZG3S_PCI_MSIRE(id)			(0x600 + (id) * 0x10)
 #define RZG3S_PCI_MSIRE_ENA			BIT(0)
 
@@ -159,10 +168,6 @@
 
 #define RZG3S_PCI_CFG_PCIEC			0x60
 
-/* System controller registers */
-#define RZG3S_SYS_PCIE_RST_RSM_B		0xd74
-#define RZG3S_SYS_PCIE_RST_RSM_B_MASK		BIT(0)
-
 /* Maximum number of windows */
 #define RZG3S_MAX_WINDOWS			8
 
@@ -174,6 +179,48 @@
 /* Timeouts experimentally determined */
 #define RZG3S_REQ_ISSUE_TIMEOUT_US		2500
 
+/**
+ * struct rzg3s_sysc_function - System Controller function descriptor
+ * @offset: Register offset from the System Controller base address
+ * @mask: Bit mask for the function within the register
+ */
+struct rzg3s_sysc_function {
+	u32 offset;
+	u32 mask;
+};
+
+/**
+ * enum rzg3s_sysc_func_id - System controller function IDs
+ * @RZG3S_SYSC_FUNC_ID_RST_RSM_B: RST_RSM_B SYSC function ID
+ * @RZG3S_SYSC_FUNC_ID_L1_ALLOW: L1 allow SYSC function ID
+ * @RZG3S_SYSC_FUNC_ID_MODE: Mode SYSC function ID
+ * @RZG3S_SYSC_FUNC_ID_MAX: Max SYSC function ID
+ */
+enum rzg3s_sysc_func_id {
+	RZG3S_SYSC_FUNC_ID_RST_RSM_B,
+	RZG3S_SYSC_FUNC_ID_L1_ALLOW,
+	RZG3S_SYSC_FUNC_ID_MODE,
+	RZG3S_SYSC_FUNC_ID_MAX,
+};
+
+/**
+ * struct rzg3s_sysc_info - RZ/G3S System Controller info
+ * @functions: SYSC function descriptors array
+ */
+struct rzg3s_sysc_info {
+	const struct rzg3s_sysc_function functions[RZG3S_SYSC_FUNC_ID_MAX];
+};
+
+/**
+ * struct rzg3s_sysc - RZ/G3S System Controller descriptor
+ * @regmap: System controller regmap
+ * @info: System controller info
+ */
+struct rzg3s_sysc {
+	struct regmap *regmap;
+	const struct rzg3s_sysc_info *info;
+};
+
 /**
  * struct rzg3s_pcie_msi - RZ/G3S PCIe MSI data structure
  * @domain: IRQ domain
@@ -199,17 +246,25 @@ struct rzg3s_pcie_host;
 /**
  * struct rzg3s_pcie_soc_data - SoC specific data
  * @init_phy: PHY initialization function
+ * @config_pre_init: Optional callback for SoC-specific pre-configuration
+ * @config_post_init: Callback for SoC-specific post-configuration
+ * @config_deinit: Callback for SoC-specific de-initialization
  * @power_resets: array with the resets that need to be de-asserted after
  *                power-on
  * @cfg_resets: array with the resets that need to be de-asserted after
  *              configuration
+ * @sysc_info: SYSC info
  * @num_power_resets: number of power resets
  * @num_cfg_resets: number of configuration resets
  */
 struct rzg3s_pcie_soc_data {
 	int (*init_phy)(struct rzg3s_pcie_host *host);
+	void (*config_pre_init)(struct rzg3s_pcie_host *host);
+	int (*config_post_init)(struct rzg3s_pcie_host *host);
+	int (*config_deinit)(struct rzg3s_pcie_host *host);
 	const char * const *power_resets;
 	const char * const *cfg_resets;
+	struct rzg3s_sysc_info sysc_info;
 	u8 num_power_resets;
 	u8 num_cfg_resets;
 };
@@ -233,7 +288,7 @@ struct rzg3s_pcie_port {
  * @dev: struct device
  * @power_resets: reset control signals that should be set after power up
  * @cfg_resets: reset control signals that should be set after configuration
- * @sysc: SYSC regmap
+ * @sysc: SYSC descriptor
  * @intx_domain: INTx IRQ domain
  * @data: SoC specific data
  * @msi: MSI data structure
@@ -248,7 +303,7 @@ struct rzg3s_pcie_host {
 	struct device *dev;
 	struct reset_control_bulk_data *power_resets;
 	struct reset_control_bulk_data *cfg_resets;
-	struct regmap *sysc;
+	struct rzg3s_sysc *sysc;
 	struct irq_domain *intx_domain;
 	const struct rzg3s_pcie_soc_data *data;
 	struct rzg3s_pcie_msi msi;
@@ -260,6 +315,23 @@ struct rzg3s_pcie_host {
 
 #define rzg3s_msi_to_host(_msi)	container_of(_msi, struct rzg3s_pcie_host, msi)
 
+static int rzg3s_sysc_config_func(struct rzg3s_sysc *sysc,
+				  enum rzg3s_sysc_func_id fid, u32 val)
+{
+	const struct rzg3s_sysc_info *info = sysc->info;
+	const struct rzg3s_sysc_function *functions = info->functions;
+
+	if (fid >= RZG3S_SYSC_FUNC_ID_MAX)
+		return -EINVAL;
+
+	if (!functions[fid].mask)
+		return 0;
+
+	return regmap_update_bits(sysc->regmap, functions[fid].offset,
+				  functions[fid].mask,
+				  field_prep(functions[fid].mask, val));
+}
+
 static void rzg3s_pcie_update_bits(void __iomem *base, u32 offset, u32 mask,
 				   u32 val)
 {
@@ -945,8 +1017,9 @@ static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
 {
 	u32 remote_supported_link_speeds, max_supported_link_speeds;
 	u32 cs2, tmp, pcie_cap = RZG3S_PCI_CFG_PCIEC;
-	u32 cur_link_speed, link_speed;
+	u32 cur_link_speed, link_speed, hw_max_speed;
 	u8 ltssm_state_l0 = 0xc;
+	u32 lnkcap;
 	int ret;
 	u16 ls;
 
@@ -966,7 +1039,22 @@ static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
 	ls = readw_relaxed(host->pcie + pcie_cap + PCI_EXP_LNKSTA);
 	cs2 = readl_relaxed(host->axi + RZG3S_PCI_PCSTAT2);
 
-	switch (pcie_link_speed[host->max_link_speed]) {
+	/* Read hardware supported link speed from Link Capabilities Register */
+	lnkcap = readl_relaxed(host->pcie + pcie_cap + PCI_EXP_LNKCAP);
+	hw_max_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, lnkcap);
+
+	/*
+	 * Use DT max-link-speed only as a limit. If specified and lower
+	 * than hardware capability, cap to that value.
+	 */
+	if (host->max_link_speed > 0 && host->max_link_speed < hw_max_speed)
+		hw_max_speed = host->max_link_speed;
+
+	switch (pcie_link_speed[hw_max_speed]) {
+	case PCIE_SPEED_8_0GT:
+		max_supported_link_speeds = GENMASK(PCI_EXP_LNKSTA_CLS_8_0GB - 1, 0);
+		link_speed = PCI_EXP_LNKCTL2_TLS_8_0GT;
+		break;
 	case PCIE_SPEED_5_0GT:
 		max_supported_link_speeds = GENMASK(PCI_EXP_LNKSTA_CLS_5_0GB - 1, 0);
 		link_speed = PCI_EXP_LNKCTL2_TLS_5_0GT;
@@ -982,10 +1070,10 @@ static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
 	remote_supported_link_speeds &= max_supported_link_speeds;
 
 	/*
-	 * Return if max link speed is already set or the connected device
+	 * Return if target link speed is already set or the connected device
 	 * doesn't support it.
 	 */
-	if (cur_link_speed == host->max_link_speed ||
+	if (cur_link_speed == hw_max_speed ||
 	    remote_supported_link_speeds != max_supported_link_speeds)
 		return 0;
 
@@ -1022,6 +1110,7 @@ static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
 static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host)
 {
 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host);
+	u32 mask = GENMASK(31, 8);
 	struct resource_entry *ft;
 	struct resource *bus;
 	u8 subordinate_bus;
@@ -1045,6 +1134,13 @@ static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host)
 	writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00L);
 	writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00U);
 
+	/*
+	 * Explicitly program class code. RZ/G3E requires this configuration.
+	 * Harmless for RZ/G3S where this matches the hardware default.
+	 */
+	rzg3s_pcie_update_bits(host->pcie, PCI_CLASS_REVISION, mask,
+			       field_prep(mask, PCI_CLASS_BRIDGE_PCI_NORMAL));
+
 	/* Disable access control to the CFGU */
 	writel_relaxed(0, host->axi + RZG3S_PCI_PERM);
 
@@ -1056,6 +1152,57 @@ static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host)
 	return 0;
 }
 
+static int rzg3s_pcie_config_post_init(struct rzg3s_pcie_host *host)
+{
+	return reset_control_bulk_deassert(host->data->num_cfg_resets,
+					   host->cfg_resets);
+}
+
+static int rzg3s_pcie_config_deinit(struct rzg3s_pcie_host *host)
+{
+	return reset_control_bulk_assert(host->data->num_cfg_resets,
+					 host->cfg_resets);
+}
+
+static void rzg3e_pcie_config_pre_init(struct rzg3s_pcie_host *host)
+{
+	u32 mask = RZG3S_PCI_RESET_RST_LOAD_B | RZG3S_PCI_RESET_RST_CFG_B;
+
+	/* De-assert LOAD_B and CFG_B */
+	rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask);
+}
+
+static int rzg3e_pcie_config_deinit(struct rzg3s_pcie_host *host)
+{
+	writel_relaxed(0, host->axi + RZG3S_PCI_RESET);
+	return 0;
+}
+
+static int rzg3e_pcie_config_post_init(struct rzg3s_pcie_host *host)
+{
+	u32 mask = RZG3S_PCI_RESET_RST_PS_B | RZG3S_PCI_RESET_RST_GP_B |
+		   RZG3S_PCI_RESET_RST_B;
+
+	/* De-assert PS_B, GP_B, RST_B */
+	rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask);
+
+	/* Flush deassert */
+	readl_relaxed(host->axi + RZG3S_PCI_RESET);
+
+	/*
+	 * According to the RZ/G3E HW manual (Rev.1.15, Table 6.6-130
+	 * Initialization Procedure (RC)), hardware requires >= 500us delay
+	 * before final reset deassert.
+	 */
+	fsleep(500);
+
+	/* De-assert OUT_B and RSM_B */
+	mask = RZG3S_PCI_RESET_RST_OUT_B | RZG3S_PCI_RESET_RST_RSM_B;
+	rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask);
+
+	return 0;
+}
+
 static void rzg3s_pcie_irq_init(struct rzg3s_pcie_host *host)
 {
 	/*
@@ -1135,9 +1282,9 @@ static int rzg3s_pcie_resets_prepare_and_get(struct rzg3s_pcie_host *host)
 	if (ret)
 		return ret;
 
-	return devm_reset_control_bulk_get_exclusive(host->dev,
-						     data->num_cfg_resets,
-						     host->cfg_resets);
+	return devm_reset_control_bulk_get_optional_exclusive(host->dev,
+							      data->num_cfg_resets,
+							      host->cfg_resets);
 }
 
 static int rzg3s_pcie_host_parse_port(struct rzg3s_pcie_host *host)
@@ -1204,22 +1351,32 @@ static int rzg3s_pcie_host_init(struct rzg3s_pcie_host *host)
 	u32 val;
 	int ret;
 
+	/* SoC-specific pre-configuration */
+	if (host->data->config_pre_init)
+		host->data->config_pre_init(host);
+
 	/* Initialize the PCIe related registers */
 	ret = rzg3s_pcie_config_init(host);
 	if (ret)
-		return ret;
+		goto config_deinit;
 
 	ret = rzg3s_pcie_host_init_port(host);
 	if (ret)
-		return ret;
+		goto config_deinit;
+
+	/* Enable ASPM L1 transition for SoCs that use it */
+	ret = rzg3s_sysc_config_func(host->sysc,
+				     RZG3S_SYSC_FUNC_ID_L1_ALLOW, 1);
+	if (ret)
+		goto config_deinit_and_refclk;
 
 	/* Initialize the interrupts */
 	rzg3s_pcie_irq_init(host);
 
-	ret = reset_control_bulk_deassert(host->data->num_cfg_resets,
-					  host->cfg_resets);
+	/* SoC-specific post-configuration */
+	ret = host->data->config_post_init(host);
 	if (ret)
-		goto disable_port_refclk;
+		goto config_deinit_and_refclk;
 
 	/* Wait for link up */
 	ret = readl_poll_timeout(host->axi + RZG3S_PCI_PCSTAT1, val,
@@ -1228,18 +1385,20 @@ static int rzg3s_pcie_host_init(struct rzg3s_pcie_host *host)
 				 PCIE_LINK_WAIT_SLEEP_MS * MILLI *
 				 PCIE_LINK_WAIT_MAX_RETRIES);
 	if (ret)
-		goto cfg_resets_deassert;
+		goto config_deinit_post;
 
 	val = readl_relaxed(host->axi + RZG3S_PCI_PCSTAT2);
 	dev_info(host->dev, "PCIe link status [0x%x]\n", val);
 
 	return 0;
 
-cfg_resets_deassert:
-	reset_control_bulk_assert(host->data->num_cfg_resets,
-				  host->cfg_resets);
-disable_port_refclk:
+config_deinit_post:
+	host->data->config_deinit(host);
+config_deinit_and_refclk:
 	clk_disable_unprepare(host->port.refclk);
+config_deinit:
+	if (host->data->config_pre_init)
+		host->data->config_deinit(host);
 	return ret;
 }
 
@@ -1271,50 +1430,55 @@ static int rzg3s_pcie_set_inbound_windows(struct rzg3s_pcie_host *host,
 	u64 pci_addr = entry->res->start - entry->offset;
 	u64 cpu_addr = entry->res->start;
 	u64 cpu_end = entry->res->end;
-	u64 size_id = 0;
 	int id = *index;
 	u64 size;
 
-	while (cpu_addr < cpu_end) {
+	/*
+	 * According to the RZ/G3S HW manual (Rev.1.10, section 34.6.6.7) and
+	 * RZ/G3E HW manual (Rev.1.15, section 6.6.7.6):
+	 * - Each window must be a single memory size of power of two
+	 * - Mask registers must be set to (2^N - 1)
+	 * - Bit carry must not occur when adding base and mask registers,
+	 *   meaning the base address must be aligned to the window size
+	 *
+	 * Split non-power-of-2 regions into multiple windows to satisfy
+	 * these constraints without over-mapping.
+	 */
+	while (cpu_addr <= cpu_end) {
+		u64 remaining_size = cpu_end - cpu_addr + 1;
+		u64 align_limit;
+
 		if (id >= RZG3S_MAX_WINDOWS)
 			return dev_err_probe(host->dev, -ENOSPC,
 					     "Failed to map inbound window for resource (%s)\n",
 					     entry->res->name);
 
-		size = resource_size(entry->res) - size_id;
-
-		/*
-		 * According to the RZ/G3S HW manual (Rev.1.10,
-		 * section 34.3.1.71 AXI Window Mask (Lower) Registers) the min
-		 * size is 4K.
-		 */
-		size = max(size, SZ_4K);
+		/* Start with largest power-of-two that fits in remaining size */
+		size = 1ULL << __fls(remaining_size);
 
 		/*
-		 * According the RZ/G3S HW manual (Rev.1.10, sections:
-		 * - 34.3.1.69 AXI Window Base (Lower) Registers
-		 * - 34.3.1.71 AXI Window Mask (Lower) Registers
-		 * - 34.3.1.73 AXI Destination (Lower) Registers)
-		 * the CPU addr, PCIe addr, size should be 4K aligned and be a
-		 * power of 2.
+		 * The "no bit carry" rule requires base addresses to be
+		 * aligned to the window size. Find the maximum window size
+		 * that both addresses can support based on their natural
+		 * alignment (lowest set bit).
 		 */
-		size = ALIGN(size, SZ_4K);
-		size = roundup_pow_of_two(size);
+		align_limit = min(cpu_addr ? (1ULL << __ffs(cpu_addr)) : ~0ULL,
+				  pci_addr ? (1ULL << __ffs(pci_addr)) : ~0ULL);
 
-		cpu_addr = ALIGN(cpu_addr, SZ_4K);
-		pci_addr = ALIGN(pci_addr, SZ_4K);
+		size = min(size, align_limit);
 
 		/*
-		 * According to the RZ/G3S HW manual (Rev.1.10, section
-		 * 34.3.1.71 AXI Window Mask (Lower) Registers) HW expects first
-		 * 12 LSB bits to be 0xfff. Subtract 1 from size for this.
+		 * Minimum window size is 4KB.
+		 * See RZ/G3S HW manual (Rev.1.10, section 34.3.1.71) and
+		 * RZ/G3E HW manual (Rev.1.15, section 6.6.4.1.3.(74)).
 		 */
+		size = max(size, SZ_4K);
+
 		rzg3s_pcie_set_inbound_window(host, cpu_addr, pci_addr,
 					      size - 1, id);
 
 		pci_addr += size;
 		cpu_addr += size;
-		size_id = size;
 		id++;
 	}
 	*index = id;
@@ -1517,6 +1681,7 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	struct device_node *sysc_np __free(device_node) =
 		of_parse_phandle(np, "renesas,sysc", 0);
 	struct rzg3s_pcie_host *host;
+	struct rzg3s_sysc *sysc;
 	int ret;
 
 	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*host));
@@ -1528,28 +1693,36 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	host->data = device_get_match_data(dev);
 	platform_set_drvdata(pdev, host);
 
+	host->sysc = devm_kzalloc(dev, sizeof(*host->sysc), GFP_KERNEL);
+	if (!host->sysc)
+		return -ENOMEM;
+
+	sysc = host->sysc;
+	sysc->info = &host->data->sysc_info;
+
 	host->axi = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(host->axi))
 		return PTR_ERR(host->axi);
 	host->pcie = host->axi + RZG3S_PCI_CFG_BASE;
 
 	host->max_link_speed = of_pci_get_max_link_speed(np);
-	if (host->max_link_speed < 0)
-		host->max_link_speed = 2;
 
 	ret = rzg3s_pcie_host_parse_port(host);
 	if (ret)
 		return ret;
 
-	host->sysc = syscon_node_to_regmap(sysc_np);
-	if (IS_ERR(host->sysc)) {
-		ret = PTR_ERR(host->sysc);
+	sysc->regmap = syscon_node_to_regmap(sysc_np);
+	if (IS_ERR(sysc->regmap)) {
+		ret = PTR_ERR(sysc->regmap);
 		goto port_refclk_put;
 	}
 
-	ret = regmap_update_bits(host->sysc, RZG3S_SYS_PCIE_RST_RSM_B,
-				 RZG3S_SYS_PCIE_RST_RSM_B_MASK,
-				 FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 1));
+	/* Put controller in RC mode */
+	ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_MODE, 1);
+	if (ret)
+		goto port_refclk_put;
+
+	ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 1);
 	if (ret)
 		goto port_refclk_put;
 
@@ -1589,8 +1762,7 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 
 host_probe_teardown:
 	rzg3s_pcie_teardown_irqdomain(host);
-	reset_control_bulk_deassert(host->data->num_cfg_resets,
-				    host->cfg_resets);
+	host->data->config_deinit(host);
 rpm_put:
 	pm_runtime_put_sync(dev);
 rpm_disable:
@@ -1602,9 +1774,7 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	 * SYSC RST_RSM_B signal need to be asserted before turning off the
 	 * power to the PHY.
 	 */
-	regmap_update_bits(host->sysc, RZG3S_SYS_PCIE_RST_RSM_B,
-			   RZG3S_SYS_PCIE_RST_RSM_B_MASK,
-			   FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 0));
+	rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0);
 port_refclk_put:
 	clk_put(host->port.refclk);
 
@@ -1616,7 +1786,7 @@ static int rzg3s_pcie_suspend_noirq(struct device *dev)
 	struct rzg3s_pcie_host *host = dev_get_drvdata(dev);
 	const struct rzg3s_pcie_soc_data *data = host->data;
 	struct rzg3s_pcie_port *port = &host->port;
-	struct regmap *sysc = host->sysc;
+	struct rzg3s_sysc *sysc = host->sysc;
 	int ret;
 
 	ret = pm_runtime_put_sync(dev);
@@ -1625,31 +1795,30 @@ static int rzg3s_pcie_suspend_noirq(struct device *dev)
 
 	clk_disable_unprepare(port->refclk);
 
-	ret = reset_control_bulk_assert(data->num_power_resets,
-					host->power_resets);
+	/* SoC-specific de-initialization */
+	ret = data->config_deinit(host);
 	if (ret)
 		goto refclk_restore;
 
-	ret = reset_control_bulk_assert(data->num_cfg_resets,
-					host->cfg_resets);
+	ret = reset_control_bulk_assert(data->num_power_resets,
+					host->power_resets);
 	if (ret)
-		goto power_resets_restore;
+		goto config_reinit;
 
-	ret = regmap_update_bits(sysc, RZG3S_SYS_PCIE_RST_RSM_B,
-				 RZG3S_SYS_PCIE_RST_RSM_B_MASK,
-				 FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 0));
+	ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0);
 	if (ret)
-		goto cfg_resets_restore;
+		goto power_resets_restore;
 
 	return 0;
 
 	/* Restore the previous state if any error happens */
-cfg_resets_restore:
-	reset_control_bulk_deassert(data->num_cfg_resets,
-				    host->cfg_resets);
 power_resets_restore:
 	reset_control_bulk_deassert(data->num_power_resets,
 				    host->power_resets);
+config_reinit:
+	if (data->config_pre_init)
+		data->config_pre_init(host);
+	data->config_post_init(host);
 refclk_restore:
 	clk_prepare_enable(port->refclk);
 	pm_runtime_resume_and_get(dev);
@@ -1660,12 +1829,14 @@ static int rzg3s_pcie_resume_noirq(struct device *dev)
 {
 	struct rzg3s_pcie_host *host = dev_get_drvdata(dev);
 	const struct rzg3s_pcie_soc_data *data = host->data;
-	struct regmap *sysc = host->sysc;
+	struct rzg3s_sysc *sysc = host->sysc;
 	int ret;
 
-	ret = regmap_update_bits(sysc, RZG3S_SYS_PCIE_RST_RSM_B,
-				 RZG3S_SYS_PCIE_RST_RSM_B_MASK,
-				 FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 1));
+	ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_MODE, 1);
+	if (ret)
+		return ret;
+
+	ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 1);
 	if (ret)
 		return ret;
 
@@ -1694,9 +1865,7 @@ static int rzg3s_pcie_resume_noirq(struct device *dev)
 	reset_control_bulk_assert(data->num_power_resets,
 				  host->power_resets);
 assert_rst_rsm_b:
-	regmap_update_bits(sysc, RZG3S_SYS_PCIE_RST_RSM_B,
-			   RZG3S_SYS_PCIE_RST_RSM_B_MASK,
-			   FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 0));
+	rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0);
 	return ret;
 }
 
@@ -1718,7 +1887,39 @@ static const struct rzg3s_pcie_soc_data rzg3s_soc_data = {
 	.num_power_resets = ARRAY_SIZE(rzg3s_soc_power_resets),
 	.cfg_resets = rzg3s_soc_cfg_resets,
 	.num_cfg_resets = ARRAY_SIZE(rzg3s_soc_cfg_resets),
+	.config_post_init = rzg3s_pcie_config_post_init,
+	.config_deinit = rzg3s_pcie_config_deinit,
 	.init_phy = rzg3s_soc_pcie_init_phy,
+	.sysc_info = {
+		.functions = {
+			[RZG3S_SYSC_FUNC_ID_RST_RSM_B] = {
+				.offset = 0xd74,
+				.mask = BIT(0),
+			},
+		},
+	},
+};
+
+static const char * const rzg3e_soc_power_resets[] = { "aresetn" };
+
+static const struct rzg3s_pcie_soc_data rzg3e_soc_data = {
+	.power_resets = rzg3e_soc_power_resets,
+	.num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets),
+	.config_pre_init = rzg3e_pcie_config_pre_init,
+	.config_post_init = rzg3e_pcie_config_post_init,
+	.config_deinit = rzg3e_pcie_config_deinit,
+	.sysc_info = {
+		.functions = {
+			[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
+				.offset = 0x1020,
+				.mask = BIT(0),
+			},
+			[RZG3S_SYSC_FUNC_ID_MODE] = {
+				.offset = 0x1024,
+				.mask = BIT(0),
+			},
+		},
+	},
 };
 
 static const struct of_device_id rzg3s_pcie_of_match[] = {
@@ -1726,6 +1927,10 @@ static const struct of_device_id rzg3s_pcie_of_match[] = {
 		.compatible = "renesas,r9a08g045-pcie",
 		.data = &rzg3s_soc_data,
 	},
+	{
+		.compatible = "renesas,r9a09g047-pcie",
+		.data = &rzg3e_soc_data,
+	},
 	{}
 };
 
diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c
index f9cf18aa5b342..7f5326925ed54 100644
--- a/drivers/pci/endpoint/functions/pci-epf-mhi.c
+++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c
@@ -367,6 +367,8 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
 		dev_err(dev, "DMA transfer timeout\n");
 		dmaengine_terminate_sync(chan);
 		ret = -ETIMEDOUT;
+	} else {
+		ret = 0;
 	}
 
 err_unmap:
@@ -438,6 +440,8 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
 		dev_err(dev, "DMA transfer timeout\n");
 		dmaengine_terminate_sync(chan);
 		ret = -ETIMEDOUT;
+	} else {
+		ret = 0;
 	}
 
 err_unmap:
diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c
index a3a588e522e71..2bdcc35b652cf 100644
--- a/drivers/pci/endpoint/functions/pci-epf-ntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c
@@ -1494,47 +1494,6 @@ static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
 	return ret;
 }
 
-/**
- * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
- * @ntb: NTB device that facilitates communication between HOST1 and HOST2
- * @type: PRIMARY interface or SECONDARY interface
- *
- * Unbind NTB function device from EPC and relinquish reference to pci_epc
- * for each of the interface.
- */
-static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,
-					  enum pci_epc_interface_type type)
-{
-	struct epf_ntb_epc *ntb_epc;
-	struct pci_epc *epc;
-	struct pci_epf *epf;
-
-	if (type < 0)
-		return;
-
-	epf = ntb->epf;
-	ntb_epc = ntb->epc[type];
-	if (!ntb_epc)
-		return;
-	epc = ntb_epc->epc;
-	pci_epc_remove_epf(epc, epf, type);
-	pci_epc_put(epc);
-}
-
-/**
- * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
- * @ntb: NTB device that facilitates communication between HOST1 and HOST2
- *
- * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
- */
-static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
-{
-	enum pci_epc_interface_type type;
-
-	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
-		epf_ntb_epc_destroy_interface(ntb, type);
-}
-
 /**
  * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
@@ -1614,15 +1573,8 @@ static int epf_ntb_epc_create(struct epf_ntb *ntb)
 
 	ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,
 					   SECONDARY_INTERFACE);
-	if (ret) {
+	if (ret)
 		dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");
-		goto err_epc_create;
-	}
-
-	return 0;
-
-err_epc_create:
-	epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);
 
 	return ret;
 }
@@ -1887,7 +1839,7 @@ static int epf_ntb_bind(struct pci_epf *epf)
 	ret = epf_ntb_init_epc_bar(ntb);
 	if (ret) {
 		dev_err(dev, "Failed to create NTB EPC\n");
-		goto err_bar_init;
+		return ret;
 	}
 
 	ret = epf_ntb_config_spad_bar_alloc_interface(ntb);
@@ -1909,9 +1861,6 @@ static int epf_ntb_bind(struct pci_epf *epf)
 err_bar_alloc:
 	epf_ntb_config_spad_bar_free(ntb);
 
-err_bar_init:
-	epf_ntb_epc_destroy(ntb);
-
 	return ret;
 }
 
@@ -1927,7 +1876,6 @@ static void epf_ntb_unbind(struct pci_epf *epf)
 
 	epf_ntb_epc_cleanup(ntb);
 	epf_ntb_config_spad_bar_free(ntb);
-	epf_ntb_epc_destroy(ntb);
 }
 
 #define EPF_NTB_R(_name)						\
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 582938b7b4f1f..06bf53b28fb7a 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -64,6 +64,13 @@
 #define CAP_MSIX			BIT(2)
 #define CAP_INTX			BIT(3)
 #define CAP_SUBRANGE_MAPPING		BIT(4)
+#define CAP_DYNAMIC_INBOUND_MAPPING	BIT(5)
+#define CAP_BAR0_RESERVED		BIT(6)
+#define CAP_BAR1_RESERVED		BIT(7)
+#define CAP_BAR2_RESERVED		BIT(8)
+#define CAP_BAR3_RESERVED		BIT(9)
+#define CAP_BAR4_RESERVED		BIT(10)
+#define CAP_BAR5_RESERVED		BIT(11)
 
 #define PCI_EPF_TEST_BAR_SUBRANGE_NSUB	2
 
@@ -715,7 +722,6 @@ static void pci_epf_test_doorbell_cleanup(struct pci_epf_test *epf_test)
 	struct pci_epf_test_reg *reg = epf_test->reg[epf_test->test_reg_bar];
 	struct pci_epf *epf = epf_test->epf;
 
-	free_irq(epf->db_msg[0].virq, epf_test);
 	reg->doorbell_bar = cpu_to_le32(NO_BAR);
 
 	pci_epf_free_doorbell(epf);
@@ -759,7 +765,7 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
 					 &epf_test->db_bar.phys_addr, &offset);
 
 	if (ret)
-		goto err_doorbell_cleanup;
+		goto err_free_irq;
 
 	reg->doorbell_offset = cpu_to_le32(offset);
 
@@ -769,12 +775,14 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
 
 	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar);
 	if (ret)
-		goto err_doorbell_cleanup;
+		goto err_free_irq;
 
 	status |= STATUS_DOORBELL_ENABLE_SUCCESS;
 	reg->status = cpu_to_le32(status);
 	return;
 
+err_free_irq:
+	free_irq(epf->db_msg[0].virq, epf_test);
 err_doorbell_cleanup:
 	pci_epf_test_doorbell_cleanup(epf_test);
 set_status_err:
@@ -794,6 +802,7 @@ static void pci_epf_test_disable_doorbell(struct pci_epf_test *epf_test,
 	if (bar < BAR_0)
 		goto set_status_err;
 
+	free_irq(epf->db_msg[0].virq, epf_test);
 	pci_epf_test_doorbell_cleanup(epf_test);
 
 	/*
@@ -894,6 +903,11 @@ static void pci_epf_test_bar_subrange_setup(struct pci_epf_test *epf_test,
 		dev_err(&epf->dev, "pci_epc_set_bar() failed: %d\n", ret);
 		bar->submap = old_submap;
 		bar->num_submap = old_nsub;
+		ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, bar);
+		if (ret)
+			dev_warn(&epf->dev, "Failed to restore the original BAR mapping: %d\n",
+				 ret);
+
 		kfree(submap);
 		goto err;
 	}
@@ -1102,10 +1116,31 @@ static void pci_epf_test_set_capabilities(struct pci_epf *epf)
 	if (epf_test->epc_features->intx_capable)
 		caps |= CAP_INTX;
 
+	if (epf_test->epc_features->dynamic_inbound_mapping)
+		caps |= CAP_DYNAMIC_INBOUND_MAPPING;
+
 	if (epf_test->epc_features->dynamic_inbound_mapping &&
 	    epf_test->epc_features->subrange_mapping)
 		caps |= CAP_SUBRANGE_MAPPING;
 
+	if (epf_test->epc_features->bar[BAR_0].type == BAR_RESERVED)
+		caps |= CAP_BAR0_RESERVED;
+
+	if (epf_test->epc_features->bar[BAR_1].type == BAR_RESERVED)
+		caps |= CAP_BAR1_RESERVED;
+
+	if (epf_test->epc_features->bar[BAR_2].type == BAR_RESERVED)
+		caps |= CAP_BAR2_RESERVED;
+
+	if (epf_test->epc_features->bar[BAR_3].type == BAR_RESERVED)
+		caps |= CAP_BAR3_RESERVED;
+
+	if (epf_test->epc_features->bar[BAR_4].type == BAR_RESERVED)
+		caps |= CAP_BAR4_RESERVED;
+
+	if (epf_test->epc_features->bar[BAR_5].type == BAR_RESERVED)
+		caps |= CAP_BAR5_RESERVED;
+
 	reg->caps = cpu_to_le32(caps);
 }
 
diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index 20a400e834392..8053535289673 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -527,20 +527,20 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 	struct msi_msg *msg;
 	size_t sz;
 	int ret;
-	int i;
+	int i, req;
 
 	ret = pci_epf_alloc_doorbell(epf,  ntb->db_count);
 	if (ret)
 		return ret;
 
-	for (i = 0; i < ntb->db_count; i++) {
-		ret = request_irq(epf->db_msg[i].virq, epf_ntb_doorbell_handler,
+	for (req = 0; req < ntb->db_count; req++) {
+		ret = request_irq(epf->db_msg[req].virq, epf_ntb_doorbell_handler,
 				  0, "pci_epf_vntb_db", ntb);
 
 		if (ret) {
 			dev_err(&epf->dev,
 				"Failed to request doorbell IRQ: %d\n",
-				epf->db_msg[i].virq);
+				epf->db_msg[req].virq);
 			goto err_free_irq;
 		}
 	}
@@ -598,8 +598,8 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 	return 0;
 
 err_free_irq:
-	for (i--; i >= 0; i--)
-		free_irq(epf->db_msg[i].virq, ntb);
+	for (req--; req >= 0; req--)
+		free_irq(epf->db_msg[req].virq, ntb);
 
 	pci_epf_free_doorbell(ntb->epf);
 	return ret;
@@ -763,19 +763,6 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
 	}
 }
 
-/**
- * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
- * @ntb: NTB device that facilitates communication between HOST and VHOST
- *
- * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
- */
-static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
-{
-	pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
-	pci_epc_put(ntb->epf->epc);
-}
-
-
 /**
  * epf_ntb_is_bar_used() - Check if a bar is used in the ntb configuration
  * @ntb: NTB device that facilitates communication between HOST and VHOST
@@ -955,6 +942,7 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb)
  */
 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
 {
+	disable_delayed_work_sync(&ntb->cmd_handler);
 	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 	epf_ntb_db_bar_clear(ntb);
 	epf_ntb_config_sspad_bar_clear(ntb);
@@ -995,17 +983,19 @@ static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
 	struct config_group *group = to_config_group(item);		\
 	struct epf_ntb *ntb = to_epf_ntb(group);			\
 	struct device *dev = &ntb->epf->dev;				\
-	int win_no;							\
+	int win_no, idx;						\
 									\
 	if (sscanf(#_name, "mw%d", &win_no) != 1)			\
 		return -EINVAL;						\
 									\
-	if (win_no <= 0 || win_no > ntb->num_mws) {			\
-		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
-		return -EINVAL;						\
+	idx = win_no - 1;						\
+	if (idx < 0 || idx >= ntb->num_mws) {				\
+		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
+			win_no, ntb->num_mws);				\
+		return -ERANGE;						\
 	}								\
-									\
-	return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);	\
+	idx = array_index_nospec(idx, ntb->num_mws);			\
+	return sprintf(page, "%llu\n", ntb->mws_size[idx]);		\
 }
 
 #define EPF_NTB_MW_W(_name)						\
@@ -1015,7 +1005,7 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
 	struct config_group *group = to_config_group(item);		\
 	struct epf_ntb *ntb = to_epf_ntb(group);			\
 	struct device *dev = &ntb->epf->dev;				\
-	int win_no;							\
+	int win_no, idx;						\
 	u64 val;							\
 	int ret;							\
 									\
@@ -1026,12 +1016,14 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
 	if (sscanf(#_name, "mw%d", &win_no) != 1)			\
 		return -EINVAL;						\
 									\
-	if (win_no <= 0 || win_no > ntb->num_mws) {			\
-		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
-		return -EINVAL;						\
+	idx = win_no - 1;						\
+	if (idx < 0 || idx >= ntb->num_mws) {				\
+		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
+			win_no, ntb->num_mws);				\
+		return -ERANGE;						\
 	}								\
-									\
-	ntb->mws_size[win_no - 1] = val;				\
+	idx = array_index_nospec(idx, ntb->num_mws);			\
+	ntb->mws_size[idx] = val;					\
 									\
 	return len;							\
 }
@@ -1525,7 +1517,7 @@ static int epf_ntb_bind(struct pci_epf *epf)
 	ret = epf_ntb_init_epc_bar(ntb);
 	if (ret) {
 		dev_err(dev, "Failed to create NTB EPC\n");
-		goto err_bar_init;
+		return ret;
 	}
 
 	ret = epf_ntb_config_spad_bar_alloc(ntb);
@@ -1565,9 +1557,6 @@ static int epf_ntb_bind(struct pci_epf *epf)
 err_bar_alloc:
 	epf_ntb_config_spad_bar_free(ntb);
 
-err_bar_init:
-	epf_ntb_epc_destroy(ntb);
-
 	return ret;
 }
 
@@ -1583,7 +1572,6 @@ static void epf_ntb_unbind(struct pci_epf *epf)
 
 	epf_ntb_epc_cleanup(ntb);
 	epf_ntb_config_spad_bar_free(ntb);
-	epf_ntb_epc_destroy(ntb);
 
 	pci_unregister_driver(&vntb_pci_driver);
 }
diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c
index 0f3921f28f17e..076d115db02b3 100644
--- a/drivers/pci/endpoint/pci-ep-cfs.c
+++ b/drivers/pci/endpoint/pci-ep-cfs.c
@@ -84,7 +84,7 @@ static void pci_secondary_epc_epf_unlink(struct config_item *epf_item,
 	pci_epc_remove_epf(epc, epf, SECONDARY_INTERFACE);
 }
 
-static struct configfs_item_operations pci_secondary_epc_item_ops = {
+static const struct configfs_item_operations pci_secondary_epc_item_ops = {
 	.allow_link	= pci_secondary_epc_epf_link,
 	.drop_link	= pci_secondary_epc_epf_unlink,
 };
@@ -148,7 +148,7 @@ static void pci_primary_epc_epf_unlink(struct config_item *epf_item,
 	pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE);
 }
 
-static struct configfs_item_operations pci_primary_epc_item_ops = {
+static const struct configfs_item_operations pci_primary_epc_item_ops = {
 	.allow_link	= pci_primary_epc_epf_link,
 	.drop_link	= pci_primary_epc_epf_unlink,
 };
@@ -256,7 +256,7 @@ static void pci_epc_epf_unlink(struct config_item *epc_item,
 	pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE);
 }
 
-static struct configfs_item_operations pci_epc_item_ops = {
+static const struct configfs_item_operations pci_epc_item_ops = {
 	.allow_link	= pci_epc_epf_link,
 	.drop_link	= pci_epc_epf_unlink,
 };
@@ -507,7 +507,7 @@ static void pci_epf_release(struct config_item *item)
 	kfree(epf_group);
 }
 
-static struct configfs_item_operations pci_epf_ops = {
+static const struct configfs_item_operations pci_epf_ops = {
 	.allow_link		= pci_epf_vepf_link,
 	.drop_link		= pci_epf_vepf_unlink,
 	.release		= pci_epf_release,
@@ -624,8 +624,8 @@ static struct config_group *pci_epf_make(struct config_group *group,
 
 	epf = pci_epf_create(epf_name);
 	if (IS_ERR(epf)) {
-		pr_err("failed to create endpoint function device\n");
-		err = -EINVAL;
+		err = PTR_ERR(epf);
+		pr_err("failed to create endpoint function device: %d\n", err);
 		goto free_name;
 	}
 
@@ -657,7 +657,7 @@ static void pci_epf_drop(struct config_group *group, struct config_item *item)
 	config_item_put(item);
 }
 
-static struct configfs_group_operations pci_epf_group_ops = {
+static const struct configfs_group_operations pci_epf_group_ops = {
 	.make_group     = &pci_epf_make,
 	.drop_item      = &pci_epf_drop,
 };
diff --git a/drivers/pci/endpoint/pci-ep-msi.c b/drivers/pci/endpoint/pci-ep-msi.c
index 51c19942a81ef..1395919571f83 100644
--- a/drivers/pci/endpoint/pci-ep-msi.c
+++ b/drivers/pci/endpoint/pci-ep-msi.c
@@ -50,6 +50,9 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
 		return -EINVAL;
 	}
 
+	if (epf->db_msg)
+		return -EBUSY;
+
 	domain = of_msi_map_get_device_domain(epc->dev.parent, 0,
 					      DOMAIN_BUS_PLATFORM_MSI);
 	if (!domain) {
@@ -79,6 +82,8 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
 	if (ret) {
 		dev_err(dev, "Failed to allocate MSI\n");
 		kfree(msg);
+		epf->db_msg = NULL;
+		epf->num_db = 0;
 		return ret;
 	}
 
diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c
index e546b3dbb2403..6c3c58185fc5d 100644
--- a/drivers/pci/endpoint/pci-epc-core.c
+++ b/drivers/pci/endpoint/pci-epc-core.c
@@ -103,8 +103,9 @@ enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features
 		bar++;
 
 	for (i = bar; i < PCI_STD_NUM_BARS; i++) {
-		/* If the BAR is not reserved, return it. */
-		if (epc_features->bar[i].type != BAR_RESERVED)
+		/* If the BAR is not reserved or disabled, return it. */
+		if (epc_features->bar[i].type != BAR_RESERVED &&
+		    epc_features->bar[i].type != BAR_DISABLED)
 			return i;
 	}
 
diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
index 83fd20c112387..6987343c9e61c 100644
--- a/drivers/pci/endpoint/pci-epf-core.c
+++ b/drivers/pci/endpoint/pci-epf-core.c
@@ -149,7 +149,7 @@ EXPORT_SYMBOL_GPL(pci_epf_bind);
  * @epf_vf: the virtual EP function to be added
  *
  * A physical endpoint function can be associated with multiple virtual
- * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint
+ * endpoint functions. Invoke pci_epf_add_vepf() to add a virtual PCI endpoint
  * function to a physical PCI endpoint function.
  */
 int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 1e9158d7bac75..2cafd3b26f344 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -79,7 +79,8 @@ static int init_slot(struct controller *ctrl)
 	snprintf(name, SLOT_NAME_SIZE, "%u", PSN(ctrl));
 
 	retval = pci_hp_initialize(&ctrl->hotplug_slot,
-				   ctrl->pcie->port->subordinate, 0, name);
+				   ctrl->pcie->port->subordinate,
+				   PCI_SLOT_ALL_DEVICES, name);
 	if (retval) {
 		ctrl_err(ctrl, "pci_hp_initialize failed: error %d\n", retval);
 		kfree(ops);
diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c
index 5c020831e318f..ff92a5c301b81 100644
--- a/drivers/pci/hotplug/pnv_php.c
+++ b/drivers/pci/hotplug/pnv_php.c
@@ -215,24 +215,19 @@ static void pnv_php_reverse_nodes(struct device_node *parent)
 static int pnv_php_populate_changeset(struct of_changeset *ocs,
 				      struct device_node *dn)
 {
-	struct device_node *child;
-	int ret = 0;
+	int ret;
 
-	for_each_child_of_node(dn, child) {
+	for_each_child_of_node_scoped(dn, child) {
 		ret = of_changeset_attach_node(ocs, child);
-		if (ret) {
-			of_node_put(child);
-			break;
-		}
+		if (ret)
+			return ret;
 
 		ret = pnv_php_populate_changeset(ocs, child);
-		if (ret) {
-			of_node_put(child);
-			break;
-		}
+		if (ret)
+			return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 static void *pnv_php_add_one_pdn(struct device_node *dn, void *data)
diff --git a/drivers/pci/hotplug/rpaphp_slot.c b/drivers/pci/hotplug/rpaphp_slot.c
index 33ca19200c1b5..67362e5b99719 100644
--- a/drivers/pci/hotplug/rpaphp_slot.c
+++ b/drivers/pci/hotplug/rpaphp_slot.c
@@ -82,7 +82,6 @@ EXPORT_SYMBOL_GPL(rpaphp_deregister_slot);
 int rpaphp_register_slot(struct slot *slot)
 {
 	struct hotplug_slot *php_slot = &slot->hotplug_slot;
-	struct device_node *child;
 	u32 my_index;
 	int retval;
 	int slotno = -1;
@@ -97,11 +96,10 @@ int rpaphp_register_slot(struct slot *slot)
 		return -EAGAIN;
 	}
 
-	for_each_child_of_node(slot->dn, child) {
+	for_each_child_of_node_scoped(slot->dn, child) {
 		retval = of_property_read_u32(child, "ibm,my-drc-index", &my_index);
 		if (my_index == slot->index) {
 			slotno = PCI_SLOT(PCI_DN(child)->devfn);
-			of_node_put(child);
 			break;
 		}
 	}
diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c
index 818d55fbad0df..c18559b6272c0 100644
--- a/drivers/pci/msi/api.c
+++ b/drivers/pci/msi/api.c
@@ -370,6 +370,11 @@ EXPORT_SYMBOL(pci_irq_get_affinity);
  * Undo the interrupt vector allocations and possible device MSI/MSI-X
  * enablement earlier done through pci_alloc_irq_vectors_affinity() or
  * pci_alloc_irq_vectors().
+ *
+ * WARNING: Do not call this function if the device has been enabled
+ * with pcim_enable_device(). In that case, IRQ vectors are automatically
+ * managed via pcim_msi_release() and calling pci_free_irq_vectors() can
+ * lead to double-free issues.
  */
 void pci_free_irq_vectors(struct pci_dev *dev)
 {
diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c
index e2412175d7af2..81d24a270a795 100644
--- a/drivers/pci/msi/msi.c
+++ b/drivers/pci/msi/msi.c
@@ -77,6 +77,16 @@ static void pcim_msi_release(void *pcidev)
 /*
  * Needs to be separate from pcim_release to prevent an ordering problem
  * vs. msi_device_data_release() in the MSI core code.
+ *
+ * TODO: Remove the legacy side-effect of pcim_enable_device() that
+ * activates automatic IRQ vector management. This design is dangerous
+ * and confusing because it switches normally un-managed functions
+ * into managed mode. Drivers should explicitly manage their IRQ vectors
+ * without this implicit behavior.
+ *
+ * The current implementation uses both pdev->is_managed and
+ * pdev->is_msi_managed flags, which adds unnecessary complexity.
+ * This should be simplified in a future kernel version.
  */
 static int pcim_setup_msi_release(struct pci_dev *dev)
 {
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 9f8eb5df279ed..0eba56e0ebcde 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -775,7 +775,7 @@ void of_pci_make_host_bridge_node(struct pci_host_bridge *bridge)
 
 	/* Check if there is a DT root node to attach the created node */
 	if (!of_root) {
-		pr_err("of_root node is NULL, cannot create PCI host bridge node\n");
+		pr_debug("of_root node is NULL, cannot create PCI host bridge node\n");
 		return;
 	}
 
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 16eaaf749ba97..a2f8a5d6190fd 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -553,7 +553,6 @@ static ssize_t reset_subordinate_store(struct device *dev,
 				const char *buf, size_t count)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
-	struct pci_bus *bus = pdev->subordinate;
 	unsigned long val;
 
 	if (!capable(CAP_SYS_ADMIN))
@@ -563,7 +562,7 @@ static ssize_t reset_subordinate_store(struct device *dev,
 		return -EINVAL;
 
 	if (val) {
-		int ret = __pci_reset_bus(bus);
+		int ret = pci_try_reset_bridge(pdev);
 
 		if (ret)
 			return ret;
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 8479c2e1f74f1..b4b99fb32731d 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -949,7 +949,7 @@ static void __pci_config_acs(struct pci_dev *dev, struct pci_acs *caps,
 
 		ret = pci_dev_str_match(dev, p, &p);
 		if (ret < 0) {
-			pr_info_once("PCI: Can't parse ACS command line parameter\n");
+			pr_warn_once("PCI: Can't parse ACS command line parameter\n");
 			break;
 		} else if (ret == 1) {
 			/* Found a match */
@@ -2241,10 +2241,9 @@ EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
 #ifdef CONFIG_PCIEAER
 void pcie_clear_device_status(struct pci_dev *dev)
 {
-	u16 sta;
-
-	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
-	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
+	pcie_capability_write_word(dev, PCI_EXP_DEVSTA,
+				   PCI_EXP_DEVSTA_CED | PCI_EXP_DEVSTA_NFED |
+				   PCI_EXP_DEVSTA_FED | PCI_EXP_DEVSTA_URD);
 }
 #endif
 
@@ -4914,12 +4913,8 @@ static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
 	 * If "dev" is below a CXL port that has SBR control masked, SBR
 	 * won't do anything, so return error.
 	 */
-	if (bridge && cxl_sbr_masked(bridge)) {
-		if (probe)
-			return 0;
-
+	if (bridge && pcie_is_cxl(bridge) && cxl_sbr_masked(bridge))
 		return -ENOTTY;
-	}
 
 	rc = pci_dev_reset_iommu_prepare(dev);
 	if (rc) {
@@ -5292,13 +5287,21 @@ static bool pci_bus_resettable(struct pci_bus *bus)
 	return true;
 }
 
+static void pci_bus_lock(struct pci_bus *bus);
+static void pci_bus_unlock(struct pci_bus *bus);
+static int pci_bus_trylock(struct pci_bus *bus);
+
 /* Lock devices from the top of the tree down */
-static void pci_bus_lock(struct pci_bus *bus)
+static void __pci_bus_lock(struct pci_bus *bus, struct pci_slot *slot)
 {
-	struct pci_dev *dev;
+	struct pci_dev *dev, *bridge = bus->self;
+
+	if (bridge)
+		pci_dev_lock(bridge);
 
-	pci_dev_lock(bus->self);
 	list_for_each_entry(dev, &bus->devices, bus_list) {
+		if (slot && (!dev->slot || dev->slot != slot))
+			continue;
 		if (dev->subordinate)
 			pci_bus_lock(dev->subordinate);
 		else
@@ -5307,28 +5310,34 @@ static void pci_bus_lock(struct pci_bus *bus)
 }
 
 /* Unlock devices from the bottom of the tree up */
-static void pci_bus_unlock(struct pci_bus *bus)
+static void __pci_bus_unlock(struct pci_bus *bus, struct pci_slot *slot)
 {
-	struct pci_dev *dev;
+	struct pci_dev *dev, *bridge = bus->self;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
+		if (slot && (!dev->slot || dev->slot != slot))
+			continue;
 		if (dev->subordinate)
 			pci_bus_unlock(dev->subordinate);
 		else
 			pci_dev_unlock(dev);
 	}
-	pci_dev_unlock(bus->self);
+
+	if (bridge)
+		pci_dev_unlock(bridge);
 }
 
 /* Return 1 on successful lock, 0 on contention */
-static int pci_bus_trylock(struct pci_bus *bus)
+static int __pci_bus_trylock(struct pci_bus *bus, struct pci_slot *slot)
 {
-	struct pci_dev *dev;
+	struct pci_dev *dev, *bridge = bus->self;
 
-	if (!pci_dev_trylock(bus->self))
+	if (bridge && !pci_dev_trylock(bridge))
 		return 0;
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
+		if (slot && (!dev->slot || dev->slot != slot))
+			continue;
 		if (dev->subordinate) {
 			if (!pci_bus_trylock(dev->subordinate))
 				goto unlock;
@@ -5339,15 +5348,37 @@ static int pci_bus_trylock(struct pci_bus *bus)
 
 unlock:
 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
+		if (slot && (!dev->slot || dev->slot != slot))
+			continue;
 		if (dev->subordinate)
 			pci_bus_unlock(dev->subordinate);
 		else
 			pci_dev_unlock(dev);
 	}
-	pci_dev_unlock(bus->self);
+
+	if (bridge)
+		pci_dev_unlock(bridge);
 	return 0;
 }
 
+/* Lock devices from the top of the tree down */
+static void pci_bus_lock(struct pci_bus *bus)
+{
+	__pci_bus_lock(bus, NULL);
+}
+
+/* Unlock devices from the bottom of the tree up */
+static void pci_bus_unlock(struct pci_bus *bus)
+{
+	__pci_bus_unlock(bus, NULL);
+}
+
+/* Return 1 on successful lock, 0 on contention */
+static int pci_bus_trylock(struct pci_bus *bus)
+{
+	return __pci_bus_trylock(bus, NULL);
+}
+
 /* Do any devices on or below this slot prevent a bus reset? */
 static bool pci_slot_resettable(struct pci_slot *slot)
 {
@@ -5370,72 +5401,19 @@ static bool pci_slot_resettable(struct pci_slot *slot)
 /* Lock devices from the top of the tree down */
 static void pci_slot_lock(struct pci_slot *slot)
 {
-	struct pci_dev *dev, *bridge = slot->bus->self;
-
-	if (bridge)
-		pci_dev_lock(bridge);
-
-	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
-		if (!dev->slot || dev->slot != slot)
-			continue;
-		if (dev->subordinate)
-			pci_bus_lock(dev->subordinate);
-		else
-			pci_dev_lock(dev);
-	}
+	__pci_bus_lock(slot->bus, slot);
 }
 
 /* Unlock devices from the bottom of the tree up */
 static void pci_slot_unlock(struct pci_slot *slot)
 {
-	struct pci_dev *dev, *bridge = slot->bus->self;
-
-	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
-		if (!dev->slot || dev->slot != slot)
-			continue;
-		if (dev->subordinate)
-			pci_bus_unlock(dev->subordinate);
-		else
-			pci_dev_unlock(dev);
-	}
-
-	if (bridge)
-		pci_dev_unlock(bridge);
+	__pci_bus_unlock(slot->bus, slot);
 }
 
 /* Return 1 on successful lock, 0 on contention */
 static int pci_slot_trylock(struct pci_slot *slot)
 {
-	struct pci_dev *dev, *bridge = slot->bus->self;
-
-	if (bridge && !pci_dev_trylock(bridge))
-		return 0;
-
-	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
-		if (!dev->slot || dev->slot != slot)
-			continue;
-		if (dev->subordinate) {
-			if (!pci_bus_trylock(dev->subordinate))
-				goto unlock;
-		} else if (!pci_dev_trylock(dev))
-			goto unlock;
-	}
-	return 1;
-
-unlock:
-	list_for_each_entry_continue_reverse(dev,
-					     &slot->bus->devices, bus_list) {
-		if (!dev->slot || dev->slot != slot)
-			continue;
-		if (dev->subordinate)
-			pci_bus_unlock(dev->subordinate);
-		else
-			pci_dev_unlock(dev);
-	}
-
-	if (bridge)
-		pci_dev_unlock(bridge);
-	return 0;
+	return __pci_bus_trylock(slot->bus, slot);
 }
 
 /*
@@ -5541,7 +5519,7 @@ int pci_probe_reset_slot(struct pci_slot *slot)
 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
 
 /**
- * __pci_reset_slot - Try to reset a PCI slot
+ * pci_try_reset_slot - Try to reset a PCI slot
  * @slot: PCI slot to reset
  *
  * A PCI bus may host multiple slots, each slot may support a reset mechanism
@@ -5555,7 +5533,7 @@ EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
  *
  * Same as above except return -EAGAIN if the slot cannot be locked
  */
-static int __pci_reset_slot(struct pci_slot *slot)
+static int pci_try_reset_slot(struct pci_slot *slot)
 {
 	int rc;
 
@@ -5597,14 +5575,44 @@ static int pci_bus_reset(struct pci_bus *bus, bool probe)
 }
 
 /**
- * pci_bus_error_reset - reset the bridge's subordinate bus
- * @bridge: The parent device that connects to the bus to reset
+ * pci_try_reset_bus - Try to reset a PCI bus
+ * @bus: top level PCI bus to reset
+ *
+ * Same as above except return -EAGAIN if the bus cannot be locked
+ */
+static int pci_try_reset_bus(struct pci_bus *bus)
+{
+	int rc;
+
+	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
+	if (rc)
+		return rc;
+
+	if (pci_bus_trylock(bus)) {
+		pci_bus_save_and_disable_locked(bus);
+		might_sleep();
+		rc = pci_bridge_secondary_bus_reset(bus->self);
+		pci_bus_restore_locked(bus);
+		pci_bus_unlock(bus);
+	} else
+		rc = -EAGAIN;
+
+	return rc;
+}
+
+#define PCI_RESET_RESTORE true
+#define PCI_RESET_NO_RESTORE false
+/**
+ * pci_reset_bridge - reset a bridge's subordinate bus
+ * @bridge: bridge that connects to the bus to reset
+ * @restore: when true use a reset method that invokes pci_dev_restore() post
+ *           reset for affected devices
  *
  * This function will first try to reset the slots on this bus if the method is
  * available. If slot reset fails or is not available, this will fall back to a
  * secondary bus reset.
  */
-int pci_bus_error_reset(struct pci_dev *bridge)
+static int pci_reset_bridge(struct pci_dev *bridge, bool restore)
 {
 	struct pci_bus *bus = bridge->subordinate;
 	struct pci_slot *slot;
@@ -5620,17 +5628,42 @@ int pci_bus_error_reset(struct pci_dev *bridge)
 		if (pci_probe_reset_slot(slot))
 			goto bus_reset;
 
-	list_for_each_entry(slot, &bus->slots, list)
-		if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
+	list_for_each_entry(slot, &bus->slots, list) {
+		int ret;
+
+		if (restore)
+			ret = pci_try_reset_slot(slot);
+		else
+			ret = pci_slot_reset(slot, PCI_RESET_DO_RESET);
+
+		if (ret)
 			goto bus_reset;
+	}
 
 	mutex_unlock(&pci_slot_mutex);
 	return 0;
 bus_reset:
 	mutex_unlock(&pci_slot_mutex);
+
+	if (restore)
+		return pci_try_reset_bus(bus);
 	return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET);
 }
 
+/**
+ * pci_bus_error_reset - reset the bridge's subordinate bus
+ * @bridge: The parent device that connects to the bus to reset
+ */
+int pci_bus_error_reset(struct pci_dev *bridge)
+{
+	return pci_reset_bridge(bridge, PCI_RESET_NO_RESTORE);
+}
+
+int pci_try_reset_bridge(struct pci_dev *bridge)
+{
+	return pci_reset_bridge(bridge, PCI_RESET_RESTORE);
+}
+
 /**
  * pci_probe_reset_bus - probe whether a PCI bus can be reset
  * @bus: PCI bus to probe
@@ -5643,32 +5676,6 @@ int pci_probe_reset_bus(struct pci_bus *bus)
 }
 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
 
-/**
- * __pci_reset_bus - Try to reset a PCI bus
- * @bus: top level PCI bus to reset
- *
- * Same as above except return -EAGAIN if the bus cannot be locked
- */
-int __pci_reset_bus(struct pci_bus *bus)
-{
-	int rc;
-
-	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
-	if (rc)
-		return rc;
-
-	if (pci_bus_trylock(bus)) {
-		pci_bus_save_and_disable_locked(bus);
-		might_sleep();
-		rc = pci_bridge_secondary_bus_reset(bus->self);
-		pci_bus_restore_locked(bus);
-		pci_bus_unlock(bus);
-	} else
-		rc = -EAGAIN;
-
-	return rc;
-}
-
 /**
  * pci_reset_bus - Try to reset a PCI bus
  * @pdev: top level PCI device to reset via slot/bus
@@ -5678,7 +5685,7 @@ int __pci_reset_bus(struct pci_bus *bus)
 int pci_reset_bus(struct pci_dev *pdev)
 {
 	return (!pci_probe_reset_slot(pdev->slot)) ?
-	    __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
+	    pci_try_reset_slot(pdev->slot) : pci_try_reset_bus(pdev->bus);
 }
 EXPORT_SYMBOL_GPL(pci_reset_bus);
 
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 13d998fbacce6..e542d1bf28538 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -231,7 +231,7 @@ bool pci_reset_supported(struct pci_dev *dev);
 void pci_init_reset_methods(struct pci_dev *dev);
 int pci_bridge_secondary_bus_reset(struct pci_dev *dev);
 int pci_bus_error_reset(struct pci_dev *dev);
-int __pci_reset_bus(struct pci_bus *bus);
+int pci_try_reset_bridge(struct pci_dev *bridge);
 
 struct pci_cap_saved_data {
 	u16		cap_nr;
@@ -859,6 +859,7 @@ struct aer_err_info {
 };
 
 int aer_get_device_error_info(struct aer_err_info *info, int i);
+void aer_print_init(struct pci_dev *dev, struct aer_err_info *info, int i);
 void aer_print_error(struct aer_err_info *info, int i);
 
 static inline const char *aer_err_bus(struct aer_err_info *info)
diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c
index d916378bc7072..bfafcc3add659 100644
--- a/drivers/pci/pcie/aer.c
+++ b/drivers/pci/pcie/aer.c
@@ -977,6 +977,28 @@ void pci_print_aer(struct pci_dev *dev, int aer_severity,
 }
 EXPORT_SYMBOL_GPL(pci_print_aer);
 
+/**
+ * aer_print_init - set flag whether error message should be printed
+ * @dev: pointer to pci_dev to be rate-limited
+ * @info: pointer to aer_error_info
+ * @i: index for dev array in aer_error_info
+ */
+void aer_print_init(struct pci_dev *dev, struct aer_err_info *info, int i)
+{
+	/*
+	 * Ratelimit AER log messages.  "dev" is either the source
+	 * identified by the root's Error Source ID or it has an unmasked
+	 * error logged in its own AER Capability.  Messages are emitted
+	 * when "ratelimit_print[i]" is non-zero.  If we will print detail
+	 * for a downstream device, make sure we print the Error Source ID
+	 * from the root as well.
+	 */
+	if (aer_ratelimit(dev, info->severity)) {
+		info->ratelimit_print[i] = 1;
+		info->root_ratelimit_print = 1;
+	}
+}
+
 /**
  * add_error_device - list device to be handled
  * @e_info: pointer to error info
@@ -992,18 +1014,7 @@ static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
 	e_info->dev[i] = pci_dev_get(dev);
 	e_info->error_dev_num++;
 
-	/*
-	 * Ratelimit AER log messages.  "dev" is either the source
-	 * identified by the root's Error Source ID or it has an unmasked
-	 * error logged in its own AER Capability.  Messages are emitted
-	 * when "ratelimit_print[i]" is non-zero.  If we will print detail
-	 * for a downstream device, make sure we print the Error Source ID
-	 * from the root as well.
-	 */
-	if (aer_ratelimit(dev, e_info->severity)) {
-		e_info->ratelimit_print[i] = 1;
-		e_info->root_ratelimit_print = 1;
-	}
+	aer_print_init(dev, e_info, i);
 	return 0;
 }
 
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 21f5d23e0b61b..925373b98dff0 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -706,22 +706,29 @@ static void aspm_calc_l12_info(struct pcie_link_state *link,
 	}
 
 	/* Program T_POWER_ON times in both ports */
-	pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
-	pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
+	pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2,
+				       PCI_L1SS_CTL2_T_PWR_ON_VALUE |
+				       PCI_L1SS_CTL2_T_PWR_ON_SCALE, ctl2);
+	pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL2,
+				       PCI_L1SS_CTL2_T_PWR_ON_VALUE |
+				       PCI_L1SS_CTL2_T_PWR_ON_SCALE, ctl2);
 
 	/* Program Common_Mode_Restore_Time in upstream device */
 	pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
-				       PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
+				       PCI_L1SS_CTL1_CM_RESTORE_TIME,
+				       ctl1 & PCI_L1SS_CTL1_CM_RESTORE_TIME);
 
 	/* Program LTR_L1.2_THRESHOLD time in both ports */
 	pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
 				       PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
 				       PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
-				       ctl1);
+				       ctl1 & (PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+					       PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
 	pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
 				       PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
 				       PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
-				       ctl1);
+				       ctl1 & (PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+					       PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
 
 	if (pl1_2_enables || cl1_2_enables) {
 		pci_clear_and_set_config_dword(parent,
diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c
index fc18349614d7c..99ddd3b8960fb 100644
--- a/drivers/pci/pcie/dpc.c
+++ b/drivers/pci/pcie/dpc.c
@@ -275,6 +275,7 @@ void dpc_process_error(struct pci_dev *pdev)
 			 status);
 		if (dpc_get_aer_uncorrect_severity(pdev, &info) &&
 		    aer_get_device_error_info(&info, 0)) {
+			aer_print_init(pdev, &info, 0);
 			aer_print_error(&info, 0);
 			pci_aer_clear_nonfatal_status(pdev);
 			pci_aer_clear_fatal_status(pdev);
@@ -372,11 +373,13 @@ static irqreturn_t dpc_handler(int irq, void *context)
 		return IRQ_HANDLED;
 	}
 
+	pci_dev_get(pdev);
 	dpc_process_error(pdev);
 
 	/* We configure DPC so it only triggers on ERR_FATAL */
 	pcie_do_recovery(pdev, pci_channel_io_frozen, dpc_reset_link);
 
+	pci_dev_put(pdev);
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c
index 91a598ed534ce..a41ffd1914de4 100644
--- a/drivers/pci/pcie/ptm.c
+++ b/drivers/pci/pcie/ptm.c
@@ -52,6 +52,7 @@ void pci_ptm_init(struct pci_dev *dev)
 		return;
 
 	dev->ptm_cap = ptm;
+	atomic_set(&dev->ptm_enable_cnt, 0);
 	pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_PTM, sizeof(u32));
 
 	pci_read_config_dword(dev, ptm + PCI_PTM_CAP, &cap);
@@ -85,10 +86,6 @@ void pci_ptm_init(struct pci_dev *dev)
 		dev->ptm_responder = 1;
 	if (cap & PCI_PTM_CAP_REQ)
 		dev->ptm_requester = 1;
-
-	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
-	    pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM)
-		pci_enable_ptm(dev, NULL);
 }
 
 void pci_save_ptm_state(struct pci_dev *dev)
@@ -129,26 +126,11 @@ void pci_restore_ptm_state(struct pci_dev *dev)
 static int __pci_enable_ptm(struct pci_dev *dev)
 {
 	u16 ptm = dev->ptm_cap;
-	struct pci_dev *ups;
 	u32 ctrl;
 
 	if (!ptm)
 		return -EINVAL;
 
-	/*
-	 * A device uses local PTM Messages to request time information
-	 * from a PTM Root that's farther upstream.  Every device along the
-	 * path must support PTM and have it enabled so it can handle the
-	 * messages.  Therefore, if this device is not a PTM Root, the
-	 * upstream link partner must have PTM enabled before we can enable
-	 * PTM.
-	 */
-	if (!dev->ptm_root) {
-		ups = pci_upstream_ptm(dev);
-		if (!ups || !ups->ptm_enabled)
-			return -EINVAL;
-	}
-
 	switch (pci_pcie_type(dev)) {
 	case PCI_EXP_TYPE_ROOT_PORT:
 		if (!dev->ptm_root)
@@ -182,27 +164,46 @@ static int __pci_enable_ptm(struct pci_dev *dev)
 /**
  * pci_enable_ptm() - Enable Precision Time Measurement
  * @dev: PCI device
- * @granularity: pointer to return granularity
  *
- * Enable Precision Time Measurement for @dev.  If successful and
- * @granularity is non-NULL, return the Effective Granularity.
+ * Enable Precision Time Measurement for @dev.
  *
  * Return: zero if successful, or -EINVAL if @dev lacks a PTM Capability or
  * is not a PTM Root and lacks an upstream path of PTM-enabled devices.
  */
-int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
+int pci_enable_ptm(struct pci_dev *dev)
 {
 	int rc;
 	char clock_desc[8];
 
-	rc = __pci_enable_ptm(dev);
-	if (rc)
-		return rc;
+	/*
+	 * A device uses local PTM Messages to request time information
+	 * from a PTM Root that's farther upstream. Every device along
+	 * the path must support PTM and have it enabled so it can
+	 * handle the messages. Therefore, if this device is not a PTM
+	 * Root, the upstream link partner must have PTM enabled before
+	 * we can enable PTM.
+	 */
+	if (!dev->ptm_root) {
+		struct pci_dev *parent;
+
+		parent = pci_upstream_ptm(dev);
+		if (!parent)
+			return -EINVAL;
+		/* Enable PTM for the parent */
+		rc = pci_enable_ptm(parent);
+		if (rc)
+			return rc;
+	}
 
-	dev->ptm_enabled = 1;
+	/* Already enabled? */
+	if (atomic_inc_return(&dev->ptm_enable_cnt) > 1)
+		return 0;
 
-	if (granularity)
-		*granularity = dev->ptm_granularity;
+	rc = __pci_enable_ptm(dev);
+	if (rc) {
+		atomic_dec(&dev->ptm_enable_cnt);
+		return rc;
+	}
 
 	switch (dev->ptm_granularity) {
 	case 0:
@@ -244,27 +245,31 @@ static void __pci_disable_ptm(struct pci_dev *dev)
  */
 void pci_disable_ptm(struct pci_dev *dev)
 {
-	if (dev->ptm_enabled) {
+	struct pci_dev *parent;
+
+	if (atomic_dec_and_test(&dev->ptm_enable_cnt))
 		__pci_disable_ptm(dev);
-		dev->ptm_enabled = 0;
-	}
+
+	parent = pci_upstream_ptm(dev);
+	if (parent)
+		pci_disable_ptm(parent);
 }
 EXPORT_SYMBOL(pci_disable_ptm);
 
 /*
- * Disable PTM, but preserve dev->ptm_enabled so we silently re-enable it on
+ * Disable PTM, but preserve dev->ptm_enable_cnt so we silently re-enable it on
  * resume if necessary.
  */
 void pci_suspend_ptm(struct pci_dev *dev)
 {
-	if (dev->ptm_enabled)
+	if (atomic_read(&dev->ptm_enable_cnt))
 		__pci_disable_ptm(dev);
 }
 
 /* If PTM was enabled before suspend, re-enable it when resuming */
 void pci_resume_ptm(struct pci_dev *dev)
 {
-	if (dev->ptm_enabled)
+	if (atomic_read(&dev->ptm_enable_cnt))
 		__pci_enable_ptm(dev);
 }
 
@@ -273,7 +278,7 @@ bool pcie_ptm_enabled(struct pci_dev *dev)
 	if (!dev)
 		return false;
 
-	return dev->ptm_enabled;
+	return atomic_read(&dev->ptm_enable_cnt);
 }
 EXPORT_SYMBOL(pcie_ptm_enabled);
 
diff --git a/drivers/pci/pwrctrl/Kconfig b/drivers/pci/pwrctrl/Kconfig
index cd3aa15bad009..9eec767cda86e 100644
--- a/drivers/pci/pwrctrl/Kconfig
+++ b/drivers/pci/pwrctrl/Kconfig
@@ -11,17 +11,18 @@ config PCI_PWRCTRL_PWRSEQ
 	select POWER_SEQUENCING
 	select PCI_PWRCTRL
 
-config PCI_PWRCTRL_SLOT
-	tristate "PCI Power Control driver for PCI slots"
+config PCI_PWRCTRL_GENERIC
+	tristate "Generic PCI Power Control driver for PCI slots and endpoints"
 	select POWER_SEQUENCING
 	select PCI_PWRCTRL
 	help
-	  Say Y here to enable the PCI Power Control driver to control the power
-	  state of PCI slots.
+	  Say Y here to enable the generic PCI Power Control driver to control
+	  the power state of PCI slots and endpoints.
 
 	  This is a generic driver that controls the power state of different
-	  PCI slots. The voltage regulators powering the rails of the PCI slots
-	  are expected to be defined in the devicetree node of the PCI bridge.
+	  PCI slots and endpoints. The voltage regulators powering the rails
+	  of the PCI slots or endpoints are expected to be defined in the
+	  devicetree node of the PCI bridge or endpoint.
 
 config PCI_PWRCTRL_TC9563
 	tristate "PCI Power Control driver for TC9563 PCIe switch"
diff --git a/drivers/pci/pwrctrl/Makefile b/drivers/pci/pwrctrl/Makefile
index 13b02282106c2..f6bb4fb9a410b 100644
--- a/drivers/pci/pwrctrl/Makefile
+++ b/drivers/pci/pwrctrl/Makefile
@@ -5,7 +5,7 @@ pci-pwrctrl-core-y			:= core.o
 
 obj-$(CONFIG_PCI_PWRCTRL_PWRSEQ)	+= pci-pwrctrl-pwrseq.o
 
-obj-$(CONFIG_PCI_PWRCTRL_SLOT)		+= pci-pwrctrl-slot.o
-pci-pwrctrl-slot-y			:= slot.o
+obj-$(CONFIG_PCI_PWRCTRL_GENERIC)	+= pci-pwrctrl-generic.o
+pci-pwrctrl-generic-y			:= generic.o
 
 obj-$(CONFIG_PCI_PWRCTRL_TC9563)	+= pci-pwrctrl-tc9563.o
diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c
index 6f7dea6746e0e..7754baed67f22 100644
--- a/drivers/pci/pwrctrl/core.c
+++ b/drivers/pci/pwrctrl/core.c
@@ -268,6 +268,46 @@ int pci_pwrctrl_power_on_devices(struct device *parent)
 }
 EXPORT_SYMBOL_GPL(pci_pwrctrl_power_on_devices);
 
+/*
+ * Check whether the pwrctrl device really needs to be created or not. The
+ * pwrctrl device will only be created if the node satisfies below requirements:
+ *
+ * 1. Presence of compatible property with "pci" prefix to match against the
+ *    pwrctrl driver (AND)
+ * 2. At least one of the power supplies defined in the devicetree node of the
+ *    device (OR) in the remote endpoint parent node to indicate pwrctrl
+ *    requirement.
+ */
+static bool pci_pwrctrl_is_required(struct device_node *np)
+{
+	struct device_node *endpoint;
+	const char *compat;
+	int ret;
+
+	ret = of_property_read_string(np, "compatible", &compat);
+	if (ret < 0)
+		return false;
+
+	if (!strstarts(compat, "pci"))
+		return false;
+
+	if (of_pci_supply_present(np))
+		return true;
+
+	if (of_graph_is_present(np)) {
+		for_each_endpoint_of_node(np, endpoint) {
+			struct device_node *remote __free(device_node) =
+				of_graph_get_remote_port_parent(endpoint);
+			if (remote) {
+				if (of_pci_supply_present(remote))
+					return true;
+			}
+		}
+	}
+
+	return false;
+}
+
 static int pci_pwrctrl_create_device(struct device_node *np,
 				     struct device *parent)
 {
@@ -287,19 +327,7 @@ static int pci_pwrctrl_create_device(struct device_node *np,
 		return 0;
 	}
 
-	/*
-	 * Sanity check to make sure that the node has the compatible property
-	 * to allow driver binding.
-	 */
-	if (!of_property_present(np, "compatible"))
-		return 0;
-
-	/*
-	 * Check whether the pwrctrl device really needs to be created or not.
-	 * This is decided based on at least one of the power supplies defined
-	 * in the devicetree node of the device or the graph property.
-	 */
-	if (!of_pci_supply_present(np) && !of_graph_is_present(np)) {
+	if (!pci_pwrctrl_is_required(np)) {
 		dev_dbg(parent, "Skipping OF node: %s\n", np->name);
 		return 0;
 	}
diff --git a/drivers/pci/pwrctrl/generic.c b/drivers/pci/pwrctrl/generic.c
new file mode 100644
index 0000000000000..fcedd435f7238
--- /dev/null
+++ b/drivers/pci/pwrctrl/generic.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Linaro Ltd.
+ * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/pci-pwrctrl.h>
+#include <linux/platform_device.h>
+#include <linux/pwrseq/consumer.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+struct slot_pwrctrl {
+	struct pci_pwrctrl pwrctrl;
+	struct regulator_bulk_data *supplies;
+	int num_supplies;
+	struct clk *clk;
+	struct pwrseq_desc *pwrseq;
+};
+
+static int slot_pwrctrl_power_on(struct pci_pwrctrl *pwrctrl)
+{
+	struct slot_pwrctrl *slot = container_of(pwrctrl,
+						struct slot_pwrctrl, pwrctrl);
+	int ret;
+
+	if (slot->pwrseq) {
+		pwrseq_power_on(slot->pwrseq);
+		return 0;
+	}
+
+	ret = regulator_bulk_enable(slot->num_supplies, slot->supplies);
+	if (ret < 0) {
+		dev_err(slot->pwrctrl.dev, "Failed to enable slot regulators\n");
+		return ret;
+	}
+
+	return clk_prepare_enable(slot->clk);
+}
+
+static int slot_pwrctrl_power_off(struct pci_pwrctrl *pwrctrl)
+{
+	struct slot_pwrctrl *slot = container_of(pwrctrl,
+						struct slot_pwrctrl, pwrctrl);
+
+	if (slot->pwrseq) {
+		pwrseq_power_off(slot->pwrseq);
+		return 0;
+	}
+
+	regulator_bulk_disable(slot->num_supplies, slot->supplies);
+	clk_disable_unprepare(slot->clk);
+
+	return 0;
+}
+
+static void devm_slot_pwrctrl_release(void *data)
+{
+	struct slot_pwrctrl *slot = data;
+
+	slot_pwrctrl_power_off(&slot->pwrctrl);
+	regulator_bulk_free(slot->num_supplies, slot->supplies);
+}
+
+static int slot_pwrctrl_probe(struct platform_device *pdev)
+{
+	struct slot_pwrctrl *slot;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	slot = devm_kzalloc(dev, sizeof(*slot), GFP_KERNEL);
+	if (!slot)
+		return -ENOMEM;
+
+	if (of_graph_is_present(dev_of_node(dev))) {
+		slot->pwrseq = devm_pwrseq_get(dev, "pcie");
+		if (IS_ERR(slot->pwrseq))
+			return dev_err_probe(dev, PTR_ERR(slot->pwrseq),
+				     "Failed to get the power sequencer\n");
+
+		goto skip_resources;
+	}
+
+	ret = of_regulator_bulk_get_all(dev, dev_of_node(dev),
+					&slot->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get slot regulators\n");
+
+	slot->num_supplies = ret;
+
+	slot->clk = devm_clk_get_optional(dev, NULL);
+	if (IS_ERR(slot->clk))
+		return dev_err_probe(dev, PTR_ERR(slot->clk),
+				     "Failed to enable slot clock\n");
+
+skip_resources:
+	slot->pwrctrl.power_on = slot_pwrctrl_power_on;
+	slot->pwrctrl.power_off = slot_pwrctrl_power_off;
+
+	ret = devm_add_action_or_reset(dev, devm_slot_pwrctrl_release, slot);
+	if (ret)
+		return ret;
+
+	pci_pwrctrl_init(&slot->pwrctrl, dev);
+
+	ret = devm_pci_pwrctrl_device_set_ready(dev, &slot->pwrctrl);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register pwrctrl driver\n");
+
+	return 0;
+}
+
+static const struct of_device_id slot_pwrctrl_of_match[] = {
+	{
+		.compatible = "pciclass,0604",
+	},
+	/* Renesas UPD720201/UPD720202 USB 3.0 xHCI Host Controller */
+	{
+		.compatible = "pci1912,0014",
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, slot_pwrctrl_of_match);
+
+static struct platform_driver slot_pwrctrl_driver = {
+	.driver = {
+		.name = "pci-pwrctrl-slot",
+		.of_match_table = slot_pwrctrl_of_match,
+	},
+	.probe = slot_pwrctrl_probe,
+};
+module_platform_driver(slot_pwrctrl_driver);
+
+MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
+MODULE_DESCRIPTION("Generic PCI Power Control driver for PCI Slots");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/pwrctrl/slot.c b/drivers/pci/pwrctrl/slot.c
deleted file mode 100644
index 082af81efe254..0000000000000
--- a/drivers/pci/pwrctrl/slot.c
+++ /dev/null
@@ -1,140 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2024 Linaro Ltd.
- * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
- */
-
-#include <linux/clk.h>
-#include <linux/device.h>
-#include <linux/mod_devicetable.h>
-#include <linux/module.h>
-#include <linux/of_graph.h>
-#include <linux/pci-pwrctrl.h>
-#include <linux/platform_device.h>
-#include <linux/pwrseq/consumer.h>
-#include <linux/regulator/consumer.h>
-#include <linux/slab.h>
-
-struct slot_pwrctrl {
-	struct pci_pwrctrl pwrctrl;
-	struct regulator_bulk_data *supplies;
-	int num_supplies;
-	struct clk *clk;
-	struct pwrseq_desc *pwrseq;
-};
-
-static int slot_pwrctrl_power_on(struct pci_pwrctrl *pwrctrl)
-{
-	struct slot_pwrctrl *slot = container_of(pwrctrl,
-						struct slot_pwrctrl, pwrctrl);
-	int ret;
-
-	if (slot->pwrseq) {
-		pwrseq_power_on(slot->pwrseq);
-		return 0;
-	}
-
-	ret = regulator_bulk_enable(slot->num_supplies, slot->supplies);
-	if (ret < 0) {
-		dev_err(slot->pwrctrl.dev, "Failed to enable slot regulators\n");
-		return ret;
-	}
-
-	return clk_prepare_enable(slot->clk);
-}
-
-static int slot_pwrctrl_power_off(struct pci_pwrctrl *pwrctrl)
-{
-	struct slot_pwrctrl *slot = container_of(pwrctrl,
-						struct slot_pwrctrl, pwrctrl);
-
-	if (slot->pwrseq) {
-		pwrseq_power_off(slot->pwrseq);
-		return 0;
-	}
-
-	regulator_bulk_disable(slot->num_supplies, slot->supplies);
-	clk_disable_unprepare(slot->clk);
-
-	return 0;
-}
-
-static void devm_slot_pwrctrl_release(void *data)
-{
-	struct slot_pwrctrl *slot = data;
-
-	slot_pwrctrl_power_off(&slot->pwrctrl);
-	regulator_bulk_free(slot->num_supplies, slot->supplies);
-}
-
-static int slot_pwrctrl_probe(struct platform_device *pdev)
-{
-	struct slot_pwrctrl *slot;
-	struct device *dev = &pdev->dev;
-	int ret;
-
-	slot = devm_kzalloc(dev, sizeof(*slot), GFP_KERNEL);
-	if (!slot)
-		return -ENOMEM;
-
-	if (of_graph_is_present(dev_of_node(dev))) {
-		slot->pwrseq = devm_pwrseq_get(dev, "pcie");
-		if (IS_ERR(slot->pwrseq))
-			return dev_err_probe(dev, PTR_ERR(slot->pwrseq),
-				     "Failed to get the power sequencer\n");
-
-		goto skip_resources;
-	}
-
-	ret = of_regulator_bulk_get_all(dev, dev_of_node(dev),
-					&slot->supplies);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "Failed to get slot regulators\n");
-		return ret;
-	}
-
-	slot->num_supplies = ret;
-
-	slot->clk = devm_clk_get_optional(dev, NULL);
-	if (IS_ERR(slot->clk)) {
-		return dev_err_probe(dev, PTR_ERR(slot->clk),
-				     "Failed to enable slot clock\n");
-	}
-
-skip_resources:
-	slot->pwrctrl.power_on = slot_pwrctrl_power_on;
-	slot->pwrctrl.power_off = slot_pwrctrl_power_off;
-
-	ret = devm_add_action_or_reset(dev, devm_slot_pwrctrl_release, slot);
-	if (ret)
-		return ret;
-
-	pci_pwrctrl_init(&slot->pwrctrl, dev);
-
-	ret = devm_pci_pwrctrl_device_set_ready(dev, &slot->pwrctrl);
-	if (ret)
-		return dev_err_probe(dev, ret, "Failed to register pwrctrl driver\n");
-
-	return 0;
-}
-
-static const struct of_device_id slot_pwrctrl_of_match[] = {
-	{
-		.compatible = "pciclass,0604",
-	},
-	{ }
-};
-MODULE_DEVICE_TABLE(of, slot_pwrctrl_of_match);
-
-static struct platform_driver slot_pwrctrl_driver = {
-	.driver = {
-		.name = "pci-pwrctrl-slot",
-		.of_match_table = slot_pwrctrl_of_match,
-	},
-	.probe = slot_pwrctrl_probe,
-};
-module_platform_driver(slot_pwrctrl_driver);
-
-MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
-MODULE_DESCRIPTION("Generic PCI Power Control driver for PCI Slots");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 48946cca4be72..757a296eae411 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -5603,6 +5603,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap);
  * AMD Starship/Matisse HD Audio Controller 0x1487
  * AMD Starship USB 3.0 Host Controller 0x148c
  * AMD Matisse USB 3.0 Host Controller 0x149c
+ * AMD Neural Processing Unit 0x1502 0x17f0
  * Intel 82579LM Gigabit Ethernet Controller 0x1502
  * Intel 82579V Gigabit Ethernet Controller 0x1503
  * Mediatek MT7922 802.11ax PCI Express Wireless Network Adapter
@@ -5615,6 +5616,8 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x149c, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x7901, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1502, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x17f0, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_MEDIATEK, 0x0616, quirk_no_flr);
diff --git a/drivers/pci/slot.c b/drivers/pci/slot.c
index 787311614e5b6..e0b7fb43423c9 100644
--- a/drivers/pci/slot.c
+++ b/drivers/pci/slot.c
@@ -42,6 +42,15 @@ static ssize_t address_read_file(struct pci_slot *slot, char *buf)
 				  pci_domain_nr(slot->bus),
 				  slot->bus->number);
 
+	/*
+	 * Preserve legacy ABI expectations that hotplug drivers that manage
+	 * multiple devices per slot emit 0 for the device number.
+	 */
+	if (slot->number == PCI_SLOT_ALL_DEVICES)
+		return sysfs_emit(buf, "%04x:%02x:00\n",
+				  pci_domain_nr(slot->bus),
+				  slot->bus->number);
+
 	return sysfs_emit(buf, "%04x:%02x:%02x\n",
 			  pci_domain_nr(slot->bus),
 			  slot->bus->number,
@@ -73,7 +82,8 @@ static void pci_slot_release(struct kobject *kobj)
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &slot->bus->devices, bus_list)
-		if (PCI_SLOT(dev->devfn) == slot->number)
+		if (slot->number == PCI_SLOT_ALL_DEVICES ||
+		    PCI_SLOT(dev->devfn) == slot->number)
 			dev->slot = NULL;
 	up_read(&pci_bus_sem);
 
@@ -166,7 +176,8 @@ void pci_dev_assign_slot(struct pci_dev *dev)
 
 	mutex_lock(&pci_slot_mutex);
 	list_for_each_entry(slot, &dev->bus->slots, list)
-		if (PCI_SLOT(dev->devfn) == slot->number)
+		if (slot->number == PCI_SLOT_ALL_DEVICES ||
+		    PCI_SLOT(dev->devfn) == slot->number)
 			dev->slot = slot;
 	mutex_unlock(&pci_slot_mutex);
 }
@@ -188,7 +199,8 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr)
 /**
  * pci_create_slot - create or increment refcount for physical PCI slot
  * @parent: struct pci_bus of parent bridge
- * @slot_nr: PCI_SLOT(pci_dev->devfn) or -1 for placeholder
+ * @slot_nr: PCI_SLOT(pci_dev->devfn), -1 for placeholder, or
+ *	PCI_SLOT_ALL_DEVICES
  * @name: user visible string presented in /sys/bus/pci/slots/<name>
  * @hotplug: set if caller is hotplug driver, NULL otherwise
  *
@@ -222,6 +234,16 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr)
  * consist solely of a dddd:bb tuple, where dddd is the PCI domain of the
  * %struct pci_bus and bb is the bus number. In other words, the devfn of
  * the 'placeholder' slot will not be displayed.
+ *
+ * Bus-wide slots:
+ * For PCIe hotplug, the physical slot encompasses the entire secondary
+ * bus, not just a single device number. If the device supports ARI and ARI
+ * Forwarding is enabled in the upstream bridge, a multi-function device
+ * may include functions that appear to have several different device
+ * numbers, i.e., PCI_SLOT() values.  Pass @slot_nr == PCI_SLOT_ALL_DEVICES
+ * to create a slot that matches all devices on the bus. Unlike placeholder
+ * slots, bus-wide slots go through normal slot lookup and reuse existing
+ * slots if present.
  */
 struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr,
 				 const char *name,
@@ -285,7 +307,8 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr,
 
 	down_read(&pci_bus_sem);
 	list_for_each_entry(dev, &parent->devices, bus_list)
-		if (PCI_SLOT(dev->devfn) == slot_nr)
+		if (slot_nr == PCI_SLOT_ALL_DEVICES ||
+		    PCI_SLOT(dev->devfn) == slot_nr)
 			dev->slot = slot;
 	up_read(&pci_bus_sem);
 
diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c
index ca4f97be75389..e896b39582818 100644
--- a/drivers/pci/tph.c
+++ b/drivers/pci/tph.c
@@ -407,10 +407,13 @@ int pcie_enable_tph(struct pci_dev *pdev, int mode)
 	else
 		pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY;
 
-	rp_req_type = get_rp_completer_type(pdev);
+	/* Check if the device is behind a Root Port */
+	if (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_END) {
+		rp_req_type = get_rp_completer_type(pdev);
 
-	/* Final req_type is the smallest value of two */
-	pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type);
+		/* Final req_type is the smallest value of two */
+		pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type);
+	}
 
 	if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE)
 		return -EINVAL;
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index 02467dfd4fb0a..227b9a4c612e8 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -6,7 +6,7 @@
 menu "PHY Subsystem"
 
 config PHY_COMMON_PROPS
-	bool
+	bool "PHY common properties" if KUNIT_ALL_TESTS
 	help
 	  This parses properties common between generic PHYs and Ethernet PHYs.
 
@@ -16,8 +16,7 @@ config PHY_COMMON_PROPS
 
 config PHY_COMMON_PROPS_TEST
 	tristate "KUnit tests for PHY common props" if !KUNIT_ALL_TESTS
-	select PHY_COMMON_PROPS
-	depends on KUNIT
+	depends on KUNIT && PHY_COMMON_PROPS
 	default KUNIT_ALL_TESTS
 	help
 	  This builds KUnit tests for the PHY common property API.
@@ -47,6 +46,26 @@ config GENERIC_PHY_MIPI_DPHY
 	  Provides a number of helpers a core functions for MIPI D-PHY
 	  drivers to us.
 
+config PHY_AIROHA_PCIE
+	tristate "Airoha PCIe-PHY Driver"
+	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on OF
+	select GENERIC_PHY
+	help
+	  Say Y here to add support for Airoha PCIe PHY driver.
+	  This driver create the basic PHY instance and provides initialize
+	  callback for PCIe GEN3 port.
+
+config PHY_CAN_TRANSCEIVER
+	tristate "CAN transceiver PHY"
+	select GENERIC_PHY
+	select MULTIPLEXER
+	help
+	  This option enables support for CAN transceivers as a PHY. This
+	  driver provides function for putting the transceivers in various
+	  functional modes using gpios and sets the attribute max link
+	  rate, for CAN drivers.
+
 config PHY_GOOGLE_USB
 	tristate "Google Tensor SoC USB PHY driver"
 	select GENERIC_PHY
@@ -58,6 +77,18 @@ config PHY_GOOGLE_USB
 	  both of which are integrated with the DWC3 USB DRD controller.
 	  This driver currently supports USB high-speed.
 
+config USB_LGM_PHY
+	tristate "INTEL Lightning Mountain USB PHY Driver"
+	depends on USB_SUPPORT
+	depends on X86 || COMPILE_TEST
+	select USB_PHY
+	select REGULATOR
+	select REGULATOR_FIXED_VOLTAGE
+	help
+	  Enable this to support Intel DWC3 PHY USB phy. This driver provides
+	  interface to interact with USB GEN-II and USB 3.x PHY that is part
+	  of the Intel network SOC.
+
 config PHY_LPC18XX_USB_OTG
 	tristate "NXP LPC18xx/43xx SoC USB OTG PHY driver"
 	depends on OF && (ARCH_LPC18XX || COMPILE_TEST)
@@ -69,6 +100,17 @@ config PHY_LPC18XX_USB_OTG
 	  This driver is need for USB0 support on LPC18xx/43xx and takes
 	  care of enabling and clock setup.
 
+config PHY_NXP_PTN3222
+	tristate "NXP PTN3222 1-port eUSB2 to USB2 redriver"
+	depends on I2C
+	depends on OF
+	select GENERIC_PHY
+	help
+	  Enable this to support NXP PTN3222 1-port eUSB2 to USB2 Redriver.
+	  This redriver performs translation between eUSB2 and USB2 signalling
+	  schemes. It supports all three USB 2.0 data rates: Low Speed, Full
+	  Speed and High Speed.
+
 config PHY_PISTACHIO_USB
 	tristate "IMG Pistachio USB2.0 PHY driver"
 	depends on MIPS || COMPILE_TEST
@@ -91,69 +133,17 @@ config PHY_XGENE
 	help
 	  This option enables support for APM X-Gene SoC multi-purpose PHY.
 
-config USB_LGM_PHY
-	tristate "INTEL Lightning Mountain USB PHY Driver"
-	depends on USB_SUPPORT
-	depends on X86 || COMPILE_TEST
-	select USB_PHY
-	select REGULATOR
-	select REGULATOR_FIXED_VOLTAGE
-	help
-	  Enable this to support Intel DWC3 PHY USB phy. This driver provides
-	  interface to interact with USB GEN-II and USB 3.x PHY that is part
-	  of the Intel network SOC.
-
-config PHY_CAN_TRANSCEIVER
-	tristate "CAN transceiver PHY"
-	select GENERIC_PHY
-	select MULTIPLEXER
-	help
-	  This option enables support for CAN transceivers as a PHY. This
-	  driver provides function for putting the transceivers in various
-	  functional modes using gpios and sets the attribute max link
-	  rate, for CAN drivers.
-
-config PHY_AIROHA_PCIE
-	tristate "Airoha PCIe-PHY Driver"
-	depends on ARCH_AIROHA || COMPILE_TEST
-	depends on OF
-	select GENERIC_PHY
-	help
-	  Say Y here to add support for Airoha PCIe PHY driver.
-	  This driver create the basic PHY instance and provides initialize
-	  callback for PCIe GEN3 port.
-
-config PHY_NXP_PTN3222
-	tristate "NXP PTN3222 1-port eUSB2 to USB2 redriver"
-	depends on I2C
-	depends on OF
-	select GENERIC_PHY
-	help
-	  Enable this to support NXP PTN3222 1-port eUSB2 to USB2 Redriver.
-	  This redriver performs translation between eUSB2 and USB2 signalling
-	  schemes. It supports all three USB 2.0 data rates: Low Speed, Full
-	  Speed and High Speed.
-
-config PHY_SPACEMIT_K1_PCIE
-	tristate "PCIe and combo PHY driver for the SpacemiT K1 SoC"
-	depends on ARCH_SPACEMIT || COMPILE_TEST
-	depends on COMMON_CLK
-	depends on HAS_IOMEM
-	depends on OF
-	select GENERIC_PHY
-	default ARCH_SPACEMIT
-	help
-	  Enable support for the PCIe and USB 3 combo PHY and two
-	  PCIe-only PHYs used in the SpacemiT K1 SoC.
-
 source "drivers/phy/allwinner/Kconfig"
 source "drivers/phy/amlogic/Kconfig"
 source "drivers/phy/apple/Kconfig"
 source "drivers/phy/broadcom/Kconfig"
 source "drivers/phy/cadence/Kconfig"
+source "drivers/phy/canaan/Kconfig"
+source "drivers/phy/eswin/Kconfig"
 source "drivers/phy/freescale/Kconfig"
 source "drivers/phy/hisilicon/Kconfig"
 source "drivers/phy/ingenic/Kconfig"
+source "drivers/phy/intel/Kconfig"
 source "drivers/phy/lantiq/Kconfig"
 source "drivers/phy/marvell/Kconfig"
 source "drivers/phy/mediatek/Kconfig"
@@ -175,7 +165,6 @@ source "drivers/phy/starfive/Kconfig"
 source "drivers/phy/sunplus/Kconfig"
 source "drivers/phy/tegra/Kconfig"
 source "drivers/phy/ti/Kconfig"
-source "drivers/phy/intel/Kconfig"
 source "drivers/phy/xilinx/Kconfig"
 
 endmenu
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index a648c2e02a830..f49d83f00a3d8 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -7,21 +7,23 @@ obj-$(CONFIG_PHY_COMMON_PROPS)		+= phy-common-props.o
 obj-$(CONFIG_PHY_COMMON_PROPS_TEST)	+= phy-common-props-test.o
 obj-$(CONFIG_GENERIC_PHY)		+= phy-core.o
 obj-$(CONFIG_GENERIC_PHY_MIPI_DPHY)	+= phy-core-mipi-dphy.o
+obj-$(CONFIG_PHY_AIROHA_PCIE)		+= phy-airoha-pcie.o
 obj-$(CONFIG_PHY_CAN_TRANSCEIVER)	+= phy-can-transceiver.o
 obj-$(CONFIG_PHY_GOOGLE_USB)		+= phy-google-usb.o
+obj-$(CONFIG_USB_LGM_PHY)		+= phy-lgm-usb.o
 obj-$(CONFIG_PHY_LPC18XX_USB_OTG)	+= phy-lpc18xx-usb-otg.o
-obj-$(CONFIG_PHY_XGENE)			+= phy-xgene.o
+obj-$(CONFIG_PHY_NXP_PTN3222)		+= phy-nxp-ptn3222.o
 obj-$(CONFIG_PHY_PISTACHIO_USB)		+= phy-pistachio-usb.o
 obj-$(CONFIG_PHY_SNPS_EUSB2)		+= phy-snps-eusb2.o
-obj-$(CONFIG_USB_LGM_PHY)		+= phy-lgm-usb.o
-obj-$(CONFIG_PHY_AIROHA_PCIE)		+= phy-airoha-pcie.o
-obj-$(CONFIG_PHY_NXP_PTN3222)		+= phy-nxp-ptn3222.o
-obj-$(CONFIG_PHY_SPACEMIT_K1_PCIE)	+= phy-spacemit-k1-pcie.o
+obj-$(CONFIG_PHY_XGENE)			+= phy-xgene.o
+
 obj-$(CONFIG_GENERIC_PHY)		+= allwinner/	\
 					   amlogic/	\
 					   apple/	\
 					   broadcom/	\
 					   cadence/	\
+					   canaan/	\
+					   eswin/	\
 					   freescale/	\
 					   hisilicon/	\
 					   ingenic/	\
diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c
index dc867f368b687..e9d106f135c51 100644
--- a/drivers/phy/apple/atc.c
+++ b/drivers/phy/apple/atc.c
@@ -2023,7 +2023,7 @@ static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, unsign
 	return 0;
 }
 
-const struct reset_control_ops atcphy_dwc3_reset_ops = {
+static const struct reset_control_ops atcphy_dwc3_reset_ops = {
 	.assert = atcphy_dwc3_reset_assert,
 	.deassert = atcphy_dwc3_reset_deassert,
 };
@@ -2202,14 +2202,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph
 		{ "pipehandler", &atcphy->regs.pipehandler, NULL },
 	};
 	struct resource *res;
+	void __iomem *addr;
 
 	for (int i = 0; i < ARRAY_SIZE(resources); i++) {
 		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, resources[i].name);
-		*resources[i].addr = devm_ioremap_resource(&pdev->dev, res);
-		if (IS_ERR(resources[i].addr))
-			return dev_err_probe(atcphy->dev, PTR_ERR(resources[i].addr),
+		addr = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(addr))
+			return dev_err_probe(atcphy->dev, PTR_ERR(addr),
 					     "Unable to map %s regs", resources[i].name);
 
+		*resources[i].addr = addr;
 		if (resources[i].res)
 			*resources[i].res = res;
 	}
diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig
index 1d89a2fd9b79b..46371a8940d76 100644
--- a/drivers/phy/broadcom/Kconfig
+++ b/drivers/phy/broadcom/Kconfig
@@ -52,7 +52,7 @@ config PHY_BCM_NS_USB3
 	tristate "Broadcom Northstar USB 3.0 PHY Driver"
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	depends on HAS_IOMEM && OF
-	depends on MDIO_BUS
+	depends on PHYLIB
 	select GENERIC_PHY
 	help
 	  Enable this to support Broadcom USB 3.0 PHY connected to the USB
@@ -60,7 +60,7 @@ config PHY_BCM_NS_USB3
 
 config PHY_NS2_PCIE
 	tristate "Broadcom Northstar2 PCIe PHY driver"
-	depends on (OF && MDIO_BUS_MUX_BCM_IPROC) || (COMPILE_TEST && MDIO_BUS)
+	depends on (OF && MDIO_BUS_MUX_BCM_IPROC) || (COMPILE_TEST && PHYLIB)
 	select GENERIC_PHY
 	default ARCH_BCM_IPROC
 	help
diff --git a/drivers/phy/canaan/Kconfig b/drivers/phy/canaan/Kconfig
new file mode 100644
index 0000000000000..1ff8831846d56
--- /dev/null
+++ b/drivers/phy/canaan/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Phy drivers for Canaan platforms
+#
+config PHY_CANAAN_USB
+	tristate "Canaan USB2 PHY Driver"
+	depends on (ARCH_CANAAN || COMPILE_TEST) && OF
+	select GENERIC_PHY
+	help
+	Enable this driver to support the USB 2.0 PHY controller
+	on Canaan K230 RISC-V SoCs. This PHY controller
+	provides physical layer functionality for USB 2.0 devices.
+	If you have a Canaan K230 board and need USB 2.0 support,
+	say Y or M here.
diff --git a/drivers/phy/canaan/Makefile b/drivers/phy/canaan/Makefile
new file mode 100644
index 0000000000000..d73857ba284e3
--- /dev/null
+++ b/drivers/phy/canaan/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_PHY_CANAAN_USB)		+= phy-k230-usb.o
diff --git a/drivers/phy/canaan/phy-k230-usb.c b/drivers/phy/canaan/phy-k230-usb.c
new file mode 100644
index 0000000000000..52dad35fc6cf1
--- /dev/null
+++ b/drivers/phy/canaan/phy-k230-usb.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Canaan usb PHY driver
+ *
+ * Copyright (C) 2026 Jiayu Du <jiayu.riscv@isrc.iscas.ac.cn>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#define MAX_PHYS		2
+
+/* Register offsets within the HiSysConfig system controller */
+#define K230_USB0_TEST_REG_BASE     0x70
+#define K230_USB0_CTL_REG_BASE      0xb0
+#define K230_USB1_TEST_REG_BASE     0x90
+#define K230_USB1_CTL_REG_BASE      0xb8
+
+/* Relative offsets within each PHY's control/test block */
+#define CTL0_OFFSET		0x00
+#define CTL1_OFFSET		0x04
+#define TEST_CTL3_OFFSET	0x0c
+
+/* Bit definitions for TEST_CTL3 */
+#define USB_IDPULLUP0		BIT(4)
+#define USB_DMPULLDOWN0		BIT(8)
+#define USB_DPPULLDOWN0		BIT(9)
+
+/* USB control register 0 in HiSysConfig system controller */
+/* PLL Integral Path Tune */
+#define USB_CTL0_PLLITUNE_MASK		GENMASK(23, 22)
+
+/* PLL Proportional Path Tune */
+#define USB_CTL0_PLLPTUNE_MASK		GENMASK(21, 18)
+
+/* PLL Bandwidth Adjustment */
+#define USB_CTL0_PLLBTUNE_MASK		GENMASK(17, 17)
+
+/* VReg18 Bypass Control */
+#define USB_CTL0_VREGBYPASS_MASK	GENMASK(16, 16)
+
+/* Retention Mode Enable */
+#define USB_CTL0_RETENABLEN_MASK	GENMASK(15, 15)
+
+/* Reserved Request Input */
+#define USB_CTL0_RESREQIN_MASK		GENMASK(14, 14)
+
+/* External VBUS Valid Select */
+#define USB_CTL0_VBUSVLDEXTSEL0_MASK	GENMASK(13, 13)
+
+/* OTG Block Disable Control */
+#define USB_CTL0_OTGDISABLE0_MASK	GENMASK(12, 12)
+
+/* Drive VBUS Enable */
+#define USB_CTL0_DRVVBUS0_MASK		GENMASK(11, 11)
+
+/* Autoresume Mode Enable */
+#define USB_CTL0_AUTORSMENB0_MASK	GENMASK(10, 10)
+
+/* HS Transceiver Asynchronous Control */
+#define USB_CTL0_HSXCVREXTCTL0_MASK	GENMASK(9, 9)
+
+/* USB 1.1 Transmit Data */
+#define USB_CTL0_FSDATAEXT0_MASK	GENMASK(8, 8)
+
+/* USB 1.1 SE0 Generation */
+#define USB_CTL0_FSSE0EXT0_MASK		GENMASK(7, 7)
+
+/* USB 1.1 Data Enable */
+#define USB_CTL0_TXENABLEN0_MASK	GENMASK(6, 6)
+
+/* Disconnect Threshold */
+#define USB_CTL0_COMPDISTUNE0_MASK	GENMASK(5, 3)
+
+/* Squelch Threshold */
+#define USB_CTL0_SQRXTUNE0_MASK		GENMASK(2, 0)
+
+/* USB control register 1 in HiSysConfig system controller */
+/* Data Detect Voltage */
+#define USB_CTL1_VDATREFTUNE0_MASK	GENMASK(23, 22)
+
+/* VBUS Valid Threshold */
+#define USB_CTL1_OTGTUNE0_MASK		GENMASK(21, 19)
+
+/* Transmitter High-Speed Crossover */
+#define USB_CTL1_TXHSXVTUNE0_MASK	GENMASK(18, 17)
+
+/* FS/LS Source Impedance */
+#define USB_CTL1_TXFSLSTUNE0_MASK	GENMASK(16, 13)
+
+/* HS DC Voltage Level */
+#define USB_CTL1_TXVREFTUNE0_MASK	GENMASK(12, 9)
+
+/* HS Transmitter Rise/Fall Time */
+#define USB_CTL1_TXRISETUNE0_MASK	GENMASK(8, 7)
+
+/* USB Source Impedance */
+#define USB_CTL1_TXRESTUNE0_MASK	GENMASK(6, 5)
+
+/* HS Transmitter Pre-Emphasis Current Control */
+#define USB_CTL1_TXPREEMPAMPTUNE0_MASK	GENMASK(4, 3)
+
+/* HS Transmitter Pre-Emphasis Duration Control */
+#define USB_CTL1_TXPREEMPPULSETUNE0_MASK	GENMASK(2, 2)
+
+/* charging detection */
+#define USB_CTL1_CHRGSRCPUENB0_MASK	GENMASK(1, 0)
+
+#define K230_PHY_CTL0_VAL \
+( \
+	FIELD_PREP(USB_CTL0_PLLITUNE_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_PLLPTUNE_MASK, 0xc) | \
+	FIELD_PREP(USB_CTL0_PLLBTUNE_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL0_VREGBYPASS_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL0_RETENABLEN_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL0_RESREQIN_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_VBUSVLDEXTSEL0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_OTGDISABLE0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_DRVVBUS0_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL0_AUTORSMENB0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_HSXCVREXTCTL0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_FSDATAEXT0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_FSSE0EXT0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_TXENABLEN0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL0_COMPDISTUNE0_MASK, 0x3) | \
+	FIELD_PREP(USB_CTL0_SQRXTUNE0_MASK, 0x3) \
+)
+
+#define K230_PHY_CTL1_VAL \
+( \
+	FIELD_PREP(USB_CTL1_VDATREFTUNE0_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL1_OTGTUNE0_MASK, 0x3) | \
+	FIELD_PREP(USB_CTL1_TXHSXVTUNE0_MASK, 0x3) | \
+	FIELD_PREP(USB_CTL1_TXFSLSTUNE0_MASK, 0x3) | \
+	FIELD_PREP(USB_CTL1_TXVREFTUNE0_MASK, 0x3) | \
+	FIELD_PREP(USB_CTL1_TXRISETUNE0_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL1_TXRESTUNE0_MASK, 0x1) | \
+	FIELD_PREP(USB_CTL1_TXPREEMPAMPTUNE0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL1_TXPREEMPPULSETUNE0_MASK, 0x0) | \
+	FIELD_PREP(USB_CTL1_CHRGSRCPUENB0_MASK, 0x0) \
+)
+
+struct k230_usb_phy_instance {
+	struct k230_usb_phy_global *global;
+	struct phy *phy;
+	u32 test_offset;
+	u32 ctl_offset;
+	int index;
+};
+
+struct k230_usb_phy_global {
+	struct k230_usb_phy_instance phys[MAX_PHYS];
+	void __iomem *base;
+};
+
+static int k230_usb_phy_power_on(struct phy *phy)
+{
+	struct k230_usb_phy_instance *inst = phy_get_drvdata(phy);
+	struct k230_usb_phy_global *global = inst->global;
+	void __iomem *base = global->base;
+	u32 val;
+
+	/* Apply recommended settings */
+	writel(K230_PHY_CTL0_VAL, base + inst->ctl_offset + CTL0_OFFSET);
+	writel(K230_PHY_CTL1_VAL, base + inst->ctl_offset + CTL1_OFFSET);
+
+	/* Configure test register (pull-ups/pull-downs) */
+	val = readl(base + inst->test_offset + TEST_CTL3_OFFSET);
+	val |= USB_IDPULLUP0;
+
+	if (inst->index == 1)
+		val |= (USB_DMPULLDOWN0 | USB_DPPULLDOWN0);
+	else
+		val &= ~(USB_DMPULLDOWN0 | USB_DPPULLDOWN0);
+
+	writel(val, base + inst->test_offset + TEST_CTL3_OFFSET);
+
+	return 0;
+}
+
+static int k230_usb_phy_power_off(struct phy *phy)
+{
+	struct k230_usb_phy_instance *inst = phy_get_drvdata(phy);
+	struct k230_usb_phy_global *global = inst->global;
+	void __iomem *base = global->base;
+	u32 val;
+
+	val = readl(base + inst->test_offset + TEST_CTL3_OFFSET);
+	val &= ~(USB_DMPULLDOWN0 | USB_DPPULLDOWN0);
+	writel(val, base + inst->test_offset + TEST_CTL3_OFFSET);
+
+	return 0;
+}
+
+static const struct phy_ops k230_usb_phy_ops = {
+	.power_on = k230_usb_phy_power_on,
+	.power_off = k230_usb_phy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static struct phy *k230_usb_phy_xlate(struct device *dev,
+				      const struct of_phandle_args *args)
+{
+	struct k230_usb_phy_global *global = dev_get_drvdata(dev);
+	unsigned int idx = args->args[0];
+
+	if (idx >= MAX_PHYS)
+		return ERR_PTR(-EINVAL);
+
+	return global->phys[idx].phy;
+}
+
+static int k230_usb_phy_probe(struct platform_device *pdev)
+{
+	struct k230_usb_phy_global *global;
+	struct device *dev = &pdev->dev;
+	struct phy_provider *provider;
+	int i;
+
+	global = devm_kzalloc(dev, sizeof(*global), GFP_KERNEL);
+	if (!global)
+		return -ENOMEM;
+	dev_set_drvdata(dev, global);
+
+	global->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(global->base))
+		return dev_err_probe(dev, PTR_ERR(global->base),
+				     "failed to map registers\n");
+
+	static const struct {
+		u32 test_offset;
+		u32 ctl_offset;
+	} phy_reg_info[MAX_PHYS] = {
+		[0] = { K230_USB0_TEST_REG_BASE, K230_USB0_CTL_REG_BASE },
+		[1] = { K230_USB1_TEST_REG_BASE, K230_USB1_CTL_REG_BASE },
+	};
+
+	for (i = 0; i < MAX_PHYS; i++) {
+		struct k230_usb_phy_instance *inst = &global->phys[i];
+		struct phy *phy;
+
+		inst->global = global;
+		inst->index = i;
+		inst->test_offset = phy_reg_info[i].test_offset;
+		inst->ctl_offset  = phy_reg_info[i].ctl_offset;
+
+		phy = devm_phy_create(dev, NULL, &k230_usb_phy_ops);
+		if (IS_ERR(phy)) {
+			dev_err(dev, "failed to create phy%d\n", i);
+			return PTR_ERR(phy);
+		}
+
+		phy_set_drvdata(phy, inst);
+		inst->phy = phy;
+	}
+
+	provider = devm_of_phy_provider_register(dev, k230_usb_phy_xlate);
+	if (IS_ERR(provider))
+		return PTR_ERR(provider);
+
+	return 0;
+}
+
+static const struct of_device_id k230_usb_phy_of_match[] = {
+	{ .compatible = "canaan,k230-usb-phy" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, k230_usb_phy_of_match);
+
+static struct platform_driver k230_usb_phy_driver = {
+	.probe = k230_usb_phy_probe,
+	.driver = {
+		.name = "k230-usb-phy",
+		.of_match_table = k230_usb_phy_of_match,
+	},
+};
+module_platform_driver(k230_usb_phy_driver);
+
+MODULE_DESCRIPTION("Canaan Kendryte K230 USB 2.0 PHY driver");
+MODULE_AUTHOR("Jiayu Du <jiayu.riscv@isrc.iscas.ac.cn>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/eswin/Kconfig b/drivers/phy/eswin/Kconfig
new file mode 100644
index 0000000000000..cf2bf2efc32f7
--- /dev/null
+++ b/drivers/phy/eswin/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Phy drivers for ESWIN platforms
+#
+config PHY_EIC7700_SATA
+	tristate "eic7700 Sata SerDes/PHY driver"
+	depends on ARCH_ESWIN || COMPILE_TEST
+	depends on HAS_IOMEM
+	select GENERIC_PHY
+	help
+	  Enable this to support SerDes/Phy found on ESWIN's
+	  EIC7700 SoC. This Phy supports SATA 1.5 Gb/s,
+	  SATA 3.0 Gb/s, SATA 6.0 Gb/s speeds.
+	  It supports one SATA host port to accept one SATA device.
diff --git a/drivers/phy/eswin/Makefile b/drivers/phy/eswin/Makefile
new file mode 100644
index 0000000000000..db08c66be8129
--- /dev/null
+++ b/drivers/phy/eswin/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_PHY_EIC7700_SATA)	+= phy-eic7700-sata.o
diff --git a/drivers/phy/eswin/phy-eic7700-sata.c b/drivers/phy/eswin/phy-eic7700-sata.c
new file mode 100644
index 0000000000000..c33653d48daaf
--- /dev/null
+++ b/drivers/phy/eswin/phy-eic7700-sata.c
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN SATA PHY driver
+ *
+ * Copyright 2026, Beijing ESWIN Computing Technology Co., Ltd..
+ * All rights reserved.
+ *
+ * Authors: Yulin Lu <luyulin@eswincomputing.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#define SATA_AXI_LP_CTRL			0x08
+#define SATA_MPLL_CTRL				0x20
+#define SATA_P0_PHY_STAT			0x24
+#define SATA_PHY_CTRL0				0x28
+#define SATA_PHY_CTRL1				0x2c
+#define SATA_REF_CTRL				0x34
+#define SATA_REF_CTRL1				0x38
+#define SATA_LOS_IDEN				0x3c
+
+#define SATA_CLK_RST_SOURCE_PHY			BIT(0)
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN1_MASK	GENMASK(6, 0)
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN1_DEFAULT	0x42
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN2_MASK	GENMASK(14, 8)
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN2_DEFAULT	0x46
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN3_MASK	GENMASK(22, 16)
+#define SATA_P0_PHY_TX_AMPLITUDE_GEN3_DEFAULT	0x73
+#define SATA_P0_PHY_TX_PREEMPH_GEN1_MASK	GENMASK(5, 0)
+#define SATA_P0_PHY_TX_PREEMPH_GEN1_DEFAULT	0x5
+#define SATA_P0_PHY_TX_PREEMPH_GEN2_MASK	GENMASK(13, 8)
+#define SATA_P0_PHY_TX_PREEMPH_GEN2_DEFAULT	0x5
+#define SATA_P0_PHY_TX_PREEMPH_GEN3_MASK	GENMASK(21, 16)
+#define SATA_P0_PHY_TX_PREEMPH_GEN3_DEFAULT	0x23
+#define SATA_LOS_LEVEL_MASK			GENMASK(4, 0)
+#define SATA_LOS_BIAS_MASK			GENMASK(18, 16)
+#define SATA_M_CSYSREQ				BIT(0)
+#define SATA_S_CSYSREQ				BIT(16)
+#define SATA_REF_REPEATCLK_EN			BIT(0)
+#define SATA_REF_USE_PAD			BIT(20)
+#define SATA_MPLL_MULTIPLIER_MASK		GENMASK(22, 16)
+#define SATA_P0_PHY_READY			BIT(0)
+
+#define PLL_LOCK_SLEEP_US			10
+#define PLL_LOCK_TIMEOUT_US			1000
+
+struct eic7700_sata_phy {
+	u32 tx_amplitude_tuning_val[3];
+	u32 tx_preemph_tuning_val[3];
+	struct reset_control *rst;
+	struct regmap *regmap;
+	struct clk *clk;
+	struct phy *phy;
+};
+
+static const struct regmap_config eic7700_sata_phy_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = SATA_LOS_IDEN,
+};
+
+static int wait_for_phy_ready(struct regmap *regmap, u32 reg, u32 checkbit,
+			      u32 status)
+{
+	u32 val;
+	int ret;
+
+	ret = regmap_read_poll_timeout(regmap, reg, val,
+				       (val & checkbit) == status,
+				       PLL_LOCK_SLEEP_US, PLL_LOCK_TIMEOUT_US);
+
+	return ret;
+}
+
+static int eic7700_sata_phy_init(struct phy *phy)
+{
+	struct eic7700_sata_phy *sata_phy = phy_get_drvdata(phy);
+	u32 val;
+	int ret;
+
+	ret = clk_prepare_enable(sata_phy->clk);
+	if (ret)
+		return ret;
+
+	regmap_write(sata_phy->regmap, SATA_REF_CTRL1, SATA_CLK_RST_SOURCE_PHY);
+
+	val = FIELD_PREP(SATA_P0_PHY_TX_AMPLITUDE_GEN1_MASK,
+			 sata_phy->tx_amplitude_tuning_val[0]) |
+	      FIELD_PREP(SATA_P0_PHY_TX_AMPLITUDE_GEN2_MASK,
+			 sata_phy->tx_amplitude_tuning_val[1]) |
+	      FIELD_PREP(SATA_P0_PHY_TX_AMPLITUDE_GEN3_MASK,
+			 sata_phy->tx_amplitude_tuning_val[2]);
+	regmap_write(sata_phy->regmap, SATA_PHY_CTRL0, val);
+
+	val = FIELD_PREP(SATA_P0_PHY_TX_PREEMPH_GEN1_MASK,
+			 sata_phy->tx_preemph_tuning_val[0]) |
+	      FIELD_PREP(SATA_P0_PHY_TX_PREEMPH_GEN2_MASK,
+			 sata_phy->tx_preemph_tuning_val[1]) |
+	      FIELD_PREP(SATA_P0_PHY_TX_PREEMPH_GEN3_MASK,
+			 sata_phy->tx_preemph_tuning_val[2]);
+	regmap_write(sata_phy->regmap, SATA_PHY_CTRL1, val);
+
+	val = FIELD_PREP(SATA_LOS_LEVEL_MASK, 0x9) |
+	      FIELD_PREP(SATA_LOS_BIAS_MASK, 0x2);
+	regmap_write(sata_phy->regmap, SATA_LOS_IDEN, val);
+
+	val = SATA_M_CSYSREQ | SATA_S_CSYSREQ;
+	regmap_write(sata_phy->regmap, SATA_AXI_LP_CTRL, val);
+
+	val = SATA_REF_REPEATCLK_EN | SATA_REF_USE_PAD;
+	regmap_write(sata_phy->regmap, SATA_REF_CTRL, val);
+
+	val = FIELD_PREP(SATA_MPLL_MULTIPLIER_MASK, 0x3c);
+	regmap_write(sata_phy->regmap, SATA_MPLL_CTRL, val);
+
+	usleep_range(15, 20);
+
+	ret = reset_control_deassert(sata_phy->rst);
+	if (ret)
+		goto disable_clk;
+
+	ret = wait_for_phy_ready(sata_phy->regmap, SATA_P0_PHY_STAT,
+				 SATA_P0_PHY_READY, 1);
+	if (ret < 0) {
+		dev_err(&sata_phy->phy->dev, "PHY READY check failed\n");
+		goto disable_clk;
+	}
+
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(sata_phy->clk);
+	return ret;
+}
+
+static int eic7700_sata_phy_exit(struct phy *phy)
+{
+	struct eic7700_sata_phy *sata_phy = phy_get_drvdata(phy);
+	int ret;
+
+	ret = reset_control_assert(sata_phy->rst);
+	if (ret)
+		return ret;
+
+	clk_disable_unprepare(sata_phy->clk);
+
+	return 0;
+}
+
+static const struct phy_ops eic7700_sata_phy_ops = {
+	.init		= eic7700_sata_phy_init,
+	.exit		= eic7700_sata_phy_exit,
+	.owner		= THIS_MODULE,
+};
+
+static void eic7700_get_tuning_param(struct device_node *np,
+				     struct eic7700_sata_phy *sata_phy)
+{
+	if (of_property_read_u32_array
+		(np, "eswin,tx-amplitude-tuning",
+		sata_phy->tx_amplitude_tuning_val,
+		ARRAY_SIZE(sata_phy->tx_amplitude_tuning_val))) {
+		sata_phy->tx_amplitude_tuning_val[0] =
+			SATA_P0_PHY_TX_AMPLITUDE_GEN1_DEFAULT;
+		sata_phy->tx_amplitude_tuning_val[1] =
+			SATA_P0_PHY_TX_AMPLITUDE_GEN2_DEFAULT;
+		sata_phy->tx_amplitude_tuning_val[2] =
+			SATA_P0_PHY_TX_AMPLITUDE_GEN3_DEFAULT;
+	}
+
+	if (of_property_read_u32_array
+		(np, "eswin,tx-preemph-tuning",
+		sata_phy->tx_preemph_tuning_val,
+		ARRAY_SIZE(sata_phy->tx_preemph_tuning_val))) {
+		sata_phy->tx_preemph_tuning_val[0] =
+			SATA_P0_PHY_TX_PREEMPH_GEN1_DEFAULT;
+		sata_phy->tx_preemph_tuning_val[1] =
+			SATA_P0_PHY_TX_PREEMPH_GEN2_DEFAULT;
+		sata_phy->tx_preemph_tuning_val[2] =
+			SATA_P0_PHY_TX_PREEMPH_GEN3_DEFAULT;
+	}
+}
+
+static int eic7700_sata_phy_probe(struct platform_device *pdev)
+{
+	struct eic7700_sata_phy *sata_phy;
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct resource *res;
+	void __iomem *regs;
+
+	sata_phy = devm_kzalloc(dev, sizeof(*sata_phy), GFP_KERNEL);
+	if (!sata_phy)
+		return -ENOMEM;
+
+	/*
+	 * Map the I/O resource with platform_get_resource and devm_ioremap
+	 * instead of the devm_platform_ioremap_resource API, because the
+	 * address region of the SATA-PHY falls into the region of the HSP
+	 * clock & reset that has already been obtained by the HSP
+	 * clock-and-reset driver.
+	 */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENOENT;
+
+	regs = devm_ioremap(dev, res->start, resource_size(res));
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	sata_phy->regmap = devm_regmap_init_mmio
+			   (dev, regs, &eic7700_sata_phy_regmap_config);
+	if (IS_ERR(sata_phy->regmap))
+		return dev_err_probe(dev, PTR_ERR(sata_phy->regmap),
+				     "failed to init regmap\n");
+
+	dev_set_drvdata(dev, sata_phy);
+
+	eic7700_get_tuning_param(np, sata_phy);
+
+	sata_phy->clk = devm_clk_get(dev, "phy");
+	if (IS_ERR(sata_phy->clk))
+		return PTR_ERR(sata_phy->clk);
+
+	sata_phy->rst = devm_reset_control_array_get_exclusive(dev);
+	if (IS_ERR(sata_phy->rst))
+		return dev_err_probe(dev, PTR_ERR(sata_phy->rst),
+				     "failed to get reset control\n");
+
+	sata_phy->phy = devm_phy_create(dev, NULL, &eic7700_sata_phy_ops);
+	if (IS_ERR(sata_phy->phy))
+		return dev_err_probe(dev, PTR_ERR(sata_phy->phy),
+				     "failed to create PHY\n");
+
+	phy_set_drvdata(sata_phy->phy, sata_phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (IS_ERR(phy_provider))
+		return dev_err_probe(dev, PTR_ERR(phy_provider),
+				     "failed to register PHY provider\n");
+
+	return 0;
+}
+
+static const struct of_device_id eic7700_sata_phy_of_match[] = {
+	{ .compatible = "eswin,eic7700-sata-phy" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, eic7700_sata_phy_of_match);
+
+static struct platform_driver eic7700_sata_phy_driver = {
+	.probe	= eic7700_sata_phy_probe,
+	.driver = {
+		.of_match_table	= eic7700_sata_phy_of_match,
+		.name  = "eic7700-sata-phy",
+	}
+};
+module_platform_driver(eic7700_sata_phy_driver);
+
+MODULE_DESCRIPTION("SATA PHY driver for the ESWIN EIC7700 SoC");
+MODULE_AUTHOR("Yulin Lu <luyulin@eswincomputing.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/freescale/phy-fsl-lynx-28g.c b/drivers/phy/freescale/phy-fsl-lynx-28g.c
index 2b0fd95ba62fd..63427fc34e268 100644
--- a/drivers/phy/freescale/phy-fsl-lynx-28g.c
+++ b/drivers/phy/freescale/phy-fsl-lynx-28g.c
@@ -1069,6 +1069,8 @@ static void lynx_28g_cdr_lock_check(struct work_struct *work)
 
 	for (i = 0; i < LYNX_28G_NUM_LANE; i++) {
 		lane = &priv->lane[i];
+		if (!lane->phy)
+			continue;
 
 		mutex_lock(&lane->phy->mutex);
 
diff --git a/drivers/phy/marvell/phy-mmp3-hsic.c b/drivers/phy/marvell/phy-mmp3-hsic.c
index 271f1a2258efb..72ab6da0ebc3d 100644
--- a/drivers/phy/marvell/phy-mmp3-hsic.c
+++ b/drivers/phy/marvell/phy-mmp3-hsic.c
@@ -14,15 +14,19 @@
 #define HSIC_ENABLE	BIT(7)
 #define PLL_BYPASS	BIT(4)
 
+struct mmp3_hsic_data {
+	void __iomem *base;
+};
+
 static int mmp3_hsic_phy_init(struct phy *phy)
 {
-	void __iomem *base = (void __iomem *)phy_get_drvdata(phy);
+	struct mmp3_hsic_data *mmp3 = phy_get_drvdata(phy);
 	u32 hsic_ctrl;
 
-	hsic_ctrl = readl_relaxed(base + HSIC_CTRL);
+	hsic_ctrl = readl_relaxed(mmp3->base + HSIC_CTRL);
 	hsic_ctrl |= HSIC_ENABLE;
 	hsic_ctrl |= PLL_BYPASS;
-	writel_relaxed(hsic_ctrl, base + HSIC_CTRL);
+	writel_relaxed(hsic_ctrl, mmp3->base + HSIC_CTRL);
 
 	return 0;
 }
@@ -41,13 +45,17 @@ MODULE_DEVICE_TABLE(of, mmp3_hsic_phy_of_match);
 static int mmp3_hsic_phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct mmp3_hsic_data *mmp3;
 	struct phy_provider *provider;
-	void __iomem *base;
 	struct phy *phy;
 
-	base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
+	mmp3 = devm_kzalloc(dev, sizeof(*mmp3), GFP_KERNEL);
+	if (!mmp3)
+		return -ENOMEM;
+
+	mmp3->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
+	if (IS_ERR(mmp3->base))
+		return PTR_ERR(mmp3->base);
 
 	phy = devm_phy_create(dev, NULL, &mmp3_hsic_phy_ops);
 	if (IS_ERR(phy)) {
@@ -55,7 +63,7 @@ static int mmp3_hsic_phy_probe(struct platform_device *pdev)
 		return PTR_ERR(phy);
 	}
 
-	phy_set_drvdata(phy, (void *)base);
+	phy_set_drvdata(phy, mmp3);
 	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
 	if (IS_ERR(provider)) {
 		dev_err(dev, "failed to register PHY provider\n");
diff --git a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c
index f6504e0ecd1a7..acf5065295072 100644
--- a/drivers/phy/mediatek/phy-mtk-tphy.c
+++ b/drivers/phy/mediatek/phy-mtk-tphy.c
@@ -276,14 +276,14 @@ enum mtk_phy_version {
 };
 
 /**
- * mtk_phy_pdata - SoC specific platform data
+ * struct mtk_phy_pdata - SoC specific platform data
  * @avoid_rx_sen_degradation: Avoid TX Sensitivity level degradation (MT6795/8173 only)
  * @sw_pll_48m_to_26m:        Workaround for V3 IP (MT8195) - switch the 48MHz PLL from
  *                            fractional mode to integer to output 26MHz for U2PHY
  * @sw_efuse_supported:       Switches off eFuse auto-load from PHY and applies values
  *                            read from different nvmem (usually different eFuse array)
  *                            that is pointed at in the device tree node for this PHY
- * @slew_ref_clk_mhz:         Default reference clock (in MHz) for slew rate calibration
+ * @slew_ref_clock_mhz:         Default reference clock (in MHz) for slew rate calibration
  * @slew_rate_coefficient:    Coefficient for slew rate calibration
  * @version:                  PHY IP Version
  */
diff --git a/drivers/phy/phy-can-transceiver.c b/drivers/phy/phy-can-transceiver.c
index 330356706ad7b..2b52e47f247a2 100644
--- a/drivers/phy/phy-can-transceiver.c
+++ b/drivers/phy/phy-can-transceiver.c
@@ -126,16 +126,6 @@ static const struct of_device_id can_transceiver_phy_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, can_transceiver_phy_ids);
 
-/* Temporary wrapper until the multiplexer subsystem supports optional muxes */
-static inline struct mux_state *
-devm_mux_state_get_optional(struct device *dev, const char *mux_name)
-{
-	if (!of_property_present(dev->of_node, "mux-states"))
-		return NULL;
-
-	return devm_mux_state_get(dev, mux_name);
-}
-
 static struct phy *can_transceiver_phy_xlate(struct device *dev,
 					     const struct of_phandle_args *args)
 {
diff --git a/drivers/phy/phy-spacemit-k1-pcie.c b/drivers/phy/phy-spacemit-k1-pcie.c
deleted file mode 100644
index 75477bea7f700..0000000000000
--- a/drivers/phy/phy-spacemit-k1-pcie.c
+++ /dev/null
@@ -1,670 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SpacemiT K1 PCIe and PCIe/USB 3 combo PHY driver
- *
- * Copyright (C) 2025 by RISCstar Solutions Corporation.  All rights reserved.
- */
-
-#include <linux/bitfield.h>
-#include <linux/clk.h>
-#include <linux/clk-provider.h>
-#include <linux/iopoll.h>
-#include <linux/kernel.h>
-#include <linux/mfd/syscon.h>
-#include <linux/module.h>
-#include <linux/phy/phy.h>
-#include <linux/platform_device.h>
-#include <linux/regmap.h>
-#include <linux/reset.h>
-
-#include <dt-bindings/phy/phy.h>
-
-/*
- * Three PCIe ports are supported in the SpacemiT K1 SoC, and this driver
- * supports their PHYs.
- *
- * The PHY for PCIe port A is different from the PHYs for ports B and C:
- * - It has one lane, while ports B and C have two
- * - It is a combo PHY can be used for PCIe or USB 3
- * - It can automatically calibrate PCIe TX and RX termination settings
- *
- * The PHY functionality for PCIe ports B and C is identical:
- * - They have two PCIe lanes (but can be restricted to 1 via device tree)
- * - They are used for PCIe only
- * - They are configured using TX and RX values computed for port A
- *
- * A given board is designed to use the combo PHY for either PCIe or USB 3.
- * Whether the combo PHY is configured for PCIe or USB 3 is specified in
- * device tree using a phandle plus an argument.  The argument indicates
- * the type (either PHY_TYPE_PCIE or PHY_TYPE_USB3).
- *
- * Each PHY has a reset that it gets and deasserts during initialization.
- * Each depends also on other clocks and resets provided by the controller
- * hardware (PCIe or USB) it is associated with.  The controller drivers
- * are required to enable any clocks and de-assert any resets that affect
- * PHY operation.  In addition each PHY implements an internal PLL, driven
- * by an external (24 MHz) oscillator.
- *
- * PCIe PHYs must be programmed with RX and TX calibration values.  The
- * combo PHY is the only one that can determine these values.  They are
- * determined by temporarily enabling the combo PHY in PCIe mode at probe
- * time (if necessary).  This calibration only needs to be done once, and
- * when it has completed the TX and RX values are saved.
- *
- * To allow the combo PHY to be enabled for calibration, the resets and
- * clocks it uses in PCIe mode must be supplied.
- */
-
-struct k1_pcie_phy {
-	struct device *dev;		/* PHY provider device */
-	struct phy *phy;
-	void __iomem *regs;
-	u32 pcie_lanes;			/* Max (1 or 2) unless limited by DT */
-	struct clk *pll;
-	struct clk_hw pll_hw;		/* Private PLL clock */
-
-	/* The remaining fields are only used for the combo PHY */
-	u32 type;			/* PHY_TYPE_PCIE or PHY_TYPE_USB3 */
-	struct regmap *pmu;		/* MMIO regmap (no errors) */
-};
-
-#define CALIBRATION_TIMEOUT		500000	/* For combo PHY (usec) */
-#define PLL_TIMEOUT			500000	/* For PHY PLL lock (usec) */
-#define POLL_DELAY			500	/* Time between polls (usec) */
-
-/* Selecting the combo PHY operating mode requires APMU regmap access */
-#define SYSCON_APMU			"spacemit,apmu"
-
-/* PMU space, for selecting between PCIe and USB 3 mode (combo PHY only) */
-
-#define PMUA_USB_PHY_CTRL0			0x0110
-#define COMBO_PHY_SEL			BIT(3)	/* 0: PCIe; 1: USB 3 */
-
-#define PCIE_CLK_RES_CTRL			0x03cc
-#define PCIE_APP_HOLD_PHY_RST		BIT(30)
-
-/* PHY register space */
-
-/* Offset between lane 0 and lane 1 registers when there are two */
-#define PHY_LANE_OFFSET				0x0400
-
-/* PHY PLL configuration */
-#define PCIE_PU_ADDR_CLK_CFG			0x0008
-#define PLL_READY			BIT(0)		/* read-only */
-#define CFG_INTERNAL_TIMER_ADJ		GENMASK(10, 7)
-#define TIMER_ADJ_USB		0x2
-#define TIMER_ADJ_PCIE		0x6
-#define CFG_SW_PHY_INIT_DONE		BIT(11)	/* We set after PLL config */
-
-#define PCIE_RC_DONE_STATUS			0x0018
-#define CFG_FORCE_RCV_RETRY		BIT(10)		/* Used for PCIe */
-
-/* PCIe PHY lane calibration; assumes 24MHz input clock */
-#define PCIE_RC_CAL_REG2			0x0020
-#define RC_CAL_TOGGLE			BIT(22)
-#define CLKSEL				GENMASK(31, 29)
-#define CLKSEL_24M		0x3
-
-/* Additional PHY PLL configuration (USB 3 and PCIe) */
-#define PCIE_PU_PLL_1				0x0048
-#define REF_100_WSSC			BIT(12)	/* 1: input is 100MHz, SSC */
-#define FREF_SEL			GENMASK(15, 13)
-#define FREF_24M		0x1
-#define SSC_DEP_SEL			GENMASK(19, 16)
-#define SSC_DEP_NONE		0x0
-#define SSC_DEP_5000PPM		0xa
-
-/* PCIe PHY configuration */
-#define PCIE_PU_PLL_2				0x004c
-#define GEN_REF100			BIT(4)	/* 1: generate 100MHz clk */
-
-#define PCIE_RX_REG1				0x0050
-#define EN_RTERM			BIT(3)
-#define AFE_RTERM_REG			GENMASK(11, 8)
-
-#define PCIE_RX_REG2				0x0054
-#define RX_RTERM_SEL			BIT(5)	/* 0: use AFE_RTERM_REG value */
-
-#define PCIE_LTSSM_DIS_ENTRY			0x005c
-#define CFG_REFCLK_MODE			GENMASK(9, 8)
-#define RFCLK_MODE_DRIVER	0x1
-#define OVRD_REFCLK_MODE		BIT(10)	/* 1: use CFG_RFCLK_MODE */
-
-#define PCIE_TX_REG1				0x0064
-#define TX_RTERM_REG			GENMASK(15, 12)
-#define TX_RTERM_SEL			BIT(25)	/* 1: use TX_RTERM_REG */
-
-/* Zeroed for the combo PHY operating in USB mode */
-#define USB3_TEST_CTRL				0x0068
-
-/* PHY calibration values, determined by the combo PHY at probe time */
-#define PCIE_RCAL_RESULT			0x0084	/* Port A PHY only */
-#define RTERM_VALUE_RX			GENMASK(3, 0)
-#define RTERM_VALUE_TX			GENMASK(7, 4)
-#define R_TUNE_DONE			BIT(10)
-
-static u32 k1_phy_rterm = ~0;     /* Invalid initial value */
-
-/* Save the RX and TX receiver termination values */
-static void k1_phy_rterm_set(u32 val)
-{
-	k1_phy_rterm = val & (RTERM_VALUE_RX | RTERM_VALUE_TX);
-}
-
-static bool k1_phy_rterm_valid(void)
-{
-	/* Valid if no bits outside those we care about are set */
-	return !(k1_phy_rterm & ~(RTERM_VALUE_RX | RTERM_VALUE_TX));
-}
-
-static u32 k1_phy_rterm_rx(void)
-{
-	return FIELD_GET(RTERM_VALUE_RX, k1_phy_rterm);
-}
-
-static u32 k1_phy_rterm_tx(void)
-{
-	return FIELD_GET(RTERM_VALUE_TX, k1_phy_rterm);
-}
-
-/* Only the combo PHY has a PMU pointer defined */
-static bool k1_phy_port_a(struct k1_pcie_phy *k1_phy)
-{
-	return !!k1_phy->pmu;
-}
-
-/* The PLL clocks are driven by the external oscillator */
-static const struct clk_parent_data k1_pcie_phy_data[] = {
-	{ .fw_name = "refclk", },
-};
-
-static struct k1_pcie_phy *clk_hw_to_k1_phy(struct clk_hw *clk_hw)
-{
-	return container_of(clk_hw, struct k1_pcie_phy, pll_hw);
-}
-
-/* USB mode only works on the combo PHY, which has only one lane */
-static void k1_pcie_phy_pll_prepare_usb(struct k1_pcie_phy *k1_phy)
-{
-	void __iomem *regs = k1_phy->regs;
-	u32 val;
-
-	val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
-	val &= ~CFG_INTERNAL_TIMER_ADJ;
-	val |= FIELD_PREP(CFG_INTERNAL_TIMER_ADJ, TIMER_ADJ_USB);
-	writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
-
-	val = readl(regs + PCIE_PU_PLL_1);
-	val &= ~SSC_DEP_SEL;
-	val |= FIELD_PREP(SSC_DEP_SEL, SSC_DEP_5000PPM);
-	writel(val, regs + PCIE_PU_PLL_1);
-}
-
-/* Perform PCIe-specific register updates before starting the PLL clock */
-static void k1_pcie_phy_pll_prepare_pcie(struct k1_pcie_phy *k1_phy)
-{
-	void __iomem *regs = k1_phy->regs;
-	u32 val;
-	u32 i;
-
-	for (i = 0; i < k1_phy->pcie_lanes; i++) {
-		val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
-		val &= ~CFG_INTERNAL_TIMER_ADJ;
-		val |= FIELD_PREP(CFG_INTERNAL_TIMER_ADJ, TIMER_ADJ_PCIE);
-		writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
-
-		regs += PHY_LANE_OFFSET;	/* Next lane */
-	}
-
-	regs = k1_phy->regs;
-	val = readl(regs + PCIE_RC_DONE_STATUS);
-	val |= CFG_FORCE_RCV_RETRY;
-	writel(val, regs + PCIE_RC_DONE_STATUS);
-
-	val = readl(regs + PCIE_PU_PLL_1);
-	val &= ~SSC_DEP_SEL;
-	val |= FIELD_PREP(SSC_DEP_SEL, SSC_DEP_NONE);
-	writel(val, regs + PCIE_PU_PLL_1);
-
-	val = readl(regs + PCIE_PU_PLL_2);
-	val |= GEN_REF100;		/* Enable 100 MHz PLL output clock */
-	writel(val, regs + PCIE_PU_PLL_2);
-}
-
-static int k1_pcie_phy_pll_prepare(struct clk_hw *clk_hw)
-{
-	struct k1_pcie_phy *k1_phy = clk_hw_to_k1_phy(clk_hw);
-	void __iomem *regs = k1_phy->regs;
-	u32 val;
-	u32 i;
-
-	if (k1_phy_port_a(k1_phy) && k1_phy->type == PHY_TYPE_USB3)
-		k1_pcie_phy_pll_prepare_usb(k1_phy);
-	else
-		k1_pcie_phy_pll_prepare_pcie(k1_phy);
-
-	/*
-	 * Disable 100 MHz input reference with spread-spectrum
-	 * clocking and select the 24 MHz clock input frequency
-	 */
-	val = readl(regs + PCIE_PU_PLL_1);
-	val &= ~REF_100_WSSC;
-	val &= ~FREF_SEL;
-	val |= FIELD_PREP(FREF_SEL, FREF_24M);
-	writel(val, regs + PCIE_PU_PLL_1);
-
-	/* Mark PLL configuration done on all lanes */
-	for (i = 0; i < k1_phy->pcie_lanes; i++) {
-		val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
-		val |= CFG_SW_PHY_INIT_DONE;
-		writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
-
-		regs += PHY_LANE_OFFSET;	/* Next lane */
-	}
-
-	/*
-	 * Wait for indication the PHY PLL is locked.  Lanes for ports
-	 * B and C share a PLL, so it's enough to sample just lane 0.
-	 */
-	return readl_poll_timeout(k1_phy->regs + PCIE_PU_ADDR_CLK_CFG,
-				  val, val & PLL_READY,
-				  POLL_DELAY, PLL_TIMEOUT);
-}
-
-/* Prepare implies enable, and once enabled, it's always on */
-static const struct clk_ops k1_pcie_phy_pll_ops = {
-	.prepare	= k1_pcie_phy_pll_prepare,
-};
-
-/* We represent the PHY PLL as a private clock */
-static int k1_pcie_phy_pll_setup(struct k1_pcie_phy *k1_phy)
-{
-	struct clk_hw *hw = &k1_phy->pll_hw;
-	struct device *dev = k1_phy->dev;
-	struct clk_init_data init = { };
-	char *name;
-	int ret;
-
-	name = kasprintf(GFP_KERNEL, "pcie%u_phy_pll", k1_phy->phy->id);
-	if (!name)
-		return -ENOMEM;
-
-	init.name = name;
-	init.ops = &k1_pcie_phy_pll_ops;
-	init.parent_data = k1_pcie_phy_data;
-	init.num_parents = ARRAY_SIZE(k1_pcie_phy_data);
-
-	hw->init = &init;
-
-	ret = devm_clk_hw_register(dev, hw);
-
-	kfree(name);	/* __clk_register() duplicates the name we provide */
-
-	if (ret)
-		return ret;
-
-	k1_phy->pll = devm_clk_hw_get_clk(dev, hw, "pll");
-	if (IS_ERR(k1_phy->pll))
-		return PTR_ERR(k1_phy->pll);
-
-	return 0;
-}
-
-/* Select PCIe or USB 3 mode for the combo PHY. */
-static void k1_combo_phy_sel(struct k1_pcie_phy *k1_phy, bool usb)
-{
-	struct regmap *pmu = k1_phy->pmu;
-
-	/* Only change it if it's not already in the desired state */
-	if (!regmap_test_bits(pmu, PMUA_USB_PHY_CTRL0, COMBO_PHY_SEL) == usb)
-		regmap_assign_bits(pmu, PMUA_USB_PHY_CTRL0, COMBO_PHY_SEL, usb);
-}
-
-static void k1_pcie_phy_init_pcie(struct k1_pcie_phy *k1_phy)
-{
-	u32 rx_rterm = k1_phy_rterm_rx();
-	u32 tx_rterm = k1_phy_rterm_tx();
-	void __iomem *regs;
-	u32 val;
-	int i;
-
-	/* For the combo PHY, set PHY to PCIe mode */
-	if (k1_phy_port_a(k1_phy))
-		k1_combo_phy_sel(k1_phy, false);
-
-	regs = k1_phy->regs;
-	for (i = 0; i < k1_phy->pcie_lanes; i++) {
-		val = readl(regs + PCIE_RX_REG1);
-
-		/* Set RX analog front-end receiver termination value */
-		val &= ~AFE_RTERM_REG;
-		val |= FIELD_PREP(AFE_RTERM_REG, rx_rterm);
-
-		/* And enable refclock receiver termination */
-		val |= EN_RTERM;
-		writel(val, regs + PCIE_RX_REG1);
-
-		val = readl(regs + PCIE_RX_REG2);
-		/* Use PCIE_RX_REG1 AFE_RTERM_REG value */
-		val &= ~RX_RTERM_SEL;
-		writel(val, regs + PCIE_RX_REG2);
-
-		val = readl(regs + PCIE_TX_REG1);
-
-		/* Set TX driver termination value */
-		val &= ~TX_RTERM_REG;
-		val |= FIELD_PREP(TX_RTERM_REG, tx_rterm);
-
-		/* Use PCIE_TX_REG1 TX_RTERM_REG value */
-		val |= TX_RTERM_SEL;
-		writel(val, regs + PCIE_TX_REG1);
-
-		/* Set the input clock to 24 MHz, and clear RC_CAL_TOGGLE */
-		val = readl(regs + PCIE_RC_CAL_REG2);
-		val &= CLKSEL;
-		val |= FIELD_PREP(CLKSEL, CLKSEL_24M);
-		val &= ~RC_CAL_TOGGLE;
-		writel(val, regs + PCIE_RC_CAL_REG2);
-
-		/* Now trigger recalibration by setting RC_CAL_TOGGLE again */
-		val |= RC_CAL_TOGGLE;
-		writel(val, regs + PCIE_RC_CAL_REG2);
-
-		val = readl(regs + PCIE_LTSSM_DIS_ENTRY);
-		/* Override the reference clock; set to refclk driver mode */
-		val |= OVRD_REFCLK_MODE;
-		val &= ~CFG_REFCLK_MODE;
-		val |= FIELD_PREP(CFG_REFCLK_MODE, RFCLK_MODE_DRIVER);
-		writel(val, regs + PCIE_LTSSM_DIS_ENTRY);
-
-		regs += PHY_LANE_OFFSET;	/* Next lane */
-	}
-}
-
-/* Only called for combo PHY */
-static void k1_pcie_phy_init_usb(struct k1_pcie_phy *k1_phy)
-{
-	k1_combo_phy_sel(k1_phy, true);
-
-	/* We're not doing any testing */
-	writel(0, k1_phy->regs + USB3_TEST_CTRL);
-}
-
-static int k1_pcie_phy_init(struct phy *phy)
-{
-	struct k1_pcie_phy *k1_phy = phy_get_drvdata(phy);
-
-	/* Note: port type is only valid for port A (both checks needed) */
-	if (k1_phy_port_a(k1_phy) && k1_phy->type == PHY_TYPE_USB3)
-		k1_pcie_phy_init_usb(k1_phy);
-	else
-		k1_pcie_phy_init_pcie(k1_phy);
-
-
-	return clk_prepare_enable(k1_phy->pll);
-}
-
-static int k1_pcie_phy_exit(struct phy *phy)
-{
-	struct k1_pcie_phy *k1_phy = phy_get_drvdata(phy);
-
-	clk_disable_unprepare(k1_phy->pll);
-
-	return 0;
-}
-
-static const struct phy_ops k1_pcie_phy_ops = {
-	.init		= k1_pcie_phy_init,
-	.exit		= k1_pcie_phy_exit,
-	.owner		= THIS_MODULE,
-};
-
-/*
- * Get values needed for calibrating PHYs operating in PCIe mode.  Only
- * the combo PHY is able to do this, and its calibration values are used
- * for configuring all PCIe PHYs.
- *
- * We always need to de-assert the "global" reset on the combo PHY,
- * because the USB driver depends on it.  If used for PCIe, that driver
- * will (also) de-assert this, but by leaving it de-asserted for the
- * combo PHY, the USB driver doesn't have to do this.  Note: although
- * SpacemiT refers to this as the global reset, we name the "phy" reset.
- *
- * In addition, we guarantee the APP_HOLD_PHY_RESET bit is clear for the
- * combo PHY, so the USB driver doesn't have to manage that either.  The
- * PCIe driver is free to change this bit for normal operation.
- *
- * Calibration only needs to be done once.  It's possible calibration has
- * already completed (e.g., it might have happened in the boot loader, or
- * -EPROBE_DEFER might result in this function being called again).  So we
- * check that early too, to avoid doing it more than once.
- *
- * Otherwise we temporarily power up the PHY using the PCIe app clocks
- * and resets, wait for the hardware to indicate calibration is done,
- * grab the value, then shut the PHY down again.
- */
-static int k1_pcie_combo_phy_calibrate(struct k1_pcie_phy *k1_phy)
-{
-	struct reset_control_bulk_data resets[] = {
-		{ .id = "dbi", },
-		{ .id = "mstr", },
-		{ .id = "slv", },
-	};
-	struct clk_bulk_data clocks[] = {
-		{ .id = "dbi", },
-		{ .id = "mstr", },
-		{ .id = "slv", },
-	};
-	struct device *dev = k1_phy->dev;
-	int ret = 0;
-	int val;
-
-	/* Nothing to do if we already set the receiver termination value */
-	if (k1_phy_rterm_valid())
-		return 0;
-
-	/*
-	 * We also guarantee the APP_HOLD_PHY_RESET bit is clear.  We can
-	 * leave this bit clear even if an error happens below.
-	 */
-	regmap_assign_bits(k1_phy->pmu, PCIE_CLK_RES_CTRL,
-			   PCIE_APP_HOLD_PHY_RST, false);
-
-	/* If the calibration already completed (e.g. by U-Boot), we're done */
-	val = readl(k1_phy->regs + PCIE_RCAL_RESULT);
-	if (val & R_TUNE_DONE)
-		goto out_tune_done;
-
-	/* Put the PHY into PCIe mode */
-	k1_combo_phy_sel(k1_phy, false);
-
-	/* Get and enable the PCIe app clocks */
-	ret = clk_bulk_get(dev, ARRAY_SIZE(clocks), clocks);
-	if (ret < 0)
-		goto out_tune_done;
-	ret = clk_bulk_prepare_enable(ARRAY_SIZE(clocks), clocks);
-	if (ret)
-		goto out_put_clocks;
-
-	/* Get the PCIe application resets (not the PHY reset) */
-	ret = reset_control_bulk_get_shared(dev, ARRAY_SIZE(resets), resets);
-	if (ret)
-		goto out_disable_clocks;
-
-	/* De-assert the PCIe application resets */
-	ret = reset_control_bulk_deassert(ARRAY_SIZE(resets), resets);
-	if (ret)
-		goto out_put_resets;
-
-	/*
-	 * This is the core activity here.  Wait for the hardware to
-	 * signal that it has completed calibration/tuning.  Once it
-	 * has, the register value will contain the values we'll
-	 * use to configure PCIe PHYs.
-	 */
-	ret = readl_poll_timeout(k1_phy->regs + PCIE_RCAL_RESULT,
-				 val, val & R_TUNE_DONE,
-				 POLL_DELAY, CALIBRATION_TIMEOUT);
-
-	/* Clean up.  We're done with the resets and clocks */
-	reset_control_bulk_assert(ARRAY_SIZE(resets), resets);
-out_put_resets:
-	reset_control_bulk_put(ARRAY_SIZE(resets), resets);
-out_disable_clocks:
-	clk_bulk_disable_unprepare(ARRAY_SIZE(clocks), clocks);
-out_put_clocks:
-	clk_bulk_put(ARRAY_SIZE(clocks), clocks);
-out_tune_done:
-	/* If we got the value without timing out, set k1_phy_rterm */
-	if (!ret)
-		k1_phy_rterm_set(val);
-
-	return ret;
-}
-
-static struct phy *
-k1_pcie_combo_phy_xlate(struct device *dev, const struct of_phandle_args *args)
-{
-	struct k1_pcie_phy *k1_phy = dev_get_drvdata(dev);
-	u32 type;
-
-	/* The argument specifying the PHY mode is required */
-	if (args->args_count != 1)
-		return ERR_PTR(-EINVAL);
-
-	/* We only support PCIe and USB 3 mode */
-	type = args->args[0];
-	if (type != PHY_TYPE_PCIE && type != PHY_TYPE_USB3)
-		return ERR_PTR(-EINVAL);
-
-	/* This PHY can only be used once */
-	if (k1_phy->type != PHY_NONE)
-		return ERR_PTR(-EBUSY);
-
-	k1_phy->type = type;
-
-	return k1_phy->phy;
-}
-
-/* Use the maximum number of PCIe lanes unless limited by device tree */
-static u32 k1_pcie_num_lanes(struct k1_pcie_phy *k1_phy, bool port_a)
-{
-	struct device *dev = k1_phy->dev;
-	u32 count = 0;
-	u32 max;
-	int ret;
-
-	ret = of_property_read_u32(dev_of_node(dev), "num-lanes", &count);
-	if (count == 1)
-		return 1;
-
-	if (count == 2 && !port_a)
-		return 2;
-
-	max = port_a ? 1 : 2;
-	if (ret != -EINVAL)
-		dev_warn(dev, "bad lane count %u for port; using %u\n",
-			 count, max);
-
-	return max;
-}
-
-static int k1_pcie_combo_phy_probe(struct k1_pcie_phy *k1_phy)
-{
-	struct device *dev = k1_phy->dev;
-	struct regmap *regmap;
-	int ret;
-
-	/* Setting the PHY mode requires access to the PMU regmap */
-	regmap = syscon_regmap_lookup_by_phandle(dev_of_node(dev), SYSCON_APMU);
-	if (IS_ERR(regmap))
-		return dev_err_probe(dev, PTR_ERR(regmap), "failed to get PMU\n");
-	k1_phy->pmu = regmap;
-
-	ret = k1_pcie_combo_phy_calibrate(k1_phy);
-	if (ret)
-		return dev_err_probe(dev, ret, "calibration failed\n");
-
-	/* Needed by k1_pcie_combo_phy_xlate(), which also sets k1_phy->type */
-	dev_set_drvdata(dev, k1_phy);
-
-	return 0;
-}
-
-static int k1_pcie_phy_probe(struct platform_device *pdev)
-{
-	struct phy *(*xlate)(struct device *dev,
-			     const struct of_phandle_args *args);
-	struct device *dev = &pdev->dev;
-	struct reset_control *phy_reset;
-	struct phy_provider *provider;
-	struct k1_pcie_phy *k1_phy;
-	bool probing_port_a;
-	int ret;
-
-	xlate = of_device_get_match_data(dev);
-	probing_port_a = xlate == k1_pcie_combo_phy_xlate;
-
-	/* Only the combo PHY can calibrate, so it must probe first */
-	if (!k1_phy_rterm_valid() && !probing_port_a)
-		return -EPROBE_DEFER;
-
-	k1_phy = devm_kzalloc(dev, sizeof(*k1_phy), GFP_KERNEL);
-	if (!k1_phy)
-		return -ENOMEM;
-	k1_phy->dev = dev;
-
-	k1_phy->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(k1_phy->regs))
-		return dev_err_probe(dev, PTR_ERR(k1_phy->regs),
-				     "error mapping registers\n");
-
-	/* De-assert the PHY (global) reset and leave it that way */
-	phy_reset = devm_reset_control_get_exclusive_deasserted(dev, "phy");
-	if (IS_ERR(phy_reset))
-		return PTR_ERR(phy_reset);
-
-	if (probing_port_a) {
-		ret = k1_pcie_combo_phy_probe(k1_phy);
-		if (ret)
-			return dev_err_probe(dev, ret,
-					     "error probing combo phy\n");
-	}
-
-	k1_phy->pcie_lanes = k1_pcie_num_lanes(k1_phy, probing_port_a);
-
-	k1_phy->phy = devm_phy_create(dev, NULL, &k1_pcie_phy_ops);
-	if (IS_ERR(k1_phy->phy))
-		return dev_err_probe(dev, PTR_ERR(k1_phy->phy),
-				     "error creating phy\n");
-	phy_set_drvdata(k1_phy->phy, k1_phy);
-
-	ret = k1_pcie_phy_pll_setup(k1_phy);
-	if (ret)
-		return dev_err_probe(dev, ret, "error initializing clock\n");
-
-	provider = devm_of_phy_provider_register(dev, xlate);
-	if (IS_ERR(provider))
-		return dev_err_probe(dev, PTR_ERR(provider),
-				     "error registering provider\n");
-	return 0;
-}
-
-static const struct of_device_id k1_pcie_phy_of_match[] = {
-	{ .compatible = "spacemit,k1-combo-phy", k1_pcie_combo_phy_xlate, },
-	{ .compatible = "spacemit,k1-pcie-phy", of_phy_simple_xlate, },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, k1_pcie_phy_of_match);
-
-static struct platform_driver k1_pcie_phy_driver = {
-	.probe	= k1_pcie_phy_probe,
-	.driver = {
-		.of_match_table	= k1_pcie_phy_of_match,
-		.name = "spacemit-k1-pcie-phy",
-	}
-};
-module_platform_driver(k1_pcie_phy_driver);
-
-MODULE_DESCRIPTION("SpacemiT K1 PCIe and USB 3 PHY driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
index 95cd3175926d5..68f1ba8fec4ad 100644
--- a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
+++ b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
@@ -83,7 +83,7 @@ static const struct m31_phy_tbl_entry m31_eusb2_setup_tbl[] = {
 	M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG0, UTMI_PHY_CMN_CTRL_OVERRIDE_EN, 1),
 	M31_EUSB_PHY_INIT_CFG(USB_PHY_UTMI_CTRL5, POR, 1),
 	M31_EUSB_PHY_INIT_CFG(USB_PHY_HS_PHY_CTRL_COMMON0, PHY_ENABLE, 1),
-	M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG1, PLL_EN, 1),
+	M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG1, PLL_EN, 0),
 	M31_EUSB_PHY_INIT_CFG(USB_PHY_FSEL_SEL, FSEL_SEL, 1),
 };
 
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
index df138a5442ebe..771bc7c2ab505 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
@@ -990,6 +990,7 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = {
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e),
@@ -999,13 +1000,11 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = {
 };
 
 static const struct qmp_phy_init_tbl sm8650_ufsphy_g4_pcs[] = {
-	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x13),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04),
 };
 
 static const struct qmp_phy_init_tbl sm8650_ufsphy_g5_pcs[] = {
-	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x05),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x05),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HS_G5_SYNC_LENGTH_CAPABILITY, 0x4d),
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
index 14feb77789b3e..c342479a37980 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
@@ -1679,7 +1679,7 @@ static int qmp_usbc_register_clocks(struct qmp_usbc *qmp, struct device_node *np
 	if (ret)
 		return ret;
 
-	if (qmp->dp_serdes != 0) {
+	if (qmp->dp_serdes) {
 		ret = phy_dp_clks_register(qmp, np);
 		if (ret)
 			return ret;
@@ -1833,7 +1833,7 @@ static int qmp_usbc_parse_dt(struct qmp_usbc *qmp)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	if (offs->dp_serdes != 0) {
+	if (offs->dp_serdes) {
 		qmp->dp_serdes = base + offs->dp_serdes;
 		qmp->dp_tx = base + offs->dp_txa;
 		qmp->dp_tx2 = base + offs->dp_txb;
@@ -1982,7 +1982,7 @@ static int qmp_usbc_probe(struct platform_device *pdev)
 
 	phy_set_drvdata(qmp->usb_phy, qmp);
 
-	if (qmp->dp_serdes != 0) {
+	if (qmp->dp_serdes) {
 		qmp->dp_phy = devm_phy_create(dev, np, &qmp_usbc_dp_phy_ops);
 		if (IS_ERR(qmp->dp_phy)) {
 			ret = PTR_ERR(qmp->dp_phy);
diff --git a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c
index 5b1c82459c126..f48faa2929a68 100644
--- a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c
+++ b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c
@@ -7,6 +7,7 @@
 #include <linux/ethtool.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/phy.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
@@ -28,7 +29,7 @@
 struct qcom_dwmac_sgmii_phy_data {
 	struct regmap *regmap;
 	struct clk *refclk;
-	int speed;
+	phy_interface_t interface;
 };
 
 static void qcom_dwmac_sgmii_phy_init_1g(struct regmap *regmap)
@@ -222,15 +223,18 @@ static int qcom_dwmac_sgmii_phy_calibrate(struct phy *phy)
 	struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy);
 	struct device *dev = phy->dev.parent;
 
-	switch (data->speed) {
-	case SPEED_10:
-	case SPEED_100:
-	case SPEED_1000:
+	switch (data->interface) {
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_1000BASEX:
+		/* 1.25Gbps mode */
 		qcom_dwmac_sgmii_phy_init_1g(data->regmap);
 		break;
-	case SPEED_2500:
+	case PHY_INTERFACE_MODE_2500BASEX:
+		/* 3.125Gbps mode */
 		qcom_dwmac_sgmii_phy_init_2p5g(data->regmap);
 		break;
+	default:
+		return -EINVAL;
 	}
 
 	if (qcom_dwmac_sgmii_phy_poll_status(data->regmap,
@@ -267,8 +271,17 @@ static int qcom_dwmac_sgmii_phy_calibrate(struct phy *phy)
 static int qcom_dwmac_sgmii_phy_power_on(struct phy *phy)
 {
 	struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy);
+	int ret;
 
-	return clk_prepare_enable(data->refclk);
+	ret = clk_prepare_enable(data->refclk);
+	if (ret < 0)
+		return ret;
+
+	ret = qcom_dwmac_sgmii_phy_calibrate(phy);
+	if (ret < 0)
+		clk_disable_unprepare(data->refclk);
+
+	return ret;
 }
 
 static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy)
@@ -286,12 +299,36 @@ static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy)
 	return 0;
 }
 
-static int qcom_dwmac_sgmii_phy_set_speed(struct phy *phy, int speed)
+static int qcom_dwmac_sgmii_phy_validate(struct phy *phy, enum phy_mode mode,
+					 int submode,
+					 union phy_configure_opts *opts)
+{
+	if (mode != PHY_MODE_ETHERNET)
+		return -EINVAL;
+
+	if (submode == PHY_INTERFACE_MODE_SGMII ||
+	    submode == PHY_INTERFACE_MODE_1000BASEX ||
+	    submode == PHY_INTERFACE_MODE_2500BASEX)
+		return 0;
+
+	return -EINVAL;
+}
+
+static int qcom_dwmac_sgmii_phy_set_mode(struct phy *phy, enum phy_mode mode,
+					 int submode)
 {
 	struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy);
+	int ret;
+
+	ret = qcom_dwmac_sgmii_phy_validate(phy, mode, submode, NULL);
+	if (ret)
+		return ret;
+
+	if (submode != data->interface)
+		data->interface = submode;
 
-	if (speed != data->speed)
-		data->speed = speed;
+	if (phy->power_count == 0)
+		return 0;
 
 	return qcom_dwmac_sgmii_phy_calibrate(phy);
 }
@@ -299,7 +336,8 @@ static int qcom_dwmac_sgmii_phy_set_speed(struct phy *phy, int speed)
 static const struct phy_ops qcom_dwmac_sgmii_phy_ops = {
 	.power_on	= qcom_dwmac_sgmii_phy_power_on,
 	.power_off	= qcom_dwmac_sgmii_phy_power_off,
-	.set_speed	= qcom_dwmac_sgmii_phy_set_speed,
+	.set_mode	= qcom_dwmac_sgmii_phy_set_mode,
+	.validate	= qcom_dwmac_sgmii_phy_validate,
 	.calibrate	= qcom_dwmac_sgmii_phy_calibrate,
 	.owner		= THIS_MODULE,
 };
@@ -324,7 +362,7 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENOMEM;
 
-	data->speed = SPEED_10;
+	data->interface = PHY_INTERFACE_MODE_SGMII;
 
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
index cfc2a8d9028d5..79e820e2fe55d 100644
--- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
@@ -939,21 +939,6 @@ static int rcar_gen3_phy_usb2_vbus_regulator_register(struct rcar_gen3_chan *cha
 	return rcar_gen3_phy_usb2_vbus_regulator_get_exclusive_enable(channel, enable);
 }
 
-/* Temporary wrapper until the multiplexer subsystem supports optional muxes */
-static inline struct mux_state *
-devm_mux_state_get_optional(struct device *dev, const char *mux_name)
-{
-	if (!of_property_present(dev->of_node, "mux-states"))
-		return NULL;
-
-	return devm_mux_state_get(dev, mux_name);
-}
-
-static void rcar_gen3_phy_mux_state_deselect(void *data)
-{
-	mux_state_deselect(data);
-}
-
 static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1036,20 +1021,9 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 		phy_set_drvdata(channel->rphys[i].phy, &channel->rphys[i]);
 	}
 
-	mux_state = devm_mux_state_get_optional(dev, NULL);
+	mux_state = devm_mux_state_get_optional_selected(dev, NULL);
 	if (IS_ERR(mux_state))
-		return PTR_ERR(mux_state);
-	if (mux_state) {
-		ret = mux_state_select(mux_state);
-		if (ret)
-			return dev_err_probe(dev, ret, "Failed to select USB mux\n");
-
-		ret = devm_add_action_or_reset(dev, rcar_gen3_phy_mux_state_deselect,
-					       mux_state);
-		if (ret)
-			return dev_err_probe(dev, ret,
-					     "Failed to register USB mux state deselect\n");
-	}
+		return dev_err_probe(dev, PTR_ERR(mux_state), "Failed to get USB mux\n");
 
 	if (channel->phy_data->no_adp_ctrl && channel->is_otg_channel) {
 		ret = rcar_gen3_phy_usb2_vbus_regulator_register(channel);
diff --git a/drivers/phy/spacemit/Kconfig b/drivers/phy/spacemit/Kconfig
index 0136aee2e8a2f..50b0005acf663 100644
--- a/drivers/phy/spacemit/Kconfig
+++ b/drivers/phy/spacemit/Kconfig
@@ -2,6 +2,18 @@
 #
 # Phy drivers for SpacemiT platforms
 #
+config PHY_SPACEMIT_K1_PCIE
+	tristate "PCIe and combo PHY driver for the SpacemiT K1 SoC"
+	depends on ARCH_SPACEMIT || COMPILE_TEST
+	depends on COMMON_CLK
+	depends on HAS_IOMEM
+	depends on OF
+	select GENERIC_PHY
+	default ARCH_SPACEMIT
+	help
+	  Enable support for the PCIe and USB 3 combo PHY and two
+	  PCIe-only PHYs used in the SpacemiT K1 SoC.
+
 config PHY_SPACEMIT_K1_USB2
 	tristate "SpacemiT K1 USB 2.0 PHY support"
 	depends on (ARCH_SPACEMIT || COMPILE_TEST) && OF
diff --git a/drivers/phy/spacemit/Makefile b/drivers/phy/spacemit/Makefile
index fec0b425a9485..a821a21d61422 100644
--- a/drivers/phy/spacemit/Makefile
+++ b/drivers/phy/spacemit/Makefile
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_PHY_SPACEMIT_K1_PCIE)		+= phy-k1-pcie.o
 obj-$(CONFIG_PHY_SPACEMIT_K1_USB2)		+= phy-k1-usb2.o
diff --git a/drivers/phy/spacemit/phy-k1-pcie.c b/drivers/phy/spacemit/phy-k1-pcie.c
new file mode 100644
index 0000000000000..75477bea7f700
--- /dev/null
+++ b/drivers/phy/spacemit/phy-k1-pcie.c
@@ -0,0 +1,670 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SpacemiT K1 PCIe and PCIe/USB 3 combo PHY driver
+ *
+ * Copyright (C) 2025 by RISCstar Solutions Corporation.  All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <dt-bindings/phy/phy.h>
+
+/*
+ * Three PCIe ports are supported in the SpacemiT K1 SoC, and this driver
+ * supports their PHYs.
+ *
+ * The PHY for PCIe port A is different from the PHYs for ports B and C:
+ * - It has one lane, while ports B and C have two
+ * - It is a combo PHY can be used for PCIe or USB 3
+ * - It can automatically calibrate PCIe TX and RX termination settings
+ *
+ * The PHY functionality for PCIe ports B and C is identical:
+ * - They have two PCIe lanes (but can be restricted to 1 via device tree)
+ * - They are used for PCIe only
+ * - They are configured using TX and RX values computed for port A
+ *
+ * A given board is designed to use the combo PHY for either PCIe or USB 3.
+ * Whether the combo PHY is configured for PCIe or USB 3 is specified in
+ * device tree using a phandle plus an argument.  The argument indicates
+ * the type (either PHY_TYPE_PCIE or PHY_TYPE_USB3).
+ *
+ * Each PHY has a reset that it gets and deasserts during initialization.
+ * Each depends also on other clocks and resets provided by the controller
+ * hardware (PCIe or USB) it is associated with.  The controller drivers
+ * are required to enable any clocks and de-assert any resets that affect
+ * PHY operation.  In addition each PHY implements an internal PLL, driven
+ * by an external (24 MHz) oscillator.
+ *
+ * PCIe PHYs must be programmed with RX and TX calibration values.  The
+ * combo PHY is the only one that can determine these values.  They are
+ * determined by temporarily enabling the combo PHY in PCIe mode at probe
+ * time (if necessary).  This calibration only needs to be done once, and
+ * when it has completed the TX and RX values are saved.
+ *
+ * To allow the combo PHY to be enabled for calibration, the resets and
+ * clocks it uses in PCIe mode must be supplied.
+ */
+
+struct k1_pcie_phy {
+	struct device *dev;		/* PHY provider device */
+	struct phy *phy;
+	void __iomem *regs;
+	u32 pcie_lanes;			/* Max (1 or 2) unless limited by DT */
+	struct clk *pll;
+	struct clk_hw pll_hw;		/* Private PLL clock */
+
+	/* The remaining fields are only used for the combo PHY */
+	u32 type;			/* PHY_TYPE_PCIE or PHY_TYPE_USB3 */
+	struct regmap *pmu;		/* MMIO regmap (no errors) */
+};
+
+#define CALIBRATION_TIMEOUT		500000	/* For combo PHY (usec) */
+#define PLL_TIMEOUT			500000	/* For PHY PLL lock (usec) */
+#define POLL_DELAY			500	/* Time between polls (usec) */
+
+/* Selecting the combo PHY operating mode requires APMU regmap access */
+#define SYSCON_APMU			"spacemit,apmu"
+
+/* PMU space, for selecting between PCIe and USB 3 mode (combo PHY only) */
+
+#define PMUA_USB_PHY_CTRL0			0x0110
+#define COMBO_PHY_SEL			BIT(3)	/* 0: PCIe; 1: USB 3 */
+
+#define PCIE_CLK_RES_CTRL			0x03cc
+#define PCIE_APP_HOLD_PHY_RST		BIT(30)
+
+/* PHY register space */
+
+/* Offset between lane 0 and lane 1 registers when there are two */
+#define PHY_LANE_OFFSET				0x0400
+
+/* PHY PLL configuration */
+#define PCIE_PU_ADDR_CLK_CFG			0x0008
+#define PLL_READY			BIT(0)		/* read-only */
+#define CFG_INTERNAL_TIMER_ADJ		GENMASK(10, 7)
+#define TIMER_ADJ_USB		0x2
+#define TIMER_ADJ_PCIE		0x6
+#define CFG_SW_PHY_INIT_DONE		BIT(11)	/* We set after PLL config */
+
+#define PCIE_RC_DONE_STATUS			0x0018
+#define CFG_FORCE_RCV_RETRY		BIT(10)		/* Used for PCIe */
+
+/* PCIe PHY lane calibration; assumes 24MHz input clock */
+#define PCIE_RC_CAL_REG2			0x0020
+#define RC_CAL_TOGGLE			BIT(22)
+#define CLKSEL				GENMASK(31, 29)
+#define CLKSEL_24M		0x3
+
+/* Additional PHY PLL configuration (USB 3 and PCIe) */
+#define PCIE_PU_PLL_1				0x0048
+#define REF_100_WSSC			BIT(12)	/* 1: input is 100MHz, SSC */
+#define FREF_SEL			GENMASK(15, 13)
+#define FREF_24M		0x1
+#define SSC_DEP_SEL			GENMASK(19, 16)
+#define SSC_DEP_NONE		0x0
+#define SSC_DEP_5000PPM		0xa
+
+/* PCIe PHY configuration */
+#define PCIE_PU_PLL_2				0x004c
+#define GEN_REF100			BIT(4)	/* 1: generate 100MHz clk */
+
+#define PCIE_RX_REG1				0x0050
+#define EN_RTERM			BIT(3)
+#define AFE_RTERM_REG			GENMASK(11, 8)
+
+#define PCIE_RX_REG2				0x0054
+#define RX_RTERM_SEL			BIT(5)	/* 0: use AFE_RTERM_REG value */
+
+#define PCIE_LTSSM_DIS_ENTRY			0x005c
+#define CFG_REFCLK_MODE			GENMASK(9, 8)
+#define RFCLK_MODE_DRIVER	0x1
+#define OVRD_REFCLK_MODE		BIT(10)	/* 1: use CFG_RFCLK_MODE */
+
+#define PCIE_TX_REG1				0x0064
+#define TX_RTERM_REG			GENMASK(15, 12)
+#define TX_RTERM_SEL			BIT(25)	/* 1: use TX_RTERM_REG */
+
+/* Zeroed for the combo PHY operating in USB mode */
+#define USB3_TEST_CTRL				0x0068
+
+/* PHY calibration values, determined by the combo PHY at probe time */
+#define PCIE_RCAL_RESULT			0x0084	/* Port A PHY only */
+#define RTERM_VALUE_RX			GENMASK(3, 0)
+#define RTERM_VALUE_TX			GENMASK(7, 4)
+#define R_TUNE_DONE			BIT(10)
+
+static u32 k1_phy_rterm = ~0;     /* Invalid initial value */
+
+/* Save the RX and TX receiver termination values */
+static void k1_phy_rterm_set(u32 val)
+{
+	k1_phy_rterm = val & (RTERM_VALUE_RX | RTERM_VALUE_TX);
+}
+
+static bool k1_phy_rterm_valid(void)
+{
+	/* Valid if no bits outside those we care about are set */
+	return !(k1_phy_rterm & ~(RTERM_VALUE_RX | RTERM_VALUE_TX));
+}
+
+static u32 k1_phy_rterm_rx(void)
+{
+	return FIELD_GET(RTERM_VALUE_RX, k1_phy_rterm);
+}
+
+static u32 k1_phy_rterm_tx(void)
+{
+	return FIELD_GET(RTERM_VALUE_TX, k1_phy_rterm);
+}
+
+/* Only the combo PHY has a PMU pointer defined */
+static bool k1_phy_port_a(struct k1_pcie_phy *k1_phy)
+{
+	return !!k1_phy->pmu;
+}
+
+/* The PLL clocks are driven by the external oscillator */
+static const struct clk_parent_data k1_pcie_phy_data[] = {
+	{ .fw_name = "refclk", },
+};
+
+static struct k1_pcie_phy *clk_hw_to_k1_phy(struct clk_hw *clk_hw)
+{
+	return container_of(clk_hw, struct k1_pcie_phy, pll_hw);
+}
+
+/* USB mode only works on the combo PHY, which has only one lane */
+static void k1_pcie_phy_pll_prepare_usb(struct k1_pcie_phy *k1_phy)
+{
+	void __iomem *regs = k1_phy->regs;
+	u32 val;
+
+	val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
+	val &= ~CFG_INTERNAL_TIMER_ADJ;
+	val |= FIELD_PREP(CFG_INTERNAL_TIMER_ADJ, TIMER_ADJ_USB);
+	writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
+
+	val = readl(regs + PCIE_PU_PLL_1);
+	val &= ~SSC_DEP_SEL;
+	val |= FIELD_PREP(SSC_DEP_SEL, SSC_DEP_5000PPM);
+	writel(val, regs + PCIE_PU_PLL_1);
+}
+
+/* Perform PCIe-specific register updates before starting the PLL clock */
+static void k1_pcie_phy_pll_prepare_pcie(struct k1_pcie_phy *k1_phy)
+{
+	void __iomem *regs = k1_phy->regs;
+	u32 val;
+	u32 i;
+
+	for (i = 0; i < k1_phy->pcie_lanes; i++) {
+		val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
+		val &= ~CFG_INTERNAL_TIMER_ADJ;
+		val |= FIELD_PREP(CFG_INTERNAL_TIMER_ADJ, TIMER_ADJ_PCIE);
+		writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
+
+		regs += PHY_LANE_OFFSET;	/* Next lane */
+	}
+
+	regs = k1_phy->regs;
+	val = readl(regs + PCIE_RC_DONE_STATUS);
+	val |= CFG_FORCE_RCV_RETRY;
+	writel(val, regs + PCIE_RC_DONE_STATUS);
+
+	val = readl(regs + PCIE_PU_PLL_1);
+	val &= ~SSC_DEP_SEL;
+	val |= FIELD_PREP(SSC_DEP_SEL, SSC_DEP_NONE);
+	writel(val, regs + PCIE_PU_PLL_1);
+
+	val = readl(regs + PCIE_PU_PLL_2);
+	val |= GEN_REF100;		/* Enable 100 MHz PLL output clock */
+	writel(val, regs + PCIE_PU_PLL_2);
+}
+
+static int k1_pcie_phy_pll_prepare(struct clk_hw *clk_hw)
+{
+	struct k1_pcie_phy *k1_phy = clk_hw_to_k1_phy(clk_hw);
+	void __iomem *regs = k1_phy->regs;
+	u32 val;
+	u32 i;
+
+	if (k1_phy_port_a(k1_phy) && k1_phy->type == PHY_TYPE_USB3)
+		k1_pcie_phy_pll_prepare_usb(k1_phy);
+	else
+		k1_pcie_phy_pll_prepare_pcie(k1_phy);
+
+	/*
+	 * Disable 100 MHz input reference with spread-spectrum
+	 * clocking and select the 24 MHz clock input frequency
+	 */
+	val = readl(regs + PCIE_PU_PLL_1);
+	val &= ~REF_100_WSSC;
+	val &= ~FREF_SEL;
+	val |= FIELD_PREP(FREF_SEL, FREF_24M);
+	writel(val, regs + PCIE_PU_PLL_1);
+
+	/* Mark PLL configuration done on all lanes */
+	for (i = 0; i < k1_phy->pcie_lanes; i++) {
+		val = readl(regs + PCIE_PU_ADDR_CLK_CFG);
+		val |= CFG_SW_PHY_INIT_DONE;
+		writel(val, regs + PCIE_PU_ADDR_CLK_CFG);
+
+		regs += PHY_LANE_OFFSET;	/* Next lane */
+	}
+
+	/*
+	 * Wait for indication the PHY PLL is locked.  Lanes for ports
+	 * B and C share a PLL, so it's enough to sample just lane 0.
+	 */
+	return readl_poll_timeout(k1_phy->regs + PCIE_PU_ADDR_CLK_CFG,
+				  val, val & PLL_READY,
+				  POLL_DELAY, PLL_TIMEOUT);
+}
+
+/* Prepare implies enable, and once enabled, it's always on */
+static const struct clk_ops k1_pcie_phy_pll_ops = {
+	.prepare	= k1_pcie_phy_pll_prepare,
+};
+
+/* We represent the PHY PLL as a private clock */
+static int k1_pcie_phy_pll_setup(struct k1_pcie_phy *k1_phy)
+{
+	struct clk_hw *hw = &k1_phy->pll_hw;
+	struct device *dev = k1_phy->dev;
+	struct clk_init_data init = { };
+	char *name;
+	int ret;
+
+	name = kasprintf(GFP_KERNEL, "pcie%u_phy_pll", k1_phy->phy->id);
+	if (!name)
+		return -ENOMEM;
+
+	init.name = name;
+	init.ops = &k1_pcie_phy_pll_ops;
+	init.parent_data = k1_pcie_phy_data;
+	init.num_parents = ARRAY_SIZE(k1_pcie_phy_data);
+
+	hw->init = &init;
+
+	ret = devm_clk_hw_register(dev, hw);
+
+	kfree(name);	/* __clk_register() duplicates the name we provide */
+
+	if (ret)
+		return ret;
+
+	k1_phy->pll = devm_clk_hw_get_clk(dev, hw, "pll");
+	if (IS_ERR(k1_phy->pll))
+		return PTR_ERR(k1_phy->pll);
+
+	return 0;
+}
+
+/* Select PCIe or USB 3 mode for the combo PHY. */
+static void k1_combo_phy_sel(struct k1_pcie_phy *k1_phy, bool usb)
+{
+	struct regmap *pmu = k1_phy->pmu;
+
+	/* Only change it if it's not already in the desired state */
+	if (!regmap_test_bits(pmu, PMUA_USB_PHY_CTRL0, COMBO_PHY_SEL) == usb)
+		regmap_assign_bits(pmu, PMUA_USB_PHY_CTRL0, COMBO_PHY_SEL, usb);
+}
+
+static void k1_pcie_phy_init_pcie(struct k1_pcie_phy *k1_phy)
+{
+	u32 rx_rterm = k1_phy_rterm_rx();
+	u32 tx_rterm = k1_phy_rterm_tx();
+	void __iomem *regs;
+	u32 val;
+	int i;
+
+	/* For the combo PHY, set PHY to PCIe mode */
+	if (k1_phy_port_a(k1_phy))
+		k1_combo_phy_sel(k1_phy, false);
+
+	regs = k1_phy->regs;
+	for (i = 0; i < k1_phy->pcie_lanes; i++) {
+		val = readl(regs + PCIE_RX_REG1);
+
+		/* Set RX analog front-end receiver termination value */
+		val &= ~AFE_RTERM_REG;
+		val |= FIELD_PREP(AFE_RTERM_REG, rx_rterm);
+
+		/* And enable refclock receiver termination */
+		val |= EN_RTERM;
+		writel(val, regs + PCIE_RX_REG1);
+
+		val = readl(regs + PCIE_RX_REG2);
+		/* Use PCIE_RX_REG1 AFE_RTERM_REG value */
+		val &= ~RX_RTERM_SEL;
+		writel(val, regs + PCIE_RX_REG2);
+
+		val = readl(regs + PCIE_TX_REG1);
+
+		/* Set TX driver termination value */
+		val &= ~TX_RTERM_REG;
+		val |= FIELD_PREP(TX_RTERM_REG, tx_rterm);
+
+		/* Use PCIE_TX_REG1 TX_RTERM_REG value */
+		val |= TX_RTERM_SEL;
+		writel(val, regs + PCIE_TX_REG1);
+
+		/* Set the input clock to 24 MHz, and clear RC_CAL_TOGGLE */
+		val = readl(regs + PCIE_RC_CAL_REG2);
+		val &= CLKSEL;
+		val |= FIELD_PREP(CLKSEL, CLKSEL_24M);
+		val &= ~RC_CAL_TOGGLE;
+		writel(val, regs + PCIE_RC_CAL_REG2);
+
+		/* Now trigger recalibration by setting RC_CAL_TOGGLE again */
+		val |= RC_CAL_TOGGLE;
+		writel(val, regs + PCIE_RC_CAL_REG2);
+
+		val = readl(regs + PCIE_LTSSM_DIS_ENTRY);
+		/* Override the reference clock; set to refclk driver mode */
+		val |= OVRD_REFCLK_MODE;
+		val &= ~CFG_REFCLK_MODE;
+		val |= FIELD_PREP(CFG_REFCLK_MODE, RFCLK_MODE_DRIVER);
+		writel(val, regs + PCIE_LTSSM_DIS_ENTRY);
+
+		regs += PHY_LANE_OFFSET;	/* Next lane */
+	}
+}
+
+/* Only called for combo PHY */
+static void k1_pcie_phy_init_usb(struct k1_pcie_phy *k1_phy)
+{
+	k1_combo_phy_sel(k1_phy, true);
+
+	/* We're not doing any testing */
+	writel(0, k1_phy->regs + USB3_TEST_CTRL);
+}
+
+static int k1_pcie_phy_init(struct phy *phy)
+{
+	struct k1_pcie_phy *k1_phy = phy_get_drvdata(phy);
+
+	/* Note: port type is only valid for port A (both checks needed) */
+	if (k1_phy_port_a(k1_phy) && k1_phy->type == PHY_TYPE_USB3)
+		k1_pcie_phy_init_usb(k1_phy);
+	else
+		k1_pcie_phy_init_pcie(k1_phy);
+
+
+	return clk_prepare_enable(k1_phy->pll);
+}
+
+static int k1_pcie_phy_exit(struct phy *phy)
+{
+	struct k1_pcie_phy *k1_phy = phy_get_drvdata(phy);
+
+	clk_disable_unprepare(k1_phy->pll);
+
+	return 0;
+}
+
+static const struct phy_ops k1_pcie_phy_ops = {
+	.init		= k1_pcie_phy_init,
+	.exit		= k1_pcie_phy_exit,
+	.owner		= THIS_MODULE,
+};
+
+/*
+ * Get values needed for calibrating PHYs operating in PCIe mode.  Only
+ * the combo PHY is able to do this, and its calibration values are used
+ * for configuring all PCIe PHYs.
+ *
+ * We always need to de-assert the "global" reset on the combo PHY,
+ * because the USB driver depends on it.  If used for PCIe, that driver
+ * will (also) de-assert this, but by leaving it de-asserted for the
+ * combo PHY, the USB driver doesn't have to do this.  Note: although
+ * SpacemiT refers to this as the global reset, we name the "phy" reset.
+ *
+ * In addition, we guarantee the APP_HOLD_PHY_RESET bit is clear for the
+ * combo PHY, so the USB driver doesn't have to manage that either.  The
+ * PCIe driver is free to change this bit for normal operation.
+ *
+ * Calibration only needs to be done once.  It's possible calibration has
+ * already completed (e.g., it might have happened in the boot loader, or
+ * -EPROBE_DEFER might result in this function being called again).  So we
+ * check that early too, to avoid doing it more than once.
+ *
+ * Otherwise we temporarily power up the PHY using the PCIe app clocks
+ * and resets, wait for the hardware to indicate calibration is done,
+ * grab the value, then shut the PHY down again.
+ */
+static int k1_pcie_combo_phy_calibrate(struct k1_pcie_phy *k1_phy)
+{
+	struct reset_control_bulk_data resets[] = {
+		{ .id = "dbi", },
+		{ .id = "mstr", },
+		{ .id = "slv", },
+	};
+	struct clk_bulk_data clocks[] = {
+		{ .id = "dbi", },
+		{ .id = "mstr", },
+		{ .id = "slv", },
+	};
+	struct device *dev = k1_phy->dev;
+	int ret = 0;
+	int val;
+
+	/* Nothing to do if we already set the receiver termination value */
+	if (k1_phy_rterm_valid())
+		return 0;
+
+	/*
+	 * We also guarantee the APP_HOLD_PHY_RESET bit is clear.  We can
+	 * leave this bit clear even if an error happens below.
+	 */
+	regmap_assign_bits(k1_phy->pmu, PCIE_CLK_RES_CTRL,
+			   PCIE_APP_HOLD_PHY_RST, false);
+
+	/* If the calibration already completed (e.g. by U-Boot), we're done */
+	val = readl(k1_phy->regs + PCIE_RCAL_RESULT);
+	if (val & R_TUNE_DONE)
+		goto out_tune_done;
+
+	/* Put the PHY into PCIe mode */
+	k1_combo_phy_sel(k1_phy, false);
+
+	/* Get and enable the PCIe app clocks */
+	ret = clk_bulk_get(dev, ARRAY_SIZE(clocks), clocks);
+	if (ret < 0)
+		goto out_tune_done;
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(clocks), clocks);
+	if (ret)
+		goto out_put_clocks;
+
+	/* Get the PCIe application resets (not the PHY reset) */
+	ret = reset_control_bulk_get_shared(dev, ARRAY_SIZE(resets), resets);
+	if (ret)
+		goto out_disable_clocks;
+
+	/* De-assert the PCIe application resets */
+	ret = reset_control_bulk_deassert(ARRAY_SIZE(resets), resets);
+	if (ret)
+		goto out_put_resets;
+
+	/*
+	 * This is the core activity here.  Wait for the hardware to
+	 * signal that it has completed calibration/tuning.  Once it
+	 * has, the register value will contain the values we'll
+	 * use to configure PCIe PHYs.
+	 */
+	ret = readl_poll_timeout(k1_phy->regs + PCIE_RCAL_RESULT,
+				 val, val & R_TUNE_DONE,
+				 POLL_DELAY, CALIBRATION_TIMEOUT);
+
+	/* Clean up.  We're done with the resets and clocks */
+	reset_control_bulk_assert(ARRAY_SIZE(resets), resets);
+out_put_resets:
+	reset_control_bulk_put(ARRAY_SIZE(resets), resets);
+out_disable_clocks:
+	clk_bulk_disable_unprepare(ARRAY_SIZE(clocks), clocks);
+out_put_clocks:
+	clk_bulk_put(ARRAY_SIZE(clocks), clocks);
+out_tune_done:
+	/* If we got the value without timing out, set k1_phy_rterm */
+	if (!ret)
+		k1_phy_rterm_set(val);
+
+	return ret;
+}
+
+static struct phy *
+k1_pcie_combo_phy_xlate(struct device *dev, const struct of_phandle_args *args)
+{
+	struct k1_pcie_phy *k1_phy = dev_get_drvdata(dev);
+	u32 type;
+
+	/* The argument specifying the PHY mode is required */
+	if (args->args_count != 1)
+		return ERR_PTR(-EINVAL);
+
+	/* We only support PCIe and USB 3 mode */
+	type = args->args[0];
+	if (type != PHY_TYPE_PCIE && type != PHY_TYPE_USB3)
+		return ERR_PTR(-EINVAL);
+
+	/* This PHY can only be used once */
+	if (k1_phy->type != PHY_NONE)
+		return ERR_PTR(-EBUSY);
+
+	k1_phy->type = type;
+
+	return k1_phy->phy;
+}
+
+/* Use the maximum number of PCIe lanes unless limited by device tree */
+static u32 k1_pcie_num_lanes(struct k1_pcie_phy *k1_phy, bool port_a)
+{
+	struct device *dev = k1_phy->dev;
+	u32 count = 0;
+	u32 max;
+	int ret;
+
+	ret = of_property_read_u32(dev_of_node(dev), "num-lanes", &count);
+	if (count == 1)
+		return 1;
+
+	if (count == 2 && !port_a)
+		return 2;
+
+	max = port_a ? 1 : 2;
+	if (ret != -EINVAL)
+		dev_warn(dev, "bad lane count %u for port; using %u\n",
+			 count, max);
+
+	return max;
+}
+
+static int k1_pcie_combo_phy_probe(struct k1_pcie_phy *k1_phy)
+{
+	struct device *dev = k1_phy->dev;
+	struct regmap *regmap;
+	int ret;
+
+	/* Setting the PHY mode requires access to the PMU regmap */
+	regmap = syscon_regmap_lookup_by_phandle(dev_of_node(dev), SYSCON_APMU);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "failed to get PMU\n");
+	k1_phy->pmu = regmap;
+
+	ret = k1_pcie_combo_phy_calibrate(k1_phy);
+	if (ret)
+		return dev_err_probe(dev, ret, "calibration failed\n");
+
+	/* Needed by k1_pcie_combo_phy_xlate(), which also sets k1_phy->type */
+	dev_set_drvdata(dev, k1_phy);
+
+	return 0;
+}
+
+static int k1_pcie_phy_probe(struct platform_device *pdev)
+{
+	struct phy *(*xlate)(struct device *dev,
+			     const struct of_phandle_args *args);
+	struct device *dev = &pdev->dev;
+	struct reset_control *phy_reset;
+	struct phy_provider *provider;
+	struct k1_pcie_phy *k1_phy;
+	bool probing_port_a;
+	int ret;
+
+	xlate = of_device_get_match_data(dev);
+	probing_port_a = xlate == k1_pcie_combo_phy_xlate;
+
+	/* Only the combo PHY can calibrate, so it must probe first */
+	if (!k1_phy_rterm_valid() && !probing_port_a)
+		return -EPROBE_DEFER;
+
+	k1_phy = devm_kzalloc(dev, sizeof(*k1_phy), GFP_KERNEL);
+	if (!k1_phy)
+		return -ENOMEM;
+	k1_phy->dev = dev;
+
+	k1_phy->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(k1_phy->regs))
+		return dev_err_probe(dev, PTR_ERR(k1_phy->regs),
+				     "error mapping registers\n");
+
+	/* De-assert the PHY (global) reset and leave it that way */
+	phy_reset = devm_reset_control_get_exclusive_deasserted(dev, "phy");
+	if (IS_ERR(phy_reset))
+		return PTR_ERR(phy_reset);
+
+	if (probing_port_a) {
+		ret = k1_pcie_combo_phy_probe(k1_phy);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "error probing combo phy\n");
+	}
+
+	k1_phy->pcie_lanes = k1_pcie_num_lanes(k1_phy, probing_port_a);
+
+	k1_phy->phy = devm_phy_create(dev, NULL, &k1_pcie_phy_ops);
+	if (IS_ERR(k1_phy->phy))
+		return dev_err_probe(dev, PTR_ERR(k1_phy->phy),
+				     "error creating phy\n");
+	phy_set_drvdata(k1_phy->phy, k1_phy);
+
+	ret = k1_pcie_phy_pll_setup(k1_phy);
+	if (ret)
+		return dev_err_probe(dev, ret, "error initializing clock\n");
+
+	provider = devm_of_phy_provider_register(dev, xlate);
+	if (IS_ERR(provider))
+		return dev_err_probe(dev, PTR_ERR(provider),
+				     "error registering provider\n");
+	return 0;
+}
+
+static const struct of_device_id k1_pcie_phy_of_match[] = {
+	{ .compatible = "spacemit,k1-combo-phy", k1_pcie_combo_phy_xlate, },
+	{ .compatible = "spacemit,k1-pcie-phy", of_phy_simple_xlate, },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, k1_pcie_phy_of_match);
+
+static struct platform_driver k1_pcie_phy_driver = {
+	.probe	= k1_pcie_phy_probe,
+	.driver = {
+		.of_match_table	= k1_pcie_phy_of_match,
+		.name = "spacemit-k1-pcie-phy",
+	}
+};
+module_platform_driver(k1_pcie_phy_driver);
+
+MODULE_DESCRIPTION("SpacemiT K1 PCIe and USB 3 PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/spacemit/phy-k1-usb2.c b/drivers/phy/spacemit/phy-k1-usb2.c
index 3420613800125..9215d0b223b2d 100644
--- a/drivers/phy/spacemit/phy-k1-usb2.c
+++ b/drivers/phy/spacemit/phy-k1-usb2.c
@@ -48,6 +48,9 @@
 #define  PHY_CLK_HSTXP_EN		BIT(3)		/* clock hstxp enable */
 #define  PHY_HSTXP_MODE			BIT(4)		/* 0: force en_txp to be 1; 1: no force */
 
+#define PHY_K1_HS_HOST_DISC		0x40
+#define  PHY_K1_HS_HOST_DISC_CLR		BIT(0)
+
 #define PHY_PLL_DIV_CFG			0x98
 #define  PHY_FDIV_FRACT_8_15		GENMASK(7, 0)
 #define  PHY_FDIV_FRACT_16_19		GENMASK(11, 8)
@@ -142,9 +145,20 @@ static int spacemit_usb2phy_exit(struct phy *phy)
 	return 0;
 }
 
+static int spacemit_usb2phy_disconnect(struct phy *phy, int port)
+{
+	struct spacemit_usb2phy *sphy = phy_get_drvdata(phy);
+
+	regmap_update_bits(sphy->regmap_base, PHY_K1_HS_HOST_DISC,
+			   PHY_K1_HS_HOST_DISC_CLR, PHY_K1_HS_HOST_DISC_CLR);
+
+	return 0;
+}
+
 static const struct phy_ops spacemit_usb2phy_ops = {
 	.init = spacemit_usb2phy_init,
 	.exit = spacemit_usb2phy_exit,
+	.disconnect = spacemit_usb2phy_disconnect,
 	.owner = THIS_MODULE,
 };
 
diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c
index 6e9ecb88dc8b7..6b584706b913a 100644
--- a/drivers/phy/ti/phy-j721e-wiz.c
+++ b/drivers/phy/ti/phy-j721e-wiz.c
@@ -1425,6 +1425,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz)
 			dev_err(dev,
 				"%s: Reading \"reg\" from \"%s\" failed: %d\n",
 				__func__, subnode->name, ret);
+			of_node_put(serdes);
 			return ret;
 		}
 		of_property_read_u32(subnode, "cdns,num-lanes", &num_lanes);
@@ -1439,6 +1440,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz)
 		}
 	}
 
+	of_node_put(serdes);
 	return 0;
 }
 
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index afecd9407f535..03f2e3ee065f4 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -77,7 +77,6 @@ config PINCTRL_APPLE_GPIO
 	select GPIOLIB_IRQCHIP
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
-	select OF_GPIO
 	help
 	  This is the driver for the GPIO controller found on Apple ARM SoCs,
 	  including M1.
@@ -126,7 +125,6 @@ config PINCTRL_AT91PIO4
 	select GENERIC_PINCONF
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
-	select OF_GPIO
 	help
 	  Say Y here to enable the at91 pinctrl/gpio driver for Atmel PIO4
 	  controller available on sama5d2 SoC.
@@ -254,11 +252,11 @@ config PINCTRL_EQUILIBRIUM
 config PINCTRL_EYEQ5
 	bool "Mobileye EyeQ5 pinctrl driver"
 	depends on OF
-	depends on MACH_EYEQ5 || COMPILE_TEST
+	depends on MACH_EYEQ5 || MACH_EYEQ6LPLUS || COMPILE_TEST
 	select PINMUX
 	select GENERIC_PINCONF
 	select AUXILIARY_BUS
-	default MACH_EYEQ5
+	default MACH_EYEQ5 || MACH_EYEQ6LPLUS
 	help
 	  Pin controller driver for the Mobileye EyeQ5 platform. It does both
 	  pin config & pin muxing. It does not handle GPIO.
@@ -293,7 +291,6 @@ config PINCTRL_K210
 	select GENERIC_PINMUX_FUNCTIONS
 	select GENERIC_PINCONF
 	select GPIOLIB
-	select OF_GPIO
 	select REGMAP_MMIO
 	default SOC_CANAAN_K210
 	help
@@ -419,7 +416,6 @@ config PINCTRL_MICROCHIP_SGPIO
 	select GENERIC_PINCONF
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
-	select OF_GPIO
 	help
 	  Support for the serial GPIO interface used on Microsemi and
 	  Microchip SoCs. By using a serial interface, the SIO
@@ -441,7 +437,6 @@ config PINCTRL_OCELOT
 	select GENERIC_PINCONF
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
-	select OF_GPIO
 	select REGMAP_MMIO
 	help
 	  Support for the internal GPIO interfaces on Microsemi Ocelot and
@@ -478,11 +473,10 @@ config PINCTRL_PEF2256
 config PINCTRL_PIC32
 	bool "Microchip PIC32 pin controller driver"
 	depends on OF
-	depends on MACH_PIC32
+	depends on MACH_PIC32 || COMPILE_TEST
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB_IRQCHIP
-	select OF_GPIO
 	help
 	  This is the pin controller and gpio driver for Microchip PIC32
 	  microcontrollers. This option is selected automatically when specific
@@ -499,7 +493,6 @@ config PINCTRL_PISTACHIO
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB_IRQCHIP
-	select OF_GPIO
 	help
 	  This support pinctrl and GPIO driver for IMG Pistachio SoC.
 
@@ -521,7 +514,6 @@ config PINCTRL_ROCKCHIP
 	select GENERIC_PINCONF
 	select GENERIC_IRQ_CHIP
 	select MFD_SYSCON
-	select OF_GPIO
 	default ARCH_ROCKCHIP
 	help
           This support pinctrl and GPIO driver for Rockchip SoCs.
@@ -557,7 +549,6 @@ config PINCTRL_ST
 config PINCTRL_STMFX
 	tristate "STMicroelectronics STMFX GPIO expander pinctrl driver"
 	depends on I2C
-	depends on OF_GPIO
 	depends on HAS_IOMEM
 	select GENERIC_PINCONF
 	select GPIOLIB_IRQCHIP
diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig
index 096d0778427e2..206f3f1249cf5 100644
--- a/drivers/pinctrl/bcm/Kconfig
+++ b/drivers/pinctrl/bcm/Kconfig
@@ -120,7 +120,7 @@ source "drivers/pinctrl/bcm/Kconfig.stb"
 
 config PINCTRL_IPROC_GPIO
 	bool "Broadcom iProc GPIO (with PINCONF) driver"
-	depends on OF_GPIO && (ARCH_BCM_IPROC || COMPILE_TEST)
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	select GPIOLIB_IRQCHIP
 	select PINCONF
 	select GENERIC_PINCONF
@@ -185,7 +185,7 @@ config PINCTRL_NS
 
 config PINCTRL_NSP_GPIO
 	bool "Broadcom NSP GPIO (with PINCONF) driver"
-	depends on OF_GPIO && (ARCH_BCM_NSP || COMPILE_TEST)
+	depends on ARCH_BCM_NSP || COMPILE_TEST
 	select GPIOLIB_IRQCHIP
 	select PINCONF
 	select GENERIC_PINCONF
diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
index b5e97689589fb..254161d52da75 100644
--- a/drivers/pinctrl/core.c
+++ b/drivers/pinctrl/core.c
@@ -1350,7 +1350,8 @@ static int pinctrl_commit_state(struct pinctrl *p, struct pinctrl_state *state)
 	goto restore_old_state;
 
 unapply_new_state:
-	dev_err(p->dev, "Error applying setting, reverse things back\n");
+	dev_err_probe(p->dev, ret,
+		      "Error applying setting, reverse things back\n");
 
 	/*
 	 * All we can do here is pinmux_disable_setting.
@@ -1991,7 +1992,7 @@ static void pinctrl_init_device_debugfs(struct pinctrl_dev *pctldev)
 	device_root = debugfs_create_dir(debugfs_name, debugfs_root);
 	pctldev->device_root = device_root;
 
-	if (IS_ERR(device_root) || !device_root) {
+	if (IS_ERR_OR_NULL(device_root)) {
 		pr_warn("failed to create debugfs directory for %s\n",
 			dev_name(pctldev->dev));
 		return;
diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c
index d0c3e26409a83..02a271dd292f2 100644
--- a/drivers/pinctrl/devicetree.c
+++ b/drivers/pinctrl/devicetree.c
@@ -175,7 +175,7 @@ static int dt_to_map_one_config(struct pinctrl *p,
 		 * return.
 		 */
 		dev_info(p->dev,
-			 "there is not valid maps for state %s\n", statename);
+			 "there are no valid maps for state %s\n", statename);
 		return 0;
 	}
 
diff --git a/drivers/pinctrl/freescale/Kconfig b/drivers/pinctrl/freescale/Kconfig
index 8d24decd3f074..fd53cf5bb843d 100644
--- a/drivers/pinctrl/freescale/Kconfig
+++ b/drivers/pinctrl/freescale/Kconfig
@@ -9,7 +9,7 @@ config PINCTRL_IMX
 
 config PINCTRL_IMX_SCMI
 	tristate "i.MX95 pinctrl driver using SCMI protocol interface"
-	depends on ARM_SCMI_PROTOCOL && OF
+	depends on (ARM_SCMI_PROTOCOL && OF && ARCH_MXC) || COMPILE_TEST
 	select PINMUX
 	select GENERIC_PINCONF
 	select GENERIC_PINCTRL_GROUPS
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 9d32bb8bc13a2..97bf5ec78db4e 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -53,8 +53,6 @@
 #define PADOWN_MASK(p)			(GENMASK(3, 0) << PADOWN_SHIFT(p))
 #define PADOWN_GPP(p)			((p) / 8)
 
-#define PWMC				0x204
-
 /* Offset from pad_regs */
 #define PADCFG0				0x000
 #define PADCFG0_RXEVCFG_MASK		GENMASK(26, 25)
@@ -205,19 +203,25 @@ static bool intel_pad_owned_by_host(const struct intel_pinctrl *pctrl, unsigned
 	community = intel_get_community(pctrl, pin);
 	if (!community)
 		return false;
-	if (!community->padown_offset)
+
+	/* If padown_offset is not provided, assume host ownership */
+	padown = community->regs + community->padown_offset;
+	if (padown == community->regs)
 		return true;
 
+	/* New HW generations have extended PAD_OWN registers */
+	if (community->features & PINCTRL_FEATURE_3BIT_PAD_OWN)
+		return !(readl(padown + pin_to_padno(community, pin) * 4) & 7);
+
 	padgrp = intel_community_get_padgroup(community, pin);
 	if (!padgrp)
 		return false;
 
 	gpp_offset = padgroup_offset(padgrp, pin);
 	gpp = PADOWN_GPP(gpp_offset);
-	offset = community->padown_offset + padgrp->padown_num * 4 + gpp * 4;
-	padown = community->regs + offset;
+	offset = padgrp->padown_num * 4 + gpp * 4;
 
-	return !(readl(padown) & PADOWN_MASK(gpp_offset));
+	return !(readl(padown + offset) & PADOWN_MASK(gpp_offset));
 }
 
 static bool intel_pad_acpi_mode(const struct intel_pinctrl *pctrl, unsigned int pin)
@@ -1549,8 +1553,10 @@ static int intel_pinctrl_pm_init(struct intel_pinctrl *pctrl)
 }
 
 static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl,
-				   struct intel_community *community)
+				   struct intel_community *community,
+				   unsigned short capability_offset)
 {
+	void __iomem *base = community->regs + capability_offset + 4;
 	static const struct pwm_lpss_boardinfo info = {
 		.clk_rate = 19200000,
 		.npwm = 1,
@@ -1564,7 +1570,7 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl,
 	if (!IS_REACHABLE(CONFIG_PWM_LPSS))
 		return 0;
 
-	chip = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info);
+	chip = devm_pwm_lpss_probe(pctrl->dev, base, &info);
 	return PTR_ERR_OR_ZERO(chip);
 }
 
@@ -1595,7 +1601,9 @@ int intel_pinctrl_probe(struct platform_device *pdev,
 
 	for (i = 0; i < pctrl->ncommunities; i++) {
 		struct intel_community *community = &pctrl->communities[i];
+		unsigned short capability_offset[6];
 		void __iomem *regs;
+		u32 revision;
 		u32 offset;
 		u32 value;
 
@@ -1610,10 +1618,14 @@ int intel_pinctrl_probe(struct platform_device *pdev,
 		value = readl(regs + REVID);
 		if (value == ~0u)
 			return -ENODEV;
-		if (((value & REVID_MASK) >> REVID_SHIFT) >= 0x94) {
+
+		revision = (value & REVID_MASK) >> REVID_SHIFT;
+		if (revision >= 0x092) {
 			community->features |= PINCTRL_FEATURE_DEBOUNCE;
 			community->features |= PINCTRL_FEATURE_1K_PD;
 		}
+		if (revision >= 0x110)
+			community->features |= PINCTRL_FEATURE_3BIT_PAD_OWN;
 
 		/* Determine community features based on the capabilities */
 		offset = CAPLIST;
@@ -1622,15 +1634,19 @@ int intel_pinctrl_probe(struct platform_device *pdev,
 			switch ((value & CAPLIST_ID_MASK) >> CAPLIST_ID_SHIFT) {
 			case CAPLIST_ID_GPIO_HW_INFO:
 				community->features |= PINCTRL_FEATURE_GPIO_HW_INFO;
+				capability_offset[CAPLIST_ID_GPIO_HW_INFO] = offset;
 				break;
 			case CAPLIST_ID_PWM:
 				community->features |= PINCTRL_FEATURE_PWM;
+				capability_offset[CAPLIST_ID_PWM] = offset;
 				break;
 			case CAPLIST_ID_BLINK:
 				community->features |= PINCTRL_FEATURE_BLINK;
+				capability_offset[CAPLIST_ID_BLINK] = offset;
 				break;
 			case CAPLIST_ID_EXP:
 				community->features |= PINCTRL_FEATURE_EXP;
+				capability_offset[CAPLIST_ID_EXP] = offset;
 				break;
 			default:
 				break;
@@ -1653,7 +1669,7 @@ int intel_pinctrl_probe(struct platform_device *pdev,
 		if (ret)
 			return ret;
 
-		ret = intel_pinctrl_probe_pwm(pctrl, community);
+		ret = intel_pinctrl_probe_pwm(pctrl, community, capability_offset[CAPLIST_ID_PWM]);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/pinctrl/intel/pinctrl-intel.h b/drivers/pinctrl/intel/pinctrl-intel.h
index 2f37109d5860b..b5476b9de0dbf 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.h
+++ b/drivers/pinctrl/intel/pinctrl-intel.h
@@ -150,6 +150,7 @@ struct intel_community {
 #define PINCTRL_FEATURE_PWM		BIT(3)
 #define PINCTRL_FEATURE_BLINK		BIT(4)
 #define PINCTRL_FEATURE_EXP		BIT(5)
+#define PINCTRL_FEATURE_3BIT_PAD_OWN	BIT(6)
 
 #define __INTEL_COMMUNITY(b, s, e, g, n, gs, gn, soc)		\
 	{							\
diff --git a/drivers/pinctrl/meson/Kconfig b/drivers/pinctrl/meson/Kconfig
index ce6aeec2fc791..1af4fc320f42b 100644
--- a/drivers/pinctrl/meson/Kconfig
+++ b/drivers/pinctrl/meson/Kconfig
@@ -8,7 +8,6 @@ menuconfig PINCTRL_MESON
 	select PINCONF
 	select GENERIC_PINCONF
 	select GPIOLIB
-	select OF_GPIO
 	select REGMAP_MMIO
 
 if PINCTRL_MESON
diff --git a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
index 3d5ffd6cb14b6..15d73ea1028cf 100644
--- a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
+++ b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
@@ -686,7 +686,7 @@ static int mpfs_pinctrl_probe(struct platform_device *pdev)
 
 	pctrl->regmap = device_node_to_regmap(pdev->dev.parent->of_node);
 	if (IS_ERR(pctrl->regmap))
-		dev_err_probe(dev, PTR_ERR(pctrl->regmap), "Failed to find syscon regmap\n");
+		return dev_err_probe(dev, PTR_ERR(pctrl->regmap), "Failed to find syscon regmap\n");
 
 	pctrl->sysreg_regmap = syscon_regmap_lookup_by_compatible("microchip,mpfs-sysreg-scb");
 	if (IS_ERR(pctrl->sysreg_regmap))
diff --git a/drivers/pinctrl/nxp/pinctrl-s32.h b/drivers/pinctrl/nxp/pinctrl-s32.h
index add3c77ddfedb..8715befd5f055 100644
--- a/drivers/pinctrl/nxp/pinctrl-s32.h
+++ b/drivers/pinctrl/nxp/pinctrl-s32.h
@@ -16,7 +16,7 @@ struct platform_device;
 /**
  * struct s32_pin_group - describes an S32 pin group
  * @data: generic data describes group name, number of pins, and a pin array in
-	this group.
+ *	this group.
  * @pin_sss: an array of source signal select configs paired with pin array.
  */
 struct s32_pin_group {
diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c
index 2b030bd0e6adc..88eacd1aa2dee 100644
--- a/drivers/pinctrl/pinconf-generic.c
+++ b/drivers/pinctrl/pinconf-generic.c
@@ -16,6 +16,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/property.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 
@@ -205,24 +206,27 @@ static const struct pinconf_generic_params dt_params[] = {
 };
 
 /**
- * parse_dt_cfg() - Parse DT pinconf parameters
- * @np:	DT node
+ * parse_fw_cfg() - Parse firmware pinconf parameters
+ * @fwnode:	firmware node
  * @params:	Array of describing generic parameters
  * @count:	Number of entries in @params
  * @cfg:	Array of parsed config options
  * @ncfg:	Number of entries in @cfg
  *
- * Parse the config options described in @params from @np and puts the result
+ * Parse the config options described in @params from @fwnode and puts the result
  * in @cfg. @cfg does not need to be empty, entries are added beginning at
  * @ncfg. @ncfg is updated to reflect the number of entries after parsing. @cfg
  * needs to have enough memory allocated to hold all possible entries.
  */
-static int parse_dt_cfg(struct device_node *np,
+static int parse_fw_cfg(struct fwnode_handle *fwnode,
 			const struct pinconf_generic_params *params,
 			unsigned int count, unsigned long *cfg,
 			unsigned int *ncfg)
 {
-	int i;
+	unsigned long *properties;
+	int i, test;
+
+	properties = bitmap_zalloc(count, GFP_KERNEL);
 
 	for (i = 0; i < count; i++) {
 		u32 val;
@@ -230,7 +234,7 @@ static int parse_dt_cfg(struct device_node *np,
 		const struct pinconf_generic_params *par = &params[i];
 
 		if (par->values && par->num_values) {
-			ret = fwnode_property_match_property_string(of_fwnode_handle(np),
+			ret = fwnode_property_match_property_string(fwnode,
 								    par->property,
 								    par->values, par->num_values);
 			if (ret == -ENOENT)
@@ -240,7 +244,7 @@ static int parse_dt_cfg(struct device_node *np,
 				ret = 0;
 			}
 		} else {
-			ret = of_property_read_u32(np, par->property, &val);
+			ret = fwnode_property_read_u32(fwnode, par->property, &val);
 		}
 
 		/* property not found */
@@ -251,11 +255,45 @@ static int parse_dt_cfg(struct device_node *np,
 		if (ret)
 			val = par->default_value;
 
+		/* if param is greater than count, these are custom properties */
+		if (par->param <= count) {
+			ret = test_and_set_bit(par->param, properties);
+			if (ret) {
+				pr_err("%pfw: conflicting setting detected for %s\n",
+				       fwnode, par->property);
+				bitmap_free(properties);
+				return -EINVAL;
+			}
+		}
+
 		pr_debug("found %s with value %u\n", par->property, val);
 		cfg[*ncfg] = pinconf_to_config_packed(par->param, val);
 		(*ncfg)++;
 	}
 
+	if (test_bit(PIN_CONFIG_DRIVE_STRENGTH, properties) &&
+			test_bit(PIN_CONFIG_DRIVE_STRENGTH_UA, properties))
+		pr_err("%pfw: cannot have multiple drive strength properties\n",
+		       fwnode);
+
+	test = test_bit(PIN_CONFIG_BIAS_BUS_HOLD, properties) +
+		test_bit(PIN_CONFIG_BIAS_DISABLE, properties) +
+		test_bit(PIN_CONFIG_BIAS_HIGH_IMPEDANCE, properties) +
+		test_bit(PIN_CONFIG_BIAS_PULL_UP, properties) +
+		test_bit(PIN_CONFIG_BIAS_PULL_PIN_DEFAULT, properties) +
+		test_bit(PIN_CONFIG_BIAS_PULL_DOWN, properties);
+	if (test > 1)
+		pr_err("%pfw: cannot have multiple bias configurations\n",
+		       fwnode);
+
+	test = test_bit(PIN_CONFIG_DRIVE_OPEN_DRAIN, properties) +
+		test_bit(PIN_CONFIG_DRIVE_OPEN_SOURCE, properties) +
+		test_bit(PIN_CONFIG_DRIVE_PUSH_PULL, properties);
+	if (test > 1)
+		pr_err("%pfw: cannot have multiple drive configurations\n",
+		       fwnode);
+
+	bitmap_free(properties);
 	return 0;
 }
 
@@ -274,17 +312,22 @@ int pinconf_generic_parse_dt_pinmux(struct device_node *np, struct device *dev,
 				    unsigned int **pid, unsigned int **pmux,
 				    unsigned int *npins)
 {
+	struct fwnode_handle *fwnode = of_fwnode_handle(np);
 	unsigned int *pid_t;
 	unsigned int *pmux_t;
-	struct property *prop;
 	unsigned int npins_t, i;
-	u32 value;
 	int ret;
 
-	prop = of_find_property(np, "pinmux", NULL);
-	if (!prop) {
+	ret = fwnode_property_count_u32(fwnode, "pinmux");
+	if (ret < 0) {
 		dev_info(dev, "Missing pinmux property\n");
-		return -ENOENT;
+		return ret;
+	}
+
+	npins_t = ret;
+	if (npins_t == 0) {
+		dev_info(dev, "pinmux property doesn't have entries\n");
+		return -ENODATA;
 	}
 
 	if (!pid || !pmux || !npins) {
@@ -292,21 +335,22 @@ int pinconf_generic_parse_dt_pinmux(struct device_node *np, struct device *dev,
 		return -EINVAL;
 	}
 
-	npins_t = prop->length / sizeof(u32);
 	pid_t = devm_kcalloc(dev, npins_t, sizeof(*pid_t), GFP_KERNEL);
 	pmux_t = devm_kcalloc(dev, npins_t, sizeof(*pmux_t), GFP_KERNEL);
 	if (!pid_t || !pmux_t) {
 		dev_err(dev, "kalloc memory fail\n");
 		return -ENOMEM;
 	}
+
+	ret = fwnode_property_read_u32_array(fwnode, "pinmux", pmux_t, npins_t);
+	if (ret) {
+		dev_err(dev, "get pinmux value fail\n");
+		goto exit;
+	}
+
 	for (i = 0; i < npins_t; i++) {
-		ret = of_property_read_u32_index(np, "pinmux", i, &value);
-		if (ret) {
-			dev_err(dev, "get pinmux value fail\n");
-			goto exit;
-		}
-		pmux_t[i] = value & 0xff;
-		pid_t[i] = (value >> 8) & 0xffffff;
+		pid_t[i] = pmux_t[i] >> 8;
+		pmux_t[i] = pmux_t[i] & 0xff;
 	}
 	*pid = pid_t;
 	*pmux = pmux_t;
@@ -336,9 +380,11 @@ int pinconf_generic_parse_dt_config(struct device_node *np,
 {
 	unsigned long *cfg;
 	unsigned int max_cfg, ncfg = 0;
+	struct fwnode_handle *fwnode;
 	int ret;
 
-	if (!np)
+	fwnode = of_fwnode_handle(np);
+	if (!fwnode)
 		return -EINVAL;
 
 	/* allocate a temporary array big enough to hold one of each option */
@@ -349,12 +395,12 @@ int pinconf_generic_parse_dt_config(struct device_node *np,
 	if (!cfg)
 		return -ENOMEM;
 
-	ret = parse_dt_cfg(np, dt_params, ARRAY_SIZE(dt_params), cfg, &ncfg);
+	ret = parse_fw_cfg(fwnode, dt_params, ARRAY_SIZE(dt_params), cfg, &ncfg);
 	if (ret)
 		goto out;
 	if (pctldev && pctldev->desc->num_custom_params &&
 		pctldev->desc->custom_params) {
-		ret = parse_dt_cfg(np, pctldev->desc->custom_params,
+		ret = parse_fw_cfg(fwnode, pctldev->desc->custom_params,
 				   pctldev->desc->num_custom_params, cfg, &ncfg);
 		if (ret)
 			goto out;
diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c
index 5c055d344ac9d..ace0be5ec6791 100644
--- a/drivers/pinctrl/pinctrl-cy8c95x0.c
+++ b/drivers/pinctrl/pinctrl-cy8c95x0.c
@@ -72,24 +72,6 @@
 #define CY8C95X0_MUX_REGMAP_TO_OFFSET(x, p) \
 	(CY8C95X0_VIRTUAL + (x) - CY8C95X0_PORTSEL + (p) * MUXED_STRIDE)
 
-static const struct i2c_device_id cy8c95x0_id[] = {
-	{ "cy8c9520", 20, },
-	{ "cy8c9540", 40, },
-	{ "cy8c9560", 60, },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, cy8c95x0_id);
-
-#define OF_CY8C95X(__nrgpio) ((void *)(__nrgpio))
-
-static const struct of_device_id cy8c95x0_dt_ids[] = {
-	{ .compatible = "cypress,cy8c9520", .data = OF_CY8C95X(20), },
-	{ .compatible = "cypress,cy8c9540", .data = OF_CY8C95X(40), },
-	{ .compatible = "cypress,cy8c9560", .data = OF_CY8C95X(60), },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, cy8c95x0_dt_ids);
-
 static const struct acpi_gpio_params cy8c95x0_irq_gpios = { 0, 0, true };
 
 static const struct acpi_gpio_mapping cy8c95x0_acpi_irq_gpios[] = {
@@ -144,11 +126,9 @@ static const struct dmi_system_id cy8c95x0_dmi_acpi_irq_info[] = {
  * @map:            Mask used to compensate for Gport2 width
  * @nport:          Number of Gports in this chip
  * @gpio_chip:      gpiolib chip
- * @driver_data:    private driver data
  * @dev:            struct device
  * @pctldev:        pin controller device
  * @pinctrl_desc:   pin controller description
- * @name:           Chip controller name
  * @tpin:           Total number of pins
  * @gpio_reset:     GPIO line handler that can reset the IC
  */
@@ -165,11 +145,9 @@ struct cy8c95x0_pinctrl {
 	DECLARE_BITMAP(map, MAX_LINE);
 	unsigned int nport;
 	struct gpio_chip gpio_chip;
-	unsigned long driver_data;
 	struct device *dev;
 	struct pinctrl_dev *pctldev;
 	struct pinctrl_desc pinctrl_desc;
-	char name[32];
 	unsigned int tpin;
 	struct gpio_desc *gpio_reset;
 };
@@ -1310,18 +1288,19 @@ static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq)
 {
 	struct gpio_irq_chip *girq = &chip->gpio_chip.irq;
 	DECLARE_BITMAP(pending_irqs, MAX_LINE);
+	struct device *dev = chip->dev;
 	int ret;
 
-	mutex_init(&chip->irq_lock);
+	ret = devm_mutex_init(chip->dev, &chip->irq_lock);
+	if (ret)
+		return ret;
 
 	bitmap_zero(pending_irqs, MAX_LINE);
 
 	/* Read IRQ status register to clear all pending interrupts */
 	ret = cy8c95x0_irq_pending(chip, pending_irqs);
-	if (ret) {
-		dev_err(chip->dev, "failed to clear irq status register\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, -EBUSY, "failed to clear irq status register\n");
 
 	/* Mask all interrupts */
 	bitmap_fill(chip->irq_mask, MAX_LINE);
@@ -1336,17 +1315,9 @@ static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq)
 	girq->handler = handle_simple_irq;
 	girq->threaded = true;
 
-	ret = devm_request_threaded_irq(chip->dev, irq,
-					NULL, cy8c95x0_irq_handler,
-					IRQF_ONESHOT | IRQF_SHARED,
-					dev_name(chip->dev), chip);
-	if (ret) {
-		dev_err(chip->dev, "failed to request irq %d\n", irq);
-		return ret;
-	}
-	dev_info(chip->dev, "Registered threaded IRQ\n");
-
-	return 0;
+	return devm_request_threaded_irq(dev, irq, NULL, cy8c95x0_irq_handler,
+					 IRQF_ONESHOT | IRQF_SHARED,
+					 dev_name(chip->dev), chip);
 }
 
 static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip)
@@ -1362,11 +1333,7 @@ static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip)
 	pd->owner = THIS_MODULE;
 
 	chip->pctldev = devm_pinctrl_register(chip->dev, pd, chip);
-	if (IS_ERR(chip->pctldev))
-		return dev_err_probe(chip->dev, PTR_ERR(chip->pctldev),
-			"can't register controller\n");
-
-	return 0;
+	return PTR_ERR_OR_ZERO(chip->pctldev);
 }
 
 static int cy8c95x0_detect(struct i2c_client *client,
@@ -1384,13 +1351,13 @@ static int cy8c95x0_detect(struct i2c_client *client,
 		return ret;
 	switch (ret & GENMASK(7, 4)) {
 	case 0x20:
-		name = cy8c95x0_id[0].name;
+		name = "cy8c9520";
 		break;
 	case 0x40:
-		name = cy8c95x0_id[1].name;
+		name = "cy8c9540";
 		break;
 	case 0x60:
-		name = cy8c95x0_id[2].name;
+		name = "cy8c9560";
 		break;
 	default:
 		return -ENODEV;
@@ -1408,6 +1375,7 @@ static int cy8c95x0_probe(struct i2c_client *client)
 	struct cy8c95x0_pinctrl *chip;
 	struct regmap_config regmap_conf;
 	struct regmap_range_cfg regmap_range_conf;
+	unsigned long driver_data;
 	int ret;
 
 	chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
@@ -1417,26 +1385,21 @@ static int cy8c95x0_probe(struct i2c_client *client)
 	chip->dev = dev;
 
 	/* Set the device type */
-	chip->driver_data = (uintptr_t)i2c_get_match_data(client);
-	if (!chip->driver_data)
-		return -ENODEV;
+	driver_data = (unsigned long)i2c_get_match_data(client);
 
-	chip->tpin = chip->driver_data & CY8C95X0_GPIO_MASK;
+	chip->tpin = driver_data & CY8C95X0_GPIO_MASK;
 	chip->nport = DIV_ROUND_UP(CY8C95X0_PIN_TO_OFFSET(chip->tpin), BANK_SZ);
 
 	memcpy(&regmap_range_conf, &cy8c95x0_ranges[0], sizeof(regmap_range_conf));
 
 	switch (chip->tpin) {
 	case 20:
-		strscpy(chip->name, cy8c95x0_id[0].name);
 		regmap_range_conf.range_max = CY8C95X0_VIRTUAL + 3 * MUXED_STRIDE - 1;
 		break;
 	case 40:
-		strscpy(chip->name, cy8c95x0_id[1].name);
 		regmap_range_conf.range_max = CY8C95X0_VIRTUAL + 6 * MUXED_STRIDE - 1;
 		break;
 	case 60:
-		strscpy(chip->name, cy8c95x0_id[2].name);
 		regmap_range_conf.range_max = CY8C95X0_VIRTUAL + 8 * MUXED_STRIDE - 1;
 		break;
 	default:
@@ -1474,7 +1437,9 @@ static int cy8c95x0_probe(struct i2c_client *client)
 	bitmap_fill(chip->map, MAX_LINE);
 	bitmap_clear(chip->map, 20, 4);
 
-	mutex_init(&chip->i2c_lock);
+	ret = devm_mutex_init(dev, &chip->i2c_lock);
+	if (ret)
+		return ret;
 
 	if (dmi_first_match(cy8c95x0_dmi_acpi_irq_info)) {
 		ret = cy8c95x0_acpi_get_irq(&client->dev);
@@ -1495,8 +1460,26 @@ static int cy8c95x0_probe(struct i2c_client *client)
 	return cy8c95x0_setup_gpiochip(chip);
 }
 
+static const struct i2c_device_id cy8c95x0_id[] = {
+	{ "cy8c9520", 20 },
+	{ "cy8c9540", 40 },
+	{ "cy8c9560", 60 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, cy8c95x0_id);
+
+#define OF_CY8C95X(__nrgpio) ((void *)(__nrgpio))
+
+static const struct of_device_id cy8c95x0_dt_ids[] = {
+	{ .compatible = "cypress,cy8c9520", .data = OF_CY8C95X(20) },
+	{ .compatible = "cypress,cy8c9540", .data = OF_CY8C95X(40) },
+	{ .compatible = "cypress,cy8c9560", .data = OF_CY8C95X(60) },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cy8c95x0_dt_ids);
+
 static const struct acpi_device_id cy8c95x0_acpi_ids[] = {
-	{ "INT3490", 40, },
+	{ "INT3490", 40 },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, cy8c95x0_acpi_ids);
diff --git a/drivers/pinctrl/pinctrl-eyeq5.c b/drivers/pinctrl/pinctrl-eyeq5.c
index 5f6af934a5169..dcdf80f07a907 100644
--- a/drivers/pinctrl/pinctrl-eyeq5.c
+++ b/drivers/pinctrl/pinctrl-eyeq5.c
@@ -26,6 +26,7 @@
 #include <linux/errno.h>
 #include <linux/io.h>
 #include <linux/mod_devicetable.h>
+#include <linux/of.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/types.h>
@@ -38,18 +39,6 @@
 #include "core.h"
 #include "pinctrl-utils.h"
 
-struct eq5p_pinctrl {
-	struct pinctrl_desc	desc;
-	void __iomem		*base;
-};
-
-enum eq5p_bank {
-	EQ5P_BANK_A,
-	EQ5P_BANK_B,
-
-	EQ5P_BANK_COUNT,
-};
-
 enum eq5p_regs {
 	EQ5P_PD,
 	EQ5P_PU,
@@ -60,9 +49,24 @@ enum eq5p_regs {
 	EQ5P_REG_COUNT,
 };
 
-static const unsigned int eq5p_regs[EQ5P_BANK_COUNT][EQ5P_REG_COUNT] = {
-	[EQ5P_BANK_A] = {0x0C0, 0x0C4, 0x0D0, 0x0D4, 0x0B0},
-	[EQ5P_BANK_B] = {0x0C8, 0x0CC, 0x0D8, 0x0DC, 0x0B4},
+struct eq5p_bank {
+	const unsigned int npins;
+	const unsigned int regs[EQ5P_REG_COUNT];
+};
+
+struct eq5p_match_data {
+	const unsigned int npins;
+	const unsigned int nfunctions;
+	const unsigned int nbanks;
+	const struct pinctrl_pin_desc *pins;
+	const struct pinfunction *functions;
+	const struct eq5p_bank *banks;
+};
+
+struct eq5p_pinctrl {
+	struct pinctrl_desc		desc;
+	void __iomem			*base;
+	const struct eq5p_match_data	*data;
 };
 
 /*
@@ -70,10 +74,18 @@ static const unsigned int eq5p_regs[EQ5P_BANK_COUNT][EQ5P_REG_COUNT] = {
  */
 #define EQ5P_DS_MASK	GENMASK(1, 0)
 
+/*
+ * The GPIO function is always the first function
+ */
+#define EQ5P_GPIO_FUNC_SELECTOR 0
+
+/* Helper to declare pinfunction */
+#define EQ5P_PINFUNCTION(func, groups) PINCTRL_PINFUNCTION(func, groups, ARRAY_SIZE(groups))
+
 /*
  * Comments to the right of each pin are the "signal name" in the datasheet.
  */
-static const struct pinctrl_pin_desc eq5p_pins[] = {
+static const struct pinctrl_pin_desc eq5p_eyeq5_pins[] = {
 	/* Bank A */
 	PINCTRL_PIN(0,  "PA0"),  /* A0_TIMER0_CK */
 	PINCTRL_PIN(1,  "PA1"),  /* A1_TIMER0_EOC */
@@ -105,35 +117,35 @@ static const struct pinctrl_pin_desc eq5p_pins[] = {
 	PINCTRL_PIN(27, "PA27"), /* A27_SPI_1_CS1 */
 	PINCTRL_PIN(28, "PA28"), /* A28_REF_CLK0 */
 
-#define EQ5P_PIN_OFFSET_BANK_B	29
+#define EQ5P_EYEQ5_PIN_OFFSET_BANK_B	29
 
 	/* Bank B */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 0,  "PB0"),  /* B0_TIMER3_CK */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 1,  "PB1"),  /* B1_TIMER3_EOC */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 2,  "PB2"),  /* B2_TIMER4_CK */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 3,  "PB3"),  /* B3_TIMER4_EOC */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 4,  "PB4"),  /* B4_TIMER6_EXT_INCAP1 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 5,  "PB5"),  /* B5_TIMER6_EXT_INCAP2 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 6,  "PB6"),  /* B6_TIMER6_EXT_OUTCMP1 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 7,  "PB7"),  /* B7_TIMER6_EXT_OUTCMP2 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 8,  "PB8"),  /* B8_UART_2_TX */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 9,  "PB9"),  /* B9_UART_2_RX */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 10, "PB10"), /* B10_CAN_2_TX */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 11, "PB11"), /* B11_CAN_2_RX */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 12, "PB12"), /* B12_SPI_2_DO */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 13, "PB13"), /* B13_SPI_2_DI */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 14, "PB14"), /* B14_SPI_2_CK */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 15, "PB15"), /* B15_SPI_2_CS0 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 16, "PB16"), /* B16_SPI_2_CS1 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 17, "PB17"), /* B17_SPI_3_DO */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 18, "PB18"), /* B18_SPI_3_DI */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 19, "PB19"), /* B19_SPI_3_CK */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 20, "PB20"), /* B20_SPI_3_CS0 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 21, "PB21"), /* B21_SPI_3_CS1 */
-	PINCTRL_PIN(EQ5P_PIN_OFFSET_BANK_B + 22, "PB22"), /* B22_MCLK0 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 0,  "PB0"),  /* B0_TIMER3_CK */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 1,  "PB1"),  /* B1_TIMER3_EOC */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 2,  "PB2"),  /* B2_TIMER4_CK */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 3,  "PB3"),  /* B3_TIMER4_EOC */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 4,  "PB4"),  /* B4_TIMER6_EXT_INCAP1 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 5,  "PB5"),  /* B5_TIMER6_EXT_INCAP2 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 6,  "PB6"),  /* B6_TIMER6_EXT_OUTCMP1 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 7,  "PB7"),  /* B7_TIMER6_EXT_OUTCMP2 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 8,  "PB8"),  /* B8_UART_2_TX */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 9,  "PB9"),  /* B9_UART_2_RX */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 10, "PB10"), /* B10_CAN_2_TX */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 11, "PB11"), /* B11_CAN_2_RX */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 12, "PB12"), /* B12_SPI_2_DO */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 13, "PB13"), /* B13_SPI_2_DI */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 14, "PB14"), /* B14_SPI_2_CK */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 15, "PB15"), /* B15_SPI_2_CS0 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 16, "PB16"), /* B16_SPI_2_CS1 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 17, "PB17"), /* B17_SPI_3_DO */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 18, "PB18"), /* B18_SPI_3_DI */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 19, "PB19"), /* B19_SPI_3_CK */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 20, "PB20"), /* B20_SPI_3_CS0 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 21, "PB21"), /* B21_SPI_3_CS1 */
+	PINCTRL_PIN(EQ5P_EYEQ5_PIN_OFFSET_BANK_B + 22, "PB22"), /* B22_MCLK0 */
 };
 
-static const char * const gpio_groups[] = {
+static const char * const eq5p_eyeq5_gpio_groups[] = {
 	/* Bank A */
 	"PA0",  "PA1",  "PA2",  "PA3",  "PA4",  "PA5",  "PA6",  "PA7",
 	"PA8",  "PA9",  "PA10", "PA11", "PA12", "PA13", "PA14", "PA15",
@@ -147,70 +159,184 @@ static const char * const gpio_groups[] = {
 };
 
 /* Groups of functions on bank A */
-static const char * const timer0_groups[] = { "PA0", "PA1" };
-static const char * const timer1_groups[] = { "PA2", "PA3" };
-static const char * const timer2_groups[] = { "PA4", "PA5" };
-static const char * const timer5_groups[] = { "PA6", "PA7", "PA8", "PA9" };
-static const char * const uart0_groups[] = { "PA10", "PA11" };
-static const char * const uart1_groups[] = { "PA12", "PA13" };
-static const char * const can0_groups[] = { "PA14", "PA15" };
-static const char * const can1_groups[] = { "PA16", "PA17" };
-static const char * const spi0_groups[] = { "PA18", "PA19", "PA20", "PA21", "PA22" };
-static const char * const spi1_groups[] = { "PA23", "PA24", "PA25", "PA26", "PA27" };
-static const char * const refclk0_groups[] = { "PA28" };
+static const char * const eq5p_eyeq5_timer0_groups[] = { "PA0", "PA1" };
+static const char * const eq5p_eyeq5_timer1_groups[] = { "PA2", "PA3" };
+static const char * const eq5p_eyeq5_timer2_groups[] = { "PA4", "PA5" };
+static const char * const eq5p_eyeq5_timer5_groups[] = { "PA6", "PA7", "PA8", "PA9" };
+static const char * const eq5p_eyeq5_uart0_groups[] = { "PA10", "PA11" };
+static const char * const eq5p_eyeq5_uart1_groups[] = { "PA12", "PA13" };
+static const char * const eq5p_eyeq5_can0_groups[] = { "PA14", "PA15" };
+static const char * const eq5p_eyeq5_can1_groups[] = { "PA16", "PA17" };
+static const char * const eq5p_eyeq5_spi0_groups[] = { "PA18", "PA19", "PA20", "PA21", "PA22" };
+static const char * const eq5p_eyeq5_spi1_groups[] = { "PA23", "PA24", "PA25", "PA26", "PA27" };
+static const char * const eq5p_eyeq5_refclk0_groups[] = { "PA28" };
 
 /* Groups of functions on bank B */
-static const char * const timer3_groups[] = { "PB0", "PB1" };
-static const char * const timer4_groups[] = { "PB2", "PB3" };
-static const char * const timer6_groups[] = { "PB4", "PB5", "PB6", "PB7" };
-static const char * const uart2_groups[] = { "PB8", "PB9" };
-static const char * const can2_groups[] = { "PB10", "PB11" };
-static const char * const spi2_groups[] = { "PB12", "PB13", "PB14", "PB15", "PB16" };
-static const char * const spi3_groups[] = { "PB17", "PB18", "PB19", "PB20", "PB21" };
-static const char * const mclk0_groups[] = { "PB22" };
-
-static const struct pinfunction eq5p_functions[] = {
-	/* GPIO having a fixed index is depended upon, see GPIO_FUNC_SELECTOR. */
-	PINCTRL_PINFUNCTION("gpio", gpio_groups, ARRAY_SIZE(gpio_groups)),
-#define GPIO_FUNC_SELECTOR 0
+static const char * const eq5p_eyeq5_timer3_groups[] = { "PB0", "PB1" };
+static const char * const eq5p_eyeq5_timer4_groups[] = { "PB2", "PB3" };
+static const char * const eq5p_eyeq5_timer6_groups[] = { "PB4", "PB5", "PB6", "PB7" };
+static const char * const eq5p_eyeq5_uart2_groups[] = { "PB8", "PB9" };
+static const char * const eq5p_eyeq5_can2_groups[] = { "PB10", "PB11" };
+static const char * const eq5p_eyeq5_spi2_groups[] = { "PB12", "PB13", "PB14", "PB15", "PB16" };
+static const char * const eq5p_eyeq5_spi3_groups[] = { "PB17", "PB18", "PB19", "PB20", "PB21" };
+static const char * const eq5p_eyeq5_mclk0_groups[] = { "PB22" };
+
+static const struct pinfunction eq5p_eyeq5_functions[] = {
+	/* GPIO having a fixed index is depended upon, see EQ5P_GPIO_FUNC_SELECTOR. */
+	EQ5P_PINFUNCTION("gpio", eq5p_eyeq5_gpio_groups),
 
 	/* Bank A functions */
-	PINCTRL_PINFUNCTION("timer0", timer0_groups, ARRAY_SIZE(timer0_groups)),
-	PINCTRL_PINFUNCTION("timer1", timer1_groups, ARRAY_SIZE(timer1_groups)),
-	PINCTRL_PINFUNCTION("timer2", timer2_groups, ARRAY_SIZE(timer2_groups)),
-	PINCTRL_PINFUNCTION("timer5", timer5_groups, ARRAY_SIZE(timer5_groups)),
-	PINCTRL_PINFUNCTION("uart0", uart0_groups, ARRAY_SIZE(uart0_groups)),
-	PINCTRL_PINFUNCTION("uart1", uart1_groups, ARRAY_SIZE(uart1_groups)),
-	PINCTRL_PINFUNCTION("can0", can0_groups, ARRAY_SIZE(can0_groups)),
-	PINCTRL_PINFUNCTION("can1", can1_groups, ARRAY_SIZE(can1_groups)),
-	PINCTRL_PINFUNCTION("spi0", spi0_groups, ARRAY_SIZE(spi0_groups)),
-	PINCTRL_PINFUNCTION("spi1", spi1_groups, ARRAY_SIZE(spi1_groups)),
-	PINCTRL_PINFUNCTION("refclk0", refclk0_groups, ARRAY_SIZE(refclk0_groups)),
+	EQ5P_PINFUNCTION("timer0", eq5p_eyeq5_timer0_groups),
+	EQ5P_PINFUNCTION("timer1", eq5p_eyeq5_timer1_groups),
+	EQ5P_PINFUNCTION("timer2", eq5p_eyeq5_timer2_groups),
+	EQ5P_PINFUNCTION("timer5", eq5p_eyeq5_timer5_groups),
+	EQ5P_PINFUNCTION("uart0", eq5p_eyeq5_uart0_groups),
+	EQ5P_PINFUNCTION("uart1", eq5p_eyeq5_uart1_groups),
+	EQ5P_PINFUNCTION("can0", eq5p_eyeq5_can0_groups),
+	EQ5P_PINFUNCTION("can1", eq5p_eyeq5_can1_groups),
+	EQ5P_PINFUNCTION("spi0", eq5p_eyeq5_spi0_groups),
+	EQ5P_PINFUNCTION("spi1", eq5p_eyeq5_spi1_groups),
+	EQ5P_PINFUNCTION("refclk0", eq5p_eyeq5_refclk0_groups),
 
 	/* Bank B functions */
-	PINCTRL_PINFUNCTION("timer3", timer3_groups, ARRAY_SIZE(timer3_groups)),
-	PINCTRL_PINFUNCTION("timer4", timer4_groups, ARRAY_SIZE(timer4_groups)),
-	PINCTRL_PINFUNCTION("timer6", timer6_groups, ARRAY_SIZE(timer6_groups)),
-	PINCTRL_PINFUNCTION("uart2", uart2_groups, ARRAY_SIZE(uart2_groups)),
-	PINCTRL_PINFUNCTION("can2", can2_groups, ARRAY_SIZE(can2_groups)),
-	PINCTRL_PINFUNCTION("spi2", spi2_groups, ARRAY_SIZE(spi2_groups)),
-	PINCTRL_PINFUNCTION("spi3", spi3_groups, ARRAY_SIZE(spi3_groups)),
-	PINCTRL_PINFUNCTION("mclk0", mclk0_groups, ARRAY_SIZE(mclk0_groups)),
+	EQ5P_PINFUNCTION("timer3", eq5p_eyeq5_timer3_groups),
+	EQ5P_PINFUNCTION("timer4", eq5p_eyeq5_timer4_groups),
+	EQ5P_PINFUNCTION("timer6", eq5p_eyeq5_timer6_groups),
+	EQ5P_PINFUNCTION("uart2", eq5p_eyeq5_uart2_groups),
+	EQ5P_PINFUNCTION("can2", eq5p_eyeq5_can2_groups),
+	EQ5P_PINFUNCTION("spi2", eq5p_eyeq5_spi2_groups),
+	EQ5P_PINFUNCTION("spi3", eq5p_eyeq5_spi3_groups),
+	EQ5P_PINFUNCTION("mclk0", eq5p_eyeq5_mclk0_groups),
+};
+
+static const struct eq5p_bank eq5p_eyeq5_banks[] = {
+	{
+		.npins = EQ5P_EYEQ5_PIN_OFFSET_BANK_B,
+		.regs = {0x0C0, 0x0C4, 0x0D0, 0x0D4, 0x0B0},
+	},
+	{
+		.npins = ARRAY_SIZE(eq5p_eyeq5_pins) - EQ5P_EYEQ5_PIN_OFFSET_BANK_B,
+		.regs = {0x0C8, 0x0CC, 0x0D8, 0x0DC, 0x0B4},
+	},
+};
+
+static const struct eq5p_match_data eq5p_eyeq5_data = {
+	.npins = ARRAY_SIZE(eq5p_eyeq5_pins),
+	.nfunctions = ARRAY_SIZE(eq5p_eyeq5_functions),
+	.nbanks = ARRAY_SIZE(eq5p_eyeq5_banks),
+	.pins = eq5p_eyeq5_pins,
+	.functions = eq5p_eyeq5_functions,
+	.banks = eq5p_eyeq5_banks,
+};
+
+static const struct pinctrl_pin_desc eq5p_eyeq6lplus_pins[] = {
+	PINCTRL_PIN(0,  "PA0"),  /* GPIO_A0_TIMER0_CK0 */
+	PINCTRL_PIN(1,  "PA1"),  /* GPIO_A1_TIMER0_EOC */
+	PINCTRL_PIN(2,  "PA2"),  /* GPIO_A2_TIMER1_CK */
+	PINCTRL_PIN(3,  "PA3"),  /* GPIO_A3_TIMER1_EOC1 */
+	PINCTRL_PIN(4,  "PA4"),  /* GPIO_A4_SSI_UART_RX */
+	PINCTRL_PIN(5,  "PA5"),  /* GPIO_A5_SSI_UART_TX */
+	PINCTRL_PIN(6,  "PA6"),  /* GPIO_A6_SPI_0_CS */
+	PINCTRL_PIN(7,  "PA7"),  /* GPIO_A7_SPI_0_DI */
+	PINCTRL_PIN(8,  "PA8"),  /* GPIO_A8_SPI_0_CK */
+	PINCTRL_PIN(9,  "PA9"),  /* GPIO_A9_SPI_0_DO */
+	PINCTRL_PIN(10, "PA10"), /* GPIO_A10_SPI_0_CS1 */
+	PINCTRL_PIN(11, "PA11"), /* GPIO_A11_UART_0_RX */
+	PINCTRL_PIN(12, "PA12"), /* GPIO_A12_UART_0_TX */
+	PINCTRL_PIN(13, "PA13"), /* GPIO_A13_TIMER2_CK */
+	PINCTRL_PIN(14, "PA14"), /* GPIO_A14_TIMER2_EOC */
+	PINCTRL_PIN(15, "PA15"), /* GPIO_A15_TIMER3_CK */
+	PINCTRL_PIN(16, "PA16"), /* GPIO_A16_TIMER_EOC */
+	PINCTRL_PIN(17, "PA17"), /* GPIO_A17_TIMER_EXT0_INCA P1 */
+	PINCTRL_PIN(18, "PA18"), /* GPIO_A18_TIMER_EXT0_INCA P2 */
+	PINCTRL_PIN(19, "PA19"), /* GPIO_A19_TIMER_EXT0_OUT CMP1 */
+	PINCTRL_PIN(20, "PA20"), /* GPIO_A20_TIMER_EXT0_OUT CMP2 */
+	PINCTRL_PIN(21, "PA21"), /* GPIO_A21_SPI_1_CS0 */
+	PINCTRL_PIN(22, "PA22"), /* GPIO_A22_SPI_1_DI */
+	PINCTRL_PIN(23, "PA23"), /* GPIO_A23_SPI_1_CK */
+	PINCTRL_PIN(24, "PA24"), /* GPIO_A24_SPI_1_DO */
+	PINCTRL_PIN(25, "PA25"), /* GPIO_A25_SPI_1_CS1 */
+	PINCTRL_PIN(26, "PA26"), /* GPIO_A26_TIMER_EXT1_INCA P1 */
+	PINCTRL_PIN(27, "PA27"), /* GPIO_A27_TIMER_EXT1_INCA P2 */
+	PINCTRL_PIN(28, "PA28"), /* GPIO_A28_TIMER_EXT1_OUTC MP1 */
+	PINCTRL_PIN(29, "PA29"), /* GPIO_A29_TIMER_EXT1_OUTC MP2 */
+	PINCTRL_PIN(30, "PA30"), /* GPIO_A30_EXT_CLK */
+	PINCTRL_PIN(31, "PA31"), /* GPIO_A31_VDI_MCLK */
+};
+
+static const char * const eq5p_eyeq6lplus_gpio_groups[] = {
+	/* Bank A */
+	"PA0",  "PA1",  "PA2",  "PA3",  "PA4",  "PA5",  "PA6",  "PA7",
+	"PA8",  "PA9",  "PA10", "PA11", "PA12", "PA13", "PA14", "PA15",
+	"PA16", "PA17", "PA18", "PA19", "PA20", "PA21", "PA22", "PA23",
+	"PA24", "PA25", "PA26", "PA27", "PA28", "PA29", "PA30", "PA31",
+};
+
+/* Groups of functions on bank A */
+static const char * const eq5p_eyeq6lplus_timer0_groups[] = { "PA0", "PA1" };
+static const char * const eq5p_eyeq6lplus_timer1_groups[] = { "PA2", "PA3" };
+static const char * const eq5p_eyeq6lplus_uart_ssi_groups[] = { "PA4", "PA5" };
+static const char * const eq5p_eyeq6lplus_spi0_groups[] = { "PA6", "PA7", "PA8", "PA9", "PA10" };
+static const char * const eq5p_eyeq6lplus_uart0_groups[] = { "PA11", "PA12" };
+static const char * const eq5p_eyeq6lplus_timer2_groups[] = { "PA13", "PA14" };
+static const char * const eq5p_eyeq6lplus_timer3_groups[] = { "PA15", "PA16" };
+static const char * const eq5p_eyeq6lplus_timer_ext0_groups[] = { "PA17", "PA18", "PA19", "PA20" };
+static const char * const eq5p_eyeq6lplus_spi1_groups[] = {
+	"PA21", "PA22", "PA23", "PA24", "PA25"
+};
+static const char * const eq5p_eyeq6lplus_timer_ext1_groups[] = { "PA26", "PA27", "PA28", "PA29" };
+static const char * const eq5p_eyeq6lplus_ext_ref_clk_groups[] = { "PA30" };
+static const char * const eq5p_eyeq6lplus_mipi_ref_clk_groups[] = { "PA31" };
+
+static const struct pinfunction eq5p_eyeq6lplus_functions[] = {
+	/* gpios function */
+	EQ5P_PINFUNCTION("gpio", eq5p_eyeq6lplus_gpio_groups),
+
+	/* Bank A alternate functions */
+	EQ5P_PINFUNCTION("timer0", eq5p_eyeq6lplus_timer0_groups),
+	EQ5P_PINFUNCTION("timer1", eq5p_eyeq6lplus_timer1_groups),
+	EQ5P_PINFUNCTION("uart_ssi", eq5p_eyeq6lplus_uart_ssi_groups),
+	EQ5P_PINFUNCTION("spi0", eq5p_eyeq6lplus_spi0_groups),
+	EQ5P_PINFUNCTION("uart0", eq5p_eyeq6lplus_uart0_groups),
+	EQ5P_PINFUNCTION("timer2", eq5p_eyeq6lplus_timer2_groups),
+	EQ5P_PINFUNCTION("timer3", eq5p_eyeq6lplus_timer3_groups),
+	EQ5P_PINFUNCTION("timer_ext0", eq5p_eyeq6lplus_timer_ext0_groups),
+	EQ5P_PINFUNCTION("spi1", eq5p_eyeq6lplus_spi1_groups),
+	EQ5P_PINFUNCTION("timer_ext1", eq5p_eyeq6lplus_timer_ext1_groups),
+	EQ5P_PINFUNCTION("ext_ref_clk", eq5p_eyeq6lplus_ext_ref_clk_groups),
+	EQ5P_PINFUNCTION("mipi_ref_clk", eq5p_eyeq6lplus_mipi_ref_clk_groups),
+};
+
+static const struct eq5p_bank eq5p_eyeq6lplus_banks[] = {
+	{
+		.npins = ARRAY_SIZE(eq5p_eyeq6lplus_pins),
+		.regs = {0x0C0, 0x0C4, 0x0D0, 0x0D4, 0x0B0},
+	},
+};
+
+static const struct eq5p_match_data eq5p_eyeq6lplus_data = {
+	.npins = ARRAY_SIZE(eq5p_eyeq6lplus_pins),
+	.nfunctions = ARRAY_SIZE(eq5p_eyeq6lplus_functions),
+	.nbanks = ARRAY_SIZE(eq5p_eyeq6lplus_banks),
+	.pins = eq5p_eyeq6lplus_pins,
+	.functions = eq5p_eyeq6lplus_functions,
+	.banks = eq5p_eyeq6lplus_banks,
 };
 
 static void eq5p_update_bits(const struct eq5p_pinctrl *pctrl,
-			     enum eq5p_bank bank, enum eq5p_regs reg,
-			     u32 mask, u32 val)
+			     const struct eq5p_bank *bank,
+			     enum eq5p_regs reg, u32 mask, u32 val)
 {
-	void __iomem *ptr = pctrl->base + eq5p_regs[bank][reg];
+	void __iomem *ptr = pctrl->base + bank->regs[reg];
 
 	writel((readl(ptr) & ~mask) | (val & mask), ptr);
 }
 
 static bool eq5p_test_bit(const struct eq5p_pinctrl *pctrl,
-			  enum eq5p_bank bank, enum eq5p_regs reg, int offset)
+			  const struct eq5p_bank *bank,
+			  enum eq5p_regs reg, int offset)
 {
-	u32 val = readl(pctrl->base + eq5p_regs[bank][reg]);
+	u32 val = readl(pctrl->base + bank->regs[reg]);
 
 	if (WARN_ON(offset > 31))
 		return false;
@@ -218,25 +344,29 @@ static bool eq5p_test_bit(const struct eq5p_pinctrl *pctrl,
 	return (val & BIT(offset)) != 0;
 }
 
-static enum eq5p_bank eq5p_pin_to_bank(unsigned int pin)
+static int eq5p_pin_to_bank_offset(const struct eq5p_pinctrl *pctrl, unsigned int pin,
+				   const struct eq5p_bank **bank, unsigned int *offset)
 {
-	if (pin < EQ5P_PIN_OFFSET_BANK_B)
-		return EQ5P_BANK_A;
-	else
-		return EQ5P_BANK_B;
-}
+	for (unsigned int i = 0; i < pctrl->data->nbanks; i++) {
+		const struct eq5p_bank *_bank = &pctrl->data->banks[i];
+		unsigned int npins = _bank->npins;
+
+		if (pin < npins) {
+			*bank = _bank;
+			*offset = pin;
+			return 0;
+		}
+		pin -= npins;
+	}
 
-static unsigned int eq5p_pin_to_offset(unsigned int pin)
-{
-	if (pin < EQ5P_PIN_OFFSET_BANK_B)
-		return pin;
-	else
-		return pin - EQ5P_PIN_OFFSET_BANK_B;
+	return -EINVAL;
 }
 
 static int eq5p_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
 {
-	return ARRAY_SIZE(eq5p_pins);
+	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->data->npins;
 }
 
 static const char *eq5p_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
@@ -260,10 +390,15 @@ static int eq5p_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 {
 	enum pin_config_param param = pinconf_to_config_param(*config);
 	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
-	unsigned int offset = eq5p_pin_to_offset(pin);
-	enum eq5p_bank bank = eq5p_pin_to_bank(pin);
+	const struct eq5p_bank *bank;
+	unsigned int offset;
 	u32 val_ds, arg;
 	bool pd, pu;
+	int ret;
+
+	ret = eq5p_pin_to_bank_offset(pctrl, pin, &bank, &offset);
+	if (ret)
+		return ret;
 
 	pd = eq5p_test_bit(pctrl, bank, EQ5P_PD, offset);
 	pu = eq5p_test_bit(pctrl, bank, EQ5P_PU, offset);
@@ -281,10 +416,10 @@ static int eq5p_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 	case PIN_CONFIG_DRIVE_STRENGTH:
 		offset *= 2; /* two bits per pin */
 		if (offset >= 32) {
-			val_ds = readl(pctrl->base + eq5p_regs[bank][EQ5P_DS_HIGH]);
+			val_ds = readl(pctrl->base + bank->regs[EQ5P_DS_HIGH]);
 			offset -= 32;
 		} else {
-			val_ds = readl(pctrl->base + eq5p_regs[bank][EQ5P_DS_LOW]);
+			val_ds = readl(pctrl->base + bank->regs[EQ5P_DS_LOW]);
 		}
 		arg = (val_ds >> offset) & EQ5P_DS_MASK;
 		break;
@@ -302,30 +437,35 @@ static void eq5p_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev,
 {
 	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 	const char *pin_name = pctrl->desc.pins[pin].name;
-	unsigned int offset = eq5p_pin_to_offset(pin);
-	enum eq5p_bank bank = eq5p_pin_to_bank(pin);
+	const struct eq5p_bank *bank;
 	const char *func_name, *bias;
 	unsigned long ds_config;
+	unsigned int offset;
 	u32 drive_strength;
 	bool pd, pu;
 	int i, j;
 
+	if (eq5p_pin_to_bank_offset(pctrl, pin, &bank, &offset)) {
+		seq_puts(s, "unknown pin");
+		return;
+	}
+
 	/*
 	 * First, let's get the function name. All pins have only two functions:
 	 * GPIO (IOCR == 0) and something else (IOCR == 1).
 	 */
 	if (eq5p_test_bit(pctrl, bank, EQ5P_IOCR, offset)) {
 		func_name = NULL;
-		for (i = 0; i < ARRAY_SIZE(eq5p_functions); i++) {
-			if (i == GPIO_FUNC_SELECTOR)
+		for (i = 0; i < pctrl->data->nfunctions; i++) {
+			if (i == EQ5P_GPIO_FUNC_SELECTOR)
 				continue;
 
-			for (j = 0; j < eq5p_functions[i].ngroups; j++) {
+			for (j = 0; j < pctrl->data->functions[i].ngroups; j++) {
 				/* Groups and pins are the same thing for us. */
-				const char *x = eq5p_functions[i].groups[j];
+				const char *x = pctrl->data->functions[i].groups[j];
 
 				if (strcmp(x, pin_name) == 0) {
-					func_name = eq5p_functions[i].name;
+					func_name = pctrl->data->functions[i].name;
 					break;
 				}
 			}
@@ -341,7 +481,7 @@ static void eq5p_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev,
 		if (!func_name)
 			func_name = "unknown";
 	} else {
-		func_name = eq5p_functions[GPIO_FUNC_SELECTOR].name;
+		func_name = pctrl->data->functions[EQ5P_GPIO_FUNC_SELECTOR].name;
 	}
 
 	/* Second, we retrieve the bias. */
@@ -376,13 +516,17 @@ static const struct pinctrl_ops eq5p_pinctrl_ops = {
 
 static int eq5p_pinmux_get_functions_count(struct pinctrl_dev *pctldev)
 {
-	return ARRAY_SIZE(eq5p_functions);
+	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->data->nfunctions;
 }
 
 static const char *eq5p_pinmux_get_function_name(struct pinctrl_dev *pctldev,
 						 unsigned int selector)
 {
-	return eq5p_functions[selector].name;
+	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->data->functions[selector].name;
 }
 
 static int eq5p_pinmux_get_function_groups(struct pinctrl_dev *pctldev,
@@ -390,8 +534,10 @@ static int eq5p_pinmux_get_function_groups(struct pinctrl_dev *pctldev,
 					   const char * const **groups,
 					   unsigned int *num_groups)
 {
-	*groups = eq5p_functions[selector].groups;
-	*num_groups = eq5p_functions[selector].ngroups;
+	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	*groups = pctrl->data->functions[selector].groups;
+	*num_groups = pctrl->data->functions[selector].ngroups;
 	return 0;
 }
 
@@ -399,12 +545,17 @@ static int eq5p_pinmux_set_mux(struct pinctrl_dev *pctldev,
 			       unsigned int func_selector, unsigned int pin)
 {
 	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
-	const char *func_name = eq5p_functions[func_selector].name;
+	const char *func_name = pctrl->data->functions[func_selector].name;
 	const char *group_name = pctldev->desc->pins[pin].name;
-	bool is_gpio = func_selector == GPIO_FUNC_SELECTOR;
-	unsigned int offset = eq5p_pin_to_offset(pin);
-	enum eq5p_bank bank = eq5p_pin_to_bank(pin);
+	bool is_gpio = func_selector == EQ5P_GPIO_FUNC_SELECTOR;
+	const struct eq5p_bank *bank;
+	unsigned int offset;
 	u32 mask, val;
+	int ret;
+
+	ret = eq5p_pin_to_bank_offset(pctrl, pin, &bank, &offset);
+	if (ret)
+		return ret;
 
 	dev_dbg(pctldev->dev, "func=%s group=%s\n", func_name, group_name);
 
@@ -419,7 +570,7 @@ static int eq5p_pinmux_gpio_request_enable(struct pinctrl_dev *pctldev,
 					   unsigned int pin)
 {
 	/* Pin numbers and group selectors are the same thing in our case. */
-	return eq5p_pinmux_set_mux(pctldev, GPIO_FUNC_SELECTOR, pin);
+	return eq5p_pinmux_set_mux(pctldev, EQ5P_GPIO_FUNC_SELECTOR, pin);
 }
 
 static const struct pinmux_ops eq5p_pinmux_ops = {
@@ -435,10 +586,15 @@ static int eq5p_pinconf_set_drive_strength(struct pinctrl_dev *pctldev,
 					   unsigned int pin, u32 arg)
 {
 	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
-	unsigned int offset = eq5p_pin_to_offset(pin);
-	enum eq5p_bank bank = eq5p_pin_to_bank(pin);
+	const struct eq5p_bank *bank;
+	unsigned int offset;
 	unsigned int reg;
 	u32 mask, val;
+	int ret;
+
+	ret = eq5p_pin_to_bank_offset(pctrl, pin, &bank, &offset);
+	if (ret)
+		return ret;
 
 	if (arg & ~EQ5P_DS_MASK) {
 		dev_err(pctldev->dev, "Unsupported drive strength: %u\n", arg);
@@ -465,12 +621,18 @@ static int eq5p_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 {
 	struct eq5p_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 	const char *pin_name = pctldev->desc->pins[pin].name;
-	unsigned int offset = eq5p_pin_to_offset(pin);
-	enum eq5p_bank bank = eq5p_pin_to_bank(pin);
 	struct device *dev = pctldev->dev;
-	u32 val = BIT(offset);
+	const struct eq5p_bank *bank;
+	unsigned int offset;
 	unsigned int i;
+	u32 val;
+	int ret;
+
+	ret = eq5p_pin_to_bank_offset(pctrl, pin, &bank, &offset);
+	if (ret)
+		return ret;
 
+	val = BIT(offset);
 	for (i = 0; i < num_configs; i++) {
 		enum pin_config_param param = pinconf_to_config_param(configs[i]);
 		u32 arg = pinconf_to_config_argument(configs[i]);
@@ -533,19 +695,26 @@ static const struct pinconf_ops eq5p_pinconf_ops = {
 static int eq5p_probe(struct auxiliary_device *adev,
 		      const struct auxiliary_device_id *id)
 {
+	const struct of_device_id *match;
 	struct device *dev = &adev->dev;
 	struct pinctrl_dev *pctldev;
 	struct eq5p_pinctrl *pctrl;
 	int ret;
 
+	/* Get match data based on parent OF node set in clk-eyeq */
+	match = of_match_node(dev->driver->of_match_table, dev->of_node);
+	if (!match || !match->data)
+		return -ENODEV;
+
 	pctrl = devm_kzalloc(dev, sizeof(*pctrl), GFP_KERNEL);
 	if (!pctrl)
 		return -ENOMEM;
 
 	pctrl->base = (void __iomem *)dev_get_platdata(dev);
+	pctrl->data = match->data;
 	pctrl->desc.name = dev_name(dev);
-	pctrl->desc.pins = eq5p_pins;
-	pctrl->desc.npins = ARRAY_SIZE(eq5p_pins);
+	pctrl->desc.pins = pctrl->data->pins;
+	pctrl->desc.npins = pctrl->data->npins;
 	pctrl->desc.pctlops = &eq5p_pinctrl_ops;
 	pctrl->desc.pmxops = &eq5p_pinmux_ops;
 	pctrl->desc.confops = &eq5p_pinconf_ops;
@@ -562,6 +731,13 @@ static int eq5p_probe(struct auxiliary_device *adev,
 	return 0;
 }
 
+static const struct of_device_id eq5p_match_table[] = {
+	{ .compatible = "mobileye,eyeq5-olb", .data = &eq5p_eyeq5_data },
+	{ .compatible = "mobileye,eyeq6lplus-olb", .data = &eq5p_eyeq6lplus_data },
+	{}
+};
+MODULE_DEVICE_TABLE(of, eq5p_match_table);
+
 static const struct auxiliary_device_id eq5p_id_table[] = {
 	{ .name = "clk_eyeq.pinctrl" },
 	{}
@@ -571,5 +747,8 @@ MODULE_DEVICE_TABLE(auxiliary, eq5p_id_table);
 static struct auxiliary_driver eq5p_driver = {
 	.probe = eq5p_probe,
 	.id_table = eq5p_id_table,
+	.driver = {
+		.of_match_table = eq5p_match_table,
+	}
 };
 module_auxiliary_driver(eq5p_driver);
diff --git a/drivers/pinctrl/pinctrl-pic32.c b/drivers/pinctrl/pinctrl-pic32.c
index 16bbbcf720628..07a24e17d35b2 100644
--- a/drivers/pinctrl/pinctrl-pic32.c
+++ b/drivers/pinctrl/pinctrl-pic32.c
@@ -1696,7 +1696,7 @@ static inline struct pic32_gpio_bank *irqd_to_bank(struct irq_data *d)
 }
 
 static inline struct pic32_gpio_bank *pctl_to_bank(struct pic32_pinctrl *pctl,
-						unsigned pin)
+						unsigned int pin)
 {
 	return &pctl->gpio_banks[pin / PINS_PER_BANK];
 }
@@ -1709,7 +1709,7 @@ static int pic32_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
 }
 
 static const char *pic32_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
-						    unsigned group)
+						    unsigned int group)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -1717,9 +1717,9 @@ static const char *pic32_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
 }
 
 static int pic32_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
-					    unsigned group,
-					    const unsigned **pins,
-					    unsigned *num_pins)
+					    unsigned int group,
+					    const unsigned int **pins,
+					    unsigned int *num_pins)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -1745,7 +1745,7 @@ static int pic32_pinmux_get_functions_count(struct pinctrl_dev *pctldev)
 }
 
 static const char *
-pic32_pinmux_get_function_name(struct pinctrl_dev *pctldev, unsigned func)
+pic32_pinmux_get_function_name(struct pinctrl_dev *pctldev, unsigned int func)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -1753,9 +1753,9 @@ pic32_pinmux_get_function_name(struct pinctrl_dev *pctldev, unsigned func)
 }
 
 static int pic32_pinmux_get_function_groups(struct pinctrl_dev *pctldev,
-						unsigned func,
+						unsigned int func,
 						const char * const **groups,
-						unsigned * const num_groups)
+						unsigned int * const num_groups)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -1766,7 +1766,7 @@ static int pic32_pinmux_get_function_groups(struct pinctrl_dev *pctldev,
 }
 
 static int pic32_pinmux_enable(struct pinctrl_dev *pctldev,
-				   unsigned func, unsigned group)
+				   unsigned int func, unsigned int group)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	const struct pic32_pin_group *pg = &pctl->groups[group];
@@ -1795,7 +1795,7 @@ static int pic32_pinmux_enable(struct pinctrl_dev *pctldev,
 
 static int pic32_gpio_request_enable(struct pinctrl_dev *pctldev,
 				     struct pinctrl_gpio_range *range,
-				     unsigned offset)
+				     unsigned int offset)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct pic32_gpio_bank *bank = gpiochip_get_data(range->gc);
@@ -1810,7 +1810,7 @@ static int pic32_gpio_request_enable(struct pinctrl_dev *pctldev,
 }
 
 static int pic32_gpio_direction_input(struct gpio_chip *chip,
-					  unsigned offset)
+					  unsigned int offset)
 {
 	struct pic32_gpio_bank *bank = gpiochip_get_data(chip);
 	u32 mask = BIT(offset);
@@ -1820,7 +1820,7 @@ static int pic32_gpio_direction_input(struct gpio_chip *chip,
 	return 0;
 }
 
-static int pic32_gpio_get(struct gpio_chip *chip, unsigned offset)
+static int pic32_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct pic32_gpio_bank *bank = gpiochip_get_data(chip);
 
@@ -1842,7 +1842,7 @@ static int pic32_gpio_set(struct gpio_chip *chip, unsigned int offset,
 }
 
 static int pic32_gpio_direction_output(struct gpio_chip *chip,
-					   unsigned offset, int value)
+					   unsigned int offset, int value)
 {
 	struct pic32_gpio_bank *bank = gpiochip_get_data(chip);
 	u32 mask = BIT(offset);
@@ -1855,7 +1855,7 @@ static int pic32_gpio_direction_output(struct gpio_chip *chip,
 
 static int pic32_gpio_set_direction(struct pinctrl_dev *pctldev,
 					      struct pinctrl_gpio_range *range,
-					      unsigned offset, bool input)
+					      unsigned int offset, bool input)
 {
 	struct gpio_chip *chip = range->gc;
 
@@ -1876,12 +1876,12 @@ static const struct pinmux_ops pic32_pinmux_ops = {
 	.gpio_set_direction = pic32_gpio_set_direction,
 };
 
-static int pic32_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin,
+static int pic32_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 				 unsigned long *config)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct pic32_gpio_bank *bank = pctl_to_bank(pctl, pin);
-	unsigned param = pinconf_to_config_param(*config);
+	unsigned int param = pinconf_to_config_param(*config);
 	u32 mask = BIT(pin - bank->gpio_chip.base);
 	u32 arg;
 
@@ -1917,12 +1917,12 @@ static int pic32_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin,
 	return 0;
 }
 
-static int pic32_pinconf_set(struct pinctrl_dev *pctldev, unsigned pin,
-				 unsigned long *configs, unsigned num_configs)
+static int pic32_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
+				 unsigned long *configs, unsigned int num_configs)
 {
 	struct pic32_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct pic32_gpio_bank *bank = pctl_to_bank(pctl, pin);
-	unsigned param;
+	unsigned int param;
 	u32 arg;
 	unsigned int i;
 	u32 offset = pin - bank->gpio_chip.base;
@@ -1938,7 +1938,7 @@ static int pic32_pinconf_set(struct pinctrl_dev *pctldev, unsigned pin,
 		switch (param) {
 		case PIN_CONFIG_BIAS_PULL_UP:
 			dev_dbg(pctl->dev, "   pullup\n");
-			writel(mask, bank->reg_base +PIC32_SET(CNPU_REG));
+			writel(mask, bank->reg_base + PIC32_SET(CNPU_REG));
 			break;
 		case PIN_CONFIG_BIAS_PULL_DOWN:
 			dev_dbg(pctl->dev, "   pulldown\n");
@@ -1987,7 +1987,7 @@ static struct pinctrl_desc pic32_pinctrl_desc = {
 	.owner = THIS_MODULE,
 };
 
-static int pic32_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+static int pic32_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
 {
 	struct pic32_gpio_bank *bank = gpiochip_get_data(chip);
 
@@ -2174,16 +2174,10 @@ static int pic32_pinctrl_probe(struct platform_device *pdev)
 	if (IS_ERR(pctl->reg_base))
 		return PTR_ERR(pctl->reg_base);
 
-	pctl->clk = devm_clk_get(&pdev->dev, NULL);
+	pctl->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(pctl->clk)) {
 		ret = PTR_ERR(pctl->clk);
-		dev_err(&pdev->dev, "clk get failed\n");
-		return ret;
-	}
-
-	ret = clk_prepare_enable(pctl->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "clk enable failed\n");
+		dev_err(&pdev->dev, "Failed to get and enable clock\n");
 		return ret;
 	}
 
@@ -2239,16 +2233,10 @@ static int pic32_gpio_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	bank->clk = devm_clk_get(&pdev->dev, NULL);
+	bank->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(bank->clk)) {
 		ret = PTR_ERR(bank->clk);
-		dev_err(&pdev->dev, "clk get failed\n");
-		return ret;
-	}
-
-	ret = clk_prepare_enable(bank->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "clk enable failed\n");
+		dev_err(&pdev->dev, "Failed to get and enable clock\n");
 		return ret;
 	}
 
@@ -2265,7 +2253,7 @@ static int pic32_gpio_probe(struct platform_device *pdev)
 	girq->default_type = IRQ_TYPE_NONE;
 	girq->handler = handle_level_irq;
 	girq->parents[0] = irq;
-	ret = gpiochip_add_data(&bank->gpio_chip, bank);
+	ret = devm_gpiochip_add_data(&pdev->dev, &bank->gpio_chip, bank);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to add GPIO chip %u: %d\n",
 			id, ret);
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index f15b18f334ee0..2f14c7f9c95a2 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -467,6 +467,22 @@ static const struct pinctrl_ops rockchip_pctrl_ops = {
  * Hardware access
  */
 
+static struct rockchip_mux_recalced_data rv1103b_mux_recalced_data[] = {
+	{
+		.num = 1,
+		.pin = 6,
+		.reg = 0x10024,
+		.bit = 8,
+		.mask = 0xf
+	}, {
+		.num = 1,
+		.pin = 7,
+		.reg = 0x10024,
+		.bit = 12,
+		.mask = 0xf
+	},
+};
+
 static struct rockchip_mux_recalced_data rv1108_mux_recalced_data[] = {
 	{
 		.num = 1,
@@ -1172,6 +1188,9 @@ static int rockchip_get_mux(struct rockchip_pin_bank *bank, int pin)
 	else
 		regmap = info->regmap_base;
 
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin >= 12)
+		return 0;
+
 	if (ctrl->type == RK3506) {
 		if (bank->bank_num == 1)
 			regmap = info->regmap_ioc1;
@@ -1298,6 +1317,9 @@ static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux)
 	else
 		regmap = info->regmap_base;
 
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin >= 12)
+		return 0;
+
 	if (ctrl->type == RK3506) {
 		if (bank->bank_num == 1)
 			regmap = info->regmap_ioc1;
@@ -1495,6 +1517,214 @@ static int px30_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank,
 	return 0;
 }
 
+#define RV1103B_DRV_BITS_PER_PIN		8
+#define RV1103B_DRV_PINS_PER_REG		2
+#define RV1103B_DRV_GPIO0_A_OFFSET		0x40100
+#define RV1103B_DRV_GPIO0_B_OFFSET		0x50110
+#define RV1103B_DRV_GPIO1_A01_OFFSET		0x140
+#define RV1103B_DRV_GPIO1_A67_OFFSET		0x1014C
+#define RV1103B_DRV_GPIO2_OFFSET		0x30180
+#define RV1103B_DRV_GPIO2_SARADC_OFFSET		0x3080C
+
+static int rv1103b_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank,
+				       int pin_num, struct regmap **regmap,
+				       int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+	int ret = 0;
+
+	*regmap = info->regmap_base;
+	switch (bank->bank_num) {
+	case 0:
+		if (pin_num < 7)
+			*reg = RV1103B_DRV_GPIO0_A_OFFSET;
+		else if (pin_num > 7 && pin_num < 14)
+			*reg = RV1103B_DRV_GPIO0_B_OFFSET - 0x10;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 1:
+		if (pin_num < 6)
+			*reg = RV1103B_DRV_GPIO1_A01_OFFSET;
+		else if (pin_num >= 6 && pin_num < 23)
+			*reg = RV1103B_DRV_GPIO1_A67_OFFSET - 0xc;
+		else if (pin_num >= 24 && pin_num < 30)
+			*reg = RV1103B_DRV_GPIO1_A67_OFFSET - 0xc;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 2:
+		if (pin_num < 12) {
+			*reg = RV1103B_DRV_GPIO2_OFFSET;
+		} else if (pin_num >= 16) {
+			ret = -EINVAL;
+		} else {
+			*reg = RV1103B_DRV_GPIO2_SARADC_OFFSET;
+			*bit = 10;
+
+			return 0;
+		}
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret) {
+		dev_err(info->dev, "unsupported bank_num %d pin_num %d\n", bank->bank_num, pin_num);
+
+		return ret;
+	}
+
+	*reg += ((pin_num / RV1103B_DRV_PINS_PER_REG) * 4);
+	*bit = pin_num % RV1103B_DRV_PINS_PER_REG;
+	*bit *= RV1103B_DRV_BITS_PER_PIN;
+
+	return 0;
+}
+
+#define RV1103B_PULL_BITS_PER_PIN		2
+#define RV1103B_PULL_PINS_PER_REG		8
+#define RV1103B_PULL_GPIO0_A_OFFSET		0x40200
+#define RV1103B_PULL_GPIO0_B_OFFSET		0x50204
+#define RV1103B_PULL_GPIO1_A01_OFFSET		0x210
+#define RV1103B_PULL_GPIO1_A67_OFFSET		0x10210
+#define RV1103B_PULL_GPIO2_OFFSET		0x30220
+#define RV1103B_PULL_GPIO2_SARADC_OFFSET	0x3080C
+
+static int rv1103b_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
+					int pin_num, struct regmap **regmap,
+					int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+	int ret = 0;
+
+	*regmap = info->regmap_base;
+	switch (bank->bank_num) {
+	case 0:
+		if (pin_num < 7)
+			*reg = RV1103B_PULL_GPIO0_A_OFFSET;
+		else if (pin_num > 7 && pin_num < 14)
+			*reg = RV1103B_PULL_GPIO0_B_OFFSET - 0x4;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 1:
+		if (pin_num < 6)
+			*reg = RV1103B_PULL_GPIO1_A01_OFFSET;
+		else if (pin_num >= 6 && pin_num < 23)
+			*reg = RV1103B_PULL_GPIO1_A67_OFFSET;
+		else if (pin_num >= 24 && pin_num < 30)
+			*reg = RV1103B_PULL_GPIO1_A67_OFFSET;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 2:
+		if (pin_num < 12) {
+			*reg = RV1103B_PULL_GPIO2_OFFSET;
+		} else if (pin_num >= 16) {
+			ret = -EINVAL;
+		} else {
+			*reg = RV1103B_PULL_GPIO2_SARADC_OFFSET;
+			*bit = 13;
+
+			return 0;
+		}
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret) {
+		dev_err(info->dev, "unsupported bank_num %d pin_num %d\n", bank->bank_num, pin_num);
+
+		return ret;
+	}
+
+	*reg += ((pin_num / RV1103B_PULL_PINS_PER_REG) * 4);
+	*bit = pin_num % RV1103B_PULL_PINS_PER_REG;
+	*bit *= RV1103B_PULL_BITS_PER_PIN;
+
+	return 0;
+}
+
+#define RV1103B_SMT_BITS_PER_PIN		1
+#define RV1103B_SMT_PINS_PER_REG		8
+#define RV1103B_SMT_GPIO0_A_OFFSET		0x40400
+#define RV1103B_SMT_GPIO0_B_OFFSET		0x50404
+#define RV1103B_SMT_GPIO1_A01_OFFSET		0x410
+#define RV1103B_SMT_GPIO1_A67_OFFSET		0x10410
+#define RV1103B_SMT_GPIO2_OFFSET		0x30420
+#define RV1103B_SMT_GPIO2_SARADC_OFFSET		0x3080C
+
+static int rv1103b_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank,
+					   int pin_num,
+					   struct regmap **regmap,
+					   int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+	int ret = 0;
+
+	*regmap = info->regmap_base;
+	switch (bank->bank_num) {
+	case 0:
+		if (pin_num < 7)
+			*reg = RV1103B_SMT_GPIO0_A_OFFSET;
+		else if (pin_num > 7 && pin_num < 14)
+			*reg = RV1103B_SMT_GPIO0_B_OFFSET - 0x4;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 1:
+		if (pin_num < 6)
+			*reg = RV1103B_SMT_GPIO1_A01_OFFSET;
+		else if (pin_num >= 6 && pin_num < 23)
+			*reg = RV1103B_SMT_GPIO1_A67_OFFSET;
+		else if (pin_num >= 24 && pin_num < 30)
+			*reg = RV1103B_SMT_GPIO1_A67_OFFSET;
+		else
+			ret = -EINVAL;
+		break;
+
+	case 2:
+		if (pin_num < 12) {
+			*reg = RV1103B_SMT_GPIO2_OFFSET;
+		} else if (pin_num >= 16) {
+			ret = -EINVAL;
+		} else {
+			*reg = RV1103B_SMT_GPIO2_SARADC_OFFSET;
+			*bit = 8;
+
+			return 0;
+		}
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret) {
+		dev_err(info->dev, "unsupported bank_num %d pin_num %d\n", bank->bank_num, pin_num);
+
+		return ret;
+	}
+
+	*reg += ((pin_num / RV1103B_SMT_PINS_PER_REG) * 4);
+	*bit = pin_num % RV1103B_SMT_PINS_PER_REG;
+	*bit *= RV1103B_SMT_BITS_PER_PIN;
+
+	return 0;
+}
+
 #define RV1108_PULL_PMU_OFFSET		0x10
 #define RV1108_PULL_OFFSET		0x110
 #define RV1108_PULL_PINS_PER_REG	8
@@ -2982,6 +3212,9 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank,
 	u8 bit;
 	int drv_type = bank->drv[pin_num / 8].drv_type;
 
+	if (ctrl->type == RV1103B && pin_num >= 12)
+		drv_type = DRV_TYPE_IO_LEVEL_2_BIT;
+
 	ret = ctrl->drv_calc_reg(bank, pin_num, &regmap, &reg, &bit);
 	if (ret)
 		return ret;
@@ -3043,6 +3276,11 @@ static int rockchip_get_drive_perpin(struct rockchip_pin_bank *bank,
 	if (ret)
 		return ret;
 
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) {
+		data = data >> 10;
+		return data & 0x3;
+	}
+
 	data >>= bit;
 	data &= (1 << rmask_bits) - 1;
 
@@ -3071,7 +3309,8 @@ static int rockchip_set_drive_perpin(struct rockchip_pin_bank *bank,
 		rmask_bits = RK3588_DRV_BITS_PER_PIN;
 		ret = strength;
 		goto config;
-	} else if (ctrl->type == RK3506 ||
+	} else if (ctrl->type == RV1103B ||
+		   ctrl->type == RK3506 ||
 		   ctrl->type == RK3528 ||
 		   ctrl->type == RK3562 ||
 		   ctrl->type == RK3568) {
@@ -3182,6 +3421,12 @@ static int rockchip_set_drive_perpin(struct rockchip_pin_bank *bank,
 			ret = strength;
 		}
 	}
+
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) {
+		rmask_bits = 2;
+		ret = strength;
+	}
+
 	/* enable the write to the equivalent lower bits */
 	data = ((1 << rmask_bits) - 1) << (bit + 16);
 	rmask = data | (data >> 16);
@@ -3236,6 +3481,7 @@ static int rockchip_get_pull(struct rockchip_pin_bank *bank, int pin_num)
 				? PIN_CONFIG_BIAS_PULL_PIN_DEFAULT
 				: PIN_CONFIG_BIAS_DISABLE;
 	case PX30:
+	case RV1103B:
 	case RV1108:
 	case RK3188:
 	case RK3288:
@@ -3251,6 +3497,9 @@ static int rockchip_get_pull(struct rockchip_pin_bank *bank, int pin_num)
 		pull_type = bank->pull_type[pin_num / 8];
 		data >>= bit;
 		data &= (1 << RK3188_PULL_BITS_PER_PIN) - 1;
+
+		if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12)
+			pull_type = 1;
 		/*
 		 * In the TRM, pull-up being 1 for everything except the GPIO0_D3-D6,
 		 * where that pull up value becomes 3.
@@ -3297,6 +3546,7 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank,
 		ret = regmap_write(regmap, reg, data);
 		break;
 	case PX30:
+	case RV1103B:
 	case RV1108:
 	case RV1126:
 	case RK3188:
@@ -3312,6 +3562,8 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank,
 	case RK3576:
 	case RK3588:
 		pull_type = bank->pull_type[pin_num / 8];
+		if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12)
+			pull_type = 1;
 		ret = -EINVAL;
 		for (i = 0; i < ARRAY_SIZE(rockchip_pull_list[pull_type]);
 			i++) {
@@ -3417,6 +3669,11 @@ static int rockchip_get_schmitt(struct rockchip_pin_bank *bank, int pin_num)
 	if (ret)
 		return ret;
 
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) {
+		data >>= 8;
+		return data & 0x3;
+	}
+
 	data >>= bit;
 	switch (ctrl->type) {
 	case RK3562:
@@ -3473,6 +3730,12 @@ static int rockchip_set_schmitt(struct rockchip_pin_bank *bank,
 		}
 	}
 
+	if (ctrl->type == RV1103B && bank->bank_num == 2 && pin_num >= 12) {
+		data = 0x3 << (bit + 16);
+		rmask = data | (data >> 16);
+		data |= ((enable ? 0x3 : 0) << bit);
+	}
+
 	return regmap_update_bits(regmap, reg, rmask, data);
 }
 
@@ -3579,6 +3842,7 @@ static bool rockchip_pinconf_pull_valid(struct rockchip_pin_ctrl *ctrl,
 	case RK3066B:
 		return pull ? false : true;
 	case PX30:
+	case RV1103B:
 	case RV1108:
 	case RV1126:
 	case RK3188:
@@ -4314,6 +4578,51 @@ static struct rockchip_pin_ctrl px30_pin_ctrl = {
 		.schmitt_calc_reg	= px30_calc_schmitt_reg_and_bit,
 };
 
+static struct rockchip_pin_bank rv1103b_pin_banks[] = {
+	PIN_BANK_IOMUX_FLAGS_OFFSET_DRV_FLAGS(0, 32, "gpio0",
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    0x40000, 0x50008, 0x50010, 0x50018,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT),
+	PIN_BANK_IOMUX_FLAGS_OFFSET_DRV_FLAGS(1, 32, "gpio1",
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    0x20, 0x10028, 0x10030, 0x10038,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT),
+	PIN_BANK_IOMUX_FLAGS_OFFSET_DRV_FLAGS(2, 32, "gpio2",
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    IOMUX_WIDTH_4BIT,
+				    0x30040, 0x30048, 0x30050, 0x30058,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT,
+				    DRV_TYPE_IO_LEVEL_8_BIT),
+};
+
+static struct rockchip_pin_ctrl rv1103b_pin_ctrl __maybe_unused = {
+	.pin_banks		= rv1103b_pin_banks,
+	.nr_banks		= ARRAY_SIZE(rv1103b_pin_banks),
+	.label			= "RV1103B-GPIO",
+	.type			= RV1103B,
+	.iomux_recalced		= rv1103b_mux_recalced_data,
+	.niomux_recalced	= ARRAY_SIZE(rv1103b_mux_recalced_data),
+	.pull_calc_reg		= rv1103b_calc_pull_reg_and_bit,
+	.drv_calc_reg		= rv1103b_calc_drv_reg_and_bit,
+	.schmitt_calc_reg	= rv1103b_calc_schmitt_reg_and_bit,
+};
+
 static struct rockchip_pin_bank rv1108_pin_banks[] = {
 	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", IOMUX_SOURCE_PMU,
 					     IOMUX_SOURCE_PMU,
@@ -4950,6 +5259,8 @@ static struct rockchip_pin_ctrl rk3588_pin_ctrl = {
 static const struct of_device_id rockchip_pinctrl_dt_match[] = {
 	{ .compatible = "rockchip,px30-pinctrl",
 		.data = &px30_pin_ctrl },
+	{ .compatible = "rockchip,rv1103b-pinctrl",
+		.data = &rv1103b_pin_ctrl },
 	{ .compatible = "rockchip,rv1108-pinctrl",
 		.data = &rv1108_pin_ctrl },
 	{ .compatible = "rockchip,rv1126-pinctrl",
diff --git a/drivers/pinctrl/pinctrl-rockchip.h b/drivers/pinctrl/pinctrl-rockchip.h
index 4f4aff42a80a9..bb0e803e3b8ab 100644
--- a/drivers/pinctrl/pinctrl-rockchip.h
+++ b/drivers/pinctrl/pinctrl-rockchip.h
@@ -185,6 +185,7 @@
 
 enum rockchip_pinctrl_type {
 	PX30,
+	RV1103B,
 	RV1108,
 	RV1126,
 	RK2928,
diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c
index d85e6c1f63218..288c9c9bce9a5 100644
--- a/drivers/pinctrl/pinctrl-single.c
+++ b/drivers/pinctrl/pinctrl-single.c
@@ -1980,6 +1980,7 @@ static const struct of_device_id pcs_of_match[] = {
 	{ .compatible = "ti,omap4-padconf", .data = &pinctrl_single_omap_wkup },
 	{ .compatible = "ti,omap5-padconf", .data = &pinctrl_single_omap_wkup },
 	{ .compatible = "ti,j7200-padconf", .data = &pinctrl_single_j7200 },
+	{ .compatible = "ti,am62l-padconf", .data = &pinctrl_single_j7200 },
 	{ .compatible = "pinctrl-single", .data = &pinctrl_single },
 	{ .compatible = "pinconf-single", .data = &pinconf_single },
 	{ },
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index f56592411cf6d..ee34ffca3917f 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -60,6 +60,16 @@ config PINCTRL_LPASS_LPI
 	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SoCs.
 
+config PINCTRL_MILOS_LPASS_LPI
+	tristate "Qualcomm Technologies Inc Milos LPASS LPI pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	depends on PINCTRL_LPASS_LPI
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+	  (Low Power Island) found on the Qualcomm Technologies Inc Milos
+	  platform.
+
 config PINCTRL_SC7280_LPASS_LPI
 	tristate "Qualcomm Technologies Inc SC7280 and SM8350 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm
index 3e9e02774001b..6df6159fa5f89 100644
--- a/drivers/pinctrl/qcom/Kconfig.msm
+++ b/drivers/pinctrl/qcom/Kconfig.msm
@@ -15,6 +15,16 @@ config PINCTRL_APQ8084
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm APQ8084 platform.
 
+config PINCTRL_ELIZA
+	tristate "Qualcomm Technologies Inc Eliza pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc Top Level Mode Multiplexer block (TLMM)
+	  block found on the Qualcomm Technologies Inc Eliza platform.
+	  Say Y here to compile statically, or M here to compile it as a module.
+	  If unsure, say N.
+
 config PINCTRL_GLYMUR
 	tristate "Qualcomm Technologies Inc Glymur pin controller driver"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 4269d1781015e..a8fd12f90d6e6 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -3,6 +3,7 @@
 obj-$(CONFIG_PINCTRL_MSM)	+= pinctrl-msm.o
 obj-$(CONFIG_PINCTRL_APQ8064)	+= pinctrl-apq8064.o
 obj-$(CONFIG_PINCTRL_APQ8084)	+= pinctrl-apq8084.o
+obj-$(CONFIG_PINCTRL_ELIZA)	+= pinctrl-eliza.o
 obj-$(CONFIG_PINCTRL_GLYMUR)	+= pinctrl-glymur.o
 obj-$(CONFIG_PINCTRL_IPQ4019)	+= pinctrl-ipq4019.o
 obj-$(CONFIG_PINCTRL_IPQ5018)	+= pinctrl-ipq5018.o
@@ -33,6 +34,7 @@ obj-$(CONFIG_PINCTRL_QDF2XXX)	+= pinctrl-qdf2xxx.o
 obj-$(CONFIG_PINCTRL_MDM9607)	+= pinctrl-mdm9607.o
 obj-$(CONFIG_PINCTRL_MDM9615)	+= pinctrl-mdm9615.o
 obj-$(CONFIG_PINCTRL_MILOS) += pinctrl-milos.o
+obj-$(CONFIG_PINCTRL_MILOS_LPASS_LPI) += pinctrl-milos-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_QCOM_SPMI_PMIC) += pinctrl-spmi-gpio.o
 obj-$(CONFIG_PINCTRL_QCOM_SPMI_PMIC) += pinctrl-spmi-mpp.o
 obj-$(CONFIG_PINCTRL_QCOM_SSBI_PMIC) += pinctrl-ssbi-gpio.o
diff --git a/drivers/pinctrl/qcom/pinctrl-eliza.c b/drivers/pinctrl/qcom/pinctrl-eliza.c
new file mode 100644
index 0000000000000..1a2e6461a69b8
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-eliza.c
@@ -0,0 +1,1548 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_SIZE 0x1000
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11)	\
+	{					        \
+		.grp = PINCTRL_PINGROUP("gpio" #id,	\
+			gpio##id##_pins,		\
+			ARRAY_SIZE(gpio##id##_pins)),	\
+		.funcs = (int[]){			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10,			\
+			msm_mux_##f11 /* egpio mode */	\
+		},					\
+		.nfuncs = 12,				\
+		.ctl_reg = REG_SIZE * id,		\
+		.io_reg = 0x4 + REG_SIZE * id,		\
+		.intr_cfg_reg = 0x8 + REG_SIZE * id,	\
+		.intr_status_reg = 0xc + REG_SIZE * id,	\
+		.intr_target_reg = 0x8 + REG_SIZE * id,	\
+		.mux_bit = 2,			\
+		.pull_bit = 0,			\
+		.drv_bit = 6,			\
+		.egpio_enable = 12,		\
+		.egpio_present = 11,		\
+		.oe_bit = 9,			\
+		.in_bit = 0,			\
+		.out_bit = 1,			\
+		.intr_enable_bit = 0,		\
+		.intr_status_bit = 0,		\
+		.intr_wakeup_present_bit = 6,	\
+		.intr_wakeup_enable_bit = 7,	\
+		.intr_target_bit = 5,		\
+		.intr_target_kpss_val = 3,	\
+		.intr_raw_status_bit = 4,	\
+		.intr_polarity_bit = 1,		\
+		.intr_detection_bit = 2,	\
+		.intr_detection_width = 2,	\
+	}
+
+#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv)	\
+	{					        \
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),	\
+		.ctl_reg = ctl,				\
+		.io_reg = 0,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = pull,			\
+		.drv_bit = drv,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = -1,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+#define UFS_RESET(pg_name, ctl, io)			\
+	{					        \
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),    \
+		.ctl_reg = ctl,				\
+		.io_reg = io,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = 3,				\
+		.drv_bit = 0,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = 0,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+static const struct pinctrl_pin_desc eliza_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "GPIO_146"),
+	PINCTRL_PIN(147, "GPIO_147"),
+	PINCTRL_PIN(148, "GPIO_148"),
+	PINCTRL_PIN(149, "GPIO_149"),
+	PINCTRL_PIN(150, "GPIO_150"),
+	PINCTRL_PIN(151, "GPIO_151"),
+	PINCTRL_PIN(152, "GPIO_152"),
+	PINCTRL_PIN(153, "GPIO_153"),
+	PINCTRL_PIN(154, "GPIO_154"),
+	PINCTRL_PIN(155, "GPIO_155"),
+	PINCTRL_PIN(156, "GPIO_156"),
+	PINCTRL_PIN(157, "GPIO_157"),
+	PINCTRL_PIN(158, "GPIO_158"),
+	PINCTRL_PIN(159, "GPIO_159"),
+	PINCTRL_PIN(160, "GPIO_160"),
+	PINCTRL_PIN(161, "GPIO_161"),
+	PINCTRL_PIN(162, "GPIO_162"),
+	PINCTRL_PIN(163, "GPIO_163"),
+	PINCTRL_PIN(164, "GPIO_164"),
+	PINCTRL_PIN(165, "GPIO_165"),
+	PINCTRL_PIN(166, "GPIO_166"),
+	PINCTRL_PIN(167, "GPIO_167"),
+	PINCTRL_PIN(168, "GPIO_168"),
+	PINCTRL_PIN(169, "GPIO_169"),
+	PINCTRL_PIN(170, "GPIO_170"),
+	PINCTRL_PIN(171, "GPIO_171"),
+	PINCTRL_PIN(172, "GPIO_172"),
+	PINCTRL_PIN(173, "GPIO_173"),
+	PINCTRL_PIN(174, "GPIO_174"),
+	PINCTRL_PIN(175, "GPIO_175"),
+	PINCTRL_PIN(176, "GPIO_176"),
+	PINCTRL_PIN(177, "GPIO_177"),
+	PINCTRL_PIN(178, "GPIO_178"),
+	PINCTRL_PIN(179, "GPIO_179"),
+	PINCTRL_PIN(180, "GPIO_180"),
+	PINCTRL_PIN(181, "GPIO_181"),
+	PINCTRL_PIN(182, "GPIO_182"),
+	PINCTRL_PIN(183, "GPIO_183"),
+	PINCTRL_PIN(184, "GPIO_184"),
+	PINCTRL_PIN(185, "UFS_RESET"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+DECLARE_MSM_GPIO_PINS(146);
+DECLARE_MSM_GPIO_PINS(147);
+DECLARE_MSM_GPIO_PINS(148);
+DECLARE_MSM_GPIO_PINS(149);
+DECLARE_MSM_GPIO_PINS(150);
+DECLARE_MSM_GPIO_PINS(151);
+DECLARE_MSM_GPIO_PINS(152);
+DECLARE_MSM_GPIO_PINS(153);
+DECLARE_MSM_GPIO_PINS(154);
+DECLARE_MSM_GPIO_PINS(155);
+DECLARE_MSM_GPIO_PINS(156);
+DECLARE_MSM_GPIO_PINS(157);
+DECLARE_MSM_GPIO_PINS(158);
+DECLARE_MSM_GPIO_PINS(159);
+DECLARE_MSM_GPIO_PINS(160);
+DECLARE_MSM_GPIO_PINS(161);
+DECLARE_MSM_GPIO_PINS(162);
+DECLARE_MSM_GPIO_PINS(163);
+DECLARE_MSM_GPIO_PINS(164);
+DECLARE_MSM_GPIO_PINS(165);
+DECLARE_MSM_GPIO_PINS(166);
+DECLARE_MSM_GPIO_PINS(167);
+DECLARE_MSM_GPIO_PINS(168);
+DECLARE_MSM_GPIO_PINS(169);
+DECLARE_MSM_GPIO_PINS(170);
+DECLARE_MSM_GPIO_PINS(171);
+DECLARE_MSM_GPIO_PINS(172);
+DECLARE_MSM_GPIO_PINS(173);
+DECLARE_MSM_GPIO_PINS(174);
+DECLARE_MSM_GPIO_PINS(175);
+DECLARE_MSM_GPIO_PINS(176);
+DECLARE_MSM_GPIO_PINS(177);
+DECLARE_MSM_GPIO_PINS(178);
+DECLARE_MSM_GPIO_PINS(179);
+DECLARE_MSM_GPIO_PINS(180);
+DECLARE_MSM_GPIO_PINS(181);
+DECLARE_MSM_GPIO_PINS(182);
+DECLARE_MSM_GPIO_PINS(183);
+DECLARE_MSM_GPIO_PINS(184);
+
+static const unsigned int ufs_reset_pins[] = { 185 };
+
+enum eliza_functions {
+	msm_mux_gpio,
+	msm_mux_aoss_cti,
+	msm_mux_atest_char,
+	msm_mux_atest_usb,
+	msm_mux_audio_ext_mclk0,
+	msm_mux_audio_ref_clk,
+	msm_mux_cam_mclk,
+	msm_mux_cci_async_in,
+	msm_mux_cci_i2c_scl,
+	msm_mux_cci_i2c_sda,
+	msm_mux_cci_timer,
+	msm_mux_coex_uart1_rx,
+	msm_mux_coex_uart1_tx,
+	msm_mux_coex_uart2_rx,
+	msm_mux_coex_uart2_tx,
+	msm_mux_dbg_out_clk,
+	msm_mux_ddr_bist_complete,
+	msm_mux_ddr_bist_fail,
+	msm_mux_ddr_bist_start,
+	msm_mux_ddr_bist_stop,
+	msm_mux_ddr_pxi0,
+	msm_mux_ddr_pxi1,
+	msm_mux_dp0_hot,
+	msm_mux_egpio,
+	msm_mux_gcc_gp1,
+	msm_mux_gcc_gp2,
+	msm_mux_gcc_gp3,
+	msm_mux_gnss_adc0,
+	msm_mux_gnss_adc1,
+	msm_mux_hdmi_ddc_scl,
+	msm_mux_hdmi_ddc_sda,
+	msm_mux_hdmi_dtest0,
+	msm_mux_hdmi_dtest1,
+	msm_mux_hdmi_hot_plug,
+	msm_mux_hdmi_pixel_clk,
+	msm_mux_hdmi_rcv_det,
+	msm_mux_hdmi_tx_cec,
+	msm_mux_host2wlan_sol,
+	msm_mux_i2s0_data0,
+	msm_mux_i2s0_data1,
+	msm_mux_i2s0_sck,
+	msm_mux_i2s0_ws,
+	msm_mux_ibi_i3c,
+	msm_mux_jitter_bist,
+	msm_mux_mdp_esync0_out,
+	msm_mux_mdp_esync1_out,
+	msm_mux_mdp_vsync,
+	msm_mux_mdp_vsync0_out,
+	msm_mux_mdp_vsync11_out,
+	msm_mux_mdp_vsync1_out,
+	msm_mux_mdp_vsync2_out,
+	msm_mux_mdp_vsync3_out,
+	msm_mux_mdp_vsync_e,
+	msm_mux_nav_gpio0,
+	msm_mux_nav_gpio1,
+	msm_mux_nav_gpio2,
+	msm_mux_nav_gpio3,
+	msm_mux_pcie0_clk_req_n,
+	msm_mux_pcie1_clk_req_n,
+	msm_mux_phase_flag,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_clk_aux,
+	msm_mux_prng_rosc0,
+	msm_mux_prng_rosc1,
+	msm_mux_prng_rosc2,
+	msm_mux_prng_rosc3,
+	msm_mux_qdss_cti,
+	msm_mux_qdss_gpio_traceclk,
+	msm_mux_qdss_gpio_tracectl,
+	msm_mux_qdss_gpio_tracedata,
+	msm_mux_qlink_big_enable,
+	msm_mux_qlink_big_request,
+	msm_mux_qlink_little_enable,
+	msm_mux_qlink_little_request,
+	msm_mux_qlink_wmss,
+	msm_mux_qspi0,
+	msm_mux_qspi_clk,
+	msm_mux_qspi_cs,
+	msm_mux_qup1_se0,
+	msm_mux_qup1_se1,
+	msm_mux_qup1_se2,
+	msm_mux_qup1_se3,
+	msm_mux_qup1_se4,
+	msm_mux_qup1_se5,
+	msm_mux_qup1_se6,
+	msm_mux_qup1_se7,
+	msm_mux_qup2_se0,
+	msm_mux_qup2_se1,
+	msm_mux_qup2_se2,
+	msm_mux_qup2_se3,
+	msm_mux_qup2_se4,
+	msm_mux_qup2_se5,
+	msm_mux_qup2_se6,
+	msm_mux_qup2_se7,
+	msm_mux_resout_gpio,
+	msm_mux_sd_write_protect,
+	msm_mux_sdc1,
+	msm_mux_sdc2,
+	msm_mux_sdc2_fb_clk,
+	msm_mux_tb_trig_sdc1,
+	msm_mux_tb_trig_sdc2,
+	msm_mux_tmess_prng0,
+	msm_mux_tmess_prng1,
+	msm_mux_tmess_prng2,
+	msm_mux_tmess_prng3,
+	msm_mux_tsense_pwm1,
+	msm_mux_tsense_pwm2,
+	msm_mux_tsense_pwm3,
+	msm_mux_tsense_pwm4,
+	msm_mux_uim0_clk,
+	msm_mux_uim0_data,
+	msm_mux_uim0_present,
+	msm_mux_uim0_reset,
+	msm_mux_uim1_clk,
+	msm_mux_uim1_data,
+	msm_mux_uim1_present,
+	msm_mux_uim1_reset,
+	msm_mux_usb0_hs,
+	msm_mux_usb_phy,
+	msm_mux_vfr_0,
+	msm_mux_vfr_1,
+	msm_mux_vsense_trigger_mirnat,
+	msm_mux_wcn_sw_ctrl,
+	msm_mux__,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0",   "gpio1",   "gpio2",	 "gpio3",   "gpio4",   "gpio5",
+	"gpio6",   "gpio7",   "gpio8",	 "gpio9",   "gpio10",  "gpio11",
+	"gpio12",  "gpio13",  "gpio16",	 "gpio17",  "gpio18",  "gpio19",
+	"gpio20",  "gpio21",  "gpio22",	 "gpio23",  "gpio26",  "gpio27",
+	"gpio28",  "gpio29",  "gpio30",	 "gpio31",  "gpio32",  "gpio33",
+	"gpio34",  "gpio35",  "gpio36",	 "gpio37",  "gpio38",  "gpio39",
+	"gpio40",  "gpio42",  "gpio44",	 "gpio45",  "gpio46",  "gpio47",
+	"gpio48",  "gpio49",  "gpio50",	 "gpio51",  "gpio52",  "gpio53",
+	"gpio54",  "gpio55",  "gpio56",	 "gpio57",  "gpio58",  "gpio59",
+	"gpio60",  "gpio61",  "gpio62",	 "gpio63",  "gpio64",  "gpio65",
+	"gpio66",  "gpio67",  "gpio68",	 "gpio69",  "gpio70",  "gpio71",
+	"gpio72",  "gpio73",  "gpio74",	 "gpio75",  "gpio76",  "gpio77",
+	"gpio78",  "gpio79",  "gpio80",	 "gpio81",  "gpio82",  "gpio84",
+	"gpio85",  "gpio86",  "gpio87",	 "gpio88",  "gpio89",  "gpio90",
+	"gpio91",  "gpio92",  "gpio93",	 "gpio94",  "gpio95",  "gpio96",
+	"gpio97",  "gpio98",  "gpio99",	 "gpio100", "gpio101", "gpio102",
+	"gpio103", "gpio104", "gpio105", "gpio106", "gpio107", "gpio108",
+	"gpio109", "gpio110", "gpio111", "gpio112", "gpio113", "gpio114",
+	"gpio115", "gpio116", "gpio117", "gpio118", "gpio119", "gpio120",
+	"gpio121", "gpio122", "gpio123", "gpio124", "gpio125", "gpio126",
+	"gpio127", "gpio128", "gpio129", "gpio130", "gpio131", "gpio132",
+	"gpio133", "gpio134", "gpio135", "gpio138", "gpio139", "gpio140",
+	"gpio141", "gpio142", "gpio143", "gpio144", "gpio145", "gpio146",
+	"gpio147", "gpio148", "gpio149", "gpio150", "gpio151", "gpio152",
+	"gpio153", "gpio154", "gpio155", "gpio156", "gpio157", "gpio158",
+	"gpio159", "gpio160", "gpio161", "gpio162", "gpio163", "gpio164",
+	"gpio165", "gpio166", "gpio167", "gpio168", "gpio169", "gpio170",
+	"gpio171", "gpio172", "gpio173", "gpio174", "gpio175", "gpio176",
+	"gpio177", "gpio178", "gpio179", "gpio180", "gpio181", "gpio182",
+	"gpio184",
+};
+
+static const char *const aoss_cti_groups[] = {
+	"gpio0", "gpio1", "gpio26", "gpio27",
+};
+
+static const char *const atest_char_groups[] = {
+	"gpio71", "gpio70", "gpio72", "gpio74", "gpio73",
+};
+
+static const char *const atest_usb_groups[] = {
+	"gpio55", "gpio54",
+};
+
+static const char *const audio_ext_mclk0_groups[] = {
+	"gpio69",
+};
+
+static const char *const audio_ref_clk_groups[] = {
+	"gpio32",
+};
+
+static const char *const cam_mclk_groups[] = {
+	"gpio65", "gpio66", "gpio67", "gpio68", "gpio69",
+};
+
+static const char *const cci_async_in_groups[] = {
+	"gpio115", "gpio31", "gpio30",
+};
+
+static const char *const cci_i2c_scl_groups[] = {
+	"gpio71", "gpio73", "gpio75", "gpio77",
+};
+
+static const char *const cci_i2c_sda_groups[] = {
+	"gpio70", "gpio72", "gpio74", "gpio76",
+};
+
+static const char *const cci_timer_groups[] = {
+	"gpio76", "gpio63", "gpio125", "gpio126", "gpio127",
+};
+
+static const char *const coex_uart1_rx_groups[] = {
+	"gpio112",
+};
+
+static const char *const coex_uart1_tx_groups[] = {
+	"gpio111",
+};
+
+static const char *const coex_uart2_rx_groups[] = {
+	"gpio116",
+};
+
+static const char *const coex_uart2_tx_groups[] = {
+	"gpio100",
+};
+
+static const char *const dbg_out_clk_groups[] = {
+	"gpio81",
+};
+
+static const char *const ddr_bist_complete_groups[] = {
+	"gpio52",
+};
+
+static const char *const ddr_bist_fail_groups[] = {
+	"gpio147",
+};
+
+static const char *const ddr_bist_start_groups[] = {
+	"gpio34",
+};
+
+static const char *const ddr_bist_stop_groups[] = {
+	"gpio53",
+};
+
+static const char *const ddr_pxi0_groups[] = {
+	"gpio54", "gpio55",
+};
+
+static const char *const ddr_pxi1_groups[] = {
+	"gpio40", "gpio42",
+};
+
+static const char *const dp0_hot_groups[] = {
+	"gpio55",
+};
+
+static const char *const egpio_groups[] = {
+	"gpio28",  "gpio29",  "gpio30",	 "gpio31",  "gpio138", "gpio139",
+	"gpio140", "gpio141", "gpio142", "gpio143", "gpio144", "gpio145",
+	"gpio146", "gpio147", "gpio148", "gpio149", "gpio150", "gpio151",
+	"gpio152", "gpio153", "gpio154", "gpio155", "gpio156", "gpio157",
+	"gpio158", "gpio159", "gpio160", "gpio161", "gpio162", "gpio163",
+	"gpio164", "gpio165", "gpio166", "gpio167", "gpio168", "gpio169",
+	"gpio170", "gpio171", "gpio172", "gpio173", "gpio174", "gpio175",
+	"gpio176", "gpio177", "gpio178", "gpio179", "gpio180", "gpio181",
+	"gpio182", "gpio184",
+};
+
+static const char *const gcc_gp1_groups[] = {
+	"gpio27", "gpio53",
+};
+
+static const char *const gcc_gp2_groups[] = {
+	"gpio32", "gpio35",
+};
+
+static const char *const gcc_gp3_groups[] = {
+	"gpio30", "gpio33",
+};
+
+static const char *const gnss_adc0_groups[] = {
+	"gpio42", "gpio55",
+};
+
+static const char *const gnss_adc1_groups[] = {
+	"gpio40", "gpio54",
+};
+
+static const char *const hdmi_ddc_scl_groups[] = {
+	"gpio6",
+};
+
+static const char *const hdmi_ddc_sda_groups[] = {
+	"gpio7",
+};
+
+static const char *const hdmi_dtest0_groups[] = {
+	"gpio132",
+};
+
+static const char *const hdmi_dtest1_groups[] = {
+	"gpio133",
+};
+
+static const char *const hdmi_hot_plug_groups[] = {
+	"gpio47",
+};
+
+static const char *const hdmi_pixel_clk_groups[] = {
+	"gpio18",
+};
+
+static const char *const hdmi_rcv_det_groups[] = {
+	"gpio19",
+};
+
+static const char *const hdmi_tx_cec_groups[] = {
+	"gpio46",
+};
+
+static const char *const host2wlan_sol_groups[] = {
+	"gpio33",
+};
+
+static const char *const i2s0_data0_groups[] = {
+	"gpio64",
+};
+
+static const char *const i2s0_data1_groups[] = {
+	"gpio63",
+};
+
+static const char *const i2s0_sck_groups[] = {
+	"gpio60",
+};
+
+static const char *const i2s0_ws_groups[] = {
+	"gpio61",
+};
+
+static const char *const ibi_i3c_groups[] = {
+	"gpio0",  "gpio1",  "gpio4",  "gpio5",	"gpio12", "gpio13",
+	"gpio28", "gpio29", "gpio32", "gpio33", "gpio36", "gpio37",
+};
+
+static const char *const jitter_bist_groups[] = {
+	"gpio77",
+};
+
+static const char *const mdp_esync0_out_groups[] = {
+	"gpio13",
+};
+
+static const char *const mdp_esync1_out_groups[] = {
+	"gpio12",
+};
+
+static const char *const mdp_vsync_groups[] = {
+	"gpio16", "gpio17", "gpio79", "gpio100", "gpio120", "gpio121",
+};
+
+static const char *const mdp_vsync0_out_groups[] = {
+	"gpio17",
+};
+
+static const char *const mdp_vsync11_out_groups[] = {
+	"gpio27",
+};
+
+static const char *const mdp_vsync1_out_groups[] = {
+	"gpio17",
+};
+
+static const char *const mdp_vsync2_out_groups[] = {
+	"gpio16",
+};
+
+static const char *const mdp_vsync3_out_groups[] = {
+	"gpio16",
+};
+
+static const char *const mdp_vsync_e_groups[] = {
+	"gpio13",
+};
+
+static const char *const nav_gpio0_groups[] = {
+	"gpio119",
+};
+
+static const char *const nav_gpio1_groups[] = {
+	"gpio117",
+};
+
+static const char *const nav_gpio2_groups[] = {
+	"gpio118",
+};
+
+static const char *const nav_gpio3_groups[] = {
+	"gpio113",
+};
+
+static const char *const pcie0_clk_req_n_groups[] = {
+	"gpio80",
+};
+
+static const char *const pcie1_clk_req_n_groups[] = {
+	"gpio52",
+};
+
+static const char *const phase_flag_groups[] = {
+	"gpio71", "gpio70", "gpio174", "gpio175", "gpio172", "gpio171",
+	"gpio170", "gpio169", "gpio168", "gpio167", "gpio166", "gpio165",
+	"gpio182", "gpio164", "gpio163", "gpio162", "gpio161", "gpio160",
+	"gpio159", "gpio158", "gpio157", "gpio80", "gpio78", "gpio181",
+	"gpio76", "gpio75", "gpio180", "gpio179", "gpio178", "gpio177",
+	"gpio176", "gpio173",
+};
+
+static const char *const pll_bist_sync_groups[] = {
+	"gpio184",
+};
+
+static const char *const pll_clk_aux_groups[] = {
+	"gpio135",
+};
+
+static const char *const prng_rosc0_groups[] = {
+	"gpio67",
+};
+
+static const char *const prng_rosc1_groups[] = {
+	"gpio69",
+};
+
+static const char *const prng_rosc2_groups[] = {
+	"gpio76",
+};
+
+static const char *const prng_rosc3_groups[] = {
+	"gpio74",
+};
+
+static const char *const qdss_cti_groups[] = {
+	"gpio18", "gpio19",  "gpio32",	"gpio73",
+	"gpio74", "gpio154", "gpio176", "gpio184",
+};
+
+static const char *const qdss_gpio_traceclk_groups[] = {
+	"gpio54", "gpio147",
+};
+
+static const char *const qdss_gpio_tracectl_groups[] = {
+	"gpio72", "gpio144",
+};
+
+static const char *const qdss_gpio_tracedata_groups[] = {
+	"gpio30", "gpio31", "gpio34", "gpio35", "gpio40", "gpio42",
+	"gpio52", "gpio53", "gpio65", "gpio66", "gpio67", "gpio114",
+	"gpio132", "gpio133", "gpio134", "gpio135", "gpio145", "gpio146",
+	"gpio155", "gpio156", "gpio163", "gpio164", "gpio167", "gpio168",
+	"gpio169", "gpio170", "gpio178", "gpio179", "gpio180", "gpio181",
+	"gpio182",
+};
+
+static const char *const qlink_big_enable_groups[] = {
+	"gpio96",
+};
+
+static const char *const qlink_big_request_groups[] = {
+	"gpio95",
+};
+
+static const char *const qlink_little_enable_groups[] = {
+	"gpio93",
+};
+
+static const char *const qlink_little_request_groups[] = {
+	"gpio92",
+};
+
+static const char *const qlink_wmss_groups[] = {
+	"gpio94",
+};
+
+static const char *const qspi0_groups[] = {
+	"gpio79", "gpio116", "gpio115", "gpio97", "gpio98",
+};
+
+static const char *const qspi_clk_groups[] = {
+	"gpio99",
+};
+
+static const char *const qspi_cs_groups[] = {
+	"gpio100",
+};
+
+static const char *const qup1_se0_groups[] = {
+	"gpio28", "gpio29", "gpio30", "gpio31",
+};
+
+static const char *const qup1_se1_groups[] = {
+	"gpio32", "gpio33", "gpio34", "gpio35",
+};
+
+static const char *const qup1_se2_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio52", "gpio55", "gpio53", "gpio40", "gpio42", "gpio30",
+};
+
+static const char *const qup1_se3_groups[] = {
+	"gpio44", "gpio45", "gpio46", "gpio47",
+};
+
+static const char *const qup1_se4_groups[] = {
+	"gpio36", "gpio37", "gpio37", "gpio36",
+};
+
+static const char *const qup1_se5_groups[] = {
+	"gpio132", "gpio133", "gpio134", "gpio135", "gpio34", "gpio35",
+};
+
+static const char *const qup1_se6_groups[] = {
+	"gpio40", "gpio42", "gpio54", "gpio42", "gpio40", "gpio55",
+};
+
+static const char *const qup1_se7_groups[] = {
+	"gpio81", "gpio78", "gpio80", "gpio114", "gpio114", "gpio78",
+};
+
+static const char *const qup2_se0_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+
+static const char *const qup2_se1_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7",
+};
+
+static const char *const qup2_se2_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio16", "gpio17", "gpio18",
+};
+
+static const char *const qup2_se3_groups[] = {
+	"gpio79", "gpio116", "gpio97", "gpio100", "gpio100", "gpio116",
+};
+
+static const char *const qup2_se4_groups[] = {
+	"gpio12", "gpio13", "gpio26", "gpio27",
+};
+
+static const char *const qup2_se5_groups[] = {
+	"gpio16", "gpio17", "gpio18", "gpio19",
+};
+
+static const char *const qup2_se6_groups[] = {
+	"gpio20", "gpio21", "gpio22", "gpio23",
+};
+
+static const char *const qup2_se7_groups[] = {
+	"gpio27", "gpio26", "gpio13", "gpio12",
+};
+
+static const char *const resout_gpio_groups[] = {
+	"gpio63",
+	"gpio69",
+	"gpio175",
+};
+
+static const char *const sd_write_protect_groups[] = {
+	"gpio57",
+};
+
+static const char *const sdc1_groups[] = {
+	"gpio121", "gpio123", "gpio124", "gpio125",
+	"gpio126", "gpio127", "gpio128", "gpio129",
+	"gpio130", "gpio131", "gpio120",
+};
+
+static const char *const sdc2_groups[] = {
+	"gpio38", "gpio39", "gpio48", "gpio49",
+	"gpio51", "gpio62",
+};
+
+static const char *const sdc2_fb_clk_groups[] = {
+	"gpio50",
+};
+
+static const char *const tb_trig_sdc1_groups[] = {
+	"gpio34",
+};
+
+static const char *const tb_trig_sdc2_groups[] = {
+	"gpio35",
+};
+
+static const char *const tmess_prng0_groups[] = {
+	"gpio73",
+};
+
+static const char *const tmess_prng1_groups[] = {
+	"gpio72",
+};
+
+static const char *const tmess_prng2_groups[] = {
+	"gpio70",
+};
+
+static const char *const tmess_prng3_groups[] = {
+	"gpio71",
+};
+
+static const char *const tsense_pwm1_groups[] = {
+	"gpio56",
+};
+
+static const char *const tsense_pwm2_groups[] = {
+	"gpio56",
+};
+
+static const char *const tsense_pwm3_groups[] = {
+	"gpio56",
+};
+
+static const char *const tsense_pwm4_groups[] = {
+	"gpio56",
+};
+
+static const char *const uim0_clk_groups[] = {
+	"gpio85",
+};
+
+static const char *const uim0_data_groups[] = {
+	"gpio84",
+};
+
+static const char *const uim0_present_groups[] = {
+	"gpio87",
+};
+
+static const char *const uim0_reset_groups[] = {
+	"gpio86",
+};
+
+static const char *const uim1_clk_groups[] = {
+	"gpio98", "gpio89",
+};
+
+static const char *const uim1_data_groups[] = {
+	"gpio97", "gpio88",
+};
+
+static const char *const uim1_present_groups[] = {
+	"gpio100", "gpio91",
+};
+
+static const char *const uim1_reset_groups[] = {
+	"gpio99", "gpio90",
+};
+
+static const char *const usb0_hs_groups[] = {
+	"gpio56",
+};
+
+static const char *const usb_phy_groups[] = {
+	"gpio122",
+};
+
+static const char *const vfr_0_groups[] = {
+	"gpio63",
+};
+
+static const char *const vfr_1_groups[] = {
+	"gpio117",
+};
+
+static const char *const vsense_trigger_mirnat_groups[] = {
+	"gpio52",
+};
+
+static const char *const wcn_sw_ctrl_groups[] = {
+	"gpio81",
+};
+
+static const struct pinfunction eliza_functions[] = {
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(aoss_cti),
+	MSM_PIN_FUNCTION(atest_char),
+	MSM_PIN_FUNCTION(atest_usb),
+	MSM_PIN_FUNCTION(audio_ext_mclk0),
+	MSM_PIN_FUNCTION(audio_ref_clk),
+	MSM_PIN_FUNCTION(cam_mclk),
+	MSM_PIN_FUNCTION(cci_async_in),
+	MSM_PIN_FUNCTION(cci_i2c_scl),
+	MSM_PIN_FUNCTION(cci_i2c_sda),
+	MSM_PIN_FUNCTION(cci_timer),
+	MSM_PIN_FUNCTION(coex_uart1_rx),
+	MSM_PIN_FUNCTION(coex_uart1_tx),
+	MSM_PIN_FUNCTION(coex_uart2_rx),
+	MSM_PIN_FUNCTION(coex_uart2_tx),
+	MSM_PIN_FUNCTION(dbg_out_clk),
+	MSM_PIN_FUNCTION(ddr_bist_complete),
+	MSM_PIN_FUNCTION(ddr_bist_fail),
+	MSM_PIN_FUNCTION(ddr_bist_start),
+	MSM_PIN_FUNCTION(ddr_bist_stop),
+	MSM_PIN_FUNCTION(ddr_pxi0),
+	MSM_PIN_FUNCTION(ddr_pxi1),
+	MSM_PIN_FUNCTION(dp0_hot),
+	MSM_PIN_FUNCTION(egpio),
+	MSM_PIN_FUNCTION(gcc_gp1),
+	MSM_PIN_FUNCTION(gcc_gp2),
+	MSM_PIN_FUNCTION(gcc_gp3),
+	MSM_PIN_FUNCTION(gnss_adc0),
+	MSM_PIN_FUNCTION(gnss_adc1),
+	MSM_PIN_FUNCTION(hdmi_ddc_scl),
+	MSM_PIN_FUNCTION(hdmi_ddc_sda),
+	MSM_PIN_FUNCTION(hdmi_dtest0),
+	MSM_PIN_FUNCTION(hdmi_dtest1),
+	MSM_PIN_FUNCTION(hdmi_hot_plug),
+	MSM_PIN_FUNCTION(hdmi_pixel_clk),
+	MSM_PIN_FUNCTION(hdmi_rcv_det),
+	MSM_PIN_FUNCTION(hdmi_tx_cec),
+	MSM_PIN_FUNCTION(host2wlan_sol),
+	MSM_PIN_FUNCTION(i2s0_data0),
+	MSM_PIN_FUNCTION(i2s0_data1),
+	MSM_PIN_FUNCTION(i2s0_sck),
+	MSM_PIN_FUNCTION(i2s0_ws),
+	MSM_PIN_FUNCTION(ibi_i3c),
+	MSM_PIN_FUNCTION(jitter_bist),
+	MSM_PIN_FUNCTION(mdp_esync0_out),
+	MSM_PIN_FUNCTION(mdp_esync1_out),
+	MSM_PIN_FUNCTION(mdp_vsync),
+	MSM_PIN_FUNCTION(mdp_vsync0_out),
+	MSM_PIN_FUNCTION(mdp_vsync11_out),
+	MSM_PIN_FUNCTION(mdp_vsync1_out),
+	MSM_PIN_FUNCTION(mdp_vsync2_out),
+	MSM_PIN_FUNCTION(mdp_vsync3_out),
+	MSM_PIN_FUNCTION(mdp_vsync_e),
+	MSM_PIN_FUNCTION(nav_gpio0),
+	MSM_PIN_FUNCTION(nav_gpio1),
+	MSM_PIN_FUNCTION(nav_gpio2),
+	MSM_PIN_FUNCTION(nav_gpio3),
+	MSM_PIN_FUNCTION(pcie0_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_clk_req_n),
+	MSM_PIN_FUNCTION(phase_flag),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_clk_aux),
+	MSM_PIN_FUNCTION(prng_rosc0),
+	MSM_PIN_FUNCTION(prng_rosc1),
+	MSM_PIN_FUNCTION(prng_rosc2),
+	MSM_PIN_FUNCTION(prng_rosc3),
+	MSM_PIN_FUNCTION(qdss_cti),
+	MSM_PIN_FUNCTION(qdss_gpio_traceclk),
+	MSM_PIN_FUNCTION(qdss_gpio_tracectl),
+	MSM_PIN_FUNCTION(qdss_gpio_tracedata),
+	MSM_PIN_FUNCTION(qlink_big_enable),
+	MSM_PIN_FUNCTION(qlink_big_request),
+	MSM_PIN_FUNCTION(qlink_little_enable),
+	MSM_PIN_FUNCTION(qlink_little_request),
+	MSM_PIN_FUNCTION(qlink_wmss),
+	MSM_PIN_FUNCTION(qspi0),
+	MSM_PIN_FUNCTION(qspi_clk),
+	MSM_PIN_FUNCTION(qspi_cs),
+	MSM_PIN_FUNCTION(qup1_se0),
+	MSM_PIN_FUNCTION(qup1_se1),
+	MSM_PIN_FUNCTION(qup1_se2),
+	MSM_PIN_FUNCTION(qup1_se3),
+	MSM_PIN_FUNCTION(qup1_se4),
+	MSM_PIN_FUNCTION(qup1_se5),
+	MSM_PIN_FUNCTION(qup1_se6),
+	MSM_PIN_FUNCTION(qup1_se7),
+	MSM_PIN_FUNCTION(qup2_se0),
+	MSM_PIN_FUNCTION(qup2_se1),
+	MSM_PIN_FUNCTION(qup2_se2),
+	MSM_PIN_FUNCTION(qup2_se3),
+	MSM_PIN_FUNCTION(qup2_se4),
+	MSM_PIN_FUNCTION(qup2_se5),
+	MSM_PIN_FUNCTION(qup2_se6),
+	MSM_PIN_FUNCTION(qup2_se7),
+	MSM_PIN_FUNCTION(resout_gpio),
+	MSM_PIN_FUNCTION(sd_write_protect),
+	MSM_PIN_FUNCTION(sdc1),
+	MSM_PIN_FUNCTION(sdc2),
+	MSM_PIN_FUNCTION(sdc2_fb_clk),
+	MSM_PIN_FUNCTION(tb_trig_sdc1),
+	MSM_PIN_FUNCTION(tb_trig_sdc2),
+	MSM_PIN_FUNCTION(tmess_prng0),
+	MSM_PIN_FUNCTION(tmess_prng1),
+	MSM_PIN_FUNCTION(tmess_prng2),
+	MSM_PIN_FUNCTION(tmess_prng3),
+	MSM_PIN_FUNCTION(tsense_pwm1),
+	MSM_PIN_FUNCTION(tsense_pwm2),
+	MSM_PIN_FUNCTION(tsense_pwm3),
+	MSM_PIN_FUNCTION(tsense_pwm4),
+	MSM_PIN_FUNCTION(uim0_clk),
+	MSM_PIN_FUNCTION(uim0_data),
+	MSM_PIN_FUNCTION(uim0_present),
+	MSM_PIN_FUNCTION(uim0_reset),
+	MSM_PIN_FUNCTION(uim1_clk),
+	MSM_PIN_FUNCTION(uim1_data),
+	MSM_PIN_FUNCTION(uim1_present),
+	MSM_PIN_FUNCTION(uim1_reset),
+	MSM_PIN_FUNCTION(usb0_hs),
+	MSM_PIN_FUNCTION(usb_phy),
+	MSM_PIN_FUNCTION(vfr_0),
+	MSM_PIN_FUNCTION(vfr_1),
+	MSM_PIN_FUNCTION(vsense_trigger_mirnat),
+	MSM_PIN_FUNCTION(wcn_sw_ctrl),
+};
+
+/* Every pin is maintained as a single group, and missing or non-existing pin
+ * would be maintained as dummy group to synchronize pin group index with
+ * pin descriptor registered with pinctrl core.
+ * Clients would not be able to request these dummy pin groups.
+ */
+static const struct msm_pingroup eliza_groups[] = {
+	[0] = PINGROUP(0, qup2_se0, ibi_i3c, aoss_cti, _, _, _, _, _, _, _, _),
+	[1] = PINGROUP(1, qup2_se0, ibi_i3c, aoss_cti, _, _, _, _, _, _, _, _),
+	[2] = PINGROUP(2, qup2_se0, _, _, _, _, _, _, _, _, _, _),
+	[3] = PINGROUP(3, qup2_se0, _, _, _, _, _, _, _, _, _, _),
+	[4] = PINGROUP(4, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[5] = PINGROUP(5, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[6] = PINGROUP(6, qup2_se1, hdmi_ddc_scl, _, _, _, _, _, _, _, _, _),
+	[7] = PINGROUP(7, qup2_se1, hdmi_ddc_sda, _, _, _, _, _, _, _, _, _),
+	[8] = PINGROUP(8, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[9] = PINGROUP(9, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[10] = PINGROUP(10, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[12] = PINGROUP(12, qup2_se4, ibi_i3c, mdp_esync1_out, qup2_se7, _, _, _, _, _, _, _),
+	[13] = PINGROUP(13, qup2_se4, ibi_i3c, mdp_vsync_e, mdp_esync0_out, qup2_se7, _, _, _, _, _, _),
+	[14] = PINGROUP(14, _, _, _, _, _, _, _, _, _, _, _),
+	[15] = PINGROUP(15, _, _, _, _, _, _, _, _, _, _, _),
+	[16] = PINGROUP(16, qup2_se5, qup2_se2, mdp_vsync, mdp_vsync2_out, mdp_vsync3_out, _, _, _, _, _, _),
+	[17] = PINGROUP(17, qup2_se5, qup2_se2, mdp_vsync, mdp_vsync0_out, mdp_vsync1_out, _, _, _, _, _, _),
+	[18] = PINGROUP(18, qup2_se5, qup2_se2, hdmi_pixel_clk, _, qdss_cti, _, _, _, _, _, _),
+	[19] = PINGROUP(19, qup2_se5, hdmi_rcv_det, _, qdss_cti, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, qup2_se6, _, _, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, qup2_se6, _, _, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, qup2_se6, _, _, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, qup2_se6, _, _, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, _, _, _, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, _, _, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, qup2_se4, aoss_cti, qup2_se7, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, qup2_se4, aoss_cti, mdp_vsync11_out, qup2_se7, gcc_gp1, _, _, _, _, _, _),
+	[28] = PINGROUP(28, qup1_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[29] = PINGROUP(29, qup1_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[30] = PINGROUP(30, qup1_se0, qup1_se2, cci_async_in, gcc_gp3, qdss_gpio_tracedata, _, _, _, _, _, egpio),
+	[31] = PINGROUP(31, qup1_se0, cci_async_in, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[32] = PINGROUP(32, qup1_se1, ibi_i3c, audio_ref_clk, gcc_gp2, qdss_cti, _, _, _, _, _, _),
+	[33] = PINGROUP(33, qup1_se1, ibi_i3c, host2wlan_sol, gcc_gp3, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, qup1_se1, qup1_se5, tb_trig_sdc1, ddr_bist_start, qdss_gpio_tracedata, _, _, _, _, _, _),
+	[35] = PINGROUP(35, qup1_se1, qup1_se5, tb_trig_sdc2, gcc_gp2, qdss_gpio_tracedata, _, _, _, _, _, _),
+	[36] = PINGROUP(36, qup1_se4, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, qup1_se4, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, _, _, _, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, _, _, _, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, qup1_se6, qup1_se2, qup1_se6, _, qdss_gpio_tracedata, gnss_adc1, ddr_pxi1, _, _, _, _),
+	[41] = PINGROUP(41, _, _, _, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, qup1_se6, qup1_se2, qup1_se6, qdss_gpio_tracedata, gnss_adc0, ddr_pxi1, _, _, _, _, _),
+	[43] = PINGROUP(43, _, _, _, _, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, qup1_se3, _, _, _, _, _, _, _, _, _, _),
+	[45] = PINGROUP(45, qup1_se3, _, _, _, _, _, _, _, _, _, _),
+	[46] = PINGROUP(46, qup1_se3, hdmi_tx_cec, _, _, _, _, _, _, _, _, _),
+	[47] = PINGROUP(47, qup1_se3, hdmi_hot_plug, _, _, _, _, _, _, _, _, _),
+	[48] = PINGROUP(48, _, _, _, _, _, _, _, _, _, _, _),
+	[49] = PINGROUP(49, _, _, _, _, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, sdc2_fb_clk, _, _, _, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, _, _, _, _, _, _, _, _, _, _, _),
+	[52] = PINGROUP(52, qup1_se2, pcie1_clk_req_n, qup1_se2, ddr_bist_complete, qdss_gpio_tracedata, _, vsense_trigger_mirnat, _, _, _, _),
+	[53] = PINGROUP(53, qup1_se2, qup1_se2, gcc_gp1, ddr_bist_stop, _, qdss_gpio_tracedata, _, _, _, _, _),
+	[54] = PINGROUP(54, qup1_se2, qup1_se6, qdss_gpio_tracedata, gnss_adc1, atest_usb, ddr_pxi0, _, _, _, _, _),
+	[55] = PINGROUP(55, qup1_se2, dp0_hot, qup1_se6, _, gnss_adc0, atest_usb, ddr_pxi0, _, _, _, _),
+	[56] = PINGROUP(56, usb0_hs, tsense_pwm1, tsense_pwm2, tsense_pwm3, tsense_pwm4, _, _, _, _, _, _),
+	[57] = PINGROUP(57, sd_write_protect, _, _, _, _, _, _, _, _, _, _),
+	[58] = PINGROUP(58, _, _, _, _, _, _, _, _, _, _, _),
+	[59] = PINGROUP(59, _, _, _, _, _, _, _, _, _, _, _),
+	[60] = PINGROUP(60, i2s0_sck, _, _, _, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, i2s0_ws, _, _, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, _, _, _, _, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, resout_gpio, i2s0_data1, cci_timer, vfr_0, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, i2s0_data0, _, _, _, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, cam_mclk, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _),
+	[66] = PINGROUP(66, cam_mclk, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _),
+	[67] = PINGROUP(67, cam_mclk, prng_rosc0, _, qdss_gpio_tracedata, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, cam_mclk, audio_ext_mclk0, resout_gpio, prng_rosc1, _, _, _, _, _, _, _),
+	[70] = PINGROUP(70, cci_i2c_sda, tmess_prng2, _, phase_flag, atest_char, _, _, _, _, _, _),
+	[71] = PINGROUP(71, cci_i2c_scl, tmess_prng3, _, phase_flag, atest_char, _, _, _, _, _, _),
+	[72] = PINGROUP(72, cci_i2c_sda, tmess_prng1, qdss_gpio_tracedata, atest_char, _, _, _, _, _, _, _),
+	[73] = PINGROUP(73, cci_i2c_scl, tmess_prng0, qdss_cti, atest_char, _, _, _, _, _, _, _),
+	[74] = PINGROUP(74, cci_i2c_sda, prng_rosc3, qdss_cti, atest_char, _, _, _, _, _, _, _),
+	[75] = PINGROUP(75, cci_i2c_scl, _, phase_flag, _, _, _, _, _, _, _, _),
+	[76] = PINGROUP(76, cci_i2c_sda, cci_timer, prng_rosc2, _, phase_flag, _, _, _, _, _, _),
+	[77] = PINGROUP(77, cci_i2c_scl, jitter_bist, _, _, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, qup1_se7, qup1_se7, _, phase_flag, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, qspi0, mdp_vsync, qup2_se3, _, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, pcie0_clk_req_n, qup1_se7, _, phase_flag, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, wcn_sw_ctrl, qup1_se7, dbg_out_clk, _, _, _, _, _, _, _, _),
+	[82] = PINGROUP(82, _, _, _, _, _, _, _, _, _, _, _),
+	[83] = PINGROUP(83, _, _, _, _, _, _, _, _, _, _, _),
+	[84] = PINGROUP(84, uim0_data, _, _, _, _, _, _, _, _, _, _),
+	[85] = PINGROUP(85, uim0_clk, _, _, _, _, _, _, _, _, _, _),
+	[86] = PINGROUP(86, uim0_reset, _, _, _, _, _, _, _, _, _, _),
+	[87] = PINGROUP(87, uim0_present, _, _, _, _, _, _, _, _, _, _),
+	[88] = PINGROUP(88, uim1_data, _, _, _, _, _, _, _, _, _, _),
+	[89] = PINGROUP(89, uim1_clk, _, _, _, _, _, _, _, _, _, _),
+	[90] = PINGROUP(90, uim1_reset, _, _, _, _, _, _, _, _, _, _),
+	[91] = PINGROUP(91, uim1_present, _, _, _, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, qlink_little_request, _, _, _, _, _, _, _, _, _, _),
+	[93] = PINGROUP(93, qlink_little_enable, _, _, _, _, _, _, _, _, _, _),
+	[94] = PINGROUP(94, qlink_wmss, _, _, _, _, _, _, _, _, _, _),
+	[95] = PINGROUP(95, qlink_big_request, _, _, _, _, _, _, _, _, _, _),
+	[96] = PINGROUP(96, qlink_big_enable, _, _, _, _, _, _, _, _, _, _),
+	[97] = PINGROUP(97, uim1_data, qspi0, qup2_se3, _, _, _, _, _, _, _, _),
+	[98] = PINGROUP(98, uim1_clk, qspi0, _, _, _, _, _, _, _, _, _),
+	[99] = PINGROUP(99, uim1_reset, qspi0, _, _, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, uim1_present, qspi0, qup2_se3, coex_uart2_tx, qup2_se3, mdp_vsync, _, _, _, _, _),
+	[101] = PINGROUP(101, _, _, _, _, _, _, _, _, _, _, _),
+	[102] = PINGROUP(102, _, _, _, _, _, _, _, _, _, _, _),
+	[103] = PINGROUP(103, _, _, _, _, _, _, _, _, _, _, _),
+	[104] = PINGROUP(104, _, _, _, _, _, _, _, _, _, _, _),
+	[105] = PINGROUP(105, _, _, _, _, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, _, _, _, _, _, _, _, _, _, _, _),
+	[107] = PINGROUP(107, _, _, _, _, _, _, _, _, _, _, _),
+	[108] = PINGROUP(108, _, _, _, _, _, _, _, _, _, _, _),
+	[109] = PINGROUP(109, _, _, _, _, _, _, _, _, _, _, _),
+	[110] = PINGROUP(110, _, _, _, _, _, _, _, _, _, _, _),
+	[111] = PINGROUP(111, coex_uart1_tx, _, _, _, _, _, _, _, _, _, _),
+	[112] = PINGROUP(112, coex_uart1_rx, _, _, _, _, _, _, _, _, _, _),
+	[113] = PINGROUP(113, _, nav_gpio3, _, _, _, _, _, _, _, _, _),
+	[114] = PINGROUP(114, qup1_se7, qup1_se7, _, qdss_gpio_tracedata, _, _, _, _, _, _, _),
+	[115] = PINGROUP(115, _, qspi0, cci_async_in, _, _, _, _, _, _, _, _),
+	[116] = PINGROUP(116, qspi0, coex_uart2_rx, qup2_se3, qup2_se3, _, _, _, _, _, _, _),
+	[117] = PINGROUP(117, nav_gpio1, _, vfr_1, _, _, _, _, _, _, _, _),
+	[118] = PINGROUP(118, nav_gpio2, _, _, _, _, _, _, _, _, _, _),
+	[119] = PINGROUP(119, nav_gpio0, _, _, _, _, _, _, _, _, _, _),
+	[120] = PINGROUP(120, sdc1, mdp_vsync, _, _, _, _, _, _, _, _, _),
+	[121] = PINGROUP(121, sdc1, mdp_vsync, _, _, _, _, _, _, _, _, _),
+	[122] = PINGROUP(122, usb_phy, _, _, _, _, _, _, _, _, _, _),
+	[123] = PINGROUP(123, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[124] = PINGROUP(124, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[125] = PINGROUP(125, sdc1, cci_timer, _, _, _, _, _, _, _, _, _),
+	[126] = PINGROUP(126, sdc1, cci_timer, _, _, _, _, _, _, _, _, _),
+	[127] = PINGROUP(127, sdc1, cci_timer, _, _, _, _, _, _, _, _, _),
+	[128] = PINGROUP(128, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[129] = PINGROUP(129, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[130] = PINGROUP(130, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[131] = PINGROUP(131, sdc1, _, _, _, _, _, _, _, _, _, _),
+	[132] = PINGROUP(132, qup1_se5, _, qdss_gpio_tracedata, hdmi_dtest0, _, _, _, _, _, _, _),
+	[133] = PINGROUP(133, qup1_se5, _, qdss_gpio_tracedata, hdmi_dtest1, _, _, _, _, _, _, _),
+	[134] = PINGROUP(134, qup1_se5, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _),
+	[135] = PINGROUP(135, qup1_se5, _, pll_clk_aux, qdss_gpio_tracedata, _, _, _, _, _, _, _),
+	[136] = PINGROUP(136, _, _, _, _, _, _, _, _, _, _, _),
+	[137] = PINGROUP(137, _, _, _, _, _, _, _, _, _, _, _),
+	[138] = PINGROUP(138, _, _, _, _, _, _, _, _, _, _, egpio),
+	[139] = PINGROUP(139, _, _, _, _, _, _, _, _, _, _, egpio),
+	[140] = PINGROUP(140, _, _, _, _, _, _, _, _, _, _, egpio),
+	[141] = PINGROUP(141, _, _, _, _, _, _, _, _, _, _, egpio),
+	[142] = PINGROUP(142, _, _, _, _, _, _, _, _, _, _, egpio),
+	[143] = PINGROUP(143, _, _, _, _, _, _, _, _, _, _, egpio),
+	[144] = PINGROUP(144, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+	[145] = PINGROUP(145, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _, egpio),
+	[146] = PINGROUP(146, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+	[147] = PINGROUP(147, ddr_bist_fail, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[148] = PINGROUP(148, _, _, _, _, _, _, _, _, _, _, egpio),
+	[149] = PINGROUP(149, _, _, _, _, _, _, _, _, _, _, egpio),
+	[150] = PINGROUP(150, _, _, _, _, _, _, _, _, _, _, egpio),
+	[151] = PINGROUP(151, _, _, _, _, _, _, _, _, _, _, egpio),
+	[152] = PINGROUP(152, _, _, _, _, _, _, _, _, _, _, egpio),
+	[153] = PINGROUP(153, _, _, _, _, _, _, _, _, _, _, egpio),
+	[154] = PINGROUP(154, qdss_cti, _, _, _, _, _, _, _, _, _, egpio),
+	[155] = PINGROUP(155, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+	[156] = PINGROUP(156, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+	[157] = PINGROUP(157, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[158] = PINGROUP(158, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[159] = PINGROUP(159, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[160] = PINGROUP(160, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[161] = PINGROUP(161, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[162] = PINGROUP(162, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[163] = PINGROUP(163, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[164] = PINGROUP(164, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[165] = PINGROUP(165, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[166] = PINGROUP(166, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[167] = PINGROUP(167, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[168] = PINGROUP(168, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[169] = PINGROUP(169, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[170] = PINGROUP(170, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[171] = PINGROUP(171, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[172] = PINGROUP(172, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[173] = PINGROUP(173, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[174] = PINGROUP(174, _, phase_flag, _, _, _, _, _, _, _, _, egpio),
+	[175] = PINGROUP(175, resout_gpio, _, phase_flag, _, _, _, _, _, _, _, egpio),
+	[176] = PINGROUP(176, _, phase_flag, qdss_cti, _, _, _, _, _, _, _, egpio),
+	[177] = PINGROUP(177, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[178] = PINGROUP(178, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[179] = PINGROUP(179, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[180] = PINGROUP(180, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[181] = PINGROUP(181, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[182] = PINGROUP(182, _, phase_flag, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[183] = PINGROUP(183, _, _, _, _, _, _, _, _, _, _, _),
+	[184] = PINGROUP(184, pll_bist_sync, qdss_cti, _, _, _, _, _, _, _, _, egpio),
+	[185] = UFS_RESET(ufs_reset, 0xc9004, 0xca000),
+};
+
+static const struct msm_gpio_wakeirq_map eliza_pdc_map[] = {
+	{ 0, 82 },    { 3, 87 },    { 4, 90 },	  { 6, 68 },	{ 7, 153 },
+	{ 11, 85 },   { 12, 107 },  { 13, 106 },  { 16, 88 },	{ 17, 70 },
+	{ 18, 134 },  { 19, 79 },   { 23, 80 },	  { 26, 91 },	{ 27, 74 },
+	{ 28, 137 },  { 29, 138 },  { 30, 139 },  { 31, 140 },	{ 32, 117 },
+	{ 34, 100 },  { 35, 98 },   { 36, 141 },  { 39, 89 },	{ 40, 142 },
+	{ 42, 143 },  { 44, 101 },  { 45, 144 },  { 46, 145 },	{ 47, 146 },
+	{ 49, 75 },   { 51, 147 },  { 52, 148 },  { 53, 149 },	{ 54, 150 },
+	{ 55, 151 },  { 56, 152 },  { 58, 71 },	  { 59, 155 },	{ 63, 99 },
+	{ 78, 156 },  { 79, 76 },   { 80, 157 },  { 81, 69 },	{ 87, 158 },
+	{ 91, 67 },   { 92, 159 },  { 95, 160 },  { 98, 161 },	{ 99, 162 },
+	{ 100, 83 },  { 108, 154 }, { 109, 84 },  { 112, 86 },	{ 113, 92 },
+	{ 114, 93 },  { 115, 110 }, { 116, 94 },  { 117, 77 },	{ 118, 108 },
+	{ 119, 95 },  { 120, 81 },  { 121, 96 },  { 122, 97 },	{ 123, 102 },
+	{ 125, 103 }, { 127, 104 }, { 128, 105 }, { 129, 78 },	{ 130, 112 },
+	{ 131, 113 }, { 133, 114 }, { 135, 115 }, { 139, 116 }, { 142, 118 },
+	{ 145, 109 }, { 147, 72 },  { 149, 111 }, { 154, 122 }, { 157, 119 },
+	{ 159, 120 }, { 161, 121 }, { 164, 123 }, { 165, 124 }, { 167, 125 },
+	{ 170, 126 }, { 171, 73 },  { 172, 127 }, { 173, 128 }, { 174, 129 },
+	{ 175, 130 }, { 176, 131 }, { 177, 132 }, { 179, 133 }, { 182, 135 },
+	{ 184, 136 },
+};
+
+static const struct msm_pinctrl_soc_data eliza_tlmm = {
+	.pins = eliza_pins,
+	.npins = ARRAY_SIZE(eliza_pins),
+	.functions = eliza_functions,
+	.nfunctions = ARRAY_SIZE(eliza_functions),
+	.groups = eliza_groups,
+	.ngroups = ARRAY_SIZE(eliza_groups),
+	.ngpios = 186,
+	.wakeirq_map = eliza_pdc_map,
+	.nwakeirq_map = ARRAY_SIZE(eliza_pdc_map),
+	.egpio_func = 11,
+};
+
+static int eliza_tlmm_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &eliza_tlmm);
+}
+
+static const struct of_device_id eliza_tlmm_of_match[] = {
+	{ .compatible = "qcom,eliza-tlmm", },
+	{},
+};
+
+static struct platform_driver eliza_tlmm_driver = {
+	.driver = {
+		.name = "eliza-tlmm",
+		.of_match_table = eliza_tlmm_of_match,
+	},
+	.probe = eliza_tlmm_probe,
+};
+
+static int __init eliza_tlmm_init(void)
+{
+	return platform_driver_register(&eliza_tlmm_driver);
+}
+arch_initcall(eliza_tlmm_init);
+
+static void __exit eliza_tlmm_exit(void)
+{
+	platform_driver_unregister(&eliza_tlmm_driver);
+}
+module_exit(eliza_tlmm_exit);
+
+MODULE_DESCRIPTION("QTI Eliza TLMM driver");
+MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(of, eliza_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-glymur.c b/drivers/pinctrl/qcom/pinctrl-glymur.c
index 44f9745325b76..2da3b513d31b3 100644
--- a/drivers/pinctrl/qcom/pinctrl-glymur.c
+++ b/drivers/pinctrl/qcom/pinctrl-glymur.c
@@ -1812,6 +1812,6 @@ static void __exit glymur_tlmm_exit(void)
 }
 module_exit(glymur_tlmm_exit);
 
-MODULE_DESCRIPTION("QTI GLYMUR TLMM driver");
+MODULE_DESCRIPTION("QTI Glymur TLMM driver");
 MODULE_LICENSE("GPL");
 MODULE_DEVICE_TABLE(of, glymur_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
new file mode 100644
index 0000000000000..3bf6fe0cf1bbe
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022-2023 Linaro Ltd.
+ * Copyright (c) 2026 Luca Weiss <luca.weiss@fairphone.com>
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+	LPI_MUX_dmic1_clk,
+	LPI_MUX_dmic1_data,
+	LPI_MUX_dmic2_clk,
+	LPI_MUX_dmic2_data,
+	LPI_MUX_dmic3_clk,
+	LPI_MUX_dmic3_data,
+	LPI_MUX_dmic4_clk,
+	LPI_MUX_dmic4_data,
+	LPI_MUX_i2s0_clk,
+	LPI_MUX_i2s0_data,
+	LPI_MUX_i2s0_ws,
+	LPI_MUX_i2s1_clk,
+	LPI_MUX_i2s1_data,
+	LPI_MUX_i2s1_ws,
+	LPI_MUX_i2s2_clk,
+	LPI_MUX_i2s2_data,
+	LPI_MUX_i2s2_ws,
+	LPI_MUX_i2s3_clk,
+	LPI_MUX_i2s3_data,
+	LPI_MUX_i2s3_ws,
+	LPI_MUX_qca_swr_clk,
+	LPI_MUX_qca_swr_data,
+	LPI_MUX_slimbus_clk,
+	LPI_MUX_slimbus_data,
+	LPI_MUX_swr_rx_clk,
+	LPI_MUX_swr_rx_data,
+	LPI_MUX_swr_tx_clk,
+	LPI_MUX_swr_tx_data,
+	LPI_MUX_wsa_swr_clk,
+	LPI_MUX_wsa_swr_data,
+	LPI_MUX_ext_mclk1_a,
+	LPI_MUX_ext_mclk1_b,
+	LPI_MUX_ext_mclk1_c,
+	LPI_MUX_ext_mclk1_d,
+	LPI_MUX_ext_mclk1_e,
+	LPI_MUX_gpio,
+	LPI_MUX__,
+};
+
+static const struct pinctrl_pin_desc milos_lpi_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+	PINCTRL_PIN(15, "gpio15"),
+	PINCTRL_PIN(16, "gpio16"),
+	PINCTRL_PIN(17, "gpio17"),
+	PINCTRL_PIN(18, "gpio18"),
+	PINCTRL_PIN(19, "gpio19"),
+	PINCTRL_PIN(20, "gpio20"),
+	PINCTRL_PIN(21, "gpio21"),
+	PINCTRL_PIN(22, "gpio22"),
+};
+
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22",
+};
+
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const dmic4_clk_groups[] = { "gpio21" };
+static const char * const dmic4_data_groups[] = { "gpio22" };
+static const char * const i2s0_clk_groups[] = { "gpio0" };
+static const char * const i2s0_ws_groups[] = { "gpio1" };
+static const char * const i2s0_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const i2s1_clk_groups[] = { "gpio6" };
+static const char * const i2s1_ws_groups[] = { "gpio7" };
+static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" };
+static const char * const i2s2_clk_groups[] = { "gpio10" };
+static const char * const i2s2_ws_groups[] = { "gpio11" };
+static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" };
+static const char * const i2s3_clk_groups[] = { "gpio19" };
+static const char * const i2s3_ws_groups[] = { "gpio20" };
+static const char * const i2s3_data_groups[] = { "gpio21", "gpio22" };
+static const char * const qca_swr_clk_groups[] = { "gpio19" };
+static const char * const qca_swr_data_groups[] = { "gpio20" };
+static const char * const slimbus_clk_groups[] = { "gpio19" };
+static const char * const slimbus_data_groups[] = { "gpio20" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" };
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_data_groups[] = { "gpio11" };
+static const char * const ext_mclk1_a_groups[] = { "gpio13" };
+static const char * const ext_mclk1_b_groups[] = { "gpio9" };
+static const char * const ext_mclk1_c_groups[] = { "gpio5" };
+static const char * const ext_mclk1_d_groups[] = { "gpio14" };
+static const char * const ext_mclk1_e_groups[] = { "gpio22" };
+
+static const struct lpi_pingroup milos_groups[] = {
+	LPI_PINGROUP(0, 0, swr_tx_clk, i2s0_clk, _, _),
+	LPI_PINGROUP(1, 2, swr_tx_data, i2s0_ws, _, _),
+	LPI_PINGROUP(2, 4, swr_tx_data, i2s0_data, _, _),
+	LPI_PINGROUP(3, 8, swr_rx_clk, i2s0_data, _, _),
+	LPI_PINGROUP(4, 10, swr_rx_data, i2s0_data, _, _),
+	LPI_PINGROUP(5, 12, swr_rx_data, ext_mclk1_c, i2s0_data, _),
+	LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _),
+	LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+	LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+	LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _),
+	LPI_PINGROUP(10, 16, wsa_swr_clk, i2s2_clk, _, _),
+	LPI_PINGROUP(11, 18, wsa_swr_data, i2s2_ws, _, _),
+	LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s2_data, _, _),
+	LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s2_data, ext_mclk1_a, _),
+	LPI_PINGROUP(14, 6, swr_tx_data, ext_mclk1_d, _, _),
+	/* gpio15 - gpio18 do not really exist */
+	LPI_PINGROUP(15, 20, _, _, _, _),
+	LPI_PINGROUP(16, 22, _, _, _, _),
+	LPI_PINGROUP(17, LPI_NO_SLEW, _, _, _, _),
+	LPI_PINGROUP(18, LPI_NO_SLEW, _, _, _, _),
+	LPI_PINGROUP(19, LPI_NO_SLEW, i2s3_clk, slimbus_clk, qca_swr_clk, _),
+	LPI_PINGROUP(20, LPI_NO_SLEW, i2s3_ws, slimbus_data, qca_swr_data, _),
+	LPI_PINGROUP(21, LPI_NO_SLEW, i2s3_data, dmic4_clk, _, _),
+	LPI_PINGROUP(22, LPI_NO_SLEW, i2s3_data, dmic4_data, ext_mclk1_e, _),
+};
+
+static const struct lpi_function milos_functions[] = {
+	LPI_FUNCTION(gpio),
+	LPI_FUNCTION(dmic1_clk),
+	LPI_FUNCTION(dmic1_data),
+	LPI_FUNCTION(dmic2_clk),
+	LPI_FUNCTION(dmic2_data),
+	LPI_FUNCTION(dmic3_clk),
+	LPI_FUNCTION(dmic3_data),
+	LPI_FUNCTION(dmic4_clk),
+	LPI_FUNCTION(dmic4_data),
+	LPI_FUNCTION(i2s0_clk),
+	LPI_FUNCTION(i2s0_data),
+	LPI_FUNCTION(i2s0_ws),
+	LPI_FUNCTION(i2s1_clk),
+	LPI_FUNCTION(i2s1_data),
+	LPI_FUNCTION(i2s1_ws),
+	LPI_FUNCTION(i2s2_clk),
+	LPI_FUNCTION(i2s2_data),
+	LPI_FUNCTION(i2s2_ws),
+	LPI_FUNCTION(i2s3_clk),
+	LPI_FUNCTION(i2s3_data),
+	LPI_FUNCTION(i2s3_ws),
+	LPI_FUNCTION(qca_swr_clk),
+	LPI_FUNCTION(qca_swr_data),
+	LPI_FUNCTION(slimbus_clk),
+	LPI_FUNCTION(slimbus_data),
+	LPI_FUNCTION(swr_rx_clk),
+	LPI_FUNCTION(swr_rx_data),
+	LPI_FUNCTION(swr_tx_clk),
+	LPI_FUNCTION(swr_tx_data),
+	LPI_FUNCTION(wsa_swr_clk),
+	LPI_FUNCTION(wsa_swr_data),
+	LPI_FUNCTION(ext_mclk1_a),
+	LPI_FUNCTION(ext_mclk1_b),
+	LPI_FUNCTION(ext_mclk1_c),
+	LPI_FUNCTION(ext_mclk1_d),
+	LPI_FUNCTION(ext_mclk1_e),
+};
+
+static const struct lpi_pinctrl_variant_data milos_lpi_data = {
+	.pins = milos_lpi_pins,
+	.npins = ARRAY_SIZE(milos_lpi_pins),
+	.groups = milos_groups,
+	.ngroups = ARRAY_SIZE(milos_groups),
+	.functions = milos_functions,
+	.nfunctions = ARRAY_SIZE(milos_functions),
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+	{
+		.compatible = "qcom,milos-lpass-lpi-pinctrl",
+		.data = &milos_lpi_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static struct platform_driver lpi_pinctrl_driver = {
+	.driver = {
+		.name = "qcom-milos-lpass-lpi-pinctrl",
+		.of_match_table = lpi_pinctrl_of_match,
+	},
+	.probe = lpi_pinctrl_probe,
+	.remove = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("Qualcomm Milos LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index 83f940fe30b26..d02d42513ebbc 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -723,6 +723,21 @@ static const struct pinconf_ops pmic_gpio_pinconf_ops = {
 	.pin_config_group_dbg_show	= pmic_gpio_config_dbg_show,
 };
 
+static int pmic_gpio_get_direction(struct gpio_chip *chip, unsigned pin)
+{
+	struct pmic_gpio_state *state = gpiochip_get_data(chip);
+	struct pmic_gpio_pad *pad;
+
+	pad = state->ctrl->desc->pins[pin].drv_data;
+
+	if (!pad->is_enabled || pad->analog_pass ||
+	    (!pad->input_enabled && !pad->output_enabled))
+		return -EINVAL;
+
+	/* Make sure the state is aligned on what pmic_gpio_get() returns */
+	return pad->input_enabled ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT;
+}
+
 static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 {
 	struct pmic_gpio_state *state = gpiochip_get_data(chip);
@@ -801,6 +816,7 @@ static void pmic_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 }
 
 static const struct gpio_chip pmic_gpio_gpio_template = {
+	.get_direction		= pmic_gpio_get_direction,
 	.direction_input	= pmic_gpio_direction_input,
 	.direction_output	= pmic_gpio_direction_output,
 	.get			= pmic_gpio_get,
diff --git a/drivers/pinctrl/realtek/pinctrl-rtd.c b/drivers/pinctrl/realtek/pinctrl-rtd.c
index 2440604863327..7836a15afe446 100644
--- a/drivers/pinctrl/realtek/pinctrl-rtd.c
+++ b/drivers/pinctrl/realtek/pinctrl-rtd.c
@@ -30,6 +30,7 @@ struct rtd_pinctrl {
 	struct pinctrl_desc desc;
 	const struct rtd_pinctrl_desc *info;
 	struct regmap *regmap_pinctrl;
+	u32 **saved_regs;
 };
 
 /* custom pinconf parameters */
@@ -293,14 +294,14 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	config_desc = rtd_pinctrl_find_config(data, pinnr);
 	if (!config_desc) {
-		dev_err(data->dev, "Not support pin config for pin: %s\n", name);
+		dev_err(data->dev, "Pin config unsupported for pin: %s\n", name);
 		return -ENOTSUPP;
 	}
 	switch ((u32)param) {
 	case PIN_CONFIG_INPUT_SCHMITT:
 	case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
 		if (config_desc->smt_offset == NA) {
-			dev_err(data->dev, "Not support input schmitt for pin: %s\n", name);
+			dev_err(data->dev, "Input schmitt unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		smt_off = config_desc->base_bit + config_desc->smt_offset;
@@ -313,7 +314,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	case PIN_CONFIG_DRIVE_PUSH_PULL:
 		if (config_desc->pud_en_offset == NA) {
-			dev_err(data->dev, "Not support push pull for pin: %s\n", name);
+			dev_err(data->dev, "Push pull unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		pulen_off = config_desc->base_bit + config_desc->pud_en_offset;
@@ -325,7 +326,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	case PIN_CONFIG_BIAS_DISABLE:
 		if (config_desc->pud_en_offset == NA) {
-			dev_err(data->dev, "Not support bias disable for pin: %s\n", name);
+			dev_err(data->dev, "Bias disable unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		pulen_off = config_desc->base_bit + config_desc->pud_en_offset;
@@ -337,7 +338,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	case PIN_CONFIG_BIAS_PULL_UP:
 		if (config_desc->pud_en_offset == NA) {
-			dev_err(data->dev, "Not support bias pull up for pin:%s\n", name);
+			dev_err(data->dev, "Bias pull up unsupported for pin:%s\n", name);
 			return -ENOTSUPP;
 		}
 		pulen_off = config_desc->base_bit + config_desc->pud_en_offset;
@@ -350,7 +351,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	case PIN_CONFIG_BIAS_PULL_DOWN:
 		if (config_desc->pud_en_offset == NA) {
-			dev_err(data->dev, "Not support bias pull down for pin: %s\n", name);
+			dev_err(data->dev, "Bias pull down unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		pulen_off = config_desc->base_bit + config_desc->pud_en_offset;
@@ -384,7 +385,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 				return -EINVAL;
 			break;
 		case NA:
-			dev_err(data->dev, "Not support drive strength for pin: %s\n", name);
+			dev_err(data->dev, "Drive strength unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		default:
 			return -EINVAL;
@@ -394,7 +395,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 
 	case PIN_CONFIG_POWER_SOURCE:
 		if (config_desc->power_offset == NA) {
-			dev_err(data->dev, "Not support power source for pin: %s\n", name);
+			dev_err(data->dev, "Power source unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		reg_off = config_desc->reg_offset;
@@ -411,7 +412,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 	case RTD_DRIVE_STRENGH_P:
 		sconfig_desc = rtd_pinctrl_find_sconfig(data, pinnr);
 		if (!sconfig_desc) {
-			dev_err(data->dev, "Not support P driving for pin: %s\n", name);
+			dev_err(data->dev, "P driving unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		set_val = arg;
@@ -428,7 +429,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 	case RTD_DRIVE_STRENGH_N:
 		sconfig_desc = rtd_pinctrl_find_sconfig(data, pinnr);
 		if (!sconfig_desc) {
-			dev_err(data->dev, "Not support N driving for pin: %s\n", name);
+			dev_err(data->dev, "N driving unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		set_val = arg;
@@ -445,7 +446,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 	case RTD_DUTY_CYCLE:
 		sconfig_desc = rtd_pinctrl_find_sconfig(data, pinnr);
 		if (!sconfig_desc || sconfig_desc->dcycle_offset == NA) {
-			dev_err(data->dev, "Not support duty cycle for pin: %s\n", name);
+			dev_err(data->dev, "Duty cycle unsupported for pin: %s\n", name);
 			return -ENOTSUPP;
 		}
 		set_val = arg;
@@ -456,8 +457,8 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 		break;
 
 	default:
-		dev_err(data->dev, "unsupported pinconf: %d\n", (u32)param);
-		return -EINVAL;
+		dev_dbg(data->dev, "unsupported pinconf: %d\n", (u32)param);
+		return -ENOTSUPP;
 	}
 
 	ret = regmap_update_bits(data->regmap_pinctrl, reg_off, mask, val);
@@ -540,18 +541,42 @@ static const struct regmap_config rtd_pinctrl_regmap_config = {
 	.use_relaxed_mmio = true,
 };
 
+static int rtd_pinctrl_init_pm(struct rtd_pinctrl *data)
+{
+	const struct rtd_pin_range *pin_range = data->info->pin_range;
+	struct device *dev = data->pcdev->dev;
+	const struct rtd_reg_range *range;
+	size_t num_entries;
+	int i;
+
+	data->saved_regs = devm_kcalloc(dev, pin_range->num_ranges, sizeof(u32 *), GFP_KERNEL);
+	if (!data->saved_regs)
+		return -ENOMEM;
+
+	for (i = 0; i < pin_range->num_ranges; i++) {
+		range = &pin_range->ranges[i];
+		num_entries = range->len / 4;
+
+		data->saved_regs[i] = devm_kzalloc(dev, num_entries * sizeof(u32), GFP_KERNEL);
+		if (!data->saved_regs[i])
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
 int rtd_pinctrl_probe(struct platform_device *pdev, const struct rtd_pinctrl_desc *desc)
 {
 	struct rtd_pinctrl *data;
-	int ret;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	data->base = of_iomap(pdev->dev.of_node, 0);
-	if (!data->base)
-		return -ENOMEM;
+	data->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(data->base))
+		return dev_err_probe(&pdev->dev, PTR_ERR(data->base),
+				     "Failed to ioremap resource\n");
 
 	data->dev = &pdev->dev;
 	data->info = desc;
@@ -567,30 +592,95 @@ int rtd_pinctrl_probe(struct platform_device *pdev, const struct rtd_pinctrl_des
 	data->regmap_pinctrl = devm_regmap_init_mmio(data->dev, data->base,
 						     &rtd_pinctrl_regmap_config);
 
-	if (IS_ERR(data->regmap_pinctrl)) {
-		dev_err(data->dev, "failed to init regmap: %ld\n",
-			PTR_ERR(data->regmap_pinctrl));
-		ret = PTR_ERR(data->regmap_pinctrl);
-		goto unmap;
-	}
+	if (IS_ERR(data->regmap_pinctrl))
+		return dev_err_probe(data->dev, PTR_ERR(data->regmap_pinctrl),
+				     "Failed to init regmap\n");
 
-	data->pcdev = pinctrl_register(&data->desc, &pdev->dev, data);
-	if (IS_ERR(data->pcdev)) {
-		ret = PTR_ERR(data->pcdev);
-		goto unmap;
-	}
+	data->pcdev = devm_pinctrl_register(&pdev->dev, &data->desc, data);
+	if (IS_ERR(data->pcdev))
+		return dev_err_probe(data->dev, PTR_ERR(data->pcdev),
+				     "Failed to register pinctrl\n");
 
 	platform_set_drvdata(pdev, data);
 
 	dev_dbg(&pdev->dev, "probed\n");
 
-	return 0;
+	if (data->info->pin_range) {
+		if (rtd_pinctrl_init_pm(data))
+			return -ENOMEM;
+	}
 
-unmap:
-	iounmap(data->base);
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL(rtd_pinctrl_probe);
 
+static int realtek_pinctrl_suspend(struct device *dev)
+{
+	struct rtd_pinctrl *data = dev_get_drvdata(dev);
+	const struct rtd_pin_range *pin_range = data->info->pin_range;
+	const struct rtd_reg_range *range;
+	u32 *range_regs;
+	int count;
+	int i, j;
+	int ret;
+
+	if (!data->saved_regs)
+		return 0;
+
+	for (i = 0; i < pin_range->num_ranges; i++) {
+		range = &pin_range->ranges[i];
+		range_regs = data->saved_regs[i];
+		count = range->len / 4;
+
+		for (j = 0; j < count; j++) {
+			ret = regmap_read(data->regmap_pinctrl, range->offset + (j * 4),
+					  &range_regs[j]);
+			if (ret) {
+				dev_err(dev, "failed to store register 0x%x: %d\n",
+					range->offset + (j * 4), ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int realtek_pinctrl_resume(struct device *dev)
+{
+	struct rtd_pinctrl *data = dev_get_drvdata(dev);
+	const struct rtd_pin_range *pin_range = data->info->pin_range;
+	const struct rtd_reg_range *range;
+	u32 *range_regs;
+	int count;
+	int i, j;
+	int ret;
+
+	if (!data->saved_regs)
+		return 0;
+
+	for (i = 0; i < pin_range->num_ranges; i++) {
+		range = &pin_range->ranges[i];
+		range_regs = data->saved_regs[i];
+		count = range->len / 4;
+
+		for (j = 0; j < count; j++) {
+			ret = regmap_write(data->regmap_pinctrl, range->offset + (j * 4),
+					   range_regs[j]);
+			if (ret) {
+				dev_err(dev, "failed to restore register 0x%x: %d\n",
+					range->offset + (j * 4), ret);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
+const struct dev_pm_ops realtek_pinctrl_pm_ops = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(realtek_pinctrl_suspend, realtek_pinctrl_resume)
+};
+EXPORT_SYMBOL_GPL(realtek_pinctrl_pm_ops);
+
 MODULE_DESCRIPTION("Realtek DHC SoC pinctrl driver");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/realtek/pinctrl-rtd.h b/drivers/pinctrl/realtek/pinctrl-rtd.h
index e15130896abc0..7fb0955ce7491 100644
--- a/drivers/pinctrl/realtek/pinctrl-rtd.h
+++ b/drivers/pinctrl/realtek/pinctrl-rtd.h
@@ -47,6 +47,16 @@ struct rtd_pin_sconfig_desc {
 	unsigned int pdrive_maskbits;
 };
 
+struct rtd_reg_range {
+	unsigned int offset;
+	size_t len;
+};
+
+struct rtd_pin_range {
+	const struct rtd_reg_range *ranges;
+	const int num_ranges;
+};
+
 struct rtd_pin_desc {
 	const char *name;
 	unsigned int mux_offset;
@@ -119,6 +129,9 @@ struct rtd_pinctrl_desc {
 	unsigned int num_sconfigs;
 	struct rtd_pin_reg_list *lists;
 	unsigned int num_regs;
+	const struct rtd_pin_range *pin_range;
 };
 
 int rtd_pinctrl_probe(struct platform_device *pdev, const struct rtd_pinctrl_desc *desc);
+
+extern const struct dev_pm_ops realtek_pinctrl_pm_ops;
diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c
index bc8b9b9ad05b9..d2949e4dbaf73 100644
--- a/drivers/pinctrl/renesas/pinctrl-rza1.c
+++ b/drivers/pinctrl/renesas/pinctrl-rza1.c
@@ -589,7 +589,7 @@ static inline unsigned int rza1_get_bit(struct rza1_port *port,
 {
 	void __iomem *mem = RZA1_ADDR(port->base, reg, port->id);
 
-	return ioread16(mem) & BIT(bit);
+	return !!(ioread16(mem) & BIT(bit));
 }
 
 /**
diff --git a/drivers/pinctrl/renesas/pinctrl-rzt2h.c b/drivers/pinctrl/renesas/pinctrl-rzt2h.c
index 9949108a35bbc..5927744c7a966 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzt2h.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzt2h.c
@@ -85,7 +85,7 @@ struct rzt2h_pinctrl {
 	struct gpio_chip		gpio_chip;
 	struct pinctrl_gpio_range	gpio_range;
 	DECLARE_BITMAP(used_irqs, RZT2H_INTERRUPTS_NUM);
-	spinlock_t			lock; /* lock read/write registers */
+	raw_spinlock_t			lock; /* lock read/write registers */
 	struct mutex			mutex; /* serialize adding groups and functions */
 	bool				safety_port_enabled;
 	atomic_t			wakeup_path;
@@ -145,7 +145,7 @@ static void rzt2h_pinctrl_set_pfc_mode(struct rzt2h_pinctrl *pctrl,
 	u64 reg64;
 	u16 reg16;
 
-	guard(spinlock_irqsave)(&pctrl->lock);
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
 	/* Set pin to 'Non-use (Hi-Z input protection)'  */
 	reg16 = rzt2h_pinctrl_readw(pctrl, port, PM(port));
@@ -474,7 +474,7 @@ static int rzt2h_gpio_request(struct gpio_chip *chip, unsigned int offset)
 	if (ret)
 		return ret;
 
-	guard(spinlock_irqsave)(&pctrl->lock);
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
 	/* Select GPIO mode in PMC Register */
 	rzt2h_pinctrl_set_gpio_en(pctrl, port, bit, true);
@@ -487,7 +487,7 @@ static void rzt2h_gpio_set_direction(struct rzt2h_pinctrl *pctrl, u32 port,
 {
 	u16 reg;
 
-	guard(spinlock_irqsave)(&pctrl->lock);
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
 	reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
 	reg &= ~PM_PIN_MASK(bit);
@@ -509,7 +509,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
 	if (ret)
 		return ret;
 
-	guard(spinlock_irqsave)(&pctrl->lock);
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
 	if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) {
 		/*
@@ -547,7 +547,7 @@ static int rzt2h_gpio_set(struct gpio_chip *chip, unsigned int offset,
 	u8 bit = RZT2H_PIN_ID_TO_PIN(offset);
 	u8 reg;
 
-	guard(spinlock_irqsave)(&pctrl->lock);
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
 	reg = rzt2h_pinctrl_readb(pctrl, port, P(port));
 	if (value)
@@ -833,6 +833,7 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl)
 	if (ret)
 		return dev_err_probe(dev, ret, "Unable to parse gpio-ranges\n");
 
+	of_node_put(of_args.np);
 	if (of_args.args[0] != 0 || of_args.args[1] != 0 ||
 	    of_args.args[2] != pctrl->data->n_port_pins)
 		return dev_err_probe(dev, -EINVAL,
@@ -964,7 +965,7 @@ static int rzt2h_pinctrl_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	spin_lock_init(&pctrl->lock);
+	raw_spin_lock_init(&pctrl->lock);
 	mutex_init(&pctrl->mutex);
 	platform_set_drvdata(pdev, pctrl);
 
diff --git a/drivers/pinctrl/spacemit/pinctrl-k1.c b/drivers/pinctrl/spacemit/pinctrl-k1.c
index 62cab6f6cd0a8..b0be62b1c8161 100644
--- a/drivers/pinctrl/spacemit/pinctrl-k1.c
+++ b/drivers/pinctrl/spacemit/pinctrl-k1.c
@@ -674,7 +674,7 @@ static int spacemit_pinconf_get(struct pinctrl_dev *pctldev,
 			arg = 0;
 		break;
 	default:
-		return -EINVAL;
+		return -ENOTSUPP;
 	}
 
 	*config = pinconf_to_config_packed(param, arg);
@@ -740,7 +740,7 @@ static int spacemit_pinconf_generate_config(struct spacemit_pinctrl *pctrl,
 			}
 			break;
 		default:
-			return -EINVAL;
+			return -ENOTSUPP;
 		}
 	}
 
@@ -814,10 +814,12 @@ static int spacemit_pinconf_set(struct pinctrl_dev *pctldev,
 	struct spacemit_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 	const struct spacemit_pin *spin = spacemit_get_pin(pctrl, pin);
 	u32 value;
+	int ret;
 
-	if (spacemit_pinconf_generate_config(pctrl, spin, pctrl->data->dconf,
-					     configs, num_configs, &value))
-		return -EINVAL;
+	ret = spacemit_pinconf_generate_config(pctrl, spin, pctrl->data->dconf,
+					       configs, num_configs, &value);
+	if (ret)
+		return ret;
 
 	return spacemit_pin_set_config(pctrl, pin, value);
 }
@@ -831,16 +833,17 @@ static int spacemit_pinconf_group_set(struct pinctrl_dev *pctldev,
 	const struct spacemit_pin *spin;
 	const struct group_desc *group;
 	u32 value;
-	int i;
+	int i, ret;
 
 	group = pinctrl_generic_get_group(pctldev, gsel);
 	if (!group)
 		return -EINVAL;
 
 	spin = spacemit_get_pin(pctrl, group->grp.pins[0]);
-	if (spacemit_pinconf_generate_config(pctrl, spin, pctrl->data->dconf,
-					     configs, num_configs, &value))
-		return -EINVAL;
+	ret = spacemit_pinconf_generate_config(pctrl, spin, pctrl->data->dconf,
+					       configs, num_configs, &value);
+	if (ret)
+		return ret;
 
 	for (i = 0; i < group->grp.npins; i++)
 		spacemit_pin_set_config(pctrl, group->grp.pins[i], value);
diff --git a/drivers/pinctrl/starfive/Kconfig b/drivers/pinctrl/starfive/Kconfig
index 8192ac2087fc3..a9a7cb1016844 100644
--- a/drivers/pinctrl/starfive/Kconfig
+++ b/drivers/pinctrl/starfive/Kconfig
@@ -9,7 +9,6 @@ config PINCTRL_STARFIVE_JH7100
 	select GENERIC_PINCONF
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
-	select OF_GPIO
 	default SOC_STARFIVE
 	help
 	  Say yes here to support pin control on the StarFive JH7100 SoC.
@@ -24,7 +23,6 @@ config PINCTRL_STARFIVE_JH7110
 	select GENERIC_PINCONF
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
-	select OF_GPIO
 
 config PINCTRL_STARFIVE_JH7110_SYS
 	tristate "System pinctrl and GPIO driver for the StarFive JH7110 SoC"
diff --git a/drivers/pinctrl/stm32/Kconfig b/drivers/pinctrl/stm32/Kconfig
index 5f67e1ee66dd9..d6a1715230121 100644
--- a/drivers/pinctrl/stm32/Kconfig
+++ b/drivers/pinctrl/stm32/Kconfig
@@ -65,6 +65,7 @@ config PINCTRL_STM32_HDP
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB
+	select GPIO_GENERIC
 	help
 	  The Hardware Debug Port allows the observation of internal signals.
 	  It uses configurable multiplexer to route signals in a dedicated observation register.
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32-hdp.c b/drivers/pinctrl/stm32/pinctrl-stm32-hdp.c
index 0b1dff01e04c1..cce477e86ef90 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32-hdp.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32-hdp.c
@@ -4,6 +4,7 @@
  * Author: Clément Le Goffic <clement.legoffic@foss.st.com> for STMicroelectronics.
  */
 #include <linux/bits.h>
+#include <linux/bus/stm32_firewall_device.h>
 #include <linux/clk.h>
 #include <linux/gpio/driver.h>
 #include <linux/gpio/generic.h>
@@ -46,9 +47,11 @@ struct stm32_hdp {
 	void __iomem *base;
 	struct clk *clk;
 	struct pinctrl_dev *pctl_dev;
+	struct stm32_firewall *firewall;
 	struct gpio_generic_chip gpio_chip;
 	u32 mux_conf;
 	u32 gposet_conf;
+	int nb_firewall_entries;
 	const char * const *func_name;
 };
 
@@ -615,6 +618,13 @@ static int stm32_hdp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	hdp->dev = dev;
 
+	if (IS_ENABLED(CONFIG_STM32_FIREWALL)) {
+		err = stm32_firewall_get_grant_all_access(dev, &hdp->firewall,
+							  &hdp->nb_firewall_entries);
+		if (err)
+			return err;
+	}
+
 	platform_set_drvdata(pdev, hdp);
 
 	hdp->base = devm_platform_ioremap_resource(pdev, 0);
@@ -670,8 +680,12 @@ static int stm32_hdp_probe(struct platform_device *pdev)
 static void stm32_hdp_remove(struct platform_device *pdev)
 {
 	struct stm32_hdp *hdp = platform_get_drvdata(pdev);
+	int i;
 
 	writel_relaxed(HDP_CTRL_DISABLE, hdp->base + HDP_CTRL);
+
+	for (i = 0; i < hdp->nb_firewall_entries; i++)
+		stm32_firewall_release_access(&hdp->firewall[i]);
 }
 
 static int stm32_hdp_suspend(struct device *dev)
diff --git a/drivers/pinctrl/sunplus/Kconfig b/drivers/pinctrl/sunplus/Kconfig
index 4b5c47c193d97..69f82590f6d24 100644
--- a/drivers/pinctrl/sunplus/Kconfig
+++ b/drivers/pinctrl/sunplus/Kconfig
@@ -13,7 +13,6 @@ config PINCTRL_SPPCTL
 	select PINCONF
 	select PINMUX
 	select GPIOLIB
-	select OF_GPIO
 	help
 	  Say Y here to support Sunplus SP7021 pinmux controller.
 	  This driver requires the pinctrl framework.
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.c b/drivers/pinctrl/tegra/pinctrl-tegra.c
index 11ecbd6a9b2a1..bac2adeb5c63b 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra.c
@@ -832,18 +832,14 @@ int tegra_pinctrl_probe(struct platform_device *pdev,
 	int fn, gn, gfn;
 	unsigned long backup_regs_size = 0;
 
-	pmx = devm_kzalloc(&pdev->dev, sizeof(*pmx), GFP_KERNEL);
+	pmx = devm_kzalloc(&pdev->dev,
+			struct_size(pmx, pingroup_configs, soc_data->ngroups), GFP_KERNEL);
 	if (!pmx)
 		return -ENOMEM;
 
 	pmx->dev = &pdev->dev;
 	pmx->soc = soc_data;
-
-	pmx->pingroup_configs = devm_kcalloc(&pdev->dev,
-					     pmx->soc->ngroups, sizeof(*pmx->pingroup_configs),
-					     GFP_KERNEL);
-	if (!pmx->pingroup_configs)
-		return -ENOMEM;
+	pmx->num_pingroup_configs = soc_data->ngroups;
 
 	/*
 	 * Each mux group will appear in 4 functions' list of groups.
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.h b/drivers/pinctrl/tegra/pinctrl-tegra.h
index bc7b70913b893..deb589cda31e0 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra.h
+++ b/drivers/pinctrl/tegra/pinctrl-tegra.h
@@ -25,8 +25,8 @@ struct tegra_pmx {
 	int nbanks;
 	void __iomem **regs;
 	u32 *backup_regs;
-	/* Array of size soc->ngroups */
-	struct tegra_pingroup_config *pingroup_configs;
+	unsigned int num_pingroup_configs;
+	struct tegra_pingroup_config pingroup_configs[] __counted_by(num_pingroup_configs);
 };
 
 enum tegra_pinconf_param {
diff --git a/drivers/platform/chrome/chromeos_privacy_screen.c b/drivers/platform/chrome/chromeos_privacy_screen.c
index bb74ddf9af4ac..abc5d189a3894 100644
--- a/drivers/platform/chrome/chromeos_privacy_screen.c
+++ b/drivers/platform/chrome/chromeos_privacy_screen.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 #include <drm/drm_privacy_screen_driver.h>
 
 /*
@@ -32,11 +33,10 @@ chromeos_privacy_screen_get_hw_state(struct drm_privacy_screen
 				     *drm_privacy_screen)
 {
 	union acpi_object *obj;
-	acpi_handle handle;
 	struct device *privacy_screen =
 		drm_privacy_screen_get_drvdata(drm_privacy_screen);
+	acpi_handle handle = ACPI_HANDLE(privacy_screen);
 
-	handle = acpi_device_handle(to_acpi_device(privacy_screen));
 	obj = acpi_evaluate_dsm(handle, &chromeos_privacy_screen_dsm_guid,
 				PRIV_SCRN_DSM_REVID,
 				PRIV_SCRN_DSM_FN_GET_STATUS, NULL);
@@ -65,11 +65,9 @@ chromeos_privacy_screen_set_sw_state(struct drm_privacy_screen
 				     enum drm_privacy_screen_status state)
 {
 	union acpi_object *obj = NULL;
-	acpi_handle handle;
 	struct device *privacy_screen =
 		drm_privacy_screen_get_drvdata(drm_privacy_screen);
-
-	handle = acpi_device_handle(to_acpi_device(privacy_screen));
+	acpi_handle handle = ACPI_HANDLE(privacy_screen);
 
 	if (state == PRIVACY_SCREEN_DISABLED) {
 		obj = acpi_evaluate_dsm(handle,
@@ -104,30 +102,28 @@ static const struct drm_privacy_screen_ops chromeos_privacy_screen_ops = {
 	.set_sw_state = chromeos_privacy_screen_set_sw_state,
 };
 
-static int chromeos_privacy_screen_add(struct acpi_device *adev)
+static int chromeos_privacy_screen_probe(struct platform_device *pdev)
 {
 	struct drm_privacy_screen *drm_privacy_screen =
-		drm_privacy_screen_register(&adev->dev,
+		drm_privacy_screen_register(&pdev->dev,
 					    &chromeos_privacy_screen_ops,
-					    &adev->dev);
+					    &pdev->dev);
 
 	if (IS_ERR(drm_privacy_screen)) {
-		dev_err(&adev->dev, "Error registering privacy-screen\n");
+		dev_err(&pdev->dev, "Error registering privacy-screen\n");
 		return PTR_ERR(drm_privacy_screen);
 	}
 
-	adev->driver_data = drm_privacy_screen;
-	dev_info(&adev->dev, "registered privacy-screen '%s'\n",
+	platform_set_drvdata(pdev, drm_privacy_screen);
+	dev_info(&pdev->dev, "registered privacy-screen '%s'\n",
 		 dev_name(&drm_privacy_screen->dev));
 
 	return 0;
 }
 
-static void chromeos_privacy_screen_remove(struct acpi_device *adev)
+static void chromeos_privacy_screen_remove(struct platform_device *pdev)
 {
-	struct drm_privacy_screen *drm_privacy_screen =	acpi_driver_data(adev);
-
-	drm_privacy_screen_unregister(drm_privacy_screen);
+	drm_privacy_screen_unregister(platform_get_drvdata(pdev));
 }
 
 static const struct acpi_device_id chromeos_privacy_screen_device_ids[] = {
@@ -136,17 +132,16 @@ static const struct acpi_device_id chromeos_privacy_screen_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, chromeos_privacy_screen_device_ids);
 
-static struct acpi_driver chromeos_privacy_screen_driver = {
-	.name = "chromeos_privacy_screen_driver",
-	.class = "ChromeOS",
-	.ids = chromeos_privacy_screen_device_ids,
-	.ops = {
-		.add = chromeos_privacy_screen_add,
-		.remove = chromeos_privacy_screen_remove,
+static struct platform_driver chromeos_privacy_screen_driver = {
+	.probe = chromeos_privacy_screen_probe,
+	.remove = chromeos_privacy_screen_remove,
+	.driver = {
+		.name = "chromeos_privacy_screen_driver",
+		.acpi_match_table = chromeos_privacy_screen_device_ids,
 	},
 };
 
-module_acpi_driver(chromeos_privacy_screen_driver);
+module_platform_driver(chromeos_privacy_screen_driver);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("ChromeOS ACPI Privacy Screen driver");
 MODULE_AUTHOR("Rajat Jain <rajatja@google.com>");
diff --git a/drivers/platform/chrome/chromeos_tbmc.c b/drivers/platform/chrome/chromeos_tbmc.c
index d1cf8f3463ce3..5133806b2d958 100644
--- a/drivers/platform/chrome/chromeos_tbmc.c
+++ b/drivers/platform/chrome/chromeos_tbmc.c
@@ -16,6 +16,7 @@
 #include <linux/input.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/printk.h>
 
 #define DRV_NAME "chromeos_tbmc"
@@ -40,20 +41,20 @@ static int chromeos_tbmc_query_switch(struct acpi_device *adev,
 
 static __maybe_unused int chromeos_tbmc_resume(struct device *dev)
 {
-	struct acpi_device *adev = to_acpi_device(dev);
-
-	return chromeos_tbmc_query_switch(adev, adev->driver_data);
+	return chromeos_tbmc_query_switch(ACPI_COMPANION(dev), dev_get_drvdata(dev));
 }
 
-static void chromeos_tbmc_notify(struct acpi_device *adev, u32 event)
+static void chromeos_tbmc_notify(acpi_handle handle, u32 event, void *data)
 {
-	acpi_pm_wakeup_event(&adev->dev);
+	struct device *dev = data;
+
+	acpi_pm_wakeup_event(dev);
 	switch (event) {
 	case 0x80:
-		chromeos_tbmc_query_switch(adev, adev->driver_data);
+		chromeos_tbmc_query_switch(ACPI_COMPANION(dev), dev_get_drvdata(dev));
 		break;
 	default:
-		dev_err(&adev->dev, "Unexpected event: 0x%08X\n", event);
+		dev_err(dev, "Unexpected event: 0x%08X\n", event);
 	}
 }
 
@@ -64,10 +65,11 @@ static int chromeos_tbmc_open(struct input_dev *idev)
 	return chromeos_tbmc_query_switch(adev, idev);
 }
 
-static int chromeos_tbmc_add(struct acpi_device *adev)
+static int chromeos_tbmc_probe(struct platform_device *pdev)
 {
 	struct input_dev *idev;
-	struct device *dev = &adev->dev;
+	struct device *dev = &pdev->dev;
+	struct acpi_device *adev = ACPI_COMPANION(dev);
 	int ret;
 
 	idev = devm_input_allocate_device(dev);
@@ -83,7 +85,7 @@ static int chromeos_tbmc_add(struct acpi_device *adev)
 	idev->open = chromeos_tbmc_open;
 
 	input_set_drvdata(idev, adev);
-	adev->driver_data = idev;
+	platform_set_drvdata(pdev, idev);
 
 	input_set_capability(idev, EV_SW, SW_TABLET_MODE);
 	ret = input_register_device(idev);
@@ -92,9 +94,25 @@ static int chromeos_tbmc_add(struct acpi_device *adev)
 		return ret;
 	}
 	device_init_wakeup(dev, true);
+
+	ret = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
+					      chromeos_tbmc_notify, dev);
+	if (ret) {
+		dev_err(dev, "cannot install ACPI notify handler\n");
+		device_init_wakeup(dev, false);
+		return ret;
+	}
+
 	return 0;
 }
 
+static void chromeos_tbmc_remove(struct platform_device *pdev)
+{
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, chromeos_tbmc_notify);
+	device_init_wakeup(&pdev->dev, false);
+}
+
 static const struct acpi_device_id chromeos_tbmc_acpi_device_ids[] = {
 	{ ACPI_DRV_NAME, 0 },
 	{ }
@@ -104,18 +122,17 @@ MODULE_DEVICE_TABLE(acpi, chromeos_tbmc_acpi_device_ids);
 static SIMPLE_DEV_PM_OPS(chromeos_tbmc_pm_ops, NULL,
 		chromeos_tbmc_resume);
 
-static struct acpi_driver chromeos_tbmc_driver = {
-	.name = DRV_NAME,
-	.class = DRV_NAME,
-	.ids = chromeos_tbmc_acpi_device_ids,
-	.ops = {
-		.add = chromeos_tbmc_add,
-		.notify = chromeos_tbmc_notify,
+static struct platform_driver chromeos_tbmc_driver = {
+	.probe = chromeos_tbmc_probe,
+	.remove = chromeos_tbmc_remove,
+	.driver = {
+		.name = DRV_NAME,
+		.acpi_match_table = chromeos_tbmc_acpi_device_ids,
+		.pm = &chromeos_tbmc_pm_ops,
 	},
-	.drv.pm = &chromeos_tbmc_pm_ops,
 };
 
-module_acpi_driver(chromeos_tbmc_driver);
+module_platform_driver(chromeos_tbmc_driver);
 
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("ChromeOS ACPI tablet switch driver");
diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c
index 2d1aa6edda1a7..f69f2f6de276c 100644
--- a/drivers/platform/chrome/cros_ec_lightbar.c
+++ b/drivers/platform/chrome/cros_ec_lightbar.c
@@ -461,6 +461,8 @@ static ssize_t sequence_store(struct device *dev, struct device_attribute *attr,
 	param = (struct ec_params_lightbar *)msg->data;
 	param->cmd = LIGHTBAR_CMD_SEQ;
 	param->seq.num = num;
+	msg->outsize = offsetof(typeof(*param), seq) + sizeof(param->seq);
+	msg->insize = 0;
 	ret = lb_throttle();
 	if (ret)
 		goto exit;
@@ -516,6 +518,7 @@ static ssize_t program_store(struct device *dev, struct device_attribute *attr,
 	if (ret)
 		goto exit;
 	param = (struct ec_params_lightbar *)msg->data;
+	msg->insize = 0;
 
 	if (lb_version < 3) {
 		dev_info(dev, "Copying %zu byte program to EC", count);
diff --git a/drivers/platform/chrome/cros_usbpd_logger.c b/drivers/platform/chrome/cros_usbpd_logger.c
index 7ce75e2e039e7..d343e1ab6f08b 100644
--- a/drivers/platform/chrome/cros_usbpd_logger.c
+++ b/drivers/platform/chrome/cros_usbpd_logger.c
@@ -5,6 +5,7 @@
  * Copyright 2018 Google LLC.
  */
 
+#include <linux/devm-helpers.h>
 #include <linux/ktime.h>
 #include <linux/math64.h>
 #include <linux/mod_devicetable.h>
@@ -199,6 +200,7 @@ static int cros_usbpd_logger_probe(struct platform_device *pd)
 	struct cros_ec_dev *ec_dev = dev_get_drvdata(pd->dev.parent);
 	struct device *dev = &pd->dev;
 	struct logger_data *logger;
+	int ret;
 
 	logger = devm_kzalloc(dev, sizeof(*logger), GFP_KERNEL);
 	if (!logger)
@@ -210,25 +212,20 @@ static int cros_usbpd_logger_probe(struct platform_device *pd)
 	platform_set_drvdata(pd, logger);
 
 	/* Retrieve PD event logs periodically */
-	INIT_DELAYED_WORK(&logger->log_work, cros_usbpd_log_check);
-	logger->log_workqueue =	create_singlethread_workqueue("cros_usbpd_log");
+	logger->log_workqueue =	devm_alloc_ordered_workqueue(dev, "cros_usbpd_log", 0);
 	if (!logger->log_workqueue)
 		return -ENOMEM;
 
+	ret = devm_delayed_work_autocancel(dev, &logger->log_work, cros_usbpd_log_check);
+	if (ret)
+		return ret;
+
 	queue_delayed_work(logger->log_workqueue, &logger->log_work,
 			   CROS_USBPD_LOG_UPDATE_DELAY);
 
 	return 0;
 }
 
-static void cros_usbpd_logger_remove(struct platform_device *pd)
-{
-	struct logger_data *logger = platform_get_drvdata(pd);
-
-	cancel_delayed_work_sync(&logger->log_work);
-	destroy_workqueue(logger->log_workqueue);
-}
-
 static int __maybe_unused cros_usbpd_logger_resume(struct device *dev)
 {
 	struct logger_data *logger = dev_get_drvdata(dev);
@@ -263,7 +260,6 @@ static struct platform_driver cros_usbpd_logger_driver = {
 		.pm = &cros_usbpd_logger_pm_ops,
 	},
 	.probe = cros_usbpd_logger_probe,
-	.remove = cros_usbpd_logger_remove,
 	.id_table = cros_usbpd_logger_id,
 };
 
diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c
index 743cd4839bff6..b6e935badc0e9 100644
--- a/drivers/platform/chrome/wilco_ec/event.c
+++ b/drivers/platform/chrome/wilco_ec/event.c
@@ -38,6 +38,7 @@
 #include <linux/io.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/poll.h>
 #include <linux/spinlock.h>
 #include <linux/uaccess.h>
@@ -198,7 +199,7 @@ struct event_device_data {
 
 /**
  * enqueue_events() - Place EC events in queue to be read by userspace.
- * @adev: Device the events came from.
+ * @dev: Device the events came from.
  * @buf: Buffer of event data.
  * @length: Length of event data buffer.
  *
@@ -209,9 +210,9 @@ struct event_device_data {
  *
  * Return: 0 on success or negative error code on failure.
  */
-static int enqueue_events(struct acpi_device *adev, const u8 *buf, u32 length)
+static int enqueue_events(struct device *dev, const u8 *buf, u32 length)
 {
-	struct event_device_data *dev_data = adev->driver_data;
+	struct event_device_data *dev_data = dev_get_drvdata(dev);
 	struct ec_event *event, *queue_event, *old_event;
 	size_t num_words, event_size;
 	u32 offset = 0;
@@ -222,14 +223,14 @@ static int enqueue_events(struct acpi_device *adev, const u8 *buf, u32 length)
 		num_words = ec_event_num_words(event);
 		event_size = ec_event_size(event);
 		if (num_words > EC_ACPI_MAX_EVENT_WORDS) {
-			dev_err(&adev->dev, "Too many event words: %zu > %d\n",
+			dev_err(dev, "Too many event words: %zu > %d\n",
 				num_words, EC_ACPI_MAX_EVENT_WORDS);
 			return -EOVERFLOW;
 		}
 
 		/* Ensure event does not overflow the available buffer */
 		if ((offset + event_size) > length) {
-			dev_err(&adev->dev, "Event exceeds buffer: %zu > %d\n",
+			dev_err(dev, "Event exceeds buffer: %zu > %d\n",
 				offset + event_size, length);
 			return -EOVERFLOW;
 		}
@@ -253,19 +254,22 @@ static int enqueue_events(struct acpi_device *adev, const u8 *buf, u32 length)
 
 /**
  * event_device_notify() - Callback when EC generates an event over ACPI.
- * @adev: The device that the event is coming from.
+ * @handle: ACPI handle of the device that the event is coming from.
  * @value: Value passed to Notify() in ACPI.
+ * @data: Notify handler data.
  *
  * This function will read the events from the device and enqueue them.
  */
-static void event_device_notify(struct acpi_device *adev, u32 value)
+static void event_device_notify(acpi_handle handle, u32 value, void *data)
 {
 	struct acpi_buffer event_buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	struct device *dev = data;
+	struct acpi_device *adev = ACPI_COMPANION(dev);
 	union acpi_object *obj;
 	acpi_status status;
 
 	if (value != EC_ACPI_NOTIFY_EVENT) {
-		dev_err(&adev->dev, "Invalid event: 0x%08x\n", value);
+		dev_err(dev, "Invalid event: 0x%08x\n", value);
 		return;
 	}
 
@@ -273,31 +277,31 @@ static void event_device_notify(struct acpi_device *adev, u32 value)
 	status = acpi_evaluate_object(adev->handle, EC_ACPI_GET_EVENT,
 				      NULL, &event_buffer);
 	if (ACPI_FAILURE(status)) {
-		dev_err(&adev->dev, "Error executing ACPI method %s()\n",
+		dev_err(dev, "Error executing ACPI method %s()\n",
 			EC_ACPI_GET_EVENT);
 		return;
 	}
 
 	obj = (union acpi_object *)event_buffer.pointer;
 	if (!obj) {
-		dev_err(&adev->dev, "Nothing returned from %s()\n",
+		dev_err(dev, "Nothing returned from %s()\n",
 			EC_ACPI_GET_EVENT);
 		return;
 	}
 	if (obj->type != ACPI_TYPE_BUFFER) {
-		dev_err(&adev->dev, "Invalid object returned from %s()\n",
+		dev_err(dev, "Invalid object returned from %s()\n",
 			EC_ACPI_GET_EVENT);
 		kfree(obj);
 		return;
 	}
 	if (obj->buffer.length < sizeof(struct ec_event)) {
-		dev_err(&adev->dev, "Invalid buffer length %d from %s()\n",
+		dev_err(dev, "Invalid buffer length %d from %s()\n",
 			obj->buffer.length, EC_ACPI_GET_EVENT);
 		kfree(obj);
 		return;
 	}
 
-	enqueue_events(adev, obj->buffer.pointer, obj->buffer.length);
+	enqueue_events(dev, obj->buffer.pointer, obj->buffer.length);
 	kfree(obj);
 }
 
@@ -432,8 +436,8 @@ static void hangup_device(struct event_device_data *dev_data)
 }
 
 /**
- * event_device_add() - Callback when creating a new device.
- * @adev: ACPI device that we will be receiving events from.
+ * event_device_probe() - Callback when creating a new device.
+ * @pdev: Platform device that we will be receiving events from.
  *
  * This finds a free minor number for the device, allocates and initializes
  * some device data, and creates a new device and char dev node.
@@ -445,7 +449,7 @@ static void hangup_device(struct event_device_data *dev_data)
  *
  * Return: 0 on success, negative error code on failure.
  */
-static int event_device_add(struct acpi_device *adev)
+static int event_device_probe(struct platform_device *pdev)
 {
 	struct event_device_data *dev_data;
 	int error, minor;
@@ -453,7 +457,7 @@ static int event_device_add(struct acpi_device *adev)
 	minor = ida_alloc_max(&event_ida, EVENT_MAX_DEV-1, GFP_KERNEL);
 	if (minor < 0) {
 		error = minor;
-		dev_err(&adev->dev, "Failed to find minor number: %d\n", error);
+		dev_err(&pdev->dev, "Failed to find minor number: %d\n", error);
 		return error;
 	}
 
@@ -464,7 +468,7 @@ static int event_device_add(struct acpi_device *adev)
 	}
 
 	/* Initialize the device data. */
-	adev->driver_data = dev_data;
+	platform_set_drvdata(pdev, dev_data);
 	dev_data->events = event_queue_new(queue_size);
 	if (!dev_data->events) {
 		kfree(dev_data);
@@ -489,8 +493,17 @@ static int event_device_add(struct acpi_device *adev)
 	if (error)
 		goto free_dev_data;
 
+	/* Install an ACPI notify handler. */
+	error = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
+						ACPI_DEVICE_NOTIFY,
+						event_device_notify, &pdev->dev);
+	if (error)
+		goto free_cdev;
+
 	return 0;
 
+free_cdev:
+	cdev_device_del(&dev_data->cdev, &dev_data->dev);
 free_dev_data:
 	hangup_device(dev_data);
 free_minor:
@@ -498,10 +511,12 @@ static int event_device_add(struct acpi_device *adev)
 	return error;
 }
 
-static void event_device_remove(struct acpi_device *adev)
+static void event_device_remove(struct platform_device *pdev)
 {
-	struct event_device_data *dev_data = adev->driver_data;
+	struct event_device_data *dev_data = platform_get_drvdata(pdev);
 
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, event_device_notify);
 	cdev_device_del(&dev_data->cdev, &dev_data->dev);
 	ida_free(&event_ida, MINOR(dev_data->dev.devt));
 	hangup_device(dev_data);
@@ -513,14 +528,12 @@ static const struct acpi_device_id event_acpi_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, event_acpi_ids);
 
-static struct acpi_driver event_driver = {
-	.name = DRV_NAME,
-	.class = DRV_NAME,
-	.ids = event_acpi_ids,
-	.ops = {
-		.add = event_device_add,
-		.notify = event_device_notify,
-		.remove = event_device_remove,
+static struct platform_driver event_driver = {
+	.probe = event_device_probe,
+	.remove = event_device_remove,
+	.driver = {
+		.name = DRV_NAME,
+		.acpi_match_table = event_acpi_ids,
 	},
 };
 
@@ -543,7 +556,7 @@ static int __init event_module_init(void)
 	}
 	event_major = MAJOR(dev_num);
 
-	ret = acpi_bus_register_driver(&event_driver);
+	ret = platform_driver_register(&event_driver);
 	if (ret < 0) {
 		pr_err(DRV_NAME ": Failed registering driver: %d\n", ret);
 		goto unregister_region;
@@ -561,7 +574,7 @@ static int __init event_module_init(void)
 
 static void __exit event_module_exit(void)
 {
-	acpi_bus_unregister_driver(&event_driver);
+	platform_driver_unregister(&event_driver);
 	unregister_chrdev_region(MKDEV(event_major, 0), EVENT_MAX_DEV);
 	class_unregister(&event_class);
 	ida_destroy(&event_ida);
diff --git a/drivers/platform/olpc/olpc-xo175-ec.c b/drivers/platform/olpc/olpc-xo175-ec.c
index fa7b3bda688a6..bee271a4fda1a 100644
--- a/drivers/platform/olpc/olpc-xo175-ec.c
+++ b/drivers/platform/olpc/olpc-xo175-ec.c
@@ -482,7 +482,7 @@ static int olpc_xo175_ec_cmd(u8 cmd, u8 *inbuf, size_t inlen, u8 *resp,
 	dev_dbg(dev, "CMD %x, %zd bytes expected\n", cmd, resp_len);
 
 	if (inlen > 5) {
-		dev_err(dev, "command len %zd too big!\n", resp_len);
+		dev_err(dev, "command len %zd too big!\n", inlen);
 		return -EOVERFLOW;
 	}
 
diff --git a/drivers/platform/surface/surfacepro3_button.c b/drivers/platform/surface/surfacepro3_button.c
index 9bd39f09c7db4..0293bc517b544 100644
--- a/drivers/platform/surface/surfacepro3_button.c
+++ b/drivers/platform/surface/surfacepro3_button.c
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/input.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 #include <acpi/button.h>
 
 #define SURFACE_PRO3_BUTTON_HID		"MSHW0028"
@@ -72,9 +73,10 @@ struct surface_button {
 	bool suspended;
 };
 
-static void surface_button_notify(struct acpi_device *device, u32 event)
+static void surface_button_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct surface_button *button = acpi_driver_data(device);
+	struct device *dev = data;
+	struct surface_button *button = dev_get_drvdata(dev);
 	struct input_dev *input;
 	int key_code = KEY_RESERVED;
 	bool pressed = false;
@@ -109,18 +111,17 @@ static void surface_button_notify(struct acpi_device *device, u32 event)
 		key_code = KEY_VOLUMEDOWN;
 		break;
 	case SURFACE_BUTTON_NOTIFY_TABLET_MODE:
-		dev_warn_once(&device->dev, "Tablet mode is not supported\n");
+		dev_warn_once(dev, "Tablet mode is not supported\n");
 		break;
 	default:
-		dev_info_ratelimited(&device->dev,
-				     "Unsupported event [0x%x]\n", event);
+		dev_info_ratelimited(dev, "Unsupported event [0x%x]\n", event);
 		break;
 	}
 	input = button->input;
 	if (key_code == KEY_RESERVED)
 		return;
 	if (pressed)
-		pm_wakeup_dev_event(&device->dev, 0, button->suspended);
+		pm_wakeup_dev_event(dev, 0, button->suspended);
 	if (button->suspended)
 		return;
 	input_report_key(input, key_code, pressed?1:0);
@@ -130,8 +131,7 @@ static void surface_button_notify(struct acpi_device *device, u32 event)
 #ifdef CONFIG_PM_SLEEP
 static int surface_button_suspend(struct device *dev)
 {
-	struct acpi_device *device = to_acpi_device(dev);
-	struct surface_button *button = acpi_driver_data(device);
+	struct surface_button *button = dev_get_drvdata(dev);
 
 	button->suspended = true;
 	return 0;
@@ -139,8 +139,7 @@ static int surface_button_suspend(struct device *dev)
 
 static int surface_button_resume(struct device *dev)
 {
-	struct acpi_device *device = to_acpi_device(dev);
-	struct surface_button *button = acpi_driver_data(device);
+	struct surface_button *button = dev_get_drvdata(dev);
 
 	button->suspended = false;
 	return 0;
@@ -155,9 +154,8 @@ static int surface_button_resume(struct device *dev)
  * Returns true if the driver should bind to this device, i.e. the device is
  * either MSWH0028 (Pro 3) or MSHW0040 on a Pro 4 or Book 1.
  */
-static bool surface_button_check_MSHW0040(struct acpi_device *dev)
+static bool surface_button_check_MSHW0040(struct device *dev, acpi_handle handle)
 {
-	acpi_handle handle = dev->handle;
 	union acpi_object *result;
 	u64 oem_platform_rev = 0;	// valid revisions are nonzero
 
@@ -179,14 +177,15 @@ static bool surface_button_check_MSHW0040(struct acpi_device *dev)
 		ACPI_FREE(result);
 	}
 
-	dev_dbg(&dev->dev, "OEM Platform Revision %llu\n", oem_platform_rev);
+	dev_dbg(dev, "OEM Platform Revision %llu\n", oem_platform_rev);
 
 	return oem_platform_rev == 0;
 }
 
 
-static int surface_button_add(struct acpi_device *device)
+static int surface_button_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct surface_button *button;
 	struct input_dev *input;
 	const char *hid = acpi_device_hid(device);
@@ -196,14 +195,14 @@ static int surface_button_add(struct acpi_device *device)
 	    strlen(SURFACE_BUTTON_OBJ_NAME)))
 		return -ENODEV;
 
-	if (!surface_button_check_MSHW0040(device))
+	if (!surface_button_check_MSHW0040(&pdev->dev, device->handle))
 		return -ENODEV;
 
 	button = kzalloc_obj(struct surface_button);
 	if (!button)
 		return -ENOMEM;
 
-	device->driver_data = button;
+	platform_set_drvdata(pdev, button);
 	button->input = input = input_allocate_device();
 	if (!input) {
 		error = -ENOMEM;
@@ -216,7 +215,7 @@ static int surface_button_add(struct acpi_device *device)
 	input->name = acpi_device_name(device);
 	input->phys = button->phys;
 	input->id.bustype = BUS_HOST;
-	input->dev.parent = &device->dev;
+	input->dev.parent = &pdev->dev;
 	input_set_capability(input, EV_KEY, KEY_POWER);
 	input_set_capability(input, EV_KEY, KEY_LEFTMETA);
 	input_set_capability(input, EV_KEY, KEY_VOLUMEUP);
@@ -226,8 +225,17 @@ static int surface_button_add(struct acpi_device *device)
 	if (error)
 		goto err_free_input;
 
-	device_init_wakeup(&device->dev, true);
-	dev_info(&device->dev, "%s [%s]\n", acpi_device_name(device),
+	device_init_wakeup(&pdev->dev, true);
+
+	error = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+						surface_button_notify, &pdev->dev);
+	if (error) {
+		device_init_wakeup(&pdev->dev, false);
+		input_unregister_device(input);
+		goto err_free_button;
+	}
+
+	dev_info(&pdev->dev, "%s [%s]\n", acpi_device_name(device),
 		 acpi_device_bid(device));
 	return 0;
 
@@ -238,10 +246,13 @@ static int surface_button_add(struct acpi_device *device)
 	return error;
 }
 
-static void surface_button_remove(struct acpi_device *device)
+static void surface_button_remove(struct platform_device *pdev)
 {
-	struct surface_button *button = acpi_driver_data(device);
+	struct surface_button *button = platform_get_drvdata(pdev);
 
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, surface_button_notify);
+	device_init_wakeup(&pdev->dev, false);
 	input_unregister_device(button->input);
 	kfree(button);
 }
@@ -249,16 +260,14 @@ static void surface_button_remove(struct acpi_device *device)
 static SIMPLE_DEV_PM_OPS(surface_button_pm,
 		surface_button_suspend, surface_button_resume);
 
-static struct acpi_driver surface_button_driver = {
-	.name = "surface_pro3_button",
-	.class = "SurfacePro3",
-	.ids = surface_button_device_ids,
-	.ops = {
-		.add = surface_button_add,
-		.remove = surface_button_remove,
-		.notify = surface_button_notify,
+static struct platform_driver surface_button_driver = {
+	.probe = surface_button_probe,
+	.remove = surface_button_remove,
+	.driver = {
+		.name = "surface_pro3_button",
+		.acpi_match_table = surface_button_device_ids,
+		.pm = &surface_button_pm,
 	},
-	.drv.pm = &surface_button_pm,
 };
 
-module_acpi_driver(surface_button_driver);
+module_platform_driver(surface_button_driver);
diff --git a/drivers/platform/x86/acer-wireless.c b/drivers/platform/x86/acer-wireless.c
index 1b5d935d085aa..f464b13a58aff 100644
--- a/drivers/platform/x86/acer-wireless.c
+++ b/drivers/platform/x86/acer-wireless.c
@@ -10,6 +10,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci_ids.h>
+#include <linux/platform_device.h>
 #include <linux/types.h>
 
 static const struct acpi_device_id acer_wireless_acpi_ids[] = {
@@ -18,13 +19,14 @@ static const struct acpi_device_id acer_wireless_acpi_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, acer_wireless_acpi_ids);
 
-static void acer_wireless_notify(struct acpi_device *adev, u32 event)
+static void acer_wireless_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct input_dev *idev = acpi_driver_data(adev);
+	struct device *dev = data;
+	struct input_dev *idev = dev_get_drvdata(dev);
 
-	dev_dbg(&adev->dev, "event=%#x\n", event);
+	dev_dbg(dev, "event=%#x\n", event);
 	if (event != 0x80) {
-		dev_notice(&adev->dev, "Unknown SMKB event: %#x\n", event);
+		dev_notice(dev, "Unknown SMKB event: %#x\n", event);
 		return;
 	}
 	input_report_key(idev, KEY_RFKILL, 1);
@@ -33,15 +35,16 @@ static void acer_wireless_notify(struct acpi_device *adev, u32 event)
 	input_sync(idev);
 }
 
-static int acer_wireless_add(struct acpi_device *adev)
+static int acer_wireless_probe(struct platform_device *pdev)
 {
 	struct input_dev *idev;
+	int ret;
 
-	idev = devm_input_allocate_device(&adev->dev);
+	idev = devm_input_allocate_device(&pdev->dev);
 	if (!idev)
 		return -ENOMEM;
 
-	adev->driver_data = idev;
+	platform_set_drvdata(pdev, idev);
 	idev->name = "Acer Wireless Radio Control";
 	idev->phys = "acer-wireless/input0";
 	idev->id.bustype = BUS_HOST;
@@ -50,19 +53,32 @@ static int acer_wireless_add(struct acpi_device *adev)
 	set_bit(EV_KEY, idev->evbit);
 	set_bit(KEY_RFKILL, idev->keybit);
 
-	return input_register_device(idev);
+	ret = input_register_device(idev);
+	if (ret)
+		return ret;
+
+	return acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
+					       ACPI_DEVICE_NOTIFY,
+					       acer_wireless_notify,
+					       &pdev->dev);
+}
+
+static void acer_wireless_remove(struct platform_device *pdev)
+{
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY,
+				       acer_wireless_notify);
 }
 
-static struct acpi_driver acer_wireless_driver = {
-	.name = "Acer Wireless Radio Control Driver",
-	.class = "hotkey",
-	.ids = acer_wireless_acpi_ids,
-	.ops = {
-		.add = acer_wireless_add,
-		.notify = acer_wireless_notify,
+static struct platform_driver acer_wireless_driver = {
+	.probe = acer_wireless_probe,
+	.remove = acer_wireless_remove,
+	.driver = {
+		.name = "Acer Wireless Radio Control Driver",
+		.acpi_match_table = acer_wireless_acpi_ids,
 	},
 };
-module_acpi_driver(acer_wireless_driver);
+module_platform_driver(acer_wireless_driver);
 
 MODULE_DESCRIPTION("Acer Wireless Radio Control Driver");
 MODULE_AUTHOR("Chris Chiu <chiu@gmail.com>");
diff --git a/drivers/platform/x86/amd/hsmp/hsmp.c b/drivers/platform/x86/amd/hsmp/hsmp.c
index 19f82c1d30905..631ffc0978d16 100644
--- a/drivers/platform/x86/amd/hsmp/hsmp.c
+++ b/drivers/platform/x86/amd/hsmp/hsmp.c
@@ -117,7 +117,7 @@ static int __hsmp_send_message(struct hsmp_socket *sock, struct hsmp_message *ms
 	}
 
 	if (unlikely(mbox_status == HSMP_STATUS_NOT_READY)) {
-		dev_err(sock->dev, "Message ID 0x%X failure : SMU tmeout (status = 0x%X)\n",
+		dev_err(sock->dev, "Message ID 0x%X failure : SMU timeout (status = 0x%X)\n",
 			msg->msg_id, mbox_status);
 		return -ETIMEDOUT;
 	} else if (unlikely(mbox_status == HSMP_ERR_INVALID_MSG)) {
diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h
index 208f6fe16168c..569743746347c 100644
--- a/drivers/platform/x86/asus-armoury.h
+++ b/drivers/platform/x86/asus-armoury.h
@@ -1080,6 +1080,20 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "GA503QM"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl1_spl_def = 35,
+				.ppt_pl1_spl_max = 80,
+				.ppt_pl2_sppt_min = 65,
+				.ppt_pl2_sppt_max = 80,
+			},
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "GA503QR"),
@@ -1518,6 +1532,35 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "GZ302EA"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 28,
+				.ppt_pl1_spl_def = 60,
+				.ppt_pl1_spl_max = 80,
+				.ppt_pl2_sppt_min = 32,
+				.ppt_pl2_sppt_def = 75,
+				.ppt_pl2_sppt_max = 92,
+				.ppt_pl3_fppt_min = 45,
+				.ppt_pl3_fppt_def = 86,
+				.ppt_pl3_fppt_max = 93,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 28,
+				.ppt_pl1_spl_def = 45,
+				.ppt_pl1_spl_max = 80,
+				.ppt_pl2_sppt_min = 32,
+				.ppt_pl2_sppt_def = 52,
+				.ppt_pl2_sppt_max = 92,
+				.ppt_pl3_fppt_min = 45,
+				.ppt_pl3_fppt_def = 71,
+				.ppt_pl3_fppt_max = 93,
+			},
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "G513I"),
@@ -1596,6 +1639,40 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "G614FP"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 30,
+				.ppt_pl1_spl_max = 120,
+				.ppt_pl2_sppt_min = 65,
+				.ppt_pl2_sppt_def = 140,
+				.ppt_pl2_sppt_max = 165,
+				.ppt_pl3_fppt_min = 65,
+				.ppt_pl3_fppt_def = 140,
+				.ppt_pl3_fppt_max = 165,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+				.nv_dynamic_boost_min = 5,
+				.nv_dynamic_boost_max = 15,
+				.nv_tgp_min = 50,
+				.nv_tgp_max = 100,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl1_spl_max = 65,
+				.ppt_pl2_sppt_min = 25,
+				.ppt_pl2_sppt_max = 65,
+				.ppt_pl3_fppt_min = 35,
+				.ppt_pl3_fppt_max = 75,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "G614J"),
diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c
index a38a65f5c550d..b4677c5bba5b4 100644
--- a/drivers/platform/x86/asus-nb-wmi.c
+++ b/drivers/platform/x86/asus-nb-wmi.c
@@ -548,7 +548,7 @@ static const struct dmi_system_id asus_quirks[] = {
 		.callback = dmi_matched,
 		.ident = "ASUS ROG Z13",
 		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow Z13"),
 		},
 		.driver_data = &quirk_asus_z13,
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index 7c0915e097bae..6ba49bd375dfa 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -5413,17 +5413,3 @@ void asus_wmi_unregister_driver(struct asus_wmi_driver *driver)
 	used = false;
 }
 EXPORT_SYMBOL_GPL(asus_wmi_unregister_driver);
-
-static int __init asus_wmi_init(void)
-{
-	pr_info("ASUS WMI generic driver loaded\n");
-	return 0;
-}
-
-static void __exit asus_wmi_exit(void)
-{
-	pr_info("ASUS WMI generic driver unloaded\n");
-}
-
-module_init(asus_wmi_init);
-module_exit(asus_wmi_exit);
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
index 9dddab6c93975..069cf958a90d0 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
@@ -11,6 +11,7 @@
 #include <linux/dmi.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
+#include <linux/sysfs.h>
 #include <linux/wmi.h>
 #include "dell-wmi-sysman.h"
 #include "../../firmware_attributes_class.h"
@@ -143,17 +144,17 @@ int map_wmi_error(int error_code)
  */
 static ssize_t reset_bios_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	char *start = buf;
+	ssize_t len = 0;
 	int i;
 
 	for (i = 0; i < MAX_TYPES; i++) {
 		if (i == reset_option)
-			buf += sprintf(buf, "[%s] ", reset_types[i]);
+			len += sysfs_emit_at(buf, len, "[%s] ", reset_types[i]);
 		else
-			buf += sprintf(buf, "%s ", reset_types[i]);
+			len += sysfs_emit_at(buf, len, "%s ", reset_types[i]);
 	}
-	buf += sprintf(buf, "\n");
-	return buf-start;
+	len += sysfs_emit_at(buf, len, "\n");
+	return len;
 }
 
 /**
@@ -194,7 +195,7 @@ static ssize_t reset_bios_store(struct kobject *kobj,
 static ssize_t pending_reboot_show(struct kobject *kobj, struct kobj_attribute *attr,
 				   char *buf)
 {
-	return sprintf(buf, "%d\n", wmi_priv.pending_changes);
+	return sysfs_emit(buf, "%d\n", wmi_priv.pending_changes);
 }
 
 static struct kobj_attribute reset_bios = __ATTR_RW(reset_bios);
@@ -220,35 +221,6 @@ static int create_attributes_level_sysfs_files(void)
 	return 0;
 }
 
-static ssize_t wmi_sysman_attr_show(struct kobject *kobj, struct attribute *attr,
-				    char *buf)
-{
-	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
-
-	kattr = container_of(attr, struct kobj_attribute, attr);
-	if (kattr->show)
-		ret = kattr->show(kobj, kattr, buf);
-	return ret;
-}
-
-static ssize_t wmi_sysman_attr_store(struct kobject *kobj, struct attribute *attr,
-				     const char *buf, size_t count)
-{
-	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
-
-	kattr = container_of(attr, struct kobj_attribute, attr);
-	if (kattr->store)
-		ret = kattr->store(kobj, kattr, buf, count);
-	return ret;
-}
-
-static const struct sysfs_ops wmi_sysman_kobj_sysfs_ops = {
-	.show	= wmi_sysman_attr_show,
-	.store	= wmi_sysman_attr_store,
-};
-
 static void attr_name_release(struct kobject *kobj)
 {
 	kfree(kobj);
@@ -256,7 +228,7 @@ static void attr_name_release(struct kobject *kobj)
 
 static const struct kobj_type attr_name_ktype = {
 	.release	= attr_name_release,
-	.sysfs_ops	= &wmi_sysman_kobj_sysfs_ops,
+	.sysfs_ops	= &kobj_sysfs_ops,
 };
 
 /**
diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c
index 974f55e0b36fd..02a71095920e5 100644
--- a/drivers/platform/x86/eeepc-laptop.c
+++ b/drivers/platform/x86/eeepc-laptop.c
@@ -1204,9 +1204,10 @@ static void eeepc_input_notify(struct eeepc_laptop *eeepc, int event)
 		pr_info("Unknown key %x pressed\n", event);
 }
 
-static void eeepc_acpi_notify(struct acpi_device *device, u32 event)
+static void eeepc_acpi_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct eeepc_laptop *eeepc = acpi_driver_data(device);
+	struct eeepc_laptop *eeepc = data;
+	struct acpi_device *device = eeepc->device;
 	int old_brightness, new_brightness;
 	u16 count;
 
@@ -1360,8 +1361,9 @@ static void eeepc_enable_camera(struct eeepc_laptop *eeepc)
 
 static bool eeepc_device_present;
 
-static int eeepc_acpi_add(struct acpi_device *device)
+static int eeepc_acpi_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct eeepc_laptop *eeepc;
 	int result;
 
@@ -1372,9 +1374,10 @@ static int eeepc_acpi_add(struct acpi_device *device)
 	eeepc->handle = device->handle;
 	strscpy(acpi_device_name(device), EEEPC_ACPI_DEVICE_NAME);
 	strscpy(acpi_device_class(device), EEEPC_ACPI_CLASS);
-	device->driver_data = eeepc;
 	eeepc->device = device;
 
+	platform_set_drvdata(pdev, eeepc);
+
 	eeepc->hotplug_disabled = hotplug_disabled;
 
 	eeepc_dmi_check(eeepc);
@@ -1422,9 +1425,16 @@ static int eeepc_acpi_add(struct acpi_device *device)
 	if (result)
 		goto fail_rfkill;
 
+	result = acpi_dev_install_notify_handler(device, ACPI_ALL_NOTIFY,
+						 eeepc_acpi_notify, eeepc);
+	if (result)
+		goto fail_acpi_notifier;
+
 	eeepc_device_present = true;
 	return 0;
 
+fail_acpi_notifier:
+	eeepc_rfkill_exit(eeepc);
 fail_rfkill:
 	eeepc_led_exit(eeepc);
 fail_led:
@@ -1440,10 +1450,12 @@ static int eeepc_acpi_add(struct acpi_device *device)
 	return result;
 }
 
-static void eeepc_acpi_remove(struct acpi_device *device)
+static void eeepc_acpi_remove(struct platform_device *pdev)
 {
-	struct eeepc_laptop *eeepc = acpi_driver_data(device);
+	struct eeepc_laptop *eeepc = platform_get_drvdata(pdev);
 
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_ALL_NOTIFY, eeepc_acpi_notify);
 	eeepc_backlight_exit(eeepc);
 	eeepc_rfkill_exit(eeepc);
 	eeepc_input_exit(eeepc);
@@ -1460,15 +1472,12 @@ static const struct acpi_device_id eeepc_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, eeepc_device_ids);
 
-static struct acpi_driver eeepc_acpi_driver = {
-	.name = EEEPC_LAPTOP_NAME,
-	.class = EEEPC_ACPI_CLASS,
-	.ids = eeepc_device_ids,
-	.flags = ACPI_DRIVER_ALL_NOTIFY_EVENTS,
-	.ops = {
-		.add = eeepc_acpi_add,
-		.remove = eeepc_acpi_remove,
-		.notify = eeepc_acpi_notify,
+static struct platform_driver eeepc_acpi_driver = {
+	.probe = eeepc_acpi_probe,
+	.remove = eeepc_acpi_remove,
+	.driver = {
+		.name = EEEPC_LAPTOP_NAME,
+		.acpi_match_table = eeepc_device_ids,
 	},
 };
 
@@ -1481,7 +1490,7 @@ static int __init eeepc_laptop_init(void)
 	if (result < 0)
 		return result;
 
-	result = acpi_bus_register_driver(&eeepc_acpi_driver);
+	result = platform_driver_register(&eeepc_acpi_driver);
 	if (result < 0)
 		goto fail_acpi_driver;
 
@@ -1493,7 +1502,7 @@ static int __init eeepc_laptop_init(void)
 	return 0;
 
 fail_no_device:
-	acpi_bus_unregister_driver(&eeepc_acpi_driver);
+	platform_driver_unregister(&eeepc_acpi_driver);
 fail_acpi_driver:
 	platform_driver_unregister(&platform_driver);
 	return result;
@@ -1501,7 +1510,7 @@ static int __init eeepc_laptop_init(void)
 
 static void __exit eeepc_laptop_exit(void)
 {
-	acpi_bus_unregister_driver(&eeepc_acpi_driver);
+	platform_driver_unregister(&eeepc_acpi_driver);
 	platform_driver_unregister(&platform_driver);
 }
 
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index 931fbcdd21b88..2e265be2267ed 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -144,11 +144,11 @@ struct fujitsu_laptop {
 	bool charge_control_supported;
 };
 
-static struct acpi_device *fext;
+static struct device *fext;
 
 /* Fujitsu ACPI interface function */
 
-static int call_fext_func(struct acpi_device *device,
+static int call_fext_func(struct device *dev,
 			  int func, int op, int feature, int state)
 {
 	union acpi_object params[4] = {
@@ -158,18 +158,17 @@ static int call_fext_func(struct acpi_device *device,
 		{ .integer.type = ACPI_TYPE_INTEGER, .integer.value = state }
 	};
 	struct acpi_object_list arg_list = { 4, params };
+	acpi_handle handle = ACPI_HANDLE(dev);
 	unsigned long long value;
 	acpi_status status;
 
-	status = acpi_evaluate_integer(device->handle, "FUNC", &arg_list,
-				       &value);
+	status = acpi_evaluate_integer(handle, "FUNC", &arg_list, &value);
 	if (ACPI_FAILURE(status)) {
-		acpi_handle_err(device->handle, "Failed to evaluate FUNC\n");
+		acpi_handle_err(handle, "Failed to evaluate FUNC\n");
 		return -ENODEV;
 	}
 
-	acpi_handle_debug(device->handle,
-			  "FUNC 0x%x (args 0x%x, 0x%x, 0x%x) returned 0x%x\n",
+	acpi_handle_debug(handle, "FUNC 0x%x (args 0x%x, 0x%x, 0x%x) returned 0x%x\n",
 			  func, op, feature, state, (int)value);
 	return value;
 }
@@ -251,9 +250,9 @@ static struct acpi_battery_hook battery_hook = {
  * These functions are intended to be called from acpi_fujitsu_laptop_add and
  * acpi_fujitsu_laptop_remove.
  */
-static int fujitsu_battery_charge_control_add(struct acpi_device *device)
+static int fujitsu_battery_charge_control_add(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
 	int s006_cc_return;
 
 	priv->charge_control_supported = false;
@@ -274,9 +273,9 @@ static int fujitsu_battery_charge_control_add(struct acpi_device *device)
 	return 0;
 }
 
-static void fujitsu_battery_charge_control_remove(struct acpi_device *device)
+static void fujitsu_battery_charge_control_remove(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
 
 	if (priv->charge_control_supported)
 		battery_hook_unregister(&battery_hook);
@@ -284,15 +283,16 @@ static void fujitsu_battery_charge_control_remove(struct acpi_device *device)
 
 /* Hardware access for LCD brightness control */
 
-static int set_lcd_level(struct acpi_device *device, int level)
+static int set_lcd_level(struct device *dev, int level)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
+	acpi_handle handle = ACPI_HANDLE(dev);
 	acpi_status status;
 	char *method;
 
 	switch (use_alt_lcd_levels) {
 	case -1:
-		if (acpi_has_method(device->handle, "SBL2"))
+		if (acpi_has_method(handle, "SBL2"))
 			method = "SBL2";
 		else
 			method = "SBLL";
@@ -305,16 +305,14 @@ static int set_lcd_level(struct acpi_device *device, int level)
 		break;
 	}
 
-	acpi_handle_debug(device->handle, "set lcd level via %s [%d]\n", method,
-			  level);
+	acpi_handle_debug(handle, "set lcd level via %s [%d]\n", method, level);
 
 	if (level < 0 || level >= priv->max_brightness)
 		return -EINVAL;
 
-	status = acpi_execute_simple_method(device->handle, method, level);
+	status = acpi_execute_simple_method(handle, method, level);
 	if (ACPI_FAILURE(status)) {
-		acpi_handle_err(device->handle, "Failed to evaluate %s\n",
-				method);
+		acpi_handle_err(handle, "Failed to evaluate %s\n", method);
 		return -ENODEV;
 	}
 
@@ -323,15 +321,16 @@ static int set_lcd_level(struct acpi_device *device, int level)
 	return 0;
 }
 
-static int get_lcd_level(struct acpi_device *device)
+static int get_lcd_level(struct device *dev)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
+	acpi_handle handle = ACPI_HANDLE(dev);
 	unsigned long long state = 0;
 	acpi_status status = AE_OK;
 
-	acpi_handle_debug(device->handle, "get lcd level via GBLL\n");
+	acpi_handle_debug(handle, "get lcd level via GBLL\n");
 
-	status = acpi_evaluate_integer(device->handle, "GBLL", NULL, &state);
+	status = acpi_evaluate_integer(handle, "GBLL", NULL, &state);
 	if (ACPI_FAILURE(status))
 		return 0;
 
@@ -340,15 +339,16 @@ static int get_lcd_level(struct acpi_device *device)
 	return priv->brightness_level;
 }
 
-static int get_max_brightness(struct acpi_device *device)
+static int get_max_brightness(struct device *dev)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
+	acpi_handle handle = ACPI_HANDLE(dev);
 	unsigned long long state = 0;
 	acpi_status status = AE_OK;
 
-	acpi_handle_debug(device->handle, "get max lcd level via RBLL\n");
+	acpi_handle_debug(handle, "get max lcd level via RBLL\n");
 
-	status = acpi_evaluate_integer(device->handle, "RBLL", NULL, &state);
+	status = acpi_evaluate_integer(handle, "RBLL", NULL, &state);
 	if (ACPI_FAILURE(status))
 		return -1;
 
@@ -361,15 +361,13 @@ static int get_max_brightness(struct acpi_device *device)
 
 static int bl_get_brightness(struct backlight_device *b)
 {
-	struct acpi_device *device = bl_get_data(b);
+	struct device *dev = bl_get_data(b);
 
-	return b->props.power == BACKLIGHT_POWER_OFF ? 0 : get_lcd_level(device);
+	return b->props.power == BACKLIGHT_POWER_OFF ? 0 : get_lcd_level(dev);
 }
 
 static int bl_update_status(struct backlight_device *b)
 {
-	struct acpi_device *device = bl_get_data(b);
-
 	if (fext) {
 		if (b->props.power == BACKLIGHT_POWER_OFF)
 			call_fext_func(fext, FUNC_BACKLIGHT, 0x1,
@@ -379,7 +377,7 @@ static int bl_update_status(struct backlight_device *b)
 				       BACKLIGHT_PARAM_POWER, BACKLIGHT_ON);
 	}
 
-	return set_lcd_level(device, b->props.brightness);
+	return set_lcd_level(bl_get_data(b), b->props.brightness);
 }
 
 static const struct backlight_ops fujitsu_bl_ops = {
@@ -455,12 +453,13 @@ static const struct key_entry keymap_backlight[] = {
 	{ KE_END, 0 }
 };
 
-static int acpi_fujitsu_bl_input_setup(struct acpi_device *device)
+static int acpi_fujitsu_bl_input_setup(struct device *dev)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
+	struct acpi_device *device = ACPI_COMPANION(dev);
 	int ret;
 
-	priv->input = devm_input_allocate_device(&device->dev);
+	priv->input = devm_input_allocate_device(dev);
 	if (!priv->input)
 		return -ENOMEM;
 
@@ -479,9 +478,9 @@ static int acpi_fujitsu_bl_input_setup(struct acpi_device *device)
 	return input_register_device(priv->input);
 }
 
-static int fujitsu_backlight_register(struct acpi_device *device)
+static int fujitsu_backlight_register(struct device *dev)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
 	const struct backlight_properties props = {
 		.brightness = priv->brightness_level,
 		.max_brightness = priv->max_brightness - 1,
@@ -489,9 +488,8 @@ static int fujitsu_backlight_register(struct acpi_device *device)
 	};
 	struct backlight_device *bd;
 
-	bd = devm_backlight_device_register(&device->dev, "fujitsu-laptop",
-					    &device->dev, device,
-					    &fujitsu_bl_ops, &props);
+	bd = devm_backlight_device_register(dev, "fujitsu-laptop",
+					    dev, dev, &fujitsu_bl_ops, &props);
 	if (IS_ERR(bd))
 		return PTR_ERR(bd);
 
@@ -500,65 +498,78 @@ static int fujitsu_backlight_register(struct acpi_device *device)
 	return 0;
 }
 
-static int acpi_fujitsu_bl_add(struct acpi_device *device)
+/* Brightness notify */
+
+static void acpi_fujitsu_bl_notify(acpi_handle handle, u32 event, void *data)
 {
+	struct device *dev = data;
+	struct fujitsu_bl *priv = dev_get_drvdata(dev);
+	int oldb, newb;
+
+	if (event != ACPI_FUJITSU_NOTIFY_CODE) {
+		acpi_handle_info(handle, "unsupported event [0x%x]\n", event);
+		sparse_keymap_report_event(priv->input, -1, 1, true);
+		return;
+	}
+
+	oldb = priv->brightness_level;
+	get_lcd_level(dev);
+	newb = priv->brightness_level;
+
+	acpi_handle_debug(handle, "brightness button event [%i -> %i]\n",
+			  oldb, newb);
+
+	if (oldb == newb)
+		return;
+
+	if (!disable_brightness_adjust)
+		set_lcd_level(dev, newb);
+
+	sparse_keymap_report_event(priv->input, oldb < newb, 1, true);
+}
+
+static int acpi_fujitsu_bl_probe(struct platform_device *pdev)
+{
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct fujitsu_bl *priv;
 	int ret;
 
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		return -ENODEV;
 
-	priv = devm_kzalloc(&device->dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
 	fujitsu_bl = priv;
 	strscpy(acpi_device_name(device), ACPI_FUJITSU_BL_DEVICE_NAME);
 	strscpy(acpi_device_class(device), ACPI_FUJITSU_CLASS);
-	device->driver_data = priv;
+
+	platform_set_drvdata(pdev, priv);
 
 	pr_info("ACPI: %s [%s]\n",
 		acpi_device_name(device), acpi_device_bid(device));
 
-	if (get_max_brightness(device) <= 0)
+	if (get_max_brightness(&pdev->dev) <= 0)
 		priv->max_brightness = FUJITSU_LCD_N_LEVELS;
-	get_lcd_level(device);
+	get_lcd_level(&pdev->dev);
 
-	ret = acpi_fujitsu_bl_input_setup(device);
+	ret = acpi_fujitsu_bl_input_setup(&pdev->dev);
 	if (ret)
 		return ret;
 
-	return fujitsu_backlight_register(device);
-}
+	ret = fujitsu_backlight_register(&pdev->dev);
+	if (ret)
+		return ret;
 
-/* Brightness notify */
+	return acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+					       acpi_fujitsu_bl_notify, &pdev->dev);
+}
 
-static void acpi_fujitsu_bl_notify(struct acpi_device *device, u32 event)
+static void acpi_fujitsu_bl_remove(struct platform_device *pdev)
 {
-	struct fujitsu_bl *priv = acpi_driver_data(device);
-	int oldb, newb;
-
-	if (event != ACPI_FUJITSU_NOTIFY_CODE) {
-		acpi_handle_info(device->handle, "unsupported event [0x%x]\n",
-				 event);
-		sparse_keymap_report_event(priv->input, -1, 1, true);
-		return;
-	}
-
-	oldb = priv->brightness_level;
-	get_lcd_level(device);
-	newb = priv->brightness_level;
-
-	acpi_handle_debug(device->handle,
-			  "brightness button event [%i -> %i]\n", oldb, newb);
-
-	if (oldb == newb)
-		return;
-
-	if (!disable_brightness_adjust)
-		set_lcd_level(device, newb);
-
-	sparse_keymap_report_event(priv->input, oldb < newb, 1, true);
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, acpi_fujitsu_bl_notify);
 }
 
 /* ACPI device for hotkey handling */
@@ -653,12 +664,13 @@ static const struct dmi_system_id fujitsu_laptop_dmi_table[] = {
 	{}
 };
 
-static int acpi_fujitsu_laptop_input_setup(struct acpi_device *device)
+static int acpi_fujitsu_laptop_input_setup(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
+	struct acpi_device *device = ACPI_COMPANION(dev);
 	int ret;
 
-	priv->input = devm_input_allocate_device(&device->dev);
+	priv->input = devm_input_allocate_device(dev);
 	if (!priv->input)
 		return -ENOMEM;
 
@@ -677,9 +689,9 @@ static int acpi_fujitsu_laptop_input_setup(struct acpi_device *device)
 	return input_register_device(priv->input);
 }
 
-static int fujitsu_laptop_platform_add(struct acpi_device *device)
+static int fujitsu_laptop_platform_add(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
 	int ret;
 
 	priv->pf_device = platform_device_alloc("fujitsu-laptop", PLATFORM_DEVID_NONE);
@@ -707,9 +719,9 @@ static int fujitsu_laptop_platform_add(struct acpi_device *device)
 	return ret;
 }
 
-static void fujitsu_laptop_platform_remove(struct acpi_device *device)
+static void fujitsu_laptop_platform_remove(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
 
 	sysfs_remove_group(&priv->pf_device->dev.kobj,
 			   &fujitsu_pf_attribute_group);
@@ -719,7 +731,7 @@ static void fujitsu_laptop_platform_remove(struct acpi_device *device)
 static int logolamp_set(struct led_classdev *cdev,
 			enum led_brightness brightness)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 	int poweron = FUNC_LED_ON, always = FUNC_LED_ON;
 	int ret;
 
@@ -729,23 +741,23 @@ static int logolamp_set(struct led_classdev *cdev,
 	if (brightness < LED_FULL)
 		always = FUNC_LED_OFF;
 
-	ret = call_fext_func(device, FUNC_LEDS, 0x1, LOGOLAMP_POWERON, poweron);
+	ret = call_fext_func(parent, FUNC_LEDS, 0x1, LOGOLAMP_POWERON, poweron);
 	if (ret < 0)
 		return ret;
 
-	return call_fext_func(device, FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, always);
+	return call_fext_func(parent, FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, always);
 }
 
 static enum led_brightness logolamp_get(struct led_classdev *cdev)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 	int ret;
 
-	ret = call_fext_func(device, FUNC_LEDS, 0x2, LOGOLAMP_ALWAYS, 0x0);
+	ret = call_fext_func(parent, FUNC_LEDS, 0x2, LOGOLAMP_ALWAYS, 0x0);
 	if (ret == FUNC_LED_ON)
 		return LED_FULL;
 
-	ret = call_fext_func(device, FUNC_LEDS, 0x2, LOGOLAMP_POWERON, 0x0);
+	ret = call_fext_func(parent, FUNC_LEDS, 0x2, LOGOLAMP_POWERON, 0x0);
 	if (ret == FUNC_LED_ON)
 		return LED_HALF;
 
@@ -755,22 +767,21 @@ static enum led_brightness logolamp_get(struct led_classdev *cdev)
 static int kblamps_set(struct led_classdev *cdev,
 		       enum led_brightness brightness)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 
 	if (brightness >= LED_FULL)
-		return call_fext_func(device, FUNC_LEDS, 0x1, KEYBOARD_LAMPS,
+		return call_fext_func(parent, FUNC_LEDS, 0x1, KEYBOARD_LAMPS,
 				      FUNC_LED_ON);
 	else
-		return call_fext_func(device, FUNC_LEDS, 0x1, KEYBOARD_LAMPS,
+		return call_fext_func(parent, FUNC_LEDS, 0x1, KEYBOARD_LAMPS,
 				      FUNC_LED_OFF);
 }
 
 static enum led_brightness kblamps_get(struct led_classdev *cdev)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
 	enum led_brightness brightness = LED_OFF;
 
-	if (call_fext_func(device,
+	if (call_fext_func(cdev->dev->parent,
 			   FUNC_LEDS, 0x2, KEYBOARD_LAMPS, 0x0) == FUNC_LED_ON)
 		brightness = LED_FULL;
 
@@ -780,22 +791,22 @@ static enum led_brightness kblamps_get(struct led_classdev *cdev)
 static int radio_led_set(struct led_classdev *cdev,
 			 enum led_brightness brightness)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 
 	if (brightness >= LED_FULL)
-		return call_fext_func(device, FUNC_FLAGS, 0x5, RADIO_LED_ON,
+		return call_fext_func(parent, FUNC_FLAGS, 0x5, RADIO_LED_ON,
 				      RADIO_LED_ON);
 	else
-		return call_fext_func(device, FUNC_FLAGS, 0x5, RADIO_LED_ON,
+		return call_fext_func(parent, FUNC_FLAGS, 0x5, RADIO_LED_ON,
 				      0x0);
 }
 
 static enum led_brightness radio_led_get(struct led_classdev *cdev)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 	enum led_brightness brightness = LED_OFF;
 
-	if (call_fext_func(device, FUNC_FLAGS, 0x4, 0x0, 0x0) & RADIO_LED_ON)
+	if (call_fext_func(parent, FUNC_FLAGS, 0x4, 0x0, 0x0) & RADIO_LED_ON)
 		brightness = LED_FULL;
 
 	return brightness;
@@ -804,60 +815,58 @@ static enum led_brightness radio_led_get(struct led_classdev *cdev)
 static int eco_led_set(struct led_classdev *cdev,
 		       enum led_brightness brightness)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 	int curr;
 
-	curr = call_fext_func(device, FUNC_LEDS, 0x2, ECO_LED, 0x0);
+	curr = call_fext_func(parent, FUNC_LEDS, 0x2, ECO_LED, 0x0);
 	if (brightness >= LED_FULL)
-		return call_fext_func(device, FUNC_LEDS, 0x1, ECO_LED,
+		return call_fext_func(parent, FUNC_LEDS, 0x1, ECO_LED,
 				      curr | ECO_LED_ON);
 	else
-		return call_fext_func(device, FUNC_LEDS, 0x1, ECO_LED,
+		return call_fext_func(parent, FUNC_LEDS, 0x1, ECO_LED,
 				      curr & ~ECO_LED_ON);
 }
 
 static enum led_brightness eco_led_get(struct led_classdev *cdev)
 {
-	struct acpi_device *device = to_acpi_device(cdev->dev->parent);
+	struct device *parent = cdev->dev->parent;
 	enum led_brightness brightness = LED_OFF;
 
-	if (call_fext_func(device, FUNC_LEDS, 0x2, ECO_LED, 0x0) & ECO_LED_ON)
+	if (call_fext_func(parent, FUNC_LEDS, 0x2, ECO_LED, 0x0) & ECO_LED_ON)
 		brightness = LED_FULL;
 
 	return brightness;
 }
 
-static int acpi_fujitsu_laptop_leds_register(struct acpi_device *device)
+static int acpi_fujitsu_laptop_leds_register(struct device *dev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
 	struct led_classdev *led;
 	int ret;
 
-	if (call_fext_func(device,
-			   FUNC_LEDS, 0x0, 0x0, 0x0) & LOGOLAMP_POWERON) {
-		led = devm_kzalloc(&device->dev, sizeof(*led), GFP_KERNEL);
+	if (call_fext_func(dev, FUNC_LEDS, 0x0, 0x0, 0x0) & LOGOLAMP_POWERON) {
+		led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
 		if (!led)
 			return -ENOMEM;
 
 		led->name = "fujitsu::logolamp";
 		led->brightness_set_blocking = logolamp_set;
 		led->brightness_get = logolamp_get;
-		ret = devm_led_classdev_register(&device->dev, led);
+		ret = devm_led_classdev_register(dev, led);
 		if (ret)
 			return ret;
 	}
 
-	if ((call_fext_func(device,
-			    FUNC_LEDS, 0x0, 0x0, 0x0) & KEYBOARD_LAMPS) &&
-	    (call_fext_func(device, FUNC_BUTTONS, 0x0, 0x0, 0x0) == 0x0)) {
-		led = devm_kzalloc(&device->dev, sizeof(*led), GFP_KERNEL);
+	if ((call_fext_func(dev, FUNC_LEDS, 0x0, 0x0, 0x0) & KEYBOARD_LAMPS) &&
+	    (call_fext_func(dev, FUNC_BUTTONS, 0x0, 0x0, 0x0) == 0x0)) {
+		led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
 		if (!led)
 			return -ENOMEM;
 
 		led->name = "fujitsu::kblamps";
 		led->brightness_set_blocking = kblamps_set;
 		led->brightness_get = kblamps_get;
-		ret = devm_led_classdev_register(&device->dev, led);
+		ret = devm_led_classdev_register(dev, led);
 		if (ret)
 			return ret;
 	}
@@ -872,7 +881,7 @@ static int acpi_fujitsu_laptop_leds_register(struct acpi_device *device)
 	 * whether given model has a radio toggle button.
 	 */
 	if (priv->flags_supported & BIT(17)) {
-		led = devm_kzalloc(&device->dev, sizeof(*led), GFP_KERNEL);
+		led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
 		if (!led)
 			return -ENOMEM;
 
@@ -880,7 +889,7 @@ static int acpi_fujitsu_laptop_leds_register(struct acpi_device *device)
 		led->brightness_set_blocking = radio_led_set;
 		led->brightness_get = radio_led_get;
 		led->default_trigger = "rfkill-any";
-		ret = devm_led_classdev_register(&device->dev, led);
+		ret = devm_led_classdev_register(dev, led);
 		if (ret)
 			return ret;
 	}
@@ -890,17 +899,16 @@ static int acpi_fujitsu_laptop_leds_register(struct acpi_device *device)
 	 * bit 14 seems to indicate presence of said led as well.
 	 * Confirm by testing the status.
 	 */
-	if ((call_fext_func(device, FUNC_LEDS, 0x0, 0x0, 0x0) & BIT(14)) &&
-	    (call_fext_func(device,
-			    FUNC_LEDS, 0x2, ECO_LED, 0x0) != UNSUPPORTED_CMD)) {
-		led = devm_kzalloc(&device->dev, sizeof(*led), GFP_KERNEL);
+	if ((call_fext_func(dev, FUNC_LEDS, 0x0, 0x0, 0x0) & BIT(14)) &&
+	    (call_fext_func(dev, FUNC_LEDS, 0x2, ECO_LED, 0x0) != UNSUPPORTED_CMD)) {
+		led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
 		if (!led)
 			return -ENOMEM;
 
 		led->name = "fujitsu::eco_led";
 		led->brightness_set_blocking = eco_led_set;
 		led->brightness_get = eco_led_get;
-		ret = devm_led_classdev_register(&device->dev, led);
+		ret = devm_led_classdev_register(dev, led);
 		if (ret)
 			return ret;
 	}
@@ -908,21 +916,98 @@ static int acpi_fujitsu_laptop_leds_register(struct acpi_device *device)
 	return 0;
 }
 
-static int acpi_fujitsu_laptop_add(struct acpi_device *device)
+static void acpi_fujitsu_laptop_press(struct device *dev, int scancode)
+{
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = kfifo_in_locked(&priv->fifo, (unsigned char *)&scancode,
+			      sizeof(scancode), &priv->fifo_lock);
+	if (ret != sizeof(scancode)) {
+		dev_info(&priv->input->dev, "Could not push scancode [0x%x]\n",
+			 scancode);
+		return;
+	}
+	sparse_keymap_report_event(priv->input, scancode, 1, false);
+	dev_dbg(&priv->input->dev, "Push scancode into ringbuffer [0x%x]\n",
+		scancode);
+}
+
+static void acpi_fujitsu_laptop_release(struct device *dev)
+{
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
+	int scancode, ret;
+
+	while (true) {
+		ret = kfifo_out_locked(&priv->fifo, (unsigned char *)&scancode,
+				       sizeof(scancode), &priv->fifo_lock);
+		if (ret != sizeof(scancode))
+			return;
+		sparse_keymap_report_event(priv->input, scancode, 0, false);
+		dev_dbg(&priv->input->dev,
+			"Pop scancode from ringbuffer [0x%x]\n", scancode);
+	}
+}
+
+static void acpi_fujitsu_laptop_notify(acpi_handle handle, u32 event, void *data)
+{
+	struct device *dev = data;
+	struct fujitsu_laptop *priv = dev_get_drvdata(dev);
+	unsigned long flags;
+	int scancode, i = 0;
+	unsigned int irb;
+
+	if (event != ACPI_FUJITSU_NOTIFY_CODE) {
+		acpi_handle_info(handle, "Unsupported event [0x%x]\n", event);
+		sparse_keymap_report_event(priv->input, -1, 1, true);
+		return;
+	}
+
+	if (priv->flags_supported)
+		priv->flags_state = call_fext_func(dev, FUNC_FLAGS, 0x4, 0x0, 0x0);
+
+	while ((irb = call_fext_func(dev, FUNC_BUTTONS, 0x1, 0x0, 0x0)) != 0 &&
+	       i++ < MAX_HOTKEY_RINGBUFFER_SIZE) {
+		scancode = irb & 0x4ff;
+		if (sparse_keymap_entry_from_scancode(priv->input, scancode))
+			acpi_fujitsu_laptop_press(dev, scancode);
+		else if (scancode == 0)
+			acpi_fujitsu_laptop_release(dev);
+		else
+			acpi_handle_info(handle, "Unknown GIRB result [%x]\n", irb);
+	}
+
+	/*
+	 * First seen on the Skylake-based Lifebook E736/E746/E756), the
+	 * touchpad toggle hotkey (Fn+F4) is handled in software. Other models
+	 * have since added additional "soft keys". These are reported in the
+	 * status flags queried using FUNC_FLAGS.
+	 */
+	if (priv->flags_supported & (FLAG_SOFTKEYS)) {
+		flags = call_fext_func(dev, FUNC_FLAGS, 0x1, 0x0, 0x0);
+		flags &= (FLAG_SOFTKEYS);
+		for_each_set_bit(i, &flags, BITS_PER_LONG)
+			sparse_keymap_report_event(priv->input, BIT(i), 1, true);
+	}
+}
+
+static int acpi_fujitsu_laptop_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct fujitsu_laptop *priv;
 	int ret, i = 0;
 
-	priv = devm_kzalloc(&device->dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
 	WARN_ONCE(fext, "More than one FUJ02E3 ACPI device was found.  Driver may not work as intended.");
-	fext = device;
+	fext = &pdev->dev;
 
 	strscpy(acpi_device_name(device), ACPI_FUJITSU_LAPTOP_DEVICE_NAME);
 	strscpy(acpi_device_class(device), ACPI_FUJITSU_CLASS);
-	device->driver_data = priv;
+
+	platform_set_drvdata(pdev, priv);
 
 	/* kfifo */
 	spin_lock_init(&priv->fifo_lock);
@@ -934,14 +1019,13 @@ static int acpi_fujitsu_laptop_add(struct acpi_device *device)
 	pr_info("ACPI: %s [%s]\n",
 		acpi_device_name(device), acpi_device_bid(device));
 
-	while (call_fext_func(device, FUNC_BUTTONS, 0x1, 0x0, 0x0) != 0 &&
+	while (call_fext_func(fext, FUNC_BUTTONS, 0x1, 0x0, 0x0) != 0 &&
 	       i++ < MAX_HOTKEY_RINGBUFFER_SIZE)
 		; /* No action, result is discarded */
 	acpi_handle_debug(device->handle, "Discarded %i ringbuffer entries\n",
 			  i);
 
-	priv->flags_supported = call_fext_func(device, FUNC_FLAGS, 0x0, 0x0,
-					       0x0);
+	priv->flags_supported = call_fext_func(fext, FUNC_FLAGS, 0x0, 0x0, 0x0);
 
 	/* Make sure our bitmask of supported functions is cleared if the
 	   RFKILL function block is not implemented, like on the S7020. */
@@ -949,12 +1033,12 @@ static int acpi_fujitsu_laptop_add(struct acpi_device *device)
 		priv->flags_supported = 0;
 
 	if (priv->flags_supported)
-		priv->flags_state = call_fext_func(device, FUNC_FLAGS, 0x4, 0x0,
+		priv->flags_state = call_fext_func(fext, FUNC_FLAGS, 0x4, 0x0,
 						   0x0);
 
 	/* Suspect this is a keymap of the application panel, print it */
 	acpi_handle_info(device->handle, "BTNI: [0x%x]\n",
-			 call_fext_func(device, FUNC_BUTTONS, 0x0, 0x0, 0x0));
+			 call_fext_func(fext, FUNC_BUTTONS, 0x0, 0x0, 0x0));
 
 	/* Sync backlight power status */
 	if (fujitsu_bl && fujitsu_bl->bl_device &&
@@ -966,117 +1050,49 @@ static int acpi_fujitsu_laptop_add(struct acpi_device *device)
 			fujitsu_bl->bl_device->props.power = BACKLIGHT_POWER_ON;
 	}
 
-	ret = acpi_fujitsu_laptop_input_setup(device);
+	ret = acpi_fujitsu_laptop_input_setup(fext);
 	if (ret)
 		goto err_free_fifo;
 
-	ret = acpi_fujitsu_laptop_leds_register(device);
+	ret = acpi_fujitsu_laptop_leds_register(fext);
 	if (ret)
 		goto err_free_fifo;
 
-	ret = fujitsu_laptop_platform_add(device);
+	ret = fujitsu_laptop_platform_add(fext);
 	if (ret)
 		goto err_free_fifo;
 
-	ret = fujitsu_battery_charge_control_add(device);
+	ret = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+					      acpi_fujitsu_laptop_notify, fext);
+	if (ret)
+		goto err_platform_remove;
+
+	ret = fujitsu_battery_charge_control_add(fext);
 	if (ret < 0)
 		pr_warn("Unable to register battery charge control: %d\n", ret);
 
 	return 0;
 
+err_platform_remove:
+	fujitsu_laptop_platform_remove(fext);
 err_free_fifo:
 	kfifo_free(&priv->fifo);
 
 	return ret;
 }
 
-static void acpi_fujitsu_laptop_remove(struct acpi_device *device)
-{
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
-
-	fujitsu_battery_charge_control_remove(device);
-
-	fujitsu_laptop_platform_remove(device);
-
-	kfifo_free(&priv->fifo);
-}
-
-static void acpi_fujitsu_laptop_press(struct acpi_device *device, int scancode)
-{
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
-	int ret;
-
-	ret = kfifo_in_locked(&priv->fifo, (unsigned char *)&scancode,
-			      sizeof(scancode), &priv->fifo_lock);
-	if (ret != sizeof(scancode)) {
-		dev_info(&priv->input->dev, "Could not push scancode [0x%x]\n",
-			 scancode);
-		return;
-	}
-	sparse_keymap_report_event(priv->input, scancode, 1, false);
-	dev_dbg(&priv->input->dev, "Push scancode into ringbuffer [0x%x]\n",
-		scancode);
-}
-
-static void acpi_fujitsu_laptop_release(struct acpi_device *device)
+static void acpi_fujitsu_laptop_remove(struct platform_device *pdev)
 {
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
-	int scancode, ret;
-
-	while (true) {
-		ret = kfifo_out_locked(&priv->fifo, (unsigned char *)&scancode,
-				       sizeof(scancode), &priv->fifo_lock);
-		if (ret != sizeof(scancode))
-			return;
-		sparse_keymap_report_event(priv->input, scancode, 0, false);
-		dev_dbg(&priv->input->dev,
-			"Pop scancode from ringbuffer [0x%x]\n", scancode);
-	}
-}
+	struct fujitsu_laptop *priv = platform_get_drvdata(pdev);
 
-static void acpi_fujitsu_laptop_notify(struct acpi_device *device, u32 event)
-{
-	struct fujitsu_laptop *priv = acpi_driver_data(device);
-	unsigned long flags;
-	int scancode, i = 0;
-	unsigned int irb;
+	fujitsu_battery_charge_control_remove(&pdev->dev);
 
-	if (event != ACPI_FUJITSU_NOTIFY_CODE) {
-		acpi_handle_info(device->handle, "Unsupported event [0x%x]\n",
-				 event);
-		sparse_keymap_report_event(priv->input, -1, 1, true);
-		return;
-	}
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev), ACPI_DEVICE_NOTIFY,
+				       acpi_fujitsu_laptop_notify);
 
-	if (priv->flags_supported)
-		priv->flags_state = call_fext_func(device, FUNC_FLAGS, 0x4, 0x0,
-						   0x0);
+	fujitsu_laptop_platform_remove(&pdev->dev);
 
-	while ((irb = call_fext_func(device,
-				     FUNC_BUTTONS, 0x1, 0x0, 0x0)) != 0 &&
-	       i++ < MAX_HOTKEY_RINGBUFFER_SIZE) {
-		scancode = irb & 0x4ff;
-		if (sparse_keymap_entry_from_scancode(priv->input, scancode))
-			acpi_fujitsu_laptop_press(device, scancode);
-		else if (scancode == 0)
-			acpi_fujitsu_laptop_release(device);
-		else
-			acpi_handle_info(device->handle,
-					 "Unknown GIRB result [%x]\n", irb);
-	}
-
-	/*
-	 * First seen on the Skylake-based Lifebook E736/E746/E756), the
-	 * touchpad toggle hotkey (Fn+F4) is handled in software. Other models
-	 * have since added additional "soft keys". These are reported in the
-	 * status flags queried using FUNC_FLAGS.
-	 */
-	if (priv->flags_supported & (FLAG_SOFTKEYS)) {
-		flags = call_fext_func(device, FUNC_FLAGS, 0x1, 0x0, 0x0);
-		flags &= (FLAG_SOFTKEYS);
-		for_each_set_bit(i, &flags, BITS_PER_LONG)
-			sparse_keymap_report_event(priv->input, BIT(i), 1, true);
-	}
+	kfifo_free(&priv->fifo);
 }
 
 /* Initialization */
@@ -1086,14 +1102,13 @@ static const struct acpi_device_id fujitsu_bl_device_ids[] = {
 	{"", 0},
 };
 
-static struct acpi_driver acpi_fujitsu_bl_driver = {
-	.name = ACPI_FUJITSU_BL_DRIVER_NAME,
-	.class = ACPI_FUJITSU_CLASS,
-	.ids = fujitsu_bl_device_ids,
-	.ops = {
-		.add = acpi_fujitsu_bl_add,
-		.notify = acpi_fujitsu_bl_notify,
-		},
+static struct platform_driver acpi_fujitsu_bl_driver = {
+	.probe = acpi_fujitsu_bl_probe,
+	.remove = acpi_fujitsu_bl_remove,
+	.driver = {
+		.name = ACPI_FUJITSU_BL_DRIVER_NAME,
+		.acpi_match_table = fujitsu_bl_device_ids,
+	},
 };
 
 static const struct acpi_device_id fujitsu_laptop_device_ids[] = {
@@ -1101,15 +1116,13 @@ static const struct acpi_device_id fujitsu_laptop_device_ids[] = {
 	{"", 0},
 };
 
-static struct acpi_driver acpi_fujitsu_laptop_driver = {
-	.name = ACPI_FUJITSU_LAPTOP_DRIVER_NAME,
-	.class = ACPI_FUJITSU_CLASS,
-	.ids = fujitsu_laptop_device_ids,
-	.ops = {
-		.add = acpi_fujitsu_laptop_add,
-		.remove = acpi_fujitsu_laptop_remove,
-		.notify = acpi_fujitsu_laptop_notify,
-		},
+static struct platform_driver acpi_fujitsu_laptop_driver = {
+	.probe = acpi_fujitsu_laptop_probe,
+	.remove = acpi_fujitsu_laptop_remove,
+	.driver = {
+		.name = ACPI_FUJITSU_LAPTOP_DRIVER_NAME,
+		.acpi_match_table = fujitsu_laptop_device_ids,
+	},
 };
 
 static const struct acpi_device_id fujitsu_ids[] __used = {
@@ -1123,7 +1136,7 @@ static int __init fujitsu_init(void)
 {
 	int ret;
 
-	ret = acpi_bus_register_driver(&acpi_fujitsu_bl_driver);
+	ret = platform_driver_register(&acpi_fujitsu_bl_driver);
 	if (ret)
 		return ret;
 
@@ -1135,7 +1148,7 @@ static int __init fujitsu_init(void)
 
 	/* Register laptop driver */
 
-	ret = acpi_bus_register_driver(&acpi_fujitsu_laptop_driver);
+	ret = platform_driver_register(&acpi_fujitsu_laptop_driver);
 	if (ret)
 		goto err_unregister_platform_driver;
 
@@ -1146,18 +1159,18 @@ static int __init fujitsu_init(void)
 err_unregister_platform_driver:
 	platform_driver_unregister(&fujitsu_pf_driver);
 err_unregister_acpi:
-	acpi_bus_unregister_driver(&acpi_fujitsu_bl_driver);
+	platform_driver_unregister(&acpi_fujitsu_bl_driver);
 
 	return ret;
 }
 
 static void __exit fujitsu_cleanup(void)
 {
-	acpi_bus_unregister_driver(&acpi_fujitsu_laptop_driver);
+	platform_driver_unregister(&acpi_fujitsu_laptop_driver);
 
 	platform_driver_unregister(&fujitsu_pf_driver);
 
-	acpi_bus_unregister_driver(&acpi_fujitsu_bl_driver);
+	platform_driver_unregister(&acpi_fujitsu_bl_driver);
 
 	pr_info("driver unloaded\n");
 }
diff --git a/drivers/platform/x86/fujitsu-tablet.c b/drivers/platform/x86/fujitsu-tablet.c
index 17f08ce7552d7..8319df28e9b8a 100644
--- a/drivers/platform/x86/fujitsu-tablet.c
+++ b/drivers/platform/x86/fujitsu-tablet.c
@@ -18,6 +18,7 @@
 #include <linux/input.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
+#include <linux/platform_device.h>
 
 #define MODULENAME "fujitsu-tablet"
 
@@ -442,14 +443,12 @@ static acpi_status fujitsu_walk_resources(struct acpi_resource *res, void *data)
 	}
 }
 
-static int acpi_fujitsu_add(struct acpi_device *adev)
+static int acpi_fujitsu_probe(struct platform_device *pdev)
 {
+	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
 	acpi_status status;
 	int error;
 
-	if (!adev)
-		return -EINVAL;
-
 	status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS,
 			fujitsu_walk_resources, NULL);
 	if (ACPI_FAILURE(status) || !fujitsu.irq || !fujitsu.io_base)
@@ -461,7 +460,7 @@ static int acpi_fujitsu_add(struct acpi_device *adev)
 	snprintf(fujitsu.phys, sizeof(fujitsu.phys),
 			"%s/input0", acpi_device_hid(adev));
 
-	error = input_fujitsu_setup(&adev->dev,
+	error = input_fujitsu_setup(&pdev->dev,
 		acpi_device_name(adev), fujitsu.phys);
 	if (error)
 		return error;
@@ -484,7 +483,7 @@ static int acpi_fujitsu_add(struct acpi_device *adev)
 	return 0;
 }
 
-static void acpi_fujitsu_remove(struct acpi_device *adev)
+static void acpi_fujitsu_remove(struct platform_device *pdev)
 {
 	free_irq(fujitsu.irq, fujitsu_interrupt);
 	release_region(fujitsu.io_base, fujitsu.io_length);
@@ -501,15 +500,14 @@ static int acpi_fujitsu_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(acpi_fujitsu_pm, NULL, acpi_fujitsu_resume);
 
-static struct acpi_driver acpi_fujitsu_driver = {
-	.name  = MODULENAME,
-	.class = "hotkey",
-	.ids   = fujitsu_ids,
-	.ops   = {
-		.add    = acpi_fujitsu_add,
-		.remove	= acpi_fujitsu_remove,
+static struct platform_driver acpi_fujitsu_driver = {
+	.probe = acpi_fujitsu_probe,
+	.remove = acpi_fujitsu_remove,
+	.driver = {
+		.name = MODULENAME,
+		.acpi_match_table = fujitsu_ids,
+		.pm = &acpi_fujitsu_pm,
 	},
-	.drv.pm = &acpi_fujitsu_pm,
 };
 
 static int __init fujitsu_module_init(void)
@@ -518,7 +516,7 @@ static int __init fujitsu_module_init(void)
 
 	dmi_check_system(dmi_ids);
 
-	error = acpi_bus_register_driver(&acpi_fujitsu_driver);
+	error = platform_driver_register(&acpi_fujitsu_driver);
 	if (error)
 		return error;
 
@@ -527,7 +525,7 @@ static int __init fujitsu_module_init(void)
 
 static void __exit fujitsu_module_exit(void)
 {
-	acpi_bus_unregister_driver(&acpi_fujitsu_driver);
+	platform_driver_unregister(&acpi_fujitsu_driver);
 }
 
 module_init(fujitsu_module_init);
diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c
index 68ede7e5757a9..7df00447bba49 100644
--- a/drivers/platform/x86/hp/hp-wmi.c
+++ b/drivers/platform/x86/hp/hp-wmi.c
@@ -58,8 +58,6 @@ enum hp_ec_offsets {
 #define HP_POWER_LIMIT_DEFAULT	 0x00
 #define HP_POWER_LIMIT_NO_CHANGE 0xFF
 
-#define ACPI_AC_CLASS "ac_adapter"
-
 #define zero_if_sup(tmp) (zero_insize_support?0:sizeof(tmp)) // use when zero insize is required
 
 enum hp_thermal_profile_omen_v0 {
@@ -120,6 +118,13 @@ static const struct thermal_profile_params omen_v1_thermal_params = {
 	.ec_tp_offset	= HP_VICTUS_S_EC_THERMAL_PROFILE_OFFSET,
 };
 
+static const struct thermal_profile_params omen_v1_legacy_thermal_params = {
+	.performance	= HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE,
+	.balanced	= HP_OMEN_V1_THERMAL_PROFILE_DEFAULT,
+	.low_power	= HP_OMEN_V1_THERMAL_PROFILE_DEFAULT,
+	.ec_tp_offset	= HP_OMEN_EC_THERMAL_PROFILE_OFFSET,
+};
+
 /*
  * A generic pointer for the currently-active board's thermal profile
  * parameters.
@@ -175,6 +180,10 @@ static const char * const victus_thermal_profile_boards[] = {
 
 /* DMI Board names of Victus 16-r and Victus 16-s laptops */
 static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = {
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A4D") },
+		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") },
 		.driver_data = (void *)&omen_v1_thermal_params,
@@ -183,6 +192,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") },
 		.driver_data = (void *)&victus_s_thermal_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCA") },
+		.driver_data = (void *)&omen_v1_thermal_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") },
 		.driver_data = (void *)&omen_v1_thermal_params,
@@ -195,6 +208,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") },
 		.driver_data = (void *)&victus_s_thermal_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") },
+		.driver_data = (void *)&omen_v1_thermal_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") },
 		.driver_data = (void *)&omen_v1_thermal_params,
diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
index 95c405c8bac02..2ddd8af8c1ce9 100644
--- a/drivers/platform/x86/intel/hid.c
+++ b/drivers/platform/x86/intel/hid.c
@@ -438,6 +438,14 @@ static int intel_hid_pl_suspend_handler(struct device *device)
 	return 0;
 }
 
+static int intel_hid_pl_freeze_handler(struct device *device)
+{
+	struct intel_hid_priv *priv = dev_get_drvdata(device);
+
+	priv->wakeup_mode = false;
+	return intel_hid_pl_suspend_handler(device);
+}
+
 static int intel_hid_pl_resume_handler(struct device *device)
 {
 	intel_hid_pm_complete(device);
@@ -452,7 +460,7 @@ static int intel_hid_pl_resume_handler(struct device *device)
 static const struct dev_pm_ops intel_hid_pl_pm_ops = {
 	.prepare = intel_hid_pm_prepare,
 	.complete = intel_hid_pm_complete,
-	.freeze  = intel_hid_pl_suspend_handler,
+	.freeze  = intel_hid_pl_freeze_handler,
 	.thaw  = intel_hid_pl_resume_handler,
 	.restore  = intel_hid_pl_resume_handler,
 	.suspend  = intel_hid_pl_suspend_handler,
diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c
index 02b303418d185..d91e1ab842d65 100644
--- a/drivers/platform/x86/intel/pmc/core.c
+++ b/drivers/platform/x86/intel/pmc/core.c
@@ -1315,7 +1315,7 @@ static struct telem_endpoint *pmc_core_register_endpoint(struct pci_dev *pcidev,
 	unsigned int i;
 
 	for (i = 0; guids[i]; i++) {
-		ep = pmt_telem_find_and_register_endpoint(pcidev, guids[i], 0);
+		ep = pmt_telem_find_and_register_endpoint(&pcidev->dev, guids[i], 0);
 		if (!IS_ERR(ep))
 			return ep;
 	}
@@ -1600,7 +1600,7 @@ static int pmc_core_get_telem_info(struct pmc_dev *pmcdev, struct pmc_dev_info *
 		if (!pmc->map->lpm_req_guid)
 			return -ENXIO;
 
-		ep = pmt_telem_find_and_register_endpoint(pcidev, pmc->map->lpm_req_guid, 0);
+		ep = pmt_telem_find_and_register_endpoint(&pcidev->dev, pmc->map->lpm_req_guid, 0);
 		if (IS_ERR(ep)) {
 			dev_dbg(&pmcdev->pdev->dev, "couldn't get telem endpoint %pe", ep);
 			return -EPROBE_DEFER;
diff --git a/drivers/platform/x86/intel/pmc/ssram_telemetry.c b/drivers/platform/x86/intel/pmc/ssram_telemetry.c
index 03fad9331fc0c..6f6e83e70fc5a 100644
--- a/drivers/platform/x86/intel/pmc/ssram_telemetry.c
+++ b/drivers/platform/x86/intel/pmc/ssram_telemetry.c
@@ -60,7 +60,7 @@ pmc_ssram_telemetry_add_pmt(struct pci_dev *pcidev, u64 ssram_base, void __iomem
 	info.base_addr = ssram_base;
 	info.parent = &pcidev->dev;
 
-	return intel_vsec_register(pcidev, &info);
+	return intel_vsec_register(&pcidev->dev, &info);
 }
 
 static inline u64 get_base(void __iomem *addr, u32 offset)
diff --git a/drivers/platform/x86/intel/pmt/class.c b/drivers/platform/x86/intel/pmt/class.c
index be3c8d9e4fff2..b4c9964df8077 100644
--- a/drivers/platform/x86/intel/pmt/class.c
+++ b/drivers/platform/x86/intel/pmt/class.c
@@ -60,11 +60,11 @@ pmt_memcpy64_fromio(void *to, const u64 __iomem *from, size_t count)
 	return count;
 }
 
-int pmt_telem_read_mmio(struct pci_dev *pdev, struct pmt_callbacks *cb, u32 guid, void *buf,
+int pmt_telem_read_mmio(struct device *dev, struct pmt_callbacks *cb, u32 guid, void *buf,
 			void __iomem *addr, loff_t off, u32 count)
 {
 	if (cb && cb->read_telem)
-		return cb->read_telem(pdev, guid, buf, off, count);
+		return cb->read_telem(dev, guid, buf, off, count);
 
 	addr += off;
 
@@ -99,7 +99,7 @@ intel_pmt_read(struct file *filp, struct kobject *kobj,
 	if (count > entry->size - off)
 		count = entry->size - off;
 
-	count = pmt_telem_read_mmio(entry->pcidev, entry->cb, entry->header.guid, buf,
+	count = pmt_telem_read_mmio(entry->ep->dev, entry->cb, entry->header.guid, buf,
 				    entry->base, off, count);
 
 	return count;
@@ -208,7 +208,7 @@ static int intel_pmt_populate_entry(struct intel_pmt_entry *entry,
 				    struct intel_vsec_device *ivdev,
 				    struct resource *disc_res)
 {
-	struct pci_dev *pci_dev = ivdev->pcidev;
+	struct pci_dev *pci_dev = to_pci_dev(ivdev->dev);
 	struct device *dev = &ivdev->auxdev.dev;
 	struct intel_pmt_header *header = &entry->header;
 	u8 bir;
diff --git a/drivers/platform/x86/intel/pmt/class.h b/drivers/platform/x86/intel/pmt/class.h
index 3c5ad5f52bca6..1ae56a5baad23 100644
--- a/drivers/platform/x86/intel/pmt/class.h
+++ b/drivers/platform/x86/intel/pmt/class.h
@@ -19,11 +19,12 @@
 #define GET_BIR(v)		((v) & GENMASK(2, 0))
 #define GET_ADDRESS(v)		((v) & GENMASK(31, 3))
 
+struct device;
 struct pci_dev;
 extern struct class intel_pmt_class;
 
 struct telem_endpoint {
-	struct pci_dev		*pcidev;
+	struct device		*dev;
 	struct telem_header	header;
 	struct pmt_callbacks	*cb;
 	void __iomem		*base;
@@ -65,7 +66,7 @@ struct intel_pmt_namespace {
 				struct intel_pmt_entry *entry);
 };
 
-int pmt_telem_read_mmio(struct pci_dev *pdev, struct pmt_callbacks *cb, u32 guid, void *buf,
+int pmt_telem_read_mmio(struct device *dev, struct pmt_callbacks *cb, u32 guid, void *buf,
 			void __iomem *addr, loff_t off, u32 count);
 bool intel_pmt_is_early_client_hw(struct device *dev);
 int intel_pmt_dev_create(struct intel_pmt_entry *entry,
diff --git a/drivers/platform/x86/intel/pmt/discovery.c b/drivers/platform/x86/intel/pmt/discovery.c
index e500aa327d237..c482368bfaaef 100644
--- a/drivers/platform/x86/intel/pmt/discovery.c
+++ b/drivers/platform/x86/intel/pmt/discovery.c
@@ -542,7 +542,7 @@ static int pmt_features_probe(struct auxiliary_device *auxdev, const struct auxi
 	if (!priv)
 		return -ENOMEM;
 
-	priv->parent = &ivdev->pcidev->dev;
+	priv->parent = ivdev->dev;
 	auxiliary_set_drvdata(auxdev, priv);
 
 	priv->dev = device_create(&intel_pmt_class, &auxdev->dev, MKDEV(0, 0), priv,
@@ -609,7 +609,7 @@ void intel_pmt_get_features(struct intel_pmt_entry *entry)
 
 	mutex_lock(&feature_list_lock);
 	list_for_each_entry(feature, &pmt_feature_list, list) {
-		if (feature->priv->parent != &entry->ep->pcidev->dev)
+		if (feature->priv->parent != entry->ep->dev)
 			continue;
 
 		pmt_get_features(entry, feature);
diff --git a/drivers/platform/x86/intel/pmt/telemetry.c b/drivers/platform/x86/intel/pmt/telemetry.c
index a52803bfe124c..bdc7c24a36785 100644
--- a/drivers/platform/x86/intel/pmt/telemetry.c
+++ b/drivers/platform/x86/intel/pmt/telemetry.c
@@ -112,7 +112,7 @@ static int pmt_telem_add_endpoint(struct intel_vsec_device *ivdev,
 		return -ENOMEM;
 
 	ep = entry->ep;
-	ep->pcidev = ivdev->pcidev;
+	ep->dev = ivdev->dev;
 	ep->header.access_type = entry->header.access_type;
 	ep->header.guid = entry->header.guid;
 	ep->header.base_offset = entry->header.base_offset;
@@ -204,7 +204,7 @@ int pmt_telem_get_endpoint_info(int devid, struct telem_endpoint_info *info)
 		goto unlock;
 	}
 
-	info->pdev = entry->ep->pcidev;
+	info->dev = entry->ep->dev;
 	info->header = entry->ep->header;
 
 unlock:
@@ -218,9 +218,10 @@ static int pmt_copy_region(struct telemetry_region *region,
 			   struct intel_pmt_entry *entry)
 {
 
+	struct pci_dev *pdev = to_pci_dev(entry->ep->dev);
 	struct oobmsm_plat_info *plat_info;
 
-	plat_info = intel_vsec_get_mapping(entry->ep->pcidev);
+	plat_info = intel_vsec_get_mapping(pdev);
 	if (IS_ERR(plat_info))
 		return PTR_ERR(plat_info);
 
@@ -308,7 +309,7 @@ int pmt_telem_read(struct telem_endpoint *ep, u32 id, u64 *data, u32 count)
 	if (offset + NUM_BYTES_QWORD(count) > size)
 		return -EINVAL;
 
-	pmt_telem_read_mmio(ep->pcidev, ep->cb, ep->header.guid, data, ep->base, offset,
+	pmt_telem_read_mmio(ep->dev, ep->cb, ep->header.guid, data, ep->base, offset,
 			    NUM_BYTES_QWORD(count));
 
 	return ep->present ? 0 : -EPIPE;
@@ -335,7 +336,7 @@ int pmt_telem_read32(struct telem_endpoint *ep, u32 id, u32 *data, u32 count)
 EXPORT_SYMBOL_NS_GPL(pmt_telem_read32, "INTEL_PMT_TELEMETRY");
 
 struct telem_endpoint *
-pmt_telem_find_and_register_endpoint(struct pci_dev *pcidev, u32 guid, u16 pos)
+pmt_telem_find_and_register_endpoint(struct device *dev, u32 guid, u16 pos)
 {
 	int devid = 0;
 	int inst = 0;
@@ -348,7 +349,7 @@ pmt_telem_find_and_register_endpoint(struct pci_dev *pcidev, u32 guid, u16 pos)
 		if (err)
 			return ERR_PTR(err);
 
-		if (ep_info.header.guid == guid && ep_info.pdev == pcidev) {
+		if (ep_info.header.guid == guid && ep_info.dev == dev) {
 			if (inst == pos)
 				return pmt_telem_register_endpoint(devid);
 			++inst;
diff --git a/drivers/platform/x86/intel/pmt/telemetry.h b/drivers/platform/x86/intel/pmt/telemetry.h
index d45af5512b4e2..0f88c5e7d90e9 100644
--- a/drivers/platform/x86/intel/pmt/telemetry.h
+++ b/drivers/platform/x86/intel/pmt/telemetry.h
@@ -6,8 +6,8 @@
 #define PMT_TELEM_TELEMETRY	0
 #define PMT_TELEM_CRASHLOG	1
 
+struct device;
 struct telem_endpoint;
-struct pci_dev;
 
 struct telem_header {
 	u8	access_type;
@@ -17,7 +17,7 @@ struct telem_header {
 };
 
 struct telem_endpoint_info {
-	struct pci_dev		*pdev;
+	struct device		*dev;
 	struct telem_header	header;
 };
 
@@ -71,8 +71,8 @@ int pmt_telem_get_endpoint_info(int devid, struct telem_endpoint_info *info);
 
 /**
  * pmt_telem_find_and_register_endpoint() - Get a telemetry endpoint from
- * pci_dev device, guid and pos
- * @pdev:   PCI device inside the Intel vsec
+ * device, guid and pos
+ * @dev:    device inside the Intel vsec
  * @guid:   GUID of the telemetry space
  * @pos:    Instance of the guid
  *
@@ -80,8 +80,8 @@ int pmt_telem_get_endpoint_info(int devid, struct telem_endpoint_info *info);
  * * endpoint    - On success returns pointer to the telemetry endpoint
  * * -ENXIO      - telemetry endpoint not found
  */
-struct telem_endpoint *pmt_telem_find_and_register_endpoint(struct pci_dev *pcidev,
-				u32 guid, u16 pos);
+struct telem_endpoint *
+pmt_telem_find_and_register_endpoint(struct device *dev, u32 guid, u16 pos);
 
 /**
  * pmt_telem_read() - Read qwords from counter sram using sample id
diff --git a/drivers/platform/x86/intel/rst.c b/drivers/platform/x86/intel/rst.c
index f3a60e14d4c17..4bd10927aad99 100644
--- a/drivers/platform/x86/intel/rst.c
+++ b/drivers/platform/x86/intel/rst.c
@@ -5,6 +5,7 @@
 
 #include <linux/acpi.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/slab.h>
 
 MODULE_DESCRIPTION("Intel Rapid Start Technology Driver");
@@ -99,8 +100,9 @@ static struct device_attribute irst_timeout_attr = {
 	.store = irst_store_wakeup_time
 };
 
-static int irst_add(struct acpi_device *acpi)
+static int irst_probe(struct platform_device *pdev)
 {
+	struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
 	int error;
 
 	error = device_create_file(&acpi->dev, &irst_timeout_attr);
@@ -114,8 +116,10 @@ static int irst_add(struct acpi_device *acpi)
 	return error;
 }
 
-static void irst_remove(struct acpi_device *acpi)
+static void irst_remove(struct platform_device *pdev)
 {
+	struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
+
 	device_remove_file(&acpi->dev, &irst_wakeup_attr);
 	device_remove_file(&acpi->dev, &irst_timeout_attr);
 }
@@ -125,16 +129,15 @@ static const struct acpi_device_id irst_ids[] = {
 	{"", 0}
 };
 
-static struct acpi_driver irst_driver = {
-	.name = "intel_rapid_start",
-	.class = "intel_rapid_start",
-	.ids = irst_ids,
-	.ops = {
-		.add = irst_add,
-		.remove = irst_remove,
+static struct platform_driver irst_driver = {
+	.probe = irst_probe,
+	.remove = irst_remove,
+	.driver = {
+		.name = "intel_rapid_start",
+		.acpi_match_table = irst_ids,
 	},
 };
 
-module_acpi_driver(irst_driver);
+module_platform_driver(irst_driver);
 
 MODULE_DEVICE_TABLE(acpi, irst_ids);
diff --git a/drivers/platform/x86/intel/sdsi.c b/drivers/platform/x86/intel/sdsi.c
index da75f53d0bcc0..d7e37d4ace238 100644
--- a/drivers/platform/x86/intel/sdsi.c
+++ b/drivers/platform/x86/intel/sdsi.c
@@ -599,13 +599,14 @@ static int sdsi_get_layout(struct sdsi_priv *priv, struct disc_table *table)
 	return 0;
 }
 
-static int sdsi_map_mbox_registers(struct sdsi_priv *priv, struct pci_dev *parent,
+static int sdsi_map_mbox_registers(struct sdsi_priv *priv, struct device *dev,
 				   struct disc_table *disc_table, struct resource *disc_res)
 {
 	u32 access_type = FIELD_GET(DT_ACCESS_TYPE, disc_table->access_info);
 	u32 size = FIELD_GET(DT_SIZE, disc_table->access_info);
 	u32 tbir = FIELD_GET(DT_TBIR, disc_table->offset);
 	u32 offset = DT_OFFSET(disc_table->offset);
+	struct pci_dev *parent = to_pci_dev(dev);
 	struct resource res = {};
 
 	/* Starting location of SDSi MMIO region based on access type */
@@ -681,7 +682,7 @@ static int sdsi_probe(struct auxiliary_device *auxdev, const struct auxiliary_de
 		return ret;
 
 	/* Map the SDSi mailbox registers */
-	ret = sdsi_map_mbox_registers(priv, intel_cap_dev->pcidev, &disc_table, disc_res);
+	ret = sdsi_map_mbox_registers(priv, intel_cap_dev->dev, &disc_table, disc_res);
 	if (ret)
 		return ret;
 
diff --git a/drivers/platform/x86/intel/smartconnect.c b/drivers/platform/x86/intel/smartconnect.c
index 31019a1a6d5e8..4d866b6366d69 100644
--- a/drivers/platform/x86/intel/smartconnect.c
+++ b/drivers/platform/x86/intel/smartconnect.c
@@ -5,22 +5,24 @@
 
 #include <linux/acpi.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 
 MODULE_DESCRIPTION("Intel Smart Connect disabling driver");
 MODULE_LICENSE("GPL");
 
-static int smartconnect_acpi_init(struct acpi_device *acpi)
+static int smartconnect_acpi_probe(struct platform_device *pdev)
 {
+	acpi_handle handle = ACPI_HANDLE(&pdev->dev);
 	unsigned long long value;
 	acpi_status status;
 
-	status = acpi_evaluate_integer(acpi->handle, "GAOS", NULL, &value);
+	status = acpi_evaluate_integer(handle, "GAOS", NULL, &value);
 	if (ACPI_FAILURE(status))
 		return -EINVAL;
 
 	if (value & 0x1) {
-		dev_info(&acpi->dev, "Disabling Intel Smart Connect\n");
-		status = acpi_execute_simple_method(acpi->handle, "SAOS", 0);
+		dev_info(&pdev->dev, "Disabling Intel Smart Connect\n");
+		status = acpi_execute_simple_method(handle, "SAOS", 0);
 	}
 
 	return 0;
@@ -32,13 +34,12 @@ static const struct acpi_device_id smartconnect_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, smartconnect_ids);
 
-static struct acpi_driver smartconnect_driver = {
-	.name = "intel_smart_connect",
-	.class = "intel_smart_connect",
-	.ids = smartconnect_ids,
-	.ops = {
-		.add = smartconnect_acpi_init,
+static struct platform_driver smartconnect_driver = {
+	.probe = smartconnect_acpi_probe,
+	.driver = {
+		.name = "intel_smart_connect",
+		.acpi_match_table = smartconnect_ids,
 	},
 };
 
-module_acpi_driver(smartconnect_driver);
+module_platform_driver(smartconnect_driver);
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
index b8cdaa233ea9c..6f46cbb2898fd 100644
--- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
+++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
@@ -558,6 +558,9 @@ static bool disable_dynamic_sst_features(void)
 {
 	u64 value;
 
+	if (!static_cpu_has(X86_FEATURE_HWP))
+		return true;
+
 	rdmsrq(MSR_PM_ENABLE, value);
 	return !(value & 0x1);
 }
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 5059d320edf8c..7d5dbc1c1d05a 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -24,7 +24,9 @@
 #include <linux/intel_vsec.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
+#include <linux/string.h>
 #include <linux/types.h>
 
 #define PMT_XA_START			0
@@ -42,7 +44,7 @@ enum vsec_device_state {
 };
 
 struct vsec_priv {
-	struct intel_vsec_platform_info *info;
+	const struct intel_vsec_platform_info *info;
 	struct device *suppliers[VSEC_FEATURE_COUNT];
 	struct oobmsm_plat_info plat_info;
 	enum vsec_device_state state[VSEC_FEATURE_COUNT];
@@ -109,6 +111,7 @@ static void intel_vsec_dev_release(struct device *dev)
 
 	ida_free(intel_vsec_dev->ida, intel_vsec_dev->auxdev.id);
 
+	kfree(intel_vsec_dev->acpi_disc);
 	kfree(intel_vsec_dev->resource);
 	kfree(intel_vsec_dev);
 }
@@ -158,18 +161,23 @@ static bool vsec_driver_present(int cap_id)
  */
 static const struct pci_device_id intel_vsec_pci_ids[];
 
-static int intel_vsec_link_devices(struct pci_dev *pdev, struct device *dev,
+static int intel_vsec_link_devices(struct device *parent, struct device *dev,
 				   int consumer_id)
 {
 	const struct vsec_feature_dependency *deps;
 	enum vsec_device_state *state;
 	struct device **suppliers;
 	struct vsec_priv *priv;
+	struct pci_dev *pdev;
 	int supplier_id;
 
 	if (!consumer_id)
 		return 0;
 
+	if (!dev_is_pci(parent))
+		return 0;
+
+	pdev = to_pci_dev(parent);
 	if (!pci_match_id(intel_vsec_pci_ids, pdev))
 		return 0;
 
@@ -204,7 +212,7 @@ static int intel_vsec_link_devices(struct pci_dev *pdev, struct device *dev,
 	return 0;
 }
 
-int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent,
+int intel_vsec_add_aux(struct device *parent,
 		       struct intel_vsec_device *intel_vsec_dev,
 		       const char *name)
 {
@@ -252,7 +260,7 @@ int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent,
 	if (ret)
 		goto cleanup_aux;
 
-	ret = intel_vsec_link_devices(pdev, &auxdev->dev, intel_vsec_dev->cap_id);
+	ret = intel_vsec_link_devices(parent, &auxdev->dev, intel_vsec_dev->cap_id);
 	if (ret)
 		goto cleanup_aux;
 
@@ -269,33 +277,32 @@ int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent,
 }
 EXPORT_SYMBOL_NS_GPL(intel_vsec_add_aux, "INTEL_VSEC");
 
-static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *header,
-			      struct intel_vsec_platform_info *info,
-			      unsigned long cap_id)
+static int intel_vsec_add_dev(struct device *dev, struct intel_vsec_header *header,
+			      const struct intel_vsec_platform_info *info,
+			      unsigned long cap_id, u64 base_addr)
 {
 	struct intel_vsec_device __free(kfree) *intel_vsec_dev = NULL;
 	struct resource __free(kfree) *res = NULL;
 	struct resource *tmp;
 	struct device *parent;
 	unsigned long quirks = info->quirks;
-	u64 base_addr;
 	int i;
 
 	if (info->parent)
 		parent = info->parent;
 	else
-		parent = &pdev->dev;
+		parent = dev;
 
 	if (!intel_vsec_supported(header->id, info->caps))
 		return -EINVAL;
 
 	if (!header->num_entries) {
-		dev_dbg(&pdev->dev, "Invalid 0 entry count for header id %d\n", header->id);
+		dev_dbg(dev, "Invalid 0 entry count for header id %d\n", header->id);
 		return -EINVAL;
 	}
 
 	if (!header->entry_size) {
-		dev_dbg(&pdev->dev, "Invalid 0 entry size for header id %d\n", header->id);
+		dev_dbg(dev, "Invalid 0 entry size for header id %d\n", header->id);
 		return -EINVAL;
 	}
 
@@ -310,17 +317,19 @@ static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *he
 	if (quirks & VSEC_QUIRK_TABLE_SHIFT)
 		header->offset >>= TABLE_OFFSET_SHIFT;
 
-	if (info->base_addr)
-		base_addr = info->base_addr;
-	else
-		base_addr = pdev->resource[header->tbir].start;
-
 	/*
 	 * The DVSEC/VSEC contains the starting offset and count for a block of
 	 * discovery tables. Create a resource array of these tables to the
 	 * auxiliary device driver.
 	 */
 	for (i = 0, tmp = res; i < header->num_entries; i++, tmp++) {
+		/*
+		 * Skip resource mapping check for ACPI-based discovery
+		 * since those tables are read from _DSD, not MMIO.
+		 */
+		if (info->src == INTEL_VSEC_DISC_ACPI)
+			break;
+
 		tmp->start = base_addr + header->offset + i * (header->entry_size * sizeof(u32));
 		tmp->end = tmp->start + (header->entry_size * sizeof(u32)) - 1;
 		tmp->flags = IORESOURCE_MEM;
@@ -332,13 +341,26 @@ static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *he
 		release_mem_region(tmp->start, resource_size(tmp));
 	}
 
-	intel_vsec_dev->pcidev = pdev;
+	intel_vsec_dev->dev = dev;
 	intel_vsec_dev->resource = no_free_ptr(res);
 	intel_vsec_dev->num_resources = header->num_entries;
 	intel_vsec_dev->quirks = info->quirks;
 	intel_vsec_dev->base_addr = info->base_addr;
 	intel_vsec_dev->priv_data = info->priv_data;
 	intel_vsec_dev->cap_id = cap_id;
+	intel_vsec_dev->src = info->src;
+
+	if (info->src == INTEL_VSEC_DISC_ACPI) {
+		size_t bytes;
+
+		if (check_mul_overflow(intel_vsec_dev->num_resources,
+				       sizeof(*info->acpi_disc), &bytes))
+			return -EOVERFLOW;
+
+		intel_vsec_dev->acpi_disc = kmemdup(info->acpi_disc, bytes, GFP_KERNEL);
+		if (!intel_vsec_dev->acpi_disc)
+			return -ENOMEM;
+	}
 
 	if (header->id == VSEC_ID_SDSI)
 		intel_vsec_dev->ida = &intel_vsec_sdsi_ida;
@@ -349,7 +371,7 @@ static int intel_vsec_add_dev(struct pci_dev *pdev, struct intel_vsec_header *he
 	 * Pass the ownership of intel_vsec_dev and resource within it to
 	 * intel_vsec_add_aux()
 	 */
-	return intel_vsec_add_aux(pdev, parent, no_free_ptr(intel_vsec_dev),
+	return intel_vsec_add_aux(parent, no_free_ptr(intel_vsec_dev),
 				  intel_vsec_name(header->id));
 }
 
@@ -410,12 +432,14 @@ static int get_cap_id(u32 header_id, unsigned long *cap_id)
 	return 0;
 }
 
-static int intel_vsec_register_device(struct pci_dev *pdev,
+static int intel_vsec_register_device(struct device *dev,
 				      struct intel_vsec_header *header,
-				      struct intel_vsec_platform_info *info)
+				      const struct intel_vsec_platform_info *info,
+				      u64 base_addr)
 {
 	const struct vsec_feature_dependency *consumer_deps;
 	struct vsec_priv *priv;
+	struct pci_dev *pdev;
 	unsigned long cap_id;
 	int ret;
 
@@ -427,8 +451,12 @@ static int intel_vsec_register_device(struct pci_dev *pdev,
 	 * Only track dependencies for devices probed by the VSEC driver.
 	 * For others using the exported APIs, add the device directly.
 	 */
+	if (!dev_is_pci(dev))
+		return intel_vsec_add_dev(dev, header, info, cap_id, base_addr);
+
+	pdev = to_pci_dev(dev);
 	if (!pci_match_id(intel_vsec_pci_ids, pdev))
-		return intel_vsec_add_dev(pdev, header, info, cap_id);
+		return intel_vsec_add_dev(dev, header, info, cap_id, base_addr);
 
 	priv = pci_get_drvdata(pdev);
 	if (priv->state[cap_id] == STATE_REGISTERED ||
@@ -444,7 +472,7 @@ static int intel_vsec_register_device(struct pci_dev *pdev,
 
 	consumer_deps = get_consumer_dependencies(priv, cap_id);
 	if (!consumer_deps || suppliers_ready(priv, consumer_deps, cap_id)) {
-		ret = intel_vsec_add_dev(pdev, header, info, cap_id);
+		ret = intel_vsec_add_dev(dev, header, info, cap_id, base_addr);
 		if (ret)
 			priv->state[cap_id] = STATE_SKIP;
 		else
@@ -456,24 +484,23 @@ static int intel_vsec_register_device(struct pci_dev *pdev,
 	return -EAGAIN;
 }
 
-static bool intel_vsec_walk_header(struct pci_dev *pdev,
-				   struct intel_vsec_platform_info *info)
+static int intel_vsec_walk_header(struct device *dev,
+				  const struct intel_vsec_platform_info *info)
 {
 	struct intel_vsec_header **header = info->headers;
-	bool have_devices = false;
 	int ret;
 
 	for ( ; *header; header++) {
-		ret = intel_vsec_register_device(pdev, *header, info);
-		if (!ret)
-			have_devices = true;
+		ret = intel_vsec_register_device(dev, *header, info, info->base_addr);
+		if (ret)
+			return ret;
 	}
 
-	return have_devices;
+	return 0;
 }
 
 static bool intel_vsec_walk_dvsec(struct pci_dev *pdev,
-				  struct intel_vsec_platform_info *info)
+				  const struct intel_vsec_platform_info *info)
 {
 	bool have_devices = false;
 	int pos = 0;
@@ -512,7 +539,8 @@ static bool intel_vsec_walk_dvsec(struct pci_dev *pdev,
 		pci_read_config_dword(pdev, pos + PCI_DVSEC_HEADER2, &hdr);
 		header.id = PCI_DVSEC_HEADER2_ID(hdr);
 
-		ret = intel_vsec_register_device(pdev, &header, info);
+		ret = intel_vsec_register_device(&pdev->dev, &header, info,
+						 pci_resource_start(pdev, header.tbir));
 		if (ret)
 			continue;
 
@@ -523,7 +551,7 @@ static bool intel_vsec_walk_dvsec(struct pci_dev *pdev,
 }
 
 static bool intel_vsec_walk_vsec(struct pci_dev *pdev,
-				 struct intel_vsec_platform_info *info)
+				 const struct intel_vsec_platform_info *info)
 {
 	bool have_devices = false;
 	int pos = 0;
@@ -557,7 +585,8 @@ static bool intel_vsec_walk_vsec(struct pci_dev *pdev,
 		header.tbir = INTEL_DVSEC_TABLE_BAR(table);
 		header.offset = INTEL_DVSEC_TABLE_OFFSET(table);
 
-		ret = intel_vsec_register_device(pdev, &header, info);
+		ret = intel_vsec_register_device(&pdev->dev, &header, info,
+						 pci_resource_start(pdev, header.tbir));
 		if (ret)
 			continue;
 
@@ -567,21 +596,18 @@ static bool intel_vsec_walk_vsec(struct pci_dev *pdev,
 	return have_devices;
 }
 
-int intel_vsec_register(struct pci_dev *pdev,
-			 struct intel_vsec_platform_info *info)
+int intel_vsec_register(struct device *dev,
+			const struct intel_vsec_platform_info *info)
 {
-	if (!pdev || !info || !info->headers)
+	if (!dev || !info || !info->headers)
 		return -EINVAL;
 
-	if (!intel_vsec_walk_header(pdev, info))
-		return -ENODEV;
-	else
-		return 0;
+	return intel_vsec_walk_header(dev, info);
 }
 EXPORT_SYMBOL_NS_GPL(intel_vsec_register, "INTEL_VSEC");
 
 static bool intel_vsec_get_features(struct pci_dev *pdev,
-				    struct intel_vsec_platform_info *info)
+				    const struct intel_vsec_platform_info *info)
 {
 	bool found = false;
 
@@ -599,7 +625,7 @@ static bool intel_vsec_get_features(struct pci_dev *pdev,
 		found = true;
 
 	if (info && (info->quirks & VSEC_QUIRK_NO_DVSEC) &&
-	    intel_vsec_walk_header(pdev, info))
+	    intel_vsec_walk_header(&pdev->dev, info))
 		found = true;
 
 	return found;
@@ -625,7 +651,7 @@ static void intel_vsec_skip_missing_dependencies(struct pci_dev *pdev)
 
 static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-	struct intel_vsec_platform_info *info;
+	const struct intel_vsec_platform_info *info;
 	struct vsec_priv *priv;
 	int num_caps, ret;
 	int run_once = 0;
@@ -636,7 +662,7 @@ static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id
 		return ret;
 
 	pci_save_state(pdev);
-	info = (struct intel_vsec_platform_info *)id->driver_data;
+	info = (const struct intel_vsec_platform_info *)id->driver_data;
 	if (!info)
 		return -EINVAL;
 
@@ -671,7 +697,10 @@ int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info,
 {
 	struct vsec_priv *priv;
 
-	priv = pci_get_drvdata(vsec_dev->pcidev);
+	if (!dev_is_pci(vsec_dev->dev))
+		return -ENODEV;
+
+	priv = pci_get_drvdata(to_pci_dev(vsec_dev->dev));
 	if (!priv)
 		return -EINVAL;
 
@@ -819,7 +848,7 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
 
 	xa_for_each(&auxdev_array, index, intel_vsec_dev) {
 		/* check if pdev doesn't match */
-		if (pdev != intel_vsec_dev->pcidev)
+		if (&pdev->dev != intel_vsec_dev->dev)
 			continue;
 		devm_release_action(&pdev->dev, intel_vsec_remove_aux,
 				    &intel_vsec_dev->auxdev);
diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c
index 98846e88d3d0c..9dddf4e5863e5 100644
--- a/drivers/platform/x86/intel/vsec_tpmi.c
+++ b/drivers/platform/x86/intel/vsec_tpmi.c
@@ -530,7 +530,7 @@ static const struct file_operations mem_write_ops = {
 	.release        = single_release,
 };
 
-#define tpmi_to_dev(info)	(&info->vsec_dev->pcidev->dev)
+#define tpmi_to_dev(info)	((info)->vsec_dev->dev)
 
 static void tpmi_dbgfs_register(struct intel_tpmi_info *tpmi_info)
 {
@@ -642,7 +642,7 @@ static int tpmi_create_device(struct intel_tpmi_info *tpmi_info,
 		tmp->flags = IORESOURCE_MEM;
 	}
 
-	feature_vsec_dev->pcidev = vsec_dev->pcidev;
+	feature_vsec_dev->dev = vsec_dev->dev;
 	feature_vsec_dev->resource = res;
 	feature_vsec_dev->num_resources = pfs->pfs_header.num_entries;
 	feature_vsec_dev->priv_data = &tpmi_info->plat_info;
@@ -655,7 +655,7 @@ static int tpmi_create_device(struct intel_tpmi_info *tpmi_info,
 	 * feature_vsec_dev and res memory are also freed as part of
 	 * device deletion.
 	 */
-	return intel_vsec_add_aux(vsec_dev->pcidev, &vsec_dev->auxdev.dev,
+	return intel_vsec_add_aux(&vsec_dev->auxdev.dev,
 				  feature_vsec_dev, feature_id_name);
 }
 
@@ -742,7 +742,7 @@ static int tpmi_fetch_pfs_header(struct intel_tpmi_pm_feature *pfs, u64 start, i
 static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev)
 {
 	struct intel_vsec_device *vsec_dev = auxdev_to_ivdev(auxdev);
-	struct pci_dev *pci_dev = vsec_dev->pcidev;
+	struct pci_dev *pci_dev = to_pci_dev(vsec_dev->dev);
 	struct intel_tpmi_info *tpmi_info;
 	u64 pfs_start = 0;
 	int ret, i;
diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c
index 8982d92dfd977..90da69b0e4170 100644
--- a/drivers/platform/x86/lenovo/thinkpad_acpi.c
+++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c
@@ -374,7 +374,7 @@ static struct {
 	u32 hotkey_poll_active:1;
 	u32 has_adaptive_kbd:1;
 	u32 kbd_lang:1;
-	u32 trackpoint_doubletap:1;
+	u32 trackpoint_doubletap_enable:1;
 	struct quirk_entry *quirks;
 } tp_features;
 
@@ -1315,7 +1315,7 @@ static ssize_t tpacpi_rfk_sysfs_enable_store(const enum tpacpi_rfk_id id,
 static int tpacpi_rfk_procfs_read(const enum tpacpi_rfk_id id, struct seq_file *m)
 {
 	if (id >= TPACPI_RFK_SW_MAX)
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	else {
 		int status;
 
@@ -1330,7 +1330,7 @@ static int tpacpi_rfk_procfs_read(const enum tpacpi_rfk_id id, struct seq_file *
 		}
 
 		seq_printf(m, "status:\t\t%s\n", str_enabled_disabled(status == TPACPI_RFK_RADIO_ON));
-		seq_printf(m, "commands:\tenable, disable\n");
+		seq_puts(m, "commands:\tenable, disable\n");
 	}
 
 	return 0;
@@ -3019,6 +3019,31 @@ static const struct attribute_group adaptive_kbd_attr_group = {
 	.attrs = adaptive_kbd_attributes,
 };
 
+/* sysfs doubletap enable --------------------------------------------- */
+static ssize_t doubletap_enable_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	return sysfs_emit(buf, "%d\n", tp_features.trackpoint_doubletap_enable);
+}
+
+static ssize_t doubletap_enable_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t count)
+{
+	bool enable;
+	int err;
+
+	err = kstrtobool(buf, &enable);
+	if (err)
+		return err;
+
+	tp_features.trackpoint_doubletap_enable = enable;
+	return count;
+}
+
+static DEVICE_ATTR_RW(doubletap_enable);
+
 /* --------------------------------------------------------------------- */
 
 static struct attribute *hotkey_attributes[] = {
@@ -3033,6 +3058,7 @@ static struct attribute *hotkey_attributes[] = {
 	&dev_attr_hotkey_recommended_mask.attr,
 	&dev_attr_hotkey_tablet_mode.attr,
 	&dev_attr_hotkey_radio_sw.attr,
+	&dev_attr_doubletap_enable.attr,
 #ifdef CONFIG_THINKPAD_ACPI_HOTKEY_POLL
 	&dev_attr_hotkey_source_mask.attr,
 	&dev_attr_hotkey_poll_freq.attr,
@@ -3558,8 +3584,8 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 
 	hotkey_poll_setup_safe(true);
 
-	/* Enable doubletap by default */
-	tp_features.trackpoint_doubletap = 1;
+	/* Enable TrackPoint doubletap event reporting by default. */
+	tp_features.trackpoint_doubletap_enable = 1;
 
 	return 0;
 }
@@ -3864,9 +3890,9 @@ static bool hotkey_notify_8xxx(const u32 hkey, bool *send_acpi_ev)
 {
 	switch (hkey) {
 	case TP_HKEY_EV_TRACK_DOUBLETAP:
-		if (tp_features.trackpoint_doubletap)
-			tpacpi_input_send_key(hkey, send_acpi_ev);
-
+		/* Only send event if doubletap is enabled */
+		if (!tp_features.trackpoint_doubletap_enable)
+			*send_acpi_ev = false;
 		return true;
 	default:
 		return false;
@@ -4017,7 +4043,7 @@ static int hotkey_read(struct seq_file *m)
 	int res, status;
 
 	if (!tp_features.hotkey) {
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 		return 0;
 	}
 
@@ -4033,10 +4059,10 @@ static int hotkey_read(struct seq_file *m)
 	seq_printf(m, "status:\t\t%s\n", str_enabled_disabled(status & BIT(0)));
 	if (hotkey_all_mask) {
 		seq_printf(m, "mask:\t\t0x%08x\n", hotkey_user_mask);
-		seq_printf(m, "commands:\tenable, disable, reset, <mask>\n");
+		seq_puts(m, "commands:\tenable, disable, reset, <mask>\n");
 	} else {
-		seq_printf(m, "mask:\t\tnot supported\n");
-		seq_printf(m, "commands:\tenable, disable, reset\n");
+		seq_puts(m, "mask:\t\tnot supported\n");
+		seq_puts(m, "commands:\tenable, disable, reset\n");
 	}
 
 	return 0;
@@ -4933,7 +4959,7 @@ static int video_read(struct seq_file *m)
 	int status, autosw;
 
 	if (video_supported == TPACPI_VIDEO_NONE) {
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 		return 0;
 	}
 
@@ -4949,18 +4975,18 @@ static int video_read(struct seq_file *m)
 	if (autosw < 0)
 		return autosw;
 
-	seq_printf(m, "status:\t\tsupported\n");
+	seq_puts(m, "status:\t\tsupported\n");
 	seq_printf(m, "lcd:\t\t%s\n", str_enabled_disabled(status & BIT(0)));
 	seq_printf(m, "crt:\t\t%s\n", str_enabled_disabled(status & BIT(1)));
 	if (video_supported == TPACPI_VIDEO_NEW)
 		seq_printf(m, "dvi:\t\t%s\n", str_enabled_disabled(status & BIT(3)));
 	seq_printf(m, "auto:\t\t%s\n", str_enabled_disabled(autosw & BIT(0)));
-	seq_printf(m, "commands:\tlcd_enable, lcd_disable\n");
-	seq_printf(m, "commands:\tcrt_enable, crt_disable\n");
+	seq_puts(m, "commands:\tlcd_enable, lcd_disable\n");
+	seq_puts(m, "commands:\tcrt_enable, crt_disable\n");
 	if (video_supported == TPACPI_VIDEO_NEW)
-		seq_printf(m, "commands:\tdvi_enable, dvi_disable\n");
-	seq_printf(m, "commands:\tauto_enable, auto_disable\n");
-	seq_printf(m, "commands:\tvideo_switch, expand_toggle\n");
+		seq_puts(m, "commands:\tdvi_enable, dvi_disable\n");
+	seq_puts(m, "commands:\tauto_enable, auto_disable\n");
+	seq_puts(m, "commands:\tvideo_switch, expand_toggle\n");
 
 	return 0;
 }
@@ -5204,14 +5230,14 @@ static int kbdlight_read(struct seq_file *m)
 	int level;
 
 	if (!tp_features.kbdlight) {
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	} else {
 		level = kbdlight_get_level();
 		if (level < 0)
 			seq_printf(m, "status:\t\terror %d\n", level);
 		else
 			seq_printf(m, "status:\t\t%d\n", level);
-		seq_printf(m, "commands:\t0, 1, 2\n");
+		seq_puts(m, "commands:\t0, 1, 2\n");
 	}
 
 	return 0;
@@ -5378,16 +5404,16 @@ static int light_read(struct seq_file *m)
 	int status;
 
 	if (!tp_features.light) {
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	} else if (!tp_features.light_status) {
-		seq_printf(m, "status:\t\tunknown\n");
-		seq_printf(m, "commands:\ton, off\n");
+		seq_puts(m, "status:\t\tunknown\n");
+		seq_puts(m, "commands:\ton, off\n");
 	} else {
 		status = light_get_status();
 		if (status < 0)
 			return status;
 		seq_printf(m, "status:\t\t%s\n", str_on_off(status & BIT(0)));
-		seq_printf(m, "commands:\ton, off\n");
+		seq_puts(m, "commands:\ton, off\n");
 	}
 
 	return 0;
@@ -5477,10 +5503,10 @@ static int cmos_read(struct seq_file *m)
 	/* cmos not supported on 570, 600e/x, 770e, 770x, A21e, A2xm/p,
 	   R30, R31, T20-22, X20-21 */
 	if (!cmos_handle)
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	else {
-		seq_printf(m, "status:\t\tsupported\n");
-		seq_printf(m, "commands:\t<cmd> (<cmd> is 0-21)\n");
+		seq_puts(m, "status:\t\tsupported\n");
+		seq_puts(m, "commands:\t<cmd> (<cmd> is 0-21)\n");
 	}
 
 	return 0;
@@ -5846,10 +5872,10 @@ static int __init led_init(struct ibm_init_struct *iibm)
 static int led_read(struct seq_file *m)
 {
 	if (!led_supported) {
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 		return 0;
 	}
-	seq_printf(m, "status:\t\tsupported\n");
+	seq_puts(m, "status:\t\tsupported\n");
 
 	if (led_supported == TPACPI_LED_570) {
 		/* 570 */
@@ -5862,7 +5888,7 @@ static int led_read(struct seq_file *m)
 		}
 	}
 
-	seq_printf(m, "commands:\t<led> on, <led> off, <led> blink (<led> is 0-15)\n");
+	seq_puts(m, "commands:\t<led> on, <led> off, <led> blink (<led> is 0-15)\n");
 
 	return 0;
 }
@@ -5946,10 +5972,10 @@ static int __init beep_init(struct ibm_init_struct *iibm)
 static int beep_read(struct seq_file *m)
 {
 	if (!beep_handle)
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	else {
-		seq_printf(m, "status:\t\tsupported\n");
-		seq_printf(m, "commands:\t<cmd> (<cmd> is 0-17)\n");
+		seq_puts(m, "status:\t\tsupported\n");
+		seq_puts(m, "commands:\t<cmd> (<cmd> is 0-17)\n");
 	}
 
 	return 0;
@@ -6398,14 +6424,14 @@ static int thermal_read(struct seq_file *m)
 	if (unlikely(n < 0))
 		return n;
 
-	seq_printf(m, "temperatures:\t");
+	seq_puts(m, "temperatures:\t");
 
 	if (n > 0) {
 		for (i = 0; i < (n - 1); i++)
 			seq_printf(m, "%d ", t.temp[i] / 1000);
 		seq_printf(m, "%d\n", t.temp[i] / 1000);
 	} else
-		seq_printf(m, "not supported\n");
+		seq_puts(m, "not supported\n");
 
 	return 0;
 }
@@ -6918,10 +6944,10 @@ static int brightness_read(struct seq_file *m)
 
 	level = brightness_get(NULL);
 	if (level < 0) {
-		seq_printf(m, "level:\t\tunreadable\n");
+		seq_puts(m, "level:\t\tunreadable\n");
 	} else {
 		seq_printf(m, "level:\t\t%d\n", level);
-		seq_printf(m, "commands:\tup, down\n");
+		seq_puts(m, "commands:\tup, down\n");
 		seq_printf(m, "commands:\tlevel <level> (<level> is 0-%d)\n",
 			       bright_maxlvl);
 	}
@@ -7637,10 +7663,10 @@ static int volume_read(struct seq_file *m)
 	u8 status;
 
 	if (volume_get_status(&status) < 0) {
-		seq_printf(m, "level:\t\tunreadable\n");
+		seq_puts(m, "level:\t\tunreadable\n");
 	} else {
 		if (tp_features.mixer_no_level_control)
-			seq_printf(m, "level:\t\tunsupported\n");
+			seq_puts(m, "level:\t\tunsupported\n");
 		else
 			seq_printf(m, "level:\t\t%d\n",
 					status & TP_EC_AUDIO_LVL_MSK);
@@ -7648,9 +7674,9 @@ static int volume_read(struct seq_file *m)
 		seq_printf(m, "mute:\t\t%s\n", str_on_off(status & BIT(TP_EC_AUDIO_MUTESW)));
 
 		if (volume_control_allowed) {
-			seq_printf(m, "commands:\tunmute, mute\n");
+			seq_puts(m, "commands:\tunmute, mute\n");
 			if (!tp_features.mixer_no_level_control) {
-				seq_printf(m, "commands:\tup, down\n");
+				seq_puts(m, "commands:\tup, down\n");
 				seq_printf(m, "commands:\tlevel <level> (<level> is 0-%d)\n",
 					      TP_EC_VOLUME_MAX);
 			}
@@ -9156,9 +9182,9 @@ static int fan_read(struct seq_file *m)
 		} else if (fan_status_access_mode == TPACPI_FAN_RD_TPEC) {
 			if (status & TP_EC_FAN_FULLSPEED)
 				/* Disengaged mode takes precedence */
-				seq_printf(m, "level:\t\tdisengaged\n");
+				seq_puts(m, "level:\t\tdisengaged\n");
 			else if (status & TP_EC_FAN_AUTO)
-				seq_printf(m, "level:\t\tauto\n");
+				seq_puts(m, "level:\t\tauto\n");
 			else
 				seq_printf(m, "level:\t\t%d\n", status);
 		}
@@ -9166,19 +9192,19 @@ static int fan_read(struct seq_file *m)
 
 	case TPACPI_FAN_NONE:
 	default:
-		seq_printf(m, "status:\t\tnot supported\n");
+		seq_puts(m, "status:\t\tnot supported\n");
 	}
 
 	if (fan_control_commands & TPACPI_FAN_CMD_LEVEL) {
-		seq_printf(m, "commands:\tlevel <level>");
+		seq_puts(m, "commands:\tlevel <level>");
 
 		switch (fan_control_access_mode) {
 		case TPACPI_FAN_WR_ACPI_SFAN:
-			seq_printf(m, " (<level> is 0-7)\n");
+			seq_puts(m, " (<level> is 0-7)\n");
 			break;
 
 		default:
-			seq_printf(m, " (<level> is 0-7, auto, disengaged, full-speed)\n");
+			seq_puts(m, " (<level> is 0-7, auto, disengaged, full-speed)\n");
 			break;
 		}
 	}
@@ -9188,7 +9214,7 @@ static int fan_read(struct seq_file *m)
 			       "commands:\twatchdog <timeout> (<timeout> is 0 (off), 1-120 (seconds))\n");
 
 	if (fan_control_commands & TPACPI_FAN_CMD_SPEED)
-		seq_printf(m, "commands:\tspeed <speed> (<speed> is 0-65535)\n");
+		seq_puts(m, "commands:\tspeed <speed> (<speed> is 0-65535)\n");
 
 	return 0;
 }
@@ -11488,7 +11514,9 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 		mutex_unlock(&tpacpi_inputdev_send_mutex);
 		return true;
 	case TP_HKEY_EV_DOUBLETAP_TOGGLE:
-		tp_features.trackpoint_doubletap = !tp_features.trackpoint_doubletap;
+		/* Toggle kernel-level doubletap event filtering */
+		tp_features.trackpoint_doubletap_enable =
+			!tp_features.trackpoint_doubletap_enable;
 		return true;
 	case TP_HKEY_EV_PROFILE_TOGGLE:
 	case TP_HKEY_EV_PROFILE_TOGGLE2:
diff --git a/drivers/platform/x86/lenovo/wmi-capdata.c b/drivers/platform/x86/lenovo/wmi-capdata.c
index ee1fb02d8e31e..b73d378f0e8b3 100644
--- a/drivers/platform/x86/lenovo/wmi-capdata.c
+++ b/drivers/platform/x86/lenovo/wmi-capdata.c
@@ -53,7 +53,6 @@
 #define LENOVO_CAPABILITY_DATA_01_GUID "7A8F5407-CB67-4D6E-B547-39B3BE018154"
 #define LENOVO_FAN_TEST_DATA_GUID "B642801B-3D21-45DE-90AE-6E86F164FB21"
 
-#define ACPI_AC_CLASS "ac_adapter"
 #define ACPI_AC_NOTIFY_STATUS 0x80
 
 #define LWMI_FEATURE_ID_FAN_TEST 0x05
diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.c b/drivers/platform/x86/lenovo/wmi-gamezone.c
index 381836d29a964..c7fe7e3c9f179 100644
--- a/drivers/platform/x86/lenovo/wmi-gamezone.c
+++ b/drivers/platform/x86/lenovo/wmi-gamezone.c
@@ -31,8 +31,6 @@
 #define LWMI_GZ_METHOD_ID_SMARTFAN_SET 44
 #define LWMI_GZ_METHOD_ID_SMARTFAN_GET 45
 
-static BLOCKING_NOTIFIER_HEAD(gz_chain_head);
-
 struct lwmi_gz_priv {
 	enum thermal_mode current_mode;
 	struct notifier_block event_nb;
diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c
index 61ef7a218a80d..9681412d694be 100644
--- a/drivers/platform/x86/lg-laptop.c
+++ b/drivers/platform/x86/lg-laptop.c
@@ -271,11 +271,6 @@ static void wmi_input_setup(void)
 	}
 }
 
-static void acpi_notify(struct acpi_device *device, u32 event)
-{
-	acpi_handle_debug(device->handle, "notify: %d\n", event);
-}
-
 static ssize_t fan_mode_store(struct device *dev,
 			      struct device_attribute *attr,
 			      const char *buffer, size_t count)
@@ -764,8 +759,9 @@ static void lg_laptop_remove_address_space_handler(void *data)
 					  &lg_laptop_address_space_handler);
 }
 
-static int acpi_add(struct acpi_device *device)
+static int acpi_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct platform_device_info pdev_info = {
 		.fwnode = acpi_fwnode_handle(device),
 		.name = PLATFORM_NAME,
@@ -781,11 +777,11 @@ static int acpi_add(struct acpi_device *device)
 
 	status = acpi_install_address_space_handler(device->handle, LG_ADDRESS_SPACE_ID,
 						    &lg_laptop_address_space_handler,
-						    NULL, &device->dev);
+						    NULL, &pdev->dev);
 	if (ACPI_FAILURE(status))
 		return -ENODEV;
 
-	ret = devm_add_action_or_reset(&device->dev, lg_laptop_remove_address_space_handler,
+	ret = devm_add_action_or_reset(&pdev->dev, lg_laptop_remove_address_space_handler,
 				       device);
 	if (ret < 0)
 		return ret;
@@ -879,7 +875,7 @@ static int acpi_add(struct acpi_device *device)
 	return ret;
 }
 
-static void acpi_remove(struct acpi_device *device)
+static void acpi_remove(struct platform_device *pdev)
 {
 	sysfs_remove_group(&pf_device->dev.kobj, &dev_attribute_group);
 
@@ -899,34 +895,13 @@ static const struct acpi_device_id device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, device_ids);
 
-static struct acpi_driver acpi_driver = {
-	.name = "LG Gram Laptop Support",
-	.class = "lg-laptop",
-	.ids = device_ids,
-	.ops = {
-		.add = acpi_add,
-		.remove = acpi_remove,
-		.notify = acpi_notify,
-		},
+static struct platform_driver acpi_driver = {
+	.probe = acpi_probe,
+	.remove = acpi_remove,
+	.driver = {
+		.name = "LG Gram Laptop Support",
+		.acpi_match_table = device_ids,
+	},
 };
 
-static int __init acpi_init(void)
-{
-	int result;
-
-	result = acpi_bus_register_driver(&acpi_driver);
-	if (result < 0) {
-		pr_debug("Error registering driver\n");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-static void __exit acpi_exit(void)
-{
-	acpi_bus_unregister_driver(&acpi_driver);
-}
-
-module_init(acpi_init);
-module_exit(acpi_exit);
+module_platform_driver(acpi_driver);
diff --git a/drivers/platform/x86/mxm-wmi.c b/drivers/platform/x86/mxm-wmi.c
index 9a457956025a5..dbc5e35ec38b6 100644
--- a/drivers/platform/x86/mxm-wmi.c
+++ b/drivers/platform/x86/mxm-wmi.c
@@ -80,15 +80,3 @@ bool mxm_wmi_supported(void)
 	return guid_valid;
 }
 EXPORT_SYMBOL_GPL(mxm_wmi_supported);
-
-static int __init mxm_wmi_init(void)
-{
-	return 0;
-}
-
-static void __exit mxm_wmi_exit(void)
-{
-}
-
-module_init(mxm_wmi_init);
-module_exit(mxm_wmi_exit);
diff --git a/drivers/platform/x86/pcengines-apuv2.c b/drivers/platform/x86/pcengines-apuv2.c
index 3b086863c6ac9..3f19589d1ba03 100644
--- a/drivers/platform/x86/pcengines-apuv2.c
+++ b/drivers/platform/x86/pcengines-apuv2.c
@@ -294,7 +294,8 @@ static int __init apu_board_init(void)
 	}
 
 	apu_gpio_pdev = apu_create_pdev(AMD_FCH_GPIO_DRIVER_NAME,
-					id->driver_data, sizeof(struct amd_fch_gpio_pdata), NULL);
+					id->driver_data, sizeof(struct amd_fch_gpio_pdata),
+					&apu2_gpiochip_node);
 	err = PTR_ERR_OR_ZERO(apu_gpio_pdev);
 	if (err)
 		goto err_unregister_swnodes;
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index d3e7a52c22a76..b18f00e9082fa 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -178,8 +178,7 @@ enum sony_nc_rfkill {
 static int sony_rfkill_handle;
 static struct rfkill *sony_rfkill_devices[N_SONY_RFKILL];
 static int sony_rfkill_address[N_SONY_RFKILL] = {0x300, 0x500, 0x700, 0x900};
-static int sony_nc_rfkill_setup(struct acpi_device *device,
-		unsigned int handle);
+static int sony_nc_rfkill_setup(struct device *dev, unsigned int handle);
 static void sony_nc_rfkill_cleanup(void);
 static void sony_nc_rfkill_update(void);
 
@@ -435,7 +434,7 @@ static void sony_laptop_report_input_event(u8 event)
 		dprintk("unknown input event %.2x\n", event);
 }
 
-static int sony_laptop_setup_input(struct acpi_device *acpi_device)
+static int sony_laptop_setup_input(struct device *parent)
 {
 	struct input_dev *jog_dev;
 	struct input_dev *key_dev;
@@ -468,7 +467,7 @@ static int sony_laptop_setup_input(struct acpi_device *acpi_device)
 	key_dev->name = "Sony Vaio Keys";
 	key_dev->id.bustype = BUS_ISA;
 	key_dev->id.vendor = PCI_VENDOR_ID_SONY;
-	key_dev->dev.parent = &acpi_device->dev;
+	key_dev->dev.parent = parent;
 
 	/* Initialize the Input Drivers: special keys */
 	input_set_capability(key_dev, EV_MSC, MSC_SCAN);
@@ -497,7 +496,7 @@ static int sony_laptop_setup_input(struct acpi_device *acpi_device)
 	jog_dev->name = "Sony Vaio Jogdial";
 	jog_dev->id.bustype = BUS_ISA;
 	jog_dev->id.vendor = PCI_VENDOR_ID_SONY;
-	jog_dev->dev.parent = &acpi_device->dev;
+	jog_dev->dev.parent = parent;
 
 	input_set_capability(jog_dev, EV_KEY, BTN_MIDDLE);
 	input_set_capability(jog_dev, EV_REL, REL_WHEEL);
@@ -1176,7 +1175,7 @@ enum event_types {
 	KILLSWITCH,
 	GFX_SWITCH
 };
-static void sony_nc_notify(struct acpi_device *device, u32 event)
+static void sony_nc_notify(acpi_handle ah, u32 event, void *data)
 {
 	u32 real_ev = event;
 	u8 ev_type = 0;
@@ -1287,7 +1286,7 @@ static acpi_status sony_walk_callback(acpi_handle handle, u32 level,
 /*
  * ACPI device
  */
-static void sony_nc_function_setup(struct acpi_device *device,
+static void sony_nc_function_setup(struct device *dev,
 		struct platform_device *pf_device)
 {
 	unsigned int i, result, bitmask, arg;
@@ -1360,7 +1359,7 @@ static void sony_nc_function_setup(struct acpi_device *device,
 			break;
 		case 0x0124:
 		case 0x0135:
-			result = sony_nc_rfkill_setup(device, handle);
+			result = sony_nc_rfkill_setup(dev, handle);
 			if (result)
 				pr_err("couldn't set up rfkill support (%d)\n",
 						result);
@@ -1600,8 +1599,7 @@ static const struct rfkill_ops sony_rfkill_ops = {
 	.set_block = sony_nc_rfkill_set,
 };
 
-static int sony_nc_setup_rfkill(struct acpi_device *device,
-				enum sony_nc_rfkill nc_type)
+static int sony_nc_setup_rfkill(struct device *parent, enum sony_nc_rfkill nc_type)
 {
 	int err;
 	struct rfkill *rfk;
@@ -1631,8 +1629,7 @@ static int sony_nc_setup_rfkill(struct acpi_device *device,
 		return -EINVAL;
 	}
 
-	rfk = rfkill_alloc(name, &device->dev, type,
-			   &sony_rfkill_ops, (void *)nc_type);
+	rfk = rfkill_alloc(name, parent, type, &sony_rfkill_ops, (void *)nc_type);
 	if (!rfk)
 		return -ENOMEM;
 
@@ -1692,8 +1689,7 @@ static void sony_nc_rfkill_update(void)
 	}
 }
 
-static int sony_nc_rfkill_setup(struct acpi_device *device,
-		unsigned int handle)
+static int sony_nc_rfkill_setup(struct device *parent, unsigned int handle)
 {
 	u64 offset;
 	int i;
@@ -1734,18 +1730,18 @@ static int sony_nc_rfkill_setup(struct acpi_device *device,
 		dprintk("Radio devices, found 0x%.2x\n", buffer[i]);
 
 		if (buffer[i] == 0 && !sony_rfkill_devices[SONY_WIFI])
-			sony_nc_setup_rfkill(device, SONY_WIFI);
+			sony_nc_setup_rfkill(parent, SONY_WIFI);
 
 		if (buffer[i] == 0x10 && !sony_rfkill_devices[SONY_BLUETOOTH])
-			sony_nc_setup_rfkill(device, SONY_BLUETOOTH);
+			sony_nc_setup_rfkill(parent, SONY_BLUETOOTH);
 
 		if (((0xf0 & buffer[i]) == 0x20 ||
 					(0xf0 & buffer[i]) == 0x50) &&
 				!sony_rfkill_devices[SONY_WWAN])
-			sony_nc_setup_rfkill(device, SONY_WWAN);
+			sony_nc_setup_rfkill(parent, SONY_WWAN);
 
 		if (buffer[i] == 0x30 && !sony_rfkill_devices[SONY_WIMAX])
-			sony_nc_setup_rfkill(device, SONY_WIMAX);
+			sony_nc_setup_rfkill(parent, SONY_WIMAX);
 	}
 	return 0;
 }
@@ -3149,8 +3145,9 @@ static void sony_nc_backlight_cleanup(void)
 	backlight_device_unregister(sony_bl_props.dev);
 }
 
-static int sony_nc_add(struct acpi_device *device)
+static int sony_nc_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	acpi_status status;
 	int result = 0;
 	struct sony_nc_value *item;
@@ -3184,7 +3181,7 @@ static int sony_nc_add(struct acpi_device *device)
 		}
 	}
 
-	result = sony_laptop_setup_input(device);
+	result = sony_laptop_setup_input(&pdev->dev);
 	if (result) {
 		pr_err("Unable to create input devices\n");
 		goto outplatform;
@@ -3201,7 +3198,7 @@ static int sony_nc_add(struct acpi_device *device)
 		/* retrieve the available handles */
 		result = sony_nc_handles_setup(sony_pf_device);
 		if (!result)
-			sony_nc_function_setup(device, sony_pf_device);
+			sony_nc_function_setup(&pdev->dev, sony_pf_device);
 	}
 
 	if (acpi_video_get_backlight_type() == acpi_backlight_vendor)
@@ -3244,6 +3241,11 @@ static int sony_nc_add(struct acpi_device *device)
 		}
 	}
 
+	result = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+						 sony_nc_notify, NULL);
+	if (result)
+		goto out_sysfs;
+
 	pr_info("SNC setup done.\n");
 	return 0;
 
@@ -3266,10 +3268,13 @@ static int sony_nc_add(struct acpi_device *device)
 	return result;
 }
 
-static void sony_nc_remove(struct acpi_device *device)
+static void sony_nc_remove(struct platform_device *pdev)
 {
 	struct sony_nc_value *item;
 
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, sony_nc_notify);
+
 	sony_nc_backlight_cleanup();
 
 	sony_nc_acpi_device = NULL;
@@ -3297,16 +3302,14 @@ static const struct acpi_device_id sony_nc_device_ids[] = {
 	{"", 0},
 };
 
-static struct acpi_driver sony_nc_driver = {
-	.name = SONY_NC_DRIVER_NAME,
-	.class = SONY_NC_CLASS,
-	.ids = sony_nc_device_ids,
-	.ops = {
-		.add = sony_nc_add,
-		.remove = sony_nc_remove,
-		.notify = sony_nc_notify,
-		},
-	.drv.pm = &sony_nc_pm,
+static struct platform_driver sony_nc_driver = {
+	.probe = sony_nc_probe,
+	.remove = sony_nc_remove,
+	.driver = {
+		.name = SONY_NC_DRIVER_NAME,
+		.acpi_match_table = sony_nc_device_ids,
+		.pm = &sony_nc_pm,
+	},
 };
 
 /*********** SPIC (SNY6001) Device ***********/
@@ -4276,9 +4279,9 @@ static int sony_pic_possible_resources(struct acpi_device *device)
 /*
  *  Disable the spic device by calling its _DIS method
  */
-static int sony_pic_disable(struct acpi_device *device)
+static int sony_pic_disable(struct device *dev)
 {
-	acpi_status ret = acpi_evaluate_object(device->handle, "_DIS", NULL,
+	acpi_status ret = acpi_evaluate_object(ACPI_HANDLE(dev), "_DIS", NULL,
 					       NULL);
 
 	if (ACPI_FAILURE(ret) && ret != AE_NOT_FOUND)
@@ -4294,7 +4297,7 @@ static int sony_pic_disable(struct acpi_device *device)
  *
  *  Call _SRS to set current resources
  */
-static int sony_pic_enable(struct acpi_device *device,
+static int sony_pic_enable(struct device *dev,
 		struct sony_pic_ioport *ioport, struct sony_pic_irq *irq)
 {
 	acpi_status status;
@@ -4376,7 +4379,7 @@ static int sony_pic_enable(struct acpi_device *device,
 
 	/* Attempt to set the resource */
 	dprintk("Evaluating _SRS\n");
-	status = acpi_set_current_resources(device->handle, &buffer);
+	status = acpi_set_current_resources(ACPI_HANDLE(dev), &buffer);
 
 	/* check for total failure */
 	if (ACPI_FAILURE(status)) {
@@ -4465,12 +4468,12 @@ static irqreturn_t sony_pic_irq(int irq, void *dev_id)
  *  ACPI driver
  *
  *****************/
-static void sony_pic_remove(struct acpi_device *device)
+static void sony_pic_remove(struct platform_device *pdev)
 {
 	struct sony_pic_ioport *io, *tmp_io;
 	struct sony_pic_irq *irq, *tmp_irq;
 
-	if (sony_pic_disable(device)) {
+	if (sony_pic_disable(&pdev->dev)) {
 		pr_err("Couldn't disable device\n");
 		return;
 	}
@@ -4504,11 +4507,12 @@ static void sony_pic_remove(struct acpi_device *device)
 	dprintk(SONY_PIC_DRIVER_NAME " removed.\n");
 }
 
-static int sony_pic_add(struct acpi_device *device)
+static int sony_pic_probe(struct platform_device *pdev)
 {
-	int result;
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct sony_pic_ioport *io, *tmp_io;
 	struct sony_pic_irq *irq, *tmp_irq;
+	int result;
 
 	spic_dev.acpi_dev = device;
 	strscpy(acpi_device_class(device), "sony/hotkey");
@@ -4523,7 +4527,7 @@ static int sony_pic_add(struct acpi_device *device)
 	}
 
 	/* setup input devices and helper fifo */
-	result = sony_laptop_setup_input(device);
+	result = sony_laptop_setup_input(&pdev->dev);
 	if (result) {
 		pr_err("Unable to create input devices\n");
 		goto err_free_resources;
@@ -4593,7 +4597,7 @@ static int sony_pic_add(struct acpi_device *device)
 	}
 
 	/* set resource status _SRS */
-	result = sony_pic_enable(device, spic_dev.cur_ioport, spic_dev.cur_irq);
+	result = sony_pic_enable(&pdev->dev, spic_dev.cur_ioport, spic_dev.cur_irq);
 	if (result) {
 		pr_err("Couldn't enable device\n");
 		goto err_free_irq;
@@ -4616,7 +4620,7 @@ static int sony_pic_add(struct acpi_device *device)
 	sony_pf_remove();
 
 err_disable_device:
-	sony_pic_disable(device);
+	sony_pic_disable(&pdev->dev);
 
 err_free_irq:
 	free_irq(spic_dev.cur_irq->irq.interrupts[0], &spic_dev);
@@ -4652,15 +4656,14 @@ static int sony_pic_add(struct acpi_device *device)
 #ifdef CONFIG_PM_SLEEP
 static int sony_pic_suspend(struct device *dev)
 {
-	if (sony_pic_disable(to_acpi_device(dev)))
+	if (sony_pic_disable(dev))
 		return -ENXIO;
 	return 0;
 }
 
 static int sony_pic_resume(struct device *dev)
 {
-	sony_pic_enable(to_acpi_device(dev),
-			spic_dev.cur_ioport, spic_dev.cur_irq);
+	sony_pic_enable(dev, spic_dev.cur_ioport, spic_dev.cur_irq);
 	return 0;
 }
 #endif
@@ -4672,15 +4675,14 @@ static const struct acpi_device_id sony_pic_device_ids[] = {
 	{"", 0},
 };
 
-static struct acpi_driver sony_pic_driver = {
-	.name = SONY_PIC_DRIVER_NAME,
-	.class = SONY_PIC_CLASS,
-	.ids = sony_pic_device_ids,
-	.ops = {
-		.add = sony_pic_add,
-		.remove = sony_pic_remove,
-		},
-	.drv.pm = &sony_pic_pm,
+static struct platform_driver sony_pic_driver = {
+	.probe = sony_pic_probe,
+	.remove = sony_pic_remove,
+	.driver = {
+		.name = SONY_PIC_DRIVER_NAME,
+		.acpi_match_table = sony_pic_device_ids,
+		.pm = &sony_pic_pm,
+	},
 };
 
 static const struct dmi_system_id sonypi_dmi_table[] __initconst = {
@@ -4706,7 +4708,7 @@ static int __init sony_laptop_init(void)
 	int result;
 
 	if (!no_spic && dmi_check_system(sonypi_dmi_table)) {
-		result = acpi_bus_register_driver(&sony_pic_driver);
+		result = platform_driver_register(&sony_pic_driver);
 		if (result) {
 			pr_err("Unable to register SPIC driver\n");
 			goto out;
@@ -4714,7 +4716,7 @@ static int __init sony_laptop_init(void)
 		spic_drv_registered = 1;
 	}
 
-	result = acpi_bus_register_driver(&sony_nc_driver);
+	result = platform_driver_register(&sony_nc_driver);
 	if (result) {
 		pr_err("Unable to register SNC driver\n");
 		goto out_unregister_pic;
@@ -4724,16 +4726,16 @@ static int __init sony_laptop_init(void)
 
 out_unregister_pic:
 	if (spic_drv_registered)
-		acpi_bus_unregister_driver(&sony_pic_driver);
+		platform_driver_unregister(&sony_pic_driver);
 out:
 	return result;
 }
 
 static void __exit sony_laptop_exit(void)
 {
-	acpi_bus_unregister_driver(&sony_nc_driver);
+	platform_driver_unregister(&sony_nc_driver);
 	if (spic_drv_registered)
-		acpi_bus_unregister_driver(&sony_pic_driver);
+		platform_driver_unregister(&sony_pic_driver);
 }
 
 module_init(sony_laptop_init);
diff --git a/drivers/platform/x86/topstar-laptop.c b/drivers/platform/x86/topstar-laptop.c
index a7b4b6c8e5492..e09d7f8ce45f9 100644
--- a/drivers/platform/x86/topstar-laptop.c
+++ b/drivers/platform/x86/topstar-laptop.c
@@ -232,9 +232,9 @@ static int topstar_acpi_fncx_switch(struct acpi_device *device, bool state)
 	return 0;
 }
 
-static void topstar_acpi_notify(struct acpi_device *device, u32 event)
+static void topstar_acpi_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct topstar_laptop *topstar = acpi_driver_data(device);
+	struct topstar_laptop *topstar = data;
 	static bool dup_evnt[2];
 	bool *dup;
 
@@ -285,8 +285,9 @@ static const struct dmi_system_id topstar_dmi_ids[] = {
 	{}
 };
 
-static int topstar_acpi_add(struct acpi_device *device)
+static int topstar_acpi_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct topstar_laptop *topstar;
 	int err;
 
@@ -296,9 +297,10 @@ static int topstar_acpi_add(struct acpi_device *device)
 	if (!topstar)
 		return -ENOMEM;
 
+	platform_set_drvdata(pdev, topstar);
+
 	strscpy(acpi_device_name(device), "Topstar TPSACPI");
 	strscpy(acpi_device_class(device), TOPSTAR_LAPTOP_CLASS);
-	device->driver_data = topstar;
 	topstar->device = device;
 
 	err = topstar_acpi_init(topstar);
@@ -313,14 +315,21 @@ static int topstar_acpi_add(struct acpi_device *device)
 	if (err)
 		goto err_platform_exit;
 
+	err = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+					      topstar_acpi_notify, topstar);
+	if (err)
+		goto err_input_exit;
+
 	if (led_workaround) {
 		err = topstar_led_init(topstar);
 		if (err)
-			goto err_input_exit;
+			goto err_notify_handler_exit;
 	}
 
 	return 0;
 
+err_notify_handler_exit:
+	acpi_dev_remove_notify_handler(device, ACPI_DEVICE_NOTIFY, topstar_acpi_notify);
 err_input_exit:
 	topstar_input_exit(topstar);
 err_platform_exit:
@@ -332,13 +341,15 @@ static int topstar_acpi_add(struct acpi_device *device)
 	return err;
 }
 
-static void topstar_acpi_remove(struct acpi_device *device)
+static void topstar_acpi_remove(struct platform_device *pdev)
 {
-	struct topstar_laptop *topstar = acpi_driver_data(device);
+	struct topstar_laptop *topstar = platform_get_drvdata(pdev);
 
 	if (led_workaround)
 		topstar_led_exit(topstar);
 
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, topstar_acpi_notify);
 	topstar_input_exit(topstar);
 	topstar_platform_exit(topstar);
 	topstar_acpi_exit(topstar);
@@ -353,14 +364,12 @@ static const struct acpi_device_id topstar_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, topstar_device_ids);
 
-static struct acpi_driver topstar_acpi_driver = {
-	.name = "Topstar laptop ACPI driver",
-	.class = TOPSTAR_LAPTOP_CLASS,
-	.ids = topstar_device_ids,
-	.ops = {
-		.add = topstar_acpi_add,
-		.remove = topstar_acpi_remove,
-		.notify = topstar_acpi_notify,
+static struct platform_driver topstar_acpi_driver = {
+	.probe = topstar_acpi_probe,
+	.remove = topstar_acpi_remove,
+	.driver = {
+		.name = "Topstar laptop ACPI driver",
+		.acpi_match_table = topstar_device_ids,
 	},
 };
 
@@ -372,7 +381,7 @@ static int __init topstar_laptop_init(void)
 	if (ret < 0)
 		return ret;
 
-	ret = acpi_bus_register_driver(&topstar_acpi_driver);
+	ret = platform_driver_register(&topstar_acpi_driver);
 	if (ret < 0)
 		goto err_driver_unreg;
 
@@ -386,7 +395,7 @@ static int __init topstar_laptop_init(void)
 
 static void __exit topstar_laptop_exit(void)
 {
-	acpi_bus_unregister_driver(&topstar_acpi_driver);
+	platform_driver_unregister(&topstar_acpi_driver);
 	platform_driver_unregister(&topstar_platform_driver);
 }
 
diff --git a/drivers/platform/x86/wireless-hotkey.c b/drivers/platform/x86/wireless-hotkey.c
index e5083c0e15150..f680d8ff8e877 100644
--- a/drivers/platform/x86/wireless-hotkey.c
+++ b/drivers/platform/x86/wireless-hotkey.c
@@ -35,16 +35,17 @@ static const struct acpi_device_id wl_ids[] = {
 	{"", 0},
 };
 
-static int wireless_input_setup(struct acpi_device *device)
+static int wireless_input_setup(struct device *dev)
 {
-	struct wl_button *button = acpi_driver_data(device);
+	struct wl_button *button = dev_get_drvdata(dev);
 	int err;
 
 	button->input_dev = input_allocate_device();
 	if (!button->input_dev)
 		return -ENOMEM;
 
-	snprintf(button->phys, sizeof(button->phys), "%s/input0", acpi_device_hid(device));
+	snprintf(button->phys, sizeof(button->phys), "%s/input0",
+		 acpi_device_hid(ACPI_COMPANION(dev)));
 
 	button->input_dev->name = "Wireless hotkeys";
 	button->input_dev->phys = button->phys;
@@ -63,17 +64,17 @@ static int wireless_input_setup(struct acpi_device *device)
 	return err;
 }
 
-static void wireless_input_destroy(struct acpi_device *device)
+static void wireless_input_destroy(struct device *dev)
 {
-	struct wl_button *button = acpi_driver_data(device);
+	struct wl_button *button = dev_get_drvdata(dev);
 
 	input_unregister_device(button->input_dev);
 	kfree(button);
 }
 
-static void wl_notify(struct acpi_device *acpi_dev, u32 event)
+static void wl_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct wl_button *button = acpi_driver_data(acpi_dev);
+	struct wl_button *button = data;
 
 	if (event != 0x80) {
 		pr_info("Received unknown event (0x%x)\n", event);
@@ -86,7 +87,7 @@ static void wl_notify(struct acpi_device *acpi_dev, u32 event)
 	input_sync(button->input_dev);
 }
 
-static int wl_add(struct acpi_device *device)
+static int wl_probe(struct platform_device *pdev)
 {
 	struct wl_button *button;
 	int err;
@@ -95,30 +96,38 @@ static int wl_add(struct acpi_device *device)
 	if (!button)
 		return -ENOMEM;
 
-	device->driver_data = button;
+	platform_set_drvdata(pdev, button);
 
-	err = wireless_input_setup(device);
+	err = wireless_input_setup(&pdev->dev);
 	if (err) {
 		pr_err("Failed to setup wireless hotkeys\n");
 		kfree(button);
+		return err;
+	}
+	err = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
+					      ACPI_DEVICE_NOTIFY, wl_notify, button);
+	if (err) {
+		pr_err("Failed to install ACPI notify handler\n");
+		wireless_input_destroy(&pdev->dev);
 	}
 
 	return err;
 }
 
-static void wl_remove(struct acpi_device *device)
+static void wl_remove(struct platform_device *pdev)
 {
-	wireless_input_destroy(device);
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, wl_notify);
+	wireless_input_destroy(&pdev->dev);
 }
 
-static struct acpi_driver wl_driver = {
-	.name	= "wireless-hotkey",
-	.ids	= wl_ids,
-	.ops	= {
-		.add	= wl_add,
-		.remove	= wl_remove,
-		.notify	= wl_notify,
+static struct platform_driver wl_driver = {
+	.probe = wl_probe,
+	.remove = wl_remove,
+	.driver = {
+		.name = "wireless-hotkey",
+		.acpi_match_table = wl_ids,
 	},
 };
 
-module_acpi_driver(wl_driver);
+module_platform_driver(wl_driver);
diff --git a/drivers/pmdomain/arm/scmi_pm_domain.c b/drivers/pmdomain/arm/scmi_pm_domain.c
index b5e2ffd5ea64c..5454faed7d5d4 100644
--- a/drivers/pmdomain/arm/scmi_pm_domain.c
+++ b/drivers/pmdomain/arm/scmi_pm_domain.c
@@ -113,6 +113,7 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev)
 		goto err_rm_genpds;
 
 	dev_set_drvdata(dev, scmi_pd_data);
+	dev_info(dev, "Initialized %d power domains", num_domains);
 
 	return 0;
 err_rm_genpds:
diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c
index 0450202bbee25..b76d74e3849be 100644
--- a/drivers/pmdomain/bcm/bcm2835-power.c
+++ b/drivers/pmdomain/bcm/bcm2835-power.c
@@ -9,6 +9,7 @@
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/mfd/bcm2835-pm.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
@@ -153,7 +154,6 @@ struct bcm2835_power {
 static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable)
 {
 	void __iomem *base = power->asb;
-	u64 start;
 	u32 val;
 
 	switch (reg) {
@@ -166,8 +166,6 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable
 		break;
 	}
 
-	start = ktime_get_ns();
-
 	/* Enable the module's async AXI bridges. */
 	if (enable) {
 		val = readl(base + reg) & ~ASB_REQ_STOP;
@@ -176,11 +174,9 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable
 	}
 	writel(PM_PASSWORD | val, base + reg);
 
-	while (!!(readl(base + reg) & ASB_ACK) == enable) {
-		cpu_relax();
-		if (ktime_get_ns() - start >= 1000)
-			return -ETIMEDOUT;
-	}
+	if (readl_poll_timeout_atomic(base + reg, val,
+				      !!(val & ASB_ACK) != enable, 0, 5))
+		return -ETIMEDOUT;
 
 	return 0;
 }
@@ -219,10 +215,10 @@ static int bcm2835_power_power_on(struct bcm2835_power_domain *pd, u32 pm_reg)
 {
 	struct bcm2835_power *power = pd->power;
 	struct device *dev = power->dev;
-	u64 start;
 	int ret;
 	int inrush;
 	bool powok;
+	u32 val;
 
 	/* We don't run this on BCM2711 */
 	if (power->rpivid_asb)
@@ -243,12 +239,8 @@ static int bcm2835_power_power_on(struct bcm2835_power_domain *pd, u32 pm_reg)
 			 (inrush << PM_INRUSH_SHIFT) |
 			 PM_POWUP);
 
-		start = ktime_get_ns();
-		while (!(powok = !!(PM_READ(pm_reg) & PM_POWOK))) {
-			cpu_relax();
-			if (ktime_get_ns() - start >= 3000)
-				break;
-		}
+		powok = !readl_poll_timeout_atomic(power->base + pm_reg,
+						   val, val & PM_POWOK, 0, 3);
 	}
 	if (!powok) {
 		dev_err(dev, "Timeout waiting for %s power OK\n",
@@ -262,15 +254,12 @@ static int bcm2835_power_power_on(struct bcm2835_power_domain *pd, u32 pm_reg)
 
 	/* Repair memory */
 	PM_WRITE(pm_reg, PM_READ(pm_reg) | PM_MEMREP);
-	start = ktime_get_ns();
-	while (!(PM_READ(pm_reg) & PM_MRDONE)) {
-		cpu_relax();
-		if (ktime_get_ns() - start >= 1000) {
-			dev_err(dev, "Timeout waiting for %s memory repair\n",
-				pd->base.name);
-			ret = -ETIMEDOUT;
-			goto err_disable_ispow;
-		}
+	if (readl_poll_timeout_atomic(power->base + pm_reg, val,
+				      val & PM_MRDONE, 0, 1)) {
+		dev_err(dev, "Timeout waiting for %s memory repair\n",
+			pd->base.name);
+		ret = -ETIMEDOUT;
+		goto err_disable_ispow;
 	}
 
 	/* Disable functional isolation */
diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c
index 52ea84e548ff6..4d32fc676aaf5 100644
--- a/drivers/pmdomain/core.c
+++ b/drivers/pmdomain/core.c
@@ -1438,6 +1438,13 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock,
 		return;
 	} else {
 		genpd->states[genpd->state_idx].usage++;
+
+		/*
+		 * The ->system_power_down_ok() callback is currently used only
+		 * for s2idle. Use it to know when to update the usage counter.
+		 */
+		if (genpd->gov && genpd->gov->system_power_down_ok)
+			genpd->states[genpd->state_idx].usage_s2idle++;
 	}
 
 	genpd->status = GENPD_STATE_OFF;
@@ -3772,11 +3779,11 @@ static int idle_states_show(struct seq_file *s, void *data)
 	if (ret)
 		return -ERESTARTSYS;
 
-	seq_puts(s, "State          Time Spent(ms) Usage      Rejected   Above      Below\n");
+	seq_puts(s, "State  Time(ms)       Usage      Rejected   Above      Below      S2idle\n");
 
 	for (i = 0; i < genpd->state_count; i++) {
 		struct genpd_power_state *state = &genpd->states[i];
-		char state_name[15];
+		char state_name[7];
 
 		idle_time += state->idle_time;
 
@@ -3788,14 +3795,45 @@ static int idle_states_show(struct seq_file *s, void *data)
 			}
 		}
 
-		if (!state->name)
-			snprintf(state_name, ARRAY_SIZE(state_name), "S%-13d", i);
-
+		snprintf(state_name, ARRAY_SIZE(state_name), "S%-5d", i);
 		do_div(idle_time, NSEC_PER_MSEC);
-		seq_printf(s, "%-14s %-14llu %-10llu %-10llu %-10llu %llu\n",
-			   state->name ?: state_name, idle_time,
-			   state->usage, state->rejected, state->above,
-			   state->below);
+		seq_printf(s, "%-6s %-14llu %-10llu %-10llu %-10llu %-10llu %llu\n",
+			   state_name, idle_time, state->usage, state->rejected,
+			   state->above, state->below, state->usage_s2idle);
+	}
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+static int idle_states_desc_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	unsigned int i;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	seq_puts(s, "State  Latency(us)  Residency(us)  Name\n");
+
+	for (i = 0; i < genpd->state_count; i++) {
+		struct genpd_power_state *state = &genpd->states[i];
+		u64 latency, residency;
+		char state_name[7];
+
+		latency = state->power_off_latency_ns +
+			state->power_on_latency_ns;
+		do_div(latency, NSEC_PER_USEC);
+
+		residency = state->residency_ns;
+		do_div(residency, NSEC_PER_USEC);
+
+		snprintf(state_name, ARRAY_SIZE(state_name), "S%-5d", i);
+		seq_printf(s, "%-6s %-12llu %-14llu %s\n",
+			   state_name, latency, residency,
+			   state->name ?: "N/A");
 	}
 
 	genpd_unlock(genpd);
@@ -3891,6 +3929,7 @@ DEFINE_SHOW_ATTRIBUTE(summary);
 DEFINE_SHOW_ATTRIBUTE(status);
 DEFINE_SHOW_ATTRIBUTE(sub_domains);
 DEFINE_SHOW_ATTRIBUTE(idle_states);
+DEFINE_SHOW_ATTRIBUTE(idle_states_desc);
 DEFINE_SHOW_ATTRIBUTE(active_time);
 DEFINE_SHOW_ATTRIBUTE(total_idle_time);
 DEFINE_SHOW_ATTRIBUTE(devices);
@@ -3911,6 +3950,8 @@ static void genpd_debug_add(struct generic_pm_domain *genpd)
 			    d, genpd, &sub_domains_fops);
 	debugfs_create_file("idle_states", 0444,
 			    d, genpd, &idle_states_fops);
+	debugfs_create_file("idle_states_desc", 0444,
+			    d, genpd, &idle_states_desc_fops);
 	debugfs_create_file("active_time", 0444,
 			    d, genpd, &active_time_fops);
 	debugfs_create_file("total_idle_time", 0444,
diff --git a/drivers/pmdomain/imx/imx93-blk-ctrl.c b/drivers/pmdomain/imx/imx93-blk-ctrl.c
index e094fe5a42bf6..1afc78b034fad 100644
--- a/drivers/pmdomain/imx/imx93-blk-ctrl.c
+++ b/drivers/pmdomain/imx/imx93-blk-ctrl.c
@@ -7,6 +7,7 @@
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
@@ -188,6 +189,20 @@ static int imx93_blk_ctrl_power_off(struct generic_pm_domain *genpd)
 	return 0;
 }
 
+static void imx93_release_genpd_provider(void *data)
+{
+	struct device_node *of_node = data;
+
+	of_genpd_del_provider(of_node);
+}
+
+static void imx93_release_pm_genpd(void *data)
+{
+	struct generic_pm_domain *genpd = data;
+
+	pm_genpd_remove(genpd);
+}
+
 static struct lock_class_key blk_ctrl_genpd_lock_class;
 
 static int imx93_blk_ctrl_probe(struct platform_device *pdev)
@@ -240,10 +255,8 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev)
 	bc->num_clks = bc_data->num_clks;
 
 	ret = devm_clk_bulk_get(dev, bc->num_clks, bc->clks);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to get bus clock\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get bus clock\n");
 
 	for (i = 0; i < bc_data->num_domains; i++) {
 		const struct imx93_blk_ctrl_domain_data *data = &bc_data->domains[i];
@@ -258,10 +271,8 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev)
 			domain->clks[j].id = data->clk_names[j];
 
 		ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks);
-		if (ret) {
-			dev_err_probe(dev, ret, "failed to get clock\n");
-			goto cleanup_pds;
-		}
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to get clock\n");
 
 		domain->genpd.name = data->name;
 		domain->genpd.power_on = imx93_blk_ctrl_power_on;
@@ -269,11 +280,12 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev)
 		domain->bc = bc;
 
 		ret = pm_genpd_init(&domain->genpd, NULL, true);
-		if (ret) {
-			dev_err_probe(dev, ret, "failed to init power domain\n");
-			goto cleanup_pds;
-		}
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to init power domain\n");
 
+		ret = devm_add_action_or_reset(dev, imx93_release_pm_genpd, &domain->genpd);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to add pm_genpd release callback\n");
 		/*
 		 * We use runtime PM to trigger power on/off of the upstream GPC
 		 * domain, as a strict hierarchical parent/child power domain
@@ -290,39 +302,23 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev)
 		bc->onecell_data.domains[i] = &domain->genpd;
 	}
 
-	pm_runtime_enable(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable pm-runtime\n");
 
 	ret = of_genpd_add_provider_onecell(dev->of_node, &bc->onecell_data);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to add power domain provider\n");
-		goto cleanup_pds;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to add power domain provider\n");
 
-	dev_set_drvdata(dev, bc);
+	ret = devm_add_action_or_reset(dev, imx93_release_genpd_provider, dev->of_node);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to add genpd_provider release callback\n");
 
-	return 0;
-
-cleanup_pds:
-	for (i--; i >= 0; i--)
-		pm_genpd_remove(&bc->domains[i].genpd);
-
-	return ret;
-}
-
-static void imx93_blk_ctrl_remove(struct platform_device *pdev)
-{
-	struct imx93_blk_ctrl *bc = dev_get_drvdata(&pdev->dev);
-	int i;
-
-	of_genpd_del_provider(pdev->dev.of_node);
+	ret = devm_of_platform_populate(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to populate blk-ctrl sub-devices\n");
 
-	pm_runtime_disable(&pdev->dev);
-
-	for (i = 0; i < bc->onecell_data.num_domains; i++) {
-		struct imx93_blk_ctrl_domain *domain = &bc->domains[i];
-
-		pm_genpd_remove(&domain->genpd);
-	}
+	return 0;
 }
 
 static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] = {
@@ -457,7 +453,6 @@ MODULE_DEVICE_TABLE(of, imx93_blk_ctrl_of_match);
 
 static struct platform_driver imx93_blk_ctrl_driver = {
 	.probe = imx93_blk_ctrl_probe,
-	.remove = imx93_blk_ctrl_remove,
 	.driver = {
 		.name = "imx93-blk-ctrl",
 		.of_match_table = imx93_blk_ctrl_of_match,
diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c
index 01d465d88f60d..3ec33667a308c 100644
--- a/drivers/pmdomain/imx/scu-pd.c
+++ b/drivers/pmdomain/imx/scu-pd.c
@@ -326,6 +326,7 @@ static void imx_sc_pd_get_console_rsrc(void)
 		return;
 
 	imx_con_rsrc = specs.args[0];
+	of_node_put(specs.np);
 }
 
 static int imx_sc_get_pd_power(struct device *dev, u32 rsrc)
diff --git a/drivers/pmdomain/marvell/pxa1908-power-controller.c b/drivers/pmdomain/marvell/pxa1908-power-controller.c
index ff5e6e82d3f8d..543e8d33ac0c8 100644
--- a/drivers/pmdomain/marvell/pxa1908-power-controller.c
+++ b/drivers/pmdomain/marvell/pxa1908-power-controller.c
@@ -24,12 +24,15 @@
 #define APMU_DEBUG		0x88
 #define DSI_PHY_DVM_MASK	BIT(31)
 
+#define APMU_AUDIO_CLK		0x80
+#define AUDIO_ULCX_ENABLE	0x0d
+
 #define POWER_ON_LATENCY_US	300
 #define POWER_OFF_LATENCY_US	20
 #define POWER_POLL_TIMEOUT_US	(25 * USEC_PER_MSEC)
 #define POWER_POLL_SLEEP_US	6
 
-#define NR_DOMAINS	5
+#define NR_DOMAINS	6
 
 #define to_pxa1908_pd(_genpd) container_of(_genpd, struct pxa1908_pd, genpd)
 
@@ -59,9 +62,14 @@ static inline bool pxa1908_pd_is_on(struct pxa1908_pd *pd)
 {
 	struct pxa1908_pd_ctrl *ctrl = pd->ctrl;
 
-	return pd->data.id != PXA1908_POWER_DOMAIN_DSI
-		? regmap_test_bits(ctrl->base, APMU_PWR_STATUS_REG, pd->data.pwr_state)
-		: regmap_test_bits(ctrl->base, APMU_DEBUG, DSI_PHY_DVM_MASK);
+	switch (pd->data.id) {
+	case PXA1908_POWER_DOMAIN_AUDIO:
+		return regmap_test_bits(ctrl->base, APMU_AUDIO_CLK, AUDIO_ULCX_ENABLE);
+	case PXA1908_POWER_DOMAIN_DSI:
+		return regmap_test_bits(ctrl->base, APMU_DEBUG, DSI_PHY_DVM_MASK);
+	default:
+		return regmap_test_bits(ctrl->base, APMU_PWR_STATUS_REG, pd->data.pwr_state);
+	}
 }
 
 static int pxa1908_pd_power_on(struct generic_pm_domain *genpd)
@@ -123,6 +131,22 @@ static inline int pxa1908_dsi_power_off(struct generic_pm_domain *genpd)
 	return regmap_clear_bits(ctrl->base, APMU_DEBUG, DSI_PHY_DVM_MASK);
 }
 
+static inline int pxa1908_audio_power_on(struct generic_pm_domain *genpd)
+{
+	struct pxa1908_pd *pd = to_pxa1908_pd(genpd);
+	struct pxa1908_pd_ctrl *ctrl = pd->ctrl;
+
+	return regmap_set_bits(ctrl->base, APMU_AUDIO_CLK, AUDIO_ULCX_ENABLE);
+}
+
+static inline int pxa1908_audio_power_off(struct generic_pm_domain *genpd)
+{
+	struct pxa1908_pd *pd = to_pxa1908_pd(genpd);
+	struct pxa1908_pd_ctrl *ctrl = pd->ctrl;
+
+	return regmap_clear_bits(ctrl->base, APMU_AUDIO_CLK, AUDIO_ULCX_ENABLE);
+}
+
 #define DOMAIN(_id, _name, ctrl, mode, state) \
 	[_id] = { \
 		.data = { \
@@ -159,6 +183,13 @@ static struct pxa1908_pd domains[NR_DOMAINS] = {
 			.keep_on = true,
 		},
 	},
+	[PXA1908_POWER_DOMAIN_AUDIO] = {
+		.genpd = {
+			.name = "audio",
+			.power_on = pxa1908_audio_power_on,
+			.power_off = pxa1908_audio_power_off,
+		},
+	},
 };
 
 static void pxa1908_pd_remove(struct auxiliary_device *auxdev)
diff --git a/drivers/pmdomain/mediatek/mt8189-pm-domains.h b/drivers/pmdomain/mediatek/mt8189-pm-domains.h
new file mode 100644
index 0000000000000..c28b9460c074a
--- /dev/null
+++ b/drivers/pmdomain/mediatek/mt8189-pm-domains.h
@@ -0,0 +1,485 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025 MediaTek Inc.
+ * Author: Qiqi Wang <qiqi.wang@mediatek.com>
+ */
+
+#ifndef __SOC_MEDIATEK_MT8189_PM_DOMAINS_H
+#define __SOC_MEDIATEK_MT8189_PM_DOMAINS_H
+
+#include "mtk-pm-domains.h"
+#include <dt-bindings/power/mediatek,mt8189-power.h>
+
+/*
+ * MT8189 power domain support
+ */
+
+#define MT8189_SPM_PWR_STATUS				0x0f40
+#define MT8189_SPM_PWR_STATUS_2ND			0x0f44
+#define MT8189_SPM_PWR_STATUS_MSB			0x0f48
+#define MT8189_SPM_PWR_STATUS_MSB_2ND			0x0f4c
+#define MT8189_SPM_XPU_PWR_STATUS			0x0f50
+#define MT8189_SPM_XPU_PWR_STATUS_2ND			0x0f54
+
+#define MT8189_PROT_EN_EMICFG_GALS_SLP_SET		0x0084
+#define MT8189_PROT_EN_EMICFG_GALS_SLP_CLR		0x0088
+#define MT8189_PROT_EN_EMICFG_GALS_SLP_RDY		0x008c
+#define MT8189_PROT_EN_MMSYS_STA_0_SET			0x0c14
+#define MT8189_PROT_EN_MMSYS_STA_0_CLR			0x0c18
+#define MT8189_PROT_EN_MMSYS_STA_0_RDY			0x0c1c
+#define MT8189_PROT_EN_MMSYS_STA_1_SET			0x0c24
+#define MT8189_PROT_EN_MMSYS_STA_1_CLR			0x0c28
+#define MT8189_PROT_EN_MMSYS_STA_1_RDY			0x0c2c
+#define MT8189_PROT_EN_INFRASYS_STA_0_SET		0x0c44
+#define MT8189_PROT_EN_INFRASYS_STA_0_CLR		0x0c48
+#define MT8189_PROT_EN_INFRASYS_STA_0_RDY		0x0c4c
+#define MT8189_PROT_EN_INFRASYS_STA_1_SET		0x0c54
+#define MT8189_PROT_EN_INFRASYS_STA_1_CLR		0x0c58
+#define MT8189_PROT_EN_INFRASYS_STA_1_RDY		0x0c5c
+#define MT8189_PROT_EN_PERISYS_STA_0_SET		0x0c84
+#define MT8189_PROT_EN_PERISYS_STA_0_CLR		0x0c88
+#define MT8189_PROT_EN_PERISYS_STA_0_RDY		0x0c8c
+#define MT8189_PROT_EN_MCU_STA_0_SET			0x0c94
+#define MT8189_PROT_EN_MCU_STA_0_CLR			0x0c98
+#define MT8189_PROT_EN_MCU_STA_0_RDY			0x0c9c
+#define MT8189_PROT_EN_MD_STA_0_SET			0x0ca4
+#define MT8189_PROT_EN_MD_STA_0_CLR			0x0ca8
+#define MT8189_PROT_EN_MD_STA_0_RDY			0x0cac
+
+#define MT8189_PROT_EN_EMISYS_STA_0_MM_INFRA		(GENMASK(21, 20))
+#define MT8189_PROT_EN_INFRASYS_STA_0_CONN		(BIT(8))
+#define MT8189_PROT_EN_INFRASYS_STA_1_CONN		(BIT(12))
+#define MT8189_PROT_EN_INFRASYS_STA_0_MM_INFRA		(BIT(16))
+#define MT8189_PROT_EN_INFRASYS_STA_1_MM_INFRA		(BIT(11))
+#define MT8189_PROT_EN_INFRASYS_STA_1_MFG1		(BIT(20))
+#define MT8189_PROT_EN_MCU_STA_0_CONN			(BIT(1))
+#define MT8189_PROT_EN_MCU_STA_0_CONN_2ND		(BIT(0))
+#define MT8189_PROT_EN_MD_STA_0_MFG1			(BIT(0) | BIT(2))
+#define MT8189_PROT_EN_MD_STA_0_MFG1_2ND		(BIT(4))
+#define MT8189_PROT_EN_MM_INFRA_IGN			(BIT(1))
+#define MT8189_PROT_EN_MM_INFRA_2_IGN			(BIT(0))
+#define MT8189_PROT_EN_MMSYS_STA_0_CAM_MAIN		(GENMASK(31, 30))
+#define MT8189_PROT_EN_MMSYS_STA_1_CAM_MAIN		(GENMASK(10, 9))
+#define MT8189_PROT_EN_MMSYS_STA_0_DISP			(GENMASK(1, 0))
+#define MT8189_PROT_EN_MMSYS_STA_0_ISP_IMG1		(BIT(3))
+#define MT8189_PROT_EN_MMSYS_STA_1_ISP_IMG1		(BIT(7))
+#define MT8189_PROT_EN_MMSYS_STA_0_ISP_IPE		(BIT(2))
+#define MT8189_PROT_EN_MMSYS_STA_1_ISP_IPE		(BIT(8))
+#define MT8189_PROT_EN_MMSYS_STA_0_MDP0			(BIT(18))
+#define MT8189_PROT_EN_MMSYS_STA_1_MM_INFRA		(GENMASK(3, 2))
+#define MT8189_PROT_EN_MMSYS_STA_1_MM_INFRA_2ND		(GENMASK(15, 7))
+#define MT8189_PROT_EN_MMSYS_STA_0_VDE0			(BIT(20))
+#define MT8189_PROT_EN_MMSYS_STA_1_VDE0			(BIT(13))
+#define MT8189_PROT_EN_MMSYS_STA_0_VEN0			(BIT(12))
+#define MT8189_PROT_EN_MMSYS_STA_1_VEN0			(BIT(12))
+#define MT8189_PROT_EN_PERISYS_STA_0_AUDIO		(BIT(6))
+#define MT8189_PROT_EN_PERISYS_STA_0_SSUSB		(BIT(7))
+#define MT8189_PROT_EN_EMICFG_GALS_SLP_MFG1		(GENMASK(5, 4))
+
+static enum scpsys_bus_prot_block scpsys_bus_prot_blocks_mt8189[] = {
+	BUS_PROT_BLOCK_INFRA, BUS_PROT_BLOCK_SMI
+};
+
+static const struct scpsys_domain_data scpsys_domain_data_mt8189[] = {
+	[MT8189_POWER_DOMAIN_CONN] = {
+		.name = "conn",
+		.sta_mask = BIT(1),
+		.ctl_offs = 0xe04,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MCU_STA_0_CONN,
+					MT8189_PROT_EN_MCU_STA_0_SET,
+					MT8189_PROT_EN_MCU_STA_0_CLR,
+					MT8189_PROT_EN_MCU_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_INFRASYS_STA_1_CONN,
+					MT8189_PROT_EN_INFRASYS_STA_1_SET,
+					MT8189_PROT_EN_INFRASYS_STA_1_CLR,
+					MT8189_PROT_EN_INFRASYS_STA_1_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MCU_STA_0_CONN_2ND,
+					MT8189_PROT_EN_MCU_STA_0_SET,
+					MT8189_PROT_EN_MCU_STA_0_CLR,
+					MT8189_PROT_EN_MCU_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_INFRASYS_STA_0_CONN,
+					MT8189_PROT_EN_INFRASYS_STA_0_SET,
+					MT8189_PROT_EN_INFRASYS_STA_0_CLR,
+					MT8189_PROT_EN_INFRASYS_STA_0_RDY),
+		},
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_AUDIO] = {
+		.name = "audio",
+		.sta_mask = BIT(6),
+		.ctl_offs = 0xe18,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_PERISYS_STA_0_AUDIO,
+					MT8189_PROT_EN_PERISYS_STA_0_SET,
+					MT8189_PROT_EN_PERISYS_STA_0_CLR,
+					MT8189_PROT_EN_PERISYS_STA_0_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_ADSP_TOP_DORMANT] = {
+		.name = "adsp-top-dormant",
+		.sta_mask = BIT(7),
+		.ctl_offs = 0xe1c,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(9),
+		.sram_pdn_ack_bits = BIT(13),
+		.caps = MTK_SCPD_SRAM_ISO | MTK_SCPD_SRAM_PDN_INVERTED |
+			MTK_SCPD_ACTIVE_WAKEUP | MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_ADSP_INFRA] = {
+		.name = "adsp-infra",
+		.sta_mask = BIT(8),
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.ctl_offs = 0xe20,
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_ADSP_AO] = {
+		.name = "adsp-ao",
+		.sta_mask = BIT(9),
+		.ctl_offs = 0xe24,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+	},
+	[MT8189_POWER_DOMAIN_ISP_IMG1] = {
+		.name = "isp-img1",
+		.sta_mask = BIT(10),
+		.ctl_offs = 0xe28,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_ISP_IMG1,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_ISP_IMG1,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_ISP_IMG2] = {
+		.name = "isp-img2",
+		.sta_mask = BIT(11),
+		.ctl_offs = 0xe2c,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_ISP_IPE] = {
+		.name = "isp-ipe",
+		.sta_mask = BIT(12),
+		.ctl_offs = 0xe30,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_ISP_IPE,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_ISP_IPE,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_VDE0] = {
+		.name = "vde0",
+		.sta_mask = BIT(14),
+		.ctl_offs = 0xe38,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_VDE0,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_VDE0,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_VEN0] = {
+		.name = "ven0",
+		.sta_mask = BIT(16),
+		.ctl_offs = 0xe40,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_VEN0,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_VEN0,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_CAM_MAIN] = {
+		.name = "cam-main",
+		.sta_mask = BIT(18),
+		.ctl_offs = 0xe48,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_CAM_MAIN,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_CAM_MAIN,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_CAM_SUBA] = {
+		.name = "cam-suba",
+		.sta_mask = BIT(20),
+		.ctl_offs = 0xe50,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_CAM_SUBB] = {
+		.name = "cam-subb",
+		.sta_mask = BIT(21),
+		.ctl_offs = 0xe54,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_MDP0] = {
+		.name = "mdp0",
+		.sta_mask = BIT(26),
+		.ctl_offs = 0xe68,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_MDP0,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_DISP] = {
+		.name = "disp",
+		.sta_mask = BIT(28),
+		.ctl_offs = 0xe70,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_0_DISP,
+					MT8189_PROT_EN_MMSYS_STA_0_SET,
+					MT8189_PROT_EN_MMSYS_STA_0_CLR,
+					MT8189_PROT_EN_MMSYS_STA_0_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_MM_INFRA] = {
+		.name = "mm-infra",
+		.sta_mask = BIT(30),
+		.ctl_offs = 0xe78,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_MM_INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MMSYS_STA_1_MM_INFRA_2ND,
+					MT8189_PROT_EN_MMSYS_STA_1_SET,
+					MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					MT8189_PROT_EN_MMSYS_STA_1_RDY),
+			BUS_PROT_WR_IGN_SUBCLK(INFRA,
+					       MT8189_PROT_EN_MM_INFRA_IGN,
+					       MT8189_PROT_EN_MMSYS_STA_1_SET,
+					       MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					       MT8189_PROT_EN_MMSYS_STA_1_RDY),
+			BUS_PROT_WR_IGN_SUBCLK(INFRA,
+					       MT8189_PROT_EN_MM_INFRA_2_IGN,
+					       MT8189_PROT_EN_MMSYS_STA_1_SET,
+					       MT8189_PROT_EN_MMSYS_STA_1_CLR,
+					       MT8189_PROT_EN_MMSYS_STA_1_RDY),
+		},
+	},
+	[MT8189_POWER_DOMAIN_DP_TX] = {
+		.name = "dp-tx",
+		.sta_mask = BIT(0),
+		.ctl_offs = 0xe80,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+	},
+	[MT8189_POWER_DOMAIN_CSI_RX] = {
+		.name = "csi-rx",
+		.sta_mask = BIT(7),
+		.ctl_offs = 0xe9c,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+		.caps = MTK_SCPD_KEEP_DEFAULT_OFF,
+	},
+	[MT8189_POWER_DOMAIN_SSUSB] = {
+		.name = "ssusb",
+		.sta_mask = BIT(10),
+		.ctl_offs = 0xea8,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_PERISYS_STA_0_SSUSB,
+					MT8189_PROT_EN_PERISYS_STA_0_SET,
+					MT8189_PROT_EN_PERISYS_STA_0_CLR,
+					MT8189_PROT_EN_PERISYS_STA_0_RDY),
+		},
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
+	},
+	[MT8189_POWER_DOMAIN_MFG0] = {
+		.name = "mfg0",
+		.sta_mask = BIT(1),
+		.ctl_offs = 0xeb4,
+		.pwr_sta_offs = MT8189_SPM_XPU_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_XPU_PWR_STATUS_2ND,
+		.caps = MTK_SCPD_DOMAIN_SUPPLY,
+	},
+	[MT8189_POWER_DOMAIN_MFG1] = {
+		.name = "mfg1",
+		.sta_mask = BIT(2),
+		.ctl_offs = 0xeb8,
+		.pwr_sta_offs = MT8189_SPM_XPU_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_XPU_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.bp_cfg = {
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_INFRASYS_STA_1_MFG1,
+					MT8189_PROT_EN_INFRASYS_STA_1_SET,
+					MT8189_PROT_EN_INFRASYS_STA_1_CLR,
+					MT8189_PROT_EN_INFRASYS_STA_1_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MD_STA_0_MFG1,
+					MT8189_PROT_EN_MD_STA_0_SET,
+					MT8189_PROT_EN_MD_STA_0_CLR,
+					MT8189_PROT_EN_MD_STA_0_RDY),
+			BUS_PROT_WR_IGN(INFRA,
+					MT8189_PROT_EN_MD_STA_0_MFG1_2ND,
+					MT8189_PROT_EN_MD_STA_0_SET,
+					MT8189_PROT_EN_MD_STA_0_CLR,
+					MT8189_PROT_EN_MD_STA_0_RDY),
+			BUS_PROT_WR_IGN(SMI,
+					MT8189_PROT_EN_EMICFG_GALS_SLP_MFG1,
+					MT8189_PROT_EN_EMICFG_GALS_SLP_SET,
+					MT8189_PROT_EN_EMICFG_GALS_SLP_CLR,
+					MT8189_PROT_EN_EMICFG_GALS_SLP_RDY),
+		},
+		.caps = MTK_SCPD_DOMAIN_SUPPLY,
+	},
+	[MT8189_POWER_DOMAIN_MFG2] = {
+		.name = "mfg2",
+		.sta_mask = BIT(3),
+		.ctl_offs = 0xebc,
+		.pwr_sta_offs = MT8189_SPM_XPU_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_XPU_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+	},
+	[MT8189_POWER_DOMAIN_MFG3] = {
+		.name = "mfg3",
+		.sta_mask = BIT(4),
+		.ctl_offs = 0xec0,
+		.pwr_sta_offs = MT8189_SPM_XPU_PWR_STATUS,
+		.pwr_sta2nd_offs = MT8189_SPM_XPU_PWR_STATUS_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+	},
+	[MT8189_POWER_DOMAIN_EDP_TX_DORMANT] = {
+		.name = "edp-tx-dormant",
+		.sta_mask = BIT(12),
+		.ctl_offs = 0xf70,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+		.sram_pdn_bits = BIT(9),
+		.sram_pdn_ack_bits = 0,
+		.caps = MTK_SCPD_SRAM_ISO | MTK_SCPD_SRAM_PDN_INVERTED,
+	},
+	[MT8189_POWER_DOMAIN_PCIE] = {
+		.name = "pcie",
+		.sta_mask = BIT(13),
+		.ctl_offs = 0xf74,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+		.sram_pdn_bits = BIT(8),
+		.sram_pdn_ack_bits = BIT(12),
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
+	},
+	[MT8189_POWER_DOMAIN_PCIE_PHY] = {
+		.name = "pcie-phy",
+		.sta_mask = BIT(14),
+		.ctl_offs = 0xf78,
+		.pwr_sta_offs = MT8189_SPM_PWR_STATUS_MSB,
+		.pwr_sta2nd_offs = MT8189_SPM_PWR_STATUS_MSB_2ND,
+	},
+};
+
+static const struct scpsys_soc_data mt8189_scpsys_data = {
+	.domains_data = scpsys_domain_data_mt8189,
+	.num_domains = ARRAY_SIZE(scpsys_domain_data_mt8189),
+	.bus_prot_blocks = scpsys_bus_prot_blocks_mt8189,
+	.num_bus_prot_blocks = ARRAY_SIZE(scpsys_bus_prot_blocks_mt8189),
+};
+
+#endif /* __SOC_MEDIATEK_MT8189_PM_DOMAINS_H */
diff --git a/drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c b/drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c
index 9bad577b3ae4b..3ce6fb74dd53f 100644
--- a/drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c
+++ b/drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c
@@ -10,6 +10,7 @@
 #include <linux/clk-provider.h>
 #include <linux/container_of.h>
 #include <linux/iopoll.h>
+#include <linux/nvmem-provider.h>
 #include <linux/mailbox_client.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -872,6 +873,60 @@ static int mtk_mfg_init_clk_provider(struct mtk_mfg *mfg)
 	return 0;
 }
 
+static int mtk_mfg_read_nvmem(void *priv, unsigned int offset, void *val, size_t bytes)
+{
+	struct mtk_mfg *mfg = priv;
+	u32 *buf = val;
+
+	if (bytes != 4)
+		return -EINVAL;
+
+	if (!mfg->shared_mem)
+		return -ENODEV;
+
+	if (offset + bytes >= mfg->shared_mem_size)
+		return -EINVAL;
+
+	*buf = readl(mfg->shared_mem + offset);
+
+	return 0;
+}
+
+static int mtk_mfg_init_nvmem_provider(struct mtk_mfg *mfg)
+{
+	struct device *dev = &mfg->pdev->dev;
+	struct nvmem_cell_info cell = {};
+	struct nvmem_config config = {};
+	struct nvmem_device *nvdev;
+	int ret;
+
+	config.reg_read = mtk_mfg_read_nvmem;
+	config.dev = dev;
+	config.read_only = true;
+	config.priv = mfg;
+	config.size = 4;
+	config.word_size = 4;
+
+	nvdev = devm_nvmem_register(dev, &config);
+	if (IS_ERR(nvdev))
+		return dev_err_probe(dev, PTR_ERR(nvdev), "Couldn't register nvmem provider\n");
+
+	cell.name = "shader-present";
+	cell.offset = GF_REG_SHADER_PRESENT;
+	cell.bytes = 4;
+	cell.np = of_get_child_by_name(dev->of_node, cell.name);
+
+	ret = nvmem_add_one_cell(nvdev, &cell);
+	if (ret) {
+		of_node_put(cell.np);
+		return dev_err_probe(dev, ret, "Couldn't add cell %s\n", cell.name);
+	}
+
+	/* cell.np purposefully not put as nvmem_add_one_cell does not increase refcount */
+
+	return 0;
+}
+
 static int mtk_mfg_probe(struct platform_device *pdev)
 {
 	struct mtk_mfg *mfg;
@@ -984,6 +1039,10 @@ static int mtk_mfg_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_power_off;
 
+	ret = mtk_mfg_init_nvmem_provider(mfg);
+	if (ret)
+		goto err_power_off;
+
 	ret = of_genpd_add_provider_simple(dev->of_node, &mfg->pd);
 	if (ret) {
 		dev_err_probe(dev, ret, "Failed to add pmdomain provider\n");
diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c
index f64f24d520ddd..d2b8d03329515 100644
--- a/drivers/pmdomain/mediatek/mtk-pm-domains.c
+++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c
@@ -26,6 +26,7 @@
 #include "mt8183-pm-domains.h"
 #include "mt8186-pm-domains.h"
 #include "mt8188-pm-domains.h"
+#include "mt8189-pm-domains.h"
 #include "mt8192-pm-domains.h"
 #include "mt8195-pm-domains.h"
 #include "mt8196-pm-domains.h"
@@ -250,7 +251,7 @@ static int scpsys_bus_protect_set(struct scpsys_domain *pd,
 					MTK_POLL_DELAY_US, MTK_POLL_TIMEOUT);
 }
 
-static int scpsys_bus_protect_enable(struct scpsys_domain *pd)
+static int scpsys_bus_protect_enable(struct scpsys_domain *pd, u8 flags)
 {
 	for (int i = 0; i < SPM_MAX_BUS_PROT_DATA; i++) {
 		const struct scpsys_bus_prot_data *bpd = &pd->data->bp_cfg[i];
@@ -259,6 +260,10 @@ static int scpsys_bus_protect_enable(struct scpsys_domain *pd)
 		if (!bpd->bus_prot_set_clr_mask)
 			break;
 
+		if ((bpd->flags & BUS_PROT_IGNORE_SUBCLK) !=
+		    (flags & BUS_PROT_IGNORE_SUBCLK))
+			continue;
+
 		if (bpd->flags & BUS_PROT_INVERTED)
 			ret = scpsys_bus_protect_clear(pd, bpd);
 		else
@@ -270,7 +275,7 @@ static int scpsys_bus_protect_enable(struct scpsys_domain *pd)
 	return 0;
 }
 
-static int scpsys_bus_protect_disable(struct scpsys_domain *pd)
+static int scpsys_bus_protect_disable(struct scpsys_domain *pd, u8 flags)
 {
 	for (int i = SPM_MAX_BUS_PROT_DATA - 1; i >= 0; i--) {
 		const struct scpsys_bus_prot_data *bpd = &pd->data->bp_cfg[i];
@@ -279,6 +284,10 @@ static int scpsys_bus_protect_disable(struct scpsys_domain *pd)
 		if (!bpd->bus_prot_set_clr_mask)
 			continue;
 
+		if ((bpd->flags & BUS_PROT_IGNORE_SUBCLK) !=
+		    (flags & BUS_PROT_IGNORE_SUBCLK))
+			continue;
+
 		if (bpd->flags & BUS_PROT_INVERTED)
 			ret = scpsys_bus_protect_set(pd, bpd);
 		else
@@ -632,6 +641,15 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	if (ret)
 		goto err_pwr_ack;
 
+	/*
+	 * In MT8189 mminfra power domain, the bus protect policy separates
+	 * into two parts, one is set before subsys clocks enabled, and another
+	 * need to enable after subsys clocks enable.
+	 */
+	ret = scpsys_bus_protect_disable(pd, BUS_PROT_IGNORE_SUBCLK);
+	if (ret < 0)
+		goto err_pwr_ack;
+
 	/*
 	 * In few Mediatek platforms(e.g. MT6779), the bus protect policy is
 	 * stricter, which leads to bus protect release must be prior to bus
@@ -648,7 +666,7 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	if (ret < 0)
 		goto err_disable_subsys_clks;
 
-	ret = scpsys_bus_protect_disable(pd);
+	ret = scpsys_bus_protect_disable(pd, 0);
 	if (ret < 0)
 		goto err_disable_sram;
 
@@ -662,7 +680,7 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	return 0;
 
 err_enable_bus_protect:
-	scpsys_bus_protect_enable(pd);
+	scpsys_bus_protect_enable(pd, 0);
 err_disable_sram:
 	scpsys_sram_disable(pd);
 err_disable_subsys_clks:
@@ -683,7 +701,7 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 	bool tmp;
 	int ret;
 
-	ret = scpsys_bus_protect_enable(pd);
+	ret = scpsys_bus_protect_enable(pd, 0);
 	if (ret < 0)
 		return ret;
 
@@ -697,6 +715,10 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 
 	clk_bulk_disable_unprepare(pd->num_subsys_clks, pd->subsys_clks);
 
+	ret = scpsys_bus_protect_enable(pd, BUS_PROT_IGNORE_SUBCLK);
+	if (ret < 0)
+		return ret;
+
 	if (MTK_SCPD_CAPS(pd, MTK_SCPD_MODEM_PWRSEQ))
 		scpsys_modem_pwrseq_off(pd);
 	else
@@ -1150,6 +1172,10 @@ static const struct of_device_id scpsys_of_match[] = {
 		.compatible = "mediatek,mt8188-power-controller",
 		.data = &mt8188_scpsys_data,
 	},
+	{
+		.compatible = "mediatek,mt8189-power-controller",
+		.data = &mt8189_scpsys_data,
+	},
 	{
 		.compatible = "mediatek,mt8192-power-controller",
 		.data = &mt8192_scpsys_data,
@@ -1182,7 +1208,6 @@ static int scpsys_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	const struct scpsys_soc_data *soc;
-	struct device_node *node;
 	struct device *parent;
 	struct scpsys *scpsys;
 	int num_domains, ret;
@@ -1203,7 +1228,7 @@ static int scpsys_probe(struct platform_device *pdev)
 	scpsys->soc_data = soc;
 
 	scpsys->pd_data.domains = scpsys->domains;
-	scpsys->pd_data.num_domains = soc->num_domains;
+	scpsys->pd_data.num_domains = num_domains;
 
 	parent = dev->parent;
 	if (!parent) {
@@ -1226,21 +1251,18 @@ static int scpsys_probe(struct platform_device *pdev)
 		return ret;
 
 	ret = -ENODEV;
-	for_each_available_child_of_node(np, node) {
+	for_each_available_child_of_node_scoped(np, node) {
 		struct generic_pm_domain *domain;
 
 		domain = scpsys_add_one_domain(scpsys, node);
 		if (IS_ERR(domain)) {
 			ret = PTR_ERR(domain);
-			of_node_put(node);
 			goto err_cleanup_domains;
 		}
 
 		ret = scpsys_add_subdomain(scpsys, node);
-		if (ret) {
-			of_node_put(node);
+		if (ret)
 			goto err_cleanup_domains;
-		}
 	}
 
 	if (ret) {
diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.h b/drivers/pmdomain/mediatek/mtk-pm-domains.h
index f608e6ec47446..a5dca24cbc2f5 100644
--- a/drivers/pmdomain/mediatek/mtk-pm-domains.h
+++ b/drivers/pmdomain/mediatek/mtk-pm-domains.h
@@ -56,6 +56,7 @@ enum scpsys_bus_prot_flags {
 	BUS_PROT_REG_UPDATE = BIT(1),
 	BUS_PROT_IGNORE_CLR_ACK = BIT(2),
 	BUS_PROT_INVERTED = BIT(3),
+	BUS_PROT_IGNORE_SUBCLK = BIT(4),
 };
 
 enum scpsys_bus_prot_block {
@@ -95,6 +96,10 @@ enum scpsys_bus_prot_block {
 		_BUS_PROT(_hwip, _mask, _set, _clr, _mask, _sta,	\
 			  BUS_PROT_REG_UPDATE)
 
+#define BUS_PROT_WR_IGN_SUBCLK(_hwip, _mask, _set, _clr, _sta)		\
+		_BUS_PROT(_hwip, _mask, _set, _clr, _mask, _sta,	\
+			  BUS_PROT_IGNORE_CLR_ACK | BUS_PROT_IGNORE_SUBCLK)
+
 #define BUS_PROT_INFRA_UPDATE_TOPAXI(_mask)			\
 		BUS_PROT_UPDATE(INFRA, _mask,			\
 				INFRA_TOPAXI_PROTECTEN,		\
diff --git a/drivers/pmdomain/mediatek/mtk-scpsys.c b/drivers/pmdomain/mediatek/mtk-scpsys.c
index 1a80c1537a43d..efc8f276ffce1 100644
--- a/drivers/pmdomain/mediatek/mtk-scpsys.c
+++ b/drivers/pmdomain/mediatek/mtk-scpsys.c
@@ -867,6 +867,16 @@ static const struct scp_domain_data scp_domain_data_mt7622[] = {
 		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_WB,
 		.caps = MTK_SCPD_ACTIVE_WAKEUP | MTK_SCPD_FWAIT_SRAM,
 	},
+	[MT7622_POWER_DOMAIN_AUDIO] = {
+		.name = "audsys",
+		.sta_mask = PWR_STATUS_AUDIO,
+		.ctl_offs = SPM_AUDIO_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_NONE},
+		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_HIF1,
+		.caps = MTK_SCPD_ACTIVE_WAKEUP,
+	},
 };
 
 /*
diff --git a/drivers/pmdomain/qcom/cpr.c b/drivers/pmdomain/qcom/cpr.c
index 3ee8184e4be36..c30690ef6919c 100644
--- a/drivers/pmdomain/qcom/cpr.c
+++ b/drivers/pmdomain/qcom/cpr.c
@@ -239,7 +239,6 @@ struct cpr_drv {
 	u32			gcnt;
 	unsigned long		flags;
 
-	struct fuse_corner	*fuse_corners;
 	struct corner		*corners;
 
 	const struct cpr_desc *desc;
@@ -247,6 +246,8 @@ struct cpr_drv {
 	const struct cpr_fuse *cpr_fuses;
 
 	struct dentry *debugfs;
+
+	struct fuse_corner	fuse_corners[];
 };
 
 static bool cpr_is_allowed(struct cpr_drv *drv)
@@ -1600,19 +1601,15 @@ static int cpr_probe(struct platform_device *pdev)
 	if (!data || !data->cpr_desc || !data->acc_desc)
 		return -EINVAL;
 
-	drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
+	drv = devm_kzalloc(dev,
+			struct_size(drv, fuse_corners, data->cpr_desc->num_fuse_corners),
+			GFP_KERNEL);
 	if (!drv)
 		return -ENOMEM;
 	drv->dev = dev;
 	drv->desc = data->cpr_desc;
 	drv->acc_desc = data->acc_desc;
 
-	drv->fuse_corners = devm_kcalloc(dev, drv->desc->num_fuse_corners,
-					 sizeof(*drv->fuse_corners),
-					 GFP_KERNEL);
-	if (!drv->fuse_corners)
-		return -ENOMEM;
-
 	np = of_parse_phandle(dev->of_node, "acc-syscon", 0);
 	if (!np)
 		return -ENODEV;
diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c
index 19849703be4ac..b0c80e4bd86c9 100644
--- a/drivers/pmdomain/qcom/rpmhpd.c
+++ b/drivers/pmdomain/qcom/rpmhpd.c
@@ -217,6 +217,25 @@ static struct rpmhpd gmxc = {
 	.res_name = "gmxc.lvl",
 };
 
+/* Eliza RPMH powerdomains */
+static struct rpmhpd *eliza_rpmhpds[] = {
+	[RPMHPD_CX] = &cx,
+	[RPMHPD_CX_AO] = &cx_ao,
+	[RPMHPD_EBI] = &ebi,
+	[RPMHPD_GFX] = &gfx,
+	[RPMHPD_LCX] = &lcx,
+	[RPMHPD_LMX] = &lmx,
+	[RPMHPD_MSS] = &mss,
+	[RPMHPD_MX] = &mx,
+	[RPMHPD_MX_AO] = &mx_ao,
+	[RPMHPD_NSP] = &nsp,
+};
+
+static const struct rpmhpd_desc eliza_desc = {
+	.rpmhpds = eliza_rpmhpds,
+	.num_pds = ARRAY_SIZE(eliza_rpmhpds),
+};
+
 /* Milos RPMH powerdomains */
 static struct rpmhpd *milos_rpmhpds[] = {
 	[RPMHPD_CX] = &cx,
@@ -795,6 +814,7 @@ static const struct rpmhpd_desc qcs615_desc = {
 };
 
 static const struct of_device_id rpmhpd_match_table[] = {
+	{ .compatible = "qcom,eliza-rpmhpd", .data = &eliza_desc },
 	{ .compatible = "qcom,glymur-rpmhpd", .data = &glymur_desc },
 	{ .compatible = "qcom,kaanapali-rpmhpd", .data = &kaanapali_desc },
 	{ .compatible = "qcom,milos-rpmhpd", .data = &milos_desc },
diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c
index 98ab4f9ea9bff..15a11ff282c3f 100644
--- a/drivers/pmdomain/qcom/rpmpd.c
+++ b/drivers/pmdomain/qcom/rpmpd.c
@@ -978,7 +978,7 @@ static int rpmpd_send_corner(struct rpmpd *pd, int state, unsigned int corner)
 
 	return qcom_rpm_smd_write(rpmpd_smd_rpm, state, pd->res_type, pd->res_id,
 				  &req, sizeof(req));
-};
+}
 
 static void to_active_sleep(struct rpmpd *pd, unsigned int corner,
 			    unsigned int *active, unsigned int *sleep)
diff --git a/drivers/pmdomain/rockchip/pm-domains.c b/drivers/pmdomain/rockchip/pm-domains.c
index 44d34840ede7a..490bbb1d1d8e8 100644
--- a/drivers/pmdomain/rockchip/pm-domains.c
+++ b/drivers/pmdomain/rockchip/pm-domains.c
@@ -705,10 +705,9 @@ static int rockchip_pd_power_on(struct generic_pm_domain *domain)
 	int ret;
 
 	ret = rockchip_pd_regulator_enable(pd);
-	if (ret) {
-		dev_err(pd->pmu->dev, "Failed to enable supply: %d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(pd->pmu->dev, ret,
+				     "Failed to enable supply: %d\n", ret);
 
 	ret = rockchip_pd_power(pd, true);
 	if (ret)
diff --git a/drivers/pmdomain/sunxi/sun55i-pck600.c b/drivers/pmdomain/sunxi/sun55i-pck600.c
index c7ab515145315..1d47bbd35ced2 100644
--- a/drivers/pmdomain/sunxi/sun55i-pck600.c
+++ b/drivers/pmdomain/sunxi/sun55i-pck600.c
@@ -52,6 +52,7 @@ struct sunxi_pck600_desc {
 	u32 logic_power_switch0_delay;
 	u32 logic_power_switch1_delay;
 	u32 off2on_delay;
+	bool has_rst_clk;
 };
 
 struct sunxi_pck600_pd {
@@ -151,9 +152,11 @@ static int sunxi_pck600_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	rst = devm_reset_control_get_exclusive_released(dev, NULL);
-	if (IS_ERR(rst))
-		return dev_err_probe(dev, PTR_ERR(rst), "failed to get reset control\n");
+	if (desc->has_rst_clk) {
+		rst = devm_reset_control_get_exclusive_released(dev, NULL);
+		if (IS_ERR(rst))
+			return dev_err_probe(dev, PTR_ERR(rst), "failed to get reset control\n");
+	}
 
 	clk = devm_clk_get_enabled(dev, NULL);
 	if (IS_ERR(clk))
@@ -206,7 +209,27 @@ static const struct sunxi_pck600_desc sun55i_a523_pck600_desc = {
 	.device_ctrl1_delay = 0xffff,
 	.logic_power_switch0_delay = 0x8080808,
 	.logic_power_switch1_delay = 0x808,
-	.off2on_delay = 0x8
+	.off2on_delay = 0x8,
+	.has_rst_clk = true,
+};
+
+static const char * const sun60i_a733_pck600_pd_names[] = {
+	"VI", "DE_SYS", "VE_DEC", "VE_ENC", "NPU",
+	"GPU_TOP", "GPU_CORE", "PCIE", "USB2", "VO", "VO1"
+};
+
+static const struct sunxi_pck600_desc sun60i_a733_pck600_desc = {
+	.pd_names = sun60i_a733_pck600_pd_names,
+	.num_domains = ARRAY_SIZE(sun60i_a733_pck600_pd_names),
+	.logic_power_switch0_delay_offset = 0xc00,
+	.logic_power_switch1_delay_offset = 0xc04,
+	.off2on_delay_offset = 0xc10,
+	.device_ctrl0_delay = 0x1f1f1f,
+	.device_ctrl1_delay = 0x1f1f,
+	.logic_power_switch0_delay = 0x8080808,
+	.logic_power_switch1_delay = 0x808,
+	.off2on_delay = 0x8,
+	.has_rst_clk = false,
 };
 
 static const struct of_device_id sunxi_pck600_of_match[] = {
@@ -214,6 +237,10 @@ static const struct of_device_id sunxi_pck600_of_match[] = {
 		.compatible	= "allwinner,sun55i-a523-pck-600",
 		.data		= &sun55i_a523_pck600_desc,
 	},
+	{
+		.compatible	= "allwinner,sun60i-a733-pck-600",
+		.data		= &sun60i_a733_pck600_desc,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, sunxi_pck600_of_match);
diff --git a/drivers/pmdomain/ti/omap_prm.c b/drivers/pmdomain/ti/omap_prm.c
index 5142f064bf5cd..64a187f79a1a7 100644
--- a/drivers/pmdomain/ti/omap_prm.c
+++ b/drivers/pmdomain/ti/omap_prm.c
@@ -655,6 +655,7 @@ static int omap_prm_domain_attach_dev(struct generic_pm_domain *domain,
 	if (pd_args.args_count != 0)
 		dev_warn(dev, "%s: unusupported #power-domain-cells: %i\n",
 			 prmd->pd.name, pd_args.args_count);
+	of_node_put(pd_args.np);
 
 	genpd_data = dev_gpd_data(dev);
 	genpd_data->data = NULL;
diff --git a/drivers/pmdomain/ti/ti_sci_pm_domains.c b/drivers/pmdomain/ti/ti_sci_pm_domains.c
index e5d1934f78d9e..18d33bc35dee1 100644
--- a/drivers/pmdomain/ti/ti_sci_pm_domains.c
+++ b/drivers/pmdomain/ti/ti_sci_pm_domains.c
@@ -91,8 +91,9 @@ static inline void ti_sci_pd_set_wkup_constraint(struct device *dev)
 		 * If device can wakeup using IO daisy chain wakeups,
 		 * we do not want to set a constraint.
 		 */
-		if (dev->power.wakeirq) {
-			dev_dbg(dev, "%s: has wake IRQ, not setting constraints\n", __func__);
+		if (device_out_band_wakeup(dev)) {
+			dev_dbg(dev, "%s: has out of band wakeup, not setting constraints\n", \
+					__func__);
 			return;
 		}
 
diff --git a/drivers/power/reset/keystone-reset.c b/drivers/power/reset/keystone-reset.c
index d9268d150e1f9..3c44cd6cee0aa 100644
--- a/drivers/power/reset/keystone-reset.c
+++ b/drivers/power/reset/keystone-reset.c
@@ -48,8 +48,7 @@ static inline int rsctrl_enable_rspll_write(void)
 				  RSCTRL_KEY_MASK, RSCTRL_KEY);
 }
 
-static int rsctrl_restart_handler(struct notifier_block *this,
-				  unsigned long mode, void *cmd)
+static int rsctrl_restart_handler(struct sys_off_data *data)
 {
 	/* enable write access to RSTCTRL */
 	rsctrl_enable_rspll_write();
@@ -61,11 +60,6 @@ static int rsctrl_restart_handler(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block rsctrl_restart_nb = {
-	.notifier_call = rsctrl_restart_handler,
-	.priority = 128,
-};
-
 static const struct of_device_id rsctrl_of_match[] = {
 	{.compatible = "ti,keystone-reset", },
 	{},
@@ -140,7 +134,8 @@ static int rsctrl_probe(struct platform_device *pdev)
 			return ret;
 	}
 
-	ret = register_restart_handler(&rsctrl_restart_nb);
+	ret = devm_register_sys_off_handler(dev, SYS_OFF_MODE_RESTART, 128,
+					    rsctrl_restart_handler, NULL);
 	if (ret)
 		dev_err(dev, "cannot register restart handler (err=%d)\n", ret);
 
diff --git a/drivers/power/reset/reboot-mode.c b/drivers/power/reset/reboot-mode.c
index fba53f638da04..d20e44db05325 100644
--- a/drivers/power/reset/reboot-mode.c
+++ b/drivers/power/reset/reboot-mode.c
@@ -4,12 +4,16 @@
  */
 
 #include <linux/device.h>
+#include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/list.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/reboot.h>
 #include <linux/reboot-mode.h>
+#include <linux/slab.h>
+#include <linux/string.h>
 
 #define PREFIX "mode-"
 
@@ -19,6 +23,54 @@ struct mode_info {
 	struct list_head list;
 };
 
+struct reboot_mode_sysfs_data {
+	struct device *reboot_mode_device;
+	struct list_head head;
+};
+
+static inline void reboot_mode_release_list(struct reboot_mode_sysfs_data *priv)
+{
+	struct mode_info *info;
+	struct mode_info *next;
+
+	list_for_each_entry_safe(info, next, &priv->head, list) {
+		list_del(&info->list);
+		kfree_const(info->mode);
+		kfree(info);
+	}
+}
+
+static ssize_t reboot_modes_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct reboot_mode_sysfs_data *priv;
+	struct mode_info *sysfs_info;
+	ssize_t size = 0;
+
+	priv = dev_get_drvdata(dev);
+	if (!priv)
+		return -ENODATA;
+
+	list_for_each_entry(sysfs_info, &priv->head, list)
+		size += sysfs_emit_at(buf, size, "%s ", sysfs_info->mode);
+
+	if (!size)
+		return -ENODATA;
+
+	return size + sysfs_emit_at(buf, size - 1, "\n");
+}
+static DEVICE_ATTR_RO(reboot_modes);
+
+static struct attribute *reboot_mode_attrs[] = {
+	&dev_attr_reboot_modes.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(reboot_mode);
+
+static const struct class reboot_mode_class = {
+	.name = "reboot-mode",
+	.dev_groups = reboot_mode_groups,
+};
+
 static unsigned int get_reboot_mode_magic(struct reboot_mode_driver *reboot,
 					  const char *cmd)
 {
@@ -62,6 +114,52 @@ static int reboot_mode_notify(struct notifier_block *this,
 	return NOTIFY_DONE;
 }
 
+static int reboot_mode_create_device(struct reboot_mode_driver *reboot)
+{
+	struct reboot_mode_sysfs_data *priv;
+	struct mode_info *sysfs_info;
+	struct mode_info *info;
+	int ret;
+
+	priv = kzalloc_obj(*priv, GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&priv->head);
+
+	list_for_each_entry(info, &reboot->head, list) {
+		sysfs_info = kzalloc_obj(*sysfs_info, GFP_KERNEL);
+		if (!sysfs_info) {
+			ret = -ENOMEM;
+			goto error;
+		}
+
+		sysfs_info->mode = kstrdup_const(info->mode, GFP_KERNEL);
+		if (!sysfs_info->mode) {
+			kfree(sysfs_info);
+			ret = -ENOMEM;
+			goto error;
+		}
+
+		list_add_tail(&sysfs_info->list, &priv->head);
+	}
+
+	priv->reboot_mode_device = device_create(&reboot_mode_class, NULL, 0,
+						 (void *)priv, "%s",
+						 reboot->dev->driver->name);
+	if (IS_ERR(priv->reboot_mode_device)) {
+		ret = PTR_ERR(priv->reboot_mode_device);
+		goto error;
+	}
+
+	return 0;
+
+error:
+	reboot_mode_release_list(priv);
+	kfree(priv);
+	return ret;
+}
+
 /**
  * reboot_mode_register - register a reboot mode driver
  * @reboot: reboot mode driver
@@ -113,16 +211,49 @@ int reboot_mode_register(struct reboot_mode_driver *reboot)
 	reboot->reboot_notifier.notifier_call = reboot_mode_notify;
 	register_reboot_notifier(&reboot->reboot_notifier);
 
+	ret = reboot_mode_create_device(reboot);
+	if (ret)
+		goto error;
+
 	return 0;
 
 error:
-	list_for_each_entry(info, &reboot->head, list)
-		kfree_const(info->mode);
-
+	reboot_mode_unregister(reboot);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(reboot_mode_register);
 
+static int reboot_mode_match_by_name(struct device *dev, const void *data)
+{
+	const char *name = data;
+
+	if (!dev || !data)
+		return 0;
+
+	return dev_name(dev) && strcmp(dev_name(dev), name) == 0;
+}
+
+static inline void reboot_mode_unregister_device(struct reboot_mode_driver *reboot)
+{
+	struct reboot_mode_sysfs_data *priv;
+	struct device *reboot_mode_device;
+
+	reboot_mode_device = class_find_device(&reboot_mode_class, NULL, reboot->dev->driver->name,
+					       reboot_mode_match_by_name);
+
+	if (!reboot_mode_device)
+		return;
+
+	priv = dev_get_drvdata(reboot_mode_device);
+	device_unregister(reboot_mode_device);
+
+	if (!priv)
+		return;
+
+	reboot_mode_release_list(priv);
+	kfree(priv);
+}
+
 /**
  * reboot_mode_unregister - unregister a reboot mode driver
  * @reboot: reboot mode driver
@@ -132,6 +263,7 @@ int reboot_mode_unregister(struct reboot_mode_driver *reboot)
 	struct mode_info *info;
 
 	unregister_reboot_notifier(&reboot->reboot_notifier);
+	reboot_mode_unregister_device(reboot);
 
 	list_for_each_entry(info, &reboot->head, list)
 		kfree_const(info->mode);
@@ -199,6 +331,19 @@ void devm_reboot_mode_unregister(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_reboot_mode_unregister);
 
+static int __init reboot_mode_init(void)
+{
+	return class_register(&reboot_mode_class);
+}
+
+static void __exit reboot_mode_exit(void)
+{
+	class_unregister(&reboot_mode_class);
+}
+
+subsys_initcall(reboot_mode_init);
+module_exit(reboot_mode_exit);
+
 MODULE_AUTHOR("Andy Yan <andy.yan@rock-chips.com>");
 MODULE_DESCRIPTION("System reboot mode core library");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig
index 92f9f7aae92f2..e8a1bdb246c4d 100644
--- a/drivers/power/supply/Kconfig
+++ b/drivers/power/supply/Kconfig
@@ -229,6 +229,17 @@ config BATTERY_SAMSUNG_SDI
 	  Say Y to enable support for Samsung SDI battery data.
 	  These batteries are used in Samsung mobile phones.
 
+config BATTERY_S2MU005
+	tristate "Samsung S2MU005 PMIC fuel gauge driver"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y to enable support for the Samsung S2MU005 PMIC integrated
+	  fuel gauge, which works indepenently of the PMIC battery charger
+	  counterpart, and reports battery metrics.
+
+	  This driver, if built as a module, will be called s2mu005-fuel-gauge.
+
 config BATTERY_COLLIE
 	tristate "Sharp SL-5500 (collie) battery"
 	depends on SA1100_COLLIE && MCP_UCB1200
@@ -1132,4 +1143,15 @@ config FUEL_GAUGE_MM8013
 	  the state of charge, temperature, cycle count, actual and design
 	  capacity, etc.
 
+config MACSMC_POWER
+	tristate "Apple SMC Battery and Power Driver"
+	depends on MFD_MACSMC
+	help
+	  This driver provides support for the battery and AC adapter on
+	  Apple Silicon machines. It exposes battery telemetry (voltage,
+	  current, health) and AC adapter status through the standard Linux
+	  power supply framework.
+
+	  Say Y or M here if you have an Apple Silicon based Mac.
+
 endif # POWER_SUPPLY
diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile
index 4b79d5abc49a7..f2efbb82707c7 100644
--- a/drivers/power/supply/Makefile
+++ b/drivers/power/supply/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_BATTERY_PMU)	+= pmu_battery.o
 obj-$(CONFIG_BATTERY_QCOM_BATTMGR)	+= qcom_battmgr.o
 obj-$(CONFIG_BATTERY_OLPC)	+= olpc_battery.o
 obj-$(CONFIG_BATTERY_SAMSUNG_SDI)	+= samsung-sdi-battery.o
+obj-$(CONFIG_BATTERY_S2MU005)	+= s2mu005-battery.o
 obj-$(CONFIG_BATTERY_COLLIE)	+= collie_battery.o
 obj-$(CONFIG_BATTERY_INGENIC)	+= ingenic-battery.o
 obj-$(CONFIG_BATTERY_INTEL_DC_TI) += intel_dc_ti_battery.o
@@ -128,3 +129,4 @@ obj-$(CONFIG_CHARGER_SURFACE)	+= surface_charger.o
 obj-$(CONFIG_BATTERY_UG3105)	+= ug3105_battery.o
 obj-$(CONFIG_CHARGER_QCOM_SMB2)	+= qcom_smbx.o
 obj-$(CONFIG_FUEL_GAUGE_MM8013)	+= mm8013.o
+obj-$(CONFIG_MACSMC_POWER)	+= macsmc-power.o
diff --git a/drivers/power/supply/axp288_charger.c b/drivers/power/supply/axp288_charger.c
index ac05942e4e6ac..ea0f5caee8f0e 100644
--- a/drivers/power/supply/axp288_charger.c
+++ b/drivers/power/supply/axp288_charger.c
@@ -10,6 +10,7 @@
 #include <linux/acpi.h>
 #include <linux/bitops.h>
 #include <linux/module.h>
+#include <linux/devm-helpers.h>
 #include <linux/device.h>
 #include <linux/regmap.h>
 #include <linux/workqueue.h>
@@ -821,14 +822,6 @@ static int charger_init_hw_regs(struct axp288_chrg_info *info)
 	return 0;
 }
 
-static void axp288_charger_cancel_work(void *data)
-{
-	struct axp288_chrg_info *info = data;
-
-	cancel_work_sync(&info->otg.work);
-	cancel_work_sync(&info->cable.work);
-}
-
 static int axp288_charger_probe(struct platform_device *pdev)
 {
 	int ret, i, pirq;
@@ -866,12 +859,10 @@ static int axp288_charger_probe(struct platform_device *pdev)
 	info->regmap_irqc = axp20x->regmap_irqc;
 
 	info->cable.edev = extcon_get_extcon_dev(AXP288_EXTCON_DEV_NAME);
-	if (IS_ERR(info->cable.edev)) {
-		dev_err_probe(dev, PTR_ERR(info->cable.edev),
-			      "extcon_get_extcon_dev(%s) failed\n",
-			      AXP288_EXTCON_DEV_NAME);
-		return PTR_ERR(info->cable.edev);
-	}
+	if (IS_ERR(info->cable.edev))
+		return dev_err_probe(dev, PTR_ERR(info->cable.edev),
+				     "extcon_get_extcon_dev(%s) failed\n",
+				     AXP288_EXTCON_DEV_NAME);
 
 	/*
 	 * On devices with broken ACPI GPIO event handlers there also is no ACPI
@@ -885,12 +876,11 @@ static int axp288_charger_probe(struct platform_device *pdev)
 
 	if (extcon_name) {
 		info->otg.cable = extcon_get_extcon_dev(extcon_name);
-		if (IS_ERR(info->otg.cable)) {
-			dev_err_probe(dev, PTR_ERR(info->otg.cable),
-				      "extcon_get_extcon_dev(%s) failed\n",
-				      USB_HOST_EXTCON_NAME);
-			return PTR_ERR(info->otg.cable);
-		}
+		if (IS_ERR(info->otg.cable))
+			return dev_err_probe(dev, PTR_ERR(info->otg.cable),
+					     "extcon_get_extcon_dev(%s) failed\n",
+					     USB_HOST_EXTCON_NAME);
+
 		dev_info(dev, "Using " USB_HOST_EXTCON_HID " extcon for usb-id\n");
 	}
 
@@ -904,38 +894,39 @@ static int axp288_charger_probe(struct platform_device *pdev)
 	charger_cfg.drv_data = info;
 	info->psy_usb = devm_power_supply_register(dev, &axp288_charger_desc,
 						   &charger_cfg);
-	if (IS_ERR(info->psy_usb)) {
-		ret = PTR_ERR(info->psy_usb);
-		dev_err(dev, "failed to register power supply: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(info->psy_usb))
+		return dev_err_probe(dev, PTR_ERR(info->psy_usb),
+				     "failed to register power supply: %d\n", ret);
 
 	/* Cancel our work on cleanup, register this before the notifiers */
-	ret = devm_add_action(dev, axp288_charger_cancel_work, info);
+	ret = devm_work_autocancel(dev, &info->cable.work,
+				   axp288_charger_extcon_evt_worker);
 	if (ret)
 		return ret;
 
 	/* Register for extcon notification */
-	INIT_WORK(&info->cable.work, axp288_charger_extcon_evt_worker);
 	info->cable.nb.notifier_call = axp288_charger_handle_cable_evt;
 	ret = devm_extcon_register_notifier_all(dev, info->cable.edev,
 						&info->cable.nb);
-	if (ret) {
-		dev_err(dev, "failed to register cable extcon notifier\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register cable extcon notifier\n");
+
 	schedule_work(&info->cable.work);
 
+	ret = devm_work_autocancel(dev, &info->otg.work,
+				   axp288_charger_otg_evt_worker);
+	if (ret)
+		return ret;
+
 	/* Register for OTG notification */
-	INIT_WORK(&info->otg.work, axp288_charger_otg_evt_worker);
 	info->otg.id_nb.notifier_call = axp288_charger_handle_otg_evt;
 	if (info->otg.cable) {
 		ret = devm_extcon_register_notifier(dev, info->otg.cable,
 					EXTCON_USB_HOST, &info->otg.id_nb);
-		if (ret) {
-			dev_err(dev, "failed to register EXTCON_USB_HOST notifier\n");
-			return ret;
-		}
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "failed to register EXTCON_USB_HOST notifier\n");
+
 		schedule_work(&info->otg.work);
 	}
 
@@ -954,11 +945,9 @@ static int axp288_charger_probe(struct platform_device *pdev)
 		ret = devm_request_threaded_irq(&info->pdev->dev, info->irq[i],
 					NULL, axp288_charger_irq_thread_handler,
 					IRQF_ONESHOT, info->pdev->name, info);
-		if (ret) {
-			dev_err(dev, "failed to request interrupt=%d\n",
-								info->irq[i]);
-			return ret;
-		}
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to request interrupt=%d\n",
+					     info->irq[i]);
 	}
 
 	return 0;
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index ed0ceae8d90b1..55da91bacc3e6 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -9,6 +9,7 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
+#include <linux/devm-helpers.h>
 #include <linux/pm_runtime.h>
 #include <linux/power_supply.h>
 #include <linux/power/bq24190_charger.h>
@@ -2087,8 +2088,11 @@ static int bq24190_probe(struct i2c_client *client)
 	bdi->charge_type = POWER_SUPPLY_CHARGE_TYPE_FAST;
 	bdi->f_reg = 0;
 	bdi->ss_reg = BQ24190_REG_SS_VBUS_STAT_MASK; /* impossible state */
-	INIT_DELAYED_WORK(&bdi->input_current_limit_work,
-			  bq24190_input_current_limit_work);
+
+	ret = devm_delayed_work_autocancel(dev, &bdi->input_current_limit_work,
+					   bq24190_input_current_limit_work);
+	if (ret)
+		return ret;
 
 	i2c_set_clientdata(client, bdi);
 
@@ -2198,7 +2202,6 @@ static void bq24190_remove(struct i2c_client *client)
 	struct bq24190_dev_info *bdi = i2c_get_clientdata(client);
 	int error;
 
-	cancel_delayed_work_sync(&bdi->input_current_limit_work);
 	error = pm_runtime_resume_and_get(bdi->dev);
 	if (error < 0)
 		dev_warn(bdi->dev, "pm_runtime_get failed: %i\n", error);
diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c
index 507fdc1c866d5..7b7bdce3162fc 100644
--- a/drivers/power/supply/cpcap-battery.c
+++ b/drivers/power/supply/cpcap-battery.c
@@ -387,7 +387,7 @@ static const struct cpcap_battery_config cpcap_battery_bw8x_data = {
  * Safe values for any lipo battery likely to fit into a mapphone
  * battery bay.
  */
-static const struct cpcap_battery_config cpcap_battery_unkown_data = {
+static const struct cpcap_battery_config cpcap_battery_unknown_data = {
 	.cd_factor = 0x3cc,
 	.info.technology = POWER_SUPPLY_TECHNOLOGY_LION,
 	.info.voltage_max_design = 4200000,
@@ -404,6 +404,30 @@ static int cpcap_battery_match_nvmem(struct device *dev, const void *data)
 		return 0;
 }
 
+static void cpcap_battery_update_battery_data(struct cpcap_battery_ddata *ddata)
+{
+	struct power_supply_battery_info *info;
+
+	if (power_supply_get_battery_info(ddata->psy, &info) < 0)
+		return;
+
+	if (info->technology > 0)
+		ddata->config.info.technology = info->technology;
+
+	if (info->voltage_max_design_uv > 0)
+		ddata->config.info.voltage_max_design = info->voltage_max_design_uv;
+
+	if (info->voltage_min_design_uv > 0)
+		ddata->config.info.voltage_min_design = info->voltage_min_design_uv;
+
+	if (info->charge_full_design_uah > 0)
+		ddata->config.info.charge_full_design = info->charge_full_design_uah;
+
+	if (info->constant_charge_voltage_max_uv > 0)
+		ddata->config.bat.constant_charge_voltage_max_uv =
+			info->constant_charge_voltage_max_uv;
+}
+
 static void cpcap_battery_detect_battery_type(struct cpcap_battery_ddata *ddata)
 {
 	struct nvmem_device *nvmem;
@@ -429,8 +453,11 @@ static void cpcap_battery_detect_battery_type(struct cpcap_battery_ddata *ddata)
 		ddata->config = cpcap_battery_bw8x_data;
 		break;
 	default:
-		ddata->config = cpcap_battery_unkown_data;
+		ddata->config = cpcap_battery_unknown_data;
 	}
+
+	if (ddata->psy)
+		cpcap_battery_update_battery_data(ddata);
 }
 
 /**
diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c
index a05dcc4a48f22..286524d2318cb 100644
--- a/drivers/power/supply/cw2015_battery.c
+++ b/drivers/power/supply/cw2015_battery.c
@@ -694,7 +694,8 @@ static int cw_bat_probe(struct i2c_client *client)
 			 "No monitored battery, some properties will be missing\n");
 	}
 
-	cw_bat->battery_workqueue = create_singlethread_workqueue("rk_battery");
+	cw_bat->battery_workqueue = devm_alloc_ordered_workqueue(&client->dev,
+								 "rk_battery", 0);
 	if (!cw_bat->battery_workqueue)
 		return -ENOMEM;
 
diff --git a/drivers/power/supply/ipaq_micro_battery.c b/drivers/power/supply/ipaq_micro_battery.c
index ff8573a5ca6d0..5e3fe3852d0ec 100644
--- a/drivers/power/supply/ipaq_micro_battery.c
+++ b/drivers/power/supply/ipaq_micro_battery.c
@@ -7,6 +7,7 @@
  * Author : Linus Walleij <linus.walleij@linaro.org>
  */
 
+#include <linux/devm-helpers.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
@@ -232,49 +233,31 @@ static int micro_batt_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	mb->micro = dev_get_drvdata(pdev->dev.parent);
-	mb->wq = alloc_workqueue("ipaq-battery-wq",
-				 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
+	mb->wq = devm_alloc_workqueue(&pdev->dev, "ipaq-battery-wq",
+				      WQ_MEM_RECLAIM | WQ_PERCPU, 0);
 	if (!mb->wq)
 		return -ENOMEM;
 
-	INIT_DELAYED_WORK(&mb->update, micro_battery_work);
+	ret = devm_delayed_work_autocancel(&pdev->dev, &mb->update, micro_battery_work);
+	if (ret)
+		return ret;
+
 	platform_set_drvdata(pdev, mb);
 	queue_delayed_work(mb->wq, &mb->update, 1);
 
-	micro_batt_power = power_supply_register(&pdev->dev,
-						 &micro_batt_power_desc, NULL);
-	if (IS_ERR(micro_batt_power)) {
-		ret = PTR_ERR(micro_batt_power);
-		goto batt_err;
-	}
+	micro_batt_power = devm_power_supply_register(&pdev->dev,
+						      &micro_batt_power_desc,
+						      NULL);
+	if (IS_ERR(micro_batt_power))
+		return PTR_ERR(micro_batt_power);
 
-	micro_ac_power = power_supply_register(&pdev->dev,
-					       &micro_ac_power_desc, NULL);
-	if (IS_ERR(micro_ac_power)) {
-		ret = PTR_ERR(micro_ac_power);
-		goto ac_err;
-	}
+	micro_ac_power = devm_power_supply_register(&pdev->dev,
+						    &micro_ac_power_desc, NULL);
+	if (IS_ERR(micro_ac_power))
+		return PTR_ERR(micro_ac_power);
 
 	dev_info(&pdev->dev, "iPAQ micro battery driver\n");
 	return 0;
-
-ac_err:
-	power_supply_unregister(micro_batt_power);
-batt_err:
-	cancel_delayed_work_sync(&mb->update);
-	destroy_workqueue(mb->wq);
-	return ret;
-}
-
-static void micro_batt_remove(struct platform_device *pdev)
-
-{
-	struct micro_battery *mb = platform_get_drvdata(pdev);
-
-	power_supply_unregister(micro_ac_power);
-	power_supply_unregister(micro_batt_power);
-	cancel_delayed_work_sync(&mb->update);
-	destroy_workqueue(mb->wq);
 }
 
 static int __maybe_unused micro_batt_suspend(struct device *dev)
@@ -303,7 +286,6 @@ static struct platform_driver micro_batt_device_driver = {
 		.pm	= &micro_batt_dev_pm_ops,
 	},
 	.probe		= micro_batt_probe,
-	.remove		= micro_batt_remove,
 };
 module_platform_driver(micro_batt_device_driver);
 
diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c
new file mode 100644
index 0000000000000..33ca07460f3a3
--- /dev/null
+++ b/drivers/power/supply/macsmc-power.c
@@ -0,0 +1,855 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Apple SMC Power/Battery Management Driver
+ *
+ * This driver exposes battery telemetry (voltage, current, temperature, health)
+ * and AC adapter status provided by the Apple SMC (System Management Controller)
+ * on Apple Silicon systems.
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+#include <linux/ctype.h>
+#include <linux/delay.h>
+#include <linux/devm-helpers.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/mfd/macsmc.h>
+#include <linux/notifier.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/power_supply.h>
+#include <linux/reboot.h>
+#include <linux/workqueue.h>
+
+#define MAX_STRING_LENGTH 256
+
+/*
+ * The SMC reports charge in mAh (Coulombs) but energy in mWh (Joules).
+ * We lack a register for "Nominal Voltage" or "Energy Accumulator".
+ * We use a fixed 3.8V/cell constant to approximate energy stats for userspace,
+ * derived from empirical data across supported MacBook models.
+ */
+#define MACSMC_NOMINAL_CELL_VOLTAGE_MV 3800
+
+/* SMC Key Flags */
+#define CHNC_BATTERY_FULL	BIT(0)
+#define CHNC_NO_CHARGER		BIT(7)
+#define CHNC_NOCHG_CH0C		BIT(14)
+#define CHNC_NOCHG_CH0B_CH0K	BIT(15)
+#define CHNC_BATTERY_FULL_2	BIT(18)
+#define CHNC_BMS_BUSY		BIT(23)
+#define CHNC_CHLS_LIMIT		BIT(24)
+#define CHNC_NOAC_CH0J		BIT(53)
+#define CHNC_NOAC_CH0I		BIT(54)
+
+#define CH0R_LOWER_FLAGS	GENMASK(15, 0)
+#define CH0R_NOAC_CH0I		BIT(0)
+#define CH0R_NOAC_DISCONNECTED	BIT(4)
+#define CH0R_NOAC_CH0J		BIT(5)
+#define CH0R_BMS_BUSY		BIT(8)
+#define CH0R_NOAC_CH0K		BIT(9)
+#define CH0R_NOAC_CHWA		BIT(11)
+
+#define CH0X_CH0C		BIT(0)
+#define CH0X_CH0B		BIT(1)
+
+#define ACSt_CAN_BOOT_AP	BIT(2)
+#define ACSt_CAN_BOOT_IBOOT	BIT(1)
+
+#define CHWA_CHLS_FIXED_START_OFFSET	5
+#define CHLS_MIN_END_THRESHOLD		10
+#define CHLS_FORCE_DISCHARGE		0x100
+#define CHWA_FIXED_END_THRESHOLD	80
+#define CHWA_PROP_WRITE_THRESHOLD	95
+
+#define MACSMC_MAX_BATT_PROPS		50
+#define MACSMC_MAX_AC_PROPS		10
+
+struct macsmc_power {
+	struct device *dev;
+	struct apple_smc *smc;
+
+	struct power_supply_desc ac_desc;
+	struct power_supply_desc batt_desc;
+
+	struct power_supply *batt;
+	struct power_supply *ac;
+
+	char model_name[MAX_STRING_LENGTH];
+	char serial_number[MAX_STRING_LENGTH];
+	char mfg_date[MAX_STRING_LENGTH];
+
+	/* Supported feature flags based on SMC key presence */
+	bool has_chwa; /* Charge limit (Modern firmware) */
+	bool has_chls; /* Charge limit (Older firmware) */
+	bool has_ch0i; /* Force discharge (Older firmware) */
+	bool has_ch0c; /* Inhibit charge (Older firmware) */
+	bool has_chte; /* Inhibit charge (Modern firmware) */
+
+	u8 num_cells;
+	int nominal_voltage_mv;
+
+	struct notifier_block nb;
+	struct work_struct critical_work;
+	bool emergency_shutdown_triggered;
+	bool orderly_shutdown_triggered;
+};
+
+static int macsmc_battery_get_status(struct macsmc_power *power)
+{
+	u64 nocharge_flags;
+	u32 nopower_flags;
+	u16 ac_current;
+	int charge_limit = 0;
+	bool limited = false;
+	bool flag;
+	int ret;
+
+	/*
+	 * B0AV (Voltage) is fundamental. If we can't read it, we assume the
+	 * battery is gone. CHCE (Hardware charger present) / CHCC (Hardware
+	 * charger capable) are fundamental status flags.
+	 * BSFC (System full charge) / CHSC (System charging) are fundamental
+	 * status flags.
+	 */
+
+	/* Check if power input is inhibited (e.g. BMS balancing cycle) */
+	ret = apple_smc_read_u32(power->smc, SMC_KEY(CH0R), &nopower_flags);
+	if (!ret && (nopower_flags & CH0R_LOWER_FLAGS & ~CH0R_BMS_BUSY))
+		return POWER_SUPPLY_STATUS_DISCHARGING;
+
+	/* Check if charger is present */
+	ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCE), &flag);
+	if (ret < 0)
+		return ret;
+	if (!flag)
+		return POWER_SUPPLY_STATUS_DISCHARGING;
+
+	/* Check if AC is charge capable */
+	ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCC), &flag);
+	if (ret < 0)
+		return ret;
+	if (!flag)
+		return POWER_SUPPLY_STATUS_DISCHARGING;
+
+	/* Check if AC input limit is too low */
+	ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &ac_current);
+	if (!ret && ac_current < 100)
+		return POWER_SUPPLY_STATUS_DISCHARGING;
+
+	/* Check if battery is full */
+	ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag);
+	if (ret < 0)
+		return ret;
+	if (flag)
+		return POWER_SUPPLY_STATUS_FULL;
+
+	/* Check for user-defined charge limits */
+	if (power->has_chls) {
+		u16 vu16;
+
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16);
+		if (ret == 0 && (vu16 & 0xff) >= CHLS_MIN_END_THRESHOLD)
+			charge_limit = (vu16 & 0xff) - CHWA_CHLS_FIXED_START_OFFSET;
+	} else if (power->has_chwa) {
+		ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag);
+		if (ret == 0 && flag)
+			charge_limit = CHWA_FIXED_END_THRESHOLD - CHWA_CHLS_FIXED_START_OFFSET;
+	}
+
+	if (charge_limit > 0) {
+		u8 buic = 0;
+
+		if (apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &buic) >= 0 &&
+		    buic >= charge_limit)
+			limited = true;
+	}
+
+	/* Check charging inhibitors */
+	ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags);
+	if (!ret) {
+		if (nocharge_flags & CHNC_BATTERY_FULL)
+			return POWER_SUPPLY_STATUS_FULL;
+		/* BMS busy shows up as inhibit, but we treat it as charging */
+		else if (nocharge_flags == CHNC_BMS_BUSY && !limited)
+			return POWER_SUPPLY_STATUS_CHARGING;
+		else if (nocharge_flags)
+			return POWER_SUPPLY_STATUS_NOT_CHARGING;
+		else
+			return POWER_SUPPLY_STATUS_CHARGING;
+	}
+
+	/* Fallback: System charging flag */
+	ret = apple_smc_read_flag(power->smc, SMC_KEY(CHSC), &flag);
+	if (ret < 0)
+		return ret;
+	if (!flag)
+		return POWER_SUPPLY_STATUS_NOT_CHARGING;
+
+	return POWER_SUPPLY_STATUS_CHARGING;
+}
+
+static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power)
+{
+	int ret;
+	u8 val8;
+	u8 chte_buf[4];
+
+	if (power->has_ch0i) {
+		ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8);
+		if (ret)
+			return ret;
+		if (val8 & CH0R_NOAC_CH0I)
+			return POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE;
+	}
+
+	if (power->has_chte) {
+		ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4);
+		if (ret < 0)
+			return ret;
+
+		if (chte_buf[0] == 0x01)
+			return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE;
+	} else if (power->has_ch0c) {
+		ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8);
+		if (ret)
+			return ret;
+		if (val8 & CH0X_CH0C)
+			return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE;
+	}
+
+	return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO;
+}
+
+static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val)
+{
+	int ret;
+
+	switch (val) {
+	case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO:
+		/* Reset all inhibitors to a known-good 'auto' state */
+		if (power->has_ch0i) {
+			ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0);
+			if (ret)
+				return ret;
+		}
+
+		if (power->has_chte) {
+			ret = apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 0);
+			if (ret)
+				return ret;
+		} else if (power->has_ch0c) {
+			ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0);
+			if (ret)
+				return ret;
+		}
+		return 0;
+
+	case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE:
+		if (power->has_chte)
+			return apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 1);
+		else if (power->has_ch0c)
+			return apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 1);
+		else
+			return -EOPNOTSUPP;
+
+	case POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE:
+		if (!power->has_ch0i)
+			return -EOPNOTSUPP;
+		return apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 1);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int macsmc_battery_get_date(const char *s, int *out)
+{
+	if (!isdigit(s[0]) || !isdigit(s[1]))
+		return -EOPNOTSUPP;
+
+	*out = (s[0] - '0') * 10 + s[1] - '0';
+	return 0;
+}
+
+static int macsmc_battery_get_capacity_level(struct macsmc_power *power)
+{
+	bool flag;
+	u32 val;
+	int ret;
+
+	/* Check for emergency shutdown condition */
+	if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val) >= 0 && val)
+		return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL;
+
+	/* Check AC status for whether we could boot in this state */
+	if (apple_smc_read_u32(power->smc, SMC_KEY(ACSt), &val) >= 0) {
+		if (!(val & ACSt_CAN_BOOT_IBOOT))
+			return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL;
+
+		if (!(val & ACSt_CAN_BOOT_AP))
+			return POWER_SUPPLY_CAPACITY_LEVEL_LOW;
+	}
+
+	/* BSFC = Battery System Full Charge */
+	ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag);
+	if (ret < 0)
+		return POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN;
+
+	if (flag)
+		return POWER_SUPPLY_CAPACITY_LEVEL_FULL;
+	else
+		return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL;
+}
+
+static int macsmc_battery_get_property(struct power_supply *psy,
+				       enum power_supply_property psp,
+				       union power_supply_propval *val)
+{
+	struct macsmc_power *power = power_supply_get_drvdata(psy);
+	int ret = 0;
+	u8 vu8;
+	u16 vu16;
+	s16 vs16;
+	s32 vs32;
+	s64 vs64;
+	bool flag;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_STATUS:
+		val->intval = macsmc_battery_get_status(power);
+		ret = val->intval < 0 ? val->intval : 0;
+		break;
+	case POWER_SUPPLY_PROP_PRESENT:
+		val->intval = 1;
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR:
+		val->intval = macsmc_battery_get_charge_behaviour(power);
+		ret = val->intval < 0 ? val->intval : 0;
+		break;
+	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16);
+		val->intval = vu16 == 0xffff ? 0 : vu16 * 60;
+		break;
+	case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16);
+		val->intval = vu16 == 0xffff ? 0 : vu16 * 60;
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY:
+		ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8);
+		val->intval = vu8;
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
+		val->intval = macsmc_battery_get_capacity_level(power);
+		ret = val->intval < 0 ? val->intval : 0;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = apple_smc_read_s16(power->smc, SMC_KEY(B0AC), &vs16);
+		val->intval = vs16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_POWER_NOW:
+		ret = apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &vs32);
+		val->intval = vs32 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(BITV), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN:
+		/* Calculate total max design voltage from per-cell maximum voltage */
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(BVVN), &vu16);
+		val->intval = vu16 * 1000 * power->num_cells;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_MIN:
+		/* Lifetime min */
+		ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPM), &vs16);
+		val->intval = vs16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_MAX:
+		/* Lifetime max */
+		ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPX), &vs16);
+		val->intval = vs16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RC), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RI), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RV), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_FULL:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16);
+		/* B0RM is Big Endian, likely pass through from TI gas gauge */
+		val->intval = (s16)swab16(vu16) * 1000;
+		break;
+	case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16);
+		val->intval = vu16 * power->nominal_voltage_mv;
+		break;
+	case POWER_SUPPLY_PROP_ENERGY_FULL:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16);
+		val->intval = vu16 * power->nominal_voltage_mv;
+		break;
+	case POWER_SUPPLY_PROP_ENERGY_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16);
+		/* B0RM is Big Endian, likely pass through from TI gas gauge */
+		val->intval = (s16)swab16(vu16) * power->nominal_voltage_mv;
+		break;
+	case POWER_SUPPLY_PROP_TEMP:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16);
+		val->intval = vu16 - 2732; /* Kelvin x10 to Celsius x10 */
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_COUNTER:
+		ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64);
+		val->intval = vs64;
+		break;
+	case POWER_SUPPLY_PROP_CYCLE_COUNT:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16);
+		val->intval = vu16;
+		break;
+	case POWER_SUPPLY_PROP_SCOPE:
+		val->intval = POWER_SUPPLY_SCOPE_SYSTEM;
+		break;
+	case POWER_SUPPLY_PROP_HEALTH:
+		flag = false;
+		ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD), &flag);
+		val->intval = flag ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD;
+		break;
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		val->strval = power->model_name;
+		break;
+	case POWER_SUPPLY_PROP_SERIAL_NUMBER:
+		val->strval = power->serial_number;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURE_YEAR:
+		ret = macsmc_battery_get_date(&power->mfg_date[0], &val->intval);
+		/* The SMC reports the manufacture year as an offset from 1992. */
+		val->intval += 1992;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURE_MONTH:
+		ret = macsmc_battery_get_date(&power->mfg_date[2], &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURE_DAY:
+		ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int macsmc_battery_set_property(struct power_supply *psy,
+				       enum power_supply_property psp,
+				       const union power_supply_propval *val)
+{
+	struct macsmc_power *power = power_supply_get_drvdata(psy);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR:
+		return macsmc_battery_set_charge_behaviour(power, val->intval);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int macsmc_battery_property_is_writeable(struct power_supply *psy,
+						enum power_supply_property psp)
+{
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct power_supply_desc macsmc_battery_desc_template = {
+	.name			= "macsmc-battery",
+	.type			= POWER_SUPPLY_TYPE_BATTERY,
+	.get_property		= macsmc_battery_get_property,
+	.set_property		= macsmc_battery_set_property,
+	.property_is_writeable	= macsmc_battery_property_is_writeable,
+};
+
+static int macsmc_ac_get_property(struct power_supply *psy,
+				  enum power_supply_property psp,
+				  union power_supply_propval *val)
+{
+	struct macsmc_power *power = power_supply_get_drvdata(psy);
+	int ret = 0;
+	u16 vu16;
+	u32 vu32;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_ONLINE:
+		ret = apple_smc_read_u32(power->smc, SMC_KEY(CHIS), &vu32);
+		val->intval = !!vu32;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &vu16);
+		val->intval = vu16 * 1000;
+		break;
+	case POWER_SUPPLY_PROP_INPUT_POWER_LIMIT:
+		ret = apple_smc_read_u32(power->smc, SMC_KEY(ACPW), &vu32);
+		val->intval = vu32 * 1000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct power_supply_desc macsmc_ac_desc_template = {
+	.name			= "macsmc-ac",
+	.type			= POWER_SUPPLY_TYPE_MAINS,
+	.get_property		= macsmc_ac_get_property,
+};
+
+static void macsmc_power_critical_work(struct work_struct *wrk)
+{
+	struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work);
+	u16 bitv, b0av;
+	u32 bcf0;
+
+	if (!power->batt)
+		return;
+
+	/*
+	 * Avoid duplicate atempts at emergency shutdown
+	 */
+	if (power->emergency_shutdown_triggered || system_state > SYSTEM_RUNNING)
+		return;
+
+	/*
+	 * EMERGENCY: Check voltage vs design minimum.
+	 * If we are below BITV, the battery is physically exhausted.
+	 * We must shut down NOW to protect the filesystem.
+	 */
+	if (apple_smc_read_u16(power->smc, SMC_KEY(BITV), &bitv) >= 0 &&
+	    apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &b0av) >= 0 &&
+	    b0av < bitv) {
+		power->emergency_shutdown_triggered = true;
+		dev_emerg(power->dev,
+			  "Battery voltage (%d mV) below design minimum (%d mV)! Emergency shutdown.\n",
+			  b0av, bitv);
+
+		/*
+		 * Shutdown is now imminent. Kick userspace again and give it some
+		 * brief time to (hopefully) flush what's needed, before forcing.
+		 */
+		hw_protection_trigger("Battery voltage below design minimum", 1500);
+	}
+
+	/*
+	 * Avoid duplicate attempts at orderly shutdown.
+	 * Voltage check is above this as we may want to
+	 * "upgrade" an orderly shutdown to a critical power
+	 * off if voltage drops.
+	 */
+	if (power->orderly_shutdown_triggered || system_state > SYSTEM_RUNNING)
+		return;
+
+	/*
+	 * Check if SMC flagged the battery as empty.
+	 * We trigger a graceful shutdown to let the OS save data.
+	 */
+	if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &bcf0) == 0 && bcf0 != 0) {
+		power->orderly_shutdown_triggered = true;
+		dev_crit(power->dev, "Battery critical (empty flag set). Triggering orderly shutdown.\n");
+		orderly_poweroff(true);
+	}
+}
+
+static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data)
+{
+	struct macsmc_power *power = container_of(nb, struct macsmc_power, nb);
+
+	/*
+	 * SMC Event IDs are correlated to physical events (e.g. charger
+	 * connect/disconnect) but the exact meaning of each ID is predicted.
+	 * 0x71... indicates power/battery events.
+	 */
+	if ((event & 0xffffff00) == 0x71010100 || /* Charger status change */
+	    (event & 0xffff0000) == 0x71060000 || /* Port charge state change */
+	    (event & 0xffff0000) == 0x71130000) { /* Connector insert/remove event */
+		if (power->batt)
+			power_supply_changed(power->batt);
+		if (power->ac)
+			power_supply_changed(power->ac);
+		return NOTIFY_OK;
+	} else if (event == 0x71020000) {
+		/* Critical battery warning */
+		if (power->batt)
+			schedule_work(&power->critical_work);
+		return NOTIFY_OK;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int macsmc_power_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent);
+	struct power_supply_config psy_cfg = {};
+	struct macsmc_power *power;
+	bool has_battery = false;
+	bool has_ac_adapter = false;
+	int ret = -ENODEV;
+	bool flag;
+	u16 vu16;
+	u32 val32;
+	enum power_supply_property *props;
+	size_t nprops;
+
+	if (!smc)
+		return -ENODEV;
+
+	power = devm_kzalloc(dev, sizeof(*power), GFP_KERNEL);
+	if (!power)
+		return -ENOMEM;
+
+	power->dev = dev;
+	power->smc = smc;
+	dev_set_drvdata(dev, power);
+
+	INIT_WORK(&power->critical_work, macsmc_power_critical_work);
+	ret = devm_work_autocancel(dev, &power->critical_work, macsmc_power_critical_work);
+	if (ret)
+		return ret;
+
+	/*
+	 * Check for battery presence.
+	 * B0AV is a fundamental key.
+	 */
+	if (apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16) == 0 &&
+	    macsmc_battery_get_status(power) > POWER_SUPPLY_STATUS_UNKNOWN)
+		has_battery = true;
+
+	/*
+	 * Check for AC adapter presence.
+	 * CHIS is a fundamental key.
+	 */
+	if (apple_smc_key_exists(smc, SMC_KEY(CHIS)))
+		has_ac_adapter = true;
+
+	if (!has_battery && !has_ac_adapter)
+		return -ENODEV;
+
+	if (has_battery) {
+		power->batt_desc = macsmc_battery_desc_template;
+		props = devm_kcalloc(dev, MACSMC_MAX_BATT_PROPS,
+				     sizeof(enum power_supply_property),
+				     GFP_KERNEL);
+		if (!props)
+			return -ENOMEM;
+
+		nprops = 0;
+
+		/* Fundamental properties */
+		props[nprops++] = POWER_SUPPLY_PROP_STATUS;
+		props[nprops++] = POWER_SUPPLY_PROP_PRESENT;
+		props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_CURRENT_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_POWER_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_CAPACITY;
+		props[nprops++] = POWER_SUPPLY_PROP_CAPACITY_LEVEL;
+		props[nprops++] = POWER_SUPPLY_PROP_TEMP;
+		props[nprops++] = POWER_SUPPLY_PROP_CYCLE_COUNT;
+		props[nprops++] = POWER_SUPPLY_PROP_HEALTH;
+		props[nprops++] = POWER_SUPPLY_PROP_SCOPE;
+		props[nprops++] = POWER_SUPPLY_PROP_MODEL_NAME;
+		props[nprops++] = POWER_SUPPLY_PROP_SERIAL_NUMBER;
+		props[nprops++] = POWER_SUPPLY_PROP_MANUFACTURE_YEAR;
+		props[nprops++] = POWER_SUPPLY_PROP_MANUFACTURE_MONTH;
+		props[nprops++] = POWER_SUPPLY_PROP_MANUFACTURE_DAY;
+
+		/* Extended properties usually present */
+		props[nprops++] = POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_TIME_TO_FULL_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN;
+		props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN;
+		props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MIN;
+		props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_MAX;
+		props[nprops++] = POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT;
+		props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX;
+		props[nprops++] = POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE;
+		props[nprops++] = POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN;
+		props[nprops++] = POWER_SUPPLY_PROP_CHARGE_FULL;
+		props[nprops++] = POWER_SUPPLY_PROP_CHARGE_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN;
+		props[nprops++] = POWER_SUPPLY_PROP_ENERGY_FULL;
+		props[nprops++] = POWER_SUPPLY_PROP_ENERGY_NOW;
+		props[nprops++] = POWER_SUPPLY_PROP_CHARGE_COUNTER;
+
+		/* Detect features based on key availability */
+		if (apple_smc_key_exists(smc, SMC_KEY(CHTE)))
+			power->has_chte = true;
+		if (apple_smc_key_exists(smc, SMC_KEY(CH0C)))
+			power->has_ch0c = true;
+		if (apple_smc_key_exists(smc, SMC_KEY(CH0I)))
+			power->has_ch0i = true;
+
+		/* Reset "Optimised Battery Charging" flags to default state */
+		if (power->has_chte)
+			apple_smc_write_u32(smc, SMC_KEY(CHTE), 0);
+		else if (power->has_ch0c)
+			apple_smc_write_u8(smc, SMC_KEY(CH0C), 0);
+
+		if (power->has_ch0i)
+			apple_smc_write_u8(smc, SMC_KEY(CH0I), 0);
+
+		apple_smc_write_u8(smc, SMC_KEY(CH0K), 0);
+		apple_smc_write_u8(smc, SMC_KEY(CH0B), 0);
+
+		/* Configure charge behaviour if supported */
+		if (power->has_ch0i || power->has_ch0c || power->has_chte) {
+			props[nprops++] = POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR;
+
+			power->batt_desc.charge_behaviours =
+				BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO);
+
+			if (power->has_ch0i)
+				power->batt_desc.charge_behaviours |=
+					BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE);
+
+			if (power->has_chte || power->has_ch0c)
+				power->batt_desc.charge_behaviours |=
+					BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE);
+		}
+
+		/* Detect charge limit method (CHWA vs CHLS) */
+		if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0)
+			power->has_chwa = true;
+		else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0)
+			power->has_chls = true;
+
+		if (nprops > MACSMC_MAX_BATT_PROPS)
+			return -ENOMEM;
+
+		power->batt_desc.properties = props;
+		power->batt_desc.num_properties = nprops;
+
+		/* Fetch identity strings */
+		apple_smc_read(smc, SMC_KEY(BMDN), power->model_name,
+			       sizeof(power->model_name) - 1);
+		apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number,
+			       sizeof(power->serial_number) - 1);
+		apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date,
+			       sizeof(power->mfg_date) - 1);
+
+		apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells);
+		power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells;
+
+		/* Enable critical shutdown notifications by reading status once */
+		apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val32);
+
+		psy_cfg.drv_data = power;
+		power->batt = devm_power_supply_register(dev, &power->batt_desc, &psy_cfg);
+		if (IS_ERR(power->batt)) {
+			dev_err_probe(dev, PTR_ERR(power->batt),
+				      "Failed to register battery\n");
+			/* Don't return failure yet; try AC registration first */
+			power->batt = NULL;
+		}
+	}
+
+	if (has_ac_adapter) {
+		power->ac_desc = macsmc_ac_desc_template;
+		props = devm_kcalloc(dev, MACSMC_MAX_AC_PROPS,
+				     sizeof(enum power_supply_property),
+				     GFP_KERNEL);
+		if (!props)
+			return -ENOMEM;
+
+		nprops = 0;
+
+		/* Online status is fundamental */
+		props[nprops++] = POWER_SUPPLY_PROP_ONLINE;
+
+		/* Input power limits are usually available */
+		if (apple_smc_key_exists(power->smc, SMC_KEY(ACPW)))
+			props[nprops++] = POWER_SUPPLY_PROP_INPUT_POWER_LIMIT;
+
+		/* macOS 15.4+ firmware dropped legacy AC keys (AC-n, AC-i) */
+		if (apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16) >= 0) {
+			props[nprops++] = POWER_SUPPLY_PROP_VOLTAGE_NOW;
+			props[nprops++] = POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT;
+		}
+
+		if (nprops > MACSMC_MAX_AC_PROPS)
+			return -ENOMEM;
+
+		power->ac_desc.properties = props;
+		power->ac_desc.num_properties = nprops;
+
+		psy_cfg.drv_data = power;
+		power->ac = devm_power_supply_register(dev, &power->ac_desc, &psy_cfg);
+		if (IS_ERR(power->ac)) {
+			dev_err_probe(dev, PTR_ERR(power->ac),
+				      "Failed to register AC adapter\n");
+			power->ac = NULL;
+		}
+	}
+
+	/* Final check: did we register anything? */
+	if (!power->batt && !power->ac)
+		return -ENODEV;
+
+	power->nb.notifier_call = macsmc_power_event;
+	blocking_notifier_chain_register(&smc->event_handlers, &power->nb);
+
+	return 0;
+}
+
+static void macsmc_power_remove(struct platform_device *pdev)
+{
+	struct macsmc_power *power = dev_get_drvdata(&pdev->dev);
+
+	blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb);
+}
+
+static const struct platform_device_id macsmc_power_id[] = {
+	{ "macsmc-power" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, macsmc_power_id);
+
+static struct platform_driver macsmc_power_driver = {
+	.driver = {
+		.name = "macsmc-power",
+	},
+	.id_table = macsmc_power_id,
+	.probe = macsmc_power_probe,
+	.remove = macsmc_power_remove,
+};
+module_platform_driver(macsmc_power_driver);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("Apple SMC battery and power management driver");
+MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
+MODULE_AUTHOR("Michael Reeves <michael.reeves077@gmail.com>");
diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c
index acea176101fa8..167fb3fb37320 100644
--- a/drivers/power/supply/max17042_battery.c
+++ b/drivers/power/supply/max17042_battery.c
@@ -61,6 +61,7 @@ struct max17042_chip {
 	struct work_struct work;
 	int    init_complete;
 	int    irq;
+	int    task_period;
 };
 
 static enum power_supply_property max17042_battery_props[] = {
@@ -88,6 +89,7 @@ static enum power_supply_property max17042_battery_props[] = {
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_SCOPE,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
+	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	// these two have to be at the end on the list
 	POWER_SUPPLY_PROP_CURRENT_NOW,
 	POWER_SUPPLY_PROP_CURRENT_AVG,
@@ -131,7 +133,7 @@ static int max17042_get_status(struct max17042_chip *chip, int *status)
 	 * FullCAP to match RepCap when it detects end of charging.
 	 *
 	 * When this cycle the battery gets charged to a higher (calculated)
-	 * capacity then the previous cycle then FullCAP will get updated
+	 * capacity than the previous cycle then FullCAP will get updated
 	 * continuously once end-of-charge detection kicks in, so allow the
 	 * 2 to differ a bit.
 	 */
@@ -201,7 +203,7 @@ static int max17042_get_battery_health(struct max17042_chip *chip, int *health)
 		goto out;
 	}
 
-	if (vbatt > chip->pdata->vmax + MAX17042_VMAX_TOLERANCE) {
+	if (vbatt > size_add(chip->pdata->vmax, MAX17042_VMAX_TOLERANCE)) {
 		*health = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
 		goto out;
 	}
@@ -331,6 +333,8 @@ static int max17042_get_property(struct power_supply *psy,
 			return ret;
 
 		data64 = data * 5000000ll;
+		data64 *= chip->task_period;
+		do_div(data64, MAX17042_DEFAULT_TASK_PERIOD);
 		do_div(data64, chip->pdata->r_sns);
 		val->intval = data64;
 		break;
@@ -340,6 +344,8 @@ static int max17042_get_property(struct power_supply *psy,
 			return ret;
 
 		data64 = data * 5000000ll;
+		data64 *= chip->task_period;
+		do_div(data64, MAX17042_DEFAULT_TASK_PERIOD);
 		do_div(data64, chip->pdata->r_sns);
 		val->intval = data64;
 		break;
@@ -349,6 +355,8 @@ static int max17042_get_property(struct power_supply *psy,
 			return ret;
 
 		data64 = data * 5000000ll;
+		data64 *= chip->task_period;
+		do_div(data64, MAX17042_DEFAULT_TASK_PERIOD);
 		do_div(data64, chip->pdata->r_sns);
 		val->intval = data64;
 		break;
@@ -358,6 +366,8 @@ static int max17042_get_property(struct power_supply *psy,
 			return ret;
 
 		data64 = sign_extend64(data, 15) * 5000000ll;
+		data64 *= chip->task_period;
+		data64 = div_s64(data64, MAX17042_DEFAULT_TASK_PERIOD);
 		val->intval = div_s64(data64, chip->pdata->r_sns);
 		break;
 	case POWER_SUPPLY_PROP_TEMP:
@@ -430,6 +440,25 @@ static int max17042_get_property(struct power_supply *psy,
 		if (ret < 0)
 			return ret;
 
+		/* when charging, the value is not meaningful */
+		if (data == U16_MAX)
+			return -ENODATA;
+
+		val->intval = data * 5625 / 1000;
+		break;
+	case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW:
+		if (chip->chip_type != MAXIM_DEVICE_TYPE_MAX17055 &&
+		    chip->chip_type != MAXIM_DEVICE_TYPE_MAX77759)
+			return -EINVAL;
+
+		ret = regmap_read(map, MAX17055_TTF, &data);
+		if (ret < 0)
+			return ret;
+
+		/* when discharging, the value is not meaningful */
+		if (data == U16_MAX)
+			return -ENODATA;
+
 		val->intval = data * 5625 / 1000;
 		break;
 	default:
@@ -646,7 +675,8 @@ static void max17042_write_config_regs(struct max17042_chip *chip)
 	regmap_write(map, MAX17042_RelaxCFG, config->relax_cfg);
 	if (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17047 ||
 			chip->chip_type == MAXIM_DEVICE_TYPE_MAX17050 ||
-			chip->chip_type == MAXIM_DEVICE_TYPE_MAX17055)
+			chip->chip_type == MAXIM_DEVICE_TYPE_MAX17055 ||
+			chip->chip_type == MAXIM_DEVICE_TYPE_MAX77759)
 		regmap_write(map, MAX17047_FullSOCThr,
 						config->full_soc_thresh);
 }
@@ -783,7 +813,8 @@ static inline void max17042_override_por_values(struct max17042_chip *chip)
 
 	if ((chip->chip_type == MAXIM_DEVICE_TYPE_MAX17042) ||
 	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17047) ||
-	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17050)) {
+	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17050) ||
+	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX77759)) {
 		max17042_override_por(map, MAX17042_IAvg_empty, config->iavg_empty);
 		max17042_override_por(map, MAX17042_TempNom, config->temp_nom);
 		max17042_override_por(map, MAX17042_TempLim, config->temp_lim);
@@ -792,7 +823,8 @@ static inline void max17042_override_por_values(struct max17042_chip *chip)
 
 	if ((chip->chip_type == MAXIM_DEVICE_TYPE_MAX17047) ||
 	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17050) ||
-	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17055)) {
+	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX17055) ||
+	    (chip->chip_type == MAXIM_DEVICE_TYPE_MAX77759)) {
 		max17042_override_por(map, MAX17047_V_empty, config->vempty);
 	}
 }
@@ -921,8 +953,12 @@ max17042_get_of_pdata(struct max17042_chip *chip)
 	/*
 	 * Require current sense resistor value to be specified for
 	 * current-sense functionality to be enabled at all.
+	 * maxim,rsns-microohm is the property name used by older DTs and kept
+	 * for compatibility.
 	 */
-	if (of_property_read_u32(np, "maxim,rsns-microohm", &prop) == 0) {
+	if ((of_property_read_u32(np, "shunt-resistor-micro-ohms",
+				  &prop) == 0) ||
+	    (of_property_read_u32(np, "maxim,rsns-microohm", &prop) == 0)) {
 		pdata->r_sns = prop;
 		pdata->enable_current_sense = true;
 	}
@@ -1011,6 +1047,45 @@ static const struct regmap_config max17042_regmap_config = {
 	.val_format_endian = REGMAP_ENDIAN_NATIVE,
 };
 
+static const struct regmap_range max77759_fg_registers[] = {
+	regmap_reg_range(MAX17042_STATUS, MAX77759_MixAtFull),
+	regmap_reg_range(MAX17042_VFSOC0Enable, MAX17042_VFSOC0Enable),
+	regmap_reg_range(MAX17042_MLOCKReg1, MAX17042_MLOCKReg2),
+	regmap_reg_range(MAX17042_MODELChrTbl, MAX17055_TimerH),
+	regmap_reg_range(MAX77759_IIn, MAX77759_IIn),
+	regmap_reg_range(MAX17055_AtQResidual, MAX17055_AtAvCap),
+	regmap_reg_range(MAX17042_OCVInternal, MAX17042_OCVInternal),
+	regmap_reg_range(MAX17042_VFSOC, MAX17042_VFSOC),
+};
+
+static const struct regmap_range max77759_fg_ro_registers[] = {
+	regmap_reg_range(MAX17042_FSTAT, MAX17042_FSTAT),
+	regmap_reg_range(MAX17042_OCVInternal, MAX17042_OCVInternal),
+	regmap_reg_range(MAX17042_VFSOC, MAX17042_VFSOC),
+};
+
+static const struct regmap_access_table max77759_fg_write_table = {
+	.yes_ranges = max77759_fg_registers,
+	.n_yes_ranges = ARRAY_SIZE(max77759_fg_registers),
+	.no_ranges = max77759_fg_ro_registers,
+	.n_no_ranges = ARRAY_SIZE(max77759_fg_ro_registers),
+};
+
+static const struct regmap_access_table max77759_fg_rd_table = {
+	.yes_ranges = max77759_fg_registers,
+	.n_yes_ranges = ARRAY_SIZE(max77759_fg_registers),
+};
+
+static const struct regmap_config max77759_fg_regmap_cfg = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.max_register = 0xff,
+	.wr_table = &max77759_fg_write_table,
+	.rd_table = &max77759_fg_rd_table,
+	.val_format_endian = REGMAP_ENDIAN_NATIVE,
+	.cache_type = REGCACHE_NONE,
+};
+
 static const struct power_supply_desc max17042_psy_desc = {
 	.name		= "max170xx_battery",
 	.type		= POWER_SUPPLY_TYPE_BATTERY,
@@ -1037,6 +1112,7 @@ static int max17042_probe(struct i2c_client *client, struct device *dev, int irq
 {
 	struct i2c_adapter *adapter = client->adapter;
 	const struct power_supply_desc *max17042_desc = &max17042_psy_desc;
+	const struct regmap_config *regmap_config;
 	struct power_supply_config psy_cfg = {};
 	struct max17042_chip *chip;
 	int ret;
@@ -1052,17 +1128,20 @@ static int max17042_probe(struct i2c_client *client, struct device *dev, int irq
 
 	chip->dev = dev;
 	chip->chip_type = chip_type;
-	chip->regmap = devm_regmap_init_i2c(client, &max17042_regmap_config);
-	if (IS_ERR(chip->regmap)) {
-		dev_err(dev, "Failed to initialize regmap\n");
-		return -EINVAL;
-	}
+
+	if (chip->chip_type == MAXIM_DEVICE_TYPE_MAX77759)
+		regmap_config = &max77759_fg_regmap_cfg;
+	else
+		regmap_config = &max17042_regmap_config;
+	chip->regmap = devm_regmap_init_i2c(client, regmap_config);
+	if (IS_ERR(chip->regmap))
+		return dev_err_probe(dev, PTR_ERR(chip->regmap),
+				     "Failed to initialize regmap\n");
 
 	chip->pdata = max17042_get_pdata(chip);
-	if (!chip->pdata) {
-		dev_err(dev, "no platform data provided\n");
-		return -EINVAL;
-	}
+	if (!chip->pdata)
+		return dev_err_probe(dev, -EINVAL,
+				     "no platform data provided\n");
 
 	dev_set_drvdata(dev, chip);
 	psy_cfg.drv_data = chip;
@@ -1088,12 +1167,22 @@ static int max17042_probe(struct i2c_client *client, struct device *dev, int irq
 		regmap_write(chip->regmap, MAX17042_LearnCFG, 0x0007);
 	}
 
+	chip->task_period = MAX17042_DEFAULT_TASK_PERIOD;
+	if (chip->chip_type == MAXIM_DEVICE_TYPE_MAX77759) {
+		ret = regmap_read(chip->regmap, MAX17042_TaskPeriod, &val);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "failed to read task period\n");
+		chip->task_period = val;
+	}
+	dev_dbg(dev, "task period: %#.4x (%d)\n", chip->task_period,
+		chip->task_period);
+
 	chip->battery = devm_power_supply_register(dev, max17042_desc,
 						   &psy_cfg);
-	if (IS_ERR(chip->battery)) {
-		dev_err(dev, "failed: power supply register\n");
-		return PTR_ERR(chip->battery);
-	}
+	if (IS_ERR(chip->battery))
+		return dev_err_probe(dev, PTR_ERR(chip->battery),
+				     "failed: power supply register\n");
 
 	if (irq) {
 		unsigned int flags = IRQF_ONESHOT | IRQF_SHARED | IRQF_PROBE_SHARED;
@@ -1236,6 +1325,8 @@ static const struct of_device_id max17042_dt_match[] __used = {
 		.data = (void *) MAXIM_DEVICE_TYPE_MAX17055 },
 	{ .compatible = "maxim,max77705-battery",
 		.data = (void *) MAXIM_DEVICE_TYPE_MAX17047 },
+	{ .compatible = "maxim,max77759-fg",
+		.data = (void *) MAXIM_DEVICE_TYPE_MAX77759 },
 	{ .compatible = "maxim,max77849-battery",
 		.data = (void *) MAXIM_DEVICE_TYPE_MAX17047 },
 	{ },
@@ -1248,6 +1339,7 @@ static const struct i2c_device_id max17042_id[] = {
 	{ "max17047", MAXIM_DEVICE_TYPE_MAX17047 },
 	{ "max17050", MAXIM_DEVICE_TYPE_MAX17050 },
 	{ "max17055", MAXIM_DEVICE_TYPE_MAX17055 },
+	{ "max77759-fg", MAXIM_DEVICE_TYPE_MAX77759 },
 	{ "max77849-battery", MAXIM_DEVICE_TYPE_MAX17047 },
 	{ }
 };
diff --git a/drivers/power/supply/max77705_charger.c b/drivers/power/supply/max77705_charger.c
index 5dd02f658f5bd..63b0b4f0cd217 100644
--- a/drivers/power/supply/max77705_charger.c
+++ b/drivers/power/supply/max77705_charger.c
@@ -646,51 +646,37 @@ static int max77705_charger_probe(struct i2c_client *i2c)
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to add irq chip\n");
 
-	chg->wqueue = create_singlethread_workqueue(dev_name(dev));
+	chg->wqueue = devm_alloc_ordered_workqueue(dev, "%s", 0, dev_name(dev));
 	if (!chg->wqueue)
 		return -ENOMEM;
 
 	ret = devm_work_autocancel(dev, &chg->chgin_work, max77705_chgin_isr_work);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to initialize interrupt work\n");
-		goto destroy_wq;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to initialize interrupt work\n");
 
 	ret = max77705_charger_initialize(chg);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to initialize charger IC\n");
-		goto destroy_wq;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to initialize charger IC\n");
 
 	ret = devm_request_threaded_irq(dev, regmap_irq_get_virq(irq_data, MAX77705_CHGIN_I),
 					NULL, max77705_chgin_irq,
 					IRQF_TRIGGER_NONE,
 					"chgin-irq", chg);
-	if (ret) {
-		dev_err_probe(dev, ret, "Failed to Request chgin IRQ\n");
-		goto destroy_wq;
-	}
+	if (ret)
+		return ret;
 
 	ret = devm_request_threaded_irq(dev, regmap_irq_get_virq(irq_data, MAX77705_AICL_I),
 					NULL, max77705_aicl_irq,
 					IRQF_TRIGGER_NONE,
 					"aicl-irq", chg);
-	if (ret) {
-		dev_err_probe(dev, ret, "Failed to Request aicl IRQ\n");
-		goto destroy_wq;
-	}
+	if (ret)
+		return ret;
 
 	ret = max77705_charger_enable(chg);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to enable charge\n");
-		goto destroy_wq;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable charge\n");
 
 	return devm_add_action_or_reset(dev, max77705_charger_disable, chg);
-
-destroy_wq:
-	destroy_workqueue(chg->wqueue);
-	return ret;
 }
 
 static const struct of_device_id max77705_charger_of_match[] = {
diff --git a/drivers/power/supply/mt6370-charger.c b/drivers/power/supply/mt6370-charger.c
index e6db961d5818d..916556baa854a 100644
--- a/drivers/power/supply/mt6370-charger.c
+++ b/drivers/power/supply/mt6370-charger.c
@@ -761,13 +761,6 @@ static int mt6370_chg_init_psy(struct mt6370_priv *priv)
 	return PTR_ERR_OR_ZERO(priv->psy);
 }
 
-static void mt6370_chg_destroy_wq(void *data)
-{
-	struct workqueue_struct *wq = data;
-
-	destroy_workqueue(wq);
-}
-
 static irqreturn_t mt6370_attach_i_handler(int irq, void *data)
 {
 	struct mt6370_priv *priv = data;
@@ -893,14 +886,10 @@ static int mt6370_chg_probe(struct platform_device *pdev)
 
 	priv->attach = MT6370_ATTACH_STAT_DETACH;
 
-	priv->wq = create_singlethread_workqueue(dev_name(priv->dev));
+	priv->wq = devm_alloc_ordered_workqueue(dev, "%s", 0, dev_name(priv->dev));
 	if (!priv->wq)
 		return -ENOMEM;
 
-	ret = devm_add_action_or_reset(dev, mt6370_chg_destroy_wq, priv->wq);
-	if (ret)
-		return ret;
-
 	ret = devm_work_autocancel(dev, &priv->bc12_work, mt6370_chg_bc12_work_func);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to init bc12 work\n");
diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index dd3a48d72d2b1..f30a7b9ccd5e9 100644
--- a/drivers/power/supply/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
@@ -70,6 +70,8 @@ static const char * const POWER_SUPPLY_USB_TYPE_TEXT[] = {
 	[POWER_SUPPLY_USB_TYPE_PD]		= "PD",
 	[POWER_SUPPLY_USB_TYPE_PD_DRP]		= "PD_DRP",
 	[POWER_SUPPLY_USB_TYPE_PD_PPS]		= "PD_PPS",
+	[POWER_SUPPLY_USB_TYPE_PD_SPR_AVS]	= "PD_SPR_AVS",
+	[POWER_SUPPLY_USB_TYPE_PD_PPS_SPR_AVS]	= "PD_PPS_SPR_AVS",
 	[POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID]	= "BrickID",
 };
 
diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c
index 80572ee945b4f..490137a23d00e 100644
--- a/drivers/power/supply/qcom_battmgr.c
+++ b/drivers/power/supply/qcom_battmgr.c
@@ -1611,6 +1611,8 @@ static void qcom_battmgr_pdr_notify(void *priv, int state)
 }
 
 static const struct of_device_id qcom_battmgr_of_variants[] = {
+	{ .compatible = "qcom,glymur-pmic-glink", .data = (void *)QCOM_BATTMGR_X1E80100 },
+	{ .compatible = "qcom,kaanapali-pmic-glink", .data = (void *)QCOM_BATTMGR_SM8550 },
 	{ .compatible = "qcom,sc8180x-pmic-glink", .data = (void *)QCOM_BATTMGR_SC8280XP },
 	{ .compatible = "qcom,sc8280xp-pmic-glink", .data = (void *)QCOM_BATTMGR_SC8280XP },
 	{ .compatible = "qcom,sm8550-pmic-glink", .data = (void *)QCOM_BATTMGR_SM8550 },
diff --git a/drivers/power/supply/s2mu005-battery.c b/drivers/power/supply/s2mu005-battery.c
new file mode 100644
index 0000000000000..64c57a14ef7d9
--- /dev/null
+++ b/drivers/power/supply/s2mu005-battery.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Battery Fuel Gauge Driver for Samsung S2MU005 PMIC.
+ *
+ * Copyright (C) 2015 Samsung Electronics
+ * Copyright (C) 2023 Yassine Oudjana <y.oudjana@protonmail.com>
+ * Copyright (C) 2025 Kaustabh Chakraborty <kauschluss@disroot.org>
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/mod_devicetable.h>
+#include <linux/mutex.h>
+#include <linux/power_supply.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/units.h>
+
+#define S2MU005_FG_REG_STATUS		0x00
+#define S2MU005_FG_REG_IRQ		0x02
+#define S2MU005_FG_REG_RVBAT		0x04
+#define S2MU005_FG_REG_RCURCC		0x06
+#define S2MU005_FG_REG_RSOC		0x08
+#define S2MU005_FG_REG_MONOUT		0x0a
+#define S2MU005_FG_REG_MONOUTSEL	0x0c
+#define S2MU005_FG_REG_RBATCAP		0x0e
+#define S2MU005_FG_REG_RZADJ		0x12
+#define S2MU005_FG_REG_RBATZ0		0x16
+#define S2MU005_FG_REG_RBATZ1		0x18
+#define S2MU005_FG_REG_IRQLVL		0x1a
+#define S2MU005_FG_REG_START		0x1e
+
+#define S2MU005_FG_MONOUTSEL_AVGCURRENT		0x26
+#define S2MU005_FG_MONOUTSEL_AVGVOLTAGE		0x27
+
+struct s2mu005_fg {
+	struct device *dev;
+	struct regmap *regmap;
+	struct power_supply *psy;
+	struct mutex monout_mutex;
+};
+
+static const struct regmap_config s2mu005_fg_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
+static irqreturn_t s2mu005_handle_irq(int irq, void *data)
+{
+	struct s2mu005_fg *priv = data;
+
+	msleep(100);
+	power_supply_changed(priv->psy);
+
+	return IRQ_HANDLED;
+}
+
+static int s2mu005_fg_get_voltage_now(struct s2mu005_fg *priv, int *value)
+{
+	struct regmap *regmap = priv->regmap;
+	u32 val;
+	int ret;
+
+	ret = regmap_read(regmap, S2MU005_FG_REG_RVBAT, &val);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to read voltage register (%d)\n", ret);
+		return ret;
+	}
+
+	*value = (val * MICRO) >> 13;
+
+	return 0;
+}
+
+static int s2mu005_fg_get_voltage_avg(struct s2mu005_fg *priv, int *value)
+{
+	struct regmap *regmap = priv->regmap;
+	u32 val;
+	int ret;
+
+	mutex_lock(&priv->monout_mutex);
+
+	ret = regmap_write(regmap, S2MU005_FG_REG_MONOUTSEL,
+			   S2MU005_FG_MONOUTSEL_AVGVOLTAGE);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to enable average voltage monitoring (%d)\n",
+			ret);
+		goto unlock;
+	}
+
+	ret = regmap_read(regmap, S2MU005_FG_REG_MONOUT, &val);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to read current register (%d)\n", ret);
+		goto unlock;
+	}
+
+	*value = (val * MICRO) >> 12;
+
+unlock:
+	mutex_unlock(&priv->monout_mutex);
+
+	return ret;
+}
+static int s2mu005_fg_get_current_now(struct s2mu005_fg *priv, int *value)
+{
+	struct regmap *regmap = priv->regmap;
+	u32 val;
+	int ret;
+
+	ret = regmap_read(regmap, S2MU005_FG_REG_RCURCC, &val);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to read current register (%d)\n", ret);
+		return ret;
+	}
+
+	*value = -((s16)val * MICRO) >> 12;
+
+	return 0;
+}
+
+static int s2mu005_fg_get_current_avg(struct s2mu005_fg *priv, int *value)
+{
+	struct regmap *regmap = priv->regmap;
+	u32 val;
+	int ret;
+
+	mutex_lock(&priv->monout_mutex);
+
+	ret = regmap_write(regmap, S2MU005_FG_REG_MONOUTSEL,
+			   S2MU005_FG_MONOUTSEL_AVGCURRENT);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to enable average current monitoring (%d)\n",
+			ret);
+		goto unlock;
+	}
+
+	ret = regmap_read(regmap, S2MU005_FG_REG_MONOUT, &val);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to read current register (%d)\n", ret);
+		goto unlock;
+	}
+
+	*value = -((s16)val * MICRO) >> 12;
+
+unlock:
+	mutex_unlock(&priv->monout_mutex);
+
+	return ret;
+}
+
+static int s2mu005_fg_get_capacity(struct s2mu005_fg *priv, int *value)
+{
+	struct regmap *regmap = priv->regmap;
+	u32 val;
+	int ret;
+
+	ret = regmap_read(regmap, S2MU005_FG_REG_RSOC, &val);
+	if (ret < 0) {
+		dev_err(priv->dev, "failed to read capacity register (%d)\n", ret);
+		return ret;
+	}
+
+	*value = (val * CENTI) >> 14;
+
+	return 0;
+}
+
+static int s2mu005_fg_get_status(struct s2mu005_fg *priv, int *value)
+{
+	int current_now, current_avg, capacity;
+	int ret;
+
+	ret = s2mu005_fg_get_current_now(priv, &current_now);
+	if (ret < 0)
+		return ret;
+
+	ret = s2mu005_fg_get_current_avg(priv, &current_avg);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Verify both current values reported to reduce inaccuracies due to
+	 * internal hysteresis.
+	 */
+	if (current_now < 0 && current_avg < 0) {
+		*value = POWER_SUPPLY_STATUS_DISCHARGING;
+	} else if (current_now == 0) {
+		*value = POWER_SUPPLY_STATUS_NOT_CHARGING;
+	} else {
+		*value = POWER_SUPPLY_STATUS_CHARGING;
+
+		ret = s2mu005_fg_get_capacity(priv, &capacity);
+		if (!ret && capacity > 98)
+			*value = POWER_SUPPLY_STATUS_FULL;
+		return ret;
+	}
+
+	return 0;
+}
+
+static const enum power_supply_property s2mu005_fg_properties[] = {
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_VOLTAGE_AVG,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CURRENT_AVG,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_STATUS,
+};
+
+static int s2mu005_fg_get_property(struct power_supply *psy,
+				   enum power_supply_property psp,
+				   union power_supply_propval *val)
+{
+	struct s2mu005_fg *priv = power_supply_get_drvdata(psy);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		return s2mu005_fg_get_voltage_now(priv, &val->intval);
+	case POWER_SUPPLY_PROP_VOLTAGE_AVG:
+		return s2mu005_fg_get_voltage_avg(priv, &val->intval);
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		return s2mu005_fg_get_current_now(priv, &val->intval);
+	case POWER_SUPPLY_PROP_CURRENT_AVG:
+		return s2mu005_fg_get_current_avg(priv, &val->intval);
+	case POWER_SUPPLY_PROP_CAPACITY:
+		return s2mu005_fg_get_capacity(priv, &val->intval);
+	case POWER_SUPPLY_PROP_STATUS:
+		return s2mu005_fg_get_status(priv, &val->intval);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct power_supply_desc s2mu005_fg_desc = {
+	.name = "s2mu005-fuel-gauge",
+	.type = POWER_SUPPLY_TYPE_BATTERY,
+	.properties = s2mu005_fg_properties,
+	.num_properties = ARRAY_SIZE(s2mu005_fg_properties),
+	.get_property = s2mu005_fg_get_property,
+};
+
+static int s2mu005_fg_i2c_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct s2mu005_fg *priv;
+	struct power_supply_config psy_cfg = {};
+	const struct power_supply_desc *psy_desc;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, priv);
+	priv->dev = dev;
+
+	priv->regmap = devm_regmap_init_i2c(client, &s2mu005_fg_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return dev_err_probe(dev, PTR_ERR(priv->regmap),
+				     "failed to initialize regmap\n");
+
+	ret = devm_mutex_init(dev, &priv->monout_mutex);
+	if (ret)
+		dev_err_probe(dev, ret, "failed to initialize MONOUT mutex\n");
+
+	psy_desc = device_get_match_data(dev);
+
+	psy_cfg.drv_data = priv;
+	psy_cfg.fwnode = dev_fwnode(dev);
+	priv->psy = devm_power_supply_register(priv->dev, psy_desc, &psy_cfg);
+	if (IS_ERR(priv->psy))
+		return dev_err_probe(dev, PTR_ERR(priv->psy),
+				     "failed to register power supply subsystem\n");
+
+	ret = devm_request_threaded_irq(priv->dev, client->irq, NULL,
+					s2mu005_handle_irq, IRQF_ONESHOT,
+					psy_desc->name, priv);
+	if (ret)
+		dev_err_probe(dev, ret, "failed to request IRQ\n");
+
+	return 0;
+}
+
+static const struct of_device_id s2mu005_fg_of_match_table[] = {
+	{
+		.compatible = "samsung,s2mu005-fuel-gauge",
+		.data = &s2mu005_fg_desc,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, s2mu005_fg_of_match_table);
+
+static struct i2c_driver s2mu005_fg_i2c_driver = {
+	.probe = s2mu005_fg_i2c_probe,
+	.driver = {
+		.name = "s2mu005-fuel-gauge",
+		.of_match_table = s2mu005_fg_of_match_table,
+	},
+};
+module_i2c_driver(s2mu005_fg_i2c_driver);
+
+MODULE_DESCRIPTION("Samsung S2MU005 PMIC Battery Fuel Gauge Driver");
+MODULE_AUTHOR("Yassine Oudjana <y.oudjana@protonmail.com>");
+MODULE_AUTHOR("Kaustabh Chakraborty <kauschluss@disroot.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/sbs-manager.c b/drivers/power/supply/sbs-manager.c
index 6fe526222f7f4..343ad4ab4082c 100644
--- a/drivers/power/supply/sbs-manager.c
+++ b/drivers/power/supply/sbs-manager.c
@@ -199,7 +199,7 @@ static int sbsm_gpio_get_value(struct gpio_chip *gc, unsigned int off)
 	if (ret < 0)
 		return ret;
 
-	return ret & BIT(off);
+	return !!(ret & BIT(off));
 }
 
 /*
diff --git a/drivers/power/supply/twl4030_madc_battery.c b/drivers/power/supply/twl4030_madc_battery.c
index 3935162e350b5..a99b3ff26929d 100644
--- a/drivers/power/supply/twl4030_madc_battery.c
+++ b/drivers/power/supply/twl4030_madc_battery.c
@@ -11,9 +11,7 @@
  */
 
 #include <linux/module.h>
-#include <linux/param.h>
 #include <linux/delay.h>
-#include <linux/workqueue.h>
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
 #include <linux/slab.h>
diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c
index 019a65a5283ad..2b5c587b222ba 100644
--- a/drivers/powercap/intel_rapl_common.c
+++ b/drivers/powercap/intel_rapl_common.c
@@ -24,77 +24,91 @@
 #include <linux/suspend.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
+#include <linux/units.h>
 
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
-#include <asm/iosf_mbi.h>
 #include <asm/msr.h>
 
 /* bitmasks for RAPL MSRs, used by primitive access functions */
-#define ENERGY_STATUS_MASK      0xffffffff
+#define ENERGY_STATUS_MASK		GENMASK(31, 0)
 
-#define POWER_LIMIT1_MASK       0x7FFF
-#define POWER_LIMIT1_ENABLE     BIT(15)
-#define POWER_LIMIT1_CLAMP      BIT(16)
+#define POWER_LIMIT1_MASK		GENMASK(14, 0)
+#define POWER_LIMIT1_ENABLE		BIT(15)
+#define POWER_LIMIT1_CLAMP		BIT(16)
 
-#define POWER_LIMIT2_MASK       (0x7FFFULL<<32)
-#define POWER_LIMIT2_ENABLE     BIT_ULL(47)
-#define POWER_LIMIT2_CLAMP      BIT_ULL(48)
-#define POWER_HIGH_LOCK         BIT_ULL(63)
-#define POWER_LOW_LOCK          BIT(31)
+#define POWER_LIMIT2_MASK		GENMASK_ULL(46, 32)
+#define POWER_LIMIT2_ENABLE		BIT_ULL(47)
+#define POWER_LIMIT2_CLAMP		BIT_ULL(48)
+#define POWER_HIGH_LOCK			BIT_ULL(63)
+#define POWER_LOW_LOCK			BIT(31)
 
-#define POWER_LIMIT4_MASK		0x1FFF
+#define POWER_LIMIT4_MASK		GENMASK(12, 0)
 
-#define TIME_WINDOW1_MASK       (0x7FULL<<17)
-#define TIME_WINDOW2_MASK       (0x7FULL<<49)
+#define TIME_WINDOW1_MASK		GENMASK_ULL(23, 17)
+#define TIME_WINDOW2_MASK		GENMASK_ULL(55, 49)
 
-#define POWER_UNIT_OFFSET	0
-#define POWER_UNIT_MASK		0x0F
+#define POWER_UNIT_OFFSET		0x00
+#define POWER_UNIT_MASK			GENMASK(3, 0)
 
-#define ENERGY_UNIT_OFFSET	0x08
-#define ENERGY_UNIT_MASK	0x1F00
+#define ENERGY_UNIT_OFFSET		0x08
+#define ENERGY_UNIT_MASK		GENMASK(12, 8)
 
-#define TIME_UNIT_OFFSET	0x10
-#define TIME_UNIT_MASK		0xF0000
+#define TIME_UNIT_OFFSET		0x10
+#define TIME_UNIT_MASK			GENMASK(19, 16)
 
-#define POWER_INFO_MAX_MASK     (0x7fffULL<<32)
-#define POWER_INFO_MIN_MASK     (0x7fffULL<<16)
-#define POWER_INFO_MAX_TIME_WIN_MASK     (0x3fULL<<48)
-#define POWER_INFO_THERMAL_SPEC_MASK     0x7fff
+#define POWER_INFO_MAX_MASK		GENMASK_ULL(46, 32)
+#define POWER_INFO_MIN_MASK		GENMASK_ULL(30, 16)
+#define POWER_INFO_MAX_TIME_WIN_MASK	GENMASK_ULL(53, 48)
+#define POWER_INFO_THERMAL_SPEC_MASK	GENMASK(14, 0)
 
-#define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
-#define PP_POLICY_MASK         0x1F
+#define PERF_STATUS_THROTTLE_TIME_MASK	GENMASK(31, 0)
+#define PP_POLICY_MASK			GENMASK(4, 0)
 
 /*
  * SPR has different layout for Psys Domain PowerLimit registers.
  * There are 17 bits of PL1 and PL2 instead of 15 bits.
  * The Enable bits and TimeWindow bits are also shifted as a result.
  */
-#define PSYS_POWER_LIMIT1_MASK       0x1FFFF
-#define PSYS_POWER_LIMIT1_ENABLE     BIT(17)
+#define PSYS_POWER_LIMIT1_MASK		GENMASK_ULL(16, 0)
+#define PSYS_POWER_LIMIT1_ENABLE	BIT(17)
 
-#define PSYS_POWER_LIMIT2_MASK       (0x1FFFFULL<<32)
-#define PSYS_POWER_LIMIT2_ENABLE     BIT_ULL(49)
+#define PSYS_POWER_LIMIT2_MASK		GENMASK_ULL(48, 32)
+#define PSYS_POWER_LIMIT2_ENABLE	BIT_ULL(49)
 
-#define PSYS_TIME_WINDOW1_MASK       (0x7FULL<<19)
-#define PSYS_TIME_WINDOW2_MASK       (0x7FULL<<51)
+#define PSYS_TIME_WINDOW1_MASK		GENMASK_ULL(25, 19)
+#define PSYS_TIME_WINDOW2_MASK		GENMASK_ULL(57, 51)
 
 /* bitmasks for RAPL TPMI, used by primitive access functions */
-#define TPMI_POWER_LIMIT_MASK	0x3FFFF
-#define TPMI_POWER_LIMIT_ENABLE	BIT_ULL(62)
-#define TPMI_TIME_WINDOW_MASK	(0x7FULL<<18)
-#define TPMI_INFO_SPEC_MASK	0x3FFFF
-#define TPMI_INFO_MIN_MASK	(0x3FFFFULL << 18)
-#define TPMI_INFO_MAX_MASK	(0x3FFFFULL << 36)
-#define TPMI_INFO_MAX_TIME_WIN_MASK	(0x7FULL << 54)
+#define TPMI_POWER_LIMIT_MASK		GENMASK_ULL(17, 0)
+#define TPMI_POWER_LIMIT_ENABLE		BIT_ULL(62)
+#define TPMI_TIME_WINDOW_MASK		GENMASK_ULL(24, 18)
+#define TPMI_INFO_SPEC_MASK		GENMASK_ULL(17, 0)
+#define TPMI_INFO_MIN_MASK		GENMASK_ULL(35, 18)
+#define TPMI_INFO_MAX_MASK		GENMASK_ULL(53, 36)
+#define TPMI_INFO_MAX_TIME_WIN_MASK	GENMASK_ULL(60, 54)
 
 /* Non HW constants */
-#define RAPL_PRIMITIVE_DERIVED       BIT(1)	/* not from raw data */
-#define RAPL_PRIMITIVE_DUMMY         BIT(2)
+#define RAPL_PRIMITIVE_DERIVED		BIT(1)	/* not from raw data */
+#define RAPL_PRIMITIVE_DUMMY		BIT(2)
+
+#define ENERGY_UNIT_SCALE		1000	/* scale from driver unit to powercap unit */
+
+/* per domain data, some are optional */
+#define NR_RAW_PRIMITIVES		(NR_RAPL_PRIMITIVES - 2)
+
+#define	DOMAIN_STATE_INACTIVE		BIT(0)
+#define	DOMAIN_STATE_POWER_LIMIT_SET	BIT(1)
+
+/* Sideband MBI registers */
+#define IOSF_CPU_POWER_BUDGET_CTL_BYT	0x02
+#define IOSF_CPU_POWER_BUDGET_CTL_TNG	0xDF
+
+#define PACKAGE_PLN_INT_SAVED		BIT(0)
+#define MAX_PRIM_NAME			32
+
+#define RAPL_EVENT_MASK			GENMASK(7, 0)
 
-#define TIME_WINDOW_MAX_MSEC 40000
-#define TIME_WINDOW_MIN_MSEC 250
-#define ENERGY_UNIT_SCALE    1000	/* scale from driver unit to powercap unit */
 enum unit_type {
 	ARBITRARY_UNIT,		/* no translation */
 	POWER_UNIT,
@@ -102,12 +116,6 @@ enum unit_type {
 	TIME_UNIT,
 };
 
-/* per domain data, some are optional */
-#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
-
-#define	DOMAIN_STATE_INACTIVE           BIT(0)
-#define	DOMAIN_STATE_POWER_LIMIT_SET    BIT(1)
-
 static const char *pl_names[NR_POWER_LIMITS] = {
 	[POWER_LIMIT1] = "long_term",
 	[POWER_LIMIT2] = "short_term",
@@ -204,31 +212,11 @@ static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim)
 #define power_zone_to_rapl_domain(_zone) \
 	container_of(_zone, struct rapl_domain, power_zone)
 
-struct rapl_defaults {
-	u8 floor_freq_reg_addr;
-	int (*check_unit)(struct rapl_domain *rd);
-	void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
-	u64 (*compute_time_window)(struct rapl_domain *rd, u64 val,
-				    bool to_raw);
-	unsigned int dram_domain_energy_unit;
-	unsigned int psys_domain_energy_unit;
-	bool spr_psys_bits;
-};
-static struct rapl_defaults *defaults_msr;
-static const struct rapl_defaults defaults_tpmi;
-
-static struct rapl_defaults *get_defaults(struct rapl_package *rp)
+static const struct rapl_defaults *get_defaults(struct rapl_package *rp)
 {
 	return rp->priv->defaults;
 }
 
-/* Sideband MBI registers */
-#define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)
-#define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)
-
-#define PACKAGE_PLN_INT_SAVED   BIT(0)
-#define MAX_PRIM_NAME (32)
-
 /* per domain data. used to describe individual knobs such that access function
  * can be consolidated into one instead of many inline functions.
  */
@@ -341,7 +329,7 @@ static int find_nr_power_limit(struct rapl_domain *rd)
 static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
 {
 	struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
-	struct rapl_defaults *defaults = get_defaults(rd->rp);
+	const struct rapl_defaults *defaults = get_defaults(rd->rp);
 	u64 val;
 	int ret;
 
@@ -630,7 +618,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
 			   u64 value, int to_raw)
 {
 	u64 units = 1;
-	struct rapl_defaults *defaults = get_defaults(rd->rp);
+	const struct rapl_defaults *defaults = get_defaults(rd->rp);
 	u64 scale = 1;
 
 	switch (type) {
@@ -659,99 +647,104 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
 /* RAPL primitives for MSR and MMIO I/F */
 static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = {
 	/* name, mask, shift, msr index, unit divisor */
-	[POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
-			    RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
-	[POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
-			    RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
-	[POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0,
-				RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
-	[ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
-			    RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
-	[FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[FW_HIGH_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_HIGH_LOCK, 63,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL1_CLAMP] = PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL2_CLAMP] = PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
-			    RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
-	[TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
-			    RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
-	[THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
-			    0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
-			    RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
-			    RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
-			    RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
-	[THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
-			    RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
-	[PRIORITY_LEVEL] = PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
-			    RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
-	[PSYS_POWER_LIMIT1] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0,
-			    RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
-	[PSYS_POWER_LIMIT2] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32,
-			    RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
-	[PSYS_PL1_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PSYS_PL2_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49,
-			    RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PSYS_TIME_WINDOW1] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19,
-			    RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
-	[PSYS_TIME_WINDOW2] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51,
-			    RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+	[POWER_LIMIT1]		= PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
+						      RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+	[POWER_LIMIT2]		= PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
+						      RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+	[POWER_LIMIT4]		= PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0,
+						      RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
+	[ENERGY_COUNTER]	= PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
+						      RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
+	[FW_LOCK]		= PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[FW_HIGH_LOCK]		= PRIMITIVE_INFO_INIT(FW_LOCK, POWER_HIGH_LOCK, 63,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL1_ENABLE]		= PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL1_CLAMP]		= PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL2_ENABLE]		= PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL2_CLAMP]		= PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[TIME_WINDOW1]		= PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
+						      RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+	[TIME_WINDOW2]		= PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
+						      RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+	[THERMAL_SPEC_POWER]	= PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER,
+						      POWER_INFO_THERMAL_SPEC_MASK, 0,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MAX_POWER]		= PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MIN_POWER]		= PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MAX_TIME_WINDOW]	= PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW,
+						      POWER_INFO_MAX_TIME_WIN_MASK, 48,
+						      RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
+	[THROTTLED_TIME]	= PRIMITIVE_INFO_INIT(THROTTLED_TIME,
+						      PERF_STATUS_THROTTLE_TIME_MASK, 0,
+						      RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
+	[PRIORITY_LEVEL]	= PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
+						      RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
+	[PSYS_POWER_LIMIT1]	= PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0,
+						      RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+	[PSYS_POWER_LIMIT2]	= PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK,
+						      32, RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+	[PSYS_PL1_ENABLE]	= PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE,
+						      17, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT,
+						      0),
+	[PSYS_PL2_ENABLE]	= PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE,
+						      49, RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT,
+						      0),
+	[PSYS_TIME_WINDOW1]	= PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK,
+						      19, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+	[PSYS_TIME_WINDOW2]	= PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK,
+						      51, RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
 	/* non-hardware */
-	[AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
-			    RAPL_PRIMITIVE_DERIVED),
+	[AVERAGE_POWER]		= PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
+						      RAPL_PRIMITIVE_DERIVED),
 };
 
 /* RAPL primitives for TPMI I/F */
 static struct rapl_primitive_info rpi_tpmi[NR_RAPL_PRIMITIVES] = {
 	/* name, mask, shift, msr index, unit divisor */
-	[POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0,
-		RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
-	[POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0,
-		RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0),
-	[POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0,
-		RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
-	[ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
-		RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
-	[PL1_LOCK] = PRIMITIVE_INFO_INIT(PL1_LOCK, POWER_HIGH_LOCK, 63,
-		RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL2_LOCK] = PRIMITIVE_INFO_INIT(PL2_LOCK, POWER_HIGH_LOCK, 63,
-		RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
-	[PL4_LOCK] = PRIMITIVE_INFO_INIT(PL4_LOCK, POWER_HIGH_LOCK, 63,
-		RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
-	[PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
-		RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
-	[PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
-		RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
-	[PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
-		RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
-	[TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18,
-		RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
-	[TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18,
-		RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0),
-	[THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0,
-		RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36,
-		RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18,
-		RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
-	[MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK, 54,
-		RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
-	[THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
-		RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
+	[POWER_LIMIT1]		= PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0,
+						      RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+	[POWER_LIMIT2]		= PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0,
+						      RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0),
+	[POWER_LIMIT4]		= PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0,
+						      RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
+	[ENERGY_COUNTER]	= PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
+						      RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
+	[PL1_LOCK]		= PRIMITIVE_INFO_INIT(PL1_LOCK, POWER_HIGH_LOCK, 63,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL2_LOCK]		= PRIMITIVE_INFO_INIT(PL2_LOCK, POWER_HIGH_LOCK, 63,
+						      RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
+	[PL4_LOCK]		= PRIMITIVE_INFO_INIT(PL4_LOCK, POWER_HIGH_LOCK, 63,
+						      RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
+	[PL1_ENABLE]		= PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
+						      RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+	[PL2_ENABLE]		= PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
+						      RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
+	[PL4_ENABLE]		= PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
+						      RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
+	[TIME_WINDOW1]		= PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18,
+						      RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+	[TIME_WINDOW2]		= PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18,
+						      RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0),
+	[THERMAL_SPEC_POWER]	= PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MAX_POWER]		= PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MIN_POWER]		= PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18,
+						      RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+	[MAX_TIME_WINDOW]	= PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK,
+						      54, RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
+	[THROTTLED_TIME]	= PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK,
+						      0, RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
 	/* non-hardware */
-	[AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0,
-		POWER_UNIT, RAPL_PRIMITIVE_DERIVED),
+	[AVERAGE_POWER]		= PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
+						      RAPL_PRIMITIVE_DERIVED),
 };
 
 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)
@@ -770,11 +763,9 @@ static int rapl_config(struct rapl_package *rp)
 	/* MMIO I/F shares the same register layout as MSR registers */
 	case RAPL_IF_MMIO:
 	case RAPL_IF_MSR:
-		rp->priv->defaults = (void *)defaults_msr;
 		rp->priv->rpi = (void *)rpi_msr;
 		break;
 	case RAPL_IF_TPMI:
-		rp->priv->defaults = (void *)&defaults_tpmi;
 		rp->priv->rpi = (void *)rpi_tpmi;
 		break;
 	default:
@@ -791,7 +782,7 @@ static int rapl_config(struct rapl_package *rp)
 static enum rapl_primitives
 prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim)
 {
-	struct rapl_defaults *defaults = get_defaults(rd->rp);
+	const struct rapl_defaults *defaults = get_defaults(rd->rp);
 
 	if (!defaults->spr_psys_bits)
 		return prim;
@@ -936,7 +927,7 @@ static int rapl_write_pl_data(struct rapl_domain *rd, int pl,
  * power unit : microWatts  : Represented in milliWatts by default
  * time unit  : microseconds: Represented in seconds by default
  */
-static int rapl_check_unit_core(struct rapl_domain *rd)
+int rapl_default_check_unit(struct rapl_domain *rd)
 {
 	struct reg_action ra;
 	u32 value;
@@ -950,47 +941,20 @@ static int rapl_check_unit_core(struct rapl_domain *rd)
 	}
 
 	value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
-	rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
+	rd->energy_unit = (ENERGY_UNIT_SCALE * MICROJOULE_PER_JOULE) >> value;
 
 	value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
-	rd->power_unit = 1000000 / (1 << value);
+	rd->power_unit = MICROWATT_PER_WATT >> value;
 
 	value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
-	rd->time_unit = 1000000 / (1 << value);
+	rd->time_unit = USEC_PER_SEC >> value;
 
 	pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
 
 	return 0;
 }
-
-static int rapl_check_unit_atom(struct rapl_domain *rd)
-{
-	struct reg_action ra;
-	u32 value;
-
-	ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
-	ra.mask = ~0;
-	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) {
-		pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
-			ra.reg.val, rd->rp->name, rd->name);
-		return -ENODEV;
-	}
-
-	value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
-	rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
-
-	value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
-	rd->power_unit = (1 << value) * 1000;
-
-	value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
-	rd->time_unit = 1000000 / (1 << value);
-
-	pr_debug("Atom %s:%s energy=%dpJ, time=%dus, power=%duW\n",
-		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
-
-	return 0;
-}
+EXPORT_SYMBOL_NS_GPL(rapl_default_check_unit, "INTEL_RAPL");
 
 static void power_limit_irq_save_cpu(void *info)
 {
@@ -1056,7 +1020,7 @@ static void package_power_limit_irq_restore(struct rapl_package *rp)
 	wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 }
 
-static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
+void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode)
 {
 	int i;
 
@@ -1070,33 +1034,9 @@ static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
 		rapl_write_pl_data(rd, i, PL_CLAMP, mode);
 	}
 }
+EXPORT_SYMBOL_NS_GPL(rapl_default_set_floor_freq, "INTEL_RAPL");
 
-static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
-{
-	static u32 power_ctrl_orig_val;
-	struct rapl_defaults *defaults = get_defaults(rd->rp);
-	u32 mdata;
-
-	if (!defaults->floor_freq_reg_addr) {
-		pr_err("Invalid floor frequency config register\n");
-		return;
-	}
-
-	if (!power_ctrl_orig_val)
-		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
-			      defaults->floor_freq_reg_addr,
-			      &power_ctrl_orig_val);
-	mdata = power_ctrl_orig_val;
-	if (enable) {
-		mdata &= ~(0x7f << 8);
-		mdata |= 1 << 8;
-	}
-	iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
-		       defaults->floor_freq_reg_addr, mdata);
-}
-
-static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value,
-					 bool to_raw)
+u64 rapl_default_compute_time_window(struct rapl_domain *rd, u64 value, bool to_raw)
 {
 	u64 f, y;		/* fraction and exp. used for time unit */
 
@@ -1107,7 +1047,7 @@ static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value,
 	if (!to_raw) {
 		f = (value & 0x60) >> 5;
 		y = value & 0x1f;
-		value = (1 << y) * (4 + f) * rd->time_unit / 4;
+		value = (1ULL << y) * (4 + f) * rd->time_unit / 4;
 	} else {
 		if (value < rd->time_unit)
 			return 0;
@@ -1122,199 +1062,12 @@ static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value,
 		if (y > 0x1f)
 			return 0x7f;
 
-		f = div64_u64(4 * (value - (1ULL << y)), 1ULL << y);
+		f = div64_u64(4 * (value - BIT_ULL(y)), BIT_ULL(y));
 		value = (y & 0x1f) | ((f & 0x3) << 5);
 	}
 	return value;
 }
-
-static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value,
-					 bool to_raw)
-{
-	/*
-	 * Atom time unit encoding is straight forward val * time_unit,
-	 * where time_unit is default to 1 sec. Never 0.
-	 */
-	if (!to_raw)
-		return (value) ? value * rd->time_unit : rd->time_unit;
-
-	value = div64_u64(value, rd->time_unit);
-
-	return value;
-}
-
-/* TPMI Unit register has different layout */
-#define TPMI_POWER_UNIT_OFFSET	POWER_UNIT_OFFSET
-#define TPMI_POWER_UNIT_MASK	POWER_UNIT_MASK
-#define TPMI_ENERGY_UNIT_OFFSET	0x06
-#define TPMI_ENERGY_UNIT_MASK	0x7C0
-#define TPMI_TIME_UNIT_OFFSET	0x0C
-#define TPMI_TIME_UNIT_MASK	0xF000
-
-static int rapl_check_unit_tpmi(struct rapl_domain *rd)
-{
-	struct reg_action ra;
-	u32 value;
-
-	ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
-	ra.mask = ~0;
-	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) {
-		pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
-			ra.reg.val, rd->rp->name, rd->name);
-		return -ENODEV;
-	}
-
-	value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET;
-	rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
-
-	value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET;
-	rd->power_unit = 1000000 / (1 << value);
-
-	value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET;
-	rd->time_unit = 1000000 / (1 << value);
-
-	pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
-		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
-
-	return 0;
-}
-
-static const struct rapl_defaults defaults_tpmi = {
-	.check_unit = rapl_check_unit_tpmi,
-	/* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */
-	.set_floor_freq = set_floor_freq_default,
-	.compute_time_window = rapl_compute_time_window_core,
-};
-
-static const struct rapl_defaults rapl_defaults_core = {
-	.floor_freq_reg_addr = 0,
-	.check_unit = rapl_check_unit_core,
-	.set_floor_freq = set_floor_freq_default,
-	.compute_time_window = rapl_compute_time_window_core,
-};
-
-static const struct rapl_defaults rapl_defaults_hsw_server = {
-	.check_unit = rapl_check_unit_core,
-	.set_floor_freq = set_floor_freq_default,
-	.compute_time_window = rapl_compute_time_window_core,
-	.dram_domain_energy_unit = 15300,
-};
-
-static const struct rapl_defaults rapl_defaults_spr_server = {
-	.check_unit = rapl_check_unit_core,
-	.set_floor_freq = set_floor_freq_default,
-	.compute_time_window = rapl_compute_time_window_core,
-	.psys_domain_energy_unit = 1000000000,
-	.spr_psys_bits = true,
-};
-
-static const struct rapl_defaults rapl_defaults_byt = {
-	.floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
-	.check_unit = rapl_check_unit_atom,
-	.set_floor_freq = set_floor_freq_atom,
-	.compute_time_window = rapl_compute_time_window_atom,
-};
-
-static const struct rapl_defaults rapl_defaults_tng = {
-	.floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
-	.check_unit = rapl_check_unit_atom,
-	.set_floor_freq = set_floor_freq_atom,
-	.compute_time_window = rapl_compute_time_window_atom,
-};
-
-static const struct rapl_defaults rapl_defaults_ann = {
-	.floor_freq_reg_addr = 0,
-	.check_unit = rapl_check_unit_atom,
-	.set_floor_freq = NULL,
-	.compute_time_window = rapl_compute_time_window_atom,
-};
-
-static const struct rapl_defaults rapl_defaults_cht = {
-	.floor_freq_reg_addr = 0,
-	.check_unit = rapl_check_unit_atom,
-	.set_floor_freq = NULL,
-	.compute_time_window = rapl_compute_time_window_atom,
-};
-
-static const struct rapl_defaults rapl_defaults_amd = {
-	.check_unit = rapl_check_unit_core,
-};
-
-static const struct x86_cpu_id rapl_ids[] __initconst = {
-	X86_MATCH_VFM(INTEL_SANDYBRIDGE,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,	&rapl_defaults_core),
-
-	X86_MATCH_VFM(INTEL_IVYBRIDGE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,	&rapl_defaults_core),
-
-	X86_MATCH_VFM(INTEL_HASWELL,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_HASWELL_L,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_HASWELL_G,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_HASWELL_X,		&rapl_defaults_hsw_server),
-
-	X86_MATCH_VFM(INTEL_BROADWELL,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_BROADWELL_G,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_BROADWELL_D,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_BROADWELL_X,	&rapl_defaults_hsw_server),
-
-	X86_MATCH_VFM(INTEL_SKYLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_SKYLAKE_X,		&rapl_defaults_hsw_server),
-	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_KABYLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_CANNONLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ICELAKE_L,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ICELAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ICELAKE_NNPI,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ICELAKE_X,		&rapl_defaults_hsw_server),
-	X86_MATCH_VFM(INTEL_ICELAKE_D,		&rapl_defaults_hsw_server),
-	X86_MATCH_VFM(INTEL_COMETLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_COMETLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_TIGERLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ROCKETLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ALDERLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_RAPTORLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_RAPTORLAKE_P,        &rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_BARTLETTLAKE,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_METEORLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&rapl_defaults_spr_server),
-	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&rapl_defaults_spr_server),
-	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_NOVALAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_NOVALAKE_L,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ARROWLAKE,		&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_LAKEFIELD,		&rapl_defaults_core),
-
-	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT,	&rapl_defaults_byt),
-	X86_MATCH_VFM(INTEL_ATOM_AIRMONT,	&rapl_defaults_cht),
-	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &rapl_defaults_tng),
-	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID2,&rapl_defaults_ann),
-	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_TREMONT,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,	&rapl_defaults_core),
-	X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,	&rapl_defaults_core),
-
-	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,	&rapl_defaults_hsw_server),
-	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,	&rapl_defaults_hsw_server),
-
-	X86_MATCH_VENDOR_FAM(AMD, 0x17, &rapl_defaults_amd),
-	X86_MATCH_VENDOR_FAM(AMD, 0x19, &rapl_defaults_amd),
-	X86_MATCH_VENDOR_FAM(AMD, 0x1A, &rapl_defaults_amd),
-	X86_MATCH_VENDOR_FAM(HYGON, 0x18, &rapl_defaults_amd),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
+EXPORT_SYMBOL_NS_GPL(rapl_default_compute_time_window, "INTEL_RAPL");
 
 /* Read once for all raw primitive data for domains */
 static void rapl_update_domain_data(struct rapl_package *rp)
@@ -1443,7 +1196,7 @@ static int rapl_check_domain(int domain, struct rapl_package *rp)
  */
 static int rapl_get_domain_unit(struct rapl_domain *rd)
 {
-	struct rapl_defaults *defaults = get_defaults(rd->rp);
+	const struct rapl_defaults *defaults = get_defaults(rd->rp);
 	int ret;
 
 	if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) {
@@ -1777,7 +1530,6 @@ enum perf_rapl_events {
 	PERF_RAPL_PSYS,		/* psys */
 	PERF_RAPL_MAX
 };
-#define RAPL_EVENT_MASK GENMASK(7, 0)
 
 static const int event_to_domain[PERF_RAPL_MAX] = {
 	[PERF_RAPL_PP0]		= RAPL_DOMAIN_PP0,
@@ -2083,7 +1835,7 @@ int rapl_package_add_pmu_locked(struct rapl_package *rp)
 
 	return rapl_pmu_update(rp);
 }
-EXPORT_SYMBOL_GPL(rapl_package_add_pmu_locked);
+EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu_locked, "INTEL_RAPL");
 
 int rapl_package_add_pmu(struct rapl_package *rp)
 {
@@ -2091,7 +1843,7 @@ int rapl_package_add_pmu(struct rapl_package *rp)
 
 	return rapl_package_add_pmu_locked(rp);
 }
-EXPORT_SYMBOL_GPL(rapl_package_add_pmu);
+EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu, "INTEL_RAPL");
 
 void rapl_package_remove_pmu_locked(struct rapl_package *rp)
 {
@@ -2109,7 +1861,7 @@ void rapl_package_remove_pmu_locked(struct rapl_package *rp)
 	perf_pmu_unregister(&rapl_pmu.pmu);
 	memset(&rapl_pmu, 0, sizeof(struct rapl_pmu));
 }
-EXPORT_SYMBOL_GPL(rapl_package_remove_pmu_locked);
+EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu_locked, "INTEL_RAPL");
 
 void rapl_package_remove_pmu(struct rapl_package *rp)
 {
@@ -2117,9 +1869,67 @@ void rapl_package_remove_pmu(struct rapl_package *rp)
 
 	rapl_package_remove_pmu_locked(rp);
 }
-EXPORT_SYMBOL_GPL(rapl_package_remove_pmu);
+EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu, "INTEL_RAPL");
 #endif
 
+/* pm notifier for saving/restoring Power Limit settings */
+static void power_limit_state_save(void)
+{
+	struct rapl_package *rp;
+	struct rapl_domain *rd;
+	int ret;
+
+	cpus_read_lock();
+	list_for_each_entry(rp, &rapl_packages, plist) {
+		if (!rp->power_zone)
+			continue;
+		rd = power_zone_to_rapl_domain(rp->power_zone);
+		for (int i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
+			ret = rapl_read_pl_data(rd, i, PL_LIMIT, true,
+						&rd->rpl[i].last_power_limit);
+			if (ret)
+				rd->rpl[i].last_power_limit = 0;
+		}
+	}
+	cpus_read_unlock();
+}
+
+static void power_limit_state_restore(void)
+{
+	struct rapl_package *rp;
+	struct rapl_domain *rd;
+
+	cpus_read_lock();
+	list_for_each_entry(rp, &rapl_packages, plist) {
+		if (!rp->power_zone)
+			continue;
+		rd = power_zone_to_rapl_domain(rp->power_zone);
+		for (int i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++)
+			if (rd->rpl[i].last_power_limit)
+				rapl_write_pl_data(rd, i, PL_LIMIT,
+						   rd->rpl[i].last_power_limit);
+	}
+	cpus_read_unlock();
+}
+
+static int rapl_pm_callback(struct notifier_block *nb,
+			    unsigned long mode, void *_unused)
+{
+	switch (mode) {
+	case PM_SUSPEND_PREPARE:
+		power_limit_state_save();
+		break;
+	case PM_POST_SUSPEND:
+		power_limit_state_restore();
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block rapl_pm_notifier = {
+	.notifier_call = rapl_pm_callback,
+};
+
 /* called from CPU hotplug notifier, hotplug lock held */
 void rapl_remove_package_cpuslocked(struct rapl_package *rp)
 {
@@ -2149,15 +1959,18 @@ void rapl_remove_package_cpuslocked(struct rapl_package *rp)
 				 &rd_package->power_zone);
 	list_del(&rp->plist);
 	kfree(rp);
+
+	if (list_empty(&rapl_packages))
+		unregister_pm_notifier(&rapl_pm_notifier);
 }
-EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked);
+EXPORT_SYMBOL_NS_GPL(rapl_remove_package_cpuslocked, "INTEL_RAPL");
 
 void rapl_remove_package(struct rapl_package *rp)
 {
 	guard(cpus_read_lock)();
 	rapl_remove_package_cpuslocked(rp);
 }
-EXPORT_SYMBOL_GPL(rapl_remove_package);
+EXPORT_SYMBOL_NS_GPL(rapl_remove_package, "INTEL_RAPL");
 
 /*
  * RAPL Package energy counter scope:
@@ -2200,14 +2013,14 @@ struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_
 
 	return NULL;
 }
-EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked);
+EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain_cpuslocked, "INTEL_RAPL");
 
 struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu)
 {
 	guard(cpus_read_lock)();
 	return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu);
 }
-EXPORT_SYMBOL_GPL(rapl_find_package_domain);
+EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain, "INTEL_RAPL");
 
 /* called from CPU hotplug notifier, hotplug lock held */
 struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu)
@@ -2251,6 +2064,8 @@ struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *pr
 	}
 	ret = rapl_package_register_powercap(rp);
 	if (!ret) {
+		if (list_empty(&rapl_packages))
+			register_pm_notifier(&rapl_pm_notifier);
 		INIT_LIST_HEAD(&rp->plist);
 		list_add(&rp->plist, &rapl_packages);
 		return rp;
@@ -2261,112 +2076,14 @@ struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *pr
 	kfree(rp);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked);
+EXPORT_SYMBOL_NS_GPL(rapl_add_package_cpuslocked, "INTEL_RAPL");
 
 struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu)
 {
 	guard(cpus_read_lock)();
 	return rapl_add_package_cpuslocked(id, priv, id_is_cpu);
 }
-EXPORT_SYMBOL_GPL(rapl_add_package);
-
-static void power_limit_state_save(void)
-{
-	struct rapl_package *rp;
-	struct rapl_domain *rd;
-	int ret, i;
-
-	cpus_read_lock();
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		if (!rp->power_zone)
-			continue;
-		rd = power_zone_to_rapl_domain(rp->power_zone);
-		for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
-			ret = rapl_read_pl_data(rd, i, PL_LIMIT, true,
-						 &rd->rpl[i].last_power_limit);
-			if (ret)
-				rd->rpl[i].last_power_limit = 0;
-		}
-	}
-	cpus_read_unlock();
-}
-
-static void power_limit_state_restore(void)
-{
-	struct rapl_package *rp;
-	struct rapl_domain *rd;
-	int i;
-
-	cpus_read_lock();
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		if (!rp->power_zone)
-			continue;
-		rd = power_zone_to_rapl_domain(rp->power_zone);
-		for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++)
-			if (rd->rpl[i].last_power_limit)
-				rapl_write_pl_data(rd, i, PL_LIMIT,
-					       rd->rpl[i].last_power_limit);
-	}
-	cpus_read_unlock();
-}
-
-static int rapl_pm_callback(struct notifier_block *nb,
-			    unsigned long mode, void *_unused)
-{
-	switch (mode) {
-	case PM_SUSPEND_PREPARE:
-		power_limit_state_save();
-		break;
-	case PM_POST_SUSPEND:
-		power_limit_state_restore();
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block rapl_pm_notifier = {
-	.notifier_call = rapl_pm_callback,
-};
-
-static struct platform_device *rapl_msr_platdev;
-
-static int __init rapl_init(void)
-{
-	const struct x86_cpu_id *id;
-	int ret;
-
-	id = x86_match_cpu(rapl_ids);
-	if (id) {
-		defaults_msr = (struct rapl_defaults *)id->driver_data;
-
-		rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0);
-		if (!rapl_msr_platdev)
-			return -ENOMEM;
-
-		ret = platform_device_add(rapl_msr_platdev);
-		if (ret) {
-			platform_device_put(rapl_msr_platdev);
-			return ret;
-		}
-	}
-
-	ret = register_pm_notifier(&rapl_pm_notifier);
-	if (ret && rapl_msr_platdev) {
-		platform_device_del(rapl_msr_platdev);
-		platform_device_put(rapl_msr_platdev);
-	}
-
-	return ret;
-}
-
-static void __exit rapl_exit(void)
-{
-	platform_device_unregister(rapl_msr_platdev);
-	unregister_pm_notifier(&rapl_pm_notifier);
-}
-
-fs_initcall(rapl_init);
-module_exit(rapl_exit);
+EXPORT_SYMBOL_NS_GPL(rapl_add_package, "INTEL_RAPL");
 
 MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");
 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c
index 3d5e7f56d68a1..b7c10ed75d69c 100644
--- a/drivers/powercap/intel_rapl_msr.c
+++ b/drivers/powercap/intel_rapl_msr.c
@@ -21,15 +21,33 @@
 #include <linux/intel_rapl.h>
 #include <linux/processor.h>
 #include <linux/platform_device.h>
+#include <linux/units.h>
+#include <linux/bits.h>
 
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
+#include <asm/iosf_mbi.h>
 #include <asm/msr.h>
 
 /* Local defines */
 #define MSR_PLATFORM_POWER_LIMIT	0x0000065C
 #define MSR_VR_CURRENT_CONFIG		0x00000601
 
+#define ENERGY_UNIT_SCALE		1000	/* scale from driver unit to powercap unit */
+
+#define POWER_UNIT_OFFSET		0x00
+#define POWER_UNIT_MASK			GENMASK(3, 0)
+
+#define ENERGY_UNIT_OFFSET		0x08
+#define ENERGY_UNIT_MASK		GENMASK(12, 8)
+
+#define TIME_UNIT_OFFSET		0x10
+#define TIME_UNIT_MASK			GENMASK(19, 16)
+
+/* Sideband MBI registers */
+#define IOSF_CPU_POWER_BUDGET_CTL_BYT	0x02
+#define IOSF_CPU_POWER_BUDGET_CTL_TNG	0xDF
+
 /* private data for RAPL MSR Interface */
 static struct rapl_if_priv *rapl_msr_priv;
 
@@ -185,6 +203,201 @@ static const struct x86_cpu_id pmu_support_ids[] = {
 	{}
 };
 
+static int rapl_check_unit_atom(struct rapl_domain *rd)
+{
+	struct reg_action ra;
+	u32 value;
+
+	ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
+	ra.mask = ~0;
+	if (rapl_msr_read_raw(rd->rp->lead_cpu, &ra, false)) {
+		pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
+			ra.reg.val, rd->rp->name, rd->name);
+		return -ENODEV;
+	}
+
+	value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
+	rd->energy_unit = ENERGY_UNIT_SCALE * (1ULL << value);
+
+	value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
+	rd->power_unit = (1ULL << value) * MILLIWATT_PER_WATT;
+
+	value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
+	rd->time_unit = USEC_PER_SEC >> value;
+
+	pr_debug("Atom %s:%s energy=%dpJ, time=%dus, power=%duW\n",
+		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
+
+	return 0;
+}
+
+static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
+{
+	static u32 power_ctrl_orig_val;
+	const struct rapl_defaults *defaults = rd->rp->priv->defaults;
+	u32 mdata;
+
+	if (!defaults->floor_freq_reg_addr) {
+		pr_err("Invalid floor frequency config register\n");
+		return;
+	}
+
+	if (!power_ctrl_orig_val)
+		iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
+			      defaults->floor_freq_reg_addr,
+			      &power_ctrl_orig_val);
+	mdata = power_ctrl_orig_val;
+	if (enable) {
+		mdata &= ~GENMASK(14, 8);
+		mdata |= BIT(8);
+	}
+	iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
+		       defaults->floor_freq_reg_addr, mdata);
+}
+
+static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value,
+					 bool to_raw)
+{
+	if (to_raw)
+		return div64_u64(value, rd->time_unit);
+
+	/*
+	 * Atom time unit encoding is straight forward val * time_unit,
+	 * where time_unit is default to 1 sec. Never 0.
+	 */
+	return value ? value * rd->time_unit : rd->time_unit;
+}
+
+static const struct rapl_defaults rapl_defaults_core = {
+	.floor_freq_reg_addr = 0,
+	.check_unit = rapl_default_check_unit,
+	.set_floor_freq = rapl_default_set_floor_freq,
+	.compute_time_window = rapl_default_compute_time_window,
+};
+
+static const struct rapl_defaults rapl_defaults_hsw_server = {
+	.check_unit = rapl_default_check_unit,
+	.set_floor_freq = rapl_default_set_floor_freq,
+	.compute_time_window = rapl_default_compute_time_window,
+	.dram_domain_energy_unit = 15300,
+};
+
+static const struct rapl_defaults rapl_defaults_spr_server = {
+	.check_unit = rapl_default_check_unit,
+	.set_floor_freq = rapl_default_set_floor_freq,
+	.compute_time_window = rapl_default_compute_time_window,
+	.psys_domain_energy_unit = NANOJOULE_PER_JOULE,
+	.spr_psys_bits = true,
+};
+
+static const struct rapl_defaults rapl_defaults_byt = {
+	.floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
+	.check_unit = rapl_check_unit_atom,
+	.set_floor_freq = set_floor_freq_atom,
+	.compute_time_window = rapl_compute_time_window_atom,
+};
+
+static const struct rapl_defaults rapl_defaults_tng = {
+	.floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
+	.check_unit = rapl_check_unit_atom,
+	.set_floor_freq = set_floor_freq_atom,
+	.compute_time_window = rapl_compute_time_window_atom,
+};
+
+static const struct rapl_defaults rapl_defaults_ann = {
+	.floor_freq_reg_addr = 0,
+	.check_unit = rapl_check_unit_atom,
+	.set_floor_freq = NULL,
+	.compute_time_window = rapl_compute_time_window_atom,
+};
+
+static const struct rapl_defaults rapl_defaults_cht = {
+	.floor_freq_reg_addr = 0,
+	.check_unit = rapl_check_unit_atom,
+	.set_floor_freq = NULL,
+	.compute_time_window = rapl_compute_time_window_atom,
+};
+
+static const struct rapl_defaults rapl_defaults_amd = {
+	.check_unit = rapl_default_check_unit,
+};
+
+static const struct x86_cpu_id rapl_ids[]  = {
+	X86_MATCH_VFM(INTEL_SANDYBRIDGE,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,		&rapl_defaults_core),
+
+	X86_MATCH_VFM(INTEL_IVYBRIDGE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,		&rapl_defaults_core),
+
+	X86_MATCH_VFM(INTEL_HASWELL,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_HASWELL_L,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_HASWELL_G,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_HASWELL_X,			&rapl_defaults_hsw_server),
+
+	X86_MATCH_VFM(INTEL_BROADWELL,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_BROADWELL_G,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_BROADWELL_D,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_BROADWELL_X,		&rapl_defaults_hsw_server),
+
+	X86_MATCH_VFM(INTEL_SKYLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_SKYLAKE_L,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_SKYLAKE_X,			&rapl_defaults_hsw_server),
+	X86_MATCH_VFM(INTEL_KABYLAKE_L,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_KABYLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_CANNONLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ICELAKE_L,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ICELAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ICELAKE_NNPI,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ICELAKE_X,			&rapl_defaults_hsw_server),
+	X86_MATCH_VFM(INTEL_ICELAKE_D,			&rapl_defaults_hsw_server),
+	X86_MATCH_VFM(INTEL_COMETLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_COMETLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_TIGERLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_TIGERLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ROCKETLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ALDERLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ALDERLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_RAPTORLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_RAPTORLAKE_P,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_BARTLETTLAKE,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_METEORLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_METEORLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,		&rapl_defaults_spr_server),
+	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,		&rapl_defaults_spr_server),
+	X86_MATCH_VFM(INTEL_LUNARLAKE_M,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_NOVALAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_NOVALAKE_L,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ARROWLAKE_H,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ARROWLAKE,			&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ARROWLAKE_U,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_LAKEFIELD,			&rapl_defaults_core),
+
+	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT,		&rapl_defaults_byt),
+	X86_MATCH_VFM(INTEL_ATOM_AIRMONT,		&rapl_defaults_cht),
+	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID,	&rapl_defaults_tng),
+	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID2,	&rapl_defaults_ann),
+	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_TREMONT,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,		&rapl_defaults_core),
+	X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,		&rapl_defaults_core),
+
+	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,		&rapl_defaults_hsw_server),
+	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,		&rapl_defaults_hsw_server),
+
+	X86_MATCH_VENDOR_FAM(AMD, 0x17,			&rapl_defaults_amd),
+	X86_MATCH_VENDOR_FAM(AMD, 0x19,			&rapl_defaults_amd),
+	X86_MATCH_VENDOR_FAM(AMD, 0x1A,			&rapl_defaults_amd),
+	X86_MATCH_VENDOR_FAM(HYGON, 0x18,		&rapl_defaults_amd),
+	{}
+};
+MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
+
 static int rapl_msr_probe(struct platform_device *pdev)
 {
 	const struct x86_cpu_id *id = x86_match_cpu(pl4_support_ids);
@@ -204,6 +417,7 @@ static int rapl_msr_probe(struct platform_device *pdev)
 	}
 	rapl_msr_priv->read_raw = rapl_msr_read_raw;
 	rapl_msr_priv->write_raw = rapl_msr_write_raw;
+	rapl_msr_priv->defaults = (const struct rapl_defaults *)pdev->dev.platform_data;
 
 	if (id) {
 		rapl_msr_priv->limits[RAPL_DOMAIN_PACKAGE] |= BIT(POWER_LIMIT4);
@@ -258,8 +472,43 @@ static struct platform_driver intel_rapl_msr_driver = {
 	},
 };
 
-module_platform_driver(intel_rapl_msr_driver);
+static struct platform_device *rapl_msr_platdev;
+
+static int intel_rapl_msr_init(void)
+{
+	const struct rapl_defaults *def;
+	const struct x86_cpu_id *id;
+	int ret;
+
+	ret = platform_driver_register(&intel_rapl_msr_driver);
+	if (ret)
+		return ret;
+
+	/* Create the MSR RAPL platform device for supported platforms */
+	id = x86_match_cpu(rapl_ids);
+	if (!id)
+		return 0;
+
+	def = (const struct rapl_defaults *)id->driver_data;
+
+	rapl_msr_platdev = platform_device_register_data(NULL, "intel_rapl_msr", 0, def,
+							 sizeof(*def));
+	if (IS_ERR(rapl_msr_platdev))
+		pr_debug("intel_rapl_msr device register failed, ret:%ld\n",
+			 PTR_ERR(rapl_msr_platdev));
+
+	return 0;
+}
+module_init(intel_rapl_msr_init);
+
+static void intel_rapl_msr_exit(void)
+{
+	platform_device_unregister(rapl_msr_platdev);
+	platform_driver_unregister(&intel_rapl_msr_driver);
+}
+module_exit(intel_rapl_msr_exit);
 
 MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit) control via MSR interface");
 MODULE_AUTHOR("Zhang Rui <rui.zhang@intel.com>");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("INTEL_RAPL");
diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c
index ba956a2571d16..c06d687366fcf 100644
--- a/drivers/powercap/intel_rapl_tpmi.c
+++ b/drivers/powercap/intel_rapl_tpmi.c
@@ -9,12 +9,14 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/auxiliary_bus.h>
+#include <linux/bits.h>
 #include <linux/intel_rapl.h>
 #include <linux/intel_tpmi.h>
 #include <linux/intel_vsec.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/units.h>
 
 #define TPMI_RAPL_MAJOR_VERSION 0
 #define TPMI_RAPL_MINOR_VERSION 1
@@ -250,6 +252,50 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset)
 	return 0;
 }
 
+/* TPMI Unit register has different layout */
+#define TPMI_ENERGY_UNIT_SCALE		1000
+#define TPMI_POWER_UNIT_OFFSET		0x00
+#define TPMI_POWER_UNIT_MASK		GENMASK(3, 0)
+#define TPMI_ENERGY_UNIT_OFFSET		0x06
+#define TPMI_ENERGY_UNIT_MASK		GENMASK_ULL(10, 6)
+#define TPMI_TIME_UNIT_OFFSET		0x0C
+#define TPMI_TIME_UNIT_MASK		GENMASK_ULL(15, 12)
+
+static int rapl_check_unit_tpmi(struct rapl_domain *rd)
+{
+	struct reg_action ra;
+	u32 value;
+
+	ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
+	ra.mask = ~0;
+	if (tpmi_rapl_read_raw(rd->rp->id, &ra, false)) {
+		pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
+			ra.reg.val, rd->rp->name, rd->name);
+		return -ENODEV;
+	}
+
+	value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET;
+	rd->energy_unit = (TPMI_ENERGY_UNIT_SCALE * MICROJOULE_PER_JOULE) >> value;
+
+	value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET;
+	rd->power_unit = MICROWATT_PER_WATT >> value;
+
+	value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET;
+	rd->time_unit = USEC_PER_SEC >> value;
+
+	pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
+		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
+
+	return 0;
+}
+
+static const struct rapl_defaults defaults_tpmi = {
+	.check_unit = rapl_check_unit_tpmi,
+	/* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */
+	.set_floor_freq = rapl_default_set_floor_freq,
+	.compute_time_window = rapl_default_compute_time_window,
+};
+
 static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev,
 				 const struct auxiliary_device_id *id)
 {
@@ -297,6 +343,7 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev,
 	trp->priv.read_raw = tpmi_rapl_read_raw;
 	trp->priv.write_raw = tpmi_rapl_write_raw;
 	trp->priv.control_type = tpmi_control_type;
+	trp->priv.defaults = &defaults_tpmi;
 
 	/* RAPL TPMI I/F is per physical package */
 	trp->rp = rapl_find_package_domain(info->package_id, &trp->priv, false);
@@ -348,6 +395,7 @@ static struct auxiliary_driver intel_rapl_tpmi_driver = {
 
 module_auxiliary_driver(intel_rapl_tpmi_driver)
 
+MODULE_IMPORT_NS("INTEL_RAPL");
 MODULE_IMPORT_NS("INTEL_TPMI");
 
 MODULE_DESCRIPTION("Intel RAPL TPMI Driver");
diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
index d88ab2f86b1bf..beacc2ffb166b 100644
--- a/drivers/ptp/ptp_ocp.c
+++ b/drivers/ptp/ptp_ocp.c
@@ -35,6 +35,7 @@
 
 #define PCI_VENDOR_ID_ADVA			0xad5a
 #define PCI_DEVICE_ID_ADVA_TIMECARD		0x0400
+#define PCI_DEVICE_ID_ADVA_TIMECARD_X1		0x0410
 
 static struct class timecard_class = {
 	.name		= "timecard",
@@ -72,6 +73,20 @@ struct ptp_ocp_servo_conf {
 	u32	servo_drift_i;
 };
 
+/*
+ * Combined servo + board-variant parameters for ADVA boards.
+ * Embedded in the resource table .extra so a single ptp_ocp_adva_board_init()
+ * can handle both ADVA and ADVA-X1 without per-variant init functions.
+ */
+struct ptp_ocp_adva_info {
+	struct ptp_ocp_servo_conf	servo;
+	u32				flash_start;
+	const struct ocp_sma_op		*sma_op;
+	u8				signals_nr;
+	u8				freq_in_nr;
+	const struct ocp_attr_group	*attr_groups;
+};
+
 #define OCP_CTRL_ENABLE		BIT(0)
 #define OCP_CTRL_ADJUST_TIME	BIT(1)
 #define OCP_CTRL_ADJUST_OFFSET	BIT(2)
@@ -293,6 +308,20 @@ struct ocp_attr_group {
 	const struct attribute_group *group;
 };
 
+struct ocp_selector {
+	const char *name;
+	int value;
+	u64 frequency;
+};
+
+struct ocp_sma_op {
+	const struct ocp_selector *tbl[2];
+	void (*init)(struct ptp_ocp *bp);
+	u32 (*get)(struct ptp_ocp *bp, int sma_nr);
+	int (*set_inputs)(struct ptp_ocp *bp, int sma_nr, u32 val);
+	int (*set_output)(struct ptp_ocp *bp, int sma_nr, u32 val);
+};
+
 #define OCP_CAP_BASIC	BIT(0)
 #define OCP_CAP_SIGNAL	BIT(1)
 #define OCP_CAP_FREQ	BIT(2)
@@ -420,12 +449,17 @@ static int ptp_ocp_art_board_init(struct ptp_ocp *bp, struct ocp_resource *r);
 
 static int ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r);
 
+static const struct ocp_sma_op ocp_adva_sma_op;
+static const struct ocp_sma_op ocp_adva_x1_sma_op;
+
 static const struct ocp_attr_group fb_timecard_groups[];
 
 static const struct ocp_attr_group art_timecard_groups[];
 
 static const struct ocp_attr_group adva_timecard_groups[];
 
+static const struct ocp_attr_group adva_timecard_x1_groups[];
+
 struct ptp_ocp_eeprom_map {
 	u16	off;
 	u16	len;
@@ -1020,11 +1054,225 @@ static struct ocp_resource ocp_adva_resource[] = {
 	},
 	{
 		.setup = ptp_ocp_adva_board_init,
-		.extra = &(struct ptp_ocp_servo_conf) {
-			.servo_offset_p = 0xc000,
-			.servo_offset_i = 0x1000,
-			.servo_drift_p = 0,
-			.servo_drift_i = 0,
+		.extra = &(struct ptp_ocp_adva_info) {
+			.servo = {
+				.servo_offset_p = 0xc000,
+				.servo_offset_i = 0x1000,
+				.servo_drift_p = 0,
+				.servo_drift_i = 0,
+			},
+			.flash_start  = 0xA00000,
+			.sma_op       = &ocp_adva_sma_op,
+			.signals_nr   = 2,
+			.freq_in_nr   = 2,
+			.attr_groups  = adva_timecard_groups,
+		},
+	},
+	{ }
+};
+
+static struct ocp_resource ocp_adva_x1_resource[] = {
+	{
+		OCP_MEM_RESOURCE(reg),
+		.offset = 0x01000000, .size = 0x10000,
+	},
+	{
+		OCP_EXT_RESOURCE(ts0),
+		.offset = 0x01010000, .size = 0x10000, .irq_vec = 1,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 0,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(ts1),
+		.offset = 0x01020000, .size = 0x10000, .irq_vec = 2,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 1,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(ts2),
+		.offset = 0x01060000, .size = 0x10000, .irq_vec = 6,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 2,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(ts3),
+		.offset = 0x01110000, .size = 0x10000, .irq_vec = 15,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 3,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(ts4),
+		.offset = 0x01120000, .size = 0x10000, .irq_vec = 16,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 4,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	/* Timestamp for PHC and/or PPS generator */
+	{
+		OCP_EXT_RESOURCE(pps),
+		.offset = 0x010C0000, .size = 0x10000, .irq_vec = 0,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 5,
+			.irq_fcn = ptp_ocp_ts_irq,
+			.enable = ptp_ocp_ts_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(signal_out[0]),
+		.offset = 0x010D0000, .size = 0x10000, .irq_vec = 11,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 1,
+			.irq_fcn = ptp_ocp_signal_irq,
+			.enable = ptp_ocp_signal_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(signal_out[1]),
+		.offset = 0x010E0000, .size = 0x10000, .irq_vec = 12,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 2,
+			.irq_fcn = ptp_ocp_signal_irq,
+			.enable = ptp_ocp_signal_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(signal_out[2]),
+		.offset = 0x010F0000, .size = 0x10000, .irq_vec = 13,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 3,
+			.irq_fcn = ptp_ocp_signal_irq,
+			.enable = ptp_ocp_signal_enable,
+		},
+	},
+	{
+		OCP_EXT_RESOURCE(signal_out[3]),
+		.offset = 0x01100000, .size = 0x10000, .irq_vec = 14,
+		.extra = &(struct ptp_ocp_ext_info) {
+			.index = 4,
+			.irq_fcn = ptp_ocp_signal_irq,
+			.enable = ptp_ocp_signal_enable,
+		},
+	},
+	{
+		OCP_MEM_RESOURCE(pps_to_ext),
+		.offset = 0x01030000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(pps_to_clk),
+		.offset = 0x01040000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(tod),
+		.offset = 0x01050000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(image),
+		.offset = 0x00020000, .size = 0x1000,
+	},
+	{
+		OCP_MEM_RESOURCE(pps_select),
+		.offset = 0x00130000, .size = 0x1000,
+	},
+	{
+		OCP_MEM_RESOURCE(sma_map1),
+		.offset = 0x00140000, .size = 0x1000,
+	},
+	{
+		OCP_MEM_RESOURCE(sma_map2),
+		.offset = 0x00220000, .size = 0x1000,
+	},
+	{
+		OCP_SERIAL_RESOURCE(port[PORT_GNSS]),
+		.offset = 0x00160000 + 0x1000, .irq_vec = 3,
+		.extra = &(struct ptp_ocp_serial_port) {
+			.baud = 9600,
+		},
+	},
+	{
+		OCP_SERIAL_RESOURCE(port[PORT_MAC]),
+		.offset = 0x00180000 + 0x1000, .irq_vec = 5,
+		.extra = &(struct ptp_ocp_serial_port) {
+			.baud = 115200,
+		},
+	},
+	{
+		OCP_MEM_RESOURCE(freq_in[0]),
+		.offset = 0x01200000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(freq_in[1]),
+		.offset = 0x01210000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(freq_in[2]),
+		.offset = 0x01220000, .size = 0x10000,
+	},
+	{
+		OCP_MEM_RESOURCE(freq_in[3]),
+		.offset = 0x01230000, .size = 0x10000,
+	},
+	{
+		OCP_SPI_RESOURCE(spi_flash),
+		.offset = 0x00310000, .size = 0x10000, .irq_vec = 9,
+		.extra = &(struct ptp_ocp_flash_info) {
+			.name = "xilinx_spi", .pci_offset = 0,
+			.data_size = sizeof(struct xspi_platform_data),
+			.data = &(struct xspi_platform_data) {
+				.num_chipselect = 1,
+				.bits_per_word = 8,
+				.num_devices = 1,
+				.force_irq = true,
+				.devices = &(struct spi_board_info) {
+					.modalias = "spi-nor",
+				},
+			},
+		},
+	},
+	{
+		OCP_I2C_RESOURCE(i2c_ctrl),
+		.offset = 0x00150000, .size = 0x10000, .irq_vec = 7,
+		.extra = &(struct ptp_ocp_i2c_info) {
+			.name = "xiic-i2c",
+			.fixed_rate = 50000000,
+			.data_size = sizeof(struct xiic_i2c_platform_data),
+			.data = &(struct xiic_i2c_platform_data) {
+				.num_devices = 2,
+				.devices = (struct i2c_board_info[]) {
+					{ I2C_BOARD_INFO("24c02", 0x50) },
+					{ I2C_BOARD_INFO("24mac402", 0x58),
+					  .platform_data = "mac" },
+				},
+			},
+		},
+	},
+	{
+		.setup = ptp_ocp_adva_board_init,
+		.extra = &(struct ptp_ocp_adva_info) {
+			.servo = {
+				.servo_offset_p = 0xc000,
+				.servo_offset_i = 0x1000,
+				.servo_drift_p = 0,
+				.servo_drift_i = 0,
+			},
+			.flash_start  = 0x1000000,
+			.sma_op       = &ocp_adva_x1_sma_op,
+			.signals_nr   = 4,
+			.freq_in_nr   = 4,
+			.attr_groups  = adva_timecard_x1_groups,
 		},
 	},
 	{ }
@@ -1035,6 +1283,7 @@ static const struct pci_device_id ptp_ocp_pcidev_id[] = {
 	{ PCI_DEVICE_DATA(CELESTICA, TIMECARD, &ocp_fb_resource) },
 	{ PCI_DEVICE_DATA(OROLIA, ARTCARD, &ocp_art_resource) },
 	{ PCI_DEVICE_DATA(ADVA, TIMECARD, &ocp_adva_resource) },
+	{ PCI_DEVICE_DATA(ADVA, TIMECARD_X1, &ocp_adva_x1_resource) },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, ptp_ocp_pcidev_id);
@@ -1042,12 +1291,6 @@ MODULE_DEVICE_TABLE(pci, ptp_ocp_pcidev_id);
 static DEFINE_MUTEX(ptp_ocp_lock);
 static DEFINE_IDR(ptp_ocp_idr);
 
-struct ocp_selector {
-	const char *name;
-	int value;
-	u64 frequency;
-};
-
 static const struct ocp_selector ptp_ocp_clock[] = {
 	{ .name = "NONE",	.value = 0 },
 	{ .name = "TOD",	.value = 1 },
@@ -1137,12 +1380,32 @@ static const struct ocp_selector ptp_ocp_adva_sma_out[] = {
 	{ }
 };
 
-struct ocp_sma_op {
-	const struct ocp_selector *tbl[2];
-	void (*init)(struct ptp_ocp *bp);
-	u32 (*get)(struct ptp_ocp *bp, int sma_nr);
-	int (*set_inputs)(struct ptp_ocp *bp, int sma_nr, u32 val);
-	int (*set_output)(struct ptp_ocp *bp, int sma_nr, u32 val);
+static const struct ocp_selector ptp_ocp_adva_x1_sma_in[] = {
+	{ .name = "PPS1",	.value = 0x0001,      .frequency = 1 },
+	{ .name = "TS1",	.value = 0x0004,      .frequency = 0 },
+	{ .name = "TS2",	.value = 0x0008,      .frequency = 0 },
+	{ .name = "TS3",    .value = 0x0040,      .frequency = 0 },
+	{ .name = "TS4",    .value = 0x0080,      .frequency = 0 },
+	{ .name = "FREQ1",	.value = 0x0100,      .frequency = 0 },
+	{ .name = "FREQ2",	.value = 0x0200,      .frequency = 0 },
+	{ .name = "FREQ3",  .value = 0x0400,      .frequency = 0 },
+	{ .name = "FREQ4",  .value = 0x0800,      .frequency = 0 },
+	{ .name = "None",	.value = SMA_DISABLE, .frequency = 0 },
+	{ }
+};
+
+static const struct ocp_selector ptp_ocp_adva_x1_sma_out[] = {
+	{ .name = "10Mhz",	.value = 0x0000,  .frequency = 10000000},
+	{ .name = "PHC",	.value = 0x0001,  .frequency = 1 },
+	{ .name = "MAC",	.value = 0x0002,  .frequency = 1 },
+	{ .name = "GNSS1",	.value = 0x0004,  .frequency = 1 },
+	{ .name = "GEN1",	.value = 0x0040 },
+	{ .name = "GEN2",	.value = 0x0080 },
+	{ .name = "GEN3",	.value = 0x0100 },
+	{ .name = "GEN4",	.value = 0x0200 },
+	{ .name = "GND",	.value = 0x2000 },
+	{ .name = "VCC",	.value = 0x4000 },
+	{ }
 };
 
 static void
@@ -2639,6 +2902,14 @@ static const struct ocp_sma_op ocp_adva_sma_op = {
 	.set_output	= ptp_ocp_sma_adva_set_output,
 };
 
+static const struct ocp_sma_op ocp_adva_x1_sma_op = {
+	.tbl		= { ptp_ocp_adva_x1_sma_in, ptp_ocp_adva_x1_sma_out },
+	.init		= ptp_ocp_sma_fb_init,
+	.get		= ptp_ocp_sma_fb_get,
+	.set_inputs	= ptp_ocp_sma_adva_set_inputs,
+	.set_output	= ptp_ocp_sma_adva_set_output,
+};
+
 static int
 ptp_ocp_set_pins(struct ptp_ocp *bp)
 {
@@ -2887,18 +3158,19 @@ ptp_ocp_art_board_init(struct ptp_ocp *bp, struct ocp_resource *r)
 	return ptp_ocp_init_clock(bp, r->extra);
 }
 
-/* ADVA specific board initializers; last "resource" registered. */
+/* ADVA board initializer; variant differences come from r->extra. */
 static int
 ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r)
 {
-	int err;
+	struct ptp_ocp_adva_info *info = r->extra;
 	u32 version;
+	int err;
 
-	bp->flash_start = 0xA00000;
-	bp->eeprom_map = fb_eeprom_map;
-	bp->sma_op = &ocp_adva_sma_op;
-	bp->signals_nr = 2;
-	bp->freq_in_nr = 2;
+	bp->flash_start = info->flash_start;
+	bp->eeprom_map  = fb_eeprom_map;
+	bp->sma_op      = info->sma_op;
+	bp->signals_nr  = info->signals_nr;
+	bp->freq_in_nr  = info->freq_in_nr;
 
 	version = ioread32(&bp->image->version);
 	/* if lower 16 bits are empty, this is the fw loader. */
@@ -2906,15 +3178,15 @@ ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r)
 		version = version >> 16;
 		bp->fw_loader = true;
 	}
-	bp->fw_tag = 3;
+	bp->fw_tag     = 3;
 	bp->fw_version = version & 0xffff;
-	bp->fw_cap = OCP_CAP_BASIC | OCP_CAP_SIGNAL | OCP_CAP_FREQ;
+	bp->fw_cap     = OCP_CAP_BASIC | OCP_CAP_SIGNAL | OCP_CAP_FREQ;
 
 	ptp_ocp_tod_init(bp);
 	ptp_ocp_nmea_out_init(bp);
 	ptp_ocp_signal_init(bp);
 
-	err = ptp_ocp_attr_group_add(bp, adva_timecard_groups);
+	err = ptp_ocp_attr_group_add(bp, info->attr_groups);
 	if (err)
 		return err;
 
@@ -2923,7 +3195,7 @@ ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r)
 		return err;
 	ptp_ocp_sma_init(bp);
 
-	return ptp_ocp_init_clock(bp, r->extra);
+	return ptp_ocp_init_clock(bp, &info->servo);
 }
 
 static ssize_t
@@ -3982,6 +4254,43 @@ static const struct ocp_attr_group adva_timecard_groups[] = {
 	{ },
 };
 
+static struct attribute *adva_timecard_x1_attrs[] = {
+	&dev_attr_serialnum.attr,
+	&dev_attr_gnss_sync.attr,
+	&dev_attr_clock_source.attr,
+	&dev_attr_available_clock_sources.attr,
+	&dev_attr_sma1.attr,
+	&dev_attr_sma2.attr,
+	&dev_attr_sma3.attr,
+	&dev_attr_sma4.attr,
+	&dev_attr_available_sma_inputs.attr,
+	&dev_attr_available_sma_outputs.attr,
+	&dev_attr_clock_status_drift.attr,
+	&dev_attr_clock_status_offset.attr,
+	&dev_attr_ts_window_adjust.attr,
+	&dev_attr_utc_tai_offset.attr,
+	&dev_attr_tod_correction.attr,
+	NULL,
+};
+
+static const struct attribute_group adva_timecard_x1_group = {
+	.attrs = adva_timecard_x1_attrs,
+};
+
+static const struct ocp_attr_group adva_timecard_x1_groups[] = {
+	{ .cap = OCP_CAP_BASIC,	    .group = &adva_timecard_x1_group },
+	{ .cap = OCP_CAP_BASIC,	    .group = &ptp_ocp_timecard_tty_group },
+	{ .cap = OCP_CAP_SIGNAL,    .group = &fb_timecard_signal0_group },
+	{ .cap = OCP_CAP_SIGNAL,    .group = &fb_timecard_signal1_group },
+	{ .cap = OCP_CAP_SIGNAL,    .group = &fb_timecard_signal2_group },
+	{ .cap = OCP_CAP_SIGNAL,    .group = &fb_timecard_signal3_group },
+	{ .cap = OCP_CAP_FREQ,	    .group = &fb_timecard_freq0_group },
+	{ .cap = OCP_CAP_FREQ,	    .group = &fb_timecard_freq1_group },
+	{ .cap = OCP_CAP_FREQ,	    .group = &fb_timecard_freq2_group },
+	{ .cap = OCP_CAP_FREQ,	    .group = &fb_timecard_freq3_group },
+	{ },
+};
+
 static void
 gpio_input_map(char *buf, struct ptp_ocp *bp, u16 map[][2], u16 bit,
 	       const char *def)
diff --git a/drivers/ptp/ptp_vmw.c b/drivers/ptp/ptp_vmw.c
index 20ab05c4daa8a..8510121d79d10 100644
--- a/drivers/ptp/ptp_vmw.c
+++ b/drivers/ptp/ptp_vmw.c
@@ -10,6 +10,7 @@
 #include <linux/acpi.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/ptp_clock_kernel.h>
 #include <asm/hypervisor.h>
 #include <asm/vmware.h>
@@ -83,7 +84,7 @@ static struct ptp_clock_info ptp_vmw_clock_info = {
  * ACPI driver ops for VMware "precision clock" virtual device.
  */
 
-static int ptp_vmw_acpi_add(struct acpi_device *device)
+static int ptp_vmw_acpi_probe(struct platform_device *pdev)
 {
 	ptp_vmw_clock = ptp_clock_register(&ptp_vmw_clock_info, NULL);
 	if (IS_ERR(ptp_vmw_clock)) {
@@ -91,11 +92,11 @@ static int ptp_vmw_acpi_add(struct acpi_device *device)
 		return PTR_ERR(ptp_vmw_clock);
 	}
 
-	ptp_vmw_acpi_device = device;
+	ptp_vmw_acpi_device = ACPI_COMPANION(&pdev->dev);
 	return 0;
 }
 
-static void ptp_vmw_acpi_remove(struct acpi_device *device)
+static void ptp_vmw_acpi_remove(struct platform_device *pdev)
 {
 	ptp_clock_unregister(ptp_vmw_clock);
 }
@@ -107,12 +108,12 @@ static const struct acpi_device_id ptp_vmw_acpi_device_ids[] = {
 
 MODULE_DEVICE_TABLE(acpi, ptp_vmw_acpi_device_ids);
 
-static struct acpi_driver ptp_vmw_acpi_driver = {
-	.name = "ptp_vmw",
-	.ids = ptp_vmw_acpi_device_ids,
-	.ops = {
-		.add = ptp_vmw_acpi_add,
-		.remove	= ptp_vmw_acpi_remove
+static struct platform_driver ptp_vmw_acpi_driver = {
+	.probe = ptp_vmw_acpi_probe,
+	.remove = ptp_vmw_acpi_remove,
+	.driver = {
+		.name = "ptp_vmw_acpi",
+		.acpi_match_table = ptp_vmw_acpi_device_ids,
 	},
 };
 
@@ -120,12 +121,12 @@ static int __init ptp_vmw_init(void)
 {
 	if (x86_hyper_type != X86_HYPER_VMWARE)
 		return -1;
-	return acpi_bus_register_driver(&ptp_vmw_acpi_driver);
+	return platform_driver_register(&ptp_vmw_acpi_driver);
 }
 
 static void __exit ptp_vmw_exit(void)
 {
-	acpi_bus_unregister_driver(&ptp_vmw_acpi_driver);
+	platform_driver_unregister(&ptp_vmw_acpi_driver);
 }
 
 module_init(ptp_vmw_init);
diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c
index 5b399de16d604..80fdb3303400f 100644
--- a/drivers/pwm/pwm-imx-tpm.c
+++ b/drivers/pwm/pwm-imx-tpm.c
@@ -352,7 +352,7 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev)
 	struct clk *clk;
 	void __iomem *base;
 	int ret;
-	unsigned int npwm;
+	unsigned int i, npwm;
 	u32 val;
 
 	base = devm_platform_ioremap_resource(pdev, 0);
@@ -382,6 +382,13 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev)
 
 	mutex_init(&tpm->lock);
 
+	/* count the enabled channels */
+	for (i = 0; i < npwm; ++i) {
+		val = readl(base + PWM_IMX_TPM_CnSC(i));
+		if (FIELD_GET(PWM_IMX_TPM_CnSC_ELS, val))
+			++tpm->enable_count;
+	}
+
 	ret = devm_pwmchip_add(&pdev->dev, chip);
 	if (ret)
 		return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n");
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index d10b6f9243d51..10e64e3ffb1f5 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -1232,7 +1232,6 @@ config REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY
 	depends on ARM || ARM64 || COMPILE_TEST
 	depends on BACKLIGHT_CLASS_DEVICE
 	depends on I2C
-	depends on OF_GPIO
 	select REGMAP_I2C
 	help
 	  This driver supports ATTINY regulator on the Raspberry Pi 7-inch
@@ -1332,7 +1331,6 @@ config REGULATOR_RT5133
 	depends on I2C && GPIOLIB && OF
 	select REGMAP
 	select CRC8
-	select OF_GPIO
 	help
 	  This driver adds support for RT5133 PMIC regulators.
 	  RT5133 is an integrated chip. It includes 8 LDOs and 3 GPOs that
diff --git a/drivers/regulator/cpcap-regulator.c b/drivers/regulator/cpcap-regulator.c
index 6958d154442be..63f5c90ddf50b 100644
--- a/drivers/regulator/cpcap-regulator.c
+++ b/drivers/regulator/cpcap-regulator.c
@@ -261,6 +261,30 @@ static const struct regulator_ops cpcap_regulator_ops = {
 };
 
 static const unsigned int unknown_val_tbl[] = { 0, };
+static const unsigned int sw_mot_val_tbl[] =  { 600000, 612500, 625000,
+						637500, 650000, 662500,
+						675000, 687500, 700000,
+						712500, 725000, 737500,
+						750000, 762500, 775000,
+						787500, 800000, 812500,
+						825000, 837500, 850000,
+						862500, 875000, 887500,
+						900000, 912500, 925000,
+						937500, 950000, 962500,
+						975000, 987500, 1000000,
+						1012500, 1025000, 1037500,
+						1050000, 1062500, 1075000,
+						1087500, 1100000, 1112500,
+						1125000, 1137500, 1150000,
+						1162500, 1175000, 1187500,
+						1200000, 1212500, 1225000,
+						1237500, 1250000, 1262500,
+						1275000, 1287500, 1300000,
+						1312500, 1325000, 1337500,
+						1350000, 1362500, 1375000,
+						1387500, 1400000, 1412500,
+						1425000, 1437500, 1450000,
+						1462500, 1475000, };
 static const unsigned int sw2_sw4_val_tbl[] = { 612500, 625000, 637500,
 						650000, 662500, 675000,
 						687500, 700000, 712500,
@@ -284,6 +308,7 @@ static const unsigned int sw2_sw4_val_tbl[] = { 612500, 625000, 637500,
 						1362500, 1375000, 1387500,
 						1400000, 1412500, 1425000,
 						1437500, 1450000, 1462500, };
+static const unsigned int sw3_val_tbl[] = { 1350000, 1800000, 1850000, 1875000, };
 static const unsigned int sw5_val_tbl[] = { 0, 5050000, };
 static const unsigned int vcam_val_tbl[] = { 2600000, 2700000, 2800000,
 					     2900000, };
@@ -402,6 +427,82 @@ static const struct cpcap_regulator omap4_regulators[] = {
 	{ /* sentinel */ },
 };
 
+static const struct cpcap_regulator mot_regulators[] = {
+	CPCAP_REG(SW1, CPCAP_REG_S1C1, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW1_SEL, sw_mot_val_tbl,
+		  0x6f00, 0x7f, 0x6800, 0, 0),
+	CPCAP_REG(SW2, CPCAP_REG_S2C1, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW2_SEL, sw_mot_val_tbl,
+		  0x6f00, 0x7f, 0x4804, 0, 0),
+	CPCAP_REG(SW3, CPCAP_REG_S3C, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW3_SEL, sw3_val_tbl,
+		  0x578, 0x3, 0x043c, 0, 0),
+	CPCAP_REG(SW4, CPCAP_REG_S4C1, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW4_SEL, sw_mot_val_tbl,
+		  0x6f00, 0x7f, 0x4909, 0, 0),
+	CPCAP_REG(SW5, CPCAP_REG_S5C, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW5_SEL, sw5_val_tbl,
+		  0x28, 0, 0x20, 0, 0),
+	CPCAP_REG(SW6, CPCAP_REG_S6C, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_SW6_SEL, unknown_val_tbl,
+		  0, 0, 0, 0, 0),
+	CPCAP_REG(VCAM, CPCAP_REG_VCAMC, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_VCAM_SEL, vcam_val_tbl,
+		  0x87, 0x30, 0x7, 0, 420),
+	CPCAP_REG(VCSI, CPCAP_REG_VCSIC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VCSI_SEL, vcsi_val_tbl,
+		  0x47, 0x10, 0x7, 0, 350),
+	CPCAP_REG(VDAC, CPCAP_REG_VDACC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VDAC_SEL, vdac_val_tbl,
+		  0x87, 0x30, 0x0, 0, 420),
+	CPCAP_REG(VDIG, CPCAP_REG_VDIGC, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_VDIG_SEL, vdig_val_tbl,
+		  0x87, 0x30, 0x0, 0, 420),
+	CPCAP_REG(VFUSE, CPCAP_REG_VFUSEC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VFUSE_SEL, vfuse_val_tbl,
+		  0xa0, 0xf, 0x0, 0, 420),
+	CPCAP_REG(VHVIO, CPCAP_REG_VHVIOC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VHVIO_SEL, vhvio_val_tbl,
+		  0x17, 0, 0x2, 0, 0),
+	CPCAP_REG(VSDIO, CPCAP_REG_VSDIOC, CPCAP_REG_ASSIGN2,
+		  CPCAP_BIT_VSDIO_SEL, vsdio_val_tbl,
+		  0x87, 0x38, 0x2, 0, 420),
+	CPCAP_REG(VPLL, CPCAP_REG_VPLLC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VPLL_SEL, vpll_val_tbl,
+		  0x47, 0x18, 0x1, 0, 420),
+	CPCAP_REG(VRF1, CPCAP_REG_VRF1C, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VRF1_SEL, vrf1_val_tbl,
+		  0xac, 0x2, 0, 0, 10),
+	CPCAP_REG(VRF2, CPCAP_REG_VRF2C, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VRF2_SEL, vrf2_val_tbl,
+		  0x23, 0x8, 0, 0, 10),
+	CPCAP_REG(VRFREF, CPCAP_REG_VRFREFC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VRFREF_SEL, vrfref_val_tbl,
+		  0x23, 0x8, 0, 0, 420),
+	CPCAP_REG(VWLAN1, CPCAP_REG_VWLAN1C, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VWLAN1_SEL, vwlan1_val_tbl,
+		  0x47, 0x10, 0x5, 0, 420),
+	CPCAP_REG(VWLAN2, CPCAP_REG_VWLAN2C, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VWLAN2_SEL, vwlan2_val_tbl,
+		  0x20c, 0xc0, 0xd, 0, 420),
+	CPCAP_REG(VSIM, CPCAP_REG_VSIMC, CPCAP_REG_ASSIGN3,
+		  0xffff, vsim_val_tbl,
+		  0x23, 0x8, 0, 0, 420),
+	CPCAP_REG(VSIMCARD, CPCAP_REG_VSIMC, CPCAP_REG_ASSIGN3,
+		  0xffff, vsimcard_val_tbl,
+		  0x1e80, 0x8, 0x1e00, 0, 420),
+	CPCAP_REG(VVIB, CPCAP_REG_VVIBC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VVIB_SEL, vvib_val_tbl,
+		  0x1, 0xc, 0x1, 0, 500),
+	CPCAP_REG(VUSB, CPCAP_REG_VUSBC, CPCAP_REG_ASSIGN3,
+		  CPCAP_BIT_VUSB_SEL, vusb_val_tbl,
+		  0x11c, 0x40, 0xc, 0, 0),
+	CPCAP_REG(VAUDIO, CPCAP_REG_VAUDIOC, CPCAP_REG_ASSIGN4,
+		  CPCAP_BIT_VAUDIO_SEL, vaudio_val_tbl,
+		  0x16, 0x1, 0x5, 0, 0),
+	{ /* sentinel */ }
+};
+
 static const struct cpcap_regulator xoom_regulators[] = {
 	CPCAP_REG(SW1, CPCAP_REG_S1C1, CPCAP_REG_ASSIGN2,
 		  CPCAP_BIT_SW1_SEL, unknown_val_tbl,
@@ -486,6 +587,10 @@ static const struct of_device_id cpcap_regulator_id_table[] = {
 		.compatible = "motorola,mapphone-cpcap-regulator",
 		.data = omap4_regulators,
 	},
+	{
+		.compatible = "motorola,mot-cpcap-regulator",
+		.data = mot_regulators,
+	},
 	{
 		.compatible = "motorola,xoom-cpcap-regulator",
 		.data = xoom_regulators,
diff --git a/drivers/regulator/da9063-regulator.c b/drivers/regulator/da9063-regulator.c
index 9d369cc45d417..bf34ea440d77c 100644
--- a/drivers/regulator/da9063-regulator.c
+++ b/drivers/regulator/da9063-regulator.c
@@ -67,7 +67,7 @@ struct da9063_regulator_data {
 
 struct da9063_regulators_pdata {
 	unsigned int			n_regulators;
-	struct da9063_regulator_data	*regulator_data;
+	struct da9063_regulator_data	regulator_data[] __counted_by(n_regulators);
 };
 
 /* Regulator capabilities and registers description */
@@ -857,15 +857,10 @@ static struct da9063_regulators_pdata *da9063_parse_regulators_dt(
 		return ERR_PTR(-EINVAL);
 	}
 
-	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	pdata = devm_kzalloc(&pdev->dev, struct_size(pdata, regulator_data, num), GFP_KERNEL);
 	if (!pdata)
 		return ERR_PTR(-ENOMEM);
 
-	pdata->regulator_data = devm_kcalloc(&pdev->dev,
-					num, sizeof(*pdata->regulator_data),
-					GFP_KERNEL);
-	if (!pdata->regulator_data)
-		return ERR_PTR(-ENOMEM);
 	pdata->n_regulators = num;
 
 	n = 0;
diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c
index 254c0a8a45559..3b2fddc1c8a39 100644
--- a/drivers/regulator/fixed.c
+++ b/drivers/regulator/fixed.c
@@ -20,7 +20,6 @@
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_opp.h>
-#include <linux/reboot.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/fixed.h>
 #include <linux/gpio/consumer.h>
diff --git a/drivers/regulator/fp9931.c b/drivers/regulator/fp9931.c
index abea3b69d8a08..002b41f53eff8 100644
--- a/drivers/regulator/fp9931.c
+++ b/drivers/regulator/fp9931.c
@@ -446,7 +446,7 @@ static int fp9931_probe(struct i2c_client *client)
 		return dev_err_probe(&client->dev, PTR_ERR(data->regmap),
 				     "failed to allocate regmap!\n");
 
-	data->vin_reg = devm_regulator_get_optional(&client->dev, "vin");
+	data->vin_reg = devm_regulator_get(&client->dev, "vin");
 	if (IS_ERR(data->vin_reg))
 		return dev_err_probe(&client->dev, PTR_ERR(data->vin_reg),
 				     "failed to get vin regulator\n");
diff --git a/drivers/regulator/max20411-regulator.c b/drivers/regulator/max20411-regulator.c
index 02d7009ea0e61..6c0ebb970e902 100644
--- a/drivers/regulator/max20411-regulator.c
+++ b/drivers/regulator/max20411-regulator.c
@@ -133,7 +133,7 @@ static int max20411_probe(struct i2c_client *client)
 
 	max20411->rdev = devm_regulator_register(max20411->dev, &max20411->desc, &cfg);
 	if (IS_ERR(max20411->rdev))
-		dev_err(max20411->dev, "Failed to register regulator\n");
+		dev_err(max20411->dev, "Failed to register regulator: %pe\n", max20411->rdev);
 
 	return PTR_ERR_OR_ZERO(max20411->rdev);
 }
diff --git a/drivers/regulator/pf0900-regulator.c b/drivers/regulator/pf0900-regulator.c
index b5effee329172..5c44d2dbcab4d 100644
--- a/drivers/regulator/pf0900-regulator.c
+++ b/drivers/regulator/pf0900-regulator.c
@@ -771,7 +771,7 @@ static const struct pf0900_regulator_desc pf0900_regulators[] = {
 	},
 };
 
-struct pf0900_regulator_irq regu_irqs[] = {
+static const struct pf0900_regulator_irq regu_irqs[] = {
 	PF0900_REGU_IRQ(PF0900_REG_SW_ILIM_INT, PF0900_SW, REGULATOR_ERROR_OVER_CURRENT_WARN),
 	PF0900_REGU_IRQ(PF0900_REG_LDO_ILIM_INT, PF0900_LDO, REGULATOR_ERROR_OVER_CURRENT_WARN),
 	PF0900_REGU_IRQ(PF0900_REG_SW_UV_INT, PF0900_SW, REGULATOR_ERROR_UNDER_VOLTAGE_WARN),
diff --git a/drivers/regulator/pf1550-regulator.c b/drivers/regulator/pf1550-regulator.c
index 037b8ec940660..1d17265284604 100644
--- a/drivers/regulator/pf1550-regulator.c
+++ b/drivers/regulator/pf1550-regulator.c
@@ -360,7 +360,6 @@ static int pf1550_regulator_probe(struct platform_device *pdev)
 				     "failed to get parent regmap\n");
 
 	config.dev = pf1550->dev;
-	config.regmap = pf1550->regmap;
 	info->dev = &pdev->dev;
 	info->pf1550 = pf1550;
 
diff --git a/drivers/regulator/pf9453-regulator.c b/drivers/regulator/pf9453-regulator.c
index eed3055d1c1ca..07cbcd312653b 100644
--- a/drivers/regulator/pf9453-regulator.c
+++ b/drivers/regulator/pf9453-regulator.c
@@ -732,6 +732,9 @@ static irqreturn_t pf9453_irq_handler(int irq, void *data)
 		return IRQ_NONE;
 	}
 
+	if (!status)
+		return IRQ_NONE;
+
 	if (status & IRQ_RSTB)
 		dev_warn(pf9453->dev, "IRQ_RSTB interrupt.\n");
 
@@ -809,7 +812,7 @@ static int pf9453_i2c_probe(struct i2c_client *i2c)
 	}
 
 	ret = devm_request_threaded_irq(pf9453->dev, pf9453->irq, NULL, pf9453_irq_handler,
-					IRQF_ONESHOT,
+					(IRQF_ONESHOT | IRQF_SHARED),
 					"pf9453-irq", pf9453);
 	if (ret)
 		return dev_err_probe(pf9453->dev, ret, "Failed to request IRQ: %d\n", pf9453->irq);
diff --git a/drivers/regulator/spacemit-p1.c b/drivers/regulator/spacemit-p1.c
index 2b585ba01a93d..57e6e00a73fa4 100644
--- a/drivers/regulator/spacemit-p1.c
+++ b/drivers/regulator/spacemit-p1.c
@@ -87,13 +87,16 @@ static const struct linear_range p1_ldo_ranges[] = {
 	}
 
 #define P1_BUCK_DESC(_n) \
-	P1_REG_DESC(BUCK, buck, _n, "vin", 0x47, BUCK_MASK, 255, p1_buck_ranges)
+	P1_REG_DESC(BUCK, buck, _n, "vin" #_n, 0x47, BUCK_MASK, 255, p1_buck_ranges)
 
 #define P1_ALDO_DESC(_n) \
-	P1_REG_DESC(ALDO, aldo, _n, "vin", 0x5b, LDO_MASK, 128, p1_ldo_ranges)
+	P1_REG_DESC(ALDO, aldo, _n, "aldoin", 0x5b, LDO_MASK, 128, p1_ldo_ranges)
 
-#define P1_DLDO_DESC(_n) \
-	P1_REG_DESC(DLDO, dldo, _n, "buck5", 0x67, LDO_MASK, 128, p1_ldo_ranges)
+#define P1_DLDO1_DESC(_n) \
+	P1_REG_DESC(DLDO, dldo, _n, "dldoin1", 0x67, LDO_MASK, 128, p1_ldo_ranges)
+
+#define P1_DLDO2_DESC(_n) \
+	P1_REG_DESC(DLDO, dldo, _n, "dldoin2", 0x67, LDO_MASK, 128, p1_ldo_ranges)
 
 static const struct regulator_desc p1_regulator_desc[] = {
 	P1_BUCK_DESC(1),
@@ -108,13 +111,13 @@ static const struct regulator_desc p1_regulator_desc[] = {
 	P1_ALDO_DESC(3),
 	P1_ALDO_DESC(4),
 
-	P1_DLDO_DESC(1),
-	P1_DLDO_DESC(2),
-	P1_DLDO_DESC(3),
-	P1_DLDO_DESC(4),
-	P1_DLDO_DESC(5),
-	P1_DLDO_DESC(6),
-	P1_DLDO_DESC(7),
+	P1_DLDO1_DESC(1),
+	P1_DLDO1_DESC(2),
+	P1_DLDO1_DESC(3),
+	P1_DLDO1_DESC(4),
+	P1_DLDO2_DESC(5),
+	P1_DLDO2_DESC(6),
+	P1_DLDO2_DESC(7),
 };
 
 static int p1_regulator_probe(struct platform_device *pdev)
diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
index e418a2bf5d2ee..23fca7176539c 100644
--- a/drivers/remoteproc/da8xx_remoteproc.c
+++ b/drivers/remoteproc/da8xx_remoteproc.c
@@ -57,11 +57,10 @@ struct da8xx_rproc_mem {
  * @mem: internal memory regions data
  * @num_mems: number of internal memory regions
  * @dsp_clk: placeholder for platform's DSP clk
- * @ack_fxn: chip-specific ack function for ack'ing irq
+ * @dsp_reset: control for local reset
  * @irq_data: ack_fxn function parameter
  * @chipsig: virt ptr to DSP interrupt registers (CHIPSIG & CHIPSIG_CLR)
  * @bootreg: virt ptr to DSP boot address register (HOST1CFG)
- * @irq: irq # used by this instance
  */
 struct da8xx_rproc {
 	struct rproc *rproc;
@@ -69,11 +68,9 @@ struct da8xx_rproc {
 	int num_mems;
 	struct clk *dsp_clk;
 	struct reset_control *dsp_reset;
-	void (*ack_fxn)(struct irq_data *data);
 	struct irq_data *irq_data;
 	void __iomem *chipsig;
 	void __iomem *bootreg;
-	int irq;
 };
 
 /**
@@ -122,7 +119,7 @@ static irqreturn_t da8xx_rproc_callback(int irq, void *p)
 		 * we need to ack it after taking down the level else we'll
 		 * be called again immediately after returning.
 		 */
-		drproc->ack_fxn(drproc->irq_data);
+		drproc->irq_data->chip->irq_ack(drproc->irq_data);
 
 		return IRQ_WAKE_THREAD;
 	}
@@ -245,45 +242,9 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct da8xx_rproc *drproc;
 	struct rproc *rproc;
-	struct irq_data *irq_data;
-	struct clk *dsp_clk;
-	struct reset_control *dsp_reset;
-	void __iomem *chipsig;
-	void __iomem *bootreg;
 	int irq;
 	int ret;
 
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	irq_data = irq_get_irq_data(irq);
-	if (!irq_data)
-		return dev_err_probe(dev, -EINVAL, "irq_get_irq_data(%d): NULL\n", irq);
-
-	bootreg = devm_platform_ioremap_resource_byname(pdev, "host1cfg");
-	if (IS_ERR(bootreg))
-		return PTR_ERR(bootreg);
-
-	chipsig = devm_platform_ioremap_resource_byname(pdev, "chipsig");
-	if (IS_ERR(chipsig))
-		return PTR_ERR(chipsig);
-
-	dsp_clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(dsp_clk))
-		return dev_err_probe(dev, PTR_ERR(dsp_clk), "clk_get error\n");
-
-	dsp_reset = devm_reset_control_get_exclusive(dev, NULL);
-	if (IS_ERR(dsp_reset))
-		return dev_err_probe(dev, PTR_ERR(dsp_reset), "unable to get reset control\n");
-
-	if (dev->of_node) {
-		ret = of_reserved_mem_device_init(dev);
-		if (ret)
-			return dev_err_probe(dev, ret, "device does not have specific CMA pool\n");
-		devm_add_action_or_reset(&pdev->dev, da8xx_rproc_mem_release, &pdev->dev);
-	}
-
 	rproc = devm_rproc_alloc(dev, "dsp", &da8xx_rproc_ops, da8xx_fw_name,
 				 sizeof(*drproc));
 	if (!rproc)
@@ -294,43 +255,63 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 
 	drproc = rproc->priv;
 	drproc->rproc = rproc;
-	drproc->dsp_clk = dsp_clk;
-	drproc->dsp_reset = dsp_reset;
 	rproc->has_iommu = false;
 
+	drproc->dsp_clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(drproc->dsp_clk))
+		return dev_err_probe(dev, PTR_ERR(drproc->dsp_clk), "clk_get error\n");
+
+	drproc->dsp_reset = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(drproc->dsp_reset))
+		return dev_err_probe(dev, PTR_ERR(drproc->dsp_reset),
+				     "unable to get reset control\n");
+
+	if (dev->of_node) {
+		ret = of_reserved_mem_device_init(dev);
+		if (ret)
+			return dev_err_probe(dev, ret, "device does not have specific CMA pool\n");
+		devm_add_action_or_reset(&pdev->dev, da8xx_rproc_mem_release, &pdev->dev);
+	}
+
 	ret = da8xx_rproc_get_internal_memories(pdev, drproc);
 	if (ret)
 		return ret;
 
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	drproc->irq_data = irq_get_irq_data(irq);
+	if (!drproc->irq_data)
+		return dev_err_probe(dev, -EINVAL, "irq_get_irq_data(%d): NULL\n", irq);
+
+	drproc->chipsig = devm_platform_ioremap_resource_byname(pdev, "chipsig");
+	if (IS_ERR(drproc->chipsig))
+		return PTR_ERR(drproc->chipsig);
+
+	drproc->bootreg = devm_platform_ioremap_resource_byname(pdev, "host1cfg");
+	if (IS_ERR(drproc->bootreg))
+		return PTR_ERR(drproc->bootreg);
+
 	/* everything the ISR needs is now setup, so hook it up */
 	ret = devm_request_threaded_irq(dev, irq, da8xx_rproc_callback,
 					handle_event, 0, "da8xx-remoteproc",
 					rproc);
-	if (ret) {
-		dev_err(dev, "devm_request_threaded_irq error: %d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "devm_request_threaded_irq error\n");
 
 	/*
 	 * rproc_add() can end up enabling the DSP's clk with the DSP
 	 * *not* in reset, but da8xx_rproc_start() needs the DSP to be
 	 * held in reset at the time it is called.
 	 */
-	ret = reset_control_assert(dsp_reset);
+	ret = reset_control_assert(drproc->dsp_reset);
 	if (ret)
 		return ret;
 
-	drproc->chipsig = chipsig;
-	drproc->bootreg = bootreg;
-	drproc->ack_fxn = irq_data->chip->irq_ack;
-	drproc->irq_data = irq_data;
-	drproc->irq = irq;
-
 	ret = devm_rproc_add(dev, rproc);
-	if (ret) {
-		dev_err(dev, "rproc_add failed: %d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "rproc_add failed\n");
 
 	return 0;
 }
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 8c8ddbf995a46..0dd80e688b0ea 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -812,7 +812,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv,
 
 		/* Not use resource version, because we might share region */
 		priv->mem[b].cpu_addr = devm_ioremap_resource_wc(&pdev->dev, &res);
-		if (!priv->mem[b].cpu_addr) {
+		if (IS_ERR(priv->mem[b].cpu_addr)) {
 			dev_err(dev, "failed to remap %pr\n", &res);
 			return -ENOMEM;
 		}
@@ -1007,7 +1007,11 @@ static int imx_rproc_mmio_detect_mode(struct rproc *rproc)
 	}
 
 	priv->regmap = regmap;
-	regmap_attach_dev(dev, regmap, &config);
+	ret = regmap_attach_dev(dev, regmap, &config);
+	if (ret) {
+		dev_err(dev, "regmap attach failed\n");
+		return ret;
+	}
 
 	if (priv->gpr) {
 		ret = regmap_read(priv->gpr, dcfg->gpr_reg, &val);
diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c
index 4d6550b485675..e7fde55097866 100644
--- a/drivers/remoteproc/keystone_remoteproc.c
+++ b/drivers/remoteproc/keystone_remoteproc.c
@@ -173,35 +173,16 @@ static int keystone_rproc_start(struct rproc *rproc)
 
 	INIT_WORK(&ksproc->workqueue, handle_event);
 
-	ret = request_irq(ksproc->irq_ring, keystone_rproc_vring_interrupt, 0,
-			  dev_name(ksproc->dev), ksproc);
-	if (ret) {
-		dev_err(ksproc->dev, "failed to enable vring interrupt, ret = %d\n",
-			ret);
-		goto out;
-	}
+	enable_irq(ksproc->irq_ring);
+	enable_irq(ksproc->irq_fault);
 
-	ret = request_irq(ksproc->irq_fault, keystone_rproc_exception_interrupt,
-			  0, dev_name(ksproc->dev), ksproc);
+	ret = keystone_rproc_dsp_boot(ksproc, rproc->bootaddr);
 	if (ret) {
-		dev_err(ksproc->dev, "failed to enable exception interrupt, ret = %d\n",
-			ret);
-		goto free_vring_irq;
+		flush_work(&ksproc->workqueue);
+		return ret;
 	}
 
-	ret = keystone_rproc_dsp_boot(ksproc, rproc->bootaddr);
-	if (ret)
-		goto free_exc_irq;
-
 	return 0;
-
-free_exc_irq:
-	free_irq(ksproc->irq_fault, ksproc);
-free_vring_irq:
-	free_irq(ksproc->irq_ring, ksproc);
-	flush_work(&ksproc->workqueue);
-out:
-	return ret;
 }
 
 /*
@@ -215,8 +196,8 @@ static int keystone_rproc_stop(struct rproc *rproc)
 	struct keystone_rproc *ksproc = rproc->priv;
 
 	keystone_rproc_dsp_reset(ksproc);
-	free_irq(ksproc->irq_fault, ksproc);
-	free_irq(ksproc->irq_ring, ksproc);
+	disable_irq(ksproc->irq_fault);
+	disable_irq(ksproc->irq_ring);
 	flush_work(&ksproc->workqueue);
 
 	return 0;
@@ -427,10 +408,18 @@ static int keystone_rproc_probe(struct platform_device *pdev)
 	ksproc->irq_ring = platform_get_irq_byname(pdev, "vring");
 	if (ksproc->irq_ring < 0)
 		return ksproc->irq_ring;
+	ret = devm_request_irq(dev, ksproc->irq_ring, keystone_rproc_vring_interrupt,
+			       IRQF_NO_AUTOEN, dev_name(dev), ksproc);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to request vring interrupt\n");
 
 	ksproc->irq_fault = platform_get_irq_byname(pdev, "exception");
 	if (ksproc->irq_fault < 0)
 		return ksproc->irq_fault;
+	ret = devm_request_irq(dev, ksproc->irq_fault, keystone_rproc_exception_interrupt,
+			       IRQF_NO_AUTOEN, dev_name(dev), ksproc);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable exception interrupt\n");
 
 	ksproc->kick_gpio = devm_gpiod_get(dev, "kick", GPIOD_ASIS);
 	ret = PTR_ERR_OR_ZERO(ksproc->kick_gpio);
diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c
index 5e3eb7b86a0e3..a4636c7bc6b7b 100644
--- a/drivers/remoteproc/pru_rproc.c
+++ b/drivers/remoteproc/pru_rproc.c
@@ -1003,11 +1003,9 @@ static int pru_rproc_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENODEV;
 
-	ret = of_property_read_string(np, "firmware-name", &fw_name);
-	if (ret) {
-		dev_err(dev, "unable to retrieve firmware-name %d\n", ret);
-		return ret;
-	}
+	ret = rproc_of_parse_firmware(dev, 0, &fw_name);
+	if (ret)
+		return dev_err_probe(dev, ret, "unable to retrieve firmware-name\n");
 
 	rproc = devm_rproc_alloc(dev, pdev->name, &pru_rproc_ops, fw_name,
 				 sizeof(*pru));
@@ -1080,14 +1078,6 @@ static int pru_rproc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static void pru_rproc_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct rproc *rproc = platform_get_drvdata(pdev);
-
-	dev_dbg(dev, "%s: removing rproc %s\n", __func__, rproc->name);
-}
-
 static const struct pru_private_data pru_data = {
 	.type = PRU_TYPE_PRU,
 };
@@ -1133,7 +1123,6 @@ static struct platform_driver pru_rproc_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe  = pru_rproc_probe,
-	.remove = pru_rproc_remove,
 };
 module_platform_driver(pru_rproc_driver);
 
diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
index 91940977ca898..4e9eb5bd11fa1 100644
--- a/drivers/remoteproc/qcom_q6v5_mss.c
+++ b/drivers/remoteproc/qcom_q6v5_mss.c
@@ -124,6 +124,7 @@
 #define QDSP6v56_CLAMP_QMC_MEM		BIT(22)
 #define QDSP6SS_XO_CBCR		0x0038
 #define QDSP6SS_ACC_OVERRIDE_VAL		0x20
+#define QDSP6SS_ACC_OVERRIDE_VAL_9607	0x80800000
 #define QDSP6v55_BHS_EN_REST_ACK	BIT(0)
 
 /* QDSP6v65 parameters */
@@ -162,6 +163,7 @@ struct rproc_hexagon_res {
 	char **proxy_pd_names;
 	int version;
 	bool need_mem_protection;
+	bool need_pas_mem_setup;
 	bool has_alt_reset;
 	bool has_mba_logs;
 	bool has_spare_reg;
@@ -240,6 +242,7 @@ struct q6v5 {
 	struct qcom_sysmon *sysmon;
 	struct platform_device *bam_dmux;
 	bool need_mem_protection;
+	bool need_pas_mem_setup;
 	bool has_alt_reset;
 	bool has_mba_logs;
 	bool has_spare_reg;
@@ -254,10 +257,14 @@ struct q6v5 {
 };
 
 enum {
+	MSS_MDM9607,
 	MSS_MSM8226,
 	MSS_MSM8909,
 	MSS_MSM8916,
+	MSS_MSM8917,
 	MSS_MSM8926,
+	MSS_MSM8937,
+	MSS_MSM8940,
 	MSS_MSM8953,
 	MSS_MSM8974,
 	MSS_MSM8996,
@@ -745,15 +752,25 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 			return ret;
 		}
 		goto pbl_wait;
-	} else if (qproc->version == MSS_MSM8909 ||
+	} else if (qproc->version == MSS_MDM9607 ||
+		   qproc->version == MSS_MSM8909 ||
+		   qproc->version == MSS_MSM8917 ||
+		   qproc->version == MSS_MSM8937 ||
+		   qproc->version == MSS_MSM8940 ||
 		   qproc->version == MSS_MSM8953 ||
 		   qproc->version == MSS_MSM8996 ||
 		   qproc->version == MSS_MSM8998 ||
 		   qproc->version == MSS_SDM660) {
 
-		if (qproc->version != MSS_MSM8909 &&
-		    qproc->version != MSS_MSM8953)
-			/* Override the ACC value if required */
+		/* Override the ACC value if required */
+		if (qproc->version == MSS_MDM9607 ||
+		    qproc->version == MSS_MSM8917 ||
+		    qproc->version == MSS_MSM8937 ||
+		    qproc->version == MSS_MSM8940)
+			writel(QDSP6SS_ACC_OVERRIDE_VAL_9607,
+			       qproc->reg_base + QDSP6SS_STRAP_ACC);
+		else if (qproc->version != MSS_MSM8909 &&
+			 qproc->version != MSS_MSM8953)
 			writel(QDSP6SS_ACC_OVERRIDE_VAL,
 			       qproc->reg_base + QDSP6SS_STRAP_ACC);
 
@@ -799,6 +816,7 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 
 		if (qproc->version != MSS_MSM8909) {
 			int mem_pwr_ctl;
+			int reverse;
 
 			/* Deassert QDSP6 compiler memory clamp */
 			val = readl(qproc->reg_base + QDSP6SS_PWR_CTL_REG);
@@ -810,17 +828,37 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 			writel(val, qproc->reg_base + QDSP6SS_PWR_CTL_REG);
 
 			/* Turn on L1, L2, ETB and JU memories 1 at a time */
-			if (qproc->version == MSS_MSM8953 ||
+			if (qproc->version == MSS_MSM8940 ||
+			    qproc->version == MSS_MSM8953 ||
 			    qproc->version == MSS_MSM8996) {
 				mem_pwr_ctl = QDSP6SS_MEM_PWR_CTL;
 				i = 19;
+				reverse = 0;
+			} else if (qproc->version == MSS_MDM9607 ||
+				   qproc->version == MSS_MSM8917 ||
+				   qproc->version == MSS_MSM8937) {
+				mem_pwr_ctl = QDSP6SS_MEM_PWR_CTL;
+				i = 19;
+				/*
+				 * Set first 5 bits in reverse to avoid
+				 * "inrush current" issues.
+				 */
+				reverse = 6;
 			} else {
 				/* MSS_MSM8998, MSS_SDM660 */
 				mem_pwr_ctl = QDSP6V6SS_MEM_PWR_CTL;
 				i = 28;
+				reverse = 0;
 			}
+
 			val = readl(qproc->reg_base + mem_pwr_ctl);
-			for (; i >= 0; i--) {
+			for (; i >= reverse; i--) {
+				val |= BIT(i);
+				writel(val, qproc->reg_base + mem_pwr_ctl);
+				val = readl(qproc->reg_base + mem_pwr_ctl);
+				udelay(1);
+			}
+			for (i = 0; i < reverse; i++) {
 				val |= BIT(i);
 				writel(val, qproc->reg_base + mem_pwr_ctl);
 				/*
@@ -828,7 +866,7 @@ static int q6v5proc_reset(struct q6v5 *qproc)
 				 * wait for 1us for both memory peripheral and data
 				 * array to turn on.
 				 */
-				val |= readl(qproc->reg_base + mem_pwr_ctl);
+				val = readl(qproc->reg_base + mem_pwr_ctl);
 				udelay(1);
 			}
 		} else {
@@ -1441,7 +1479,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 			max_addr = ALIGN(phdr->p_paddr + phdr->p_memsz, SZ_4K);
 	}
 
-	if (qproc->version == MSS_MSM8953) {
+	if (qproc->need_pas_mem_setup) {
 		ret = qcom_scm_pas_mem_setup(MPSS_PAS_ID, qproc->mpss_phys, qproc->mpss_size);
 		if (ret) {
 			dev_err(qproc->dev,
@@ -2224,6 +2262,7 @@ static const struct rproc_hexagon_res sc7180_mss = {
 		NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = true,
 	.has_spare_reg = true,
@@ -2253,6 +2292,7 @@ static const struct rproc_hexagon_res sc7280_mss = {
 		NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = true,
 	.has_spare_reg = false,
@@ -2285,6 +2325,7 @@ static const struct rproc_hexagon_res sdm660_mss = {
 			NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2321,6 +2362,7 @@ static const struct rproc_hexagon_res sdm845_mss = {
 			NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = true,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2353,6 +2395,7 @@ static const struct rproc_hexagon_res msm8998_mss = {
 			NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2392,6 +2435,7 @@ static const struct rproc_hexagon_res msm8996_mss = {
 			NULL
 	},
 	.need_mem_protection = true,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2402,6 +2446,41 @@ static const struct rproc_hexagon_res msm8996_mss = {
 	.version = MSS_MSM8996,
 };
 
+static const struct rproc_hexagon_res mdm9607_mss = {
+	.hexagon_mba_image = "mba.mbn",
+	.proxy_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "pll",
+			.uA = 100000,
+		},
+		{}
+	},
+	.proxy_clk_names = (char*[]){
+		"xo",
+		NULL
+	},
+	.active_clk_names = (char*[]){
+		"iface",
+		"bus",
+		"mem",
+		NULL
+	},
+	.proxy_pd_names = (char*[]){
+		"mx",
+		"cx",
+		NULL
+	},
+	.need_mem_protection = false,
+	.has_alt_reset = false,
+	.has_mba_logs = false,
+	.has_spare_reg = false,
+	.has_qaccept_regs = false,
+	.has_ext_bhs_reg = false,
+	.has_ext_cntl_regs = false,
+	.has_vq6 = false,
+	.version = MSS_MDM9607,
+};
+
 static const struct rproc_hexagon_res msm8909_mss = {
 	.hexagon_mba_image = "mba.mbn",
 	.proxy_supply = (struct qcom_mss_reg_res[]) {
@@ -2427,6 +2506,7 @@ static const struct rproc_hexagon_res msm8909_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2473,6 +2553,7 @@ static const struct rproc_hexagon_res msm8916_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2483,6 +2564,138 @@ static const struct rproc_hexagon_res msm8916_mss = {
 	.version = MSS_MSM8916,
 };
 
+static const struct rproc_hexagon_res msm8917_mss = {
+	.hexagon_mba_image = "mba.mbn",
+	.proxy_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "pll",
+			.uA = 100000,
+		},
+		{}
+	},
+	.active_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "mss",
+			.uV = 1050000,
+			.uA = 100000,
+		},
+		{}
+	},
+	.proxy_clk_names = (char*[]){
+		"xo",
+		NULL
+	},
+	.active_clk_names = (char*[]){
+		"iface",
+		"bus",
+		"mem",
+		NULL
+	},
+	.proxy_pd_names = (char*[]) {
+		"cx",
+		"mx",
+		NULL
+	},
+	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
+	.has_alt_reset = false,
+	.has_mba_logs = false,
+	.has_spare_reg = false,
+	.has_qaccept_regs = false,
+	.has_ext_bhs_reg = false,
+	.has_ext_cntl_regs = false,
+	.has_vq6 = false,
+	.version = MSS_MSM8917,
+};
+
+static const struct rproc_hexagon_res msm8937_mss = {
+	.hexagon_mba_image = "mba.mbn",
+	.proxy_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "pll",
+			.uA = 100000,
+		},
+		{}
+	},
+	.active_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "mss",
+			.uV = 1050000,
+			.uA = 100000,
+		},
+		{}
+	},
+	.proxy_clk_names = (char*[]){
+		"xo",
+		NULL
+	},
+	.active_clk_names = (char*[]){
+		"iface",
+		"bus",
+		"mem",
+		NULL
+	},
+	.proxy_pd_names = (char*[]) {
+		"cx",
+		"mx",
+		NULL
+	},
+	.need_mem_protection = false,
+	.need_pas_mem_setup = true,
+	.has_alt_reset = false,
+	.has_mba_logs = false,
+	.has_spare_reg = false,
+	.has_qaccept_regs = false,
+	.has_ext_bhs_reg = false,
+	.has_ext_cntl_regs = false,
+	.has_vq6 = false,
+	.version = MSS_MSM8937,
+};
+
+static const struct rproc_hexagon_res msm8940_mss = {
+	.hexagon_mba_image = "mba.mbn",
+	.proxy_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "pll",
+			.uA = 100000,
+		},
+		{}
+	},
+	.active_supply = (struct qcom_mss_reg_res[]) {
+		{
+			.supply = "mss",
+			.uV = 1050000,
+			.uA = 100000,
+		},
+		{}
+	},
+	.proxy_clk_names = (char*[]){
+		"xo",
+		NULL
+	},
+	.active_clk_names = (char*[]){
+		"iface",
+		"bus",
+		"mem",
+		NULL
+	},
+	.proxy_pd_names = (char*[]) {
+		"cx",
+		"mx",
+		NULL
+	},
+	.need_mem_protection = false,
+	.need_pas_mem_setup = true,
+	.has_alt_reset = false,
+	.has_mba_logs = false,
+	.has_spare_reg = false,
+	.has_qaccept_regs = false,
+	.has_ext_bhs_reg = false,
+	.has_ext_cntl_regs = false,
+	.has_vq6 = false,
+	.version = MSS_MSM8940,
+};
+
 static const struct rproc_hexagon_res msm8953_mss = {
 	.hexagon_mba_image = "mba.mbn",
 	.proxy_supply = (struct qcom_mss_reg_res[]) {
@@ -2509,6 +2722,7 @@ static const struct rproc_hexagon_res msm8953_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = true,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2562,6 +2776,7 @@ static const struct rproc_hexagon_res msm8974_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2600,6 +2815,7 @@ static const struct rproc_hexagon_res msm8226_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2646,6 +2862,7 @@ static const struct rproc_hexagon_res msm8926_mss = {
 		NULL
 	},
 	.need_mem_protection = false,
+	.need_pas_mem_setup = false,
 	.has_alt_reset = false,
 	.has_mba_logs = false,
 	.has_spare_reg = false,
@@ -2658,10 +2875,14 @@ static const struct rproc_hexagon_res msm8926_mss = {
 
 static const struct of_device_id q6v5_of_match[] = {
 	{ .compatible = "qcom,q6v5-pil", .data = &msm8916_mss},
+	{ .compatible = "qcom,mdm9607-mss-pil", .data = &mdm9607_mss},
 	{ .compatible = "qcom,msm8226-mss-pil", .data = &msm8226_mss},
 	{ .compatible = "qcom,msm8909-mss-pil", .data = &msm8909_mss},
 	{ .compatible = "qcom,msm8916-mss-pil", .data = &msm8916_mss},
+	{ .compatible = "qcom,msm8917-mss-pil", .data = &msm8917_mss},
 	{ .compatible = "qcom,msm8926-mss-pil", .data = &msm8926_mss},
+	{ .compatible = "qcom,msm8937-mss-pil", .data = &msm8937_mss},
+	{ .compatible = "qcom,msm8940-mss-pil", .data = &msm8940_mss},
 	{ .compatible = "qcom,msm8953-mss-pil", .data = &msm8953_mss},
 	{ .compatible = "qcom,msm8974-mss-pil", .data = &msm8974_mss},
 	{ .compatible = "qcom,msm8996-mss-pil", .data = &msm8996_mss},
diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c
index 3ceec1fd6d998..913e3b750a869 100644
--- a/drivers/remoteproc/qcom_sysmon.c
+++ b/drivers/remoteproc/qcom_sysmon.c
@@ -677,7 +677,7 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc,
 		return ERR_PTR(ret);
 	}
 
-	qmi_add_lookup(&sysmon->qmi, 43, 0, 0);
+	qmi_add_lookup(&sysmon->qmi, QMI_SERVICE_ID_SSCTL, 0, 0);
 
 	sysmon->subdev.prepare = sysmon_prepare;
 	sysmon->subdev.start = sysmon_start;
diff --git a/drivers/remoteproc/ti_k3_common.c b/drivers/remoteproc/ti_k3_common.c
index 32aa954dc5be6..3cb8ae5d72f67 100644
--- a/drivers/remoteproc/ti_k3_common.c
+++ b/drivers/remoteproc/ti_k3_common.c
@@ -513,7 +513,7 @@ int k3_reserved_mem_init(struct k3_rproc *kproc)
 		kproc->rmem[i].dev_addr = (u32)res.start;
 		kproc->rmem[i].size = resource_size(&res);
 		kproc->rmem[i].cpu_addr = devm_ioremap_resource_wc(dev, &res);
-		if (!kproc->rmem[i].cpu_addr) {
+		if (IS_ERR(kproc->rmem[i].cpu_addr)) {
 			dev_err(dev, "failed to map reserved memory#%d at %pR\n",
 				i + 1, &res);
 			return -ENOMEM;
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
index b71ce69afe9f4..50a9974f3202e 100644
--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
+++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
@@ -232,17 +232,19 @@ static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg)
 
 	ipi = container_of(cl, struct mbox_info, mbox_cl);
 
-	/* copy data from ipi buffer to r5_core */
+	/* copy data from ipi buffer to r5_core if IPI is buffered. */
 	ipi_msg = (struct zynqmp_ipi_message *)msg;
-	buf_msg = (struct zynqmp_ipi_message *)ipi->rx_mc_buf;
-	len = ipi_msg->len;
-	if (len > IPI_BUF_LEN_MAX) {
-		dev_warn(cl->dev, "msg size exceeded than %d\n",
-			 IPI_BUF_LEN_MAX);
-		len = IPI_BUF_LEN_MAX;
+	if (ipi_msg) {
+		buf_msg = (struct zynqmp_ipi_message *)ipi->rx_mc_buf;
+		len = ipi_msg->len;
+		if (len > IPI_BUF_LEN_MAX) {
+			dev_warn(cl->dev, "msg size exceeded than %d\n",
+				 IPI_BUF_LEN_MAX);
+			len = IPI_BUF_LEN_MAX;
+		}
+		buf_msg->len = len;
+		memcpy(buf_msg->data, ipi_msg->data, len);
 	}
-	buf_msg->len = len;
-	memcpy(buf_msg->data, ipi_msg->data, len);
 
 	/* received and processed interrupt ack */
 	if (mbox_send_message(ipi->rx_chan, NULL) < 0)
@@ -265,6 +267,10 @@ static struct mbox_info *zynqmp_r5_setup_mbox(struct device *cdev)
 	struct mbox_client *mbox_cl;
 	struct mbox_info *ipi;
 
+	if (!of_property_present(dev_of_node(cdev), "mboxes") ||
+	    !of_property_present(dev_of_node(cdev), "mbox-names"))
+		return NULL;
+
 	ipi = kzalloc_obj(*ipi);
 	if (!ipi)
 		return NULL;
@@ -1005,7 +1011,7 @@ static int zynqmp_r5_get_sram_banks(struct zynqmp_r5_core *r5_core)
 		}
 
 		/* Get SRAM device address */
-		ret = of_property_read_reg(sram_np, i, &abs_addr, &size);
+		ret = of_property_read_reg(sram_np, 0, &abs_addr, &size);
 		if (ret) {
 			dev_err(dev, "failed to get reg property\n");
 			goto fail_sram_get;
@@ -1484,6 +1490,8 @@ static void zynqmp_r5_remoteproc_shutdown(struct platform_device *pdev)
 			dev_err(cluster->dev, "failed to %s rproc %d\n",
 				rproc_state_str, rproc->index);
 		}
+
+		zynqmp_r5_free_mbox(r5_core->ipi);
 	}
 }
 
diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c
index 1eebc26021871..0666be6b0e88d 100644
--- a/drivers/resctrl/mpam_devices.c
+++ b/drivers/resctrl/mpam_devices.c
@@ -1428,6 +1428,7 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
 static int mpam_restore_mbwu_state(void *_ris)
 {
 	int i;
+	u64 val;
 	struct mon_read mwbu_arg;
 	struct mpam_msc_ris *ris = _ris;
 	struct mpam_class *class = ris->vmsc->comp->class;
@@ -1437,6 +1438,7 @@ static int mpam_restore_mbwu_state(void *_ris)
 			mwbu_arg.ris = ris;
 			mwbu_arg.ctx = &ris->mbwu_state[i].cfg;
 			mwbu_arg.type = mpam_msmon_choose_counter(class);
+			mwbu_arg.val = &val;
 
 			__ris_msmon_read(&mwbu_arg);
 		}
diff --git a/drivers/resctrl/test_mpam_devices.c b/drivers/resctrl/test_mpam_devices.c
index 3e8d564a0c647..31871f5197299 100644
--- a/drivers/resctrl/test_mpam_devices.c
+++ b/drivers/resctrl/test_mpam_devices.c
@@ -322,9 +322,17 @@ static void test_mpam_enable_merge_features(struct kunit *test)
 	mutex_unlock(&mpam_list_lock);
 }
 
+static void __test_mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd)
+{
+	/* Avoid warnings when running with CONFIG_DEBUG_PREEMPT */
+	guard(preempt)();
+
+	mpam_reset_msc_bitmap(msc, reg, wd);
+}
+
 static void test_mpam_reset_msc_bitmap(struct kunit *test)
 {
-	char __iomem *buf = kunit_kzalloc(test, SZ_16K, GFP_KERNEL);
+	char __iomem *buf = (__force char __iomem *)kunit_kzalloc(test, SZ_16K, GFP_KERNEL);
 	struct mpam_msc fake_msc = {};
 	u32 *test_result;
 
@@ -339,33 +347,33 @@ static void test_mpam_reset_msc_bitmap(struct kunit *test)
 	mutex_init(&fake_msc.part_sel_lock);
 	mutex_lock(&fake_msc.part_sel_lock);
 
-	test_result = (u32 *)(buf + MPAMCFG_CPBM);
+	test_result = (__force u32 *)(buf + MPAMCFG_CPBM);
 
-	mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 0);
+	__test_mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 0);
 	KUNIT_EXPECT_EQ(test, test_result[0], 0);
 	KUNIT_EXPECT_EQ(test, test_result[1], 0);
 	test_result[0] = 0;
 	test_result[1] = 0;
 
-	mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 1);
+	__test_mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 1);
 	KUNIT_EXPECT_EQ(test, test_result[0], 1);
 	KUNIT_EXPECT_EQ(test, test_result[1], 0);
 	test_result[0] = 0;
 	test_result[1] = 0;
 
-	mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 16);
+	__test_mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 16);
 	KUNIT_EXPECT_EQ(test, test_result[0], 0xffff);
 	KUNIT_EXPECT_EQ(test, test_result[1], 0);
 	test_result[0] = 0;
 	test_result[1] = 0;
 
-	mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 32);
+	__test_mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 32);
 	KUNIT_EXPECT_EQ(test, test_result[0], 0xffffffff);
 	KUNIT_EXPECT_EQ(test, test_result[1], 0);
 	test_result[0] = 0;
 	test_result[1] = 0;
 
-	mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 33);
+	__test_mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 33);
 	KUNIT_EXPECT_EQ(test, test_result[0], 0xffffffff);
 	KUNIT_EXPECT_EQ(test, test_result[1], 1);
 	test_result[0] = 0;
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 7ce151f6a7e4d..5165006be6931 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -291,6 +291,13 @@ config RESET_SIMPLE
 	   - SiFive FU740 SoCs
 	   - Sophgo SoCs
 
+config RESET_SKY1
+	bool "Cix Sky1 reset controller"
+	depends on ARCH_CIX || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  This enables the reset controller for Cix Sky1.
+
 config RESET_SOCFPGA
 	bool "SoCFPGA Reset Driver" if COMPILE_TEST && (!ARM || !ARCH_INTEL_SOCFPGA)
 	default ARM && ARCH_INTEL_SOCFPGA
@@ -315,6 +322,17 @@ config RESET_SUNXI
 	help
 	  This enables the reset driver for Allwinner SoCs.
 
+config RESET_TENSTORRENT_ATLANTIS
+	tristate "Tenstorrent atlantis reset driver"
+	depends on ARCH_TENSTORRENT || COMPILE_TEST
+	select AUXILIARY_BUS
+	default ARCH_TENSTORRENT
+	help
+	  This enables the driver for the reset controller
+	  present in the Tenstorrent Atlantis SoC.
+	  Enable this option to be able to use hardware
+	  resets on Atalantis based systems.
+
 config RESET_TH1520
 	tristate "T-HEAD TH1520 reset controller"
 	depends on ARCH_THEAD || COMPILE_TEST
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index fc0cc99f8514c..3e52569bd2768 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -38,9 +38,11 @@ obj-$(CONFIG_RESET_RZG2L_USBPHY_CTRL) += reset-rzg2l-usbphy-ctrl.o
 obj-$(CONFIG_RESET_RZV2H_USB2PHY) += reset-rzv2h-usb2phy.o
 obj-$(CONFIG_RESET_SCMI) += reset-scmi.o
 obj-$(CONFIG_RESET_SIMPLE) += reset-simple.o
+obj-$(CONFIG_RESET_SKY1) += reset-sky1.o
 obj-$(CONFIG_RESET_SOCFPGA) += reset-socfpga.o
 obj-$(CONFIG_RESET_SUNPLUS) += reset-sunplus.o
 obj-$(CONFIG_RESET_SUNXI) += reset-sunxi.o
+obj-$(CONFIG_RESET_TENSTORRENT_ATLANTIS) += reset-tenstorrent-atlantis.o
 obj-$(CONFIG_RESET_TH1520) += reset-th1520.o
 obj-$(CONFIG_RESET_TI_SCI) += reset-ti-sci.o
 obj-$(CONFIG_RESET_TI_SYSCON) += reset-ti-syscon.o
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index fceec45c8afc1..a0d8c68f2afc2 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -12,6 +12,7 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/export.h>
+#include <linux/fwnode.h>
 #include <linux/gpio/driver.h>
 #include <linux/gpio/machine.h>
 #include <linux/gpio/property.h>
@@ -20,9 +21,11 @@
 #include <linux/kref.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/property.h>
 #include <linux/reset.h>
 #include <linux/reset-controller.h>
 #include <linux/slab.h>
+#include <linux/srcu.h>
 
 static DEFINE_MUTEX(reset_list_mutex);
 static LIST_HEAD(reset_controller_list);
@@ -36,6 +39,7 @@ static DEFINE_IDA(reset_gpio_ida);
  * struct reset_control - a reset control
  * @rcdev: a pointer to the reset controller device
  *         this reset control belongs to
+ * @srcu: protects the rcdev pointer from removal during consumer access
  * @list: list entry for the rcdev's reset controller list
  * @id: ID of the reset controller in the reset
  *      controller device
@@ -47,9 +51,11 @@ static DEFINE_IDA(reset_gpio_ida);
  * @triggered_count: Number of times this reset line has been reset. Currently
  *                   only used for shared resets, which means that the value
  *                   will be either 0 or 1.
+ * @lock: serializes the internals of reset_control_acquire()
  */
 struct reset_control {
-	struct reset_controller_dev *rcdev;
+	struct reset_controller_dev __rcu *rcdev;
+	struct srcu_struct srcu;
 	struct list_head list;
 	unsigned int id;
 	struct kref refcnt;
@@ -58,6 +64,7 @@ struct reset_control {
 	bool array;
 	atomic_t deassert_count;
 	atomic_t triggered_count;
+	struct mutex lock;
 };
 
 /**
@@ -74,14 +81,16 @@ struct reset_control_array {
 
 /**
  * struct reset_gpio_lookup - lookup key for ad-hoc created reset-gpio devices
- * @of_args: phandle to the reset controller with all the args like GPIO number
+ * @ref_args: Reference to the reset controller with all the args like GPIO number
  * @swnode: Software node containing the reference to the GPIO provider
  * @list: list entry for the reset_gpio_lookup_list
+ * @adev: Auxiliary device representing the reset controller
  */
 struct reset_gpio_lookup {
-	struct of_phandle_args of_args;
+	struct fwnode_reference_args ref_args;
 	struct fwnode_handle *swnode;
 	struct list_head list;
+	struct auxiliary_device adev;
 };
 
 static const char *rcdev_name(struct reset_controller_dev *rcdev)
@@ -89,27 +98,24 @@ static const char *rcdev_name(struct reset_controller_dev *rcdev)
 	if (rcdev->dev)
 		return dev_name(rcdev->dev);
 
-	if (rcdev->of_node)
-		return rcdev->of_node->full_name;
-
-	if (rcdev->of_args)
-		return rcdev->of_args->np->full_name;
+	if (rcdev->fwnode)
+		return fwnode_get_name(rcdev->fwnode);
 
 	return NULL;
 }
 
 /**
- * of_reset_simple_xlate - translate reset_spec to the reset line number
+ * fwnode_reset_simple_xlate - translate reset_spec to the reset line number
  * @rcdev: a pointer to the reset controller device
- * @reset_spec: reset line specifier as found in the device tree
+ * @reset_spec: reset line specifier as found in firmware
  *
- * This static translation function is used by default if of_xlate in
- * :c:type:`reset_controller_dev` is not set. It is useful for all reset
- * controllers with 1:1 mapping, where reset lines can be indexed by number
- * without gaps.
+ * This static translation function is used by default if neither fwnode_xlate
+ * not of_xlate in :c:type:`reset_controller_dev` is not set. It is useful for
+ * all reset controllers with 1:1 mapping, where reset lines can be indexed by
+ * number without gaps.
  */
-static int of_reset_simple_xlate(struct reset_controller_dev *rcdev,
-				 const struct of_phandle_args *reset_spec)
+static int fwnode_reset_simple_xlate(struct reset_controller_dev *rcdev,
+				     const struct fwnode_reference_args *reset_spec)
 {
 	if (reset_spec->args[0] >= rcdev->nr_resets)
 		return -EINVAL;
@@ -123,33 +129,71 @@ static int of_reset_simple_xlate(struct reset_controller_dev *rcdev,
  */
 int reset_controller_register(struct reset_controller_dev *rcdev)
 {
-	if (rcdev->of_node && rcdev->of_args)
+	if ((rcdev->of_node && rcdev->fwnode) || (rcdev->of_xlate && rcdev->fwnode_xlate))
 		return -EINVAL;
 
-	if (!rcdev->of_xlate) {
-		rcdev->of_reset_n_cells = 1;
-		rcdev->of_xlate = of_reset_simple_xlate;
+	if (rcdev->of_node && !rcdev->fwnode)
+		rcdev->fwnode = of_fwnode_handle(rcdev->of_node);
+
+	if (!rcdev->fwnode) {
+		rcdev->fwnode = dev_fwnode(rcdev->dev);
+		if (!rcdev->fwnode)
+			return -EINVAL;
+	}
+
+	if (rcdev->of_xlate)
+		rcdev->fwnode_reset_n_cells = rcdev->of_reset_n_cells;
+
+	if (!rcdev->fwnode_xlate && !rcdev->of_xlate) {
+		rcdev->fwnode_xlate = fwnode_reset_simple_xlate;
+		rcdev->fwnode_reset_n_cells = 1;
 	}
 
 	INIT_LIST_HEAD(&rcdev->reset_control_head);
+	mutex_init(&rcdev->lock);
+
+	guard(mutex)(&reset_list_mutex);
 
-	mutex_lock(&reset_list_mutex);
 	list_add(&rcdev->list, &reset_controller_list);
-	mutex_unlock(&reset_list_mutex);
 
 	return 0;
 }
 EXPORT_SYMBOL_GPL(reset_controller_register);
 
+static void reset_controller_remove(struct reset_controller_dev *rcdev,
+				    struct reset_control *rstc)
+{
+	lockdep_assert_held(&rcdev->lock);
+
+	list_del(&rstc->list);
+	module_put(rcdev->owner);
+	put_device(rcdev->dev);
+}
+
 /**
  * reset_controller_unregister - unregister a reset controller device
  * @rcdev: a pointer to the reset controller device
  */
 void reset_controller_unregister(struct reset_controller_dev *rcdev)
 {
-	mutex_lock(&reset_list_mutex);
-	list_del(&rcdev->list);
-	mutex_unlock(&reset_list_mutex);
+	struct reset_control *rstc, *pos;
+
+	scoped_guard(mutex, &reset_list_mutex)
+		list_del(&rcdev->list);
+
+	scoped_guard(mutex, &rcdev->lock) {
+		/*
+		 * Numb but don't free the remaining reset control handles that are
+		 * still held by consumers.
+		 */
+		list_for_each_entry_safe(rstc, pos, &rcdev->reset_control_head, list) {
+			rcu_assign_pointer(rstc->rcdev, NULL);
+			synchronize_srcu(&rstc->srcu);
+			reset_controller_remove(rcdev, rstc);
+		}
+	}
+
+	mutex_destroy(&rcdev->lock);
 }
 EXPORT_SYMBOL_GPL(reset_controller_unregister);
 
@@ -326,6 +370,7 @@ static inline bool reset_control_is_array(struct reset_control *rstc)
  */
 int reset_control_reset(struct reset_control *rstc)
 {
+	struct reset_controller_dev *rcdev;
 	int ret;
 
 	if (!rstc)
@@ -337,7 +382,13 @@ int reset_control_reset(struct reset_control *rstc)
 	if (reset_control_is_array(rstc))
 		return reset_control_array_reset(rstc_to_array(rstc));
 
-	if (!rstc->rcdev->ops->reset)
+	guard(srcu)(&rstc->srcu);
+
+	rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+	if (!rcdev)
+		return -ENODEV;
+
+	if (!rcdev->ops->reset)
 		return -ENOTSUPP;
 
 	if (rstc->shared) {
@@ -351,7 +402,7 @@ int reset_control_reset(struct reset_control *rstc)
 			return -EPERM;
 	}
 
-	ret = rstc->rcdev->ops->reset(rstc->rcdev, rstc->id);
+	ret = rcdev->ops->reset(rcdev, rstc->id);
 	if (rstc->shared && ret)
 		atomic_dec(&rstc->triggered_count);
 
@@ -441,6 +492,8 @@ EXPORT_SYMBOL_GPL(reset_control_rearm);
  */
 int reset_control_assert(struct reset_control *rstc)
 {
+	struct reset_controller_dev *rcdev;
+
 	if (!rstc)
 		return 0;
 
@@ -450,6 +503,12 @@ int reset_control_assert(struct reset_control *rstc)
 	if (reset_control_is_array(rstc))
 		return reset_control_array_assert(rstc_to_array(rstc));
 
+	guard(srcu)(&rstc->srcu);
+
+	rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+	if (!rcdev)
+		return -ENODEV;
+
 	if (rstc->shared) {
 		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
 			return -EINVAL;
@@ -464,7 +523,7 @@ int reset_control_assert(struct reset_control *rstc)
 		 * Shared reset controls allow the reset line to be in any state
 		 * after this call, so doing nothing is a valid option.
 		 */
-		if (!rstc->rcdev->ops->assert)
+		if (!rcdev->ops->assert)
 			return 0;
 	} else {
 		/*
@@ -472,17 +531,17 @@ int reset_control_assert(struct reset_control *rstc)
 		 * is no way to guarantee that the reset line is asserted after
 		 * this call.
 		 */
-		if (!rstc->rcdev->ops->assert)
+		if (!rcdev->ops->assert)
 			return -ENOTSUPP;
 
 		if (!rstc->acquired) {
 			WARN(1, "reset %s (ID: %u) is not acquired\n",
-			     rcdev_name(rstc->rcdev), rstc->id);
+			     rcdev_name(rcdev), rstc->id);
 			return -EPERM;
 		}
 	}
 
-	return rstc->rcdev->ops->assert(rstc->rcdev, rstc->id);
+	return rcdev->ops->assert(rcdev, rstc->id);
 }
 EXPORT_SYMBOL_GPL(reset_control_assert);
 
@@ -529,6 +588,8 @@ EXPORT_SYMBOL_GPL(reset_control_bulk_assert);
  */
 int reset_control_deassert(struct reset_control *rstc)
 {
+	struct reset_controller_dev *rcdev;
+
 	if (!rstc)
 		return 0;
 
@@ -538,6 +599,12 @@ int reset_control_deassert(struct reset_control *rstc)
 	if (reset_control_is_array(rstc))
 		return reset_control_array_deassert(rstc_to_array(rstc));
 
+	guard(srcu)(&rstc->srcu);
+
+	rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+	if (!rcdev)
+		return -ENODEV;
+
 	if (rstc->shared) {
 		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
 			return -EINVAL;
@@ -547,7 +614,7 @@ int reset_control_deassert(struct reset_control *rstc)
 	} else {
 		if (!rstc->acquired) {
 			WARN(1, "reset %s (ID: %u) is not acquired\n",
-			     rcdev_name(rstc->rcdev), rstc->id);
+			     rcdev_name(rcdev), rstc->id);
 			return -EPERM;
 		}
 	}
@@ -559,10 +626,10 @@ int reset_control_deassert(struct reset_control *rstc)
 	 * case, the reset controller driver should implement .deassert() and
 	 * return -ENOTSUPP.
 	 */
-	if (!rstc->rcdev->ops->deassert)
+	if (!rcdev->ops->deassert)
 		return 0;
 
-	return rstc->rcdev->ops->deassert(rstc->rcdev, rstc->id);
+	return rcdev->ops->deassert(rcdev, rstc->id);
 }
 EXPORT_SYMBOL_GPL(reset_control_deassert);
 
@@ -604,14 +671,22 @@ EXPORT_SYMBOL_GPL(reset_control_bulk_deassert);
  */
 int reset_control_status(struct reset_control *rstc)
 {
+	struct reset_controller_dev *rcdev;
+
 	if (!rstc)
 		return 0;
 
 	if (WARN_ON(IS_ERR(rstc)) || reset_control_is_array(rstc))
 		return -EINVAL;
 
-	if (rstc->rcdev->ops->status)
-		return rstc->rcdev->ops->status(rstc->rcdev, rstc->id);
+	guard(srcu)(&rstc->srcu);
+
+	rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+	if (!rcdev)
+		return -ENODEV;
+
+	if (rcdev->ops->status)
+		return rcdev->ops->status(rcdev, rstc->id);
 
 	return -ENOTSUPP;
 }
@@ -639,6 +714,7 @@ EXPORT_SYMBOL_GPL(reset_control_status);
  */
 int reset_control_acquire(struct reset_control *rstc)
 {
+	struct reset_controller_dev *rcdev;
 	struct reset_control *rc;
 
 	if (!rstc)
@@ -650,25 +726,28 @@ int reset_control_acquire(struct reset_control *rstc)
 	if (reset_control_is_array(rstc))
 		return reset_control_array_acquire(rstc_to_array(rstc));
 
-	mutex_lock(&reset_list_mutex);
+	guard(mutex)(&rstc->lock);
 
-	if (rstc->acquired) {
-		mutex_unlock(&reset_list_mutex);
+	if (rstc->acquired)
 		return 0;
-	}
 
-	list_for_each_entry(rc, &rstc->rcdev->reset_control_head, list) {
-		if (rstc != rc && rstc->id == rc->id) {
-			if (rc->acquired) {
-				mutex_unlock(&reset_list_mutex);
-				return -EBUSY;
+	guard(srcu)(&rstc->srcu);
+
+	rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+	if (!rcdev)
+		return -ENODEV;
+
+	scoped_guard(mutex, &rcdev->lock) {
+		list_for_each_entry(rc, &rcdev->reset_control_head, list) {
+			if (rstc != rc && rstc->id == rc->id) {
+				if (rc->acquired)
+					return -EBUSY;
 			}
 		}
 	}
 
 	rstc->acquired = true;
 
-	mutex_unlock(&reset_list_mutex);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(reset_control_acquire);
@@ -752,8 +831,9 @@ __reset_control_get_internal(struct reset_controller_dev *rcdev,
 	bool shared = flags & RESET_CONTROL_FLAGS_BIT_SHARED;
 	bool acquired = flags & RESET_CONTROL_FLAGS_BIT_ACQUIRED;
 	struct reset_control *rstc;
+	int ret;
 
-	lockdep_assert_held(&reset_list_mutex);
+	lockdep_assert_held(&rcdev->lock);
 
 	/* Expect callers to filter out OPTIONAL and DEASSERTED bits */
 	if (WARN_ON(flags & ~(RESET_CONTROL_FLAGS_BIT_SHARED |
@@ -782,15 +862,23 @@ __reset_control_get_internal(struct reset_controller_dev *rcdev,
 	if (!rstc)
 		return ERR_PTR(-ENOMEM);
 
+	ret = init_srcu_struct(&rstc->srcu);
+	if (ret) {
+		kfree(rstc);
+		return ERR_PTR(ret);
+	}
+
 	if (!try_module_get(rcdev->owner)) {
+		cleanup_srcu_struct(&rstc->srcu);
 		kfree(rstc);
 		return ERR_PTR(-ENODEV);
 	}
 
-	rstc->rcdev = rcdev;
+	rcu_assign_pointer(rstc->rcdev, rcdev);
 	list_add(&rstc->list, &rcdev->reset_control_head);
 	rstc->id = index;
 	kref_init(&rstc->refcnt);
+	mutex_init(&rstc->lock);
 	rstc->acquired = acquired;
 	rstc->shared = shared;
 	get_device(rcdev->dev);
@@ -802,77 +890,133 @@ static void __reset_control_release(struct kref *kref)
 {
 	struct reset_control *rstc = container_of(kref, struct reset_control,
 						  refcnt);
+	struct reset_controller_dev *rcdev;
 
-	lockdep_assert_held(&reset_list_mutex);
+	lockdep_assert_held(&rstc->srcu);
 
-	module_put(rstc->rcdev->owner);
+	rcdev = rcu_replace_pointer(rstc->rcdev, NULL, true);
+	if (rcdev) {
+		lockdep_assert_held(&rcdev->lock);
+		reset_controller_remove(rcdev, rstc);
+	}
 
-	list_del(&rstc->list);
-	put_device(rstc->rcdev->dev);
-	kfree(rstc);
+	mutex_destroy(&rstc->lock);
 }
 
-static void __reset_control_put_internal(struct reset_control *rstc)
+static void reset_control_put_internal(struct reset_control *rstc)
 {
-	lockdep_assert_held(&reset_list_mutex);
+	struct reset_controller_dev *rcdev;
+	int ret = 0;
 
 	if (IS_ERR_OR_NULL(rstc))
 		return;
 
-	kref_put(&rstc->refcnt, __reset_control_release);
+	scoped_guard(srcu, &rstc->srcu) {
+		rcdev = srcu_dereference(rstc->rcdev, &rstc->srcu);
+		if (!rcdev)
+			/* Already released. */
+			return;
+
+		guard(mutex)(&rcdev->lock);
+		ret = kref_put(&rstc->refcnt, __reset_control_release);
+	}
+
+	if (ret) {
+		synchronize_srcu(&rstc->srcu);
+		cleanup_srcu_struct(&rstc->srcu);
+		kfree(rstc);
+	}
 }
 
 static void reset_gpio_aux_device_release(struct device *dev)
 {
-	struct auxiliary_device *adev = to_auxiliary_dev(dev);
-
-	kfree(adev);
+	WARN(1, "reset-gpio device %s should never have been removed", dev_name(dev));
 }
 
-static int reset_add_gpio_aux_device(struct device *parent,
-				     struct fwnode_handle *swnode,
-				     int id, void *pdata)
+static int reset_create_gpio_aux_device(struct reset_gpio_lookup *rgpio_dev,
+					struct device *parent)
 {
-	struct auxiliary_device *adev;
-	int ret;
+	struct auxiliary_device *adev = &rgpio_dev->adev;
+	int ret, id;
 
-	adev = kzalloc_obj(*adev);
-	if (!adev)
+	id = ida_alloc(&reset_gpio_ida, GFP_KERNEL);
+	if (id < 0)
 		return -ENOMEM;
 
 	adev->id = id;
 	adev->name = "gpio";
 	adev->dev.parent = parent;
-	adev->dev.platform_data = pdata;
+	adev->dev.platform_data = &rgpio_dev->ref_args;
 	adev->dev.release = reset_gpio_aux_device_release;
-	device_set_node(&adev->dev, swnode);
+	device_set_node(&adev->dev, rgpio_dev->swnode);
 
 	ret = auxiliary_device_init(adev);
 	if (ret) {
-		kfree(adev);
+		ida_free(&reset_gpio_ida, id);
 		return ret;
 	}
 
 	ret = __auxiliary_device_add(adev, "reset");
 	if (ret) {
 		auxiliary_device_uninit(adev);
-		kfree(adev);
+		ida_free(&reset_gpio_ida, id);
 		return ret;
 	}
 
-	return ret;
+	return 0;
+}
+
+static void reset_gpio_add_devlink(struct fwnode_handle *fwnode,
+				   struct reset_gpio_lookup *rgpio_dev)
+{
+	struct device *consumer;
+
+	/*
+	 * We must use get_dev_from_fwnode() and not ref_find_device_by_node()
+	 * because the latter only considers the platform bus while we want to
+	 * get consumers of any kind that can be associated with firmware
+	 * nodes: auxiliary, soundwire, etc.
+	 */
+	consumer = get_dev_from_fwnode(fwnode);
+	if (consumer) {
+		if (!device_link_add(consumer, &rgpio_dev->adev.dev,
+				     DL_FLAG_AUTOREMOVE_CONSUMER))
+			pr_warn("Failed to create a device link between reset-gpio and its consumer");
+
+		put_device(consumer);
+	}
+	/*
+	 * else { }
+	 *
+	 * TODO: If ever there's a case where we need to support shared
+	 * reset-gpios retrieved from a device node for which there's no
+	 * device present yet, this is where we'd set up a notifier waiting
+	 * for the device to appear in the system. This would be a lot of code
+	 * that would go unused for now so let's cross that bridge when and if
+	 * we get there.
+	 */
+}
+
+/* TODO: move it out into drivers/base/ */
+static bool fwnode_reference_args_equal(const struct fwnode_reference_args *left,
+					const struct fwnode_reference_args *right)
+{
+	return left->fwnode == right->fwnode && left->nargs == right->nargs &&
+	       !memcmp(left->args, right->args, sizeof(left->args[0]) * left->nargs);
 }
 
 /*
- * @args:	phandle to the GPIO provider with all the args like GPIO number
+ * @np: OF-node associated with the consumer
+ * @args: Reference to the GPIO provider with all the args like GPIO number
  */
-static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
+static int __reset_add_reset_gpio_device(struct fwnode_handle *fwnode,
+					 const struct fwnode_reference_args *args)
 {
 	struct property_entry properties[3] = { };
-	unsigned int offset, of_flags, lflags;
+	unsigned int offset, flags, lflags;
 	struct reset_gpio_lookup *rgpio_dev;
 	struct device *parent;
-	int id, ret, prop = 0;
+	int ret, prop = 0;
 
 	/*
 	 * Currently only #gpio-cells=2 is supported with the meaning of:
@@ -880,7 +1024,7 @@ static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
 	 * args[1]: GPIO flags
 	 * TODO: Handle other cases.
 	 */
-	if (args->args_count != 2)
+	if (args->nargs != 2)
 		return -ENOENT;
 
 	/*
@@ -891,7 +1035,7 @@ static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
 	lockdep_assert_not_held(&reset_list_mutex);
 
 	offset = args->args[0];
-	of_flags = args->args[1];
+	flags = args->args[1];
 
 	/*
 	 * Later we map GPIO flags between OF and Linux, however not all
@@ -901,90 +1045,89 @@ static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
 	 * FIXME: Find a better way of translating OF flags to GPIO lookup
 	 * flags.
 	 */
-	if (of_flags > GPIO_ACTIVE_LOW) {
+	if (flags > GPIO_ACTIVE_LOW) {
 		pr_err("reset-gpio code does not support GPIO flags %u for GPIO %u\n",
-		       of_flags, offset);
+		       flags, offset);
 		return -EINVAL;
 	}
 
 	struct gpio_device *gdev __free(gpio_device_put) =
-		gpio_device_find_by_fwnode(of_fwnode_handle(args->np));
+			gpio_device_find_by_fwnode(args->fwnode);
 	if (!gdev)
 		return -EPROBE_DEFER;
 
 	guard(mutex)(&reset_gpio_lookup_mutex);
 
 	list_for_each_entry(rgpio_dev, &reset_gpio_lookup_list, list) {
-		if (args->np == rgpio_dev->of_args.np) {
-			if (of_phandle_args_equal(args, &rgpio_dev->of_args))
-				return 0; /* Already on the list, done */
+		if (fwnode_reference_args_equal(args, &rgpio_dev->ref_args)) {
+			/*
+			 * Already on the list, create the device link
+			 * and stop here.
+			 */
+			reset_gpio_add_devlink(fwnode, rgpio_dev);
+			return 0;
 		}
 	}
 
-	lflags = GPIO_PERSISTENT | (of_flags & GPIO_ACTIVE_LOW);
+	lflags = GPIO_PERSISTENT | (flags & GPIO_ACTIVE_LOW);
 	parent = gpio_device_to_device(gdev);
 	properties[prop++] = PROPERTY_ENTRY_STRING("compatible", "reset-gpio");
 	properties[prop++] = PROPERTY_ENTRY_GPIO("reset-gpios", parent->fwnode, offset, lflags);
 
-	id = ida_alloc(&reset_gpio_ida, GFP_KERNEL);
-	if (id < 0)
-		return id;
-
 	/* Not freed on success, because it is persisent subsystem data. */
 	rgpio_dev = kzalloc_obj(*rgpio_dev);
-	if (!rgpio_dev) {
-		ret = -ENOMEM;
-		goto err_ida_free;
-	}
+	if (!rgpio_dev)
+		return -ENOMEM;
 
-	rgpio_dev->of_args = *args;
+	rgpio_dev->ref_args = *args;
 	/*
-	 * We keep the device_node reference, but of_args.np is put at the end
-	 * of __of_reset_control_get(), so get it one more time.
+	 * We keep the fwnode_handle reference, but ref_args.fwnode is put at
+	 * the end of __fwnode_reset_control_get(), so get it one more time.
 	 * Hold reference as long as rgpio_dev memory is valid.
 	 */
-	of_node_get(rgpio_dev->of_args.np);
+	fwnode_handle_get(rgpio_dev->ref_args.fwnode);
 
 	rgpio_dev->swnode = fwnode_create_software_node(properties, NULL);
 	if (IS_ERR(rgpio_dev->swnode)) {
 		ret = PTR_ERR(rgpio_dev->swnode);
-		goto err_put_of_node;
+		goto err_put_fwnode;
 	}
 
-	ret = reset_add_gpio_aux_device(parent, rgpio_dev->swnode, id,
-					&rgpio_dev->of_args);
+	ret = reset_create_gpio_aux_device(rgpio_dev, parent);
 	if (ret)
 		goto err_del_swnode;
 
+	reset_gpio_add_devlink(fwnode, rgpio_dev);
 	list_add(&rgpio_dev->list, &reset_gpio_lookup_list);
 
 	return 0;
 
 err_del_swnode:
 	fwnode_remove_software_node(rgpio_dev->swnode);
-err_put_of_node:
-	of_node_put(rgpio_dev->of_args.np);
+err_put_fwnode:
+	fwnode_handle_put(rgpio_dev->ref_args.fwnode);
 	kfree(rgpio_dev);
-err_ida_free:
-	ida_free(&reset_gpio_ida, id);
 
 	return ret;
 }
 
-static struct reset_controller_dev *__reset_find_rcdev(const struct of_phandle_args *args,
-						       bool gpio_fallback)
+static struct reset_controller_dev *
+__reset_find_rcdev(const struct fwnode_reference_args *args, bool gpio_fallback)
 {
+	struct fwnode_reference_args *rc_args;
 	struct reset_controller_dev *rcdev;
 
 	lockdep_assert_held(&reset_list_mutex);
 
 	list_for_each_entry(rcdev, &reset_controller_list, list) {
-		if (gpio_fallback) {
-			if (rcdev->of_args && of_phandle_args_equal(args,
-								    rcdev->of_args))
+		if (gpio_fallback && rcdev->dev &&
+		    device_is_compatible(rcdev->dev, "reset-gpio")) {
+			rc_args = dev_get_platdata(rcdev->dev);
+
+			if (fwnode_reference_args_equal(args, rc_args))
 				return rcdev;
 		} else {
-			if (args->np == rcdev->of_node)
+			if (args->fwnode == rcdev->fwnode)
 				return rcdev;
 		}
 	}
@@ -993,31 +1136,31 @@ static struct reset_controller_dev *__reset_find_rcdev(const struct of_phandle_a
 }
 
 struct reset_control *
-__of_reset_control_get(struct device_node *node, const char *id, int index,
-		       enum reset_control_flags flags)
+__fwnode_reset_control_get(struct fwnode_handle *fwnode, const char *id, int index,
+			   enum reset_control_flags flags)
 {
 	bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL;
 	bool gpio_fallback = false;
-	struct reset_control *rstc;
+	struct reset_control *rstc = ERR_PTR(-EINVAL);
 	struct reset_controller_dev *rcdev;
-	struct of_phandle_args args;
-	int rstc_id;
+	struct fwnode_reference_args args;
+	struct of_phandle_args of_args;
+	int rstc_id = -EINVAL;
 	int ret;
 
-	if (!node)
+	if (!fwnode)
 		return ERR_PTR(-EINVAL);
 
 	if (id) {
-		index = of_property_match_string(node,
-						 "reset-names", id);
+		index = fwnode_property_match_string(fwnode, "reset-names", id);
 		if (index == -EILSEQ)
 			return ERR_PTR(index);
 		if (index < 0)
 			return optional ? NULL : ERR_PTR(-ENOENT);
 	}
 
-	ret = of_parse_phandle_with_args(node, "resets", "#reset-cells",
-					 index, &args);
+	ret = fwnode_property_get_reference_args(fwnode, "resets", "#reset-cells",
+						 0, index, &args);
 	if (ret == -EINVAL)
 		return ERR_PTR(ret);
 	if (ret) {
@@ -1028,51 +1171,65 @@ __of_reset_control_get(struct device_node *node, const char *id, int index,
 		 * There can be only one reset-gpio for regular devices, so
 		 * don't bother with the "reset-gpios" phandle index.
 		 */
-		ret = of_parse_phandle_with_args(node, "reset-gpios", "#gpio-cells",
-						 0, &args);
+		ret = fwnode_property_get_reference_args(fwnode, "reset-gpios",
+							 "#gpio-cells", 0, 0, &args);
 		if (ret)
 			return optional ? NULL : ERR_PTR(ret);
 
 		gpio_fallback = true;
 
-		ret = __reset_add_reset_gpio_device(&args);
+		ret = __reset_add_reset_gpio_device(fwnode, &args);
 		if (ret) {
-			rstc = ERR_PTR(ret);
-			goto out_put;
+			fwnode_handle_put(args.fwnode);
+			return ERR_PTR(ret);
 		}
 	}
 
-	mutex_lock(&reset_list_mutex);
+	guard(mutex)(&reset_list_mutex);
+
 	rcdev = __reset_find_rcdev(&args, gpio_fallback);
 	if (!rcdev) {
 		rstc = ERR_PTR(-EPROBE_DEFER);
-		goto out_unlock;
+		goto out_put;
 	}
 
-	if (WARN_ON(args.args_count != rcdev->of_reset_n_cells)) {
+	if (WARN_ON(args.nargs != rcdev->fwnode_reset_n_cells)) {
 		rstc = ERR_PTR(-EINVAL);
-		goto out_unlock;
+		goto out_put;
 	}
 
-	rstc_id = rcdev->of_xlate(rcdev, &args);
+	if (rcdev->of_xlate && is_of_node(fwnode)) {
+		ret = of_parse_phandle_with_args(to_of_node(fwnode),
+					 gpio_fallback ? "reset-gpios" : "resets",
+					 gpio_fallback ? "#gpio-cells" : "#reset-cells",
+					 gpio_fallback ? 0 : index,
+					 &of_args);
+		if (ret) {
+			rstc = ERR_PTR(ret);
+			goto out_put;
+		}
+
+		rstc_id = rcdev->of_xlate(rcdev, &of_args);
+		of_node_put(of_args.np);
+	} else if (rcdev->fwnode_xlate) {
+		rstc_id = rcdev->fwnode_xlate(rcdev, &args);
+	}
 	if (rstc_id < 0) {
 		rstc = ERR_PTR(rstc_id);
-		goto out_unlock;
+		goto out_put;
 	}
 
 	flags &= ~RESET_CONTROL_FLAGS_BIT_OPTIONAL;
 
-	/* reset_list_mutex also protects the rcdev's reset_control list */
-	rstc = __reset_control_get_internal(rcdev, rstc_id, flags);
+	scoped_guard(mutex, &rcdev->lock)
+		rstc = __reset_control_get_internal(rcdev, rstc_id, flags);
 
-out_unlock:
-	mutex_unlock(&reset_list_mutex);
 out_put:
-	of_node_put(args.np);
+	fwnode_handle_put(args.fwnode);
 
 	return rstc;
 }
-EXPORT_SYMBOL_GPL(__of_reset_control_get);
+EXPORT_SYMBOL_GPL(__fwnode_reset_control_get);
 
 struct reset_control *__reset_control_get(struct device *dev, const char *id,
 					  int index, enum reset_control_flags flags)
@@ -1080,12 +1237,13 @@ struct reset_control *__reset_control_get(struct device *dev, const char *id,
 	bool shared = flags & RESET_CONTROL_FLAGS_BIT_SHARED;
 	bool acquired = flags & RESET_CONTROL_FLAGS_BIT_ACQUIRED;
 	bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL;
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
 
 	if (WARN_ON(shared && acquired))
 		return ERR_PTR(-EINVAL);
 
-	if (dev->of_node)
-		return __of_reset_control_get(dev->of_node, id, index, flags);
+	if (fwnode)
+		return __fwnode_reset_control_get(fwnode, id, index, flags);
 
 	return optional ? NULL : ERR_PTR(-ENOENT);
 }
@@ -1108,10 +1266,9 @@ int __reset_control_bulk_get(struct device *dev, int num_rstcs,
 	return 0;
 
 err:
-	mutex_lock(&reset_list_mutex);
 	while (i--)
-		__reset_control_put_internal(rstcs[i].rstc);
-	mutex_unlock(&reset_list_mutex);
+		reset_control_put_internal(rstcs[i].rstc);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__reset_control_bulk_get);
@@ -1120,10 +1277,8 @@ static void reset_control_array_put(struct reset_control_array *resets)
 {
 	int i;
 
-	mutex_lock(&reset_list_mutex);
 	for (i = 0; i < resets->num_rstcs; i++)
-		__reset_control_put_internal(resets->rstc[i]);
-	mutex_unlock(&reset_list_mutex);
+		reset_control_put_internal(resets->rstc[i]);
 	kfree(resets);
 }
 
@@ -1141,9 +1296,7 @@ void reset_control_put(struct reset_control *rstc)
 		return;
 	}
 
-	mutex_lock(&reset_list_mutex);
-	__reset_control_put_internal(rstc);
-	mutex_unlock(&reset_list_mutex);
+	reset_control_put_internal(rstc);
 }
 EXPORT_SYMBOL_GPL(reset_control_put);
 
@@ -1154,10 +1307,8 @@ EXPORT_SYMBOL_GPL(reset_control_put);
  */
 void reset_control_bulk_put(int num_rstcs, struct reset_control_bulk_data *rstcs)
 {
-	mutex_lock(&reset_list_mutex);
 	while (num_rstcs--)
-		__reset_control_put_internal(rstcs[num_rstcs].rstc);
-	mutex_unlock(&reset_list_mutex);
+		reset_control_put_internal(rstcs[num_rstcs].rstc);
 }
 EXPORT_SYMBOL_GPL(reset_control_bulk_put);
 
@@ -1318,21 +1469,35 @@ EXPORT_SYMBOL_GPL(__device_reset);
  */
 
 /**
- * of_reset_control_get_count - Count number of resets available with a device
+ * fwnode_reset_control_get_count - Count number of resets available with a device
  *
- * @node: device node that contains 'resets'.
+ * @fwnode: firmware node that contains 'resets'.
  *
  * Returns positive reset count on success, or error number on failure and
  * on count being zero.
  */
-static int of_reset_control_get_count(struct device_node *node)
+static int fwnode_reset_control_get_count(struct fwnode_handle *fwnode)
 {
-	int count;
+	struct fwnode_reference_args args;
+	int count = 0, ret;
 
-	if (!node)
+	if (!fwnode)
 		return -EINVAL;
 
-	count = of_count_phandle_with_args(node, "resets", "#reset-cells");
+	for (;;) {
+		ret = fwnode_property_get_reference_args(fwnode, "resets", "#reset-cells",
+							 0, count, &args);
+		if (ret) {
+			if (ret == -ENOENT)
+				break;
+
+			return ret;
+		}
+
+		fwnode_handle_put(args.fwnode);
+		count++;
+	}
+
 	if (count == 0)
 		count = -ENOENT;
 
@@ -1340,23 +1505,24 @@ static int of_reset_control_get_count(struct device_node *node)
 }
 
 /**
- * of_reset_control_array_get - Get a list of reset controls using
- *				device node.
+ * fwnode_reset_control_array_get - Get a list of reset controls using
+ *                                  a firmware node.
  *
- * @np: device node for the device that requests the reset controls array
+ * @fwnode: firmware node for the device that requests the reset controls array
  * @flags: whether reset controls are shared, optional, acquired
  *
  * Returns pointer to allocated reset_control on success or error on failure
  */
 struct reset_control *
-of_reset_control_array_get(struct device_node *np, enum reset_control_flags flags)
+fwnode_reset_control_array_get(struct fwnode_handle *fwnode,
+			       enum reset_control_flags flags)
 {
 	bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL;
 	struct reset_control_array *resets;
 	struct reset_control *rstc;
 	int num, i;
 
-	num = of_reset_control_get_count(np);
+	num = fwnode_reset_control_get_count(fwnode);
 	if (num < 0)
 		return optional ? NULL : ERR_PTR(num);
 
@@ -1366,7 +1532,7 @@ of_reset_control_array_get(struct device_node *np, enum reset_control_flags flag
 	resets->num_rstcs = num;
 
 	for (i = 0; i < num; i++) {
-		rstc = __of_reset_control_get(np, NULL, i, flags);
+		rstc = __fwnode_reset_control_get(fwnode, NULL, i, flags);
 		if (IS_ERR(rstc))
 			goto err_rst;
 		resets->rstc[i] = rstc;
@@ -1376,16 +1542,14 @@ of_reset_control_array_get(struct device_node *np, enum reset_control_flags flag
 	return &resets->base;
 
 err_rst:
-	mutex_lock(&reset_list_mutex);
 	while (--i >= 0)
-		__reset_control_put_internal(resets->rstc[i]);
-	mutex_unlock(&reset_list_mutex);
+		reset_control_put_internal(resets->rstc[i]);
 
 	kfree(resets);
 
 	return rstc;
 }
-EXPORT_SYMBOL_GPL(of_reset_control_array_get);
+EXPORT_SYMBOL_GPL(fwnode_reset_control_array_get);
 
 /**
  * devm_reset_control_array_get - Resource managed reset control array get
@@ -1409,7 +1573,7 @@ devm_reset_control_array_get(struct device *dev, enum reset_control_flags flags)
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
-	rstc = of_reset_control_array_get(dev->of_node, flags);
+	rstc = fwnode_reset_control_array_get(dev_fwnode(dev), flags);
 	if (IS_ERR_OR_NULL(rstc)) {
 		devres_free(ptr);
 		return rstc;
@@ -1432,8 +1596,10 @@ EXPORT_SYMBOL_GPL(devm_reset_control_array_get);
  */
 int reset_control_get_count(struct device *dev)
 {
-	if (dev->of_node)
-		return of_reset_control_get_count(dev->of_node);
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
+
+	if (fwnode)
+		return fwnode_reset_control_get_count(fwnode);
 
 	return -ENOENT;
 }
diff --git a/drivers/reset/reset-ath79.c b/drivers/reset/reset-ath79.c
index b5d620132052d..4c4e69eb32bbc 100644
--- a/drivers/reset/reset-ath79.c
+++ b/drivers/reset/reset-ath79.c
@@ -15,7 +15,6 @@
 
 struct ath79_reset {
 	struct reset_controller_dev rcdev;
-	struct notifier_block restart_nb;
 	void __iomem *base;
 	spinlock_t lock;
 };
@@ -72,11 +71,9 @@ static const struct reset_control_ops ath79_reset_ops = {
 	.status = ath79_reset_status,
 };
 
-static int ath79_reset_restart_handler(struct notifier_block *nb,
-				unsigned long action, void *data)
+static int ath79_reset_restart_handler(struct sys_off_data *data)
 {
-	struct ath79_reset *ath79_reset =
-		container_of(nb, struct ath79_reset, restart_nb);
+	struct ath79_reset *ath79_reset = data->cb_data;
 
 	ath79_reset_assert(&ath79_reset->rcdev, FULL_CHIP_RESET);
 
@@ -108,10 +105,7 @@ static int ath79_reset_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	ath79_reset->restart_nb.notifier_call = ath79_reset_restart_handler;
-	ath79_reset->restart_nb.priority = 128;
-
-	err = register_restart_handler(&ath79_reset->restart_nb);
+	err = devm_register_restart_handler(&pdev->dev, ath79_reset_restart_handler, ath79_reset);
 	if (err)
 		dev_warn(&pdev->dev, "Failed to register restart handler\n");
 
diff --git a/drivers/reset/reset-gpio.c b/drivers/reset/reset-gpio.c
index 0a1610d9e78a3..26aa2c3a2e689 100644
--- a/drivers/reset/reset-gpio.c
+++ b/drivers/reset/reset-gpio.c
@@ -4,7 +4,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/reset-controller.h>
 
 struct reset_gpio_priv {
@@ -46,34 +46,22 @@ static const struct reset_control_ops reset_gpio_ops = {
 	.status = reset_gpio_status,
 };
 
-static int reset_gpio_of_xlate(struct reset_controller_dev *rcdev,
-			       const struct of_phandle_args *reset_spec)
+static int reset_gpio_fwnode_xlate(struct reset_controller_dev *rcdev,
+				   const struct fwnode_reference_args *reset_spec)
 {
 	return reset_spec->args[0];
 }
 
-static void reset_gpio_of_node_put(void *data)
-{
-	of_node_put(data);
-}
-
 static int reset_gpio_probe(struct auxiliary_device *adev,
 			    const struct auxiliary_device_id *id)
 {
 	struct device *dev = &adev->dev;
-	struct of_phandle_args *platdata = dev_get_platdata(dev);
 	struct reset_gpio_priv *priv;
-	int ret;
-
-	if (!platdata)
-		return -EINVAL;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	auxiliary_set_drvdata(adev, &priv->rc);
-
 	priv->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(priv->reset))
 		return dev_err_probe(dev, PTR_ERR(priv->reset),
@@ -82,15 +70,10 @@ static int reset_gpio_probe(struct auxiliary_device *adev,
 	priv->rc.ops = &reset_gpio_ops;
 	priv->rc.owner = THIS_MODULE;
 	priv->rc.dev = dev;
-	priv->rc.of_args = platdata;
-	ret = devm_add_action_or_reset(dev, reset_gpio_of_node_put,
-				       priv->rc.of_node);
-	if (ret)
-		return ret;
 
 	/* Cells to match GPIO specifier, but it's not really used */
-	priv->rc.of_reset_n_cells = 2;
-	priv->rc.of_xlate = reset_gpio_of_xlate;
+	priv->rc.fwnode_reset_n_cells = 2;
+	priv->rc.fwnode_xlate = reset_gpio_fwnode_xlate;
 	priv->rc.nr_resets = 1;
 
 	return devm_reset_controller_register(dev, &priv->rc);
diff --git a/drivers/reset/reset-intel-gw.c b/drivers/reset/reset-intel-gw.c
index a5ce3350cb5ea..0db64cc8a2822 100644
--- a/drivers/reset/reset-intel-gw.c
+++ b/drivers/reset/reset-intel-gw.c
@@ -28,7 +28,6 @@ struct intel_reset_soc {
 
 struct intel_reset_data {
 	struct reset_controller_dev rcdev;
-	struct notifier_block restart_nb;
 	const struct intel_reset_soc *soc_data;
 	struct regmap *regmap;
 	struct device *dev;
@@ -153,12 +152,10 @@ static int intel_reset_xlate(struct reset_controller_dev *rcdev,
 	return id;
 }
 
-static int intel_reset_restart_handler(struct notifier_block *nb,
-				       unsigned long action, void *data)
+static int intel_reset_restart_handler(struct sys_off_data *data)
 {
-	struct intel_reset_data *reset_data;
+	struct intel_reset_data *reset_data = data->cb_data;
 
-	reset_data = container_of(nb, struct intel_reset_data, restart_nb);
 	intel_assert_device(&reset_data->rcdev, reset_data->reboot_id);
 
 	return NOTIFY_DONE;
@@ -215,9 +212,7 @@ static int intel_reset_probe(struct platform_device *pdev)
 	if (data->soc_data->legacy)
 		data->reboot_id |= FIELD_PREP(STAT_BIT_OFFSET_MASK, rb_id[2]);
 
-	data->restart_nb.notifier_call =	intel_reset_restart_handler;
-	data->restart_nb.priority =		128;
-	register_restart_handler(&data->restart_nb);
+	devm_register_restart_handler(&pdev->dev, intel_reset_restart_handler, data);
 
 	return 0;
 }
diff --git a/drivers/reset/reset-lpc18xx.c b/drivers/reset/reset-lpc18xx.c
index e42b2f24a93da..8ac9f237e1ceb 100644
--- a/drivers/reset/reset-lpc18xx.c
+++ b/drivers/reset/reset-lpc18xx.c
@@ -31,7 +31,6 @@
 
 struct lpc18xx_rgu_data {
 	struct reset_controller_dev rcdev;
-	struct notifier_block restart_nb;
 	struct clk *clk_delay;
 	struct clk *clk_reg;
 	void __iomem *base;
@@ -41,11 +40,9 @@ struct lpc18xx_rgu_data {
 
 #define to_rgu_data(p) container_of(p, struct lpc18xx_rgu_data, rcdev)
 
-static int lpc18xx_rgu_restart(struct notifier_block *nb, unsigned long mode,
-			       void *cmd)
+static int lpc18xx_rgu_restart(struct sys_off_data *data)
 {
-	struct lpc18xx_rgu_data *rc = container_of(nb, struct lpc18xx_rgu_data,
-						   restart_nb);
+	struct lpc18xx_rgu_data *rc = data->cb_data;
 
 	writel(BIT(LPC18XX_RGU_CORE_RST), rc->base + LPC18XX_RGU_CTRL0);
 	mdelay(2000);
@@ -178,9 +175,8 @@ static int lpc18xx_rgu_probe(struct platform_device *pdev)
 	if (ret)
 		return dev_err_probe(&pdev->dev, ret, "unable to register device\n");
 
-	rc->restart_nb.priority = 192,
-	rc->restart_nb.notifier_call = lpc18xx_rgu_restart,
-	ret = register_restart_handler(&rc->restart_nb);
+	ret = devm_register_sys_off_handler(&pdev->dev, SYS_OFF_MODE_RESTART, 192,
+					    lpc18xx_rgu_restart, rc);
 	if (ret)
 		dev_warn(&pdev->dev, "failed to register restart handler\n");
 
diff --git a/drivers/reset/reset-ma35d1.c b/drivers/reset/reset-ma35d1.c
index 54e53863c98aa..4ee901f001326 100644
--- a/drivers/reset/reset-ma35d1.c
+++ b/drivers/reset/reset-ma35d1.c
@@ -19,7 +19,6 @@
 
 struct ma35d1_reset_data {
 	struct reset_controller_dev rcdev;
-	struct notifier_block restart_handler;
 	void __iomem *base;
 	/* protect registers against concurrent read-modify-write */
 	spinlock_t lock;
@@ -125,10 +124,9 @@ static const struct {
 	[MA35D1_RESET_SSPCC] =   {0x2C, 31}
 };
 
-static int ma35d1_restart_handler(struct notifier_block *this, unsigned long mode, void *cmd)
+static int ma35d1_restart_handler(struct sys_off_data *sys_off_data)
 {
-	struct ma35d1_reset_data *data =
-				 container_of(this, struct ma35d1_reset_data, restart_handler);
+	struct ma35d1_reset_data *data = sys_off_data->cb_data;
 	u32 id = MA35D1_RESET_CHIP;
 
 	writel_relaxed(BIT(ma35d1_reset_map[id].bit),
@@ -213,11 +211,10 @@ static int ma35d1_reset_probe(struct platform_device *pdev)
 	reset_data->rcdev.nr_resets = MA35D1_RESET_COUNT;
 	reset_data->rcdev.ops = &ma35d1_reset_ops;
 	reset_data->rcdev.of_node = dev->of_node;
-	reset_data->restart_handler.notifier_call = ma35d1_restart_handler;
-	reset_data->restart_handler.priority = 192;
 	spin_lock_init(&reset_data->lock);
 
-	err = register_restart_handler(&reset_data->restart_handler);
+	err = devm_register_sys_off_handler(dev, SYS_OFF_MODE_RESTART, 192,
+					    ma35d1_restart_handler, reset_data);
 	if (err)
 		dev_warn(&pdev->dev, "failed to register restart handler\n");
 
diff --git a/drivers/reset/reset-npcm.c b/drivers/reset/reset-npcm.c
index 4070a4b6663f1..a762d0215e76e 100644
--- a/drivers/reset/reset-npcm.c
+++ b/drivers/reset/reset-npcm.c
@@ -89,7 +89,6 @@ static const struct npcm_reset_info npxm8xx_reset_info[] = {
 
 struct npcm_rc_data {
 	struct reset_controller_dev rcdev;
-	struct notifier_block restart_nb;
 	const struct npcm_reset_info *info;
 	struct regmap *gcr_regmap;
 	u32 sw_reset_number;
@@ -100,11 +99,9 @@ struct npcm_rc_data {
 
 #define to_rc_data(p) container_of(p, struct npcm_rc_data, rcdev)
 
-static int npcm_rc_restart(struct notifier_block *nb, unsigned long mode,
-			   void *cmd)
+static int npcm_rc_restart(struct sys_off_data *data)
 {
-	struct npcm_rc_data *rc = container_of(nb, struct npcm_rc_data,
-					       restart_nb);
+	struct npcm_rc_data *rc = data->cb_data;
 
 	writel(NPCM_SWRST << rc->sw_reset_number, rc->base + NPCM_SWRSTR);
 	mdelay(1000);
@@ -472,9 +469,8 @@ static int npcm_rc_probe(struct platform_device *pdev)
 	if (!of_property_read_u32(pdev->dev.of_node, "nuvoton,sw-reset-number",
 				  &rc->sw_reset_number)) {
 		if (rc->sw_reset_number && rc->sw_reset_number < 5) {
-			rc->restart_nb.priority = 192;
-			rc->restart_nb.notifier_call = npcm_rc_restart;
-			ret = register_restart_handler(&rc->restart_nb);
+			ret = devm_register_sys_off_handler(&pdev->dev, SYS_OFF_MODE_RESTART, 192,
+							    npcm_rc_restart, rc);
 			if (ret) {
 				dev_warn(&pdev->dev, "failed to register restart handler\n");
 				return ret;
diff --git a/drivers/reset/reset-rzg2l-usbphy-ctrl.c b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
index 32bc268c9149e..fd75d9601a3bf 100644
--- a/drivers/reset/reset-rzg2l-usbphy-ctrl.c
+++ b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
@@ -136,6 +136,9 @@ static int rzg2l_usbphy_ctrl_set_pwrrdy(struct regmap_field *pwrrdy,
 {
 	u32 val = power_on ? 0 : 1;
 
+	if (!pwrrdy)
+		return 0;
+
 	/* The initialization path guarantees that the mask is 1 bit long. */
 	return regmap_field_update_bits(pwrrdy, 1, val);
 }
@@ -347,4 +350,4 @@ module_platform_driver(rzg2l_usbphy_ctrl_driver);
 
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Renesas RZ/G2L USBPHY Control");
-MODULE_AUTHOR("biju.das.jz@bp.renesas.com>");
+MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
diff --git a/drivers/reset/reset-sky1.c b/drivers/reset/reset-sky1.c
new file mode 100644
index 0000000000000..78e80a533c399
--- /dev/null
+++ b/drivers/reset/reset-sky1.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *
+ * CIX System Reset Controller (SRC) driver
+ *
+ * Author: Jerry Zhu <jerry.zhu@cixtech.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/reset/cix,sky1-system-control.h>
+#include <dt-bindings/reset/cix,sky1-s5-system-control.h>
+
+#define SKY1_RESET_SLEEP_MIN_US		50
+#define SKY1_RESET_SLEEP_MAX_US		100
+
+struct sky1_src_signal {
+	unsigned int offset;
+	unsigned int bit;
+};
+
+struct sky1_src_variant {
+	const struct sky1_src_signal *signals;
+	unsigned int signals_num;
+};
+
+struct sky1_src {
+	struct reset_controller_dev rcdev;
+	const struct sky1_src_signal *signals;
+	struct regmap *regmap;
+};
+
+enum {
+	CSU_PM_RESET				= 0x304,
+	SENSORFUSION_RESET			= 0x308,
+	SENSORFUSION_NOC_RESET			= 0x30c,
+	RESET_GROUP0_S0_DOMAIN_0		= 0x400,
+	RESET_GROUP0_S0_DOMAIN_1		= 0x404,
+	RESET_GROUP1_USB_PHYS			= 0x408,
+	RESET_GROUP1_USB_CONTROLLERS		= 0x40c,
+	RESET_GROUP0_RCSU			= 0x800,
+	RESET_GROUP1_RCSU			= 0x804,
+};
+
+static const struct sky1_src_signal sky1_src_signals[] = {
+	/* reset group1 for s0 domain modules */
+	[SKY1_CSU_PM_RESET_N]		= { CSU_PM_RESET, BIT(0) },
+	[SKY1_SENSORFUSION_RESET_N]	= { SENSORFUSION_RESET, BIT(0) },
+	[SKY1_SENSORFUSION_NOC_RESET_N]	= { SENSORFUSION_NOC_RESET, BIT(0) },
+	[SKY1_DDRC_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(0) },
+	[SKY1_GIC_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(1) },
+	[SKY1_CI700_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(2) },
+	[SKY1_SYS_NI700_RESET_N]	= { RESET_GROUP0_S0_DOMAIN_0, BIT(3) },
+	[SKY1_MM_NI700_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(4) },
+	[SKY1_PCIE_NI700_RESET_N]	= { RESET_GROUP0_S0_DOMAIN_0, BIT(5) },
+	[SKY1_GPU_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(6) },
+	[SKY1_NPUTOP_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(7) },
+	[SKY1_NPUCORE0_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(8) },
+	[SKY1_NPUCORE1_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(9) },
+	[SKY1_NPUCORE2_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(10) },
+	[SKY1_VPU_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(11) },
+	[SKY1_ISP_SRESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(12) },
+	[SKY1_ISP_ARESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(13) },
+	[SKY1_ISP_HRESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(14) },
+	[SKY1_ISP_GDCRESET_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(15) },
+	[SKY1_DPU_RESET0_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(16) },
+	[SKY1_DPU_RESET1_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(17) },
+	[SKY1_DPU_RESET2_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(18) },
+	[SKY1_DPU_RESET3_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(19) },
+	[SKY1_DPU_RESET4_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(20) },
+	[SKY1_DP_RESET0_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(21) },
+	[SKY1_DP_RESET1_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(22) },
+	[SKY1_DP_RESET2_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(23) },
+	[SKY1_DP_RESET3_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(24) },
+	[SKY1_DP_RESET4_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(25) },
+	[SKY1_DP_PHY_RST_N]		= { RESET_GROUP0_S0_DOMAIN_0, BIT(26) },
+
+	/* reset group1 for s0 domain modules */
+	[SKY1_AUDIO_HIFI5_RESET_N]	= { RESET_GROUP0_S0_DOMAIN_1, BIT(0) },
+	[SKY1_AUDIO_HIFI5_NOC_RESET_N]	= { RESET_GROUP0_S0_DOMAIN_1, BIT(1) },
+	[SKY1_CSIDPHY_PRST0_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(2) },
+	[SKY1_CSIDPHY_CMNRST0_N]	= { RESET_GROUP0_S0_DOMAIN_1, BIT(3) },
+	[SKY1_CSI0_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(4) },
+	[SKY1_CSIDPHY_PRST1_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(5) },
+	[SKY1_CSIDPHY_CMNRST1_N]	= { RESET_GROUP0_S0_DOMAIN_1, BIT(6) },
+	[SKY1_CSI1_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(7) },
+	[SKY1_CSI2_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(8) },
+	[SKY1_CSI3_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(9) },
+	[SKY1_CSIBRDGE0_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(10) },
+	[SKY1_CSIBRDGE1_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(11) },
+	[SKY1_CSIBRDGE2_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(12) },
+	[SKY1_CSIBRDGE3_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(13) },
+	[SKY1_GMAC0_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(14) },
+	[SKY1_GMAC1_RST_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(15) },
+	[SKY1_PCIE0_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(16) },
+	[SKY1_PCIE1_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(17) },
+	[SKY1_PCIE2_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(18) },
+	[SKY1_PCIE3_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(19) },
+	[SKY1_PCIE4_RESET_N]		= { RESET_GROUP0_S0_DOMAIN_1, BIT(20) },
+
+	/* reset group1 for usb phys */
+	[SKY1_USB_DP_PHY0_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(0) },
+	[SKY1_USB_DP_PHY1_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(1) },
+	[SKY1_USB_DP_PHY2_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(2) },
+	[SKY1_USB_DP_PHY3_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(3) },
+	[SKY1_USB_DP_PHY0_RST_N]		= { RESET_GROUP1_USB_PHYS, BIT(4) },
+	[SKY1_USB_DP_PHY1_RST_N]		= { RESET_GROUP1_USB_PHYS, BIT(5) },
+	[SKY1_USB_DP_PHY2_RST_N]		= { RESET_GROUP1_USB_PHYS, BIT(6) },
+	[SKY1_USB_DP_PHY3_RST_N]		= { RESET_GROUP1_USB_PHYS, BIT(7) },
+	[SKY1_USBPHY_SS_PST_N]			= { RESET_GROUP1_USB_PHYS, BIT(8) },
+	[SKY1_USBPHY_SS_RST_N]			= { RESET_GROUP1_USB_PHYS, BIT(9) },
+	[SKY1_USBPHY_HS0_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(10) },
+	[SKY1_USBPHY_HS1_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(11) },
+	[SKY1_USBPHY_HS2_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(12) },
+	[SKY1_USBPHY_HS3_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(13) },
+	[SKY1_USBPHY_HS4_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(14) },
+	[SKY1_USBPHY_HS5_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(15) },
+	[SKY1_USBPHY_HS6_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(16) },
+	[SKY1_USBPHY_HS7_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(17) },
+	[SKY1_USBPHY_HS8_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(18) },
+	[SKY1_USBPHY_HS9_PRST_N]		= { RESET_GROUP1_USB_PHYS, BIT(19) },
+
+	/* reset group1 for usb controllers */
+	[SKY1_USBC_SS0_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(0) },
+	[SKY1_USBC_SS1_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(1) },
+	[SKY1_USBC_SS2_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(2) },
+	[SKY1_USBC_SS3_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(3) },
+	[SKY1_USBC_SS4_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(4) },
+	[SKY1_USBC_SS5_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(5) },
+	[SKY1_USBC_SS0_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(6) },
+	[SKY1_USBC_SS1_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(7) },
+	[SKY1_USBC_SS2_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(8) },
+	[SKY1_USBC_SS3_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(9) },
+	[SKY1_USBC_SS4_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(10) },
+	[SKY1_USBC_SS5_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(11) },
+	[SKY1_USBC_HS0_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(12) },
+	[SKY1_USBC_HS1_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(13) },
+	[SKY1_USBC_HS2_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(14) },
+	[SKY1_USBC_HS3_PRST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(15) },
+	[SKY1_USBC_HS0_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(16) },
+	[SKY1_USBC_HS1_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(17) },
+	[SKY1_USBC_HS2_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(18) },
+	[SKY1_USBC_HS3_RST_N]		= { RESET_GROUP1_USB_CONTROLLERS, BIT(19) },
+
+	/* reset group0 for rcsu */
+	[SKY1_AUDIO_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(0) },
+	[SKY1_CI700_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(1) },
+	[SKY1_CSI_RCSU0_RESET_N]		= { RESET_GROUP0_RCSU, BIT(2) },
+	[SKY1_CSI_RCSU1_RESET_N]		= { RESET_GROUP0_RCSU, BIT(3) },
+	[SKY1_CSU_PM_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(4) },
+	[SKY1_DDR_BROADCAST_RCSU_RESET_N]	= { RESET_GROUP0_RCSU, BIT(5) },
+	[SKY1_DDR_CTRL_RCSU_0_RESET_N]		= { RESET_GROUP0_RCSU, BIT(6) },
+	[SKY1_DDR_CTRL_RCSU_1_RESET_N]		= { RESET_GROUP0_RCSU, BIT(7) },
+	[SKY1_DDR_CTRL_RCSU_2_RESET_N]		= { RESET_GROUP0_RCSU, BIT(8) },
+	[SKY1_DDR_CTRL_RCSU_3_RESET_N]		= { RESET_GROUP0_RCSU, BIT(9) },
+	[SKY1_DDR_TZC400_RCSU_0_RESET_N]	= { RESET_GROUP0_RCSU, BIT(10) },
+	[SKY1_DDR_TZC400_RCSU_1_RESET_N]	= { RESET_GROUP0_RCSU, BIT(11) },
+	[SKY1_DDR_TZC400_RCSU_2_RESET_N]	= { RESET_GROUP0_RCSU, BIT(12) },
+	[SKY1_DDR_TZC400_RCSU_3_RESET_N]	= { RESET_GROUP0_RCSU, BIT(13) },
+	[SKY1_DP0_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(14) },
+	[SKY1_DP1_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(15) },
+	[SKY1_DP2_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(16) },
+	[SKY1_DP3_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(17) },
+	[SKY1_DP4_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(18) },
+	[SKY1_DPU0_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(19) },
+	[SKY1_DPU1_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(20) },
+	[SKY1_DPU2_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(21) },
+	[SKY1_DPU3_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(22) },
+	[SKY1_DPU4_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(23) },
+	[SKY1_DSU_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(24) },
+	[SKY1_FCH_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(25) },
+	[SKY1_GICD_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(26) },
+	[SKY1_GMAC_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(27) },
+	[SKY1_GPU_RCSU_RESET_N]			= { RESET_GROUP0_RCSU, BIT(28) },
+	[SKY1_ISP_RCSU0_RESET_N]		= { RESET_GROUP0_RCSU, BIT(29) },
+	[SKY1_ISP_RCSU1_RESET_N]		= { RESET_GROUP0_RCSU, BIT(30) },
+	[SKY1_NI700_MMHUB_RCSU_RESET_N]		= { RESET_GROUP0_RCSU, BIT(31) },
+
+	/* reset group1 for rcsu */
+	[SKY1_NPU_RCSU_RESET_N]			= { RESET_GROUP1_RCSU, BIT(0) },
+	[SKY1_NI700_PCIE_RCSU_RESET_N]		= { RESET_GROUP1_RCSU, BIT(1) },
+	[SKY1_PCIE_X421_RCSU_RESET_N]		= { RESET_GROUP1_RCSU, BIT(2) },
+	[SKY1_PCIE_X8_RCSU_RESET_N]		= { RESET_GROUP1_RCSU, BIT(3) },
+	[SKY1_SF_RCSU_RESET_N]			= { RESET_GROUP1_RCSU, BIT(4) },
+	[SKY1_RCSU_SMMU_MMHUB_RESET_N]		= { RESET_GROUP1_RCSU, BIT(5) },
+	[SKY1_RCSU_SMMU_PCIEHUB_RESET_N]	= { RESET_GROUP1_RCSU, BIT(6) },
+	[SKY1_RCSU_SYSHUB_RESET_N]		= { RESET_GROUP1_RCSU, BIT(7) },
+	[SKY1_NI700_SMN_RCSU_RESET_N]		= { RESET_GROUP1_RCSU, BIT(8) },
+	[SKY1_NI700_SYSHUB_RCSU_RESET_N]	= { RESET_GROUP1_RCSU, BIT(9) },
+	[SKY1_RCSU_USB2_HOST0_RESET_N]		= { RESET_GROUP1_RCSU, BIT(10) },
+	[SKY1_RCSU_USB2_HOST1_RESET_N]		= { RESET_GROUP1_RCSU, BIT(11) },
+	[SKY1_RCSU_USB2_HOST2_RESET_N]		= { RESET_GROUP1_RCSU, BIT(12) },
+	[SKY1_RCSU_USB2_HOST3_RESET_N]		= { RESET_GROUP1_RCSU, BIT(13) },
+	[SKY1_RCSU_USB3_TYPEA_DRD_RESET_N]	= { RESET_GROUP1_RCSU, BIT(14) },
+	[SKY1_RCSU_USB3_TYPEC_DRD_RESET_N]	= { RESET_GROUP1_RCSU, BIT(15) },
+	[SKY1_RCSU_USB3_TYPEC_HOST0_RESET_N]	= { RESET_GROUP1_RCSU, BIT(16) },
+	[SKY1_RCSU_USB3_TYPEC_HOST1_RESET_N]	= { RESET_GROUP1_RCSU, BIT(17) },
+	[SKY1_RCSU_USB3_TYPEC_HOST2_RESET_N]	= { RESET_GROUP1_RCSU, BIT(18) },
+	[SKY1_VPU_RCSU_RESET_N]			= { RESET_GROUP1_RCSU, BIT(19) },
+};
+
+static const struct sky1_src_variant variant_sky1 = {
+	.signals = sky1_src_signals,
+	.signals_num = ARRAY_SIZE(sky1_src_signals),
+};
+
+enum {
+	FCH_SW_RST_FUNC			= 0x8,
+	FCH_SW_RST_BUS			= 0xc,
+	FCH_SW_XSPI			= 0x10,
+};
+
+static const struct sky1_src_signal sky1_src_fch_signals[] = {
+	/* resets for fch_sw_rst_func */
+	[SW_I3C0_RST_FUNC_G_N]	= { FCH_SW_RST_FUNC, BIT(0) },
+	[SW_I3C0_RST_FUNC_I_N]	= { FCH_SW_RST_FUNC, BIT(1) },
+	[SW_I3C1_RST_FUNC_G_N]	= { FCH_SW_RST_FUNC, BIT(2) },
+	[SW_I3C1_RST_FUNC_I_N]	= { FCH_SW_RST_FUNC, BIT(3) },
+	[SW_UART0_RST_FUNC_N]	= { FCH_SW_RST_FUNC, BIT(4) },
+	[SW_UART1_RST_FUNC_N]	= { FCH_SW_RST_FUNC, BIT(5) },
+	[SW_UART2_RST_FUNC_N]	= { FCH_SW_RST_FUNC, BIT(6) },
+	[SW_UART3_RST_FUNC_N]	= { FCH_SW_RST_FUNC, BIT(7) },
+	[SW_TIMER_RST_FUNC_N]	= { FCH_SW_RST_FUNC, BIT(20) },
+
+	/* resets for fch_sw_rst_bus */
+	[SW_I3C0_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(0) },
+	[SW_I3C1_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(1) },
+	[SW_DMA_RST_AXI_N]	= { FCH_SW_RST_BUS, BIT(2) },
+	[SW_UART0_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(4) },
+	[SW_UART1_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(5) },
+	[SW_UART2_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(6) },
+	[SW_UART3_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(7) },
+	[SW_SPI0_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(8) },
+	[SW_SPI1_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(9) },
+	[SW_I2C0_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(12) },
+	[SW_I2C1_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(13) },
+	[SW_I2C2_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(14) },
+	[SW_I2C3_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(15) },
+	[SW_I2C4_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(16) },
+	[SW_I2C5_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(17) },
+	[SW_I2C6_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(18) },
+	[SW_I2C7_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(19) },
+	[SW_GPIO_RST_APB_N]	= { FCH_SW_RST_BUS, BIT(21) },
+
+	/* resets for fch_sw_xspi */
+	[SW_XSPI_REG_RST_N]	= { FCH_SW_XSPI, BIT(0) },
+	[SW_XSPI_SYS_RST_N]	= { FCH_SW_XSPI, BIT(1) },
+};
+
+static const struct sky1_src_variant variant_sky1_fch = {
+	.signals = sky1_src_fch_signals,
+	.signals_num = ARRAY_SIZE(sky1_src_fch_signals),
+};
+
+static struct sky1_src *to_sky1_src(struct reset_controller_dev *rcdev)
+{
+	return container_of(rcdev, struct sky1_src, rcdev);
+}
+
+static int sky1_reset_set(struct reset_controller_dev *rcdev,
+			  unsigned long id, bool assert)
+{
+	struct sky1_src *sky1src = to_sky1_src(rcdev);
+	const struct sky1_src_signal *signal = &sky1src->signals[id];
+	unsigned int value = assert ? 0 : signal->bit;
+
+	return regmap_update_bits(sky1src->regmap,
+				  signal->offset, signal->bit, value);
+}
+
+static int sky1_reset_assert(struct reset_controller_dev *rcdev,
+			     unsigned long id)
+{
+	sky1_reset_set(rcdev, id, true);
+	usleep_range(SKY1_RESET_SLEEP_MIN_US,
+		     SKY1_RESET_SLEEP_MAX_US);
+	return 0;
+}
+
+static int sky1_reset_deassert(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	sky1_reset_set(rcdev, id, false);
+	usleep_range(SKY1_RESET_SLEEP_MIN_US,
+		     SKY1_RESET_SLEEP_MAX_US);
+	return 0;
+}
+
+static int sky1_reset(struct reset_controller_dev *rcdev,
+		      unsigned long id)
+{
+	sky1_reset_assert(rcdev, id);
+	sky1_reset_deassert(rcdev, id);
+	return 0;
+}
+
+static int sky1_reset_status(struct reset_controller_dev *rcdev,
+			     unsigned long id)
+{
+	unsigned int value = 0;
+	struct sky1_src *sky1src = to_sky1_src(rcdev);
+	const struct sky1_src_signal *signal = &sky1src->signals[id];
+
+	regmap_read(sky1src->regmap, signal->offset, &value);
+	return !(value & signal->bit);
+}
+
+static const struct reset_control_ops sky1_src_ops = {
+	.reset    = sky1_reset,
+	.assert   = sky1_reset_assert,
+	.deassert = sky1_reset_deassert,
+	.status   = sky1_reset_status
+};
+
+static int sky1_reset_probe(struct platform_device *pdev)
+{
+	struct sky1_src *sky1src;
+	struct device *dev = &pdev->dev;
+	const struct sky1_src_variant *variant;
+
+	sky1src = devm_kzalloc(dev, sizeof(*sky1src), GFP_KERNEL);
+	if (!sky1src)
+		return -ENOMEM;
+
+	variant = of_device_get_match_data(dev);
+
+	sky1src->regmap = device_node_to_regmap(dev->of_node);
+	if (IS_ERR(sky1src->regmap)) {
+		return dev_err_probe(dev, PTR_ERR(sky1src->regmap),
+				     "Unable to get sky1-src regmap");
+	}
+
+	sky1src->signals = variant->signals;
+	sky1src->rcdev.owner     = THIS_MODULE;
+	sky1src->rcdev.nr_resets = variant->signals_num;
+	sky1src->rcdev.ops       = &sky1_src_ops;
+	sky1src->rcdev.of_node   = dev->of_node;
+	sky1src->rcdev.dev       = dev;
+
+	return devm_reset_controller_register(dev, &sky1src->rcdev);
+}
+
+static const struct of_device_id sky1_sysreg_of_match[] = {
+	{ .compatible = "cix,sky1-system-control", .data = &variant_sky1_fch},
+	{ .compatible = "cix,sky1-s5-system-control", .data = &variant_sky1},
+	{},
+};
+MODULE_DEVICE_TABLE(of, sky1_sysreg_of_match);
+
+static struct platform_driver sky1_reset_driver = {
+	.probe	= sky1_reset_probe,
+	.driver = {
+		.name		= "cix,sky1-rst",
+		.of_match_table = sky1_sysreg_of_match,
+	},
+};
+module_platform_driver(sky1_reset_driver)
+
+MODULE_AUTHOR("Jerry Zhu <jerry.zhu@cixtech.com>");
+MODULE_DESCRIPTION("Cix Sky1 reset driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/reset/reset-sunplus.c b/drivers/reset/reset-sunplus.c
index df58decab64da..58b0874337598 100644
--- a/drivers/reset/reset-sunplus.c
+++ b/drivers/reset/reset-sunplus.c
@@ -100,7 +100,6 @@ static const u32 sp_resets[] = {
 
 struct sp_reset {
 	struct reset_controller_dev rcdev;
-	struct notifier_block notifier;
 	void __iomem *base;
 };
 
@@ -154,10 +153,9 @@ static const struct reset_control_ops sp_reset_ops = {
 	.status   = sp_reset_status,
 };
 
-static int sp_restart(struct notifier_block *nb, unsigned long mode,
-		      void *cmd)
+static int sp_restart(struct sys_off_data *data)
 {
-	struct sp_reset *reset = container_of(nb, struct sp_reset, notifier);
+	struct sp_reset *reset = data->cb_data;
 
 	sp_reset_assert(&reset->rcdev, 0);
 	sp_reset_deassert(&reset->rcdev, 0);
@@ -189,10 +187,8 @@ static int sp_reset_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	reset->notifier.notifier_call = sp_restart;
-	reset->notifier.priority = 192;
-
-	return register_restart_handler(&reset->notifier);
+	return devm_register_sys_off_handler(&pdev->dev, SYS_OFF_MODE_RESTART,
+					     192, sp_restart, reset);
 }
 
 static const struct of_device_id sp_reset_dt_ids[] = {
diff --git a/drivers/reset/reset-tenstorrent-atlantis.c b/drivers/reset/reset-tenstorrent-atlantis.c
new file mode 100644
index 0000000000000..ab8be52fdd5e4
--- /dev/null
+++ b/drivers/reset/reset-tenstorrent-atlantis.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Tenstorrent Atlantis PRCM Reset Driver
+ *
+ * Copyright (c) 2026 Tenstorrent
+ */
+
+#include <dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/reset-controller.h>
+#include <linux/regmap.h>
+
+/* RCPU Reset Register Offsets */
+#define RCPU_BLK_RST_REG 0x001c
+#define LSIO_BLK_RST_REG 0x0020
+#define HSIO_BLK_RST_REG 0x000c
+#define PCIE_SUBS_RST_REG 0x0000
+#define MM_RSTN_REG 0x0014
+
+struct atlantis_reset_data {
+	u8 bit;
+	u16 reg;
+	bool active_low;
+};
+
+struct atlantis_reset_controller_data {
+	const struct atlantis_reset_data *reset_data;
+	size_t count;
+};
+
+struct atlantis_reset_controller {
+	struct reset_controller_dev rcdev;
+	const struct atlantis_reset_controller_data *data;
+	struct regmap *regmap;
+};
+
+static inline struct atlantis_reset_controller *
+to_atlantis_reset_controller(struct reset_controller_dev *rcdev)
+{
+	return container_of(rcdev, struct atlantis_reset_controller, rcdev);
+}
+
+#define RESET_DATA(_reg, _bit, _active_low)                          \
+	{                                                            \
+		.bit = _bit, .reg = _reg, .active_low = _active_low, \
+	}
+
+static const struct atlantis_reset_data atlantis_rcpu_resets[] = {
+	[RST_SMNDMA0]	= RESET_DATA(RCPU_BLK_RST_REG, 0, true),
+	[RST_SMNDMA1]	= RESET_DATA(RCPU_BLK_RST_REG, 1, true),
+	[RST_WDT0]	= RESET_DATA(RCPU_BLK_RST_REG, 2, true),
+	[RST_WDT1]	= RESET_DATA(RCPU_BLK_RST_REG, 3, true),
+	[RST_TMR]	= RESET_DATA(RCPU_BLK_RST_REG, 4, true),
+	[RST_PVTC]	= RESET_DATA(RCPU_BLK_RST_REG, 12, true),
+	[RST_PMU]	= RESET_DATA(RCPU_BLK_RST_REG, 13, true),
+	[RST_MAILBOX]	= RESET_DATA(RCPU_BLK_RST_REG, 14, true),
+	[RST_SPACC]	= RESET_DATA(RCPU_BLK_RST_REG, 26, true),
+	[RST_OTP]	= RESET_DATA(RCPU_BLK_RST_REG, 28, true),
+	[RST_TRNG]	= RESET_DATA(RCPU_BLK_RST_REG, 29, true),
+	[RST_CRC]	= RESET_DATA(RCPU_BLK_RST_REG, 30, true),
+	[RST_QSPI]	= RESET_DATA(LSIO_BLK_RST_REG, 0, true),
+	[RST_I2C0]	= RESET_DATA(LSIO_BLK_RST_REG, 1, true),
+	[RST_I2C1]	= RESET_DATA(LSIO_BLK_RST_REG, 2, true),
+	[RST_I2C2]	= RESET_DATA(LSIO_BLK_RST_REG, 3, true),
+	[RST_I2C3]	= RESET_DATA(LSIO_BLK_RST_REG, 4, true),
+	[RST_I2C4]	= RESET_DATA(LSIO_BLK_RST_REG, 5, true),
+	[RST_UART0]	= RESET_DATA(LSIO_BLK_RST_REG, 6, true),
+	[RST_UART1]	= RESET_DATA(LSIO_BLK_RST_REG, 7, true),
+	[RST_UART2]	= RESET_DATA(LSIO_BLK_RST_REG, 8, true),
+	[RST_UART3]	= RESET_DATA(LSIO_BLK_RST_REG, 9, true),
+	[RST_UART4]	= RESET_DATA(LSIO_BLK_RST_REG, 10, true),
+	[RST_SPI0]	= RESET_DATA(LSIO_BLK_RST_REG, 11, true),
+	[RST_SPI1]	= RESET_DATA(LSIO_BLK_RST_REG, 12, true),
+	[RST_SPI2]	= RESET_DATA(LSIO_BLK_RST_REG, 13, true),
+	[RST_SPI3]	= RESET_DATA(LSIO_BLK_RST_REG, 14, true),
+	[RST_GPIO]	= RESET_DATA(LSIO_BLK_RST_REG, 15, true),
+	[RST_CAN0]	= RESET_DATA(LSIO_BLK_RST_REG, 17, true),
+	[RST_CAN1]	= RESET_DATA(LSIO_BLK_RST_REG, 18, true),
+	[RST_I2S0]	= RESET_DATA(LSIO_BLK_RST_REG, 19, true),
+	[RST_I2S1]	= RESET_DATA(LSIO_BLK_RST_REG, 20, true),
+
+};
+
+static const struct atlantis_reset_controller_data atlantis_rcpu_reset_data = {
+	.reset_data = atlantis_rcpu_resets,
+	.count = ARRAY_SIZE(atlantis_rcpu_resets),
+};
+
+static int atlantis_reset_update(struct reset_controller_dev *rcdev,
+				 unsigned long id, bool assert)
+{
+	unsigned int val;
+	struct atlantis_reset_controller *rst =
+		to_atlantis_reset_controller(rcdev);
+	const struct atlantis_reset_data *data = &rst->data->reset_data[id];
+	unsigned int mask = BIT(data->bit);
+	struct regmap *regmap = rst->regmap;
+
+	if (data->active_low ^ assert)
+		val = mask;
+	else
+		val = 0;
+
+	return regmap_update_bits(regmap, data->reg, mask, val);
+}
+
+static int atlantis_reset_assert(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	return atlantis_reset_update(rcdev, id, true);
+}
+
+static int atlantis_reset_deassert(struct reset_controller_dev *rcdev,
+				   unsigned long id)
+{
+	return atlantis_reset_update(rcdev, id, false);
+}
+
+static const struct reset_control_ops atlantis_reset_control_ops = {
+	.assert = atlantis_reset_assert,
+	.deassert = atlantis_reset_deassert,
+};
+
+static int
+atlantis_reset_controller_register(struct device *dev,
+				   struct atlantis_reset_controller *controller)
+{
+	struct reset_controller_dev *rcdev = &controller->rcdev;
+
+	rcdev->ops = &atlantis_reset_control_ops;
+	rcdev->owner = THIS_MODULE;
+	rcdev->of_node = dev->of_node;
+	rcdev->nr_resets = controller->data->count;
+
+	return devm_reset_controller_register(dev, &controller->rcdev);
+}
+static int atlantis_reset_probe(struct auxiliary_device *adev,
+				const struct auxiliary_device_id *id)
+{
+	struct atlantis_reset_controller *controller;
+	struct device *dev = &adev->dev;
+	struct regmap *regmap;
+
+	regmap = dev_get_regmap(dev->parent, NULL);
+	if (!regmap)
+		return -ENODEV;
+
+	controller = devm_kzalloc(dev, sizeof(*controller), GFP_KERNEL);
+	if (!controller)
+		return -ENOMEM;
+	controller->data =
+		(const struct atlantis_reset_controller_data *)id->driver_data;
+	controller->regmap = regmap;
+
+	return atlantis_reset_controller_register(dev, controller);
+}
+
+static const struct auxiliary_device_id atlantis_reset_ids[] = {
+	{ .name = "atlantis_prcm.rcpu-reset",
+	  .driver_data = (kernel_ulong_t)&atlantis_rcpu_reset_data },
+	{},
+};
+MODULE_DEVICE_TABLE(auxiliary, atlantis_reset_ids);
+
+static struct auxiliary_driver atlantis_reset_driver = {
+	.probe = atlantis_reset_probe,
+	.id_table = atlantis_reset_ids,
+};
+module_auxiliary_driver(atlantis_reset_driver);
+
+MODULE_AUTHOR("Anirudh Srinivasan <asrinivasan@oss.tenstorrent.com>");
+MODULE_DESCRIPTION("Atlantis PRCM reset controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/reset/sti/reset-syscfg.c b/drivers/reset/sti/reset-syscfg.c
index 2324060b747c8..38f78d78fa4fb 100644
--- a/drivers/reset/sti/reset-syscfg.c
+++ b/drivers/reset/sti/reset-syscfg.c
@@ -41,7 +41,7 @@ struct syscfg_reset_channel {
 struct syscfg_reset_controller {
 	struct reset_controller_dev rst;
 	bool active_low;
-	struct syscfg_reset_channel *channels;
+	struct syscfg_reset_channel channels[];
 };
 
 #define to_syscfg_reset_controller(_rst) \
@@ -135,15 +135,10 @@ static int syscfg_reset_controller_register(struct device *dev,
 	struct syscfg_reset_controller *rc;
 	int i, err;
 
-	rc = devm_kzalloc(dev, sizeof(*rc), GFP_KERNEL);
+	rc = devm_kzalloc(dev, struct_size(rc, channels, data->nr_channels), GFP_KERNEL);
 	if (!rc)
 		return -ENOMEM;
 
-	rc->channels = devm_kcalloc(dev, data->nr_channels,
-				    sizeof(*rc->channels), GFP_KERNEL);
-	if (!rc->channels)
-		return -ENOMEM;
-
 	rc->rst.ops = &syscfg_reset_ops;
 	rc->rst.of_node = dev->of_node;
 	rc->rst.nr_resets = data->nr_channels;
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index 96964745065b1..948541656950a 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -650,7 +650,7 @@ static int __init rpmsg_init(void)
 	ret = bus_register(&rpmsg_bus);
 	if (ret) {
 		pr_err("failed to register rpmsg bus: %d\n", ret);
-		class_destroy(&rpmsg_class);
+		class_unregister(&rpmsg_class);
 	}
 	return ret;
 }
@@ -659,7 +659,7 @@ postcore_initcall(rpmsg_init);
 static void __exit rpmsg_fini(void)
 {
 	bus_unregister(&rpmsg_bus);
-	class_destroy(&rpmsg_class);
+	class_unregister(&rpmsg_class);
 }
 module_exit(rpmsg_fini);
 
diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c
index 3fee27914ba80..eca09872ea971 100644
--- a/drivers/rtc/rtc-abx80x.c
+++ b/drivers/rtc/rtc-abx80x.c
@@ -772,8 +772,7 @@ static int abx80x_probe(struct i2c_client *client)
 	struct abx80x_priv *priv;
 	int i, data, err, trickle_cfg = -EINVAL;
 	char buf[7];
-	const struct i2c_device_id *id = i2c_match_id(abx80x_id, client);
-	unsigned int part = id->driver_data;
+	unsigned int part = (uintptr_t)i2c_get_match_data(client);
 	unsigned int partnumber;
 	unsigned int majrev, minrev;
 	unsigned int lot;
diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c
index 0743c6acd6e2c..917b07799d942 100644
--- a/drivers/rtc/rtc-cmos.c
+++ b/drivers/rtc/rtc-cmos.c
@@ -27,6 +27,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/acpi.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -1369,85 +1370,6 @@ static int __maybe_unused cmos_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(cmos_pm_ops, cmos_suspend, cmos_resume);
 
-/*----------------------------------------------------------------*/
-
-/* On non-x86 systems, a "CMOS" RTC lives most naturally on platform_bus.
- * ACPI systems always list these as PNPACPI devices, and pre-ACPI PCs
- * probably list them in similar PNPBIOS tables; so PNP is more common.
- *
- * We don't use legacy "poke at the hardware" probing.  Ancient PCs that
- * predate even PNPBIOS should set up platform_bus devices.
- */
-
-#ifdef	CONFIG_PNP
-
-#include <linux/pnp.h>
-
-static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
-{
-	int irq;
-
-	if (pnp_port_start(pnp, 0) == 0x70 && !pnp_irq_valid(pnp, 0)) {
-		irq = 0;
-#ifdef CONFIG_X86
-		/* Some machines contain a PNP entry for the RTC, but
-		 * don't define the IRQ. It should always be safe to
-		 * hardcode it on systems with a legacy PIC.
-		 */
-		if (nr_legacy_irqs())
-			irq = RTC_IRQ;
-#endif
-	} else {
-		irq = pnp_irq(pnp, 0);
-	}
-
-	return cmos_do_probe(&pnp->dev, pnp_get_resource(pnp, IORESOURCE_IO, 0), irq);
-}
-
-static void cmos_pnp_remove(struct pnp_dev *pnp)
-{
-	cmos_do_remove(&pnp->dev);
-}
-
-static void cmos_pnp_shutdown(struct pnp_dev *pnp)
-{
-	struct device *dev = &pnp->dev;
-	struct cmos_rtc	*cmos = dev_get_drvdata(dev);
-
-	if (system_state == SYSTEM_POWER_OFF) {
-		int retval = cmos_poweroff(dev);
-
-		if (cmos_aie_poweroff(dev) < 0 && !retval)
-			return;
-	}
-
-	cmos_do_shutdown(cmos->irq);
-}
-
-static const struct pnp_device_id rtc_ids[] = {
-	{ .id = "PNP0b00", },
-	{ .id = "PNP0b01", },
-	{ .id = "PNP0b02", },
-	{ },
-};
-MODULE_DEVICE_TABLE(pnp, rtc_ids);
-
-static struct pnp_driver cmos_pnp_driver = {
-	.name		= driver_name,
-	.id_table	= rtc_ids,
-	.probe		= cmos_pnp_probe,
-	.remove		= cmos_pnp_remove,
-	.shutdown	= cmos_pnp_shutdown,
-
-	/* flag ensures resume() gets called, and stops syslog spam */
-	.flags		= PNP_DRIVER_RES_DO_NOT_CHANGE,
-	.driver		= {
-			.pm = &cmos_pm_ops,
-	},
-};
-
-#endif	/* CONFIG_PNP */
-
 #ifdef CONFIG_OF
 static const struct of_device_id of_cmos_match[] = {
 	{
@@ -1476,6 +1398,14 @@ static __init void cmos_of_init(struct platform_device *pdev)
 #else
 static inline void cmos_of_init(struct platform_device *pdev) {}
 #endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id acpi_cmos_rtc_ids[] = {
+	ACPI_CMOS_RTC_IDS
+};
+MODULE_DEVICE_TABLE(acpi, acpi_cmos_rtc_ids);
+#endif
+
 /*----------------------------------------------------------------*/
 
 /* Platform setup should have set up an RTC device, when PNP is
@@ -1493,9 +1423,18 @@ static int __init cmos_platform_probe(struct platform_device *pdev)
 		resource = platform_get_resource(pdev, IORESOURCE_IO, 0);
 	else
 		resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
+	irq = platform_get_irq_optional(pdev, 0);
+	if (irq < 0) {
 		irq = -1;
+#ifdef CONFIG_X86
+		/*
+		 * On some x86 systems the IRQ is not defined, but it should
+		 * always be safe to hardcode it on systems with a legacy PIC.
+		 */
+		if (nr_legacy_irqs())
+			irq = RTC_IRQ;
+#endif
+	}
 
 	return cmos_do_probe(&pdev->dev, resource, irq);
 }
@@ -1530,48 +1469,31 @@ static struct platform_driver cmos_platform_driver = {
 		.name		= driver_name,
 		.pm		= &cmos_pm_ops,
 		.of_match_table = of_match_ptr(of_cmos_match),
+		.acpi_match_table = ACPI_PTR(acpi_cmos_rtc_ids),
 	}
 };
 
-#ifdef CONFIG_PNP
-static bool pnp_driver_registered;
-#endif
 static bool platform_driver_registered;
 
 static int __init cmos_init(void)
 {
-	int retval = 0;
+	int retval;
 
-#ifdef	CONFIG_PNP
-	retval = pnp_register_driver(&cmos_pnp_driver);
-	if (retval == 0)
-		pnp_driver_registered = true;
-#endif
+	if (cmos_rtc.dev)
+		return 0;
 
-	if (!cmos_rtc.dev) {
-		retval = platform_driver_probe(&cmos_platform_driver,
-					       cmos_platform_probe);
-		if (retval == 0)
-			platform_driver_registered = true;
-	}
+	retval = platform_driver_probe(&cmos_platform_driver, cmos_platform_probe);
+	if (retval)
+		return retval;
 
-	if (retval == 0)
-		return 0;
+	platform_driver_registered = true;
 
-#ifdef	CONFIG_PNP
-	if (pnp_driver_registered)
-		pnp_unregister_driver(&cmos_pnp_driver);
-#endif
-	return retval;
+	return 0;
 }
 module_init(cmos_init);
 
 static void __exit cmos_exit(void)
 {
-#ifdef	CONFIG_PNP
-	if (pnp_driver_registered)
-		pnp_unregister_driver(&cmos_pnp_driver);
-#endif
 	if (platform_driver_registered)
 		platform_driver_unregister(&cmos_platform_driver);
 }
diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c
index 740cab013f590..b26afef37d9cf 100644
--- a/drivers/rtc/rtc-m41t80.c
+++ b/drivers/rtc/rtc-m41t80.c
@@ -924,13 +924,7 @@ static int m41t80_probe(struct i2c_client *client)
 		return -ENOMEM;
 
 	m41t80_data->client = client;
-	if (client->dev.of_node) {
-		m41t80_data->features = (unsigned long)
-			of_device_get_match_data(&client->dev);
-	} else {
-		const struct i2c_device_id *id = i2c_match_id(m41t80_id, client);
-		m41t80_data->features = id->driver_data;
-	}
+	m41t80_data->features = (unsigned long)i2c_get_match_data(client);
 	i2c_set_clientdata(client, m41t80_data);
 
 	m41t80_data->rtc =  devm_rtc_allocate_device(&client->dev);
diff --git a/drivers/rtc/rtc-max77686.c b/drivers/rtc/rtc-max77686.c
index 69ea3ce75b5a5..3cdfd78a07ccc 100644
--- a/drivers/rtc/rtc-max77686.c
+++ b/drivers/rtc/rtc-max77686.c
@@ -686,6 +686,11 @@ static int max77686_rtc_init_reg(struct max77686_rtc_info *info)
 	return ret;
 }
 
+static void max77686_rtc_release_dev(void *client)
+{
+	i2c_unregister_device(client);
+}
+
 static int max77686_init_rtc_regmap(struct max77686_rtc_info *info)
 {
 	struct device *parent = info->dev->parent;
@@ -713,12 +718,17 @@ static int max77686_init_rtc_regmap(struct max77686_rtc_info *info)
 		goto add_rtc_irq;
 	}
 
-	client = devm_i2c_new_dummy_device(info->dev, parent_i2c->adapter,
-					   info->drv_data->rtc_i2c_addr);
+	client = i2c_new_ancillary_device(parent_i2c, "rtc",
+					  info->drv_data->rtc_i2c_addr);
 	if (IS_ERR(client))
 		return dev_err_probe(info->dev, PTR_ERR(client),
 				     "Failed to allocate I2C device for RTC\n");
 
+	ret = devm_add_action_or_reset(info->dev, max77686_rtc_release_dev,
+				       client);
+	if (ret)
+		return ret;
+
 	info->rtc_regmap = devm_regmap_init_i2c(client,
 						info->drv_data->regmap_config);
 	if (IS_ERR(info->rtc_regmap))
diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c
index bb4fe81d3d62c..e4785c5a55d03 100644
--- a/drivers/rtc/rtc-pcf2127.c
+++ b/drivers/rtc/rtc-pcf2127.c
@@ -1449,10 +1449,10 @@ static const struct regmap_bus pcf2127_i2c_regmap = {
 static struct i2c_driver pcf2127_i2c_driver;
 
 static const struct i2c_device_id pcf2127_i2c_id[] = {
-	{ "pcf2127", PCF2127 },
-	{ "pcf2129", PCF2129 },
-	{ "pca2129", PCF2129 },
-	{ "pcf2131", PCF2131 },
+	{ "pcf2127", (kernel_ulong_t)&pcf21xx_cfg[PCF2127] },
+	{ "pcf2129", (kernel_ulong_t)&pcf21xx_cfg[PCF2129] },
+	{ "pca2129", (kernel_ulong_t)&pcf21xx_cfg[PCF2129] },
+	{ "pcf2131", (kernel_ulong_t)&pcf21xx_cfg[PCF2131] },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pcf2127_i2c_id);
@@ -1469,18 +1469,9 @@ static int pcf2127_i2c_probe(struct i2c_client *client)
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
 		return -ENODEV;
 
-	if (client->dev.of_node) {
-		variant = of_device_get_match_data(&client->dev);
-		if (!variant)
-			return -ENODEV;
-	} else {
-		enum pcf21xx_type type =
-			i2c_match_id(pcf2127_i2c_id, client)->driver_data;
-
-		if (type >= PCF21XX_LAST_ID)
-			return -ENODEV;
-		variant = &pcf21xx_cfg[type];
-	}
+	variant = i2c_get_match_data(client);
+	if (!variant)
+		return -ENODEV;
 
 	config.max_register = variant->max_register,
 
diff --git a/drivers/rtc/rtc-rs5c372.c b/drivers/rtc/rtc-rs5c372.c
index f8fab0205f8cc..936f4f05c8c7a 100644
--- a/drivers/rtc/rtc-rs5c372.c
+++ b/drivers/rtc/rtc-rs5c372.c
@@ -825,12 +825,7 @@ static int rs5c372_probe(struct i2c_client *client)
 
 	rs5c372->client = client;
 	i2c_set_clientdata(client, rs5c372);
-	if (client->dev.of_node) {
-		rs5c372->type = (uintptr_t)of_device_get_match_data(&client->dev);
-	} else {
-		const struct i2c_device_id *id = i2c_match_id(rs5c372_id, client);
-		rs5c372->type = id->driver_data;
-	}
+	rs5c372->type = (uintptr_t)i2c_get_match_data(client);
 
 	/* we read registers 0x0f then 0x00-0x0f; skip the first one */
 	rs5c372->regs = &rs5c372->buf[1];
diff --git a/drivers/rtc/rtc-rv8803.c b/drivers/rtc/rtc-rv8803.c
index 4e9e04cbec89a..2bf988a89fd7b 100644
--- a/drivers/rtc/rtc-rv8803.c
+++ b/drivers/rtc/rtc-rv8803.c
@@ -667,13 +667,7 @@ static int rv8803_probe(struct i2c_client *client)
 
 	mutex_init(&rv8803->flags_lock);
 	rv8803->client = client;
-	if (client->dev.of_node) {
-		rv8803->type = (uintptr_t)of_device_get_match_data(&client->dev);
-	} else {
-		const struct i2c_device_id *id = i2c_match_id(rv8803_id, client);
-
-		rv8803->type = id->driver_data;
-	}
+	rv8803->type = (uintptr_t)i2c_get_match_data(client);
 	i2c_set_clientdata(client, rv8803);
 
 	flags = rv8803_read_reg(client, RV8803_FLAG);
diff --git a/drivers/rtc/rtc-rx8025.c b/drivers/rtc/rtc-rx8025.c
index ced6e7adfe8d0..c57081f9e02b4 100644
--- a/drivers/rtc/rtc-rx8025.c
+++ b/drivers/rtc/rtc-rx8025.c
@@ -522,7 +522,6 @@ static const struct attribute_group rx8025_attr_group = {
 
 static int rx8025_probe(struct i2c_client *client)
 {
-	const struct i2c_device_id *id = i2c_match_id(rx8025_id, client);
 	struct i2c_adapter *adapter = client->adapter;
 	struct rx8025_data *rx8025;
 	int err = 0;
@@ -540,8 +539,7 @@ static int rx8025_probe(struct i2c_client *client)
 
 	i2c_set_clientdata(client, rx8025);
 
-	if (id)
-		rx8025->model = id->driver_data;
+	rx8025->model = (uintptr_t)i2c_get_match_data(client);
 
 	err = rx8025_init_client(client);
 	if (err)
diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c
index da6599ae3d0d8..5304d2febd63e 100644
--- a/drivers/scsi/BusLogic.c
+++ b/drivers/scsi/BusLogic.c
@@ -1632,8 +1632,8 @@ static bool __init blogic_rdconfig(struct blogic_adapter *adapter)
 	/*
 	   Initialize the Host Adapter Full Model Name from the Model Name.
 	 */
-	strcpy(adapter->full_model, "BusLogic ");
-	strcat(adapter->full_model, adapter->model);
+	scnprintf(adapter->full_model, sizeof(adapter->full_model),
+		  "BusLogic %s", adapter->model);
 	/*
 	   Select an appropriate value for the Tagged Queue Depth either from a
 	   BusLogic Driver Options specification, or based on whether this Host
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 19d0884479a24..f811ce473c2ab 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1151,6 +1151,7 @@ config SCSI_LPFC
 	depends on NVME_TARGET_FC || NVME_TARGET_FC=n
 	depends on NVME_FC || NVME_FC=n
 	select CRC_T10DIF
+	select CRC32
 	select IRQ_POLL
 	help
           This lpfc driver supports the Emulex LightPulse
diff --git a/drivers/scsi/elx/efct/efct_lio.c b/drivers/scsi/elx/efct/efct_lio.c
index d6e35ee8fee01..67d686dd6fb33 100644
--- a/drivers/scsi/elx/efct/efct_lio.c
+++ b/drivers/scsi/elx/efct/efct_lio.c
@@ -1612,6 +1612,7 @@ static const struct target_core_fabric_ops efct_lio_ops = {
 	.sess_get_initiator_sid		= NULL,
 	.tfc_tpg_base_attrs		= efct_lio_tpg_attrs,
 	.tfc_tpg_attrib_attrs           = efct_lio_tpg_attrib_attrs,
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
@@ -1650,6 +1651,7 @@ static const struct target_core_fabric_ops efct_lio_npiv_ops = {
 	.tfc_tpg_base_attrs		= efct_lio_npiv_tpg_attrs,
 	.tfc_tpg_attrib_attrs		= efct_lio_npiv_tpg_attrib_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/scsi/fnic/fdls_disc.c b/drivers/scsi/fnic/fdls_disc.c
index 455426564ca09..554dea7678854 100644
--- a/drivers/scsi/fnic/fdls_disc.c
+++ b/drivers/scsi/fnic/fdls_disc.c
@@ -4613,7 +4613,7 @@ void fnic_fdls_disc_start(struct fnic_iport_s *iport)
 	if (!iport->usefip) {
 		if (iport->flags & FNIC_FIRST_LINK_UP) {
 			spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
-			fnic_scsi_fcpio_reset(iport->fnic);
+			fnic_fcpio_reset(iport->fnic);
 			spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
 
 			iport->flags &= ~FNIC_FIRST_LINK_UP;
@@ -5072,7 +5072,7 @@ void fnic_fdls_link_down(struct fnic_iport_s *iport)
 	iport->fabric.flags = 0;
 
 	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
-	fnic_scsi_fcpio_reset(iport->fnic);
+	fnic_fcpio_reset(iport->fnic);
 	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
 	list_for_each_entry_safe(tport, next, &iport->tport_list, links) {
 		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
diff --git a/drivers/scsi/fnic/fip.c b/drivers/scsi/fnic/fip.c
index ccd0da7d490fd..132f00512ee16 100644
--- a/drivers/scsi/fnic/fip.c
+++ b/drivers/scsi/fnic/fip.c
@@ -737,7 +737,7 @@ void fnic_work_on_fip_timer(struct work_struct *work)
 		if (memcmp(iport->selected_fcf.fcf_mac, zmac, ETH_ALEN) != 0) {
 
 			if (iport->flags & FNIC_FIRST_LINK_UP) {
-				fnic_scsi_fcpio_reset(iport->fnic);
+				fnic_fcpio_reset(iport->fnic);
 				iport->flags &= ~FNIC_FIRST_LINK_UP;
 			}
 
diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h
index 42237eb3222ff..8724d64f25253 100644
--- a/drivers/scsi/fnic/fnic.h
+++ b/drivers/scsi/fnic/fnic.h
@@ -30,7 +30,7 @@
 
 #define DRV_NAME		"fnic"
 #define DRV_DESCRIPTION		"Cisco FCoE HBA Driver"
-#define DRV_VERSION		"1.8.0.2"
+#define DRV_VERSION		"1.8.0.3"
 #define PFX			DRV_NAME ": "
 #define DFX                     DRV_NAME "%d: "
 
@@ -438,6 +438,7 @@ struct fnic {
 	struct list_head tx_queue;
 	mempool_t *frame_pool;
 	mempool_t *frame_elem_pool;
+	mempool_t *frame_recv_pool;
 	struct work_struct tport_work;
 	struct list_head tport_event_list;
 
@@ -512,7 +513,6 @@ int fnic_host_reset(struct Scsi_Host *shost);
 void fnic_reset(struct Scsi_Host *shost);
 int fnic_issue_fc_host_lip(struct Scsi_Host *shost);
 void fnic_get_host_port_state(struct Scsi_Host *shost);
-void fnic_scsi_fcpio_reset(struct fnic *fnic);
 int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do, unsigned int cq_index);
 int fnic_wq_cmpl_handler(struct fnic *fnic, int);
 int fnic_flogi_reg_handler(struct fnic *fnic, u32);
@@ -541,7 +541,8 @@ fnic_chk_state_flags_locked(struct fnic *fnic, unsigned long st_flags)
 }
 void __fnic_set_state_flags(struct fnic *, unsigned long, unsigned long);
 void fnic_dump_fchost_stats(struct Scsi_Host *, struct fc_host_statistics *);
-void fnic_free_txq(struct list_head *head);
+void fnic_free_txq(struct fnic *fnic);
+void fnic_free_rxq(struct fnic *fnic);
 int fnic_get_desc_by_devid(struct pci_dev *pdev, char **desc,
 						   char **subsys_desc);
 void fnic_fdls_link_status_change(struct fnic *fnic, int linkup);
diff --git a/drivers/scsi/fnic/fnic_fcs.c b/drivers/scsi/fnic/fnic_fcs.c
index 405b341b73d7a..063eb864a5cd0 100644
--- a/drivers/scsi/fnic/fnic_fcs.c
+++ b/drivers/scsi/fnic/fnic_fcs.c
@@ -291,7 +291,7 @@ void fnic_handle_frame(struct work_struct *work)
 		if (fnic->stop_rx_link_events) {
 			list_del(&cur_frame->links);
 			spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
-			kfree(cur_frame->fp);
+			mempool_free(cur_frame->fp, fnic->frame_recv_pool);
 			mempool_free(cur_frame, fnic->frame_elem_pool);
 			return;
 		}
@@ -317,7 +317,7 @@ void fnic_handle_frame(struct work_struct *work)
 		fnic_fdls_recv_frame(&fnic->iport, cur_frame->fp,
 							 cur_frame->frame_len, fchdr_offset);
 
-		kfree(cur_frame->fp);
+		mempool_free(cur_frame->fp, fnic->frame_recv_pool);
 		mempool_free(cur_frame, fnic->frame_elem_pool);
 	}
 	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
@@ -337,8 +337,8 @@ void fnic_handle_fip_frame(struct work_struct *work)
 		if (fnic->stop_rx_link_events) {
 			list_del(&cur_frame->links);
 			spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
-			kfree(cur_frame->fp);
-			kfree(cur_frame);
+			mempool_free(cur_frame->fp, fnic->frame_recv_pool);
+			mempool_free(cur_frame, fnic->frame_elem_pool);
 			return;
 		}
 
@@ -355,8 +355,8 @@ void fnic_handle_fip_frame(struct work_struct *work)
 		list_del(&cur_frame->links);
 
 		if (fdls_fip_recv_frame(fnic, cur_frame->fp)) {
-			kfree(cur_frame->fp);
-			kfree(cur_frame);
+			mempool_free(cur_frame->fp, fnic->frame_recv_pool);
+			mempool_free(cur_frame, fnic->frame_elem_pool);
 		}
 	}
 	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
@@ -375,10 +375,10 @@ static inline int fnic_import_rq_eth_pkt(struct fnic *fnic, void *fp)
 
 	eh = (struct ethhdr *) fp;
 	if ((eh->h_proto == cpu_to_be16(ETH_P_FIP)) && (fnic->iport.usefip)) {
-		fip_fr_elem = (struct fnic_frame_list *)
-			kzalloc_obj(struct fnic_frame_list, GFP_ATOMIC);
+		fip_fr_elem = mempool_alloc(fnic->frame_elem_pool, GFP_ATOMIC);
 		if (!fip_fr_elem)
 			return 0;
+		memset(fip_fr_elem, 0, sizeof(struct fnic_frame_list));
 		fip_fr_elem->fp = fp;
 		spin_lock_irqsave(&fnic->fnic_lock, flags);
 		list_add_tail(&fip_fr_elem->links, &fnic->fip_frame_queue);
@@ -519,13 +519,13 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc
 
 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 
-	frame_elem = mempool_alloc(fnic->frame_elem_pool,
-					GFP_ATOMIC | __GFP_ZERO);
+	frame_elem = mempool_alloc(fnic->frame_elem_pool, GFP_ATOMIC);
 	if (!frame_elem) {
 		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
 				 "Failed to allocate memory for frame elem");
 		goto drop;
 	}
+	memset(frame_elem, 0, sizeof(struct fnic_frame_list));
 	frame_elem->fp = fp;
 	frame_elem->rx_ethhdr_stripped = ethhdr_stripped;
 	frame_elem->frame_len = bytes_written;
@@ -538,7 +538,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc
 	return;
 
 drop:
-	kfree(fp);
+	mempool_free(fp, fnic->frame_recv_pool);
 }
 
 static int fnic_rq_cmpl_handler_cont(struct vnic_dev *vdev,
@@ -591,7 +591,7 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq)
 	int ret;
 
 	len = FNIC_FRAME_HT_ROOM;
-	buf = kmalloc(len, GFP_ATOMIC);
+	buf = mempool_alloc(fnic->frame_recv_pool, GFP_ATOMIC);
 	if (!buf) {
 		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
 					 "Unable to allocate RQ buffer of size: %d\n", len);
@@ -609,7 +609,7 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq)
 	fnic_queue_rq_desc(rq, buf, pa, len);
 	return 0;
 free_buf:
-	kfree(buf);
+	mempool_free(buf, fnic->frame_recv_pool);
 	return ret;
 }
 
@@ -621,7 +621,7 @@ void fnic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf)
 	dma_unmap_single(&fnic->pdev->dev, buf->dma_addr, buf->len,
 			 DMA_FROM_DEVICE);
 
-	kfree(rq_buf);
+	mempool_free(rq_buf, fnic->frame_recv_pool);
 	buf->os_buf = NULL;
 }
 
@@ -704,13 +704,13 @@ fdls_send_fcoe_frame(struct fnic *fnic, void *frame, int frame_size,
 	 */
 	if ((fnic->state != FNIC_IN_FC_MODE)
 		&& (fnic->state != FNIC_IN_ETH_MODE)) {
-		frame_elem = mempool_alloc(fnic->frame_elem_pool,
-						GFP_ATOMIC | __GFP_ZERO);
+		frame_elem = mempool_alloc(fnic->frame_elem_pool, GFP_ATOMIC);
 		if (!frame_elem) {
 			FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
 				 "Failed to allocate memory for frame elem");
 			return -ENOMEM;
 		}
+		memset(frame_elem, 0, sizeof(struct fnic_frame_list));
 
 		FNIC_FCS_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num,
 			"Queueing FC frame: sid/did/type/oxid = 0x%x/0x%x/0x%x/0x%x\n",
@@ -836,14 +836,34 @@ fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id,
 	return 0;
 }
 
-void fnic_free_txq(struct list_head *head)
+void fnic_free_txq(struct fnic *fnic)
+{
+	struct fnic_frame_list *cur_frame, *next;
+
+	list_for_each_entry_safe(cur_frame, next, &fnic->tx_queue, links) {
+		list_del(&cur_frame->links);
+		mempool_free(cur_frame->fp, fnic->frame_pool);
+		mempool_free(cur_frame, fnic->frame_elem_pool);
+	}
+}
+
+void fnic_free_rxq(struct fnic *fnic)
 {
 	struct fnic_frame_list *cur_frame, *next;
 
-	list_for_each_entry_safe(cur_frame, next, head, links) {
+	list_for_each_entry_safe(cur_frame, next, &fnic->frame_queue, links) {
 		list_del(&cur_frame->links);
-		kfree(cur_frame->fp);
-		kfree(cur_frame);
+		mempool_free(cur_frame->fp, fnic->frame_recv_pool);
+		mempool_free(cur_frame, fnic->frame_elem_pool);
+	}
+
+	if (fnic->config.flags & VFCF_FIP_CAPABLE) {
+		list_for_each_entry_safe(cur_frame, next,
+				&fnic->fip_frame_queue, links) {
+			list_del(&cur_frame->links);
+			mempool_free(cur_frame->fp, fnic->frame_recv_pool);
+			mempool_free(cur_frame, fnic->frame_elem_pool);
+		}
 	}
 }
 
@@ -898,7 +918,7 @@ void fnic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
 	dma_unmap_single(&fnic->pdev->dev, buf->dma_addr, buf->len,
 			 DMA_TO_DEVICE);
 
-	kfree(buf->os_buf);
+	mempool_free(buf->os_buf, fnic->frame_pool);
 	buf->os_buf = NULL;
 }
 
@@ -1108,3 +1128,53 @@ void fnic_reset_work_handler(struct work_struct *work)
 	spin_unlock_irqrestore(&reset_fnic_list_lock,
 						   reset_fnic_list_lock_flags);
 }
+
+void fnic_fcpio_reset(struct fnic *fnic)
+{
+	unsigned long flags;
+	enum fnic_state old_state;
+	struct fnic_iport_s *iport = &fnic->iport;
+	DECLARE_COMPLETION_ONSTACK(fw_reset_done);
+	int time_remain;
+
+	/* issue fw reset */
+	spin_lock_irqsave(&fnic->fnic_lock, flags);
+	if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) {
+		/* fw reset is in progress, poll for its completion */
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
+			  "fnic is in unexpected state: %d for fw_reset\n",
+			  fnic->state);
+		return;
+	}
+
+	old_state = fnic->state;
+	fnic->state = FNIC_IN_FC_TRANS_ETH_MODE;
+
+	fnic_update_mac_locked(fnic, iport->hwmac);
+	fnic->fw_reset_done = &fw_reset_done;
+	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+
+	FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
+				"Issuing fw reset\n");
+	if (fnic_fw_reset_handler(fnic)) {
+		spin_lock_irqsave(&fnic->fnic_lock, flags);
+		if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)
+			fnic->state = old_state;
+		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
+	} else {
+		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
+					  "Waiting for fw completion\n");
+		time_remain = wait_for_completion_timeout(&fw_reset_done,
+						  msecs_to_jiffies(FNIC_FW_RESET_TIMEOUT));
+		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
+					  "Woken up after fw completion timeout\n");
+		if (time_remain == 0) {
+			FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
+				  "FW reset completion timed out after %d ms\n",
+				  FNIC_FW_RESET_TIMEOUT);
+		}
+		atomic64_inc(&fnic->fnic_stats.reset_stats.fw_reset_timeouts);
+	}
+	fnic->fw_reset_done = NULL;
+}
diff --git a/drivers/scsi/fnic/fnic_fdls.h b/drivers/scsi/fnic/fnic_fdls.h
index 531d0b37e450f..e2959120c4f9f 100644
--- a/drivers/scsi/fnic/fnic_fdls.h
+++ b/drivers/scsi/fnic/fnic_fdls.h
@@ -410,6 +410,7 @@ void fnic_fdls_add_tport(struct fnic_iport_s *iport,
 void fnic_fdls_remove_tport(struct fnic_iport_s *iport,
 			    struct fnic_tport_s *tport,
 			    unsigned long flags);
+void fnic_fcpio_reset(struct fnic *fnic);
 
 /* fip.c */
 void fnic_fcoe_send_vlan_req(struct fnic *fnic);
@@ -422,7 +423,6 @@ void fnic_handle_fip_timer(struct timer_list *t);
 extern void fdls_fabric_timer_callback(struct timer_list *t);
 
 /* fnic_scsi.c */
-void fnic_scsi_fcpio_reset(struct fnic *fnic);
 extern void fdls_fabric_timer_callback(struct timer_list *t);
 void fnic_rport_exch_reset(struct fnic *fnic, u32 fcid);
 int fnic_fdls_register_portid(struct fnic_iport_s *iport, u32 port_id,
diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c
index 8b551b79e0878..24d62c0874ac4 100644
--- a/drivers/scsi/fnic/fnic_main.c
+++ b/drivers/scsi/fnic/fnic_main.c
@@ -40,6 +40,7 @@ static struct kmem_cache *fnic_sgl_cache[FNIC_SGL_NUM_CACHES];
 static struct kmem_cache *fnic_io_req_cache;
 static struct kmem_cache *fdls_frame_cache;
 static struct kmem_cache *fdls_frame_elem_cache;
+static struct kmem_cache *fdls_frame_recv_cache;
 static LIST_HEAD(fnic_list);
 static DEFINE_SPINLOCK(fnic_list_lock);
 static DEFINE_IDA(fnic_ida);
@@ -554,6 +555,7 @@ static int fnic_cleanup(struct fnic *fnic)
 	mempool_destroy(fnic->io_req_pool);
 	mempool_destroy(fnic->frame_pool);
 	mempool_destroy(fnic->frame_elem_pool);
+	mempool_destroy(fnic->frame_recv_pool);
 	for (i = 0; i < FNIC_SGL_NUM_CACHES; i++)
 		mempool_destroy(fnic->io_sgl_pool[i]);
 
@@ -928,6 +930,14 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	fnic->frame_elem_pool = pool;
 
+	pool = mempool_create_slab_pool(FDLS_MIN_FRAMES,
+						fdls_frame_recv_cache);
+	if (!pool) {
+		err = -ENOMEM;
+		goto err_out_fdls_frame_recv_pool;
+	}
+	fnic->frame_recv_pool = pool;
+
 	/* setup vlan config, hw inserts vlan header */
 	fnic->vlan_hw_insert = 1;
 	fnic->vlan_id = 0;
@@ -1085,6 +1095,8 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	vnic_dev_notify_unset(fnic->vdev);
 err_out_fnic_notify_set:
+	mempool_destroy(fnic->frame_recv_pool);
+err_out_fdls_frame_recv_pool:
 	mempool_destroy(fnic->frame_elem_pool);
 err_out_fdls_frame_elem_pool:
 	mempool_destroy(fnic->frame_pool);
@@ -1157,7 +1169,6 @@ static void fnic_remove(struct pci_dev *pdev)
 		timer_delete_sync(&fnic->enode_ka_timer);
 		timer_delete_sync(&fnic->vn_ka_timer);
 
-		fnic_free_txq(&fnic->fip_frame_queue);
 		fnic_fcoe_reset_vlans(fnic);
 	}
 
@@ -1177,8 +1188,8 @@ static void fnic_remove(struct pci_dev *pdev)
 	list_del(&fnic->list);
 	spin_unlock_irqrestore(&fnic_list_lock, flags);
 
-	fnic_free_txq(&fnic->frame_queue);
-	fnic_free_txq(&fnic->tx_queue);
+	fnic_free_rxq(fnic);
+	fnic_free_txq(fnic);
 
 	vnic_dev_notify_unset(fnic->vdev);
 	fnic_free_intr(fnic);
@@ -1287,6 +1298,15 @@ static int __init fnic_init_module(void)
 		goto err_create_fdls_frame_cache_elem;
 	}
 
+	fdls_frame_recv_cache = kmem_cache_create("fdls_frame_recv",
+						  FNIC_FRAME_HT_ROOM,
+						  0, SLAB_HWCACHE_ALIGN, NULL);
+	if (!fdls_frame_recv_cache) {
+		pr_err("fnic fdls frame recv cach create failed\n");
+		err = -ENOMEM;
+		goto err_create_fdls_frame_recv_cache;
+	}
+
 	fnic_event_queue =
 		alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, "fnic_event_wq");
 	if (!fnic_event_queue) {
@@ -1339,6 +1359,8 @@ static int __init fnic_init_module(void)
 	if (pc_rscn_handling_feature_flag == PC_RSCN_HANDLING_FEATURE_ON)
 		destroy_workqueue(reset_fnic_work_queue);
 err_create_reset_fnic_workq:
+	kmem_cache_destroy(fdls_frame_recv_cache);
+err_create_fdls_frame_recv_cache:
 	destroy_workqueue(fnic_event_queue);
 err_create_fnic_workq:
 	kmem_cache_destroy(fdls_frame_elem_cache);
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index 29d7aca06958e..6ee3c559e129c 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -471,7 +471,6 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost,
 	int sg_count = 0;
 	unsigned long flags = 0;
 	unsigned long ptr;
-	int io_lock_acquired = 0;
 	uint16_t hwq = 0;
 	struct fnic_tport_s *tport = NULL;
 	struct rport_dd_data_s *rdd_data;
@@ -636,7 +635,6 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost,
 	spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags);
 
 	/* initialize rest of io_req */
-	io_lock_acquired = 1;
 	io_req->port_id = rport->port_id;
 	io_req->start_time = jiffies;
 	fnic_priv(sc)->state = FNIC_IOREQ_CMD_PENDING;
@@ -689,6 +687,9 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost,
 		/* REVISIT: Use per IO lock in the final code */
 		fnic_priv(sc)->flags |= FNIC_IO_ISSUED;
 	}
+
+	spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags);
+
 out:
 	cmd_trace = ((u64)sc->cmnd[0] << 56 | (u64)sc->cmnd[7] << 40 |
 			(u64)sc->cmnd[8] << 32 | (u64)sc->cmnd[2] << 24 |
@@ -699,10 +700,6 @@ enum scsi_qc_status fnic_queuecommand(struct Scsi_Host *shost,
 		   mqtag, sc, io_req, sg_count, cmd_trace,
 		   fnic_flags_and_state(sc));
 
-	/* if only we issued IO, will we have the io lock */
-	if (io_lock_acquired)
-		spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags);
-
 	atomic_dec(&fnic->in_flight);
 	atomic_dec(&tport->in_flight);
 
@@ -777,7 +774,7 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic,
 	 */
 	if (ret) {
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-		fnic_free_txq(&fnic->tx_queue);
+		fnic_free_txq(fnic);
 		goto reset_cmpl_handler_end;
 	}
 
@@ -1972,15 +1969,11 @@ void fnic_scsi_unload(struct fnic *fnic)
 	 */
 	spin_lock_irqsave(&fnic->fnic_lock, flags);
 	fnic->iport.state = FNIC_IPORT_STATE_LINK_WAIT;
-	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-
-	if (fdls_get_state(&fnic->iport.fabric) != FDLS_STATE_INIT)
-		fnic_scsi_fcpio_reset(fnic);
-
-	spin_lock_irqsave(&fnic->fnic_lock, flags);
 	fnic->in_remove = 1;
 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 
+	fnic_fcpio_reset(fnic);
+
 	fnic_flush_tport_event_list(fnic);
 	fnic_delete_fcp_tports(fnic);
 }
@@ -3040,54 +3033,3 @@ int fnic_eh_host_reset_handler(struct scsi_cmnd *sc)
 	ret = fnic_host_reset(shost);
 	return ret;
 }
-
-
-void fnic_scsi_fcpio_reset(struct fnic *fnic)
-{
-	unsigned long flags;
-	enum fnic_state old_state;
-	struct fnic_iport_s *iport = &fnic->iport;
-	DECLARE_COMPLETION_ONSTACK(fw_reset_done);
-	int time_remain;
-
-	/* issue fw reset */
-	spin_lock_irqsave(&fnic->fnic_lock, flags);
-	if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) {
-		/* fw reset is in progress, poll for its completion */
-		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-		FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
-			  "fnic is in unexpected state: %d for fw_reset\n",
-			  fnic->state);
-		return;
-	}
-
-	old_state = fnic->state;
-	fnic->state = FNIC_IN_FC_TRANS_ETH_MODE;
-
-	fnic_update_mac_locked(fnic, iport->hwmac);
-	fnic->fw_reset_done = &fw_reset_done;
-	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-
-	FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
-				  "Issuing fw reset\n");
-	if (fnic_fw_reset_handler(fnic)) {
-		spin_lock_irqsave(&fnic->fnic_lock, flags);
-		if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)
-			fnic->state = old_state;
-		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
-	} else {
-		FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
-					  "Waiting for fw completion\n");
-		time_remain = wait_for_completion_timeout(&fw_reset_done,
-						  msecs_to_jiffies(FNIC_FW_RESET_TIMEOUT));
-		FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
-					  "Woken up after fw completion timeout\n");
-		if (time_remain == 0) {
-			FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
-				  "FW reset completion timed out after %d ms)\n",
-				  FNIC_FW_RESET_TIMEOUT);
-		}
-		atomic64_inc(&fnic->fnic_stats.reset_stats.fw_reset_timeouts);
-	}
-	fnic->fw_reset_done = NULL;
-}
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index c2b082f1252c3..944ce19ae2fc1 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -1326,7 +1326,7 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 
 	if (sts && !wait_for_completion_timeout(&completion,
 		HISI_SAS_WAIT_PHYUP_TIMEOUT)) {
-		dev_warn(dev, "phy%d wait phyup timed out for func %d\n",
+		dev_warn(dev, "phy%d wait phyup timed out for func %u\n",
 			 phy_no, func);
 		if (phy->in_reset)
 			ret = -ETIMEDOUT;
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index f69efc6494b8e..fda07b193137a 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -432,7 +432,7 @@
 #define CMPLT_HDR_IPTT_OFF		0
 #define CMPLT_HDR_IPTT_MSK		(0xffff << CMPLT_HDR_IPTT_OFF)
 #define CMPLT_HDR_DEV_ID_OFF		16
-#define CMPLT_HDR_DEV_ID_MSK		(0xffff << CMPLT_HDR_DEV_ID_OFF)
+#define CMPLT_HDR_DEV_ID_MSK		(0xffffU << CMPLT_HDR_DEV_ID_OFF)
 /* dw3 */
 #define SATA_DISK_IN_ERROR_STATUS_OFF	8
 #define SATA_DISK_IN_ERROR_STATUS_MSK	(0x1 << SATA_DISK_IN_ERROR_STATUS_OFF)
@@ -444,7 +444,7 @@
 #define FIS_ATA_STATUS_ERR_OFF		18
 #define FIS_ATA_STATUS_ERR_MSK		(0x1 << FIS_ATA_STATUS_ERR_OFF)
 #define FIS_TYPE_SDB_OFF		31
-#define FIS_TYPE_SDB_MSK		(0x1 << FIS_TYPE_SDB_OFF)
+#define FIS_TYPE_SDB_MSK		(0x1U << FIS_TYPE_SDB_OFF)
 
 /* ITCT header */
 /* qw0 */
@@ -896,7 +896,7 @@ static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 			qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 		break;
 	default:
-		dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
+		dev_warn(dev, "setup itct: unsupported dev type (%u)\n",
 			 sas_dev->dev_type);
 	}
 
@@ -2847,7 +2847,7 @@ static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
 static ssize_t intr_conv_v3_hw_show(struct device *dev,
 				    struct device_attribute *attr, char *buf)
 {
-	return scnprintf(buf, PAGE_SIZE, "%u\n", hisi_sas_intr_conv);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", hisi_sas_intr_conv);
 }
 static DEVICE_ATTR_RO(intr_conv_v3_hw);
 
@@ -3293,7 +3293,7 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
 	u32 *fix_code = &hisi_hba->debugfs_bist_fixed_code[0];
 	struct device *dev = hisi_hba->dev;
 
-	dev_info(dev, "BIST info:phy%d link_rate=%d code_mode=%d path_mode=%d ffe={0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x} fixed_code={0x%x, 0x%x}\n",
+	dev_info(dev, "BIST info:phy%u link_rate=%u code_mode=%u path_mode=%u ffe={0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x} fixed_code={0x%x, 0x%x}\n",
 		 phy_no, linkrate, code_mode, path_mode,
 		 ffe[FFE_SAS_1_5_GBPS], ffe[FFE_SAS_3_0_GBPS],
 		 ffe[FFE_SAS_6_0_GBPS], ffe[FFE_SAS_12_0_GBPS],
@@ -3650,7 +3650,7 @@ static void debugfs_print_reg_v3_hw(u32 *regs_val, struct seq_file *s,
 	int i;
 
 	for (i = 0; i < reg->count; i++) {
-		int off = i * HISI_SAS_REG_MEM_SIZE;
+		u32 off = i * HISI_SAS_REG_MEM_SIZE;
 		const char *name;
 
 		name = debugfs_to_reg_name_v3_hw(off, reg->base_off,
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index a20fce04fe79b..3dd2adda195ee 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -4966,7 +4966,8 @@ static void ibmvfc_discover_targets_done(struct ibmvfc_event *evt)
 	switch (mad_status) {
 	case IBMVFC_MAD_SUCCESS:
 		ibmvfc_dbg(vhost, "Discover Targets succeeded\n");
-		vhost->num_targets = be32_to_cpu(rsp->num_written);
+		vhost->num_targets = min_t(u32, be32_to_cpu(rsp->num_written),
+					   max_targets);
 		ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_ALLOC_TGTS);
 		break;
 	case IBMVFC_MAD_FAILED:
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index b395a9d7c6401..61f6828007654 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -3968,6 +3968,7 @@ static const struct target_core_fabric_ops ibmvscsis_ops = {
 
 	.tfc_wwn_attrs			= ibmvscsis_wwn_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 689793d03c20b..49ed55db1e475 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -552,8 +552,6 @@ struct lpfc_cgn_info {
 	);
 
 	__le32   cgn_info_crc;
-#define LPFC_CGN_CRC32_MAGIC_NUMBER	0x1EDC6F41
-#define LPFC_CGN_CRC32_SEED		0xFFFFFFFF
 };
 
 #define LPFC_CGN_INFO_SZ	(sizeof(struct lpfc_cgn_info) -  \
@@ -1667,8 +1665,9 @@ lpfc_phba_elsring(struct lpfc_hba *phba)
  * @mask: Pointer to phba's cpumask member.
  * @start: starting cpu index
  *
- * Note: If no valid cpu found, then nr_cpu_ids is returned.
+ * Returns: next online CPU in @mask on success
  *
+ * Note: If no valid cpu found, then nr_cpu_ids is returned.
  **/
 static __always_inline unsigned int
 lpfc_next_online_cpu(const struct cpumask *mask, unsigned int start)
@@ -1680,8 +1679,9 @@ lpfc_next_online_cpu(const struct cpumask *mask, unsigned int start)
  * lpfc_next_present_cpu - Finds next present CPU after n
  * @n: the cpu prior to search
  *
- * Note: If no next present cpu, then fallback to first present cpu.
+ * Returns: next present CPU after CPU @n
  *
+ * Note: If no next present cpu, then fallback to first present cpu.
  **/
 static __always_inline unsigned int lpfc_next_present_cpu(int n)
 {
@@ -1691,7 +1691,7 @@ static __always_inline unsigned int lpfc_next_present_cpu(int n)
 /**
  * lpfc_sli4_mod_hba_eq_delay - update EQ delay
  * @phba: Pointer to HBA context object.
- * @q: The Event Queue to update.
+ * @eq: The Event Queue to update.
  * @delay: The delay value (in us) to be written.
  *
  **/
@@ -1753,8 +1753,9 @@ static const char *routine(enum enum_name table_key)			\
  * Pr Tag     1               0              N
  * Pr Tag     1               1              Y
  * Pr Tag     2               *              Y
- ---------------------------------------------------
+ * ---------------------------------------------------
  *
+ * Returns: whether VMID is enabled
  **/
 static inline int lpfc_is_vmid_enabled(struct lpfc_hba *phba)
 {
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index efeb61b15a5b5..8a5b76bdea061 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -86,7 +86,7 @@ void lpfc_cmf_stop(struct lpfc_hba *phba);
 void lpfc_init_congestion_stat(struct lpfc_hba *phba);
 void lpfc_init_congestion_buf(struct lpfc_hba *phba);
 int lpfc_sli4_cgn_params_read(struct lpfc_hba *phba);
-uint32_t lpfc_cgn_calc_crc32(void *bufp, uint32_t sz, uint32_t seed);
+uint32_t lpfc_cgn_calc_crc32(const void *data, size_t size);
 int lpfc_config_cgn_signal(struct lpfc_hba *phba);
 int lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total);
 void lpfc_cgn_dump_rxmonitor(struct lpfc_hba *phba);
@@ -660,6 +660,7 @@ void lpfc_wqe_cmd_template(void);
 void lpfc_nvmet_cmd_template(void);
 void lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 			   uint32_t stat, uint32_t param);
+void lpfc_nvme_flush_abts_list(struct lpfc_hba *phba);
 void lpfc_nvmels_flush_cmd(struct lpfc_hba *phba);
 extern int lpfc_enable_nvmet_cnt;
 extern unsigned long long lpfc_enable_nvmet[];
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index d64f4acfcdae9..c7853e7fe0719 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -2427,13 +2427,14 @@ lpfc_cmpl_ct_disc_fdmi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 
 			/* CGN is only for the physical port, no vports */
 			if (lpfc_fdmi_cmd(vport, ndlp, cmd,
-					  LPFC_FDMI_VENDOR_ATTR_mi) == 0)
+					  LPFC_FDMI_VENDOR_ATTR_mi) == 0) {
 				phba->link_flag |= LS_CT_VEN_RPA;
-			lpfc_printf_log(phba, KERN_INFO,
+				lpfc_printf_log(phba, KERN_INFO,
 					LOG_DISCOVERY | LOG_ELS,
 					"6458 Send MI FDMI:%x Flag x%x\n",
 					phba->sli4_hba.pc_sli4_params.mi_ver,
 					phba->link_flag);
+			}
 		} else {
 			lpfc_printf_log(phba, KERN_INFO,
 					LOG_DISCOVERY | LOG_ELS,
@@ -3214,7 +3215,7 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		     struct lpfc_iocbq *rspiocb);
 
 	if (!ndlp)
-		return 0;
+		goto fdmi_cmd_exit;
 
 	cmpl = lpfc_cmpl_ct_disc_fdmi; /* called from discovery */
 
@@ -3320,7 +3321,7 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		if (vport->port_type != LPFC_PHYSICAL_PORT) {
 			ndlp = lpfc_findnode_did(phba->pport, FDMI_DID);
 			if (!ndlp)
-				return 0;
+				goto fdmi_cmd_free_rspvirt;
 		}
 		fallthrough;
 	case SLI_MGMT_RPA:
@@ -3396,7 +3397,7 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		if (vport->port_type != LPFC_PHYSICAL_PORT) {
 			ndlp = lpfc_findnode_did(phba->pport, FDMI_DID);
 			if (!ndlp)
-				return 0;
+				goto fdmi_cmd_free_rspvirt;
 		}
 		fallthrough;
 	case SLI_MGMT_DPA:
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index de0adeecf668a..a377e97cbe651 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2013 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -137,7 +137,8 @@ struct lpfc_nodelist {
 	uint16_t	nlp_maxframe;		/* Max RCV frame size */
 	uint8_t		nlp_class_sup;		/* Supported Classes */
 	uint8_t         nlp_retry;		/* used for ELS retries */
-	uint8_t         nlp_fcp_info;	        /* class info, bits 0-3 */
+	uint8_t         nlp_fcp_info;	        /* class info, bits 0-2 */
+#define NLP_FCP_CLASS_MASK 0x07			/* class info bitmask */
 #define NLP_FCP_2_DEVICE   0x10			/* FCP-2 device */
 	u8		nlp_nvme_info;	        /* NVME NSLER Support */
 	uint8_t		vmid_support;		/* destination VMID support */
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index cee709617a313..d70a4039a345c 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -1107,7 +1107,7 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		vport->vmid_flag = 0;
 	}
 	if (sp->cmn.priority_tagging)
-		vport->phba->pport->vmid_flag |= (LPFC_VMID_ISSUE_QFPA |
+		vport->vmid_flag |= (LPFC_VMID_ISSUE_QFPA |
 						  LPFC_VMID_TYPE_PRIO);
 
 	/*
@@ -1303,8 +1303,12 @@ lpfc_issue_els_flogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 	elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp,
 				     ndlp->nlp_DID, ELS_CMD_FLOGI);
 
-	if (!elsiocb)
+	if (!elsiocb) {
+		lpfc_vport_set_state(vport, FC_VPORT_FAILED);
+		lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS | LOG_DISCOVERY,
+				 "4296 Unable to prepare FLOGI iocb\n");
 		return 1;
+	}
 
 	wqe = &elsiocb->wqe;
 	pcmd = (uint8_t *)elsiocb->cmd_dmabuf->virt;
@@ -1394,10 +1398,8 @@ lpfc_issue_els_flogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		phba->sli3_options, 0, 0);
 
 	elsiocb->ndlp = lpfc_nlp_get(ndlp);
-	if (!elsiocb->ndlp) {
-		lpfc_els_free_iocb(phba, elsiocb);
-		return 1;
-	}
+	if (!elsiocb->ndlp)
+		goto err_out;
 
 	/* Avoid race with FLOGI completion and hba_flags. */
 	set_bit(HBA_FLOGI_ISSUED, &phba->hba_flag);
@@ -1407,9 +1409,8 @@ lpfc_issue_els_flogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 	if (rc == IOCB_ERROR) {
 		clear_bit(HBA_FLOGI_ISSUED, &phba->hba_flag);
 		clear_bit(HBA_FLOGI_OUTSTANDING, &phba->hba_flag);
-		lpfc_els_free_iocb(phba, elsiocb);
 		lpfc_nlp_put(ndlp);
-		return 1;
+		goto err_out;
 	}
 
 	/* Clear external loopback plug detected flag */
@@ -1474,6 +1475,13 @@ lpfc_issue_els_flogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 	}
 
 	return 0;
+
+ err_out:
+	lpfc_els_free_iocb(phba, elsiocb);
+	lpfc_vport_set_state(vport, FC_VPORT_FAILED);
+	lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS | LOG_DISCOVERY,
+			 "4297 Issue FLOGI: Cannot send IOCB\n");
+	return 1;
 }
 
 /**
@@ -2641,7 +2649,9 @@ lpfc_issue_els_prli(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		}
 		npr->estabImagePair = 1;
 		npr->readXferRdyDis = 1;
-		if (vport->cfg_first_burst_size)
+		if (phba->sli_rev == LPFC_SLI_REV4 &&
+		    !test_bit(HBA_FCOE_MODE, &phba->hba_flag) &&
+		    vport->cfg_first_burst_size)
 			npr->writeXferRdyDis = 1;
 
 		/* For FCP support */
@@ -10291,10 +10301,8 @@ lpfc_els_rcv_fpin_cgn(struct lpfc_hba *phba, struct fc_tlv_desc *tlv)
 						cpu_to_le16(value);
 					cp->cgn_warn_freq =
 						cpu_to_le16(value);
-					crc = lpfc_cgn_calc_crc32
-						(cp,
-						LPFC_CGN_INFO_SZ,
-						LPFC_CGN_CRC32_SEED);
+					crc = lpfc_cgn_calc_crc32(
+						cp, LPFC_CGN_INFO_SZ);
 					cp->cgn_info_crc = cpu_to_le32(crc);
 				}
 
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 8aaf05d7bb0af..73e78e633d41b 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -425,7 +425,6 @@ lpfc_check_nlp_post_devloss(struct lpfc_vport *vport,
 {
 	if (test_and_clear_bit(NLP_IN_RECOV_POST_DEV_LOSS, &ndlp->save_flags)) {
 		clear_bit(NLP_DROPPED, &ndlp->nlp_flag);
-		lpfc_nlp_get(ndlp);
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY | LOG_NODE,
 				 "8438 Devloss timeout reversed on DID x%x "
 				 "refcnt %d ndlp %p flag x%lx "
@@ -3174,7 +3173,11 @@ lpfc_init_vfi_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 		return;
 	}
 
-	lpfc_initial_flogi(vport);
+	if (!lpfc_initial_flogi(vport)) {
+		lpfc_printf_vlog(vport, KERN_ERR, LOG_MBOX | LOG_ELS,
+				 "2345 Can't issue initial FLOGI\n");
+		lpfc_vport_set_state(vport, FC_VPORT_FAILED);
+	}
 	mempool_free(mboxq, phba->mbox_mem_pool);
 	return;
 }
@@ -3247,8 +3250,14 @@ lpfc_init_vpi_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 			return;
 	}
 
-	if (phba->link_flag & LS_NPIV_FAB_SUPPORTED)
-		lpfc_initial_fdisc(vport);
+	if (phba->link_flag & LS_NPIV_FAB_SUPPORTED) {
+		if (!lpfc_initial_fdisc(vport)) {
+			lpfc_printf_vlog(vport, KERN_WARNING,
+					 LOG_MBOX | LOG_ELS,
+					 "2346 Can't issue initial FDISC\n");
+			lpfc_vport_set_state(vport, FC_VPORT_FAILED);
+		}
+	}
 	else {
 		lpfc_vport_set_state(vport, FC_VPORT_NO_FABRIC_SUPP);
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
@@ -6599,11 +6608,6 @@ lpfc_nlp_get(struct lpfc_nodelist *ndlp)
 	unsigned long flags;
 
 	if (ndlp) {
-		lpfc_debugfs_disc_trc(ndlp->vport, LPFC_DISC_TRC_NODE,
-			"node get:        did:x%x flg:x%lx refcnt:x%x",
-			ndlp->nlp_DID, ndlp->nlp_flag,
-			kref_read(&ndlp->kref));
-
 		/* The check of ndlp usage to prevent incrementing the
 		 * ndlp reference count that is in the process of being
 		 * released.
@@ -6611,9 +6615,8 @@ lpfc_nlp_get(struct lpfc_nodelist *ndlp)
 		spin_lock_irqsave(&ndlp->lock, flags);
 		if (!kref_get_unless_zero(&ndlp->kref)) {
 			spin_unlock_irqrestore(&ndlp->lock, flags);
-			lpfc_printf_vlog(ndlp->vport, KERN_WARNING, LOG_NODE,
-				"0276 %s: ndlp:x%px refcnt:%d\n",
-				__func__, (void *)ndlp, kref_read(&ndlp->kref));
+			pr_info("0276 %s: NDLP x%px has zero reference count. "
+				"Exiting\n", __func__, ndlp);
 			return NULL;
 		}
 		spin_unlock_irqrestore(&ndlp->lock, flags);
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index e9d9ac7da485b..5b7955b508358 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -22,6 +22,7 @@
  *******************************************************************/
 
 #include <linux/blkdev.h>
+#include <linux/crc32.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/idr.h>
@@ -1087,7 +1088,6 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba)
 	struct lpfc_async_xchg_ctx *ctxp, *ctxp_next;
 	struct lpfc_sli4_hdw_queue *qp;
 	LIST_HEAD(aborts);
-	LIST_HEAD(nvme_aborts);
 	LIST_HEAD(nvmet_aborts);
 	struct lpfc_sglq *sglq_entry = NULL;
 	int cnt, idx;
@@ -1946,6 +1946,7 @@ lpfc_sli4_port_sta_fn_reset(struct lpfc_hba *phba, int mbx_action,
 
 	lpfc_offline_prep(phba, mbx_action);
 	lpfc_sli_flush_io_rings(phba);
+	lpfc_nvme_flush_abts_list(phba);
 	lpfc_nvmels_flush_cmd(phba);
 	lpfc_offline(phba);
 	/* release interrupt for possible resource change */
@@ -5634,8 +5635,7 @@ lpfc_cgn_update_stat(struct lpfc_hba *phba, uint32_t dtag)
 		cp->cgn_stat_npm = value;
 	}
 
-	value = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ,
-				    LPFC_CGN_CRC32_SEED);
+	value = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ);
 	cp->cgn_info_crc = cpu_to_le32(value);
 }
 
@@ -5897,8 +5897,7 @@ lpfc_cmf_stats_timer(struct hrtimer *timer)
 	cp->cgn_warn_freq = cpu_to_le16(value);
 	cp->cgn_alarm_freq = cpu_to_le16(value);
 
-	lvalue = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ,
-				     LPFC_CGN_CRC32_SEED);
+	lvalue = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ);
 	cp->cgn_info_crc = cpu_to_le32(lvalue);
 
 	hrtimer_forward_now(timer, ktime_set(0, LPFC_SEC_MIN * NSEC_PER_SEC));
@@ -7121,8 +7120,7 @@ lpfc_cgn_params_parse(struct lpfc_hba *phba,
 			cp->cgn_info_level0 = phba->cgn_p.cgn_param_level0;
 			cp->cgn_info_level1 = phba->cgn_p.cgn_param_level1;
 			cp->cgn_info_level2 = phba->cgn_p.cgn_param_level2;
-			crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ,
-						  LPFC_CGN_CRC32_SEED);
+			crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ);
 			cp->cgn_info_crc = cpu_to_le32(crc);
 		}
 		spin_unlock_irq(&phba->hbalock);
@@ -8283,7 +8281,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 			phba->cfg_total_seg_cnt,  phba->cfg_scsi_seg_cnt,
 			phba->cfg_nvme_seg_cnt);
 
-	i = min(phba->cfg_sg_dma_buf_size, SLI4_PAGE_SIZE);
+	i = min_t(u32, phba->cfg_sg_dma_buf_size, SLI4_PAGE_SIZE);
 
 	phba->lpfc_sg_dma_buf_pool =
 			dma_pool_create("lpfc_sg_dma_buf_pool",
@@ -13495,54 +13493,14 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba)
 		phba->pport->work_port_events = 0;
 }
 
-static uint32_t
-lpfc_cgn_crc32(uint32_t crc, u8 byte)
-{
-	uint32_t msb = 0;
-	uint32_t bit;
-
-	for (bit = 0; bit < 8; bit++) {
-		msb = (crc >> 31) & 1;
-		crc <<= 1;
-
-		if (msb ^ (byte & 1)) {
-			crc ^= LPFC_CGN_CRC32_MAGIC_NUMBER;
-			crc |= 1;
-		}
-		byte >>= 1;
-	}
-	return crc;
-}
-
-static uint32_t
-lpfc_cgn_reverse_bits(uint32_t wd)
-{
-	uint32_t result = 0;
-	uint32_t i;
-
-	for (i = 0; i < 32; i++) {
-		result <<= 1;
-		result |= (1 & (wd >> i));
-	}
-	return result;
-}
-
 /*
  * The routine corresponds with the algorithm the HBA firmware
  * uses to validate the data integrity.
  */
 uint32_t
-lpfc_cgn_calc_crc32(void *ptr, uint32_t byteLen, uint32_t crc)
+lpfc_cgn_calc_crc32(const void *data, size_t size)
 {
-	uint32_t  i;
-	uint32_t result;
-	uint8_t  *data = (uint8_t *)ptr;
-
-	for (i = 0; i < byteLen; ++i)
-		crc = lpfc_cgn_crc32(crc, data[i]);
-
-	result = ~lpfc_cgn_reverse_bits(crc);
-	return result;
+	return ~crc32c(~0, data, size);
 }
 
 void
@@ -13591,7 +13549,7 @@ lpfc_init_congestion_buf(struct lpfc_hba *phba)
 
 	cp->cgn_warn_freq = cpu_to_le16(LPFC_FPIN_INIT_FREQ);
 	cp->cgn_alarm_freq = cpu_to_le16(LPFC_FPIN_INIT_FREQ);
-	crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ, LPFC_CGN_CRC32_SEED);
+	crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ);
 	cp->cgn_info_crc = cpu_to_le32(crc);
 
 	phba->cgn_evt_timestamp = jiffies +
@@ -13614,7 +13572,7 @@ lpfc_init_congestion_stat(struct lpfc_hba *phba)
 	memset(&cp->cgn_stat, 0, sizeof(cp->cgn_stat));
 
 	lpfc_cgn_update_tstamp(phba, &cp->stat_start);
-	crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ, LPFC_CGN_CRC32_SEED);
+	crc = lpfc_cgn_calc_crc32(cp, LPFC_CGN_INFO_SZ);
 	cp->cgn_info_crc = cpu_to_le32(crc);
 }
 
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index a6b3b16f870da..74c2820c64f3a 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -2846,6 +2846,54 @@ lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 #endif
 }
 
+/**
+ * lpfc_nvme_flush_abts_list - Clean up nvme commands from the abts list
+ * @phba: Pointer to HBA context object.
+ *
+ **/
+void
+lpfc_nvme_flush_abts_list(struct lpfc_hba *phba)
+{
+#if (IS_ENABLED(CONFIG_NVME_FC))
+	struct lpfc_io_buf *psb, *psb_next;
+	struct lpfc_sli4_hdw_queue *qp;
+	LIST_HEAD(aborts);
+	int i;
+
+	/* abts_xxxx_buf_list_lock required because worker thread uses this
+	 * list.
+	 */
+	spin_lock_irq(&phba->hbalock);
+	for (i = 0; i < phba->cfg_hdw_queue; i++) {
+		qp = &phba->sli4_hba.hdwq[i];
+
+		spin_lock(&qp->abts_io_buf_list_lock);
+		list_for_each_entry_safe(psb, psb_next,
+					 &qp->lpfc_abts_io_buf_list, list) {
+			if (!(psb->cur_iocbq.cmd_flag & LPFC_IO_NVME))
+				continue;
+			list_move(&psb->list, &aborts);
+			qp->abts_nvme_io_bufs--;
+		}
+		spin_unlock(&qp->abts_io_buf_list_lock);
+	}
+	spin_unlock_irq(&phba->hbalock);
+
+	list_for_each_entry_safe(psb, psb_next, &aborts, list) {
+		list_del_init(&psb->list);
+		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
+				"6195 %s: lpfc_ncmd x%px flags x%x "
+				"cmd_flag x%x xri x%x\n", __func__,
+				psb, psb->flags,
+				psb->cur_iocbq.cmd_flag,
+				psb->cur_iocbq.sli4_xritag);
+		psb->flags &= ~LPFC_SBUF_XBUSY;
+		psb->status = IOSTAT_SUCCESS;
+		lpfc_sli4_nvme_pci_offline_aborted(phba, psb);
+	}
+#endif
+}
+
 /**
  * lpfc_nvmels_flush_cmd - Clean up outstanding nvmels commands for a port
  * @phba: Pointer to HBA context object.
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 69bf1ac6f846c..e9d27703bc447 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -4665,7 +4665,7 @@ static int lpfc_scsi_prep_cmnd_buf_s3(struct lpfc_vport *vport,
 	else
 		piocbq->iocb.ulpFCP2Rcvy = 0;
 
-	piocbq->iocb.ulpClass = (pnode->nlp_fcp_info & 0x0f);
+	piocbq->iocb.ulpClass = (pnode->nlp_fcp_info & NLP_FCP_CLASS_MASK);
 	piocbq->io_buf  = lpfc_cmd;
 	if (!piocbq->cmd_cmpl)
 		piocbq->cmd_cmpl = lpfc_scsi_cmd_iocb_cmpl;
@@ -4777,7 +4777,7 @@ static int lpfc_scsi_prep_cmnd_buf_s4(struct lpfc_vport *vport,
 		bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
 
 	bf_set(wqe_class, &wqe->generic.wqe_com,
-	       (pnode->nlp_fcp_info & 0x0f));
+	       (pnode->nlp_fcp_info & NLP_FCP_CLASS_MASK));
 
 	 /* Word 8 */
 	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
@@ -4877,7 +4877,7 @@ lpfc_scsi_prep_task_mgmt_cmd_s3(struct lpfc_vport *vport,
 	piocb->ulpCommand = CMD_FCP_ICMND64_CR;
 	piocb->ulpContext = ndlp->nlp_rpi;
 	piocb->ulpFCP2Rcvy = (ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0;
-	piocb->ulpClass = (ndlp->nlp_fcp_info & 0x0f);
+	piocb->ulpClass = (ndlp->nlp_fcp_info & NLP_FCP_CLASS_MASK);
 	piocb->ulpPU = 0;
 	piocb->un.fcpi.fcpi_parm = 0;
 
@@ -4945,7 +4945,7 @@ lpfc_scsi_prep_task_mgmt_cmd_s4(struct lpfc_vport *vport,
 	bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com,
 	       ((ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0));
 	bf_set(wqe_class, &wqe->fcp_icmd.wqe_com,
-	       (ndlp->nlp_fcp_info & 0x0f));
+	       (ndlp->nlp_fcp_info & NLP_FCP_CLASS_MASK));
 
 	/* ulpTimeout is only one byte */
 	if (lpfc_cmd->timeout > 0xff) {
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 303523f754b86..218714ca6af7c 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -4572,59 +4572,41 @@ void
 lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 {
 	LIST_HEAD(tx_completions);
-	LIST_HEAD(txcmplq_completions);
+	spinlock_t *plock;		/* for transmit queue access */
 	struct lpfc_iocbq *iocb, *next_iocb;
 	int offline;
 
-	if (pring->ringno == LPFC_ELS_RING) {
+	if (phba->sli_rev >= LPFC_SLI_REV4)
+		plock = &pring->ring_lock;
+	else
+		plock = &phba->hbalock;
+
+	if (pring->ringno == LPFC_ELS_RING)
 		lpfc_fabric_abort_hba(phba);
-	}
+
 	offline = pci_channel_offline(phba->pcidev);
 
-	/* Error everything on txq and txcmplq
-	 * First do the txq.
-	 */
-	if (phba->sli_rev >= LPFC_SLI_REV4) {
-		spin_lock_irq(&pring->ring_lock);
-		list_splice_init(&pring->txq, &tx_completions);
-		pring->txq_cnt = 0;
+	/* Cancel everything on txq */
+	spin_lock_irq(plock);
+	list_splice_init(&pring->txq, &tx_completions);
+	pring->txq_cnt = 0;
 
-		if (offline) {
-			list_splice_init(&pring->txcmplq,
-					 &txcmplq_completions);
-		} else {
-			/* Next issue ABTS for everything on the txcmplq */
-			list_for_each_entry_safe(iocb, next_iocb,
-						 &pring->txcmplq, list)
-				lpfc_sli_issue_abort_iotag(phba, pring,
-							   iocb, NULL);
-		}
-		spin_unlock_irq(&pring->ring_lock);
+	if (offline) {
+		/* Cancel everything on txcmplq */
+		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
+			iocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
+		list_splice_init(&pring->txcmplq, &tx_completions);
+		pring->txcmplq_cnt = 0;
 	} else {
-		spin_lock_irq(&phba->hbalock);
-		list_splice_init(&pring->txq, &tx_completions);
-		pring->txq_cnt = 0;
-
-		if (offline) {
-			list_splice_init(&pring->txcmplq, &txcmplq_completions);
-		} else {
-			/* Next issue ABTS for everything on the txcmplq */
-			list_for_each_entry_safe(iocb, next_iocb,
-						 &pring->txcmplq, list)
-				lpfc_sli_issue_abort_iotag(phba, pring,
-							   iocb, NULL);
-		}
-		spin_unlock_irq(&phba->hbalock);
+		/* Issue ABTS for everything on the txcmplq */
+		list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
+			lpfc_sli_issue_abort_iotag(phba, pring, iocb, NULL);
 	}
+	spin_unlock_irq(plock);
 
-	if (offline) {
-		/* Cancel all the IOCBs from the completions list */
-		lpfc_sli_cancel_iocbs(phba, &txcmplq_completions,
-				      IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED);
-	} else {
-		/* Make sure HBA is alive */
+	if (!offline)
 		lpfc_issue_hb_tmo(phba);
-	}
+
 	/* Cancel all the IOCBs from the completions list */
 	lpfc_sli_cancel_iocbs(phba, &tx_completions, IOSTAT_LOCAL_REJECT,
 			      IOERR_SLI_ABORTED);
@@ -14736,11 +14718,22 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
 					atomic_read(&tgtp->rcv_fcp_cmd_out),
 					atomic_read(&tgtp->xmt_fcp_release));
 		}
+		hrq->RQ_discard_frm++;
 		fallthrough;
-
 	case FC_STATUS_INSUFF_BUF_NEED_BUF:
+		/* Unexpected event - bump the counter for support. */
 		hrq->RQ_no_posted_buf++;
-		/* Post more buffers if possible */
+
+		lpfc_log_msg(phba, KERN_WARNING,
+			     LOG_ELS | LOG_DISCOVERY | LOG_SLI,
+			     "6423 RQE completion Status x%x, needed x%x "
+			     "discarded x%x\n", status,
+			     hrq->RQ_no_posted_buf - hrq->RQ_discard_frm,
+			     hrq->RQ_discard_frm);
+
+		/* For SLI3, post more buffers if possible. No action for SLI4.
+		 * SLI4 is reposting immediately after processing the RQE.
+		 */
 		set_bit(HBA_POST_RECEIVE_BUFFER, &phba->hba_flag);
 		workposted = true;
 		break;
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index b6d90604bb61b..2a5462a3ff641 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2009-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -246,6 +246,8 @@ struct lpfc_queue {
 	uint32_t q_cnt_2;
 	uint32_t q_cnt_3;
 	uint64_t q_cnt_4;
+	uint32_t q_cnt_5;
+
 /* defines for EQ stats */
 #define	EQ_max_eqe		q_cnt_1
 #define	EQ_no_entry		q_cnt_2
@@ -268,6 +270,7 @@ struct lpfc_queue {
 #define	RQ_no_buf_found		q_cnt_2
 #define	RQ_buf_posted		q_cnt_3
 #define	RQ_rcv_buf		q_cnt_4
+#define RQ_discard_frm		q_cnt_5
 
 	struct work_struct	irqwork;
 	struct work_struct	spwork;
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index c4ca8bf5843a6..31a0cd9db1c2f 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2026 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "14.4.0.13"
+#define LPFC_DRIVER_VERSION "14.4.0.14"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
@@ -32,6 +32,6 @@
 
 #define LPFC_MODULE_DESC "Emulex LightPulse Fibre Channel SCSI driver " \
 		LPFC_DRIVER_VERSION
-#define LPFC_COPYRIGHT "Copyright (C) 2017-2025 Broadcom. All Rights " \
+#define LPFC_COPYRIGHT "Copyright (C) 2017-2026 Broadcom. All Rights " \
 		"Reserved. The term \"Broadcom\" refers to Broadcom Inc. " \
 		"and/or its subsidiaries."
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index ac71ea4898b22..ecd365d78ae3c 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -6365,11 +6365,13 @@ static int megasas_init_fw(struct megasas_instance *instance)
 
 	megasas_setup_jbod_map(instance);
 
-	if (megasas_get_device_list(instance) != SUCCESS) {
-		dev_err(&instance->pdev->dev,
-			"%s: megasas_get_device_list failed\n",
-			__func__);
-		goto fail_get_ld_pd_list;
+	scoped_guard(mutex, &instance->reset_mutex) {
+		if (megasas_get_device_list(instance) != SUCCESS) {
+			dev_err(&instance->pdev->dev,
+				"%s: megasas_get_device_list failed\n",
+				__func__);
+			goto fail_get_ld_pd_list;
+		}
 	}
 
 	/* stream detection initialization */
@@ -6468,7 +6470,8 @@ static int megasas_init_fw(struct megasas_instance *instance)
 	}
 
 	if (instance->snapdump_wait_time) {
-		megasas_get_snapdump_properties(instance);
+		scoped_guard(mutex, &instance->reset_mutex)
+			megasas_get_snapdump_properties(instance);
 		dev_info(&instance->pdev->dev, "Snap dump wait time\t: %d\n",
 			 instance->snapdump_wait_time);
 	}
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index 2e584a8bf66b2..6a05ce195aa05 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -1638,7 +1638,7 @@ qla2x00_fw_state_show(struct device *dev, struct device_attribute *attr,
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	int rval = QLA_FUNCTION_FAILED;
-	uint16_t state[6];
+	uint16_t state[16];
 	uint32_t pstate;
 
 	if (IS_QLAFX00(vha->hw)) {
@@ -2402,6 +2402,63 @@ qla2x00_dport_diagnostics_show(struct device *dev,
 	    vha->dport_data[0], vha->dport_data[1],
 	    vha->dport_data[2], vha->dport_data[3]);
 }
+
+static ssize_t
+qla2x00_mpi_fw_state_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	int rval = QLA_FUNCTION_FAILED;
+	u16 state[16];
+	u16 mpi_state;
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!(IS_QLA27XX(ha) || IS_QLA28XX(ha)))
+		return scnprintf(buf, PAGE_SIZE,
+				"MPI state reporting is not supported for this HBA.\n");
+
+	memset(state, 0, sizeof(state));
+
+	mutex_lock(&vha->hw->optrom_mutex);
+	if (qla2x00_chip_is_down(vha)) {
+		mutex_unlock(&vha->hw->optrom_mutex);
+		ql_dbg(ql_dbg_user, vha, 0x70df,
+		       "ISP reset is in progress, failing mpi_fw_state.\n");
+		return -EBUSY;
+	} else if (vha->hw->flags.eeh_busy) {
+		mutex_unlock(&vha->hw->optrom_mutex);
+		ql_dbg(ql_dbg_user, vha, 0x70ea,
+		       "HBA in PCI error state, failing mpi_fw_state.\n");
+		return -EBUSY;
+	}
+
+	rval = qla2x00_get_firmware_state(vha, state);
+	mutex_unlock(&vha->hw->optrom_mutex);
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_user, vha, 0x70eb,
+		       "MB Command to retrieve MPI state failed (%d), failing mpi_fw_state.\n",
+				rval);
+		return -EIO;
+	}
+
+	mpi_state = state[11];
+
+	if (!(mpi_state & BIT_15))
+		return scnprintf(buf, PAGE_SIZE,
+				 "MPI firmware state reporting is not supported by this firmware. (0x%02x)\n",
+				mpi_state);
+
+	if (!(mpi_state & BIT_8))
+		return scnprintf(buf, PAGE_SIZE,
+				 "MPI firmware is disabled. (0x%02x)\n",
+				mpi_state);
+
+	return scnprintf(buf, PAGE_SIZE,
+			 "MPI firmware is enabled, state is %s. (0x%02x)\n",
+			 mpi_state & BIT_9 ? "active" : "inactive",
+			 mpi_state);
+}
+
 static DEVICE_ATTR(dport_diagnostics, 0444,
 	   qla2x00_dport_diagnostics_show, NULL);
 
@@ -2469,6 +2526,8 @@ static DEVICE_ATTR(port_speed, 0644, qla2x00_port_speed_show,
     qla2x00_port_speed_store);
 static DEVICE_ATTR(port_no, 0444, qla2x00_port_no_show, NULL);
 static DEVICE_ATTR(fw_attr, 0444, qla2x00_fw_attr_show, NULL);
+static DEVICE_ATTR(mpi_fw_state, 0444, qla2x00_mpi_fw_state_show, NULL);
+
 
 static struct attribute *qla2x00_host_attrs[] = {
 	&dev_attr_driver_version.attr.attr,
@@ -2517,6 +2576,7 @@ static struct attribute *qla2x00_host_attrs[] = {
 	&dev_attr_qlini_mode.attr,
 	&dev_attr_ql2xiniexchg.attr,
 	&dev_attr_ql2xexchoffld.attr,
+	&dev_attr_mpi_fw_state.attr,
 	NULL,
 };
 
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 730c42b1a7b9d..e746c9274cded 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -4914,7 +4914,7 @@ qla2x00_fw_ready(scsi_qla_host_t *vha)
 	unsigned long	wtime, mtime, cs84xx_time;
 	uint16_t	min_wait;	/* Minimum wait time if loop is down */
 	uint16_t	wait_time;	/* Wait time if loop is coming ready */
-	uint16_t	state[6];
+	uint16_t	state[16];
 	struct qla_hw_data *ha = vha->hw;
 
 	if (IS_QLAFX00(vha->hw))
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index 0d598be6f3eab..44e310f1a3708 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -2268,6 +2268,13 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states)
 		mcp->in_mb = MBX_6|MBX_5|MBX_4|MBX_3|MBX_2|MBX_1|MBX_0;
 	else
 		mcp->in_mb = MBX_1|MBX_0;
+
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		mcp->mb[12] = 0;
+		mcp->out_mb |= MBX_12;
+		mcp->in_mb |= MBX_12;
+	}
+
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
 	rval = qla2x00_mailbox_command(vha, mcp);
@@ -2280,6 +2287,8 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states)
 		states[3] = mcp->mb[4];
 		states[4] = mcp->mb[5];
 		states[5] = mcp->mb[6];  /* DPORT status */
+		if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+			states[11] = mcp->mb[12]; /* MPI state. */
 	}
 
 	if (rval != QLA_SUCCESS) {
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 28df9025def04..3be23ed067e63 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -1841,6 +1841,7 @@ static const struct target_core_fabric_ops tcm_qla2xxx_ops = {
 	.tfc_tpg_base_attrs		= tcm_qla2xxx_tpg_attrs,
 	.tfc_tpg_attrib_attrs		= tcm_qla2xxx_tpg_attrib_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
@@ -1881,6 +1882,7 @@ static const struct target_core_fabric_ops tcm_qla2xxx_npiv_ops = {
 
 	.tfc_wwn_attrs			= tcm_qla2xxx_wwn_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/scsi/scsi_bsg.c b/drivers/scsi/scsi_bsg.c
index a9a9ec086a7e3..c3ce497a3b94f 100644
--- a/drivers/scsi/scsi_bsg.c
+++ b/drivers/scsi/scsi_bsg.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/bsg.h>
+#include <linux/io_uring/cmd.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_ioctl.h>
 #include <scsi/scsi_cmnd.h>
@@ -9,6 +10,178 @@
 
 #define uptr64(val) ((void __user *)(uintptr_t)(val))
 
+/*
+ * Per-command BSG SCSI PDU stored in io_uring_cmd.pdu[32].
+ * Holds temporary state between submission, completion and task_work.
+ */
+struct scsi_bsg_uring_cmd_pdu {
+	struct bio *bio;		/* mapped user buffer, unmap in task work */
+	struct request *req;		/* block request, freed in task work */
+	u64 response_addr;		/* user space response buffer address */
+};
+static_assert(sizeof(struct scsi_bsg_uring_cmd_pdu) <= sizeof_field(struct io_uring_cmd, pdu));
+
+static inline struct scsi_bsg_uring_cmd_pdu *scsi_bsg_uring_cmd_pdu(
+	struct io_uring_cmd *ioucmd)
+{
+	return io_uring_cmd_to_pdu(ioucmd, struct scsi_bsg_uring_cmd_pdu);
+}
+
+/* Task work: build res2 (layout in uapi/linux/bsg.h) and copy sense to user. */
+static void scsi_bsg_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
+{
+	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
+	struct scsi_bsg_uring_cmd_pdu *pdu = scsi_bsg_uring_cmd_pdu(ioucmd);
+	struct request *rq = pdu->req;
+	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
+	u64 res2;
+	int ret = 0;
+	u8 driver_status = 0;
+	u8 sense_len_wr = 0;
+
+	if (pdu->bio)
+		blk_rq_unmap_user(pdu->bio);
+
+	if (scsi_status_is_check_condition(scmd->result)) {
+		driver_status = DRIVER_SENSE;
+		if (pdu->response_addr)
+			sense_len_wr = min_t(u8, scmd->sense_len,
+					     SCSI_SENSE_BUFFERSIZE);
+	}
+
+	if (sense_len_wr) {
+		if (copy_to_user(uptr64(pdu->response_addr), scmd->sense_buffer,
+				 sense_len_wr))
+			ret = -EFAULT;
+	}
+
+	res2 = bsg_scsi_res2_build(status_byte(scmd->result), driver_status,
+				  host_byte(scmd->result), sense_len_wr,
+				  scmd->resid_len);
+
+	blk_mq_free_request(rq);
+	io_uring_cmd_done32(ioucmd, ret, res2,
+			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+}
+
+static enum rq_end_io_ret scsi_bsg_uring_cmd_done(struct request *req,
+						  blk_status_t status,
+						  const struct io_comp_batch *iocb)
+{
+	struct io_uring_cmd *ioucmd = req->end_io_data;
+
+	io_uring_cmd_do_in_task_lazy(ioucmd, scsi_bsg_uring_task_cb);
+	return RQ_END_IO_NONE;
+}
+
+static int scsi_bsg_map_user_buffer(struct request *req,
+				    struct io_uring_cmd *ioucmd,
+				    unsigned int issue_flags, gfp_t gfp_mask)
+{
+	const struct bsg_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct bsg_uring_cmd);
+	bool is_write = cmd->dout_xfer_len > 0;
+	u64 buf_addr = is_write ? cmd->dout_xferp : cmd->din_xferp;
+	unsigned long buf_len = is_write ? cmd->dout_xfer_len : cmd->din_xfer_len;
+	struct iov_iter iter;
+	int ret;
+
+	if (ioucmd->flags & IORING_URING_CMD_FIXED) {
+		ret = io_uring_cmd_import_fixed(buf_addr, buf_len,
+						is_write ? WRITE : READ,
+						&iter, ioucmd, issue_flags);
+		if (ret < 0)
+			return ret;
+		ret = blk_rq_map_user_iov(req->q, req, NULL, &iter, gfp_mask);
+	} else {
+		ret = blk_rq_map_user(req->q, req, NULL, uptr64(buf_addr),
+				      buf_len, gfp_mask);
+	}
+
+	return ret;
+}
+
+static int scsi_bsg_uring_cmd(struct request_queue *q, struct io_uring_cmd *ioucmd,
+			       unsigned int issue_flags, bool open_for_write)
+{
+	struct scsi_bsg_uring_cmd_pdu *pdu = scsi_bsg_uring_cmd_pdu(ioucmd);
+	const struct bsg_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct bsg_uring_cmd);
+	struct scsi_cmnd *scmd;
+	struct request *req;
+	blk_mq_req_flags_t blk_flags = 0;
+	gfp_t gfp_mask = GFP_KERNEL;
+	int ret;
+
+	if (cmd->protocol != BSG_PROTOCOL_SCSI ||
+	    cmd->subprotocol != BSG_SUB_PROTOCOL_SCSI_CMD)
+		return -EINVAL;
+
+	if (!cmd->request || cmd->request_len == 0)
+		return -EINVAL;
+
+	if (cmd->dout_xfer_len && cmd->din_xfer_len) {
+		pr_warn_once("BIDI support in bsg has been removed.\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (cmd->dout_iovec_count > 0 || cmd->din_iovec_count > 0)
+		return -EOPNOTSUPP;
+
+	if (issue_flags & IO_URING_F_NONBLOCK) {
+		blk_flags = BLK_MQ_REQ_NOWAIT;
+		gfp_mask = GFP_NOWAIT;
+	}
+
+	req = scsi_alloc_request(q, cmd->dout_xfer_len ?
+				 REQ_OP_DRV_OUT : REQ_OP_DRV_IN, blk_flags);
+	if (IS_ERR(req))
+		return PTR_ERR(req);
+
+	scmd = blk_mq_rq_to_pdu(req);
+	scmd->cmd_len = cmd->request_len;
+	if (scmd->cmd_len > sizeof(scmd->cmnd)) {
+		ret = -EINVAL;
+		goto out_free_req;
+	}
+	scmd->allowed = SG_DEFAULT_RETRIES;
+
+	if (copy_from_user(scmd->cmnd, uptr64(cmd->request), cmd->request_len)) {
+		ret = -EFAULT;
+		goto out_free_req;
+	}
+
+	if (!scsi_cmd_allowed(scmd->cmnd, open_for_write)) {
+		ret = -EPERM;
+		goto out_free_req;
+	}
+
+	pdu->response_addr = cmd->response;
+	scmd->sense_len = cmd->max_response_len ?
+		min(cmd->max_response_len, SCSI_SENSE_BUFFERSIZE) : SCSI_SENSE_BUFFERSIZE;
+
+	if (cmd->dout_xfer_len || cmd->din_xfer_len) {
+		ret = scsi_bsg_map_user_buffer(req, ioucmd, issue_flags, gfp_mask);
+		if (ret)
+			goto out_free_req;
+		pdu->bio = req->bio;
+	} else {
+		pdu->bio = NULL;
+	}
+
+	req->timeout = cmd->timeout_ms ?
+		msecs_to_jiffies(cmd->timeout_ms) : BLK_DEFAULT_SG_TIMEOUT;
+
+	req->end_io = scsi_bsg_uring_cmd_done;
+	req->end_io_data = ioucmd;
+	pdu->req = req;
+
+	blk_execute_rq_nowait(req, false);
+	return -EIOCBQUEUED;
+
+out_free_req:
+	blk_mq_free_request(req);
+	return ret;
+}
+
 static int scsi_bsg_sg_io_fn(struct request_queue *q, struct sg_io_v4 *hdr,
 		bool open_for_write, unsigned int timeout)
 {
@@ -99,5 +272,6 @@ static int scsi_bsg_sg_io_fn(struct request_queue *q, struct sg_io_v4 *hdr,
 struct bsg_device *scsi_bsg_register_queue(struct scsi_device *sdev)
 {
 	return bsg_register_queue(sdev->request_queue, &sdev->sdev_gendev,
-			dev_name(&sdev->sdev_gendev), scsi_bsg_sg_io_fn);
+			dev_name(&sdev->sdev_gendev), scsi_bsg_sg_io_fn,
+			scsi_bsg_uring_cmd);
 }
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index d3a8cd4166f92..6e8c7a42603e5 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -13,6 +13,7 @@
 #include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/completion.h>
+#include <linux/ctype.h>
 #include <linux/kernel.h>
 #include <linux/export.h>
 #include <linux/init.h>
@@ -3460,6 +3461,52 @@ int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)
 }
 EXPORT_SYMBOL(scsi_vpd_lun_id);
 
+/**
+ * scsi_vpd_lun_serial - return a unique device serial number
+ * @sdev: SCSI device
+ * @sn:   buffer for the serial number
+ * @sn_size: size of the buffer
+ *
+ * Copies the device serial number into @sn based on the information in
+ * the VPD page 0x80 of the device. The string will be null terminated
+ * and have leading and trailing whitespace stripped.
+ *
+ * Returns the length of the serial number or error on failure.
+ */
+int scsi_vpd_lun_serial(struct scsi_device *sdev, char *sn, size_t sn_size)
+{
+	const struct scsi_vpd *vpd_pg80;
+	const unsigned char *d;
+	int len;
+
+	guard(rcu)();
+	vpd_pg80 = rcu_dereference(sdev->vpd_pg80);
+	if (!vpd_pg80)
+		return -ENXIO;
+
+	len = vpd_pg80->len - 4;
+	d = vpd_pg80->data + 4;
+
+	/* Skip leading spaces */
+	while (len > 0 && isspace(*d)) {
+		len--;
+		d++;
+	}
+
+	/* Skip trailing spaces */
+	while (len > 0 && isspace(d[len - 1]))
+		len--;
+
+	if (sn_size < len + 1)
+		return -EINVAL;
+
+	memcpy(sn, d, len);
+	sn[len] = '\0';
+
+	return len;
+}
+EXPORT_SYMBOL(scsi_vpd_lun_serial);
+
 /**
  * scsi_vpd_tpg_id - return a target port group identifier
  * @sdev: SCSI device
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 7b11bc7de0e39..ef22a4228b855 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -1940,7 +1940,6 @@ static void scsi_sysfs_add_devices(struct Scsi_Host *shost)
 static struct async_scan_data *scsi_prep_async_scan(struct Scsi_Host *shost)
 {
 	struct async_scan_data *data = NULL;
-	unsigned long flags;
 
 	if (strncmp(scsi_scan_type, "sync", 4) == 0)
 		return NULL;
@@ -1959,9 +1958,7 @@ static struct async_scan_data *scsi_prep_async_scan(struct Scsi_Host *shost)
 		goto err;
 	init_completion(&data->prev_finished);
 
-	spin_lock_irqsave(shost->host_lock, flags);
-	shost->async_scan = 1;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	shost->async_scan = true;
 	mutex_unlock(&shost->scan_mutex);
 
 	spin_lock(&async_scan_lock);
@@ -1989,7 +1986,6 @@ static struct async_scan_data *scsi_prep_async_scan(struct Scsi_Host *shost)
 static void scsi_finish_async_scan(struct async_scan_data *data)
 {
 	struct Scsi_Host *shost;
-	unsigned long flags;
 
 	if (!data)
 		return;
@@ -2009,9 +2005,7 @@ static void scsi_finish_async_scan(struct async_scan_data *data)
 
 	scsi_sysfs_add_devices(shost);
 
-	spin_lock_irqsave(shost->host_lock, flags);
-	shost->async_scan = 0;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	shost->async_scan = false;
 
 	mutex_unlock(&shost->scan_mutex);
 
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 6b8c5c05f2944..dfc3559e7e04f 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -1051,6 +1051,21 @@ sdev_show_wwid(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR(wwid, S_IRUGO, sdev_show_wwid, NULL);
 
+static ssize_t
+sdev_show_serial(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct scsi_device *sdev = to_scsi_device(dev);
+	ssize_t ret;
+
+	ret = scsi_vpd_lun_serial(sdev, buf, PAGE_SIZE - 1);
+	if (ret < 0)
+		return ret;
+
+	buf[ret] = '\n';
+	return ret + 1;
+}
+static DEVICE_ATTR(serial, S_IRUGO, sdev_show_serial, NULL);
+
 #define BLIST_FLAG_NAME(name)					\
 	[const_ilog2((__force __u64)BLIST_##name)] = #name
 static const char *const sdev_bflags_name[] = {
@@ -1295,6 +1310,7 @@ static struct attribute *scsi_sdev_attrs[] = {
 	&dev_attr_device_busy.attr,
 	&dev_attr_vendor.attr,
 	&dev_attr_model.attr,
+	&dev_attr_serial.attr,
 	&dev_attr_rev.attr,
 	&dev_attr_rescan.attr,
 	&dev_attr_delete.attr,
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 12124f9d5ccd0..13412702188e4 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -1734,7 +1734,7 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel,
 		break;
 
 	default:
-		if (channel < shost->max_channel) {
+		if (channel <= shost->max_channel) {
 			res = scsi_scan_host_selected(shost, channel, id, lun,
 						      SCSI_SCAN_MANUAL);
 		} else {
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 628a1d0a74bac..205877b1f8aae 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -107,8 +107,11 @@ static void sd_config_write_same(struct scsi_disk *sdkp,
 static void  sd_revalidate_disk(struct gendisk *);
 
 static DEFINE_IDA(sd_index_ida);
+static DEFINE_MUTEX(sd_mutex_lock);
 
 static mempool_t *sd_page_pool;
+static mempool_t *sd_large_page_pool;
+static atomic_t sd_large_page_pool_users = ATOMIC_INIT(0);
 static struct lock_class_key sd_bio_compl_lkclass;
 
 static const char *sd_cache_types[] = {
@@ -116,6 +119,33 @@ static const char *sd_cache_types[] = {
 	"write back, no read (daft)"
 };
 
+static int sd_large_pool_create(void)
+{
+	mutex_lock(&sd_mutex_lock);
+	if (!sd_large_page_pool) {
+		sd_large_page_pool = mempool_create_page_pool(
+			SD_MEMPOOL_SIZE, get_order(BLK_MAX_BLOCK_SIZE));
+		if (!sd_large_page_pool) {
+			printk(KERN_ERR "sd: can't create large page mempool\n");
+			mutex_unlock(&sd_mutex_lock);
+			return -ENOMEM;
+		}
+	}
+	atomic_inc(&sd_large_page_pool_users);
+	mutex_unlock(&sd_mutex_lock);
+	return 0;
+}
+
+static void sd_large_pool_destroy(void)
+{
+	mutex_lock(&sd_mutex_lock);
+	if (atomic_dec_and_test(&sd_large_page_pool_users)) {
+		mempool_destroy(sd_large_page_pool);
+		sd_large_page_pool = NULL;
+	}
+	mutex_unlock(&sd_mutex_lock);
+}
+
 static void sd_disable_discard(struct scsi_disk *sdkp)
 {
 	sdkp->provisioning_mode = SD_LBP_DISABLE;
@@ -928,14 +958,24 @@ static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd,
 	return protect;
 }
 
-static void *sd_set_special_bvec(struct request *rq, unsigned int data_len)
+static void *sd_set_special_bvec(struct scsi_cmnd *cmd, unsigned int data_len)
 {
 	struct page *page;
+	struct request *rq = scsi_cmd_to_rq(cmd);
+	struct scsi_device *sdp = cmd->device;
+	unsigned sector_size = sdp->sector_size;
+	unsigned int nr_pages = DIV_ROUND_UP(sector_size, PAGE_SIZE);
+	int n;
 
-	page = mempool_alloc(sd_page_pool, GFP_ATOMIC);
+	if (sector_size > PAGE_SIZE)
+		page = mempool_alloc(sd_large_page_pool, GFP_ATOMIC);
+	else
+		page = mempool_alloc(sd_page_pool, GFP_ATOMIC);
 	if (!page)
 		return NULL;
-	clear_highpage(page);
+
+	for (n = 0; n < nr_pages; n++)
+		clear_highpage(page + n);
 	bvec_set_page(&rq->special_vec, page, data_len, 0);
 	rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
 	return bvec_virt(&rq->special_vec);
@@ -951,7 +991,7 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd)
 	unsigned int data_len = 24;
 	char *buf;
 
-	buf = sd_set_special_bvec(rq, data_len);
+	buf = sd_set_special_bvec(cmd, data_len);
 	if (!buf)
 		return BLK_STS_RESOURCE;
 
@@ -1040,7 +1080,7 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd,
 	u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq));
 	u32 data_len = sdp->sector_size;
 
-	if (!sd_set_special_bvec(rq, data_len))
+	if (!sd_set_special_bvec(cmd, data_len))
 		return BLK_STS_RESOURCE;
 
 	cmd->cmd_len = 16;
@@ -1067,7 +1107,7 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd,
 	u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq));
 	u32 data_len = sdp->sector_size;
 
-	if (!sd_set_special_bvec(rq, data_len))
+	if (!sd_set_special_bvec(cmd, data_len))
 		return BLK_STS_RESOURCE;
 
 	cmd->cmd_len = 10;
@@ -1513,9 +1553,15 @@ static blk_status_t sd_init_command(struct scsi_cmnd *cmd)
 static void sd_uninit_command(struct scsi_cmnd *SCpnt)
 {
 	struct request *rq = scsi_cmd_to_rq(SCpnt);
+	struct scsi_device *sdp = SCpnt->device;
+	unsigned sector_size = sdp->sector_size;
 
-	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
-		mempool_free(rq->special_vec.bv_page, sd_page_pool);
+	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
+		if (sector_size > PAGE_SIZE)
+			mempool_free(rq->special_vec.bv_page, sd_large_page_pool);
+		else
+			mempool_free(rq->special_vec.bv_page, sd_page_pool);
+	}
 }
 
 static bool sd_need_revalidate(struct gendisk *disk, struct scsi_disk *sdkp)
@@ -2912,10 +2958,7 @@ sd_read_capacity(struct scsi_disk *sdkp, struct queue_limits *lim,
 			  "Sector size 0 reported, assuming 512.\n");
 	}
 
-	if (sector_size != 512 &&
-	    sector_size != 1024 &&
-	    sector_size != 2048 &&
-	    sector_size != 4096) {
+	if (blk_validate_block_size(sector_size)) {
 		sd_printk(KERN_NOTICE, sdkp, "Unsupported sector size %d.\n",
 			  sector_size);
 		/*
@@ -4043,6 +4086,12 @@ static int sd_probe(struct scsi_device *sdp)
 	sdkp->max_medium_access_timeouts = SD_MAX_MEDIUM_TIMEOUTS;
 
 	sd_revalidate_disk(gd);
+	if (sdp->sector_size > PAGE_SIZE) {
+		if (sd_large_pool_create()) {
+			error = -ENOMEM;
+			goto out_free_index;
+		}
+	}
 
 	if (sdp->removable) {
 		gd->flags |= GENHD_FL_REMOVABLE;
@@ -4060,6 +4109,8 @@ static int sd_probe(struct scsi_device *sdp)
 	if (error) {
 		device_unregister(&sdkp->disk_dev);
 		put_disk(gd);
+		if (sdp->sector_size > PAGE_SIZE)
+			sd_large_pool_destroy();
 		goto out;
 	}
 
@@ -4212,6 +4263,9 @@ static void sd_remove(struct scsi_device *sdp)
 		sd_shutdown(sdp);
 
 	put_disk(sdkp->disk);
+
+	if (sdp->sector_size > PAGE_SIZE)
+		sd_large_pool_destroy();
 }
 
 static inline bool sd_do_start_stop(struct scsi_device *sdev, bool runtime)
@@ -4435,6 +4489,8 @@ static void __exit exit_sd(void)
 
 	scsi_unregister_driver(&sd_template);
 	mempool_destroy(sd_page_pool);
+	if (sd_large_page_pool)
+		mempool_destroy(sd_large_page_pool);
 
 	class_unregister(&sd_disk_class);
 
diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c
index 8e1686358e25c..4c348645b04ee 100644
--- a/drivers/scsi/ses.c
+++ b/drivers/scsi/ses.c
@@ -215,7 +215,7 @@ static unsigned char *ses_get_page2_descriptor(struct enclosure_device *edev,
 	unsigned char *type_ptr = ses_dev->page1_types;
 	unsigned char *desc_ptr = ses_dev->page2 + 8;
 
-	if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len) < 0)
+	if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len))
 		return NULL;
 
 	for (i = 0; i < ses_dev->page1_num_types; i++, type_ptr += 4) {
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 37bac49f30f0d..2b4b2a1a8e442 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -81,14 +81,14 @@ static int sg_proc_init(void);
 
 #define SG_DEFAULT_TIMEOUT mult_frac(SG_DEFAULT_TIMEOUT_USER, HZ, USER_HZ)
 
-static int sg_big_buff = SG_DEF_RESERVED_SIZE;
 /* N.B. This variable is readable and writeable via
-   /proc/scsi/sg/def_reserved_size . Each time sg_open() is called a buffer
-   of this size (or less if there is not enough memory) will be reserved
-   for use by this file descriptor. [Deprecated usage: this variable is also
-   readable via /proc/sys/kernel/sg-big-buff if the sg driver is built into
-   the kernel (i.e. it is not a module).] */
-static int def_reserved_size = -1;	/* picks up init parameter */
+ * /proc/scsi/sg/def_reserved_size . Each time sg_open() is called a buffer
+ * of this size (or less if there is not enough memory) will be reserved
+ * for use by this file descriptor.
+ */
+
+/* picks up init parameter */
+static int def_reserved_size = SG_DEF_RESERVED_SIZE;
 static int sg_allow_dio = SG_ALLOW_DIO_DEF;
 
 static int scatter_elem_sz = SG_SCATTER_SZ;
@@ -1623,10 +1623,35 @@ sg_remove_device(struct device *cl_dev)
 }
 
 module_param_named(scatter_elem_sz, scatter_elem_sz, int, S_IRUGO | S_IWUSR);
-module_param_named(def_reserved_size, def_reserved_size, int,
-		   S_IRUGO | S_IWUSR);
 module_param_named(allow_dio, sg_allow_dio, int, S_IRUGO | S_IWUSR);
 
+static int def_reserved_size_set(const char *val, const struct kernel_param *kp)
+{
+	int size, ret;
+
+	if (!val)
+		return -EINVAL;
+
+	ret = kstrtoint(val, 0, &size);
+	if (ret)
+		return ret;
+
+	/* limit to 1 MB */
+	if (size < 0 || size > 1048576)
+		return -ERANGE;
+
+	def_reserved_size = size;
+	return 0;
+}
+
+static const struct kernel_param_ops def_reserved_size_ops = {
+	.set	= def_reserved_size_set,
+	.get	= param_get_int,
+};
+
+module_param_cb(def_reserved_size, &def_reserved_size_ops, &def_reserved_size,
+		   S_IRUGO | S_IWUSR);
+
 MODULE_AUTHOR("Douglas Gilbert");
 MODULE_DESCRIPTION("SCSI generic (sg) driver");
 MODULE_LICENSE("GPL");
@@ -1638,35 +1663,6 @@ MODULE_PARM_DESC(scatter_elem_sz, "scatter gather element "
 MODULE_PARM_DESC(def_reserved_size, "size of buffer reserved for each fd");
 MODULE_PARM_DESC(allow_dio, "allow direct I/O (default: 0 (disallow))");
 
-#ifdef CONFIG_SYSCTL
-#include <linux/sysctl.h>
-
-static const struct ctl_table sg_sysctls[] = {
-	{
-		.procname	= "sg-big-buff",
-		.data		= &sg_big_buff,
-		.maxlen		= sizeof(int),
-		.mode		= 0444,
-		.proc_handler	= proc_dointvec,
-	},
-};
-
-static struct ctl_table_header *hdr;
-static void register_sg_sysctls(void)
-{
-	if (!hdr)
-		hdr = register_sysctl("kernel", sg_sysctls);
-}
-
-static void unregister_sg_sysctls(void)
-{
-	unregister_sysctl_table(hdr);
-}
-#else
-#define register_sg_sysctls() do { } while (0)
-#define unregister_sg_sysctls() do { } while (0)
-#endif /* CONFIG_SYSCTL */
-
 static int __init
 init_sg(void)
 {
@@ -1676,10 +1672,6 @@ init_sg(void)
 		scatter_elem_sz = PAGE_SIZE;
 		scatter_elem_sz_prev = scatter_elem_sz;
 	}
-	if (def_reserved_size >= 0)
-		sg_big_buff = def_reserved_size;
-	else
-		def_reserved_size = sg_big_buff;
 
 	rc = register_chrdev_region(MKDEV(SCSI_GENERIC_MAJOR, 0), 
 				    SG_MAX_DEVS, "sg");
@@ -1697,7 +1689,6 @@ init_sg(void)
 		return 0;
 	}
 	class_unregister(&sg_sysfs_class);
-	register_sg_sysctls();
 err_out:
 	unregister_chrdev_region(MKDEV(SCSI_GENERIC_MAJOR, 0), SG_MAX_DEVS);
 	return rc;
@@ -1706,7 +1697,6 @@ init_sg(void)
 static void __exit
 exit_sg(void)
 {
-	unregister_sg_sysctls();
 #ifdef CONFIG_SCSI_PROC_FS
 	remove_proc_subtree("scsi/sg", NULL);
 #endif				/* CONFIG_SCSI_PROC_FS */
@@ -2182,10 +2172,8 @@ sg_add_sfp(Sg_device * sdp)
 	write_unlock_irqrestore(&sdp->sfd_lock, iflags);
 	SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sdp,
 				      "sg_add_sfp: sfp=0x%p\n", sfp));
-	if (unlikely(sg_big_buff != def_reserved_size))
-		sg_big_buff = def_reserved_size;
 
-	bufflen = min_t(int, sg_big_buff,
+	bufflen = min_t(int, def_reserved_size,
 			max_sectors_bytes(sdp->device->request_queue));
 	sg_build_reserve(sfp, bufflen);
 	SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sdp,
@@ -2413,7 +2401,7 @@ sg_proc_write_adio(struct file *filp, const char __user *buffer,
 
 static int sg_proc_single_open_dressz(struct inode *inode, struct file *file)
 {
-	return single_open(file, sg_proc_seq_show_int, &sg_big_buff);
+	return single_open(file, sg_proc_seq_show_int, &def_reserved_size);
 }
 
 static ssize_t 
@@ -2430,7 +2418,7 @@ sg_proc_write_dressz(struct file *filp, const char __user *buffer,
 	if (err)
 		return err;
 	if (k <= 1048576) {	/* limit "big buff" to 1 MB */
-		sg_big_buff = k;
+		def_reserved_size = k;
 		return count;
 	}
 	return -ERANGE;
@@ -2603,7 +2591,7 @@ static int sg_proc_seq_show_debug(struct seq_file *s, void *v)
 
 	if (it && (0 == it->index))
 		seq_printf(s, "max_active_device=%d  def_reserved_size=%d\n",
-			   (int)it->max, sg_big_buff);
+			   (int)it->max, def_reserved_size);
 
 	read_lock_irqsave(&sg_index_lock, iflags);
 	sdp = it ? sg_lookup_dev(it->index) : NULL;
diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c
index 9aa7218b4e8d2..1ed6be6e85d2c 100644
--- a/drivers/slimbus/qcom-ngd-ctrl.c
+++ b/drivers/slimbus/qcom-ngd-ctrl.c
@@ -1535,10 +1535,8 @@ static int of_qcom_slim_ngd_register(struct device *parent,
 		ngd->id = id;
 		ngd->pdev->dev.parent = parent;
 
-		ret = driver_set_override(&ngd->pdev->dev,
-					  &ngd->pdev->driver_override,
-					  QCOM_SLIM_NGD_DRV_NAME,
-					  strlen(QCOM_SLIM_NGD_DRV_NAME));
+		ret = device_set_driver_override(&ngd->pdev->dev,
+						 QCOM_SLIM_NGD_DRV_NAME);
 		if (ret) {
 			platform_device_put(ngd->pdev);
 			kfree(ngd);
diff --git a/drivers/soc/aspeed/aspeed-socinfo.c b/drivers/soc/aspeed/aspeed-socinfo.c
index 5e34e01ad26de..fb8fde94b6513 100644
--- a/drivers/soc/aspeed/aspeed-socinfo.c
+++ b/drivers/soc/aspeed/aspeed-socinfo.c
@@ -39,7 +39,7 @@ static const char *siliconid_to_name(u32 siliconid)
 	unsigned int i;
 
 	for (i = 0 ; i < ARRAY_SIZE(rev_table) ; ++i) {
-		if (rev_table[i].id == id)
+		if ((rev_table[i].id & 0xff00ffff) == id)
 			return rev_table[i].name;
 	}
 
diff --git a/drivers/soc/fsl/guts.c b/drivers/soc/fsl/guts.c
index 40afb27b582b1..9bee7baec2b9b 100644
--- a/drivers/soc/fsl/guts.c
+++ b/drivers/soc/fsl/guts.c
@@ -186,7 +186,6 @@ static int __init fsl_guts_init(void)
 	const struct fsl_soc_data *soc_data;
 	const struct of_device_id *match;
 	struct ccsr_guts __iomem *regs;
-	const char *machine = NULL;
 	struct device_node *np;
 	bool little_endian;
 	u64 soc_uid = 0;
@@ -217,13 +216,9 @@ static int __init fsl_guts_init(void)
 	if (!soc_dev_attr)
 		return -ENOMEM;
 
-	if (of_property_read_string(of_root, "model", &machine))
-		of_property_read_string_index(of_root, "compatible", 0, &machine);
-	if (machine) {
-		soc_dev_attr->machine = kstrdup(machine, GFP_KERNEL);
-		if (!soc_dev_attr->machine)
-			goto err_nomem;
-	}
+	ret = soc_attr_read_machine(soc_dev_attr);
+	if (ret)
+		of_machine_read_compatible(&soc_dev_attr->machine, 0);
 
 	soc_die = fsl_soc_die_match(svr, fsl_soc_die);
 	if (soc_die) {
@@ -267,7 +262,6 @@ static int __init fsl_guts_init(void)
 err_nomem:
 	ret = -ENOMEM;
 err:
-	kfree(soc_dev_attr->machine);
 	kfree(soc_dev_attr->family);
 	kfree(soc_dev_attr->soc_id);
 	kfree(soc_dev_attr->revision);
diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c
index 411381f1a1c4b..9ddafcb18f1c7 100644
--- a/drivers/soc/fsl/qbman/qman.c
+++ b/drivers/soc/fsl/qbman/qman.c
@@ -1827,6 +1827,8 @@ EXPORT_SYMBOL(qman_create_fq);
 
 void qman_destroy_fq(struct qman_fq *fq)
 {
+	int leaked;
+
 	/*
 	 * We don't need to lock the FQ as it is a pre-condition that the FQ be
 	 * quiesced. Instead, run some checks.
@@ -1834,11 +1836,29 @@ void qman_destroy_fq(struct qman_fq *fq)
 	switch (fq->state) {
 	case qman_fq_state_parked:
 	case qman_fq_state_oos:
-		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
-			qman_release_fqid(fq->fqid);
+		/*
+		 * There's a race condition here on releasing the fqid,
+		 * setting the fq_table to NULL, and freeing the fqid.
+		 * To prevent it, this order should be respected:
+		 */
+		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) {
+			leaked = qman_shutdown_fq(fq->fqid);
+			if (leaked)
+				pr_debug("FQID %d leaked\n", fq->fqid);
+		}
 
 		DPAA_ASSERT(fq_table[fq->idx]);
 		fq_table[fq->idx] = NULL;
+
+		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID) && !leaked) {
+			/*
+			 * fq_table[fq->idx] should be set to null before
+			 * freeing fq->fqid otherwise it could by allocated by
+			 * qman_alloc_fqid() while still being !NULL
+			 */
+			smp_wmb();
+			gen_pool_free(qm_fqalloc, fq->fqid | DPAA_GENALLOC_OFF, 1);
+		}
 		return;
 	default:
 		break;
diff --git a/drivers/soc/fsl/qe/qmc.c b/drivers/soc/fsl/qe/qmc.c
index c4587b32a59bf..672adff8e35f4 100644
--- a/drivers/soc/fsl/qe/qmc.c
+++ b/drivers/soc/fsl/qe/qmc.c
@@ -1790,8 +1790,8 @@ static int qmc_qe_init_resources(struct qmc *qmc, struct platform_device *pdev)
 		return -EINVAL;
 	qmc->dpram_offset = res->start - qe_muram_dma(qe_muram_addr(0));
 	qmc->dpram = devm_ioremap_resource(qmc->dev, res);
-	if (IS_ERR(qmc->scc_pram))
-		return PTR_ERR(qmc->scc_pram);
+	if (IS_ERR(qmc->dpram))
+		return PTR_ERR(qmc->dpram);
 
 	return 0;
 }
diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c
index 8e2322999f099..77763a107edbd 100644
--- a/drivers/soc/imx/soc-imx8m.c
+++ b/drivers/soc/imx/soc-imx8m.c
@@ -226,7 +226,6 @@ static int imx8m_soc_probe(struct platform_device *pdev)
 	const struct imx8_soc_data *data;
 	struct imx8_soc_drvdata *drvdata;
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *id;
 	struct soc_device *soc_dev;
 	u32 soc_rev = 0;
 	u64 soc_uid[2] = {0, 0};
@@ -244,15 +243,11 @@ static int imx8m_soc_probe(struct platform_device *pdev)
 
 	soc_dev_attr->family = "Freescale i.MX";
 
-	ret = of_property_read_string(of_root, "model", &soc_dev_attr->machine);
+	ret = soc_attr_read_machine(soc_dev_attr);
 	if (ret)
 		return ret;
 
-	id = of_match_node(imx8_soc_match, of_root);
-	if (!id)
-		return -ENODEV;
-
-	data = id->data;
+	data = device_get_match_data(dev);
 	if (data) {
 		soc_dev_attr->soc_id = data->name;
 		ret = imx8m_soc_prepare(pdev, data->ocotp_compatible);
@@ -326,7 +321,7 @@ static int __init imx8_soc_init(void)
 	int ret;
 
 	/* No match means this is non-i.MX8M hardware, do nothing. */
-	if (!of_match_node(imx8_soc_match, of_root))
+	if (!of_machine_device_match(imx8_soc_match))
 		return 0;
 
 	ret = platform_driver_register(&imx8m_soc_driver);
diff --git a/drivers/soc/imx/soc-imx9.c b/drivers/soc/imx/soc-imx9.c
index d67bc7402b10e..58eef7d4f9089 100644
--- a/drivers/soc/imx/soc-imx9.c
+++ b/drivers/soc/imx/soc-imx9.c
@@ -30,7 +30,7 @@ static int imx9_soc_probe(struct platform_device *pdev)
 	if (!attr)
 		return -ENOMEM;
 
-	err = of_property_read_string(of_root, "model", &attr->machine);
+	err = soc_attr_read_machine(attr);
 	if (err)
 		return dev_err_probe(dev, err, "%s: missing model property\n", __func__);
 
@@ -89,7 +89,7 @@ static int __init imx9_soc_init(void)
 	struct platform_device *pdev;
 
 	/* No match means it is not an i.MX 9 series SoC, do nothing. */
-	if (!of_match_node(imx9_soc_match, of_root))
+	if (!of_machine_device_match(imx9_soc_match))
 		return 0;
 
 	ret = platform_driver_register(&imx9_soc_driver);
diff --git a/drivers/soc/microchip/mpfs-control-scb.c b/drivers/soc/microchip/mpfs-control-scb.c
index f0b84b1f49cbc..8dda5704a389f 100644
--- a/drivers/soc/microchip/mpfs-control-scb.c
+++ b/drivers/soc/microchip/mpfs-control-scb.c
@@ -14,8 +14,10 @@ static int mpfs_control_scb_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 
-	return mfd_add_devices(dev, PLATFORM_DEVID_NONE, mpfs_control_scb_devs,
-			       ARRAY_SIZE(mpfs_control_scb_devs), NULL, 0, NULL);
+	return devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE,
+				    mpfs_control_scb_devs,
+				    ARRAY_SIZE(mpfs_control_scb_devs), NULL, 0,
+				    NULL);
 }
 
 static const struct of_device_id mpfs_control_scb_of_match[] = {
diff --git a/drivers/soc/microchip/mpfs-mss-top-sysreg.c b/drivers/soc/microchip/mpfs-mss-top-sysreg.c
index b2244e44ff0fa..b0f42b8dd3ed6 100644
--- a/drivers/soc/microchip/mpfs-mss-top-sysreg.c
+++ b/drivers/soc/microchip/mpfs-mss-top-sysreg.c
@@ -16,8 +16,10 @@ static int mpfs_mss_top_sysreg_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	int ret;
 
-	ret = mfd_add_devices(dev, PLATFORM_DEVID_NONE, mpfs_mss_top_sysreg_devs,
-			      ARRAY_SIZE(mpfs_mss_top_sysreg_devs) , NULL, 0, NULL);
+	ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE,
+				   mpfs_mss_top_sysreg_devs,
+				   ARRAY_SIZE(mpfs_mss_top_sysreg_devs), NULL,
+				   0, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/soc/microchip/mpfs-sys-controller.c b/drivers/soc/microchip/mpfs-sys-controller.c
index 8e7ae3cb92fff..92d1142a59e68 100644
--- a/drivers/soc/microchip/mpfs-sys-controller.c
+++ b/drivers/soc/microchip/mpfs-sys-controller.c
@@ -36,6 +36,11 @@ struct mpfs_sys_controller {
 	struct kref consumers;
 };
 
+struct mpfs_syscon_config {
+	unsigned int nb_subdevs;
+	struct platform_device *subdevs;
+};
+
 int mpfs_blocking_transaction(struct mpfs_sys_controller *sys_controller, struct mpfs_mss_msg *msg)
 {
 	unsigned long timeout = msecs_to_jiffies(MPFS_SYS_CTRL_TIMEOUT_MS);
@@ -110,25 +115,11 @@ struct mtd_info *mpfs_sys_controller_get_flash(struct mpfs_sys_controller *mpfs_
 }
 EXPORT_SYMBOL(mpfs_sys_controller_get_flash);
 
-static struct platform_device subdevs[] = {
-	{
-		.name		= "mpfs-rng",
-		.id		= -1,
-	},
-	{
-		.name		= "mpfs-generic-service",
-		.id		= -1,
-	},
-	{
-		.name		= "mpfs-auto-update",
-		.id		= -1,
-	},
-};
-
 static int mpfs_sys_controller_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct mpfs_sys_controller *sys_controller;
+	struct mpfs_syscon_config *of_data;
 	struct device_node *np;
 	int i, ret;
 
@@ -142,8 +133,10 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 
 	sys_controller->flash = of_get_mtd_device_by_node(np);
 	of_node_put(np);
-	if (IS_ERR(sys_controller->flash))
-		return dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n");
+	if (IS_ERR(sys_controller->flash)) {
+		ret = dev_err_probe(dev, PTR_ERR(sys_controller->flash), "Failed to get flash\n");
+		goto out_free;
+	}
 
 no_flash:
 	sys_controller->client.dev = dev;
@@ -155,8 +148,7 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 	if (IS_ERR(sys_controller->chan)) {
 		ret = dev_err_probe(dev, PTR_ERR(sys_controller->chan),
 				    "Failed to get mbox channel\n");
-		kfree(sys_controller);
-		return ret;
+		goto out_free;
 	}
 
 	init_completion(&sys_controller->c);
@@ -164,16 +156,26 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sys_controller);
 
+	of_data = (struct mpfs_syscon_config *) device_get_match_data(dev);
+	if (!of_data) {
+		dev_err(dev, "Error getting match data\n");
+		return -EINVAL;
+	}
 
-	for (i = 0; i < ARRAY_SIZE(subdevs); i++) {
-		subdevs[i].dev.parent = dev;
-		if (platform_device_register(&subdevs[i]))
-			dev_warn(dev, "Error registering sub device %s\n", subdevs[i].name);
+	for (i = 0; i < of_data->nb_subdevs; i++) {
+		of_data->subdevs[i].dev.parent = dev;
+		if (platform_device_register(&of_data->subdevs[i]))
+			dev_warn(dev, "Error registering sub device %s\n",
+				 of_data->subdevs[i].name);
 	}
 
 	dev_info(&pdev->dev, "Registered MPFS system controller\n");
 
 	return 0;
+
+out_free:
+	kfree(sys_controller);
+	return ret;
 }
 
 static void mpfs_sys_controller_remove(struct platform_device *pdev)
@@ -183,8 +185,45 @@ static void mpfs_sys_controller_remove(struct platform_device *pdev)
 	mpfs_sys_controller_put(sys_controller);
 }
 
+static struct platform_device mpfs_subdevs[] = {
+	{
+		.name		= "mpfs-rng",
+		.id		= -1,
+	},
+	{
+		.name		= "mpfs-generic-service",
+		.id		= -1,
+	},
+	{
+		.name		= "mpfs-auto-update",
+		.id		= -1,
+	},
+};
+
+static struct platform_device pic64gx_subdevs[] = {
+	{
+		.name		= "mpfs-rng",
+		.id		= -1,
+	},
+	{
+		.name		= "mpfs-generic-service",
+		.id		= -1,
+	},
+};
+
+static const struct mpfs_syscon_config mpfs_config = {
+	.nb_subdevs = ARRAY_SIZE(mpfs_subdevs),
+	.subdevs = mpfs_subdevs,
+};
+
+static const struct mpfs_syscon_config pic64gx_config = {
+	.nb_subdevs = ARRAY_SIZE(pic64gx_subdevs),
+	.subdevs = pic64gx_subdevs,
+};
+
 static const struct of_device_id mpfs_sys_controller_of_match[] = {
-	{.compatible = "microchip,mpfs-sys-controller", },
+	{.compatible = "microchip,mpfs-sys-controller", .data = &mpfs_config},
+	{.compatible = "microchip,pic64gx-sys-controller", .data = &pic64gx_config},
 	{},
 };
 MODULE_DEVICE_TABLE(of, mpfs_sys_controller_of_match);
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index ad5899d083f3f..68a3416692e0a 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -5,7 +5,6 @@
  */
 
 #include <linux/bitfield.h>
-#include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <linux/cleanup.h>
 #include <linux/device.h>
@@ -1782,6 +1781,94 @@ static const struct llcc_slice_config sc8280xp_data[] = {
 	},
 };
 
+static const struct llcc_slice_config sdm670_data[] = {
+	{
+		.usecase_id = LLCC_CPUSS,
+		.slice_id = 1,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_ROTATOR,
+		.slice_id = 4,
+		.max_cap = 384,
+		.priority = 2,
+		.fixed_size = true,
+		.bonus_ways = 0x0,
+		.res_ways = 0xe,
+		.cache_mode = 2,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	}, {
+		.usecase_id = LLCC_VOICE,
+		.slice_id = 5,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	}, {
+		.usecase_id = LLCC_AUDIO,
+		.slice_id = 6,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	}, {
+		.usecase_id = LLCC_MDM,
+		.slice_id = 8,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	}, {
+		.usecase_id = LLCC_GPU,
+		.slice_id = 12,
+		.max_cap = 384,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0x0,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	}, {
+		.usecase_id = LLCC_MMUHWT,
+		.slice_id = 13,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_AUDHW,
+		.slice_id = 22,
+		.max_cap = 512,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xf,
+		.res_ways = 0x0,
+		.cache_mode = 0,
+		.dis_cap_alloc = true,
+		.retain_on_pc = true,
+	},
+};
+
 static const struct llcc_slice_config sdm845_data[] =  {{
 		.usecase_id = LLCC_CPUSS,
 		.slice_id = 1,
@@ -4196,6 +4283,17 @@ static const struct qcom_llcc_config sc8280xp_cfg[] = {
 	},
 };
 
+static const struct qcom_llcc_config sdm670_cfg[] = {
+	{
+		.sct_data	= sdm670_data,
+		.size		= ARRAY_SIZE(sdm670_data),
+		.skip_llcc_cfg	= true,
+		.reg_offset	= llcc_v1_reg_offset,
+		.edac_reg_offset = &llcc_v1_edac_reg_offset,
+		.no_edac	= true,
+	},
+};
+
 static const struct qcom_llcc_config sdm845_cfg[] = {
 	{
 		.sct_data	= sdm845_data,
@@ -4364,6 +4462,11 @@ static const struct qcom_sct_config sc8280xp_cfgs = {
 	.num_config	= ARRAY_SIZE(sc8280xp_cfg),
 };
 
+static const struct qcom_sct_config sdm670_cfgs = {
+	.llcc_config	= sdm670_cfg,
+	.num_config	= ARRAY_SIZE(sdm670_cfg),
+};
+
 static const struct qcom_sct_config sdm845_cfgs = {
 	.llcc_config	= sdm845_cfg,
 	.num_config	= ARRAY_SIZE(sdm845_cfg),
@@ -4431,8 +4534,7 @@ static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER;
 struct llcc_slice_desc *llcc_slice_getd(u32 uid)
 {
 	const struct llcc_slice_config *cfg;
-	struct llcc_slice_desc *desc;
-	u32 sz, count;
+	u32 sz, i;
 
 	if (IS_ERR(drv_data))
 		return ERR_CAST(drv_data);
@@ -4440,21 +4542,14 @@ struct llcc_slice_desc *llcc_slice_getd(u32 uid)
 	cfg = drv_data->cfg;
 	sz = drv_data->cfg_size;
 
-	for (count = 0; cfg && count < sz; count++, cfg++)
+	for (i = 0; cfg && i < sz; i++, cfg++)
 		if (cfg->usecase_id == uid)
 			break;
 
-	if (count == sz || !cfg)
+	if (i == sz)
 		return ERR_PTR(-ENODEV);
 
-	desc = kzalloc_obj(*desc);
-	if (!desc)
-		return ERR_PTR(-ENOMEM);
-
-	desc->slice_id = cfg->slice_id;
-	desc->slice_size = cfg->max_cap;
-
-	return desc;
+	return &drv_data->desc[i];
 }
 EXPORT_SYMBOL_GPL(llcc_slice_getd);
 
@@ -4465,7 +4560,7 @@ EXPORT_SYMBOL_GPL(llcc_slice_getd);
 void llcc_slice_putd(struct llcc_slice_desc *desc)
 {
 	if (!IS_ERR_OR_NULL(desc))
-		kfree(desc);
+		return;
 }
 EXPORT_SYMBOL_GPL(llcc_slice_putd);
 
@@ -4540,25 +4635,21 @@ int llcc_slice_activate(struct llcc_slice_desc *desc)
 	if (IS_ERR_OR_NULL(desc))
 		return -EINVAL;
 
-	mutex_lock(&drv_data->lock);
-	if (test_bit(desc->slice_id, drv_data->bitmap)) {
-		mutex_unlock(&drv_data->lock);
+	guard(mutex)(&drv_data->lock);
+	/* Already active; try to take another reference. */
+	if (refcount_inc_not_zero(&desc->refcount))
 		return 0;
-	}
 
 	act_ctrl_val = ACT_CTRL_OPCODE_ACTIVATE << ACT_CTRL_OPCODE_SHIFT;
-
 	ret = llcc_update_act_ctrl(desc->slice_id, act_ctrl_val,
 				  DEACTIVATE);
-	if (ret) {
-		mutex_unlock(&drv_data->lock);
+	if (ret)
 		return ret;
-	}
 
-	__set_bit(desc->slice_id, drv_data->bitmap);
-	mutex_unlock(&drv_data->lock);
+	/* Set first reference */
+	refcount_set(&desc->refcount, 1);
 
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(llcc_slice_activate);
 
@@ -4580,24 +4671,21 @@ int llcc_slice_deactivate(struct llcc_slice_desc *desc)
 	if (IS_ERR_OR_NULL(desc))
 		return -EINVAL;
 
-	mutex_lock(&drv_data->lock);
-	if (!test_bit(desc->slice_id, drv_data->bitmap)) {
-		mutex_unlock(&drv_data->lock);
+	guard(mutex)(&drv_data->lock);
+	/* refcount > 1, drop one ref and we’re done. */
+	if (refcount_dec_not_one(&desc->refcount))
 		return 0;
-	}
-	act_ctrl_val = ACT_CTRL_OPCODE_DEACTIVATE << ACT_CTRL_OPCODE_SHIFT;
 
+	act_ctrl_val = ACT_CTRL_OPCODE_DEACTIVATE << ACT_CTRL_OPCODE_SHIFT;
 	ret = llcc_update_act_ctrl(desc->slice_id, act_ctrl_val,
 				  ACTIVATE);
-	if (ret) {
-		mutex_unlock(&drv_data->lock);
+	if (ret)
 		return ret;
-	}
 
-	__clear_bit(desc->slice_id, drv_data->bitmap);
-	mutex_unlock(&drv_data->lock);
+	/* Finalize: atomically transition 1 -> 0 */
+	WARN_ON_ONCE(!refcount_dec_if_one(&desc->refcount));
 
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(llcc_slice_deactivate);
 
@@ -4638,7 +4726,7 @@ static int _qcom_llcc_cfg_program(const struct llcc_slice_config *config,
 	u32 attr1_val;
 	u32 attr0_val;
 	u32 max_cap_cacheline;
-	struct llcc_slice_desc desc;
+	struct llcc_slice_desc *desc;
 
 	attr1_val = config->cache_mode;
 	attr1_val |= config->probe_target_ways << ATTR1_PROBE_TARGET_WAYS_SHIFT;
@@ -4787,8 +4875,11 @@ static int _qcom_llcc_cfg_program(const struct llcc_slice_config *config,
 	}
 
 	if (config->activate_on_init) {
-		desc.slice_id = config->slice_id;
-		ret = llcc_slice_activate(&desc);
+		desc = llcc_slice_getd(config->usecase_id);
+		if (IS_ERR(desc))
+			return PTR_ERR(desc);
+
+		ret = llcc_slice_activate(desc);
 	}
 
 	return ret;
@@ -5101,18 +5192,18 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 
 	llcc_cfg = cfg->sct_data;
 	sz = cfg->size;
-
-	for (i = 0; i < sz; i++)
-		if (llcc_cfg[i].slice_id > drv_data->max_slices)
-			drv_data->max_slices = llcc_cfg[i].slice_id;
-
-	drv_data->bitmap = devm_bitmap_zalloc(dev, drv_data->max_slices,
-					      GFP_KERNEL);
-	if (!drv_data->bitmap) {
+	drv_data->desc = devm_kcalloc(dev, sz, sizeof(struct llcc_slice_desc), GFP_KERNEL);
+	if (!drv_data->desc) {
 		ret = -ENOMEM;
 		goto err;
 	}
 
+	for (i = 0; i < sz; i++) {
+		drv_data->desc[i].slice_id = llcc_cfg[i].slice_id;
+		drv_data->desc[i].slice_size = llcc_cfg[i].max_cap;
+		refcount_set(&drv_data->desc[i].refcount, 0);
+	}
+
 	drv_data->cfg = llcc_cfg;
 	drv_data->cfg_size = sz;
 	drv_data->edac_reg_offset = cfg->edac_reg_offset;
@@ -5160,6 +5251,7 @@ static const struct of_device_id qcom_llcc_of_match[] = {
 	{ .compatible = "qcom,sc7280-llcc", .data = &sc7280_cfgs },
 	{ .compatible = "qcom,sc8180x-llcc", .data = &sc8180x_cfgs },
 	{ .compatible = "qcom,sc8280xp-llcc", .data = &sc8280xp_cfgs },
+	{ .compatible = "qcom,sdm670-llcc", .data = &sdm670_cfgs },
 	{ .compatible = "qcom,sdm845-llcc", .data = &sdm845_cfgs },
 	{ .compatible = "qcom,sm6350-llcc", .data = &sm6350_cfgs },
 	{ .compatible = "qcom,sm7150-llcc", .data = &sm7150_cfgs },
diff --git a/drivers/soc/qcom/pdr_interface.c b/drivers/soc/qcom/pdr_interface.c
index 72259f489075f..6d879e1540b0a 100644
--- a/drivers/soc/qcom/pdr_interface.c
+++ b/drivers/soc/qcom/pdr_interface.c
@@ -523,7 +523,7 @@ struct pdr_service *pdr_add_lookup(struct pdr_handle *pdr,
 	if (!pds)
 		return ERR_PTR(-ENOMEM);
 
-	pds->service = SERVREG_NOTIFIER_SERVICE;
+	pds->service = QMI_SERVICE_ID_SERVREG_NOTIF;
 	strscpy(pds->service_name, service_name, sizeof(pds->service_name));
 	strscpy(pds->service_path, service_path, sizeof(pds->service_path));
 	pds->need_locator_lookup = true;
@@ -678,7 +678,7 @@ struct pdr_handle *pdr_handle_alloc(void (*status)(int state,
 	if (ret < 0)
 		goto destroy_indack;
 
-	ret = qmi_add_lookup(&pdr->locator_hdl, SERVREG_LOCATOR_SERVICE, 1, 1);
+	ret = qmi_add_lookup(&pdr->locator_hdl, QMI_SERVICE_ID_SERVREG_LOC, 1, 1);
 	if (ret < 0)
 		goto release_qmi_handle;
 
diff --git a/drivers/soc/qcom/pdr_internal.h b/drivers/soc/qcom/pdr_internal.h
index 039508c1bbf7d..d867e9b5e973c 100644
--- a/drivers/soc/qcom/pdr_internal.h
+++ b/drivers/soc/qcom/pdr_internal.h
@@ -4,9 +4,6 @@
 
 #include <linux/soc/qcom/pdr.h>
 
-#define SERVREG_LOCATOR_SERVICE				0x40
-#define SERVREG_NOTIFIER_SERVICE			0x42
-
 #define SERVREG_REGISTER_LISTENER_REQ			0x20
 #define SERVREG_GET_DOMAIN_LIST_REQ			0x21
 #define SERVREG_STATE_UPDATED_IND_ID			0x22
@@ -84,7 +81,7 @@ struct servreg_set_ack_resp {
 
 struct servreg_loc_pfr_req {
 	char service[SERVREG_NAME_LENGTH + 1];
-	char reason[257];
+	char reason[SERVREG_PFR_LENGTH + 1];
 };
 
 struct servreg_loc_pfr_resp {
diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c
index 627f96ca322e1..3042261578aad 100644
--- a/drivers/soc/qcom/pmic_glink.c
+++ b/drivers/soc/qcom/pmic_glink.c
@@ -23,13 +23,19 @@ enum {
 	PMIC_GLINK_CLIENT_UCSI,
 };
 
+struct pmic_glink_data {
+	unsigned long	client_mask;
+	const char	*charger_pdr_service_name;
+	const char	*charger_pdr_service_path;
+};
+
 struct pmic_glink {
 	struct device *dev;
 	struct pdr_handle *pdr;
 
 	struct rpmsg_endpoint *ept;
 
-	unsigned long client_mask;
+	const struct pmic_glink_data *data;
 
 	struct auxiliary_device altmode_aux;
 	struct auxiliary_device ps_aux;
@@ -292,7 +298,6 @@ static struct rpmsg_driver pmic_glink_rpmsg_driver = {
 
 static int pmic_glink_probe(struct platform_device *pdev)
 {
-	const unsigned long *match_data;
 	struct pdr_service *service;
 	struct pmic_glink *pg;
 	int ret;
@@ -309,12 +314,10 @@ static int pmic_glink_probe(struct platform_device *pdev)
 	spin_lock_init(&pg->client_lock);
 	mutex_init(&pg->state_lock);
 
-	match_data = (unsigned long *)of_device_get_match_data(&pdev->dev);
-	if (!match_data)
+	pg->data = of_device_get_match_data(&pdev->dev);
+	if (!pg->data)
 		return -EINVAL;
 
-	pg->client_mask = *match_data;
-
 	pg->pdr = pdr_handle_alloc(pmic_glink_pdr_callback, pg);
 	if (IS_ERR(pg->pdr)) {
 		ret = dev_err_probe(&pdev->dev, PTR_ERR(pg->pdr),
@@ -322,27 +325,30 @@ static int pmic_glink_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) {
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI)) {
 		ret = pmic_glink_add_aux_device(pg, &pg->ucsi_aux, "ucsi");
 		if (ret)
 			goto out_release_pdr_handle;
 	}
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) {
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE)) {
 		ret = pmic_glink_add_aux_device(pg, &pg->altmode_aux, "altmode");
 		if (ret)
 			goto out_release_ucsi_aux;
 	}
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) {
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT)) {
 		ret = pmic_glink_add_aux_device(pg, &pg->ps_aux, "power-supply");
 		if (ret)
 			goto out_release_altmode_aux;
 	}
 
-	service = pdr_add_lookup(pg->pdr, "tms/servreg", "msm/adsp/charger_pd");
-	if (IS_ERR(service)) {
-		ret = dev_err_probe(&pdev->dev, PTR_ERR(service),
-				    "failed adding pdr lookup for charger_pd\n");
-		goto out_release_aux_devices;
+	if (pg->data->charger_pdr_service_name && pg->data->charger_pdr_service_path) {
+		service = pdr_add_lookup(pg->pdr, pg->data->charger_pdr_service_name,
+					 pg->data->charger_pdr_service_path);
+		if (IS_ERR(service)) {
+			ret = dev_err_probe(&pdev->dev, PTR_ERR(service),
+					    "failed adding pdr lookup for charger_pd\n");
+			goto out_release_aux_devices;
+		}
 	}
 
 	mutex_lock(&__pmic_glink_lock);
@@ -352,13 +358,13 @@ static int pmic_glink_probe(struct platform_device *pdev)
 	return 0;
 
 out_release_aux_devices:
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
 		pmic_glink_del_aux_device(pg, &pg->ps_aux);
 out_release_altmode_aux:
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
 		pmic_glink_del_aux_device(pg, &pg->altmode_aux);
 out_release_ucsi_aux:
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
 		pmic_glink_del_aux_device(pg, &pg->ucsi_aux);
 out_release_pdr_handle:
 	pdr_handle_release(pg->pdr);
@@ -372,23 +378,35 @@ static void pmic_glink_remove(struct platform_device *pdev)
 
 	pdr_handle_release(pg->pdr);
 
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_BATT))
 		pmic_glink_del_aux_device(pg, &pg->ps_aux);
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_ALTMODE))
 		pmic_glink_del_aux_device(pg, &pg->altmode_aux);
-	if (pg->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
+	if (pg->data->client_mask & BIT(PMIC_GLINK_CLIENT_UCSI))
 		pmic_glink_del_aux_device(pg, &pg->ucsi_aux);
 
 	guard(mutex)(&__pmic_glink_lock);
 	__pmic_glink = NULL;
 }
 
-static const unsigned long pmic_glink_sm8450_client_mask = BIT(PMIC_GLINK_CLIENT_BATT) |
-							   BIT(PMIC_GLINK_CLIENT_ALTMODE) |
-							   BIT(PMIC_GLINK_CLIENT_UCSI);
+static const struct pmic_glink_data pmic_glink_adsp_data = {
+	.client_mask = BIT(PMIC_GLINK_CLIENT_BATT) |
+		       BIT(PMIC_GLINK_CLIENT_ALTMODE) |
+		       BIT(PMIC_GLINK_CLIENT_UCSI),
+	.charger_pdr_service_name = "tms/servreg",
+	.charger_pdr_service_path = "msm/adsp/charger_pd",
+};
+
+static const struct pmic_glink_data pmic_glink_soccp_data = {
+	.client_mask = BIT(PMIC_GLINK_CLIENT_BATT) |
+		       BIT(PMIC_GLINK_CLIENT_ALTMODE) |
+		       BIT(PMIC_GLINK_CLIENT_UCSI),
+};
 
 static const struct of_device_id pmic_glink_of_match[] = {
-	{ .compatible = "qcom,pmic-glink", .data = &pmic_glink_sm8450_client_mask },
+	{ .compatible = "qcom,glymur-pmic-glink", .data = &pmic_glink_soccp_data },
+	{ .compatible = "qcom,kaanapali-pmic-glink", .data = &pmic_glink_soccp_data },
+	{ .compatible = "qcom,pmic-glink", .data = &pmic_glink_adsp_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, pmic_glink_of_match);
diff --git a/drivers/soc/qcom/pmic_glink_altmode.c b/drivers/soc/qcom/pmic_glink_altmode.c
index d0afdcb96ee11..619bad2c27eeb 100644
--- a/drivers/soc/qcom/pmic_glink_altmode.c
+++ b/drivers/soc/qcom/pmic_glink_altmode.c
@@ -62,6 +62,9 @@ struct usbc_notify {
 	u8 orientation;
 	u8 mux_ctrl;
 #define MUX_CTRL_STATE_NO_CONN			0
+#define MUX_CTRL_STATE_USB3_ONLY		1
+#define MUX_CTRL_STATE_DP4LN			2
+#define MUX_CTRL_STATE_USB3_DP			3
 #define MUX_CTRL_STATE_TUNNELING		4
 
 	u8 res;
@@ -350,15 +353,20 @@ static void pmic_glink_altmode_worker(struct work_struct *work)
 
 	typec_switch_set(alt_port->typec_switch, alt_port->orientation);
 
-	if (alt_port->mux_ctrl == MUX_CTRL_STATE_NO_CONN) {
-		pmic_glink_altmode_safe(altmode, alt_port);
-	} else if (alt_port->svid == USB_TYPEC_TBT_SID) {
-		pmic_glink_altmode_enable_tbt(altmode, alt_port);
-	} else if (alt_port->svid == USB_TYPEC_DP_SID) {
-		pmic_glink_altmode_enable_dp(altmode, alt_port,
-					     alt_port->mode,
-					     alt_port->hpd_state,
-					     alt_port->hpd_irq);
+	/*
+	 * MUX_CTRL_STATE_DP4LN/USB3_DP may only be set if SVID=DP, but we need
+	 * to special-case the SVID=DP && mux_ctrl=NO_CONN case to deliver a
+	 * HPD notification
+	 */
+	if (alt_port->svid == USB_TYPEC_DP_SID) {
+		if (alt_port->mux_ctrl == MUX_CTRL_STATE_NO_CONN) {
+			pmic_glink_altmode_safe(altmode, alt_port);
+		} else {
+			pmic_glink_altmode_enable_dp(altmode, alt_port,
+						     alt_port->mode,
+						     alt_port->hpd_state,
+						     alt_port->hpd_irq);
+		}
 
 		if (alt_port->hpd_state)
 			conn_status = connector_status_connected;
@@ -367,9 +375,18 @@ static void pmic_glink_altmode_worker(struct work_struct *work)
 
 		drm_aux_hpd_bridge_notify(&alt_port->bridge->dev, conn_status);
 	} else if (alt_port->mux_ctrl == MUX_CTRL_STATE_TUNNELING) {
-		pmic_glink_altmode_enable_usb4(altmode, alt_port);
-	} else {
+		if (alt_port->svid == USB_TYPEC_TBT_SID)
+			pmic_glink_altmode_enable_tbt(altmode, alt_port);
+		else
+			pmic_glink_altmode_enable_usb4(altmode, alt_port);
+	} else if (alt_port->mux_ctrl == MUX_CTRL_STATE_USB3_ONLY) {
 		pmic_glink_altmode_enable_usb(altmode, alt_port);
+	} else if (alt_port->mux_ctrl == MUX_CTRL_STATE_NO_CONN) {
+		pmic_glink_altmode_safe(altmode, alt_port);
+	} else {
+		dev_err(altmode->dev, "Got unknown mux_ctrl: %u on port %u, forcing safe mode\n",
+			alt_port->mux_ctrl, alt_port->index);
+		pmic_glink_altmode_safe(altmode, alt_port);
 	}
 
 	pmic_glink_altmode_request(altmode, ALTMODE_PAN_ACK, alt_port->index);
diff --git a/drivers/soc/qcom/qcom_pd_mapper.c b/drivers/soc/qcom/qcom_pd_mapper.c
index dc10bc859ff41..05e66c497691f 100644
--- a/drivers/soc/qcom/qcom_pd_mapper.c
+++ b/drivers/soc/qcom/qcom_pd_mapper.c
@@ -401,6 +401,16 @@ static const struct qcom_pdm_domain_data *qcs404_domains[] = {
 	NULL,
 };
 
+static const struct qcom_pdm_domain_data *qcs615_domains[] = {
+	&adsp_audio_pd,
+	&adsp_root_pd,
+	&adsp_sensor_pd,
+	&cdsp_root_pd,
+	&mpss_root_pd,
+	&mpss_wlan_pd,
+	NULL,
+};
+
 static const struct qcom_pdm_domain_data *sc7180_domains[] = {
 	&adsp_audio_pd,
 	&adsp_root_pd_pdr,
@@ -562,6 +572,7 @@ static const struct of_device_id qcom_pdm_domains[] __maybe_unused = {
 	{ .compatible = "qcom,apq8084", .data = NULL, },
 	{ .compatible = "qcom,apq8096", .data = msm8996_domains, },
 	{ .compatible = "qcom,kaanapali", .data = kaanapali_domains, },
+	{ .compatible = "qcom,milos", .data = sm8550_domains, },
 	{ .compatible = "qcom,msm8226", .data = NULL, },
 	{ .compatible = "qcom,msm8909", .data = NULL, },
 	{ .compatible = "qcom,msm8916", .data = NULL, },
@@ -572,6 +583,7 @@ static const struct of_device_id qcom_pdm_domains[] __maybe_unused = {
 	{ .compatible = "qcom,qcm2290", .data = qcm2290_domains, },
 	{ .compatible = "qcom,qcm6490", .data = sc7280_domains, },
 	{ .compatible = "qcom,qcs404", .data = qcs404_domains, },
+	{ .compatible = "qcom,qcs615", .data = qcs615_domains, },
 	{ .compatible = "qcom,sc7180", .data = sc7180_domains, },
 	{ .compatible = "qcom,sc7280", .data = sc7280_domains, },
 	{ .compatible = "qcom,sc8180x", .data = sc8180x_domains, },
@@ -615,15 +627,9 @@ static struct qcom_pdm_data *qcom_pdm_start(void)
 	const struct qcom_pdm_domain_data * const *domains;
 	const struct of_device_id *match;
 	struct qcom_pdm_data *data;
-	struct device_node *root;
 	int ret, i;
 
-	root = of_find_node_by_path("/");
-	if (!root)
-		return ERR_PTR(-ENODEV);
-
-	match = of_match_node(qcom_pdm_domains, root);
-	of_node_put(root);
+	match = of_machine_get_match(qcom_pdm_domains);
 	if (!match) {
 		pr_notice("PDM: no support for the platform, userspace daemon might be required.\n");
 		return ERR_PTR(-ENODEV);
@@ -656,7 +662,7 @@ static struct qcom_pdm_data *qcom_pdm_start(void)
 			goto err_stop;
 	}
 
-	ret = qmi_add_server(&data->handle, SERVREG_LOCATOR_SERVICE,
+	ret = qmi_add_server(&data->handle, QMI_SERVICE_ID_SERVREG_LOC,
 			     SERVREG_QMI_VERSION, SERVREG_QMI_INSTANCE);
 	if (ret) {
 		pr_err("PDM: error adding server %d\n", ret);
diff --git a/drivers/soc/qcom/qcom_pdr_msg.c b/drivers/soc/qcom/qcom_pdr_msg.c
index ca98932140d87..02022b11ecf05 100644
--- a/drivers/soc/qcom/qcom_pdr_msg.c
+++ b/drivers/soc/qcom/qcom_pdr_msg.c
@@ -325,7 +325,7 @@ const struct qmi_elem_info servreg_loc_pfr_req_ei[] = {
 	},
 	{
 		.data_type = QMI_STRING,
-		.elem_len = SERVREG_NAME_LENGTH + 1,
+		.elem_len = SERVREG_PFR_LENGTH + 1,
 		.elem_size = sizeof(char),
 		.array_type = VAR_LEN_ARRAY,
 		.tlv_type = 0x02,
diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c
index cb515c2340c10..af0ceeaf6e073 100644
--- a/drivers/soc/qcom/smp2p.c
+++ b/drivers/soc/qcom/smp2p.c
@@ -36,6 +36,10 @@
  * The driver uses the Linux GPIO and interrupt framework to expose a virtual
  * GPIO for each outbound entry and a virtual interrupt controller for each
  * inbound entry.
+ *
+ * V2 of SMP2P allows remote processors to write to outbound smp2p items before
+ * the full smp2p connection is negotiated. This is important for processors
+ * started before linux runs.
  */
 
 #define SMP2P_MAX_ENTRY 16
@@ -47,11 +51,12 @@
 
 #define SMP2P_MAGIC 0x504d5324
 #define SMP2P_ALL_FEATURES	SMP2P_FEATURE_SSR_ACK
+#define MAX_VERSION 2
 
 /**
  * struct smp2p_smem_item - in memory communication structure
  * @magic:		magic number
- * @version:		version - must be 1
+ * @version:		version
  * @features:		features flag - currently unused
  * @local_pid:		processor id of sending end
  * @remote_pid:		processor id of receiving end
@@ -180,14 +185,22 @@ static void qcom_smp2p_kick(struct qcom_smp2p *smp2p)
 static bool qcom_smp2p_check_ssr(struct qcom_smp2p *smp2p)
 {
 	struct smp2p_smem_item *in = smp2p->in;
+	struct smp2p_entry *entry;
+	bool restart_done;
 	bool restart;
 
 	if (!smp2p->ssr_ack_enabled)
 		return false;
 
-	restart = in->flags & BIT(SMP2P_FLAGS_RESTART_DONE_BIT);
+	restart_done = in->flags & BIT(SMP2P_FLAGS_RESTART_DONE_BIT);
+	restart = restart_done != smp2p->ssr_ack;
+	list_for_each_entry(entry, &smp2p->inbound, node) {
+		if (!entry->value)
+			continue;
+		entry->last_value = 0;
+	}
 
-	return restart != smp2p->ssr_ack;
+	return restart;
 }
 
 static void qcom_smp2p_do_ssr_ack(struct qcom_smp2p *smp2p)
@@ -219,7 +232,57 @@ static void qcom_smp2p_negotiate(struct qcom_smp2p *smp2p)
 
 		smp2p->negotiation_done = true;
 		trace_smp2p_negotiate(smp2p->dev, out->features);
+	} else if (in->version && in->version < out->version) {
+		out->version = in->version;
+		qcom_smp2p_kick(smp2p);
+	}
+}
+
+static int qcom_smp2p_in_version(struct qcom_smp2p *smp2p)
+{
+	unsigned int smem_id = smp2p->smem_items[SMP2P_INBOUND];
+	unsigned int pid = smp2p->remote_pid;
+	struct smp2p_smem_item *in;
+	size_t size;
+
+	in = qcom_smem_get(pid, smem_id, &size);
+	if (IS_ERR(in))
+		return 0;
+
+	return in->version;
+}
+
+static void qcom_smp2p_start_in(struct qcom_smp2p *smp2p)
+{
+	unsigned int smem_id = smp2p->smem_items[SMP2P_INBOUND];
+	unsigned int pid = smp2p->remote_pid;
+	char buf[SMP2P_MAX_ENTRY_NAME];
+	struct smp2p_smem_item *in;
+	struct smp2p_entry *entry;
+	size_t size;
+	int i;
+
+	in = qcom_smem_get(pid, smem_id, &size);
+	if (IS_ERR(in))
+		return;
+
+	smp2p->in = in;
+
+	/* Check if version is initialized by the remote. */
+	if (in->version == 0)
+		return;
+
+	for (i = smp2p->valid_entries; i < in->valid_entries; i++) {
+		list_for_each_entry(entry, &smp2p->inbound, node) {
+			memcpy(buf, in->entries[i].name, sizeof(buf));
+			if (!strcmp(buf, entry->name)) {
+				entry->value = &in->entries[i].value;
+				entry->last_value = readl(entry->value);
+				break;
+			}
+		}
 	}
+	smp2p->valid_entries = i;
 }
 
 static void qcom_smp2p_notify_in(struct qcom_smp2p *smp2p)
@@ -368,12 +431,31 @@ static void smp2p_irq_print_chip(struct irq_data *irqd, struct seq_file *p)
 	seq_printf(p, "%8s", dev_name(entry->smp2p->dev));
 }
 
+static int smp2p_irq_get_irqchip_state(struct irq_data *irqd, enum irqchip_irq_state which,
+				       bool *state)
+{
+	struct smp2p_entry *entry = irq_data_get_irq_chip_data(irqd);
+	u32 val;
+
+	if (which != IRQCHIP_STATE_LINE_LEVEL)
+		return -EINVAL;
+
+	if (!entry->value)
+		return -ENODEV;
+
+	val = readl(entry->value);
+	*state = !!(val & BIT(irqd_to_hwirq(irqd)));
+
+	return 0;
+}
+
 static struct irq_chip smp2p_irq_chip = {
 	.name           = "smp2p",
 	.irq_mask       = smp2p_mask_irq,
 	.irq_unmask     = smp2p_unmask_irq,
 	.irq_set_type	= smp2p_set_irq_type,
 	.irq_print_chip = smp2p_irq_print_chip,
+	.irq_get_irqchip_state = smp2p_irq_get_irqchip_state,
 };
 
 static int smp2p_irq_map(struct irq_domain *d,
@@ -464,6 +546,7 @@ static int qcom_smp2p_alloc_outbound_item(struct qcom_smp2p *smp2p)
 	struct smp2p_smem_item *out;
 	unsigned smem_id = smp2p->smem_items[SMP2P_OUTBOUND];
 	unsigned pid = smp2p->remote_pid;
+	u8 in_version;
 	int ret;
 
 	ret = qcom_smem_alloc(pid, smem_id, sizeof(*out));
@@ -485,12 +568,21 @@ static int qcom_smp2p_alloc_outbound_item(struct qcom_smp2p *smp2p)
 	out->valid_entries = 0;
 	out->features = SMP2P_ALL_FEATURES;
 
+	in_version = qcom_smp2p_in_version(smp2p);
+	if (in_version > MAX_VERSION) {
+		dev_err(smp2p->dev, "Unsupported smp2p version %d\n", in_version);
+		return -EINVAL;
+	}
+
 	/*
 	 * Make sure the rest of the header is written before we validate the
 	 * item by writing a valid version number.
 	 */
 	wmb();
-	out->version = 1;
+	if (in_version && in_version <= 2)
+		out->version = in_version;
+	else
+		out->version = 2;
 
 	qcom_smp2p_kick(smp2p);
 
@@ -618,6 +710,9 @@ static int qcom_smp2p_probe(struct platform_device *pdev)
 		}
 	}
 
+	/* Check inbound entries in the case of early boot processor */
+	qcom_smp2p_start_in(smp2p);
+
 	/* Kick the outgoing edge after allocating entries */
 	qcom_smp2p_kick(smp2p);
 
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index 003a2304d535c..802bf289626f3 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -182,6 +182,7 @@ static const char *const pmic_models[] = {
 	[72] = "PMR735D",
 	[73] = "PM8550",
 	[74] = "PMK8550",
+	[76] = "PM7550BA",
 	[78] = "PMM8650AU",
 	[79] = "PMM8650AU_PSAIL",
 	[80] = "PM7550",
@@ -473,6 +474,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(IPQ5000) },
 	{ qcom_board_id(IPQ0509) },
 	{ qcom_board_id(IPQ0518) },
+	{ qcom_board_id(SM7450) },
 	{ qcom_board_id(SM6375) },
 	{ qcom_board_id(IPQ9514) },
 	{ qcom_board_id(IPQ9550) },
@@ -492,6 +494,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(QRU1000) },
 	{ qcom_board_id(SM8475_2) },
 	{ qcom_board_id(QDU1000) },
+	{ qcom_board_id(SM7450P) },
 	{ qcom_board_id(X1E80100) },
 	{ qcom_board_id(SM8650) },
 	{ qcom_board_id(SM4450) },
@@ -522,6 +525,13 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(QCS8275) },
 	{ qcom_board_id(QCS9075) },
 	{ qcom_board_id(QCS615) },
+	{ qcom_board_id(CQ7790M) },
+	{ qcom_board_id(CQ7790S) },
+	{ qcom_board_id(IPQ5200) },
+	{ qcom_board_id(IPQ5210) },
+	{ qcom_board_id(QCF2200) },
+	{ qcom_board_id(QCF3200) },
+	{ qcom_board_id(QCF3210) },
 };
 
 static const char *socinfo_machine(struct device *dev, unsigned int id)
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 1e50dc7c31cd7..26bed0fdceb00 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -390,6 +390,14 @@ config ARCH_R9A08G045
 	help
 	  This enables support for the Renesas RZ/G3S SoC variants.
 
+config ARCH_R9A08G046
+	bool "ARM64 Platform support for R9A08G046 (RZ/G3L)"
+	default y if ARCH_RENESAS
+	select ARCH_RZG2L
+	select SYSC_R9A08G046
+	help
+	  This enables support for the Renesas RZ/G3L SoC variants.
+
 config ARCH_R9A09G011
 	bool "ARM64 Platform support for R9A09G011 (RZ/V2M)"
 	default y if ARCH_RENESAS
@@ -474,6 +482,10 @@ config SYSC_R9A08G045
 	bool "Renesas System controller support for R9A08G045 (RZ/G3S)" if COMPILE_TEST
 	select SYSC_RZ
 
+config SYSC_R9A08G046
+	bool "Renesas System controller support for R9A08G046 (RZ/G3L)" if COMPILE_TEST
+	select SYSC_RZ
+
 config SYS_R9A09G047
 	bool "Renesas System controller support for R9A09G047 (RZ/G3E)" if COMPILE_TEST
 	select SYSC_RZ
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index 33d44d964d61b..655dbcb087472 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -7,6 +7,7 @@ ifdef CONFIG_SMP
 obj-$(CONFIG_ARCH_R9A06G032)	+= r9a06g032-smp.o
 endif
 obj-$(CONFIG_SYSC_R9A08G045)	+= r9a08g045-sysc.o
+obj-$(CONFIG_SYSC_R9A08G046)	+= r9a08g046-sysc.o
 obj-$(CONFIG_SYS_R9A09G047)	+= r9a09g047-sys.o
 obj-$(CONFIG_SYS_R9A09G056)	+= r9a09g056-sys.o
 obj-$(CONFIG_SYS_R9A09G057)	+= r9a09g057-sys.o
diff --git a/drivers/soc/renesas/r9a08g046-sysc.c b/drivers/soc/renesas/r9a08g046-sysc.c
new file mode 100644
index 0000000000000..fd98df196d0a5
--- /dev/null
+++ b/drivers/soc/renesas/r9a08g046-sysc.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RZ/G3L System controller (SYSC) driver
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/init.h>
+
+#include "rz-sysc.h"
+
+#define SYS_XSPI_MAP_STAADD_CS0		0x348
+#define SYS_XSPI_MAP_ENDADD_CS0		0x34c
+#define SYS_XSPI_MAP_STAADD_CS1		0x350
+#define SYS_XSPI_MAP_ENDADD_CS1		0x354
+#define SYS_GETH0_CFG			0x380
+#define SYS_GETH1_CFG			0x390
+#define SYS_PCIE_CFG			0x3a0
+#define SYS_PCIE_MON			0x3a4
+#define SYS_PCIE_PHY			0x3b4
+#define SYS_I2C0_CFG			0x400
+#define SYS_I2C1_CFG			0x410
+#define SYS_I2C2_CFG			0x420
+#define SYS_I2C3_CFG			0x430
+#define SYS_I3C_CFG			0x440
+#define SYS_PWRRDY_N			0xd70
+#define SYS_IPCONT_SEL_CLONECH		0xe2c
+
+static bool rzg3l_regmap_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case SYS_XSPI_MAP_STAADD_CS0:
+	case SYS_XSPI_MAP_ENDADD_CS0:
+	case SYS_XSPI_MAP_STAADD_CS1:
+	case SYS_XSPI_MAP_ENDADD_CS1:
+	case SYS_GETH0_CFG:
+	case SYS_GETH1_CFG:
+	case SYS_PCIE_CFG:
+	case SYS_PCIE_MON:
+	case SYS_PCIE_PHY:
+	case SYS_I2C0_CFG:
+	case SYS_I2C1_CFG:
+	case SYS_I2C2_CFG:
+	case SYS_I2C3_CFG:
+	case SYS_I3C_CFG:
+	case SYS_PWRRDY_N:
+	case SYS_IPCONT_SEL_CLONECH:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool rzg3l_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case SYS_XSPI_MAP_STAADD_CS0:
+	case SYS_XSPI_MAP_ENDADD_CS0:
+	case SYS_XSPI_MAP_STAADD_CS1:
+	case SYS_XSPI_MAP_ENDADD_CS1:
+	case SYS_PCIE_CFG:
+	case SYS_PCIE_PHY:
+	case SYS_I2C0_CFG:
+	case SYS_I2C1_CFG:
+	case SYS_I2C2_CFG:
+	case SYS_I2C3_CFG:
+	case SYS_I3C_CFG:
+	case SYS_PWRRDY_N:
+	case SYS_IPCONT_SEL_CLONECH:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct rz_sysc_soc_id_init_data rzg3l_sysc_soc_id_init_data __initconst = {
+	.family = "RZ/G3L",
+	.id = 0x87d9447,
+	.devid_offset = 0xa04,
+	.revision_mask = GENMASK(31, 28),
+	.specific_id_mask = GENMASK(27, 0),
+};
+
+const struct rz_sysc_init_data rzg3l_sysc_init_data __initconst = {
+	.soc_id_init_data = &rzg3l_sysc_soc_id_init_data,
+	.readable_reg = rzg3l_regmap_readable_reg,
+	.writeable_reg = rzg3l_regmap_writeable_reg,
+	.max_register = 0xe2c,
+};
diff --git a/drivers/soc/renesas/r9a09g047-sys.c b/drivers/soc/renesas/r9a09g047-sys.c
index e413b0eff9bfd..ea3ca10fcc33f 100644
--- a/drivers/soc/renesas/r9a09g047-sys.c
+++ b/drivers/soc/renesas/r9a09g047-sys.c
@@ -139,7 +139,7 @@ static bool rzg3e_regmap_writeable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-const struct rz_sysc_init_data rzg3e_sys_init_data = {
+const struct rz_sysc_init_data rzg3e_sys_init_data __initconst = {
 	.soc_id_init_data = &rzg3e_sys_soc_id_init_data,
 	.readable_reg = rzg3e_regmap_readable_reg,
 	.writeable_reg = rzg3e_regmap_writeable_reg,
diff --git a/drivers/soc/renesas/r9a09g056-sys.c b/drivers/soc/renesas/r9a09g056-sys.c
index 42f5eff291fd1..2a8ebc2099610 100644
--- a/drivers/soc/renesas/r9a09g056-sys.c
+++ b/drivers/soc/renesas/r9a09g056-sys.c
@@ -136,7 +136,7 @@ static bool rzv2n_regmap_writeable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-const struct rz_sysc_init_data rzv2n_sys_init_data = {
+const struct rz_sysc_init_data rzv2n_sys_init_data __initconst = {
 	.soc_id_init_data = &rzv2n_sys_soc_id_init_data,
 	.readable_reg = rzv2n_regmap_readable_reg,
 	.writeable_reg = rzv2n_regmap_writeable_reg,
diff --git a/drivers/soc/renesas/r9a09g057-sys.c b/drivers/soc/renesas/r9a09g057-sys.c
index 827c718ac7c54..f3e054206acbd 100644
--- a/drivers/soc/renesas/r9a09g057-sys.c
+++ b/drivers/soc/renesas/r9a09g057-sys.c
@@ -161,7 +161,7 @@ static bool rzv2h_regmap_writeable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-const struct rz_sysc_init_data rzv2h_sys_init_data = {
+const struct rz_sysc_init_data rzv2h_sys_init_data __initconst = {
 	.soc_id_init_data = &rzv2h_sys_soc_id_init_data,
 	.readable_reg = rzv2h_regmap_readable_reg,
 	.writeable_reg = rzv2h_regmap_writeable_reg,
diff --git a/drivers/soc/renesas/rz-sysc.c b/drivers/soc/renesas/rz-sysc.c
index 7471dc8736e63..161e8c38eea69 100644
--- a/drivers/soc/renesas/rz-sysc.c
+++ b/drivers/soc/renesas/rz-sysc.c
@@ -88,6 +88,9 @@ static const struct of_device_id rz_sysc_match[] = {
 #ifdef CONFIG_SYSC_R9A08G045
 	{ .compatible = "renesas,r9a08g045-sysc", .data = &rzg3s_sysc_init_data },
 #endif
+#ifdef CONFIG_SYSC_R9A08G046
+	{ .compatible = "renesas,r9a08g046-sysc", .data = &rzg3l_sysc_init_data },
+#endif
 #ifdef CONFIG_SYS_R9A09G047
 	{ .compatible = "renesas,r9a09g047-sys", .data = &rzg3e_sys_init_data },
 #endif
diff --git a/drivers/soc/renesas/rz-sysc.h b/drivers/soc/renesas/rz-sysc.h
index 88929bf21cb11..921ee0d26c470 100644
--- a/drivers/soc/renesas/rz-sysc.h
+++ b/drivers/soc/renesas/rz-sysc.h
@@ -46,6 +46,7 @@ struct rz_sysc_init_data {
 };
 
 extern const struct rz_sysc_init_data rzg3e_sys_init_data;
+extern const struct rz_sysc_init_data rzg3l_sysc_init_data;
 extern const struct rz_sysc_init_data rzg3s_sysc_init_data;
 extern const struct rz_sysc_init_data rzv2h_sys_init_data;
 extern const struct rz_sysc_init_data rzv2n_sys_init_data;
diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c
index 04937c40da471..b459607c118aa 100644
--- a/drivers/soc/rockchip/grf.c
+++ b/drivers/soc/rockchip/grf.c
@@ -231,6 +231,7 @@ static int __init rockchip_grf_init(void)
 		grf = syscon_node_to_regmap(np);
 		if (IS_ERR(grf)) {
 			pr_err("%s: could not get grf syscon\n", __func__);
+			of_node_put(np);
 			return PTR_ERR(grf);
 		}
 
diff --git a/drivers/soc/sunxi/sunxi_mbus.c b/drivers/soc/sunxi/sunxi_mbus.c
index 1734da357ca21..8bc5f62ff2588 100644
--- a/drivers/soc/sunxi/sunxi_mbus.c
+++ b/drivers/soc/sunxi/sunxi_mbus.c
@@ -118,7 +118,7 @@ static const char * const sunxi_mbus_platforms[] __initconst = {
 
 static int __init sunxi_mbus_init(void)
 {
-	if (!of_device_compatible_match(of_root, sunxi_mbus_platforms))
+	if (!of_machine_compatible_match(sunxi_mbus_platforms))
 		return 0;
 
 	bus_register_notifier(&platform_bus_type, &sunxi_mbus_nb);
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index c0fc54c3cd35e..0a84bfd5c657c 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -125,6 +125,15 @@ config ARCH_TEGRA_234_SOC
 	help
 	  Enable support for the NVIDIA Tegra234 SoC.
 
+config ARCH_TEGRA_238_SOC
+	bool "NVIDIA Tegra238 SoC"
+	default ARCH_TEGRA
+	depends on !CPU_BIG_ENDIAN
+	select MAILBOX
+	select SOC_TEGRA_PMC
+	help
+	  Enable support for the NVIDIA Tegra238 SoC.
+
 config ARCH_TEGRA_241_SOC
 	bool "NVIDIA Tegra241 SoC"
 	help
diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c
index b9ccb33438961..a3316efdf8ac2 100644
--- a/drivers/soundwire/amd_manager.c
+++ b/drivers/soundwire/amd_manager.c
@@ -27,6 +27,36 @@
 
 #define to_amd_sdw(b)	container_of(b, struct amd_sdw_manager, bus)
 
+static int amd_sdw_clk_init_ctrl(struct amd_sdw_manager *amd_manager)
+{
+	struct sdw_bus *bus = &amd_manager->bus;
+	struct sdw_master_prop *prop = &bus->prop;
+	u32 divider;
+
+	dev_dbg(amd_manager->dev, "mclk %d max %d row %d col %d frame_rate:%d\n",
+		prop->mclk_freq, prop->max_clk_freq, prop->default_row,
+		prop->default_col, prop->default_frame_rate);
+
+	if (!prop->default_frame_rate || !prop->default_row) {
+		dev_err(amd_manager->dev, "Default frame_rate %d or row %d is invalid\n",
+			prop->default_frame_rate, prop->default_row);
+		return -EINVAL;
+	}
+
+	/* Set clock divider */
+	divider = (prop->mclk_freq / bus->params.curr_dr_freq);
+	writel(divider, amd_manager->mmio + ACP_SW_CLK_FREQUENCY_CTRL);
+
+	/* Set frame shape base on the actual bus frequency. */
+	prop->default_col = bus->params.curr_dr_freq /
+			    prop->default_frame_rate / prop->default_row;
+	amd_manager->cols_index = sdw_find_col_index(prop->default_col);
+	amd_manager->rows_index = sdw_find_row_index(prop->default_row);
+	bus->params.col = prop->default_col;
+	bus->params.row = prop->default_row;
+	return 0;
+}
+
 static int amd_init_sdw_manager(struct amd_sdw_manager *amd_manager)
 {
 	u32 val;
@@ -437,12 +467,16 @@ static u32 amd_sdw_read_ping_status(struct sdw_bus *bus)
 
 static int amd_sdw_compute_params(struct sdw_bus *bus, struct sdw_stream_runtime *stream)
 {
+	struct amd_sdw_manager *amd_manager = to_amd_sdw(bus);
 	struct sdw_transport_data t_data = {0};
 	struct sdw_master_runtime *m_rt;
 	struct sdw_port_runtime *p_rt;
 	struct sdw_bus_params *b_params = &bus->params;
 	int port_bo, hstart, hstop, sample_int;
-	unsigned int rate, bps;
+	unsigned int rate, bps, channels;
+	unsigned int stream_slot_size, max_slots;
+	static unsigned int next_offset[AMD_SDW_MAX_MANAGER_COUNT] = {1};
+	unsigned int inst_id = amd_manager->instance;
 
 	port_bo = 0;
 	hstart = 1;
@@ -453,11 +487,51 @@ static int amd_sdw_compute_params(struct sdw_bus *bus, struct sdw_stream_runtime
 	list_for_each_entry(m_rt, &bus->m_rt_list, bus_node) {
 		rate = m_rt->stream->params.rate;
 		bps = m_rt->stream->params.bps;
+		channels = m_rt->stream->params.ch_count;
 		sample_int = (bus->params.curr_dr_freq / rate);
+
+		/* Compute slots required for this stream dynamically */
+		stream_slot_size = bps * channels;
+
 		list_for_each_entry(p_rt, &m_rt->port_list, port_node) {
-			port_bo = (p_rt->num * 64) + 1;
-			dev_dbg(bus->dev, "p_rt->num=%d hstart=%d hstop=%d port_bo=%d\n",
-				p_rt->num, hstart, hstop, port_bo);
+			if (p_rt->num >= amd_manager->max_ports) {
+				dev_err(bus->dev, "Port %d exceeds max ports %d\n",
+					p_rt->num, amd_manager->max_ports);
+				return -EINVAL;
+			}
+
+			if (!amd_manager->port_offset_map[p_rt->num]) {
+				/*
+				 * port block offset calculation for 6MHz bus clock frequency with
+				 * different frame sizes 50 x 10 and 125 x 2
+				 */
+				if (bus->params.curr_dr_freq == 12000000) {
+					max_slots = bus->params.row * (bus->params.col - 1);
+					if (next_offset[inst_id] + stream_slot_size <=
+					    (max_slots - 1)) {
+						amd_manager->port_offset_map[p_rt->num] =
+									next_offset[inst_id];
+						next_offset[inst_id] += stream_slot_size;
+					} else {
+						dev_err(bus->dev,
+							"No space for port %d\n", p_rt->num);
+						return -ENOMEM;
+					}
+				} else {
+					 /*
+					  * port block offset calculation for 12MHz bus clock
+					  * frequency
+					  */
+					amd_manager->port_offset_map[p_rt->num] =
+									(p_rt->num * 64) + 1;
+				}
+			}
+			port_bo = amd_manager->port_offset_map[p_rt->num];
+			dev_dbg(bus->dev,
+				"Port=%d hstart=%d hstop=%d port_bo=%d slots=%d max_ports=%d\n",
+				p_rt->num, hstart, hstop, port_bo, stream_slot_size,
+				amd_manager->max_ports);
+
 			sdw_fill_xport_params(&p_rt->transport_params, p_rt->num,
 					      false, SDW_BLK_GRP_CNT_1, sample_int,
 					      port_bo, port_bo >> 8, hstart, hstop,
@@ -960,6 +1034,9 @@ int amd_sdw_manager_start(struct amd_sdw_manager *amd_manager)
 
 	prop = &amd_manager->bus.prop;
 	if (!prop->hw_disabled) {
+		ret = amd_sdw_clk_init_ctrl(amd_manager);
+		if (ret)
+			return ret;
 		ret = amd_init_sdw_manager(amd_manager);
 		if (ret)
 			return ret;
@@ -984,7 +1061,6 @@ static int amd_sdw_manager_probe(struct platform_device *pdev)
 	struct resource *res;
 	struct device *dev = &pdev->dev;
 	struct sdw_master_prop *prop;
-	struct sdw_bus_params *params;
 	struct amd_sdw_manager *amd_manager;
 	int ret;
 
@@ -1047,15 +1123,14 @@ static int amd_sdw_manager_probe(struct platform_device *pdev)
 	default:
 		return -EINVAL;
 	}
+	amd_manager->max_ports = amd_manager->num_dout_ports + amd_manager->num_din_ports;
+	amd_manager->port_offset_map = devm_kcalloc(dev, amd_manager->max_ports,
+						    sizeof(int), GFP_KERNEL);
+	if (!amd_manager->port_offset_map)
+		return -ENOMEM;
 
-	params = &amd_manager->bus.params;
-
-	params->col = AMD_SDW_DEFAULT_COLUMNS;
-	params->row = AMD_SDW_DEFAULT_ROWS;
 	prop = &amd_manager->bus.prop;
-	prop->clk_freq = &amd_sdw_freq_tbl[0];
 	prop->mclk_freq = AMD_SDW_BUS_BASE_FREQ;
-	prop->max_clk_freq = AMD_SDW_DEFAULT_CLK_FREQ;
 
 	ret = sdw_bus_master_add(&amd_manager->bus, dev, dev->fwnode);
 	if (ret) {
@@ -1347,6 +1422,9 @@ static int __maybe_unused amd_resume_runtime(struct device *dev)
 			}
 		}
 		sdw_clear_slave_status(bus, SDW_UNATTACH_REQUEST_MASTER_RESET);
+		ret = amd_sdw_clk_init_ctrl(amd_manager);
+		if (ret)
+			return ret;
 		amd_init_sdw_manager(amd_manager);
 		amd_enable_sdw_interrupts(amd_manager);
 		ret = amd_enable_sdw_manager(amd_manager);
diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h
index 6cc916b0c8206..88cf8a426a0c4 100644
--- a/drivers/soundwire/amd_manager.h
+++ b/drivers/soundwire/amd_manager.h
@@ -203,10 +203,6 @@
 #define AMD_SDW_DEVICE_STATE_D3				3
 #define ACP_PME_EN					0x0001400
 
-static u32 amd_sdw_freq_tbl[AMD_SDW_MAX_FREQ_NUM] = {
-	AMD_SDW_DEFAULT_CLK_FREQ,
-};
-
 struct sdw_manager_dp_reg {
 	u32 frame_fmt_reg;
 	u32 sample_int_reg;
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index fb68738dfb9b8..fe5316d93fefe 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -1899,8 +1899,8 @@ int sdw_handle_slave_status(struct sdw_bus *bus,
 
 		if (status[i] == SDW_SLAVE_UNATTACHED &&
 		    slave->status != SDW_SLAVE_UNATTACHED) {
-			dev_warn(&slave->dev, "Slave %d state check1: UNATTACHED, status was %d\n",
-				 i, slave->status);
+			dev_dbg(&slave->dev, "Slave %d state check1: UNATTACHED, status was %d\n",
+			i, slave->status);
 			sdw_modify_slave_status(slave, SDW_SLAVE_UNATTACHED);
 
 			/* Ensure driver knows that peripheral unattached */
@@ -1951,8 +1951,8 @@ int sdw_handle_slave_status(struct sdw_bus *bus,
 			if (slave->status == SDW_SLAVE_UNATTACHED)
 				break;
 
-			dev_warn(&slave->dev, "Slave %d state check2: UNATTACHED, status was %d\n",
-				 i, slave->status);
+			dev_dbg(&slave->dev, "Slave %d state check2: UNATTACHED, status was %d\n",
+			i, slave->status);
 
 			sdw_modify_slave_status(slave, SDW_SLAVE_UNATTACHED);
 			break;
diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c
index f245c3ffb9e97..b8b62735c8938 100644
--- a/drivers/soundwire/cadence_master.c
+++ b/drivers/soundwire/cadence_master.c
@@ -933,6 +933,14 @@ irqreturn_t sdw_cdns_irq(int irq, void *dev_id)
 
 		cdns_read_response(cdns);
 
+		/*
+		 * Clear interrupt before signalling the completion to avoid
+		 * a race between this thread and the main thread starting
+		 * another TX.
+		 */
+		cdns_writel(cdns, CDNS_MCP_INTSTAT, CDNS_MCP_INT_RX_WL);
+		int_status &= ~CDNS_MCP_INT_RX_WL;
+
 		if (defer && defer->msg) {
 			cdns_fill_msg_resp(cdns, defer->msg,
 					   defer->length, 0);
diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c
index 7f01e43ae978a..20422534baf19 100644
--- a/drivers/soundwire/intel_ace2x.c
+++ b/drivers/soundwire/intel_ace2x.c
@@ -82,6 +82,11 @@ static int intel_ace2x_bpt_open_stream(struct sdw_intel *sdw, struct sdw_slave *
 	int len;
 	int i;
 
+	if (cdns->bus.bpt_stream) {
+		dev_err(cdns->dev, "%s: BPT stream already exists\n", __func__);
+		return -EAGAIN;
+	}
+
 	stream = sdw_alloc_stream("BPT", SDW_STREAM_BPT);
 	if (!stream)
 		return -ENOMEM;
diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c
index 8752b0e3ce74c..913e95207ee1a 100644
--- a/drivers/soundwire/intel_auxdevice.c
+++ b/drivers/soundwire/intel_auxdevice.c
@@ -53,6 +53,8 @@ struct wake_capable_part {
 static struct wake_capable_part wake_capable_list[] = {
 	{0x01fa, 0x4243},
 	{0x01fa, 0x4245},
+	{0x01fa, 0x4249},
+	{0x01fa, 0x4747},
 	{0x025d, 0x5682},
 	{0x025d, 0x700},
 	{0x025d, 0x711},
diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c
index f5a3ca3b9dda4..ff763b692078d 100644
--- a/drivers/soundwire/slave.c
+++ b/drivers/soundwire/slave.c
@@ -115,6 +115,9 @@ static bool find_slave(struct sdw_bus *bus,
 	u64 addr;
 	int ret;
 
+	if (acpi_bus_get_status(adev) || !acpi_dev_ready_for_enumeration(adev))
+		return false;
+
 	ret = acpi_get_local_u64_address(adev->handle, &addr);
 	if (ret < 0)
 		return false;
diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c
index b6982f8dcf17c..4ed8fb7663ad4 100644
--- a/drivers/soundwire/stream.c
+++ b/drivers/soundwire/stream.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/init.h>
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/slab.h>
@@ -18,6 +19,8 @@
 #include <sound/soc.h>
 #include "bus.h"
 
+#define SDW_PORT_PREP_POLL_USEC	1000
+
 /*
  * Array of supported rows and columns as per MIPI SoundWire Specification 1.1
  *
@@ -443,7 +446,6 @@ static int sdw_prep_deprep_slave_ports(struct sdw_bus *bus,
 				       struct sdw_port_runtime *p_rt,
 				       bool prep)
 {
-	struct completion *port_ready;
 	struct sdw_dpn_prop *dpn_prop;
 	struct sdw_prepare_ch prep_ch;
 	u32 imp_def_interrupts;
@@ -518,14 +520,18 @@ static int sdw_prep_deprep_slave_ports(struct sdw_bus *bus,
 			return ret;
 		}
 
-		/* Wait for completion on port ready */
-		port_ready = &s_rt->slave->port_ready[prep_ch.num];
-		wait_for_completion_timeout(port_ready,
-			msecs_to_jiffies(ch_prep_timeout));
+		/*
+		 * Poll for NOT_PREPARED==0. Cannot use the interrupt because
+		 * this code holds bus_lock which blocks interrupt handling.
+		 */
+		ret = read_poll_timeout(sdw_read_no_pm, val,
+					(val < 0) || ((val & p_rt->ch_mask) == 0),
+					SDW_PORT_PREP_POLL_USEC, ch_prep_timeout * USEC_PER_MSEC,
+					false, s_rt->slave, SDW_DPN_PREPARESTATUS(p_rt->num));
+		if (ret || (val < 0)) {
+			if (val < 0)
+				ret = val;
 
-		val = sdw_read_no_pm(s_rt->slave, SDW_DPN_PREPARESTATUS(p_rt->num));
-		if ((val < 0) || (val & p_rt->ch_mask)) {
-			ret = (val < 0) ? val : -ETIMEDOUT;
 			dev_err(&s_rt->slave->dev,
 				"Chn prep failed for port %d: %d\n", prep_ch.num, ret);
 			return ret;
diff --git a/drivers/spi/spi-amlogic-spifc-a4.c b/drivers/spi/spi-amlogic-spifc-a4.c
index 3956869cfec11..1aabafa36e48e 100644
--- a/drivers/spi/spi-amlogic-spifc-a4.c
+++ b/drivers/spi/spi-amlogic-spifc-a4.c
@@ -1083,14 +1083,6 @@ static int aml_sfc_clk_init(struct aml_sfc *sfc)
 	return clk_set_rate(sfc->core_clk, SFC_BUS_DEFAULT_CLK);
 }
 
-static int aml_sfc_disable_clk(struct aml_sfc *sfc)
-{
-	clk_disable_unprepare(sfc->core_clk);
-	clk_disable_unprepare(sfc->gate_clk);
-
-	return 0;
-}
-
 static int aml_sfc_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -1141,16 +1133,12 @@ static int aml_sfc_probe(struct platform_device *pdev)
 
 	/* Enable Amlogic flash controller spi mode */
 	ret = regmap_write(sfc->regmap_base, SFC_SPI_CFG, SPI_MODE_EN);
-	if (ret) {
-		dev_err(dev, "failed to enable SPI mode\n");
-		goto err_out;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable SPI mode\n");
 
 	ret = dma_set_mask(sfc->dev, DMA_BIT_MASK(32));
-	if (ret) {
-		dev_err(sfc->dev, "failed to set dma mask\n");
-		goto err_out;
-	}
+	if (ret)
+		return dev_err_probe(sfc->dev, ret, "failed to set dma mask\n");
 
 	sfc->ecc_eng.dev = &pdev->dev;
 	sfc->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED;
@@ -1158,10 +1146,8 @@ static int aml_sfc_probe(struct platform_device *pdev)
 	sfc->ecc_eng.priv = sfc;
 
 	ret = nand_ecc_register_on_host_hw_engine(&sfc->ecc_eng);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to register Aml host ecc engine.\n");
-		goto err_out;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "failed to register Aml host ecc engine.\n");
 
 	ret = of_property_read_u32(np, "amlogic,rx-adj", &val);
 	if (!ret)
@@ -1177,24 +1163,7 @@ static int aml_sfc_probe(struct platform_device *pdev)
 	ctrl->min_speed_hz = SFC_MIN_FREQUENCY;
 	ctrl->num_chipselect = SFC_MAX_CS_NUM;
 
-	ret = devm_spi_register_controller(dev, ctrl);
-	if (ret)
-		goto err_out;
-
-	return 0;
-
-err_out:
-	aml_sfc_disable_clk(sfc);
-
-	return ret;
-}
-
-static void aml_sfc_remove(struct platform_device *pdev)
-{
-	struct spi_controller *ctlr = platform_get_drvdata(pdev);
-	struct aml_sfc *sfc = spi_controller_get_devdata(ctlr);
-
-	aml_sfc_disable_clk(sfc);
+	return devm_spi_register_controller(dev, ctrl);
 }
 
 static const struct of_device_id aml_sfc_of_match[] = {
@@ -1212,7 +1181,6 @@ static struct platform_driver aml_sfc_driver = {
 		.of_match_table = aml_sfc_of_match,
 	},
 	.probe = aml_sfc_probe,
-	.remove = aml_sfc_remove,
 };
 module_platform_driver(aml_sfc_driver);
 
diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c
index 1509df2b17aed..da8ec35115da5 100644
--- a/drivers/spi/spi-amlogic-spisg.c
+++ b/drivers/spi/spi-amlogic-spisg.c
@@ -647,13 +647,13 @@ static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base)
 	int ret, i;
 
 	spisg->core = devm_clk_get_enabled(dev, "core");
-	if (IS_ERR_OR_NULL(spisg->core)) {
+	if (IS_ERR(spisg->core)) {
 		dev_err(dev, "core clock request failed\n");
 		return PTR_ERR(spisg->core);
 	}
 
 	spisg->pclk = devm_clk_get_enabled(dev, "pclk");
-	if (IS_ERR_OR_NULL(spisg->pclk)) {
+	if (IS_ERR(spisg->pclk)) {
 		dev_err(dev, "pclk clock request failed\n");
 		return PTR_ERR(spisg->pclk);
 	}
@@ -703,7 +703,7 @@ static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base)
 	}
 
 	spisg->sclk = devm_clk_hw_get_clk(dev, &div->hw, NULL);
-	if (IS_ERR_OR_NULL(spisg->sclk)) {
+	if (IS_ERR(spisg->sclk)) {
 		dev_err(dev, "get clock failed\n");
 		return PTR_ERR(spisg->sclk);
 	}
@@ -729,9 +729,9 @@ static int aml_spisg_probe(struct platform_device *pdev)
 	};
 
 	if (of_property_read_bool(dev->of_node, "spi-slave"))
-		ctlr = spi_alloc_target(dev, sizeof(*spisg));
+		ctlr = devm_spi_alloc_target(dev, sizeof(*spisg));
 	else
-		ctlr = spi_alloc_host(dev, sizeof(*spisg));
+		ctlr = devm_spi_alloc_host(dev, sizeof(*spisg));
 	if (!ctlr)
 		return -ENOMEM;
 
@@ -750,10 +750,8 @@ static int aml_spisg_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(spisg->map), "regmap init failed\n");
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto out_controller;
-	}
+	if (irq < 0)
+		return irq;
 
 	ret = device_reset_optional(dev);
 	if (ret)
@@ -817,8 +815,6 @@ static int aml_spisg_probe(struct platform_device *pdev)
 	if (spisg->core)
 		clk_disable_unprepare(spisg->core);
 	clk_disable_unprepare(spisg->pclk);
-out_controller:
-	spi_controller_put(ctlr);
 
 	return ret;
 }
diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c
index 2665f31a49ceb..3832d9db3cbf8 100644
--- a/drivers/spi/spi-atcspi200.c
+++ b/drivers/spi/spi-atcspi200.c
@@ -494,11 +494,6 @@ static int atcspi_init_resources(struct platform_device *pdev,
 		return dev_err_probe(spi->dev, PTR_ERR(spi->regmap),
 				     "Failed to init regmap\n");
 
-	spi->clk = devm_clk_get(spi->dev, NULL);
-	if (IS_ERR(spi->clk))
-		return dev_err_probe(spi->dev, PTR_ERR(spi->clk),
-				     "Failed to get SPI clock\n");
-
 	spi->sclk_rate = ATCSPI_MAX_SPEED_HZ;
 	return 0;
 }
@@ -520,13 +515,10 @@ static int atcspi_configure_dma(struct atcspi_dev *spi)
 
 static int atcspi_enable_clk(struct atcspi_dev *spi)
 {
-	int ret;
-
-	ret = clk_prepare_enable(spi->clk);
-	if (ret)
-		return dev_err_probe(spi->dev, ret,
-				     "Failed to enable clock\n");
-
+	spi->clk = devm_clk_get_enabled(spi->dev, NULL);
+	if (IS_ERR(spi->clk))
+		return dev_err_probe(spi->dev, PTR_ERR(spi->clk),
+				     "Failed to get SPI clock\n");
 	spi->clk_rate = clk_get_rate(spi->clk);
 	if (!spi->clk_rate)
 		return dev_err_probe(spi->dev, -EINVAL,
@@ -567,6 +559,8 @@ static int atcspi_probe(struct platform_device *pdev)
 	spi->dev = &pdev->dev;
 	dev_set_drvdata(&pdev->dev, host);
 
+	mutex_init(&spi->mutex_lock);
+
 	ret = atcspi_init_resources(pdev, spi, &mem_res);
 	if (ret)
 		goto free_controller;
@@ -579,15 +573,14 @@ static int atcspi_probe(struct platform_device *pdev)
 
 	ret = atcspi_setup(spi);
 	if (ret)
-		goto disable_clk;
+		goto free_controller;
 
 	ret = devm_spi_register_controller(&pdev->dev, host);
 	if (ret) {
 		dev_err_probe(spi->dev, ret,
 			      "Failed to register SPI controller\n");
-		goto disable_clk;
+		goto free_controller;
 	}
-
 	spi->use_dma = false;
 	if (ATCSPI_DMA_SUPPORT) {
 		ret = atcspi_configure_dma(spi);
@@ -597,14 +590,11 @@ static int atcspi_probe(struct platform_device *pdev)
 		else
 			spi->use_dma = true;
 	}
-	mutex_init(&spi->mutex_lock);
 
 	return 0;
 
-disable_clk:
-	clk_disable_unprepare(spi->clk);
-
 free_controller:
+	mutex_destroy(&spi->mutex_lock);
 	spi_controller_put(host);
 	return ret;
 }
@@ -661,7 +651,6 @@ static struct platform_driver atcspi_driver = {
 	.probe = atcspi_probe,
 	.driver = {
 		.name = "atcspi200",
-		.owner	= THIS_MODULE,
 		.of_match_table = atcspi_of_match,
 		.pm = pm_sleep_ptr(&atcspi_pm_ops)
 	}
diff --git a/drivers/spi/spi-axiado.c b/drivers/spi/spi-axiado.c
index 8cea81432c5ba..dc55c55ae63c8 100644
--- a/drivers/spi/spi-axiado.c
+++ b/drivers/spi/spi-axiado.c
@@ -765,30 +765,22 @@ static int ax_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, ctlr);
 
 	xspi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(xspi->regs)) {
-		ret = PTR_ERR(xspi->regs);
-		goto remove_ctlr;
-	}
+	if (IS_ERR(xspi->regs))
+		return PTR_ERR(xspi->regs);
 
 	xspi->pclk = devm_clk_get(&pdev->dev, "pclk");
-	if (IS_ERR(xspi->pclk)) {
-		dev_err(&pdev->dev, "pclk clock not found.\n");
-		ret = PTR_ERR(xspi->pclk);
-		goto remove_ctlr;
-	}
+	if (IS_ERR(xspi->pclk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(xspi->pclk),
+				     "pclk clock not found.\n");
 
 	xspi->ref_clk = devm_clk_get(&pdev->dev, "ref");
-	if (IS_ERR(xspi->ref_clk)) {
-		dev_err(&pdev->dev, "ref clock not found.\n");
-		ret = PTR_ERR(xspi->ref_clk);
-		goto remove_ctlr;
-	}
+	if (IS_ERR(xspi->ref_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(xspi->ref_clk),
+				     "ref clock not found.\n");
 
 	ret = clk_prepare_enable(xspi->pclk);
-	if (ret) {
-		dev_err(&pdev->dev, "Unable to enable APB clock.\n");
-		goto remove_ctlr;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "Unable to enable APB clock.\n");
 
 	ret = clk_prepare_enable(xspi->ref_clk);
 	if (ret) {
@@ -850,7 +842,6 @@ static int ax_spi_probe(struct platform_device *pdev)
 
 	ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
 
-	pm_runtime_mark_last_busy(&pdev->dev);
 	pm_runtime_put_autosuspend(&pdev->dev);
 
 	ctlr->mem_ops = &ax_spi_mem_ops;
@@ -869,8 +860,7 @@ static int ax_spi_probe(struct platform_device *pdev)
 	clk_disable_unprepare(xspi->ref_clk);
 clk_dis_apb:
 	clk_disable_unprepare(xspi->pclk);
-remove_ctlr:
-	spi_controller_put(ctlr);
+
 	return ret;
 }
 
diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c
index a4a650c8d7405..68f208ef1e01f 100644
--- a/drivers/spi/spi-cs42l43.c
+++ b/drivers/spi/spi-cs42l43.c
@@ -367,10 +367,10 @@ static int cs42l43_spi_probe(struct platform_device *pdev)
 		ret = devm_add_action_or_reset(priv->dev, cs42l43_release_of_node, fwnode);
 		if (ret)
 			return ret;
+	} else {
+		fwnode_property_read_u32(xu_fwnode, "01fa-sidecar-instances", &nsidecars);
 	}
 
-	fwnode_property_read_u32(xu_fwnode, "01fa-sidecar-instances", &nsidecars);
-
 	/*
 	 * Depending on the value of nsidecars we either create a software node
 	 * or assign an fwnode. We don't want software node to be attached to
diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c
index a223b4bc6e637..57358851029ba 100644
--- a/drivers/spi/spi-fsl-qspi.c
+++ b/drivers/spi/spi-fsl-qspi.c
@@ -633,7 +633,7 @@ static int fsl_qspi_do_op(struct fsl_qspi *q, const struct spi_mem_op *op)
 	void __iomem *base = q->iobase;
 	int err = 0;
 
-	init_completion(&q->c);
+	reinit_completion(&q->c);
 
 	/*
 	 * Always start the sequence at the same index since we update
@@ -965,6 +965,7 @@ static int fsl_qspi_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
+	init_completion(&q->c);
 	ret = devm_request_irq(dev, ret,
 			fsl_qspi_irq_handler, 0, pdev->name, q);
 	if (ret) {
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index 43ce47f2454cf..d5fb0edc8e0c8 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -359,9 +359,9 @@ static int setup_fifo_params(struct spi_device *spi_slv,
 		writel((spi_slv->mode & SPI_LOOP) ? LOOPBACK_ENABLE : 0, se->base + SE_SPI_LOOPBACK);
 	if (cs_changed)
 		writel(chipselect, se->base + SE_SPI_DEMUX_SEL);
-	if (mode_changed & SE_SPI_CPHA)
+	if (mode_changed & SPI_CPHA)
 		writel((spi_slv->mode & SPI_CPHA) ? CPHA : 0, se->base + SE_SPI_CPHA);
-	if (mode_changed & SE_SPI_CPOL)
+	if (mode_changed & SPI_CPOL)
 		writel((spi_slv->mode & SPI_CPOL) ? CPOL : 0, se->base + SE_SPI_CPOL);
 	if ((mode_changed & SPI_CS_HIGH) || (cs_changed && (spi_slv->mode & SPI_CS_HIGH)))
 		writel((spi_slv->mode & SPI_CS_HIGH) ? BIT(chipselect) : 0, se->base + SE_SPI_DEMUX_OUTPUT_INV);
@@ -906,10 +906,13 @@ static irqreturn_t geni_spi_isr(int irq, void *data)
 	struct spi_controller *spi = data;
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
 	struct geni_se *se = &mas->se;
-	u32 m_irq;
+	u32 m_irq, dma_tx_status, dma_rx_status;
 
 	m_irq = readl(se->base + SE_GENI_M_IRQ_STATUS);
-	if (!m_irq)
+	dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT);
+	dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT);
+
+	if (!m_irq && !dma_tx_status && !dma_rx_status)
 		return IRQ_NONE;
 
 	if (m_irq & (M_CMD_OVERRUN_EN | M_ILLEGAL_CMD_EN | M_CMD_FAILURE_EN |
@@ -957,8 +960,6 @@ static irqreturn_t geni_spi_isr(int irq, void *data)
 		}
 	} else if (mas->cur_xfer_mode == GENI_SE_DMA) {
 		const struct spi_transfer *xfer = mas->cur_xfer;
-		u32 dma_tx_status = readl_relaxed(se->base + SE_DMA_TX_IRQ_STAT);
-		u32 dma_rx_status = readl_relaxed(se->base + SE_DMA_RX_IRQ_STAT);
 
 		if (dma_tx_status)
 			writel(dma_tx_status, se->base + SE_DMA_TX_IRQ_CLR);
diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c
index 320b3d93df571..1e36ae084dd86 100644
--- a/drivers/spi/spi-nxp-fspi.c
+++ b/drivers/spi/spi-nxp-fspi.c
@@ -996,7 +996,7 @@ static int nxp_fspi_do_op(struct nxp_fspi *f, const struct spi_mem_op *op)
 	reg = reg | FSPI_IPRXFCR_CLR;
 	fspi_writel(f, reg, base + FSPI_IPRXFCR);
 
-	init_completion(&f->c);
+	reinit_completion(&f->c);
 
 	fspi_writel(f, op->addr.val, base + FSPI_IPCR0);
 	/*
@@ -1365,6 +1365,7 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return dev_err_probe(dev, ret, "Failed to disable clock");
 
+	init_completion(&f->c);
 	ret = devm_request_irq(dev, irq,
 			nxp_fspi_irq_handler, 0, pdev->name, f);
 	if (ret)
diff --git a/drivers/spi/spi-nxp-xspi.c b/drivers/spi/spi-nxp-xspi.c
index 06fcdf22990b8..385302a6e62f2 100644
--- a/drivers/spi/spi-nxp-xspi.c
+++ b/drivers/spi/spi-nxp-xspi.c
@@ -958,7 +958,7 @@ static int nxp_xspi_do_op(struct nxp_xspi *xspi, const struct spi_mem_op *op)
 		writel(reg, base + XSPI_RBCT);
 	}
 
-	init_completion(&xspi->c);
+	reinit_completion(&xspi->c);
 
 	/* Config the data address */
 	writel(op->addr.val + xspi->memmap_phy, base + XSPI_SFP_TG_SFAR);
@@ -1273,6 +1273,7 @@ static int nxp_xspi_probe(struct platform_device *pdev)
 
 	nxp_xspi_default_setup(xspi);
 
+	init_completion(&xspi->c);
 	ret = devm_request_irq(dev, irq,
 			nxp_xspi_irq_handler, 0, pdev->name, xspi);
 	if (ret)
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index 2207e05c9d062..a3468e47e77d0 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -942,10 +942,16 @@ static int omap2_mcspi_setup_transfer(struct spi_device *spi,
 
 	l = mcspi_cached_chconf0(spi);
 
-	/* standard 4-wire host mode:  SCK, MOSI/out, MISO/in, nCS
-	 * REVISIT: this controller could support SPI_3WIRE mode.
-	 */
-	if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
+	if (spi->mode & SPI_3WIRE) {
+		if (t && !t->tx_buf) {
+			l &= ~OMAP2_MCSPI_CHCONF_IS;
+			l |= OMAP2_MCSPI_CHCONF_DPE0;
+		} else if (t && !t->rx_buf) {
+			l |= OMAP2_MCSPI_CHCONF_IS;
+			l &= ~OMAP2_MCSPI_CHCONF_DPE0;
+		}
+		l |= OMAP2_MCSPI_CHCONF_DPE1;
+	} else if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
 		l &= ~OMAP2_MCSPI_CHCONF_IS;
 		l &= ~OMAP2_MCSPI_CHCONF_DPE1;
 		l |= OMAP2_MCSPI_CHCONF_DPE0;
@@ -1178,6 +1184,7 @@ static int omap2_mcspi_transfer_one(struct spi_controller *ctlr,
 		omap2_mcspi_set_cs(spi, spi->mode & SPI_CS_HIGH);
 
 	if (par_override ||
+	    (spi->mode & SPI_3WIRE) ||
 	    (t->speed_hz != spi->max_speed_hz) ||
 	    (t->bits_per_word != spi->bits_per_word)) {
 		par_override = 1;
@@ -1484,7 +1491,7 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	/* the spi->mode bits understood by this driver: */
-	ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
+	ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_3WIRE;
 	ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
 	ctlr->setup = omap2_mcspi_setup;
 	ctlr->auto_runtime_pm = true;
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index f7881a31e4cc0..6291d7c2e06fc 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -796,7 +796,7 @@ static irqreturn_t ssp_int(int irq, void *dev_id)
  * The function calculates parameters for all cases and chooses the one closest
  * to the asked baud rate.
  */
-static unsigned int quark_x1000_get_clk_div(int rate, u32 *dds)
+static unsigned int quark_x1000_get_clk_div(u32 rate, u32 *dds)
 {
 	unsigned long xtal = 200000000;
 	unsigned long fref = xtal / 2;		/* mandatory division by 2,
@@ -885,12 +885,12 @@ static unsigned int quark_x1000_get_clk_div(int rate, u32 *dds)
 	return q - 1;
 }
 
-static unsigned int ssp_get_clk_div(struct driver_data *drv_data, int rate)
+static unsigned int ssp_get_clk_div(struct driver_data *drv_data, u32 rate)
 {
-	unsigned long ssp_clk = drv_data->controller->max_speed_hz;
+	u32 ssp_clk = drv_data->controller->max_speed_hz;
 	const struct ssp_device *ssp = drv_data->ssp;
 
-	rate = min_t(int, ssp_clk, rate);
+	rate = min(ssp_clk, rate);
 
 	/*
 	 * Calculate the divisor for the SCR (Serial Clock Rate), avoiding
@@ -902,8 +902,7 @@ static unsigned int ssp_get_clk_div(struct driver_data *drv_data, int rate)
 		return (DIV_ROUND_UP(ssp_clk, rate) - 1)  & 0xfff;
 }
 
-static unsigned int pxa2xx_ssp_get_clk_div(struct driver_data *drv_data,
-					   int rate)
+static unsigned int pxa2xx_ssp_get_clk_div(struct driver_data *drv_data, u32 rate)
 {
 	struct chip_data *chip =
 		spi_get_ctldata(drv_data->controller->cur_msg->spi);
diff --git a/drivers/spi/spi-stm32-ospi.c b/drivers/spi/spi-stm32-ospi.c
index c98afe02a1b6b..acf2d182e8b1c 100644
--- a/drivers/spi/spi-stm32-ospi.c
+++ b/drivers/spi/spi-stm32-ospi.c
@@ -469,11 +469,6 @@ static int stm32_ospi_send(struct spi_device *spi, const struct spi_mem_op *op)
 	int timeout, err = 0, err_poll_status = 0;
 	u8 cs = spi->chip_select[ffs(spi->cs_index_mask) - 1];
 
-	dev_dbg(ospi->dev, "cmd:%#x mode:%d.%d.%d.%d addr:%#llx len:%#x\n",
-		op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth,
-		op->dummy.buswidth, op->data.buswidth,
-		op->addr.val, op->data.nbytes);
-
 	cr = readl_relaxed(ospi->regs_base + OSPI_CR);
 	cr &= ~CR_CSSEL;
 	cr |= FIELD_PREP(CR_CSSEL, cs);
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index 33f211e159ef1..8a7f5a10d4af0 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -1659,7 +1659,7 @@ static int stm32_spi_transfer_one_dma(struct stm32_spi *spi,
 			ret = stm32_spi_prepare_rx_dma_mdma_chaining(spi, xfer, &rx_dma_conf,
 								     &rx_dma_desc, &rx_mdma_desc);
 			if (ret) { /* RX DMA MDMA chaining not possible, fallback to DMA only */
-				rx_dma_conf.peripheral_config = 0;
+				rx_dma_conf.peripheral_config = NULL;
 				rx_dma_desc = NULL;
 			}
 		}
@@ -2505,7 +2505,7 @@ static int stm32_spi_probe(struct platform_device *pdev)
 			spi->dma_rx = NULL;
 		} else {
 			dev_err_probe(&pdev->dev, ret, "failed to request rx dma channel\n");
-			goto err_dma_release;
+			goto err_dma_tx_release;
 		}
 	} else {
 		ctrl->dma_rx = spi->dma_rx;
@@ -2574,11 +2574,11 @@ static int stm32_spi_probe(struct platform_device *pdev)
 	if (spi->sram_pool)
 		gen_pool_free(spi->sram_pool, (unsigned long)spi->sram_rx_buf,
 			      spi->sram_rx_buf_size);
-err_dma_release:
-	if (spi->dma_tx)
-		dma_release_channel(spi->dma_tx);
 	if (spi->dma_rx)
 		dma_release_channel(spi->dma_rx);
+err_dma_tx_release:
+	if (spi->dma_tx)
+		dma_release_channel(spi->dma_tx);
 err_clk_disable:
 	clk_disable_unprepare(spi->clk);
 
diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c
index 7ea5aa993596f..7cca5578eba31 100644
--- a/drivers/spi/spi-tegra210-quad.c
+++ b/drivers/spi/spi-tegra210-quad.c
@@ -1000,7 +1000,7 @@ static int tegra_qspi_setup(struct spi_device *spi)
 
 	spin_unlock_irqrestore(&tqspi->lock, flags);
 
-	pm_runtime_put(tqspi->dev);
+	pm_runtime_put_autosuspend(tqspi->dev);
 
 	return 0;
 }
@@ -1765,6 +1765,14 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 	init_completion(&tqspi->rx_dma_complete);
 	init_completion(&tqspi->xfer_completion);
 
+	/*
+	 * Set autosuspend delay to 500ms. Testing shows this value eliminates
+	 * suspend/resume overhead during burst operations while allowing quick
+	 * suspension during idle. For longer operations, the overhead is negligible.
+	 */
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
+	pm_runtime_use_autosuspend(&pdev->dev);
+
 	pm_runtime_enable(&pdev->dev);
 	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret < 0) {
@@ -1781,7 +1789,7 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 	tqspi->spi_cs_timing2 = tegra_qspi_readl(tqspi, QSPI_CS_TIMING2);
 	tqspi->def_command2_reg = tegra_qspi_readl(tqspi, QSPI_COMMAND2);
 
-	pm_runtime_put(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
 
 	ret = request_threaded_irq(tqspi->irq, NULL,
 				   tegra_qspi_isr_thread, IRQF_ONESHOT,
@@ -1802,6 +1810,7 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 exit_free_irq:
 	free_irq(qspi_irq, tqspi);
 exit_pm_disable:
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_force_suspend(&pdev->dev);
 	tegra_qspi_deinit_dma(tqspi);
 	return ret;
@@ -1814,6 +1823,7 @@ static void tegra_qspi_remove(struct platform_device *pdev)
 
 	spi_unregister_controller(host);
 	free_irq(tqspi->irq, tqspi);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_force_suspend(&pdev->dev);
 	tegra_qspi_deinit_dma(tqspi);
 }
@@ -1839,7 +1849,7 @@ static int __maybe_unused tegra_qspi_resume(struct device *dev)
 
 	tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1);
 	tegra_qspi_writel(tqspi, tqspi->def_command2_reg, QSPI_COMMAND2);
-	pm_runtime_put(dev);
+	pm_runtime_put_autosuspend(dev);
 
 	return spi_controller_resume(host);
 }
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 61f7bde8c7fbb..bf1df365f795e 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -96,25 +96,22 @@ static ssize_t driver_override_show(struct device *dev,
 }
 static DEVICE_ATTR_RW(driver_override);
 
-static struct spi_statistics __percpu *spi_alloc_pcpu_stats(struct device *dev)
+static struct spi_statistics __percpu *spi_alloc_pcpu_stats(void)
 {
 	struct spi_statistics __percpu *pcpu_stats;
+	int cpu;
 
-	if (dev)
-		pcpu_stats = devm_alloc_percpu(dev, struct spi_statistics);
-	else
-		pcpu_stats = alloc_percpu_gfp(struct spi_statistics, GFP_KERNEL);
-
-	if (pcpu_stats) {
-		int cpu;
+	pcpu_stats = alloc_percpu_gfp(struct spi_statistics, GFP_KERNEL);
+	if (!pcpu_stats)
+		return NULL;
 
-		for_each_possible_cpu(cpu) {
-			struct spi_statistics *stat;
+	for_each_possible_cpu(cpu) {
+		struct spi_statistics *stat;
 
-			stat = per_cpu_ptr(pcpu_stats, cpu);
-			u64_stats_init(&stat->syncp);
-		}
+		stat = per_cpu_ptr(pcpu_stats, cpu);
+		u64_stats_init(&stat->syncp);
 	}
+
 	return pcpu_stats;
 }
 
@@ -574,7 +571,7 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr)
 		return NULL;
 	}
 
-	spi->pcpu_statistics = spi_alloc_pcpu_stats(NULL);
+	spi->pcpu_statistics = spi_alloc_pcpu_stats();
 	if (!spi->pcpu_statistics) {
 		kfree(spi);
 		spi_controller_put(ctlr);
@@ -641,12 +638,26 @@ static inline int spi_dev_check_cs(struct device *dev,
 	return 0;
 }
 
+struct spi_dev_check_info {
+	struct spi_device *new_spi;
+	struct spi_device *parent;	/* set for ancillary devices */
+};
+
 static int spi_dev_check(struct device *dev, void *data)
 {
 	struct spi_device *spi = to_spi_device(dev);
-	struct spi_device *new_spi = data;
+	struct spi_dev_check_info *info = data;
+	struct spi_device *new_spi = info->new_spi;
 	int status, idx;
 
+	/*
+	 * When registering an ancillary device, skip checking against the
+	 * parent device since the ancillary is intentionally using one of
+	 * the parent's chip selects.
+	 */
+	if (info->parent && spi == info->parent)
+		return 0;
+
 	if (spi->controller == new_spi->controller) {
 		for (idx = 0; idx < spi->num_chipselect; idx++) {
 			status = spi_dev_check_cs(dev, spi, idx, new_spi, 0);
@@ -663,10 +674,11 @@ static void spi_cleanup(struct spi_device *spi)
 		spi->controller->cleanup(spi);
 }
 
-static int __spi_add_device(struct spi_device *spi)
+static int __spi_add_device(struct spi_device *spi, struct spi_device *parent)
 {
 	struct spi_controller *ctlr = spi->controller;
 	struct device *dev = ctlr->dev.parent;
+	struct spi_dev_check_info check_info;
 	int status, idx;
 	u8 cs;
 
@@ -710,7 +722,9 @@ static int __spi_add_device(struct spi_device *spi)
 	 * chipselect **BEFORE** we call setup(), else we'll trash
 	 * its configuration.
 	 */
-	status = bus_for_each_dev(&spi_bus_type, NULL, spi, spi_dev_check);
+	check_info.new_spi = spi;
+	check_info.parent = parent;
+	status = bus_for_each_dev(&spi_bus_type, NULL, &check_info, spi_dev_check);
 	if (status)
 		return status;
 
@@ -772,7 +786,7 @@ int spi_add_device(struct spi_device *spi)
 	spi_dev_set_name(spi);
 
 	mutex_lock(&ctlr->add_lock);
-	status = __spi_add_device(spi);
+	status = __spi_add_device(spi, NULL);
 	mutex_unlock(&ctlr->add_lock);
 	return status;
 }
@@ -2715,8 +2729,8 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
 
 	WARN_ON(!mutex_is_locked(&ctlr->add_lock));
 
-	/* Register the new device */
-	rc = __spi_add_device(ancillary);
+	/* Register the new device, passing the parent to skip CS conflict check */
+	rc = __spi_add_device(ancillary, spi);
 	if (rc) {
 		dev_err(&spi->dev, "failed to register ancillary device\n");
 		goto err_out;
@@ -2730,6 +2744,46 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
 }
 EXPORT_SYMBOL_GPL(spi_new_ancillary_device);
 
+static void devm_spi_unregister_device(void *spi)
+{
+	spi_unregister_device(spi);
+}
+
+/**
+ * devm_spi_new_ancillary_device() - Register managed ancillary SPI device
+ * @spi:         Pointer to the main SPI device registering the ancillary device
+ * @chip_select: Chip Select of the ancillary device
+ *
+ * Register an ancillary SPI device; for example some chips have a chip-select
+ * for normal device usage and another one for setup/firmware upload.
+ *
+ * This is the managed version of spi_new_ancillary_device(). The ancillary
+ * device will be unregistered automatically when the parent SPI device is
+ * unregistered.
+ *
+ * This may only be called from main SPI device's probe routine.
+ *
+ * Return: Pointer to new ancillary device on success; ERR_PTR on failure
+ */
+struct spi_device *devm_spi_new_ancillary_device(struct spi_device *spi,
+						 u8 chip_select)
+{
+	struct spi_device *ancillary;
+	int ret;
+
+	ancillary = spi_new_ancillary_device(spi, chip_select);
+	if (IS_ERR(ancillary))
+		return ancillary;
+
+	ret = devm_add_action_or_reset(&spi->dev, devm_spi_unregister_device,
+				       ancillary);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return ancillary;
+}
+EXPORT_SYMBOL_GPL(devm_spi_new_ancillary_device);
+
 #ifdef CONFIG_ACPI
 struct acpi_spi_lookup {
 	struct spi_controller 	*ctlr;
@@ -3049,6 +3103,8 @@ static void spi_controller_release(struct device *dev)
 	struct spi_controller *ctlr;
 
 	ctlr = container_of(dev, struct spi_controller, dev);
+
+	free_percpu(ctlr->pcpu_statistics);
 	kfree(ctlr);
 }
 
@@ -3192,6 +3248,12 @@ struct spi_controller *__spi_alloc_controller(struct device *dev,
 	if (!ctlr)
 		return NULL;
 
+	ctlr->pcpu_statistics = spi_alloc_pcpu_stats();
+	if (!ctlr->pcpu_statistics) {
+		kfree(ctlr);
+		return NULL;
+	}
+
 	device_initialize(&ctlr->dev);
 	INIT_LIST_HEAD(&ctlr->queue);
 	spin_lock_init(&ctlr->queue_lock);
@@ -3383,8 +3445,8 @@ static int spi_controller_id_alloc(struct spi_controller *ctlr, int start, int e
  * device identification, boards need configuration tables telling which
  * chip is at which address.
  *
- * This must be called from context that can sleep.  It returns zero on
- * success, else a negative error code (dropping the controller's refcount).
+ * This must be called from context that can sleep.
+ *
  * After a successful return, the caller is responsible for calling
  * spi_unregister_controller().
  *
@@ -3480,17 +3542,8 @@ int spi_register_controller(struct spi_controller *ctlr)
 		dev_info(dev, "controller is unqueued, this is deprecated\n");
 	} else if (ctlr->transfer_one || ctlr->transfer_one_message) {
 		status = spi_controller_initialize_queue(ctlr);
-		if (status) {
-			device_del(&ctlr->dev);
-			goto free_bus_id;
-		}
-	}
-	/* Add statistics */
-	ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev);
-	if (!ctlr->pcpu_statistics) {
-		dev_err(dev, "Error allocating per-cpu statistics\n");
-		status = -ENOMEM;
-		goto destroy_queue;
+		if (status)
+			goto del_ctrl;
 	}
 
 	mutex_lock(&board_lock);
@@ -3504,8 +3557,8 @@ int spi_register_controller(struct spi_controller *ctlr)
 	acpi_register_spi_devices(ctlr);
 	return status;
 
-destroy_queue:
-	spi_destroy_queue(ctlr);
+del_ctrl:
+	device_del(&ctlr->dev);
 free_bus_id:
 	mutex_lock(&board_lock);
 	idr_remove(&spi_controller_idr, ctlr->bus_num);
@@ -3527,7 +3580,8 @@ static void devm_spi_unregister_controller(void *ctlr)
  * Context: can sleep
  *
  * Register a SPI device as with spi_register_controller() which will
- * automatically be unregistered and freed.
+ * automatically be unregistered (and freed unless it has been allocated using
+ * devm_spi_alloc_host/target()).
  *
  * Return: zero on success, else a negative error code.
  */
@@ -3561,7 +3615,8 @@ static int __unregister(struct device *dev, void *null)
  *
  * This must be called from context that can sleep.
  *
- * Note that this function also drops a reference to the controller.
+ * Note that this function also drops a reference to the controller unless it
+ * has been allocated using devm_spi_alloc_host/target().
  */
 void spi_unregister_controller(struct spi_controller *ctlr)
 {
diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c
index aa90b27197cf9..3aa2aa870ea91 100644
--- a/drivers/staging/axis-fifo/axis-fifo.c
+++ b/drivers/staging/axis-fifo/axis-fifo.c
@@ -71,8 +71,8 @@ struct axis_fifo {
 
 	unsigned int rx_fifo_depth;
 	unsigned int tx_fifo_depth;
-	int has_rx_fifo;
-	int has_tx_fifo;
+	u32 has_rx_fifo;
+	u32 has_tx_fifo;
 
 	wait_queue_head_t read_queue;
 	struct mutex read_lock; /* lock for reading */
@@ -392,60 +392,39 @@ static int axis_fifo_parse_dt(struct axis_fifo *fifo)
 
 	ret = of_property_read_u32(node, "xlnx,axi-str-rxd-tdata-width",
 				   &value);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,axi-str-rxd-tdata-width property\n");
-		goto end;
-	} else if (value != 32) {
-		dev_err(fifo->dt_device, "xlnx,axi-str-rxd-tdata-width only supports 32 bits\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
+	if (value != 32)
+		return -EINVAL;
 
 	ret = of_property_read_u32(node, "xlnx,axi-str-txd-tdata-width",
 				   &value);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,axi-str-txd-tdata-width property\n");
-		goto end;
-	} else if (value != 32) {
-		dev_err(fifo->dt_device, "xlnx,axi-str-txd-tdata-width only supports 32 bits\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
+	if (value != 32)
+		return -EINVAL;
 
 	ret = of_property_read_u32(node, "xlnx,rx-fifo-depth",
 				   &fifo->rx_fifo_depth);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,rx-fifo-depth property\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
 
 	ret = of_property_read_u32(node, "xlnx,tx-fifo-depth",
 				   &fifo->tx_fifo_depth);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,tx-fifo-depth property\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
 
 	ret = of_property_read_u32(node, "xlnx,use-rx-data",
 				   &fifo->has_rx_fifo);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,use-rx-data property\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
 
 	ret = of_property_read_u32(node, "xlnx,use-tx-data",
 				   &fifo->has_tx_fifo);
-	if (ret) {
-		dev_err(fifo->dt_device, "missing xlnx,use-tx-data property\n");
-		ret = -EIO;
-		goto end;
-	}
+	if (ret)
+		return ret;
 
-end:
-	return ret;
+	return 0;
 }
 
 static int axis_fifo_probe(struct platform_device *pdev)
diff --git a/drivers/staging/fbtft/Kconfig b/drivers/staging/fbtft/Kconfig
index 578412a2f379e..92943564cb91b 100644
--- a/drivers/staging/fbtft/Kconfig
+++ b/drivers/staging/fbtft/Kconfig
@@ -86,7 +86,11 @@ config FB_TFT_PCD8544
 config FB_TFT_RA8875
 	tristate "FB driver for the RA8875 LCD Controller"
 	help
-	  Generic Framebuffer support for RA8875
+	  This enables generic framebuffer support for the RAiO RA8875
+	  display controller. The controller is intended for medium size text/graphic
+	  mixed displays with a resolution of up to 800x480 pixels.
+
+	  Say Y if you have such a display that utilizes this controller.
 
 config FB_TFT_S6D02A1
 	tristate "FB driver for the S6D02A1 LCD Controller"
diff --git a/drivers/staging/fbtft/README b/drivers/staging/fbtft/README
index ba4c74c92e4c3..91f152d622bd5 100644
--- a/drivers/staging/fbtft/README
+++ b/drivers/staging/fbtft/README
@@ -6,27 +6,12 @@ The module 'fbtft' makes writing drivers for some of these displays very easy.
 
 Development is done on a Raspberry Pi running the Raspbian "wheezy" distribution.
 
-INSTALLATION
-  Download kernel sources
+For new hardware support consider using DRM subsystem (see TODO).
 
-  From Linux 3.15
-    cd drivers/video/fbdev/fbtft
-    git clone https://github.com/notro/fbtft.git
+NOTE:
 
-    Add to drivers/video/fbdev/Kconfig:   source "drivers/video/fbdev/fbtft/Kconfig"
-    Add to drivers/video/fbdev/Makefile:  obj-y += fbtft/
-
-  Before Linux 3.15
-    cd drivers/video
-    git clone https://github.com/notro/fbtft.git
-
-    Add to drivers/video/Kconfig:   source "drivers/video/fbtft/Kconfig"
-    Add to drivers/video/Makefile:  obj-y += fbtft/
-
-  Enable driver(s) in menuconfig and build the kernel
-
-
-See wiki for more information: https://github.com/notro/fbtft/wiki
-
-
-Source: https://github.com/notro/fbtft/
+The driver is in maintenance mode, only performance issue or bug fixes
+are accepted, which effectively means the patches must be tested on
+the real hardware (the patch must be accompanied with the information
+what hardware is that). The treewide changes may also be accepted as
+an exception.
diff --git a/drivers/staging/fbtft/fb_agm1264k-fl.c b/drivers/staging/fbtft/fb_agm1264k-fl.c
index af2dbebefc72b..6fc8f4e9c814d 100644
--- a/drivers/staging/fbtft/fb_agm1264k-fl.c
+++ b/drivers/staging/fbtft/fb_agm1264k-fl.c
@@ -376,7 +376,7 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 
 			/* write bitmap */
 			gpiod_set_value(par->RS, 1); /* RS->1 (data mode) */
-			par->fbtftops.write(par, buf, len);
+			ret = par->fbtftops.write(par, buf, len);
 			if (ret < 0)
 				dev_err(par->info->device,
 					"write failed and returned: %d\n",
diff --git a/drivers/staging/fbtft/fb_hx8340bn.c b/drivers/staging/fbtft/fb_hx8340bn.c
index 2fd7b87ea0cef..ca27914f14122 100644
--- a/drivers/staging/fbtft/fb_hx8340bn.c
+++ b/drivers/staging/fbtft/fb_hx8340bn.c
@@ -106,13 +106,6 @@ static int init_display(struct fbtft_par *par)
 	return 0;
 }
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS, 0x00, xs, 0x00, xe);
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS, 0x00, ys, 0x00, ye);
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 static int set_var(struct fbtft_par *par)
 {
 	/* MADCTL - Memory data access control */
@@ -207,7 +200,6 @@ static struct fbtft_display display = {
 	.gamma = DEFAULT_GAMMA,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 		.set_gamma = set_gamma,
 	},
diff --git a/drivers/staging/fbtft/fb_hx8353d.c b/drivers/staging/fbtft/fb_hx8353d.c
index 3e73b69b6a276..f6cd82df4da60 100644
--- a/drivers/staging/fbtft/fb_hx8353d.c
+++ b/drivers/staging/fbtft/fb_hx8353d.c
@@ -61,18 +61,6 @@ static int init_display(struct fbtft_par *par)
 	return 0;
 };
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	/* column address */
-	write_reg(par, 0x2a, xs >> 8, xs & 0xff, xe >> 8, xe & 0xff);
-
-	/* Row address */
-	write_reg(par, 0x2b, ys >> 8, ys & 0xff, ye >> 8, ye & 0xff);
-
-	/* memory write */
-	write_reg(par, 0x2c);
-}
-
 #define my BIT(7)
 #define mx BIT(6)
 #define mv BIT(5)
@@ -130,7 +118,6 @@ static struct fbtft_display display = {
 	.gamma = DEFAULT_GAMMA,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 		.set_gamma = set_gamma,
 	},
diff --git a/drivers/staging/fbtft/fb_hx8357d.c b/drivers/staging/fbtft/fb_hx8357d.c
index 94a357e8fdf6a..7b9f020a956f4 100644
--- a/drivers/staging/fbtft/fb_hx8357d.c
+++ b/drivers/staging/fbtft/fb_hx8357d.c
@@ -129,19 +129,6 @@ static int init_display(struct fbtft_par *par)
 	return 0;
 }
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xff,  /* XSTART */
-		  xe >> 8, xe & 0xff); /* XEND */
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xff,  /* YSTART */
-		  ye >> 8, ye & 0xff); /* YEND */
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define HX8357D_MADCTL_MY  0x80
 #define HX8357D_MADCTL_MX  0x40
 #define HX8357D_MADCTL_MV  0x20
@@ -184,7 +171,6 @@ static struct fbtft_display display = {
 	.gamma_len = 14,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fb_ili9163.c b/drivers/staging/fbtft/fb_ili9163.c
index 6582a2c90aafc..a2b5033a98603 100644
--- a/drivers/staging/fbtft/fb_ili9163.c
+++ b/drivers/staging/fbtft/fb_ili9163.c
@@ -60,7 +60,7 @@
  * configure to constrain the memory and resolution to a fixed dimension (in
  * that case 128x128) but they leaved those pins configured for 128x160 so
  * there was several pixel memory addressing problems.
- * I solved by setup several parameters that dinamically fix the resolution as
+ * I solved by setup several parameters that dynamically fix the resolution as
  * needit so below the parameters for this display. If you have a strain or a
  * correct display (can happen with chinese) you can copy those parameters and
  * create setup for different displays.
diff --git a/drivers/staging/fbtft/fb_ili9340.c b/drivers/staging/fbtft/fb_ili9340.c
index 704236bcaf3ff..023d8cb96f955 100644
--- a/drivers/staging/fbtft/fb_ili9340.c
+++ b/drivers/staging/fbtft/fb_ili9340.c
@@ -78,17 +78,6 @@ static int init_display(struct fbtft_par *par)
 	return 0;
 }
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xFF, xe >> 8, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xFF, ye >> 8, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define ILI9340_MADCTL_MV  0x20
 #define ILI9340_MADCTL_MX  0x40
 #define ILI9340_MADCTL_MY  0x80
@@ -122,7 +111,6 @@ static struct fbtft_display display = {
 	.height = HEIGHT,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fb_ili9341.c b/drivers/staging/fbtft/fb_ili9341.c
index 47e72b87d76d9..428922dee9f90 100644
--- a/drivers/staging/fbtft/fb_ili9341.c
+++ b/drivers/staging/fbtft/fb_ili9341.c
@@ -65,17 +65,6 @@ static int init_display(struct fbtft_par *par)
 	return 0;
 }
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  (xs >> 8) & 0xFF, xs & 0xFF, (xe >> 8) & 0xFF, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  (ys >> 8) & 0xFF, ys & 0xFF, (ye >> 8) & 0xFF, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define MEM_Y   BIT(7) /* MY row address order */
 #define MEM_X   BIT(6) /* MX column address order */
 #define MEM_V   BIT(5) /* MV row / column exchange */
@@ -139,7 +128,6 @@ static struct fbtft_display display = {
 	.gamma = DEFAULT_GAMMA,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 		.set_gamma = set_gamma,
 	},
diff --git a/drivers/staging/fbtft/fb_ili9481.c b/drivers/staging/fbtft/fb_ili9481.c
index 19eba085ea53e..5f31b5d5590f0 100644
--- a/drivers/staging/fbtft/fb_ili9481.c
+++ b/drivers/staging/fbtft/fb_ili9481.c
@@ -42,17 +42,6 @@ static const s16 default_init_sequence[] = {
 	-3
 };
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xff, xe >> 8, xe & 0xff);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xff, ye >> 8, ye & 0xff);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define HFLIP 0x01
 #define VFLIP 0x02
 #define ROW_X_COL 0x20
@@ -86,7 +75,6 @@ static struct fbtft_display display = {
 	.height = HEIGHT,
 	.init_sequence = default_init_sequence,
 	.fbtftops = {
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fb_ili9486.c b/drivers/staging/fbtft/fb_ili9486.c
index 66210a7137fcc..a4d699ef57e4d 100644
--- a/drivers/staging/fbtft/fb_ili9486.c
+++ b/drivers/staging/fbtft/fb_ili9486.c
@@ -43,17 +43,6 @@ static const s16 default_init_sequence[] = {
 	-3
 };
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xFF, xe >> 8, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xFF, ye >> 8, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 static int set_var(struct fbtft_par *par)
 {
 	switch (par->info->var.rotate) {
@@ -86,7 +75,6 @@ static struct fbtft_display display = {
 	.height = HEIGHT,
 	.init_sequence = default_init_sequence,
 	.fbtftops = {
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fb_s6d02a1.c b/drivers/staging/fbtft/fb_s6d02a1.c
index d3d6871d8c47f..d8ddc804d626e 100644
--- a/drivers/staging/fbtft/fb_s6d02a1.c
+++ b/drivers/staging/fbtft/fb_s6d02a1.c
@@ -97,17 +97,6 @@ static const s16 default_init_sequence[] = {
 
 };
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xFF, xe >> 8, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xFF, ye >> 8, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define MY BIT(7)
 #define MX BIT(6)
 #define MV BIT(5)
@@ -149,7 +138,6 @@ static struct fbtft_display display = {
 	.height = 160,
 	.init_sequence = default_init_sequence,
 	.fbtftops = {
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fb_st7735r.c b/drivers/staging/fbtft/fb_st7735r.c
index 9670a8989b917..6d9735fa23324 100644
--- a/drivers/staging/fbtft/fb_st7735r.c
+++ b/drivers/staging/fbtft/fb_st7735r.c
@@ -83,17 +83,6 @@ static const s16 default_init_sequence[] = {
 	-3
 };
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xFF, xe >> 8, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xFF, ye >> 8, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 #define MY BIT(7)
 #define MX BIT(6)
 #define MV BIT(5)
@@ -168,7 +157,6 @@ static struct fbtft_display display = {
 	.gamma_len = 16,
 	.gamma = DEFAULT_GAMMA,
 	.fbtftops = {
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 		.set_gamma = set_gamma,
 	},
diff --git a/drivers/staging/fbtft/fb_tinylcd.c b/drivers/staging/fbtft/fb_tinylcd.c
index 9469248f2c507..afa8f1c740c56 100644
--- a/drivers/staging/fbtft/fb_tinylcd.c
+++ b/drivers/staging/fbtft/fb_tinylcd.c
@@ -41,23 +41,12 @@ static int init_display(struct fbtft_par *par)
 		       0x00, 0x35, 0x33, 0x00, 0x00, 0x00);
 	write_reg(par, MIPI_DCS_SET_PIXEL_FORMAT, 0x55);
 	write_reg(par, MIPI_DCS_EXIT_SLEEP_MODE);
-	udelay(250);
+	fsleep(250);
 	write_reg(par, MIPI_DCS_SET_DISPLAY_ON);
 
 	return 0;
 }
 
-static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
-{
-	write_reg(par, MIPI_DCS_SET_COLUMN_ADDRESS,
-		  xs >> 8, xs & 0xFF, xe >> 8, xe & 0xFF);
-
-	write_reg(par, MIPI_DCS_SET_PAGE_ADDRESS,
-		  ys >> 8, ys & 0xFF, ye >> 8, ye & 0xFF);
-
-	write_reg(par, MIPI_DCS_WRITE_MEMORY_START);
-}
-
 static int set_var(struct fbtft_par *par)
 {
 	switch (par->info->var.rotate) {
@@ -88,7 +77,6 @@ static struct fbtft_display display = {
 	.height = HEIGHT,
 	.fbtftops = {
 		.init_display = init_display,
-		.set_addr_win = set_addr_win,
 		.set_var = set_var,
 	},
 };
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index f427c0914907e..3da42c8ca6e33 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -300,12 +300,6 @@ static void fbtft_mkdirty(struct fb_info *info, int y, int height)
 	struct fbtft_par *par = info->par;
 	struct fb_deferred_io *fbdefio = info->fbdefio;
 
-	/* special case, needed ? */
-	if (y == -1) {
-		y = 0;
-		height = info->var.yres;
-	}
-
 	/* Mark display lines/area as dirty */
 	spin_lock(&par->dirty_lock);
 	if (y < par->dirty_lines_start)
@@ -413,9 +407,12 @@ static int fbtft_fb_blank(int blank, struct fb_info *info)
 static void fbtft_ops_damage_range(struct fb_info *info, off_t off, size_t len)
 {
 	struct fbtft_par *par = info->par;
+	u32 start, end;
+
+	start = off / info->fix.line_length;
+	end = (off + len - 1) / info->fix.line_length;
 
-	/* TODO: only mark changed area update all for now */
-	par->fbtftops.mkdirty(info, -1, 0);
+	par->fbtftops.mkdirty(info, start, end - start + 1);
 }
 
 static void fbtft_ops_damage_area(struct fb_info *info, u32 x, u32 y, u32 width, u32 height)
diff --git a/drivers/staging/greybus/Documentation/firmware/authenticate.c b/drivers/staging/greybus/Documentation/firmware/authenticate.c
index 3d2c6f88a138a..0ef88b7d24de0 100644
--- a/drivers/staging/greybus/Documentation/firmware/authenticate.c
+++ b/drivers/staging/greybus/Documentation/firmware/authenticate.c
@@ -58,7 +58,7 @@ int main(int argc, char *argv[])
 		goto close_fd;
 	}
 
-	printf("UID received: 0x%llx\n", *(unsigned long long int *)(uid.uid));
+	printf("UID received: 0x%llx\n", *(unsigned long long *)(uid.uid));
 
 	/* Get certificate */
 	printf("Get IMS certificate\n");
diff --git a/drivers/staging/greybus/Kconfig b/drivers/staging/greybus/Kconfig
index 1e745a8d439c8..7635f3e850faf 100644
--- a/drivers/staging/greybus/Kconfig
+++ b/drivers/staging/greybus/Kconfig
@@ -123,7 +123,7 @@ menuconfig GREYBUS_BRIDGED_PHY
 	help
 	  Select this option to pick from a variety of Greybus Bridged
 	  PHY class drivers.  These drivers emulate a number of
-	  different "traditional" busses by tunneling them over Greybus.
+	  different "traditional" buses by tunneling them over Greybus.
 	  Examples of this include serial, SPI, USB, and others.
 
 	  To compile this code as a module, chose M here: the module
diff --git a/drivers/staging/greybus/arche-apb-ctrl.c b/drivers/staging/greybus/arche-apb-ctrl.c
index 33f26a65f0cc0..19a6e59b6d5c1 100644
--- a/drivers/staging/greybus/arche-apb-ctrl.c
+++ b/drivers/staging/greybus/arche-apb-ctrl.c
@@ -300,16 +300,16 @@ static ssize_t state_show(struct device *dev,
 
 	switch (apb->state) {
 	case ARCHE_PLATFORM_STATE_OFF:
-		return sprintf(buf, "off%s\n",
+		return sysfs_emit(buf, "off%s\n",
 				apb->init_disabled ? ",disabled" : "");
 	case ARCHE_PLATFORM_STATE_ACTIVE:
-		return sprintf(buf, "active\n");
+		return sysfs_emit(buf, "active\n");
 	case ARCHE_PLATFORM_STATE_STANDBY:
-		return sprintf(buf, "standby\n");
+		return sysfs_emit(buf, "standby\n");
 	case ARCHE_PLATFORM_STATE_FW_FLASHING:
-		return sprintf(buf, "fw_flashing\n");
+		return sysfs_emit(buf, "fw_flashing\n");
 	default:
-		return sprintf(buf, "unknown state\n");
+		return sysfs_emit(buf, "unknown state\n");
 	}
 }
 
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
index f669a7e2eb11d..de5de59ea8ab2 100644
--- a/drivers/staging/greybus/arche-platform.c
+++ b/drivers/staging/greybus/arche-platform.c
@@ -374,15 +374,15 @@ static ssize_t state_show(struct device *dev,
 
 	switch (arche_pdata->state) {
 	case ARCHE_PLATFORM_STATE_OFF:
-		return sprintf(buf, "off\n");
+		return sysfs_emit(buf, "off\n");
 	case ARCHE_PLATFORM_STATE_ACTIVE:
-		return sprintf(buf, "active\n");
+		return sysfs_emit(buf, "active\n");
 	case ARCHE_PLATFORM_STATE_STANDBY:
-		return sprintf(buf, "standby\n");
+		return sysfs_emit(buf, "standby\n");
 	case ARCHE_PLATFORM_STATE_FW_FLASHING:
-		return sprintf(buf, "fw_flashing\n");
+		return sysfs_emit(buf, "fw_flashing\n");
 	default:
-		return sprintf(buf, "unknown state\n");
+		return sysfs_emit(buf, "unknown state\n");
 	}
 }
 
diff --git a/drivers/staging/greybus/audio_manager.c b/drivers/staging/greybus/audio_manager.c
index 27ca5f796c5f3..118ada9b909b6 100644
--- a/drivers/staging/greybus/audio_manager.c
+++ b/drivers/staging/greybus/audio_manager.c
@@ -105,18 +105,6 @@ void gb_audio_manager_remove_all(void)
 }
 EXPORT_SYMBOL_GPL(gb_audio_manager_remove_all);
 
-struct gb_audio_manager_module *gb_audio_manager_get_module(int id)
-{
-	struct gb_audio_manager_module *module;
-
-	down_read(&modules_rwsem);
-	module = gb_audio_manager_get_locked(id);
-	kobject_get(&module->kobj);
-	up_read(&modules_rwsem);
-	return module;
-}
-EXPORT_SYMBOL_GPL(gb_audio_manager_get_module);
-
 void gb_audio_manager_put_module(struct gb_audio_manager_module *module)
 {
 	kobject_put(&module->kobj);
diff --git a/drivers/staging/greybus/audio_manager.h b/drivers/staging/greybus/audio_manager.h
index be605485a8cea..c3ef62ee22c88 100644
--- a/drivers/staging/greybus/audio_manager.h
+++ b/drivers/staging/greybus/audio_manager.h
@@ -54,13 +54,6 @@ int gb_audio_manager_remove(int id);
  */
 void gb_audio_manager_remove_all(void);
 
-/*
- * Retrieves a gb_audio_manager_module_descriptor for the specified id.
- * Returns the gb_audio_manager_module_descriptor structure,
- * or NULL if there is no module with the specified ID.
- */
-struct gb_audio_manager_module *gb_audio_manager_get_module(int id);
-
 /*
  * Decreases the refcount of the module, obtained by the get function.
  * Modules are removed via gb_audio_manager_remove
diff --git a/drivers/staging/greybus/audio_manager_module.c b/drivers/staging/greybus/audio_manager_module.c
index e87b82ca6d8a2..dc90cc2d23084 100644
--- a/drivers/staging/greybus/audio_manager_module.c
+++ b/drivers/staging/greybus/audio_manager_module.c
@@ -69,14 +69,13 @@ static void gb_audio_module_release(struct kobject *kobj)
 	struct gb_audio_manager_module *module = to_gb_audio_module(kobj);
 
 	pr_info("Destroying audio module #%d\n", module->id);
-	/* TODO -> delete from list */
 	kfree(module);
 }
 
 static ssize_t gb_audio_module_name_show(struct gb_audio_manager_module *module,
 					 struct gb_audio_manager_module_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%s", module->desc.name);
+	return sysfs_emit(buf, "%s", module->desc.name);
 }
 
 static struct gb_audio_manager_module_attribute gb_audio_module_name_attribute =
@@ -85,7 +84,7 @@ static struct gb_audio_manager_module_attribute gb_audio_module_name_attribute =
 static ssize_t gb_audio_module_vid_show(struct gb_audio_manager_module *module,
 					struct gb_audio_manager_module_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d", module->desc.vid);
+	return sysfs_emit(buf, "%d", module->desc.vid);
 }
 
 static struct gb_audio_manager_module_attribute gb_audio_module_vid_attribute =
@@ -94,7 +93,7 @@ static struct gb_audio_manager_module_attribute gb_audio_module_vid_attribute =
 static ssize_t gb_audio_module_pid_show(struct gb_audio_manager_module *module,
 					struct gb_audio_manager_module_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d", module->desc.pid);
+	return sysfs_emit(buf, "%d", module->desc.pid);
 }
 
 static struct gb_audio_manager_module_attribute gb_audio_module_pid_attribute =
@@ -104,7 +103,7 @@ static ssize_t gb_audio_module_intf_id_show(struct gb_audio_manager_module *modu
 					    struct gb_audio_manager_module_attribute *attr,
 					    char *buf)
 {
-	return sprintf(buf, "%d", module->desc.intf_id);
+	return sysfs_emit(buf, "%d", module->desc.intf_id);
 }
 
 static struct gb_audio_manager_module_attribute
@@ -115,7 +114,7 @@ static ssize_t gb_audio_module_ip_devices_show(struct gb_audio_manager_module *m
 					       struct gb_audio_manager_module_attribute *attr,
 					       char *buf)
 {
-	return sprintf(buf, "0x%X", module->desc.ip_devices);
+	return sysfs_emit(buf, "0x%X", module->desc.ip_devices);
 }
 
 static struct gb_audio_manager_module_attribute
@@ -126,7 +125,7 @@ static ssize_t gb_audio_module_op_devices_show(struct gb_audio_manager_module *m
 					       struct gb_audio_manager_module_attribute *attr,
 					       char *buf)
 {
-	return sprintf(buf, "0x%X", module->desc.op_devices);
+	return sysfs_emit(buf, "0x%X", module->desc.op_devices);
 }
 
 static struct gb_audio_manager_module_attribute
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
index aa9c73cb0ae5a..ea57b1f5d1562 100644
--- a/drivers/staging/greybus/loopback.c
+++ b/drivers/staging/greybus/loopback.c
@@ -125,7 +125,7 @@ static ssize_t field##_show(struct device *dev,			\
 			    char *buf)					\
 {									\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	return sprintf(buf, "%u\n", gb->field);			\
+	return sysfs_emit(buf, "%u\n", gb->field);			\
 }									\
 static DEVICE_ATTR_RO(field)
 
@@ -137,8 +137,8 @@ static ssize_t name##_##field##_show(struct device *dev,	\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
 	/* Report 0 for min and max if no transfer succeeded */		\
 	if (!gb->requests_completed)					\
-		return sprintf(buf, "0\n");				\
-	return sprintf(buf, "%" #type "\n", gb->name.field);		\
+		return sysfs_emit(buf, "0\n");				\
+	return sysfs_emit(buf, "%" #type "\n", gb->name.field);		\
 }									\
 static DEVICE_ATTR_RO(name##_##field)
 
@@ -158,7 +158,7 @@ static ssize_t name##_avg_show(struct device *dev,		\
 	rem = do_div(avg, count);					\
 	rem *= 1000000;							\
 	do_div(rem, count);						\
-	return sprintf(buf, "%llu.%06u\n", avg, (u32)rem);		\
+	return sysfs_emit(buf, "%llu.%06u\n", avg, (u32)rem);		\
 }									\
 static DEVICE_ATTR_RO(name##_avg)
 
@@ -173,7 +173,7 @@ static ssize_t field##_show(struct device *dev,				\
 			    char *buf)					\
 {									\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	return sprintf(buf, "%" #type "\n", gb->field);			\
+	return sysfs_emit(buf, "%" #type "\n", gb->field);			\
 }									\
 static ssize_t field##_store(struct device *dev,			\
 			    struct device_attribute *attr,		\
@@ -193,13 +193,13 @@ static ssize_t field##_store(struct device *dev,			\
 }									\
 static DEVICE_ATTR_RW(field)
 
-#define gb_dev_loopback_ro_attr(field, conn)				\
+#define gb_dev_loopback_ro_attr(field)				\
 static ssize_t field##_show(struct device *dev,		\
 			    struct device_attribute *attr,		\
 			    char *buf)					\
 {									\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	return sprintf(buf, "%u\n", gb->field);				\
+	return sysfs_emit(buf, "%u\n", gb->field);				\
 }									\
 static DEVICE_ATTR_RO(field)
 
@@ -209,7 +209,7 @@ static ssize_t field##_show(struct device *dev,				\
 			    char *buf)					\
 {									\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	return sprintf(buf, "%" #type "\n", gb->field);			\
+	return sysfs_emit(buf, "%" #type "\n", gb->field);			\
 }									\
 static ssize_t field##_store(struct device *dev,			\
 			    struct device_attribute *attr,		\
@@ -305,7 +305,7 @@ gb_dev_loopback_rw_attr(us_wait, d);
 /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */
 gb_dev_loopback_rw_attr(iteration_max, u);
 /* The current index of the for (i = 0; i < iteration_max; i++) loop */
-gb_dev_loopback_ro_attr(iteration_count, false);
+gb_dev_loopback_ro_attr(iteration_count);
 /* A flag to indicate synchronous or asynchronous operations */
 gb_dev_loopback_rw_attr(async, u);
 /* Timeout of an individual asynchronous request */
diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c
index 1cb7b9089ead1..52515a4acb121 100644
--- a/drivers/staging/greybus/pwm.c
+++ b/drivers/staging/greybus/pwm.c
@@ -215,7 +215,7 @@ static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	/*
 	 * Set period and duty cycle
 	 *
-	 * PWM privodes 64-bit period and duty_cycle, but greybus only accepts
+	 * PWM provides 64-bit period and duty_cycle, but greybus only accepts
 	 * 32-bit, so their values have to be limited to U32_MAX.
 	 */
 	if (period > U32_MAX)
diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c
index 12c36a5e1d8ce..3952f3d225dbe 100644
--- a/drivers/staging/greybus/sdio.c
+++ b/drivers/staging/greybus/sdio.c
@@ -206,7 +206,7 @@ static int gb_sdio_request_handler(struct gb_operation *op)
 	struct gb_message *request;
 	struct gb_sdio_event_request *payload;
 	u8 type = op->type;
-	int ret =  0;
+	int ret = 0;
 	u8 event;
 
 	if (type != GB_SDIO_TYPE_EVENT) {
@@ -372,7 +372,7 @@ static int gb_sdio_transfer(struct gb_sdio_host *host, struct mmc_data *data)
 	left = data->blksz * data->blocks;
 
 	while (left) {
-		/* check is a stop transmission is pending */
+		/* check if a stop transmission is pending */
 		spin_lock(&host->xfer);
 		if (host->xfer_stop) {
 			host->xfer_stop = false;
diff --git a/drivers/staging/iio/adc/ad7816.c b/drivers/staging/iio/adc/ad7816.c
index 172acf135f3bd..0e32a2295990d 100644
--- a/drivers/staging/iio/adc/ad7816.c
+++ b/drivers/staging/iio/adc/ad7816.c
@@ -124,8 +124,8 @@ static ssize_t ad7816_show_mode(struct device *dev,
 	struct ad7816_chip_info *chip = iio_priv(indio_dev);
 
 	if (chip->mode)
-		return sprintf(buf, "power-save\n");
-	return sprintf(buf, "full\n");
+		return sysfs_emit(buf, "power-save\n");
+	return sysfs_emit(buf, "full\n");
 }
 
 static ssize_t ad7816_store_mode(struct device *dev,
@@ -156,7 +156,7 @@ static ssize_t ad7816_show_available_modes(struct device *dev,
 					   struct device_attribute *attr,
 					   char *buf)
 {
-	return sprintf(buf, "full\npower-save\n");
+	return sysfs_emit(buf, "full\npower-save\n");
 }
 
 static IIO_DEVICE_ATTR(available_modes, 0444, ad7816_show_available_modes,
@@ -169,7 +169,7 @@ static ssize_t ad7816_show_channel(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7816_chip_info *chip = iio_priv(indio_dev);
 
-	return sprintf(buf, "%d\n", chip->channel_id);
+	return sysfs_emit(buf, "%d\n", chip->channel_id);
 }
 
 static ssize_t ad7816_store_channel(struct device *dev,
@@ -231,9 +231,9 @@ static ssize_t ad7816_show_value(struct device *dev,
 		data &= AD7816_TEMP_FLOAT_MASK;
 		if (value < 0)
 			data = BIT(AD7816_TEMP_FLOAT_OFFSET) - data;
-		return sprintf(buf, "%d.%.2d\n", value, data * 25);
+		return sysfs_emit(buf, "%d.%.2d\n", value, data * 25);
 	}
-	return sprintf(buf, "%u\n", data);
+	return sysfs_emit(buf, "%u\n", data);
 }
 
 static IIO_DEVICE_ATTR(value, 0444, ad7816_show_value, NULL, 0);
@@ -281,9 +281,9 @@ static ssize_t ad7816_show_oti(struct device *dev,
 		value = AD7816_BOUND_VALUE_MIN +
 			(chip->oti_data[chip->channel_id] -
 			AD7816_BOUND_VALUE_BASE);
-		return sprintf(buf, "%d\n", value);
+		return sysfs_emit(buf, "%d\n", value);
 	}
-	return sprintf(buf, "%u\n", chip->oti_data[chip->channel_id]);
+	return sysfs_emit(buf, "%u\n", chip->oti_data[chip->channel_id]);
 }
 
 static inline ssize_t ad7816_set_oti(struct device *dev,
diff --git a/drivers/staging/iio/impedance-analyzer/ad5933.c b/drivers/staging/iio/impedance-analyzer/ad5933.c
index 85a4223295cd0..f6d3d98b6c6a9 100644
--- a/drivers/staging/iio/impedance-analyzer/ad5933.c
+++ b/drivers/staging/iio/impedance-analyzer/ad5933.c
@@ -5,6 +5,7 @@
  * Copyright 2011 Analog Devices Inc.
  */
 
+#include <linux/bits.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/device.h>
@@ -194,8 +195,7 @@ static int ad5933_set_freq(struct ad5933_state *st,
 		u8 d8[4];
 	} dat;
 
-	freqreg = (u64)freq * (u64)(1 << 27);
-	do_div(freqreg, st->mclk_hz / 4);
+	freqreg = div64_ul(BIT_ULL(27) * freq, st->mclk_hz / 4);
 
 	switch (reg) {
 	case AD5933_REG_FREQ_START:
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index ab250c89cd4d4..1aa31bddf9703 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -34,8 +34,6 @@ source "drivers/staging/media/max96712/Kconfig"
 
 source "drivers/staging/media/meson/vdec/Kconfig"
 
-source "drivers/staging/media/starfive/Kconfig"
-
 source "drivers/staging/media/sunxi/Kconfig"
 
 source "drivers/staging/media/tegra-video/Kconfig"
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index 4a073938b2b2d..6f78b0edde1e3 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -4,7 +4,6 @@ obj-$(CONFIG_INTEL_ATOMISP)     += atomisp/
 obj-$(CONFIG_VIDEO_IMX_MEDIA)	+= imx/
 obj-$(CONFIG_VIDEO_MAX96712)	+= max96712/
 obj-$(CONFIG_VIDEO_MESON_VDEC)	+= meson/vdec/
-obj-$(CONFIG_VIDEO_STARFIVE_CAMSS)	+= starfive/
 obj-$(CONFIG_VIDEO_SUNXI)	+= sunxi/
 obj-$(CONFIG_VIDEO_TEGRA)	+= tegra-video/
 obj-$(CONFIG_VIDEO_IPU3_IMGU)	+= ipu3/
diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
index a7f22de1c8899..50366bf10f327 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
@@ -1356,6 +1356,10 @@ static int atomisp_s_parm(struct file *file, void *fh,
 static long atomisp_vidioc_default(struct file *file, void *fh,
 				   bool valid_prio, unsigned int cmd, void *arg)
 {
+	/* Disable all private IOCTLs for now! */
+	if (cmd)
+		return -EINVAL;
+
 	struct video_device *vdev = video_devdata(file);
 	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
 	int err;
diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c
index 607992100baf2..014d0c6f0a8ba 100644
--- a/drivers/staging/media/av7110/av7110.c
+++ b/drivers/staging/media/av7110/av7110.c
@@ -460,7 +460,7 @@ static void gpioirq(struct tasklet_struct *t)
 
 	if (saa7146_wait_for_debi_done(av7110->dev, 0)) {
 		pr_err("%s(): saa7146_wait_for_debi_done timed out\n", __func__);
-		BUG(); /* maybe we should try resetting the debi? */
+		return;
 	}
 
 	spin_lock(&av7110->debilock);
diff --git a/drivers/staging/media/av7110/av7110_ir.c b/drivers/staging/media/av7110/av7110_ir.c
index 68b3979ba5f20..fdae467fd7ab8 100644
--- a/drivers/staging/media/av7110/av7110_ir.c
+++ b/drivers/staging/media/av7110/av7110_ir.c
@@ -151,6 +151,7 @@ int av7110_ir_init(struct av7110 *av7110)
 void av7110_ir_exit(struct av7110 *av7110)
 {
 	rc_unregister_device(av7110->ir.rcdev);
+	rc_free_device(av7110->ir.rcdev);
 }
 
 //MODULE_AUTHOR("Holger Waechtler <holger@convergence.de>, Oliver Endriss <o.endriss@gmx.de>");
diff --git a/drivers/staging/media/ipu3/ipu3.c b/drivers/staging/media/ipu3/ipu3.c
index bdf5a457752b2..84c4d0bf027d5 100644
--- a/drivers/staging/media/ipu3/ipu3.c
+++ b/drivers/staging/media/ipu3/ipu3.c
@@ -151,7 +151,7 @@ static int imgu_dummybufs_init(struct imgu_device *imgu, unsigned int pipe)
 
 /* May be called from atomic context */
 static struct imgu_css_buffer *imgu_dummybufs_get(struct imgu_device *imgu,
-						   int queue, unsigned int pipe)
+						  int queue, unsigned int pipe)
 {
 	unsigned int i;
 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
diff --git a/drivers/staging/media/starfive/Kconfig b/drivers/staging/media/starfive/Kconfig
deleted file mode 100644
index 34727cf560722..0000000000000
--- a/drivers/staging/media/starfive/Kconfig
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-comment "StarFive media platform drivers"
-
-source "drivers/staging/media/starfive/camss/Kconfig"
diff --git a/drivers/staging/media/starfive/Makefile b/drivers/staging/media/starfive/Makefile
deleted file mode 100644
index 4639fa1bca325..0000000000000
--- a/drivers/staging/media/starfive/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-obj-y += camss/
diff --git a/drivers/staging/media/starfive/camss/Kconfig b/drivers/staging/media/starfive/camss/Kconfig
deleted file mode 100644
index 9ea5708fe4095..0000000000000
--- a/drivers/staging/media/starfive/camss/Kconfig
+++ /dev/null
@@ -1,18 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config VIDEO_STARFIVE_CAMSS
-	tristate "Starfive Camera Subsystem driver"
-	depends on V4L_PLATFORM_DRIVERS
-	depends on VIDEO_DEV && OF
-	depends on HAS_DMA
-	depends on PM
-	depends on ARCH_STARFIVE || COMPILE_TEST
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	select VIDEOBUF2_DMA_CONTIG
-	select V4L2_FWNODE
-	help
-	   Enable this to support for the Starfive Camera subsystem
-	   found on Starfive JH7110 SoC.
-
-	   To compile this driver as a module, choose M here: the
-	   module will be called starfive-camss.
diff --git a/drivers/staging/media/starfive/camss/Makefile b/drivers/staging/media/starfive/camss/Makefile
deleted file mode 100644
index 005790202e7b8..0000000000000
--- a/drivers/staging/media/starfive/camss/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for StarFive Camera Subsystem driver
-#
-
-starfive-camss-objs += \
-		stf-camss.o \
-		stf-capture.o \
-		stf-isp.o \
-		stf-isp-hw-ops.o \
-		stf-video.o
-
-obj-$(CONFIG_VIDEO_STARFIVE_CAMSS) += starfive-camss.o
diff --git a/drivers/staging/media/starfive/camss/TODO.txt b/drivers/staging/media/starfive/camss/TODO.txt
deleted file mode 100644
index 7d459f4f09d00..0000000000000
--- a/drivers/staging/media/starfive/camss/TODO.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-Unstaging requirements:
-- Add userspace support which demonstrates the ability to receive statistics and
-  adapt hardware modules configuration accordingly;
-- Add documentation for description of the statistics data structures;
diff --git a/drivers/staging/media/starfive/camss/stf-camss.c b/drivers/staging/media/starfive/camss/stf-camss.c
deleted file mode 100644
index 259aaad010d2f..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-camss.c
+++ /dev/null
@@ -1,438 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * stf_camss.c
- *
- * Starfive Camera Subsystem driver
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- *
- * Author: Jack Zhu <jack.zhu@starfivetech.com>
- * Author: Changhuang Liang <changhuang.liang@starfivetech.com>
- *
- */
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_graph.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-fwnode.h>
-#include <media/v4l2-mc.h>
-
-#include "stf-camss.h"
-
-static const char * const stfcamss_clocks[] = {
-	"wrapper_clk_c",
-	"ispcore_2x",
-	"isp_axi",
-};
-
-static const char * const stfcamss_resets[] = {
-	"wrapper_p",
-	"wrapper_c",
-	"axiwr",
-	"isp_top_n",
-	"isp_top_axi",
-};
-
-static const struct stf_isr_data stf_isrs[] = {
-	{"wr_irq", stf_wr_irq_handler},
-	{"isp_irq", stf_isp_irq_handler},
-	{"line_irq", stf_line_irq_handler},
-};
-
-static int stfcamss_get_mem_res(struct stfcamss *stfcamss)
-{
-	struct platform_device *pdev = to_platform_device(stfcamss->dev);
-
-	stfcamss->syscon_base =
-		devm_platform_ioremap_resource_byname(pdev, "syscon");
-	if (IS_ERR(stfcamss->syscon_base))
-		return PTR_ERR(stfcamss->syscon_base);
-
-	stfcamss->isp_base = devm_platform_ioremap_resource_byname(pdev, "isp");
-	if (IS_ERR(stfcamss->isp_base))
-		return PTR_ERR(stfcamss->isp_base);
-
-	return 0;
-}
-
-/*
- * stfcamss_of_parse_endpoint_node - Parse port endpoint node
- * @dev: Device
- * @node: Device node to be parsed
- * @csd: Parsed data from port endpoint node
- *
- * Return 0 on success or a negative error code on failure
- */
-static int stfcamss_of_parse_endpoint_node(struct stfcamss *stfcamss,
-					   struct device_node *node,
-					   struct stfcamss_async_subdev *csd)
-{
-	struct v4l2_fwnode_endpoint vep = { { 0 } };
-	int ret;
-
-	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(node), &vep);
-	if (ret) {
-		dev_err(stfcamss->dev, "endpoint not defined at %pOF\n", node);
-		return ret;
-	}
-
-	csd->port = vep.base.port;
-
-	return 0;
-}
-
-/*
- * stfcamss_of_parse_ports - Parse ports node
- * @stfcamss: STFCAMSS device
- *
- * Return number of "port" nodes found in "ports" node
- */
-static int stfcamss_of_parse_ports(struct stfcamss *stfcamss)
-{
-	struct device_node *node = NULL;
-	int ret, num_subdevs = 0;
-
-	for_each_endpoint_of_node(stfcamss->dev->of_node, node) {
-		struct stfcamss_async_subdev *csd;
-
-		if (!of_device_is_available(node))
-			continue;
-
-		csd = v4l2_async_nf_add_fwnode_remote(&stfcamss->notifier,
-						      of_fwnode_handle(node),
-						      struct stfcamss_async_subdev);
-		if (IS_ERR(csd)) {
-			ret = PTR_ERR(csd);
-			dev_err(stfcamss->dev, "failed to add async notifier\n");
-			goto err_cleanup;
-		}
-
-		ret = stfcamss_of_parse_endpoint_node(stfcamss, node, csd);
-		if (ret)
-			goto err_cleanup;
-
-		num_subdevs++;
-	}
-
-	return num_subdevs;
-
-err_cleanup:
-	of_node_put(node);
-	return ret;
-}
-
-static int stfcamss_register_devs(struct stfcamss *stfcamss)
-{
-	struct stf_capture *cap_yuv = &stfcamss->captures[STF_CAPTURE_YUV];
-	struct stf_isp_dev *isp_dev = &stfcamss->isp_dev;
-	int ret;
-
-	ret = stf_isp_register(isp_dev, &stfcamss->v4l2_dev);
-	if (ret < 0) {
-		dev_err(stfcamss->dev,
-			"failed to register stf isp%d entity: %d\n", 0, ret);
-		return ret;
-	}
-
-	ret = stf_capture_register(stfcamss, &stfcamss->v4l2_dev);
-	if (ret < 0) {
-		dev_err(stfcamss->dev,
-			"failed to register capture: %d\n", ret);
-		goto err_isp_unregister;
-	}
-
-	ret = media_create_pad_link(&isp_dev->subdev.entity, STF_ISP_PAD_SRC,
-				    &cap_yuv->video.vdev.entity, 0, 0);
-	if (ret)
-		goto err_cap_unregister;
-
-	cap_yuv->video.source_subdev = &isp_dev->subdev;
-
-	return ret;
-
-err_cap_unregister:
-	stf_capture_unregister(stfcamss);
-err_isp_unregister:
-	stf_isp_unregister(&stfcamss->isp_dev);
-
-	return ret;
-}
-
-static void stfcamss_unregister_devs(struct stfcamss *stfcamss)
-{
-	struct stf_capture *cap_yuv = &stfcamss->captures[STF_CAPTURE_YUV];
-	struct stf_isp_dev *isp_dev = &stfcamss->isp_dev;
-
-	media_entity_remove_links(&isp_dev->subdev.entity);
-	media_entity_remove_links(&cap_yuv->video.vdev.entity);
-
-	stf_isp_unregister(&stfcamss->isp_dev);
-	stf_capture_unregister(stfcamss);
-}
-
-static int stfcamss_subdev_notifier_bound(struct v4l2_async_notifier *async,
-					  struct v4l2_subdev *subdev,
-					  struct v4l2_async_connection *asc)
-{
-	struct stfcamss *stfcamss =
-		container_of(async, struct stfcamss, notifier);
-	struct stfcamss_async_subdev *csd =
-		container_of(asc, struct stfcamss_async_subdev, asd);
-	enum stf_port_num port = csd->port;
-	struct stf_isp_dev *isp_dev = &stfcamss->isp_dev;
-	struct stf_capture *cap_raw = &stfcamss->captures[STF_CAPTURE_RAW];
-	struct media_pad *pad;
-	int ret;
-
-	if (port == STF_PORT_CSI2RX) {
-		pad = &isp_dev->pads[STF_ISP_PAD_SINK];
-	} else {
-		dev_err(stfcamss->dev, "not support port %d\n", port);
-		return -EPERM;
-	}
-
-	ret = v4l2_create_fwnode_links_to_pad(subdev, pad, 0);
-	if (ret)
-		return ret;
-
-	ret = media_create_pad_link(&subdev->entity, 1,
-				    &cap_raw->video.vdev.entity, 0, 0);
-	if (ret)
-		return ret;
-
-	isp_dev->source_subdev = subdev;
-	cap_raw->video.source_subdev = subdev;
-
-	return 0;
-}
-
-static int stfcamss_subdev_notifier_complete(struct v4l2_async_notifier *ntf)
-{
-	struct stfcamss *stfcamss =
-		container_of(ntf, struct stfcamss, notifier);
-
-	return v4l2_device_register_subdev_nodes(&stfcamss->v4l2_dev);
-}
-
-static const struct v4l2_async_notifier_operations
-stfcamss_subdev_notifier_ops = {
-	.bound = stfcamss_subdev_notifier_bound,
-	.complete = stfcamss_subdev_notifier_complete,
-};
-
-static void stfcamss_mc_init(struct platform_device *pdev,
-			     struct stfcamss *stfcamss)
-{
-	stfcamss->media_dev.dev = stfcamss->dev;
-	strscpy(stfcamss->media_dev.model, "Starfive Camera Subsystem",
-		sizeof(stfcamss->media_dev.model));
-	media_device_init(&stfcamss->media_dev);
-
-	stfcamss->v4l2_dev.mdev = &stfcamss->media_dev;
-}
-
-/*
- * stfcamss_probe - Probe STFCAMSS platform device
- * @pdev: Pointer to STFCAMSS platform device
- *
- * Return 0 on success or a negative error code on failure
- */
-static int stfcamss_probe(struct platform_device *pdev)
-{
-	struct stfcamss *stfcamss;
-	struct device *dev = &pdev->dev;
-	int ret, num_subdevs;
-	unsigned int i;
-
-	stfcamss = devm_kzalloc(dev, sizeof(*stfcamss), GFP_KERNEL);
-	if (!stfcamss)
-		return -ENOMEM;
-
-	stfcamss->dev = dev;
-
-	for (i = 0; i < ARRAY_SIZE(stf_isrs); ++i) {
-		int irq;
-
-		irq = platform_get_irq(pdev, i);
-		if (irq < 0)
-			return irq;
-
-		ret = devm_request_irq(stfcamss->dev, irq, stf_isrs[i].isr, 0,
-				       stf_isrs[i].name, stfcamss);
-		if (ret) {
-			dev_err(dev, "request irq failed: %d\n", ret);
-			return ret;
-		}
-	}
-
-	stfcamss->nclks = ARRAY_SIZE(stfcamss->sys_clk);
-	for (i = 0; i < stfcamss->nclks; ++i)
-		stfcamss->sys_clk[i].id = stfcamss_clocks[i];
-	ret = devm_clk_bulk_get(dev, stfcamss->nclks, stfcamss->sys_clk);
-	if (ret) {
-		dev_err(dev, "Failed to get clk controls\n");
-		return ret;
-	}
-
-	stfcamss->nrsts = ARRAY_SIZE(stfcamss->sys_rst);
-	for (i = 0; i < stfcamss->nrsts; ++i)
-		stfcamss->sys_rst[i].id = stfcamss_resets[i];
-	ret = devm_reset_control_bulk_get_shared(dev, stfcamss->nrsts,
-						 stfcamss->sys_rst);
-	if (ret) {
-		dev_err(dev, "Failed to get reset controls\n");
-		return ret;
-	}
-
-	ret = stfcamss_get_mem_res(stfcamss);
-	if (ret) {
-		dev_err(dev, "Could not map registers\n");
-		return ret;
-	}
-
-	platform_set_drvdata(pdev, stfcamss);
-
-	v4l2_async_nf_init(&stfcamss->notifier, &stfcamss->v4l2_dev);
-
-	num_subdevs = stfcamss_of_parse_ports(stfcamss);
-	if (num_subdevs < 0) {
-		ret = -ENODEV;
-		dev_err(dev, "Failed to get sub devices: %d\n", ret);
-		goto err_cleanup_notifier;
-	}
-
-	ret = stf_isp_init(stfcamss);
-	if (ret < 0) {
-		dev_err(dev, "Failed to init isp: %d\n", ret);
-		goto err_cleanup_notifier;
-	}
-
-	stfcamss_mc_init(pdev, stfcamss);
-
-	ret = v4l2_device_register(stfcamss->dev, &stfcamss->v4l2_dev);
-	if (ret < 0) {
-		dev_err(dev, "Failed to register V4L2 device: %d\n", ret);
-		goto err_cleanup_media_device;
-	}
-
-	ret = media_device_register(&stfcamss->media_dev);
-	if (ret) {
-		dev_err(dev, "Failed to register media device: %d\n", ret);
-		goto err_unregister_device;
-	}
-
-	ret = stfcamss_register_devs(stfcamss);
-	if (ret < 0) {
-		dev_err(dev, "Failed to register subdevice: %d\n", ret);
-		goto err_unregister_media_dev;
-	}
-
-	pm_runtime_enable(dev);
-
-	stfcamss->notifier.ops = &stfcamss_subdev_notifier_ops;
-	ret = v4l2_async_nf_register(&stfcamss->notifier);
-	if (ret) {
-		dev_err(dev, "Failed to register async subdev nodes: %d\n",
-			ret);
-		pm_runtime_disable(dev);
-		goto err_unregister_subdevs;
-	}
-
-	return 0;
-
-err_unregister_subdevs:
-	stfcamss_unregister_devs(stfcamss);
-err_unregister_media_dev:
-	media_device_unregister(&stfcamss->media_dev);
-err_unregister_device:
-	v4l2_device_unregister(&stfcamss->v4l2_dev);
-err_cleanup_media_device:
-	media_device_cleanup(&stfcamss->media_dev);
-err_cleanup_notifier:
-	v4l2_async_nf_cleanup(&stfcamss->notifier);
-	return ret;
-}
-
-/*
- * stfcamss_remove - Remove STFCAMSS platform device
- * @pdev: Pointer to STFCAMSS platform device
- */
-static void stfcamss_remove(struct platform_device *pdev)
-{
-	struct stfcamss *stfcamss = platform_get_drvdata(pdev);
-
-	stfcamss_unregister_devs(stfcamss);
-	v4l2_device_unregister(&stfcamss->v4l2_dev);
-	media_device_cleanup(&stfcamss->media_dev);
-	v4l2_async_nf_cleanup(&stfcamss->notifier);
-	pm_runtime_disable(&pdev->dev);
-}
-
-static const struct of_device_id stfcamss_of_match[] = {
-	{ .compatible = "starfive,jh7110-camss" },
-	{ /* sentinel */ },
-};
-
-MODULE_DEVICE_TABLE(of, stfcamss_of_match);
-
-static int __maybe_unused stfcamss_runtime_suspend(struct device *dev)
-{
-	struct stfcamss *stfcamss = dev_get_drvdata(dev);
-	int ret;
-
-	ret = reset_control_bulk_assert(stfcamss->nrsts, stfcamss->sys_rst);
-	if (ret) {
-		dev_err(dev, "reset assert failed\n");
-		return ret;
-	}
-
-	clk_bulk_disable_unprepare(stfcamss->nclks, stfcamss->sys_clk);
-
-	return 0;
-}
-
-static int __maybe_unused stfcamss_runtime_resume(struct device *dev)
-{
-	struct stfcamss *stfcamss = dev_get_drvdata(dev);
-	int ret;
-
-	ret = clk_bulk_prepare_enable(stfcamss->nclks, stfcamss->sys_clk);
-	if (ret) {
-		dev_err(dev, "clock prepare enable failed\n");
-		return ret;
-	}
-
-	ret = reset_control_bulk_deassert(stfcamss->nrsts, stfcamss->sys_rst);
-	if (ret < 0) {
-		dev_err(dev, "cannot deassert resets\n");
-		clk_bulk_disable_unprepare(stfcamss->nclks, stfcamss->sys_clk);
-		return ret;
-	}
-
-	return 0;
-}
-
-static const struct dev_pm_ops stfcamss_pm_ops = {
-	SET_RUNTIME_PM_OPS(stfcamss_runtime_suspend,
-			   stfcamss_runtime_resume,
-			   NULL)
-};
-
-static struct platform_driver stfcamss_driver = {
-	.probe = stfcamss_probe,
-	.remove = stfcamss_remove,
-	.driver = {
-		.name = "starfive-camss",
-		.pm = &stfcamss_pm_ops,
-		.of_match_table = stfcamss_of_match,
-	},
-};
-
-module_platform_driver(stfcamss_driver);
-
-MODULE_AUTHOR("Jack Zhu <jack.zhu@starfivetech.com>");
-MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>");
-MODULE_DESCRIPTION("StarFive Camera Subsystem driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/starfive/camss/stf-camss.h b/drivers/staging/media/starfive/camss/stf-camss.h
deleted file mode 100644
index e2b0cfb437bdb..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-camss.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * stf_camss.h
- *
- * Starfive Camera Subsystem driver
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#ifndef STF_CAMSS_H
-#define STF_CAMSS_H
-
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/reset.h>
-#include <media/media-device.h>
-#include <media/media-entity.h>
-#include <media/v4l2-async.h>
-#include <media/v4l2-device.h>
-
-#include "stf-isp.h"
-#include "stf-capture.h"
-
-enum stf_port_num {
-	STF_PORT_DVP = 0,
-	STF_PORT_CSI2RX
-};
-
-enum stf_clk {
-	STF_CLK_WRAPPER_CLK_C = 0,
-	STF_CLK_ISPCORE_2X,
-	STF_CLK_ISP_AXI,
-	STF_CLK_NUM
-};
-
-enum stf_rst {
-	STF_RST_WRAPPER_P = 0,
-	STF_RST_WRAPPER_C,
-	STF_RST_AXIWR,
-	STF_RST_ISP_TOP_N,
-	STF_RST_ISP_TOP_AXI,
-	STF_RST_NUM
-};
-
-struct stf_isr_data {
-	const char *name;
-	irqreturn_t (*isr)(int irq, void *priv);
-};
-
-struct stfcamss {
-	struct v4l2_device v4l2_dev;
-	struct media_device media_dev;
-	struct media_pipeline pipe;
-	struct device *dev;
-	struct stf_isp_dev isp_dev;
-	struct stf_capture captures[STF_CAPTURE_NUM];
-	struct v4l2_async_notifier notifier;
-	void __iomem *syscon_base;
-	void __iomem *isp_base;
-	struct clk_bulk_data sys_clk[STF_CLK_NUM];
-	int nclks;
-	struct reset_control_bulk_data sys_rst[STF_RST_NUM];
-	int nrsts;
-};
-
-struct stfcamss_async_subdev {
-	struct v4l2_async_connection asd;  /* must be first */
-	enum stf_port_num port;
-};
-
-static inline u32 stf_isp_reg_read(struct stfcamss *stfcamss, u32 reg)
-{
-	return ioread32(stfcamss->isp_base + reg);
-}
-
-static inline void stf_isp_reg_write(struct stfcamss *stfcamss,
-				     u32 reg, u32 val)
-{
-	iowrite32(val, stfcamss->isp_base + reg);
-}
-
-static inline void stf_isp_reg_write_delay(struct stfcamss *stfcamss,
-					   u32 reg, u32 val, u32 delay)
-{
-	iowrite32(val, stfcamss->isp_base + reg);
-	usleep_range(1000 * delay, 1000 * delay + 100);
-}
-
-static inline void stf_isp_reg_set_bit(struct stfcamss *stfcamss,
-				       u32 reg, u32 mask, u32 val)
-{
-	u32 value;
-
-	value = ioread32(stfcamss->isp_base + reg) & ~mask;
-	val &= mask;
-	val |= value;
-	iowrite32(val, stfcamss->isp_base + reg);
-}
-
-static inline void stf_isp_reg_set(struct stfcamss *stfcamss, u32 reg, u32 mask)
-{
-	iowrite32(ioread32(stfcamss->isp_base + reg) | mask,
-		  stfcamss->isp_base + reg);
-}
-
-static inline u32 stf_syscon_reg_read(struct stfcamss *stfcamss, u32 reg)
-{
-	return ioread32(stfcamss->syscon_base + reg);
-}
-
-static inline void stf_syscon_reg_write(struct stfcamss *stfcamss,
-					u32 reg, u32 val)
-{
-	iowrite32(val, stfcamss->syscon_base + reg);
-}
-
-static inline void stf_syscon_reg_set_bit(struct stfcamss *stfcamss,
-					  u32 reg, u32 bit_mask)
-{
-	u32 value;
-
-	value = ioread32(stfcamss->syscon_base + reg);
-	iowrite32(value | bit_mask, stfcamss->syscon_base + reg);
-}
-
-static inline void stf_syscon_reg_clear_bit(struct stfcamss *stfcamss,
-					    u32 reg, u32 bit_mask)
-{
-	u32 value;
-
-	value = ioread32(stfcamss->syscon_base + reg);
-	iowrite32(value & ~bit_mask, stfcamss->syscon_base + reg);
-}
-#endif /* STF_CAMSS_H */
diff --git a/drivers/staging/media/starfive/camss/stf-capture.c b/drivers/staging/media/starfive/camss/stf-capture.c
deleted file mode 100644
index e15d2e97eb0b6..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-capture.c
+++ /dev/null
@@ -1,605 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * stf_capture.c
- *
- * StarFive Camera Subsystem - capture device
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#include "stf-camss.h"
-
-static const char * const stf_cap_names[] = {
-	"capture_raw",
-	"capture_yuv",
-};
-
-static const struct stfcamss_format_info stf_wr_fmts[] = {
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10,
-		.planes = 1,
-		.vsub = { 1 },
-		.bpp = 16,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10,
-		.planes = 1,
-		.vsub = { 1 },
-		.bpp = 16,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10,
-		.planes = 1,
-		.vsub = { 1 },
-		.bpp = 16,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10,
-		.planes = 1,
-		.vsub = { 1 },
-		.bpp = 16,
-	},
-};
-
-static const struct stfcamss_format_info stf_isp_fmts[] = {
-	{
-		.code = MEDIA_BUS_FMT_YUYV8_1_5X8,
-		.pixelformat = V4L2_PIX_FMT_NV12,
-		.planes = 2,
-		.vsub = { 1, 2 },
-		.bpp = 8,
-	},
-};
-
-static inline struct stf_capture *to_stf_capture(struct stfcamss_video *video)
-{
-	return container_of(video, struct stf_capture, video);
-}
-
-static void stf_set_raw_addr(struct stfcamss *stfcamss, dma_addr_t addr)
-{
-	stf_syscon_reg_write(stfcamss, VIN_START_ADDR_O, (long)addr);
-	stf_syscon_reg_write(stfcamss, VIN_START_ADDR_N, (long)addr);
-}
-
-static void stf_set_yuv_addr(struct stfcamss *stfcamss,
-			     dma_addr_t y_addr, dma_addr_t uv_addr)
-{
-	stf_isp_reg_write(stfcamss, ISP_REG_Y_PLANE_START_ADDR, y_addr);
-	stf_isp_reg_write(stfcamss, ISP_REG_UV_PLANE_START_ADDR, uv_addr);
-}
-
-static void stf_init_addrs(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *output = &cap->buffers;
-	dma_addr_t addr0, addr1;
-
-	output->active_buf = 0;
-
-	if (!output->buf[0])
-		return;
-
-	addr0 = output->buf[0]->addr[0];
-	addr1 = output->buf[0]->addr[1];
-
-	if (cap->type == STF_CAPTURE_RAW)
-		stf_set_raw_addr(video->stfcamss, addr0);
-	else if (cap->type == STF_CAPTURE_YUV)
-		stf_set_yuv_addr(video->stfcamss, addr0, addr1);
-}
-
-static struct stfcamss_buffer *stf_buf_get_pending(struct stf_v_buf *output)
-{
-	struct stfcamss_buffer *buffer = NULL;
-
-	if (!list_empty(&output->pending_bufs)) {
-		buffer = list_first_entry(&output->pending_bufs,
-					  struct stfcamss_buffer,
-					  queue);
-		list_del(&buffer->queue);
-	}
-
-	return buffer;
-}
-
-static void stf_cap_s_cfg(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *output = &cap->buffers;
-	unsigned long flags;
-
-	spin_lock_irqsave(&output->lock, flags);
-
-	output->state = STF_OUTPUT_IDLE;
-	output->buf[0] = stf_buf_get_pending(output);
-
-	if (!output->buf[0] && output->buf[1]) {
-		output->buf[0] = output->buf[1];
-		output->buf[1] = NULL;
-	}
-
-	if (output->buf[0])
-		output->state = STF_OUTPUT_SINGLE;
-
-	output->sequence = 0;
-	stf_init_addrs(video);
-
-	spin_unlock_irqrestore(&output->lock, flags);
-}
-
-static int stf_cap_s_cleanup(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *output = &cap->buffers;
-	unsigned long flags;
-
-	spin_lock_irqsave(&output->lock, flags);
-
-	output->state = STF_OUTPUT_OFF;
-
-	spin_unlock_irqrestore(&output->lock, flags);
-
-	return 0;
-}
-
-static void stf_wr_data_en(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stfcamss *stfcamss = cap->video.stfcamss;
-
-	stf_syscon_reg_set_bit(stfcamss, VIN_CHANNEL_SEL_EN, U0_VIN_AXIWR0_EN);
-}
-
-static void stf_wr_irq_enable(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stfcamss *stfcamss = cap->video.stfcamss;
-
-	stf_syscon_reg_clear_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_M);
-}
-
-static void stf_wr_irq_disable(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stfcamss *stfcamss = cap->video.stfcamss;
-
-	stf_syscon_reg_set_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_CLEAN);
-	stf_syscon_reg_clear_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_CLEAN);
-	stf_syscon_reg_set_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_M);
-}
-
-static void stf_channel_set(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stfcamss *stfcamss = cap->video.stfcamss;
-	u32 val;
-
-	if (cap->type == STF_CAPTURE_RAW) {
-		const struct v4l2_pix_format *pix = &video->active_fmt.fmt.pix;
-
-		val = stf_syscon_reg_read(stfcamss, VIN_CHANNEL_SEL_EN);
-		val &= ~U0_VIN_CHANNEL_SEL_MASK;
-		val |= CHANNEL(0);
-		stf_syscon_reg_write(stfcamss, VIN_CHANNEL_SEL_EN, val);
-
-		val = stf_syscon_reg_read(stfcamss, VIN_INRT_PIX_CFG);
-		val &= ~U0_VIN_PIX_CT_MASK;
-		val |= PIX_CT(1);
-
-		val &= ~U0_VIN_PIXEL_HEIGH_BIT_SEL_MAKS;
-		val |= PIXEL_HEIGH_BIT_SEL(0);
-
-		val &= ~U0_VIN_PIX_CNT_END_MASK;
-		val |= PIX_CNT_END(pix->width / 4 - 1);
-
-		stf_syscon_reg_write(stfcamss, VIN_INRT_PIX_CFG, val);
-	} else if (cap->type == STF_CAPTURE_YUV) {
-		val = stf_syscon_reg_read(stfcamss, VIN_CFG_REG);
-		val &= ~U0_VIN_MIPI_BYTE_EN_ISP0_MASK;
-		val |= U0_VIN_MIPI_BYTE_EN_ISP0(0);
-
-		val &= ~U0_VIN_MIPI_CHANNEL_SEL0_MASK;
-		val |= U0_VIN_MIPI_CHANNEL_SEL0(0);
-
-		val &= ~U0_VIN_PIX_NUM_MASK;
-		val |= U0_VIN_PIX_NUM(0);
-
-		val &= ~U0_VIN_P_I_MIPI_HAEDER_EN0_MASK;
-		val |= U0_VIN_P_I_MIPI_HAEDER_EN0(1);
-
-		stf_syscon_reg_write(stfcamss, VIN_CFG_REG, val);
-	}
-}
-
-static void stf_capture_start(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-
-	stf_channel_set(video);
-	if (cap->type == STF_CAPTURE_RAW) {
-		stf_wr_irq_enable(video);
-		stf_wr_data_en(video);
-	}
-
-	stf_cap_s_cfg(video);
-}
-
-static void stf_capture_stop(struct stfcamss_video *video)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-
-	if (cap->type == STF_CAPTURE_RAW)
-		stf_wr_irq_disable(video);
-
-	stf_cap_s_cleanup(video);
-}
-
-static void stf_capture_init(struct stfcamss *stfcamss, struct stf_capture *cap)
-{
-	cap->buffers.state = STF_OUTPUT_OFF;
-	cap->buffers.buf[0] = NULL;
-	cap->buffers.buf[1] = NULL;
-	cap->buffers.active_buf = 0;
-	atomic_set(&cap->buffers.frame_skip, 4);
-	INIT_LIST_HEAD(&cap->buffers.pending_bufs);
-	INIT_LIST_HEAD(&cap->buffers.ready_bufs);
-	spin_lock_init(&cap->buffers.lock);
-
-	cap->video.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	cap->video.stfcamss = stfcamss;
-	cap->video.bpl_alignment = 16 * 8;
-
-	if (cap->type == STF_CAPTURE_RAW) {
-		cap->video.formats = stf_wr_fmts;
-		cap->video.nformats = ARRAY_SIZE(stf_wr_fmts);
-		cap->video.bpl_alignment = 8;
-	} else if (cap->type == STF_CAPTURE_YUV) {
-		cap->video.formats = stf_isp_fmts;
-		cap->video.nformats = ARRAY_SIZE(stf_isp_fmts);
-		cap->video.bpl_alignment = 1;
-	}
-}
-
-static void stf_buf_add_ready(struct stf_v_buf *output,
-			      struct stfcamss_buffer *buffer)
-{
-	INIT_LIST_HEAD(&buffer->queue);
-	list_add_tail(&buffer->queue, &output->ready_bufs);
-}
-
-static struct stfcamss_buffer *stf_buf_get_ready(struct stf_v_buf *output)
-{
-	struct stfcamss_buffer *buffer = NULL;
-
-	if (!list_empty(&output->ready_bufs)) {
-		buffer = list_first_entry(&output->ready_bufs,
-					  struct stfcamss_buffer,
-					  queue);
-		list_del(&buffer->queue);
-	}
-
-	return buffer;
-}
-
-static void stf_buf_add_pending(struct stf_v_buf *output,
-				struct stfcamss_buffer *buffer)
-{
-	INIT_LIST_HEAD(&buffer->queue);
-	list_add_tail(&buffer->queue, &output->pending_bufs);
-}
-
-static void stf_buf_update_on_last(struct stf_v_buf *output)
-{
-	switch (output->state) {
-	case STF_OUTPUT_CONTINUOUS:
-		output->state = STF_OUTPUT_SINGLE;
-		output->active_buf = !output->active_buf;
-		break;
-	case STF_OUTPUT_SINGLE:
-		output->state = STF_OUTPUT_STOPPING;
-		break;
-	default:
-		break;
-	}
-}
-
-static void stf_buf_update_on_next(struct stf_v_buf *output)
-{
-	switch (output->state) {
-	case STF_OUTPUT_CONTINUOUS:
-		output->active_buf = !output->active_buf;
-		break;
-	case STF_OUTPUT_SINGLE:
-	default:
-		break;
-	}
-}
-
-static void stf_buf_update_on_new(struct stfcamss_video *video,
-				  struct stfcamss_buffer *new_buf)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *output = &cap->buffers;
-
-	switch (output->state) {
-	case STF_OUTPUT_SINGLE:
-		stf_buf_add_pending(output, new_buf);
-		break;
-	case STF_OUTPUT_IDLE:
-		if (!output->buf[0]) {
-			output->buf[0] = new_buf;
-			stf_init_addrs(video);
-			output->state = STF_OUTPUT_SINGLE;
-		} else {
-			stf_buf_add_pending(output, new_buf);
-		}
-		break;
-	case STF_OUTPUT_STOPPING:
-		if (output->last_buffer) {
-			output->buf[output->active_buf] = output->last_buffer;
-			output->last_buffer = NULL;
-		}
-
-		output->state = STF_OUTPUT_SINGLE;
-		stf_buf_add_pending(output, new_buf);
-		break;
-	case STF_OUTPUT_CONTINUOUS:
-	default:
-		stf_buf_add_pending(output, new_buf);
-		break;
-	}
-}
-
-static void stf_buf_flush(struct stf_v_buf *output, enum vb2_buffer_state state)
-{
-	struct stfcamss_buffer *buf;
-	struct stfcamss_buffer *t;
-
-	list_for_each_entry_safe(buf, t, &output->pending_bufs, queue) {
-		vb2_buffer_done(&buf->vb.vb2_buf, state);
-		list_del(&buf->queue);
-	}
-	list_for_each_entry_safe(buf, t, &output->ready_bufs, queue) {
-		vb2_buffer_done(&buf->vb.vb2_buf, state);
-		list_del(&buf->queue);
-	}
-}
-
-static void stf_buf_done(struct stf_v_buf *output)
-{
-	struct stfcamss_buffer *ready_buf;
-	u64 ts = ktime_get_ns();
-	unsigned long flags;
-
-	if (output->state == STF_OUTPUT_OFF ||
-	    output->state == STF_OUTPUT_RESERVED)
-		return;
-
-	spin_lock_irqsave(&output->lock, flags);
-
-	while ((ready_buf = stf_buf_get_ready(output))) {
-		ready_buf->vb.vb2_buf.timestamp = ts;
-		ready_buf->vb.sequence = output->sequence++;
-
-		vb2_buffer_done(&ready_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
-	}
-
-	spin_unlock_irqrestore(&output->lock, flags);
-}
-
-static void stf_change_buffer(struct stf_v_buf *output)
-{
-	struct stf_capture *cap = container_of(output, struct stf_capture,
-					       buffers);
-	struct stfcamss *stfcamss = cap->video.stfcamss;
-	struct stfcamss_buffer *ready_buf;
-	dma_addr_t *new_addr;
-	unsigned long flags;
-	u32 active_index;
-
-	if (output->state == STF_OUTPUT_OFF ||
-	    output->state == STF_OUTPUT_STOPPING ||
-	    output->state == STF_OUTPUT_RESERVED ||
-	    output->state == STF_OUTPUT_IDLE)
-		return;
-
-	spin_lock_irqsave(&output->lock, flags);
-
-	active_index = output->active_buf;
-
-	ready_buf = output->buf[active_index];
-	if (!ready_buf) {
-		dev_dbg(stfcamss->dev, "missing ready buf %d %d.\n",
-			active_index, output->state);
-		active_index = !active_index;
-		ready_buf = output->buf[active_index];
-		if (!ready_buf) {
-			dev_dbg(stfcamss->dev,
-				"missing ready buf2 %d %d.\n",
-				active_index, output->state);
-			goto out_unlock;
-		}
-	}
-
-	/* Get next buffer */
-	output->buf[active_index] = stf_buf_get_pending(output);
-	if (!output->buf[active_index]) {
-		new_addr = ready_buf->addr;
-		stf_buf_update_on_last(output);
-	} else {
-		new_addr = output->buf[active_index]->addr;
-		stf_buf_update_on_next(output);
-	}
-
-	if (output->state == STF_OUTPUT_STOPPING) {
-		output->last_buffer = ready_buf;
-	} else {
-		if (cap->type == STF_CAPTURE_RAW)
-			stf_set_raw_addr(stfcamss, new_addr[0]);
-		else if (cap->type == STF_CAPTURE_YUV)
-			stf_set_yuv_addr(stfcamss, new_addr[0], new_addr[1]);
-
-		stf_buf_add_ready(output, ready_buf);
-	}
-
-out_unlock:
-	spin_unlock_irqrestore(&output->lock, flags);
-}
-
-irqreturn_t stf_wr_irq_handler(int irq, void *priv)
-{
-	struct stfcamss *stfcamss = priv;
-	struct stf_capture *cap = &stfcamss->captures[STF_CAPTURE_RAW];
-
-	if (atomic_dec_if_positive(&cap->buffers.frame_skip) < 0) {
-		stf_change_buffer(&cap->buffers);
-		stf_buf_done(&cap->buffers);
-	}
-
-	stf_syscon_reg_set_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_CLEAN);
-	stf_syscon_reg_clear_bit(stfcamss, VIN_INRT_PIX_CFG, U0_VIN_INTR_CLEAN);
-
-	return IRQ_HANDLED;
-}
-
-irqreturn_t stf_isp_irq_handler(int irq, void *priv)
-{
-	struct stfcamss *stfcamss = priv;
-	struct stf_capture *cap = &stfcamss->captures[STF_CAPTURE_YUV];
-	u32 status;
-
-	status = stf_isp_reg_read(stfcamss, ISP_REG_ISP_CTRL_0);
-	if (status & ISPC_ISP) {
-		if (status & ISPC_ENUO)
-			stf_buf_done(&cap->buffers);
-
-		stf_isp_reg_write(stfcamss, ISP_REG_ISP_CTRL_0,
-				  (status & ~ISPC_INT_ALL_MASK) |
-				  ISPC_ISP | ISPC_CSI | ISPC_SC);
-	}
-
-	return IRQ_HANDLED;
-}
-
-irqreturn_t stf_line_irq_handler(int irq, void *priv)
-{
-	struct stfcamss *stfcamss = priv;
-	struct stf_capture *cap = &stfcamss->captures[STF_CAPTURE_YUV];
-	u32 status;
-
-	status = stf_isp_reg_read(stfcamss, ISP_REG_ISP_CTRL_0);
-	if (status & ISPC_LINE) {
-		if (atomic_dec_if_positive(&cap->buffers.frame_skip) < 0) {
-			if ((status & ISPC_ENUO))
-				stf_change_buffer(&cap->buffers);
-		}
-
-		stf_isp_reg_set_bit(stfcamss, ISP_REG_CSIINTS,
-				    CSI_INTS_MASK, CSI_INTS(0x3));
-		stf_isp_reg_set_bit(stfcamss, ISP_REG_IESHD,
-				    SHAD_UP_M | SHAD_UP_EN, 0x3);
-
-		stf_isp_reg_write(stfcamss, ISP_REG_ISP_CTRL_0,
-				  (status & ~ISPC_INT_ALL_MASK) | ISPC_LINE);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static int stf_queue_buffer(struct stfcamss_video *video,
-			    struct stfcamss_buffer *buf)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *v_bufs = &cap->buffers;
-	unsigned long flags;
-
-	spin_lock_irqsave(&v_bufs->lock, flags);
-	stf_buf_update_on_new(video, buf);
-	spin_unlock_irqrestore(&v_bufs->lock, flags);
-
-	return 0;
-}
-
-static int stf_flush_buffers(struct stfcamss_video *video,
-			     enum vb2_buffer_state state)
-{
-	struct stf_capture *cap = to_stf_capture(video);
-	struct stf_v_buf *v_bufs = &cap->buffers;
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&v_bufs->lock, flags);
-
-	stf_buf_flush(v_bufs, state);
-
-	for (i = 0; i < ARRAY_SIZE(v_bufs->buf); i++) {
-		if (v_bufs->buf[i])
-			vb2_buffer_done(&v_bufs->buf[i]->vb.vb2_buf, state);
-
-		v_bufs->buf[i] = NULL;
-	}
-
-	if (v_bufs->last_buffer) {
-		vb2_buffer_done(&v_bufs->last_buffer->vb.vb2_buf, state);
-		v_bufs->last_buffer = NULL;
-	}
-
-	spin_unlock_irqrestore(&v_bufs->lock, flags);
-	return 0;
-}
-
-static const struct stfcamss_video_ops stf_capture_ops = {
-	.queue_buffer = stf_queue_buffer,
-	.flush_buffers = stf_flush_buffers,
-	.start_streaming = stf_capture_start,
-	.stop_streaming = stf_capture_stop,
-};
-
-static void stf_capture_unregister_one(struct stf_capture *cap)
-{
-	if (!video_is_registered(&cap->video.vdev))
-		return;
-
-	media_entity_cleanup(&cap->video.vdev.entity);
-	vb2_video_unregister_device(&cap->video.vdev);
-}
-
-void stf_capture_unregister(struct stfcamss *stfcamss)
-{
-	struct stf_capture *cap_raw = &stfcamss->captures[STF_CAPTURE_RAW];
-	struct stf_capture *cap_yuv = &stfcamss->captures[STF_CAPTURE_YUV];
-
-	stf_capture_unregister_one(cap_raw);
-	stf_capture_unregister_one(cap_yuv);
-}
-
-int stf_capture_register(struct stfcamss *stfcamss,
-			 struct v4l2_device *v4l2_dev)
-{
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < ARRAY_SIZE(stfcamss->captures); i++) {
-		struct stf_capture *capture = &stfcamss->captures[i];
-
-		capture->type = i;
-		capture->video.ops = &stf_capture_ops;
-		stf_capture_init(stfcamss, capture);
-
-		ret = stf_video_register(&capture->video, v4l2_dev,
-					 stf_cap_names[i]);
-		if (ret < 0) {
-			dev_err(stfcamss->dev,
-				"Failed to register video node: %d\n", ret);
-			stf_capture_unregister(stfcamss);
-			return ret;
-		}
-	}
-
-	return 0;
-}
diff --git a/drivers/staging/media/starfive/camss/stf-capture.h b/drivers/staging/media/starfive/camss/stf-capture.h
deleted file mode 100644
index 2f9740b7e500a..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-capture.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * stf_capture.h
- *
- * Starfive Camera Subsystem driver
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#ifndef STF_CAPTURE_H
-#define STF_CAPTURE_H
-
-#include "stf-video.h"
-
-#define VIN_CHANNEL_SEL_EN			0x14
-#define VIN_START_ADDR_N			0x18
-#define VIN_INRT_PIX_CFG			0x1c
-#define VIN_START_ADDR_O			0x20
-#define VIN_CFG_REG				0x24
-
-#define U0_VIN_CNFG_AXI_DVP_EN			BIT(2)
-
-#define U0_VIN_CHANNEL_SEL_MASK			GENMASK(3, 0)
-#define U0_VIN_AXIWR0_EN			BIT(4)
-#define CHANNEL(x)				((x) << 0)
-
-#define U0_VIN_INTR_CLEAN			BIT(0)
-#define U0_VIN_INTR_M				BIT(1)
-#define U0_VIN_PIX_CNT_END_MASK			GENMASK(12, 2)
-#define U0_VIN_PIX_CT_MASK			GENMASK(14, 13)
-#define U0_VIN_PIXEL_HEIGH_BIT_SEL_MAKS		GENMASK(16, 15)
-
-#define PIX_CNT_END(x)				((x) << 2)
-#define PIX_CT(x)				((x) << 13)
-#define PIXEL_HEIGH_BIT_SEL(x)			((x) << 15)
-
-#define U0_VIN_CNFG_DVP_HS_POS			BIT(1)
-#define U0_VIN_CNFG_DVP_SWAP_EN			BIT(2)
-#define U0_VIN_CNFG_DVP_VS_POS			BIT(3)
-#define U0_VIN_CNFG_GEN_EN_AXIRD		BIT(4)
-#define U0_VIN_CNFG_ISP_DVP_EN0			BIT(5)
-#define U0_VIN_MIPI_BYTE_EN_ISP0(n)		((n) << 6)
-#define U0_VIN_MIPI_CHANNEL_SEL0(n)		((n) << 8)
-#define U0_VIN_P_I_MIPI_HAEDER_EN0(n)		((n) << 12)
-#define U0_VIN_PIX_NUM(n)			((n) << 13)
-#define U0_VIN_MIPI_BYTE_EN_ISP0_MASK		GENMASK(7, 6)
-#define U0_VIN_MIPI_CHANNEL_SEL0_MASK		GENMASK(11, 8)
-#define U0_VIN_P_I_MIPI_HAEDER_EN0_MASK		BIT(12)
-#define U0_VIN_PIX_NUM_MASK			GENMASK(16, 13)
-
-enum stf_v_state {
-	STF_OUTPUT_OFF,
-	STF_OUTPUT_RESERVED,
-	STF_OUTPUT_SINGLE,
-	STF_OUTPUT_CONTINUOUS,
-	STF_OUTPUT_IDLE,
-	STF_OUTPUT_STOPPING
-};
-
-struct stf_v_buf {
-	int active_buf;
-	struct stfcamss_buffer *buf[2];
-	struct stfcamss_buffer *last_buffer;
-	struct list_head pending_bufs;
-	struct list_head ready_bufs;
-	enum stf_v_state state;
-	unsigned int sequence;
-	/* protects the above member variables */
-	spinlock_t lock;
-	atomic_t frame_skip;
-};
-
-struct stf_capture {
-	struct stfcamss_video video;
-	struct stf_v_buf buffers;
-	enum stf_capture_type type;
-};
-
-irqreturn_t stf_wr_irq_handler(int irq, void *priv);
-irqreturn_t stf_isp_irq_handler(int irq, void *priv);
-irqreturn_t stf_line_irq_handler(int irq, void *priv);
-int stf_capture_register(struct stfcamss *stfcamss,
-			 struct v4l2_device *v4l2_dev);
-void stf_capture_unregister(struct stfcamss *stfcamss);
-
-#endif
diff --git a/drivers/staging/media/starfive/camss/stf-isp-hw-ops.c b/drivers/staging/media/starfive/camss/stf-isp-hw-ops.c
deleted file mode 100644
index c34631ff94227..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-isp-hw-ops.c
+++ /dev/null
@@ -1,445 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * stf_isp_hw_ops.c
- *
- * Register interface file for StarFive ISP driver
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- *
- */
-
-#include "stf-camss.h"
-
-static void stf_isp_config_obc(struct stfcamss *stfcamss)
-{
-	u32 reg_val, reg_add;
-
-	stf_isp_reg_write(stfcamss, ISP_REG_OBC_CFG, OBC_W_H(11) | OBC_W_W(11));
-
-	reg_val = GAIN_D_POINT(0x40) | GAIN_C_POINT(0x40) |
-		  GAIN_B_POINT(0x40) | GAIN_A_POINT(0x40);
-	for (reg_add = ISP_REG_OBCG_CFG_0; reg_add <= ISP_REG_OBCG_CFG_3;) {
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-	}
-
-	reg_val = OFFSET_D_POINT(0) | OFFSET_C_POINT(0) |
-		  OFFSET_B_POINT(0) | OFFSET_A_POINT(0);
-	for (reg_add = ISP_REG_OBCO_CFG_0; reg_add <= ISP_REG_OBCO_CFG_3;) {
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-	}
-}
-
-static void stf_isp_config_oecf(struct stfcamss *stfcamss)
-{
-	u32 reg_add, par_val;
-	u16 par_h, par_l;
-
-	par_h = 0x10; par_l = 0;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG0; reg_add <= ISP_REG_OECF_Y3_CFG0;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x40; par_l = 0x20;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG1; reg_add <= ISP_REG_OECF_Y3_CFG1;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x80; par_l = 0x60;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG2; reg_add <= ISP_REG_OECF_Y3_CFG2;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0xc0; par_l = 0xa0;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG3; reg_add <= ISP_REG_OECF_Y3_CFG3;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x100; par_l = 0xe0;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG4; reg_add <= ISP_REG_OECF_Y3_CFG4;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x200; par_l = 0x180;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG5; reg_add <= ISP_REG_OECF_Y3_CFG5;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x300; par_l = 0x280;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG6; reg_add <= ISP_REG_OECF_Y3_CFG6;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x3fe; par_l = 0x380;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_X0_CFG7; reg_add <= ISP_REG_OECF_Y3_CFG7;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 0x20;
-	}
-
-	par_h = 0x80; par_l = 0x80;
-	par_val = OCEF_PAR_H(par_h) | OCEF_PAR_L(par_l);
-	for (reg_add = ISP_REG_OECF_S0_CFG0; reg_add <= ISP_REG_OECF_S3_CFG7;) {
-		stf_isp_reg_write(stfcamss, reg_add, par_val);
-		reg_add += 4;
-	}
-}
-
-static void stf_isp_config_lccf(struct stfcamss *stfcamss)
-{
-	u32 reg_add;
-
-	stf_isp_reg_write(stfcamss, ISP_REG_LCCF_CFG_0,
-			  Y_DISTANCE(0x21C) | X_DISTANCE(0x3C0));
-	stf_isp_reg_write(stfcamss, ISP_REG_LCCF_CFG_1, LCCF_MAX_DIS(0xb));
-
-	for (reg_add = ISP_REG_LCCF_CFG_2; reg_add <= ISP_REG_LCCF_CFG_5;) {
-		stf_isp_reg_write(stfcamss, reg_add,
-				  LCCF_F2_PAR(0x0) | LCCF_F1_PAR(0x0));
-		reg_add += 4;
-	}
-}
-
-static void stf_isp_config_awb(struct stfcamss *stfcamss)
-{
-	u32 reg_val, reg_add;
-	u16 symbol_h, symbol_l;
-
-	symbol_h = 0x0; symbol_l = 0x0;
-	reg_val = AWB_X_SYMBOL_H(symbol_h) | AWB_X_SYMBOL_L(symbol_l);
-
-	for (reg_add = ISP_REG_AWB_X0_CFG_0; reg_add <= ISP_REG_AWB_X3_CFG_1;) {
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-	}
-
-	symbol_h = 0x0, symbol_l = 0x0;
-	reg_val = AWB_Y_SYMBOL_H(symbol_h) | AWB_Y_SYMBOL_L(symbol_l);
-
-	for (reg_add = ISP_REG_AWB_Y0_CFG_0; reg_add <= ISP_REG_AWB_Y3_CFG_1;) {
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-	}
-
-	symbol_h = 0x80, symbol_l = 0x80;
-	reg_val = AWB_S_SYMBOL_H(symbol_h) | AWB_S_SYMBOL_L(symbol_l);
-
-	for (reg_add = ISP_REG_AWB_S0_CFG_0; reg_add <= ISP_REG_AWB_S3_CFG_1;) {
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-	}
-}
-
-static void stf_isp_config_grgb(struct stfcamss *stfcamss)
-{
-	stf_isp_reg_write(stfcamss, ISP_REG_ICTC,
-			  GF_MODE(1) | MAXGT(0x140) | MINGT(0x40));
-	stf_isp_reg_write(stfcamss, ISP_REG_IDBC, BADGT(0x200) | BADXT(0x200));
-}
-
-static void stf_isp_config_cfa(struct stfcamss *stfcamss)
-{
-	stf_isp_reg_write(stfcamss, ISP_REG_RAW_FORMAT_CFG,
-			  SMY13(0) | SMY12(1) | SMY11(0) | SMY10(1) | SMY3(2) |
-			  SMY2(3) | SMY1(2) | SMY0(3));
-	stf_isp_reg_write(stfcamss, ISP_REG_ICFAM, CROSS_COV(3) | HV_W(2));
-}
-
-static void stf_isp_config_ccm(struct stfcamss *stfcamss)
-{
-	u32 reg_add;
-
-	stf_isp_reg_write(stfcamss, ISP_REG_ICAMD_0, DNRM_F(6) | CCM_M_DAT(0));
-
-	for (reg_add = ISP_REG_ICAMD_12; reg_add <= ISP_REG_ICAMD_20;) {
-		stf_isp_reg_write(stfcamss, reg_add, CCM_M_DAT(0x80));
-		reg_add += 0x10;
-	}
-
-	stf_isp_reg_write(stfcamss, ISP_REG_ICAMD_24, CCM_M_DAT(0x700));
-	stf_isp_reg_write(stfcamss, ISP_REG_ICAMD_25, CCM_M_DAT(0x200));
-}
-
-static void stf_isp_config_gamma(struct stfcamss *stfcamss)
-{
-	u32 reg_val, reg_add;
-	u16 gamma_slope_v, gamma_v;
-
-	gamma_slope_v = 0x2400; gamma_v = 0x0;
-	reg_val = GAMMA_S_VAL(gamma_slope_v) | GAMMA_VAL(gamma_v);
-	stf_isp_reg_write(stfcamss, ISP_REG_GAMMA_VAL0, reg_val);
-
-	gamma_slope_v = 0x800; gamma_v = 0x20;
-	for (reg_add = ISP_REG_GAMMA_VAL1; reg_add <= ISP_REG_GAMMA_VAL7;) {
-		reg_val = GAMMA_S_VAL(gamma_slope_v) | GAMMA_VAL(gamma_v);
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-		gamma_v += 0x20;
-	}
-
-	gamma_v = 0x100;
-	for (reg_add = ISP_REG_GAMMA_VAL8; reg_add <= ISP_REG_GAMMA_VAL13;) {
-		reg_val = GAMMA_S_VAL(gamma_slope_v) | GAMMA_VAL(gamma_v);
-		stf_isp_reg_write(stfcamss, reg_add, reg_val);
-		reg_add += 4;
-		gamma_v += 0x80;
-	}
-
-	gamma_v = 0x3fe;
-	reg_val = GAMMA_S_VAL(gamma_slope_v) | GAMMA_VAL(gamma_v);
-	stf_isp_reg_write(stfcamss, ISP_REG_GAMMA_VAL14, reg_val);
-}
-
-static void stf_isp_config_r2y(struct stfcamss *stfcamss)
-{
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_0, 0x4C);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_1, 0x97);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_2, 0x1d);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_3, 0x1d5);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_4, 0x1ac);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_5, 0x80);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_6, 0x80);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_7, 0x194);
-	stf_isp_reg_write(stfcamss, ISP_REG_R2Y_8, 0x1ec);
-}
-
-static void stf_isp_config_y_curve(struct stfcamss *stfcamss)
-{
-	u32 reg_add;
-	u16 y_curve;
-
-	y_curve = 0x0;
-	for (reg_add = ISP_REG_YCURVE_0; reg_add <= ISP_REG_YCURVE_63;) {
-		stf_isp_reg_write(stfcamss, reg_add, y_curve);
-		reg_add += 4;
-		y_curve += 0x10;
-	}
-}
-
-static void stf_isp_config_sharpen(struct stfcamss *sc)
-{
-	u32 reg_add;
-
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN0, S_DELTA(0x7) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN1, S_DELTA(0x18) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN2, S_DELTA(0x80) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN3, S_DELTA(0x100) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN4, S_DELTA(0x10) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN5, S_DELTA(0x60) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN6, S_DELTA(0x100) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN7, S_DELTA(0x190) | S_WEIGHT(0xf));
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN8, S_DELTA(0x0) | S_WEIGHT(0xf));
-
-	for (reg_add = ISP_REG_SHARPEN9; reg_add <= ISP_REG_SHARPEN14;) {
-		stf_isp_reg_write(sc, reg_add, S_WEIGHT(0xf));
-		reg_add += 4;
-	}
-
-	for (reg_add = ISP_REG_SHARPEN_FS0; reg_add <= ISP_REG_SHARPEN_FS5;) {
-		stf_isp_reg_write(sc, reg_add, S_FACTOR(0x10) | S_SLOPE(0x0));
-		reg_add += 4;
-	}
-
-	stf_isp_reg_write(sc, ISP_REG_SHARPEN_WN,
-			  PDIRF(0x8) | NDIRF(0x8) | WSUM(0xd7c));
-	stf_isp_reg_write(sc, ISP_REG_IUVS1, UVDIFF2(0xC0) | UVDIFF1(0x40));
-	stf_isp_reg_write(sc, ISP_REG_IUVS2, UVF(0xff) | UVSLOPE(0x0));
-	stf_isp_reg_write(sc, ISP_REG_IUVCKS1,
-			  UVCKDIFF2(0xa0) | UVCKDIFF1(0x40));
-}
-
-static void stf_isp_config_dnyuv(struct stfcamss *stfcamss)
-{
-	u32 reg_val;
-
-	reg_val = YUVSW5(7) | YUVSW4(7) | YUVSW3(7) | YUVSW2(7) |
-		  YUVSW1(7) | YUVSW0(7);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_YSWR0, reg_val);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_CSWR0, reg_val);
-
-	reg_val = YUVSW3(7) | YUVSW2(7) | YUVSW1(7) | YUVSW0(7);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_YSWR1, reg_val);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_CSWR1, reg_val);
-
-	reg_val = CURVE_D_H(0x60) | CURVE_D_L(0x40);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_YDR0, reg_val);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_CDR0, reg_val);
-
-	reg_val = CURVE_D_H(0xd8) | CURVE_D_L(0x90);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_YDR1, reg_val);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_CDR1, reg_val);
-
-	reg_val = CURVE_D_H(0x1e6) | CURVE_D_L(0x144);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_YDR2, reg_val);
-	stf_isp_reg_write(stfcamss, ISP_REG_DNYUV_CDR2, reg_val);
-}
-
-static void stf_isp_config_sat(struct stfcamss *stfcamss)
-{
-	stf_isp_reg_write(stfcamss, ISP_REG_CS_GAIN, CMAD(0x0) | CMAB(0x100));
-	stf_isp_reg_write(stfcamss, ISP_REG_CS_THRESHOLD, CMD(0x1f) | CMB(0x1));
-	stf_isp_reg_write(stfcamss, ISP_REG_CS_OFFSET, VOFF(0x0) | UOFF(0x0));
-	stf_isp_reg_write(stfcamss, ISP_REG_CS_HUE_F, SIN(0x0) | COS(0x100));
-	stf_isp_reg_write(stfcamss, ISP_REG_CS_SCALE, 0x8);
-	stf_isp_reg_write(stfcamss, ISP_REG_YADJ0, YOIR(0x401) | YIMIN(0x1));
-	stf_isp_reg_write(stfcamss, ISP_REG_YADJ1, YOMAX(0x3ff) | YOMIN(0x1));
-}
-
-int stf_isp_reset(struct stf_isp_dev *isp_dev)
-{
-	stf_isp_reg_set_bit(isp_dev->stfcamss, ISP_REG_ISP_CTRL_0,
-			    ISPC_RST_MASK, ISPC_RST);
-	stf_isp_reg_set_bit(isp_dev->stfcamss, ISP_REG_ISP_CTRL_0,
-			    ISPC_RST_MASK, 0);
-
-	return 0;
-}
-
-void stf_isp_init_cfg(struct stf_isp_dev *isp_dev)
-{
-	stf_isp_reg_write(isp_dev->stfcamss, ISP_REG_DC_CFG_1, DC_AXI_ID(0x0));
-	stf_isp_reg_write(isp_dev->stfcamss, ISP_REG_DEC_CFG,
-			  DEC_V_KEEP(0x0) |
-			  DEC_V_PERIOD(0x0) |
-			  DEC_H_KEEP(0x0) |
-			  DEC_H_PERIOD(0x0));
-
-	stf_isp_config_obc(isp_dev->stfcamss);
-	stf_isp_config_oecf(isp_dev->stfcamss);
-	stf_isp_config_lccf(isp_dev->stfcamss);
-	stf_isp_config_awb(isp_dev->stfcamss);
-	stf_isp_config_grgb(isp_dev->stfcamss);
-	stf_isp_config_cfa(isp_dev->stfcamss);
-	stf_isp_config_ccm(isp_dev->stfcamss);
-	stf_isp_config_gamma(isp_dev->stfcamss);
-	stf_isp_config_r2y(isp_dev->stfcamss);
-	stf_isp_config_y_curve(isp_dev->stfcamss);
-	stf_isp_config_sharpen(isp_dev->stfcamss);
-	stf_isp_config_dnyuv(isp_dev->stfcamss);
-	stf_isp_config_sat(isp_dev->stfcamss);
-
-	stf_isp_reg_write(isp_dev->stfcamss, ISP_REG_CSI_MODULE_CFG,
-			  CSI_DUMP_EN | CSI_SC_EN | CSI_AWB_EN |
-			  CSI_LCCF_EN | CSI_OECF_EN | CSI_OBC_EN | CSI_DEC_EN);
-	stf_isp_reg_write(isp_dev->stfcamss, ISP_REG_ISP_CTRL_1,
-			  CTRL_SAT(1) | CTRL_DBC | CTRL_CTC | CTRL_YHIST |
-			  CTRL_YCURVE | CTRL_BIYUV | CTRL_SCE | CTRL_EE |
-			  CTRL_CCE | CTRL_RGE | CTRL_CME | CTRL_AE | CTRL_CE);
-}
-
-static void stf_isp_config_crop(struct stfcamss *stfcamss,
-				struct v4l2_rect *crop)
-{
-	u32 bpp = stfcamss->isp_dev.current_fmt->bpp;
-	u32 val;
-
-	val = VSTART_CAP(crop->top) | HSTART_CAP(crop->left);
-	stf_isp_reg_write(stfcamss, ISP_REG_PIC_CAPTURE_START_CFG, val);
-
-	val = VEND_CAP(crop->height + crop->top - 1) |
-	      HEND_CAP(crop->width + crop->left - 1);
-	stf_isp_reg_write(stfcamss, ISP_REG_PIC_CAPTURE_END_CFG, val);
-
-	val = H_ACT_CAP(crop->height) | W_ACT_CAP(crop->width);
-	stf_isp_reg_write(stfcamss, ISP_REG_PIPELINE_XY_SIZE, val);
-
-	val = ALIGN(crop->width * bpp / 8, STFCAMSS_FRAME_WIDTH_ALIGN_8);
-	stf_isp_reg_write(stfcamss, ISP_REG_STRIDE, val);
-}
-
-static void stf_isp_config_raw_fmt(struct stfcamss *stfcamss, u32 mcode)
-{
-	u32 val, val1;
-
-	switch (mcode) {
-	case MEDIA_BUS_FMT_SRGGB10_1X10:
-	case MEDIA_BUS_FMT_SRGGB8_1X8:
-		/* 3 2 3 2 1 0 1 0 B Gb B Gb Gr R Gr R */
-		val = SMY13(3) | SMY12(2) | SMY11(3) | SMY10(2) |
-		      SMY3(1) | SMY2(0) | SMY1(1) | SMY0(0);
-		val1 = CTRL_SAT(0x0);
-		break;
-	case MEDIA_BUS_FMT_SGRBG10_1X10:
-	case MEDIA_BUS_FMT_SGRBG8_1X8:
-		/* 2 3 2 3 0 1 0 1, Gb B Gb B R Gr R Gr */
-		val = SMY13(2) | SMY12(3) | SMY11(2) | SMY10(3) |
-		      SMY3(0) | SMY2(1) | SMY1(0) | SMY0(1);
-		val1 = CTRL_SAT(0x2);
-		break;
-	case MEDIA_BUS_FMT_SGBRG10_1X10:
-	case MEDIA_BUS_FMT_SGBRG8_1X8:
-		/* 1 0 1 0 3 2 3 2, Gr R Gr R B Gb B Gb */
-		val = SMY13(1) | SMY12(0) | SMY11(1) | SMY10(0) |
-		      SMY3(3) | SMY2(2) | SMY1(3) | SMY0(2);
-		val1 = CTRL_SAT(0x3);
-		break;
-	case MEDIA_BUS_FMT_SBGGR10_1X10:
-	case MEDIA_BUS_FMT_SBGGR8_1X8:
-		/* 0 1 0 1 2 3 2 3 R Gr R Gr Gb B Gb B */
-		val = SMY13(0) | SMY12(1) | SMY11(0) | SMY10(1) |
-		      SMY3(2) | SMY2(3) | SMY1(2) | SMY0(3);
-		val1 = CTRL_SAT(0x1);
-		break;
-	default:
-		val = SMY13(0) | SMY12(1) | SMY11(0) | SMY10(1) |
-		      SMY3(2) | SMY2(3) | SMY1(2) | SMY0(3);
-		val1 = CTRL_SAT(0x1);
-		break;
-	}
-	stf_isp_reg_write(stfcamss, ISP_REG_RAW_FORMAT_CFG, val);
-	stf_isp_reg_set_bit(stfcamss, ISP_REG_ISP_CTRL_1, CTRL_SAT_MASK, val1);
-}
-
-void stf_isp_settings(struct stf_isp_dev *isp_dev,
-		      struct v4l2_rect *crop, u32 mcode)
-{
-	struct stfcamss *stfcamss = isp_dev->stfcamss;
-
-	stf_isp_config_crop(stfcamss, crop);
-	stf_isp_config_raw_fmt(stfcamss, mcode);
-
-	stf_isp_reg_set_bit(stfcamss, ISP_REG_DUMP_CFG_1,
-			    DUMP_BURST_LEN_MASK | DUMP_SD_MASK,
-			    DUMP_BURST_LEN(3));
-
-	stf_isp_reg_write(stfcamss, ISP_REG_ITIIWSR,
-			  ITI_HSIZE(IMAGE_MAX_HEIGH) |
-			  ITI_WSIZE(IMAGE_MAX_WIDTH));
-	stf_isp_reg_write(stfcamss, ISP_REG_ITIDWLSR, 0x960);
-	stf_isp_reg_write(stfcamss, ISP_REG_ITIDRLSR, 0x960);
-	stf_isp_reg_write(stfcamss, ISP_REG_SENSOR, IMAGER_SEL(1));
-}
-
-void stf_isp_stream_set(struct stf_isp_dev *isp_dev)
-{
-	struct stfcamss *stfcamss = isp_dev->stfcamss;
-
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_ISP_CTRL_0,
-				ISPC_ENUO | ISPC_ENLS | ISPC_RST, 10);
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_ISP_CTRL_0,
-				ISPC_ENUO | ISPC_ENLS, 10);
-	stf_isp_reg_write(stfcamss, ISP_REG_IESHD, SHAD_UP_M);
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_ISP_CTRL_0,
-				ISPC_ENUO | ISPC_ENLS | ISPC_EN, 10);
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_CSIINTS,
-				CSI_INTS(1) | CSI_SHA_M(4), 10);
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_CSIINTS,
-				CSI_INTS(2) | CSI_SHA_M(4), 10);
-	stf_isp_reg_write_delay(stfcamss, ISP_REG_CSI_INPUT_EN_AND_STATUS,
-				CSI_EN_S, 10);
-}
diff --git a/drivers/staging/media/starfive/camss/stf-isp.c b/drivers/staging/media/starfive/camss/stf-isp.c
deleted file mode 100644
index df7a903fbb1b0..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-isp.c
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * stf_isp.c
- *
- * StarFive Camera Subsystem - ISP Module
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-#include <media/v4l2-rect.h>
-
-#include "stf-camss.h"
-
-static int isp_set_selection(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_state *state,
-			     struct v4l2_subdev_selection *sel);
-
-static const struct stf_isp_format isp_formats_sink[] = {
-	{ MEDIA_BUS_FMT_SRGGB10_1X10, 10 },
-	{ MEDIA_BUS_FMT_SGRBG10_1X10, 10 },
-	{ MEDIA_BUS_FMT_SGBRG10_1X10, 10 },
-	{ MEDIA_BUS_FMT_SBGGR10_1X10, 10 },
-};
-
-static const struct stf_isp_format isp_formats_source[] = {
-	{ MEDIA_BUS_FMT_YUYV8_1_5X8, 8 },
-};
-
-static const struct stf_isp_format_table isp_formats_st7110[] = {
-	{ isp_formats_sink, ARRAY_SIZE(isp_formats_sink) },
-	{ isp_formats_source, ARRAY_SIZE(isp_formats_source) },
-};
-
-static const struct stf_isp_format *
-stf_g_fmt_by_mcode(const struct stf_isp_format_table *fmt_table, u32 mcode)
-{
-	unsigned int i;
-
-	for (i = 0; i < fmt_table->nfmts; i++) {
-		if (fmt_table->fmts[i].code == mcode)
-			return &fmt_table->fmts[i];
-	}
-
-	return NULL;
-}
-
-int stf_isp_init(struct stfcamss *stfcamss)
-{
-	struct stf_isp_dev *isp_dev = &stfcamss->isp_dev;
-
-	isp_dev->stfcamss = stfcamss;
-	isp_dev->formats = isp_formats_st7110;
-	isp_dev->nformats = ARRAY_SIZE(isp_formats_st7110);
-	isp_dev->current_fmt = &isp_formats_source[0];
-
-	return 0;
-}
-
-static int isp_set_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct stf_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
-	struct v4l2_subdev_state *sd_state;
-	struct v4l2_mbus_framefmt *fmt;
-	struct v4l2_rect *crop;
-
-	sd_state = v4l2_subdev_lock_and_get_active_state(sd);
-	fmt = v4l2_subdev_state_get_format(sd_state, STF_ISP_PAD_SINK);
-	crop = v4l2_subdev_state_get_crop(sd_state, STF_ISP_PAD_SRC);
-
-	if (enable) {
-		stf_isp_reset(isp_dev);
-		stf_isp_init_cfg(isp_dev);
-		stf_isp_settings(isp_dev, crop, fmt->code);
-		stf_isp_stream_set(isp_dev);
-	}
-
-	v4l2_subdev_call(isp_dev->source_subdev, video, s_stream, enable);
-
-	v4l2_subdev_unlock_state(sd_state);
-	return 0;
-}
-
-static void isp_try_format(struct stf_isp_dev *isp_dev,
-			   struct v4l2_subdev_state *state,
-			   unsigned int pad,
-			   struct v4l2_mbus_framefmt *fmt)
-{
-	const struct stf_isp_format_table *formats;
-
-	if (pad >= STF_ISP_PAD_MAX) {
-		fmt->colorspace = V4L2_COLORSPACE_SRGB;
-		return;
-	}
-
-	formats = &isp_dev->formats[pad];
-
-	fmt->width = clamp_t(u32, fmt->width, STFCAMSS_FRAME_MIN_WIDTH,
-			     STFCAMSS_FRAME_MAX_WIDTH);
-	fmt->height = clamp_t(u32, fmt->height, STFCAMSS_FRAME_MIN_HEIGHT,
-			      STFCAMSS_FRAME_MAX_HEIGHT);
-	fmt->height &= ~0x1;
-	fmt->field = V4L2_FIELD_NONE;
-	fmt->colorspace = V4L2_COLORSPACE_SRGB;
-	fmt->flags = 0;
-
-	if (!stf_g_fmt_by_mcode(formats, fmt->code))
-		fmt->code = formats->fmts[0].code;
-}
-
-static int isp_enum_mbus_code(struct v4l2_subdev *sd,
-			      struct v4l2_subdev_state *state,
-			      struct v4l2_subdev_mbus_code_enum *code)
-{
-	struct stf_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
-	const struct stf_isp_format_table *formats;
-
-	if (code->pad == STF_ISP_PAD_SINK) {
-		if (code->index >= ARRAY_SIZE(isp_formats_sink))
-			return -EINVAL;
-
-		formats = &isp_dev->formats[code->pad];
-		code->code = formats->fmts[code->index].code;
-	} else {
-		struct v4l2_mbus_framefmt *sink_fmt;
-
-		if (code->index >= ARRAY_SIZE(isp_formats_source))
-			return -EINVAL;
-
-		sink_fmt = v4l2_subdev_state_get_format(state,
-							STF_ISP_PAD_SRC);
-
-		code->code = sink_fmt->code;
-		if (!code->code)
-			return -EINVAL;
-	}
-	code->flags = 0;
-
-	return 0;
-}
-
-static int isp_set_format(struct v4l2_subdev *sd,
-			  struct v4l2_subdev_state *state,
-			  struct v4l2_subdev_format *fmt)
-{
-	struct stf_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
-	struct v4l2_mbus_framefmt *format;
-
-	format = v4l2_subdev_state_get_format(state, fmt->pad);
-	if (!format)
-		return -EINVAL;
-
-	isp_try_format(isp_dev, state, fmt->pad, &fmt->format);
-	*format = fmt->format;
-
-	isp_dev->current_fmt = stf_g_fmt_by_mcode(&isp_dev->formats[fmt->pad],
-						  fmt->format.code);
-
-	/* Propagate to in crop */
-	if (fmt->pad == STF_ISP_PAD_SINK) {
-		struct v4l2_subdev_selection sel = { 0 };
-
-		/* Reset sink pad compose selection */
-		sel.which = fmt->which;
-		sel.pad = STF_ISP_PAD_SINK;
-		sel.target = V4L2_SEL_TGT_CROP;
-		sel.r.width = fmt->format.width;
-		sel.r.height = fmt->format.height;
-		isp_set_selection(sd, state, &sel);
-	}
-
-	return 0;
-}
-
-static const struct v4l2_rect stf_frame_min_crop = {
-	.width = STFCAMSS_FRAME_MIN_WIDTH,
-	.height = STFCAMSS_FRAME_MIN_HEIGHT,
-	.top = 0,
-	.left = 0,
-};
-
-static void isp_try_crop(struct stf_isp_dev *isp_dev,
-			 struct v4l2_subdev_state *state,
-			 struct v4l2_rect *crop)
-{
-	struct v4l2_mbus_framefmt *fmt =
-		v4l2_subdev_state_get_format(state, STF_ISP_PAD_SINK);
-
-	const struct v4l2_rect bounds = {
-		.width = fmt->width,
-		.height = fmt->height,
-		.left = 0,
-		.top = 0,
-	};
-
-	v4l2_rect_set_min_size(crop, &stf_frame_min_crop);
-	v4l2_rect_map_inside(crop, &bounds);
-}
-
-static int isp_get_selection(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_state *state,
-			     struct v4l2_subdev_selection *sel)
-{
-	struct v4l2_subdev_format fmt = { 0 };
-	struct v4l2_rect *rect;
-
-	switch (sel->target) {
-	case V4L2_SEL_TGT_CROP_BOUNDS:
-		if (sel->pad == STF_ISP_PAD_SINK) {
-			fmt.format = *v4l2_subdev_state_get_format(state,
-								   sel->pad);
-			sel->r.left = 0;
-			sel->r.top = 0;
-			sel->r.width = fmt.format.width;
-			sel->r.height = fmt.format.height;
-		} else if (sel->pad == STF_ISP_PAD_SRC) {
-			rect = v4l2_subdev_state_get_crop(state, sel->pad);
-			sel->r = *rect;
-		}
-		break;
-
-	case V4L2_SEL_TGT_CROP:
-		rect = v4l2_subdev_state_get_crop(state, sel->pad);
-		if (!rect)
-			return -EINVAL;
-
-		sel->r = *rect;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int isp_set_selection(struct v4l2_subdev *sd,
-			     struct v4l2_subdev_state *state,
-			     struct v4l2_subdev_selection *sel)
-{
-	struct stf_isp_dev *isp_dev = v4l2_get_subdevdata(sd);
-	struct v4l2_rect *rect;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	if (sel->target == V4L2_SEL_TGT_CROP &&
-	    sel->pad == STF_ISP_PAD_SINK) {
-		struct v4l2_subdev_selection crop = { 0 };
-
-		rect = v4l2_subdev_state_get_crop(state, sel->pad);
-		if (!rect)
-			return -EINVAL;
-
-		isp_try_crop(isp_dev, state, &sel->r);
-		*rect = sel->r;
-
-		/* Reset source crop selection */
-		crop.which = sel->which;
-		crop.pad = STF_ISP_PAD_SRC;
-		crop.target = V4L2_SEL_TGT_CROP;
-		crop.r = *rect;
-		isp_set_selection(sd, state, &crop);
-	} else if (sel->target == V4L2_SEL_TGT_CROP &&
-		   sel->pad == STF_ISP_PAD_SRC) {
-		struct v4l2_subdev_format fmt = { 0 };
-
-		rect = v4l2_subdev_state_get_crop(state, sel->pad);
-		if (!rect)
-			return -EINVAL;
-
-		isp_try_crop(isp_dev, state, &sel->r);
-		*rect = sel->r;
-
-		/* Reset source pad format width and height */
-		fmt.which = sel->which;
-		fmt.pad = STF_ISP_PAD_SRC;
-		fmt.format.width = rect->width;
-		fmt.format.height = rect->height;
-		isp_set_format(sd, state, &fmt);
-	}
-
-	dev_dbg(isp_dev->stfcamss->dev, "pad: %d sel(%d,%d)/%ux%u\n",
-		sel->pad, sel->r.left, sel->r.top, sel->r.width, sel->r.height);
-
-	return 0;
-}
-
-static int isp_init_formats(struct v4l2_subdev *sd,
-			    struct v4l2_subdev_state *sd_state)
-{
-	struct v4l2_subdev_format format = {
-		.pad = STF_ISP_PAD_SINK,
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-		.format = {
-			.code = MEDIA_BUS_FMT_SRGGB10_1X10,
-			.width = 1920,
-			.height = 1080
-		}
-	};
-
-	return isp_set_format(sd, sd_state, &format);
-}
-
-static const struct v4l2_subdev_video_ops isp_video_ops = {
-	.s_stream = isp_set_stream,
-};
-
-static const struct v4l2_subdev_pad_ops isp_pad_ops = {
-	.enum_mbus_code = isp_enum_mbus_code,
-	.get_fmt = v4l2_subdev_get_fmt,
-	.set_fmt = isp_set_format,
-	.get_selection = isp_get_selection,
-	.set_selection = isp_set_selection,
-};
-
-static const struct v4l2_subdev_ops isp_v4l2_ops = {
-	.video = &isp_video_ops,
-	.pad = &isp_pad_ops,
-};
-
-static const struct v4l2_subdev_internal_ops isp_internal_ops = {
-	.init_state = isp_init_formats,
-};
-
-static const struct media_entity_operations isp_media_ops = {
-	.link_validate = v4l2_subdev_link_validate,
-};
-
-int stf_isp_register(struct stf_isp_dev *isp_dev, struct v4l2_device *v4l2_dev)
-{
-	struct v4l2_subdev *sd = &isp_dev->subdev;
-	struct media_pad *pads = isp_dev->pads;
-	int ret;
-
-	v4l2_subdev_init(sd, &isp_v4l2_ops);
-	sd->internal_ops = &isp_internal_ops;
-	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	snprintf(sd->name, ARRAY_SIZE(sd->name), "stf_isp");
-	v4l2_set_subdevdata(sd, isp_dev);
-
-	pads[STF_ISP_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
-	pads[STF_ISP_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
-
-	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_ISP;
-	sd->entity.ops = &isp_media_ops;
-	ret = media_entity_pads_init(&sd->entity, STF_ISP_PAD_MAX, pads);
-	if (ret) {
-		dev_err(isp_dev->stfcamss->dev,
-			"Failed to init media entity: %d\n", ret);
-		return ret;
-	}
-
-	ret = v4l2_subdev_init_finalize(sd);
-	if (ret)
-		goto err_entity_cleanup;
-
-	ret = v4l2_device_register_subdev(v4l2_dev, sd);
-	if (ret) {
-		dev_err(isp_dev->stfcamss->dev,
-			"Failed to register subdev: %d\n", ret);
-		goto err_subdev_cleanup;
-	}
-
-	return 0;
-
-err_subdev_cleanup:
-	v4l2_subdev_cleanup(sd);
-err_entity_cleanup:
-	media_entity_cleanup(&sd->entity);
-	return ret;
-}
-
-int stf_isp_unregister(struct stf_isp_dev *isp_dev)
-{
-	v4l2_device_unregister_subdev(&isp_dev->subdev);
-	v4l2_subdev_cleanup(&isp_dev->subdev);
-	media_entity_cleanup(&isp_dev->subdev.entity);
-
-	return 0;
-}
diff --git a/drivers/staging/media/starfive/camss/stf-isp.h b/drivers/staging/media/starfive/camss/stf-isp.h
deleted file mode 100644
index 955cbb0483637..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-isp.h
+++ /dev/null
@@ -1,428 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * stf_isp.h
- *
- * StarFive Camera Subsystem - ISP Module
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#ifndef STF_ISP_H
-#define STF_ISP_H
-
-#include <media/v4l2-subdev.h>
-
-#include "stf-video.h"
-
-#define ISP_RAW_DATA_BITS			12
-#define SCALER_RATIO_MAX			1
-#define STF_ISP_REG_OFFSET_MAX			0x0fff
-#define STF_ISP_REG_DELAY_MAX			100
-
-/* isp registers */
-#define ISP_REG_CSI_INPUT_EN_AND_STATUS		0x000
-#define CSI_SCD_ERR				BIT(6)
-#define CSI_ITU656_ERR				BIT(4)
-#define CSI_ITU656_F				BIT(3)
-#define CSI_SCD_DONE				BIT(2)
-#define CSI_BUSY_S				BIT(1)
-#define CSI_EN_S				BIT(0)
-
-#define ISP_REG_CSIINTS				0x008
-#define CSI_INTS(n)				((n) << 16)
-#define CSI_SHA_M(n)				((n) << 0)
-#define CSI_INTS_MASK				GENMASK(17, 16)
-
-#define ISP_REG_CSI_MODULE_CFG			0x010
-#define CSI_DUMP_EN				BIT(19)
-#define CSI_VS_EN				BIT(18)
-#define CSI_SC_EN				BIT(17)
-#define CSI_OBA_EN				BIT(16)
-#define CSI_AWB_EN				BIT(7)
-#define CSI_LCCF_EN				BIT(6)
-#define CSI_OECFHM_EN				BIT(5)
-#define CSI_OECF_EN				BIT(4)
-#define CSI_LCBQ_EN				BIT(3)
-#define CSI_OBC_EN				BIT(2)
-#define CSI_DEC_EN				BIT(1)
-#define CSI_DC_EN				BIT(0)
-
-#define ISP_REG_SENSOR				0x014
-#define DVP_SYNC_POL(n)				((n) << 2)
-#define ITU656_EN(n)				((n) << 1)
-#define IMAGER_SEL(n)				((n) << 0)
-
-#define ISP_REG_RAW_FORMAT_CFG			0x018
-#define SMY13(n)				((n) << 14)
-#define SMY12(n)				((n) << 12)
-#define SMY11(n)				((n) << 10)
-#define SMY10(n)				((n) << 8)
-#define SMY3(n)					((n) << 6)
-#define SMY2(n)					((n) << 4)
-#define SMY1(n)					((n) << 2)
-#define SMY0(n)					((n) << 0)
-
-#define ISP_REG_PIC_CAPTURE_START_CFG		0x01c
-#define VSTART_CAP(n)				((n) << 16)
-#define HSTART_CAP(n)				((n) << 0)
-
-#define ISP_REG_PIC_CAPTURE_END_CFG		0x020
-#define VEND_CAP(n)				((n) << 16)
-#define HEND_CAP(n)				((n) << 0)
-
-#define ISP_REG_DUMP_CFG_0			0x024
-#define ISP_REG_DUMP_CFG_1			0x028
-#define DUMP_ID(n)				((n) << 24)
-#define DUMP_SHT(n)				((n) << 20)
-#define DUMP_BURST_LEN(n)			((n) << 16)
-#define DUMP_SD(n)				((n) << 0)
-#define DUMP_BURST_LEN_MASK			GENMASK(17, 16)
-#define DUMP_SD_MASK				GENMASK(15, 0)
-
-#define ISP_REG_DEC_CFG				0x030
-#define DEC_V_KEEP(n)				((n) << 24)
-#define DEC_V_PERIOD(n)				((n) << 16)
-#define DEC_H_KEEP(n)				((n) << 8)
-#define DEC_H_PERIOD(n)				((n) << 0)
-
-#define ISP_REG_OBC_CFG				0x034
-#define OBC_W_H(y)				((y) << 4)
-#define OBC_W_W(x)				((x) << 0)
-
-#define ISP_REG_DC_CFG_1			0x044
-#define DC_AXI_ID(n)				((n) << 0)
-
-#define ISP_REG_LCCF_CFG_0			0x050
-#define Y_DISTANCE(y)				((y) << 16)
-#define X_DISTANCE(x)				((x) << 0)
-
-#define ISP_REG_LCCF_CFG_1			0x058
-#define LCCF_MAX_DIS(n)				((n) << 0)
-
-#define ISP_REG_LCBQ_CFG_0			0x074
-#define H_LCBQ(y)				((y) << 12)
-#define W_LCBQ(x)				((x) << 8)
-
-#define ISP_REG_LCBQ_CFG_1			0x07c
-#define Y_COOR(y)				((y) << 16)
-#define X_COOR(x)				((x) << 0)
-
-#define ISP_REG_LCCF_CFG_2			0x0e0
-#define ISP_REG_LCCF_CFG_3			0x0e4
-#define ISP_REG_LCCF_CFG_4			0x0e8
-#define ISP_REG_LCCF_CFG_5			0x0ec
-#define LCCF_F2_PAR(n)				((n) << 16)
-#define LCCF_F1_PAR(n)				((n) << 0)
-
-#define ISP_REG_OECF_X0_CFG0			0x100
-#define ISP_REG_OECF_X0_CFG1			0x104
-#define ISP_REG_OECF_X0_CFG2			0x108
-#define ISP_REG_OECF_X0_CFG3			0x10c
-#define ISP_REG_OECF_X0_CFG4			0x110
-#define ISP_REG_OECF_X0_CFG5			0x114
-#define ISP_REG_OECF_X0_CFG6			0x118
-#define ISP_REG_OECF_X0_CFG7			0x11c
-
-#define ISP_REG_OECF_Y3_CFG0			0x1e0
-#define ISP_REG_OECF_Y3_CFG1			0x1e4
-#define ISP_REG_OECF_Y3_CFG2			0x1e8
-#define ISP_REG_OECF_Y3_CFG3			0x1ec
-#define ISP_REG_OECF_Y3_CFG4			0x1f0
-#define ISP_REG_OECF_Y3_CFG5			0x1f4
-#define ISP_REG_OECF_Y3_CFG6			0x1f8
-#define ISP_REG_OECF_Y3_CFG7			0x1fc
-
-#define ISP_REG_OECF_S0_CFG0			0x200
-#define ISP_REG_OECF_S3_CFG7			0x27c
-#define OCEF_PAR_H(n)				((n) << 16)
-#define OCEF_PAR_L(n)				((n) << 0)
-
-#define ISP_REG_AWB_X0_CFG_0			0x280
-#define ISP_REG_AWB_X0_CFG_1			0x284
-#define ISP_REG_AWB_X1_CFG_0			0x288
-#define ISP_REG_AWB_X1_CFG_1			0x28c
-#define ISP_REG_AWB_X2_CFG_0			0x290
-#define ISP_REG_AWB_X2_CFG_1			0x294
-#define ISP_REG_AWB_X3_CFG_0			0x298
-#define ISP_REG_AWB_X3_CFG_1			0x29c
-#define AWB_X_SYMBOL_H(n)			((n) << 16)
-#define AWB_X_SYMBOL_L(n)			((n) << 0)
-
-#define ISP_REG_AWB_Y0_CFG_0			0x2a0
-#define ISP_REG_AWB_Y0_CFG_1			0x2a4
-#define ISP_REG_AWB_Y1_CFG_0			0x2a8
-#define ISP_REG_AWB_Y1_CFG_1			0x2ac
-#define ISP_REG_AWB_Y2_CFG_0			0x2b0
-#define ISP_REG_AWB_Y2_CFG_1			0x2b4
-#define ISP_REG_AWB_Y3_CFG_0			0x2b8
-#define ISP_REG_AWB_Y3_CFG_1			0x2bc
-#define AWB_Y_SYMBOL_H(n)			((n) << 16)
-#define AWB_Y_SYMBOL_L(n)			((n) << 0)
-
-#define ISP_REG_AWB_S0_CFG_0			0x2c0
-#define ISP_REG_AWB_S0_CFG_1			0x2c4
-#define ISP_REG_AWB_S1_CFG_0			0x2c8
-#define ISP_REG_AWB_S1_CFG_1			0x2cc
-#define ISP_REG_AWB_S2_CFG_0			0x2d0
-#define ISP_REG_AWB_S2_CFG_1			0x2d4
-#define ISP_REG_AWB_S3_CFG_0			0x2d8
-#define ISP_REG_AWB_S3_CFG_1			0x2dc
-#define AWB_S_SYMBOL_H(n)			((n) << 16)
-#define AWB_S_SYMBOL_L(n)			((n) << 0)
-
-#define ISP_REG_OBCG_CFG_0			0x2e0
-#define ISP_REG_OBCG_CFG_1			0x2e4
-#define ISP_REG_OBCG_CFG_2			0x2e8
-#define ISP_REG_OBCG_CFG_3			0x2ec
-#define ISP_REG_OBCO_CFG_0			0x2f0
-#define ISP_REG_OBCO_CFG_1			0x2f4
-#define ISP_REG_OBCO_CFG_2			0x2f8
-#define ISP_REG_OBCO_CFG_3			0x2fc
-#define GAIN_D_POINT(x)				((x) << 24)
-#define GAIN_C_POINT(x)				((x) << 16)
-#define GAIN_B_POINT(x)				((x) << 8)
-#define GAIN_A_POINT(x)				((x) << 0)
-#define OFFSET_D_POINT(x)			((x) << 24)
-#define OFFSET_C_POINT(x)			((x) << 16)
-#define OFFSET_B_POINT(x)			((x) << 8)
-#define OFFSET_A_POINT(x)			((x) << 0)
-
-#define ISP_REG_ISP_CTRL_0			0xa00
-#define ISPC_LINE				BIT(27)
-#define ISPC_SC					BIT(26)
-#define ISPC_CSI				BIT(25)
-#define ISPC_ISP				BIT(24)
-#define ISPC_ENUO				BIT(20)
-#define ISPC_ENLS				BIT(17)
-#define ISPC_ENSS1				BIT(12)
-#define ISPC_ENSS0				BIT(11)
-#define ISPC_RST				BIT(1)
-#define ISPC_EN					BIT(0)
-#define ISPC_RST_MASK				BIT(1)
-#define ISPC_INT_ALL_MASK			GENMASK(27, 24)
-
-#define ISP_REG_ISP_CTRL_1			0xa08
-#define CTRL_SAT(n)				((n) << 28)
-#define CTRL_DBC				BIT(22)
-#define CTRL_CTC				BIT(21)
-#define CTRL_YHIST				BIT(20)
-#define CTRL_YCURVE				BIT(19)
-#define CTRL_CTM				BIT(18)
-#define CTRL_BIYUV				BIT(17)
-#define CTRL_SCE				BIT(8)
-#define CTRL_EE					BIT(7)
-#define CTRL_CCE				BIT(5)
-#define CTRL_RGE				BIT(4)
-#define CTRL_CME				BIT(3)
-#define CTRL_AE					BIT(2)
-#define CTRL_CE					BIT(1)
-#define CTRL_SAT_MASK				GENMASK(31, 28)
-
-#define ISP_REG_PIPELINE_XY_SIZE		0xa0c
-#define H_ACT_CAP(n)				((n) << 16)
-#define W_ACT_CAP(n)				((n) << 0)
-
-#define ISP_REG_ICTC				0xa10
-#define GF_MODE(n)				((n) << 30)
-#define MAXGT(n)				((n) << 16)
-#define MINGT(n)				((n) << 0)
-
-#define ISP_REG_IDBC				0xa14
-#define BADGT(n)				((n) << 16)
-#define BADXT(n)				((n) << 0)
-
-#define ISP_REG_ICFAM				0xa1c
-#define CROSS_COV(n)				((n) << 4)
-#define HV_W(n)					((n) << 0)
-
-#define ISP_REG_CS_GAIN				0xa30
-#define CMAD(n)					((n) << 16)
-#define CMAB(n)					((n) << 0)
-
-#define ISP_REG_CS_THRESHOLD			0xa34
-#define CMD(n)					((n) << 16)
-#define CMB(n)					((n) << 0)
-
-#define ISP_REG_CS_OFFSET			0xa38
-#define VOFF(n)					((n) << 16)
-#define UOFF(n)					((n) << 0)
-
-#define ISP_REG_CS_HUE_F			0xa3c
-#define SIN(n)					((n) << 16)
-#define COS(n)					((n) << 0)
-
-#define ISP_REG_CS_SCALE			0xa40
-
-#define ISP_REG_IESHD				0xa50
-#define SHAD_UP_M				BIT(1)
-#define SHAD_UP_EN				BIT(0)
-
-#define ISP_REG_YADJ0				0xa54
-#define YOIR(n)					((n) << 16)
-#define YIMIN(n)				((n) << 0)
-
-#define ISP_REG_YADJ1				0xa58
-#define YOMAX(n)				((n) << 16)
-#define YOMIN(n)				((n) << 0)
-
-#define ISP_REG_Y_PLANE_START_ADDR		0xa80
-#define ISP_REG_UV_PLANE_START_ADDR		0xa84
-#define ISP_REG_STRIDE				0xa88
-
-#define ISP_REG_ITIIWSR				0xb20
-#define ITI_HSIZE(n)				((n) << 16)
-#define ITI_WSIZE(n)				((n) << 0)
-
-#define ISP_REG_ITIDWLSR			0xb24
-#define ISP_REG_ITIPDFR				0xb38
-#define ISP_REG_ITIDRLSR			0xb3C
-
-#define ISP_REG_DNYUV_YSWR0			0xc00
-#define ISP_REG_DNYUV_YSWR1			0xc04
-#define ISP_REG_DNYUV_CSWR0			0xc08
-#define ISP_REG_DNYUV_CSWR1			0xc0c
-#define YUVSW5(n)				((n) << 20)
-#define YUVSW4(n)				((n) << 16)
-#define YUVSW3(n)				((n) << 12)
-#define YUVSW2(n)				((n) << 8)
-#define YUVSW1(n)				((n) << 4)
-#define YUVSW0(n)				((n) << 0)
-
-#define ISP_REG_DNYUV_YDR0			0xc10
-#define ISP_REG_DNYUV_YDR1			0xc14
-#define ISP_REG_DNYUV_YDR2			0xc18
-#define ISP_REG_DNYUV_CDR0			0xc1c
-#define ISP_REG_DNYUV_CDR1			0xc20
-#define ISP_REG_DNYUV_CDR2			0xc24
-#define CURVE_D_H(n)				((n) << 16)
-#define CURVE_D_L(n)				((n) << 0)
-
-#define ISP_REG_ICAMD_0				0xc40
-#define ISP_REG_ICAMD_12			0xc70
-#define ISP_REG_ICAMD_20			0xc90
-#define ISP_REG_ICAMD_24			0xca0
-#define ISP_REG_ICAMD_25			0xca4
-#define DNRM_F(n)				((n) << 16)
-#define CCM_M_DAT(n)				((n) << 0)
-
-#define ISP_REG_GAMMA_VAL0			0xe00
-#define ISP_REG_GAMMA_VAL1			0xe04
-#define ISP_REG_GAMMA_VAL2			0xe08
-#define ISP_REG_GAMMA_VAL3			0xe0c
-#define ISP_REG_GAMMA_VAL4			0xe10
-#define ISP_REG_GAMMA_VAL5			0xe14
-#define ISP_REG_GAMMA_VAL6			0xe18
-#define ISP_REG_GAMMA_VAL7			0xe1c
-#define ISP_REG_GAMMA_VAL8			0xe20
-#define ISP_REG_GAMMA_VAL9			0xe24
-#define ISP_REG_GAMMA_VAL10			0xe28
-#define ISP_REG_GAMMA_VAL11			0xe2c
-#define ISP_REG_GAMMA_VAL12			0xe30
-#define ISP_REG_GAMMA_VAL13			0xe34
-#define ISP_REG_GAMMA_VAL14			0xe38
-#define GAMMA_S_VAL(n)				((n) << 16)
-#define GAMMA_VAL(n)				((n) << 0)
-
-#define ISP_REG_R2Y_0				0xe40
-#define ISP_REG_R2Y_1				0xe44
-#define ISP_REG_R2Y_2				0xe48
-#define ISP_REG_R2Y_3				0xe4c
-#define ISP_REG_R2Y_4				0xe50
-#define ISP_REG_R2Y_5				0xe54
-#define ISP_REG_R2Y_6				0xe58
-#define ISP_REG_R2Y_7				0xe5c
-#define ISP_REG_R2Y_8				0xe60
-
-#define ISP_REG_SHARPEN0			0xe80
-#define ISP_REG_SHARPEN1			0xe84
-#define ISP_REG_SHARPEN2			0xe88
-#define ISP_REG_SHARPEN3			0xe8c
-#define ISP_REG_SHARPEN4			0xe90
-#define ISP_REG_SHARPEN5			0xe94
-#define ISP_REG_SHARPEN6			0xe98
-#define ISP_REG_SHARPEN7			0xe9c
-#define ISP_REG_SHARPEN8			0xea0
-#define ISP_REG_SHARPEN9			0xea4
-#define ISP_REG_SHARPEN10			0xea8
-#define ISP_REG_SHARPEN11			0xeac
-#define ISP_REG_SHARPEN12			0xeb0
-#define ISP_REG_SHARPEN13			0xeb4
-#define ISP_REG_SHARPEN14			0xeb8
-#define S_DELTA(n)				((n) << 16)
-#define S_WEIGHT(n)				((n) << 8)
-
-#define ISP_REG_SHARPEN_FS0			0xebc
-#define ISP_REG_SHARPEN_FS1			0xec0
-#define ISP_REG_SHARPEN_FS2			0xec4
-#define ISP_REG_SHARPEN_FS3			0xec8
-#define ISP_REG_SHARPEN_FS4			0xecc
-#define ISP_REG_SHARPEN_FS5			0xed0
-#define S_FACTOR(n)				((n) << 24)
-#define S_SLOPE(n)				((n) << 0)
-
-#define ISP_REG_SHARPEN_WN			0xed4
-#define PDIRF(n)				((n) << 28)
-#define NDIRF(n)				((n) << 24)
-#define WSUM(n)					((n) << 0)
-
-#define ISP_REG_IUVS1				0xed8
-#define UVDIFF2(n)				((n) << 16)
-#define UVDIFF1(n)				((n) << 0)
-
-#define ISP_REG_IUVS2				0xedc
-#define UVF(n)					((n) << 24)
-#define UVSLOPE(n)				((n) << 0)
-
-#define ISP_REG_IUVCKS1				0xee0
-#define UVCKDIFF2(n)				((n) << 16)
-#define UVCKDIFF1(n)				((n) << 0)
-
-#define ISP_REG_IUVCKS2				0xee4
-
-#define ISP_REG_ISHRPET				0xee8
-#define TH(n)					((n) << 8)
-#define EN(n)					((n) << 0)
-
-#define ISP_REG_YCURVE_0			0xf00
-#define ISP_REG_YCURVE_63			0xffc
-
-#define IMAGE_MAX_WIDTH				1920
-#define IMAGE_MAX_HEIGH				1080
-
-/* pad id for media framework */
-enum stf_isp_pad_id {
-	STF_ISP_PAD_SINK = 0,
-	STF_ISP_PAD_SRC,
-	STF_ISP_PAD_MAX
-};
-
-struct stf_isp_format {
-	u32 code;
-	u8 bpp;
-};
-
-struct stf_isp_format_table {
-	const struct stf_isp_format *fmts;
-	int nfmts;
-};
-
-struct stf_isp_dev {
-	struct stfcamss *stfcamss;
-	struct v4l2_subdev subdev;
-	struct media_pad pads[STF_ISP_PAD_MAX];
-	const struct stf_isp_format_table *formats;
-	unsigned int nformats;
-	struct v4l2_subdev *source_subdev;
-	const struct stf_isp_format *current_fmt;
-};
-
-int stf_isp_reset(struct stf_isp_dev *isp_dev);
-void stf_isp_init_cfg(struct stf_isp_dev *isp_dev);
-void stf_isp_settings(struct stf_isp_dev *isp_dev,
-		      struct v4l2_rect *crop, u32 mcode);
-void stf_isp_stream_set(struct stf_isp_dev *isp_dev);
-int stf_isp_init(struct stfcamss *stfcamss);
-int stf_isp_register(struct stf_isp_dev *isp_dev, struct v4l2_device *v4l2_dev);
-int stf_isp_unregister(struct stf_isp_dev *isp_dev);
-
-#endif /* STF_ISP_H */
diff --git a/drivers/staging/media/starfive/camss/stf-video.c b/drivers/staging/media/starfive/camss/stf-video.c
deleted file mode 100644
index a0420eb6a0aa0..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-video.c
+++ /dev/null
@@ -1,570 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * stf_video.c
- *
- * StarFive Camera Subsystem - V4L2 device node
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#include <linux/pm_runtime.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-event.h>
-#include <media/v4l2-mc.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "stf-camss.h"
-#include "stf-video.h"
-
-/* -----------------------------------------------------------------------------
- * Helper functions
- */
-
-static inline struct stfcamss_buffer *
-to_stfcamss_buffer(struct vb2_v4l2_buffer *vbuf)
-{
-	return container_of(vbuf, struct stfcamss_buffer, vb);
-}
-
-static const struct stfcamss_format_info *
-video_g_fi_by_index(struct stfcamss_video *video, int index)
-{
-	if (index >= video->nformats)
-		return NULL;
-
-	return &video->formats[index];
-}
-
-static const struct stfcamss_format_info *
-video_g_fi_by_mcode(struct stfcamss_video *video, u32 mcode)
-{
-	unsigned int i;
-
-	for (i = 0; i < video->nformats; i++) {
-		if (video->formats[i].code == mcode)
-			return &video->formats[i];
-	}
-
-	return NULL;
-}
-
-static const struct stfcamss_format_info *
-video_g_fi_by_pfmt(struct stfcamss_video *video, u32 pixelformat)
-{
-	unsigned int i;
-
-	for (i = 0; i < video->nformats; i++) {
-		if (video->formats[i].pixelformat == pixelformat)
-			return &video->formats[i];
-	}
-
-	return NULL;
-}
-
-static int __video_try_fmt(struct stfcamss_video *video, struct v4l2_format *f)
-{
-	struct v4l2_pix_format *pix = &f->fmt.pix;
-	const struct stfcamss_format_info *fi;
-	u32 width, height;
-	u32 bpl;
-	unsigned int i;
-
-	fi = video_g_fi_by_pfmt(video, pix->pixelformat);
-	if (!fi)
-		fi = &video->formats[0]; /* default format */
-
-	width = pix->width;
-	height = pix->height;
-
-	memset(pix, 0, sizeof(*pix));
-
-	pix->pixelformat = fi->pixelformat;
-	pix->width = clamp_t(u32, width, STFCAMSS_FRAME_MIN_WIDTH,
-			     STFCAMSS_FRAME_MAX_WIDTH);
-	pix->height = clamp_t(u32, height, STFCAMSS_FRAME_MIN_HEIGHT,
-			      STFCAMSS_FRAME_MAX_HEIGHT);
-	bpl = pix->width * fi->bpp / 8;
-	bpl = ALIGN(bpl, video->bpl_alignment);
-	pix->bytesperline = bpl;
-
-	for (i = 0; i < fi->planes; ++i)
-		pix->sizeimage += bpl * pix->height / fi->vsub[i];
-
-	pix->field = V4L2_FIELD_NONE;
-	pix->colorspace = V4L2_COLORSPACE_SRGB;
-	pix->flags = 0;
-	pix->ycbcr_enc =
-		V4L2_MAP_YCBCR_ENC_DEFAULT(pix->colorspace);
-	pix->quantization = V4L2_MAP_QUANTIZATION_DEFAULT(true,
-							  pix->colorspace,
-							  pix->ycbcr_enc);
-	pix->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix->colorspace);
-
-	return 0;
-}
-
-static int stf_video_init_format(struct stfcamss_video *video)
-{
-	int ret;
-	struct v4l2_format format = {
-		.type = video->type,
-		.fmt.pix = {
-			.width = 1920,
-			.height = 1080,
-			.pixelformat = V4L2_PIX_FMT_NV12,
-		},
-	};
-
-	ret = __video_try_fmt(video, &format);
-
-	if (ret < 0)
-		return ret;
-
-	video->active_fmt = format;
-
-	return 0;
-}
-
-/* -----------------------------------------------------------------------------
- * Video queue operations
- */
-
-static int video_queue_setup(struct vb2_queue *q,
-			     unsigned int *num_buffers,
-			     unsigned int *num_planes,
-			     unsigned int sizes[],
-			     struct device *alloc_devs[])
-{
-	struct stfcamss_video *video = vb2_get_drv_priv(q);
-	const struct v4l2_pix_format *format = &video->active_fmt.fmt.pix;
-
-	if (*num_planes) {
-		if (*num_planes != 1)
-			return -EINVAL;
-
-		if (sizes[0] < format->sizeimage)
-			return -EINVAL;
-	} else {
-		*num_planes = 1;
-		sizes[0] = format->sizeimage;
-	}
-
-	if (!sizes[0]) {
-		dev_dbg(video->stfcamss->dev,
-			"%s: error size is zero.\n", __func__);
-		return -EINVAL;
-	}
-
-	dev_dbg(video->stfcamss->dev, "planes = %d, size = %d\n",
-		*num_planes, sizes[0]);
-
-	return 0;
-}
-
-static int video_buf_init(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct stfcamss_video *video = vb2_get_drv_priv(vb->vb2_queue);
-	struct stfcamss_buffer *buffer = to_stfcamss_buffer(vbuf);
-	const struct v4l2_pix_format *fmt = &video->active_fmt.fmt.pix;
-	dma_addr_t *paddr;
-
-	paddr = vb2_plane_cookie(vb, 0);
-	buffer->addr[0] = *paddr;
-
-	if (fmt->pixelformat == V4L2_PIX_FMT_NV12)
-		buffer->addr[1] =
-			buffer->addr[0] + fmt->bytesperline * fmt->height;
-
-	return 0;
-}
-
-static int video_buf_prepare(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct stfcamss_video *video = vb2_get_drv_priv(vb->vb2_queue);
-	const struct v4l2_pix_format *fmt = &video->active_fmt.fmt.pix;
-
-	if (fmt->sizeimage > vb2_plane_size(vb, 0)) {
-		dev_dbg(video->stfcamss->dev,
-			"sizeimage = %u, plane size = %u\n",
-			fmt->sizeimage, (unsigned int)vb2_plane_size(vb, 0));
-		return -EINVAL;
-	}
-	vb2_set_plane_payload(vb, 0, fmt->sizeimage);
-
-	vbuf->field = V4L2_FIELD_NONE;
-
-	return 0;
-}
-
-static void video_buf_queue(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct stfcamss_video *video = vb2_get_drv_priv(vb->vb2_queue);
-	struct stfcamss_buffer *buffer = to_stfcamss_buffer(vbuf);
-
-	video->ops->queue_buffer(video, buffer);
-}
-
-static int video_get_subdev_format(struct stfcamss_video *video,
-				   struct v4l2_subdev_format *fmt)
-{
-	struct v4l2_subdev *subdev;
-	struct media_pad *pad;
-	struct media_entity *entity;
-	int ret;
-
-	entity = &video->vdev.entity;
-	while (1) {
-		pad = &entity->pads[0];
-		if (!(pad->flags & MEDIA_PAD_FL_SINK))
-			break;
-
-		pad = media_pad_remote_pad_first(pad);
-		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
-			break;
-
-		entity = pad->entity;
-		subdev = media_entity_to_v4l2_subdev(entity);
-
-		fmt->pad = pad->index;
-
-		ret = v4l2_subdev_call_state_active(subdev, pad, get_fmt, fmt);
-		if (ret < 0 && ret != -ENOIOCTLCMD)
-			return ret;
-		else if (!ret)
-			break;
-	}
-
-	return 0;
-}
-
-static int stf_video_check_format(struct stfcamss_video *video)
-{
-	struct v4l2_pix_format *pix = &video->active_fmt.fmt.pix;
-	const struct stfcamss_format_info *fi;
-	int ret;
-	struct v4l2_subdev_format sd_fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-
-	ret = video_get_subdev_format(video, &sd_fmt);
-	if (ret < 0)
-		return ret;
-
-	fi = video_g_fi_by_mcode(video, sd_fmt.format.code);
-	if (!fi)
-		return -EINVAL;
-
-	if (pix->pixelformat != fi->pixelformat ||
-	    pix->height != sd_fmt.format.height ||
-	    pix->width != sd_fmt.format.width ||
-	    pix->field != sd_fmt.format.field)
-		return -EPIPE;
-
-	return 0;
-}
-
-static int video_start_streaming(struct vb2_queue *q, unsigned int count)
-{
-	struct stfcamss_video *video = vb2_get_drv_priv(q);
-	struct video_device *vdev = &video->vdev;
-	int ret;
-
-	ret = video_device_pipeline_start(vdev, &video->stfcamss->pipe);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev,
-			"Failed to media_pipeline_start: %d\n", ret);
-		goto err_ret_buffers;
-	}
-
-	ret = pm_runtime_resume_and_get(video->stfcamss->dev);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev, "power up failed %d\n", ret);
-		goto err_pipeline_stop;
-	}
-
-	video->ops->start_streaming(video);
-
-	ret = v4l2_subdev_call(video->source_subdev, video, s_stream, true);
-	if (ret) {
-		dev_err(video->stfcamss->dev, "stream on failed\n");
-		goto err_pm_put;
-	}
-
-	return 0;
-
-err_pm_put:
-	pm_runtime_put(video->stfcamss->dev);
-err_pipeline_stop:
-	video_device_pipeline_stop(vdev);
-err_ret_buffers:
-	video->ops->flush_buffers(video, VB2_BUF_STATE_QUEUED);
-	return ret;
-}
-
-static void video_stop_streaming(struct vb2_queue *q)
-{
-	struct stfcamss_video *video = vb2_get_drv_priv(q);
-	struct video_device *vdev = &video->vdev;
-
-	video->ops->stop_streaming(video);
-
-	v4l2_subdev_call(video->source_subdev, video, s_stream, false);
-
-	pm_runtime_put(video->stfcamss->dev);
-
-	video_device_pipeline_stop(vdev);
-	video->ops->flush_buffers(video, VB2_BUF_STATE_ERROR);
-}
-
-static const struct vb2_ops stf_video_vb2_q_ops = {
-	.queue_setup     = video_queue_setup,
-	.buf_init        = video_buf_init,
-	.buf_prepare     = video_buf_prepare,
-	.buf_queue       = video_buf_queue,
-	.start_streaming = video_start_streaming,
-	.stop_streaming  = video_stop_streaming,
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 ioctls
- */
-
-static int video_querycap(struct file *file, void *fh,
-			  struct v4l2_capability *cap)
-{
-	strscpy(cap->driver, "starfive-camss", sizeof(cap->driver));
-	strscpy(cap->card, "Starfive Camera Subsystem", sizeof(cap->card));
-
-	return 0;
-}
-
-static int video_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
-{
-	struct stfcamss_video *video = video_drvdata(file);
-	const struct stfcamss_format_info *fi;
-
-	if (f->index >= video->nformats)
-		return -EINVAL;
-
-	if (f->mbus_code) {
-		/* Each entry in formats[] table has unique mbus_code */
-		if (f->index > 0)
-			return -EINVAL;
-
-		fi = video_g_fi_by_mcode(video, f->mbus_code);
-	} else {
-		fi = video_g_fi_by_index(video, f->index);
-	}
-
-	if (!fi)
-		return -EINVAL;
-
-	f->pixelformat = fi->pixelformat;
-
-	return 0;
-}
-
-static int video_enum_framesizes(struct file *file, void *fh,
-				 struct v4l2_frmsizeenum *fsize)
-{
-	struct stfcamss_video *video = video_drvdata(file);
-	unsigned int i;
-
-	if (fsize->index)
-		return -EINVAL;
-
-	for (i = 0; i < video->nformats; i++) {
-		if (video->formats[i].pixelformat == fsize->pixel_format)
-			break;
-	}
-
-	if (i == video->nformats)
-		return -EINVAL;
-
-	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
-	fsize->stepwise.min_width = STFCAMSS_FRAME_MIN_WIDTH;
-	fsize->stepwise.max_width = STFCAMSS_FRAME_MAX_WIDTH;
-	fsize->stepwise.min_height = STFCAMSS_FRAME_MIN_HEIGHT;
-	fsize->stepwise.max_height = STFCAMSS_FRAME_MAX_HEIGHT;
-	fsize->stepwise.step_width = 1;
-	fsize->stepwise.step_height = 1;
-
-	return 0;
-}
-
-static int video_g_fmt(struct file *file, void *fh, struct v4l2_format *f)
-{
-	struct stfcamss_video *video = video_drvdata(file);
-
-	*f = video->active_fmt;
-
-	return 0;
-}
-
-static int video_s_fmt(struct file *file, void *fh, struct v4l2_format *f)
-{
-	struct stfcamss_video *video = video_drvdata(file);
-	int ret;
-
-	if (vb2_is_busy(&video->vb2_q))
-		return -EBUSY;
-
-	ret = __video_try_fmt(video, f);
-	if (ret < 0)
-		return ret;
-
-	video->active_fmt = *f;
-
-	return 0;
-}
-
-static int video_try_fmt(struct file *file, void *fh, struct v4l2_format *f)
-{
-	struct stfcamss_video *video = video_drvdata(file);
-
-	return __video_try_fmt(video, f);
-}
-
-static const struct v4l2_ioctl_ops stf_vid_ioctl_ops = {
-	.vidioc_querycap                = video_querycap,
-	.vidioc_enum_fmt_vid_cap        = video_enum_fmt,
-	.vidioc_enum_framesizes         = video_enum_framesizes,
-	.vidioc_g_fmt_vid_cap           = video_g_fmt,
-	.vidioc_s_fmt_vid_cap           = video_s_fmt,
-	.vidioc_try_fmt_vid_cap         = video_try_fmt,
-	.vidioc_reqbufs                 = vb2_ioctl_reqbufs,
-	.vidioc_querybuf                = vb2_ioctl_querybuf,
-	.vidioc_qbuf                    = vb2_ioctl_qbuf,
-	.vidioc_expbuf                  = vb2_ioctl_expbuf,
-	.vidioc_dqbuf                   = vb2_ioctl_dqbuf,
-	.vidioc_create_bufs             = vb2_ioctl_create_bufs,
-	.vidioc_prepare_buf             = vb2_ioctl_prepare_buf,
-	.vidioc_streamon                = vb2_ioctl_streamon,
-	.vidioc_streamoff               = vb2_ioctl_streamoff,
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 file operations
- */
-
-static const struct v4l2_file_operations stf_vid_fops = {
-	.owner          = THIS_MODULE,
-	.unlocked_ioctl = video_ioctl2,
-	.open           = v4l2_fh_open,
-	.release        = vb2_fop_release,
-	.poll           = vb2_fop_poll,
-	.mmap           = vb2_fop_mmap,
-	.read           = vb2_fop_read,
-};
-
-/* -----------------------------------------------------------------------------
- * STFCAMSS video core
- */
-
-static int stf_link_validate(struct media_link *link)
-{
-	struct video_device *vdev =
-		media_entity_to_video_device(link->sink->entity);
-	struct stfcamss_video *video = video_get_drvdata(vdev);
-	int ret;
-
-	ret = stf_video_check_format(video);
-
-	return ret;
-}
-
-static const struct media_entity_operations stf_media_ops = {
-	.link_validate = stf_link_validate,
-};
-
-static void stf_video_release(struct video_device *vdev)
-{
-	struct stfcamss_video *video = video_get_drvdata(vdev);
-
-	media_entity_cleanup(&vdev->entity);
-
-	mutex_destroy(&video->q_lock);
-	mutex_destroy(&video->lock);
-}
-
-int stf_video_register(struct stfcamss_video *video,
-		       struct v4l2_device *v4l2_dev, const char *name)
-{
-	struct video_device *vdev = &video->vdev;
-	struct vb2_queue *q;
-	struct media_pad *pad = &video->pad;
-	int ret;
-
-	mutex_init(&video->q_lock);
-	mutex_init(&video->lock);
-
-	q = &video->vb2_q;
-	q->drv_priv = video;
-	q->mem_ops = &vb2_dma_contig_memops;
-	q->ops = &stf_video_vb2_q_ops;
-	q->type = video->type;
-	q->io_modes = VB2_DMABUF | VB2_MMAP;
-	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-	q->buf_struct_size = sizeof(struct stfcamss_buffer);
-	q->dev = video->stfcamss->dev;
-	q->lock = &video->q_lock;
-	q->min_queued_buffers = STFCAMSS_MIN_BUFFERS;
-	ret = vb2_queue_init(q);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev,
-			"Failed to init vb2 queue: %d\n", ret);
-		goto err_mutex_destroy;
-	}
-
-	pad->flags = MEDIA_PAD_FL_SINK;
-	ret = media_entity_pads_init(&vdev->entity, 1, pad);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev,
-			"Failed to init video entity: %d\n", ret);
-		goto err_mutex_destroy;
-	}
-
-	ret = stf_video_init_format(video);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev,
-			"Failed to init format: %d\n", ret);
-		goto err_media_cleanup;
-	}
-
-	vdev->fops = &stf_vid_fops;
-	vdev->ioctl_ops = &stf_vid_ioctl_ops;
-	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
-	vdev->entity.ops = &stf_media_ops;
-	vdev->vfl_dir = VFL_DIR_RX;
-	vdev->release = stf_video_release;
-	vdev->v4l2_dev = v4l2_dev;
-	vdev->queue = &video->vb2_q;
-	vdev->lock = &video->lock;
-	strscpy(vdev->name, name, sizeof(vdev->name));
-
-	video_set_drvdata(vdev, video);
-
-	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
-	if (ret < 0) {
-		dev_err(video->stfcamss->dev,
-			"Failed to register video device: %d\n", ret);
-		goto err_media_cleanup;
-	}
-
-	return 0;
-
-err_media_cleanup:
-	media_entity_cleanup(&vdev->entity);
-err_mutex_destroy:
-	mutex_destroy(&video->lock);
-	mutex_destroy(&video->q_lock);
-	return ret;
-}
-
-void stf_video_unregister(struct stfcamss_video *video)
-{
-	vb2_video_unregister_device(&video->vdev);
-}
diff --git a/drivers/staging/media/starfive/camss/stf-video.h b/drivers/staging/media/starfive/camss/stf-video.h
deleted file mode 100644
index 8052b77e3ad83..0000000000000
--- a/drivers/staging/media/starfive/camss/stf-video.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * stf_video.h
- *
- * StarFive Camera Subsystem - V4L2 device node
- *
- * Copyright (C) 2021-2023 StarFive Technology Co., Ltd.
- */
-
-#ifndef STF_VIDEO_H
-#define STF_VIDEO_H
-
-#include <linux/list.h>
-#include <linux/mutex.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-dev.h>
-#include <media/v4l2-fh.h>
-#include <media/v4l2-ioctl.h>
-#include <media/videobuf2-v4l2.h>
-
-#define STFCAMSS_FRAME_MIN_WIDTH		64
-#define STFCAMSS_FRAME_MAX_WIDTH		1920
-#define STFCAMSS_FRAME_MIN_HEIGHT		64
-#define STFCAMSS_FRAME_MAX_HEIGHT		1080
-#define STFCAMSS_FRAME_WIDTH_ALIGN_8		8
-#define STFCAMSS_FRAME_WIDTH_ALIGN_128		128
-#define STFCAMSS_MIN_BUFFERS			2
-
-#define STFCAMSS_MAX_ENTITY_NAME_LEN		27
-
-enum stf_v_line_id {
-	STF_V_LINE_WR = 0,
-	STF_V_LINE_ISP,
-	STF_V_LINE_MAX,
-};
-
-enum stf_capture_type {
-	STF_CAPTURE_RAW = 0,
-	STF_CAPTURE_YUV,
-	STF_CAPTURE_NUM,
-};
-
-struct stfcamss_buffer {
-	struct vb2_v4l2_buffer vb;
-	dma_addr_t addr[2];
-	struct list_head queue;
-};
-
-struct fract {
-	u8 numerator;
-	u8 denominator;
-};
-
-/*
- * struct stfcamss_format_info - ISP media bus format information
- * @code: V4L2 media bus format code
- * @pixelformat: V4L2 pixel format FCC identifier
- * @planes: Number of planes
- * @vsub: Vertical subsampling (for each plane)
- * @bpp: Bits per pixel when stored in memory (for each plane)
- */
-struct stfcamss_format_info {
-	u32 code;
-	u32 pixelformat;
-	u8 planes;
-	u8 vsub[3];
-	u8 bpp;
-};
-
-struct stfcamss_video {
-	struct stfcamss *stfcamss;
-	struct vb2_queue vb2_q;
-	struct video_device vdev;
-	struct media_pad pad;
-	struct v4l2_format active_fmt;
-	enum v4l2_buf_type type;
-	const struct stfcamss_video_ops *ops;
-	struct mutex lock;	 /* serialize device access */
-	struct mutex q_lock;	 /* protects the queue */
-	unsigned int bpl_alignment;
-	const struct stfcamss_format_info *formats;
-	unsigned int nformats;
-	struct v4l2_subdev *source_subdev;
-};
-
-struct stfcamss_video_ops {
-	int (*queue_buffer)(struct stfcamss_video *video,
-			    struct stfcamss_buffer *buf);
-	int (*flush_buffers)(struct stfcamss_video *video,
-			     enum vb2_buffer_state state);
-	void (*start_streaming)(struct stfcamss_video *video);
-	void (*stop_streaming)(struct stfcamss_video *video);
-};
-
-int stf_video_register(struct stfcamss_video *video,
-		       struct v4l2_device *v4l2_dev, const char *name);
-
-void stf_video_unregister(struct stfcamss_video *video);
-
-#endif /* STF_VIDEO_H */
diff --git a/drivers/staging/media/tegra-video/Makefile b/drivers/staging/media/tegra-video/Makefile
index 6c7552e05109e..96380b5dbd8bb 100644
--- a/drivers/staging/media/tegra-video/Makefile
+++ b/drivers/staging/media/tegra-video/Makefile
@@ -6,5 +6,6 @@ tegra-video-objs := \
 		csi.o
 
 tegra-video-$(CONFIG_ARCH_TEGRA_2x_SOC)  += tegra20.o
+tegra-video-$(CONFIG_ARCH_TEGRA_3x_SOC)  += tegra20.o
 tegra-video-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o
 obj-$(CONFIG_VIDEO_TEGRA) += tegra-video.o
diff --git a/drivers/staging/media/tegra-video/csi.c b/drivers/staging/media/tegra-video/csi.c
index 7842104ca933c..41d57dac61a71 100644
--- a/drivers/staging/media/tegra-video/csi.c
+++ b/drivers/staging/media/tegra-video/csi.c
@@ -12,6 +12,7 @@
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/tegra-mipi-cal.h>
 
 #include <media/v4l2-fwnode.h>
 
@@ -20,17 +21,6 @@
 
 #define MHZ			1000000
 
-static inline struct tegra_csi *
-host1x_client_to_csi(struct host1x_client *client)
-{
-	return container_of(client, struct tegra_csi, client);
-}
-
-static inline struct tegra_csi_channel *to_csi_chan(struct v4l2_subdev *subdev)
-{
-	return container_of(subdev, struct tegra_csi_channel, subdev);
-}
-
 /*
  * CSI is a separate subdevice which has 6 source pads to generate
  * test pattern. CSI subdevice pad ops are used only for TPG and
@@ -455,6 +445,22 @@ static const struct v4l2_subdev_ops tegra_csi_ops = {
 	.pad    = &tegra_csi_pad_ops,
 };
 
+struct v4l2_subdev *tegra_channel_get_remote_csi_subdev(struct tegra_vi_channel *chan)
+{
+	struct media_pad *pad;
+	struct v4l2_subdev *subdev;
+
+	pad = media_pad_remote_pad_first(&chan->pad);
+	if (!pad)
+		return NULL;
+
+	subdev = media_entity_to_v4l2_subdev(pad->entity);
+	if (!subdev)
+		return NULL;
+
+	return subdev->ops == &tegra_csi_ops ? subdev : NULL;
+}
+
 static int tegra_csi_channel_alloc(struct tegra_csi *csi,
 				   struct device_node *node,
 				   unsigned int port_num, unsigned int lanes,
@@ -704,6 +710,8 @@ static int __maybe_unused csi_runtime_suspend(struct device *dev)
 
 	clk_bulk_disable_unprepare(csi->soc->num_clks, csi->clks);
 
+	regulator_disable(csi->vdd);
+
 	return 0;
 }
 
@@ -712,13 +720,23 @@ static int __maybe_unused csi_runtime_resume(struct device *dev)
 	struct tegra_csi *csi = dev_get_drvdata(dev);
 	int ret;
 
+	ret = regulator_enable(csi->vdd);
+	if (ret) {
+		dev_err(dev, "failed to enable VDD supply: %d\n", ret);
+		return ret;
+	}
+
 	ret = clk_bulk_prepare_enable(csi->soc->num_clks, csi->clks);
 	if (ret < 0) {
 		dev_err(csi->dev, "failed to enable clocks: %d\n", ret);
-		return ret;
+		goto disable_vdd;
 	}
 
 	return 0;
+
+disable_vdd:
+	regulator_disable(csi->vdd);
+	return ret;
 }
 
 static int tegra_csi_init(struct host1x_client *client)
@@ -796,6 +814,11 @@ static int tegra_csi_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	csi->vdd = devm_regulator_get(&pdev->dev, "avdd-dsi-csi");
+	if (IS_ERR(csi->vdd))
+		return dev_err_probe(&pdev->dev, PTR_ERR(csi->vdd),
+				     "failed to get VDD supply");
+
 	if (!pdev->dev.pm_domain) {
 		ret = -ENOENT;
 		dev_warn(&pdev->dev, "PM domain is not attached: %d\n", ret);
@@ -804,6 +827,17 @@ static int tegra_csi_probe(struct platform_device *pdev)
 
 	csi->dev = &pdev->dev;
 	csi->ops = csi->soc->ops;
+
+	if (csi->soc->mipi_ops) {
+		ret = devm_tegra_mipi_add_provider(&pdev->dev, pdev->dev.of_node,
+						   csi->soc->mipi_ops);
+		if (ret)
+			return dev_err_probe(&pdev->dev, ret,
+					     "failed to add MIPI calibration operations\n");
+	}
+
+	mutex_init(&csi->mipi_lock);
+
 	platform_set_drvdata(pdev, csi);
 	pm_runtime_enable(&pdev->dev);
 
@@ -836,6 +870,12 @@ static void tegra_csi_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id tegra_csi_of_id_table[] = {
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+	{ .compatible = "nvidia,tegra20-csi", .data = &tegra20_csi_soc },
+#endif
+#if defined(CONFIG_ARCH_TEGRA_3x_SOC)
+	{ .compatible = "nvidia,tegra30-csi", .data = &tegra30_csi_soc },
+#endif
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
 	{ .compatible = "nvidia,tegra210-csi", .data = &tegra210_csi_soc },
 #endif
diff --git a/drivers/staging/media/tegra-video/csi.h b/drivers/staging/media/tegra-video/csi.h
index 609c5952e050d..f0277cb424469 100644
--- a/drivers/staging/media/tegra-video/csi.h
+++ b/drivers/staging/media/tegra-video/csi.h
@@ -115,6 +115,7 @@ struct tegra_csi_ops {
  * struct tegra_csi_soc - NVIDIA Tegra CSI SoC structure
  *
  * @ops: csi hardware operations
+ * @mipi_ops: MIPI calibration operations
  * @csi_max_channels: supported max streaming channels
  * @clk_names: csi and cil clock names
  * @num_clks: total clocks count
@@ -123,6 +124,7 @@ struct tegra_csi_ops {
  */
 struct tegra_csi_soc {
 	const struct tegra_csi_ops *ops;
+	const struct tegra_mipi_ops *mipi_ops;
 	unsigned int csi_max_channels;
 	const char * const *clk_names;
 	unsigned int num_clks;
@@ -130,6 +132,12 @@ struct tegra_csi_soc {
 	unsigned int tpg_frmrate_table_size;
 };
 
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+extern const struct tegra_csi_soc tegra20_csi_soc;
+#endif
+#if defined(CONFIG_ARCH_TEGRA_3x_SOC)
+extern const struct tegra_csi_soc tegra30_csi_soc;
+#endif
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
 extern const struct tegra_csi_soc tegra210_csi_soc;
 #endif
@@ -141,8 +149,10 @@ extern const struct tegra_csi_soc tegra210_csi_soc;
  * @client: host1x_client struct
  * @iomem: register base
  * @clks: clock for CSI and CIL
+ * @vdd: vdd regulator for CSI hardware, usually avdd_dsi_csi
  * @soc: pointer to SoC data structure
  * @ops: csi operations
+ * @mipi_lock: for MIPI calibration operations
  * @csi_chans: list head for CSI channels
  */
 struct tegra_csi {
@@ -150,11 +160,23 @@ struct tegra_csi {
 	struct host1x_client client;
 	void __iomem *iomem;
 	struct clk_bulk_data *clks;
+	struct regulator *vdd;
 	const struct tegra_csi_soc *soc;
 	const struct tegra_csi_ops *ops;
+	struct mutex mipi_lock; /* for register access */
 	struct list_head csi_chans;
 };
 
+static inline struct tegra_csi *host1x_client_to_csi(struct host1x_client *client)
+{
+	return container_of(client, struct tegra_csi, client);
+}
+
+static inline struct tegra_csi_channel *to_csi_chan(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct tegra_csi_channel, subdev);
+}
+
 void tegra_csi_error_recover(struct v4l2_subdev *subdev);
 void tegra_csi_calc_settle_time(struct tegra_csi_channel *csi_chan,
 				u8 csi_port_num,
diff --git a/drivers/staging/media/tegra-video/tegra20.c b/drivers/staging/media/tegra-video/tegra20.c
index aa9ff7fec4f9c..e513e6ccb776f 100644
--- a/drivers/staging/media/tegra-video/tegra20.c
+++ b/drivers/staging/media/tegra-video/tegra20.c
@@ -4,6 +4,9 @@
  *
  * Copyright (C) 2023 SKIDATA GmbH
  * Author: Luca Ceresoli <luca.ceresoli@bootlin.com>
+ *
+ * Copyright (c) 2025 Svyatoslav Ryhel <clamor95@gmail.com>
+ * Copyright (c) 2025 Jonas Schwöbel <jonasschwoebel@yahoo.de>
  */
 
 /*
@@ -12,10 +15,15 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/clk/tegra.h>
 #include <linux/delay.h>
 #include <linux/host1x.h>
+#include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/kthread.h>
+#include <linux/pm_runtime.h>
+#include <linux/tegra-mipi-cal.h>
 #include <linux/v4l2-mediabus.h>
 
 #include "vip.h"
@@ -23,19 +31,27 @@
 
 #define TEGRA_VI_SYNCPT_WAIT_TIMEOUT			msecs_to_jiffies(200)
 
-/* This are just good-sense numbers. The actual min/max is not documented. */
 #define TEGRA20_MIN_WIDTH	32U
+#define TEGRA20_MAX_WIDTH	8190U
 #define TEGRA20_MIN_HEIGHT	32U
-#define TEGRA20_MAX_WIDTH	2048U
-#define TEGRA20_MAX_HEIGHT	2048U
+#define TEGRA20_MAX_HEIGHT	8190U
+
+/* Tegra20/Tegra30 has 2 outputs in VI */
+enum tegra_vi_out {
+	TEGRA_VI_OUT_1 = 0,
+	TEGRA_VI_OUT_2 = 1,
+};
 
 /* --------------------------------------------------------------------------
  * Registers
  */
 
-#define TEGRA_VI_CONT_SYNCPT_OUT_1			0x0060
-#define       VI_CONT_SYNCPT_OUT_1_CONTINUOUS_SYNCPT	BIT(8)
-#define       VI_CONT_SYNCPT_OUT_1_SYNCPT_IDX_SFT	0
+#define TEGRA_VI_CONT_SYNCPT_OUT(n)			(0x0060 + (n) * 4)
+#define       VI_CONT_SYNCPT_OUT_CONTINUOUS_SYNCPT	BIT(8)
+#define       VI_CONT_SYNCPT_OUT_SYNCPT_IDX_SFT		0
+
+#define TEGRA_VI_CONT_SYNCPT_CSI_PP_FRAME_START(n)	(0x0070 + (n) * 8)
+#define TEGRA_VI_CONT_SYNCPT_CSI_PP_FRAME_END(n)	(0x0074 + (n) * 8)
 
 #define TEGRA_VI_VI_INPUT_CONTROL			0x0088
 #define       VI_INPUT_FIELD_DETECT			BIT(27)
@@ -47,6 +63,7 @@
 #define       VI_INPUT_YUV_INPUT_FORMAT_YVYU		(3 << VI_INPUT_YUV_INPUT_FORMAT_SFT)
 #define       VI_INPUT_INPUT_FORMAT_SFT			2  /* bits [5:2] */
 #define       VI_INPUT_INPUT_FORMAT_YUV422		(0 << VI_INPUT_INPUT_FORMAT_SFT)
+#define       VI_INPUT_INPUT_FORMAT_BAYER		(2 << VI_INPUT_INPUT_FORMAT_SFT)
 #define       VI_INPUT_VIP_INPUT_ENABLE			BIT(1)
 
 #define TEGRA_VI_VI_CORE_CONTROL			0x008c
@@ -67,7 +84,7 @@
 #define       VI_VI_CORE_CONTROL_OUTPUT_TO_EPP_SFT	2
 #define       VI_VI_CORE_CONTROL_OUTPUT_TO_ISP_SFT	0
 
-#define TEGRA_VI_VI_FIRST_OUTPUT_CONTROL		0x0090
+#define TEGRA_VI_VI_OUTPUT_CONTROL(n)			(0x0090 + (n) * 4)
 #define       VI_OUTPUT_FORMAT_EXT			BIT(22)
 #define       VI_OUTPUT_V_DIRECTION			BIT(20)
 #define       VI_OUTPUT_H_DIRECTION			BIT(19)
@@ -81,6 +98,10 @@
 #define       VI_OUTPUT_OUTPUT_FORMAT_SFT		0
 #define       VI_OUTPUT_OUTPUT_FORMAT_YUV422POST	(3 << VI_OUTPUT_OUTPUT_FORMAT_SFT)
 #define       VI_OUTPUT_OUTPUT_FORMAT_YUV420PLANAR	(6 << VI_OUTPUT_OUTPUT_FORMAT_SFT)
+/* TEGRA_VI_OUT_2 supported formats */
+#define       VI_OUTPUT_OUTPUT_FORMAT_CSI_PPA_BAYER	(7 << VI_OUTPUT_OUTPUT_FORMAT_SFT)
+#define       VI_OUTPUT_OUTPUT_FORMAT_CSI_PPB_BAYER	(8 << VI_OUTPUT_OUTPUT_FORMAT_SFT)
+#define       VI_OUTPUT_OUTPUT_FORMAT_VIP_BAYER_DIRECT	(9 << VI_OUTPUT_OUTPUT_FORMAT_SFT)
 
 #define TEGRA_VI_VIP_H_ACTIVE				0x00a4
 #define       VI_VIP_H_ACTIVE_PERIOD_SFT		16 /* active pixels/line, must be even */
@@ -90,26 +111,26 @@
 #define       VI_VIP_V_ACTIVE_PERIOD_SFT		16 /* active lines */
 #define       VI_VIP_V_ACTIVE_START_SFT			0
 
-#define TEGRA_VI_VB0_START_ADDRESS_FIRST		0x00c4
-#define TEGRA_VI_VB0_BASE_ADDRESS_FIRST			0x00c8
+#define TEGRA_VI_VB0_START_ADDRESS(n)			(0x00c4 + (n) * 44)
+#define TEGRA_VI_VB0_BASE_ADDRESS(n)			(0x00c8 + (n) * 44)
 #define TEGRA_VI_VB0_START_ADDRESS_U			0x00cc
 #define TEGRA_VI_VB0_BASE_ADDRESS_U			0x00d0
 #define TEGRA_VI_VB0_START_ADDRESS_V			0x00d4
 #define TEGRA_VI_VB0_BASE_ADDRESS_V			0x00d8
 
-#define TEGRA_VI_FIRST_OUTPUT_FRAME_SIZE		0x00e0
-#define       VI_FIRST_OUTPUT_FRAME_HEIGHT_SFT		16
-#define       VI_FIRST_OUTPUT_FRAME_WIDTH_SFT		0
+#define TEGRA_VI_OUTPUT_FRAME_SIZE(n)			(0x00e0 + (n) * 24)
+#define       VI_OUTPUT_FRAME_HEIGHT_SFT		16
+#define       VI_OUTPUT_FRAME_WIDTH_SFT			0
 
-#define TEGRA_VI_VB0_COUNT_FIRST			0x00e4
+#define TEGRA_VI_VB0_COUNT(n)				(0x00e4 + (n) * 24)
 
-#define TEGRA_VI_VB0_SIZE_FIRST				0x00e8
-#define       VI_VB0_SIZE_FIRST_V_SFT			16
-#define       VI_VB0_SIZE_FIRST_H_SFT			0
+#define TEGRA_VI_VB0_SIZE(n)				(0x00e8 + (n) * 24)
+#define       VI_VB0_SIZE_V_SFT				16
+#define       VI_VB0_SIZE_H_SFT				0
 
-#define TEGRA_VI_VB0_BUFFER_STRIDE_FIRST		0x00ec
-#define       VI_VB0_BUFFER_STRIDE_FIRST_CHROMA_SFT	30
-#define       VI_VB0_BUFFER_STRIDE_FIRST_LUMA_SFT	0
+#define TEGRA_VI_VB0_BUFFER_STRIDE(n)			(0x00ec + (n) * 24)
+#define       VI_VB0_BUFFER_STRIDE_CHROMA_SFT		30
+#define       VI_VB0_BUFFER_STRIDE_LUMA_SFT		0
 
 #define TEGRA_VI_H_LPF_CONTROL				0x0108
 #define       VI_H_LPF_CONTROL_CHROMA_SFT		16
@@ -137,15 +158,113 @@
 #define       VI_CAMERA_CONTROL_TEST_MODE		BIT(1)
 #define       VI_CAMERA_CONTROL_VIP_ENABLE		BIT(0)
 
-#define TEGRA_VI_VI_ENABLE				0x01a4
+#define TEGRA_VI_VI_ENABLE(n)				(0x01a4 + (n) * 4)
 #define       VI_VI_ENABLE_SW_FLOW_CONTROL_OUT1		BIT(1)
 #define       VI_VI_ENABLE_FIRST_OUTPUT_TO_MEM_DISABLE	BIT(0)
 
 #define TEGRA_VI_VI_RAISE				0x01ac
 #define       VI_VI_RAISE_ON_EDGE			BIT(0)
 
+#define TEGRA_VI_CSI_PP_RAISE_FRAME_START(n)		(0x01d8 + (n) * 8)
+#define TEGRA_VI_CSI_PP_RAISE_FRAME_END(n)		(0x01dc + (n) * 8)
+#define TEGRA_VI_CSI_PP_H_ACTIVE(n)			(0x01e8 + (n) * 8)
+#define TEGRA_VI_CSI_PP_V_ACTIVE(n)			(0x01ec + (n) * 8)
+
+/* Tegra20 CSI registers: Starts from 0x800, offset 0x0 */
+#define TEGRA_CSI_VI_INPUT_STREAM_CONTROL		0x0000
+#define TEGRA_CSI_HOST_INPUT_STREAM_CONTROL		0x0008
+#define TEGRA_CSI_INPUT_STREAM_CONTROL(n)		(0x0010 + (n) * 0x2c)
+#define       CSI_SKIP_PACKET_THRESHOLD(n)		(((n) & 0xff) << 16)
+#define TEGRA_CSI_PIXEL_STREAM_CONTROL0(n)		(0x0018 + (n) * 0x2c)
+#define       CSI_PP_PAD_FRAME_PAD0S			(0 << 28)
+#define       CSI_PP_PAD_FRAME_PAD1S			(1 << 28)
+#define       CSI_PP_PAD_FRAME_NOPAD			(2 << 28)
+#define       CSI_PP_HEADER_EC_ENABLE			BIT(27)
+#define       CSI_PP_PAD_SHORT_LINE_PAD0S		(0 << 24)
+#define       CSI_PP_PAD_SHORT_LINE_PAD1S		(1 << 24)
+#define       CSI_PP_PAD_SHORT_LINE_NOPAD		(2 << 24)
+#define       CSI_PP_EMBEDDED_DATA_EMBEDDED		BIT(20)
+#define       CSI_PP_OUTPUT_FORMAT_ARBITRARY		(0 << 16)
+#define       CSI_PP_OUTPUT_FORMAT_PIXEL		(1 << 16)
+#define       CSI_PP_OUTPUT_FORMAT_PIXEL_REP		(2 << 16)
+#define       CSI_PP_OUTPUT_FORMAT_STORE		(3 << 16)
+#define       CSI_PP_VIRTUAL_CHANNEL_ID(n)		(((n) - 1) << 14)
+#define       CSI_PP_DATA_TYPE(n)			((n) << 8)
+#define       CSI_PP_CRC_CHECK_ENABLE			BIT(7)
+#define       CSI_PP_WORD_COUNT_HEADER			BIT(6)
+#define       CSI_PP_DATA_IDENTIFIER_ENABLE		BIT(5)
+#define       CSI_PP_PACKET_HEADER_SENT			BIT(4)
+#define TEGRA_CSI_PIXEL_STREAM_CONTROL1(n)		(0x001c + (n) * 0x2c)
+#define TEGRA_CSI_PIXEL_STREAM_WORD_COUNT(n)		(0x0020 + (n) * 0x2c)
+#define TEGRA_CSI_PIXEL_STREAM_GAP(n)			(0x0024 + (n) * 0x2c)
+#define       CSI_PP_FRAME_MIN_GAP(n)			(((n) & 0xffff) << 16)
+#define       CSI_PP_LINE_MIN_GAP(n)			(((n) & 0xffff))
+#define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(n)		(0x0028 + (n) * 0x2c)
+#define       CSI_PP_START_MARKER_FRAME_MAX(n)		(((n) & 0xf) << 12)
+#define       CSI_PP_START_MARKER_FRAME_MIN(n)		(((n) & 0xf) << 8)
+#define       CSI_PP_VSYNC_START_MARKER			BIT(4)
+#define       CSI_PP_SINGLE_SHOT			BIT(2)
+#define       CSI_PP_NOP				0
+#define       CSI_PP_ENABLE				1
+#define       CSI_PP_DISABLE				2
+#define       CSI_PP_RESET				3
+#define TEGRA_CSI_PHY_CIL_COMMAND			0x0068
+#define       CSI_A_PHY_CIL_NOP				0x0
+#define       CSI_A_PHY_CIL_ENABLE			0x1
+#define       CSI_A_PHY_CIL_DISABLE			0x2
+#define       CSI_A_PHY_CIL_ENABLE_MASK			0x3
+#define       CSI_B_PHY_CIL_NOP				(0x0 << 16)
+#define       CSI_B_PHY_CIL_ENABLE			(0x1 << 16)
+#define       CSI_B_PHY_CIL_DISABLE			(0x2 << 16)
+#define       CSI_B_PHY_CIL_ENABLE_MASK			(0x3 << 16)
+#define TEGRA_CSI_PHY_CIL_CONTROL0(n)			(0x006c + (n) * 4)
+#define       CSI_CONTINUOUS_CLOCK_MODE_ENABLE		BIT(5)
+#define TEGRA_CSI_CSI_PIXEL_PARSER_STATUS		0x0078
+#define TEGRA_CSI_CSI_CIL_STATUS			0x007c
+#define       CSI_MIPI_AUTO_CAL_DONE			BIT(15)
+#define TEGRA_CSI_CSI_PIXEL_PARSER_INTERRUPT_MASK	0x0080
+#define TEGRA_CSI_CSI_CIL_INTERRUPT_MASK		0x0084
+#define TEGRA_CSI_CSI_READONLY_STATUS			0x0088
+#define TEGRA_CSI_ESCAPE_MODE_COMMAND			0x008c
+#define TEGRA_CSI_ESCAPE_MODE_DATA			0x0090
+#define TEGRA_CSI_CIL_PAD_CONFIG0(n)			(0x0094 + (n) * 8)
+#define TEGRA_CSI_CIL_PAD_CONFIG1(n)			(0x0098 + (n) * 8)
+#define TEGRA_CSI_CIL_PAD_CONFIG			0x00a4
+#define TEGRA_CSI_CILA_MIPI_CAL_CONFIG			0x00a8
+#define TEGRA_CSI_CILB_MIPI_CAL_CONFIG			0x00ac
+#define       CSI_CIL_MIPI_CAL_STARTCAL			BIT(31)
+#define       CSI_CIL_MIPI_CAL_OVERIDE_A		BIT(30)
+#define       CSI_CIL_MIPI_CAL_OVERIDE_B		BIT(30)
+#define       CSI_CIL_MIPI_CAL_NOISE_FLT(n)		(((n) & 0xf) << 26)
+#define       CSI_CIL_MIPI_CAL_PRESCALE(n)		(((n) & 0x3) << 24)
+#define       CSI_CIL_MIPI_CAL_SEL_A			BIT(21)
+#define       CSI_CIL_MIPI_CAL_SEL_B			BIT(21)
+#define       CSI_CIL_MIPI_CAL_HSPDOS(n)		(((n) & 0x1f) << 16)
+#define       CSI_CIL_MIPI_CAL_HSPUOS(n)		(((n) & 0x1f) << 8)
+#define       CSI_CIL_MIPI_CAL_TERMOS(n)		(((n) & 0x1f))
+#define TEGRA_CSI_CIL_MIPI_CAL_STATUS			0x00b0
+#define TEGRA_CSI_CLKEN_OVERRIDE			0x00b4
+#define TEGRA_CSI_DEBUG_CONTROL				0x00b8
+#define       CSI_DEBUG_CONTROL_DEBUG_EN_ENABLED	BIT(0)
+#define       CSI_DEBUG_CONTROL_CLR_DBG_CNT_0		BIT(4)
+#define       CSI_DEBUG_CONTROL_CLR_DBG_CNT_1		BIT(5)
+#define       CSI_DEBUG_CONTROL_CLR_DBG_CNT_2		BIT(6)
+#define       CSI_DEBUG_CONTROL_DBG_CNT_SEL(n, v)	((v) << (8 + 8 * (n)))
+#define TEGRA_CSI_DEBUG_COUNTER(n)			(0x00bc + (n) * 4)
+#define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME(n)	(0x00c8 + (n) * 4)
+#define       CSI_PP_EXP_FRAME_HEIGHT(n)		(((n) & 0x1fff) << 16)
+#define       CSI_PP_MAX_CLOCKS(n)			(((n) & 0xfff) << 4)
+#define       CSI_PP_LINE_TIMEOUT_ENABLE		BIT(0)
+#define TEGRA_CSI_DSI_MIPI_CAL_CONFIG			0x00d0
+#define TEGRA_CSI_MIPIBIAS_PAD_CONFIG			0x00d4
+#define       CSI_PAD_DRIV_DN_REF(n)			(((n) & 0x7) << 16)
+#define       CSI_PAD_DRIV_UP_REF(n)			(((n) & 0x7) << 8)
+#define       CSI_PAD_TERM_REF(n)			(((n) & 0x7) << 0)
+#define TEGRA_CSI_CSI_CILA_STATUS			0x00d8
+#define TEGRA_CSI_CSI_CILB_STATUS			0x00dc
+
 /* --------------------------------------------------------------------------
- * VI
+ * Read and Write helpers
  */
 
 static void tegra20_vi_write(struct tegra_vi_channel *chan, unsigned int addr, u32 val)
@@ -153,6 +272,35 @@ static void tegra20_vi_write(struct tegra_vi_channel *chan, unsigned int addr, u
 	writel(val, chan->vi->iomem + addr);
 }
 
+static int __maybe_unused tegra20_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
+{
+	return readl(chan->vi->iomem + addr);
+}
+
+static void tegra20_csi_write(struct tegra_csi_channel *csi_chan, unsigned int addr, u32 val)
+{
+	writel(val, csi_chan->csi->iomem + addr);
+}
+
+static int __maybe_unused tegra20_csi_read(struct tegra_csi_channel *csi_chan, unsigned int addr)
+{
+	return readl(csi_chan->csi->iomem + addr);
+}
+
+static void tegra20_mipi_write(struct tegra_csi *csi, unsigned int addr, u32 val)
+{
+	writel(val, csi->iomem + addr);
+}
+
+static int __maybe_unused tegra20_mipi_read(struct tegra_csi *csi, unsigned int addr)
+{
+	return readl(csi->iomem + addr);
+}
+
+/* --------------------------------------------------------------------------
+ * VI
+ */
+
 /*
  * Get the main input format (YUV/RGB...) and the YUV variant as values to
  * be written into registers for the current VI input mbus code.
@@ -179,6 +327,18 @@ static void tegra20_vi_get_input_formats(struct tegra_vi_channel *chan,
 	case MEDIA_BUS_FMT_YVYU8_2X8:
 		(*yuv_input_format) = VI_INPUT_YUV_INPUT_FORMAT_YVYU;
 		break;
+	/* RAW8 */
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+	/* RAW10 */
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+		(*main_input_format) = VI_INPUT_INPUT_FORMAT_BAYER;
+		break;
 	}
 }
 
@@ -213,6 +373,18 @@ static void tegra20_vi_get_output_formats(struct tegra_vi_channel *chan,
 	case V4L2_PIX_FMT_YVU420:
 		(*main_output_format) = VI_OUTPUT_OUTPUT_FORMAT_YUV420PLANAR;
 		break;
+	/* RAW8 */
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SRGGB8:
+	/* RAW10 */
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SRGGB10:
+		(*main_output_format) = VI_OUTPUT_OUTPUT_FORMAT_VIP_BAYER_DIRECT;
+		break;
 	}
 }
 
@@ -251,20 +423,27 @@ static int tegra20_vi_enable(struct tegra_vi *vi, bool on)
 static int tegra20_channel_host1x_syncpt_init(struct tegra_vi_channel *chan)
 {
 	struct tegra_vi *vi = chan->vi;
-	struct host1x_syncpt *out_sp;
+	struct host1x_syncpt *out_sp, *fs_sp;
 
 	out_sp = host1x_syncpt_request(&vi->client, HOST1X_SYNCPT_CLIENT_MANAGED);
 	if (!out_sp)
-		return -ENOMEM;
+		return dev_err_probe(vi->dev, -EBUSY, "failed to request mw ack syncpoint\n");
 
 	chan->mw_ack_sp[0] = out_sp;
 
+	fs_sp = host1x_syncpt_request(&vi->client, HOST1X_SYNCPT_CLIENT_MANAGED);
+	if (!fs_sp)
+		return dev_err_probe(vi->dev, -EBUSY, "failed to request frame start syncpoint\n");
+
+	chan->frame_start_sp[0] = fs_sp;
+
 	return 0;
 }
 
 static void tegra20_channel_host1x_syncpt_free(struct tegra_vi_channel *chan)
 {
 	host1x_syncpt_put(chan->mw_ack_sp[0]);
+	host1x_syncpt_put(chan->frame_start_sp[0]);
 }
 
 static void tegra20_fmt_align(struct v4l2_pix_format *pix, unsigned int bpp)
@@ -272,18 +451,13 @@ static void tegra20_fmt_align(struct v4l2_pix_format *pix, unsigned int bpp)
 	pix->width  = clamp(pix->width,  TEGRA20_MIN_WIDTH,  TEGRA20_MAX_WIDTH);
 	pix->height = clamp(pix->height, TEGRA20_MIN_HEIGHT, TEGRA20_MAX_HEIGHT);
 
+	pix->bytesperline = roundup(pix->width, 8) * bpp;
+	pix->sizeimage = pix->bytesperline * pix->height;
+
 	switch (pix->pixelformat) {
-	case V4L2_PIX_FMT_UYVY:
-	case V4L2_PIX_FMT_VYUY:
-	case V4L2_PIX_FMT_YUYV:
-	case V4L2_PIX_FMT_YVYU:
-		pix->bytesperline = roundup(pix->width, 2) * 2;
-		pix->sizeimage = roundup(pix->width, 2) * 2 * pix->height;
-		break;
 	case V4L2_PIX_FMT_YUV420:
 	case V4L2_PIX_FMT_YVU420:
-		pix->bytesperline = roundup(pix->width, 8);
-		pix->sizeimage = roundup(pix->width, 8) * pix->height * 3 / 2;
+		pix->sizeimage = pix->sizeimage * 3 / 2;
 		break;
 	}
 }
@@ -305,6 +479,16 @@ static void tegra20_channel_queue_setup(struct tegra_vi_channel *chan)
 	case V4L2_PIX_FMT_VYUY:
 	case V4L2_PIX_FMT_YUYV:
 	case V4L2_PIX_FMT_YVYU:
+	/* RAW8 */
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SBGGR8:
+	/* RAW10 */
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SBGGR10:
 		if (chan->vflip)
 			chan->start_offset += stride * (height - 1);
 		if (chan->hflip)
@@ -367,48 +551,111 @@ static void tegra20_channel_vi_buffer_setup(struct tegra_vi_channel *chan,
 	case V4L2_PIX_FMT_VYUY:
 	case V4L2_PIX_FMT_YUYV:
 	case V4L2_PIX_FMT_YVYU:
-		tegra20_vi_write(chan, TEGRA_VI_VB0_BASE_ADDRESS_FIRST,  base);
-		tegra20_vi_write(chan, TEGRA_VI_VB0_START_ADDRESS_FIRST, base + chan->start_offset);
+		tegra20_vi_write(chan, TEGRA_VI_VB0_BASE_ADDRESS(TEGRA_VI_OUT_1),  base);
+		tegra20_vi_write(chan, TEGRA_VI_VB0_START_ADDRESS(TEGRA_VI_OUT_1), base + chan->start_offset);
+		break;
+	/* RAW8 */
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SBGGR8:
+	/* RAW10 */
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SBGGR10:
+		tegra20_vi_write(chan, TEGRA_VI_VB0_BASE_ADDRESS(TEGRA_VI_OUT_2),  base);
+		tegra20_vi_write(chan, TEGRA_VI_VB0_START_ADDRESS(TEGRA_VI_OUT_2), base + chan->start_offset);
 		break;
 	}
 }
 
 static int tegra20_channel_capture_frame(struct tegra_vi_channel *chan,
-					 struct tegra_channel_buffer *buf)
+					 struct tegra_channel_buffer *buf,
+					 struct tegra_csi_channel *csi_chan)
 {
+	u32 val;
 	int err;
 
-	chan->next_out_sp_idx++;
-
 	tegra20_channel_vi_buffer_setup(chan, buf);
 
-	tegra20_vi_write(chan, TEGRA_VI_CAMERA_CONTROL, VI_CAMERA_CONTROL_VIP_ENABLE);
+	if (csi_chan) {
+		u32 port = csi_chan->csi_port_nums[0] & 1;
+
+		tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port),
+				  CSI_PP_START_MARKER_FRAME_MAX(0xf) |
+				  CSI_PP_SINGLE_SHOT | CSI_PP_ENABLE);
+
+		/*
+		 * ERESTARTSYS workaround for syncpoints is used because host1x_syncpt_wait
+		 * is unconditionally interruptible. This is not an issue with single shots
+		 * or low resolution capture, but -ERESTARTSYS occurs quite often with high
+		 * resolution or high framerate captures and if not addressed here will
+		 * cause capture to fail entirely.
+		 *
+		 * TODO: once uninterruptible version of host1x_syncpt_wait is available,
+		 * host1x_syncpt_wait should be swapped and ERESTARTSYS workaround can be
+		 * removed.
+		 */
+
+		val = host1x_syncpt_read(chan->frame_start_sp[0]);
+		do {
+			err = host1x_syncpt_wait(chan->frame_start_sp[0],
+						 val + 1, TEGRA_VI_SYNCPT_WAIT_TIMEOUT, NULL);
+		} while (err == -ERESTARTSYS);
+
+		if (err) {
+			if (err != -ERESTARTSYS)
+				dev_err_ratelimited(&chan->video.dev,
+						    "frame start syncpt timeout: %d\n", err);
+
+			tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port),
+					  CSI_PP_START_MARKER_FRAME_MAX(0xf) | CSI_PP_RESET);
+			goto exit;
+		}
+
+		tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port),
+				  CSI_PP_START_MARKER_FRAME_MAX(0xf) |
+				  CSI_PP_DISABLE);
+	} else {
+		tegra20_vi_write(chan, TEGRA_VI_CAMERA_CONTROL, VI_CAMERA_CONTROL_VIP_ENABLE);
+	}
+
+	val = host1x_syncpt_read(chan->mw_ack_sp[0]);
+	do {
+		err = host1x_syncpt_wait(chan->mw_ack_sp[0], val + 1,
+					 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, NULL);
+	} while (err == -ERESTARTSYS);
 
-	/* Wait for syncpt counter to reach frame start event threshold */
-	err = host1x_syncpt_wait(chan->mw_ack_sp[0], chan->next_out_sp_idx,
-				 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, NULL);
 	if (err) {
-		host1x_syncpt_incr(chan->mw_ack_sp[0]);
-		dev_err_ratelimited(&chan->video.dev, "frame start syncpt timeout: %d\n", err);
-		release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
-		return err;
+		if (err != -ERESTARTSYS)
+			dev_err_ratelimited(&chan->video.dev, "mw ack syncpt timeout: %d\n", err);
+		goto exit;
 	}
 
-	tegra20_vi_write(chan, TEGRA_VI_CAMERA_CONTROL,
-			 VI_CAMERA_CONTROL_STOP_CAPTURE | VI_CAMERA_CONTROL_VIP_ENABLE);
+	if (!csi_chan)
+		tegra20_vi_write(chan, TEGRA_VI_CAMERA_CONTROL,
+				 VI_CAMERA_CONTROL_STOP_CAPTURE | VI_CAMERA_CONTROL_VIP_ENABLE);
 
+exit:
 	release_buffer(chan, buf, VB2_BUF_STATE_DONE);
 
-	return 0;
+	return err;
 }
 
 static int tegra20_chan_capture_kthread_start(void *data)
 {
 	struct tegra_vi_channel *chan = data;
 	struct tegra_channel_buffer *buf;
+	struct v4l2_subdev *csi_subdev = NULL;
+	struct tegra_csi_channel *csi_chan = NULL;
 	unsigned int retries = 0;
 	int err = 0;
 
+	csi_subdev = tegra_channel_get_remote_csi_subdev(chan);
+	if (csi_subdev)
+		csi_chan = to_csi_chan(csi_subdev);
+
 	while (1) {
 		/*
 		 * Source is not streaming if error is non-zero.
@@ -433,7 +680,7 @@ static int tegra20_chan_capture_kthread_start(void *data)
 		list_del_init(&buf->queue);
 		spin_unlock(&chan->start_lock);
 
-		err = tegra20_channel_capture_frame(chan, buf);
+		err = tegra20_channel_capture_frame(chan, buf, csi_chan);
 		if (!err) {
 			retries = 0;
 			continue;
@@ -451,55 +698,37 @@ static int tegra20_chan_capture_kthread_start(void *data)
 static void tegra20_camera_capture_setup(struct tegra_vi_channel *chan)
 {
 	u32 output_fourcc = chan->format.pixelformat;
+	u32 data_type = chan->fmtinfo->img_dt;
 	int width  = chan->format.width;
 	int height = chan->format.height;
-	int stride_l = chan->format.bytesperline;
+	int stride_l = chan->format.bytesperline * height;
 	int stride_c = (output_fourcc == V4L2_PIX_FMT_YUV420 ||
 			output_fourcc == V4L2_PIX_FMT_YVU420) ? 1 : 0;
-	int main_output_format;
-	int yuv_output_format;
-
-	tegra20_vi_get_output_formats(chan, &main_output_format, &yuv_output_format);
-
-	/*
-	 * Set up low pass filter.  Use 0x240 for chromaticity and 0x240
-	 * for luminance, which is the default and means not to touch
-	 * anything.
-	 */
-	tegra20_vi_write(chan, TEGRA_VI_H_LPF_CONTROL,
-			 0x0240 << VI_H_LPF_CONTROL_LUMA_SFT |
-			 0x0240 << VI_H_LPF_CONTROL_CHROMA_SFT);
-
-	/* Set up raise-on-edge, so we get an interrupt on end of frame. */
-	tegra20_vi_write(chan, TEGRA_VI_VI_RAISE, VI_VI_RAISE_ON_EDGE);
-
-	tegra20_vi_write(chan, TEGRA_VI_VI_FIRST_OUTPUT_CONTROL,
-			 (chan->vflip ? VI_OUTPUT_V_DIRECTION : 0) |
-			 (chan->hflip ? VI_OUTPUT_H_DIRECTION : 0) |
-			 yuv_output_format << VI_OUTPUT_YUV_OUTPUT_FORMAT_SFT |
-			 main_output_format << VI_OUTPUT_OUTPUT_FORMAT_SFT);
+	enum tegra_vi_out output_channel = (data_type == TEGRA_IMAGE_DT_RAW8 ||
+					    data_type == TEGRA_IMAGE_DT_RAW10) ?
+					    TEGRA_VI_OUT_2 : TEGRA_VI_OUT_1;
 
 	/* Set up frame size */
-	tegra20_vi_write(chan, TEGRA_VI_FIRST_OUTPUT_FRAME_SIZE,
-			 height << VI_FIRST_OUTPUT_FRAME_HEIGHT_SFT |
-			 width  << VI_FIRST_OUTPUT_FRAME_WIDTH_SFT);
+	tegra20_vi_write(chan, TEGRA_VI_OUTPUT_FRAME_SIZE(output_channel),
+			 height << VI_OUTPUT_FRAME_HEIGHT_SFT |
+			 width  << VI_OUTPUT_FRAME_WIDTH_SFT);
 
 	/* First output memory enabled */
-	tegra20_vi_write(chan, TEGRA_VI_VI_ENABLE, 0);
+	tegra20_vi_write(chan, TEGRA_VI_VI_ENABLE(output_channel), 0);
 
 	/* Set the number of frames in the buffer */
-	tegra20_vi_write(chan, TEGRA_VI_VB0_COUNT_FIRST, 1);
+	tegra20_vi_write(chan, TEGRA_VI_VB0_COUNT(output_channel), 1);
 
 	/* Set up buffer frame size */
-	tegra20_vi_write(chan, TEGRA_VI_VB0_SIZE_FIRST,
-			 height << VI_VB0_SIZE_FIRST_V_SFT |
-			 width  << VI_VB0_SIZE_FIRST_H_SFT);
+	tegra20_vi_write(chan, TEGRA_VI_VB0_SIZE(output_channel),
+			 height << VI_VB0_SIZE_V_SFT |
+			 width  << VI_VB0_SIZE_H_SFT);
 
-	tegra20_vi_write(chan, TEGRA_VI_VB0_BUFFER_STRIDE_FIRST,
-			 stride_l << VI_VB0_BUFFER_STRIDE_FIRST_LUMA_SFT |
-			 stride_c << VI_VB0_BUFFER_STRIDE_FIRST_CHROMA_SFT);
+	tegra20_vi_write(chan, TEGRA_VI_VB0_BUFFER_STRIDE(output_channel),
+			 stride_l << VI_VB0_BUFFER_STRIDE_LUMA_SFT |
+			 stride_c << VI_VB0_BUFFER_STRIDE_CHROMA_SFT);
 
-	tegra20_vi_write(chan, TEGRA_VI_VI_ENABLE, 0);
+	tegra20_vi_write(chan, TEGRA_VI_VI_ENABLE(output_channel), 0);
 }
 
 static int tegra20_vi_start_streaming(struct vb2_queue *vq, u32 count)
@@ -508,18 +737,28 @@ static int tegra20_vi_start_streaming(struct vb2_queue *vq, u32 count)
 	struct media_pipeline *pipe = &chan->video.pipe;
 	int err;
 
-	chan->next_out_sp_idx = host1x_syncpt_read(chan->mw_ack_sp[0]);
-
 	err = video_device_pipeline_start(&chan->video, pipe);
 	if (err)
 		goto error_pipeline_start;
 
-	tegra20_camera_capture_setup(chan);
+	/*
+	 * Set up low pass filter.  Use 0x240 for chromaticity and 0x240
+	 * for luminance, which is the default and means not to touch
+	 * anything.
+	 */
+	tegra20_vi_write(chan, TEGRA_VI_H_LPF_CONTROL,
+			 0x0240 << VI_H_LPF_CONTROL_LUMA_SFT |
+			 0x0240 << VI_H_LPF_CONTROL_CHROMA_SFT);
+
+	/* Set up raise-on-edge, so we get an interrupt on end of frame. */
+	tegra20_vi_write(chan, TEGRA_VI_VI_RAISE, VI_VI_RAISE_ON_EDGE);
 
 	err = tegra_channel_set_stream(chan, true);
 	if (err)
 		goto error_set_stream;
 
+	tegra20_camera_capture_setup(chan);
+
 	chan->sequence = 0;
 
 	chan->kthread_start_capture = kthread_run(tegra20_chan_capture_kthread_start,
@@ -567,20 +806,40 @@ static const struct tegra_vi_ops tegra20_vi_ops = {
 	.vi_stop_streaming = tegra20_vi_stop_streaming,
 };
 
-#define TEGRA20_VIDEO_FMT(MBUS_CODE, BPP, FOURCC)	\
-{							\
-	.code    = MEDIA_BUS_FMT_##MBUS_CODE,		\
-	.bpp     = BPP,					\
-	.fourcc  = V4L2_PIX_FMT_##FOURCC,		\
+#define TEGRA20_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP, FOURCC)	\
+{									\
+	.img_dt		= TEGRA_IMAGE_DT_##DATA_TYPE,			\
+	.bit_width	= BIT_WIDTH,					\
+	.code		= MEDIA_BUS_FMT_##MBUS_CODE,			\
+	.bpp		= BPP,						\
+	.fourcc		= V4L2_PIX_FMT_##FOURCC,			\
 }
 
 static const struct tegra_video_format tegra20_video_formats[] = {
-	TEGRA20_VIDEO_FMT(UYVY8_2X8, 2, UYVY),
-	TEGRA20_VIDEO_FMT(VYUY8_2X8, 2, VYUY),
-	TEGRA20_VIDEO_FMT(YUYV8_2X8, 2, YUYV),
-	TEGRA20_VIDEO_FMT(YVYU8_2X8, 2, YVYU),
-	TEGRA20_VIDEO_FMT(UYVY8_2X8, 1, YUV420),
-	TEGRA20_VIDEO_FMT(UYVY8_2X8, 1, YVU420),
+	/* YUV422 */
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, UYVY),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, VYUY),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, YUYV),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, YVYU),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, UYVY),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, VYUY),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, YUYV),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, YVYU),
+	/* YUV420P */
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 1, YUV420),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 1, YVU420),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, YUV420),
+	TEGRA20_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, YVU420),
+	/* RAW 8 */
+	TEGRA20_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 2, SRGGB8),
+	TEGRA20_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 2, SGRBG8),
+	TEGRA20_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 2, SGBRG8),
+	TEGRA20_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 2, SBGGR8),
+	/* RAW 10 */
+	TEGRA20_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, SRGGB10),
+	TEGRA20_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, SGRBG10),
+	TEGRA20_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, SGBRG10),
+	TEGRA20_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, SBGGR10),
 };
 
 const struct tegra_vi_soc tegra20_vi_soc = {
@@ -588,11 +847,351 @@ const struct tegra_vi_soc tegra20_vi_soc = {
 	.nformats = ARRAY_SIZE(tegra20_video_formats),
 	.default_video_format = &tegra20_video_formats[0],
 	.ops = &tegra20_vi_ops,
-	.vi_max_channels = 1, /* parallel input (VIP) */
-	.vi_max_clk_hz = 150000000,
+	.hw_revision = 1,
+	.vi_max_channels = 2, /* TEGRA_VI_OUT_1 and TEGRA_VI_OUT_2 */
+	.vi_max_clk_hz = 450000000,
 	.has_h_v_flip = true,
 };
 
+/* --------------------------------------------------------------------------
+ * MIPI Calibration
+ */
+static int tegra20_start_pad_calibration(struct tegra_mipi_device *mipi)
+{
+	struct tegra_csi *csi = platform_get_drvdata(mipi->pdev);
+	unsigned int port = mipi->pads;
+	u32 value;
+	int ret;
+
+	guard(mutex)(&csi->mipi_lock);
+
+	ret = pm_runtime_resume_and_get(csi->dev);
+	if (ret < 0) {
+		dev_err(csi->dev, "failed to get runtime PM: %d\n", ret);
+		return ret;
+	}
+
+	tegra20_mipi_write(csi, TEGRA_CSI_DSI_MIPI_CAL_CONFIG,
+			   CSI_CIL_MIPI_CAL_HSPDOS(4) |
+			   CSI_CIL_MIPI_CAL_HSPUOS(3) |
+			   CSI_CIL_MIPI_CAL_TERMOS(0));
+	tegra20_mipi_write(csi, TEGRA_CSI_MIPIBIAS_PAD_CONFIG,
+			   CSI_PAD_DRIV_DN_REF(5) |
+			   CSI_PAD_DRIV_UP_REF(7) |
+			   CSI_PAD_TERM_REF(0));
+
+	/* CSI B */
+	value = CSI_CIL_MIPI_CAL_HSPDOS(0) |
+		CSI_CIL_MIPI_CAL_HSPUOS(0) |
+		CSI_CIL_MIPI_CAL_TERMOS(4);
+
+	if (port == PORT_B)
+		value |= CSI_CIL_MIPI_CAL_SEL_B;
+
+	tegra20_mipi_write(csi, TEGRA_CSI_CILB_MIPI_CAL_CONFIG, value);
+
+	/* CSI A */
+	value = CSI_CIL_MIPI_CAL_STARTCAL |
+		CSI_CIL_MIPI_CAL_NOISE_FLT(0xa) |
+		CSI_CIL_MIPI_CAL_PRESCALE(0x2) |
+		CSI_CIL_MIPI_CAL_HSPDOS(0) |
+		CSI_CIL_MIPI_CAL_HSPUOS(0) |
+		CSI_CIL_MIPI_CAL_TERMOS(4);
+
+	if (port == PORT_A)
+		value |= CSI_CIL_MIPI_CAL_SEL_A;
+
+	tegra20_mipi_write(csi, TEGRA_CSI_CILA_MIPI_CAL_CONFIG, value);
+
+	tegra20_mipi_write(csi, TEGRA_CSI_CIL_PAD_CONFIG, 0);
+
+	return 0;
+}
+
+static int tegra20_finish_pad_calibration(struct tegra_mipi_device *mipi)
+{
+	struct tegra_csi *csi = platform_get_drvdata(mipi->pdev);
+	void __iomem *cil_status_reg = csi->iomem + TEGRA_CSI_CSI_CIL_STATUS;
+	unsigned int port = mipi->pads;
+	u32 value, pp = 0, cil = 0;
+	int ret;
+
+	/* This part is only for CSI */
+	if (port > PORT_B) {
+		pm_runtime_put(csi->dev);
+
+		return 0;
+	}
+
+	guard(mutex)(&csi->mipi_lock);
+
+	ret = readl_relaxed_poll_timeout(cil_status_reg, value,
+					 value & CSI_MIPI_AUTO_CAL_DONE, 50, 250000);
+	if (ret < 0) {
+		dev_warn(csi->dev, "MIPI calibration timeout!\n");
+		goto exit;
+	}
+
+	/* clear status */
+	tegra20_mipi_write(csi, TEGRA_CSI_CSI_CIL_STATUS, value);
+	ret = readl_relaxed_poll_timeout(cil_status_reg, value,
+					 !(value & CSI_MIPI_AUTO_CAL_DONE), 50, 250000);
+	if (ret < 0) {
+		dev_warn(csi->dev, "MIPI calibration status timeout!\n");
+		goto exit;
+	}
+
+	pp = tegra20_mipi_read(csi, TEGRA_CSI_CSI_PIXEL_PARSER_STATUS);
+	cil = tegra20_mipi_read(csi, TEGRA_CSI_CSI_CIL_STATUS);
+	if (pp | cil) {
+		dev_warn(csi->dev, "Calibration status not been cleared!\n");
+		ret = -EINVAL;
+		goto exit;
+	}
+
+exit:
+	tegra20_mipi_write(csi, TEGRA_CSI_CSI_CIL_STATUS, pp);
+
+	/* un-select to avoid interference with DSI */
+	tegra20_mipi_write(csi, TEGRA_CSI_CILB_MIPI_CAL_CONFIG,
+			   CSI_CIL_MIPI_CAL_HSPDOS(0) |
+			   CSI_CIL_MIPI_CAL_HSPUOS(0) |
+			   CSI_CIL_MIPI_CAL_TERMOS(4));
+
+	tegra20_mipi_write(csi, TEGRA_CSI_CILA_MIPI_CAL_CONFIG,
+			   CSI_CIL_MIPI_CAL_NOISE_FLT(0xa) |
+			   CSI_CIL_MIPI_CAL_PRESCALE(0x2) |
+			   CSI_CIL_MIPI_CAL_HSPDOS(0) |
+			   CSI_CIL_MIPI_CAL_HSPUOS(0) |
+			   CSI_CIL_MIPI_CAL_TERMOS(4));
+
+	pm_runtime_put(csi->dev);
+
+	return ret;
+}
+
+static const struct tegra_mipi_ops tegra20_mipi_ops = {
+	.start_calibration = tegra20_start_pad_calibration,
+	.finish_calibration = tegra20_finish_pad_calibration,
+};
+
+/* --------------------------------------------------------------------------
+ * CSI
+ */
+static void tegra20_csi_capture_clean(struct tegra_csi_channel *csi_chan)
+{
+	tegra20_csi_write(csi_chan, TEGRA_CSI_VI_INPUT_STREAM_CONTROL, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_HOST_INPUT_STREAM_CONTROL, 0);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_PIXEL_PARSER_STATUS, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_CIL_STATUS, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_CIL_INTERRUPT_MASK, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_READONLY_STATUS, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_ESCAPE_MODE_COMMAND, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_ESCAPE_MODE_DATA, 0);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CIL_PAD_CONFIG, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CIL_MIPI_CAL_STATUS, 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CLKEN_OVERRIDE, 0);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_DEBUG_CONTROL,
+			  CSI_DEBUG_CONTROL_CLR_DBG_CNT_0 |
+			  CSI_DEBUG_CONTROL_CLR_DBG_CNT_1 |
+			  CSI_DEBUG_CONTROL_CLR_DBG_CNT_2);
+}
+
+static int tegra20_csi_port_start_streaming(struct tegra_csi_channel *csi_chan,
+					    u8 portno)
+{
+	struct tegra_vi_channel *vi_chan = v4l2_get_subdev_hostdata(&csi_chan->subdev);
+	int width  = vi_chan->format.width;
+	int height = vi_chan->format.height;
+	u32 data_type = vi_chan->fmtinfo->img_dt;
+	u32 word_count = (width * vi_chan->fmtinfo->bit_width) / 8;
+	enum tegra_vi_out output_channel = TEGRA_VI_OUT_1;
+
+	unsigned int main_output_format, yuv_output_format;
+	unsigned int port = portno & 1;
+	u32 value;
+
+	tegra20_vi_get_output_formats(vi_chan, &main_output_format, &yuv_output_format);
+
+	switch (data_type) {
+	case TEGRA_IMAGE_DT_RAW8:
+	case TEGRA_IMAGE_DT_RAW10:
+		output_channel = TEGRA_VI_OUT_2;
+		if (port == PORT_A)
+			main_output_format = VI_OUTPUT_OUTPUT_FORMAT_CSI_PPA_BAYER;
+		else
+			main_output_format = VI_OUTPUT_OUTPUT_FORMAT_CSI_PPB_BAYER;
+		break;
+	}
+
+	tegra20_csi_capture_clean(csi_chan);
+
+	/* CSI port cleanup */
+	tegra20_csi_write(csi_chan, TEGRA_CSI_INPUT_STREAM_CONTROL(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_CONTROL0(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_CONTROL1(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_WORD_COUNT(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_GAP(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PHY_CIL_CONTROL0(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CIL_PAD_CONFIG0(port), 0);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CIL_PAD_CONFIG1(port), 0);
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_VI_CORE_CONTROL, BIT(25 + port)); /* CSI_PP_YUV422 */
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_H_DOWNSCALE_CONTROL, BIT(2 + port)); /* CSI_PP */
+	tegra20_vi_write(vi_chan, TEGRA_VI_V_DOWNSCALE_CONTROL, BIT(2 + port)); /* CSI_PP */
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_CSI_PP_H_ACTIVE(port), width << 16);
+	tegra20_vi_write(vi_chan, TEGRA_VI_CSI_PP_V_ACTIVE(port), height << 16);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_CONTROL1(port), 0x1);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_WORD_COUNT(port), word_count);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_GAP(port),
+			  CSI_PP_FRAME_MIN_GAP(0x14)); /* 14 vi clks between frames */
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME(port),
+			  CSI_PP_EXP_FRAME_HEIGHT(height) |
+			  CSI_PP_MAX_CLOCKS(0x300) | /* wait 0x300 vi clks for timeout */
+			  CSI_PP_LINE_TIMEOUT_ENABLE);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_CONTROL0(port),
+			  CSI_PP_OUTPUT_FORMAT_PIXEL |
+			  CSI_PP_DATA_TYPE(data_type) |
+			  CSI_PP_CRC_CHECK_ENABLE |
+			  CSI_PP_WORD_COUNT_HEADER |
+			  CSI_PP_DATA_IDENTIFIER_ENABLE |
+			  CSI_PP_PACKET_HEADER_SENT |
+			  port);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_INPUT_STREAM_CONTROL(port),
+			  CSI_SKIP_PACKET_THRESHOLD(0x3f) |
+			  (csi_chan->numlanes - 1));
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PHY_CIL_CONTROL0(port),
+			  CSI_CONTINUOUS_CLOCK_MODE_ENABLE |
+			  0x5); /* Clock settle time */
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_CONT_SYNCPT_CSI_PP_FRAME_START(port),
+			 VI_CONT_SYNCPT_OUT_CONTINUOUS_SYNCPT |
+			 host1x_syncpt_id(vi_chan->frame_start_sp[0])
+			 << VI_CONT_SYNCPT_OUT_SYNCPT_IDX_SFT);
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_CONT_SYNCPT_OUT(output_channel),
+			 VI_CONT_SYNCPT_OUT_CONTINUOUS_SYNCPT |
+			 host1x_syncpt_id(vi_chan->mw_ack_sp[0])
+			 << VI_CONT_SYNCPT_OUT_SYNCPT_IDX_SFT);
+
+	value = (port == PORT_A) ? CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_DISABLE :
+		CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_DISABLE;
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PHY_CIL_COMMAND, value);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port),
+			  CSI_PP_START_MARKER_FRAME_MAX(0xf) |
+			  CSI_PP_DISABLE);
+
+	tegra20_vi_write(vi_chan, TEGRA_VI_VI_OUTPUT_CONTROL(output_channel),
+			 (vi_chan->vflip ? VI_OUTPUT_V_DIRECTION : 0) |
+			 (vi_chan->hflip ? VI_OUTPUT_H_DIRECTION : 0) |
+			 yuv_output_format | main_output_format);
+
+	return 0;
+};
+
+static void tegra20_csi_port_stop_streaming(struct tegra_csi_channel *csi_chan, u8 portno)
+{
+	struct tegra_csi *csi = csi_chan->csi;
+	unsigned int port = portno & 1;
+	u32 value;
+
+	value = tegra20_csi_read(csi_chan, TEGRA_CSI_CSI_PIXEL_PARSER_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CSI_PIXEL_PARSER_STATUS 0x%08x\n", value);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_PIXEL_PARSER_STATUS, value);
+
+	value = tegra20_csi_read(csi_chan, TEGRA_CSI_CSI_CIL_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CSI_CIL_STATUS 0x%08x\n", value);
+	tegra20_csi_write(csi_chan, TEGRA_CSI_CSI_CIL_STATUS, value);
+
+	tegra20_csi_write(csi_chan, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND(port),
+			  CSI_PP_START_MARKER_FRAME_MAX(0xf) |
+			  CSI_PP_DISABLE);
+
+	if (csi_chan->numlanes == 4) {
+		tegra20_csi_write(csi_chan, TEGRA_CSI_PHY_CIL_COMMAND,
+				  CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_DISABLE);
+	} else {
+		value = (port == PORT_A) ? CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
+			CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
+		tegra20_csi_write(csi_chan, TEGRA_CSI_PHY_CIL_COMMAND, value);
+	}
+}
+
+static int tegra20_csi_start_streaming(struct tegra_csi_channel *csi_chan)
+{
+	u8 *portnos = csi_chan->csi_port_nums;
+	int ret, i;
+
+	for (i = 0; i < csi_chan->numgangports; i++) {
+		ret = tegra20_csi_port_start_streaming(csi_chan, portnos[i]);
+		if (ret)
+			goto stream_start_fail;
+	}
+
+	return 0;
+
+stream_start_fail:
+	for (i = i - 1; i >= 0; i--)
+		tegra20_csi_port_stop_streaming(csi_chan, portnos[i]);
+
+	return ret;
+}
+
+static void tegra20_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
+{
+	u8 *portnos = csi_chan->csi_port_nums;
+	int i;
+
+	for (i = 0; i < csi_chan->numgangports; i++)
+		tegra20_csi_port_stop_streaming(csi_chan, portnos[i]);
+}
+
+static const struct tegra_csi_ops tegra20_csi_ops = {
+	.csi_start_streaming = tegra20_csi_start_streaming,
+	.csi_stop_streaming = tegra20_csi_stop_streaming,
+};
+
+static const char * const tegra20_csi_clks[] = {
+	NULL,
+};
+
+const struct tegra_csi_soc tegra20_csi_soc = {
+	.ops = &tegra20_csi_ops,
+	.mipi_ops = &tegra20_mipi_ops,
+	.csi_max_channels = 2, /* CSI-A and CSI-B */
+	.clk_names = tegra20_csi_clks,
+	.num_clks = ARRAY_SIZE(tegra20_csi_clks),
+};
+
+static const char * const tegra30_csi_clks[] = {
+	"csi",
+	"csia-pad",
+	"csib-pad",
+};
+
+const struct tegra_csi_soc tegra30_csi_soc = {
+	.ops = &tegra20_csi_ops,
+	.mipi_ops = &tegra20_mipi_ops,
+	.csi_max_channels = 2, /* CSI-A and CSI-B */
+	.clk_names = tegra30_csi_clks,
+	.num_clks = ARRAY_SIZE(tegra30_csi_clks),
+};
+
 /* --------------------------------------------------------------------------
  * VIP
  */
@@ -606,13 +1205,17 @@ const struct tegra_vi_soc tegra20_vi_soc = {
 static int tegra20_vip_start_streaming(struct tegra_vip_channel *vip_chan)
 {
 	struct tegra_vi_channel *vi_chan = v4l2_get_subdev_hostdata(&vip_chan->subdev);
+	u32 data_type = vi_chan->fmtinfo->img_dt;
 	int width  = vi_chan->format.width;
 	int height = vi_chan->format.height;
-
-	unsigned int main_input_format;
-	unsigned int yuv_input_format;
+	enum tegra_vi_out output_channel = (data_type == TEGRA_IMAGE_DT_RAW8 ||
+					    data_type == TEGRA_IMAGE_DT_RAW10) ?
+					    TEGRA_VI_OUT_2 : TEGRA_VI_OUT_1;
+	unsigned int main_input_format, yuv_input_format;
+	unsigned int main_output_format, yuv_output_format;
 
 	tegra20_vi_get_input_formats(vi_chan, &main_input_format, &yuv_input_format);
+	tegra20_vi_get_output_formats(vi_chan, &main_output_format, &yuv_output_format);
 
 	tegra20_vi_write(vi_chan, TEGRA_VI_VI_CORE_CONTROL, 0);
 
@@ -638,13 +1241,18 @@ static int tegra20_vip_start_streaming(struct tegra_vip_channel *vip_chan)
 			 GENMASK(9, 2) << VI_DATA_INPUT_SFT);
 	tegra20_vi_write(vi_chan, TEGRA_VI_PIN_INVERSION, 0);
 
-	tegra20_vi_write(vi_chan, TEGRA_VI_CONT_SYNCPT_OUT_1,
-			 VI_CONT_SYNCPT_OUT_1_CONTINUOUS_SYNCPT |
+	tegra20_vi_write(vi_chan, TEGRA_VI_CONT_SYNCPT_OUT(output_channel),
+			 VI_CONT_SYNCPT_OUT_CONTINUOUS_SYNCPT |
 			 host1x_syncpt_id(vi_chan->mw_ack_sp[0])
-			 << VI_CONT_SYNCPT_OUT_1_SYNCPT_IDX_SFT);
+			 << VI_CONT_SYNCPT_OUT_SYNCPT_IDX_SFT);
 
 	tegra20_vi_write(vi_chan, TEGRA_VI_CAMERA_CONTROL, VI_CAMERA_CONTROL_STOP_CAPTURE);
 
+	tegra20_vi_write(vi_chan, TEGRA_VI_VI_OUTPUT_CONTROL(output_channel),
+			 (vi_chan->vflip ? VI_OUTPUT_V_DIRECTION : 0) |
+			 (vi_chan->hflip ? VI_OUTPUT_H_DIRECTION : 0) |
+			  yuv_output_format | main_output_format);
+
 	return 0;
 }
 
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
index 9c0b38585d63e..f14cdc7b52113 100644
--- a/drivers/staging/media/tegra-video/vi.c
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -160,8 +160,8 @@ static void tegra_channel_buffer_queue(struct vb2_buffer *vb)
 	wake_up_interruptible(&chan->start_wait);
 }
 
-struct v4l2_subdev *
-tegra_channel_get_remote_csi_subdev(struct tegra_vi_channel *chan)
+static struct v4l2_subdev *
+tegra_channel_get_remote_bridge_subdev(struct tegra_vi_channel *chan)
 {
 	struct media_pad *pad;
 
@@ -182,7 +182,7 @@ tegra_channel_get_remote_source_subdev(struct tegra_vi_channel *chan)
 	struct v4l2_subdev *subdev;
 	struct media_entity *entity;
 
-	subdev = tegra_channel_get_remote_csi_subdev(chan);
+	subdev = tegra_channel_get_remote_bridge_subdev(chan);
 	if (!subdev)
 		return NULL;
 
@@ -204,7 +204,7 @@ static int tegra_channel_enable_stream(struct tegra_vi_channel *chan)
 	struct v4l2_subdev *subdev;
 	int ret;
 
-	subdev = tegra_channel_get_remote_csi_subdev(chan);
+	subdev = tegra_channel_get_remote_bridge_subdev(chan);
 	ret = v4l2_subdev_call(subdev, video, s_stream, true);
 	if (ret < 0 && ret != -ENOIOCTLCMD)
 		return ret;
@@ -217,7 +217,7 @@ static int tegra_channel_disable_stream(struct tegra_vi_channel *chan)
 	struct v4l2_subdev *subdev;
 	int ret;
 
-	subdev = tegra_channel_get_remote_csi_subdev(chan);
+	subdev = tegra_channel_get_remote_bridge_subdev(chan);
 	ret = v4l2_subdev_call(subdev, video, s_stream, false);
 	if (ret < 0 && ret != -ENOIOCTLCMD)
 		return ret;
@@ -476,17 +476,11 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
 	fse.code = fmtinfo->code;
 	ret = v4l2_subdev_call(subdev, pad, enum_frame_size, sd_state, &fse);
 	if (ret) {
-		if (!v4l2_subdev_has_op(subdev, pad, get_selection)) {
+		if (!v4l2_subdev_has_op(subdev, pad, get_selection) ||
+		    v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel)) {
 			try_crop->width = 0;
 			try_crop->height = 0;
 		} else {
-			ret = v4l2_subdev_call(subdev, pad, get_selection,
-					       NULL, &sdsel);
-			if (ret) {
-				ret = -EINVAL;
-				goto out_free;
-			}
-
 			try_crop->width = sdsel.r.width;
 			try_crop->height = sdsel.r.height;
 		}
@@ -968,6 +962,7 @@ static int tegra_channel_setup_ctrl_handler(struct tegra_vi_channel *chan)
 	}
 #else
 	struct v4l2_subdev *subdev;
+	struct v4l2_ctrl *hflip, *vflip;
 
 	/* custom control */
 	v4l2_ctrl_new_custom(&chan->ctrl_handler, &syncpt_timeout_ctrl, NULL);
@@ -993,11 +988,13 @@ static int tegra_channel_setup_ctrl_handler(struct tegra_vi_channel *chan)
 		return ret;
 	}
 
-	if (chan->vi->soc->has_h_v_flip) {
+	hflip = v4l2_ctrl_find(subdev->ctrl_handler, V4L2_CID_HFLIP);
+	if (chan->vi->soc->has_h_v_flip && !hflip)
 		v4l2_ctrl_new_std(&chan->ctrl_handler, &vi_ctrl_ops, V4L2_CID_HFLIP, 0, 1, 1, 0);
-		v4l2_ctrl_new_std(&chan->ctrl_handler, &vi_ctrl_ops, V4L2_CID_VFLIP, 0, 1, 1, 0);
-	}
 
+	vflip = v4l2_ctrl_find(subdev->ctrl_handler, V4L2_CID_VFLIP);
+	if (chan->vi->soc->has_h_v_flip && !vflip)
+		v4l2_ctrl_new_std(&chan->ctrl_handler, &vi_ctrl_ops, V4L2_CID_VFLIP, 0, 1, 1, 0);
 #endif
 
 	/* setup the controls */
@@ -1421,29 +1418,19 @@ static int __maybe_unused vi_runtime_resume(struct device *dev)
 	struct tegra_vi *vi = dev_get_drvdata(dev);
 	int ret;
 
-	ret = regulator_enable(vi->vdd);
-	if (ret) {
-		dev_err(dev, "failed to enable VDD supply: %d\n", ret);
-		return ret;
-	}
-
 	ret = clk_set_rate(vi->clk, vi->soc->vi_max_clk_hz);
 	if (ret) {
 		dev_err(dev, "failed to set vi clock rate: %d\n", ret);
-		goto disable_vdd;
+		return ret;
 	}
 
 	ret = clk_prepare_enable(vi->clk);
 	if (ret) {
 		dev_err(dev, "failed to enable vi clock: %d\n", ret);
-		goto disable_vdd;
+		return ret;
 	}
 
 	return 0;
-
-disable_vdd:
-	regulator_disable(vi->vdd);
-	return ret;
 }
 
 static int __maybe_unused vi_runtime_suspend(struct device *dev)
@@ -1452,8 +1439,6 @@ static int __maybe_unused vi_runtime_suspend(struct device *dev)
 
 	clk_disable_unprepare(vi->clk);
 
-	regulator_disable(vi->vdd);
-
 	return 0;
 }
 
@@ -1634,11 +1619,11 @@ static int tegra_vi_graph_notify_complete(struct v4l2_async_notifier *notifier)
 		goto unregister_video;
 	}
 
-	subdev = tegra_channel_get_remote_csi_subdev(chan);
+	subdev = tegra_channel_get_remote_bridge_subdev(chan);
 	if (!subdev) {
 		ret = -ENODEV;
 		dev_err(vi->dev,
-			"failed to get remote csi subdev: %d\n", ret);
+			"failed to get remote bridge subdev: %d\n", ret);
 		goto unregister_video;
 	}
 
@@ -1898,13 +1883,6 @@ static int tegra_vi_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	vi->vdd = devm_regulator_get(&pdev->dev, "avdd-dsi-csi");
-	if (IS_ERR(vi->vdd)) {
-		ret = PTR_ERR(vi->vdd);
-		dev_err(&pdev->dev, "failed to get VDD supply: %d\n", ret);
-		return ret;
-	}
-
 	if (!pdev->dev.pm_domain) {
 		ret = -ENOENT;
 		dev_warn(&pdev->dev, "PM domain is not attached: %d\n", ret);
@@ -1959,7 +1937,7 @@ static void tegra_vi_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id tegra_vi_of_id_table[] = {
-#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
 	{ .compatible = "nvidia,tegra20-vi",  .data = &tegra20_vi_soc },
 #endif
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
diff --git a/drivers/staging/media/tegra-video/vi.h b/drivers/staging/media/tegra-video/vi.h
index 1e6a5caa70827..46328e488aa38 100644
--- a/drivers/staging/media/tegra-video/vi.h
+++ b/drivers/staging/media/tegra-video/vi.h
@@ -94,7 +94,6 @@ struct tegra_vi_soc {
  * @client: host1x_client struct
  * @iomem: register base
  * @clk: main clock for VI block
- * @vdd: vdd regulator for VI hardware, normally it is avdd_dsi_csi
  * @soc: pointer to SoC data structure
  * @ops: vi operations
  * @vi_chans: list head for VI channels
@@ -104,7 +103,6 @@ struct tegra_vi {
 	struct host1x_client client;
 	void __iomem *iomem;
 	struct clk *clk;
-	struct regulator *vdd;
 	const struct tegra_vi_soc *soc;
 	const struct tegra_vi_ops *ops;
 	struct list_head vi_chans;
@@ -127,7 +125,6 @@ struct tegra_vi {
  *		frame through host1x syncpoint counters (On Tegra20 used for the
  *              OUT_1 syncpt)
  * @sp_incr_lock: protects cpu syncpoint increment.
- * @next_out_sp_idx: next expected value for mw_ack_sp[0], i.e. OUT_1 (Tegra20)
  *
  * @kthread_start_capture: kthread to start capture of single frame when
  *		vb buffer is available. This thread programs VI CSI hardware
@@ -190,7 +187,6 @@ struct tegra_vi_channel {
 	struct host1x_syncpt *mw_ack_sp[GANG_PORTS_MAX];
 	/* protects the cpu syncpoint increment */
 	spinlock_t sp_incr_lock[GANG_PORTS_MAX];
-	u32 next_out_sp_idx;
 
 	struct task_struct *kthread_start_capture;
 	wait_queue_head_t start_wait;
@@ -296,7 +292,7 @@ struct tegra_video_format {
 	u32 fourcc;
 };
 
-#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
 extern const struct tegra_vi_soc tegra20_vi_soc;
 #endif
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
diff --git a/drivers/staging/media/tegra-video/video.c b/drivers/staging/media/tegra-video/video.c
index ae1ae03fa9de1..2e84385708bac 100644
--- a/drivers/staging/media/tegra-video/video.c
+++ b/drivers/staging/media/tegra-video/video.c
@@ -121,10 +121,16 @@ static void host1x_video_remove(struct host1x_device *dev)
 }
 
 static const struct of_device_id host1x_video_subdevs[] = {
-#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
 	{ .compatible = "nvidia,tegra20-vip", },
 	{ .compatible = "nvidia,tegra20-vi", },
 #endif
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+	{ .compatible = "nvidia,tegra20-csi", },
+#endif
+#if defined(CONFIG_ARCH_TEGRA_3x_SOC)
+	{ .compatible = "nvidia,tegra30-csi", },
+#endif
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
 	{ .compatible = "nvidia,tegra210-csi", },
 	{ .compatible = "nvidia,tegra210-vi", },
diff --git a/drivers/staging/media/tegra-video/vip.c b/drivers/staging/media/tegra-video/vip.c
index 80cd3b1131259..9ff1f1750a154 100644
--- a/drivers/staging/media/tegra-video/vip.c
+++ b/drivers/staging/media/tegra-video/vip.c
@@ -264,7 +264,7 @@ static void tegra_vip_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id tegra_vip_of_id_table[] = {
-#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
 	{ .compatible = "nvidia,tegra20-vip", .data = &tegra20_vip_soc },
 #endif
 	{ }
diff --git a/drivers/staging/media/tegra-video/vip.h b/drivers/staging/media/tegra-video/vip.h
index fdded00447e44..563059cbf5b50 100644
--- a/drivers/staging/media/tegra-video/vip.h
+++ b/drivers/staging/media/tegra-video/vip.h
@@ -50,7 +50,7 @@ struct tegra_vip_soc {
 	const struct tegra_vip_ops *ops;
 };
 
-#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
 extern const struct tegra_vip_soc tegra20_vip_soc;
 #endif
 
diff --git a/drivers/staging/most/Documentation/driver_usage.txt b/drivers/staging/most/Documentation/driver_usage.txt
index 2fa8dea1da4d2..f1b6977f45430 100644
--- a/drivers/staging/most/Documentation/driver_usage.txt
+++ b/drivers/staging/most/Documentation/driver_usage.txt
@@ -101,11 +101,11 @@ following components are available
 	   Userspace can access the driver by means of character devices.
 
 	2) Networking
-	   Standard networking applications (e.g. iperf) can by used to access
+	   Standard networking applications (e.g. iperf) can be used to access
 	   the driver via the networking subsystem.
 
 	3) Video4Linux (v4l2)
-	   Standard video applications (e.g. VLC) can by used to access the
+	   Standard video applications (e.g. VLC) can be used to access the
 	   driver via the V4L subsystem.
 
 	4) Advanced Linux Sound Architecture (ALSA)
@@ -121,7 +121,7 @@ The driver is to be configured via configfs. Each loaded component kernel
 object (see section 1.3) registers a subsystem with configfs, which is used to
 configure and establish communication pathways (links) to attached devices on
 the bus. To do so, the user has to descend into the component's configuration
-directory and create a new directory (child config itmes). The name of this
+directory and create a new directory (child config items). The name of this
 directory will be used as a reference for the link and it will contain the
 following attributes:
 
@@ -129,7 +129,7 @@ following attributes:
 	  configure the buffer size for this channel
 	- subbuffer_size
 	  configure the sub-buffer size for this channel (needed for
-	  synchronous and isochrnous data)
+	  synchronous and isochronous data)
 	- num_buffers
 	  configure number of buffers used for this channel
 	- datatype
@@ -229,7 +229,7 @@ packets within one USB transaction. This renders
 bytes for padding.
 
 Note that at least (2 * subbuffer_size) bytes for isochronous data or
-(subbuffer_size * packts_per_xact) bytes for synchronous data need to
+(subbuffer_size * packets_per_xact) bytes for synchronous data need to
 be put in the transmission buffer and passed to the driver.
 
 Since adapter drivers are allowed to change a chosen configuration to best
diff --git a/drivers/staging/most/dim2/dim2.c b/drivers/staging/most/dim2/dim2.c
index 8d649d9204338..66617e89e0283 100644
--- a/drivers/staging/most/dim2/dim2.c
+++ b/drivers/staging/most/dim2/dim2.c
@@ -7,9 +7,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/module.h>
 #include <linux/printk.h>
-#include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
@@ -17,7 +15,6 @@
 #include <linux/io.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
-#include <linux/sched.h>
 #include <linux/kthread.h>
 #include <linux/most.h>
 #include <linux/of.h>
@@ -246,9 +243,9 @@ static void retrieve_netinfo(struct dim2_hdm *dev, struct mbo *mbo)
 {
 	u8 *data = mbo->virt_address;
 
-	pr_info("Node Address: 0x%03x\n", (u16)data[16] << 8 | data[17]);
+	dev_dbg(&dev->dev, "Node Address: 0x%03x\n", (u16)data[16] << 8 | data[17]);
 	dev->link_state = data[18];
-	pr_info("NIState: %d\n", dev->link_state);
+	dev_dbg(&dev->dev, "NIState: %d\n", dev->link_state);
 	memcpy(dev->mac_addrs, data + 19, 6);
 	dev->deliver_netinfo++;
 	wake_up_interruptible(&dev->netinfo_waitq);
@@ -472,13 +469,13 @@ static int configure_channel(struct most_interface *most_iface, int ch_idx,
 	case MOST_CH_CONTROL:
 		new_size = dim_norm_ctrl_async_buffer_size(buf_size);
 		if (new_size == 0) {
-			pr_err("%s: too small buffer size\n", hdm_ch->name);
+			dev_err(&dev->dev, "%s: too small buffer size\n", hdm_ch->name);
 			return -EINVAL;
 		}
 		ccfg->buffer_size = new_size;
 		if (new_size != buf_size)
-			pr_warn("%s: fixed buffer size (%d -> %d)\n",
-				hdm_ch->name, buf_size, new_size);
+			dev_warn(&dev->dev, "%s: fixed buffer size (%d -> %d)\n",
+				 hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
 		hal_ret = dim_init_control(&hdm_ch->ch, is_tx, ch_addr,
 					   is_tx ? new_size * 2 : new_size);
@@ -486,13 +483,13 @@ static int configure_channel(struct most_interface *most_iface, int ch_idx,
 	case MOST_CH_ASYNC:
 		new_size = dim_norm_ctrl_async_buffer_size(buf_size);
 		if (new_size == 0) {
-			pr_err("%s: too small buffer size\n", hdm_ch->name);
+			dev_err(&dev->dev, "%s: too small buffer size\n", hdm_ch->name);
 			return -EINVAL;
 		}
 		ccfg->buffer_size = new_size;
 		if (new_size != buf_size)
-			pr_warn("%s: fixed buffer size (%d -> %d)\n",
-				hdm_ch->name, buf_size, new_size);
+			dev_warn(&dev->dev, "%s: fixed buffer size (%d -> %d)\n",
+				 hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
 		hal_ret = dim_init_async(&hdm_ch->ch, is_tx, ch_addr,
 					 is_tx ? new_size * 2 : new_size);
@@ -500,41 +497,41 @@ static int configure_channel(struct most_interface *most_iface, int ch_idx,
 	case MOST_CH_ISOC:
 		new_size = dim_norm_isoc_buffer_size(buf_size, sub_size);
 		if (new_size == 0) {
-			pr_err("%s: invalid sub-buffer size or too small buffer size\n",
-			       hdm_ch->name);
+			dev_err(&dev->dev, "%s: invalid sub-buffer size or too small buffer size\n",
+				hdm_ch->name);
 			return -EINVAL;
 		}
 		ccfg->buffer_size = new_size;
 		if (new_size != buf_size)
-			pr_warn("%s: fixed buffer size (%d -> %d)\n",
-				hdm_ch->name, buf_size, new_size);
+			dev_warn(&dev->dev, "%s: fixed buffer size (%d -> %d)\n",
+				 hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
 		hal_ret = dim_init_isoc(&hdm_ch->ch, is_tx, ch_addr, sub_size);
 		break;
 	case MOST_CH_SYNC:
 		new_size = dim_norm_sync_buffer_size(buf_size, sub_size);
 		if (new_size == 0) {
-			pr_err("%s: invalid sub-buffer size or too small buffer size\n",
-			       hdm_ch->name);
+			dev_err(&dev->dev, "%s: invalid sub-buffer size or too small buffer size\n",
+				hdm_ch->name);
 			return -EINVAL;
 		}
 		ccfg->buffer_size = new_size;
 		if (new_size != buf_size)
-			pr_warn("%s: fixed buffer size (%d -> %d)\n",
-				hdm_ch->name, buf_size, new_size);
+			dev_warn(&dev->dev, "%s: fixed buffer size (%d -> %d)\n",
+				 hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
 		hal_ret = dim_init_sync(&hdm_ch->ch, is_tx, ch_addr, sub_size);
 		break;
 	default:
-		pr_err("%s: configure failed, bad channel type: %d\n",
-		       hdm_ch->name, ccfg->data_type);
+		dev_err(&dev->dev, "%s: configure failed, bad channel type: %d\n",
+			hdm_ch->name, ccfg->data_type);
 		return -EINVAL;
 	}
 
 	if (hal_ret != DIM_NO_ERROR) {
 		spin_unlock_irqrestore(&dim_lock, flags);
-		pr_err("%s: configure failed (%d), type: %d, is_tx: %d\n",
-		       hdm_ch->name, hal_ret, ccfg->data_type, (int)is_tx);
+		dev_err(&dev->dev, "%s: configure failed (%d), type: %d, is_tx: %d\n",
+			hdm_ch->name, hal_ret, ccfg->data_type, (int)is_tx);
 		return -ENODEV;
 	}
 
@@ -608,7 +605,7 @@ static void request_netinfo(struct most_interface *most_iface, int ch_idx,
 		return;
 
 	if (dev->atx_idx < 0) {
-		pr_err("Async Tx Not initialized\n");
+		dev_err(&dev->dev, "Async Tx Not initialized\n");
 		return;
 	}
 
@@ -657,7 +654,7 @@ static int poison_channel(struct most_interface *most_iface, int ch_idx)
 		dev->atx_idx = -1;
 	spin_unlock_irqrestore(&dim_lock, flags);
 	if (hal_ret != DIM_NO_ERROR) {
-		pr_err("HAL Failed to close channel %s\n", hdm_ch->name);
+		dev_err(&dev->dev, "HAL Failed to close channel %s\n", hdm_ch->name);
 		ret = -EFAULT;
 	}
 
@@ -799,8 +796,7 @@ static int dim2_probe(struct platform_device *pdev)
 			dev_fcnt = pdata->fcnt;
 	}
 
-	dev_info(&pdev->dev, "sync: num of frames per sub-buffer: %u\n",
-		 dev_fcnt);
+	dev_dbg(&pdev->dev, "sync: num of frames per sub-buffer: %u\n", dev_fcnt);
 	hal_ret = dim_startup(dev->io_base, dev->clk_speed, dev_fcnt);
 	if (hal_ret != DIM_NO_ERROR) {
 		dev_err(&pdev->dev, "dim_startup failed: %d\n", hal_ret);
@@ -925,28 +921,32 @@ static int fsl_mx6_enable(struct platform_device *pdev)
 	int ret;
 
 	dev->clk = devm_clk_get(&pdev->dev, "mlb");
-	if (IS_ERR_OR_NULL(dev->clk)) {
-		dev_err(&pdev->dev, "unable to get mlb clock\n");
-		return -EFAULT;
-	}
+	if (IS_ERR(dev->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk),
+				     "unable to get mlb clock\n");
 
 	ret = clk_prepare_enable(dev->clk);
 	if (ret) {
-		dev_err(&pdev->dev, "%s\n", "clk_prepare_enable failed");
+		dev_err(&pdev->dev, "clk_prepare_enable failed\n");
 		return ret;
 	}
 
 	if (dev->clk_speed >= CLK_2048FS) {
 		/* enable pll */
 		dev->clk_pll = devm_clk_get(&pdev->dev, "pll8_mlb");
-		if (IS_ERR_OR_NULL(dev->clk_pll)) {
-			dev_err(&pdev->dev, "unable to get mlb pll clock\n");
+		if (IS_ERR(dev->clk_pll)) {
 			clk_disable_unprepare(dev->clk);
-			return -EFAULT;
+			return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_pll),
+					     "unable to get mlb pll clock\n");
 		}
 
 		writel(0x888, dev->io_base + 0x38);
-		clk_prepare_enable(dev->clk_pll);
+		ret = clk_prepare_enable(dev->clk_pll);
+		if (ret) {
+			dev_err(&pdev->dev, "failed to enable pll clock\n");
+			clk_disable_unprepare(dev->clk);
+			return ret;
+		}
 	}
 
 	return 0;
@@ -975,7 +975,7 @@ static int rcar_gen2_enable(struct platform_device *pdev)
 
 	ret = clk_prepare_enable(dev->clk);
 	if (ret) {
-		dev_err(&pdev->dev, "%s\n", "clk_prepare_enable failed");
+		dev_err(&pdev->dev, "clk_prepare_enable failed\n");
 		return ret;
 	}
 
@@ -1020,7 +1020,7 @@ static int rcar_gen3_enable(struct platform_device *pdev)
 
 	ret = clk_prepare_enable(dev->clk);
 	if (ret) {
-		dev_err(&pdev->dev, "%s\n", "clk_prepare_enable failed");
+		dev_err(&pdev->dev, "clk_prepare_enable failed\n");
 		return ret;
 	}
 
diff --git a/drivers/staging/most/dim2/hal.c b/drivers/staging/most/dim2/hal.c
index 6abe3ab2b2cf4..38376d3e3f0d4 100644
--- a/drivers/staging/most/dim2/hal.c
+++ b/drivers/staging/most/dim2/hal.c
@@ -11,9 +11,8 @@
 #include "hal.h"
 #include "errors.h"
 #include "reg.h"
-#include <linux/stddef.h>
-#include <linux/kernel.h>
 #include <linux/io.h>
+#include <linux/kernel.h>
 
 /*
  * Size factor for isochronous DBR buffer.
@@ -45,8 +44,6 @@
 #define DBR_SIZE  (16 * 1024) /* specified by IP */
 #define DBR_BLOCK_SIZE  (DBR_SIZE / 32 / DBR_MAP_SIZE)
 
-#define ROUND_UP_TO(x, d)  (DIV_ROUND_UP(x, (d)) * (d))
-
 /* -------------------------------------------------------------------------- */
 /* generic helper functions and macros */
 
@@ -758,7 +755,7 @@ static u8 init_ctrl_async(struct dim_channel *ch, u8 type, u8 is_tx,
 		return DIM_INIT_ERR_CHANNEL_ADDRESS;
 
 	if (!ch->dbr_size)
-		ch->dbr_size = ROUND_UP_TO(hw_buffer_size, DBR_BLOCK_SIZE);
+		ch->dbr_size = round_up(hw_buffer_size, DBR_BLOCK_SIZE);
 	ch->dbr_addr = alloc_dbr(ch->dbr_size);
 	if (ch->dbr_addr >= DBR_SIZE)
 		return DIM_INIT_ERR_OUT_OF_MEMORY;
diff --git a/drivers/staging/most/net/net.c b/drivers/staging/most/net/net.c
index 1d1fe8bff7ee4..fffdb60cd230b 100644
--- a/drivers/staging/most/net/net.c
+++ b/drivers/staging/most/net/net.c
@@ -10,7 +10,6 @@
 #include <linux/module.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
-#include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/list.h>
 #include <linux/wait.h>
diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c
index e9af66a084483..952c5a849a563 100644
--- a/drivers/staging/nvec/nvec.c
+++ b/drivers/staging/nvec/nvec.c
@@ -659,8 +659,10 @@ static irqreturn_t nvec_interrupt(int irq, void *dev)
 			nvec_tx_set(nvec);
 			to_send = nvec->tx->data[0];
 			nvec->tx->pos = 1;
-			/* delay ACK due to AP20 HW Bug
-			   do not replace by usleep_range */
+			/*
+			 * delay ACK due to AP20 HW Bug
+			 * do not replace by usleep_range
+			 */
 			udelay(33);
 		} else if (status == (I2C_SL_IRQ)) {
 			nvec->rx->data[1] = received;
@@ -811,13 +813,12 @@ static int tegra_nvec_probe(struct platform_device *pdev)
 
 	nvec->irq = platform_get_irq(pdev, 0);
 	if (nvec->irq < 0)
-		return -ENODEV;
+		return nvec->irq;
 
 	i2c_clk = devm_clk_get(dev, "div-clk");
-	if (IS_ERR(i2c_clk)) {
-		dev_err(dev, "failed to get controller clock\n");
-		return -ENODEV;
-	}
+	if (IS_ERR(i2c_clk))
+		return dev_err_probe(dev, PTR_ERR(i2c_clk),
+				     "failed to get controller clock\n");
 
 	nvec->rst = devm_reset_control_get_exclusive(dev, "i2c");
 	if (IS_ERR(nvec->rst)) {
@@ -849,10 +850,8 @@ static int tegra_nvec_probe(struct platform_device *pdev)
 
 	err = devm_request_irq(dev, nvec->irq, nvec_interrupt, IRQF_NO_AUTOEN,
 			       "nvec", nvec);
-	if (err) {
-		dev_err(dev, "couldn't request irq\n");
-		return -ENODEV;
-	}
+	if (err)
+		return dev_err_probe(dev, err, "couldn't request irq\n");
 
 	tegra_init_i2c_slave(nvec);
 
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 965330eec80a8..d0b43d50b83ce 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -535,7 +535,7 @@ void cvm_oct_rx_shutdown(void)
 			cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0);
 
 		/* Free the interrupt handler */
-		free_irq(oct_rx_group[i].irq, cvm_oct_device);
+		free_irq(oct_rx_group[i].irq, &oct_rx_group[i].napi);
 
 		netif_napi_del(&oct_rx_group[i].napi);
 	}
diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c
index f5bbedac6a653..14d10659bce71 100644
--- a/drivers/staging/octeon/ethernet-tx.c
+++ b/drivers/staging/octeon/ethernet-tx.c
@@ -449,8 +449,6 @@ netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev)
 	case QUEUE_CORE:
 		__skb_queue_tail(&priv->tx_free_list[qos], skb);
 		break;
-	default:
-		BUG();
 	}
 
 	while (skb_to_free > 0) {
diff --git a/drivers/staging/octeon/octeon-stubs.h b/drivers/staging/octeon/octeon-stubs.h
index 35b5078ba51e9..291eaffd2543a 100644
--- a/drivers/staging/octeon/octeon-stubs.h
+++ b/drivers/staging/octeon/octeon-stubs.h
@@ -124,65 +124,65 @@ union cvmx_pip_wqe_word2 {
 
 union cvmx_pip_wqe_word0 {
 	struct {
-		uint64_t next_ptr:40;
-		uint8_t unused;
+		u64 next_ptr:40;
+		u8 unused;
 		__wsum hw_chksum;
 	} cn38xx;
 	struct {
-		uint64_t pknd:6;        /* 0..5 */
-		uint64_t unused2:2;     /* 6..7 */
-		uint64_t bpid:6;        /* 8..13 */
-		uint64_t unused1:18;    /* 14..31 */
-		uint64_t l2ptr:8;       /* 32..39 */
-		uint64_t l3ptr:8;       /* 40..47 */
-		uint64_t unused0:8;     /* 48..55 */
-		uint64_t l4ptr:8;       /* 56..63 */
+		u64 pknd:6;        /* 0..5 */
+		u64 unused2:2;     /* 6..7 */
+		u64 bpid:6;        /* 8..13 */
+		u64 unused1:18;    /* 14..31 */
+		u64 l2ptr:8;       /* 32..39 */
+		u64 l3ptr:8;       /* 40..47 */
+		u64 unused0:8;     /* 48..55 */
+		u64 l4ptr:8;       /* 56..63 */
 	} cn68xx;
 };
 
 union cvmx_wqe_word0 {
-	uint64_t u64;
+	u64 u64;
 	union cvmx_pip_wqe_word0 pip;
 };
 
 union cvmx_wqe_word1 {
-	uint64_t u64;
+	u64 u64;
 	struct {
-		uint64_t tag:32;
-		uint64_t tag_type:2;
-		uint64_t varies:14;
-		uint64_t len:16;
+		u64 tag:32;
+		u64 tag_type:2;
+		u64 varies:14;
+		u64 len:16;
 	};
 	struct {
-		uint64_t tag:32;
-		uint64_t tag_type:2;
-		uint64_t zero_2:3;
-		uint64_t grp:6;
-		uint64_t zero_1:1;
-		uint64_t qos:3;
-		uint64_t zero_0:1;
-		uint64_t len:16;
+		u64 tag:32;
+		u64 tag_type:2;
+		u64 zero_2:3;
+		u64 grp:6;
+		u64 zero_1:1;
+		u64 qos:3;
+		u64 zero_0:1;
+		u64 len:16;
 	} cn68xx;
 	struct {
-		uint64_t tag:32;
-		uint64_t tag_type:2;
-		uint64_t zero_2:1;
-		uint64_t grp:4;
-		uint64_t qos:3;
-		uint64_t ipprt:6;
-		uint64_t len:16;
+		u64 tag:32;
+		u64 tag_type:2;
+		u64 zero_2:1;
+		u64 grp:4;
+		u64 qos:3;
+		u64 ipprt:6;
+		u64 len:16;
 	} cn38xx;
 };
 
 union cvmx_buf_ptr {
 	void *ptr;
-	uint64_t u64;
+	u64 u64;
 	struct {
-		uint64_t i:1;
-		uint64_t back:4;
-		uint64_t pool:3;
-		uint64_t size:16;
-		uint64_t addr:40;
+		u64 i:1;
+		u64 back:4;
+		u64 pool:3;
+		u64 size:16;
+		u64 addr:40;
 	} s;
 };
 
@@ -191,16 +191,16 @@ struct cvmx_wqe {
 	union cvmx_wqe_word1 word1;
 	union cvmx_pip_wqe_word2 word2;
 	union cvmx_buf_ptr packet_ptr;
-	uint8_t packet_data[96];
+	u8 packet_data[96];
 };
 
 union cvmx_helper_link_info {
-	uint64_t u64;
+	u64 u64;
 	struct {
-		uint64_t reserved_20_63:44;
-		uint64_t link_up:1;	    /**< Is the physical link up? */
-		uint64_t full_duplex:1;	    /**< 1 if the link is full duplex */
-		uint64_t speed:18;	    /**< Speed of the link in Mbps */
+		u64 reserved_20_63:44;
+		u64 link_up:1;	    /**< Is the physical link up? */
+		u64 full_duplex:1;	    /**< 1 if the link is full duplex */
+		u64 speed:18;	    /**< Speed of the link in Mbps */
 	} s;
 };
 
@@ -264,921 +264,921 @@ enum cvmx_pow_tag_type {
 };
 
 union cvmx_ipd_ctl_status {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_ipd_ctl_status_s {
-		uint64_t reserved_18_63:46;
-		uint64_t use_sop:1;
-		uint64_t rst_done:1;
-		uint64_t clken:1;
-		uint64_t no_wptr:1;
-		uint64_t pq_apkt:1;
-		uint64_t pq_nabuf:1;
-		uint64_t ipd_full:1;
-		uint64_t pkt_off:1;
-		uint64_t len_m8:1;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_18_63:46;
+		u64 use_sop:1;
+		u64 rst_done:1;
+		u64 clken:1;
+		u64 no_wptr:1;
+		u64 pq_apkt:1;
+		u64 pq_nabuf:1;
+		u64 ipd_full:1;
+		u64 pkt_off:1;
+		u64 len_m8:1;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} s;
 	struct cvmx_ipd_ctl_status_cn30xx {
-		uint64_t reserved_10_63:54;
-		uint64_t len_m8:1;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_10_63:54;
+		u64 len_m8:1;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} cn30xx;
 	struct cvmx_ipd_ctl_status_cn38xxp2 {
-		uint64_t reserved_9_63:55;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_9_63:55;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} cn38xxp2;
 	struct cvmx_ipd_ctl_status_cn50xx {
-		uint64_t reserved_15_63:49;
-		uint64_t no_wptr:1;
-		uint64_t pq_apkt:1;
-		uint64_t pq_nabuf:1;
-		uint64_t ipd_full:1;
-		uint64_t pkt_off:1;
-		uint64_t len_m8:1;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_15_63:49;
+		u64 no_wptr:1;
+		u64 pq_apkt:1;
+		u64 pq_nabuf:1;
+		u64 ipd_full:1;
+		u64 pkt_off:1;
+		u64 len_m8:1;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} cn50xx;
 	struct cvmx_ipd_ctl_status_cn58xx {
-		uint64_t reserved_12_63:52;
-		uint64_t ipd_full:1;
-		uint64_t pkt_off:1;
-		uint64_t len_m8:1;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_12_63:52;
+		u64 ipd_full:1;
+		u64 pkt_off:1;
+		u64 len_m8:1;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} cn58xx;
 	struct cvmx_ipd_ctl_status_cn63xxp1 {
-		uint64_t reserved_16_63:48;
-		uint64_t clken:1;
-		uint64_t no_wptr:1;
-		uint64_t pq_apkt:1;
-		uint64_t pq_nabuf:1;
-		uint64_t ipd_full:1;
-		uint64_t pkt_off:1;
-		uint64_t len_m8:1;
-		uint64_t reset:1;
-		uint64_t addpkt:1;
-		uint64_t naddbuf:1;
-		uint64_t pkt_lend:1;
-		uint64_t wqe_lend:1;
-		uint64_t pbp_en:1;
-		uint64_t opc_mode:2;
-		uint64_t ipd_en:1;
+		u64 reserved_16_63:48;
+		u64 clken:1;
+		u64 no_wptr:1;
+		u64 pq_apkt:1;
+		u64 pq_nabuf:1;
+		u64 ipd_full:1;
+		u64 pkt_off:1;
+		u64 len_m8:1;
+		u64 reset:1;
+		u64 addpkt:1;
+		u64 naddbuf:1;
+		u64 pkt_lend:1;
+		u64 wqe_lend:1;
+		u64 pbp_en:1;
+		u64 opc_mode:2;
+		u64 ipd_en:1;
 	} cn63xxp1;
 };
 
 union cvmx_ipd_sub_port_fcs {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_ipd_sub_port_fcs_s {
-		uint64_t port_bit:32;
-		uint64_t reserved_32_35:4;
-		uint64_t port_bit2:4;
-		uint64_t reserved_40_63:24;
+		u64 port_bit:32;
+		u64 reserved_32_35:4;
+		u64 port_bit2:4;
+		u64 reserved_40_63:24;
 	} s;
 	struct cvmx_ipd_sub_port_fcs_cn30xx {
-		uint64_t port_bit:3;
-		uint64_t reserved_3_63:61;
+		u64 port_bit:3;
+		u64 reserved_3_63:61;
 	} cn30xx;
 	struct cvmx_ipd_sub_port_fcs_cn38xx {
-		uint64_t port_bit:32;
-		uint64_t reserved_32_63:32;
+		u64 port_bit:32;
+		u64 reserved_32_63:32;
 	} cn38xx;
 };
 
 union cvmx_ipd_sub_port_qos_cnt {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_ipd_sub_port_qos_cnt_s {
-		uint64_t cnt:32;
-		uint64_t port_qos:9;
-		uint64_t reserved_41_63:23;
+		u64 cnt:32;
+		u64 port_qos:9;
+		u64 reserved_41_63:23;
 	} s;
 };
 
 typedef struct {
-	uint32_t dropped_octets;
-	uint32_t dropped_packets;
-	uint32_t pci_raw_packets;
-	uint32_t octets;
-	uint32_t packets;
-	uint32_t multicast_packets;
-	uint32_t broadcast_packets;
-	uint32_t len_64_packets;
-	uint32_t len_65_127_packets;
-	uint32_t len_128_255_packets;
-	uint32_t len_256_511_packets;
-	uint32_t len_512_1023_packets;
-	uint32_t len_1024_1518_packets;
-	uint32_t len_1519_max_packets;
-	uint32_t fcs_align_err_packets;
-	uint32_t runt_packets;
-	uint32_t runt_crc_packets;
-	uint32_t oversize_packets;
-	uint32_t oversize_crc_packets;
-	uint32_t inb_packets;
-	uint64_t inb_octets;
-	uint16_t inb_errors;
+	u32 dropped_octets;
+	u32 dropped_packets;
+	u32 pci_raw_packets;
+	u32 octets;
+	u32 packets;
+	u32 multicast_packets;
+	u32 broadcast_packets;
+	u32 len_64_packets;
+	u32 len_65_127_packets;
+	u32 len_128_255_packets;
+	u32 len_256_511_packets;
+	u32 len_512_1023_packets;
+	u32 len_1024_1518_packets;
+	u32 len_1519_max_packets;
+	u32 fcs_align_err_packets;
+	u32 runt_packets;
+	u32 runt_crc_packets;
+	u32 oversize_packets;
+	u32 oversize_crc_packets;
+	u32 inb_packets;
+	u64 inb_octets;
+	u16 inb_errors;
 } cvmx_pip_port_status_t;
 
 typedef struct {
-	uint32_t packets;
-	uint64_t octets;
-	uint64_t doorbell;
+	u32 packets;
+	u64 octets;
+	u64 doorbell;
 } cvmx_pko_port_status_t;
 
 union cvmx_pip_frm_len_chkx {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_pip_frm_len_chkx_s {
-		uint64_t reserved_32_63:32;
-		uint64_t maxlen:16;
-		uint64_t minlen:16;
+		u64 reserved_32_63:32;
+		u64 maxlen:16;
+		u64 minlen:16;
 	} s;
 };
 
 union cvmx_gmxx_rxx_frm_ctl {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_gmxx_rxx_frm_ctl_s {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t vlan_len:1;
-		uint64_t pad_len:1;
-		uint64_t pre_align:1;
-		uint64_t null_dis:1;
-		uint64_t reserved_11_11:1;
-		uint64_t ptp_mode:1;
-		uint64_t reserved_13_63:51;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 vlan_len:1;
+		u64 pad_len:1;
+		u64 pre_align:1;
+		u64 null_dis:1;
+		u64 reserved_11_11:1;
+		u64 ptp_mode:1;
+		u64 reserved_13_63:51;
 	} s;
 	struct cvmx_gmxx_rxx_frm_ctl_cn30xx {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t vlan_len:1;
-		uint64_t pad_len:1;
-		uint64_t reserved_9_63:55;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 vlan_len:1;
+		u64 pad_len:1;
+		u64 reserved_9_63:55;
 	} cn30xx;
 	struct cvmx_gmxx_rxx_frm_ctl_cn31xx {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t vlan_len:1;
-		uint64_t reserved_8_63:56;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 vlan_len:1;
+		u64 reserved_8_63:56;
 	} cn31xx;
 	struct cvmx_gmxx_rxx_frm_ctl_cn50xx {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t reserved_7_8:2;
-		uint64_t pre_align:1;
-		uint64_t null_dis:1;
-		uint64_t reserved_11_63:53;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 reserved_7_8:2;
+		u64 pre_align:1;
+		u64 null_dis:1;
+		u64 reserved_11_63:53;
 	} cn50xx;
 	struct cvmx_gmxx_rxx_frm_ctl_cn56xxp1 {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t reserved_7_8:2;
-		uint64_t pre_align:1;
-		uint64_t reserved_10_63:54;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 reserved_7_8:2;
+		u64 pre_align:1;
+		u64 reserved_10_63:54;
 	} cn56xxp1;
 	struct cvmx_gmxx_rxx_frm_ctl_cn58xx {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t vlan_len:1;
-		uint64_t pad_len:1;
-		uint64_t pre_align:1;
-		uint64_t null_dis:1;
-		uint64_t reserved_11_63:53;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 vlan_len:1;
+		u64 pad_len:1;
+		u64 pre_align:1;
+		u64 null_dis:1;
+		u64 reserved_11_63:53;
 	} cn58xx;
 	struct cvmx_gmxx_rxx_frm_ctl_cn61xx {
-		uint64_t pre_chk:1;
-		uint64_t pre_strp:1;
-		uint64_t ctl_drp:1;
-		uint64_t ctl_bck:1;
-		uint64_t ctl_mcst:1;
-		uint64_t ctl_smac:1;
-		uint64_t pre_free:1;
-		uint64_t reserved_7_8:2;
-		uint64_t pre_align:1;
-		uint64_t null_dis:1;
-		uint64_t reserved_11_11:1;
-		uint64_t ptp_mode:1;
-		uint64_t reserved_13_63:51;
+		u64 pre_chk:1;
+		u64 pre_strp:1;
+		u64 ctl_drp:1;
+		u64 ctl_bck:1;
+		u64 ctl_mcst:1;
+		u64 ctl_smac:1;
+		u64 pre_free:1;
+		u64 reserved_7_8:2;
+		u64 pre_align:1;
+		u64 null_dis:1;
+		u64 reserved_11_11:1;
+		u64 ptp_mode:1;
+		u64 reserved_13_63:51;
 	} cn61xx;
 };
 
 union cvmx_gmxx_rxx_int_reg {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_gmxx_rxx_int_reg_s {
-		uint64_t minerr:1;
-		uint64_t carext:1;
-		uint64_t maxerr:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t alnerr:1;
-		uint64_t lenerr:1;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t niberr:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t phy_link:1;
-		uint64_t phy_spd:1;
-		uint64_t phy_dupx:1;
-		uint64_t pause_drp:1;
-		uint64_t loc_fault:1;
-		uint64_t rem_fault:1;
-		uint64_t bad_seq:1;
-		uint64_t bad_term:1;
-		uint64_t unsop:1;
-		uint64_t uneop:1;
-		uint64_t undat:1;
-		uint64_t hg2fld:1;
-		uint64_t hg2cc:1;
-		uint64_t reserved_29_63:35;
+		u64 minerr:1;
+		u64 carext:1;
+		u64 maxerr:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 alnerr:1;
+		u64 lenerr:1;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 niberr:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 phy_link:1;
+		u64 phy_spd:1;
+		u64 phy_dupx:1;
+		u64 pause_drp:1;
+		u64 loc_fault:1;
+		u64 rem_fault:1;
+		u64 bad_seq:1;
+		u64 bad_term:1;
+		u64 unsop:1;
+		u64 uneop:1;
+		u64 undat:1;
+		u64 hg2fld:1;
+		u64 hg2cc:1;
+		u64 reserved_29_63:35;
 	} s;
 	struct cvmx_gmxx_rxx_int_reg_cn30xx {
-		uint64_t minerr:1;
-		uint64_t carext:1;
-		uint64_t maxerr:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t alnerr:1;
-		uint64_t lenerr:1;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t niberr:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t phy_link:1;
-		uint64_t phy_spd:1;
-		uint64_t phy_dupx:1;
-		uint64_t reserved_19_63:45;
+		u64 minerr:1;
+		u64 carext:1;
+		u64 maxerr:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 alnerr:1;
+		u64 lenerr:1;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 niberr:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 phy_link:1;
+		u64 phy_spd:1;
+		u64 phy_dupx:1;
+		u64 reserved_19_63:45;
 	} cn30xx;
 	struct cvmx_gmxx_rxx_int_reg_cn50xx {
-		uint64_t reserved_0_0:1;
-		uint64_t carext:1;
-		uint64_t reserved_2_2:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t alnerr:1;
-		uint64_t reserved_6_6:1;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t niberr:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t phy_link:1;
-		uint64_t phy_spd:1;
-		uint64_t phy_dupx:1;
-		uint64_t pause_drp:1;
-		uint64_t reserved_20_63:44;
+		u64 reserved_0_0:1;
+		u64 carext:1;
+		u64 reserved_2_2:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 alnerr:1;
+		u64 reserved_6_6:1;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 niberr:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 phy_link:1;
+		u64 phy_spd:1;
+		u64 phy_dupx:1;
+		u64 pause_drp:1;
+		u64 reserved_20_63:44;
 	} cn50xx;
 	struct cvmx_gmxx_rxx_int_reg_cn52xx {
-		uint64_t reserved_0_0:1;
-		uint64_t carext:1;
-		uint64_t reserved_2_2:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t reserved_5_6:2;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t reserved_9_9:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t reserved_16_18:3;
-		uint64_t pause_drp:1;
-		uint64_t loc_fault:1;
-		uint64_t rem_fault:1;
-		uint64_t bad_seq:1;
-		uint64_t bad_term:1;
-		uint64_t unsop:1;
-		uint64_t uneop:1;
-		uint64_t undat:1;
-		uint64_t hg2fld:1;
-		uint64_t hg2cc:1;
-		uint64_t reserved_29_63:35;
+		u64 reserved_0_0:1;
+		u64 carext:1;
+		u64 reserved_2_2:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 reserved_5_6:2;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 reserved_9_9:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 reserved_16_18:3;
+		u64 pause_drp:1;
+		u64 loc_fault:1;
+		u64 rem_fault:1;
+		u64 bad_seq:1;
+		u64 bad_term:1;
+		u64 unsop:1;
+		u64 uneop:1;
+		u64 undat:1;
+		u64 hg2fld:1;
+		u64 hg2cc:1;
+		u64 reserved_29_63:35;
 	} cn52xx;
 	struct cvmx_gmxx_rxx_int_reg_cn56xxp1 {
-		uint64_t reserved_0_0:1;
-		uint64_t carext:1;
-		uint64_t reserved_2_2:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t reserved_5_6:2;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t reserved_9_9:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t reserved_16_18:3;
-		uint64_t pause_drp:1;
-		uint64_t loc_fault:1;
-		uint64_t rem_fault:1;
-		uint64_t bad_seq:1;
-		uint64_t bad_term:1;
-		uint64_t unsop:1;
-		uint64_t uneop:1;
-		uint64_t undat:1;
-		uint64_t reserved_27_63:37;
+		u64 reserved_0_0:1;
+		u64 carext:1;
+		u64 reserved_2_2:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 reserved_5_6:2;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 reserved_9_9:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 reserved_16_18:3;
+		u64 pause_drp:1;
+		u64 loc_fault:1;
+		u64 rem_fault:1;
+		u64 bad_seq:1;
+		u64 bad_term:1;
+		u64 unsop:1;
+		u64 uneop:1;
+		u64 undat:1;
+		u64 reserved_27_63:37;
 	} cn56xxp1;
 	struct cvmx_gmxx_rxx_int_reg_cn58xx {
-		uint64_t minerr:1;
-		uint64_t carext:1;
-		uint64_t maxerr:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t alnerr:1;
-		uint64_t lenerr:1;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t niberr:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t phy_link:1;
-		uint64_t phy_spd:1;
-		uint64_t phy_dupx:1;
-		uint64_t pause_drp:1;
-		uint64_t reserved_20_63:44;
+		u64 minerr:1;
+		u64 carext:1;
+		u64 maxerr:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 alnerr:1;
+		u64 lenerr:1;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 niberr:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 phy_link:1;
+		u64 phy_spd:1;
+		u64 phy_dupx:1;
+		u64 pause_drp:1;
+		u64 reserved_20_63:44;
 	} cn58xx;
 	struct cvmx_gmxx_rxx_int_reg_cn61xx {
-		uint64_t minerr:1;
-		uint64_t carext:1;
-		uint64_t reserved_2_2:1;
-		uint64_t jabber:1;
-		uint64_t fcserr:1;
-		uint64_t reserved_5_6:2;
-		uint64_t rcverr:1;
-		uint64_t skperr:1;
-		uint64_t reserved_9_9:1;
-		uint64_t ovrerr:1;
-		uint64_t pcterr:1;
-		uint64_t rsverr:1;
-		uint64_t falerr:1;
-		uint64_t coldet:1;
-		uint64_t ifgerr:1;
-		uint64_t reserved_16_18:3;
-		uint64_t pause_drp:1;
-		uint64_t loc_fault:1;
-		uint64_t rem_fault:1;
-		uint64_t bad_seq:1;
-		uint64_t bad_term:1;
-		uint64_t unsop:1;
-		uint64_t uneop:1;
-		uint64_t undat:1;
-		uint64_t hg2fld:1;
-		uint64_t hg2cc:1;
-		uint64_t reserved_29_63:35;
+		u64 minerr:1;
+		u64 carext:1;
+		u64 reserved_2_2:1;
+		u64 jabber:1;
+		u64 fcserr:1;
+		u64 reserved_5_6:2;
+		u64 rcverr:1;
+		u64 skperr:1;
+		u64 reserved_9_9:1;
+		u64 ovrerr:1;
+		u64 pcterr:1;
+		u64 rsverr:1;
+		u64 falerr:1;
+		u64 coldet:1;
+		u64 ifgerr:1;
+		u64 reserved_16_18:3;
+		u64 pause_drp:1;
+		u64 loc_fault:1;
+		u64 rem_fault:1;
+		u64 bad_seq:1;
+		u64 bad_term:1;
+		u64 unsop:1;
+		u64 uneop:1;
+		u64 undat:1;
+		u64 hg2fld:1;
+		u64 hg2cc:1;
+		u64 reserved_29_63:35;
 	} cn61xx;
 };
 
 union cvmx_gmxx_prtx_cfg {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_gmxx_prtx_cfg_s {
-		uint64_t reserved_22_63:42;
-		uint64_t pknd:6;
-		uint64_t reserved_14_15:2;
-		uint64_t tx_idle:1;
-		uint64_t rx_idle:1;
-		uint64_t reserved_9_11:3;
-		uint64_t speed_msb:1;
-		uint64_t reserved_4_7:4;
-		uint64_t slottime:1;
-		uint64_t duplex:1;
-		uint64_t speed:1;
-		uint64_t en:1;
+		u64 reserved_22_63:42;
+		u64 pknd:6;
+		u64 reserved_14_15:2;
+		u64 tx_idle:1;
+		u64 rx_idle:1;
+		u64 reserved_9_11:3;
+		u64 speed_msb:1;
+		u64 reserved_4_7:4;
+		u64 slottime:1;
+		u64 duplex:1;
+		u64 speed:1;
+		u64 en:1;
 	} s;
 	struct cvmx_gmxx_prtx_cfg_cn30xx {
-		uint64_t reserved_4_63:60;
-		uint64_t slottime:1;
-		uint64_t duplex:1;
-		uint64_t speed:1;
-		uint64_t en:1;
+		u64 reserved_4_63:60;
+		u64 slottime:1;
+		u64 duplex:1;
+		u64 speed:1;
+		u64 en:1;
 	} cn30xx;
 	struct cvmx_gmxx_prtx_cfg_cn52xx {
-		uint64_t reserved_14_63:50;
-		uint64_t tx_idle:1;
-		uint64_t rx_idle:1;
-		uint64_t reserved_9_11:3;
-		uint64_t speed_msb:1;
-		uint64_t reserved_4_7:4;
-		uint64_t slottime:1;
-		uint64_t duplex:1;
-		uint64_t speed:1;
-		uint64_t en:1;
+		u64 reserved_14_63:50;
+		u64 tx_idle:1;
+		u64 rx_idle:1;
+		u64 reserved_9_11:3;
+		u64 speed_msb:1;
+		u64 reserved_4_7:4;
+		u64 slottime:1;
+		u64 duplex:1;
+		u64 speed:1;
+		u64 en:1;
 	} cn52xx;
 };
 
 union cvmx_gmxx_rxx_adr_ctl {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_gmxx_rxx_adr_ctl_s {
-		uint64_t reserved_4_63:60;
-		uint64_t cam_mode:1;
-		uint64_t mcst:2;
-		uint64_t bcst:1;
+		u64 reserved_4_63:60;
+		u64 cam_mode:1;
+		u64 mcst:2;
+		u64 bcst:1;
 	} s;
 };
 
 union cvmx_pip_prt_tagx {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_pip_prt_tagx_s {
-		uint64_t reserved_54_63:10;
-		uint64_t portadd_en:1;
-		uint64_t inc_hwchk:1;
-		uint64_t reserved_50_51:2;
-		uint64_t grptagbase_msb:2;
-		uint64_t reserved_46_47:2;
-		uint64_t grptagmask_msb:2;
-		uint64_t reserved_42_43:2;
-		uint64_t grp_msb:2;
-		uint64_t grptagbase:4;
-		uint64_t grptagmask:4;
-		uint64_t grptag:1;
-		uint64_t grptag_mskip:1;
-		uint64_t tag_mode:2;
-		uint64_t inc_vs:2;
-		uint64_t inc_vlan:1;
-		uint64_t inc_prt_flag:1;
-		uint64_t ip6_dprt_flag:1;
-		uint64_t ip4_dprt_flag:1;
-		uint64_t ip6_sprt_flag:1;
-		uint64_t ip4_sprt_flag:1;
-		uint64_t ip6_nxth_flag:1;
-		uint64_t ip4_pctl_flag:1;
-		uint64_t ip6_dst_flag:1;
-		uint64_t ip4_dst_flag:1;
-		uint64_t ip6_src_flag:1;
-		uint64_t ip4_src_flag:1;
-		uint64_t tcp6_tag_type:2;
-		uint64_t tcp4_tag_type:2;
-		uint64_t ip6_tag_type:2;
-		uint64_t ip4_tag_type:2;
-		uint64_t non_tag_type:2;
-		uint64_t grp:4;
+		u64 reserved_54_63:10;
+		u64 portadd_en:1;
+		u64 inc_hwchk:1;
+		u64 reserved_50_51:2;
+		u64 grptagbase_msb:2;
+		u64 reserved_46_47:2;
+		u64 grptagmask_msb:2;
+		u64 reserved_42_43:2;
+		u64 grp_msb:2;
+		u64 grptagbase:4;
+		u64 grptagmask:4;
+		u64 grptag:1;
+		u64 grptag_mskip:1;
+		u64 tag_mode:2;
+		u64 inc_vs:2;
+		u64 inc_vlan:1;
+		u64 inc_prt_flag:1;
+		u64 ip6_dprt_flag:1;
+		u64 ip4_dprt_flag:1;
+		u64 ip6_sprt_flag:1;
+		u64 ip4_sprt_flag:1;
+		u64 ip6_nxth_flag:1;
+		u64 ip4_pctl_flag:1;
+		u64 ip6_dst_flag:1;
+		u64 ip4_dst_flag:1;
+		u64 ip6_src_flag:1;
+		u64 ip4_src_flag:1;
+		u64 tcp6_tag_type:2;
+		u64 tcp4_tag_type:2;
+		u64 ip6_tag_type:2;
+		u64 ip4_tag_type:2;
+		u64 non_tag_type:2;
+		u64 grp:4;
 	} s;
 	struct cvmx_pip_prt_tagx_cn30xx {
-		uint64_t reserved_40_63:24;
-		uint64_t grptagbase:4;
-		uint64_t grptagmask:4;
-		uint64_t grptag:1;
-		uint64_t reserved_30_30:1;
-		uint64_t tag_mode:2;
-		uint64_t inc_vs:2;
-		uint64_t inc_vlan:1;
-		uint64_t inc_prt_flag:1;
-		uint64_t ip6_dprt_flag:1;
-		uint64_t ip4_dprt_flag:1;
-		uint64_t ip6_sprt_flag:1;
-		uint64_t ip4_sprt_flag:1;
-		uint64_t ip6_nxth_flag:1;
-		uint64_t ip4_pctl_flag:1;
-		uint64_t ip6_dst_flag:1;
-		uint64_t ip4_dst_flag:1;
-		uint64_t ip6_src_flag:1;
-		uint64_t ip4_src_flag:1;
-		uint64_t tcp6_tag_type:2;
-		uint64_t tcp4_tag_type:2;
-		uint64_t ip6_tag_type:2;
-		uint64_t ip4_tag_type:2;
-		uint64_t non_tag_type:2;
-		uint64_t grp:4;
+		u64 reserved_40_63:24;
+		u64 grptagbase:4;
+		u64 grptagmask:4;
+		u64 grptag:1;
+		u64 reserved_30_30:1;
+		u64 tag_mode:2;
+		u64 inc_vs:2;
+		u64 inc_vlan:1;
+		u64 inc_prt_flag:1;
+		u64 ip6_dprt_flag:1;
+		u64 ip4_dprt_flag:1;
+		u64 ip6_sprt_flag:1;
+		u64 ip4_sprt_flag:1;
+		u64 ip6_nxth_flag:1;
+		u64 ip4_pctl_flag:1;
+		u64 ip6_dst_flag:1;
+		u64 ip4_dst_flag:1;
+		u64 ip6_src_flag:1;
+		u64 ip4_src_flag:1;
+		u64 tcp6_tag_type:2;
+		u64 tcp4_tag_type:2;
+		u64 ip6_tag_type:2;
+		u64 ip4_tag_type:2;
+		u64 non_tag_type:2;
+		u64 grp:4;
 	} cn30xx;
 	struct cvmx_pip_prt_tagx_cn50xx {
-		uint64_t reserved_40_63:24;
-		uint64_t grptagbase:4;
-		uint64_t grptagmask:4;
-		uint64_t grptag:1;
-		uint64_t grptag_mskip:1;
-		uint64_t tag_mode:2;
-		uint64_t inc_vs:2;
-		uint64_t inc_vlan:1;
-		uint64_t inc_prt_flag:1;
-		uint64_t ip6_dprt_flag:1;
-		uint64_t ip4_dprt_flag:1;
-		uint64_t ip6_sprt_flag:1;
-		uint64_t ip4_sprt_flag:1;
-		uint64_t ip6_nxth_flag:1;
-		uint64_t ip4_pctl_flag:1;
-		uint64_t ip6_dst_flag:1;
-		uint64_t ip4_dst_flag:1;
-		uint64_t ip6_src_flag:1;
-		uint64_t ip4_src_flag:1;
-		uint64_t tcp6_tag_type:2;
-		uint64_t tcp4_tag_type:2;
-		uint64_t ip6_tag_type:2;
-		uint64_t ip4_tag_type:2;
-		uint64_t non_tag_type:2;
-		uint64_t grp:4;
+		u64 reserved_40_63:24;
+		u64 grptagbase:4;
+		u64 grptagmask:4;
+		u64 grptag:1;
+		u64 grptag_mskip:1;
+		u64 tag_mode:2;
+		u64 inc_vs:2;
+		u64 inc_vlan:1;
+		u64 inc_prt_flag:1;
+		u64 ip6_dprt_flag:1;
+		u64 ip4_dprt_flag:1;
+		u64 ip6_sprt_flag:1;
+		u64 ip4_sprt_flag:1;
+		u64 ip6_nxth_flag:1;
+		u64 ip4_pctl_flag:1;
+		u64 ip6_dst_flag:1;
+		u64 ip4_dst_flag:1;
+		u64 ip6_src_flag:1;
+		u64 ip4_src_flag:1;
+		u64 tcp6_tag_type:2;
+		u64 tcp4_tag_type:2;
+		u64 ip6_tag_type:2;
+		u64 ip4_tag_type:2;
+		u64 non_tag_type:2;
+		u64 grp:4;
 	} cn50xx;
 };
 
 union cvmx_spxx_int_reg {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_spxx_int_reg_s {
-		uint64_t reserved_32_63:32;
-		uint64_t spf:1;
-		uint64_t reserved_12_30:19;
-		uint64_t calerr:1;
-		uint64_t syncerr:1;
-		uint64_t diperr:1;
-		uint64_t tpaovr:1;
-		uint64_t rsverr:1;
-		uint64_t drwnng:1;
-		uint64_t clserr:1;
-		uint64_t spiovr:1;
-		uint64_t reserved_2_3:2;
-		uint64_t abnorm:1;
-		uint64_t prtnxa:1;
+		u64 reserved_32_63:32;
+		u64 spf:1;
+		u64 reserved_12_30:19;
+		u64 calerr:1;
+		u64 syncerr:1;
+		u64 diperr:1;
+		u64 tpaovr:1;
+		u64 rsverr:1;
+		u64 drwnng:1;
+		u64 clserr:1;
+		u64 spiovr:1;
+		u64 reserved_2_3:2;
+		u64 abnorm:1;
+		u64 prtnxa:1;
 	} s;
 };
 
 union cvmx_spxx_int_msk {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_spxx_int_msk_s {
-		uint64_t reserved_12_63:52;
-		uint64_t calerr:1;
-		uint64_t syncerr:1;
-		uint64_t diperr:1;
-		uint64_t tpaovr:1;
-		uint64_t rsverr:1;
-		uint64_t drwnng:1;
-		uint64_t clserr:1;
-		uint64_t spiovr:1;
-		uint64_t reserved_2_3:2;
-		uint64_t abnorm:1;
-		uint64_t prtnxa:1;
+		u64 reserved_12_63:52;
+		u64 calerr:1;
+		u64 syncerr:1;
+		u64 diperr:1;
+		u64 tpaovr:1;
+		u64 rsverr:1;
+		u64 drwnng:1;
+		u64 clserr:1;
+		u64 spiovr:1;
+		u64 reserved_2_3:2;
+		u64 abnorm:1;
+		u64 prtnxa:1;
 	} s;
 };
 
 union cvmx_pow_wq_int {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_pow_wq_int_s {
-		uint64_t wq_int:16;
-		uint64_t iq_dis:16;
-		uint64_t reserved_32_63:32;
+		u64 wq_int:16;
+		u64 iq_dis:16;
+		u64 reserved_32_63:32;
 	} s;
 };
 
 union cvmx_sso_wq_int_thrx {
-	uint64_t u64;
+	u64 u64;
 	struct {
-		uint64_t iq_thr:12;
-		uint64_t reserved_12_13:2;
-		uint64_t ds_thr:12;
-		uint64_t reserved_26_27:2;
-		uint64_t tc_thr:4;
-		uint64_t tc_en:1;
-		uint64_t reserved_33_63:31;
+		u64 iq_thr:12;
+		u64 reserved_12_13:2;
+		u64 ds_thr:12;
+		u64 reserved_26_27:2;
+		u64 tc_thr:4;
+		u64 tc_en:1;
+		u64 reserved_33_63:31;
 	} s;
 };
 
 union cvmx_stxx_int_reg {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_stxx_int_reg_s {
-		uint64_t reserved_9_63:55;
-		uint64_t syncerr:1;
-		uint64_t frmerr:1;
-		uint64_t unxfrm:1;
-		uint64_t nosync:1;
-		uint64_t diperr:1;
-		uint64_t datovr:1;
-		uint64_t ovrbst:1;
-		uint64_t calpar1:1;
-		uint64_t calpar0:1;
+		u64 reserved_9_63:55;
+		u64 syncerr:1;
+		u64 frmerr:1;
+		u64 unxfrm:1;
+		u64 nosync:1;
+		u64 diperr:1;
+		u64 datovr:1;
+		u64 ovrbst:1;
+		u64 calpar1:1;
+		u64 calpar0:1;
 	} s;
 };
 
 union cvmx_stxx_int_msk {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_stxx_int_msk_s {
-		uint64_t reserved_8_63:56;
-		uint64_t frmerr:1;
-		uint64_t unxfrm:1;
-		uint64_t nosync:1;
-		uint64_t diperr:1;
-		uint64_t datovr:1;
-		uint64_t ovrbst:1;
-		uint64_t calpar1:1;
-		uint64_t calpar0:1;
+		u64 reserved_8_63:56;
+		u64 frmerr:1;
+		u64 unxfrm:1;
+		u64 nosync:1;
+		u64 diperr:1;
+		u64 datovr:1;
+		u64 ovrbst:1;
+		u64 calpar1:1;
+		u64 calpar0:1;
 	} s;
 };
 
 union cvmx_pow_wq_int_pc {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_pow_wq_int_pc_s {
-		uint64_t reserved_0_7:8;
-		uint64_t pc_thr:20;
-		uint64_t reserved_28_31:4;
-		uint64_t pc:28;
-		uint64_t reserved_60_63:4;
+		u64 reserved_0_7:8;
+		u64 pc_thr:20;
+		u64 reserved_28_31:4;
+		u64 pc:28;
+		u64 reserved_60_63:4;
 	} s;
 };
 
 union cvmx_pow_wq_int_thrx {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_pow_wq_int_thrx_s {
-		uint64_t reserved_29_63:35;
-		uint64_t tc_en:1;
-		uint64_t tc_thr:4;
-		uint64_t reserved_23_23:1;
-		uint64_t ds_thr:11;
-		uint64_t reserved_11_11:1;
-		uint64_t iq_thr:11;
+		u64 reserved_29_63:35;
+		u64 tc_en:1;
+		u64 tc_thr:4;
+		u64 reserved_23_23:1;
+		u64 ds_thr:11;
+		u64 reserved_11_11:1;
+		u64 iq_thr:11;
 	} s;
 	struct cvmx_pow_wq_int_thrx_cn30xx {
-		uint64_t reserved_29_63:35;
-		uint64_t tc_en:1;
-		uint64_t tc_thr:4;
-		uint64_t reserved_18_23:6;
-		uint64_t ds_thr:6;
-		uint64_t reserved_6_11:6;
-		uint64_t iq_thr:6;
+		u64 reserved_29_63:35;
+		u64 tc_en:1;
+		u64 tc_thr:4;
+		u64 reserved_18_23:6;
+		u64 ds_thr:6;
+		u64 reserved_6_11:6;
+		u64 iq_thr:6;
 	} cn30xx;
 	struct cvmx_pow_wq_int_thrx_cn31xx {
-		uint64_t reserved_29_63:35;
-		uint64_t tc_en:1;
-		uint64_t tc_thr:4;
-		uint64_t reserved_20_23:4;
-		uint64_t ds_thr:8;
-		uint64_t reserved_8_11:4;
-		uint64_t iq_thr:8;
+		u64 reserved_29_63:35;
+		u64 tc_en:1;
+		u64 tc_thr:4;
+		u64 reserved_20_23:4;
+		u64 ds_thr:8;
+		u64 reserved_8_11:4;
+		u64 iq_thr:8;
 	} cn31xx;
 	struct cvmx_pow_wq_int_thrx_cn52xx {
-		uint64_t reserved_29_63:35;
-		uint64_t tc_en:1;
-		uint64_t tc_thr:4;
-		uint64_t reserved_21_23:3;
-		uint64_t ds_thr:9;
-		uint64_t reserved_9_11:3;
-		uint64_t iq_thr:9;
+		u64 reserved_29_63:35;
+		u64 tc_en:1;
+		u64 tc_thr:4;
+		u64 reserved_21_23:3;
+		u64 ds_thr:9;
+		u64 reserved_9_11:3;
+		u64 iq_thr:9;
 	} cn52xx;
 	struct cvmx_pow_wq_int_thrx_cn63xx {
-		uint64_t reserved_29_63:35;
-		uint64_t tc_en:1;
-		uint64_t tc_thr:4;
-		uint64_t reserved_22_23:2;
-		uint64_t ds_thr:10;
-		uint64_t reserved_10_11:2;
-		uint64_t iq_thr:10;
+		u64 reserved_29_63:35;
+		u64 tc_en:1;
+		u64 tc_thr:4;
+		u64 reserved_22_23:2;
+		u64 ds_thr:10;
+		u64 reserved_10_11:2;
+		u64 iq_thr:10;
 	} cn63xx;
 };
 
 union cvmx_npi_rsl_int_blocks {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_npi_rsl_int_blocks_s {
-		uint64_t reserved_32_63:32;
-		uint64_t rint_31:1;
-		uint64_t iob:1;
-		uint64_t reserved_28_29:2;
-		uint64_t rint_27:1;
-		uint64_t rint_26:1;
-		uint64_t rint_25:1;
-		uint64_t rint_24:1;
-		uint64_t asx1:1;
-		uint64_t asx0:1;
-		uint64_t rint_21:1;
-		uint64_t pip:1;
-		uint64_t spx1:1;
-		uint64_t spx0:1;
-		uint64_t lmc:1;
-		uint64_t l2c:1;
-		uint64_t rint_15:1;
-		uint64_t reserved_13_14:2;
-		uint64_t pow:1;
-		uint64_t tim:1;
-		uint64_t pko:1;
-		uint64_t ipd:1;
-		uint64_t rint_8:1;
-		uint64_t zip:1;
-		uint64_t dfa:1;
-		uint64_t fpa:1;
-		uint64_t key:1;
-		uint64_t npi:1;
-		uint64_t gmx1:1;
-		uint64_t gmx0:1;
-		uint64_t mio:1;
+		u64 reserved_32_63:32;
+		u64 rint_31:1;
+		u64 iob:1;
+		u64 reserved_28_29:2;
+		u64 rint_27:1;
+		u64 rint_26:1;
+		u64 rint_25:1;
+		u64 rint_24:1;
+		u64 asx1:1;
+		u64 asx0:1;
+		u64 rint_21:1;
+		u64 pip:1;
+		u64 spx1:1;
+		u64 spx0:1;
+		u64 lmc:1;
+		u64 l2c:1;
+		u64 rint_15:1;
+		u64 reserved_13_14:2;
+		u64 pow:1;
+		u64 tim:1;
+		u64 pko:1;
+		u64 ipd:1;
+		u64 rint_8:1;
+		u64 zip:1;
+		u64 dfa:1;
+		u64 fpa:1;
+		u64 key:1;
+		u64 npi:1;
+		u64 gmx1:1;
+		u64 gmx0:1;
+		u64 mio:1;
 	} s;
 	struct cvmx_npi_rsl_int_blocks_cn30xx {
-		uint64_t reserved_32_63:32;
-		uint64_t rint_31:1;
-		uint64_t iob:1;
-		uint64_t rint_29:1;
-		uint64_t rint_28:1;
-		uint64_t rint_27:1;
-		uint64_t rint_26:1;
-		uint64_t rint_25:1;
-		uint64_t rint_24:1;
-		uint64_t asx1:1;
-		uint64_t asx0:1;
-		uint64_t rint_21:1;
-		uint64_t pip:1;
-		uint64_t spx1:1;
-		uint64_t spx0:1;
-		uint64_t lmc:1;
-		uint64_t l2c:1;
-		uint64_t rint_15:1;
-		uint64_t rint_14:1;
-		uint64_t usb:1;
-		uint64_t pow:1;
-		uint64_t tim:1;
-		uint64_t pko:1;
-		uint64_t ipd:1;
-		uint64_t rint_8:1;
-		uint64_t zip:1;
-		uint64_t dfa:1;
-		uint64_t fpa:1;
-		uint64_t key:1;
-		uint64_t npi:1;
-		uint64_t gmx1:1;
-		uint64_t gmx0:1;
-		uint64_t mio:1;
+		u64 reserved_32_63:32;
+		u64 rint_31:1;
+		u64 iob:1;
+		u64 rint_29:1;
+		u64 rint_28:1;
+		u64 rint_27:1;
+		u64 rint_26:1;
+		u64 rint_25:1;
+		u64 rint_24:1;
+		u64 asx1:1;
+		u64 asx0:1;
+		u64 rint_21:1;
+		u64 pip:1;
+		u64 spx1:1;
+		u64 spx0:1;
+		u64 lmc:1;
+		u64 l2c:1;
+		u64 rint_15:1;
+		u64 rint_14:1;
+		u64 usb:1;
+		u64 pow:1;
+		u64 tim:1;
+		u64 pko:1;
+		u64 ipd:1;
+		u64 rint_8:1;
+		u64 zip:1;
+		u64 dfa:1;
+		u64 fpa:1;
+		u64 key:1;
+		u64 npi:1;
+		u64 gmx1:1;
+		u64 gmx0:1;
+		u64 mio:1;
 	} cn30xx;
 	struct cvmx_npi_rsl_int_blocks_cn38xx {
-		uint64_t reserved_32_63:32;
-		uint64_t rint_31:1;
-		uint64_t iob:1;
-		uint64_t rint_29:1;
-		uint64_t rint_28:1;
-		uint64_t rint_27:1;
-		uint64_t rint_26:1;
-		uint64_t rint_25:1;
-		uint64_t rint_24:1;
-		uint64_t asx1:1;
-		uint64_t asx0:1;
-		uint64_t rint_21:1;
-		uint64_t pip:1;
-		uint64_t spx1:1;
-		uint64_t spx0:1;
-		uint64_t lmc:1;
-		uint64_t l2c:1;
-		uint64_t rint_15:1;
-		uint64_t rint_14:1;
-		uint64_t rint_13:1;
-		uint64_t pow:1;
-		uint64_t tim:1;
-		uint64_t pko:1;
-		uint64_t ipd:1;
-		uint64_t rint_8:1;
-		uint64_t zip:1;
-		uint64_t dfa:1;
-		uint64_t fpa:1;
-		uint64_t key:1;
-		uint64_t npi:1;
-		uint64_t gmx1:1;
-		uint64_t gmx0:1;
-		uint64_t mio:1;
+		u64 reserved_32_63:32;
+		u64 rint_31:1;
+		u64 iob:1;
+		u64 rint_29:1;
+		u64 rint_28:1;
+		u64 rint_27:1;
+		u64 rint_26:1;
+		u64 rint_25:1;
+		u64 rint_24:1;
+		u64 asx1:1;
+		u64 asx0:1;
+		u64 rint_21:1;
+		u64 pip:1;
+		u64 spx1:1;
+		u64 spx0:1;
+		u64 lmc:1;
+		u64 l2c:1;
+		u64 rint_15:1;
+		u64 rint_14:1;
+		u64 rint_13:1;
+		u64 pow:1;
+		u64 tim:1;
+		u64 pko:1;
+		u64 ipd:1;
+		u64 rint_8:1;
+		u64 zip:1;
+		u64 dfa:1;
+		u64 fpa:1;
+		u64 key:1;
+		u64 npi:1;
+		u64 gmx1:1;
+		u64 gmx0:1;
+		u64 mio:1;
 	} cn38xx;
 	struct cvmx_npi_rsl_int_blocks_cn50xx {
-		uint64_t reserved_31_63:33;
-		uint64_t iob:1;
-		uint64_t lmc1:1;
-		uint64_t agl:1;
-		uint64_t reserved_24_27:4;
-		uint64_t asx1:1;
-		uint64_t asx0:1;
-		uint64_t reserved_21_21:1;
-		uint64_t pip:1;
-		uint64_t spx1:1;
-		uint64_t spx0:1;
-		uint64_t lmc:1;
-		uint64_t l2c:1;
-		uint64_t reserved_15_15:1;
-		uint64_t rad:1;
-		uint64_t usb:1;
-		uint64_t pow:1;
-		uint64_t tim:1;
-		uint64_t pko:1;
-		uint64_t ipd:1;
-		uint64_t reserved_8_8:1;
-		uint64_t zip:1;
-		uint64_t dfa:1;
-		uint64_t fpa:1;
-		uint64_t key:1;
-		uint64_t npi:1;
-		uint64_t gmx1:1;
-		uint64_t gmx0:1;
-		uint64_t mio:1;
+		u64 reserved_31_63:33;
+		u64 iob:1;
+		u64 lmc1:1;
+		u64 agl:1;
+		u64 reserved_24_27:4;
+		u64 asx1:1;
+		u64 asx0:1;
+		u64 reserved_21_21:1;
+		u64 pip:1;
+		u64 spx1:1;
+		u64 spx0:1;
+		u64 lmc:1;
+		u64 l2c:1;
+		u64 reserved_15_15:1;
+		u64 rad:1;
+		u64 usb:1;
+		u64 pow:1;
+		u64 tim:1;
+		u64 pko:1;
+		u64 ipd:1;
+		u64 reserved_8_8:1;
+		u64 zip:1;
+		u64 dfa:1;
+		u64 fpa:1;
+		u64 key:1;
+		u64 npi:1;
+		u64 gmx1:1;
+		u64 gmx0:1;
+		u64 mio:1;
 	} cn50xx;
 };
 
 union cvmx_pko_command_word0 {
-	uint64_t u64;
+	u64 u64;
 	struct {
-		uint64_t total_bytes:16;
-		uint64_t segs:6;
-		uint64_t dontfree:1;
-		uint64_t ignore_i:1;
-		uint64_t ipoffp1:7;
-		uint64_t gather:1;
-		uint64_t rsp:1;
-		uint64_t wqp:1;
-		uint64_t n2:1;
-		uint64_t le:1;
-		uint64_t reg0:11;
-		uint64_t subone0:1;
-		uint64_t reg1:11;
-		uint64_t subone1:1;
-		uint64_t size0:2;
-		uint64_t size1:2;
+		u64 total_bytes:16;
+		u64 segs:6;
+		u64 dontfree:1;
+		u64 ignore_i:1;
+		u64 ipoffp1:7;
+		u64 gather:1;
+		u64 rsp:1;
+		u64 wqp:1;
+		u64 n2:1;
+		u64 le:1;
+		u64 reg0:11;
+		u64 subone0:1;
+		u64 reg1:11;
+		u64 subone1:1;
+		u64 size0:2;
+		u64 size1:2;
 	} s;
 };
 
 union cvmx_ciu_timx {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_ciu_timx_s {
-		uint64_t reserved_37_63:27;
-		uint64_t one_shot:1;
-		uint64_t len:36;
+		u64 reserved_37_63:27;
+		u64 one_shot:1;
+		u64 len:36;
 	} s;
 };
 
 union cvmx_gmxx_rxx_rx_inbnd {
-	uint64_t u64;
+	u64 u64;
 	struct cvmx_gmxx_rxx_rx_inbnd_s {
-		uint64_t status:1;
-		uint64_t speed:2;
-		uint64_t duplex:1;
-		uint64_t reserved_4_63:60;
+		u64 status:1;
+		u64 speed:2;
+		u64 duplex:1;
+		u64 reserved_4_63:60;
 	} s;
 };
 
@@ -1196,12 +1196,12 @@ static inline void cvmx_fau_atomic_write32(enum cvmx_fau_reg_32 reg,
 					   int32_t value)
 { }
 
-static inline uint64_t cvmx_scratch_read64(uint64_t address)
+static inline u64 cvmx_scratch_read64(u64 address)
 {
 	return 0;
 }
 
-static inline void cvmx_scratch_write64(uint64_t address, uint64_t value)
+static inline void cvmx_scratch_write64(u64 address, u64 value)
 { }
 
 static inline int cvmx_wqe_get_grp(struct cvmx_wqe *work)
@@ -1209,7 +1209,7 @@ static inline int cvmx_wqe_get_grp(struct cvmx_wqe *work)
 	return 0;
 }
 
-static inline void *cvmx_phys_to_ptr(uint64_t physical_address)
+static inline void *cvmx_phys_to_ptr(u64 physical_address)
 {
 	return (void *)(uintptr_t)(physical_address);
 }
@@ -1232,12 +1232,12 @@ static inline int cvmx_helper_get_interface_index_num(int ipd_port)
 static inline void cvmx_fpa_enable(void)
 { }
 
-static inline uint64_t cvmx_read_csr(uint64_t csr_addr)
+static inline u64 cvmx_read_csr(u64 csr_addr)
 {
 	return 0;
 }
 
-static inline void cvmx_write_csr(uint64_t csr_addr, uint64_t val)
+static inline void cvmx_write_csr(u64 csr_addr, u64 val)
 { }
 
 static inline int cvmx_helper_setup_red(int pass_thresh, int drop_thresh)
@@ -1245,13 +1245,13 @@ static inline int cvmx_helper_setup_red(int pass_thresh, int drop_thresh)
 	return 0;
 }
 
-static inline void *cvmx_fpa_alloc(uint64_t pool)
+static inline void *cvmx_fpa_alloc(u64 pool)
 {
 	return NULL;
 }
 
-static inline void cvmx_fpa_free(void *ptr, uint64_t pool,
-				 uint64_t num_cache_lines)
+static inline void cvmx_fpa_free(void *ptr, u64 pool,
+				 u64 num_cache_lines)
 { }
 
 static inline int octeon_is_simulation(void)
@@ -1259,11 +1259,11 @@ static inline int octeon_is_simulation(void)
 	return 1;
 }
 
-static inline void cvmx_pip_get_port_status(uint64_t port_num, uint64_t clear,
+static inline void cvmx_pip_get_port_status(u64 port_num, u64 clear,
 					    cvmx_pip_port_status_t *status)
 { }
 
-static inline void cvmx_pko_get_port_status(uint64_t port_num, uint64_t clear,
+static inline void cvmx_pko_get_port_status(u64 port_num, u64 clear,
 					    cvmx_pko_port_status_t *status)
 { }
 
@@ -1369,7 +1369,7 @@ static inline int cvmx_spi_restart_interface(int interface,
 	return 0;
 }
 
-static inline void cvmx_fau_async_fetch_and_add32(uint64_t scraddr,
+static inline void cvmx_fau_async_fetch_and_add32(u64 scraddr,
 						  enum cvmx_fau_reg_32 reg,
 						  int32_t value)
 { }
@@ -1382,12 +1382,12 @@ static inline union cvmx_gmxx_rxx_rx_inbnd cvmx_spi4000_check_speed(int interfac
 	return r;
 }
 
-static inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue,
+static inline void cvmx_pko_send_packet_prepare(u64 port, u64 queue,
 						cvmx_pko_lock_t use_locking)
 { }
 
-static inline cvmx_pko_status_t cvmx_pko_send_packet_finish(uint64_t port,
-		uint64_t queue, union cvmx_pko_command_word0 pko_command,
+static inline cvmx_pko_status_t cvmx_pko_send_packet_finish(u64 port,
+		u64 queue, union cvmx_pko_command_word0 pko_command,
 		union cvmx_buf_ptr packet, cvmx_pko_lock_t use_locking)
 {
 	return 0;
@@ -1407,9 +1407,9 @@ static inline int cvmx_wqe_get_qos(struct cvmx_wqe *work)
 static inline void cvmx_wqe_set_grp(struct cvmx_wqe *work, int grp)
 { }
 
-static inline void cvmx_pow_work_submit(struct cvmx_wqe *wqp, uint32_t tag,
+static inline void cvmx_pow_work_submit(struct cvmx_wqe *wqp, u32 tag,
 					enum cvmx_pow_tag_type tag_type,
-					uint64_t qos, uint64_t grp)
+					u64 qos, u64 grp)
 { }
 
 #define CVMX_ASXX_RX_CLK_SETX(a, b)	((a) + (b))
diff --git a/drivers/staging/rtl8723bs/core/rtw_ap.c b/drivers/staging/rtl8723bs/core/rtw_ap.c
index 864cd8b6d1f19..4cdcdddf6b33e 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ap.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ap.c
@@ -367,7 +367,6 @@ void add_ratid(struct adapter *padapter, struct sta_info *psta, u8 rssi_level)
 void update_bmc_sta(struct adapter *padapter)
 {
 	unsigned char network_type;
-	int support_rate_num = 0;
 	unsigned int tx_ra_bitmap = 0;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
@@ -388,12 +387,10 @@ void update_bmc_sta(struct adapter *padapter)
 
 		psta->ieee8021x_blocked = false;
 
-		memset((void *)&psta->sta_stats, 0, sizeof(struct stainfo_stats));
+		memset(&psta->sta_stats, 0, sizeof(struct stainfo_stats));
 
 		/* prepare for add_ratid */
-		support_rate_num = rtw_get_rateset_len((u8 *)&pcur_network->supported_rates);
 		network_type = rtw_check_network_type((u8 *)&pcur_network->supported_rates,
-						      support_rate_num,
 						      pcur_network->configuration.ds_config
 		);
 		if (is_supported_tx_cck(network_type)) {
@@ -545,7 +542,7 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 
 	/* todo: init other variables */
 
-	memset((void *)&psta->sta_stats, 0, sizeof(struct stainfo_stats));
+	memset(&psta->sta_stats, 0, sizeof(struct stainfo_stats));
 
 	/* add ratid */
 	/* add_ratid(padapter, psta); move to ap_sta_info_defer_update() */
@@ -800,7 +797,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 {
 	int ret = _SUCCESS;
 	u8 *p;
-	u8 *pHT_caps_ie = NULL;
+	u8 *ht_caps_ie = NULL;
 	u8 *pHT_info_ie = NULL;
 	struct sta_info *psta = NULL;
 	u16 cap, ht_cap = false;
@@ -885,12 +882,10 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		       WLAN_EID_EXT_SUPP_RATES,
 		       &ie_len,
 		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
-	if (p) {
+	if (p)
 		memcpy(support_rate + support_rate_num, p + 2, ie_len);
-		support_rate_num += ie_len;
-	}
 
-	network_type = rtw_check_network_type(support_rate, support_rate_num, channel);
+	network_type = rtw_check_network_type(support_rate, channel);
 
 	rtw_set_supported_rate(pbss_network->supported_rates, network_type);
 
@@ -1006,17 +1001,17 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		u8 max_rx_ampdu_factor = 0;
 		struct ieee80211_ht_cap *pht_cap = (struct ieee80211_ht_cap *)(p + 2);
 
-		pHT_caps_ie = p;
+		ht_caps_ie = p;
 
 		ht_cap = true;
 		network_type |= WIRELESS_11_24N;
 
 		rtw_ht_use_default_setting(padapter);
 
-		if (pmlmepriv->htpriv.sgi_20m == false)
+		if (!pmlmepriv->htpriv.sgi_20m)
 			pht_cap->cap_info &= cpu_to_le16(~(IEEE80211_HT_CAP_SGI_20));
 
-		if (pmlmepriv->htpriv.sgi_40m == false)
+		if (!pmlmepriv->htpriv.sgi_40m)
 			pht_cap->cap_info &= cpu_to_le16(~(IEEE80211_HT_CAP_SGI_40));
 
 		if (!TEST_FLAG(pmlmepriv->htpriv.ldpc_cap, LDPC_HT_ENABLE_RX))
@@ -1094,7 +1089,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		if (pregistrypriv->ampdu_enable == 1)
 			pmlmepriv->htpriv.ampdu_enable = true;
 
-		HT_caps_handler(padapter, (struct ndis_80211_var_ie *)pHT_caps_ie);
+		HT_caps_handler(padapter, (struct ndis_80211_var_ie *)ht_caps_ie);
 
 		HT_info_handler(padapter, (struct ndis_80211_var_ie *)pHT_info_ie);
 	}
@@ -1156,7 +1151,7 @@ int rtw_acl_add_sta(struct adapter *padapter, u8 *addr)
 		paclnode = list_entry(plist, struct rtw_wlan_acl_node, list);
 
 		if (!memcmp(paclnode->addr, addr, ETH_ALEN)) {
-			if (paclnode->valid == true) {
+			if (paclnode->valid) {
 				added = true;
 				break;
 			}
@@ -1439,7 +1434,7 @@ static void update_bcn_wps_ie(struct adapter *padapter)
 
 	remainder_ielen = ielen - wps_offset - wps_ielen;
 
-	if (premainder_ie && remainder_ielen)
+	if (remainder_ielen)
 		pbackup_remainder_ie = kmemdup(premainder_ie, remainder_ielen, GFP_ATOMIC);
 
 	wps_ielen = (uint)pwps_ie_src[1];/* to get ie data len */
@@ -1573,8 +1568,8 @@ static int rtw_ht_operation_update(struct adapter *padapter)
 	if (pmlmepriv->htpriv.ht_option)
 		return 0;
 
-	if (!(pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT)
-	    && pmlmepriv->num_sta_ht_no_gf) {
+	if (!(pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT) &&
+	    pmlmepriv->num_sta_ht_no_gf) {
 		pmlmepriv->ht_op_mode |=
 			IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT;
 		op_mode_changes++;
@@ -1780,8 +1775,8 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 	if (psta->no_short_preamble_set) {
 		psta->no_short_preamble_set = 0;
 		pmlmepriv->num_sta_no_short_preamble--;
-		if (pmlmeext->cur_wireless_mode > WIRELESS_11B
-		    && pmlmepriv->num_sta_no_short_preamble == 0){
+		if (pmlmeext->cur_wireless_mode > WIRELESS_11B &&
+		    pmlmepriv->num_sta_no_short_preamble == 0){
 			beacon_updated = true;
 			update_beacon(padapter, 0xFF, NULL, true);
 		}
@@ -1799,8 +1794,8 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 	if (psta->no_short_slot_time_set) {
 		psta->no_short_slot_time_set = 0;
 		pmlmepriv->num_sta_no_short_slot_time--;
-		if (pmlmeext->cur_wireless_mode > WIRELESS_11B
-		    && pmlmepriv->num_sta_no_short_slot_time == 0){
+		if (pmlmeext->cur_wireless_mode > WIRELESS_11B &&
+		    pmlmepriv->num_sta_no_short_slot_time == 0){
 			beacon_updated = true;
 			update_beacon(padapter, 0xFF, NULL, true);
 		}
diff --git a/drivers/staging/rtl8723bs/core/rtw_btcoex.c b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
index f4b19ef7b3416..0191a943f0a45 100644
--- a/drivers/staging/rtl8723bs/core/rtw_btcoex.c
+++ b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
@@ -54,7 +54,6 @@ void rtw_btcoex_LPS_Enter(struct adapter *padapter)
 	struct pwrctrl_priv *pwrpriv;
 	u8 lps_val;
 
-
 	pwrpriv = adapter_to_pwrctl(padapter);
 
 	pwrpriv->bpower_saving = true;
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index abb84f8aecbeb..c1185c25ed369 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -8,6 +8,7 @@
 #include <hal_btcoex.h>
 #include <linux/jiffies.h>
 #include <linux/align.h>
+#include <linux/delay.h>
 
 static struct _cmd_callback rtw_cmd_callback[] = {
 	{GEN_CMD_CODE(_Read_MACREG), NULL}, /*0*/
@@ -214,7 +215,7 @@ void _rtw_free_evt_priv(struct	evt_priv *pevtpriv)
 {
 	_cancel_workitem_sync(&pevtpriv->c2h_wk);
 	while (pevtpriv->c2h_wk_alive)
-		msleep(10);
+		fsleep(10 * USEC_PER_MSEC);
 
 	while (!rtw_cbuf_empty(pevtpriv->c2h_queue)) {
 		void *c2h = rtw_cbuf_pop(pevtpriv->c2h_queue);
@@ -1125,14 +1126,18 @@ static void collect_traffic_statistics(struct adapter *padapter)
 	pdvobjpriv->traffic_stat.cur_rx_tp = (u32)(pdvobjpriv->traffic_stat.cur_rx_bytes * 8 / 2 / 1024 / 1024);
 }
 
-u8 traffic_status_watchdog(struct adapter *padapter, u8 from_timer)
+bool traffic_status_watchdog(struct adapter *padapter, bool from_timer)
 {
-	u8 bEnterPS = false;
-	u16 BusyThresholdHigh = 25;
-	u16 BusyThresholdLow = 10;
-	u16 BusyThreshold = BusyThresholdHigh;
-	u8 bBusyTraffic = false, bTxBusyTraffic = false, bRxBusyTraffic = false;
-	u8 bHigherBusyTraffic = false, bHigherBusyRxTraffic = false, bHigherBusyTxTraffic = false;
+	bool should_enter_ps = false;
+	u16 busy_threshold_high = 25;
+	u16 busy_threshold_low = 10;
+	u16 busy_threshold = busy_threshold_high;
+	bool busy_traffic = false;
+	bool tx_busy_traffic = false;
+	bool rx_busy_traffic = false;
+	bool higher_busy_traffic = false;
+	bool higher_busy_rx_traffic = false;
+	bool higher_busy_tx_traffic = false;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	collect_traffic_statistics(padapter);
@@ -1142,57 +1147,60 @@ u8 traffic_status_watchdog(struct adapter *padapter, u8 from_timer)
 	/*  */
 	if ((check_fwstate(pmlmepriv, _FW_LINKED))
 		/*&& !MgntInitAdapterInProgress(pMgntInfo)*/) {
-		/*  if we raise bBusyTraffic in last watchdog, using lower threshold. */
-		if (pmlmepriv->LinkDetectInfo.bBusyTraffic)
-			BusyThreshold = BusyThresholdLow;
+		/*  if we raise busy_traffic in last watchdog, using lower threshold. */
+		if (pmlmepriv->link_detect_info.busy_traffic)
+			busy_threshold = busy_threshold_low;
 
-		if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > BusyThreshold ||
-		    pmlmepriv->LinkDetectInfo.NumTxOkInPeriod > BusyThreshold) {
-			bBusyTraffic = true;
+		if (pmlmepriv->link_detect_info.num_rx_ok_in_period > busy_threshold ||
+		    pmlmepriv->link_detect_info.num_tx_ok_in_period > busy_threshold) {
+			busy_traffic = true;
 
-			if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > pmlmepriv->LinkDetectInfo.NumTxOkInPeriod)
-				bRxBusyTraffic = true;
+			if (pmlmepriv->link_detect_info.num_rx_ok_in_period >
+			    pmlmepriv->link_detect_info.num_tx_ok_in_period)
+				rx_busy_traffic = true;
 			else
-				bTxBusyTraffic = true;
+				tx_busy_traffic = true;
 		}
 
 		/*  Higher Tx/Rx data. */
-		if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > 4000 ||
-		    pmlmepriv->LinkDetectInfo.NumTxOkInPeriod > 4000) {
-			bHigherBusyTraffic = true;
+		if (pmlmepriv->link_detect_info.num_rx_ok_in_period > 4000 ||
+		    pmlmepriv->link_detect_info.num_tx_ok_in_period > 4000) {
+			higher_busy_traffic = true;
 
-			if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > pmlmepriv->LinkDetectInfo.NumTxOkInPeriod)
-				bHigherBusyRxTraffic = true;
+			if (pmlmepriv->link_detect_info.num_rx_ok_in_period >
+			    pmlmepriv->link_detect_info.num_tx_ok_in_period)
+				higher_busy_rx_traffic = true;
 			else
-				bHigherBusyTxTraffic = true;
+				higher_busy_tx_traffic = true;
 		}
 
 		/*  check traffic for  powersaving. */
-		if (((pmlmepriv->LinkDetectInfo.NumRxUnicastOkInPeriod + pmlmepriv->LinkDetectInfo.NumTxOkInPeriod) > 8) ||
-		    (pmlmepriv->LinkDetectInfo.NumRxUnicastOkInPeriod > 2)) {
-			bEnterPS = false;
+		if (((pmlmepriv->link_detect_info.num_rx_unicast_ok_in_period +
+		      pmlmepriv->link_detect_info.num_tx_ok_in_period) > 8) ||
+		    (pmlmepriv->link_detect_info.num_rx_unicast_ok_in_period > 2)) {
+			should_enter_ps = false;
 
-			if (bBusyTraffic) {
-				if (pmlmepriv->LinkDetectInfo.TrafficTransitionCount <= 4)
-					pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 4;
+			if (busy_traffic) {
+				if (pmlmepriv->link_detect_info.traffic_transition_count <= 4)
+					pmlmepriv->link_detect_info.traffic_transition_count = 4;
 
-				pmlmepriv->LinkDetectInfo.TrafficTransitionCount++;
+				pmlmepriv->link_detect_info.traffic_transition_count++;
 
-				if (pmlmepriv->LinkDetectInfo.TrafficTransitionCount > 30/*TrafficTransitionLevel*/)
-					pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 30;
+				if (pmlmepriv->link_detect_info.traffic_transition_count > 30)
+					pmlmepriv->link_detect_info.traffic_transition_count = 30;
 			}
 		} else {
-			if (pmlmepriv->LinkDetectInfo.TrafficTransitionCount >= 2)
-				pmlmepriv->LinkDetectInfo.TrafficTransitionCount -= 2;
+			if (pmlmepriv->link_detect_info.traffic_transition_count >= 2)
+				pmlmepriv->link_detect_info.traffic_transition_count -= 2;
 			else
-				pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
+				pmlmepriv->link_detect_info.traffic_transition_count = 0;
 
-			if (pmlmepriv->LinkDetectInfo.TrafficTransitionCount == 0)
-				bEnterPS = true;
+			if (pmlmepriv->link_detect_info.traffic_transition_count == 0)
+				should_enter_ps = true;
 		}
 
 		/*  LeisurePS only work in infra mode. */
-		if (bEnterPS) {
+		if (should_enter_ps) {
 			if (!from_timer)
 				LPS_Enter(padapter, "TRAFFIC_IDLE");
 		} else {
@@ -1212,17 +1220,17 @@ u8 traffic_status_watchdog(struct adapter *padapter, u8 from_timer)
 			LPS_Leave(padapter, "NON_LINKED");
 	}
 
-	pmlmepriv->LinkDetectInfo.NumRxOkInPeriod = 0;
-	pmlmepriv->LinkDetectInfo.NumTxOkInPeriod = 0;
-	pmlmepriv->LinkDetectInfo.NumRxUnicastOkInPeriod = 0;
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = bBusyTraffic;
-	pmlmepriv->LinkDetectInfo.bTxBusyTraffic = bTxBusyTraffic;
-	pmlmepriv->LinkDetectInfo.bRxBusyTraffic = bRxBusyTraffic;
-	pmlmepriv->LinkDetectInfo.bHigherBusyTraffic = bHigherBusyTraffic;
-	pmlmepriv->LinkDetectInfo.bHigherBusyRxTraffic = bHigherBusyRxTraffic;
-	pmlmepriv->LinkDetectInfo.bHigherBusyTxTraffic = bHigherBusyTxTraffic;
-
-	return bEnterPS;
+	pmlmepriv->link_detect_info.num_rx_ok_in_period = 0;
+	pmlmepriv->link_detect_info.num_tx_ok_in_period = 0;
+	pmlmepriv->link_detect_info.num_rx_unicast_ok_in_period = 0;
+	pmlmepriv->link_detect_info.busy_traffic = busy_traffic;
+	pmlmepriv->link_detect_info.tx_busy_traffic = tx_busy_traffic;
+	pmlmepriv->link_detect_info.rx_busy_traffic = rx_busy_traffic;
+	pmlmepriv->link_detect_info.higher_busy_traffic = higher_busy_traffic;
+	pmlmepriv->link_detect_info.higher_busy_rx_traffic = higher_busy_rx_traffic;
+	pmlmepriv->link_detect_info.higher_busy_tx_traffic = higher_busy_tx_traffic;
+
+	return should_enter_ps;
 }
 
 static void dynamic_chk_wk_hdl(struct adapter *padapter)
@@ -1239,7 +1247,7 @@ static void dynamic_chk_wk_hdl(struct adapter *padapter)
 	/* if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING|_FW_UNDER_SURVEY) ==false) */
 	{
 		linked_status_chk(padapter);
-		traffic_status_watchdog(padapter, 0);
+		traffic_status_watchdog(padapter, false);
 	}
 	rtw_hal_dm_watchdog(padapter);
 
@@ -1495,7 +1503,7 @@ static void rtw_chk_hi_queue_hdl(struct adapter *padapter)
 	rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
 
 	while (!empty && jiffies_to_msecs(jiffies - start) < g_wait_hiq_empty) {
-		msleep(100);
+		fsleep(100 * USEC_PER_MSEC);
 		rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
 	}
 
@@ -1761,7 +1769,8 @@ u8 rtw_drvextra_cmd_hdl(struct adapter *padapter, unsigned char *pbuf)
 		rtw_free_assoc_resources(padapter, 1);
 		break;
 	case C2H_WK_CID:
-		rtw_hal_set_hwreg_with_buf(padapter, HW_VAR_C2H_HANDLE, pdrvextra_cmd->pbuf, pdrvextra_cmd->size);
+		rtw_hal_set_hwreg_with_buf(padapter, HW_VAR_C2H_HANDLE,
+					   pdrvextra_cmd->pbuf, pdrvextra_cmd->size);
 		break;
 	case DM_RA_MSK_WK_CID:
 		rtw_dm_ra_mask_hdl(padapter, (struct sta_info *)pdrvextra_cmd->pbuf);
diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c
index 98b15ca100746..099320e4eb500 100644
--- a/drivers/staging/rtl8723bs/core/rtw_efuse.c
+++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c
@@ -8,24 +8,6 @@
 #include <hal_data.h>
 #include <linux/jiffies.h>
 
-
-/* Define global variables */
-u8 fakeEfuseBank;
-u32 fakeEfuseUsedBytes;
-u8 fakeEfuseContent[EFUSE_MAX_HW_SIZE] = {0};
-u8 fakeEfuseInitMap[EFUSE_MAX_MAP_LEN] = {0};
-u8 fakeEfuseModifiedMap[EFUSE_MAX_MAP_LEN] = {0};
-
-u32 BTEfuseUsedBytes;
-u8 BTEfuseContent[EFUSE_MAX_BT_BANK][EFUSE_MAX_HW_SIZE];
-u8 BTEfuseInitMap[EFUSE_BT_MAX_MAP_LEN] = {0};
-u8 BTEfuseModifiedMap[EFUSE_BT_MAX_MAP_LEN] = {0};
-
-u32 fakeBTEfuseUsedBytes;
-u8 fakeBTEfuseContent[EFUSE_MAX_BT_BANK][EFUSE_MAX_HW_SIZE];
-u8 fakeBTEfuseInitMap[EFUSE_BT_MAX_MAP_LEN] = {0};
-u8 fakeBTEfuseModifiedMap[EFUSE_BT_MAX_MAP_LEN] = {0};
-
 /*  11/16/2008 MH Add description. Get current efuse area enabled word!!. */
 u8
 Efuse_CalculateWordCnts(u8 word_en)
@@ -69,7 +51,7 @@ u16		Address)
 	u32 k = 0;
 	u16 contentLen = 0;
 
-	Hal_GetEfuseDefinition(Adapter, EFUSE_WIFI, TYPE_EFUSE_REAL_CONTENT_LEN, (void *)&contentLen);
+	Hal_GetEfuseDefinition(Adapter, EFUSE_WIFI, TYPE_EFUSE_REAL_CONTENT_LEN, &contentLen);
 
 	if (Address < contentLen) {/* E-fuse 512Byte */
 		/* Write E-fuse Register address bit0~7 */
@@ -163,7 +145,7 @@ static void Efuse_ReadAllMap(struct adapter *padapter, u8 efuseType, u8 *Efuse)
 
 	Hal_EfusePowerSwitch(padapter, true);
 
-	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
+	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, &mapLen);
 
 	Hal_ReadEFuse(padapter, efuseType, 0, mapLen, Efuse);
 
@@ -239,7 +221,7 @@ void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType)
 	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
 	u16 mapLen = 0;
 
-	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
+	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, &mapLen);
 
 	if (pEEPROM->bautoload_fail_flag)
 		memset(pEEPROM->efuse_eeprom_data, 0xFF, mapLen);
diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
index 3e2b5e6b07f93..72b7f731dd471 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
@@ -94,7 +94,7 @@ bool rtw_is_cckratesonly_included(u8 *rate)
 	return true;
 }
 
-int rtw_check_network_type(unsigned char *rate, int ratelen, int channel)
+int rtw_check_network_type(unsigned char *rate, int channel)
 {
 	if (channel > 14)
 		return WIRELESS_INVALID;
@@ -109,7 +109,7 @@ int rtw_check_network_type(unsigned char *rate, int ratelen, int channel)
 u8 *rtw_set_fixed_ie(unsigned char *pbuf, unsigned int len, unsigned char *source,
 				unsigned int *frlen)
 {
-	memcpy((void *)pbuf, (void *)source, len);
+	memcpy(pbuf, source, len);
 	*frlen = *frlen + len;
 	return pbuf + len;
 }
@@ -126,7 +126,7 @@ u8 *rtw_set_ie(u8 *pbuf,
 	*(pbuf + 1) = (u8)len;
 
 	if (len > 0)
-		memcpy((void *)(pbuf + 2), (void *)source, len);
+		memcpy(pbuf + 2, source, len);
 
 	*frlen = *frlen + (len + 2);
 
@@ -192,8 +192,8 @@ u8 *rtw_get_ie_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len, u8 *ie, u
 		if (cnt + 2 + ie_len > in_len)
 			break;
 
-		if (eid == in_ie[cnt]
-			&& (!oui || (ie_len >= oui_len && !memcmp(&in_ie[cnt + 2], oui, oui_len)))) {
+		if (eid == in_ie[cnt] &&
+		    (!oui || (ie_len >= oui_len && !memcmp(&in_ie[cnt + 2], oui, oui_len)))) {
 			target_ie = &in_ie[cnt];
 
 			if (ie)
@@ -291,7 +291,7 @@ uint rtw_get_rateset_len(u8 *rateset)
 int rtw_generate_ie(struct registry_priv *pregistrypriv)
 {
 	u8 wireless_mode;
-	int	sz = 0, rateLen;
+	int	sz = 0, rate_len;
 	struct wlan_bssid_ex *pdev_network = &pregistrypriv->dev_network;
 	u8 *ie = pdev_network->ies;
 
@@ -326,13 +326,13 @@ int rtw_generate_ie(struct registry_priv *pregistrypriv)
 
 	rtw_set_supported_rate(pdev_network->supported_rates, wireless_mode);
 
-	rateLen = rtw_get_rateset_len(pdev_network->supported_rates);
+	rate_len = rtw_get_rateset_len(pdev_network->supported_rates);
 
-	if (rateLen > 8) {
+	if (rate_len > 8) {
 		ie = rtw_set_ie(ie, WLAN_EID_SUPP_RATES, 8, pdev_network->supported_rates, &sz);
-		/* ie = rtw_set_ie(ie, WLAN_EID_EXT_SUPP_RATES, (rateLen - 8), (pdev_network->supported_rates + 8), &sz); */
+		/* ie = rtw_set_ie(ie, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (pdev_network->supported_rates + 8), &sz); */
 	} else {
-		ie = rtw_set_ie(ie, WLAN_EID_SUPP_RATES, rateLen, pdev_network->supported_rates, &sz);
+		ie = rtw_set_ie(ie, WLAN_EID_SUPP_RATES, rate_len, pdev_network->supported_rates, &sz);
 	}
 
 	/* DS parameter set */
@@ -342,8 +342,8 @@ int rtw_generate_ie(struct registry_priv *pregistrypriv)
 
 	ie = rtw_set_ie(ie, WLAN_EID_IBSS_PARAMS, 2, (u8 *)&(pdev_network->configuration.atim_window), &sz);
 
-	if (rateLen > 8)
-		ie = rtw_set_ie(ie, WLAN_EID_EXT_SUPP_RATES, (rateLen - 8), (pdev_network->supported_rates + 8), &sz);
+	if (rate_len > 8)
+		ie = rtw_set_ie(ie, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (pdev_network->supported_rates + 8), &sz);
 
 	/* HT Cap. */
 	if ((pregistrypriv->wireless_mode & WIRELESS_11_24N) &&
@@ -455,10 +455,10 @@ int rtw_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwis
 		return _FAIL;
 	}
 
-	if ((*wpa_ie != WLAN_EID_VENDOR_SPECIFIC) || (*(wpa_ie+1) != (u8)(wpa_ie_len - 2)) ||
-	   (memcmp(wpa_ie+2, RTW_WPA_OUI_TYPE, WPA_SELECTOR_LEN))) {
+	if ((*wpa_ie != WLAN_EID_VENDOR_SPECIFIC) ||
+	    (*(wpa_ie + 1) != (u8)(wpa_ie_len - 2)) ||
+	    (memcmp(wpa_ie + 2, RTW_WPA_OUI_TYPE, WPA_SELECTOR_LEN)))
 		return _FAIL;
-	}
 
 	pos = wpa_ie;
 
@@ -518,7 +518,7 @@ int rtw_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher, int *pairwi
 		return _FAIL;
 	}
 
-	if ((*rsn_ie != WLAN_EID_RSN) || (*(rsn_ie+1) != (u8)(rsn_ie_len - 2)))
+	if ((*rsn_ie != WLAN_EID_RSN) || (*(rsn_ie + 1) != (u8)(rsn_ie_len - 2)))
 		return _FAIL;
 
 	pos = rsn_ie;
@@ -585,19 +585,18 @@ int rtw_get_wapi_ie(u8 *in_ie, uint in_len, u8 *wapi_ie, u16 *wapi_len)
 	while (cnt < in_len) {
 		authmode = in_ie[cnt];
 
-		/* if (authmode == WLAN_EID_BSS_AC_ACCESS_DELAY) */
-		if (authmode == WLAN_EID_BSS_AC_ACCESS_DELAY && (!memcmp(&in_ie[cnt+6], wapi_oui1, 4) ||
-					!memcmp(&in_ie[cnt+6], wapi_oui2, 4))) {
+		if (authmode == WLAN_EID_BSS_AC_ACCESS_DELAY &&
+		    (!memcmp(&in_ie[cnt + 6], wapi_oui1, 4) ||
+		     !memcmp(&in_ie[cnt + 6], wapi_oui2, 4))) {
 			if (wapi_ie)
-				memcpy(wapi_ie, &in_ie[cnt], in_ie[cnt+1]+2);
+				memcpy(wapi_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
 
 			if (wapi_len)
-				*wapi_len = in_ie[cnt+1]+2;
+				*wapi_len = in_ie[cnt + 1] + 2;
 
-			cnt += in_ie[cnt+1]+2;  /* get next */
-		} else {
-			cnt += in_ie[cnt+1]+2;   /* get next */
 		}
+
+		cnt += in_ie[cnt + 1] + 2;   /* get next */
 	}
 
 	if (wapi_len)
@@ -619,23 +618,20 @@ void rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie
 	while (cnt < in_len) {
 		authmode = in_ie[cnt];
 
-		if ((authmode == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&in_ie[cnt+2], &wpa_oui[0], 4))) {
+		if ((authmode == WLAN_EID_VENDOR_SPECIFIC) &&
+		    (!memcmp(&in_ie[cnt + 2], &wpa_oui[0], 4))) {
 			if (wpa_ie)
-				memcpy(wpa_ie, &in_ie[cnt], in_ie[cnt+1]+2);
+				memcpy(wpa_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
 
 			*wpa_len = in_ie[cnt + 1] + 2;
-			cnt += in_ie[cnt + 1] + 2;  /* get next */
-		} else {
-			if (authmode == WLAN_EID_RSN) {
-				if (rsn_ie)
-					memcpy(rsn_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
-
-				*rsn_len = in_ie[cnt+1]+2;
-				cnt += in_ie[cnt+1]+2;  /* get next */
-			} else {
-				cnt += in_ie[cnt+1]+2;   /* get next */
-			}
+		} else if (authmode == WLAN_EID_RSN) {
+			if (rsn_ie)
+				memcpy(rsn_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			*rsn_len = in_ie[cnt + 1] + 2;
 		}
+
+		cnt += in_ie[cnt + 1] + 2;   /* get next */
 	}
 }
 
@@ -665,20 +661,20 @@ u8 *rtw_get_wps_ie(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen)
 	while (cnt < in_len) {
 		eid = in_ie[cnt];
 
-		if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&in_ie[cnt+2], wps_oui, 4))) {
+		if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&in_ie[cnt + 2], wps_oui, 4))) {
 			wpsie_ptr = &in_ie[cnt];
 
 			if (wps_ie)
-				memcpy(wps_ie, &in_ie[cnt], in_ie[cnt+1]+2);
+				memcpy(wps_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
 
 			if (wps_ielen)
-				*wps_ielen = in_ie[cnt+1]+2;
+				*wps_ielen = in_ie[cnt + 1] + 2;
 
-			cnt += in_ie[cnt+1]+2;
+			cnt += in_ie[cnt + 1] + 2;
 
 			break;
 		}
-		cnt += in_ie[cnt+1]+2; /* goto next */
+		cnt += in_ie[cnt + 1] + 2; /* goto next */
 	}
 
 	return wpsie_ptr;
@@ -756,12 +752,12 @@ u8 *rtw_get_wps_attr_content(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8
 
 	if (attr_ptr && attr_len) {
 		if (buf_content)
-			memcpy(buf_content, attr_ptr+4, attr_len-4);
+			memcpy(buf_content, attr_ptr + 4, attr_len - 4);
 
 		if (len_content)
-			*len_content = attr_len-4;
+			*len_content = attr_len - 4;
 
-		return attr_ptr+4;
+		return attr_ptr + 4;
 	}
 
 	return NULL;
@@ -851,9 +847,9 @@ static int rtw_ieee802_11_parse_vendor_specific(u8 *pos, uint elen,
  * @show_errors: Whether to show parsing errors in debug log
  * Returns: Parsing result
  */
-enum ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
-				struct rtw_ieee802_11_elems *elems,
-				int show_errors)
+enum parse_result rtw_ieee802_11_parse_elems(u8 *start, uint len,
+					     struct rtw_ieee802_11_elems *elems,
+					     int show_errors)
 {
 	uint left = len;
 	u8 *pos = start;
@@ -869,7 +865,7 @@ enum ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
 		left -= 2;
 
 		if (elen > left)
-			return ParseFailed;
+			return PARSE_FAILED;
 
 		switch (id) {
 		case WLAN_EID_SSID:
@@ -972,9 +968,9 @@ enum ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
 	}
 
 	if (left)
-		return ParseFailed;
+		return PARSE_FAILED;
 
-	return unknown ? ParseUnknown : ParseOK;
+	return unknown ? PARSE_UNKNOWN : PARSE_OK;
 }
 
 void rtw_macaddr_cfg(struct device *dev, u8 *mac_addr)
@@ -1012,20 +1008,25 @@ static int rtw_get_cipher_info(struct wlan_network *pnetwork)
 	int group_cipher = 0, pairwise_cipher = 0, is8021x = 0;
 	int ret = _FAIL;
 
-	pbuf = rtw_get_wpa_ie(&pnetwork->network.ies[12], &wpa_ielen, pnetwork->network.ie_length-12);
+	pbuf = rtw_get_wpa_ie(&pnetwork->network.ies[12],
+			      &wpa_ielen,
+			      pnetwork->network.ie_length - 12);
 
 	if (pbuf && (wpa_ielen > 0)) {
-		if (rtw_parse_wpa_ie(pbuf, wpa_ielen+2, &group_cipher, &pairwise_cipher, &is8021x) == _SUCCESS) {
+		if (rtw_parse_wpa_ie(pbuf, wpa_ielen + 2, &group_cipher,
+				     &pairwise_cipher, &is8021x) == _SUCCESS) {
 			pnetwork->bcn_info.pairwise_cipher = pairwise_cipher;
 			pnetwork->bcn_info.group_cipher = group_cipher;
 			pnetwork->bcn_info.is_8021x = is8021x;
 			ret = _SUCCESS;
 		}
 	} else {
-		pbuf = rtw_get_wpa2_ie(&pnetwork->network.ies[12], &wpa_ielen, pnetwork->network.ie_length-12);
+		pbuf = rtw_get_wpa2_ie(&pnetwork->network.ies[12], &wpa_ielen,
+				       pnetwork->network.ie_length - 12);
 
 		if (pbuf && (wpa_ielen > 0)) {
-			if (rtw_parse_wpa2_ie(pbuf, wpa_ielen+2, &group_cipher, &pairwise_cipher, &is8021x) == _SUCCESS) {
+			if (rtw_parse_wpa2_ie(pbuf, wpa_ielen + 2, &group_cipher,
+					      &pairwise_cipher, &is8021x) == _SUCCESS) {
 				pnetwork->bcn_info.pairwise_cipher = pairwise_cipher;
 				pnetwork->bcn_info.group_cipher = group_cipher;
 				pnetwork->bcn_info.is_8021x = is8021x;
@@ -1094,21 +1095,21 @@ u16 rtw_mcs_rate(u8 bw_40MHz, u8 short_GI, unsigned char *MCS_rate)
 	u16 max_rate = 0;
 
 	if (MCS_rate[0] & BIT(7))
-		max_rate = (bw_40MHz) ? ((short_GI)?1500:1350):((short_GI)?722:650);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 1500 : 1350) : ((short_GI) ? 722 : 650);
 	else if (MCS_rate[0] & BIT(6))
-		max_rate = (bw_40MHz) ? ((short_GI)?1350:1215):((short_GI)?650:585);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 1350 : 1215) : ((short_GI) ? 650 : 585);
 	else if (MCS_rate[0] & BIT(5))
-		max_rate = (bw_40MHz) ? ((short_GI)?1200:1080):((short_GI)?578:520);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 1200 : 1080) : ((short_GI) ? 578 : 520);
 	else if (MCS_rate[0] & BIT(4))
-		max_rate = (bw_40MHz) ? ((short_GI)?900:810):((short_GI)?433:390);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 900 : 810) : ((short_GI) ? 433 : 390);
 	else if (MCS_rate[0] & BIT(3))
-		max_rate = (bw_40MHz) ? ((short_GI)?600:540):((short_GI)?289:260);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 600 : 540) : ((short_GI) ? 289 : 260);
 	else if (MCS_rate[0] & BIT(2))
-		max_rate = (bw_40MHz) ? ((short_GI)?450:405):((short_GI)?217:195);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 450 : 405) : ((short_GI) ? 217 : 195);
 	else if (MCS_rate[0] & BIT(1))
-		max_rate = (bw_40MHz) ? ((short_GI)?300:270):((short_GI)?144:130);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 300 : 270) : ((short_GI) ? 144 : 130);
 	else if (MCS_rate[0] & BIT(0))
-		max_rate = (bw_40MHz) ? ((short_GI)?150:135):((short_GI)?72:65);
+		max_rate = (bw_40MHz) ? ((short_GI) ? 150 : 135) : ((short_GI) ? 72 : 65);
 
 	return max_rate;
 }
@@ -1122,11 +1123,9 @@ int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *act
 
 	fc = le16_to_cpu(((struct ieee80211_hdr_3addr *)frame)->frame_control);
 
-	if ((fc & (IEEE80211_FCTL_FTYPE|IEEE80211_FCTL_STYPE))
-		!= (IEEE80211_FTYPE_MGMT|IEEE80211_STYPE_ACTION)
-	) {
+	if ((fc & (IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) !=
+	    (IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ACTION))
 		return false;
-	}
 
 	c = frame_body[0];
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_io.c b/drivers/staging/rtl8723bs/core/rtw_io.c
index 965c3cfea103e..53ba99e1de00a 100644
--- a/drivers/staging/rtl8723bs/core/rtw_io.c
+++ b/drivers/staging/rtl8723bs/core/rtw_io.c
@@ -75,6 +75,7 @@ int rtw_write8(struct adapter *adapter, u32 addr, u8 val)
 
 	return RTW_STATUS_CODE(ret);
 }
+
 int rtw_write16(struct adapter *adapter, u32 addr, u16 val)
 {
 	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
@@ -88,6 +89,7 @@ int rtw_write16(struct adapter *adapter, u32 addr, u16 val)
 	ret = _write16(pintfhdl, addr, val);
 	return RTW_STATUS_CODE(ret);
 }
+
 int rtw_write32(struct adapter *adapter, u32 addr, u32 val)
 {
 	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
@@ -114,7 +116,9 @@ u32 rtw_write_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem)
 	return _write_port(pintfhdl, addr, cnt, pmem);
 }
 
-int rtw_init_io_priv(struct adapter *padapter, void (*set_intf_ops)(struct adapter *padapter, struct _io_ops *pops))
+int rtw_init_io_priv(struct adapter *padapter,
+		     void (*set_intf_ops)(struct adapter *padapter,
+					  struct _io_ops *pops))
 {
 	struct io_priv *piopriv = &padapter->iopriv;
 	struct intf_hdl *pintf = &piopriv->intf;
@@ -132,7 +136,8 @@ int rtw_init_io_priv(struct adapter *padapter, void (*set_intf_ops)(struct adapt
 }
 
 /*
- * Increase and check if the continual_io_error of this @param dvobjprive is larger than MAX_CONTINUAL_IO_ERR
+ * Increase and check if the continual_io_error of this @param dvobjprive
+ * is larger than MAX_CONTINUAL_IO_ERR
  * @return true:
  * @return false:
  */
diff --git a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
index 587a87fbffeb4..d92ec9949d001 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
@@ -61,9 +61,7 @@ u8 rtw_do_join(struct adapter *padapter)
 		/* when set_ssid/set_bssid for rtw_do_join(), but scanning queue is empty */
 		/* we try to issue sitesurvey firstly */
 
-		if (pmlmepriv->LinkDetectInfo.bBusyTraffic == false
-			|| rtw_to_roam(padapter) > 0
-		) {
+		if (!pmlmepriv->link_detect_info.busy_traffic || rtw_to_roam(padapter) > 0) {
 			/*  submit site_survey_cmd */
 			ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0);
 			if (ret != _SUCCESS)
@@ -113,9 +111,8 @@ u8 rtw_do_join(struct adapter *padapter)
 
 				/* when set_ssid/set_bssid for rtw_do_join(), but there are no desired bss in scanning queue */
 				/* we try to issue sitesurvey firstly */
-				if (pmlmepriv->LinkDetectInfo.bBusyTraffic == false
-					|| rtw_to_roam(padapter) > 0
-				) {
+				if (!pmlmepriv->link_detect_info.busy_traffic ||
+				    rtw_to_roam(padapter) > 0) {
 					ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0);
 					if (ret != _SUCCESS)
 						pmlmepriv->to_join = false;
@@ -379,8 +376,8 @@ u8 rtw_set_802_11_bssid_list_scan(struct adapter *padapter, struct ndis_802_11_s
 		goto exit;
 	}
 
-	if ((check_fwstate(pmlmepriv, _FW_UNDER_SURVEY|_FW_UNDER_LINKING) == true) ||
-		(pmlmepriv->LinkDetectInfo.bBusyTraffic == true)) {
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING) ||
+	    pmlmepriv->link_detect_info.busy_traffic) {
 		/*  Scan or linking is in progress, do nothing. */
 		res = true;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index 1ef48bf6581c4..5516593bc8aba 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -262,17 +262,13 @@ void rtw_free_network_queue(struct adapter *padapter, u8 isfreeall)
 	spin_unlock_bh(&scanned_queue->lock);
 }
 
-signed int rtw_if_up(struct adapter *padapter)
+bool rtw_if_up(struct adapter *padapter)
 {
-	signed int res;
-
 	if (padapter->bDriverStopped || padapter->bSurpriseRemoved ||
 		!check_fwstate(&padapter->mlmepriv, _FW_LINKED))
-		res = false;
-	else
-		res =  true;
+		return false;
 
-	return res;
+	return true;
 }
 
 void rtw_generate_random_ibss(u8 *pibss)
@@ -330,21 +326,18 @@ struct	wlan_network *rtw_find_network(struct __queue *scanned_queue, u8 *addr)
 	return pnetwork;
 }
 
-int rtw_is_same_ibss(struct adapter *adapter, struct wlan_network *pnetwork)
+bool rtw_is_same_ibss(struct adapter *adapter, struct wlan_network *pnetwork)
 {
-	int ret = true;
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 
 	if ((psecuritypriv->dot11PrivacyAlgrthm != _NO_PRIVACY_) &&
 		    (pnetwork->network.privacy == 0))
-		ret = false;
+		return false;
 	else if ((psecuritypriv->dot11PrivacyAlgrthm == _NO_PRIVACY_) &&
 		 (pnetwork->network.privacy == 1))
-		ret = false;
-	else
-		ret = true;
+		return false;
 
-	return ret;
+	return true;
 }
 
 inline int is_same_ess(struct wlan_bssid_ex *a, struct wlan_bssid_ex *b)
@@ -595,15 +588,14 @@ void rtw_add_network(struct adapter *adapter, struct wlan_bssid_ex *pnetwork)
  * (4) HT
  * (5) others
  */
-int rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwork);
-int rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwork)
+static bool rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwork)
 {
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
 	u32 desired_encmode;
 	u32 privacy;
 	uint wps_ielen;
-	int bselected = true;
+	bool bselected = true;
 
 	desired_encmode = psecuritypriv->ndisencryptstatus;
 	privacy = pnetwork->network.privacy;
@@ -877,11 +869,10 @@ void rtw_indicate_connect(struct adapter *padapter)
 		set_fwstate(pmlmepriv, _FW_LINKED);
 
 		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-		    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
+		    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
 			rtw_cfg80211_ibss_indicate_connect(padapter);
-		} else {
+		else
 			rtw_cfg80211_indicate_connect(padapter);
-		}
 
 		netif_carrier_on(padapter->pnetdev);
 
@@ -1174,8 +1165,8 @@ void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
 
 	spin_lock_bh(&pmlmepriv->lock);
 
-	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
-	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+	pmlmepriv->link_detect_info.traffic_transition_count = 0;
+	pmlmepriv->link_detect_info.low_power_transition_count = 0;
 
 	if (pnetwork->join_res > 0) {
 		spin_lock_bh(&pmlmepriv->scanned_queue.lock);
@@ -1615,7 +1606,7 @@ static void rtw_auto_scan_handler(struct adapter *padapter)
 			if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING))
 				goto exit;
 
-			if (pmlmepriv->LinkDetectInfo.bBusyTraffic)
+			if (pmlmepriv->link_detect_info.busy_traffic)
 				goto exit;
 		}
 
@@ -1643,12 +1634,12 @@ void rtw_dynamic_check_timer_handler(struct adapter *adapter)
 	if ((adapter_to_pwrctl(adapter)->fw_current_in_ps_mode)
 		&& !(hal_btcoex_IsBtControlLps(adapter))
 		) {
-		u8 bEnterPS;
+		bool should_enter_ps;
 
 		linked_status_chk(adapter);
 
-		bEnterPS = traffic_status_watchdog(adapter, 1);
-		if (bEnterPS) {
+		should_enter_ps = traffic_status_watchdog(adapter, true);
+		if (should_enter_ps) {
 			/* rtw_lps_ctrl_wk_cmd(adapter, LPS_CTRL_ENTER, 1); */
 			rtw_hal_dm_watchdog_in_lps(adapter);
 		} else {
@@ -2507,8 +2498,7 @@ void rtw_issue_addbareq_cmd(struct adapter *padapter, struct xmit_frame *pxmitfr
 	struct pkt_attrib *pattrib = &pxmitframe->attrib;
 	s32 bmcst = is_multicast_ether_addr(pattrib->ra);
 
-	/* if (bmcst || (padapter->mlmepriv.LinkDetectInfo.bTxBusyTraffic == false)) */
-	if (bmcst || (padapter->mlmepriv.LinkDetectInfo.NumTxOkInPeriod < 100))
+	if (bmcst || (padapter->mlmepriv.link_detect_info.num_tx_ok_in_period < 100))
 		return;
 
 	priority = pattrib->priority;
@@ -2595,7 +2585,7 @@ void _rtw_roaming(struct adapter *padapter, struct wlan_network *tgt_network)
 	}
 }
 
-signed int rtw_linked_check(struct adapter *padapter)
+bool rtw_linked_check(struct adapter *padapter)
 {
 	if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) ||
 	    check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) {
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index b1f20aa81efbf..5f00fe282d1be 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -8,6 +8,7 @@
 #include <rtw_wifi_regd.h>
 #include <hal_btcoex.h>
 #include <linux/kernel.h>
+#include <linux/minmax.h>
 #include <linux/unaligned.h>
 
 static struct mlme_handler mlme_sta_tbl[] = {
@@ -183,12 +184,12 @@ int rtw_ch_set_search_ch(struct rt_channel_info *ch_set, const u32 ch)
 
 /* Following are the initialization functions for WiFi MLME */
 
-int init_hw_mlme_ext(struct adapter *padapter)
+void init_hw_mlme_ext(struct adapter *padapter)
 {
-	struct	mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 
-	set_channel_bwmode(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-	return _SUCCESS;
+	set_channel_bwmode(padapter, pmlmeext->cur_channel,
+			   pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
 }
 
 void init_mlme_default_rate_set(struct adapter *padapter)
@@ -525,7 +526,7 @@ unsigned int OnProbeReq(struct adapter *padapter, union recv_frame *precv_frame)
 		if (is_valid_p2p_probereq)
 			goto _issue_probersp;
 
-		if ((ielen != 0 && false == !memcmp((void *)(p+2), (void *)cur->ssid.ssid, cur->ssid.ssid_length))
+		if ((ielen != 0 && false == !memcmp((p+2), cur->ssid.ssid, cur->ssid.ssid_length))
 			|| (ielen == 0 && pmlmeinfo->hidden_ssid_mode)
 		)
 			return _SUCCESS;
@@ -791,7 +792,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	} else { /*  shared system or auto authentication */
 		if (seq == 1) {
 			/* prepare for the challenging txt... */
-			memset((void *)pstat->chg_txt, 78, 128);
+			memset(pstat->chg_txt, 78, 128);
 
 			pstat->state &= ~WIFI_FW_AUTH_NULL;
 			pstat->state |= WIFI_FW_AUTH_STATE;
@@ -806,7 +807,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 				goto auth_fail;
 			}
 
-			if (!memcmp((void *)(p + 2), pstat->chg_txt, 128)) {
+			if (!memcmp((p + 2), pstat->chg_txt, 128)) {
 				pstat->state &= (~WIFI_FW_AUTH_STATE);
 				pstat->state |= WIFI_FW_AUTH_SUCCESS;
 				/*  challenging txt is correct... */
@@ -893,7 +894,7 @@ unsigned int OnAuthClient(struct adapter *padapter, union recv_frame *precv_fram
 			if (!p)
 				goto authclnt_fail;
 
-			memcpy((void *)(pmlmeinfo->chg_txt), (void *)(p + 2), len);
+			memcpy(pmlmeinfo->chg_txt, p + 2, len);
 			pmlmeinfo->auth_seq = 3;
 			issue_auth(padapter, NULL, 0);
 			set_link_timer(pmlmeext, REAUTH_TO);
@@ -933,9 +934,10 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	struct sta_info *pstat;
 	unsigned char *p, *pos, *wpa_ie;
 	unsigned char WMM_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01};
-	int		i, ie_len, wpa_ie_len, left;
+	int		i, ie_len, left;
+	u8 wpa_ie_len;
 	unsigned char supportRate[16];
-	int					supportRateNum;
+	int					support_rate_num;
 	unsigned short		status = WLAN_STATUS_SUCCESS;
 	unsigned short		frame_type, ie_offset = 0;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
@@ -990,7 +992,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	pstat->capability = capab_info;
 
 	/* now parse all ieee802_11 ie to point to elems */
-	if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed ||
+	if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == PARSE_FAILED ||
 	    !elems.ssid) {
 		status = WLAN_STATUS_CHALLENGE_FAIL;
 		goto OnAssocReqFail;
@@ -1007,7 +1009,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		goto OnAssocReqFail;
 	} else {
 		/*  check if ssid match */
-		if (memcmp((void *)(p+2), cur->ssid.ssid, cur->ssid.ssid_length))
+		if (memcmp(p+2, cur->ssid.ssid, cur->ssid.ssid_length))
 			status = WLAN_STATUS_CHALLENGE_FAIL;
 
 		if (ie_len != cur->ssid.ssid_length)
@@ -1022,7 +1024,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	if (!p) {
 		/*  use our own rate set as statoin used */
 		/* memcpy(supportRate, AP_BSSRATE, AP_BSSRATE_LEN); */
-		/* supportRateNum = AP_BSSRATE_LEN; */
+		/* support_rate_num = AP_BSSRATE_LEN; */
 
 		status = WLAN_STATUS_CHALLENGE_FAIL;
 		goto OnAssocReqFail;
@@ -1031,25 +1033,25 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 			ie_len = sizeof(supportRate);
 
 		memcpy(supportRate, p+2, ie_len);
-		supportRateNum = ie_len;
+		support_rate_num = ie_len;
 
 		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + ie_offset, WLAN_EID_EXT_SUPP_RATES, &ie_len,
 				pkt_len - WLAN_HDR_A3_LEN - ie_offset);
 		if (p) {
 
-			if (supportRateNum + ie_len <= sizeof(supportRate)) {
-				memcpy(supportRate+supportRateNum, p+2, ie_len);
-				supportRateNum += ie_len;
+			if (support_rate_num + ie_len <= sizeof(supportRate)) {
+				memcpy(supportRate + support_rate_num, p + 2, ie_len);
+				support_rate_num += ie_len;
 			}
 		}
 	}
 
 	/* todo: mask supportRate between AP & STA -> move to update raid */
-	/* get_matched_rate(pmlmeext, supportRate, &supportRateNum, 0); */
+	/* get_matched_rate(pmlmeext, supportRate, &support_rate_num, 0); */
 
 	/* update station supportRate */
-	pstat->bssratelen = supportRateNum;
-	memcpy(pstat->bssrateset, supportRate, supportRateNum);
+	pstat->bssratelen = support_rate_num;
+	memcpy(pstat->bssrateset, supportRate, support_rate_num);
 	update_basic_rate_table_soft_ap(pstat->bssrateset, pstat->bssratelen);
 
 	/* check RSN/WPA/WPS */
@@ -1154,7 +1156,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 			pstat->flags |= WLAN_STA_WPS;
 			copy_len = 0;
 		} else {
-			copy_len = ((wpa_ie_len+2) > sizeof(pstat->wpa_ie)) ? (sizeof(pstat->wpa_ie)):(wpa_ie_len+2);
+			copy_len = min(sizeof(pstat->wpa_ie), wpa_ie_len + 2u);
 		}
 
 
@@ -1337,7 +1339,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 
 asoc_class2_error:
 
-	issue_deauth(padapter, (void *)GetAddr2Ptr(pframe), status);
+	issue_deauth(padapter, GetAddr2Ptr(pframe), status);
 
 	return _FAIL;
 
@@ -1511,7 +1513,7 @@ unsigned int OnDeAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	if (ignore_received_deauth == 0)
 		receive_disconnect(padapter, GetAddr3Ptr(pframe), reason);
 
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
+	pmlmepriv->link_detect_info.busy_traffic = false;
 	return _SUCCESS;
 }
 
@@ -1563,7 +1565,7 @@ unsigned int OnDisassoc(struct adapter *padapter, union recv_frame *precv_frame)
 
 	receive_disconnect(padapter, GetAddr3Ptr(pframe), reason);
 
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
+	pmlmepriv->link_detect_info.busy_traffic = false;
 	return _SUCCESS;
 
 }
@@ -2715,9 +2717,9 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 	fctrl = &(pwlanhdr->frame_control);
 	*(fctrl) = 0;
 
-	memcpy((void *)GetAddr1Ptr(pwlanhdr), pstat->hwaddr, ETH_ALEN);
-	memcpy((void *)GetAddr2Ptr(pwlanhdr), myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy((void *)GetAddr3Ptr(pwlanhdr), get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+	memcpy(GetAddr1Ptr(pwlanhdr), pstat->hwaddr, ETH_ALEN);
+	memcpy(GetAddr2Ptr(pwlanhdr), myid(&(padapter->eeprompriv)), ETH_ALEN);
+	memcpy(GetAddr3Ptr(pwlanhdr), get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
 
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
@@ -4756,9 +4758,8 @@ static void rtw_mlmeext_disconnect(struct adapter *padapter)
 
 	timer_delete_sync(&pmlmeext->link_timer);
 
-	/* pmlmepriv->LinkDetectInfo.TrafficBusyState = false; */
-	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
-	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+	pmlmepriv->link_detect_info.traffic_transition_count = 0;
+	pmlmepriv->link_detect_info.low_power_transition_count = 0;
 
 }
 
@@ -4885,7 +4886,7 @@ void mlmeext_sta_add_event_callback(struct adapter *padapter, struct sta_info *p
 	rtw_hal_update_sta_rate_mask(padapter, psta);
 
 	/*  ToDo: HT for Ad-hoc */
-	psta->wireless_mode = rtw_check_network_type(psta->bssrateset, psta->bssratelen, pmlmeext->cur_channel);
+	psta->wireless_mode = rtw_check_network_type(psta->bssrateset, pmlmeext->cur_channel);
 	psta->raid = networktype_to_raid_ex(padapter, psta);
 
 	/* rate radaptive */
diff --git a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
index 1c9e02732687e..f074ea6e0f38e 100644
--- a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
@@ -29,14 +29,12 @@ void _ips_enter(struct adapter *padapter)
 		pwrpriv->rf_pwrstate = rf_off;
 	}
 	pwrpriv->bips_processing = false;
-
 }
 
 void ips_enter(struct adapter *padapter)
 {
 	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
 
-
 	hal_btcoex_IpsNotify(padapter, pwrpriv->ips_mode_req);
 
 	mutex_lock(&pwrpriv->lock);
@@ -84,7 +82,7 @@ int ips_leave(struct adapter *padapter)
 static bool rtw_pwr_unassociated_idle(struct adapter *adapter)
 {
 	struct adapter *buddy = adapter->pbuddy_adapter;
-	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
 	struct xmit_priv *pxmit_priv = &adapter->xmitpriv;
 
 	bool ret = false;
@@ -95,27 +93,27 @@ static bool rtw_pwr_unassociated_idle(struct adapter *adapter)
 	if (time_before(jiffies, adapter_to_pwrctl(adapter)->ips_deny_time))
 		goto exit;
 
-	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE|WIFI_SITE_MONITOR)
-		|| check_fwstate(pmlmepriv, WIFI_UNDER_LINKING|WIFI_UNDER_WPS)
-		|| check_fwstate(pmlmepriv, WIFI_AP_STATE)
-		|| check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE|WIFI_ADHOC_STATE)
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE | WIFI_SITE_MONITOR) ||
+	    check_fwstate(pmlmepriv, WIFI_UNDER_LINKING | WIFI_UNDER_WPS) ||
+	    check_fwstate(pmlmepriv, WIFI_AP_STATE) ||
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE | WIFI_ADHOC_STATE)
 	)
 		goto exit;
 
 	/* consider buddy, if exist */
 	if (buddy) {
-		struct mlme_priv *b_pmlmepriv = &(buddy->mlmepriv);
+		struct mlme_priv *b_pmlmepriv = &buddy->mlmepriv;
 
-		if (check_fwstate(b_pmlmepriv, WIFI_ASOC_STATE|WIFI_SITE_MONITOR)
-			|| check_fwstate(b_pmlmepriv, WIFI_UNDER_LINKING|WIFI_UNDER_WPS)
-			|| check_fwstate(b_pmlmepriv, WIFI_AP_STATE)
-			|| check_fwstate(b_pmlmepriv, WIFI_ADHOC_MASTER_STATE|WIFI_ADHOC_STATE)
+		if (check_fwstate(b_pmlmepriv, WIFI_ASOC_STATE | WIFI_SITE_MONITOR) ||
+		    check_fwstate(b_pmlmepriv, WIFI_UNDER_LINKING | WIFI_UNDER_WPS) ||
+		    check_fwstate(b_pmlmepriv, WIFI_AP_STATE) ||
+		    check_fwstate(b_pmlmepriv, WIFI_ADHOC_MASTER_STATE | WIFI_ADHOC_STATE)
 		)
 			goto exit;
 	}
 
 	if (pxmit_priv->free_xmitbuf_cnt != NR_XMITBUFF ||
-		pxmit_priv->free_xmit_extbuf_cnt != NR_XMIT_EXTBUFF) {
+	    pxmit_priv->free_xmit_extbuf_cnt != NR_XMIT_EXTBUFF) {
 		netdev_dbg(adapter->pnetdev,
 			   "There are some pkts to transmit\n");
 		netdev_dbg(adapter->pnetdev,
@@ -131,7 +129,6 @@ static bool rtw_pwr_unassociated_idle(struct adapter *adapter)
 	return ret;
 }
 
-
 /*
  * ATTENTION:
  *rtw_ps_processor() doesn't handle LPS.
@@ -158,7 +155,7 @@ void rtw_ps_processor(struct adapter *padapter)
 	if (!rtw_pwr_unassociated_idle(padapter))
 		goto exit;
 
-	if ((pwrpriv->rf_pwrstate == rf_on) && ((pwrpriv->pwr_state_check_cnts%4) == 0)) {
+	if ((pwrpriv->rf_pwrstate == rf_on) && ((pwrpriv->pwr_state_check_cnts % 4) == 0)) {
 		pwrpriv->change_rfpwrstate = rf_off;
 		{
 			ips_enter(padapter);
@@ -184,8 +181,6 @@ void traffic_check_for_leave_lps(struct adapter *padapter, u8 tx, u32 tx_packets
 	u8 bLeaveLPS = false;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
-
-
 	if (tx) { /* from tx */
 		xmit_cnt += tx_packets;
 
@@ -194,9 +189,9 @@ void traffic_check_for_leave_lps(struct adapter *padapter, u8 tx, u32 tx_packets
 
 		if (jiffies_to_msecs(jiffies - start_time) > 2000) { /*  2 sec == watch dog timer */
 			if (xmit_cnt > 8) {
-				if (adapter_to_pwrctl(padapter)->bLeisurePs
-				    && (adapter_to_pwrctl(padapter)->pwr_mode != PS_MODE_ACTIVE)
-				    && !(hal_btcoex_IsBtControlLps(padapter))) {
+				if (adapter_to_pwrctl(padapter)->bLeisurePs &&
+				    (adapter_to_pwrctl(padapter)->pwr_mode != PS_MODE_ACTIVE) &&
+				    !(hal_btcoex_IsBtControlLps(padapter))) {
 					bLeaveLPS = true;
 				}
 			}
@@ -206,17 +201,17 @@ void traffic_check_for_leave_lps(struct adapter *padapter, u8 tx, u32 tx_packets
 		}
 
 	} else { /*  from rx path */
-		if (pmlmepriv->LinkDetectInfo.NumRxUnicastOkInPeriod > 4/*2*/) {
-			if (adapter_to_pwrctl(padapter)->bLeisurePs
-			    && (adapter_to_pwrctl(padapter)->pwr_mode != PS_MODE_ACTIVE)
-			    && !(hal_btcoex_IsBtControlLps(padapter)))
+		if (pmlmepriv->link_detect_info.num_rx_unicast_ok_in_period > 4) {
+			if (adapter_to_pwrctl(padapter)->bLeisurePs &&
+			    (adapter_to_pwrctl(padapter)->pwr_mode != PS_MODE_ACTIVE) &&
+			    !(hal_btcoex_IsBtControlLps(padapter)))
 				bLeaveLPS = true;
 		}
 	}
 
 	if (bLeaveLPS)
 		/* rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_LEAVE, 1); */
-		rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_LEAVE, tx?0:1);
+		rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_LEAVE, tx ? 0 : 1);
 }
 
 /*
@@ -240,12 +235,10 @@ void rtw_set_rpwm(struct adapter *padapter, u8 pslv)
 		if (pwrpriv->rpwm == pslv ||
 		    (pwrpriv->rpwm >= PS_STATE_S2 && pslv >= PS_STATE_S2))
 			return;
-
 	}
 
 	if ((padapter->bSurpriseRemoved) || !(padapter->hw_init_completed)) {
 		pwrpriv->cpwm = PS_STATE_S4;
-
 		return;
 	}
 
@@ -293,15 +286,16 @@ void rtw_set_rpwm(struct adapter *padapter, u8 pslv)
 				break;
 			}
 		} while (1);
-	} else
+	} else {
 		pwrpriv->cpwm = pslv;
+	}
 }
 
 static u8 PS_RDY_CHECK(struct adapter *padapter)
 {
 	unsigned long curr_time, delta_time;
 	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (pwrpriv->bInSuspend)
 		return false;
@@ -313,11 +307,11 @@ static u8 PS_RDY_CHECK(struct adapter *padapter)
 	if (delta_time < LPS_DELAY_TIME)
 		return false;
 
-	if (check_fwstate(pmlmepriv, WIFI_SITE_MONITOR)
-		|| check_fwstate(pmlmepriv, WIFI_UNDER_LINKING|WIFI_UNDER_WPS)
-		|| check_fwstate(pmlmepriv, WIFI_AP_STATE)
-		|| check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE|WIFI_ADHOC_STATE)
-		|| rtw_is_scan_deny(padapter)
+	if (check_fwstate(pmlmepriv, WIFI_SITE_MONITOR) ||
+	    check_fwstate(pmlmepriv, WIFI_UNDER_LINKING | WIFI_UNDER_WPS) ||
+	    check_fwstate(pmlmepriv, WIFI_AP_STATE) ||
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE | WIFI_ADHOC_STATE) ||
+	    rtw_is_scan_deny(padapter)
 	)
 		return false;
 
@@ -331,7 +325,8 @@ static u8 PS_RDY_CHECK(struct adapter *padapter)
 	return true;
 }
 
-void rtw_set_ps_mode(struct adapter *padapter, u8 ps_mode, u8 smart_ps, u8 bcn_ant_mode, const char *msg)
+void rtw_set_ps_mode(struct adapter *padapter, u8 ps_mode,
+		     u8 smart_ps, u8 bcn_ant_mode, const char *msg)
 {
 	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
 
@@ -342,14 +337,13 @@ void rtw_set_ps_mode(struct adapter *padapter, u8 ps_mode, u8 smart_ps, u8 bcn_a
 		if (ps_mode == PS_MODE_ACTIVE)
 			return;
 
-
 	mutex_lock(&pwrpriv->lock);
 
 	/* if (pwrpriv->pwr_mode == PS_MODE_ACTIVE) */
 	if (ps_mode == PS_MODE_ACTIVE) {
-		if (!(hal_btcoex_IsBtControlLps(padapter))
-				|| (hal_btcoex_IsBtControlLps(padapter)
-					&& !(hal_btcoex_IsLpsOn(padapter)))) {
+		if (!(hal_btcoex_IsBtControlLps(padapter)) ||
+		    (hal_btcoex_IsBtControlLps(padapter) &&
+		    !(hal_btcoex_IsLpsOn(padapter)))) {
 			pwrpriv->pwr_mode = ps_mode;
 			rtw_set_rpwm(padapter, PS_STATE_S4);
 
@@ -359,7 +353,8 @@ void rtw_set_ps_mode(struct adapter *padapter, u8 ps_mode, u8 smart_ps, u8 bcn_a
 			hal_btcoex_LpsNotify(padapter, ps_mode);
 		}
 	} else {
-		if ((PS_RDY_CHECK(padapter) && check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) ||
+		if ((PS_RDY_CHECK(padapter) &&
+		     check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) ||
 		    ((hal_btcoex_IsBtControlLps(padapter)) && (hal_btcoex_IsLpsOn(padapter)))
 			) {
 			u8 pslv;
@@ -404,7 +399,6 @@ s32 LPS_RF_ON_check(struct adapter *padapter, u32 delay_ms)
 	u8 bAwake = false;
 	s32 err = 0;
 
-
 	start_time = jiffies;
 	while (1) {
 		rtw_hal_get_hwreg(padapter, HW_VAR_FWLPS_RF_ON, &bAwake);
@@ -438,7 +432,7 @@ void LPS_Enter(struct adapter *padapter, const char *msg)
 		return;
 
 	/* Skip lps enter request if number of associated adapters is not 1 */
-	if (check_fwstate(&(dvobj->padapters->mlmepriv), WIFI_ASOC_STATE))
+	if (check_fwstate(&dvobj->padapters->mlmepriv, WIFI_ASOC_STATE))
 		n_assoc_iface++;
 	if (n_assoc_iface != 1)
 		return;
@@ -452,10 +446,12 @@ void LPS_Enter(struct adapter *padapter, const char *msg)
 			if (pwrpriv->pwr_mode == PS_MODE_ACTIVE) {
 				scnprintf(buf, sizeof(buf), "WIFI-%s", msg);
 				pwrpriv->bpower_saving = true;
-				rtw_set_ps_mode(padapter, pwrpriv->power_mgnt, padapter->registrypriv.smart_ps, 0, buf);
+				rtw_set_ps_mode(padapter, pwrpriv->power_mgnt,
+						padapter->registrypriv.smart_ps, 0, buf);
 			}
-		} else
+		} else {
 			pwrpriv->LpsIdleCount++;
+		}
 	}
 }
 
@@ -487,7 +483,7 @@ void LPS_Leave(struct adapter *padapter, const char *msg)
 void LeaveAllPowerSaveModeDirect(struct adapter *Adapter)
 {
 	struct adapter *pri_padapter = GET_PRIMARY_ADAPTER(Adapter);
-	struct mlme_priv *pmlmepriv = &(Adapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &Adapter->mlmepriv;
 	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(Adapter);
 
 	if (Adapter->bSurpriseRemoved)
@@ -527,7 +523,7 @@ void LeaveAllPowerSaveMode(struct adapter *Adapter)
 	if (Adapter->bSurpriseRemoved)
 		return;
 
-	if (check_fwstate(&(dvobj->padapters->mlmepriv), WIFI_ASOC_STATE))
+	if (check_fwstate(&dvobj->padapters->mlmepriv, WIFI_ASOC_STATE))
 		n_assoc_iface++;
 
 	if (n_assoc_iface) { /* connect */
@@ -606,7 +602,6 @@ void cpwm_int_hdl(struct adapter *padapter, struct reportpwrstate_parm *preportp
 
 exit:
 	mutex_unlock(&pwrpriv->lock);
-
 }
 
 static void cpwm_event_callback(struct work_struct *work)
@@ -626,7 +621,6 @@ static void rpwmtimeout_workitem_callback(struct work_struct *work)
 	struct dvobj_priv *dvobj;
 	struct pwrctrl_priv *pwrpriv;
 
-
 	pwrpriv = container_of(work, struct pwrctrl_priv, rpwmtimeoutwi);
 	dvobj = pwrctl_to_dvobj(pwrpriv);
 	padapter = dvobj->if1;
@@ -683,7 +677,6 @@ static inline void unregister_task_alive(struct pwrctrl_priv *pwrctrl, u32 tag)
 	pwrctrl->alives &= ~tag;
 }
 
-
 /*
  * Description:
  *Check if the fw_pwrstate is okay for I/O.
@@ -762,10 +755,10 @@ void rtw_unregister_task_alive(struct adapter *padapter, u32 task)
 	unregister_task_alive(pwrctrl, task);
 
 	if ((pwrctrl->pwr_mode != PS_MODE_ACTIVE) && pwrctrl->fw_current_in_ps_mode) {
-		if (pwrctrl->cpwm > pslv)
+		if (pwrctrl->cpwm > pslv) {
 			if ((pslv >= PS_STATE_S2) || (pwrctrl->alives == 0))
 				rtw_set_rpwm(padapter, pslv);
-
+		}
 	}
 
 	mutex_unlock(&pwrctrl->lock);
@@ -991,6 +984,7 @@ void rtw_free_pwrctrl_priv(struct adapter *adapter)
 inline void rtw_set_ips_deny(struct adapter *padapter, u32 ms)
 {
 	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+
 	pwrpriv->ips_deny_time = jiffies + msecs_to_jiffies(ms);
 }
 
@@ -1020,7 +1014,6 @@ int _rtw_pwr_wakeup(struct adapter *padapter, u32 ips_deffer_ms, const char *cal
 	if (time_before(pwrpriv->ips_deny_time, deny_time))
 		pwrpriv->ips_deny_time = deny_time;
 
-
 	if (pwrpriv->ps_processing)
 		while (pwrpriv->ps_processing && jiffies_to_msecs(jiffies - start) <= 3000)
 			mdelay(10);
@@ -1068,7 +1061,6 @@ int _rtw_pwr_wakeup(struct adapter *padapter, u32 ips_deffer_ms, const char *cal
 	if (time_before(pwrpriv->ips_deny_time, deny_time))
 		pwrpriv->ips_deny_time = deny_time;
 	return ret;
-
 }
 
 int rtw_pm_set_lps(struct adapter *padapter, u8 mode)
@@ -1087,8 +1079,9 @@ int rtw_pm_set_lps(struct adapter *padapter, u8 mode)
 			pwrctrlpriv->bLeisurePs =
 				pwrctrlpriv->power_mgnt != PS_MODE_ACTIVE;
 		}
-	} else
+	} else {
 		ret = -EINVAL;
+	}
 
 	return ret;
 }
@@ -1104,8 +1097,9 @@ int rtw_pm_set_ips(struct adapter *padapter, u8 mode)
 		rtw_ips_mode_req(pwrctrlpriv, mode);
 		if ((padapter->bSurpriseRemoved == 0) && (rtw_pwr_wakeup(padapter) == _FAIL))
 			return -EFAULT;
-	} else
+	} else {
 		return -EINVAL;
+	}
 
 	return 0;
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index 337671b1211f0..a52884d2129ff 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -322,7 +322,7 @@ static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup)
 	} else {
 		cur_time = jiffies;
 
-		if (cur_time - psecuritypriv->last_mic_err_time < 60*HZ) {
+		if (cur_time - psecuritypriv->last_mic_err_time < 60 * HZ) {
 			psecuritypriv->btkip_countermeasure = true;
 			psecuritypriv->last_mic_err_time = 0;
 			psecuritypriv->btkip_countermeasure_time = cur_time;
@@ -390,13 +390,13 @@ static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *p
 				mickey = &stainfo->dot11tkiprxmickey.skey[0];
 			}
 
-			datalen = precvframe->u.hdr.len-prxattrib->hdrlen-prxattrib->iv_len-prxattrib->icv_len-8;/* icv_len included the mic code */
+			datalen = precvframe->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len - prxattrib->icv_len - 8;/* icv_len included the mic code */
 			pframe = precvframe->u.hdr.rx_data;
-			payload = pframe+prxattrib->hdrlen+prxattrib->iv_len;
+			payload = pframe + prxattrib->hdrlen + prxattrib->iv_len;
 
 			rtw_seccalctkipmic(mickey, pframe, payload, datalen, &miccode[0], (unsigned char)prxattrib->priority); /* care the length of the data */
 
-			pframemic = payload+datalen;
+			pframemic = payload + datalen;
 
 			bmic_err = false;
 
@@ -444,9 +444,9 @@ static union recv_frame *decryptor(struct adapter *padapter, union recv_frame *p
 	u32  res = _SUCCESS;
 
 	if (prxattrib->encrypt > 0) {
-		u8 *iv = precv_frame->u.hdr.rx_data+prxattrib->hdrlen;
+		u8 *iv = precv_frame->u.hdr.rx_data + prxattrib->hdrlen;
 
-		prxattrib->key_index = (((iv[3])>>6)&0x3);
+		prxattrib->key_index = (((iv[3]) >> 6) & 0x3);
 
 		if (prxattrib->key_index > WEP_KEYS) {
 			switch (prxattrib->encrypt) {
@@ -568,7 +568,7 @@ static signed int recv_decache(union recv_frame *precv_frame, u8 bretry, struct
 {
 	signed int tid = precv_frame->u.hdr.attrib.priority;
 
-	u16 seq_ctrl = ((precv_frame->u.hdr.attrib.seq_num&0xffff) << 4) |
+	u16 seq_ctrl = ((precv_frame->u.hdr.attrib.seq_num & 0xffff) << 4) |
 		(precv_frame->u.hdr.attrib.frag_num & 0xf);
 
 	if (tid > 15)
@@ -632,29 +632,29 @@ static void process_wmmps_data(struct adapter *padapter, union recv_frame *precv
 	if (!psta->qos_option)
 		return;
 
-	if (!(psta->qos_info&0xf))
+	if (!(psta->qos_info & 0xf))
 		return;
 
-	if (psta->state&WIFI_SLEEP_STATE) {
+	if (psta->state & WIFI_SLEEP_STATE) {
 		u8 wmmps_ac = 0;
 
 		switch (pattrib->priority) {
 		case 1:
 		case 2:
-			wmmps_ac = psta->uapsd_bk&BIT(1);
+			wmmps_ac = psta->uapsd_bk & BIT(1);
 			break;
 		case 4:
 		case 5:
-			wmmps_ac = psta->uapsd_vi&BIT(1);
+			wmmps_ac = psta->uapsd_vi & BIT(1);
 			break;
 		case 6:
 		case 7:
-			wmmps_ac = psta->uapsd_vo&BIT(1);
+			wmmps_ac = psta->uapsd_vo & BIT(1);
 			break;
 		case 0:
 		case 3:
 		default:
-			wmmps_ac = psta->uapsd_be&BIT(1);
+			wmmps_ac = psta->uapsd_be & BIT(1);
 			break;
 		}
 
@@ -680,10 +680,10 @@ static void count_rx_stats(struct adapter *padapter, union recv_frame *prframe,
 	sz = get_recvframe_len(prframe);
 	precvpriv->rx_bytes += sz;
 
-	padapter->mlmepriv.LinkDetectInfo.NumRxOkInPeriod++;
+	padapter->mlmepriv.link_detect_info.num_rx_ok_in_period++;
 
 	if ((!is_broadcast_ether_addr(pattrib->dst)) && (!is_multicast_ether_addr(pattrib->dst)))
-		padapter->mlmepriv.LinkDetectInfo.NumRxUnicastOkInPeriod++;
+		padapter->mlmepriv.link_detect_info.num_rx_unicast_ok_in_period++;
 
 	if (sta)
 		psta = sta;
@@ -979,20 +979,20 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 		switch (pattrib->priority) {
 		case 1:
 		case 2:
-			wmmps_ac = psta->uapsd_bk&BIT(0);
+			wmmps_ac = psta->uapsd_bk & BIT(0);
 			break;
 		case 4:
 		case 5:
-			wmmps_ac = psta->uapsd_vi&BIT(0);
+			wmmps_ac = psta->uapsd_vi & BIT(0);
 			break;
 		case 6:
 		case 7:
-			wmmps_ac = psta->uapsd_vo&BIT(0);
+			wmmps_ac = psta->uapsd_vo & BIT(0);
 			break;
 		case 0:
 		case 3:
 		default:
-			wmmps_ac = psta->uapsd_be&BIT(0);
+			wmmps_ac = psta->uapsd_be & BIT(0);
 			break;
 		}
 
@@ -1004,7 +1004,7 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 			psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
 		}
 
-		if ((psta->state&WIFI_SLEEP_STATE) && (pstapriv->sta_dz_bitmap&BIT(psta->aid))) {
+		if ((psta->state & WIFI_SLEEP_STATE) && (pstapriv->sta_dz_bitmap & BIT(psta->aid))) {
 			struct list_head	*xmitframe_plist, *xmitframe_phead;
 			struct xmit_frame *pxmitframe = NULL;
 			struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
@@ -1048,7 +1048,7 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 				/* spin_unlock_bh(&psta->sleep_q.lock); */
 				spin_unlock_bh(&pxmitpriv->lock);
 
-				if (pstapriv->tim_bitmap&BIT(psta->aid)) {
+				if (pstapriv->tim_bitmap & BIT(psta->aid)) {
 					if (psta->sleepq_len == 0) {
 						/* issue nulldata with More data bit = 0 to indicate we have no buffered packets */
 						issue_nulldata_in_interrupt(padapter, psta->hwaddr);
@@ -1339,11 +1339,8 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f
 
 	}
 
-	if (ret == _FAIL) {
+	if (ret == _FAIL || ret == RTW_RX_HANDLED)
 		goto exit;
-	} else if (ret == RTW_RX_HANDLED) {
-		goto exit;
-	}
 
 
 	if (!psta) {
@@ -1430,9 +1427,9 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame
 				goto validate_80211w_fail;
 			precv_frame = decryptor(adapter, precv_frame);
 			/* save actual management data frame body */
-			memcpy(mgmt_DATA, ptr+pattrib->hdrlen+pattrib->iv_len, data_len);
+			memcpy(mgmt_DATA, ptr + pattrib->hdrlen + pattrib->iv_len, data_len);
 			/* overwrite the iv field */
-			memcpy(ptr+pattrib->hdrlen, mgmt_DATA, data_len);
+			memcpy(ptr + pattrib->hdrlen, mgmt_DATA, data_len);
 			/* remove the iv and icv length */
 			pattrib->pkt_len = pattrib->pkt_len - pattrib->iv_len - pattrib->icv_len;
 			kfree(mgmt_DATA);
@@ -1488,7 +1485,7 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
 
 	u8 *ptr = precv_frame->u.hdr.rx_data;
-	u8  ver = (unsigned char) (*ptr)&0x3;
+	u8  ver = (unsigned char) (*ptr) & 0x3;
 
 	/* add version chk */
 	if (ver != 0) {
@@ -1526,7 +1523,7 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 		retval = _FAIL; /*  only data frame return _SUCCESS */
 		break;
 	case WIFI_DATA_TYPE: /* data */
-		pattrib->qos = (subtype & BIT(7)) ? 1:0;
+		pattrib->qos = (subtype & BIT(7)) ? 1 : 0;
 		retval = validate_recv_data_frame(adapter, precv_frame);
 		if (retval == _FAIL) {
 			struct recv_priv *precvpriv = &adapter->recvpriv;
@@ -1571,8 +1568,8 @@ static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	if (pattrib->encrypt)
 		recvframe_pull_tail(precvframe, pattrib->icv_len);
 
-	psnap = (struct ieee80211_snap_hdr	*)(ptr+pattrib->hdrlen + pattrib->iv_len);
-	psnap_type = ptr+pattrib->hdrlen + pattrib->iv_len+SNAP_SIZE;
+	psnap = (struct ieee80211_snap_hdr	*)(ptr + pattrib->hdrlen + pattrib->iv_len);
+	psnap_type = ptr + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
 	/* convert hdr + possible LLC headers into Ethernet header */
 	/* eth_type = (psnap_type[0] << 8) | psnap_type[1]; */
 	if ((!memcmp(psnap, rfc1042_header, SNAP_SIZE) &&
@@ -1586,37 +1583,37 @@ static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 		/* Leave Ethernet header part of hdr and full payload */
 		bsnaphdr = false;
 
-	rmv_len = pattrib->hdrlen + pattrib->iv_len + (bsnaphdr?SNAP_SIZE:0);
+	rmv_len = pattrib->hdrlen + pattrib->iv_len + (bsnaphdr ? SNAP_SIZE : 0);
 	len = precvframe->u.hdr.len - rmv_len;
 
-	memcpy(&be_tmp, ptr+rmv_len, 2);
+	memcpy(&be_tmp, ptr + rmv_len, 2);
 	eth_type = ntohs(be_tmp); /* pattrib->ether_type */
 	pattrib->eth_type = eth_type;
 
 	if ((check_fwstate(pmlmepriv, WIFI_MP_STATE) == true)) {
 		ptr += rmv_len;
 		*ptr = 0x87;
-		*(ptr+1) = 0x12;
+		*(ptr + 1) = 0x12;
 
 		eth_type = 0x8712;
 		/*  append rx status for mp test packets */
-		ptr = recvframe_pull(precvframe, (rmv_len-sizeof(struct ethhdr)+2)-24);
+		ptr = recvframe_pull(precvframe, (rmv_len - sizeof(struct ethhdr) + 2) - 24);
 		if (!ptr)
 			return _FAIL;
 		memcpy(ptr, get_rxmem(precvframe), 24);
 		ptr += 24;
 	} else {
-		ptr = recvframe_pull(precvframe, (rmv_len-sizeof(struct ethhdr) + (bsnaphdr?2:0)));
+		ptr = recvframe_pull(precvframe, (rmv_len - sizeof(struct ethhdr) + (bsnaphdr ? 2 : 0)));
 		if (!ptr)
 			return _FAIL;
 	}
 
 	memcpy(ptr, pattrib->dst, ETH_ALEN);
-	memcpy(ptr+ETH_ALEN, pattrib->src, ETH_ALEN);
+	memcpy(ptr + ETH_ALEN, pattrib->src, ETH_ALEN);
 
 	if (!bsnaphdr) {
 		be_tmp = htons(len);
-		memcpy(ptr+12, &be_tmp, 2);
+		memcpy(ptr + 12, &be_tmp, 2);
 	}
 
 	return _SUCCESS;
@@ -1754,7 +1751,7 @@ static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 		pdata += nSubframe_Length;
 		a_len -= nSubframe_Length;
 		if (a_len != 0) {
-			padding_len = 4 - ((nSubframe_Length + ETH_HLEN) & (4-1));
+			padding_len = 4 - ((nSubframe_Length + ETH_HLEN) & (4 - 1));
 			if (padding_len == 4)
 				padding_len = 0;
 
@@ -2004,7 +2001,7 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 
 			rtw_recv_indicatepkt(padapter, prframe);
 
-			preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1)%4096;
+			preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1) % 4096;
 
 			return _SUCCESS;
 		}
@@ -2014,7 +2011,7 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 
 			retval = amsdu_to_msdu(padapter, prframe);
 
-			preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1)%4096;
+			preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1) % 4096;
 
 			return retval;
 		}
@@ -2200,7 +2197,7 @@ static int recv_func(struct adapter *padapter, union recv_frame *rframe)
 			!psecuritypriv->busetkipkey) {
 			rtw_enqueue_recvframe(rframe, &padapter->recvpriv.uc_swdec_pending_queue);
 
-			if (recvpriv->free_recvframe_cnt < NR_RECVFRAME/4) {
+			if (recvpriv->free_recvframe_cnt < NR_RECVFRAME / 4) {
 				/* to prevent from recvframe starvation, get recvframe from uc_swdec_pending_queue to free_recvframe_cnt  */
 				rframe = rtw_alloc_recvframe(&padapter->recvpriv.uc_swdec_pending_queue);
 				if (rframe)
@@ -2289,19 +2286,19 @@ static void rtw_signal_stat_timer_hdl(struct timer_list *t)
 		}
 
 		/* update value of signal_strength, rssi, signal_qual */
-		tmp_s = (avg_signal_strength+(_alpha-1)*recvpriv->signal_strength);
+		tmp_s = (avg_signal_strength + (_alpha - 1) * recvpriv->signal_strength);
 		if (tmp_s % _alpha)
-			tmp_s = tmp_s/_alpha + 1;
+			tmp_s = tmp_s / _alpha + 1;
 		else
-			tmp_s = tmp_s/_alpha;
+			tmp_s = tmp_s / _alpha;
 		if (tmp_s > 100)
 			tmp_s = 100;
 
-		tmp_q = (avg_signal_qual+(_alpha-1)*recvpriv->signal_qual);
+		tmp_q = (avg_signal_qual + (_alpha - 1) * recvpriv->signal_qual);
 		if (tmp_q % _alpha)
-			tmp_q = tmp_q/_alpha + 1;
+			tmp_q = tmp_q / _alpha + 1;
 		else
-			tmp_q = tmp_q/_alpha;
+			tmp_q = tmp_q / _alpha;
 		if (tmp_q > 100)
 			tmp_q = 100;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index b489babe74324..b98bc1aa9cbe5 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -196,7 +196,6 @@ void rtw_secgetmic(struct mic_data *pmicdata, u8 *dst)
 	secmicclear(pmicdata);
 }
 
-
 void rtw_seccalctkipmic(u8 *key, u8 *header, u8 *data, u32 data_len, u8 *mic_code, u8 pri)
 {
 
@@ -222,7 +221,6 @@ void rtw_seccalctkipmic(u8 *key, u8 *header, u8 *data, u32 data_len, u8 *mic_cod
 	}
 	rtw_secmicappend(&micdata, &priority[0], 4);
 
-
 	rtw_secmicappend(&micdata, data, data_len);
 
 	rtw_secgetmic(&micdata, mic_code);
@@ -282,7 +280,6 @@ static const unsigned short Sbox1[2][256] = {      /* Sbox for hash (can be in R
 	 0x82C3, 0x29B0, 0x5A77, 0x1E11, 0x7BCB, 0xA8FC, 0x6DD6, 0x2C3A,
 	},
 
-
 	{  /* second half of table is unsigned char-reversed version of first! */
 	 0xA5C6, 0x84F8, 0x99EE, 0x8DF6, 0x0DFF, 0xBDD6, 0xB1DE, 0x5491,
 	 0x5060, 0x0302, 0xA9CE, 0x7D56, 0x19E7, 0x62B5, 0xE64D, 0x9AEC,
@@ -357,7 +354,6 @@ static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32)
 	}
 }
 
-
 /*
  * Routine: Phase 2 -- generate RC4KEY, given TK, P1K, IV16
  *
@@ -662,8 +658,8 @@ static void construct_mic_iv(u8 *mic_iv,
 		for (i = 8; i < 14; i++)
 			mic_iv[i] = pn_vector[13 - i];          /* mic_iv[8:13] = PN[5:0] */
 		#endif
-		mic_iv[14] = (unsigned char) (payload_length / 256);
-		mic_iv[15] = (unsigned char) (payload_length % 256);
+		mic_iv[14] = (unsigned char)(payload_length / 256);
+		mic_iv[15] = (unsigned char)(payload_length % 256);
 }
 
 /************************************************/
@@ -785,8 +781,8 @@ static void construct_ctr_preload(u8 *ctr_preload,
 	for (i = 8; i < 14; i++)
 		ctr_preload[i] =    pn_vector[13 - i];          /* ctr_preload[8:13] = PN[5:0] */
 #endif
-	ctr_preload[14] =  (unsigned char) (c / 256); /* Ctr */
-	ctr_preload[15] =  (unsigned char) (c % 256);
+	ctr_preload[14] = (unsigned char)(c / 256); /* Ctr */
+	ctr_preload[15] = (unsigned char)(c % 256);
 }
 
 static signed int aes_cipher(u8 *key, uint	hdrlen,
@@ -1091,7 +1087,7 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 
 	/* start to calculate the mic */
 	if ((hdrlen + plen + 8) <= MAX_MSG_SIZE)
-		memcpy((void *)message, pframe, (hdrlen + plen + 8)); /* 8 is for ext iv len */
+		memcpy(message, pframe, (hdrlen + plen + 8)); /* 8 is for ext iv len */
 
 	pn_vector[0] = pframe[hdrlen];
 	pn_vector[1] = pframe[hdrlen + 1];
@@ -1339,7 +1335,7 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe)
 			goto BIP_exit;
 
 		/* MIC field should be last 8 bytes of packet (packet without FCS) */
-		if (!memcmp(mic, pframe+pattrib->pkt_len-8, 8)) {
+		if (!memcmp(mic, pframe + pattrib->pkt_len - 8, 8)) {
 			pmlmeext->mgnt_80211w_IPN_rx = temp_ipn;
 			res = _SUCCESS;
 		} else {
diff --git a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
index bdd4b6d8fd2e0..07a6db1d2317a 100644
--- a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+++ b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
@@ -79,7 +79,7 @@ u32 _rtw_init_sta_priv(struct	sta_priv *pstapriv)
 	for (i = 0; i < NUM_STA; i++) {
 		_rtw_init_stainfo(psta);
 
-		INIT_LIST_HEAD(&(pstapriv->sta_hash[i]));
+		INIT_LIST_HEAD(&pstapriv->sta_hash[i]);
 
 		list_add_tail(&psta->list, get_list_head(&pstapriv->free_sta_queue));
 
@@ -149,7 +149,7 @@ u32 _rtw_free_sta_priv(struct	sta_priv *pstapriv)
 		/*delete all reordering_ctrl_timer		*/
 		spin_lock_bh(&pstapriv->sta_hash_lock);
 		for (index = 0; index < NUM_STA; index++) {
-			phead = &(pstapriv->sta_hash[index]);
+			phead = &pstapriv->sta_hash[index];
 			list_for_each(plist, phead) {
 				int i;
 
@@ -186,15 +186,15 @@ struct	sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr)
 	pfree_sta_queue = &pstapriv->free_sta_queue;
 
 	/* spin_lock_bh(&(pfree_sta_queue->lock)); */
-	spin_lock_bh(&(pstapriv->sta_hash_lock));
+	spin_lock_bh(&pstapriv->sta_hash_lock);
 	if (list_empty(&pfree_sta_queue->queue)) {
 		/* spin_unlock_bh(&(pfree_sta_queue->lock)); */
-		spin_unlock_bh(&(pstapriv->sta_hash_lock));
+		spin_unlock_bh(&pstapriv->sta_hash_lock);
 		return NULL;
 	}
 	psta = container_of(get_next(&pfree_sta_queue->queue), struct sta_info, list);
 
-	list_del_init(&(psta->list));
+	list_del_init(&psta->list);
 
 	/* spin_unlock_bh(&(pfree_sta_queue->lock)); */
 
@@ -207,11 +207,11 @@ struct	sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr)
 	index = wifi_mac_hash(hwaddr);
 
 	if (index >= NUM_STA) {
-		spin_unlock_bh(&(pstapriv->sta_hash_lock));
+		spin_unlock_bh(&pstapriv->sta_hash_lock);
 		psta = NULL;
 		goto exit;
 	}
-	phash_list = &(pstapriv->sta_hash[index]);
+	phash_list = &pstapriv->sta_hash[index];
 
 	/* spin_lock_bh(&(pstapriv->sta_hash_lock)); */
 
@@ -258,7 +258,7 @@ struct	sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr)
 
 	/* init for the sequence number of received management frame */
 	psta->RxMgmtFrameSeqNum = 0xffff;
-	spin_unlock_bh(&(pstapriv->sta_hash_lock));
+	spin_unlock_bh(&pstapriv->sta_hash_lock);
 	/* alloc mac id for non-bc/mc station, */
 	rtw_alloc_macid(pstapriv->padapter, psta);
 
@@ -300,7 +300,7 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 	/* vo */
 	/* spin_lock_bh(&(pxmitpriv->vo_pending.lock)); */
 	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->vo_q.sta_pending);
-	list_del_init(&(pstaxmitpriv->vo_q.tx_pending));
+	list_del_init(&pstaxmitpriv->vo_q.tx_pending);
 	phwxmit = pxmitpriv->hwxmits;
 	phwxmit->accnt -= pstaxmitpriv->vo_q.qcnt;
 	pstaxmitpriv->vo_q.qcnt = 0;
@@ -309,7 +309,7 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 	/* vi */
 	/* spin_lock_bh(&(pxmitpriv->vi_pending.lock)); */
 	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->vi_q.sta_pending);
-	list_del_init(&(pstaxmitpriv->vi_q.tx_pending));
+	list_del_init(&pstaxmitpriv->vi_q.tx_pending);
 	phwxmit = pxmitpriv->hwxmits + 1;
 	phwxmit->accnt -= pstaxmitpriv->vi_q.qcnt;
 	pstaxmitpriv->vi_q.qcnt = 0;
@@ -318,7 +318,7 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 	/* be */
 	/* spin_lock_bh(&(pxmitpriv->be_pending.lock)); */
 	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->be_q.sta_pending);
-	list_del_init(&(pstaxmitpriv->be_q.tx_pending));
+	list_del_init(&pstaxmitpriv->be_q.tx_pending);
 	phwxmit = pxmitpriv->hwxmits + 2;
 	phwxmit->accnt -= pstaxmitpriv->be_q.qcnt;
 	pstaxmitpriv->be_q.qcnt = 0;
@@ -327,7 +327,7 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 	/* bk */
 	/* spin_lock_bh(&(pxmitpriv->bk_pending.lock)); */
 	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->bk_q.sta_pending);
-	list_del_init(&(pstaxmitpriv->bk_q.tx_pending));
+	list_del_init(&pstaxmitpriv->bk_q.tx_pending);
 	phwxmit = pxmitpriv->hwxmits + 3;
 	phwxmit->accnt -= pstaxmitpriv->bk_q.qcnt;
 	pstaxmitpriv->bk_q.qcnt = 0;
@@ -369,7 +369,7 @@ u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 
 			plist = get_next(plist);
 
-			list_del_init(&(prframe->u.hdr.list));
+			list_del_init(&prframe->u.hdr.list);
 
 			rtw_free_recvframe(prframe, pfree_recv_queue);
 		}
@@ -435,7 +435,7 @@ void rtw_free_all_stainfo(struct adapter *padapter)
 	spin_lock_bh(&pstapriv->sta_hash_lock);
 
 	for (index = 0; index < NUM_STA; index++) {
-		phead = &(pstapriv->sta_hash[index]);
+		phead = &pstapriv->sta_hash[index];
 		list_for_each_safe(plist, tmp, phead) {
 			psta = list_entry(plist, struct sta_info, hash_list);
 
@@ -473,7 +473,7 @@ struct sta_info *rtw_get_stainfo(struct sta_priv *pstapriv, u8 *hwaddr)
 
 	spin_lock_bh(&pstapriv->sta_hash_lock);
 
-	phead = &(pstapriv->sta_hash[index]);
+	phead = &pstapriv->sta_hash[index];
 	list_for_each(plist, phead) {
 		psta = list_entry(plist, struct sta_info, hash_list);
 
@@ -491,7 +491,7 @@ struct sta_info *rtw_get_stainfo(struct sta_priv *pstapriv, u8 *hwaddr)
 u32 rtw_init_bcmc_stainfo(struct adapter *padapter)
 {
 	struct sta_info *psta;
-	NDIS_802_11_MAC_ADDRESS	bcast_addr = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 bcast_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
 	struct	sta_priv *pstapriv = &padapter->stapriv;
 	/* struct __queue	*pstapending = &padapter->xmitpriv.bm_pending; */
@@ -525,7 +525,7 @@ u8 rtw_access_ctrl(struct adapter *padapter, u8 *mac_addr)
 	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
 	struct __queue	*pacl_node_q = &pacl_list->acl_node_q;
 
-	spin_lock_bh(&(pacl_node_q->lock));
+	spin_lock_bh(&pacl_node_q->lock);
 	phead = get_list_head(pacl_node_q);
 	list_for_each(plist, phead) {
 		paclnode = list_entry(plist, struct rtw_wlan_acl_node, list);
@@ -536,7 +536,7 @@ u8 rtw_access_ctrl(struct adapter *padapter, u8 *mac_addr)
 				break;
 			}
 	}
-	spin_unlock_bh(&(pacl_node_q->lock));
+	spin_unlock_bh(&pacl_node_q->lock);
 
 	if (pacl_list->mode == 1) /* accept unless in deny list */
 		res = !match;
diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
index 3242978da36ca..6a7c09db4cd93 100644
--- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
@@ -572,7 +572,7 @@ s16 rtw_camid_alloc(struct adapter *adapter, struct sta_info *sta, u8 kid)
 
 	if ((((mlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) || ((mlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE))
 		&& !sta) {
-		/* AP/Ad-hoc mode group key: static alloction to default key by key ID */
+		/* AP/Ad-hoc mode group key: static allocation to default key by key ID */
 		if (kid > 3) {
 			netdev_dbg(adapter->pnetdev,
 				   FUNC_ADPT_FMT " group key with invalid key id:%u\n",
@@ -597,7 +597,9 @@ s16 rtw_camid_alloc(struct adapter *adapter, struct sta_info *sta, u8 kid)
 
 		i = _rtw_camid_search(adapter, addr, kid);
 		if (i >= 0) {
-			/* Fix issue that pairwise and group key have same key id. Pairwise key first, group key can overwirte group only(ex: rekey) */
+			/* Fix issue that pairwise and group key have same key id.
+			 * Pairwise key first, group key can overwrite group only(ex: rekey)
+			 */
 			if (sta || _rtw_camid_is_gk(adapter, i))
 				cam_id = i;
 			else
@@ -704,7 +706,7 @@ static void sort_wmm_ac_params(u32 *inx, u32 *edca)
 {
 	u32 i, j, change_inx = false;
 
-	/* entry indx: 0->vo, 1->vi, 2->be, 3->bk. */
+	/* entry index: 0->vo, 1->vi, 2->be, 3->bk. */
 	for (i = 0; i < 4; i++) {
 		for (j = i + 1; j < 4; j++) {
 			/* compare CW and AIFS */
@@ -1593,7 +1595,7 @@ int update_sta_support_rate(struct adapter *padapter, u8 *pvar_ie, uint var_ie_l
 {
 	unsigned int	ie_len;
 	struct ndis_80211_var_ie *pIE;
-	int	supportRateNum = 0;
+	int	support_rate_num = 0;
 	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
 	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
 
@@ -1604,11 +1606,11 @@ int update_sta_support_rate(struct adapter *padapter, u8 *pvar_ie, uint var_ie_l
 		return _FAIL;
 
 	memcpy(pmlmeinfo->FW_sta_info[cam_idx].SupportedRates, pIE->data, ie_len);
-	supportRateNum = ie_len;
+	support_rate_num = ie_len;
 
 	pIE = (struct ndis_80211_var_ie *)rtw_get_ie(pvar_ie, WLAN_EID_EXT_SUPP_RATES, &ie_len, var_ie_len);
-	if (pIE && (ie_len <= sizeof(pmlmeinfo->FW_sta_info[cam_idx].SupportedRates) - supportRateNum))
-		memcpy((pmlmeinfo->FW_sta_info[cam_idx].SupportedRates + supportRateNum), pIE->data, ie_len);
+	if (pIE && (ie_len <= sizeof(pmlmeinfo->FW_sta_info[cam_idx].SupportedRates) - support_rate_num))
+		memcpy((pmlmeinfo->FW_sta_info[cam_idx].SupportedRates + support_rate_num), pIE->data, ie_len);
 
 	return _SUCCESS;
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c
index 7b18be8912e69..114b2c057f28a 100644
--- a/drivers/staging/rtl8723bs/core/rtw_xmit.c
+++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c
@@ -5,6 +5,8 @@
  *
  ******************************************************************************/
 #include <drv_types.h>
+#include <linux/delay.h>
+#include <linux/if_ether.h>
 
 static u8 P802_1H_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0xf8 };
 static u8 RFC1042_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0x00 };
@@ -128,7 +130,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		/* Tx buf allocation may fail sometimes, so sleep and retry. */
 		res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ), true);
 		if (res == _FAIL) {
-			msleep(10);
+			fsleep(10 * USEC_PER_MSEC);
 			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ), true);
 			if (res == _FAIL)
 				goto exit;
@@ -1414,7 +1416,7 @@ void rtw_count_tx_stats(struct adapter *padapter, struct xmit_frame *pxmitframe,
 	if ((pxmitframe->frame_tag & 0x0f) == DATA_FRAMETAG) {
 		pkt_num = pxmitframe->agg_num;
 
-		pmlmepriv->LinkDetectInfo.NumTxOkInPeriod += pkt_num;
+		pmlmepriv->link_detect_info.num_tx_ok_in_period += pkt_num;
 
 		pxmitpriv->tx_pkts += pkt_num;
 
@@ -2026,8 +2028,8 @@ inline bool xmitframe_hiq_filter(struct xmit_frame *xmitframe)
 	if (registry->hiq_filter == RTW_HIQ_FILTER_ALLOW_SPECIAL) {
 		struct pkt_attrib *attrib = &xmitframe->attrib;
 
-		if (attrib->ether_type == 0x0806 ||
-		    attrib->ether_type == 0x888e ||
+		if (attrib->ether_type == ETH_P_ARP ||
+		    attrib->ether_type == ETH_P_PAE ||
 		    attrib->dhcp_pkt
 		)
 			allow = true;
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
index b3d7f50fac4c6..b3e34f97cfc6b 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
@@ -684,52 +684,22 @@ static void halbtc8723b1ant_CoexTableWithType(
 	struct btc_coexist *pBtCoexist, bool bForceExec, u8 type
 )
 {
+	static const u32 table[8][2] = {
+		{0x55555555, 0x55555555},
+		{0x55555555, 0x5a5a5a5a},
+		{0x5a5a5a5a, 0x5a5a5a5a},
+		{0xaaaa5555, 0xaaaa5a5a},
+		{0x55555555, 0xaaaa5a5a},
+		{0x5a5a5a5a, 0xaaaa5a5a},
+		{0x55555555, 0xaaaaaaaa},
+		{0xaaaaaaaa, 0xaaaaaaaa}
+	};
+
 	pCoexSta->nCoexTableType = type;
 
-	switch (type) {
-	case 0:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x55555555, 0x55555555, 0xffffff, 0x3
-		);
-		break;
-	case 1:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x55555555, 0x5a5a5a5a, 0xffffff, 0x3
-		);
-		break;
-	case 2:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x5a5a5a5a, 0x5a5a5a5a, 0xffffff, 0x3
-		);
-		break;
-	case 3:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0xaaaa5555, 0xaaaa5a5a, 0xffffff, 0x3
-		);
-		break;
-	case 4:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x55555555, 0xaaaa5a5a, 0xffffff, 0x3
-		);
-		break;
-	case 5:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x5a5a5a5a, 0xaaaa5a5a, 0xffffff, 0x3
-		);
-		break;
-	case 6:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0x55555555, 0xaaaaaaaa, 0xffffff, 0x3
-		);
-		break;
-	case 7:
-		halbtc8723b1ant_CoexTable(
-			pBtCoexist, bForceExec, 0xaaaaaaaa, 0xaaaaaaaa, 0xffffff, 0x3
-		);
-		break;
-	default:
-		break;
-	}
+	if (type < 8)
+		halbtc8723b1ant_CoexTable(pBtCoexist, bForceExec, table[type][0],
+					  table[type][1], 0xffffff, 0x3);
 }
 
 static void halbtc8723b1ant_SetFwIgnoreWlanAct(
@@ -2316,7 +2286,7 @@ void EXhalbtc8723b1ant_ConnectNotify(struct btc_coexist *pBtCoexist, u8 type)
 
 	if (type == BTC_ASSOCIATE_START) {
 		pCoexSta->bWiFiIsHighPriTask = true;
-		 pCoexDm->nArpCnt = 0;
+		pCoexDm->nArpCnt = 0;
 	} else {
 		pCoexSta->bWiFiIsHighPriTask = false;
 		/* pCoexDm->nArpCnt = 0; */
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
index 9df3274c10480..8f6849f2277e1 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
@@ -121,25 +121,15 @@ static void setIqkMatrix_8723B(
 
 static void setCCKFilterCoefficient(struct dm_odm_t *pDM_Odm, u8 CCKSwingIndex)
 {
-	if (!pDM_Odm->RFCalibrateInfo.bCCKinCH14) {
-		rtw_write8(pDM_Odm->Adapter, 0xa22, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][0]);
-		rtw_write8(pDM_Odm->Adapter, 0xa23, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][1]);
-		rtw_write8(pDM_Odm->Adapter, 0xa24, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][2]);
-		rtw_write8(pDM_Odm->Adapter, 0xa25, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][3]);
-		rtw_write8(pDM_Odm->Adapter, 0xa26, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][4]);
-		rtw_write8(pDM_Odm->Adapter, 0xa27, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][5]);
-		rtw_write8(pDM_Odm->Adapter, 0xa28, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][6]);
-		rtw_write8(pDM_Odm->Adapter, 0xa29, CCKSwingTable_Ch1_Ch13_New[CCKSwingIndex][7]);
-	} else {
-		rtw_write8(pDM_Odm->Adapter, 0xa22, CCKSwingTable_Ch14_New[CCKSwingIndex][0]);
-		rtw_write8(pDM_Odm->Adapter, 0xa23, CCKSwingTable_Ch14_New[CCKSwingIndex][1]);
-		rtw_write8(pDM_Odm->Adapter, 0xa24, CCKSwingTable_Ch14_New[CCKSwingIndex][2]);
-		rtw_write8(pDM_Odm->Adapter, 0xa25, CCKSwingTable_Ch14_New[CCKSwingIndex][3]);
-		rtw_write8(pDM_Odm->Adapter, 0xa26, CCKSwingTable_Ch14_New[CCKSwingIndex][4]);
-		rtw_write8(pDM_Odm->Adapter, 0xa27, CCKSwingTable_Ch14_New[CCKSwingIndex][5]);
-		rtw_write8(pDM_Odm->Adapter, 0xa28, CCKSwingTable_Ch14_New[CCKSwingIndex][6]);
-		rtw_write8(pDM_Odm->Adapter, 0xa29, CCKSwingTable_Ch14_New[CCKSwingIndex][7]);
-	}
+	u8 (*swingtable)[8];
+
+	if (!pDM_Odm->RFCalibrateInfo.bCCKinCH14)
+		swingtable = CCKSwingTable_Ch1_Ch13_New;
+	else
+		swingtable = CCKSwingTable_Ch14_New;
+
+	for (int i = 0; i < 8; i++)
+		rtw_write8(pDM_Odm->Adapter, 0xa22 + i, swingtable[CCKSwingIndex][i]);
 }
 
 /*-----------------------------------------------------------------------------
@@ -208,8 +198,6 @@ void ODM_TxPwrTrackSetPwr_8723B(
 	}
 
 	if (Method == TXAGC) {
-		struct adapter *Adapter = pDM_Odm->Adapter;
-
 		pDM_Odm->Remnant_OFDMSwingIdx[RFPath] = pDM_Odm->Absolute_OFDMSwingIdx[RFPath];
 
 		pDM_Odm->Modify_TxAGC_Flag_PathA = true;
@@ -1407,8 +1395,10 @@ static void phy_IQCalibrate_8723B(
 			PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 			pDM_Odm->RFCalibrateInfo.TxLOK[RF_PATH_A] = PHY_QueryRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x8, bRFRegOffsetMask);
 
-				result[t][0] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_Before_IQK_A, bMaskDWord)&0x3FF0000)>>16;
-				result[t][1] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_After_IQK_A, bMaskDWord)&0x3FF0000)>>16;
+			result[t][0] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_Before_IQK_A,
+						       bMaskDWord) & 0x3FF0000) >> 16;
+			result[t][1] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_After_IQK_A,
+						       bMaskDWord) & 0x3FF0000) >> 16;
 			break;
 		}
 	}
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index 9105594d2dde0..2a2dd60be8bb9 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -167,7 +167,7 @@ static u8 halbtcoutsrc_IsWifiBusy(struct adapter *padapter)
 	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == true) {
 		if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
 			return true;
-		if (pmlmepriv->LinkDetectInfo.bBusyTraffic)
+		if (pmlmepriv->link_detect_info.busy_traffic)
 			return true;
 	}
 
@@ -364,9 +364,10 @@ static u8 halbtcoutsrc_Get(void *pBtcContext, u8 getType, void *pOutBuf)
 	case BTC_GET_U4_WIFI_TRAFFIC_DIRECTION:
 		{
 			struct rt_link_detect_t *plinkinfo;
-			plinkinfo = &padapter->mlmepriv.LinkDetectInfo;
 
-			if (plinkinfo->NumTxOkInPeriod > plinkinfo->NumRxOkInPeriod)
+			plinkinfo = &padapter->mlmepriv.link_detect_info;
+
+			if (plinkinfo->num_tx_ok_in_period > plinkinfo->num_rx_ok_in_period)
 				*pU4Tmp = BTC_WIFI_TRAFFIC_TX;
 			else
 				*pU4Tmp = BTC_WIFI_TRAFFIC_RX;
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index 31b3e880ae6ab..50370b14ce7c5 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -616,6 +616,7 @@ void SetHwReg(struct adapter *adapter, u8 variable, u8 *val)
 	case HW_VAR_DM_FUNC_SET:
 		if (*((u32 *)val) == DYNAMIC_ALL_FUNC_ENABLE) {
 			struct dm_priv *dm = &hal_data->dmpriv;
+
 			dm->DMFlag = dm->InitDMFlag;
 			odm->SupportAbility = dm->InitODMFlag;
 		} else {
@@ -727,6 +728,7 @@ void SetHalODMVar(
 	case HAL_ODM_STA_INFO:
 		{
 			struct sta_info *psta = pValue1;
+
 			if (bSet) {
 				ODM_CmnInfoPtrArrayHook(podmpriv, ODM_CMNINFO_STA_STATUS, psta->mac_id, psta);
 			} else {
@@ -750,18 +752,6 @@ void SetHalODMVar(
 }
 
 
-bool eqNByte(u8 *str1, u8 *str2, u32 num)
-{
-	if (num == 0)
-		return false;
-	while (num > 0) {
-		num--;
-		if (str1[num] != str2[num])
-			return false;
-	}
-	return true;
-}
-
 bool GetU1ByteIntegerFromStringInDecimal(char *Str, u8 *pInt)
 {
 	u16 i = 0;
diff --git a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
index dc2da49e6738e..bdd595a99b980 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
@@ -8,6 +8,7 @@
 #include <drv_types.h>
 #include <hal_data.h>
 #include <linux/kernel.h>
+#include <linux/string.h>
 
 u8 PHY_GetTxPowerByRateBase(struct adapter *Adapter, u8 RfPath,
 			    enum rate_section RateSection)
@@ -423,6 +424,7 @@ void PHY_SetTxPowerIndexByRateSection(
 
 	if (RateSection == CCK) {
 		u8 cckRates[]   = {MGN_1M, MGN_2M, MGN_5_5M, MGN_11M};
+
 		PHY_SetTxPowerIndexByRateArray(padapter, RFPath,
 					     pHalData->CurrentChannelBW,
 					     Channel, cckRates,
@@ -430,6 +432,7 @@ void PHY_SetTxPowerIndexByRateSection(
 
 	} else if (RateSection == OFDM) {
 		u8 ofdmRates[]  = {MGN_6M, MGN_9M, MGN_12M, MGN_18M, MGN_24M, MGN_36M, MGN_48M, MGN_54M};
+
 		PHY_SetTxPowerIndexByRateArray(padapter, RFPath,
 					       pHalData->CurrentChannelBW,
 					       Channel, ofdmRates,
@@ -437,6 +440,7 @@ void PHY_SetTxPowerIndexByRateSection(
 
 	} else if (RateSection == HT_MCS0_MCS7) {
 		u8 htRates1T[]  = {MGN_MCS0, MGN_MCS1, MGN_MCS2, MGN_MCS3, MGN_MCS4, MGN_MCS5, MGN_MCS6, MGN_MCS7};
+
 		PHY_SetTxPowerIndexByRateArray(padapter, RFPath,
 					       pHalData->CurrentChannelBW,
 					       Channel, htRates1T,
@@ -500,6 +504,7 @@ s8 PHY_GetTxPowerTrackingOffset(struct adapter *padapter, u8 RFPath, u8 Rate)
 u8 PHY_GetRateIndexOfTxPowerByRate(u8 Rate)
 {
 	u8 index = 0;
+
 	switch (Rate) {
 	case MGN_1M:
 		index = 0;
@@ -819,27 +824,27 @@ void PHY_SetTxPowerLimit(
 
 	powerLimit = powerLimit > MAX_POWER_INDEX ? MAX_POWER_INDEX : powerLimit;
 
-	if (eqNByte(Regulation, (u8 *)("FCC"), 3))
+	if (strcmp(Regulation, "FCC") == 0)
 		regulation = 0;
-	else if (eqNByte(Regulation, (u8 *)("MKK"), 3))
+	else if (strcmp(Regulation, "MKK") == 0)
 		regulation = 1;
-	else if (eqNByte(Regulation, (u8 *)("ETSI"), 4))
+	else if (strcmp(Regulation, "ETSI") == 0)
 		regulation = 2;
-	else if (eqNByte(Regulation, (u8 *)("WW13"), 4))
+	else if (strcmp(Regulation, "WW13") == 0)
 		regulation = 3;
 
-	if (eqNByte(RateSection, (u8 *)("CCK"), 3) && eqNByte(RfPath, (u8 *)("1T"), 2))
+	if (strcmp(RateSection, "CCK") == 0 && strcmp(RfPath, "1T") == 0)
 		rateSection = 0;
-	else if (eqNByte(RateSection, (u8 *)("OFDM"), 4) && eqNByte(RfPath, (u8 *)("1T"), 2))
+	else if (strcmp(RateSection, "OFDM") == 0 && strcmp(RfPath, "1T") == 0)
 		rateSection = 1;
-	else if (eqNByte(RateSection, (u8 *)("HT"), 2) && eqNByte(RfPath, (u8 *)("1T"), 2))
+	else if (strcmp(RateSection, "HT") == 0 && strcmp(RfPath, "1T") == 0)
 		rateSection = 2;
 	else
 		return;
 
-	if (eqNByte(Bandwidth, (u8 *)("20M"), 3))
+	if (strcmp(Bandwidth, "20M") == 0)
 		bandwidth = 0;
-	else if (eqNByte(Bandwidth, (u8 *)("40M"), 3))
+	else if (strcmp(Bandwidth, "40M") == 0)
 		bandwidth = 1;
 
 	channelIndex = phy_GetChannelIndexOfTxPowerLimit(channel);
@@ -856,6 +861,7 @@ void PHY_SetTxPowerLimit(
 void Hal_ChannelPlanToRegulation(struct adapter *Adapter, u16 ChannelPlan)
 {
 	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
+
 	pHalData->Regulation2_4G = TXPWR_LMT_WW;
 
 	switch (ChannelPlan) {
diff --git a/drivers/staging/rtl8723bs/hal/odm.h b/drivers/staging/rtl8723bs/hal/odm.h
index 1c929d88e5963..714bab71d8211 100644
--- a/drivers/staging/rtl8723bs/hal/odm.h
+++ b/drivers/staging/rtl8723bs/hal/odm.h
@@ -202,14 +202,7 @@ struct odm_rate_adaptive {
 /*  */
 /*  Declare for common info */
 /*  */
-#define MAX_PATH_NUM_92CS		2
-#define MAX_PATH_NUM_8188E		1
-#define MAX_PATH_NUM_8192E		2
 #define MAX_PATH_NUM_8723B		1
-#define MAX_PATH_NUM_8812A		2
-#define MAX_PATH_NUM_8821A		1
-#define MAX_PATH_NUM_8814A		4
-#define MAX_PATH_NUM_8822B		2
 
 #define IQK_THRESHOLD			8
 #define DPK_THRESHOLD			4
@@ -985,7 +978,7 @@ struct dm_odm_t { /* DM_Out_Source_Dynamic_Mechanism_Structure */
 	#endif
 };
 
- enum odm_rf_content {
+enum odm_rf_content {
 	odm_radioa_txt = 0x1000,
 	odm_radiob_txt = 0x1001,
 	odm_radioc_txt = 0x1002,
diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.c b/drivers/staging/rtl8723bs/hal/odm_DIG.c
index f10427abd8490..33661182d85a7 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DIG.c
+++ b/drivers/staging/rtl8723bs/hal/odm_DIG.c
@@ -220,9 +220,8 @@ void odm_Adaptivity(void *pDM_VOID, u8 IGI)
 	s8 Diff, IGI_target;
 	bool EDCCA_State = false;
 
-	if (!(pDM_Odm->SupportAbility & ODM_BB_ADAPTIVITY)) {
+	if (!(pDM_Odm->SupportAbility & ODM_BB_ADAPTIVITY))
 		return;
-	}
 
 	if (*pDM_Odm->pBandWidth == ODM_BW20M) /* CHANNEL_WIDTH_20 */
 		IGI_target = pDM_Odm->IGI_Base;
@@ -286,16 +285,14 @@ void ODM_Write_DIG(void *pDM_VOID, u8 CurrentIGI)
 	struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID;
 	struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable;
 
-	if (pDM_DigTable->bStopDIG) {
+	if (pDM_DigTable->bStopDIG)
 		return;
-	}
 
 	if (pDM_DigTable->CurIGValue != CurrentIGI) {
 		/* 1 Check initial gain by upper bound */
 		if (!pDM_DigTable->bPSDInProgress) {
-			if (CurrentIGI > pDM_DigTable->rx_gain_range_max) {
+			if (CurrentIGI > pDM_DigTable->rx_gain_range_max)
 				CurrentIGI = pDM_DigTable->rx_gain_range_max;
-			}
 
 		}
 
@@ -314,24 +311,20 @@ bool odm_DigAbort(void *pDM_VOID)
 	struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID;
 
 	/* SupportAbility */
-	if (!(pDM_Odm->SupportAbility & ODM_BB_FA_CNT)) {
+	if (!(pDM_Odm->SupportAbility & ODM_BB_FA_CNT))
 		return	true;
-	}
 
 	/* SupportAbility */
-	if (!(pDM_Odm->SupportAbility & ODM_BB_DIG)) {
+	if (!(pDM_Odm->SupportAbility & ODM_BB_DIG))
 		return	true;
-	}
 
 	/* ScanInProcess */
-	if (*(pDM_Odm->pbScanInProcess)) {
+	if (*(pDM_Odm->pbScanInProcess))
 		return	true;
-	}
 
 	/* add by Neil Chen to avoid PSD is processing */
-	if (pDM_Odm->bDMInitialGainEnable == false) {
+	if (pDM_Odm->bDMInitialGainEnable == false)
 		return	true;
-	}
 
 	return	false;
 }
@@ -413,9 +406,9 @@ void odm_DIG(void *pDM_VOID)
 	if (pDM_Odm->bLinked && bPerformance) {
 		/* 2 Modify DIG upper bound */
 		/* 4 Modify DIG upper bound for 92E, 8723A\B, 8821 & 8812 BT */
-		if (pDM_Odm->bBtLimitedDig == 1) {
+		if (pDM_Odm->bBtLimitedDig == 1)
 			offset = 10;
-		} else
+		else
 			offset = 15;
 
 		if ((pDM_Odm->RSSI_Min + offset) > dm_dig_max)
@@ -475,9 +468,8 @@ void odm_DIG(void *pDM_VOID)
 	}
 
 	/* 2 Abnormal lower bound case */
-	if (pDM_DigTable->rx_gain_range_min > pDM_DigTable->rx_gain_range_max) {
+	if (pDM_DigTable->rx_gain_range_min > pDM_DigTable->rx_gain_range_max)
 		pDM_DigTable->rx_gain_range_min = pDM_DigTable->rx_gain_range_max;
-	}
 
 
 	/* 1 False alarm threshold decision */
diff --git a/drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c b/drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c
index 1df42069bd5cb..b0ca46aec1a52 100644
--- a/drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c
+++ b/drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.c
@@ -125,9 +125,8 @@ void odm_ConfigBB_PHY_REG_PG_8723B(
 {
 	if (Addr == 0xfe || Addr == 0xffe)
 		msleep(50);
-	else {
+	else
 		PHY_StoreTxPowerByRate(pDM_Odm->Adapter, RfPath, Addr, Bitmask, Data);
-	}
 }
 
 void odm_ConfigBB_PHY_8723B(
@@ -149,9 +148,8 @@ void odm_ConfigBB_PHY_8723B(
 		udelay(5);
 	else if (Addr == 0xf9)
 		udelay(1);
-	else {
+	else
 		PHY_SetBBReg(pDM_Odm->Adapter, Addr, Bitmask, Data);
-	}
 
 	/*  Add 1us delay between BB/RF register setting. */
 	udelay(1);
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
index af6cdda8238d8..12416e499ac35 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
@@ -672,6 +672,7 @@ void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 		if (padapter->bSurpriseRemoved || padapter->bDriverStopped) {
 		} else {
 			struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+
 			pwrctl->fw_psmode_iface_id = padapter->iface_id;
 		}
 
@@ -935,6 +936,7 @@ void rtl8723b_download_BTCoex_AP_mode_rsvd_page(struct adapter *padapter)
 
 	if (bcn_valid) {
 		struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+
 		pwrctl->fw_psmode_iface_id = padapter->iface_id;
 	}
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index 8d259820f1037..e794fe3caf9d6 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -1185,12 +1185,12 @@ void Hal_InitPGData(struct adapter *padapter, u8 *PROMContent)
 		if (!pEEPROM->EepromOrEfuse) {
 			/*  Read EFUSE real map to shadow. */
 			EFUSE_ShadowMapUpdate(padapter, EFUSE_WIFI);
-			memcpy((void *)PROMContent, (void *)pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
+			memcpy(PROMContent, pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
 		}
 	} else {/* autoload fail */
 		if (!pEEPROM->EepromOrEfuse)
 			EFUSE_ShadowMapUpdate(padapter, EFUSE_WIFI);
-		memcpy((void *)PROMContent, (void *)pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
+		memcpy(PROMContent, pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
 	}
 }
 
@@ -1631,6 +1631,7 @@ static void rtl8723b_cal_txdesc_chksum(struct tx_desc *ptxdesc)
 static u8 fill_txdesc_sectype(struct pkt_attrib *pattrib)
 {
 	u8 sectype = 0;
+
 	if ((pattrib->encrypt > 0) && !pattrib->bswenc) {
 		switch (pattrib->encrypt) {
 		/*  SEC_TYPE */
@@ -2031,6 +2032,7 @@ static void hw_var_set_bcn_func(struct adapter *padapter, u8 variable, u8 *val)
 		rtw_write8(padapter, bcn_ctrl_reg, (EN_BCN_FUNCTION | EN_TXBCN_RPT));
 	else {
 		u8 val8;
+
 		val8 = rtw_read8(padapter, bcn_ctrl_reg);
 		val8 &= ~(EN_BCN_FUNCTION | EN_TXBCN_RPT);
 
@@ -2225,6 +2227,7 @@ s32 c2h_id_filter_ccx_8723b(u8 *buf)
 {
 	struct c2h_evt_hdr_88xx *c2h_evt = (struct c2h_evt_hdr_88xx *)buf;
 	s32 ret = false;
+
 	if (c2h_evt->id == C2H_CCX_TX_RPT)
 		ret = true;
 
@@ -2314,6 +2317,7 @@ void C2HPacketHandler_8723B(struct adapter *padapter, u8 *pbuffer, u16 length)
 {
 	struct c2h_evt_hdr_t	C2hEvent;
 	u8 *tmpBuf = NULL;
+
 	C2hEvent.CmdID = pbuffer[0];
 	C2hEvent.CmdSeq = pbuffer[1];
 	C2hEvent.CmdLen = length-2;
@@ -2397,6 +2401,7 @@ void SetHwReg8723B(struct adapter *padapter, u8 variable, u8 *val)
 	case HW_VAR_CHECK_BSSID:
 		{
 			u32 val32;
+
 			val32 = rtw_read32(padapter, REG_RCR);
 			if (*val)
 				val32 |= RCR_CBSSID_DATA|RCR_CBSSID_BCN;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
index 6d5e531505f9a..7fac1c2ba8e0f 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
@@ -575,6 +575,7 @@ static void phy_SetRegBW_8723B(
 )
 {
 	u16 RegRfMod_BW, u2tmp = 0;
+
 	RegRfMod_BW = rtw_read16(Adapter, REG_TRXPTCL_CTL_8723B);
 
 	switch (CurrentBW) {
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index 5faac9f28b024..cab4707091e2b 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -143,6 +143,7 @@ static void update_recvframe_phyinfo(union recv_frame *precvframe,
 		}
 	} else if (pkt_info.to_self || pkt_info.is_beacon) {
 		u32 adhoc_state = WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE;
+
 		if (check_fwstate(&padapter->mlmepriv, adhoc_state))
 			if (psta)
 				precvframe->u.hdr.psta = psta;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
index a1f2cbf2cf550..f50726d2ed0c9 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
@@ -202,7 +202,7 @@ static s32 xmit_xmitframes(struct adapter *padapter, struct xmit_priv *pxmitpriv
 
 		if (
 			(check_pending_xmitbuf(pxmitpriv)) &&
-			(padapter->mlmepriv.LinkDetectInfo.bHigherBusyTxTraffic)
+			(padapter->mlmepriv.link_detect_info.higher_busy_tx_traffic)
 		) {
 			if ((phwxmit->accnt > 0) && (phwxmit->accnt < 5)) {
 				err = -2;
@@ -240,6 +240,7 @@ static s32 xmit_xmitframes(struct adapter *padapter, struct xmit_priv *pxmitpriv
 						if (pxmitbuf->len > 0 &&
 						    pxmitbuf->priv_data) {
 							struct xmit_frame *pframe;
+
 							pframe = (struct xmit_frame *)pxmitbuf->priv_data;
 							pframe->agg_num = k;
 							pxmitbuf->agg_num = k;
@@ -324,6 +325,7 @@ static s32 xmit_xmitframes(struct adapter *padapter, struct xmit_priv *pxmitpriv
 		if (pxmitbuf) {
 			if (pxmitbuf->len > 0) {
 				struct xmit_frame *pframe;
+
 				pframe = (struct xmit_frame *)pxmitbuf->priv_data;
 				pframe->agg_num = k;
 				pxmitbuf->agg_num = k;
@@ -482,7 +484,7 @@ s32 rtl8723bs_hal_xmit(
 		(pxmitframe->attrib.ether_type != 0x888e) &&
 		(pxmitframe->attrib.dhcp_pkt != 1)
 	) {
-		if (padapter->mlmepriv.LinkDetectInfo.bBusyTraffic)
+		if (padapter->mlmepriv.link_detect_info.busy_traffic)
 			rtw_issue_addbareq_cmd(padapter, pxmitframe);
 	}
 
diff --git a/drivers/staging/rtl8723bs/hal/sdio_halinit.c b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
index 0fa1b22fdf9a1..f2f73c65a636a 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_halinit.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
@@ -30,7 +30,7 @@ static u8 CardEnable(struct adapter *padapter)
 
 		ret = HalPwrSeqCmdParsing(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, rtl8723B_card_enable_flow);
 		if (ret == _SUCCESS) {
-			u8 bMacPwrCtrlOn = true;
+			bMacPwrCtrlOn = true;
 			rtw_hal_set_hwreg(padapter, HW_VAR_APFM_ON_MAC, &bMacPwrCtrlOn);
 		}
 	} else
@@ -214,6 +214,7 @@ static void _InitNormalChipOneOutEpPriority(struct adapter *Adapter)
 	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
 
 	u16 value = 0;
+
 	switch (pHalData->OutEpQueueSel) {
 	case TX_SELE_HQ:
 		value = QUEUE_HIGH;
@@ -341,6 +342,7 @@ static void _InitTransferPageSize(struct adapter *padapter)
 	/*  Tx page size is always 128. */
 
 	u8 value8;
+
 	value8 = _PSRX(PBP_128) | _PSTX(PBP_128);
 	rtw_write8(padapter, REG_PBP, value8);
 }
@@ -388,7 +390,7 @@ static void _InitWMACSetting(struct adapter *padapter)
 
 	/*  2010.09.08 hpfan */
 	/*  Since ADF is removed from RCR, ps-poll will not be indicate to driver, */
-	/*  RxFilterMap should mask ps-poll to gurantee AP mode can rx ps-poll. */
+	/*  RxFilterMap should mask ps-poll to guarantee AP mode can rx ps-poll. */
 	value16 = 0x400;
 	rtw_write16(padapter, REG_RXFLTMAP1, value16);
 
@@ -606,7 +608,7 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 		cpwm_orig = 0;
 		rtw_hal_get_hwreg(padapter, HW_VAR_CPWM, &cpwm_orig);
 
-		/* ser rpwm */
+		/* set rpwm */
 		val8 = rtw_read8(padapter, SDIO_LOCAL_BASE | SDIO_REG_HRPWM1);
 		val8 &= 0x80;
 		val8 += 0x80;
@@ -901,7 +903,7 @@ u32 rtl8723bs_hal_deinit(struct adapter *padapter)
 				} while (cnt < 100 && (val8 != 0));
 				/* H2C done, enter 32k */
 				if (val8 == 0) {
-					/* ser rpwm to enter 32k */
+					/* set rpwm to enter 32k */
 					val8 = rtw_read8(padapter, SDIO_LOCAL_BASE | SDIO_REG_HRPWM1);
 					val8 += 0x80;
 					val8 |= BIT(0);
@@ -1147,6 +1149,7 @@ void SetHwReg8723BS(struct adapter *padapter, u8 variable, u8 *val)
 	case HW_VAR_SET_REQ_FW_PS:
 		{
 			u8 req_fw_ps = 0;
+
 			req_fw_ps = rtw_read8(padapter, 0x8f);
 			req_fw_ps |= 0x10;
 			rtw_write8(padapter, 0x8f, req_fw_ps);
diff --git a/drivers/staging/rtl8723bs/hal/sdio_ops.c b/drivers/staging/rtl8723bs/hal/sdio_ops.c
index c9cb20c61a2b2..8cece69dbf792 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_ops.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_ops.c
@@ -13,12 +13,8 @@
 /*  */
 /*  Creadted by Roger, 2011.01.31. */
 /*  */
-static void hal_sdio_get_cmd_addr_8723b(
-	struct adapter *adapter,
-	u8 device_id,
-	u32 addr,
-	u32 *cmdaddr
-)
+static void hal_sdio_get_cmd_addr_8723b(struct adapter *adapter, u8 device_id,
+					u32 addr, u32 *cmdaddr)
 {
 	switch (device_id) {
 	case SDIO_LOCAL_DEVICE_ID:
@@ -583,12 +579,10 @@ s32 sdio_local_write(
 	)
 		return sd_cmd52_write(intfhdl, addr, cnt, buf);
 
-	tmpbuf = kmalloc(cnt, GFP_ATOMIC);
+	tmpbuf = kmemdup(buf, cnt, GFP_ATOMIC);
 	if (!tmpbuf)
 		return -ENOMEM;
 
-	memcpy(tmpbuf, buf, cnt);
-
 	err = sd_write(intfhdl, addr, cnt, tmpbuf);
 
 	kfree(tmpbuf);
diff --git a/drivers/staging/rtl8723bs/include/hal_com.h b/drivers/staging/rtl8723bs/include/hal_com.h
index 74d6c892c4013..483f0390addca 100644
--- a/drivers/staging/rtl8723bs/include/hal_com.h
+++ b/drivers/staging/rtl8723bs/include/hal_com.h
@@ -141,8 +141,6 @@ void rtw_hal_check_rxfifo_full(struct adapter *adapter);
 u8 GetHalDefVar(struct adapter *adapter, enum hal_def_variable variable,
 		void *value);
 
-bool eqNByte(u8 *str1, u8 *str2, u32 num);
-
 bool GetU1ByteIntegerFromStringInDecimal(char *str, u8 *in);
 
 #define		HWSET_MAX_SIZE			512
diff --git a/drivers/staging/rtl8723bs/include/ieee80211.h b/drivers/staging/rtl8723bs/include/ieee80211.h
index 0f28c904a7147..14c806a22b38e 100644
--- a/drivers/staging/rtl8723bs/include/ieee80211.h
+++ b/drivers/staging/rtl8723bs/include/ieee80211.h
@@ -11,7 +11,6 @@
 
 #define MGMT_QUEUE_NUM 5
 
-#define ETH_ALEN	6
 #define ETH_TYPE_LEN		2
 #define PAYLOAD_TYPE_LEN	1
 
@@ -723,11 +722,15 @@ struct rtw_ieee802_11_elems {
 	u8 vht_op_mode_notify_len;
 };
 
-enum ParseRes { ParseOK = 0, ParseUnknown = 1, ParseFailed = -1 };
+enum parse_result {
+	PARSE_OK = 0,
+	PARSE_UNKNOWN = 1,
+	PARSE_FAILED = -1
+};
 
-enum ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
-				struct rtw_ieee802_11_elems *elems,
-				int show_errors);
+enum parse_result rtw_ieee802_11_parse_elems(u8 *start, uint len,
+					     struct rtw_ieee802_11_elems *elems,
+					     int show_errors);
 
 u8 *rtw_set_fixed_ie(unsigned char *pbuf, unsigned int len, unsigned char *source, unsigned int *frlen);
 u8 *rtw_set_ie(u8 *pbuf, signed int index, uint len, u8 *source, uint *frlen);
@@ -774,7 +777,7 @@ bool rtw_is_cckrates_included(u8 *rate);
 
 bool rtw_is_cckratesonly_included(u8 *rate);
 
-int rtw_check_network_type(unsigned char *rate, int ratelen, int channel);
+int rtw_check_network_type(unsigned char *rate, int channel);
 
 void rtw_get_bcn_info(struct wlan_network *pnetwork);
 
diff --git a/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h b/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
index 753009b074511..bc62b7285950f 100644
--- a/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
+++ b/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
@@ -7,58 +7,6 @@
 #ifndef __IOCTL_CFG80211_H__
 #define __IOCTL_CFG80211_H__
 
-struct rtw_wdev_invit_info {
-	u8 state; /* 0: req, 1:rep */
-	u8 peer_mac[ETH_ALEN];
-	u8 active;
-	u8 token;
-	u8 flags;
-	u8 status;
-	u8 req_op_ch;
-	u8 rsp_op_ch;
-};
-
-#define rtw_wdev_invit_info_init(invit_info) \
-	do { \
-		(invit_info)->state = 0xff; \
-		memset((invit_info)->peer_mac, 0, ETH_ALEN); \
-		(invit_info)->active = 0xff; \
-		(invit_info)->token = 0; \
-		(invit_info)->flags = 0x00; \
-		(invit_info)->status = 0xff; \
-		(invit_info)->req_op_ch = 0; \
-		(invit_info)->rsp_op_ch = 0; \
-	} while (0)
-
-struct rtw_wdev_nego_info {
-	u8 state; /* 0: req, 1:rep, 2:conf */
-	u8 peer_mac[ETH_ALEN];
-	u8 active;
-	u8 token;
-	u8 status;
-	u8 req_intent;
-	u8 req_op_ch;
-	u8 req_listen_ch;
-	u8 rsp_intent;
-	u8 rsp_op_ch;
-	u8 conf_op_ch;
-};
-
-#define rtw_wdev_nego_info_init(nego_info) \
-	do { \
-		(nego_info)->state = 0xff; \
-		memset((nego_info)->peer_mac, 0, ETH_ALEN); \
-		(nego_info)->active = 0xff; \
-		(nego_info)->token = 0; \
-		(nego_info)->status = 0xff; \
-		(nego_info)->req_intent = 0xff; \
-		(nego_info)->req_op_ch = 0; \
-		(nego_info)->req_listen_ch = 0; \
-		(nego_info)->rsp_intent = 0xff; \
-		(nego_info)->rsp_op_ch = 0; \
-		(nego_info)->conf_op_ch = 0; \
-	} while (0)
-
 struct rtw_wdev_priv {
 	struct wireless_dev *rtw_wdev;
 
@@ -70,14 +18,6 @@ struct rtw_wdev_priv {
 	struct net_device *pmon_ndev;/* for monitor interface */
 	char ifname_mon[IFNAMSIZ + 1]; /* interface name for monitor interface */
 
-	u8 p2p_enabled;
-
-	u8 provdisc_req_issued;
-
-	struct rtw_wdev_invit_info invit_info;
-	struct rtw_wdev_nego_info nego_info;
-
-	u8 bandroid_scan;
 	bool block;
 	bool power_mgmt;
 };
diff --git a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
index 9664758e21bef..e2e9aa03ffdf2 100644
--- a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
@@ -7,10 +7,6 @@
 #ifndef _RTL8192C_RECV_H_
 #define _RTL8192C_RECV_H_
 
-#define RECV_BLK_SZ 512
-#define RECV_BLK_CNT 16
-#define RECV_BLK_TH RECV_BLK_CNT
-
 #define MAX_RECVBUF_SZ (10240)
 
 struct phy_stat {
diff --git a/drivers/staging/rtl8723bs/include/rtw_cmd.h b/drivers/staging/rtl8723bs/include/rtw_cmd.h
index cb44119ce9a91..c4c3edee809d4 100644
--- a/drivers/staging/rtl8723bs/include/rtw_cmd.h
+++ b/drivers/staging/rtl8723bs/include/rtw_cmd.h
@@ -559,7 +559,7 @@ struct RunInThread_param {
 Result:
 0x00: success
 0x01: success, and check Response.
-0x02: cmd ignored due to duplicated sequcne number
+0x02: cmd ignored due to duplicated sequence number
 0x03: cmd dropped due to invalid cmd code
 0x04: reserved.
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_efuse.h b/drivers/staging/rtl8723bs/include/rtw_efuse.h
index 936b204b88306..191ffdf593d40 100644
--- a/drivers/staging/rtl8723bs/include/rtw_efuse.h
+++ b/drivers/staging/rtl8723bs/include/rtw_efuse.h
@@ -68,25 +68,6 @@ struct efuse_hal {
 	u8 fakeBTEfuseModifiedMap[EFUSE_BT_MAX_MAP_LEN];
 };
 
-
-/*------------------------Export global variable----------------------------*/
-extern u8 fakeEfuseBank;
-extern u32 fakeEfuseUsedBytes;
-extern u8 fakeEfuseContent[];
-extern u8 fakeEfuseInitMap[];
-extern u8 fakeEfuseModifiedMap[];
-
-extern u32 BTEfuseUsedBytes;
-extern u8 BTEfuseContent[EFUSE_MAX_BT_BANK][EFUSE_MAX_HW_SIZE];
-extern u8 BTEfuseInitMap[];
-extern u8 BTEfuseModifiedMap[];
-
-extern u32 fakeBTEfuseUsedBytes;
-extern u8 fakeBTEfuseContent[EFUSE_MAX_BT_BANK][EFUSE_MAX_HW_SIZE];
-extern u8 fakeBTEfuseInitMap[];
-extern u8 fakeBTEfuseModifiedMap[];
-/*------------------------Export global variable----------------------------*/
-
 u8 Efuse_CalculateWordCnts(u8 word_en);
 u8 efuse_OneByteRead(struct adapter *padapter, u16 addr, u8 *data);
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme.h b/drivers/staging/rtl8723bs/include/rtw_mlme.h
index 2a128568c6df5..ac3ba746b64c9 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme.h
@@ -76,7 +76,7 @@ Each struct __queue has its own locks, already.
 Other items in mlme_priv are protected by mlme_priv.lock, while items in
 xmit_priv are protected by xmit_priv.lock.
 
-To avoid possible dead lock, any thread trying to modifiying mlme_priv
+To avoid possible dead lock, any thread trying to modifying mlme_priv
 SHALL not lock up more than one locks at a time!
 
 The only exception is that queue functions which take the __queue.lock
@@ -93,18 +93,20 @@ struct sitesurvey_ctrl {
 };
 
 struct rt_link_detect_t {
-	u32 			NumTxOkInPeriod;
-	u32 			NumRxOkInPeriod;
-	u32 			NumRxUnicastOkInPeriod;
-	bool			bBusyTraffic;
-	bool			bTxBusyTraffic;
-	bool			bRxBusyTraffic;
-	bool			bHigherBusyTraffic; /*  For interrupt migration purpose. */
-	bool			bHigherBusyRxTraffic; /*  We may disable Tx interrupt according as Rx traffic. */
-	bool			bHigherBusyTxTraffic; /*  We may disable Tx interrupt according as Tx traffic. */
-	/* u8 TrafficBusyState; */
-	u8 TrafficTransitionCount;
-	u32 LowPowerTransitionCount;
+	u32			num_tx_ok_in_period;
+	u32			num_rx_ok_in_period;
+	u32			num_rx_unicast_ok_in_period;
+	bool			busy_traffic;
+	bool			tx_busy_traffic;
+	bool			rx_busy_traffic;
+	/* For interrupt migration purpose. */
+	bool			higher_busy_traffic;
+	/* We may disable Tx interrupt according as Rx traffic. */
+	bool			higher_busy_rx_traffic;
+	/* We may disable Tx interrupt according as Tx traffic. */
+	bool			higher_busy_tx_traffic;
+	u8			traffic_transition_count;
+	u32			low_power_transition_count;
 };
 
 /* used for mlme_priv.roam_flags */
@@ -171,7 +173,7 @@ struct mlme_priv {
 
 	struct ht_priv htpriv;
 
-	struct rt_link_detect_t	LinkDetectInfo;
+	struct rt_link_detect_t	link_detect_info;
 	struct timer_list	dynamic_chk_timer; /* dynamic/periodic check timer */
 
 	u8 acm_mask; /*  for wmm acm mask */
@@ -360,9 +362,9 @@ extern void _rtw_free_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_ne
 
 extern struct wlan_network *_rtw_find_network(struct __queue *scanned_queue, u8 *addr);
 
-extern signed int rtw_if_up(struct adapter *padapter);
+bool rtw_if_up(struct adapter *padapter);
 
-signed int rtw_linked_check(struct adapter *padapter);
+bool rtw_linked_check(struct adapter *padapter);
 
 u8 *rtw_get_capability_from_ie(u8 *ie);
 u8 *rtw_get_beacon_interval_from_ie(u8 *ie);
@@ -377,7 +379,7 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len, u8 channe
 void rtw_issue_addbareq_cmd(struct adapter *padapter, struct xmit_frame *pxmitframe);
 void rtw_append_exented_cap(struct adapter *padapter, u8 *out_ie, uint *pout_len);
 
-int rtw_is_same_ibss(struct adapter *adapter, struct wlan_network *pnetwork);
+bool rtw_is_same_ibss(struct adapter *adapter, struct wlan_network *pnetwork);
 int is_same_network(struct wlan_bssid_ex *src, struct wlan_bssid_ex *dst, u8 feature);
 
 #define rtw_roam_flags(adapter) ((adapter)->mlmepriv.roam_flags)
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
index afa5631a441a0..95769f90d1961 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
@@ -424,7 +424,7 @@ struct mlme_ext_priv {
 
 void init_mlme_default_rate_set(struct adapter *padapter);
 void init_mlme_ext_priv(struct adapter *padapter);
-int init_hw_mlme_ext(struct adapter *padapter);
+void init_hw_mlme_ext(struct adapter *padapter);
 void free_mlme_ext_priv(struct mlme_ext_priv *pmlmeext);
 extern struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv);
 
@@ -615,7 +615,7 @@ extern void process_addba_req(struct adapter *padapter, u8 *paddba_req, u8 *addr
 extern void update_TSF(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
 extern void correct_TSF(struct adapter *padapter, struct mlme_ext_priv *pmlmeext);
 extern void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
-extern u8 traffic_status_watchdog(struct adapter *padapter, u8 from_timer);
+extern bool traffic_status_watchdog(struct adapter *padapter, bool from_timer);
 
 int rtw_chk_start_clnt_join(struct adapter *padapter, u8 *ch, u8 *bw, u8 *offset);
 
diff --git a/drivers/staging/rtl8723bs/include/wlan_bssdef.h b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
index eb38594c8f5c1..812a683942689 100644
--- a/drivers/staging/rtl8723bs/include/wlan_bssdef.h
+++ b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
@@ -15,7 +15,6 @@
 #define NDIS_802_11_LENGTH_RATES        8
 #define NDIS_802_11_LENGTH_RATES_EX     16
 
-typedef unsigned char   NDIS_802_11_MAC_ADDRESS[6];
 typedef unsigned char   NDIS_802_11_RATES[NDIS_802_11_LENGTH_RATES];        /*  Set of 8 data rates */
 typedef unsigned char   NDIS_802_11_RATES_EX[NDIS_802_11_LENGTH_RATES_EX];  /*  Set of 16 data rates */
 
@@ -64,7 +63,7 @@ struct ndis_80211_var_ie {
 };
 
 /* Length is the 4 bytes multiples of the sum of
- * sizeof (NDIS_802_11_MAC_ADDRESS) + 2 +
+ * ETH_ALEN + 2 +
  * sizeof (struct ndis_802_11_ssid) + sizeof (u32) +
  * sizeof (long) + sizeof (enum ndis_802_11_network_type) +
  * sizeof (struct ndis_802_11_conf) + sizeof (NDIS_802_11_RATES_EX) + ie_length
@@ -155,7 +154,7 @@ struct wlan_bcn_info {
 */
 struct wlan_bssid_ex {
 	u32  length;
-	NDIS_802_11_MAC_ADDRESS  mac_address;
+	u8 mac_address[ETH_ALEN];
 	u8  reserved[2];/* 0]: IS beacon frame */
 	struct ndis_802_11_ssid  ssid;
 	u32  privacy;
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index 7cb0c6f22bf3b..fd3bae31b0edb 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -93,7 +93,7 @@ static struct ieee80211_channel rtw_2ghz_channels[] = {
 
 static void rtw_2g_channels_init(struct ieee80211_channel *channels)
 {
-	memcpy((void *)channels, (void *)rtw_2ghz_channels,
+	memcpy(channels, rtw_2ghz_channels,
 	       sizeof(struct ieee80211_channel) * RTW_2G_CHANNELS_NUM
 	);
 }
@@ -380,9 +380,9 @@ void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 				rtw_warn_on(1);
 				return;
 			}
-			if (!memcmp(&(scanned->network.ssid), &(pnetwork->ssid), sizeof(struct ndis_802_11_ssid))
-				&& !memcmp(scanned->network.mac_address, pnetwork->mac_address, sizeof(NDIS_802_11_MAC_ADDRESS))
-			)
+			if (!memcmp(&scanned->network.ssid, &pnetwork->ssid,
+				    sizeof(struct ndis_802_11_ssid)) &&
+			    !memcmp(scanned->network.mac_address, pnetwork->mac_address, ETH_ALEN))
 				rtw_cfg80211_inform_bss(padapter, scanned);
 			else
 				rtw_warn_on(1);
@@ -422,9 +422,9 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 			goto check_bss;
 		}
 
-		if (!memcmp(scanned->network.mac_address, pnetwork->mac_address, sizeof(NDIS_802_11_MAC_ADDRESS))
-			&& !memcmp(&(scanned->network.ssid), &(pnetwork->ssid), sizeof(struct ndis_802_11_ssid))
-		)
+		if (!memcmp(scanned->network.mac_address, pnetwork->mac_address, ETH_ALEN) &&
+		    !memcmp(&scanned->network.ssid, &pnetwork->ssid,
+			    sizeof(struct ndis_802_11_ssid)))
 			rtw_cfg80211_inform_bss(padapter, scanned);
 		else
 			rtw_warn_on(1);
@@ -831,7 +831,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 	return ret;
 }
 
-static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
+static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				int link_id, u8 key_index, bool pairwise,
 				const u8 *mac_addr, struct key_params *params)
 {
@@ -839,6 +839,7 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
 	u32 param_len;
 	struct ieee_param *param = NULL;
 	int ret = 0;
+	struct net_device *ndev = wdev->netdev;
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
@@ -895,7 +896,7 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
 		ret =  rtw_cfg80211_set_encryption(ndev, param, param_len);
 	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
 		if (mac_addr)
-			memcpy(param->sta_addr, (void *)mac_addr, ETH_ALEN);
+			memcpy(param->sta_addr, mac_addr, ETH_ALEN);
 
 		ret = rtw_cfg80211_ap_set_encryption(ndev, param, param_len);
 	} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true
@@ -909,7 +910,7 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
 	return ret;
 }
 
-static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev,
+static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				int link_id, u8 key_index, bool pairwise,
 				const u8 *mac_addr, void *cookie,
 				void (*callback)(void *cookie,
@@ -918,11 +919,11 @@ static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev,
 	return 0;
 }
 
-static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev,
+static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 				int link_id, u8 key_index, bool pairwise,
 				const u8 *mac_addr)
 {
-	struct adapter *padapter = rtw_netdev_priv(ndev);
+	struct adapter *padapter = rtw_netdev_priv(wdev->netdev);
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 
 	if (key_index == psecuritypriv->dot11PrivacyKeyIndex) {
@@ -960,11 +961,12 @@ static int cfg80211_rtw_set_default_key(struct wiphy *wiphy,
 }
 
 static int cfg80211_rtw_get_station(struct wiphy *wiphy,
-				    struct net_device *ndev,
+				    struct wireless_dev *wdev,
 				const u8 *mac,
 				struct station_info *sinfo)
 {
 	int ret = 0;
+	struct net_device *ndev = wdev_to_ndev(wdev);
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_info *psta = NULL;
@@ -1231,7 +1233,7 @@ static int cfg80211_rtw_scan(struct wiphy *wiphy
 		goto check_need_indicate_scan_done;
 	}
 
-	if (pmlmepriv->LinkDetectInfo.bBusyTraffic == true) {
+	if (pmlmepriv->link_detect_info.busy_traffic) {
 		static unsigned long lastscantime;
 		unsigned long passtime;
 
@@ -1430,14 +1432,12 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel
 		goto exit;
 	}
 
-	buf = kzalloc(ielen, GFP_KERNEL);
+	buf = kmemdup(pie, ielen, GFP_KERNEL);
 	if (!buf) {
 		ret =  -ENOMEM;
 		goto exit;
 	}
 
-	memcpy(buf, pie, ielen);
-
 	if (ielen < RSN_HEADER_LEN) {
 		ret  = -1;
 		goto exit;
@@ -1735,7 +1735,7 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 		pwep->key_index = wep_key_idx;
 		pwep->key_index |= 0x80000000;
 
-		memcpy(pwep->key_material,  (void *)sme->key, pwep->key_length);
+		memcpy(pwep->key_material, sme->key, pwep->key_length);
 
 		if (rtw_set_802_11_add_wep(padapter, pwep) == (u8)_FAIL)
 			ret = -EOPNOTSUPP;
@@ -1912,7 +1912,7 @@ static int cfg80211_rtw_flush_pmksa(struct wiphy *wiphy,
 
 void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, uint frame_len)
 {
-	struct net_device *ndev = padapter->pnetdev;
+	struct wireless_dev *wdev = padapter->rtw_wdev;
 
 	{
 		struct station_info sinfo = {};
@@ -1926,15 +1926,15 @@ void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame,
 		sinfo.filled = 0;
 		sinfo.assoc_req_ies = pmgmt_frame + WLAN_HDR_A3_LEN + ie_offset;
 		sinfo.assoc_req_ies_len = frame_len - WLAN_HDR_A3_LEN - ie_offset;
-		cfg80211_new_sta(ndev, GetAddr2Ptr(pmgmt_frame), &sinfo, GFP_ATOMIC);
+		cfg80211_new_sta(wdev, GetAddr2Ptr(pmgmt_frame), &sinfo, GFP_ATOMIC);
 	}
 }
 
 void rtw_cfg80211_indicate_sta_disassoc(struct adapter *padapter, unsigned char *da, unsigned short reason)
 {
-	struct net_device *ndev = padapter->pnetdev;
+	struct wireless_dev *wdev = padapter->rtw_wdev;
 
-	cfg80211_del_sta(ndev, da, GFP_ATOMIC);
+	cfg80211_del_sta(wdev, da, GFP_ATOMIC);
 }
 
 static u8 rtw_get_chan_type(struct adapter *adapter)
@@ -2084,7 +2084,7 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc
 
 		pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
 
-		memcpy(pframe, (void *)buf, len);
+		memcpy(pframe, buf, len);
 		pattrib->pktlen = len;
 
 		pwlanhdr = (struct ieee80211_hdr *)pframe;
@@ -2259,8 +2259,8 @@ static int rtw_add_beacon(struct adapter *adapter, const u8 *head, size_t head_l
 	if (!pbuf)
 		return -ENOMEM;
 
-	memcpy(pbuf, (void *)head + 24, head_len - 24);/*  24 =beacon header len. */
-	memcpy(pbuf + head_len - 24, (void *)tail, tail_len);
+	memcpy(pbuf, head + 24, head_len - 24);/*  24 =beacon header len. */
+	memcpy(pbuf + head_len - 24, tail, tail_len);
 
 	len = head_len + tail_len - 24;
 
@@ -2297,9 +2297,9 @@ static int cfg80211_rtw_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 		struct wlan_bssid_ex *pbss_network = &adapter->mlmepriv.cur_network.network;
 		struct wlan_bssid_ex *pbss_network_ext = &adapter->mlmeextpriv.mlmext_info.network;
 
-		memcpy(pbss_network->ssid.ssid, (void *)settings->ssid, settings->ssid_len);
+		memcpy(pbss_network->ssid.ssid, settings->ssid, settings->ssid_len);
 		pbss_network->ssid.ssid_length = settings->ssid_len;
-		memcpy(pbss_network_ext->ssid.ssid, (void *)settings->ssid, settings->ssid_len);
+		memcpy(pbss_network_ext->ssid.ssid, settings->ssid, settings->ssid_len);
 		pbss_network_ext->ssid.ssid_length = settings->ssid_len;
 	}
 
@@ -2323,21 +2323,22 @@ static int cfg80211_rtw_stop_ap(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int	cfg80211_rtw_add_station(struct wiphy *wiphy,
-					 struct net_device *ndev,
+					 struct wireless_dev *wdev,
 					 const u8 *mac,
 					 struct station_parameters *params)
 {
 	return 0;
 }
 
-static int cfg80211_rtw_del_station(struct wiphy *wiphy, struct net_device *ndev,
+static int cfg80211_rtw_del_station(struct wiphy *wiphy,
+				    struct wireless_dev *wdev,
 				    struct station_del_parameters *params)
 {
 	int ret = 0;
 	struct list_head *phead, *plist, *tmp;
 	u8 updated = false;
 	struct sta_info *psta = NULL;
-	struct adapter *padapter = rtw_netdev_priv(ndev);
+	struct adapter *padapter = rtw_netdev_priv(wdev->netdev);
 	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	const u8 *mac = params->mac;
@@ -2388,7 +2389,7 @@ static int cfg80211_rtw_del_station(struct wiphy *wiphy, struct net_device *ndev
 }
 
 static int cfg80211_rtw_change_station(struct wiphy *wiphy,
-				       struct net_device *ndev,
+				       struct wireless_dev *wdev,
 				       const u8 *mac,
 				       struct station_parameters *params)
 {
@@ -2416,12 +2417,12 @@ static struct sta_info *rtw_sta_info_get_by_idx(const int idx, struct sta_priv *
 }
 
 static int	cfg80211_rtw_dump_station(struct wiphy *wiphy,
-					  struct net_device *ndev,
+					 struct wireless_dev *wdev,
 					  int idx, u8 *mac,
 					  struct station_info *sinfo)
 {
 	int ret = 0;
-	struct adapter *padapter = rtw_netdev_priv(ndev);
+	struct adapter *padapter = rtw_netdev_priv(wdev_to_ndev(wdev));
 	struct sta_info *psta = NULL;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 
@@ -2492,7 +2493,7 @@ static int _cfg80211_rtw_mgmt_tx(struct adapter *padapter, u8 tx_ch, const u8 *b
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
 
-	memcpy(pframe, (void *)buf, len);
+	memcpy(pframe, buf, len);
 	pattrib->pktlen = len;
 
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
@@ -2749,13 +2750,6 @@ int rtw_wdev_alloc(struct adapter *padapter, struct device *dev)
 	pwdev_priv->scan_request = NULL;
 	spin_lock_init(&pwdev_priv->scan_req_lock);
 
-	pwdev_priv->p2p_enabled = false;
-	pwdev_priv->provdisc_req_issued = false;
-	rtw_wdev_invit_info_init(&pwdev_priv->invit_info);
-	rtw_wdev_nego_info_init(&pwdev_priv->nego_info);
-
-	pwdev_priv->bandroid_scan = false;
-
 	if (padapter->registrypriv.power_mgnt != PS_MODE_ACTIVE)
 		pwdev_priv->power_mgmt = true;
 	else
diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
index 29939bf5a1569..7ba689f2dfc88 100644
--- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
@@ -492,7 +492,7 @@ void rtw_stop_drv_threads(struct adapter *padapter)
 {
 	rtw_stop_cmd_thread(padapter);
 
-	/*  Below is to termindate tx_thread... */
+	/*  Below is to terminate tx_thread... */
 	complete(&padapter->xmitpriv.xmit_comp);
 	wait_for_completion(&padapter->xmitpriv.terminate_xmitthread_comp);
 
@@ -618,11 +618,10 @@ void rtw_reset_drv_sw(struct adapter *padapter)
 	padapter->xmitpriv.tx_pkts = 0;
 	padapter->recvpriv.rx_pkts = 0;
 
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
+	pmlmepriv->link_detect_info.busy_traffic = false;
 
-	/* pmlmepriv->LinkDetectInfo.TrafficBusyState = false; */
-	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
-	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+	pmlmepriv->link_detect_info.traffic_transition_count = 0;
+	pmlmepriv->link_detect_info.low_power_transition_count = 0;
 
 	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING);
 
@@ -960,7 +959,6 @@ static int netdev_close(struct net_device *pnetdev)
 	}
 
 	rtw_scan_abort(padapter);
-	adapter_wdev_data(padapter)->bandroid_scan = false;
 
 	return 0;
 }
diff --git a/drivers/staging/sm750fb/ddk750_mode.c b/drivers/staging/sm750fb/ddk750_mode.c
index 3b25892af7133..7163232c0701a 100644
--- a/drivers/staging/sm750fb/ddk750_mode.c
+++ b/drivers/staging/sm750fb/ddk750_mode.c
@@ -74,7 +74,7 @@ display_control_adjust_SM750LE(struct mode_parameter *mode_param,
 	return disp_control;
 }
 
-/* only timing related registers will be  programed */
+/* only timing related registers will be programmed */
 static void program_mode_registers(struct mode_parameter *mode_param,
 				   struct pll_value *pll)
 {
diff --git a/drivers/staging/sm750fb/ddk750_swi2c.c b/drivers/staging/sm750fb/ddk750_swi2c.c
index 0ef8d4ff2ef95..e63f3b00ec4c5 100644
--- a/drivers/staging/sm750fb/ddk750_swi2c.c
+++ b/drivers/staging/sm750fb/ddk750_swi2c.c
@@ -27,7 +27,7 @@
  *
  * I.e. the SCL may only be changed in section 1. and section 3. while
  * the SDA may only be changed in section 2. and section 4. The table
- * below gives the changes for these 2 lines in the varios sections.
+ * below gives the changes for these 2 lines in the various sections.
  *
  * Section changes Table:
  * ======================
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 62f6e0cdff4d4..9a42a08c8ed59 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -375,7 +375,6 @@ static int lynxfb_ops_set_par(struct fb_info *info)
 	line_length = var->xres_virtual * var->bits_per_pixel / 8;
 	line_length = ALIGN(line_length, crtc->line_pad);
 	fix->line_length = line_length;
-	pr_info("fix->line_length = %d\n", fix->line_length);
 
 	/*
 	 * var->red,green,blue,transp are need to be set by driver
@@ -389,7 +388,8 @@ static int lynxfb_ops_set_par(struct fb_info *info)
 	var->accel_flags = 0;/*FB_ACCELF_TEXT;*/
 
 	if (ret) {
-		pr_err("bpp %d not supported\n", var->bits_per_pixel);
+		dev_err(info->device, "bpp %d not supported\n",
+			var->bits_per_pixel);
 		return ret;
 	}
 	ret = hw_sm750_crtc_set_mode(crtc, var, fix);
@@ -485,15 +485,11 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 	par = info->par;
 	crtc = &par->crtc;
 
-	pr_debug("check var:%dx%d-%d\n",
-		 var->xres,
-		 var->yres,
-		 var->bits_per_pixel);
-
 	ret = lynxfb_set_color_offsets(info);
 
 	if (ret) {
-		pr_err("bpp %d not supported\n", var->bits_per_pixel);
+		dev_err(info->device, "bpp %d not supported\n",
+			var->bits_per_pixel);
 		return ret;
 	}
 
@@ -508,7 +504,7 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 	request = ALIGN(request, crtc->line_pad);
 	request = request * var->yres_virtual;
 	if (crtc->vidmem_size < request) {
-		pr_err("not enough video memory for mode\n");
+		dev_err(info->device, "not enough video memory for mode\n");
 		return -ENOMEM;
 	}
 
@@ -533,7 +529,7 @@ static int lynxfb_ops_setcolreg(unsigned int regno,
 	ret = 0;
 
 	if (regno > 256) {
-		pr_err("regno = %d\n", regno);
+		dev_err(info->device, "regno = %d\n", regno);
 		return -EINVAL;
 	}
 
@@ -580,7 +576,6 @@ static int lynxfb_ops_blank(int blank, struct fb_info *info)
 	struct lynxfb_par *par;
 	struct lynxfb_output *output;
 
-	pr_debug("blank = %d.\n", blank);
 	par = info->par;
 	output = &par->output;
 	sm750_dev = par->dev;
@@ -625,7 +620,6 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 		crtc->channel = sm750_primary;
 		crtc->o_screen = 0;
 		crtc->v_screen = sm750_dev->pvMem;
-		pr_info("use simul primary mode\n");
 		break;
 	case sm750_simul_sec:
 		output->paths = sm750_pnc;
@@ -734,12 +728,6 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 		lynx750_ext, NULL, vesa_modes,
 	};
 	int cdb[] = {ARRAY_SIZE(lynx750_ext), 0, VESA_MODEDB_SIZE};
-	static const char * const mdb_desc[] = {
-		"driver prepared modes",
-		"kernel prepared default modedb",
-		"kernel HELPERS prepared vesa_modes",
-	};
-
 	static const char *fix_id[2] = {
 		"sm750_fb1", "sm750_fb2",
 	};
@@ -767,7 +755,6 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	crtc->cursor.mmio = sm750_dev->pvReg +
 		0x800f0 + (int)crtc->channel * 0x140;
 
-	pr_info("crtc->cursor.mmio = %p\n", crtc->cursor.mmio);
 	crtc->cursor.max_h = 64;
 	crtc->cursor.max_w = 64;
 	crtc->cursor.size = crtc->cursor.max_h * crtc->cursor.max_w * 2 / 8;
@@ -801,47 +788,10 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 		ret = fb_find_mode(var, info, g_fbmode[index],
 				   pdb[i], cdb[i], NULL, 8);
 
-		if (ret == 1) {
-			pr_info("success! use specified mode:%s in %s\n",
-				g_fbmode[index],
-				mdb_desc[i]);
+		if (ret == 1 || ret == 2)
 			break;
-		} else if (ret == 2) {
-			pr_warn("use specified mode:%s in %s,with an ignored refresh rate\n",
-				g_fbmode[index],
-				mdb_desc[i]);
-			break;
-		} else if (ret == 3) {
-			pr_warn("wanna use default mode\n");
-			/*break;*/
-		} else if (ret == 4) {
-			pr_warn("fall back to any valid mode\n");
-		} else {
-			pr_warn("ret = %d,fb_find_mode failed,with %s\n",
-				ret,
-				mdb_desc[i]);
-		}
 	}
 
-	/* some member of info->var had been set by fb_find_mode */
-
-	pr_info("Member of info->var is :\n"
-		"xres=%d\n"
-		"yres=%d\n"
-		"xres_virtual=%d\n"
-		"yres_virtual=%d\n"
-		"xoffset=%d\n"
-		"yoffset=%d\n"
-		"bits_per_pixel=%d\n"
-		" ...\n",
-		var->xres,
-		var->yres,
-		var->xres_virtual,
-		var->yres_virtual,
-		var->xoffset,
-		var->yoffset,
-		var->bits_per_pixel);
-
 	/* set par */
 	par->info = info;
 
@@ -851,7 +801,6 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 
 	info->pseudo_palette = &par->pseudo_palette[0];
 	info->screen_base = crtc->v_screen;
-	pr_debug("screen_base vaddr = %p\n", info->screen_base);
 	info->screen_size = line_length * var->yres_virtual;
 
 	/* set info->fix */
@@ -865,7 +814,6 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	strscpy(fix->id, fix_id[index], sizeof(fix->id));
 
 	fix->smem_start = crtc->o_screen + sm750_dev->vidmem_start;
-	pr_info("fix->smem_start = %lx\n", fix->smem_start);
 	/*
 	 * according to mmap experiment from user space application,
 	 * fix->mmio_len should not larger than virtual size
@@ -874,13 +822,10 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	 * data into the bound over virtual size
 	 */
 	fix->smem_len = crtc->vidmem_size;
-	pr_info("fix->smem_len = %x\n", fix->smem_len);
 	info->screen_size = fix->smem_len;
 	fix->line_length = line_length;
 	fix->mmio_start = sm750_dev->vidreg_start;
-	pr_info("fix->mmio_start = %lx\n", fix->mmio_start);
 	fix->mmio_len = sm750_dev->vidreg_size;
-	pr_info("fix->mmio_len = %x\n", fix->mmio_len);
 
 	lynxfb_set_visual_mode(info);
 
@@ -889,22 +834,12 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	var->accel_flags = 0;
 	var->vmode = FB_VMODE_NONINTERLACED;
 
-	pr_debug("#1 show info->cmap :\nstart=%d,len=%d,red=%p,green=%p,blue=%p,transp=%p\n",
-		 info->cmap.start, info->cmap.len,
-		 info->cmap.red, info->cmap.green, info->cmap.blue,
-		 info->cmap.transp);
-
 	ret = fb_alloc_cmap(&info->cmap, 256, 0);
 	if (ret < 0) {
-		pr_err("Could not allocate memory for cmap.\n");
+		dev_err(info->device, "Could not allocate memory for cmap.\n");
 		goto exit;
 	}
 
-	pr_debug("#2 show info->cmap :\nstart=%d,len=%d,red=%p,green=%p,blue=%p,transp=%p\n",
-		 info->cmap.start, info->cmap.len,
-		 info->cmap.red, info->cmap.green, info->cmap.blue,
-		 info->cmap.transp);
-
 exit:
 	lynxfb_ops_check_var(var, info);
 	return ret;
@@ -1132,12 +1067,8 @@ static int __init lynxfb_setup(char *options)
 	int len;
 	char *opt, *tmp;
 
-	if (!options || !*options) {
-		pr_warn("no options.\n");
+	if (!options || !*options)
 		return 0;
-	}
-
-	pr_info("options:%s\n", options);
 
 	len = strlen(options) + 1;
 	g_settings = kzalloc(len, GFP_KERNEL);
@@ -1174,7 +1105,6 @@ static int __init lynxfb_setup(char *options)
 	}
 
 	/* misc g_settings are transport to chip specific routines */
-	pr_info("parameter left for chip specific analysis:%s\n", g_settings);
 	return 0;
 }
 
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index f60b152a647d3..a2798d428663e 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -34,12 +34,10 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	sm750_dev->vidreg_start = pci_resource_start(pdev, 1);
 	sm750_dev->vidreg_size = SZ_2M;
 
-	pr_info("mmio phyAddr = %lx\n", sm750_dev->vidreg_start);
-
 	/* reserve the vidreg space of smi adaptor */
 	ret = pci_request_region(pdev, 1, "sm750fb");
 	if (ret) {
-		pr_err("Can not request PCI regions.\n");
+		dev_err(&pdev->dev, "Can not request PCI regions.\n");
 		return ret;
 	}
 
@@ -47,11 +45,10 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	sm750_dev->pvReg =
 		ioremap(sm750_dev->vidreg_start, sm750_dev->vidreg_size);
 	if (!sm750_dev->pvReg) {
-		pr_err("mmio failed\n");
+		dev_err(&pdev->dev, "mmio failed\n");
 		ret = -EFAULT;
 		goto err_release_region;
 	}
-	pr_info("mmio virtual addr = %p\n", sm750_dev->pvReg);
 
 	sm750_dev->accel.dpr_base = sm750_dev->pvReg + DE_BASE_ADDR_TYPE1;
 	sm750_dev->accel.dp_port_base = sm750_dev->pvReg + DE_PORT_ADDR_TYPE1;
@@ -67,18 +64,15 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	 * @ddk750_get_vm_size function can be safe.
 	 */
 	sm750_dev->vidmem_size = ddk750_get_vm_size();
-	pr_info("video memory phyAddr = %lx, size = %u bytes\n",
-		sm750_dev->vidmem_start, sm750_dev->vidmem_size);
 
 	/* reserve the vidmem space of smi adaptor */
 	sm750_dev->pvMem =
 		ioremap_wc(sm750_dev->vidmem_start, sm750_dev->vidmem_size);
 	if (!sm750_dev->pvMem) {
-		pr_err("Map video memory failed\n");
+		dev_err(&pdev->dev, "Map video memory failed\n");
 		ret = -EFAULT;
 		goto err_unmap_reg;
 	}
-	pr_info("video memory vaddr = %p\n", sm750_dev->pvMem);
 
 	return 0;
 
@@ -163,11 +157,9 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 			 * The following register values for CH7301 are from
 			 * Chrontel app note and our experiment.
 			 */
-			pr_info("yes,CH7301 DVI chip found\n");
 			sm750_sw_i2c_write_reg(0xec, 0x1d, 0x16);
 			sm750_sw_i2c_write_reg(0xec, 0x21, 0x9);
 			sm750_sw_i2c_write_reg(0xec, 0x49, 0xC0);
-			pr_info("okay,CH7301 DVI chip setup done\n");
 		}
 	}
 
@@ -192,14 +184,12 @@ int hw_sm750_output_set_mode(struct lynxfb_output *output,
 
 	if (sm750_get_chip_type() != SM750LE) {
 		if (channel == sm750_primary) {
-			pr_info("primary channel\n");
 			if (output->paths & sm750_panel)
 				disp_set |= do_LCD1_PRI;
 			if (output->paths & sm750_crt)
 				disp_set |= do_CRT_PRI;
 
 		} else {
-			pr_info("secondary channel\n");
 			if (output->paths & sm750_panel)
 				disp_set |= do_LCD1_SEC;
 			if (output->paths & sm750_crt)
@@ -215,7 +205,6 @@ int hw_sm750_output_set_mode(struct lynxfb_output *output,
 		poke32(DISPLAY_CONTROL_750LE, reg);
 	}
 
-	pr_info("ddk setlogicdispout done\n");
 	return ret;
 }
 
@@ -232,10 +221,8 @@ int hw_sm750_crtc_check_mode(struct lynxfb_crtc *crtc,
 	case 16:
 		break;
 	case 32:
-		if (sm750_dev->revid == SM750LE_REVISION_ID) {
-			pr_debug("750le do not support 32bpp\n");
+		if (sm750_dev->revid == SM750LE_REVISION_ID)
 			return -EINVAL;
-		}
 		break;
 	default:
 		return -EINVAL;
@@ -302,10 +289,9 @@ int hw_sm750_crtc_set_mode(struct lynxfb_crtc *crtc,
 	else
 		clock = SECONDARY_PLL;
 
-	pr_debug("Request pixel clock = %lu\n", modparm.pixel_clock);
 	ret = ddk750_set_mode_timing(&modparm, clock);
 	if (ret) {
-		pr_err("Set mode timing failed\n");
+		dev_err(&sm750_dev->pdev->dev, "Set mode timing failed\n");
 		goto exit;
 	}
 
@@ -431,12 +417,10 @@ int hw_sm750_set_blank(struct lynxfb_output *output, int blank)
 
 	switch (blank) {
 	case FB_BLANK_UNBLANK:
-		pr_debug("flag = FB_BLANK_UNBLANK\n");
 		dpms = SYSTEM_CTRL_DPMS_VPHP;
 		pps = PANEL_DISPLAY_CTRL_DATA;
 		break;
 	case FB_BLANK_NORMAL:
-		pr_debug("flag = FB_BLANK_NORMAL\n");
 		dpms = SYSTEM_CTRL_DPMS_VPHP;
 		crtdb = CRT_DISPLAY_CTRL_BLANK;
 		break;
diff --git a/drivers/staging/vme_user/vme.c b/drivers/staging/vme_user/vme.c
index f10a00c05f128..7220aba7b919b 100644
--- a/drivers/staging/vme_user/vme.c
+++ b/drivers/staging/vme_user/vme.c
@@ -735,9 +735,9 @@ unsigned int vme_master_rmw(struct vme_resource *resource, unsigned int mask,
 EXPORT_SYMBOL(vme_master_rmw);
 
 /**
- * vme_master_mmap - Mmap region of VME master window.
+ * vme_master_mmap_prepare - Mmap region of VME master window.
  * @resource: Pointer to VME master resource.
- * @vma: Pointer to definition of user mapping.
+ * @desc: Pointer to descriptor of user mapping.
  *
  * Memory map a region of the VME master window into user space.
  *
@@ -745,12 +745,13 @@ EXPORT_SYMBOL(vme_master_rmw);
  *         resource or -EFAULT if map exceeds window size. Other generic mmap
  *         errors may also be returned.
  */
-int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma)
+int vme_master_mmap_prepare(struct vme_resource *resource,
+			    struct vm_area_desc *desc)
 {
+	const unsigned long vma_size = vma_desc_size(desc);
 	struct vme_bridge *bridge = find_bridge(resource);
 	struct vme_master_resource *image;
 	phys_addr_t phys_addr;
-	unsigned long vma_size;
 
 	if (resource->type != VME_MASTER) {
 		dev_err(bridge->parent, "Not a master resource\n");
@@ -758,19 +759,18 @@ int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma)
 	}
 
 	image = list_entry(resource->entry, struct vme_master_resource, list);
-	phys_addr = image->bus_resource.start + (vma->vm_pgoff << PAGE_SHIFT);
-	vma_size = vma->vm_end - vma->vm_start;
+	phys_addr = image->bus_resource.start + (desc->pgoff << PAGE_SHIFT);
 
 	if (phys_addr + vma_size > image->bus_resource.end + 1) {
 		dev_err(bridge->parent, "Map size cannot exceed the window size\n");
 		return -EFAULT;
 	}
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-
-	return vm_iomap_memory(vma, phys_addr, vma->vm_end - vma->vm_start);
+	desc->page_prot = pgprot_noncached(desc->page_prot);
+	mmap_action_simple_ioremap(desc, phys_addr, vma_size);
+	return 0;
 }
-EXPORT_SYMBOL(vme_master_mmap);
+EXPORT_SYMBOL(vme_master_mmap_prepare);
 
 /**
  * vme_master_free - Free VME master window
diff --git a/drivers/staging/vme_user/vme.h b/drivers/staging/vme_user/vme.h
index 797e9940fdd19..b6413605ea49c 100644
--- a/drivers/staging/vme_user/vme.h
+++ b/drivers/staging/vme_user/vme.h
@@ -151,7 +151,7 @@ ssize_t vme_master_read(struct vme_resource *resource, void *buf, size_t count,
 ssize_t vme_master_write(struct vme_resource *resource, void *buf, size_t count, loff_t offset);
 unsigned int vme_master_rmw(struct vme_resource *resource, unsigned int mask, unsigned int compare,
 			    unsigned int swap, loff_t offset);
-int vme_master_mmap(struct vme_resource *resource, struct vm_area_struct *vma);
+int vme_master_mmap_prepare(struct vme_resource *resource, struct vm_area_desc *desc);
 void vme_master_free(struct vme_resource *resource);
 
 struct vme_resource *vme_dma_request(struct vme_dev *vdev, u32 route);
diff --git a/drivers/staging/vme_user/vme_user.c b/drivers/staging/vme_user/vme_user.c
index d95dd7d9190ad..11e25c2f6b0aa 100644
--- a/drivers/staging/vme_user/vme_user.c
+++ b/drivers/staging/vme_user/vme_user.c
@@ -446,24 +446,14 @@ static void vme_user_vm_close(struct vm_area_struct *vma)
 	kfree(vma_priv);
 }
 
-static const struct vm_operations_struct vme_user_vm_ops = {
-	.open = vme_user_vm_open,
-	.close = vme_user_vm_close,
-};
-
-static int vme_user_master_mmap(unsigned int minor, struct vm_area_struct *vma)
+static int vme_user_vm_mapped(unsigned long start, unsigned long end, pgoff_t pgoff,
+			      const struct file *file, void **vm_private_data)
 {
-	int err;
+	const unsigned int minor = iminor(file_inode(file));
 	struct vme_user_vma_priv *vma_priv;
 
 	mutex_lock(&image[minor].mutex);
 
-	err = vme_master_mmap(image[minor].resource, vma);
-	if (err) {
-		mutex_unlock(&image[minor].mutex);
-		return err;
-	}
-
 	vma_priv = kmalloc_obj(*vma_priv);
 	if (!vma_priv) {
 		mutex_unlock(&image[minor].mutex);
@@ -472,22 +462,41 @@ static int vme_user_master_mmap(unsigned int minor, struct vm_area_struct *vma)
 
 	vma_priv->minor = minor;
 	refcount_set(&vma_priv->refcnt, 1);
-	vma->vm_ops = &vme_user_vm_ops;
-	vma->vm_private_data = vma_priv;
-
+	*vm_private_data = vma_priv;
 	image[minor].mmap_count++;
 
 	mutex_unlock(&image[minor].mutex);
-
 	return 0;
 }
 
-static int vme_user_mmap(struct file *file, struct vm_area_struct *vma)
+static const struct vm_operations_struct vme_user_vm_ops = {
+	.mapped = vme_user_vm_mapped,
+	.open = vme_user_vm_open,
+	.close = vme_user_vm_close,
+};
+
+static int vme_user_master_mmap_prepare(unsigned int minor,
+					struct vm_area_desc *desc)
+{
+	int err;
+
+	mutex_lock(&image[minor].mutex);
+
+	err = vme_master_mmap_prepare(image[minor].resource, desc);
+	if (!err)
+		desc->vm_ops = &vme_user_vm_ops;
+
+	mutex_unlock(&image[minor].mutex);
+	return err;
+}
+
+static int vme_user_mmap_prepare(struct vm_area_desc *desc)
 {
-	unsigned int minor = iminor(file_inode(file));
+	const struct file *file = desc->file;
+	const unsigned int minor = iminor(file_inode(file));
 
 	if (type[minor] == MASTER_MINOR)
-		return vme_user_master_mmap(minor, vma);
+		return vme_user_master_mmap_prepare(minor, desc);
 
 	return -ENODEV;
 }
@@ -498,7 +507,7 @@ static const struct file_operations vme_user_fops = {
 	.llseek = vme_user_llseek,
 	.unlocked_ioctl = vme_user_unlocked_ioctl,
 	.compat_ioctl = compat_ptr_ioctl,
-	.mmap = vme_user_mmap,
+	.mmap_prepare = vme_user_mmap_prepare,
 };
 
 static int vme_user_match(struct vme_dev *vdev)
diff --git a/drivers/staging/vme_user/vme_user.h b/drivers/staging/vme_user/vme_user.h
index 19ecb05781cce..9e010974351dc 100644
--- a/drivers/staging/vme_user/vme_user.h
+++ b/drivers/staging/vme_user/vme_user.h
@@ -14,11 +14,6 @@ struct vme_master {
 	__u32 aspace;		/* Address Space */
 	__u32 cycle;		/* Cycle properties */
 	__u32 dwidth;		/* Maximum Data Width */
-#if 0
-	char prefetchenable;		/* Prefetch Read Enable State */
-	int prefetchsize;		/* Prefetch Read Size (Cache Lines) */
-	char wrpostenable;		/* Write Post State */
-#endif
 } __packed;
 
 /*
@@ -35,11 +30,6 @@ struct vme_slave {
 	__u64 size;		/* Window Size */
 	__u32 aspace;		/* Address Space */
 	__u32 cycle;		/* Cycle properties */
-#if 0
-	char wrpostenable;		/* Write Post State */
-	char rmwlock;			/* Lock PCI during RMW Cycles */
-	char data64bitcapable;		/* non-VMEbus capable of 64-bit Data */
-#endif
 } __packed;
 
 struct vme_irq_id {
diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
index efe8cdb200609..704ec94383c3c 100644
--- a/drivers/target/iscsi/iscsi_target_configfs.c
+++ b/drivers/target/iscsi/iscsi_target_configfs.c
@@ -1591,6 +1591,7 @@ const struct target_core_fabric_ops iscsi_ops = {
 
 	.write_pending_must_be_called	= 1,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index d668bd19fd4ae..a25fd826b5420 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -26,6 +26,7 @@
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/configfs.h>
+#include <linux/blk-mq.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_tcq.h>
 #include <scsi/scsi_host.h>
@@ -269,15 +270,27 @@ static int tcm_loop_device_reset(struct scsi_cmnd *sc)
 	return (ret == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED;
 }
 
+static bool tcm_loop_flush_work_iter(struct request *rq, void *data)
+{
+	struct scsi_cmnd *sc = blk_mq_rq_to_pdu(rq);
+	struct tcm_loop_cmd *tl_cmd = scsi_cmd_priv(sc);
+	struct se_cmd *se_cmd = &tl_cmd->tl_se_cmd;
+
+	flush_work(&se_cmd->work);
+	return true;
+}
+
 static int tcm_loop_target_reset(struct scsi_cmnd *sc)
 {
 	struct tcm_loop_hba *tl_hba;
 	struct tcm_loop_tpg *tl_tpg;
+	struct Scsi_Host *sh = sc->device->host;
+	int ret;
 
 	/*
 	 * Locate the tcm_loop_hba_t pointer
 	 */
-	tl_hba = *(struct tcm_loop_hba **)shost_priv(sc->device->host);
+	tl_hba = *(struct tcm_loop_hba **)shost_priv(sh);
 	if (!tl_hba) {
 		pr_err("Unable to perform device reset without active I_T Nexus\n");
 		return FAILED;
@@ -286,11 +299,38 @@ static int tcm_loop_target_reset(struct scsi_cmnd *sc)
 	 * Locate the tl_tpg pointer from TargetID in sc->device->id
 	 */
 	tl_tpg = &tl_hba->tl_hba_tpgs[sc->device->id];
-	if (tl_tpg) {
-		tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE;
-		return SUCCESS;
-	}
-	return FAILED;
+	if (!tl_tpg)
+		return FAILED;
+
+	/*
+	 * Issue a LUN_RESET to drain all commands that the target core
+	 * knows about.  This handles commands not yet marked CMD_T_COMPLETE.
+	 */
+	ret = tcm_loop_issue_tmr(tl_tpg, sc->device->lun, 0, TMR_LUN_RESET);
+	if (ret != TMR_FUNCTION_COMPLETE)
+		return FAILED;
+
+	/*
+	 * Flush any deferred target core completion work that may still be
+	 * queued.  Commands that already had CMD_T_COMPLETE set before the TMR
+	 * are skipped by the TMR drain, but their async completion work
+	 * (transport_lun_remove_cmd → percpu_ref_put, release_cmd → scsi_done)
+	 * may still be pending in target_completion_wq.
+	 *
+	 * The SCSI EH will reuse in-flight scsi_cmnd structures for recovery
+	 * commands (e.g. TUR) immediately after this handler returns SUCCESS —
+	 * if deferred work is still pending, the memset in queuecommand would
+	 * zero the se_cmd while the work accesses it, leaking the LUN
+	 * percpu_ref and hanging configfs unlink forever.
+	 *
+	 * Use blk_mq_tagset_busy_iter() to find all started requests and
+	 * flush_work() on each — the same pattern used by mpi3mr, scsi_debug,
+	 * and other SCSI drivers to drain outstanding commands during reset.
+	 */
+	blk_mq_tagset_busy_iter(&sh->tag_set, tcm_loop_flush_work_iter, NULL);
+
+	tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE;
+	return SUCCESS;
 }
 
 static const struct scsi_host_template tcm_loop_driver_template = {
@@ -1107,6 +1147,7 @@ static const struct target_core_fabric_ops loop_ops = {
 	.tfc_wwn_attrs			= tcm_loop_wwn_attrs,
 	.tfc_tpg_base_attrs		= tcm_loop_tpg_attrs,
 	.tfc_tpg_attrib_attrs		= tcm_loop_tpg_attrib_attrs,
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_QUEUE_SUBMIT,
 	.direct_submit_supp		= 0,
 };
diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c
index ad1da7edbb089..896fc0f0379f7 100644
--- a/drivers/target/sbp/sbp_target.c
+++ b/drivers/target/sbp/sbp_target.c
@@ -2278,6 +2278,7 @@ static const struct target_core_fabric_ops sbp_ops = {
 	.tfc_tpg_base_attrs		= sbp_tpg_base_attrs,
 	.tfc_tpg_attrib_attrs		= sbp_tpg_attrib_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index a1c91d4515bc5..d93773b3227c3 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -575,6 +575,7 @@ DEF_CONFIGFS_ATTRIB_SHOW(unmap_zeroes_data);
 DEF_CONFIGFS_ATTRIB_SHOW(max_write_same_len);
 DEF_CONFIGFS_ATTRIB_SHOW(emulate_rsoc);
 DEF_CONFIGFS_ATTRIB_SHOW(submit_type);
+DEF_CONFIGFS_ATTRIB_SHOW(complete_type);
 DEF_CONFIGFS_ATTRIB_SHOW(atomic_max_len);
 DEF_CONFIGFS_ATTRIB_SHOW(atomic_alignment);
 DEF_CONFIGFS_ATTRIB_SHOW(atomic_granularity);
@@ -1266,6 +1267,24 @@ static ssize_t submit_type_store(struct config_item *item, const char *page,
 	return count;
 }
 
+static ssize_t complete_type_store(struct config_item *item, const char *page,
+				   size_t count)
+{
+	struct se_dev_attrib *da = to_attrib(item);
+	int ret;
+	u8 val;
+
+	ret = kstrtou8(page, 0, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val > TARGET_QUEUE_COMPL)
+		return -EINVAL;
+
+	da->complete_type = val;
+	return count;
+}
+
 CONFIGFS_ATTR(, emulate_model_alias);
 CONFIGFS_ATTR(, emulate_dpo);
 CONFIGFS_ATTR(, emulate_fua_write);
@@ -1302,6 +1321,7 @@ CONFIGFS_ATTR(, max_write_same_len);
 CONFIGFS_ATTR(, alua_support);
 CONFIGFS_ATTR(, pgr_support);
 CONFIGFS_ATTR(, submit_type);
+CONFIGFS_ATTR(, complete_type);
 CONFIGFS_ATTR_RO(, atomic_max_len);
 CONFIGFS_ATTR_RO(, atomic_alignment);
 CONFIGFS_ATTR_RO(, atomic_granularity);
@@ -1350,6 +1370,7 @@ struct configfs_attribute *sbc_attrib_attrs[] = {
 	&attr_pgr_support,
 	&attr_emulate_rsoc,
 	&attr_submit_type,
+	&attr_complete_type,
 	&attr_atomic_alignment,
 	&attr_atomic_max_len,
 	&attr_atomic_granularity,
@@ -1373,6 +1394,7 @@ struct configfs_attribute *passthrough_attrib_attrs[] = {
 	&attr_alua_support,
 	&attr_pgr_support,
 	&attr_submit_type,
+	&attr_complete_type,
 	NULL,
 };
 EXPORT_SYMBOL(passthrough_attrib_attrs);
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 74c6383f9eede..9db2201aa553f 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -813,6 +813,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 				DA_UNMAP_ZEROES_DATA_DEFAULT;
 	dev->dev_attrib.max_write_same_len = DA_MAX_WRITE_SAME_LEN;
 	dev->dev_attrib.submit_type = TARGET_FABRIC_DEFAULT_SUBMIT;
+	dev->dev_attrib.complete_type = TARGET_FABRIC_DEFAULT_COMPL;
 
 	/* Skip allocating lun_stats since we can't export them. */
 	xcopy_lun = &dev->xcopy_lun;
diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c
index 331689b30f854..166dbf4c40616 100644
--- a/drivers/target/target_core_fabric_configfs.c
+++ b/drivers/target/target_core_fabric_configfs.c
@@ -1065,6 +1065,28 @@ target_fabric_wwn_cmd_completion_affinity_store(struct config_item *item,
 }
 CONFIGFS_ATTR(target_fabric_wwn_, cmd_completion_affinity);
 
+static ssize_t
+target_fabric_wwn_default_complete_type_show(struct config_item *item,
+					     char *page)
+{
+	struct se_wwn *wwn = container_of(to_config_group(item), struct se_wwn,
+					  param_group);
+	return sysfs_emit(page, "%u\n",
+			  wwn->wwn_tf->tf_ops->default_compl_type);
+}
+CONFIGFS_ATTR_RO(target_fabric_wwn_, default_complete_type);
+
+static ssize_t
+target_fabric_wwn_direct_complete_supported_show(struct config_item *item,
+						 char *page)
+{
+	struct se_wwn *wwn = container_of(to_config_group(item), struct se_wwn,
+					  param_group);
+	return sysfs_emit(page, "%u\n",
+			  wwn->wwn_tf->tf_ops->direct_compl_supp);
+}
+CONFIGFS_ATTR_RO(target_fabric_wwn_, direct_complete_supported);
+
 static ssize_t
 target_fabric_wwn_default_submit_type_show(struct config_item *item,
 					   char *page)
@@ -1089,6 +1111,8 @@ CONFIGFS_ATTR_RO(target_fabric_wwn_, direct_submit_supported);
 
 static struct configfs_attribute *target_fabric_wwn_param_attrs[] = {
 	&target_fabric_wwn_attr_cmd_completion_affinity,
+	&target_fabric_wwn_attr_default_complete_type,
+	&target_fabric_wwn_attr_direct_complete_supported,
 	&target_fabric_wwn_attr_default_submit_type,
 	&target_fabric_wwn_attr_direct_submit_supported,
 	NULL,
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 3ae1f7137d9d5..62ced9f5102ff 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -173,7 +173,7 @@ static int fd_configure_device(struct se_device *dev)
 		 */
 		dev->dev_attrib.max_write_same_len = 0xFFFF;
 
-		if (bdev_nonrot(bdev))
+		if (!bdev_rot(bdev))
 			dev->dev_attrib.is_nonrot = 1;
 	} else {
 		if (!(fd_dev->fbd_flags & FBDF_HAS_SIZE)) {
@@ -276,7 +276,7 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 	ssize_t len = 0;
 	int ret = 0, i;
 
-	aio_cmd = kmalloc_flex(*aio_cmd, bvecs, sgl_nents);
+	aio_cmd = kzalloc_flex(*aio_cmd, bvecs, sgl_nents);
 	if (!aio_cmd)
 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index 3c92f94497b47..1087d1d17c36d 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -148,7 +148,7 @@ static int iblock_configure_device(struct se_device *dev)
 	else
 		dev->dev_attrib.max_write_same_len = 0xFFFF;
 
-	if (bdev_nonrot(bd))
+	if (!bdev_rot(bd))
 		dev->dev_attrib.is_nonrot = 1;
 
 	target_configure_write_atomic_from_bdev(&dev->dev_attrib, bd);
diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c
index abe91dc8722e4..21f5cb86d70c0 100644
--- a/drivers/target/target_core_sbc.c
+++ b/drivers/target/target_core_sbc.c
@@ -1187,7 +1187,8 @@ sbc_execute_unmap(struct se_cmd *cmd)
 			goto err;
 		}
 
-		if (lba + range > dev->transport->get_blocks(dev) + 1) {
+		if (lba + range < lba ||
+		    lba + range > dev->transport->get_blocks(dev) + 1) {
 			ret = TCM_ADDRESS_OUT_OF_RANGE;
 			goto err;
 		}
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index a7330c4fedde3..4e8d779dda5e2 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -902,13 +902,59 @@ static bool target_cmd_interrupted(struct se_cmd *cmd)
 	return false;
 }
 
+static void target_complete(struct se_cmd *cmd, int success)
+{
+	struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn;
+	struct se_dev_attrib *da;
+	u8 compl_type;
+	int cpu;
+
+	if (!wwn) {
+		cpu = cmd->cpuid;
+		goto queue_work;
+	}
+
+	da = &cmd->se_dev->dev_attrib;
+	if (da->complete_type == TARGET_FABRIC_DEFAULT_COMPL)
+		compl_type = wwn->wwn_tf->tf_ops->default_compl_type;
+	else if (da->complete_type == TARGET_DIRECT_COMPL &&
+		 wwn->wwn_tf->tf_ops->direct_compl_supp)
+		compl_type = TARGET_DIRECT_COMPL;
+	else
+		compl_type = TARGET_QUEUE_COMPL;
+
+	if (compl_type == TARGET_DIRECT_COMPL) {
+		/*
+		 * Failure handling and processing secondary stages of
+		 * complex commands can be too heavy to handle from the
+		 * fabric driver so always defer.
+		 */
+		if (success && !cmd->transport_complete_callback) {
+			target_complete_ok_work(&cmd->work);
+			return;
+		}
+
+		compl_type = TARGET_QUEUE_COMPL;
+	}
+
+queue_work:
+	INIT_WORK(&cmd->work, success ? target_complete_ok_work :
+		  target_complete_failure_work);
+
+	if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID)
+		cpu = cmd->cpuid;
+	else
+		cpu = wwn->cmd_compl_affinity;
+
+	queue_work_on(cpu, target_completion_wq, &cmd->work);
+}
+
 /* May be called from interrupt context so must not sleep. */
 void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status,
 				    sense_reason_t sense_reason)
 {
-	struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn;
-	int success, cpu;
 	unsigned long flags;
+	int success;
 
 	if (target_cmd_interrupted(cmd))
 		return;
@@ -933,15 +979,7 @@ void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status,
 	cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
 	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 
-	INIT_WORK(&cmd->work, success ? target_complete_ok_work :
-		  target_complete_failure_work);
-
-	if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID)
-		cpu = cmd->cpuid;
-	else
-		cpu = wwn->cmd_compl_affinity;
-
-	queue_work_on(cpu, target_completion_wq, &cmd->work);
+	target_complete(cmd, success);
 }
 EXPORT_SYMBOL(target_complete_cmd_with_sense);
 
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index af95531ddd351..edc2afd5f4ee2 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -1860,6 +1860,17 @@ static struct page *tcmu_try_get_data_page(struct tcmu_dev *udev, uint32_t dpi)
 	return NULL;
 }
 
+static int tcmu_vma_mapped(unsigned long start, unsigned long end, pgoff_t pgoff,
+			   const struct file *file, void **vm_private_data)
+{
+	struct tcmu_dev *udev = *vm_private_data;
+
+	pr_debug("vma_mapped\n");
+
+	kref_get(&udev->kref);
+	return 0;
+}
+
 static void tcmu_vma_open(struct vm_area_struct *vma)
 {
 	struct tcmu_dev *udev = vma->vm_private_data;
@@ -1919,26 +1930,25 @@ static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
 }
 
 static const struct vm_operations_struct tcmu_vm_ops = {
+	.mapped = tcmu_vma_mapped,
 	.open = tcmu_vma_open,
 	.close = tcmu_vma_close,
 	.fault = tcmu_vma_fault,
 };
 
-static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
+static int tcmu_mmap_prepare(struct uio_info *info, struct vm_area_desc *desc)
 {
 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
 
-	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_ops = &tcmu_vm_ops;
+	vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
+	desc->vm_ops = &tcmu_vm_ops;
 
-	vma->vm_private_data = udev;
+	desc->private_data = udev;
 
 	/* Ensure the mmap is exactly the right size */
-	if (vma_pages(vma) != udev->mmap_pages)
+	if (vma_desc_pages(desc) != udev->mmap_pages)
 		return -EINVAL;
 
-	tcmu_vma_open(vma);
-
 	return 0;
 }
 
@@ -2253,7 +2263,7 @@ static int tcmu_configure_device(struct se_device *dev)
 	info->irqcontrol = tcmu_irqcontrol;
 	info->irq = UIO_IRQ_CUSTOM;
 
-	info->mmap = tcmu_mmap;
+	info->mmap_prepare = tcmu_mmap_prepare;
 	info->open = tcmu_open;
 	info->release = tcmu_release;
 
diff --git a/drivers/target/tcm_fc/tfc_conf.c b/drivers/target/tcm_fc/tfc_conf.c
index 88cf1e5a5810a..3920fb02d9fc0 100644
--- a/drivers/target/tcm_fc/tfc_conf.c
+++ b/drivers/target/tcm_fc/tfc_conf.c
@@ -434,6 +434,7 @@ static const struct target_core_fabric_ops ft_fabric_ops = {
 	.tfc_wwn_attrs			= ft_wwn_attrs,
 	.tfc_tpg_nacl_base_attrs	= ft_nacl_base_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/tee/optee/device.c b/drivers/tee/optee/device.c
index bae954e79fdc1..f751d743a120c 100644
--- a/drivers/tee/optee/device.c
+++ b/drivers/tee/optee/device.c
@@ -13,10 +13,7 @@
 
 static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data)
 {
-	if (ver->impl_id == TEE_IMPL_ID_OPTEE)
-		return 1;
-	else
-		return 0;
+	return (ver->impl_id == TEE_IMPL_ID_OPTEE);
 }
 
 static int get_devices(struct tee_context *ctx, u32 session,
diff --git a/drivers/tee/optee/supp.c b/drivers/tee/optee/supp.c
index a3d11b1f90fa5..06747e90c2309 100644
--- a/drivers/tee/optee/supp.c
+++ b/drivers/tee/optee/supp.c
@@ -10,7 +10,11 @@
 struct optee_supp_req {
 	struct list_head link;
 
+	int id;
+
 	bool in_queue;
+	bool processed;
+
 	u32 func;
 	u32 ret;
 	size_t num_params;
@@ -19,6 +23,9 @@ struct optee_supp_req {
 	struct completion c;
 };
 
+/* It is temporary request used for revoked pending request in supp->idr. */
+#define INVALID_REQ_PTR ((struct optee_supp_req *)ERR_PTR(-EBADF))
+
 void optee_supp_init(struct optee_supp *supp)
 {
 	memset(supp, 0, sizeof(*supp));
@@ -39,21 +46,23 @@ void optee_supp_release(struct optee_supp *supp)
 {
 	int id;
 	struct optee_supp_req *req;
-	struct optee_supp_req *req_tmp;
 
 	mutex_lock(&supp->mutex);
 
-	/* Abort all request retrieved by supplicant */
+	/* Abort all request */
 	idr_for_each_entry(&supp->idr, req, id) {
 		idr_remove(&supp->idr, id);
-		req->ret = TEEC_ERROR_COMMUNICATION;
-		complete(&req->c);
-	}
+		/* Skip if request was already marked invalid */
+		if (IS_ERR(req))
+			continue;
 
-	/* Abort all queued requests */
-	list_for_each_entry_safe(req, req_tmp, &supp->reqs, link) {
-		list_del(&req->link);
-		req->in_queue = false;
+		/* For queued requests where supplicant has not seen it */
+		if (req->in_queue) {
+			list_del(&req->link);
+			req->in_queue = false;
+		}
+
+		req->processed = true;
 		req->ret = TEEC_ERROR_COMMUNICATION;
 		complete(&req->c);
 	}
@@ -100,8 +109,16 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
 
 	/* Insert the request in the request list */
 	mutex_lock(&supp->mutex);
+	req->id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
+	if (req->id < 0) {
+		mutex_unlock(&supp->mutex);
+		kfree(req);
+		return TEEC_ERROR_OUT_OF_MEMORY;
+	}
+
 	list_add_tail(&req->link, &supp->reqs);
 	req->in_queue = true;
+	req->processed = false;
 	mutex_unlock(&supp->mutex);
 
 	/* Tell an eventual waiter there's a new request */
@@ -117,21 +134,43 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
 	if (wait_for_completion_killable(&req->c)) {
 		mutex_lock(&supp->mutex);
 		if (req->in_queue) {
+			/* Supplicant has not seen this request yet. */
+			idr_remove(&supp->idr, req->id);
 			list_del(&req->link);
 			req->in_queue = false;
+
+			ret = TEEC_ERROR_COMMUNICATION;
+		} else if (req->processed) {
+			/*
+			 * Supplicant has processed this request. Ignore the
+			 * kill signal for now and submit the result. req is not
+			 * in supp->reqs (removed by supp_pop_entry()) nor in
+			 * supp->idr (removed by supp_pop_req()).
+			 */
+			ret = req->ret;
+		} else {
+			/*
+			 * Supplicant is in the middle of processing this
+			 * request. Replace req with INVALID_REQ_PTR so that
+			 * the ID remains busy, causing optee_supp_send() to
+			 * fail on the next call to supp_pop_req() with this ID.
+			 */
+			idr_replace(&supp->idr, INVALID_REQ_PTR, req->id);
+			ret = TEEC_ERROR_COMMUNICATION;
 		}
+
 		mutex_unlock(&supp->mutex);
-		req->ret = TEEC_ERROR_COMMUNICATION;
+	} else {
+		ret = req->ret;
 	}
 
-	ret = req->ret;
 	kfree(req);
 
 	return ret;
 }
 
 static struct optee_supp_req  *supp_pop_entry(struct optee_supp *supp,
-					      int num_params, int *id)
+					      int num_params)
 {
 	struct optee_supp_req *req;
 
@@ -153,10 +192,6 @@ static struct optee_supp_req  *supp_pop_entry(struct optee_supp *supp,
 		return ERR_PTR(-EINVAL);
 	}
 
-	*id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
-	if (*id < 0)
-		return ERR_PTR(-ENOMEM);
-
 	list_del(&req->link);
 	req->in_queue = false;
 
@@ -214,7 +249,6 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 	struct optee *optee = tee_get_drvdata(teedev);
 	struct optee_supp *supp = &optee->supp;
 	struct optee_supp_req *req = NULL;
-	int id;
 	size_t num_meta;
 	int rc;
 
@@ -224,15 +258,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 
 	while (true) {
 		mutex_lock(&supp->mutex);
-		req = supp_pop_entry(supp, *num_params - num_meta, &id);
+		req = supp_pop_entry(supp, *num_params - num_meta);
+		if (req)
+			break; /* Keep mutex held. */
 		mutex_unlock(&supp->mutex);
 
-		if (req) {
-			if (IS_ERR(req))
-				return PTR_ERR(req);
-			break;
-		}
-
 		/*
 		 * If we didn't get a request we'll block in
 		 * wait_for_completion() to avoid needless spinning.
@@ -245,6 +275,13 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 			return -ERESTARTSYS;
 	}
 
+	/* supp->mutex held and req != NULL. */
+
+	if (IS_ERR(req)) {
+		mutex_unlock(&supp->mutex);
+		return PTR_ERR(req);
+	}
+
 	if (num_meta) {
 		/*
 		 * tee-supplicant support meta parameters -> requsts can be
@@ -252,13 +289,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 		 */
 		param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT |
 			      TEE_IOCTL_PARAM_ATTR_META;
-		param->u.value.a = id;
+		param->u.value.a = req->id;
 		param->u.value.b = 0;
 		param->u.value.c = 0;
 	} else {
-		mutex_lock(&supp->mutex);
-		supp->req_id = id;
-		mutex_unlock(&supp->mutex);
+		supp->req_id = req->id;
 	}
 
 	*func = req->func;
@@ -266,6 +301,7 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 	memcpy(param + num_meta, req->param,
 	       sizeof(struct tee_param) * req->num_params);
 
+	mutex_unlock(&supp->mutex);
 	return 0;
 }
 
@@ -297,12 +333,17 @@ static struct optee_supp_req *supp_pop_req(struct optee_supp *supp,
 	if (!req)
 		return ERR_PTR(-ENOENT);
 
+	/* optee_supp_thrd_req() already returned to optee. */
+	if (IS_ERR(req))
+		goto failed_req;
+
 	if ((num_params - nm) != req->num_params)
 		return ERR_PTR(-EINVAL);
 
+	*num_meta = nm;
+failed_req:
 	idr_remove(&supp->idr, id);
 	supp->req_id = -1;
-	*num_meta = nm;
 
 	return req;
 }
@@ -328,10 +369,9 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
 
 	mutex_lock(&supp->mutex);
 	req = supp_pop_req(supp, num_params, param, &num_meta);
-	mutex_unlock(&supp->mutex);
-
 	if (IS_ERR(req)) {
-		/* Something is wrong, let supplicant restart. */
+		mutex_unlock(&supp->mutex);
+		/* Something is wrong, let supplicant handel it. */
 		return PTR_ERR(req);
 	}
 
@@ -355,9 +395,10 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
 		}
 	}
 	req->ret = ret;
-
+	req->processed = true;
 	/* Let the requesting thread continue */
 	complete(&req->c);
+	mutex_unlock(&supp->mutex);
 
 	return 0;
 }
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index 6c5b9e352e5e0..e9ea9f80cfd9a 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -23,29 +23,11 @@ struct tee_shm_dma_mem {
 	struct page *page;
 };
 
-static void shm_put_kernel_pages(struct page **pages, size_t page_count)
-{
-	size_t n;
-
-	for (n = 0; n < page_count; n++)
-		put_page(pages[n]);
-}
-
-static void shm_get_kernel_pages(struct page **pages, size_t page_count)
-{
-	size_t n;
-
-	for (n = 0; n < page_count; n++)
-		get_page(pages[n]);
-}
-
 static void release_registered_pages(struct tee_shm *shm)
 {
 	if (shm->pages) {
 		if (shm->flags & TEE_SHM_USER_MAPPED)
 			unpin_user_pages(shm->pages, shm->num_pages);
-		else
-			shm_put_kernel_pages(shm->pages, shm->num_pages);
 
 		kfree(shm->pages);
 	}
@@ -477,13 +459,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags,
 		goto err_put_shm_pages;
 	}
 
-	/*
-	 * iov_iter_extract_kvec_pages does not get reference on the pages,
-	 * get a reference on them.
-	 */
-	if (iov_iter_is_kvec(iter))
-		shm_get_kernel_pages(shm->pages, num_pages);
-
 	shm->offset = off;
 	shm->size = len;
 	shm->num_pages = num_pages;
@@ -499,8 +474,6 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags,
 err_put_shm_pages:
 	if (!iov_iter_is_kvec(iter))
 		unpin_user_pages(shm->pages, shm->num_pages);
-	else
-		shm_put_kernel_pages(shm->pages, shm->num_pages);
 err_free_shm_pages:
 	kfree(shm->pages);
 err_free_shm:
diff --git a/drivers/thermal/imx91_thermal.c b/drivers/thermal/imx91_thermal.c
index 9b20be03d6dec..25915bb702bea 100644
--- a/drivers/thermal/imx91_thermal.c
+++ b/drivers/thermal/imx91_thermal.c
@@ -17,6 +17,8 @@
 #include <linux/thermal.h>
 #include <linux/units.h>
 
+#include "thermal_hwmon.h"
+
 #define REG_SET					0x4
 #define REG_CLR					0x8
 #define REG_TOG					0xc
@@ -318,6 +320,8 @@ static int imx91_tmu_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(tmu->tzd),
 				     "failed to register thermal zone sensor\n");
 
+	devm_thermal_add_hwmon_sysfs(dev, tmu->tzd);
+
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c
index bf51a17c5be61..5dbeb0a43c8c0 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c
@@ -19,6 +19,13 @@ static const struct rapl_mmio_regs rapl_mmio_default = {
 	.limits[RAPL_DOMAIN_DRAM] = BIT(POWER_LIMIT2),
 };
 
+static const struct rapl_defaults rapl_defaults_mmio = {
+	.floor_freq_reg_addr = 0,
+	.check_unit = rapl_default_check_unit,
+	.set_floor_freq = rapl_default_set_floor_freq,
+	.compute_time_window = rapl_default_compute_time_window,
+};
+
 static int rapl_mmio_read_raw(int cpu, struct reg_action *ra, bool atomic)
 {
 	if (!ra->reg.mmio)
@@ -67,6 +74,7 @@ int proc_thermal_rapl_add(struct pci_dev *pdev, struct proc_thermal_device *proc
 
 	rapl_mmio_priv.read_raw = rapl_mmio_read_raw;
 	rapl_mmio_priv.write_raw = rapl_mmio_write_raw;
+	rapl_mmio_priv.defaults = &rapl_defaults_mmio;
 
 	rapl_mmio_priv.control_type = powercap_register_control_type(NULL, "intel-rapl-mmio", NULL);
 	if (IS_ERR(rapl_mmio_priv.control_type)) {
@@ -111,4 +119,5 @@ void proc_thermal_rapl_remove(void)
 EXPORT_SYMBOL_GPL(proc_thermal_rapl_remove);
 
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("INTEL_RAPL");
 MODULE_DESCRIPTION("RAPL interface using MMIO");
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
index 314fbc1f490f1..1a7e134dfcf8d 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c
@@ -402,17 +402,31 @@ static ssize_t rfi_restriction_show(struct device *dev,
 	return sysfs_emit(buf, "%llu\n", resp);
 }
 
+ /* ddr_data_rate */
+static const struct mmio_reg nvl_ddr_data_rate_reg = { 1, 0xE0, 10, 0x3FF, 2};
+
+static const struct mmio_reg *ddr_data_rate_reg;
+
 static ssize_t ddr_data_rate_show(struct device *dev,
 				  struct device_attribute *attr,
 				  char *buf)
 {
-	u16 id = 0x0107;
 	u64 resp;
-	int ret;
 
-	ret = processor_thermal_send_mbox_read_cmd(to_pci_dev(dev), id, &resp);
-	if (ret)
-		return ret;
+	if (ddr_data_rate_reg) {
+		u16 reg_val;
+
+		pci_read_config_word(to_pci_dev(dev), ddr_data_rate_reg->offset, &reg_val);
+		resp = (reg_val >> ddr_data_rate_reg->shift) & ddr_data_rate_reg->mask;
+		resp = (resp * 3333) / 100;
+	} else {
+		const u16 id = 0x0107;
+		int ret;
+
+		ret = processor_thermal_send_mbox_read_cmd(to_pci_dev(dev), id, &resp);
+		if (ret)
+			return ret;
+	}
 
 	return sysfs_emit(buf, "%llu\n", resp);
 }
@@ -461,6 +475,7 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc
 		case PCI_DEVICE_ID_INTEL_NVL_H_THERMAL:
 		case PCI_DEVICE_ID_INTEL_NVL_S_THERMAL:
 			dlvr_mmio_regs_table = nvl_dlvr_mmio_regs;
+			ddr_data_rate_reg = &nvl_ddr_data_rate_reg;
 			break;
 		default:
 			dlvr_mmio_regs_table = dlvr_mmio_regs;
diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c
index 1a1a95b394053..8c4ae75231f8e 100644
--- a/drivers/thermal/intel/intel_hfi.c
+++ b/drivers/thermal/intel/intel_hfi.c
@@ -526,7 +526,7 @@ void intel_hfi_offline(unsigned int cpu)
 	mutex_lock(&hfi_instance_lock);
 	cpumask_clear_cpu(cpu, hfi_instance->cpus);
 
-	if (!cpumask_weight(hfi_instance->cpus))
+	if (cpumask_empty(hfi_instance->cpus))
 		hfi_disable();
 
 	mutex_unlock(&hfi_instance_lock);
diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c
index e546067c96218..44fa45f74da72 100644
--- a/drivers/thermal/sprd_thermal.c
+++ b/drivers/thermal/sprd_thermal.c
@@ -178,7 +178,7 @@ static int sprd_thm_sensor_calibration(struct device_node *np,
 static int sprd_thm_rawdata_to_temp(struct sprd_thermal_sensor *sen,
 				    u32 rawdata)
 {
-	clamp(rawdata, (u32)SPRD_THM_RAW_DATA_LOW, (u32)SPRD_THM_RAW_DATA_HIGH);
+	rawdata = clamp(rawdata, SPRD_THM_RAW_DATA_LOW, SPRD_THM_RAW_DATA_HIGH);
 
 	/*
 	 * According to the thermal datasheet, the formula of converting
@@ -192,7 +192,7 @@ static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen)
 {
 	u32 val;
 
-	clamp(temp, (int)SPRD_THM_TEMP_LOW, (int)SPRD_THM_TEMP_HIGH);
+	temp = clamp(temp, SPRD_THM_TEMP_LOW, SPRD_THM_TEMP_HIGH);
 
 	/*
 	 * According to the thermal datasheet, the formula of converting
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index b7d706ed7ed96..479916b67f480 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -861,7 +861,7 @@ static int thermal_bind_cdev_to_trip(struct thermal_zone_device *tz,
 		goto free_mem;
 
 	dev->id = result;
-	sprintf(dev->name, "cdev%d", dev->id);
+	snprintf(dev->name, sizeof(dev->name), "cdev%d", dev->id);
 	result =
 	    sysfs_create_link(&tz->device.kobj, &cdev->device.kobj, dev->name);
 	if (result)
diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
index 6ebb83cb70b2f..99085c806a1f6 100644
--- a/drivers/thermal/thermal_of.c
+++ b/drivers/thermal/thermal_of.c
@@ -144,7 +144,7 @@ static struct device_node *of_thermal_zone_find(struct device_node *sensor, int
 		count = of_count_phandle_with_args(child, "thermal-sensors",
 						   "#thermal-sensor-cells");
 		if (count <= 0) {
-			pr_err("%pOFn: missing thermal sensor\n", child);
+			pr_err("%pOFP: missing thermal sensor\n", child);
 			return ERR_PTR(-EINVAL);
 		}
 
@@ -156,14 +156,14 @@ static struct device_node *of_thermal_zone_find(struct device_node *sensor, int
 							 "#thermal-sensor-cells",
 							 i, &sensor_specs);
 			if (ret < 0) {
-				pr_err("%pOFn: Failed to read thermal-sensors cells: %d\n", child, ret);
+				pr_err("%pOFP: Failed to read thermal-sensors cells: %d\n", child, ret);
 				return ERR_PTR(ret);
 			}
 
 			of_node_put(sensor_specs.np);
 			if ((sensor == sensor_specs.np) && id == (sensor_specs.args_count ?
 								  sensor_specs.args[0] : 0)) {
-				pr_debug("sensor %pOFn id=%d belongs to %pOFn\n", sensor, id, child);
+				pr_debug("sensor %pOFP id=%d belongs to %pOFP\n", sensor, id, child);
 				return no_free_ptr(child);
 			}
 		}
@@ -180,7 +180,7 @@ static int thermal_of_monitor_init(struct device_node *np, int *delay, int *pdel
 	if (ret == -EINVAL) {
 		*pdelay = 0;
 	} else if (ret < 0) {
-		pr_err("%pOFn: Couldn't get polling-delay-passive: %d\n", np, ret);
+		pr_err("%pOFP: Couldn't get polling-delay-passive: %d\n", np, ret);
 		return ret;
 	}
 
@@ -188,7 +188,7 @@ static int thermal_of_monitor_init(struct device_node *np, int *delay, int *pdel
 	if (ret == -EINVAL) {
 		*delay = 0;
 	} else if (ret < 0) {
-		pr_err("%pOFn: Couldn't get polling-delay: %d\n", np, ret);
+		pr_err("%pOFP: Couldn't get polling-delay: %d\n", np, ret);
 		return ret;
 	}
 
@@ -380,23 +380,23 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node *
 	np = of_thermal_zone_find(sensor, id);
 	if (IS_ERR(np)) {
 		if (PTR_ERR(np) != -ENODEV)
-			pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id);
+			pr_err("Failed to find thermal zone for %pOFP id=%d\n", sensor, id);
 		return ERR_CAST(np);
 	}
 
 	trips = thermal_of_trips_init(np, &ntrips);
 	if (IS_ERR(trips)) {
-		pr_err("Failed to parse trip points for %pOFn id=%d\n", sensor, id);
+		pr_err("Failed to parse trip points for %pOFP id=%d\n", sensor, id);
 		ret = PTR_ERR(trips);
 		goto out_of_node_put;
 	}
 
 	if (!trips)
-		pr_info("No trip points found for %pOFn id=%d\n", sensor, id);
+		pr_info("No trip points found for %pOFP id=%d\n", sensor, id);
 
 	ret = thermal_of_monitor_init(np, &delay, &pdelay);
 	if (ret) {
-		pr_err("Failed to initialize monitoring delays from %pOFn\n", np);
+		pr_err("Failed to initialize monitoring delays from %pOFP\n", np);
 		goto out_kfree_trips;
 	}
 
@@ -417,7 +417,7 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node *
 						     pdelay, delay);
 	if (IS_ERR(tz)) {
 		ret = PTR_ERR(tz);
-		pr_err("Failed to register thermal zone %pOFn: %d\n", np, ret);
+		pr_err("Failed to register thermal zone %pOFP: %d\n", np, ret);
 		goto out_kfree_trips;
 	}
 
diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c
index 46b3b2df935c9..5eecae13f07d8 100644
--- a/drivers/thermal/thermal_sysfs.c
+++ b/drivers/thermal/thermal_sysfs.c
@@ -18,6 +18,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/string_choices.h>
 #include <linux/jiffies.h>
 
 #include "thermal_core.h"
@@ -56,10 +57,8 @@ mode_show(struct device *dev, struct device_attribute *attr, char *buf)
 
 	guard(thermal_zone)(tz);
 
-	if (tz->mode == THERMAL_DEVICE_ENABLED)
-		return sysfs_emit(buf, "enabled\n");
-
-	return sysfs_emit(buf, "disabled\n");
+	return sysfs_emit(buf, "%s\n",
+		str_enabled_disabled(tz->mode == THERMAL_DEVICE_ENABLED));
 }
 
 static ssize_t
diff --git a/drivers/thunderbolt/dma_port.c b/drivers/thunderbolt/dma_port.c
index 334fefe212552..c7c2942fa7bed 100644
--- a/drivers/thunderbolt/dma_port.c
+++ b/drivers/thunderbolt/dma_port.c
@@ -55,7 +55,7 @@ struct tb_dma_port {
 	struct tb_switch *sw;
 	u8 port;
 	u32 base;
-	u8 *buf;
+	u8 buf[];
 };
 
 /*
@@ -209,16 +209,10 @@ struct tb_dma_port *dma_port_alloc(struct tb_switch *sw)
 	if (port < 0)
 		return NULL;
 
-	dma = kzalloc_obj(*dma);
+	dma = kzalloc_flex(*dma, buf, MAIL_DATA_DWORDS);
 	if (!dma)
 		return NULL;
 
-	dma->buf = kmalloc_array(MAIL_DATA_DWORDS, sizeof(u32), GFP_KERNEL);
-	if (!dma->buf) {
-		kfree(dma);
-		return NULL;
-	}
-
 	dma->sw = sw;
 	dma->port = port;
 	dma->base = DMA_PORT_CAP;
@@ -232,10 +226,7 @@ struct tb_dma_port *dma_port_alloc(struct tb_switch *sw)
  */
 void dma_port_free(struct tb_dma_port *dma)
 {
-	if (dma) {
-		kfree(dma->buf);
-		kfree(dma);
-	}
+	kfree(dma);
 }
 
 static int dma_port_wait_for_completion(struct tb_dma_port *dma,
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index ccce020a2432f..2bb2e79ca3cb3 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -1020,7 +1020,7 @@ static bool nhi_wake_supported(struct pci_dev *pdev)
 	 * If power rails are sustainable for wakeup from S4 this
 	 * property is set by the BIOS.
 	 */
-	if (device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val))
+	if (!device_property_read_u8(&pdev->dev, "WAKE_SUPPORTED", &val))
 		return !!val;
 
 	return true;
diff --git a/drivers/thunderbolt/quirks.c b/drivers/thunderbolt/quirks.c
index e81de9c30eac9..9f7914ac2f48c 100644
--- a/drivers/thunderbolt/quirks.c
+++ b/drivers/thunderbolt/quirks.c
@@ -23,6 +23,9 @@ static void quirk_dp_credit_allocation(struct tb_switch *sw)
 
 static void quirk_clx_disable(struct tb_switch *sw)
 {
+	if (tb_switch_is_titan_ridge(sw) && sw->nvm && sw->nvm->major >= 0x65)
+		return;
+
 	sw->quirks |= QUIRK_NO_CLX;
 	tb_sw_dbg(sw, "disabling CL states\n");
 }
@@ -61,6 +64,10 @@ static const struct tb_quirk tb_quirks[] = {
 	/* Dell WD19TB supports self-authentication on unplug */
 	{ 0x0000, 0x0000, 0x00d4, 0xb070, quirk_force_power_link },
 	{ 0x0000, 0x0000, 0x00d4, 0xb071, quirk_force_power_link },
+
+	/* Intel Titan Ridge CLx is unstable on early firmware versions */
+	{ 0x8086, PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_BRIDGE, 0x0000, 0x0000,
+		  quirk_clx_disable },
 	/*
 	 * Intel Goshen Ridge NVM 27 and before report wrong number of
 	 * DP buffers.
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index e5b48a331c583..c2ad58b19e7b1 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -347,7 +347,7 @@ static int nvm_write(void *priv, unsigned int offset, void *val, size_t bytes)
 	return ret;
 }
 
-static int tb_switch_nvm_add(struct tb_switch *sw)
+static int tb_switch_nvm_init(struct tb_switch *sw)
 {
 	struct tb_nvm *nvm;
 	int ret;
@@ -365,6 +365,26 @@ static int tb_switch_nvm_add(struct tb_switch *sw)
 	if (ret)
 		goto err_nvm;
 
+	sw->nvm = nvm;
+	return 0;
+
+err_nvm:
+	tb_sw_dbg(sw, "NVM upgrade disabled\n");
+	sw->no_nvm_upgrade = true;
+	if (!IS_ERR(nvm))
+		tb_nvm_free(nvm);
+
+	return ret;
+}
+
+static int tb_switch_nvm_add(struct tb_switch *sw)
+{
+	struct tb_nvm *nvm = sw->nvm;
+	int ret;
+
+	if (!nvm)
+		return 0;
+
 	/*
 	 * If the switch is in safe-mode the only accessible portion of
 	 * the NVM is the non-active one where userspace is expected to
@@ -383,14 +403,12 @@ static int tb_switch_nvm_add(struct tb_switch *sw)
 			goto err_nvm;
 	}
 
-	sw->nvm = nvm;
 	return 0;
 
 err_nvm:
 	tb_sw_dbg(sw, "NVM upgrade disabled\n");
 	sw->no_nvm_upgrade = true;
-	if (!IS_ERR(nvm))
-		tb_nvm_free(nvm);
+	tb_nvm_free(nvm);
 
 	return ret;
 }
@@ -3311,6 +3329,10 @@ int tb_switch_add(struct tb_switch *sw)
 		return ret;
 	}
 
+	ret = tb_switch_nvm_init(sw);
+	if (ret)
+		return ret;
+
 	if (!sw->safe_mode) {
 		tb_switch_credits_init(sw);
 
diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c
index 1dcdb9e99bd8a..37db8a3e5158e 100644
--- a/drivers/tty/hvc/hvc_iucv.c
+++ b/drivers/tty/hvc/hvc_iucv.c
@@ -130,7 +130,7 @@ static struct iucv_handler hvc_iucv_handler = {
  */
 static struct hvc_iucv_private *hvc_iucv_get_private(uint32_t num)
 {
-	if (num > hvc_iucv_devices)
+	if (num >= hvc_iucv_devices)
 		return NULL;
 	return hvc_iucv_table[num];
 }
diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c
index cb427e93372da..cc7f7091ed9a2 100644
--- a/drivers/tty/pty.c
+++ b/drivers/tty/pty.c
@@ -532,14 +532,16 @@ static void __init legacy_pty_init(void)
 	pty_driver = tty_alloc_driver(legacy_count,
 			TTY_DRIVER_RESET_TERMIOS |
 			TTY_DRIVER_REAL_RAW |
-			TTY_DRIVER_DYNAMIC_ALLOC);
+			TTY_DRIVER_DYNAMIC_ALLOC |
+			TTY_DRIVER_NO_WORKQUEUE);
 	if (IS_ERR(pty_driver))
 		panic("Couldn't allocate pty driver");
 
 	pty_slave_driver = tty_alloc_driver(legacy_count,
 			TTY_DRIVER_RESET_TERMIOS |
 			TTY_DRIVER_REAL_RAW |
-			TTY_DRIVER_DYNAMIC_ALLOC);
+			TTY_DRIVER_DYNAMIC_ALLOC |
+			TTY_DRIVER_NO_WORKQUEUE);
 	if (IS_ERR(pty_slave_driver))
 		panic("Couldn't allocate pty slave driver");
 
@@ -849,7 +851,8 @@ static void __init unix98_pty_init(void)
 			TTY_DRIVER_REAL_RAW |
 			TTY_DRIVER_DYNAMIC_DEV |
 			TTY_DRIVER_DEVPTS_MEM |
-			TTY_DRIVER_DYNAMIC_ALLOC);
+			TTY_DRIVER_DYNAMIC_ALLOC |
+			TTY_DRIVER_NO_WORKQUEUE);
 	if (IS_ERR(ptm_driver))
 		panic("Couldn't allocate Unix98 ptm driver");
 	pts_driver = tty_alloc_driver(NR_UNIX98_PTY_MAX,
@@ -857,7 +860,8 @@ static void __init unix98_pty_init(void)
 			TTY_DRIVER_REAL_RAW |
 			TTY_DRIVER_DYNAMIC_DEV |
 			TTY_DRIVER_DEVPTS_MEM |
-			TTY_DRIVER_DYNAMIC_ALLOC);
+			TTY_DRIVER_DYNAMIC_ALLOC |
+			TTY_DRIVER_NO_WORKQUEUE);
 	if (IS_ERR(pts_driver))
 		panic("Couldn't allocate Unix98 pts driver");
 
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index 8caecfc85d933..77fe0588fd6bd 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -175,7 +175,9 @@ static unsigned int __maybe_unused serial_icr_read(struct uart_8250_port *up,
 	return value;
 }
 
+void serial8250_clear_fifos(struct uart_8250_port *p);
 void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p);
+void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count);
 
 void serial8250_rpm_get(struct uart_8250_port *p);
 void serial8250_rpm_put(struct uart_8250_port *p);
@@ -400,6 +402,26 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p)
 
 	return dma && dma->tx_running;
 }
+
+static inline void serial8250_tx_dma_pause(struct uart_8250_port *p)
+{
+	struct uart_8250_dma *dma = p->dma;
+
+	if (!dma->tx_running)
+		return;
+
+	dmaengine_pause(dma->txchan);
+}
+
+static inline void serial8250_tx_dma_resume(struct uart_8250_port *p)
+{
+	struct uart_8250_dma *dma = p->dma;
+
+	if (!dma->tx_running)
+		return;
+
+	dmaengine_resume(dma->txchan);
+}
 #else
 static inline int serial8250_tx_dma(struct uart_8250_port *p)
 {
@@ -421,6 +443,9 @@ static inline bool serial8250_tx_dma_running(struct uart_8250_port *p)
 {
 	return false;
 }
+
+static inline void serial8250_tx_dma_pause(struct uart_8250_port *p) { }
+static inline void serial8250_tx_dma_resume(struct uart_8250_port *p) { }
 #endif
 
 static inline int ns16550a_goto_highspeed(struct uart_8250_port *up)
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index d2e2c5dfef995..a428e88938eb7 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -524,7 +524,7 @@ console_initcall(univ8250_console_init);
 
 struct uart_driver serial8250_reg = {
 	.owner			= THIS_MODULE,
-	.driver_name		= "serial",
+	.driver_name		= "serial_8250",
 	.dev_name		= "ttyS",
 	.major			= TTY_MAJOR,
 	.minor			= 64,
diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c
index bdd26c9f34bdf..3b6452e759d5b 100644
--- a/drivers/tty/serial/8250/8250_dma.c
+++ b/drivers/tty/serial/8250/8250_dma.c
@@ -162,7 +162,22 @@ void serial8250_tx_dma_flush(struct uart_8250_port *p)
 	 */
 	dma->tx_size = 0;
 
+	/*
+	 * We can't use `dmaengine_terminate_sync` because `uart_flush_buffer` is
+	 * holding the uart port spinlock.
+	 */
 	dmaengine_terminate_async(dma->txchan);
+
+	/*
+	 * The callback might or might not run. If it doesn't run, we need to ensure
+	 * that `tx_running` is cleared so that we can schedule new transactions.
+	 * If it does run, then the zombie callback will clear `tx_running` again
+	 * and perform a no-op since `tx_size` was cleared above.
+	 *
+	 * In either case, we ASSUME the DMA transaction will terminate before we
+	 * issue a new `serial8250_tx_dma`.
+	 */
+	dma->tx_running = 0;
 }
 
 int serial8250_rx_dma(struct uart_8250_port *p)
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index db73b2ae17fa3..94beadb4024df 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -9,10 +9,14 @@
  * LCR is written whilst busy.  If it is, then a busy detect interrupt is
  * raised, the LCR needs to be rewritten and the uart status register read.
  */
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/lockdep.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/notifier.h>
@@ -40,8 +44,12 @@
 #define RZN1_UART_RDMACR 0x110 /* DMA Control Register Receive Mode */
 
 /* DesignWare specific register fields */
+#define DW_UART_IIR_IID			GENMASK(3, 0)
+
 #define DW_UART_MCR_SIRE		BIT(6)
 
+#define DW_UART_USR_BUSY		BIT(0)
+
 /* Renesas specific register fields */
 #define RZN1_UART_xDMACR_DMA_EN		BIT(0)
 #define RZN1_UART_xDMACR_1_WORD_BURST	(0 << 1)
@@ -56,6 +64,13 @@
 #define DW_UART_QUIRK_IS_DMA_FC		BIT(3)
 #define DW_UART_QUIRK_APMC0D08		BIT(4)
 #define DW_UART_QUIRK_CPR_VALUE		BIT(5)
+#define DW_UART_QUIRK_IER_KICK		BIT(6)
+
+/*
+ * Number of consecutive IIR_NO_INT interrupts required to trigger interrupt
+ * storm prevention code.
+ */
+#define DW_UART_QUIRK_IER_KICK_THRES	4
 
 struct dw8250_platform_data {
 	u8 usr_reg;
@@ -77,6 +92,9 @@ struct dw8250_data {
 
 	unsigned int		skip_autocfg:1;
 	unsigned int		uart_16550_compatible:1;
+	unsigned int		in_idle:1;
+
+	u8			no_int_count;
 };
 
 static inline struct dw8250_data *to_dw8250_data(struct dw8250_port_data *data)
@@ -107,78 +125,167 @@ static inline u32 dw8250_modify_msr(struct uart_port *p, unsigned int offset, u3
 	return value;
 }
 
+static void dw8250_idle_exit(struct uart_port *p)
+{
+	struct dw8250_data *d = to_dw8250_data(p->private_data);
+	struct uart_8250_port *up = up_to_u8250p(p);
+
+	if (d->uart_16550_compatible)
+		return;
+
+	if (up->capabilities & UART_CAP_FIFO)
+		serial_port_out(p, UART_FCR, up->fcr);
+	serial_port_out(p, UART_MCR, up->mcr);
+	serial_port_out(p, UART_IER, up->ier);
+
+	/* DMA Rx is restarted by IRQ handler as needed. */
+	if (up->dma)
+		serial8250_tx_dma_resume(up);
+
+	d->in_idle = 0;
+}
+
 /*
- * This function is being called as part of the uart_port::serial_out()
- * routine. Hence, it must not call serial_port_out() or serial_out()
- * against the modified registers here, i.e. LCR.
+ * Ensure BUSY is not asserted. If DW UART is configured with
+ * !uart_16550_compatible, the writes to LCR, DLL, and DLH fail while
+ * BUSY is asserted.
+ *
+ * Context: port's lock must be held
  */
-static void dw8250_force_idle(struct uart_port *p)
+static int dw8250_idle_enter(struct uart_port *p)
 {
+	struct dw8250_data *d = to_dw8250_data(p->private_data);
+	unsigned int usr_reg = d->pdata ? d->pdata->usr_reg : DW_UART_USR;
 	struct uart_8250_port *up = up_to_u8250p(p);
-	unsigned int lsr;
+	int retries;
+	u32 lsr;
 
-	/*
-	 * The following call currently performs serial_out()
-	 * against the FCR register. Because it differs to LCR
-	 * there will be no infinite loop, but if it ever gets
-	 * modified, we might need a new custom version of it
-	 * that avoids infinite recursion.
-	 */
-	serial8250_clear_and_reinit_fifos(up);
+	lockdep_assert_held_once(&p->lock);
+
+	if (d->uart_16550_compatible)
+		return 0;
+
+	d->in_idle = 1;
+
+	/* Prevent triggering interrupt from RBR filling */
+	serial_port_out(p, UART_IER, 0);
+
+	if (up->dma) {
+		serial8250_rx_dma_flush(up);
+		if (serial8250_tx_dma_running(up))
+			serial8250_tx_dma_pause(up);
+	}
 
 	/*
-	 * With PSLVERR_RESP_EN parameter set to 1, the device generates an
-	 * error response when an attempt to read an empty RBR with FIFO
-	 * enabled.
+	 * Wait until Tx becomes empty + one extra frame time to ensure all bits
+	 * have been sent on the wire.
+	 *
+	 * FIXME: frame_time delay is too long with very low baudrates.
 	 */
-	if (up->fcr & UART_FCR_ENABLE_FIFO) {
-		lsr = serial_port_in(p, UART_LSR);
-		if (!(lsr & UART_LSR_DR))
-			return;
+	serial8250_fifo_wait_for_lsr_thre(up, p->fifosize);
+	ndelay(p->frame_time);
+
+	serial_port_out(p, UART_MCR, up->mcr | UART_MCR_LOOP);
+
+	retries = 4;	/* Arbitrary limit, 2 was always enough in tests */
+	do {
+		serial8250_clear_fifos(up);
+		if (!(serial_port_in(p, usr_reg) & DW_UART_USR_BUSY))
+			break;
+		/* FIXME: frame_time delay is too long with very low baudrates. */
+		ndelay(p->frame_time);
+	} while (--retries);
+
+	lsr = serial_lsr_in(up);
+	if (lsr & UART_LSR_DR) {
+		serial_port_in(p, UART_RX);
+		up->lsr_saved_flags = 0;
+	}
+
+	/* Now guaranteed to have BUSY deasserted? Just sanity check */
+	if (serial_port_in(p, usr_reg) & DW_UART_USR_BUSY) {
+		dw8250_idle_exit(p);
+		return -EBUSY;
 	}
 
-	serial_port_in(p, UART_RX);
+	return 0;
+}
+
+static void dw8250_set_divisor(struct uart_port *p, unsigned int baud,
+			       unsigned int quot, unsigned int quot_frac)
+{
+	struct uart_8250_port *up = up_to_u8250p(p);
+	int ret;
+
+	ret = dw8250_idle_enter(p);
+	if (ret < 0)
+		return;
+
+	serial_port_out(p, UART_LCR, up->lcr | UART_LCR_DLAB);
+	if (!(serial_port_in(p, UART_LCR) & UART_LCR_DLAB))
+		goto idle_failed;
+
+	serial_dl_write(up, quot);
+	serial_port_out(p, UART_LCR, up->lcr);
+
+idle_failed:
+	dw8250_idle_exit(p);
 }
 
 /*
  * This function is being called as part of the uart_port::serial_out()
- * routine. Hence, it must not call serial_port_out() or serial_out()
- * against the modified registers here, i.e. LCR.
+ * routine. Hence, special care must be taken when serial_port_out() or
+ * serial_out() against the modified registers here, i.e. LCR (d->in_idle is
+ * used to break recursion loop).
  */
 static void dw8250_check_lcr(struct uart_port *p, unsigned int offset, u32 value)
 {
 	struct dw8250_data *d = to_dw8250_data(p->private_data);
-	void __iomem *addr = p->membase + (offset << p->regshift);
-	int tries = 1000;
+	u32 lcr;
+	int ret;
 
 	if (offset != UART_LCR || d->uart_16550_compatible)
 		return;
 
+	lcr = serial_port_in(p, UART_LCR);
+
 	/* Make sure LCR write wasn't ignored */
-	while (tries--) {
-		u32 lcr = serial_port_in(p, offset);
+	if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR))
+		return;
 
-		if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR))
-			return;
+	if (d->in_idle)
+		goto write_err;
 
-		dw8250_force_idle(p);
+	ret = dw8250_idle_enter(p);
+	if (ret < 0)
+		goto write_err;
 
-#ifdef CONFIG_64BIT
-		if (p->type == PORT_OCTEON)
-			__raw_writeq(value & 0xff, addr);
-		else
-#endif
-		if (p->iotype == UPIO_MEM32)
-			writel(value, addr);
-		else if (p->iotype == UPIO_MEM32BE)
-			iowrite32be(value, addr);
-		else
-			writeb(value, addr);
-	}
+	serial_port_out(p, UART_LCR, value);
+	dw8250_idle_exit(p);
+	return;
+
+write_err:
 	/*
 	 * FIXME: this deadlocks if port->lock is already held
 	 * dev_err(p->dev, "Couldn't set LCR to %d\n", value);
 	 */
+	return;		/* Silences "label at the end of compound statement" */
+}
+
+/*
+ * With BUSY, LCR writes can be very expensive (IRQ + complex retry logic).
+ * If the write does not change the value of the LCR register, skip it entirely.
+ */
+static bool dw8250_can_skip_reg_write(struct uart_port *p, unsigned int offset, u32 value)
+{
+	struct dw8250_data *d = to_dw8250_data(p->private_data);
+	u32 lcr;
+
+	if (offset != UART_LCR || d->uart_16550_compatible)
+		return false;
+
+	lcr = serial_port_in(p, offset);
+	return lcr == value;
 }
 
 /* Returns once the transmitter is empty or we run out of retries */
@@ -207,12 +314,18 @@ static void dw8250_tx_wait_empty(struct uart_port *p)
 
 static void dw8250_serial_out(struct uart_port *p, unsigned int offset, u32 value)
 {
+	if (dw8250_can_skip_reg_write(p, offset, value))
+		return;
+
 	writeb(value, p->membase + (offset << p->regshift));
 	dw8250_check_lcr(p, offset, value);
 }
 
 static void dw8250_serial_out38x(struct uart_port *p, unsigned int offset, u32 value)
 {
+	if (dw8250_can_skip_reg_write(p, offset, value))
+		return;
+
 	/* Allow the TX to drain before we reconfigure */
 	if (offset == UART_LCR)
 		dw8250_tx_wait_empty(p);
@@ -237,6 +350,9 @@ static u32 dw8250_serial_inq(struct uart_port *p, unsigned int offset)
 
 static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 value)
 {
+	if (dw8250_can_skip_reg_write(p, offset, value))
+		return;
+
 	value &= 0xff;
 	__raw_writeq(value, p->membase + (offset << p->regshift));
 	/* Read back to ensure register write ordering. */
@@ -248,6 +364,9 @@ static void dw8250_serial_outq(struct uart_port *p, unsigned int offset, u32 val
 
 static void dw8250_serial_out32(struct uart_port *p, unsigned int offset, u32 value)
 {
+	if (dw8250_can_skip_reg_write(p, offset, value))
+		return;
+
 	writel(value, p->membase + (offset << p->regshift));
 	dw8250_check_lcr(p, offset, value);
 }
@@ -261,6 +380,9 @@ static u32 dw8250_serial_in32(struct uart_port *p, unsigned int offset)
 
 static void dw8250_serial_out32be(struct uart_port *p, unsigned int offset, u32 value)
 {
+	if (dw8250_can_skip_reg_write(p, offset, value))
+		return;
+
 	iowrite32be(value, p->membase + (offset << p->regshift));
 	dw8250_check_lcr(p, offset, value);
 }
@@ -272,6 +394,29 @@ static u32 dw8250_serial_in32be(struct uart_port *p, unsigned int offset)
        return dw8250_modify_msr(p, offset, value);
 }
 
+/*
+ * INTC10EE UART can IRQ storm while reporting IIR_NO_INT. Inducing IIR value
+ * change has been observed to break the storm.
+ *
+ * If Tx is empty (THRE asserted), we use here IER_THRI to cause IIR_NO_INT ->
+ * IIR_THRI transition.
+ */
+static void dw8250_quirk_ier_kick(struct uart_port *p)
+{
+	struct uart_8250_port *up = up_to_u8250p(p);
+	u32 lsr;
+
+	if (up->ier & UART_IER_THRI)
+		return;
+
+	lsr = serial_lsr_in(up);
+	if (!(lsr & UART_LSR_THRE))
+		return;
+
+	serial_port_out(p, UART_IER, up->ier | UART_IER_THRI);
+	serial_port_in(p, UART_LCR);		/* safe, no side-effects */
+	serial_port_out(p, UART_IER, up->ier);
+}
 
 static int dw8250_handle_irq(struct uart_port *p)
 {
@@ -281,7 +426,31 @@ static int dw8250_handle_irq(struct uart_port *p)
 	bool rx_timeout = (iir & 0x3f) == UART_IIR_RX_TIMEOUT;
 	unsigned int quirks = d->pdata->quirks;
 	unsigned int status;
-	unsigned long flags;
+
+	guard(uart_port_lock_irqsave)(p);
+
+	switch (FIELD_GET(DW_UART_IIR_IID, iir)) {
+	case UART_IIR_NO_INT:
+		if (d->uart_16550_compatible || up->dma)
+			return 0;
+
+		if (quirks & DW_UART_QUIRK_IER_KICK &&
+		    d->no_int_count == (DW_UART_QUIRK_IER_KICK_THRES - 1))
+			dw8250_quirk_ier_kick(p);
+		d->no_int_count = (d->no_int_count + 1) % DW_UART_QUIRK_IER_KICK_THRES;
+
+		return 0;
+
+	case UART_IIR_BUSY:
+		/* Clear the USR */
+		serial_port_in(p, d->pdata->usr_reg);
+
+		d->no_int_count = 0;
+
+		return 1;
+	}
+
+	d->no_int_count = 0;
 
 	/*
 	 * There are ways to get Designware-based UARTs into a state where
@@ -294,20 +463,15 @@ static int dw8250_handle_irq(struct uart_port *p)
 	 * so we limit the workaround only to non-DMA mode.
 	 */
 	if (!up->dma && rx_timeout) {
-		uart_port_lock_irqsave(p, &flags);
 		status = serial_lsr_in(up);
 
 		if (!(status & (UART_LSR_DR | UART_LSR_BI)))
 			serial_port_in(p, UART_RX);
-
-		uart_port_unlock_irqrestore(p, flags);
 	}
 
 	/* Manually stop the Rx DMA transfer when acting as flow controller */
 	if (quirks & DW_UART_QUIRK_IS_DMA_FC && up->dma && up->dma->rx_running && rx_timeout) {
-		uart_port_lock_irqsave(p, &flags);
 		status = serial_lsr_in(up);
-		uart_port_unlock_irqrestore(p, flags);
 
 		if (status & (UART_LSR_DR | UART_LSR_BI)) {
 			dw8250_writel_ext(p, RZN1_UART_RDMACR, 0);
@@ -315,17 +479,9 @@ static int dw8250_handle_irq(struct uart_port *p)
 		}
 	}
 
-	if (serial8250_handle_irq(p, iir))
-		return 1;
-
-	if ((iir & UART_IIR_BUSY) == UART_IIR_BUSY) {
-		/* Clear the USR */
-		serial_port_in(p, d->pdata->usr_reg);
-
-		return 1;
-	}
+	serial8250_handle_irq_locked(p, iir);
 
-	return 0;
+	return 1;
 }
 
 static void dw8250_clk_work_cb(struct work_struct *work)
@@ -527,6 +683,14 @@ static void dw8250_reset_control_assert(void *data)
 	reset_control_assert(data);
 }
 
+static void dw8250_shutdown(struct uart_port *port)
+{
+	struct dw8250_data *d = to_dw8250_data(port->private_data);
+
+	serial8250_do_shutdown(port);
+	d->no_int_count = 0;
+}
+
 static int dw8250_probe(struct platform_device *pdev)
 {
 	struct uart_8250_port uart = {}, *up = &uart;
@@ -545,8 +709,10 @@ static int dw8250_probe(struct platform_device *pdev)
 	p->type		= PORT_8250;
 	p->flags	= UPF_FIXED_PORT;
 	p->dev		= dev;
+
 	p->set_ldisc	= dw8250_set_ldisc;
 	p->set_termios	= dw8250_set_termios;
+	p->set_divisor	= dw8250_set_divisor;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
@@ -654,10 +820,12 @@ static int dw8250_probe(struct platform_device *pdev)
 		dw8250_quirks(p, data);
 
 	/* If the Busy Functionality is not implemented, don't handle it */
-	if (data->uart_16550_compatible)
+	if (data->uart_16550_compatible) {
 		p->handle_irq = NULL;
-	else if (data->pdata)
+	} else if (data->pdata) {
 		p->handle_irq = dw8250_handle_irq;
+		p->shutdown = dw8250_shutdown;
+	}
 
 	dw8250_setup_dma_filter(p, data);
 
@@ -789,6 +957,11 @@ static const struct dw8250_platform_data dw8250_skip_set_rate_data = {
 	.quirks = DW_UART_QUIRK_SKIP_SET_RATE,
 };
 
+static const struct dw8250_platform_data dw8250_intc10ee = {
+	.usr_reg = DW_UART_USR,
+	.quirks = DW_UART_QUIRK_IER_KICK,
+};
+
 static const struct of_device_id dw8250_of_match[] = {
 	{ .compatible = "snps,dw-apb-uart", .data = &dw8250_dw_apb },
 	{ .compatible = "cavium,octeon-3860-uart", .data = &dw8250_octeon_3860_data },
@@ -818,7 +991,7 @@ static const struct acpi_device_id dw8250_acpi_match[] = {
 	{ "INT33C5", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "INT3434", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "INT3435", (kernel_ulong_t)&dw8250_dw_apb },
-	{ "INTC10EE", (kernel_ulong_t)&dw8250_dw_apb },
+	{ "INTC10EE", (kernel_ulong_t)&dw8250_intc10ee },
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, dw8250_acpi_match);
@@ -836,6 +1009,7 @@ static struct platform_driver dw8250_platform_driver = {
 
 module_platform_driver(dw8250_platform_driver);
 
+MODULE_IMPORT_NS("SERIAL_8250");
 MODULE_AUTHOR("Jamie Iles");
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Synopsys DesignWare 8250 serial port driver");
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index aa1ab4da9ff1b..2fbd8f2603b5f 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -100,6 +100,8 @@
 #define PCI_DEVICE_ID_ADDIDATA_CPCI7420_NG     0x7025
 #define PCI_DEVICE_ID_ADDIDATA_CPCI7300_NG     0x7026
 
+#define PCI_VENDOR_ID_SYSTEMBASE	0x14a1
+
 /* Unknown vendors/cards - this should not be in linux/pci_ids.h */
 #define PCI_SUBDEVICE_ID_UNKNOWN_0x1584	0x1584
 #define PCI_SUBDEVICE_ID_UNKNOWN_0x1588	0x1588
@@ -137,6 +139,8 @@ struct serial_private {
 };
 
 #define PCI_DEVICE_ID_HPE_PCI_SERIAL	0x37e
+#define PCIE_VENDOR_ID_ASIX		0x125B
+#define PCIE_DEVICE_ID_AX99100		0x9100
 
 static const struct pci_device_id pci_use_msi[] = {
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900,
@@ -149,6 +153,8 @@ static const struct pci_device_id pci_use_msi[] = {
 			 0xA000, 0x1000) },
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL,
 			 PCI_ANY_ID, PCI_ANY_ID) },
+	{ PCI_DEVICE_SUB(PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100,
+			 0xA000, 0x1000) },
 	{ }
 };
 
@@ -920,6 +926,7 @@ static int pci_netmos_init(struct pci_dev *dev)
 	case PCI_DEVICE_ID_NETMOS_9912:
 	case PCI_DEVICE_ID_NETMOS_9922:
 	case PCI_DEVICE_ID_NETMOS_9900:
+	case PCIE_DEVICE_ID_AX99100:
 		num_serial = pci_netmos_9900_numports(dev);
 		break;
 
@@ -2128,6 +2135,35 @@ pci_moxa_setup(struct serial_private *priv,
 	return setup_port(priv, port, bar, offset, 0);
 }
 
+#define SB_OPTR_IMR0	0x0c /* Interrupt mask register, p0 to p7 */
+static int pci_systembase_init(struct pci_dev *dev)
+{
+	resource_size_t iobase;
+
+	if (!IS_ENABLED(CONFIG_HAS_IOPORT))
+		return serial_8250_warn_need_ioport(dev);
+
+	iobase = pci_resource_start(dev, 1);
+
+	/* This will support up to 8 ports */
+	outb(0xff, iobase + SB_OPTR_IMR0);
+
+	return 0;
+}
+
+static void pci_systembase_exit(struct pci_dev *dev)
+{
+	resource_size_t iobase;
+
+	if (!IS_ENABLED(CONFIG_HAS_IOPORT)) {
+		serial_8250_warn_need_ioport(dev);
+		return;
+	}
+
+	iobase = pci_resource_start(dev, 0);
+	outb(0x00, iobase + SB_OPTR_IMR0);
+}
+
 /*
  * Master list of serial port init/setup/exit quirks.
  * This does not describe the general nature of the port.
@@ -2476,6 +2512,16 @@ static struct pci_serial_quirk pci_serial_quirks[] = {
 		.init		= pci_siig_init,
 		.setup		= pci_siig_setup,
 	},
+	/* Systembase */
+	{
+		.vendor		= PCI_VENDOR_ID_SYSTEMBASE,
+		.device		= 0x0008,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.init		= pci_systembase_init,
+		.setup		= pci_default_setup,
+		.exit		= pci_systembase_exit,
+	},
 	/*
 	 * Titan cards
 	 */
@@ -2544,6 +2590,14 @@ static struct pci_serial_quirk pci_serial_quirks[] = {
 		.init		= pci_netmos_init,
 		.setup		= pci_netmos_9900_setup,
 	},
+	{
+		.vendor		= PCIE_VENDOR_ID_ASIX,
+		.device		= PCI_ANY_ID,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.init		= pci_netmos_init,
+		.setup		= pci_netmos_9900_setup,
+	},
 	/*
 	 * EndRun Technologies
 	*/
@@ -3041,6 +3095,7 @@ enum pci_board_num_t {
 	pbn_b0_1_921600,
 	pbn_b0_2_921600,
 	pbn_b0_4_921600,
+	pbn_b0_8_921600,
 
 	pbn_b0_2_1130000,
 
@@ -3241,6 +3296,12 @@ static struct pciserial_board pci_boards[] = {
 		.base_baud	= 921600,
 		.uart_offset	= 8,
 	},
+	[pbn_b0_8_921600] = {
+		.flags		= FL_BASE0,
+		.num_ports	= 8,
+		.base_baud	= 921600,
+		.uart_offset	= 8,
+	},
 
 	[pbn_b0_2_1130000] = {
 		.flags          = FL_BASE0,
@@ -6065,6 +6126,10 @@ static const struct pci_device_id serial_pci_tbl[] = {
 		0xA000, 0x3002,
 		0, 0, pbn_NETMOS9900_2s_115200 },
 
+	{	PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100,
+		0xA000, 0x1000,
+		0, 0, pbn_b0_1_115200 },
+
 	/*
 	 * Best Connectivity and Rosewill PCI Multi I/O cards
 	 */
@@ -6152,6 +6217,9 @@ static const struct pci_device_id serial_pci_tbl[] = {
 		PCI_ANY_ID, PCI_ANY_ID,
 		0, 0, pbn_b0_1_115200 },
 
+	/* Systembase Multi I/O cards */
+	{ PCI_VDEVICE(SYSTEMBASE, 0x0008), pbn_b0_8_921600 },
+
 	/* Fintek PCI serial cards */
 	{ PCI_DEVICE(0x1c29, 0x1104), .driver_data = pbn_fintek_4 },
 	{ PCI_DEVICE(0x1c29, 0x1108), .driver_data = pbn_fintek_8 },
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index cc94af2d578a6..af78cc02f38e7 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -18,6 +18,7 @@
 #include <linux/irq.h>
 #include <linux/console.h>
 #include <linux/gpio/consumer.h>
+#include <linux/lockdep.h>
 #include <linux/sysrq.h>
 #include <linux/delay.h>
 #include <linux/platform_device.h>
@@ -488,7 +489,7 @@ serial_port_out_sync(struct uart_port *p, int offset, int value)
 /*
  * FIFO support.
  */
-static void serial8250_clear_fifos(struct uart_8250_port *p)
+void serial8250_clear_fifos(struct uart_8250_port *p)
 {
 	if (p->capabilities & UART_CAP_FIFO) {
 		serial_out(p, UART_FCR, UART_FCR_ENABLE_FIFO);
@@ -497,6 +498,7 @@ static void serial8250_clear_fifos(struct uart_8250_port *p)
 		serial_out(p, UART_FCR, 0);
 	}
 }
+EXPORT_SYMBOL_NS_GPL(serial8250_clear_fifos, "SERIAL_8250");
 
 static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t);
 static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t);
@@ -1782,20 +1784,16 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
 }
 
 /*
- * This handles the interrupt from one port.
+ * Context: port's lock must be held by the caller.
  */
-int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
+void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
 	struct tty_port *tport = &port->state->port;
 	bool skip_rx = false;
-	unsigned long flags;
 	u16 status;
 
-	if (iir & UART_IIR_NO_INT)
-		return 0;
-
-	uart_port_lock_irqsave(port, &flags);
+	lockdep_assert_held_once(&port->lock);
 
 	status = serial_lsr_in(up);
 
@@ -1828,8 +1826,19 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
 		else if (!up->dma->tx_running)
 			__stop_tx(up);
 	}
+}
+EXPORT_SYMBOL_NS_GPL(serial8250_handle_irq_locked, "SERIAL_8250");
+
+/*
+ * This handles the interrupt from one port.
+ */
+int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
+{
+	if (iir & UART_IIR_NO_INT)
+		return 0;
 
-	uart_unlock_and_check_sysrq_irqrestore(port, flags);
+	guard(uart_port_lock_irqsave)(port);
+	serial8250_handle_irq_locked(port, iir);
 
 	return 1;
 }
@@ -2147,8 +2156,7 @@ static void serial8250_THRE_test(struct uart_port *port)
 	if (up->port.flags & UPF_NO_THRE_TEST)
 		return;
 
-	if (port->irqflags & IRQF_SHARED)
-		disable_irq_nosync(port->irq);
+	disable_irq(port->irq);
 
 	/*
 	 * Test for UARTs that do not reassert THRE when the transmitter is idle and the interrupt
@@ -2170,8 +2178,7 @@ static void serial8250_THRE_test(struct uart_port *port)
 		serial_port_out(port, UART_IER, 0);
 	}
 
-	if (port->irqflags & IRQF_SHARED)
-		enable_irq(port->irq);
+	enable_irq(port->irq);
 
 	/*
 	 * If the interrupt is not reasserted, or we otherwise don't trust the iir, setup a timer to
@@ -2350,6 +2357,7 @@ static int serial8250_startup(struct uart_port *port)
 void serial8250_do_shutdown(struct uart_port *port)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
+	u32 lcr;
 
 	serial8250_rpm_get(up);
 	/*
@@ -2364,8 +2372,8 @@ void serial8250_do_shutdown(struct uart_port *port)
 
 	synchronize_irq(port->irq);
 
-	if (up->dma)
-		serial8250_release_dma(up);
+	serial8250_release_dma(up);
+	up->dma = NULL;
 
 	scoped_guard(uart_port_lock_irqsave, port) {
 		if (port->flags & UPF_FOURPORT) {
@@ -2376,13 +2384,13 @@ void serial8250_do_shutdown(struct uart_port *port)
 			port->mctrl &= ~TIOCM_OUT2;
 
 		serial8250_set_mctrl(port, port->mctrl);
+
+		/* Disable break condition */
+		lcr = serial_port_in(port, UART_LCR);
+		lcr &= ~UART_LCR_SBC;
+		serial_port_out(port, UART_LCR, lcr);
 	}
 
-	/*
-	 * Disable break condition and FIFOs
-	 */
-	serial_port_out(port, UART_LCR,
-			serial_port_in(port, UART_LCR) & ~UART_LCR_SBC);
 	serial8250_clear_fifos(up);
 
 	rsa_disable(up);
@@ -2392,6 +2400,12 @@ void serial8250_do_shutdown(struct uart_port *port)
 	 * the IRQ chain.
 	 */
 	serial_port_in(port, UART_RX);
+	/*
+	 * LCR writes on DW UART can trigger late (unmaskable) IRQs.
+	 * Handle them before releasing the handler.
+	 */
+	synchronize_irq(port->irq);
+
 	serial8250_rpm_put(up);
 
 	up->ops->release_irq(up);
@@ -3185,6 +3199,17 @@ void serial8250_set_defaults(struct uart_8250_port *up)
 }
 EXPORT_SYMBOL_GPL(serial8250_set_defaults);
 
+void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up, unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++) {
+		if (wait_for_lsr(up, UART_LSR_THRE))
+			return;
+	}
+}
+EXPORT_SYMBOL_NS_GPL(serial8250_fifo_wait_for_lsr_thre, "SERIAL_8250");
+
 #ifdef CONFIG_SERIAL_8250_CONSOLE
 
 static void serial8250_console_putchar(struct uart_port *port, unsigned char ch)
@@ -3226,16 +3251,6 @@ static void serial8250_console_restore(struct uart_8250_port *up)
 	serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS);
 }
 
-static void fifo_wait_for_lsr(struct uart_8250_port *up, unsigned int count)
-{
-	unsigned int i;
-
-	for (i = 0; i < count; i++) {
-		if (wait_for_lsr(up, UART_LSR_THRE))
-			return;
-	}
-}
-
 /*
  * Print a string to the serial port using the device FIFO
  *
@@ -3254,7 +3269,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up,
 
 	while (s != end) {
 		/* Allow timeout for each byte of a possibly full FIFO */
-		fifo_wait_for_lsr(up, fifosize);
+		serial8250_fifo_wait_for_lsr_thre(up, fifosize);
 
 		for (i = 0; i < fifosize && s != end; ++i) {
 			if (*s == '\n' && !cr_sent) {
@@ -3272,7 +3287,7 @@ static void serial8250_console_fifo_write(struct uart_8250_port *up,
 	 * Allow timeout for each byte written since the caller will only wait
 	 * for UART_LSR_BOTH_EMPTY using the timeout of a single character
 	 */
-	fifo_wait_for_lsr(up, tx_count);
+	serial8250_fifo_wait_for_lsr_thre(up, tx_count);
 }
 
 /*
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index f86775cfdcc9e..9aa61c93d7bc6 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -255,14 +255,13 @@ config SERIAL_SAMSUNG_CONSOLE
 
 config SERIAL_TEGRA
 	tristate "NVIDIA Tegra20/30 SoC serial controller"
-	depends on (ARCH_TEGRA && TEGRA20_APB_DMA) || COMPILE_TEST
+	depends on ARCH_TEGRA || COMPILE_TEST
 	select SERIAL_CORE
 	help
 	  Support for the on-chip UARTs on the NVIDIA Tegra series SOCs
 	  providing /dev/ttyTHS0, 1, 2, 3 and 4 (note, some machines may not
 	  provide all of these ports, depending on how the serial port
-	  are enabled). This driver uses the APB DMA to achieve higher baudrate
-	  and better performance.
+	  are enabled).
 
 config SERIAL_TEGRA_TCU
 	tristate "NVIDIA Tegra Combined UART"
@@ -803,7 +802,7 @@ config SERIAL_CPM_CONSOLE
 
 config SERIAL_PIC32
 	tristate "Microchip PIC32 serial support"
-	depends on MACH_PIC32 || (MIPS && COMPILE_TEST)
+	depends on MACH_PIC32 || COMPILE_TEST
 	select SERIAL_CORE
 	help
 	  If you have a PIC32, this driver supports the serial ports.
@@ -1593,32 +1592,6 @@ config SERIAL_NUVOTON_MA35D1_CONSOLE
 	  but you can alter that using a kernel command line option such as
 	  "console=ttyNVTx".
 
-config SERIAL_ESP32
-	tristate "Espressif ESP32 UART support"
-	depends on XTENSA_PLATFORM_ESP32 || (COMPILE_TEST && OF)
-	select SERIAL_CORE
-	select SERIAL_CORE_CONSOLE
-	select SERIAL_EARLYCON
-	help
-	  Driver for the UART controllers of the Espressif ESP32xx SoCs.
-	  When earlycon option is enabled the following kernel command line
-	  snippets may be used:
-	    earlycon=esp32s3uart,mmio32,0x60000000,115200n8,40000000
-	    earlycon=esp32uart,mmio32,0x3ff40000,115200n8
-
-config SERIAL_ESP32_ACM
-	tristate "Espressif ESP32 USB ACM gadget support"
-	depends on XTENSA_PLATFORM_ESP32 || (COMPILE_TEST && OF)
-	select SERIAL_CORE
-	select SERIAL_CORE_CONSOLE
-	select SERIAL_EARLYCON
-	help
-	  Driver for the CDC ACM gadget controller of the Espressif ESP32S3
-	  SoCs that share separate USB controller with the JTAG adapter.
-	  When earlycon option is enabled the following kernel command line
-	  snippet may be used:
-	    earlycon=esp32s3acm,mmio32,0x60038000
-
 endmenu
 
 config SERIAL_MCTRL_GPIO
diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile
index a2ccbc508ec57..bba7b21a4a1d3 100644
--- a/drivers/tty/serial/Makefile
+++ b/drivers/tty/serial/Makefile
@@ -37,8 +37,6 @@ obj-$(CONFIG_SERIAL_CLPS711X)		+= clps711x.o
 obj-$(CONFIG_SERIAL_CPM)		+= cpm_uart.o
 obj-$(CONFIG_SERIAL_CONEXANT_DIGICOLOR)	+= digicolor-usart.o
 obj-$(CONFIG_SERIAL_DZ)			+= dz.o
-obj-$(CONFIG_SERIAL_ESP32)		+= esp32_uart.o
-obj-$(CONFIG_SERIAL_ESP32_ACM)		+= esp32_acm.o
 obj-$(CONFIG_SERIAL_FSL_LINFLEXUART)	+= fsl_linflexuart.o
 obj-$(CONFIG_SERIAL_FSL_LPUART)		+= fsl_lpuart.o
 obj-$(CONFIG_SERIAL_ICOM)		+= icom.o
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index 7f17d288c8076..028e37ad8d79f 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -114,6 +114,8 @@ struct vendor_data {
 	bool			cts_event_workaround;
 	bool			always_enabled;
 	bool			fixed_options;
+	bool			skip_ibrd_fbrd;
+	bool			set_uartclk_rate;
 
 	unsigned int (*get_fifosize)(struct amba_device *dev);
 };
@@ -216,6 +218,28 @@ static struct vendor_data vendor_st = {
 	.get_fifosize		= get_fifosize_st,
 };
 
+static unsigned int get_fifosize_nvidia(struct amba_device *dev)
+{
+	return 32;
+}
+
+static struct vendor_data vendor_nvidia = {
+	.reg_offset		= pl011_std_offsets,
+	.ifls			= UART011_IFLS_RX4_8 | UART011_IFLS_TX4_8,
+	.fr_busy		= UART01x_FR_BUSY,
+	.fr_dsr			= UART01x_FR_DSR,
+	.fr_cts			= UART01x_FR_CTS,
+	.fr_ri			= UART011_FR_RI,
+	.oversampling		= false,
+	.dma_threshold		= false,
+	.cts_event_workaround	= false,
+	.always_enabled		= false,
+	.fixed_options		= false,
+	.skip_ibrd_fbrd		= true,
+	.set_uartclk_rate	= true,
+	.get_fifosize		= get_fifosize_nvidia,
+};
+
 /* Deals with DMA transactions */
 
 struct pl011_dmabuf {
@@ -625,6 +649,15 @@ static int pl011_dma_tx_refill(struct uart_amba_port *uap)
 		count = PL011_DMA_BUFFER_SIZE;
 
 	count = kfifo_out_peek(&tport->xmit_fifo, dmatx->buf, count);
+
+	/*
+	 * Align the TX buffer length to the DMA controller's copy_align
+	 * requirements. Some DMA controllers (e.g., Tegra GPC DMA) require
+	 * word-aligned transfers. Unaligned bytes will be sent via PIO.
+	 */
+	if (chan->device->copy_align)
+		count = ALIGN_DOWN(count, 1 << chan->device->copy_align);
+
 	dmatx->len = count;
 	dmatx->dma = dma_map_single(dma_dev->dev, dmatx->buf, count,
 				    DMA_TO_DEVICE);
@@ -2095,6 +2128,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
 	unsigned int lcr_h, old_cr;
 	unsigned long flags;
 	unsigned int baud, quot, clkdiv;
+	unsigned int max_baud;
 	unsigned int bits;
 
 	if (uap->vendor->oversampling)
@@ -2102,11 +2136,34 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
 	else
 		clkdiv = 16;
 
+	max_baud = port->uartclk / clkdiv;
+
+	if (uap->vendor->set_uartclk_rate) {
+		long max_clkrate = clk_round_rate(uap->clk, UINT_MAX);
+
+		/*
+		 * Clock is reprogrammable - determine max baud from the clock's
+		 * maximum rate, not the current uartclk.
+		 */
+		if (max_clkrate > 0)
+			max_baud = max_clkrate / clkdiv;
+	}
+
 	/*
 	 * Ask the core to calculate the divisor for us.
 	 */
-	baud = uart_get_baud_rate(port, termios, old, 0,
-				  port->uartclk / clkdiv);
+	baud = uart_get_baud_rate(port, termios, old, 0, max_baud);
+
+	if (uap->vendor->set_uartclk_rate) {
+		int err;
+
+		err = clk_set_rate(uap->clk, baud * clkdiv);
+		if (err) {
+			dev_err(port->dev, "Failed to set clock rate: %d\n", err);
+			return;
+		}
+	}
+
 #ifdef CONFIG_DMA_ENGINE
 	/*
 	 * Adjust RX DMA polling rate with baud rate if not specified.
@@ -2115,11 +2172,6 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
 		uap->dmarx.poll_rate = DIV_ROUND_UP(10000000, baud);
 #endif
 
-	if (baud > port->uartclk / 16)
-		quot = DIV_ROUND_CLOSEST(port->uartclk * 8, baud);
-	else
-		quot = DIV_ROUND_CLOSEST(port->uartclk * 4, baud);
-
 	switch (termios->c_cflag & CSIZE) {
 	case CS5:
 		lcr_h = UART01x_LCRH_WLEN_5;
@@ -2190,21 +2242,28 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios,
 			old_cr &= ~ST_UART011_CR_OVSFACT;
 	}
 
-	/*
-	 * Workaround for the ST Micro oversampling variants to
-	 * increase the bitrate slightly, by lowering the divisor,
-	 * to avoid delayed sampling of start bit at high speeds,
-	 * else we see data corruption.
-	 */
-	if (uap->vendor->oversampling) {
-		if (baud >= 3000000 && baud < 3250000 && quot > 1)
-			quot -= 1;
-		else if (baud > 3250000 && quot > 2)
-			quot -= 2;
+	if (!uap->vendor->skip_ibrd_fbrd) {
+		if (baud > port->uartclk / 16)
+			quot = DIV_ROUND_CLOSEST(port->uartclk * 8, baud);
+		else
+			quot = DIV_ROUND_CLOSEST(port->uartclk * 4, baud);
+
+		/*
+		 * Workaround for the ST Micro oversampling variants to
+		 * increase the bitrate slightly, by lowering the divisor,
+		 * to avoid delayed sampling of start bit at high speeds,
+		 * else we see data corruption.
+		 */
+		if (uap->vendor->oversampling) {
+			if (baud >= 3000000 && baud < 3250000 && quot > 1)
+				quot -= 1;
+			else if (baud > 3250000 && quot > 2)
+				quot -= 2;
+		}
+		/* Set baud rate */
+		pl011_write(quot & 0x3f, uap, REG_FBRD);
+		pl011_write(quot >> 6, uap, REG_IBRD);
 	}
-	/* Set baud rate */
-	pl011_write(quot & 0x3f, uap, REG_FBRD);
-	pl011_write(quot >> 6, uap, REG_IBRD);
 
 	/*
 	 * ----------v----------v----------v----------v-----
@@ -2374,6 +2433,7 @@ static void pl011_console_get_options(struct uart_amba_port *uap, int *baud,
 				      int *parity, int *bits)
 {
 	unsigned int lcr_h, ibrd, fbrd;
+	unsigned int clkdiv;
 
 	if (!(pl011_read(uap, REG_CR) & UART01x_CR_UARTEN))
 		return;
@@ -2393,10 +2453,15 @@ static void pl011_console_get_options(struct uart_amba_port *uap, int *baud,
 	else
 		*bits = 8;
 
-	ibrd = pl011_read(uap, REG_IBRD);
-	fbrd = pl011_read(uap, REG_FBRD);
+	if (uap->vendor->skip_ibrd_fbrd) {
+		clkdiv = 64;
+	} else {
+		ibrd = pl011_read(uap, REG_IBRD);
+		fbrd = pl011_read(uap, REG_FBRD);
+		clkdiv = 64 * ibrd + fbrd;
+	}
 
-	*baud = uap->port.uartclk * 4 / (64 * ibrd + fbrd);
+	*baud = uap->port.uartclk * 4 / clkdiv;
 
 	if (uap->vendor->oversampling &&
 	    (pl011_read(uap, REG_CR) & ST_UART011_CR_OVSFACT))
@@ -2701,18 +2766,47 @@ static int pl011_early_read(struct console *con, char *s, unsigned int n)
 static int __init pl011_early_console_setup(struct earlycon_device *device,
 					    const char *opt)
 {
+	unsigned int cr;
+
 	if (!device->port.membase)
 		return -ENODEV;
 
 	device->con->write = pl011_early_write;
 	device->con->read = pl011_early_read;
 
+	if (device->port.iotype == UPIO_MEM32)
+		cr = readl(device->port.membase + UART011_CR);
+	else
+		cr = readw(device->port.membase + UART011_CR);
+	cr &= UART011_CR_RTS | UART011_CR_DTR;
+	cr |= UART01x_CR_UARTEN | UART011_CR_RXE | UART011_CR_TXE;
+	if (device->port.iotype == UPIO_MEM32)
+		writel(cr, device->port.membase + UART011_CR);
+	else
+		writew(cr, device->port.membase + UART011_CR);
+
 	return 0;
 }
 
 OF_EARLYCON_DECLARE(pl011, "arm,pl011", pl011_early_console_setup);
 
-OF_EARLYCON_DECLARE(pl011, "arm,sbsa-uart", pl011_early_console_setup);
+/*
+ * The SBSA UART has no defined control register and is assumed to
+ * be pre-enabled by firmware, so we do not write to UART011_CR.
+ */
+static int __init sbsa_uart_early_console_setup(struct earlycon_device *device,
+						const char *opt)
+{
+	if (!device->port.membase)
+		return -ENODEV;
+
+	device->con->write = pl011_early_write;
+	device->con->read = pl011_early_read;
+
+	return 0;
+}
+
+OF_EARLYCON_DECLARE(pl011, "arm,sbsa-uart", sbsa_uart_early_console_setup);
 
 /*
  * On Qualcomm Datacenter Technologies QDF2400 SOCs affected by
@@ -3081,6 +3175,11 @@ static const struct amba_id pl011_ids[] = {
 		.mask	= 0x00ffffff,
 		.data	= &vendor_st,
 	},
+	{
+		.id	= 0x0006b011,
+		.mask	= 0x000fffff,
+		.data	= &vendor_nvidia,
+	},
 	{ 0, 0 },
 };
 
diff --git a/drivers/tty/serial/apbuart.c b/drivers/tty/serial/apbuart.c
index 364599f256db8..3e46341cfff8a 100644
--- a/drivers/tty/serial/apbuart.c
+++ b/drivers/tty/serial/apbuart.c
@@ -505,7 +505,7 @@ console_initcall(apbuart_console_init);
 
 static struct uart_driver grlib_apbuart_driver = {
 	.owner = THIS_MODULE,
-	.driver_name = "serial",
+	.driver_name = "serial_apbuart",
 	.dev_name = "ttyS",
 	.major = SERIAL_APBUART_MAJOR,
 	.minor = SERIAL_APBUART_MINOR,
diff --git a/drivers/tty/serial/dz.c b/drivers/tty/serial/dz.c
index eba91daedef81..e53c54353c3e4 100644
--- a/drivers/tty/serial/dz.c
+++ b/drivers/tty/serial/dz.c
@@ -914,7 +914,7 @@ console_initcall(dz_serial_console_init);
 
 static struct uart_driver dz_reg = {
 	.owner			= THIS_MODULE,
-	.driver_name		= "serial",
+	.driver_name		= "serial_dz",
 	.dev_name		= "ttyS",
 	.major			= TTY_MAJOR,
 	.minor			= 64,
diff --git a/drivers/tty/serial/esp32_acm.c b/drivers/tty/serial/esp32_acm.c
deleted file mode 100644
index bb7cc65427f0b..0000000000000
--- a/drivers/tty/serial/esp32_acm.c
+++ /dev/null
@@ -1,459 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include <linux/bitfield.h>
-#include <linux/bits.h>
-#include <linux/console.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/irq.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/serial_core.h>
-#include <linux/slab.h>
-#include <linux/tty_flip.h>
-#include <asm/serial.h>
-
-#define DRIVER_NAME	"esp32s3-acm"
-#define DEV_NAME	"ttyGS"
-#define UART_NR		4
-
-#define ESP32S3_ACM_TX_FIFO_SIZE	64
-
-#define USB_SERIAL_JTAG_EP1_REG		0x00
-#define USB_SERIAL_JTAG_EP1_CONF_REG	0x04
-#define USB_SERIAL_JTAG_WR_DONE				BIT(0)
-#define USB_SERIAL_JTAG_SERIAL_IN_EP_DATA_FREE		BIT(1)
-#define USB_SERIAL_JTAG_INT_ST_REG	0x0c
-#define USB_SERIAL_JTAG_SERIAL_OUT_RECV_PKT_INT_ST	BIT(2)
-#define USB_SERIAL_JTAG_SERIAL_IN_EMPTY_INT_ST		BIT(3)
-#define USB_SERIAL_JTAG_INT_ENA_REG	0x10
-#define USB_SERIAL_JTAG_SERIAL_OUT_RECV_PKT_INT_ENA	BIT(2)
-#define USB_SERIAL_JTAG_SERIAL_IN_EMPTY_INT_ENA		BIT(3)
-#define USB_SERIAL_JTAG_INT_CLR_REG	0x14
-#define USB_SERIAL_JTAG_IN_EP1_ST_REG	0x2c
-#define USB_SERIAL_JTAG_IN_EP1_WR_ADDR			GENMASK(8, 2)
-#define USB_SERIAL_JTAG_OUT_EP1_ST_REG	0x3c
-#define USB_SERIAL_JTAG_OUT_EP1_REC_DATA_CNT		GENMASK(22, 16)
-
-static const struct of_device_id esp32s3_acm_dt_ids[] = {
-	{
-		.compatible = "esp,esp32s3-acm",
-	}, { /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, esp32s3_acm_dt_ids);
-
-static struct uart_port *esp32s3_acm_ports[UART_NR];
-
-static void esp32s3_acm_write(struct uart_port *port, unsigned long reg, u32 v)
-{
-	writel(v, port->membase + reg);
-}
-
-static u32 esp32s3_acm_read(struct uart_port *port, unsigned long reg)
-{
-	return readl(port->membase + reg);
-}
-
-static u32 esp32s3_acm_tx_fifo_free(struct uart_port *port)
-{
-	u32 status = esp32s3_acm_read(port, USB_SERIAL_JTAG_EP1_CONF_REG);
-
-	return status & USB_SERIAL_JTAG_SERIAL_IN_EP_DATA_FREE;
-}
-
-static u32 esp32s3_acm_tx_fifo_cnt(struct uart_port *port)
-{
-	u32 status = esp32s3_acm_read(port, USB_SERIAL_JTAG_IN_EP1_ST_REG);
-
-	return FIELD_GET(USB_SERIAL_JTAG_IN_EP1_WR_ADDR, status);
-}
-
-static u32 esp32s3_acm_rx_fifo_cnt(struct uart_port *port)
-{
-	u32 status = esp32s3_acm_read(port, USB_SERIAL_JTAG_OUT_EP1_ST_REG);
-
-	return FIELD_GET(USB_SERIAL_JTAG_OUT_EP1_REC_DATA_CNT, status);
-}
-
-/* return TIOCSER_TEMT when transmitter is not busy */
-static unsigned int esp32s3_acm_tx_empty(struct uart_port *port)
-{
-	return esp32s3_acm_tx_fifo_cnt(port) == 0 ? TIOCSER_TEMT : 0;
-}
-
-static void esp32s3_acm_set_mctrl(struct uart_port *port, unsigned int mctrl)
-{
-}
-
-static unsigned int esp32s3_acm_get_mctrl(struct uart_port *port)
-{
-	return TIOCM_CAR;
-}
-
-static void esp32s3_acm_stop_tx(struct uart_port *port)
-{
-	u32 int_ena;
-
-	int_ena = esp32s3_acm_read(port, USB_SERIAL_JTAG_INT_ENA_REG);
-	int_ena &= ~USB_SERIAL_JTAG_SERIAL_IN_EMPTY_INT_ENA;
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_ENA_REG, int_ena);
-}
-
-static void esp32s3_acm_rxint(struct uart_port *port)
-{
-	struct tty_port *tty_port = &port->state->port;
-	u32 rx_fifo_cnt = esp32s3_acm_rx_fifo_cnt(port);
-	unsigned long flags;
-	u32 i;
-
-	if (!rx_fifo_cnt)
-		return;
-
-	spin_lock_irqsave(&port->lock, flags);
-
-	for (i = 0; i < rx_fifo_cnt; ++i) {
-		u32 rx = esp32s3_acm_read(port, USB_SERIAL_JTAG_EP1_REG);
-
-		++port->icount.rx;
-		tty_insert_flip_char(tty_port, rx, TTY_NORMAL);
-	}
-	spin_unlock_irqrestore(&port->lock, flags);
-
-	tty_flip_buffer_push(tty_port);
-}
-
-static void esp32s3_acm_push(struct uart_port *port)
-{
-	if (esp32s3_acm_tx_fifo_free(port))
-		esp32s3_acm_write(port, USB_SERIAL_JTAG_EP1_CONF_REG,
-				  USB_SERIAL_JTAG_WR_DONE);
-}
-
-static void esp32s3_acm_put_char(struct uart_port *port, u8 c)
-{
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_EP1_REG, c);
-}
-
-static void esp32s3_acm_put_char_sync(struct uart_port *port, u8 c)
-{
-	unsigned long timeout = jiffies + HZ;
-
-	while (!esp32s3_acm_tx_fifo_free(port)) {
-		if (time_after(jiffies, timeout)) {
-			dev_warn(port->dev, "timeout waiting for TX FIFO\n");
-			return;
-		}
-		cpu_relax();
-	}
-	esp32s3_acm_put_char(port, c);
-	esp32s3_acm_push(port);
-}
-
-static void esp32s3_acm_transmit_buffer(struct uart_port *port)
-{
-	u32 tx_fifo_used;
-	unsigned int pending;
-	u8 ch;
-
-	if (!esp32s3_acm_tx_fifo_free(port))
-		return;
-
-	tx_fifo_used = esp32s3_acm_tx_fifo_cnt(port);
-	pending = uart_port_tx_limited(port, ch,
-				       ESP32S3_ACM_TX_FIFO_SIZE - tx_fifo_used,
-				       true, esp32s3_acm_put_char(port, ch),
-				       ({}));
-	if (pending) {
-		u32 int_ena;
-
-		int_ena = esp32s3_acm_read(port, USB_SERIAL_JTAG_INT_ENA_REG);
-		int_ena |= USB_SERIAL_JTAG_SERIAL_IN_EMPTY_INT_ENA;
-		esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_ENA_REG, int_ena);
-	}
-	esp32s3_acm_push(port);
-}
-
-static void esp32s3_acm_txint(struct uart_port *port)
-{
-	esp32s3_acm_transmit_buffer(port);
-}
-
-static irqreturn_t esp32s3_acm_int(int irq, void *dev_id)
-{
-	struct uart_port *port = dev_id;
-	u32 status;
-
-	status = esp32s3_acm_read(port, USB_SERIAL_JTAG_INT_ST_REG);
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_CLR_REG, status);
-
-	if (status & USB_SERIAL_JTAG_SERIAL_OUT_RECV_PKT_INT_ST)
-		esp32s3_acm_rxint(port);
-	if (status & USB_SERIAL_JTAG_SERIAL_IN_EMPTY_INT_ST)
-		esp32s3_acm_txint(port);
-
-	return IRQ_RETVAL(status);
-}
-
-static void esp32s3_acm_start_tx(struct uart_port *port)
-{
-	esp32s3_acm_transmit_buffer(port);
-}
-
-static void esp32s3_acm_stop_rx(struct uart_port *port)
-{
-	u32 int_ena;
-
-	int_ena = esp32s3_acm_read(port, USB_SERIAL_JTAG_INT_ENA_REG);
-	int_ena &= ~USB_SERIAL_JTAG_SERIAL_OUT_RECV_PKT_INT_ENA;
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_ENA_REG, int_ena);
-}
-
-static int esp32s3_acm_startup(struct uart_port *port)
-{
-	int ret;
-
-	ret = request_irq(port->irq, esp32s3_acm_int, 0, DRIVER_NAME, port);
-	if (ret)
-		return ret;
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_ENA_REG,
-			  USB_SERIAL_JTAG_SERIAL_OUT_RECV_PKT_INT_ENA);
-
-	return 0;
-}
-
-static void esp32s3_acm_shutdown(struct uart_port *port)
-{
-	esp32s3_acm_write(port, USB_SERIAL_JTAG_INT_ENA_REG, 0);
-	free_irq(port->irq, port);
-}
-
-static void esp32s3_acm_set_termios(struct uart_port *port,
-				    struct ktermios *termios,
-				    const struct ktermios *old)
-{
-}
-
-static const char *esp32s3_acm_type(struct uart_port *port)
-{
-	return "ESP32S3 ACM";
-}
-
-/* configure/auto-configure the port */
-static void esp32s3_acm_config_port(struct uart_port *port, int flags)
-{
-	if (flags & UART_CONFIG_TYPE)
-		port->type = PORT_GENERIC;
-}
-
-#ifdef CONFIG_CONSOLE_POLL
-static void esp32s3_acm_poll_put_char(struct uart_port *port, unsigned char c)
-{
-	esp32s3_acm_put_char_sync(port, c);
-}
-
-static int esp32s3_acm_poll_get_char(struct uart_port *port)
-{
-	if (esp32s3_acm_rx_fifo_cnt(port))
-		return esp32s3_acm_read(port, USB_SERIAL_JTAG_EP1_REG);
-	else
-		return NO_POLL_CHAR;
-}
-#endif
-
-static const struct uart_ops esp32s3_acm_pops = {
-	.tx_empty	= esp32s3_acm_tx_empty,
-	.set_mctrl	= esp32s3_acm_set_mctrl,
-	.get_mctrl	= esp32s3_acm_get_mctrl,
-	.stop_tx	= esp32s3_acm_stop_tx,
-	.start_tx	= esp32s3_acm_start_tx,
-	.stop_rx	= esp32s3_acm_stop_rx,
-	.startup	= esp32s3_acm_startup,
-	.shutdown	= esp32s3_acm_shutdown,
-	.set_termios	= esp32s3_acm_set_termios,
-	.type		= esp32s3_acm_type,
-	.config_port	= esp32s3_acm_config_port,
-#ifdef CONFIG_CONSOLE_POLL
-	.poll_put_char	= esp32s3_acm_poll_put_char,
-	.poll_get_char	= esp32s3_acm_poll_get_char,
-#endif
-};
-
-static void esp32s3_acm_string_write(struct uart_port *port, const char *s,
-				     unsigned int count)
-{
-	uart_console_write(port, s, count, esp32s3_acm_put_char_sync);
-}
-
-static void
-esp32s3_acm_console_write(struct console *co, const char *s, unsigned int count)
-{
-	struct uart_port *port = esp32s3_acm_ports[co->index];
-	unsigned long flags;
-	bool locked = true;
-
-	if (port->sysrq)
-		locked = false;
-	else if (oops_in_progress)
-		locked = spin_trylock_irqsave(&port->lock, flags);
-	else
-		spin_lock_irqsave(&port->lock, flags);
-
-	esp32s3_acm_string_write(port, s, count);
-
-	if (locked)
-		spin_unlock_irqrestore(&port->lock, flags);
-}
-
-static struct uart_driver esp32s3_acm_reg;
-static struct console esp32s3_acm_console = {
-	.name		= DEV_NAME,
-	.write		= esp32s3_acm_console_write,
-	.device		= uart_console_device,
-	.flags		= CON_PRINTBUFFER,
-	.index		= -1,
-	.data		= &esp32s3_acm_reg,
-};
-
-static void esp32s3_acm_earlycon_write(struct console *con, const char *s,
-				      unsigned int n)
-{
-	struct earlycon_device *dev = con->data;
-
-	uart_console_write(&dev->port, s, n, esp32s3_acm_put_char_sync);
-}
-
-#ifdef CONFIG_CONSOLE_POLL
-static int esp32s3_acm_earlycon_read(struct console *con, char *s, unsigned int n)
-{
-	struct earlycon_device *dev = con->data;
-	unsigned int num_read = 0;
-
-	while (num_read < n) {
-		int c = esp32s3_acm_poll_get_char(&dev->port);
-
-		if (c == NO_POLL_CHAR)
-			break;
-		s[num_read++] = c;
-	}
-	return num_read;
-}
-#endif
-
-static int __init esp32s3_acm_early_console_setup(struct earlycon_device *device,
-						   const char *options)
-{
-	if (!device->port.membase)
-		return -ENODEV;
-
-	device->con->write = esp32s3_acm_earlycon_write;
-#ifdef CONFIG_CONSOLE_POLL
-	device->con->read = esp32s3_acm_earlycon_read;
-#endif
-	return 0;
-}
-
-OF_EARLYCON_DECLARE(esp32s3acm, "esp,esp32s3-acm",
-		    esp32s3_acm_early_console_setup);
-
-static struct uart_driver esp32s3_acm_reg = {
-	.owner		= THIS_MODULE,
-	.driver_name	= DRIVER_NAME,
-	.dev_name	= DEV_NAME,
-	.nr		= ARRAY_SIZE(esp32s3_acm_ports),
-	.cons		= &esp32s3_acm_console,
-};
-
-static int esp32s3_acm_probe(struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	struct uart_port *port;
-	struct resource *res;
-	int ret;
-
-	port = devm_kzalloc(&pdev->dev, sizeof(*port), GFP_KERNEL);
-	if (!port)
-		return -ENOMEM;
-
-	ret = of_alias_get_id(np, "serial");
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);
-		return ret;
-	}
-	if (ret >= UART_NR) {
-		dev_err(&pdev->dev, "driver limited to %d serial ports\n",
-			UART_NR);
-		return -ENOMEM;
-	}
-
-	port->line = ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	port->mapbase = res->start;
-	port->membase = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(port->membase))
-		return PTR_ERR(port->membase);
-
-	port->dev = &pdev->dev;
-	port->type = PORT_GENERIC;
-	port->iotype = UPIO_MEM;
-	port->irq = platform_get_irq(pdev, 0);
-	port->ops = &esp32s3_acm_pops;
-	port->flags = UPF_BOOT_AUTOCONF;
-	port->has_sysrq = 1;
-	port->fifosize = ESP32S3_ACM_TX_FIFO_SIZE;
-
-	esp32s3_acm_ports[port->line] = port;
-
-	platform_set_drvdata(pdev, port);
-
-	return uart_add_one_port(&esp32s3_acm_reg, port);
-}
-
-static void esp32s3_acm_remove(struct platform_device *pdev)
-{
-	struct uart_port *port = platform_get_drvdata(pdev);
-
-	uart_remove_one_port(&esp32s3_acm_reg, port);
-}
-
-
-static struct platform_driver esp32s3_acm_driver = {
-	.probe		= esp32s3_acm_probe,
-	.remove		= esp32s3_acm_remove,
-	.driver		= {
-		.name	= DRIVER_NAME,
-		.of_match_table	= esp32s3_acm_dt_ids,
-	},
-};
-
-static int __init esp32s3_acm_init(void)
-{
-	int ret;
-
-	ret = uart_register_driver(&esp32s3_acm_reg);
-	if (ret)
-		return ret;
-
-	ret = platform_driver_register(&esp32s3_acm_driver);
-	if (ret)
-		uart_unregister_driver(&esp32s3_acm_reg);
-
-	return ret;
-}
-
-static void __exit esp32s3_acm_exit(void)
-{
-	platform_driver_unregister(&esp32s3_acm_driver);
-	uart_unregister_driver(&esp32s3_acm_reg);
-}
-
-module_init(esp32s3_acm_init);
-module_exit(esp32s3_acm_exit);
-
-MODULE_AUTHOR("Max Filippov <jcmvbkbc@gmail.com>");
-MODULE_DESCRIPTION("Espressif ESP32 USB ACM gadget support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/tty/serial/esp32_uart.c b/drivers/tty/serial/esp32_uart.c
deleted file mode 100644
index 667c2198a03a5..0000000000000
--- a/drivers/tty/serial/esp32_uart.c
+++ /dev/null
@@ -1,779 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include <linux/bitfield.h>
-#include <linux/bits.h>
-#include <linux/clk.h>
-#include <linux/console.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/irq.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/property.h>
-#include <linux/serial_core.h>
-#include <linux/slab.h>
-#include <linux/tty_flip.h>
-#include <asm/serial.h>
-
-#define DRIVER_NAME	"esp32-uart"
-#define DEV_NAME	"ttyS"
-#define UART_NR		3
-
-#define ESP32_UART_TX_FIFO_SIZE	127
-#define ESP32_UART_RX_FIFO_SIZE	127
-
-#define UART_FIFO_REG			0x00
-#define UART_INT_RAW_REG		0x04
-#define UART_INT_ST_REG			0x08
-#define UART_INT_ENA_REG		0x0c
-#define UART_INT_CLR_REG		0x10
-#define UART_RXFIFO_FULL_INT			BIT(0)
-#define UART_TXFIFO_EMPTY_INT			BIT(1)
-#define UART_BRK_DET_INT			BIT(7)
-#define UART_CLKDIV_REG			0x14
-#define ESP32_UART_CLKDIV			GENMASK(19, 0)
-#define ESP32S3_UART_CLKDIV			GENMASK(11, 0)
-#define UART_CLKDIV_SHIFT			0
-#define UART_CLKDIV_FRAG			GENMASK(23, 20)
-#define UART_STATUS_REG			0x1c
-#define ESP32_UART_RXFIFO_CNT			GENMASK(7, 0)
-#define ESP32S3_UART_RXFIFO_CNT			GENMASK(9, 0)
-#define UART_RXFIFO_CNT_SHIFT			0
-#define UART_DSRN				BIT(13)
-#define UART_CTSN				BIT(14)
-#define ESP32_UART_TXFIFO_CNT			GENMASK(23, 16)
-#define ESP32S3_UART_TXFIFO_CNT			GENMASK(25, 16)
-#define UART_TXFIFO_CNT_SHIFT			16
-#define UART_CONF0_REG			0x20
-#define UART_PARITY				BIT(0)
-#define UART_PARITY_EN				BIT(1)
-#define UART_BIT_NUM				GENMASK(3, 2)
-#define UART_BIT_NUM_5				0
-#define UART_BIT_NUM_6				1
-#define UART_BIT_NUM_7				2
-#define UART_BIT_NUM_8				3
-#define UART_STOP_BIT_NUM			GENMASK(5, 4)
-#define UART_STOP_BIT_NUM_1			1
-#define UART_STOP_BIT_NUM_2			3
-#define UART_SW_RTS				BIT(6)
-#define UART_SW_DTR				BIT(7)
-#define UART_LOOPBACK				BIT(14)
-#define UART_TX_FLOW_EN				BIT(15)
-#define UART_RTS_INV				BIT(23)
-#define UART_DTR_INV				BIT(24)
-#define UART_CONF1_REG			0x24
-#define UART_RXFIFO_FULL_THRHD_SHIFT		0
-#define ESP32_UART_TXFIFO_EMPTY_THRHD_SHIFT	8
-#define ESP32S3_UART_TXFIFO_EMPTY_THRHD_SHIFT	10
-#define ESP32_UART_RX_FLOW_EN			BIT(23)
-#define ESP32S3_UART_RX_FLOW_EN			BIT(22)
-#define ESP32S3_UART_CLK_CONF_REG	0x78
-#define ESP32S3_UART_SCLK_DIV_B			GENMASK(5, 0)
-#define ESP32S3_UART_SCLK_DIV_A			GENMASK(11, 6)
-#define ESP32S3_UART_SCLK_DIV_NUM		GENMASK(19, 12)
-#define ESP32S3_UART_SCLK_SEL			GENMASK(21, 20)
-#define APB_CLK					1
-#define RC_FAST_CLK				2
-#define XTAL_CLK				3
-#define ESP32S3_UART_SCLK_EN			BIT(22)
-#define ESP32S3_UART_RST_CORE			BIT(23)
-#define ESP32S3_UART_TX_SCLK_EN			BIT(24)
-#define ESP32S3_UART_RX_SCLK_EN			BIT(25)
-#define ESP32S3_UART_TX_RST_CORE		BIT(26)
-#define ESP32S3_UART_RX_RST_CORE		BIT(27)
-
-#define ESP32S3_UART_CLK_CONF_DEFAULT \
-	(ESP32S3_UART_RX_SCLK_EN | \
-	 ESP32S3_UART_TX_SCLK_EN | \
-	 ESP32S3_UART_SCLK_EN | \
-	 FIELD_PREP(ESP32S3_UART_SCLK_SEL, XTAL_CLK))
-
-struct esp32_port {
-	struct uart_port port;
-	struct clk *clk;
-};
-
-struct esp32_uart_variant {
-	u32 clkdiv_mask;
-	u32 rxfifo_cnt_mask;
-	u32 txfifo_cnt_mask;
-	u32 txfifo_empty_thrhd_shift;
-	u32 rx_flow_en;
-	const char *type;
-	bool has_clkconf;
-};
-
-static const struct esp32_uart_variant esp32_variant = {
-	.clkdiv_mask = ESP32_UART_CLKDIV,
-	.rxfifo_cnt_mask = ESP32_UART_RXFIFO_CNT,
-	.txfifo_cnt_mask = ESP32_UART_TXFIFO_CNT,
-	.txfifo_empty_thrhd_shift = ESP32_UART_TXFIFO_EMPTY_THRHD_SHIFT,
-	.rx_flow_en = ESP32_UART_RX_FLOW_EN,
-	.type = "ESP32 UART",
-};
-
-static const struct esp32_uart_variant esp32s3_variant = {
-	.clkdiv_mask = ESP32S3_UART_CLKDIV,
-	.rxfifo_cnt_mask = ESP32S3_UART_RXFIFO_CNT,
-	.txfifo_cnt_mask = ESP32S3_UART_TXFIFO_CNT,
-	.txfifo_empty_thrhd_shift = ESP32S3_UART_TXFIFO_EMPTY_THRHD_SHIFT,
-	.rx_flow_en = ESP32S3_UART_RX_FLOW_EN,
-	.type = "ESP32S3 UART",
-	.has_clkconf = true,
-};
-
-static const struct of_device_id esp32_uart_dt_ids[] = {
-	{
-		.compatible = "esp,esp32-uart",
-		.data = &esp32_variant,
-	}, {
-		.compatible = "esp,esp32s3-uart",
-		.data = &esp32s3_variant,
-	}, { /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, esp32_uart_dt_ids);
-
-static struct esp32_port *esp32_uart_ports[UART_NR];
-
-static const struct esp32_uart_variant *port_variant(struct uart_port *port)
-{
-	return port->private_data;
-}
-
-static void esp32_uart_write(struct uart_port *port, unsigned long reg, u32 v)
-{
-	writel(v, port->membase + reg);
-}
-
-static u32 esp32_uart_read(struct uart_port *port, unsigned long reg)
-{
-	return readl(port->membase + reg);
-}
-
-static u32 esp32_uart_tx_fifo_cnt(struct uart_port *port)
-{
-	u32 status = esp32_uart_read(port, UART_STATUS_REG);
-
-	return (status & port_variant(port)->txfifo_cnt_mask) >> UART_TXFIFO_CNT_SHIFT;
-}
-
-static u32 esp32_uart_rx_fifo_cnt(struct uart_port *port)
-{
-	u32 status = esp32_uart_read(port, UART_STATUS_REG);
-
-	return (status & port_variant(port)->rxfifo_cnt_mask) >> UART_RXFIFO_CNT_SHIFT;
-}
-
-/* return TIOCSER_TEMT when transmitter is not busy */
-static unsigned int esp32_uart_tx_empty(struct uart_port *port)
-{
-	return esp32_uart_tx_fifo_cnt(port) ? 0 : TIOCSER_TEMT;
-}
-
-static void esp32_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
-{
-	u32 conf0 = esp32_uart_read(port, UART_CONF0_REG);
-
-	conf0 &= ~(UART_LOOPBACK |
-		   UART_SW_RTS | UART_RTS_INV |
-		   UART_SW_DTR | UART_DTR_INV);
-
-	if (mctrl & TIOCM_RTS)
-		conf0 |= UART_SW_RTS;
-	if (mctrl & TIOCM_DTR)
-		conf0 |= UART_SW_DTR;
-	if (mctrl & TIOCM_LOOP)
-		conf0 |= UART_LOOPBACK;
-
-	esp32_uart_write(port, UART_CONF0_REG, conf0);
-}
-
-static unsigned int esp32_uart_get_mctrl(struct uart_port *port)
-{
-	u32 status = esp32_uart_read(port, UART_STATUS_REG);
-	unsigned int ret = TIOCM_CAR;
-
-	if (status & UART_DSRN)
-		ret |= TIOCM_DSR;
-	if (status & UART_CTSN)
-		ret |= TIOCM_CTS;
-
-	return ret;
-}
-
-static void esp32_uart_stop_tx(struct uart_port *port)
-{
-	u32 int_ena;
-
-	int_ena = esp32_uart_read(port, UART_INT_ENA_REG);
-	int_ena &= ~UART_TXFIFO_EMPTY_INT;
-	esp32_uart_write(port, UART_INT_ENA_REG, int_ena);
-}
-
-static void esp32_uart_rxint(struct uart_port *port)
-{
-	struct tty_port *tty_port = &port->state->port;
-	u32 rx_fifo_cnt = esp32_uart_rx_fifo_cnt(port);
-	unsigned long flags;
-	u32 i;
-
-	if (!rx_fifo_cnt)
-		return;
-
-	spin_lock_irqsave(&port->lock, flags);
-
-	for (i = 0; i < rx_fifo_cnt; ++i) {
-		u32 rx = esp32_uart_read(port, UART_FIFO_REG);
-
-		if (!rx &&
-		    (esp32_uart_read(port, UART_INT_ST_REG) & UART_BRK_DET_INT)) {
-			esp32_uart_write(port, UART_INT_CLR_REG, UART_BRK_DET_INT);
-			++port->icount.brk;
-			uart_handle_break(port);
-		} else {
-			if (uart_handle_sysrq_char(port, (unsigned char)rx))
-				continue;
-			tty_insert_flip_char(tty_port, rx, TTY_NORMAL);
-			++port->icount.rx;
-		}
-	}
-	spin_unlock_irqrestore(&port->lock, flags);
-
-	tty_flip_buffer_push(tty_port);
-}
-
-static void esp32_uart_put_char(struct uart_port *port, u8 c)
-{
-	esp32_uart_write(port, UART_FIFO_REG, c);
-}
-
-static void esp32_uart_put_char_sync(struct uart_port *port, u8 c)
-{
-	unsigned long timeout = jiffies + HZ;
-
-	while (esp32_uart_tx_fifo_cnt(port) >= ESP32_UART_TX_FIFO_SIZE) {
-		if (time_after(jiffies, timeout)) {
-			dev_warn(port->dev, "timeout waiting for TX FIFO\n");
-			return;
-		}
-		cpu_relax();
-	}
-	esp32_uart_put_char(port, c);
-}
-
-static void esp32_uart_transmit_buffer(struct uart_port *port)
-{
-	u32 tx_fifo_used = esp32_uart_tx_fifo_cnt(port);
-	unsigned int pending;
-	u8 ch;
-
-	if (tx_fifo_used >= ESP32_UART_TX_FIFO_SIZE)
-		return;
-
-	pending = uart_port_tx_limited(port, ch,
-				       ESP32_UART_TX_FIFO_SIZE - tx_fifo_used,
-				       true, esp32_uart_put_char(port, ch),
-				       ({}));
-	if (pending) {
-		u32 int_ena;
-
-		int_ena = esp32_uart_read(port, UART_INT_ENA_REG);
-		int_ena |= UART_TXFIFO_EMPTY_INT;
-		esp32_uart_write(port, UART_INT_ENA_REG, int_ena);
-	}
-}
-
-static void esp32_uart_txint(struct uart_port *port)
-{
-	esp32_uart_transmit_buffer(port);
-}
-
-static irqreturn_t esp32_uart_int(int irq, void *dev_id)
-{
-	struct uart_port *port = dev_id;
-	u32 status;
-
-	status = esp32_uart_read(port, UART_INT_ST_REG);
-
-	if (status & (UART_RXFIFO_FULL_INT | UART_BRK_DET_INT))
-		esp32_uart_rxint(port);
-	if (status & UART_TXFIFO_EMPTY_INT)
-		esp32_uart_txint(port);
-
-	esp32_uart_write(port, UART_INT_CLR_REG, status);
-
-	return IRQ_RETVAL(status);
-}
-
-static void esp32_uart_start_tx(struct uart_port *port)
-{
-	esp32_uart_transmit_buffer(port);
-}
-
-static void esp32_uart_stop_rx(struct uart_port *port)
-{
-	u32 int_ena;
-
-	int_ena = esp32_uart_read(port, UART_INT_ENA_REG);
-	int_ena &= ~UART_RXFIFO_FULL_INT;
-	esp32_uart_write(port, UART_INT_ENA_REG, int_ena);
-}
-
-static int esp32_uart_startup(struct uart_port *port)
-{
-	int ret = 0;
-	unsigned long flags;
-	struct esp32_port *sport = container_of(port, struct esp32_port, port);
-
-	ret = clk_prepare_enable(sport->clk);
-	if (ret)
-		return ret;
-
-	ret = request_irq(port->irq, esp32_uart_int, 0, DRIVER_NAME, port);
-	if (ret) {
-		clk_disable_unprepare(sport->clk);
-		return ret;
-	}
-
-	spin_lock_irqsave(&port->lock, flags);
-	if (port_variant(port)->has_clkconf)
-		esp32_uart_write(port, ESP32S3_UART_CLK_CONF_REG,
-				 ESP32S3_UART_CLK_CONF_DEFAULT);
-	esp32_uart_write(port, UART_CONF1_REG,
-			 (1 << UART_RXFIFO_FULL_THRHD_SHIFT) |
-			 (1 << port_variant(port)->txfifo_empty_thrhd_shift));
-	esp32_uart_write(port, UART_INT_CLR_REG, UART_RXFIFO_FULL_INT | UART_BRK_DET_INT);
-	esp32_uart_write(port, UART_INT_ENA_REG, UART_RXFIFO_FULL_INT | UART_BRK_DET_INT);
-	spin_unlock_irqrestore(&port->lock, flags);
-
-	return ret;
-}
-
-static void esp32_uart_shutdown(struct uart_port *port)
-{
-	struct esp32_port *sport = container_of(port, struct esp32_port, port);
-
-	esp32_uart_write(port, UART_INT_ENA_REG, 0);
-	free_irq(port->irq, port);
-	clk_disable_unprepare(sport->clk);
-}
-
-static bool esp32_uart_set_baud(struct uart_port *port, u32 baud)
-{
-	u32 sclk = port->uartclk;
-	u32 div = sclk / baud;
-
-	if (port_variant(port)->has_clkconf) {
-		u32 sclk_div = div / port_variant(port)->clkdiv_mask;
-
-		if (div > port_variant(port)->clkdiv_mask) {
-			sclk /= (sclk_div + 1);
-			div = sclk / baud;
-		}
-		esp32_uart_write(port, ESP32S3_UART_CLK_CONF_REG,
-				 FIELD_PREP(ESP32S3_UART_SCLK_DIV_NUM, sclk_div) |
-				 ESP32S3_UART_CLK_CONF_DEFAULT);
-	}
-
-	if (div <= port_variant(port)->clkdiv_mask) {
-		u32 frag = (sclk * 16) / baud - div * 16;
-
-		esp32_uart_write(port, UART_CLKDIV_REG,
-				 div | FIELD_PREP(UART_CLKDIV_FRAG, frag));
-		return true;
-	}
-
-	return false;
-}
-
-static void esp32_uart_set_termios(struct uart_port *port,
-				   struct ktermios *termios,
-				   const struct ktermios *old)
-{
-	unsigned long flags;
-	u32 conf0, conf1;
-	u32 baud;
-	const u32 rx_flow_en = port_variant(port)->rx_flow_en;
-	u32 max_div = port_variant(port)->clkdiv_mask;
-
-	termios->c_cflag &= ~CMSPAR;
-
-	if (port_variant(port)->has_clkconf)
-		max_div *= FIELD_MAX(ESP32S3_UART_SCLK_DIV_NUM);
-
-	baud = uart_get_baud_rate(port, termios, old,
-				  port->uartclk / max_div,
-				  port->uartclk / 16);
-
-	spin_lock_irqsave(&port->lock, flags);
-
-	conf0 = esp32_uart_read(port, UART_CONF0_REG);
-	conf0 &= ~(UART_PARITY_EN | UART_PARITY | UART_BIT_NUM | UART_STOP_BIT_NUM);
-
-	conf1 = esp32_uart_read(port, UART_CONF1_REG);
-	conf1 &= ~rx_flow_en;
-
-	if (termios->c_cflag & PARENB) {
-		conf0 |= UART_PARITY_EN;
-		if (termios->c_cflag & PARODD)
-			conf0 |= UART_PARITY;
-	}
-
-	switch (termios->c_cflag & CSIZE) {
-	case CS5:
-		conf0 |= FIELD_PREP(UART_BIT_NUM, UART_BIT_NUM_5);
-		break;
-	case CS6:
-		conf0 |= FIELD_PREP(UART_BIT_NUM, UART_BIT_NUM_6);
-		break;
-	case CS7:
-		conf0 |= FIELD_PREP(UART_BIT_NUM, UART_BIT_NUM_7);
-		break;
-	case CS8:
-		conf0 |= FIELD_PREP(UART_BIT_NUM, UART_BIT_NUM_8);
-		break;
-	}
-
-	if (termios->c_cflag & CSTOPB)
-		conf0 |= FIELD_PREP(UART_STOP_BIT_NUM, UART_STOP_BIT_NUM_2);
-	else
-		conf0 |= FIELD_PREP(UART_STOP_BIT_NUM, UART_STOP_BIT_NUM_1);
-
-	if (termios->c_cflag & CRTSCTS)
-		conf1 |= rx_flow_en;
-
-	esp32_uart_write(port, UART_CONF0_REG, conf0);
-	esp32_uart_write(port, UART_CONF1_REG, conf1);
-
-	if (baud) {
-		esp32_uart_set_baud(port, baud);
-		uart_update_timeout(port, termios->c_cflag, baud);
-	} else {
-		if (esp32_uart_set_baud(port, 115200)) {
-			baud = 115200;
-			tty_termios_encode_baud_rate(termios, baud, baud);
-			uart_update_timeout(port, termios->c_cflag, baud);
-		} else {
-			dev_warn(port->dev,
-				 "unable to set speed to %d baud or the default 115200\n",
-				 baud);
-		}
-	}
-	spin_unlock_irqrestore(&port->lock, flags);
-}
-
-static const char *esp32_uart_type(struct uart_port *port)
-{
-	return port_variant(port)->type;
-}
-
-/* configure/auto-configure the port */
-static void esp32_uart_config_port(struct uart_port *port, int flags)
-{
-	if (flags & UART_CONFIG_TYPE)
-		port->type = PORT_GENERIC;
-}
-
-#ifdef CONFIG_CONSOLE_POLL
-static int esp32_uart_poll_init(struct uart_port *port)
-{
-	struct esp32_port *sport = container_of(port, struct esp32_port, port);
-
-	return clk_prepare_enable(sport->clk);
-}
-
-static void esp32_uart_poll_put_char(struct uart_port *port, unsigned char c)
-{
-	esp32_uart_put_char_sync(port, c);
-}
-
-static int esp32_uart_poll_get_char(struct uart_port *port)
-{
-	if (esp32_uart_rx_fifo_cnt(port))
-		return esp32_uart_read(port, UART_FIFO_REG);
-	else
-		return NO_POLL_CHAR;
-
-}
-#endif
-
-static const struct uart_ops esp32_uart_pops = {
-	.tx_empty	= esp32_uart_tx_empty,
-	.set_mctrl	= esp32_uart_set_mctrl,
-	.get_mctrl	= esp32_uart_get_mctrl,
-	.stop_tx	= esp32_uart_stop_tx,
-	.start_tx	= esp32_uart_start_tx,
-	.stop_rx	= esp32_uart_stop_rx,
-	.startup	= esp32_uart_startup,
-	.shutdown	= esp32_uart_shutdown,
-	.set_termios	= esp32_uart_set_termios,
-	.type		= esp32_uart_type,
-	.config_port	= esp32_uart_config_port,
-#ifdef CONFIG_CONSOLE_POLL
-	.poll_init	= esp32_uart_poll_init,
-	.poll_put_char	= esp32_uart_poll_put_char,
-	.poll_get_char	= esp32_uart_poll_get_char,
-#endif
-};
-
-static void esp32_uart_console_putchar(struct uart_port *port, u8 c)
-{
-	esp32_uart_put_char_sync(port, c);
-}
-
-static void esp32_uart_string_write(struct uart_port *port, const char *s,
-				    unsigned int count)
-{
-	uart_console_write(port, s, count, esp32_uart_console_putchar);
-}
-
-static void
-esp32_uart_console_write(struct console *co, const char *s, unsigned int count)
-{
-	struct esp32_port *sport = esp32_uart_ports[co->index];
-	struct uart_port *port = &sport->port;
-	unsigned long flags;
-	bool locked = true;
-
-	if (port->sysrq)
-		locked = false;
-	else if (oops_in_progress)
-		locked = spin_trylock_irqsave(&port->lock, flags);
-	else
-		spin_lock_irqsave(&port->lock, flags);
-
-	esp32_uart_string_write(port, s, count);
-
-	if (locked)
-		spin_unlock_irqrestore(&port->lock, flags);
-}
-
-static int __init esp32_uart_console_setup(struct console *co, char *options)
-{
-	struct esp32_port *sport;
-	int baud = 115200;
-	int bits = 8;
-	int parity = 'n';
-	int flow = 'n';
-	int ret;
-
-	/*
-	 * check whether an invalid uart number has been specified, and
-	 * if so, search for the first available port that does have
-	 * console support.
-	 */
-	if (co->index == -1 || co->index >= ARRAY_SIZE(esp32_uart_ports))
-		co->index = 0;
-
-	sport = esp32_uart_ports[co->index];
-	if (!sport)
-		return -ENODEV;
-
-	ret = clk_prepare_enable(sport->clk);
-	if (ret)
-		return ret;
-
-	if (options)
-		uart_parse_options(options, &baud, &parity, &bits, &flow);
-
-	return uart_set_options(&sport->port, co, baud, parity, bits, flow);
-}
-
-static int esp32_uart_console_exit(struct console *co)
-{
-	struct esp32_port *sport = esp32_uart_ports[co->index];
-
-	clk_disable_unprepare(sport->clk);
-	return 0;
-}
-
-static struct uart_driver esp32_uart_reg;
-static struct console esp32_uart_console = {
-	.name		= DEV_NAME,
-	.write		= esp32_uart_console_write,
-	.device		= uart_console_device,
-	.setup		= esp32_uart_console_setup,
-	.exit		= esp32_uart_console_exit,
-	.flags		= CON_PRINTBUFFER,
-	.index		= -1,
-	.data		= &esp32_uart_reg,
-};
-
-static void esp32_uart_earlycon_putchar(struct uart_port *port, u8 c)
-{
-	esp32_uart_put_char_sync(port, c);
-}
-
-static void esp32_uart_earlycon_write(struct console *con, const char *s,
-				      unsigned int n)
-{
-	struct earlycon_device *dev = con->data;
-
-	uart_console_write(&dev->port, s, n, esp32_uart_earlycon_putchar);
-}
-
-#ifdef CONFIG_CONSOLE_POLL
-static int esp32_uart_earlycon_read(struct console *con, char *s, unsigned int n)
-{
-	struct earlycon_device *dev = con->data;
-	unsigned int num_read = 0;
-
-	while (num_read < n) {
-		int c = esp32_uart_poll_get_char(&dev->port);
-
-		if (c == NO_POLL_CHAR)
-			break;
-		s[num_read++] = c;
-	}
-	return num_read;
-}
-#endif
-
-static int __init esp32xx_uart_early_console_setup(struct earlycon_device *device,
-						   const char *options)
-{
-	if (!device->port.membase)
-		return -ENODEV;
-
-	device->con->write = esp32_uart_earlycon_write;
-#ifdef CONFIG_CONSOLE_POLL
-	device->con->read = esp32_uart_earlycon_read;
-#endif
-	if (device->port.uartclk != BASE_BAUD * 16)
-		esp32_uart_set_baud(&device->port, device->baud);
-
-	return 0;
-}
-
-static int __init esp32_uart_early_console_setup(struct earlycon_device *device,
-						 const char *options)
-{
-	device->port.private_data = (void *)&esp32_variant;
-
-	return esp32xx_uart_early_console_setup(device, options);
-}
-
-OF_EARLYCON_DECLARE(esp32uart, "esp,esp32-uart",
-		    esp32_uart_early_console_setup);
-
-static int __init esp32s3_uart_early_console_setup(struct earlycon_device *device,
-						   const char *options)
-{
-	device->port.private_data = (void *)&esp32s3_variant;
-
-	return esp32xx_uart_early_console_setup(device, options);
-}
-
-OF_EARLYCON_DECLARE(esp32s3uart, "esp,esp32s3-uart",
-		    esp32s3_uart_early_console_setup);
-
-static struct uart_driver esp32_uart_reg = {
-	.owner		= THIS_MODULE,
-	.driver_name	= DRIVER_NAME,
-	.dev_name	= DEV_NAME,
-	.nr		= ARRAY_SIZE(esp32_uart_ports),
-	.cons		= &esp32_uart_console,
-};
-
-static int esp32_uart_probe(struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	struct uart_port *port;
-	struct esp32_port *sport;
-	struct resource *res;
-	int ret;
-
-	sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);
-	if (!sport)
-		return -ENOMEM;
-
-	port = &sport->port;
-
-	ret = of_alias_get_id(np, "serial");
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);
-		return ret;
-	}
-	if (ret >= UART_NR) {
-		dev_err(&pdev->dev, "driver limited to %d serial ports\n", UART_NR);
-		return -ENOMEM;
-	}
-
-	port->line = ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	port->mapbase = res->start;
-	port->membase = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(port->membase))
-		return PTR_ERR(port->membase);
-
-	sport->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(sport->clk))
-		return PTR_ERR(sport->clk);
-
-	port->uartclk = clk_get_rate(sport->clk);
-	port->dev = &pdev->dev;
-	port->type = PORT_GENERIC;
-	port->iotype = UPIO_MEM;
-	port->irq = platform_get_irq(pdev, 0);
-	port->ops = &esp32_uart_pops;
-	port->flags = UPF_BOOT_AUTOCONF;
-	port->has_sysrq = 1;
-	port->fifosize = ESP32_UART_TX_FIFO_SIZE;
-	port->private_data = (void *)device_get_match_data(&pdev->dev);
-
-	esp32_uart_ports[port->line] = sport;
-
-	platform_set_drvdata(pdev, port);
-
-	return uart_add_one_port(&esp32_uart_reg, port);
-}
-
-static void esp32_uart_remove(struct platform_device *pdev)
-{
-	struct uart_port *port = platform_get_drvdata(pdev);
-
-	uart_remove_one_port(&esp32_uart_reg, port);
-}
-
-
-static struct platform_driver esp32_uart_driver = {
-	.probe		= esp32_uart_probe,
-	.remove		= esp32_uart_remove,
-	.driver		= {
-		.name	= DRIVER_NAME,
-		.of_match_table	= esp32_uart_dt_ids,
-	},
-};
-
-static int __init esp32_uart_init(void)
-{
-	int ret;
-
-	ret = uart_register_driver(&esp32_uart_reg);
-	if (ret)
-		return ret;
-
-	ret = platform_driver_register(&esp32_uart_driver);
-	if (ret)
-		uart_unregister_driver(&esp32_uart_reg);
-
-	return ret;
-}
-
-static void __exit esp32_uart_exit(void)
-{
-	platform_driver_unregister(&esp32_uart_driver);
-	uart_unregister_driver(&esp32_uart_reg);
-}
-
-module_init(esp32_uart_init);
-module_exit(esp32_uart_exit);
-
-MODULE_AUTHOR("Max Filippov <jcmvbkbc@gmail.com>");
-MODULE_DESCRIPTION("Espressif ESP32 UART support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
index c488e5d372ffd..251a50c8aa387 100644
--- a/drivers/tty/serial/imx.c
+++ b/drivers/tty/serial/imx.c
@@ -1442,9 +1442,9 @@ static void imx_uart_enable_dma(struct imx_port *sport)
 
 	imx_uart_setup_ufcr(sport, TXTL_DMA, RXTL_DMA);
 
-	/* set UCR1 */
+	/* set UCR1 except TXDMAEN which would be enabled in imx_uart_dma_tx */
 	ucr1 = imx_uart_readl(sport, UCR1);
-	ucr1 |= UCR1_RXDMAEN | UCR1_TXDMAEN | UCR1_ATDMAEN;
+	ucr1 |= UCR1_RXDMAEN | UCR1_ATDMAEN;
 	imx_uart_writel(sport, ucr1, UCR1);
 
 	sport->dma_is_enabled = 1;
@@ -1567,8 +1567,9 @@ static int imx_uart_startup(struct uart_port *port)
 	imx_uart_enable_ms(&sport->port);
 
 	if (dma_is_inited) {
-		imx_uart_enable_dma(sport);
+		/* Note: enable dma request after transfer start! */
 		imx_uart_start_rx_dma(sport);
+		imx_uart_enable_dma(sport);
 	} else {
 		ucr1 = imx_uart_readl(sport, UCR1);
 		ucr1 |= UCR1_RRDYEN;
diff --git a/drivers/tty/serial/ip22zilog.c b/drivers/tty/serial/ip22zilog.c
index 6e19c6713849a..a69b06893d9e3 100644
--- a/drivers/tty/serial/ip22zilog.c
+++ b/drivers/tty/serial/ip22zilog.c
@@ -1015,7 +1015,7 @@ static struct console ip22zilog_console = {
 
 static struct uart_driver ip22zilog_reg = {
 	.owner		= THIS_MODULE,
-	.driver_name	= "serial",
+	.driver_name	= "serial_ip22zilog",
 	.dev_name	= "ttyS",
 	.major		= TTY_MAJOR,
 	.minor		= 64,
diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c
index cc65c9fb6446c..693b491f1e755 100644
--- a/drivers/tty/serial/mxs-auart.c
+++ b/drivers/tty/serial/mxs-auart.c
@@ -1318,7 +1318,8 @@ auart_console_write(struct console *co, const char *str, unsigned int count)
 	s = auart_port[co->index];
 	port = &s->port;
 
-	clk_enable(s->clk);
+	if (clk_enable(s->clk))
+		return;
 
 	/* First save the CR then disable the interrupts */
 	old_ctrl2 = mxs_read(s, REG_CTRL2);
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index e6b0a55f0cfb2..9854bb2406e3f 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -146,6 +146,7 @@ struct qcom_geni_serial_port {
 	int wakeup_irq;
 	bool rx_tx_swap;
 	bool cts_rts_swap;
+	bool manual_flow;
 
 	struct qcom_geni_private_data private_data;
 	const struct qcom_geni_device_data *dev_data;
@@ -250,7 +251,7 @@ static void qcom_geni_serial_set_mctrl(struct uart_port *uport,
 	if (mctrl & TIOCM_LOOP)
 		port->loopback = RX_TX_CTS_RTS_SORTED;
 
-	if (!(mctrl & TIOCM_RTS) && !uport->suspended)
+	if (port->manual_flow && !(mctrl & TIOCM_RTS) && !uport->suspended)
 		uart_manual_rfr = UART_MANUAL_RFR_EN | UART_RFR_NOT_READY;
 	writel(uart_manual_rfr, uport->membase + SE_UART_MANUAL_RFR);
 }
@@ -1401,11 +1402,21 @@ static void qcom_geni_serial_set_termios(struct uart_port *uport,
 	else
 		stop_bit_len = TX_STOP_BIT_LEN_1;
 
-	/* flow control, clear the CTS_MASK bit if using flow control. */
-	if (termios->c_cflag & CRTSCTS)
+	/* Configure flow control based on CRTSCTS flag.
+	 * When CRTSCTS is set, use HW/auto flow control mode, where HW
+	 * controls the RTS/CTS pin based FIFO state.
+	 * When CRTSCTS is clear, the CTS pin value is ignored for TX
+	 * path and RTS pin can be set/cleared using registers, for RX
+	 * path.
+	 */
+
+	if (termios->c_cflag & CRTSCTS) {
 		tx_trans_cfg &= ~UART_CTS_MASK;
-	else
+		port->manual_flow = false;
+	} else {
 		tx_trans_cfg |= UART_CTS_MASK;
+		port->manual_flow = true;
+	}
 
 	if (baud) {
 		uart_update_timeout(uport, termios->c_cflag, baud);
diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c
index c1fabad6ba1fa..e27806bf2cf3e 100644
--- a/drivers/tty/serial/samsung_tty.c
+++ b/drivers/tty/serial/samsung_tty.c
@@ -1562,12 +1562,12 @@ static void s3c24xx_serial_set_termios(struct uart_port *port,
 		ulcon |= S3C2410_LCON_PNONE;
 	}
 
-	uart_port_lock_irqsave(port, &flags);
-
 	dev_dbg(port->dev,
 		"setting ulcon to %08x, brddiv to %d, udivslot %08x\n",
 		ulcon, quot, udivslot);
 
+	uart_port_lock_irqsave(port, &flags);
+
 	wr_regl(port, S3C2410_ULCON, ulcon);
 	wr_regl(port, S3C2410_UBRDIV, quot);
 
@@ -1587,12 +1587,6 @@ static void s3c24xx_serial_set_termios(struct uart_port *port,
 	if (ourport->info->has_divslot)
 		wr_regl(port, S3C2443_DIVSLOT, udivslot);
 
-	dev_dbg(port->dev,
-		"uart: ulcon = 0x%08x, ucon = 0x%08x, ufcon = 0x%08x\n",
-		rd_regl(port, S3C2410_ULCON),
-		rd_regl(port, S3C2410_UCON),
-		rd_regl(port, S3C2410_UFCON));
-
 	/*
 	 * Update the per-port timeout.
 	 */
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 487756947a962..89cebdd278410 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -643,7 +643,10 @@ static unsigned int uart_write_room(struct tty_struct *tty)
 	unsigned int ret;
 
 	port = uart_port_ref_lock(state, &flags);
-	ret = kfifo_avail(&state->port.xmit_fifo);
+	if (!state->port.xmit_buf)
+		ret = 0;
+	else
+		ret = kfifo_avail(&state->port.xmit_fifo);
 	uart_port_unlock_deref(port, flags);
 	return ret;
 }
diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c
index 39c1fd1ff9ced..6240c3d4dfd79 100644
--- a/drivers/tty/serial/uartlite.c
+++ b/drivers/tty/serial/uartlite.c
@@ -878,6 +878,7 @@ static int ulite_probe(struct platform_device *pdev)
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_autosuspend_delay(&pdev->dev, UART_AUTOSUSPEND_TIMEOUT);
 	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_get_noresume(&pdev->dev);
 	pm_runtime_enable(&pdev->dev);
 
 	ret = ulite_assign(&pdev->dev, id, res->start, irq, pdata);
diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c
index 79ea7108a0f33..72a3c0d90f409 100644
--- a/drivers/tty/serial/zs.c
+++ b/drivers/tty/serial/zs.c
@@ -1252,7 +1252,7 @@ console_initcall(zs_serial_console_init);
 
 static struct uart_driver zs_reg = {
 	.owner			= THIS_MODULE,
-	.driver_name		= "serial",
+	.driver_name		= "serial_zs",
 	.dev_name		= "ttyS",
 	.major			= TTY_MAJOR,
 	.minor			= 64,
diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c
index 79ec953824d50..96be90db53b79 100644
--- a/drivers/tty/tty_buffer.c
+++ b/drivers/tty/tty_buffer.c
@@ -59,6 +59,13 @@ void tty_buffer_lock_exclusive(struct tty_port *port)
 }
 EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive);
 
+static bool tty_buffer_queue_work(struct tty_bufhead *buf)
+{
+	struct workqueue_struct *flip_wq = READ_ONCE(buf->flip_wq);
+
+	return queue_work(flip_wq ?: system_dfl_wq, &buf->work);
+}
+
 /**
  * tty_buffer_unlock_exclusive	-	release exclusive access
  * @port: tty port owning the flip buffer
@@ -76,7 +83,7 @@ void tty_buffer_unlock_exclusive(struct tty_port *port)
 	mutex_unlock(&buf->lock);
 
 	if (restart)
-		queue_work(system_dfl_wq, &buf->work);
+		tty_buffer_queue_work(buf);
 }
 EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive);
 
@@ -530,7 +537,7 @@ void tty_flip_buffer_push(struct tty_port *port)
 	struct tty_bufhead *buf = &port->buf;
 
 	tty_flip_buffer_commit(buf->tail);
-	queue_work(system_dfl_wq, &buf->work);
+	tty_buffer_queue_work(buf);
 }
 EXPORT_SYMBOL(tty_flip_buffer_push);
 
@@ -560,7 +567,7 @@ int tty_insert_flip_string_and_push_buffer(struct tty_port *port,
 		tty_flip_buffer_commit(buf->tail);
 	spin_unlock_irqrestore(&port->lock, flags);
 
-	queue_work(system_dfl_wq, &buf->work);
+	tty_buffer_queue_work(buf);
 
 	return size;
 }
@@ -613,7 +620,7 @@ void tty_buffer_set_lock_subclass(struct tty_port *port)
 
 bool tty_buffer_restart_work(struct tty_port *port)
 {
-	return queue_work(system_dfl_wq, &port->buf.work);
+	return tty_buffer_queue_work(&port->buf);
 }
 
 bool tty_buffer_cancel_work(struct tty_port *port)
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index a5d0457e0e284..6b283fd03ff82 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -3443,10 +3443,27 @@ int tty_register_driver(struct tty_driver *driver)
 	if (error < 0)
 		goto err;
 
+	/*
+	 * Drivers that do not define driver_name are potentially in-memory devices
+	 * like vty, which generally do not require special workqueue settings.
+	 */
+	if (!(driver->flags & TTY_DRIVER_NO_WORKQUEUE) && driver->driver_name) {
+		driver->flip_wq = alloc_workqueue("%s-%s", WQ_UNBOUND | WQ_SYSFS,
+						  0, driver->name, driver->driver_name);
+		if (!driver->flip_wq) {
+			error = -ENOMEM;
+			goto err_unreg_char;
+		}
+		for (i = 0; i < driver->num; i++) {
+			if (driver->ports[i])
+				tty_port_link_driver_wq(driver->ports[i], driver);
+		}
+	}
+
 	if (driver->flags & TTY_DRIVER_DYNAMIC_ALLOC) {
 		error = tty_cdev_add(driver, dev, 0, driver->num);
 		if (error)
-			goto err_unreg_char;
+			goto err_destroy_wq;
 	}
 
 	scoped_guard(mutex, &tty_mutex)
@@ -3472,6 +3489,10 @@ int tty_register_driver(struct tty_driver *driver)
 	scoped_guard(mutex, &tty_mutex)
 		list_del(&driver->tty_drivers);
 
+err_destroy_wq:
+	if (driver->flip_wq)
+		destroy_workqueue(driver->flip_wq);
+
 err_unreg_char:
 	unregister_chrdev_region(dev, driver->num);
 err:
@@ -3491,6 +3512,8 @@ void tty_unregister_driver(struct tty_driver *driver)
 				driver->num);
 	scoped_guard(mutex, &tty_mutex)
 		list_del(&driver->tty_drivers);
+	if (driver->flip_wq)
+		destroy_workqueue(driver->flip_wq);
 }
 EXPORT_SYMBOL(tty_unregister_driver);
 
diff --git a/drivers/tty/tty_ldisc.c b/drivers/tty/tty_ldisc.c
index 888f2f8f94819..27fe8236f6623 100644
--- a/drivers/tty/tty_ldisc.c
+++ b/drivers/tty/tty_ldisc.c
@@ -44,7 +44,7 @@ enum {
 
 static DEFINE_RAW_SPINLOCK(tty_ldiscs_lock);
 /* Line disc dispatch table */
-static struct tty_ldisc_ops *tty_ldiscs[NR_LDISCS];
+static const struct tty_ldisc_ops *tty_ldiscs[NR_LDISCS];
 
 /**
  * tty_register_ldisc	-	install a line discipline
@@ -55,7 +55,7 @@ static struct tty_ldisc_ops *tty_ldiscs[NR_LDISCS];
  *
  * Locking: takes %tty_ldiscs_lock to guard against ldisc races
  */
-int tty_register_ldisc(struct tty_ldisc_ops *new_ldisc)
+int tty_register_ldisc(const struct tty_ldisc_ops *new_ldisc)
 {
 	unsigned long flags;
 
@@ -80,7 +80,7 @@ EXPORT_SYMBOL(tty_register_ldisc);
  * Locking: takes %tty_ldiscs_lock to guard against ldisc races
  */
 
-void tty_unregister_ldisc(struct tty_ldisc_ops *ldisc)
+void tty_unregister_ldisc(const struct tty_ldisc_ops *ldisc)
 {
 	unsigned long flags;
 
@@ -90,10 +90,10 @@ void tty_unregister_ldisc(struct tty_ldisc_ops *ldisc)
 }
 EXPORT_SYMBOL(tty_unregister_ldisc);
 
-static struct tty_ldisc_ops *get_ldops(int disc)
+static const struct tty_ldisc_ops *get_ldops(int disc)
 {
 	unsigned long flags;
-	struct tty_ldisc_ops *ldops, *ret;
+	const struct tty_ldisc_ops *ldops, *ret;
 
 	raw_spin_lock_irqsave(&tty_ldiscs_lock, flags);
 	ret = ERR_PTR(-EINVAL);
@@ -107,7 +107,7 @@ static struct tty_ldisc_ops *get_ldops(int disc)
 	return ret;
 }
 
-static void put_ldops(struct tty_ldisc_ops *ldops)
+static void put_ldops(const struct tty_ldisc_ops *ldops)
 {
 	unsigned long flags;
 
@@ -139,7 +139,7 @@ int tty_ldisc_autoload = IS_BUILTIN(CONFIG_LDISC_AUTOLOAD);
 static struct tty_ldisc *tty_ldisc_get(struct tty_struct *tty, int disc)
 {
 	struct tty_ldisc *ld;
-	struct tty_ldisc_ops *ldops;
+	const struct tty_ldisc_ops *ldops;
 
 	if (disc < N_TTY || disc >= NR_LDISCS)
 		return ERR_PTR(-EINVAL);
@@ -202,7 +202,7 @@ static void tty_ldiscs_seq_stop(struct seq_file *m, void *v)
 static int tty_ldiscs_seq_show(struct seq_file *m, void *v)
 {
 	int i = *(loff_t *)v;
-	struct tty_ldisc_ops *ldops;
+	const struct tty_ldisc_ops *ldops;
 
 	ldops = get_ldops(i);
 	if (IS_ERR(ldops))
diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c
index fe67c5cb0a3f0..54359310e293c 100644
--- a/drivers/tty/tty_port.c
+++ b/drivers/tty/tty_port.c
@@ -99,6 +99,23 @@ void tty_port_init(struct tty_port *port)
 }
 EXPORT_SYMBOL(tty_port_init);
 
+/**
+ * tty_port_link_wq - link tty_port and flip workqueue
+ * @port: tty_port of the device
+ * @flip_wq: workqueue to queue flip buffer work on
+ *
+ * Whenever %TTY_DRIVER_NO_WORKQUEUE is used, every tty_port can be linked to
+ * a workqueue manually by this function.
+ * tty_port will use system_dfl_wq when buf.flip_wq is NULL.
+ *
+ * Note that tty_port API will NOT destroy the workqueue.
+ */
+void tty_port_link_wq(struct tty_port *port, struct workqueue_struct *flip_wq)
+{
+	port->buf.flip_wq = flip_wq;
+}
+EXPORT_SYMBOL_GPL(tty_port_link_wq);
+
 /**
  * tty_port_link_device - link tty and tty_port
  * @port: tty_port of the device
@@ -157,6 +174,7 @@ struct device *tty_port_register_device_attr(struct tty_port *port,
 		const struct attribute_group **attr_grp)
 {
 	tty_port_link_device(port, driver, index);
+	tty_port_link_driver_wq(port, driver);
 	return tty_register_device_attr(driver, index, device, drvdata,
 			attr_grp);
 }
@@ -183,6 +201,7 @@ struct device *tty_port_register_device_attr_serdev(struct tty_port *port,
 	struct device *dev;
 
 	tty_port_link_device(port, driver, index);
+	tty_port_link_driver_wq(port, driver);
 
 	dev = serdev_tty_port_register(port, host, parent, driver, index);
 	if (PTR_ERR(dev) != -ENODEV) {
@@ -210,6 +229,7 @@ void tty_port_unregister_device(struct tty_port *port,
 {
 	int ret;
 
+	WRITE_ONCE(port->buf.flip_wq, NULL);
 	ret = serdev_tty_port_unregister(port);
 	if (ret == 0)
 		return;
@@ -257,6 +277,7 @@ void tty_port_destroy(struct tty_port *port)
 {
 	tty_buffer_cancel_work(port);
 	tty_buffer_free_all(port);
+	WRITE_ONCE(port->buf.flip_wq, NULL);
 }
 EXPORT_SYMBOL(tty_port_destroy);
 
@@ -703,6 +724,7 @@ int tty_port_install(struct tty_port *port, struct tty_driver *driver,
 		struct tty_struct *tty)
 {
 	tty->port = port;
+	tty_port_link_driver_wq(port, driver);
 	return tty_standard_install(driver, tty);
 }
 EXPORT_SYMBOL_GPL(tty_port_install);
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index 13bc048f45e86..dfdea08421495 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -74,7 +74,7 @@ static inline int kbd_defleds(void)
 	k_self,		k_fn,		k_spec,		k_pad,\
 	k_dead,		k_cons,		k_cur,		k_shift,\
 	k_meta,		k_ascii,	k_lock,		k_lowercase,\
-	k_slock,	k_dead2,	k_brl,		k_ignore
+	k_slock,	k_dead2,	k_brl,		k_csi
 
 typedef void (k_handler_fn)(struct vc_data *vc, unsigned char value,
 			    char up_flag);
@@ -127,6 +127,7 @@ static const unsigned char max_vals[] = {
 	[ KT_SLOCK	] = NR_LOCK - 1,
 	[ KT_DEAD2	] = 255,
 	[ KT_BRL	] = NR_BRL - 1,
+	[ KT_CSI	] = 99,
 };
 
 static const int NR_TYPES = ARRAY_SIZE(max_vals);
@@ -644,10 +645,6 @@ static void fn_null(struct vc_data *vc)
 /*
  * Special key handlers
  */
-static void k_ignore(struct vc_data *vc, unsigned char value, char up_flag)
-{
-}
-
 static void k_spec(struct vc_data *vc, unsigned char value, char up_flag)
 {
 	if (up_flag)
@@ -765,14 +762,39 @@ static void k_fn(struct vc_data *vc, unsigned char value, char up_flag)
 		pr_err("k_fn called with value=%d\n", value);
 }
 
+/*
+ * Compute xterm-style modifier parameter for CSI sequences.
+ * Returns 1 + (shift ? 1 : 0) + (alt ? 2 : 0) + (ctrl ? 4 : 0)
+ */
+static int csi_modifier_param(void)
+{
+	int mod = 1;
+
+	if (shift_state & (BIT(KG_SHIFT) | BIT(KG_SHIFTL) | BIT(KG_SHIFTR)))
+		mod += 1;
+	if (shift_state & (BIT(KG_ALT) | BIT(KG_ALTGR)))
+		mod += 2;
+	if (shift_state & (BIT(KG_CTRL) | BIT(KG_CTRLL) | BIT(KG_CTRLR)))
+		mod += 4;
+	return mod;
+}
+
 static void k_cur(struct vc_data *vc, unsigned char value, char up_flag)
 {
 	static const char cur_chars[] = "BDCA";
+	int mod;
 
 	if (up_flag)
 		return;
 
-	applkey(vc, cur_chars[value], vc_kbd_mode(kbd, VC_CKMODE));
+	mod = csi_modifier_param();
+	if (mod > 1) {
+		char buf[] = { 0x1b, '[', '1', ';', '0' + mod, cur_chars[value], 0x00 };
+
+		puts_queue(vc, buf);
+	} else {
+		applkey(vc, cur_chars[value], vc_kbd_mode(kbd, VC_CKMODE));
+	}
 }
 
 static void k_pad(struct vc_data *vc, unsigned char value, char up_flag)
@@ -1004,6 +1026,37 @@ static void k_brl(struct vc_data *vc, unsigned char value, char up_flag)
 	}
 }
 
+/*
+ * Handle KT_CSI keysym type: generate CSI tilde sequences with modifier
+ * support. The value encodes the CSI parameter number, producing sequences
+ * like ESC [ <value> ~ or ESC [ <value> ; <mod> ~ when modifiers are held.
+ */
+static void k_csi(struct vc_data *vc, unsigned char value, char up_flag)
+{
+	char buf[10];
+	int i = 0;
+	int mod;
+
+	if (up_flag)
+		return;
+
+	mod = csi_modifier_param();
+
+	buf[i++] = 0x1b;
+	buf[i++] = '[';
+	if (value >= 10)
+		buf[i++] = '0' + value / 10;
+	buf[i++] = '0' + value % 10;
+	if (mod > 1) {
+		buf[i++] = ';';
+		buf[i++] = '0' + mod;
+	}
+	buf[i++] = '~';
+	buf[i] = 0x00;
+
+	puts_queue(vc, buf);
+}
+
 #if IS_ENABLED(CONFIG_INPUT_LEDS) && IS_ENABLED(CONFIG_LEDS_TRIGGERS)
 
 struct kbd_led_trigger {
@@ -1445,6 +1498,21 @@ static void kbd_keycode(unsigned int keycode, int down, bool hw_raw)
 	param.ledstate = kbd->ledflagstate;
 	key_map = key_maps[shift_final];
 
+	/*
+	 * Fall back to the plain map if modifiers are active, the modifier-
+	 * specific map is missing or has no entry, and the plain map has a
+	 * modifier-aware key type (KT_CUR or KT_CSI). These handlers encode
+	 * the modifier state into the emitted escape sequence.
+	 */
+	if (shift_final && keycode < NR_KEYS &&
+	    (!key_map || key_map[keycode] == K_HOLE) && key_maps[0]) {
+		unsigned short plain = key_maps[0][keycode];
+		unsigned char type = KTYP(plain);
+
+		if (type >= 0xf0 && (type - 0xf0 == KT_CUR || type - 0xf0 == KT_CSI))
+			key_map = key_maps[0];
+	}
+
 	rc = atomic_notifier_call_chain(&keyboard_notifier_list,
 					KBD_KEYCODE, &param);
 	if (rc == NOTIFY_STOP || !key_map) {
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index c1f152d8b03b6..a9d904f474a0e 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -1339,6 +1339,8 @@ struct vc_data *vc_deallocate(unsigned int currcons)
 			kfree(vc->vc_saved_screen);
 			vc->vc_saved_screen = NULL;
 		}
+		vc_uniscr_free(vc->vc_saved_uni_lines);
+		vc->vc_saved_uni_lines = NULL;
 	}
 	return vc;
 }
@@ -1644,9 +1646,7 @@ static void rgb_background(struct vc_data *vc, const struct rgb *c)
 
 /*
  * ITU T.416 Higher colour modes. They break the usual properties of SGR codes
- * and thus need to be detected and ignored by hand. That standard also
- * wants : rather than ; as separators but sequences containing : are currently
- * completely ignored by the parser.
+ * and thus need to be detected and ignored by hand.
  *
  * Subcommands 3 (CMY) and 4 (CMYK) are so insane there's no point in
  * supporting them.
@@ -1703,6 +1703,7 @@ enum {
 	CSI_m_BG_COLOR_END		= 47,
 	CSI_m_BG_COLOR			= 48,
 	CSI_m_DEFAULT_BG_COLOR		= 49,
+	CSI_m_UNDERLINE_COLOR		= 58,
 	CSI_m_BRIGHT_FG_COLOR_BEG	= 90,
 	CSI_m_BRIGHT_FG_COLOR_END	= 97,
 	CSI_m_BRIGHT_FG_COLOR_OFF	= CSI_m_BRIGHT_FG_COLOR_BEG - CSI_m_FG_COLOR_BEG,
@@ -1884,6 +1885,8 @@ static void enter_alt_screen(struct vc_data *vc)
 	vc->vc_saved_screen = kmemdup((u16 *)vc->vc_origin, size, GFP_KERNEL);
 	if (vc->vc_saved_screen == NULL)
 		return;
+	vc->vc_saved_uni_lines = vc->vc_uni_lines;
+	vc->vc_uni_lines = NULL;
 	vc->vc_saved_rows = vc->vc_rows;
 	vc->vc_saved_cols = vc->vc_cols;
 	save_cur(vc);
@@ -1905,6 +1908,8 @@ static void leave_alt_screen(struct vc_data *vc)
 		dest = ((u16 *)vc->vc_origin) + r * vc->vc_cols;
 		memcpy(dest, src, 2 * cols);
 	}
+	vc_uniscr_set(vc, vc->vc_saved_uni_lines);
+	vc->vc_saved_uni_lines = NULL;
 	restore_cur(vc);
 	/* Update the entire screen */
 	if (con_should_update(vc))
@@ -2160,6 +2165,7 @@ static void restore_cur(struct vc_data *vc)
  * @ESesc:		ESC parsed
  * @ESsquare:		CSI parsed -- modifiers/parameters/ctrl chars expected
  * @ESgetpars:		CSI parsed -- parameters/ctrl chars expected
+ * @ESgetsubpars:	CSI m parsed -- subparameters expected
  * @ESfunckey:		CSI [ parsed
  * @EShash:		ESC # parsed
  * @ESsetG0:		ESC ( parsed
@@ -2180,6 +2186,7 @@ enum vc_ctl_state {
 	ESesc,
 	ESsquare,
 	ESgetpars,
+	ESgetsubpars,
 	ESfunckey,
 	EShash,
 	ESsetG0,
@@ -2227,6 +2234,8 @@ static void reset_terminal(struct vc_data *vc, int do_clear)
 	if (vc->vc_saved_screen != NULL) {
 		kfree(vc->vc_saved_screen);
 		vc->vc_saved_screen = NULL;
+		vc_uniscr_free(vc->vc_saved_uni_lines);
+		vc->vc_saved_uni_lines = NULL;
 		vc->vc_saved_rows = 0;
 		vc->vc_saved_cols = 0;
 	}
@@ -2699,6 +2708,47 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, u8 c)
 		fallthrough;
 	case ESgetpars: /* ESC [ aka CSI, parameters expected */
 		switch (c) {
+		case ':': /* ITU-T T.416 color subparameters */
+			if (vc->vc_par[vc->vc_npar] == CSI_m_FG_COLOR ||
+			    vc->vc_par[vc->vc_npar] == CSI_m_BG_COLOR ||
+			    vc->vc_par[vc->vc_npar] == CSI_m_UNDERLINE_COLOR)
+				vc->vc_state = ESgetsubpars;
+			else
+				break;
+			fallthrough;
+		case ';':
+			if (vc->vc_npar < NPAR - 1) {
+				vc->vc_npar++;
+				return;
+			}
+			break;
+		case '0' ... '9':
+			vc->vc_par[vc->vc_npar] *= 10;
+			vc->vc_par[vc->vc_npar] += c - '0';
+			return;
+		}
+		if (c >= ASCII_CSI_IGNORE_FIRST && c <= ASCII_CSI_IGNORE_LAST) {
+			vc->vc_state = EScsiignore;
+			return;
+		}
+
+		/* parameters done, handle the control char @c */
+
+		vc->vc_state = ESnormal;
+
+		switch (vc->vc_priv) {
+		case EPdec:
+			csi_DEC(tty, vc, c);
+			return;
+		case EPecma:
+			csi_ECMA(tty, vc, c);
+			return;
+		default:
+			return;
+		}
+	case ESgetsubpars: /* ESC [ 38/48/58, subparameters expected */
+		switch (c) {
+		case ':':
 		case ';':
 			if (vc->vc_npar < NPAR - 1) {
 				vc->vc_npar++;
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index 18a95b728633f..1b30625779458 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -31,7 +31,8 @@
 
 #define UFSHCD_ENABLE_MCQ_INTRS	(UTP_TASK_REQ_COMPL |\
 				 UFSHCD_ERROR_MASK |\
-				 MCQ_CQ_EVENT_STATUS)
+				 MCQ_CQ_EVENT_STATUS |\
+				 MCQ_IAG_EVENT_STATUS)
 
 /* Max mcq register polling time in microseconds */
 #define MCQ_POLL_US 500000
@@ -272,14 +273,29 @@ void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i)
 }
 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis);
 
+u32 ufshcd_mcq_read_mcqiacr(struct ufs_hba *hba, int i)
+{
+	return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_MCQIACR);
+}
+
+void ufshcd_mcq_write_mcqiacr(struct ufs_hba *hba, u32 val, int i)
+{
+	writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_MCQIACR);
+}
+
 /*
- * Current MCQ specification doesn't provide a Task Tag or its equivalent in
+ * UFSHCI 4.0 MCQ specification doesn't provide a Task Tag or its equivalent in
  * the Completion Queue Entry. Find the Task Tag using an indirect method.
+ * UFSHCI 4.1 and above can directly return the Task Tag in the Completion Queue
+ * Entry.
  */
 static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe)
 {
 	u64 addr;
 
+	if (hba->ufs_version >= ufshci_version(4, 1))
+		return cqe->task_tag;
+
 	/* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */
 	BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0));
 
@@ -301,6 +317,8 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
 		ufshcd_compl_one_cqe(hba, tag, cqe);
 		/* After processed the cqe, mark it empty (invalid) entry */
 		cqe->command_desc_base_addr = 0;
+	} else {
+		dev_err(hba->dev, "Abnormal CQ entry!\n");
 	}
 }
 
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 384d958615d76..99af3c73f1af7 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -605,6 +605,34 @@ static ssize_t device_lvl_exception_id_show(struct device *dev,
 	return sysfs_emit(buf, "%llu\n", exception_id);
 }
 
+static ssize_t dme_qos_notification_show(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "0x%x\n", atomic_read(&hba->dme_qos_notification));
+}
+
+static ssize_t dme_qos_notification_store(struct device *dev,
+					  struct device_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+	unsigned int value;
+
+	if (kstrtouint(buf, 0, &value))
+		return -EINVAL;
+
+	/* the only supported usecase is to reset the dme_qos_notification */
+	if (value)
+		return -EINVAL;
+
+	atomic_set(&hba->dme_qos_notification, 0);
+
+	return count;
+}
+
 static DEVICE_ATTR_RW(rpm_lvl);
 static DEVICE_ATTR_RO(rpm_target_dev_state);
 static DEVICE_ATTR_RO(rpm_target_link_state);
@@ -621,6 +649,7 @@ static DEVICE_ATTR_RW(pm_qos_enable);
 static DEVICE_ATTR_RO(critical_health);
 static DEVICE_ATTR_RW(device_lvl_exception_count);
 static DEVICE_ATTR_RO(device_lvl_exception_id);
+static DEVICE_ATTR_RW(dme_qos_notification);
 
 static struct attribute *ufs_sysfs_ufshcd_attrs[] = {
 	&dev_attr_rpm_lvl.attr,
@@ -639,6 +668,7 @@ static struct attribute *ufs_sysfs_ufshcd_attrs[] = {
 	&dev_attr_critical_health.attr,
 	&dev_attr_device_lvl_exception_count.attr,
 	&dev_attr_device_lvl_exception_id.attr,
+	&dev_attr_dme_qos_notification.attr,
 	NULL
 };
 
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 37c32071e7540..6d3d14e883b8b 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -76,6 +76,8 @@ void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
 int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
+u32 ufshcd_mcq_read_mcqiacr(struct ufs_hba *hba, int i);
+void ufshcd_mcq_write_mcqiacr(struct ufs_hba *hba, u32 val, int i);
 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
 void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
 
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 9ceb6d6d479d0..a930e9afc8aba 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -5580,8 +5580,11 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
 
 	guard(spinlock_irqsave)(hba->host->host_lock);
 	cmd = hba->active_uic_cmd;
-	if (!cmd)
+	if (!cmd) {
+		dev_err(hba->dev,
+			"No active UIC command. Maybe a timeout occurred?\n");
 		return retval;
+	}
 
 	if (ufshcd_is_auto_hibern8_error(hba, intr_status))
 		hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
@@ -6976,10 +6979,19 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
 	}
 
 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
-	if ((reg & UIC_DME_ERROR) &&
-	    (reg & UIC_DME_ERROR_CODE_MASK)) {
+	if (reg & UIC_DME_ERROR) {
 		ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
-		hba->uic_error |= UFSHCD_UIC_DME_ERROR;
+
+		if (reg & UIC_DME_ERROR_CODE_MASK)
+			hba->uic_error |= UFSHCD_UIC_DME_ERROR;
+
+		if (reg & UIC_DME_QOS_MASK) {
+			atomic_set(&hba->dme_qos_notification,
+				   reg & UIC_DME_QOS_MASK);
+			if (hba->dme_qos_sysfs_handle)
+				sysfs_notify_dirent(hba->dme_qos_sysfs_handle);
+		}
+
 		retval |= IRQ_HANDLED;
 	}
 
@@ -7097,16 +7109,17 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
 /**
  * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
  * @hba: per adapter instance
+ * @reset_iag: true, to reset MCQ IAG counter and timer of the CQ
  *
  * Return: IRQ_HANDLED if interrupt is handled.
  */
-static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
+static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba, bool reset_iag)
 {
 	struct ufs_hw_queue *hwq;
 	unsigned long outstanding_cqs;
 	unsigned int nr_queues;
 	int i, ret;
-	u32 events;
+	u32 events, reg;
 
 	ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs);
 	if (ret)
@@ -7121,6 +7134,12 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
 		if (events)
 			ufshcd_mcq_write_cqis(hba, events, i);
 
+		if (reset_iag) {
+			reg = ufshcd_mcq_read_mcqiacr(hba, i);
+			reg |= INT_AGGR_COUNTER_AND_TIMER_RESET;
+			ufshcd_mcq_write_mcqiacr(hba, reg, i);
+		}
+
 		if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
 			ufshcd_mcq_poll_cqe_lock(hba, hwq);
 	}
@@ -7154,7 +7173,10 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
 		retval |= ufshcd_transfer_req_compl(hba);
 
 	if (intr_status & MCQ_CQ_EVENT_STATUS)
-		retval |= ufshcd_handle_mcq_cq_events(hba);
+		retval |= ufshcd_handle_mcq_cq_events(hba, false);
+
+	if (intr_status & MCQ_IAG_EVENT_STATUS)
+		retval |= ufshcd_handle_mcq_cq_events(hba, true);
 
 	return retval;
 }
@@ -7222,8 +7244,12 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba)
 	struct ufs_hba *hba = __hba;
 	u32 intr_status, enabled_intr_status;
 
-	/* Move interrupt handling to thread when MCQ & ESI are not enabled */
-	if (!hba->mcq_enabled || !hba->mcq_esi_enabled)
+	/*
+	 * Handle interrupt in thread if MCQ or ESI is disabled,
+	 * and no active UIC command.
+	 */
+	if ((!hba->mcq_enabled || !hba->mcq_esi_enabled) &&
+	    !hba->active_uic_cmd)
 		return IRQ_WAKE_THREAD;
 
 	intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
@@ -9111,6 +9137,12 @@ static int ufshcd_post_device_init(struct ufs_hba *hba)
 
 	/* UFS device is also active now */
 	ufshcd_set_ufs_dev_active(hba);
+
+	/* Indicate that DME QoS Monitor has been reset */
+	atomic_set(&hba->dme_qos_notification, 0x1);
+	if (hba->dme_qos_sysfs_handle)
+		sysfs_notify_dirent(hba->dme_qos_sysfs_handle);
+
 	ufshcd_force_reset_auto_bkops(hba);
 
 	ufshcd_set_timestamp_attr(hba);
@@ -9743,6 +9775,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba)
 		hba->is_powered = false;
 		ufs_put_device_desc(hba);
 	}
+	sysfs_put(hba->dme_qos_sysfs_handle);
 }
 
 static int ufshcd_execute_start_stop(struct scsi_device *sdev,
@@ -9942,11 +9975,13 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
 #ifdef CONFIG_PM
 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
 {
+	bool vcc_on = false;
 	int ret = 0;
 
 	if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
 	    !hba->dev_info.is_lu_power_on_wp) {
 		ret = ufshcd_setup_vreg(hba, true);
+		vcc_on = true;
 	} else if (!ufshcd_is_ufs_dev_active(hba)) {
 		if (!ufshcd_is_link_active(hba)) {
 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
@@ -9957,6 +9992,7 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
 				goto vccq_lpm;
 		}
 		ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
+		vcc_on = true;
 	}
 	goto out;
 
@@ -9965,6 +10001,15 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
 vcc_disable:
 	ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
 out:
+	/*
+	 * On platforms with a slow VCC ramp-up, a delay is needed after
+	 * turning on VCC to ensure the voltage is stable before the
+	 * reference clock is enabled.
+	 */
+	if (hba->quirks & UFSHCD_QUIRK_VCC_ON_DELAY && !ret && vcc_on &&
+	    hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on)
+		usleep_range(1000, 1100);
+
 	return ret;
 }
 #endif /* CONFIG_PM */
@@ -11070,6 +11115,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 		goto out_disable;
 
 	ufs_sysfs_add_nodes(hba->dev);
+	hba->dme_qos_sysfs_handle = sysfs_get_dirent(hba->dev->kobj.sd,
+						     "dme_qos_notification");
 	async_schedule(ufshcd_async_scan, hba);
 
 	device_enable_async_suspend(dev);
diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c
index b3daaa07e9252..4618d7834414d 100644
--- a/drivers/ufs/host/ufs-mediatek.c
+++ b/drivers/ufs/host/ufs-mediatek.c
@@ -1960,6 +1960,8 @@ static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
 
 static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
 {
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
 	ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
 
 	if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc) {
@@ -1971,6 +1973,15 @@ static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
 		hba->dev_quirks &= ~UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM;
 	}
 
+	/*
+	 * Add a delay after enabling UFS5 VCC to ensure the voltage
+	 * is stable before the refclk is enabled.
+	 */
+	if (hba->dev_info.wspecversion >= 0x0500 &&
+	    (host->ip_ver == IP_VER_MT6995_A0 ||
+	     host->ip_ver == IP_VER_MT6995_B0))
+		hba->quirks |= UFSHCD_QUIRK_VCC_ON_DELAY;
+
 	ufs_mtk_vreg_fix_vcc(hba);
 	ufs_mtk_vreg_fix_vccqx(hba);
 	ufs_mtk_fix_ahit(hba);
diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h
index 9747277f11e89..8547a6f049909 100644
--- a/drivers/ufs/host/ufs-mediatek.h
+++ b/drivers/ufs/host/ufs-mediatek.h
@@ -220,6 +220,10 @@ enum {
 	IP_VER_MT6991_B0 = 0x10470000,
 	IP_VER_MT6993    = 0x10480000,
 
+	/* UFSHCI 5.0 */
+	IP_VER_MT6995_A0 = 0x10490000,
+	IP_VER_MT6995_B0 = 0x10500000,
+
 	IP_VER_NONE      = 0xFFFFFFFF
 };
 
diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c
index 5f65dfad1a71a..63f6b36b912fc 100644
--- a/drivers/ufs/host/ufshcd-pci.c
+++ b/drivers/ufs/host/ufshcd-pci.c
@@ -695,6 +695,7 @@ static const struct pci_device_id ufshcd_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x7747), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
 	{ PCI_VDEVICE(INTEL, 0xE447), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
 	{ PCI_VDEVICE(INTEL, 0x4D47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0xD335), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
 	{ }	/* terminate list */
 };
 
diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c
index 5a4998e2caf87..1e4ade78ed849 100644
--- a/drivers/uio/uio.c
+++ b/drivers/uio/uio.c
@@ -850,8 +850,14 @@ static int uio_mmap(struct file *filep, struct vm_area_struct *vma)
 		goto out;
 	}
 
-	if (idev->info->mmap) {
-		ret = idev->info->mmap(idev->info, vma);
+	if (idev->info->mmap_prepare) {
+		struct vm_area_desc desc;
+
+		compat_set_desc_from_vma(&desc, filep, vma);
+		ret = idev->info->mmap_prepare(idev->info, &desc);
+		if (ret)
+			goto out;
+		ret = __compat_vma_mmap(&desc, vma);
 		goto out;
 	}
 
diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c
index 3f8e2e27697fb..29ec2d15ada86 100644
--- a/drivers/uio/uio_hv_generic.c
+++ b/drivers/uio/uio_hv_generic.c
@@ -154,15 +154,16 @@ static void hv_uio_rescind(struct vmbus_channel *channel)
  * The ring buffer is allocated as contiguous memory by vmbus_open
  */
 static int
-hv_uio_ring_mmap(struct vmbus_channel *channel, struct vm_area_struct *vma)
+hv_uio_ring_mmap_prepare(struct vmbus_channel *channel, struct vm_area_desc *desc)
 {
 	void *ring_buffer = page_address(channel->ringbuffer_page);
 
 	if (channel->state != CHANNEL_OPENED_STATE)
 		return -ENODEV;
 
-	return vm_iomap_memory(vma, virt_to_phys(ring_buffer),
-			       channel->ringbuffer_pagecount << PAGE_SHIFT);
+	mmap_action_simple_ioremap(desc, virt_to_phys(ring_buffer),
+			channel->ringbuffer_pagecount << PAGE_SHIFT);
+	return 0;
 }
 
 /* Callback from VMBUS subsystem when new channel created. */
@@ -183,7 +184,7 @@ hv_uio_new_channel(struct vmbus_channel *new_sc)
 	}
 
 	set_channel_read_mode(new_sc, HV_CALL_ISR);
-	ret = hv_create_ring_sysfs(new_sc, hv_uio_ring_mmap);
+	ret = hv_create_ring_sysfs(new_sc, hv_uio_ring_mmap_prepare);
 	if (ret) {
 		dev_err(device, "sysfs create ring bin file failed; %d\n", ret);
 		vmbus_close(new_sc);
@@ -366,7 +367,7 @@ hv_uio_probe(struct hv_device *dev,
 	 * or decoupled from uio_hv_generic probe. Userspace programs can make use of inotify
 	 * APIs to make sure that ring is created.
 	 */
-	hv_create_ring_sysfs(channel, hv_uio_ring_mmap);
+	hv_create_ring_sysfs(channel, hv_uio_ring_mmap_prepare);
 
 	hv_set_drvdata(dev, pdata);
 
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index 7ede29d4c7c13..cf3c3eede1a51 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -1225,6 +1225,12 @@ static int acm_probe(struct usb_interface *intf,
 		if (!data_interface || !control_interface)
 			return -ENODEV;
 		goto skip_normal_probe;
+	} else if (quirks == NO_UNION_12) {
+		data_interface = usb_ifnum_to_if(usb_dev, 2);
+		control_interface = usb_ifnum_to_if(usb_dev, 1);
+		if (!data_interface || !control_interface)
+			 return -ENODEV;
+		goto skip_normal_probe;
 	}
 
 	/* normal probing*/
@@ -1748,6 +1754,9 @@ static const struct usb_device_id acm_ids[] = {
 	{ USB_DEVICE(0x045b, 0x024D),	/* Renesas R-Car E3 USB Download mode */
 	.driver_info = DISABLE_ECHO,	/* Don't echo banner */
 	},
+	{ USB_DEVICE(0x04b8, 0x0d12),	/* EPSON HMD Com&Sens */
+	.driver_info = NO_UNION_12,	/* union descriptor is garbage */
+	},
 	{ USB_DEVICE(0x0e8d, 0x0003), /* FIREFLY, MediaTek Inc; andrey.arapov@gmail.com */
 	.driver_info = NO_UNION_NORMAL, /* has no union descriptor */
 	},
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 76f73853a60b6..25fd5329a8781 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -114,3 +114,4 @@ struct acm {
 #define SEND_ZERO_PACKET		BIT(6)
 #define DISABLE_ECHO			BIT(7)
 #define MISSING_CAP_BRK			BIT(8)
+#define NO_UNION_12			BIT(9)
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index d39bbfd7fd18a..bd9347804dec6 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -254,6 +254,9 @@ static int usbtmc_release(struct inode *inode, struct file *file)
 	list_del(&file_data->file_elem);
 
 	spin_unlock_irq(&file_data->data->dev_lock);
+
+	/* flush anchored URBs */
+	usbtmc_draw_down(file_data);
 	mutex_unlock(&file_data->data->io_mutex);
 
 	kref_put(&file_data->data->kref, usbtmc_delete);
diff --git a/drivers/usb/core/endpoint.c b/drivers/usb/core/endpoint.c
index 4137ab47f1cd0..e00eaf9e22cd0 100644
--- a/drivers/usb/core/endpoint.c
+++ b/drivers/usb/core/endpoint.c
@@ -26,14 +26,6 @@ struct ep_device {
 #define to_ep_device(_dev) \
 	container_of(_dev, struct ep_device, dev)
 
-struct ep_attribute {
-	struct attribute attr;
-	ssize_t (*show)(struct usb_device *,
-			struct usb_endpoint_descriptor *, char *);
-};
-#define to_ep_attribute(_attr) \
-	container_of(_attr, struct ep_attribute, attr)
-
 #define usb_ep_attr(field, format_string)			\
 static ssize_t field##_show(struct device *dev,			\
 			       struct device_attribute *attr,	\
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index dee842ea69311..89221f1ce7694 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -2403,7 +2403,7 @@ void usb_hcd_resume_root_hub (struct usb_hcd *hcd)
 	if (hcd->rh_registered) {
 		pm_wakeup_event(&hcd->self.root_hub->dev, 0);
 		set_bit(HCD_FLAG_WAKEUP_PENDING, &hcd->flags);
-		queue_work(pm_wq, &hcd->wakeup_work);
+		queue_work(system_freezable_wq, &hcd->wakeup_work);
 	}
 	spin_unlock_irqrestore (&hcd_root_hub_lock, flags);
 }
diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c
index 2ab120ce2fa85..75e2bfd744a98 100644
--- a/drivers/usb/core/message.c
+++ b/drivers/usb/core/message.c
@@ -1063,7 +1063,7 @@ int usb_string(struct usb_device *dev, int index, char *buf, size_t size)
 }
 EXPORT_SYMBOL_GPL(usb_string);
 
-/* one UTF-8-encoded 16-bit character has at most three bytes */
+/* one 16-bit character, when UTF-8-encoded, has at most three bytes */
 #define MAX_USB_STRING_SIZE (127 * 3 + 1)
 
 /**
@@ -1084,16 +1084,18 @@ char *usb_cache_string(struct usb_device *udev, int index)
 		return NULL;
 
 	buf = kmalloc(MAX_USB_STRING_SIZE, GFP_NOIO);
-	if (buf) {
-		len = usb_string(udev, index, buf, MAX_USB_STRING_SIZE);
-		if (len > 0) {
-			smallbuf = kmalloc(++len, GFP_NOIO);
-			if (!smallbuf)
-				return buf;
-			memcpy(smallbuf, buf, len);
-		}
+	if (!buf)
+		return NULL;
+
+	len = usb_string(udev, index, buf, MAX_USB_STRING_SIZE);
+	if (len <= 0) {
 		kfree(buf);
+		return NULL;
 	}
+
+	smallbuf = krealloc(buf, len + 1, GFP_NOIO);
+	if (unlikely(!smallbuf))
+		return buf;
 	return smallbuf;
 }
 EXPORT_SYMBOL_GPL(usb_cache_string);
diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c
index 44e38f922bc55..b1364f0c384ce 100644
--- a/drivers/usb/core/port.c
+++ b/drivers/usb/core/port.c
@@ -21,6 +21,20 @@ static int usb_port_block_power_off;
 
 static const struct attribute_group *port_dev_group[];
 
+static bool usb_port_allow_power_off(struct usb_device *hdev,
+				     struct usb_hub *hub,
+				     struct usb_port *port_dev)
+{
+	if (hub_is_port_power_switchable(hub))
+		return true;
+
+	if (!IS_ENABLED(CONFIG_ACPI))
+		return false;
+
+	return port_dev->connect_type == USB_PORT_CONNECT_TYPE_HARD_WIRED &&
+	       usb_acpi_power_manageable(hdev, port_dev->portnum - 1);
+}
+
 static ssize_t early_stop_show(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
@@ -141,6 +155,7 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr,
 		usb_disconnect(&port_dev->child);
 
 	rc = usb_hub_set_port_power(hdev, hub, port1, !disabled);
+	msleep(2 * hub_power_on_good_delay(hub));
 
 	if (disabled) {
 		usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION);
@@ -805,10 +820,10 @@ int usb_hub_create_port_device(struct usb_hub *hub, int port1)
 	device_enable_async_suspend(&port_dev->dev);
 
 	/*
-	 * Keep hidden the ability to enable port-poweroff if the hub
-	 * does not support power switching.
+	 * Keep hidden the ability to enable port-poweroff if neither the
+	 * USB hub nor platform firmware can manage downstream port power.
 	 */
-	if (!hub_is_port_power_switchable(hub))
+	if (!usb_port_allow_power_off(hdev, hub, port_dev))
 		return 0;
 
 	/* Attempt to let userspace take over the policy. */
diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
index 240b15bc52cbd..18169727a413e 100644
--- a/drivers/usb/dwc3/Kconfig
+++ b/drivers/usb/dwc3/Kconfig
@@ -150,6 +150,18 @@ config USB_DWC3_IMX8MP
 	  functionality.
 	  Say 'Y' or 'M' if you have one such device.
 
+config USB_DWC3_IMX
+	tristate "NXP iMX Platform"
+	depends on OF && COMMON_CLK
+	depends on (ARCH_MXC && ARM64) || COMPILE_TEST
+	default USB_DWC3
+	help
+	  NXP iMX SoC use DesignWare Core IP for USB2/3
+	  functionality.
+	  This driver also handles the wakeup feature outside
+	  of DesignWare Core.
+	  Say 'Y' or 'M' if you have one such device.
+
 config USB_DWC3_XILINX
 	tristate "Xilinx Platforms"
 	depends on (ARCH_ZYNQMP || COMPILE_TEST) && OF
diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile
index 073bef5309b5b..f37971197203e 100644
--- a/drivers/usb/dwc3/Makefile
+++ b/drivers/usb/dwc3/Makefile
@@ -55,6 +55,7 @@ obj-$(CONFIG_USB_DWC3_ST)		+= dwc3-st.o
 obj-$(CONFIG_USB_DWC3_QCOM)		+= dwc3-qcom.o
 obj-$(CONFIG_USB_DWC3_QCOM)		+= dwc3-qcom-legacy.o
 obj-$(CONFIG_USB_DWC3_IMX8MP)		+= dwc3-imx8mp.o
+obj-$(CONFIG_USB_DWC3_IMX)		+= dwc3-imx.o
 obj-$(CONFIG_USB_DWC3_XILINX)		+= dwc3-xilinx.o
 obj-$(CONFIG_USB_DWC3_OCTEON)		+= dwc3-octeon.o
 obj-$(CONFIG_USB_DWC3_RTK)		+= dwc3-rtk.o
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index 161a4d58b2cec..58899b1fa96d2 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -782,6 +782,24 @@ static int dwc3_hs_phy_setup(struct dwc3 *dwc, int index)
 	return 0;
 }
 
+static void dwc3_ulpi_setup(struct dwc3 *dwc)
+{
+	int index;
+	u32 reg;
+
+	/* Don't do anything if there is no ULPI PHY */
+	if (!dwc->ulpi)
+		return;
+
+	if (dwc->enable_usb2_transceiver_delay) {
+		for (index = 0; index < dwc->num_usb2_ports; index++) {
+			reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(index));
+			reg |= DWC3_GUSB2PHYCFG_XCVRDLY;
+			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(index), reg);
+		}
+	}
+}
+
 /**
  * dwc3_phy_setup - Configure USB PHY Interface of DWC3 Core
  * @dwc: Pointer to our controller context structure
@@ -1363,6 +1381,8 @@ int dwc3_core_init(struct dwc3 *dwc)
 		dwc->ulpi_ready = true;
 	}
 
+	dwc3_ulpi_setup(dwc);
+
 	if (!dwc->phys_ready) {
 		ret = dwc3_core_get_phy(dwc);
 		if (ret)
@@ -1675,6 +1695,9 @@ static void dwc3_get_software_properties(struct dwc3 *dwc,
 	u16 gsbuscfg0_reqinfo;
 	int ret;
 
+	if (properties->needs_full_reinit)
+		dwc->needs_full_reinit = true;
+
 	dwc->gsbuscfg0_reqinfo = DWC3_GSBUSCFG0_REQINFO_UNSPECIFIED;
 
 	if (properties->gsbuscfg0_reqinfo !=
@@ -2479,7 +2502,8 @@ static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg)
 		dwc3_core_exit(dwc);
 		break;
 	case DWC3_GCTL_PRTCAP_HOST:
-		if (!PMSG_IS_AUTO(msg) && !device_may_wakeup(dwc->dev)) {
+		if (!PMSG_IS_AUTO(msg) &&
+		    (!device_may_wakeup(dwc->dev) || dwc->needs_full_reinit)) {
 			dwc3_core_exit(dwc);
 			break;
 		}
@@ -2542,7 +2566,8 @@ static int dwc3_resume_common(struct dwc3 *dwc, pm_message_t msg)
 		dwc3_gadget_resume(dwc);
 		break;
 	case DWC3_GCTL_PRTCAP_HOST:
-		if (!PMSG_IS_AUTO(msg) && !device_may_wakeup(dwc->dev)) {
+		if (!PMSG_IS_AUTO(msg) &&
+		    (!device_may_wakeup(dwc->dev) || dwc->needs_full_reinit)) {
 			ret = dwc3_core_init_for_resume(dwc);
 			if (ret)
 				return ret;
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index a35b3db1f9f3e..e0dee9d287401 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -302,6 +302,7 @@
 #define DWC3_GUSB2PHYCFG_SUSPHY		BIT(6)
 #define DWC3_GUSB2PHYCFG_ULPI_UTMI	BIT(4)
 #define DWC3_GUSB2PHYCFG_ENBLSLPM	BIT(8)
+#define DWC3_GUSB2PHYCFG_XCVRDLY	BIT(9)
 #define DWC3_GUSB2PHYCFG_PHYIF(n)	(n << 3)
 #define DWC3_GUSB2PHYCFG_PHYIF_MASK	DWC3_GUSB2PHYCFG_PHYIF(1)
 #define DWC3_GUSB2PHYCFG_USBTRDTIM(n)	(n << 10)
@@ -1119,6 +1120,8 @@ struct dwc3_glue_ops {
  * @usb3_lpm_capable: set if hadrware supports Link Power Management
  * @usb2_lpm_disable: set to disable usb2 lpm for host
  * @usb2_gadget_lpm_disable: set to disable usb2 lpm for gadget
+ * @needs_full_reinit: set to indicate the core may lose power and need full
+ *			initialization during system pm
  * @disable_scramble_quirk: set if we enable the disable scramble quirk
  * @u2exit_lfps_quirk: set if we enable u2exit lfps quirk
  * @u2ss_inp3_quirk: set if we enable P3 OK for U2/SS Inactive quirk
@@ -1149,7 +1152,7 @@ struct dwc3_glue_ops {
  *			VBUS with an external supply.
  * @parkmode_disable_ss_quirk: set if we need to disable all SuperSpeed
  *			instances in park mode.
- * @parkmode_disable_hs_quirk: set if we need to disable all HishSpeed
+ * @parkmode_disable_hs_quirk: set if we need to disable all HighSpeed
  *			instances in park mode.
  * @gfladj_refclk_lpm_sel: set if we need to enable SOF/ITP counter
  *                          running based on ref_clk
@@ -1161,6 +1164,8 @@ struct dwc3_glue_ops {
  *	3	- Reserved
  * @dis_metastability_quirk: set to disable metastability quirk.
  * @dis_split_quirk: set to disable split boundary.
+ * @enable_usb2_transceiver_delay: Set to insert a delay before the
+ *			assertion of the TxValid signal during a HS Chirp.
  * @sys_wakeup: set if the device may do system wakeup.
  * @wakeup_configured: set if the device is configured for remote wakeup.
  * @suspended: set to track suspend event due to U3/L2.
@@ -1373,6 +1378,7 @@ struct dwc3 {
 	unsigned		usb3_lpm_capable:1;
 	unsigned		usb2_lpm_disable:1;
 	unsigned		usb2_gadget_lpm_disable:1;
+	unsigned		needs_full_reinit:1;
 
 	unsigned		disable_scramble_quirk:1;
 	unsigned		u2exit_lfps_quirk:1;
@@ -1403,6 +1409,7 @@ struct dwc3 {
 	unsigned		dis_metastability_quirk:1;
 
 	unsigned		dis_split_quirk:1;
+	unsigned		enable_usb2_transceiver_delay:1;
 	unsigned		async_callbacks:1;
 	unsigned		sys_wakeup:1;
 	unsigned		wakeup_configured:1;
diff --git a/drivers/usb/dwc3/dwc3-google.c b/drivers/usb/dwc3/dwc3-google.c
index 2105c72af7532..4ca567ec01d07 100644
--- a/drivers/usb/dwc3/dwc3-google.c
+++ b/drivers/usb/dwc3/dwc3-google.c
@@ -385,8 +385,9 @@ static int dwc3_google_probe(struct platform_device *pdev)
 						     "google,usb-cfg-csr",
 						     ARRAY_SIZE(args), args);
 	if (IS_ERR(google->usb_cfg_regmap)) {
-		return dev_err_probe(dev, PTR_ERR(google->usb_cfg_regmap),
-				     "invalid usb cfg csr\n");
+		ret = dev_err_probe(dev, PTR_ERR(google->usb_cfg_regmap),
+				    "invalid usb cfg csr\n");
+		goto err_deinit_pdom;
 	}
 
 	google->host_cfg_offset = args[0];
diff --git a/drivers/usb/dwc3/dwc3-imx.c b/drivers/usb/dwc3/dwc3-imx.c
new file mode 100644
index 0000000000000..973a486b544d5
--- /dev/null
+++ b/drivers/usb/dwc3/dwc3-imx.c
@@ -0,0 +1,448 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * dwc3-imx.c - NXP i.MX Soc USB3 Specific Glue layer
+ *
+ * Copyright 2026 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "core.h"
+#include "glue.h"
+
+/* USB wakeup registers */
+#define USB_WAKEUP_CTRL			0x00
+
+/* Global wakeup interrupt enable, also used to clear interrupt */
+#define USB_WAKEUP_EN			BIT(31)
+/* Wakeup from connect or disconnect, only for superspeed */
+#define USB_WAKEUP_SS_CONN		BIT(5)
+/* 0 select vbus_valid, 1 select sessvld */
+#define USB_WAKEUP_VBUS_SRC_SESS_VAL	BIT(4)
+/* Enable signal for wake up from u3 state */
+#define USB_WAKEUP_U3_EN		BIT(3)
+/* Enable signal for wake up from id change */
+#define USB_WAKEUP_ID_EN		BIT(2)
+/* Enable signal for wake up from vbus change */
+#define	USB_WAKEUP_VBUS_EN		BIT(1)
+/* Enable signal for wake up from dp/dm change */
+#define USB_WAKEUP_DPDM_EN		BIT(0)
+
+#define USB_WAKEUP_EN_MASK		GENMASK(5, 0)
+
+/* USB glue registers */
+#define USB_CTRL0		0x00
+#define USB_CTRL1		0x04
+
+#define USB_CTRL0_PORTPWR_EN	BIT(12) /* 1 - PPC enabled (default) */
+#define USB_CTRL0_USB3_FIXED	BIT(22) /* 1 - USB3 permanent attached */
+#define USB_CTRL0_USB2_FIXED	BIT(23) /* 1 - USB2 permanent attached */
+
+#define USB_CTRL1_OC_POLARITY	BIT(16) /* 0 - HIGH / 1 - LOW */
+#define USB_CTRL1_PWR_POLARITY	BIT(17) /* 0 - HIGH / 1 - LOW */
+
+struct dwc3_imx {
+	struct dwc3	dwc;
+	struct device	*dev;
+	void __iomem	*blkctl_base;
+	void __iomem	*glue_base;
+	struct clk	*hsio_clk;
+	struct clk	*suspend_clk;
+	int		irq;
+	bool		pm_suspended;
+	bool		wakeup_pending;
+	unsigned	permanent_attached:1;
+	unsigned	disable_pwr_ctrl:1;
+	unsigned	overcur_active_low:1;
+	unsigned	power_active_low:1;
+};
+
+#define to_dwc3_imx(d) container_of((d), struct dwc3_imx, dwc)
+
+static void dwc3_imx_get_property(struct dwc3_imx *dwc_imx)
+{
+	struct device	*dev = dwc_imx->dev;
+
+	dwc_imx->permanent_attached =
+		device_property_read_bool(dev, "fsl,permanently-attached");
+	dwc_imx->disable_pwr_ctrl =
+		device_property_read_bool(dev, "fsl,disable-port-power-control");
+	dwc_imx->overcur_active_low =
+		device_property_read_bool(dev, "fsl,over-current-active-low");
+	dwc_imx->power_active_low =
+		device_property_read_bool(dev, "fsl,power-active-low");
+}
+
+static void dwc3_imx_configure_glue(struct dwc3_imx *dwc_imx)
+{
+	u32		value;
+
+	if (!dwc_imx->glue_base)
+		return;
+
+	value = readl(dwc_imx->glue_base + USB_CTRL0);
+
+	if (dwc_imx->permanent_attached)
+		value |= USB_CTRL0_USB2_FIXED | USB_CTRL0_USB3_FIXED;
+	else
+		value &= ~(USB_CTRL0_USB2_FIXED | USB_CTRL0_USB3_FIXED);
+
+	if (dwc_imx->disable_pwr_ctrl)
+		value &= ~USB_CTRL0_PORTPWR_EN;
+	else
+		value |= USB_CTRL0_PORTPWR_EN;
+
+	writel(value, dwc_imx->glue_base + USB_CTRL0);
+
+	value = readl(dwc_imx->glue_base + USB_CTRL1);
+	if (dwc_imx->overcur_active_low)
+		value |= USB_CTRL1_OC_POLARITY;
+	else
+		value &= ~USB_CTRL1_OC_POLARITY;
+
+	if (dwc_imx->power_active_low)
+		value |= USB_CTRL1_PWR_POLARITY;
+	else
+		value &= ~USB_CTRL1_PWR_POLARITY;
+
+	writel(value, dwc_imx->glue_base + USB_CTRL1);
+}
+
+static void dwc3_imx_wakeup_enable(struct dwc3_imx *dwc_imx, pm_message_t msg)
+{
+	struct dwc3	*dwc = &dwc_imx->dwc;
+	u32		val;
+
+	val = readl(dwc_imx->blkctl_base + USB_WAKEUP_CTRL);
+
+	if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_HOST && dwc->xhci) {
+		val |= USB_WAKEUP_EN | USB_WAKEUP_DPDM_EN;
+		if (PMSG_IS_AUTO(msg))
+			val |= USB_WAKEUP_SS_CONN | USB_WAKEUP_U3_EN;
+	} else {
+		val |= USB_WAKEUP_EN | USB_WAKEUP_VBUS_EN |
+		       USB_WAKEUP_VBUS_SRC_SESS_VAL;
+	}
+
+	writel(val, dwc_imx->blkctl_base + USB_WAKEUP_CTRL);
+}
+
+static void dwc3_imx_wakeup_disable(struct dwc3_imx *dwc_imx)
+{
+	u32	val;
+
+	val = readl(dwc_imx->blkctl_base + USB_WAKEUP_CTRL);
+	val &= ~(USB_WAKEUP_EN | USB_WAKEUP_EN_MASK);
+	writel(val, dwc_imx->blkctl_base + USB_WAKEUP_CTRL);
+}
+
+static irqreturn_t dwc3_imx_interrupt(int irq, void *data)
+{
+	struct dwc3_imx	*dwc_imx = data;
+	struct dwc3	*dwc = &dwc_imx->dwc;
+
+	if (!dwc_imx->pm_suspended)
+		return IRQ_HANDLED;
+
+	disable_irq_nosync(dwc_imx->irq);
+	dwc_imx->wakeup_pending = true;
+
+	if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_HOST && dwc->xhci)
+		pm_runtime_resume(&dwc->xhci->dev);
+	else if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_DEVICE)
+		pm_runtime_get(dwc->dev);
+
+	return IRQ_HANDLED;
+}
+
+static void dwc3_imx_pre_set_role(struct dwc3 *dwc, enum usb_role role)
+{
+	if (role == USB_ROLE_HOST)
+		/*
+		 * For xhci host, we need disable dwc core auto
+		 * suspend, because during this auto suspend delay(5s),
+		 * xhci host RUN_STOP is cleared and wakeup is not
+		 * enabled, if device is inserted, xhci host can't
+		 * response the connection.
+		 */
+		pm_runtime_dont_use_autosuspend(dwc->dev);
+	else
+		pm_runtime_use_autosuspend(dwc->dev);
+}
+
+static struct dwc3_glue_ops dwc3_imx_glue_ops = {
+	.pre_set_role = dwc3_imx_pre_set_role,
+};
+
+static const struct property_entry dwc3_imx_properties[] = {
+	PROPERTY_ENTRY_BOOL("xhci-missing-cas-quirk"),
+	PROPERTY_ENTRY_BOOL("xhci-skip-phy-init-quirk"),
+	{},
+};
+
+static const struct software_node dwc3_imx_swnode = {
+	.properties = dwc3_imx_properties,
+};
+
+static int dwc3_imx_probe(struct platform_device *pdev)
+{
+	struct device		*dev = &pdev->dev;
+	struct dwc3_imx		*dwc_imx;
+	struct dwc3		*dwc;
+	struct resource		*res;
+	const char		*irq_name;
+	struct dwc3_probe_data	probe_data = {};
+	int			ret, irq;
+
+	dwc_imx = devm_kzalloc(dev, sizeof(*dwc_imx), GFP_KERNEL);
+	if (!dwc_imx)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, dwc_imx);
+	dwc_imx->dev = dev;
+
+	dwc3_imx_get_property(dwc_imx);
+
+	dwc_imx->blkctl_base = devm_platform_ioremap_resource_byname(pdev, "blkctl");
+	if (IS_ERR(dwc_imx->blkctl_base))
+		return PTR_ERR(dwc_imx->blkctl_base);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "glue");
+	if (!res) {
+		dev_warn(dev, "Base address for glue layer missing\n");
+	} else {
+		dwc_imx->glue_base = devm_ioremap_resource(dev, res);
+		if (IS_ERR(dwc_imx->glue_base))
+			return PTR_ERR(dwc_imx->glue_base);
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core");
+	if (!res)
+		return dev_err_probe(dev, -ENODEV, "missing core memory resource\n");
+
+	dwc_imx->hsio_clk = devm_clk_get_enabled(dev, "hsio");
+	if (IS_ERR(dwc_imx->hsio_clk))
+		return dev_err_probe(dev, PTR_ERR(dwc_imx->hsio_clk),
+				     "Failed to get hsio clk\n");
+
+	dwc_imx->suspend_clk = devm_clk_get_enabled(dev, "suspend");
+	if (IS_ERR(dwc_imx->suspend_clk))
+		return dev_err_probe(dev, PTR_ERR(dwc_imx->suspend_clk),
+				     "Failed to get suspend clk\n");
+
+	irq = platform_get_irq_byname(pdev, "wakeup");
+	if (irq < 0)
+		return irq;
+	dwc_imx->irq = irq;
+
+	irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s:wakeup", dev_name(dev));
+	if (!irq_name)
+		return dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n");
+
+	ret = devm_request_threaded_irq(dev, irq, NULL, dwc3_imx_interrupt,
+					IRQF_ONESHOT | IRQF_NO_AUTOEN,
+					irq_name, dwc_imx);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to request IRQ #%d\n", irq);
+
+	ret = device_add_software_node(dev, &dwc3_imx_swnode);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to add software node\n");
+
+	dwc3_imx_configure_glue(dwc_imx);
+
+	dwc = &dwc_imx->dwc;
+	dwc->dev = dev;
+	dwc->glue_ops = &dwc3_imx_glue_ops;
+
+	probe_data.res = res;
+	probe_data.dwc = dwc;
+	probe_data.properties = DWC3_DEFAULT_PROPERTIES;
+	probe_data.properties.needs_full_reinit = true;
+
+	ret = dwc3_core_probe(&probe_data);
+	if (ret) {
+		device_remove_software_node(dev);
+		return ret;
+	}
+
+	device_set_wakeup_capable(dev, true);
+	return 0;
+}
+
+static void dwc3_imx_remove(struct platform_device *pdev)
+{
+	struct device	*dev = &pdev->dev;
+	struct dwc3	*dwc = dev_get_drvdata(dev);
+
+	dwc3_core_remove(dwc);
+	device_remove_software_node(dev);
+}
+
+static void dwc3_imx_suspend(struct dwc3_imx *dwc_imx, pm_message_t msg)
+{
+	if (dwc_imx->pm_suspended)
+		return;
+
+	if (PMSG_IS_AUTO(msg) || device_may_wakeup(dwc_imx->dev))
+		dwc3_imx_wakeup_enable(dwc_imx, msg);
+
+	enable_irq(dwc_imx->irq);
+	dwc_imx->pm_suspended = true;
+}
+
+static void dwc3_imx_resume(struct dwc3_imx *dwc_imx, pm_message_t msg)
+{
+	struct dwc3	*dwc = &dwc_imx->dwc;
+
+	if (!dwc_imx->pm_suspended)
+		return;
+
+	dwc_imx->pm_suspended = false;
+	if (!dwc_imx->wakeup_pending)
+		disable_irq_nosync(dwc_imx->irq);
+
+	dwc3_imx_wakeup_disable(dwc_imx);
+
+	/* Upon power loss any previous configuration is lost, restore it */
+	dwc3_imx_configure_glue(dwc_imx);
+
+	if (dwc_imx->wakeup_pending) {
+		dwc_imx->wakeup_pending = false;
+		if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_DEVICE)
+			pm_runtime_put_autosuspend(dwc->dev);
+		else
+			/*
+			 * Add wait for xhci switch from suspend
+			 * clock to normal clock to detect connection.
+			 */
+			usleep_range(9000, 10000);
+	}
+}
+
+static int dwc3_imx_runtime_suspend(struct device *dev)
+{
+	struct dwc3	*dwc = dev_get_drvdata(dev);
+	struct dwc3_imx	*dwc_imx = to_dwc3_imx(dwc);
+	int		ret;
+
+	ret = dwc3_runtime_suspend(dwc);
+	if (ret)
+		return ret;
+
+	dwc3_imx_suspend(dwc_imx, PMSG_AUTO_SUSPEND);
+	return 0;
+}
+
+static int dwc3_imx_runtime_resume(struct device *dev)
+{
+	struct dwc3	*dwc = dev_get_drvdata(dev);
+	struct dwc3_imx	*dwc_imx = to_dwc3_imx(dwc);
+
+	dwc3_imx_resume(dwc_imx, PMSG_AUTO_RESUME);
+	return dwc3_runtime_resume(dwc);
+}
+
+static int dwc3_imx_runtime_idle(struct device *dev)
+{
+	return dwc3_runtime_idle(dev_get_drvdata(dev));
+}
+
+static int dwc3_imx_pm_suspend(struct device *dev)
+{
+	struct dwc3	*dwc = dev_get_drvdata(dev);
+	struct dwc3_imx *dwc_imx = to_dwc3_imx(dwc);
+	int		ret;
+
+	ret = dwc3_pm_suspend(dwc);
+	if (ret)
+		return ret;
+
+	dwc3_imx_suspend(dwc_imx, PMSG_SUSPEND);
+
+	if (device_may_wakeup(dev)) {
+		enable_irq_wake(dwc_imx->irq);
+		device_set_out_band_wakeup(dev);
+	} else {
+		clk_disable_unprepare(dwc_imx->suspend_clk);
+	}
+
+	clk_disable_unprepare(dwc_imx->hsio_clk);
+
+	return 0;
+}
+
+static int dwc3_imx_pm_resume(struct device *dev)
+{
+	struct dwc3	*dwc = dev_get_drvdata(dev);
+	struct dwc3_imx *dwc_imx = to_dwc3_imx(dwc);
+	int		ret;
+
+	if (device_may_wakeup(dwc_imx->dev)) {
+		disable_irq_wake(dwc_imx->irq);
+	} else {
+		ret = clk_prepare_enable(dwc_imx->suspend_clk);
+		if (ret)
+			return ret;
+	}
+
+	ret = clk_prepare_enable(dwc_imx->hsio_clk);
+	if (ret) {
+		clk_disable_unprepare(dwc_imx->suspend_clk);
+		return ret;
+	}
+
+	dwc3_imx_resume(dwc_imx, PMSG_RESUME);
+
+	ret = dwc3_pm_resume(dwc);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void dwc3_imx_complete(struct device *dev)
+{
+	dwc3_pm_complete(dev_get_drvdata(dev));
+}
+
+static int dwc3_imx_prepare(struct device *dev)
+{
+	return dwc3_pm_prepare(dev_get_drvdata(dev));
+}
+
+static const struct dev_pm_ops dwc3_imx_dev_pm_ops = {
+	SYSTEM_SLEEP_PM_OPS(dwc3_imx_pm_suspend, dwc3_imx_pm_resume)
+	RUNTIME_PM_OPS(dwc3_imx_runtime_suspend, dwc3_imx_runtime_resume,
+		       dwc3_imx_runtime_idle)
+	.complete = pm_sleep_ptr(dwc3_imx_complete),
+	.prepare = pm_sleep_ptr(dwc3_imx_prepare),
+};
+
+static const struct of_device_id dwc3_imx_of_match[] = {
+	{ .compatible = "nxp,imx8mp-dwc3", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dwc3_imx_of_match);
+
+static struct platform_driver dwc3_imx_driver = {
+	.probe		= dwc3_imx_probe,
+	.remove		= dwc3_imx_remove,
+	.driver		= {
+		.name	= "imx-dwc3",
+		.pm	= pm_ptr(&dwc3_imx_dev_pm_ops),
+		.of_match_table	= dwc3_imx_of_match,
+	},
+};
+
+module_platform_driver(dwc3_imx_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("DesignWare USB3 i.MX Glue Layer");
diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c
index 9ac75547820d9..f43f73ac36ff1 100644
--- a/drivers/usb/dwc3/dwc3-qcom.c
+++ b/drivers/usb/dwc3/dwc3-qcom.c
@@ -526,14 +526,14 @@ static int dwc3_qcom_find_num_ports(struct platform_device *pdev)
 	int irq;
 
 	irq = platform_get_irq_byname_optional(pdev, "dp_hs_phy_1");
-	if (irq <= 0)
+	if (irq < 0)
 		return 1;
 
 	for (port_num = 2; port_num <= DWC3_QCOM_MAX_PORTS; port_num++) {
 		sprintf(irq_name, "dp_hs_phy_%d", port_num);
 
 		irq = platform_get_irq_byname_optional(pdev, irq_name);
-		if (irq <= 0)
+		if (irq < 0)
 			return port_num - 1;
 	}
 
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 0a688904ce8c5..3d4ca68e584c7 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -1688,7 +1688,7 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
 	 * transfer, there's no need to update the transfer.
 	 */
 	if (!ret && !starting)
-		return ret;
+		return 0;
 
 	req = next_request(&dep->started_list);
 	if (!req) {
diff --git a/drivers/usb/dwc3/glue.h b/drivers/usb/dwc3/glue.h
index df86e14cb706c..d738e1739ae01 100644
--- a/drivers/usb/dwc3/glue.h
+++ b/drivers/usb/dwc3/glue.h
@@ -12,9 +12,12 @@
 /**
  * dwc3_properties: DWC3 core properties
  * @gsbuscfg0_reqinfo: Value to be programmed in the GSBUSCFG0.REQINFO field
+ * @needs_full_reinit: indicate the controller may not remain power during system
+ *			pm and need full initialization
  */
 struct dwc3_properties {
 	u32 gsbuscfg0_reqinfo;
+	unsigned needs_full_reinit:1;
 };
 
 #define DWC3_DEFAULT_PROPERTIES ((struct dwc3_properties){		\
diff --git a/drivers/usb/dwc3/ulpi.c b/drivers/usb/dwc3/ulpi.c
index 57daad15f502d..a256b7f5d78b4 100644
--- a/drivers/usb/dwc3/ulpi.c
+++ b/drivers/usb/dwc3/ulpi.c
@@ -10,10 +10,13 @@
 #include <linux/delay.h>
 #include <linux/time64.h>
 #include <linux/ulpi/regs.h>
+#include <linux/ulpi/driver.h>
 
 #include "core.h"
 #include "io.h"
 
+#define USB_VENDOR_MICROCHIP 0x0424
+
 #define DWC3_ULPI_ADDR(a) \
 		((a >= ULPI_EXT_VENDOR_SPECIFIC) ? \
 		DWC3_GUSB2PHYACC_ADDR(ULPI_ACCESS_EXTENDED) | \
@@ -83,6 +86,26 @@ static const struct ulpi_ops dwc3_ulpi_ops = {
 	.write = dwc3_ulpi_write,
 };
 
+static void dwc3_ulpi_detect_config(struct dwc3 *dwc)
+{
+	struct ulpi *ulpi = dwc->ulpi;
+
+	switch (ulpi->id.vendor) {
+	case USB_VENDOR_MICROCHIP:
+		switch (ulpi->id.product) {
+		case 0x0009:
+			/* Microchip USB3340 ULPI PHY */
+			dwc->enable_usb2_transceiver_delay = true;
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
 int dwc3_ulpi_init(struct dwc3 *dwc)
 {
 	/* Register the interface */
@@ -92,6 +115,8 @@ int dwc3_ulpi_init(struct dwc3 *dwc)
 		return PTR_ERR(dwc->ulpi);
 	}
 
+	dwc3_ulpi_detect_config(dwc);
+
 	return 0;
 }
 
diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c
index b5f0defde95d6..19fdac0243439 100644
--- a/drivers/usb/gadget/function/f_midi2.c
+++ b/drivers/usb/gadget/function/f_midi2.c
@@ -1541,9 +1541,9 @@ static int f_midi2_create_card(struct f_midi2 *midi2)
 		return err;
 	midi2->card = card;
 
-	strcpy(card->driver, "f_midi2");
-	strcpy(card->shortname, "MIDI 2.0 Gadget");
-	strcpy(card->longname, "MIDI 2.0 Gadget");
+	strscpy(card->driver, "f_midi2");
+	strscpy(card->shortname, "MIDI 2.0 Gadget");
+	strscpy(card->longname, "MIDI 2.0 Gadget");
 
 	id = 0;
 	for (i = 0; i < midi2->num_eps; i++) {
diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c
index a7853dcbb14ca..34d9f49e99874 100644
--- a/drivers/usb/gadget/function/f_tcm.c
+++ b/drivers/usb/gadget/function/f_tcm.c
@@ -2030,6 +2030,7 @@ static const struct target_core_fabric_ops usbg_ops = {
 	.tfc_wwn_attrs			= usbg_wwn_attrs,
 	.tfc_tpg_base_attrs		= usbg_base_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index 1a9e7c495e2e8..59d85d6a84a89 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -113,8 +113,10 @@ static void eth_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *p)
 
 	strscpy(p->driver, "g_ether", sizeof(p->driver));
 	strscpy(p->version, UETH__VERSION, sizeof(p->version));
-	strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version));
-	strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info));
+	if (dev->gadget) {
+		strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version));
+		strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info));
+	}
 }
 
 /* REVISIT can also support:
@@ -1223,6 +1225,11 @@ void gether_disconnect(struct gether *link)
 
 	DBG(dev, "%s\n", __func__);
 
+	spin_lock(&dev->lock);
+	dev->port_usb = NULL;
+	link->is_suspend = false;
+	spin_unlock(&dev->lock);
+
 	netif_stop_queue(dev->net);
 	netif_carrier_off(dev->net);
 
@@ -1260,11 +1267,6 @@ void gether_disconnect(struct gether *link)
 	dev->header_len = 0;
 	dev->unwrap = NULL;
 	dev->wrap = NULL;
-
-	spin_lock(&dev->lock);
-	dev->port_usb = NULL;
-	link->is_suspend = false;
-	spin_unlock(&dev->lock);
 }
 EXPORT_SYMBOL_GPL(gether_disconnect);
 
diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
index e43ad63738465..cdd1dfc666c4f 100644
--- a/drivers/usb/gadget/function/u_serial.c
+++ b/drivers/usb/gadget/function/u_serial.c
@@ -1086,7 +1086,7 @@ static int gs_console_init(struct gs_port *port)
 	if (!cons)
 		return -ENOMEM;
 
-	strcpy(cons->console.name, "ttyGS");
+	strscpy(cons->console.name, "ttyGS");
 	cons->console.write = gs_console_write;
 	cons->console.device = gs_console_device;
 	cons->console.flags = CON_PRINTBUFFER;
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index c9eca90376e2b..f094491b1041a 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -462,8 +462,13 @@ static void set_link_state(struct dummy_hcd *dum_hcd)
 
 		/* Report reset and disconnect events to the driver */
 		if (dum->ints_enabled && (disconnect || reset)) {
-			stop_activity(dum);
 			++dum->callback_usage;
+			/*
+			 * stop_activity() can drop dum->lock, so it must
+			 * not come between the dum->ints_enabled test
+			 * and the ++dum->callback_usage.
+			 */
+			stop_activity(dum);
 			spin_unlock(&dum->lock);
 			if (reset)
 				usb_gadget_udc_reset(&dum->gadget, dum->driver);
@@ -908,21 +913,6 @@ static int dummy_pullup(struct usb_gadget *_gadget, int value)
 	spin_lock_irqsave(&dum->lock, flags);
 	dum->pullup = (value != 0);
 	set_link_state(dum_hcd);
-	if (value == 0) {
-		/*
-		 * Emulate synchronize_irq(): wait for callbacks to finish.
-		 * This seems to be the best place to emulate the call to
-		 * synchronize_irq() that's in usb_gadget_remove_driver().
-		 * Doing it in dummy_udc_stop() would be too late since it
-		 * is called after the unbind callback and unbind shouldn't
-		 * be invoked until all the other callbacks are finished.
-		 */
-		while (dum->callback_usage > 0) {
-			spin_unlock_irqrestore(&dum->lock, flags);
-			usleep_range(1000, 2000);
-			spin_lock_irqsave(&dum->lock, flags);
-		}
-	}
 	spin_unlock_irqrestore(&dum->lock, flags);
 
 	usb_hcd_poll_rh_status(dummy_hcd_to_hcd(dum_hcd));
@@ -945,6 +935,20 @@ static void dummy_udc_async_callbacks(struct usb_gadget *_gadget, bool enable)
 
 	spin_lock_irq(&dum->lock);
 	dum->ints_enabled = enable;
+	if (!enable) {
+		/*
+		 * Emulate synchronize_irq(): wait for callbacks to finish.
+		 * This has to happen after emulated interrupts are disabled
+		 * (dum->ints_enabled is clear) and before the unbind callback,
+		 * just like the call to synchronize_irq() in
+		 * gadget/udc/core:gadget_unbind_driver().
+		 */
+		while (dum->callback_usage > 0) {
+			spin_unlock_irq(&dum->lock);
+			usleep_range(1000, 2000);
+			spin_lock_irq(&dum->lock);
+		}
+	}
 	spin_unlock_irq(&dum->lock);
 }
 
@@ -1534,6 +1538,12 @@ static int transfer(struct dummy_hcd *dum_hcd, struct urb *urb,
 		/* rescan to continue with any other queued i/o */
 		if (rescan)
 			goto top;
+
+		/* request not fully transferred; stop iterating to
+		 * preserve data ordering across queued requests.
+		 */
+		if (req->req.actual < req->req.length)
+			break;
 	}
 	return sent;
 }
diff --git a/drivers/usb/gadget/udc/pxa27x_udc.c b/drivers/usb/gadget/udc/pxa27x_udc.c
index 4fac477d24c05..1abea0d48c35d 100644
--- a/drivers/usb/gadget/udc/pxa27x_udc.c
+++ b/drivers/usb/gadget/udc/pxa27x_udc.c
@@ -1462,7 +1462,7 @@ static int pxa_udc_wakeup(struct usb_gadget *_gadget)
 	return 0;
 }
 
-static void udc_enable(struct pxa_udc *udc);
+static int udc_enable(struct pxa_udc *udc);
 static void udc_disable(struct pxa_udc *udc);
 
 /**
@@ -1519,14 +1519,20 @@ static int should_disable_udc(struct pxa_udc *udc)
 static int pxa_udc_pullup(struct usb_gadget *_gadget, int is_active)
 {
 	struct pxa_udc *udc = to_gadget_udc(_gadget);
+	int ret;
 
 	if (!udc->gpiod && !udc->udc_command)
 		return -EOPNOTSUPP;
 
 	dplus_pullup(udc, is_active);
 
-	if (should_enable_udc(udc))
-		udc_enable(udc);
+	if (should_enable_udc(udc)) {
+		ret = udc_enable(udc);
+		if (ret) {
+			dplus_pullup(udc, !is_active);
+			return ret;
+		}
+	}
 	if (should_disable_udc(udc))
 		udc_disable(udc);
 	return 0;
@@ -1545,10 +1551,16 @@ static int pxa_udc_pullup(struct usb_gadget *_gadget, int is_active)
 static int pxa_udc_vbus_session(struct usb_gadget *_gadget, int is_active)
 {
 	struct pxa_udc *udc = to_gadget_udc(_gadget);
+	int ret;
 
 	udc->vbus_sensed = is_active;
-	if (should_enable_udc(udc))
-		udc_enable(udc);
+	if (should_enable_udc(udc)) {
+		ret = udc_enable(udc);
+		if (ret) {
+			udc->vbus_sensed = !is_active;
+			return ret;
+		}
+	}
 	if (should_disable_udc(udc))
 		udc_disable(udc);
 
@@ -1691,12 +1703,18 @@ static void udc_init_data(struct pxa_udc *dev)
  * Enables the udc device : enables clocks, udc interrupts, control endpoint
  * interrupts, sets usb as UDC client and setups endpoints.
  */
-static void udc_enable(struct pxa_udc *udc)
+static int udc_enable(struct pxa_udc *udc)
 {
+	int ret;
+
 	if (udc->enabled)
-		return;
+		return 0;
 
-	clk_enable(udc->clk);
+	ret = clk_enable(udc->clk);
+	if (ret) {
+		dev_err(udc->dev, "clk_enable failed: %d\n", ret);
+		return ret;
+	}
 	udc_writel(udc, UDCICR0, 0);
 	udc_writel(udc, UDCICR1, 0);
 	udc_clear_mask_UDCCR(udc, UDCCR_UDE);
@@ -1726,6 +1744,8 @@ static void udc_enable(struct pxa_udc *udc)
 	pio_irq_enable(&udc->pxa_ep[0]);
 
 	udc->enabled = 1;
+
+	return 0;
 }
 
 /**
@@ -1761,10 +1781,16 @@ static int pxa27x_udc_start(struct usb_gadget *g,
 		}
 	}
 
-	if (should_enable_udc(udc))
-		udc_enable(udc);
+	if (should_enable_udc(udc)) {
+		retval = udc_enable(udc);
+		if (retval)
+			goto fail_enable;
+	}
 	return 0;
 
+fail_enable:
+	if (!IS_ERR_OR_NULL(udc->transceiver))
+		otg_set_peripheral(udc->transceiver->otg, NULL);
 fail:
 	udc->driver = NULL;
 	return retval;
@@ -2430,10 +2456,16 @@ static int pxa_udc_probe(struct platform_device *pdev)
 		goto err_add_gadget;
 
 	pxa_init_debugfs(udc);
-	if (should_enable_udc(udc))
-		udc_enable(udc);
+	if (should_enable_udc(udc)) {
+		retval = udc_enable(udc);
+		if (retval)
+			goto err_enable;
+	}
 	return 0;
 
+err_enable:
+	usb_del_gadget_udc(&udc->gadget);
+	pxa_cleanup_debugfs(udc);
 err_add_gadget:
 	if (!IS_ERR_OR_NULL(udc->transceiver))
 		usb_unregister_notifier(udc->transceiver, &pxa27x_udc_phy);
@@ -2509,13 +2541,19 @@ static int pxa_udc_resume(struct platform_device *_dev)
 {
 	struct pxa_udc *udc = platform_get_drvdata(_dev);
 	struct pxa_ep *ep;
+	int ret;
 
 	ep = &udc->pxa_ep[0];
 	udc_ep_writel(ep, UDCCSR, udc->udccsr0 & (UDCCSR0_FST | UDCCSR0_DME));
 
 	dplus_pullup(udc, udc->pullup_resume);
-	if (should_enable_udc(udc))
-		udc_enable(udc);
+	if (should_enable_udc(udc)) {
+		ret = udc_enable(udc);
+		if (ret) {
+			dplus_pullup(udc, !udc->pullup_resume);
+			return ret;
+		}
+	}
 	/*
 	 * We do not handle OTG yet.
 	 *
diff --git a/drivers/usb/gadget/udc/snps_udc_core.c b/drivers/usb/gadget/udc/snps_udc_core.c
index 5f9514623956d..0e0db68e0b27a 100644
--- a/drivers/usb/gadget/udc/snps_udc_core.c
+++ b/drivers/usb/gadget/udc/snps_udc_core.c
@@ -3151,7 +3151,7 @@ int udc_probe(struct udc *dev)
 			 tmp, dev->phys_addr, dev->chiprev,
 			 (dev->chiprev == UDC_HSA0_REV) ?
 			 "A0" : "B1");
-		strcpy(tmp, UDC_DRIVER_VERSION_STRING);
+		strscpy(tmp, UDC_DRIVER_VERSION_STRING);
 		if (dev->chiprev == UDC_HSA0_REV) {
 			dev_err(dev->dev, "chip revision is A0; too old\n");
 			retval = -ENODEV;
diff --git a/drivers/usb/host/ehci-orion.c b/drivers/usb/host/ehci-orion.c
index 34abff8669f8b..eaaa49712a8c6 100644
--- a/drivers/usb/host/ehci-orion.c
+++ b/drivers/usb/host/ehci-orion.c
@@ -12,7 +12,6 @@
 #include <linux/clk.h>
 #include <linux/platform_data/usb-ehci-orion.h>
 #include <linux/of.h>
-#include <linux/phy/phy.h>
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
 #include <linux/io.h>
@@ -60,7 +59,6 @@
 
 struct orion_ehci_hcd {
 	struct clk *clk;
-	struct phy *phy;
 };
 
 static struct hc_driver __read_mostly ehci_orion_hc_driver;
@@ -276,13 +274,6 @@ static int ehci_orion_drv_probe(struct platform_device *pdev)
 			goto err_put_hcd;
 	}
 
-	priv->phy = devm_phy_optional_get(&pdev->dev, "usb");
-	if (IS_ERR(priv->phy)) {
-		err = PTR_ERR(priv->phy);
-		if (err != -ENOSYS)
-			goto err_dis_clk;
-	}
-
 	/*
 	 * (Re-)program MBUS remapping windows if we are asked to.
 	 */
diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c
index 271bcbe9b3262..71e785f445a30 100644
--- a/drivers/usb/host/fhci-hcd.c
+++ b/drivers/usb/host/fhci-hcd.c
@@ -426,16 +426,11 @@ static int fhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 	}
 
 	/* allocate the private part of the URB */
-	urb_priv = kzalloc_obj(*urb_priv, mem_flags);
+	urb_priv = kzalloc_flex(*urb_priv, tds, size, mem_flags);
 	if (!urb_priv)
 		return -ENOMEM;
 
-	/* allocate the private part of the URB */
-	urb_priv->tds = kzalloc_objs(*urb_priv->tds, size, mem_flags);
-	if (!urb_priv->tds) {
-		kfree(urb_priv);
-		return -ENOMEM;
-	}
+	urb_priv->num_of_tds = size;
 
 	spin_lock_irqsave(&fhci->lock, flags);
 
@@ -444,8 +439,6 @@ static int fhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 		goto err;
 
 	/* fill the private part of the URB */
-	urb_priv->num_of_tds = size;
-
 	urb->status = -EINPROGRESS;
 	urb->actual_length = 0;
 	urb->error_count = 0;
@@ -453,10 +446,8 @@ static int fhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 
 	fhci_queue_urb(fhci, urb);
 err:
-	if (ret) {
-		kfree(urb_priv->tds);
+	if (ret)
 		kfree(urb_priv);
-	}
 	spin_unlock_irqrestore(&fhci->lock, flags);
 	return ret;
 }
diff --git a/drivers/usb/host/fhci.h b/drivers/usb/host/fhci.h
index 1f57b0989485a..e221b28e8608b 100644
--- a/drivers/usb/host/fhci.h
+++ b/drivers/usb/host/fhci.h
@@ -387,9 +387,10 @@ struct urb_priv {
 	int tds_cnt;
 	int state;
 
-	struct td **tds;
 	struct ed *ed;
 	struct timer_list time_out;
+
+	struct td *tds[] __counted_by(num_of_tds);
 };
 
 struct endpoint {
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index 3f6aa2440b05b..ed4b11f8d298e 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -1357,15 +1357,11 @@ static void tegra_xhci_id_work(struct work_struct *work)
 
 	dev_dbg(tegra->dev, "host mode %s\n", str_on_off(tegra->host_mode));
 
-	mutex_lock(&tegra->lock);
-
 	if (tegra->host_mode)
 		phy_set_mode_ext(phy, PHY_MODE_USB_OTG, USB_ROLE_HOST);
 	else
 		phy_set_mode_ext(phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
 
-	mutex_unlock(&tegra->lock);
-
 	tegra->otg_usb3_port = tegra_xusb_padctl_get_usb3_companion(tegra->padctl,
 								    tegra->otg_usb2_port);
 
diff --git a/drivers/usb/image/microtek.c b/drivers/usb/image/microtek.c
index 45dc209f5fe5e..1f0b3c44d3881 100644
--- a/drivers/usb/image/microtek.c
+++ b/drivers/usb/image/microtek.c
@@ -55,7 +55,6 @@
  * Status:
  *
  *	Untested with multiple scanners.
- *	Untested on SMP.
  *	Untested on a bigendian machine.
  *
  * History:
@@ -170,25 +169,13 @@ static struct usb_driver mts_usb_driver = {
 #define MTS_VERSION	"0.4.3"
 #define MTS_NAME	"microtek usb (rev " MTS_VERSION "): "
 
-#define MTS_WARNING(x...) \
-	printk( KERN_WARNING MTS_NAME x )
-#define MTS_ERROR(x...) \
-	printk( KERN_ERR MTS_NAME x )
-#define MTS_INT_ERROR(x...) \
-	MTS_ERROR(x)
-#define MTS_MESSAGE(x...) \
-	printk( KERN_INFO MTS_NAME x )
-
 #if defined MTS_DO_DEBUG
 
 #define MTS_DEBUG(x...) \
 	printk( KERN_DEBUG MTS_NAME x )
 
-#define MTS_DEBUG_GOT_HERE() \
-	MTS_DEBUG("got to %s:%d (%s)\n", __FILE__, (int)__LINE__, __func__ )
 #define MTS_DEBUG_INT() \
-	do { MTS_DEBUG_GOT_HERE(); \
-	     MTS_DEBUG("transfer = 0x%x context = 0x%x\n",(int)transfer,(int)context ); \
+	do { MTS_DEBUG("transfer = 0x%x context = 0x%x\n",(int)transfer,(int)context ); \
 	     MTS_DEBUG("status = 0x%x data-length = 0x%x sent = 0x%x\n",transfer->status,(int)context->data_length, (int)transfer->actual_length ); \
              mts_debug_dump(context->instance);\
 	   } while(0)
@@ -197,7 +184,6 @@ static struct usb_driver mts_usb_driver = {
 #define MTS_NUL_STATEMENT do { } while(0)
 
 #define MTS_DEBUG(x...)	MTS_NUL_STATEMENT
-#define MTS_DEBUG_GOT_HERE() MTS_NUL_STATEMENT
 #define MTS_DEBUG_INT() MTS_NUL_STATEMENT
 
 #endif
@@ -316,7 +302,6 @@ static inline void mts_debug_dump(struct mts_desc* dummy)
 #endif
 
 static inline void mts_urb_abort(struct mts_desc* desc) {
-	MTS_DEBUG_GOT_HERE();
 	mts_debug_dump(desc);
 
 	usb_kill_urb( desc->urb );
@@ -332,8 +317,6 @@ static int mts_scsi_abort(struct scsi_cmnd *srb)
 {
 	struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);
 
-	MTS_DEBUG_GOT_HERE();
-
 	mts_urb_abort(desc);
 
 	return FAILED;
@@ -344,7 +327,6 @@ static int mts_scsi_host_reset(struct scsi_cmnd *srb)
 	struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);
 	int result;
 
-	MTS_DEBUG_GOT_HERE();
 	mts_debug_dump(desc);
 
 	result = usb_lock_device_for_reset(desc->usb_dev, desc->usb_intf);
@@ -386,7 +368,8 @@ void mts_int_submit_urb (struct urb* transfer,
 
 	res = usb_submit_urb( transfer, GFP_ATOMIC );
 	if ( unlikely(res) ) {
-		MTS_INT_ERROR( "could not submit URB! Error was %d\n",(int)res );
+		dev_err(&context->instance->usb_dev->dev,
+			"could not submit URB! Error was %d\n",(int)res );
 		set_host_byte(context->srb, DID_ERROR);
 		mts_transfer_cleanup(transfer);
 	}
@@ -452,12 +435,9 @@ static void mts_command_done( struct urb *transfer )
 	if ( unlikely(status) ) {
 	        if (status == -ENOENT) {
 		        /* We are being killed */
-			MTS_DEBUG_GOT_HERE();
 			set_host_byte(context->srb, DID_ABORT);
                 } else {
 		        /* A genuine error has occurred */
-			MTS_DEBUG_GOT_HERE();
-
 		        set_host_byte(context->srb, DID_ERROR);
                 }
 		mts_transfer_cleanup(transfer);
@@ -523,8 +503,6 @@ mts_build_transfer_context(struct scsi_cmnd *srb, struct mts_desc* desc)
 {
 	int pipe;
 
-	MTS_DEBUG_GOT_HERE();
-
 	desc->context.instance = desc;
 	desc->context.srb = srb;
 
@@ -565,7 +543,6 @@ static enum scsi_qc_status mts_scsi_queuecommand_lck(struct scsi_cmnd *srb)
 	struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);
 	int res;
 
-	MTS_DEBUG_GOT_HERE();
 	mts_show_command(srb);
 	mts_debug_dump(desc);
 
@@ -601,7 +578,7 @@ static enum scsi_qc_status mts_scsi_queuecommand_lck(struct scsi_cmnd *srb)
 	res=usb_submit_urb(desc->urb, GFP_ATOMIC);
 
 	if(unlikely(res)){
-		MTS_ERROR("error %d submitting URB\n",(int)res);
+		dev_err(&desc->usb_dev->dev, "error %d submitting URB\n",(int)res);
 		set_host_byte(srb, DID_ERROR);
 
 		if(likely(callback != NULL))
@@ -666,59 +643,47 @@ static int mts_usb_probe(struct usb_interface *intf,
 	/* the current altsetting on the interface we're probing */
 	struct usb_host_interface *altsetting;
 
-	MTS_DEBUG_GOT_HERE();
 	MTS_DEBUG( "usb-device descriptor at %x\n", (int)dev );
 
 	MTS_DEBUG( "product id = 0x%x, vendor id = 0x%x\n",
 		   le16_to_cpu(dev->descriptor.idProduct),
 		   le16_to_cpu(dev->descriptor.idVendor) );
 
-	MTS_DEBUG_GOT_HERE();
-
 	/* the current altsetting on the interface we're probing */
 	altsetting = intf->cur_altsetting;
 
-
 	/* Check if the config is sane */
 
 	if ( altsetting->desc.bNumEndpoints != MTS_EP_TOTAL ) {
-		MTS_WARNING( "expecting %d got %d endpoints! Bailing out.\n",
+		dev_warn(&dev->dev, "expecting %d got %d endpoints! Bailing out.\n",
 			     (int)MTS_EP_TOTAL, (int)altsetting->desc.bNumEndpoints );
 		return -ENODEV;
 	}
 
 	for( i = 0; i < altsetting->desc.bNumEndpoints; i++ ) {
-		if ((altsetting->endpoint[i].desc.bmAttributes &
-		     USB_ENDPOINT_XFERTYPE_MASK) != USB_ENDPOINT_XFER_BULK) {
-
-			MTS_WARNING( "can only deal with bulk endpoints; endpoint %d is not bulk.\n",
-			     (int)altsetting->endpoint[i].desc.bEndpointAddress );
-		} else {
-			if (altsetting->endpoint[i].desc.bEndpointAddress &
-			    USB_DIR_IN)
-				*ep_in_current++
-					= altsetting->endpoint[i].desc.bEndpointAddress &
-					USB_ENDPOINT_NUMBER_MASK;
-			else {
-				if ( ep_out != -1 ) {
-					MTS_WARNING( "can only deal with one output endpoints. Bailing out." );
-					return -ENODEV;
-				}
-
-				ep_out = altsetting->endpoint[i].desc.bEndpointAddress &
-					USB_ENDPOINT_NUMBER_MASK;
+		if (usb_endpoint_is_bulk_in(&altsetting->endpoint[i].desc)) {
+			*ep_in_current++ = usb_endpoint_num(&altsetting->endpoint[i].desc);
+		} else if (usb_endpoint_is_bulk_out(&altsetting->endpoint[i].desc)) {
+			if (ep_out == -1) {
+				ep_out = usb_endpoint_num(&altsetting->endpoint[i].desc);
+			} else {
+				dev_warn(&dev->dev, "can only deal with bulk endpoints; endpoint %d is not bulk.\n",
+						usb_endpoint_num(&altsetting->endpoint[i].desc));
+				return -ENODEV;
 			}
+		} else {
+			dev_warn(&dev->dev, "can only deal with bulk endpoints; endpoint %d is not bulk.\n",
+					usb_endpoint_num(&altsetting->endpoint[i].desc));
 		}
-
 	}
 
 	if (ep_in_current != &ep_in_set[2]) {
-		MTS_WARNING("couldn't find two input bulk endpoints. Bailing out.\n");
+		dev_warn(&dev->dev, "couldn't find two input bulk endpoints. Bailing out.\n");
 		return -ENODEV;
 	}
 
 	if ( ep_out == -1 ) {
-		MTS_WARNING( "couldn't find an output bulk endpoint. Bailing out.\n" );
+		dev_warn(&dev->dev, "couldn't find an output bulk endpoint. Bailing out.\n" );
 		return -ENODEV;
 	}
 
@@ -744,15 +709,15 @@ static int mts_usb_probe(struct usb_interface *intf,
 	new_desc->ep_image = ep_in_set[1];
 
 	if ( new_desc->ep_out != MTS_EP_OUT )
-		MTS_WARNING( "will this work? Command EP is not usually %d\n",
+		dev_warn(&dev->dev, "will this work? Command EP is not usually %d\n",
 			     (int)new_desc->ep_out );
 
 	if ( new_desc->ep_response != MTS_EP_RESPONSE )
-		MTS_WARNING( "will this work? Response EP is not usually %d\n",
+		dev_warn(&dev->dev, "will this work? Response EP is not usually %d\n",
 			     (int)new_desc->ep_response );
 
 	if ( new_desc->ep_image != MTS_EP_IMAGE )
-		MTS_WARNING( "will this work? Image data EP is not usually %d\n",
+		dev_warn(&dev->dev, "will this work? Image data EP is not usually %d\n",
 			     (int)new_desc->ep_image );
 
 	new_desc->host = scsi_host_alloc(&mts_scsi_host_template,
diff --git a/drivers/usb/misc/apple-mfi-fastcharge.c b/drivers/usb/misc/apple-mfi-fastcharge.c
index 339f6cd2e9b23..af266e19f2fdc 100644
--- a/drivers/usb/misc/apple-mfi-fastcharge.c
+++ b/drivers/usb/misc/apple-mfi-fastcharge.c
@@ -210,7 +210,7 @@ static int mfi_fc_probe(struct usb_device *udev)
 		goto err_free_name;
 	}
 
-	mfi->udev = usb_get_dev(udev);
+	mfi->udev = udev;
 	dev_set_drvdata(&udev->dev, mfi);
 
 	return 0;
@@ -231,7 +231,6 @@ static void mfi_fc_disconnect(struct usb_device *udev)
 		power_supply_unregister(mfi->battery);
 	kfree(mfi->battery_desc.name);
 	dev_set_drvdata(&udev->dev, NULL);
-	usb_put_dev(mfi->udev);
 	kfree(mfi);
 }
 
diff --git a/drivers/usb/misc/cypress_cy7c63.c b/drivers/usb/misc/cypress_cy7c63.c
index 99185fc3e9dff..4a7f955ba85b8 100644
--- a/drivers/usb/misc/cypress_cy7c63.c
+++ b/drivers/usb/misc/cypress_cy7c63.c
@@ -215,7 +215,7 @@ static int cypress_probe(struct usb_interface *interface,
 	if (!dev)
 		goto error_mem;
 
-	dev->udev = usb_get_dev(interface_to_usbdev(interface));
+	dev->udev = interface_to_usbdev(interface);
 
 	/* save our data pointer in this interface device */
 	usb_set_intfdata(interface, dev);
@@ -239,8 +239,6 @@ static void cypress_disconnect(struct usb_interface *interface)
 	 * device files have been removed */
 	usb_set_intfdata(interface, NULL);
 
-	usb_put_dev(dev->udev);
-
 	dev_info(&interface->dev,
 		 "Cypress CY7C63xxx device now disconnected\n");
 
diff --git a/drivers/usb/misc/cytherm.c b/drivers/usb/misc/cytherm.c
index 2bf082474e9db..b183df9826bc6 100644
--- a/drivers/usb/misc/cytherm.c
+++ b/drivers/usb/misc/cytherm.c
@@ -311,7 +311,7 @@ static int cytherm_probe(struct usb_interface *interface,
 	if (!dev)
 		goto error_mem;
 
-	dev->udev = usb_get_dev(udev);
+	dev->udev = udev;
 
 	usb_set_intfdata(interface, dev);
 
@@ -334,8 +334,6 @@ static void cytherm_disconnect(struct usb_interface *interface)
 	/* first remove the files, then NULL the pointer */
 	usb_set_intfdata(interface, NULL);
 
-	usb_put_dev(dev->udev);
-
 	kfree(dev);
 
 	dev_info(&interface->dev, "Cypress thermometer now disconnected\n");
diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c
index 18670dfed2e7d..54f1bb0f71234 100644
--- a/drivers/usb/misc/iowarrior.c
+++ b/drivers/usb/misc/iowarrior.c
@@ -74,7 +74,6 @@ struct iowarrior {
 	struct mutex mutex;			/* locks this structure */
 	struct usb_device *udev;		/* save off the usb device pointer */
 	struct usb_interface *interface;	/* the interface for this device */
-	unsigned char minor;			/* the starting minor number for this device */
 	struct usb_endpoint_descriptor *int_out_endpoint;	/* endpoint for reading (needed for IOW56 only) */
 	struct usb_endpoint_descriptor *int_in_endpoint;	/* endpoint for reading */
 	struct urb *int_in_urb;		/* the urb for reading data */
@@ -234,8 +233,7 @@ static void iowarrior_write_callback(struct urb *urb)
 			"nonzero write bulk status received: %d\n", status);
 	}
 	/* free up our allocated buffer */
-	usb_free_coherent(urb->dev, urb->transfer_buffer_length,
-			  urb->transfer_buffer, urb->transfer_dma);
+	kfree(urb->transfer_buffer);
 	/* tell a waiting writer the interrupt-out-pipe is available again */
 	atomic_dec(&dev->write_busy);
 	wake_up_interruptible(&dev->write_wait);
@@ -246,7 +244,6 @@ static void iowarrior_write_callback(struct urb *urb)
  */
 static inline void iowarrior_delete(struct iowarrior *dev)
 {
-	dev_dbg(&dev->interface->dev, "minor %d\n", dev->minor);
 	kfree(dev->int_in_buffer);
 	usb_free_urb(dev->int_in_urb);
 	kfree(dev->read_queue);
@@ -297,9 +294,6 @@ static ssize_t iowarrior_read(struct file *file, char __user *buffer,
 		goto exit;
 	}
 
-	dev_dbg(&dev->interface->dev, "minor %d, count = %zd\n",
-		dev->minor, count);
-
 	/* read count must be packet size (+ time stamp) */
 	if ((count != dev->report_size)
 	    && (count != (dev->report_size + 1))) {
@@ -367,20 +361,21 @@ static ssize_t iowarrior_write(struct file *file,
 			       size_t count, loff_t *ppos)
 {
 	struct iowarrior *dev;
-	int retval = 0;
+	int retval;
 	char *buf = NULL;	/* for IOW24 and IOW56 we need a buffer */
 	struct urb *int_out_urb = NULL;
 
 	dev = file->private_data;
 
-	mutex_lock(&dev->mutex);
+	retval = mutex_lock_interruptible(&dev->mutex);
+	if (retval < 0)
+		return -EINTR;
+
 	/* verify that the device wasn't unplugged */
 	if (!dev->present) {
 		retval = -ENODEV;
 		goto exit;
 	}
-	dev_dbg(&dev->interface->dev, "minor %d, count = %zd\n",
-		dev->minor, count);
 	/* if count is 0 we're already done */
 	if (count == 0) {
 		retval = 0;
@@ -443,8 +438,7 @@ static ssize_t iowarrior_write(struct file *file,
 			retval = -ENOMEM;
 			goto error_no_urb;
 		}
-		buf = usb_alloc_coherent(dev->udev, dev->report_size,
-					 GFP_KERNEL, &int_out_urb->transfer_dma);
+		buf = kmalloc(dev->report_size, GFP_KERNEL);
 		if (!buf) {
 			retval = -ENOMEM;
 			dev_dbg(&dev->interface->dev,
@@ -457,7 +451,6 @@ static ssize_t iowarrior_write(struct file *file,
 				 buf, dev->report_size,
 				 iowarrior_write_callback, dev,
 				 dev->int_out_endpoint->bInterval);
-		int_out_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 		if (copy_from_user(buf, user_buffer, count)) {
 			retval = -EFAULT;
 			goto error;
@@ -483,8 +476,7 @@ static ssize_t iowarrior_write(struct file *file,
 		goto exit;
 	}
 error:
-	usb_free_coherent(dev->udev, dev->report_size, buf,
-			  int_out_urb->transfer_dma);
+	kfree(buf);
 error_no_buffer:
 	usb_free_urb(int_out_urb);
 error_no_urb:
@@ -523,9 +515,6 @@ static long iowarrior_ioctl(struct file *file, unsigned int cmd,
 		goto error_out;
 	}
 
-	dev_dbg(&dev->interface->dev, "minor %d, cmd 0x%.4x, arg %ld\n",
-		dev->minor, cmd, arg);
-
 	retval = 0;
 	switch (cmd) {
 	case IOW_WRITE:
@@ -671,8 +660,6 @@ static int iowarrior_release(struct inode *inode, struct file *file)
 	if (!dev)
 		return -ENODEV;
 
-	dev_dbg(&dev->interface->dev, "minor %d\n", dev->minor);
-
 	/* lock our device */
 	mutex_lock(&dev->mutex);
 
@@ -775,6 +762,7 @@ static int iowarrior_probe(struct usb_interface *interface,
 	struct usb_host_interface *iface_desc;
 	int retval = -ENOMEM;
 	int res;
+	int minor;
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc_obj(struct iowarrior);
@@ -890,12 +878,12 @@ static int iowarrior_probe(struct usb_interface *interface,
 		goto error;
 	}
 
-	dev->minor = interface->minor;
+	minor = interface->minor;
 
 	/* let the user know what node this device is now attached to */
 	dev_info(&interface->dev, "IOWarrior product=0x%x, serial=%s interface=%d "
 		 "now attached to iowarrior%d\n", dev->product_id, dev->chip_serial,
-		 iface_desc->desc.bInterfaceNumber, dev->minor - IOWARRIOR_MINOR_BASE);
+		 iface_desc->desc.bInterfaceNumber, minor - IOWARRIOR_MINOR_BASE);
 	return retval;
 
 error:
diff --git a/drivers/usb/misc/onboard_usb_dev.c b/drivers/usb/misc/onboard_usb_dev.c
index ba37eb99efbaf..6dd73f23e9be8 100644
--- a/drivers/usb/misc/onboard_usb_dev.c
+++ b/drivers/usb/misc/onboard_usb_dev.c
@@ -565,6 +565,7 @@ static struct platform_driver onboard_dev_driver = {
 /************************** USB driver **************************/
 
 #define VENDOR_ID_BISON		0x5986
+#define VENDOR_ID_CORECHIPS	0x3431
 #define VENDOR_ID_CYPRESS	0x04b4
 #define VENDOR_ID_GENESYS	0x05e3
 #define VENDOR_ID_MICROCHIP	0x0424
@@ -649,6 +650,8 @@ static void onboard_dev_usbdev_disconnect(struct usb_device *udev)
 
 static const struct usb_device_id onboard_dev_id_table[] = {
 	{ USB_DEVICE(VENDOR_ID_BISON, 0x1198) }, /* Bison Electronics Inc. Integrated Camera */
+	{ USB_DEVICE(VENDOR_ID_CORECHIPS, 0x6241) }, /* SL6341 2.0 HUB */
+	{ USB_DEVICE(VENDOR_ID_CORECHIPS, 0x6341) }, /* SL6341 3.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_CYPRESS, 0x6500) }, /* CYUSB330x 3.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_CYPRESS, 0x6502) }, /* CYUSB330x 2.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_CYPRESS, 0x6503) }, /* CYUSB33{0,1}x 2.0 HUB, Vendor Mode */
diff --git a/drivers/usb/misc/onboard_usb_dev.h b/drivers/usb/misc/onboard_usb_dev.h
index 1a1e86e60e042..56cb5b1621412 100644
--- a/drivers/usb/misc/onboard_usb_dev.h
+++ b/drivers/usb/misc/onboard_usb_dev.h
@@ -66,13 +66,6 @@ static const struct onboard_dev_pdata ti_tusb8020b_data = {
 	.is_hub = true,
 };
 
-static const struct onboard_dev_pdata ti_tusb8041_data = {
-	.reset_us = 3000,
-	.num_supplies = 1,
-	.supply_names = { "vdd" },
-	.is_hub = true,
-};
-
 static const struct onboard_dev_pdata bison_intcamera_data = {
 	.reset_us = 1000,
 	.num_supplies = 1,
@@ -80,17 +73,17 @@ static const struct onboard_dev_pdata bison_intcamera_data = {
 	.is_hub = false,
 };
 
-static const struct onboard_dev_pdata cypress_hx3_data = {
+static const struct onboard_dev_pdata corechips_sl6341_data = {
 	.reset_us = 10000,
 	.num_supplies = 2,
-	.supply_names = { "vdd", "vdd2" },
+	.supply_names = { "vdd1v1", "vdd3v3" },
 	.is_hub = true,
 };
 
-static const struct onboard_dev_pdata cypress_hx2vl_data = {
-	.reset_us = 1,
-	.num_supplies = 1,
-	.supply_names = { "vdd" },
+static const struct onboard_dev_pdata cypress_hx3_data = {
+	.reset_us = 10000,
+	.num_supplies = 2,
+	.supply_names = { "vdd", "vdd2" },
 	.is_hub = true,
 };
 
@@ -108,6 +101,11 @@ static const struct onboard_dev_pdata genesys_gl852g_data = {
 	.is_hub = true,
 };
 
+static const struct onboard_dev_pdata usb_a_conn_data = {
+	.num_supplies = 1,
+	.supply_names = { "vbus" },
+};
+
 static const struct onboard_dev_pdata vialab_vl817_data = {
 	.reset_us = 10,
 	.num_supplies = 1,
@@ -130,6 +128,7 @@ static const struct onboard_dev_pdata xmos_xvf3500_data = {
 };
 
 static const struct of_device_id onboard_dev_match[] = {
+	{ .compatible = "usb-a-connector", .data = &usb_a_conn_data, },
 	{ .compatible = "usb424,2412", .data = &microchip_usb424_data, },
 	{ .compatible = "usb424,2514", .data = &microchip_usb2514_data, },
 	{ .compatible = "usb424,2517", .data = &microchip_usb424_data, },
@@ -137,13 +136,13 @@ static const struct of_device_id onboard_dev_match[] = {
 	{ .compatible = "usb424,5744", .data = &microchip_usb5744_data, },
 	{ .compatible = "usb451,8025", .data = &ti_tusb8020b_data, },
 	{ .compatible = "usb451,8027", .data = &ti_tusb8020b_data, },
-	{ .compatible = "usb451,8140", .data = &ti_tusb8041_data, },
-	{ .compatible = "usb451,8142", .data = &ti_tusb8041_data, },
-	{ .compatible = "usb451,8440", .data = &ti_tusb8041_data, },
-	{ .compatible = "usb451,8442", .data = &ti_tusb8041_data, },
+	{ .compatible = "usb451,8140", .data = &ti_tusb8020b_data, },
+	{ .compatible = "usb451,8142", .data = &ti_tusb8020b_data, },
+	{ .compatible = "usb451,8440", .data = &ti_tusb8020b_data, },
+	{ .compatible = "usb451,8442", .data = &ti_tusb8020b_data, },
 	{ .compatible = "usb4b4,6504", .data = &cypress_hx3_data, },
 	{ .compatible = "usb4b4,6506", .data = &cypress_hx3_data, },
-	{ .compatible = "usb4b4,6570", .data = &cypress_hx2vl_data, },
+	{ .compatible = "usb4b4,6570", .data = &microchip_usb424_data, },
 	{ .compatible = "usb5e3,608", .data = &genesys_gl850g_data, },
 	{ .compatible = "usb5e3,610", .data = &genesys_gl852g_data, },
 	{ .compatible = "usb5e3,620", .data = &genesys_gl852g_data, },
@@ -159,6 +158,8 @@ static const struct of_device_id onboard_dev_match[] = {
 	{ .compatible = "usb2109,817", .data = &vialab_vl817_data, },
 	{ .compatible = "usb2109,2817", .data = &vialab_vl817_data, },
 	{ .compatible = "usb20b1,0013", .data = &xmos_xvf3500_data, },
+	{ .compatible = "usb3431,6241", .data = &corechips_sl6341_data, },
+	{ .compatible = "usb3431,6341", .data = &corechips_sl6341_data, },
 	{ .compatible = "usb5986,1198", .data = &bison_intcamera_data, },
 	{}
 };
diff --git a/drivers/usb/misc/trancevibrator.c b/drivers/usb/misc/trancevibrator.c
index 6aaec2db360b9..37f6b79889a64 100644
--- a/drivers/usb/misc/trancevibrator.c
+++ b/drivers/usb/misc/trancevibrator.c
@@ -92,7 +92,7 @@ static int tv_probe(struct usb_interface *interface,
 		goto error;
 	}
 
-	dev->udev = usb_get_dev(udev);
+	dev->udev = udev;
 	usb_set_intfdata(interface, dev);
 
 	return 0;
@@ -108,7 +108,6 @@ static void tv_disconnect(struct usb_interface *interface)
 
 	dev = usb_get_intfdata (interface);
 	usb_set_intfdata(interface, NULL);
-	usb_put_dev(dev->udev);
 	kfree(dev);
 }
 
diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c
index 7e85fd12da564..c60121faa3dae 100644
--- a/drivers/usb/misc/usb-ljca.c
+++ b/drivers/usb/misc/usb-ljca.c
@@ -776,7 +776,7 @@ static int ljca_probe(struct usb_interface *interface,
 	init_completion(&adap->cmd_completion);
 	INIT_LIST_HEAD(&adap->client_list);
 
-	adap->intf = usb_get_intf(interface);
+	adap->intf = interface;
 	adap->usb_dev = usb_dev;
 	adap->dev = dev;
 
@@ -787,7 +787,7 @@ static int ljca_probe(struct usb_interface *interface,
 	ret = usb_find_common_endpoints(alt, &ep_in, &ep_out, NULL, NULL);
 	if (ret) {
 		dev_err(dev, "bulk endpoints not found\n");
-		goto err_put;
+		goto err_destroy_mutex;
 	}
 	adap->rx_pipe = usb_rcvbulkpipe(usb_dev, usb_endpoint_num(ep_in));
 	adap->tx_pipe = usb_sndbulkpipe(usb_dev, usb_endpoint_num(ep_out));
@@ -797,14 +797,14 @@ static int ljca_probe(struct usb_interface *interface,
 	adap->rx_buf = devm_kzalloc(dev, adap->rx_len, GFP_KERNEL);
 	if (!adap->rx_buf) {
 		ret = -ENOMEM;
-		goto err_put;
+		goto err_destroy_mutex;
 	}
 
 	/* alloc rx urb */
 	adap->rx_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!adap->rx_urb) {
 		ret = -ENOMEM;
-		goto err_put;
+		goto err_destroy_mutex;
 	}
 	usb_fill_bulk_urb(adap->rx_urb, usb_dev, adap->rx_pipe,
 			  adap->rx_buf, adap->rx_len, ljca_recv, adap);
@@ -836,10 +836,7 @@ static int ljca_probe(struct usb_interface *interface,
 
 err_free:
 	usb_free_urb(adap->rx_urb);
-
-err_put:
-	usb_put_intf(adap->intf);
-
+err_destroy_mutex:
 	mutex_destroy(&adap->mutex);
 
 	return ret;
@@ -864,8 +861,6 @@ static void ljca_disconnect(struct usb_interface *interface)
 
 	usb_free_urb(adap->rx_urb);
 
-	usb_put_intf(adap->intf);
-
 	mutex_destroy(&adap->mutex);
 }
 
diff --git a/drivers/usb/misc/usbsevseg.c b/drivers/usb/misc/usbsevseg.c
index b37bf53dd54f1..89d25fcef6424 100644
--- a/drivers/usb/misc/usbsevseg.c
+++ b/drivers/usb/misc/usbsevseg.c
@@ -312,7 +312,7 @@ static int sevseg_probe(struct usb_interface *interface,
 	if (!mydev)
 		goto error_mem;
 
-	mydev->udev = usb_get_dev(udev);
+	mydev->udev = udev;
 	mydev->intf = interface;
 	usb_set_intfdata(interface, mydev);
 
@@ -338,7 +338,6 @@ static void sevseg_disconnect(struct usb_interface *interface)
 
 	mydev = usb_get_intfdata(interface);
 	usb_set_intfdata(interface, NULL);
-	usb_put_dev(mydev->udev);
 	kfree(mydev);
 	dev_info(&interface->dev, "USB 7 Segment now disconnected\n");
 }
diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c
index a8af7615b1bfb..b7d3c44b970e9 100644
--- a/drivers/usb/misc/uss720.c
+++ b/drivers/usb/misc/uss720.c
@@ -677,35 +677,32 @@ static int uss720_probe(struct usb_interface *intf,
 	struct parport_uss720_private *priv;
 	struct parport *pp;
 	unsigned char reg;
-	int ret;
+	int ret = -ENODEV;
 
 	dev_dbg(&intf->dev, "probe: vendor id 0x%x, device id 0x%x\n",
 		le16_to_cpu(usbdev->descriptor.idVendor),
 		le16_to_cpu(usbdev->descriptor.idProduct));
 
 	/* our known interfaces have 3 alternate settings */
-	if (intf->num_altsetting != 3) {
-		usb_put_dev(usbdev);
-		return -ENODEV;
-	}
+	if (intf->num_altsetting != 3)
+		goto bail_out_early;
+
 	ret = usb_set_interface(usbdev, intf->altsetting->desc.bInterfaceNumber, 2);
 	dev_dbg(&intf->dev, "set interface result %d\n", ret);
 
 	interface = intf->cur_altsetting;
 
-	if (interface->desc.bNumEndpoints < 2) {
-		usb_put_dev(usbdev);
-		return -ENODEV;
-	}
+	if (interface->desc.bNumEndpoints < 2)
+		goto bail_out_early;
 
 	/*
 	 * Allocate parport interface 
 	 */
+	ret = -ENOMEM;
 	priv = kzalloc_obj(struct parport_uss720_private);
-	if (!priv) {
-		usb_put_dev(usbdev);
-		return -ENOMEM;
-	}
+	if (!priv)
+		goto bail_out_early;
+
 	priv->pp = NULL;
 	priv->usbdev = usbdev;
 	kref_init(&priv->ref_count);
@@ -752,6 +749,10 @@ static int uss720_probe(struct usb_interface *intf,
 	kill_all_async_requests_priv(priv);
 	kref_put(&priv->ref_count, destroy_priv);
 	return -ENODEV;
+
+bail_out_early:
+	usb_put_dev(usbdev);
+	return ret;
 }
 
 static void uss720_disconnect(struct usb_interface *intf)
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index 0acc62569ae51..73ac25f53698f 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -1600,7 +1600,7 @@ static int musb_core_init(u16 musb_type, struct musb *musb)
 	/* log core options (read using indexed model) */
 	reg = musb_read_configdata(mbase);
 
-	strcpy(aInfo, (reg & MUSB_CONFIGDATA_UTMIDW) ? "UTMI-16" : "UTMI-8");
+	strscpy(aInfo, (reg & MUSB_CONFIGDATA_UTMIDW) ? "UTMI-16" : "UTMI-8");
 	if (reg & MUSB_CONFIGDATA_DYNFIFO) {
 		strcat(aInfo, ", dyn FIFOs");
 		musb->dyn_fifo = true;
diff --git a/drivers/usb/renesas_usbhs/mod_gadget.c b/drivers/usb/renesas_usbhs/mod_gadget.c
index 1539e8e6901d9..0c7fe109d5c79 100644
--- a/drivers/usb/renesas_usbhs/mod_gadget.c
+++ b/drivers/usb/renesas_usbhs/mod_gadget.c
@@ -40,7 +40,6 @@ struct usbhsg_gpriv {
 	struct usb_gadget	 gadget;
 	struct usbhs_mod	 mod;
 
-	struct usbhsg_uep	*uep;
 	int			 uep_size;
 
 	struct usb_gadget_driver	*driver;
@@ -53,6 +52,7 @@ struct usbhsg_gpriv {
 #define USBHSG_STATUS_WEDGE		(1 << 2)
 #define USBHSG_STATUS_SELF_POWERED	(1 << 3)
 #define USBHSG_STATUS_SOFT_CONNECT	(1 << 4)
+	struct usbhsg_uep	uep[] __counted_by(uep_size);
 };
 
 struct usbhsg_recip_handle {
@@ -1084,15 +1084,11 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 	int i;
 	int ret;
 
-	gpriv = kzalloc_obj(struct usbhsg_gpriv);
+	gpriv = kzalloc_flex(*gpriv, uep, pipe_size);
 	if (!gpriv)
 		return -ENOMEM;
 
-	uep = kzalloc_objs(struct usbhsg_uep, pipe_size);
-	if (!uep) {
-		ret = -ENOMEM;
-		goto usbhs_mod_gadget_probe_err_gpriv;
-	}
+	gpriv->uep_size	= pipe_size;
 
 	gpriv->transceiver = devm_usb_get_phy(dev, USB_PHY_TYPE_UNDEFINED);
 	dev_info(dev, "%stransceiver found\n",
@@ -1115,8 +1111,6 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 	gpriv->mod.name		= "gadget";
 	gpriv->mod.start	= usbhsg_start;
 	gpriv->mod.stop		= usbhsg_stop;
-	gpriv->uep		= uep;
-	gpriv->uep_size		= pipe_size;
 	usbhsg_status_init(gpriv);
 
 	/*
@@ -1175,9 +1169,6 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 	return 0;
 
 err_add_udc:
-	kfree(gpriv->uep);
-
-usbhs_mod_gadget_probe_err_gpriv:
 	kfree(gpriv);
 
 	return ret;
@@ -1189,6 +1180,5 @@ void usbhs_mod_gadget_remove(struct usbhs_priv *priv)
 
 	usb_del_gadget_udc(&gpriv->gadget);
 
-	kfree(gpriv->uep);
 	kfree(gpriv);
 }
diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c
index 58694b8943d19..3f5889145e519 100644
--- a/drivers/usb/serial/io_edgeport.c
+++ b/drivers/usb/serial/io_edgeport.c
@@ -73,6 +73,7 @@ static const struct usb_device_id edgeport_4port_id_table[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_22I) },
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_4) },
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_COMPATIBLE) },
+	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) },
 	{ }
 };
 
@@ -121,6 +122,7 @@ static const struct usb_device_id id_table_combined[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8R) },
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_8RR) },
 	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_EDGEPORT_412_8) },
+	{ USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_BLACKBOX_IC135A) },
 	{ USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0202) },
 	{ USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0203) },
 	{ USB_DEVICE(USB_VENDOR_ID_NCR, NCR_DEVICE_ID_EPIC_0310) },
@@ -470,6 +472,7 @@ static void get_product_info(struct edgeport_serial *edge_serial)
 	case ION_DEVICE_ID_EDGEPORT_2_DIN:
 	case ION_DEVICE_ID_EDGEPORT_4_DIN:
 	case ION_DEVICE_ID_EDGEPORT_16_DUAL_CPU:
+	case ION_DEVICE_ID_BLACKBOX_IC135A:
 		product_info->IsRS232 = 1;
 		break;
 
diff --git a/drivers/usb/serial/io_usbvend.h b/drivers/usb/serial/io_usbvend.h
index 9a6f742ad3abd..c82a275e8e76e 100644
--- a/drivers/usb/serial/io_usbvend.h
+++ b/drivers/usb/serial/io_usbvend.h
@@ -211,6 +211,7 @@
 
 //
 // Definitions for other product IDs
+#define ION_DEVICE_ID_BLACKBOX_IC135A		0x0801	// OEM device (rebranded Edgeport/4)
 #define ION_DEVICE_ID_MT4X56USB			0x1403	// OEM device
 #define ION_DEVICE_ID_E5805A			0x1A01  // OEM device (rebranded Edgeport/4)
 
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index e349ed66d2ac7..d222bf2c25391 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -2461,6 +2461,7 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) },			/* Rolling RW101R-GL (laptop MBIM) */
 	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff),			/* Rolling RW350-GL (laptop MBIM) */
 	  .driver_info = RSVD(5) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff) },			/* Rolling RW135R-GL (laptop MBIM) */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) },	/* NetPrisma LCUK54-WWD for Global */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) },
diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c
index 658b54d8fcef7..b3591d6d7645a 100644
--- a/drivers/usb/serial/ti_usb_3410_5052.c
+++ b/drivers/usb/serial/ti_usb_3410_5052.c
@@ -1600,29 +1600,29 @@ static int ti_download_firmware(struct ti_device *tdev)
 		if (le16_to_cpu(dev->descriptor.idVendor) == MTS_VENDOR_ID) {
 			switch (le16_to_cpu(dev->descriptor.idProduct)) {
 			case MTS_CDMA_PRODUCT_ID:
-				strcpy(buf, "mts_cdma.fw");
+				strscpy(buf, "mts_cdma.fw");
 				break;
 			case MTS_GSM_PRODUCT_ID:
-				strcpy(buf, "mts_gsm.fw");
+				strscpy(buf, "mts_gsm.fw");
 				break;
 			case MTS_EDGE_PRODUCT_ID:
-				strcpy(buf, "mts_edge.fw");
+				strscpy(buf, "mts_edge.fw");
 				break;
 			case MTS_MT9234MU_PRODUCT_ID:
-				strcpy(buf, "mts_mt9234mu.fw");
+				strscpy(buf, "mts_mt9234mu.fw");
 				break;
 			case MTS_MT9234ZBA_PRODUCT_ID:
-				strcpy(buf, "mts_mt9234zba.fw");
+				strscpy(buf, "mts_mt9234zba.fw");
 				break;
 			case MTS_MT9234ZBAOLD_PRODUCT_ID:
-				strcpy(buf, "mts_mt9234zba.fw");
+				strscpy(buf, "mts_mt9234zba.fw");
 				break;			}
 		}
 		if (buf[0] == '\0') {
 			if (tdev->td_is_3410)
-				strcpy(buf, "ti_3410.fw");
+				strscpy(buf, "ti_3410.fw");
 			else
-				strcpy(buf, "ti_5052.fw");
+				strscpy(buf, "ti_5052.fw");
 		}
 		status = request_firmware(&fw_p, buf, &dev->dev);
 	}
diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c
index 0a9902d2b118c..2651629812694 100644
--- a/drivers/usb/storage/uas.c
+++ b/drivers/usb/storage/uas.c
@@ -772,7 +772,7 @@ static int uas_eh_abort_handler(struct scsi_cmnd *cmnd)
 	return FAILED;
 }
 
-static int uas_eh_device_reset_handler(struct scsi_cmnd *cmnd)
+static int uas_eh_host_reset_handler(struct scsi_cmnd *cmnd)
 {
 	struct scsi_device *sdev = cmnd->device;
 	struct uas_dev_info *devinfo = sdev->hostdata;
@@ -918,7 +918,7 @@ static const struct scsi_host_template uas_host_template = {
 	.sdev_init = uas_sdev_init,
 	.sdev_configure = uas_sdev_configure,
 	.eh_abort_handler = uas_eh_abort_handler,
-	.eh_device_reset_handler = uas_eh_device_reset_handler,
+	.eh_host_reset_handler = uas_eh_host_reset_handler,
 	.this_id = -1,
 	.skip_settle_delay = 1,
 	/*
diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c
index 58fb97ea6877a..db5e4a4c0a996 100644
--- a/drivers/usb/typec/mux.c
+++ b/drivers/usb/typec/mux.c
@@ -35,7 +35,9 @@ static int switch_fwnode_match(struct device *dev, const void *fwnode)
 static void *typec_switch_match(const struct fwnode_handle *fwnode,
 				const char *id, void *data)
 {
+	struct typec_switch_dev **sw_devs = data;
 	struct device *dev;
+	int i;
 
 	/*
 	 * Device graph (OF graph) does not give any means to identify the
@@ -56,6 +58,13 @@ static void *typec_switch_match(const struct fwnode_handle *fwnode,
 	dev = class_find_device(&typec_mux_class, NULL, fwnode,
 				switch_fwnode_match);
 
+	/* Skip duplicates */
+	for (i = 0; i < TYPEC_MUX_MAX_DEVS; i++)
+		if (to_typec_switch_dev(dev) == sw_devs[i]) {
+			put_device(dev);
+			return NULL;
+		}
+
 	return dev ? to_typec_switch_dev(dev) : ERR_PTR(-EPROBE_DEFER);
 }
 
@@ -80,7 +89,8 @@ struct typec_switch *fwnode_typec_switch_get(struct fwnode_handle *fwnode)
 	if (!sw)
 		return ERR_PTR(-ENOMEM);
 
-	count = fwnode_connection_find_matches(fwnode, "orientation-switch", NULL,
+	count = fwnode_connection_find_matches(fwnode, "orientation-switch",
+					       (void **)sw_devs,
 					       typec_switch_match,
 					       (void **)sw_devs,
 					       ARRAY_SIZE(sw_devs));
@@ -265,7 +275,9 @@ static int mux_fwnode_match(struct device *dev, const void *fwnode)
 static void *typec_mux_match(const struct fwnode_handle *fwnode,
 			     const char *id, void *data)
 {
+	struct typec_mux_dev **mux_devs = data;
 	struct device *dev;
+	int i;
 
 	/*
 	 * Device graph (OF graph) does not give any means to identify the
@@ -281,6 +293,14 @@ static void *typec_mux_match(const struct fwnode_handle *fwnode,
 	dev = class_find_device(&typec_mux_class, NULL, fwnode,
 				mux_fwnode_match);
 
+	/* Skip duplicates */
+	for (i = 0; i < TYPEC_MUX_MAX_DEVS; i++)
+		if (to_typec_mux_dev(dev) == mux_devs[i]) {
+			put_device(dev);
+			return NULL;
+		}
+
+
 	return dev ? to_typec_mux_dev(dev) : ERR_PTR(-EPROBE_DEFER);
 }
 
@@ -306,7 +326,8 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode)
 		return ERR_PTR(-ENOMEM);
 
 	count = fwnode_connection_find_matches(fwnode, "mode-switch",
-					       NULL, typec_mux_match,
+					       (void **)mux_devs,
+					       typec_mux_match,
 					       (void **)mux_devs,
 					       ARRAY_SIZE(mux_devs));
 	if (count <= 0) {
diff --git a/drivers/usb/typec/mux/ps883x.c b/drivers/usb/typec/mux/ps883x.c
index 5f2879749769e..1256252eceedc 100644
--- a/drivers/usb/typec/mux/ps883x.c
+++ b/drivers/usb/typec/mux/ps883x.c
@@ -444,6 +444,7 @@ static int ps883x_retimer_probe(struct i2c_client *client)
 		goto err_switch_unregister;
 	}
 
+	i2c_set_clientdata(client, retimer);
 	return 0;
 
 err_switch_unregister:
diff --git a/drivers/usb/typec/tcpm/Kconfig b/drivers/usb/typec/tcpm/Kconfig
index 8cdd84ca5d6f7..00baa7503d452 100644
--- a/drivers/usb/typec/tcpm/Kconfig
+++ b/drivers/usb/typec/tcpm/Kconfig
@@ -58,6 +58,8 @@ config TYPEC_FUSB302
 	tristate "Fairchild FUSB302 Type-C chip driver"
 	depends on I2C
 	depends on EXTCON || !EXTCON
+	depends on DRM || DRM=n
+	select DRM_AUX_HPD_BRIDGE if DRM_BRIDGE && OF
 	help
 	  The Fairchild FUSB302 Type-C chip driver that works with
 	  Type-C Port Controller Manager to provide USB PD and USB
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index 19ff8217818e7..ce7069fb4be69 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -5,6 +5,7 @@
  * Fairchild FUSB302 Type-C Chip Driver
  */
 
+#include <drm/bridge/aux-bridge.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
@@ -1689,6 +1690,7 @@ static int fusb302_probe(struct i2c_client *client)
 {
 	struct fusb302_chip *chip;
 	struct i2c_adapter *adapter = client->adapter;
+	struct auxiliary_device *bridge_dev;
 	struct device *dev = &client->dev;
 	const char *name;
 	int ret = 0;
@@ -1747,6 +1749,13 @@ static int fusb302_probe(struct i2c_client *client)
 		goto destroy_workqueue;
 	}
 
+	bridge_dev = devm_drm_dp_hpd_bridge_alloc(chip->dev, to_of_node(chip->tcpc_dev.fwnode));
+	if (IS_ERR(bridge_dev)) {
+		ret = PTR_ERR(bridge_dev);
+		dev_err_probe(chip->dev, ret, "failed to alloc bridge\n");
+		goto destroy_workqueue;
+	}
+
 	chip->tcpm_port = tcpm_register_port(&client->dev, &chip->tcpc_dev);
 	if (IS_ERR(chip->tcpm_port)) {
 		fwnode_handle_put(chip->tcpc_dev.fwnode);
@@ -1764,6 +1773,10 @@ static int fusb302_probe(struct i2c_client *client)
 	enable_irq_wake(chip->gpio_int_n_irq);
 	i2c_set_clientdata(client, chip);
 
+	ret = devm_drm_dp_hpd_bridge_add(chip->dev, bridge_dev);
+	if (ret)
+		return ret;
+
 	return ret;
 
 tcpm_unregister_port:
diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c
index 2a951c585e92b..8b7e6eb92ca2e 100644
--- a/drivers/usb/typec/tcpm/tcpci.c
+++ b/drivers/usb/typec/tcpm/tcpci.c
@@ -999,7 +999,7 @@ static int tcpci_resume(struct device *dev)
 	return ret;
 }
 
-DEFINE_SIMPLE_DEV_PM_OPS(tcpci_pm_ops, tcpci_suspend, tcpci_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tcpci_pm_ops, tcpci_suspend, tcpci_resume);
 
 static const struct i2c_device_id tcpci_id[] = {
 	{ "tcpci" },
diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c
index 88c50b984e8a3..37cf55ad74f84 100644
--- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c
+++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c
@@ -19,9 +19,11 @@
 #include <linux/regulator/consumer.h>
 
 #define RT1711H_VID		0x29CF
+#define ET7304_VID		0x6DCF
 #define RT1711H_PID		0x1711
 #define RT1711H_DID		0x2171
 #define RT1715_DID		0x2173
+#define ET7304_DID		0x2173
 
 #define RT1711H_PHYCTRL1	0x80
 #define RT1711H_PHYCTRL2	0x81
@@ -55,6 +57,7 @@
 
 struct rt1711h_chip_info {
 	u32 rxdz_sel;
+	u16 vid;
 	u16 did;
 	bool enable_pd30_extended_message;
 };
@@ -308,7 +311,7 @@ static int rt1711h_check_revision(struct i2c_client *i2c, struct rt1711h_chip *c
 	ret = i2c_smbus_read_word_data(i2c, TCPC_VENDOR_ID);
 	if (ret < 0)
 		return ret;
-	if (ret != RT1711H_VID) {
+	if (ret != chip->info->vid) {
 		dev_err(&i2c->dev, "vid is not correct, 0x%04x\n", ret);
 		return -ENODEV;
 	}
@@ -405,17 +408,27 @@ static void rt1711h_remove(struct i2c_client *client)
 	tcpci_unregister_port(chip->tcpci);
 }
 
+static const struct rt1711h_chip_info et7304 = {
+	.rxdz_sel = RT1711H_BMCIO_RXDZSEL,
+	.vid = ET7304_VID,
+	.did = ET7304_DID,
+	.enable_pd30_extended_message = true,
+};
+
 static const struct rt1711h_chip_info rt1711h = {
+	.vid = RT1711H_VID,
 	.did = RT1711H_DID,
 };
 
 static const struct rt1711h_chip_info rt1715 = {
 	.rxdz_sel = RT1711H_BMCIO_RXDZSEL,
+	.vid = RT1711H_VID,
 	.did = RT1715_DID,
 	.enable_pd30_extended_message = true,
 };
 
 static const struct i2c_device_id rt1711h_id[] = {
+	{ "et7304", (kernel_ulong_t)&et7304 },
 	{ "rt1711h", (kernel_ulong_t)&rt1711h },
 	{ "rt1715", (kernel_ulong_t)&rt1715 },
 	{}
@@ -423,6 +436,7 @@ static const struct i2c_device_id rt1711h_id[] = {
 MODULE_DEVICE_TABLE(i2c, rt1711h_id);
 
 static const struct of_device_id rt1711h_of_match[] = {
+	{ .compatible = "etekmicro,et7304", .data = &et7304 },
 	{ .compatible = "richtek,rt1711h", .data = &rt1711h },
 	{ .compatible = "richtek,rt1715", .data = &rt1715 },
 	{}
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 8e0e14a2704e0..dfbb94ddc98a6 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -12,6 +12,7 @@
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/kthread.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/power_supply.h>
@@ -61,6 +62,7 @@
 	S(SNK_WAIT_CAPABILITIES_TIMEOUT),	\
 	S(SNK_NEGOTIATE_CAPABILITIES),		\
 	S(SNK_NEGOTIATE_PPS_CAPABILITIES),	\
+	S(SNK_NEGOTIATE_SPR_AVS_CAPABILITIES),	\
 	S(SNK_TRANSITION_SINK),			\
 	S(SNK_TRANSITION_SINK_VBUS),		\
 	S(SNK_READY),				\
@@ -188,7 +190,8 @@
 	S(STRUCTURED_VDMS),			\
 	S(COUNTRY_INFO),			\
 	S(COUNTRY_CODES),			\
-	S(REVISION_INFORMATION)
+	S(REVISION_INFORMATION),		\
+	S(GETTING_SINK_EXTENDED_CAPABILITIES)
 
 #define GENERATE_ENUM(e)	e
 #define GENERATE_STRING(s)	#s
@@ -229,6 +232,7 @@ enum pd_msg_request {
 	PD_MSG_DATA_SINK_CAP,
 	PD_MSG_DATA_SOURCE_CAP,
 	PD_MSG_DATA_REV,
+	PD_MSG_EXT_SINK_CAP_EXT
 };
 
 enum adev_actions {
@@ -305,6 +309,51 @@ struct pd_pps_data {
 	bool active;
 };
 
+enum spr_avs_status {
+	SPR_AVS_UNKNOWN,
+	SPR_AVS_NOT_SUPPORTED,
+	SPR_AVS_SUPPORTED
+};
+
+static const char * const spr_avs_status_strings[] = {
+	[SPR_AVS_UNKNOWN]	= "Unknown",
+	[SPR_AVS_SUPPORTED]	= "Supported",
+	[SPR_AVS_NOT_SUPPORTED]	= "Not Supported",
+};
+
+/*
+ * Standard Power Range Adjustable Voltage Supply (SPR - AVS) data
+ * @max_current_ma_9v_to_15v: Max current for 9V to 15V range derived from
+ *                            source cap & sink cap
+ * @max_current_ma_15v_to_20v: Max current for 15V to 20V range derived from
+ *                             source cap & sink cap
+ * @req_op_curr_ma: Requested operating current to the port partner acting as source
+ * @req_out_volt_mv: Requested output voltage to the port partner acting as source
+ * @max_out_volt_mv: Max SPR voltage supported by the port and the port partner
+ * @max_current_ma; MAX SPR current supported by the port and the port partner
+ * @port_partner_src_status: SPR AVS status of port partner acting as source
+ * @port_partner_src_pdo_index: PDO index of SPR AVS cap of the port partner
+ *                              acting as source. Valid only when
+ *                              port_partner_src_status is SPR_AVS_SUPPORTED.
+ * @port_snk_status: SPR AVS status of the local port acting as sink.
+ * @port_snk_pdo_index: PDO index of SPR AVS cap of local port acting as sink
+ * @active: True when the local port acting as the sink has negotiated SPR AVS
+ *          with the partner acting as source.
+ */
+struct pd_spr_avs_data {
+	u32 max_current_ma_9v_to_15v;
+	u32 max_current_ma_15v_to_20v;
+	u32 req_op_curr_ma;
+	u32 req_out_volt_mv;
+	u32 max_out_volt_mv;
+	u32 max_current_ma;
+	enum spr_avs_status port_partner_src_status;
+	unsigned int port_partner_src_pdo_index;
+	enum spr_avs_status port_snk_status;
+	unsigned int port_snk_pdo_index;
+	bool active;
+};
+
 struct pd_data {
 	struct usb_power_delivery *pd;
 	struct usb_power_delivery_capabilities *source_cap;
@@ -337,6 +386,47 @@ struct pd_timings {
 	u32 snk_bc12_cmpletion_time;
 };
 
+/* Convert microwatt to watt */
+#define UW_TO_W(pow)					((pow) / 1000000)
+
+/*
+ * struct pd_identifier - Contains info about PD identifiers
+ * @vid: Vendor ID (assigned by USB-IF)
+ * @pid: Product ID (assigned by manufacturer)
+ * @xid: Value assigned by USB-IF for product
+ */
+struct pd_identifier {
+	u16 vid;
+	u16 pid;
+	u32 xid;
+};
+
+/*
+ * struct sink_caps_ext_data - Sink extended capability data
+ * @load_step: Indicates the load step slew rate. Value of 0 indicates 150mA/us
+ *             & 1 indicates 500 mA/us
+ * @load_char: Snk overload characteristics
+ * @compliance: Types of sources the sink has been tested & certified on
+ * @modes: Charging caps & power sources supported
+ * @spr_min_pdp: Sink Minimum PDP for SPR mode (in Watts)
+ * @spr_op_pdp: Sink Operational PDP for SPR mode (in Watts)
+ * @spr_max_pdp: Sink Maximum PDP for SPR mode (in Watts)
+ */
+struct sink_caps_ext_data {
+	u8 load_step;
+	u16 load_char;
+	u8 compliance;
+	u8 modes;
+	u8 spr_min_pdp;
+	u8 spr_op_pdp;
+	u8 spr_max_pdp;
+};
+
+enum aug_req_type {
+	PD_PPS,
+	PD_SPR_AVS,
+};
+
 struct tcpm_port {
 	struct device *dev;
 
@@ -499,9 +589,14 @@ struct tcpm_port {
 
 	/* PPS */
 	struct pd_pps_data pps_data;
-	struct completion pps_complete;
-	bool pps_pending;
-	int pps_status;
+
+	/* SPR AVS */
+	struct pd_spr_avs_data spr_avs_data;
+
+	/* Augmented supply request - PPS; SPR_AVS */
+	struct completion aug_supply_req_complete;
+	bool aug_supply_req_pending;
+	int aug_supply_req_status;
 
 	/* Alternate mode data */
 	struct pd_mode_data mode_data;
@@ -585,6 +680,9 @@ struct tcpm_port {
 
 	/* Indicates maximum (revision, version) supported */
 	struct pd_revision_info pd_rev;
+
+	struct pd_identifier pd_ident;
+	struct sink_caps_ext_data sink_caps_ext;
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dentry;
 	struct mutex logbuffer_lock;	/* log buffer access lock */
@@ -605,9 +703,9 @@ struct altmode_vdm_event {
 	struct kthread_work work;
 	struct tcpm_port *port;
 	u32 header;
-	u32 *data;
 	int cnt;
 	enum tcpm_transmit_type tx_sop_type;
+	u32 data[] __counted_by(cnt);
 };
 
 static const char * const pd_rev[] = {
@@ -725,7 +823,7 @@ static void _tcpm_log(struct tcpm_port *port, const char *fmt, va_list args)
 
 	if (tcpm_log_full(port)) {
 		port->logbuffer_head = max(port->logbuffer_head - 1, 0);
-		strcpy(tmpbuffer, "overflow");
+		strscpy(tmpbuffer, "overflow");
 	}
 
 	if (port->logbuffer_head < 0 ||
@@ -841,10 +939,10 @@ static void tcpm_log_source_caps(struct tcpm_port *port)
 					  pdo_spr_avs_apdo_15v_to_20v_max_current_ma(pdo),
 					  pdo_spr_avs_apdo_src_peak_current(pdo));
 			else
-				strcpy(msg, "undefined APDO");
+				strscpy(msg, "undefined APDO");
 			break;
 		default:
-			strcpy(msg, "undefined");
+			strscpy(msg, "undefined");
 			break;
 		}
 		tcpm_log(port, " PDO %d: type %d, %s",
@@ -1367,6 +1465,64 @@ static int tcpm_pd_send_sink_caps(struct tcpm_port *port)
 	return tcpm_pd_transmit(port, TCPC_TX_SOP, &msg);
 }
 
+static int tcpm_pd_send_sink_cap_ext(struct tcpm_port *port)
+{
+	u16 operating_snk_watt = port->operating_snk_mw / 1000;
+	struct sink_caps_ext_data *data = &port->sink_caps_ext;
+	struct pd_identifier *pd_ident = &port->pd_ident;
+	struct sink_caps_ext_msg skedb = {0};
+	struct pd_message msg;
+	u8 data_obj_cnt;
+
+	if (!port->self_powered)
+		data->spr_op_pdp = operating_snk_watt;
+
+	/*
+	 * SPR Sink Minimum PDP indicates the minimum power required to operate
+	 * a sink device in its lowest level of functionality without requiring
+	 * power from the battery. We can use the operating_snk_watt value to
+	 * populate it, as operating_snk_watt indicates device's min operating
+	 * power.
+	 */
+	data->spr_min_pdp = operating_snk_watt;
+
+	if (data->spr_op_pdp < data->spr_min_pdp ||
+	    data->spr_max_pdp < data->spr_op_pdp) {
+		tcpm_log(port,
+			 "Invalid PDP values, Min PDP:%u, Op PDP:%u, Max PDP:%u",
+			 data->spr_min_pdp, data->spr_op_pdp, data->spr_max_pdp);
+		return -EOPNOTSUPP;
+	}
+
+	memset(&msg, 0, sizeof(msg));
+	skedb.vid = cpu_to_le16(pd_ident->vid);
+	skedb.pid = cpu_to_le16(pd_ident->pid);
+	skedb.xid = cpu_to_le32(pd_ident->xid);
+	skedb.skedb_ver = SKEDB_VER_1_0;
+	skedb.load_step = data->load_step;
+	skedb.load_char = cpu_to_le16(data->load_char);
+	skedb.compliance = data->compliance;
+	skedb.modes = data->modes;
+	skedb.spr_min_pdp = data->spr_min_pdp;
+	skedb.spr_op_pdp = data->spr_op_pdp;
+	skedb.spr_max_pdp = data->spr_max_pdp;
+	memcpy(msg.ext_msg.data, &skedb, sizeof(skedb));
+	msg.ext_msg.header = PD_EXT_HDR_LE(sizeof(skedb),
+					   0, /* Denotes if request chunk */
+					   0, /* Chunk Number */
+					   1  /* Chunked */);
+
+	data_obj_cnt = count_chunked_data_objs(sizeof(skedb));
+	msg.header = cpu_to_le16(PD_HEADER(PD_EXT_SINK_CAP_EXT,
+					   port->pwr_role,
+					   port->data_role,
+					   port->negotiated_rev,
+					   port->message_id,
+					   data_obj_cnt,
+					   1 /* Denotes if ext header */));
+	return tcpm_pd_transmit(port, TCPC_TX_SOP, &msg);
+}
+
 static void mod_tcpm_delayed_work(struct tcpm_port *port, unsigned int delay_ms)
 {
 	if (delay_ms) {
@@ -1653,7 +1809,6 @@ static void tcpm_queue_vdm_work(struct kthread_work *work)
 	tcpm_queue_vdm(port, event->header, event->data, event->cnt, event->tx_sop_type);
 
 port_unlock:
-	kfree(event->data);
 	kfree(event);
 	mutex_unlock(&port->lock);
 }
@@ -1662,35 +1817,27 @@ static int tcpm_queue_vdm_unlocked(struct tcpm_port *port, const u32 header,
 				   const u32 *data, int cnt, enum tcpm_transmit_type tx_sop_type)
 {
 	struct altmode_vdm_event *event;
-	u32 *data_cpy;
 	int ret = -ENOMEM;
 
-	event = kzalloc_obj(*event);
+	event = kzalloc_flex(*event, data, cnt);
 	if (!event)
 		goto err_event;
 
-	data_cpy = kcalloc(cnt, sizeof(u32), GFP_KERNEL);
-	if (!data_cpy)
-		goto err_data;
-
 	kthread_init_work(&event->work, tcpm_queue_vdm_work);
+	event->cnt = cnt;
 	event->port = port;
 	event->header = header;
-	memcpy(data_cpy, data, sizeof(u32) * cnt);
-	event->data = data_cpy;
-	event->cnt = cnt;
+	memcpy(event->data, data, sizeof(u32) * cnt);
 	event->tx_sop_type = tx_sop_type;
 
 	ret = kthread_queue_work(port->wq, &event->work);
 	if (!ret) {
 		ret = -EBUSY;
-		goto err_queue;
+		goto err_data;
 	}
 
 	return 0;
 
-err_queue:
-	kfree(data_cpy);
 err_data:
 	kfree(event);
 err_event:
@@ -3194,6 +3341,7 @@ static void tcpm_pd_data_request(struct tcpm_port *port,
 
 	switch (type) {
 	case PD_DATA_SOURCE_CAP:
+		port->spr_avs_data.port_partner_src_status = SPR_AVS_UNKNOWN;
 		for (i = 0; i < cnt; i++)
 			port->source_caps[i] = le32_to_cpu(msg->payload[i]);
 
@@ -3365,12 +3513,12 @@ static void tcpm_pd_data_request(struct tcpm_port *port,
 	}
 }
 
-static void tcpm_pps_complete(struct tcpm_port *port, int result)
+static void tcpm_aug_supply_req_complete(struct tcpm_port *port, int result)
 {
-	if (port->pps_pending) {
-		port->pps_status = result;
-		port->pps_pending = false;
-		complete(&port->pps_complete);
+	if (port->aug_supply_req_pending) {
+		port->aug_supply_req_status = result;
+		port->aug_supply_req_pending = false;
+		complete(&port->aug_supply_req_complete);
 	}
 }
 
@@ -3468,7 +3616,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 			/* Revert data back from any requested PPS updates */
 			port->pps_data.req_out_volt = port->supply_voltage;
 			port->pps_data.req_op_curr = port->current_limit;
-			port->pps_status = (type == PD_CTRL_WAIT ?
+			port->aug_supply_req_status = (type == PD_CTRL_WAIT ?
 					    -EAGAIN : -EOPNOTSUPP);
 
 			/* Threshold was relaxed before sending Request. Restore it back. */
@@ -3476,6 +3624,20 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 							       port->pps_data.active,
 							       port->supply_voltage);
 
+			tcpm_set_state(port, SNK_READY, 0);
+			break;
+		case SNK_NEGOTIATE_SPR_AVS_CAPABILITIES:
+			/* Revert data back from any requested SPR AVS updates */
+			port->spr_avs_data.req_out_volt_mv = port->supply_voltage;
+			port->spr_avs_data.req_op_curr_ma = port->current_limit;
+			port->aug_supply_req_status = (type == PD_CTRL_WAIT ?
+					      -EAGAIN : -EOPNOTSUPP);
+
+			/* Threshold was relaxed before sending Request. Restore it back. */
+			tcpm_set_auto_vbus_discharge_threshold(port, TYPEC_PWR_MODE_PD,
+							       port->spr_avs_data.active,
+							       port->supply_voltage);
+
 			tcpm_set_state(port, SNK_READY, 0);
 			break;
 		case DR_SWAP_SEND:
@@ -3530,6 +3692,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 		switch (port->state) {
 		case SNK_NEGOTIATE_CAPABILITIES:
 			port->pps_data.active = false;
+			port->spr_avs_data.active = false;
 			tcpm_set_state(port, SNK_TRANSITION_SINK, 0);
 			break;
 		case SNK_NEGOTIATE_PPS_CAPABILITIES:
@@ -3542,6 +3705,13 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 			power_supply_changed(port->psy);
 			tcpm_set_state(port, SNK_TRANSITION_SINK, 0);
 			break;
+		case SNK_NEGOTIATE_SPR_AVS_CAPABILITIES:
+			port->spr_avs_data.active = true;
+			port->req_supply_voltage = port->spr_avs_data.req_out_volt_mv;
+			port->req_current_limit = port->spr_avs_data.req_op_curr_ma;
+			power_supply_changed(port->psy);
+			tcpm_set_state(port, SNK_TRANSITION_SINK, 0);
+			break;
 		case SOFT_RESET_SEND:
 			if (port->ams == SOFT_RESET_AMS)
 				tcpm_ams_finish(port);
@@ -3655,6 +3825,19 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 					   PD_MSG_CTRL_NOT_SUPP,
 					   NONE_AMS);
 		break;
+	case PD_CTRL_GET_SINK_CAP_EXT:
+		/* This is an unsupported message if port type is SRC */
+		if (port->negotiated_rev >= PD_REV30 &&
+		    port->port_type != TYPEC_PORT_SRC)
+			tcpm_pd_handle_msg(port, PD_MSG_EXT_SINK_CAP_EXT,
+					   GETTING_SINK_EXTENDED_CAPABILITIES);
+		else
+			tcpm_pd_handle_msg(port,
+					   port->negotiated_rev < PD_REV30 ?
+					   PD_MSG_CTRL_REJECT :
+					   PD_MSG_CTRL_NOT_SUPP,
+					   NONE_AMS);
+		break;
 	default:
 		tcpm_pd_handle_msg(port,
 				   port->negotiated_rev < PD_REV30 ?
@@ -3907,6 +4090,16 @@ static bool tcpm_send_queued_message(struct tcpm_port *port)
 					 ret);
 			tcpm_ams_finish(port);
 			break;
+		case PD_MSG_EXT_SINK_CAP_EXT:
+			ret = tcpm_pd_send_sink_cap_ext(port);
+			if (ret == -EOPNOTSUPP)
+				tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP, TCPC_TX_SOP);
+			else if (ret < 0)
+				tcpm_log(port,
+					 "Unable to transmit sink cap extended, ret=%d",
+					 ret);
+			tcpm_ams_finish(port);
+			break;
 		default:
 			break;
 		}
@@ -4016,9 +4209,9 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo,
 		case PDO_TYPE_APDO:
 			if (pdo_apdo_type(pdo) == APDO_TYPE_PPS) {
 				port->pps_data.supported = true;
-				port->usb_type =
-					POWER_SUPPLY_USB_TYPE_PD_PPS;
-				power_supply_changed(port->psy);
+			} else if (pdo_apdo_type(pdo) == APDO_TYPE_SPR_AVS) {
+				port->spr_avs_data.port_partner_src_status = SPR_AVS_SUPPORTED;
+				port->spr_avs_data.port_partner_src_pdo_index = i;
 			}
 			continue;
 		default:
@@ -4056,6 +4249,10 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo,
 				min_snk_mv = pdo_min_voltage(pdo);
 				break;
 			case PDO_TYPE_APDO:
+				if (pdo_apdo_type(pdo) == APDO_TYPE_SPR_AVS) {
+					port->spr_avs_data.port_snk_status = SPR_AVS_SUPPORTED;
+					port->spr_avs_data.port_snk_pdo_index = j;
+				}
 				continue;
 			default:
 				tcpm_log(port, "Invalid sink PDO type, ignoring");
@@ -4077,6 +4274,23 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo,
 		}
 	}
 
+	if (port->spr_avs_data.port_snk_status == SPR_AVS_UNKNOWN)
+		port->spr_avs_data.port_snk_status = SPR_AVS_NOT_SUPPORTED;
+
+	if (port->spr_avs_data.port_partner_src_status == SPR_AVS_UNKNOWN)
+		port->spr_avs_data.port_partner_src_status = SPR_AVS_NOT_SUPPORTED;
+
+	if (port->pps_data.supported &&
+	    port->spr_avs_data.port_partner_src_status == SPR_AVS_SUPPORTED)
+		port->usb_type = POWER_SUPPLY_USB_TYPE_PD_PPS_SPR_AVS;
+	else if (port->pps_data.supported)
+		port->usb_type = POWER_SUPPLY_USB_TYPE_PD_PPS;
+	else if (port->spr_avs_data.port_partner_src_status == SPR_AVS_SUPPORTED)
+		port->usb_type = POWER_SUPPLY_USB_TYPE_PD_SPR_AVS;
+
+	if (port->usb_type != POWER_SUPPLY_USB_TYPE_PD)
+		power_supply_changed(port->psy);
+
 	return ret;
 }
 
@@ -4127,6 +4341,88 @@ static unsigned int tcpm_pd_select_pps_apdo(struct tcpm_port *port)
 	return src_pdo;
 }
 
+static int tcpm_pd_select_spr_avs_apdo(struct tcpm_port *port)
+{
+	u32 req_out_volt_mv, req_op_curr_ma, src_max_curr_ma = 0, source_cap;
+	u32 snk_max_curr_ma = 0, src_pdo_index, snk_pdo_index, snk_pdo;
+
+	if (port->spr_avs_data.port_snk_status != SPR_AVS_SUPPORTED ||
+	    port->spr_avs_data.port_partner_src_status !=
+	    SPR_AVS_SUPPORTED) {
+		tcpm_log(port, "SPR AVS not supported. port:%s partner:%s",
+			 spr_avs_status_strings[port->spr_avs_data.port_snk_status],
+			 spr_avs_status_strings[port->spr_avs_data.port_partner_src_status]);
+		return -EOPNOTSUPP;
+	}
+
+	/* Round up to SPR_AVS_VOLT_MV_STEP */
+	req_out_volt_mv = port->spr_avs_data.req_out_volt_mv;
+	if (req_out_volt_mv % SPR_AVS_VOLT_MV_STEP) {
+		req_out_volt_mv += SPR_AVS_VOLT_MV_STEP -
+			(req_out_volt_mv % SPR_AVS_VOLT_MV_STEP);
+		port->spr_avs_data.req_out_volt_mv = req_out_volt_mv;
+	}
+
+	/* Round up to RDO_SPR_AVS_CURR_MA_STEP */
+	req_op_curr_ma = port->spr_avs_data.req_op_curr_ma;
+	if (req_op_curr_ma % RDO_SPR_AVS_CURR_MA_STEP) {
+		req_op_curr_ma += RDO_SPR_AVS_CURR_MA_STEP -
+			(req_op_curr_ma % RDO_SPR_AVS_CURR_MA_STEP);
+		port->spr_avs_data.req_op_curr_ma = req_op_curr_ma;
+	}
+
+	src_pdo_index = port->spr_avs_data.port_partner_src_pdo_index;
+	snk_pdo_index = port->spr_avs_data.port_snk_pdo_index;
+	source_cap = port->source_caps[src_pdo_index];
+	snk_pdo = port->snk_pdo[snk_pdo_index];
+	tcpm_log(port,
+		 "SPR AVS src_pdo_index:%d snk_pdo_index:%d req_op_curr_ma roundup:%u req_out_volt_mv roundup:%u",
+		 src_pdo_index, snk_pdo_index, req_op_curr_ma, req_out_volt_mv);
+
+	if (req_out_volt_mv >= SPR_AVS_TIER1_MIN_VOLT_MV &&
+	    req_out_volt_mv <= SPR_AVS_TIER1_MAX_VOLT_MV) {
+		src_max_curr_ma =
+			pdo_spr_avs_apdo_9v_to_15v_max_current_ma(source_cap);
+		snk_max_curr_ma =
+			pdo_spr_avs_apdo_9v_to_15v_max_current_ma(snk_pdo);
+	} else if (req_out_volt_mv > SPR_AVS_TIER1_MAX_VOLT_MV &&
+		   req_out_volt_mv <= SPR_AVS_TIER2_MAX_VOLT_MV) {
+		src_max_curr_ma =
+			pdo_spr_avs_apdo_15v_to_20v_max_current_ma(source_cap);
+		snk_max_curr_ma =
+			pdo_spr_avs_apdo_15v_to_20v_max_current_ma(snk_pdo);
+	} else {
+		tcpm_log(port, "Invalid SPR AVS req_volt:%umV", req_out_volt_mv);
+		return -EINVAL;
+	}
+
+	if (req_op_curr_ma > src_max_curr_ma ||
+	    req_op_curr_ma > snk_max_curr_ma) {
+		tcpm_log(port,
+			 "Invalid SPR AVS request. req_volt:%umV req_curr:%umA src_max_cur:%umA snk_max_cur:%umA",
+			 req_out_volt_mv, req_op_curr_ma, src_max_curr_ma,
+			 snk_max_curr_ma);
+		return -EINVAL;
+	}
+
+	/* Max SPR voltage based on both the port and the partner caps */
+	if (pdo_spr_avs_apdo_15v_to_20v_max_current_ma(snk_pdo) &&
+	    pdo_spr_avs_apdo_15v_to_20v_max_current_ma(source_cap))
+		port->spr_avs_data.max_out_volt_mv = SPR_AVS_TIER2_MAX_VOLT_MV;
+	else
+		port->spr_avs_data.max_out_volt_mv = SPR_AVS_TIER1_MAX_VOLT_MV;
+
+	/*
+	 * Max SPR AVS curr based on 9V to 15V. This should be higher than or
+	 * equal to 15V to 20V range.
+	 */
+	port->spr_avs_data.max_current_ma =
+		min(pdo_spr_avs_apdo_9v_to_15v_max_current_ma(source_cap),
+		    pdo_spr_avs_apdo_9v_to_15v_max_current_ma(snk_pdo));
+
+	return src_pdo_index;
+}
+
 static int tcpm_pd_build_request(struct tcpm_port *port, u32 *rdo)
 {
 	unsigned int mv, ma, mw, flags;
@@ -4294,13 +4590,74 @@ static int tcpm_pd_build_pps_request(struct tcpm_port *port, u32 *rdo)
 	return 0;
 }
 
-static int tcpm_pd_send_pps_request(struct tcpm_port *port)
+static int tcpm_pd_build_spr_avs_request(struct tcpm_port *port, u32 *rdo)
+{
+	u32 out_mv, op_ma, flags, snk_pdo_index, source_cap;
+	unsigned int src_power_mw, snk_power_mw;
+	int src_pdo_index;
+	u32 snk_pdo;
+
+	src_pdo_index = tcpm_pd_select_spr_avs_apdo(port);
+	if (src_pdo_index < 0)
+		return src_pdo_index;
+	snk_pdo_index = port->spr_avs_data.port_snk_pdo_index;
+	source_cap = port->source_caps[src_pdo_index];
+	snk_pdo = port->snk_pdo[snk_pdo_index];
+	out_mv = port->spr_avs_data.req_out_volt_mv;
+	op_ma = port->spr_avs_data.req_op_curr_ma;
+
+	flags = RDO_USB_COMM | RDO_NO_SUSPEND;
+
+	/*
+	 * Set capability mismatch when the maximum power needs in the current
+	 * requested AVS voltage tier range is greater than
+	 * port->operating_snk_mw, however, the maximum power offered by the
+	 * source at the current requested AVS voltage tier is less than
+	 * port->operating_sink_mw.
+	 */
+	if (out_mv > SPR_AVS_TIER1_MAX_VOLT_MV) {
+		src_power_mw =
+			pdo_spr_avs_apdo_15v_to_20v_max_current_ma(source_cap) *
+			SPR_AVS_TIER2_MAX_VOLT_MV / 1000;
+		snk_power_mw =
+			pdo_spr_avs_apdo_15v_to_20v_max_current_ma(snk_pdo) *
+			SPR_AVS_TIER2_MAX_VOLT_MV / 1000;
+	} else {
+		src_power_mw =
+			pdo_spr_avs_apdo_9v_to_15v_max_current_ma(source_cap) *
+			SPR_AVS_TIER1_MAX_VOLT_MV / 1000;
+		snk_power_mw =
+			pdo_spr_avs_apdo_9v_to_15v_max_current_ma(snk_pdo) *
+			SPR_AVS_TIER1_MAX_VOLT_MV / 1000;
+	}
+
+	if (snk_power_mw >= port->operating_snk_mw &&
+	    src_power_mw < port->operating_snk_mw)
+		flags |= RDO_CAP_MISMATCH;
+
+	*rdo = RDO_AVS(src_pdo_index + 1, out_mv, op_ma, flags);
+
+	tcpm_log(port, "Requesting APDO SPR AVS %d: %u mV, %u mA",
+		 src_pdo_index, out_mv, op_ma);
+
+	return 0;
+}
+
+static int tcpm_pd_send_aug_supply_request(struct tcpm_port *port,
+					   enum aug_req_type type)
 {
 	struct pd_message msg;
 	int ret;
 	u32 rdo;
 
-	ret = tcpm_pd_build_pps_request(port, &rdo);
+	if (type == PD_PPS) {
+		ret = tcpm_pd_build_pps_request(port, &rdo);
+	} else if (type == PD_SPR_AVS) {
+		ret = tcpm_pd_build_spr_avs_request(port, &rdo);
+	} else {
+		tcpm_log(port, "Invalid aug_req_type %d", type);
+		ret = -EOPNOTSUPP;
+	}
 	if (ret < 0)
 		return ret;
 
@@ -4523,6 +4880,14 @@ static void tcpm_set_partner_usb_comm_capable(struct tcpm_port *port, bool capab
 		port->tcpc->set_partner_usb_comm_capable(port->tcpc, capable);
 }
 
+static void tcpm_partner_source_caps_reset(struct tcpm_port *port)
+{
+	usb_power_delivery_unregister_capabilities(port->partner_source_caps);
+	port->partner_source_caps = NULL;
+	port->spr_avs_data.port_partner_src_status = SPR_AVS_UNKNOWN;
+	port->spr_avs_data.active = false;
+}
+
 static void tcpm_reset_port(struct tcpm_port *port)
 {
 	tcpm_enable_auto_vbus_discharge(port, false);
@@ -4562,8 +4927,7 @@ static void tcpm_reset_port(struct tcpm_port *port)
 
 	usb_power_delivery_unregister_capabilities(port->partner_sink_caps);
 	port->partner_sink_caps = NULL;
-	usb_power_delivery_unregister_capabilities(port->partner_source_caps);
-	port->partner_source_caps = NULL;
+	tcpm_partner_source_caps_reset(port);
 	usb_power_delivery_unregister(port->partner_pd);
 	port->partner_pd = NULL;
 }
@@ -5055,7 +5419,7 @@ static void run_state_machine(struct tcpm_port *port)
 	case SNK_UNATTACHED:
 		if (!port->non_pd_role_swap)
 			tcpm_swap_complete(port, -ENOTCONN);
-		tcpm_pps_complete(port, -ENOTCONN);
+		tcpm_aug_supply_req_complete(port, -ENOTCONN);
 		tcpm_snk_detach(port);
 		if (port->potential_contaminant) {
 			tcpm_set_state(port, CHECK_CONTAMINANT, 0);
@@ -5286,13 +5650,16 @@ static void run_state_machine(struct tcpm_port *port)
 		}
 		break;
 	case SNK_NEGOTIATE_PPS_CAPABILITIES:
-		ret = tcpm_pd_send_pps_request(port);
+	case SNK_NEGOTIATE_SPR_AVS_CAPABILITIES:
+		ret = tcpm_pd_send_aug_supply_request(port, port->state ==
+						      SNK_NEGOTIATE_PPS_CAPABILITIES ?
+						      PD_PPS : PD_SPR_AVS);
 		if (ret < 0) {
 			/* Restore back to the original state */
 			tcpm_set_auto_vbus_discharge_threshold(port, TYPEC_PWR_MODE_PD,
 							       port->pps_data.active,
 							       port->supply_voltage);
-			port->pps_status = ret;
+			port->aug_supply_req_status = ret;
 			/*
 			 * If this was called due to updates to sink
 			 * capabilities, and pps is no longer valid, we should
@@ -5308,23 +5675,58 @@ static void run_state_machine(struct tcpm_port *port)
 		}
 		break;
 	case SNK_TRANSITION_SINK:
-		/* From the USB PD spec:
-		 * "The Sink Shall transition to Sink Standby before a positive or
-		 * negative voltage transition of VBUS. During Sink Standby
-		 * the Sink Shall reduce its power draw to pSnkStdby."
-		 *
-		 * This is not applicable to PPS though as the port can continue
-		 * to draw negotiated power without switching to standby.
-		 */
-		if (port->supply_voltage != port->req_supply_voltage && !port->pps_data.active &&
-		    port->current_limit * port->supply_voltage / 1000 > PD_P_SNK_STDBY_MW) {
-			u32 stdby_ma = PD_P_SNK_STDBY_MW * 1000 / port->supply_voltage;
+		if (port->spr_avs_data.active) {
+			if (abs(port->req_supply_voltage - port->supply_voltage) >
+			    SPR_AVS_AVS_SMALL_STEP_V * 1000) {
+				/*
+				 * The Sink Shall reduce its current draw to
+				 * iSnkStdby within tSnkStdby. The reduction to
+				 * iSnkStdby is not required if the voltage
+				 * increase is less than or equal to
+				 * vAvsSmallStep.
+				 */
+				tcpm_log(port,
+					 "SPR AVS Setting iSnkstandby. Req vol: %u mV Curr vol: %u mV",
+					 port->req_supply_voltage,
+					 port->supply_voltage);
+				tcpm_set_current_limit(port, PD_I_SNK_STBY_MA,
+						       port->supply_voltage);
+			}
+			/*
+			 * Although tAvsSrcTransSmall is expected to be used
+			 * for voltage transistions smaller than 1V, using
+			 * tAvsSrcTransLarge to be resilient against chargers
+			 * which strictly cannot honor tAvsSrcTransSmall to
+			 * improve interoperability.
+			 */
+			tcpm_set_state(port, hard_reset_state(port),
+				       PD_T_AVS_SRC_TRANS_LARGE);
+			/*
+			 * From the USB PD spec:
+			 * "The Sink Shall transition to Sink Standby before a
+			 * positive ornegative voltage transition of VBUS.
+			 * During Sink Standby the Sink Shall reduce its power
+			 * draw to pSnkStdby."
+			 *
+			 * This is not applicable to PPS though as the port can
+			 * continue to draw negotiated power without switching
+			 * to standby.
+			 */
+		} else if (port->supply_voltage != port->req_supply_voltage &&
+			   !port->pps_data.active &&
+			   (port->current_limit * port->supply_voltage / 1000 >
+			   PD_P_SNK_STDBY_MW)) {
+			u32 stdby_ma = PD_P_SNK_STDBY_MW * 1000 /
+				port->supply_voltage;
 
 			tcpm_log(port, "Setting standby current %u mV @ %u mA",
 				 port->supply_voltage, stdby_ma);
-			tcpm_set_current_limit(port, stdby_ma, port->supply_voltage);
+			tcpm_set_current_limit(port, stdby_ma,
+					       port->supply_voltage);
+			tcpm_set_state(port, hard_reset_state(port),
+				       PD_T_PS_TRANSITION);
 		}
-		fallthrough;
+		break;
 	case SNK_TRANSITION_SINK_VBUS:
 		tcpm_set_state(port, hard_reset_state(port),
 			       PD_T_PS_TRANSITION);
@@ -5344,7 +5746,7 @@ static void run_state_machine(struct tcpm_port *port)
 		tcpm_typec_connect(port);
 		if (port->pd_capable && port->source_caps[0] & PDO_FIXED_DUAL_ROLE)
 			mod_enable_frs_delayed_work(port, 0);
-		tcpm_pps_complete(port, port->pps_status);
+		tcpm_aug_supply_req_complete(port, port->aug_supply_req_status);
 
 		if (port->ams != NONE_AMS)
 			tcpm_ams_finish(port);
@@ -5531,8 +5933,7 @@ static void run_state_machine(struct tcpm_port *port)
 		port->message_id = 0;
 		port->rx_msgid = -1;
 		/* remove existing capabilities */
-		usb_power_delivery_unregister_capabilities(port->partner_source_caps);
-		port->partner_source_caps = NULL;
+		tcpm_partner_source_caps_reset(port);
 		tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP);
 		tcpm_ams_finish(port);
 		if (port->pwr_role == TYPEC_SOURCE) {
@@ -5565,8 +5966,7 @@ static void run_state_machine(struct tcpm_port *port)
 			port->message_id = 0;
 			port->rx_msgid = -1;
 			/* remove existing capabilities */
-			usb_power_delivery_unregister_capabilities(port->partner_source_caps);
-			port->partner_source_caps = NULL;
+			tcpm_partner_source_caps_reset(port);
 			if (tcpm_pd_send_control(port, PD_CTRL_SOFT_RESET, TCPC_TX_SOP))
 				tcpm_set_state_cond(port, hard_reset_state(port), 0);
 			else
@@ -5703,8 +6103,7 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	case PR_SWAP_SNK_SRC_SINK_OFF:
 		/* will be source, remove existing capabilities */
-		usb_power_delivery_unregister_capabilities(port->partner_source_caps);
-		port->partner_source_caps = NULL;
+		tcpm_partner_source_caps_reset(port);
 		/*
 		 * Prevent vbus discharge circuit from turning on during PR_SWAP
 		 * as this is not a disconnect.
@@ -5852,7 +6251,7 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	case ERROR_RECOVERY:
 		tcpm_swap_complete(port, -EPROTO);
-		tcpm_pps_complete(port, -EPROTO);
+		tcpm_aug_supply_req_complete(port, -EPROTO);
 		tcpm_set_state(port, PORT_RESET, 0);
 		break;
 	case PORT_RESET:
@@ -6826,7 +7225,7 @@ static int tcpm_try_role(struct typec_port *p, int role)
 	return ret;
 }
 
-static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 req_op_curr)
+static int tcpm_aug_set_op_curr(struct tcpm_port *port, u16 req_op_curr_ma)
 {
 	unsigned int target_mw;
 	int ret;
@@ -6834,7 +7233,19 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 req_op_curr)
 	mutex_lock(&port->swap_lock);
 	mutex_lock(&port->lock);
 
-	if (!port->pps_data.active) {
+	if (port->pps_data.active) {
+		req_op_curr_ma = req_op_curr_ma -
+				 (req_op_curr_ma % RDO_PROG_CURR_MA_STEP);
+		if (req_op_curr_ma > port->pps_data.max_curr) {
+			ret = -EINVAL;
+			goto port_unlock;
+		}
+		target_mw = (req_op_curr_ma * port->supply_voltage) / 1000;
+		if (target_mw < port->operating_snk_mw) {
+			ret = -EINVAL;
+			goto port_unlock;
+		}
+	} else if (!port->spr_avs_data.active) {
 		ret = -EOPNOTSUPP;
 		goto port_unlock;
 	}
@@ -6844,38 +7255,31 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 req_op_curr)
 		goto port_unlock;
 	}
 
-	if (req_op_curr > port->pps_data.max_curr) {
-		ret = -EINVAL;
-		goto port_unlock;
-	}
-
-	target_mw = (req_op_curr * port->supply_voltage) / 1000;
-	if (target_mw < port->operating_snk_mw) {
-		ret = -EINVAL;
-		goto port_unlock;
-	}
+	if (port->pps_data.active)
+		port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES;
+	else
+		port->upcoming_state = SNK_NEGOTIATE_SPR_AVS_CAPABILITIES;
 
-	port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES;
 	ret = tcpm_ams_start(port, POWER_NEGOTIATION);
 	if (ret == -EAGAIN) {
 		port->upcoming_state = INVALID_STATE;
 		goto port_unlock;
 	}
 
-	/* Round down operating current to align with PPS valid steps */
-	req_op_curr = req_op_curr - (req_op_curr % RDO_PROG_CURR_MA_STEP);
-
-	reinit_completion(&port->pps_complete);
-	port->pps_data.req_op_curr = req_op_curr;
-	port->pps_status = 0;
-	port->pps_pending = true;
+	reinit_completion(&port->aug_supply_req_complete);
+	if (port->pps_data.active)
+		port->pps_data.req_op_curr = req_op_curr_ma;
+	else
+		port->spr_avs_data.req_op_curr_ma = req_op_curr_ma;
+	port->aug_supply_req_status = 0;
+	port->aug_supply_req_pending = true;
 	mutex_unlock(&port->lock);
 
-	if (!wait_for_completion_timeout(&port->pps_complete,
-				msecs_to_jiffies(PD_PPS_CTRL_TIMEOUT)))
+	if (!wait_for_completion_timeout(&port->aug_supply_req_complete,
+					 msecs_to_jiffies(PD_AUG_PSY_CTRL_TIMEOUT)))
 		ret = -ETIMEDOUT;
 	else
-		ret = port->pps_status;
+		ret = port->aug_supply_req_status;
 
 	goto swap_unlock;
 
@@ -6887,7 +7291,7 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 req_op_curr)
 	return ret;
 }
 
-static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 req_out_volt)
+static int tcpm_aug_set_out_volt(struct tcpm_port *port, u16 req_out_volt_mv)
 {
 	unsigned int target_mw;
 	int ret;
@@ -6895,7 +7299,16 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 req_out_volt)
 	mutex_lock(&port->swap_lock);
 	mutex_lock(&port->lock);
 
-	if (!port->pps_data.active) {
+	if (port->pps_data.active) {
+		req_out_volt_mv = req_out_volt_mv - (req_out_volt_mv %
+						     RDO_PROG_VOLT_MV_STEP);
+		/* Round down output voltage to align with PPS valid steps */
+		target_mw = (port->current_limit * req_out_volt_mv) / 1000;
+		if (target_mw < port->operating_snk_mw) {
+			ret = -EINVAL;
+			goto port_unlock;
+		}
+	} else if (!port->spr_avs_data.active) {
 		ret = -EOPNOTSUPP;
 		goto port_unlock;
 	}
@@ -6905,33 +7318,31 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 req_out_volt)
 		goto port_unlock;
 	}
 
-	target_mw = (port->current_limit * req_out_volt) / 1000;
-	if (target_mw < port->operating_snk_mw) {
-		ret = -EINVAL;
-		goto port_unlock;
-	}
+	if (port->pps_data.active)
+		port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES;
+	else
+		port->upcoming_state = SNK_NEGOTIATE_SPR_AVS_CAPABILITIES;
 
-	port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES;
 	ret = tcpm_ams_start(port, POWER_NEGOTIATION);
 	if (ret == -EAGAIN) {
 		port->upcoming_state = INVALID_STATE;
 		goto port_unlock;
 	}
 
-	/* Round down output voltage to align with PPS valid steps */
-	req_out_volt = req_out_volt - (req_out_volt % RDO_PROG_VOLT_MV_STEP);
-
-	reinit_completion(&port->pps_complete);
-	port->pps_data.req_out_volt = req_out_volt;
-	port->pps_status = 0;
-	port->pps_pending = true;
+	reinit_completion(&port->aug_supply_req_complete);
+	if (port->pps_data.active)
+		port->pps_data.req_out_volt = req_out_volt_mv;
+	else
+		port->spr_avs_data.req_out_volt_mv = req_out_volt_mv;
+	port->aug_supply_req_status = 0;
+	port->aug_supply_req_pending = true;
 	mutex_unlock(&port->lock);
 
-	if (!wait_for_completion_timeout(&port->pps_complete,
-				msecs_to_jiffies(PD_PPS_CTRL_TIMEOUT)))
+	if (!wait_for_completion_timeout(&port->aug_supply_req_complete,
+					 msecs_to_jiffies(PD_AUG_PSY_CTRL_TIMEOUT)))
 		ret = -ETIMEDOUT;
 	else
-		ret = port->pps_status;
+		ret = port->aug_supply_req_status;
 
 	goto swap_unlock;
 
@@ -6974,9 +7385,9 @@ static int tcpm_pps_activate(struct tcpm_port *port, bool activate)
 		goto port_unlock;
 	}
 
-	reinit_completion(&port->pps_complete);
-	port->pps_status = 0;
-	port->pps_pending = true;
+	reinit_completion(&port->aug_supply_req_complete);
+	port->aug_supply_req_status = 0;
+	port->aug_supply_req_pending = true;
 
 	/* Trigger PPS request or move back to standard PDO contract */
 	if (activate) {
@@ -6985,11 +7396,75 @@ static int tcpm_pps_activate(struct tcpm_port *port, bool activate)
 	}
 	mutex_unlock(&port->lock);
 
-	if (!wait_for_completion_timeout(&port->pps_complete,
-				msecs_to_jiffies(PD_PPS_CTRL_TIMEOUT)))
+	if (!wait_for_completion_timeout(&port->aug_supply_req_complete,
+					 msecs_to_jiffies(PD_AUG_PSY_CTRL_TIMEOUT)))
 		ret = -ETIMEDOUT;
 	else
-		ret = port->pps_status;
+		ret = port->aug_supply_req_status;
+
+	goto swap_unlock;
+
+port_unlock:
+	mutex_unlock(&port->lock);
+swap_unlock:
+	mutex_unlock(&port->swap_lock);
+
+	return ret;
+}
+
+static int tcpm_spr_avs_activate(struct tcpm_port *port, bool activate)
+{
+	int ret = 0;
+
+	mutex_lock(&port->swap_lock);
+	mutex_lock(&port->lock);
+
+	if (port->spr_avs_data.port_snk_status == SPR_AVS_NOT_SUPPORTED ||
+	    port->spr_avs_data.port_partner_src_status == SPR_AVS_NOT_SUPPORTED) {
+		tcpm_log(port, "SPR_AVS not supported");
+		ret = -EOPNOTSUPP;
+		goto port_unlock;
+	}
+
+	/* Trying to deactivate SPR AVS when already deactivated so just bail */
+	if (!port->spr_avs_data.active && !activate)
+		goto port_unlock;
+
+	if (port->state != SNK_READY) {
+		tcpm_log(port,
+			 "SPR_AVS cannot be activated. Port not in SNK_READY");
+		ret = -EAGAIN;
+		goto port_unlock;
+	}
+
+	if (activate)
+		port->upcoming_state = SNK_NEGOTIATE_SPR_AVS_CAPABILITIES;
+	else
+		port->upcoming_state = SNK_NEGOTIATE_CAPABILITIES;
+	ret = tcpm_ams_start(port, POWER_NEGOTIATION);
+	if (ret == -EAGAIN) {
+		tcpm_log(port, "SPR_AVS cannot be %s. AMS start failed",
+			 activate ? "activated" : "deactivated");
+		port->upcoming_state = INVALID_STATE;
+		goto port_unlock;
+	}
+
+	reinit_completion(&port->aug_supply_req_complete);
+	port->aug_supply_req_status = 0;
+	port->aug_supply_req_pending = true;
+
+	/* Trigger AVS request or move back to standard PDO contract */
+	if (activate) {
+		port->spr_avs_data.req_out_volt_mv = port->supply_voltage;
+		port->spr_avs_data.req_op_curr_ma = port->current_limit;
+	}
+	mutex_unlock(&port->lock);
+
+	if (!wait_for_completion_timeout(&port->aug_supply_req_complete,
+					 msecs_to_jiffies(PD_AUG_PSY_CTRL_TIMEOUT)))
+		ret = -ETIMEDOUT;
+	else
+		ret = port->aug_supply_req_status;
 
 	goto swap_unlock;
 
@@ -7145,16 +7620,26 @@ static int tcpm_pd_set(struct typec_port *p, struct usb_power_delivery *pd)
 		break;
 	case SNK_NEGOTIATE_CAPABILITIES:
 	case SNK_NEGOTIATE_PPS_CAPABILITIES:
+	case SNK_NEGOTIATE_SPR_AVS_CAPABILITIES:
 	case SNK_READY:
 	case SNK_TRANSITION_SINK:
 	case SNK_TRANSITION_SINK_VBUS:
-		if (port->pps_data.active)
+		if (port->pps_data.active) {
 			port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES;
-		else if (port->pd_capable)
+		} else if (port->pd_capable) {
 			port->upcoming_state = SNK_NEGOTIATE_CAPABILITIES;
-		else
+			if (port->spr_avs_data.active) {
+				/*
+				 * De-activate AVS and fallback to PD to
+				 * re-evaluate whether AVS is supported in the
+				 * current sink cap set.
+				 */
+				port->spr_avs_data.active = false;
+				port->spr_avs_data.port_snk_status = SPR_AVS_UNKNOWN;
+			}
+		} else {
 			break;
-
+		}
 		port->update_sink_caps = true;
 
 		ret = tcpm_ams_start(port, POWER_NEGOTIATION);
@@ -7291,6 +7776,129 @@ static void tcpm_fw_get_timings(struct tcpm_port *port, struct fwnode_handle *fw
 		port->timings.snk_bc12_cmpletion_time = val;
 }
 
+static void tcpm_fw_get_pd_ident(struct tcpm_port *port)
+{
+	struct pd_identifier *pd_ident = &port->pd_ident;
+	u32 *vdo;
+
+	/* First 3 vdo values contain info regarding USB PID, VID & XID */
+	if (port->nr_snk_vdo >= 3)
+		vdo = port->snk_vdo;
+	else if (port->nr_snk_vdo_v1 >= 3)
+		vdo = port->snk_vdo_v1;
+	else
+		return;
+
+	pd_ident->vid = PD_IDH_VID(vdo[0]);
+	pd_ident->pid = PD_PRODUCT_PID(vdo[2]);
+	pd_ident->xid = PD_CSTAT_XID(vdo[1]);
+	tcpm_log(port, "vid:%#x pid:%#x xid:%#x",
+		 pd_ident->vid, pd_ident->pid, pd_ident->xid);
+}
+
+static void tcpm_parse_snk_pdos(struct tcpm_port *port)
+{
+	struct sink_caps_ext_data *caps = &port->sink_caps_ext;
+	u32 max_mv, max_ma;
+	u8 avs_tier1_pdp, avs_tier2_pdp;
+	int i, pdo_itr;
+	u32 *snk_pdos;
+
+	for (i = 0; i < port->pd_count; ++i) {
+		snk_pdos = port->pd_list[i]->sink_desc.pdo;
+		for (pdo_itr = 0; pdo_itr < PDO_MAX_OBJECTS && snk_pdos[pdo_itr];
+		     ++pdo_itr) {
+			u32 pdo = snk_pdos[pdo_itr];
+			u8 curr_snk_pdp = 0;
+
+			switch (pdo_type(pdo)) {
+			case PDO_TYPE_FIXED:
+				max_mv = pdo_fixed_voltage(pdo);
+				max_ma = pdo_fixed_current(pdo);
+				curr_snk_pdp = UW_TO_W(max_mv * max_ma);
+				break;
+			case PDO_TYPE_BATT:
+				curr_snk_pdp = UW_TO_W(pdo_max_power(pdo));
+				break;
+			case PDO_TYPE_VAR:
+				max_mv = pdo_max_voltage(pdo);
+				max_ma = pdo_max_current(pdo);
+				curr_snk_pdp = UW_TO_W(max_mv * max_ma);
+				break;
+			case PDO_TYPE_APDO:
+				if (pdo_apdo_type(pdo) == APDO_TYPE_PPS) {
+					max_mv = pdo_pps_apdo_max_voltage(pdo);
+					max_ma = pdo_pps_apdo_max_current(pdo);
+					curr_snk_pdp = UW_TO_W(max_mv * max_ma);
+					caps->modes |= SINK_MODE_PPS;
+				} else if (pdo_apdo_type(pdo) ==
+					   APDO_TYPE_SPR_AVS) {
+					avs_tier1_pdp = UW_TO_W(SPR_AVS_TIER1_MAX_VOLT_MV
+						* pdo_spr_avs_apdo_9v_to_15v_max_current_ma(pdo));
+					avs_tier2_pdp = UW_TO_W(SPR_AVS_TIER2_MAX_VOLT_MV
+						* pdo_spr_avs_apdo_15v_to_20v_max_current_ma(pdo));
+					curr_snk_pdp = max(avs_tier1_pdp, avs_tier2_pdp);
+					caps->modes |= SINK_MODE_AVS;
+				}
+				break;
+			default:
+				tcpm_log(port, "Invalid source PDO type, ignoring");
+				continue;
+			}
+
+			caps->spr_max_pdp = max(caps->spr_max_pdp,
+						curr_snk_pdp);
+		}
+	}
+}
+
+static void tcpm_fw_get_sink_caps_ext(struct tcpm_port *port,
+				      struct fwnode_handle *fwnode)
+{
+	struct sink_caps_ext_data *caps = &port->sink_caps_ext;
+	int ret;
+	u32 val;
+
+	/*
+	 * Load step represents the change in current per usec that a given
+	 * source can tolerate while maintaining Vbus within the vSrcValid
+	 * range. For a sink this represents the "preferred" load-step value. It
+	 * can only have 2 values (150 mA/usec or 500 mA/usec) with 150 mA/usec
+	 * being the default.
+	 */
+	ret = fwnode_property_read_u32(fwnode, "sink-load-step", &val);
+	if (!ret)
+		caps->load_step = val == 500 ? 1 : 0;
+
+	fwnode_property_read_u16(fwnode, "sink-load-characteristics",
+				 &caps->load_char);
+	fwnode_property_read_u8(fwnode, "sink-compliance", &caps->compliance);
+	caps->modes = SINK_MODE_VBUS;
+
+	/*
+	 * As per "6.5.13.14" SPR Sink Operational PDP definition, for battery
+	 * powered devices, this value will correspond to the PDP of the
+	 * charging adapter either shipped or recommended for use with it. For
+	 * batteryless sink devices SPR Operational PDP indicates the power
+	 * required to operate all the device's functional modes. Hence, this
+	 * value may be considered equal to port's operating_snk_mw. As
+	 * operating_sink_mw can change as per the pd set used thus, OP PDP
+	 * is determined when populating Sink Caps Extended Data Block.
+	 */
+	if (port->self_powered) {
+		fwnode_property_read_u32(fwnode, "charging-adapter-pdp-milliwatt",
+					 &val);
+		caps->spr_op_pdp = (u8)(val / 1000);
+		caps->modes |= SINK_MODE_BATT;
+	}
+
+	tcpm_parse_snk_pdos(port);
+	tcpm_log(port,
+		 "load-step:%#x load-char:%#x compl:%#x op-pdp:%#x max-pdp:%#x",
+		 caps->load_step, caps->load_char, caps->compliance,
+		 caps->spr_op_pdp, caps->spr_max_pdp);
+}
+
 static int tcpm_fw_get_caps(struct tcpm_port *port, struct fwnode_handle *fwnode)
 {
 	struct fwnode_handle *capabilities, *caps = NULL;
@@ -7464,6 +8072,9 @@ static int tcpm_fw_get_caps(struct tcpm_port *port, struct fwnode_handle *fwnode
 		}
 	}
 
+	if (port->port_type != TYPEC_PORT_SRC)
+		tcpm_fw_get_sink_caps_ext(port, fwnode);
+
 put_caps:
 	if (caps != fwnode)
 		fwnode_handle_put(caps);
@@ -7506,6 +8117,8 @@ static int tcpm_fw_get_snk_vdos(struct tcpm_port *port, struct fwnode_handle *fw
 			return ret;
 	}
 
+	tcpm_fw_get_pd_ident(port);
+
 	return 0;
 }
 
@@ -7536,7 +8149,8 @@ static void tcpm_fw_get_pd_revision(struct tcpm_port *port, struct fwnode_handle
 enum tcpm_psy_online_states {
 	TCPM_PSY_OFFLINE = 0,
 	TCPM_PSY_FIXED_ONLINE,
-	TCPM_PSY_PROG_ONLINE,
+	TCPM_PSY_PPS_ONLINE,
+	TCPM_PSY_SPR_AVS_ONLINE,
 };
 
 static enum power_supply_property tcpm_psy_props[] = {
@@ -7554,7 +8168,9 @@ static int tcpm_psy_get_online(struct tcpm_port *port,
 {
 	if (port->vbus_charge) {
 		if (port->pps_data.active)
-			val->intval = TCPM_PSY_PROG_ONLINE;
+			val->intval = TCPM_PSY_PPS_ONLINE;
+		else if (port->spr_avs_data.active)
+			val->intval = TCPM_PSY_SPR_AVS_ONLINE;
 		else
 			val->intval = TCPM_PSY_FIXED_ONLINE;
 	} else {
@@ -7569,6 +8185,8 @@ static int tcpm_psy_get_voltage_min(struct tcpm_port *port,
 {
 	if (port->pps_data.active)
 		val->intval = port->pps_data.min_volt * 1000;
+	else if (port->spr_avs_data.active)
+		val->intval = SPR_AVS_TIER1_MIN_VOLT_MV * 1000;
 	else
 		val->intval = port->supply_voltage * 1000;
 
@@ -7580,6 +8198,8 @@ static int tcpm_psy_get_voltage_max(struct tcpm_port *port,
 {
 	if (port->pps_data.active)
 		val->intval = port->pps_data.max_volt * 1000;
+	else if (port->spr_avs_data.active)
+		val->intval = port->spr_avs_data.max_out_volt_mv * 1000;
 	else
 		val->intval = port->supply_voltage * 1000;
 
@@ -7599,6 +8219,8 @@ static int tcpm_psy_get_current_max(struct tcpm_port *port,
 {
 	if (port->pps_data.active)
 		val->intval = port->pps_data.max_curr * 1000;
+	else if (port->spr_avs_data.active)
+		val->intval = port->spr_avs_data.max_current_ma * 1000;
 	else
 		val->intval = port->current_limit * 1000;
 
@@ -7674,17 +8296,41 @@ static int tcpm_psy_get_prop(struct power_supply *psy,
 	return ret;
 }
 
+static int tcpm_disable_pps_avs(struct tcpm_port *port)
+{
+	int ret = 0;
+
+	if (port->pps_data.active)
+		ret = tcpm_pps_activate(port, false);
+	else if (port->spr_avs_data.active)
+		ret = tcpm_spr_avs_activate(port, false);
+
+	return ret;
+}
+
 static int tcpm_psy_set_online(struct tcpm_port *port,
 			       const union power_supply_propval *val)
 {
-	int ret;
+	int ret = 0;
 
 	switch (val->intval) {
 	case TCPM_PSY_FIXED_ONLINE:
-		ret = tcpm_pps_activate(port, false);
+		ret = tcpm_disable_pps_avs(port);
+		break;
+	case TCPM_PSY_PPS_ONLINE:
+		if (port->spr_avs_data.active)
+			ret = tcpm_spr_avs_activate(port, false);
+		if (!ret)
+			ret = tcpm_pps_activate(port, true);
 		break;
-	case TCPM_PSY_PROG_ONLINE:
-		ret = tcpm_pps_activate(port, true);
+	case TCPM_PSY_SPR_AVS_ONLINE:
+		tcpm_log(port, "request to set AVS online");
+		if (port->spr_avs_data.active)
+			return 0;
+		ret = tcpm_disable_pps_avs(port);
+		if (ret)
+			break;
+		ret = tcpm_spr_avs_activate(port, true);
 		break;
 	default:
 		ret = -EINVAL;
@@ -7713,13 +8359,10 @@ static int tcpm_psy_set_prop(struct power_supply *psy,
 		ret = tcpm_psy_set_online(port, val);
 		break;
 	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
-		ret = tcpm_pps_set_out_volt(port, val->intval / 1000);
+		ret = tcpm_aug_set_out_volt(port, val->intval / 1000);
 		break;
 	case POWER_SUPPLY_PROP_CURRENT_NOW:
-		if (val->intval > port->pps_data.max_curr * 1000)
-			ret = -EINVAL;
-		else
-			ret = tcpm_pps_set_op_curr(port, val->intval / 1000);
+		ret = tcpm_aug_set_op_curr(port, val->intval / 1000);
 		break;
 	default:
 		ret = -EINVAL;
@@ -7764,7 +8407,9 @@ static int devm_tcpm_psy_register(struct tcpm_port *port)
 	port->psy_desc.type = POWER_SUPPLY_TYPE_USB;
 	port->psy_desc.usb_types = BIT(POWER_SUPPLY_USB_TYPE_C)  |
 				   BIT(POWER_SUPPLY_USB_TYPE_PD) |
-				   BIT(POWER_SUPPLY_USB_TYPE_PD_PPS);
+				   BIT(POWER_SUPPLY_USB_TYPE_PD_PPS) |
+				   BIT(POWER_SUPPLY_USB_TYPE_PD_PPS_SPR_AVS) |
+				   BIT(POWER_SUPPLY_USB_TYPE_PD_SPR_AVS);
 	port->psy_desc.properties = tcpm_psy_props;
 	port->psy_desc.num_properties = ARRAY_SIZE(tcpm_psy_props);
 	port->psy_desc.get_property = tcpm_psy_get_prop;
@@ -7859,7 +8504,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
 
 	init_completion(&port->tx_complete);
 	init_completion(&port->swap_complete);
-	init_completion(&port->pps_complete);
+	init_completion(&port->aug_supply_req_complete);
 	tcpm_debugfs_init(port);
 
 	err = tcpm_fw_get_caps(port, tcpc->fwnode);
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index e2b26af2b84a8..43faec794b95a 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -820,8 +820,10 @@ static void cd321x_update_work(struct work_struct *work)
 			desc.identity = &st.partner_identity;
 
 		tps->partner = typec_register_partner(tps->port, &desc);
-		if (IS_ERR(tps->partner))
-			dev_warn(tps->dev, "%s: failed to register partnet\n", __func__);
+		if (IS_ERR(tps->partner)) {
+			dev_warn(tps->dev, "%s: failed to register partner\n", __func__);
+			return;
+		}
 
 		if (desc.identity) {
 			typec_partner_set_identity(tps->partner);
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index f38a4d7ebc424..f6bb88b1ccee4 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -43,8 +43,13 @@ void ucsi_notify_common(struct ucsi *ucsi, u32 cci)
 	if (cci & UCSI_CCI_BUSY)
 		return;
 
-	if (UCSI_CCI_CONNECTOR(cci))
-		ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci));
+	if (UCSI_CCI_CONNECTOR(cci)) {
+		if (UCSI_CCI_CONNECTOR(cci) <= ucsi->cap.num_connectors)
+			ucsi_connector_change(ucsi, UCSI_CCI_CONNECTOR(cci));
+		else
+			dev_err(ucsi->dev, "bogus connector number in CCI: %lu\n",
+				UCSI_CCI_CONNECTOR(cci));
+	}
 
 	if (cci & UCSI_CCI_ACK_COMPLETE &&
 	    test_and_clear_bit(ACK_PENDING, &ucsi->flags))
@@ -235,6 +240,8 @@ static int ucsi_send_command_common(struct ucsi *ucsi, u64 cmd,
 	if (cci & UCSI_CCI_ERROR)
 		ret = ucsi_read_error(ucsi, connector_num);
 
+	trace_ucsi_run_command(cmd, ret);
+
 	mutex_unlock(&ucsi->ppm_lock);
 	return ret;
 }
@@ -1182,8 +1189,10 @@ static void ucsi_partner_change(struct ucsi_connector *con)
 		}
 	}
 
-	/* Only notify USB controller if partner supports USB data */
-	if (!(UCSI_CONSTAT(con, PARTNER_FLAG_USB)))
+	if ((!UCSI_CONSTAT(con, PARTNER_FLAG_USB)) &&
+	    ((con->ucsi->quirks & UCSI_USB4_IMPLIES_USB) &&
+	     (!(UCSI_CONSTAT(con, PARTNER_FLAG_USB4_GEN3) ||
+		UCSI_CONSTAT(con, PARTNER_FLAG_USB4_GEN4)))))
 		u_role = USB_ROLE_NONE;
 
 	ret = usb_role_switch_set_role(con->usb_role_sw, u_role);
diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h
index 43a0d01ade8ff..cff9ddc2ae215 100644
--- a/drivers/usb/typec/ucsi/ucsi.h
+++ b/drivers/usb/typec/ucsi/ucsi.h
@@ -497,6 +497,9 @@ struct ucsi {
 	unsigned long quirks;
 #define UCSI_NO_PARTNER_PDOS	BIT(0)	/* Don't read partner's PDOs */
 #define UCSI_DELAY_DEVICE_PDOS	BIT(1)	/* Reading PDOs fails until the parter is in PD mode */
+
+/* USB4 connection can imply that USB communcation is supported */
+#define UCSI_USB4_IMPLIES_USB	BIT(2)
 };
 
 #define UCSI_MAX_DATA_LENGTH(u) (((u)->version < UCSI_VERSION_2_0) ? 0x10 : 0xff)
diff --git a/drivers/usb/typec/ucsi/ucsi_glink.c b/drivers/usb/typec/ucsi/ucsi_glink.c
index 11b3e24e34e2b..12e07b9fe6228 100644
--- a/drivers/usb/typec/ucsi/ucsi_glink.c
+++ b/drivers/usb/typec/ucsi/ucsi_glink.c
@@ -371,14 +371,18 @@ static void pmic_glink_ucsi_destroy(void *data)
 static unsigned long quirk_sc8180x = UCSI_NO_PARTNER_PDOS;
 static unsigned long quirk_sc8280xp = UCSI_NO_PARTNER_PDOS | UCSI_DELAY_DEVICE_PDOS;
 static unsigned long quirk_sm8450 = UCSI_DELAY_DEVICE_PDOS;
+static unsigned long quirk_x1e80100 = UCSI_DELAY_DEVICE_PDOS | UCSI_USB4_IMPLIES_USB;
 
 static const struct of_device_id pmic_glink_ucsi_of_quirks[] = {
+	{ .compatible = "qcom,glymur-pmic-glink", .data = &quirk_sm8450, },
+	{ .compatible = "qcom,kaanapali-pmic-glink", .data = &quirk_sm8450, },
 	{ .compatible = "qcom,qcm6490-pmic-glink", .data = &quirk_sc8280xp, },
 	{ .compatible = "qcom,sc8180x-pmic-glink", .data = &quirk_sc8180x, },
 	{ .compatible = "qcom,sc8280xp-pmic-glink", .data = &quirk_sc8280xp, },
 	{ .compatible = "qcom,sm8350-pmic-glink", .data = &quirk_sc8180x, },
 	{ .compatible = "qcom,sm8450-pmic-glink", .data = &quirk_sm8450, },
 	{ .compatible = "qcom,sm8550-pmic-glink", .data = &quirk_sm8450, },
+	{ .compatible = "qcom,x1e80100-pmic-glink", .data = &quirk_x1e80100, },
 	{}
 };
 
diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
index c5965656babad..ca749fde49bd0 100644
--- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
+++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
@@ -193,7 +193,7 @@ static void gaokun_ucsi_connector_status(struct ucsi_connector *con)
 	gaokun_set_orientation(con, &uec->ports[idx]);
 }
 
-const struct ucsi_operations gaokun_ucsi_ops = {
+static const struct ucsi_operations gaokun_ucsi_ops = {
 	.read_version = gaokun_ucsi_read_version,
 	.read_cci = gaokun_ucsi_read_cci,
 	.poll_cci = gaokun_ucsi_read_cci,
diff --git a/drivers/usb/usbip/stub_dev.c b/drivers/usb/usbip/stub_dev.c
index 34990b7e2d189..abfa11d6bde79 100644
--- a/drivers/usb/usbip/stub_dev.c
+++ b/drivers/usb/usbip/stub_dev.c
@@ -267,7 +267,7 @@ static struct stub_device *stub_device_alloc(struct usb_device *udev)
 	if (!sdev)
 		return NULL;
 
-	sdev->udev = usb_get_dev(udev);
+	sdev->udev = udev;
 
 	/*
 	 * devid is defined with devnum when this driver is first allocated.
@@ -409,7 +409,6 @@ static int stub_probe(struct usb_device *udev)
 	put_busid_priv(busid_priv);
 
 sdev_free:
-	usb_put_dev(udev);
 	stub_device_free(sdev);
 
 	return rc;
@@ -488,8 +487,6 @@ static void stub_disconnect(struct usb_device *udev)
 	/* shutdown the current connection */
 	shutdown_busid(busid_priv);
 
-	usb_put_dev(sdev->udev);
-
 	/* we already have busid_priv, just lock busid_lock */
 	spin_lock(&busid_priv->busid_lock);
 	/* free sdev */
diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c
index bfc10f665e52a..5bc8c47788d45 100644
--- a/drivers/usb/usbip/vhci_sysfs.c
+++ b/drivers/usb/usbip/vhci_sysfs.c
@@ -463,7 +463,7 @@ static void set_status_attr(int id)
 
 	status = status_attrs + id;
 	if (id == 0)
-		strcpy(status->name, "status");
+		strscpy(status->name, "status");
 	else
 		snprintf(status->name, MAX_STATUS_NAME+1, "status.%d", id);
 	status->attr.attr.name = status->name;
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index 478beafc6ac3f..3a803923141be 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -17,19 +17,11 @@ struct vfio_pci_dma_buf {
 	struct phys_vec *phys_vec;
 	struct p2pdma_provider *provider;
 	u32 nr_ranges;
+	struct kref kref;
+	struct completion comp;
 	u8 revoked : 1;
 };
 
-static int vfio_pci_dma_buf_pin(struct dma_buf_attachment *attachment)
-{
-	return -EOPNOTSUPP;
-}
-
-static void vfio_pci_dma_buf_unpin(struct dma_buf_attachment *attachment)
-{
-	/* Do nothing */
-}
-
 static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 				   struct dma_buf_attachment *attachment)
 {
@@ -41,30 +33,52 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 	if (priv->revoked)
 		return -ENODEV;
 
+	if (!dma_buf_attach_revocable(attachment))
+		return -EOPNOTSUPP;
+
 	return 0;
 }
 
+static void vfio_pci_dma_buf_done(struct kref *kref)
+{
+	struct vfio_pci_dma_buf *priv =
+		container_of(kref, struct vfio_pci_dma_buf, kref);
+
+	complete(&priv->comp);
+}
+
 static struct sg_table *
 vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,
 		     enum dma_data_direction dir)
 {
 	struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv;
+	struct sg_table *ret;
 
 	dma_resv_assert_held(priv->dmabuf->resv);
 
 	if (priv->revoked)
 		return ERR_PTR(-ENODEV);
 
-	return dma_buf_phys_vec_to_sgt(attachment, priv->provider,
-				       priv->phys_vec, priv->nr_ranges,
-				       priv->size, dir);
+	ret = dma_buf_phys_vec_to_sgt(attachment, priv->provider,
+				      priv->phys_vec, priv->nr_ranges,
+				      priv->size, dir);
+	if (IS_ERR(ret))
+		return ret;
+
+	kref_get(&priv->kref);
+	return ret;
 }
 
 static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment,
 				   struct sg_table *sgt,
 				   enum dma_data_direction dir)
 {
+	struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv;
+
+	dma_resv_assert_held(priv->dmabuf->resv);
+
 	dma_buf_free_sgt(attachment, sgt, dir);
+	kref_put(&priv->kref, vfio_pci_dma_buf_done);
 }
 
 static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)
@@ -86,8 +100,6 @@ static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)
 }
 
 static const struct dma_buf_ops vfio_pci_dmabuf_ops = {
-	.pin = vfio_pci_dma_buf_pin,
-	.unpin = vfio_pci_dma_buf_unpin,
 	.attach = vfio_pci_dma_buf_attach,
 	.map_dma_buf = vfio_pci_dma_buf_map,
 	.unmap_dma_buf = vfio_pci_dma_buf_unmap,
@@ -286,6 +298,9 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 		goto err_dev_put;
 	}
 
+	kref_init(&priv->kref);
+	init_completion(&priv->comp);
+
 	/* dma_buf_put() now frees priv */
 	INIT_LIST_HEAD(&priv->dmabufs_elm);
 	down_write(&vdev->memory_lock);
@@ -330,9 +345,33 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
 
 		if (priv->revoked != revoked) {
 			dma_resv_lock(priv->dmabuf->resv, NULL);
-			priv->revoked = revoked;
-			dma_buf_move_notify(priv->dmabuf);
+			if (revoked)
+				priv->revoked = true;
+			dma_buf_invalidate_mappings(priv->dmabuf);
+			dma_resv_wait_timeout(priv->dmabuf->resv,
+					      DMA_RESV_USAGE_BOOKKEEP, false,
+					      MAX_SCHEDULE_TIMEOUT);
 			dma_resv_unlock(priv->dmabuf->resv);
+			if (revoked) {
+				kref_put(&priv->kref, vfio_pci_dma_buf_done);
+				wait_for_completion(&priv->comp);
+			} else {
+				/*
+				 * Kref is initialize again, because when revoke
+				 * was performed the reference counter was decreased
+				 * to zero to trigger completion.
+				 */
+				kref_init(&priv->kref);
+				/*
+				 * There is no need to wait as no mapping was
+				 * performed when the previous status was
+				 * priv->revoked == true.
+				 */
+				reinit_completion(&priv->comp);
+				dma_resv_lock(priv->dmabuf->resv, NULL);
+				priv->revoked = false;
+				dma_resv_unlock(priv->dmabuf->resv);
+			}
 		}
 		fput(priv->dmabuf->file);
 	}
@@ -352,8 +391,13 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
 		list_del_init(&priv->dmabufs_elm);
 		priv->vdev = NULL;
 		priv->revoked = true;
-		dma_buf_move_notify(priv->dmabuf);
+		dma_buf_invalidate_mappings(priv->dmabuf);
+		dma_resv_wait_timeout(priv->dmabuf->resv,
+				      DMA_RESV_USAGE_BOOKKEEP, false,
+				      MAX_SCHEDULE_TIMEOUT);
 		dma_resv_unlock(priv->dmabuf->resv);
+		kref_put(&priv->kref, vfio_pci_dma_buf_done);
+		wait_for_completion(&priv->comp);
 		vfio_device_put_registration(&vdev->vdev);
 		fput(priv->dmabuf->file);
 	}
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 1c22880e72267..9a1253b9d8c50 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -2950,6 +2950,8 @@ static const struct target_core_fabric_ops vhost_scsi_ops = {
 	.tfc_tpg_base_attrs		= vhost_scsi_tpg_attrs,
 	.tfc_tpg_attrib_attrs		= vhost_scsi_tpg_attrib_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
+	.direct_compl_supp		= 1,
 	.default_submit_type		= TARGET_QUEUE_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index 054f7a718f50f..1d8ec6bed53e4 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -91,6 +91,18 @@ static struct vhost_vsock *vhost_vsock_get(u32 guest_cid, struct net *net)
 	return NULL;
 }
 
+static bool vhost_transport_has_remote_cid(struct vsock_sock *vsk, u32 cid)
+{
+	struct sock *sk = sk_vsock(vsk);
+	struct net *net = sock_net(sk);
+	bool found;
+
+	rcu_read_lock();
+	found = !!vhost_vsock_get(cid, net);
+	rcu_read_unlock();
+	return found;
+}
+
 static void
 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
 			    struct vhost_virtqueue *vq)
@@ -424,6 +436,7 @@ static struct virtio_transport vhost_transport = {
 		.module                   = THIS_MODULE,
 
 		.get_local_cid            = vhost_transport_get_local_cid,
+		.has_remote_cid           = vhost_transport_has_remote_cid,
 
 		.init                     = virtio_transport_do_socket_init,
 		.destruct                 = virtio_transport_destruct,
diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig
index 9884f003247d8..a7144d275f54e 100644
--- a/drivers/video/Kconfig
+++ b/drivers/video/Kconfig
@@ -37,6 +37,7 @@ source "drivers/char/agp/Kconfig"
 
 source "drivers/gpu/vga/Kconfig"
 
+source "drivers/gpu/Kconfig"
 source "drivers/gpu/host1x/Kconfig"
 source "drivers/gpu/ipu-v3/Kconfig"
 source "drivers/gpu/nova-core/Kconfig"
diff --git a/drivers/video/backlight/sky81452-backlight.c b/drivers/video/backlight/sky81452-backlight.c
index 2749231f03854..b2679b24de14b 100644
--- a/drivers/video/backlight/sky81452-backlight.c
+++ b/drivers/video/backlight/sky81452-backlight.c
@@ -202,6 +202,9 @@ static struct sky81452_bl_platform_data *sky81452_bl_parse_dt(
 	pdata->dpwm_mode = of_property_read_bool(np, "skyworks,dpwm-mode");
 	pdata->phase_shift = of_property_read_bool(np, "skyworks,phase-shift");
 	pdata->gpiod_enable = devm_gpiod_get_optional(dev, NULL, GPIOD_OUT_HIGH);
+	if (IS_ERR(pdata->gpiod_enable))
+		return dev_err_cast_probe(dev, pdata->gpiod_enable,
+					  "failed to get gpio\n");
 
 	ret = of_property_count_u32_elems(np, "led-sources");
 	if (ret < 0) {
diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c
index 337e04236d6d3..e88ff3a93b779 100644
--- a/drivers/video/console/newport_con.c
+++ b/drivers/video/console/newport_con.c
@@ -33,9 +33,9 @@
 
 #define NEWPORT_LEN	0x10000
 
-#define FONT_DATA ((unsigned char *)font_vga_8x16.data)
+#define FONT_DATA font_vga_8x16.data
 
-static unsigned char *font_data[MAX_NR_CONSOLES];
+static font_data_t *font_data[MAX_NR_CONSOLES];
 
 static struct newport_regs *npregs;
 static unsigned long newport_addr;
@@ -370,9 +370,9 @@ static void newport_clear(struct vc_data *vc, unsigned int sy, unsigned int sx,
 static void newport_putc(struct vc_data *vc, u16 charattr, unsigned int ypos,
 			 unsigned int xpos)
 {
-	unsigned char *p;
+	const unsigned char *p;
 
-	p = &font_data[vc->vc_num][(charattr & 0xff) << 4];
+	p = &font_data_buf(font_data[vc->vc_num])[(charattr & 0xff) << 4];
 	charattr = (charattr >> 8) & 0xff;
 	xpos <<= 3;
 	ypos <<= 4;
@@ -400,7 +400,7 @@ static void newport_putcs(struct vc_data *vc, const u16 *s,
 			  unsigned int count, unsigned int ypos,
 			  unsigned int xpos)
 {
-	unsigned char *p;
+	const unsigned char *p;
 	unsigned int i;
 	u16 charattr;
 
@@ -424,7 +424,7 @@ static void newport_putcs(struct vc_data *vc, const u16 *s,
 				 NPORT_DMODE0_L32);
 
 	for (i = 0; i < count; i++, xpos += 8) {
-		p = &font_data[vc->vc_num][(scr_readw(s++) & 0xff) << 4];
+		p = &font_data_buf(font_data[vc->vc_num])[(scr_readw(s++) & 0xff) << 4];
 
 		newport_wait(npregs);
 
@@ -501,52 +501,32 @@ static int newport_set_font(int unit, const struct console_font *op,
 {
 	int w = op->width;
 	int h = op->height;
-	int size = h * op->charcount;
 	int i;
-	unsigned char *new_data, *data = op->data, *p;
+	font_data_t *new_data;
 
 	/* ladis: when I grow up, there will be a day... and more sizes will
 	 * be supported ;-) */
-	if ((w != 8) || (h != 16) || (vpitch != 32)
-	    || (op->charcount != 256 && op->charcount != 512))
+	if (w != 8 || h != 16 || (op->charcount != 256 && op->charcount != 512))
 		return -EINVAL;
 
-	if (!(new_data = kmalloc(FONT_EXTRA_WORDS * sizeof(int) + size,
-	     GFP_USER))) return -ENOMEM;
-
-	new_data += FONT_EXTRA_WORDS * sizeof(int);
-	FNTSIZE(new_data) = size;
-	FNTCHARCNT(new_data) = op->charcount;
-	REFCOUNT(new_data) = 0;	/* usage counter */
-	FNTSUM(new_data) = 0;
-
-	p = new_data;
-	for (i = 0; i < op->charcount; i++) {
-		memcpy(p, data, h);
-		data += 32;
-		p += h;
-	}
+	new_data = font_data_import(op, vpitch, NULL);
+	if (IS_ERR(new_data))
+		return PTR_ERR(new_data);
 
 	/* check if font is already used by other console */
 	for (i = 0; i < MAX_NR_CONSOLES; i++) {
-		if (font_data[i] != FONT_DATA
-		    && FNTSIZE(font_data[i]) == size
-		    && !memcmp(font_data[i], new_data, size)) {
-			kfree(new_data - FONT_EXTRA_WORDS * sizeof(int));
+		if (font_data_is_equal(font_data[i], new_data)) {
+			font_data_put(new_data);
 			/* current font is the same as the new one */
 			if (i == unit)
 				return 0;
 			new_data = font_data[i];
+			font_data_get(new_data);
 			break;
 		}
 	}
-	/* old font is user font */
-	if (font_data[unit] != FONT_DATA) {
-		if (--REFCOUNT(font_data[unit]) == 0)
-			kfree(font_data[unit] -
-			      FONT_EXTRA_WORDS * sizeof(int));
-	}
-	REFCOUNT(new_data)++;
+
+	font_data_put(font_data[unit]);
 	font_data[unit] = new_data;
 
 	return 0;
@@ -554,12 +534,9 @@ static int newport_set_font(int unit, const struct console_font *op,
 
 static int newport_set_def_font(int unit, struct console_font *op)
 {
-	if (font_data[unit] != FONT_DATA) {
-		if (--REFCOUNT(font_data[unit]) == 0)
-			kfree(font_data[unit] -
-			      FONT_EXTRA_WORDS * sizeof(int));
-		font_data[unit] = FONT_DATA;
-	}
+	font_data_put(font_data[unit]);
+	font_data[unit] = FONT_DATA;
+	font_data_get(font_data[unit]);
 
 	return 0;
 }
diff --git a/drivers/video/fbdev/core/bitblit.c b/drivers/video/fbdev/core/bitblit.c
index 085ffb44c51a9..7478accea8ecf 100644
--- a/drivers/video/fbdev/core/bitblit.c
+++ b/drivers/video/fbdev/core/bitblit.c
@@ -22,8 +22,7 @@
 /*
  * Accelerated handlers.
  */
-static void update_attr(u8 *dst, u8 *src, int attribute,
-			       struct vc_data *vc)
+static void update_attr(u8 *dst, const u8 *src, int attribute, struct vc_data *vc)
 {
 	int i, offset = (vc->vc_font.height < 10) ? 1 : 2;
 	int width = DIV_ROUND_UP(vc->vc_font.width, 8);
@@ -81,7 +80,7 @@ static inline void bit_putcs_aligned(struct vc_data *vc, struct fb_info *info,
 	u16 charmask = vc->vc_hi_font_mask ? 0x1ff : 0xff;
 	unsigned int charcnt = vc->vc_font.charcount;
 	u32 idx = vc->vc_font.width >> 3;
-	u8 *src;
+	const u8 *src;
 
 	while (cnt--) {
 		u16 ch = scr_readw(s++) & charmask;
@@ -120,7 +119,7 @@ static inline void bit_putcs_unaligned(struct vc_data *vc,
 	u32 shift_low = 0, mod = vc->vc_font.width % 8;
 	u32 shift_high = 8;
 	u32 idx = vc->vc_font.width >> 3;
-	u8 *src;
+	const u8 *src;
 
 	while (cnt--) {
 		u16 ch = scr_readw(s++) & charmask;
@@ -267,7 +266,7 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
 	int y = real_y(par->p, vc->state.y);
 	int attribute, use_sw = vc->vc_cursor_type & CUR_SW;
 	int err = 1;
-	char *src;
+	const u8 *src;
 
 	cursor.set = 0;
 
@@ -278,7 +277,7 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable,
 	attribute = get_attribute(info, c);
 	src = vc->vc_font.data + ((c & charmask) * (w * vc->vc_font.height));
 
-	if (par->cursor_state.image.data != src ||
+	if (par->cursor_state.image.data != (const char *)src ||
 	    par->cursor_reset) {
 		par->cursor_state.image.data = src;
 		cursor.set |= FB_CUR_SETIMAGE;
diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c
index ca48b89a323d3..a12dd25ab6979 100644
--- a/drivers/video/fbdev/core/fb_defio.c
+++ b/drivers/video/fbdev/core/fb_defio.c
@@ -14,6 +14,7 @@
 #include <linux/export.h>
 #include <linux/string.h>
 #include <linux/mm.h>
+#include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
@@ -24,6 +25,104 @@
 #include <linux/rmap.h>
 #include <linux/pagemap.h>
 
+struct address_space;
+
+/*
+ * struct fb_deferred_io_state
+ */
+
+struct fb_deferred_io_state {
+	struct kref ref;
+
+	int open_count; /* number of opened files; protected by fb_info lock */
+	struct address_space *mapping; /* page cache object for fb device */
+
+	struct mutex lock; /* mutex that protects the pageref list */
+	/* fields protected by lock */
+	struct fb_info *info;
+	struct list_head pagereflist; /* list of pagerefs for touched pages */
+	unsigned long npagerefs;
+	struct fb_deferred_io_pageref *pagerefs;
+};
+
+static struct fb_deferred_io_state *fb_deferred_io_state_alloc(unsigned long len)
+{
+	struct fb_deferred_io_state *fbdefio_state;
+	struct fb_deferred_io_pageref *pagerefs;
+	unsigned long npagerefs;
+
+	fbdefio_state = kzalloc_obj(*fbdefio_state);
+	if (!fbdefio_state)
+		return NULL;
+
+	npagerefs = DIV_ROUND_UP(len, PAGE_SIZE);
+
+	/* alloc a page ref for each page of the display memory */
+	pagerefs = kvzalloc_objs(*pagerefs, npagerefs);
+	if (!pagerefs)
+		goto err_kfree;
+	fbdefio_state->npagerefs = npagerefs;
+	fbdefio_state->pagerefs = pagerefs;
+
+	kref_init(&fbdefio_state->ref);
+	mutex_init(&fbdefio_state->lock);
+
+	INIT_LIST_HEAD(&fbdefio_state->pagereflist);
+
+	return fbdefio_state;
+
+err_kfree:
+	kfree(fbdefio_state);
+	return NULL;
+}
+
+static void fb_deferred_io_state_release(struct fb_deferred_io_state *fbdefio_state)
+{
+	WARN_ON(!list_empty(&fbdefio_state->pagereflist));
+	mutex_destroy(&fbdefio_state->lock);
+	kvfree(fbdefio_state->pagerefs);
+
+	kfree(fbdefio_state);
+}
+
+static void fb_deferred_io_state_get(struct fb_deferred_io_state *fbdefio_state)
+{
+	kref_get(&fbdefio_state->ref);
+}
+
+static void __fb_deferred_io_state_release(struct kref *ref)
+{
+	struct fb_deferred_io_state *fbdefio_state =
+		container_of(ref, struct fb_deferred_io_state, ref);
+
+	fb_deferred_io_state_release(fbdefio_state);
+}
+
+static void fb_deferred_io_state_put(struct fb_deferred_io_state *fbdefio_state)
+{
+	kref_put(&fbdefio_state->ref, __fb_deferred_io_state_release);
+}
+
+/*
+ * struct vm_operations_struct
+ */
+
+static void fb_deferred_io_vm_open(struct vm_area_struct *vma)
+{
+	struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data;
+
+	WARN_ON_ONCE(!try_module_get(THIS_MODULE));
+	fb_deferred_io_state_get(fbdefio_state);
+}
+
+static void fb_deferred_io_vm_close(struct vm_area_struct *vma)
+{
+	struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data;
+
+	fb_deferred_io_state_put(fbdefio_state);
+	module_put(THIS_MODULE);
+}
+
 static struct page *fb_deferred_io_get_page(struct fb_info *info, unsigned long offs)
 {
 	struct fb_deferred_io *fbdefio = info->fbdefio;
@@ -44,18 +143,19 @@ static struct page *fb_deferred_io_get_page(struct fb_info *info, unsigned long
 	return page;
 }
 
-static struct fb_deferred_io_pageref *fb_deferred_io_pageref_lookup(struct fb_info *info,
-								    unsigned long offset,
-								    struct page *page)
+static struct fb_deferred_io_pageref *
+fb_deferred_io_pageref_lookup(struct fb_deferred_io_state *fbdefio_state, unsigned long offset,
+			      struct page *page)
 {
+	struct fb_info *info = fbdefio_state->info;
 	unsigned long pgoff = offset >> PAGE_SHIFT;
 	struct fb_deferred_io_pageref *pageref;
 
-	if (fb_WARN_ON_ONCE(info, pgoff >= info->npagerefs))
+	if (fb_WARN_ON_ONCE(info, pgoff >= fbdefio_state->npagerefs))
 		return NULL; /* incorrect allocation size */
 
 	/* 1:1 mapping between pageref and page offset */
-	pageref = &info->pagerefs[pgoff];
+	pageref = &fbdefio_state->pagerefs[pgoff];
 
 	if (pageref->page)
 		goto out;
@@ -75,10 +175,11 @@ static struct fb_deferred_io_pageref *fb_deferred_io_pageref_get(struct fb_info
 								 struct page *page)
 {
 	struct fb_deferred_io *fbdefio = info->fbdefio;
-	struct list_head *pos = &fbdefio->pagereflist;
+	struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state;
+	struct list_head *pos = &fbdefio_state->pagereflist;
 	struct fb_deferred_io_pageref *pageref, *cur;
 
-	pageref = fb_deferred_io_pageref_lookup(info, offset, page);
+	pageref = fb_deferred_io_pageref_lookup(fbdefio_state, offset, page);
 	if (!pageref)
 		return NULL;
 
@@ -99,7 +200,7 @@ static struct fb_deferred_io_pageref *fb_deferred_io_pageref_get(struct fb_info
 		 * pages. If possible, drivers should try to work with
 		 * unsorted page lists instead.
 		 */
-		list_for_each_entry(cur, &fbdefio->pagereflist, list) {
+		list_for_each_entry(cur, &fbdefio_state->pagereflist, list) {
 			if (cur->offset > pageref->offset)
 				break;
 		}
@@ -121,25 +222,46 @@ static void fb_deferred_io_pageref_put(struct fb_deferred_io_pageref *pageref,
 /* this is to find and return the vmalloc-ed fb pages */
 static vm_fault_t fb_deferred_io_fault(struct vm_fault *vmf)
 {
+	struct fb_info *info;
 	unsigned long offset;
 	struct page *page;
-	struct fb_info *info = vmf->vma->vm_private_data;
+	vm_fault_t ret;
+	struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data;
+
+	mutex_lock(&fbdefio_state->lock);
+
+	info = fbdefio_state->info;
+	if (!info) {
+		ret = VM_FAULT_SIGBUS; /* our device is gone */
+		goto err_mutex_unlock;
+	}
 
 	offset = vmf->pgoff << PAGE_SHIFT;
-	if (offset >= info->fix.smem_len)
-		return VM_FAULT_SIGBUS;
+	if (offset >= info->fix.smem_len) {
+		ret = VM_FAULT_SIGBUS;
+		goto err_mutex_unlock;
+	}
 
 	page = fb_deferred_io_get_page(info, offset);
-	if (!page)
-		return VM_FAULT_SIGBUS;
+	if (!page) {
+		ret = VM_FAULT_SIGBUS;
+		goto err_mutex_unlock;
+	}
 
 	if (!vmf->vma->vm_file)
 		fb_err(info, "no mapping available\n");
 
-	BUG_ON(!info->fbdefio->mapping);
+	fb_WARN_ON_ONCE(info, !fbdefio_state->mapping);
+
+	mutex_unlock(&fbdefio_state->lock);
 
 	vmf->page = page;
+
 	return 0;
+
+err_mutex_unlock:
+	mutex_unlock(&fbdefio_state->lock);
+	return ret;
 }
 
 int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasync)
@@ -166,15 +288,24 @@ EXPORT_SYMBOL_GPL(fb_deferred_io_fsync);
  * Adds a page to the dirty list. Call this from struct
  * vm_operations_struct.page_mkwrite.
  */
-static vm_fault_t fb_deferred_io_track_page(struct fb_info *info, unsigned long offset,
-					    struct page *page)
+static vm_fault_t fb_deferred_io_track_page(struct fb_deferred_io_state *fbdefio_state,
+					    unsigned long offset, struct page *page)
 {
-	struct fb_deferred_io *fbdefio = info->fbdefio;
+	struct fb_info *info;
+	struct fb_deferred_io *fbdefio;
 	struct fb_deferred_io_pageref *pageref;
 	vm_fault_t ret;
 
 	/* protect against the workqueue changing the page list */
-	mutex_lock(&fbdefio->lock);
+	mutex_lock(&fbdefio_state->lock);
+
+	info = fbdefio_state->info;
+	if (!info) {
+		ret = VM_FAULT_SIGBUS; /* our device is gone */
+		goto err_mutex_unlock;
+	}
+
+	fbdefio = info->fbdefio;
 
 	pageref = fb_deferred_io_pageref_get(info, offset, page);
 	if (WARN_ON_ONCE(!pageref)) {
@@ -192,50 +323,38 @@ static vm_fault_t fb_deferred_io_track_page(struct fb_info *info, unsigned long
 	 */
 	lock_page(pageref->page);
 
-	mutex_unlock(&fbdefio->lock);
+	mutex_unlock(&fbdefio_state->lock);
 
 	/* come back after delay to process the deferred IO */
 	schedule_delayed_work(&info->deferred_work, fbdefio->delay);
 	return VM_FAULT_LOCKED;
 
 err_mutex_unlock:
-	mutex_unlock(&fbdefio->lock);
+	mutex_unlock(&fbdefio_state->lock);
 	return ret;
 }
 
-/*
- * fb_deferred_io_page_mkwrite - Mark a page as written for deferred I/O
- * @fb_info: The fbdev info structure
- * @vmf: The VM fault
- *
- * This is a callback we get when userspace first tries to
- * write to the page. We schedule a workqueue. That workqueue
- * will eventually mkclean the touched pages and execute the
- * deferred framebuffer IO. Then if userspace touches a page
- * again, we repeat the same scheme.
- *
- * Returns:
- * VM_FAULT_LOCKED on success, or a VM_FAULT error otherwise.
- */
-static vm_fault_t fb_deferred_io_page_mkwrite(struct fb_info *info, struct vm_fault *vmf)
+static vm_fault_t fb_deferred_io_page_mkwrite(struct fb_deferred_io_state *fbdefio_state,
+					      struct vm_fault *vmf)
 {
 	unsigned long offset = vmf->pgoff << PAGE_SHIFT;
 	struct page *page = vmf->page;
 
 	file_update_time(vmf->vma->vm_file);
 
-	return fb_deferred_io_track_page(info, offset, page);
+	return fb_deferred_io_track_page(fbdefio_state, offset, page);
 }
 
-/* vm_ops->page_mkwrite handler */
 static vm_fault_t fb_deferred_io_mkwrite(struct vm_fault *vmf)
 {
-	struct fb_info *info = vmf->vma->vm_private_data;
+	struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data;
 
-	return fb_deferred_io_page_mkwrite(info, vmf);
+	return fb_deferred_io_page_mkwrite(fbdefio_state, vmf);
 }
 
 static const struct vm_operations_struct fb_deferred_io_vm_ops = {
+	.open		= fb_deferred_io_vm_open,
+	.close		= fb_deferred_io_vm_close,
 	.fault		= fb_deferred_io_fault,
 	.page_mkwrite	= fb_deferred_io_mkwrite,
 };
@@ -248,11 +367,17 @@ int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 
+	if (!try_module_get(THIS_MODULE))
+		return -EINVAL;
+
 	vma->vm_ops = &fb_deferred_io_vm_ops;
 	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
 	if (!(info->flags & FBINFO_VIRTFB))
 		vm_flags_set(vma, VM_IO);
-	vma->vm_private_data = info;
+	vma->vm_private_data = info->fbdefio_state;
+
+	fb_deferred_io_state_get(info->fbdefio_state); /* released in vma->vm_ops->close() */
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(fb_deferred_io_mmap);
@@ -263,63 +388,52 @@ static void fb_deferred_io_work(struct work_struct *work)
 	struct fb_info *info = container_of(work, struct fb_info, deferred_work.work);
 	struct fb_deferred_io_pageref *pageref, *next;
 	struct fb_deferred_io *fbdefio = info->fbdefio;
+	struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state;
 
 	/* here we wrprotect the page's mappings, then do all deferred IO. */
-	mutex_lock(&fbdefio->lock);
+	mutex_lock(&fbdefio_state->lock);
 #ifdef CONFIG_MMU
-	list_for_each_entry(pageref, &fbdefio->pagereflist, list) {
+	list_for_each_entry(pageref, &fbdefio_state->pagereflist, list) {
 		struct page *page = pageref->page;
 		pgoff_t pgoff = pageref->offset >> PAGE_SHIFT;
 
-		mapping_wrprotect_range(fbdefio->mapping, pgoff,
+		mapping_wrprotect_range(fbdefio_state->mapping, pgoff,
 					page_to_pfn(page), 1);
 	}
 #endif
 
 	/* driver's callback with pagereflist */
-	fbdefio->deferred_io(info, &fbdefio->pagereflist);
+	fbdefio->deferred_io(info, &fbdefio_state->pagereflist);
 
 	/* clear the list */
-	list_for_each_entry_safe(pageref, next, &fbdefio->pagereflist, list)
+	list_for_each_entry_safe(pageref, next, &fbdefio_state->pagereflist, list)
 		fb_deferred_io_pageref_put(pageref, info);
 
-	mutex_unlock(&fbdefio->lock);
+	mutex_unlock(&fbdefio_state->lock);
 }
 
 int fb_deferred_io_init(struct fb_info *info)
 {
 	struct fb_deferred_io *fbdefio = info->fbdefio;
-	struct fb_deferred_io_pageref *pagerefs;
-	unsigned long npagerefs;
-	int ret;
+	struct fb_deferred_io_state *fbdefio_state;
 
 	BUG_ON(!fbdefio);
 
 	if (WARN_ON(!info->fix.smem_len))
 		return -EINVAL;
 
-	mutex_init(&fbdefio->lock);
+	fbdefio_state = fb_deferred_io_state_alloc(info->fix.smem_len);
+	if (!fbdefio_state)
+		return -ENOMEM;
+	fbdefio_state->info = info;
+
 	INIT_DELAYED_WORK(&info->deferred_work, fb_deferred_io_work);
-	INIT_LIST_HEAD(&fbdefio->pagereflist);
 	if (fbdefio->delay == 0) /* set a default of 1 s */
 		fbdefio->delay = HZ;
 
-	npagerefs = DIV_ROUND_UP(info->fix.smem_len, PAGE_SIZE);
-
-	/* alloc a page ref for each page of the display memory */
-	pagerefs = kvzalloc_objs(*pagerefs, npagerefs);
-	if (!pagerefs) {
-		ret = -ENOMEM;
-		goto err;
-	}
-	info->npagerefs = npagerefs;
-	info->pagerefs = pagerefs;
+	info->fbdefio_state = fbdefio_state;
 
 	return 0;
-
-err:
-	mutex_destroy(&fbdefio->lock);
-	return ret;
 }
 EXPORT_SYMBOL_GPL(fb_deferred_io_init);
 
@@ -327,11 +441,11 @@ void fb_deferred_io_open(struct fb_info *info,
 			 struct inode *inode,
 			 struct file *file)
 {
-	struct fb_deferred_io *fbdefio = info->fbdefio;
+	struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state;
 
-	fbdefio->mapping = file->f_mapping;
+	fbdefio_state->mapping = file->f_mapping;
 	file->f_mapping->a_ops = &fb_deferred_io_aops;
-	fbdefio->open_count++;
+	fbdefio_state->open_count++;
 }
 EXPORT_SYMBOL_GPL(fb_deferred_io_open);
 
@@ -342,21 +456,25 @@ static void fb_deferred_io_lastclose(struct fb_info *info)
 
 void fb_deferred_io_release(struct fb_info *info)
 {
-	struct fb_deferred_io *fbdefio = info->fbdefio;
+	struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state;
 
-	if (!--fbdefio->open_count)
+	if (!--fbdefio_state->open_count)
 		fb_deferred_io_lastclose(info);
 }
 EXPORT_SYMBOL_GPL(fb_deferred_io_release);
 
 void fb_deferred_io_cleanup(struct fb_info *info)
 {
-	struct fb_deferred_io *fbdefio = info->fbdefio;
+	struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state;
 
 	fb_deferred_io_lastclose(info);
 
-	kvfree(info->pagerefs);
-	mutex_destroy(&fbdefio->lock);
-	fbdefio->mapping = NULL;
+	info->fbdefio_state = NULL;
+
+	mutex_lock(&fbdefio_state->lock);
+	fbdefio_state->info = NULL;
+	mutex_unlock(&fbdefio_state->lock);
+
+	fb_deferred_io_state_put(fbdefio_state);
 }
 EXPORT_SYMBOL_GPL(fb_deferred_io_cleanup);
diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
index 666261ae59d82..8641b0b3edc4c 100644
--- a/drivers/video/fbdev/core/fbcon.c
+++ b/drivers/video/fbdev/core/fbcon.c
@@ -1019,8 +1019,11 @@ static const char *fbcon_startup(void)
 						info->pixmap.blit_y);
 		vc->vc_font.width = font->width;
 		vc->vc_font.height = font->height;
-		vc->vc_font.data = (void *)(p->fontdata = font->data);
+		vc->vc_font.data = font_data_buf(font->data);
 		vc->vc_font.charcount = font->charcount;
+
+		p->fontdata = font->data;
+		font_data_get(p->fontdata);
 	}
 
 	cols = FBCON_SWAP(par->rotate, info->var.xres, info->var.yres);
@@ -1078,15 +1081,13 @@ static void fbcon_init(struct vc_data *vc, bool init)
 		if (t->fontdata) {
 			struct vc_data *fvc = vc_cons[fg_console].d;
 
-			vc->vc_font.data = (void *)(p->fontdata =
-						    fvc->vc_font.data);
+			vc->vc_font.data = fvc->vc_font.data;
 			vc->vc_font.width = fvc->vc_font.width;
 			vc->vc_font.height = fvc->vc_font.height;
 			vc->vc_font.charcount = fvc->vc_font.charcount;
-			p->userfont = t->userfont;
 
-			if (p->userfont)
-				REFCOUNT(p->fontdata)++;
+			p->fontdata = t->fontdata;
+			font_data_get(p->fontdata);
 		} else {
 			const struct font_desc *font = NULL;
 
@@ -1097,8 +1098,11 @@ static void fbcon_init(struct vc_data *vc, bool init)
 							info->pixmap.blit_y);
 			vc->vc_font.width = font->width;
 			vc->vc_font.height = font->height;
-			vc->vc_font.data = (void *)(p->fontdata = font->data);
+			vc->vc_font.data = font_data_buf(font->data);
 			vc->vc_font.charcount = font->charcount;
+
+			p->fontdata = font->data;
+			font_data_get(p->fontdata);
 		}
 	}
 
@@ -1189,10 +1193,10 @@ static void fbcon_init(struct vc_data *vc, bool init)
 
 static void fbcon_free_font(struct fbcon_display *p)
 {
-	if (p->userfont && p->fontdata && (--REFCOUNT(p->fontdata) == 0))
-		kfree(p->fontdata - FONT_EXTRA_WORDS * sizeof(int));
-	p->fontdata = NULL;
-	p->userfont = 0;
+	if (p->fontdata) {
+		font_data_put(p->fontdata);
+		p->fontdata = NULL;
+	}
 }
 
 static void set_vc_hi_font(struct vc_data *vc, bool set);
@@ -1409,14 +1413,13 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
 	svc = *default_mode;
 	t = &fb_display[svc->vc_num];
 
-	if (!vc->vc_font.data) {
-		vc->vc_font.data = (void *)(p->fontdata = t->fontdata);
+	if (!p->fontdata) {
+		vc->vc_font.data = font_data_buf(t->fontdata);
 		vc->vc_font.width = (*default_mode)->vc_font.width;
 		vc->vc_font.height = (*default_mode)->vc_font.height;
 		vc->vc_font.charcount = (*default_mode)->vc_font.charcount;
-		p->userfont = t->userfont;
-		if (p->userfont)
-			REFCOUNT(p->fontdata)++;
+		p->fontdata = t->fontdata;
+		font_data_get(p->fontdata);
 	}
 
 	var->activate = FB_ACTIVATE_NOW;
@@ -2036,9 +2039,6 @@ static void updatescrollmode(struct fbcon_display *p,
 	updatescrollmode_accel(p, info, vc);
 }
 
-#define PITCH(w) (((w) + 7) >> 3)
-#define CALC_FONTSZ(h, p, c) ((h) * (p) * (c)) /* size = height * pitch * charcount */
-
 static int fbcon_resize(struct vc_data *vc, unsigned int width,
 			unsigned int height, bool from_user)
 {
@@ -2048,9 +2048,8 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
 	struct fb_var_screeninfo var = info->var;
 	int x_diff, y_diff, virt_w, virt_h, virt_fw, virt_fh;
 
-	if (p->userfont && FNTSIZE(vc->vc_font.data)) {
-		int size;
-		int pitch = PITCH(vc->vc_font.width);
+	if (font_data_size(p->fontdata)) {
+		unsigned int size = vc_font_size(&vc->vc_font);
 
 		/*
 		 * If user font, ensure that a possible change to user font
@@ -2059,10 +2058,7 @@ static int fbcon_resize(struct vc_data *vc, unsigned int width,
 		 * charcount can change and cannot be used to determine the
 		 * font data allocated size.
 		 */
-		if (pitch <= 0)
-			return -EINVAL;
-		size = CALC_FONTSZ(vc->vc_font.height, pitch, vc->vc_font.charcount);
-		if (size > FNTSIZE(vc->vc_font.data))
+		if (!size || size > font_data_size(p->fontdata))
 			return -EINVAL;
 	}
 
@@ -2286,67 +2282,15 @@ static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank,
 
 static int fbcon_get_font(struct vc_data *vc, struct console_font *font, unsigned int vpitch)
 {
-	u8 *fontdata = vc->vc_font.data;
-	u8 *data = font->data;
-	int i, j;
+	const struct fbcon_display *p = &fb_display[vc->vc_num];
 
 	font->width = vc->vc_font.width;
 	font->height = vc->vc_font.height;
 	if (font->height > vpitch)
 		return -ENOSPC;
 	font->charcount = vc->vc_hi_font_mask ? 512 : 256;
-	if (!font->data)
-		return 0;
-
-	if (font->width <= 8) {
-		j = vc->vc_font.height;
-		if (font->charcount * j > FNTSIZE(fontdata))
-			return -EINVAL;
-
-		for (i = 0; i < font->charcount; i++) {
-			memcpy(data, fontdata, j);
-			memset(data + j, 0, vpitch - j);
-			data += vpitch;
-			fontdata += j;
-		}
-	} else if (font->width <= 16) {
-		j = vc->vc_font.height * 2;
-		if (font->charcount * j > FNTSIZE(fontdata))
-			return -EINVAL;
-
-		for (i = 0; i < font->charcount; i++) {
-			memcpy(data, fontdata, j);
-			memset(data + j, 0, 2*vpitch - j);
-			data += 2*vpitch;
-			fontdata += j;
-		}
-	} else if (font->width <= 24) {
-		if (font->charcount * (vc->vc_font.height * sizeof(u32)) > FNTSIZE(fontdata))
-			return -EINVAL;
 
-		for (i = 0; i < font->charcount; i++) {
-			for (j = 0; j < vc->vc_font.height; j++) {
-				*data++ = fontdata[0];
-				*data++ = fontdata[1];
-				*data++ = fontdata[2];
-				fontdata += sizeof(u32);
-			}
-			memset(data, 0, 3 * (vpitch - j));
-			data += 3 * (vpitch - j);
-		}
-	} else {
-		j = vc->vc_font.height * 4;
-		if (font->charcount * j > FNTSIZE(fontdata))
-			return -EINVAL;
-
-		for (i = 0; i < font->charcount; i++) {
-			memcpy(data, fontdata, j);
-			memset(data + j, 0, 4 * vpitch - j);
-			data += 4 * vpitch;
-			fontdata += j;
-		}
-	}
-	return 0;
+	return font_data_export(p->fontdata, font, vpitch);
 }
 
 /* set/clear vc_hi_font_mask and update vc attrs accordingly */
@@ -2411,20 +2355,20 @@ static void set_vc_hi_font(struct vc_data *vc, bool set)
 }
 
 static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount,
-			     const u8 * data, int userfont)
+			     font_data_t *data)
 {
 	struct fb_info *info = fbcon_info_from_console(vc->vc_num);
 	struct fbcon_par *par = info->fbcon_par;
 	struct fbcon_display *p = &fb_display[vc->vc_num];
-	int resize, ret, old_userfont, old_width, old_height, old_charcount;
-	u8 *old_data = vc->vc_font.data;
+	int resize, ret, old_width, old_height, old_charcount;
+	font_data_t *old_fontdata = p->fontdata;
+	const u8 *old_data = vc->vc_font.data;
 
-	resize = (w != vc->vc_font.width) || (h != vc->vc_font.height);
-	vc->vc_font.data = (void *)(p->fontdata = data);
-	old_userfont = p->userfont;
-	if ((p->userfont = userfont))
-		REFCOUNT(data)++;
+	font_data_get(data);
 
+	resize = (w != vc->vc_font.width) || (h != vc->vc_font.height);
+	p->fontdata = data;
+	vc->vc_font.data = font_data_buf(p->fontdata);
 	old_width = vc->vc_font.width;
 	old_height = vc->vc_font.height;
 	old_charcount = vc->vc_font.charcount;
@@ -2453,24 +2397,20 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount,
 		update_screen(vc);
 	}
 
-	if (old_userfont && (--REFCOUNT(old_data) == 0))
-		kfree(old_data - FONT_EXTRA_WORDS * sizeof(int));
+	if (old_fontdata)
+		font_data_put(old_fontdata);
+
 	return 0;
 
 err_out:
-	p->fontdata = old_data;
+	p->fontdata = old_fontdata;
 	vc->vc_font.data = old_data;
-
-	if (userfont) {
-		p->userfont = old_userfont;
-		if (--REFCOUNT(data) == 0)
-			kfree(data - FONT_EXTRA_WORDS * sizeof(int));
-	}
-
 	vc->vc_font.width = old_width;
 	vc->vc_font.height = old_height;
 	vc->vc_font.charcount = old_charcount;
 
+	font_data_put(data);
+
 	return ret;
 }
 
@@ -2486,10 +2426,8 @@ static int fbcon_set_font(struct vc_data *vc, const struct console_font *font,
 	unsigned charcount = font->charcount;
 	int w = font->width;
 	int h = font->height;
-	int size, alloc_size;
-	int i, csum;
-	u8 *new_data, *data = font->data;
-	int pitch = PITCH(font->width);
+	int i, ret;
+	font_data_t *new_data;
 
 	/* Is there a reason why fbconsole couldn't handle any charcount >256?
 	 * If not this check should be changed to charcount < 256 */
@@ -2513,50 +2451,24 @@ static int fbcon_set_font(struct vc_data *vc, const struct console_font *font,
 	if (fbcon_invalid_charcount(info, charcount))
 		return -EINVAL;
 
-	/* Check for integer overflow in font size calculation */
-	if (check_mul_overflow(h, pitch, &size) ||
-	    check_mul_overflow(size, charcount, &size))
-		return -EINVAL;
+	new_data = font_data_import(font, vpitch, crc32);
+	if (IS_ERR(new_data))
+		return PTR_ERR(new_data);
 
-	/* Check for overflow in allocation size calculation */
-	if (check_add_overflow(FONT_EXTRA_WORDS * sizeof(int), size, &alloc_size))
-		return -EINVAL;
-
-	new_data = kmalloc(alloc_size, GFP_USER);
-
-	if (!new_data)
-		return -ENOMEM;
-
-	memset(new_data, 0, FONT_EXTRA_WORDS * sizeof(int));
-
-	new_data += FONT_EXTRA_WORDS * sizeof(int);
-	FNTSIZE(new_data) = size;
-	REFCOUNT(new_data) = 0;	/* usage counter */
-	for (i=0; i< charcount; i++) {
-		memcpy(new_data + i*h*pitch, data +  i*vpitch*pitch, h*pitch);
-	}
-
-	/* Since linux has a nice crc32 function use it for counting font
-	 * checksums. */
-	csum = crc32(0, new_data, size);
-
-	FNTSUM(new_data) = csum;
 	/* Check if the same font is on some other console already */
 	for (i = first_fb_vc; i <= last_fb_vc; i++) {
-		struct vc_data *tmp = vc_cons[i].d;
-
-		if (fb_display[i].userfont &&
-		    fb_display[i].fontdata &&
-		    FNTSUM(fb_display[i].fontdata) == csum &&
-		    FNTSIZE(fb_display[i].fontdata) == size &&
-		    tmp->vc_font.width == w &&
-		    !memcmp(fb_display[i].fontdata, new_data, size)) {
-			kfree(new_data - FONT_EXTRA_WORDS * sizeof(int));
-			new_data = (u8 *)fb_display[i].fontdata;
+		if (fb_display[i].fontdata &&
+		    font_data_is_equal(fb_display[i].fontdata, new_data)) {
+			font_data_get(fb_display[i].fontdata);
+			font_data_put(new_data);
+			new_data = fb_display[i].fontdata;
 			break;
 		}
 	}
-	return fbcon_do_set_font(vc, font->width, font->height, charcount, new_data, 1);
+	ret = fbcon_do_set_font(vc, font->width, font->height, charcount, new_data);
+	font_data_put(new_data);
+
+	return ret;
 }
 
 static int fbcon_set_def_font(struct vc_data *vc, struct console_font *font,
@@ -2573,7 +2485,7 @@ static int fbcon_set_def_font(struct vc_data *vc, struct console_font *font,
 
 	font->width = f->width;
 	font->height = f->height;
-	return fbcon_do_set_font(vc, f->width, f->height, f->charcount, f->data, 0);
+	return fbcon_do_set_font(vc, f->width, f->height, f->charcount, f->data);
 }
 
 static u16 palette_red[16];
diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h
index fca14e9b729b9..1e3c1ef84762e 100644
--- a/drivers/video/fbdev/core/fbcon.h
+++ b/drivers/video/fbdev/core/fbcon.h
@@ -11,6 +11,7 @@
 #ifndef _VIDEO_FBCON_H
 #define _VIDEO_FBCON_H
 
+#include <linux/font.h>
 #include <linux/types.h>
 #include <linux/vt_buffer.h>
 #include <linux/vt_kern.h>
@@ -25,8 +26,7 @@
 
 struct fbcon_display {
     /* Filled in by the low-level console driver */
-    const u_char *fontdata;
-    int userfont;                   /* != 0 if fontdata kmalloc()ed */
+    font_data_t *fontdata;
 #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION
     u_short scrollmode;             /* Scroll Method, use fb_scrollmode() */
 #endif
@@ -82,8 +82,8 @@ struct fbcon_par {
 	int    rotate;
 	int    cur_rotate;
 	char  *cursor_data;
-	u8    *fontbuffer;
-	u8    *fontdata;
+	u8          *fontbuffer;
+	const u8    *fontdata;
 	u8    *cursor_src;
 	u32    cursor_size;
 	u32    fd_size;
diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index cf199038f069e..30f2b59c47bf9 100644
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -91,14 +91,14 @@ EXPORT_SYMBOL(fb_get_color_depth);
 /*
  * Data padding functions.
  */
-void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 s_pitch, u32 height)
+void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 s_pitch, u32 height)
 {
 	__fb_pad_aligned_buffer(dst, d_pitch, src, s_pitch, height);
 }
 EXPORT_SYMBOL(fb_pad_aligned_buffer);
 
-void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 idx, u32 height,
-				u32 shift_high, u32 shift_low, u32 mod)
+void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 idx, u32 height,
+			     u32 shift_high, u32 shift_low, u32 mod)
 {
 	u8 mask = (u8) (0xff << shift_high), tmp;
 	int i, j;
diff --git a/drivers/video/fbdev/goldfishfb.c b/drivers/video/fbdev/goldfishfb.c
index ffe33a36b944d..c9871281bc1da 100644
--- a/drivers/video/fbdev/goldfishfb.c
+++ b/drivers/video/fbdev/goldfishfb.c
@@ -174,7 +174,6 @@ static const struct fb_ops goldfish_fb_ops = {
 static int goldfish_fb_probe(struct platform_device *pdev)
 {
 	int ret;
-	struct resource *r;
 	struct goldfish_fb *fb;
 	size_t framesize;
 	u32 width, height;
@@ -189,14 +188,9 @@ static int goldfish_fb_probe(struct platform_device *pdev)
 	init_waitqueue_head(&fb->wait);
 	platform_set_drvdata(pdev, fb);
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (r == NULL) {
-		ret = -ENODEV;
-		goto err_no_io_base;
-	}
-	fb->reg_base = ioremap(r->start, PAGE_SIZE);
-	if (fb->reg_base == NULL) {
-		ret = -ENOMEM;
+	fb->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(fb->reg_base)) {
+		ret = PTR_ERR(fb->reg_base);
 		goto err_no_io_base;
 	}
 
@@ -273,7 +267,6 @@ static int goldfish_fb_probe(struct platform_device *pdev)
 				fb->fb.fix.smem_start);
 err_alloc_screen_base_failed:
 err_no_irq:
-	iounmap(fb->reg_base);
 err_no_io_base:
 	kfree(fb);
 err_fb_alloc_failed:
@@ -291,7 +284,6 @@ static void goldfish_fb_remove(struct platform_device *pdev)
 
 	dma_free_coherent(&pdev->dev, framesize, (void *)fb->fb.screen_base,
 						fb->fb.fix.smem_start);
-	iounmap(fb->reg_base);
 	kfree(fb);
 }
 
diff --git a/drivers/video/fbdev/macfb.c b/drivers/video/fbdev/macfb.c
index 887fffdccd245..ef3d2304e2f44 100644
--- a/drivers/video/fbdev/macfb.c
+++ b/drivers/video/fbdev/macfb.c
@@ -668,19 +668,19 @@ static int __init macfb_init(void)
 
 		switch(ndev->dr_hw) {
 		case NUBUS_DRHW_APPLE_MDC:
-			strcpy(macfb_fix.id, "Mac Disp. Card");
+			strscpy(macfb_fix.id, "Mac Disp. Card");
 			macfb_setpalette = mdc_setpalette;
 			break;
 		case NUBUS_DRHW_APPLE_TFB:
-			strcpy(macfb_fix.id, "Toby");
+			strscpy(macfb_fix.id, "Toby");
 			macfb_setpalette = toby_setpalette;
 			break;
 		case NUBUS_DRHW_APPLE_JET:
-			strcpy(macfb_fix.id, "Jet");
+			strscpy(macfb_fix.id, "Jet");
 			macfb_setpalette = jet_setpalette;
 			break;
 		default:
-			strcpy(macfb_fix.id, "Generic NuBus");
+			strscpy(macfb_fix.id, "Generic NuBus");
 			break;
 		}
 	}
@@ -707,7 +707,7 @@ static int __init macfb_init(void)
 		case MAC_MODEL_Q700:
 		case MAC_MODEL_Q900:
 		case MAC_MODEL_Q950:
-			strcpy(macfb_fix.id, "DAFB");
+			strscpy(macfb_fix.id, "DAFB");
 			macfb_setpalette = dafb_setpalette;
 			dafb_cmap_regs = ioremap(DAFB_BASE, 0x1000);
 			break;
@@ -716,7 +716,7 @@ static int __init macfb_init(void)
 		 * LC II uses the V8 framebuffer
 		 */
 		case MAC_MODEL_LCII:
-			strcpy(macfb_fix.id, "V8");
+			strscpy(macfb_fix.id, "V8");
 			macfb_setpalette = v8_brazil_setpalette;
 			v8_brazil_cmap_regs = ioremap(DAC_BASE, 0x1000);
 			break;
@@ -729,7 +729,7 @@ static int __init macfb_init(void)
 		case MAC_MODEL_IIVI:
 		case MAC_MODEL_IIVX:
 		case MAC_MODEL_P600:
-			strcpy(macfb_fix.id, "Brazil");
+			strscpy(macfb_fix.id, "Brazil");
 			macfb_setpalette = v8_brazil_setpalette;
 			v8_brazil_cmap_regs = ioremap(DAC_BASE, 0x1000);
 			break;
@@ -745,7 +745,7 @@ static int __init macfb_init(void)
 		case MAC_MODEL_P520:
 		case MAC_MODEL_P550:
 		case MAC_MODEL_P460:
-			strcpy(macfb_fix.id, "Sonora");
+			strscpy(macfb_fix.id, "Sonora");
 			macfb_setpalette = v8_brazil_setpalette;
 			v8_brazil_cmap_regs = ioremap(DAC_BASE, 0x1000);
 			break;
@@ -757,7 +757,7 @@ static int __init macfb_init(void)
 		 */
 		case MAC_MODEL_IICI:
 		case MAC_MODEL_IISI:
-			strcpy(macfb_fix.id, "RBV");
+			strscpy(macfb_fix.id, "RBV");
 			macfb_setpalette = rbv_setpalette;
 			rbv_cmap_regs = ioremap(DAC_BASE, 0x1000);
 			break;
@@ -767,7 +767,7 @@ static int __init macfb_init(void)
 		 */
 		case MAC_MODEL_Q840:
 		case MAC_MODEL_C660:
-			strcpy(macfb_fix.id, "Civic");
+			strscpy(macfb_fix.id, "Civic");
 			macfb_setpalette = civic_setpalette;
 			civic_cmap_regs = ioremap(CIVIC_BASE, 0x1000);
 			break;
@@ -778,7 +778,7 @@ static int __init macfb_init(void)
 		 * We think this may be like the LC II
 		 */
 		case MAC_MODEL_LC:
-			strcpy(macfb_fix.id, "LC");
+			strscpy(macfb_fix.id, "LC");
 			if (vidtest) {
 				macfb_setpalette = v8_brazil_setpalette;
 				v8_brazil_cmap_regs =
@@ -790,7 +790,7 @@ static int __init macfb_init(void)
 		 * We think this may be like the LC II
 		 */
 		case MAC_MODEL_CCL:
-			strcpy(macfb_fix.id, "Color Classic");
+			strscpy(macfb_fix.id, "Color Classic");
 			if (vidtest) {
 				macfb_setpalette = v8_brazil_setpalette;
 				v8_brazil_cmap_regs =
@@ -802,7 +802,7 @@ static int __init macfb_init(void)
 		 * And we *do* mean "weirdos"
 		 */
 		case MAC_MODEL_TV:
-			strcpy(macfb_fix.id, "Mac TV");
+			strscpy(macfb_fix.id, "Mac TV");
 			break;
 
 		/*
@@ -810,7 +810,7 @@ static int __init macfb_init(void)
 		 */
 		case MAC_MODEL_SE30:
 		case MAC_MODEL_CLII:
-			strcpy(macfb_fix.id, "Monochrome");
+			strscpy(macfb_fix.id, "Monochrome");
 			break;
 
 		/*
@@ -828,7 +828,7 @@ static int __init macfb_init(void)
 		case MAC_MODEL_PB140:
 		case MAC_MODEL_PB145:
 		case MAC_MODEL_PB170:
-			strcpy(macfb_fix.id, "DDC");
+			strscpy(macfb_fix.id, "DDC");
 			break;
 
 		/*
@@ -840,7 +840,7 @@ static int __init macfb_init(void)
 		case MAC_MODEL_PB180:
 		case MAC_MODEL_PB210:
 		case MAC_MODEL_PB230:
-			strcpy(macfb_fix.id, "GSC");
+			strscpy(macfb_fix.id, "GSC");
 			break;
 
 		/*
@@ -848,7 +848,7 @@ static int __init macfb_init(void)
 		 */
 		case MAC_MODEL_PB165C:
 		case MAC_MODEL_PB180C:
-			strcpy(macfb_fix.id, "TIM");
+			strscpy(macfb_fix.id, "TIM");
 			break;
 
 		/*
@@ -860,13 +860,13 @@ static int __init macfb_init(void)
 		case MAC_MODEL_PB270C:
 		case MAC_MODEL_PB280:
 		case MAC_MODEL_PB280C:
-			strcpy(macfb_fix.id, "CSC");
+			strscpy(macfb_fix.id, "CSC");
 			macfb_setpalette = csc_setpalette;
 			csc_cmap_regs = ioremap(CSC_BASE, 0x1000);
 			break;
 
 		default:
-			strcpy(macfb_fix.id, "Unknown");
+			strscpy(macfb_fix.id, "Unknown");
 			break;
 		}
 
diff --git a/drivers/video/fbdev/omap/hwa742.c b/drivers/video/fbdev/omap/hwa742.c
index 64e76e1f5388a..68a677f824e95 100644
--- a/drivers/video/fbdev/omap/hwa742.c
+++ b/drivers/video/fbdev/omap/hwa742.c
@@ -950,6 +950,8 @@ static int hwa742_init(struct omapfb_device *fbdev, int ext_mode,
 	omapfb_conf = dev_get_platdata(fbdev->dev);
 
 	hwa742.sys_ck = clk_get(NULL, "hwa_sys_ck");
+	if (IS_ERR(hwa742.sys_ck))
+		return PTR_ERR(hwa742.sys_ck);
 
 	spin_lock_init(&hwa742.req_lock);
 
@@ -1028,6 +1030,7 @@ static int hwa742_init(struct omapfb_device *fbdev, int ext_mode,
 err2:
 	hwa742.int_ctrl->cleanup();
 err1:
+	clk_put(hwa742.sys_ck);
 	return r;
 }
 
@@ -1037,6 +1040,7 @@ static void hwa742_cleanup(void)
 	hwa742.extif->cleanup();
 	hwa742.int_ctrl->cleanup();
 	clk_disable_unprepare(hwa742.sys_ck);
+	clk_put(hwa742.sys_ck);
 }
 
 struct lcd_ctrl hwa742_ctrl = {
diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c
index 51ebe78359ec3..7606e024fa3f4 100644
--- a/drivers/video/fbdev/tdfxfb.c
+++ b/drivers/video/fbdev/tdfxfb.c
@@ -77,6 +77,7 @@
 #include <asm/io.h>
 
 #include <video/tdfx.h>
+#include <video/vga.h>
 
 #define DPRINTK(a, b...) pr_debug("fb: %s: " a, __func__ , ## b)
 
@@ -591,7 +592,7 @@ static int tdfxfb_set_par(struct fb_info *info)
 		vt = ve + (info->var.upper_margin << 1) - 1;
 		reg.screensize = info->var.xres | (info->var.yres << 13);
 		reg.vidcfg |= VIDCFG_HALF_MODE;
-		reg.crt[0x09] = 0x80;
+		reg.crt[VGA_CRTC_MAX_SCAN] = 0x80;
 	} else {
 		vd = info->var.yres - 1;
 		vs  = vd + info->var.lower_margin;
@@ -609,59 +610,59 @@ static int tdfxfb_set_par(struct fb_info *info)
 			 info->var.xres < 480 ? 0x60 :
 			 info->var.xres < 768 ? 0xe0 : 0x20);
 
-	reg.gra[0x05] = 0x40;
-	reg.gra[0x06] = 0x05;
-	reg.gra[0x07] = 0x0f;
-	reg.gra[0x08] = 0xff;
-
-	reg.att[0x00] = 0x00;
-	reg.att[0x01] = 0x01;
-	reg.att[0x02] = 0x02;
-	reg.att[0x03] = 0x03;
-	reg.att[0x04] = 0x04;
-	reg.att[0x05] = 0x05;
-	reg.att[0x06] = 0x06;
-	reg.att[0x07] = 0x07;
-	reg.att[0x08] = 0x08;
-	reg.att[0x09] = 0x09;
-	reg.att[0x0a] = 0x0a;
-	reg.att[0x0b] = 0x0b;
-	reg.att[0x0c] = 0x0c;
-	reg.att[0x0d] = 0x0d;
-	reg.att[0x0e] = 0x0e;
-	reg.att[0x0f] = 0x0f;
-	reg.att[0x10] = 0x41;
-	reg.att[0x12] = 0x0f;
-
-	reg.seq[0x00] = 0x03;
-	reg.seq[0x01] = 0x01; /* fixme: clkdiv2? */
-	reg.seq[0x02] = 0x0f;
-	reg.seq[0x03] = 0x00;
-	reg.seq[0x04] = 0x0e;
-
-	reg.crt[0x00] = ht - 4;
-	reg.crt[0x01] = hd;
-	reg.crt[0x02] = hbs;
-	reg.crt[0x03] = 0x80 | (hbe & 0x1f);
-	reg.crt[0x04] = hs;
-	reg.crt[0x05] = ((hbe & 0x20) << 2) | (he & 0x1f);
-	reg.crt[0x06] = vt;
-	reg.crt[0x07] = ((vs & 0x200) >> 2) |
-			((vd & 0x200) >> 3) |
-			((vt & 0x200) >> 4) | 0x10 |
-			((vbs & 0x100) >> 5) |
-			((vs & 0x100) >> 6) |
-			((vd & 0x100) >> 7) |
-			((vt & 0x100) >> 8);
-	reg.crt[0x09] |= 0x40 | ((vbs & 0x200) >> 4);
-	reg.crt[0x10] = vs;
-	reg.crt[0x11] = (ve & 0x0f) | 0x20;
-	reg.crt[0x12] = vd;
-	reg.crt[0x13] = wd;
-	reg.crt[0x15] = vbs;
-	reg.crt[0x16] = vbe + 1;
-	reg.crt[0x17] = 0xc3;
-	reg.crt[0x18] = 0xff;
+	reg.gra[VGA_GFX_MODE]         = 0x40;
+	reg.gra[VGA_GFX_MISC]         = 0x05;
+	reg.gra[VGA_GFX_COMPARE_MASK] = 0x0f;
+	reg.gra[VGA_GFX_BIT_MASK]     = 0xff;
+
+	reg.att[VGA_ATC_PALETTE0]     = 0x00;
+	reg.att[VGA_ATC_PALETTE1]     = 0x01;
+	reg.att[VGA_ATC_PALETTE2]     = 0x02;
+	reg.att[VGA_ATC_PALETTE3]     = 0x03;
+	reg.att[VGA_ATC_PALETTE4]     = 0x04;
+	reg.att[VGA_ATC_PALETTE5]     = 0x05;
+	reg.att[VGA_ATC_PALETTE6]     = 0x06;
+	reg.att[VGA_ATC_PALETTE7]     = 0x07;
+	reg.att[VGA_ATC_PALETTE8]     = 0x08;
+	reg.att[VGA_ATC_PALETTE9]     = 0x09;
+	reg.att[VGA_ATC_PALETTEA]     = 0x0a;
+	reg.att[VGA_ATC_PALETTEB]     = 0x0b;
+	reg.att[VGA_ATC_PALETTEC]     = 0x0c;
+	reg.att[VGA_ATC_PALETTED]     = 0x0d;
+	reg.att[VGA_ATC_PALETTEE]     = 0x0e;
+	reg.att[VGA_ATC_PALETTEF]     = 0x0f;
+	reg.att[VGA_ATC_MODE]         = 0x41;
+	reg.att[VGA_ATC_PLANE_ENABLE] = 0x0f;
+
+	reg.seq[VGA_SEQ_RESET]         = 0x03;
+	reg.seq[VGA_SEQ_CLOCK_MODE]    = 0x01; /* fixme: clkdiv2? */
+	reg.seq[VGA_SEQ_PLANE_WRITE]   = 0x0f;
+	reg.seq[VGA_SEQ_CHARACTER_MAP] = 0x00;
+	reg.seq[VGA_SEQ_MEMORY_MODE]   = 0x0e;
+
+	reg.crt[VGA_CRTC_H_TOTAL]       = ht - 4;
+	reg.crt[VGA_CRTC_H_DISP]        = hd;
+	reg.crt[VGA_CRTC_H_BLANK_START] = hbs;
+	reg.crt[VGA_CRTC_H_BLANK_END]   = 0x80 | (hbe & 0x1f);
+	reg.crt[VGA_CRTC_H_SYNC_START]  = hs;
+	reg.crt[VGA_CRTC_H_SYNC_END]    = ((hbe & 0x20) << 2) | (he & 0x1f);
+	reg.crt[VGA_CRTC_V_TOTAL]       = vt;
+	reg.crt[VGA_CRTC_OVERFLOW]      = ((vs & 0x200) >> 2) |
+					  ((vd & 0x200) >> 3) |
+					  ((vt & 0x200) >> 4) | 0x10 |
+					  ((vbs & 0x100) >> 5) |
+					  ((vs & 0x100) >> 6) |
+					  ((vd & 0x100) >> 7) |
+					  ((vt & 0x100) >> 8);
+	reg.crt[VGA_CRTC_MAX_SCAN]     |= 0x40 | ((vbs & 0x200) >> 4);
+	reg.crt[VGA_CRTC_V_SYNC_START]  = vs;
+	reg.crt[VGA_CRTC_V_SYNC_END]    = (ve & 0x0f) | 0x20;
+	reg.crt[VGA_CRTC_V_DISP_END]    = vd;
+	reg.crt[VGA_CRTC_OFFSET]        = wd;
+	reg.crt[VGA_CRTC_V_BLANK_START] = vbs;
+	reg.crt[VGA_CRTC_V_BLANK_END]   = vbe + 1;
+	reg.crt[VGA_CRTC_MODE]          = 0xc3;
+	reg.crt[VGA_CRTC_LINE_COMPARE]  = 0xff;
 
 	/* Banshee's nonvga stuff */
 	reg.ext[0x00] = (((ht & 0x100) >> 8) |
diff --git a/drivers/video/fbdev/via/lcd.c b/drivers/video/fbdev/via/lcd.c
index 8673fced87492..3fa2304fbda7e 100644
--- a/drivers/video/fbdev/via/lcd.c
+++ b/drivers/video/fbdev/via/lcd.c
@@ -954,6 +954,9 @@ bool viafb_lcd_get_mobile_state(bool *mobile)
 	u16 start_pattern;
 
 	biosptr = ioremap(romaddr, 0x10000);
+	if (!biosptr)
+		return false;
+
 	start_pattern = readw(biosptr);
 
 	/* Compare pattern */
diff --git a/drivers/video/fbdev/wmt_ge_rops.c b/drivers/video/fbdev/wmt_ge_rops.c
index 2bd26bfb2b46b..0cf78bcadfa6c 100644
--- a/drivers/video/fbdev/wmt_ge_rops.c
+++ b/drivers/video/fbdev/wmt_ge_rops.c
@@ -148,25 +148,15 @@ EXPORT_SYMBOL_GPL(wmt_ge_sync);
 
 static int wmt_ge_rops_probe(struct platform_device *pdev)
 {
-	struct resource *res;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "no I/O memory resource defined\n");
-		return -ENODEV;
-	}
-
 	/* Only one ROP engine is presently supported. */
 	if (unlikely(regbase)) {
 		WARN_ON(1);
 		return -EBUSY;
 	}
 
-	regbase = ioremap(res->start, resource_size(res));
-	if (regbase == NULL) {
-		dev_err(&pdev->dev, "failed to map I/O memory\n");
-		return -EBUSY;
-	}
+	regbase = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regbase))
+		return PTR_ERR(regbase);
 
 	writel(1, regbase + GE_ENABLE_OFF);
 	printk(KERN_INFO "Enabled support for WMT GE raster acceleration\n");
@@ -176,7 +166,7 @@ static int wmt_ge_rops_probe(struct platform_device *pdev)
 
 static void wmt_ge_rops_remove(struct platform_device *pdev)
 {
-	iounmap(regbase);
+	regbase = NULL;
 }
 
 static const struct of_device_id wmt_dt_ids[] = {
diff --git a/drivers/virt/coco/guest/Kconfig b/drivers/virt/coco/guest/Kconfig
index 3d5e1d05bf343..da570dc4bd480 100644
--- a/drivers/virt/coco/guest/Kconfig
+++ b/drivers/virt/coco/guest/Kconfig
@@ -13,5 +13,4 @@ config TSM_REPORTS
 config TSM_MEASUREMENTS
 	select TSM_GUEST
 	select CRYPTO_HASH_INFO
-	select CRYPTO
 	bool
diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c
index 4252b147593ad..23ef3991c4d5a 100644
--- a/drivers/virt/coco/tdx-guest/tdx-guest.c
+++ b/drivers/virt/coco/tdx-guest/tdx-guest.c
@@ -306,6 +306,11 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data)
 		return ret;
 	}
 
+	if (quote_buf->status != GET_QUOTE_SUCCESS) {
+		pr_debug("GetQuote request failed, status:%llx\n", quote_buf->status);
+		return -EIO;
+	}
+
 	buf = kvmemdup(quote_buf->data, quote_buf->out_len, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c
index 66135eac3abff..2cf0096aa217f 100644
--- a/drivers/virt/vmgenid.c
+++ b/drivers/virt/vmgenid.c
@@ -75,7 +75,8 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state)
 	phys_addr = (obj->package.elements[0].integer.value << 0) |
 		    (obj->package.elements[1].integer.value << 32);
 
-	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB);
+	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE,
+				  MEMREMAP_WB | MEMREMAP_DEC);
 	if (IS_ERR(virt_addr)) {
 		ret = PTR_ERR(virt_addr);
 		goto out;
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index d1fbc8fe84701..f6c2dff33f8af 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -369,13 +369,13 @@ static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb)
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_ALLOC_STALL, stall);
 
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_SCAN,
-		    pages_to_bytes(events[PGSCAN_KSWAPD]));
+		    pages_to_bytes(global_node_page_state(PGSCAN_KSWAPD)));
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_SCAN,
-		    pages_to_bytes(events[PGSCAN_DIRECT]));
+		    pages_to_bytes(global_node_page_state(PGSCAN_DIRECT)));
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_RECLAIM,
-		    pages_to_bytes(events[PGSTEAL_KSWAPD]));
+		    pages_to_bytes(global_node_page_state(PGSTEAL_KSWAPD)));
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_RECLAIM,
-		    pages_to_bytes(events[PGSTEAL_DIRECT]));
+		    pages_to_bytes(global_node_page_state(PGSTEAL_DIRECT)));
 
 #ifdef CONFIG_HUGETLB_PAGE
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
@@ -1022,6 +1022,8 @@ static int virtballoon_probe(struct virtio_device *vdev)
 			goto out_unregister_oom;
 		}
 
+		vb->pr_dev_info.order = PAGE_REPORTING_ORDER_UNSPECIFIED;
+
 		/*
 		 * The default page reporting order is @pageblock_order, which
 		 * corresponds to 512MB in size on ARM64 when 64KB base page
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 335692d41617a..fbca7ce1c6bf0 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -2912,10 +2912,10 @@ EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
  * @data: the token identifying the buffer.
  * @gfp: how to do memory allocations (if necessary).
  *
- * Same as virtqueue_add_inbuf but passes DMA_ATTR_CPU_CACHE_CLEAN to indicate
- * that the CPU will not dirty any cacheline overlapping this buffer while it
- * is available, and to suppress overlapping cacheline warnings in DMA debug
- * builds.
+ * Same as virtqueue_add_inbuf but passes DMA_ATTR_DEBUGGING_IGNORE_CACHELINES
+ * to indicate that the CPU will not dirty any cacheline overlapping this buffer
+ * while it is available, and to suppress overlapping cacheline warnings in DMA
+ * debug builds.
  *
  * Caller must ensure we don't call this with other virtqueue operations
  * at the same time (except where noted).
@@ -2928,7 +2928,7 @@ int virtqueue_add_inbuf_cache_clean(struct virtqueue *vq,
 				    gfp_t gfp)
 {
 	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, false, gfp,
-			     DMA_ATTR_CPU_CACHE_CLEAN);
+			     DMA_ATTR_DEBUGGING_IGNORE_CACHELINES);
 }
 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_cache_clean);
 
diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c
index 35e75d07ef5d6..aa1f57f743979 100644
--- a/drivers/w1/masters/ds2490.c
+++ b/drivers/w1/masters/ds2490.c
@@ -1022,11 +1022,8 @@ static int ds_probe(struct usb_interface *intf,
 	if (!dev)
 		return -ENOMEM;
 
-	dev->udev = usb_get_dev(udev);
-	if (!dev->udev) {
-		err = -ENOMEM;
-		goto err_out_free;
-	}
+	dev->udev = udev;
+
 	memset(dev->ep, 0, sizeof(dev->ep));
 
 	usb_set_intfdata(intf, dev);
@@ -1085,9 +1082,8 @@ static int ds_probe(struct usb_interface *intf,
 
 err_out_clear:
 	usb_set_intfdata(intf, NULL);
-	usb_put_dev(dev->udev);
-err_out_free:
 	kfree(dev);
+
 	return err;
 }
 
@@ -1107,7 +1103,6 @@ static void ds_disconnect(struct usb_interface *intf)
 
 	usb_set_intfdata(intf, NULL);
 
-	usb_put_dev(dev->udev);
 	kfree(dev);
 }
 
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index ccf25027bec19..b79917e785a59 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -262,10 +262,12 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
 
 done:
 	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
-		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr))))
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr)))) {
 			arch_sync_dma_for_device(phys, size, dir);
-		else
+			arch_sync_dma_flush();
+		} else {
 			xen_dma_sync_for_device(dev, dev_addr, size, dir);
+		}
 	}
 	return dev_addr;
 }
@@ -287,10 +289,12 @@ static void xen_swiotlb_unmap_phys(struct device *hwdev, dma_addr_t dev_addr,
 	BUG_ON(dir == DMA_NONE);
 
 	if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
-		if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr))))
+		if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr)))) {
 			arch_sync_dma_for_cpu(paddr, size, dir);
-		else
+			arch_sync_dma_flush();
+		} else {
 			xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir);
+		}
 	}
 
 	/* NOTE: We use dev_addr here, not paddr! */
@@ -308,10 +312,12 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
 	struct io_tlb_pool *pool;
 
 	if (!dev_is_dma_coherent(dev)) {
-		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr)))) {
 			arch_sync_dma_for_cpu(paddr, size, dir);
-		else
+			arch_sync_dma_flush();
+		} else {
 			xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
+		}
 	}
 
 	pool = xen_swiotlb_find_pool(dev, dma_addr);
@@ -331,10 +337,12 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
 		__swiotlb_sync_single_for_device(dev, paddr, size, dir, pool);
 
 	if (!dev_is_dma_coherent(dev)) {
-		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr)))) {
 			arch_sync_dma_for_device(paddr, size, dir);
-		else
+			arch_sync_dma_flush();
+		} else {
 			xen_dma_sync_for_device(dev, dma_addr, size, dir);
+		}
 	}
 }
 
diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c
index 3035c7d0f1b77..e33f95c91b096 100644
--- a/drivers/xen/xen-scsiback.c
+++ b/drivers/xen/xen-scsiback.c
@@ -1832,6 +1832,7 @@ static const struct target_core_fabric_ops scsiback_ops = {
 	.tfc_tpg_base_attrs		= scsiback_tpg_attrs,
 	.tfc_tpg_param_attrs		= scsiback_param_attrs,
 
+	.default_compl_type		= TARGET_QUEUE_COMPL,
 	.default_submit_type		= TARGET_DIRECT_SUBMIT,
 	.direct_submit_supp		= 1,
 };
diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c
index 862164181baca..c21d33830f5f0 100644
--- a/fs/9p/vfs_addr.c
+++ b/fs/9p/vfs_addr.c
@@ -36,7 +36,7 @@ static void v9fs_begin_writeback(struct netfs_io_request *wreq)
 
 	fid = v9fs_fid_find_inode(wreq->inode, true, INVALID_UID, true);
 	if (!fid) {
-		WARN_ONCE(1, "folio expected an open fid inode->i_ino=%lx\n",
+		WARN_ONCE(1, "folio expected an open fid inode->i_ino=%llx\n",
 			  wreq->inode->i_ino);
 		return;
 	}
@@ -133,7 +133,7 @@ static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file)
 	return 0;
 
 no_fid:
-	WARN_ONCE(1, "folio expected an open fid inode->i_ino=%lx\n",
+	WARN_ONCE(1, "folio expected an open fid inode->i_ino=%llx\n",
 		  rreq->inode->i_ino);
 	return -EINVAL;
 }
diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
index 97abe65bf7c1f..d1508b1fe1092 100644
--- a/fs/9p/vfs_inode.c
+++ b/fs/9p/vfs_inode.c
@@ -1245,7 +1245,7 @@ static int
 v9fs_vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 		 struct dentry *dentry, const char *symname)
 {
-	p9_debug(P9_DEBUG_VFS, " %lu,%pd,%s\n",
+	p9_debug(P9_DEBUG_VFS, " %llu,%pd,%s\n",
 		 dir->i_ino, dentry, symname);
 
 	return v9fs_vfs_mkspecial(dir, dentry, P9_DMSYMLINK, symname);
@@ -1269,7 +1269,7 @@ v9fs_vfs_link(struct dentry *old_dentry, struct inode *dir,
 	char name[1 + U32_MAX_DIGITS + 2]; /* sign + number + \n + \0 */
 	struct p9_fid *oldfid;
 
-	p9_debug(P9_DEBUG_VFS, " %lu,%pd,%pd\n",
+	p9_debug(P9_DEBUG_VFS, " %llu,%pd,%pd\n",
 		 dir->i_ino, dentry, old_dentry);
 
 	oldfid = v9fs_fid_clone(old_dentry);
@@ -1305,7 +1305,7 @@ v9fs_vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	char name[2 + U32_MAX_DIGITS + 1 + U32_MAX_DIGITS + 1];
 	u32 perm;
 
-	p9_debug(P9_DEBUG_VFS, " %lu,%pd mode: %x MAJOR: %u MINOR: %u\n",
+	p9_debug(P9_DEBUG_VFS, " %llu,%pd mode: %x MAJOR: %u MINOR: %u\n",
 		 dir->i_ino, dentry, mode,
 		 MAJOR(rdev), MINOR(rdev));
 
diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
index 643e759eacb2a..71796a89bcf47 100644
--- a/fs/9p/vfs_inode_dotl.c
+++ b/fs/9p/vfs_inode_dotl.c
@@ -691,7 +691,7 @@ v9fs_vfs_symlink_dotl(struct mnt_idmap *idmap, struct inode *dir,
 	struct p9_fid *fid = NULL;
 
 	name = dentry->d_name.name;
-	p9_debug(P9_DEBUG_VFS, "%lu,%s,%s\n", dir->i_ino, name, symname);
+	p9_debug(P9_DEBUG_VFS, "%llu,%s,%s\n", dir->i_ino, name, symname);
 
 	dfid = v9fs_parent_fid(dentry);
 	if (IS_ERR(dfid)) {
@@ -734,7 +734,7 @@ v9fs_vfs_link_dotl(struct dentry *old_dentry, struct inode *dir,
 	struct p9_fid *dfid, *oldfid;
 	struct v9fs_session_info *v9ses;
 
-	p9_debug(P9_DEBUG_VFS, "dir ino: %lu, old_name: %pd, new_name: %pd\n",
+	p9_debug(P9_DEBUG_VFS, "dir ino: %llu, old_name: %pd, new_name: %pd\n",
 		 dir->i_ino, old_dentry, dentry);
 
 	v9ses = v9fs_inode2v9ses(dir);
@@ -798,7 +798,7 @@ v9fs_vfs_mknod_dotl(struct mnt_idmap *idmap, struct inode *dir,
 	struct p9_qid qid;
 	struct posix_acl *dacl = NULL, *pacl = NULL;
 
-	p9_debug(P9_DEBUG_VFS, " %lu,%pd mode: %x MAJOR: %u MINOR: %u\n",
+	p9_debug(P9_DEBUG_VFS, " %llu,%pd mode: %x MAJOR: %u MINOR: %u\n",
 		 dir->i_ino, dentry, omode,
 		 MAJOR(rdev), MINOR(rdev));
 
diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c
index 0a1c4f7cb001d..431f24938a1d3 100644
--- a/fs/9p/vfs_super.c
+++ b/fs/9p/vfs_super.c
@@ -312,6 +312,9 @@ static int v9fs_init_fs_context(struct fs_context *fc)
 	if (!ctx)
 		return -ENOMEM;
 
+	fc->ops = &v9fs_context_ops;
+	fc->fs_private = ctx;
+
 	/* initialize core options */
 	ctx->session_opts.afid = ~0;
 	ctx->session_opts.cache = CACHE_NONE;
@@ -345,9 +348,6 @@ static int v9fs_init_fs_context(struct fs_context *fc)
 	ctx->rdma_opts.timeout = P9_RDMA_TIMEOUT;
 	ctx->rdma_opts.privport = false;
 
-	fc->ops = &v9fs_context_ops;
-	fc->fs_private = ctx;
-
 	return 0;
 error:
 	fc->need_free = 1;
diff --git a/fs/Kconfig b/fs/Kconfig
index 0bfdaecaa8775..43cb06de297ff 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -152,6 +152,7 @@ menu "DOS/FAT/EXFAT/NT Filesystems"
 
 source "fs/fat/Kconfig"
 source "fs/exfat/Kconfig"
+source "fs/ntfs/Kconfig"
 source "fs/ntfs3/Kconfig"
 
 endmenu
diff --git a/fs/Makefile b/fs/Makefile
index cf4a745e96798..ae1b07f9c6a0c 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -90,6 +90,7 @@ obj-$(CONFIG_NLS)		+= nls/
 obj-y				+= unicode/
 obj-$(CONFIG_SMBFS)		+= smb/
 obj-$(CONFIG_HPFS_FS)		+= hpfs/
+obj-$(CONFIG_NTFS_FS)		+= ntfs/
 obj-$(CONFIG_NTFS3_FS)		+= ntfs3/
 obj-$(CONFIG_UFS_FS)		+= ufs/
 obj-$(CONFIG_EFS_FS)		+= efs/
diff --git a/fs/affs/amigaffs.c b/fs/affs/amigaffs.c
index fd669daa4e7bc..d8a96d8cc8264 100644
--- a/fs/affs/amigaffs.c
+++ b/fs/affs/amigaffs.c
@@ -33,7 +33,7 @@ affs_insert_hash(struct inode *dir, struct buffer_head *bh)
 	ino = bh->b_blocknr;
 	offset = affs_hash_name(sb, AFFS_TAIL(sb, bh)->name + 1, AFFS_TAIL(sb, bh)->name[0]);
 
-	pr_debug("%s(dir=%lu, ino=%d)\n", __func__, dir->i_ino, ino);
+	pr_debug("%s(dir=%llu, ino=%d)\n", __func__, dir->i_ino, ino);
 
 	dir_bh = affs_bread(sb, dir->i_ino);
 	if (!dir_bh)
@@ -83,7 +83,7 @@ affs_remove_hash(struct inode *dir, struct buffer_head *rem_bh)
 	sb = dir->i_sb;
 	rem_ino = rem_bh->b_blocknr;
 	offset = affs_hash_name(sb, AFFS_TAIL(sb, rem_bh)->name+1, AFFS_TAIL(sb, rem_bh)->name[0]);
-	pr_debug("%s(dir=%lu, ino=%d, hashval=%d)\n", __func__, dir->i_ino,
+	pr_debug("%s(dir=%llu, ino=%d, hashval=%d)\n", __func__, dir->i_ino,
 		 rem_ino, offset);
 
 	bh = affs_bread(sb, dir->i_ino);
@@ -128,7 +128,7 @@ affs_fix_dcache(struct inode *inode, u32 entry_ino)
 	spin_lock(&inode->i_lock);
 	hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
 		if (entry_ino == (u32)(long)dentry->d_fsdata) {
-			dentry->d_fsdata = (void *)inode->i_ino;
+			dentry->d_fsdata = (void *)(unsigned long)inode->i_ino;
 			break;
 		}
 	}
@@ -147,7 +147,7 @@ affs_remove_link(struct dentry *dentry)
 	u32 link_ino, ino;
 	int retval;
 
-	pr_debug("%s(key=%ld)\n", __func__, inode->i_ino);
+	pr_debug("%s(key=%llu)\n", __func__, inode->i_ino);
 	retval = -EIO;
 	bh = affs_bread(sb, inode->i_ino);
 	if (!bh)
@@ -279,7 +279,7 @@ affs_remove_header(struct dentry *dentry)
 	if (!inode)
 		goto done;
 
-	pr_debug("%s(key=%ld)\n", __func__, inode->i_ino);
+	pr_debug("%s(key=%llu)\n", __func__, inode->i_ino);
 	retval = -EIO;
 	bh = affs_bread(sb, (u32)(long)dentry->d_fsdata);
 	if (!bh)
diff --git a/fs/affs/bitmap.c b/fs/affs/bitmap.c
index 5ba9ef2742f6e..40bc4ce6af4ac 100644
--- a/fs/affs/bitmap.c
+++ b/fs/affs/bitmap.c
@@ -125,7 +125,7 @@ affs_alloc_block(struct inode *inode, u32 goal)
 	sb = inode->i_sb;
 	sbi = AFFS_SB(sb);
 
-	pr_debug("balloc(inode=%lu,goal=%u): ", inode->i_ino, goal);
+	pr_debug("balloc(inode=%llu,goal=%u): ", inode->i_ino, goal);
 
 	if (AFFS_I(inode)->i_pa_cnt) {
 		pr_debug("%d\n", AFFS_I(inode)->i_lastalloc+1);
diff --git a/fs/affs/dir.c b/fs/affs/dir.c
index 5c8d83387a394..11e2bac2e3918 100644
--- a/fs/affs/dir.c
+++ b/fs/affs/dir.c
@@ -90,7 +90,7 @@ affs_readdir(struct file *file, struct dir_context *ctx)
 	u32			 ino;
 	int			 error = 0;
 
-	pr_debug("%s(ino=%lu,f_pos=%llx)\n", __func__, inode->i_ino, ctx->pos);
+	pr_debug("%s(ino=%llu,f_pos=%llx)\n", __func__, inode->i_ino, ctx->pos);
 
 	if (ctx->pos < 2) {
 		data->ino = 0;
diff --git a/fs/affs/file.c b/fs/affs/file.c
index 6c9258359ddb9..a51dee9d7d7ec 100644
--- a/fs/affs/file.c
+++ b/fs/affs/file.c
@@ -24,7 +24,7 @@ static struct buffer_head *affs_get_extblock_slow(struct inode *inode, u32 ext);
 static int
 affs_file_open(struct inode *inode, struct file *filp)
 {
-	pr_debug("open(%lu,%d)\n",
+	pr_debug("open(%llu,%d)\n",
 		 inode->i_ino, atomic_read(&AFFS_I(inode)->i_opencnt));
 	atomic_inc(&AFFS_I(inode)->i_opencnt);
 	return 0;
@@ -33,7 +33,7 @@ affs_file_open(struct inode *inode, struct file *filp)
 static int
 affs_file_release(struct inode *inode, struct file *filp)
 {
-	pr_debug("release(%lu, %d)\n",
+	pr_debug("release(%llu, %d)\n",
 		 inode->i_ino, atomic_read(&AFFS_I(inode)->i_opencnt));
 
 	if (atomic_dec_and_test(&AFFS_I(inode)->i_opencnt)) {
@@ -301,7 +301,7 @@ affs_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_resul
 	struct buffer_head	*ext_bh;
 	u32			 ext;
 
-	pr_debug("%s(%lu, %llu)\n", __func__, inode->i_ino,
+	pr_debug("%s(%llu, %llu)\n", __func__, inode->i_ino,
 		 (unsigned long long)block);
 
 	BUG_ON(block > (sector_t)0x7fffffffUL);
@@ -534,7 +534,7 @@ static int affs_do_read_folio_ofs(struct folio *folio, size_t to, int create)
 	size_t bidx, boff, bsize;
 	u32 tmp;
 
-	pr_debug("%s(%lu, %ld, 0, %zu)\n", __func__, inode->i_ino,
+	pr_debug("%s(%llu, %ld, 0, %zu)\n", __func__, inode->i_ino,
 		 folio->index, to);
 	BUG_ON(to > folio_size(folio));
 	bsize = AFFS_SB(sb)->s_data_blksize;
@@ -566,7 +566,7 @@ affs_extent_file_ofs(struct inode *inode, u32 newsize)
 	u32 size, bsize;
 	u32 tmp;
 
-	pr_debug("%s(%lu, %d)\n", __func__, inode->i_ino, newsize);
+	pr_debug("%s(%llu, %d)\n", __func__, inode->i_ino, newsize);
 	bsize = AFFS_SB(sb)->s_data_blksize;
 	bh = NULL;
 	size = AFFS_I(inode)->mmu_private;
@@ -634,7 +634,7 @@ static int affs_read_folio_ofs(struct file *file, struct folio *folio)
 	size_t to;
 	int err;
 
-	pr_debug("%s(%lu, %ld)\n", __func__, inode->i_ino, folio->index);
+	pr_debug("%s(%llu, %ld)\n", __func__, inode->i_ino, folio->index);
 	to = folio_size(folio);
 	if (folio_pos(folio) + to > inode->i_size) {
 		to = inode->i_size - folio_pos(folio);
@@ -658,7 +658,7 @@ static int affs_write_begin_ofs(const struct kiocb *iocb,
 	pgoff_t index;
 	int err = 0;
 
-	pr_debug("%s(%lu, %llu, %llu)\n", __func__, inode->i_ino, pos,
+	pr_debug("%s(%llu, %llu, %llu)\n", __func__, inode->i_ino, pos,
 		 pos + len);
 	if (pos > AFFS_I(inode)->mmu_private) {
 		/* XXX: this probably leaves a too-big i_size in case of
@@ -710,7 +710,7 @@ static int affs_write_end_ofs(const struct kiocb *iocb,
 	 * due to write_begin.
 	 */
 
-	pr_debug("%s(%lu, %llu, %llu)\n", __func__, inode->i_ino, pos,
+	pr_debug("%s(%llu, %llu, %llu)\n", __func__, inode->i_ino, pos,
 		 pos + len);
 	bsize = AFFS_SB(sb)->s_data_blksize;
 	data = folio_address(folio);
@@ -854,7 +854,7 @@ affs_free_prealloc(struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
 
-	pr_debug("free_prealloc(ino=%lu)\n", inode->i_ino);
+	pr_debug("free_prealloc(ino=%llu)\n", inode->i_ino);
 
 	while (AFFS_I(inode)->i_pa_cnt) {
 		AFFS_I(inode)->i_pa_cnt--;
@@ -874,7 +874,7 @@ affs_truncate(struct inode *inode)
 	struct buffer_head *ext_bh;
 	int i;
 
-	pr_debug("truncate(inode=%lu, oldsize=%llu, newsize=%llu)\n",
+	pr_debug("truncate(inode=%llu, oldsize=%llu, newsize=%llu)\n",
 		 inode->i_ino, AFFS_I(inode)->mmu_private, inode->i_size);
 
 	last_blk = 0;
diff --git a/fs/affs/inode.c b/fs/affs/inode.c
index 0bfc7d151dcd3..561fc0185e89c 100644
--- a/fs/affs/inode.c
+++ b/fs/affs/inode.c
@@ -32,7 +32,7 @@ struct inode *affs_iget(struct super_block *sb, unsigned long ino)
 	if (!(inode_state_read_once(inode) & I_NEW))
 		return inode;
 
-	pr_debug("affs_iget(%lu)\n", inode->i_ino);
+	pr_debug("affs_iget(%llu)\n", inode->i_ino);
 
 	block = inode->i_ino;
 	bh = affs_bread(sb, block);
@@ -171,14 +171,14 @@ affs_write_inode(struct inode *inode, struct writeback_control *wbc)
 	uid_t			 uid;
 	gid_t			 gid;
 
-	pr_debug("write_inode(%lu)\n", inode->i_ino);
+	pr_debug("write_inode(%llu)\n", inode->i_ino);
 
 	if (!inode->i_nlink)
 		// possibly free block
 		return 0;
 	bh = affs_bread(sb, inode->i_ino);
 	if (!bh) {
-		affs_error(sb,"write_inode","Cannot read block %lu",inode->i_ino);
+		affs_error(sb, "write_inode", "Cannot read block %llu", inode->i_ino);
 		return -EIO;
 	}
 	tail = AFFS_TAIL(sb, bh);
@@ -219,7 +219,7 @@ affs_notify_change(struct mnt_idmap *idmap, struct dentry *dentry,
 	struct inode *inode = d_inode(dentry);
 	int error;
 
-	pr_debug("notify_change(%lu,0x%x)\n", inode->i_ino, attr->ia_valid);
+	pr_debug("notify_change(%llu,0x%x)\n", inode->i_ino, attr->ia_valid);
 
 	error = setattr_prepare(&nop_mnt_idmap, dentry, attr);
 	if (error)
@@ -260,7 +260,7 @@ void
 affs_evict_inode(struct inode *inode)
 {
 	unsigned long cache_page;
-	pr_debug("evict_inode(ino=%lu, nlink=%u)\n",
+	pr_debug("evict_inode(ino=%llu, nlink=%u)\n",
 		 inode->i_ino, inode->i_nlink);
 	truncate_inode_pages_final(&inode->i_data);
 
@@ -353,7 +353,7 @@ affs_add_entry(struct inode *dir, struct inode *inode, struct dentry *dentry, s3
 	u32 block = 0;
 	int retval;
 
-	pr_debug("%s(dir=%lu, inode=%lu, \"%pd\", type=%d)\n", __func__,
+	pr_debug("%s(dir=%llu, inode=%llu, \"%pd\", type=%d)\n", __func__,
 		 dir->i_ino, inode->i_ino, dentry, type);
 
 	retval = -EIO;
diff --git a/fs/affs/namei.c b/fs/affs/namei.c
index f883be50db122..8705321926007 100644
--- a/fs/affs/namei.c
+++ b/fs/affs/namei.c
@@ -235,7 +235,7 @@ affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
 int
 affs_unlink(struct inode *dir, struct dentry *dentry)
 {
-	pr_debug("%s(dir=%lu, %lu \"%pd\")\n", __func__, dir->i_ino,
+	pr_debug("%s(dir=%llu, %llu \"%pd\")\n", __func__, dir->i_ino,
 		 d_inode(dentry)->i_ino, dentry);
 
 	return affs_remove_header(dentry);
@@ -249,7 +249,7 @@ affs_create(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode	*inode;
 	int		 error;
 
-	pr_debug("%s(%lu,\"%pd\",0%ho)\n",
+	pr_debug("%s(%llu,\"%pd\",0%ho)\n",
 		 __func__, dir->i_ino, dentry, mode);
 
 	inode = affs_new_inode(dir);
@@ -280,7 +280,7 @@ affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode		*inode;
 	int			 error;
 
-	pr_debug("%s(%lu,\"%pd\",0%ho)\n",
+	pr_debug("%s(%llu,\"%pd\",0%ho)\n",
 		 __func__, dir->i_ino, dentry, mode);
 
 	inode = affs_new_inode(dir);
@@ -306,7 +306,7 @@ affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 int
 affs_rmdir(struct inode *dir, struct dentry *dentry)
 {
-	pr_debug("%s(dir=%lu, %lu \"%pd\")\n", __func__, dir->i_ino,
+	pr_debug("%s(dir=%llu, %llu \"%pd\")\n", __func__, dir->i_ino,
 		 d_inode(dentry)->i_ino, dentry);
 
 	return affs_remove_header(dentry);
@@ -323,7 +323,7 @@ affs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	int			 i, maxlen, error;
 	char			 c, lc;
 
-	pr_debug("%s(%lu,\"%pd\" -> \"%s\")\n",
+	pr_debug("%s(%llu,\"%pd\" -> \"%s\")\n",
 		 __func__, dir->i_ino, dentry, symname);
 
 	maxlen = AFFS_SB(sb)->s_hashsize * sizeof(u32) - 1;
@@ -395,7 +395,7 @@ affs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
 {
 	struct inode *inode = d_inode(old_dentry);
 
-	pr_debug("%s(%lu, %lu, \"%pd\")\n", __func__, inode->i_ino, dir->i_ino,
+	pr_debug("%s(%llu, %llu, \"%pd\")\n", __func__, inode->i_ino, dir->i_ino,
 		 dentry);
 
 	return affs_add_entry(dir, inode, dentry, ST_LINKFILE);
@@ -511,7 +511,7 @@ int affs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE))
 		return -EINVAL;
 
-	pr_debug("%s(old=%lu,\"%pd\" to new=%lu,\"%pd\")\n", __func__,
+	pr_debug("%s(old=%llu,\"%pd\" to new=%llu,\"%pd\")\n", __func__,
 		 old_dir->i_ino, old_dentry, new_dir->i_ino, new_dentry);
 
 	if (flags & RENAME_EXCHANGE)
diff --git a/fs/affs/symlink.c b/fs/affs/symlink.c
index 094aec8d17b88..de31ed2e71dfc 100644
--- a/fs/affs/symlink.c
+++ b/fs/affs/symlink.c
@@ -21,7 +21,7 @@ static int affs_symlink_read_folio(struct file *file, struct folio *folio)
 	char			 c;
 	char			 lc;
 
-	pr_debug("get_link(ino=%lu)\n", inode->i_ino);
+	pr_debug("get_link(ino=%llu)\n", inode->i_ino);
 
 	bh = affs_bread(inode->i_sb, inode->i_ino);
 	if (!bh)
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 78caef3f13388..aaaa55878ffd2 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -148,7 +148,7 @@ static bool afs_dir_check_block(struct afs_vnode *dvnode, size_t progress,
 				union afs_xdr_dir_block *block)
 {
 	if (block->hdr.magic != AFS_DIR_MAGIC) {
-		pr_warn("%s(%lx): [%zx] bad magic %04x\n",
+		pr_warn("%s(%llx): [%zx] bad magic %04x\n",
 		       __func__, dvnode->netfs.inode.i_ino,
 		       progress, ntohs(block->hdr.magic));
 		trace_afs_dir_check_failed(dvnode, progress);
@@ -214,7 +214,7 @@ static int afs_dir_check(struct afs_vnode *dvnode)
  */
 static int afs_dir_open(struct inode *inode, struct file *file)
 {
-	_enter("{%lu}", inode->i_ino);
+	_enter("{%llu}", inode->i_ino);
 
 	BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048);
 	BUILD_BUG_ON(sizeof(union afs_xdr_dirent) != 32);
@@ -523,7 +523,7 @@ static int afs_dir_iterate(struct inode *dir, struct dir_context *ctx,
 	int retry_limit = 100;
 	int ret;
 
-	_enter("{%lu},%llx,,", dir->i_ino, ctx->pos);
+	_enter("{%llu},%llx,,", dir->i_ino, ctx->pos);
 
 	do {
 		if (--retry_limit < 0) {
@@ -610,7 +610,7 @@ static int afs_do_lookup_one(struct inode *dir, const struct qstr *name,
 	};
 	int ret;
 
-	_enter("{%lu},{%.*s},", dir->i_ino, name->len, name->name);
+	_enter("{%llu},{%.*s},", dir->i_ino, name->len, name->name);
 
 	/* search the directory */
 	ret = afs_dir_iterate(dir, &cookie.ctx, NULL, _dir_version);
@@ -783,7 +783,7 @@ static struct inode *afs_do_lookup(struct inode *dir, struct dentry *dentry)
 	long ret;
 	int i;
 
-	_enter("{%lu},%p{%pd},", dir->i_ino, dentry, dentry);
+	_enter("{%llu},%p{%pd},", dir->i_ino, dentry, dentry);
 
 	cookie = kzalloc_obj(struct afs_lookup_cookie);
 	if (!cookie)
diff --git a/fs/afs/dir_search.c b/fs/afs/dir_search.c
index d2516e55b5edb..104411c0692f5 100644
--- a/fs/afs/dir_search.c
+++ b/fs/afs/dir_search.c
@@ -194,7 +194,7 @@ int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name,
 	struct afs_dir_iter iter = { .dvnode = dvnode, };
 	int ret, retry_limit = 3;
 
-	_enter("{%lu},,,", dvnode->netfs.inode.i_ino);
+	_enter("{%llu},,,", dvnode->netfs.inode.i_ino);
 
 	if (!afs_dir_init_iter(&iter, name))
 		return -ENOENT;
diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c
index aa56e8951e037..1d5e33bc75022 100644
--- a/fs/afs/dynroot.c
+++ b/fs/afs/dynroot.c
@@ -59,7 +59,7 @@ static struct inode *afs_iget_pseudo_dir(struct super_block *sb, ino_t ino)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	_debug("GOT INODE %p { ino=%lu, vl=%llx, vn=%llx, u=%x }",
+	_debug("GOT INODE %p { ino=%llu, vl=%llx, vn=%llx, u=%x }",
 	       inode, inode->i_ino, fid.vid, fid.vnode, fid.unique);
 
 	vnode = AFS_FS_I(inode);
diff --git a/fs/afs/file.c b/fs/afs/file.c
index f609366fd2ac0..85696ac984cc3 100644
--- a/fs/afs/file.c
+++ b/fs/afs/file.c
@@ -28,6 +28,8 @@ static ssize_t afs_file_splice_read(struct file *in, loff_t *ppos,
 static void afs_vm_open(struct vm_area_struct *area);
 static void afs_vm_close(struct vm_area_struct *area);
 static vm_fault_t afs_vm_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff);
+static int afs_mapped(unsigned long start, unsigned long end, pgoff_t pgoff,
+		      const struct file *file, void **vm_private_data);
 
 const struct file_operations afs_file_operations = {
 	.open		= afs_open,
@@ -61,6 +63,7 @@ const struct address_space_operations afs_file_aops = {
 };
 
 static const struct vm_operations_struct afs_vm_ops = {
+	.mapped		= afs_mapped,
 	.open		= afs_vm_open,
 	.close		= afs_vm_close,
 	.fault		= filemap_fault,
@@ -494,32 +497,45 @@ static void afs_drop_open_mmap(struct afs_vnode *vnode)
  */
 static int afs_file_mmap_prepare(struct vm_area_desc *desc)
 {
-	struct afs_vnode *vnode = AFS_FS_I(file_inode(desc->file));
 	int ret;
 
-	afs_add_open_mmap(vnode);
-
 	ret = generic_file_mmap_prepare(desc);
-	if (ret == 0)
-		desc->vm_ops = &afs_vm_ops;
-	else
-		afs_drop_open_mmap(vnode);
+	if (ret)
+		return ret;
+
+	desc->vm_ops = &afs_vm_ops;
 	return ret;
 }
 
+static int afs_mapped(unsigned long start, unsigned long end, pgoff_t pgoff,
+		      const struct file *file, void **vm_private_data)
+{
+	struct afs_vnode *vnode = AFS_FS_I(file_inode(file));
+
+	afs_add_open_mmap(vnode);
+	return 0;
+}
+
 static void afs_vm_open(struct vm_area_struct *vma)
 {
-	afs_add_open_mmap(AFS_FS_I(file_inode(vma->vm_file)));
+	struct file *file = vma->vm_file;
+	struct afs_vnode *vnode = AFS_FS_I(file_inode(file));
+
+	afs_add_open_mmap(vnode);
 }
 
 static void afs_vm_close(struct vm_area_struct *vma)
 {
-	afs_drop_open_mmap(AFS_FS_I(file_inode(vma->vm_file)));
+	struct file *file = vma->vm_file;
+	struct afs_vnode *vnode = AFS_FS_I(file_inode(file));
+
+	afs_drop_open_mmap(vnode);
 }
 
 static vm_fault_t afs_vm_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff)
 {
-	struct afs_vnode *vnode = AFS_FS_I(file_inode(vmf->vma->vm_file));
+	struct file *file = vmf->vma->vm_file;
+	struct afs_vnode *vnode = AFS_FS_I(file_inode(file));
 
 	if (afs_check_validity(vnode))
 		return filemap_map_pages(vmf, start_pgoff, end_pgoff);
diff --git a/fs/afs/inode.c b/fs/afs/inode.c
index dde1857fcabb3..a5173434f7862 100644
--- a/fs/afs/inode.c
+++ b/fs/afs/inode.c
@@ -683,7 +683,7 @@ int afs_getattr(struct mnt_idmap *idmap, const struct path *path,
 	struct key *key;
 	int ret, seq;
 
-	_enter("{ ino=%lu v=%u }", inode->i_ino, inode->i_generation);
+	_enter("{ ino=%llu v=%u }", inode->i_ino, inode->i_generation);
 
 	if (vnode->volume &&
 	    !(query_flags & AT_STATX_DONT_SYNC) &&
diff --git a/fs/afs/internal.h b/fs/afs/internal.h
index 009064b8d6616..599353c333372 100644
--- a/fs/afs/internal.h
+++ b/fs/afs/internal.h
@@ -31,7 +31,6 @@
 
 #define AFS_CELL_MAX_ADDRS 15
 
-struct pagevec;
 struct afs_call;
 struct afs_vnode;
 struct afs_server_probe;
diff --git a/fs/afs/write.c b/fs/afs/write.c
index 93ad86ff33453..fcfed9d24e0a3 100644
--- a/fs/afs/write.c
+++ b/fs/afs/write.c
@@ -10,7 +10,6 @@
 #include <linux/fs.h>
 #include <linux/pagemap.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
 #include <linux/netfs.h>
 #include <trace/events/netfs.h>
 #include "internal.h"
diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c
index c53dc551053ba..c1e210cec4369 100644
--- a/fs/autofs/inode.c
+++ b/fs/autofs/inode.c
@@ -92,7 +92,7 @@ static int autofs_show_options(struct seq_file *m, struct dentry *root)
 		seq_puts(m, ",ignore");
 #ifdef CONFIG_CHECKPOINT_RESTORE
 	if (sbi->pipe)
-		seq_printf(m, ",pipe_ino=%ld", file_inode(sbi->pipe)->i_ino);
+		seq_printf(m, ",pipe_ino=%llu", file_inode(sbi->pipe)->i_ino);
 	else
 		seq_puts(m, ",pipe_ino=-1");
 #endif
diff --git a/fs/backing-file.c b/fs/backing-file.c
index 45da8600d5644..d0a64c2103907 100644
--- a/fs/backing-file.c
+++ b/fs/backing-file.c
@@ -11,16 +11,19 @@
 #include <linux/fs.h>
 #include <linux/backing-file.h>
 #include <linux/splice.h>
+#include <linux/uio.h>
 #include <linux/mm.h>
+#include <linux/security.h>
 
 #include "internal.h"
 
 /**
  * backing_file_open - open a backing file for kernel internal use
  * @user_path:	path that the user reuqested to open
+ * @user_cred:	credentials that the user used for open
  * @flags:	open flags
  * @real_path:	path of the backing file
- * @cred:	credentials for open
+ * @cred:	credentials for open of the backing file
  *
  * Open a backing file for a stackable filesystem (e.g., overlayfs).
  * @user_path may be on the stackable filesystem and @real_path on the
@@ -29,20 +32,21 @@
  * returned file into a container structure that also stores the stacked
  * file's path, which can be retrieved using backing_file_user_path().
  */
-struct file *backing_file_open(const struct path *user_path, int flags,
+struct file *backing_file_open(const struct path *user_path,
+			       const struct cred *user_cred, int flags,
 			       const struct path *real_path,
 			       const struct cred *cred)
 {
 	struct file *f;
 	int error;
 
-	f = alloc_empty_backing_file(flags, cred);
+	f = alloc_empty_backing_file(flags, cred, user_cred);
 	if (IS_ERR(f))
 		return f;
 
-	path_get(user_path);
-	backing_file_set_user_path(f, user_path);
-	error = vfs_open(real_path, f);
+	error = backing_file_open_user_path(f, user_path);
+	if (!error)
+		error = vfs_open(real_path, f);
 	if (error) {
 		fput(f);
 		f = ERR_PTR(error);
@@ -52,7 +56,8 @@ struct file *backing_file_open(const struct path *user_path, int flags,
 }
 EXPORT_SYMBOL_GPL(backing_file_open);
 
-struct file *backing_tmpfile_open(const struct path *user_path, int flags,
+struct file *backing_tmpfile_open(const struct path *user_path,
+				  const struct cred *user_cred, int flags,
 				  const struct path *real_parentpath,
 				  umode_t mode, const struct cred *cred)
 {
@@ -60,13 +65,13 @@ struct file *backing_tmpfile_open(const struct path *user_path, int flags,
 	struct file *f;
 	int error;
 
-	f = alloc_empty_backing_file(flags, cred);
+	f = alloc_empty_backing_file(flags, cred, user_cred);
 	if (IS_ERR(f))
 		return f;
 
-	path_get(user_path);
-	backing_file_set_user_path(f, user_path);
-	error = vfs_tmpfile(real_idmap, real_parentpath, f, mode);
+	error = backing_file_open_user_path(f, user_path);
+	if (!error)
+		error = vfs_tmpfile(real_idmap, real_parentpath, f, mode);
 	if (error) {
 		fput(f);
 		f = ERR_PTR(error);
@@ -336,8 +341,13 @@ int backing_file_mmap(struct file *file, struct vm_area_struct *vma,
 
 	vma_set_file(vma, file);
 
-	scoped_with_creds(ctx->cred)
+	scoped_with_creds(ctx->cred) {
+		ret = security_mmap_backing_file(vma, file, user_file);
+		if (ret)
+			return ret;
+
 		ret = vfs_mmap(vma->vm_file, vma);
+	}
 
 	if (ctx->accessed)
 		ctx->accessed(user_file);
diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c
index cecbc92f959aa..c12caae9a9679 100644
--- a/fs/befs/linuxvfs.c
+++ b/fs/befs/linuxvfs.c
@@ -140,20 +140,20 @@ befs_get_block(struct inode *inode, sector_t block,
 	int res;
 	ulong disk_off;
 
-	befs_debug(sb, "---> befs_get_block() for inode %lu, block %ld",
-		   (unsigned long)inode->i_ino, (long)block);
+	befs_debug(sb, "---> befs_get_block() for inode %llu, block %ld",
+		   inode->i_ino, (long)block);
 	if (create) {
 		befs_error(sb, "befs_get_block() was asked to write to "
-			   "block %ld in inode %lu", (long)block,
-			   (unsigned long)inode->i_ino);
+			   "block %ld in inode %llu", (long)block,
+			   inode->i_ino);
 		return -EPERM;
 	}
 
 	res = befs_fblock2brun(sb, ds, block, &run);
 	if (res != BEFS_OK) {
 		befs_error(sb,
-			   "<--- %s for inode %lu, block %ld ERROR",
-			   __func__, (unsigned long)inode->i_ino,
+			   "<--- %s for inode %llu, block %ld ERROR",
+			   __func__, inode->i_ino,
 			   (long)block);
 		return -EFBIG;
 	}
@@ -162,8 +162,8 @@ befs_get_block(struct inode *inode, sector_t block,
 
 	map_bh(bh_result, inode->i_sb, disk_off);
 
-	befs_debug(sb, "<--- %s for inode %lu, block %ld, disk address %lu",
-		  __func__, (unsigned long)inode->i_ino, (long)block,
+	befs_debug(sb, "<--- %s for inode %llu, block %ld, disk address %lu",
+		  __func__, inode->i_ino, (long)block,
 		  (unsigned long)disk_off);
 
 	return 0;
@@ -181,7 +181,7 @@ befs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
 	char *utfname;
 	const char *name = dentry->d_name.name;
 
-	befs_debug(sb, "---> %s name %pd inode %ld", __func__,
+	befs_debug(sb, "---> %s name %pd inode %llu", __func__,
 		   dentry, dir->i_ino);
 
 	/* Convert to UTF-8 */
@@ -224,7 +224,7 @@ befs_readdir(struct file *file, struct dir_context *ctx)
 	size_t keysize;
 	char keybuf[BEFS_NAME_LEN + 1];
 
-	befs_debug(sb, "---> %s name %pD, inode %ld, ctx->pos %lld",
+	befs_debug(sb, "---> %s name %pD, inode %llu, ctx->pos %lld",
 		  __func__, file, inode->i_ino, ctx->pos);
 
 	while (1) {
@@ -233,7 +233,7 @@ befs_readdir(struct file *file, struct dir_context *ctx)
 
 		if (result == BEFS_ERR) {
 			befs_debug(sb, "<--- %s ERROR", __func__);
-			befs_error(sb, "IO error reading %pD (inode %lu)",
+			befs_error(sb, "IO error reading %pD (inode %llu)",
 				   file, inode->i_ino);
 			return -EIO;
 
@@ -324,7 +324,7 @@ static struct inode *befs_iget(struct super_block *sb, unsigned long ino)
 	bh = sb_bread(sb, inode->i_ino);
 	if (!bh) {
 		befs_error(sb, "unable to read inode block - "
-			   "inode = %lu", inode->i_ino);
+			   "inode = %llu", inode->i_ino);
 		goto unacquire_none;
 	}
 
@@ -333,7 +333,7 @@ static struct inode *befs_iget(struct super_block *sb, unsigned long ino)
 	befs_dump_inode(sb, raw_inode);
 
 	if (befs_check_inode(sb, raw_inode, inode->i_ino) != BEFS_OK) {
-		befs_error(sb, "Bad inode: %lu", inode->i_ino);
+		befs_error(sb, "Bad inode: %llu", inode->i_ino);
 		goto unacquire_bh;
 	}
 
@@ -407,7 +407,7 @@ static struct inode *befs_iget(struct super_block *sb, unsigned long ino)
 			inode->i_op = &simple_symlink_inode_operations;
 		}
 	} else {
-		befs_error(sb, "Inode %lu is not a regular file, "
+		befs_error(sb, "Inode %llu is not a regular file, "
 			   "directory or symlink. THAT IS WRONG! BeFS has no "
 			   "on disk special files", inode->i_ino);
 		goto unacquire_bh;
diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c
index c375e22c4c0c1..481514db4eae3 100644
--- a/fs/bfs/dir.c
+++ b/fs/bfs/dir.c
@@ -35,7 +35,7 @@ static int bfs_readdir(struct file *f, struct dir_context *ctx)
 	int block;
 
 	if (ctx->pos & (BFS_DIRENT_SIZE - 1)) {
-		printf("Bad f_pos=%08lx for %s:%08lx\n",
+		printf("Bad f_pos=%08lx for %s:%08llx\n",
 					(unsigned long)ctx->pos,
 					dir->i_sb->s_id, dir->i_ino);
 		return -EINVAL;
@@ -180,7 +180,7 @@ static int bfs_unlink(struct inode *dir, struct dentry *dentry)
 		goto out_brelse;
 
 	if (!inode->i_nlink) {
-		printf("unlinking non-existent file %s:%lu (nlink=%d)\n",
+		printf("unlinking non-existent file %s:%llu (nlink=%d)\n",
 					inode->i_sb->s_id, inode->i_ino,
 					inode->i_nlink);
 		set_nlink(inode, 1);
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index fb857faaf0d62..107e5149a6ec4 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -247,7 +247,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
 	 */
 	ARCH_DLINFO;
 #endif
-	NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP);
+	NEW_AUX_ENT(AT_HWCAP, bprm->hwcap);
 	NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE);
 	NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC);
 	NEW_AUX_ENT(AT_PHDR, phdr_addr);
@@ -265,13 +265,13 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
 	NEW_AUX_ENT(AT_SECURE, bprm->secureexec);
 	NEW_AUX_ENT(AT_RANDOM, (elf_addr_t)(unsigned long)u_rand_bytes);
 #ifdef ELF_HWCAP2
-	NEW_AUX_ENT(AT_HWCAP2, ELF_HWCAP2);
+	NEW_AUX_ENT(AT_HWCAP2, bprm->hwcap2);
 #endif
 #ifdef ELF_HWCAP3
-	NEW_AUX_ENT(AT_HWCAP3, ELF_HWCAP3);
+	NEW_AUX_ENT(AT_HWCAP3, bprm->hwcap3);
 #endif
 #ifdef ELF_HWCAP4
-	NEW_AUX_ENT(AT_HWCAP4, ELF_HWCAP4);
+	NEW_AUX_ENT(AT_HWCAP4, bprm->hwcap4);
 #endif
 	NEW_AUX_ENT(AT_EXECFN, bprm->exec);
 	if (k_platform) {
@@ -453,14 +453,13 @@ static unsigned long elf_load(struct file *filep, unsigned long addr,
 		zero_end = ELF_PAGEALIGN(zero_end);
 
 		error = vm_brk_flags(zero_start, zero_end - zero_start,
-				     prot & PROT_EXEC ? VM_EXEC : 0);
+				     prot & PROT_EXEC);
 		if (error)
 			map_addr = error;
 	}
 	return map_addr;
 }
 
-
 static unsigned long total_mapping_size(const struct elf_phdr *phdr, int nr)
 {
 	elf_addr_t min_addr = -1;
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index 95b1d08521354..73d593f2c8955 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -595,6 +595,12 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm,
 #ifdef ELF_HWCAP2
 	nitems++;
 #endif
+#ifdef ELF_HWCAP3
+	nitems++;
+#endif
+#ifdef ELF_HWCAP4
+	nitems++;
+#endif
 
 	csp = sp;
 	sp -= nitems * 2 * sizeof(unsigned long);
@@ -623,15 +629,15 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm,
 	 */
 	ARCH_DLINFO;
 #endif
-	NEW_AUX_ENT(AT_HWCAP,	ELF_HWCAP);
+	NEW_AUX_ENT(AT_HWCAP,	bprm->hwcap);
 #ifdef ELF_HWCAP2
-	NEW_AUX_ENT(AT_HWCAP2,	ELF_HWCAP2);
+	NEW_AUX_ENT(AT_HWCAP2,	bprm->hwcap2);
 #endif
 #ifdef ELF_HWCAP3
-	NEW_AUX_ENT(AT_HWCAP3,	ELF_HWCAP3);
+	NEW_AUX_ENT(AT_HWCAP3,	bprm->hwcap3);
 #endif
 #ifdef ELF_HWCAP4
-	NEW_AUX_ENT(AT_HWCAP4,	ELF_HWCAP4);
+	NEW_AUX_ENT(AT_HWCAP4,	bprm->hwcap4);
 #endif
 	NEW_AUX_ENT(AT_PAGESZ,	PAGE_SIZE);
 	NEW_AUX_ENT(AT_CLKTCK,	CLOCKS_PER_SEC);
@@ -911,7 +917,7 @@ static int elf_fdpic_map_file(struct elf_fdpic_params *params,
 	return 0;
 
 dynamic_error:
-	printk("ELF FDPIC %s with invalid DYNAMIC section (inode=%lu)\n",
+	printk("ELF FDPIC %s with invalid DYNAMIC section (inode=%llu)\n",
 	       what, file_inode(file)->i_ino);
 	return -ELIBBAD;
 }
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
index ede184b6eda16..04677c06ebc45 100644
--- a/fs/btrfs/Kconfig
+++ b/fs/btrfs/Kconfig
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
+# misc-next marker
 
 config BTRFS_FS
 	tristate "Btrfs filesystem support"
diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
index 31b00b9325884..b15122aa26f95 100644
--- a/fs/btrfs/Makefile
+++ b/fs/btrfs/Makefile
@@ -45,3 +45,7 @@ btrfs-$(CONFIG_BTRFS_FS_RUN_SANITY_TESTS) += tests/free-space-tests.o \
 	tests/free-space-tree-tests.o tests/extent-map-tests.o \
 	tests/raid-stripe-tree-tests.o tests/delayed-refs-tests.o \
 	tests/chunk-allocation-tests.o
+
+ifeq ($(CONFIG_BLK_DEV_ZONED),y)
+btrfs-$(CONFIG_BTRFS_FS_RUN_SANITY_TESTS) += tests/zoned-tests.o
+endif
diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c
index af98421e79227..273924ca912c2 100644
--- a/fs/btrfs/backref.c
+++ b/fs/btrfs/backref.c
@@ -858,11 +858,6 @@ static int add_missing_keys(struct btrfs_fs_info *fs_info,
 			free_pref(ref);
 			return PTR_ERR(eb);
 		}
-		if (unlikely(!extent_buffer_uptodate(eb))) {
-			free_pref(ref);
-			free_extent_buffer(eb);
-			return -EIO;
-		}
 
 		if (lock)
 			btrfs_tree_read_lock(eb);
@@ -1393,6 +1388,13 @@ static int find_parent_nodes(struct btrfs_backref_walk_ctx *ctx,
 		.indirect_missing_keys = PREFTREE_INIT
 	};
 
+	if (unlikely(!root)) {
+		btrfs_err(ctx->fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  ctx->bytenr);
+		return -EUCLEAN;
+	}
+
 	/* Roots ulist is not needed when using a sharedness check context. */
 	if (sc)
 		ASSERT(ctx->roots == NULL);
@@ -1613,11 +1615,6 @@ static int find_parent_nodes(struct btrfs_backref_walk_ctx *ctx,
 					ret = PTR_ERR(eb);
 					goto out;
 				}
-				if (unlikely(!extent_buffer_uptodate(eb))) {
-					free_extent_buffer(eb);
-					ret = -EIO;
-					goto out;
-				}
 
 				if (!path->skip_locking)
 					btrfs_tree_read_lock(eb);
@@ -2204,6 +2201,13 @@ int extent_from_logical(struct btrfs_fs_info *fs_info, u64 logical,
 	struct btrfs_extent_item *ei;
 	struct btrfs_key key;
 
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  logical);
+		return -EUCLEAN;
+	}
+
 	key.objectid = logical;
 	if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
 		key.type = BTRFS_METADATA_ITEM_KEY;
@@ -2851,6 +2855,13 @@ int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr)
 	struct btrfs_key key;
 	int ret;
 
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	key.type = BTRFS_METADATA_ITEM_KEY;
 	key.offset = (u64)-1;
@@ -2987,6 +2998,13 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter)
 
 	/* We're at keyed items, there is no inline item, go to the next one */
 	extent_root = btrfs_extent_root(iter->fs_info, iter->bytenr);
+	if (unlikely(!extent_root)) {
+		btrfs_err(iter->fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  iter->bytenr);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_next_item(extent_root, iter->path);
 	if (ret)
 		return ret;
diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c
index 2a2a21aec8178..08b1d62603d0a 100644
--- a/fs/btrfs/bio.c
+++ b/fs/btrfs/bio.c
@@ -352,8 +352,7 @@ static void btrfs_log_dev_io_error(const struct bio *bio, struct btrfs_device *d
 {
 	if (!dev || !dev->bdev)
 		return;
-	if (bio->bi_status != BLK_STS_IOERR && bio->bi_status != BLK_STS_TARGET)
-		return;
+	ASSERT(bio->bi_status);
 
 	if (btrfs_op(bio) == BTRFS_MAP_WRITE)
 		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c
index 2a886bece8105..fa6e49a4ba37d 100644
--- a/fs/btrfs/block-group.c
+++ b/fs/btrfs/block-group.c
@@ -728,7 +728,7 @@ static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
 	struct extent_buffer *leaf;
 	struct btrfs_key key;
 	u64 total_found = 0;
-	u64 last = 0;
+	u64 last = block_group->start;
 	u32 nritems;
 	int ret;
 	bool wakeup = true;
@@ -737,8 +737,13 @@ static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
 	if (!path)
 		return -ENOMEM;
 
-	last = max_t(u64, block_group->start, BTRFS_SUPER_INFO_OFFSET);
 	extent_root = btrfs_extent_root(fs_info, last);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for block group at offset %llu",
+			  block_group->start);
+		return -EUCLEAN;
+	}
 
 #ifdef CONFIG_BTRFS_DEBUG
 	/*
@@ -1061,6 +1066,11 @@ static int remove_block_group_item(struct btrfs_trans_handle *trans,
 	int ret;
 
 	root = btrfs_block_group_root(fs_info);
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return -EUCLEAN;
+	}
+
 	key.objectid = block_group->start;
 	key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
 	key.offset = block_group->length;
@@ -1349,6 +1359,11 @@ struct btrfs_trans_handle *btrfs_start_trans_remove_block_group(
 	struct btrfs_chunk_map *map;
 	unsigned int num_items;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return ERR_PTR(-EUCLEAN);
+	}
+
 	map = btrfs_find_chunk_map(fs_info, chunk_offset, 1);
 	ASSERT(map != NULL);
 	ASSERT(map->start == chunk_offset);
@@ -1597,6 +1612,24 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 
 		spin_lock(&space_info->lock);
 		spin_lock(&block_group->lock);
+
+		if (btrfs_is_zoned(fs_info) && btrfs_is_block_group_used(block_group) &&
+		    block_group->zone_unusable >= div_u64(block_group->length, 2)) {
+			/*
+			 * If the block group has data left, but at least half
+			 * of the block group is zone_unusable, mark it as
+			 * reclaimable before continuing with the next block group.
+			 */
+
+			spin_unlock(&block_group->lock);
+			spin_unlock(&space_info->lock);
+			up_write(&space_info->groups_sem);
+
+			btrfs_mark_bg_to_reclaim(block_group);
+
+			goto next;
+		}
+
 		if (btrfs_is_block_group_used(block_group) ||
 		    (block_group->ro && !(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) ||
 		    list_is_singular(&block_group->list) ||
@@ -1663,7 +1696,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 		spin_unlock(&space_info->lock);
 
 		/* We don't want to force the issue, only flip if it's ok. */
-		ret = inc_block_group_ro(block_group, 0);
+		ret = inc_block_group_ro(block_group, false);
 		up_write(&space_info->groups_sem);
 		if (ret < 0) {
 			ret = 0;
@@ -1876,13 +1909,145 @@ static bool should_reclaim_block_group(const struct btrfs_block_group *bg, u64 b
 	return true;
 }
 
-void btrfs_reclaim_bgs_work(struct work_struct *work)
+static int btrfs_reclaim_block_group(struct btrfs_block_group *bg, int *reclaimed)
+{
+	struct btrfs_fs_info *fs_info = bg->fs_info;
+	struct btrfs_space_info *space_info = bg->space_info;
+	u64 used;
+	u64 reserved;
+	u64 old_total;
+	int ret = 0;
+
+	/* Don't race with allocators so take the groups_sem */
+	down_write(&space_info->groups_sem);
+
+	spin_lock(&space_info->lock);
+	spin_lock(&bg->lock);
+	if (bg->reserved || bg->pinned || bg->ro) {
+		/*
+		 * We want to bail if we made new allocations or have
+		 * outstanding allocations in this block group.  We do
+		 * the ro check in case balance is currently acting on
+		 * this block group.
+		 */
+		spin_unlock(&bg->lock);
+		spin_unlock(&space_info->lock);
+		up_write(&space_info->groups_sem);
+		return 0;
+	}
+
+	if (bg->used == 0) {
+		/*
+		 * It is possible that we trigger relocation on a block
+		 * group as its extents are deleted and it first goes
+		 * below the threshold, then shortly after goes empty.
+		 *
+		 * In this case, relocating it does delete it, but has
+		 * some overhead in relocation specific metadata, looking
+		 * for the non-existent extents and running some extra
+		 * transactions, which we can avoid by using one of the
+		 * other mechanisms for dealing with empty block groups.
+		 */
+		if (!btrfs_test_opt(fs_info, DISCARD_ASYNC))
+			btrfs_mark_bg_unused(bg);
+		spin_unlock(&bg->lock);
+		spin_unlock(&space_info->lock);
+		up_write(&space_info->groups_sem);
+		return 0;
+	}
+
+	/*
+	 * The block group might no longer meet the reclaim condition by
+	 * the time we get around to reclaiming it, so to avoid
+	 * reclaiming overly full block_groups, skip reclaiming them.
+	 *
+	 * Since the decision making process also depends on the amount
+	 * being freed, pass in a fake giant value to skip that extra
+	 * check, which is more meaningful when adding to the list in
+	 * the first place.
+	 */
+	if (!should_reclaim_block_group(bg, bg->length)) {
+		spin_unlock(&bg->lock);
+		spin_unlock(&space_info->lock);
+		up_write(&space_info->groups_sem);
+		return 0;
+	}
+
+	spin_unlock(&bg->lock);
+	old_total = space_info->total_bytes;
+	spin_unlock(&space_info->lock);
+
+	/*
+	 * Get out fast, in case we're read-only or unmounting the
+	 * filesystem. It is OK to drop block groups from the list even
+	 * for the read-only case. As we did take the super write lock,
+	 * "mount -o remount,ro" won't happen and read-only filesystem
+	 * means it is forced read-only due to a fatal error. So, it
+	 * never gets back to read-write to let us reclaim again.
+	 */
+	if (btrfs_need_cleaner_sleep(fs_info)) {
+		up_write(&space_info->groups_sem);
+		return 0;
+	}
+
+	ret = inc_block_group_ro(bg, false);
+	up_write(&space_info->groups_sem);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * The amount of bytes reclaimed corresponds to the sum of the
+	 * "used" and "reserved" counters. We have set the block group
+	 * to RO above, which prevents reservations from happening but
+	 * we may have existing reservations for which allocation has
+	 * not yet been done - btrfs_update_block_group() was not yet
+	 * called, which is where we will transfer a reserved extent's
+	 * size from the "reserved" counter to the "used" counter - this
+	 * happens when running delayed references. When we relocate the
+	 * chunk below, relocation first flushes delalloc, waits for
+	 * ordered extent completion (which is where we create delayed
+	 * references for data extents) and commits the current
+	 * transaction (which runs delayed references), and only after
+	 * it does the actual work to move extents out of the block
+	 * group. So the reported amount of reclaimed bytes is
+	 * effectively the sum of the 'used' and 'reserved' counters.
+	 */
+	spin_lock(&bg->lock);
+	used = bg->used;
+	reserved = bg->reserved;
+	spin_unlock(&bg->lock);
+
+	trace_btrfs_reclaim_block_group(bg);
+	ret = btrfs_relocate_chunk(fs_info, bg->start, false);
+	if (ret) {
+		btrfs_dec_block_group_ro(bg);
+		btrfs_err(fs_info, "error relocating chunk %llu",
+			  bg->start);
+		used = 0;
+		reserved = 0;
+		spin_lock(&space_info->lock);
+		space_info->reclaim_errors++;
+		spin_unlock(&space_info->lock);
+	}
+	spin_lock(&space_info->lock);
+	space_info->reclaim_count++;
+	space_info->reclaim_bytes += used;
+	space_info->reclaim_bytes += reserved;
+	if (space_info->total_bytes < old_total)
+		btrfs_set_periodic_reclaim_ready(space_info, true);
+	spin_unlock(&space_info->lock);
+	if (!ret)
+		(*reclaimed)++;
+
+	return ret;
+}
+
+void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit)
 {
-	struct btrfs_fs_info *fs_info =
-		container_of(work, struct btrfs_fs_info, reclaim_bgs_work);
 	struct btrfs_block_group *bg;
 	struct btrfs_space_info *space_info;
 	LIST_HEAD(retry_list);
+	int reclaimed = 0;
 
 	if (!btrfs_should_reclaim(fs_info))
 		return;
@@ -1909,10 +2074,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
 	 */
 	list_sort(NULL, &fs_info->reclaim_bgs, reclaim_bgs_cmp);
 	while (!list_empty(&fs_info->reclaim_bgs)) {
-		u64 used;
-		u64 reserved;
-		u64 old_total;
-		int ret = 0;
+		int ret;
 
 		bg = list_first_entry(&fs_info->reclaim_bgs,
 				      struct btrfs_block_group,
@@ -1921,126 +2083,8 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
 
 		space_info = bg->space_info;
 		spin_unlock(&fs_info->unused_bgs_lock);
+		ret = btrfs_reclaim_block_group(bg, &reclaimed);
 
-		/* Don't race with allocators so take the groups_sem */
-		down_write(&space_info->groups_sem);
-
-		spin_lock(&space_info->lock);
-		spin_lock(&bg->lock);
-		if (bg->reserved || bg->pinned || bg->ro) {
-			/*
-			 * We want to bail if we made new allocations or have
-			 * outstanding allocations in this block group.  We do
-			 * the ro check in case balance is currently acting on
-			 * this block group.
-			 */
-			spin_unlock(&bg->lock);
-			spin_unlock(&space_info->lock);
-			up_write(&space_info->groups_sem);
-			goto next;
-		}
-		if (bg->used == 0) {
-			/*
-			 * It is possible that we trigger relocation on a block
-			 * group as its extents are deleted and it first goes
-			 * below the threshold, then shortly after goes empty.
-			 *
-			 * In this case, relocating it does delete it, but has
-			 * some overhead in relocation specific metadata, looking
-			 * for the non-existent extents and running some extra
-			 * transactions, which we can avoid by using one of the
-			 * other mechanisms for dealing with empty block groups.
-			 */
-			if (!btrfs_test_opt(fs_info, DISCARD_ASYNC))
-				btrfs_mark_bg_unused(bg);
-			spin_unlock(&bg->lock);
-			spin_unlock(&space_info->lock);
-			up_write(&space_info->groups_sem);
-			goto next;
-
-		}
-		/*
-		 * The block group might no longer meet the reclaim condition by
-		 * the time we get around to reclaiming it, so to avoid
-		 * reclaiming overly full block_groups, skip reclaiming them.
-		 *
-		 * Since the decision making process also depends on the amount
-		 * being freed, pass in a fake giant value to skip that extra
-		 * check, which is more meaningful when adding to the list in
-		 * the first place.
-		 */
-		if (!should_reclaim_block_group(bg, bg->length)) {
-			spin_unlock(&bg->lock);
-			spin_unlock(&space_info->lock);
-			up_write(&space_info->groups_sem);
-			goto next;
-		}
-
-		spin_unlock(&bg->lock);
-		old_total = space_info->total_bytes;
-		spin_unlock(&space_info->lock);
-
-		/*
-		 * Get out fast, in case we're read-only or unmounting the
-		 * filesystem. It is OK to drop block groups from the list even
-		 * for the read-only case. As we did take the super write lock,
-		 * "mount -o remount,ro" won't happen and read-only filesystem
-		 * means it is forced read-only due to a fatal error. So, it
-		 * never gets back to read-write to let us reclaim again.
-		 */
-		if (btrfs_need_cleaner_sleep(fs_info)) {
-			up_write(&space_info->groups_sem);
-			goto next;
-		}
-
-		ret = inc_block_group_ro(bg, 0);
-		up_write(&space_info->groups_sem);
-		if (ret < 0)
-			goto next;
-
-		/*
-		 * The amount of bytes reclaimed corresponds to the sum of the
-		 * "used" and "reserved" counters. We have set the block group
-		 * to RO above, which prevents reservations from happening but
-		 * we may have existing reservations for which allocation has
-		 * not yet been done - btrfs_update_block_group() was not yet
-		 * called, which is where we will transfer a reserved extent's
-		 * size from the "reserved" counter to the "used" counter - this
-		 * happens when running delayed references. When we relocate the
-		 * chunk below, relocation first flushes delalloc, waits for
-		 * ordered extent completion (which is where we create delayed
-		 * references for data extents) and commits the current
-		 * transaction (which runs delayed references), and only after
-		 * it does the actual work to move extents out of the block
-		 * group. So the reported amount of reclaimed bytes is
-		 * effectively the sum of the 'used' and 'reserved' counters.
-		 */
-		spin_lock(&bg->lock);
-		used = bg->used;
-		reserved = bg->reserved;
-		spin_unlock(&bg->lock);
-
-		trace_btrfs_reclaim_block_group(bg);
-		ret = btrfs_relocate_chunk(fs_info, bg->start, false);
-		if (ret) {
-			btrfs_dec_block_group_ro(bg);
-			btrfs_err(fs_info, "error relocating chunk %llu",
-				  bg->start);
-			used = 0;
-			reserved = 0;
-			spin_lock(&space_info->lock);
-			space_info->reclaim_errors++;
-			spin_unlock(&space_info->lock);
-		}
-		spin_lock(&space_info->lock);
-		space_info->reclaim_count++;
-		space_info->reclaim_bytes += used;
-		space_info->reclaim_bytes += reserved;
-		if (space_info->total_bytes < old_total)
-			btrfs_set_periodic_reclaim_ready(space_info, true);
-		spin_unlock(&space_info->lock);
-
-next:
 		if (ret && !READ_ONCE(space_info->periodic_reclaim))
 			btrfs_link_bg_list(bg, &retry_list);
 		btrfs_put_block_group(bg);
@@ -2058,6 +2102,8 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
 		if (!mutex_trylock(&fs_info->reclaim_bgs_lock))
 			goto end;
 		spin_lock(&fs_info->unused_bgs_lock);
+		if (reclaimed >= limit)
+			break;
 	}
 	spin_unlock(&fs_info->unused_bgs_lock);
 	mutex_unlock(&fs_info->reclaim_bgs_lock);
@@ -2068,6 +2114,14 @@ void btrfs_reclaim_bgs_work(struct work_struct *work)
 	btrfs_exclop_finish(fs_info);
 }
 
+void btrfs_reclaim_bgs_work(struct work_struct *work)
+{
+	struct btrfs_fs_info *fs_info =
+		container_of(work, struct btrfs_fs_info, reclaim_bgs_work);
+
+	btrfs_reclaim_block_groups(fs_info, -1);
+}
+
 void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info)
 {
 	btrfs_reclaim_sweep(fs_info);
@@ -2140,6 +2194,11 @@ static int find_first_block_group(struct btrfs_fs_info *fs_info,
 	int ret;
 	struct btrfs_key found_key;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return -EUCLEAN;
+	}
+
 	btrfs_for_each_slot(root, key, &found_key, path, ret) {
 		if (found_key.objectid >= key->objectid &&
 		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
@@ -2201,7 +2260,7 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
 	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
 		io_stripe_size = btrfs_stripe_nr_to_offset(nr_data_stripes(map));
 
-	buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
+	buf = kzalloc_objs(u64, map->num_stripes, GFP_NOFS);
 	if (!buf) {
 		ret = -ENOMEM;
 		goto out;
@@ -2517,7 +2576,7 @@ static int read_one_block_group(struct btrfs_fs_info *info,
 				btrfs_mark_bg_unused(cache);
 		}
 	} else {
-		inc_block_group_ro(cache, 1);
+		inc_block_group_ro(cache, true);
 	}
 
 	return 0;
@@ -2673,11 +2732,11 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info)
 		list_for_each_entry(cache,
 				&space_info->block_groups[BTRFS_RAID_RAID0],
 				list)
-			inc_block_group_ro(cache, 1);
+			inc_block_group_ro(cache, true);
 		list_for_each_entry(cache,
 				&space_info->block_groups[BTRFS_RAID_SINGLE],
 				list)
-			inc_block_group_ro(cache, 1);
+			inc_block_group_ro(cache, true);
 	}
 
 	btrfs_init_global_block_rsv(info);
@@ -2713,6 +2772,11 @@ static int insert_block_group_item(struct btrfs_trans_handle *trans,
 	size_t size;
 	int ret;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return -EUCLEAN;
+	}
+
 	spin_lock(&block_group->lock);
 	btrfs_set_stack_block_group_v2_used(&bgi, block_group->used);
 	btrfs_set_stack_block_group_v2_chunk_objectid(&bgi, block_group->global_root_id);
@@ -3048,6 +3112,11 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
 	int ret;
 	bool dirty_bg_running;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return -EUCLEAN;
+	}
+
 	/*
 	 * This can only happen when we are doing read-only scrub on read-only
 	 * mount.
@@ -3056,7 +3125,7 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
 	 */
 	if (sb_rdonly(fs_info->sb)) {
 		mutex_lock(&fs_info->ro_block_group_mutex);
-		ret = inc_block_group_ro(cache, 0);
+		ret = inc_block_group_ro(cache, false);
 		mutex_unlock(&fs_info->ro_block_group_mutex);
 		return ret;
 	}
@@ -3107,7 +3176,7 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
 		}
 	}
 
-	ret = inc_block_group_ro(cache, 0);
+	ret = inc_block_group_ro(cache, false);
 	if (!ret)
 		goto out;
 	if (ret == -ETXTBSY)
@@ -3134,7 +3203,7 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
 	if (ret < 0)
 		goto out;
 
-	ret = inc_block_group_ro(cache, 0);
+	ret = inc_block_group_ro(cache, false);
 	if (ret == -ETXTBSY)
 		goto unlock_out;
 out:
@@ -3192,6 +3261,11 @@ static int update_block_group_item(struct btrfs_trans_handle *trans,
 	u64 used, remap_bytes;
 	u32 identity_remap_count;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info, "missing block group root");
+		return -EUCLEAN;
+	}
+
 	/*
 	 * Block group items update can be triggered out of commit transaction
 	 * critical section, thus we need a consistent view of used bytes.
@@ -3269,9 +3343,9 @@ static int update_block_group_item(struct btrfs_trans_handle *trans,
 
 }
 
-static int cache_save_setup(struct btrfs_block_group *block_group,
-			    struct btrfs_trans_handle *trans,
-			    struct btrfs_path *path)
+static void cache_save_setup(struct btrfs_block_group *block_group,
+			     struct btrfs_trans_handle *trans,
+			     struct btrfs_path *path)
 {
 	struct btrfs_fs_info *fs_info = block_group->fs_info;
 	struct inode *inode = NULL;
@@ -3283,7 +3357,7 @@ static int cache_save_setup(struct btrfs_block_group *block_group,
 	int ret = 0;
 
 	if (!btrfs_test_opt(fs_info, SPACE_CACHE))
-		return 0;
+		return;
 
 	/*
 	 * If this block group is smaller than 100 megs don't bother caching the
@@ -3293,11 +3367,11 @@ static int cache_save_setup(struct btrfs_block_group *block_group,
 		spin_lock(&block_group->lock);
 		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
 		spin_unlock(&block_group->lock);
-		return 0;
+		return;
 	}
 
 	if (TRANS_ABORTED(trans))
-		return 0;
+		return;
 again:
 	inode = lookup_free_space_inode(block_group, path);
 	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
@@ -3378,10 +3452,8 @@ static int cache_save_setup(struct btrfs_block_group *block_group,
 	 * We hit an ENOSPC when setting up the cache in this transaction, just
 	 * skip doing the setup, we've already cleared the cache so we're safe.
 	 */
-	if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
-		ret = -ENOSPC;
+	if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags))
 		goto out_put;
-	}
 
 	/*
 	 * Try to preallocate enough space based on how big the block group is.
@@ -3429,7 +3501,6 @@ static int cache_save_setup(struct btrfs_block_group *block_group,
 	spin_unlock(&block_group->lock);
 
 	extent_changeset_free(data_reserved);
-	return ret;
 }
 
 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
@@ -4547,7 +4618,7 @@ static void check_removing_space_info(struct btrfs_space_info *space_info)
 		for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) {
 			if (space_info->sub_group[i]) {
 				check_removing_space_info(space_info->sub_group[i]);
-				kfree(space_info->sub_group[i]);
+				btrfs_sysfs_remove_space_info(space_info->sub_group[i]);
 				space_info->sub_group[i] = NULL;
 			}
 		}
diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h
index c03e042929002..0504cb3579921 100644
--- a/fs/btrfs/block-group.h
+++ b/fs/btrfs/block-group.h
@@ -350,6 +350,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 			     struct btrfs_chunk_map *map);
 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info);
 void btrfs_mark_bg_unused(struct btrfs_block_group *bg);
+void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit);
 void btrfs_reclaim_bgs_work(struct work_struct *work);
 void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info);
 void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg);
diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c
index 6064dd00d041b..9efb3016ef116 100644
--- a/fs/btrfs/block-rsv.c
+++ b/fs/btrfs/block-rsv.c
@@ -541,6 +541,31 @@ struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans,
 					   BTRFS_RESERVE_NO_FLUSH);
 	if (!ret)
 		return block_rsv;
+
+	/*
+	 * If we are being used for updating a log tree, fail immediately, which
+	 * makes the fsync fallback to a transaction commit.
+	 *
+	 * We don't want to consume from the global block reserve, as that is
+	 * precious space that may be needed to do updates to some trees for
+	 * which we don't reserve space during a transaction commit (update root
+	 * items in the root tree, device stat items in the device tree and
+	 * quota tree updates, see btrfs_init_root_block_rsv()), or to fallback
+	 * to in case we did not reserve enough space to run delayed items,
+	 * delayed references, or anything else we need in order to avoid a
+	 * transaction abort.
+	 *
+	 * We also don't want to do a reservation in flush emergency mode, as
+	 * we end up using metadata that could be critical to allow a
+	 * transaction to complete successfully and therefore increase the
+	 * chances for a transaction abort.
+	 *
+	 * Log trees are an optimization and should never consume from the
+	 * global reserve or be allowed overcommitting metadata.
+	 */
+	if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID)
+		return ERR_PTR(ret);
+
 	/*
 	 * If we couldn't reserve metadata bytes try and use some from
 	 * the global reserve if its space type is the same as the global
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 790518a8c8033..1c1a08820c9d9 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -8,7 +8,7 @@
 #include <linux/file.h>
 #include <linux/fs.h>
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/highmem.h>
 #include <linux/kthread.h>
 #include <linux/time.h>
@@ -224,7 +224,6 @@ void btrfs_free_compr_folio(struct folio *folio)
 		return;
 
 free:
-	ASSERT(folio_ref_count(folio) == 1);
 	folio_put(folio);
 }
 
@@ -292,7 +291,7 @@ static void end_bbio_compressed_write(struct btrfs_bio *bbio)
 	struct compressed_bio *cb = to_compressed_bio(bbio);
 	struct folio_iter fi;
 
-	btrfs_finish_ordered_extent(cb->bbio.ordered, NULL, cb->start, cb->len,
+	btrfs_finish_ordered_extent(cb->bbio.ordered, cb->start, cb->len,
 				    cb->bbio.bio.bi_status == BLK_STS_OK);
 
 	if (cb->writeback)
@@ -320,11 +319,16 @@ void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered,
 
 	ASSERT(IS_ALIGNED(ordered->file_offset, fs_info->sectorsize));
 	ASSERT(IS_ALIGNED(ordered->num_bytes, fs_info->sectorsize));
-	ASSERT(cb->writeback);
+	/*
+	 * This flag determines if we should clear the writeback flag from the
+	 * page cache. But this function is only utilized by encoded writes, it
+	 * never goes through the page cache.
+	 */
+	ASSERT(!cb->writeback);
 
 	cb->start = ordered->file_offset;
 	cb->len = ordered->num_bytes;
-	cb->compressed_len = ordered->disk_num_bytes;
+	ASSERT(cb->bbio.bio.bi_iter.bi_size == ordered->disk_num_bytes);
 	cb->bbio.bio.bi_iter.bi_sector = ordered->disk_bytenr >> SECTOR_SHIFT;
 	cb->bbio.ordered = ordered;
 
@@ -345,8 +349,7 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode,
 	cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE, end_bbio_compressed_write);
 	cb->start = start;
 	cb->len = len;
-	cb->writeback = true;
-
+	cb->writeback = false;
 	return cb;
 }
 
@@ -555,7 +558,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio)
 	em_start = em->start;
 
 	cb->len = bbio->bio.bi_iter.bi_size;
-	cb->compressed_len = compressed_len;
 	cb->compress_type = btrfs_extent_map_compression(em);
 	cb->orig_bbio = bbio;
 	cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root;
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
index 65b8bc4bbe0bf..973530e9ce6c2 100644
--- a/fs/btrfs/compression.h
+++ b/fs/btrfs/compression.h
@@ -36,6 +36,9 @@ struct btrfs_ordered_extent;
 #define BTRFS_MAX_COMPRESSED_PAGES	(BTRFS_MAX_COMPRESSED / PAGE_SIZE)
 static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0);
 
+/* The max size for a single worker to compress. */
+#define BTRFS_COMPRESSION_CHUNK_SIZE	(SZ_512K)
+
 /* Maximum size of data before compression */
 #define BTRFS_MAX_UNCOMPRESSED		(SZ_128K)
 
@@ -48,9 +51,6 @@ struct compressed_bio {
 	/* Number of bytes in the inode we're working on */
 	unsigned int len;
 
-	/* Number of bytes on disk */
-	unsigned int compressed_len;
-
 	/* The compression algorithm for this bio */
 	u8 compress_type;
 
diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 7267b25026657..cdd6c1422b53a 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -21,6 +21,7 @@
 #include "fs.h"
 #include "accessors.h"
 #include "extent-tree.h"
+#include "extent_io.h"
 #include "relocation.h"
 #include "file-item.h"
 
@@ -590,6 +591,9 @@ int btrfs_force_cow_block(struct btrfs_trans_handle *trans,
 		btrfs_tree_unlock(buf);
 	free_extent_buffer_stale(buf);
 	btrfs_mark_buffer_dirty(trans, cow);
+
+	btrfs_inhibit_eb_writeback(trans, cow);
+
 	*cow_ret = cow;
 	return 0;
 
@@ -599,9 +603,9 @@ int btrfs_force_cow_block(struct btrfs_trans_handle *trans,
 	return ret;
 }
 
-static inline bool should_cow_block(const struct btrfs_trans_handle *trans,
+static inline bool should_cow_block(struct btrfs_trans_handle *trans,
 				    const struct btrfs_root *root,
-				    const struct extent_buffer *buf)
+				    struct extent_buffer *buf)
 {
 	if (btrfs_is_testing(root->fs_info))
 		return false;
@@ -635,6 +639,7 @@ static inline bool should_cow_block(const struct btrfs_trans_handle *trans,
 	if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))
 		return true;
 
+	btrfs_inhibit_eb_writeback(trans, buf);
 	return false;
 }
 
@@ -822,7 +827,6 @@ struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent,
 {
 	int level = btrfs_header_level(parent);
 	struct btrfs_tree_parent_check check = { 0 };
-	struct extent_buffer *eb;
 
 	if (slot < 0 || slot >= btrfs_header_nritems(parent))
 		return ERR_PTR(-ENOENT);
@@ -835,16 +839,8 @@ struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent,
 	check.has_first_key = true;
 	btrfs_node_key_to_cpu(parent, &check.first_key, slot);
 
-	eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot),
-			     &check);
-	if (IS_ERR(eb))
-		return eb;
-	if (unlikely(!extent_buffer_uptodate(eb))) {
-		free_extent_buffer(eb);
-		return ERR_PTR(-EIO);
-	}
-
-	return eb;
+	return read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot),
+			       &check);
 }
 
 /*
@@ -1503,7 +1499,7 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
 			reada_for_search(fs_info, p, parent_level, slot, key->objectid);
 
 		/* first we do an atomic uptodate check */
-		if (btrfs_buffer_uptodate(tmp, check.transid, true) > 0) {
+		if (btrfs_buffer_uptodate(tmp, check.transid, true, NULL) > 0) {
 			/*
 			 * Do extra check for first_key, eb can be stale due to
 			 * being cached, read from scrub, or have multiple
@@ -2106,6 +2102,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 			    p->nodes[level + 1])) {
 				write_lock_level = level + 1;
 				btrfs_release_path(p);
+				trace_btrfs_search_slot_restart(root, level, "write_lock");
 				goto again;
 			}
 
@@ -2168,8 +2165,10 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		p->slots[level] = slot;
 		ret2 = setup_nodes_for_search(trans, root, p, b, level, ins_len,
 					      &write_lock_level);
-		if (ret2 == -EAGAIN)
+		if (ret2 == -EAGAIN) {
+			trace_btrfs_search_slot_restart(root, level, "setup_nodes");
 			goto again;
+		}
 		if (ret2) {
 			ret = ret2;
 			goto done;
@@ -2185,6 +2184,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		if (slot == 0 && ins_len && write_lock_level < level + 1) {
 			write_lock_level = level + 1;
 			btrfs_release_path(p);
+			trace_btrfs_search_slot_restart(root, level, "slot_zero");
 			goto again;
 		}
 
@@ -2198,8 +2198,10 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		}
 
 		ret2 = read_block_for_search(root, p, &b, slot, key);
-		if (ret2 == -EAGAIN && !p->nowait)
+		if (ret2 == -EAGAIN && !p->nowait) {
+			trace_btrfs_search_slot_restart(root, level, "read_block");
 			goto again;
+		}
 		if (ret2) {
 			ret = ret2;
 			goto done;
@@ -3896,7 +3898,7 @@ static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
 			goto err;
 	}
 
-	ret = split_leaf(trans, root, &key, path, ins_len, 1);
+	ret = split_leaf(trans, root, &key, path, ins_len, true);
 	if (ret)
 		goto err;
 
diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c
index 3766ff29fbbb1..605858c2d9a95 100644
--- a/fs/btrfs/delayed-ref.c
+++ b/fs/btrfs/delayed-ref.c
@@ -207,6 +207,30 @@ void btrfs_dec_delayed_refs_rsv_bg_updates(struct btrfs_fs_info *fs_info)
  * This will refill the delayed block_rsv up to 1 items size worth of space and
  * will return -ENOSPC if we can't make the reservation.
  */
+static int btrfs_zoned_cap_metadata_reservation(struct btrfs_space_info *space_info)
+{
+	struct btrfs_fs_info *fs_info = space_info->fs_info;
+	struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
+	u64 usable;
+	u64 cap;
+	int ret = 0;
+
+	if (!btrfs_is_zoned(fs_info))
+		return 0;
+
+	spin_lock(&space_info->lock);
+	usable = space_info->total_bytes - space_info->bytes_zone_unusable;
+	spin_unlock(&space_info->lock);
+	cap = usable >> 1;
+
+	spin_lock(&block_rsv->lock);
+	if (block_rsv->size > cap)
+		ret = -EAGAIN;
+	spin_unlock(&block_rsv->lock);
+
+	return ret;
+}
+
 int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
 				  enum btrfs_reserve_flush_enum flush)
 {
@@ -228,6 +252,10 @@ int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
 	if (!num_bytes)
 		return 0;
 
+	ret = btrfs_zoned_cap_metadata_reservation(space_info);
+	if (ret)
+		return ret;
+
 	ret = btrfs_reserve_metadata_bytes(space_info, num_bytes, flush);
 	if (ret)
 		return ret;
diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
index b6c7da8e1bc8b..8f8fa14886ded 100644
--- a/fs/btrfs/dev-replace.c
+++ b/fs/btrfs/dev-replace.c
@@ -697,7 +697,7 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
 	/* the disk copy procedure reuses the scrub code */
 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
 			      btrfs_device_get_total_bytes(src_device),
-			      &dev_replace->scrub_progress, 0, 1);
+			      &dev_replace->scrub_progress, false, true);
 
 	ret = btrfs_dev_replace_finishing(fs_info, ret);
 	if (ret == -EINPROGRESS)
@@ -1255,7 +1255,7 @@ static int btrfs_dev_replace_kthread(void *data)
 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
 			      dev_replace->committed_cursor_left,
 			      btrfs_device_get_total_bytes(dev_replace->srcdev),
-			      &dev_replace->scrub_progress, 0, 1);
+			      &dev_replace->scrub_progress, false, true);
 	ret = btrfs_dev_replace_finishing(fs_info, ret);
 	WARN_ON(ret && ret != -ECANCELED);
 
diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c
index 085a83ae9e62f..84f1c64423d32 100644
--- a/fs/btrfs/dir-item.c
+++ b/fs/btrfs/dir-item.c
@@ -253,9 +253,7 @@ int btrfs_check_dir_item_collision(struct btrfs_root *root, u64 dir_ino,
 		/* Nothing found, we're safe */
 		if (ret == -ENOENT)
 			return 0;
-
-		if (ret < 0)
-			return ret;
+		return ret;
 	}
 
 	/* we found an item, look for our name in the item */
diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c
index 9a63200d7a53d..57167d56dc727 100644
--- a/fs/btrfs/direct-io.c
+++ b/fs/btrfs/direct-io.c
@@ -107,7 +107,7 @@ static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
 				btrfs_start_ordered_extent(ordered);
 			else
-				ret = nowait ? -EAGAIN : -ENOTBLK;
+				ret = -ENOTBLK;
 			btrfs_put_ordered_extent(ordered);
 		} else {
 			/*
@@ -625,7 +625,7 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 		pos += submitted;
 		length -= submitted;
 		if (write)
-			btrfs_finish_ordered_extent(dio_data->ordered, NULL,
+			btrfs_finish_ordered_extent(dio_data->ordered,
 						    pos, length, false);
 		else
 			btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos,
@@ -657,9 +657,8 @@ static void btrfs_dio_end_io(struct btrfs_bio *bbio)
 	}
 
 	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
-		btrfs_finish_ordered_extent(bbio->ordered, NULL,
-					    dip->file_offset, dip->bytes,
-					    !bio->bi_status);
+		btrfs_finish_ordered_extent(bbio->ordered, dip->file_offset,
+					    dip->bytes, !bio->bi_status);
 	} else {
 		btrfs_unlock_dio_extent(&inode->io_tree, dip->file_offset,
 					dip->file_offset + dip->bytes - 1, NULL);
@@ -735,7 +734,7 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio,
 
 		ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);
 		if (ret) {
-			btrfs_finish_ordered_extent(dio_data->ordered, NULL,
+			btrfs_finish_ordered_extent(dio_data->ordered,
 						    file_offset, dip->bytes,
 						    !ret);
 			bio->bi_status = errno_to_blk_status(ret);
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 407830d86d0da..45e6dde602744 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -50,7 +50,6 @@
 #include "relocation.h"
 #include "scrub.h"
 #include "super.h"
-#include "delayed-inode.h"
 
 #define BTRFS_SUPER_FLAG_SUPP	(BTRFS_HEADER_FLAG_WRITTEN |\
 				 BTRFS_HEADER_FLAG_RELOC |\
@@ -110,19 +109,26 @@ static void csum_tree_block(struct extent_buffer *buf, u8 *result)
  * detect blocks that either didn't get written at all or got written
  * in the wrong place.
  */
-int btrfs_buffer_uptodate(struct extent_buffer *eb, u64 parent_transid, bool atomic)
+int btrfs_buffer_uptodate(struct extent_buffer *eb, u64 parent_transid, bool atomic,
+			  const struct btrfs_tree_parent_check *check)
 {
 	if (!extent_buffer_uptodate(eb))
 		return 0;
 
-	if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
+	if (!parent_transid || btrfs_header_generation(eb) == parent_transid) {
+		/*
+		 * On a cache hit, the caller may still need tree parent
+		 * verification before reusing the buffer.
+		 */
+		if (unlikely(check && btrfs_verify_level_key(eb, check)))
+			return -EUCLEAN;
 		return 1;
+	}
 
 	if (atomic)
 		return -EAGAIN;
 
-	if (!extent_buffer_uptodate(eb) ||
-	    btrfs_header_generation(eb) != parent_transid) {
+	if (btrfs_header_generation(eb) != parent_transid) {
 		btrfs_err_rl(eb->fs_info,
 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 			eb->start, eb->read_mirror,
@@ -730,7 +736,7 @@ void btrfs_global_root_delete(struct btrfs_root *root)
 }
 
 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
-				     struct btrfs_key *key)
+				     const struct btrfs_key *key)
 {
 	struct rb_node *node;
 	struct btrfs_root *root = NULL;
@@ -767,7 +773,7 @@ static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr)
 
 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
 {
-	struct btrfs_key key = {
+	const struct btrfs_key key = {
 		.objectid = BTRFS_CSUM_TREE_OBJECTID,
 		.type = BTRFS_ROOT_ITEM_KEY,
 		.offset = btrfs_global_root_id(fs_info, bytenr),
@@ -778,7 +784,7 @@ struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
 
 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr)
 {
-	struct btrfs_key key = {
+	const struct btrfs_key key = {
 		.objectid = BTRFS_EXTENT_TREE_OBJECTID,
 		.type = BTRFS_ROOT_ITEM_KEY,
 		.offset = btrfs_global_root_id(fs_info, bytenr),
@@ -994,8 +1000,11 @@ static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root,
 		root->node = NULL;
 		goto fail;
 	}
-	if (unlikely(!btrfs_buffer_uptodate(root->node, generation, false))) {
-		ret = -EIO;
+
+	ret = btrfs_buffer_uptodate(root->node, generation, false, &check);
+	if (unlikely(ret <= 0)) {
+		if (ret == 0)
+			ret = -EIO;
 		goto fail;
 	}
 
@@ -1591,7 +1600,7 @@ static int find_newest_super_backup(struct btrfs_fs_info *info)
  * this will bump the backup pointer by one when it is
  * done
  */
-static void backup_super_roots(struct btrfs_fs_info *info)
+static int backup_super_roots(struct btrfs_fs_info *info)
 {
 	const int next_backup = info->backup_root_index;
 	struct btrfs_root_backup *root_backup;
@@ -1623,6 +1632,15 @@ static void backup_super_roots(struct btrfs_fs_info *info)
 		struct btrfs_root *extent_root = btrfs_extent_root(info, 0);
 		struct btrfs_root *csum_root = btrfs_csum_root(info, 0);
 
+		if (unlikely(!extent_root)) {
+			btrfs_err(info, "missing extent root for extent at bytenr 0");
+			return -EUCLEAN;
+		}
+		if (unlikely(!csum_root)) {
+			btrfs_err(info, "missing csum root for extent at bytenr 0");
+			return -EUCLEAN;
+		}
+
 		btrfs_set_backup_extent_root(root_backup,
 					     extent_root->node->start);
 		btrfs_set_backup_extent_root_gen(root_backup,
@@ -1670,6 +1688,8 @@ static void backup_super_roots(struct btrfs_fs_info *info)
 	memcpy(&info->super_copy->super_roots,
 	       &info->super_for_commit->super_roots,
 	       sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
+
+	return 0;
 }
 
 /*
@@ -2014,11 +2034,6 @@ static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
 		btrfs_put_root(log_tree_root);
 		return ret;
 	}
-	if (unlikely(!extent_buffer_uptodate(log_tree_root->node))) {
-		btrfs_err(fs_info, "failed to read log tree");
-		btrfs_put_root(log_tree_root);
-		return -EIO;
-	}
 
 	/* returns with log_tree_root freed on success */
 	ret = btrfs_recover_log_trees(log_tree_root);
@@ -2288,6 +2303,15 @@ static int validate_sys_chunk_array(const struct btrfs_fs_info *fs_info,
 		return -EUCLEAN;
 	}
 
+	/* It must hold at least one key and one chunk. */
+	if (unlikely(sys_array_size < sizeof(struct btrfs_disk_key) +
+		     sizeof(struct btrfs_chunk))) {
+		btrfs_err(fs_info, "system chunk array too small %u < %zu",
+			  sys_array_size,
+			  sizeof(struct btrfs_disk_key) + sizeof(struct btrfs_chunk));
+		return -EUCLEAN;
+	}
+
 	while (cur < sys_array_size) {
 		struct btrfs_disk_key *disk_key;
 		struct btrfs_chunk *chunk;
@@ -2354,11 +2378,11 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 	int ret = 0;
 	const bool ignore_flags = btrfs_test_opt(fs_info, IGNORESUPERFLAGS);
 
-	if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
+	if (unlikely(btrfs_super_magic(sb) != BTRFS_MAGIC)) {
 		btrfs_err(fs_info, "no valid FS found");
 		ret = -EINVAL;
 	}
-	if ((btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP)) {
+	if (unlikely(btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP)) {
 		if (!ignore_flags) {
 			btrfs_err(fs_info,
 			"unrecognized or unsupported super flag 0x%llx",
@@ -2370,17 +2394,17 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 				   btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
 		}
 	}
-	if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
+	if (unlikely(btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL)) {
 		btrfs_err(fs_info, "tree_root level too big: %d >= %d",
 				btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
 		ret = -EINVAL;
 	}
-	if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
+	if (unlikely(btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL)) {
 		btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
 				btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
 		ret = -EINVAL;
 	}
-	if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
+	if (unlikely(btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL)) {
 		btrfs_err(fs_info, "log_root level too big: %d >= %d",
 				btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
 		ret = -EINVAL;
@@ -2390,65 +2414,65 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 	 * Check sectorsize and nodesize first, other check will need it.
 	 * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
 	 */
-	if (!is_power_of_2(sectorsize) || sectorsize < BTRFS_MIN_BLOCKSIZE ||
-	    sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
+	if (unlikely(!is_power_of_2(sectorsize) || sectorsize < BTRFS_MIN_BLOCKSIZE ||
+		     sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE)) {
 		btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
 		ret = -EINVAL;
 	}
 
-	if (!btrfs_supported_blocksize(sectorsize)) {
+	if (unlikely(!btrfs_supported_blocksize(sectorsize))) {
 		btrfs_err(fs_info,
 			"sectorsize %llu not yet supported for page size %lu",
 			sectorsize, PAGE_SIZE);
 		ret = -EINVAL;
 	}
 
-	if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
-	    nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
+	if (unlikely(!is_power_of_2(nodesize) || nodesize < sectorsize ||
+		     nodesize > BTRFS_MAX_METADATA_BLOCKSIZE)) {
 		btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
 		ret = -EINVAL;
 	}
-	if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
+	if (unlikely(nodesize != le32_to_cpu(sb->__unused_leafsize))) {
 		btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
 			  le32_to_cpu(sb->__unused_leafsize), nodesize);
 		ret = -EINVAL;
 	}
 
 	/* Root alignment check */
-	if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
+	if (unlikely(!IS_ALIGNED(btrfs_super_root(sb), sectorsize))) {
 		btrfs_err(fs_info, "tree_root block unaligned: %llu",
 			  btrfs_super_root(sb));
 		ret = -EINVAL;
 	}
-	if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
+	if (unlikely(!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize))) {
 		btrfs_err(fs_info, "chunk_root block unaligned: %llu",
 			   btrfs_super_chunk_root(sb));
 		ret = -EINVAL;
 	}
-	if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
+	if (unlikely(!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize))) {
 		btrfs_err(fs_info, "log_root block unaligned: %llu",
 			  btrfs_super_log_root(sb));
 		ret = -EINVAL;
 	}
 
-	if (!fs_info->fs_devices->temp_fsid &&
-	    memcmp(fs_info->fs_devices->fsid, sb->fsid, BTRFS_FSID_SIZE) != 0) {
+	if (unlikely(!fs_info->fs_devices->temp_fsid &&
+		     memcmp(fs_info->fs_devices->fsid, sb->fsid, BTRFS_FSID_SIZE) != 0)) {
 		btrfs_err(fs_info,
 		"superblock fsid doesn't match fsid of fs_devices: %pU != %pU",
 			  sb->fsid, fs_info->fs_devices->fsid);
 		ret = -EINVAL;
 	}
 
-	if (memcmp(fs_info->fs_devices->metadata_uuid, btrfs_sb_fsid_ptr(sb),
-		   BTRFS_FSID_SIZE) != 0) {
+	if (unlikely(memcmp(fs_info->fs_devices->metadata_uuid, btrfs_sb_fsid_ptr(sb),
+			    BTRFS_FSID_SIZE) != 0)) {
 		btrfs_err(fs_info,
 "superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU",
 			  btrfs_sb_fsid_ptr(sb), fs_info->fs_devices->metadata_uuid);
 		ret = -EINVAL;
 	}
 
-	if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
-		   BTRFS_FSID_SIZE) != 0) {
+	if (unlikely(memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
+			    BTRFS_FSID_SIZE) != 0)) {
 		btrfs_err(fs_info,
 			"dev_item UUID does not match metadata fsid: %pU != %pU",
 			fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
@@ -2459,9 +2483,9 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 	 * Artificial requirement for block-group-tree to force newer features
 	 * (free-space-tree, no-holes) so the test matrix is smaller.
 	 */
-	if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
-	    (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
-	     !btrfs_fs_incompat(fs_info, NO_HOLES))) {
+	if (unlikely(btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
+		     (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
+		      !btrfs_fs_incompat(fs_info, NO_HOLES)))) {
 		btrfs_err(fs_info,
 		"block-group-tree feature requires free-space-tree and no-holes");
 		ret = -EINVAL;
@@ -2472,25 +2496,25 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 		 * Reduce test matrix for remap tree by requiring block-group-tree
 		 * and no-holes. Free-space-tree is a hard requirement.
 		 */
-		if (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
-		    !btrfs_fs_incompat(fs_info, NO_HOLES) ||
-		    !btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) {
+		if (unlikely(!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
+			     !btrfs_fs_incompat(fs_info, NO_HOLES) ||
+			     !btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE))) {
 			btrfs_err(fs_info,
 "remap-tree feature requires free-space-tree, no-holes, and block-group-tree");
 			ret = -EINVAL;
 		}
 
-		if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
+		if (unlikely(btrfs_fs_incompat(fs_info, MIXED_GROUPS))) {
 			btrfs_err(fs_info, "remap-tree not supported with mixed-bg");
 			ret = -EINVAL;
 		}
 
-		if (btrfs_fs_incompat(fs_info, ZONED)) {
+		if (unlikely(btrfs_fs_incompat(fs_info, ZONED))) {
 			btrfs_err(fs_info, "remap-tree not supported with zoned devices");
 			ret = -EINVAL;
 		}
 
-		if (sectorsize > PAGE_SIZE) {
+		if (unlikely(sectorsize > PAGE_SIZE)) {
 			btrfs_err(fs_info, "remap-tree not supported when block size > page size");
 			ret = -EINVAL;
 		}
@@ -2500,66 +2524,47 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 	 * Hint to catch really bogus numbers, bitflips or so, more exact checks are
 	 * done later
 	 */
-	if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
+	if (unlikely(btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb))) {
 		btrfs_err(fs_info, "bytes_used is too small %llu",
 			  btrfs_super_bytes_used(sb));
 		ret = -EINVAL;
 	}
-	if (!is_power_of_2(btrfs_super_stripesize(sb))) {
+	if (unlikely(!is_power_of_2(btrfs_super_stripesize(sb)))) {
 		btrfs_err(fs_info, "invalid stripesize %u",
 			  btrfs_super_stripesize(sb));
 		ret = -EINVAL;
 	}
-	if (btrfs_super_num_devices(sb) > (1UL << 31))
+	if (unlikely(btrfs_super_num_devices(sb) > (1UL << 31)))
 		btrfs_warn(fs_info, "suspicious number of devices: %llu",
 			   btrfs_super_num_devices(sb));
-	if (btrfs_super_num_devices(sb) == 0) {
+	if (unlikely(btrfs_super_num_devices(sb) == 0)) {
 		btrfs_err(fs_info, "number of devices is 0");
 		ret = -EINVAL;
 	}
 
-	if (mirror_num >= 0 &&
-	    btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
-		btrfs_err(fs_info, "super offset mismatch %llu != %u",
-			  btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
+	if (unlikely(mirror_num >= 0 &&
+		     btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num))) {
+		btrfs_err(fs_info, "super offset mismatch %llu != %llu",
+			  btrfs_super_bytenr(sb), btrfs_sb_offset(mirror_num));
 		ret = -EINVAL;
 	}
 
-	if (ret)
+	if (unlikely(ret))
 		return ret;
 
 	ret = validate_sys_chunk_array(fs_info, sb);
 
-	/*
-	 * Obvious sys_chunk_array corruptions, it must hold at least one key
-	 * and one chunk
-	 */
-	if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
-		btrfs_err(fs_info, "system chunk array too big %u > %u",
-			  btrfs_super_sys_array_size(sb),
-			  BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
-		ret = -EINVAL;
-	}
-	if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
-			+ sizeof(struct btrfs_chunk)) {
-		btrfs_err(fs_info, "system chunk array too small %u < %zu",
-			  btrfs_super_sys_array_size(sb),
-			  sizeof(struct btrfs_disk_key)
-			  + sizeof(struct btrfs_chunk));
-		ret = -EINVAL;
-	}
-
 	/*
 	 * The generation is a global counter, we'll trust it more than the others
 	 * but it's still possible that it's the one that's wrong.
 	 */
-	if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
+	if (unlikely(btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb)))
 		btrfs_warn(fs_info,
 			"suspicious: generation < chunk_root_generation: %llu < %llu",
 			btrfs_super_generation(sb),
 			btrfs_super_chunk_root_generation(sb));
-	if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
-	    && btrfs_super_cache_generation(sb) != (u64)-1)
+	if (unlikely(btrfs_super_generation(sb) < btrfs_super_cache_generation(sb) &&
+		     btrfs_super_cache_generation(sb) != (u64)-1))
 		btrfs_warn(fs_info,
 			"suspicious: generation < cache_generation: %llu < %llu",
 			btrfs_super_generation(sb),
@@ -2590,7 +2595,7 @@ static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
 	int ret;
 
 	ret = btrfs_validate_super(fs_info, sb, -1);
-	if (ret < 0)
+	if (unlikely(ret < 0))
 		goto out;
 	if (unlikely(!btrfs_supported_super_csum(btrfs_super_csum_type(sb)))) {
 		ret = -EUCLEAN;
@@ -2607,7 +2612,7 @@ static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
 		goto out;
 	}
 out:
-	if (ret < 0)
+	if (unlikely(ret < 0))
 		btrfs_err(fs_info,
 		"super block corruption detected before writing it to disk");
 	return ret;
@@ -2628,11 +2633,6 @@ static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int lev
 		root->node = NULL;
 		return ret;
 	}
-	if (unlikely(!extent_buffer_uptodate(root->node))) {
-		free_extent_buffer(root->node);
-		root->node = NULL;
-		return -EIO;
-	}
 
 	btrfs_set_root_node(&root->root_item, root->node);
 	root->commit_root = btrfs_root_node(root);
@@ -3755,8 +3755,7 @@ static void btrfs_end_super_write(struct bio *bio)
  * Write superblock @sb to the @device. Do not wait for completion, all the
  * folios we use for writing are locked.
  *
- * Write @max_mirrors copies of the superblock, where 0 means default that fit
- * the expected device size at commit time. Note that max_mirrors must be
+ * Write @max_mirrors copies of the superblock. Note that max_mirrors must be
  * same for write and wait phases.
  *
  * Return number of errors when folio is not found or submission fails.
@@ -3772,9 +3771,6 @@ static int write_dev_supers(struct btrfs_device *device,
 
 	atomic_set(&device->sb_write_errors, 0);
 
-	if (max_mirrors == 0)
-		max_mirrors = BTRFS_SUPER_MIRROR_MAX;
-
 	for (i = 0; i < max_mirrors; i++) {
 		struct folio *folio;
 		struct bio *bio;
@@ -3859,16 +3855,13 @@ static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
 	int ret;
 	u64 bytenr;
 
-	if (max_mirrors == 0)
-		max_mirrors = BTRFS_SUPER_MIRROR_MAX;
-
 	for (i = 0; i < max_mirrors; i++) {
 		struct folio *folio;
 
 		ret = btrfs_sb_log_location(device, i, READ, &bytenr);
 		if (ret == -ENOENT) {
 			break;
-		} else if (ret < 0) {
+		} else if (unlikely(ret < 0)) {
 			errors++;
 			if (i == 0)
 				primary_failed = true;
@@ -3890,9 +3883,8 @@ static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
 	}
 
 	errors += atomic_read(&device->sb_write_errors);
-	if (errors >= BTRFS_SUPER_PRIMARY_WRITE_ERROR)
-		primary_failed = true;
-	if (primary_failed) {
+
+	if (unlikely(primary_failed || errors >= BTRFS_SUPER_PRIMARY_WRITE_ERROR)) {
 		btrfs_err(device->fs_info, "error writing primary super block to device %llu",
 			  device->devid);
 		return -1;
@@ -3943,7 +3935,7 @@ static bool wait_dev_flush(struct btrfs_device *device)
 
 	wait_for_completion_io(&device->flush_wait);
 
-	if (bio->bi_status) {
+	if (unlikely(bio->bi_status)) {
 		set_bit(BTRFS_DEV_STATE_FLUSH_FAILED, &device->dev_state);
 		btrfs_dev_stat_inc_and_print(device, BTRFS_DEV_STAT_FLUSH_ERRS);
 		return true;
@@ -3981,7 +3973,7 @@ static int barrier_all_devices(struct btrfs_fs_info *info)
 	list_for_each_entry(dev, head, dev_list) {
 		if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
 			continue;
-		if (!dev->bdev) {
+		if (unlikely(!dev->bdev)) {
 			errors_wait++;
 			continue;
 		}
@@ -3989,7 +3981,7 @@ static int barrier_all_devices(struct btrfs_fs_info *info)
 		    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
 			continue;
 
-		if (wait_dev_flush(dev))
+		if (unlikely(wait_dev_flush(dev)))
 			errors_wait++;
 	}
 
@@ -4032,27 +4024,31 @@ int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
 	return min_tolerated;
 }
 
-int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
+int write_all_supers(struct btrfs_trans_handle *trans)
 {
+	struct btrfs_fs_info *fs_info = trans->fs_info;
 	struct list_head *head;
 	struct btrfs_device *dev;
 	struct btrfs_super_block *sb;
 	struct btrfs_dev_item *dev_item;
+	int max_mirrors;
 	int ret;
 	int do_barriers;
 	int max_errors;
 	int total_errors = 0;
-	u64 flags;
 
 	do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
 
-	/*
-	 * max_mirrors == 0 indicates we're from commit_transaction,
-	 * not from fsync where the tree roots in fs_info have not
-	 * been consistent on disk.
-	 */
-	if (max_mirrors == 0)
-		backup_super_roots(fs_info);
+	if (trans->transaction->state < TRANS_STATE_UNBLOCKED) {
+		/* We are called from fsync. */
+		max_mirrors = 1;
+	} else {
+		/* We are called from transaction commit. */
+		max_mirrors = BTRFS_SUPER_MIRROR_MAX;
+		ret = backup_super_roots(fs_info);
+		if (ret < 0)
+			return ret;
+	}
 
 	sb = fs_info->super_for_commit;
 	dev_item = &sb->dev_item;
@@ -4063,17 +4059,19 @@ int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
 
 	if (do_barriers) {
 		ret = barrier_all_devices(fs_info);
-		if (ret) {
+		if (unlikely(ret)) {
 			mutex_unlock(
 				&fs_info->fs_devices->device_list_mutex);
-			btrfs_handle_fs_error(fs_info, ret,
-					      "errors while submitting device barriers.");
+			btrfs_abort_transaction(trans, ret);
+			btrfs_err(fs_info, "error while submitting device barriers");
 			return ret;
 		}
 	}
 
+	btrfs_set_super_flags(sb, btrfs_super_flags(sb) | BTRFS_HEADER_FLAG_WRITTEN);
+
 	list_for_each_entry(dev, head, dev_list) {
-		if (!dev->bdev) {
+		if (unlikely(!dev->bdev)) {
 			total_errors++;
 			continue;
 		}
@@ -4095,19 +4093,17 @@ int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
 		memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
 		       BTRFS_FSID_SIZE);
 
-		flags = btrfs_super_flags(sb);
-		btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
-
 		ret = btrfs_validate_write_super(fs_info, sb);
 		if (unlikely(ret < 0)) {
 			mutex_unlock(&fs_info->fs_devices->device_list_mutex);
-			btrfs_handle_fs_error(fs_info, -EUCLEAN,
-				"unexpected superblock corruption detected");
-			return -EUCLEAN;
+			btrfs_abort_transaction(trans, ret);
+			btrfs_err(fs_info,
+			  "unexpected superblock corruption before writing it");
+			return ret;
 		}
 
 		ret = write_dev_supers(dev, sb, max_mirrors);
-		if (ret)
+		if (unlikely(ret))
 			total_errors++;
 	}
 	if (unlikely(total_errors > max_errors)) {
@@ -4116,29 +4112,27 @@ int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
 
 		/* FUA is masked off if unsupported and can't be the reason */
-		btrfs_handle_fs_error(fs_info, -EIO,
-				      "%d errors while writing supers",
-				      total_errors);
+		btrfs_abort_transaction(trans, -EIO);
+		btrfs_err(fs_info, "%d errors while writing supers", total_errors);
 		return -EIO;
 	}
 
 	total_errors = 0;
 	list_for_each_entry(dev, head, dev_list) {
-		if (!dev->bdev)
+		if (unlikely(!dev->bdev))
 			continue;
 		if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
 		    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
 			continue;
 
 		ret = wait_dev_supers(dev, max_mirrors);
-		if (ret)
+		if (unlikely(ret))
 			total_errors++;
 	}
 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
 	if (unlikely(total_errors > max_errors)) {
-		btrfs_handle_fs_error(fs_info, -EIO,
-				      "%d errors while writing supers",
-				      total_errors);
+		btrfs_abort_transaction(trans, -EIO);
+		btrfs_err(fs_info, "%d errors while writing supers", total_errors);
 		return -EIO;
 	}
 	return 0;
diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h
index 5320da83d0cf8..343e332b17c0b 100644
--- a/fs/btrfs/disk-io.h
+++ b/fs/btrfs/disk-io.h
@@ -58,7 +58,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info);
 int btrfs_validate_super(const struct btrfs_fs_info *fs_info,
 			 const struct btrfs_super_block *sb, int mirror_num);
 int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount);
-int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors);
+int write_all_supers(struct btrfs_trans_handle *trans);
 int btrfs_commit_super(struct btrfs_fs_info *fs_info);
 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
 					const struct btrfs_key *key);
@@ -76,7 +76,7 @@ struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info,
 int btrfs_global_root_insert(struct btrfs_root *root);
 void btrfs_global_root_delete(struct btrfs_root *root);
 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
-				     struct btrfs_key *key);
+				     const struct btrfs_key *key);
 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr);
 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr);
 
@@ -107,7 +107,8 @@ static inline struct btrfs_root *btrfs_grab_root(struct btrfs_root *root)
 void btrfs_put_root(struct btrfs_root *root);
 void btrfs_mark_buffer_dirty(struct btrfs_trans_handle *trans,
 			     struct extent_buffer *buf);
-int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, bool atomic);
+int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, bool atomic,
+			  const struct btrfs_tree_parent_check *check);
 int btrfs_read_extent_buffer(struct extent_buffer *buf,
 			     const struct btrfs_tree_parent_check *check);
 
diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index b0d9baf5b4128..a0a0212c5f402 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -75,6 +75,12 @@ int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len)
 	struct btrfs_key key;
 	BTRFS_PATH_AUTO_FREE(path);
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu", start);
+		return -EUCLEAN;
+	}
+
 	path = btrfs_alloc_path();
 	if (!path)
 		return -ENOMEM;
@@ -131,6 +137,12 @@ int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
 	key.offset = offset;
 
 	extent_root = btrfs_extent_root(fs_info, bytenr);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
 	if (ret < 0)
 		return ret;
@@ -436,6 +448,12 @@ static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
 	int recow;
 	int ret;
 
+	if (unlikely(!root)) {
+		btrfs_err(trans->fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	if (parent) {
 		key.type = BTRFS_SHARED_DATA_REF_KEY;
@@ -510,6 +528,12 @@ static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
 	u32 num_refs;
 	int ret;
 
+	if (unlikely(!root)) {
+		btrfs_err(trans->fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	if (node->parent) {
 		key.type = BTRFS_SHARED_DATA_REF_KEY;
@@ -668,6 +692,12 @@ static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
 	struct btrfs_key key;
 	int ret;
 
+	if (unlikely(!root)) {
+		btrfs_err(trans->fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	if (parent) {
 		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
@@ -692,6 +722,12 @@ static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
 	struct btrfs_key key;
 	int ret;
 
+	if (unlikely(!root)) {
+		btrfs_err(trans->fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	if (node->parent) {
 		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
@@ -782,6 +818,12 @@ int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
 	bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
 	int needed;
 
+	if (unlikely(!root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	key.type = BTRFS_EXTENT_ITEM_KEY;
 	key.offset = num_bytes;
@@ -1680,6 +1722,12 @@ static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
 	}
 
 	root = btrfs_extent_root(fs_info, key.objectid);
+	if (unlikely(!root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  key.objectid);
+		return -EUCLEAN;
+	}
 again:
 	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
 	if (ret < 0) {
@@ -1926,8 +1974,15 @@ static int cleanup_ref_head(struct btrfs_trans_handle *trans,
 			struct btrfs_root *csum_root;
 
 			csum_root = btrfs_csum_root(fs_info, head->bytenr);
-			ret = btrfs_del_csums(trans, csum_root, head->bytenr,
-					      head->num_bytes);
+			if (unlikely(!csum_root)) {
+				btrfs_err(fs_info,
+					  "missing csum root for extent at bytenr %llu",
+					  head->bytenr);
+				ret = -EUCLEAN;
+			} else {
+				ret = btrfs_del_csums(trans, csum_root, head->bytenr,
+						      head->num_bytes);
+			}
 		}
 	}
 
@@ -2379,6 +2434,12 @@ static noinline int check_committed_ref(struct btrfs_inode *inode,
 	int type;
 	int ret;
 
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
+
 	key.objectid = bytenr;
 	key.type = BTRFS_EXTENT_ITEM_KEY;
 	key.offset = (u64)-1;
@@ -3093,6 +3154,15 @@ static int do_free_extent_accounting(struct btrfs_trans_handle *trans,
 		struct btrfs_root *csum_root;
 
 		csum_root = btrfs_csum_root(trans->fs_info, bytenr);
+		if (unlikely(!csum_root)) {
+			ret = -EUCLEAN;
+			btrfs_abort_transaction(trans, ret);
+			btrfs_err(trans->fs_info,
+				  "missing csum root for extent at bytenr %llu",
+				  bytenr);
+			return ret;
+		}
+
 		ret = btrfs_del_csums(trans, csum_root, bytenr, num_bytes);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
@@ -3222,7 +3292,11 @@ static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
 	u64 delayed_ref_root = href->owning_root;
 
 	extent_root = btrfs_extent_root(info, bytenr);
-	ASSERT(extent_root);
+	if (unlikely(!extent_root)) {
+		btrfs_err(info,
+			  "missing extent root for extent at bytenr %llu", bytenr);
+		return -EUCLEAN;
+	}
 
 	path = btrfs_alloc_path();
 	if (!path)
@@ -3939,9 +4013,8 @@ static int do_allocation_clustered(struct btrfs_block_group *block_group,
  * Lock nesting
  * ============
  *
- * space_info::lock
- *   block_group::lock
- *     fs_info::treelog_bg_lock
+ * block_group::lock
+ *   fs_info::treelog_bg_lock
  */
 
 /*
@@ -3954,7 +4027,6 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
 			       struct btrfs_block_group **bg_ret)
 {
 	struct btrfs_fs_info *fs_info = block_group->fs_info;
-	struct btrfs_space_info *space_info = block_group->space_info;
 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
 	u64 start = block_group->start;
 	u64 num_bytes = ffe_ctl->num_bytes;
@@ -4015,7 +4087,6 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
 		 */
 	}
 
-	spin_lock(&space_info->lock);
 	spin_lock(&block_group->lock);
 	spin_lock(&fs_info->treelog_bg_lock);
 	spin_lock(&fs_info->relocation_bg_lock);
@@ -4117,7 +4188,6 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group,
 	spin_unlock(&fs_info->relocation_bg_lock);
 	spin_unlock(&fs_info->treelog_bg_lock);
 	spin_unlock(&block_group->lock);
-	spin_unlock(&space_info->lock);
 	return ret;
 }
 
@@ -4939,11 +5009,18 @@ static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
 		size += btrfs_extent_inline_ref_size(BTRFS_EXTENT_OWNER_REF_KEY);
 	size += btrfs_extent_inline_ref_size(type);
 
+	extent_root = btrfs_extent_root(fs_info, ins->objectid);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  ins->objectid);
+		return -EUCLEAN;
+	}
+
 	path = btrfs_alloc_path();
 	if (!path)
 		return -ENOMEM;
 
-	extent_root = btrfs_extent_root(fs_info, ins->objectid);
 	ret = btrfs_insert_empty_item(trans, extent_root, path, ins, size);
 	if (ret) {
 		btrfs_free_path(path);
@@ -5019,11 +5096,18 @@ static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
 		size += sizeof(*block_info);
 	}
 
+	extent_root = btrfs_extent_root(fs_info, extent_key.objectid);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  extent_key.objectid);
+		return -EUCLEAN;
+	}
+
 	path = btrfs_alloc_path();
 	if (!path)
 		return -ENOMEM;
 
-	extent_root = btrfs_extent_root(fs_info, extent_key.objectid);
 	ret = btrfs_insert_empty_item(trans, extent_root, path, &extent_key,
 				      size);
 	if (ret) {
@@ -5696,7 +5780,7 @@ static int check_next_block_uptodate(struct btrfs_trans_handle *trans,
 
 	generation = btrfs_node_ptr_generation(path->nodes[level], path->slots[level]);
 
-	if (btrfs_buffer_uptodate(next, generation, false))
+	if (btrfs_buffer_uptodate(next, generation, false, NULL))
 		return 0;
 
 	check.level = level - 1;
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 5f97a3d2a8d72..fd78b92873880 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -11,9 +11,10 @@
 #include <linux/blkdev.h>
 #include <linux/swap.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/prefetch.h>
 #include <linux/fsverity.h>
+#include <linux/lockdep.h>
 #include "extent_io.h"
 #include "extent-io-tree.h"
 #include "extent_map.h"
@@ -520,7 +521,7 @@ static void end_bbio_data_write(struct btrfs_bio *bbio)
 	struct bio *bio = &bbio->bio;
 	int error = blk_status_to_errno(bio->bi_status);
 	struct folio_iter fi;
-	const u32 sectorsize = fs_info->sectorsize;
+	u32 bio_size = 0;
 
 	ASSERT(!bio_flagged(bio, BIO_CLONED));
 	bio_for_each_folio_all(fi, bio) {
@@ -528,23 +529,16 @@ static void end_bbio_data_write(struct btrfs_bio *bbio)
 		u64 start = folio_pos(folio) + fi.offset;
 		u32 len = fi.length;
 
-		/* Our read/write should always be sector aligned. */
-		if (!IS_ALIGNED(fi.offset, sectorsize))
-			btrfs_err(fs_info,
-		"partial page write in btrfs with offset %zu and length %zu",
-				  fi.offset, fi.length);
-		else if (!IS_ALIGNED(fi.length, sectorsize))
-			btrfs_info(fs_info,
-		"incomplete page write with offset %zu and length %zu",
-				   fi.offset, fi.length);
-
-		btrfs_finish_ordered_extent(bbio->ordered, folio, start, len,
-					    !error);
-		if (error)
-			mapping_set_error(folio->mapping, error);
+		bio_size += len;
+		ASSERT(btrfs_folio_test_ordered(fs_info, folio, start, len));
+		btrfs_folio_clear_ordered(fs_info, folio, start, len);
 		btrfs_folio_clear_writeback(fs_info, folio, start, len);
 	}
 
+	if (error)
+		mapping_set_error(bbio->inode->vfs_inode.i_mapping, error);
+
+	btrfs_finish_ordered_extent(bbio->ordered, bbio->file_offset, bio_size, !error);
 	bio_put(bio);
 }
 
@@ -1587,7 +1581,8 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 			u64 start = page_start + (start_bit << fs_info->sectorsize_bits);
 			u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits;
 
-			btrfs_mark_ordered_io_finished(inode, folio, start, len, false);
+			btrfs_folio_clear_ordered(fs_info, folio, start, len);
+			btrfs_mark_ordered_io_finished(inode, start, len, false);
 		}
 		return ret;
 	}
@@ -1663,6 +1658,7 @@ static int submit_one_sector(struct btrfs_inode *inode,
 		 * ordered extent.
 		 */
 		btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
+		btrfs_folio_clear_ordered(fs_info, folio, filepos, sectorsize);
 		btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
 		btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize);
 
@@ -1670,8 +1666,8 @@ static int submit_one_sector(struct btrfs_inode *inode,
 		 * Since there is no bio submitted to finish the ordered
 		 * extent, we have to manually finish this sector.
 		 */
-		btrfs_mark_ordered_io_finished(inode, folio, filepos,
-					       fs_info->sectorsize, false);
+		btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize,
+					       false);
 		return PTR_ERR(em);
 	}
 
@@ -1783,8 +1779,8 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 			spin_unlock(&inode->ordered_tree_lock);
 			btrfs_put_ordered_extent(ordered);
 
-			btrfs_mark_ordered_io_finished(inode, folio, cur,
-						       fs_info->sectorsize, true);
+			btrfs_folio_clear_ordered(fs_info, folio, cur, fs_info->sectorsize);
+			btrfs_mark_ordered_io_finished(inode, cur, fs_info->sectorsize, true);
 			/*
 			 * This range is beyond i_size, thus we don't need to
 			 * bother writing back.
@@ -1949,7 +1945,9 @@ static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *e
 	 * of time.
 	 */
 	spin_lock(&eb->refs_lock);
-	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
+	if ((wbc->sync_mode == WB_SYNC_ALL ||
+	     atomic_read(&eb->writeback_inhibitors) == 0) &&
+	    test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
 		XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
 		unsigned long flags;
 
@@ -2095,13 +2093,13 @@ static void buffer_tree_tag_for_writeback(struct btrfs_fs_info *fs_info,
 struct eb_batch {
 	unsigned int nr;
 	unsigned int cur;
-	struct extent_buffer *ebs[PAGEVEC_SIZE];
+	struct extent_buffer *ebs[FOLIO_BATCH_SIZE];
 };
 
 static inline bool eb_batch_add(struct eb_batch *batch, struct extent_buffer *eb)
 {
 	batch->ebs[batch->nr++] = eb;
-	return (batch->nr < PAGEVEC_SIZE);
+	return (batch->nr < FOLIO_BATCH_SIZE);
 }
 
 static inline void eb_batch_init(struct eb_batch *batch)
@@ -2396,38 +2394,12 @@ int btree_writepages(struct address_space *mapping, struct writeback_control *wb
 		index = 0;
 		goto retry;
 	}
+
 	/*
-	 * If something went wrong, don't allow any metadata write bio to be
-	 * submitted.
-	 *
-	 * This would prevent use-after-free if we had dirty pages not
-	 * cleaned up, which can still happen by fuzzed images.
-	 *
-	 * - Bad extent tree
-	 *   Allowing existing tree block to be allocated for other trees.
-	 *
-	 * - Log tree operations
-	 *   Exiting tree blocks get allocated to log tree, bumps its
-	 *   generation, then get cleaned in tree re-balance.
-	 *   Such tree block will not be written back, since it's clean,
-	 *   thus no WRITTEN flag set.
-	 *   And after log writes back, this tree block is not traced by
-	 *   any dirty extent_io_tree.
-	 *
-	 * - Offending tree block gets re-dirtied from its original owner
-	 *   Since it has bumped generation, no WRITTEN flag, it can be
-	 *   reused without COWing. This tree block will not be traced
-	 *   by btrfs_transaction::dirty_pages.
-	 *
-	 *   Now such dirty tree block will not be cleaned by any dirty
-	 *   extent io tree. Thus we don't want to submit such wild eb
-	 *   if the fs already has error.
-	 *
-	 * We can get ret > 0 from submit_extent_folio() indicating how many ebs
-	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
+	 * Only btrfs_check_meta_write_pointer() can update @ret,
+	 * and it only returns 0 or errors.
 	 */
-	if (ret > 0)
-		ret = 0;
+	ASSERT(ret <= 0);
 	if (!ret && BTRFS_FS_ERROR(fs_info))
 		ret = -EROFS;
 
@@ -2659,8 +2631,7 @@ void extent_write_locked_range(struct inode *inode, const struct folio *locked_f
 		if (IS_ERR(folio)) {
 			cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
 			cur_len = cur_end + 1 - cur;
-			btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
-						       cur, cur_len, false);
+			btrfs_mark_ordered_io_finished(BTRFS_I(inode), cur, cur_len, false);
 			mapping_set_error(mapping, PTR_ERR(folio));
 			cur = cur_end;
 			continue;
@@ -3011,6 +2982,64 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
 	kmem_cache_free(extent_buffer_cache, eb);
 }
 
+/*
+ * Inhibit writeback on buffer during transaction.
+ *
+ * @trans:  transaction handle that will own the inhibitor
+ * @eb:      extent buffer to inhibit writeback on
+ *
+ * Attempt to track this extent buffer in the transaction's inhibited set.  If
+ * memory allocation fails, the buffer is simply not tracked. It may be written
+ * back and need re-COW, which is the original behavior.  This is acceptable
+ * since inhibiting writeback is an optimization.
+ */
+void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_buffer *eb)
+{
+	unsigned long index = eb->start >> trans->fs_info->nodesize_bits;
+	void *old;
+
+	lockdep_assert_held(&eb->lock);
+	/* Check if already inhibited by this handle. */
+	old = xa_load(&trans->writeback_inhibited_ebs, index);
+	if (old == eb)
+		return;
+
+	/* Take reference for the xarray entry. */
+	refcount_inc(&eb->refs);
+
+	old = xa_store(&trans->writeback_inhibited_ebs, index, eb, GFP_NOFS);
+	if (xa_is_err(old)) {
+		/* Allocation failed, just skip inhibiting this buffer. */
+		free_extent_buffer(eb);
+		return;
+	}
+
+	/* Handle replacement of different eb at same index. */
+	if (old && old != eb) {
+		struct extent_buffer *old_eb = old;
+
+		atomic_dec(&old_eb->writeback_inhibitors);
+		free_extent_buffer(old_eb);
+	}
+
+	atomic_inc(&eb->writeback_inhibitors);
+}
+
+/*
+ * Uninhibit writeback on all extent buffers.
+ */
+void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans)
+{
+	struct extent_buffer *eb;
+	unsigned long index;
+
+	xa_for_each(&trans->writeback_inhibited_ebs, index, eb) {
+		atomic_dec(&eb->writeback_inhibitors);
+		free_extent_buffer(eb);
+	}
+	xa_destroy(&trans->writeback_inhibited_ebs);
+}
+
 static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info,
 						   u64 start)
 {
@@ -3021,6 +3050,7 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info
 	eb->len = fs_info->nodesize;
 	eb->fs_info = fs_info;
 	init_rwsem(&eb->lock);
+	atomic_set(&eb->writeback_inhibitors, 0);
 
 	btrfs_leak_debug_add_eb(eb);
 
@@ -3871,8 +3901,17 @@ int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
 	struct btrfs_fs_info *fs_info = eb->fs_info;
 	struct btrfs_bio *bbio;
 
-	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
+	if (extent_buffer_uptodate(eb)) {
+		int ret;
+
+		ret = btrfs_buffer_uptodate(eb, 0, true, check);
+		if (unlikely(ret <= 0)) {
+			if (ret == 0)
+				ret = -EIO;
+			return ret;
+		}
 		return 0;
+	}
 
 	/*
 	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
@@ -3892,8 +3931,16 @@ int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
 	 * started and finished reading the same eb.  In this case, UPTODATE
 	 * will now be set, and we shouldn't read it in again.
 	 */
-	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
+	if (unlikely(extent_buffer_uptodate(eb))) {
+		int ret;
+
 		clear_extent_buffer_reading(eb);
+		ret = btrfs_buffer_uptodate(eb, 0, true, check);
+		if (unlikely(ret <= 0)) {
+			if (ret == 0)
+				ret = -EIO;
+			return ret;
+		}
 		return 0;
 	}
 
@@ -3929,7 +3976,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num,
 		return ret;
 
 	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
-	if (unlikely(!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)))
+	if (unlikely(!extent_buffer_uptodate(eb)))
 		return -EIO;
 	return 0;
 }
@@ -4606,7 +4653,7 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
 	if (IS_ERR(eb))
 		return;
 
-	if (btrfs_buffer_uptodate(eb, gen, true)) {
+	if (btrfs_buffer_uptodate(eb, gen, true, NULL)) {
 		free_extent_buffer(eb);
 		return;
 	}
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index 8d05f1a58b7c3..a3b0ab5013617 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -99,6 +99,8 @@ struct extent_buffer {
 	spinlock_t refs_lock;
 	refcount_t refs;
 	int read_mirror;
+	/* Inhibit WB_SYNC_NONE writeback when > 0. */
+	atomic_t writeback_inhibitors;
 	/* >= 0 if eb belongs to a log tree, -1 otherwise */
 	s8 log_index;
 	u8 folio_shift;
@@ -298,7 +300,7 @@ static inline int __pure num_extent_folios(const struct extent_buffer *eb)
 	return num_extent_pages(eb);
 }
 
-static inline int extent_buffer_uptodate(const struct extent_buffer *eb)
+static inline bool extent_buffer_uptodate(const struct extent_buffer *eb)
 {
 	return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
 }
@@ -381,4 +383,8 @@ void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info);
 #define btrfs_extent_buffer_leak_debug_check(fs_info)	do {} while (0)
 #endif
 
+void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans,
+			       struct extent_buffer *eb);
+void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans);
+
 #endif
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 927324a2175aa..d72249390030f 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -308,6 +308,13 @@ static int search_csum_tree(struct btrfs_fs_info *fs_info,
 	/* Current item doesn't contain the desired range, search again */
 	btrfs_release_path(path);
 	csum_root = btrfs_csum_root(fs_info, disk_bytenr);
+	if (unlikely(!csum_root)) {
+		btrfs_err(fs_info,
+			  "missing csum root for extent at bytenr %llu",
+			  disk_bytenr);
+		return -EUCLEAN;
+	}
+
 	item = btrfs_lookup_csum(NULL, csum_root, path, disk_bytenr, 0);
 	if (IS_ERR(item)) {
 		ret = PTR_ERR(item);
@@ -1090,9 +1097,9 @@ static int find_next_csum_offset(struct btrfs_root *root,
 	return 0;
 }
 
-int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
-			   struct btrfs_root *root,
-			   struct btrfs_ordered_sum *sums)
+int btrfs_insert_data_csums(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root,
+			    struct btrfs_ordered_sum *sums)
 {
 	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct btrfs_key file_key;
diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h
index 5645c5e3abdb7..6c678787c7701 100644
--- a/fs/btrfs/file-item.h
+++ b/fs/btrfs/file-item.h
@@ -61,9 +61,9 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
 			     struct btrfs_root *root,
 			     struct btrfs_path *path, u64 objectid,
 			     u64 bytenr, int mod);
-int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
-			   struct btrfs_root *root,
-			   struct btrfs_ordered_sum *sums);
+int btrfs_insert_data_csums(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root,
+			    struct btrfs_ordered_sum *sums);
 int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async);
 int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio);
 int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end,
diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index a4cb9d3cfc4e8..06cc6e13cb621 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -3316,8 +3316,8 @@ static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end
 			*delalloc_start_ret = start;
 			delalloc_len = btrfs_count_range_bits(&inode->io_tree,
 							      delalloc_start_ret, end,
-							      len, EXTENT_DELALLOC, 1,
-							      cached_state);
+							      len, EXTENT_DELALLOC,
+							      true, cached_state);
 		} else {
 			spin_unlock(&inode->lock);
 		}
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index ecddfca92b2b5..9efd1ec90f031 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -1073,6 +1073,14 @@ static int populate_free_space_tree(struct btrfs_trans_handle *trans,
 	if (ret)
 		return ret;
 
+	extent_root = btrfs_extent_root(trans->fs_info, block_group->start);
+	if (unlikely(!extent_root)) {
+		btrfs_err(trans->fs_info,
+			  "missing extent root for block group at offset %llu",
+			  block_group->start);
+		return -EUCLEAN;
+	}
+
 	mutex_lock(&block_group->free_space_lock);
 
 	/*
@@ -1086,7 +1094,6 @@ static int populate_free_space_tree(struct btrfs_trans_handle *trans,
 	key.type = BTRFS_EXTENT_ITEM_KEY;
 	key.offset = 0;
 
-	extent_root = btrfs_extent_root(trans->fs_info, key.objectid);
 	ret = btrfs_search_slot_for_read(extent_root, &key, path, 1, 0);
 	if (ret < 0)
 		goto out_locked;
diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index 3de3b517810ed..a4758d94b32e9 100644
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -27,6 +27,7 @@
 #include <linux/sched.h>
 #include <linux/rbtree.h>
 #include <linux/xxhash.h>
+#include <linux/fserror.h>
 #include <uapi/linux/btrfs.h>
 #include <uapi/linux/btrfs_tree.h>
 #include "extent-io-tree.h"
@@ -966,13 +967,13 @@ struct btrfs_fs_info {
 #define inode_to_fs_info(_inode) (BTRFS_I(_Generic((_inode),			\
 					   struct inode *: (_inode)))->root->fs_info)
 
-static inline gfp_t btrfs_alloc_write_mask(struct address_space *mapping)
+static inline gfp_t btrfs_alloc_write_mask(const struct address_space *mapping)
 {
 	return mapping_gfp_constraint(mapping, ~__GFP_FS);
 }
 
 /* Return the minimal folio size of the fs. */
-static inline unsigned int btrfs_min_folio_size(struct btrfs_fs_info *fs_info)
+static inline unsigned int btrfs_min_folio_size(const struct btrfs_fs_info *fs_info)
 {
 	return 1U << (PAGE_SHIFT + fs_info->block_min_order);
 }
@@ -1199,8 +1200,10 @@ static inline void btrfs_force_shutdown(struct btrfs_fs_info *fs_info)
 	 * So here we only mark the fs error without flipping it RO.
 	 */
 	WRITE_ONCE(fs_info->fs_error, -EIO);
-	if (!test_and_set_bit(BTRFS_FS_STATE_EMERGENCY_SHUTDOWN, &fs_info->fs_state))
+	if (!test_and_set_bit(BTRFS_FS_STATE_EMERGENCY_SHUTDOWN, &fs_info->fs_state)) {
 		btrfs_crit(fs_info, "emergency shutdown");
+		fserror_report_shutdown(fs_info->sb, GFP_KERNEL);
+	}
 }
 
 /*
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index a6da98435ef7c..8d97a8ad3858b 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -74,7 +74,6 @@
 #include "delayed-inode.h"
 
 #define COW_FILE_RANGE_KEEP_LOCKED	(1UL << 0)
-#define COW_FILE_RANGE_NO_INLINE	(1UL << 1)
 
 struct btrfs_iget_args {
 	u64 ino;
@@ -424,7 +423,7 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,
 		folio_put(folio);
 	}
 
-	return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false);
+	return btrfs_mark_ordered_io_finished(inode, offset, bytes, false);
 }
 
 static int btrfs_dirty_inode(struct btrfs_inode *inode);
@@ -622,6 +621,10 @@ static bool can_cow_file_range_inline(struct btrfs_inode *inode,
  *
  * If being used directly, you must have already checked we're allowed to cow
  * the range by getting true from can_cow_file_range_inline().
+ *
+ * Return 0 if the inlined extent is created successfully.
+ * Return <0 for critical error, and should be considered as an writeback error.
+ * Return >0 if can not create an inlined extent (mostly due to lack of meta space).
  */
 static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
 					    u64 size, size_t compressed_size,
@@ -703,55 +706,6 @@ static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
 	return ret;
 }
 
-static noinline int cow_file_range_inline(struct btrfs_inode *inode,
-					  struct folio *locked_folio,
-					  u64 offset, u64 end,
-					  size_t compressed_size,
-					  int compress_type,
-					  struct folio *compressed_folio,
-					  bool update_i_size)
-{
-	struct extent_state *cached = NULL;
-	unsigned long clear_flags = EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
-		EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING | EXTENT_LOCKED;
-	u64 size = min_t(u64, i_size_read(&inode->vfs_inode), end + 1);
-	int ret;
-
-	if (!can_cow_file_range_inline(inode, offset, size, compressed_size))
-		return 1;
-
-	btrfs_lock_extent(&inode->io_tree, offset, end, &cached);
-	ret = __cow_file_range_inline(inode, size, compressed_size,
-				      compress_type, compressed_folio,
-				      update_i_size);
-	if (ret > 0) {
-		btrfs_unlock_extent(&inode->io_tree, offset, end, &cached);
-		return ret;
-	}
-
-	/*
-	 * In the successful case (ret == 0 here), cow_file_range will return 1.
-	 *
-	 * Quite a bit further up the callstack in extent_writepage(), ret == 1
-	 * is treated as a short circuited success and does not unlock the folio,
-	 * so we must do it here.
-	 *
-	 * In the failure case, the locked_folio does get unlocked by
-	 * btrfs_folio_end_all_writers, which asserts that it is still locked
-	 * at that point, so we must *not* unlock it here.
-	 *
-	 * The other two callsites in compress_file_range do not have a
-	 * locked_folio, so they are not relevant to this logic.
-	 */
-	if (ret == 0)
-		locked_folio = NULL;
-
-	extent_clear_unlock_delalloc(inode, offset, end, locked_folio, &cached,
-				     clear_flags, PAGE_UNLOCK |
-				     PAGE_START_WRITEBACK | PAGE_END_WRITEBACK);
-	return ret;
-}
-
 struct async_extent {
 	u64 start;
 	u64 ram_size;
@@ -797,7 +751,7 @@ static int add_async_extent(struct async_chunk *cow, u64 start, u64 ram_size,
  * options, defragmentation, properties or heuristics.
  */
 static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
-				      u64 end)
+				      u64 end, bool check_inline)
 {
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 
@@ -811,8 +765,10 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
 	 * do not even bother try compression, as there will be no space saving
 	 * and will always fallback to regular write later.
 	 */
-	if (start != 0 && end + 1 - start <= fs_info->sectorsize)
+	if (end + 1 - start <= fs_info->sectorsize &&
+	    (!check_inline || (start > 0 || end + 1 < inode->disk_i_size)))
 		return 0;
+
 	/* Defrag ioctl takes precedence over mount options and properties. */
 	if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS)
 		return 0;
@@ -890,28 +846,20 @@ static struct folio *compressed_bio_last_folio(struct compressed_bio *cb)
 	return page_folio(phys_to_page(paddr));
 }
 
-static void zero_last_folio(struct compressed_bio *cb)
-{
-	struct bio *bio = &cb->bbio.bio;
-	struct folio *last_folio = compressed_bio_last_folio(cb);
-	const u32 bio_size = bio->bi_iter.bi_size;
-	const u32 foffset = offset_in_folio(last_folio, bio_size);
-
-	folio_zero_range(last_folio, foffset, folio_size(last_folio) - foffset);
-}
-
 static void round_up_last_block(struct compressed_bio *cb, u32 blocksize)
 {
 	struct bio *bio = &cb->bbio.bio;
 	struct folio *last_folio = compressed_bio_last_folio(cb);
 	const u32 bio_size = bio->bi_iter.bi_size;
 	const u32 foffset = offset_in_folio(last_folio, bio_size);
+	const u32 padding_len = round_up(foffset, blocksize) - foffset;
 	bool ret;
 
 	if (IS_ALIGNED(bio_size, blocksize))
 		return;
 
-	ret = bio_add_folio(bio, last_folio, round_up(foffset, blocksize) - foffset, foffset);
+	folio_zero_range(last_folio, foffset, padding_len);
+	ret = bio_add_folio(bio, last_folio, padding_len, foffset);
 	/* The remaining part should be merged thus never fail. */
 	ASSERT(ret);
 }
@@ -935,9 +883,7 @@ static void compress_file_range(struct btrfs_work *work)
 		container_of(work, struct async_chunk, work);
 	struct btrfs_inode *inode = async_chunk->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	struct address_space *mapping = inode->vfs_inode.i_mapping;
 	struct compressed_bio *cb = NULL;
-	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
 	u64 blocksize = fs_info->sectorsize;
 	u64 start = async_chunk->start;
 	u64 end = async_chunk->end;
@@ -947,7 +893,6 @@ static void compress_file_range(struct btrfs_work *work)
 	int ret = 0;
 	unsigned long total_compressed = 0;
 	unsigned long total_in = 0;
-	unsigned int loff;
 	int compress_type = fs_info->compress_type;
 	int compress_level = fs_info->compress_level;
 
@@ -1009,7 +954,7 @@ static void compress_file_range(struct btrfs_work *work)
 	 * been flagged as NOCOMPRESS.  This flag can change at any time if we
 	 * discover bad compression ratios.
 	 */
-	if (!inode_need_compress(inode, start, end))
+	if (!inode_need_compress(inode, start, end, false))
 		goto cleanup_and_bail_uncompressed;
 
 	if (0 < inode->defrag_compress && inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
@@ -1030,43 +975,13 @@ static void compress_file_range(struct btrfs_work *work)
 	total_compressed = cb->bbio.bio.bi_iter.bi_size;
 	total_in = cur_len;
 
-	/*
-	 * Zero the tail end of the last folio, as we might be sending it down
-	 * to disk.
-	 */
-	loff = (total_compressed & (min_folio_size - 1));
-	if (loff)
-		zero_last_folio(cb);
-
-	/*
-	 * Try to create an inline extent.
-	 *
-	 * If we didn't compress the entire range, try to create an uncompressed
-	 * inline extent, else a compressed one.
-	 *
-	 * Check cow_file_range() for why we don't even try to create inline
-	 * extent for the subpage case.
-	 */
-	if (total_in < actual_end)
-		ret = cow_file_range_inline(inode, NULL, start, end, 0,
-					    BTRFS_COMPRESS_NONE, NULL, false);
-	else
-		ret = cow_file_range_inline(inode, NULL, start, end, total_compressed,
-					    compress_type,
-					    bio_first_folio_all(&cb->bbio.bio), false);
-	if (ret <= 0) {
-		cleanup_compressed_bio(cb);
-		if (ret < 0)
-			mapping_set_error(mapping, -EIO);
-		return;
-	}
-
 	/*
 	 * We aren't doing an inline extent. Round the compressed size up to a
 	 * block size boundary so the allocator does sane things.
 	 */
-	total_compressed = ALIGN(total_compressed, blocksize);
 	round_up_last_block(cb, blocksize);
+	total_compressed = cb->bbio.bio.bi_iter.bi_size;
+	ASSERT(IS_ALIGNED(total_compressed, blocksize));
 
 	/*
 	 * One last check to make sure the compression is really a win, compare
@@ -1437,11 +1352,6 @@ static int cow_one_range(struct btrfs_inode *inode, struct folio *locked_folio,
  *
  * When this function fails, it unlocks all folios except @locked_folio.
  *
- * When this function successfully creates an inline extent, it returns 1 and
- * unlocks all folios including locked_folio and starts I/O on them.
- * (In reality inline extents are limited to a single block, so locked_folio is
- * the only folio handled anyway).
- *
  * When this function succeed and creates a normal extent, the folio locking
  * status depends on the passed in flags:
  *
@@ -1485,25 +1395,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 	ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
 
 	inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
-
-	if (!(flags & COW_FILE_RANGE_NO_INLINE)) {
-		/* lets try to make an inline extent */
-		ret = cow_file_range_inline(inode, locked_folio, start, end, 0,
-					    BTRFS_COMPRESS_NONE, NULL, false);
-		if (ret <= 0) {
-			/*
-			 * We succeeded, return 1 so the caller knows we're done
-			 * with this page and already handled the IO.
-			 *
-			 * If there was an error then cow_file_range_inline() has
-			 * already done the cleanup.
-			 */
-			if (ret == 0)
-				ret = 1;
-			goto done;
-		}
-	}
-
 	alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);
 
 	/*
@@ -1581,7 +1472,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 	}
 	extent_clear_unlock_delalloc(inode, orig_start, end, locked_folio, &cached,
 				     EXTENT_LOCKED | EXTENT_DELALLOC, page_ops);
-done:
 	if (done_offset)
 		*done_offset = end;
 	return ret;
@@ -1701,7 +1591,7 @@ static bool run_delalloc_compressed(struct btrfs_inode *inode,
 	struct async_cow *ctx;
 	struct async_chunk *async_chunk;
 	unsigned long nr_pages;
-	u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);
+	u64 num_chunks = DIV_ROUND_UP(end - start, BTRFS_COMPRESSION_CHUNK_SIZE);
 	int i;
 	unsigned nofs_flag;
 	const blk_opf_t write_flags = wbc_to_write_flags(wbc);
@@ -1718,7 +1608,7 @@ static bool run_delalloc_compressed(struct btrfs_inode *inode,
 	atomic_set(&ctx->num_chunks, num_chunks);
 
 	for (i = 0; i < num_chunks; i++) {
-		u64 cur_end = min(end, start + SZ_512K - 1);
+		u64 cur_end = min(end, start + BTRFS_COMPRESSION_CHUNK_SIZE - 1);
 
 		/*
 		 * igrab is called higher up in the call chain, take only the
@@ -1853,7 +1743,7 @@ static int fallback_to_cow(struct btrfs_inode *inode,
 	 */
 	btrfs_lock_extent(io_tree, start, end, &cached_state);
 	count = btrfs_count_range_bits(io_tree, &range_start, end, range_bytes,
-				       EXTENT_NORESERVE, 0, NULL);
+				       EXTENT_NORESERVE, false, NULL);
 	if (count > 0 || is_space_ino || is_reloc_ino) {
 		u64 bytes = count;
 		struct btrfs_fs_info *fs_info = inode->root->fs_info;
@@ -1884,7 +1774,7 @@ static int fallback_to_cow(struct btrfs_inode *inode,
 	 * a locked folio, which can race with writeback.
 	 */
 	ret = cow_file_range(inode, locked_folio, start, end, NULL,
-			     COW_FILE_RANGE_NO_INLINE | COW_FILE_RANGE_KEEP_LOCKED);
+			     COW_FILE_RANGE_KEEP_LOCKED);
 	ASSERT(ret != 1);
 	return ret;
 }
@@ -1936,6 +1826,11 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 	int ret = 0;
 	bool nowait = path->nowait;
 
+	/* If there are pending snapshots for this root, we must do COW. */
+	if (args->writeback_path && !is_freespace_inode &&
+	    atomic_read(&root->snapshot_force_cow))
+		goto out;
+
 	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
 	extent_type = btrfs_file_extent_type(leaf, fi);
 
@@ -1997,11 +1892,6 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 		path = NULL;
 	}
 
-	/* If there are pending snapshots for this root, we must COW. */
-	if (args->writeback_path && !is_freespace_inode &&
-	    atomic_read(&root->snapshot_force_cow))
-		goto out;
-
 	args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start;
 	args->file_extent.offset += args->start - key->offset;
 	io_start = args->file_extent.disk_bytenr + args->file_extent.offset;
@@ -2012,6 +1902,13 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 	 */
 
 	csum_root = btrfs_csum_root(root->fs_info, io_start);
+	if (unlikely(!csum_root)) {
+		btrfs_err(root->fs_info,
+			  "missing csum root for extent at bytenr %llu", io_start);
+		ret = -EUCLEAN;
+		goto out;
+	}
+
 	ret = btrfs_lookup_csums_list(csum_root, io_start,
 				      io_start + args->file_extent.num_bytes - 1,
 				      NULL, nowait);
@@ -2428,6 +2325,91 @@ static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end)
 	return false;
 }
 
+/*
+ * Return 0 if an inlined extent is created successfully.
+ * Return <0 if critical error happened.
+ * Return >0 if an inline extent can not be created.
+ */
+static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_folio)
+{
+	struct btrfs_fs_info *fs_info = inode->root->fs_info;
+	struct compressed_bio *cb = NULL;
+	struct extent_state *cached = NULL;
+	const u64 i_size = i_size_read(&inode->vfs_inode);
+	const u32 blocksize = fs_info->sectorsize;
+	int compress_type = fs_info->compress_type;
+	int compress_level = fs_info->compress_level;
+	u32 compressed_size = 0;
+	int ret;
+
+	ASSERT(folio_pos(locked_folio) == 0);
+
+	if (btrfs_inode_can_compress(inode) &&
+	    inode_need_compress(inode, 0, blocksize, true)) {
+		if (inode->defrag_compress > 0 &&
+		    inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
+			compress_type = inode->defrag_compress;
+			compress_level = inode->defrag_compress_level;
+		} else if (inode->prop_compress) {
+			compress_type = inode->prop_compress;
+		}
+		cb = btrfs_compress_bio(inode, 0, blocksize, compress_type, compress_level, 0);
+		if (IS_ERR(cb)) {
+			cb = NULL;
+			/* Just fall back to non-compressed case. */
+		} else {
+			compressed_size = cb->bbio.bio.bi_iter.bi_size;
+		}
+	}
+	if (!can_cow_file_range_inline(inode, 0, i_size, compressed_size)) {
+		if (cb)
+			cleanup_compressed_bio(cb);
+		return 1;
+	}
+
+	btrfs_lock_extent(&inode->io_tree, 0, blocksize - 1, &cached);
+	if (cb) {
+		ret = __cow_file_range_inline(inode, i_size, compressed_size, compress_type,
+					      bio_first_folio_all(&cb->bbio.bio), false);
+		cleanup_compressed_bio(cb);
+		cb = NULL;
+	} else {
+		ret = __cow_file_range_inline(inode, i_size, 0, BTRFS_COMPRESS_NONE,
+					      NULL, false);
+	}
+	/*
+	 * We failed to insert inline extent due to lack of meta space.
+	 * Just unlock the extent io range and fallback to regular COW/NOCOW path.
+	 */
+	if (ret > 0) {
+		btrfs_unlock_extent(&inode->io_tree, 0, blocksize - 1, &cached);
+		return ret;
+	}
+
+	/*
+	 * In the successful case (ret == 0 here), btrfs_run_delalloc_range()
+	 * will return 1.
+	 *
+	 * Quite a bit further up the callstack in extent_writepage(), ret == 1
+	 * is treated as a short circuited success and does not unlock the folio,
+	 * so we must do it here.
+	 *
+	 * For failure case, the @locked_folio does get unlocked by
+	 * btrfs_folio_end_lock_bitmap(), so we must *not* unlock it here.
+	 *
+	 * So if ret == 0, we let extent_clear_unlock_delalloc() to unlock the
+	 * folio by passing NULL as @locked_folio.
+	 * Otherwise pass @locked_folio as usual.
+	 */
+	if (ret == 0)
+		locked_folio = NULL;
+	extent_clear_unlock_delalloc(inode, 0, blocksize - 1, locked_folio, &cached,
+				     EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
+				     EXTENT_DO_ACCOUNTING | EXTENT_LOCKED,
+				     PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK);
+	return ret;
+}
+
 /*
  * Function to process delayed allocation (create CoW) for ranges which are
  * being touched for the first time.
@@ -2444,11 +2426,26 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol
 	ASSERT(!(end <= folio_pos(locked_folio) ||
 		 start >= folio_next_pos(locked_folio)));
 
+	if (start == 0 && end + 1 <= inode->root->fs_info->sectorsize &&
+	    end + 1 >= inode->disk_i_size) {
+		int ret;
+
+		ret = run_delalloc_inline(inode, locked_folio);
+		if (ret < 0)
+			return ret;
+		if (ret == 0)
+			return 1;
+		/*
+		 * Continue regular handling if we can not create an
+		 * inlined extent.
+		 */
+	}
+
 	if (should_nocow(inode, start, end))
 		return run_delalloc_nocow(inode, locked_folio, start, end);
 
 	if (btrfs_inode_can_compress(inode) &&
-	    inode_need_compress(inode, start, end) &&
+	    inode_need_compress(inode, start, end, false) &&
 	    run_delalloc_compressed(inode, locked_folio, start, end, wbc))
 		return 1;
 
@@ -2738,22 +2735,31 @@ void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
 }
 
 /*
- * given a list of ordered sums record them in the inode.  This happens
- * at IO completion time based on sums calculated at bio submission time.
+ * Given an ordered extent and insert all its checksums into the csum tree.
+ *
+ * This happens at IO completion time based on sums calculated at bio
+ * submission time.
  */
 static int add_pending_csums(struct btrfs_trans_handle *trans,
-			     struct list_head *list)
+			     struct btrfs_ordered_extent *oe)
 {
 	struct btrfs_ordered_sum *sum;
 	struct btrfs_root *csum_root = NULL;
 	int ret;
 
-	list_for_each_entry(sum, list, list) {
-		trans->adding_csums = true;
-		if (!csum_root)
+	list_for_each_entry(sum, &oe->csum_list, list) {
+		if (!csum_root) {
 			csum_root = btrfs_csum_root(trans->fs_info,
 						    sum->logical);
-		ret = btrfs_csum_file_blocks(trans, csum_root, sum);
+			if (unlikely(!csum_root)) {
+				btrfs_err(trans->fs_info,
+				  "missing csum root for extent at bytenr %llu",
+					  sum->logical);
+				return -EUCLEAN;
+			}
+		}
+		trans->adding_csums = true;
+		ret = btrfs_insert_data_csums(trans, csum_root, sum);
 		trans->adding_csums = false;
 		if (ret)
 			return ret;
@@ -2942,7 +2948,9 @@ static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
 		 * to reflect the errors and clean the page.
 		 */
 		mapping_set_error(folio->mapping, ret);
-		btrfs_mark_ordered_io_finished(inode, folio, page_start,
+		btrfs_folio_clear_ordered(fs_info, folio, page_start,
+					  folio_size(folio));
+		btrfs_mark_ordered_io_finished(inode, page_start,
 					       folio_size(folio), !ret);
 		folio_clear_dirty_for_io(folio);
 	}
@@ -3257,8 +3265,8 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 
 	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
 		/* Logic error */
-		ASSERT(list_empty(&ordered_extent->list));
-		if (unlikely(!list_empty(&ordered_extent->list))) {
+		ASSERT(list_empty(&ordered_extent->csum_list));
+		if (unlikely(!list_empty(&ordered_extent->csum_list))) {
 			ret = -EINVAL;
 			btrfs_abort_transaction(trans, ret);
 			goto out;
@@ -3307,7 +3315,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 		goto out;
 	}
 
-	ret = add_pending_csums(trans, &ordered_extent->list);
+	ret = add_pending_csums(trans, ordered_extent);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out;
@@ -3413,7 +3421,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 	 * This needs to be done to make sure anybody waiting knows we are done
 	 * updating everything for this ordered extent.
 	 */
-	btrfs_remove_ordered_extent(inode, ordered_extent);
+	btrfs_remove_ordered_extent(ordered_extent);
 
 	/* once for us */
 	btrfs_put_ordered_extent(ordered_extent);
@@ -4683,7 +4691,7 @@ static noinline int may_destroy_subvol(struct btrfs_root *root)
 	dir_id = btrfs_super_root_dir(fs_info->super_copy);
 	di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,
 				   dir_id, &name, 0);
-	if (di && !IS_ERR(di)) {
+	if (!IS_ERR_OR_NULL(di)) {
 		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
 		if (key.objectid == btrfs_root_id(root)) {
 			ret = -EPERM;
@@ -6845,7 +6853,7 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
 		}
 	} else {
 		ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
-				     0, BTRFS_I(inode)->dir_index);
+				     false, BTRFS_I(inode)->dir_index);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			goto discard;
@@ -7061,7 +7069,7 @@ static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
 	inode_set_ctime_current(inode);
 
 	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
-			     &fname.disk_name, 1, index);
+			     &fname.disk_name, true, index);
 	if (ret)
 		goto fail;
 
@@ -8159,7 +8167,7 @@ void btrfs_destroy_inode(struct inode *vfs_inode)
 			if (!freespace_inode)
 				btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
 
-			btrfs_remove_ordered_extent(inode, ordered);
+			btrfs_remove_ordered_extent(ordered);
 			btrfs_put_ordered_extent(ordered);
 			btrfs_put_ordered_extent(ordered);
 		}
@@ -8481,14 +8489,14 @@ static int btrfs_rename_exchange(struct inode *old_dir,
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
-			     new_name, 0, old_idx);
+			     new_name, false, old_idx);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
-			     old_name, 0, new_idx);
+			     old_name, false, new_idx);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
@@ -8779,7 +8787,7 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
-			     &new_fname.disk_name, 0, index);
+			     &new_fname.disk_name, false, index);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
@@ -9874,6 +9882,7 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
 	int compression;
 	size_t orig_count;
 	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
+	const u32 blocksize = fs_info->sectorsize;
 	u64 start, end;
 	u64 num_bytes, ram_bytes, disk_num_bytes;
 	struct btrfs_key ins;
@@ -9984,9 +9993,9 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
 			ret = -EFAULT;
 			goto out_cb;
 		}
-		if (bytes < min_folio_size)
-			folio_zero_range(folio, bytes, min_folio_size - bytes);
-		ret = bio_add_folio(&cb->bbio.bio, folio, folio_size(folio), 0);
+		if (!IS_ALIGNED(bytes, blocksize))
+			folio_zero_range(folio, bytes, round_up(bytes, blocksize) - bytes);
+		ret = bio_add_folio(&cb->bbio.bio, folio, round_up(bytes, blocksize), 0);
 		if (unlikely(!ret)) {
 			folio_put(folio);
 			ret = -EINVAL;
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index b805dd9227efa..a4d715bbed57b 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -3038,7 +3038,7 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg)
 
 	ret = btrfs_scrub_dev(fs_info, sa->devid, sa->start, sa->end,
 			      &sa->progress, sa->flags & BTRFS_SCRUB_READONLY,
-			      0);
+			      false);
 
 	/*
 	 * Copy scrub args to user space even if btrfs_scrub_dev() returned an
@@ -3617,7 +3617,8 @@ static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg)
 		}
 	}
 
-	trans = btrfs_join_transaction(root);
+	/* 2 BTRFS_QGROUP_RELATION_KEY items. */
+	trans = btrfs_start_transaction(root, 2);
 	if (IS_ERR(trans)) {
 		ret = PTR_ERR(trans);
 		goto out;
@@ -3689,7 +3690,11 @@ static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg)
 		goto out;
 	}
 
-	trans = btrfs_join_transaction(root);
+	/*
+	 * 1 BTRFS_QGROUP_INFO_KEY item.
+	 * 1 BTRFS_QGROUP_LIMIT_KEY item.
+	 */
+	trans = btrfs_start_transaction(root, 2);
 	if (IS_ERR(trans)) {
 		ret = PTR_ERR(trans);
 		goto out;
@@ -3738,7 +3743,8 @@ static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg)
 		goto drop_write;
 	}
 
-	trans = btrfs_join_transaction(root);
+	/* 1 BTRFS_QGROUP_LIMIT_KEY item. */
+	trans = btrfs_start_transaction(root, 1);
 	if (IS_ERR(trans)) {
 		ret = PTR_ERR(trans);
 		goto out;
@@ -3922,7 +3928,7 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file,
 		ret = btrfs_uuid_tree_add(trans, sa->uuid,
 					  BTRFS_UUID_KEY_RECEIVED_SUBVOL,
 					  btrfs_root_id(root));
-		if (unlikely(ret < 0 && ret != -EEXIST)) {
+		if (unlikely(ret < 0)) {
 			btrfs_abort_transaction(trans, ret);
 			btrfs_end_transaction(trans);
 			goto out;
diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c
index 049a940ba4499..0c90937707395 100644
--- a/fs/btrfs/lzo.c
+++ b/fs/btrfs/lzo.c
@@ -429,8 +429,9 @@ static void copy_compressed_segment(struct compressed_bio *cb,
 int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 {
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
-	const struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info;
+	struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info;
 	const u32 sectorsize = fs_info->sectorsize;
+	const u32 compressed_len = bio_get_size(&cb->bbio.bio);
 	struct folio_iter fi;
 	char *kaddr;
 	int ret;
@@ -447,7 +448,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	/* There must be a compressed folio and matches the sectorsize. */
 	if (unlikely(!fi.folio))
 		return -EINVAL;
-	ASSERT(folio_size(fi.folio) == sectorsize);
+	ASSERT(folio_size(fi.folio) == btrfs_min_folio_size(fs_info));
 	kaddr = kmap_local_folio(fi.folio, 0);
 	len_in = read_compress_length(kaddr);
 	kunmap_local(kaddr);
@@ -460,14 +461,14 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	 * and all sectors should be used.
 	 * If this happens, it means the compressed extent is corrupted.
 	 */
-	if (unlikely(len_in > min_t(size_t, BTRFS_MAX_COMPRESSED, cb->compressed_len) ||
-		     round_up(len_in, sectorsize) < cb->compressed_len)) {
+	if (unlikely(len_in > min_t(size_t, BTRFS_MAX_COMPRESSED, compressed_len) ||
+		     round_up(len_in, sectorsize) < compressed_len)) {
 		struct btrfs_inode *inode = cb->bbio.inode;
 
 		btrfs_err(fs_info,
 "lzo header invalid, root %llu inode %llu offset %llu lzo len %u compressed len %u",
 			  btrfs_root_id(inode->root), btrfs_ino(inode),
-			  cb->start, len_in, cb->compressed_len);
+			  cb->start, len_in, compressed_len);
 		return -EUCLEAN;
 	}
 
diff --git a/fs/btrfs/messages.h b/fs/btrfs/messages.h
index c8e92efce4058..556d4e79cde60 100644
--- a/fs/btrfs/messages.h
+++ b/fs/btrfs/messages.h
@@ -144,11 +144,11 @@ do {										\
 	verify_assert_printk_format("check the format string" args);		\
 	if (!likely(cond)) {							\
 		if (("" __FIRST_ARG(args) [0]) == 0) {				\
-			pr_err("assertion failed: %s :: %ld, in %s:%d\n",	\
-				#cond, (long)(cond), __FILE__, __LINE__);	\
+			pr_err("assertion failed: %s, in %s:%d\n",		\
+				#cond, __FILE__, __LINE__);			\
 		} else {							\
-			pr_err("assertion failed: %s :: %ld, in %s:%d (" __FIRST_ARG(args) ")\n", \
-				#cond, (long)(cond), __FILE__, __LINE__ __REST_ARGS(args)); \
+			pr_err("assertion failed: %s, in %s:%d (" __FIRST_ARG(args) ")\n", \
+				#cond, __FILE__, __LINE__ __REST_ARGS(args));	\
 		}								\
 		BUG();								\
 	}									\
diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h
index 12c5a9d6564f9..694be6d0562a4 100644
--- a/fs/btrfs/misc.h
+++ b/fs/btrfs/misc.h
@@ -28,7 +28,8 @@
 	name = (1U << __ ## name ## _BIT),              \
 	__ ## name ## _SEQ = __ ## name ## _BIT
 
-static inline phys_addr_t bio_iter_phys(struct bio *bio, struct bvec_iter *iter)
+static inline phys_addr_t bio_iter_phys(const struct bio *bio,
+					const struct bvec_iter *iter)
 {
 	struct bio_vec bv = bio_iter_iovec(bio, *iter);
 
@@ -52,15 +53,22 @@ static inline phys_addr_t bio_iter_phys(struct bio *bio, struct bvec_iter *iter)
 	     (paddr = bio_iter_phys((bio), (iter)), 1);			\
 	     bio_advance_iter_single((bio), (iter), (blocksize)))
 
-/* Initialize a bvec_iter to the size of the specified bio. */
-static inline struct bvec_iter init_bvec_iter_for_bio(struct bio *bio)
+/* Can only be called on a non-cloned bio. */
+static inline u32 bio_get_size(struct bio *bio)
 {
 	struct bio_vec *bvec;
-	u32 bio_size = 0;
+	u32 ret = 0;
 	int i;
 
 	bio_for_each_bvec_all(bvec, bio, i)
-		bio_size += bvec->bv_len;
+		ret += bvec->bv_len;
+	return ret;
+}
+
+/* Initialize a bvec_iter to the size of the specified bio. */
+static inline struct bvec_iter init_bvec_iter_for_bio(struct bio *bio)
+{
+	const u32 bio_size = bio_get_size(bio);
 
 	return (struct bvec_iter) {
 		.bi_sector = 0,
diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index 5df02c707aee6..d39f1c49d1cf8 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -156,6 +156,19 @@ static struct btrfs_ordered_extent *alloc_ordered_extent(
 	const bool is_nocow = (flags &
 	       ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC)));
 
+	/* Only one type flag can be set. */
+	ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS));
+
+	/* DIRECT cannot be set with COMPRESSED nor ENCODED. */
+	if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) {
+		ASSERT(!test_bit(BTRFS_ORDERED_COMPRESSED, &flags));
+		ASSERT(!test_bit(BTRFS_ORDERED_ENCODED, &flags));
+	}
+
+	/* ENCODED must be set with COMPRESSED. */
+	if (test_bit(BTRFS_ORDERED_ENCODED, &flags))
+		ASSERT(test_bit(BTRFS_ORDERED_COMPRESSED, &flags));
+
 	/*
 	 * For a NOCOW write we can free the qgroup reserve right now. For a COW
 	 * one we transfer the reserved space from the inode's iotree into the
@@ -197,7 +210,7 @@ static struct btrfs_ordered_extent *alloc_ordered_extent(
 	entry->flags = flags;
 	refcount_set(&entry->refs, 1);
 	init_waitqueue_head(&entry->wait);
-	INIT_LIST_HEAD(&entry->list);
+	INIT_LIST_HEAD(&entry->csum_list);
 	INIT_LIST_HEAD(&entry->log_list);
 	INIT_LIST_HEAD(&entry->root_extent_list);
 	INIT_LIST_HEAD(&entry->work_list);
@@ -240,10 +253,15 @@ static void insert_ordered_extent(struct btrfs_ordered_extent *entry)
 	spin_lock(&inode->ordered_tree_lock);
 	node = tree_insert(&inode->ordered_tree, entry->file_offset,
 			   &entry->rb_node);
-	if (unlikely(node))
+	if (unlikely(node)) {
+		struct btrfs_ordered_extent *exist =
+			rb_entry(node, struct btrfs_ordered_extent, rb_node);
+
 		btrfs_panic(fs_info, -EEXIST,
-				"inconsistency in ordered tree at offset %llu",
-				entry->file_offset);
+"overlapping ordered extents, existing oe file_offset %llu num_bytes %llu flags 0x%lx, new oe file_offset %llu num_bytes %llu flags 0x%lx",
+			    exist->file_offset, exist->num_bytes, exist->flags,
+			    entry->file_offset, entry->num_bytes, entry->flags);
+	}
 	spin_unlock(&inode->ordered_tree_lock);
 
 	spin_lock(&root->ordered_extent_lock);
@@ -329,7 +347,7 @@ void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry,
 	struct btrfs_inode *inode = entry->inode;
 
 	spin_lock(&inode->ordered_tree_lock);
-	list_add_tail(&sum->list, &entry->list);
+	list_add_tail(&sum->list, &entry->csum_list);
 	spin_unlock(&inode->ordered_tree_lock);
 }
 
@@ -348,30 +366,13 @@ static void finish_ordered_fn(struct btrfs_work *work)
 }
 
 static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
-				      struct folio *folio, u64 file_offset,
-				      u64 len, bool uptodate)
+				      u64 file_offset, u64 len, bool uptodate)
 {
 	struct btrfs_inode *inode = ordered->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 
 	lockdep_assert_held(&inode->ordered_tree_lock);
 
-	if (folio) {
-		ASSERT(folio->mapping);
-		ASSERT(folio_pos(folio) <= file_offset);
-		ASSERT(file_offset + len <= folio_next_pos(folio));
-
-		/*
-		 * Ordered flag indicates whether we still have
-		 * pending io unfinished for the ordered extent.
-		 *
-		 * If it's not set, we need to skip to next range.
-		 */
-		if (!btrfs_folio_test_ordered(fs_info, folio, file_offset, len))
-			return false;
-		btrfs_folio_clear_ordered(fs_info, folio, file_offset, len);
-	}
-
 	/* Now we're fine to update the accounting. */
 	if (WARN_ON_ONCE(len > ordered->bytes_left)) {
 		btrfs_crit(fs_info,
@@ -413,8 +414,7 @@ static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered)
 }
 
 void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
-				 struct folio *folio, u64 file_offset, u64 len,
-				 bool uptodate)
+				 u64 file_offset, u64 len, bool uptodate)
 {
 	struct btrfs_inode *inode = ordered->inode;
 	bool ret;
@@ -422,7 +422,7 @@ void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
 	trace_btrfs_finish_ordered_extent(inode, file_offset, len, uptodate);
 
 	spin_lock(&inode->ordered_tree_lock);
-	ret = can_finish_ordered_extent(ordered, folio, file_offset, len,
+	ret = can_finish_ordered_extent(ordered, file_offset, len,
 					uptodate);
 	spin_unlock(&inode->ordered_tree_lock);
 
@@ -475,8 +475,7 @@ void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
  * extent(s) covering it.
  */
 void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
-				    struct folio *folio, u64 file_offset,
-				    u64 num_bytes, bool uptodate)
+				    u64 file_offset, u64 num_bytes, bool uptodate)
 {
 	struct rb_node *node;
 	struct btrfs_ordered_extent *entry = NULL;
@@ -536,7 +535,7 @@ void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
 		len = this_end - cur;
 		ASSERT(len < U32_MAX);
 
-		if (can_finish_ordered_extent(entry, folio, cur, len, uptodate)) {
+		if (can_finish_ordered_extent(entry, cur, len, uptodate)) {
 			spin_unlock(&inode->ordered_tree_lock);
 			btrfs_queue_ordered_fn(entry);
 			spin_lock(&inode->ordered_tree_lock);
@@ -628,7 +627,7 @@ void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry)
 		ASSERT(list_empty(&entry->log_list));
 		ASSERT(RB_EMPTY_NODE(&entry->rb_node));
 		btrfs_add_delayed_iput(entry->inode);
-		list_for_each_entry_safe(sum, tmp, &entry->list, list)
+		list_for_each_entry_safe(sum, tmp, &entry->csum_list, list)
 			kvfree(sum);
 		kmem_cache_free(btrfs_ordered_extent_cache, entry);
 	}
@@ -638,9 +637,9 @@ void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry)
  * remove an ordered extent from the tree.  No references are dropped
  * and waiters are woken up.
  */
-void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode,
-				 struct btrfs_ordered_extent *entry)
+void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry)
 {
+	struct btrfs_inode *btrfs_inode = entry->inode;
 	struct btrfs_root *root = btrfs_inode->root;
 	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct rb_node *node;
@@ -1323,10 +1322,10 @@ struct btrfs_ordered_extent *btrfs_split_ordered_extent(
 		}
 	}
 
-	list_for_each_entry_safe(sum, tmpsum, &ordered->list, list) {
+	list_for_each_entry_safe(sum, tmpsum, &ordered->csum_list, list) {
 		if (offset == len)
 			break;
-		list_move_tail(&sum->list, &new->list);
+		list_move_tail(&sum->list, &new->csum_list);
 		offset += sum->len;
 	}
 
diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h
index 1e6b0b182b295..03e12380a2fda 100644
--- a/fs/btrfs/ordered-data.h
+++ b/fs/btrfs/ordered-data.h
@@ -47,8 +47,25 @@ struct btrfs_ordered_sum {
  * IO is done and any metadata is inserted into the tree.
  */
 enum {
+	/* Extra status bits for ordered extents */
+
+	/* Set when all the pages are written. */
+	BTRFS_ORDERED_IO_DONE,
+	/* Set when removed from the tree. */
+	BTRFS_ORDERED_COMPLETE,
+	/* We had an io error when writing this out. */
+	BTRFS_ORDERED_IOERR,
+	/* Set when we have to truncate an extent. */
+	BTRFS_ORDERED_TRUNCATED,
+	/* Used during fsync to track already logged extents. */
+	BTRFS_ORDERED_LOGGED,
+	/* We have already logged all the csums of the ordered extent. */
+	BTRFS_ORDERED_LOGGED_CSUM,
+	/* We wait for this extent to complete in the current transaction. */
+	BTRFS_ORDERED_PENDING,
+
 	/*
-	 * Different types for ordered extents, one and only one of the 4 types
+	 * Different types for ordered extents, one and only one of these types
 	 * need to be set when creating ordered extent.
 	 *
 	 * REGULAR:	For regular non-compressed COW write
@@ -61,37 +78,27 @@ enum {
 	BTRFS_ORDERED_PREALLOC,
 	BTRFS_ORDERED_COMPRESSED,
 
+	/* Extra bit for encoded write, must be set with COMPRESSED. */
+	BTRFS_ORDERED_ENCODED,
+
 	/*
 	 * Extra bit for direct io, can only be set for
-	 * REGULAR/NOCOW/PREALLOC. No direct io for compressed extent.
+	 * REGULAR/NOCOW/PREALLOC. Must not be set for COMPRESSED nor ENCODED.
 	 */
 	BTRFS_ORDERED_DIRECT,
 
-	/* Extra status bits for ordered extents */
-
-	/* set when all the pages are written */
-	BTRFS_ORDERED_IO_DONE,
-	/* set when removed from the tree */
-	BTRFS_ORDERED_COMPLETE,
-	/* We had an io error when writing this out */
-	BTRFS_ORDERED_IOERR,
-	/* Set when we have to truncate an extent */
-	BTRFS_ORDERED_TRUNCATED,
-	/* Used during fsync to track already logged extents */
-	BTRFS_ORDERED_LOGGED,
-	/* We have already logged all the csums of the ordered extent */
-	BTRFS_ORDERED_LOGGED_CSUM,
-	/* We wait for this extent to complete in the current transaction */
-	BTRFS_ORDERED_PENDING,
-	/* BTRFS_IOC_ENCODED_WRITE */
-	BTRFS_ORDERED_ENCODED,
+	BTRFS_ORDERED_NR_FLAGS,
 };
+static_assert(BTRFS_ORDERED_NR_FLAGS <= BITS_PER_LONG);
+
+/* One and only one flag can be set. */
+#define BTRFS_ORDERED_EXCLUSIVE_FLAGS ((1UL << BTRFS_ORDERED_REGULAR) |		\
+				       (1UL << BTRFS_ORDERED_NOCOW) |		\
+				       (1UL << BTRFS_ORDERED_PREALLOC) |	\
+				       (1UL << BTRFS_ORDERED_COMPRESSED))
 
 /* BTRFS_ORDERED_* flags that specify the type of the extent. */
-#define BTRFS_ORDERED_TYPE_FLAGS ((1UL << BTRFS_ORDERED_REGULAR) |	\
-				  (1UL << BTRFS_ORDERED_NOCOW) |	\
-				  (1UL << BTRFS_ORDERED_PREALLOC) |	\
-				  (1UL << BTRFS_ORDERED_COMPRESSED) |	\
+#define BTRFS_ORDERED_TYPE_FLAGS (BTRFS_ORDERED_EXCLUSIVE_FLAGS |	\
 				  (1UL << BTRFS_ORDERED_DIRECT) |	\
 				  (1UL << BTRFS_ORDERED_ENCODED))
 
@@ -134,7 +141,7 @@ struct btrfs_ordered_extent {
 	struct btrfs_inode *inode;
 
 	/* list of checksums for insertion when the extent io is done */
-	struct list_head list;
+	struct list_head csum_list;
 
 	/* used for fast fsyncs */
 	struct list_head log_list;
@@ -161,14 +168,11 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent);
 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
 
 void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry);
-void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode,
-				struct btrfs_ordered_extent *entry);
+void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry);
 void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
-				 struct folio *folio, u64 file_offset, u64 len,
-				 bool uptodate);
+				 u64 file_offset, u64 len, bool uptodate);
 void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
-				    struct folio *folio, u64 file_offset,
-				    u64 num_bytes, bool uptodate);
+				    u64 file_offset, u64 num_bytes, bool uptodate);
 bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,
 				    struct btrfs_ordered_extent **cached,
 				    u64 file_offset, u64 io_size);
diff --git a/fs/btrfs/print-tree.c b/fs/btrfs/print-tree.c
index b7dfe877cf8d9..87e60a2d4bd81 100644
--- a/fs/btrfs/print-tree.c
+++ b/fs/btrfs/print-tree.c
@@ -626,10 +626,6 @@ void btrfs_print_tree(const struct extent_buffer *c, bool follow)
 		next = read_tree_block(fs_info, btrfs_node_blockptr(c, i), &check);
 		if (IS_ERR(next))
 			continue;
-		if (!extent_buffer_uptodate(next)) {
-			free_extent_buffer(next);
-			continue;
-		}
 
 		if (btrfs_is_leaf(next) &&
 		   level != 1)
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index 3b2a6517d0b53..65ec458595071 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -2740,8 +2740,6 @@ static void qgroup_iterator_nested_clean(struct list_head *head)
 	}
 }
 
-#define UPDATE_NEW	0
-#define UPDATE_OLD	1
 /*
  * Walk all of the roots that points to the bytenr and adjust their refcnts.
  */
@@ -2980,10 +2978,10 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr,
 	seq = fs_info->qgroup_seq;
 
 	/* Update old refcnts using old_roots */
-	qgroup_update_refcnt(fs_info, old_roots, &qgroups, seq, UPDATE_OLD);
+	qgroup_update_refcnt(fs_info, old_roots, &qgroups, seq, true);
 
 	/* Update new refcnts using new_roots */
-	qgroup_update_refcnt(fs_info, new_roots, &qgroups, seq, UPDATE_NEW);
+	qgroup_update_refcnt(fs_info, new_roots, &qgroups, seq, false);
 
 	qgroup_update_counters(fs_info, &qgroups, nr_old_roots, nr_new_roots,
 			       num_bytes, seq);
@@ -3739,6 +3737,14 @@ static int qgroup_rescan_leaf(struct btrfs_trans_handle *trans,
 	mutex_lock(&fs_info->qgroup_rescan_lock);
 	extent_root = btrfs_extent_root(fs_info,
 				fs_info->qgroup_rescan_progress.objectid);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  fs_info->qgroup_rescan_progress.objectid);
+		mutex_unlock(&fs_info->qgroup_rescan_lock);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_search_slot_for_read(extent_root,
 					 &fs_info->qgroup_rescan_progress,
 					 path, 1, 0);
@@ -4875,10 +4881,6 @@ int btrfs_qgroup_trace_subtree_after_cow(struct btrfs_trans_handle *trans,
 		reloc_eb = NULL;
 		goto free_out;
 	}
-	if (unlikely(!extent_buffer_uptodate(reloc_eb))) {
-		ret = -EIO;
-		goto free_out;
-	}
 
 	ret = qgroup_trace_subtree_swap(trans, reloc_eb, subvol_eb,
 			block->last_snapshot, block->trace_leaf);
diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c
index 2987cb7c686ea..638c4ad572c99 100644
--- a/fs/btrfs/raid-stripe-tree.c
+++ b/fs/btrfs/raid-stripe-tree.c
@@ -300,7 +300,7 @@ int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans,
 	int ret;
 
 	stripe_extent = kzalloc(item_size, GFP_NOFS);
-	if (!unlikely(stripe_extent)) {
+	if (unlikely(!stripe_extent)) {
 		btrfs_abort_transaction(trans, -ENOMEM);
 		btrfs_end_transaction(trans);
 		return -ENOMEM;
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index b4511f560e929..baebd9f733e9e 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -617,26 +617,6 @@ static void cache_rbio(struct btrfs_raid_bio *rbio)
 	spin_unlock(&table->cache_lock);
 }
 
-/*
- * helper function to run the xor_blocks api.  It is only
- * able to do MAX_XOR_BLOCKS at a time, so we need to
- * loop through.
- */
-static void run_xor(void **pages, int src_cnt, ssize_t len)
-{
-	int src_off = 0;
-	int xor_src_cnt = 0;
-	void *dest = pages[src_cnt];
-
-	while(src_cnt > 0) {
-		xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
-		xor_blocks(xor_src_cnt, len, dest, pages + src_off);
-
-		src_cnt -= xor_src_cnt;
-		src_off += xor_src_cnt;
-	}
-}
-
 /*
  * Returns true if the bio list inside this rbio covers an entire stripe (no
  * rmw required).
@@ -1434,7 +1414,8 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int
 	} else {
 		/* raid5 */
 		memcpy(pointers[rbio->nr_data], pointers[0], step);
-		run_xor(pointers + 1, rbio->nr_data - 1, step);
+		xor_gen(pointers[rbio->nr_data], pointers + 1, rbio->nr_data - 1,
+				step);
 	}
 	for (stripe = stripe - 1; stripe >= 0; stripe--)
 		kunmap_local(pointers[stripe]);
@@ -1653,12 +1634,7 @@ static int get_bio_sector_nr(struct btrfs_raid_bio *rbio, struct bio *bio)
 static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bio)
 {
 	int total_sector_nr = get_bio_sector_nr(rbio, bio);
-	u32 bio_size = 0;
-	struct bio_vec *bvec;
-	int i;
-
-	bio_for_each_bvec_all(bvec, bio, i)
-		bio_size += bvec->bv_len;
+	const u32 bio_size = bio_get_size(bio);
 
 	/*
 	 * Since we can have multiple bios touching the error_bitmap, we cannot
@@ -1666,7 +1642,7 @@ static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bi
 	 *
 	 * Instead use set_bit() for each bit, as set_bit() itself is atomic.
 	 */
-	for (i = total_sector_nr; i < total_sector_nr +
+	for (int i = total_sector_nr; i < total_sector_nr +
 	     (bio_size >> rbio->bioc->fs_info->sectorsize_bits); i++)
 		set_bit(i, rbio->error_bitmap);
 }
@@ -2034,7 +2010,7 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio,
 		pointers[rbio->nr_data - 1] = p;
 
 		/* Xor in the rest */
-		run_xor(pointers, rbio->nr_data - 1, step);
+		xor_gen(p, pointers, rbio->nr_data - 1, step);
 	}
 
 cleanup:
@@ -2110,8 +2086,8 @@ static int recover_sectors(struct btrfs_raid_bio *rbio)
 	 * @unmap_array stores copy of pointers that does not get reordered
 	 * during reconstruction so that kunmap_local works.
 	 */
-	pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
-	unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
+	pointers = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS);
+	unmap_array = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS);
 	if (!pointers || !unmap_array) {
 		ret = -ENOMEM;
 		goto out;
@@ -2297,8 +2273,7 @@ void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc,
 static void fill_data_csums(struct btrfs_raid_bio *rbio)
 {
 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
-	struct btrfs_root *csum_root = btrfs_csum_root(fs_info,
-						       rbio->bioc->full_stripe_logical);
+	struct btrfs_root *csum_root;
 	const u64 start = rbio->bioc->full_stripe_logical;
 	const u32 len = (rbio->nr_data * rbio->stripe_nsectors) <<
 			fs_info->sectorsize_bits;
@@ -2331,6 +2306,15 @@ static void fill_data_csums(struct btrfs_raid_bio *rbio)
 		goto error;
 	}
 
+	csum_root = btrfs_csum_root(fs_info, rbio->bioc->full_stripe_logical);
+	if (unlikely(!csum_root)) {
+		btrfs_err(fs_info,
+			  "missing csum root for extent at bytenr %llu",
+			  rbio->bioc->full_stripe_logical);
+		ret = -EUCLEAN;
+		goto error;
+	}
+
 	ret = btrfs_lookup_csums_bitmap(csum_root, NULL, start, start + len - 1,
 					rbio->csum_buf, rbio->csum_bitmap);
 	if (ret < 0)
@@ -2664,7 +2648,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio,
 	} else {
 		/* RAID5. */
 		memcpy(pointers[nr_data], pointers[0], step);
-		run_xor(pointers + 1, nr_data - 1, step);
+		xor_gen(pointers[nr_data], pointers + 1, nr_data - 1, step);
 	}
 
 	/* Check scrubbing parity and repair it. */
@@ -2836,8 +2820,8 @@ static int recover_scrub_rbio(struct btrfs_raid_bio *rbio)
 	 * @unmap_array stores copy of pointers that does not get reordered
 	 * during reconstruction so that kunmap_local works.
 	 */
-	pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
-	unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
+	pointers = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS);
+	unmap_array = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS);
 	if (!pointers || !unmap_array) {
 		ret = -ENOMEM;
 		goto out;
diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c
index 314cb95ba846d..fca00c0f53874 100644
--- a/fs/btrfs/reflink.c
+++ b/fs/btrfs/reflink.c
@@ -646,7 +646,7 @@ static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len,
 	 */
 	btrfs_lock_extent(&dst->io_tree, dst_loff, end, &cached_state);
 	ret = btrfs_clone(&src->vfs_inode, &dst->vfs_inode, loff, len,
-			  ALIGN(len, bs), dst_loff, 1);
+			  ALIGN(len, bs), dst_loff, true);
 	btrfs_unlock_extent(&dst->io_tree, dst_loff, end, &cached_state);
 
 	btrfs_btree_balance_dirty(fs_info);
@@ -747,7 +747,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src,
 	 */
 	end = destoff + len - 1;
 	btrfs_lock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state);
-	ret = btrfs_clone(src, inode, off, olen, len, destoff, 0);
+	ret = btrfs_clone(src, inode, off, olen, len, destoff, false);
 	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, destoff, end, &cached_state);
 	if (ret < 0)
 		return ret;
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index b2343aed7a5d0..1c42c5180bddd 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -2440,10 +2440,7 @@ static int get_tree_block_key(struct btrfs_fs_info *fs_info,
 	eb = read_tree_block(fs_info, block->bytenr, &check);
 	if (IS_ERR(eb))
 		return PTR_ERR(eb);
-	if (unlikely(!extent_buffer_uptodate(eb))) {
-		free_extent_buffer(eb);
-		return -EIO;
-	}
+
 	if (block->level == 0)
 		btrfs_item_key_to_cpu(eb, &block->key, 0);
 	else
@@ -3645,12 +3642,7 @@ static noinline_for_stack int relocate_block_group(struct reloc_control *rc)
 	btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL);
 
 	/* get rid of pinned extents */
-	trans = btrfs_join_transaction(rc->extent_root);
-	if (IS_ERR(trans)) {
-		err = PTR_ERR(trans);
-		goto out_free;
-	}
-	ret = btrfs_commit_transaction(trans);
+	ret = btrfs_commit_current_transaction(rc->extent_root);
 	if (ret && !err)
 		err = ret;
 out_free:
@@ -4185,6 +4177,8 @@ static int move_existing_remap(struct btrfs_fs_info *fs_info,
 	dest_addr = ins.objectid;
 	dest_length = ins.offset;
 
+	dest_bg = btrfs_lookup_block_group(fs_info, dest_addr);
+
 	if (!is_data && !IS_ALIGNED(dest_length, fs_info->nodesize)) {
 		u64 new_length = ALIGN_DOWN(dest_length, fs_info->nodesize);
 
@@ -4295,15 +4289,12 @@ static int move_existing_remap(struct btrfs_fs_info *fs_info,
 	if (unlikely(ret))
 		goto end;
 
-	dest_bg = btrfs_lookup_block_group(fs_info, dest_addr);
-
 	adjust_block_group_remap_bytes(trans, dest_bg, dest_length);
 
 	mutex_lock(&dest_bg->free_space_lock);
 	bg_needs_free_space = test_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE,
 				       &dest_bg->runtime_flags);
 	mutex_unlock(&dest_bg->free_space_lock);
-	btrfs_put_block_group(dest_bg);
 
 	if (bg_needs_free_space) {
 		ret = btrfs_add_block_group_free_space(trans, dest_bg);
@@ -4333,13 +4324,13 @@ static int move_existing_remap(struct btrfs_fs_info *fs_info,
 			btrfs_end_transaction(trans);
 		}
 	} else {
-		dest_bg = btrfs_lookup_block_group(fs_info, dest_addr);
 		btrfs_free_reserved_bytes(dest_bg, dest_length, 0);
-		btrfs_put_block_group(dest_bg);
 
 		ret = btrfs_commit_transaction(trans);
 	}
 
+	btrfs_put_block_group(dest_bg);
+
 	return ret;
 }
 
@@ -4954,6 +4945,12 @@ static int do_remap_reloc_trans(struct btrfs_fs_info *fs_info,
 	struct btrfs_space_info *sinfo = src_bg->space_info;
 
 	extent_root = btrfs_extent_root(fs_info, src_bg->start);
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for block group at offset %llu",
+			  src_bg->start);
+		return -EUCLEAN;
+	}
 
 	trans = btrfs_start_transaction(extent_root, 0);
 	if (IS_ERR(trans))
@@ -5306,6 +5303,13 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start,
 	int ret;
 	bool bg_is_ro = false;
 
+	if (unlikely(!extent_root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for block group at offset %llu",
+			  group_start);
+		return -EUCLEAN;
+	}
+
 	/*
 	 * This only gets set if we had a half-deleted snapshot on mount.  We
 	 * cannot allow relocation to start while we're still trying to clean up
@@ -5536,12 +5540,17 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info)
 		goto out;
 	}
 
+	rc->extent_root = btrfs_extent_root(fs_info, 0);
+	if (unlikely(!rc->extent_root)) {
+		btrfs_err(fs_info, "missing extent root for extent at bytenr 0");
+		ret = -EUCLEAN;
+		goto out;
+	}
+
 	ret = reloc_chunk_start(fs_info);
 	if (ret < 0)
 		goto out_end;
 
-	rc->extent_root = btrfs_extent_root(fs_info, 0);
-
 	set_reloc_control(rc);
 
 	trans = btrfs_join_transaction(rc->extent_root);
@@ -5636,6 +5645,14 @@ int btrfs_reloc_clone_csums(struct btrfs_ordered_extent *ordered)
 	LIST_HEAD(list);
 	int ret;
 
+	if (unlikely(!csum_root)) {
+		btrfs_mark_ordered_extent_error(ordered);
+		btrfs_err(fs_info,
+			  "missing csum root for extent at bytenr %llu",
+			  disk_bytenr);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_lookup_csums_list(csum_root, disk_bytenr,
 				      disk_bytenr + ordered->num_bytes - 1,
 				      &list, false);
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index bc94bbc007720..0234971894fa3 100644
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -891,16 +891,11 @@ static void scrub_repair_read_endio(struct btrfs_bio *bbio)
 {
 	struct scrub_stripe *stripe = bbio->private;
 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
-	struct bio_vec *bvec;
 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
-	u32 bio_size = 0;
-	int i;
+	const u32 bio_size = bio_get_size(&bbio->bio);
 
 	ASSERT(sector_nr < stripe->nr_sectors);
 
-	bio_for_each_bvec_all(bvec, &bbio->bio, i)
-		bio_size += bvec->bv_len;
-
 	if (bbio->bio.bi_status) {
 		scrub_bitmap_set_io_error(stripe, sector_nr,
 					  bio_size >> fs_info->sectorsize_bits);
@@ -1249,15 +1244,11 @@ static void scrub_stripe_read_repair_worker(struct work_struct *work)
 static void scrub_read_endio(struct btrfs_bio *bbio)
 {
 	struct scrub_stripe *stripe = bbio->private;
-	struct bio_vec *bvec;
 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
 	int num_sectors;
-	u32 bio_size = 0;
-	int i;
+	const u32 bio_size = bio_get_size(&bbio->bio);
 
 	ASSERT(sector_nr < stripe->nr_sectors);
-	bio_for_each_bvec_all(bvec, &bbio->bio, i)
-		bio_size += bvec->bv_len;
 	num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits;
 
 	if (bbio->bio.bi_status) {
@@ -1278,13 +1269,8 @@ static void scrub_write_endio(struct btrfs_bio *bbio)
 {
 	struct scrub_stripe *stripe = bbio->private;
 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
-	struct bio_vec *bvec;
 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
-	u32 bio_size = 0;
-	int i;
-
-	bio_for_each_bvec_all(bvec, &bbio->bio, i)
-		bio_size += bvec->bv_len;
+	const u32 bio_size = bio_get_size(&bbio->bio);
 
 	if (bbio->bio.bi_status) {
 		unsigned long flags;
@@ -1293,7 +1279,7 @@ static void scrub_write_endio(struct btrfs_bio *bbio)
 		bitmap_set(&stripe->write_error_bitmap, sector_nr,
 			   bio_size >> fs_info->sectorsize_bits);
 		spin_unlock_irqrestore(&stripe->write_error_lock, flags);
-		for (i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++)
+		for (int i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++)
 			btrfs_dev_stat_inc_and_print(stripe->dev,
 						     BTRFS_DEV_STAT_WRITE_ERRS);
 	}
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 904a2f57f86dc..89d72d8cb85fd 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -7201,7 +7201,7 @@ static int changed_cb(struct btrfs_path *left_path,
 	sctx->right_path = right_path;
 	sctx->cmp_key = key;
 
-	ret = finish_inode_if_needed(sctx, 0);
+	ret = finish_inode_if_needed(sctx, false);
 	if (ret < 0)
 		return ret;
 
@@ -7328,7 +7328,7 @@ static int full_send_tree(struct send_ctx *sctx)
 	}
 
 out_finish:
-	return finish_inode_if_needed(sctx, 1);
+	return finish_inode_if_needed(sctx, true);
 }
 
 static int replace_node_with_clone(struct btrfs_path *path, int level)
@@ -7879,7 +7879,7 @@ static int send_subvol(struct send_ctx *sctx)
 		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
 		if (ret < 0)
 			goto out;
-		ret = finish_inode_if_needed(sctx, 1);
+		ret = finish_inode_if_needed(sctx, true);
 		if (ret < 0)
 			goto out;
 	} else {
diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
index 87cbc051cb12f..e017bb182c8cd 100644
--- a/fs/btrfs/space-info.c
+++ b/fs/btrfs/space-info.c
@@ -129,6 +129,15 @@
  *     churn a lot and we can avoid making some extent tree modifications if we
  *     are able to delay for as long as possible.
  *
+ *   RECLAIM_ZONES
+ *     This state only works for the zoned mode. In zoned mode, we cannot reuse
+ *     regions that have once been allocated and then been freed until we reset
+ *     the zone, due to the sequential write requirement. The RECLAIM_ZONES state
+ *     calls the reclaim machinery, evacuating the still valid data in these
+ *     block-groups and relocates it to the data_reloc_bg. Afterwards these
+ *     block-groups get deleted and the transaction is committed. This frees up
+ *     space to use for new allocations.
+ *
  *   RESET_ZONES
  *     This state works only for the zoned mode. On the zoned mode, we cannot
  *     reuse once allocated then freed region until we reset the zone, due to
@@ -203,6 +212,8 @@ void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
 
 #define BTRFS_UNALLOC_BLOCK_GROUP_TARGET			(10ULL)
 
+#define BTRFS_ZONED_SYNC_RECLAIM_BATCH				(5)
+
 /*
  * Calculate chunk size depending on volume type (regular or zoned).
  */
@@ -403,10 +414,10 @@ void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info,
 	up_write(&space_info->groups_sem);
 }
 
-struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info,
+struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info,
 					       u64 flags)
 {
-	struct list_head *head = &info->space_info;
+	const struct list_head *head = &info->space_info;
 	struct btrfs_space_info *found;
 
 	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
@@ -418,7 +429,7 @@ struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info,
 	return NULL;
 }
 
-static u64 calc_effective_data_chunk_size(struct btrfs_fs_info *fs_info)
+static u64 calc_effective_data_chunk_size(const struct btrfs_fs_info *fs_info)
 {
 	struct btrfs_space_info *data_sinfo;
 	u64 data_chunk_size;
@@ -444,6 +455,7 @@ static u64 calc_available_free_space(const struct btrfs_space_info *space_info,
 				     enum btrfs_reserve_flush_enum flush)
 {
 	struct btrfs_fs_info *fs_info = space_info->fs_info;
+	bool has_per_profile;
 	u64 profile;
 	u64 avail;
 	u64 data_chunk_size;
@@ -454,19 +466,21 @@ static u64 calc_available_free_space(const struct btrfs_space_info *space_info,
 	else
 		profile = btrfs_metadata_alloc_profile(fs_info);
 
-	avail = atomic64_read(&fs_info->free_chunk_space);
-
-	/*
-	 * If we have dup, raid1 or raid10 then only half of the free
-	 * space is actually usable.  For raid56, the space info used
-	 * doesn't include the parity drive, so we don't have to
-	 * change the math
-	 */
-	factor = btrfs_bg_type_to_factor(profile);
-	avail = div_u64(avail, factor);
-	if (avail == 0)
-		return 0;
+	has_per_profile = btrfs_get_per_profile_avail(fs_info, profile, &avail);
+	if (!has_per_profile) {
+		avail = atomic64_read(&fs_info->free_chunk_space);
 
+		/*
+		 * If we have dup, raid1 or raid10 then only half of the free
+		 * space is actually usable.  For raid56, the space info used
+		 * doesn't include the parity drive, so we don't have to
+		 * change the math
+		 */
+		factor = btrfs_bg_type_to_factor(profile);
+		avail = div_u64(avail, factor);
+		if (avail == 0)
+			return 0;
+	}
 	data_chunk_size = calc_effective_data_chunk_size(fs_info);
 
 	/*
@@ -489,10 +503,10 @@ static u64 calc_available_free_space(const struct btrfs_space_info *space_info,
 	/*
 	 * If we aren't flushing all things, let us overcommit up to
 	 * 1/2th of the space. If we can flush, don't let us overcommit
-	 * too much, let it overcommit up to 1/8 of the space.
+	 * too much, let it overcommit up to 1/64th of the space.
 	 */
-	if (flush == BTRFS_RESERVE_FLUSH_ALL)
-		avail >>= 3;
+	if (flush == BTRFS_RESERVE_FLUSH_ALL || flush == BTRFS_RESERVE_FLUSH_ALL_STEAL)
+		avail >>= 6;
 	else
 		avail >>= 1;
 
@@ -902,6 +916,18 @@ static void flush_space(struct btrfs_space_info *space_info, u64 num_bytes,
 		if (ret > 0 || ret == -ENOSPC)
 			ret = 0;
 		break;
+	case RECLAIM_ZONES:
+		if (btrfs_is_zoned(fs_info)) {
+			btrfs_reclaim_sweep(fs_info);
+			btrfs_delete_unused_bgs(fs_info);
+			btrfs_reclaim_block_groups(fs_info,
+						   BTRFS_ZONED_SYNC_RECLAIM_BATCH);
+			ASSERT(current->journal_info == NULL);
+			ret = btrfs_commit_current_transaction(root);
+		} else {
+			ret = 0;
+		}
+		break;
 	case RUN_DELAYED_IPUTS:
 		/*
 		 * If we have pending delayed iputs then we could free up a
@@ -1400,6 +1426,7 @@ static const enum btrfs_flush_state data_flush_states[] = {
 	FLUSH_DELALLOC_FULL,
 	RUN_DELAYED_IPUTS,
 	COMMIT_TRANS,
+	RECLAIM_ZONES,
 	RESET_ZONES,
 	ALLOC_CHUNK_FORCE,
 };
diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h
index 0703f24b23f70..24f45072ca4b1 100644
--- a/fs/btrfs/space-info.h
+++ b/fs/btrfs/space-info.h
@@ -21,7 +21,24 @@ struct btrfs_block_group;
  * The higher the level, the more methods we try to reclaim space.
  */
 enum btrfs_reserve_flush_enum {
-	/* If we are in the transaction, we can't flush anything.*/
+	/*
+	 * Used when we can't flush or don't need:
+	 *
+	 * 1) We are holding a transaction handle open, so we can't flush as
+	 *    that could deadlock.
+	 *
+	 * 2) For a nowait write we don't want to block when reserving delalloc.
+	 *
+	 * 3) Joining a transaction or attaching a transaction, we don't want
+	 *    to wait and we don't need to reserve anything (any needed space
+	 *    was reserved before in a dedicated block reserve, or we rely on
+	 *    the global block reserve, see btrfs_init_root_block_rsv()).
+	 *
+	 * 4) Starting a transaction when we don't need to reserve space, as
+	 *    we don't need it because we previously reserved in a dedicated
+	 *    block reserve or rely on the global block reserve, like the above
+	 *    case.
+	 */
 	BTRFS_RESERVE_NO_FLUSH,
 
 	/*
@@ -96,6 +113,7 @@ enum btrfs_flush_state {
 	RUN_DELAYED_IPUTS	= 10,
 	COMMIT_TRANS		= 11,
 	RESET_ZONES		= 12,
+	RECLAIM_ZONES		= 13,
 };
 
 enum btrfs_space_info_sub_group {
@@ -274,7 +292,7 @@ void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info,
 				struct btrfs_block_group *block_group);
 void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info,
 					u64 chunk_size);
-struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info,
+struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info,
 					       u64 flags);
 void btrfs_clear_space_info_full(struct btrfs_fs_info *info);
 void btrfs_dump_space_info(struct btrfs_space_info *info, u64 bytes,
diff --git a/fs/btrfs/super.h b/fs/btrfs/super.h
index d80a86acfbbee..f85f8a8a7bfe3 100644
--- a/fs/btrfs/super.h
+++ b/fs/btrfs/super.h
@@ -18,7 +18,7 @@ char *btrfs_get_subvol_name_from_objectid(struct btrfs_fs_info *fs_info,
 					  u64 subvol_objectid);
 void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info);
 
-static inline struct btrfs_fs_info *btrfs_sb(struct super_block *sb)
+static inline struct btrfs_fs_info *btrfs_sb(const struct super_block *sb)
 {
 	return sb->s_fs_info;
 }
diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c
index 90e50e62dd172..19c127ac6d10c 100644
--- a/fs/btrfs/tests/btrfs-tests.c
+++ b/fs/btrfs/tests/btrfs-tests.c
@@ -303,6 +303,9 @@ int btrfs_run_sanity_tests(void)
 		}
 	}
 	ret = btrfs_test_extent_map();
+	if (ret)
+		goto out;
+	ret = btrfs_test_zoned();
 
 out:
 	btrfs_destroy_test_fs();
diff --git a/fs/btrfs/tests/btrfs-tests.h b/fs/btrfs/tests/btrfs-tests.h
index b03d85a6e5ef7..cea58fe84a6d5 100644
--- a/fs/btrfs/tests/btrfs-tests.h
+++ b/fs/btrfs/tests/btrfs-tests.h
@@ -63,6 +63,16 @@ void btrfs_init_dummy_trans(struct btrfs_trans_handle *trans,
 			    struct btrfs_fs_info *fs_info);
 void btrfs_init_dummy_transaction(struct btrfs_transaction *trans, struct btrfs_fs_info *fs_info);
 struct btrfs_device *btrfs_alloc_dummy_device(struct btrfs_fs_info *fs_info);
+
+#ifdef CONFIG_BLK_DEV_ZONED
+int btrfs_test_zoned(void);
+#else
+static inline int btrfs_test_zoned(void)
+{
+	return 0;
+}
+#endif
+
 #else
 static inline int btrfs_run_sanity_tests(void)
 {
diff --git a/fs/btrfs/tests/extent-io-tests.c b/fs/btrfs/tests/extent-io-tests.c
index a0187d6163df4..b2aacf846c8b7 100644
--- a/fs/btrfs/tests/extent-io-tests.c
+++ b/fs/btrfs/tests/extent-io-tests.c
@@ -4,7 +4,7 @@
  */
 
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/sizes.h>
diff --git a/fs/btrfs/tests/zoned-tests.c b/fs/btrfs/tests/zoned-tests.c
new file mode 100644
index 0000000000000..2bc3b14baa410
--- /dev/null
+++ b/fs/btrfs/tests/zoned-tests.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Western Digital.  All rights reserved.
+ */
+
+#include <linux/cleanup.h>
+#include <linux/sizes.h>
+
+#include "btrfs-tests.h"
+#include "../space-info.h"
+#include "../volumes.h"
+#include "../zoned.h"
+
+#define WP_MISSING_DEV				((u64)-1)
+#define WP_CONVENTIONAL				((u64)-2)
+#define ZONE_SIZE				SZ_256M
+
+#define HALF_STRIPE_LEN				(BTRFS_STRIPE_LEN >> 1)
+
+struct load_zone_info_test_vector {
+	u64 raid_type;
+	u64 num_stripes;
+	u64 alloc_offsets[8];
+	u64 last_alloc;
+	u64 bg_length;
+	bool degraded;
+
+	int expected_result;
+	u64 expected_alloc_offset;
+
+	const char *description;
+};
+
+struct zone_info {
+	u64 physical;
+	u64 capacity;
+	u64 alloc_offset;
+};
+
+static int test_load_zone_info(struct btrfs_fs_info *fs_info,
+			       const struct load_zone_info_test_vector *test)
+{
+	struct btrfs_block_group *bg __free(btrfs_free_dummy_block_group) = NULL;
+	struct btrfs_chunk_map *map __free(btrfs_free_chunk_map) = NULL;
+	struct zone_info AUTO_KFREE(zone_info);
+	unsigned long AUTO_KFREE(active);
+	int ret;
+
+	bg = btrfs_alloc_dummy_block_group(fs_info, test->bg_length);
+	if (!bg) {
+		test_std_err(TEST_ALLOC_BLOCK_GROUP);
+		return -ENOMEM;
+	}
+
+	map = btrfs_alloc_chunk_map(test->num_stripes, GFP_KERNEL);
+	if (!map) {
+		test_std_err(TEST_ALLOC_EXTENT_MAP);
+		return -ENOMEM;
+	}
+
+	zone_info = kzalloc_objs(*zone_info, test->num_stripes, GFP_KERNEL);
+	if (!zone_info) {
+		test_err("cannot allocate zone info");
+		return -ENOMEM;
+	}
+
+	active = bitmap_zalloc(test->num_stripes, GFP_KERNEL);
+	if (!zone_info) {
+		test_err("cannot allocate active bitmap");
+		return -ENOMEM;
+	}
+
+	map->type = test->raid_type;
+	map->num_stripes = test->num_stripes;
+	if (test->raid_type == BTRFS_BLOCK_GROUP_RAID10)
+		map->sub_stripes = 2;
+	for (int i = 0; i < test->num_stripes; i++) {
+		zone_info[i].physical = 0;
+		zone_info[i].alloc_offset = test->alloc_offsets[i];
+		zone_info[i].capacity = ZONE_SIZE;
+		if (zone_info[i].alloc_offset && zone_info[i].alloc_offset < ZONE_SIZE)
+			__set_bit(i, active);
+	}
+	if (test->degraded)
+		btrfs_set_opt(fs_info->mount_opt, DEGRADED);
+	else
+		btrfs_clear_opt(fs_info->mount_opt, DEGRADED);
+
+	ret = btrfs_load_block_group_by_raid_type(bg, map, zone_info, active,
+						  test->last_alloc);
+
+	if (ret != test->expected_result) {
+		test_err("unexpected return value: ret %d expected %d", ret,
+			 test->expected_result);
+		return -EINVAL;
+	}
+
+	if (!ret && bg->alloc_offset != test->expected_alloc_offset) {
+		test_err("unexpected alloc_offset: alloc_offset %llu expected %llu",
+			 bg->alloc_offset, test->expected_alloc_offset);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct load_zone_info_test_vector load_zone_info_tests[] = {
+	/* SINGLE */
+	{
+		.description = "SINGLE: load write pointer from sequential zone",
+		.raid_type = 0,
+		.num_stripes = 1,
+		.alloc_offsets = {
+			SZ_1M,
+		},
+		.expected_alloc_offset = SZ_1M,
+	},
+	/*
+	 * SINGLE block group on a conventional zone sets last_alloc outside of
+	 * btrfs_load_block_group_*(). Do not test that case.
+	 */
+
+	/* DUP */
+	/* Normal case */
+	{
+		.description = "DUP: having matching write pointers",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, SZ_1M,
+		},
+		.expected_alloc_offset = SZ_1M,
+	},
+	/*
+	 * One sequential zone and one conventional zone, having matching
+	 * last_alloc.
+	 */
+	{
+		.description = "DUP: seq zone and conv zone, matching last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_1M,
+		.expected_alloc_offset = SZ_1M,
+	},
+	/*
+	 * One sequential and one conventional zone, but having smaller
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "DUP: seq zone and conv zone, smaller last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = 0,
+		.expected_alloc_offset = SZ_1M,
+	},
+	/* Error case: having different write pointers. */
+	{
+		.description = "DUP: fail: different write pointers",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, SZ_2M,
+		},
+		.expected_result = -EIO,
+	},
+	/* Error case: partial missing device should not happen on DUP. */
+	{
+		.description = "DUP: fail: missing device",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_MISSING_DEV,
+		},
+		.expected_result = -EIO,
+	},
+	/*
+	 * Error case: one sequential and one conventional zone, but having larger
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "DUP: fail: seq zone and conv zone, larger last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_DUP,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M,
+		.expected_result = -EIO,
+	},
+
+	/* RAID1 */
+	/* Normal case */
+	{
+		.description = "RAID1: having matching write pointers",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, SZ_1M,
+		},
+		.expected_alloc_offset = SZ_1M,
+	},
+	/*
+	 * One sequential zone and one conventional zone, having matching
+	 * last_alloc.
+	 */
+	{
+		.description = "RAID1: seq zone and conv zone, matching last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_1M,
+		.expected_alloc_offset = SZ_1M,
+	},
+	/*
+	 * One sequential and one conventional zone, but having smaller
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "RAID1: seq zone and conv zone, smaller last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = 0,
+		.expected_alloc_offset = SZ_1M,
+	},
+	/* Partial missing device should be recovered on DEGRADED mount */
+	{
+		.description = "RAID1: fail: missing device on DEGRADED",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_MISSING_DEV,
+		},
+		.degraded = true,
+		.expected_alloc_offset = SZ_1M,
+	},
+	/* Error case: having different write pointers. */
+	{
+		.description = "RAID1: fail: different write pointers",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, SZ_2M,
+		},
+		.expected_result = -EIO,
+	},
+	/*
+	 * Partial missing device is not allowed on non-DEGRADED mount never happen
+	 * as it is rejected beforehand.
+	 */
+	/*
+	 * Error case: one sequential and one conventional zone, but having larger
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "RAID1: fail: seq zone and conv zone, larger last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID1,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M,
+		.expected_result = -EIO,
+	},
+
+	/* RAID0 */
+	/* Normal case */
+	{
+		.description = "RAID0: initial partial write",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			HALF_STRIPE_LEN, 0, 0, 0,
+		},
+		.expected_alloc_offset = HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID0: while in second stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN + HALF_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 5 + HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID0: one stripe advanced",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M + BTRFS_STRIPE_LEN, SZ_1M,
+		},
+		.expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN,
+	},
+	/* Error case: having different write pointers. */
+	{
+		.description = "RAID0: fail: disordered stripes",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN * 2,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_result = -EIO,
+	},
+	{
+		.description = "RAID0: fail: far distance",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_result = -EIO,
+	},
+	{
+		.description = "RAID0: fail: too many partial write",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			HALF_STRIPE_LEN, HALF_STRIPE_LEN, 0, 0,
+		},
+		.expected_result = -EIO,
+	},
+	/*
+	 * Error case: Partial missing device is not allowed even on non-DEGRADED
+	 * mount.
+	 */
+	{
+		.description = "RAID0: fail: missing device on DEGRADED",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_MISSING_DEV,
+		},
+		.degraded = true,
+		.expected_result = -EIO,
+	},
+
+	/*
+	 * One sequential zone and one conventional zone, having matching
+	 * last_alloc.
+	 */
+	{
+		.description = "RAID0: seq zone and conv zone, partially written stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M - SZ_4K,
+		.expected_alloc_offset = SZ_2M - SZ_4K,
+	},
+	{
+		.description = "RAID0: conv zone and seq zone, partially written stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			WP_CONVENTIONAL, SZ_1M,
+		},
+		.last_alloc = SZ_2M + SZ_4K,
+		.expected_alloc_offset = SZ_2M + SZ_4K,
+	},
+	/*
+	 * Error case: one sequential and one conventional zone, but having larger
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "RAID0: fail: seq zone and conv zone, larger last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 2,
+		.alloc_offsets = {
+			SZ_1M, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M + BTRFS_STRIPE_LEN * 2,
+		.expected_result = -EIO,
+	},
+
+	/* RAID0, 4 stripes with seq zones and conv zones. */
+	{
+		.description = "RAID0: stripes [2, 2, ?, ?] last_alloc = 6",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 6,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 6,
+	},
+	{
+		.description = "RAID0: stripes [2, 2, ?, ?] last_alloc = 7.5",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID0: stripes [3, ?, ?, ?] last_alloc = 1",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 3, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 9,
+	},
+	{
+		.description = "RAID0: stripes [2, ?, 1, ?] last_alloc = 5",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, WP_CONVENTIONAL,
+			BTRFS_STRIPE_LEN, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 5,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 5,
+	},
+	{
+		.description = "RAID0: fail: stripes [2, ?, 1, ?] last_alloc = 7",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID0,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, WP_CONVENTIONAL,
+			BTRFS_STRIPE_LEN, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 7,
+		.expected_result = -EIO,
+	},
+
+	/* RAID10 */
+	/* Normal case */
+	{
+		.description = "RAID10: initial partial write",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			HALF_STRIPE_LEN, HALF_STRIPE_LEN, 0, 0,
+		},
+		.expected_alloc_offset = HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID10: while in second stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			BTRFS_STRIPE_LEN + HALF_STRIPE_LEN,
+			BTRFS_STRIPE_LEN + HALF_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 5 + HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID10: one stripe advanced",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			SZ_1M + BTRFS_STRIPE_LEN, SZ_1M + BTRFS_STRIPE_LEN,
+			SZ_1M, SZ_1M,
+		},
+		.expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN,
+	},
+	{
+		.description = "RAID10: one stripe advanced, with conventional zone",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			SZ_1M + BTRFS_STRIPE_LEN, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, SZ_1M,
+		},
+		.expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN,
+	},
+	/* Error case: having different write pointers. */
+	{
+		.description = "RAID10: fail: disordered stripes",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_result = -EIO,
+	},
+	{
+		.description = "RAID10: fail: far distance",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN * 3,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+		},
+		.expected_result = -EIO,
+	},
+	{
+		.description = "RAID10: fail: too many partial write",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			HALF_STRIPE_LEN, HALF_STRIPE_LEN,
+			HALF_STRIPE_LEN, HALF_STRIPE_LEN,
+			0, 0, 0, 0,
+		},
+		.expected_result = -EIO,
+	},
+	/*
+	 * Error case: Partial missing device in RAID0 level is not allowed even on
+	 * non-DEGRADED mount.
+	 */
+	{
+		.description = "RAID10: fail: missing device on DEGRADED",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			SZ_1M, SZ_1M,
+			WP_MISSING_DEV, WP_MISSING_DEV,
+		},
+		.degraded = true,
+		.expected_result = -EIO,
+	},
+
+	/*
+	 * One sequential zone and one conventional zone, having matching
+	 * last_alloc.
+	 */
+	{
+		.description = "RAID10: seq zone and conv zone, partially written stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			SZ_1M, SZ_1M,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M - SZ_4K,
+		.expected_alloc_offset = SZ_2M - SZ_4K,
+	},
+	{
+		.description = "RAID10: conv zone and seq zone, partially written stripe",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			SZ_1M, SZ_1M,
+		},
+		.last_alloc = SZ_2M + SZ_4K,
+		.expected_alloc_offset = SZ_2M + SZ_4K,
+	},
+	/*
+	 * Error case: one sequential and one conventional zone, but having larger
+	 * last_alloc than write pointer.
+	 */
+	{
+		.description = "RAID10: fail: seq zone and conv zone, larger last_alloc",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 4,
+		.alloc_offsets = {
+			SZ_1M, SZ_1M,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = SZ_2M + BTRFS_STRIPE_LEN * 2,
+		.expected_result = -EIO,
+	},
+
+	/* RAID10, 4 stripes with seq zones and conv zones. */
+	{
+		.description = "RAID10: stripes [2, 2, ?, ?] last_alloc = 6",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 6,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 6,
+	},
+	{
+		.description = "RAID10: stripes [2, 2, ?, ?] last_alloc = 7.5",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN,
+	},
+	{
+		.description = "RAID10: stripes [3, ?, ?, ?] last_alloc = 1",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN * 3,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 9,
+	},
+	{
+		.description = "RAID10: stripes [2, ?, 1, ?] last_alloc = 5",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 5,
+		.expected_alloc_offset = BTRFS_STRIPE_LEN * 5,
+	},
+	{
+		.description = "RAID10: fail: stripes [2, ?, 1, ?] last_alloc = 7",
+		.raid_type = BTRFS_BLOCK_GROUP_RAID10,
+		.num_stripes = 8,
+		.alloc_offsets = {
+			BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+			BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN,
+			WP_CONVENTIONAL, WP_CONVENTIONAL,
+		},
+		.last_alloc = BTRFS_STRIPE_LEN * 7,
+		.expected_result = -EIO,
+	},
+};
+
+int btrfs_test_zoned(void)
+{
+	struct btrfs_fs_info *fs_info __free(btrfs_free_dummy_fs_info) = NULL;
+	int ret;
+
+	test_msg("running zoned tests (error messages are expected)");
+
+	fs_info = btrfs_alloc_dummy_fs_info(PAGE_SIZE, PAGE_SIZE);
+	if (!fs_info) {
+		test_std_err(TEST_ALLOC_FS_INFO);
+		return -ENOMEM;
+	}
+
+	for (int i = 0; i < ARRAY_SIZE(load_zone_info_tests); i++) {
+		ret = test_load_zone_info(fs_info, &load_zone_info_tests[i]);
+		if (ret) {
+			test_err("test case \"%s\" failed", load_zone_info_tests[i].description);
+			return ret;
+		}
+	}
+
+	return 0;
+}
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index 8dd77c431974d..4358f4b630572 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -15,6 +15,7 @@
 #include "misc.h"
 #include "ctree.h"
 #include "disk-io.h"
+#include "extent_io.h"
 #include "transaction.h"
 #include "locking.h"
 #include "tree-log.h"
@@ -503,7 +504,7 @@ int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans,
 		return 0;
 
 	mutex_lock(&fs_info->reloc_mutex);
-	ret = record_root_in_trans(trans, root, 0);
+	ret = record_root_in_trans(trans, root, false);
 	mutex_unlock(&fs_info->reloc_mutex);
 
 	return ret;
@@ -678,6 +679,14 @@ start_transaction(struct btrfs_root *root, unsigned int num_items,
 		 * here.
 		 */
 		ret = btrfs_delayed_refs_rsv_refill(fs_info, flush);
+		if (ret == -EAGAIN) {
+			ASSERT(btrfs_is_zoned(fs_info));
+			ret = btrfs_commit_current_transaction(root);
+			if (ret)
+				goto reserve_fail;
+			ret = btrfs_delayed_refs_rsv_refill(fs_info, flush);
+		}
+
 		if (ret)
 			goto reserve_fail;
 	}
@@ -688,6 +697,8 @@ start_transaction(struct btrfs_root *root, unsigned int num_items,
 		goto alloc_fail;
 	}
 
+	xa_init(&h->writeback_inhibited_ebs);
+
 	/*
 	 * If we are JOIN_NOLOCK we're already committing a transaction and
 	 * waiting on this guy, so we don't need to do the sb_start_intwrite
@@ -1084,6 +1095,13 @@ static int __btrfs_end_transaction(struct btrfs_trans_handle *trans,
 	if (trans->type & __TRANS_FREEZABLE)
 		sb_end_intwrite(info->sb);
 
+	/*
+	 * Uninhibit extent buffer writeback before decrementing num_writers,
+	 * since the decrement wakes the committing thread which needs all
+	 * buffers uninhibited to write them to disk.
+	 */
+	btrfs_uninhibit_all_eb_writeback(trans);
+
 	WARN_ON(cur_trans != info->running_transaction);
 	WARN_ON(atomic_read(&cur_trans->num_writers) < 1);
 	atomic_dec(&cur_trans->num_writers);
@@ -1571,7 +1589,7 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans,
 	 * recorded root will never be updated again, causing an outdated root
 	 * item.
 	 */
-	ret = record_root_in_trans(trans, src, 1);
+	ret = record_root_in_trans(trans, src, true);
 	if (ret)
 		return ret;
 
@@ -1594,16 +1612,16 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans,
 
 	ret = commit_fs_roots(trans);
 	if (ret)
-		goto out;
+		return ret;
 	ret = btrfs_qgroup_account_extents(trans);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	/* Now qgroup are all updated, we can inherit it to new qgroups */
 	ret = btrfs_qgroup_inherit(trans, btrfs_root_id(src), dst_objectid,
 				   btrfs_root_id(parent), inherit);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	/*
 	 * Now we do a simplified commit transaction, which will:
@@ -1619,23 +1637,22 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans,
 	 */
 	ret = commit_cowonly_roots(trans);
 	if (ret)
-		goto out;
+		return ret;
 	switch_commit_roots(trans);
 	ret = btrfs_write_and_wait_transaction(trans);
-	if (unlikely(ret))
+	if (unlikely(ret)) {
 		btrfs_err(fs_info,
 "error while writing out transaction during qgroup snapshot accounting: %d", ret);
+		return ret;
+	}
 
-out:
 	/*
 	 * Force parent root to be updated, as we recorded it before so its
 	 * last_trans == cur_transid.
 	 * Or it won't be committed again onto disk after later
 	 * insert_dir_item()
 	 */
-	if (!ret)
-		ret = record_root_in_trans(trans, parent, 1);
-	return ret;
+	return record_root_in_trans(trans, parent, true);
 }
 
 /*
@@ -1662,7 +1679,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 	BTRFS_PATH_AUTO_FREE(path);
 	struct btrfs_dir_item *dir_item;
 	struct extent_buffer *tmp;
-	struct extent_buffer *old;
+	struct extent_buffer *root_eb;
 	struct timespec64 cur_time;
 	int ret = 0;
 	u64 to_reserve = 0;
@@ -1719,7 +1736,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 				      trans->transid,
 				      trans->bytes_reserved, 1);
 	parent_root = parent_inode->root;
-	ret = record_root_in_trans(trans, parent_root, 0);
+	ret = record_root_in_trans(trans, parent_root, false);
 	if (unlikely(ret))
 		goto fail;
 	cur_time = current_time(&parent_inode->vfs_inode);
@@ -1737,7 +1754,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 	dir_item = btrfs_lookup_dir_item(NULL, parent_root, path,
 					 btrfs_ino(parent_inode),
 					 &fname.disk_name, 0);
-	if (unlikely(dir_item != NULL && !IS_ERR(dir_item))) {
+	if (!IS_ERR_OR_NULL(dir_item)) {
 		pending->error = -EEXIST;
 		goto dir_item_existed;
 	} else if (IS_ERR(dir_item)) {
@@ -1767,7 +1784,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 		goto fail;
 	}
 
-	ret = record_root_in_trans(trans, root, 0);
+	ret = record_root_in_trans(trans, root, false);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto fail;
@@ -1800,20 +1817,10 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 	btrfs_set_stack_timespec_nsec(&new_root_item->otime, cur_time.tv_nsec);
 	btrfs_set_root_otransid(new_root_item, trans->transid);
 
-	old = btrfs_lock_root_node(root);
-	ret = btrfs_cow_block(trans, root, old, NULL, 0, &old,
-			      BTRFS_NESTING_COW);
-	if (unlikely(ret)) {
-		btrfs_tree_unlock(old);
-		free_extent_buffer(old);
-		btrfs_abort_transaction(trans, ret);
-		goto fail;
-	}
-
-	ret = btrfs_copy_root(trans, root, old, &tmp, objectid);
-	/* clean up in any case */
-	btrfs_tree_unlock(old);
-	free_extent_buffer(old);
+	root_eb = btrfs_lock_root_node(root);
+	ret = btrfs_copy_root(trans, root, root_eb, &tmp, objectid);
+	btrfs_tree_unlock(root_eb);
+	free_extent_buffer(root_eb);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto fail;
@@ -1921,7 +1928,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 		 */
 		if (ret == -EOVERFLOW)
 			ret = 0;
-		if (unlikely(ret && ret != -EEXIST)) {
+		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			goto fail;
 		}
@@ -2127,6 +2134,7 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err)
 	if (!test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags))
 		btrfs_scrub_cancel(fs_info);
 
+	btrfs_uninhibit_all_eb_writeback(trans);
 	kmem_cache_free(btrfs_trans_handle_cachep, trans);
 }
 
@@ -2566,6 +2574,14 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 	    fs_info->cleaner_kthread)
 		wake_up_process(fs_info->cleaner_kthread);
 
+	/*
+	 * Uninhibit writeback on all extent buffers inhibited during this
+	 * transaction before writing them to disk. Inhibiting prevented
+	 * writeback while the transaction was building, but now we need
+	 * them written.
+	 */
+	btrfs_uninhibit_all_eb_writeback(trans);
+
 	ret = btrfs_write_and_wait_transaction(trans);
 	if (unlikely(ret)) {
 		btrfs_err(fs_info, "error while writing out transaction: %d", ret);
@@ -2573,7 +2589,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 		goto scrub_continue;
 	}
 
-	ret = write_all_supers(fs_info, 0);
+	ret = write_all_supers(trans);
 	/*
 	 * the super is written, we can safely allow the tree-loggers
 	 * to go about their business
diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h
index 18ef069197e5b..7d70fe4867587 100644
--- a/fs/btrfs/transaction.h
+++ b/fs/btrfs/transaction.h
@@ -12,6 +12,7 @@
 #include <linux/time64.h>
 #include <linux/mutex.h>
 #include <linux/wait.h>
+#include <linux/xarray.h>
 #include "btrfs_inode.h"
 #include "delayed-ref.h"
 
@@ -162,6 +163,8 @@ struct btrfs_trans_handle {
 	struct btrfs_fs_info *fs_info;
 	struct list_head new_bgs;
 	struct btrfs_block_rsv delayed_rsv;
+	/* Extent buffers with writeback inhibited by this handle. */
+	struct xarray writeback_inhibited_ebs;
 };
 
 /*
diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
index ac4c4573ee391..495d1dd61e66c 100644
--- a/fs/btrfs/tree-checker.c
+++ b/fs/btrfs/tree-checker.c
@@ -1284,10 +1284,27 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key,
 	}
 	if (unlikely(btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL)) {
 		generic_err(leaf, slot,
-			    "invalid root level, have %u expect [0, %u]",
+			    "invalid root drop_level, have %u expect [0, %u]",
 			    btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1);
 		return -EUCLEAN;
 	}
+	/*
+	 * If drop_progress.objectid is non-zero, a btrfs_drop_snapshot() was
+	 * interrupted and the resume point was recorded in drop_progress and
+	 * drop_level.  In that case drop_level must be >= 1: level 0 is the
+	 * leaf level and drop_snapshot never saves a checkpoint there (it
+	 * only records checkpoints at internal node levels in DROP_REFERENCE
+	 * stage).  A zero drop_level combined with a non-zero drop_progress
+	 * objectid indicates on-disk corruption and would cause a BUG_ON in
+	 * merge_reloc_root() and btrfs_drop_snapshot() at mount time.
+	 */
+	if (unlikely(btrfs_disk_key_objectid(&ri.drop_progress) != 0 &&
+		     btrfs_root_drop_level(&ri) == 0)) {
+		generic_err(leaf, slot,
+			    "invalid root drop_level 0 with non-zero drop_progress objectid %llu",
+			    btrfs_disk_key_objectid(&ri.drop_progress));
+		return -EUCLEAN;
+	}
 
 	/* Flags check */
 	if (unlikely(btrfs_root_flags(&ri) & ~valid_root_flags)) {
@@ -1928,6 +1945,119 @@ static int check_dev_extent_item(const struct extent_buffer *leaf,
 	return 0;
 }
 
+static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *key,
+				 int slot)
+{
+	struct btrfs_fs_info *fs_info = leaf->fs_info;
+	struct btrfs_free_space_info *fsi;
+	const u32 blocksize = fs_info->sectorsize;
+	u32 flags;
+
+	if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) {
+		generic_err(leaf, slot,
+		"free space info key objectid is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(!IS_ALIGNED(key->offset, blocksize))) {
+		generic_err(leaf, slot,
+		"free space info key offset is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(btrfs_item_size(leaf, slot) !=
+		     sizeof(struct btrfs_free_space_info))) {
+		generic_err(leaf, slot,
+		"invalid item size for free space info, has %u expect %zu",
+			    btrfs_item_size(leaf, slot),
+			    sizeof(struct btrfs_free_space_info));
+		return -EUCLEAN;
+	}
+	fsi = btrfs_item_ptr(leaf, slot, struct btrfs_free_space_info);
+	flags = btrfs_free_space_flags(leaf, fsi);
+	if (unlikely(flags & ~BTRFS_FREE_SPACE_FLAGS_MASK)) {
+		generic_err(leaf, slot,
+		"unknown flags for free space info, has 0x%x valid mask 0x%lx",
+			    flags, BTRFS_FREE_SPACE_FLAGS_MASK);
+		return -EUCLEAN;
+	}
+	if (unlikely(btrfs_free_space_extent_count(leaf, fsi) >
+		     key->offset >> fs_info->sectorsize_bits)) {
+		generic_err(leaf, slot,
+			    "suspicious extent count, has %u max valid %llu",
+			    btrfs_free_space_extent_count(leaf, fsi),
+			    key->offset >> fs_info->sectorsize_bits);
+		return -EUCLEAN;
+	}
+	return 0;
+}
+
+static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot)
+{
+	struct btrfs_fs_info *fs_info = leaf->fs_info;
+	const u32 blocksize = fs_info->sectorsize;
+
+	if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) {
+		generic_err(leaf, slot,
+		"free space extent key objectid is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(!IS_ALIGNED(key->offset, blocksize))) {
+		generic_err(leaf, slot,
+		"free space extent key offset is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(btrfs_item_size(leaf, slot) != 0)) {
+		generic_err(leaf, slot,
+			    "invalid item size for free space info, has %u expect 0",
+			    btrfs_item_size(leaf, slot));
+		return -EUCLEAN;
+	}
+	return 0;
+}
+
+static int check_free_space_bitmap(struct extent_buffer *leaf,
+				   struct btrfs_key *key, int slot)
+{
+	struct btrfs_fs_info *fs_info = leaf->fs_info;
+	const u32 blocksize = fs_info->sectorsize;
+	u32 expected_item_size;
+
+	if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) {
+		generic_err(leaf, slot,
+		"free space bitmap key objectid is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(!IS_ALIGNED(key->offset, blocksize))) {
+		generic_err(leaf, slot,
+		"free space bitmap key offset is not aligned to %u, has " BTRFS_KEY_FMT,
+			    blocksize, BTRFS_KEY_FMT_VALUE(key));
+		return -EUCLEAN;
+	}
+	if (unlikely(key->offset == 0)) {
+		generic_err(leaf, slot, "free space bitmap length is 0");
+		return -EUCLEAN;
+	}
+	/*
+	 * The item must hold exactly the right number of bitmap bytes for the
+	 * range described by key->offset.  A mismatch means the item was
+	 * truncated or the key is corrupt; either way the bitmap data is not
+	 * safe to access.
+	 */
+	expected_item_size = DIV_ROUND_UP(key->offset >> fs_info->sectorsize_bits,
+					  BITS_PER_BYTE);
+	if (unlikely(btrfs_item_size(leaf, slot) != expected_item_size)) {
+		generic_err(leaf, slot,
+			    "invalid item size for free space bitmap, has %u expect %u",
+			    btrfs_item_size(leaf, slot), expected_item_size);
+		return -EUCLEAN;
+	}
+	return 0;
+}
+
 /*
  * Common point to switch the item-specific validation.
  */
@@ -1991,6 +2121,15 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf,
 	case BTRFS_RAID_STRIPE_KEY:
 		ret = check_raid_stripe_extent(leaf, key, slot);
 		break;
+	case BTRFS_FREE_SPACE_INFO_KEY:
+		ret = check_free_space_info(leaf, key, slot);
+		break;
+	case BTRFS_FREE_SPACE_EXTENT_KEY:
+		ret = check_free_space_extent(leaf, key, slot);
+		break;
+	case BTRFS_FREE_SPACE_BITMAP_KEY:
+		ret = check_free_space_bitmap(leaf, key, slot);
+		break;
 	}
 
 	if (unlikely(ret))
diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
index 780a06d592409..6cf2828ba5001 100644
--- a/fs/btrfs/tree-log.c
+++ b/fs/btrfs/tree-log.c
@@ -457,7 +457,7 @@ static int process_one_buffer(struct extent_buffer *eb,
 			return ret;
 		}
 
-		if (btrfs_buffer_uptodate(eb, gen, false) && level == 0) {
+		if (btrfs_buffer_uptodate(eb, gen, false, NULL) && level == 0) {
 			ret = btrfs_exclude_logged_extents(eb);
 			if (ret)
 				btrfs_abort_transaction(trans, ret);
@@ -984,6 +984,13 @@ static noinline int replay_one_extent(struct walk_control *wc)
 
 		sums = list_first_entry(&ordered_sums, struct btrfs_ordered_sum, list);
 		csum_root = btrfs_csum_root(fs_info, sums->logical);
+		if (unlikely(!csum_root)) {
+			btrfs_err(fs_info,
+				  "missing csum root for extent at bytenr %llu",
+				  sums->logical);
+			ret = -EUCLEAN;
+		}
+
 		if (!ret) {
 			ret = btrfs_del_csums(trans, csum_root, sums->logical,
 					      sums->len);
@@ -996,7 +1003,7 @@ static noinline int replay_one_extent(struct walk_control *wc)
 						       btrfs_root_id(root));
 		}
 		if (!ret) {
-			ret = btrfs_csum_file_blocks(trans, csum_root, sums);
+			ret = btrfs_insert_data_csums(trans, csum_root, sums);
 			if (ret)
 				btrfs_abort_log_replay(wc, ret,
 	       "failed to add csums for range [%llu, %llu) inode %llu root %llu",
@@ -1704,7 +1711,7 @@ static noinline int add_inode_ref(struct walk_control *wc)
 			}
 
 			/* insert our name */
-			ret = btrfs_add_link(trans, dir, inode, &name, 0, ref_index);
+			ret = btrfs_add_link(trans, dir, inode, &name, false, ref_index);
 			if (ret) {
 				btrfs_abort_log_replay(wc, ret,
 "failed to add link for inode %llu in dir %llu ref_index %llu name %.*s root %llu",
@@ -2052,7 +2059,7 @@ static noinline int insert_one_name(struct btrfs_trans_handle *trans,
 		return PTR_ERR(dir);
 	}
 
-	ret = btrfs_add_link(trans, dir, inode, name, 1, index);
+	ret = btrfs_add_link(trans, dir, inode, name, true, index);
 
 	/* FIXME, put inode into FIXUP list */
 
@@ -3569,7 +3576,7 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans,
 
 	btrfs_set_super_log_root(fs_info->super_for_commit, log_root_start);
 	btrfs_set_super_log_root_level(fs_info->super_for_commit, log_root_level);
-	ret = write_all_supers(fs_info, 1);
+	ret = write_all_supers(trans);
 	mutex_unlock(&fs_info->tree_log_mutex);
 	if (unlikely(ret)) {
 		btrfs_set_log_full_commit(trans);
@@ -4606,40 +4613,52 @@ static int truncate_inode_items(struct btrfs_trans_handle *trans,
 static void fill_inode_item(struct btrfs_trans_handle *trans,
 			    struct extent_buffer *leaf,
 			    struct btrfs_inode_item *item,
-			    struct inode *inode, bool log_inode_only,
+			    struct btrfs_inode *inode, bool log_inode_only,
 			    u64 logged_isize)
 {
+	struct inode *vfs_inode = &inode->vfs_inode;
+	u64 gen = inode->generation;
 	u64 flags;
 
 	if (log_inode_only) {
-		/* set the generation to zero so the recover code
-		 * can tell the difference between an logging
-		 * just to say 'this inode exists' and a logging
-		 * to say 'update this inode with these values'
+		/*
+		 * Set the generation to zero so the recover code can tell the
+		 * difference between a logging just to say 'this inode exists'
+		 * and a logging to say 'update this inode with these values'.
+		 * But only if the inode was not already logged before.
+		 * We access ->logged_trans directly since it was already set
+		 * up in the call chain by btrfs_log_inode(), and data_race()
+		 * to avoid false alerts from KCSAN and since it was set already
+		 * and one can set it to 0 since that only happens on eviction
+		 * and we are holding a ref on the inode.
 		 */
-		btrfs_set_inode_generation(leaf, item, 0);
+		ASSERT(data_race(inode->logged_trans) > 0);
+		if (data_race(inode->logged_trans) < trans->transid)
+			gen = 0;
+
 		btrfs_set_inode_size(leaf, item, logged_isize);
 	} else {
-		btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation);
-		btrfs_set_inode_size(leaf, item, inode->i_size);
+		btrfs_set_inode_size(leaf, item, vfs_inode->i_size);
 	}
 
-	btrfs_set_inode_uid(leaf, item, i_uid_read(inode));
-	btrfs_set_inode_gid(leaf, item, i_gid_read(inode));
-	btrfs_set_inode_mode(leaf, item, inode->i_mode);
-	btrfs_set_inode_nlink(leaf, item, inode->i_nlink);
+	btrfs_set_inode_generation(leaf, item, gen);
+
+	btrfs_set_inode_uid(leaf, item, i_uid_read(vfs_inode));
+	btrfs_set_inode_gid(leaf, item, i_gid_read(vfs_inode));
+	btrfs_set_inode_mode(leaf, item, vfs_inode->i_mode);
+	btrfs_set_inode_nlink(leaf, item, vfs_inode->i_nlink);
 
-	btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(inode));
-	btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(inode));
+	btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(vfs_inode));
+	btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(vfs_inode));
 
-	btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(inode));
-	btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(inode));
+	btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(vfs_inode));
+	btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(vfs_inode));
 
-	btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(inode));
-	btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(inode));
+	btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(vfs_inode));
+	btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(vfs_inode));
 
-	btrfs_set_timespec_sec(leaf, &item->otime, BTRFS_I(inode)->i_otime_sec);
-	btrfs_set_timespec_nsec(leaf, &item->otime, BTRFS_I(inode)->i_otime_nsec);
+	btrfs_set_timespec_sec(leaf, &item->otime, inode->i_otime_sec);
+	btrfs_set_timespec_nsec(leaf, &item->otime, inode->i_otime_nsec);
 
 	/*
 	 * We do not need to set the nbytes field, in fact during a fast fsync
@@ -4650,11 +4669,10 @@ static void fill_inode_item(struct btrfs_trans_handle *trans,
 	 * inode item in subvolume tree as needed (see overwrite_item()).
 	 */
 
-	btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(inode));
+	btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(vfs_inode));
 	btrfs_set_inode_transid(leaf, item, trans->transid);
-	btrfs_set_inode_rdev(leaf, item, inode->i_rdev);
-	flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags,
-					  BTRFS_I(inode)->ro_flags);
+	btrfs_set_inode_rdev(leaf, item, vfs_inode->i_rdev);
+	flags = btrfs_inode_combine_flags(inode->flags, inode->ro_flags);
 	btrfs_set_inode_flags(leaf, item, flags);
 	btrfs_set_inode_block_group(leaf, item, 0);
 }
@@ -4701,8 +4719,7 @@ static int log_inode_item(struct btrfs_trans_handle *trans,
 		return ret;
 	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
 				    struct btrfs_inode_item);
-	fill_inode_item(trans, path->nodes[0], inode_item, &inode->vfs_inode,
-			false, 0);
+	fill_inode_item(trans, path->nodes[0], inode_item, inode, false, 0);
 	btrfs_release_path(path);
 	return 0;
 }
@@ -4722,7 +4739,7 @@ static int log_csums(struct btrfs_trans_handle *trans,
 	 * worry about logging checksum items with overlapping ranges.
 	 */
 	if (inode->last_reflink_trans < trans->transid)
-		return btrfs_csum_file_blocks(trans, log_root, sums);
+		return btrfs_insert_data_csums(trans, log_root, sums);
 
 	/*
 	 * Serialize logging for checksums. This is to avoid racing with the
@@ -4745,7 +4762,7 @@ static int log_csums(struct btrfs_trans_handle *trans,
 	 */
 	ret = btrfs_del_csums(trans, log_root, sums->logical, sums->len);
 	if (!ret)
-		ret = btrfs_csum_file_blocks(trans, log_root, sums);
+		ret = btrfs_insert_data_csums(trans, log_root, sums);
 
 	btrfs_unlock_extent(&log_root->log_csum_range, sums->logical, lock_end,
 			    &cached_state);
@@ -4890,6 +4907,13 @@ static noinline int copy_items(struct btrfs_trans_handle *trans,
 		}
 
 		csum_root = btrfs_csum_root(trans->fs_info, disk_bytenr);
+		if (unlikely(!csum_root)) {
+			btrfs_err(trans->fs_info,
+				  "missing csum root for extent at bytenr %llu",
+				  disk_bytenr);
+			return -EUCLEAN;
+		}
+
 		disk_bytenr += extent_offset;
 		ret = btrfs_lookup_csums_list(csum_root, disk_bytenr,
 					      disk_bytenr + extent_num_bytes - 1,
@@ -4964,8 +4988,7 @@ static noinline int copy_items(struct btrfs_trans_handle *trans,
 			inode_item = btrfs_item_ptr(dst_path->nodes[0], dst_slot,
 						    struct btrfs_inode_item);
 			fill_inode_item(trans, dst_path->nodes[0], inode_item,
-					&inode->vfs_inode,
-					inode_only == LOG_INODE_EXISTS,
+					inode, inode_only == LOG_INODE_EXISTS,
 					logged_isize);
 		} else {
 			copy_extent_buffer(dst_path->nodes[0], src, dst_offset,
@@ -5063,7 +5086,7 @@ static int log_extent_csums(struct btrfs_trans_handle *trans,
 		if (test_and_set_bit(BTRFS_ORDERED_LOGGED_CSUM, &ordered->flags))
 			continue;
 
-		list_for_each_entry(sums, &ordered->list, list) {
+		list_for_each_entry(sums, &ordered->csum_list, list) {
 			ret = log_csums(trans, inode, log_root, sums);
 			if (ret)
 				return ret;
@@ -5086,6 +5109,13 @@ static int log_extent_csums(struct btrfs_trans_handle *trans,
 	/* block start is already adjusted for the file extent offset. */
 	block_start = btrfs_extent_map_block_start(em);
 	csum_root = btrfs_csum_root(trans->fs_info, block_start);
+	if (unlikely(!csum_root)) {
+		btrfs_err(trans->fs_info,
+			  "missing csum root for extent at bytenr %llu",
+			  block_start);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_lookup_csums_list(csum_root, block_start + csum_offset,
 				      block_start + csum_offset + csum_len - 1,
 				      &ordered_sums, false);
@@ -5427,42 +5457,63 @@ static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
-static int logged_inode_size(struct btrfs_root *log, struct btrfs_inode *inode,
-			     struct btrfs_path *path, u64 *size_ret)
+static int get_inode_size_to_log(struct btrfs_trans_handle *trans,
+				 struct btrfs_inode *inode,
+				 struct btrfs_path *path, u64 *size_ret)
 {
 	struct btrfs_key key;
+	struct btrfs_inode_item *item;
 	int ret;
 
 	key.objectid = btrfs_ino(inode);
 	key.type = BTRFS_INODE_ITEM_KEY;
 	key.offset = 0;
 
-	ret = btrfs_search_slot(NULL, log, &key, path, 0, 0);
-	if (ret < 0) {
-		return ret;
-	} else if (ret > 0) {
-		*size_ret = 0;
-	} else {
-		struct btrfs_inode_item *item;
+	/*
+	 * Our caller called inode_logged(), so logged_trans is up to date.
+	 * Use data_race() to silence any warning from KCSAN. Once logged_trans
+	 * is set, it can only be reset to 0 after inode eviction.
+	 */
+	if (data_race(inode->logged_trans) == trans->transid) {
+		ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0);
+	} else if (inode->generation < trans->transid) {
+		path->search_commit_root = true;
+		path->skip_locking = true;
+		ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0);
+		path->search_commit_root = false;
+		path->skip_locking = false;
 
-		item = btrfs_item_ptr(path->nodes[0], path->slots[0],
-				      struct btrfs_inode_item);
-		*size_ret = btrfs_inode_size(path->nodes[0], item);
-		/*
-		 * If the in-memory inode's i_size is smaller then the inode
-		 * size stored in the btree, return the inode's i_size, so
-		 * that we get a correct inode size after replaying the log
-		 * when before a power failure we had a shrinking truncate
-		 * followed by addition of a new name (rename / new hard link).
-		 * Otherwise return the inode size from the btree, to avoid
-		 * data loss when replaying a log due to previously doing a
-		 * write that expands the inode's size and logging a new name
-		 * immediately after.
-		 */
-		if (*size_ret > inode->vfs_inode.i_size)
-			*size_ret = inode->vfs_inode.i_size;
+	} else {
+		*size_ret = 0;
+		return 0;
 	}
 
+	/*
+	 * If the inode was logged before or is from a past transaction, then
+	 * its inode item must exist in the log root or in the commit root.
+	 */
+	ASSERT(ret <= 0);
+	if (WARN_ON_ONCE(ret > 0))
+		ret = -ENOENT;
+
+	if (ret < 0)
+		return ret;
+
+	item = btrfs_item_ptr(path->nodes[0], path->slots[0],
+			      struct btrfs_inode_item);
+	*size_ret = btrfs_inode_size(path->nodes[0], item);
+	/*
+	 * If the in-memory inode's i_size is smaller then the inode size stored
+	 * in the btree, return the inode's i_size, so that we get a correct
+	 * inode size after replaying the log when before a power failure we had
+	 * a shrinking truncate followed by addition of a new name (rename / new
+	 * hard link). Otherwise return the inode size from the btree, to avoid
+	 * data loss when replaying a log due to previously doing a write that
+	 * expands the inode's size and logging a new name immediately after.
+	 */
+	if (*size_ret > inode->vfs_inode.i_size)
+		*size_ret = inode->vfs_inode.i_size;
+
 	btrfs_release_path(path);
 	return 0;
 }
@@ -5750,7 +5801,7 @@ static int btrfs_check_ref_name_override(struct extent_buffer *eb,
 		name_str.len = this_name_len;
 		di = btrfs_lookup_dir_item(NULL, inode->root, search_path,
 				parent, &name_str, 0);
-		if (di && !IS_ERR(di)) {
+		if (!IS_ERR_OR_NULL(di)) {
 			struct btrfs_key di_key;
 
 			btrfs_dir_item_key_to_cpu(search_path->nodes[0],
@@ -6195,6 +6246,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans,
 				  struct btrfs_root *root,
 				  struct btrfs_log_ctx *ctx)
 {
+	const bool orig_log_new_dentries = ctx->log_new_dentries;
 	int ret = 0;
 
 	/*
@@ -6256,7 +6308,11 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans,
 			 * dir index key range logged for the directory. So we
 			 * must make sure the deletion is recorded.
 			 */
+			ctx->log_new_dentries = false;
 			ret = btrfs_log_inode(trans, inode, LOG_INODE_ALL, ctx);
+			if (!ret && ctx->log_new_dentries)
+				ret = log_new_dir_dentries(trans, inode, ctx);
+
 			btrfs_add_delayed_iput(inode);
 			if (ret)
 				break;
@@ -6291,6 +6347,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans,
 			break;
 	}
 
+	ctx->log_new_dentries = orig_log_new_dentries;
 	ctx->logging_conflict_inodes = false;
 	if (ret)
 		free_conflicting_inodes(ctx);
@@ -6969,7 +7026,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans,
 			ret = drop_inode_items(trans, log, path, inode,
 					       BTRFS_XATTR_ITEM_KEY);
 	} else {
-		if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) {
+		if (inode_only == LOG_INODE_EXISTS) {
 			/*
 			 * Make sure the new inode item we write to the log has
 			 * the same isize as the current one (if it exists).
@@ -6983,7 +7040,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans,
 			 * (zeroes), as if an expanding truncate happened,
 			 * instead of getting a file of 4Kb only.
 			 */
-			ret = logged_inode_size(log, inode, path, &logged_isize);
+			ret = get_inode_size_to_log(trans, inode, path, &logged_isize);
 			if (ret)
 				goto out_unlock;
 		}
diff --git a/fs/btrfs/tree-mod-log.c b/fs/btrfs/tree-mod-log.c
index 603c1457130eb..a8094928f4c9b 100644
--- a/fs/btrfs/tree-mod-log.c
+++ b/fs/btrfs/tree-mod-log.c
@@ -1042,12 +1042,10 @@ struct extent_buffer *btrfs_get_old_root(struct btrfs_root *root, u64 time_seq)
 		check.owner_root = btrfs_root_id(root);
 
 		old = read_tree_block(fs_info, logical, &check);
-		if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) {
-			if (!IS_ERR(old))
-				free_extent_buffer(old);
+		if (WARN_ON(IS_ERR(old))) {
 			btrfs_warn(fs_info,
-				   "failed to read tree block %llu from get_old_root",
-				   logical);
+				   "failed to read tree block %llu from get_old_root: %ld",
+				   logical, PTR_ERR(old));
 		} else {
 			struct tree_mod_elem *tm2;
 
diff --git a/fs/btrfs/uuid-tree.c b/fs/btrfs/uuid-tree.c
index 43c17a1d34513..467dff7212d69 100644
--- a/fs/btrfs/uuid-tree.c
+++ b/fs/btrfs/uuid-tree.c
@@ -35,7 +35,7 @@ static int btrfs_uuid_tree_lookup(struct btrfs_root *uuid_root, const u8 *uuid,
 	struct btrfs_key key;
 
 	if (WARN_ON_ONCE(!uuid_root))
-		return -ENOENT;
+		return -EINVAL;
 
 	path = btrfs_alloc_path();
 	if (!path)
@@ -92,9 +92,6 @@ int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, const u8 *uuid, u8 typ
 	if (ret != -ENOENT)
 		return ret;
 
-	if (WARN_ON_ONCE(!uuid_root))
-		return -EINVAL;
-
 	btrfs_uuid_to_key(uuid, type, &key);
 
 	path = btrfs_alloc_path();
@@ -516,7 +513,7 @@ int btrfs_uuid_scan_kthread(void *data)
 
 out:
 	btrfs_free_path(path);
-	if (trans && !IS_ERR(trans))
+	if (!IS_ERR_OR_NULL(trans))
 		btrfs_end_transaction(trans);
 	if (ret)
 		btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 648bb09fc4166..8dab290b36946 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -392,6 +392,7 @@ static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
 	INIT_LIST_HEAD(&fs_devs->alloc_list);
 	INIT_LIST_HEAD(&fs_devs->fs_list);
 	INIT_LIST_HEAD(&fs_devs->seed_list);
+	spin_lock_init(&fs_devs->per_profile_lock);
 
 	if (fsid) {
 		memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
@@ -694,7 +695,7 @@ static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
 			set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
 	}
 
-	if (!bdev_nonrot(file_bdev(bdev_file)))
+	if (bdev_rot(file_bdev(bdev_file)))
 		fs_devices->rotating = true;
 
 	if (bdev_max_discard_sectors(file_bdev(bdev_file)))
@@ -2339,6 +2340,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 		mutex_lock(&fs_info->chunk_mutex);
 		list_del_init(&device->dev_alloc_list);
 		device->fs_devices->rw_devices--;
+		btrfs_update_per_profile_avail(fs_info);
 		mutex_unlock(&fs_info->chunk_mutex);
 	}
 
@@ -2450,6 +2452,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 		list_add(&device->dev_alloc_list,
 			 &fs_devices->alloc_list);
 		device->fs_devices->rw_devices++;
+		btrfs_update_per_profile_avail(fs_info);
 		mutex_unlock(&fs_info->chunk_mutex);
 	}
 	return ret;
@@ -2919,7 +2922,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 
 	atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
 
-	if (!bdev_nonrot(device->bdev))
+	if (bdev_rot(device->bdev))
 		fs_devices->rotating = true;
 
 	orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
@@ -2937,6 +2940,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 	 */
 	btrfs_clear_space_info_full(fs_info);
 
+	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 
 	/* Add sysfs device entry */
@@ -2947,6 +2951,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 	if (seeding_dev) {
 		mutex_lock(&fs_info->chunk_mutex);
 		ret = init_first_rw_device(trans);
+		btrfs_update_per_profile_avail(fs_info);
 		mutex_unlock(&fs_info->chunk_mutex);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
@@ -3029,6 +3034,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 				    orig_super_total_bytes);
 	btrfs_set_super_num_devices(fs_info->super_copy,
 				    orig_super_num_devices);
+	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
 error_trans:
@@ -3121,6 +3127,7 @@ int btrfs_grow_device(struct btrfs_trans_handle *trans,
 	if (list_empty(&device->post_commit_list))
 		list_add_tail(&device->post_commit_list,
 			      &trans->transaction->dev_update_list);
+	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 
 	btrfs_reserve_chunk_metadata(trans, false);
@@ -3497,6 +3504,7 @@ int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
 		}
 	}
 
+	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 	trans->removing_chunk = false;
 
@@ -3587,7 +3595,7 @@ int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset, bool v
 
 	/* step one, relocate all the extents inside this chunk */
 	btrfs_scrub_pause(fs_info);
-	ret = btrfs_relocate_block_group(fs_info, chunk_offset, true);
+	ret = btrfs_relocate_block_group(fs_info, chunk_offset, verbose);
 	btrfs_scrub_continue(fs_info);
 	if (ret) {
 		/*
@@ -4277,20 +4285,29 @@ static int balance_remap_chunks(struct btrfs_fs_info *fs_info, struct btrfs_path
 end:
 	while (!list_empty(chunks)) {
 		bool is_unused;
+		struct btrfs_block_group *bg;
 
 		rci = list_first_entry(chunks, struct remap_chunk_info, list);
 
-		spin_lock(&rci->bg->lock);
-		is_unused = !btrfs_is_block_group_used(rci->bg);
-		spin_unlock(&rci->bg->lock);
+		bg = rci->bg;
+		if (bg) {
+			/*
+			 * This is a bit racy and the 'used' status can change
+			 * but this is not a problem as later functions will
+			 * verify it again.
+			 */
+			spin_lock(&bg->lock);
+			is_unused = !btrfs_is_block_group_used(bg);
+			spin_unlock(&bg->lock);
 
-		if (is_unused)
-			btrfs_mark_bg_unused(rci->bg);
+			if (is_unused)
+				btrfs_mark_bg_unused(bg);
 
-		if (rci->made_ro)
-			btrfs_dec_block_group_ro(rci->bg);
+			if (rci->made_ro)
+				btrfs_dec_block_group_ro(bg);
 
-		btrfs_put_block_group(rci->bg);
+			btrfs_put_block_group(bg);
+		}
 
 		list_del(&rci->list);
 		kfree(rci);
@@ -5191,6 +5208,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
 		atomic64_sub(free_diff, &fs_info->free_chunk_space);
 	}
 
+	btrfs_update_per_profile_avail(fs_info);
 	/*
 	 * Once the device's size has been set to the new size, ensure all
 	 * in-memory chunks are synced to disk so that the loop below sees them
@@ -5306,6 +5324,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
 	WARN_ON(diff > old_total);
 	btrfs_set_super_total_bytes(super_copy,
 			round_down(old_total - diff, fs_info->sectorsize));
+	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 
 	btrfs_reserve_chunk_metadata(trans, false);
@@ -5378,6 +5397,168 @@ static int btrfs_cmp_device_info(const void *a, const void *b)
 	return 0;
 }
 
+/*
+ * Return 0 if we allocated any virtual(*) chunk, and restore the size to
+ * @allocated.
+ * Return -ENOSPC if we have no more space to allocate virtual chunk
+ *
+ * *: A virtual chunk is a chunk that only exists during per-profile available
+ *    estimation.
+ *    Those numbers won't really take on-disk space, but only to emulate
+ *    chunk allocator behavior to get accurate estimation on available space.
+ *
+ *    Another difference is, a virtual chunk has no size limit and doesn't care
+ *    about holes in the device tree, allowing us to exhaust device space
+ *    much faster.
+ */
+static int alloc_virtual_chunk(struct btrfs_fs_info *fs_info,
+			       struct btrfs_device_info *devices_info,
+			       enum btrfs_raid_types type,
+			       u64 *allocated)
+{
+	const struct btrfs_raid_attr *raid_attr = &btrfs_raid_array[type];
+	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+	struct btrfs_device *device;
+	u64 stripe_size;
+	int ndevs = 0;
+
+	lockdep_assert_held(&fs_info->chunk_mutex);
+
+	/* Go through devices to collect their unallocated space. */
+	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
+		u64 avail;
+
+		if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
+					&device->dev_state) ||
+		    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
+			continue;
+
+		if (device->total_bytes > device->bytes_used +
+				device->per_profile_allocated)
+			avail = device->total_bytes - device->bytes_used -
+				device->per_profile_allocated;
+		else
+			avail = 0;
+
+		avail = round_down(avail, fs_info->sectorsize);
+
+		/* And exclude the [0, 1M) reserved space. */
+		if (avail > BTRFS_DEVICE_RANGE_RESERVED)
+			avail -= BTRFS_DEVICE_RANGE_RESERVED;
+		else
+			avail = 0;
+
+		/*
+		 * Not enough to support a single stripe, this device
+		 * can not be utilized for chunk allocation.
+		 */
+		if (avail < BTRFS_STRIPE_LEN)
+			continue;
+
+		/*
+		 * Unlike chunk allocator, we don't care about stripe or hole
+		 * size, so here we use @avail directly.
+		 */
+		devices_info[ndevs].dev_offset = 0;
+		devices_info[ndevs].total_avail = avail;
+		devices_info[ndevs].max_avail = avail;
+		devices_info[ndevs].dev = device;
+		++ndevs;
+	}
+	sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
+	     btrfs_cmp_device_info, NULL);
+	ndevs = rounddown(ndevs, raid_attr->devs_increment);
+	if (ndevs < raid_attr->devs_min)
+		return -ENOSPC;
+	if (raid_attr->devs_max)
+		ndevs = min(ndevs, (int)raid_attr->devs_max);
+	else
+		ndevs = min(ndevs, (int)BTRFS_MAX_DEVS(fs_info));
+
+	/*
+	 * Stripe size will be determined by the device with the least
+	 * unallocated space.
+	 */
+	stripe_size = devices_info[ndevs - 1].total_avail;
+
+	for (int i = 0; i < ndevs; i++)
+		devices_info[i].dev->per_profile_allocated += stripe_size;
+	*allocated = div_u64(stripe_size * (ndevs - raid_attr->nparity),
+			     raid_attr->ncopies);
+	return 0;
+}
+
+static int calc_one_profile_avail(struct btrfs_fs_info *fs_info,
+				  enum btrfs_raid_types type,
+				  u64 *result_ret)
+{
+	struct btrfs_device_info *devices_info = NULL;
+	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+	struct btrfs_device *device;
+	u64 allocated;
+	u64 result = 0;
+	int ret = 0;
+
+	lockdep_assert_held(&fs_info->chunk_mutex);
+	ASSERT(type >= 0 && type < BTRFS_NR_RAID_TYPES);
+
+	/* Not enough devices, quick exit, just update the result. */
+	if (fs_devices->rw_devices < btrfs_raid_array[type].devs_min) {
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	devices_info = kzalloc_objs(*devices_info, fs_devices->rw_devices, GFP_NOFS);
+	if (!devices_info) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	/* Clear virtual chunk used space for each device. */
+	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list)
+		device->per_profile_allocated = 0;
+
+	while (!alloc_virtual_chunk(fs_info, devices_info, type, &allocated))
+		result += allocated;
+
+out:
+	kfree(devices_info);
+	if (ret < 0 && ret != -ENOSPC)
+		return ret;
+	*result_ret = result;
+	return 0;
+}
+
+/* Update the per-profile available space array. */
+void btrfs_update_per_profile_avail(struct btrfs_fs_info *fs_info)
+{
+	u64 results[BTRFS_NR_RAID_TYPES];
+	int ret;
+
+	/*
+	 * Zoned is more complex as we can not simply get the amount of
+	 * available space for each device.
+	 */
+	if (btrfs_is_zoned(fs_info))
+		goto error;
+
+	for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
+		ret = calc_one_profile_avail(fs_info, i, &results[i]);
+		if (ret < 0)
+			goto error;
+	}
+
+	spin_lock(&fs_info->fs_devices->per_profile_lock);
+	for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++)
+		fs_info->fs_devices->per_profile_avail[i] = results[i];
+	spin_unlock(&fs_info->fs_devices->per_profile_lock);
+	return;
+error:
+	spin_lock(&fs_info->fs_devices->per_profile_lock);
+	for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++)
+		fs_info->fs_devices->per_profile_avail[i] = U64_MAX;
+	spin_unlock(&fs_info->fs_devices->per_profile_lock);
+}
+
 static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
 {
 	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
@@ -5855,6 +6036,8 @@ static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
 	check_raid56_incompat_flag(info, type);
 	check_raid1c34_incompat_flag(info, type);
 
+	btrfs_update_per_profile_avail(info);
+
 	return block_group;
 }
 
@@ -5892,8 +6075,7 @@ struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
 	ctl.space_info = space_info;
 	init_alloc_chunk_ctl(fs_devices, &ctl);
 
-	devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
-			       GFP_NOFS);
+	devices_info = kzalloc_objs(*devices_info, fs_devices->rw_devices, GFP_NOFS);
 	if (!devices_info)
 		return ERR_PTR(-ENOMEM);
 
@@ -8429,7 +8611,14 @@ int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
 	}
 
 	/* Ensure all chunks have corresponding dev extents */
-	return verify_chunk_dev_extent_mapping(fs_info);
+	ret = verify_chunk_dev_extent_mapping(fs_info);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&fs_info->chunk_mutex);
+	btrfs_update_per_profile_avail(fs_info);
+	mutex_unlock(&fs_info->chunk_mutex);
+	return 0;
 }
 
 /*
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 8288d79372a56..0082c166af91f 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -22,6 +22,7 @@
 #include <uapi/linux/btrfs_tree.h>
 #include "messages.h"
 #include "extent-io-tree.h"
+#include "fs.h"
 
 struct block_device;
 struct bdev_handle;
@@ -213,6 +214,12 @@ struct btrfs_device {
 
 	/* Bandwidth limit for scrub, in bytes */
 	u64 scrub_speed_max;
+
+	/*
+	 * A temporary number of allocated space during per-profile
+	 * available space calculation.
+	 */
+	u64 per_profile_allocated;
 };
 
 /*
@@ -458,6 +465,15 @@ struct btrfs_fs_devices {
 	/* Device to be used for reading in case of RAID1. */
 	u64 read_devid;
 #endif
+
+	/*
+	 * Each value indicates the available space for that profile.
+	 * U64_MAX means the estimation is unavailable.
+	 *
+	 * Protected by per_profile_lock;
+	 */
+	u64 per_profile_avail[BTRFS_NR_RAID_TYPES];
+	spinlock_t per_profile_lock;
 };
 
 #define BTRFS_MAX_DEVS(info) ((BTRFS_MAX_ITEM_SIZE(info)	\
@@ -887,6 +903,24 @@ int btrfs_bg_type_to_factor(u64 flags);
 const char *btrfs_bg_type_to_raid_name(u64 flags);
 int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info);
 bool btrfs_verify_dev_items(const struct btrfs_fs_info *fs_info);
+void btrfs_update_per_profile_avail(struct btrfs_fs_info *fs_info);
+
+static inline bool btrfs_get_per_profile_avail(struct btrfs_fs_info *fs_info,
+					       u64 profile, u64 *avail_ret)
+{
+	enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(profile);
+	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+	bool uptodate = false;
+
+	spin_lock(&fs_devices->per_profile_lock);
+	if (fs_devices->per_profile_avail[index] != U64_MAX) {
+		uptodate = true;
+		*avail_ret = fs_devices->per_profile_avail[index];
+	}
+	spin_unlock(&fs_info->fs_devices->per_profile_lock);
+	return uptodate;
+}
+
 bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical);
 
 bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr);
diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c
index 1b7544aa88a2b..1a5093525e32b 100644
--- a/fs/btrfs/zlib.c
+++ b/fs/btrfs/zlib.c
@@ -349,7 +349,7 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	int wbits = MAX_WBITS;
 	char *data_in;
 	size_t total_out = 0;
-	size_t srclen = cb->compressed_len;
+	const size_t srclen = bio_get_size(&cb->bbio.bio);
 	unsigned long buf_start;
 
 	bio_first_folio(&fi, &cb->bbio.bio, 0);
diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c
index 39930d99943c1..f7ca01a32a031 100644
--- a/fs/btrfs/zoned.c
+++ b/fs/btrfs/zoned.c
@@ -337,7 +337,10 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
 	if (!btrfs_fs_incompat(fs_info, ZONED))
 		return 0;
 
-	mutex_lock(&fs_devices->device_list_mutex);
+	/*
+	 * No need to take the device_list mutex here, we're still in the mount
+	 * path and devices cannot be added to or removed from the list yet.
+	 */
 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
 		/* We can skip reading of zone info for missing devices */
 		if (!device->bdev)
@@ -347,7 +350,6 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
 		if (ret)
 			break;
 	}
-	mutex_unlock(&fs_devices->device_list_mutex);
 
 	return ret;
 }
@@ -1259,6 +1261,13 @@ static int calculate_alloc_pointer(struct btrfs_block_group *cache,
 	key.offset = 0;
 
 	root = btrfs_extent_root(fs_info, key.objectid);
+	if (unlikely(!root)) {
+		btrfs_err(fs_info,
+			  "missing extent root for extent at bytenr %llu",
+			  key.objectid);
+		return -EUCLEAN;
+	}
+
 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 	/* We should not find the exact match */
 	if (unlikely(!ret))
@@ -1690,8 +1699,7 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg,
 		return -EINVAL;
 	}
 
-	raid0_allocs = kcalloc(map->num_stripes / map->sub_stripes, sizeof(*raid0_allocs),
-			       GFP_NOFS);
+	raid0_allocs = kzalloc_objs(*raid0_allocs, map->num_stripes / map->sub_stripes, GFP_NOFS);
 	if (!raid0_allocs)
 		return -ENOMEM;
 
@@ -1909,7 +1917,7 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new)
 
 	cache->physical_map = map;
 
-	zone_info = kcalloc(map->num_stripes, sizeof(*zone_info), GFP_NOFS);
+	zone_info = kzalloc_objs(*zone_info, map->num_stripes, GFP_NOFS);
 	if (!zone_info) {
 		ret = -ENOMEM;
 		goto out;
@@ -2114,9 +2122,8 @@ void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered)
 	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered->flags))
 		return;
 
-	ASSERT(!list_empty(&ordered->list));
-	/* The ordered->list can be empty in the above pre-alloc case. */
-	sum = list_first_entry(&ordered->list, struct btrfs_ordered_sum, list);
+	ASSERT(!list_empty(&ordered->csum_list));
+	sum = list_first_entry(&ordered->csum_list, struct btrfs_ordered_sum, list);
 	logical = sum->logical;
 	len = sum->len;
 
@@ -2147,7 +2154,7 @@ void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered)
 	 */
 	if ((inode->flags & BTRFS_INODE_NODATASUM) ||
 	    test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) {
-		while ((sum = list_first_entry_or_null(&ordered->list,
+		while ((sum = list_first_entry_or_null(&ordered->csum_list,
 						       typeof(*sum), list))) {
 			list_del(&sum->list);
 			kfree(sum);
@@ -2377,6 +2384,9 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group)
 	if (!btrfs_is_zoned(block_group->fs_info))
 		return true;
 
+	if (unlikely(btrfs_is_testing(fs_info)))
+		return true;
+
 	map = block_group->physical_map;
 
 	spin_lock(&fs_info->zone_active_bgs_lock);
diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
index 6f076fa8f3f9a..41547ff187f65 100644
--- a/fs/btrfs/zstd.c
+++ b/fs/btrfs/zstd.c
@@ -587,7 +587,7 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
 	struct folio_iter fi;
-	size_t srclen = cb->compressed_len;
+	size_t srclen = bio_get_size(&cb->bbio.bio);
 	zstd_dstream *stream;
 	int ret = 0;
 	const u32 blocksize = fs_info->sectorsize;
@@ -600,7 +600,7 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	bio_first_folio(&fi, &cb->bbio.bio, 0);
 	if (unlikely(!fi.folio))
 		return -EINVAL;
-	ASSERT(folio_size(fi.folio) == blocksize);
+	ASSERT(folio_size(fi.folio) == min_folio_size);
 
 	stream = zstd_init_dstream(
 			ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
diff --git a/fs/buffer.c b/fs/buffer.c
index 22b43642ba574..2d2e3ecec6b2b 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -45,7 +45,7 @@
 #include <linux/bitops.h>
 #include <linux/mpage.h>
 #include <linux/bit_spinlock.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/sched/mm.h>
 #include <trace/events/block.h>
 #include <linux/fscrypt.h>
@@ -922,8 +922,7 @@ struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
 	long offset;
 	struct mem_cgroup *memcg, *old_memcg;
 
-	/* The folio lock pins the memcg */
-	memcg = folio_memcg(folio);
+	memcg = get_mem_cgroup_from_folio(folio);
 	old_memcg = set_active_memcg(memcg);
 
 	head = NULL;
@@ -944,6 +943,7 @@ struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
 	}
 out:
 	set_active_memcg(old_memcg);
+	mem_cgroup_put(memcg);
 	return head;
 /*
  * In case anything failed, we just free everything we got.
@@ -2774,6 +2774,21 @@ static void end_bio_bh_io_sync(struct bio *bio)
 	bio_put(bio);
 }
 
+static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh,
+				  gfp_t gfp_mask)
+{
+	const struct address_space *mapping = folio_mapping(bh->b_folio);
+
+	/*
+	 * The ext4 journal (jbd2) can submit a buffer_head it directly created
+	 * for a non-pagecache page.  fscrypt doesn't care about these.
+	 */
+	if (!mapping)
+		return;
+	fscrypt_set_bio_crypt_ctx(bio, mapping->host,
+			folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask);
+}
+
 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
 			  enum rw_hint write_hint,
 			  struct writeback_control *wbc)
@@ -2800,7 +2815,8 @@ static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
 
 	bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
 
-	fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
+	if (IS_ENABLED(CONFIG_FS_ENCRYPTION))
+		buffer_set_crypto_ctx(bio, bh, GFP_NOIO);
 
 	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
 	bio->bi_write_hint = write_hint;
diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c
index eaf47851c65f4..d879b80a0bedc 100644
--- a/fs/cachefiles/io.c
+++ b/fs/cachefiles/io.c
@@ -93,7 +93,7 @@ static int cachefiles_read(struct netfs_cache_resources *cres,
 	object = cachefiles_cres_object(cres);
 	file = cachefiles_cres_file(cres);
 
-	_enter("%pD,%li,%llx,%zx/%llx",
+	_enter("%pD,%llu,%llx,%zx/%llx",
 	       file, file_inode(file)->i_ino, start_pos, len,
 	       i_size_read(file_inode(file)));
 
@@ -214,7 +214,7 @@ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
 	file = cachefiles_cres_file(cres);
 	granularity = max_t(size_t, object->volume->cache->bsize, granularity);
 
-	_enter("%pD,%li,%llx,%zx/%llx",
+	_enter("%pD,%llu,%llx,%zx/%llx",
 	       file, file_inode(file)->i_ino, start, len,
 	       i_size_read(file_inode(file)));
 
@@ -294,7 +294,7 @@ int __cachefiles_write(struct cachefiles_object *object,
 	fscache_count_write();
 	cache = object->volume->cache;
 
-	_enter("%pD,%li,%llx,%zx/%llx",
+	_enter("%pD,%llu,%llx,%zx/%llx",
 	       file, file_inode(file)->i_ino, start_pos, len,
 	       i_size_read(file_inode(file)));
 
diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c
index e5ec90dccc27f..b498c210112b3 100644
--- a/fs/cachefiles/namei.c
+++ b/fs/cachefiles/namei.c
@@ -147,7 +147,7 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
 		}
 		ASSERT(d_backing_inode(subdir));
 
-		_debug("mkdir -> %pd{ino=%lu}",
+		_debug("mkdir -> %pd{ino=%llu}",
 		       subdir, d_backing_inode(subdir)->i_ino);
 		if (_is_new)
 			*_is_new = true;
@@ -158,7 +158,7 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
 	end_creating_keep(subdir);
 
 	if (!__cachefiles_mark_inode_in_use(NULL, d_inode(subdir))) {
-		pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n",
+		pr_notice("cachefiles: Inode already in use: %pd (B=%llx)\n",
 			  subdir, d_inode(subdir)->i_ino);
 		goto mark_error;
 	}
@@ -183,7 +183,7 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
 	    !d_backing_inode(subdir)->i_op->unlink)
 		goto check_error;
 
-	_leave(" = [%lu]", d_backing_inode(subdir)->i_ino);
+	_leave(" = [%llu]", d_backing_inode(subdir)->i_ino);
 	return subdir;
 
 check_error:
@@ -270,7 +270,8 @@ int cachefiles_bury_object(struct cachefiles_cache *cache,
 			   struct dentry *rep,
 			   enum fscache_why_object_killed why)
 {
-	struct dentry *grave, *trap;
+	struct dentry *grave;
+	struct renamedata rd = {};
 	struct path path, path_to_graveyard;
 	char nbuffer[8 + 8 + 1];
 	int ret;
@@ -302,77 +303,55 @@ int cachefiles_bury_object(struct cachefiles_cache *cache,
 		(uint32_t) ktime_get_real_seconds(),
 		(uint32_t) atomic_inc_return(&cache->gravecounter));
 
-	/* do the multiway lock magic */
-	trap = lock_rename(cache->graveyard, dir);
-	if (IS_ERR(trap))
-		return PTR_ERR(trap);
-
-	/* do some checks before getting the grave dentry */
-	if (rep->d_parent != dir || IS_DEADDIR(d_inode(rep))) {
-		/* the entry was probably culled when we dropped the parent dir
-		 * lock */
-		unlock_rename(cache->graveyard, dir);
-		_leave(" = 0 [culled?]");
-		return 0;
-	}
-
-	if (!d_can_lookup(cache->graveyard)) {
-		unlock_rename(cache->graveyard, dir);
-		cachefiles_io_error(cache, "Graveyard no longer a directory");
-		return -EIO;
-	}
+	rd.mnt_idmap = &nop_mnt_idmap;
+	rd.old_parent = dir;
+	rd.new_parent = cache->graveyard;
+	rd.flags = 0;
+	ret = start_renaming_dentry(&rd, 0, rep, &QSTR(nbuffer));
+	if (ret) {
+		/* Some errors aren't fatal */
+		if (ret == -EXDEV)
+			/* double-lock failed */
+			return ret;
+		if (d_unhashed(rep) || rep->d_parent != dir || IS_DEADDIR(d_inode(rep))) {
+			/* the entry was probably culled when we dropped the parent dir
+			 * lock */
+			_leave(" = 0 [culled?]");
+			return 0;
+		}
+		if (ret == -EINVAL || ret == -ENOTEMPTY) {
+			cachefiles_io_error(cache, "May not make directory loop");
+			return -EIO;
+		}
+		if (ret == -ENOMEM) {
+			_leave(" = -ENOMEM");
+			return -ENOMEM;
+		}
 
-	if (trap == rep) {
-		unlock_rename(cache->graveyard, dir);
-		cachefiles_io_error(cache, "May not make directory loop");
+		cachefiles_io_error(cache, "Lookup error %d", ret);
 		return -EIO;
 	}
 
 	if (d_mountpoint(rep)) {
-		unlock_rename(cache->graveyard, dir);
+		end_renaming(&rd);
 		cachefiles_io_error(cache, "Mountpoint in cache");
 		return -EIO;
 	}
 
-	grave = lookup_one(&nop_mnt_idmap, &QSTR(nbuffer), cache->graveyard);
-	if (IS_ERR(grave)) {
-		unlock_rename(cache->graveyard, dir);
-		trace_cachefiles_vfs_error(object, d_inode(cache->graveyard),
-					   PTR_ERR(grave),
-					   cachefiles_trace_lookup_error);
-
-		if (PTR_ERR(grave) == -ENOMEM) {
-			_leave(" = -ENOMEM");
-			return -ENOMEM;
-		}
-
-		cachefiles_io_error(cache, "Lookup error %ld", PTR_ERR(grave));
-		return -EIO;
-	}
-
+	grave = rd.new_dentry;
 	if (d_is_positive(grave)) {
-		unlock_rename(cache->graveyard, dir);
-		dput(grave);
+		end_renaming(&rd);
 		grave = NULL;
 		cond_resched();
 		goto try_again;
 	}
 
 	if (d_mountpoint(grave)) {
-		unlock_rename(cache->graveyard, dir);
-		dput(grave);
+		end_renaming(&rd);
 		cachefiles_io_error(cache, "Mountpoint in graveyard");
 		return -EIO;
 	}
 
-	/* target should not be an ancestor of source */
-	if (trap == grave) {
-		unlock_rename(cache->graveyard, dir);
-		dput(grave);
-		cachefiles_io_error(cache, "May not make directory loop");
-		return -EIO;
-	}
-
 	/* attempt the rename */
 	path.mnt = cache->mnt;
 	path.dentry = dir;
@@ -382,13 +361,6 @@ int cachefiles_bury_object(struct cachefiles_cache *cache,
 	if (ret < 0) {
 		cachefiles_io_error(cache, "Rename security error %d", ret);
 	} else {
-		struct renamedata rd = {
-			.mnt_idmap	= &nop_mnt_idmap,
-			.old_parent	= dir,
-			.old_dentry	= rep,
-			.new_parent	= cache->graveyard,
-			.new_dentry	= grave,
-		};
 		trace_cachefiles_rename(object, d_inode(rep)->i_ino, why);
 		ret = cachefiles_inject_read_error();
 		if (ret == 0)
@@ -402,8 +374,7 @@ int cachefiles_bury_object(struct cachefiles_cache *cache,
 	}
 
 	__cachefiles_unmark_inode_in_use(object, d_inode(rep));
-	unlock_rename(cache->graveyard, dir);
-	dput(grave);
+	end_renaming(&rd);
 	_leave(" = 0");
 	return 0;
 }
@@ -529,7 +500,7 @@ static bool cachefiles_create_file(struct cachefiles_object *object)
 
 	set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags);
 	set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags);
-	_debug("create -> %pD{ino=%lu}", file, file_inode(file)->i_ino);
+	_debug("create -> %pD{ino=%llu}", file, file_inode(file)->i_ino);
 	object->file = file;
 	return true;
 }
@@ -549,7 +520,7 @@ static bool cachefiles_open_file(struct cachefiles_object *object,
 	_enter("%pd", dentry);
 
 	if (!cachefiles_mark_inode_in_use(object, d_inode(dentry))) {
-		pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n",
+		pr_notice("cachefiles: Inode already in use: %pd (B=%llx)\n",
 			  dentry, d_inode(dentry)->i_ino);
 		return false;
 	}
@@ -657,7 +628,7 @@ bool cachefiles_look_up_object(struct cachefiles_object *object)
 	if (!ret)
 		return false;
 
-	_leave(" = t [%lu]", file_inode(object->file)->i_ino);
+	_leave(" = t [%llu]", file_inode(object->file)->i_ino);
 	return true;
 
 new_file:
diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c
index 52383b1d0ba63..f8ae78b3f7b6d 100644
--- a/fs/cachefiles/xattr.c
+++ b/fs/cachefiles/xattr.c
@@ -179,7 +179,7 @@ int cachefiles_remove_object_xattr(struct cachefiles_cache *cache,
 			ret = 0;
 		else if (ret != -ENOMEM)
 			cachefiles_io_error(cache,
-					    "Can't remove xattr from %lu"
+					    "Can't remove xattr from %llu"
 					    " (error %d)",
 					    d_backing_inode(dentry)->i_ino, -ret);
 	}
diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 2090fc78529cb..fa4f25f994092 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -7,7 +7,7 @@
 #include <linux/swap.h>
 #include <linux/pagemap.h>
 #include <linux/slab.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/task_io_accounting_ops.h>
 #include <linux/signal.h>
 #include <linux/iversion.h>
@@ -1365,6 +1365,10 @@ void ceph_process_folio_batch(struct address_space *mapping,
 		rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc,
 				folio);
 		if (rc) {
+			/* Did we just begin a new contiguous op? Nevermind! */
+			if (ceph_wbc->len == 0)
+				ceph_wbc->num_ops--;
+
 			folio_redirty_for_writepage(wbc, folio);
 			folio_unlock(folio);
 			break;
diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c
index f3de43ccb470d..64d2407592772 100644
--- a/fs/ceph/crypto.c
+++ b/fs/ceph/crypto.c
@@ -272,7 +272,7 @@ int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int elen)
 	/* To understand the 240 limit, see CEPH_NOHASH_NAME_MAX comments */
 	WARN_ON(elen > 240);
 	if (dir != parent) // leading _ is already there; append _<inum>
-		elen += 1 + sprintf(p + elen, "_%ld", dir->i_ino);
+		elen += 1 + sprintf(p + elen, "_%llu", dir->i_ino);
 
 out:
 	kfree(cryptbuf);
@@ -377,7 +377,7 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname,
 	if (!ret && (dir != fname->dir)) {
 		char tmp_buf[BASE64_CHARS(NAME_MAX)];
 
-		name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%ld",
+		name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%llu",
 				    oname->len, oname->name, dir->i_ino);
 		memcpy(oname->name, tmp_buf, name_len);
 		oname->len = name_len;
diff --git a/fs/coda/dir.c b/fs/coda/dir.c
index c64b8cd81568d..d6b9fc3cc1ca9 100644
--- a/fs/coda/dir.c
+++ b/fs/coda/dir.c
@@ -533,7 +533,7 @@ int coda_revalidate_inode(struct inode *inode)
 		coda_vattr_to_iattr(inode, &attr);
 
 		if ((old_mode & S_IFMT) != (inode->i_mode & S_IFMT)) {
-			pr_warn("inode %ld, fid %s changed type!\n",
+			pr_warn("inode %llu, fid %s changed type!\n",
 				inode->i_ino, coda_f2s(&(cii->c_fid)));
 		}
 
diff --git a/fs/coda/inode.c b/fs/coda/inode.c
index ad1654f3adf8a..40b43866e6a54 100644
--- a/fs/coda/inode.c
+++ b/fs/coda/inode.c
@@ -257,7 +257,7 @@ static int coda_fill_super(struct super_block *sb, struct fs_context *fc)
 		goto error;
 	} 
 
-	pr_info("%s: rootinode is %ld dev %s\n",
+	pr_info("%s: rootinode is %llu dev %s\n",
 		__func__, root->i_ino, root->i_sb->s_id);
 	sb->s_root = d_make_root(root);
 	if (!sb->s_root) {
diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c
index e0ba9cd640dcf..4edbfccd0bbea 100644
--- a/fs/cramfs/inode.c
+++ b/fs/cramfs/inode.c
@@ -125,7 +125,7 @@ static struct inode *get_cramfs_inode(struct super_block *sb,
 				old_decode_dev(cramfs_inode->size));
 		break;
 	default:
-		printk(KERN_DEBUG "CRAMFS: Invalid file type 0%04o for inode %lu.\n",
+		printk(KERN_DEBUG "CRAMFS: Invalid file type 0%04o for inode %llu.\n",
 		       inode->i_mode, inode->i_ino);
 		iget_failed(inode);
 		return ERR_PTR(-EIO);
diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c
index 6da683ea69dc9..d07740680602a 100644
--- a/fs/crypto/bio.c
+++ b/fs/crypto/bio.c
@@ -70,11 +70,9 @@ static void fscrypt_zeroout_range_end_io(struct bio *bio)
 }
 
 static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
-					      pgoff_t lblk, sector_t sector,
-					      unsigned int len)
+					      loff_t pos, sector_t sector,
+					      u64 len)
 {
-	const unsigned int blockbits = inode->i_blkbits;
-	const unsigned int blocks_per_page = 1 << (PAGE_SHIFT - blockbits);
 	struct fscrypt_zero_done done = {
 		.pending	= ATOMIC_INIT(1),
 		.done		= COMPLETION_INITIALIZER_ONSTACK(done.done),
@@ -89,18 +87,16 @@ static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
 		bio->bi_iter.bi_sector = sector;
 		bio->bi_private = &done;
 		bio->bi_end_io = fscrypt_zeroout_range_end_io;
-		fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
+		fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_NOFS);
 
 		for (n = 0; n < BIO_MAX_VECS; n++) {
-			unsigned int blocks_this_page =
-				min(len, blocks_per_page);
-			unsigned int bytes_this_page = blocks_this_page << blockbits;
+			unsigned int bytes_this_page = min(len, PAGE_SIZE);
 
 			__bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
-			len -= blocks_this_page;
-			lblk += blocks_this_page;
+			len -= bytes_this_page;
+			pos += bytes_this_page;
 			sector += (bytes_this_page >> SECTOR_SHIFT);
-			if (!len || !fscrypt_mergeable_bio(bio, inode, lblk))
+			if (!len || !fscrypt_mergeable_bio(bio, inode, pos))
 				break;
 		}
 
@@ -117,31 +113,31 @@ static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
 /**
  * fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file
  * @inode: the file's inode
- * @lblk: the first file logical block to zero out
- * @pblk: the first filesystem physical block to zero out
- * @len: number of blocks to zero out
+ * @pos: the first file position (in bytes) to zero out
+ * @sector: the first sector to zero out
+ * @len: bytes to zero out
  *
  * Zero out filesystem blocks in an encrypted regular file on-disk, i.e. write
  * ciphertext blocks which decrypt to the all-zeroes block.  The blocks must be
  * both logically and physically contiguous.  It's also assumed that the
- * filesystem only uses a single block device, ->s_bdev.
+ * filesystem only uses a single block device, ->s_bdev.  @len must be a
+ * multiple of the file system logical block size.
  *
  * Note that since each block uses a different IV, this involves writing a
  * different ciphertext to each block; we can't simply reuse the same one.
  *
  * Return: 0 on success; -errno on failure.
  */
-int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
-			  sector_t pblk, unsigned int len)
+int fscrypt_zeroout_range(const struct inode *inode, loff_t pos,
+			  sector_t sector, u64 len)
 {
 	const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode);
 	const unsigned int du_bits = ci->ci_data_unit_bits;
 	const unsigned int du_size = 1U << du_bits;
 	const unsigned int du_per_page_bits = PAGE_SHIFT - du_bits;
 	const unsigned int du_per_page = 1U << du_per_page_bits;
-	u64 du_index = (u64)lblk << (inode->i_blkbits - du_bits);
-	u64 du_remaining = (u64)len << (inode->i_blkbits - du_bits);
-	sector_t sector = pblk << (inode->i_blkbits - SECTOR_SHIFT);
+	u64 du_index = pos >> du_bits;
+	u64 du_remaining = len >> du_bits;
 	struct page *pages[16]; /* write up to 16 pages at a time */
 	unsigned int nr_pages;
 	unsigned int i;
@@ -153,7 +149,7 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
 		return 0;
 
 	if (fscrypt_inode_uses_inline_crypto(inode))
-		return fscrypt_zeroout_range_inline_crypt(inode, lblk, sector,
+		return fscrypt_zeroout_range_inline_crypt(inode, pos, sector,
 							  len);
 
 	BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_VECS);
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c
index 07f9cbfe3ea41..570a2231c945a 100644
--- a/fs/crypto/crypto.c
+++ b/fs/crypto/crypto.c
@@ -365,7 +365,7 @@ void fscrypt_msg(const struct inode *inode, const char *level,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 	if (inode && inode->i_ino)
-		printk("%sfscrypt (%s, inode %lu): %pV\n",
+		printk("%sfscrypt (%s, inode %llu): %pV\n",
 		       level, inode->i_sb->s_id, inode->i_ino, &vaf);
 	else if (inode)
 		printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf);
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 4e8e82a9ccf9a..8d3c278a7591e 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -278,9 +278,6 @@ struct fscrypt_inode_info {
 	 */
 	u8 ci_data_unit_bits;
 
-	/* Cached value: log2 of number of data units per FS block */
-	u8 ci_data_units_per_block_bits;
-
 	/* Hashed inode number.  Only set for IV_INO_LBLK_32 */
 	u32 ci_hashed_ino;
 
diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c
index b97de0d1430fd..a7a8a3f581a04 100644
--- a/fs/crypto/hooks.c
+++ b/fs/crypto/hooks.c
@@ -62,7 +62,7 @@ int fscrypt_file_open(struct inode *inode, struct file *filp)
 	dentry_parent = dget_parent(dentry);
 	if (!fscrypt_has_permitted_context(d_inode(dentry_parent), inode)) {
 		fscrypt_warn(inode,
-			     "Inconsistent encryption context (parent directory: %lu)",
+			     "Inconsistent encryption context (parent directory: %llu)",
 			     d_inode(dentry_parent)->i_ino);
 		err = -EPERM;
 	}
diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c
index c0852b920dbc2..37d42d357925e 100644
--- a/fs/crypto/inline_crypt.c
+++ b/fs/crypto/inline_crypt.c
@@ -268,14 +268,12 @@ bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
 EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto);
 
 static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci,
-				 u64 lblk_num,
-				 u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
+				 loff_t pos, u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
 {
-	u64 index = lblk_num << ci->ci_data_units_per_block_bits;
 	union fscrypt_iv iv;
 	int i;
 
-	fscrypt_generate_iv(&iv, index, ci);
+	fscrypt_generate_iv(&iv, pos >> ci->ci_data_unit_bits, ci);
 
 	BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE);
 	memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE);
@@ -287,7 +285,7 @@ static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci,
  * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
  * @bio: a bio which will eventually be submitted to the file
  * @inode: the file's inode
- * @first_lblk: the first file logical block number in the I/O
+ * @pos: the first file position (in bytes) in the I/O
  * @gfp_mask: memory allocation flags - these must be a waiting mask so that
  *					bio_crypt_set_ctx can't fail.
  *
@@ -300,7 +298,7 @@ static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci,
  * The encryption context will be freed automatically when the bio is freed.
  */
 void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
-			       u64 first_lblk, gfp_t gfp_mask)
+			       loff_t pos, gfp_t gfp_mask)
 {
 	const struct fscrypt_inode_info *ci;
 	u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
@@ -309,61 +307,16 @@ void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
 		return;
 	ci = fscrypt_get_inode_info_raw(inode);
 
-	fscrypt_generate_dun(ci, first_lblk, dun);
+	fscrypt_generate_dun(ci, pos, dun);
 	bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask);
 }
 EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
 
-/* Extract the inode and logical block number from a buffer_head. */
-static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh,
-				      const struct inode **inode_ret,
-				      u64 *lblk_num_ret)
-{
-	struct folio *folio = bh->b_folio;
-	const struct address_space *mapping;
-	const struct inode *inode;
-
-	/*
-	 * The ext4 journal (jbd2) can submit a buffer_head it directly created
-	 * for a non-pagecache page.  fscrypt doesn't care about these.
-	 */
-	mapping = folio_mapping(folio);
-	if (!mapping)
-		return false;
-	inode = mapping->host;
-
-	*inode_ret = inode;
-	*lblk_num_ret = (folio_pos(folio) + bh_offset(bh)) >> inode->i_blkbits;
-	return true;
-}
-
-/**
- * fscrypt_set_bio_crypt_ctx_bh() - prepare a file contents bio for inline
- *				    crypto
- * @bio: a bio which will eventually be submitted to the file
- * @first_bh: the first buffer_head for which I/O will be submitted
- * @gfp_mask: memory allocation flags
- *
- * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead
- * of an inode and block number directly.
- */
-void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
-				  const struct buffer_head *first_bh,
-				  gfp_t gfp_mask)
-{
-	const struct inode *inode;
-	u64 first_lblk;
-
-	if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk))
-		fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask);
-}
-EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh);
-
 /**
  * fscrypt_mergeable_bio() - test whether data can be added to a bio
  * @bio: the bio being built up
  * @inode: the inode for the next part of the I/O
- * @next_lblk: the next file logical block number in the I/O
+ * @pos: the next file position (in bytes) in the I/O
  *
  * When building a bio which may contain data which should undergo inline
  * encryption (or decryption) via fscrypt, filesystems should call this function
@@ -381,7 +334,7 @@ EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh);
  * Return: true iff the I/O is mergeable
  */
 bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
-			   u64 next_lblk)
+			   loff_t pos)
 {
 	const struct bio_crypt_ctx *bc = bio->bi_crypt_context;
 	const struct fscrypt_inode_info *ci;
@@ -401,34 +354,11 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
 	if (bc->bc_key != ci->ci_enc_key.blk_key)
 		return false;
 
-	fscrypt_generate_dun(ci, next_lblk, next_dun);
+	fscrypt_generate_dun(ci, pos, next_dun);
 	return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun);
 }
 EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio);
 
-/**
- * fscrypt_mergeable_bio_bh() - test whether data can be added to a bio
- * @bio: the bio being built up
- * @next_bh: the next buffer_head for which I/O will be submitted
- *
- * Same as fscrypt_mergeable_bio(), except this takes a buffer_head instead of
- * an inode and block number directly.
- *
- * Return: true iff the I/O is mergeable
- */
-bool fscrypt_mergeable_bio_bh(struct bio *bio,
-			      const struct buffer_head *next_bh)
-{
-	const struct inode *inode;
-	u64 next_lblk;
-
-	if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk))
-		return !bio->bi_crypt_context;
-
-	return fscrypt_mergeable_bio(bio, inode, next_lblk);
-}
-EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh);
-
 /**
  * fscrypt_dio_supported() - check whether DIO (direct I/O) is supported on an
  *			     inode, as far as encryption is concerned
diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c
index 9ec6e5ef0947e..be8e6e8011f21 100644
--- a/fs/crypto/keyring.c
+++ b/fs/crypto/keyring.c
@@ -969,8 +969,8 @@ static int check_for_busy_inodes(struct super_block *sb,
 {
 	struct list_head *pos;
 	size_t busy_count = 0;
-	unsigned long ino;
 	char ino_str[50] = "";
+	u64 ino;
 
 	spin_lock(&mk->mk_decrypted_inodes_lock);
 
@@ -994,7 +994,7 @@ static int check_for_busy_inodes(struct super_block *sb,
 
 	/* If the inode is currently being created, ino may still be 0. */
 	if (ino)
-		snprintf(ino_str, sizeof(ino_str), ", including ino %lu", ino);
+		snprintf(ino_str, sizeof(ino_str), ", including ino %llu", ino);
 
 	fscrypt_warn(NULL,
 		     "%s: %zu inode(s) still busy after removing key with %s %*phN%s",
diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
index 40fa05688d3a1..ce327bfdada46 100644
--- a/fs/crypto/keysetup.c
+++ b/fs/crypto/keysetup.c
@@ -91,7 +91,7 @@ select_encryption_mode(const union fscrypt_policy *policy,
 	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
 		return &fscrypt_modes[fscrypt_policy_fnames_mode(policy)];
 
-	WARN_ONCE(1, "fscrypt: filesystem tried to load encryption info for inode %lu, which is not encryptable (file type %d)\n",
+	WARN_ONCE(1, "fscrypt: filesystem tried to load encryption info for inode %llu, which is not encryptable (file type %d)\n",
 		  inode->i_ino, (inode->i_mode & S_IFMT));
 	return ERR_PTR(-EINVAL);
 }
@@ -609,8 +609,6 @@ fscrypt_setup_encryption_info(struct inode *inode,
 
 	crypt_info->ci_data_unit_bits =
 		fscrypt_policy_du_bits(&crypt_info->ci_policy, inode);
-	crypt_info->ci_data_units_per_block_bits =
-		inode->i_blkbits - crypt_info->ci_data_unit_bits;
 
 	res = setup_file_encryption_key(crypt_info, need_dirhash_key, &mk);
 	if (res)
diff --git a/fs/dax.c b/fs/dax.c
index 289e6254aa30b..a5237169b4679 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -15,7 +15,6 @@
 #include <linux/memcontrol.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
-#include <linux/pagevec.h>
 #include <linux/sched.h>
 #include <linux/sched/signal.h>
 #include <linux/uio.h>
@@ -1360,7 +1359,7 @@ static vm_fault_t dax_load_hole(struct xa_state *xas, struct vm_fault *vmf,
 {
 	struct inode *inode = iter->inode;
 	unsigned long vaddr = vmf->address;
-	unsigned long pfn = my_zero_pfn(vaddr);
+	unsigned long pfn = zero_pfn(vaddr);
 	vm_fault_t ret;
 
 	*entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, DAX_ZERO_PAGE);
diff --git a/fs/dcache.c b/fs/dcache.c
index 7ba1801d81329..9ceab142896f6 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -1637,11 +1637,11 @@ static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
 	if (dentry == _data && dentry->d_lockref.count == 1)
 		return D_WALK_CONTINUE;
 
-	WARN(1, "BUG: Dentry %p{i=%lx,n=%pd} "
+	WARN(1, "BUG: Dentry %p{i=%llx,n=%pd} "
 			" still in use (%d) [unmount of %s %s]\n",
 		       dentry,
 		       dentry->d_inode ?
-		       dentry->d_inode->i_ino : 0UL,
+		       dentry->d_inode->i_ino : (u64)0,
 		       dentry,
 		       dentry->d_lockref.count,
 		       dentry->d_sb->s_type->name,
@@ -3257,7 +3257,7 @@ static void __init dcache_init_early(void)
 					HASH_EARLY | HASH_ZERO,
 					&d_hash_shift,
 					NULL,
-					0,
+					2,
 					0);
 	d_hash_shift = 32 - d_hash_shift;
 
@@ -3289,7 +3289,7 @@ static void __init dcache_init(void)
 					HASH_ZERO,
 					&d_hash_shift,
 					NULL,
-					0,
+					2,
 					0);
 	d_hash_shift = 32 - d_hash_shift;
 
diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c
index 3b59346d68c59..f25c9a49e251a 100644
--- a/fs/ecryptfs/crypto.c
+++ b/fs/ecryptfs/crypto.c
@@ -1313,7 +1313,7 @@ int ecryptfs_read_metadata(struct dentry *ecryptfs_dentry)
 		rc = ecryptfs_read_xattr_region(page_virt, ecryptfs_inode);
 		if (rc) {
 			printk(KERN_DEBUG "Valid eCryptfs headers not found in "
-			       "file header region or xattr region, inode %lu\n",
+			       "file header region or xattr region, inode %llu\n",
 				ecryptfs_inode->i_ino);
 			rc = -EINVAL;
 			goto out;
@@ -1323,7 +1323,7 @@ int ecryptfs_read_metadata(struct dentry *ecryptfs_dentry)
 						ECRYPTFS_DONT_VALIDATE_HEADER_SIZE);
 		if (rc) {
 			printk(KERN_DEBUG "Valid eCryptfs headers not found in "
-			       "file xattr region either, inode %lu\n",
+			       "file xattr region either, inode %llu\n",
 				ecryptfs_inode->i_ino);
 			rc = -EINVAL;
 		}
@@ -1335,7 +1335,7 @@ int ecryptfs_read_metadata(struct dentry *ecryptfs_dentry)
 			       "crypto metadata only in the extended attribute "
 			       "region, but eCryptfs was mounted without "
 			       "xattr support enabled. eCryptfs will not treat "
-			       "this like an encrypted file, inode %lu\n",
+			       "this like an encrypted file, inode %llu\n",
 				ecryptfs_inode->i_ino);
 			rc = -EINVAL;
 		}
diff --git a/fs/ecryptfs/file.c b/fs/ecryptfs/file.c
index 7929411837cf5..49b0fbe0428a2 100644
--- a/fs/ecryptfs/file.c
+++ b/fs/ecryptfs/file.c
@@ -253,7 +253,7 @@ static int ecryptfs_open(struct inode *inode, struct file *file)
 	if (rc)
 		goto out_put;
 	ecryptfs_printk(KERN_DEBUG, "inode w/ addr = [0x%p], i_ino = "
-			"[0x%.16lx] size: [0x%.16llx]\n", inode, inode->i_ino,
+			"[0x%.16llx] size: [0x%.16llx]\n", inode, inode->i_ino,
 			(unsigned long long)i_size_read(inode));
 	goto out;
 out_put:
diff --git a/fs/efs/inode.c b/fs/efs/inode.c
index 28407578f83ad..4b132729e6385 100644
--- a/fs/efs/inode.c
+++ b/fs/efs/inode.c
@@ -132,7 +132,7 @@ struct inode *efs_iget(struct super_block *super, unsigned long ino)
 	for(i = 0; i < EFS_DIRECTEXTENTS; i++) {
 		extent_copy(&(efs_inode->di_u.di_extents[i]), &(in->extents[i]));
 		if (i < in->numextents && in->extents[i].cooked.ex_magic != 0) {
-			pr_warn("extent %d has bad magic number in inode %lu\n",
+			pr_warn("extent %d has bad magic number in inode %llu\n",
 				i, inode->i_ino);
 			brelse(bh);
 			goto read_inode_error;
@@ -140,7 +140,7 @@ struct inode *efs_iget(struct super_block *super, unsigned long ino)
 	}
 
 	brelse(bh);
-	pr_debug("efs_iget(): inode %lu, extents %d, mode %o\n",
+	pr_debug("efs_iget(): inode %llu, extents %d, mode %o\n",
 		 inode->i_ino, in->numextents, inode->i_mode);
 	switch (inode->i_mode & S_IFMT) {
 		case S_IFDIR: 
@@ -171,7 +171,7 @@ struct inode *efs_iget(struct super_block *super, unsigned long ino)
 	return inode;
         
 read_inode_error:
-	pr_warn("failed to read inode %lu\n", inode->i_ino);
+	pr_warn("failed to read inode %llu\n", inode->i_ino);
 	iget_failed(inode);
 	return ERR_PTR(-EIO);
 }
diff --git a/fs/erofs/data.c b/fs/erofs/data.c
index f79ee80627d95..b33dd4d8710e4 100644
--- a/fs/erofs/data.c
+++ b/fs/erofs/data.c
@@ -473,8 +473,7 @@ static int erofs_file_mmap_prepare(struct vm_area_desc *desc)
 	if (!IS_DAX(file_inode(desc->file)))
 		return generic_file_readonly_mmap_prepare(desc);
 
-	if (vma_desc_test_flags(desc, VMA_SHARED_BIT) &&
-	    vma_desc_test_flags(desc, VMA_MAYWRITE_BIT))
+	if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT))
 		return -EINVAL;
 
 	desc->vm_ops = &erofs_dax_vm_ops;
diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c
index abe873f01297b..98cdaa1cd1a7c 100644
--- a/fs/erofs/fileio.c
+++ b/fs/erofs/fileio.c
@@ -25,10 +25,8 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret)
 			container_of(iocb, struct erofs_fileio_rq, iocb);
 	struct folio_iter fi;
 
-	if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) {
-		bio_advance(&rq->bio, ret);
-		zero_fill_bio(&rq->bio);
-	}
+	if (ret >= 0 && ret != rq->bio.bi_iter.bi_size)
+		ret = -EIO;
 	if (!rq->bio.bi_end_io) {
 		bio_for_each_folio_all(fi, &rq->bio) {
 			DBG_BUGON(folio_test_uptodate(fi.folio));
diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c
index 829d50d5c717d..1957841b0c789 100644
--- a/fs/erofs/ishare.c
+++ b/fs/erofs/ishare.c
@@ -4,6 +4,7 @@
  */
 #include <linux/xxhash.h>
 #include <linux/mount.h>
+#include <linux/security.h>
 #include "internal.h"
 #include "xattr.h"
 
@@ -103,18 +104,25 @@ static int erofs_ishare_file_open(struct inode *inode, struct file *file)
 {
 	struct inode *sharedinode = EROFS_I(inode)->sharedinode;
 	struct file *realfile;
+	int err;
 
 	if (file->f_flags & O_DIRECT)
 		return -EINVAL;
-	realfile = alloc_empty_backing_file(O_RDONLY|O_NOATIME, current_cred());
+	realfile = alloc_empty_backing_file(O_RDONLY|O_NOATIME, current_cred(),
+					    file->f_cred);
 	if (IS_ERR(realfile))
 		return PTR_ERR(realfile);
+
+	err = backing_file_open_user_path(realfile, &file->f_path);
+	if (err) {
+		fput(realfile);
+		return err;
+	}
+
 	ihold(sharedinode);
 	realfile->f_op = &erofs_file_fops;
 	realfile->f_inode = sharedinode;
 	realfile->f_mapping = sharedinode->i_mapping;
-	path_get(&file->f_path);
-	backing_file_set_user_path(realfile, &file->f_path);
 
 	file_ra_state_init(&realfile->f_ra, file->f_mapping);
 	realfile->private_data = EROFS_I(inode);
@@ -150,8 +158,14 @@ static ssize_t erofs_ishare_file_read_iter(struct kiocb *iocb,
 static int erofs_ishare_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct file *realfile = file->private_data;
+	int err;
 
 	vma_set_file(vma, realfile);
+
+	err = security_mmap_backing_file(vma, realfile, file);
+	if (err)
+		return err;
+
 	return generic_file_readonly_mmap(file, vma);
 }
 
diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c
index c411df5d9dfc7..41e311019a251 100644
--- a/fs/erofs/xattr.c
+++ b/fs/erofs/xattr.c
@@ -85,6 +85,14 @@ static int erofs_init_inode_xattrs(struct inode *inode)
 	}
 	vi->xattr_name_filter = le32_to_cpu(ih->h_name_filter);
 	vi->xattr_shared_count = ih->h_shared_count;
+	if ((u32)vi->xattr_shared_count * sizeof(__le32) >
+	    vi->xattr_isize - sizeof(struct erofs_xattr_ibody_header)) {
+		erofs_err(sb, "invalid h_shared_count %u @ nid %llu",
+			  vi->xattr_shared_count, vi->nid);
+		erofs_put_metabuf(&buf);
+		ret = -EFSCORRUPTED;
+		goto out_unlock;
+	}
 	vi->xattr_shared_xattrs = kmalloc_objs(uint, vi->xattr_shared_count);
 	if (!vi->xattr_shared_xattrs) {
 		erofs_put_metabuf(&buf);
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 3977e42b95168..fe8121df9ef2f 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1445,6 +1445,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
 				       int bios)
 {
 	struct erofs_sb_info *const sbi = EROFS_SB(io->sb);
+	int gfp_flag;
 
 	/* wake up the caller thread for sync decompression */
 	if (io->sync) {
@@ -1477,7 +1478,9 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
 			sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON;
 		return;
 	}
+	gfp_flag = memalloc_noio_save();
 	z_erofs_decompressqueue_work(&io->u.work);
+	memalloc_noio_restore(gfp_flag);
 }
 
 static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index 5714e900567c4..4ccd4d2e31adf 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -1080,7 +1080,7 @@ static void ep_show_fdinfo(struct seq_file *m, struct file *f)
 		struct inode *inode = file_inode(epi->ffd.file);
 
 		seq_printf(m, "tfd: %8d events: %8x data: %16llx "
-			   " pos:%lli ino:%lx sdev:%x\n",
+			   " pos:%lli ino:%llx sdev:%x\n",
 			   epi->ffd.fd, epi->event.events,
 			   (long long)epi->event.data,
 			   (long long)epi->ffd.file->f_pos,
diff --git a/fs/exec.c b/fs/exec.c
index 9ea3a775d51e9..8503c7f11658b 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1454,6 +1454,17 @@ static struct linux_binprm *alloc_bprm(int fd, struct filename *filename, int fl
 	 */
 	bprm->is_check = !!(flags & AT_EXECVE_CHECK);
 
+	bprm->hwcap = ELF_HWCAP;
+#ifdef ELF_HWCAP2
+	bprm->hwcap2 = ELF_HWCAP2;
+#endif
+#ifdef ELF_HWCAP3
+	bprm->hwcap3 = ELF_HWCAP3;
+#endif
+#ifdef ELF_HWCAP4
+	bprm->hwcap4 = ELF_HWCAP4;
+#endif
+
 	retval = bprm_mm_init(bprm);
 	if (!retval)
 		return bprm;
@@ -1775,6 +1786,57 @@ static int bprm_execve(struct linux_binprm *bprm)
 	return retval;
 }
 
+static void inherit_hwcap(struct linux_binprm *bprm)
+{
+	struct mm_struct *mm = current->mm;
+	int i, n;
+
+#ifdef ELF_HWCAP4
+	n = 4;
+#elif defined(ELF_HWCAP3)
+	n = 3;
+#elif defined(ELF_HWCAP2)
+	n = 2;
+#else
+	n = 1;
+#endif
+
+	spin_lock(&mm->arg_lock);
+	for (i = 0; n && i < AT_VECTOR_SIZE; i += 2) {
+		unsigned long type = mm->saved_auxv[i];
+		unsigned long val = mm->saved_auxv[i + 1];
+
+		switch (type) {
+		case AT_NULL:
+			goto done;
+		case AT_HWCAP:
+			bprm->hwcap = val & ELF_HWCAP;
+			break;
+#ifdef ELF_HWCAP2
+		case AT_HWCAP2:
+			bprm->hwcap2 = val & ELF_HWCAP2;
+			break;
+#endif
+#ifdef ELF_HWCAP3
+		case AT_HWCAP3:
+			bprm->hwcap3 = val & ELF_HWCAP3;
+			break;
+#endif
+#ifdef ELF_HWCAP4
+		case AT_HWCAP4:
+			bprm->hwcap4 = val & ELF_HWCAP4;
+			break;
+#endif
+		default:
+			continue;
+		}
+		n--;
+	}
+done:
+	spin_unlock(&mm->arg_lock);
+	mm_flags_set(MMF_USER_HWCAP, bprm->mm);
+}
+
 static int do_execveat_common(int fd, struct filename *filename,
 			      struct user_arg_ptr argv,
 			      struct user_arg_ptr envp,
@@ -1843,6 +1905,9 @@ static int do_execveat_common(int fd, struct filename *filename,
 			     current->comm, bprm->filename);
 	}
 
+	if (mm_flags_test(MMF_USER_HWCAP, current->mm))
+		inherit_hwcap(bprm);
+
 	return bprm_execve(bprm);
 }
 
diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c
index 3a32f49f9dbd1..625f2f14d4fe0 100644
--- a/fs/exfat/balloc.c
+++ b/fs/exfat/balloc.c
@@ -74,11 +74,10 @@ static int exfat_allocate_bitmap(struct super_block *sb,
 		struct exfat_dentry *ep)
 {
 	struct exfat_sb_info *sbi = EXFAT_SB(sb);
-	struct blk_plug plug;
 	long long map_size;
 	unsigned int i, j, need_map_size;
-	sector_t sector;
-	unsigned int max_ra_count;
+	sector_t sector, end, ra;
+	blkcnt_t ra_cnt = 0;
 
 	sbi->map_clu = le32_to_cpu(ep->dentry.bitmap.start_clu);
 	map_size = le64_to_cpu(ep->dentry.bitmap.size);
@@ -100,17 +99,12 @@ static int exfat_allocate_bitmap(struct super_block *sb,
 	if (!sbi->vol_amap)
 		return -ENOMEM;
 
-	sector = exfat_cluster_to_sector(sbi, sbi->map_clu);
-	max_ra_count = min(sb->s_bdi->ra_pages, sb->s_bdi->io_pages) <<
-		(PAGE_SHIFT - sb->s_blocksize_bits);
+	sector = ra = exfat_cluster_to_sector(sbi, sbi->map_clu);
+	end = sector + sbi->map_sectors - 1;
+
 	for (i = 0; i < sbi->map_sectors; i++) {
 		/* Trigger the next readahead in advance. */
-		if (max_ra_count && 0 == (i % max_ra_count)) {
-			blk_start_plug(&plug);
-			for (j = i; j < min(max_ra_count, sbi->map_sectors - i) + i; j++)
-				sb_breadahead(sb, sector + j);
-			blk_finish_plug(&plug);
-		}
+		exfat_blk_readahead(sb, sector + i, &ra, &ra_cnt, end);
 
 		sbi->vol_amap[i] = sb_bread(sb, sector + i);
 		if (!sbi->vol_amap[i])
diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c
index 3a4853693d8bf..a2c2b998808ca 100644
--- a/fs/exfat/dir.c
+++ b/fs/exfat/dir.c
@@ -249,7 +249,7 @@ static int exfat_iterate(struct file *file, struct dir_context *ctx)
 		 */
 		if (err == -EIO) {
 			cpos += 1 << (sb->s_blocksize_bits);
-			cpos &= ~(sb->s_blocksize - 1);
+			cpos &= ~(loff_t)(sb->s_blocksize - 1);
 		}
 
 		err = -EIO;
@@ -623,44 +623,11 @@ static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir
 	return 0;
 }
 
-#define EXFAT_MAX_RA_SIZE     (128*1024)
-static int exfat_dir_readahead(struct super_block *sb, sector_t sec)
-{
-	struct exfat_sb_info *sbi = EXFAT_SB(sb);
-	struct buffer_head *bh;
-	unsigned int max_ra_count = EXFAT_MAX_RA_SIZE >> sb->s_blocksize_bits;
-	unsigned int page_ra_count = PAGE_SIZE >> sb->s_blocksize_bits;
-	unsigned int adj_ra_count = max(sbi->sect_per_clus, page_ra_count);
-	unsigned int ra_count = min(adj_ra_count, max_ra_count);
-
-	/* Read-ahead is not required */
-	if (sbi->sect_per_clus == 1)
-		return 0;
-
-	if (sec < sbi->data_start_sector) {
-		exfat_err(sb, "requested sector is invalid(sect:%llu, root:%llu)",
-			  (unsigned long long)sec, sbi->data_start_sector);
-		return -EIO;
-	}
-
-	/* Not sector aligned with ra_count, resize ra_count to page size */
-	if ((sec - sbi->data_start_sector) & (ra_count - 1))
-		ra_count = page_ra_count;
-
-	bh = sb_find_get_block(sb, sec);
-	if (!bh || !buffer_uptodate(bh)) {
-		unsigned int i;
-
-		for (i = 0; i < ra_count; i++)
-			sb_breadahead(sb, (sector_t)(sec + i));
-	}
-	brelse(bh);
-	return 0;
-}
-
 struct exfat_dentry *exfat_get_dentry(struct super_block *sb,
 		struct exfat_chain *p_dir, int entry, struct buffer_head **bh)
 {
+	struct exfat_sb_info *sbi = EXFAT_SB(sb);
+	unsigned int sect_per_clus = sbi->sect_per_clus;
 	unsigned int dentries_per_page = EXFAT_B_TO_DEN(PAGE_SIZE);
 	int off;
 	sector_t sec;
@@ -673,9 +640,18 @@ struct exfat_dentry *exfat_get_dentry(struct super_block *sb,
 	if (exfat_find_location(sb, p_dir, entry, &sec, &off))
 		return NULL;
 
-	if (p_dir->dir != EXFAT_FREE_CLUSTER &&
-			!(entry & (dentries_per_page - 1)))
-		exfat_dir_readahead(sb, sec);
+	if (sect_per_clus > 1 &&
+	    (entry & (dentries_per_page - 1)) == 0) {
+		sector_t ra = sec;
+		blkcnt_t cnt = 0;
+		unsigned int ra_count = sect_per_clus;
+
+		/* Not sector aligned with ra_count, resize ra_count to page size */
+		if ((sec - sbi->data_start_sector) & (ra_count - 1))
+			ra_count = PAGE_SIZE >> sb->s_blocksize_bits;
+
+		exfat_blk_readahead(sb, sec, &ra, &cnt, sec + ra_count - 1);
+	}
 
 	*bh = sb_bread(sb, sec);
 	if (!*bh)
diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h
index 2dbed5f8ec267..9fed9fb33caec 100644
--- a/fs/exfat/exfat_fs.h
+++ b/fs/exfat/exfat_fs.h
@@ -10,6 +10,7 @@
 #include <linux/ratelimit.h>
 #include <linux/nls.h>
 #include <linux/blkdev.h>
+#include <linux/backing-dev.h>
 #include <uapi/linux/exfat.h>
 
 #define EXFAT_ROOT_INO		1
@@ -79,6 +80,10 @@ enum {
 #define EXFAT_HINT_NONE		-1
 #define EXFAT_MIN_SUBDIR	2
 
+#define EXFAT_BLK_RA_SIZE(sb)		\
+	(min_t(blkcnt_t, (sb)->s_bdi->ra_pages, (sb)->s_bdi->io_pages) \
+	<< (PAGE_SHIFT - (sb)->s_blocksize_bits))
+
 /*
  * helpers for cluster size to byte conversion.
  */
@@ -117,9 +122,9 @@ enum {
 #define FAT_ENT_SIZE (4)
 #define FAT_ENT_SIZE_BITS (2)
 #define FAT_ENT_OFFSET_SECTOR(sb, loc) (EXFAT_SB(sb)->FAT1_start_sector + \
-	(((u64)loc << FAT_ENT_SIZE_BITS) >> sb->s_blocksize_bits))
+	(((u64)(loc) << FAT_ENT_SIZE_BITS) >> sb->s_blocksize_bits))
 #define FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc)	\
-	((loc << FAT_ENT_SIZE_BITS) & (sb->s_blocksize - 1))
+	(((loc) << FAT_ENT_SIZE_BITS) & (sb->s_blocksize - 1))
 
 /*
  * helpers for bitmap.
@@ -448,6 +453,8 @@ int exfat_find_last_cluster(struct super_block *sb, struct exfat_chain *p_chain,
 		unsigned int *ret_clu);
 int exfat_count_num_clusters(struct super_block *sb,
 		struct exfat_chain *p_chain, unsigned int *ret_count);
+int exfat_blk_readahead(struct super_block *sb, sector_t sec,
+		sector_t *ra, blkcnt_t *ra_cnt, sector_t end);
 
 /* balloc.c */
 int exfat_load_bitmap(struct super_block *sb);
@@ -577,7 +584,7 @@ void exfat_set_entry_time(struct exfat_sb_info *sbi, struct timespec64 *ts,
 		u8 *tz, __le16 *time, __le16 *date, u8 *time_cs);
 u16 exfat_calc_chksum16(void *data, int len, u16 chksum, int type);
 u32 exfat_calc_chksum32(void *data, int len, u32 chksum, int type);
-void exfat_update_bh(struct buffer_head *bh, int sync);
+int exfat_update_bh(struct buffer_head *bh, int sync);
 int exfat_update_bhs(struct buffer_head **bhs, int nr_bhs, int sync);
 void exfat_chain_set(struct exfat_chain *ec, unsigned int dir,
 		unsigned int size, unsigned char flags);
diff --git a/fs/exfat/exfat_raw.h b/fs/exfat/exfat_raw.h
index 4082fa7b8c142..ec70cd35bba0c 100644
--- a/fs/exfat/exfat_raw.h
+++ b/fs/exfat/exfat_raw.h
@@ -25,6 +25,7 @@
 #define EXFAT_FIRST_CLUSTER	2
 #define EXFAT_DATA_CLUSTER_COUNT(sbi)	\
 	((sbi)->num_clusters - EXFAT_RESERVED_CLUSTERS)
+#define EXFAT_MAX_NUM_CLUSTER		(0xFFFFFFF5)
 
 /* AllocationPossible and NoFatChain field in GeneralSecondaryFlags Field */
 #define ALLOC_POSSIBLE		0x01
diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c
index f87576ca70329..f2e5d5dde3936 100644
--- a/fs/exfat/fatent.c
+++ b/fs/exfat/fatent.c
@@ -11,11 +11,11 @@
 #include "exfat_raw.h"
 #include "exfat_fs.h"
 
-static int exfat_mirror_bh(struct super_block *sb, sector_t sec,
-		struct buffer_head *bh)
+static int exfat_mirror_bh(struct super_block *sb, struct buffer_head *bh)
 {
 	struct buffer_head *c_bh;
 	struct exfat_sb_info *sbi = EXFAT_SB(sb);
+	sector_t sec = bh->b_blocknr;
 	sector_t sec2;
 	int err = 0;
 
@@ -25,16 +25,24 @@ static int exfat_mirror_bh(struct super_block *sb, sector_t sec,
 		if (!c_bh)
 			return -ENOMEM;
 		memcpy(c_bh->b_data, bh->b_data, sb->s_blocksize);
-		set_buffer_uptodate(c_bh);
-		mark_buffer_dirty(c_bh);
-		if (sb->s_flags & SB_SYNCHRONOUS)
-			err = sync_dirty_buffer(c_bh);
+		err = exfat_update_bh(c_bh, sb->s_flags & SB_SYNCHRONOUS);
 		brelse(c_bh);
 	}
 
 	return err;
 }
 
+static int exfat_end_bh(struct super_block *sb, struct buffer_head *bh)
+{
+	int err;
+
+	err = exfat_update_bh(bh, sb->s_flags & SB_SYNCHRONOUS);
+	if (!err)
+		err = exfat_mirror_bh(sb, bh);
+	brelse(bh);
+	return err;
+}
+
 static int __exfat_ent_get(struct super_block *sb, unsigned int loc,
 		unsigned int *content, struct buffer_head **last)
 {
@@ -65,29 +73,40 @@ static int __exfat_ent_get(struct super_block *sb, unsigned int loc,
 	return 0;
 }
 
-int exfat_ent_set(struct super_block *sb, unsigned int loc,
-		unsigned int content)
+static int __exfat_ent_set(struct super_block *sb, unsigned int loc,
+		unsigned int content, struct buffer_head **cache)
 {
-	unsigned int off;
 	sector_t sec;
 	__le32 *fat_entry;
-	struct buffer_head *bh;
+	struct buffer_head *bh = cache ? *cache : NULL;
+	unsigned int off;
 
 	sec = FAT_ENT_OFFSET_SECTOR(sb, loc);
 	off = FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc);
 
-	bh = sb_bread(sb, sec);
-	if (!bh)
-		return -EIO;
+	if (!bh || bh->b_blocknr != sec || !buffer_uptodate(bh)) {
+		if (bh)
+			exfat_end_bh(sb, bh);
+		bh = sb_bread(sb, sec);
+		if (cache)
+			*cache = bh;
+		if (unlikely(!bh))
+			return -EIO;
+	}
 
 	fat_entry = (__le32 *)&(bh->b_data[off]);
 	*fat_entry = cpu_to_le32(content);
-	exfat_update_bh(bh, sb->s_flags & SB_SYNCHRONOUS);
-	exfat_mirror_bh(sb, sec, bh);
-	brelse(bh);
+	if (!cache)
+		exfat_end_bh(sb, bh);
 	return 0;
 }
 
+int exfat_ent_set(struct super_block *sb, unsigned int loc,
+		unsigned int content)
+{
+	return __exfat_ent_set(sb, loc, content, NULL);
+}
+
 /*
  * Caller must release the buffer_head if no error return.
  */
@@ -142,21 +161,61 @@ int exfat_ent_get(struct super_block *sb, unsigned int loc,
 	return -EIO;
 }
 
+int exfat_blk_readahead(struct super_block *sb, sector_t sec,
+		sector_t *ra, blkcnt_t *ra_cnt, sector_t end)
+{
+	struct blk_plug plug;
+
+	if (sec < *ra)
+		return 0;
+
+	*ra += *ra_cnt;
+
+	/* No blocks left (or only the last block), skip readahead. */
+	if (*ra >= end)
+		return 0;
+
+	*ra_cnt = min(end - *ra + 1, EXFAT_BLK_RA_SIZE(sb));
+	if (*ra_cnt == 0) {
+		/* Move 'ra' to the end to disable readahead. */
+		*ra = end;
+		return 0;
+	}
+
+	blk_start_plug(&plug);
+	for (unsigned int i = 0; i < *ra_cnt; i++)
+		sb_breadahead(sb, *ra + i);
+	blk_finish_plug(&plug);
+	return 0;
+}
+
 int exfat_chain_cont_cluster(struct super_block *sb, unsigned int chain,
 		unsigned int len)
 {
+	struct buffer_head *bh = NULL;
+	sector_t sec, end, ra;
+	blkcnt_t ra_cnt = 0;
+
 	if (!len)
 		return 0;
 
+	ra = FAT_ENT_OFFSET_SECTOR(sb, chain);
+	end = FAT_ENT_OFFSET_SECTOR(sb, chain + len - 1);
+
 	while (len > 1) {
-		if (exfat_ent_set(sb, chain, chain + 1))
+		sec = FAT_ENT_OFFSET_SECTOR(sb, chain);
+		exfat_blk_readahead(sb, sec, &ra, &ra_cnt, end);
+
+		if (__exfat_ent_set(sb, chain, chain + 1, &bh))
 			return -EIO;
 		chain++;
 		len--;
 	}
 
-	if (exfat_ent_set(sb, chain, EXFAT_EOF_CLUSTER))
+	if (__exfat_ent_set(sb, chain, EXFAT_EOF_CLUSTER, &bh))
 		return -EIO;
+
+	exfat_end_bh(sb, bh);
 	return 0;
 }
 
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 90cd540afeaa7..6fa720e99103c 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -13,6 +13,7 @@
 #include <linux/msdos_fs.h>
 #include <linux/writeback.h>
 #include <linux/filelock.h>
+#include <linux/falloc.h>
 
 #include "exfat_raw.h"
 #include "exfat_fs.h"
@@ -33,6 +34,7 @@ static int exfat_cont_expand(struct inode *inode, loff_t size)
 		return ret;
 
 	num_clusters = EXFAT_B_TO_CLU(exfat_ondisk_size(inode), sbi);
+	/* integer overflow is already checked in inode_newsize_ok(). */
 	new_num_clusters = EXFAT_B_TO_CLU_ROUND_UP(size, sbi);
 
 	if (new_num_clusters == num_clusters)
@@ -90,6 +92,45 @@ static int exfat_cont_expand(struct inode *inode, loff_t size)
 	return -EIO;
 }
 
+/*
+ * Preallocate space for a file. This implements exfat's fallocate file
+ * operation, which gets called from sys_fallocate system call. User space
+ * requests len bytes at offset. In contrary to fat, we only support
+ * FALLOC_FL_ALLOCATE_RANGE because by leaving the valid data length(VDL)
+ * field, it is unnecessary to zero out the newly allocated clusters.
+ */
+static long exfat_fallocate(struct file *file, int mode,
+			  loff_t offset, loff_t len)
+{
+	struct inode *inode = file->f_mapping->host;
+	loff_t newsize = offset + len;
+	int err = 0;
+
+	/* No support for other modes */
+	if (mode != FALLOC_FL_ALLOCATE_RANGE)
+		return -EOPNOTSUPP;
+
+	/* No support for dir */
+	if (!S_ISREG(inode->i_mode))
+		return -EOPNOTSUPP;
+
+	if (unlikely(exfat_forced_shutdown(inode->i_sb)))
+		return -EIO;
+
+	inode_lock(inode);
+
+	if (newsize <= i_size_read(inode))
+		goto error;
+
+	/* This is just an expanding truncate */
+	err = exfat_cont_expand(inode, newsize);
+
+error:
+	inode_unlock(inode);
+
+	return err;
+}
+
 static bool exfat_allow_set_time(struct mnt_idmap *idmap,
 				 struct exfat_sb_info *sbi, struct inode *inode)
 {
@@ -771,6 +812,7 @@ const struct file_operations exfat_file_operations = {
 	.fsync		= exfat_file_fsync,
 	.splice_read	= exfat_splice_read,
 	.splice_write	= iter_file_splice_write,
+	.fallocate	= exfat_fallocate,
 	.setlease	= generic_setlease,
 };
 
diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c
index 2fb2d2d5d503a..beb9ea7cca9f9 100644
--- a/fs/exfat/inode.c
+++ b/fs/exfat/inode.c
@@ -204,8 +204,9 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset,
 				 * so fat-chain should be synced with
 				 * alloc-bitmap
 				 */
-				exfat_chain_cont_cluster(sb, ei->start_clu,
-					num_clusters);
+				if (exfat_chain_cont_cluster(sb, ei->start_clu,
+						num_clusters))
+					return -EIO;
 				ei->flags = ALLOC_FAT_CHAIN;
 			}
 			if (new_clu.flags == ALLOC_FAT_CHAIN)
@@ -213,7 +214,6 @@ static int exfat_map_cluster(struct inode *inode, unsigned int clu_offset,
 					return -EIO;
 		}
 
-		num_clusters += num_to_be_allocated;
 		*clu = new_clu.dir;
 
 		inode->i_blocks += EXFAT_CLU_TO_B(num_to_be_allocated, sbi) >> 9;
@@ -686,7 +686,7 @@ struct inode *exfat_build_inode(struct super_block *sb,
 
 void exfat_evict_inode(struct inode *inode)
 {
-	truncate_inode_pages(&inode->i_data, 0);
+	truncate_inode_pages_final(&inode->i_data);
 
 	if (!inode->i_nlink) {
 		i_size_write(inode, 0);
diff --git a/fs/exfat/misc.c b/fs/exfat/misc.c
index fa84598280461..6f11a96a4ffa8 100644
--- a/fs/exfat/misc.c
+++ b/fs/exfat/misc.c
@@ -161,13 +161,17 @@ u32 exfat_calc_chksum32(void *data, int len, u32 chksum, int type)
 	return chksum;
 }
 
-void exfat_update_bh(struct buffer_head *bh, int sync)
+int exfat_update_bh(struct buffer_head *bh, int sync)
 {
+	int err = 0;
+
 	set_buffer_uptodate(bh);
 	mark_buffer_dirty(bh);
 
 	if (sync)
-		sync_dirty_buffer(bh);
+		err = sync_dirty_buffer(bh);
+
+	return err;
 }
 
 int exfat_update_bhs(struct buffer_head **bhs, int nr_bhs, int sync)
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index 670116ae9ec85..ef2a3488c1b37 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -365,7 +365,8 @@ int exfat_find_empty_entry(struct inode *inode,
 			/* no-fat-chain bit is disabled,
 			 * so fat-chain should be synced with alloc-bitmap
 			 */
-			exfat_chain_cont_cluster(sb, p_dir->dir, p_dir->size);
+			if (exfat_chain_cont_cluster(sb, p_dir->dir, p_dir->size))
+				return -EIO;
 			p_dir->flags = ALLOC_FAT_CHAIN;
 			hint_femp.cur.flags = ALLOC_FAT_CHAIN;
 		}
diff --git a/fs/exfat/super.c b/fs/exfat/super.c
index 83396fd265cda..95d87e2d7717f 100644
--- a/fs/exfat/super.c
+++ b/fs/exfat/super.c
@@ -531,9 +531,14 @@ static int exfat_read_boot_sector(struct super_block *sb)
 	if (sbi->vol_flags & MEDIA_FAILURE)
 		exfat_warn(sb, "Medium has reported failures. Some data may be lost.");
 
-	/* exFAT file size is limited by a disk volume size */
-	sb->s_maxbytes = (u64)(sbi->num_clusters - EXFAT_RESERVED_CLUSTERS) <<
-		sbi->cluster_size_bits;
+	/*
+	 * Set to the max possible volume size for this volume's cluster size so
+	 * that any integer overflow from bytes to cluster size conversion is
+	 * checked in inode_newsize_ok(). Clamped to MAX_LFS_FILESIZE for 32-bit
+	 * machines.
+	 */
+	sb->s_maxbytes = min(MAX_LFS_FILESIZE,
+			     EXFAT_CLU_TO_B((loff_t)EXFAT_MAX_NUM_CLUSTER, sbi));
 
 	/* check logical sector size */
 	if (exfat_calibrate_blocksize(sb, 1 << p_boot->sect_size_bits))
diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c
index 6c9be60a3e48d..5c3183ce350eb 100644
--- a/fs/exportfs/expfs.c
+++ b/fs/exportfs/expfs.c
@@ -130,12 +130,12 @@ static struct dentry *reconnect_one(struct vfsmount *mnt,
 		parent = mnt->mnt_sb->s_export_op->get_parent(dentry);
 
 	if (IS_ERR(parent)) {
-		dprintk("get_parent of %lu failed, err %ld\n",
+		dprintk("get_parent of %llu failed, err %ld\n",
 			dentry->d_inode->i_ino, PTR_ERR(parent));
 		return parent;
 	}
 
-	dprintk("%s: find name of %lu in %lu\n", __func__,
+	dprintk("%s: find name of %llu in %llu\n", __func__,
 		dentry->d_inode->i_ino, parent->d_inode->i_ino);
 	err = exportfs_get_name(mnt, parent, nbuf, dentry);
 	if (err == -ENOENT)
diff --git a/fs/ext2/balloc.c b/fs/ext2/balloc.c
index 007eee794bd17..adf0f31fbddd1 100644
--- a/fs/ext2/balloc.c
+++ b/fs/ext2/balloc.c
@@ -201,7 +201,7 @@ static void group_adjust_blocks(struct super_block *sb, int group_no,
  * windows(start, end). Otherwise, it will only print out the "bad" windows,
  * those windows that overlap with their immediate neighbors.
  */
-#if 1
+#ifdef EXT2FS_DEBUG
 static void __rsv_window_dump(struct rb_root *root, int verbose,
 			      const char *fn)
 {
@@ -248,7 +248,7 @@ static void __rsv_window_dump(struct rb_root *root, int verbose,
 	__rsv_window_dump((root), (verbose), __func__)
 #else
 #define rsv_window_dump(root, verbose) do {} while (0)
-#endif
+#endif /* EXT2FS_DEBUG */
 
 /**
  * goal_in_my_reservation()
diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c
index 395fc36c089b7..278d4be8ecbe7 100644
--- a/fs/ext2/dir.c
+++ b/fs/ext2/dir.c
@@ -141,7 +141,7 @@ static bool ext2_check_folio(struct folio *folio, int quiet, char *kaddr)
 Ebadsize:
 	if (!quiet)
 		ext2_error(sb, __func__,
-			"size of directory #%lu is not a multiple "
+			"size of directory #%llu is not a multiple "
 			"of chunk size", dir->i_ino);
 	goto fail;
 Eshort:
@@ -160,7 +160,7 @@ static bool ext2_check_folio(struct folio *folio, int quiet, char *kaddr)
 	error = "inode out of bounds";
 bad_entry:
 	if (!quiet)
-		ext2_error(sb, __func__, "bad entry in directory #%lu: : %s - "
+		ext2_error(sb, __func__, "bad entry in directory #%llu: : %s - "
 			"offset=%llu, inode=%lu, rec_len=%d, name_len=%d",
 			dir->i_ino, error, folio_pos(folio) + offs,
 			(unsigned long) le32_to_cpu(p->inode),
@@ -170,7 +170,7 @@ static bool ext2_check_folio(struct folio *folio, int quiet, char *kaddr)
 	if (!quiet) {
 		p = (ext2_dirent *)(kaddr + offs);
 		ext2_error(sb, "ext2_check_folio",
-			"entry in directory #%lu spans the page boundary"
+			"entry in directory #%llu spans the page boundary"
 			"offset=%llu, inode=%lu",
 			dir->i_ino, folio_pos(folio) + offs,
 			(unsigned long) le32_to_cpu(p->inode));
@@ -281,7 +281,7 @@ ext2_readdir(struct file *file, struct dir_context *ctx)
 
 		if (IS_ERR(kaddr)) {
 			ext2_error(sb, __func__,
-				   "bad page in #%lu",
+				   "bad page in #%llu",
 				   inode->i_ino);
 			ctx->pos += PAGE_SIZE - offset;
 			return PTR_ERR(kaddr);
@@ -383,7 +383,7 @@ struct ext2_dir_entry_2 *ext2_find_entry (struct inode *dir,
 		/* next folio is past the blocks we've got */
 		if (unlikely(n > (dir->i_blocks >> (PAGE_SHIFT - 9)))) {
 			ext2_error(dir->i_sb, __func__,
-				"dir %lu size %lld exceeds block count %llu",
+				"dir %llu size %lld exceeds block count %llu",
 				dir->i_ino, dir->i_size,
 				(unsigned long long)dir->i_blocks);
 			goto out;
diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c
index fdf63e9c6e7ca..bf21b57cf98cd 100644
--- a/fs/ext2/ialloc.c
+++ b/fs/ext2/ialloc.c
@@ -169,9 +169,10 @@ static void ext2_preread_inode(struct inode *inode)
 	unsigned long block_group;
 	unsigned long offset;
 	unsigned long block;
+	unsigned int ino = inode->i_ino;
 	struct ext2_group_desc * gdp;
 
-	block_group = (inode->i_ino - 1) / EXT2_INODES_PER_GROUP(inode->i_sb);
+	block_group = (ino - 1) / EXT2_INODES_PER_GROUP(inode->i_sb);
 	gdp = ext2_get_group_desc(inode->i_sb, block_group, NULL);
 	if (gdp == NULL)
 		return;
@@ -179,7 +180,7 @@ static void ext2_preread_inode(struct inode *inode)
 	/*
 	 * Figure out the offset within the block group inode table
 	 */
-	offset = ((inode->i_ino - 1) % EXT2_INODES_PER_GROUP(inode->i_sb)) *
+	offset = ((ino - 1) % EXT2_INODES_PER_GROUP(inode->i_sb)) *
 				EXT2_INODE_SIZE(inode->i_sb);
 	block = le32_to_cpu(gdp->bg_inode_table) +
 				(offset >> EXT2_BLOCK_SIZE_BITS(inode->i_sb));
@@ -381,7 +382,7 @@ static int find_group_other(struct super_block *sb, struct inode *parent)
 	 *
 	 * So add our directory's i_ino into the starting point for the hash.
 	 */
-	group = (group + parent->i_ino) % ngroups;
+	group = (group + (unsigned int)parent->i_ino) % ngroups;
 
 	/*
 	 * Use a quadratic hash to find a group with a free inode and some
@@ -589,7 +590,7 @@ struct inode *ext2_new_inode(struct inode *dir, umode_t mode,
 		goto fail_free_drop;
 
 	mark_inode_dirty(inode);
-	ext2_debug("allocating inode %lu\n", inode->i_ino);
+	ext2_debug("allocating inode %llu\n", inode->i_ino);
 	ext2_preread_inode(inode);
 	return inode;
 
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index dbfe9098a1245..997ea4373a412 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -638,7 +638,8 @@ static int ext2_get_blocks(struct inode *inode,
 	int count = 0;
 	ext2_fsblk_t first_block = 0;
 
-	BUG_ON(maxblocks == 0);
+	if (WARN_ON_ONCE(maxblocks == 0))
+		return -EINVAL;
 
 	depth = ext2_block_to_path(inode,iblock,offsets,&blocks_to_boundary);
 
@@ -1152,7 +1153,7 @@ static void ext2_free_branches(struct inode *inode, __le32 *p, __le32 *q, int de
 			 */ 
 			if (!bh) {
 				ext2_error(inode->i_sb, "ext2_free_branches",
-					"Read failure, inode=%ld, block=%ld",
+					"Read failure, inode=%llu, block=%ld",
 					inode->i_ino, nr);
 				continue;
 			}
diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c
index bde617a66cecd..0d09d22fe7086 100644
--- a/fs/ext2/namei.c
+++ b/fs/ext2/namei.c
@@ -14,8 +14,6 @@
  *
  * The only non-static object here is ext2_dir_inode_operations.
  *
- * TODO: get rid of kmap() use, add readahead.
- *
  * Copyright (C) 1992, 1993, 1994, 1995
  * Remy Card (card@masi.ibp.fr)
  * Laboratoire MASI - Institut Blaise Pascal
@@ -293,7 +291,10 @@ static int ext2_unlink(struct inode *dir, struct dentry *dentry)
 		goto out;
 
 	inode_set_ctime_to_ts(inode, inode_get_ctime(dir));
-	inode_dec_link_count(inode);
+
+	if (inode->i_nlink)
+		inode_dec_link_count(inode);
+
 	err = 0;
 out:
 	return err;
diff --git a/fs/ext2/trace.h b/fs/ext2/trace.h
index 7d230e13576e7..0922c0e6aab85 100644
--- a/fs/ext2/trace.h
+++ b/fs/ext2/trace.h
@@ -13,7 +13,7 @@ DECLARE_EVENT_CLASS(ext2_dio_class,
 	TP_ARGS(iocb, iter, ret),
 	TP_STRUCT__entry(
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, isize)
 		__field(loff_t, pos)
 		__field(size_t,	count)
@@ -31,7 +31,7 @@ DECLARE_EVENT_CLASS(ext2_dio_class,
 		__entry->aio = !is_sync_kiocb(iocb);
 		__entry->ret = ret;
 	),
-	TP_printk("dev %d:%d ino 0x%lx isize 0x%llx pos 0x%llx len %zu flags %s aio %d ret %zd",
+	TP_printk("dev %d:%d ino 0x%llx isize 0x%llx pos 0x%llx len %zu flags %s aio %d ret %zd",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->ino,
 		  __entry->isize,
@@ -57,7 +57,7 @@ TRACE_EVENT(ext2_dio_write_endio,
 	TP_ARGS(iocb, size, ret),
 	TP_STRUCT__entry(
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, isize)
 		__field(loff_t, pos)
 		__field(ssize_t, size)
@@ -75,7 +75,7 @@ TRACE_EVENT(ext2_dio_write_endio,
 		__entry->aio = !is_sync_kiocb(iocb);
 		__entry->ret = ret;
 	),
-	TP_printk("dev %d:%d ino 0x%lx isize 0x%llx pos 0x%llx len %zd flags %s aio %d ret %d",
+	TP_printk("dev %d:%d ino 0x%llx isize 0x%llx pos 0x%llx len %zd flags %s aio %d ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->ino,
 		  __entry->isize,
diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c
index c885dcc3bd0d1..e55d16abf422f 100644
--- a/fs/ext2/xattr.c
+++ b/fs/ext2/xattr.c
@@ -73,7 +73,7 @@
 
 #ifdef EXT2_XATTR_DEBUG
 # define ea_idebug(inode, f...) do { \
-		printk(KERN_DEBUG "inode %s:%ld: ", \
+		printk(KERN_DEBUG "inode %s:%llu: ", \
 			inode->i_sb->s_id, inode->i_ino); \
 		printk(f); \
 		printk("\n"); \
@@ -227,7 +227,7 @@ ext2_xattr_get(struct inode *inode, int name_index, const char *name,
 	if (!ext2_xattr_header_valid(HDR(bh))) {
 bad_block:
 		ext2_error(inode->i_sb, "ext2_xattr_get",
-			"inode %ld: bad block %d", inode->i_ino,
+			"inode %llu: bad block %d", inode->i_ino,
 			EXT2_I(inode)->i_file_acl);
 		error = -EIO;
 		goto cleanup;
@@ -313,7 +313,7 @@ ext2_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size)
 	if (!ext2_xattr_header_valid(HDR(bh))) {
 bad_block:
 		ext2_error(inode->i_sb, "ext2_xattr_list",
-			"inode %ld: bad block %d", inode->i_ino,
+			"inode %llu: bad block %d", inode->i_ino,
 			EXT2_I(inode)->i_file_acl);
 		error = -EIO;
 		goto cleanup;
@@ -454,7 +454,7 @@ ext2_xattr_set(struct inode *inode, int name_index, const char *name,
 		if (!ext2_xattr_header_valid(header)) {
 bad_block:
 			ext2_error(sb, "ext2_xattr_set",
-				"inode %ld: bad block %d", inode->i_ino, 
+				"inode %llu: bad block %d", inode->i_ino,
 				   EXT2_I(inode)->i_file_acl);
 			error = -EIO;
 			goto cleanup;
@@ -833,7 +833,7 @@ ext2_xattr_delete_inode(struct inode *inode)
 
 	if (!ext2_data_block_valid(sbi, EXT2_I(inode)->i_file_acl, 1)) {
 		ext2_error(inode->i_sb, "ext2_xattr_delete_inode",
-			"inode %ld: xattr block %d is out of data blocks range",
+			"inode %llu: xattr block %d is out of data blocks range",
 			inode->i_ino, EXT2_I(inode)->i_file_acl);
 		goto cleanup;
 	}
@@ -841,14 +841,14 @@ ext2_xattr_delete_inode(struct inode *inode)
 	bh = sb_bread(inode->i_sb, EXT2_I(inode)->i_file_acl);
 	if (!bh) {
 		ext2_error(inode->i_sb, "ext2_xattr_delete_inode",
-			"inode %ld: block %d read error", inode->i_ino,
+			"inode %llu: block %d read error", inode->i_ino,
 			EXT2_I(inode)->i_file_acl);
 		goto cleanup;
 	}
 	ea_bdebug(bh, "b_count=%d", atomic_read(&(bh->b_count)));
 	if (!ext2_xattr_header_valid(HDR(bh))) {
 		ext2_error(inode->i_sb, "ext2_xattr_delete_inode",
-			"inode %ld: bad block %d", inode->i_ino,
+			"inode %llu: bad block %d", inode->i_ino,
 			EXT2_I(inode)->i_file_acl);
 		goto cleanup;
 	}
@@ -952,7 +952,7 @@ ext2_xattr_cache_find(struct inode *inode, struct ext2_xattr_header *header)
 		bh = sb_bread(inode->i_sb, ce->e_value);
 		if (!bh) {
 			ext2_error(inode->i_sb, "ext2_xattr_cache_find",
-				"inode %ld: block %ld read error",
+				"inode %llu: block %ld read error",
 				inode->i_ino, (unsigned long) ce->e_value);
 		} else {
 			lock_buffer(bh);
diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c
index 28b2a3deb954f..17edd678fa87b 100644
--- a/fs/ext4/dir.c
+++ b/fs/ext4/dir.c
@@ -535,7 +535,7 @@ static int call_filldir(struct file *file, struct dir_context *ctx,
 	struct super_block *sb = inode->i_sb;
 
 	if (!fname) {
-		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: comm %s: "
+		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: comm %s: "
 			 "called with null fname?!?", __func__, __LINE__,
 			 inode->i_ino, current->comm);
 		return 0;
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 293f698b70424..85e6c2b543a82 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -92,7 +92,7 @@
  */
 #ifdef CONFIG_EXT4_DEBUG
 #define ext_debug(ino, fmt, ...)					\
-	pr_debug("[%s/%d] EXT4-fs (%s): ino %lu: (%s, %d): %s:" fmt,	\
+	pr_debug("[%s/%d] EXT4-fs (%s): ino %llu: (%s, %d): %s:" fmt,	\
 		 current->comm, task_pid_nr(current),			\
 		 ino->i_sb->s_id, ino->i_ino, __FILE__, __LINE__,	\
 		 __func__, ##__VA_ARGS__)
@@ -3229,7 +3229,7 @@ extern void __dump_mmp_msg(struct super_block *, struct mmp_struct *mmp,
 extern __printf(7, 8)
 void __ext4_grp_locked_error(const char *, unsigned int,
 			     struct super_block *, ext4_group_t,
-			     unsigned long, ext4_fsblk_t,
+			     u64, ext4_fsblk_t,
 			     const char *, ...);
 
 #define EXT4_ERROR_INODE(inode, fmt, a...) \
diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c
index 7c4690eb7dade..a6b3e6b592a5a 100644
--- a/fs/ext4/extents-test.c
+++ b/fs/ext4/extents-test.c
@@ -142,8 +142,10 @@ static struct file_system_type ext_fs_type = {
 
 static void extents_kunit_exit(struct kunit *test)
 {
-	struct ext4_sb_info *sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info;
+	struct super_block *sb = k_ctx.k_ei->vfs_inode.i_sb;
+	struct ext4_sb_info *sbi = sb->s_fs_info;
 
+	ext4_es_unregister_shrinker(sbi);
 	kfree(sbi);
 	kfree(k_ctx.k_ei);
 	kfree(k_ctx.k_data);
diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index ae3804f36535a..042e1555a6741 100644
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -4603,7 +4603,7 @@ static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
 		}
 		ret = ext4_map_blocks(handle, inode, &map, flags);
 		if (ret <= 0) {
-			ext4_debug("inode #%lu: block %u: len %u: "
+			ext4_debug("inode #%llu: block %u: len %u: "
 				   "ext4_ext_map_blocks returned %d",
 				   inode->i_ino, map.m_lblk,
 				   map.m_len, ret);
@@ -4955,7 +4955,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
 		ret = ext4_map_blocks(handle, inode, &map, flags);
 		if (ret != max_blocks)
 			ext4_msg(inode->i_sb, KERN_INFO,
-				     "inode #%lu: block %u: len %u: "
+				     "inode #%llu: block %u: len %u: "
 				     "split block mapping found for atomic write, "
 				     "ret = %d",
 				     inode->i_ino, map.m_lblk,
@@ -4974,7 +4974,7 @@ int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
 
 	if (ret <= 0 || ret2)
 		ext4_warning(inode->i_sb,
-			     "inode #%lu: block %u: len %u: "
+			     "inode #%llu: block %u: len %u: "
 			     "returned %d or %d",
 			     inode->i_ino, map.m_lblk,
 			     map.m_len, ret, ret2);
@@ -5031,7 +5031,7 @@ int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
 				      EXT4_EX_NOCACHE);
 		if (ret <= 0)
 			ext4_warning(inode->i_sb,
-				     "inode #%lu: block %u: len %u: "
+				     "inode #%llu: block %u: len %u: "
 				     "ext4_ext_map_blocks returned %d",
 				     inode->i_ino, map.m_lblk,
 				     map.m_len, ret);
diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c
index a1538bac51c61..6e4a191e82191 100644
--- a/fs/ext4/extents_status.c
+++ b/fs/ext4/extents_status.c
@@ -214,7 +214,7 @@ static void ext4_es_print_tree(struct inode *inode)
 	struct ext4_es_tree *tree;
 	struct rb_node *node;
 
-	printk(KERN_DEBUG "status extents for inode %lu:", inode->i_ino);
+	printk(KERN_DEBUG "status extents for inode %llu:", inode->i_ino);
 	tree = &EXT4_I(inode)->i_es_tree;
 	node = rb_first(&tree->root);
 	while (node) {
@@ -703,7 +703,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode,
 		if (!ext4_es_is_written(es) && !ext4_es_is_unwritten(es)) {
 			if (in_range(es->es_lblk, ee_block, ee_len)) {
 				pr_warn("ES insert assertion failed for "
-					"inode: %lu we can find an extent "
+					"inode: %llu we can find an extent "
 					"at block [%d/%d/%llu/%c], but we "
 					"want to add a delayed/hole extent "
 					"[%d/%d/%llu/%x]\n",
@@ -721,7 +721,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode,
 		 */
 		if (es->es_lblk < ee_block ||
 		    ext4_es_pblock(es) != ee_start + es->es_lblk - ee_block) {
-			pr_warn("ES insert assertion failed for inode: %lu "
+			pr_warn("ES insert assertion failed for inode: %llu "
 				"ex_status [%d/%d/%llu/%c] != "
 				"es_status [%d/%d/%llu/%c]\n", inode->i_ino,
 				ee_block, ee_len, ee_start,
@@ -731,7 +731,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode,
 		}
 
 		if (ee_status ^ es_status) {
-			pr_warn("ES insert assertion failed for inode: %lu "
+			pr_warn("ES insert assertion failed for inode: %llu "
 				"ex_status [%d/%d/%llu/%c] != "
 				"es_status [%d/%d/%llu/%c]\n", inode->i_ino,
 				ee_block, ee_len, ee_start,
@@ -744,7 +744,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode,
 		 * that we don't want to add an written/unwritten extent.
 		 */
 		if (!ext4_es_is_delayed(es) && !ext4_es_is_hole(es)) {
-			pr_warn("ES insert assertion failed for inode: %lu "
+			pr_warn("ES insert assertion failed for inode: %llu "
 				"can't find an extent at block %d but we want "
 				"to add a written/unwritten extent "
 				"[%d/%d/%llu/%x]\n", inode->i_ino,
@@ -779,7 +779,7 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode,
 			 * We want to add a delayed/hole extent but this
 			 * block has been allocated.
 			 */
-			pr_warn("ES insert assertion failed for inode: %lu "
+			pr_warn("ES insert assertion failed for inode: %llu "
 				"We can find blocks but we want to add a "
 				"delayed/hole extent [%d/%d/%llu/%x]\n",
 				inode->i_ino, es->es_lblk, es->es_len,
@@ -788,13 +788,13 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode,
 		} else if (ext4_es_is_written(es)) {
 			if (retval != es->es_len) {
 				pr_warn("ES insert assertion failed for "
-					"inode: %lu retval %d != es_len %d\n",
+					"inode: %llu retval %d != es_len %d\n",
 					inode->i_ino, retval, es->es_len);
 				return;
 			}
 			if (map.m_pblk != ext4_es_pblock(es)) {
 				pr_warn("ES insert assertion failed for "
-					"inode: %lu m_pblk %llu != "
+					"inode: %llu m_pblk %llu != "
 					"es_pblk %llu\n",
 					inode->i_ino, map.m_pblk,
 					ext4_es_pblock(es));
@@ -809,7 +809,7 @@ static void ext4_es_insert_extent_ind_check(struct inode *inode,
 		}
 	} else if (retval == 0) {
 		if (ext4_es_is_written(es)) {
-			pr_warn("ES insert assertion failed for inode: %lu "
+			pr_warn("ES insert assertion failed for inode: %llu "
 				"We can't find the block but we want to add "
 				"a written extent [%d/%d/%llu/%x]\n",
 				inode->i_ino, es->es_lblk, es->es_len,
@@ -919,7 +919,7 @@ void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk,
 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
 		return;
 
-	es_debug("add [%u/%u) %llu %x %d to extent status tree of inode %lu\n",
+	es_debug("add [%u/%u) %llu %x %d to extent status tree of inode %llu\n",
 		 lblk, len, pblk, status, delalloc_reserve_used, inode->i_ino);
 
 	if (!len)
@@ -1631,7 +1631,7 @@ void ext4_es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
 		return;
 
-	es_debug("remove [%u/%u) from extent status tree of inode %lu\n",
+	es_debug("remove [%u/%u) from extent status tree of inode %llu\n",
 		 lblk, len, inode->i_ino);
 
 	if (!len)
@@ -1821,7 +1821,7 @@ int ext4_seq_es_shrinker_info_show(struct seq_file *seq, void *v)
 	seq_printf(seq, "  %lu shrunk objects\n", es_stats->es_stats_shrunk);
 	if (inode_cnt)
 		seq_printf(seq,
-		    "maximum:\n  %lu inode (%u objects, %u reclaimable)\n"
+		    "maximum:\n  %llu inode (%u objects, %u reclaimable)\n"
 		    "  %llu us max scan time\n",
 		    max->vfs_inode.i_ino, max->i_es_all_nr, max->i_es_shk_nr,
 		    div_u64(es_stats->es_stats_max_scan_time, 1000));
@@ -1998,7 +1998,7 @@ static void ext4_print_pending_tree(struct inode *inode)
 	struct rb_node *node;
 	struct pending_reservation *pr;
 
-	printk(KERN_DEBUG "pending reservations for inode %lu:", inode->i_ino);
+	printk(KERN_DEBUG "pending reservations for inode %llu:", inode->i_ino);
 	tree = &EXT4_I(inode)->i_pending_tree;
 	node = rb_first(&tree->root);
 	while (node) {
@@ -2214,7 +2214,7 @@ void ext4_es_insert_delayed_extent(struct inode *inode, ext4_lblk_t lblk,
 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
 		return;
 
-	es_debug("add [%u/%u) delayed to extent status tree of inode %lu\n",
+	es_debug("add [%u/%u) delayed to extent status tree of inode %llu\n",
 		 lblk, len, inode->i_ino);
 	if (!len)
 		return;
diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c
index f575751f1cae4..379fb66dedbcd 100644
--- a/fs/ext4/fast_commit.c
+++ b/fs/ext4/fast_commit.c
@@ -616,7 +616,7 @@ static int __track_range(handle_t *handle, struct inode *inode, void *arg,
 		(struct __track_range_args *)arg;
 
 	if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
-		ext4_debug("Special inode %ld being modified\n", inode->i_ino);
+		ext4_debug("Special inode %llu being modified\n", inode->i_ino);
 		return -ECANCELED;
 	}
 
@@ -914,7 +914,7 @@ static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
 	spin_unlock(&ei->i_fc_lock);
 
 	cur_lblk_off = old_blk_size;
-	ext4_debug("will try writing %d to %d for inode %ld\n",
+	ext4_debug("will try writing %d to %d for inode %llu\n",
 		   cur_lblk_off, new_blk_size, inode->i_ino);
 
 	while (cur_lblk_off <= new_blk_size) {
@@ -1792,7 +1792,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb,
 
 	cur = start;
 	remaining = len;
-	ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
+	ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %llu\n",
 		  start, start_pblk, len, ext4_ext_is_unwritten(ex),
 		  inode->i_ino);
 
@@ -1903,7 +1903,7 @@ ext4_fc_replay_del_range(struct super_block *sb,
 	if (ret)
 		goto out;
 
-	ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n",
+	ext4_debug("DEL_RANGE, inode %llu, lblk %d, len %d\n",
 			inode->i_ino, le32_to_cpu(lrange.fc_lblk),
 			le32_to_cpu(lrange.fc_len));
 	while (remaining > 0) {
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index f1dc5ce791a7d..5e02f6cf653e4 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -27,7 +27,6 @@
 #include <linux/dax.h>
 #include <linux/filelock.h>
 #include <linux/quotaops.h>
-#include <linux/pagevec.h>
 #include <linux/uio.h>
 #include <linux/mman.h>
 #include <linux/backing-dev.h>
diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
index b20a1bf866abe..628a74b2bbe62 100644
--- a/fs/ext4/ialloc.c
+++ b/fs/ext4/ialloc.c
@@ -253,13 +253,13 @@ void ext4_free_inode(handle_t *handle, struct inode *inode)
 		return;
 	}
 	if (icount_read(inode) > 1) {
-		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: count=%d",
+		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d",
 			 __func__, __LINE__, inode->i_ino,
 			 icount_read(inode));
 		return;
 	}
 	if (inode->i_nlink) {
-		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%lu: nlink=%d\n",
+		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: nlink=%d\n",
 			 __func__, __LINE__, inode->i_ino, inode->i_nlink);
 		return;
 	}
@@ -631,7 +631,7 @@ static int find_group_other(struct super_block *sb, struct inode *parent,
 	 *
 	 * So add our directory's i_ino into the starting point for the hash.
 	 */
-	*group = (*group + parent->i_ino) % ngroups;
+	*group = (*group + (unsigned int)parent->i_ino) % ngroups;
 
 	/*
 	 * Use a quadratic hash to find a group with a free inode and some free
@@ -1275,7 +1275,7 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap,
 		 * twice.
 		 */
 		err = -EIO;
-		ext4_error(sb, "failed to insert inode %lu: doubly allocated?",
+		ext4_error(sb, "failed to insert inode %llu: doubly allocated?",
 			   inode->i_ino);
 		ext4_mark_group_bitmap_corrupted(sb, group,
 					EXT4_GROUP_INFO_IBITMAP_CORRUPT);
@@ -1344,7 +1344,7 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap,
 		goto fail_free_drop;
 	}
 
-	ext4_debug("allocating inode %lu\n", inode->i_ino);
+	ext4_debug("allocating inode %llu\n", inode->i_ino);
 	trace_ext4_allocate_inode(inode, dir, mode);
 	brelse(inode_bitmap_bh);
 	return ret;
diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c
index da76353b3a575..5aec759eed705 100644
--- a/fs/ext4/indirect.c
+++ b/fs/ext4/indirect.c
@@ -102,7 +102,7 @@ static int ext4_block_to_path(struct inode *inode,
 		offsets[n++] = i_block & (ptrs - 1);
 		final = ptrs;
 	} else {
-		ext4_warning(inode->i_sb, "block %lu > max in inode %lu",
+		ext4_warning(inode->i_sb, "block %lu > max in inode %llu",
 			     i_block + direct_blocks +
 			     indirect_blocks + double_blocks, inode->i_ino);
 	}
diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index 1f6bc05593df1..f846fcb7db244 100644
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -119,7 +119,7 @@ int ext4_get_max_inline_size(struct inode *inode)
 	error = ext4_get_inode_loc(inode, &iloc);
 	if (error) {
 		ext4_error_inode_err(inode, __func__, __LINE__, 0, -error,
-				     "can't get inode location %lu",
+				     "can't get inode location %llu",
 				     inode->i_ino);
 		return 0;
 	}
@@ -512,7 +512,7 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio)
 	BUG_ON(folio->index);
 
 	if (!EXT4_I(inode)->i_inline_off) {
-		ext4_warning(inode->i_sb, "inode %lu doesn't have inline data.",
+		ext4_warning(inode->i_sb, "inode %llu doesn't have inline data.",
 			     inode->i_ino);
 		goto out;
 	}
@@ -934,7 +934,7 @@ void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh,
 	struct ext4_dir_entry_2 *de = inline_start;
 	void *dlimit = inline_start + inline_size;
 
-	trace_printk("inode %lu\n", dir->i_ino);
+	trace_printk("inode %llu\n", dir->i_ino);
 	offset = 0;
 	while ((void *)de < dlimit) {
 		de_len = ext4_rec_len_from_disk(de->rec_len, inline_size);
@@ -1071,7 +1071,7 @@ static void ext4_restore_inline_data(handle_t *handle, struct inode *inode,
 	ret = ext4_create_inline_data(handle, inode, inline_size);
 	if (ret) {
 		ext4_msg(inode->i_sb, KERN_EMERG,
-			"error restoring inline_data for inode -- potential data loss! (inode %lu, error %d)",
+			"error restoring inline_data for inode -- potential data loss! (inode %llu, error %d)",
 			inode->i_ino, ret);
 		return;
 	}
@@ -1740,7 +1740,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
 	err = ext4_get_inode_loc(dir, &iloc);
 	if (err) {
 		EXT4_ERROR_INODE_ERR(dir, -err,
-				     "error %d getting inode %lu block",
+				     "error %d getting inode %llu block",
 				     err, dir->i_ino);
 		return false;
 	}
@@ -1755,7 +1755,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
 	de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block;
 	if (!le32_to_cpu(de->inode)) {
 		ext4_warning(dir->i_sb,
-			     "bad inline directory (dir #%lu) - no `..'",
+			     "bad inline directory (dir #%llu) - no `..'",
 			     dir->i_ino);
 		goto out;
 	}
@@ -1769,7 +1769,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data)
 					 iloc.bh, inline_pos,
 					 inline_size, offset)) {
 			ext4_warning(dir->i_sb,
-				     "bad inline directory (dir #%lu) - "
+				     "bad inline directory (dir #%llu) - "
 				     "inode %u, rec_len %u, name_len %d"
 				     "inline size %d",
 				     dir->i_ino, le32_to_cpu(de->inode),
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 396dc3a5d16b4..35b8c1541702d 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/buffer_head.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/mpage.h>
 #include <linux/rmap.h>
 #include <linux/namei.h>
@@ -262,7 +262,7 @@ void ext4_evict_inode(struct inode *inode)
 		err = ext4_truncate(inode);
 		if (err) {
 			ext4_error_err(inode->i_sb, -err,
-				       "couldn't truncate inode %lu (err %d)",
+				       "couldn't truncate inode %llu (err %d)",
 				       inode->i_ino, err);
 			goto stop_handle;
 		}
@@ -342,7 +342,7 @@ void ext4_da_update_reserve_space(struct inode *inode,
 	spin_lock(&ei->i_block_reservation_lock);
 	trace_ext4_da_update_reserve_space(inode, used, quota_claim);
 	if (unlikely(used > ei->i_reserved_data_blocks)) {
-		ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
+		ext4_warning(inode->i_sb, "%s: ino %llu, used %d "
 			 "with only %d reserved data blocks",
 			 __func__, inode->i_ino, used,
 			 ei->i_reserved_data_blocks);
@@ -405,7 +405,10 @@ int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
 	KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len);
 
 	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
-		return fscrypt_zeroout_range(inode, lblk, pblk, len);
+		return fscrypt_zeroout_range(inode,
+				(loff_t)lblk << inode->i_blkbits,
+				pblk << (inode->i_blkbits - SECTOR_SHIFT),
+				(u64)len << inode->i_blkbits);
 
 	ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
 	if (ret > 0)
@@ -475,7 +478,7 @@ static void ext4_map_blocks_es_recheck(handle_t *handle,
 	if (es_map->m_lblk != map->m_lblk ||
 	    es_map->m_flags != map->m_flags ||
 	    es_map->m_pblk != map->m_pblk) {
-		printk("ES cache assertion failed for inode: %lu "
+		printk("ES cache assertion failed for inode: %llu "
 		       "es_cached ex [%d/%d/%llu/%x] != "
 		       "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
 		       inode->i_ino, es_map->m_lblk, es_map->m_len,
@@ -515,7 +518,7 @@ static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
 	if (unlikely(retval != map2.m_len)) {
 		ext4_warning(inode->i_sb,
 			     "ES len assertion failed for inode "
-			     "%lu: retval %d != map->m_len %d",
+			     "%llu: retval %d != map->m_len %d",
 			     inode->i_ino, retval, map2.m_len);
 		WARN_ON(1);
 	}
@@ -563,7 +566,7 @@ int ext4_map_query_blocks(handle_t *handle, struct inode *inode,
 	if (unlikely(retval != map->m_len)) {
 		ext4_warning(inode->i_sb,
 			     "ES len assertion failed for inode "
-			     "%lu: retval %d != map->m_len %d",
+			     "%llu: retval %d != map->m_len %d",
 			     inode->i_ino, retval, map->m_len);
 		WARN_ON(1);
 	}
@@ -630,7 +633,7 @@ int ext4_map_create_blocks(handle_t *handle, struct inode *inode,
 
 	if (unlikely(retval != map->m_len)) {
 		ext4_warning(inode->i_sb,
-			     "ES len assertion failed for inode %lu: "
+			     "ES len assertion failed for inode %llu: "
 			     "retval %d != map->m_len %d",
 			     inode->i_ino, retval, map->m_len);
 		WARN_ON(1);
@@ -937,7 +940,7 @@ int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
 {
 	int ret = 0;
 
-	ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
+	ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n",
 		   inode->i_ino, create);
 	ret = _ext4_get_block(inode, iblock, bh_result,
 			       EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
@@ -1659,7 +1662,7 @@ void ext4_da_release_space(struct inode *inode, int to_free)
 		 * harmless to return without any action.
 		 */
 		ext4_warning(inode->i_sb, "ext4_da_release_space: "
-			 "ino %lu, to_free %d with only %d reserved "
+			 "ino %llu, to_free %d with only %d reserved "
 			 "data blocks", inode->i_ino, to_free,
 			 ei->i_reserved_data_blocks);
 		WARN_ON(1);
@@ -2491,7 +2494,7 @@ static int mpage_map_and_submit_extent(handle_t *handle,
 			}
 			ext4_msg(sb, KERN_CRIT,
 				 "Delayed block allocation failed for "
-				 "inode %lu at logical offset %llu with"
+				 "inode %llu at logical offset %llu with"
 				 " max blocks %u with error %d",
 				 inode->i_ino,
 				 (unsigned long long)map->m_lblk,
@@ -2535,7 +2538,7 @@ static int mpage_map_and_submit_extent(handle_t *handle,
 		err2 = ext4_mark_inode_dirty(handle, inode);
 		if (err2) {
 			ext4_error_err(inode->i_sb, -err2,
-				       "Failed to mark inode %lu dirty",
+				       "Failed to mark inode %llu dirty",
 				       inode->i_ino);
 		}
 		if (!err)
@@ -2909,7 +2912,7 @@ static int ext4_do_writepages(struct mpage_da_data *mpd)
 		if (IS_ERR(handle)) {
 			ret = PTR_ERR(handle);
 			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
-			       "%ld pages, ino %lu; err %d", __func__,
+			       "%ld pages, ino %llu; err %d", __func__,
 				wbc->nr_to_write, inode->i_ino, ret);
 			/* Release allocated io_end */
 			ext4_put_io_end(mpd->io_submit.io_end);
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index 3ae9cb50a0c05..1d0c3d4bdf472 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -477,7 +477,7 @@ static long swap_inode_boot_loader(struct super_block *sb,
 	if (err < 0) {
 		/* No need to update quota information. */
 		ext4_warning(inode->i_sb,
-			"couldn't mark inode #%lu dirty (err %d)",
+			"couldn't mark inode #%llu dirty (err %d)",
 			inode->i_ino, err);
 		/* Revert all changes: */
 		swap_inode_data(inode, inode_bl);
@@ -493,7 +493,7 @@ static long swap_inode_boot_loader(struct super_block *sb,
 	if (err < 0) {
 		/* No need to update quota information. */
 		ext4_warning(inode_bl->i_sb,
-			"couldn't mark inode #%lu dirty (err %d)",
+			"couldn't mark inode #%llu dirty (err %d)",
 			inode_bl->i_ino, err);
 		goto revert;
 	}
diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c
index 9fbdf6a09489c..b9f22e3a8d5cb 100644
--- a/fs/ext4/mballoc-test.c
+++ b/fs/ext4/mballoc-test.c
@@ -72,7 +72,7 @@ static int mbt_mb_init(struct super_block *sb)
 	ext4_fsblk_t block;
 	int ret;
 
-	/* needed by ext4_mb_init->bdev_nonrot(sb->s_bdev) */
+	/* needed by ext4_mb_init->bdev_rot(sb->s_bdev) */
 	sb->s_bdev = kzalloc_obj(*sb->s_bdev);
 	if (sb->s_bdev == NULL)
 		return -ENOMEM;
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index 20e9fdaf4301b..51476d121f72c 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -2266,7 +2266,7 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
 	folio_get(ac->ac_buddy_folio);
 	/* store last allocated for subsequent stream allocation */
 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
-		int hash = ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals;
+		int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals;
 
 		WRITE_ONCE(sbi->s_mb_last_groups[hash], ac->ac_f_ex.fe_group);
 	}
@@ -3032,7 +3032,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
 
 	/* if stream allocation is enabled, use global goal */
 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
-		int hash = ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals;
+		int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals;
 
 		ac->ac_g_ex.fe_group = READ_ONCE(sbi->s_mb_last_groups[hash]);
 		ac->ac_g_ex.fe_start = -1;
@@ -3836,7 +3836,7 @@ int ext4_mb_init(struct super_block *sb)
 		spin_lock_init(&lg->lg_prealloc_lock);
 	}
 
-	if (bdev_nonrot(sb->s_bdev))
+	if (!bdev_rot(sb->s_bdev))
 		sbi->s_mb_max_linear_groups = 0;
 	else
 		sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT;
@@ -5628,7 +5628,7 @@ void ext4_discard_preallocations(struct inode *inode)
 	if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)
 		return;
 
-	mb_debug(sb, "discard preallocation for inode %lu\n",
+	mb_debug(sb, "discard preallocation for inode %llu\n",
 		 inode->i_ino);
 	trace_ext4_discard_preallocations(inode,
 			atomic_read(&ei->i_prealloc_active));
diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c
index 96ab95167bd6e..477d43d7e2942 100644
--- a/fs/ext4/migrate.c
+++ b/fs/ext4/migrate.c
@@ -455,7 +455,7 @@ int ext4_ext_migrate(struct inode *inode)
 	 * log, so disable fast commits for this transaction.
 	 */
 	ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_MIGRATE, handle);
-	goal = (((inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) *
+	goal = ((((u32)inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) *
 		EXT4_INODES_PER_GROUP(inode->i_sb)) + 1;
 	owner[0] = i_uid_read(inode);
 	owner[1] = i_gid_read(inode);
diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c
index ce1f738dff938..ab17c1d3a7b51 100644
--- a/fs/ext4/move_extent.c
+++ b/fs/ext4/move_extent.c
@@ -420,21 +420,21 @@ static int mext_check_validity(struct inode *orig_inode,
 
 	/* origin and donor should be different inodes */
 	if (orig_inode == donor_inode) {
-		ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
 
 	/* origin and donor should belone to the same filesystem */
 	if (orig_inode->i_sb != donor_inode->i_sb) {
-		ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
 
 	/* Regular file check */
 	if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) {
-		ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
@@ -477,26 +477,26 @@ static int mext_check_validity(struct inode *orig_inode,
 	}
 
 	if (donor_inode->i_mode & (S_ISUID|S_ISGID)) {
-		ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
 
 	if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) {
-		ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EPERM;
 	}
 
 	/* Ext4 move extent does not support swap files */
 	if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) {
-		ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -ETXTBSY;
 	}
 
 	if (ext4_is_quota_file(orig_inode) || ext4_is_quota_file(donor_inode)) {
-		ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EOPNOTSUPP;
 	}
@@ -523,7 +523,7 @@ static int mext_check_adjust_range(struct inode *orig_inode,
 	/* Start offset should be same */
 	if ((orig_start & ~(PAGE_MASK >> orig_inode->i_blkbits)) !=
 	    (donor_start & ~(PAGE_MASK >> orig_inode->i_blkbits))) {
-		ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
@@ -533,7 +533,7 @@ static int mext_check_adjust_range(struct inode *orig_inode,
 	    (*len > EXT_MAX_BLOCKS) ||
 	    (donor_start + *len >= EXT_MAX_BLOCKS) ||
 	    (orig_start + *len >= EXT_MAX_BLOCKS))  {
-		ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %llu, donor %llu]\n",
 			   EXT_MAX_BLOCKS,
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
@@ -550,7 +550,7 @@ static int mext_check_adjust_range(struct inode *orig_inode,
 	else if (donor_eof < donor_start + *len - 1)
 		*len = donor_eof - donor_start;
 	if (!*len) {
-		ext4_debug("ext4 move extent: len should not be 0 [ino:orig %lu, donor %lu]\n",
+		ext4_debug("ext4 move extent: len should not be 0 [ino:orig %llu, donor %llu]\n",
 			   orig_inode->i_ino, donor_inode->i_ino);
 		return -EINVAL;
 	}
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index c4b5e252af0ef..503dc9ffd6146 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -144,7 +144,7 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,
 		bh = ext4_bread(NULL, inode, block, 0);
 	if (IS_ERR(bh)) {
 		__ext4_warning(inode->i_sb, func, line,
-			       "inode #%lu: lblock %lu: comm %s: "
+			       "inode #%llu: lblock %lu: comm %s: "
 			       "error %ld reading directory block",
 			       inode->i_ino, (unsigned long)block,
 			       current->comm, PTR_ERR(bh));
@@ -841,7 +841,7 @@ dx_probe(struct ext4_filename *fname, struct inode *dir,
 	indirect = root->info.indirect_levels;
 	if (indirect >= ext4_dir_htree_level(dir->i_sb)) {
 		ext4_warning(dir->i_sb,
-			     "Directory (ino: %lu) htree depth %#06x exceed"
+			     "Directory (ino: %llu) htree depth %#06x exceed"
 			     "supported value", dir->i_ino,
 			     ext4_dir_htree_level(dir->i_sb));
 		if (ext4_dir_htree_level(dir->i_sb) < EXT4_HTREE_LEVEL) {
@@ -1793,7 +1793,7 @@ static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsi
 		    (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
 		    !fscrypt_has_permitted_context(dir, inode)) {
 			ext4_warning(inode->i_sb,
-				     "Inconsistent encryption contexts: %lu/%lu",
+				     "Inconsistent encryption contexts: %llu/%llu",
 				     dir->i_ino, inode->i_ino);
 			iput(inode);
 			return ERR_PTR(-EPERM);
@@ -2227,7 +2227,7 @@ static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname,
 		csum_size = sizeof(struct ext4_dir_entry_tail);
 
 	blocksize =  dir->i_sb->s_blocksize;
-	dxtrace(printk(KERN_DEBUG "Creating index: inode %lu\n", dir->i_ino));
+	dxtrace(printk(KERN_DEBUG "Creating index: inode %llu\n", dir->i_ino));
 	BUFFER_TRACE(bh, "get_write_access");
 	retval = ext4_journal_get_write_access(handle, dir->i_sb, bh,
 					       EXT4_JTR_NONE);
@@ -2523,7 +2523,7 @@ static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
 			restart = 1;
 		}
 		if (add_level && levels == ext4_dir_htree_level(sb)) {
-			ext4_warning(sb, "Directory (ino: %lu) index full, "
+			ext4_warning(sb, "Directory (ino: %llu) index full, "
 					 "reach max htree level :%d",
 					 dir->i_ino, levels);
 			if (ext4_dir_htree_level(sb) < EXT4_HTREE_LEVEL) {
diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c
index c0022f0bff874..64ea476242338 100644
--- a/fs/ext4/orphan.c
+++ b/fs/ext4/orphan.c
@@ -179,8 +179,8 @@ int ext4_orphan_add(handle_t *handle, struct inode *inode)
 	} else
 		brelse(iloc.bh);
 
-	ext4_debug("superblock will point to %lu\n", inode->i_ino);
-	ext4_debug("orphan inode %lu will point to %d\n",
+	ext4_debug("superblock will point to %llu\n", inode->i_ino);
+	ext4_debug("orphan inode %llu will point to %d\n",
 			inode->i_ino, NEXT_ORPHAN(inode));
 out:
 	ext4_std_error(sb, err);
@@ -249,7 +249,7 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode)
 	}
 
 	mutex_lock(&sbi->s_orphan_lock);
-	ext4_debug("remove inode %lu from orphan list\n", inode->i_ino);
+	ext4_debug("remove inode %llu from orphan list\n", inode->i_ino);
 
 	prev = ei->i_orphan.prev;
 	list_del_init(&ei->i_orphan);
@@ -284,7 +284,7 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode)
 		struct inode *i_prev =
 			&list_entry(prev, struct ext4_inode_info, i_orphan)->vfs_inode;
 
-		ext4_debug("orphan inode %lu will point to %u\n",
+		ext4_debug("orphan inode %llu will point to %u\n",
 			  i_prev->i_ino, ino_next);
 		err = ext4_reserve_inode_write(handle, i_prev, &iloc2);
 		if (err) {
@@ -328,9 +328,9 @@ static void ext4_process_orphan(struct inode *inode,
 	if (inode->i_nlink) {
 		if (test_opt(sb, DEBUG))
 			ext4_msg(sb, KERN_DEBUG,
-				"%s: truncating inode %lu to %lld bytes",
+				"%s: truncating inode %llu to %lld bytes",
 				__func__, inode->i_ino, inode->i_size);
-		ext4_debug("truncating inode %lu to %lld bytes\n",
+		ext4_debug("truncating inode %llu to %lld bytes\n",
 			   inode->i_ino, inode->i_size);
 		inode_lock(inode);
 		truncate_inode_pages(inode->i_mapping, inode->i_size);
@@ -349,9 +349,9 @@ static void ext4_process_orphan(struct inode *inode,
 	} else {
 		if (test_opt(sb, DEBUG))
 			ext4_msg(sb, KERN_DEBUG,
-				"%s: deleting unreferenced inode %lu",
+				"%s: deleting unreferenced inode %llu",
 				__func__, inode->i_ino);
-		ext4_debug("deleting unreferenced inode %lu\n",
+		ext4_debug("deleting unreferenced inode %llu\n",
 			   inode->i_ino);
 		(*nr_orphans)++;
 	}
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index a8c95eee91b79..a90df42d153e7 100644
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -16,7 +16,6 @@
 #include <linux/string.h>
 #include <linux/buffer_head.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
 #include <linux/mpage.h>
 #include <linux/namei.h>
 #include <linux/uio.h>
@@ -180,7 +179,7 @@ static int ext4_end_io_end(ext4_io_end_t *io_end)
 	struct super_block *sb = inode->i_sb;
 	int ret = 0;
 
-	ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %lu,list->next 0x%p,"
+	ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %llu,list->next 0x%p,"
 		   "list->prev 0x%p\n",
 		   io_end, inode->i_ino, io_end->list.next, io_end->list.prev);
 
@@ -204,7 +203,7 @@ static int ext4_end_io_end(ext4_io_end_t *io_end)
 		ext4_msg(sb, KERN_EMERG,
 			 "failed to convert unwritten extents to written "
 			 "extents -- potential data loss!  "
-			 "(inode %lu, error %d)", inode->i_ino, ret);
+			 "(inode %llu, error %d)", inode->i_ino, ret);
 	}
 
 	ext4_clear_io_unwritten_flag(io_end);
@@ -221,7 +220,7 @@ static void dump_completed_IO(struct inode *inode, struct list_head *head)
 	if (list_empty(head))
 		return;
 
-	ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
+	ext4_debug("Dump inode %llu completed io list\n", inode->i_ino);
 	list_for_each_entry(io_end, head, list) {
 		cur = &io_end->list;
 		before = cur->prev;
@@ -229,7 +228,7 @@ static void dump_completed_IO(struct inode *inode, struct list_head *head)
 		after = cur->next;
 		io_end1 = container_of(after, ext4_io_end_t, list);
 
-		ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
+		ext4_debug("io 0x%p from inode %llu,prev 0x%p,next 0x%p\n",
 			    io_end, inode->i_ino, io_end0, io_end1);
 	}
 #endif
@@ -366,7 +365,7 @@ static void ext4_end_bio(struct bio *bio)
 	if (bio->bi_status) {
 		struct inode *inode = io_end->inode;
 
-		ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
+		ext4_warning(inode->i_sb, "I/O error %d writing to inode %llu "
 			     "starting block %llu)",
 			     bio->bi_status, inode->i_ino,
 			     (unsigned long long)
@@ -416,6 +415,8 @@ void ext4_io_submit_init(struct ext4_io_submit *io,
 }
 
 static void io_submit_init_bio(struct ext4_io_submit *io,
+			       struct inode *inode,
+			       struct folio *folio,
 			       struct buffer_head *bh)
 {
 	struct bio *bio;
@@ -425,30 +426,42 @@ static void io_submit_init_bio(struct ext4_io_submit *io,
 	 * __GFP_DIRECT_RECLAIM is set, see comments for bio_alloc_bioset().
 	 */
 	bio = bio_alloc(bh->b_bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOIO);
-	fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
+	fscrypt_set_bio_crypt_ctx(bio, inode, folio_pos(folio) + bh_offset(bh),
+				  GFP_NOIO);
 	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
 	bio->bi_end_io = ext4_end_bio;
 	bio->bi_private = ext4_get_io_end(io->io_end);
+	bio->bi_write_hint = inode->i_write_hint;
 	io->io_bio = bio;
 	io->io_next_block = bh->b_blocknr;
 	wbc_init_bio(io->io_wbc, bio);
 }
 
+static bool io_submit_need_new_bio(struct ext4_io_submit *io,
+				   struct inode *inode,
+				   struct folio *folio,
+				   struct buffer_head *bh)
+{
+	if (bh->b_blocknr != io->io_next_block)
+		return true;
+	if (!fscrypt_mergeable_bio(io->io_bio, inode,
+			folio_pos(folio) + bh_offset(bh)))
+		return true;
+	return false;
+}
+
 static void io_submit_add_bh(struct ext4_io_submit *io,
 			     struct inode *inode,
 			     struct folio *folio,
 			     struct folio *io_folio,
 			     struct buffer_head *bh)
 {
-	if (io->io_bio && (bh->b_blocknr != io->io_next_block ||
-			   !fscrypt_mergeable_bio_bh(io->io_bio, bh))) {
+	if (io->io_bio && io_submit_need_new_bio(io, inode, folio, bh)) {
 submit_and_retry:
 		ext4_io_submit(io);
 	}
-	if (io->io_bio == NULL) {
-		io_submit_init_bio(io, bh);
-		io->io_bio->bi_write_hint = inode->i_write_hint;
-	}
+	if (io->io_bio == NULL)
+		io_submit_init_bio(io, inode, folio, bh);
 	if (!bio_add_folio(io->io_bio, io_folio, bh->b_size, bh_offset(bh)))
 		goto submit_and_retry;
 	wbc_account_cgroup_owner(io->io_wbc, folio, bh->b_size);
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index 830f3b8a321f9..dd3627c717329 100644
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -43,7 +43,6 @@
 #include <linux/mpage.h>
 #include <linux/writeback.h>
 #include <linux/backing-dev.h>
-#include <linux/pagevec.h>
 
 #include "ext4.h"
 #include <trace/events/ext4.h>
@@ -215,11 +214,11 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
 	sector_t last_block_in_bio = 0;
 	const unsigned blkbits = inode->i_blkbits;
 	const unsigned blocksize = 1 << blkbits;
-	sector_t next_block;
 	sector_t block_in_file;
 	sector_t last_block;
 	sector_t last_block_in_file;
 	sector_t first_block;
+	loff_t pos;
 	unsigned page_block;
 	struct block_device *bdev = inode->i_sb->s_bdev;
 	int length;
@@ -249,7 +248,8 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
 
 		blocks_per_folio = folio_size(folio) >> blkbits;
 		first_hole = blocks_per_folio;
-		block_in_file = next_block = EXT4_PG_TO_LBLK(inode, folio->index);
+		pos = folio_pos(folio);
+		block_in_file = pos >> blkbits;
 		last_block = EXT4_PG_TO_LBLK(inode, folio->index + nr_pages);
 		last_block_in_file = (ext4_readpage_limit(inode) +
 				      blocksize - 1) >> blkbits;
@@ -342,7 +342,7 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
 		 * BIO off first?
 		 */
 		if (bio && (last_block_in_bio != first_block - 1 ||
-			    !fscrypt_mergeable_bio(bio, inode, next_block))) {
+			    !fscrypt_mergeable_bio(bio, inode, pos))) {
 		submit_and_realloc:
 			blk_crypto_submit_bio(bio);
 			bio = NULL;
@@ -354,8 +354,7 @@ static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi,
 			 */
 			bio = bio_alloc(bdev, bio_max_segs(nr_pages),
 					REQ_OP_READ, GFP_KERNEL);
-			fscrypt_set_bio_crypt_ctx(bio, inode, next_block,
-						  GFP_KERNEL);
+			fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL);
 			ext4_set_bio_post_read_ctx(bio, inode, vi);
 			bio->bi_iter.bi_sector = first_block << (blkbits - 9);
 			bio->bi_end_io = mpage_end_io;
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 43f680c750ae6..781c083000c2e 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -848,12 +848,12 @@ void __ext4_error_inode(struct inode *inode, const char *function,
 		vaf.va = &args;
 		if (block)
 			printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
-			       "inode #%lu: block %llu: comm %s: %pV\n",
+			       "inode #%llu: block %llu: comm %s: %pV\n",
 			       inode->i_sb->s_id, function, line, inode->i_ino,
 			       block, current->comm, &vaf);
 		else
 			printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: "
-			       "inode #%lu: comm %s: %pV\n",
+			       "inode #%llu: comm %s: %pV\n",
 			       inode->i_sb->s_id, function, line, inode->i_ino,
 			       current->comm, &vaf);
 		va_end(args);
@@ -888,13 +888,13 @@ void __ext4_error_file(struct file *file, const char *function,
 		vaf.va = &args;
 		if (block)
 			printk(KERN_CRIT
-			       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
+			       "EXT4-fs error (device %s): %s:%d: inode #%llu: "
 			       "block %llu: comm %s: path %s: %pV\n",
 			       inode->i_sb->s_id, function, line, inode->i_ino,
 			       block, current->comm, path, &vaf);
 		else
 			printk(KERN_CRIT
-			       "EXT4-fs error (device %s): %s:%d: inode #%lu: "
+			       "EXT4-fs error (device %s): %s:%d: inode #%llu: "
 			       "comm %s: path %s: %pV\n",
 			       inode->i_sb->s_id, function, line, inode->i_ino,
 			       current->comm, path, &vaf);
@@ -1035,14 +1035,14 @@ void __ext4_warning_inode(const struct inode *inode, const char *function,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 	printk(KERN_WARNING "EXT4-fs warning (device %s): %s:%d: "
-	       "inode #%lu: comm %s: %pV\n", inode->i_sb->s_id,
+	       "inode #%llu: comm %s: %pV\n", inode->i_sb->s_id,
 	       function, line, inode->i_ino, current->comm, &vaf);
 	va_end(args);
 }
 
 void __ext4_grp_locked_error(const char *function, unsigned int line,
 			     struct super_block *sb, ext4_group_t grp,
-			     unsigned long ino, ext4_fsblk_t block,
+			     u64 ino, ext4_fsblk_t block,
 			     const char *fmt, ...)
 __releases(bitlock)
 __acquires(bitlock)
@@ -1061,7 +1061,7 @@ __acquires(bitlock)
 		printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: group %u, ",
 		       sb->s_id, function, line, grp);
 		if (ino)
-			printk(KERN_CONT "inode %lu: ", ino);
+			printk(KERN_CONT "inode %llu: ", ino);
 		if (block)
 			printk(KERN_CONT "block %llu:",
 			       (unsigned long long) block);
@@ -1170,7 +1170,7 @@ static void dump_orphan_list(struct super_block *sb, struct ext4_sb_info *sbi)
 	list_for_each(l, &sbi->s_orphan) {
 		struct inode *inode = orphan_list_entry(l);
 		printk(KERN_ERR "  "
-		       "inode %s:%lu at %p: mode %o, nlink %d, next %d\n",
+		       "inode %s:%llu at %p: mode %o, nlink %d, next %d\n",
 		       inode->i_sb->s_id, inode->i_ino, inode,
 		       inode->i_mode, inode->i_nlink,
 		       NEXT_ORPHAN(inode));
@@ -1446,7 +1446,7 @@ static void ext4_free_in_core_inode(struct inode *inode)
 {
 	fscrypt_free_inode(inode);
 	if (!list_empty(&(EXT4_I(inode)->i_fc_list))) {
-		pr_warn("%s: inode %ld still in fc list",
+		pr_warn("%s: inode %llu still in fc list",
 			__func__, inode->i_ino);
 	}
 	kmem_cache_free(ext4_inode_cachep, EXT4_I(inode));
@@ -1456,7 +1456,7 @@ static void ext4_destroy_inode(struct inode *inode)
 {
 	if (ext4_inode_orphan_tracked(inode)) {
 		ext4_msg(inode->i_sb, KERN_ERR,
-			 "Inode %lu (%p): inode tracked as orphan!",
+			 "Inode %llu (%p): inode tracked as orphan!",
 			 inode->i_ino, EXT4_I(inode));
 		print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 4,
 				EXT4_I(inode), sizeof(struct ext4_inode_info),
@@ -1467,7 +1467,7 @@ static void ext4_destroy_inode(struct inode *inode)
 	if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ERROR_FS) &&
 	    WARN_ON_ONCE(EXT4_I(inode)->i_reserved_data_blocks))
 		ext4_msg(inode->i_sb, KERN_ERR,
-			 "Inode %lu (%p): i_reserved_data_blocks (%u) not cleared!",
+			 "Inode %llu (%p): i_reserved_data_blocks (%u) not cleared!",
 			 inode->i_ino, EXT4_I(inode),
 			 EXT4_I(inode)->i_reserved_data_blocks);
 }
diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c
index 7bf9ba19a89db..60aec4712f7fc 100644
--- a/fs/ext4/xattr.c
+++ b/fs/ext4/xattr.c
@@ -64,7 +64,7 @@
 
 #ifdef EXT4_XATTR_DEBUG
 # define ea_idebug(inode, fmt, ...)					\
-	printk(KERN_DEBUG "inode %s:%lu: " fmt "\n",			\
+	printk(KERN_DEBUG "inode %s:%llu: " fmt "\n",			\
 	       inode->i_sb->s_id, inode->i_ino, ##__VA_ARGS__)
 # define ea_bdebug(bh, fmt, ...)					\
 	printk(KERN_DEBUG "block %pg:%lu: " fmt "\n",			\
@@ -1035,7 +1035,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode,
 	ref_count = ext4_xattr_inode_get_ref(ea_inode);
 	if ((ref_count == 0 && ref_change < 0) || (ref_count == U64_MAX && ref_change > 0)) {
 		ext4_error_inode(ea_inode, __func__, __LINE__, 0,
-			"EA inode %lu ref wraparound: ref_count=%lld ref_change=%d",
+			"EA inode %llu ref wraparound: ref_count=%lld ref_change=%d",
 			ea_inode->i_ino, ref_count, ref_change);
 		brelse(iloc.bh);
 		ret = -EFSCORRUPTED;
@@ -1046,7 +1046,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode,
 
 	if (ref_change > 0) {
 		if (ref_count == 1) {
-			WARN_ONCE(ea_inode->i_nlink, "EA inode %lu i_nlink=%u",
+			WARN_ONCE(ea_inode->i_nlink, "EA inode %llu i_nlink=%u",
 				  ea_inode->i_ino, ea_inode->i_nlink);
 
 			set_nlink(ea_inode, 1);
@@ -1055,7 +1055,7 @@ static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode,
 	} else {
 		if (ref_count == 0) {
 			WARN_ONCE(ea_inode->i_nlink != 1,
-				  "EA inode %lu i_nlink=%u",
+				  "EA inode %llu i_nlink=%u",
 				  ea_inode->i_ino, ea_inode->i_nlink);
 
 			clear_nlink(ea_inode);
@@ -2854,7 +2854,7 @@ int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize,
 
 cleanup:
 	if (error && (mnt_count != le16_to_cpu(sbi->s_es->s_mnt_count))) {
-		ext4_warning(inode->i_sb, "Unable to expand inode %lu. Delete some EAs or run e2fsck.",
+		ext4_warning(inode->i_sb, "Unable to expand inode %llu. Delete some EAs or run e2fsck.",
 			     inode->i_ino);
 		mnt_count = le16_to_cpu(sbi->s_es->s_mnt_count);
 	}
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 6dd39b7de11a4..0143365c07dc1 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -11,7 +11,7 @@
 #include <linux/writeback.h>
 #include <linux/blkdev.h>
 #include <linux/f2fs_fs.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/swap.h>
 #include <linux/kthread.h>
 #include <linux/delayacct.h>
diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index 8c76400ba6317..50fac72734ac2 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -13,7 +13,7 @@
 #include <linux/lzo.h>
 #include <linux/lz4.h>
 #include <linux/zstd.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -773,7 +773,7 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)
 			if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) {
 				set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT);
 				f2fs_info_ratelimited(sbi,
-					"checksum invalid, nid = %lu, %x vs %x",
+					"checksum invalid, nid = %llu, %x vs %x",
 					dic->inode->i_ino,
 					provided, calculated);
 			}
@@ -932,7 +932,7 @@ bool f2fs_sanity_check_cluster(struct dnode_of_data *dn)
 
 	return false;
 out:
-	f2fs_warn(sbi, "access invalid cluster, ino:%lu, nid:%u, ofs_in_node:%u, reason:%s",
+	f2fs_warn(sbi, "access invalid cluster, ino:%llu, nid:%u, ofs_in_node:%u, reason:%s",
 			dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason);
 	set_sbi_flag(sbi, SBI_NEED_FSCK);
 	return true;
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 338df7a2aea6b..609fe0bbdefe5 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -10,7 +10,7 @@
 #include <linux/sched/mm.h>
 #include <linux/mpage.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/blkdev.h>
 #include <linux/bio.h>
 #include <linux/blk-crypto.h>
@@ -386,6 +386,8 @@ static void f2fs_write_end_io(struct bio *bio)
 				folio->index, NODE_TYPE_REGULAR, true);
 			f2fs_bug_on(sbi, folio->index != nid_of_node(folio));
 		}
+		if (f2fs_in_warm_node_list(folio))
+			f2fs_del_fsync_node_entry(sbi, folio);
 
 		dec_page_count(sbi, type);
 
@@ -397,8 +399,6 @@ static void f2fs_write_end_io(struct bio *bio)
 				wq_has_sleeper(&sbi->cp_wait))
 			wake_up(&sbi->cp_wait);
 
-		if (f2fs_in_warm_node_list(sbi, folio))
-			f2fs_del_fsync_node_entry(sbi, folio);
 		folio_clear_f2fs_gcing(folio);
 		folio_end_writeback(folio);
 	}
@@ -527,7 +527,9 @@ static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
 	 * read/write raw data without encryption.
 	 */
 	if (!fio || !fio->encrypted_page)
-		fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);
+		fscrypt_set_bio_crypt_ctx(bio, inode,
+				(loff_t)first_idx << inode->i_blkbits,
+				gfp_mask);
 }
 
 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
@@ -541,7 +543,8 @@ static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
 	if (fio && fio->encrypted_page)
 		return !bio_has_crypt_ctx(bio);
 
-	return fscrypt_mergeable_bio(bio, inode, next_idx);
+	return fscrypt_mergeable_bio(bio, inode,
+			(loff_t)next_idx << inode->i_blkbits);
 }
 
 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
@@ -2787,7 +2790,6 @@ int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
 	struct inode *inode = fio_inode(fio);
 	struct folio *mfolio;
 	struct page *page;
-	gfp_t gfp_flags = GFP_NOFS;
 
 	if (!f2fs_encrypted_file(inode))
 		return 0;
@@ -2797,19 +2799,10 @@ int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
 	if (fscrypt_inode_uses_inline_crypto(inode))
 		return 0;
 
-retry_encrypt:
 	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page_folio(page),
-					PAGE_SIZE, 0, gfp_flags);
-	if (IS_ERR(fio->encrypted_page)) {
-		/* flush pending IOs and wait for a while in the ENOMEM case */
-		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
-			f2fs_flush_merged_writes(fio->sbi);
-			memalloc_retry_wait(GFP_NOFS);
-			gfp_flags |= __GFP_NOFAIL;
-			goto retry_encrypt;
-		}
+					PAGE_SIZE, 0, GFP_NOFS);
+	if (IS_ERR(fio->encrypted_page))
 		return PTR_ERR(fio->encrypted_page);
-	}
 
 	mfolio = filemap_lock_folio(META_MAPPING(fio->sbi), fio->old_blkaddr);
 	if (!IS_ERR(mfolio)) {
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index 8e1040e375a7b..af88db8fdb711 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -659,6 +659,7 @@ static int stat_show(struct seq_file *s, void *v)
 				si->bg_node_blks);
 		seq_printf(s, "BG skip : IO: %u, Other: %u\n",
 				si->io_skip_bggc, si->other_skip_bggc);
+		seq_printf(s, "defrag blocks : %u\n", si->defrag_blks);
 		seq_puts(s, "\nExtent Cache (Read):\n");
 		seq_printf(s, "  - Hit Count: L1-1:%llu L1-2:%llu L2:%llu\n",
 				si->hit_largest, si->hit_cached[EX_READ],
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index f70092e231f0f..38802ee2e40de 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -368,7 +368,7 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
 
 	max_depth = F2FS_I(dir)->i_current_depth;
 	if (unlikely(max_depth > MAX_DIR_HASH_DEPTH)) {
-		f2fs_warn(F2FS_I_SB(dir), "Corrupted max_depth of %lu: %u",
+		f2fs_warn(F2FS_I_SB(dir), "Corrupted max_depth of %llu: %u",
 			  dir->i_ino, max_depth);
 		max_depth = MAX_DIR_HASH_DEPTH;
 		f2fs_i_depth_write(dir, max_depth);
diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c
index 0ed84cc065a7e..d73aeef333a24 100644
--- a/fs/f2fs/extent_cache.c
+++ b/fs/f2fs/extent_cache.c
@@ -34,7 +34,7 @@ bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio)
 	if (!f2fs_is_valid_blkaddr(sbi, ei.blk, DATA_GENERIC_ENHANCE) ||
 	    !f2fs_is_valid_blkaddr(sbi, ei.blk + ei.len - 1,
 					DATA_GENERIC_ENHANCE)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx) extent info [%u, %u, %u] is incorrect, run fsck to fix",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) extent info [%u, %u, %u] is incorrect, run fsck to fix",
 			  __func__, inode->i_ino,
 			  ei.blk, ei.fofs, ei.len);
 		return false;
@@ -50,14 +50,14 @@ bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio)
 
 		if (devi == 0) {
 			f2fs_warn(sbi,
-			    "%s: inode (ino=%lx) is an alias of meta device",
+			    "%s: inode (ino=%llx) is an alias of meta device",
 			    __func__, inode->i_ino);
 			return false;
 		}
 
 		if (bdev_is_zoned(FDEV(devi).bdev)) {
 			f2fs_warn(sbi,
-			    "%s: device alias inode (ino=%lx)'s extent info "
+			    "%s: device alias inode (ino=%llx)'s extent info "
 			    "[%u, %u, %u] maps to zoned block device",
 			    __func__, inode->i_ino, ei.blk, ei.fofs, ei.len);
 			return false;
@@ -65,7 +65,7 @@ bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio)
 		return true;
 	}
 
-	f2fs_warn(sbi, "%s: device alias inode (ino=%lx)'s extent info "
+	f2fs_warn(sbi, "%s: device alias inode (ino=%llx)'s extent info "
 			"[%u, %u, %u] is inconsistent w/ any devices",
 			__func__, inode->i_ino, ei.blk, ei.fofs, ei.len);
 	return false;
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index bb34e864d0efc..ea0d99352ffb2 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -28,8 +28,6 @@
 #include <linux/fscrypt.h>
 #include <linux/fsverity.h>
 
-struct pagevec;
-
 #ifdef CONFIG_F2FS_CHECK_FS
 #define f2fs_bug_on(sbi, condition)	BUG_ON(condition)
 #else
@@ -2706,7 +2704,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 
 	spin_lock(&sbi->stat_lock);
 	if (unlikely(sbi->total_valid_block_count < count)) {
-		f2fs_warn(sbi, "Inconsistent total_valid_block_count:%u, ino:%lu, count:%u",
+		f2fs_warn(sbi, "Inconsistent total_valid_block_count:%u, ino:%llu, count:%u",
 			  sbi->total_valid_block_count, inode->i_ino, count);
 		sbi->total_valid_block_count = 0;
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
@@ -2719,7 +2717,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 					sbi->current_reserved_blocks + count);
 	spin_unlock(&sbi->stat_lock);
 	if (unlikely(inode->i_blocks < sectors)) {
-		f2fs_warn(sbi, "Inconsistent i_blocks, ino:%lu, iblocks:%llu, sectors:%llu",
+		f2fs_warn(sbi, "Inconsistent i_blocks, ino:%llu, iblocks:%llu, sectors:%llu",
 			  inode->i_ino,
 			  (unsigned long long)inode->i_blocks,
 			  (unsigned long long)sectors);
@@ -2993,7 +2991,7 @@ static inline void dec_valid_node_count(struct f2fs_sb_info *sbi,
 		dquot_free_inode(inode);
 	} else {
 		if (unlikely(inode->i_blocks == 0)) {
-			f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%lu, iblocks:%llu",
+			f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%llu, iblocks:%llu",
 				  inode->i_ino,
 				  (unsigned long long)inode->i_blocks);
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
@@ -3921,7 +3919,7 @@ enum node_type;
 
 int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
 bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
-bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct folio *folio);
+bool f2fs_in_warm_node_list(struct folio *folio);
 void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
 void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio);
 void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
@@ -4288,6 +4286,7 @@ struct f2fs_stat_info {
 	int gc_secs[2][2];
 	int tot_blks, data_blks, node_blks;
 	int bg_data_blks, bg_node_blks;
+	unsigned int defrag_blks;
 	int blkoff[NR_CURSEG_TYPE];
 	int curseg[NR_CURSEG_TYPE];
 	int cursec[NR_CURSEG_TYPE];
@@ -4422,6 +4421,9 @@ static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
 		si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0;	\
 	} while (0)
 
+#define stat_inc_defrag_blk_count(sbi, blks)				\
+	(F2FS_STAT(sbi)->defrag_blks += (blks))
+
 int f2fs_build_stats(struct f2fs_sb_info *sbi);
 void f2fs_destroy_stats(struct f2fs_sb_info *sbi);
 void __init f2fs_create_root_stats(void);
@@ -4463,6 +4465,7 @@ void f2fs_update_sit_info(struct f2fs_sb_info *sbi);
 #define stat_inc_tot_blk_count(si, blks)		do { } while (0)
 #define stat_inc_data_blk_count(sbi, blks, gc_type)	do { } while (0)
 #define stat_inc_node_blk_count(sbi, blks, gc_type)	do { } while (0)
+#define stat_inc_defrag_blk_count(sbi, blks)		do { } while (0)
 
 static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; }
 static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { }
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index c8a2f17a8f11c..08a37a2f3508d 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -17,7 +17,6 @@
 #include <linux/compat.h>
 #include <linux/uaccess.h>
 #include <linux/mount.h>
-#include <linux/pagevec.h>
 #include <linux/uio.h>
 #include <linux/uuid.h>
 #include <linux/file.h>
@@ -1917,7 +1916,7 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset,
 				f2fs_up_write(&sbi->pin_sem);
 				err = -ENOSPC;
 				f2fs_warn_ratelimited(sbi,
-					"ino:%lu, start:%lu, end:%lu, need to trigger GC to "
+					"ino:%llu, start:%lu, end:%lu, need to trigger GC to "
 					"reclaim enough free segment when checkpoint is enabled",
 					inode->i_ino, pg_start, pg_end);
 				goto out_err;
@@ -2307,7 +2306,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate)
 	 * f2fs_is_atomic_file.
 	 */
 	if (get_dirty_pages(inode))
-		f2fs_warn(sbi, "Unexpected flush for atomic writes: ino=%lu, npages=%u",
+		f2fs_warn(sbi, "Unexpected flush for atomic writes: ino=%llu, npages=%u",
 			  inode->i_ino, get_dirty_pages(inode));
 	ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
 	if (ret)
@@ -3043,8 +3042,10 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi,
 	clear_inode_flag(inode, FI_OPU_WRITE);
 unlock_out:
 	inode_unlock(inode);
-	if (!err)
+	if (!err) {
 		range->len = (u64)total << PAGE_SHIFT;
+		stat_inc_defrag_blk_count(sbi, total);
+	}
 	return err;
 }
 
@@ -3494,7 +3495,7 @@ int f2fs_pin_file_control(struct inode *inode, bool inc)
 		return -EINVAL;
 
 	if (fi->i_gc_failures >= sbi->gc_pin_file_threshold) {
-		f2fs_warn(sbi, "%s: Enable GC = ino %lx after %x GC trials",
+		f2fs_warn(sbi, "%s: Enable GC = ino %llx after %x GC trials",
 			  __func__, inode->i_ino, fi->i_gc_failures);
 		clear_inode_flag(inode, FI_PIN_FILE);
 		return -EAGAIN;
@@ -3679,7 +3680,7 @@ static int f2fs_ioc_enable_verity(struct file *filp, unsigned long arg)
 
 	if (!f2fs_sb_has_verity(F2FS_I_SB(inode))) {
 		f2fs_warn(F2FS_I_SB(inode),
-			  "Can't enable fs-verity on inode %lu: the verity feature is not enabled on this filesystem",
+			  "Can't enable fs-verity on inode %llu: the verity feature is not enabled on this filesystem",
 			  inode->i_ino);
 		return -EOPNOTSUPP;
 	}
@@ -3950,7 +3951,7 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg)
 	} else if (released_blocks &&
 			atomic_read(&fi->i_compr_blocks)) {
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
-		f2fs_warn(sbi, "%s: partial blocks were released i_ino=%lx "
+		f2fs_warn(sbi, "%s: partial blocks were released i_ino=%llx "
 			"iblocks=%llu, released=%u, compr_blocks=%u, "
 			"run fsck to fix.",
 			__func__, inode->i_ino, inode->i_blocks,
@@ -4133,7 +4134,7 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg)
 	} else if (reserved_blocks &&
 			atomic_read(&fi->i_compr_blocks)) {
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
-		f2fs_warn(sbi, "%s: partial blocks were reserved i_ino=%lx "
+		f2fs_warn(sbi, "%s: partial blocks were reserved i_ino=%llx "
 			"iblocks=%llu, reserved=%u, compr_blocks=%u, "
 			"run fsck to fix.",
 			__func__, inode->i_ino, inode->i_blocks,
@@ -4162,7 +4163,9 @@ static int f2fs_secure_erase(struct block_device *bdev, struct inode *inode,
 
 	if (!ret && (flags & F2FS_TRIM_FILE_ZEROOUT)) {
 		if (IS_ENCRYPTED(inode))
-			ret = fscrypt_zeroout_range(inode, off, block, len);
+			ret = fscrypt_zeroout_range(inode,
+					(loff_t)off << inode->i_blkbits, sector,
+					(u64)len << inode->i_blkbits);
 		else
 			ret = blkdev_issue_zeroout(bdev, sector, nr_sects,
 					GFP_NOFS, 0);
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index f46b2673d31f5..f16ef0e284c75 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -316,10 +316,11 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type,
 		p->max_search = sbi->max_victim_search;
 
 	/* let's select beginning hot/small space first. */
-	if (f2fs_need_rand_seg(sbi))
+	if (f2fs_need_rand_seg(sbi)) {
 		p->offset = get_random_u32_below(MAIN_SECS(sbi) *
 						SEGS_PER_SEC(sbi));
-	else if (type == CURSEG_HOT_DATA || IS_NODESEG(type))
+		SIT_I(sbi)->last_victim[p->gc_mode] = p->offset;
+	} else if (type == CURSEG_HOT_DATA || IS_NODESEG(type))
 		p->offset = 0;
 	else
 		p->offset = SIT_I(sbi)->last_victim[p->gc_mode];
@@ -1230,7 +1231,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 		.encrypted_page = NULL,
 		.in_list = 0,
 	};
-	int err;
+	int err = 0;
 
 	folio = f2fs_grab_cache_folio(mapping, index, true);
 	if (IS_ERR(folio))
@@ -1283,6 +1284,9 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 
 	fio.encrypted_page = &efolio->page;
 
+	if (folio_test_uptodate(efolio))
+		goto put_encrypted_page;
+
 	err = f2fs_submit_page_bio(&fio);
 	if (err)
 		goto put_encrypted_page;
@@ -1622,7 +1626,7 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 				iput(inode);
 				set_sbi_flag(sbi, SBI_NEED_FSCK);
 				f2fs_err_ratelimited(sbi,
-					"inode %lx has both inline_data flag and "
+					"inode %llu has both inline_data flag and "
 					"data block, nid=%u, ofs_in_node=%u",
 					inode->i_ino, dni.nid, ofs_in_node);
 				continue;
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 0a1052d5ee624..39f340f52b920 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -176,7 +176,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio)
 	if (unlikely(dn->data_blkaddr != NEW_ADDR)) {
 		f2fs_put_dnode(dn);
 		set_sbi_flag(fio.sbi, SBI_NEED_FSCK);
-		f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
+		f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.",
 			  __func__, dn->inode->i_ino, dn->data_blkaddr);
 		f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR);
 		return -EFSCORRUPTED;
@@ -431,7 +431,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 	if (unlikely(dn.data_blkaddr != NEW_ADDR)) {
 		f2fs_put_dnode(&dn);
 		set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK);
-		f2fs_warn(F2FS_F_SB(folio), "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
+		f2fs_warn(F2FS_F_SB(folio), "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.",
 			  __func__, dir->i_ino, dn.data_blkaddr);
 		f2fs_handle_error(F2FS_F_SB(folio), ERROR_INVALID_BLKADDR);
 		err = -EFSCORRUPTED;
@@ -792,7 +792,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 int f2fs_inline_data_fiemap(struct inode *inode,
 		struct fiemap_extent_info *fieinfo, __u64 start, __u64 len)
 {
-	__u64 byteaddr, ilen;
+	__u64 byteaddr = 0, ilen;
 	__u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED |
 		FIEMAP_EXTENT_LAST;
 	struct node_info ni;
@@ -825,9 +825,14 @@ int f2fs_inline_data_fiemap(struct inode *inode,
 	if (err)
 		goto out;
 
-	byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits;
-	byteaddr += (char *)inline_data_addr(inode, ifolio) -
-					(char *)F2FS_INODE(ifolio);
+	if (__is_valid_data_blkaddr(ni.blk_addr)) {
+		byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits;
+		byteaddr += (char *)inline_data_addr(inode, ifolio) -
+						(char *)F2FS_INODE(ifolio);
+	} else {
+		f2fs_bug_on(F2FS_I_SB(inode), ni.blk_addr != NEW_ADDR);
+		flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN;
+	}
 	err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags);
 	trace_f2fs_fiemap(inode, start, byteaddr, ilen, flags, err);
 out:
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index e0f850b3f0c38..f27198d6695b9 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -203,14 +203,14 @@ static bool sanity_check_compress_inode(struct inode *inode,
 
 	if (ri->i_compress_algorithm >= COMPRESS_MAX) {
 		f2fs_warn(sbi,
-			"%s: inode (ino=%lx) has unsupported compress algorithm: %u, run fsck to fix",
+			"%s: inode (ino=%llx) has unsupported compress algorithm: %u, run fsck to fix",
 			__func__, inode->i_ino, ri->i_compress_algorithm);
 		return false;
 	}
 	if (le64_to_cpu(ri->i_compr_blocks) >
 			SECTOR_TO_BLOCK(inode->i_blocks)) {
 		f2fs_warn(sbi,
-			"%s: inode (ino=%lx) has inconsistent i_compr_blocks:%llu, i_blocks:%llu, run fsck to fix",
+			"%s: inode (ino=%llx) has inconsistent i_compr_blocks:%llu, i_blocks:%llu, run fsck to fix",
 			__func__, inode->i_ino, le64_to_cpu(ri->i_compr_blocks),
 			SECTOR_TO_BLOCK(inode->i_blocks));
 		return false;
@@ -218,7 +218,7 @@ static bool sanity_check_compress_inode(struct inode *inode,
 	if (ri->i_log_cluster_size < MIN_COMPRESS_LOG_SIZE ||
 		ri->i_log_cluster_size > MAX_COMPRESS_LOG_SIZE) {
 		f2fs_warn(sbi,
-			"%s: inode (ino=%lx) has unsupported log cluster size: %u, run fsck to fix",
+			"%s: inode (ino=%llx) has unsupported log cluster size: %u, run fsck to fix",
 			__func__, inode->i_ino, ri->i_log_cluster_size);
 		return false;
 	}
@@ -262,7 +262,7 @@ static bool sanity_check_compress_inode(struct inode *inode,
 
 	return true;
 err_level:
-	f2fs_warn(sbi, "%s: inode (ino=%lx) has unsupported compress level: %u, run fsck to fix",
+	f2fs_warn(sbi, "%s: inode (ino=%llx) has unsupported compress level: %u, run fsck to fix",
 		  __func__, inode->i_ino, clevel);
 	return false;
 }
@@ -276,40 +276,40 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
 
 	iblocks = le64_to_cpu(F2FS_INODE(node_folio)->i_blocks);
 	if (!iblocks) {
-		f2fs_warn(sbi, "%s: corrupted inode i_blocks i_ino=%lx iblocks=%llu, run fsck to fix.",
+		f2fs_warn(sbi, "%s: corrupted inode i_blocks i_ino=%llx iblocks=%llu, run fsck to fix.",
 			  __func__, inode->i_ino, iblocks);
 		return false;
 	}
 
 	if (ino_of_node(node_folio) != nid_of_node(node_folio)) {
-		f2fs_warn(sbi, "%s: corrupted inode footer i_ino=%lx, ino,nid: [%u, %u] run fsck to fix.",
+		f2fs_warn(sbi, "%s: corrupted inode footer i_ino=%llx, ino,nid: [%u, %u] run fsck to fix.",
 			  __func__, inode->i_ino,
 			  ino_of_node(node_folio), nid_of_node(node_folio));
 		return false;
 	}
 
 	if (ino_of_node(node_folio) == fi->i_xattr_nid) {
-		f2fs_warn(sbi, "%s: corrupted inode i_ino=%lx, xnid=%x, run fsck to fix.",
+		f2fs_warn(sbi, "%s: corrupted inode i_ino=%llx, xnid=%x, run fsck to fix.",
 			  __func__, inode->i_ino, fi->i_xattr_nid);
 		return false;
 	}
 
 	if (S_ISDIR(inode->i_mode) && unlikely(inode->i_nlink == 1)) {
-		f2fs_warn(sbi, "%s: directory inode (ino=%lx) has a single i_nlink",
+		f2fs_warn(sbi, "%s: directory inode (ino=%llx) has a single i_nlink",
 			  __func__, inode->i_ino);
 		return false;
 	}
 
 	if (f2fs_has_extra_attr(inode)) {
 		if (!f2fs_sb_has_extra_attr(sbi)) {
-			f2fs_warn(sbi, "%s: inode (ino=%lx) is with extra_attr, but extra_attr feature is off",
+			f2fs_warn(sbi, "%s: inode (ino=%llx) is with extra_attr, but extra_attr feature is off",
 				  __func__, inode->i_ino);
 			return false;
 		}
 		if (fi->i_extra_isize > F2FS_TOTAL_EXTRA_ATTR_SIZE ||
 			fi->i_extra_isize < F2FS_MIN_EXTRA_ATTR_SIZE ||
 			fi->i_extra_isize % sizeof(__le32)) {
-			f2fs_warn(sbi, "%s: inode (ino=%lx) has corrupted i_extra_isize: %d, max: %zu",
+			f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_extra_isize: %d, max: %zu",
 				  __func__, inode->i_ino, fi->i_extra_isize,
 				  F2FS_TOTAL_EXTRA_ATTR_SIZE);
 			return false;
@@ -327,7 +327,7 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
 		f2fs_has_inline_xattr(inode) &&
 		(fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE ||
 		fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu",
 			  __func__, inode->i_ino, fi->i_inline_xattr_size,
 			  MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE);
 		return false;
@@ -335,64 +335,64 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
 
 	if (!f2fs_sb_has_extra_attr(sbi)) {
 		if (f2fs_sb_has_project_quota(sbi)) {
-			f2fs_warn(sbi, "%s: corrupted inode ino=%lx, wrong feature flag: %u, run fsck to fix.",
+			f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.",
 				  __func__, inode->i_ino, F2FS_FEATURE_PRJQUOTA);
 			return false;
 		}
 		if (f2fs_sb_has_inode_chksum(sbi)) {
-			f2fs_warn(sbi, "%s: corrupted inode ino=%lx, wrong feature flag: %u, run fsck to fix.",
+			f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.",
 				  __func__, inode->i_ino, F2FS_FEATURE_INODE_CHKSUM);
 			return false;
 		}
 		if (f2fs_sb_has_flexible_inline_xattr(sbi)) {
-			f2fs_warn(sbi, "%s: corrupted inode ino=%lx, wrong feature flag: %u, run fsck to fix.",
+			f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.",
 				  __func__, inode->i_ino, F2FS_FEATURE_FLEXIBLE_INLINE_XATTR);
 			return false;
 		}
 		if (f2fs_sb_has_inode_crtime(sbi)) {
-			f2fs_warn(sbi, "%s: corrupted inode ino=%lx, wrong feature flag: %u, run fsck to fix.",
+			f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.",
 				  __func__, inode->i_ino, F2FS_FEATURE_INODE_CRTIME);
 			return false;
 		}
 		if (f2fs_sb_has_compression(sbi)) {
-			f2fs_warn(sbi, "%s: corrupted inode ino=%lx, wrong feature flag: %u, run fsck to fix.",
+			f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.",
 				  __func__, inode->i_ino, F2FS_FEATURE_COMPRESSION);
 			return false;
 		}
 	}
 
 	if (f2fs_sanity_check_inline_data(inode, node_folio)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx, mode=%u) should not have inline_data, run fsck to fix",
+		f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_data, run fsck to fix",
 			  __func__, inode->i_ino, inode->i_mode);
 		return false;
 	}
 
 	if (f2fs_has_inline_dentry(inode) && !S_ISDIR(inode->i_mode)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx, mode=%u) should not have inline_dentry, run fsck to fix",
+		f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_dentry, run fsck to fix",
 			  __func__, inode->i_ino, inode->i_mode);
 		return false;
 	}
 
 	if ((fi->i_flags & F2FS_CASEFOLD_FL) && !f2fs_sb_has_casefold(sbi)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx) has casefold flag, but casefold feature is off",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) has casefold flag, but casefold feature is off",
 			  __func__, inode->i_ino);
 		return false;
 	}
 
 	if (fi->i_xattr_nid && f2fs_check_nid_range(sbi, fi->i_xattr_nid)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx) has corrupted i_xattr_nid: %u, run fsck to fix.",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_xattr_nid: %u, run fsck to fix.",
 			  __func__, inode->i_ino, fi->i_xattr_nid);
 		return false;
 	}
 
 	if (IS_DEVICE_ALIASING(inode)) {
 		if (!f2fs_sb_has_device_alias(sbi)) {
-			f2fs_warn(sbi, "%s: inode (ino=%lx) has device alias flag, but the feature is off",
+			f2fs_warn(sbi, "%s: inode (ino=%llx) has device alias flag, but the feature is off",
 				  __func__, inode->i_ino);
 			return false;
 		}
 		if (!f2fs_is_pinned_file(inode)) {
-			f2fs_warn(sbi, "%s: inode (ino=%lx) has device alias flag, but is not pinned",
+			f2fs_warn(sbi, "%s: inode (ino=%llx) has device alias flag, but is not pinned",
 				  __func__, inode->i_ino);
 			return false;
 		}
@@ -925,7 +925,7 @@ void f2fs_evict_inode(struct inode *inode)
 			 */
 			if (is_inode_flag_set(inode, FI_DIRTY_INODE)) {
 				f2fs_warn(F2FS_I_SB(inode),
-					"f2fs_evict_inode: inconsistent node id, ino:%lu",
+					"f2fs_evict_inode: inconsistent node id, ino:%llu",
 					inode->i_ino);
 				f2fs_inode_synced(inode);
 				set_sbi_flag(sbi, SBI_NEED_FSCK);
@@ -954,7 +954,7 @@ void f2fs_evict_inode(struct inode *inode)
 		 */
 		if (is_inode_flag_set(inode, FI_DIRTY_INODE)) {
 			f2fs_warn(sbi,
-				"f2fs_evict_inode: inode is dirty, ino:%lu",
+				"f2fs_evict_inode: inode is dirty, ino:%llu",
 				inode->i_ino);
 			f2fs_inode_synced(inode);
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index e360f08a9586a..da8ded8a66be7 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -505,7 +505,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 	}
 
 	if (inode->i_nlink == 0) {
-		f2fs_warn(F2FS_I_SB(inode), "%s: inode (ino=%lx) has zero i_nlink",
+		f2fs_warn(F2FS_I_SB(inode), "%s: inode (ino=%llx) has zero i_nlink",
 			  __func__, inode->i_ino);
 		err = -EFSCORRUPTED;
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
@@ -515,7 +515,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 	if (IS_ENCRYPTED(dir) &&
 	    (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
 	    !fscrypt_has_permitted_context(dir, inode)) {
-		f2fs_warn(F2FS_I_SB(inode), "Inconsistent encryption contexts: %lu/%lu",
+		f2fs_warn(F2FS_I_SB(inode), "Inconsistent encryption contexts: %llu/%llu",
 			  dir->i_ino, inode->i_ino);
 		err = -EPERM;
 		goto out_iput;
@@ -573,11 +573,11 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
 	}
 
 	if (unlikely(inode->i_nlink == 0)) {
-		f2fs_warn(sbi, "%s: inode (ino=%lx) has zero i_nlink",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) has zero i_nlink",
 			  __func__, inode->i_ino);
 		goto corrupted;
 	} else if (S_ISDIR(inode->i_mode) && unlikely(inode->i_nlink == 1)) {
-		f2fs_warn(sbi, "%s: directory inode (ino=%lx) has a single i_nlink",
+		f2fs_warn(sbi, "%s: directory inode (ino=%llx) has a single i_nlink",
 			  __func__, inode->i_ino);
 		goto corrupted;
 	}
@@ -964,6 +964,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 			return err;
 
 		err = f2fs_create_whiteout(idmap, old_dir, &whiteout, &fname);
+		f2fs_free_filename(&fname);
 		if (err)
 			return err;
 	}
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 74992fd9c9b66..d1e3b7fc399c4 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -10,7 +10,7 @@
 #include <linux/mpage.h>
 #include <linux/sched/mm.h>
 #include <linux/blkdev.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/swap.h>
 
 #include "f2fs.h"
@@ -78,7 +78,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 		if (excess_cached_nats(sbi))
 			res = false;
 	} else if (type == DIRTY_DENTS) {
-		if (sbi->sb->s_bdi->wb.dirty_exceeded)
+		if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi))
 			return false;
 		mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
@@ -119,7 +119,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 		res = false;
 #endif
 	} else {
-		if (!sbi->sb->s_bdi->wb.dirty_exceeded)
+		if (!bdi_wb_dirty_exceeded(sbi->sb->s_bdi))
 			return true;
 	}
 	return res;
@@ -325,7 +325,7 @@ static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,
 							start, nr);
 }
 
-bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct folio *folio)
+bool f2fs_in_warm_node_list(struct folio *folio)
 {
 	return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio);
 }
@@ -847,7 +847,7 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 			err = -EFSCORRUPTED;
 			f2fs_err_ratelimited(sbi,
 				"inode mapping table is corrupted, run fsck to fix it, "
-				"ino:%lu, nid:%u, level:%d, offset:%d",
+				"ino:%llu, nid:%u, level:%d, offset:%d",
 				dn->inode->i_ino, nids[i], level, offset[level]);
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
 			goto release_pages;
@@ -1013,7 +1013,7 @@ static int truncate_dnode(struct dnode_of_data *dn)
 		return PTR_ERR(folio);
 
 	if (IS_INODE(folio) || ino_of_node(folio) != dn->inode->i_ino) {
-		f2fs_err(sbi, "incorrect node reference, ino: %lu, nid: %u, ino_of_node: %u",
+		f2fs_err(sbi, "incorrect node reference, ino: %llu, nid: %u, ino_of_node: %u",
 				dn->inode->i_ino, dn->nid, ino_of_node(folio));
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
 		f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE);
@@ -1194,7 +1194,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 	if (level <= 0) {
 		if (!level) {
 			level = -EFSCORRUPTED;
-			f2fs_err(sbi, "%s: inode ino=%lx has corrupted node block, from:%lu addrs:%u",
+			f2fs_err(sbi, "%s: inode ino=%llx has corrupted node block, from:%lu addrs:%u",
 					__func__, inode->i_ino,
 					from, ADDRS_PER_INODE(inode));
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
@@ -1265,7 +1265,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 			set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK);
 			f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
 			f2fs_err_ratelimited(sbi,
-				"truncate node fail, ino:%lu, nid:%u, "
+				"truncate node fail, ino:%llu, nid:%u, "
 				"offset[0]:%d, offset[1]:%d, nofs:%d",
 				inode->i_ino, dn.nid, offset[0],
 				offset[1], nofs);
@@ -1351,7 +1351,7 @@ int f2fs_remove_inode_page(struct inode *inode)
 
 	if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) {
 		f2fs_warn(F2FS_I_SB(inode),
-			"f2fs_remove_inode_page: inconsistent i_blocks, ino:%lu, iblocks:%llu",
+			"f2fs_remove_inode_page: inconsistent i_blocks, ino:%llu, iblocks:%llu",
 			inode->i_ino, (unsigned long long)inode->i_blocks);
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 	}
@@ -1810,7 +1810,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted
 	}
 
 	/* should add to global list before clearing PAGECACHE status */
-	if (f2fs_in_warm_node_list(sbi, folio)) {
+	if (f2fs_in_warm_node_list(folio)) {
 		seq = f2fs_add_fsync_node_entry(sbi, folio);
 		if (seq_id)
 			*seq_id = seq;
@@ -1997,7 +1997,7 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
 	return ret;
 }
 
-static int f2fs_match_ino(struct inode *inode, unsigned long ino, void *data)
+static int f2fs_match_ino(struct inode *inode, u64 ino, void *data)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	bool clean;
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index a26071f2b0bc8..3d3dacec94825 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -232,7 +232,7 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio,
 		name = "<encrypted>";
 	else
 		name = raw_inode->i_name;
-	f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %lx, err = %d",
+	f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %llu, err = %d",
 		    __func__, ino_of_node(ifolio), name,
 		    IS_ERR(dir) ? 0 : dir->i_ino, err);
 	return err;
@@ -532,7 +532,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 
 	max_addrs = ADDRS_PER_PAGE(dn->node_folio, dn->inode);
 	if (ofs_in_node >= max_addrs) {
-		f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%lu, nid:%u, max:%u",
+		f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%llu, nid:%u, max:%u",
 			ofs_in_node, dn->inode->i_ino, nid, max_addrs);
 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUMMARY);
 		return -EFSCORRUPTED;
@@ -674,7 +674,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 	f2fs_bug_on(sbi, ni.ino != ino_of_node(folio));
 
 	if (ofs_of_node(dn.node_folio) != ofs_of_node(folio)) {
-		f2fs_warn(sbi, "Inconsistent ofs_of_node, ino:%lu, ofs:%u, %u",
+		f2fs_warn(sbi, "Inconsistent ofs_of_node, ino:%llu, ofs:%u, %u",
 			  inode->i_ino, ofs_of_node(dn.node_folio),
 			  ofs_of_node(folio));
 		err = -EFSCORRUPTED;
@@ -748,7 +748,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 
 			if (f2fs_is_valid_blkaddr(sbi, dest,
 					DATA_GENERIC_ENHANCE_UPDATE)) {
-				f2fs_err(sbi, "Inconsistent dest blkaddr:%u, ino:%lu, ofs:%u",
+				f2fs_err(sbi, "Inconsistent dest blkaddr:%u, ino:%llu, ofs:%u",
 					dest, inode->i_ino, dn.ofs_in_node);
 				err = -EFSCORRUPTED;
 				goto err;
@@ -768,7 +768,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 err:
 	f2fs_put_dnode(&dn);
 out:
-	f2fs_notice(sbi, "recover_data: ino = %lx, nid = %x (i_size: %s), "
+	f2fs_notice(sbi, "recover_data: ino = %llx, nid = %x (i_size: %s), "
 		    "range (%u, %u), recovered = %d, err = %d",
 		    inode->i_ino, nid_of_node(folio),
 		    file_keep_isize(inode) ? "keep" : "recover",
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 6a97fe76712b5..23faf67256329 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -3980,7 +3980,7 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
 		if (fscrypt_inode_uses_fs_layer_crypto(folio->mapping->host))
 			fscrypt_finalize_bounce_page(&fio->encrypted_page);
 		folio_end_writeback(folio);
-		if (f2fs_in_warm_node_list(fio->sbi, folio))
+		if (f2fs_in_warm_node_list(folio))
 			f2fs_del_fsync_node_entry(fio->sbi, folio);
 		f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi,
 							CP_ERROR_FLAG));
diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index 068845660b0fa..08735a165433c 100644
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -992,7 +992,7 @@ static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
  */
 static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
 {
-	if (sbi->sb->s_bdi->wb.dirty_exceeded)
+	if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi))
 		return 0;
 
 	if (type == DATA)
diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c
index 5fbfdc96e502d..969e06b65b046 100644
--- a/fs/f2fs/sysfs.c
+++ b/fs/f2fs/sysfs.c
@@ -338,6 +338,14 @@ static ssize_t avg_vblocks_show(struct f2fs_attr *a,
 	f2fs_update_sit_info(sbi);
 	return sysfs_emit(buf, "%llu\n", (unsigned long long)(si->avg_vblocks));
 }
+
+static ssize_t defrag_blocks_show(struct f2fs_attr *a,
+				struct f2fs_sb_info *sbi, char *buf)
+{
+	struct f2fs_stat_info *si = F2FS_STAT(sbi);
+
+	return sysfs_emit(buf, "%llu\n", (unsigned long long)(si->defrag_blks));
+}
 #endif
 
 static ssize_t main_blkaddr_show(struct f2fs_attr *a,
@@ -1351,6 +1359,7 @@ F2FS_GENERAL_RO_ATTR(gc_mode);
 F2FS_GENERAL_RO_ATTR(moved_blocks_background);
 F2FS_GENERAL_RO_ATTR(moved_blocks_foreground);
 F2FS_GENERAL_RO_ATTR(avg_vblocks);
+F2FS_GENERAL_RO_ATTR(defrag_blocks);
 #endif
 
 #ifdef CONFIG_FS_ENCRYPTION
@@ -1473,6 +1482,7 @@ static struct attribute *f2fs_attrs[] = {
 	ATTR_LIST(moved_blocks_foreground),
 	ATTR_LIST(moved_blocks_background),
 	ATTR_LIST(avg_vblocks),
+	ATTR_LIST(defrag_blocks),
 #endif
 #ifdef CONFIG_BLK_DEV_ZONED
 	ATTR_LIST(unusable_blocks_per_sec),
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index 941dc62a6d6f7..610d5810074dc 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -365,7 +365,7 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio,
 
 	*xe = __find_xattr(cur_addr, last_txattr_addr, NULL, index, len, name);
 	if (!*xe) {
-		f2fs_err(F2FS_I_SB(inode), "lookup inode (%lu) has corrupted xattr",
+		f2fs_err(F2FS_I_SB(inode), "lookup inode (%llu) has corrupted xattr",
 								inode->i_ino);
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 		err = -ENODATA;
@@ -585,7 +585,7 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size)
 
 		if ((void *)(entry) + sizeof(__u32) > last_base_addr ||
 			(void *)XATTR_NEXT_ENTRY(entry) > last_base_addr) {
-			f2fs_err(F2FS_I_SB(inode), "list inode (%lu) has corrupted xattr",
+			f2fs_err(F2FS_I_SB(inode), "list inode (%llu) has corrupted xattr",
 						inode->i_ino);
 			set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 			f2fs_handle_error(F2FS_I_SB(inode),
@@ -664,14 +664,14 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 		if (!F2FS_I(inode)->i_xattr_nid) {
 			error = f2fs_recover_xattr_data(inode, NULL);
 			f2fs_notice(F2FS_I_SB(inode),
-				"recover xattr in inode (%lu), error(%d)",
+				"recover xattr in inode (%llu), error(%d)",
 					inode->i_ino, error);
 			if (!error) {
 				kfree(base_addr);
 				goto retry;
 			}
 		}
-		f2fs_err(F2FS_I_SB(inode), "set inode (%lu) has corrupted xattr",
+		f2fs_err(F2FS_I_SB(inode), "set inode (%llu) has corrupted xattr",
 								inode->i_ino);
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 		error = -EFSCORRUPTED;
@@ -699,7 +699,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 	while (!IS_XATTR_LAST_ENTRY(last)) {
 		if ((void *)(last) + sizeof(__u32) > last_base_addr ||
 			(void *)XATTR_NEXT_ENTRY(last) > last_base_addr) {
-			f2fs_err(F2FS_I_SB(inode), "inode (%lu) has invalid last xattr entry, entry_size: %zu",
+			f2fs_err(F2FS_I_SB(inode), "inode (%llu) has invalid last xattr entry, entry_size: %zu",
 					inode->i_ino, ENTRY_SIZE(last));
 			set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 			error = -EFSCORRUPTED;
diff --git a/fs/file_table.c b/fs/file_table.c
index aaa5faaace1e9..e8b4eb2bbff85 100644
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -27,6 +27,7 @@
 #include <linux/task_work.h>
 #include <linux/swap.h>
 #include <linux/kmemleak.h>
+#include <linux/backing-file.h>
 
 #include <linux/atomic.h>
 
@@ -43,11 +44,11 @@ static struct kmem_cache *bfilp_cachep __ro_after_init;
 
 static struct percpu_counter nr_files __cacheline_aligned_in_smp;
 
-/* Container for backing file with optional user path */
+/* Container for backing file with optional user path file */
 struct backing_file {
 	struct file file;
 	union {
-		struct path user_path;
+		struct file user_path_file;
 		freeptr_t bf_freeptr;
 	};
 };
@@ -56,24 +57,44 @@ struct backing_file {
 
 const struct path *backing_file_user_path(const struct file *f)
 {
-	return &backing_file(f)->user_path;
+	return &backing_file(f)->user_path_file.f_path;
 }
 EXPORT_SYMBOL_GPL(backing_file_user_path);
 
-void backing_file_set_user_path(struct file *f, const struct path *path)
+const struct file *backing_file_user_path_file(const struct file *f)
 {
-	backing_file(f)->user_path = *path;
+	return &backing_file(f)->user_path_file;
 }
-EXPORT_SYMBOL_GPL(backing_file_set_user_path);
+EXPORT_SYMBOL_GPL(backing_file_user_path_file);
 
-static inline void file_free(struct file *f)
+int backing_file_open_user_path(struct file *f, const struct path *path)
+{
+	/* open an O_PATH file to reference the user path - should not fail */
+	return WARN_ON(vfs_open(path, &backing_file(f)->user_path_file));
+}
+EXPORT_SYMBOL_GPL(backing_file_open_user_path);
+
+static void destroy_file(struct file *f)
 {
 	security_file_free(f);
+	put_cred(f->f_cred);
+}
+
+static inline void file_free(struct file *f)
+{
+	destroy_file(f);
 	if (likely(!(f->f_mode & FMODE_NOACCOUNT)))
 		percpu_counter_dec(&nr_files);
-	put_cred(f->f_cred);
 	if (unlikely(f->f_mode & FMODE_BACKING)) {
-		path_put(backing_file_user_path(f));
+		struct file *user_path_file = &backing_file(f)->user_path_file;
+
+		/*
+		 * no refcount on the user_path_file - they die together,
+		 * so __fput() is not called for user_path_file. path_put()
+		 * is the only relevant cleanup from __fput().
+		 */
+		destroy_file(user_path_file);
+		path_put(&user_path_file->__f_path);
 		kmem_cache_free(bfilp_cachep, backing_file(f));
 	} else {
 		kmem_cache_free(filp_cachep, f);
@@ -201,7 +222,7 @@ static int init_file(struct file *f, int flags, const struct cred *cred)
 	 * fget-rcu pattern users need to be able to handle spurious
 	 * refcount bumps we should reinitialize the reused file first.
 	 */
-	file_ref_init(&f->f_ref, 1);
+	file_ref_init(&f->f_ref, FILE_REF_ONEREF);
 	return 0;
 }
 
@@ -290,7 +311,8 @@ struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred)
  * This is only for kernel internal use, and the allocate file must not be
  * installed into file tables or such.
  */
-struct file *alloc_empty_backing_file(int flags, const struct cred *cred)
+struct file *alloc_empty_backing_file(int flags, const struct cred *cred,
+				      const struct cred *user_cred)
 {
 	struct backing_file *ff;
 	int error;
@@ -305,6 +327,15 @@ struct file *alloc_empty_backing_file(int flags, const struct cred *cred)
 		return ERR_PTR(error);
 	}
 
+	error = init_file(&ff->user_path_file, O_PATH, user_cred);
+	/* user_path_file is not refcounterd - it dies with the backing file */
+	file_ref_init(&ff->user_path_file.f_ref, FILE_REF_DEAD);
+	if (unlikely(error)) {
+		destroy_file(&ff->file);
+		kmem_cache_free(bfilp_cachep, ff);
+		return ERR_PTR(error);
+	}
+
 	ff->file.f_mode |= FMODE_BACKING | FMODE_NOACCOUNT;
 	return &ff->file;
 }
diff --git a/fs/freevxfs/vxfs_bmap.c b/fs/freevxfs/vxfs_bmap.c
index 26d367e3668da..e85222892038f 100644
--- a/fs/freevxfs/vxfs_bmap.c
+++ b/fs/freevxfs/vxfs_bmap.c
@@ -260,12 +260,12 @@ vxfs_bmap1(struct inode *ip, long iblock)
 	if (VXFS_ISIMMED(vip))
 		goto unsupp;
 
-	printk(KERN_WARNING "vxfs: inode %ld has no valid orgtype (%x)\n",
+	printk(KERN_WARNING "vxfs: inode %llu has no valid orgtype (%x)\n",
 			ip->i_ino, vip->vii_orgtype);
 	BUG();
 
 unsupp:
-	printk(KERN_WARNING "vxfs: inode %ld has an unsupported orgtype (%x)\n",
+	printk(KERN_WARNING "vxfs: inode %llu has an unsupported orgtype (%x)\n",
 			ip->i_ino, vip->vii_orgtype);
 	return 0;
 }
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 7c75ed7e89799..c3442a38450ca 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -280,15 +280,13 @@ void __inode_attach_wb(struct inode *inode, struct folio *folio)
 	if (inode_cgwb_enabled(inode)) {
 		struct cgroup_subsys_state *memcg_css;
 
-		if (folio) {
-			memcg_css = mem_cgroup_css_from_folio(folio);
-			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
-		} else {
-			/* must pin memcg_css, see wb_get_create() */
+		/* must pin memcg_css, see wb_get_create() */
+		if (folio)
+			memcg_css = get_mem_cgroup_css_from_folio(folio);
+		else
 			memcg_css = task_get_css(current, memory_cgrp_id);
-			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
-			css_put(memcg_css);
-		}
+		wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
+		css_put(memcg_css);
 	}
 
 	if (!wb)
@@ -979,16 +977,16 @@ void wbc_account_cgroup_owner(struct writeback_control *wbc, struct folio *folio
 	if (!wbc->wb || wbc->no_cgroup_owner)
 		return;
 
-	css = mem_cgroup_css_from_folio(folio);
+	css = get_mem_cgroup_css_from_folio(folio);
 	/* dead cgroups shouldn't contribute to inode ownership arbitration */
 	if (!css_is_online(css))
-		return;
+		goto out;
 
 	id = css->id;
 
 	if (id == wbc->wb_id) {
 		wbc->wb_bytes += bytes;
-		return;
+		goto out;
 	}
 
 	if (id == wbc->wb_lcand_id)
@@ -1001,6 +999,8 @@ void wbc_account_cgroup_owner(struct writeback_control *wbc, struct folio *folio
 		wbc->wb_tcand_bytes += bytes;
 	else
 		wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);
+out:
+	css_put(css);
 }
 EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner);
 
diff --git a/fs/fs_context.c b/fs/fs_context.c
index a37b0a0935052..23ad66cd94e1f 100644
--- a/fs/fs_context.c
+++ b/fs/fs_context.c
@@ -318,7 +318,6 @@ struct fs_context *fs_context_for_reconfigure(struct dentry *dentry,
 	return alloc_fs_context(dentry->d_sb->s_type, dentry, sb_flags,
 				sb_flags_mask, FS_CONTEXT_FOR_RECONFIGURE);
 }
-EXPORT_SYMBOL(fs_context_for_reconfigure);
 
 /**
  * fs_context_for_submount: allocate a new fs_context for a submount
diff --git a/fs/fs_parser.c b/fs/fs_parser.c
index c092a9f79e324..b4cc4cce518a2 100644
--- a/fs/fs_parser.c
+++ b/fs/fs_parser.c
@@ -13,7 +13,7 @@
 #include <linux/namei.h>
 #include "internal.h"
 
-const struct constant_table bool_names[] = {
+static const struct constant_table bool_names[] = {
 	{ "0",		false },
 	{ "1",		true },
 	{ "false",	false },
@@ -22,7 +22,6 @@ const struct constant_table bool_names[] = {
 	{ "yes",	true },
 	{ },
 };
-EXPORT_SYMBOL(bool_names);
 
 static const struct constant_table *
 __lookup_constant(const struct constant_table *tbl, const char *name)
@@ -278,15 +277,6 @@ int fs_param_is_string(struct p_log *log, const struct fs_parameter_spec *p,
 }
 EXPORT_SYMBOL(fs_param_is_string);
 
-int fs_param_is_blob(struct p_log *log, const struct fs_parameter_spec *p,
-		     struct fs_parameter *param, struct fs_parse_result *result)
-{
-	if (param->type != fs_value_is_blob)
-		return fs_param_bad_value(log, param);
-	return 0;
-}
-EXPORT_SYMBOL(fs_param_is_blob);
-
 int fs_param_is_fd(struct p_log *log, const struct fs_parameter_spec *p,
 		  struct fs_parameter *param, struct fs_parse_result *result)
 {
@@ -371,13 +361,6 @@ int fs_param_is_blockdev(struct p_log *log, const struct fs_parameter_spec *p,
 }
 EXPORT_SYMBOL(fs_param_is_blockdev);
 
-int fs_param_is_path(struct p_log *log, const struct fs_parameter_spec *p,
-		     struct fs_parameter *param, struct fs_parse_result *result)
-{
-	return 0;
-}
-EXPORT_SYMBOL(fs_param_is_path);
-
 #ifdef CONFIG_VALIDATE_FS_PARSER
 /**
  * fs_validate_description - Validate a parameter specification array
diff --git a/fs/fs_struct.c b/fs/fs_struct.c
index 394875d06fd60..e1487ca6256d7 100644
--- a/fs/fs_struct.c
+++ b/fs/fs_struct.c
@@ -33,20 +33,24 @@ void set_fs_root(struct fs_struct *fs, const struct path *path)
 void set_fs_pwd(struct fs_struct *fs, const struct path *path)
 {
 	struct path old_pwd;
+	int count;
 
 	path_get(path);
 	write_seqlock(&fs->seq);
 	old_pwd = fs->pwd;
 	fs->pwd = *path;
+	count = fs->pwd_refs + 1;
+	fs->pwd_refs = 0;
 	write_sequnlock(&fs->seq);
 
 	if (old_pwd.dentry)
-		path_put(&old_pwd);
+		while (count--)
+			path_put(&old_pwd);
 }
 
 static inline int replace_path(struct path *p, const struct path *old, const struct path *new)
 {
-	if (likely(p->dentry != old->dentry || p->mnt != old->mnt))
+	if (likely(!path_equal(p, old)))
 		return 0;
 	*p = *new;
 	return 1;
@@ -63,10 +67,15 @@ void chroot_fs_refs(const struct path *old_root, const struct path *new_root)
 		task_lock(p);
 		fs = p->fs;
 		if (fs) {
-			int hits = 0;
+			int hits;
+
 			write_seqlock(&fs->seq);
+			hits = replace_path(&fs->pwd, old_root, new_root);
+			if (hits && fs->pwd_refs) {
+				count += fs->pwd_refs;
+				fs->pwd_refs = 0;
+			}
 			hits += replace_path(&fs->root, old_root, new_root);
-			hits += replace_path(&fs->pwd, old_root, new_root);
 			while (hits--) {
 				count++;
 				path_get(new_root);
@@ -80,9 +89,22 @@ void chroot_fs_refs(const struct path *old_root, const struct path *new_root)
 		path_put(old_root);
 }
 
+/*
+ * Drain extra pwd references from the pool. The caller must ensure
+ * exclusive access to @fs (e.g., fs->users == 1 or under write_seqlock).
+ */
+void drain_fs_pwd_pool(struct fs_struct *fs)
+{
+	while (fs->pwd_refs) {
+		path_put(&fs->pwd);
+		fs->pwd_refs--;
+	}
+}
+
 void free_fs_struct(struct fs_struct *fs)
 {
 	path_put(&fs->root);
+	drain_fs_pwd_pool(fs);
 	path_put(&fs->pwd);
 	kmem_cache_free(fs_cachep, fs);
 }
@@ -111,14 +133,14 @@ struct fs_struct *copy_fs_struct(struct fs_struct *old)
 	if (fs) {
 		fs->users = 1;
 		fs->in_exec = 0;
+		fs->pwd_refs = 0;
 		seqlock_init(&fs->seq);
 		fs->umask = old->umask;
 
 		read_seqlock_excl(&old->seq);
 		fs->root = old->root;
 		path_get(&fs->root);
-		fs->pwd = old->pwd;
-		path_get(&fs->pwd);
+		get_fs_pwd_pool_locked(old, &fs->pwd);
 		read_sequnlock_excl(&old->seq);
 	}
 	return fs;
diff --git a/fs/fserror.c b/fs/fserror.c
index 06ca86adab9b7..1e4d11fd9562f 100644
--- a/fs/fserror.c
+++ b/fs/fserror.c
@@ -176,7 +176,7 @@ void fserror_report(struct super_block *sb, struct inode *inode,
 lost:
 	if (inode)
 		pr_err_ratelimited(
- "%s: lost file I/O error report for ino %lu type %u pos 0x%llx len 0x%llx error %d",
+ "%s: lost file I/O error report for ino %llu type %u pos 0x%llx len 0x%llx error %d",
 		       sb->s_id, inode->i_ino, type, pos, len, error);
 	else
 		pr_err_ratelimited(
diff --git a/fs/fuse/control.c b/fs/fuse/control.c
index 140bd5730d998..f902a7fb4630c 100644
--- a/fs/fuse/control.c
+++ b/fs/fuse/control.c
@@ -121,7 +121,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file,
 					      const char __user *buf,
 					      size_t count, loff_t *ppos)
 {
-	unsigned val;
+	unsigned int val = 0;
 	ssize_t ret;
 
 	ret = fuse_conn_limit_write(file, buf, count, ppos, &val,
@@ -163,7 +163,7 @@ static ssize_t fuse_conn_congestion_threshold_write(struct file *file,
 						    const char __user *buf,
 						    size_t count, loff_t *ppos)
 {
-	unsigned val;
+	unsigned int val = 0;
 	struct fuse_conn *fc;
 	ssize_t ret;
 
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 0b0241f47170d..b212565a78cf4 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -570,6 +570,11 @@ static void request_wait_answer(struct fuse_req *req)
 		if (!err)
 			return;
 
+		if (req->args->abort_on_kill) {
+			fuse_abort_conn(fc);
+			return;
+		}
+
 		if (test_bit(FR_URING, &req->flags))
 			removed = fuse_uring_remove_pending_req(req);
 		else
@@ -676,7 +681,8 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
 			fuse_force_creds(req);
 
 		__set_bit(FR_WAITING, &req->flags);
-		__set_bit(FR_FORCE, &req->flags);
+		if (!args->abort_on_kill)
+			__set_bit(FR_FORCE, &req->flags);
 	} else {
 		WARN_ON(args->nocreds);
 		req = fuse_get_req(idmap, fm, false);
@@ -1011,6 +1017,9 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop
 	folio_clear_uptodate(newfolio);
 	folio_clear_mappedtodisk(newfolio);
 
+	if (folio_test_large(newfolio))
+		goto out_fallback_unlock;
+
 	if (fuse_check_folio(newfolio) != 0)
 		goto out_fallback_unlock;
 
@@ -1764,10 +1773,9 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 	struct address_space *mapping;
 	u64 nodeid;
 	int err;
-	pgoff_t index;
-	unsigned int offset;
 	unsigned int num;
 	loff_t file_size;
+	loff_t pos;
 	loff_t end;
 
 	if (size < sizeof(outarg))
@@ -1780,7 +1788,12 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 	if (size - sizeof(outarg) != outarg.size)
 		return -EINVAL;
 
+	if (outarg.offset >= MAX_LFS_FILESIZE)
+		return -EINVAL;
+
 	nodeid = outarg.nodeid;
+	pos = outarg.offset;
+	num = min(outarg.size, MAX_LFS_FILESIZE - pos);
 
 	down_read(&fc->killsb);
 
@@ -1790,33 +1803,29 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 		goto out_up_killsb;
 
 	mapping = inode->i_mapping;
-	index = outarg.offset >> PAGE_SHIFT;
-	offset = outarg.offset & ~PAGE_MASK;
 	file_size = i_size_read(inode);
-	end = outarg.offset + outarg.size;
+	end = pos + num;
 	if (end > file_size) {
 		file_size = end;
-		fuse_write_update_attr(inode, file_size, outarg.size);
+		fuse_write_update_attr(inode, file_size, num);
 	}
 
-	num = outarg.size;
 	while (num) {
 		struct folio *folio;
 		unsigned int folio_offset;
 		unsigned int nr_bytes;
-		unsigned int nr_pages;
+		pgoff_t index = pos >> PAGE_SHIFT;
 
 		folio = filemap_grab_folio(mapping, index);
 		err = PTR_ERR(folio);
 		if (IS_ERR(folio))
 			goto out_iput;
 
-		folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
+		folio_offset = offset_in_folio(folio, pos);
 		nr_bytes = min(num, folio_size(folio) - folio_offset);
-		nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
-		if (!folio_test_uptodate(folio) && !err && offset == 0 &&
+		if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
 		    (nr_bytes == folio_size(folio) || file_size == end)) {
 			folio_zero_segment(folio, nr_bytes, folio_size(folio));
 			folio_mark_uptodate(folio);
@@ -1827,9 +1836,8 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 		if (err)
 			goto out_iput;
 
+		pos += nr_bytes;
 		num -= nr_bytes;
-		offset = 0;
-		index += nr_pages;
 	}
 
 	err = 0;
@@ -1861,7 +1869,6 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
 {
 	int err;
 	struct address_space *mapping = inode->i_mapping;
-	pgoff_t index;
 	loff_t file_size;
 	unsigned int num;
 	unsigned int offset;
@@ -1872,17 +1879,18 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
 	size_t args_size = sizeof(*ra);
 	struct fuse_args_pages *ap;
 	struct fuse_args *args;
+	loff_t pos = outarg->offset;
 
-	offset = outarg->offset & ~PAGE_MASK;
+	offset = offset_in_page(pos);
 	file_size = i_size_read(inode);
 
 	num = min(outarg->size, fc->max_write);
-	if (outarg->offset > file_size)
+	if (pos > file_size)
 		num = 0;
-	else if (outarg->offset + num > file_size)
-		num = file_size - outarg->offset;
+	else if (num > file_size - pos)
+		num = file_size - pos;
 
-	num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE);
 	num_pages = min(num_pages, fc->max_pages);
 	num = min(num, num_pages << PAGE_SHIFT);
 
@@ -1903,31 +1911,27 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
 	args->in_pages = true;
 	args->end = fuse_retrieve_end;
 
-	index = outarg->offset >> PAGE_SHIFT;
-
 	while (num && ap->num_folios < num_pages) {
 		struct folio *folio;
 		unsigned int folio_offset;
 		unsigned int nr_bytes;
-		unsigned int nr_pages;
+		pgoff_t index = pos >> PAGE_SHIFT;
 
 		folio = filemap_get_folio(mapping, index);
 		if (IS_ERR(folio))
 			break;
 
-		folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
+		folio_offset = offset_in_folio(folio, pos);
 		nr_bytes = min(folio_size(folio) - folio_offset, num);
-		nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 		ap->folios[ap->num_folios] = folio;
 		ap->descs[ap->num_folios].offset = folio_offset;
 		ap->descs[ap->num_folios].length = nr_bytes;
 		ap->num_folios++;
 
-		offset = 0;
+		pos += nr_bytes;
 		num -= nr_bytes;
 		total_len += nr_bytes;
-		index += nr_pages;
 	}
 	ra->inarg.offset = outarg->offset;
 	ra->inarg.size = total_len;
@@ -1961,6 +1965,9 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
 
 	fuse_copy_finish(cs);
 
+	if (outarg.offset >= MAX_LFS_FILESIZE)
+		return -EINVAL;
+
 	down_read(&fc->killsb);
 	err = -ENOENT;
 	nodeid = outarg.nodeid;
diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 3a38b61aac26f..7b9822e8837bc 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -397,6 +397,20 @@ static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue)
 				     FRRS_AVAILABLE);
 }
 
+static void fuse_uring_teardown_all_queues(struct fuse_ring *ring)
+{
+	int qid;
+
+	for (qid = 0; qid < ring->nr_queues; qid++) {
+		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);
+
+		if (!queue)
+			continue;
+
+		fuse_uring_teardown_entries(queue);
+	}
+}
+
 /*
  * Log state debug info
  */
@@ -431,19 +445,10 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring)
 
 static void fuse_uring_async_stop_queues(struct work_struct *work)
 {
-	int qid;
 	struct fuse_ring *ring =
 		container_of(work, struct fuse_ring, async_teardown_work.work);
 
-	/* XXX code dup */
-	for (qid = 0; qid < ring->nr_queues; qid++) {
-		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);
-
-		if (!queue)
-			continue;
-
-		fuse_uring_teardown_entries(queue);
-	}
+	fuse_uring_teardown_all_queues(ring);
 
 	/*
 	 * Some ring entries might be in the middle of IO operations,
@@ -469,16 +474,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work)
  */
 void fuse_uring_stop_queues(struct fuse_ring *ring)
 {
-	int qid;
-
-	for (qid = 0; qid < ring->nr_queues; qid++) {
-		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);
-
-		if (!queue)
-			continue;
-
-		fuse_uring_teardown_entries(queue);
-	}
+	fuse_uring_teardown_all_queues(ring);
 
 	if (atomic_read(&ring->queue_refs) > 0) {
 		ring->teardown_time = jiffies;
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index 7ac6b232ef123..b658b6baf72fe 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -354,8 +354,8 @@ static void fuse_invalidate_entry(struct dentry *entry)
 	fuse_invalidate_entry_cache(entry);
 }
 
-static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args,
-			     u64 nodeid, const struct qstr *name,
+static void fuse_lookup_init(struct fuse_args *args, u64 nodeid,
+			     const struct qstr *name,
 			     struct fuse_entry_out *outarg)
 {
 	memset(outarg, 0, sizeof(struct fuse_entry_out));
@@ -421,8 +421,7 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name,
 
 		attr_version = fuse_get_attr_version(fm->fc);
 
-		fuse_lookup_init(fm->fc, &args, get_node_id(dir),
-				 name, &outarg);
+		fuse_lookup_init(&args, get_node_id(dir), name, &outarg);
 		ret = fuse_simple_request(fm, &args);
 		/* Zero nodeid is same as -ENOENT */
 		if (!ret && !outarg.nodeid)
@@ -481,6 +480,11 @@ static int fuse_dentry_init(struct dentry *dentry)
 	fd->dentry = dentry;
 	RB_CLEAR_NODE(&fd->node);
 	dentry->d_fsdata = fd;
+	/*
+	 * Initialising d_time (epoch) to '0' ensures the dentry is invalid
+	 * if compared to fc->epoch, which is initialized to '1'.
+	 */
+	dentry->d_time = 0;
 
 	return 0;
 }
@@ -570,7 +574,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name
 	attr_version = fuse_get_attr_version(fm->fc);
 	evict_ctr = fuse_get_evict_ctr(fm->fc);
 
-	fuse_lookup_init(fm->fc, &args, nodeid, name, outarg);
+	fuse_lookup_init(&args, nodeid, name, outarg);
 	err = fuse_simple_request(fm, &args);
 	/* Zero nodeid is same as -ENOENT, but with valid timeout */
 	if (err || !outarg->nodeid)
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index b1bb7153cb785..bb5acfa0b1dda 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -117,6 +117,12 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
 			fuse_simple_request(ff->fm, args);
 			fuse_release_end(ff->fm, args, 0);
 		} else {
+			/*
+			 * DAX inodes may need to issue a number of synchronous
+			 * request for clearing the mappings.
+			 */
+			if (ra && ra->inode && FUSE_IS_DAX(ra->inode))
+				args->may_block = true;
 			args->end = fuse_release_end;
 			if (fuse_simple_background(ff->fm, args,
 						   GFP_KERNEL | __GFP_NOFAIL))
@@ -947,7 +953,8 @@ static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter,
 	return ret;
 }
 
-static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx)
+static void fuse_iomap_submit_read(const struct iomap_iter *iter,
+		struct iomap_read_folio_ctx *ctx)
 {
 	struct fuse_fill_read_data *data = ctx->read_ctx;
 
@@ -958,7 +965,7 @@ static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx)
 
 static const struct iomap_read_ops fuse_iomap_read_ops = {
 	.read_folio_range = fuse_iomap_read_folio_range_async,
-	.submit_read = fuse_iomap_read_submit,
+	.submit_read = fuse_iomap_submit_read,
 };
 
 static int fuse_read_folio(struct file *file, struct folio *folio)
@@ -1242,7 +1249,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
 {
 	struct fuse_args_pages *ap = &ia->ap;
 	struct fuse_conn *fc = get_fuse_conn(mapping->host);
-	unsigned offset = pos & (PAGE_SIZE - 1);
 	size_t count = 0;
 	unsigned int num;
 	int err = 0;
@@ -1269,7 +1275,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
 		if (mapping_writably_mapped(mapping))
 			flush_dcache_folio(folio);
 
-		folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
+		folio_offset = offset_in_folio(folio, pos);
 		bytes = min(folio_size(folio) - folio_offset, num);
 
 		tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii);
@@ -1299,9 +1305,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
 		count += tmp;
 		pos += tmp;
 		num -= tmp;
-		offset += tmp;
-		if (offset == folio_size(folio))
-			offset = 0;
 
 		/* If we copied full folio, mark it uptodate */
 		if (tmp == folio_size(folio))
@@ -1313,7 +1316,9 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
 			ia->write.folio_locked = true;
 			break;
 		}
-		if (!fc->big_writes || offset != 0)
+		if (!fc->big_writes)
+			break;
+		if (folio_offset + tmp != folio_size(folio))
 			break;
 	}
 
@@ -2172,7 +2177,7 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
 	WARN_ON(!ap->num_folios);
 
 	/* Reached max pages */
-	if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages)
+	if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages)
 		return true;
 
 	if (bytes > max_bytes)
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 7f16049387d15..23a241f18623a 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -345,6 +345,7 @@ struct fuse_args {
 	bool is_ext:1;
 	bool is_pinned:1;
 	bool invalidate_vmap:1;
+	bool abort_on_kill:1;
 	struct fuse_in_arg in_args[4];
 	struct fuse_arg out_args[2];
 	void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index e57b8af06be93..84f78fb89d353 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -1551,6 +1551,7 @@ int fuse_send_init(struct fuse_mount *fm)
 	int err;
 
 	if (fm->fc->sync_init) {
+		ia->args.abort_on_kill = true;
 		err = fuse_simple_request(fm, &ia->args);
 		/* Ignore size of init reply */
 		if (err > 0)
diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c
index 72de97c03d0ee..60018c6359342 100644
--- a/fs/fuse/passthrough.c
+++ b/fs/fuse/passthrough.c
@@ -10,6 +10,7 @@
 #include <linux/file.h>
 #include <linux/backing-file.h>
 #include <linux/splice.h>
+#include <linux/uio.h>
 
 static void fuse_file_accessed(struct file *file)
 {
@@ -167,7 +168,7 @@ struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id)
 		goto out;
 
 	/* Allocate backing file per fuse file to store fuse path */
-	backing_file = backing_file_open(&file->f_path, file->f_flags,
+	backing_file = backing_file_open(&file->f_path, file->f_cred, file->f_flags,
 					 &fb->file->f_path, fb->cred);
 	err = PTR_ERR(backing_file);
 	if (IS_ERR(backing_file)) {
diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c
index c2aae2eef0868..c88194e52d18f 100644
--- a/fs/fuse/readdir.c
+++ b/fs/fuse/readdir.c
@@ -52,7 +52,7 @@ static void fuse_add_dirent_to_cache(struct file *file,
 	}
 	version = fi->rdc.version;
 	size = fi->rdc.size;
-	offset = size & ~PAGE_MASK;
+	offset = offset_in_page(size);
 	index = size >> PAGE_SHIFT;
 	/* Dirent doesn't fit in current page?  Jump to next page. */
 	if (offset + reclen > PAGE_SIZE) {
@@ -392,7 +392,7 @@ static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff,
 					       void *addr, unsigned int size,
 					       struct dir_context *ctx)
 {
-	unsigned int offset = ff->readdir.cache_off & ~PAGE_MASK;
+	unsigned int offset = offset_in_page(ff->readdir.cache_off);
 	enum fuse_parse_result res = FOUND_NONE;
 
 	WARN_ON(offset >= size);
@@ -518,13 +518,13 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx)
 	index = ff->readdir.cache_off >> PAGE_SHIFT;
 
 	if (index == (fi->rdc.size >> PAGE_SHIFT))
-		size = fi->rdc.size & ~PAGE_MASK;
+		size = offset_in_page(fi->rdc.size);
 	else
 		size = PAGE_SIZE;
 	spin_unlock(&fi->rdc.lock);
 
 	/* EOF? */
-	if ((ff->readdir.cache_off & ~PAGE_MASK) == size)
+	if (offset_in_page(ff->readdir.cache_off) == size)
 		return 0;
 
 	page = find_get_page_flags(file->f_mapping, index,
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index 057e65b51b99d..2f7485ffac527 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -758,6 +758,27 @@ static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
 	req->argbuf = NULL;
 }
 
+/* Verify that the server properly follows the FUSE protocol */
+static bool virtio_fs_verify_response(struct fuse_req *req, unsigned int len)
+{
+	struct fuse_out_header *oh = &req->out.h;
+
+	if (len < sizeof(*oh)) {
+		pr_warn("virtio-fs: response too short (%u)\n", len);
+		return false;
+	}
+	if (oh->len != len) {
+		pr_warn("virtio-fs: oh.len mismatch (%u != %u)\n", oh->len, len);
+		return false;
+	}
+	if (oh->unique != req->in.h.unique) {
+		pr_warn("virtio-fs: oh.unique mismatch (%llu != %llu)\n",
+			oh->unique, req->in.h.unique);
+		return false;
+	}
+	return true;
+}
+
 /* Work function for request completion */
 static void virtio_fs_request_complete(struct fuse_req *req,
 				       struct virtio_fs_vq *fsvq)
@@ -767,10 +788,6 @@ static void virtio_fs_request_complete(struct fuse_req *req,
 	unsigned int len, i, thislen;
 	struct folio *folio;
 
-	/*
-	 * TODO verify that server properly follows FUSE protocol
-	 * (oh.uniq, oh.len)
-	 */
 	args = req->args;
 	copy_args_from_argbuf(args, req);
 
@@ -824,6 +841,10 @@ static void virtio_fs_requests_done_work(struct work_struct *work)
 		virtqueue_disable_cb(vq);
 
 		while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
+			if (!virtio_fs_verify_response(req, len)) {
+				req->out.h.error = -EIO;
+				req->out.h.len = sizeof(struct fuse_out_header);
+			}
 			spin_lock(&fpq->lock);
 			list_move_tail(&req->list, &reqs);
 			spin_unlock(&fpq->lock);
diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c
index e79ad087512a0..7d8db5a88b297 100644
--- a/fs/gfs2/aops.c
+++ b/fs/gfs2/aops.c
@@ -10,7 +10,7 @@
 #include <linux/completion.h>
 #include <linux/buffer_head.h>
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/mpage.h>
 #include <linux/fs.h>
 #include <linux/writeback.h>
@@ -158,6 +158,7 @@ static int gfs2_writepages(struct address_space *mapping,
 			   struct writeback_control *wbc)
 {
 	struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping);
+	long initial_nr_to_write = wbc->nr_to_write;
 	struct iomap_writepage_ctx wpc = {
 		.inode		= mapping->host,
 		.wbc		= wbc,
@@ -166,13 +167,13 @@ static int gfs2_writepages(struct address_space *mapping,
 	int ret;
 
 	/*
-	 * Even if we didn't write enough pages here, we might still be holding
+	 * Even if we didn't write any pages here, we might still be holding
 	 * dirty pages in the ail. We forcibly flush the ail because we don't
 	 * want balance_dirty_pages() to loop indefinitely trying to write out
 	 * pages held in the ail that it can't find.
 	 */
 	ret = iomap_writepages(&wpc);
-	if (ret == 0 && wbc->nr_to_write > 0)
+	if (ret == 0 && wbc->nr_to_write == initial_nr_to_write)
 		set_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags);
 	return ret;
 }
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index 8344040ecaf79..e9bf4879c07f7 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -892,7 +892,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 		goto fail_gunlock4;
 
 	mark_inode_dirty(inode);
-	d_instantiate(dentry, inode);
+	d_instantiate_new(dentry, inode);
 	/* After instantiate, errors should result in evict which will destroy
 	 * both inode and iopen glocks properly. */
 	if (file) {
@@ -904,7 +904,6 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	gfs2_glock_dq_uninit(&gh);
 	gfs2_glock_put(io_gl);
 	gfs2_qa_put(dip);
-	unlock_new_inode(inode);
 	return error;
 
 fail_gunlock4:
diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
index bc191b55b18e5..180b5c6858ae6 100644
--- a/fs/gfs2/super.c
+++ b/fs/gfs2/super.c
@@ -449,7 +449,7 @@ static int gfs2_write_inode(struct inode *inode, struct writeback_control *wbc)
 		gfs2_log_flush(GFS2_SB(inode), ip->i_gl,
 			       GFS2_LOG_HEAD_FLUSH_NORMAL |
 			       GFS2_LFC_WRITE_INODE);
-	if (bdi->wb.dirty_exceeded)
+	if (bdi_wb_dirty_exceeded(bdi))
 		gfs2_ail1_flush(sdp, wbc);
 	else
 		filemap_fdatawrite(metamapping);
@@ -1241,6 +1241,9 @@ static enum evict_behavior evict_should_delete(struct inode *inode,
 	struct gfs2_sbd *sdp = sb->s_fs_info;
 	int ret;
 
+	if (inode->i_nlink)
+		return EVICT_SHOULD_SKIP_DELETE;
+
 	if (gfs2_holder_initialized(&ip->i_iopen_gh) &&
 	    test_bit(GLF_DEFER_DELETE, &ip->i_iopen_gh.gh_gl->gl_flags))
 		return EVICT_SHOULD_DEFER_DELETE;
@@ -1279,12 +1282,18 @@ static enum evict_behavior evict_should_delete(struct inode *inode,
 /**
  * evict_unlinked_inode - delete the pieces of an unlinked evicted inode
  * @inode: The inode to evict
+ * @gh: The glock holder structure
  */
-static int evict_unlinked_inode(struct inode *inode)
+static int evict_unlinked_inode(struct inode *inode, struct gfs2_holder *gh)
 {
 	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = ip->i_gl;
 	int ret;
 
+	/* The inode glock must be held exclusively and be instantiated. */
+	BUG_ON(!gfs2_holder_initialized(gh) ||
+	       test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags));
+
 	if (S_ISDIR(inode->i_mode) &&
 	    (ip->i_diskflags & GFS2_DIF_EXHASH)) {
 		ret = gfs2_dir_exhash_dealloc(ip);
@@ -1317,8 +1326,8 @@ static int evict_unlinked_inode(struct inode *inode)
 	 */
 
 	ret = gfs2_dinode_dealloc(ip);
-	if (!ret && ip->i_gl)
-		gfs2_inode_remember_delete(ip->i_gl, ip->i_no_formal_ino);
+	if (!ret)
+		gfs2_inode_remember_delete(gl, ip->i_no_formal_ino);
 
 out:
 	return ret;
@@ -1327,33 +1336,40 @@ static int evict_unlinked_inode(struct inode *inode)
 /*
  * evict_linked_inode - evict an inode whose dinode has not been unlinked
  * @inode: The inode to evict
+ * @gh: The glock holder structure
  */
-static int evict_linked_inode(struct inode *inode)
+static int evict_linked_inode(struct inode *inode, struct gfs2_holder *gh)
 {
 	struct super_block *sb = inode->i_sb;
 	struct gfs2_sbd *sdp = sb->s_fs_info;
 	struct gfs2_inode *ip = GFS2_I(inode);
-	struct address_space *metamapping;
+	struct gfs2_glock *gl = ip->i_gl;
+	struct address_space *metamapping = gfs2_glock2aspace(gl);
 	int ret;
 
-	gfs2_log_flush(sdp, ip->i_gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
+	if (!(test_bit(GLF_DIRTY, &gl->gl_flags) || inode->i_flags & I_DIRTY))
+		goto clean;
+
+	/* The inode glock must be held exclusively and be instantiated. */
+	if (!gfs2_holder_initialized(gh))
+		ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, gh);
+	else
+		ret = gfs2_instantiate(gh);
+	if (ret)
+		return ret;
+
+	gfs2_log_flush(sdp, gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
 		       GFS2_LFC_EVICT_INODE);
-	metamapping = gfs2_glock2aspace(ip->i_gl);
-	if (test_bit(GLF_DIRTY, &ip->i_gl->gl_flags)) {
+	if (test_bit(GLF_DIRTY, &gl->gl_flags)) {
 		filemap_fdatawrite(metamapping);
 		filemap_fdatawait(metamapping);
 	}
 	write_inode_now(inode, 1);
-	gfs2_ail_flush(ip->i_gl, 0);
+	gfs2_ail_flush(gl, 0);
 
-	ret = gfs2_trans_begin(sdp, 0, sdp->sd_jdesc->jd_blocks);
-	if (ret)
-		return ret;
-
-	/* Needs to be done before glock release & also in a transaction */
+clean:
 	truncate_inode_pages(&inode->i_data, 0);
 	truncate_inode_pages(metamapping, 0);
-	gfs2_trans_end(sdp);
 	return 0;
 }
 
@@ -1388,7 +1404,7 @@ static void gfs2_evict_inode(struct inode *inode)
 	int ret;
 
 	gfs2_holder_mark_uninitialized(&gh);
-	if (inode->i_nlink || sb_rdonly(sb) || !ip->i_no_addr)
+	if (sb_rdonly(sb) || !ip->i_no_addr || !ip->i_gl)
 		goto out;
 
 	/*
@@ -1413,14 +1429,14 @@ static void gfs2_evict_inode(struct inode *inode)
 		behavior = EVICT_SHOULD_SKIP_DELETE;
 	}
 	if (behavior == EVICT_SHOULD_DELETE)
-		ret = evict_unlinked_inode(inode);
+		ret = evict_unlinked_inode(inode, &gh);
 	else
-		ret = evict_linked_inode(inode);
+		ret = evict_linked_inode(inode, &gh);
 
 	if (gfs2_rs_active(&ip->i_res))
 		gfs2_rs_deltree(&ip->i_res);
 
-	if (ret && ret != GLR_TRYFAILED && ret != -EROFS)
+	if (ret && ret != -EROFS)
 		fs_warn(sdp, "gfs2_evict_inode: %d\n", ret);
 out:
 	if (gfs2_holder_initialized(&gh))
diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c
index b80ba40e38776..7f5339ee57c15 100644
--- a/fs/hfs/catalog.c
+++ b/fs/hfs/catalog.c
@@ -417,7 +417,7 @@ int hfs_cat_move(u32 cnid, struct inode *src_dir, const struct qstr *src_name,
 	int entry_size, type;
 	int err;
 
-	hfs_dbg("cnid %u - (ino %lu, name %s) - (ino %lu, name %s)\n",
+	hfs_dbg("cnid %u - (ino %llu, name %s) - (ino %llu, name %s)\n",
 		cnid, src_dir->i_ino, src_name->name,
 		dst_dir->i_ino, dst_name->name);
 	sb = src_dir->i_sb;
diff --git a/fs/hfs/extent.c b/fs/hfs/extent.c
index a097908b269d0..f066a99a863bc 100644
--- a/fs/hfs/extent.c
+++ b/fs/hfs/extent.c
@@ -411,7 +411,7 @@ int hfs_extend_file(struct inode *inode)
 		goto out;
 	}
 
-	hfs_dbg("ino %lu, start %u, len %u\n", inode->i_ino, start, len);
+	hfs_dbg("ino %llu, start %u, len %u\n", inode->i_ino, start, len);
 	if (HFS_I(inode)->alloc_blocks == HFS_I(inode)->first_blocks) {
 		if (!HFS_I(inode)->first_blocks) {
 			hfs_dbg("first_extent: start %u, len %u\n",
@@ -482,7 +482,7 @@ void hfs_file_truncate(struct inode *inode)
 	u32 size;
 	int res;
 
-	hfs_dbg("ino %lu, phys_size %llu -> i_size %llu\n",
+	hfs_dbg("ino %llu, phys_size %llu -> i_size %llu\n",
 		inode->i_ino, (long long)HFS_I(inode)->phys_size,
 		inode->i_size);
 	if (inode->i_size > HFS_I(inode)->phys_size) {
diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c
index 878535db64d67..95f0333a608b0 100644
--- a/fs/hfs/inode.c
+++ b/fs/hfs/inode.c
@@ -270,7 +270,7 @@ void hfs_delete_inode(struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
 
-	hfs_dbg("ino %lu\n", inode->i_ino);
+	hfs_dbg("ino %llu\n", inode->i_ino);
 	if (S_ISDIR(inode->i_mode)) {
 		atomic64_dec(&HFS_SB(sb)->folder_count);
 		if (HFS_I(inode)->cat_key.ParID == cpu_to_be32(HFS_ROOT_CNID))
@@ -455,7 +455,7 @@ int hfs_write_inode(struct inode *inode, struct writeback_control *wbc)
 	hfs_cat_rec rec;
 	int res;
 
-	hfs_dbg("ino %lu\n", inode->i_ino);
+	hfs_dbg("ino %llu\n", inode->i_ino);
 	res = hfs_ext_write_extent(inode);
 	if (res)
 		return res;
diff --git a/fs/hfs/super.c b/fs/hfs/super.c
index a4f2a2bfa6d30..2e52acf282b0f 100644
--- a/fs/hfs/super.c
+++ b/fs/hfs/super.c
@@ -361,7 +361,7 @@ static int hfs_fill_super(struct super_block *sb, struct fs_context *fc)
 			goto bail_hfs_find;
 		}
 		hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, fd.entrylength);
-		if (rec.type != HFS_CDR_DIR)
+		if (rec.type != HFS_CDR_DIR || rec.dir.DirID != cpu_to_be32(HFS_ROOT_CNID))
 			res = -EIO;
 	}
 	if (res)
diff --git a/fs/hfsplus/attributes.c b/fs/hfsplus/attributes.c
index 4b79cd606276e..174cd13106ad6 100644
--- a/fs/hfsplus/attributes.c
+++ b/fs/hfsplus/attributes.c
@@ -203,7 +203,7 @@ int hfsplus_create_attr_nolock(struct inode *inode, const char *name,
 	int entry_size;
 	int err;
 
-	hfs_dbg("name %s, ino %ld\n",
+	hfs_dbg("name %s, ino %llu\n",
 		name ? name : NULL, inode->i_ino);
 
 	if (name) {
@@ -255,7 +255,7 @@ int hfsplus_create_attr(struct inode *inode,
 	hfsplus_attr_entry *entry_ptr;
 	int err;
 
-	hfs_dbg("name %s, ino %ld\n",
+	hfs_dbg("name %s, ino %llu\n",
 		name ? name : NULL, inode->i_ino);
 
 	if (!HFSPLUS_SB(sb)->attr_tree) {
@@ -337,7 +337,7 @@ int hfsplus_delete_attr_nolock(struct inode *inode, const char *name,
 	struct super_block *sb = inode->i_sb;
 	int err;
 
-	hfs_dbg("name %s, ino %ld\n",
+	hfs_dbg("name %s, ino %llu\n",
 		name ? name : NULL, inode->i_ino);
 
 	if (name) {
@@ -367,7 +367,7 @@ int hfsplus_delete_attr(struct inode *inode, const char *name)
 	struct super_block *sb = inode->i_sb;
 	struct hfs_find_data fd;
 
-	hfs_dbg("name %s, ino %ld\n",
+	hfs_dbg("name %s, ino %llu\n",
 		name ? name : NULL, inode->i_ino);
 
 	if (!HFSPLUS_SB(sb)->attr_tree) {
@@ -436,7 +436,7 @@ int hfsplus_replace_attr(struct inode *inode,
 	hfsplus_attr_entry *entry_ptr;
 	int err = 0;
 
-	hfs_dbg("name %s, ino %ld\n",
+	hfs_dbg("name %s, ino %llu\n",
 		name ? name : NULL, inode->i_ino);
 
 	if (!HFSPLUS_SB(sb)->attr_tree) {
diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c
index 02c1eee4a4b86..0e961e99b9856 100644
--- a/fs/hfsplus/catalog.c
+++ b/fs/hfsplus/catalog.c
@@ -441,7 +441,7 @@ int hfsplus_rename_cat(u32 cnid,
 	int entry_size, type;
 	int err;
 
-	hfs_dbg("cnid %u - ino %lu, name %s - ino %lu, name %s\n",
+	hfs_dbg("cnid %u - ino %llu, name %s - ino %llu, name %s\n",
 		cnid, src_dir->i_ino, src_name->name,
 		dst_dir->i_ino, dst_name->name);
 	err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &src_fd);
diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c
index d559bf8625f85..054f6da460334 100644
--- a/fs/hfsplus/dir.c
+++ b/fs/hfsplus/dir.c
@@ -313,7 +313,7 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir,
 	if (!S_ISREG(inode->i_mode))
 		return -EPERM;
 
-	hfs_dbg("src_dir->i_ino %lu, dst_dir->i_ino %lu, inode->i_ino %lu\n",
+	hfs_dbg("src_dir->i_ino %llu, dst_dir->i_ino %llu, inode->i_ino %llu\n",
 		src_dir->i_ino, dst_dir->i_ino, inode->i_ino);
 
 	mutex_lock(&sbi->vh_mutex);
@@ -385,7 +385,7 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry)
 	if (HFSPLUS_IS_RSRC(inode))
 		return -EPERM;
 
-	hfs_dbg("dir->i_ino %lu, inode->i_ino %lu\n",
+	hfs_dbg("dir->i_ino %llu, inode->i_ino %llu\n",
 		dir->i_ino, inode->i_ino);
 
 	mutex_lock(&sbi->vh_mutex);
@@ -393,7 +393,7 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry)
 	if (inode->i_ino == cnid &&
 	    atomic_read(&HFSPLUS_I(inode)->opencnt)) {
 		str.name = name;
-		str.len = sprintf(name, "temp%lu", inode->i_ino);
+		str.len = sprintf(name, "temp%llu", inode->i_ino);
 		res = hfsplus_rename_cat(inode->i_ino,
 					 dir, &dentry->d_name,
 					 sbi->hidden_dir, &str);
diff --git a/fs/hfsplus/extents.c b/fs/hfsplus/extents.c
index 8e886514d27f1..474fde1a1653b 100644
--- a/fs/hfsplus/extents.c
+++ b/fs/hfsplus/extents.c
@@ -275,7 +275,7 @@ int hfsplus_get_block(struct inode *inode, sector_t iblock,
 	mutex_unlock(&hip->extents_lock);
 
 done:
-	hfs_dbg("ino %lu, iblock %llu - dblock %u\n",
+	hfs_dbg("ino %llu, iblock %llu - dblock %u\n",
 		inode->i_ino, (long long)iblock, dblock);
 
 	mask = (1 << sbi->fs_shift) - 1;
@@ -476,7 +476,7 @@ int hfsplus_file_extend(struct inode *inode, bool zeroout)
 			goto out;
 	}
 
-	hfs_dbg("ino %lu, start %u, len %u\n", inode->i_ino, start, len);
+	hfs_dbg("ino %llu, start %u, len %u\n", inode->i_ino, start, len);
 
 	if (hip->alloc_blocks <= hip->first_blocks) {
 		if (!hip->first_blocks) {
@@ -545,7 +545,7 @@ void hfsplus_file_truncate(struct inode *inode)
 	u32 alloc_cnt, blk_cnt, start;
 	int res;
 
-	hfs_dbg("ino %lu, phys_size %llu -> i_size %llu\n",
+	hfs_dbg("ino %llu, phys_size %llu -> i_size %llu\n",
 		inode->i_ino, (long long)hip->phys_size, inode->i_size);
 
 	if (inode->i_size > hip->phys_size) {
diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c
index 922ff41df042a..02be32dc6833d 100644
--- a/fs/hfsplus/inode.c
+++ b/fs/hfsplus/inode.c
@@ -230,7 +230,7 @@ static int hfsplus_get_perms(struct inode *inode,
 		inode->i_flags &= ~S_APPEND;
 	return 0;
 bad_type:
-	pr_err("invalid file type 0%04o for inode %lu\n", mode, inode->i_ino);
+	pr_err("invalid file type 0%04o for inode %llu\n", mode, inode->i_ino);
 	return -EIO;
 }
 
@@ -328,7 +328,7 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end,
 	struct hfsplus_vh *vhdr = sbi->s_vhdr;
 	int error = 0, error2;
 
-	hfs_dbg("inode->i_ino %lu, start %llu, end %llu\n",
+	hfs_dbg("inode->i_ino %llu, start %llu, end %llu\n",
 		inode->i_ino, start, end);
 
 	error = file_write_and_wait_range(file, start, end);
@@ -639,7 +639,7 @@ int hfsplus_cat_write_inode(struct inode *inode)
 	hfsplus_cat_entry entry;
 	int res = 0;
 
-	hfs_dbg("inode->i_ino %lu\n", inode->i_ino);
+	hfs_dbg("inode->i_ino %llu\n", inode->i_ino);
 
 	if (HFSPLUS_IS_RSRC(inode))
 		main_inode = HFSPLUS_I(inode)->rsrc_inode;
@@ -716,7 +716,7 @@ int hfsplus_cat_write_inode(struct inode *inode)
 	if (!res) {
 		res = hfs_btree_write(tree);
 		if (res) {
-			pr_err("b-tree write err: %d, ino %lu\n",
+			pr_err("b-tree write err: %d, ino %llu\n",
 			       res, inode->i_ino);
 		}
 	}
diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c
index 7229a8ae89f94..b3917249c206c 100644
--- a/fs/hfsplus/super.c
+++ b/fs/hfsplus/super.c
@@ -156,7 +156,7 @@ static int hfsplus_system_write_inode(struct inode *inode)
 		int err = hfs_btree_write(tree);
 
 		if (err) {
-			pr_err("b-tree write err: %d, ino %lu\n",
+			pr_err("b-tree write err: %d, ino %llu\n",
 			       err, inode->i_ino);
 			return err;
 		}
@@ -169,7 +169,7 @@ static int hfsplus_write_inode(struct inode *inode,
 {
 	int err;
 
-	hfs_dbg("ino %lu\n", inode->i_ino);
+	hfs_dbg("ino %llu\n", inode->i_ino);
 
 	err = hfsplus_ext_write_extent(inode);
 	if (err)
@@ -184,7 +184,7 @@ static int hfsplus_write_inode(struct inode *inode,
 
 static void hfsplus_evict_inode(struct inode *inode)
 {
-	hfs_dbg("ino %lu\n", inode->i_ino);
+	hfs_dbg("ino %llu\n", inode->i_ino);
 	truncate_inode_pages_final(&inode->i_data);
 	clear_inode(inode);
 	if (HFSPLUS_IS_RSRC(inode)) {
diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c
index 9904944cbd54e..c70bb6f494b22 100644
--- a/fs/hfsplus/xattr.c
+++ b/fs/hfsplus/xattr.c
@@ -277,7 +277,7 @@ int __hfsplus_setxattr(struct inode *inode, const char *name,
 	u16 folder_finderinfo_len = sizeof(DInfo) + sizeof(DXInfo);
 	u16 file_finderinfo_len = sizeof(FInfo) + sizeof(FXInfo);
 
-	hfs_dbg("ino %lu, name %s, value %p, size %zu\n",
+	hfs_dbg("ino %llu, name %s, value %p, size %zu\n",
 		inode->i_ino, name ? name : NULL,
 		value, size);
 
@@ -447,7 +447,7 @@ int hfsplus_setxattr(struct inode *inode, const char *name,
 		NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN + 1;
 	int res;
 
-	hfs_dbg("ino %lu, name %s, prefix %s, prefixlen %zu, "
+	hfs_dbg("ino %llu, name %s, prefix %s, prefixlen %zu, "
 		"value %p, size %zu\n",
 		inode->i_ino, name ? name : NULL,
 		prefix ? prefix : NULL, prefixlen,
@@ -607,7 +607,7 @@ ssize_t hfsplus_getxattr(struct inode *inode, const char *name,
 	int res;
 	char *xattr_name;
 
-	hfs_dbg("ino %lu, name %s, prefix %s\n",
+	hfs_dbg("ino %llu, name %s, prefix %s\n",
 		inode->i_ino, name ? name : NULL,
 		prefix ? prefix : NULL);
 
@@ -717,7 +717,7 @@ ssize_t hfsplus_listxattr(struct dentry *dentry, char *buffer, size_t size)
 	size_t strbuf_size;
 	int xattr_name_len;
 
-	hfs_dbg("ino %lu\n", inode->i_ino);
+	hfs_dbg("ino %llu\n", inode->i_ino);
 
 	if (!is_xattr_operation_supported(inode))
 		return -EOPNOTSUPP;
@@ -819,7 +819,7 @@ static int hfsplus_removexattr(struct inode *inode, const char *name)
 	int is_xattr_acl_deleted;
 	int is_all_xattrs_deleted;
 
-	hfs_dbg("ino %lu, name %s\n",
+	hfs_dbg("ino %llu, name %s\n",
 		inode->i_ino, name ? name : NULL);
 
 	if (!HFSPLUS_SB(inode->i_sb)->attr_tree)
diff --git a/fs/hpfs/dir.c b/fs/hpfs/dir.c
index ceb50b2dc91a9..3bf11202e2d3c 100644
--- a/fs/hpfs/dir.c
+++ b/fs/hpfs/dir.c
@@ -96,8 +96,8 @@ static int hpfs_readdir(struct file *file, struct dir_context *ctx)
 		}
 		if (!fnode_is_dir(fno)) {
 			e = 1;
-			hpfs_error(inode->i_sb, "not a directory, fnode %08lx",
-					(unsigned long)inode->i_ino);
+			hpfs_error(inode->i_sb, "not a directory, fnode %08llx",
+					inode->i_ino);
 		}
 		if (hpfs_inode->i_dno != le32_to_cpu(fno->u.external[0].disk_secno)) {
 			e = 1;
diff --git a/fs/hpfs/dnode.c b/fs/hpfs/dnode.c
index dde764ebe2467..8c6aa060fd874 100644
--- a/fs/hpfs/dnode.c
+++ b/fs/hpfs/dnode.c
@@ -550,9 +550,9 @@ static void delete_empty_dnode(struct inode *i, dnode_secno dno)
 			if (hpfs_sb(i->i_sb)->sb_chk)
 				if (up != i->i_ino) {
 					hpfs_error(i->i_sb,
-						   "bad pointer to fnode, dnode %08x, pointing to %08x, should be %08lx",
+						   "bad pointer to fnode, dnode %08x, pointing to %08x, should be %08llx",
 						   dno, up,
-						   (unsigned long)i->i_ino);
+						   i->i_ino);
 					return;
 				}
 			if ((d1 = hpfs_map_dnode(i->i_sb, down, &qbh1))) {
diff --git a/fs/hpfs/ea.c b/fs/hpfs/ea.c
index 2149d3ca530b6..4664f9ab06eef 100644
--- a/fs/hpfs/ea.c
+++ b/fs/hpfs/ea.c
@@ -245,8 +245,8 @@ void hpfs_set_ea(struct inode *inode, struct fnode *fnode, const char *key,
 		fnode->ea_offs = cpu_to_le16(0xc4);
 	}
 	if (le16_to_cpu(fnode->ea_offs) < 0xc4 || le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s) + le16_to_cpu(fnode->ea_size_s) > 0x200) {
-		hpfs_error(s, "fnode %08lx: ea_offs == %03x, ea_size_s == %03x",
-			(unsigned long)inode->i_ino,
+		hpfs_error(s, "fnode %08llx: ea_offs == %03x, ea_size_s == %03x",
+			inode->i_ino,
 			le16_to_cpu(fnode->ea_offs), le16_to_cpu(fnode->ea_size_s));
 		return;
 	}
diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c
index 93d528f4f4f28..0e932cc8be1b2 100644
--- a/fs/hpfs/inode.c
+++ b/fs/hpfs/inode.c
@@ -250,8 +250,8 @@ void hpfs_write_inode_nolock(struct inode *i)
 			hpfs_brelse4(&qbh);
 		} else
 			hpfs_error(i->i_sb,
-				"directory %08lx doesn't have '.' entry",
-				(unsigned long)i->i_ino);
+				"directory %08llx doesn't have '.' entry",
+				i->i_ino);
 	}
 	mark_buffer_dirty(bh);
 	brelse(bh);
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 3f70c47981deb..cd6b22f6e2b18 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -25,7 +25,7 @@
 #include <linux/ctype.h>
 #include <linux/backing-dev.h>
 #include <linux/hugetlb.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/fs_parser.h>
 #include <linux/mman.h>
 #include <linux/slab.h>
@@ -164,7 +164,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 		goto out;
 
 	ret = 0;
-	if (vma_desc_test_flags(desc, VMA_WRITE_BIT) && inode->i_size < len)
+	if (vma_desc_test(desc, VMA_WRITE_BIT) && inode->i_size < len)
 		i_size_write(inode, len);
 out:
 	inode_unlock(inode);
@@ -513,15 +513,11 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
 
 /*
  * Called with hugetlb fault mutex held.
- * Returns true if page was actually removed, false otherwise.
  */
-static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
-					struct address_space *mapping,
-					struct folio *folio, pgoff_t index,
-					bool truncate_op)
+static void remove_inode_single_folio(struct hstate *h, struct inode *inode,
+		struct address_space *mapping, struct folio *folio,
+		pgoff_t index, bool truncate_op)
 {
-	bool ret = false;
-
 	/*
 	 * If folio is mapped, it was faulted in after being
 	 * unmapped in caller or hugetlb_vmdelete_list() skips
@@ -543,7 +539,6 @@ static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
 	 */
 	VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio);
 	hugetlb_delete_from_page_cache(folio);
-	ret = true;
 	if (!truncate_op) {
 		if (unlikely(hugetlb_unreserve_pages(inode, index,
 							index + 1, 1)))
@@ -551,7 +546,6 @@ static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
 	}
 
 	folio_unlock(folio);
-	return ret;
 }
 
 /*
@@ -599,9 +593,9 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
 			/*
 			 * Remove folio that was part of folio_batch.
 			 */
-			if (remove_inode_single_folio(h, inode, mapping, folio,
-							index, truncate_op))
-				freed++;
+			remove_inode_single_folio(h, inode, mapping, folio,
+						  index, truncate_op);
+			freed++;
 
 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 		}
diff --git a/fs/inode.c b/fs/inode.c
index cc12b68e021b2..5ad169d51728c 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -672,7 +672,7 @@ static inline void inode_sb_list_del(struct inode *inode)
 	}
 }
 
-static unsigned long hash(struct super_block *sb, unsigned long hashval)
+static unsigned long hash(struct super_block *sb, u64 hashval)
 {
 	unsigned long tmp;
 
@@ -685,12 +685,12 @@ static unsigned long hash(struct super_block *sb, unsigned long hashval)
 /**
  *	__insert_inode_hash - hash an inode
  *	@inode: unhashed inode
- *	@hashval: unsigned long value used to locate this object in the
+ *	@hashval: u64 value used to locate this object in the
  *		inode_hashtable.
  *
  *	Add an inode to the inode hash for this superblock.
  */
-void __insert_inode_hash(struct inode *inode, unsigned long hashval)
+void __insert_inode_hash(struct inode *inode, u64 hashval)
 {
 	struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
 
@@ -726,7 +726,7 @@ void dump_mapping(const struct address_space *mapping)
 	struct dentry *dentry_ptr;
 	struct dentry dentry;
 	char fname[64] = {};
-	unsigned long ino;
+	u64 ino;
 
 	/*
 	 * If mapping is an invalid pointer, we don't want to crash
@@ -750,14 +750,14 @@ void dump_mapping(const struct address_space *mapping)
 	}
 
 	if (!dentry_first) {
-		pr_warn("aops:%ps ino:%lx\n", a_ops, ino);
+		pr_warn("aops:%ps ino:%llx\n", a_ops, ino);
 		return;
 	}
 
 	dentry_ptr = container_of(dentry_first, struct dentry, d_u.d_alias);
 	if (get_kernel_nofault(dentry, dentry_ptr) ||
 	    !dentry.d_parent || !dentry.d_name.name) {
-		pr_warn("aops:%ps ino:%lx invalid dentry:%px\n",
+		pr_warn("aops:%ps ino:%llx invalid dentry:%px\n",
 				a_ops, ino, dentry_ptr);
 		return;
 	}
@@ -768,7 +768,7 @@ void dump_mapping(const struct address_space *mapping)
 	 * Even if strncpy_from_kernel_nofault() succeeded,
 	 * the fname could be unreliable
 	 */
-	pr_warn("aops:%ps ino:%lx dentry name(?):\"%s\"\n",
+	pr_warn("aops:%ps ino:%llx dentry name(?):\"%s\"\n",
 		a_ops, ino, fname);
 }
 
@@ -1087,7 +1087,7 @@ static struct inode *find_inode(struct super_block *sb,
  * iget_locked for details.
  */
 static struct inode *find_inode_fast(struct super_block *sb,
-				struct hlist_head *head, unsigned long ino,
+				struct hlist_head *head, u64 ino,
 				bool hash_locked, bool *isnew)
 {
 	struct inode *inode = NULL;
@@ -1301,7 +1301,7 @@ EXPORT_SYMBOL(unlock_two_nondirectories);
  * Note that both @test and @set are called with the inode_hash_lock held, so
  * they can't sleep.
  */
-struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
+struct inode *inode_insert5(struct inode *inode, u64 hashval,
 			    int (*test)(struct inode *, void *),
 			    int (*set)(struct inode *, void *), void *data)
 {
@@ -1378,7 +1378,7 @@ EXPORT_SYMBOL(inode_insert5);
  * Note that both @test and @set are called with the inode_hash_lock held, so
  * they can't sleep.
  */
-struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
+struct inode *iget5_locked(struct super_block *sb, u64 hashval,
 		int (*test)(struct inode *, void *),
 		int (*set)(struct inode *, void *), void *data)
 {
@@ -1408,7 +1408,7 @@ EXPORT_SYMBOL(iget5_locked);
  * This is equivalent to iget5_locked, except the @test callback must
  * tolerate the inode not being stable, including being mid-teardown.
  */
-struct inode *iget5_locked_rcu(struct super_block *sb, unsigned long hashval,
+struct inode *iget5_locked_rcu(struct super_block *sb, u64 hashval,
 		int (*test)(struct inode *, void *),
 		int (*set)(struct inode *, void *), void *data)
 {
@@ -1455,7 +1455,7 @@ EXPORT_SYMBOL_GPL(iget5_locked_rcu);
  * hashed, and with the I_NEW flag set.  The file system gets to fill it in
  * before unlocking it via unlock_new_inode().
  */
-struct inode *iget_locked(struct super_block *sb, unsigned long ino)
+struct inode *iget_locked(struct super_block *sb, u64 ino)
 {
 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
 	struct inode *inode;
@@ -1527,7 +1527,7 @@ EXPORT_SYMBOL(iget_locked);
  *
  * Returns 1 if the inode number is unique, 0 if it is not.
  */
-static int test_inode_iunique(struct super_block *sb, unsigned long ino)
+static int test_inode_iunique(struct super_block *sb, u64 ino)
 {
 	struct hlist_head *b = inode_hashtable + hash(sb, ino);
 	struct inode *inode;
@@ -1616,7 +1616,7 @@ EXPORT_SYMBOL(igrab);
  *
  * Note2: @test is called with the inode_hash_lock held, so can't sleep.
  */
-struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
+struct inode *ilookup5_nowait(struct super_block *sb, u64 hashval,
 		int (*test)(struct inode *, void *), void *data, bool *isnew)
 {
 	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
@@ -1647,7 +1647,7 @@ EXPORT_SYMBOL(ilookup5_nowait);
  *
  * Note: @test is called with the inode_hash_lock held, so can't sleep.
  */
-struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
+struct inode *ilookup5(struct super_block *sb, u64 hashval,
 		int (*test)(struct inode *, void *), void *data)
 {
 	struct inode *inode;
@@ -1677,7 +1677,7 @@ EXPORT_SYMBOL(ilookup5);
  * Search for the inode @ino in the inode cache, and if the inode is in the
  * cache, the inode is returned with an incremented reference count.
  */
-struct inode *ilookup(struct super_block *sb, unsigned long ino)
+struct inode *ilookup(struct super_block *sb, u64 ino)
 {
 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
 	struct inode *inode;
@@ -1726,8 +1726,8 @@ EXPORT_SYMBOL(ilookup);
  * very carefully implemented.
  */
 struct inode *find_inode_nowait(struct super_block *sb,
-				unsigned long hashval,
-				int (*match)(struct inode *, unsigned long,
+				u64 hashval,
+				int (*match)(struct inode *, u64,
 					     void *),
 				void *data)
 {
@@ -1773,7 +1773,7 @@ EXPORT_SYMBOL(find_inode_nowait);
  *
  * The caller must hold the RCU read lock.
  */
-struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
+struct inode *find_inode_rcu(struct super_block *sb, u64 hashval,
 			     int (*test)(struct inode *, void *), void *data)
 {
 	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
@@ -1812,7 +1812,7 @@ EXPORT_SYMBOL(find_inode_rcu);
  * The caller must hold the RCU read lock.
  */
 struct inode *find_inode_by_ino_rcu(struct super_block *sb,
-				    unsigned long ino)
+				    u64 ino)
 {
 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
 	struct inode *inode;
@@ -1833,7 +1833,7 @@ EXPORT_SYMBOL(find_inode_by_ino_rcu);
 int insert_inode_locked(struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
-	ino_t ino = inode->i_ino;
+	u64 ino = inode->i_ino;
 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
 	bool isnew;
 
@@ -1884,7 +1884,7 @@ int insert_inode_locked(struct inode *inode)
 }
 EXPORT_SYMBOL(insert_inode_locked);
 
-int insert_inode_locked4(struct inode *inode, unsigned long hashval,
+int insert_inode_locked4(struct inode *inode, u64 hashval,
 		int (*test)(struct inode *, void *), void *data)
 {
 	struct inode *old;
@@ -2641,9 +2641,8 @@ void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
 		/* leave it no_open_fops */
 		break;
 	default:
-		printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
-				  " inode %s:%lu\n", mode, inode->i_sb->s_id,
-				  inode->i_ino);
+		pr_debug("init_special_inode: bogus i_mode (%o) for inode %s:%llu\n",
+			 mode, inode->i_sb->s_id, inode->i_ino);
 		break;
 	}
 }
diff --git a/fs/internal.h b/fs/internal.h
index cbc384a1aa096..7c44a58627ba3 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -106,8 +106,9 @@ extern void chroot_fs_refs(const struct path *, const struct path *);
  */
 struct file *alloc_empty_file(int flags, const struct cred *cred);
 struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred);
-struct file *alloc_empty_backing_file(int flags, const struct cred *cred);
-void backing_file_set_user_path(struct file *f, const struct path *path);
+struct file *alloc_empty_backing_file(int flags, const struct cred *cred,
+				      const struct cred *user_cred);
+int backing_file_open_user_path(struct file *f, const struct path *path);
 
 static inline void file_put_write_access(struct file *file)
 {
diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c
index fc045f2e4c459..f989ffcaac96d 100644
--- a/fs/iomap/bio.c
+++ b/fs/iomap/bio.c
@@ -3,74 +3,111 @@
  * Copyright (C) 2010 Red Hat, Inc.
  * Copyright (C) 2016-2023 Christoph Hellwig.
  */
+#include <linux/bio-integrity.h>
 #include <linux/iomap.h>
 #include <linux/pagemap.h>
 #include "internal.h"
 #include "trace.h"
 
-static void iomap_read_end_io(struct bio *bio)
+static u32 __iomap_read_end_io(struct bio *bio, int error)
 {
-	int error = blk_status_to_errno(bio->bi_status);
 	struct folio_iter fi;
+	u32 folio_count = 0;
 
-	bio_for_each_folio_all(fi, bio)
+	bio_for_each_folio_all(fi, bio) {
 		iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
+		folio_count++;
+	}
+	if (bio_integrity(bio))
+		fs_bio_integrity_free(bio);
 	bio_put(bio);
+	return folio_count;
+}
+
+static void iomap_read_end_io(struct bio *bio)
+{
+	__iomap_read_end_io(bio, blk_status_to_errno(bio->bi_status));
 }
 
-static void iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx)
+u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend)
+{
+	return __iomap_read_end_io(&ioend->io_bio, ioend->io_error);
+}
+
+static void iomap_bio_submit_read(const struct iomap_iter *iter,
+		struct iomap_read_folio_ctx *ctx)
 {
 	struct bio *bio = ctx->read_ctx;
 
-	if (bio)
-		submit_bio(bio);
+	if (iter->iomap.flags & IOMAP_F_INTEGRITY)
+		fs_bio_integrity_alloc(bio);
+	submit_bio(bio);
+}
+
+static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx)
+{
+	if (ctx->ops && ctx->ops->bio_set)
+		return ctx->ops->bio_set;
+	return &fs_bio_set;
+}
+
+static void iomap_read_alloc_bio(const struct iomap_iter *iter,
+		struct iomap_read_folio_ctx *ctx, size_t plen)
+{
+	const struct iomap *iomap = &iter->iomap;
+	unsigned int nr_vecs = DIV_ROUND_UP(iomap_length(iter), PAGE_SIZE);
+	struct bio_set *bio_set = iomap_read_bio_set(ctx);
+	struct folio *folio = ctx->cur_folio;
+	gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
+	gfp_t orig_gfp = gfp;
+	struct bio *bio;
+
+	/* Submit the existing range if there was one. */
+	if (ctx->read_ctx)
+		ctx->ops->submit_read(iter, ctx);
+
+	/* Same as readahead_gfp_mask: */
+	if (ctx->rac)
+		gfp |= __GFP_NORETRY | __GFP_NOWARN;
+
+	/*
+	 * If the bio_alloc fails, try it again for a single page to avoid
+	 * having to deal with partial page reads.  This emulates what
+	 * do_mpage_read_folio does.
+	 */
+	bio = bio_alloc_bioset(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
+			gfp, bio_set);
+	if (!bio)
+		bio = bio_alloc_bioset(iomap->bdev, 1, REQ_OP_READ, orig_gfp,
+				bio_set);
+	if (ctx->rac)
+		bio->bi_opf |= REQ_RAHEAD;
+	bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos);
+	bio->bi_end_io = iomap_read_end_io;
+	bio_add_folio_nofail(bio, folio, plen,
+			offset_in_folio(folio, iter->pos));
+	ctx->read_ctx = bio;
+	ctx->read_ctx_file_offset = iter->pos;
 }
 
-static int iomap_bio_read_folio_range(const struct iomap_iter *iter,
+int iomap_bio_read_folio_range(const struct iomap_iter *iter,
 		struct iomap_read_folio_ctx *ctx, size_t plen)
 {
 	struct folio *folio = ctx->cur_folio;
-	const struct iomap *iomap = &iter->iomap;
-	loff_t pos = iter->pos;
-	size_t poff = offset_in_folio(folio, pos);
-	loff_t length = iomap_length(iter);
-	sector_t sector;
 	struct bio *bio = ctx->read_ctx;
 
-	sector = iomap_sector(iomap, pos);
-	if (!bio || bio_end_sector(bio) != sector ||
-	    !bio_add_folio(bio, folio, plen, poff)) {
-		gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
-		gfp_t orig_gfp = gfp;
-		unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
-
-		if (bio)
-			submit_bio(bio);
-
-		if (ctx->rac) /* same as readahead_gfp_mask */
-			gfp |= __GFP_NORETRY | __GFP_NOWARN;
-		bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
-				     gfp);
-		/*
-		 * If the bio_alloc fails, try it again for a single page to
-		 * avoid having to deal with partial page reads.  This emulates
-		 * what do_mpage_read_folio does.
-		 */
-		if (!bio)
-			bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp);
-		if (ctx->rac)
-			bio->bi_opf |= REQ_RAHEAD;
-		bio->bi_iter.bi_sector = sector;
-		bio->bi_end_io = iomap_read_end_io;
-		bio_add_folio_nofail(bio, folio, plen, poff);
-		ctx->read_ctx = bio;
-	}
+	if (!bio ||
+	    bio_end_sector(bio) != iomap_sector(&iter->iomap, iter->pos) ||
+	    bio->bi_iter.bi_size > iomap_max_bio_size(&iter->iomap) - plen ||
+	    !bio_add_folio(bio, folio, plen, offset_in_folio(folio, iter->pos)))
+		iomap_read_alloc_bio(iter, ctx, plen);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(iomap_bio_read_folio_range);
 
 const struct iomap_read_ops iomap_bio_read_ops = {
-	.read_folio_range = iomap_bio_read_folio_range,
-	.submit_read = iomap_bio_submit_read,
+	.read_folio_range	= iomap_bio_read_folio_range,
+	.submit_read		= iomap_bio_submit_read,
 };
 EXPORT_SYMBOL_GPL(iomap_bio_read_ops);
 
@@ -78,11 +115,21 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
 		struct folio *folio, loff_t pos, size_t len)
 {
 	const struct iomap *srcmap = iomap_iter_srcmap(iter);
+	sector_t sector = iomap_sector(srcmap, pos);
 	struct bio_vec bvec;
 	struct bio bio;
+	int error;
 
 	bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ);
-	bio.bi_iter.bi_sector = iomap_sector(srcmap, pos);
+	bio.bi_iter.bi_sector = sector;
 	bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
-	return submit_bio_wait(&bio);
+	if (srcmap->flags & IOMAP_F_INTEGRITY)
+		fs_bio_integrity_alloc(&bio);
+	error = submit_bio_wait(&bio);
+	if (srcmap->flags & IOMAP_F_INTEGRITY) {
+		if (!error)
+			error = fs_bio_integrity_verify(&bio, sector, len);
+		fs_bio_integrity_free(&bio);
+	}
+	return error;
 }
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 00f0efaf12b22..3cf93ab2e38a5 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -596,8 +596,8 @@ void iomap_read_folio(const struct iomap_ops *ops,
 		iter.status = iomap_read_folio_iter(&iter, ctx,
 				&bytes_submitted);
 
-	if (ctx->ops->submit_read)
-		ctx->ops->submit_read(ctx);
+	if (ctx->read_ctx && ctx->ops->submit_read)
+		ctx->ops->submit_read(&iter, ctx);
 
 	if (ctx->cur_folio)
 		iomap_read_end(ctx->cur_folio, bytes_submitted);
@@ -663,8 +663,8 @@ void iomap_readahead(const struct iomap_ops *ops,
 		iter.status = iomap_readahead_iter(&iter, ctx,
 					&cur_bytes_submitted);
 
-	if (ctx->ops->submit_read)
-		ctx->ops->submit_read(ctx);
+	if (ctx->read_ctx && ctx->ops->submit_read)
+		ctx->ops->submit_read(&iter, ctx);
 
 	if (ctx->cur_folio)
 		iomap_read_end(ctx->cur_folio, cur_bytes_submitted);
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index e911daedff65a..b0a6549b38487 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2010 Red Hat, Inc.
  * Copyright (c) 2016-2025 Christoph Hellwig.
  */
+#include <linux/bio-integrity.h>
 #include <linux/blk-crypto.h>
 #include <linux/fscrypt.h>
 #include <linux/pagemap.h>
@@ -240,6 +241,9 @@ static void __iomap_dio_bio_end_io(struct bio *bio, bool inline_completion)
 {
 	struct iomap_dio *dio = bio->bi_private;
 
+	if (bio_integrity(bio))
+		fs_bio_integrity_free(bio);
+
 	if (dio->flags & IOMAP_DIO_BOUNCE) {
 		bio_iov_iter_unbounce(bio, !!dio->error,
 				dio->flags & IOMAP_DIO_USER_BACKED);
@@ -311,8 +315,7 @@ static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
 
 	bio = iomap_dio_alloc_bio(iter, dio, nr_vecs,
 				  REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);
-	fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
-				  GFP_KERNEL);
+	fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL);
 	bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
 	bio->bi_private = dio;
 	bio->bi_end_io = iomap_dio_bio_end_io;
@@ -342,16 +345,17 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 		nr_vecs = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
 
 	bio = iomap_dio_alloc_bio(iter, dio, nr_vecs, op);
-	fscrypt_set_bio_crypt_ctx(bio, iter->inode,
-			pos >> iter->inode->i_blkbits, GFP_KERNEL);
+	fscrypt_set_bio_crypt_ctx(bio, iter->inode, pos, GFP_KERNEL);
 	bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
 	bio->bi_write_hint = iter->inode->i_write_hint;
 	bio->bi_ioprio = dio->iocb->ki_ioprio;
 	bio->bi_private = dio;
 	bio->bi_end_io = iomap_dio_bio_end_io;
 
+
 	if (dio->flags & IOMAP_DIO_BOUNCE)
-		ret = bio_iov_iter_bounce(bio, dio->submit.iter);
+		ret = bio_iov_iter_bounce(bio, dio->submit.iter,
+				iomap_max_bio_size(&iter->iomap));
 	else
 		ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
 					     alignment - 1);
@@ -368,6 +372,13 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 		goto out_put_bio;
 	}
 
+	if (iter->iomap.flags & IOMAP_F_INTEGRITY) {
+		if (dio->flags & IOMAP_DIO_WRITE)
+			fs_bio_integrity_generate(bio);
+		else
+			fs_bio_integrity_alloc(bio);
+	}
+
 	if (dio->flags & IOMAP_DIO_WRITE)
 		task_io_account_write(ret);
 	else if ((dio->flags & IOMAP_DIO_USER_BACKED) &&
diff --git a/fs/iomap/internal.h b/fs/iomap/internal.h
index 3a4e4aad2bd12..74e898b196dc1 100644
--- a/fs/iomap/internal.h
+++ b/fs/iomap/internal.h
@@ -4,6 +4,20 @@
 
 #define IOEND_BATCH_SIZE	4096
 
+/*
+ * Normally we can build bios as big as the data structure supports.
+ *
+ * But for integrity protected I/O we need to respect the maximum size of the
+ * single contiguous allocation for the integrity buffer.
+ */
+static inline size_t iomap_max_bio_size(const struct iomap *iomap)
+{
+	if (iomap->flags & IOMAP_F_INTEGRITY)
+		return max_integrity_io_size(bdev_limits(iomap->bdev));
+	return BIO_MAX_SIZE;
+}
+
+u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend);
 u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend);
 
 #ifdef CONFIG_BLOCK
diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c
index 60546fa14dfe4..acf3cf98b23a9 100644
--- a/fs/iomap/ioend.c
+++ b/fs/iomap/ioend.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2016-2025 Christoph Hellwig.
  */
+#include <linux/bio-integrity.h>
 #include <linux/iomap.h>
 #include <linux/list_sort.h>
 #include <linux/pagemap.h>
@@ -37,7 +38,7 @@ EXPORT_SYMBOL_GPL(iomap_init_ioend);
  * state, release holds on bios, and finally free up memory.  Do not use the
  * ioend after this.
  */
-static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
+static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend)
 {
 	struct inode *inode = ioend->io_inode;
 	struct bio *bio = &ioend->io_bio;
@@ -48,7 +49,7 @@ static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
 		mapping_set_error(inode->i_mapping, ioend->io_error);
 		if (!bio_flagged(bio, BIO_QUIET)) {
 			pr_err_ratelimited(
-"%s: writeback error on inode %lu, offset %lld, sector %llu",
+"%s: writeback error on inode %llu, offset %lld, sector %llu",
 				inode->i_sb->s_id, inode->i_ino,
 				ioend->io_offset, ioend->io_sector);
 		}
@@ -65,6 +66,8 @@ static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
 		folio_count++;
 	}
 
+	if (bio_integrity(bio))
+		fs_bio_integrity_free(bio);
 	bio_put(bio);	/* frees the ioend */
 	return folio_count;
 }
@@ -87,7 +90,7 @@ iomap_fail_ioends(
 	while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
 			io_list))) {
 		list_del_init(&ioend->io_list);
-		iomap_finish_ioend_buffered(ioend);
+		iomap_finish_ioend_buffered_write(ioend);
 		cond_resched();
 	}
 }
@@ -120,7 +123,7 @@ static void ioend_writeback_end_bio(struct bio *bio)
 		return;
 	}
 
-	iomap_finish_ioend_buffered(ioend);
+	iomap_finish_ioend_buffered_write(ioend);
 }
 
 /*
@@ -144,6 +147,8 @@ int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
 		return error;
 	}
 
+	if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
+		fs_bio_integrity_generate(&ioend->io_bio);
 	submit_bio(&ioend->io_bio);
 	return 0;
 }
@@ -165,10 +170,13 @@ static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
 }
 
 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
-		u16 ioend_flags)
+		unsigned int map_len, u16 ioend_flags)
 {
 	struct iomap_ioend *ioend = wpc->wb_ctx;
 
+	if (ioend->io_bio.bi_iter.bi_size >
+	    iomap_max_bio_size(&wpc->iomap) - map_len)
+		return false;
 	if (ioend_flags & IOMAP_IOEND_BOUNDARY)
 		return false;
 	if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
@@ -234,7 +242,7 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
 	if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
 		ioend_flags |= IOMAP_IOEND_BOUNDARY;
 
-	if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
+	if (!ioend || !iomap_can_add_to_ioend(wpc, pos, map_len, ioend_flags)) {
 new_ioend:
 		if (ioend) {
 			error = wpc->ops->writeback_submit(wpc, 0);
@@ -311,9 +319,19 @@ static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
 
 	if (!atomic_dec_and_test(&ioend->io_remaining))
 		return 0;
+
+	if (!ioend->io_error &&
+	    bio_integrity(&ioend->io_bio) &&
+	    bio_op(&ioend->io_bio) == REQ_OP_READ) {
+		ioend->io_error = fs_bio_integrity_verify(&ioend->io_bio,
+			ioend->io_sector, ioend->io_size);
+	}
+
 	if (ioend->io_flags & IOMAP_IOEND_DIRECT)
 		return iomap_finish_ioend_direct(ioend);
-	return iomap_finish_ioend_buffered(ioend);
+	if (bio_op(&ioend->io_bio) == REQ_OP_READ)
+		return iomap_finish_ioend_buffered_read(ioend);
+	return iomap_finish_ioend_buffered_write(ioend);
 }
 
 /*
diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h
index 532787277b168..097773c6db80e 100644
--- a/fs/iomap/trace.h
+++ b/fs/iomap/trace.h
@@ -257,7 +257,7 @@ TRACE_EVENT(iomap_dio_rw_begin,
 	TP_ARGS(iocb, iter, dio_flags, done_before),
 	TP_STRUCT__entry(
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, isize)
 		__field(loff_t, pos)
 		__field(size_t,	count)
@@ -277,7 +277,7 @@ TRACE_EVENT(iomap_dio_rw_begin,
 		__entry->dio_flags = dio_flags;
 		__entry->aio = !is_sync_kiocb(iocb);
 	),
-	TP_printk("dev %d:%d ino 0x%lx size 0x%llx offset 0x%llx length 0x%zx done_before 0x%zx flags %s dio_flags %s aio %d",
+	TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx length 0x%zx done_before 0x%zx flags %s dio_flags %s aio %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->ino,
 		  __entry->isize,
@@ -294,7 +294,7 @@ TRACE_EVENT(iomap_dio_complete,
 	TP_ARGS(iocb, error, ret),
 	TP_STRUCT__entry(
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, isize)
 		__field(loff_t, pos)
 		__field(int,	ki_flags)
@@ -312,7 +312,7 @@ TRACE_EVENT(iomap_dio_complete,
 		__entry->error = error;
 		__entry->ret = ret;
 	),
-	TP_printk("dev %d:%d ino 0x%lx size 0x%llx offset 0x%llx flags %s aio %d error %d ret %zd",
+	TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx flags %s aio %d error %d ret %zd",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->ino,
 		  __entry->isize,
diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c
index 50b4cb3aea87c..397568b9c7e7d 100644
--- a/fs/isofs/compress.c
+++ b/fs/isofs/compress.c
@@ -156,7 +156,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 				else {
 					printk(KERN_DEBUG
 					       "zisofs: zisofs_inflate returned"
-					       " %d, inode = %lu,"
+					       " %d, inode = %llu,"
 					       " page idx = %d, bh idx = %d,"
 					       " avail_in = %ld,"
 					       " avail_out = %ld\n",
diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c
index 2ca16c3fe5ef3..2fd9948d606e9 100644
--- a/fs/isofs/dir.c
+++ b/fs/isofs/dir.c
@@ -152,7 +152,7 @@ static int do_isofs_readdir(struct inode *inode, struct file *file,
 		    de_len < de->name_len[0] +
 					sizeof(struct iso_directory_record)) {
 			printk(KERN_NOTICE "iso9660: Corrupted directory entry"
-			       " in block %lu of inode %lu\n", block,
+			       " in block %lu of inode %llu\n", block,
 			       inode->i_ino);
 			brelse(bh);
 			return -EIO;
diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c
index 5c01536c5e8fe..efee53717f1cd 100644
--- a/fs/isofs/inode.c
+++ b/fs/isofs/inode.c
@@ -1261,7 +1261,7 @@ static int isofs_read_level3_size(struct inode *inode)
 
 out_toomany:
 	printk(KERN_INFO "%s: More than 100 file sections ?!?, aborting...\n"
-		"isofs_read_level3_size: inode=%lu\n",
+		"isofs_read_level3_size: inode=%llu\n",
 		__func__, inode->i_ino);
 	goto out;
 }
@@ -1380,7 +1380,7 @@ static int isofs_read_inode(struct inode *inode, int relocated)
 	/* I have no idea what file_unit_size is used for, so
 	   we will flag it for now */
 	if (de->file_unit_size[0] != 0) {
-		printk(KERN_DEBUG "ISOFS: File unit size != 0 for ISO file (%ld).\n",
+		printk(KERN_DEBUG "ISOFS: File unit size != 0 for ISO file (%llu).\n",
 			inode->i_ino);
 	}
 
@@ -1389,7 +1389,7 @@ static int isofs_read_inode(struct inode *inode, int relocated)
 #ifdef DEBUG
 	if((de->flags[-high_sierra] & ~2)!= 0){
 		printk(KERN_DEBUG "ISOFS: Unusual flag settings for ISO file "
-				"(%ld %x).\n",
+				"(%llu %x).\n",
 			inode->i_ino, de->flags[-high_sierra]);
 	}
 #endif
@@ -1450,7 +1450,7 @@ static int isofs_read_inode(struct inode *inode, int relocated)
 		/* XXX - parse_rock_ridge_inode() had already set i_rdev. */
 		init_special_inode(inode, inode->i_mode, inode->i_rdev);
 	} else {
-		printk(KERN_DEBUG "ISOFS: Invalid file type 0%04o for inode %lu.\n",
+		printk(KERN_DEBUG "ISOFS: Invalid file type 0%04o for inode %llu.\n",
 			inode->i_mode, inode->i_ino);
 		ret = -EIO;
 		goto fail;
diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c
index 58f80e1b3ac0d..8dd3911717e0c 100644
--- a/fs/isofs/namei.c
+++ b/fs/isofs/namei.c
@@ -100,7 +100,7 @@ isofs_find_entry(struct inode *dir, struct dentry *dentry,
 		/* Basic sanity check, whether name doesn't exceed dir entry */
 		if (de_len < dlen + sizeof(struct iso_directory_record)) {
 			printk(KERN_NOTICE "iso9660: Corrupted directory entry"
-			       " in block %lu of inode %lu\n", block,
+			       " in block %lu of inode %llu\n", block,
 			       dir->i_ino);
 			brelse(bh);
 			return 0;
diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c
index cb2c529a8f1be..b60918ed8a996 100644
--- a/fs/jbd2/journal.c
+++ b/fs/jbd2/journal.c
@@ -1677,7 +1677,7 @@ journal_t *jbd2_journal_init_inode(struct inode *inode)
 		return err ? ERR_PTR(err) : ERR_PTR(-EINVAL);
 	}
 
-	jbd2_debug(1, "JBD2: inode %s/%ld, size %lld, bits %d, blksize %ld\n",
+	jbd2_debug(1, "JBD2: inode %s/%llu, size %lld, bits %d, blksize %ld\n",
 		  inode->i_sb->s_id, inode->i_ino, (long long) inode->i_size,
 		  inode->i_sb->s_blocksize_bits, inode->i_sb->s_blocksize);
 
@@ -1689,7 +1689,7 @@ journal_t *jbd2_journal_init_inode(struct inode *inode)
 
 	journal->j_inode = inode;
 	snprintf(journal->j_devname, sizeof(journal->j_devname),
-		 "%pg-%lu", journal->j_dev, journal->j_inode->i_ino);
+		 "%pg-%llu", journal->j_dev, journal->j_inode->i_ino);
 	strreplace(journal->j_devname, '/', '!');
 	jbd2_stats_proc_init(journal);
 
diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c
index dca4b5d8aaaa3..a90f9092706cc 100644
--- a/fs/jbd2/transaction.c
+++ b/fs/jbd2/transaction.c
@@ -2651,7 +2651,7 @@ static int jbd2_journal_file_inode(handle_t *handle, struct jbd2_inode *jinode,
 		return -EROFS;
 	journal = transaction->t_journal;
 
-	jbd2_debug(4, "Adding inode %lu, tid:%d\n", jinode->i_vfs_inode->i_ino,
+	jbd2_debug(4, "Adding inode %llu, tid:%d\n", jinode->i_vfs_inode->i_ino,
 			transaction->t_tid);
 
 	spin_lock(&journal->j_list_lock);
diff --git a/fs/jffs2/dir.c b/fs/jffs2/dir.c
index 2b38ce1fd8e8d..c4088c3b4ac03 100644
--- a/fs/jffs2/dir.c
+++ b/fs/jffs2/dir.c
@@ -129,7 +129,7 @@ static int jffs2_readdir(struct file *file, struct dir_context *ctx)
 	struct jffs2_full_dirent *fd;
 	unsigned long curofs = 1;
 
-	jffs2_dbg(1, "jffs2_readdir() for dir_i #%lu\n", inode->i_ino);
+	jffs2_dbg(1, "jffs2_readdir() for dir_i #%llu\n", inode->i_ino);
 
 	if (!dir_emit_dots(file, ctx))
 		return 0;
@@ -211,7 +211,7 @@ static int jffs2_create(struct mnt_idmap *idmap, struct inode *dir_i,
 
 	jffs2_free_raw_inode(ri);
 
-	jffs2_dbg(1, "%s(): Created ino #%lu with mode %o, nlink %d(%d). nrpages %ld\n",
+	jffs2_dbg(1, "%s(): Created ino #%llu with mode %o, nlink %d(%d). nrpages %ld\n",
 		  __func__, inode->i_ino, inode->i_mode, inode->i_nlink,
 		  f->inocache->pino_nlink, inode->i_mapping->nrpages);
 
diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c
index 5e1ef4bc009b6..1e18d3a798409 100644
--- a/fs/jffs2/file.c
+++ b/fs/jffs2/file.c
@@ -88,7 +88,7 @@ static int jffs2_do_readpage_nolock(struct inode *inode, struct folio *folio)
 	unsigned char *kaddr;
 	int ret;
 
-	jffs2_dbg(2, "%s(): ino #%lu, page at offset 0x%lx\n",
+	jffs2_dbg(2, "%s(): ino #%llu, page at offset 0x%lx\n",
 		  __func__, inode->i_ino, folio->index << PAGE_SHIFT);
 
 	BUG_ON(!folio_test_locked(folio));
@@ -259,7 +259,7 @@ static int jffs2_write_end(const struct kiocb *iocb,
 	uint32_t writtenlen = 0;
 	void *buf;
 
-	jffs2_dbg(1, "%s(): ino #%lu, page at 0x%llx, range %d-%d, flags %lx\n",
+	jffs2_dbg(1, "%s(): ino #%llu, page at 0x%llx, range %d-%d, flags %lx\n",
 		  __func__, inode->i_ino, folio_pos(folio),
 		  start, end, folio->flags.f);
 
diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c
index c3ce2c868f7a3..6ada8369a7622 100644
--- a/fs/jffs2/fs.c
+++ b/fs/jffs2/fs.c
@@ -43,7 +43,7 @@ int jffs2_do_setattr (struct inode *inode, struct iattr *iattr)
 	int ret;
 	int alloc_type = ALLOC_NORMAL;
 
-	jffs2_dbg(1, "%s(): ino #%lu\n", __func__, inode->i_ino);
+	jffs2_dbg(1, "%s(): ino #%llu\n", __func__, inode->i_ino);
 
 	/* Special cases - we don't want more than one data node
 	   for these types on the medium at any time. So setattr
@@ -243,7 +243,7 @@ void jffs2_evict_inode (struct inode *inode)
 	struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb);
 	struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode);
 
-	jffs2_dbg(1, "%s(): ino #%lu mode %o\n",
+	jffs2_dbg(1, "%s(): ino #%llu mode %o\n",
 		  __func__, inode->i_ino, inode->i_mode);
 	truncate_inode_pages_final(&inode->i_data);
 	clear_inode(inode);
@@ -334,8 +334,8 @@ struct inode *jffs2_iget(struct super_block *sb, unsigned long ino)
 		ret = jffs2_read_dnode(c, f, f->metadata, (char *)&jdev, 0, f->metadata->size);
 		if (ret < 0) {
 			/* Eep */
-			pr_notice("Read device numbers for inode %lu failed\n",
-				  (unsigned long)inode->i_ino);
+			pr_notice("Read device numbers for inode %llu failed\n",
+				  inode->i_ino);
 			goto error;
 		}
 		if (f->metadata->size == sizeof(jdev.old_id))
@@ -351,8 +351,8 @@ struct inode *jffs2_iget(struct super_block *sb, unsigned long ino)
 		break;
 
 	default:
-		pr_warn("%s(): Bogus i_mode %o for ino %lu\n",
-			__func__, inode->i_mode, (unsigned long)inode->i_ino);
+		pr_warn("%s(): Bogus i_mode %o for ino %llu\n",
+			__func__, inode->i_mode, inode->i_ino);
 	}
 
 	mutex_unlock(&f->sem);
@@ -374,12 +374,12 @@ void jffs2_dirty_inode(struct inode *inode, int flags)
 	struct iattr iattr;
 
 	if (!(inode_state_read_once(inode) & I_DIRTY_DATASYNC)) {
-		jffs2_dbg(2, "%s(): not calling setattr() for ino #%lu\n",
+		jffs2_dbg(2, "%s(): not calling setattr() for ino #%llu\n",
 			  __func__, inode->i_ino);
 		return;
 	}
 
-	jffs2_dbg(1, "%s(): calling setattr() for ino #%lu\n",
+	jffs2_dbg(1, "%s(): calling setattr() for ino #%llu\n",
 		  __func__, inode->i_ino);
 
 	iattr.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_ATIME|ATTR_MTIME|ATTR_CTIME;
@@ -428,7 +428,7 @@ struct inode *jffs2_new_inode (struct inode *dir_i, umode_t mode, struct jffs2_r
 	struct jffs2_inode_info *f;
 	int ret;
 
-	jffs2_dbg(1, "%s(): dir_i %ld, mode 0x%x\n",
+	jffs2_dbg(1, "%s(): dir_i %llu, mode 0x%x\n",
 		  __func__, dir_i->i_ino, mode);
 
 	c = JFFS2_SB_INFO(sb);
diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c
index 4709762713efc..c7914dbc91ed9 100644
--- a/fs/jfs/inode.c
+++ b/fs/jfs/inode.c
@@ -64,7 +64,7 @@ struct inode *jfs_iget(struct super_block *sb, unsigned long ino)
 		inode->i_op = &jfs_file_inode_operations;
 		init_special_inode(inode, inode->i_mode, inode->i_rdev);
 	} else {
-		printk(KERN_DEBUG "JFS: Invalid file type 0%04o for inode %lu.\n",
+		printk(KERN_DEBUG "JFS: Invalid file type 0%04o for inode %llu.\n",
 		       inode->i_mode, inode->i_ino);
 		iget_failed(inode);
 		return ERR_PTR(-EIO);
diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c
index 2abe8cc02ee6f..a841cf21da7de 100644
--- a/fs/jfs/jfs_dmap.c
+++ b/fs/jfs/jfs_dmap.c
@@ -133,6 +133,93 @@ static const s8 budtab[256] = {
 	2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, -1
 };
 
+/*
+ * check_dmapctl - Validate integrity of a dmapctl structure
+ * @dcp: Pointer to the dmapctl structure to check
+ *
+ * Return: true if valid, false if corrupted
+ */
+static bool check_dmapctl(struct dmapctl *dcp)
+{
+	s8 budmin = dcp->budmin;
+	u32 nleafs, l2nleafs, leafidx, height;
+	int i;
+
+	nleafs = le32_to_cpu(dcp->nleafs);
+	/* Check basic field ranges */
+	if (unlikely(nleafs > LPERCTL)) {
+		jfs_err("dmapctl: invalid nleafs %u (max %u)",
+			nleafs, LPERCTL);
+		return false;
+	}
+
+	l2nleafs = le32_to_cpu(dcp->l2nleafs);
+	if (unlikely(l2nleafs > L2LPERCTL)) {
+		jfs_err("dmapctl: invalid l2nleafs %u (max %u)",
+			l2nleafs, L2LPERCTL);
+		return false;
+	}
+
+	/* Verify nleafs matches l2nleafs (must be power of two) */
+	if (unlikely((1U << l2nleafs) != nleafs)) {
+		jfs_err("dmapctl: nleafs %u != 2^%u",
+			nleafs, l2nleafs);
+		return false;
+	}
+
+	leafidx = le32_to_cpu(dcp->leafidx);
+	/* Check leaf index matches expected position */
+	if (unlikely(leafidx != CTLLEAFIND)) {
+		jfs_err("dmapctl: invalid leafidx %u (expected %u)",
+			leafidx, CTLLEAFIND);
+		return false;
+	}
+
+	height = le32_to_cpu(dcp->height);
+	/* Check tree height is within valid range */
+	if (unlikely(height > (L2LPERCTL >> 1))) {
+		jfs_err("dmapctl: invalid height %u (max %u)",
+			height, L2LPERCTL >> 1);
+		return false;
+	}
+
+	/* Check budmin is valid (cannot be NOFREE for non-empty tree) */
+	if (budmin == NOFREE) {
+		if (unlikely(nleafs > 0)) {
+			jfs_err("dmapctl: budmin is NOFREE but nleafs %u",
+				nleafs);
+			return false;
+		}
+	} else if (unlikely(budmin < BUDMIN)) {
+		jfs_err("dmapctl: invalid budmin %d (min %d)",
+			budmin, BUDMIN);
+		return false;
+	}
+
+	/* Check leaf nodes fit within stree array */
+	if (unlikely(leafidx + nleafs > CTLTREESIZE)) {
+		jfs_err("dmapctl: leaf range exceeds stree size (end %u > %u)",
+			leafidx + nleafs, CTLTREESIZE);
+		return false;
+	}
+
+	/* Check leaf nodes have valid values */
+	for (i = leafidx; i < leafidx + nleafs; i++) {
+		s8 val = dcp->stree[i];
+
+		if (unlikely(val < NOFREE)) {
+			jfs_err("dmapctl: invalid leaf value %d at index %d",
+					val, i);
+			return false;
+		} else if (unlikely(val > 31)) {
+			jfs_err("dmapctl: leaf value %d too large at index %d", val, i);
+			return false;
+		}
+	}
+
+	return true;
+}
+
 /*
  * NAME:	dbMount()
  *
@@ -1372,7 +1459,7 @@ dbAllocAG(struct bmap * bmp, int agno, s64 nblocks, int l2nb, s64 * results)
 	dcp = (struct dmapctl *) mp->data;
 	budmin = dcp->budmin;
 
-	if (dcp->leafidx != cpu_to_le32(CTLLEAFIND)) {
+	if (unlikely(!check_dmapctl(dcp))) {
 		jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n");
 		release_metapage(mp);
 		return -EIO;
@@ -1702,7 +1789,7 @@ static int dbFindCtl(struct bmap * bmp, int l2nb, int level, s64 * blkno)
 		dcp = (struct dmapctl *) mp->data;
 		budmin = dcp->budmin;
 
-		if (dcp->leafidx != cpu_to_le32(CTLLEAFIND)) {
+		if (unlikely(!check_dmapctl(dcp))) {
 			jfs_error(bmp->db_ipbmap->i_sb,
 				  "Corrupt dmapctl page\n");
 			release_metapage(mp);
@@ -2485,7 +2572,7 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level)
 		return -EIO;
 	dcp = (struct dmapctl *) mp->data;
 
-	if (dcp->leafidx != cpu_to_le32(CTLLEAFIND)) {
+	if (unlikely(!check_dmapctl(dcp))) {
 		jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n");
 		release_metapage(mp);
 		return -EIO;
@@ -3454,6 +3541,11 @@ int dbExtendFS(struct inode *ipbmap, s64 blkno,	s64 nblocks)
 		return -EIO;
 	}
 	l2dcp = (struct dmapctl *) l2mp->data;
+	if (unlikely(!check_dmapctl(l2dcp))) {
+		jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n");
+		release_metapage(l2mp);
+		return -EIO;
+	}
 
 	/* compute start L1 */
 	k = blkno >> L2MAXL1SIZE;
@@ -3471,6 +3563,10 @@ int dbExtendFS(struct inode *ipbmap, s64 blkno,	s64 nblocks)
 			if (l1mp == NULL)
 				goto errout;
 			l1dcp = (struct dmapctl *) l1mp->data;
+			if (unlikely(!check_dmapctl(l1dcp))) {
+				jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n");
+				goto errout;
+			}
 
 			/* compute start L0 */
 			j = (blkno & (MAXL1SIZE - 1)) >> L2MAXL0SIZE;
@@ -3484,6 +3580,10 @@ int dbExtendFS(struct inode *ipbmap, s64 blkno,	s64 nblocks)
 				goto errout;
 
 			l1dcp = (struct dmapctl *) l1mp->data;
+			if (unlikely(!check_dmapctl(l1dcp))) {
+				jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n");
+				goto errout;
+			}
 
 			/* compute start L0 */
 			j = 0;
@@ -3503,6 +3603,10 @@ int dbExtendFS(struct inode *ipbmap, s64 blkno,	s64 nblocks)
 				if (l0mp == NULL)
 					goto errout;
 				l0dcp = (struct dmapctl *) l0mp->data;
+				if (unlikely(!check_dmapctl(l0dcp))) {
+					jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n");
+					goto errout;
+				}
 
 				/* compute start dmap */
 				i = (blkno & (MAXL0SIZE - 1)) >>
@@ -3518,6 +3622,10 @@ int dbExtendFS(struct inode *ipbmap, s64 blkno,	s64 nblocks)
 					goto errout;
 
 				l0dcp = (struct dmapctl *) l0mp->data;
+				if (unlikely(!check_dmapctl(l0dcp))) {
+					jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n");
+					goto errout;
+				}
 
 				/* compute start dmap */
 				i = 0;
diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c
index 9ab3f2fc61d17..ac0f79fafaca0 100644
--- a/fs/jfs/jfs_dtree.c
+++ b/fs/jfs/jfs_dtree.c
@@ -115,10 +115,7 @@ struct dtsplit {
 do {									\
 	BT_GETPAGE(IP, BN, MP, dtpage_t, SIZE, P, RC, i_dtroot);	\
 	if (!(RC)) {							\
-		if (((P)->header.nextindex >				\
-		     (((BN) == 0) ? DTROOTMAXSLOT : (P)->header.maxslot)) || \
-		    ((BN) && (((P)->header.maxslot > DTPAGEMAXSLOT) ||	\
-		    ((P)->header.stblindex >= DTPAGEMAXSLOT)))) {	\
+		if ((BN) && !check_dtpage(P)) {				\
 			BT_PUTPAGE(MP);					\
 			jfs_error((IP)->i_sb,				\
 				  "DT_GETPAGE: dtree page corrupt\n");	\
@@ -4297,3 +4294,190 @@ int dtModify(tid_t tid, struct inode *ip,
 
 	return 0;
 }
+
+bool check_dtroot(dtroot_t *p)
+{
+	DECLARE_BITMAP(bitmap, DTROOTMAXSLOT) = {0};
+	int i;
+
+	/* freecnt cannot be negative or exceed DTROOTMAXSLOT-1
+	 * (since slot[0] is occupied by the header).
+	 */
+	if (unlikely(p->header.freecnt < 0 ||
+				p->header.freecnt > DTROOTMAXSLOT - 1)) {
+		jfs_err("Bad freecnt:%d in dtroot\n", p->header.freecnt);
+		return false;
+	} else if (p->header.freecnt == 0) {
+		/* No free slots: freelist must be -1 */
+		if (unlikely(p->header.freelist != -1)) {
+			jfs_err("freecnt=0, but freelist=%d in dtroot\n",
+					p->header.freelist);
+			return false;
+		}
+	} else {
+		int fsi, i;
+		/* When there are free slots, freelist must be a valid slot index in
+		 * 1~DTROOTMAXSLOT-1(since slot[0] is occupied by the header).
+		 */
+		if (unlikely(p->header.freelist < 1 ||
+					p->header.freelist >= DTROOTMAXSLOT)) {
+			jfs_err("Bad freelist:%d in dtroot\n", p->header.freelist);
+			return false;
+		}
+
+		/* Traverse the free list to check validity of all node indices */
+		fsi = p->header.freelist;
+		for (i = 0; i < p->header.freecnt - 1; i++) {
+			/* Check for duplicate indices in the free list */
+			if (unlikely(__test_and_set_bit(fsi, bitmap))) {
+				jfs_err("duplicate index%d in slot in dtroot\n", fsi);
+				return false;
+			}
+			fsi = p->slot[fsi].next;
+
+			/* Ensure the next slot index in the free list is valid */
+			if (unlikely(fsi < 1 || fsi >= DTROOTMAXSLOT)) {
+				jfs_err("Bad index:%d in slot in dtroot\n", fsi);
+				return false;
+			}
+		}
+
+		/* The last node in the free list must terminate with next = -1 */
+		if (unlikely(p->slot[fsi].next != -1)) {
+			jfs_err("Bad next:%d of the last slot in dtroot\n",
+					p->slot[fsi].next);
+			return false;
+		}
+	}
+
+	/* Validate nextindex (next free entry index in stbl)
+	 * stbl array has size 8 (indices 0~7).
+	 * It may get set to 8 when the last free slot has been filled.
+	 */
+	if (unlikely(p->header.nextindex > ARRAY_SIZE(p->header.stbl))) {
+		jfs_err("Bad nextindex:%d in dtroot\n", p->header.nextindex);
+		return false;
+	}
+
+	/* Validate index validity of stbl array (8 elements)
+	 * Each entry in stbl is a slot index, with valid range: -1 (invalid)
+	 * or 0~8 (slot[0]~slot[8])
+	 */
+	for (i = 0; i < p->header.nextindex; i++) {
+		int idx = p->header.stbl[i];
+
+		if (unlikely(idx < 0 || idx >= 9)) {
+			jfs_err("Bad index:%d of stbl[%d] in dtroot\n", idx, i);
+			return false; /* stbl entry points out of slot array range */
+		}
+
+		/* Check for duplicate valid indices (skip check for idx=0) */
+		if (unlikely(idx && __test_and_set_bit(idx, bitmap))) {
+			jfs_err("Duplicate index:%d in stbl in dtroot\n", idx);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+bool check_dtpage(dtpage_t *p)
+{
+	DECLARE_BITMAP(bitmap, DTPAGEMAXSLOT) = {0};
+	const int stblsize = ((PSIZE >> L2DTSLOTSIZE) + 31) >> L2DTSLOTSIZE;
+	int i;
+
+	/* Validate maxslot (maximum number of slots in the page)
+	 * dtpage_t slot array is defined to hold up to DTPAGEMAXSLOT (128) slots
+	 */
+	if (unlikely(p->header.maxslot != DTPAGEMAXSLOT)) {
+		jfs_err("Bad maxslot:%d in dtpage (expected %d)\n",
+				p->header.maxslot, DTPAGEMAXSLOT);
+		return false;
+	}
+
+	/* freecnt cannot be negative or exceed DTPAGEMAXSLOT-1
+	 * (since slot[0] is occupied by the header).
+	 */
+	if (unlikely(p->header.freecnt < 0 ||
+				p->header.freecnt > DTPAGEMAXSLOT - 1)) {
+		jfs_err("Bad freecnt:%d in dtpage\n", p->header.freecnt);
+		return false;
+	} else if (p->header.freecnt == 0) {
+		/* No free slots: freelist must be -1 */
+		if (unlikely(p->header.freelist != -1)) {
+			jfs_err("freecnt=0 but freelist=%d in dtpage\n",
+					p->header.freelist);
+			return false;
+		}
+	} else {
+		int fsi;
+
+		if (unlikely(p->header.freelist < 1)) {
+			jfs_err("Bad freelist:%d in dtpage\n", p->header.freelist);
+			return false;
+		}
+
+		/* Traverse the free list to check validity of all node indices */
+		fsi = p->header.freelist;
+		for (i = 0; i < p->header.freecnt - 1; i++) {
+			/* Check for duplicate indices in the free list */
+			if (unlikely(__test_and_set_bit(fsi, bitmap))) {
+				jfs_err("duplicate index%d in slot in dtpage\n", fsi);
+				return false;
+			}
+			fsi = p->slot[fsi].next;
+
+			/* Ensure the next slot index in the free list is valid */
+			if (unlikely(fsi < 1 || fsi >= DTPAGEMAXSLOT)) {
+				jfs_err("Bad index:%d in slot in dtpage\n", fsi);
+				return false;
+			}
+		}
+
+		/* The last node in the free list must terminate with next = -1 */
+		if (unlikely(p->slot[fsi].next != -1)) {
+			jfs_err("Bad next:%d of the last slot in dtpage\n",
+					p->slot[fsi].next);
+			return false;
+		}
+	}
+
+	/* stbl must be little then DTPAGEMAXSLOT */
+	if (unlikely(p->header.stblindex >= DTPAGEMAXSLOT - stblsize)) {
+		jfs_err("Bad stblindex:%d in dtpage (stbl size %d)\n",
+				p->header.stblindex, stblsize);
+		return false;
+	}
+
+	/* nextindex must be little then stblsize*32 */
+	if (unlikely(p->header.nextindex > (stblsize << L2DTSLOTSIZE))) {
+		jfs_err("Bad nextindex:%d in dtpage (stbl size %d)\n",
+				p->header.nextindex, stblsize);
+		return false;
+	}
+
+	/* Validate stbl entries
+	 * Each entry is a slot index, valid range: -1 (invalid) or
+	 * [0, nextindex-1] (valid data slots)
+	 * (stblindex and higher slots are reserved for stbl itself)
+	 */
+	for (i = 0; i < p->header.nextindex; i++) {
+		int idx = DT_GETSTBL(p)[i];
+
+		/* Check if index is out of valid data slot range */
+		if (unlikely(idx < 1 || idx >= DTPAGEMAXSLOT)) {
+			jfs_err("Bad stbl[%d] index:%d (stblindex %d) in dtpage\n",
+					i, idx, p->header.stblindex);
+			return false;
+		}
+
+		/* Check for duplicate valid indices (skip -1) */
+		if (unlikely(__test_and_set_bit(idx, bitmap))) {
+			jfs_err("Duplicate index:%d in stbl of dtpage\n", idx);
+			return false;
+		}
+	}
+
+	return true;
+}
diff --git a/fs/jfs/jfs_dtree.h b/fs/jfs/jfs_dtree.h
index 1758289647a0e..dfc87b6690a9a 100644
--- a/fs/jfs/jfs_dtree.h
+++ b/fs/jfs/jfs_dtree.h
@@ -253,4 +253,8 @@ extern int dtModify(tid_t tid, struct inode *ip, struct component_name * key,
 		    ino_t * orig_ino, ino_t new_ino, int flag);
 
 extern int jfs_readdir(struct file *file, struct dir_context *ctx);
+
+extern bool check_dtroot(dtroot_t *p);
+
+extern bool check_dtpage(dtpage_t *p);
 #endif				/* !_H_JFS_DTREE */
diff --git a/fs/jfs/jfs_imap.c b/fs/jfs/jfs_imap.c
index 294a67327c735..b84ba4d7dfb44 100644
--- a/fs/jfs/jfs_imap.c
+++ b/fs/jfs/jfs_imap.c
@@ -302,7 +302,7 @@ int diRead(struct inode *ip)
 	unsigned long pageno;
 	int rel_inode;
 
-	jfs_info("diRead: ino = %ld", ip->i_ino);
+	jfs_info("diRead: ino = %llu", ip->i_ino);
 
 	ipimap = sbi->ipimap;
 	JFS_IP(ip)->ipimap = ipimap;
@@ -3102,6 +3102,10 @@ static int copy_from_dinode(struct dinode * dip, struct inode *ip)
 
 	if (S_ISDIR(ip->i_mode)) {
 		memcpy(&jfs_ip->u.dir, &dip->u._dir, 384);
+		if (!check_dtroot(&jfs_ip->i_dtroot)) {
+			jfs_error(ip->i_sb, "Corrupt dtroot\n");
+			return -EIO;
+		}
 	} else if (S_ISREG(ip->i_mode) || S_ISLNK(ip->i_mode)) {
 		memcpy(&jfs_ip->i_xtroot, &dip->di_xtroot, 288);
 	} else
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index ada00d5bc2146..306165e61438c 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -74,12 +74,6 @@ static struct lbuf *log_redrive_list;
 static DEFINE_SPINLOCK(log_redrive_lock);
 
 
-/*
- *	log read/write serialization (per log)
- */
-#define LOG_LOCK_INIT(log)	mutex_init(&(log)->loglock)
-#define LOG_LOCK(log)		mutex_lock(&((log)->loglock))
-#define LOG_UNLOCK(log)		mutex_unlock(&((log)->loglock))
 
 
 /*
@@ -204,9 +198,13 @@ static void write_special_inodes(struct jfs_log *log,
 	struct jfs_sb_info *sbi;
 
 	list_for_each_entry(sbi, &log->sb_list, log_list) {
-		writer(sbi->ipbmap->i_mapping);
-		writer(sbi->ipimap->i_mapping);
-		writer(sbi->direct_inode->i_mapping);
+		/* These pointers can be NULL before list_del during umount */
+		if (sbi->ipbmap)
+			writer(sbi->ipbmap->i_mapping);
+		if (sbi->ipimap)
+			writer(sbi->ipimap->i_mapping);
+		if (sbi->direct_inode)
+			writer(sbi->direct_inode->i_mapping);
 	}
 }
 
@@ -2180,8 +2178,6 @@ static void lbmIODone(struct bio *bio)
 
 	LCACHE_LOCK(flags);		/* disable+lock */
 
-	bp->l_flag |= lbmDONE;
-
 	if (bio->bi_status) {
 		bp->l_flag |= lbmERROR;
 
@@ -2196,12 +2192,10 @@ static void lbmIODone(struct bio *bio)
 	if (bp->l_flag & lbmREAD) {
 		bp->l_flag &= ~lbmREAD;
 
-		LCACHE_UNLOCK(flags);	/* unlock+enable */
-
 		/* wakeup I/O initiator */
 		LCACHE_WAKEUP(&bp->l_ioevent);
 
-		return;
+		goto out;
 	}
 
 	/*
@@ -2225,8 +2219,7 @@ static void lbmIODone(struct bio *bio)
 
 	if (bp->l_flag & lbmDIRECT) {
 		LCACHE_WAKEUP(&bp->l_ioevent);
-		LCACHE_UNLOCK(flags);
-		return;
+		goto out;
 	}
 
 	tail = log->wqueue;
@@ -2278,8 +2271,6 @@ static void lbmIODone(struct bio *bio)
 	 * leave buffer for i/o initiator to dispose
 	 */
 	if (bp->l_flag & lbmSYNC) {
-		LCACHE_UNLOCK(flags);	/* unlock+enable */
-
 		/* wakeup I/O initiator */
 		LCACHE_WAKEUP(&bp->l_ioevent);
 	}
@@ -2290,6 +2281,7 @@ static void lbmIODone(struct bio *bio)
 	else if (bp->l_flag & lbmGC) {
 		LCACHE_UNLOCK(flags);
 		lmPostGC(bp);
+		LCACHE_LOCK(flags);		/* disable+lock */
 	}
 
 	/*
@@ -2302,9 +2294,11 @@ static void lbmIODone(struct bio *bio)
 		assert(bp->l_flag & lbmRELEASE);
 		assert(bp->l_flag & lbmFREE);
 		lbmfree(bp);
-
-		LCACHE_UNLOCK(flags);	/* unlock+enable */
 	}
+
+out:
+	bp->l_flag |= lbmDONE;
+	LCACHE_UNLOCK(flags);
 }
 
 int jfsIOWait(void *arg)
diff --git a/fs/jfs/jfs_logmgr.h b/fs/jfs/jfs_logmgr.h
index 8b8994e48cd08..09e0ef6aeccef 100644
--- a/fs/jfs/jfs_logmgr.h
+++ b/fs/jfs/jfs_logmgr.h
@@ -402,6 +402,13 @@ struct jfs_log {
 	int no_integrity;	/* 3: flag to disable journaling to disk */
 };
 
+/*
+ * log read/write serialization (per log)
+ */
+#define LOG_LOCK_INIT(log)	mutex_init(&(log)->loglock)
+#define LOG_LOCK(log)		mutex_lock(&((log)->loglock))
+#define LOG_UNLOCK(log)		mutex_unlock(&((log)->loglock))
+
 /*
  * Log flag
  */
diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c
index 64c6eaa7f3f26..41fe12e641cee 100644
--- a/fs/jfs/jfs_metapage.c
+++ b/fs/jfs/jfs_metapage.c
@@ -270,6 +270,7 @@ static inline struct metapage *alloc_metapage(gfp_t gfp_mask)
 		mp->clsn = 0;
 		mp->log = NULL;
 		init_waitqueue_head(&mp->wait);
+		INIT_LIST_HEAD(&mp->synclist);
 	}
 	return mp;
 }
@@ -379,7 +380,7 @@ static void remove_from_logsync(struct metapage *mp)
 		mp->lsn = 0;
 		mp->clsn = 0;
 		log->count--;
-		list_del(&mp->synclist);
+		list_del_init(&mp->synclist);
 	}
 	LOGSYNC_UNLOCK(log, flags);
 }
@@ -692,7 +693,7 @@ struct metapage *__get_metapage(struct inode *inode, unsigned long lblock,
 	unsigned long page_index;
 	unsigned long page_offset;
 
-	jfs_info("__get_metapage: ino = %ld, lblock = 0x%lx, abs=%d",
+	jfs_info("__get_metapage: ino = %llu, lblock = 0x%lx, abs=%d",
 		 inode->i_ino, lblock, absolute);
 
 	l2bsize = inode->i_blkbits;
diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c
index 52e6b58c5dbd2..dac822f150701 100644
--- a/fs/jfs/jfs_mount.c
+++ b/fs/jfs/jfs_mount.c
@@ -378,11 +378,12 @@ static int chkSuper(struct super_block *sb)
 	sbi->nbperpage = PSIZE >> sbi->l2bsize;
 	sbi->l2nbperpage = L2PSIZE - sbi->l2bsize;
 	sbi->l2niperblk = sbi->l2bsize - L2DISIZE;
+	uuid_copy(&sbi->uuid, &j_sb->s_uuid);
+
 	if (sbi->mntflag & JFS_INLINELOG)
 		sbi->logpxd = j_sb->s_logpxd;
 	else {
 		sbi->logdev = new_decode_dev(le32_to_cpu(j_sb->s_logdev));
-		uuid_copy(&sbi->uuid, &j_sb->s_uuid);
 		uuid_copy(&sbi->loguuid, &j_sb->s_loguuid);
 	}
 	sbi->fsckpxd = j_sb->s_fsckpxd;
diff --git a/fs/jfs/jfs_txnmgr.c b/fs/jfs/jfs_txnmgr.c
index c16578af3a77e..083dbbb0c3268 100644
--- a/fs/jfs/jfs_txnmgr.c
+++ b/fs/jfs/jfs_txnmgr.c
@@ -275,6 +275,7 @@ int txInit(void)
 	for (k = 0; k < nTxBlock; k++) {
 		init_waitqueue_head(&TxBlock[k].gcwait);
 		init_waitqueue_head(&TxBlock[k].waitor);
+		INIT_LIST_HEAD(&TxBlock[k].synclist);
 	}
 
 	for (k = 1; k < nTxBlock - 1; k++) {
@@ -974,7 +975,7 @@ static void txUnlock(struct tblock * tblk)
 	if (tblk->lsn) {
 		LOGSYNC_LOCK(log, flags);
 		log->count--;
-		list_del(&tblk->synclist);
+		list_del_init(&tblk->synclist);
 		LOGSYNC_UNLOCK(log, flags);
 	}
 }
diff --git a/fs/jfs/jfs_umount.c b/fs/jfs/jfs_umount.c
index 8ec43f53f6865..18569f1eaabdb 100644
--- a/fs/jfs/jfs_umount.c
+++ b/fs/jfs/jfs_umount.c
@@ -20,6 +20,7 @@
 #include "jfs_superblock.h"
 #include "jfs_dmap.h"
 #include "jfs_imap.h"
+#include "jfs_logmgr.h"
 #include "jfs_metapage.h"
 #include "jfs_debug.h"
 
@@ -57,6 +58,12 @@ int jfs_umount(struct super_block *sb)
 		 */
 		jfs_flush_journal(log, 2);
 
+	/*
+	 * Hold log lock so write_special_inodes (lmLogSync) cannot see
+	 * this sbi with a NULL inode pointer while iterating log->sb_list.
+	 */
+	if (log)
+		LOG_LOCK(log);
 	/*
 	 * close fileset inode allocation map (aka fileset inode)
 	 */
@@ -95,6 +102,9 @@ int jfs_umount(struct super_block *sb)
 	 */
 	filemap_write_and_wait(sbi->direct_inode->i_mapping);
 
+	if (log)
+		LOG_UNLOCK(log);
+
 	/*
 	 * ensure all file system file pages are propagated to their
 	 * home blocks on disk (and their in-memory buffer pages are
diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c
index 8d40c4b1db9ff..b2ee03450226d 100644
--- a/fs/kernfs/dir.c
+++ b/fs/kernfs/dir.c
@@ -481,12 +481,14 @@ void kernfs_put_active(struct kernfs_node *kn)
 /**
  * kernfs_drain - drain kernfs_node
  * @kn: kernfs_node to drain
+ * @drop_supers: Set to true if this function is called with the
+ *               kernfs_supers_rwsem locked.
  *
  * Drain existing usages and nuke all existing mmaps of @kn.  Multiple
  * removers may invoke this function concurrently on @kn and all will
  * return after draining is complete.
  */
-static void kernfs_drain(struct kernfs_node *kn)
+static void kernfs_drain(struct kernfs_node *kn, bool drop_supers)
 	__releases(&kernfs_root(kn)->kernfs_rwsem)
 	__acquires(&kernfs_root(kn)->kernfs_rwsem)
 {
@@ -506,6 +508,8 @@ static void kernfs_drain(struct kernfs_node *kn)
 		return;
 
 	up_write(&root->kernfs_rwsem);
+	if (drop_supers)
+		up_read(&root->kernfs_supers_rwsem);
 
 	if (kernfs_lockdep(kn)) {
 		rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);
@@ -524,6 +528,8 @@ static void kernfs_drain(struct kernfs_node *kn)
 	if (kernfs_should_drain_open_files(kn))
 		kernfs_drain_open_files(kn);
 
+	if (drop_supers)
+		down_read(&root->kernfs_supers_rwsem);
 	down_write(&root->kernfs_rwsem);
 }
 
@@ -547,10 +553,8 @@ static void kernfs_free_rcu(struct rcu_head *rcu)
 	/* If the whole node goes away, then name can't be used outside */
 	kfree_const(rcu_access_pointer(kn->name));
 
-	if (kn->iattr) {
-		simple_xattrs_free(&kn->iattr->xattrs, NULL);
+	if (kn->iattr)
 		kmem_cache_free(kernfs_iattrs_cache, kn->iattr);
-	}
 
 	kmem_cache_free(kernfs_node_cache, kn);
 }
@@ -584,6 +588,12 @@ void kernfs_put(struct kernfs_node *kn)
 	if (kernfs_type(kn) == KERNFS_LINK)
 		kernfs_put(kn->symlink.target_kn);
 
+	if (kn->iattr && kn->iattr->xattrs) {
+		simple_xattrs_free(kn->iattr->xattrs, NULL);
+		kfree(kn->iattr->xattrs);
+		kn->iattr->xattrs = NULL;
+	}
+
 	spin_lock(&root->kernfs_idr_lock);
 	idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));
 	spin_unlock(&root->kernfs_idr_lock);
@@ -682,7 +692,10 @@ static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root,
 
  err_out4:
 	if (kn->iattr) {
-		simple_xattrs_free(&kn->iattr->xattrs, NULL);
+		if (kn->iattr->xattrs) {
+			simple_xattrs_free(kn->iattr->xattrs, NULL);
+			kfree(kn->iattr->xattrs);
+		}
 		kmem_cache_free(kernfs_iattrs_cache, kn->iattr);
 	}
  err_out3:
@@ -1465,12 +1478,43 @@ void kernfs_show(struct kernfs_node *kn, bool show)
 		kn->flags |= KERNFS_HIDDEN;
 		if (kernfs_active(kn))
 			atomic_add(KN_DEACTIVATED_BIAS, &kn->active);
-		kernfs_drain(kn);
+		kernfs_drain(kn, false);
 	}
 
 	up_write(&root->kernfs_rwsem);
 }
 
+/*
+ * This function enables VFS to send fsnotify events for deletions.
+ * There is gap in this implementation for certain file removals due their
+ * unique nature in kernfs. Directory removals that trigger file removals occur
+ * through vfs_rmdir, which shrinks the dcache and emits fsnotify events after
+ * the rmdir operation; there is no issue here. However kernfs writes to
+ * particular files (e.g. cgroup.subtree_control) can also cause file removal,
+ * but vfs_write does not attempt to emit fsnotify events after the write
+ * operation, even if i_nlink counts are 0. As a usecase for monitoring this
+ * category of file removals is not known, they are left without having
+ * IN_DELETE or IN_DELETE_SELF events generated.
+ * Fanotify recursive monitoring also does not work for kernfs nodes that do not
+ * have inodes attached, as they are created on-demand in kernfs.
+ */
+static void kernfs_clear_inode_nlink(struct kernfs_node *kn)
+{
+	struct kernfs_root *root = kernfs_root(kn);
+	struct kernfs_super_info *info;
+
+	lockdep_assert_held_read(&root->kernfs_supers_rwsem);
+
+	list_for_each_entry(info, &root->supers, node) {
+		struct inode *inode = ilookup(info->sb, kernfs_ino(kn));
+
+		if (inode) {
+			clear_nlink(inode);
+			iput(inode);
+		}
+	}
+}
+
 static void __kernfs_remove(struct kernfs_node *kn)
 {
 	struct kernfs_node *pos, *parent;
@@ -1479,6 +1523,7 @@ static void __kernfs_remove(struct kernfs_node *kn)
 	if (!kn)
 		return;
 
+	lockdep_assert_held_read(&kernfs_root(kn)->kernfs_supers_rwsem);
 	lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem);
 
 	/*
@@ -1491,12 +1536,14 @@ static void __kernfs_remove(struct kernfs_node *kn)
 	pr_debug("kernfs %s: removing\n", kernfs_rcu_name(kn));
 
 	/* prevent new usage by marking all nodes removing and deactivating */
+	down_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
 	pos = NULL;
 	while ((pos = kernfs_next_descendant_post(pos, kn))) {
 		pos->flags |= KERNFS_REMOVING;
 		if (kernfs_active(pos))
 			atomic_add(KN_DEACTIVATED_BIAS, &pos->active);
 	}
+	up_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
 
 	/* deactivate and unlink the subtree node-by-node */
 	do {
@@ -1510,7 +1557,7 @@ static void __kernfs_remove(struct kernfs_node *kn)
 		 */
 		kernfs_get(pos);
 
-		kernfs_drain(pos);
+		kernfs_drain(pos, true);
 		parent = kernfs_parent(pos);
 		/*
 		 * kernfs_unlink_sibling() succeeds once per node.  Use it
@@ -1520,9 +1567,11 @@ static void __kernfs_remove(struct kernfs_node *kn)
 			struct kernfs_iattrs *ps_iattr =
 				parent ? parent->iattr : NULL;
 
-			/* update timestamps on the parent */
 			down_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
 
+			kernfs_clear_inode_nlink(pos);
+
+			/* update timestamps on the parent */
 			if (ps_iattr) {
 				ktime_get_real_ts64(&ps_iattr->ia_ctime);
 				ps_iattr->ia_mtime = ps_iattr->ia_ctime;
@@ -1551,9 +1600,11 @@ void kernfs_remove(struct kernfs_node *kn)
 
 	root = kernfs_root(kn);
 
+	down_read(&root->kernfs_supers_rwsem);
 	down_write(&root->kernfs_rwsem);
 	__kernfs_remove(kn);
 	up_write(&root->kernfs_rwsem);
+	up_read(&root->kernfs_supers_rwsem);
 }
 
 /**
@@ -1644,6 +1695,7 @@ bool kernfs_remove_self(struct kernfs_node *kn)
 	bool ret;
 	struct kernfs_root *root = kernfs_root(kn);
 
+	down_read(&root->kernfs_supers_rwsem);
 	down_write(&root->kernfs_rwsem);
 	kernfs_break_active_protection(kn);
 
@@ -1673,7 +1725,9 @@ bool kernfs_remove_self(struct kernfs_node *kn)
 				break;
 
 			up_write(&root->kernfs_rwsem);
+			up_read(&root->kernfs_supers_rwsem);
 			schedule();
+			down_read(&root->kernfs_supers_rwsem);
 			down_write(&root->kernfs_rwsem);
 		}
 		finish_wait(waitq, &wait);
@@ -1688,6 +1742,7 @@ bool kernfs_remove_self(struct kernfs_node *kn)
 	kernfs_unbreak_active_protection(kn);
 
 	up_write(&root->kernfs_rwsem);
+	up_read(&root->kernfs_supers_rwsem);
 	return ret;
 }
 
@@ -1714,6 +1769,7 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
 	}
 
 	root = kernfs_root(parent);
+	down_read(&root->kernfs_supers_rwsem);
 	down_write(&root->kernfs_rwsem);
 
 	kn = kernfs_find_ns(parent, name, ns);
@@ -1724,6 +1780,7 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
 	}
 
 	up_write(&root->kernfs_rwsem);
+	up_read(&root->kernfs_supers_rwsem);
 
 	if (kn)
 		return 0;
diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c
index a36aaee98dcec..38b28aa7cd023 100644
--- a/fs/kernfs/inode.c
+++ b/fs/kernfs/inode.c
@@ -45,9 +45,7 @@ static struct kernfs_iattrs *__kernfs_iattrs(struct kernfs_node *kn, bool alloc)
 	ret->ia_mtime = ret->ia_atime;
 	ret->ia_ctime = ret->ia_atime;
 
-	simple_xattrs_init(&ret->xattrs);
-	atomic_set(&ret->nr_user_xattrs, 0);
-	atomic_set(&ret->user_xattr_size, 0);
+	simple_xattr_limits_init(&ret->xattr_limits);
 
 	/* If someone raced us, recognize it. */
 	if (!try_cmpxchg(&kn->iattr, &attr, ret))
@@ -146,7 +144,8 @@ ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size)
 	if (!attrs)
 		return -ENOMEM;
 
-	return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size);
+	return simple_xattr_list(d_inode(dentry), READ_ONCE(attrs->xattrs),
+				 buf, size);
 }
 
 static inline void set_default_inode_attr(struct inode *inode, umode_t mode)
@@ -178,7 +177,7 @@ static void kernfs_refresh_inode(struct kernfs_node *kn, struct inode *inode)
 		 */
 		set_inode_attr(inode, attrs);
 
-	if (kernfs_type(kn) == KERNFS_DIR)
+	if (kernfs_type(kn) == KERNFS_DIR && !(kn->flags & KERNFS_REMOVING))
 		set_nlink(inode, kn->dir.subdirs + 2);
 }
 
@@ -298,27 +297,38 @@ int kernfs_xattr_get(struct kernfs_node *kn, const char *name,
 		     void *value, size_t size)
 {
 	struct kernfs_iattrs *attrs = kernfs_iattrs_noalloc(kn);
+	struct simple_xattrs *xattrs;
+
 	if (!attrs)
 		return -ENODATA;
 
-	return simple_xattr_get(&attrs->xattrs, name, value, size);
+	xattrs = READ_ONCE(attrs->xattrs);
+	if (!xattrs)
+		return -ENODATA;
+
+	return simple_xattr_get(xattrs, name, value, size);
 }
 
 int kernfs_xattr_set(struct kernfs_node *kn, const char *name,
 		     const void *value, size_t size, int flags)
 {
 	struct simple_xattr *old_xattr;
+	struct simple_xattrs *xattrs;
 	struct kernfs_iattrs *attrs;
 
 	attrs = kernfs_iattrs(kn);
 	if (!attrs)
 		return -ENOMEM;
 
-	old_xattr = simple_xattr_set(&attrs->xattrs, name, value, size, flags);
+	xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags);
+	if (IS_ERR_OR_NULL(xattrs))
+		return PTR_ERR(xattrs);
+
+	old_xattr = simple_xattr_set(xattrs, name, value, size, flags);
 	if (IS_ERR(old_xattr))
 		return PTR_ERR(old_xattr);
 
-	simple_xattr_free(old_xattr);
+	simple_xattr_free_rcu(old_xattr);
 	return 0;
 }
 
@@ -344,69 +354,6 @@ static int kernfs_vfs_xattr_set(const struct xattr_handler *handler,
 	return kernfs_xattr_set(kn, name, value, size, flags);
 }
 
-static int kernfs_vfs_user_xattr_add(struct kernfs_node *kn,
-				     const char *full_name,
-				     struct simple_xattrs *xattrs,
-				     const void *value, size_t size, int flags)
-{
-	struct kernfs_iattrs *attr = kernfs_iattrs_noalloc(kn);
-	atomic_t *sz = &attr->user_xattr_size;
-	atomic_t *nr = &attr->nr_user_xattrs;
-	struct simple_xattr *old_xattr;
-	int ret;
-
-	if (atomic_inc_return(nr) > KERNFS_MAX_USER_XATTRS) {
-		ret = -ENOSPC;
-		goto dec_count_out;
-	}
-
-	if (atomic_add_return(size, sz) > KERNFS_USER_XATTR_SIZE_LIMIT) {
-		ret = -ENOSPC;
-		goto dec_size_out;
-	}
-
-	old_xattr = simple_xattr_set(xattrs, full_name, value, size, flags);
-	if (!old_xattr)
-		return 0;
-
-	if (IS_ERR(old_xattr)) {
-		ret = PTR_ERR(old_xattr);
-		goto dec_size_out;
-	}
-
-	ret = 0;
-	size = old_xattr->size;
-	simple_xattr_free(old_xattr);
-dec_size_out:
-	atomic_sub(size, sz);
-dec_count_out:
-	atomic_dec(nr);
-	return ret;
-}
-
-static int kernfs_vfs_user_xattr_rm(struct kernfs_node *kn,
-				    const char *full_name,
-				    struct simple_xattrs *xattrs,
-				    const void *value, size_t size, int flags)
-{
-	struct kernfs_iattrs *attr = kernfs_iattrs_noalloc(kn);
-	atomic_t *sz = &attr->user_xattr_size;
-	atomic_t *nr = &attr->nr_user_xattrs;
-	struct simple_xattr *old_xattr;
-
-	old_xattr = simple_xattr_set(xattrs, full_name, value, size, flags);
-	if (!old_xattr)
-		return 0;
-
-	if (IS_ERR(old_xattr))
-		return PTR_ERR(old_xattr);
-
-	atomic_sub(old_xattr->size, sz);
-	atomic_dec(nr);
-	simple_xattr_free(old_xattr);
-	return 0;
-}
-
 static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler,
 				     struct mnt_idmap *idmap,
 				     struct dentry *unused, struct inode *inode,
@@ -415,6 +362,7 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler,
 {
 	const char *full_name = xattr_full_name(handler, suffix);
 	struct kernfs_node *kn = inode->i_private;
+	struct simple_xattrs *xattrs;
 	struct kernfs_iattrs *attrs;
 
 	if (!(kernfs_root(kn)->flags & KERNFS_ROOT_SUPPORT_USER_XATTR))
@@ -424,13 +372,12 @@ static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler,
 	if (!attrs)
 		return -ENOMEM;
 
-	if (value)
-		return kernfs_vfs_user_xattr_add(kn, full_name, &attrs->xattrs,
-						 value, size, flags);
-	else
-		return kernfs_vfs_user_xattr_rm(kn, full_name, &attrs->xattrs,
-						value, size, flags);
+	xattrs = simple_xattrs_lazy_alloc(&attrs->xattrs, value, flags);
+	if (IS_ERR_OR_NULL(xattrs))
+		return PTR_ERR(xattrs);
 
+	return simple_xattr_set_limited(xattrs, &attrs->xattr_limits,
+					full_name, value, size, flags);
 }
 
 static const struct xattr_handler kernfs_trusted_xattr_handler = {
diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h
index 6061b6f70d2a3..1d3831e3a270c 100644
--- a/fs/kernfs/kernfs-internal.h
+++ b/fs/kernfs/kernfs-internal.h
@@ -26,9 +26,8 @@ struct kernfs_iattrs {
 	struct timespec64	ia_mtime;
 	struct timespec64	ia_ctime;
 
-	struct simple_xattrs	xattrs;
-	atomic_t		nr_user_xattrs;
-	atomic_t		user_xattr_size;
+	struct simple_xattrs	*xattrs;
+	struct simple_xattr_limits xattr_limits;
 };
 
 struct kernfs_root {
diff --git a/fs/libfs.c b/fs/libfs.c
index 74134ba2e8d1e..63b4fb082435f 100644
--- a/fs/libfs.c
+++ b/fs/libfs.c
@@ -2318,7 +2318,6 @@ EXPORT_SYMBOL(simple_start_creating);
 /* parent must have been held exclusive since simple_start_creating() */
 void simple_done_creating(struct dentry *child)
 {
-	inode_unlock(child->d_parent->d_inode);
-	dput(child);
+	end_creating(child);
 }
 EXPORT_SYMBOL(simple_done_creating);
diff --git a/fs/lockd/Makefile b/fs/lockd/Makefile
index 51bbe22d21e35..808f0f2a7be13 100644
--- a/fs/lockd/Makefile
+++ b/fs/lockd/Makefile
@@ -9,5 +9,33 @@ obj-$(CONFIG_LOCKD) += lockd.o
 
 lockd-y := clntlock.o clntproc.o clntxdr.o host.o svc.o svclock.o \
 	   svcshare.o svcproc.o svcsubs.o mon.o trace.o xdr.o netlink.o
-lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o xdr4.o svc4proc.o
+lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o svc4proc.o nlm4xdr_gen.o
 lockd-$(CONFIG_PROC_FS) += procfs.o
+
+#
+# XDR code generation (requires Python and additional packages)
+#
+# The generated *xdr_gen.{h,c} files are checked into git. Normal kernel
+# builds do not require the xdrgen tool or its Python dependencies.
+#
+# Developers modifying .x files in Documentation/sunrpc/xdr/ should run
+# "make xdrgen" to regenerate the affected files.
+#
+.PHONY: xdrgen
+
+XDRGEN			= ../../tools/net/sunrpc/xdrgen/xdrgen
+
+XDRGEN_DEFINITIONS	= ../../include/linux/sunrpc/xdrgen/nlm4.h
+XDRGEN_DECLARATIONS	= nlm4xdr_gen.h
+XDRGEN_SOURCE		= nlm4xdr_gen.c
+
+xdrgen: $(XDRGEN_DEFINITIONS) $(XDRGEN_DECLARATIONS) $(XDRGEN_SOURCE)
+
+../../include/linux/sunrpc/xdrgen/nlm4.h: ../../Documentation/sunrpc/xdr/nlm4.x
+	$(XDRGEN) definitions $< > $@
+
+nlm4xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm4.x
+	$(XDRGEN) declarations $< > $@
+
+nlm4xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm4.x
+	$(XDRGEN) source --peer server $< > $@
diff --git a/fs/lockd/clnt4xdr.c b/fs/lockd/clnt4xdr.c
index 527458db4525a..2058733eacf83 100644
--- a/fs/lockd/clnt4xdr.c
+++ b/fs/lockd/clnt4xdr.c
@@ -13,7 +13,8 @@
 #include <linux/sunrpc/xdr.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/stats.h>
-#include <linux/lockd/lockd.h>
+
+#include "lockd.h"
 
 #include <uapi/linux/nfs3.h>
 
@@ -284,7 +285,7 @@ static int decode_nlm4_holder(struct xdr_stream *xdr, struct nlm_res *result)
 	fl->c.flc_type  = exclusive != 0 ? F_WRLCK : F_RDLCK;
 	p = xdr_decode_hyper(p, &l_offset);
 	xdr_decode_hyper(p, &l_len);
-	nlm4svc_set_file_lock_range(fl, l_offset, l_len);
+	lockd_set_file_lock_range4(fl, l_offset, l_len);
 	error = 0;
 out:
 	return error;
diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c
index 85bc0f3e91dfe..8fa30c42c92a2 100644
--- a/fs/lockd/clntlock.c
+++ b/fs/lockd/clntlock.c
@@ -15,9 +15,9 @@
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/svc.h>
 #include <linux/sunrpc/svc_xprt.h>
-#include <linux/lockd/lockd.h>
 #include <linux/kthread.h>
 
+#include "lockd.h"
 #include "trace.h"
 
 #define NLMDBG_FACILITY		NLMDBG_CLIENT
diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c
index fb4d0752c9bbd..7f211008a5d2f 100644
--- a/fs/lockd/clntproc.c
+++ b/fs/lockd/clntproc.c
@@ -18,8 +18,8 @@
 #include <linux/freezer.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/svc.h>
-#include <linux/lockd/lockd.h>
 
+#include "lockd.h"
 #include "trace.h"
 
 #define NLMDBG_FACILITY		NLMDBG_CLIENT
diff --git a/fs/lockd/clntxdr.c b/fs/lockd/clntxdr.c
index 6ea3448d2d31e..65555f5224b14 100644
--- a/fs/lockd/clntxdr.c
+++ b/fs/lockd/clntxdr.c
@@ -15,7 +15,8 @@
 #include <linux/sunrpc/xdr.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/stats.h>
-#include <linux/lockd/lockd.h>
+
+#include "lockd.h"
 
 #include <uapi/linux/nfs2.h>
 
diff --git a/fs/lockd/host.c b/fs/lockd/host.c
index 1a9582a10a86f..ea8a8e166f7e4 100644
--- a/fs/lockd/host.c
+++ b/fs/lockd/host.c
@@ -16,13 +16,13 @@
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/svc.h>
-#include <linux/lockd/lockd.h>
 #include <linux/mutex.h>
 
 #include <linux/sunrpc/svc_xprt.h>
 
 #include <net/ipv6.h>
 
+#include "lockd.h"
 #include "netns.h"
 
 #define NLMDBG_FACILITY		NLMDBG_HOSTCACHE
@@ -306,6 +306,35 @@ void nlmclnt_release_host(struct nlm_host *host)
 	}
 }
 
+/* Callback for rpc_cancel_tasks() - matches all tasks for cancellation */
+static bool nlmclnt_match_all(const struct rpc_task *task, const void *data)
+{
+	return true;
+}
+
+/**
+ * nlmclnt_shutdown_rpc_clnt - safely shut down NLM client RPC operations
+ * @host: nlm_host to shut down
+ *
+ * Cancels outstanding RPC tasks and marks the client as shut down.
+ * Synchronizes with nlmclnt_release_host() via nlm_host_mutex to prevent
+ * races between shutdown and host destruction. Safe to call if h_rpcclnt
+ * is NULL or already shut down.
+ */
+void nlmclnt_shutdown_rpc_clnt(struct nlm_host *host)
+{
+	struct rpc_clnt *clnt;
+
+	mutex_lock(&nlm_host_mutex);
+	clnt = host->h_rpcclnt;
+	if (clnt) {
+		clnt->cl_shutdown = 1;
+		rpc_cancel_tasks(clnt, -EIO, nlmclnt_match_all, NULL);
+	}
+	mutex_unlock(&nlm_host_mutex);
+}
+EXPORT_SYMBOL_GPL(nlmclnt_shutdown_rpc_clnt);
+
 /**
  * nlmsvc_lookup_host - Find an NLM host handle matching a remote client
  * @rqstp: incoming NLM request
diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h
new file mode 100644
index 0000000000000..a7c85ab6d4b5e
--- /dev/null
+++ b/fs/lockd/lockd.h
@@ -0,0 +1,452 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
+ */
+
+#ifndef _LOCKD_LOCKD_H
+#define _LOCKD_LOCKD_H
+
+#include <linux/exportfs.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <net/ipv6.h>
+#include <linux/fs.h>
+#include <linux/kref.h>
+#include <linux/refcount.h>
+#include <linux/utsname.h>
+#include "nlm.h"
+#include <linux/lockd/bind.h>
+#include "xdr.h"
+#include <linux/sunrpc/debug.h>
+#include <linux/sunrpc/svc.h>
+
+/*
+ * Enable lockd debugging.
+ * Requires CONFIG_SUNRPC_DEBUG.
+ */
+#undef ifdebug
+#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
+# define ifdebug(flag)		if (unlikely(nlm_debug & NLMDBG_##flag))
+#else
+# define ifdebug(flag)		if (0)
+#endif
+
+#define NLMDBG_SVC		0x0001
+#define NLMDBG_CLIENT		0x0002
+#define NLMDBG_CLNTLOCK		0x0004
+#define NLMDBG_SVCLOCK		0x0008
+#define NLMDBG_MONITOR		0x0010
+#define NLMDBG_CLNTSUBS		0x0020
+#define NLMDBG_SVCSUBS		0x0040
+#define NLMDBG_HOSTCACHE	0x0080
+#define NLMDBG_XDR		0x0100
+#define NLMDBG_ALL		0x7fff
+
+/*
+ * Version string
+ */
+#define LOCKD_VERSION		"0.5"
+
+/*
+ * Default timeout for RPC calls (seconds)
+ */
+#define LOCKD_DFLT_TIMEO	10
+
+/* error codes new to NLMv4 */
+#define	nlm4_deadlock		cpu_to_be32(NLM_DEADLCK)
+#define	nlm4_rofs		cpu_to_be32(NLM_ROFS)
+#define	nlm4_stale_fh		cpu_to_be32(NLM_STALE_FH)
+#define	nlm4_fbig		cpu_to_be32(NLM_FBIG)
+#define	nlm4_failed		cpu_to_be32(NLM_FAILED)
+
+/*
+ * Internal-use status codes, not to be placed on the wire.
+ * Version handlers translate these to appropriate wire values.
+ */
+#define nlm__int__drop_reply	cpu_to_be32(30000)
+#define nlm__int__deadlock	cpu_to_be32(30001)
+#define nlm__int__stale_fh	cpu_to_be32(30002)
+#define nlm__int__failed	cpu_to_be32(30003)
+
+/*
+ * Lockd host handle (used both by the client and server personality).
+ */
+struct nlm_host {
+	struct hlist_node	h_hash;		/* doubly linked list */
+	struct sockaddr_storage	h_addr;		/* peer address */
+	size_t			h_addrlen;
+	struct sockaddr_storage	h_srcaddr;	/* our address (optional) */
+	size_t			h_srcaddrlen;
+	struct rpc_clnt		*h_rpcclnt;	/* RPC client to talk to peer */
+	char			*h_name;		/* remote hostname */
+	u32			h_version;	/* interface version */
+	unsigned short		h_proto;	/* transport proto */
+	unsigned short		h_reclaiming : 1,
+				h_server     : 1, /* server side, not client side */
+				h_noresvport : 1,
+				h_inuse      : 1;
+	wait_queue_head_t	h_gracewait;	/* wait while reclaiming */
+	struct rw_semaphore	h_rwsem;	/* Reboot recovery lock */
+	u32			h_state;	/* pseudo-state counter */
+	u32			h_nsmstate;	/* true remote NSM state */
+	u32			h_pidcount;	/* Pseudopids */
+	refcount_t		h_count;	/* reference count */
+	struct mutex		h_mutex;	/* mutex for pmap binding */
+	unsigned long		h_nextrebind;	/* next portmap call */
+	unsigned long		h_expires;	/* eligible for GC */
+	struct list_head	h_lockowners;	/* Lockowners for the client */
+	spinlock_t		h_lock;
+	struct list_head	h_granted;	/* Locks in GRANTED state */
+	struct list_head	h_reclaim;	/* Locks in RECLAIM state */
+	struct nsm_handle	*h_nsmhandle;	/* NSM status handle */
+	char			*h_addrbuf;	/* address eyecatcher */
+	struct net		*net;		/* host net */
+	const struct cred	*h_cred;
+	char			nodename[UNX_MAXNODENAME + 1];
+	const struct nlmclnt_operations	*h_nlmclnt_ops;	/* Callback ops for NLM users */
+};
+
+/*
+ * The largest string sm_addrbuf should hold is a full-size IPv6 address
+ * (no "::" anywhere) with a scope ID.  The buffer size is computed to
+ * hold eight groups of colon-separated four-hex-digit numbers, a
+ * percent sign, a scope id (at most 32 bits, in decimal), and NUL.
+ */
+#define NSM_ADDRBUF		((8 * 4 + 7) + (1 + 10) + 1)
+
+struct nsm_handle {
+	struct list_head	sm_link;
+	refcount_t		sm_count;
+	char			*sm_mon_name;
+	char			*sm_name;
+	struct sockaddr_storage	sm_addr;
+	size_t			sm_addrlen;
+	unsigned int		sm_monitored : 1,
+				sm_sticky : 1;	/* don't unmonitor */
+	struct nsm_private	sm_priv;
+	char			sm_addrbuf[NSM_ADDRBUF];
+};
+
+/*
+ * Rigorous type checking on sockaddr type conversions
+ */
+static inline struct sockaddr *nlm_addr(const struct nlm_host *host)
+{
+	return (struct sockaddr *)&host->h_addr;
+}
+
+static inline struct sockaddr *nlm_srcaddr(const struct nlm_host *host)
+{
+	return (struct sockaddr *)&host->h_srcaddr;
+}
+
+/*
+ * Map an fl_owner_t into a unique 32-bit "pid"
+ */
+struct nlm_lockowner {
+	struct list_head list;
+	refcount_t count;
+
+	struct nlm_host *host;
+	fl_owner_t owner;
+	uint32_t pid;
+};
+
+/*
+ * This is the representation of a blocked client lock.
+ */
+struct nlm_wait {
+	struct list_head	b_list;		/* linked list */
+	wait_queue_head_t	b_wait;		/* where to wait on */
+	struct nlm_host		*b_host;
+	struct file_lock	*b_lock;	/* local file lock */
+	__be32			b_status;	/* grant callback status */
+};
+
+/*
+ * Memory chunk for NLM client RPC request.
+ */
+#define NLMCLNT_OHSIZE		((__NEW_UTS_LEN) + 10u)
+struct nlm_rqst {
+	refcount_t		a_count;
+	unsigned int		a_flags;	/* initial RPC task flags */
+	struct nlm_host *	a_host;		/* host handle */
+	struct nlm_args		a_args;		/* arguments */
+	struct nlm_res		a_res;		/* result */
+	struct nlm_block *	a_block;
+	unsigned int		a_retries;	/* Retry count */
+	u8			a_owner[NLMCLNT_OHSIZE];
+	void *	a_callback_data; /* sent to nlmclnt_operations callbacks */
+};
+
+struct nlm_share;
+
+/*
+ * This struct describes a file held open by lockd on behalf of
+ * an NFS client.
+ */
+struct nlm_file {
+	struct hlist_node	f_list;		/* linked list */
+	struct nfs_fh		f_handle;	/* NFS file handle */
+	struct file *		f_file[2];	/* VFS file pointers,
+						   indexed by O_ flags */
+	struct nlm_share *	f_shares;	/* DOS shares */
+	struct list_head	f_blocks;	/* blocked locks */
+	unsigned int		f_locks;	/* guesstimate # of locks */
+	unsigned int		f_count;	/* reference count */
+	struct mutex		f_mutex;	/* avoid concurrent access */
+};
+
+/*
+ * This is a server block (i.e. a lock requested by some client which
+ * couldn't be granted because of a conflicting lock).
+ */
+#define NLM_NEVER		(~(unsigned long) 0)
+/* timeout on non-blocking call: */
+#define NLM_TIMEOUT		(7 * HZ)
+
+struct nlm_block {
+	struct kref		b_count;	/* Reference count */
+	struct list_head	b_list;		/* linked list of all blocks */
+	struct list_head	b_flist;	/* linked list (per file) */
+	struct nlm_rqst	*	b_call;		/* RPC args & callback info */
+	struct svc_serv *	b_daemon;	/* NLM service */
+	struct nlm_host *	b_host;		/* host handle for RPC clnt */
+	unsigned long		b_when;		/* next re-xmit */
+	unsigned int		b_id;		/* block id */
+	unsigned char		b_granted;	/* VFS granted lock */
+	struct nlm_file *	b_file;		/* file in question */
+	struct cache_req *	b_cache_req;	/* deferred request handling */
+	struct cache_deferred_req * b_deferred_req;
+	unsigned int		b_flags;	/* block flags */
+#define B_QUEUED		1	/* lock queued */
+#define B_GOT_CALLBACK		2	/* got lock or conflicting lock */
+#define B_TIMED_OUT		4	/* filesystem too slow to respond */
+};
+
+/*
+ * Global variables
+ */
+extern const struct rpc_program	nlm_program;
+extern const struct svc_version nlmsvc_version1;
+extern const struct svc_version nlmsvc_version3;
+#ifdef CONFIG_LOCKD_V4
+extern const struct svc_version nlmsvc_version4;
+#endif
+extern int			nlmsvc_grace_period;
+extern unsigned long		nlm_timeout;
+extern bool			nsm_use_hostnames;
+extern u32			nsm_local_state;
+
+extern struct timer_list	nlmsvc_retry;
+
+/*
+ * Lockd client functions
+ */
+struct nlm_rqst * nlm_alloc_call(struct nlm_host *host);
+int		  nlm_async_call(struct nlm_rqst *, u32, const struct rpc_call_ops *);
+int		  nlm_async_reply(struct nlm_rqst *, u32, const struct rpc_call_ops *);
+void		  nlmclnt_release_call(struct nlm_rqst *);
+void		  nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host,
+					struct file_lock *fl);
+void		  nlmclnt_queue_block(struct nlm_wait *block);
+__be32		  nlmclnt_dequeue_block(struct nlm_wait *block);
+int		  nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout);
+__be32		  nlmclnt_grant(const struct sockaddr *addr,
+				const struct nlm_lock *lock);
+void		  nlmclnt_recovery(struct nlm_host *);
+int		  nlmclnt_reclaim(struct nlm_host *, struct file_lock *,
+				  struct nlm_rqst *);
+void		  nlmclnt_next_cookie(struct nlm_cookie *);
+
+#ifdef CONFIG_LOCKD_V4
+extern const struct rpc_version nlm_version4;
+#endif
+
+/*
+ * Host cache
+ */
+struct nlm_host  *nlmclnt_lookup_host(const struct sockaddr *sap,
+					const size_t salen,
+					const unsigned short protocol,
+					const u32 version,
+					const char *hostname,
+					int noresvport,
+					struct net *net,
+					const struct cred *cred);
+void		  nlmclnt_release_host(struct nlm_host *);
+struct nlm_host  *nlmsvc_lookup_host(const struct svc_rqst *rqstp,
+					const char *hostname,
+					const size_t hostname_len);
+void		  nlmsvc_release_host(struct nlm_host *);
+struct rpc_clnt * nlm_bind_host(struct nlm_host *);
+void		  nlm_rebind_host(struct nlm_host *);
+struct nlm_host * nlm_get_host(struct nlm_host *);
+void		  nlm_shutdown_hosts(void);
+void		  nlm_shutdown_hosts_net(struct net *net);
+void		  nlm_host_rebooted(const struct net *net,
+					const struct nlm_reboot *);
+
+/*
+ * Host monitoring
+ */
+int		  nsm_monitor(const struct nlm_host *host);
+void		  nsm_unmonitor(const struct nlm_host *host);
+
+struct nsm_handle *nsm_get_handle(const struct net *net,
+					const struct sockaddr *sap,
+					const size_t salen,
+					const char *hostname,
+					const size_t hostname_len);
+struct nsm_handle *nsm_reboot_lookup(const struct net *net,
+					const struct nlm_reboot *info);
+void		  nsm_release(struct nsm_handle *nsm);
+
+/*
+ * This is used in garbage collection and resource reclaim
+ * A return value != 0 means destroy the lock/block/share
+ */
+typedef int	  (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref);
+
+/*
+ * Server-side lock handling
+ */
+int		  lock_to_openmode(struct file_lock *);
+__be32		  nlmsvc_lock(struct svc_rqst *, struct nlm_file *,
+			      struct nlm_host *, struct nlm_lock *, int,
+			      struct nlm_cookie *, int);
+__be32		  nlmsvc_unlock(struct net *net, struct nlm_file *, struct nlm_lock *);
+__be32		  nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file,
+			struct nlm_host *host, struct nlm_lock *lock,
+			struct nlm_lock *conflock);
+__be32		  nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct nlm_lock *);
+void		  nlmsvc_retry_blocked(struct svc_rqst *rqstp);
+void		  nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *,
+					nlm_host_match_fn_t match);
+void		  nlmsvc_grant_reply(struct nlm_cookie *, __be32);
+void		  nlmsvc_release_call(struct nlm_rqst *);
+void		  nlmsvc_locks_init_private(struct file_lock *, struct nlm_host *, pid_t);
+int		  nlmsvc_dispatch(struct svc_rqst *rqstp);
+
+/*
+ * File handling for the server personality
+ */
+__be32		  nlm_lookup_file(struct svc_rqst *, struct nlm_file **,
+					struct nlm_lock *);
+void		  nlm_release_file(struct nlm_file *);
+void		  nlmsvc_put_lockowner(struct nlm_lockowner *);
+void		  nlmsvc_release_lockowner(struct nlm_lock *);
+void		  nlmsvc_mark_resources(struct net *);
+void		  nlmsvc_free_host_resources(struct nlm_host *);
+void		  nlmsvc_invalidate_all(void);
+
+static inline struct file *nlmsvc_file_file(const struct nlm_file *file)
+{
+	return file->f_file[O_RDONLY] ?
+	       file->f_file[O_RDONLY] : file->f_file[O_WRONLY];
+}
+
+static inline struct inode *nlmsvc_file_inode(struct nlm_file *file)
+{
+	return file_inode(nlmsvc_file_file(file));
+}
+
+static inline bool
+nlmsvc_file_cannot_lock(const struct nlm_file *file)
+{
+	return exportfs_cannot_lock(nlmsvc_file_file(file)->f_path.dentry->d_sb->s_export_op);
+}
+
+static inline int __nlm_privileged_request4(const struct sockaddr *sap)
+{
+	const struct sockaddr_in *sin = (struct sockaddr_in *)sap;
+
+	if (ntohs(sin->sin_port) > 1023)
+		return 0;
+
+	return ipv4_is_loopback(sin->sin_addr.s_addr);
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static inline int __nlm_privileged_request6(const struct sockaddr *sap)
+{
+	const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap;
+
+	if (ntohs(sin6->sin6_port) > 1023)
+		return 0;
+
+	if (ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_MAPPED)
+		return ipv4_is_loopback(sin6->sin6_addr.s6_addr32[3]);
+
+	return ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LOOPBACK;
+}
+#else	/* IS_ENABLED(CONFIG_IPV6) */
+static inline int __nlm_privileged_request6(const struct sockaddr *sap)
+{
+	return 0;
+}
+#endif	/* IS_ENABLED(CONFIG_IPV6) */
+
+/*
+ * Ensure incoming requests are from local privileged callers.
+ *
+ * Return TRUE if sender is local and is connecting via a privileged port;
+ * otherwise return FALSE.
+ */
+static inline int nlm_privileged_requester(const struct svc_rqst *rqstp)
+{
+	const struct sockaddr *sap = svc_addr(rqstp);
+
+	switch (sap->sa_family) {
+	case AF_INET:
+		return __nlm_privileged_request4(sap);
+	case AF_INET6:
+		return __nlm_privileged_request6(sap);
+	default:
+		return 0;
+	}
+}
+
+/*
+ * Compare two NLM locks.
+ * When the second lock is of type F_UNLCK, this acts like a wildcard.
+ */
+static inline int nlm_compare_locks(const struct file_lock *fl1,
+				    const struct file_lock *fl2)
+{
+	return file_inode(fl1->c.flc_file) == file_inode(fl2->c.flc_file)
+	     && fl1->c.flc_pid   == fl2->c.flc_pid
+	     && fl1->c.flc_owner == fl2->c.flc_owner
+	     && fl1->fl_start == fl2->fl_start
+	     && fl1->fl_end   == fl2->fl_end
+	     &&(fl1->c.flc_type  == fl2->c.flc_type || fl2->c.flc_type == F_UNLCK);
+}
+
+/**
+ * lockd_set_file_lock_range4 - set the byte range of a file_lock
+ * @fl: file_lock whose length fields are to be initialized
+ * @off: starting offset of the lock, in bytes
+ * @len: length of the byte range, in bytes, or zero
+ *
+ * The NLMv4 protocol represents lock byte ranges as (start, length),
+ * where length zero means "lock to end of file." The kernel's file_lock
+ * structure uses (start, end) representation. Convert from NLMv4 format
+ * to file_lock format, clamping the starting offset and treating
+ * arithmetic overflow as "lock to EOF."
+ */
+static inline void
+lockd_set_file_lock_range4(struct file_lock *fl, u64 off, u64 len)
+{
+	u64 clamped_off = (off > OFFSET_MAX) ? OFFSET_MAX : off;
+	s64 end = clamped_off + len - 1;
+
+	fl->fl_start = clamped_off;
+	if (len == 0 || end < 0)
+		fl->fl_end = OFFSET_MAX;
+	else
+		fl->fl_end = end;
+}
+
+extern const struct lock_manager_operations nlmsvc_lock_operations;
+
+#endif /* _LOCKD_LOCKD_H */
diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c
index b8fc732e1c677..3d3ee88ca4dc2 100644
--- a/fs/lockd/mon.c
+++ b/fs/lockd/mon.c
@@ -16,10 +16,10 @@
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/xprtsock.h>
 #include <linux/sunrpc/svc.h>
-#include <linux/lockd/lockd.h>
 
 #include <linux/unaligned.h>
 
+#include "lockd.h"
 #include "netns.h"
 
 #define NLMDBG_FACILITY		NLMDBG_MONITOR
diff --git a/fs/lockd/nlm.h b/fs/lockd/nlm.h
new file mode 100644
index 0000000000000..47be65d0111fd
--- /dev/null
+++ b/fs/lockd/nlm.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Declarations for the Network Lock Manager protocol.
+ *
+ * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
+ */
+
+#ifndef _LOCKD_NLM_H
+#define _LOCKD_NLM_H
+
+
+/* Maximum file offset in file_lock.fl_end */
+# define NLM_OFFSET_MAX		((s32) 0x7fffffff)
+# define NLM4_OFFSET_MAX	((s64) ((~(u64)0) >> 1))
+
+/* Return states for NLM */
+enum {
+	NLM_LCK_GRANTED			= 0,
+	NLM_LCK_DENIED			= 1,
+	NLM_LCK_DENIED_NOLOCKS		= 2,
+	NLM_LCK_BLOCKED			= 3,
+	NLM_LCK_DENIED_GRACE_PERIOD	= 4,
+#ifdef CONFIG_LOCKD_V4
+	NLM_DEADLCK			= 5,
+	NLM_ROFS			= 6,
+	NLM_STALE_FH			= 7,
+	NLM_FBIG			= 8,
+	NLM_FAILED			= 9,
+#endif
+};
+
+#define NLM_PROGRAM		100021
+
+#define NLMPROC_NULL		0
+#define NLMPROC_TEST		1
+#define NLMPROC_LOCK		2
+#define NLMPROC_CANCEL		3
+#define NLMPROC_UNLOCK		4
+#define NLMPROC_GRANTED		5
+#define NLMPROC_TEST_MSG	6
+#define NLMPROC_LOCK_MSG	7
+#define NLMPROC_CANCEL_MSG	8
+#define NLMPROC_UNLOCK_MSG	9
+#define NLMPROC_GRANTED_MSG	10
+#define NLMPROC_TEST_RES	11
+#define NLMPROC_LOCK_RES	12
+#define NLMPROC_CANCEL_RES	13
+#define NLMPROC_UNLOCK_RES	14
+#define NLMPROC_GRANTED_RES	15
+#define NLMPROC_NSM_NOTIFY	16		/* statd callback */
+#define NLMPROC_SHARE		20
+#define NLMPROC_UNSHARE		21
+#define NLMPROC_NM_LOCK		22
+#define NLMPROC_FREE_ALL	23
+
+#endif /* _LOCKD_NLM_H */
diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c
new file mode 100644
index 0000000000000..1c8c221db456c
--- /dev/null
+++ b/fs/lockd/nlm4xdr_gen.c
@@ -0,0 +1,724 @@
+// SPDX-License-Identifier: GPL-2.0
+// Generated by xdrgen. Manual edits will be lost.
+// XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x
+// XDR specification modification time: Thu Dec 25 13:10:19 2025
+
+#include <linux/sunrpc/svc.h>
+
+#include "nlm4xdr_gen.h"
+
+static bool __maybe_unused
+xdrgen_decode_netobj(struct xdr_stream *xdr, netobj *ptr)
+{
+	return xdrgen_decode_opaque(xdr, ptr, MAXNETOBJ_SZ);
+}
+
+static bool __maybe_unused
+xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr)
+{
+	u32 val;
+
+	if (xdr_stream_decode_u32(xdr, &val) < 0)
+		return false;
+	*ptr = val;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr)
+{
+	u32 val;
+
+	if (xdr_stream_decode_u32(xdr, &val) < 0)
+		return false;
+	*ptr = val;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_uint64(struct xdr_stream *xdr, uint64 *ptr)
+{
+	return xdrgen_decode_unsigned_hyper(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_int64(struct xdr_stream *xdr, int64 *ptr)
+{
+	return xdrgen_decode_hyper(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_uint32(struct xdr_stream *xdr, uint32 *ptr)
+{
+	return xdrgen_decode_unsigned_long(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr)
+{
+	return xdrgen_decode_long(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr)
+{
+	return xdr_stream_decode_be32(xdr, ptr) == 0;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_holder(struct xdr_stream *xdr, struct nlm4_holder *ptr)
+{
+	if (!xdrgen_decode_bool(xdr, &ptr->exclusive))
+		return false;
+	if (!xdrgen_decode_int32(xdr, &ptr->svid))
+		return false;
+	if (!xdrgen_decode_netobj(xdr, &ptr->oh))
+		return false;
+	if (!xdrgen_decode_uint64(xdr, &ptr->l_offset))
+		return false;
+	if (!xdrgen_decode_uint64(xdr, &ptr->l_len))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_testrply(struct xdr_stream *xdr, struct nlm4_testrply *ptr)
+{
+	if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat))
+		return false;
+	switch (ptr->stat) {
+	case __constant_cpu_to_be32(NLM4_DENIED):
+		if (!xdrgen_decode_nlm4_holder(xdr, &ptr->u.holder))
+			return false;
+		break;
+	default:
+		break;
+	}
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_stat(struct xdr_stream *xdr, struct nlm4_stat *ptr)
+{
+	if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_res(struct xdr_stream *xdr, struct nlm4_res *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_nlm4_stat(xdr, &ptr->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_testres(struct xdr_stream *xdr, struct nlm4_testres *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_nlm4_testrply(xdr, &ptr->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_lock(struct xdr_stream *xdr, struct nlm4_lock *ptr)
+{
+	if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN))
+		return false;
+	if (!xdrgen_decode_netobj(xdr, &ptr->fh))
+		return false;
+	if (!xdrgen_decode_netobj(xdr, &ptr->oh))
+		return false;
+	if (!xdrgen_decode_int32(xdr, &ptr->svid))
+		return false;
+	if (!xdrgen_decode_uint64(xdr, &ptr->l_offset))
+		return false;
+	if (!xdrgen_decode_uint64(xdr, &ptr->l_len))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_lockargs(struct xdr_stream *xdr, struct nlm4_lockargs *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->block))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->exclusive))
+		return false;
+	if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->reclaim))
+		return false;
+	if (!xdrgen_decode_int32(xdr, &ptr->state))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_cancargs(struct xdr_stream *xdr, struct nlm4_cancargs *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->block))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->exclusive))
+		return false;
+	if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_testargs(struct xdr_stream *xdr, struct nlm4_testargs *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->exclusive))
+		return false;
+	if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_unlockargs(struct xdr_stream *xdr, struct nlm4_unlockargs *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_share(struct xdr_stream *xdr, struct nlm4_share *ptr)
+{
+	if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN))
+		return false;
+	if (!xdrgen_decode_netobj(xdr, &ptr->fh))
+		return false;
+	if (!xdrgen_decode_netobj(xdr, &ptr->oh))
+		return false;
+	if (!xdrgen_decode_fsh4_mode(xdr, &ptr->mode))
+		return false;
+	if (!xdrgen_decode_fsh4_access(xdr, &ptr->access))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_shareargs(struct xdr_stream *xdr, struct nlm4_shareargs *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_nlm4_share(xdr, &ptr->share))
+		return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->reclaim))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_shareres(struct xdr_stream *xdr, struct nlm4_shareres *ptr)
+{
+	if (!xdrgen_decode_netobj(xdr, &ptr->cookie))
+		return false;
+	if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat))
+		return false;
+	if (!xdrgen_decode_int32(xdr, &ptr->sequence))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_notify(struct xdr_stream *xdr, struct nlm4_notify *ptr)
+{
+	if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXNAMELEN))
+		return false;
+	if (!xdrgen_decode_int32(xdr, &ptr->state))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nlm4_notifyargs(struct xdr_stream *xdr, struct nlm4_notifyargs *ptr)
+{
+	if (!xdrgen_decode_nlm4_notify(xdr, &ptr->notify))
+		return false;
+	if (xdr_stream_decode_opaque_fixed(xdr, ptr->private, SM_PRIV_SIZE) < 0)
+		return false;
+	return true;
+}
+
+/**
+ * nlm4_svc_decode_void - Decode a void argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	return xdrgen_decode_void(xdr);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_testargs - Decode a nlm4_testargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_testargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_testargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_lockargs - Decode a nlm4_lockargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_lockargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_lockargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_cancargs - Decode a nlm4_cancargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_cancargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_cancargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_unlockargs - Decode a nlm4_unlockargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_unlockargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_unlockargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_testres - Decode a nlm4_testres argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_testres *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_testres(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_res - Decode a nlm4_res argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_res *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_res(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_notifyargs - Decode a nlm4_notifyargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_notifyargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_notifyargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_shareargs - Decode a nlm4_shareargs argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_shareargs *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_shareargs(xdr, argp);
+}
+
+/**
+ * nlm4_svc_decode_nlm4_notify - Decode a nlm4_notify argument
+ * @rqstp: RPC transaction context
+ * @xdr: source XDR data stream
+ *
+ * Return values:
+ *   %true: procedure arguments decoded successfully
+ *   %false: decode failed
+ */
+bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_notify *argp = rqstp->rq_argp;
+
+	return xdrgen_decode_nlm4_notify(xdr, argp);
+}
+
+static bool __maybe_unused
+xdrgen_encode_netobj(struct xdr_stream *xdr, const netobj value)
+{
+	return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0;
+}
+
+static bool __maybe_unused
+xdrgen_encode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode value)
+{
+	return xdr_stream_encode_u32(xdr, value) == XDR_UNIT;
+}
+
+static bool __maybe_unused
+xdrgen_encode_fsh4_access(struct xdr_stream *xdr, fsh4_access value)
+{
+	return xdr_stream_encode_u32(xdr, value) == XDR_UNIT;
+}
+
+static bool __maybe_unused
+xdrgen_encode_uint64(struct xdr_stream *xdr, const uint64 value)
+{
+	return xdrgen_encode_unsigned_hyper(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_int64(struct xdr_stream *xdr, const int64 value)
+{
+	return xdrgen_encode_hyper(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_uint32(struct xdr_stream *xdr, const uint32 value)
+{
+	return xdrgen_encode_unsigned_long(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_int32(struct xdr_stream *xdr, const int32 value)
+{
+	return xdrgen_encode_long(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats value)
+{
+	return xdr_stream_encode_be32(xdr, value) == XDR_UNIT;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_holder(struct xdr_stream *xdr, const struct nlm4_holder *value)
+{
+	if (!xdrgen_encode_bool(xdr, value->exclusive))
+		return false;
+	if (!xdrgen_encode_int32(xdr, value->svid))
+		return false;
+	if (!xdrgen_encode_netobj(xdr, value->oh))
+		return false;
+	if (!xdrgen_encode_uint64(xdr, value->l_offset))
+		return false;
+	if (!xdrgen_encode_uint64(xdr, value->l_len))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_testrply(struct xdr_stream *xdr, const struct nlm4_testrply *ptr)
+{
+	if (!xdrgen_encode_nlm4_stats(xdr, ptr->stat))
+		return false;
+	switch (ptr->stat) {
+	case __constant_cpu_to_be32(NLM4_DENIED):
+		if (!xdrgen_encode_nlm4_holder(xdr, &ptr->u.holder))
+			return false;
+		break;
+	default:
+		break;
+	}
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_stat(struct xdr_stream *xdr, const struct nlm4_stat *value)
+{
+	if (!xdrgen_encode_nlm4_stats(xdr, value->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_res(struct xdr_stream *xdr, const struct nlm4_res *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_nlm4_stat(xdr, &value->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_testres(struct xdr_stream *xdr, const struct nlm4_testres *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_nlm4_testrply(xdr, &value->stat))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_lock(struct xdr_stream *xdr, const struct nlm4_lock *value)
+{
+	if (value->caller_name.len > LM_MAXSTRLEN)
+		return false;
+	if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0)
+		return false;
+	if (!xdrgen_encode_netobj(xdr, value->fh))
+		return false;
+	if (!xdrgen_encode_netobj(xdr, value->oh))
+		return false;
+	if (!xdrgen_encode_int32(xdr, value->svid))
+		return false;
+	if (!xdrgen_encode_uint64(xdr, value->l_offset))
+		return false;
+	if (!xdrgen_encode_uint64(xdr, value->l_len))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_lockargs(struct xdr_stream *xdr, const struct nlm4_lockargs *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->block))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->exclusive))
+		return false;
+	if (!xdrgen_encode_nlm4_lock(xdr, &value->alock))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->reclaim))
+		return false;
+	if (!xdrgen_encode_int32(xdr, value->state))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_cancargs(struct xdr_stream *xdr, const struct nlm4_cancargs *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->block))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->exclusive))
+		return false;
+	if (!xdrgen_encode_nlm4_lock(xdr, &value->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_testargs(struct xdr_stream *xdr, const struct nlm4_testargs *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->exclusive))
+		return false;
+	if (!xdrgen_encode_nlm4_lock(xdr, &value->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_unlockargs(struct xdr_stream *xdr, const struct nlm4_unlockargs *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_nlm4_lock(xdr, &value->alock))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_share(struct xdr_stream *xdr, const struct nlm4_share *value)
+{
+	if (value->caller_name.len > LM_MAXSTRLEN)
+		return false;
+	if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0)
+		return false;
+	if (!xdrgen_encode_netobj(xdr, value->fh))
+		return false;
+	if (!xdrgen_encode_netobj(xdr, value->oh))
+		return false;
+	if (!xdrgen_encode_fsh4_mode(xdr, value->mode))
+		return false;
+	if (!xdrgen_encode_fsh4_access(xdr, value->access))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_shareargs(struct xdr_stream *xdr, const struct nlm4_shareargs *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_nlm4_share(xdr, &value->share))
+		return false;
+	if (!xdrgen_encode_bool(xdr, value->reclaim))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_shareres(struct xdr_stream *xdr, const struct nlm4_shareres *value)
+{
+	if (!xdrgen_encode_netobj(xdr, value->cookie))
+		return false;
+	if (!xdrgen_encode_nlm4_stats(xdr, value->stat))
+		return false;
+	if (!xdrgen_encode_int32(xdr, value->sequence))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_notify(struct xdr_stream *xdr, const struct nlm4_notify *value)
+{
+	if (value->name.len > LM_MAXNAMELEN)
+		return false;
+	if (xdr_stream_encode_opaque(xdr, value->name.data, value->name.len) < 0)
+		return false;
+	if (!xdrgen_encode_int32(xdr, value->state))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nlm4_notifyargs(struct xdr_stream *xdr, const struct nlm4_notifyargs *value)
+{
+	if (!xdrgen_encode_nlm4_notify(xdr, &value->notify))
+		return false;
+	if (xdr_stream_encode_opaque_fixed(xdr, value->private, SM_PRIV_SIZE) < 0)
+		return false;
+	return true;
+}
+
+/**
+ * nlm4_svc_encode_void - Encode a void result
+ * @rqstp: RPC transaction context
+ * @xdr: target XDR data stream
+ *
+ * Return values:
+ *   %true: procedure results encoded successfully
+ *   %false: encode failed
+ */
+bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	return xdrgen_encode_void(xdr);
+}
+
+/**
+ * nlm4_svc_encode_nlm4_testres - Encode a nlm4_testres result
+ * @rqstp: RPC transaction context
+ * @xdr: target XDR data stream
+ *
+ * Return values:
+ *   %true: procedure results encoded successfully
+ *   %false: encode failed
+ */
+bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_testres *resp = rqstp->rq_resp;
+
+	return xdrgen_encode_nlm4_testres(xdr, resp);
+}
+
+/**
+ * nlm4_svc_encode_nlm4_res - Encode a nlm4_res result
+ * @rqstp: RPC transaction context
+ * @xdr: target XDR data stream
+ *
+ * Return values:
+ *   %true: procedure results encoded successfully
+ *   %false: encode failed
+ */
+bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_res *resp = rqstp->rq_resp;
+
+	return xdrgen_encode_nlm4_res(xdr, resp);
+}
+
+/**
+ * nlm4_svc_encode_nlm4_shareres - Encode a nlm4_shareres result
+ * @rqstp: RPC transaction context
+ * @xdr: target XDR data stream
+ *
+ * Return values:
+ *   %true: procedure results encoded successfully
+ *   %false: encode failed
+ */
+bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
+{
+	struct nlm4_shareres *resp = rqstp->rq_resp;
+
+	return xdrgen_encode_nlm4_shareres(xdr, resp);
+}
diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h
new file mode 100644
index 0000000000000..b6008b296a3e3
--- /dev/null
+++ b/fs/lockd/nlm4xdr_gen.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Generated by xdrgen. Manual edits will be lost. */
+/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */
+/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */
+
+#ifndef _LINUX_XDRGEN_NLM4_DECL_H
+#define _LINUX_XDRGEN_NLM4_DECL_H
+
+#include <linux/types.h>
+
+#include <linux/sunrpc/xdr.h>
+#include <linux/sunrpc/xdrgen/_defs.h>
+#include <linux/sunrpc/xdrgen/_builtins.h>
+#include <linux/sunrpc/xdrgen/nlm4.h>
+
+bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+
+bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+
+#endif /* _LINUX_XDRGEN_NLM4_DECL_H */
diff --git a/fs/lockd/share.h b/fs/lockd/share.h
new file mode 100644
index 0000000000000..20ea8ee491682
--- /dev/null
+++ b/fs/lockd/share.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * DOS share management for lockd.
+ *
+ * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
+ */
+
+#ifndef _LOCKD_SHARE_H
+#define _LOCKD_SHARE_H
+
+/* Synthetic svid for lockowner lookup during share operations */
+#define LOCKD_SHARE_SVID	(~(u32)0)
+
+/*
+ * DOS share for a specific file
+ */
+struct nlm_share {
+	struct nlm_share *	s_next;		/* linked list */
+	struct nlm_host *	s_host;		/* client host */
+	struct nlm_file *	s_file;		/* shared file */
+	struct xdr_netobj	s_owner;	/* owner handle */
+	u32			s_access;	/* access mode */
+	u32			s_mode;		/* deny mode */
+};
+
+__be32	nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file,
+			  struct xdr_netobj *oh, u32 access, u32 mode);
+__be32	nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file,
+			    struct xdr_netobj *oh);
+void	nlmsvc_traverse_shares(struct nlm_host *, struct nlm_file *,
+					       nlm_host_match_fn_t);
+
+#endif /* _LOCKD_SHARE_H */
diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index dcd80c4e74c94..490551369ef2a 100644
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -36,15 +36,14 @@
 #include <net/ip.h>
 #include <net/addrconf.h>
 #include <net/ipv6.h>
-#include <linux/lockd/lockd.h>
 #include <linux/nfs.h>
 
+#include "lockd.h"
 #include "netns.h"
 #include "procfs.h"
 #include "netlink.h"
 
 #define NLMDBG_FACILITY		NLMDBG_SVC
-#define LOCKD_BUFSIZE		(1024 + NLMSVC_XDRSIZE)
 
 static struct svc_program	nlmsvc_program;
 
@@ -319,6 +318,7 @@ static struct notifier_block lockd_inet6addr_notifier = {
 static int lockd_get(void)
 {
 	struct svc_serv *serv;
+	unsigned int bufsize;
 	int error;
 
 	if (nlmsvc_serv) {
@@ -334,7 +334,15 @@ static int lockd_get(void)
 		printk(KERN_WARNING
 			"lockd_up: no pid, %d users??\n", nlmsvc_users);
 
-	serv = svc_create(&nlmsvc_program, LOCKD_BUFSIZE, lockd);
+#ifdef CONFIG_LOCKD_V4
+	bufsize = 1024 + max3(nlmsvc_version1.vs_xdrsize,
+			      nlmsvc_version3.vs_xdrsize,
+			      nlmsvc_version4.vs_xdrsize);
+#else
+	bufsize = 1024 + max(nlmsvc_version1.vs_xdrsize,
+			     nlmsvc_version3.vs_xdrsize);
+#endif
+	serv = svc_create(&nlmsvc_program, bufsize, lockd);
 	if (!serv) {
 		printk(KERN_WARNING "lockd_up: create service failed\n");
 		return -ENOMEM;
@@ -640,7 +648,7 @@ module_exit(exit_nlm);
  *  %0: Processing complete; do not send a Reply
  *  %1: Processing complete; send Reply in rqstp->rq_res
  */
-static int nlmsvc_dispatch(struct svc_rqst *rqstp)
+int nlmsvc_dispatch(struct svc_rqst *rqstp)
 {
 	const struct svc_procedure *procp = rqstp->rq_procinfo;
 	__be32 *statp = rqstp->rq_accept_statp;
@@ -671,40 +679,6 @@ static int nlmsvc_dispatch(struct svc_rqst *rqstp)
 /*
  * Define NLM program and procedures
  */
-static DEFINE_PER_CPU_ALIGNED(unsigned long, nlmsvc_version1_count[17]);
-static const struct svc_version	nlmsvc_version1 = {
-	.vs_vers	= 1,
-	.vs_nproc	= 17,
-	.vs_proc	= nlmsvc_procedures,
-	.vs_count	= nlmsvc_version1_count,
-	.vs_dispatch	= nlmsvc_dispatch,
-	.vs_xdrsize	= NLMSVC_XDRSIZE,
-};
-
-static DEFINE_PER_CPU_ALIGNED(unsigned long,
-			      nlmsvc_version3_count[ARRAY_SIZE(nlmsvc_procedures)]);
-static const struct svc_version	nlmsvc_version3 = {
-	.vs_vers	= 3,
-	.vs_nproc	= ARRAY_SIZE(nlmsvc_procedures),
-	.vs_proc	= nlmsvc_procedures,
-	.vs_count	= nlmsvc_version3_count,
-	.vs_dispatch	= nlmsvc_dispatch,
-	.vs_xdrsize	= NLMSVC_XDRSIZE,
-};
-
-#ifdef CONFIG_LOCKD_V4
-static DEFINE_PER_CPU_ALIGNED(unsigned long,
-			      nlmsvc_version4_count[ARRAY_SIZE(nlmsvc_procedures4)]);
-static const struct svc_version	nlmsvc_version4 = {
-	.vs_vers	= 4,
-	.vs_nproc	= ARRAY_SIZE(nlmsvc_procedures4),
-	.vs_proc	= nlmsvc_procedures4,
-	.vs_count	= nlmsvc_version4_count,
-	.vs_dispatch	= nlmsvc_dispatch,
-	.vs_xdrsize	= NLMSVC_XDRSIZE,
-};
-#endif
-
 static const struct svc_version *nlmsvc_version[] = {
 	[1] = &nlmsvc_version1,
 	[3] = &nlmsvc_version3,
diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c
index 4b6f18d977343..5de41e2495341 100644
--- a/fs/lockd/svc4proc.c
+++ b/fs/lockd/svc4proc.c
@@ -10,257 +10,528 @@
 
 #include <linux/types.h>
 #include <linux/time.h>
-#include <linux/lockd/lockd.h>
-#include <linux/lockd/share.h>
 #include <linux/sunrpc/svc_xprt.h>
 
-#define NLMDBG_FACILITY		NLMDBG_CLIENT
+#include "lockd.h"
 
 /*
- * Obtain client and file from arguments
+ * xdr.h defines SM_MAXSTRLEN and SM_PRIV_SIZE as macros.
+ * nlm4xdr_gen.h defines them as enum constants. Undefine the
+ * macros to allow the xdrgen enum definitions to be used.
  */
-static __be32
-nlm4svc_retrieve_args(struct svc_rqst *rqstp, struct nlm_args *argp,
-			struct nlm_host **hostp, struct nlm_file **filp)
-{
-	struct nlm_host		*host = NULL;
-	struct nlm_file		*file = NULL;
-	struct nlm_lock		*lock = &argp->lock;
-	__be32			error = 0;
+#undef SM_MAXSTRLEN
+#undef SM_PRIV_SIZE
 
-	/* nfsd callbacks must have been installed for this procedure */
-	if (!nlmsvc_ops)
-		return nlm_lck_denied_nolocks;
+#include "share.h"
+#include "nlm4xdr_gen.h"
 
-	if (lock->lock_start > OFFSET_MAX ||
-	    (lock->lock_len && ((lock->lock_len - 1) > (OFFSET_MAX - lock->lock_start))))
-		return nlm4_fbig;
+/*
+ * Wrapper structures combine xdrgen types with legacy nlm_lock.
+ * The xdrgen field must be first so the structure can be cast
+ * to its XDR type for the RPC dispatch layer.
+ */
+struct nlm4_testargs_wrapper {
+	struct nlm4_testargs		xdrgen;
+	struct nlm_lock			lock;
+};
 
-	/* Obtain host handle */
-	if (!(host = nlmsvc_lookup_host(rqstp, lock->caller, lock->len))
-	 || (argp->monitor && nsm_monitor(host) < 0))
-		goto no_locks;
-	*hostp = host;
-
-	/* Obtain file pointer. Not used by FREE_ALL call. */
-	if (filp != NULL) {
-		int mode = lock_to_openmode(&lock->fl);
-
-		lock->fl.c.flc_flags = FL_POSIX;
-
-		error = nlm_lookup_file(rqstp, &file, lock);
-		if (error)
-			goto no_locks;
-		*filp = file;
-
-		/* Set up the missing parts of the file_lock structure */
-		lock->fl.c.flc_file = file->f_file[mode];
-		lock->fl.c.flc_pid = current->tgid;
-		lock->fl.fl_start = (loff_t)lock->lock_start;
-		lock->fl.fl_end = lock->lock_len ?
-				   (loff_t)(lock->lock_start + lock->lock_len - 1) :
-				   OFFSET_MAX;
-		lock->fl.fl_lmops = &nlmsvc_lock_operations;
-		nlmsvc_locks_init_private(&lock->fl, host, (pid_t)lock->svid);
-		if (!lock->fl.c.flc_owner) {
-			/* lockowner allocation has failed */
-			nlmsvc_release_host(host);
-			return nlm_lck_denied_nolocks;
-		}
-	}
+static_assert(offsetof(struct nlm4_testargs_wrapper, xdrgen) == 0);
 
-	return 0;
+struct nlm4_lockargs_wrapper {
+	struct nlm4_lockargs		xdrgen;
+	struct nlm_cookie		cookie;
+	struct nlm_lock			lock;
+};
 
-no_locks:
-	nlmsvc_release_host(host);
- 	if (error)
-		return error;	
-	return nlm_lck_denied_nolocks;
-}
+static_assert(offsetof(struct nlm4_lockargs_wrapper, xdrgen) == 0);
 
-/*
- * NULL: Test for presence of service
- */
-static __be32
-nlm4svc_proc_null(struct svc_rqst *rqstp)
-{
-	dprintk("lockd: NULL          called\n");
-	return rpc_success;
-}
+struct nlm4_cancargs_wrapper {
+	struct nlm4_cancargs		xdrgen;
+	struct nlm_lock			lock;
+};
 
-/*
- * TEST: Check for conflicting lock
- */
-static __be32
-__nlm4svc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_host	*host;
-	struct nlm_file	*file;
-	__be32 rc = rpc_success;
+static_assert(offsetof(struct nlm4_cancargs_wrapper, xdrgen) == 0);
 
-	dprintk("lockd: TEST4        called\n");
-	resp->cookie = argp->cookie;
+struct nlm4_unlockargs_wrapper {
+	struct nlm4_unlockargs		xdrgen;
+	struct nlm_lock			lock;
+};
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+static_assert(offsetof(struct nlm4_unlockargs_wrapper, xdrgen) == 0);
 
-	/* Now check for conflicting locks */
-	resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock,
-				       &resp->lock);
-	if (resp->status == nlm_drop_reply)
-		rc = rpc_drop_reply;
-	else
-		dprintk("lockd: TEST4        status %d\n", ntohl(resp->status));
+struct nlm4_notifyargs_wrapper {
+	struct nlm4_notifyargs		xdrgen;
+	struct nlm_reboot		reboot;
+};
 
-	nlmsvc_release_lockowner(&argp->lock);
-	nlmsvc_release_host(host);
-	nlm_release_file(file);
-	return rc;
-}
+static_assert(offsetof(struct nlm4_notifyargs_wrapper, xdrgen) == 0);
 
-static __be32
-nlm4svc_proc_test(struct svc_rqst *rqstp)
-{
-	return __nlm4svc_proc_test(rqstp, rqstp->rq_resp);
-}
+struct nlm4_notify_wrapper {
+	struct nlm4_notify		xdrgen;
+};
 
-static __be32
-__nlm4svc_proc_lock(struct svc_rqst *rqstp, struct nlm_res *resp)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_host	*host;
-	struct nlm_file	*file;
-	__be32 rc = rpc_success;
+static_assert(offsetof(struct nlm4_notify_wrapper, xdrgen) == 0);
 
-	dprintk("lockd: LOCK          called\n");
+struct nlm4_testres_wrapper {
+	struct nlm4_testres		xdrgen;
+	struct nlm_lock			lock;
+};
 
-	resp->cookie = argp->cookie;
+struct nlm4_shareargs_wrapper {
+	struct nlm4_shareargs		xdrgen;
+	struct nlm_lock			lock;
+};
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+static_assert(offsetof(struct nlm4_shareargs_wrapper, xdrgen) == 0);
 
-	/* Now try to lock the file */
-	resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock,
-					argp->block, &argp->cookie,
-					argp->reclaim);
-	if (resp->status == nlm_drop_reply)
-		rc = rpc_drop_reply;
-	else
-		dprintk("lockd: LOCK         status %d\n", ntohl(resp->status));
+static_assert(offsetof(struct nlm4_testres_wrapper, xdrgen) == 0);
 
-	nlmsvc_release_lockowner(&argp->lock);
-	nlmsvc_release_host(host);
-	nlm_release_file(file);
-	return rc;
+struct nlm4_res_wrapper {
+	struct nlm4_res			xdrgen;
+	struct nlm_cookie		cookie;
+};
+
+static_assert(offsetof(struct nlm4_res_wrapper, xdrgen) == 0);
+
+struct nlm4_shareres_wrapper {
+	struct nlm4_shareres		xdrgen;
+};
+
+static_assert(offsetof(struct nlm4_shareres_wrapper, xdrgen) == 0);
+
+static __be32
+nlm4_netobj_to_cookie(struct nlm_cookie *cookie, netobj *object)
+{
+	if (object->len > NLM_MAXCOOKIELEN)
+		return nlm_lck_denied_nolocks;
+	cookie->len = object->len;
+	memcpy(cookie->data, object->data, object->len);
+	return nlm_granted;
 }
 
 static __be32
-nlm4svc_proc_lock(struct svc_rqst *rqstp)
+nlm4_lock_to_nlm_lock(struct nlm_lock *lock, struct nlm4_lock *alock)
 {
-	return __nlm4svc_proc_lock(rqstp, rqstp->rq_resp);
+	if (alock->fh.len > NFS_MAXFHSIZE)
+		return nlm_lck_denied;
+	lock->fh.size = alock->fh.len;
+	memcpy(lock->fh.data, alock->fh.data, alock->fh.len);
+	lock->oh.len = alock->oh.len;
+	lock->oh.data = alock->oh.data;
+	lock->svid = alock->svid;
+	locks_init_lock(&lock->fl);
+	lockd_set_file_lock_range4(&lock->fl, alock->l_offset, alock->l_len);
+	return nlm_granted;
+}
+
+static struct nlm_host *
+nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored)
+{
+	struct nlm_host *host;
+
+	if (!nlmsvc_ops)
+		return NULL;
+	host = nlmsvc_lookup_host(rqstp, caller.data, caller.len);
+	if (!host)
+		return NULL;
+	if (monitored && nsm_monitor(host) < 0) {
+		nlmsvc_release_host(host);
+		return NULL;
+	}
+	return host;
 }
 
 static __be32
-__nlm4svc_proc_cancel(struct svc_rqst *rqstp, struct nlm_res *resp)
+nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host,
+		    struct nlm_lock *lock, struct nlm_file **filp,
+		    struct nlm4_lock *xdr_lock, unsigned char type)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_host	*host;
-	struct nlm_file	*file;
+	struct file_lock *fl = &lock->fl;
+	struct nlm_file *file = NULL;
+	__be32 error;
 
-	dprintk("lockd: CANCEL        called\n");
+	if (xdr_lock->fh.len > NFS_MAXFHSIZE)
+		return nlm_lck_denied_nolocks;
+	lock->fh.size = xdr_lock->fh.len;
+	memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len);
 
-	resp->cookie = argp->cookie;
+	lock->oh.len = xdr_lock->oh.len;
+	lock->oh.data = xdr_lock->oh.data;
 
-	/* Don't accept requests during grace period */
-	if (locks_in_grace(SVC_NET(rqstp))) {
-		resp->status = nlm_lck_denied_grace_period;
-		return rpc_success;
-	}
+	lock->svid = xdr_lock->svid;
+	lock->lock_start = xdr_lock->l_offset;
+	lock->lock_len = xdr_lock->l_len;
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+	if (lock->lock_start > OFFSET_MAX ||
+	    (lock->lock_len && ((lock->lock_len - 1) > (OFFSET_MAX - lock->lock_start))))
+		return nlm4_fbig;
 
-	/* Try to cancel request. */
-	resp->status = nlmsvc_cancel_blocked(SVC_NET(rqstp), file, &argp->lock);
+	locks_init_lock(fl);
+	fl->c.flc_type = type;
+	lockd_set_file_lock_range4(fl, lock->lock_start, lock->lock_len);
+
+	error = nlm_lookup_file(rqstp, &file, lock);
+	switch (error) {
+	case nlm_granted:
+		break;
+	case nlm__int__stale_fh:
+		return nlm4_stale_fh;
+	case nlm__int__failed:
+		return nlm4_failed;
+	default:
+		return error;
+	}
+	*filp = file;
+
+	fl->c.flc_flags = FL_POSIX;
+	fl->c.flc_file = file->f_file[lock_to_openmode(fl)];
+	fl->c.flc_pid = current->tgid;
+	fl->fl_lmops = &nlmsvc_lock_operations;
+	nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid);
+	if (!fl->c.flc_owner)
+		return nlm_lck_denied_nolocks;
 
-	dprintk("lockd: CANCEL        status %d\n", ntohl(resp->status));
-	nlmsvc_release_lockowner(&argp->lock);
-	nlmsvc_release_host(host);
-	nlm_release_file(file);
-	return rpc_success;
+	return nlm_granted;
 }
 
+/**
+ * nlm4svc_proc_null - NULL: Test for presence of service
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_NULL(void) = 0;
+ */
 static __be32
-nlm4svc_proc_cancel(struct svc_rqst *rqstp)
+nlm4svc_proc_null(struct svc_rqst *rqstp)
 {
-	return __nlm4svc_proc_cancel(rqstp, rqstp->rq_resp);
+	return rpc_success;
 }
 
-/*
- * UNLOCK: release a lock
+/**
+ * nlm4svc_proc_test - TEST: Check for conflicting lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_testres NLMPROC4_TEST(nlm4_testargs) = 1;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The server would be able to grant the
+ *				requested lock.
+ *   %NLM4_DENIED:		The requested lock conflicted with existing
+ *				lock reservations for the file.
+ *   %NLM4_DENIED_NOLOCKS:	The server could not allocate the resources
+ *				needed to process the request.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
  */
-static __be32
-__nlm4svc_proc_unlock(struct svc_rqst *rqstp, struct nlm_res *resp)
+static __be32 nlm4svc_proc_test(struct svc_rqst *rqstp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct nlm4_testres_wrapper *resp = rqstp->rq_resp;
+	struct nlm_file	*file = NULL;
 	struct nlm_host	*host;
-	struct nlm_file	*file;
 
-	dprintk("lockd: UNLOCK        called\n");
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
 
-	resp->cookie = argp->cookie;
+	resp->xdrgen.stat.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
 
-	/* Don't accept new lock requests during grace period */
-	if (locks_in_grace(SVC_NET(rqstp))) {
-		resp->status = nlm_lck_denied_grace_period;
-		return rpc_success;
+	resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+						     &file, &argp->xdrgen.alock,
+						     type);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlmsvc_testlock(rqstp, file, host,
+						 &argp->lock, &resp->lock);
+	nlmsvc_release_lockowner(&argp->lock);
+
+	if (resp->xdrgen.stat.stat == nlm_lck_denied) {
+		struct nlm_lock *conf = &resp->lock;
+		struct nlm4_holder *holder = &resp->xdrgen.stat.u.holder;
+
+		holder->exclusive = (conf->fl.c.flc_type != F_RDLCK);
+		holder->svid = conf->svid;
+		holder->oh.len = conf->oh.len;
+		holder->oh.data = conf->oh.data;
+		holder->l_offset = conf->fl.fl_start;
+		if (conf->fl.fl_end == OFFSET_MAX)
+			holder->l_len = 0;
+		else
+			holder->l_len = conf->fl.fl_end - conf->fl.fl_start + 1;
 	}
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->xdrgen.stat.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
+}
 
-	/* Now try to remove the lock */
-	resp->status = nlmsvc_unlock(SVC_NET(rqstp), file, &argp->lock);
+static __be32
+nlm4svc_do_lock(struct svc_rqst *rqstp, bool monitored)
+{
+	struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct nlm4_res_wrapper *resp = rqstp->rq_resp;
+	struct nlm_file	*file = NULL;
+	struct nlm_host	*host = NULL;
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat.stat = nlm4_netobj_to_cookie(&argp->cookie,
+						       &argp->xdrgen.cookie);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name,
+				   monitored);
+	if (!host)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+						     &file, &argp->xdrgen.alock,
+						     type);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlmsvc_lock(rqstp, file, host, &argp->lock,
+					     argp->xdrgen.block, &argp->cookie,
+					     argp->xdrgen.reclaim);
+	if (resp->xdrgen.stat.stat == nlm__int__deadlock)
+		resp->xdrgen.stat.stat = nlm4_deadlock;
 
-	dprintk("lockd: UNLOCK        status %d\n", ntohl(resp->status));
 	nlmsvc_release_lockowner(&argp->lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
 	nlmsvc_release_host(host);
-	nlm_release_file(file);
-	return rpc_success;
+	return resp->xdrgen.stat.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
 }
 
+/**
+ * nlm4svc_proc_lock - LOCK: Establish a monitored lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_res NLMPROC4_LOCK(nlm4_lockargs) = 2;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The requested lock was granted.
+ *   %NLM4_DENIED:		The requested lock conflicted with existing
+ *				lock reservations for the file.
+ *   %NLM4_DENIED_NOLOCKS:	The server could not allocate the resources
+ *				needed to process the request.
+ *   %NLM4_BLOCKED:		The blocking request cannot be granted
+ *				immediately. The server will send an
+ *				NLMPROC4_GRANTED callback to the client when
+ *				the lock can be granted.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DEADLCK:		The request could not be granted and
+ *				blocking would cause a deadlock.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
+ */
 static __be32
-nlm4svc_proc_unlock(struct svc_rqst *rqstp)
+nlm4svc_proc_lock(struct svc_rqst *rqstp)
 {
-	return __nlm4svc_proc_unlock(rqstp, rqstp->rq_resp);
+	return nlm4svc_do_lock(rqstp, true);
 }
 
-/*
- * GRANTED: A server calls us to tell that a process' lock request
- * was granted
+/**
+ * nlm4svc_proc_cancel - CANCEL: Cancel an outstanding blocked lock request
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully
+ *   %rpc_drop_reply:		Do not send an RPC reply
+ *
+ * RPC synopsis:
+ *   nlm4_res NLMPROC4_CANCEL(nlm4_cancargs) = 3;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_LCK_GRANTED:		The requested lock was canceled.
+ *   %NLM4_LCK_DENIED:		There was no lock to cancel.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DENIED_NOLOCKS:	A needed resource could not be allocated.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
  */
 static __be32
-__nlm4svc_proc_granted(struct svc_rqst *rqstp, struct nlm_res *resp)
+nlm4svc_proc_cancel(struct svc_rqst *rqstp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
+	struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct nlm4_res_wrapper *resp = rqstp->rq_resp;
+	struct net *net = SVC_NET(rqstp);
+	struct nlm_host	*host = NULL;
+	struct nlm_file	*file = NULL;
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat.stat = nlm_lck_denied_grace_period;
+	if (locks_in_grace(net))
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+						     &file, &argp->xdrgen.alock,
+						     type);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlmsvc_cancel_blocked(net, file, &argp->lock);
+	nlmsvc_release_lockowner(&argp->lock);
 
-	resp->cookie = argp->cookie;
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->xdrgen.stat.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
+}
 
-	dprintk("lockd: GRANTED       called\n");
-	resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock);
-	dprintk("lockd: GRANTED       status %d\n", ntohl(resp->status));
-	return rpc_success;
+/**
+ * nlm4svc_proc_unlock - UNLOCK: Remove a lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_res NLMPROC4_UNLOCK(nlm4_unlockargs) = 4;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The requested lock was released.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DENIED_NOLOCKS:	A needed resource could not be allocated.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
+ */
+static __be32
+nlm4svc_proc_unlock(struct svc_rqst *rqstp)
+{
+	struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm4_res_wrapper *resp = rqstp->rq_resp;
+	struct net *net = SVC_NET(rqstp);
+	struct nlm_host	*host = NULL;
+	struct nlm_file	*file = NULL;
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat.stat = nlm_lck_denied_grace_period;
+	if (locks_in_grace(net))
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+						     &file, &argp->xdrgen.alock,
+						     F_UNLCK);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlmsvc_unlock(net, file, &argp->lock);
+	nlmsvc_release_lockowner(&argp->lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->xdrgen.stat.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
 }
 
+/**
+ * nlm4svc_proc_granted - GRANTED: Server grants a previously blocked lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *
+ * RPC synopsis:
+ *   nlm4_res NLMPROC4_GRANTED(nlm4_testargs) = 5;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The requested lock was granted.
+ *   %NLM4_DENIED:		The server could not allocate the resources
+ *				needed to process the request.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ */
 static __be32
 nlm4svc_proc_granted(struct svc_rqst *rqstp)
 {
-	return __nlm4svc_proc_granted(rqstp, rqstp->rq_resp);
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm4_res_wrapper *resp = rqstp->rq_resp;
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat.stat = nlm4_lock_to_nlm_lock(&argp->lock,
+						       &argp->xdrgen.alock);
+	if (resp->xdrgen.stat.stat)
+		goto out;
+
+	resp->xdrgen.stat.stat = nlmclnt_grant(svc_addr(rqstp), &argp->lock);
+
+out:
+	return rpc_success;
 }
 
 /*
@@ -281,24 +552,17 @@ static const struct rpc_call_ops nlm4svc_callback_ops = {
 };
 
 /*
- * `Async' versions of the above service routines. They aren't really,
- * because we send the callback before the reply proper. I hope this
- * doesn't break any clients.
+ * Dispatch an async callback RPC to a client with a pre-resolved host.
+ * Caller provides a reference to @host; this function takes ownership
+ * and releases it via nlmsvc_release_host() before returning.
  */
-static __be32 nlm4svc_callback(struct svc_rqst *rqstp, u32 proc,
-		__be32 (*func)(struct svc_rqst *,  struct nlm_res *))
+static __be32
+nlm4svc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc,
+		 __be32 (*func)(struct svc_rqst *,  struct nlm_res *))
 {
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_host	*host;
 	struct nlm_rqst	*call;
 	__be32 stat;
 
-	host = nlmsvc_lookup_host(rqstp,
-				  argp->lock.caller,
-				  argp->lock.len);
-	if (host == NULL)
-		return rpc_system_err;
-
 	call = nlm_alloc_call(host);
 	nlmsvc_release_host(host);
 	if (call == NULL)
@@ -316,433 +580,845 @@ static __be32 nlm4svc_callback(struct svc_rqst *rqstp, u32 proc,
 	return rpc_success;
 }
 
-static __be32 nlm4svc_proc_test_msg(struct svc_rqst *rqstp)
+static __be32
+__nlm4svc_proc_test_msg(struct svc_rqst *rqstp, struct nlm_res *resp)
 {
-	dprintk("lockd: TEST_MSG      called\n");
-	return nlm4svc_callback(rqstp, NLMPROC_TEST_RES, __nlm4svc_proc_test);
-}
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct nlm_lockowner *owner;
+	struct nlm_file	*file = NULL;
+	struct nlm_host	*host = NULL;
+
+	resp->status = nlm_lck_denied_nolocks;
+	if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie))
+		goto out;
+
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
+
+	resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+					   &file, &argp->xdrgen.alock, type);
+	if (resp->status)
+		goto out;
+
+	owner = argp->lock.fl.c.flc_owner;
+	resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock,
+				       &resp->lock);
+	nlmsvc_put_lockowner(owner);
 
-static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp)
-{
-	dprintk("lockd: LOCK_MSG      called\n");
-	return nlm4svc_callback(rqstp, NLMPROC_LOCK_RES, __nlm4svc_proc_lock);
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success;
 }
 
-static __be32 nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp)
+/**
+ * nlm4svc_proc_test_msg - TEST_MSG: Check for conflicting lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_TEST_MSG(nlm4_testargs) = 6;
+ *
+ * The response to this request is delivered via the TEST_RES procedure.
+ */
+static __be32 nlm4svc_proc_test_msg(struct svc_rqst *rqstp)
 {
-	dprintk("lockd: CANCEL_MSG    called\n");
-	return nlm4svc_callback(rqstp, NLMPROC_CANCEL_RES, __nlm4svc_proc_cancel);
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host *host;
+
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		return rpc_system_err;
+
+	return nlm4svc_callback(rqstp, host, NLMPROC4_TEST_RES,
+				__nlm4svc_proc_test_msg);
 }
 
-static __be32 nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp)
+static __be32
+__nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct nlm_res *resp)
 {
-	dprintk("lockd: UNLOCK_MSG    called\n");
-	return nlm4svc_callback(rqstp, NLMPROC_UNLOCK_RES, __nlm4svc_proc_unlock);
+	struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct nlm_file	*file = NULL;
+	struct nlm_host	*host = NULL;
+
+	resp->status = nlm_lck_denied_nolocks;
+	if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie))
+		goto out;
+
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true);
+	if (!host)
+		goto out;
+
+	resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+					   &file, &argp->xdrgen.alock, type);
+	if (resp->status)
+		goto out;
+
+	resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock,
+				   argp->xdrgen.block, &resp->cookie,
+				   argp->xdrgen.reclaim);
+	nlmsvc_release_lockowner(&argp->lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->status == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
 }
 
-static __be32 nlm4svc_proc_granted_msg(struct svc_rqst *rqstp)
+/**
+ * nlm4svc_proc_lock_msg - LOCK_MSG: Establish a monitored lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_LOCK_MSG(nlm4_lockargs) = 7;
+ *
+ * The response to this request is delivered via the LOCK_RES procedure.
+ */
+static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp)
 {
-	dprintk("lockd: GRANTED_MSG   called\n");
-	return nlm4svc_callback(rqstp, NLMPROC_GRANTED_RES, __nlm4svc_proc_granted);
+	struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host *host;
+
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true);
+	if (!host)
+		return rpc_system_err;
+
+	return nlm4svc_callback(rqstp, host, NLMPROC4_LOCK_RES,
+				__nlm4svc_proc_lock_msg);
 }
 
-/*
- * SHARE: create a DOS share or alter existing share.
- */
 static __be32
-nlm4svc_proc_share(struct svc_rqst *rqstp)
+__nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp, struct nlm_res *resp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_res *resp = rqstp->rq_resp;
-	struct nlm_host	*host;
-	struct nlm_file	*file;
-
-	dprintk("lockd: SHARE         called\n");
+	struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp;
+	unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK;
+	struct net *net = SVC_NET(rqstp);
+	struct nlm_file	*file = NULL;
+	struct nlm_host	*host = NULL;
+
+	resp->status = nlm_lck_denied_nolocks;
+	if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie))
+		goto out;
+
+	resp->status = nlm_lck_denied_grace_period;
+	if (locks_in_grace(net))
+		goto out;
+
+	resp->status = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
+
+	resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+					   &file, &argp->xdrgen.alock, type);
+	if (resp->status)
+		goto out;
+
+	resp->status = nlmsvc_cancel_blocked(net, file, &argp->lock);
+	nlmsvc_release_lockowner(&argp->lock);
 
-	resp->cookie = argp->cookie;
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->status == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
+}
 
-	/* Don't accept new lock requests during grace period */
-	if (locks_in_grace(SVC_NET(rqstp)) && !argp->reclaim) {
-		resp->status = nlm_lck_denied_grace_period;
-		return rpc_success;
-	}
+/**
+ * nlm4svc_proc_cancel_msg - CANCEL_MSG: Cancel an outstanding lock request
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_CANCEL_MSG(nlm4_cancargs) = 8;
+ *
+ * The response to this request is delivered via the CANCEL_RES procedure.
+ */
+static __be32 nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp)
+{
+	struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host *host;
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		return rpc_system_err;
 
-	/* Now try to create the share */
-	resp->status = nlmsvc_share_file(host, file, argp);
+	return nlm4svc_callback(rqstp, host, NLMPROC4_CANCEL_RES,
+				__nlm4svc_proc_cancel_msg);
+}
 
-	dprintk("lockd: SHARE         status %d\n", ntohl(resp->status));
+static __be32
+__nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp, struct nlm_res *resp)
+{
+	struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp;
+	struct net *net = SVC_NET(rqstp);
+	struct nlm_file	*file = NULL;
+	struct nlm_host	*host = NULL;
+
+	resp->status = nlm_lck_denied_nolocks;
+	if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie))
+		goto out;
+
+	resp->status = nlm_lck_denied_grace_period;
+	if (locks_in_grace(net))
+		goto out;
+
+	resp->status = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		goto out;
+
+	resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock,
+					   &file, &argp->xdrgen.alock, F_UNLCK);
+	if (resp->status)
+		goto out;
+
+	resp->status = nlmsvc_unlock(net, file, &argp->lock);
 	nlmsvc_release_lockowner(&argp->lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
 	nlmsvc_release_host(host);
-	nlm_release_file(file);
-	return rpc_success;
+	return resp->status == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
 }
 
-/*
- * UNSHARE: Release a DOS share.
+/**
+ * nlm4svc_proc_unlock_msg - UNLOCK_MSG: Remove an existing lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_UNLOCK_MSG(nlm4_unlockargs) = 9;
+ *
+ * The response to this request is delivered via the UNLOCK_RES procedure.
  */
-static __be32
-nlm4svc_proc_unshare(struct svc_rqst *rqstp)
+static __be32 nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_res *resp = rqstp->rq_resp;
-	struct nlm_host	*host;
-	struct nlm_file	*file;
+	struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host *host;
 
-	dprintk("lockd: UNSHARE       called\n");
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		return rpc_system_err;
 
-	resp->cookie = argp->cookie;
+	return nlm4svc_callback(rqstp, host, NLMPROC4_UNLOCK_RES,
+				__nlm4svc_proc_unlock_msg);
+}
 
-	/* Don't accept requests during grace period */
-	if (locks_in_grace(SVC_NET(rqstp))) {
-		resp->status = nlm_lck_denied_grace_period;
-		return rpc_success;
-	}
+static __be32
+__nlm4svc_proc_granted_msg(struct svc_rqst *rqstp, struct nlm_res *resp)
+{
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
 
-	/* Obtain client and file */
-	if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+	resp->status = nlm_lck_denied;
+	if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie))
+		goto out;
 
-	/* Now try to lock the file */
-	resp->status = nlmsvc_unshare_file(host, file, argp);
+	if (nlm4_lock_to_nlm_lock(&argp->lock, &argp->xdrgen.alock))
+		goto out;
 
-	dprintk("lockd: UNSHARE       status %d\n", ntohl(resp->status));
-	nlmsvc_release_lockowner(&argp->lock);
-	nlmsvc_release_host(host);
-	nlm_release_file(file);
+	resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock);
+
+out:
 	return rpc_success;
 }
 
-/*
- * NM_LOCK: Create an unmonitored lock
+/**
+ * nlm4svc_proc_granted_msg - GRANTED_MSG: Blocked lock has been granted
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_GRANTED_MSG(nlm4_testargs) = 10;
+ *
+ * The response to this request is delivered via the GRANTED_RES procedure.
  */
-static __be32
-nlm4svc_proc_nm_lock(struct svc_rqst *rqstp)
+static __be32 nlm4svc_proc_granted_msg(struct svc_rqst *rqstp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
+	struct nlm4_testargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host *host;
 
-	dprintk("lockd: NM_LOCK       called\n");
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false);
+	if (!host)
+		return rpc_system_err;
 
-	argp->monitor = 0;		/* just clean the monitor flag */
-	return nlm4svc_proc_lock(rqstp);
+	return nlm4svc_callback(rqstp, host, NLMPROC4_GRANTED_RES,
+				__nlm4svc_proc_granted_msg);
 }
 
-/*
- * FREE_ALL: Release all locks and shares held by client
+/**
+ * nlm4svc_proc_granted_res - GRANTED_RES: Lock Granted result
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_GRANTED_RES(nlm4_res) = 15;
  */
-static __be32
-nlm4svc_proc_free_all(struct svc_rqst *rqstp)
+static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp)
 {
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_host	*host;
+	struct nlm4_res_wrapper *argp = rqstp->rq_argp;
 
-	/* Obtain client */
-	if (nlm4svc_retrieve_args(rqstp, argp, &host, NULL))
+	if (!nlmsvc_ops)
 		return rpc_success;
 
-	nlmsvc_free_host_resources(host);
-	nlmsvc_release_host(host);
+	if (nlm4_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie))
+		return rpc_success;
+	nlmsvc_grant_reply(&argp->cookie, argp->xdrgen.stat.stat);
+
 	return rpc_success;
 }
 
-/*
- * SM_NOTIFY: private callback from statd (not part of official NLM proto)
+/**
+ * nlm4svc_proc_sm_notify - SM_NOTIFY: Peer has rebooted
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_system_err:		RPC execution failed.
+ *
+ * The SM_NOTIFY procedure is a private callback from Linux statd and is
+ * not part of the official NLM protocol.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_SM_NOTIFY(nlm4_notifyargs) = 16;
  */
-static __be32
-nlm4svc_proc_sm_notify(struct svc_rqst *rqstp)
+static __be32 nlm4svc_proc_sm_notify(struct svc_rqst *rqstp)
 {
-	struct nlm_reboot *argp = rqstp->rq_argp;
-
-	dprintk("lockd: SM_NOTIFY     called\n");
+	struct nlm4_notifyargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm_reboot *reboot = &argp->reboot;
 
 	if (!nlm_privileged_requester(rqstp)) {
 		char buf[RPC_MAX_ADDRBUFLEN];
-		printk(KERN_WARNING "lockd: rejected NSM callback from %s\n",
-				svc_print_addr(rqstp, buf, sizeof(buf)));
+
+		pr_warn("lockd: rejected NSM callback from %s\n",
+			svc_print_addr(rqstp, buf, sizeof(buf)));
 		return rpc_system_err;
 	}
 
-	nlm_host_rebooted(SVC_NET(rqstp), argp);
+	reboot->len = argp->xdrgen.notify.name.len;
+	reboot->mon = (char *)argp->xdrgen.notify.name.data;
+	reboot->state = argp->xdrgen.notify.state;
+	memcpy(&reboot->priv.data, argp->xdrgen.private,
+	       sizeof(reboot->priv.data));
+
+	nlm_host_rebooted(SVC_NET(rqstp), reboot);
+
 	return rpc_success;
 }
 
-/*
- * client sent a GRANTED_RES, let's remove the associated block
+/**
+ * nlm4svc_proc_unused - stub for unused procedures
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_proc_unavail:	Program can't support procedure.
  */
-static __be32
-nlm4svc_proc_granted_res(struct svc_rqst *rqstp)
+static __be32 nlm4svc_proc_unused(struct svc_rqst *rqstp)
 {
-	struct nlm_res *argp = rqstp->rq_argp;
+	return rpc_proc_unavail;
+}
 
-        if (!nlmsvc_ops)
-                return rpc_success;
+/**
+ * nlm4svc_proc_share - SHARE: Open a file using DOS file-sharing modes
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_shareres NLMPROC4_SHARE(nlm4_shareargs) = 20;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The requested share lock was granted.
+ *   %NLM4_DENIED:		The requested lock conflicted with existing
+ *				lock reservations for the file.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DENIED_NOLOCKS:	A needed resource could not be allocated.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
+ */
+static __be32 nlm4svc_proc_share(struct svc_rqst *rqstp)
+{
+	struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm4_shareres_wrapper *resp = rqstp->rq_resp;
+	struct nlm_lock	*lock = &argp->lock;
+	struct nlm_host	*host = NULL;
+	struct nlm_file	*file = NULL;
+	struct nlm4_lock xdr_lock = {
+		.fh		= argp->xdrgen.share.fh,
+		.oh		= argp->xdrgen.share.oh,
+		.svid		= LOCKD_SHARE_SVID,
+	};
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat = nlm_lck_denied_grace_period;
+	if (locks_in_grace(SVC_NET(rqstp)) && !argp->xdrgen.reclaim)
+		goto out;
+
+	resp->xdrgen.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, true);
+	if (!host)
+		goto out;
+
+	resp->xdrgen.stat = nlm4svc_lookup_file(rqstp, host, lock, &file,
+						&xdr_lock, F_RDLCK);
+	if (resp->xdrgen.stat)
+		goto out;
+
+	resp->xdrgen.stat = nlmsvc_share_file(host, file, &lock->oh,
+					      argp->xdrgen.share.access,
+					      argp->xdrgen.share.mode);
+
+	nlmsvc_release_lockowner(lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->xdrgen.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
+}
 
-        dprintk("lockd: GRANTED_RES   called\n");
+/**
+ * nlm4svc_proc_unshare - UNSHARE: Release a share reservation
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_shareres NLMPROC4_UNSHARE(nlm4_shareargs) = 21;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The share reservation was released.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DENIED_NOLOCKS:	A needed resource could not be allocated.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
+ */
+static __be32 nlm4svc_proc_unshare(struct svc_rqst *rqstp)
+{
+	struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp;
+	struct nlm4_shareres_wrapper *resp = rqstp->rq_resp;
+	struct nlm_lock	*lock = &argp->lock;
+	struct nlm4_lock xdr_lock = {
+		.fh		= argp->xdrgen.share.fh,
+		.oh		= argp->xdrgen.share.oh,
+		.svid		= LOCKD_SHARE_SVID,
+	};
+	struct nlm_host	*host = NULL;
+	struct nlm_file	*file = NULL;
+
+	resp->xdrgen.cookie = argp->xdrgen.cookie;
+
+	resp->xdrgen.stat = nlm_lck_denied_grace_period;
+	if (locks_in_grace(SVC_NET(rqstp)))
+		goto out;
+
+	resp->xdrgen.stat = nlm_lck_denied_nolocks;
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, true);
+	if (!host)
+		goto out;
+
+	resp->xdrgen.stat = nlm4svc_lookup_file(rqstp, host, lock, &file,
+						&xdr_lock, F_RDLCK);
+	if (resp->xdrgen.stat)
+		goto out;
+
+	resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh);
+
+	nlmsvc_release_lockowner(lock);
+
+out:
+	if (file)
+		nlm_release_file(file);
+	nlmsvc_release_host(host);
+	return resp->xdrgen.stat == nlm__int__drop_reply ?
+		rpc_drop_reply : rpc_success;
+}
 
-        nlmsvc_grant_reply(&argp->cookie, argp->status);
-        return rpc_success;
+/**
+ * nlm4svc_proc_nm_lock - NM_LOCK: Establish a non-monitored lock
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *   %rpc_drop_reply:		Do not send an RPC reply.
+ *
+ * RPC synopsis:
+ *   nlm4_res NLMPROC4_NM_LOCK(nlm4_lockargs) = 22;
+ *
+ * Permissible procedure status codes:
+ *   %NLM4_GRANTED:		The requested lock was granted.
+ *   %NLM4_DENIED:		The requested lock conflicted with existing
+ *				lock reservations for the file.
+ *   %NLM4_DENIED_NOLOCKS:	The server could not allocate the resources
+ *				needed to process the request.
+ *   %NLM4_BLOCKED:		The blocking request cannot be granted
+ *				immediately. The server will send an
+ *				NLMPROC4_GRANTED callback to the client when
+ *				the lock can be granted.
+ *   %NLM4_DENIED_GRACE_PERIOD:	The server has recently restarted and is
+ *				re-establishing existing locks, and is not
+ *				yet ready to accept normal service requests.
+ *
+ * The Linux NLM server implementation also returns:
+ *   %NLM4_DEADLCK:		The request could not be granted and
+ *				blocking would cause a deadlock.
+ *   %NLM4_STALE_FH:		The request specified an invalid file handle.
+ *   %NLM4_FBIG:		The request specified a length or offset
+ *				that exceeds the range supported by the
+ *				server.
+ *   %NLM4_FAILED:		The request failed for an unspecified reason.
+ */
+static __be32 nlm4svc_proc_nm_lock(struct svc_rqst *rqstp)
+{
+	return nlm4svc_do_lock(rqstp, false);
 }
 
-static __be32
-nlm4svc_proc_unused(struct svc_rqst *rqstp)
+/**
+ * nlm4svc_proc_free_all - FREE_ALL: Discard client's lock and share state
+ * @rqstp: RPC transaction context
+ *
+ * Returns:
+ *   %rpc_success:		RPC executed successfully.
+ *
+ * RPC synopsis:
+ *   void NLMPROC4_FREE_ALL(nlm4_notify) = 23;
+ */
+static __be32 nlm4svc_proc_free_all(struct svc_rqst *rqstp)
 {
-	return rpc_proc_unavail;
+	struct nlm4_notify_wrapper *argp = rqstp->rq_argp;
+	struct nlm_host	*host;
+
+	host = nlm4svc_lookup_host(rqstp, argp->xdrgen.name, false);
+	if (!host)
+		goto out;
+
+	nlmsvc_free_host_resources(host);
+
+	nlmsvc_release_host(host);
+
+out:
+	return rpc_success;
 }
 
 
 /*
- * NLM Server procedures.
+ * NLMv4 Server procedures.
  */
 
-struct nlm_void			{ int dummy; };
-
-#define	Ck	(1+XDR_QUADLEN(NLM_MAXCOOKIELEN))	/* cookie */
-#define	No	(1+1024/4)				/* netobj */
-#define	St	1					/* status */
-#define	Rg	4					/* range (offset + length) */
-
-const struct svc_procedure nlmsvc_procedures4[24] = {
-	[NLMPROC_NULL] = {
-		.pc_func = nlm4svc_proc_null,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_void),
-		.pc_argzero = sizeof(struct nlm_void),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "NULL",
+static const struct svc_procedure nlm4svc_procedures[24] = {
+	[NLMPROC4_NULL] = {
+		.pc_func	= nlm4svc_proc_null,
+		.pc_decode	= nlm4_svc_decode_void,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= XDR_void,
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "NULL",
 	},
-	[NLMPROC_TEST] = {
-		.pc_func = nlm4svc_proc_test,
-		.pc_decode = nlm4svc_decode_testargs,
-		.pc_encode = nlm4svc_encode_testres,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St+2+No+Rg,
-		.pc_name = "TEST",
+	[NLMPROC4_TEST] = {
+		.pc_func	= nlm4svc_proc_test,
+		.pc_decode	= nlm4_svc_decode_nlm4_testargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_testres,
+		.pc_argsize	= sizeof(struct nlm4_testargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_testres_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_testres_sz,
+		.pc_name	= "TEST",
 	},
-	[NLMPROC_LOCK] = {
-		.pc_func = nlm4svc_proc_lock,
-		.pc_decode = nlm4svc_decode_lockargs,
-		.pc_encode = nlm4svc_encode_res,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St,
-		.pc_name = "LOCK",
+	[NLMPROC4_LOCK] = {
+		.pc_func	= nlm4svc_proc_lock,
+		.pc_decode	= nlm4_svc_decode_nlm4_lockargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_res,
+		.pc_argsize	= sizeof(struct nlm4_lockargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_res_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_res_sz,
+		.pc_name	= "LOCK",
 	},
-	[NLMPROC_CANCEL] = {
-		.pc_func = nlm4svc_proc_cancel,
-		.pc_decode = nlm4svc_decode_cancargs,
-		.pc_encode = nlm4svc_encode_res,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St,
-		.pc_name = "CANCEL",
+	[NLMPROC4_CANCEL] = {
+		.pc_func	= nlm4svc_proc_cancel,
+		.pc_decode	= nlm4_svc_decode_nlm4_cancargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_res,
+		.pc_argsize	= sizeof(struct nlm4_cancargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_res_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_res_sz,
+		.pc_name	= "CANCEL",
 	},
-	[NLMPROC_UNLOCK] = {
-		.pc_func = nlm4svc_proc_unlock,
-		.pc_decode = nlm4svc_decode_unlockargs,
-		.pc_encode = nlm4svc_encode_res,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St,
-		.pc_name = "UNLOCK",
+	[NLMPROC4_UNLOCK] = {
+		.pc_func	= nlm4svc_proc_unlock,
+		.pc_decode	= nlm4_svc_decode_nlm4_unlockargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_res,
+		.pc_argsize	= sizeof(struct nlm4_unlockargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_res_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_res_sz,
+		.pc_name	= "UNLOCK",
 	},
-	[NLMPROC_GRANTED] = {
-		.pc_func = nlm4svc_proc_granted,
-		.pc_decode = nlm4svc_decode_testargs,
-		.pc_encode = nlm4svc_encode_res,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St,
-		.pc_name = "GRANTED",
+	[NLMPROC4_GRANTED] = {
+		.pc_func	= nlm4svc_proc_granted,
+		.pc_decode	= nlm4_svc_decode_nlm4_testargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_res,
+		.pc_argsize	= sizeof(struct nlm4_testargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_res_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_res_sz,
+		.pc_name	= "GRANTED",
 	},
-	[NLMPROC_TEST_MSG] = {
-		.pc_func = nlm4svc_proc_test_msg,
-		.pc_decode = nlm4svc_decode_testargs,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "TEST_MSG",
+	[NLMPROC4_TEST_MSG] = {
+		.pc_func	= nlm4svc_proc_test_msg,
+		.pc_decode	= nlm4_svc_decode_nlm4_testargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_testargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "TEST_MSG",
 	},
-	[NLMPROC_LOCK_MSG] = {
-		.pc_func = nlm4svc_proc_lock_msg,
-		.pc_decode = nlm4svc_decode_lockargs,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "LOCK_MSG",
+	[NLMPROC4_LOCK_MSG] = {
+		.pc_func	= nlm4svc_proc_lock_msg,
+		.pc_decode	= nlm4_svc_decode_nlm4_lockargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_lockargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "LOCK_MSG",
 	},
-	[NLMPROC_CANCEL_MSG] = {
-		.pc_func = nlm4svc_proc_cancel_msg,
-		.pc_decode = nlm4svc_decode_cancargs,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "CANCEL_MSG",
+	[NLMPROC4_CANCEL_MSG] = {
+		.pc_func	= nlm4svc_proc_cancel_msg,
+		.pc_decode	= nlm4_svc_decode_nlm4_cancargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_cancargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "CANCEL_MSG",
 	},
-	[NLMPROC_UNLOCK_MSG] = {
-		.pc_func = nlm4svc_proc_unlock_msg,
-		.pc_decode = nlm4svc_decode_unlockargs,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "UNLOCK_MSG",
+	[NLMPROC4_UNLOCK_MSG] = {
+		.pc_func	= nlm4svc_proc_unlock_msg,
+		.pc_decode	= nlm4_svc_decode_nlm4_unlockargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_unlockargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "UNLOCK_MSG",
 	},
-	[NLMPROC_GRANTED_MSG] = {
-		.pc_func = nlm4svc_proc_granted_msg,
-		.pc_decode = nlm4svc_decode_testargs,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "GRANTED_MSG",
+	[NLMPROC4_GRANTED_MSG] = {
+		.pc_func	= nlm4svc_proc_granted_msg,
+		.pc_decode	= nlm4_svc_decode_nlm4_testargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_testargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "GRANTED_MSG",
 	},
-	[NLMPROC_TEST_RES] = {
-		.pc_func = nlm4svc_proc_null,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_res),
-		.pc_argzero = sizeof(struct nlm_res),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "TEST_RES",
+	[NLMPROC4_TEST_RES] = {
+		.pc_func	= nlm4svc_proc_null,
+		.pc_decode	= nlm4_svc_decode_nlm4_testres,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_testres),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "TEST_RES",
 	},
-	[NLMPROC_LOCK_RES] = {
-		.pc_func = nlm4svc_proc_null,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_res),
-		.pc_argzero = sizeof(struct nlm_res),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "LOCK_RES",
+	[NLMPROC4_LOCK_RES] = {
+		.pc_func	= nlm4svc_proc_null,
+		.pc_decode	= nlm4_svc_decode_nlm4_res,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_res),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "LOCK_RES",
 	},
-	[NLMPROC_CANCEL_RES] = {
-		.pc_func = nlm4svc_proc_null,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_res),
-		.pc_argzero = sizeof(struct nlm_res),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "CANCEL_RES",
+	[NLMPROC4_CANCEL_RES] = {
+		.pc_func	= nlm4svc_proc_null,
+		.pc_decode	= nlm4_svc_decode_nlm4_res,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_res),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "CANCEL_RES",
 	},
-	[NLMPROC_UNLOCK_RES] = {
-		.pc_func = nlm4svc_proc_null,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_res),
-		.pc_argzero = sizeof(struct nlm_res),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "UNLOCK_RES",
+	[NLMPROC4_UNLOCK_RES] = {
+		.pc_func	= nlm4svc_proc_null,
+		.pc_decode	= nlm4_svc_decode_nlm4_res,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_res),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "UNLOCK_RES",
 	},
-	[NLMPROC_GRANTED_RES] = {
-		.pc_func = nlm4svc_proc_granted_res,
-		.pc_decode = nlm4svc_decode_res,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_res),
-		.pc_argzero = sizeof(struct nlm_res),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "GRANTED_RES",
+	[NLMPROC4_GRANTED_RES] = {
+		.pc_func	= nlm4svc_proc_granted_res,
+		.pc_decode	= nlm4_svc_decode_nlm4_res,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_res_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "GRANTED_RES",
 	},
-	[NLMPROC_NSM_NOTIFY] = {
-		.pc_func = nlm4svc_proc_sm_notify,
-		.pc_decode = nlm4svc_decode_reboot,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_reboot),
-		.pc_argzero = sizeof(struct nlm_reboot),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "SM_NOTIFY",
+	[NLMPROC4_SM_NOTIFY] = {
+		.pc_func	= nlm4svc_proc_sm_notify,
+		.pc_decode	= nlm4_svc_decode_nlm4_notifyargs,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_notifyargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "SM_NOTIFY",
 	},
 	[17] = {
-		.pc_func = nlm4svc_proc_unused,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_void),
-		.pc_argzero = sizeof(struct nlm_void),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = 0,
-		.pc_name = "UNUSED",
+		.pc_func	= nlm4svc_proc_unused,
+		.pc_decode	= nlm4_svc_decode_void,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= 0,
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "UNUSED",
 	},
 	[18] = {
-		.pc_func = nlm4svc_proc_unused,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_void),
-		.pc_argzero = sizeof(struct nlm_void),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = 0,
-		.pc_name = "UNUSED",
+		.pc_func	= nlm4svc_proc_unused,
+		.pc_decode	= nlm4_svc_decode_void,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= 0,
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "UNUSED",
 	},
 	[19] = {
-		.pc_func = nlm4svc_proc_unused,
-		.pc_decode = nlm4svc_decode_void,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_void),
-		.pc_argzero = sizeof(struct nlm_void),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = 0,
-		.pc_name = "UNUSED",
+		.pc_func	= nlm4svc_proc_unused,
+		.pc_decode	= nlm4_svc_decode_void,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= 0,
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "UNUSED",
 	},
-	[NLMPROC_SHARE] = {
-		.pc_func = nlm4svc_proc_share,
-		.pc_decode = nlm4svc_decode_shareargs,
-		.pc_encode = nlm4svc_encode_shareres,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St+1,
-		.pc_name = "SHARE",
+	[NLMPROC4_SHARE] = {
+		.pc_func	= nlm4svc_proc_share,
+		.pc_decode	= nlm4_svc_decode_nlm4_shareargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_shareres,
+		.pc_argsize	= sizeof(struct nlm4_shareargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_shareres_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_shareres_sz,
+		.pc_name	= "SHARE",
 	},
-	[NLMPROC_UNSHARE] = {
-		.pc_func = nlm4svc_proc_unshare,
-		.pc_decode = nlm4svc_decode_shareargs,
-		.pc_encode = nlm4svc_encode_shareres,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St+1,
-		.pc_name = "UNSHARE",
+	[NLMPROC4_UNSHARE] = {
+		.pc_func	= nlm4svc_proc_unshare,
+		.pc_decode	= nlm4_svc_decode_nlm4_shareargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_shareres,
+		.pc_argsize	= sizeof(struct nlm4_shareargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_shareres_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_shareres_sz,
+		.pc_name	= "UNSHARE",
 	},
-	[NLMPROC_NM_LOCK] = {
-		.pc_func = nlm4svc_proc_nm_lock,
-		.pc_decode = nlm4svc_decode_lockargs,
-		.pc_encode = nlm4svc_encode_res,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_res),
-		.pc_xdrressize = Ck+St,
-		.pc_name = "NM_LOCK",
+	[NLMPROC4_NM_LOCK] = {
+		.pc_func	= nlm4svc_proc_nm_lock,
+		.pc_decode	= nlm4_svc_decode_nlm4_lockargs,
+		.pc_encode	= nlm4_svc_encode_nlm4_res,
+		.pc_argsize	= sizeof(struct nlm4_lockargs_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= sizeof(struct nlm4_res_wrapper),
+		.pc_xdrressize	= NLM4_nlm4_res_sz,
+		.pc_name	= "NM_LOCK",
 	},
-	[NLMPROC_FREE_ALL] = {
-		.pc_func = nlm4svc_proc_free_all,
-		.pc_decode = nlm4svc_decode_notify,
-		.pc_encode = nlm4svc_encode_void,
-		.pc_argsize = sizeof(struct nlm_args),
-		.pc_argzero = sizeof(struct nlm_args),
-		.pc_ressize = sizeof(struct nlm_void),
-		.pc_xdrressize = St,
-		.pc_name = "FREE_ALL",
+	[NLMPROC4_FREE_ALL] = {
+		.pc_func	= nlm4svc_proc_free_all,
+		.pc_decode	= nlm4_svc_decode_nlm4_notify,
+		.pc_encode	= nlm4_svc_encode_void,
+		.pc_argsize	= sizeof(struct nlm4_notify_wrapper),
+		.pc_argzero	= 0,
+		.pc_ressize	= 0,
+		.pc_xdrressize	= XDR_void,
+		.pc_name	= "FREE_ALL",
 	},
 };
+
+/*
+ * Storage requirements for XDR arguments and results
+ */
+union nlm4svc_xdrstore {
+	struct nlm4_testargs_wrapper	testargs;
+	struct nlm4_lockargs_wrapper	lockargs;
+	struct nlm4_cancargs_wrapper	cancargs;
+	struct nlm4_unlockargs_wrapper	unlockargs;
+	struct nlm4_notifyargs_wrapper	notifyargs;
+	struct nlm4_shareargs_wrapper	shareargs;
+	struct nlm4_notify_wrapper	notify;
+	struct nlm4_testres_wrapper	testres;
+	struct nlm4_res_wrapper		res;
+	struct nlm4_shareres_wrapper	shareres;
+};
+
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+			      nlm4svc_call_counters[ARRAY_SIZE(nlm4svc_procedures)]);
+
+const struct svc_version nlmsvc_version4 = {
+	.vs_vers	= 4,
+	.vs_nproc	= ARRAY_SIZE(nlm4svc_procedures),
+	.vs_proc	= nlm4svc_procedures,
+	.vs_count	= nlm4svc_call_counters,
+	.vs_dispatch	= nlmsvc_dispatch,
+	.vs_xdrsize	= sizeof(union nlm4svc_xdrstore),
+};
diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c
index 255a847ca0b6b..b98b1d0ada35f 100644
--- a/fs/lockd/svclock.c
+++ b/fs/lockd/svclock.c
@@ -28,16 +28,10 @@
 #include <linux/sched.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/svc_xprt.h>
-#include <linux/lockd/nlm.h>
-#include <linux/lockd/lockd.h>
 
-#define NLMDBG_FACILITY		NLMDBG_SVCLOCK
+#include "lockd.h"
 
-#ifdef CONFIG_LOCKD_V4
-#define nlm_deadlock	nlm4_deadlock
-#else
-#define nlm_deadlock	nlm_lck_denied
-#endif
+#define NLMDBG_FACILITY		NLMDBG_SVCLOCK
 
 static void nlmsvc_release_block(struct nlm_block *block);
 static void	nlmsvc_insert_block(struct nlm_block *block, unsigned long);
@@ -80,6 +74,11 @@ static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie)
 
 	return buf;
 }
+#else
+static inline const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie)
+{
+	return "???";
+}
 #endif
 
 /*
@@ -463,7 +462,7 @@ nlmsvc_defer_lock_rqst(struct svc_rqst *rqstp, struct nlm_block *block)
 		block->b_deferred_req =
 			rqstp->rq_chandle.defer(block->b_cache_req);
 		if (block->b_deferred_req != NULL)
-			status = nlm_drop_reply;
+			status = nlm__int__drop_reply;
 	}
 	dprintk("lockd: nlmsvc_defer_lock_rqst block %p flags %d status %d\n",
 		block, block->b_flags, ntohl(status));
@@ -487,7 +486,7 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file,
 	int			async_block = 0;
 	__be32			ret;
 
-	dprintk("lockd: nlmsvc_lock(%s/%ld, ty=%d, pi=%d, %Ld-%Ld, bl=%d)\n",
+	dprintk("lockd: nlmsvc_lock(%s/%llu, ty=%d, pi=%d, %Ld-%Ld, bl=%d)\n",
 				inode->i_sb->s_id, inode->i_ino,
 				lock->fl.c.flc_type,
 				lock->fl.c.flc_pid,
@@ -531,7 +530,7 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file,
 			ret = nlm_lck_denied;
 			goto out;
 		}
-		ret = nlm_drop_reply;
+		ret = nlm__int__drop_reply;
 		goto out;
 	}
 
@@ -589,7 +588,7 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file,
 			goto out;
 		case -EDEADLK:
 			nlmsvc_remove_block(block);
-			ret = nlm_deadlock;
+			ret = nlm__int__deadlock;
 			goto out;
 		default:			/* includes ENOLCK */
 			nlmsvc_remove_block(block);
@@ -617,7 +616,7 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file,
 	int			mode;
 	__be32			ret;
 
-	dprintk("lockd: nlmsvc_testlock(%s/%ld, ty=%d, %Ld-%Ld)\n",
+	dprintk("lockd: nlmsvc_testlock(%s/%llu, ty=%d, %Ld-%Ld)\n",
 				nlmsvc_file_inode(file)->i_sb->s_id,
 				nlmsvc_file_inode(file)->i_ino,
 				lock->fl.c.flc_type,
@@ -676,7 +675,7 @@ nlmsvc_unlock(struct net *net, struct nlm_file *file, struct nlm_lock *lock)
 {
 	int	error = 0;
 
-	dprintk("lockd: nlmsvc_unlock(%s/%ld, pi=%d, %Ld-%Ld)\n",
+	dprintk("lockd: nlmsvc_unlock(%s/%llu, pi=%d, %Ld-%Ld)\n",
 				nlmsvc_file_inode(file)->i_sb->s_id,
 				nlmsvc_file_inode(file)->i_ino,
 				lock->fl.c.flc_pid,
@@ -716,7 +715,7 @@ nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct nlm_lock *l
 	int status = 0;
 	int mode;
 
-	dprintk("lockd: nlmsvc_cancel(%s/%ld, pi=%d, %Ld-%Ld)\n",
+	dprintk("lockd: nlmsvc_cancel(%s/%llu, pi=%d, %Ld-%Ld)\n",
 				nlmsvc_file_inode(file)->i_sb->s_id,
 				nlmsvc_file_inode(file)->i_ino,
 				lock->fl.c.flc_pid,
diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c
index 5817ef272332d..749abf8886ba7 100644
--- a/fs/lockd/svcproc.c
+++ b/fs/lockd/svcproc.c
@@ -10,39 +10,52 @@
 
 #include <linux/types.h>
 #include <linux/time.h>
-#include <linux/lockd/lockd.h>
-#include <linux/lockd/share.h>
 #include <linux/sunrpc/svc_xprt.h>
 
+#include "lockd.h"
+#include "share.h"
+
 #define NLMDBG_FACILITY		NLMDBG_CLIENT
 
 #ifdef CONFIG_LOCKD_V4
-static __be32
-cast_to_nlm(__be32 status, u32 vers)
+static inline __be32 cast_status(__be32 status)
 {
-	/* Note: status is assumed to be in network byte order !!! */
-	if (vers != 4){
-		switch (status) {
-		case nlm_granted:
-		case nlm_lck_denied:
-		case nlm_lck_denied_nolocks:
-		case nlm_lck_blocked:
-		case nlm_lck_denied_grace_period:
-		case nlm_drop_reply:
-			break;
-		case nlm4_deadlock:
-			status = nlm_lck_denied;
-			break;
-		default:
-			status = nlm_lck_denied_nolocks;
-		}
+	switch (status) {
+	case nlm_granted:
+	case nlm_lck_denied:
+	case nlm_lck_denied_nolocks:
+	case nlm_lck_blocked:
+	case nlm_lck_denied_grace_period:
+	case nlm__int__drop_reply:
+		break;
+	case nlm__int__deadlock:
+		status = nlm_lck_denied;
+		break;
+	default:
+		status = nlm_lck_denied_nolocks;
 	}
 
-	return (status);
+	return status;
 }
-#define	cast_status(status) (cast_to_nlm(status, rqstp->rq_vers))
 #else
-#define cast_status(status) (status)
+static inline __be32 cast_status(__be32 status)
+{
+	switch (status) {
+	case nlm__int__deadlock:
+		status = nlm_lck_denied;
+		break;
+	case nlm__int__stale_fh:
+	case nlm__int__failed:
+		status = nlm_lck_denied_nolocks;
+		break;
+	default:
+		if (be32_to_cpu(status) >= 30000)
+			pr_warn_once("lockd: unhandled internal status %u\n",
+				     be32_to_cpu(status));
+		break;
+	}
+	return status;
+}
 #endif
 
 /*
@@ -124,12 +137,13 @@ __nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Now check for conflicting locks */
 	resp->status = cast_status(nlmsvc_testlock(rqstp, file, host,
 						   &argp->lock, &resp->lock));
-	if (resp->status == nlm_drop_reply)
+	if (resp->status == nlm__int__drop_reply)
 		rc = rpc_drop_reply;
 	else
 		dprintk("lockd: TEST          status %d vers %d\n",
@@ -161,13 +175,14 @@ __nlmsvc_proc_lock(struct svc_rqst *rqstp, struct nlm_res *resp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Now try to lock the file */
 	resp->status = cast_status(nlmsvc_lock(rqstp, file, host, &argp->lock,
 					       argp->block, &argp->cookie,
 					       argp->reclaim));
-	if (resp->status == nlm_drop_reply)
+	if (resp->status == nlm__int__drop_reply)
 		rc = rpc_drop_reply;
 	else
 		dprintk("lockd: LOCK         status %d\n", ntohl(resp->status));
@@ -204,7 +219,8 @@ __nlmsvc_proc_cancel(struct svc_rqst *rqstp, struct nlm_res *resp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Try to cancel request. */
 	resp->status = cast_status(nlmsvc_cancel_blocked(net, file, &argp->lock));
@@ -245,7 +261,8 @@ __nlmsvc_proc_unlock(struct svc_rqst *rqstp, struct nlm_res *resp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Now try to remove the lock */
 	resp->status = cast_status(nlmsvc_unlock(net, file, &argp->lock));
@@ -402,10 +419,13 @@ nlmsvc_proc_share(struct svc_rqst *rqstp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Now try to create the share */
-	resp->status = cast_status(nlmsvc_share_file(host, file, argp));
+	resp->status = cast_status(nlmsvc_share_file(host, file, &argp->lock.oh,
+						     argp->fsm_access,
+						     argp->fsm_mode));
 
 	dprintk("lockd: SHARE         status %d\n", ntohl(resp->status));
 	nlmsvc_release_lockowner(&argp->lock);
@@ -437,10 +457,12 @@ nlmsvc_proc_unshare(struct svc_rqst *rqstp)
 
 	/* Obtain client and file */
 	if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file)))
-		return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success;
+		return resp->status == nlm__int__drop_reply ?
+			rpc_drop_reply : rpc_success;
 
 	/* Now try to unshare the file */
-	resp->status = cast_status(nlmsvc_unshare_file(host, file, argp));
+	resp->status = cast_status(nlmsvc_unshare_file(host, file,
+						       &argp->lock.oh));
 
 	dprintk("lockd: UNSHARE       status %d\n", ntohl(resp->status));
 	nlmsvc_release_lockowner(&argp->lock);
@@ -536,7 +558,7 @@ struct nlm_void			{ int dummy; };
 #define	No	(1+1024/4)			/* Net Obj */
 #define	Rg	2				/* range - offset + size */
 
-const struct svc_procedure nlmsvc_procedures[24] = {
+static const struct svc_procedure nlmsvc_procedures[24] = {
 	[NLMPROC_NULL] = {
 		.pc_func = nlmsvc_proc_null,
 		.pc_decode = nlmsvc_decode_void,
@@ -778,3 +800,39 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_name = "FREE_ALL",
 	},
 };
+
+/*
+ * Storage requirements for XDR arguments and results
+ */
+union nlmsvc_xdrstore {
+	struct nlm_args			args;
+	struct nlm_res			res;
+	struct nlm_reboot		reboot;
+};
+
+/*
+ * NLMv1 defines only procedures 1 - 15. Linux lockd also implements
+ * procedures 0 (NULL) and 16 (SM_NOTIFY).
+ */
+static DEFINE_PER_CPU_ALIGNED(unsigned long, nlm1svc_call_counters[17]);
+
+const struct svc_version nlmsvc_version1 = {
+	.vs_vers	= 1,
+	.vs_nproc	= 17,
+	.vs_proc	= nlmsvc_procedures,
+	.vs_count	= nlm1svc_call_counters,
+	.vs_dispatch	= nlmsvc_dispatch,
+	.vs_xdrsize	= sizeof(union nlmsvc_xdrstore),
+};
+
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+			      nlm3svc_call_counters[ARRAY_SIZE(nlmsvc_procedures)]);
+
+const struct svc_version nlmsvc_version3 = {
+	.vs_vers	= 3,
+	.vs_nproc	= ARRAY_SIZE(nlmsvc_procedures),
+	.vs_proc	= nlmsvc_procedures,
+	.vs_count	= nlm3svc_call_counters,
+	.vs_dispatch	= nlmsvc_dispatch,
+	.vs_xdrsize	= sizeof(union nlmsvc_xdrstore),
+};
diff --git a/fs/lockd/svcshare.c b/fs/lockd/svcshare.c
index 88c81ce1148d9..53f5655c128c4 100644
--- a/fs/lockd/svcshare.c
+++ b/fs/lockd/svcshare.c
@@ -14,8 +14,9 @@
 
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/svc.h>
-#include <linux/lockd/lockd.h>
-#include <linux/lockd/share.h>
+
+#include "lockd.h"
+#include "share.h"
 
 static inline int
 nlm_cmp_owner(struct nlm_share *share, struct xdr_netobj *oh)
@@ -24,12 +25,21 @@ nlm_cmp_owner(struct nlm_share *share, struct xdr_netobj *oh)
 	    && !memcmp(share->s_owner.data, oh->data, oh->len);
 }
 
+/**
+ * nlmsvc_share_file - create a share
+ * @host: Network client peer
+ * @file: File to be shared
+ * @oh: Share owner handle
+ * @access: Requested access mode
+ * @mode: Requested file sharing mode
+ *
+ * Returns an NLM status code.
+ */
 __be32
 nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file,
-			struct nlm_args *argp)
+		  struct xdr_netobj *oh, u32 access, u32 mode)
 {
 	struct nlm_share	*share;
-	struct xdr_netobj	*oh = &argp->lock.oh;
 	u8			*ohdata;
 
 	if (nlmsvc_file_cannot_lock(file))
@@ -38,13 +48,11 @@ nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file,
 	for (share = file->f_shares; share; share = share->s_next) {
 		if (share->s_host == host && nlm_cmp_owner(share, oh))
 			goto update;
-		if ((argp->fsm_access & share->s_mode)
-		 || (argp->fsm_mode   & share->s_access ))
+		if ((access & share->s_mode) || (mode & share->s_access))
 			return nlm_lck_denied;
 	}
 
-	share = kmalloc(sizeof(*share) + oh->len,
-						GFP_KERNEL);
+	share = kmalloc(sizeof(*share) + oh->len, GFP_KERNEL);
 	if (share == NULL)
 		return nlm_lck_denied_nolocks;
 
@@ -60,20 +68,24 @@ nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file,
 	file->f_shares      = share;
 
 update:
-	share->s_access = argp->fsm_access;
-	share->s_mode   = argp->fsm_mode;
+	share->s_access = access;
+	share->s_mode = mode;
 	return nlm_granted;
 }
 
-/*
- * Delete a share.
+/**
+ * nlmsvc_unshare_file - delete a share
+ * @host: Network client peer
+ * @file: File to be unshared
+ * @oh: Share owner handle
+ *
+ * Returns an NLM status code.
  */
 __be32
 nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file,
-			struct nlm_args *argp)
+		    struct xdr_netobj *oh)
 {
 	struct nlm_share	*share, **shpp;
-	struct xdr_netobj	*oh = &argp->lock.oh;
 
 	if (nlmsvc_file_cannot_lock(file))
 		return nlm_lck_denied_nolocks;
diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c
index dd0214dcb6950..344e6c187cde2 100644
--- a/fs/lockd/svcsubs.c
+++ b/fs/lockd/svcsubs.c
@@ -15,12 +15,13 @@
 #include <linux/mutex.h>
 #include <linux/sunrpc/svc.h>
 #include <linux/sunrpc/addr.h>
-#include <linux/lockd/lockd.h>
-#include <linux/lockd/share.h>
 #include <linux/module.h>
 #include <linux/mount.h>
 #include <uapi/linux/nfs2.h>
 
+#include "lockd.h"
+#include "share.h"
+
 #define NLMDBG_FACILITY		NLMDBG_SVCSUBS
 
 
@@ -47,7 +48,7 @@ static inline void nlm_debug_print_file(char *msg, struct nlm_file *file)
 {
 	struct inode *inode = nlmsvc_file_inode(file);
 
-	dprintk("lockd: %s %s/%ld\n",
+	dprintk("lockd: %s %s/%llu\n",
 		msg, inode->i_sb->s_id, inode->i_ino);
 }
 #else
@@ -87,14 +88,29 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
 			   struct nlm_file *file, int mode)
 {
 	struct file **fp = &file->f_file[mode];
-	__be32	nfserr;
+	__be32 nlmerr = nlm_granted;
+	int error;
 
 	if (*fp)
-		return 0;
-	nfserr = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, mode);
-	if (nfserr)
-		dprintk("lockd: open failed (error %d)\n", nfserr);
-	return nfserr;
+		return nlmerr;
+
+	error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, mode);
+	if (error) {
+		dprintk("lockd: open failed (errno %d)\n", error);
+		switch (error) {
+		case -EWOULDBLOCK:
+			nlmerr = nlm__int__drop_reply;
+			break;
+		case -ESTALE:
+			nlmerr = nlm__int__stale_fh;
+			break;
+		default:
+			nlmerr = nlm__int__failed;
+			break;
+		}
+	}
+
+	return nlmerr;
 }
 
 /*
diff --git a/fs/lockd/trace.h b/fs/lockd/trace.h
index 7461b13b6e744..7214d7e96a424 100644
--- a/fs/lockd/trace.h
+++ b/fs/lockd/trace.h
@@ -8,7 +8,8 @@
 #include <linux/tracepoint.h>
 #include <linux/crc32.h>
 #include <linux/nfs.h>
-#include <linux/lockd/lockd.h>
+
+#include "lockd.h"
 
 #ifdef CONFIG_LOCKD_V4
 #define NLM_STATUS_LIST					\
diff --git a/fs/lockd/xdr.c b/fs/lockd/xdr.c
index adfcce2bf11ba..dfca8b8dab733 100644
--- a/fs/lockd/xdr.c
+++ b/fs/lockd/xdr.c
@@ -15,13 +15,13 @@
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/svc.h>
 #include <linux/sunrpc/stats.h>
-#include <linux/lockd/lockd.h>
 
 #include <uapi/linux/nfs2.h>
 
+#include "lockd.h"
+#include "share.h"
 #include "svcxdr.h"
 
-
 static inline loff_t
 s32_to_loff_t(__s32 offset)
 {
@@ -275,7 +275,7 @@ nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 
 	memset(lock, 0, sizeof(*lock));
 	locks_init_lock(&lock->fl);
-	lock->svid = ~(u32)0;
+	lock->svid = LOCKD_SHARE_SVID;
 
 	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
 		return false;
diff --git a/fs/lockd/xdr.h b/fs/lockd/xdr.h
new file mode 100644
index 0000000000000..3c60817c4349a
--- /dev/null
+++ b/fs/lockd/xdr.h
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * XDR types for the NLM protocol
+ *
+ * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
+ */
+
+#ifndef _LOCKD_XDR_H
+#define _LOCKD_XDR_H
+
+#include <linux/fs.h>
+#include <linux/filelock.h>
+#include <linux/nfs.h>
+#include <linux/sunrpc/xdr.h>
+
+#define SM_MAXSTRLEN		1024
+#define SM_PRIV_SIZE		16
+
+struct nsm_private {
+	unsigned char		data[SM_PRIV_SIZE];
+};
+
+struct svc_rqst;
+
+#define NLM_MAXCOOKIELEN    	32
+#define NLM_MAXSTRLEN		1024
+
+#define	nlm_granted		cpu_to_be32(NLM_LCK_GRANTED)
+#define	nlm_lck_denied		cpu_to_be32(NLM_LCK_DENIED)
+#define	nlm_lck_denied_nolocks	cpu_to_be32(NLM_LCK_DENIED_NOLOCKS)
+#define	nlm_lck_blocked		cpu_to_be32(NLM_LCK_BLOCKED)
+#define	nlm_lck_denied_grace_period	cpu_to_be32(NLM_LCK_DENIED_GRACE_PERIOD)
+
+/* Lock info passed via NLM */
+struct nlm_lock {
+	char *			caller;
+	unsigned int		len; 	/* length of "caller" */
+	struct nfs_fh		fh;
+	struct xdr_netobj	oh;
+	u32			svid;
+	u64			lock_start;
+	u64			lock_len;
+	struct file_lock	fl;
+};
+
+/*
+ *	NLM cookies. Technically they can be 1K, but Linux only uses 8 bytes.
+ *	FreeBSD uses 16, Apple Mac OS X 10.3 uses 20. Therefore we set it to
+ *	32 bytes.
+ */
+
+struct nlm_cookie
+{
+	unsigned char data[NLM_MAXCOOKIELEN];
+	unsigned int len;
+};
+
+/*
+ * Generic lockd arguments for all but sm_notify
+ */
+struct nlm_args {
+	struct nlm_cookie	cookie;
+	struct nlm_lock		lock;
+	u32			block;
+	u32			reclaim;
+	u32			state;
+	u32			monitor;
+	u32			fsm_access;
+	u32			fsm_mode;
+};
+
+/*
+ * Generic lockd result
+ */
+struct nlm_res {
+	struct nlm_cookie	cookie;
+	__be32			status;
+	struct nlm_lock		lock;
+};
+
+/*
+ * statd callback when client has rebooted
+ */
+struct nlm_reboot {
+	char			*mon;
+	unsigned int		len;
+	u32			state;
+	struct nsm_private	priv;
+};
+
+bool	nlmsvc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+
+bool	nlmsvc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+bool	nlmsvc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
+
+#endif /* _LOCKD_XDR_H */
diff --git a/fs/lockd/xdr4.c b/fs/lockd/xdr4.c
deleted file mode 100644
index e343c820301fd..0000000000000
--- a/fs/lockd/xdr4.c
+++ /dev/null
@@ -1,347 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * linux/fs/lockd/xdr4.c
- *
- * XDR support for lockd and the lock client.
- *
- * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
- * Copyright (C) 1999, Trond Myklebust <trond.myklebust@fys.uio.no>
- */
-
-#include <linux/types.h>
-#include <linux/sched.h>
-#include <linux/nfs.h>
-
-#include <linux/sunrpc/xdr.h>
-#include <linux/sunrpc/clnt.h>
-#include <linux/sunrpc/svc.h>
-#include <linux/sunrpc/stats.h>
-#include <linux/lockd/lockd.h>
-
-#include "svcxdr.h"
-
-static inline s64
-loff_t_to_s64(loff_t offset)
-{
-	s64 res;
-	if (offset > NLM4_OFFSET_MAX)
-		res = NLM4_OFFSET_MAX;
-	else if (offset < -NLM4_OFFSET_MAX)
-		res = -NLM4_OFFSET_MAX;
-	else
-		res = offset;
-	return res;
-}
-
-void nlm4svc_set_file_lock_range(struct file_lock *fl, u64 off, u64 len)
-{
-	s64 end = off + len - 1;
-
-	fl->fl_start = off;
-	if (len == 0 || end < 0)
-		fl->fl_end = OFFSET_MAX;
-	else
-		fl->fl_end = end;
-}
-
-/*
- * NLM file handles are defined by specification to be a variable-length
- * XDR opaque no longer than 1024 bytes. However, this implementation
- * limits their length to the size of an NFSv3 file handle.
- */
-static bool
-svcxdr_decode_fhandle(struct xdr_stream *xdr, struct nfs_fh *fh)
-{
-	__be32 *p;
-	u32 len;
-
-	if (xdr_stream_decode_u32(xdr, &len) < 0)
-		return false;
-	if (len > NFS_MAXFHSIZE)
-		return false;
-
-	p = xdr_inline_decode(xdr, len);
-	if (!p)
-		return false;
-	fh->size = len;
-	memcpy(fh->data, p, len);
-	memset(fh->data + len, 0, sizeof(fh->data) - len);
-
-	return true;
-}
-
-static bool
-svcxdr_decode_lock(struct xdr_stream *xdr, struct nlm_lock *lock)
-{
-	struct file_lock *fl = &lock->fl;
-
-	if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len))
-		return false;
-	if (!svcxdr_decode_fhandle(xdr, &lock->fh))
-		return false;
-	if (!svcxdr_decode_owner(xdr, &lock->oh))
-		return false;
-	if (xdr_stream_decode_u32(xdr, &lock->svid) < 0)
-		return false;
-	if (xdr_stream_decode_u64(xdr, &lock->lock_start) < 0)
-		return false;
-	if (xdr_stream_decode_u64(xdr, &lock->lock_len) < 0)
-		return false;
-
-	locks_init_lock(fl);
-	fl->c.flc_type  = F_RDLCK;
-	nlm4svc_set_file_lock_range(fl, lock->lock_start, lock->lock_len);
-	return true;
-}
-
-static bool
-svcxdr_encode_holder(struct xdr_stream *xdr, const struct nlm_lock *lock)
-{
-	const struct file_lock *fl = &lock->fl;
-	s64 start, len;
-
-	/* exclusive */
-	if (xdr_stream_encode_bool(xdr, fl->c.flc_type != F_RDLCK) < 0)
-		return false;
-	if (xdr_stream_encode_u32(xdr, lock->svid) < 0)
-		return false;
-	if (!svcxdr_encode_owner(xdr, &lock->oh))
-		return false;
-	start = loff_t_to_s64(fl->fl_start);
-	if (fl->fl_end == OFFSET_MAX)
-		len = 0;
-	else
-		len = loff_t_to_s64(fl->fl_end - fl->fl_start + 1);
-	if (xdr_stream_encode_u64(xdr, start) < 0)
-		return false;
-	if (xdr_stream_encode_u64(xdr, len) < 0)
-		return false;
-
-	return true;
-}
-
-static bool
-svcxdr_encode_testrply(struct xdr_stream *xdr, const struct nlm_res *resp)
-{
-	if (!svcxdr_encode_stats(xdr, resp->status))
-		return false;
-	switch (resp->status) {
-	case nlm_lck_denied:
-		if (!svcxdr_encode_holder(xdr, &resp->lock))
-			return false;
-	}
-
-	return true;
-}
-
-
-/*
- * Decode Call arguments
- */
-
-bool
-nlm4svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	return true;
-}
-
-bool
-nlm4svc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	u32 exclusive;
-
-	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
-		return false;
-	if (xdr_stream_decode_bool(xdr, &exclusive) < 0)
-		return false;
-	if (!svcxdr_decode_lock(xdr, &argp->lock))
-		return false;
-	if (exclusive)
-		argp->lock.fl.c.flc_type = F_WRLCK;
-
-	return true;
-}
-
-bool
-nlm4svc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	u32 exclusive;
-
-	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
-		return false;
-	if (xdr_stream_decode_bool(xdr, &argp->block) < 0)
-		return false;
-	if (xdr_stream_decode_bool(xdr, &exclusive) < 0)
-		return false;
-	if (!svcxdr_decode_lock(xdr, &argp->lock))
-		return false;
-	if (exclusive)
-		argp->lock.fl.c.flc_type = F_WRLCK;
-	if (xdr_stream_decode_bool(xdr, &argp->reclaim) < 0)
-		return false;
-	if (xdr_stream_decode_u32(xdr, &argp->state) < 0)
-		return false;
-	argp->monitor = 1;		/* monitor client by default */
-
-	return true;
-}
-
-bool
-nlm4svc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	u32 exclusive;
-
-	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
-		return false;
-	if (xdr_stream_decode_bool(xdr, &argp->block) < 0)
-		return false;
-	if (xdr_stream_decode_bool(xdr, &exclusive) < 0)
-		return false;
-	if (!svcxdr_decode_lock(xdr, &argp->lock))
-		return false;
-	if (exclusive)
-		argp->lock.fl.c.flc_type = F_WRLCK;
-
-	return true;
-}
-
-bool
-nlm4svc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-
-	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
-		return false;
-	if (!svcxdr_decode_lock(xdr, &argp->lock))
-		return false;
-	argp->lock.fl.c.flc_type = F_UNLCK;
-
-	return true;
-}
-
-bool
-nlm4svc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_res *resp = rqstp->rq_argp;
-
-	if (!svcxdr_decode_cookie(xdr, &resp->cookie))
-		return false;
-	if (!svcxdr_decode_stats(xdr, &resp->status))
-		return false;
-
-	return true;
-}
-
-bool
-nlm4svc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_reboot *argp = rqstp->rq_argp;
-	__be32 *p;
-	u32 len;
-
-	if (xdr_stream_decode_u32(xdr, &len) < 0)
-		return false;
-	if (len > SM_MAXSTRLEN)
-		return false;
-	p = xdr_inline_decode(xdr, len);
-	if (!p)
-		return false;
-	argp->len = len;
-	argp->mon = (char *)p;
-	if (xdr_stream_decode_u32(xdr, &argp->state) < 0)
-		return false;
-	p = xdr_inline_decode(xdr, SM_PRIV_SIZE);
-	if (!p)
-		return false;
-	memcpy(&argp->priv.data, p, sizeof(argp->priv.data));
-
-	return true;
-}
-
-bool
-nlm4svc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_lock	*lock = &argp->lock;
-
-	locks_init_lock(&lock->fl);
-	lock->svid = ~(u32)0;
-
-	if (!svcxdr_decode_cookie(xdr, &argp->cookie))
-		return false;
-	if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len))
-		return false;
-	if (!svcxdr_decode_fhandle(xdr, &lock->fh))
-		return false;
-	if (!svcxdr_decode_owner(xdr, &lock->oh))
-		return false;
-	/* XXX: Range checks are missing in the original code */
-	if (xdr_stream_decode_u32(xdr, &argp->fsm_mode) < 0)
-		return false;
-	if (xdr_stream_decode_u32(xdr, &argp->fsm_access) < 0)
-		return false;
-
-	return true;
-}
-
-bool
-nlm4svc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_args *argp = rqstp->rq_argp;
-	struct nlm_lock	*lock = &argp->lock;
-
-	if (!svcxdr_decode_string(xdr, &lock->caller, &lock->len))
-		return false;
-	if (xdr_stream_decode_u32(xdr, &argp->state) < 0)
-		return false;
-
-	return true;
-}
-
-
-/*
- * Encode Reply results
- */
-
-bool
-nlm4svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	return true;
-}
-
-bool
-nlm4svc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_res *resp = rqstp->rq_resp;
-
-	return svcxdr_encode_cookie(xdr, &resp->cookie) &&
-		svcxdr_encode_testrply(xdr, resp);
-}
-
-bool
-nlm4svc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_res *resp = rqstp->rq_resp;
-
-	return svcxdr_encode_cookie(xdr, &resp->cookie) &&
-		svcxdr_encode_stats(xdr, resp->status);
-}
-
-bool
-nlm4svc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
-{
-	struct nlm_res *resp = rqstp->rq_resp;
-
-	if (!svcxdr_encode_cookie(xdr, &resp->cookie))
-		return false;
-	if (!svcxdr_encode_stats(xdr, resp->status))
-		return false;
-	/* sequence */
-	if (xdr_stream_encode_u32(xdr, 0) < 0)
-		return false;
-
-	return true;
-}
diff --git a/fs/locks.c b/fs/locks.c
index d13ec930b7bb0..fead53474c300 100644
--- a/fs/locks.c
+++ b/fs/locks.c
@@ -234,7 +234,7 @@ locks_check_ctx_lists(struct inode *inode)
 	if (unlikely(!list_empty(&ctx->flc_flock) ||
 		     !list_empty(&ctx->flc_posix) ||
 		     !list_empty(&ctx->flc_lease))) {
-		pr_warn("Leaked locks on dev=0x%x:0x%x ino=0x%lx:\n",
+		pr_warn("Leaked locks on dev=0x%x:0x%x ino=0x%llx:\n",
 			MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev),
 			inode->i_ino);
 		locks_dump_ctx_list(&ctx->flc_flock, "FLOCK");
@@ -251,7 +251,7 @@ locks_check_ctx_file_list(struct file *filp, struct list_head *list, char *list_
 
 	list_for_each_entry(flc, list, flc_list)
 		if (flc->flc_file == filp)
-			pr_warn("Leaked %s lock on dev=0x%x:0x%x ino=0x%lx "
+			pr_warn("Leaked %s lock on dev=0x%x:0x%x ino=0x%llx "
 				" fl_owner=%p fl_flags=0x%x fl_type=0x%x fl_pid=%u\n",
 				list_type, MAJOR(inode->i_sb->s_dev),
 				MINOR(inode->i_sb->s_dev), inode->i_ino,
@@ -1534,6 +1534,7 @@ static void time_out_leases(struct inode *inode, struct list_head *dispose)
 {
 	struct file_lock_context *ctx = inode->i_flctx;
 	struct file_lease *fl, *tmp;
+	bool remove;
 
 	lockdep_assert_held(&ctx->flc_lock);
 
@@ -1541,8 +1542,19 @@ static void time_out_leases(struct inode *inode, struct list_head *dispose)
 		trace_time_out_leases(inode, fl);
 		if (past_time(fl->fl_downgrade_time))
 			lease_modify(fl, F_RDLCK, dispose);
-		if (past_time(fl->fl_break_time))
-			lease_modify(fl, F_UNLCK, dispose);
+
+		remove = true;
+		if (past_time(fl->fl_break_time)) {
+			/*
+			 * Consult the lease manager when a lease break times
+			 * out to determine whether the lease should be disposed
+			 * of.
+			 */
+			if (fl->fl_lmops && fl->fl_lmops->lm_breaker_timedout)
+				remove = fl->fl_lmops->lm_breaker_timedout(fl);
+			if (remove)
+				lease_modify(fl, F_UNLCK, dispose);
+		}
 	}
 }
 
@@ -1670,9 +1682,13 @@ int __break_lease(struct inode *inode, unsigned int flags)
 restart:
 	fl = list_first_entry(&ctx->flc_lease, struct file_lease, c.flc_list);
 	break_time = fl->fl_break_time;
-	if (break_time != 0)
-		break_time -= jiffies;
-	if (break_time == 0)
+	if (break_time != 0) {
+		if (time_after(jiffies, break_time)) {
+			fl->fl_break_time = jiffies + lease_break_time * HZ;
+			break_time = lease_break_time * HZ;
+		} else
+			break_time -= jiffies;
+	} else
 		break_time++;
 	locks_insert_block(&fl->c, &new_fl->c, leases_conflict);
 	trace_break_lease_block(inode, new_fl);
@@ -2896,7 +2912,7 @@ static void lock_get_status(struct seq_file *f, struct file_lock_core *flc,
 			     (type == F_RDLCK) ? "READ" : "UNLCK");
 	if (inode) {
 		/* userspace relies on this representation of dev_t */
-		seq_printf(f, "%d %02x:%02x:%lu ", pid,
+		seq_printf(f, "%d %02x:%02x:%llu ", pid,
 				MAJOR(inode->i_sb->s_dev),
 				MINOR(inode->i_sb->s_dev), inode->i_ino);
 	} else {
diff --git a/fs/minix/inode.c b/fs/minix/inode.c
index 99541c6a5bbf1..838b072b6cf0b 100644
--- a/fs/minix/inode.c
+++ b/fs/minix/inode.c
@@ -36,7 +36,7 @@ void __minix_error_inode(struct inode *inode, const char *function,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 	printk(KERN_CRIT "minix-fs error (device %s): %s:%d: "
-	       "inode #%lu: comm %s: %pV\n",
+	       "inode #%llu: comm %s: %pV\n",
 	       inode->i_sb->s_id, function, line, inode->i_ino,
 	       current->comm, &vaf);
 	va_end(args);
@@ -520,7 +520,7 @@ void minix_set_inode(struct inode *inode, dev_t rdev)
 		   S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
 		init_special_inode(inode, inode->i_mode, rdev);
 	} else {
-		printk(KERN_DEBUG "MINIX-fs: Invalid file type 0%04o for inode %lu.\n",
+		printk(KERN_DEBUG "MINIX-fs: Invalid file type 0%04o for inode %llu.\n",
 		       inode->i_mode, inode->i_ino);
 		make_bad_inode(inode);
 	}
@@ -542,7 +542,7 @@ static struct inode *V1_minix_iget(struct inode *inode)
 		return ERR_PTR(-EIO);
 	}
 	if (raw_inode->i_nlinks == 0) {
-		printk("MINIX-fs: deleted inode referenced: %lu\n",
+		printk("MINIX-fs: deleted inode referenced: %llu\n",
 		       inode->i_ino);
 		brelse(bh);
 		iget_failed(inode);
@@ -580,7 +580,7 @@ static struct inode *V2_minix_iget(struct inode *inode)
 		return ERR_PTR(-EIO);
 	}
 	if (raw_inode->i_nlinks == 0) {
-		printk("MINIX-fs: deleted inode referenced: %lu\n",
+		printk("MINIX-fs: deleted inode referenced: %llu\n",
 		       inode->i_ino);
 		brelse(bh);
 		iget_failed(inode);
@@ -692,7 +692,7 @@ static int minix_write_inode(struct inode *inode, struct writeback_control *wbc)
 	if (wbc->sync_mode == WB_SYNC_ALL && buffer_dirty(bh)) {
 		sync_dirty_buffer(bh);
 		if (buffer_req(bh) && !buffer_uptodate(bh)) {
-			printk("IO error syncing minix inode [%s:%08lx]\n",
+			printk("IO error syncing minix inode [%s:%08llx]\n",
 				inode->i_sb->s_id, inode->i_ino);
 			err = -EIO;
 		}
diff --git a/fs/mpage.c b/fs/mpage.c
index 7dae5afc2b9ee..e5285fbfcf095 100644
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -28,7 +28,6 @@
 #include <linux/mm_inline.h>
 #include <linux/writeback.h>
 #include <linux/backing-dev.h>
-#include <linux/pagevec.h>
 #include "internal.h"
 
 /*
diff --git a/fs/namei.c b/fs/namei.c
index 58f715f7657e2..77189335bbcc3 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1782,8 +1782,8 @@ static struct dentry *lookup_dcache(const struct qstr *name,
  * Will return -ENOENT if name isn't found and LOOKUP_CREATE wasn't passed.
  * Will return -EEXIST if name is found and LOOKUP_EXCL was passed.
  */
-struct dentry *lookup_one_qstr_excl(const struct qstr *name,
-				    struct dentry *base, unsigned int flags)
+static struct dentry *lookup_one_qstr_excl(const struct qstr *name,
+					   struct dentry *base, unsigned int flags)
 {
 	struct dentry *dentry;
 	struct dentry *old;
@@ -1820,7 +1820,6 @@ struct dentry *lookup_one_qstr_excl(const struct qstr *name,
 	}
 	return dentry;
 }
-EXPORT_SYMBOL(lookup_one_qstr_excl);
 
 /**
  * lookup_fast - do fast lockless (but racy) lookup of a dentry
@@ -2893,20 +2892,6 @@ static int filename_parentat(int dfd, struct filename *name,
 	return __filename_parentat(dfd, name, flags, parent, last, type, NULL);
 }
 
-/**
- * __start_dirop - begin a create or remove dirop, performing locking and lookup
- * @parent:       the dentry of the parent in which the operation will occur
- * @name:         a qstr holding the name within that parent
- * @lookup_flags: intent and other lookup flags.
- * @state:        task state bitmask
- *
- * The lookup is performed and necessary locks are taken so that, on success,
- * the returned dentry can be operated on safely.
- * The qstr must already have the hash value calculated.
- *
- * Returns: a locked dentry, or an error.
- *
- */
 static struct dentry *__start_dirop(struct dentry *parent, struct qstr *name,
 				    unsigned int lookup_flags,
 				    unsigned int state)
@@ -2928,6 +2913,19 @@ static struct dentry *__start_dirop(struct dentry *parent, struct qstr *name,
 	return dentry;
 }
 
+/**
+ * start_dirop - begin a create or remove dirop, performing locking and lookup
+ * @parent:       the dentry of the parent in which the operation will occur
+ * @name:         a qstr holding the name within that parent
+ * @lookup_flags: intent and other lookup flags.
+ *
+ * The lookup is performed and necessary locks are taken so that, on success,
+ * the returned dentry can be operated on safely.
+ * The qstr must already have the hash value calculated.
+ *
+ * Returns: a locked dentry, or an error.
+ *
+ */
 struct dentry *start_dirop(struct dentry *parent, struct qstr *name,
 			   unsigned int lookup_flags)
 {
@@ -3124,7 +3122,8 @@ static int lookup_one_common(struct mnt_idmap *idmap,
  * @base:	base directory to lookup from
  *
  * Look up a dentry by name in the dcache, returning NULL if it does not
- * currently exist.  The function does not try to create a dentry and if one
+ * currently exist or an error if there is a problem with the name.
+ * The function does not try to create a dentry and if one
  * is found it doesn't try to revalidate it.
  *
  * Note that this routine is purely a helper for filesystem usage and should
@@ -3132,6 +3131,11 @@ static int lookup_one_common(struct mnt_idmap *idmap,
  *
  * No locks need be held - only a counted reference to @base is needed.
  *
+ * Returns:
+ *   - ref-counted dentry on success, or
+ *   - %NULL if name could not be found, or
+ *   - ERR_PTR(-EACCES) if name is dot or dotdot or contains a slash or nul, or
+ *   - ERR_PTR() if fs provide ->d_hash, and this returned an error.
  */
 struct dentry *try_lookup_noperm(struct qstr *name, struct dentry *base)
 {
@@ -3208,6 +3212,11 @@ EXPORT_SYMBOL(lookup_one);
  *
  * Unlike lookup_one, it should be called without the parent
  * i_rwsem held, and will take the i_rwsem itself if necessary.
+ *
+ * Returns: - A dentry, possibly negative, or
+ *	    - same errors as try_lookup_noperm() or
+ *	    - ERR_PTR(-ENOENT) if parent has been removed, or
+ *	    - ERR_PTR(-EACCES) if parent directory is not searchable.
  */
 struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap, struct qstr *name,
 				   struct dentry *base)
@@ -3244,6 +3253,10 @@ EXPORT_SYMBOL(lookup_one_unlocked);
  * It should be called without the parent i_rwsem held, and will take
  * the i_rwsem itself if necessary.  If a fatal signal is pending or
  * delivered, it will return %-EINTR if the lock is needed.
+ *
+ * Returns: A dentry, possibly negative, or
+ *	   - same errors as lookup_one_unlocked() or
+ *	   - ERR_PTR(-EINTR) if a fatal signal is pending.
  */
 struct dentry *lookup_one_positive_killable(struct mnt_idmap *idmap,
 					    struct qstr *name,
@@ -3283,6 +3296,10 @@ EXPORT_SYMBOL(lookup_one_positive_killable);
  * This can be used for in-kernel filesystem clients such as file servers.
  *
  * The helper should be called without i_rwsem held.
+ *
+ * Returns: A positive dentry, or
+ *	   - ERR_PTR(-ENOENT) if the name could not be found, or
+ *	   - same errors as lookup_one_unlocked().
  */
 struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap,
 					    struct qstr *name,
@@ -3311,6 +3328,10 @@ EXPORT_SYMBOL(lookup_one_positive_unlocked);
  *
  * Unlike try_lookup_noperm() it *does* revalidate the dentry if it already
  * existed.
+ *
+ * Returns: A dentry, possibly negative, or
+ *	   - ERR_PTR(-ENOENT) if parent has been removed, or
+ *	   - same errors as try_lookup_noperm()
  */
 struct dentry *lookup_noperm_unlocked(struct qstr *name, struct dentry *base)
 {
@@ -3335,6 +3356,10 @@ EXPORT_SYMBOL(lookup_noperm_unlocked);
  * _can_ become positive at any time, so callers of lookup_noperm_unlocked()
  * need to be very careful; pinned positives have ->d_inode stable, so
  * this one avoids such problems.
+ *
+ * Returns: A positive dentry, or
+ *	   - ERR_PTR(-ENOENT) if name cannot be found or parent has been removed, or
+ *	   - same errors as try_lookup_noperm()
  */
 struct dentry *lookup_noperm_positive_unlocked(struct qstr *name,
 					       struct dentry *base)
@@ -3750,7 +3775,7 @@ static struct dentry *lock_two_directories(struct dentry *p1, struct dentry *p2)
 /*
  * p1 and p2 should be directories on the same fs.
  */
-struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
+static struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
 {
 	if (p1 == p2) {
 		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
@@ -3760,12 +3785,11 @@ struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
 	mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
 	return lock_two_directories(p1, p2);
 }
-EXPORT_SYMBOL(lock_rename);
 
 /*
  * c1 and p2 should be on the same fs.
  */
-struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2)
+static struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2)
 {
 	if (READ_ONCE(c1->d_parent) == p2) {
 		/*
@@ -3802,9 +3826,8 @@ struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2)
 	mutex_unlock(&c1->d_sb->s_vfs_rename_mutex);
 	return NULL;
 }
-EXPORT_SYMBOL(lock_rename_child);
 
-void unlock_rename(struct dentry *p1, struct dentry *p2)
+static void unlock_rename(struct dentry *p1, struct dentry *p2)
 {
 	inode_unlock(p1->d_inode);
 	if (p1 != p2) {
@@ -3812,7 +3835,6 @@ void unlock_rename(struct dentry *p1, struct dentry *p2)
 		mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
 	}
 }
-EXPORT_SYMBOL(unlock_rename);
 
 /**
  * __start_renaming - lookup and lock names for rename
diff --git a/fs/namespace.c b/fs/namespace.c
index 854f4fc664697..cf4d94e822c65 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -2646,6 +2646,19 @@ static int attach_recursive_mnt(struct mount *source_mnt,
 
 			if (unlikely(shorter) && child != source_mnt)
 				mp = shorter;
+			/*
+			 * If @q was locked it was meant to hide
+			 * whatever was under it. Let @child take over
+			 * that job and lock it, then we can unlock @q.
+			 * That'll allow another namespace to shed @q
+			 * and reveal @child. Clearly, that mounter
+			 * consented to this by not severing the mount
+			 * relationship. Otherwise, what's the point.
+			 */
+			if (IS_MNT_LOCKED(q)) {
+				child->mnt.mnt_flags |= MNT_LOCKED;
+				q->mnt.mnt_flags &= ~MNT_LOCKED;
+			}
 			mnt_change_mountpoint(r, mp, q);
 		}
 	}
@@ -2722,7 +2735,7 @@ static inline struct mount *where_to_mount(const struct path *path,
  * In all cases the location must not have been unmounted and the
  * chosen mountpoint must be allowed to be mounted on.  For "beneath"
  * case we also require the location to be at the root of a mount
- * that has a parent (i.e. is not a root of some namespace).
+ * that has something mounted on top of it (i.e. has an overmount).
  */
 static void do_lock_mount(const struct path *path,
 			  struct pinned_mountpoint *res,
@@ -2958,10 +2971,9 @@ static inline bool may_copy_tree(const struct path *path)
 }
 
 static struct mount *__do_loopback(const struct path *old_path,
-				   unsigned int flags, unsigned int copy_flags)
+				   bool recurse, unsigned int copy_flags)
 {
 	struct mount *old = real_mount(old_path->mnt);
-	bool recurse = flags & AT_RECURSIVE;
 
 	if (IS_MNT_UNBINDABLE(old))
 		return ERR_PTR(-EINVAL);
@@ -2972,18 +2984,6 @@ static struct mount *__do_loopback(const struct path *old_path,
 	if (!recurse && __has_locked_children(old, old_path->dentry))
 		return ERR_PTR(-EINVAL);
 
-	/*
-	 * When creating a new mount namespace we don't want to copy over
-	 * mounts of mount namespaces to avoid the risk of cycles and also to
-	 * minimize the default complex interdependencies between mount
-	 * namespaces.
-	 *
-	 * We could ofc just check whether all mount namespace files aren't
-	 * creating cycles but really let's keep this simple.
-	 */
-	if (!(flags & OPEN_TREE_NAMESPACE))
-		copy_flags |= CL_COPY_MNT_NS_FILE;
-
 	if (recurse)
 		return copy_tree(old, old_path->dentry, copy_flags);
 
@@ -2998,7 +2998,6 @@ static int do_loopback(const struct path *path, const char *old_name,
 {
 	struct path old_path __free(path_put) = {};
 	struct mount *mnt = NULL;
-	unsigned int flags = recurse ? AT_RECURSIVE : 0;
 	int err;
 
 	if (!old_name || !*old_name)
@@ -3017,7 +3016,7 @@ static int do_loopback(const struct path *path, const char *old_name,
 	if (!check_mnt(mp.parent))
 		return -EINVAL;
 
-	mnt = __do_loopback(&old_path, flags, 0);
+	mnt = __do_loopback(&old_path, recurse, CL_COPY_MNT_NS_FILE);
 	if (IS_ERR(mnt))
 		return PTR_ERR(mnt);
 
@@ -3055,7 +3054,7 @@ static struct mnt_namespace *get_detached_copy(const struct path *path, unsigned
 			ns->seq_origin = src_mnt_ns->ns.ns_id;
 	}
 
-	mnt = __do_loopback(path, flags, 0);
+	mnt = __do_loopback(path, (flags & AT_RECURSIVE), CL_COPY_MNT_NS_FILE);
 	if (IS_ERR(mnt)) {
 		emptied_ns = ns;
 		return ERR_CAST(mnt);
@@ -3087,7 +3086,8 @@ static struct file *open_detached_copy(struct path *path, unsigned int flags)
 	return file;
 }
 
-static struct mnt_namespace *create_new_namespace(struct path *path, unsigned int flags)
+static struct mnt_namespace *create_new_namespace(struct path *path,
+						  bool recurse)
 {
 	struct mnt_namespace *ns = current->nsproxy->mnt_ns;
 	struct user_namespace *user_ns = current_user_ns();
@@ -3131,11 +3131,26 @@ static struct mnt_namespace *create_new_namespace(struct path *path, unsigned in
 	}
 
 	/*
-	 * We don't emulate unshare()ing a mount namespace. We stick
-	 * to the restrictions of creating detached bind-mounts. It
-	 * has a lot saner and simpler semantics.
+	 * We don't emulate unshare()ing a mount namespace. We stick to
+	 * the restrictions of creating detached bind-mounts. It has a
+	 * lot saner and simpler semantics.
 	 */
-	mnt = __do_loopback(path, flags, copy_flags);
+	mnt = real_mount(path->mnt);
+	if (!mnt->mnt_ns) {
+		/*
+		 * If we're moving into a new mount namespace via
+		 * fsmount() swap the mount ids so the nullfs mount id
+		 * is the lowest in the mount namespace avoiding another
+		 * useless copy. This is fine we're not attached to any
+		 * mount namespace so the mount ids are pure decoration
+		 * at that point.
+		 */
+		swap(mnt->mnt_id_unique, new_ns_root->mnt_id_unique);
+		swap(mnt->mnt_id, new_ns_root->mnt_id);
+		mntget(&mnt->mnt);
+	} else {
+		mnt = __do_loopback(path, recurse, copy_flags);
+	}
 	scoped_guard(mount_writer) {
 		if (IS_ERR(mnt)) {
 			emptied_ns = new_ns;
@@ -3164,11 +3179,11 @@ static struct mnt_namespace *create_new_namespace(struct path *path, unsigned in
 	return new_ns;
 }
 
-static struct file *open_new_namespace(struct path *path, unsigned int flags)
+static struct file *open_new_namespace(struct path *path, bool recurse)
 {
 	struct mnt_namespace *new_ns;
 
-	new_ns = create_new_namespace(path, flags);
+	new_ns = create_new_namespace(path, recurse);
 	if (IS_ERR(new_ns))
 		return ERR_CAST(new_ns);
 	return open_namespace_file(to_ns_common(new_ns));
@@ -3217,7 +3232,7 @@ static struct file *vfs_open_tree(int dfd, const char __user *filename, unsigned
 		return ERR_PTR(ret);
 
 	if (flags & OPEN_TREE_NAMESPACE)
-		return open_new_namespace(&path, flags);
+		return open_new_namespace(&path, (flags & AT_RECURSIVE));
 
 	if (flags & OPEN_TREE_CLONE)
 		return open_detached_copy(&path, flags);
@@ -3513,8 +3528,6 @@ static bool mount_is_ancestor(const struct mount *p1, const struct mount *p2)
  * @mnt_to:   mount under which to mount
  * @mp:   mountpoint of @mnt_to
  *
- * - Make sure that nothing can be mounted beneath the caller's current
- *   root or the rootfs of the namespace.
  * - Make sure that the caller can unmount the topmost mount ensuring
  *   that the caller could reveal the underlying mountpoint.
  * - Ensure that nothing has been mounted on top of @mnt_from before we
@@ -3528,26 +3541,14 @@ static bool mount_is_ancestor(const struct mount *p1, const struct mount *p2)
  */
 static int can_move_mount_beneath(const struct mount *mnt_from,
 				  const struct mount *mnt_to,
-				  const struct mountpoint *mp)
+				  struct pinned_mountpoint *mp)
 {
 	struct mount *parent_mnt_to = mnt_to->mnt_parent;
 
-	if (IS_MNT_LOCKED(mnt_to))
-		return -EINVAL;
-
 	/* Avoid creating shadow mounts during mount propagation. */
 	if (mnt_from->overmount)
 		return -EINVAL;
 
-	/*
-	 * Mounting beneath the rootfs only makes sense when the
-	 * semantics of pivot_root(".", ".") are used.
-	 */
-	if (&mnt_to->mnt == current->fs->root.mnt)
-		return -EINVAL;
-	if (parent_mnt_to == current->nsproxy->mnt_ns->root)
-		return -EINVAL;
-
 	if (mount_is_ancestor(mnt_to, mnt_from))
 		return -EINVAL;
 
@@ -3557,7 +3558,7 @@ static int can_move_mount_beneath(const struct mount *mnt_from,
 	 * propagating a copy @c of @mnt_from on top of @mnt_to. This
 	 * defeats the whole purpose of mounting beneath another mount.
 	 */
-	if (propagation_would_overmount(parent_mnt_to, mnt_to, mp))
+	if (propagation_would_overmount(parent_mnt_to, mnt_to, mp->mp))
 		return -EINVAL;
 
 	/*
@@ -3573,7 +3574,7 @@ static int can_move_mount_beneath(const struct mount *mnt_from,
 	 * @mnt_from beneath @mnt_to.
 	 */
 	if (check_mnt(mnt_from) &&
-	    propagation_would_overmount(parent_mnt_to, mnt_from, mp))
+	    propagation_would_overmount(parent_mnt_to, mnt_from, mp->mp))
 		return -EINVAL;
 
 	return 0;
@@ -3682,7 +3683,7 @@ static int do_move_mount(const struct path *old_path,
 
 		if (mp.parent != over->mnt_parent)
 			over = mp.parent->overmount;
-		err = can_move_mount_beneath(old, over, mp.mp);
+		err = can_move_mount_beneath(old, over, &mp);
 		if (err)
 			return err;
 	}
@@ -4272,6 +4273,16 @@ struct mnt_namespace *copy_mnt_ns(u64 flags, struct mnt_namespace *ns,
 	 * as belonging to new namespace.  We have already acquired a private
 	 * fs_struct, so tsk->fs->lock is not needed.
 	 */
+	if (new_fs)
+		WARN_ON_ONCE(new_fs->users != 1);
+
+	/*
+	 * Drain the pwd reference pool. The pool must be empty before we
+	 * update new_fs->pwd.mnt below since the pooled references belong
+	 * to the old mount. Safe to access without locking: new_fs->users == 1.
+	 */
+	if (new_fs)
+		drain_fs_pwd_pool(new_fs);
 	p = old;
 	q = new;
 	while (p) {
@@ -4414,11 +4425,15 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
 	unsigned int mnt_flags = 0;
 	long ret;
 
-	if (!may_mount())
+	if ((flags & ~(FSMOUNT_CLOEXEC | FSMOUNT_NAMESPACE)) != 0)
+		return -EINVAL;
+
+	if ((flags & FSMOUNT_NAMESPACE) &&
+	    !ns_capable(current_user_ns(), CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if ((flags & ~(FSMOUNT_CLOEXEC)) != 0)
-		return -EINVAL;
+	if (!(flags & FSMOUNT_NAMESPACE) && !may_mount())
+		return -EPERM;
 
 	if (attr_flags & ~FSMOUNT_VALID_FLAGS)
 		return -EINVAL;
@@ -4485,6 +4500,10 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
 	 */
 	vfs_clean_context(fc);
 
+	if (flags & FSMOUNT_NAMESPACE)
+		return FD_ADD((flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0,
+			      open_new_namespace(&new_path, 0));
+
 	ns = alloc_mnt_ns(current->nsproxy->mnt_ns->user_ns, true);
 	if (IS_ERR(ns))
 		return PTR_ERR(ns);
@@ -5649,14 +5668,14 @@ static int grab_requested_root(struct mnt_namespace *ns, struct path *root)
 	if (mnt_ns_empty(ns))
 		return -ENOENT;
 
-	first = child = ns->root;
-	for (;;) {
-		child = listmnt_next(child, false);
-		if (!child)
-			return -ENOENT;
-		if (child->mnt_parent == first)
+	first = ns->root;
+	for (child = node_to_mount(ns->mnt_first_node); child;
+	     child = listmnt_next(child, false)) {
+		if (child != first && child->mnt_parent == first)
 			break;
 	}
+	if (!child)
+		return -ENOENT;
 
 	root->mnt = mntget(&child->mnt);
 	root->dentry = dget(root->mnt->mnt_root);
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 22a4d61631c9d..05ea5b0cc0e8b 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -10,7 +10,6 @@
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/slab.h>
-#include <linux/pagevec.h>
 #include "internal.h"
 
 static void __netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index dd1451bf7543d..4d9760e36c119 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -186,10 +186,18 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
 		stream->sreq_max_segs	= INT_MAX;
 
 		netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
-		stream->prepare_write(subreq);
 
-		__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
-		netfs_stat(&netfs_n_wh_retry_write_subreq);
+		if (stream->prepare_write) {
+			stream->prepare_write(subreq);
+			__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
+			netfs_stat(&netfs_n_wh_retry_write_subreq);
+		} else {
+			struct iov_iter source;
+
+			netfs_reset_iter(subreq);
+			source = subreq->io_iter;
+			netfs_reissue_write(stream, subreq, &source);
+		}
 	}
 
 	netfs_unbuffered_write_done(wreq);
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 72a435e5fc6da..154a14bb2d7f7 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -142,6 +142,47 @@ static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset,
 	return min(span, max_size);
 }
 
+/*
+ * Select the span of a kvec iterator we're going to use.  Limit it by both
+ * maximum size and maximum number of segments.  Returns the size of the span
+ * in bytes.
+ */
+static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset,
+			       size_t max_size, size_t max_segs)
+{
+	const struct kvec *kvecs = iter->kvec;
+	unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0;
+	size_t len, span = 0, n = iter->count;
+	size_t skip = iter->iov_offset + start_offset;
+
+	if (WARN_ON(!iov_iter_is_kvec(iter)) ||
+	    WARN_ON(start_offset > n) ||
+	    n == 0)
+		return 0;
+
+	while (n && ix < nkv && skip) {
+		len = kvecs[ix].iov_len;
+		if (skip < len)
+			break;
+		skip -= len;
+		n -= len;
+		ix++;
+	}
+
+	while (n && ix < nkv) {
+		len = min3(n, kvecs[ix].iov_len - skip, max_size);
+		span += len;
+		nsegs++;
+		ix++;
+		if (span >= max_size || nsegs >= max_segs)
+			break;
+		skip = 0;
+		n -= len;
+	}
+
+	return min(span, max_size);
+}
+
 /*
  * Select the span of an xarray iterator we're going to use.  Limit it by both
  * maximum size and maximum number of segments.  It is assumed that segments
@@ -245,6 +286,8 @@ size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
 		return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
 	if (iov_iter_is_xarray(iter))
 		return netfs_limit_xarray(iter, start_offset, max_size, max_segs);
+	if (iov_iter_is_kvec(iter))
+		return netfs_limit_kvec(iter, start_offset, max_size, max_segs);
 	BUG();
 }
 EXPORT_SYMBOL(netfs_limit_iter);
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index cb0a645aeb501..d54a141a89b3b 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -36,7 +36,6 @@
 #include <linux/namei.h>
 #include <linux/bio.h>		/* struct bio */
 #include <linux/prefetch.h>
-#include <linux/pagevec.h>
 
 #include "../pnfs.h"
 #include "../nfs4session.h"
@@ -381,14 +380,13 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync)
 	sector_t isect, extent_length = 0;
 	struct parallel_io *par = NULL;
 	loff_t offset = header->args.offset;
-	size_t count = header->args.count;
 	struct page **pages = header->args.pages;
 	int pg_index = header->args.pgbase >> PAGE_SHIFT;
 	unsigned int pg_len;
 	struct blk_plug plug;
 	int i;
 
-	dprintk("%s enter, %zu@%lld\n", __func__, count, offset);
+	dprintk("%s enter, %u@%lld\n", __func__, header->args.count, offset);
 
 	/* At this point, header->page_aray is a (sequential) list of nfs_pages.
 	 * We want to write each, and if there is an error set pnfs_error
@@ -429,7 +427,6 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync)
 		}
 
 		offset += pg_len;
-		count -= pg_len;
 		isect += (pg_len >> SECTOR_SHIFT);
 		extent_length -= (pg_len >> SECTOR_SHIFT);
 	}
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 2402f57c8e7d9..c1e629fbd131e 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -32,7 +32,6 @@
 #include <linux/nfs_fs.h>
 #include <linux/nfs_mount.h>
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
 #include <linux/namei.h>
 #include <linux/mount.h>
 #include <linux/swap.h>
@@ -1906,7 +1905,7 @@ static int nfs_weak_revalidate(struct dentry *dentry, unsigned int flags)
 	}
 
 	error = nfs_lookup_verify_inode(inode, flags);
-	dfprintk(LOOKUPCACHE, "NFS: %s: inode %lu is %s\n",
+	dfprintk(LOOKUPCACHE, "NFS: %s: inode %llu is %s\n",
 			__func__, inode->i_ino, error ? "invalid" : "valid");
 	return !error;
 }
@@ -2121,7 +2120,7 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry,
 	/* Expect a negative dentry */
 	BUG_ON(d_inode(dentry));
 
-	dfprintk(VFS, "NFS: atomic_open(%s/%lu), %pd\n",
+	dfprintk(VFS, "NFS: atomic_open(%s/%llu), %pd\n",
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	err = nfs_check_flags(open_flags);
@@ -2404,7 +2403,7 @@ static int nfs_do_create(struct inode *dir, struct dentry *dentry,
 
 	open_flags |= O_CREAT;
 
-	dfprintk(VFS, "NFS: create(%s/%lu), %pd\n",
+	dfprintk(VFS, "NFS: create(%s/%llu), %pd\n",
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	attr.ia_mode = mode;
@@ -2442,7 +2441,7 @@ nfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	struct iattr attr;
 	int status;
 
-	dfprintk(VFS, "NFS: mknod(%s/%lu), %pd\n",
+	dfprintk(VFS, "NFS: mknod(%s/%llu), %pd\n",
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	attr.ia_mode = mode;
@@ -2469,7 +2468,7 @@ struct dentry *nfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	struct iattr attr;
 	struct dentry *ret;
 
-	dfprintk(VFS, "NFS: mkdir(%s/%lu), %pd\n",
+	dfprintk(VFS, "NFS: mkdir(%s/%llu), %pd\n",
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	attr.ia_valid = ATTR_MODE;
@@ -2507,7 +2506,7 @@ int nfs_rmdir(struct inode *dir, struct dentry *dentry)
 {
 	int error;
 
-	dfprintk(VFS, "NFS: rmdir(%s/%lu), %pd\n",
+	dfprintk(VFS, "NFS: rmdir(%s/%llu), %pd\n",
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	trace_nfs_rmdir_enter(dir, dentry);
@@ -2578,7 +2577,7 @@ int nfs_unlink(struct inode *dir, struct dentry *dentry)
 {
 	int error;
 
-	dfprintk(VFS, "NFS: unlink(%s/%lu, %pd)\n", dir->i_sb->s_id,
+	dfprintk(VFS, "NFS: unlink(%s/%llu, %pd)\n", dir->i_sb->s_id,
 		dir->i_ino, dentry);
 
 	trace_nfs_unlink_enter(dir, dentry);
@@ -2638,7 +2637,7 @@ int nfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	unsigned int pathlen = strlen(symname);
 	int error;
 
-	dfprintk(VFS, "NFS: symlink(%s/%lu, %pd, %s)\n", dir->i_sb->s_id,
+	dfprintk(VFS, "NFS: symlink(%s/%llu, %pd, %s)\n", dir->i_sb->s_id,
 		dir->i_ino, dentry, symname);
 
 	if (pathlen > PAGE_SIZE)
@@ -2660,7 +2659,7 @@ int nfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	error = NFS_PROTO(dir)->symlink(dir, dentry, folio, pathlen, &attr);
 	trace_nfs_symlink_exit(dir, dentry, error);
 	if (error != 0) {
-		dfprintk(VFS, "NFS: symlink(%s/%lu, %pd, %s) error %d\n",
+		dfprintk(VFS, "NFS: symlink(%s/%llu, %pd, %s) error %d\n",
 			dir->i_sb->s_id, dir->i_ino,
 			dentry, symname, error);
 		d_drop(dentry);
@@ -3414,7 +3413,7 @@ int nfs_permission(struct mnt_idmap *idmap,
 	if (!res && (mask & MAY_EXEC))
 		res = nfs_execute_ok(inode, mask);
 
-	dfprintk(VFS, "NFS: permission(%s/%lu), mask=0x%x, res=%d\n",
+	dfprintk(VFS, "NFS: permission(%s/%llu), mask=0x%x, res=%d\n",
 		inode->i_sb->s_id, inode->i_ino, mask, res);
 	return res;
 out_notsup:
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index 5d08b6409c284..25048a3c23643 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -391,7 +391,7 @@ static int nfs_write_begin(const struct kiocb *iocb,
 
 	trace_nfs_write_begin(file_inode(file), pos, len);
 
-	dfprintk(PAGECACHE, "NFS: write_begin(%pD2(%lu), %u@%lld)\n",
+	dfprintk(PAGECACHE, "NFS: write_begin(%pD2(%llu), %u@%lld)\n",
 		file, mapping->host->i_ino, len, (long long) pos);
 	nfs_truncate_last_folio(mapping, i_size_read(mapping->host), pos);
 
@@ -432,7 +432,7 @@ static int nfs_write_end(const struct kiocb *iocb,
 	int status;
 
 	trace_nfs_write_end(file_inode(file), pos, len);
-	dfprintk(PAGECACHE, "NFS: write_end(%pD2(%lu), %u@%lld)\n",
+	dfprintk(PAGECACHE, "NFS: write_end(%pD2(%llu), %u@%lld)\n",
 		file, mapping->host->i_ino, len, (long long) pos);
 
 	/*
@@ -557,7 +557,7 @@ static int nfs_launder_folio(struct folio *folio)
 	struct inode *inode = folio->mapping->host;
 	int ret;
 
-	dfprintk(PAGECACHE, "NFS: launder_folio(%ld, %llu)\n",
+	dfprintk(PAGECACHE, "NFS: launder_folio(%llu, %llu)\n",
 		inode->i_ino, folio_pos(folio));
 
 	folio_wait_private_2(folio); /* [DEPRECATED] */
@@ -647,7 +647,7 @@ static vm_fault_t nfs_vm_page_mkwrite(struct vm_fault *vmf)
 	struct address_space *mapping;
 	struct folio *folio = page_folio(vmf->page);
 
-	dfprintk(PAGECACHE, "NFS: vm_page_mkwrite(%pD2(%lu), offset %lld)\n",
+	dfprintk(PAGECACHE, "NFS: vm_page_mkwrite(%pD2(%llu), offset %lld)\n",
 		 filp, filp->f_mapping->host->i_ino,
 		 (long long)folio_pos(folio));
 
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 90a11afa5d054..e85380e3b11d7 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -241,7 +241,7 @@ filelayout_set_layoutcommit(struct nfs_pgio_header *hdr)
 
 	/* Note: if the write is unstable, don't set end_offs until commit */
 	pnfs_set_layoutcommit(hdr->inode, hdr->lseg, end_offs);
-	dprintk("%s inode %lu pls_end_pos %lu\n", __func__, hdr->inode->i_ino,
+	dprintk("%s inode %llu pls_end_pos %lu\n", __func__, hdr->inode->i_ino,
 		(unsigned long) NFS_I(hdr->inode)->layout->plh_lwb);
 }
 
@@ -456,7 +456,7 @@ filelayout_read_pagelist(struct nfs_pgio_header *hdr)
 	u32 j, idx;
 	struct nfs_fh *fh;
 
-	dprintk("--> %s ino %lu pgbase %u req %zu@%llu\n",
+	dprintk("--> %s ino %llu pgbase %u req %zu@%llu\n",
 		__func__, hdr->inode->i_ino,
 		hdr->args.pgbase, (size_t)hdr->args.count, offset);
 
@@ -514,7 +514,7 @@ filelayout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 	if (IS_ERR(ds_clnt))
 		return PNFS_NOT_ATTEMPTED;
 
-	dprintk("%s ino %lu sync %d req %zu@%llu DS: %s cl_count %d\n",
+	dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d\n",
 		__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
 		offset, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count));
 
@@ -1001,7 +1001,7 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
 	if (IS_ERR(ds_clnt))
 		goto out_err;
 
-	dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
+	dprintk("%s ino %llu, how %d cl_count %d\n", __func__,
 		data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count));
 	data->commit_done_cb = filelayout_commit_done_cb;
 	refcount_inc(&ds->ds_clp->cl_count);
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index f67773d52830d..8b1559171fe39 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -1631,7 +1631,7 @@ ff_layout_set_layoutcommit(struct inode *inode,
 		return;
 
 	pnfs_set_layoutcommit(inode, lseg, end_offset);
-	dprintk("%s inode %lu pls_end_pos %llu\n", __func__, inode->i_ino,
+	dprintk("%s inode %llu pls_end_pos %llu\n", __func__, inode->i_ino,
 		(unsigned long long) NFS_I(inode)->layout->plh_lwb);
 }
 
@@ -2136,7 +2136,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 	u32 dss_id;
 	bool ds_fatal_error = false;
 
-	dprintk("--> %s ino %lu pgbase %u req %zu@%llu\n",
+	dprintk("--> %s ino %llu pgbase %u req %zu@%llu\n",
 		__func__, hdr->inode->i_ino,
 		hdr->args.pgbase, (size_t)hdr->args.count, offset);
 
@@ -2245,7 +2245,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 
 	vers = nfs4_ff_layout_ds_version(mirror, dss_id);
 
-	dprintk("%s ino %lu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n",
+	dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n",
 		__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
 		offset, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count),
 		vers);
@@ -2336,7 +2336,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
 
 	vers = nfs4_ff_layout_ds_version(mirror, dss_id);
 
-	dprintk("%s ino %lu, how %d cl_count %d vers %d\n", __func__,
+	dprintk("%s ino %llu, how %d cl_count %d vers %d\n", __func__,
 		data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count),
 		vers);
 	data->commit_done_cb = ff_layout_commit_done_cb;
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 4786343eeee0f..98a8f0de11991 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -2258,7 +2258,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
 	bool attr_changed = false;
 	bool have_delegation;
 
-	dfprintk(VFS, "NFS: %s(%s/%lu fh_crc=0x%08x ct=%d info=0x%llx)\n",
+	dfprintk(VFS, "NFS: %s(%s/%llu fh_crc=0x%08x ct=%d info=0x%llx)\n",
 			__func__, inode->i_sb->s_id, inode->i_ino,
 			nfs_display_fhandle_hash(NFS_FH(inode)),
 			icount_read(inode), fattr->valid);
@@ -2288,7 +2288,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
 		/*
 		* Big trouble! The inode has become a different object.
 		*/
-		printk(KERN_DEBUG "NFS: %s: inode %lu mode changed, %07o to %07o\n",
+		printk(KERN_DEBUG "NFS: %s: inode %llu mode changed, %07o to %07o\n",
 				__func__, inode->i_ino, inode->i_mode, fattr->mode);
 		goto out_err;
 	}
@@ -2358,7 +2358,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
 				if (S_ISDIR(inode->i_mode))
 					nfs_force_lookup_revalidate(inode);
 				attr_changed = true;
-				dprintk("NFS: change_attr change on server for file %s/%ld\n",
+				dprintk("NFS: change_attr change on server for file %s/%llu\n",
 						inode->i_sb->s_id,
 						inode->i_ino);
 			} else if (!have_delegation) {
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 63e09dfc27a8f..fc54563771604 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -859,7 +859,7 @@ static inline void nfs_folio_mark_unstable(struct folio *folio,
 		 * writeback is happening on the server now.
 		 */
 		node_stat_mod_folio(folio, NR_WRITEBACK, nr);
-		wb_stat_mod(&inode_to_bdi(inode)->wb, WB_WRITEBACK, nr);
+		bdi_wb_stat_mod(inode, WB_WRITEBACK, nr);
 		__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
 	}
 }
diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c
index be2aebf62056b..95d7cd564b746 100644
--- a/fs/nfs/nfs3proc.c
+++ b/fs/nfs/nfs3proc.c
@@ -16,6 +16,7 @@
 #include <linux/nfs3.h>
 #include <linux/nfs_fs.h>
 #include <linux/nfs_page.h>
+#include <linux/filelock.h>
 #include <linux/lockd/bind.h>
 #include <linux/nfs_mount.h>
 #include <linux/freezer.h>
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 91bcf67bd743f..d839a97df822a 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4714,7 +4714,7 @@ static int _nfs4_proc_lookupp(struct inode *inode,
 	nfs_fattr_init(fattr);
 	nfs4_init_sequence(server->nfs_client, &args.seq_args, &res.seq_res, 0, 0);
 
-	dprintk("NFS call  lookupp ino=0x%lx\n", inode->i_ino);
+	dprintk("NFS call  lookupp ino=0x%llx\n", inode->i_ino);
 	status = nfs4_do_call_sync(clnt, server, &msg, &args.seq_args,
 				   &res.seq_res, task_flags);
 	dprintk("NFS reply lookupp: %d\n", status);
@@ -10019,7 +10019,7 @@ nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync)
 	int status = 0;
 
 	dprintk("NFS: initiating layoutcommit call. sync %d "
-		"lbw: %llu inode %lu\n", sync,
+		"lbw: %llu inode %llu\n", sync,
 		data->args.lastbytewritten,
 		data->args.inode->i_ino);
 
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index bc13d1e69449c..e79deb9bf664d 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -891,7 +891,7 @@ pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
 	while (!list_empty(layout_list)) {
 		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
 				plh_bulk_destroy);
-		dprintk("%s freeing layout for inode %lu\n", __func__,
+		dprintk("%s freeing layout for inode %llu\n", __func__,
 			lo->plh_inode->i_ino);
 		inode = lo->plh_inode;
 
@@ -1440,7 +1440,7 @@ _pnfs_return_layout(struct inode *ino)
 	int status = 0;
 	bool send, valid_layout;
 
-	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
+	dprintk("NFS: %s for inode %llu\n", __func__, ino->i_ino);
 
 	spin_lock(&ino->i_lock);
 	lo = nfsi->layout;
@@ -3055,7 +3055,7 @@ pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
 
 	hdr->mds_ops = call_ops;
 
-	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
+	dprintk("%s: Writing ino:%llu %u@%llu (how %d)\n", __func__,
 		inode->i_ino, hdr->args.count, hdr->args.offset, how);
 	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
 	if (trypnfs != PNFS_NOT_ATTEMPTED)
@@ -3181,7 +3181,7 @@ pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
 
 	hdr->mds_ops = call_ops;
 
-	dprintk("%s: Reading ino:%lu %u@%llu\n",
+	dprintk("%s: Reading ino:%llu %u@%llu\n",
 		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
 
 	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
@@ -3314,7 +3314,7 @@ pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
 	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
 		nfsi->layout->plh_lwb = end_pos;
 		mark_as_dirty = true;
-		dprintk("%s: Set layoutcommit for inode %lu ",
+		dprintk("%s: Set layoutcommit for inode %llu ",
 			__func__, inode->i_ino);
 	} else if (end_pos > nfsi->layout->plh_lwb)
 		nfsi->layout->plh_lwb = end_pos;
@@ -3363,7 +3363,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync)
 	if (!pnfs_layoutcommit_outstanding(inode))
 		return 0;
 
-	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
+	dprintk("--> %s inode %llu\n", __func__, inode->i_ino);
 
 	status = -EAGAIN;
 	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c
index 8c3d2efa26365..70795684b8e84 100644
--- a/fs/nfs/proc.c
+++ b/fs/nfs/proc.c
@@ -41,6 +41,7 @@
 #include <linux/nfs2.h>
 #include <linux/nfs_fs.h>
 #include <linux/nfs_page.h>
+#include <linux/filelock.h>
 #include <linux/lockd/bind.h>
 #include <linux/freezer.h>
 #include "internal.h"
diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c
index 7d8921f524a69..051da37770d8d 100644
--- a/fs/nfs/sysfs.c
+++ b/fs/nfs/sysfs.c
@@ -12,7 +12,7 @@
 #include <linux/string.h>
 #include <linux/nfs_fs.h>
 #include <linux/rcupdate.h>
-#include <linux/lockd/lockd.h>
+#include <linux/lockd/bind.h>
 
 #include "internal.h"
 #include "nfs4_fs.h"
@@ -285,7 +285,7 @@ shutdown_store(struct kobject *kobj, struct kobj_attribute *attr,
 		shutdown_client(server->client_acl);
 
 	if (server->nlm_host)
-		shutdown_client(server->nlm_host->h_rpcclnt);
+		nlmclnt_shutdown_rpc_clnt(server->nlm_host);
 out:
 	shutdown_nfs_client(server->nfs_client);
 	return count;
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 1ed4b3590b1ac..cc02b57de3c76 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -872,8 +872,7 @@ static void nfs_folio_clear_commit(struct folio *folio)
 		long nr = folio_nr_pages(folio);
 
 		node_stat_mod_folio(folio, NR_WRITEBACK, -nr);
-		wb_stat_mod(&inode_to_bdi(folio->mapping->host)->wb,
-			    WB_WRITEBACK, -nr);
+		bdi_wb_stat_mod(folio->mapping->host, WB_WRITEBACK, -nr);
 	}
 }
 
diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig
index 4fd6e818565ea..fc0e87eaa2571 100644
--- a/fs/nfsd/Kconfig
+++ b/fs/nfsd/Kconfig
@@ -177,16 +177,6 @@ config NFSD_LEGACY_CLIENT_TRACKING
 	  and will be removed in the future. Say Y here if you need support
 	  for them in the interim.
 
-config NFSD_V4_DELEG_TIMESTAMPS
-	bool "Support delegated timestamps"
-	depends on NFSD_V4
-	default n
-	help
-	  NFSD implements delegated timestamps according to
-	  draft-ietf-nfsv4-delstid-08 "Extending the Opening of Files". This
-	  is currently an experimental feature and is therefore left disabled
-	  by default.
-
 config NFSD_V4_POSIX_ACLS
 	bool "Support NFSv4 POSIX draft ACLs"
 	depends on NFSD_V4
diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
index a7cfba29990e4..9d829c84f374e 100644
--- a/fs/nfsd/blocklayout.c
+++ b/fs/nfsd/blocklayout.c
@@ -273,6 +273,52 @@ const struct nfsd4_layout_ops bl_layout_ops = {
 #endif /* CONFIG_NFSD_BLOCKLAYOUT */
 
 #ifdef CONFIG_NFSD_SCSILAYOUT
+
+#define NFSD_MDS_PR_FENCED	XA_MARK_0
+
+/*
+ * Clear the fence flag if the device already has an entry. This occurs
+ * when a client re-registers after a previous fence, allowing new
+ * layouts for this device.
+ *
+ * Insert only on first registration. This bounds cl_dev_fences to the
+ * count of devices this client has accessed, preventing unbounded growth.
+ */
+static inline int nfsd4_scsi_fence_insert(struct nfs4_client *clp,
+					  dev_t device)
+{
+	struct xarray *xa = &clp->cl_dev_fences;
+	int ret;
+
+	xa_lock(xa);
+	ret = __xa_insert(xa, device, XA_ZERO_ENTRY, GFP_KERNEL);
+	if (ret == -EBUSY) {
+		__xa_clear_mark(xa, device, NFSD_MDS_PR_FENCED);
+		ret = 0;
+	}
+	xa_unlock(xa);
+	clp->cl_fence_retry_warn = false;
+	return ret;
+}
+
+static inline bool nfsd4_scsi_fence_set(struct nfs4_client *clp, dev_t device)
+{
+	struct xarray *xa = &clp->cl_dev_fences;
+	bool skip;
+
+	xa_lock(xa);
+	skip = xa_get_mark(xa, device, NFSD_MDS_PR_FENCED);
+	if (!skip)
+		__xa_set_mark(xa, device, NFSD_MDS_PR_FENCED);
+	xa_unlock(xa);
+	return skip;
+}
+
+static inline void nfsd4_scsi_fence_clear(struct nfs4_client *clp, dev_t device)
+{
+	xa_clear_mark(&clp->cl_dev_fences, device, NFSD_MDS_PR_FENCED);
+}
+
 #define NFSD_MDS_PR_KEY		0x0100000000000000ULL
 
 /*
@@ -342,6 +388,10 @@ nfsd4_block_get_device_info_scsi(struct super_block *sb,
 		goto out_free_dev;
 	}
 
+	ret = nfsd4_scsi_fence_insert(clp, sb->s_bdev->bd_dev);
+	if (ret < 0)
+		goto out_free_dev;
+
 	ret = ops->pr_register(sb->s_bdev, 0, NFSD_MDS_PR_KEY, true);
 	if (ret) {
 		pr_err("pNFS: failed to register key for device %s.\n",
@@ -394,17 +444,67 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode, struct svc_rqst *rqstp,
 	return nfsd4_block_commit_blocks(inode, lcp, iomaps, nr_iomaps);
 }
 
-static void
+/*
+ * Perform the fence operation to prevent the client from accessing the
+ * block device. If a fence operation is already in progress, wait for
+ * it to complete before checking the NFSD_MDS_PR_FENCED flag. Once the
+ * operation is complete, check the flag. If NFSD_MDS_PR_FENCED is set,
+ * update the layout stateid by setting the ls_fenced flag to indicate
+ * that the client has been fenced.
+ *
+ * The cl_fence_mutex ensures that the fence operation has been fully
+ * completed, rather than just in progress, when returning from this
+ * function.
+ *
+ * Return true if client was fenced otherwise return false.
+ */
+static bool
 nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
 {
 	struct nfs4_client *clp = ls->ls_stid.sc_client;
 	struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
 	int status;
+	bool ret;
+
+	mutex_lock(&clp->cl_fence_mutex);
+	if (nfsd4_scsi_fence_set(clp, bdev->bd_dev)) {
+		mutex_unlock(&clp->cl_fence_mutex);
+		return true;
+	}
 
 	status = bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
 			nfsd4_scsi_pr_key(clp),
 			PR_EXCLUSIVE_ACCESS_REG_ONLY, true);
+	/*
+	 * Reset to allow retry only when the command could not have
+	 * reached the device. Negative status means a local error
+	 * (e.g., -ENOMEM) prevented the command from being sent.
+	 * PR_STS_PATH_FAILED, PR_STS_PATH_FAST_FAILED, and
+	 * PR_STS_RETRY_PATH_FAILURE indicate transport path failures
+	 * before device delivery.
+	 *
+	 * For all other errors, the command may have reached the device
+	 * and the preempt may have succeeded. Avoid resetting, since
+	 * retrying a successful preempt returns PR_STS_IOERR or
+	 * PR_STS_RESERVATION_CONFLICT, which would cause an infinite
+	 * retry loop.
+	 */
+	switch (status) {
+	case 0:
+	case PR_STS_IOERR:
+	case PR_STS_RESERVATION_CONFLICT:
+		ret = true;
+		break;
+	default:
+		/* retry-able and other errors */
+		ret = false;
+		nfsd4_scsi_fence_clear(clp, bdev->bd_dev);
+		break;
+	}
+	mutex_unlock(&clp->cl_fence_mutex);
+
 	trace_nfsd_pnfs_fence(clp, bdev->bd_disk->disk_name, status);
+	return ret;
 }
 
 const struct nfsd4_layout_ops scsi_layout_ops = {
diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c
index 7f44689e0a53e..386fd1c54f527 100644
--- a/fs/nfsd/debugfs.c
+++ b/fs/nfsd/debugfs.c
@@ -140,4 +140,8 @@ void nfsd_debugfs_init(void)
 
 	debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL,
 			    &nfsd_io_cache_write_fops);
+#ifdef CONFIG_NFSD_V4
+	debugfs_create_bool("delegated_timestamps", 0644, nfsd_top_dir,
+			    &nfsd_delegts_enabled);
+#endif
 }
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index 8fdbba7cad964..1aadfa8e0406c 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -36,19 +36,30 @@
  * second map contains a reference to the entry in the first map.
  */
 
+static struct workqueue_struct *nfsd_export_wq;
+
 #define	EXPKEY_HASHBITS		8
 #define	EXPKEY_HASHMAX		(1 << EXPKEY_HASHBITS)
 #define	EXPKEY_HASHMASK		(EXPKEY_HASHMAX -1)
 
-static void expkey_put(struct kref *ref)
+static void expkey_release(struct work_struct *work)
 {
-	struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref);
+	struct svc_expkey *key = container_of(to_rcu_work(work),
+					      struct svc_expkey, ek_rwork);
 
 	if (test_bit(CACHE_VALID, &key->h.flags) &&
 	    !test_bit(CACHE_NEGATIVE, &key->h.flags))
 		path_put(&key->ek_path);
 	auth_domain_put(key->ek_client);
-	kfree_rcu(key, ek_rcu);
+	kfree(key);
+}
+
+static void expkey_put(struct kref *ref)
+{
+	struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref);
+
+	INIT_RCU_WORK(&key->ek_rwork, expkey_release);
+	queue_rcu_work(nfsd_export_wq, &key->ek_rwork);
 }
 
 static int expkey_upcall(struct cache_detail *cd, struct cache_head *h)
@@ -353,11 +364,13 @@ static void export_stats_destroy(struct export_stats *stats)
 					    EXP_STATS_COUNTERS_NUM);
 }
 
-static void svc_export_release(struct rcu_head *rcu_head)
+static void svc_export_release(struct work_struct *work)
 {
-	struct svc_export *exp = container_of(rcu_head, struct svc_export,
-			ex_rcu);
+	struct svc_export *exp = container_of(to_rcu_work(work),
+					      struct svc_export, ex_rwork);
 
+	path_put(&exp->ex_path);
+	auth_domain_put(exp->ex_client);
 	nfsd4_fslocs_free(&exp->ex_fslocs);
 	export_stats_destroy(exp->ex_stats);
 	kfree(exp->ex_stats);
@@ -369,9 +382,8 @@ static void svc_export_put(struct kref *ref)
 {
 	struct svc_export *exp = container_of(ref, struct svc_export, h.ref);
 
-	path_put(&exp->ex_path);
-	auth_domain_put(exp->ex_client);
-	call_rcu(&exp->ex_rcu, svc_export_release);
+	INIT_RCU_WORK(&exp->ex_rwork, svc_export_release);
+	queue_rcu_work(nfsd_export_wq, &exp->ex_rwork);
 }
 
 static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h)
@@ -1039,7 +1051,7 @@ exp_rootfh(struct net *net, struct auth_domain *clp, char *name,
 	}
 	inode = d_inode(path.dentry);
 
-	dprintk("nfsd: exp_rootfh(%s [%p] %s:%s/%ld)\n",
+	dprintk("nfsd: exp_rootfh(%s [%p] %s:%s/%llu)\n",
 		 name, path.dentry, clp->name,
 		 inode->i_sb->s_id, inode->i_ino);
 	exp = exp_parent(cd, clp, &path);
@@ -1479,6 +1491,36 @@ const struct seq_operations nfs_exports_op = {
 	.show	= e_show,
 };
 
+/**
+ * nfsd_export_wq_init - allocate the export release workqueue
+ *
+ * Called once at module load. The workqueue runs deferred svc_export and
+ * svc_expkey release work scheduled by queue_rcu_work() in the cache put
+ * callbacks.
+ *
+ * Return values:
+ *   %0: workqueue allocated
+ *   %-ENOMEM: allocation failed
+ */
+int nfsd_export_wq_init(void)
+{
+	nfsd_export_wq = alloc_workqueue("nfsd_export", WQ_UNBOUND, 0);
+	if (!nfsd_export_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+/**
+ * nfsd_export_wq_shutdown - drain and free the export release workqueue
+ *
+ * Called once at module unload. Per-namespace teardown in
+ * nfsd_export_shutdown() has already drained all deferred work.
+ */
+void nfsd_export_wq_shutdown(void)
+{
+	destroy_workqueue(nfsd_export_wq);
+}
+
 /*
  * Initialize the exports module.
  */
@@ -1540,6 +1582,9 @@ nfsd_export_shutdown(struct net *net)
 
 	cache_unregister_net(nn->svc_expkey_cache, net);
 	cache_unregister_net(nn->svc_export_cache, net);
+	/* Drain deferred export and expkey release work. */
+	rcu_barrier();
+	flush_workqueue(nfsd_export_wq);
 	cache_destroy_net(nn->svc_expkey_cache, net);
 	cache_destroy_net(nn->svc_export_cache, net);
 	svcauth_unix_purge(net);
diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h
index d2b09cd761453..b053993745749 100644
--- a/fs/nfsd/export.h
+++ b/fs/nfsd/export.h
@@ -7,6 +7,7 @@
 
 #include <linux/sunrpc/cache.h>
 #include <linux/percpu_counter.h>
+#include <linux/workqueue.h>
 #include <uapi/linux/nfsd/export.h>
 #include <linux/nfs4.h>
 
@@ -75,7 +76,7 @@ struct svc_export {
 	u32			ex_layout_types;
 	struct nfsd4_deviceid_map *ex_devid_map;
 	struct cache_detail	*cd;
-	struct rcu_head		ex_rcu;
+	struct rcu_work		ex_rwork;
 	unsigned long		ex_xprtsec_modes;
 	struct export_stats	*ex_stats;
 };
@@ -92,7 +93,7 @@ struct svc_expkey {
 	u32			ek_fsid[6];
 
 	struct path		ek_path;
-	struct rcu_head		ek_rcu;
+	struct rcu_work		ek_rwork;
 };
 
 #define EX_ISSYNC(exp)		(!((exp)->ex_flags & NFSEXP_ASYNC))
@@ -110,6 +111,8 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp,
 /*
  * Function declarations
  */
+int			nfsd_export_wq_init(void);
+void			nfsd_export_wq_shutdown(void);
 int			nfsd_export_init(struct net *);
 void			nfsd_export_shutdown(struct net *);
 void			nfsd_export_flush(struct net *);
diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c
index c774ce9aa2965..6fe1325815e0e 100644
--- a/fs/nfsd/lockd.c
+++ b/fs/nfsd/lockd.c
@@ -14,19 +14,20 @@
 
 #define NFSDDBG_FACILITY		NFSDDBG_LOCKD
 
-#ifdef CONFIG_LOCKD_V4
-#define nlm_stale_fh	nlm4_stale_fh
-#define nlm_failed	nlm4_failed
-#else
-#define nlm_stale_fh	nlm_lck_denied_nolocks
-#define nlm_failed	nlm_lck_denied_nolocks
-#endif
-/*
- * Note: we hold the dentry use count while the file is open.
+/**
+ * nlm_fopen - Open an NFSD file
+ * @rqstp: NLM RPC procedure execution context
+ * @f: NFS file handle to be opened
+ * @filp: OUT: an opened struct file
+ * @flags: the POSIX open flags to use
+ *
+ * nlm_fopen() holds the dentry reference until nlm_fclose() releases it.
+ *
+ * Returns zero on success or a negative errno value if the file
+ * cannot be opened.
  */
-static __be32
-nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp,
-		int mode)
+static int nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f,
+		     struct file **filp, int flags)
 {
 	__be32		nfserr;
 	int		access;
@@ -47,18 +48,17 @@ nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp,
 	 * if NFSEXP_NOAUTHNLM is set.  Some older clients use AUTH_NULL
 	 * for NLM requests.
 	 */
-	access = (mode == O_WRONLY) ? NFSD_MAY_WRITE : NFSD_MAY_READ;
+	access = (flags == O_WRONLY) ? NFSD_MAY_WRITE : NFSD_MAY_READ;
 	access |= NFSD_MAY_NLM | NFSD_MAY_OWNER_OVERRIDE | NFSD_MAY_BYPASS_GSS;
 	nfserr = nfsd_open(rqstp, &fh, S_IFREG, access, filp);
 	fh_put(&fh);
-	/* We return nlm error codes as nlm doesn't know
-	 * about nfsd, but nfsd does know about nlm..
-	 */
+
 	switch (nfserr) {
 	case nfs_ok:
-		return 0;
+		break;
 	case nfserr_jukebox:
-		/* this error can indicate a presence of a conflicting
+		/*
+		 * This error can indicate a presence of a conflicting
 		 * delegation to an NLM lock request. Options are:
 		 * (1) For now, drop this request and make the client
 		 * retry. When delegation is returned, client's lock retry
@@ -66,19 +66,25 @@ nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp,
 		 * (2) NLM4_DENIED as per "spec" signals to the client
 		 * that the lock is unavailable now but client can retry.
 		 * Linux client implementation does not. It treats
-		 * NLM4_DENIED same as NLM4_FAILED and errors the request.
+		 * NLM4_DENIED same as NLM4_FAILED and fails the request.
 		 * (3) For the future, treat this as blocked lock and try
 		 * to callback when the delegation is returned but might
 		 * not have a proper lock request to block on.
 		 */
-		return nlm_drop_reply;
+		return -EWOULDBLOCK;
 	case nfserr_stale:
-		return nlm_stale_fh;
+		return -ESTALE;
 	default:
-		return nlm_failed;
+		return -ENOLCK;
 	}
+
+	return 0;
 }
 
+/**
+ * nlm_fclose - Close an NFSD file
+ * @filp: a struct file that was opened by nlm_fopen()
+ */
 static void
 nlm_fclose(struct file *filp)
 {
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index ad7af8cfcf1f9..69e41105efdd5 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -27,6 +27,8 @@ static struct kmem_cache *nfs4_layout_stateid_cache;
 static const struct nfsd4_callback_ops nfsd4_cb_layout_ops;
 static const struct lease_manager_operations nfsd4_layouts_lm_ops;
 
+static void nfsd4_layout_fence_worker(struct work_struct *work);
+
 const struct nfsd4_layout_ops *nfsd4_layout_ops[LAYOUT_TYPE_MAX] =  {
 #ifdef CONFIG_NFSD_FLEXFILELAYOUT
 	[LAYOUT_FLEX_FILES]	= &ff_layout_ops,
@@ -177,6 +179,13 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
 
 	trace_nfsd_layoutstate_free(&ls->ls_stid.sc_stateid);
 
+	spin_lock(&ls->ls_lock);
+	if (delayed_work_pending(&ls->ls_fence_work)) {
+		spin_unlock(&ls->ls_lock);
+		cancel_delayed_work_sync(&ls->ls_fence_work);
+	} else
+		spin_unlock(&ls->ls_lock);
+
 	spin_lock(&clp->cl_lock);
 	list_del_init(&ls->ls_perclnt);
 	spin_unlock(&clp->cl_lock);
@@ -271,6 +280,10 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate,
 	list_add(&ls->ls_perfile, &fp->fi_lo_states);
 	spin_unlock(&fp->fi_lock);
 
+	ls->ls_fenced = false;
+	ls->ls_fence_delay = 0;
+	INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker);
+
 	trace_nfsd_layoutstate_alloc(&ls->ls_stid.sc_stateid);
 	return ls;
 }
@@ -747,11 +760,9 @@ static bool
 nfsd4_layout_lm_break(struct file_lease *fl)
 {
 	/*
-	 * We don't want the locks code to timeout the lease for us;
-	 * we'll remove it ourself if a layout isn't returned
-	 * in time:
+	 * Enforce break lease timeout to prevent NFSD
+	 * thread from hanging in __break_lease.
 	 */
-	fl->fl_break_time = 0;
 	nfsd4_recall_file_layout(fl->c.flc_owner);
 	return false;
 }
@@ -782,10 +793,143 @@ nfsd4_layout_lm_open_conflict(struct file *filp, int arg)
 	return 0;
 }
 
+static void
+nfsd4_layout_fence_worker(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct nfs4_layout_stateid *ls = container_of(dwork,
+			struct nfs4_layout_stateid, ls_fence_work);
+	struct nfsd_file *nf;
+	struct block_device *bdev;
+	struct nfs4_client *clp;
+	struct nfsd_net *nn;
+
+	/*
+	 * The workqueue clears WORK_STRUCT_PENDING before invoking
+	 * this callback. Re-arm immediately so that
+	 * delayed_work_pending() returns true while the fence
+	 * operation is in progress, preventing
+	 * lm_breaker_timedout() from taking a duplicate reference.
+	 */
+	mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0);
+
+	spin_lock(&ls->ls_lock);
+	if (list_empty(&ls->ls_layouts)) {
+		spin_unlock(&ls->ls_lock);
+dispose:
+		cancel_delayed_work(&ls->ls_fence_work);
+		/* unlock the lease so that tasks waiting on it can proceed */
+		nfsd4_close_layout(ls);
+
+		ls->ls_fenced = true;
+		nfs4_put_stid(&ls->ls_stid);
+		return;
+	}
+	spin_unlock(&ls->ls_lock);
+
+	rcu_read_lock();
+	nf = nfsd_file_get(ls->ls_file);
+	rcu_read_unlock();
+	if (!nf)
+		goto dispose;
+
+	clp = ls->ls_stid.sc_client;
+	nn = net_generic(clp->net, nfsd_net_id);
+	bdev = nf->nf_file->f_path.mnt->mnt_sb->s_bdev;
+	if (nfsd4_layout_ops[ls->ls_layout_type]->fence_client(ls, nf)) {
+		/* fenced ok */
+		nfsd_file_put(nf);
+		pr_warn("%s: FENCED client[%pISpc] clid[%d] to device[%s]\n",
+			__func__, (struct sockaddr *)&clp->cl_addr,
+			clp->cl_clientid.cl_id - nn->clientid_base,
+			bdev->bd_disk->disk_name);
+		goto dispose;
+	}
+	/* fence failed */
+	nfsd_file_put(nf);
+
+	if (!clp->cl_fence_retry_warn) {
+		pr_warn("%s: FENCE failed client[%pISpc] clid[%d] device[%s]\n",
+			__func__, (struct sockaddr *)&clp->cl_addr,
+			clp->cl_clientid.cl_id - nn->clientid_base,
+			bdev->bd_disk->disk_name);
+		clp->cl_fence_retry_warn = true;
+	}
+	/*
+	 * The fence worker retries the fencing operation indefinitely to
+	 * prevent data corruption. The admin needs to take the following
+	 * actions to restore access to the file for other clients:
+	 *
+	 *  . shutdown or power off the client being fenced.
+	 *  . manually expire the client to release all its state on the server;
+	 *    echo 'expire' > /proc/fs/nfsd/clients/clid/ctl'.
+	 *
+	 *    Where:
+	 *
+	 *    clid: is the unique client identifier displayed in
+	 *          the warning message above.
+	 */
+	if (!ls->ls_fence_delay)
+		ls->ls_fence_delay = HZ;
+	else
+		ls->ls_fence_delay = min(ls->ls_fence_delay << 1,
+					 MAX_FENCE_DELAY);
+	mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, ls->ls_fence_delay);
+}
+
+/**
+ * nfsd4_layout_lm_breaker_timedout - The layout recall has timed out.
+ * @fl: file to check
+ *
+ * If the layout type supports a fence operation, schedule a worker to
+ * fence the client from accessing the block device.
+ *
+ * This function runs under the protection of the spin_lock flc_lock.
+ * At this time, the file_lease associated with the layout stateid is
+ * on the flc_list. A reference count is incremented on the layout
+ * stateid to prevent it from being freed while the fence worker is
+ * executing. Once the fence worker finishes its operation, it releases
+ * this reference.
+ *
+ * The fence worker continues to run until either the client has been
+ * fenced or the layout becomes invalid. The layout can become invalid
+ * as a result of a LAYOUTRETURN or when the CB_LAYOUT recall callback
+ * has completed.
+ *
+ * Return true if the file_lease should be disposed of by the caller;
+ * otherwise, return false.
+ */
+static bool
+nfsd4_layout_lm_breaker_timedout(struct file_lease *fl)
+{
+	struct nfs4_layout_stateid *ls = fl->c.flc_owner;
+
+	if ((!nfsd4_layout_ops[ls->ls_layout_type]->fence_client) ||
+			ls->ls_fenced)
+		return true;
+	if (delayed_work_pending(&ls->ls_fence_work))
+		return false;
+	/*
+	 * Make sure layout has not been returned yet before
+	 * taking a reference count on the layout stateid.
+	 */
+	spin_lock(&ls->ls_lock);
+	if (list_empty(&ls->ls_layouts) ||
+			!refcount_inc_not_zero(&ls->ls_stid.sc_count)) {
+		spin_unlock(&ls->ls_lock);
+		return true;
+	}
+	spin_unlock(&ls->ls_lock);
+
+	mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0);
+	return false;
+}
+
 static const struct lease_manager_operations nfsd4_layouts_lm_ops = {
 	.lm_break		= nfsd4_layout_lm_break,
 	.lm_change		= nfsd4_layout_lm_change,
 	.lm_open_conflict	= nfsd4_layout_lm_open_conflict,
+	.lm_breaker_timedout	= nfsd4_layout_lm_breaker_timedout,
 };
 
 int
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 6880c5c520e75..85e94c30285a2 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -3043,6 +3043,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 	struct svc_fh *current_fh = &cstate->current_fh;
 	struct svc_fh *save_fh = &cstate->save_fh;
 	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+	struct nfsd_thread_local_info *ntli = rqstp->rq_private;
 	__be32		status;
 
 	resp->xdr = &rqstp->rq_res_stream;
@@ -3081,7 +3082,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 	}
 	check_if_stalefh_allowed(args);
 
-	rqstp->rq_lease_breaker = (void **)&cstate->clp;
+	ntli->ntli_lease_breaker = &cstate->clp;
 
 	trace_nfsd_compound(rqstp, args->tag, args->taglen, args->client_opcnt);
 	while (!status && resp->opcnt < args->opcnt) {
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index b4bf85f96f6ec..b338473d6e52d 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -352,16 +352,14 @@ purge_old(struct dentry *parent, char *cname, struct nfsd_net *nn)
 	if (nfs4_has_reclaimed_state(name, nn))
 		goto out_free;
 
-	inode_lock_nested(d_inode(parent), I_MUTEX_PARENT);
-	child = lookup_one(&nop_mnt_idmap, &QSTR(cname), parent);
+	child = start_removing_noperm(parent, &QSTR(cname));
 	if (!IS_ERR(child)) {
 		status = vfs_rmdir(&nop_mnt_idmap, d_inode(parent), child, NULL);
 		if (status)
 			printk("failed to remove client recovery directory %pd\n",
 			       child);
-		dput(child);
 	}
-	inode_unlock(d_inode(parent));
+	end_removing(child);
 
 out_free:
 	kfree(name.data);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 6b9c399b89dfb..7d0f72e4b9524 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -76,6 +76,8 @@ static const stateid_t close_stateid = {
 
 static u64 current_sessionid = 1;
 
+bool nfsd_delegts_enabled __read_mostly = true;
+
 #define ZERO_STATEID(stateid) (!memcmp((stateid), &zero_stateid, sizeof(stateid_t)))
 #define ONE_STATEID(stateid)  (!memcmp((stateid), &one_stateid, sizeof(stateid_t)))
 #define CURRENT_STATEID(stateid) (!memcmp((stateid), &currentstateid, sizeof(stateid_t)))
@@ -1253,7 +1255,7 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f
 	if (ret) {
 		struct inode *inode = file_inode(f);
 
-		pr_notice_ratelimited("nfsd: Unable to update timestamps on inode %02x:%02x:%lu: %d\n",
+		pr_notice_ratelimited("nfsd: Unable to update timestamps on inode %02x:%02x:%llu: %d\n",
 					MAJOR(inode->i_sb->s_dev),
 					MINOR(inode->i_sb->s_dev),
 					inode->i_ino, ret);
@@ -2381,6 +2383,10 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name,
 	INIT_LIST_HEAD(&clp->cl_revoked);
 #ifdef CONFIG_NFSD_PNFS
 	INIT_LIST_HEAD(&clp->cl_lo_states);
+#endif
+#ifdef CONFIG_NFSD_SCSILAYOUT
+	xa_init(&clp->cl_dev_fences);
+	mutex_init(&clp->cl_fence_mutex);
 #endif
 	INIT_LIST_HEAD(&clp->async_copies);
 	spin_lock_init(&clp->async_lock);
@@ -2543,6 +2549,9 @@ __destroy_client(struct nfs4_client *clp)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
 	atomic_add_unless(&nn->nfs4_client_count, -1, 0);
 	nfsd4_dec_courtesy_client_count(nn, clp);
+#ifdef CONFIG_NFSD_SCSILAYOUT
+	xa_destroy(&clp->cl_dev_fences);
+#endif
 	free_client(clp);
 	wake_up_all(&expiry_wq);
 }
@@ -2888,7 +2897,7 @@ static void nfs4_show_superblock(struct seq_file *s, struct nfsd_file *f)
 {
 	struct inode *inode = file_inode(f->nf_file);
 
-	seq_printf(s, "superblock: \"%02x:%02x:%ld\"",
+	seq_printf(s, "superblock: \"%02x:%02x:%llu\"",
 					MAJOR(inode->i_sb->s_dev),
 					 MINOR(inode->i_sb->s_dev),
 					 inode->i_ino);
@@ -5535,13 +5544,15 @@ nfsd_break_deleg_cb(struct file_lease *fl)
 static bool nfsd_breaker_owns_lease(struct file_lease *fl)
 {
 	struct nfs4_delegation *dl = fl->c.flc_owner;
+	struct nfsd_thread_local_info *ntli;
 	struct svc_rqst *rqst;
 	struct nfs4_client *clp;
 
 	rqst = nfsd_current_rqst();
 	if (!nfsd_v4client(rqst))
 		return false;
-	clp = *(rqst->rq_lease_breaker);
+	ntli = rqst->rq_private;
+	clp = *ntli->ntli_lease_breaker;
 	return dl->dl_stid.sc_client == clp;
 }
 
@@ -6036,17 +6047,16 @@ nfsd4_verify_setuid_write(struct nfsd4_open *open, struct nfsd_file *nf)
 	return 0;
 }
 
-#ifdef CONFIG_NFSD_V4_DELEG_TIMESTAMPS
+/*
+ * Timestamp delegation was introduced in RFC7862. Runtime switch for disabling
+ * this feature is /sys/kernel/debug/nfsd/delegated_timestamps.
+ */
 static bool nfsd4_want_deleg_timestamps(const struct nfsd4_open *open)
 {
+	if (!nfsd_delegts_enabled)
+		return false;
 	return open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS;
 }
-#else /* CONFIG_NFSD_V4_DELEG_TIMESTAMPS */
-static bool nfsd4_want_deleg_timestamps(const struct nfsd4_open *open)
-{
-	return false;
-}
-#endif /* CONFIG NFSD_V4_DELEG_TIMESTAMPS */
 
 static struct nfs4_delegation *
 nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
@@ -6257,12 +6267,12 @@ nfsd4_add_rdaccess_to_wrdeleg(struct svc_rqst *rqstp, struct nfsd4_open *open,
 			return (false);
 		fp = stp->st_stid.sc_file;
 		spin_lock(&fp->fi_lock);
-		__nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ);
 		if (!fp->fi_fds[O_RDONLY]) {
+			__nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ);
 			fp->fi_fds[O_RDONLY] = nf;
+			fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]);
 			nf = NULL;
 		}
-		fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]);
 		spin_unlock(&fp->fi_lock);
 		if (nf)
 			nfsd_file_put(nf);
@@ -9348,13 +9358,14 @@ __be32
 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
 			     struct nfs4_delegation **pdp)
 {
-	__be32 status;
 	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+	struct nfsd_thread_local_info *ntli = rqstp->rq_private;
 	struct file_lock_context *ctx;
 	struct nfs4_delegation *dp = NULL;
 	struct file_lease *fl;
 	struct nfs4_cb_fattr *ncf;
 	struct inode *inode = d_inode(dentry);
+	__be32 status;
 
 	ctx = locks_inode_context(inode);
 	if (!ctx)
@@ -9375,7 +9386,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
 		break;
 	}
 	if (dp == NULL || dp == NON_NFSD_LEASE ||
-	    dp->dl_recall.cb_clp == *(rqstp->rq_lease_breaker)) {
+	    dp->dl_recall.cb_clp == *(ntli->ntli_lease_breaker)) {
 		spin_unlock(&ctx->flc_lock);
 		if (dp == NON_NFSD_LEASE) {
 			status = nfserrno(nfsd_open_break_lease(inode,
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 41dfba5ab8b81..690f7a3122ec5 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -2598,6 +2598,7 @@ nfsd4_opnum_in_range(struct nfsd4_compoundargs *argp, struct nfsd4_op *op)
 static bool
 nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 {
+	struct nfsd_thread_local_info *ntli = argp->rqstp->rq_private;
 	struct nfsd4_op *op;
 	bool cachethis = false;
 	int auth_slack= argp->rqstp->rq_auth_slack;
@@ -2690,7 +2691,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 	if (argp->minorversion)
 		cachethis = false;
 	svc_reserve_auth(argp->rqstp, max_reply + readbytes);
-	argp->rqstp->rq_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE;
+	ntli->ntli_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE;
 
 	argp->splice_ok = nfsd_read_splice_ok(argp->rqstp);
 	if (readcount > 1 || max_reply > PAGE_SIZE - auth_slack)
@@ -6281,9 +6282,14 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op)
 		int len = xdr->buf->len - (op_status_offset + XDR_UNIT);
 
 		so->so_replay.rp_status = op->status;
-		so->so_replay.rp_buflen = len;
-		read_bytes_from_xdr_buf(xdr->buf, op_status_offset + XDR_UNIT,
+		if (len <= NFSD4_REPLAY_ISIZE) {
+			so->so_replay.rp_buflen = len;
+			read_bytes_from_xdr_buf(xdr->buf,
+						op_status_offset + XDR_UNIT,
 						so->so_replay.rp_buf, len);
+		} else {
+			so->so_replay.rp_buflen = 0;
+		}
 	}
 status:
 	op->status = nfsd4_map_status(op->status,
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index ab13ee9c7fd84..154468ceccdc1 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -467,10 +467,11 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start,
 		      unsigned int len, struct nfsd_cacherep **cacherep)
 {
 	struct nfsd_net		*nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+	struct nfsd_thread_local_info *ntli = rqstp->rq_private;
 	struct nfsd_cacherep	*rp, *found;
 	__wsum			csum;
 	struct nfsd_drc_bucket	*b;
-	int type = rqstp->rq_cachetype;
+	int type = ntli->ntli_cachetype;
 	LIST_HEAD(dispose);
 	int rtn = RC_DOIT;
 
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 4cc8a58fa56a0..0bf01ae411c56 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -11,7 +11,7 @@
 #include <linux/fs_context.h>
 
 #include <linux/sunrpc/svcsock.h>
-#include <linux/lockd/lockd.h>
+#include <linux/lockd/bind.h>
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/gss_api.h>
 #include <linux/sunrpc/rpc_pipe_fs.h>
@@ -149,9 +149,19 @@ static int exports_net_open(struct net *net, struct file *file)
 
 	seq = file->private_data;
 	seq->private = nn->svc_export_cache;
+	get_net(net);
 	return 0;
 }
 
+static int exports_release(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq = file->private_data;
+	struct cache_detail *cd = seq->private;
+
+	put_net(cd->net);
+	return seq_release(inode, file);
+}
+
 static int exports_nfsd_open(struct inode *inode, struct file *file)
 {
 	return exports_net_open(inode->i_sb->s_fs_info, file);
@@ -161,7 +171,7 @@ static const struct file_operations exports_nfsd_operations = {
 	.open		= exports_nfsd_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
-	.release	= seq_release,
+	.release	= exports_release,
 };
 
 static int export_features_show(struct seq_file *m, void *v)
@@ -1376,7 +1386,7 @@ static const struct proc_ops exports_proc_ops = {
 	.proc_open	= exports_proc_open,
 	.proc_read	= seq_read,
 	.proc_lseek	= seq_lseek,
-	.proc_release	= seq_release,
+	.proc_release	= exports_release,
 };
 
 static int create_proc_exports_entry(void)
@@ -2259,9 +2269,12 @@ static int __init init_nfsd(void)
 	if (retval)
 		goto out_free_pnfs;
 	nfsd_lockd_init();	/* lockd->nfsd callbacks */
+	retval = nfsd_export_wq_init();
+	if (retval)
+		goto out_free_lockd;
 	retval = register_pernet_subsys(&nfsd_net_ops);
 	if (retval < 0)
-		goto out_free_lockd;
+		goto out_free_export_wq;
 	retval = register_cld_notifier();
 	if (retval)
 		goto out_free_subsys;
@@ -2290,6 +2303,8 @@ static int __init init_nfsd(void)
 	unregister_cld_notifier();
 out_free_subsys:
 	unregister_pernet_subsys(&nfsd_net_ops);
+out_free_export_wq:
+	nfsd_export_wq_shutdown();
 out_free_lockd:
 	nfsd_lockd_shutdown();
 	nfsd_drc_slab_free();
@@ -2310,6 +2325,7 @@ static void __exit exit_nfsd(void)
 	nfsd4_destroy_laundry_wq();
 	unregister_cld_notifier();
 	unregister_pernet_subsys(&nfsd_net_ops);
+	nfsd_export_wq_shutdown();
 	nfsd_drc_slab_free();
 	nfsd_lockd_shutdown();
 	nfsd4_free_slabs();
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index a01d709533585..7c009f07c90b5 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -82,6 +82,11 @@ extern atomic_t			nfsd_th_cnt;		/* number of available threads */
 
 extern const struct seq_operations nfs_exports_op;
 
+struct nfsd_thread_local_info {
+	struct nfs4_client	**ntli_lease_breaker;
+	int			ntli_cachetype;
+};
+
 /*
  * Common void argument and result helpers
  */
@@ -155,6 +160,7 @@ static inline void nfsd_debugfs_exit(void) {}
 #endif
 
 extern bool nfsd_disable_splice_read __read_mostly;
+extern bool nfsd_delegts_enabled __read_mostly;
 
 enum {
 	/* Any new NFSD_IO enum value must be added at the end */
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index ed85dd43da18e..1ff2a6b40deca 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -105,9 +105,12 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp,
 {
 	/* Check if the request originated from a secure port. */
 	if (rqstp && !nfsd_originating_port_ok(rqstp, cred, exp)) {
-		RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]);
-		dprintk("nfsd: request from insecure port %s!\n",
-		        svc_print_addr(rqstp, buf, sizeof(buf)));
+		if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) {
+			char buf[RPC_MAX_ADDRBUFLEN];
+
+			dprintk("nfsd: request from insecure port %s!\n",
+			        svc_print_addr(rqstp, buf, sizeof(buf)));
+		}
 		return nfserr_perm;
 	}
 
@@ -601,9 +604,9 @@ fh_compose(struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry,
 	struct inode * inode = d_inode(dentry);
 	dev_t ex_dev = exp_sb(exp)->s_dev;
 
-	dprintk("nfsd: fh_compose(exp %02x:%02x/%ld %pd2, ino=%ld)\n",
+	dprintk("nfsd: fh_compose(exp %02x:%02x/%llu %pd2, ino=%llu)\n",
 		MAJOR(ex_dev), MINOR(ex_dev),
-		(long) d_inode(exp->ex_path.dentry)->i_ino,
+		d_inode(exp->ex_path.dentry)->i_ino,
 		dentry,
 		(inode ? inode->i_ino : 0));
 
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 4a04208393b82..4f1ab3222a4d4 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -887,6 +887,7 @@ nfsd(void *vrqstp)
 	struct svc_xprt *perm_sock = list_entry(rqstp->rq_server->sv_permsocks.next, typeof(struct svc_xprt), xpt_list);
 	struct net *net = perm_sock->xpt_net;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+	struct nfsd_thread_local_info ntli = { };
 	bool have_mutex = false;
 
 	/* At this point, the thread shares current->fs
@@ -901,6 +902,10 @@ nfsd(void *vrqstp)
 
 	set_freezable();
 
+	/* use dynamic allocation if ntli should ever become large */
+	static_assert(sizeof(struct nfsd_thread_local_info) < 256);
+	rqstp->rq_private = &ntli;
+
 	/*
 	 * The main request loop
 	 */
@@ -967,6 +972,7 @@ nfsd(void *vrqstp)
  */
 int nfsd_dispatch(struct svc_rqst *rqstp)
 {
+	struct nfsd_thread_local_info *ntli = rqstp->rq_private;
 	const struct svc_procedure *proc = rqstp->rq_procinfo;
 	__be32 *statp = rqstp->rq_accept_statp;
 	struct nfsd_cacherep *rp;
@@ -977,7 +983,7 @@ int nfsd_dispatch(struct svc_rqst *rqstp)
 	 * Give the xdr decoder a chance to change this if it wants
 	 * (necessary in the NFSv4.0 compound case)
 	 */
-	rqstp->rq_cachetype = proc->pc_cachetype;
+	ntli->ntli_cachetype = proc->pc_cachetype;
 
 	/*
 	 * ->pc_decode advances the argument stream past the NFS
@@ -1022,7 +1028,7 @@ int nfsd_dispatch(struct svc_rqst *rqstp)
 	 */
 	smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1);
 
-	nfsd_cache_update(rqstp, rp, rqstp->rq_cachetype, nfs_reply);
+	nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply);
 out_cached_reply:
 	return 1;
 
diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
index db9af780438b5..f7bee4dc5d3d2 100644
--- a/fs/nfsd/pnfs.h
+++ b/fs/nfsd/pnfs.h
@@ -11,6 +11,9 @@
 
 struct xdr_stream;
 
+/* Cap exponential backoff between fence retries at 3 minutes */
+#define	MAX_FENCE_DELAY		((unsigned int)(3 * 60 * HZ))
+
 struct nfsd4_deviceid_map {
 	struct list_head	hash;
 	u64			idx;
@@ -38,7 +41,7 @@ struct nfsd4_layout_ops {
 			struct svc_rqst *rqstp,
 			struct nfsd4_layoutcommit *lcp);
 
-	void (*fence_client)(struct nfs4_layout_stateid *ls,
+	bool (*fence_client)(struct nfs4_layout_stateid *ls,
 			     struct nfsd_file *file);
 };
 
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 6fcbf1e427d4d..ec1c5467012ec 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -456,6 +456,7 @@ struct nfs4_client {
 	struct list_head        cl_lru;         /* tail queue */
 #ifdef CONFIG_NFSD_PNFS
 	struct list_head	cl_lo_states;	/* outstanding layout states */
+	bool			cl_fence_retry_warn;
 #endif
 	struct xdr_netobj	cl_name; 	/* id generated by client */
 	nfs4_verifier		cl_verifier; 	/* generated by client */
@@ -527,6 +528,10 @@ struct nfs4_client {
 
 	struct nfsd4_cb_recall_any	*cl_ra;
 	time64_t		cl_ra_time;
+#ifdef CONFIG_NFSD_SCSILAYOUT
+	struct xarray		cl_dev_fences;
+	struct mutex		cl_fence_mutex;
+#endif
 };
 
 /* struct nfs4_client_reset
@@ -541,11 +546,18 @@ struct nfs4_client_reclaim {
 	struct xdr_netobj	cr_princhash;
 };
 
-/* A reasonable value for REPLAY_ISIZE was estimated as follows:  
- * The OPEN response, typically the largest, requires 
- *   4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) +  8(verifier) + 
- *   4(deleg. type) + 8(deleg. stateid) + 4(deleg. recall flag) + 
- *   20(deleg. space limit) + ~32(deleg. ace) = 112 bytes 
+/*
+ * REPLAY_ISIZE is sized for an OPEN response with delegation:
+ *   4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) +
+ *   8(verifier) + 4(deleg. type) + 8(deleg. stateid) +
+ *   4(deleg. recall flag) + 20(deleg. space limit) +
+ *   ~32(deleg. ace) = 112 bytes
+ *
+ * Some responses can exceed this. A LOCK denial includes the conflicting
+ * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). Responses
+ * larger than REPLAY_ISIZE are not cached in rp_ibuf; only rp_status is
+ * saved. Enlarging this constant increases the size of every
+ * nfs4_stateowner.
  */
 
 #define NFSD4_REPLAY_ISIZE       112 
@@ -735,6 +747,10 @@ struct nfs4_layout_stateid {
 	stateid_t			ls_recall_sid;
 	bool				ls_recalled;
 	struct mutex			ls_mutex;
+
+	struct delayed_work		ls_fence_work;
+	unsigned int			ls_fence_delay;
+	bool				ls_fenced;
 };
 
 static inline struct nfs4_layout_stateid *layoutstateid(struct nfs4_stid *s)
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index c884c3f34afb0..eafdf7b7890fd 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -1163,7 +1163,7 @@ nfsd_direct_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	} else if (unlikely(host_err == -EINVAL)) {
 		struct inode *inode = d_inode(fhp->fh_dentry);
 
-		pr_info_ratelimited("nfsd: Direct I/O alignment failure on %s/%ld\n",
+		pr_info_ratelimited("nfsd: Direct I/O alignment failure on %s/%llu\n",
 				    inode->i_sb->s_id, inode->i_ino);
 		host_err = -ESERVERFAULT;
 	}
diff --git a/fs/nilfs2/alloc.c b/fs/nilfs2/alloc.c
index e7eebb04f9a40..7b1cd2baefcf2 100644
--- a/fs/nilfs2/alloc.c
+++ b/fs/nilfs2/alloc.c
@@ -707,7 +707,7 @@ void nilfs_palloc_commit_free_entry(struct inode *inode,
 
 	if (!nilfs_clear_bit_atomic(lock, group_offset, bitmap))
 		nilfs_warn(inode->i_sb,
-			   "%s (ino=%lu): entry number %llu already freed",
+			   "%s (ino=%llu): entry number %llu already freed",
 			   __func__, inode->i_ino,
 			   (unsigned long long)req->pr_entry_nr);
 	else
@@ -748,7 +748,7 @@ void nilfs_palloc_abort_alloc_entry(struct inode *inode,
 
 	if (!nilfs_clear_bit_atomic(lock, group_offset, bitmap))
 		nilfs_warn(inode->i_sb,
-			   "%s (ino=%lu): entry number %llu already freed",
+			   "%s (ino=%llu): entry number %llu already freed",
 			   __func__, inode->i_ino,
 			   (unsigned long long)req->pr_entry_nr);
 	else
@@ -861,7 +861,7 @@ int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems)
 			if (!nilfs_clear_bit_atomic(lock, group_offset,
 						    bitmap)) {
 				nilfs_warn(inode->i_sb,
-					   "%s (ino=%lu): entry number %llu already freed",
+					   "%s (ino=%llu): entry number %llu already freed",
 					   __func__, inode->i_ino,
 					   (unsigned long long)entry_nrs[j]);
 			} else {
@@ -906,7 +906,7 @@ int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems)
 							      last_nrs[k]);
 			if (ret && ret != -ENOENT)
 				nilfs_warn(inode->i_sb,
-					   "error %d deleting block that object (entry=%llu, ino=%lu) belongs to",
+					   "error %d deleting block that object (entry=%llu, ino=%llu) belongs to",
 					   ret, (unsigned long long)last_nrs[k],
 					   inode->i_ino);
 		}
@@ -923,7 +923,7 @@ int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems)
 			ret = nilfs_palloc_delete_bitmap_block(inode, group);
 			if (ret && ret != -ENOENT)
 				nilfs_warn(inode->i_sb,
-					   "error %d deleting bitmap block of group=%lu, ino=%lu",
+					   "error %d deleting bitmap block of group=%lu, ino=%llu",
 					   ret, group, inode->i_ino);
 		}
 	}
diff --git a/fs/nilfs2/bmap.c b/fs/nilfs2/bmap.c
index ccc1a7aa52d20..824f2bd91c167 100644
--- a/fs/nilfs2/bmap.c
+++ b/fs/nilfs2/bmap.c
@@ -33,7 +33,7 @@ static int nilfs_bmap_convert_error(struct nilfs_bmap *bmap,
 
 	if (err == -EINVAL) {
 		__nilfs_error(inode->i_sb, fname,
-			      "broken bmap (inode number=%lu)", inode->i_ino);
+			      "broken bmap (inode number=%llu)", inode->i_ino);
 		err = -EIO;
 	}
 	return err;
diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c
index 5683671290920..2e553d698d0f3 100644
--- a/fs/nilfs2/btnode.c
+++ b/fs/nilfs2/btnode.c
@@ -64,7 +64,7 @@ nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr)
 		 * clearing of an abandoned b-tree node is missing somewhere).
 		 */
 		nilfs_error(inode->i_sb,
-			    "state inconsistency probably due to duplicate use of b-tree node block address %llu (ino=%lu)",
+			    "state inconsistency probably due to duplicate use of b-tree node block address %llu (ino=%llu)",
 			    (unsigned long long)blocknr, inode->i_ino);
 		goto failed;
 	}
diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c
index dd0c8e560ef6a..64d5f7c5ab440 100644
--- a/fs/nilfs2/btree.c
+++ b/fs/nilfs2/btree.c
@@ -10,7 +10,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/errno.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include "nilfs.h"
 #include "page.h"
 #include "btnode.h"
@@ -353,7 +353,7 @@ static int nilfs_btree_node_broken(const struct nilfs_btree_node *node,
 		     nchildren <= 0 ||
 		     nchildren > NILFS_BTREE_NODE_NCHILDREN_MAX(size))) {
 		nilfs_crit(inode->i_sb,
-			   "bad btree node (ino=%lu, blocknr=%llu): level = %d, flags = 0x%x, nchildren = %d",
+			   "bad btree node (ino=%llu, blocknr=%llu): level = %d, flags = 0x%x, nchildren = %d",
 			   inode->i_ino, (unsigned long long)blocknr, level,
 			   flags, nchildren);
 		ret = 1;
@@ -384,7 +384,7 @@ static int nilfs_btree_root_broken(const struct nilfs_btree_node *node,
 		     nchildren > NILFS_BTREE_ROOT_NCHILDREN_MAX ||
 		     (nchildren == 0 && level > NILFS_BTREE_LEVEL_NODE_MIN))) {
 		nilfs_crit(inode->i_sb,
-			   "bad btree root (ino=%lu): level = %d, flags = 0x%x, nchildren = %d",
+			   "bad btree root (ino=%llu): level = %d, flags = 0x%x, nchildren = %d",
 			   inode->i_ino, level, flags, nchildren);
 		ret = 1;
 	}
@@ -453,7 +453,7 @@ static int nilfs_btree_bad_node(const struct nilfs_bmap *btree,
 	if (unlikely(nilfs_btree_node_get_level(node) != level)) {
 		dump_stack();
 		nilfs_crit(btree->b_inode->i_sb,
-			   "btree level mismatch (ino=%lu): %d != %d",
+			   "btree level mismatch (ino=%llu): %d != %d",
 			   btree->b_inode->i_ino,
 			   nilfs_btree_node_get_level(node), level);
 		return 1;
@@ -521,7 +521,7 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr,
  out_no_wait:
 	if (!buffer_uptodate(bh)) {
 		nilfs_err(btree->b_inode->i_sb,
-			  "I/O error reading b-tree node block (ino=%lu, blocknr=%llu)",
+			  "I/O error reading b-tree node block (ino=%llu, blocknr=%llu)",
 			  btree->b_inode->i_ino, (unsigned long long)ptr);
 		brelse(bh);
 		return -EIO;
@@ -2104,7 +2104,7 @@ static int nilfs_btree_propagate(struct nilfs_bmap *btree,
 	if (ret < 0) {
 		if (unlikely(ret == -ENOENT)) {
 			nilfs_crit(btree->b_inode->i_sb,
-				   "writing node/leaf block does not appear in b-tree (ino=%lu) at key=%llu, level=%d",
+				   "writing node/leaf block does not appear in b-tree (ino=%llu) at key=%llu, level=%d",
 				   btree->b_inode->i_ino,
 				   (unsigned long long)key, level);
 			ret = -EINVAL;
@@ -2146,7 +2146,7 @@ static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree,
 	    level >= NILFS_BTREE_LEVEL_MAX) {
 		dump_stack();
 		nilfs_warn(btree->b_inode->i_sb,
-			   "invalid btree level: %d (key=%llu, ino=%lu, blocknr=%llu)",
+			   "invalid btree level: %d (key=%llu, ino=%llu, blocknr=%llu)",
 			   level, (unsigned long long)key,
 			   btree->b_inode->i_ino,
 			   (unsigned long long)bh->b_blocknr);
diff --git a/fs/nilfs2/dir.c b/fs/nilfs2/dir.c
index b243199036dfa..3653db5cdb651 100644
--- a/fs/nilfs2/dir.c
+++ b/fs/nilfs2/dir.c
@@ -150,7 +150,7 @@ static bool nilfs_check_folio(struct folio *folio, char *kaddr)
 
 Ebadsize:
 	nilfs_error(sb,
-		    "size of directory #%lu is not a multiple of chunk size",
+		    "size of directory #%llu is not a multiple of chunk size",
 		    dir->i_ino);
 	goto fail;
 Eshort:
@@ -169,7 +169,7 @@ static bool nilfs_check_folio(struct folio *folio, char *kaddr)
 	error = "disallowed inode number";
 bad_entry:
 	nilfs_error(sb,
-		    "bad entry in directory #%lu: %s - offset=%lu, inode=%lu, rec_len=%zd, name_len=%d",
+		    "bad entry in directory #%llu: %s - offset=%lu, inode=%lu, rec_len=%zd, name_len=%d",
 		    dir->i_ino, error, (folio->index << PAGE_SHIFT) + offs,
 		    (unsigned long)le64_to_cpu(p->inode),
 		    rec_len, p->name_len);
@@ -177,7 +177,7 @@ static bool nilfs_check_folio(struct folio *folio, char *kaddr)
 Eend:
 	p = (struct nilfs_dir_entry *)(kaddr + offs);
 	nilfs_error(sb,
-		    "entry in directory #%lu spans the page boundary offset=%lu, inode=%lu",
+		    "entry in directory #%llu spans the page boundary offset=%lu, inode=%lu",
 		    dir->i_ino, (folio->index << PAGE_SHIFT) + offs,
 		    (unsigned long)le64_to_cpu(p->inode));
 fail:
@@ -251,7 +251,7 @@ static int nilfs_readdir(struct file *file, struct dir_context *ctx)
 
 		kaddr = nilfs_get_folio(inode, n, &folio);
 		if (IS_ERR(kaddr)) {
-			nilfs_error(sb, "bad page in #%lu", inode->i_ino);
+			nilfs_error(sb, "bad page in #%llu", inode->i_ino);
 			ctx->pos += PAGE_SIZE - offset;
 			return -EIO;
 		}
@@ -336,7 +336,7 @@ struct nilfs_dir_entry *nilfs_find_entry(struct inode *dir,
 		/* next folio is past the blocks we've got */
 		if (unlikely(n > (dir->i_blocks >> (PAGE_SHIFT - 9)))) {
 			nilfs_error(dir->i_sb,
-			       "dir %lu size %lld exceeds block count %llu",
+			       "dir %llu size %lld exceeds block count %llu",
 			       dir->i_ino, dir->i_size,
 			       (unsigned long long)dir->i_blocks);
 			goto out;
@@ -382,7 +382,7 @@ struct nilfs_dir_entry *nilfs_dotdot(struct inode *dir, struct folio **foliop)
 	return next_de;
 
 fail:
-	nilfs_error(dir->i_sb, "directory #%lu %s", dir->i_ino, msg);
+	nilfs_error(dir->i_sb, "directory #%llu %s", dir->i_ino, msg);
 	folio_release_kmap(folio, de);
 	return NULL;
 }
diff --git a/fs/nilfs2/direct.c b/fs/nilfs2/direct.c
index 2d8dc6b35b547..8bd0b1374e25f 100644
--- a/fs/nilfs2/direct.c
+++ b/fs/nilfs2/direct.c
@@ -338,7 +338,7 @@ static int nilfs_direct_assign(struct nilfs_bmap *bmap,
 	key = nilfs_bmap_data_get_key(bmap, *bh);
 	if (unlikely(key > NILFS_DIRECT_KEY_MAX)) {
 		nilfs_crit(bmap->b_inode->i_sb,
-			   "%s (ino=%lu): invalid key: %llu",
+			   "%s (ino=%llu): invalid key: %llu",
 			   __func__,
 			   bmap->b_inode->i_ino, (unsigned long long)key);
 		return -EINVAL;
@@ -346,7 +346,7 @@ static int nilfs_direct_assign(struct nilfs_bmap *bmap,
 	ptr = nilfs_direct_get_ptr(bmap, key);
 	if (unlikely(ptr == NILFS_BMAP_INVALID_PTR)) {
 		nilfs_crit(bmap->b_inode->i_sb,
-			   "%s (ino=%lu): invalid pointer: %llu",
+			   "%s (ino=%llu): invalid pointer: %llu",
 			   __func__,
 			   bmap->b_inode->i_ino, (unsigned long long)ptr);
 		return -EINVAL;
diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c
index 561c220799c7a..62d4c1b787e95 100644
--- a/fs/nilfs2/gcinode.c
+++ b/fs/nilfs2/gcinode.c
@@ -137,7 +137,7 @@ int nilfs_gccache_wait_and_mark_dirty(struct buffer_head *bh)
 		struct inode *inode = bh->b_folio->mapping->host;
 
 		nilfs_err(inode->i_sb,
-			  "I/O error reading %s block for GC (ino=%lu, vblocknr=%llu)",
+			  "I/O error reading %s block for GC (ino=%llu, vblocknr=%llu)",
 			  buffer_nilfs_node(bh) ? "node" : "data",
 			  inode->i_ino, (unsigned long long)bh->b_blocknr);
 		return -EIO;
diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c
index 51bde45d58650..51f7e125a311b 100644
--- a/fs/nilfs2/inode.c
+++ b/fs/nilfs2/inode.c
@@ -108,7 +108,7 @@ int nilfs_get_block(struct inode *inode, sector_t blkoff,
 				 * be locked in this case.
 				 */
 				nilfs_warn(inode->i_sb,
-					   "%s (ino=%lu): a race condition while inserting a data block at offset=%llu",
+					   "%s (ino=%llu): a race condition while inserting a data block at offset=%llu",
 					   __func__, inode->i_ino,
 					   (unsigned long long)blkoff);
 				err = -EAGAIN;
@@ -789,7 +789,7 @@ static void nilfs_truncate_bmap(struct nilfs_inode_info *ii,
 		goto repeat;
 
 failed:
-	nilfs_warn(ii->vfs_inode.i_sb, "error %d truncating bmap (ino=%lu)",
+	nilfs_warn(ii->vfs_inode.i_sb, "error %d truncating bmap (ino=%llu)",
 		   ret, ii->vfs_inode.i_ino);
 }
 
@@ -1026,7 +1026,7 @@ int nilfs_set_file_dirty(struct inode *inode, unsigned int nr_dirty)
 			 * this inode.
 			 */
 			nilfs_warn(inode->i_sb,
-				   "cannot set file dirty (ino=%lu): the file is being freed",
+				   "cannot set file dirty (ino=%llu): the file is being freed",
 				   inode->i_ino);
 			spin_unlock(&nilfs->ns_inode_lock);
 			return -EINVAL; /*
@@ -1057,7 +1057,7 @@ int __nilfs_mark_inode_dirty(struct inode *inode, int flags)
 	err = nilfs_load_inode_block(inode, &ibh);
 	if (unlikely(err)) {
 		nilfs_warn(inode->i_sb,
-			   "cannot mark inode dirty (ino=%lu): error %d loading inode block",
+			   "cannot mark inode dirty (ino=%llu): error %d loading inode block",
 			   inode->i_ino, err);
 		return err;
 	}
diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c
index 946b0d3534a5f..09adb40c65e50 100644
--- a/fs/nilfs2/mdt.c
+++ b/fs/nilfs2/mdt.c
@@ -203,7 +203,7 @@ static int nilfs_mdt_read_block(struct inode *inode, unsigned long block,
 	err = -EIO;
 	if (!buffer_uptodate(first_bh)) {
 		nilfs_err(inode->i_sb,
-			  "I/O error reading meta-data file (ino=%lu, block-offset=%lu)",
+			  "I/O error reading meta-data file (ino=%llu, block-offset=%lu)",
 			  inode->i_ino, block);
 		goto failed_bh;
 	}
diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c
index 40f4b1a28705b..40ac679ec56e4 100644
--- a/fs/nilfs2/namei.c
+++ b/fs/nilfs2/namei.c
@@ -292,7 +292,7 @@ static int nilfs_do_unlink(struct inode *dir, struct dentry *dentry)
 
 	if (!inode->i_nlink) {
 		nilfs_warn(inode->i_sb,
-			   "deleting nonexistent file (ino=%lu), %d",
+			   "deleting nonexistent file (ino=%llu), %d",
 			   inode->i_ino, inode->i_nlink);
 		set_nlink(inode, 1);
 	}
diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c
index 56c4da417b6a1..a9d8aa65416fd 100644
--- a/fs/nilfs2/page.c
+++ b/fs/nilfs2/page.c
@@ -14,7 +14,7 @@
 #include <linux/page-flags.h>
 #include <linux/list.h>
 #include <linux/highmem.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/gfp.h>
 #include "nilfs.h"
 #include "page.h"
diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c
index 098a3bd103e04..1491a4d4b1e19 100644
--- a/fs/nilfs2/segment.c
+++ b/fs/nilfs2/segment.c
@@ -19,7 +19,7 @@
 #include <linux/freezer.h>
 #include <linux/kthread.h>
 #include <linux/crc32.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
 
@@ -2024,7 +2024,7 @@ static int nilfs_segctor_collect_dirty_files(struct nilfs_sc_info *sci,
 				ifile, ii->vfs_inode.i_ino, &ibh);
 			if (unlikely(err)) {
 				nilfs_warn(sci->sc_super,
-					   "log writer: error %d getting inode block (ino=%lu)",
+					   "log writer: error %d getting inode block (ino=%llu)",
 					   err, ii->vfs_inode.i_ino);
 				return err;
 			}
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
index c2dcb25151deb..ae904451dfc09 100644
--- a/fs/notify/fanotify/fanotify_user.c
+++ b/fs/notify/fanotify/fanotify_user.c
@@ -1210,6 +1210,7 @@ static int fanotify_find_path(int dfd, const char __user *filename,
 
 		*path = fd_file(f)->f_path;
 		path_get(path);
+		ret = 0;
 	} else {
 		unsigned int lookup_flags = 0;
 
@@ -1219,22 +1220,7 @@ static int fanotify_find_path(int dfd, const char __user *filename,
 			lookup_flags |= LOOKUP_DIRECTORY;
 
 		ret = user_path_at(dfd, filename, lookup_flags, path);
-		if (ret)
-			goto out;
-	}
-
-	/* you can only watch an inode if you have read permissions on it */
-	ret = path_permission(path, MAY_READ);
-	if (ret) {
-		path_put(path);
-		goto out;
 	}
-
-	ret = security_path_notify(path, mask, obj_type);
-	if (ret)
-		path_put(path);
-
-out:
 	return ret;
 }
 
@@ -1615,17 +1601,18 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags)
 	pr_debug("%s: flags=%x event_f_flags=%x\n",
 		 __func__, flags, event_f_flags);
 
-	if (!capable(CAP_SYS_ADMIN)) {
-		/*
-		 * An unprivileged user can setup an fanotify group with
-		 * limited functionality - an unprivileged group is limited to
-		 * notification events with file handles or mount ids and it
-		 * cannot use unlimited queue/marks.
-		 */
-		if ((flags & FANOTIFY_ADMIN_INIT_FLAGS) ||
-		    !(flags & (FANOTIFY_FID_BITS | FAN_REPORT_MNT)))
-			return -EPERM;
+	/*
+	 * An unprivileged user can setup an fanotify group with limited
+	 * functionality - an unprivileged group is limited to notification
+	 * events with file handles or mount ids and it cannot use unlimited
+	 * queue/marks.
+	 */
+	if (((flags & FANOTIFY_ADMIN_INIT_FLAGS) ||
+	     !(flags & (FANOTIFY_FID_BITS | FAN_REPORT_MNT))) &&
+	    !capable(CAP_SYS_ADMIN))
+		return -EPERM;
 
+	if (!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {
 		/*
 		 * Setting the internal flag FANOTIFY_UNPRIV on the group
 		 * prevents setting mount/filesystem marks on this group and
@@ -1990,8 +1977,8 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask,
 	 * A user is allowed to setup sb/mount/mntns marks only if it is
 	 * capable in the user ns where the group was created.
 	 */
-	if (!ns_capable(group->user_ns, CAP_SYS_ADMIN) &&
-	    mark_type != FAN_MARK_INODE)
+	if (mark_type != FAN_MARK_INODE &&
+	    !ns_capable(group->user_ns, CAP_SYS_ADMIN))
 		return -EPERM;
 
 	/*
@@ -2057,6 +2044,15 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask,
 			goto path_put_and_out;
 	}
 
+	/* you can only watch an inode if you have read permissions on it */
+	ret = path_permission(&path, MAY_READ);
+	if (ret)
+		goto path_put_and_out;
+
+	ret = security_path_notify(&path, mask, obj_type);
+	if (ret)
+		goto path_put_and_out;
+
 	if (fid_mode) {
 		ret = fanotify_test_fsid(path.dentry, flags, &__fsid);
 		if (ret)
diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c
index 9cc7eb8636437..0f731eddeb8be 100644
--- a/fs/notify/fdinfo.c
+++ b/fs/notify/fdinfo.c
@@ -84,7 +84,7 @@ static void inotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark)
 	inode_mark = container_of(mark, struct inotify_inode_mark, fsn_mark);
 	inode = igrab(fsnotify_conn_inode(mark->connector));
 	if (inode) {
-		seq_printf(m, "inotify wd:%x ino:%lx sdev:%x mask:%x ignored_mask:0 ",
+		seq_printf(m, "inotify wd:%x ino:%llx sdev:%x mask:%x ignored_mask:0 ",
 			   inode_mark->wd, inode->i_ino, inode->i_sb->s_dev,
 			   inotify_mark_user_mask(mark));
 		show_mark_fhandle(m, inode);
@@ -111,7 +111,7 @@ static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark)
 		inode = igrab(fsnotify_conn_inode(mark->connector));
 		if (!inode)
 			return;
-		seq_printf(m, "fanotify ino:%lx sdev:%x mflags:%x mask:%x ignored_mask:%x ",
+		seq_printf(m, "fanotify ino:%llx sdev:%x mflags:%x mask:%x ignored_mask:%x ",
 			   inode->i_ino, inode->i_sb->s_dev,
 			   mflags, mark->mask, mark->ignore_mask);
 		show_mark_fhandle(m, inode);
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c
index 5e1845f2c25dd..ed37491c16189 100644
--- a/fs/notify/inotify/inotify_user.c
+++ b/fs/notify/inotify/inotify_user.c
@@ -573,7 +573,7 @@ static int inotify_update_existing_watch(struct fsnotify_group *group,
 
 		/* update the inode with this new fsn_mark */
 		if (dropped || do_inode)
-			fsnotify_recalc_mask(inode->i_fsnotify_marks);
+			fsnotify_recalc_mask(fsn_mark->connector);
 
 	}
 
@@ -621,6 +621,7 @@ static int inotify_new_watch(struct fsnotify_group *group,
 	if (ret) {
 		/* we failed to get on the inode, get off the idr */
 		inotify_remove_from_idr(group, tmp_i_mark);
+		dec_inotify_watches(group->inotify_data.ucounts);
 		goto out_err;
 	}
 
diff --git a/fs/nsfs.c b/fs/nsfs.c
index c215878d55e87..51e8c9430477b 100644
--- a/fs/nsfs.c
+++ b/fs/nsfs.c
@@ -46,7 +46,7 @@ static char *ns_dname(struct dentry *dentry, char *buffer, int buflen)
 	struct ns_common *ns = inode->i_private;
 	const struct proc_ns_operations *ns_ops = ns->ops;
 
-	return dynamic_dname(buffer, buflen, "%s:[%lu]",
+	return dynamic_dname(buffer, buflen, "%s:[%llu]",
 		ns_ops->name, inode->i_ino);
 }
 
@@ -407,7 +407,7 @@ static int nsfs_show_path(struct seq_file *seq, struct dentry *dentry)
 	const struct ns_common *ns = inode->i_private;
 	const struct proc_ns_operations *ns_ops = ns->ops;
 
-	seq_printf(seq, "%s:[%lu]", ns_ops->name, inode->i_ino);
+	seq_printf(seq, "%s:[%llu]", ns_ops->name, inode->i_ino);
 	return 0;
 }
 
diff --git a/fs/ntfs/Kconfig b/fs/ntfs/Kconfig
new file mode 100644
index 0000000000000..6a6acde9ba91e
--- /dev/null
+++ b/fs/ntfs/Kconfig
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config NTFS_FS
+	tristate "NTFS file system support"
+	select NLS
+	select FS_IOMAP
+	help
+	  NTFS is the file system of Microsoft Windows NT, 2000, XP and 2003.
+	  This allows you to mount devices formatted with the ntfs file system.
+
+	  To compile this as a module, choose M here: the module will be called
+	  ntfs.
+
+config NTFS_DEBUG
+	bool "NTFS debugging support"
+	depends on NTFS_FS
+	help
+	  If you are experiencing any problems with the NTFS file system, say
+	  Y here.  This will result in additional consistency checks to be
+	  performed by the driver as well as additional debugging messages to
+	  be written to the system log.  Note that debugging messages are
+	  disabled by default.  To enable them, supply the option debug_msgs=1
+	  at the kernel command line when booting the kernel or as an option
+	  to insmod when loading the ntfs module.  Once the driver is active,
+	  you can enable debugging messages by doing (as root):
+	  echo 1 > /proc/sys/fs/ntfs-debug
+	  Replacing the "1" with "0" would disable debug messages.
+
+	  If you leave debugging messages disabled, this results in little
+	  overhead, but enabling debug messages results in very significant
+	  slowdown of the system.
+
+	  When reporting bugs, please try to have available a full dump of
+	  debugging messages while the misbehaviour was occurring.
+
+config NTFS_FS_POSIX_ACL
+	bool "NTFS POSIX Access Control Lists"
+	depends on NTFS_FS
+	select FS_POSIX_ACL
+	help
+	  POSIX Access Control Lists (ACLs) support additional access rights
+	  for users and groups beyond the standard owner/group/world scheme.
+
+	  This option enables ACL support for ntfs, providing functional parity
+	  with ntfs3 drivier.
+
+	  NOTE: this is linux only feature. Windows will ignore these ACLs.
+
+	  If you don't know what Access Control Lists are, say N.
diff --git a/fs/ntfs/Makefile b/fs/ntfs/Makefile
new file mode 100644
index 0000000000000..0ce4d9a9388af
--- /dev/null
+++ b/fs/ntfs/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_NTFS_FS) += ntfs.o
+
+ntfs-y := aops.o attrib.o collate.o dir.o file.o index.o inode.o \
+	  mft.o mst.o namei.o runlist.o super.o unistr.o attrlist.o ea.o \
+	  upcase.o bitmap.o lcnalloc.o logfile.o reparse.o compress.o \
+	  iomap.o debug.o sysctl.o quota.o object_id.o bdev-io.o
+
+ccflags-$(CONFIG_NTFS_DEBUG) += -DDEBUG
diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c
new file mode 100644
index 0000000000000..78d1ce41958e0
--- /dev/null
+++ b/fs/ntfs/aops.c
@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel address space operations and page cache handling.
+ *
+ * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/writeback.h>
+
+#include "attrib.h"
+#include "mft.h"
+#include "ntfs.h"
+#include "debug.h"
+#include "iomap.h"
+
+/*
+ * ntfs_read_folio - Read data for a folio from the device
+ * @file:	open file to which the folio @folio belongs or NULL
+ * @folio:	page cache folio to fill with data
+ *
+ * This function handles reading data into the page cache. It first checks
+ * for specific ntfs attribute type like encryption and compression.
+ *
+ * - If the attribute is encrypted, access is denied (-EACCES) because
+ *   decryption is not supported in this path.
+ * - If the attribute is non-resident and compressed, the read operation is
+ *   delegated to ntfs_read_compressed_block().
+ * - For normal resident or non-resident attribute, it utilizes the generic
+ *   iomap infrastructure via iomap_bio_read_folio() to perform the I/O.
+ *
+ * Return: 0 on success, or -errno on error.
+ */
+static int ntfs_read_folio(struct file *file, struct folio *folio)
+{
+	struct ntfs_inode *ni = NTFS_I(folio->mapping->host);
+
+	/*
+	 * Only $DATA attributes can be encrypted and only unnamed $DATA
+	 * attributes can be compressed.  Index root can have the flags set but
+	 * this means to create compressed/encrypted files, not that the
+	 * attribute is compressed/encrypted.  Note we need to check for
+	 * AT_INDEX_ALLOCATION since this is the type of both directory and
+	 * index inodes.
+	 */
+	if (ni->type != AT_INDEX_ALLOCATION) {
+		/*
+		 * EFS-encrypted files are not supported.
+		 * (decryption/encryption is not implemented yet)
+		 */
+		if (NInoEncrypted(ni)) {
+			folio_unlock(folio);
+			return -EOPNOTSUPP;
+		}
+		/* Compressed data streams are handled in compress.c. */
+		if (NInoNonResident(ni) && NInoCompressed(ni))
+			return ntfs_read_compressed_block(folio);
+	}
+
+	iomap_bio_read_folio(folio, &ntfs_read_iomap_ops);
+	return 0;
+}
+
+/*
+ * ntfs_bmap - map logical file block to physical device block
+ * @mapping:	address space mapping to which the block to be mapped belongs
+ * @block:	logical block to map to its physical device block
+ *
+ * For regular, non-resident files (i.e. not compressed and not encrypted), map
+ * the logical @block belonging to the file described by the address space
+ * mapping @mapping to its physical device block.
+ *
+ * The size of the block is equal to the @s_blocksize field of the super block
+ * of the mounted file system which is guaranteed to be smaller than or equal
+ * to the cluster size thus the block is guaranteed to fit entirely inside the
+ * cluster which means we do not need to care how many contiguous bytes are
+ * available after the beginning of the block.
+ *
+ * Return the physical device block if the mapping succeeded or 0 if the block
+ * is sparse or there was an error.
+ *
+ * Note: This is a problem if someone tries to run bmap() on $Boot system file
+ * as that really is in block zero but there is nothing we can do.  bmap() is
+ * just broken in that respect (just like it cannot distinguish sparse from
+ * not available or error).
+ */
+static sector_t ntfs_bmap(struct address_space *mapping, sector_t block)
+{
+	s64 ofs, size;
+	loff_t i_size;
+	s64 lcn;
+	unsigned long blocksize, flags;
+	struct ntfs_inode *ni = NTFS_I(mapping->host);
+	struct ntfs_volume *vol = ni->vol;
+	unsigned int delta;
+	unsigned char blocksize_bits;
+
+	ntfs_debug("Entering for mft_no 0x%llx, logical block 0x%llx.",
+			ni->mft_no, (unsigned long long)block);
+	if (ni->type != AT_DATA || !NInoNonResident(ni) || NInoEncrypted(ni) ||
+	    NInoMstProtected(ni)) {
+		ntfs_error(vol->sb, "BMAP does not make sense for %s attributes, returning 0.",
+				(ni->type != AT_DATA) ? "non-data" :
+				(!NInoNonResident(ni) ? "resident" :
+				"encrypted"));
+		return 0;
+	}
+	/* None of these can happen. */
+	blocksize = vol->sb->s_blocksize;
+	blocksize_bits = vol->sb->s_blocksize_bits;
+	ofs = (s64)block << blocksize_bits;
+	read_lock_irqsave(&ni->size_lock, flags);
+	size = ni->initialized_size;
+	i_size = i_size_read(VFS_I(ni));
+	read_unlock_irqrestore(&ni->size_lock, flags);
+	/*
+	 * If the offset is outside the initialized size or the block straddles
+	 * the initialized size then pretend it is a hole unless the
+	 * initialized size equals the file size.
+	 */
+	if (unlikely(ofs >= size || (ofs + blocksize > size && size < i_size)))
+		goto hole;
+	down_read(&ni->runlist.lock);
+	lcn = ntfs_attr_vcn_to_lcn_nolock(ni, ntfs_bytes_to_cluster(vol, ofs),
+			false);
+	up_read(&ni->runlist.lock);
+	if (unlikely(lcn < LCN_HOLE)) {
+		/*
+		 * Step down to an integer to avoid gcc doing a long long
+		 * comparision in the switch when we know @lcn is between
+		 * LCN_HOLE and LCN_EIO (i.e. -1 to -5).
+		 *
+		 * Otherwise older gcc (at least on some architectures) will
+		 * try to use __cmpdi2() which is of course not available in
+		 * the kernel.
+		 */
+		switch ((int)lcn) {
+		case LCN_ENOENT:
+			/*
+			 * If the offset is out of bounds then pretend it is a
+			 * hole.
+			 */
+			goto hole;
+		case LCN_ENOMEM:
+			ntfs_error(vol->sb,
+				"Not enough memory to complete mapping for inode 0x%llx. Returning 0.",
+				ni->mft_no);
+			break;
+		default:
+			ntfs_error(vol->sb,
+				"Failed to complete mapping for inode 0x%llx.  Run chkdsk. Returning 0.",
+				ni->mft_no);
+			break;
+		}
+		return 0;
+	}
+	if (lcn < 0) {
+		/* It is a hole. */
+hole:
+		ntfs_debug("Done (returning hole).");
+		return 0;
+	}
+	/*
+	 * The block is really allocated and fullfils all our criteria.
+	 * Convert the cluster to units of block size and return the result.
+	 */
+	delta = ofs & vol->cluster_size_mask;
+	if (unlikely(sizeof(block) < sizeof(lcn))) {
+		block = lcn = (ntfs_cluster_to_bytes(vol, lcn) + delta) >>
+				blocksize_bits;
+		/* If the block number was truncated return 0. */
+		if (unlikely(block != lcn)) {
+			ntfs_error(vol->sb,
+				"Physical block 0x%llx is too large to be returned, returning 0.",
+				(long long)lcn);
+			return 0;
+		}
+	} else
+		block = (ntfs_cluster_to_bytes(vol, lcn) + delta) >>
+				blocksize_bits;
+	ntfs_debug("Done (returning block 0x%llx).", (unsigned long long)lcn);
+	return block;
+}
+
+static void ntfs_readahead(struct readahead_control *rac)
+{
+	struct address_space *mapping = rac->mapping;
+	struct inode *inode = mapping->host;
+	struct ntfs_inode *ni = NTFS_I(inode);
+
+	/*
+	 * Resident files are not cached in the page cache,
+	 * and readahead is not implemented for compressed files.
+	 */
+	if (!NInoNonResident(ni) || NInoCompressed(ni))
+		return;
+	iomap_bio_readahead(rac, &ntfs_read_iomap_ops);
+}
+
+static int ntfs_writepages(struct address_space *mapping,
+		struct writeback_control *wbc)
+{
+	struct inode *inode = mapping->host;
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct iomap_writepage_ctx wpc = {
+		.inode		= mapping->host,
+		.wbc		= wbc,
+		.ops		= &ntfs_writeback_ops,
+	};
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	if (!NInoNonResident(ni))
+		return 0;
+
+	/*
+	 * EFS-encrypted files are not supported.
+	 * (decryption/encryption is not implemented yet)
+	 */
+	if (NInoEncrypted(ni)) {
+		ntfs_debug("Encrypted I/O not supported");
+		return -EOPNOTSUPP;
+	}
+
+	return iomap_writepages(&wpc);
+}
+
+static int ntfs_swap_activate(struct swap_info_struct *sis,
+		struct file *swap_file, sector_t *span)
+{
+	return iomap_swapfile_activate(sis, swap_file, span,
+			&ntfs_read_iomap_ops);
+}
+
+const struct address_space_operations ntfs_aops = {
+	.read_folio		= ntfs_read_folio,
+	.readahead		= ntfs_readahead,
+	.writepages		= ntfs_writepages,
+	.direct_IO		= noop_direct_IO,
+	.dirty_folio		= iomap_dirty_folio,
+	.bmap			= ntfs_bmap,
+	.migrate_folio		= filemap_migrate_folio,
+	.is_partially_uptodate	= iomap_is_partially_uptodate,
+	.error_remove_folio	= generic_error_remove_folio,
+	.release_folio		= iomap_release_folio,
+	.invalidate_folio	= iomap_invalidate_folio,
+	.swap_activate          = ntfs_swap_activate,
+};
+
+const struct address_space_operations ntfs_mft_aops = {
+	.read_folio		= ntfs_read_folio,
+	.readahead		= ntfs_readahead,
+	.writepages		= ntfs_mft_writepages,
+	.dirty_folio		= iomap_dirty_folio,
+	.bmap			= ntfs_bmap,
+	.migrate_folio		= filemap_migrate_folio,
+	.is_partially_uptodate	= iomap_is_partially_uptodate,
+	.error_remove_folio	= generic_error_remove_folio,
+	.release_folio		= iomap_release_folio,
+	.invalidate_folio	= iomap_invalidate_folio,
+};
diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c
new file mode 100644
index 0000000000000..1477dbd3af824
--- /dev/null
+++ b/fs/ntfs/attrib.c
@@ -0,0 +1,5424 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS attribute operations.
+ *
+ * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2000-2010 Anton Altaparmakov
+ * Copyright (c) 2002-2005 Richard Russon
+ * Copyright (c) 2002-2008 Szabolcs Szakacsits
+ * Copyright (c) 2004-2007 Yura Pakhuchiy
+ * Copyright (c) 2007-2021 Jean-Pierre Andre
+ * Copyright (c) 2010 Erik Larsson
+ */
+
+#include <linux/writeback.h>
+#include <linux/iomap.h>
+
+#include "attrib.h"
+#include "attrlist.h"
+#include "lcnalloc.h"
+#include "debug.h"
+#include "mft.h"
+#include "ntfs.h"
+#include "iomap.h"
+
+__le16 AT_UNNAMED[] = { cpu_to_le16('\0') };
+
+/*
+ * ntfs_map_runlist_nolock - map (a part of) a runlist of an ntfs inode
+ * @ni:		ntfs inode for which to map (part of) a runlist
+ * @vcn:	map runlist part containing this vcn
+ * @ctx:	active attribute search context if present or NULL if not
+ *
+ * Map the part of a runlist containing the @vcn of the ntfs inode @ni.
+ *
+ * If @ctx is specified, it is an active search context of @ni and its base mft
+ * record.  This is needed when ntfs_map_runlist_nolock() encounters unmapped
+ * runlist fragments and allows their mapping.  If you do not have the mft
+ * record mapped, you can specify @ctx as NULL and ntfs_map_runlist_nolock()
+ * will perform the necessary mapping and unmapping.
+ *
+ * Note, ntfs_map_runlist_nolock() saves the state of @ctx on entry and
+ * restores it before returning.  Thus, @ctx will be left pointing to the same
+ * attribute on return as on entry.  However, the actual pointers in @ctx may
+ * point to different memory locations on return, so you must remember to reset
+ * any cached pointers from the @ctx, i.e. after the call to
+ * ntfs_map_runlist_nolock(), you will probably want to do:
+ *	m = ctx->mrec;
+ *	a = ctx->attr;
+ * Assuming you cache ctx->attr in a variable @a of type struct attr_record *
+ * and that you cache ctx->mrec in a variable @m of type struct mft_record *.
+ *
+ * Return 0 on success and -errno on error.  There is one special error code
+ * which is not an error as such.  This is -ENOENT.  It means that @vcn is out
+ * of bounds of the runlist.
+ *
+ * Note the runlist can be NULL after this function returns if @vcn is zero and
+ * the attribute has zero allocated size, i.e. there simply is no runlist.
+ *
+ * WARNING: If @ctx is supplied, regardless of whether success or failure is
+ *	    returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx
+ *	    is no longer valid, i.e. you need to either call
+ *	    ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it.
+ *	    In that case PTR_ERR(@ctx->mrec) will give you the error code for
+ *	    why the mapping of the old inode failed.
+ *
+ * Locking: - The runlist described by @ni must be locked for writing on entry
+ *	      and is locked on return.  Note the runlist will be modified.
+ *	    - If @ctx is NULL, the base mft record of @ni must not be mapped on
+ *	      entry and it will be left unmapped on return.
+ *	    - If @ctx is not NULL, the base mft record must be mapped on entry
+ *	      and it will be left mapped on return.
+ */
+int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn, struct ntfs_attr_search_ctx *ctx)
+{
+	s64 end_vcn;
+	unsigned long flags;
+	struct ntfs_inode *base_ni;
+	struct mft_record *m;
+	struct attr_record *a;
+	struct runlist_element *rl;
+	struct folio *put_this_folio = NULL;
+	int err = 0;
+	bool ctx_is_temporary = false, ctx_needs_reset;
+	struct ntfs_attr_search_ctx old_ctx = { NULL, };
+	size_t new_rl_count;
+
+	ntfs_debug("Mapping runlist part containing vcn 0x%llx.",
+			(unsigned long long)vcn);
+	if (!NInoAttr(ni))
+		base_ni = ni;
+	else
+		base_ni = ni->ext.base_ntfs_ino;
+	if (!ctx) {
+		ctx_is_temporary = ctx_needs_reset = true;
+		m = map_mft_record(base_ni);
+		if (IS_ERR(m))
+			return PTR_ERR(m);
+		ctx = ntfs_attr_get_search_ctx(base_ni, m);
+		if (unlikely(!ctx)) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+	} else {
+		s64 allocated_size_vcn;
+
+		WARN_ON(IS_ERR(ctx->mrec));
+		a = ctx->attr;
+		if (!a->non_resident) {
+			err = -EIO;
+			goto err_out;
+		}
+		end_vcn = le64_to_cpu(a->data.non_resident.highest_vcn);
+		read_lock_irqsave(&ni->size_lock, flags);
+		allocated_size_vcn =
+			ntfs_bytes_to_cluster(ni->vol, ni->allocated_size);
+		read_unlock_irqrestore(&ni->size_lock, flags);
+		if (!a->data.non_resident.lowest_vcn && end_vcn <= 0)
+			end_vcn = allocated_size_vcn - 1;
+		/*
+		 * If we already have the attribute extent containing @vcn in
+		 * @ctx, no need to look it up again.  We slightly cheat in
+		 * that if vcn exceeds the allocated size, we will refuse to
+		 * map the runlist below, so there is definitely no need to get
+		 * the right attribute extent.
+		 */
+		if (vcn >= allocated_size_vcn || (a->type == ni->type &&
+				a->name_length == ni->name_len &&
+				!memcmp((u8 *)a + le16_to_cpu(a->name_offset),
+				ni->name, ni->name_len) &&
+				le64_to_cpu(a->data.non_resident.lowest_vcn)
+				<= vcn && end_vcn >= vcn))
+			ctx_needs_reset = false;
+		else {
+			/* Save the old search context. */
+			old_ctx = *ctx;
+			/*
+			 * If the currently mapped (extent) inode is not the
+			 * base inode we will unmap it when we reinitialize the
+			 * search context which means we need to get a
+			 * reference to the page containing the mapped mft
+			 * record so we do not accidentally drop changes to the
+			 * mft record when it has not been marked dirty yet.
+			 */
+			if (old_ctx.base_ntfs_ino && old_ctx.ntfs_ino !=
+					old_ctx.base_ntfs_ino) {
+				put_this_folio = old_ctx.ntfs_ino->folio;
+				folio_get(put_this_folio);
+			}
+			/*
+			 * Reinitialize the search context so we can lookup the
+			 * needed attribute extent.
+			 */
+			ntfs_attr_reinit_search_ctx(ctx);
+			ctx_needs_reset = true;
+		}
+	}
+	if (ctx_needs_reset) {
+		err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+				CASE_SENSITIVE, vcn, NULL, 0, ctx);
+		if (unlikely(err)) {
+			if (err == -ENOENT)
+				err = -EIO;
+			goto err_out;
+		}
+		WARN_ON(!ctx->attr->non_resident);
+	}
+	a = ctx->attr;
+	/*
+	 * Only decompress the mapping pairs if @vcn is inside it.  Otherwise
+	 * we get into problems when we try to map an out of bounds vcn because
+	 * we then try to map the already mapped runlist fragment and
+	 * ntfs_mapping_pairs_decompress() fails.
+	 */
+	end_vcn = le64_to_cpu(a->data.non_resident.highest_vcn) + 1;
+	if (unlikely(vcn && vcn >= end_vcn)) {
+		err = -ENOENT;
+		goto err_out;
+	}
+	rl = ntfs_mapping_pairs_decompress(ni->vol, a, &ni->runlist, &new_rl_count);
+	if (IS_ERR(rl))
+		err = PTR_ERR(rl);
+	else {
+		ni->runlist.rl = rl;
+		ni->runlist.count = new_rl_count;
+	}
+err_out:
+	if (ctx_is_temporary) {
+		if (likely(ctx))
+			ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(base_ni);
+	} else if (ctx_needs_reset) {
+		/*
+		 * If there is no attribute list, restoring the search context
+		 * is accomplished simply by copying the saved context back over
+		 * the caller supplied context.  If there is an attribute list,
+		 * things are more complicated as we need to deal with mapping
+		 * of mft records and resulting potential changes in pointers.
+		 */
+		if (NInoAttrList(base_ni)) {
+			/*
+			 * If the currently mapped (extent) inode is not the
+			 * one we had before, we need to unmap it and map the
+			 * old one.
+			 */
+			if (ctx->ntfs_ino != old_ctx.ntfs_ino) {
+				/*
+				 * If the currently mapped inode is not the
+				 * base inode, unmap it.
+				 */
+				if (ctx->base_ntfs_ino && ctx->ntfs_ino !=
+						ctx->base_ntfs_ino) {
+					unmap_extent_mft_record(ctx->ntfs_ino);
+					ctx->mrec = ctx->base_mrec;
+					WARN_ON(!ctx->mrec);
+				}
+				/*
+				 * If the old mapped inode is not the base
+				 * inode, map it.
+				 */
+				if (old_ctx.base_ntfs_ino &&
+				    old_ctx.ntfs_ino !=	old_ctx.base_ntfs_ino) {
+retry_map:
+					ctx->mrec = map_mft_record(old_ctx.ntfs_ino);
+					/*
+					 * Something bad has happened.  If out
+					 * of memory retry till it succeeds.
+					 * Any other errors are fatal and we
+					 * return the error code in ctx->mrec.
+					 * Let the caller deal with it...  We
+					 * just need to fudge things so the
+					 * caller can reinit and/or put the
+					 * search context safely.
+					 */
+					if (IS_ERR(ctx->mrec)) {
+						if (PTR_ERR(ctx->mrec) == -ENOMEM) {
+							schedule();
+							goto retry_map;
+						} else
+							old_ctx.ntfs_ino =
+								old_ctx.base_ntfs_ino;
+					}
+				}
+			}
+			/* Update the changed pointers in the saved context. */
+			if (ctx->mrec != old_ctx.mrec) {
+				if (!IS_ERR(ctx->mrec))
+					old_ctx.attr = (struct attr_record *)(
+							(u8 *)ctx->mrec +
+							((u8 *)old_ctx.attr -
+							(u8 *)old_ctx.mrec));
+				old_ctx.mrec = ctx->mrec;
+			}
+		}
+		/* Restore the search context to the saved one. */
+		*ctx = old_ctx;
+		/*
+		 * We drop the reference on the page we took earlier.  In the
+		 * case that IS_ERR(ctx->mrec) is true this means we might lose
+		 * some changes to the mft record that had been made between
+		 * the last time it was marked dirty/written out and now.  This
+		 * at this stage is not a problem as the mapping error is fatal
+		 * enough that the mft record cannot be written out anyway and
+		 * the caller is very likely to shutdown the whole inode
+		 * immediately and mark the volume dirty for chkdsk to pick up
+		 * the pieces anyway.
+		 */
+		if (put_this_folio)
+			folio_put(put_this_folio);
+	}
+	return err;
+}
+
+/*
+ * ntfs_map_runlist - map (a part of) a runlist of an ntfs inode
+ * @ni:		ntfs inode for which to map (part of) a runlist
+ * @vcn:	map runlist part containing this vcn
+ *
+ * Map the part of a runlist containing the @vcn of the ntfs inode @ni.
+ *
+ * Return 0 on success and -errno on error.  There is one special error code
+ * which is not an error as such.  This is -ENOENT.  It means that @vcn is out
+ * of bounds of the runlist.
+ *
+ * Locking: - The runlist must be unlocked on entry and is unlocked on return.
+ *	    - This function takes the runlist lock for writing and may modify
+ *	      the runlist.
+ */
+int ntfs_map_runlist(struct ntfs_inode *ni, s64 vcn)
+{
+	int err = 0;
+
+	down_write(&ni->runlist.lock);
+	/* Make sure someone else didn't do the work while we were sleeping. */
+	if (likely(ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn) <=
+			LCN_RL_NOT_MAPPED))
+		err = ntfs_map_runlist_nolock(ni, vcn, NULL);
+	up_write(&ni->runlist.lock);
+	return err;
+}
+
+struct runlist_element *ntfs_attr_vcn_to_rl(struct ntfs_inode *ni, s64 vcn, s64 *lcn)
+{
+	struct runlist_element *rl = ni->runlist.rl;
+	int err;
+	bool is_retry = false;
+
+	if (!rl) {
+		err = ntfs_attr_map_whole_runlist(ni);
+		if (err)
+			return ERR_PTR(-ENOENT);
+		rl = ni->runlist.rl;
+	}
+
+remap_rl:
+	/* Seek to element containing target vcn. */
+	while (rl->length && rl[1].vcn <= vcn)
+		rl++;
+	*lcn = ntfs_rl_vcn_to_lcn(rl, vcn);
+
+	if (*lcn <= LCN_RL_NOT_MAPPED && is_retry == false) {
+		is_retry = true;
+		if (!ntfs_map_runlist_nolock(ni, vcn, NULL)) {
+			rl = ni->runlist.rl;
+			goto remap_rl;
+		}
+	}
+
+	return rl;
+}
+
+/*
+ * ntfs_attr_vcn_to_lcn_nolock - convert a vcn into a lcn given an ntfs inode
+ * @ni:			ntfs inode of the attribute whose runlist to search
+ * @vcn:		vcn to convert
+ * @write_locked:	true if the runlist is locked for writing
+ *
+ * Find the virtual cluster number @vcn in the runlist of the ntfs attribute
+ * described by the ntfs inode @ni and return the corresponding logical cluster
+ * number (lcn).
+ *
+ * If the @vcn is not mapped yet, the attempt is made to map the attribute
+ * extent containing the @vcn and the vcn to lcn conversion is retried.
+ *
+ * If @write_locked is true the caller has locked the runlist for writing and
+ * if false for reading.
+ *
+ * Since lcns must be >= 0, we use negative return codes with special meaning:
+ *
+ * Return code	Meaning / Description
+ * ==========================================
+ *  LCN_HOLE	Hole / not allocated on disk.
+ *  LCN_ENOENT	There is no such vcn in the runlist, i.e. @vcn is out of bounds.
+ *  LCN_ENOMEM	Not enough memory to map runlist.
+ *  LCN_EIO	Critical error (runlist/file is corrupt, i/o error, etc).
+ *
+ * Locking: - The runlist must be locked on entry and is left locked on return.
+ *	    - If @write_locked is 'false', i.e. the runlist is locked for reading,
+ *	      the lock may be dropped inside the function so you cannot rely on
+ *	      the runlist still being the same when this function returns.
+ */
+s64 ntfs_attr_vcn_to_lcn_nolock(struct ntfs_inode *ni, const s64 vcn,
+		const bool write_locked)
+{
+	s64 lcn;
+	unsigned long flags;
+	bool is_retry = false;
+
+	ntfs_debug("Entering for i_ino 0x%llx, vcn 0x%llx, %s_locked.",
+			ni->mft_no, (unsigned long long)vcn,
+			write_locked ? "write" : "read");
+	if (!ni->runlist.rl) {
+		read_lock_irqsave(&ni->size_lock, flags);
+		if (!ni->allocated_size) {
+			read_unlock_irqrestore(&ni->size_lock, flags);
+			return LCN_ENOENT;
+		}
+		read_unlock_irqrestore(&ni->size_lock, flags);
+	}
+retry_remap:
+	/* Convert vcn to lcn.  If that fails map the runlist and retry once. */
+	lcn = ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn);
+	if (likely(lcn >= LCN_HOLE)) {
+		ntfs_debug("Done, lcn 0x%llx.", (long long)lcn);
+		return lcn;
+	}
+	if (lcn != LCN_RL_NOT_MAPPED) {
+		if (lcn != LCN_ENOENT)
+			lcn = LCN_EIO;
+	} else if (!is_retry) {
+		int err;
+
+		if (!write_locked) {
+			up_read(&ni->runlist.lock);
+			down_write(&ni->runlist.lock);
+			if (unlikely(ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn) !=
+					LCN_RL_NOT_MAPPED)) {
+				up_write(&ni->runlist.lock);
+				down_read(&ni->runlist.lock);
+				goto retry_remap;
+			}
+		}
+		err = ntfs_map_runlist_nolock(ni, vcn, NULL);
+		if (!write_locked) {
+			up_write(&ni->runlist.lock);
+			down_read(&ni->runlist.lock);
+		}
+		if (likely(!err)) {
+			is_retry = true;
+			goto retry_remap;
+		}
+		if (err == -ENOENT)
+			lcn = LCN_ENOENT;
+		else if (err == -ENOMEM)
+			lcn = LCN_ENOMEM;
+		else
+			lcn = LCN_EIO;
+	}
+	if (lcn != LCN_ENOENT)
+		ntfs_error(ni->vol->sb, "Failed with error code %lli.",
+				(long long)lcn);
+	return lcn;
+}
+
+struct runlist_element *__ntfs_attr_find_vcn_nolock(struct runlist *runlist, const s64 vcn)
+{
+	size_t lower_idx, upper_idx, idx;
+	struct runlist_element *run;
+	int rh = runlist->rl_hint;
+
+	if (runlist->count <= 1)
+		return ERR_PTR(-ENOENT);
+
+	if (runlist->count - 1 > rh && runlist->rl[rh].vcn <= vcn) {
+		if (vcn < runlist->rl[rh].vcn + runlist->rl[rh].length)
+			return &runlist->rl[rh];
+		if (runlist->count - 2 == rh)
+			return ERR_PTR(-ENOENT);
+
+		lower_idx = rh + 1;
+	} else {
+		run = &runlist->rl[0];
+		if (vcn < run->vcn)
+			return ERR_PTR(-ENOENT);
+		else if (vcn < run->vcn + run->length) {
+			runlist->rl_hint = 0;
+			return run;
+		}
+
+		lower_idx = 1;
+	}
+
+	run = &runlist->rl[runlist->count - 2];
+	if (vcn >= run->vcn && vcn < run->vcn + run->length) {
+		runlist->rl_hint = runlist->count - 2;
+		return run;
+	}
+	if (vcn >= run->vcn + run->length)
+		return ERR_PTR(-ENOENT);
+
+	upper_idx = runlist->count - 2;
+
+	while (lower_idx <= upper_idx) {
+		idx = (lower_idx + upper_idx) >> 1;
+		run = &runlist->rl[idx];
+
+		if (vcn < run->vcn)
+			upper_idx = idx - 1;
+		else if (vcn >= run->vcn + run->length)
+			lower_idx = idx + 1;
+		else {
+			runlist->rl_hint = idx;
+			return run;
+		}
+	}
+
+	return ERR_PTR(-ENOENT);
+}
+
+/*
+ * ntfs_attr_find_vcn_nolock - find a vcn in the runlist of an ntfs inode
+ * @ni:		ntfs inode describing the runlist to search
+ * @vcn:	vcn to find
+ * @ctx:	active attribute search context if present or NULL if not
+ *
+ * Find the virtual cluster number @vcn in the runlist described by the ntfs
+ * inode @ni and return the address of the runlist element containing the @vcn.
+ *
+ * If the @vcn is not mapped yet, the attempt is made to map the attribute
+ * extent containing the @vcn and the vcn to lcn conversion is retried.
+ *
+ * If @ctx is specified, it is an active search context of @ni and its base mft
+ * record.  This is needed when ntfs_attr_find_vcn_nolock() encounters unmapped
+ * runlist fragments and allows their mapping.  If you do not have the mft
+ * record mapped, you can specify @ctx as NULL and ntfs_attr_find_vcn_nolock()
+ * will perform the necessary mapping and unmapping.
+ *
+ * Note, ntfs_attr_find_vcn_nolock() saves the state of @ctx on entry and
+ * restores it before returning.  Thus, @ctx will be left pointing to the same
+ * attribute on return as on entry.  However, the actual pointers in @ctx may
+ * point to different memory locations on return, so you must remember to reset
+ * any cached pointers from the @ctx, i.e. after the call to
+ * ntfs_attr_find_vcn_nolock(), you will probably want to do:
+ *	m = ctx->mrec;
+ *	a = ctx->attr;
+ * Assuming you cache ctx->attr in a variable @a of type attr_record * and that
+ * you cache ctx->mrec in a variable @m of type struct mft_record *.
+ * Note you need to distinguish between the lcn of the returned runlist element
+ * being >= 0 and LCN_HOLE.  In the later case you have to return zeroes on
+ * read and allocate clusters on write.
+ */
+struct runlist_element *ntfs_attr_find_vcn_nolock(struct ntfs_inode *ni, const s64 vcn,
+		struct ntfs_attr_search_ctx *ctx)
+{
+	unsigned long flags;
+	struct runlist_element *rl;
+	int err = 0;
+	bool is_retry = false;
+
+	ntfs_debug("Entering for i_ino 0x%llx, vcn 0x%llx, with%s ctx.",
+			ni->mft_no, (unsigned long long)vcn, ctx ? "" : "out");
+	if (!ni->runlist.rl) {
+		read_lock_irqsave(&ni->size_lock, flags);
+		if (!ni->allocated_size) {
+			read_unlock_irqrestore(&ni->size_lock, flags);
+			return ERR_PTR(-ENOENT);
+		}
+		read_unlock_irqrestore(&ni->size_lock, flags);
+	}
+
+retry_remap:
+	rl = ni->runlist.rl;
+	if (likely(rl && vcn >= rl[0].vcn)) {
+		rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn);
+		if (IS_ERR(rl))
+			err = PTR_ERR(rl);
+		else if (rl->lcn >= LCN_HOLE)
+			return rl;
+		else if (rl->lcn <= LCN_ENOENT)
+			err = -EIO;
+	}
+	if (!err && !is_retry) {
+		/*
+		 * If the search context is invalid we cannot map the unmapped
+		 * region.
+		 */
+		if (ctx && IS_ERR(ctx->mrec))
+			err = PTR_ERR(ctx->mrec);
+		else {
+			/*
+			 * The @vcn is in an unmapped region, map the runlist
+			 * and retry.
+			 */
+			err = ntfs_map_runlist_nolock(ni, vcn, ctx);
+			if (likely(!err)) {
+				is_retry = true;
+				goto retry_remap;
+			}
+		}
+		if (err == -EINVAL)
+			err = -EIO;
+	} else if (!err)
+		err = -EIO;
+	if (err != -ENOENT)
+		ntfs_error(ni->vol->sb, "Failed with error code %i.", err);
+	return ERR_PTR(err);
+}
+
+/*
+ * ntfs_attr_find - find (next) attribute in mft record
+ * @type:	attribute type to find
+ * @name:	attribute name to find (optional, i.e. NULL means don't care)
+ * @name_len:	attribute name length (only needed if @name present)
+ * @ic:		IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present)
+ * @val:	attribute value to find (optional, resident attributes only)
+ * @val_len:	attribute value length
+ * @ctx:	search context with mft record and attribute to search from
+ *
+ * You should not need to call this function directly.  Use ntfs_attr_lookup()
+ * instead.
+ *
+ * ntfs_attr_find() takes a search context @ctx as parameter and searches the
+ * mft record specified by @ctx->mrec, beginning at @ctx->attr, for an
+ * attribute of @type, optionally @name and @val.
+ *
+ * If the attribute is found, ntfs_attr_find() returns 0 and @ctx->attr will
+ * point to the found attribute.
+ *
+ * If the attribute is not found, ntfs_attr_find() returns -ENOENT and
+ * @ctx->attr will point to the attribute before which the attribute being
+ * searched for would need to be inserted if such an action were to be desired.
+ *
+ * On actual error, ntfs_attr_find() returns -EIO.  In this case @ctx->attr is
+ * undefined and in particular do not rely on it not changing.
+ *
+ * If @ctx->is_first is 'true', the search begins with @ctx->attr itself.  If it
+ * is 'false', the search begins after @ctx->attr.
+ *
+ * If @ic is IGNORE_CASE, the @name comparisson is not case sensitive and
+ * @ctx->ntfs_ino must be set to the ntfs inode to which the mft record
+ * @ctx->mrec belongs.  This is so we can get at the ntfs volume and hence at
+ * the upcase table.  If @ic is CASE_SENSITIVE, the comparison is case
+ * sensitive.  When @name is present, @name_len is the @name length in Unicode
+ * characters.
+ *
+ * If @name is not present (NULL), we assume that the unnamed attribute is
+ * being searched for.
+ *
+ * Finally, the resident attribute value @val is looked for, if present.  If
+ * @val is not present (NULL), @val_len is ignored.
+ *
+ * ntfs_attr_find() only searches the specified mft record and it ignores the
+ * presence of an attribute list attribute (unless it is the one being searched
+ * for, obviously).  If you need to take attribute lists into consideration,
+ * use ntfs_attr_lookup() instead (see below).  This also means that you cannot
+ * use ntfs_attr_find() to search for extent records of non-resident
+ * attributes, as extents with lowest_vcn != 0 are usually described by the
+ * attribute list attribute only. - Note that it is possible that the first
+ * extent is only in the attribute list while the last extent is in the base
+ * mft record, so do not rely on being able to find the first extent in the
+ * base mft record.
+ *
+ * Warning: Never use @val when looking for attribute types which can be
+ *	    non-resident as this most likely will result in a crash!
+ */
+static int ntfs_attr_find(const __le32 type, const __le16 *name,
+		const u32 name_len, const u32 ic,
+		const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx)
+{
+	struct attr_record *a;
+	struct ntfs_volume *vol = ctx->ntfs_ino->vol;
+	__le16 *upcase = vol->upcase;
+	u32 upcase_len = vol->upcase_len;
+	unsigned int space;
+
+	/*
+	 * Iterate over attributes in mft record starting at @ctx->attr, or the
+	 * attribute following that, if @ctx->is_first is 'true'.
+	 */
+	if (ctx->is_first) {
+		a = ctx->attr;
+		ctx->is_first = false;
+	} else
+		a = (struct attr_record *)((u8 *)ctx->attr +
+				le32_to_cpu(ctx->attr->length));
+	for (;;	a = (struct attr_record *)((u8 *)a + le32_to_cpu(a->length))) {
+		if ((u8 *)a < (u8 *)ctx->mrec || (u8 *)a > (u8 *)ctx->mrec +
+				le32_to_cpu(ctx->mrec->bytes_allocated))
+			break;
+
+		space = le32_to_cpu(ctx->mrec->bytes_in_use) - ((u8 *)a - (u8 *)ctx->mrec);
+		if ((space < offsetof(struct attr_record, data.resident.reserved) + 1 ||
+		      space < le32_to_cpu(a->length)) && (space < 4 || a->type != AT_END))
+			break;
+
+		ctx->attr = a;
+		if (((type != AT_UNUSED) && (le32_to_cpu(a->type) > le32_to_cpu(type))) ||
+				a->type == AT_END)
+			return -ENOENT;
+		if (unlikely(!a->length))
+			break;
+		if (type == AT_UNUSED)
+			return 0;
+		if (a->type != type)
+			continue;
+		/*
+		 * If @name is present, compare the two names.  If @name is
+		 * missing, assume we want an unnamed attribute.
+		 */
+		if (!name || name == AT_UNNAMED) {
+			/* The search failed if the found attribute is named. */
+			if (a->name_length)
+				return -ENOENT;
+		} else {
+			if (a->name_length && ((le16_to_cpu(a->name_offset) +
+					       a->name_length * sizeof(__le16)) >
+						le32_to_cpu(a->length))) {
+				ntfs_error(vol->sb, "Corrupt attribute name in MFT record %llu\n",
+					   ctx->ntfs_ino->mft_no);
+				break;
+			}
+
+			if (!ntfs_are_names_equal(name, name_len,
+					(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+					a->name_length, ic, upcase, upcase_len)) {
+				register int rc;
+
+				rc = ntfs_collate_names(name, name_len,
+						(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+						a->name_length, 1, IGNORE_CASE,
+						upcase, upcase_len);
+				/*
+				 * If @name collates before a->name, there is no
+				 * matching attribute.
+				 */
+				if (rc == -1)
+					return -ENOENT;
+				/* If the strings are not equal, continue search. */
+				if (rc)
+					continue;
+				rc = ntfs_collate_names(name, name_len,
+						(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+						a->name_length, 1, CASE_SENSITIVE,
+						upcase, upcase_len);
+				if (rc == -1)
+					return -ENOENT;
+				if (rc)
+					continue;
+			}
+		}
+		/*
+		 * The names match or @name not present and attribute is
+		 * unnamed.  If no @val specified, we have found the attribute
+		 * and are done.
+		 */
+		if (!val)
+			return 0;
+		/* @val is present; compare values. */
+		else {
+			register int rc;
+
+			rc = memcmp(val, (u8 *)a + le16_to_cpu(
+					a->data.resident.value_offset),
+					min_t(u32, val_len, le32_to_cpu(
+					a->data.resident.value_length)));
+			/*
+			 * If @val collates before the current attribute's
+			 * value, there is no matching attribute.
+			 */
+			if (!rc) {
+				register u32 avl;
+
+				avl = le32_to_cpu(a->data.resident.value_length);
+				if (val_len == avl)
+					return 0;
+				if (val_len < avl)
+					return -ENOENT;
+			} else if (rc < 0)
+				return -ENOENT;
+		}
+	}
+	ntfs_error(vol->sb, "Inode is corrupt.  Run chkdsk.");
+	NVolSetErrors(vol);
+	return -EIO;
+}
+
+void ntfs_attr_name_free(unsigned char **name)
+{
+	if (*name) {
+		kfree(*name);
+		*name = NULL;
+	}
+}
+
+char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname,
+		const int uname_len)
+{
+	unsigned char *name = NULL;
+	int name_len;
+
+	name_len = ntfs_ucstonls(vol, uname, uname_len, &name, 0);
+	if (name_len < 0) {
+		ntfs_error(vol->sb, "ntfs_ucstonls error");
+		/* This function when returns -1, memory for name might
+		 * be allocated. So lets free this memory.
+		 */
+		ntfs_attr_name_free(&name);
+		return NULL;
+
+	} else if (name_len > 0)
+		return name;
+
+	ntfs_attr_name_free(&name);
+	return NULL;
+}
+
+int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size)
+{
+	struct inode *attr_vi = NULL;
+	u8 *al;
+	struct attr_list_entry *ale;
+
+	if (!al_start || size <= 0)
+		return -EINVAL;
+
+	attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0);
+	if (IS_ERR(attr_vi)) {
+		ntfs_error(base_ni->vol->sb,
+			   "Failed to open an inode for Attribute list, mft = %llu",
+			   base_ni->mft_no);
+		return PTR_ERR(attr_vi);
+	}
+
+	if (ntfs_inode_attr_pread(attr_vi, 0, size, al_start) != size) {
+		iput(attr_vi);
+		ntfs_error(base_ni->vol->sb,
+			   "Failed to read attribute list, mft = %llu",
+			   base_ni->mft_no);
+		return -EIO;
+	}
+	iput(attr_vi);
+
+	for (al = al_start; al < al_start + size; al += le16_to_cpu(ale->length)) {
+		ale = (struct attr_list_entry *)al;
+		if (ale->name_offset != sizeof(struct attr_list_entry))
+			break;
+		if (le16_to_cpu(ale->length) <= ale->name_offset + ale->name_length ||
+		    al + le16_to_cpu(ale->length) > al_start + size)
+			break;
+		if (ale->type == AT_UNUSED)
+			break;
+		if (MSEQNO_LE(ale->mft_reference) == 0)
+			break;
+	}
+	if (al != al_start + size) {
+		ntfs_error(base_ni->vol->sb, "Corrupt attribute list, mft = %llu",
+			   base_ni->mft_no);
+		return -EIO;
+	}
+	return 0;
+}
+
+/*
+ * ntfs_external_attr_find - find an attribute in the attribute list of an inode
+ * @type:	attribute type to find
+ * @name:	attribute name to find (optional, i.e. NULL means don't care)
+ * @name_len:	attribute name length (only needed if @name present)
+ * @ic:		IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present)
+ * @lowest_vcn:	lowest vcn to find (optional, non-resident attributes only)
+ * @val:	attribute value to find (optional, resident attributes only)
+ * @val_len:	attribute value length
+ * @ctx:	search context with mft record and attribute to search from
+ *
+ * You should not need to call this function directly.  Use ntfs_attr_lookup()
+ * instead.
+ *
+ * Find an attribute by searching the attribute list for the corresponding
+ * attribute list entry.  Having found the entry, map the mft record if the
+ * attribute is in a different mft record/inode, ntfs_attr_find() the attribute
+ * in there and return it.
+ *
+ * On first search @ctx->ntfs_ino must be the base mft record and @ctx must
+ * have been obtained from a call to ntfs_attr_get_search_ctx().  On subsequent
+ * calls @ctx->ntfs_ino can be any extent inode, too (@ctx->base_ntfs_ino is
+ * then the base inode).
+ *
+ * After finishing with the attribute/mft record you need to call
+ * ntfs_attr_put_search_ctx() to cleanup the search context (unmapping any
+ * mapped inodes, etc).
+ *
+ * If the attribute is found, ntfs_external_attr_find() returns 0 and
+ * @ctx->attr will point to the found attribute.  @ctx->mrec will point to the
+ * mft record in which @ctx->attr is located and @ctx->al_entry will point to
+ * the attribute list entry for the attribute.
+ *
+ * If the attribute is not found, ntfs_external_attr_find() returns -ENOENT and
+ * @ctx->attr will point to the attribute in the base mft record before which
+ * the attribute being searched for would need to be inserted if such an action
+ * were to be desired.  @ctx->mrec will point to the mft record in which
+ * @ctx->attr is located and @ctx->al_entry will point to the attribute list
+ * entry of the attribute before which the attribute being searched for would
+ * need to be inserted if such an action were to be desired.
+ *
+ * Thus to insert the not found attribute, one wants to add the attribute to
+ * @ctx->mrec (the base mft record) and if there is not enough space, the
+ * attribute should be placed in a newly allocated extent mft record.  The
+ * attribute list entry for the inserted attribute should be inserted in the
+ * attribute list attribute at @ctx->al_entry.
+ *
+ * On actual error, ntfs_external_attr_find() returns -EIO.  In this case
+ * @ctx->attr is undefined and in particular do not rely on it not changing.
+ */
+static int ntfs_external_attr_find(const __le32 type,
+		const __le16 *name, const u32 name_len,
+		const u32 ic, const s64 lowest_vcn,
+		const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_inode *base_ni = ctx->base_ntfs_ino, *ni = ctx->ntfs_ino;
+	struct ntfs_volume *vol;
+	struct attr_list_entry *al_entry, *next_al_entry;
+	u8 *al_start, *al_end;
+	struct attr_record *a;
+	__le16 *al_name;
+	u32 al_name_len;
+	bool is_first_search = false;
+	int err = 0;
+	static const char *es = " Unmount and run chkdsk.";
+
+	ntfs_debug("Entering for inode 0x%llx, type 0x%x.", ni->mft_no, type);
+	if (!base_ni) {
+		/* First call happens with the base mft record. */
+		base_ni = ctx->base_ntfs_ino = ctx->ntfs_ino;
+		ctx->base_mrec = ctx->mrec;
+		ctx->mapped_base_mrec = ctx->mapped_mrec;
+	}
+	if (ni == base_ni)
+		ctx->base_attr = ctx->attr;
+	if (type == AT_END)
+		goto not_found;
+	vol = base_ni->vol;
+	al_start = base_ni->attr_list;
+	al_end = al_start + base_ni->attr_list_size;
+	if (!ctx->al_entry) {
+		ctx->al_entry = (struct attr_list_entry *)al_start;
+		is_first_search = true;
+	}
+	/*
+	 * Iterate over entries in attribute list starting at @ctx->al_entry,
+	 * or the entry following that, if @ctx->is_first is 'true'.
+	 */
+	if (ctx->is_first) {
+		al_entry = ctx->al_entry;
+		ctx->is_first = false;
+		/*
+		 * If an enumeration and the first attribute is higher than
+		 * the attribute list itself, need to return the attribute list
+		 * attribute.
+		 */
+		if ((type == AT_UNUSED) && is_first_search &&
+				le32_to_cpu(al_entry->type) >
+				le32_to_cpu(AT_ATTRIBUTE_LIST))
+			goto find_attr_list_attr;
+	} else {
+		/* Check for small entry */
+		if (((al_end - (u8 *)ctx->al_entry) <
+		      (long)offsetof(struct attr_list_entry, name)) ||
+		    (le16_to_cpu(ctx->al_entry->length) & 7) ||
+		    (le16_to_cpu(ctx->al_entry->length) < offsetof(struct attr_list_entry, name)))
+			goto corrupt;
+
+		al_entry = (struct attr_list_entry *)((u8 *)ctx->al_entry +
+				le16_to_cpu(ctx->al_entry->length));
+
+		if ((u8 *)al_entry == al_end)
+			goto not_found;
+
+		/* Preliminary check for small entry */
+		if ((al_end - (u8 *)al_entry) <
+		    (long)offsetof(struct attr_list_entry, name))
+			goto corrupt;
+
+		/*
+		 * If this is an enumeration and the attribute list attribute
+		 * is the next one in the enumeration sequence, just return the
+		 * attribute list attribute from the base mft record as it is
+		 * not listed in the attribute list itself.
+		 */
+		if ((type == AT_UNUSED) && le32_to_cpu(ctx->al_entry->type) <
+				le32_to_cpu(AT_ATTRIBUTE_LIST) &&
+				le32_to_cpu(al_entry->type) >
+				le32_to_cpu(AT_ATTRIBUTE_LIST)) {
+find_attr_list_attr:
+
+			/* Check for bogus calls. */
+			if (name || name_len || val || val_len || lowest_vcn)
+				return -EINVAL;
+
+			/* We want the base record. */
+			if (ctx->ntfs_ino != base_ni)
+				unmap_mft_record(ctx->ntfs_ino);
+			ctx->ntfs_ino = base_ni;
+			ctx->mapped_mrec = ctx->mapped_base_mrec;
+			ctx->mrec = ctx->base_mrec;
+			ctx->is_first = true;
+
+			/* Sanity checks are performed elsewhere. */
+			ctx->attr = (struct attr_record *)((u8 *)ctx->mrec +
+					le16_to_cpu(ctx->mrec->attrs_offset));
+
+			/* Find the attribute list attribute. */
+			err = ntfs_attr_find(AT_ATTRIBUTE_LIST, NULL, 0,
+					IGNORE_CASE, NULL, 0, ctx);
+
+			/*
+			 * Setup the search context so the correct
+			 * attribute is returned next time round.
+			 */
+			ctx->al_entry = al_entry;
+			ctx->is_first = true;
+
+			/* Got it. Done. */
+			if (!err)
+				return 0;
+
+			/* Error! If other than not found return it. */
+			if (err != -ENOENT)
+				return err;
+
+			/* Not found?!? Absurd! */
+			ntfs_error(ctx->ntfs_ino->vol->sb, "Attribute list wasn't found");
+			return -EIO;
+		}
+	}
+	for (;; al_entry = next_al_entry) {
+		/* Out of bounds check. */
+		if ((u8 *)al_entry < base_ni->attr_list ||
+				(u8 *)al_entry > al_end)
+			break;	/* Inode is corrupt. */
+		ctx->al_entry = al_entry;
+		/* Catch the end of the attribute list. */
+		if ((u8 *)al_entry == al_end)
+			goto not_found;
+
+		if ((((u8 *)al_entry + offsetof(struct attr_list_entry, name)) > al_end) ||
+		    ((u8 *)al_entry + le16_to_cpu(al_entry->length) > al_end) ||
+		    (le16_to_cpu(al_entry->length) & 7) ||
+		    (le16_to_cpu(al_entry->length) <
+		     offsetof(struct attr_list_entry, name_length)) ||
+		    (al_entry->name_length && ((u8 *)al_entry + al_entry->name_offset +
+					       al_entry->name_length * sizeof(__le16)) > al_end))
+			break; /* corrupt */
+
+		next_al_entry = (struct attr_list_entry *)((u8 *)al_entry +
+				le16_to_cpu(al_entry->length));
+		if (type != AT_UNUSED) {
+			if (le32_to_cpu(al_entry->type) > le32_to_cpu(type))
+				goto not_found;
+			if (type != al_entry->type)
+				continue;
+		}
+		/*
+		 * If @name is present, compare the two names.  If @name is
+		 * missing, assume we want an unnamed attribute.
+		 */
+		al_name_len = al_entry->name_length;
+		al_name = (__le16 *)((u8 *)al_entry + al_entry->name_offset);
+
+		/*
+		 * If !@type we want the attribute represented by this
+		 * attribute list entry.
+		 */
+		if (type == AT_UNUSED)
+			goto is_enumeration;
+
+		if (!name || name == AT_UNNAMED) {
+			if (al_name_len)
+				goto not_found;
+		} else if (!ntfs_are_names_equal(al_name, al_name_len, name,
+				name_len, ic, vol->upcase, vol->upcase_len)) {
+			register int rc;
+
+			rc = ntfs_collate_names(name, name_len, al_name,
+					al_name_len, 1, IGNORE_CASE,
+					vol->upcase, vol->upcase_len);
+			/*
+			 * If @name collates before al_name, there is no
+			 * matching attribute.
+			 */
+			if (rc == -1)
+				goto not_found;
+			/* If the strings are not equal, continue search. */
+			if (rc)
+				continue;
+
+			rc = ntfs_collate_names(name, name_len, al_name,
+					al_name_len, 1, CASE_SENSITIVE,
+					vol->upcase, vol->upcase_len);
+			if (rc == -1)
+				goto not_found;
+			if (rc)
+				continue;
+		}
+		/*
+		 * The names match or @name not present and attribute is
+		 * unnamed.  Now check @lowest_vcn.  Continue search if the
+		 * next attribute list entry still fits @lowest_vcn.  Otherwise
+		 * we have reached the right one or the search has failed.
+		 */
+		if (lowest_vcn && (u8 *)next_al_entry >= al_start &&
+				(u8 *)next_al_entry + 6 < al_end &&
+				(u8 *)next_al_entry + le16_to_cpu(
+					next_al_entry->length) <= al_end &&
+				le64_to_cpu(next_al_entry->lowest_vcn) <=
+					lowest_vcn &&
+				next_al_entry->type == al_entry->type &&
+				next_al_entry->name_length == al_name_len &&
+				ntfs_are_names_equal((__le16 *)((u8 *)
+					next_al_entry +
+					next_al_entry->name_offset),
+					next_al_entry->name_length,
+					al_name, al_name_len, CASE_SENSITIVE,
+					vol->upcase, vol->upcase_len))
+			continue;
+
+is_enumeration:
+		if (MREF_LE(al_entry->mft_reference) == ni->mft_no) {
+			if (MSEQNO_LE(al_entry->mft_reference) != ni->seq_no) {
+				ntfs_error(vol->sb,
+					"Found stale mft reference in attribute list of base inode 0x%llx.%s",
+					base_ni->mft_no, es);
+				err = -EIO;
+				break;
+			}
+		} else { /* Mft references do not match. */
+			/* If there is a mapped record unmap it first. */
+			if (ni != base_ni)
+				unmap_extent_mft_record(ni);
+			/* Do we want the base record back? */
+			if (MREF_LE(al_entry->mft_reference) ==
+					base_ni->mft_no) {
+				ni = ctx->ntfs_ino = base_ni;
+				ctx->mrec = ctx->base_mrec;
+				ctx->mapped_mrec = ctx->mapped_base_mrec;
+			} else {
+				/* We want an extent record. */
+				ctx->mrec = map_extent_mft_record(base_ni,
+						le64_to_cpu(
+						al_entry->mft_reference), &ni);
+				if (IS_ERR(ctx->mrec)) {
+					ntfs_error(vol->sb,
+							"Failed to map extent mft record 0x%lx of base inode 0x%llx.%s",
+							MREF_LE(al_entry->mft_reference),
+							base_ni->mft_no, es);
+					err = PTR_ERR(ctx->mrec);
+					if (err == -ENOENT)
+						err = -EIO;
+					/* Cause @ctx to be sanitized below. */
+					ni = NULL;
+					break;
+				}
+				ctx->ntfs_ino = ni;
+				ctx->mapped_mrec = true;
+
+			}
+		}
+		a = ctx->attr = (struct attr_record *)((u8 *)ctx->mrec +
+					le16_to_cpu(ctx->mrec->attrs_offset));
+		/*
+		 * ctx->vfs_ino, ctx->mrec, and ctx->attr now point to the
+		 * mft record containing the attribute represented by the
+		 * current al_entry.
+		 */
+		/*
+		 * We could call into ntfs_attr_find() to find the right
+		 * attribute in this mft record but this would be less
+		 * efficient and not quite accurate as ntfs_attr_find() ignores
+		 * the attribute instance numbers for example which become
+		 * important when one plays with attribute lists.  Also,
+		 * because a proper match has been found in the attribute list
+		 * entry above, the comparison can now be optimized.  So it is
+		 * worth re-implementing a simplified ntfs_attr_find() here.
+		 */
+		/*
+		 * Use a manual loop so we can still use break and continue
+		 * with the same meanings as above.
+		 */
+do_next_attr_loop:
+		if ((u8 *)a < (u8 *)ctx->mrec || (u8 *)a > (u8 *)ctx->mrec +
+				le32_to_cpu(ctx->mrec->bytes_allocated))
+			break;
+		if (a->type == AT_END)
+			continue;
+		if (!a->length)
+			break;
+		if (al_entry->instance != a->instance)
+			goto do_next_attr;
+		/*
+		 * If the type and/or the name are mismatched between the
+		 * attribute list entry and the attribute record, there is
+		 * corruption so we break and return error EIO.
+		 */
+		if (al_entry->type != a->type)
+			break;
+		if (!ntfs_are_names_equal((__le16 *)((u8 *)a +
+				le16_to_cpu(a->name_offset)), a->name_length,
+				al_name, al_name_len, CASE_SENSITIVE,
+				vol->upcase, vol->upcase_len))
+			break;
+		ctx->attr = a;
+		/*
+		 * If no @val specified or @val specified and it matches, we
+		 * have found it!
+		 */
+		if ((type == AT_UNUSED) || !val || (!a->non_resident && le32_to_cpu(
+				a->data.resident.value_length) == val_len &&
+				!memcmp((u8 *)a +
+				le16_to_cpu(a->data.resident.value_offset),
+				val, val_len))) {
+			ntfs_debug("Done, found.");
+			return 0;
+		}
+do_next_attr:
+		/* Proceed to the next attribute in the current mft record. */
+		a = (struct attr_record *)((u8 *)a + le32_to_cpu(a->length));
+		goto do_next_attr_loop;
+	}
+
+corrupt:
+	if (ni != base_ni) {
+		if (ni)
+			unmap_extent_mft_record(ni);
+		ctx->ntfs_ino = base_ni;
+		ctx->mrec = ctx->base_mrec;
+		ctx->attr = ctx->base_attr;
+		ctx->mapped_mrec = ctx->mapped_base_mrec;
+	}
+
+	if (!err) {
+		ntfs_error(vol->sb,
+			"Base inode 0x%llx contains corrupt attribute list attribute.%s",
+			base_ni->mft_no, es);
+		err = -EIO;
+	}
+
+	if (err != -ENOMEM)
+		NVolSetErrors(vol);
+	return err;
+not_found:
+	/*
+	 * If we were looking for AT_END, we reset the search context @ctx and
+	 * use ntfs_attr_find() to seek to the end of the base mft record.
+	 */
+	if (type == AT_UNUSED || type == AT_END) {
+		ntfs_attr_reinit_search_ctx(ctx);
+		return ntfs_attr_find(AT_END, name, name_len, ic, val, val_len,
+				ctx);
+	}
+	/*
+	 * The attribute was not found.  Before we return, we want to ensure
+	 * @ctx->mrec and @ctx->attr indicate the position at which the
+	 * attribute should be inserted in the base mft record.  Since we also
+	 * want to preserve @ctx->al_entry we cannot reinitialize the search
+	 * context using ntfs_attr_reinit_search_ctx() as this would set
+	 * @ctx->al_entry to NULL.  Thus we do the necessary bits manually (see
+	 * ntfs_attr_init_search_ctx() below).  Note, we _only_ preserve
+	 * @ctx->al_entry as the remaining fields (base_*) are identical to
+	 * their non base_ counterparts and we cannot set @ctx->base_attr
+	 * correctly yet as we do not know what @ctx->attr will be set to by
+	 * the call to ntfs_attr_find() below.
+	 */
+	if (ni != base_ni)
+		unmap_extent_mft_record(ni);
+	ctx->mrec = ctx->base_mrec;
+	ctx->attr = (struct attr_record *)((u8 *)ctx->mrec +
+			le16_to_cpu(ctx->mrec->attrs_offset));
+	ctx->is_first = true;
+	ctx->ntfs_ino = base_ni;
+	ctx->base_ntfs_ino = NULL;
+	ctx->base_mrec = NULL;
+	ctx->base_attr = NULL;
+	ctx->mapped_mrec = ctx->mapped_base_mrec;
+	/*
+	 * In case there are multiple matches in the base mft record, need to
+	 * keep enumerating until we get an attribute not found response (or
+	 * another error), otherwise we would keep returning the same attribute
+	 * over and over again and all programs using us for enumeration would
+	 * lock up in a tight loop.
+	 */
+	do {
+		err = ntfs_attr_find(type, name, name_len, ic, val, val_len,
+				ctx);
+	} while (!err);
+	ntfs_debug("Done, not found.");
+	return err;
+}
+
+/*
+ * ntfs_attr_lookup - find an attribute in an ntfs inode
+ * @type:	attribute type to find
+ * @name:	attribute name to find (optional, i.e. NULL means don't care)
+ * @name_len:	attribute name length (only needed if @name present)
+ * @ic:		IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present)
+ * @lowest_vcn:	lowest vcn to find (optional, non-resident attributes only)
+ * @val:	attribute value to find (optional, resident attributes only)
+ * @val_len:	attribute value length
+ * @ctx:	search context with mft record and attribute to search from
+ *
+ * Find an attribute in an ntfs inode.  On first search @ctx->ntfs_ino must
+ * be the base mft record and @ctx must have been obtained from a call to
+ * ntfs_attr_get_search_ctx().
+ *
+ * This function transparently handles attribute lists and @ctx is used to
+ * continue searches where they were left off at.
+ *
+ * After finishing with the attribute/mft record you need to call
+ * ntfs_attr_put_search_ctx() to cleanup the search context (unmapping any
+ * mapped inodes, etc).
+ *
+ * Return 0 if the search was successful and -errno if not.
+ *
+ * When 0, @ctx->attr is the found attribute and it is in mft record
+ * @ctx->mrec.  If an attribute list attribute is present, @ctx->al_entry is
+ * the attribute list entry of the found attribute.
+ *
+ * When -ENOENT, @ctx->attr is the attribute which collates just after the
+ * attribute being searched for, i.e. if one wants to add the attribute to the
+ * mft record this is the correct place to insert it into.  If an attribute
+ * list attribute is present, @ctx->al_entry is the attribute list entry which
+ * collates just after the attribute list entry of the attribute being searched
+ * for, i.e. if one wants to add the attribute to the mft record this is the
+ * correct place to insert its attribute list entry into.
+ */
+int ntfs_attr_lookup(const __le32 type, const __le16 *name,
+		const u32 name_len, const u32 ic,
+		const s64 lowest_vcn, const u8 *val, const u32 val_len,
+		struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_inode *base_ni;
+
+	ntfs_debug("Entering.");
+	if (ctx->base_ntfs_ino)
+		base_ni = ctx->base_ntfs_ino;
+	else
+		base_ni = ctx->ntfs_ino;
+	/* Sanity check, just for debugging really. */
+	if (!base_ni || !NInoAttrList(base_ni) || type == AT_ATTRIBUTE_LIST)
+		return ntfs_attr_find(type, name, name_len, ic, val, val_len,
+				ctx);
+	return ntfs_external_attr_find(type, name, name_len, ic, lowest_vcn,
+			val, val_len, ctx);
+}
+
+/**
+ * ntfs_attr_init_search_ctx - initialize an attribute search context
+ * @ctx:        attribute search context to initialize
+ * @ni:         ntfs inode with which to initialize the search context
+ * @mrec:       mft record with which to initialize the search context
+ *
+ * Initialize the attribute search context @ctx with @ni and @mrec.
+ */
+static bool ntfs_attr_init_search_ctx(struct ntfs_attr_search_ctx *ctx,
+		struct ntfs_inode *ni, struct mft_record *mrec)
+{
+	if (!mrec) {
+		mrec = map_mft_record(ni);
+		if (IS_ERR(mrec))
+			return false;
+		ctx->mapped_mrec = true;
+	} else {
+		ctx->mapped_mrec = false;
+	}
+
+	ctx->mrec = mrec;
+	/* Sanity checks are performed elsewhere. */
+	ctx->attr = (struct attr_record *)((u8 *)mrec + le16_to_cpu(mrec->attrs_offset));
+	ctx->is_first = true;
+	ctx->ntfs_ino = ni;
+	ctx->al_entry = NULL;
+	ctx->base_ntfs_ino = NULL;
+	ctx->base_mrec = NULL;
+	ctx->base_attr = NULL;
+	ctx->mapped_base_mrec = false;
+	return true;
+}
+
+/*
+ * ntfs_attr_reinit_search_ctx - reinitialize an attribute search context
+ * @ctx:	attribute search context to reinitialize
+ *
+ * Reinitialize the attribute search context @ctx, unmapping an associated
+ * extent mft record if present, and initialize the search context again.
+ *
+ * This is used when a search for a new attribute is being started to reset
+ * the search context to the beginning.
+ */
+void ntfs_attr_reinit_search_ctx(struct ntfs_attr_search_ctx *ctx)
+{
+	bool mapped_mrec;
+
+	if (likely(!ctx->base_ntfs_ino)) {
+		/* No attribute list. */
+		ctx->is_first = true;
+		/* Sanity checks are performed elsewhere. */
+		ctx->attr = (struct attr_record *)((u8 *)ctx->mrec +
+				le16_to_cpu(ctx->mrec->attrs_offset));
+		/*
+		 * This needs resetting due to ntfs_external_attr_find() which
+		 * can leave it set despite having zeroed ctx->base_ntfs_ino.
+		 */
+		ctx->al_entry = NULL;
+		return;
+	} /* Attribute list. */
+	if (ctx->ntfs_ino != ctx->base_ntfs_ino && ctx->ntfs_ino)
+		unmap_extent_mft_record(ctx->ntfs_ino);
+
+	mapped_mrec = ctx->mapped_base_mrec;
+	ntfs_attr_init_search_ctx(ctx, ctx->base_ntfs_ino, ctx->base_mrec);
+	ctx->mapped_mrec = mapped_mrec;
+}
+
+/*
+ * ntfs_attr_get_search_ctx - allocate/initialize a new attribute search context
+ * @ni:		ntfs inode with which to initialize the search context
+ * @mrec:	mft record with which to initialize the search context
+ *
+ * Allocate a new attribute search context, initialize it with @ni and @mrec,
+ * and return it. Return NULL if allocation failed.
+ */
+struct ntfs_attr_search_ctx *ntfs_attr_get_search_ctx(struct ntfs_inode *ni,
+		struct mft_record *mrec)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	bool init;
+
+	ctx = kmem_cache_alloc(ntfs_attr_ctx_cache, GFP_NOFS);
+	if (ctx) {
+		init = ntfs_attr_init_search_ctx(ctx, ni, mrec);
+		if (init == false) {
+			kmem_cache_free(ntfs_attr_ctx_cache, ctx);
+			ctx = NULL;
+		}
+	}
+
+	return ctx;
+}
+
+/*
+ * ntfs_attr_put_search_ctx - release an attribute search context
+ * @ctx:	attribute search context to free
+ *
+ * Release the attribute search context @ctx, unmapping an associated extent
+ * mft record if present.
+ */
+void ntfs_attr_put_search_ctx(struct ntfs_attr_search_ctx *ctx)
+{
+	if (ctx->mapped_mrec)
+		unmap_mft_record(ctx->ntfs_ino);
+
+	if (ctx->mapped_base_mrec && ctx->base_ntfs_ino &&
+	    ctx->ntfs_ino != ctx->base_ntfs_ino)
+		unmap_extent_mft_record(ctx->base_ntfs_ino);
+	kmem_cache_free(ntfs_attr_ctx_cache, ctx);
+}
+
+/*
+ * ntfs_attr_find_in_attrdef - find an attribute in the $AttrDef system file
+ * @vol:	ntfs volume to which the attribute belongs
+ * @type:	attribute type which to find
+ *
+ * Search for the attribute definition record corresponding to the attribute
+ * @type in the $AttrDef system file.
+ *
+ * Return the attribute type definition record if found and NULL if not found.
+ */
+static struct attr_def *ntfs_attr_find_in_attrdef(const struct ntfs_volume *vol,
+		const __le32 type)
+{
+	struct attr_def *ad;
+
+	WARN_ON(!type);
+	for (ad = vol->attrdef; (u8 *)ad - (u8 *)vol->attrdef <
+			vol->attrdef_size && ad->type; ++ad) {
+		/* We have not found it yet, carry on searching. */
+		if (likely(le32_to_cpu(ad->type) < le32_to_cpu(type)))
+			continue;
+		/* We found the attribute; return it. */
+		if (likely(ad->type == type))
+			return ad;
+		/* We have gone too far already.  No point in continuing. */
+		break;
+	}
+	/* Attribute not found. */
+	ntfs_debug("Attribute type 0x%x not found in $AttrDef.",
+			le32_to_cpu(type));
+	return NULL;
+}
+
+/*
+ * ntfs_attr_size_bounds_check - check a size of an attribute type for validity
+ * @vol:	ntfs volume to which the attribute belongs
+ * @type:	attribute type which to check
+ * @size:	size which to check
+ *
+ * Check whether the @size in bytes is valid for an attribute of @type on the
+ * ntfs volume @vol.  This information is obtained from $AttrDef system file.
+ */
+int ntfs_attr_size_bounds_check(const struct ntfs_volume *vol, const __le32 type,
+		const s64 size)
+{
+	struct attr_def *ad;
+
+	if (size < 0)
+		return -EINVAL;
+
+	/*
+	 * $ATTRIBUTE_LIST has a maximum size of 256kiB, but this is not
+	 * listed in $AttrDef.
+	 */
+	if (unlikely(type == AT_ATTRIBUTE_LIST && size > 256 * 1024))
+		return -ERANGE;
+	/* Get the $AttrDef entry for the attribute @type. */
+	ad = ntfs_attr_find_in_attrdef(vol, type);
+	if (unlikely(!ad))
+		return -ENOENT;
+	/* Do the bounds check. */
+	if (((le64_to_cpu(ad->min_size) > 0) &&
+			size < le64_to_cpu(ad->min_size)) ||
+			((le64_to_cpu(ad->max_size) > 0) && size >
+			le64_to_cpu(ad->max_size)))
+		return -ERANGE;
+	return 0;
+}
+
+/*
+ * ntfs_attr_can_be_non_resident - check if an attribute can be non-resident
+ * @vol:	ntfs volume to which the attribute belongs
+ * @type:	attribute type which to check
+ *
+ * Check whether the attribute of @type on the ntfs volume @vol is allowed to
+ * be non-resident.  This information is obtained from $AttrDef system file.
+ */
+static int ntfs_attr_can_be_non_resident(const struct ntfs_volume *vol,
+		const __le32 type)
+{
+	struct attr_def *ad;
+
+	/* Find the attribute definition record in $AttrDef. */
+	ad = ntfs_attr_find_in_attrdef(vol, type);
+	if (unlikely(!ad))
+		return -ENOENT;
+	/* Check the flags and return the result. */
+	if (ad->flags & ATTR_DEF_RESIDENT)
+		return -EPERM;
+	return 0;
+}
+
+/*
+ * ntfs_attr_can_be_resident - check if an attribute can be resident
+ * @vol:	ntfs volume to which the attribute belongs
+ * @type:	attribute type which to check
+ *
+ * Check whether the attribute of @type on the ntfs volume @vol is allowed to
+ * be resident.  This information is derived from our ntfs knowledge and may
+ * not be completely accurate, especially when user defined attributes are
+ * present.  Basically we allow everything to be resident except for index
+ * allocation and $EA attributes.
+ *
+ * Return 0 if the attribute is allowed to be non-resident and -EPERM if not.
+ *
+ * Warning: In the system file $MFT the attribute $Bitmap must be non-resident
+ *	    otherwise windows will not boot (blue screen of death)!  We cannot
+ *	    check for this here as we do not know which inode's $Bitmap is
+ *	    being asked about so the caller needs to special case this.
+ */
+int ntfs_attr_can_be_resident(const struct ntfs_volume *vol, const __le32 type)
+{
+	if (type == AT_INDEX_ALLOCATION)
+		return -EPERM;
+	return 0;
+}
+
+/*
+ * ntfs_attr_record_resize - resize an attribute record
+ * @m:		mft record containing attribute record
+ * @a:		attribute record to resize
+ * @new_size:	new size in bytes to which to resize the attribute record @a
+ *
+ * Resize the attribute record @a, i.e. the resident part of the attribute, in
+ * the mft record @m to @new_size bytes.
+ */
+int ntfs_attr_record_resize(struct mft_record *m, struct attr_record *a, u32 new_size)
+{
+	u32 old_size, alloc_size, attr_size;
+
+	old_size   = le32_to_cpu(m->bytes_in_use);
+	alloc_size = le32_to_cpu(m->bytes_allocated);
+	attr_size  = le32_to_cpu(a->length);
+
+	ntfs_debug("Sizes: old=%u alloc=%u attr=%u new=%u\n",
+			(unsigned int)old_size, (unsigned int)alloc_size,
+			(unsigned int)attr_size, (unsigned int)new_size);
+
+	/* Align to 8 bytes if it is not already done. */
+	if (new_size & 7)
+		new_size = (new_size + 7) & ~7;
+	/* If the actual attribute length has changed, move things around. */
+	if (new_size != attr_size) {
+		u32 new_muse = le32_to_cpu(m->bytes_in_use) -
+				attr_size + new_size;
+		/* Not enough space in this mft record. */
+		if (new_muse > le32_to_cpu(m->bytes_allocated))
+			return -ENOSPC;
+
+		if (a->type == AT_INDEX_ROOT && new_size > attr_size &&
+			new_muse + 120 > alloc_size && old_size + 120 <= alloc_size) {
+			ntfs_debug("Too big struct index_root (%u > %u)\n",
+					new_muse, alloc_size);
+			return -ENOSPC;
+		}
+
+		/* Move attributes following @a to their new location. */
+		memmove((u8 *)a + new_size, (u8 *)a + le32_to_cpu(a->length),
+				le32_to_cpu(m->bytes_in_use) - ((u8 *)a -
+				(u8 *)m) - attr_size);
+		/* Adjust @m to reflect the change in used space. */
+		m->bytes_in_use = cpu_to_le32(new_muse);
+		/* Adjust @a to reflect the new size. */
+		if (new_size >= offsetof(struct attr_record, length) + sizeof(a->length))
+			a->length = cpu_to_le32(new_size);
+	}
+	return 0;
+}
+
+/*
+ * ntfs_resident_attr_value_resize - resize the value of a resident attribute
+ * @m:		mft record containing attribute record
+ * @a:		attribute record whose value to resize
+ * @new_size:	new size in bytes to which to resize the attribute value of @a
+ *
+ * Resize the value of the attribute @a in the mft record @m to @new_size bytes.
+ * If the value is made bigger, the newly allocated space is cleared.
+ */
+int ntfs_resident_attr_value_resize(struct mft_record *m, struct attr_record *a,
+		const u32 new_size)
+{
+	u32 old_size;
+
+	/* Resize the resident part of the attribute record. */
+	if (ntfs_attr_record_resize(m, a,
+			le16_to_cpu(a->data.resident.value_offset) + new_size))
+		return -ENOSPC;
+	/*
+	 * The resize succeeded!  If we made the attribute value bigger, clear
+	 * the area between the old size and @new_size.
+	 */
+	old_size = le32_to_cpu(a->data.resident.value_length);
+	if (new_size > old_size)
+		memset((u8 *)a + le16_to_cpu(a->data.resident.value_offset) +
+				old_size, 0, new_size - old_size);
+	/* Finally update the length of the attribute value. */
+	a->data.resident.value_length = cpu_to_le32(new_size);
+	return 0;
+}
+
+/*
+ * ntfs_attr_make_non_resident - convert a resident to a non-resident attribute
+ * @ni:		ntfs inode describing the attribute to convert
+ * @data_size:	size of the resident data to copy to the non-resident attribute
+ *
+ * Convert the resident ntfs attribute described by the ntfs inode @ni to a
+ * non-resident one.
+ *
+ * @data_size must be equal to the attribute value size.  This is needed since
+ * we need to know the size before we can map the mft record and our callers
+ * always know it.  The reason we cannot simply read the size from the vfs
+ * inode i_size is that this is not necessarily uptodate.  This happens when
+ * ntfs_attr_make_non_resident() is called in the ->truncate call path(s).
+ */
+int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size)
+{
+	s64 new_size;
+	struct inode *vi = VFS_I(ni);
+	struct ntfs_volume *vol = ni->vol;
+	struct ntfs_inode *base_ni;
+	struct mft_record *m;
+	struct attr_record *a;
+	struct ntfs_attr_search_ctx *ctx;
+	struct folio *folio;
+	struct runlist_element *rl;
+	unsigned long flags;
+	int mp_size, mp_ofs, name_ofs, arec_size, err, err2;
+	u32 attr_size;
+	u8 old_res_attr_flags;
+
+	if (NInoNonResident(ni)) {
+		ntfs_warning(vol->sb,
+			"Trying to make non-resident attribute non-resident.  Aborting...\n");
+		return -EINVAL;
+	}
+
+	/* Check that the attribute is allowed to be non-resident. */
+	err = ntfs_attr_can_be_non_resident(vol, ni->type);
+	if (unlikely(err)) {
+		if (err == -EPERM)
+			ntfs_debug("Attribute is not allowed to be non-resident.");
+		else
+			ntfs_debug("Attribute not defined on the NTFS volume!");
+		return err;
+	}
+
+	if (NInoEncrypted(ni))
+		return -EIO;
+
+	if (!NInoAttr(ni))
+		base_ni = ni;
+	else
+		base_ni = ni->ext.base_ntfs_ino;
+	m = map_mft_record(base_ni);
+	if (IS_ERR(m)) {
+		err = PTR_ERR(m);
+		m = NULL;
+		ctx = NULL;
+		goto err_out;
+	}
+	ctx = ntfs_attr_get_search_ctx(base_ni, m);
+	if (unlikely(!ctx)) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err)) {
+		if (err == -ENOENT)
+			err = -EIO;
+		goto err_out;
+	}
+	m = ctx->mrec;
+	a = ctx->attr;
+
+	/*
+	 * The size needs to be aligned to a cluster boundary for allocation
+	 * purposes.
+	 */
+	new_size = (data_size + vol->cluster_size - 1) &
+			~(vol->cluster_size - 1);
+	if (new_size > 0) {
+		if ((a->flags & ATTR_COMPRESSION_MASK) == ATTR_IS_COMPRESSED) {
+			/* must allocate full compression blocks */
+			new_size =
+				((new_size - 1) |
+				 ((1L << (STANDARD_COMPRESSION_UNIT +
+					  vol->cluster_size_bits)) - 1)) + 1;
+		}
+
+		/*
+		 * Will need folio later and since folio lock nests
+		 * outside all ntfs locks, we need to get the folio now.
+		 */
+		folio = __filemap_get_folio(vi->i_mapping, 0,
+					    FGP_CREAT | FGP_LOCK,
+					    mapping_gfp_mask(vi->i_mapping));
+		if (IS_ERR(folio)) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+
+		/* Start by allocating clusters to hold the attribute value. */
+		rl = ntfs_cluster_alloc(vol, 0,
+				ntfs_bytes_to_cluster(vol, new_size),
+				-1, DATA_ZONE, true, false, false);
+		if (IS_ERR(rl)) {
+			err = PTR_ERR(rl);
+			ntfs_debug("Failed to allocate cluster%s, error code %i.",
+					ntfs_bytes_to_cluster(vol, new_size) > 1 ? "s" : "",
+					err);
+			goto folio_err_out;
+		}
+	} else {
+		rl = NULL;
+		folio = NULL;
+	}
+
+	down_write(&ni->runlist.lock);
+	/* Determine the size of the mapping pairs array. */
+	mp_size = ntfs_get_size_for_mapping_pairs(vol, rl, 0, -1, -1);
+	if (unlikely(mp_size < 0)) {
+		err = mp_size;
+		ntfs_debug("Failed to get size for mapping pairs array, error code %i.\n", err);
+		goto rl_err_out;
+	}
+
+	if (NInoNonResident(ni) || a->non_resident) {
+		err = -EIO;
+		goto rl_err_out;
+	}
+
+	/*
+	 * Calculate new offsets for the name and the mapping pairs array.
+	 */
+	if (NInoSparse(ni) || NInoCompressed(ni))
+		name_ofs = (offsetof(struct attr_record,
+				data.non_resident.compressed_size) +
+				sizeof(a->data.non_resident.compressed_size) +
+				7) & ~7;
+	else
+		name_ofs = (offsetof(struct attr_record,
+				data.non_resident.compressed_size) + 7) & ~7;
+	mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7;
+	/*
+	 * Determine the size of the resident part of the now non-resident
+	 * attribute record.
+	 */
+	arec_size = (mp_ofs + mp_size + 7) & ~7;
+	/*
+	 * If the folio is not uptodate bring it uptodate by copying from the
+	 * attribute value.
+	 */
+	attr_size = le32_to_cpu(a->data.resident.value_length);
+	WARN_ON(attr_size != data_size);
+	if (folio && !folio_test_uptodate(folio)) {
+		folio_fill_tail(folio, 0, (u8 *)a +
+				le16_to_cpu(a->data.resident.value_offset),
+				attr_size);
+		folio_mark_uptodate(folio);
+	}
+
+	/* Backup the attribute flag. */
+	old_res_attr_flags = a->data.resident.flags;
+	/* Resize the resident part of the attribute record. */
+	err = ntfs_attr_record_resize(m, a, arec_size);
+	if (unlikely(err))
+		goto rl_err_out;
+
+	/*
+	 * Convert the resident part of the attribute record to describe a
+	 * non-resident attribute.
+	 */
+	a->non_resident = 1;
+	/* Move the attribute name if it exists and update the offset. */
+	if (a->name_length)
+		memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset),
+				a->name_length * sizeof(__le16));
+	a->name_offset = cpu_to_le16(name_ofs);
+	/* Setup the fields specific to non-resident attributes. */
+	a->data.non_resident.lowest_vcn = 0;
+	a->data.non_resident.highest_vcn =
+		cpu_to_le64(ntfs_bytes_to_cluster(vol, new_size - 1));
+	a->data.non_resident.mapping_pairs_offset = cpu_to_le16(mp_ofs);
+	memset(&a->data.non_resident.reserved, 0,
+			sizeof(a->data.non_resident.reserved));
+	a->data.non_resident.allocated_size = cpu_to_le64(new_size);
+	a->data.non_resident.data_size =
+			a->data.non_resident.initialized_size =
+			cpu_to_le64(attr_size);
+	if (NInoSparse(ni) || NInoCompressed(ni)) {
+		a->data.non_resident.compression_unit = 0;
+		if (NInoCompressed(ni) || vol->major_ver < 3)
+			a->data.non_resident.compression_unit = 4;
+		a->data.non_resident.compressed_size =
+				a->data.non_resident.allocated_size;
+	} else
+		a->data.non_resident.compression_unit = 0;
+	/* Generate the mapping pairs array into the attribute record. */
+	err = ntfs_mapping_pairs_build(vol, (u8 *)a + mp_ofs,
+			arec_size - mp_ofs, rl, 0, -1, NULL, NULL, NULL);
+	if (unlikely(err)) {
+		ntfs_error(vol->sb, "Failed to build mapping pairs, error code %i.",
+				err);
+		goto undo_err_out;
+	}
+
+	/* Setup the in-memory attribute structure to be non-resident. */
+	ni->runlist.rl = rl;
+	if (rl) {
+		for (ni->runlist.count = 1; rl->length != 0; rl++)
+			ni->runlist.count++;
+	} else
+		ni->runlist.count = 0;
+	write_lock_irqsave(&ni->size_lock, flags);
+	ni->allocated_size = new_size;
+	if (NInoSparse(ni) || NInoCompressed(ni)) {
+		ni->itype.compressed.size = ni->allocated_size;
+		if (a->data.non_resident.compression_unit) {
+			ni->itype.compressed.block_size = 1U <<
+				(a->data.non_resident.compression_unit +
+				 vol->cluster_size_bits);
+			ni->itype.compressed.block_size_bits =
+					ffs(ni->itype.compressed.block_size) -
+					1;
+			ni->itype.compressed.block_clusters = 1U <<
+					a->data.non_resident.compression_unit;
+		} else {
+			ni->itype.compressed.block_size = 0;
+			ni->itype.compressed.block_size_bits = 0;
+			ni->itype.compressed.block_clusters = 0;
+		}
+		vi->i_blocks = ni->itype.compressed.size >> 9;
+	} else
+		vi->i_blocks = ni->allocated_size >> 9;
+	write_unlock_irqrestore(&ni->size_lock, flags);
+	/*
+	 * This needs to be last since the address space operations ->read_folio
+	 * and ->writepage can run concurrently with us as they are not
+	 * serialized on i_mutex.  Note, we are not allowed to fail once we flip
+	 * this switch, which is another reason to do this last.
+	 */
+	NInoSetNonResident(ni);
+	NInoSetFullyMapped(ni);
+	/* Mark the mft record dirty, so it gets written back. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(base_ni);
+	up_write(&ni->runlist.lock);
+	if (folio) {
+		iomap_dirty_folio(vi->i_mapping, folio);
+		folio_unlock(folio);
+		folio_put(folio);
+	}
+	ntfs_debug("Done.");
+	return 0;
+undo_err_out:
+	/* Convert the attribute back into a resident attribute. */
+	a->non_resident = 0;
+	/* Move the attribute name if it exists and update the offset. */
+	name_ofs = (offsetof(struct attr_record, data.resident.reserved) +
+			sizeof(a->data.resident.reserved) + 7) & ~7;
+	if (a->name_length)
+		memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset),
+				a->name_length * sizeof(__le16));
+	mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7;
+	a->name_offset = cpu_to_le16(name_ofs);
+	arec_size = (mp_ofs + attr_size + 7) & ~7;
+	/* Resize the resident part of the attribute record. */
+	err2 = ntfs_attr_record_resize(m, a, arec_size);
+	if (unlikely(err2)) {
+		/*
+		 * This cannot happen (well if memory corruption is at work it
+		 * could happen in theory), but deal with it as well as we can.
+		 * If the old size is too small, truncate the attribute,
+		 * otherwise simply give it a larger allocated size.
+		 */
+		arec_size = le32_to_cpu(a->length);
+		if ((mp_ofs + attr_size) > arec_size) {
+			err2 = attr_size;
+			attr_size = arec_size - mp_ofs;
+			ntfs_error(vol->sb,
+				"Failed to undo partial resident to non-resident attribute conversion.  Truncating inode 0x%llx, attribute type 0x%x from %i bytes to %i bytes to maintain metadata consistency.  THIS MEANS YOU ARE LOSING %i BYTES DATA FROM THIS %s.",
+					ni->mft_no,
+					(unsigned int)le32_to_cpu(ni->type),
+					err2, attr_size, err2 - attr_size,
+					((ni->type == AT_DATA) &&
+					!ni->name_len) ? "FILE" : "ATTRIBUTE");
+			write_lock_irqsave(&ni->size_lock, flags);
+			ni->initialized_size = attr_size;
+			i_size_write(vi, attr_size);
+			write_unlock_irqrestore(&ni->size_lock, flags);
+		}
+	}
+	/* Setup the fields specific to resident attributes. */
+	a->data.resident.value_length = cpu_to_le32(attr_size);
+	a->data.resident.value_offset = cpu_to_le16(mp_ofs);
+	a->data.resident.flags = old_res_attr_flags;
+	memset(&a->data.resident.reserved, 0,
+			sizeof(a->data.resident.reserved));
+	/* Copy the data from folio back to the attribute value. */
+	if (folio)
+		memcpy_from_folio((u8 *)a + mp_ofs, folio, 0, attr_size);
+	/* Setup the allocated size in the ntfs inode in case it changed. */
+	write_lock_irqsave(&ni->size_lock, flags);
+	ni->allocated_size = arec_size - mp_ofs;
+	write_unlock_irqrestore(&ni->size_lock, flags);
+	/* Mark the mft record dirty, so it gets written back. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+rl_err_out:
+	up_write(&ni->runlist.lock);
+	if (rl) {
+		if (ntfs_cluster_free_from_rl(vol, rl) < 0) {
+			ntfs_error(vol->sb,
+				"Failed to release allocated cluster(s) in error code path.  Run chkdsk to recover the lost cluster(s).");
+			NVolSetErrors(vol);
+		}
+		kvfree(rl);
+folio_err_out:
+		folio_unlock(folio);
+		folio_put(folio);
+	}
+err_out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	if (m)
+		unmap_mft_record(base_ni);
+	ni->runlist.rl = NULL;
+
+	if (err == -EINVAL)
+		err = -EIO;
+	return err;
+}
+
+/*
+ * ntfs_attr_set - fill (a part of) an attribute with a byte
+ * @ni:		ntfs inode describing the attribute to fill
+ * @ofs:	offset inside the attribute at which to start to fill
+ * @cnt:	number of bytes to fill
+ * @val:	the unsigned 8-bit value with which to fill the attribute
+ *
+ * Fill @cnt bytes of the attribute described by the ntfs inode @ni starting at
+ * byte offset @ofs inside the attribute with the constant byte @val.
+ *
+ * This function is effectively like memset() applied to an ntfs attribute.
+ * Note thie function actually only operates on the page cache pages belonging
+ * to the ntfs attribute and it marks them dirty after doing the memset().
+ * Thus it relies on the vm dirty page write code paths to cause the modified
+ * pages to be written to the mft record/disk.
+ */
+int ntfs_attr_set(struct ntfs_inode *ni, s64 ofs, s64 cnt, const u8 val)
+{
+	struct address_space *mapping = VFS_I(ni)->i_mapping;
+	struct folio *folio;
+	pgoff_t index;
+	u8 *addr;
+	unsigned long offset;
+	size_t attr_len;
+	int ret = 0;
+
+	index = ofs >> PAGE_SHIFT;
+	while (cnt) {
+		folio = read_mapping_folio(mapping, index, NULL);
+		if (IS_ERR(folio)) {
+			ret = PTR_ERR(folio);
+			ntfs_error(VFS_I(ni)->i_sb, "Failed to read a page %lu for attr %#x: %ld",
+				   index, ni->type, PTR_ERR(folio));
+			break;
+		}
+
+		offset = offset_in_folio(folio, ofs);
+		attr_len = min_t(size_t, (size_t)cnt, folio_size(folio) - offset);
+
+		folio_lock(folio);
+		addr = kmap_local_folio(folio, offset);
+		memset(addr, val, attr_len);
+		kunmap_local(addr);
+
+		folio_mark_dirty(folio);
+		folio_unlock(folio);
+		folio_put(folio);
+
+		ofs += attr_len;
+		cnt -= attr_len;
+		index++;
+		cond_resched();
+	}
+
+	return ret;
+}
+
+int ntfs_attr_set_initialized_size(struct ntfs_inode *ni, loff_t new_size)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	int err = 0;
+
+	if (!NInoNonResident(ni))
+		return -EINVAL;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx)
+		return -ENOMEM;
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			       CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (err)
+		goto out_ctx;
+
+	ctx->attr->data.non_resident.initialized_size = cpu_to_le64(new_size);
+	ni->initialized_size = new_size;
+	mark_mft_record_dirty(ctx->ntfs_ino);
+out_ctx:
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_make_room_for_attr - make room for an attribute inside an mft record
+ * @m:		mft record
+ * @pos:	position at which to make space
+ * @size:	byte size to make available at this position
+ *
+ * @pos points to the attribute in front of which we want to make space.
+ */
+static int ntfs_make_room_for_attr(struct mft_record *m, u8 *pos, u32 size)
+{
+	u32 biu;
+
+	ntfs_debug("Entering for pos 0x%x, size %u.\n",
+			(int)(pos - (u8 *)m), (unsigned int) size);
+
+	/* Make size 8-byte alignment. */
+	size = (size + 7) & ~7;
+
+	/* Rigorous consistency checks. */
+	if (!m || !pos || pos < (u8 *)m) {
+		pr_err("%s: pos=%p  m=%p\n", __func__, pos, m);
+		return -EINVAL;
+	}
+
+	/* The -8 is for the attribute terminator. */
+	if (pos - (u8 *)m > (int)le32_to_cpu(m->bytes_in_use) - 8)
+		return -EINVAL;
+	/* Nothing to do. */
+	if (!size)
+		return 0;
+
+	biu = le32_to_cpu(m->bytes_in_use);
+	/* Do we have enough space? */
+	if (biu + size > le32_to_cpu(m->bytes_allocated) ||
+	    pos + size > (u8 *)m + le32_to_cpu(m->bytes_allocated)) {
+		ntfs_debug("No enough space in the MFT record\n");
+		return -ENOSPC;
+	}
+	/* Move everything after pos to pos + size. */
+	memmove(pos + size, pos, biu - (pos - (u8 *)m));
+	/* Update mft record. */
+	m->bytes_in_use = cpu_to_le32(biu + size);
+	return 0;
+}
+
+/*
+ * ntfs_resident_attr_record_add - add resident attribute to inode
+ * @ni:		opened ntfs inode to which MFT record add attribute
+ * @type:	type of the new attribute
+ * @name:	name of the new attribute
+ * @name_len:	name length of the new attribute
+ * @val:	value of the new attribute
+ * @size:	size of new attribute (length of @val, if @val != NULL)
+ * @flags:	flags of the new attribute
+ */
+int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type,
+		__le16 *name, u8 name_len, u8 *val, u32 size,
+		__le16 flags)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	u32 length;
+	struct attr_record *a;
+	struct mft_record *m;
+	int err, offset;
+	struct ntfs_inode *base_ni;
+
+	if (!ni || (!name && name_len))
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, flags 0x%x.\n",
+			(long long) ni->mft_no, (unsigned int) le32_to_cpu(type),
+			(unsigned int) le16_to_cpu(flags));
+
+	err = ntfs_attr_can_be_resident(ni->vol, type);
+	if (err) {
+		if (err == -EPERM)
+			ntfs_debug("Attribute can't be resident.\n");
+		else
+			ntfs_debug("ntfs_attr_can_be_resident failed.\n");
+		return err;
+	}
+
+	/* Locate place where record should be. */
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ntfs_error(ni->vol->sb, "%s: Failed to get search context",
+				__func__);
+		return -ENOMEM;
+	}
+	/*
+	 * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for
+	 * attribute in @ni->mrec, not any extent inode in case if @ni is base
+	 * file record.
+	 */
+	err = ntfs_attr_find(type, name, name_len, CASE_SENSITIVE, val, size, ctx);
+	if (!err) {
+		err = -EEXIST;
+		ntfs_debug("Attribute already present.\n");
+		goto put_err_out;
+	}
+	if (err != -ENOENT) {
+		err = -EIO;
+		goto put_err_out;
+	}
+	a = ctx->attr;
+	m = ctx->mrec;
+
+	/* Make room for attribute. */
+	length = offsetof(struct attr_record, data.resident.reserved) +
+			  sizeof(a->data.resident.reserved) +
+		((name_len * sizeof(__le16) + 7) & ~7) +
+		((size + 7) & ~7);
+	err = ntfs_make_room_for_attr(ctx->mrec, (u8 *) ctx->attr, length);
+	if (err) {
+		ntfs_debug("Failed to make room for attribute.\n");
+		goto put_err_out;
+	}
+
+	/* Setup record fields. */
+	offset = ((u8 *)a - (u8 *)m);
+	a->type = type;
+	a->length = cpu_to_le32(length);
+	a->non_resident = 0;
+	a->name_length = name_len;
+	a->name_offset =
+		name_len ? cpu_to_le16((offsetof(struct attr_record, data.resident.reserved) +
+				sizeof(a->data.resident.reserved))) : cpu_to_le16(0);
+
+	a->flags = flags;
+	a->instance = m->next_attr_instance;
+	a->data.resident.value_length = cpu_to_le32(size);
+	a->data.resident.value_offset = cpu_to_le16(length - ((size + 7) & ~7));
+	if (val)
+		memcpy((u8 *)a + le16_to_cpu(a->data.resident.value_offset), val, size);
+	else
+		memset((u8 *)a + le16_to_cpu(a->data.resident.value_offset), 0, size);
+	if (type == AT_FILE_NAME)
+		a->data.resident.flags = RESIDENT_ATTR_IS_INDEXED;
+	else
+		a->data.resident.flags = 0;
+	if (name_len)
+		memcpy((u8 *)a + le16_to_cpu(a->name_offset),
+				name, sizeof(__le16) * name_len);
+	m->next_attr_instance =
+		cpu_to_le16((le16_to_cpu(m->next_attr_instance) + 1) & 0xffff);
+	if (ni->nr_extents == -1)
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+	if (type != AT_ATTRIBUTE_LIST && NInoAttrList(base_ni)) {
+		err = ntfs_attrlist_entry_add(ni, a);
+		if (err) {
+			ntfs_attr_record_resize(m, a, 0);
+			mark_mft_record_dirty(ctx->ntfs_ino);
+			ntfs_debug("Failed add attribute entry to ATTRIBUTE_LIST.\n");
+			goto put_err_out;
+		}
+	}
+	mark_mft_record_dirty(ni);
+	ntfs_attr_put_search_ctx(ctx);
+	return offset;
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	return -EIO;
+}
+
+/*
+ * ntfs_non_resident_attr_record_add - add extent of non-resident attribute
+ * @ni:			opened ntfs inode to which MFT record add attribute
+ * @type:		type of the new attribute extent
+ * @name:		name of the new attribute extent
+ * @name_len:		name length of the new attribute extent
+ * @lowest_vcn:		lowest vcn of the new attribute extent
+ * @dataruns_size:	dataruns size of the new attribute extent
+ * @flags:		flags of the new attribute extent
+ */
+static int ntfs_non_resident_attr_record_add(struct ntfs_inode *ni, __le32 type,
+		__le16 *name, u8 name_len, s64 lowest_vcn, int dataruns_size,
+		__le16 flags)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	u32 length;
+	struct attr_record *a;
+	struct mft_record *m;
+	struct ntfs_inode *base_ni;
+	int err, offset;
+
+	if (!ni || dataruns_size <= 0 || (!name && name_len))
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, lowest_vcn %lld, dataruns_size %d, flags 0x%x.\n",
+			(long long) ni->mft_no, (unsigned int) le32_to_cpu(type),
+			(long long) lowest_vcn, dataruns_size,
+			(unsigned int) le16_to_cpu(flags));
+
+	err = ntfs_attr_can_be_non_resident(ni->vol, type);
+	if (err) {
+		if (err == -EPERM)
+			pr_err("Attribute can't be non resident\n");
+		else
+			pr_err("ntfs_attr_can_be_non_resident failed\n");
+		return err;
+	}
+
+	/* Locate place where record should be. */
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		pr_err("%s: Failed to get search context\n", __func__);
+		return -ENOMEM;
+	}
+	/*
+	 * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for
+	 * attribute in @ni->mrec, not any extent inode in case if @ni is base
+	 * file record.
+	 */
+	err = ntfs_attr_find(type, name, name_len, CASE_SENSITIVE, NULL, 0, ctx);
+	if (!err) {
+		err = -EEXIST;
+		pr_err("Attribute 0x%x already present\n", type);
+		goto put_err_out;
+	}
+	if (err != -ENOENT) {
+		pr_err("ntfs_attr_find failed\n");
+		err = -EIO;
+		goto put_err_out;
+	}
+	a = ctx->attr;
+	m = ctx->mrec;
+
+	/* Make room for attribute. */
+	dataruns_size = (dataruns_size + 7) & ~7;
+	length = offsetof(struct attr_record, data.non_resident.compressed_size) +
+		((sizeof(__le16) * name_len + 7) & ~7) + dataruns_size +
+		((flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE)) ?
+		 sizeof(a->data.non_resident.compressed_size) : 0);
+	err = ntfs_make_room_for_attr(ctx->mrec, (u8 *) ctx->attr, length);
+	if (err) {
+		pr_err("Failed to make room for attribute\n");
+		goto put_err_out;
+	}
+
+	/* Setup record fields. */
+	a->type = type;
+	a->length = cpu_to_le32(length);
+	a->non_resident = 1;
+	a->name_length = name_len;
+	a->name_offset = cpu_to_le16(offsetof(struct attr_record,
+					      data.non_resident.compressed_size) +
+			((flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE)) ?
+			 sizeof(a->data.non_resident.compressed_size) : 0));
+	a->flags = flags;
+	a->instance = m->next_attr_instance;
+	a->data.non_resident.lowest_vcn = cpu_to_le64(lowest_vcn);
+	a->data.non_resident.mapping_pairs_offset = cpu_to_le16(length - dataruns_size);
+	a->data.non_resident.compression_unit =
+		(flags & ATTR_IS_COMPRESSED) ? STANDARD_COMPRESSION_UNIT : 0;
+	/* If @lowest_vcn == 0, than setup empty attribute. */
+	if (!lowest_vcn) {
+		a->data.non_resident.highest_vcn = cpu_to_le64(-1);
+		a->data.non_resident.allocated_size = 0;
+		a->data.non_resident.data_size = 0;
+		a->data.non_resident.initialized_size = 0;
+		/* Set empty mapping pairs. */
+		*((u8 *)a + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)) = 0;
+	}
+	if (name_len)
+		memcpy((u8 *)a + le16_to_cpu(a->name_offset),
+				name, sizeof(__le16) * name_len);
+	m->next_attr_instance =
+		cpu_to_le16((le16_to_cpu(m->next_attr_instance) + 1) & 0xffff);
+	if (ni->nr_extents == -1)
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+	if (type != AT_ATTRIBUTE_LIST && NInoAttrList(base_ni)) {
+		err = ntfs_attrlist_entry_add(ni, a);
+		if (err) {
+			pr_err("Failed add attr entry to attrlist\n");
+			ntfs_attr_record_resize(m, a, 0);
+			goto put_err_out;
+		}
+	}
+	mark_mft_record_dirty(ni);
+	/*
+	 * Locate offset from start of the MFT record where new attribute is
+	 * placed. We need relookup it, because record maybe moved during
+	 * update of attribute list.
+	 */
+	ntfs_attr_reinit_search_ctx(ctx);
+	err = ntfs_attr_lookup(type, name, name_len, CASE_SENSITIVE,
+				lowest_vcn, NULL, 0, ctx);
+	if (err) {
+		pr_err("%s: attribute lookup failed\n", __func__);
+		ntfs_attr_put_search_ctx(ctx);
+		return err;
+
+	}
+	offset = (u8 *)ctx->attr - (u8 *)ctx->mrec;
+	ntfs_attr_put_search_ctx(ctx);
+	return offset;
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	return -1;
+}
+
+/*
+ * ntfs_attr_record_rm - remove attribute extent
+ * @ctx:	search context describing the attribute which should be removed
+ *
+ * If this function succeed, user should reinit search context if he/she wants
+ * use it anymore.
+ */
+int ntfs_attr_record_rm(struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_inode *base_ni, *ni;
+	__le32 type;
+	int err;
+
+	if (!ctx || !ctx->ntfs_ino || !ctx->mrec || !ctx->attr)
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			(long long) ctx->ntfs_ino->mft_no,
+			(unsigned int) le32_to_cpu(ctx->attr->type));
+	type = ctx->attr->type;
+	ni = ctx->ntfs_ino;
+	if (ctx->base_ntfs_ino)
+		base_ni = ctx->base_ntfs_ino;
+	else
+		base_ni = ctx->ntfs_ino;
+
+	/* Remove attribute itself. */
+	if (ntfs_attr_record_resize(ctx->mrec, ctx->attr, 0)) {
+		ntfs_debug("Couldn't remove attribute record. Bug or damaged MFT record.\n");
+		return -EIO;
+	}
+	mark_mft_record_dirty(ni);
+
+	/*
+	 * Remove record from $ATTRIBUTE_LIST if present and we don't want
+	 * delete $ATTRIBUTE_LIST itself.
+	 */
+	if (NInoAttrList(base_ni) && type != AT_ATTRIBUTE_LIST) {
+		err = ntfs_attrlist_entry_rm(ctx);
+		if (err) {
+			ntfs_debug("Couldn't delete record from $ATTRIBUTE_LIST.\n");
+			return err;
+		}
+	}
+
+	/* Post $ATTRIBUTE_LIST delete setup. */
+	if (type == AT_ATTRIBUTE_LIST) {
+		if (NInoAttrList(base_ni) && base_ni->attr_list)
+			kvfree(base_ni->attr_list);
+		base_ni->attr_list = NULL;
+		NInoClearAttrList(base_ni);
+	}
+
+	/* Free MFT record, if it doesn't contain attributes. */
+	if (le32_to_cpu(ctx->mrec->bytes_in_use) -
+			le16_to_cpu(ctx->mrec->attrs_offset) == 8) {
+		if (ntfs_mft_record_free(ni->vol, ni)) {
+			ntfs_debug("Couldn't free MFT record.\n");
+			return -EIO;
+		}
+		/* Remove done if we freed base inode. */
+		if (ni == base_ni)
+			return 0;
+		ntfs_inode_close(ni);
+		ctx->ntfs_ino = ni = NULL;
+	}
+
+	if (type == AT_ATTRIBUTE_LIST || !NInoAttrList(base_ni))
+		return 0;
+
+	/* Remove attribute list if we don't need it any more. */
+	if (!ntfs_attrlist_need(base_ni)) {
+		struct ntfs_attr na;
+		struct inode *attr_vi;
+
+		ntfs_attr_reinit_search_ctx(ctx);
+		if (ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, CASE_SENSITIVE,
+					0, NULL, 0, ctx)) {
+			ntfs_debug("Couldn't find attribute list. Succeed anyway.\n");
+			return 0;
+		}
+		/* Deallocate clusters. */
+		if (ctx->attr->non_resident) {
+			struct runlist_element *al_rl;
+			size_t new_rl_count;
+
+			al_rl = ntfs_mapping_pairs_decompress(base_ni->vol,
+					ctx->attr, NULL, &new_rl_count);
+			if (IS_ERR(al_rl)) {
+				ntfs_debug("Couldn't decompress attribute list runlist. Succeed anyway.\n");
+				return 0;
+			}
+			if (ntfs_cluster_free_from_rl(base_ni->vol, al_rl))
+				ntfs_debug("Leaking clusters! Run chkdsk. Couldn't free clusters from attribute list runlist.\n");
+			kvfree(al_rl);
+		}
+		/* Remove attribute record itself. */
+		if (ntfs_attr_record_rm(ctx)) {
+			ntfs_debug("Couldn't remove attribute list. Succeed anyway.\n");
+			return 0;
+		}
+
+		na.mft_no = VFS_I(base_ni)->i_ino;
+		na.type = AT_ATTRIBUTE_LIST;
+		na.name = NULL;
+		na.name_len = 0;
+
+		attr_vi = ilookup5(VFS_I(base_ni)->i_sb, VFS_I(base_ni)->i_ino,
+				   ntfs_test_inode, &na);
+		if (attr_vi) {
+			clear_nlink(attr_vi);
+			iput(attr_vi);
+		}
+
+	}
+	return 0;
+}
+
+/*
+ * ntfs_attr_add - add attribute to inode
+ * @ni:		opened ntfs inode to which add attribute
+ * @type:	type of the new attribute
+ * @name:	name in unicode of the new attribute
+ * @name_len:	name length in unicode characters of the new attribute
+ * @val:	value of new attribute
+ * @size:	size of the new attribute / length of @val (if specified)
+ *
+ * @val should always be specified for always resident attributes (eg. FILE_NAME
+ * attribute), for attributes that can become non-resident @val can be NULL
+ * (eg. DATA attribute). @size can be specified even if @val is NULL, in this
+ * case data size will be equal to @size and initialized size will be equal
+ * to 0.
+ *
+ * If inode haven't got enough space to add attribute, add attribute to one of
+ * it extents, if no extents present or no one of them have enough space, than
+ * allocate new extent and add attribute to it.
+ *
+ * If on one of this steps attribute list is needed but not present, than it is
+ * added transparently to caller. So, this function should not be called with
+ * @type == AT_ATTRIBUTE_LIST, if you really need to add attribute list call
+ * ntfs_inode_add_attrlist instead.
+ *
+ * On success return 0. On error return -1 with errno set to the error code.
+ */
+int ntfs_attr_add(struct ntfs_inode *ni, __le32 type,
+		__le16 *name, u8 name_len, u8 *val, s64 size)
+{
+	struct super_block *sb;
+	u32 attr_rec_size;
+	int err, i, offset;
+	bool is_resident;
+	bool can_be_non_resident = false;
+	struct ntfs_inode *attr_ni;
+	struct inode *attr_vi;
+	struct mft_record *ni_mrec;
+
+	if (!ni || size < 0 || type == AT_ATTRIBUTE_LIST)
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode 0x%llx, attr %x, size %lld.\n",
+			(long long) ni->mft_no, type, size);
+
+	if (ni->nr_extents == -1)
+		ni = ni->ext.base_ntfs_ino;
+
+	/* Check the attribute type and the size. */
+	err = ntfs_attr_size_bounds_check(ni->vol, type, size);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		return err;
+	}
+
+	sb = ni->vol->sb;
+	/* Sanity checks for always resident attributes. */
+	err = ntfs_attr_can_be_non_resident(ni->vol, type);
+	if (err) {
+		if (err != -EPERM) {
+			ntfs_error(sb, "ntfs_attr_can_be_non_resident failed");
+			goto err_out;
+		}
+		/* @val is mandatory. */
+		if (!val) {
+			ntfs_error(sb,
+				"val is mandatory for always resident attributes");
+			return -EINVAL;
+		}
+		if (size > ni->vol->mft_record_size) {
+			ntfs_error(sb, "Attribute is too big");
+			return -ERANGE;
+		}
+	} else
+		can_be_non_resident = true;
+
+	/*
+	 * Determine resident or not will be new attribute. We add 8 to size in
+	 * non resident case for mapping pairs.
+	 */
+	err = ntfs_attr_can_be_resident(ni->vol, type);
+	if (!err) {
+		is_resident = true;
+	} else {
+		if (err != -EPERM) {
+			ntfs_error(sb, "ntfs_attr_can_be_resident failed");
+			goto err_out;
+		}
+		is_resident = false;
+	}
+
+	/* Calculate attribute record size. */
+	if (is_resident)
+		attr_rec_size = offsetof(struct attr_record, data.resident.reserved) +
+			1 +
+			((name_len * sizeof(__le16) + 7) & ~7) +
+			((size + 7) & ~7);
+	else
+		attr_rec_size = offsetof(struct attr_record, data.non_resident.compressed_size) +
+			((name_len * sizeof(__le16) + 7) & ~7) + 8;
+
+	/*
+	 * If we have enough free space for the new attribute in the base MFT
+	 * record, then add attribute to it.
+	 */
+retry:
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec)) {
+		err = -EIO;
+		goto err_out;
+	}
+
+	if (le32_to_cpu(ni_mrec->bytes_allocated) -
+			le32_to_cpu(ni_mrec->bytes_in_use) >= attr_rec_size) {
+		attr_ni = ni;
+		unmap_mft_record(ni);
+		goto add_attr_record;
+	}
+	unmap_mft_record(ni);
+
+	/* Try to add to extent inodes. */
+	err = ntfs_inode_attach_all_extents(ni);
+	if (err) {
+		ntfs_error(sb, "Failed to attach all extents to inode");
+		goto err_out;
+	}
+
+	for (i = 0; i < ni->nr_extents; i++) {
+		attr_ni = ni->ext.extent_ntfs_inos[i];
+		ni_mrec = map_mft_record(attr_ni);
+		if (IS_ERR(ni_mrec)) {
+			err = -EIO;
+			goto err_out;
+		}
+
+		if (le32_to_cpu(ni_mrec->bytes_allocated) -
+				le32_to_cpu(ni_mrec->bytes_in_use) >=
+				attr_rec_size) {
+			unmap_mft_record(attr_ni);
+			goto add_attr_record;
+		}
+		unmap_mft_record(attr_ni);
+	}
+
+	/* There is no extent that contain enough space for new attribute. */
+	if (!NInoAttrList(ni)) {
+		/* Add attribute list not present, add it and retry. */
+		err = ntfs_inode_add_attrlist(ni);
+		if (err) {
+			ntfs_error(sb, "Failed to add attribute list");
+			goto err_out;
+		}
+		goto retry;
+	}
+
+	attr_ni = NULL;
+	/* Allocate new extent. */
+	err = ntfs_mft_record_alloc(ni->vol, 0, &attr_ni, ni, NULL);
+	if (err) {
+		ntfs_error(sb, "Failed to allocate extent record");
+		goto err_out;
+	}
+	unmap_mft_record(attr_ni);
+
+add_attr_record:
+	if (is_resident) {
+		/* Add resident attribute. */
+		offset = ntfs_resident_attr_record_add(attr_ni, type, name,
+				name_len, val, size, 0);
+		if (offset < 0) {
+			if (offset == -ENOSPC && can_be_non_resident)
+				goto add_non_resident;
+			err = offset;
+			ntfs_error(sb, "Failed to add resident attribute");
+			goto free_err_out;
+		}
+		return 0;
+	}
+
+add_non_resident:
+	/* Add non resident attribute. */
+	offset = ntfs_non_resident_attr_record_add(attr_ni, type, name,
+			name_len, 0, 8, 0);
+	if (offset < 0) {
+		err = offset;
+		ntfs_error(sb, "Failed to add non resident attribute");
+		goto free_err_out;
+	}
+
+	/* If @size == 0, we are done. */
+	if (!size)
+		return 0;
+
+	/* Open new attribute and resize it. */
+	attr_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len);
+	if (IS_ERR(attr_vi)) {
+		err = PTR_ERR(attr_vi);
+		ntfs_error(sb, "Failed to open just added attribute");
+		goto rm_attr_err_out;
+	}
+	attr_ni = NTFS_I(attr_vi);
+
+	/* Resize and set attribute value. */
+	if (ntfs_attr_truncate(attr_ni, size) ||
+		(val && (ntfs_inode_attr_pwrite(attr_vi, 0, size, val, false) != size))) {
+		err = -EIO;
+		ntfs_error(sb, "Failed to initialize just added attribute");
+		if (ntfs_attr_rm(attr_ni))
+			ntfs_error(sb, "Failed to remove just added attribute");
+		iput(attr_vi);
+		goto err_out;
+	}
+	iput(attr_vi);
+	return 0;
+
+rm_attr_err_out:
+	/* Remove just added attribute. */
+	ni_mrec = map_mft_record(attr_ni);
+	if (!IS_ERR(ni_mrec)) {
+		if (ntfs_attr_record_resize(ni_mrec,
+					(struct attr_record *)((u8 *)ni_mrec + offset), 0))
+			ntfs_error(sb, "Failed to remove just added attribute #2");
+		unmap_mft_record(attr_ni);
+	} else
+		pr_err("EIO when try to remove new added attr\n");
+
+free_err_out:
+	/* Free MFT record, if it doesn't contain attributes. */
+	ni_mrec = map_mft_record(attr_ni);
+	if (!IS_ERR(ni_mrec)) {
+		int attr_size;
+
+		attr_size = le32_to_cpu(ni_mrec->bytes_in_use) -
+			le16_to_cpu(ni_mrec->attrs_offset);
+		unmap_mft_record(attr_ni);
+		if (attr_size == 8) {
+			if (ntfs_mft_record_free(attr_ni->vol, attr_ni))
+				ntfs_error(sb, "Failed to free MFT record");
+			if (attr_ni->nr_extents < 0)
+				ntfs_inode_close(attr_ni);
+		}
+	} else
+		pr_err("EIO when testing mft record is free-able\n");
+
+err_out:
+	return err;
+}
+
+/*
+ * __ntfs_attr_init - primary initialization of an ntfs attribute structure
+ * @ni:		ntfs attribute inode to initialize
+ * @ni:		ntfs inode with which to initialize the ntfs attribute
+ * @type:	attribute type
+ * @name:	attribute name in little endian Unicode or NULL
+ * @name_len:	length of attribute @name in Unicode characters (if @name given)
+ *
+ * Initialize the ntfs attribute @na with @ni, @type, @name, and @name_len.
+ */
+static void __ntfs_attr_init(struct ntfs_inode *ni,
+		const __le32 type, __le16 *name, const u32 name_len)
+{
+	ni->runlist.rl = NULL;
+	ni->type = type;
+	ni->name = name;
+	if (name)
+		ni->name_len = name_len;
+	else
+		ni->name_len = 0;
+}
+
+/*
+ * ntfs_attr_init - initialize an ntfs_attr with data sizes and status
+ * @ni: ntfs inode to initialize
+ * @non_resident: true if attribute is non-resident
+ * @compressed: true if attribute is compressed
+ * @encrypted: true if attribute is encrypted
+ * @sparse: true if attribute is sparse
+ * @allocated_size: allocated size of the attribute
+ * @data_size: actual data size of the attribute
+ * @initialized_size: initialized size of the attribute
+ * @compressed_size: compressed size (if compressed or sparse)
+ * @compression_unit: compression unit size (log2 of clusters)
+ *
+ * Final initialization for an ntfs attribute.
+ */
+static void ntfs_attr_init(struct ntfs_inode *ni, const bool non_resident,
+		const bool compressed, const bool encrypted, const bool sparse,
+		const s64 allocated_size, const s64 data_size,
+		const s64 initialized_size, const s64 compressed_size,
+		const u8 compression_unit)
+{
+	if (non_resident)
+		NInoSetNonResident(ni);
+	if (compressed) {
+		NInoSetCompressed(ni);
+		ni->flags |= FILE_ATTR_COMPRESSED;
+	}
+	if (encrypted) {
+		NInoSetEncrypted(ni);
+		ni->flags |= FILE_ATTR_ENCRYPTED;
+	}
+	if (sparse) {
+		NInoSetSparse(ni);
+		ni->flags |= FILE_ATTR_SPARSE_FILE;
+	}
+	ni->allocated_size = allocated_size;
+	ni->data_size = data_size;
+	ni->initialized_size = initialized_size;
+	if (compressed || sparse) {
+		struct ntfs_volume *vol = ni->vol;
+
+		ni->itype.compressed.size = compressed_size;
+		ni->itype.compressed.block_clusters = 1 << compression_unit;
+		ni->itype.compressed.block_size = 1 << (compression_unit +
+				vol->cluster_size_bits);
+		ni->itype.compressed.block_size_bits = ffs(
+				ni->itype.compressed.block_size) - 1;
+	}
+}
+
+/*
+ * ntfs_attr_open - open an ntfs attribute for access
+ * @ni:		open ntfs inode in which the ntfs attribute resides
+ * @type:	attribute type
+ * @name:	attribute name in little endian Unicode or AT_UNNAMED or NULL
+ * @name_len:	length of attribute @name in Unicode characters (if @name given)
+ */
+int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type,
+		__le16 *name, u32 name_len)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	__le16 *newname = NULL;
+	struct attr_record *a;
+	bool cs;
+	struct ntfs_inode *base_ni;
+	int err;
+
+	ntfs_debug("Entering for inode %lld, attr 0x%x.\n",
+			(unsigned long long)ni->mft_no, type);
+
+	if (!ni || !ni->vol)
+		return -EINVAL;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	if (name && name != AT_UNNAMED && name != I30) {
+		name = ntfs_ucsndup(name, name_len);
+		if (!name) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+		newname = name;
+	}
+
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		pr_err("%s: Failed to get search context\n", __func__);
+		goto err_out;
+	}
+
+	err = ntfs_attr_lookup(type, name, name_len, 0, 0, NULL, 0, ctx);
+	if (err)
+		goto put_err_out;
+
+	a = ctx->attr;
+
+	if (!name) {
+		if (a->name_length) {
+			name = ntfs_ucsndup((__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+					    a->name_length);
+			if (!name)
+				goto put_err_out;
+			newname = name;
+			name_len = a->name_length;
+		} else {
+			name = AT_UNNAMED;
+			name_len = 0;
+		}
+	}
+
+	__ntfs_attr_init(ni, type, name, name_len);
+
+	/*
+	 * Wipe the flags in case they are not zero for an attribute list
+	 * attribute.  Windows does not complain about invalid flags and chkdsk
+	 * does not detect or fix them so we need to cope with it, too.
+	 */
+	if (type == AT_ATTRIBUTE_LIST)
+		a->flags = 0;
+
+	if ((type == AT_DATA) &&
+	    (a->non_resident ? !a->data.non_resident.initialized_size :
+	     !a->data.resident.value_length)) {
+		/*
+		 * Define/redefine the compression state if stream is
+		 * empty, based on the compression mark on parent
+		 * directory (for unnamed data streams) or on current
+		 * inode (for named data streams). The compression mark
+		 * may change any time, the compression state can only
+		 * change when stream is wiped out.
+		 *
+		 * Also prevent compression on NTFS version < 3.0
+		 * or cluster size > 4K or compression is disabled
+		 */
+		a->flags &= ~ATTR_COMPRESSION_MASK;
+		if (NInoCompressed(ni)
+				&& (ni->vol->major_ver >= 3)
+				&& NVolCompression(ni->vol)
+				&& (ni->vol->cluster_size <= MAX_COMPRESSION_CLUSTER_SIZE))
+			a->flags |= ATTR_IS_COMPRESSED;
+	}
+
+	cs = a->flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE);
+
+	if (ni->type == AT_DATA && ni->name == AT_UNNAMED &&
+	    ((!(a->flags & ATTR_IS_COMPRESSED) != !NInoCompressed(ni)) ||
+	     (!(a->flags & ATTR_IS_SPARSE)     != !NInoSparse(ni)) ||
+	     (!(a->flags & ATTR_IS_ENCRYPTED)  != !NInoEncrypted(ni)))) {
+		err = -EIO;
+		pr_err("Inode %lld has corrupt attribute flags (0x%x <> 0x%x)\n",
+				(unsigned long long)ni->mft_no,
+				a->flags, ni->flags);
+		goto put_err_out;
+	}
+
+	if (a->non_resident) {
+		if (((a->flags & ATTR_COMPRESSION_MASK) || a->data.non_resident.compression_unit) &&
+				(ni->vol->major_ver < 3)) {
+			err = -EIO;
+			pr_err("Compressed inode %lld not allowed  on NTFS %d.%d\n",
+					(unsigned long long)ni->mft_no,
+					ni->vol->major_ver,
+					ni->vol->major_ver);
+			goto put_err_out;
+		}
+
+		if ((a->flags & ATTR_IS_COMPRESSED) && !a->data.non_resident.compression_unit) {
+			err = -EIO;
+			pr_err("Compressed inode %lld attr 0x%x has no compression unit\n",
+					(unsigned long long)ni->mft_no, type);
+			goto put_err_out;
+		}
+		if ((a->flags & ATTR_COMPRESSION_MASK) &&
+		    (a->data.non_resident.compression_unit != STANDARD_COMPRESSION_UNIT)) {
+			err = -EIO;
+			pr_err("Compressed inode %lld attr 0x%lx has an unsupported compression unit %d\n",
+					(unsigned long long)ni->mft_no,
+					(long)le32_to_cpu(type),
+					(int)a->data.non_resident.compression_unit);
+			goto put_err_out;
+		}
+		ntfs_attr_init(ni, true, a->flags & ATTR_IS_COMPRESSED,
+				a->flags & ATTR_IS_ENCRYPTED,
+				a->flags & ATTR_IS_SPARSE,
+				le64_to_cpu(a->data.non_resident.allocated_size),
+				le64_to_cpu(a->data.non_resident.data_size),
+				le64_to_cpu(a->data.non_resident.initialized_size),
+				cs ? le64_to_cpu(a->data.non_resident.compressed_size) : 0,
+				cs ? a->data.non_resident.compression_unit : 0);
+	} else {
+		s64 l = le32_to_cpu(a->data.resident.value_length);
+
+		ntfs_attr_init(ni, false, a->flags & ATTR_IS_COMPRESSED,
+				a->flags & ATTR_IS_ENCRYPTED,
+				a->flags & ATTR_IS_SPARSE, (l + 7) & ~7, l, l,
+				cs ? (l + 7) & ~7 : 0, 0);
+	}
+	ntfs_attr_put_search_ctx(ctx);
+out:
+	ntfs_debug("\n");
+	return err;
+
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+err_out:
+	kfree(newname);
+	goto out;
+}
+
+/*
+ * ntfs_attr_close - free an ntfs attribute structure
+ * @ni:		ntfs inode to free
+ *
+ * Release all memory associated with the ntfs attribute @na and then release
+ * @na itself.
+ */
+void ntfs_attr_close(struct ntfs_inode *ni)
+{
+	if (NInoNonResident(ni) && ni->runlist.rl)
+		kvfree(ni->runlist.rl);
+	/* Don't release if using an internal constant. */
+	if (ni->name != AT_UNNAMED && ni->name != I30)
+		kfree(ni->name);
+}
+
+/*
+ * ntfs_attr_map_whole_runlist - map the whole runlist of an ntfs attribute
+ * @ni:		ntfs inode for which to map the runlist
+ *
+ * Map the whole runlist of the ntfs attribute @na.  For an attribute made up
+ * of only one attribute extent this is the same as calling
+ * ntfs_map_runlist(ni, 0) but for an attribute with multiple extents this
+ * will map the runlist fragments from each of the extents thus giving access
+ * to the entirety of the disk allocation of an attribute.
+ */
+int ntfs_attr_map_whole_runlist(struct ntfs_inode *ni)
+{
+	s64 next_vcn, last_vcn, highest_vcn;
+	struct ntfs_attr_search_ctx *ctx;
+	struct ntfs_volume *vol = ni->vol;
+	struct super_block *sb = vol->sb;
+	struct attr_record *a;
+	int err;
+	struct ntfs_inode *base_ni;
+	int not_mapped;
+	size_t new_rl_count;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			(unsigned long long)ni->mft_no, ni->type);
+
+	if (NInoFullyMapped(ni) && ni->runlist.rl)
+		return 0;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		ntfs_error(sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	/* Map all attribute extents one by one. */
+	next_vcn = last_vcn = highest_vcn = 0;
+	a = NULL;
+	while (1) {
+		struct runlist_element *rl;
+
+		not_mapped = 0;
+		if (ntfs_rl_vcn_to_lcn(ni->runlist.rl, next_vcn) == LCN_RL_NOT_MAPPED)
+			not_mapped = 1;
+
+		err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+					CASE_SENSITIVE, next_vcn, NULL, 0, ctx);
+		if (err)
+			break;
+
+		a = ctx->attr;
+
+		if (not_mapped) {
+			/* Decode the runlist. */
+			rl = ntfs_mapping_pairs_decompress(ni->vol, a, &ni->runlist,
+							   &new_rl_count);
+			if (IS_ERR(rl)) {
+				err = PTR_ERR(rl);
+				goto err_out;
+			}
+			ni->runlist.rl = rl;
+			ni->runlist.count = new_rl_count;
+		}
+
+		/* Are we in the first extent? */
+		if (!next_vcn) {
+			if (a->data.non_resident.lowest_vcn) {
+				err = -EIO;
+				ntfs_error(sb,
+					"First extent of inode %llu attribute has non-zero lowest_vcn",
+					(unsigned long long)ni->mft_no);
+				goto err_out;
+			}
+			/* Get the last vcn in the attribute. */
+			last_vcn = ntfs_bytes_to_cluster(vol,
+					le64_to_cpu(a->data.non_resident.allocated_size));
+		}
+
+		/* Get the lowest vcn for the next extent. */
+		highest_vcn = le64_to_cpu(a->data.non_resident.highest_vcn);
+		next_vcn = highest_vcn + 1;
+
+		/* Only one extent or error, which we catch below. */
+		if (next_vcn <= 0) {
+			err = -ENOENT;
+			break;
+		}
+
+		/* Avoid endless loops due to corruption. */
+		if (next_vcn < le64_to_cpu(a->data.non_resident.lowest_vcn)) {
+			err = -EIO;
+			ntfs_error(sb, "Inode %llu has corrupt attribute list",
+					(unsigned long long)ni->mft_no);
+			goto err_out;
+		}
+	}
+	if (!a) {
+		ntfs_error(sb, "Couldn't find attribute for runlist mapping");
+		goto err_out;
+	}
+	if (not_mapped && highest_vcn && highest_vcn != last_vcn - 1) {
+		err = -EIO;
+		ntfs_error(sb,
+			"Failed to load full runlist: inode: %llu highest_vcn: 0x%llx last_vcn: 0x%llx",
+			(unsigned long long)ni->mft_no,
+			(long long)highest_vcn, (long long)last_vcn);
+		goto err_out;
+	}
+	ntfs_attr_put_search_ctx(ctx);
+	if (err == -ENOENT) {
+		NInoSetFullyMapped(ni);
+		return 0;
+	}
+
+	return err;
+
+err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_attr_record_move_to - move attribute record to target inode
+ * @ctx:	attribute search context describing the attribute record
+ * @ni:		opened ntfs inode to which move attribute record
+ */
+int ntfs_attr_record_move_to(struct ntfs_attr_search_ctx *ctx, struct ntfs_inode *ni)
+{
+	struct ntfs_attr_search_ctx *nctx;
+	struct attr_record *a;
+	int err;
+	struct mft_record *ni_mrec;
+	struct super_block *sb;
+
+	if (!ctx || !ctx->attr || !ctx->ntfs_ino || !ni) {
+		ntfs_debug("Invalid arguments passed.\n");
+		return -EINVAL;
+	}
+
+	sb = ni->vol->sb;
+	ntfs_debug("Entering for ctx->attr->type 0x%x, ctx->ntfs_ino->mft_no 0x%llx, ni->mft_no 0x%llx.\n",
+			(unsigned int) le32_to_cpu(ctx->attr->type),
+			(long long) ctx->ntfs_ino->mft_no,
+			(long long) ni->mft_no);
+
+	if (ctx->ntfs_ino == ni)
+		return 0;
+
+	if (!ctx->al_entry) {
+		ntfs_debug("Inode should contain attribute list to use this function.\n");
+		return -EINVAL;
+	}
+
+	/* Find place in MFT record where attribute will be moved. */
+	a = ctx->attr;
+	nctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!nctx) {
+		ntfs_error(sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	/*
+	 * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for
+	 * attribute in @ni->mrec, not any extent inode in case if @ni is base
+	 * file record.
+	 */
+	err = ntfs_attr_find(a->type, (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+				a->name_length, CASE_SENSITIVE, NULL,
+				0, nctx);
+	if (!err) {
+		ntfs_debug("Attribute of such type, with same name already present in this MFT record.\n");
+		err = -EEXIST;
+		goto put_err_out;
+	}
+	if (err != -ENOENT) {
+		ntfs_debug("Attribute lookup failed.\n");
+		goto put_err_out;
+	}
+
+	/* Make space and move attribute. */
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec)) {
+		err = -EIO;
+		goto put_err_out;
+	}
+
+	err = ntfs_make_room_for_attr(ni_mrec, (u8 *) nctx->attr,
+				le32_to_cpu(a->length));
+	if (err) {
+		ntfs_debug("Couldn't make space for attribute.\n");
+		unmap_mft_record(ni);
+		goto put_err_out;
+	}
+	memcpy(nctx->attr, a, le32_to_cpu(a->length));
+	nctx->attr->instance = nctx->mrec->next_attr_instance;
+	nctx->mrec->next_attr_instance =
+		cpu_to_le16((le16_to_cpu(nctx->mrec->next_attr_instance) + 1) & 0xffff);
+	ntfs_attr_record_resize(ctx->mrec, a, 0);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	mark_mft_record_dirty(ni);
+
+	/* Update attribute list. */
+	ctx->al_entry->mft_reference =
+		MK_LE_MREF(ni->mft_no, le16_to_cpu(ni_mrec->sequence_number));
+	ctx->al_entry->instance = nctx->attr->instance;
+	unmap_mft_record(ni);
+put_err_out:
+	ntfs_attr_put_search_ctx(nctx);
+	return err;
+}
+
+/*
+ * ntfs_attr_record_move_away - move away attribute record from it's mft record
+ * @ctx:	attribute search context describing the attribute record
+ * @extra:	minimum amount of free space in the new holder of record
+ */
+int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra)
+{
+	struct ntfs_inode *base_ni, *ni = NULL;
+	struct mft_record *m;
+	int i, err;
+	struct super_block *sb;
+
+	if (!ctx || !ctx->attr || !ctx->ntfs_ino || extra < 0)
+		return -EINVAL;
+
+	ntfs_debug("Entering for attr 0x%x, inode %llu\n",
+			(unsigned int) le32_to_cpu(ctx->attr->type),
+			(unsigned long long)ctx->ntfs_ino->mft_no);
+
+	if (ctx->ntfs_ino->nr_extents == -1)
+		base_ni = ctx->base_ntfs_ino;
+	else
+		base_ni = ctx->ntfs_ino;
+
+	sb = ctx->ntfs_ino->vol->sb;
+	if (!NInoAttrList(base_ni)) {
+		ntfs_error(sb, "Inode %llu has no attrlist",
+				(unsigned long long)base_ni->mft_no);
+		return -EINVAL;
+	}
+
+	err = ntfs_inode_attach_all_extents(ctx->ntfs_ino);
+	if (err) {
+		ntfs_error(sb, "Couldn't attach extents, inode=%llu",
+			(unsigned long long)base_ni->mft_no);
+		return err;
+	}
+
+	mutex_lock(&base_ni->extent_lock);
+	/* Walk through all extents and try to move attribute to them. */
+	for (i = 0; i < base_ni->nr_extents; i++) {
+		ni = base_ni->ext.extent_ntfs_inos[i];
+
+		if (ctx->ntfs_ino->mft_no == ni->mft_no)
+			continue;
+		m = map_mft_record(ni);
+		if (IS_ERR(m)) {
+			ntfs_error(sb, "Can not map mft record for mft_no %lld",
+					(unsigned long long)ni->mft_no);
+			mutex_unlock(&base_ni->extent_lock);
+			return -EIO;
+		}
+		if (le32_to_cpu(m->bytes_allocated) -
+		    le32_to_cpu(m->bytes_in_use) < le32_to_cpu(ctx->attr->length) + extra) {
+			unmap_mft_record(ni);
+			continue;
+		}
+		unmap_mft_record(ni);
+
+		/*
+		 * ntfs_attr_record_move_to can fail if extent with other lowest
+		 * s64 already present in inode we trying move record to. So,
+		 * do not return error.
+		 */
+		if (!ntfs_attr_record_move_to(ctx, ni)) {
+			mutex_unlock(&base_ni->extent_lock);
+			return 0;
+		}
+	}
+	mutex_unlock(&base_ni->extent_lock);
+
+	/*
+	 * Failed to move attribute to one of the current extents, so allocate
+	 * new extent and move attribute to it.
+	 */
+	ni = NULL;
+	err = ntfs_mft_record_alloc(base_ni->vol, 0, &ni, base_ni, NULL);
+	if (err) {
+		ntfs_error(sb, "Couldn't allocate MFT record, err : %d", err);
+		return err;
+	}
+	unmap_mft_record(ni);
+
+	err = ntfs_attr_record_move_to(ctx, ni);
+	if (err)
+		ntfs_error(sb, "Couldn't move attribute to MFT record");
+
+	return err;
+}
+
+/*
+ * If we are in the first extent, then set/clean sparse bit,
+ * update allocated and compressed size.
+ */
+static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni,
+		struct mft_record *m, struct ntfs_attr_search_ctx *ctx)
+{
+	int sparse, err = 0;
+	struct ntfs_inode *base_ni;
+	struct super_block *sb = ni->vol->sb;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x\n",
+			(unsigned long long)ni->mft_no, ni->type);
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	if (a->data.non_resident.lowest_vcn)
+		goto out;
+
+	a->data.non_resident.allocated_size = cpu_to_le64(ni->allocated_size);
+
+	sparse = ntfs_rl_sparse(ni->runlist.rl);
+	if (sparse < 0) {
+		err = -EIO;
+		goto out;
+	}
+
+	/* Attribute become sparse. */
+	if (sparse && !(a->flags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED))) {
+		/*
+		 * Move attribute to another mft record, if attribute is too
+		 * small to add compressed_size field to it and we have no
+		 * free space in the current mft record.
+		 */
+		if ((le32_to_cpu(a->length) -
+		     le16_to_cpu(a->data.non_resident.mapping_pairs_offset) == 8) &&
+		    !(le32_to_cpu(m->bytes_allocated) - le32_to_cpu(m->bytes_in_use))) {
+
+			if (!NInoAttrList(base_ni)) {
+				err = ntfs_inode_add_attrlist(base_ni);
+				if (err)
+					goto out;
+				err = -EAGAIN;
+				goto out;
+			}
+			err = ntfs_attr_record_move_away(ctx, 8);
+			if (err) {
+				ntfs_error(sb, "Failed to move attribute");
+				goto out;
+			}
+
+			err = ntfs_attrlist_update(base_ni);
+			if (err)
+				goto out;
+			err = -EAGAIN;
+			goto out;
+		}
+		if (!(le32_to_cpu(a->length) -
+		    le16_to_cpu(a->data.non_resident.mapping_pairs_offset))) {
+			err = -EIO;
+			ntfs_error(sb, "Mapping pairs space is 0");
+			goto out;
+		}
+
+		NInoSetSparse(ni);
+		ni->flags |= FILE_ATTR_SPARSE_FILE;
+		a->flags |= ATTR_IS_SPARSE;
+		a->data.non_resident.compression_unit = 0;
+
+		memmove((u8 *)a + le16_to_cpu(a->name_offset) + 8,
+				(u8 *)a + le16_to_cpu(a->name_offset),
+				a->name_length * sizeof(__le16));
+
+		a->name_offset = cpu_to_le16(le16_to_cpu(a->name_offset) + 8);
+
+		a->data.non_resident.mapping_pairs_offset =
+			cpu_to_le16(le16_to_cpu(a->data.non_resident.mapping_pairs_offset) + 8);
+	}
+
+	/* Attribute no longer sparse. */
+	if (!sparse && (a->flags & ATTR_IS_SPARSE) &&
+	    !(a->flags & ATTR_IS_COMPRESSED)) {
+		NInoClearSparse(ni);
+		ni->flags &= ~FILE_ATTR_SPARSE_FILE;
+		a->flags &= ~ATTR_IS_SPARSE;
+		a->data.non_resident.compression_unit = 0;
+
+		memmove((u8 *)a + le16_to_cpu(a->name_offset) - 8,
+				(u8 *)a + le16_to_cpu(a->name_offset),
+				a->name_length * sizeof(__le16));
+
+		if (le16_to_cpu(a->name_offset) >= 8)
+			a->name_offset = cpu_to_le16(le16_to_cpu(a->name_offset) - 8);
+
+		a->data.non_resident.mapping_pairs_offset =
+			cpu_to_le16(le16_to_cpu(a->data.non_resident.mapping_pairs_offset) - 8);
+	}
+
+	/* Update compressed size if required. */
+	if (NInoFullyMapped(ni) && (sparse || NInoCompressed(ni))) {
+		s64 new_compr_size;
+
+		new_compr_size = ntfs_rl_get_compressed_size(ni->vol, ni->runlist.rl);
+		if (new_compr_size < 0) {
+			err = new_compr_size;
+			goto out;
+		}
+
+		ni->itype.compressed.size = new_compr_size;
+		a->data.non_resident.compressed_size = cpu_to_le64(new_compr_size);
+	}
+
+	if (NInoSparse(ni) || NInoCompressed(ni))
+		VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9;
+	else
+		VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9;
+	/*
+	 * Set FILE_NAME dirty flag, to update sparse bit and
+	 * allocated size in the index.
+	 */
+	if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
+		NInoSetFileNameDirty(ni);
+out:
+	return err;
+}
+
+#define NTFS_VCN_DELETE_MARK -2
+/*
+ * ntfs_attr_update_mapping_pairs - update mapping pairs for ntfs attribute
+ * @ni:		non-resident ntfs inode for which we need update
+ * @from_vcn:	update runlist starting this VCN
+ *
+ * Build mapping pairs from @na->rl and write them to the disk. Also, this
+ * function updates sparse bit, allocated and compressed size (allocates/frees
+ * space for this field if required).
+ *
+ * @na->allocated_size should be set to correct value for the new runlist before
+ * call to this function. Vice-versa @na->compressed_size will be calculated and
+ * set to correct value during this function.
+ */
+int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	struct ntfs_inode *base_ni;
+	struct mft_record *m;
+	struct attr_record *a;
+	s64 stop_vcn;
+	int err = 0, mp_size, cur_max_mp_size, exp_max_mp_size;
+	bool finished_build;
+	bool first_updated = false;
+	struct super_block *sb;
+	struct runlist_element *start_rl;
+	unsigned int de_cluster_count = 0;
+
+retry:
+	if (!ni || !ni->runlist.rl)
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode %llu, attr 0x%x\n",
+			(unsigned long long)ni->mft_no, ni->type);
+
+	sb = ni->vol->sb;
+	if (!NInoNonResident(ni)) {
+		ntfs_error(sb, "%s: resident attribute", __func__);
+		return -EINVAL;
+	}
+
+	if (ni->nr_extents == -1)
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		ntfs_error(sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	/* Fill attribute records with new mapping pairs. */
+	stop_vcn = 0;
+	finished_build = false;
+	start_rl = ni->runlist.rl;
+	while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+				CASE_SENSITIVE, from_vcn, NULL, 0, ctx))) {
+		unsigned int de_cnt = 0;
+
+		a = ctx->attr;
+		m = ctx->mrec;
+		if (!a->data.non_resident.lowest_vcn)
+			first_updated = true;
+
+		/*
+		 * If runlist is updating not from the beginning, then set
+		 * @stop_vcn properly, i.e. to the lowest vcn of record that
+		 * contain @from_vcn. Also we do not need @from_vcn anymore,
+		 * set it to 0 to make ntfs_attr_lookup enumerate attributes.
+		 */
+		if (from_vcn) {
+			s64 first_lcn;
+
+			stop_vcn = le64_to_cpu(a->data.non_resident.lowest_vcn);
+			from_vcn = 0;
+			/*
+			 * Check whether the first run we need to update is
+			 * the last run in runlist, if so, then deallocate
+			 * all attrubute extents starting this one.
+			 */
+			first_lcn = ntfs_rl_vcn_to_lcn(ni->runlist.rl, stop_vcn);
+			if (first_lcn == LCN_EINVAL) {
+				err = -EIO;
+				ntfs_error(sb, "Bad runlist");
+				goto put_err_out;
+			}
+			if (first_lcn == LCN_ENOENT ||
+			    first_lcn == LCN_RL_NOT_MAPPED)
+				finished_build = true;
+		}
+
+		/*
+		 * Check whether we finished mapping pairs build, if so mark
+		 * extent as need to delete (by setting highest vcn to
+		 * NTFS_VCN_DELETE_MARK (-2), we shall check it later and
+		 * delete extent) and continue search.
+		 */
+		if (finished_build) {
+			ntfs_debug("Mark attr 0x%x for delete in inode 0x%llx.\n",
+				(unsigned int)le32_to_cpu(a->type), ctx->ntfs_ino->mft_no);
+			a->data.non_resident.highest_vcn = cpu_to_le64(NTFS_VCN_DELETE_MARK);
+			mark_mft_record_dirty(ctx->ntfs_ino);
+			continue;
+		}
+
+		err = ntfs_attr_update_meta(a, ni, m, ctx);
+		if (err < 0) {
+			if (err == -EAGAIN) {
+				ntfs_attr_put_search_ctx(ctx);
+				goto retry;
+			}
+			goto put_err_out;
+		}
+
+		/*
+		 * Determine maximum possible length of mapping pairs,
+		 * if we shall *not* expand space for mapping pairs.
+		 */
+		cur_max_mp_size = le32_to_cpu(a->length) -
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset);
+		/*
+		 * Determine maximum possible length of mapping pairs in the
+		 * current mft record, if we shall expand space for mapping
+		 * pairs.
+		 */
+		exp_max_mp_size = le32_to_cpu(m->bytes_allocated) -
+			le32_to_cpu(m->bytes_in_use) + cur_max_mp_size;
+
+		/* Get the size for the rest of mapping pairs array. */
+		mp_size = ntfs_get_size_for_mapping_pairs(ni->vol, start_rl,
+				stop_vcn, -1, exp_max_mp_size);
+		if (mp_size <= 0) {
+			err = mp_size;
+			ntfs_error(sb, "%s: get MP size failed", __func__);
+			goto put_err_out;
+		}
+		/* Test mapping pairs for fitting in the current mft record. */
+		if (mp_size > exp_max_mp_size) {
+			/*
+			 * Mapping pairs of $ATTRIBUTE_LIST attribute must fit
+			 * in the base mft record. Try to move out other
+			 * attributes and try again.
+			 */
+			if (ni->type == AT_ATTRIBUTE_LIST) {
+				ntfs_attr_put_search_ctx(ctx);
+				if (ntfs_inode_free_space(base_ni, mp_size -
+							cur_max_mp_size)) {
+					ntfs_debug("Attribute list is too big. Defragment the volume\n");
+					return -ENOSPC;
+				}
+				if (ntfs_attrlist_update(base_ni))
+					return -EIO;
+				goto retry;
+			}
+
+			/* Add attribute list if it isn't present, and retry. */
+			if (!NInoAttrList(base_ni)) {
+				ntfs_attr_put_search_ctx(ctx);
+				if (ntfs_inode_add_attrlist(base_ni)) {
+					ntfs_error(sb, "Can not add attrlist");
+					return -EIO;
+				}
+				goto retry;
+			}
+
+			/*
+			 * Set mapping pairs size to maximum possible for this
+			 * mft record. We shall write the rest of mapping pairs
+			 * to another MFT records.
+			 */
+			mp_size = exp_max_mp_size;
+		}
+
+		/* Change space for mapping pairs if we need it. */
+		if (((mp_size + 7) & ~7) != cur_max_mp_size) {
+			if (ntfs_attr_record_resize(m, a,
+					le16_to_cpu(a->data.non_resident.mapping_pairs_offset) +
+						mp_size)) {
+				err = -EIO;
+				ntfs_error(sb, "Failed to resize attribute");
+				goto put_err_out;
+			}
+		}
+
+		/* Update lowest vcn. */
+		a->data.non_resident.lowest_vcn = cpu_to_le64(stop_vcn);
+		mark_mft_record_dirty(ctx->ntfs_ino);
+		if ((ctx->ntfs_ino->nr_extents == -1 || NInoAttrList(ctx->ntfs_ino)) &&
+		    ctx->attr->type != AT_ATTRIBUTE_LIST) {
+			ctx->al_entry->lowest_vcn = cpu_to_le64(stop_vcn);
+			err = ntfs_attrlist_update(base_ni);
+			if (err)
+				goto put_err_out;
+		}
+
+		/*
+		 * Generate the new mapping pairs array directly into the
+		 * correct destination, i.e. the attribute record itself.
+		 */
+		err = ntfs_mapping_pairs_build(ni->vol,
+				(u8 *)a + le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
+				mp_size, start_rl, stop_vcn, -1, &stop_vcn, &start_rl, &de_cnt);
+		if (!err)
+			finished_build = true;
+		if (!finished_build && err != -ENOSPC) {
+			ntfs_error(sb, "Failed to build mapping pairs");
+			goto put_err_out;
+		}
+		a->data.non_resident.highest_vcn = cpu_to_le64(stop_vcn - 1);
+		mark_mft_record_dirty(ctx->ntfs_ino);
+		de_cluster_count += de_cnt;
+	}
+
+	/* Check whether error occurred. */
+	if (err && err != -ENOENT) {
+		ntfs_error(sb, "%s: Attribute lookup failed", __func__);
+		goto put_err_out;
+	}
+
+	/*
+	 * If the base extent was skipped in the above process,
+	 * we still may have to update the sizes.
+	 */
+	if (!first_updated) {
+		ntfs_attr_reinit_search_ctx(ctx);
+		err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+				CASE_SENSITIVE, 0, NULL, 0, ctx);
+		if (!err) {
+			a = ctx->attr;
+			a->data.non_resident.allocated_size = cpu_to_le64(ni->allocated_size);
+			if (NInoCompressed(ni) || NInoSparse(ni))
+				a->data.non_resident.compressed_size =
+					cpu_to_le64(ni->itype.compressed.size);
+			/* Updating sizes taints the extent holding the attr */
+			if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
+				NInoSetFileNameDirty(ni);
+			mark_mft_record_dirty(ctx->ntfs_ino);
+		} else {
+			ntfs_error(sb, "Failed to update sizes in base extent\n");
+			goto put_err_out;
+		}
+	}
+
+	/* Deallocate not used attribute extents and return with success. */
+	if (finished_build) {
+		ntfs_attr_reinit_search_ctx(ctx);
+		ntfs_debug("Deallocate marked extents.\n");
+		while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+				CASE_SENSITIVE, 0, NULL, 0, ctx))) {
+			if (le64_to_cpu(ctx->attr->data.non_resident.highest_vcn) !=
+					NTFS_VCN_DELETE_MARK)
+				continue;
+			/* Remove unused attribute record. */
+			err = ntfs_attr_record_rm(ctx);
+			if (err) {
+				ntfs_error(sb, "Could not remove unused attr");
+				goto put_err_out;
+			}
+			ntfs_attr_reinit_search_ctx(ctx);
+		}
+		if (err && err != -ENOENT) {
+			ntfs_error(sb, "%s: Attr lookup failed", __func__);
+			goto put_err_out;
+		}
+		ntfs_debug("Deallocate done.\n");
+		ntfs_attr_put_search_ctx(ctx);
+		goto out;
+	}
+	ntfs_attr_put_search_ctx(ctx);
+	ctx = NULL;
+
+	/* Allocate new MFT records for the rest of mapping pairs. */
+	while (1) {
+		struct ntfs_inode *ext_ni = NULL;
+		unsigned int de_cnt = 0;
+
+		/* Allocate new mft record. */
+		err = ntfs_mft_record_alloc(ni->vol, 0, &ext_ni, base_ni, NULL);
+		if (err) {
+			ntfs_error(sb, "Failed to allocate extent record");
+			goto put_err_out;
+		}
+		unmap_mft_record(ext_ni);
+
+		m = map_mft_record(ext_ni);
+		if (IS_ERR(m)) {
+			ntfs_error(sb, "Could not map new MFT record");
+			if (ntfs_mft_record_free(ni->vol, ext_ni))
+				ntfs_error(sb, "Could not free MFT record");
+			ntfs_inode_close(ext_ni);
+			err = -ENOMEM;
+			ext_ni = NULL;
+			goto put_err_out;
+		}
+		/*
+		 * If mapping size exceed available space, set them to
+		 * possible maximum.
+		 */
+		cur_max_mp_size = le32_to_cpu(m->bytes_allocated) -
+			le32_to_cpu(m->bytes_in_use) -
+			(sizeof(struct attr_record) +
+			 ((NInoCompressed(ni) || NInoSparse(ni)) ?
+			  sizeof(a->data.non_resident.compressed_size) : 0)) -
+			((sizeof(__le16) * ni->name_len + 7) & ~7);
+
+		/* Calculate size of rest mapping pairs. */
+		mp_size = ntfs_get_size_for_mapping_pairs(ni->vol,
+				start_rl, stop_vcn, -1, cur_max_mp_size);
+		if (mp_size <= 0) {
+			unmap_mft_record(ext_ni);
+			ntfs_inode_close(ext_ni);
+			err = mp_size;
+			ntfs_error(sb, "%s: get mp size failed", __func__);
+			goto put_err_out;
+		}
+
+		if (mp_size > cur_max_mp_size)
+			mp_size = cur_max_mp_size;
+		/* Add attribute extent to new record. */
+		err = ntfs_non_resident_attr_record_add(ext_ni, ni->type,
+				ni->name, ni->name_len, stop_vcn, mp_size, 0);
+		if (err < 0) {
+			ntfs_error(sb, "Could not add attribute extent");
+			unmap_mft_record(ext_ni);
+			if (ntfs_mft_record_free(ni->vol, ext_ni))
+				ntfs_error(sb, "Could not free MFT record");
+			ntfs_inode_close(ext_ni);
+			goto put_err_out;
+		}
+		a = (struct attr_record *)((u8 *)m + err);
+
+		err = ntfs_mapping_pairs_build(ni->vol, (u8 *)a +
+				le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
+				mp_size, start_rl, stop_vcn, -1, &stop_vcn, &start_rl,
+				&de_cnt);
+		if (err < 0 && err != -ENOSPC) {
+			ntfs_error(sb, "Failed to build MP");
+			unmap_mft_record(ext_ni);
+			if (ntfs_mft_record_free(ni->vol, ext_ni))
+				ntfs_error(sb, "Couldn't free MFT record");
+			goto put_err_out;
+		}
+		a->data.non_resident.highest_vcn = cpu_to_le64(stop_vcn - 1);
+		mark_mft_record_dirty(ext_ni);
+		unmap_mft_record(ext_ni);
+
+		de_cluster_count += de_cnt;
+		/* All mapping pairs has been written. */
+		if (!err)
+			break;
+	}
+out:
+	if (from_vcn == 0)
+		ni->i_dealloc_clusters = de_cluster_count;
+	return 0;
+
+put_err_out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_attr_make_resident - convert a non-resident to a resident attribute
+ * @ni:		open ntfs attribute to make resident
+ * @ctx:	ntfs search context describing the attribute
+ *
+ * Convert a non-resident ntfs attribute to a resident one.
+ */
+static int ntfs_attr_make_resident(struct ntfs_inode *ni, struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct super_block *sb = vol->sb;
+	struct attr_record *a = ctx->attr;
+	int name_ofs, val_ofs, err;
+	s64 arec_size;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			(unsigned long long)ni->mft_no, ni->type);
+
+	/* Should be called for the first extent of the attribute. */
+	if (le64_to_cpu(a->data.non_resident.lowest_vcn)) {
+		ntfs_debug("Eeek!  Should be called for the first extent of the attribute.  Aborting...\n");
+		return -EINVAL;
+	}
+
+	/* Some preliminary sanity checking. */
+	if (!NInoNonResident(ni)) {
+		ntfs_debug("Eeek!  Trying to make resident attribute resident. Aborting...\n");
+		return -EINVAL;
+	}
+
+	/* Make sure this is not $MFT/$BITMAP or Windows will not boot! */
+	if (ni->type == AT_BITMAP && ni->mft_no == FILE_MFT)
+		return -EPERM;
+
+	/* Check that the attribute is allowed to be resident. */
+	err = ntfs_attr_can_be_resident(vol, ni->type);
+	if (err)
+		return err;
+
+	if (NInoCompressed(ni) || NInoEncrypted(ni)) {
+		ntfs_debug("Making compressed or encrypted files resident is not implemented yet.\n");
+		return -EOPNOTSUPP;
+	}
+
+	/* Work out offsets into and size of the resident attribute. */
+	name_ofs = 24; /* = sizeof(resident_struct attr_record); */
+	val_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7;
+	arec_size = (val_ofs + ni->data_size + 7) & ~7;
+
+	/* Sanity check the size before we start modifying the attribute. */
+	if (le32_to_cpu(ctx->mrec->bytes_in_use) - le32_to_cpu(a->length) +
+	    arec_size > le32_to_cpu(ctx->mrec->bytes_allocated)) {
+		ntfs_debug("Not enough space to make attribute resident\n");
+		return -ENOSPC;
+	}
+
+	/* Read and cache the whole runlist if not already done. */
+	err = ntfs_attr_map_whole_runlist(ni);
+	if (err)
+		return err;
+
+	/* Move the attribute name if it exists and update the offset. */
+	if (a->name_length) {
+		memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset),
+				a->name_length * sizeof(__le16));
+	}
+	a->name_offset = cpu_to_le16(name_ofs);
+
+	/* Resize the resident part of the attribute record. */
+	if (ntfs_attr_record_resize(ctx->mrec, a, arec_size) < 0) {
+		/*
+		 * Bug, because ntfs_attr_record_resize should not fail (we
+		 * already checked that attribute fits MFT record).
+		 */
+		ntfs_error(ctx->ntfs_ino->vol->sb, "BUG! Failed to resize attribute record. ");
+		return -EIO;
+	}
+
+	/* Convert the attribute record to describe a resident attribute. */
+	a->non_resident = 0;
+	a->flags = 0;
+	a->data.resident.value_length = cpu_to_le32(ni->data_size);
+	a->data.resident.value_offset = cpu_to_le16(val_ofs);
+	/*
+	 * File names cannot be non-resident so we would never see this here
+	 * but at least it serves as a reminder that there may be attributes
+	 * for which we do need to set this flag. (AIA)
+	 */
+	if (a->type == AT_FILE_NAME)
+		a->data.resident.flags = RESIDENT_ATTR_IS_INDEXED;
+	else
+		a->data.resident.flags = 0;
+	a->data.resident.reserved = 0;
+
+	/*
+	 * Deallocate clusters from the runlist.
+	 *
+	 * NOTE: We can use ntfs_cluster_free() because we have already mapped
+	 * the whole run list and thus it doesn't matter that the attribute
+	 * record is in a transiently corrupted state at this moment in time.
+	 */
+	err = ntfs_cluster_free(ni, 0, -1, ctx);
+	if (err) {
+		ntfs_error(sb, "Eeek! Failed to release allocated clusters");
+		ntfs_debug("Ignoring error and leaving behind wasted clusters.\n");
+	}
+
+	/* Throw away the now unused runlist. */
+	kvfree(ni->runlist.rl);
+	ni->runlist.rl = NULL;
+	ni->runlist.count = 0;
+	/* Update in-memory struct ntfs_attr. */
+	NInoClearNonResident(ni);
+	NInoClearCompressed(ni);
+	ni->flags &= ~FILE_ATTR_COMPRESSED;
+	NInoClearSparse(ni);
+	ni->flags &= ~FILE_ATTR_SPARSE_FILE;
+	NInoClearEncrypted(ni);
+	ni->flags &= ~FILE_ATTR_ENCRYPTED;
+	ni->initialized_size = ni->data_size;
+	ni->allocated_size = ni->itype.compressed.size = (ni->data_size + 7) & ~7;
+	ni->itype.compressed.block_size = 0;
+	ni->itype.compressed.block_size_bits = ni->itype.compressed.block_clusters = 0;
+	return 0;
+}
+
+/*
+ * ntfs_non_resident_attr_shrink - shrink a non-resident, open ntfs attribute
+ * @ni:		non-resident ntfs attribute to shrink
+ * @newsize:	new size (in bytes) to which to shrink the attribute
+ *
+ * Reduce the size of a non-resident, open ntfs attribute @na to @newsize bytes.
+ */
+static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsize)
+{
+	struct ntfs_volume *vol;
+	struct ntfs_attr_search_ctx *ctx;
+	s64 first_free_vcn;
+	s64 nr_freed_clusters;
+	int err;
+	struct ntfs_inode *base_ni;
+
+	ntfs_debug("Inode 0x%llx attr 0x%x new size %lld\n",
+		(unsigned long long)ni->mft_no, ni->type, (long long)newsize);
+
+	vol = ni->vol;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	/*
+	 * Check the attribute type and the corresponding minimum size
+	 * against @newsize and fail if @newsize is too small.
+	 */
+	err = ntfs_attr_size_bounds_check(vol, ni->type, newsize);
+	if (err) {
+		if (err == -ERANGE)
+			ntfs_debug("Eeek! Size bounds check failed. Aborting...\n");
+		else if (err == -ENOENT)
+			err = -EIO;
+		return err;
+	}
+
+	/* The first cluster outside the new allocation. */
+	if (NInoCompressed(ni))
+		/*
+		 * For compressed files we must keep full compressions blocks,
+		 * but currently we do not decompress/recompress the last
+		 * block to truncate the data, so we may leave more allocated
+		 * clusters than really needed.
+		 */
+		first_free_vcn = ntfs_bytes_to_cluster(vol,
+				((newsize - 1) | (ni->itype.compressed.block_size - 1)) + 1);
+	else
+		first_free_vcn =
+			ntfs_bytes_to_cluster(vol, newsize + vol->cluster_size - 1);
+
+	if (first_free_vcn < 0)
+		return -EINVAL;
+	/*
+	 * Compare the new allocation with the old one and only deallocate
+	 * clusters if there is a change.
+	 */
+	if (ntfs_bytes_to_cluster(vol, ni->allocated_size) != first_free_vcn) {
+		struct ntfs_attr_search_ctx *ctx;
+
+		err = ntfs_attr_map_whole_runlist(ni);
+		if (err) {
+			ntfs_debug("Eeek! ntfs_attr_map_whole_runlist failed.\n");
+			return err;
+		}
+
+		ctx = ntfs_attr_get_search_ctx(ni, NULL);
+		if (!ctx) {
+			ntfs_error(vol->sb, "%s: Failed to get search context", __func__);
+			return -ENOMEM;
+		}
+
+		/* Deallocate all clusters starting with the first free one. */
+		nr_freed_clusters = ntfs_cluster_free(ni, first_free_vcn, -1, ctx);
+		if (nr_freed_clusters < 0) {
+			ntfs_debug("Eeek! Freeing of clusters failed. Aborting...\n");
+			ntfs_attr_put_search_ctx(ctx);
+			return (int)nr_freed_clusters;
+		}
+		ntfs_attr_put_search_ctx(ctx);
+
+		/* Truncate the runlist itself. */
+		if (ntfs_rl_truncate_nolock(vol, &ni->runlist, first_free_vcn)) {
+			/*
+			 * Failed to truncate the runlist, so just throw it
+			 * away, it will be mapped afresh on next use.
+			 */
+			kvfree(ni->runlist.rl);
+			ni->runlist.rl = NULL;
+			ntfs_error(vol->sb, "Eeek! Run list truncation failed.\n");
+			return -EIO;
+		}
+
+		/* Prepare to mapping pairs update. */
+		ni->allocated_size = ntfs_cluster_to_bytes(vol, first_free_vcn);
+
+		if (NInoSparse(ni) || NInoCompressed(ni)) {
+			if (nr_freed_clusters) {
+				ni->itype.compressed.size -=
+					ntfs_cluster_to_bytes(vol, nr_freed_clusters);
+				VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9;
+			}
+		} else
+			VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9;
+
+		/* Write mapping pairs for new runlist. */
+		err = ntfs_attr_update_mapping_pairs(ni, 0 /*first_free_vcn*/);
+		if (err) {
+			ntfs_debug("Eeek! Mapping pairs update failed. Leaving inconstant metadata. Run chkdsk.\n");
+			return err;
+		}
+	}
+
+	/* Get the first attribute record. */
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		ntfs_error(vol->sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+				0, NULL, 0, ctx);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		ntfs_debug("Eeek! Lookup of first attribute extent failed. Leaving inconstant metadata.\n");
+		goto put_err_out;
+	}
+
+	/* Update data and initialized size. */
+	ni->data_size = newsize;
+	ctx->attr->data.non_resident.data_size = cpu_to_le64(newsize);
+	if (newsize < ni->initialized_size) {
+		ni->initialized_size = newsize;
+		ctx->attr->data.non_resident.initialized_size = cpu_to_le64(newsize);
+	}
+	/* Update data size in the index. */
+	if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
+		NInoSetFileNameDirty(ni);
+
+	/* If the attribute now has zero size, make it resident. */
+	if (!newsize && !NInoEncrypted(ni) && !NInoCompressed(ni)) {
+		err = ntfs_attr_make_resident(ni, ctx);
+		if (err) {
+			/* If couldn't make resident, just continue. */
+			if (err != -EPERM)
+				ntfs_error(ni->vol->sb,
+					"Failed to make attribute resident. Leaving as is...\n");
+		}
+	}
+
+	/* Set the inode dirty so it is written out later. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	/* Done! */
+	ntfs_attr_put_search_ctx(ctx);
+	return 0;
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_non_resident_attr_expand - expand a non-resident, open ntfs attribute
+ * @ni:			non-resident ntfs attribute to expand
+ * @prealloc_size:	preallocation size (in bytes) to which to expand the attribute
+ * @newsize:		new size (in bytes) to which to expand the attribute
+ * @holes:		how to create a hole if expanding
+ * @need_lock:		whether mrec lock is needed or not
+ *
+ * Expand the size of a non-resident, open ntfs attribute @na to @newsize bytes,
+ * by allocating new clusters.
+ */
+static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsize,
+		const s64 prealloc_size, unsigned int holes, bool need_lock)
+{
+	s64 lcn_seek_from;
+	s64 first_free_vcn;
+	struct ntfs_volume *vol;
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct runlist_element *rl, *rln;
+	s64 org_alloc_size, org_compressed_size;
+	int err, err2;
+	struct ntfs_inode *base_ni;
+	struct super_block *sb = ni->vol->sb;
+	size_t new_rl_count;
+
+	ntfs_debug("Inode 0x%llx, attr 0x%x, new size %lld old size %lld\n",
+			(unsigned long long)ni->mft_no, ni->type,
+			(long long)newsize, (long long)ni->data_size);
+
+	vol = ni->vol;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	/*
+	 * Check the attribute type and the corresponding maximum size
+	 * against @newsize and fail if @newsize is too big.
+	 */
+	err = ntfs_attr_size_bounds_check(vol, ni->type, newsize);
+	if (err	< 0) {
+		ntfs_error(sb, "%s: bounds check failed", __func__);
+		return err;
+	}
+
+	/* Save for future use. */
+	org_alloc_size = ni->allocated_size;
+	org_compressed_size = ni->itype.compressed.size;
+
+	/* The first cluster outside the new allocation. */
+	if (prealloc_size)
+		first_free_vcn =
+			ntfs_bytes_to_cluster(vol, prealloc_size + vol->cluster_size - 1);
+	else
+		first_free_vcn =
+			ntfs_bytes_to_cluster(vol, newsize + vol->cluster_size - 1);
+	if (first_free_vcn < 0)
+		return -EFBIG;
+
+	/*
+	 * Compare the new allocation with the old one and only allocate
+	 * clusters if there is a change.
+	 */
+	if (ntfs_bytes_to_cluster(vol, ni->allocated_size) < first_free_vcn) {
+		err = ntfs_attr_map_whole_runlist(ni);
+		if (err) {
+			ntfs_error(sb, "ntfs_attr_map_whole_runlist failed");
+			return err;
+		}
+
+		/*
+		 * If we extend $DATA attribute on NTFS 3+ volume, we can add
+		 * sparse runs instead of real allocation of clusters.
+		 */
+		if ((ni->type == AT_DATA && (vol->major_ver >= 3 || !NInoSparseDisabled(ni))) &&
+		    (holes != HOLES_NO)) {
+			if (NInoCompressed(ni)) {
+				int last = 0, i = 0;
+				s64 alloc_size;
+				u64 more_entries = round_up(first_free_vcn -
+						 ntfs_bytes_to_cluster(vol, ni->allocated_size),
+						 ni->itype.compressed.block_clusters);
+
+				do_div(more_entries, ni->itype.compressed.block_clusters);
+
+				while (ni->runlist.rl[last].length)
+					last++;
+
+				rl = ntfs_rl_realloc(ni->runlist.rl, last + 1,
+						last + more_entries + 1);
+				if (IS_ERR(rl)) {
+					err = -ENOMEM;
+					goto put_err_out;
+				}
+
+				alloc_size = ni->allocated_size;
+				while (i++ < more_entries) {
+					rl[last].vcn = ntfs_bytes_to_cluster(vol,
+							round_up(alloc_size, vol->cluster_size));
+					rl[last].length = ni->itype.compressed.block_clusters -
+						(rl[last].vcn &
+						 (ni->itype.compressed.block_clusters - 1));
+					rl[last].lcn = LCN_HOLE;
+					last++;
+					alloc_size += ni->itype.compressed.block_size;
+				}
+
+				rl[last].vcn = first_free_vcn;
+				rl[last].lcn = LCN_ENOENT;
+				rl[last].length = 0;
+
+				ni->runlist.rl = rl;
+				ni->runlist.count += more_entries;
+			} else {
+				rl = kmalloc(sizeof(struct runlist_element) * 2, GFP_NOFS);
+				if (!rl) {
+					err = -ENOMEM;
+					goto put_err_out;
+				}
+
+				rl[0].vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size);
+				rl[0].lcn = LCN_HOLE;
+				rl[0].length = first_free_vcn -
+					ntfs_bytes_to_cluster(vol, ni->allocated_size);
+				rl[1].vcn = first_free_vcn;
+				rl[1].lcn = LCN_ENOENT;
+				rl[1].length = 0;
+			}
+		} else {
+			/*
+			 * Determine first after last LCN of attribute.
+			 * We will start seek clusters from this LCN to avoid
+			 * fragmentation.  If there are no valid LCNs in the
+			 * attribute let the cluster allocator choose the
+			 * starting LCN.
+			 */
+			lcn_seek_from = -1;
+			if (ni->runlist.rl->length) {
+				/* Seek to the last run list element. */
+				for (rl = ni->runlist.rl; (rl + 1)->length; rl++)
+					;
+				/*
+				 * If the last LCN is a hole or similar seek
+				 * back to last valid LCN.
+				 */
+				while (rl->lcn < 0 && rl != ni->runlist.rl)
+					rl--;
+				/*
+				 * Only set lcn_seek_from it the LCN is valid.
+				 */
+				if (rl->lcn >= 0)
+					lcn_seek_from = rl->lcn + rl->length;
+			}
+
+			rl = ntfs_cluster_alloc(vol,
+					ntfs_bytes_to_cluster(vol, ni->allocated_size),
+					first_free_vcn -
+					ntfs_bytes_to_cluster(vol, ni->allocated_size),
+					lcn_seek_from, DATA_ZONE, false, false, false);
+			if (IS_ERR(rl)) {
+				ntfs_debug("Cluster allocation failed (%lld)",
+						(long long)first_free_vcn -
+						ntfs_bytes_to_cluster(vol, ni->allocated_size));
+				return PTR_ERR(rl);
+			}
+		}
+
+		if (!NInoCompressed(ni)) {
+			/* Append new clusters to attribute runlist. */
+			rln = ntfs_runlists_merge(&ni->runlist, rl, 0, &new_rl_count);
+			if (IS_ERR(rln)) {
+				/* Failed, free just allocated clusters. */
+				ntfs_error(sb, "Run list merge failed");
+				ntfs_cluster_free_from_rl(vol, rl);
+				kvfree(rl);
+				return -EIO;
+			}
+			ni->runlist.rl = rln;
+			ni->runlist.count = new_rl_count;
+		}
+
+		/* Prepare to mapping pairs update. */
+		ni->allocated_size = ntfs_cluster_to_bytes(vol, first_free_vcn);
+		err = ntfs_attr_update_mapping_pairs(ni, 0);
+		if (err) {
+			ntfs_debug("Mapping pairs update failed");
+			goto rollback;
+		}
+	}
+
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		if (ni->allocated_size == org_alloc_size)
+			return err;
+		goto rollback;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+			       0, NULL, 0, ctx);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		if (ni->allocated_size != org_alloc_size)
+			goto rollback;
+		goto put_err_out;
+	}
+
+	/* Update data size. */
+	ni->data_size = newsize;
+	ctx->attr->data.non_resident.data_size = cpu_to_le64(newsize);
+	/* Update data size in the index. */
+	if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
+		NInoSetFileNameDirty(ni);
+	/* Set the inode dirty so it is written out later. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	/* Done! */
+	ntfs_attr_put_search_ctx(ctx);
+	return 0;
+rollback:
+	/* Free allocated clusters. */
+	err2 = ntfs_cluster_free(ni, ntfs_bytes_to_cluster(vol, org_alloc_size),
+				-1, ctx);
+	if (err2)
+		ntfs_debug("Leaking clusters");
+
+	/* Now, truncate the runlist itself. */
+	if (need_lock)
+		down_write(&ni->runlist.lock);
+	err2 = ntfs_rl_truncate_nolock(vol, &ni->runlist,
+			ntfs_bytes_to_cluster(vol, org_alloc_size));
+	if (need_lock)
+		up_write(&ni->runlist.lock);
+	if (err2) {
+		/*
+		 * Failed to truncate the runlist, so just throw it away, it
+		 * will be mapped afresh on next use.
+		 */
+		kvfree(ni->runlist.rl);
+		ni->runlist.rl = NULL;
+		ntfs_error(sb, "Couldn't truncate runlist. Rollback failed");
+	} else {
+		/* Prepare to mapping pairs update. */
+		ni->allocated_size = org_alloc_size;
+		/* Restore mapping pairs. */
+		if (need_lock)
+			down_read(&ni->runlist.lock);
+		if (ntfs_attr_update_mapping_pairs(ni, 0))
+			ntfs_error(sb, "Failed to restore old mapping pairs");
+		if (need_lock)
+			up_read(&ni->runlist.lock);
+
+		if (NInoSparse(ni) || NInoCompressed(ni)) {
+			ni->itype.compressed.size =  org_compressed_size;
+			VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9;
+		} else
+			VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9;
+	}
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	return err;
+put_err_out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_resident_attr_resize - resize a resident, open ntfs attribute
+ * @attr_ni:		resident ntfs inode to resize
+ * @newsize:		new size (in bytes) to which to resize the attribute
+ * @prealloc_size:	preallocation size (in bytes) to which to resize the attribute
+ * @holes:		flags indicating how to handle holes
+ *
+ * Change the size of a resident, open ntfs attribute @na to @newsize bytes.
+ */
+static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsize,
+		const s64 prealloc_size, unsigned int holes)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	struct ntfs_volume *vol = attr_ni->vol;
+	struct super_block *sb = vol->sb;
+	int err = -EIO;
+	struct ntfs_inode *base_ni, *ext_ni = NULL;
+
+attr_resize_again:
+	ntfs_debug("Inode 0x%llx attr 0x%x new size %lld\n",
+			(unsigned long long)attr_ni->mft_no, attr_ni->type,
+			(long long)newsize);
+
+	if (NInoAttr(attr_ni))
+		base_ni = attr_ni->ext.base_ntfs_ino;
+	else
+		base_ni = attr_ni;
+
+	/* Get the attribute record that needs modification. */
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		ntfs_error(sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	err = ntfs_attr_lookup(attr_ni->type, attr_ni->name, attr_ni->name_len,
+			0, 0, NULL, 0, ctx);
+	if (err) {
+		ntfs_error(sb, "ntfs_attr_lookup failed");
+		goto put_err_out;
+	}
+
+	/*
+	 * Check the attribute type and the corresponding minimum and maximum
+	 * sizes against @newsize and fail if @newsize is out of bounds.
+	 */
+	err = ntfs_attr_size_bounds_check(vol, attr_ni->type, newsize);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		ntfs_debug("%s: bounds check failed", __func__);
+		goto put_err_out;
+	}
+	/*
+	 * If @newsize is bigger than the mft record we need to make the
+	 * attribute non-resident if the attribute type supports it. If it is
+	 * smaller we can go ahead and attempt the resize.
+	 */
+	if (newsize < vol->mft_record_size) {
+		/* Perform the resize of the attribute record. */
+		err = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr,
+					newsize);
+		if (!err) {
+			/* Update attribute size everywhere. */
+			attr_ni->data_size = attr_ni->initialized_size = newsize;
+			attr_ni->allocated_size = (newsize + 7) & ~7;
+			if (NInoCompressed(attr_ni) || NInoSparse(attr_ni))
+				attr_ni->itype.compressed.size = attr_ni->allocated_size;
+			if (attr_ni->type == AT_DATA && attr_ni->name == AT_UNNAMED)
+				NInoSetFileNameDirty(attr_ni);
+			goto resize_done;
+		}
+
+		/* Prefer AT_INDEX_ALLOCATION instead of AT_ATTRIBUTE_LIST */
+		if (err == -ENOSPC && ctx->attr->type == AT_INDEX_ROOT)
+			goto put_err_out;
+
+	}
+	/* There is not enough space in the mft record to perform the resize. */
+
+	/* Make the attribute non-resident if possible. */
+	err = ntfs_attr_make_non_resident(attr_ni,
+			le32_to_cpu(ctx->attr->data.resident.value_length));
+	if (!err) {
+		mark_mft_record_dirty(ctx->ntfs_ino);
+		ntfs_attr_put_search_ctx(ctx);
+		/* Resize non-resident attribute */
+		return ntfs_non_resident_attr_expand(attr_ni, newsize, prealloc_size, holes, true);
+	} else if (err != -ENOSPC && err != -EPERM) {
+		ntfs_error(sb, "Failed to make attribute non-resident");
+		goto put_err_out;
+	}
+
+	/* Try to make other attributes non-resident and retry each time. */
+	ntfs_attr_reinit_search_ctx(ctx);
+	while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) {
+		struct inode *tvi;
+		struct attr_record *a;
+
+		a = ctx->attr;
+		if (a->non_resident || a->type == AT_ATTRIBUTE_LIST)
+			continue;
+
+		if (ntfs_attr_can_be_non_resident(vol, a->type))
+			continue;
+
+		/*
+		 * Check out whether convert is reasonable. Assume that mapping
+		 * pairs will take 8 bytes.
+		 */
+		if (le32_to_cpu(a->length) <= (sizeof(struct attr_record) - sizeof(s64)) +
+				((a->name_length * sizeof(__le16) + 7) & ~7) + 8)
+			continue;
+
+		if (a->type == AT_DATA)
+			tvi = ntfs_iget(sb, base_ni->mft_no);
+		else
+			tvi = ntfs_attr_iget(VFS_I(base_ni), a->type,
+				(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
+				a->name_length);
+		if (IS_ERR(tvi)) {
+			ntfs_error(sb, "Couldn't open attribute");
+			continue;
+		}
+
+		if (ntfs_attr_make_non_resident(NTFS_I(tvi),
+		    le32_to_cpu(ctx->attr->data.resident.value_length))) {
+			iput(tvi);
+			continue;
+		}
+
+		mark_mft_record_dirty(ctx->ntfs_ino);
+		iput(tvi);
+		ntfs_attr_put_search_ctx(ctx);
+		goto attr_resize_again;
+	}
+
+	/* Check whether error occurred. */
+	if (err != -ENOENT) {
+		ntfs_error(sb, "%s: Attribute lookup failed 1", __func__);
+		goto put_err_out;
+	}
+
+	/*
+	 * The standard information and attribute list attributes can't be
+	 * moved out from the base MFT record, so try to move out others.
+	 */
+	if (attr_ni->type == AT_STANDARD_INFORMATION ||
+	    attr_ni->type == AT_ATTRIBUTE_LIST) {
+		ntfs_attr_put_search_ctx(ctx);
+
+		if (!NInoAttrList(base_ni)) {
+			err = ntfs_inode_add_attrlist(base_ni);
+			if (err)
+				return err;
+		}
+
+		err = ntfs_inode_free_space(base_ni, sizeof(struct attr_record));
+		if (err) {
+			err = -ENOSPC;
+			ntfs_error(sb,
+				"Couldn't free space in the MFT record to make attribute list non resident");
+			return err;
+		}
+		err = ntfs_attrlist_update(base_ni);
+		if (err)
+			return err;
+		goto attr_resize_again;
+	}
+
+	/*
+	 * Move the attribute to a new mft record, creating an attribute list
+	 * attribute or modifying it if it is already present.
+	 */
+
+	/* Point search context back to attribute which we need resize. */
+	ntfs_attr_reinit_search_ctx(ctx);
+	err = ntfs_attr_lookup(attr_ni->type, attr_ni->name, attr_ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (err) {
+		ntfs_error(sb, "%s: Attribute lookup failed 2", __func__);
+		goto put_err_out;
+	}
+
+	/*
+	 * Check whether attribute is already single in this MFT record.
+	 * 8 added for the attribute terminator.
+	 */
+	if (le32_to_cpu(ctx->mrec->bytes_in_use) ==
+	    le16_to_cpu(ctx->mrec->attrs_offset) + le32_to_cpu(ctx->attr->length) + 8) {
+		err = -ENOSPC;
+		ntfs_debug("MFT record is filled with one attribute\n");
+		goto put_err_out;
+	}
+
+	/* Add attribute list if not present. */
+	if (!NInoAttrList(base_ni)) {
+		ntfs_attr_put_search_ctx(ctx);
+		err = ntfs_inode_add_attrlist(base_ni);
+		if (err)
+			return err;
+		goto attr_resize_again;
+	}
+
+	/* Allocate new mft record. */
+	err = ntfs_mft_record_alloc(base_ni->vol, 0, &ext_ni, base_ni, NULL);
+	if (err) {
+		ntfs_error(sb, "Couldn't allocate MFT record");
+		goto put_err_out;
+	}
+	unmap_mft_record(ext_ni);
+
+	/* Move attribute to it. */
+	err = ntfs_attr_record_move_to(ctx, ext_ni);
+	if (err) {
+		ntfs_error(sb, "Couldn't move attribute to new MFT record");
+		err = -ENOMEM;
+		goto put_err_out;
+	}
+
+	err = ntfs_attrlist_update(base_ni);
+	if (err < 0)
+		goto put_err_out;
+
+	ntfs_attr_put_search_ctx(ctx);
+	/* Try to perform resize once again. */
+	goto attr_resize_again;
+
+resize_done:
+	/*
+	 * Set the inode (and its base inode if it exists) dirty so it is
+	 * written out later.
+	 */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	return 0;
+
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize,
+		const s64 i_size)
+{
+	int err = 0;
+
+	if (newsize < 0 ||
+	    (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) {
+		ntfs_debug("Invalid arguments passed.\n");
+		return -EINVAL;
+	}
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n",
+			(unsigned long long)ni->mft_no, ni->type, newsize);
+
+	if (NInoNonResident(ni)) {
+		if (newsize > i_size) {
+			down_write(&ni->runlist.lock);
+			err = ntfs_non_resident_attr_expand(ni, newsize, 0,
+							    NVolDisableSparse(ni->vol) ?
+							    HOLES_NO : HOLES_OK,
+							    false);
+			up_write(&ni->runlist.lock);
+		} else
+			err = ntfs_non_resident_attr_shrink(ni, newsize);
+	} else
+		err = ntfs_resident_attr_resize(ni, newsize, 0,
+						NVolDisableSparse(ni->vol) ?
+						HOLES_NO : HOLES_OK);
+	ntfs_debug("Return status %d\n", err);
+	return err;
+}
+
+int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size)
+{
+	int err = 0;
+
+	if (newsize < 0 ||
+	    (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) {
+		ntfs_debug("Invalid arguments passed.\n");
+		return -EINVAL;
+	}
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n",
+			(unsigned long long)ni->mft_no, ni->type, newsize);
+
+	if (ni->data_size == newsize) {
+		ntfs_debug("Size is already ok\n");
+		return 0;
+	}
+
+	/*
+	 * Encrypted attributes are not supported. We return access denied,
+	 * which is what Windows NT4 does, too.
+	 */
+	if (NInoEncrypted(ni)) {
+		pr_err("Failed to truncate encrypted attribute\n");
+		return -EACCES;
+	}
+
+	if (NInoNonResident(ni)) {
+		if (newsize > ni->data_size)
+			err = ntfs_non_resident_attr_expand(ni, newsize, prealloc_size,
+							    NVolDisableSparse(ni->vol) ?
+							    HOLES_NO : HOLES_OK, true);
+	} else
+		err = ntfs_resident_attr_resize(ni, newsize, prealloc_size,
+						NVolDisableSparse(ni->vol) ?
+						HOLES_NO : HOLES_OK);
+	if (!err)
+		i_size_write(VFS_I(ni), newsize);
+	ntfs_debug("Return status %d\n", err);
+	return err;
+}
+
+/*
+ * ntfs_attr_truncate_i - resize an ntfs attribute
+ * @ni:		open ntfs inode to resize
+ * @newsize:	new size (in bytes) to which to resize the attribute
+ * @holes:	how to create a hole if expanding
+ *
+ * Change the size of an open ntfs attribute @na to @newsize bytes. If the
+ * attribute is made bigger and the attribute is resident the newly
+ * "allocated" space is cleared and if the attribute is non-resident the
+ * newly allocated space is marked as not initialised and no real allocation
+ * on disk is performed.
+ */
+int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes)
+{
+	int err;
+
+	if (newsize < 0 ||
+	    (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) {
+		ntfs_debug("Invalid arguments passed.\n");
+		return -EINVAL;
+	}
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n",
+			(unsigned long long)ni->mft_no, ni->type, newsize);
+
+	if (ni->data_size == newsize) {
+		ntfs_debug("Size is already ok\n");
+		return 0;
+	}
+
+	/*
+	 * Encrypted attributes are not supported. We return access denied,
+	 * which is what Windows NT4 does, too.
+	 */
+	if (NInoEncrypted(ni)) {
+		pr_err("Failed to truncate encrypted attribute\n");
+		return -EACCES;
+	}
+
+	if (NInoCompressed(ni)) {
+		pr_err("Failed to truncate compressed attribute\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (NInoNonResident(ni)) {
+		if (newsize > ni->data_size)
+			err = ntfs_non_resident_attr_expand(ni, newsize, 0, holes, true);
+		else
+			err = ntfs_non_resident_attr_shrink(ni, newsize);
+	} else
+		err = ntfs_resident_attr_resize(ni, newsize, 0, holes);
+	ntfs_debug("Return status %d\n", err);
+	return err;
+}
+
+/*
+ * Resize an attribute, creating a hole if relevant
+ */
+int ntfs_attr_truncate(struct ntfs_inode *ni, const s64 newsize)
+{
+	return ntfs_attr_truncate_i(ni, newsize,
+				    NVolDisableSparse(ni->vol) ?
+				    HOLES_NO : HOLES_OK);
+}
+
+int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start,
+		s64 *lcn_count, s64 max_clu_count, bool *balloc, bool update_mp,
+		bool skip_holes)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct ntfs_attr_search_ctx *ctx;
+	struct runlist_element *rl, *rlc;
+	s64 vcn = vcn_start, lcn, clu_count;
+	s64 lcn_seek_from = -1;
+	int err = 0;
+	size_t new_rl_count;
+
+	err = ntfs_attr_map_whole_runlist(ni);
+	if (err)
+		return err;
+
+	if (NInoAttr(ni))
+		ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL);
+	else
+		ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ntfs_error(vol->sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, vcn, NULL, 0, ctx);
+	if (err) {
+		ntfs_error(vol->sb,
+			   "ntfs_attr_lookup failed, ntfs inode(mft_no : %llu) type : 0x%x, err : %d",
+			   ni->mft_no, ni->type, err);
+		goto out;
+	}
+
+	rl = ntfs_attr_find_vcn_nolock(ni, vcn, ctx);
+	if (IS_ERR(rl)) {
+		ntfs_error(vol->sb, "Failed to find run after mapping runlist.");
+		err = PTR_ERR(rl);
+		goto out;
+	}
+
+	lcn = ntfs_rl_vcn_to_lcn(rl, vcn);
+	clu_count = min(max_clu_count, rl->length - (vcn - rl->vcn));
+	if (lcn >= LCN_HOLE) {
+		if (lcn > LCN_DELALLOC ||
+		    (lcn == LCN_HOLE && skip_holes)) {
+			*lcn_start = lcn;
+			*lcn_count = clu_count;
+			*balloc = false;
+			goto out;
+		}
+	} else {
+		WARN_ON(lcn == LCN_RL_NOT_MAPPED);
+		if (lcn == LCN_ENOENT)
+			err = -ENOENT;
+		else
+			err = -EIO;
+		goto out;
+	}
+
+	/* Search backwards to find the best lcn to start seek from. */
+	rlc = rl;
+	while (rlc->vcn) {
+		rlc--;
+		if (rlc->lcn >= 0) {
+			/*
+			 * avoid fragmenting a compressed file
+			 * Windows does not do that, and that may
+			 * not be desirable for files which can
+			 * be updated
+			 */
+			if (NInoCompressed(ni))
+				lcn_seek_from = rlc->lcn + rlc->length;
+			else
+				lcn_seek_from = rlc->lcn + (vcn - rlc->vcn);
+			break;
+		}
+	}
+
+	if (lcn_seek_from == -1) {
+		/* Backwards search failed, search forwards. */
+		rlc = rl;
+		while (rlc->length) {
+			rlc++;
+			if (rlc->lcn >= 0) {
+				lcn_seek_from = rlc->lcn - (rlc->vcn - vcn);
+				if (lcn_seek_from < -1)
+					lcn_seek_from = -1;
+				break;
+			}
+		}
+	}
+
+	rlc = ntfs_cluster_alloc(vol, vcn, clu_count, lcn_seek_from, DATA_ZONE,
+			false, true, true);
+	if (IS_ERR(rlc)) {
+		err = PTR_ERR(rlc);
+		goto out;
+	}
+
+	WARN_ON(rlc->vcn != vcn);
+	lcn = rlc->lcn;
+	clu_count = rlc->length;
+
+	rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, &new_rl_count);
+	if (IS_ERR(rl)) {
+		ntfs_error(vol->sb, "Failed to merge runlists");
+		err = PTR_ERR(rl);
+		if (ntfs_cluster_free_from_rl(vol, rlc))
+			ntfs_error(vol->sb, "Failed to free hot clusters.");
+		kvfree(rlc);
+		goto out;
+	}
+	ni->runlist.rl = rl;
+	ni->runlist.count = new_rl_count;
+
+	if (!update_mp) {
+		u64 free = atomic64_read(&vol->free_clusters) * 100;
+
+		do_div(free, vol->nr_clusters);
+		if (free <= 5)
+			update_mp = true;
+	}
+
+	if (update_mp) {
+		ntfs_attr_reinit_search_ctx(ctx);
+		err = ntfs_attr_update_mapping_pairs(ni, 0);
+		if (err) {
+			int err2;
+
+			err2 = ntfs_cluster_free(ni, vcn, clu_count, ctx);
+			if (err2 < 0)
+				ntfs_error(vol->sb,
+					   "Failed to free cluster allocation. Leaving inconstant metadata.\n");
+			goto out;
+		}
+	} else {
+		VFS_I(ni)->i_blocks += clu_count << (vol->cluster_size_bits - 9);
+		NInoSetRunlistDirty(ni);
+		mark_mft_record_dirty(ni);
+	}
+
+	*lcn_start = lcn;
+	*lcn_count = clu_count;
+	*balloc = true;
+out:
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+/*
+ * ntfs_attr_rm - remove attribute from ntfs inode
+ * @ni:		opened ntfs attribute to delete
+ *
+ * Remove attribute and all it's extents from ntfs inode. If attribute was non
+ * resident also free all clusters allocated by attribute.
+ */
+int ntfs_attr_rm(struct ntfs_inode *ni)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	int err = 0, ret = 0;
+	struct ntfs_inode *base_ni;
+	struct super_block *sb = ni->vol->sb;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			(long long) ni->mft_no, ni->type);
+
+	/* Free cluster allocation. */
+	if (NInoNonResident(ni)) {
+		struct ntfs_attr_search_ctx *ctx;
+
+		err = ntfs_attr_map_whole_runlist(ni);
+		if (err)
+			return err;
+		ctx = ntfs_attr_get_search_ctx(ni, NULL);
+		if (!ctx) {
+			ntfs_error(sb, "%s: Failed to get search context", __func__);
+			return -ENOMEM;
+		}
+
+		ret = ntfs_cluster_free(ni, 0, -1, ctx);
+		if (ret < 0)
+			ntfs_error(sb,
+				"Failed to free cluster allocation. Leaving inconstant metadata.\n");
+		ntfs_attr_put_search_ctx(ctx);
+	}
+
+	/* Search for attribute extents and remove them all. */
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		ntfs_error(sb, "%s: Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+	while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+				CASE_SENSITIVE, 0, NULL, 0, ctx))) {
+		err = ntfs_attr_record_rm(ctx);
+		if (err) {
+			ntfs_error(sb,
+				"Failed to remove attribute extent. Leaving inconstant metadata.\n");
+			ret = err;
+		}
+		ntfs_attr_reinit_search_ctx(ctx);
+	}
+	ntfs_attr_put_search_ctx(ctx);
+	if (err != -ENOENT) {
+		ntfs_error(sb, "Attribute lookup failed. Probably leaving inconstant metadata.\n");
+		ret = err;
+	}
+
+	return ret;
+}
+
+int ntfs_attr_exist(struct ntfs_inode *ni, const __le32 type, __le16 *name,
+		u32 name_len)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ntfs_error(ni->vol->sb, "%s: Failed to get search context",
+				__func__);
+		return 0;
+	}
+
+	ret = ntfs_attr_lookup(type, name, name_len, CASE_SENSITIVE,
+			0, NULL, 0, ctx);
+	ntfs_attr_put_search_ctx(ctx);
+
+	return !ret;
+}
+
+int ntfs_attr_remove(struct ntfs_inode *ni, const __le32 type, __le16 *name,
+		u32 name_len)
+{
+	int err;
+	struct inode *attr_vi;
+	struct ntfs_inode *attr_ni;
+
+	ntfs_debug("Entering\n");
+
+	if (!ni)
+		return -EINVAL;
+
+	attr_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len);
+	if (IS_ERR(attr_vi)) {
+		err = PTR_ERR(attr_vi);
+		ntfs_error(ni->vol->sb, "Failed to open attribute 0x%02x of inode 0x%llx",
+				type, (unsigned long long)ni->mft_no);
+		return err;
+	}
+	attr_ni = NTFS_I(attr_vi);
+
+	err = ntfs_attr_rm(attr_ni);
+	if (err)
+		ntfs_error(ni->vol->sb, "Failed to remove attribute 0x%02x of inode 0x%llx",
+				type, (unsigned long long)ni->mft_no);
+	iput(attr_vi);
+	return err;
+}
+
+/*
+ * ntfs_attr_readall - read the entire data from an ntfs attribute
+ * @ni:		open ntfs inode in which the ntfs attribute resides
+ * @type:	attribute type
+ * @name:	attribute name in little endian Unicode or AT_UNNAMED or NULL
+ * @name_len:	length of attribute @name in Unicode characters (if @name given)
+ * @data_size:	if non-NULL then store here the data size
+ *
+ * This function will read the entire content of an ntfs attribute.
+ * If @name is AT_UNNAMED then look specifically for an unnamed attribute.
+ * If @name is NULL then the attribute could be either named or not.
+ * In both those cases @name_len is not used at all.
+ *
+ * On success a buffer is allocated with the content of the attribute
+ * and which needs to be freed when it's not needed anymore. If the
+ * @data_size parameter is non-NULL then the data size is set there.
+ */
+void *ntfs_attr_readall(struct ntfs_inode *ni, const __le32 type,
+		__le16 *name, u32 name_len, s64 *data_size)
+{
+	struct ntfs_inode *bmp_ni;
+	struct inode *bmp_vi;
+	void *data, *ret = NULL;
+	s64 size;
+	struct super_block *sb = ni->vol->sb;
+
+	ntfs_debug("Entering\n");
+
+	bmp_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len);
+	if (IS_ERR(bmp_vi)) {
+		ntfs_debug("ntfs_attr_iget failed");
+		goto err_exit;
+	}
+	bmp_ni = NTFS_I(bmp_vi);
+
+	data = kvmalloc(bmp_ni->data_size, GFP_NOFS);
+	if (!data)
+		goto out;
+
+	size = ntfs_inode_attr_pread(VFS_I(bmp_ni), 0, bmp_ni->data_size,
+			(u8 *)data);
+	if (size != bmp_ni->data_size) {
+		ntfs_error(sb, "ntfs_attr_pread failed");
+		kvfree(data);
+		goto out;
+	}
+	ret = data;
+	if (data_size)
+		*data_size = size;
+out:
+	iput(bmp_vi);
+err_exit:
+	ntfs_debug("\n");
+	return ret;
+}
+
+int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s64 len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct runlist_element *hole_rl, *rl;
+	struct ntfs_attr_search_ctx *ctx;
+	int ret;
+	size_t new_rl_count;
+
+	if (NInoAttr(ni) || ni->type != AT_DATA)
+		return -EOPNOTSUPP;
+	if (start_vcn > ntfs_bytes_to_cluster(vol, ni->allocated_size))
+		return -EINVAL;
+
+	hole_rl = kmalloc(sizeof(*hole_rl) * 2, GFP_NOFS);
+	if (!hole_rl)
+		return -ENOMEM;
+	hole_rl[0].vcn = start_vcn;
+	hole_rl[0].lcn = LCN_HOLE;
+	hole_rl[0].length = len;
+	hole_rl[1].vcn = start_vcn + len;
+	hole_rl[1].lcn = LCN_ENOENT;
+	hole_rl[1].length = 0;
+
+	down_write(&ni->runlist.lock);
+	ret = ntfs_attr_map_whole_runlist(ni);
+	if (ret) {
+		up_write(&ni->runlist.lock);
+		return ret;
+	}
+
+	rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn);
+	if (!rl) {
+		up_write(&ni->runlist.lock);
+		kfree(hole_rl);
+		return -EIO;
+	}
+
+	rl = ntfs_rl_insert_range(ni->runlist.rl, (int)ni->runlist.count,
+				  hole_rl, 1, &new_rl_count);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		kfree(hole_rl);
+		return PTR_ERR(rl);
+	}
+	ni->runlist.rl =  rl;
+	ni->runlist.count = new_rl_count;
+
+	ni->allocated_size += ntfs_cluster_to_bytes(vol, len);
+	ni->data_size += ntfs_cluster_to_bytes(vol, len);
+	if (ntfs_cluster_to_bytes(vol, start_vcn) < ni->initialized_size)
+		ni->initialized_size += ntfs_cluster_to_bytes(vol, len);
+	ret = ntfs_attr_update_mapping_pairs(ni, 0);
+	up_write(&ni->runlist.lock);
+	if (ret)
+		return ret;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ret = -ENOMEM;
+		return ret;
+	}
+
+	ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+			       0, NULL, 0, ctx);
+	if (ret) {
+		ntfs_attr_put_search_ctx(ctx);
+		return ret;
+	}
+
+	ctx->attr->data.non_resident.data_size = cpu_to_le64(ni->data_size);
+	ctx->attr->data.non_resident.initialized_size = cpu_to_le64(ni->initialized_size);
+	if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
+		NInoSetFileNameDirty(ni);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	return ret;
+}
+
+int ntfs_non_resident_attr_collapse_range(struct ntfs_inode *ni, s64 start_vcn, s64 len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct runlist_element *punch_rl, *rl;
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	s64 end_vcn;
+	int dst_cnt;
+	int ret;
+	size_t new_rl_cnt;
+
+	if (NInoAttr(ni) || ni->type != AT_DATA)
+		return -EOPNOTSUPP;
+
+	end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size);
+	if (start_vcn >= end_vcn)
+		return -EINVAL;
+
+	down_write(&ni->runlist.lock);
+	ret = ntfs_attr_map_whole_runlist(ni);
+	if (ret) {
+		up_write(&ni->runlist.lock);
+		return ret;
+	}
+
+	len = min(len, end_vcn - start_vcn);
+	for (rl = ni->runlist.rl, dst_cnt = 0; rl && rl->length; rl++)
+		dst_cnt++;
+	rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn);
+	if (!rl) {
+		up_write(&ni->runlist.lock);
+		return -EIO;
+	}
+
+	rl = ntfs_rl_collapse_range(ni->runlist.rl, dst_cnt + 1,
+				    start_vcn, len, &punch_rl, &new_rl_cnt);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		return PTR_ERR(rl);
+	}
+	ni->runlist.rl = rl;
+	ni->runlist.count = new_rl_cnt;
+
+	ni->allocated_size -= ntfs_cluster_to_bytes(vol, len);
+	if (ni->data_size > ntfs_cluster_to_bytes(vol, start_vcn)) {
+		if (ni->data_size > ntfs_cluster_to_bytes(vol, (start_vcn + len)))
+			ni->data_size -= ntfs_cluster_to_bytes(vol, len);
+		else
+			ni->data_size = ntfs_cluster_to_bytes(vol, start_vcn);
+	}
+	if (ni->initialized_size > ntfs_cluster_to_bytes(vol, start_vcn)) {
+		if (ni->initialized_size >
+		    ntfs_cluster_to_bytes(vol, start_vcn + len))
+			ni->initialized_size -= ntfs_cluster_to_bytes(vol, len);
+		else
+			ni->initialized_size = ntfs_cluster_to_bytes(vol, start_vcn);
+	}
+
+	if (ni->allocated_size > 0) {
+		ret = ntfs_attr_update_mapping_pairs(ni, 0);
+		if (ret) {
+			up_write(&ni->runlist.lock);
+			goto out_rl;
+		}
+	}
+	up_write(&ni->runlist.lock);
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ret = -ENOMEM;
+		goto out_rl;
+	}
+
+	ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+			       0, NULL, 0, ctx);
+	if (ret)
+		goto out_ctx;
+
+	ctx->attr->data.non_resident.data_size = cpu_to_le64(ni->data_size);
+	ctx->attr->data.non_resident.initialized_size = cpu_to_le64(ni->initialized_size);
+	if (ni->allocated_size == 0)
+		ntfs_attr_make_resident(ni, ctx);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+
+	ret = ntfs_cluster_free_from_rl(vol, punch_rl);
+	if (ret)
+		ntfs_error(vol->sb, "Freeing of clusters failed");
+out_ctx:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+out_rl:
+	kvfree(punch_rl);
+	mark_mft_record_dirty(ni);
+	return ret;
+}
+
+int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct runlist_element *punch_rl, *rl;
+	s64 end_vcn;
+	int dst_cnt;
+	int ret;
+	size_t new_rl_count;
+
+	if (NInoAttr(ni) || ni->type != AT_DATA)
+		return -EOPNOTSUPP;
+
+	end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size);
+	if (start_vcn >= end_vcn)
+		return -EINVAL;
+
+	down_write(&ni->runlist.lock);
+	ret = ntfs_attr_map_whole_runlist(ni);
+	if (ret) {
+		up_write(&ni->runlist.lock);
+		return ret;
+	}
+
+	len = min(len, end_vcn - start_vcn + 1);
+	for (rl = ni->runlist.rl, dst_cnt = 0; rl && rl->length; rl++)
+		dst_cnt++;
+	rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn);
+	if (!rl) {
+		up_write(&ni->runlist.lock);
+		return -EIO;
+	}
+
+	rl = ntfs_rl_punch_hole(ni->runlist.rl, dst_cnt + 1,
+				start_vcn, len, &punch_rl, &new_rl_count);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		return PTR_ERR(rl);
+	}
+	ni->runlist.rl = rl;
+	ni->runlist.count = new_rl_count;
+
+	ret = ntfs_attr_update_mapping_pairs(ni, 0);
+	up_write(&ni->runlist.lock);
+	if (ret) {
+		kvfree(punch_rl);
+		return ret;
+	}
+
+	ret = ntfs_cluster_free_from_rl(vol, punch_rl);
+	if (ret)
+		ntfs_error(vol->sb, "Freeing of clusters failed");
+
+	kvfree(punch_rl);
+	mark_mft_record_dirty(ni);
+	return ret;
+}
+
+int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bool keep_size)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct mft_record *mrec;
+	struct ntfs_attr_search_ctx *ctx;
+	s64 old_data_size;
+	s64 vcn_start, vcn_end, vcn_uninit, vcn, try_alloc_cnt;
+	s64 lcn, alloc_cnt;
+	int err = 0;
+	struct runlist_element *rl;
+	bool balloc;
+
+	if (NInoAttr(ni) || ni->type != AT_DATA)
+		return -EINVAL;
+
+	if (NInoNonResident(ni) && !NInoFullyMapped(ni)) {
+		down_write(&ni->runlist.lock);
+		err = ntfs_attr_map_whole_runlist(ni);
+		up_write(&ni->runlist.lock);
+		if (err)
+			return err;
+	}
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	mrec = map_mft_record(ni);
+	if (IS_ERR(mrec)) {
+		mutex_unlock(&ni->mrec_lock);
+		return PTR_ERR(mrec);
+	}
+
+	ctx = ntfs_attr_get_search_ctx(ni, mrec);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto out_unmap;
+	}
+
+	err = ntfs_attr_lookup(AT_DATA, AT_UNNAMED, 0, 0, 0, NULL, 0, ctx);
+	if (err) {
+		err = -EIO;
+		goto out_unmap;
+	}
+
+	old_data_size = ni->data_size;
+	if (start + byte_len > ni->data_size) {
+		err = ntfs_attr_truncate(ni, start + byte_len);
+		if (err)
+			goto out_unmap;
+		if (keep_size) {
+			ntfs_attr_reinit_search_ctx(ctx);
+			err = ntfs_attr_lookup(AT_DATA, AT_UNNAMED, 0, 0, 0, NULL, 0, ctx);
+			if (err) {
+				err = -EIO;
+				goto out_unmap;
+			}
+			ni->data_size = old_data_size;
+			if (NInoNonResident(ni))
+				ctx->attr->data.non_resident.data_size =
+					cpu_to_le64(old_data_size);
+			else
+				ctx->attr->data.resident.value_length =
+					cpu_to_le32((u32)old_data_size);
+			mark_mft_record_dirty(ni);
+		}
+	}
+
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(ni);
+	mutex_unlock(&ni->mrec_lock);
+
+	if (!NInoNonResident(ni))
+		goto out;
+
+	vcn_start = (s64)ntfs_bytes_to_cluster(vol, start);
+	vcn_end = (s64)ntfs_bytes_to_cluster(vol,
+			round_up(start + byte_len, vol->cluster_size));
+	vcn_uninit = (s64)ntfs_bytes_to_cluster(vol,
+			round_up(ni->initialized_size, vol->cluster_size));
+	vcn_uninit = min_t(s64, vcn_uninit, vcn_end);
+
+	/*
+	 * we have to allocate clusters for holes and delayed within initialized_size,
+	 * and zero out the clusters only for the holes.
+	 */
+	vcn = vcn_start;
+	while (vcn < vcn_uninit) {
+		down_read(&ni->runlist.lock);
+		rl = ntfs_attr_find_vcn_nolock(ni, vcn, NULL);
+		up_read(&ni->runlist.lock);
+		if (IS_ERR(rl)) {
+			err = PTR_ERR(rl);
+			goto out;
+		}
+
+		if (rl->lcn > 0) {
+			vcn += rl->length - (vcn - rl->vcn);
+		} else if (rl->lcn == LCN_DELALLOC || rl->lcn == LCN_HOLE) {
+			try_alloc_cnt = min(rl->length - (vcn - rl->vcn),
+					    vcn_uninit - vcn);
+
+			if (rl->lcn == LCN_DELALLOC) {
+				vcn += try_alloc_cnt;
+				continue;
+			}
+
+			while (try_alloc_cnt > 0) {
+				mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+				down_write(&ni->runlist.lock);
+				err = ntfs_attr_map_cluster(ni, vcn, &lcn, &alloc_cnt,
+							    try_alloc_cnt, &balloc, false, false);
+				up_write(&ni->runlist.lock);
+				mutex_unlock(&ni->mrec_lock);
+				if (err)
+					goto out;
+
+				err = ntfs_dio_zero_range(VFS_I(ni),
+							  lcn << vol->cluster_size_bits,
+							  alloc_cnt << vol->cluster_size_bits);
+				if (err > 0)
+					goto out;
+
+				if (signal_pending(current))
+					goto out;
+
+				vcn += alloc_cnt;
+				try_alloc_cnt -= alloc_cnt;
+			}
+		} else {
+			err = -EIO;
+			goto out;
+		}
+	}
+
+	/* allocate clusters outside of initialized_size */
+	try_alloc_cnt = vcn_end - vcn;
+	while (try_alloc_cnt > 0) {
+		mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+		down_write(&ni->runlist.lock);
+		err = ntfs_attr_map_cluster(ni, vcn, &lcn, &alloc_cnt,
+					    try_alloc_cnt, &balloc, false, false);
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+		if (err || signal_pending(current))
+			goto out;
+
+		vcn += alloc_cnt;
+		try_alloc_cnt -= alloc_cnt;
+		cond_resched();
+	}
+
+	if (NInoRunlistDirty(ni)) {
+		mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+		down_write(&ni->runlist.lock);
+		err = ntfs_attr_update_mapping_pairs(ni, 0);
+		if (err)
+			ntfs_error(ni->vol->sb, "Updating mapping pairs failed");
+		else
+			NInoClearRunlistDirty(ni);
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+	}
+	return err;
+out_unmap:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(ni);
+	mutex_unlock(&ni->mrec_lock);
+out:
+	return err >= 0 ? 0 : err;
+}
diff --git a/fs/ntfs/attrib.h b/fs/ntfs/attrib.h
new file mode 100644
index 0000000000000..f7acc7986b090
--- /dev/null
+++ b/fs/ntfs/attrib.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for attribute handling in NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2005 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_ATTRIB_H
+#define _LINUX_NTFS_ATTRIB_H
+
+#include "ntfs.h"
+#include "dir.h"
+
+extern __le16 AT_UNNAMED[];
+
+/*
+ * ntfs_attr_search_ctx - used in attribute search functions
+ * @mrec: buffer containing mft record to search
+ * @mapped_mrec: true if @mrec was mapped by the search functions
+ * @attr: attribute record in @mrec where to begin/continue search
+ * @is_first: if true ntfs_attr_lookup() begins search with @attr, else after
+ * @ntfs_ino: Inode owning this attribute search
+ * @al_entry: Current attribute list entry
+ * @base_ntfs_ino: Base inode
+ * @mapped_base_mrec: true if @base_mrec was mapped by the search
+ * @base_attr: Base attribute record pointer
+ *
+ * Structure must be initialized to zero before the first call to one of the
+ * attribute search functions. Initialize @mrec to point to the mft record to
+ * search, and @attr to point to the first attribute within @mrec (not necessary
+ * if calling the _first() functions), and set @is_first to 'true' (not necessary
+ * if calling the _first() functions).
+ *
+ * If @is_first is 'true', the search begins with @attr. If @is_first is 'false',
+ * the search begins after @attr. This is so that, after the first call to one
+ * of the search attribute functions, we can call the function again, without
+ * any modification of the search context, to automagically get the next
+ * matching attribute.
+ */
+struct ntfs_attr_search_ctx {
+	struct mft_record *mrec;
+	bool mapped_mrec;
+	struct attr_record *attr;
+	bool is_first;
+	struct ntfs_inode *ntfs_ino;
+	struct attr_list_entry *al_entry;
+	struct ntfs_inode *base_ntfs_ino;
+	struct mft_record *base_mrec;
+	bool mapped_base_mrec;
+	struct attr_record *base_attr;
+};
+
+enum {                  /* ways of processing holes when expanding */
+	HOLES_NO,
+	HOLES_OK,
+};
+
+int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn,
+		struct ntfs_attr_search_ctx *ctx);
+int ntfs_map_runlist(struct ntfs_inode *ni, s64 vcn);
+s64 ntfs_attr_vcn_to_lcn_nolock(struct ntfs_inode *ni, const s64 vcn,
+		const bool write_locked);
+struct runlist_element *ntfs_attr_find_vcn_nolock(struct ntfs_inode *ni,
+		const s64 vcn, struct ntfs_attr_search_ctx *ctx);
+struct runlist_element *__ntfs_attr_find_vcn_nolock(struct runlist *runlist,
+		const s64 vcn);
+int ntfs_attr_map_whole_runlist(struct ntfs_inode *ni);
+int ntfs_attr_lookup(const __le32 type, const __le16 *name,
+		const u32 name_len, const u32 ic,
+		const s64 lowest_vcn, const u8 *val, const u32 val_len,
+		struct ntfs_attr_search_ctx *ctx);
+int load_attribute_list(struct ntfs_inode *base_ni,
+			       u8 *al_start, const s64 size);
+
+static inline s64 ntfs_attr_size(const struct attr_record *a)
+{
+	if (!a->non_resident)
+		return (s64)le32_to_cpu(a->data.resident.value_length);
+	return le64_to_cpu(a->data.non_resident.data_size);
+}
+
+void ntfs_attr_reinit_search_ctx(struct ntfs_attr_search_ctx *ctx);
+struct ntfs_attr_search_ctx *ntfs_attr_get_search_ctx(struct ntfs_inode *ni,
+		struct mft_record *mrec);
+void ntfs_attr_put_search_ctx(struct ntfs_attr_search_ctx *ctx);
+int ntfs_attr_size_bounds_check(const struct ntfs_volume *vol,
+		const __le32 type, const s64 size);
+int ntfs_attr_can_be_resident(const struct ntfs_volume *vol,
+		const __le32 type);
+int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start,
+		s64 *lcn_count, s64 max_clu_count, bool *balloc, bool update_mp, bool skip_holes);
+int ntfs_attr_record_resize(struct mft_record *m, struct attr_record *a, u32 new_size);
+int ntfs_resident_attr_value_resize(struct mft_record *m, struct attr_record *a,
+		const u32 new_size);
+int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size);
+int ntfs_attr_set(struct ntfs_inode *ni, const s64 ofs, const s64 cnt,
+		const u8 val);
+int ntfs_attr_set_initialized_size(struct ntfs_inode *ni, loff_t new_size);
+int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type,
+		__le16 *name, u32 name_len);
+void ntfs_attr_close(struct ntfs_inode *n);
+int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bool keep_size);
+int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s64 len);
+int ntfs_non_resident_attr_collapse_range(struct ntfs_inode *ni, s64 start_vcn, s64 len);
+int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 len);
+int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize,
+		const s64 i_size);
+int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size);
+int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes);
+int ntfs_attr_truncate(struct ntfs_inode *ni, const s64 newsize);
+int ntfs_attr_rm(struct ntfs_inode *ni);
+int ntfs_attr_exist(struct ntfs_inode *ni, const __le32 type, __le16 *name,
+		u32 name_len);
+int ntfs_attr_remove(struct ntfs_inode *ni, const __le32 type, __le16 *name,
+		u32 name_len);
+int ntfs_attr_record_rm(struct ntfs_attr_search_ctx *ctx);
+int ntfs_attr_record_move_to(struct ntfs_attr_search_ctx *ctx, struct ntfs_inode *ni);
+int ntfs_attr_add(struct ntfs_inode *ni, __le32 type,
+		__le16 *name, u8 name_len, u8 *val, s64 size);
+int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra);
+char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname,
+		const int uname_len);
+void ntfs_attr_name_free(unsigned char **name);
+void *ntfs_attr_readall(struct ntfs_inode *ni, const __le32 type,
+		__le16 *name, u32 name_len, s64 *data_size);
+int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type,
+		__le16 *name, u8 name_len, u8 *val, u32 size,
+		__le16 flags);
+int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn);
+struct runlist_element *ntfs_attr_vcn_to_rl(struct ntfs_inode *ni, s64 vcn, s64 *lcn);
+
+/*
+ * ntfs_attrs_walk - syntactic sugar for walking all attributes in an inode
+ * @ctx:	initialised attribute search context
+ *
+ * Syntactic sugar for walking attributes in an inode.
+ *
+ * Return 0 on success and -1 on error with errno set to the error code from
+ * ntfs_attr_lookup().
+ *
+ * Example: When you want to enumerate all attributes in an open ntfs inode
+ *	    @ni, you can simply do:
+ *
+ *	int err;
+ *	struct ntfs_attr_search_ctx *ctx = ntfs_attr_get_search_ctx(ni, NULL);
+ *	if (!ctx)
+ *		// Error code is in errno. Handle this case.
+ *	while (!(err = ntfs_attrs_walk(ctx))) {
+ *		struct attr_record *attr = ctx->attr;
+ *		// attr now contains the next attribute. Do whatever you want
+ *		// with it and then just continue with the while loop.
+ *	}
+ *	if (err && errno != ENOENT)
+ *		// Ooops. An error occurred! You should handle this case.
+ *	// Now finished with all attributes in the inode.
+ */
+static inline int ntfs_attrs_walk(struct ntfs_attr_search_ctx *ctx)
+{
+	return ntfs_attr_lookup(AT_UNUSED, NULL, 0, CASE_SENSITIVE, 0,
+			NULL, 0, ctx);
+}
+#endif /* _LINUX_NTFS_ATTRIB_H */
diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c
new file mode 100644
index 0000000000000..bd501e8a628c4
--- /dev/null
+++ b/fs/ntfs/attrlist.c
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Attribute list attribute handling code.
+ * Part of this file is based on code from the NTFS-3G.
+ *
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2004-2005 Yura Pakhuchiy
+ * Copyright (c)      2006 Szabolcs Szakacsits
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include "mft.h"
+#include "attrib.h"
+#include "attrlist.h"
+
+/*
+ * ntfs_attrlist_need - check whether inode need attribute list
+ * @ni:	opened ntfs inode for which perform check
+ *
+ * Check whether all are attributes belong to one MFT record, in that case
+ * attribute list is not needed.
+ *
+ * Return 1 if inode need attribute list, 0 if not, or -errno on error.
+ */
+int ntfs_attrlist_need(struct ntfs_inode *ni)
+{
+	struct attr_list_entry *ale;
+
+	if (!ni) {
+		ntfs_debug("Invalid arguments.\n");
+		return -EINVAL;
+	}
+	ntfs_debug("Entering for inode 0x%llx.\n", (long long) ni->mft_no);
+
+	if (!NInoAttrList(ni)) {
+		ntfs_debug("Inode haven't got attribute list.\n");
+		return -EINVAL;
+	}
+
+	if (!ni->attr_list) {
+		ntfs_debug("Corrupt in-memory struct.\n");
+		return -EINVAL;
+	}
+
+	ale = (struct attr_list_entry *)ni->attr_list;
+	while ((u8 *)ale < ni->attr_list + ni->attr_list_size) {
+		if (MREF_LE(ale->mft_reference) != ni->mft_no)
+			return 1;
+		ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length));
+	}
+	return 0;
+}
+
+int ntfs_attrlist_update(struct ntfs_inode *base_ni)
+{
+	struct inode *attr_vi;
+	struct ntfs_inode *attr_ni;
+	int err;
+
+	attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0);
+	if (IS_ERR(attr_vi)) {
+		err = PTR_ERR(attr_vi);
+		return err;
+	}
+	attr_ni = NTFS_I(attr_vi);
+
+	err = ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO);
+	if (err == -ENOSPC && attr_ni->mft_no == FILE_MFT) {
+		err = ntfs_attr_truncate(attr_ni, 0);
+		if (err || ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO) != 0) {
+			iput(attr_vi);
+			ntfs_error(base_ni->vol->sb,
+					"Failed to truncate attribute list of inode %#llx",
+					(long long)base_ni->mft_no);
+			return -EIO;
+		}
+	} else if (err) {
+		iput(attr_vi);
+		ntfs_error(base_ni->vol->sb,
+			   "Failed to truncate attribute list of inode %#llx",
+			   (long long)base_ni->mft_no);
+		return -EIO;
+	}
+
+	i_size_write(attr_vi, base_ni->attr_list_size);
+
+	if (NInoNonResident(attr_ni) && !NInoAttrListNonResident(base_ni))
+		NInoSetAttrListNonResident(base_ni);
+
+	if (ntfs_inode_attr_pwrite(attr_vi, 0, base_ni->attr_list_size,
+				   base_ni->attr_list, false) !=
+	    base_ni->attr_list_size) {
+		iput(attr_vi);
+		ntfs_error(base_ni->vol->sb,
+			   "Failed to write attribute list of inode %#llx",
+			   (long long)base_ni->mft_no);
+		return -EIO;
+	}
+
+	NInoSetAttrListDirty(base_ni);
+	iput(attr_vi);
+	return 0;
+}
+
+/*
+ * ntfs_attrlist_entry_add - add an attribute list attribute entry
+ * @ni:	opened ntfs inode, which contains that attribute
+ * @attr: attribute record to add to attribute list
+ *
+ * Return 0 on success and -errno on error.
+ */
+int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr)
+{
+	struct attr_list_entry *ale;
+	__le64 mref;
+	struct ntfs_attr_search_ctx *ctx;
+	u8 *new_al;
+	int entry_len, entry_offset, err;
+	struct mft_record *ni_mrec;
+	u8 *old_al;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			(long long) ni->mft_no,
+			(unsigned int) le32_to_cpu(attr->type));
+
+	if (!ni || !attr) {
+		ntfs_debug("Invalid arguments.\n");
+		return -EINVAL;
+	}
+
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec)) {
+		ntfs_debug("Invalid arguments.\n");
+		return -EIO;
+	}
+
+	mref = MK_LE_MREF(ni->mft_no, le16_to_cpu(ni_mrec->sequence_number));
+	unmap_mft_record(ni);
+
+	if (ni->nr_extents == -1)
+		ni = ni->ext.base_ntfs_ino;
+
+	if (!NInoAttrList(ni)) {
+		ntfs_debug("Attribute list isn't present.\n");
+		return -ENOENT;
+	}
+
+	/* Determine size and allocate memory for new attribute list. */
+	entry_len = (sizeof(struct attr_list_entry) + sizeof(__le16) *
+			attr->name_length + 7) & ~7;
+	new_al = kvzalloc(ni->attr_list_size + entry_len, GFP_NOFS);
+	if (!new_al)
+		return -ENOMEM;
+
+	/* Find place for the new entry. */
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		ntfs_error(ni->vol->sb, "Failed to get search context");
+		goto err_out;
+	}
+
+	err = ntfs_attr_lookup(attr->type, (attr->name_length) ? (__le16 *)
+			((u8 *)attr + le16_to_cpu(attr->name_offset)) :
+			AT_UNNAMED, attr->name_length, CASE_SENSITIVE,
+			(attr->non_resident) ? le64_to_cpu(attr->data.non_resident.lowest_vcn) :
+			0, (attr->non_resident) ? NULL : ((u8 *)attr +
+			le16_to_cpu(attr->data.resident.value_offset)), (attr->non_resident) ?
+			0 : le32_to_cpu(attr->data.resident.value_length), ctx);
+	if (!err) {
+		/* Found some extent, check it to be before new extent. */
+		if (ctx->al_entry->lowest_vcn == attr->data.non_resident.lowest_vcn) {
+			err = -EEXIST;
+			ntfs_debug("Such attribute already present in the attribute list.\n");
+			ntfs_attr_put_search_ctx(ctx);
+			goto err_out;
+		}
+		/* Add new entry after this extent. */
+		ale = (struct attr_list_entry *)((u8 *)ctx->al_entry +
+				le16_to_cpu(ctx->al_entry->length));
+	} else {
+		/* Check for real errors. */
+		if (err != -ENOENT) {
+			ntfs_debug("Attribute lookup failed.\n");
+			ntfs_attr_put_search_ctx(ctx);
+			goto err_out;
+		}
+		/* No previous extents found. */
+		ale = ctx->al_entry;
+	}
+	/* Don't need it anymore, @ctx->al_entry points to @ni->attr_list. */
+	ntfs_attr_put_search_ctx(ctx);
+
+	/* Determine new entry offset. */
+	entry_offset = ((u8 *)ale - ni->attr_list);
+	/* Set pointer to new entry. */
+	ale = (struct attr_list_entry *)(new_al + entry_offset);
+	memset(ale, 0, entry_len);
+	/* Form new entry. */
+	ale->type = attr->type;
+	ale->length = cpu_to_le16(entry_len);
+	ale->name_length = attr->name_length;
+	ale->name_offset = offsetof(struct attr_list_entry, name);
+	if (attr->non_resident)
+		ale->lowest_vcn = attr->data.non_resident.lowest_vcn;
+	else
+		ale->lowest_vcn = 0;
+	ale->mft_reference = mref;
+	ale->instance = attr->instance;
+	memcpy(ale->name, (u8 *)attr + le16_to_cpu(attr->name_offset),
+			attr->name_length * sizeof(__le16));
+
+	/* Copy entries from old attribute list to new. */
+	memcpy(new_al, ni->attr_list, entry_offset);
+	memcpy(new_al + entry_offset + entry_len, ni->attr_list +
+			entry_offset, ni->attr_list_size - entry_offset);
+
+	/* Set new runlist. */
+	old_al = ni->attr_list;
+	ni->attr_list = new_al;
+	ni->attr_list_size = ni->attr_list_size + entry_len;
+
+	err = ntfs_attrlist_update(ni);
+	if (err) {
+		ni->attr_list = old_al;
+		ni->attr_list_size -= entry_len;
+		goto err_out;
+	}
+	kvfree(old_al);
+	return 0;
+err_out:
+	kvfree(new_al);
+	return err;
+}
+
+/*
+ * ntfs_attrlist_entry_rm - remove an attribute list attribute entry
+ * @ctx:	attribute search context describing the attribute list entry
+ *
+ * Remove the attribute list entry @ctx->al_entry from the attribute list.
+ *
+ * Return 0 on success and -errno on error.
+ */
+int ntfs_attrlist_entry_rm(struct ntfs_attr_search_ctx *ctx)
+{
+	u8 *new_al;
+	int new_al_len;
+	struct ntfs_inode *base_ni;
+	struct attr_list_entry *ale;
+
+	if (!ctx || !ctx->ntfs_ino || !ctx->al_entry) {
+		ntfs_debug("Invalid arguments.\n");
+		return -EINVAL;
+	}
+
+	if (ctx->base_ntfs_ino)
+		base_ni = ctx->base_ntfs_ino;
+	else
+		base_ni = ctx->ntfs_ino;
+	ale = ctx->al_entry;
+
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x, lowest_vcn %lld.\n",
+			(long long)ctx->ntfs_ino->mft_no,
+			(unsigned int)le32_to_cpu(ctx->al_entry->type),
+			(long long)le64_to_cpu(ctx->al_entry->lowest_vcn));
+
+	if (!NInoAttrList(base_ni)) {
+		ntfs_debug("Attribute list isn't present.\n");
+		return -ENOENT;
+	}
+
+	/* Allocate memory for new attribute list. */
+	new_al_len = base_ni->attr_list_size - le16_to_cpu(ale->length);
+	new_al = kvzalloc(new_al_len, GFP_NOFS);
+	if (!new_al)
+		return -ENOMEM;
+
+	/* Copy entries from old attribute list to new. */
+	memcpy(new_al, base_ni->attr_list, (u8 *)ale - base_ni->attr_list);
+	memcpy(new_al + ((u8 *)ale - base_ni->attr_list), (u8 *)ale + le16_to_cpu(
+				ale->length), new_al_len - ((u8 *)ale - base_ni->attr_list));
+
+	/* Set new runlist. */
+	kvfree(base_ni->attr_list);
+	base_ni->attr_list = new_al;
+	base_ni->attr_list_size = new_al_len;
+
+	return ntfs_attrlist_update(base_ni);
+}
diff --git a/fs/ntfs/attrlist.h b/fs/ntfs/attrlist.h
new file mode 100644
index 0000000000000..1892a3934d3ab
--- /dev/null
+++ b/fs/ntfs/attrlist.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Exports for attribute list attribute handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ * Copyright (c) 2004 Yura Pakhuchiy
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _NTFS_ATTRLIST_H
+#define _NTFS_ATTRLIST_H
+
+#include "attrib.h"
+
+int ntfs_attrlist_need(struct ntfs_inode *ni);
+int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr);
+int ntfs_attrlist_entry_rm(struct ntfs_attr_search_ctx *ctx);
+int ntfs_attrlist_update(struct ntfs_inode *base_ni);
+
+#endif /* defined _NTFS_ATTRLIST_H */
diff --git a/fs/ntfs/bdev-io.c b/fs/ntfs/bdev-io.c
new file mode 100644
index 0000000000000..67e65c88d6818
--- /dev/null
+++ b/fs/ntfs/bdev-io.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS block device I/O.
+ *
+ * Copyright (c) 2026 LG Electronics Co., Ltd.
+ */
+
+#include <linux/blkdev.h>
+
+#include "ntfs.h"
+
+/*
+ * ntfs_bdev_read - Read data directly from block device using bio
+ * @bdev:	block device to read from
+ * @data:	destination buffer
+ * @start:	starting byte offset on the block device
+ * @size:	number of bytes to read
+ *
+ * Reads @size bytes starting from byte offset @start directly from the block
+ * device using one or more BIOs. This function bypasses the page cache
+ * completely and performs synchronous I/O with REQ_META | REQ_SYNC flags set.
+ *
+ * The @start offset must be sector-aligned (512 bytes). If it is not aligned,
+ * the function will return -EINVAL.
+ *
+ * If the destination buffer @data is not a vmalloc address, it falls back
+ * to the more efficient bdev_rw_virt() helper.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int ntfs_bdev_read(struct block_device *bdev, char *data, loff_t start, size_t size)
+{
+	unsigned int done = 0, added;
+	int error;
+	struct bio *bio;
+	enum req_op op;
+	sector_t sector = start >> SECTOR_SHIFT;
+
+	if (start & (SECTOR_SIZE - 1))
+		return -EINVAL;
+
+	op = REQ_OP_READ | REQ_META | REQ_SYNC;
+	if (!is_vmalloc_addr(data))
+		return bdev_rw_virt(bdev, sector, data, size, op);
+
+	bio = bio_alloc(bdev,
+			bio_max_segs(DIV_ROUND_UP(size, PAGE_SIZE)),
+			op, GFP_KERNEL);
+	bio->bi_iter.bi_sector = sector;
+
+	do {
+		added = bio_add_vmalloc_chunk(bio, data + done, size - done);
+		if (!added) {
+			struct bio	*prev = bio;
+
+			bio = bio_alloc(prev->bi_bdev,
+					bio_max_segs(DIV_ROUND_UP(size - done, PAGE_SIZE)),
+					prev->bi_opf, GFP_KERNEL);
+			bio->bi_iter.bi_sector = bio_end_sector(prev);
+			bio_chain(prev, bio);
+			submit_bio(prev);
+		}
+		done += added;
+	} while (done < size);
+
+	error = submit_bio_wait(bio);
+	bio_put(bio);
+
+	if (op == REQ_OP_READ)
+		invalidate_kernel_vmap_range(data, size);
+	return error;
+}
+
+/*
+ * ntfs_bdev_write - Update block device contents via page cache
+ * @sb:		super block of the mounted NTFS filesystem
+ * @buf:	source buffer containing data to write
+ * @start:	starting byte offset on the block device
+ * @size:	number of bytes to write
+ *
+ * Writes @size bytes from @buf to the block device (sb->s_bdev) starting
+ * at byte offset @start. The write is performed entirely through the page
+ * cache of the block device's address space.
+ */
+int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size)
+{
+	pgoff_t idx, idx_end;
+	loff_t offset, end = start + size;
+	u32 from, to, buf_off = 0;
+	struct folio *folio;
+
+	idx = start >> PAGE_SHIFT;
+	idx_end = end >> PAGE_SHIFT;
+	from = start & ~PAGE_MASK;
+
+	if (idx == idx_end)
+		idx_end++;
+
+	for (; idx < idx_end; idx++, from = 0) {
+		folio = read_mapping_folio(sb->s_bdev->bd_mapping, idx, NULL);
+		if (IS_ERR(folio)) {
+			ntfs_error(sb, "Unable to read %ld page", idx);
+			return PTR_ERR(folio);
+		}
+
+		offset = (loff_t)idx << PAGE_SHIFT;
+		to = min_t(u32, end - offset, PAGE_SIZE);
+
+		memcpy_to_folio(folio, from, buf + buf_off, to);
+		buf_off += to;
+		folio_mark_uptodate(folio);
+		folio_mark_dirty(folio);
+		folio_put(folio);
+	}
+
+	return 0;
+}
diff --git a/fs/ntfs/bitmap.c b/fs/ntfs/bitmap.c
new file mode 100644
index 0000000000000..656d802333e35
--- /dev/null
+++ b/fs/ntfs/bitmap.c
@@ -0,0 +1,287 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel bitmap handling.
+ *
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/bitops.h>
+#include <linux/blkdev.h>
+
+#include "bitmap.h"
+#include "ntfs.h"
+
+int ntfs_trim_fs(struct ntfs_volume *vol, struct fstrim_range *range)
+{
+	size_t buf_clusters;
+	pgoff_t index, start_index, end_index;
+	struct file_ra_state *ra;
+	struct folio *folio;
+	unsigned long *bitmap;
+	char *kaddr;
+	u64 end, trimmed = 0, start_buf, end_buf, end_cluster;
+	u64 start_cluster = ntfs_bytes_to_cluster(vol, range->start);
+	u32 dq = bdev_discard_granularity(vol->sb->s_bdev);
+	int ret = 0;
+
+	if (!dq)
+		dq = vol->cluster_size;
+
+	if (start_cluster >= vol->nr_clusters)
+		return -EINVAL;
+
+	if (range->len == (u64)-1)
+		end_cluster = vol->nr_clusters;
+	else {
+		end_cluster = ntfs_bytes_to_cluster(vol,
+				(range->start + range->len + vol->cluster_size - 1));
+		if (end_cluster > vol->nr_clusters)
+			end_cluster = vol->nr_clusters;
+	}
+
+	ra = kzalloc(sizeof(*ra), GFP_NOFS);
+	if (!ra)
+		return -ENOMEM;
+
+	buf_clusters = PAGE_SIZE * 8;
+	start_index = start_cluster >> 15;
+	end_index = (end_cluster + buf_clusters - 1) >> 15;
+
+	for (index = start_index; index < end_index; index++) {
+		folio = ntfs_get_locked_folio(vol->lcnbmp_ino->i_mapping,
+				index, end_index, ra);
+		if (IS_ERR(folio)) {
+			ret = PTR_ERR(folio);
+			goto out_free;
+		}
+
+		kaddr = kmap_local_folio(folio, 0);
+		bitmap = (unsigned long *)kaddr;
+
+		start_buf = max_t(u64, index * buf_clusters, start_cluster);
+		end_buf = min_t(u64, (index + 1) * buf_clusters, end_cluster);
+
+		end = start_buf;
+		while (end < end_buf) {
+			u64 aligned_start, aligned_count;
+			u64 start = find_next_zero_bit(bitmap, end_buf - start_buf,
+					end - start_buf) + start_buf;
+			if (start >= end_buf)
+				break;
+
+			end = find_next_bit(bitmap, end_buf - start_buf,
+					start - start_buf) + start_buf;
+
+			aligned_start = ALIGN(ntfs_cluster_to_bytes(vol, start), dq);
+			aligned_count =
+				ALIGN_DOWN(ntfs_cluster_to_bytes(vol, end - start), dq);
+			if (aligned_count >= range->minlen) {
+				ret = blkdev_issue_discard(vol->sb->s_bdev, aligned_start >> 9,
+						aligned_count >> 9, GFP_NOFS);
+				if (ret)
+					goto out_unmap;
+				trimmed += aligned_count;
+			}
+		}
+
+out_unmap:
+		kunmap_local(kaddr);
+		folio_unlock(folio);
+		folio_put(folio);
+
+		if (ret)
+			goto out_free;
+	}
+
+	range->len = trimmed;
+
+out_free:
+	kfree(ra);
+	return ret;
+}
+
+/*
+ * __ntfs_bitmap_set_bits_in_run - set a run of bits in a bitmap to a value
+ * @vi:			vfs inode describing the bitmap
+ * @start_bit:		first bit to set
+ * @count:		number of bits to set
+ * @value:		value to set the bits to (i.e. 0 or 1)
+ * @is_rollback:	if 'true' this is a rollback operation
+ *
+ * Set @count bits starting at bit @start_bit in the bitmap described by the
+ * vfs inode @vi to @value, where @value is either 0 or 1.
+ *
+ * @is_rollback should always be 'false', it is for internal use to rollback
+ * errors.  You probably want to use ntfs_bitmap_set_bits_in_run() instead.
+ *
+ * Return 0 on success and -errno on error.
+ */
+int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
+		const s64 count, const u8 value, const bool is_rollback)
+{
+	s64 cnt = count;
+	pgoff_t index, end_index;
+	struct address_space *mapping;
+	struct folio *folio;
+	u8 *kaddr;
+	int pos, len;
+	u8 bit;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+
+	ntfs_debug("Entering for i_ino 0x%llx, start_bit 0x%llx, count 0x%llx, value %u.%s",
+			ni->mft_no, (unsigned long long)start_bit,
+			(unsigned long long)cnt, (unsigned int)value,
+			is_rollback ? " (rollback)" : "");
+
+	if (start_bit < 0 || cnt < 0 || value > 1)
+		return -EINVAL;
+
+	/*
+	 * Calculate the indices for the pages containing the first and last
+	 * bits, i.e. @start_bit and @start_bit + @cnt - 1, respectively.
+	 */
+	index = start_bit >> (3 + PAGE_SHIFT);
+	end_index = (start_bit + cnt - 1) >> (3 + PAGE_SHIFT);
+
+	/* Get the page containing the first bit (@start_bit). */
+	mapping = vi->i_mapping;
+	folio = read_mapping_folio(mapping, index, NULL);
+	if (IS_ERR(folio)) {
+		if (!is_rollback)
+			ntfs_error(vi->i_sb,
+				"Failed to map first page (error %li), aborting.",
+				PTR_ERR(folio));
+		return PTR_ERR(folio);
+	}
+
+	folio_lock(folio);
+	kaddr = kmap_local_folio(folio, 0);
+
+	/* Set @pos to the position of the byte containing @start_bit. */
+	pos = (start_bit >> 3) & ~PAGE_MASK;
+
+	/* Calculate the position of @start_bit in the first byte. */
+	bit = start_bit & 7;
+
+	/* If the first byte is partial, modify the appropriate bits in it. */
+	if (bit) {
+		u8 *byte = kaddr + pos;
+
+		if (ni->mft_no == FILE_Bitmap)
+			ntfs_set_lcn_empty_bits(vol, index, value, min_t(s64, 8 - bit, cnt));
+		while ((bit & 7) && cnt) {
+			cnt--;
+			if (value)
+				*byte |= 1 << bit++;
+			else
+				*byte &= ~(1 << bit++);
+		}
+		/* If we are done, unmap the page and return success. */
+		if (!cnt)
+			goto done;
+
+		/* Update @pos to the new position. */
+		pos++;
+	}
+	/*
+	 * Depending on @value, modify all remaining whole bytes in the page up
+	 * to @cnt.
+	 */
+	len = min_t(s64, cnt >> 3, PAGE_SIZE - pos);
+	memset(kaddr + pos, value ? 0xff : 0, len);
+	cnt -= len << 3;
+	if (ni->mft_no == FILE_Bitmap)
+		ntfs_set_lcn_empty_bits(vol, index, value, len << 3);
+
+	/* Update @len to point to the first not-done byte in the page. */
+	if (cnt < 8)
+		len += pos;
+
+	/* If we are not in the last page, deal with all subsequent pages. */
+	while (index < end_index) {
+		if (cnt <= 0)
+			goto rollback;
+
+		/* Update @index and get the next folio. */
+		folio_mark_dirty(folio);
+		folio_unlock(folio);
+		kunmap_local(kaddr);
+		folio_put(folio);
+		folio = read_mapping_folio(mapping, ++index, NULL);
+		if (IS_ERR(folio)) {
+			ntfs_error(vi->i_sb,
+				   "Failed to map subsequent page (error %li), aborting.",
+				   PTR_ERR(folio));
+			goto rollback;
+		}
+
+		folio_lock(folio);
+		kaddr = kmap_local_folio(folio, 0);
+		/*
+		 * Depending on @value, modify all remaining whole bytes in the
+		 * page up to @cnt.
+		 */
+		len = min_t(s64, cnt >> 3, PAGE_SIZE);
+		memset(kaddr, value ? 0xff : 0, len);
+		cnt -= len << 3;
+		if (ni->mft_no == FILE_Bitmap)
+			ntfs_set_lcn_empty_bits(vol, index, value, len << 3);
+	}
+	/*
+	 * The currently mapped page is the last one.  If the last byte is
+	 * partial, modify the appropriate bits in it.  Note, @len is the
+	 * position of the last byte inside the page.
+	 */
+	if (cnt) {
+		u8 *byte;
+
+		WARN_ON(cnt > 7);
+
+		bit = cnt;
+		byte = kaddr + len;
+		if (ni->mft_no == FILE_Bitmap)
+			ntfs_set_lcn_empty_bits(vol, index, value, bit);
+		while (bit--) {
+			if (value)
+				*byte |= 1 << bit;
+			else
+				*byte &= ~(1 << bit);
+		}
+	}
+done:
+	/* We are done.  Unmap the folio and return success. */
+	folio_mark_dirty(folio);
+	folio_unlock(folio);
+	kunmap_local(kaddr);
+	folio_put(folio);
+	ntfs_debug("Done.");
+	return 0;
+rollback:
+	/*
+	 * Current state:
+	 *	- no pages are mapped
+	 *	- @count - @cnt is the number of bits that have been modified
+	 */
+	if (is_rollback)
+		return PTR_ERR(folio);
+	if (count != cnt)
+		pos = __ntfs_bitmap_set_bits_in_run(vi, start_bit, count - cnt,
+				value ? 0 : 1, true);
+	else
+		pos = 0;
+	if (!pos) {
+		/* Rollback was successful. */
+		ntfs_error(vi->i_sb,
+			"Failed to map subsequent page (error %li), aborting.",
+			PTR_ERR(folio));
+	} else {
+		/* Rollback failed. */
+		ntfs_error(vi->i_sb,
+			"Failed to map subsequent page (error %li) and rollback failed (error %i). Aborting and leaving inconsistent metadata. Unmount and run chkdsk.",
+			PTR_ERR(folio), pos);
+		NVolSetErrors(NTFS_SB(vi->i_sb));
+	}
+	return PTR_ERR(folio);
+}
diff --git a/fs/ntfs/bitmap.h b/fs/ntfs/bitmap.h
new file mode 100644
index 0000000000000..31177fe61da69
--- /dev/null
+++ b/fs/ntfs/bitmap.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS kernel bitmap handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_BITMAP_H
+#define _LINUX_NTFS_BITMAP_H
+
+#include <linux/fs.h>
+
+#include "volume.h"
+
+int ntfs_trim_fs(struct ntfs_volume *vol, struct fstrim_range *range);
+int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
+		const s64 count, const u8 value, const bool is_rollback);
+
+/*
+ * ntfs_bitmap_set_bits_in_run - set a run of bits in a bitmap to a value
+ * @vi:			vfs inode describing the bitmap
+ * @start_bit:		first bit to set
+ * @count:		number of bits to set
+ * @value:		value to set the bits to (i.e. 0 or 1)
+ *
+ * Set @count bits starting at bit @start_bit in the bitmap described by the
+ * vfs inode @vi to @value, where @value is either 0 or 1.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static inline int ntfs_bitmap_set_bits_in_run(struct inode *vi,
+		const s64 start_bit, const s64 count, const u8 value)
+{
+	return __ntfs_bitmap_set_bits_in_run(vi, start_bit, count, value,
+			false);
+}
+
+/*
+ * ntfs_bitmap_set_run - set a run of bits in a bitmap
+ * @vi:		vfs inode describing the bitmap
+ * @start_bit:	first bit to set
+ * @count:	number of bits to set
+ *
+ * Set @count bits starting at bit @start_bit in the bitmap described by the
+ * vfs inode @vi.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static inline int ntfs_bitmap_set_run(struct inode *vi, const s64 start_bit,
+		const s64 count)
+{
+	return ntfs_bitmap_set_bits_in_run(vi, start_bit, count, 1);
+}
+
+/*
+ * ntfs_bitmap_clear_run - clear a run of bits in a bitmap
+ * @vi:		vfs inode describing the bitmap
+ * @start_bit:	first bit to clear
+ * @count:	number of bits to clear
+ *
+ * Clear @count bits starting at bit @start_bit in the bitmap described by the
+ * vfs inode @vi.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static inline int ntfs_bitmap_clear_run(struct inode *vi, const s64 start_bit,
+		const s64 count)
+{
+	return ntfs_bitmap_set_bits_in_run(vi, start_bit, count, 0);
+}
+
+/*
+ * ntfs_bitmap_set_bit - set a bit in a bitmap
+ * @vi:		vfs inode describing the bitmap
+ * @bit:	bit to set
+ *
+ * Set bit @bit in the bitmap described by the vfs inode @vi.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static inline int ntfs_bitmap_set_bit(struct inode *vi, const s64 bit)
+{
+	return ntfs_bitmap_set_run(vi, bit, 1);
+}
+
+/*
+ * ntfs_bitmap_clear_bit - clear a bit in a bitmap
+ * @vi:		vfs inode describing the bitmap
+ * @bit:	bit to clear
+ *
+ * Clear bit @bit in the bitmap described by the vfs inode @vi.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static inline int ntfs_bitmap_clear_bit(struct inode *vi, const s64 bit)
+{
+	return ntfs_bitmap_clear_run(vi, bit, 1);
+}
+
+#endif /* defined _LINUX_NTFS_BITMAP_H */
diff --git a/fs/ntfs/collate.c b/fs/ntfs/collate.c
new file mode 100644
index 0000000000000..77e34038902d1
--- /dev/null
+++ b/fs/ntfs/collate.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel collation handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2004 Anton Altaparmakov
+ * Copyright (c) 2005 Yura Pakhuchiy
+ */
+
+#include "collate.h"
+#include "debug.h"
+#include "ntfs.h"
+
+#include <linux/sort.h>
+
+static int ntfs_collate_binary(struct ntfs_volume *vol,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len)
+{
+	int rc;
+
+	rc = memcmp(data1, data2, min(data1_len, data2_len));
+	if (!rc && (data1_len != data2_len)) {
+		if (data1_len < data2_len)
+			rc = -1;
+		else
+			rc = 1;
+	}
+	return rc;
+}
+
+static int ntfs_collate_ntofs_ulong(struct ntfs_volume *vol,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len)
+{
+	int rc;
+	u32 d1 = le32_to_cpup(data1), d2 = le32_to_cpup(data2);
+
+	if (data1_len != data2_len || data1_len != 4)
+		return -EINVAL;
+
+	if (d1 < d2)
+		rc = -1;
+	else {
+		if (d1 == d2)
+			rc = 0;
+		else
+			rc = 1;
+	}
+	return rc;
+}
+
+/*
+ * ntfs_collate_ntofs_ulongs - Which of two le32 arrays should be listed first
+ * @vol: ntfs volume
+ * @data1: first ulong array to collate
+ * @data1_len: length in bytes of @data1
+ * @data2: second ulong array to collate
+ * @data2_len: length in bytes of @data2
+ *
+ * Returns: -1, 0 or 1 depending of how the arrays compare
+ */
+static int ntfs_collate_ntofs_ulongs(struct ntfs_volume *vol,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len)
+{
+	int len;
+	const __le32 *p1 = data1, *p2 = data2;
+	u32 d1, d2;
+
+	if (data1_len != data2_len || data1_len & 3) {
+		ntfs_error(vol->sb, "data1_len or data2_len not valid\n");
+		return -1;
+	}
+
+	len = data1_len;
+	do {
+		d1 = le32_to_cpup(p1);
+		p1++;
+		d2 = le32_to_cpup(p2);
+		p2++;
+	} while (d1 == d2 && (len -= 4) > 0);
+	return cmp_int(d1, d2);
+}
+
+/*
+ * ntfs_collate_file_name - Which of two filenames should be listed first
+ * @vol: ntfs volume
+ * @data1: first filename to collate
+ * @data1_len: length in bytes of @data1(unused)
+ * @data2: second filename to collate
+ * @data2_len: length in bytes of @data2(unused)
+ */
+static int ntfs_collate_file_name(struct ntfs_volume *vol,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len)
+{
+	int rc;
+
+	rc = ntfs_file_compare_values(data1, data2, -EINVAL,
+			IGNORE_CASE, vol->upcase, vol->upcase_len);
+	if (!rc)
+		rc = ntfs_file_compare_values(data1, data2,
+			-EINVAL, CASE_SENSITIVE, vol->upcase, vol->upcase_len);
+	return rc;
+}
+
+/*
+ * ntfs_collate - collate two data items using a specified collation rule
+ * @vol:	ntfs volume to which the data items belong
+ * @cr:		collation rule to use when comparing the items
+ * @data1:	first data item to collate
+ * @data1_len:	length in bytes of @data1
+ * @data2:	second data item to collate
+ * @data2_len:	length in bytes of @data2
+ *
+ * Collate the two data items @data1 and @data2 using the collation rule @cr
+ * and return -1, 0, ir 1 if @data1 is found, respectively, to collate before,
+ * to match, or to collate after @data2. return -EINVAL if an error occurred.
+ */
+int ntfs_collate(struct ntfs_volume *vol, __le32 cr,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len)
+{
+	switch (le32_to_cpu(cr)) {
+	case le32_to_cpu(COLLATION_BINARY):
+		return ntfs_collate_binary(vol, data1, data1_len,
+					   data2, data2_len);
+	case le32_to_cpu(COLLATION_FILE_NAME):
+		return ntfs_collate_file_name(vol, data1, data1_len,
+					      data2, data2_len);
+	case le32_to_cpu(COLLATION_NTOFS_ULONG):
+		return ntfs_collate_ntofs_ulong(vol, data1, data1_len,
+						data2, data2_len);
+	case le32_to_cpu(COLLATION_NTOFS_ULONGS):
+		return ntfs_collate_ntofs_ulongs(vol, data1, data1_len,
+						 data2, data2_len);
+	default:
+		ntfs_error(vol->sb, "Unknown collation rule 0x%x",
+			   le32_to_cpu(cr));
+		return -EINVAL;
+	}
+}
diff --git a/fs/ntfs/collate.h b/fs/ntfs/collate.h
new file mode 100644
index 0000000000000..8ad1f04be675c
--- /dev/null
+++ b/fs/ntfs/collate.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS kernel collation handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2004 Anton Altaparmakov
+ * Copyright (c) 2005 Yura Pakhuchiy
+ */
+
+#ifndef _LINUX_NTFS_COLLATE_H
+#define _LINUX_NTFS_COLLATE_H
+
+#include "volume.h"
+
+static inline bool ntfs_is_collation_rule_supported(__le32 cr)
+{
+	int i;
+
+	if (unlikely(cr != COLLATION_BINARY && cr != COLLATION_NTOFS_ULONG &&
+		     cr != COLLATION_FILE_NAME) && cr != COLLATION_NTOFS_ULONGS)
+		return false;
+	i = le32_to_cpu(cr);
+	if (likely(((i >= 0) && (i <= 0x02)) ||
+			((i >= 0x10) && (i <= 0x13))))
+		return true;
+	return false;
+}
+
+int ntfs_collate(struct ntfs_volume *vol, __le32 cr,
+		const void *data1, const u32 data1_len,
+		const void *data2, const u32 data2_len);
+
+#endif /* _LINUX_NTFS_COLLATE_H */
diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c
new file mode 100644
index 0000000000000..71a8d9c42674f
--- /dev/null
+++ b/fs/ntfs/compress.c
@@ -0,0 +1,1577 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel compressed attributes handling.
+ *
+ * Copyright (c) 2001-2004 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2004-2006 Szabolcs Szakacsits
+ * Copyright (c)      2005 Yura Pakhuchiy
+ * Copyright (c) 2009-2014 Jean-Pierre Andre
+ * Copyright (c)      2014 Eric Biggers
+ */
+
+#include <linux/fs.h>
+#include <linux/blkdev.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+
+#include "attrib.h"
+#include "inode.h"
+#include "debug.h"
+#include "ntfs.h"
+#include "lcnalloc.h"
+#include "mft.h"
+
+/*
+ * Constants used in the compression code
+ */
+enum {
+	/* Token types and access mask. */
+	NTFS_SYMBOL_TOKEN	=	0,
+	NTFS_PHRASE_TOKEN	=	1,
+	NTFS_TOKEN_MASK		=	1,
+
+	/* Compression sub-block constants. */
+	NTFS_SB_SIZE_MASK	=	0x0fff,
+	NTFS_SB_SIZE		=	0x1000,
+	NTFS_SB_IS_COMPRESSED	=	0x8000,
+
+	/*
+	 * The maximum compression block size is by definition 16 * the cluster
+	 * size, with the maximum supported cluster size being 4kiB. Thus the
+	 * maximum compression buffer size is 64kiB, so we use this when
+	 * initializing the compression buffer.
+	 */
+	NTFS_MAX_CB_SIZE	= 64 * 1024,
+};
+
+/*
+ * ntfs_compression_buffer - one buffer for the decompression engine
+ */
+static u8 *ntfs_compression_buffer;
+
+/*
+ * ntfs_cb_lock - mutex lock which protects ntfs_compression_buffer
+ */
+static DEFINE_MUTEX(ntfs_cb_lock);
+
+/*
+ * allocate_compression_buffers - allocate the decompression buffers
+ *
+ * Caller has to hold the ntfs_lock mutex.
+ *
+ * Return 0 on success or -ENOMEM if the allocations failed.
+ */
+int allocate_compression_buffers(void)
+{
+	if (ntfs_compression_buffer)
+		return 0;
+
+	ntfs_compression_buffer = vmalloc(NTFS_MAX_CB_SIZE);
+	if (!ntfs_compression_buffer)
+		return -ENOMEM;
+	return 0;
+}
+
+/*
+ * free_compression_buffers - free the decompression buffers
+ *
+ * Caller has to hold the ntfs_lock mutex.
+ */
+void free_compression_buffers(void)
+{
+	mutex_lock(&ntfs_cb_lock);
+	if (!ntfs_compression_buffer) {
+		mutex_unlock(&ntfs_cb_lock);
+		return;
+	}
+
+	vfree(ntfs_compression_buffer);
+	ntfs_compression_buffer = NULL;
+	mutex_unlock(&ntfs_cb_lock);
+}
+
+/*
+ * zero_partial_compressed_page - zero out of bounds compressed page region
+ * @page: page to zero
+ * @initialized_size: initialized size of the attribute
+ */
+static void zero_partial_compressed_page(struct page *page,
+		const s64 initialized_size)
+{
+	u8 *kp = page_address(page);
+	unsigned int kp_ofs;
+
+	ntfs_debug("Zeroing page region outside initialized size.");
+	if (((s64)page->__folio_index << PAGE_SHIFT) >= initialized_size) {
+		clear_page(kp);
+		return;
+	}
+	kp_ofs = initialized_size & ~PAGE_MASK;
+	memset(kp + kp_ofs, 0, PAGE_SIZE - kp_ofs);
+}
+
+/*
+ * handle_bounds_compressed_page - test for&handle out of bounds compressed page
+ * @page: page to check and handle
+ * @i_size: file size
+ * @initialized_size: initialized size of the attribute
+ */
+static inline void handle_bounds_compressed_page(struct page *page,
+		const loff_t i_size, const s64 initialized_size)
+{
+	if ((page->__folio_index >= (initialized_size >> PAGE_SHIFT)) &&
+			(initialized_size < i_size))
+		zero_partial_compressed_page(page, initialized_size);
+}
+
+/*
+ * ntfs_decompress - decompress a compression block into an array of pages
+ * @dest_pages:		destination array of pages
+ * @completed_pages:	scratch space to track completed pages
+ * @dest_index:		current index into @dest_pages (IN/OUT)
+ * @dest_ofs:		current offset within @dest_pages[@dest_index] (IN/OUT)
+ * @dest_max_index:	maximum index into @dest_pages (IN)
+ * @dest_max_ofs:	maximum offset within @dest_pages[@dest_max_index] (IN)
+ * @xpage:		the target page (-1 if none) (IN)
+ * @xpage_done:		set to 1 if xpage was completed successfully (IN/OUT)
+ * @cb_start:		compression block to decompress (IN)
+ * @cb_size:		size of compression block @cb_start in bytes (IN)
+ * @i_size:		file size when we started the read (IN)
+ * @initialized_size:	initialized file size when we started the read (IN)
+ *
+ * The caller must have disabled preemption. ntfs_decompress() reenables it when
+ * the critical section is finished.
+ *
+ * This decompresses the compression block @cb_start into the array of
+ * destination pages @dest_pages starting at index @dest_index into @dest_pages
+ * and at offset @dest_pos into the page @dest_pages[@dest_index].
+ *
+ * When the page @dest_pages[@xpage] is completed, @xpage_done is set to 1.
+ * If xpage is -1 or @xpage has not been completed, @xpage_done is not modified.
+ *
+ * @cb_start is a pointer to the compression block which needs decompressing
+ * and @cb_size is the size of @cb_start in bytes (8-64kiB).
+ *
+ * Return 0 if success or -EOVERFLOW on error in the compressed stream.
+ * @xpage_done indicates whether the target page (@dest_pages[@xpage]) was
+ * completed during the decompression of the compression block (@cb_start).
+ *
+ * Warning: This function *REQUIRES* PAGE_SIZE >= 4096 or it will blow up
+ * unpredicatbly! You have been warned!
+ *
+ * Note to hackers: This function may not sleep until it has finished accessing
+ * the compression block @cb_start as it is a per-CPU buffer.
+ */
+static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
+		int *dest_index, int *dest_ofs, const int dest_max_index,
+		const int dest_max_ofs, const int xpage, char *xpage_done,
+		u8 *const cb_start, const u32 cb_size, const loff_t i_size,
+		const s64 initialized_size)
+{
+	/*
+	 * Pointers into the compressed data, i.e. the compression block (cb),
+	 * and the therein contained sub-blocks (sb).
+	 */
+	u8 *cb_end = cb_start + cb_size; /* End of cb. */
+	u8 *cb = cb_start;	/* Current position in cb. */
+	u8 *cb_sb_start = cb;	/* Beginning of the current sb in the cb. */
+	u8 *cb_sb_end;		/* End of current sb / beginning of next sb. */
+
+	/* Variables for uncompressed data / destination. */
+	struct page *dp;	/* Current destination page being worked on. */
+	u8 *dp_addr;		/* Current pointer into dp. */
+	u8 *dp_sb_start;	/* Start of current sub-block in dp. */
+	u8 *dp_sb_end;		/* End of current sb in dp (dp_sb_start + NTFS_SB_SIZE). */
+	u16 do_sb_start;	/* @dest_ofs when starting this sub-block. */
+	u16 do_sb_end;		/* @dest_ofs of end of this sb (do_sb_start + NTFS_SB_SIZE). */
+
+	/* Variables for tag and token parsing. */
+	u8 tag;			/* Current tag. */
+	int token;		/* Loop counter for the eight tokens in tag. */
+	int nr_completed_pages = 0;
+
+	/* Default error code. */
+	int err = -EOVERFLOW;
+
+	ntfs_debug("Entering, cb_size = 0x%x.", cb_size);
+do_next_sb:
+	ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.",
+			cb - cb_start);
+	/*
+	 * Have we reached the end of the compression block or the end of the
+	 * decompressed data?  The latter can happen for example if the current
+	 * position in the compression block is one byte before its end so the
+	 * first two checks do not detect it.
+	 */
+	if (cb == cb_end || !le16_to_cpup((__le16 *)cb) ||
+			(*dest_index == dest_max_index &&
+			*dest_ofs == dest_max_ofs)) {
+		int i;
+
+		ntfs_debug("Completed. Returning success (0).");
+		err = 0;
+return_error:
+		/* We can sleep from now on, so we drop lock. */
+		mutex_unlock(&ntfs_cb_lock);
+		/* Second stage: finalize completed pages. */
+		if (nr_completed_pages > 0) {
+			for (i = 0; i < nr_completed_pages; i++) {
+				int di = completed_pages[i];
+
+				dp = dest_pages[di];
+				/*
+				 * If we are outside the initialized size, zero
+				 * the out of bounds page range.
+				 */
+				handle_bounds_compressed_page(dp, i_size,
+						initialized_size);
+				flush_dcache_page(dp);
+				kunmap_local(page_address(dp));
+				SetPageUptodate(dp);
+				unlock_page(dp);
+				if (di == xpage)
+					*xpage_done = 1;
+				else
+					put_page(dp);
+				dest_pages[di] = NULL;
+			}
+		}
+		return err;
+	}
+
+	/* Setup offsets for the current sub-block destination. */
+	do_sb_start = *dest_ofs;
+	do_sb_end = do_sb_start + NTFS_SB_SIZE;
+
+	/* Check that we are still within allowed boundaries. */
+	if (*dest_index == dest_max_index && do_sb_end > dest_max_ofs)
+		goto return_overflow;
+
+	/* Does the minimum size of a compressed sb overflow valid range? */
+	if (cb + 6 > cb_end)
+		goto return_overflow;
+
+	/* Setup the current sub-block source pointers and validate range. */
+	cb_sb_start = cb;
+	cb_sb_end = cb_sb_start + (le16_to_cpup((__le16 *)cb) & NTFS_SB_SIZE_MASK)
+			+ 3;
+	if (cb_sb_end > cb_end)
+		goto return_overflow;
+
+	/* Get the current destination page. */
+	dp = dest_pages[*dest_index];
+	if (!dp) {
+		/* No page present. Skip decompression of this sub-block. */
+		cb = cb_sb_end;
+
+		/* Advance destination position to next sub-block. */
+		*dest_ofs = (*dest_ofs + NTFS_SB_SIZE) & ~PAGE_MASK;
+		if (!*dest_ofs && (++*dest_index > dest_max_index))
+			goto return_overflow;
+		goto do_next_sb;
+	}
+
+	/* We have a valid destination page. Setup the destination pointers. */
+	dp_addr = (u8 *)page_address(dp) + do_sb_start;
+
+	/* Now, we are ready to process the current sub-block (sb). */
+	if (!(le16_to_cpup((__le16 *)cb) & NTFS_SB_IS_COMPRESSED)) {
+		ntfs_debug("Found uncompressed sub-block.");
+		/* This sb is not compressed, just copy it into destination. */
+
+		/* Advance source position to first data byte. */
+		cb += 2;
+
+		/* An uncompressed sb must be full size. */
+		if (cb_sb_end - cb != NTFS_SB_SIZE)
+			goto return_overflow;
+
+		/* Copy the block and advance the source position. */
+		memcpy(dp_addr, cb, NTFS_SB_SIZE);
+		cb += NTFS_SB_SIZE;
+
+		/* Advance destination position to next sub-block. */
+		*dest_ofs += NTFS_SB_SIZE;
+		*dest_ofs &= ~PAGE_MASK;
+		if (!(*dest_ofs)) {
+finalize_page:
+			/*
+			 * First stage: add current page index to array of
+			 * completed pages.
+			 */
+			completed_pages[nr_completed_pages++] = *dest_index;
+			if (++*dest_index > dest_max_index)
+				goto return_overflow;
+		}
+		goto do_next_sb;
+	}
+	ntfs_debug("Found compressed sub-block.");
+	/* This sb is compressed, decompress it into destination. */
+
+	/* Setup destination pointers. */
+	dp_sb_start = dp_addr;
+	dp_sb_end = dp_sb_start + NTFS_SB_SIZE;
+
+	/* Forward to the first tag in the sub-block. */
+	cb += 2;
+do_next_tag:
+	if (cb == cb_sb_end) {
+		/* Check if the decompressed sub-block was not full-length. */
+		if (dp_addr < dp_sb_end) {
+			int nr_bytes = do_sb_end - *dest_ofs;
+
+			ntfs_debug("Filling incomplete sub-block with zeroes.");
+			/* Zero remainder and update destination position. */
+			memset(dp_addr, 0, nr_bytes);
+			*dest_ofs += nr_bytes;
+		}
+		/* We have finished the current sub-block. */
+		*dest_ofs &= ~PAGE_MASK;
+		if (!(*dest_ofs))
+			goto finalize_page;
+		goto do_next_sb;
+	}
+
+	/* Check we are still in range. */
+	if (cb > cb_sb_end || dp_addr > dp_sb_end)
+		goto return_overflow;
+
+	/* Get the next tag and advance to first token. */
+	tag = *cb++;
+
+	/* Parse the eight tokens described by the tag. */
+	for (token = 0; token < 8; token++, tag >>= 1) {
+		register u16 i;
+		u16 lg, pt, length, max_non_overlap;
+		u8 *dp_back_addr;
+
+		/* Check if we are done / still in range. */
+		if (cb >= cb_sb_end || dp_addr > dp_sb_end)
+			break;
+
+		/* Determine token type and parse appropriately.*/
+		if ((tag & NTFS_TOKEN_MASK) == NTFS_SYMBOL_TOKEN) {
+			/*
+			 * We have a symbol token, copy the symbol across, and
+			 * advance the source and destination positions.
+			 */
+			*dp_addr++ = *cb++;
+			++*dest_ofs;
+
+			/* Continue with the next token. */
+			continue;
+		}
+
+		/*
+		 * We have a phrase token. Make sure it is not the first tag in
+		 * the sb as this is illegal and would confuse the code below.
+		 */
+		if (dp_addr == dp_sb_start)
+			goto return_overflow;
+
+		/*
+		 * Determine the number of bytes to go back (p) and the number
+		 * of bytes to copy (l). We use an optimized algorithm in which
+		 * we first calculate log2(current destination position in sb),
+		 * which allows determination of l and p in O(1) rather than
+		 * O(n). We just need an arch-optimized log2() function now.
+		 */
+		lg = 0;
+		for (i = *dest_ofs - do_sb_start - 1; i >= 0x10; i >>= 1)
+			lg++;
+
+		/* Get the phrase token into i. */
+		pt = le16_to_cpup((__le16 *)cb);
+
+		/*
+		 * Calculate starting position of the byte sequence in
+		 * the destination using the fact that p = (pt >> (12 - lg)) + 1
+		 * and make sure we don't go too far back.
+		 */
+		dp_back_addr = dp_addr - (pt >> (12 - lg)) - 1;
+		if (dp_back_addr < dp_sb_start)
+			goto return_overflow;
+
+		/* Now calculate the length of the byte sequence. */
+		length = (pt & (0xfff >> lg)) + 3;
+
+		/* Advance destination position and verify it is in range. */
+		*dest_ofs += length;
+		if (*dest_ofs > do_sb_end)
+			goto return_overflow;
+
+		/* The number of non-overlapping bytes. */
+		max_non_overlap = dp_addr - dp_back_addr;
+
+		if (length <= max_non_overlap) {
+			/* The byte sequence doesn't overlap, just copy it. */
+			memcpy(dp_addr, dp_back_addr, length);
+
+			/* Advance destination pointer. */
+			dp_addr += length;
+		} else {
+			/*
+			 * The byte sequence does overlap, copy non-overlapping
+			 * part and then do a slow byte by byte copy for the
+			 * overlapping part. Also, advance the destination
+			 * pointer.
+			 */
+			memcpy(dp_addr, dp_back_addr, max_non_overlap);
+			dp_addr += max_non_overlap;
+			dp_back_addr += max_non_overlap;
+			length -= max_non_overlap;
+			while (length--)
+				*dp_addr++ = *dp_back_addr++;
+		}
+
+		/* Advance source position and continue with the next token. */
+		cb += 2;
+	}
+
+	/* No tokens left in the current tag. Continue with the next tag. */
+	goto do_next_tag;
+
+return_overflow:
+	ntfs_error(NULL, "Failed. Returning -EOVERFLOW.");
+	goto return_error;
+}
+
+/*
+ * ntfs_read_compressed_block - read a compressed block into the page cache
+ * @folio:	locked folio in the compression block(s) we need to read
+ *
+ * When we are called the page has already been verified to be locked and the
+ * attribute is known to be non-resident, not encrypted, but compressed.
+ *
+ * 1. Determine which compression block(s) @page is in.
+ * 2. Get hold of all pages corresponding to this/these compression block(s).
+ * 3. Read the (first) compression block.
+ * 4. Decompress it into the corresponding pages.
+ * 5. Throw the compressed data away and proceed to 3. for the next compression
+ *    block or return success if no more compression blocks left.
+ *
+ * Warning: We have to be careful what we do about existing pages. They might
+ * have been written to so that we would lose data if we were to just overwrite
+ * them with the out-of-date uncompressed data.
+ */
+int ntfs_read_compressed_block(struct folio *folio)
+{
+	struct page *page = &folio->page;
+	loff_t i_size;
+	s64 initialized_size;
+	struct address_space *mapping = page->mapping;
+	struct ntfs_inode *ni = NTFS_I(mapping->host);
+	struct ntfs_volume *vol = ni->vol;
+	struct super_block *sb = vol->sb;
+	struct runlist_element *rl;
+	unsigned long flags;
+	u8 *cb, *cb_pos, *cb_end;
+	unsigned long offset, index = page->__folio_index;
+	u32 cb_size = ni->itype.compressed.block_size;
+	u64 cb_size_mask = cb_size - 1UL;
+	s64 vcn;
+	s64 lcn;
+	/* The first wanted vcn (minimum alignment is PAGE_SIZE). */
+	s64 start_vcn = (((s64)index << PAGE_SHIFT) & ~cb_size_mask) >>
+			vol->cluster_size_bits;
+	/*
+	 * The first vcn after the last wanted vcn (minimum alignment is again
+	 * PAGE_SIZE.
+	 */
+	s64 end_vcn = ((((s64)(index + 1UL) << PAGE_SHIFT) + cb_size - 1)
+			& ~cb_size_mask) >> vol->cluster_size_bits;
+	/* Number of compression blocks (cbs) in the wanted vcn range. */
+	unsigned int nr_cbs = ntfs_cluster_to_bytes(vol, end_vcn - start_vcn) >>
+			ni->itype.compressed.block_size_bits;
+	/*
+	 * Number of pages required to store the uncompressed data from all
+	 * compression blocks (cbs) overlapping @page. Due to alignment
+	 * guarantees of start_vcn and end_vcn, no need to round up here.
+	 */
+	unsigned int nr_pages = ntfs_cluster_to_pidx(vol, end_vcn - start_vcn);
+	unsigned int xpage, max_page, cur_page, cur_ofs, i, page_ofs, page_index;
+	unsigned int cb_clusters, cb_max_ofs;
+	int cb_max_page, err = 0;
+	struct page **pages;
+	int *completed_pages;
+	unsigned char xpage_done = 0;
+	struct page *lpage;
+
+	ntfs_debug("Entering, page->index = 0x%lx, cb_size = 0x%x, nr_pages = %i.",
+			index, cb_size, nr_pages);
+	/*
+	 * Bad things happen if we get here for anything that is not an
+	 * unnamed $DATA attribute.
+	 */
+	if (ni->type != AT_DATA || ni->name_len) {
+		unlock_page(page);
+		return -EIO;
+	}
+
+	pages = kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS);
+	completed_pages = kmalloc_array(nr_pages + 1, sizeof(int), GFP_NOFS);
+
+	if (unlikely(!pages || !completed_pages)) {
+		kfree(pages);
+		kfree(completed_pages);
+		unlock_page(page);
+		ntfs_error(vol->sb, "Failed to allocate internal buffers.");
+		return -ENOMEM;
+	}
+
+	/*
+	 * We have already been given one page, this is the one we must do.
+	 * Once again, the alignment guarantees keep it simple.
+	 */
+	offset = ntfs_cluster_to_pidx(vol, start_vcn);
+	xpage = index - offset;
+	pages[xpage] = page;
+	/*
+	 * The remaining pages need to be allocated and inserted into the page
+	 * cache, alignment guarantees keep all the below much simpler. (-8
+	 */
+	read_lock_irqsave(&ni->size_lock, flags);
+	i_size = i_size_read(VFS_I(ni));
+	initialized_size = ni->initialized_size;
+	read_unlock_irqrestore(&ni->size_lock, flags);
+	max_page = ((i_size + PAGE_SIZE - 1) >> PAGE_SHIFT) -
+			offset;
+	/* Is the page fully outside i_size? (truncate in progress) */
+	if (xpage >= max_page) {
+		kfree(pages);
+		kfree(completed_pages);
+		zero_user_segments(page, 0, PAGE_SIZE, 0, 0);
+		ntfs_debug("Compressed read outside i_size - truncated?");
+		SetPageUptodate(page);
+		unlock_page(page);
+		return 0;
+	}
+	if (nr_pages < max_page)
+		max_page = nr_pages;
+
+	for (i = 0; i < max_page; i++, offset++) {
+		if (i != xpage)
+			pages[i] = grab_cache_page_nowait(mapping, offset);
+		page = pages[i];
+		if (page) {
+			/*
+			 * We only (re)read the page if it isn't already read
+			 * in and/or dirty or we would be losing data or at
+			 * least wasting our time.
+			 */
+			if (!PageDirty(page) && (!PageUptodate(page))) {
+				kmap_local_page(page);
+				continue;
+			}
+			unlock_page(page);
+			put_page(page);
+			pages[i] = NULL;
+		}
+	}
+
+	/*
+	 * We have the runlist, and all the destination pages we need to fill.
+	 * Now read the first compression block.
+	 */
+	cur_page = 0;
+	cur_ofs = 0;
+	cb_clusters = ni->itype.compressed.block_clusters;
+do_next_cb:
+	nr_cbs--;
+
+	mutex_lock(&ntfs_cb_lock);
+	if (!ntfs_compression_buffer)
+		if (allocate_compression_buffers()) {
+			mutex_unlock(&ntfs_cb_lock);
+			goto err_out;
+		}
+
+
+	cb = ntfs_compression_buffer;
+	cb_pos = cb;
+	cb_end = cb + cb_size;
+
+	rl = NULL;
+	for (vcn = start_vcn, start_vcn += cb_clusters; vcn < start_vcn;
+			vcn++) {
+		bool is_retry = false;
+
+		if (!rl) {
+lock_retry_remap:
+			down_read(&ni->runlist.lock);
+			rl = ni->runlist.rl;
+		}
+		if (likely(rl != NULL)) {
+			/* Seek to element containing target vcn. */
+			while (rl->length && rl[1].vcn <= vcn)
+				rl++;
+			lcn = ntfs_rl_vcn_to_lcn(rl, vcn);
+		} else
+			lcn = LCN_RL_NOT_MAPPED;
+		ntfs_debug("Reading vcn = 0x%llx, lcn = 0x%llx.",
+				(unsigned long long)vcn,
+				(unsigned long long)lcn);
+		if (lcn < 0) {
+			/*
+			 * When we reach the first sparse cluster we have
+			 * finished with the cb.
+			 */
+			if (lcn == LCN_HOLE)
+				break;
+			if (is_retry || lcn != LCN_RL_NOT_MAPPED) {
+				mutex_unlock(&ntfs_cb_lock);
+				goto rl_err;
+			}
+			is_retry = true;
+			/*
+			 * Attempt to map runlist, dropping lock for the
+			 * duration.
+			 */
+			up_read(&ni->runlist.lock);
+			if (!ntfs_map_runlist(ni, vcn))
+				goto lock_retry_remap;
+			mutex_unlock(&ntfs_cb_lock);
+			goto map_rl_err;
+		}
+
+		page_ofs = ntfs_cluster_to_poff(vol, lcn);
+		page_index = ntfs_cluster_to_pidx(vol, lcn);
+
+		lpage = read_mapping_page(sb->s_bdev->bd_mapping,
+					  page_index, NULL);
+		if (IS_ERR(lpage)) {
+			err = PTR_ERR(lpage);
+			mutex_unlock(&ntfs_cb_lock);
+			goto read_err;
+		}
+
+		lock_page(lpage);
+		memcpy(cb_pos, page_address(lpage) + page_ofs,
+		       vol->cluster_size);
+		unlock_page(lpage);
+		put_page(lpage);
+		cb_pos += vol->cluster_size;
+	}
+
+	/* Release the lock if we took it. */
+	if (rl)
+		up_read(&ni->runlist.lock);
+
+	/* Just a precaution. */
+	if (cb_pos + 2 <= cb + cb_size)
+		*(u16 *)cb_pos = 0;
+
+	/* Reset cb_pos back to the beginning. */
+	cb_pos = cb;
+
+	/* We now have both source (if present) and destination. */
+	ntfs_debug("Successfully read the compression block.");
+
+	/* The last page and maximum offset within it for the current cb. */
+	cb_max_page = (cur_page << PAGE_SHIFT) + cur_ofs + cb_size;
+	cb_max_ofs = cb_max_page & ~PAGE_MASK;
+	cb_max_page >>= PAGE_SHIFT;
+
+	/* Catch end of file inside a compression block. */
+	if (cb_max_page > max_page)
+		cb_max_page = max_page;
+
+	if (vcn == start_vcn - cb_clusters) {
+		/* Sparse cb, zero out page range overlapping the cb. */
+		ntfs_debug("Found sparse compression block.");
+		/* We can sleep from now on, so we drop lock. */
+		mutex_unlock(&ntfs_cb_lock);
+		if (cb_max_ofs)
+			cb_max_page--;
+		for (; cur_page < cb_max_page; cur_page++) {
+			page = pages[cur_page];
+			if (page) {
+				if (likely(!cur_ofs))
+					clear_page(page_address(page));
+				else
+					memset(page_address(page) + cur_ofs, 0,
+							PAGE_SIZE -
+							cur_ofs);
+				flush_dcache_page(page);
+				kunmap_local(page_address(page));
+				SetPageUptodate(page);
+				unlock_page(page);
+				if (cur_page == xpage)
+					xpage_done = 1;
+				else
+					put_page(page);
+				pages[cur_page] = NULL;
+			}
+			cb_pos += PAGE_SIZE - cur_ofs;
+			cur_ofs = 0;
+			if (cb_pos >= cb_end)
+				break;
+		}
+		/* If we have a partial final page, deal with it now. */
+		if (cb_max_ofs && cb_pos < cb_end) {
+			page = pages[cur_page];
+			if (page)
+				memset(page_address(page) + cur_ofs, 0,
+						cb_max_ofs - cur_ofs);
+			/*
+			 * No need to update cb_pos at this stage:
+			 *	cb_pos += cb_max_ofs - cur_ofs;
+			 */
+			cur_ofs = cb_max_ofs;
+		}
+	} else if (vcn == start_vcn) {
+		/* We can't sleep so we need two stages. */
+		unsigned int cur2_page = cur_page;
+		unsigned int cur_ofs2 = cur_ofs;
+		u8 *cb_pos2 = cb_pos;
+
+		ntfs_debug("Found uncompressed compression block.");
+		/* Uncompressed cb, copy it to the destination pages. */
+		if (cb_max_ofs)
+			cb_max_page--;
+		/* First stage: copy data into destination pages. */
+		for (; cur_page < cb_max_page; cur_page++) {
+			page = pages[cur_page];
+			if (page)
+				memcpy(page_address(page) + cur_ofs, cb_pos,
+						PAGE_SIZE - cur_ofs);
+			cb_pos += PAGE_SIZE - cur_ofs;
+			cur_ofs = 0;
+			if (cb_pos >= cb_end)
+				break;
+		}
+		/* If we have a partial final page, deal with it now. */
+		if (cb_max_ofs && cb_pos < cb_end) {
+			page = pages[cur_page];
+			if (page)
+				memcpy(page_address(page) + cur_ofs, cb_pos,
+						cb_max_ofs - cur_ofs);
+			cb_pos += cb_max_ofs - cur_ofs;
+			cur_ofs = cb_max_ofs;
+		}
+		/* We can sleep from now on, so drop lock. */
+		mutex_unlock(&ntfs_cb_lock);
+		/* Second stage: finalize pages. */
+		for (; cur2_page < cb_max_page; cur2_page++) {
+			page = pages[cur2_page];
+			if (page) {
+				/*
+				 * If we are outside the initialized size, zero
+				 * the out of bounds page range.
+				 */
+				handle_bounds_compressed_page(page, i_size,
+						initialized_size);
+				flush_dcache_page(page);
+				kunmap_local(page_address(page));
+				SetPageUptodate(page);
+				unlock_page(page);
+				if (cur2_page == xpage)
+					xpage_done = 1;
+				else
+					put_page(page);
+				pages[cur2_page] = NULL;
+			}
+			cb_pos2 += PAGE_SIZE - cur_ofs2;
+			cur_ofs2 = 0;
+			if (cb_pos2 >= cb_end)
+				break;
+		}
+	} else {
+		/* Compressed cb, decompress it into the destination page(s). */
+		unsigned int prev_cur_page = cur_page;
+
+		ntfs_debug("Found compressed compression block.");
+		err = ntfs_decompress(pages, completed_pages, &cur_page,
+				&cur_ofs, cb_max_page, cb_max_ofs, xpage,
+				&xpage_done, cb_pos, cb_size - (cb_pos - cb),
+				i_size, initialized_size);
+		/*
+		 * We can sleep from now on, lock already dropped by
+		 * ntfs_decompress().
+		 */
+		if (err) {
+			ntfs_error(vol->sb,
+				"ntfs_decompress() failed in inode 0x%llx with error code %i. Skipping this compression block.",
+				ni->mft_no, -err);
+			/* Release the unfinished pages. */
+			for (; prev_cur_page < cur_page; prev_cur_page++) {
+				page = pages[prev_cur_page];
+				if (page) {
+					flush_dcache_page(page);
+					kunmap_local(page_address(page));
+					unlock_page(page);
+					if (prev_cur_page != xpage)
+						put_page(page);
+					pages[prev_cur_page] = NULL;
+				}
+			}
+		}
+	}
+
+	/* Do we have more work to do? */
+	if (nr_cbs)
+		goto do_next_cb;
+
+	/* Clean up if we have any pages left. Should never happen. */
+	for (cur_page = 0; cur_page < max_page; cur_page++) {
+		page = pages[cur_page];
+		if (page) {
+			ntfs_error(vol->sb,
+				"Still have pages left! Terminating them with extreme prejudice.  Inode 0x%llx, page index 0x%lx.",
+				ni->mft_no, page->__folio_index);
+			flush_dcache_page(page);
+			kunmap_local(page_address(page));
+			unlock_page(page);
+			if (cur_page != xpage)
+				put_page(page);
+			pages[cur_page] = NULL;
+		}
+	}
+
+	/* We no longer need the list of pages. */
+	kfree(pages);
+	kfree(completed_pages);
+
+	/* If we have completed the requested page, we return success. */
+	if (likely(xpage_done))
+		return 0;
+
+	ntfs_debug("Failed. Returning error code %s.", err == -EOVERFLOW ?
+			"EOVERFLOW" : (!err ? "EIO" : "unknown error"));
+	return err < 0 ? err : -EIO;
+
+map_rl_err:
+	ntfs_error(vol->sb, "ntfs_map_runlist() failed. Cannot read compression block.");
+	goto err_out;
+
+rl_err:
+	up_read(&ni->runlist.lock);
+	ntfs_error(vol->sb, "ntfs_rl_vcn_to_lcn() failed. Cannot read compression block.");
+	goto err_out;
+
+read_err:
+	up_read(&ni->runlist.lock);
+	ntfs_error(vol->sb, "IO error while reading compressed data.");
+
+err_out:
+	for (i = cur_page; i < max_page; i++) {
+		page = pages[i];
+		if (page) {
+			flush_dcache_page(page);
+			kunmap_local(page_address(page));
+			unlock_page(page);
+			if (i != xpage)
+				put_page(page);
+		}
+	}
+	kfree(pages);
+	kfree(completed_pages);
+	return -EIO;
+}
+
+/*
+ * Match length at or above which ntfs_best_match() will stop searching for
+ * longer matches.
+ */
+#define NICE_MATCH_LEN		18
+
+/*
+ * Maximum number of potential matches that ntfs_best_match() will consider at
+ * each position.
+ */
+#define MAX_SEARCH_DEPTH	24
+
+/* log base 2 of the number of entries in the hash table for match-finding.  */
+#define HASH_SHIFT		14
+
+/*
+ * Constant for the multiplicative hash function. These hashing constants
+ * are used solely for the match-finding algorithm during compression.
+ * They are NOT part of the on-disk format. The decompressor does not
+ * utilize this hash.
+ */
+#define HASH_MULTIPLIER		0x1E35A7BD
+
+struct compress_context {
+	const unsigned char *inbuf;
+	int bufsize;
+	int size;
+	int rel;
+	int mxsz;
+	s16 head[1 << HASH_SHIFT];
+	s16 prev[NTFS_SB_SIZE];
+};
+
+/*
+ * Hash the next 3-byte sequence in the input buffer
+ */
+static inline unsigned int ntfs_hash(const u8 *p)
+{
+	u32 str;
+	u32 hash;
+
+	/*
+	 * Unaligned access allowed, and little endian CPU.
+	 * Callers ensure that at least 4 (not 3) bytes are remaining.
+	 */
+	str = *(const u32 *)p & 0xFFFFFF;
+	hash = str * HASH_MULTIPLIER;
+
+	/* High bits are more random than the low bits.  */
+	return hash >> (32 - HASH_SHIFT);
+}
+
+/*
+ * Search for the longest sequence matching current position
+ *
+ * A hash table, each entry of which points to a chain of sequence
+ * positions sharing the corresponding hash code, is maintained to speed up
+ * searching for matches.  To maintain the hash table, either
+ * ntfs_best_match() or ntfs_skip_position() has to be called for each
+ * consecutive position.
+ *
+ * This function is heavily used; it has to be optimized carefully.
+ *
+ * This function sets pctx->size and pctx->rel to the length and offset,
+ * respectively, of the longest match found.
+ *
+ * The minimum match length is assumed to be 3, and the maximum match
+ * length is assumed to be pctx->mxsz.  If this function produces
+ * pctx->size < 3, then no match was found.
+ *
+ * Note: for the following reasons, this function is not guaranteed to find
+ * *the* longest match up to pctx->mxsz:
+ *
+ *      (1) If this function finds a match of NICE_MATCH_LEN bytes or greater,
+ *          it ends early because a match this long is good enough and it's not
+ *          worth spending more time searching.
+ *
+ *      (2) If this function considers MAX_SEARCH_DEPTH matches with a single
+ *          position, it ends early and returns the longest match found so far.
+ *          This saves a lot of time on degenerate inputs.
+ */
+static void ntfs_best_match(struct compress_context *pctx, const int i,
+		int best_len)
+{
+	const u8 * const inbuf = pctx->inbuf;
+	const u8 * const strptr = &inbuf[i]; /* String we're matching against */
+	s16 * const prev = pctx->prev;
+	const int max_len = min(pctx->bufsize - i, pctx->mxsz);
+	const int nice_len = min(NICE_MATCH_LEN, max_len);
+	int depth_remaining = MAX_SEARCH_DEPTH;
+	const u8 *best_matchptr = strptr;
+	unsigned int hash;
+	s16 cur_match;
+	const u8 *matchptr;
+	int len;
+
+	if (max_len < 4)
+		goto out;
+
+	/* Insert the current sequence into the appropriate hash chain. */
+	hash = ntfs_hash(strptr);
+	cur_match = pctx->head[hash];
+	prev[i] = cur_match;
+	pctx->head[hash] = i;
+
+	if (best_len >= max_len) {
+		/*
+		 * Lazy match is being attempted, but there aren't enough length
+		 * bits remaining to code a longer match.
+		 */
+		goto out;
+	}
+
+	/* Search the appropriate hash chain for matches. */
+
+	for (; cur_match >= 0 && depth_remaining--; cur_match = prev[cur_match]) {
+		matchptr = &inbuf[cur_match];
+
+		/*
+		 * Considering the potential match at 'matchptr':  is it longer
+		 * than 'best_len'?
+		 *
+		 * The bytes at index 'best_len' are the most likely to differ,
+		 * so check them first.
+		 *
+		 * The bytes at indices 'best_len - 1' and '0' are less
+		 * important to check separately.  But doing so still gives a
+		 * slight performance improvement, at least on x86_64, probably
+		 * because they create separate branches for the CPU to predict
+		 * independently of the branches in the main comparison loops.
+		 */
+		if (matchptr[best_len] != strptr[best_len] ||
+				matchptr[best_len - 1] != strptr[best_len - 1] ||
+				matchptr[0] != strptr[0])
+			goto next_match;
+
+		for (len = 1; len < best_len - 1; len++)
+			if (matchptr[len] != strptr[len])
+				goto next_match;
+
+		/*
+		 * The match is the longest found so far ---
+		 * at least 'best_len' + 1 bytes.  Continue extending it.
+		 */
+
+		best_matchptr = matchptr;
+
+		do {
+			if (++best_len >= nice_len) {
+				/*
+				 * 'nice_len' reached; don't waste time
+				 * searching for longer matches.  Extend the
+				 * match as far as possible and terminate the
+				 * search.
+				 */
+				while (best_len < max_len &&
+				       (best_matchptr[best_len] ==
+					strptr[best_len]))
+					best_len++;
+				goto out;
+			}
+		} while (best_matchptr[best_len] == strptr[best_len]);
+
+		/* Found a longer match, but 'nice_len' not yet reached.  */
+
+next_match:
+		/* Continue to next match in the chain.  */
+		;
+	}
+
+	/*
+	 * Reached end of chain, or ended early due to reaching the maximum
+	 * search depth.
+	 */
+
+out:
+	/* Return the longest match we were able to find.  */
+	pctx->size = best_len;
+	pctx->rel = best_matchptr - strptr; /* given as a negative number! */
+}
+
+/*
+ * Advance the match-finder, but don't search for matches.
+ */
+static void ntfs_skip_position(struct compress_context *pctx, const int i)
+{
+	unsigned int hash;
+
+	if (pctx->bufsize - i < 4)
+		return;
+
+	/* Insert the current sequence into the appropriate hash chain.  */
+	hash = ntfs_hash(pctx->inbuf + i);
+	pctx->prev[i] = pctx->head[hash];
+	pctx->head[hash] = i;
+}
+
+/*
+ * Compress a 4096-byte block
+ *
+ * Returns a header of two bytes followed by the compressed data.
+ * If compression is not effective, the header and an uncompressed
+ * block is returned.
+ *
+ * Note : two bytes may be output before output buffer overflow
+ * is detected, so a 4100-bytes output buffer must be reserved.
+ *
+ * Returns the size of the compressed block, including the
+ * header (minimal size is 2, maximum size is 4098)
+ * 0 if an error has been met.
+ */
+static unsigned int ntfs_compress_block(const char *inbuf, const int bufsize,
+		char *outbuf)
+{
+	struct compress_context *pctx;
+	int i; /* current position */
+	int j; /* end of best match from current position */
+	int k; /* end of best match from next position */
+	int offs; /* offset to best match */
+	int bp; /* bits to store offset */
+	int bp_cur; /* saved bits to store offset at current position */
+	int mxoff; /* max match offset : 1 << bp */
+	unsigned int xout;
+	unsigned int q; /* aggregated offset and size */
+	int have_match; /* do we have a match at the current position? */
+	char *ptag; /* location reserved for a tag */
+	int tag;    /* current value of tag */
+	int ntag;   /* count of bits still undefined in tag */
+
+	pctx = kvzalloc(sizeof(struct compress_context), GFP_NOFS);
+	if (!pctx)
+		return -ENOMEM;
+
+	/*
+	 * All hash chains start as empty.  The special value '-1' indicates the
+	 * end of each hash chain.
+	 */
+	memset(pctx->head, 0xFF, sizeof(pctx->head));
+
+	pctx->inbuf = (const unsigned char *)inbuf;
+	pctx->bufsize = bufsize;
+	xout = 2;
+	i = 0;
+	bp = 4;
+	mxoff = 1 << bp;
+	pctx->mxsz = (1 << (16 - bp)) + 2;
+	have_match = 0;
+	tag = 0;
+	ntag = 8;
+	ptag = &outbuf[xout++];
+
+	while ((i < bufsize) && (xout < (NTFS_SB_SIZE + 2))) {
+
+		/*
+		 * This implementation uses "lazy" parsing: it always chooses
+		 * the longest match, unless the match at the next position is
+		 * longer.  This is the same strategy used by the high
+		 * compression modes of zlib.
+		 */
+		if (!have_match) {
+			/*
+			 * Find the longest match at the current position.  But
+			 * first adjust the maximum match length if needed.
+			 * (This loop might need to run more than one time in
+			 * the case that we just output a long match.)
+			 */
+			while (mxoff < i) {
+				bp++;
+				mxoff <<= 1;
+				pctx->mxsz = (pctx->mxsz + 2) >> 1;
+			}
+			ntfs_best_match(pctx, i, 2);
+		}
+
+		if (pctx->size >= 3) {
+			/* Found a match at the current position.  */
+			j = i + pctx->size;
+			bp_cur = bp;
+			offs = pctx->rel;
+
+			if (pctx->size >= NICE_MATCH_LEN) {
+				/* Choose long matches immediately.  */
+				q = (~offs << (16 - bp_cur)) + (j - i - 3);
+				outbuf[xout++] = q & 255;
+				outbuf[xout++] = (q >> 8) & 255;
+				tag |= (1 << (8 - ntag));
+
+				if (j == bufsize) {
+					/*
+					 * Shortcut if the match extends to the
+					 * end of the buffer.
+					 */
+					i = j;
+					--ntag;
+					break;
+				}
+				i += 1;
+				do {
+					ntfs_skip_position(pctx, i);
+				} while (++i != j);
+				have_match = 0;
+			} else {
+				/*
+				 * Check for a longer match at the next
+				 * position.
+				 */
+
+				/*
+				 * Doesn't need to be while() since we just
+				 * adjusted the maximum match length at the
+				 * previous position.
+				 */
+				if (mxoff < i + 1) {
+					bp++;
+					mxoff <<= 1;
+					pctx->mxsz = (pctx->mxsz + 2) >> 1;
+				}
+				ntfs_best_match(pctx, i + 1, pctx->size);
+				k = i + 1 + pctx->size;
+
+				if (k > (j + 1)) {
+					/*
+					 * Next match is longer.
+					 * Output a literal.
+					 */
+					outbuf[xout++] = inbuf[i++];
+					have_match = 1;
+				} else {
+					/*
+					 * Next match isn't longer.
+					 * Output the current match.
+					 */
+					q = (~offs << (16 - bp_cur)) +
+						(j - i - 3);
+					outbuf[xout++] = q & 255;
+					outbuf[xout++] = (q >> 8) & 255;
+					tag |= (1 << (8 - ntag));
+
+					/*
+					 * The minimum match length is 3, and
+					 * we've run two bytes through the
+					 * matchfinder already.  So the minimum
+					 * number of positions we need to skip
+					 * is 1.
+					 */
+					i += 2;
+					do {
+						ntfs_skip_position(pctx, i);
+					} while (++i != j);
+					have_match = 0;
+				}
+			}
+		} else {
+			/* No match at current position.  Output a literal. */
+			outbuf[xout++] = inbuf[i++];
+			have_match = 0;
+		}
+
+		/* Store the tag if fully used. */
+		if (!--ntag) {
+			*ptag = tag;
+			ntag = 8;
+			ptag = &outbuf[xout++];
+			tag = 0;
+		}
+	}
+
+	/* Store the last tag if partially used. */
+	if (ntag == 8)
+		xout--;
+	else
+		*ptag = tag;
+
+	/* Determine whether to store the data compressed or uncompressed. */
+	if ((i >= bufsize) && (xout < (NTFS_SB_SIZE + 2))) {
+		/* Compressed. */
+		outbuf[0] = (xout - 3) & 255;
+		outbuf[1] = 0xb0 + (((xout - 3) >> 8) & 15);
+	} else {
+		/* Uncompressed.  */
+		memcpy(&outbuf[2], inbuf, bufsize);
+		if (bufsize < NTFS_SB_SIZE)
+			memset(&outbuf[bufsize + 2], 0, NTFS_SB_SIZE - bufsize);
+		outbuf[0] = 0xff;
+		outbuf[1] = 0x3f;
+		xout = NTFS_SB_SIZE + 2;
+	}
+
+	/*
+	 * Free the compression context and return the total number of bytes
+	 * written to 'outbuf'.
+	 */
+	kvfree(pctx);
+	return xout;
+}
+
+static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages,
+		int pages_per_cb)
+{
+	struct ntfs_volume *vol = ni->vol;
+	char *outbuf = NULL, *pbuf, *inbuf;
+	u32 compsz, p, insz = pages_per_cb << PAGE_SHIFT;
+	s32 rounded, bio_size;
+	unsigned int sz, bsz;
+	bool fail = false, allzeroes;
+	/* a single compressed zero */
+	static char onezero[] = {0x01, 0xb0, 0x00, 0x00};
+	/* a couple of compressed zeroes */
+	static char twozeroes[] = {0x02, 0xb0, 0x00, 0x00, 0x00};
+	/* more compressed zeroes, to be followed by some count */
+	static char morezeroes[] = {0x03, 0xb0, 0x02, 0x00};
+	struct page **pages_disk = NULL, *pg;
+	s64 bio_lcn;
+	struct runlist_element *rlc, *rl;
+	int i, err;
+	int pages_count = (round_up(ni->itype.compressed.block_size + 2 *
+		(ni->itype.compressed.block_size / NTFS_SB_SIZE) + 2, PAGE_SIZE)) / PAGE_SIZE;
+	size_t new_rl_count;
+	struct bio *bio = NULL;
+	loff_t new_length;
+	s64 new_vcn;
+
+	inbuf = vmap(pages, pages_per_cb, VM_MAP, PAGE_KERNEL_RO);
+	if (!inbuf)
+		return -ENOMEM;
+
+	/* may need 2 extra bytes per block and 2 more bytes */
+	pages_disk = kcalloc(pages_count, sizeof(struct page *), GFP_NOFS);
+	if (!pages_disk) {
+		vunmap(inbuf);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < pages_count; i++) {
+		pg = alloc_page(GFP_KERNEL);
+		if (!pg) {
+			err = -ENOMEM;
+			goto out;
+		}
+		pages_disk[i] = pg;
+		lock_page(pg);
+		kmap_local_page(pg);
+	}
+
+	outbuf = vmap(pages_disk, pages_count, VM_MAP, PAGE_KERNEL);
+	if (!outbuf) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	compsz = 0;
+	allzeroes = true;
+	for (p = 0; (p < insz) && !fail; p += NTFS_SB_SIZE) {
+		if ((p + NTFS_SB_SIZE) < insz)
+			bsz = NTFS_SB_SIZE;
+		else
+			bsz = insz - p;
+		pbuf = &outbuf[compsz];
+		sz = ntfs_compress_block(&inbuf[p], bsz, pbuf);
+		/* fail if all the clusters (or more) are needed */
+		if (!sz || ((compsz + sz + vol->cluster_size + 2) >
+			    ni->itype.compressed.block_size))
+			fail = true;
+		else {
+			if (allzeroes) {
+				/* check whether this is all zeroes */
+				switch (sz) {
+				case 4:
+					allzeroes = !memcmp(pbuf, onezero, 4);
+					break;
+				case 5:
+					allzeroes = !memcmp(pbuf, twozeroes, 5);
+					break;
+				case 6:
+					allzeroes = !memcmp(pbuf, morezeroes, 4);
+					break;
+				default:
+					allzeroes = false;
+					break;
+				}
+			}
+			compsz += sz;
+		}
+	}
+
+	if (!fail && !allzeroes) {
+		outbuf[compsz++] = 0;
+		outbuf[compsz++] = 0;
+		rounded = ((compsz - 1) | (vol->cluster_size - 1)) + 1;
+		memset(&outbuf[compsz], 0, rounded - compsz);
+		bio_size = rounded;
+		pages = pages_disk;
+	} else if (allzeroes) {
+		err = 0;
+		goto out;
+	} else {
+		bio_size = insz;
+	}
+
+	new_vcn = ntfs_bytes_to_cluster(vol, pos & ~(ni->itype.compressed.block_size - 1));
+	new_length = ntfs_bytes_to_cluster(vol, round_up(bio_size, vol->cluster_size));
+
+	err = ntfs_non_resident_attr_punch_hole(ni, new_vcn, ni->itype.compressed.block_clusters);
+	if (err < 0)
+		goto out;
+
+	rlc = ntfs_cluster_alloc(vol, new_vcn, new_length, -1, DATA_ZONE,
+			false, true, true);
+	if (IS_ERR(rlc)) {
+		err = PTR_ERR(rlc);
+		goto out;
+	}
+
+	bio_lcn = rlc->lcn;
+	down_write(&ni->runlist.lock);
+	rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, &new_rl_count);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		ntfs_error(vol->sb, "Failed to merge runlists");
+		err = PTR_ERR(rl);
+		if (ntfs_cluster_free_from_rl(vol, rlc))
+			ntfs_error(vol->sb, "Failed to free hot clusters.");
+		kvfree(rlc);
+		goto out;
+	}
+
+	ni->runlist.count = new_rl_count;
+	ni->runlist.rl = rl;
+
+	err = ntfs_attr_update_mapping_pairs(ni, 0);
+	up_write(&ni->runlist.lock);
+	if (err) {
+		err = -EIO;
+		goto out;
+	}
+
+	i = 0;
+	while (bio_size > 0) {
+		int page_size;
+
+		if (bio_size >= PAGE_SIZE) {
+			page_size = PAGE_SIZE;
+			bio_size -= PAGE_SIZE;
+		} else {
+			page_size = bio_size;
+			bio_size = 0;
+		}
+
+setup_bio:
+		if (!bio) {
+			bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE,
+					GFP_NOIO);
+			bio->bi_iter.bi_sector =
+				ntfs_bytes_to_sector(vol,
+						ntfs_cluster_to_bytes(vol, bio_lcn + i));
+		}
+
+		if (!bio_add_page(bio, pages[i], page_size, 0)) {
+			err = submit_bio_wait(bio);
+			bio_put(bio);
+			if (err)
+				goto out;
+			bio = NULL;
+			goto setup_bio;
+		}
+		i++;
+	}
+
+	err = submit_bio_wait(bio);
+	bio_put(bio);
+out:
+	vunmap(outbuf);
+	for (i = 0; i < pages_count; i++) {
+		pg = pages_disk[i];
+		if (pg) {
+			kunmap_local(page_address(pg));
+			unlock_page(pg);
+			put_page(pg);
+		}
+	}
+	kfree(pages_disk);
+	vunmap(inbuf);
+	NInoSetFileNameDirty(ni);
+	mark_mft_record_dirty(ni);
+
+	return err;
+}
+
+int ntfs_compress_write(struct ntfs_inode *ni, loff_t pos, size_t count,
+		struct iov_iter *from)
+{
+	struct folio *folio;
+	struct page **pages = NULL, *page;
+	int pages_per_cb = ni->itype.compressed.block_size >> PAGE_SHIFT;
+	int cb_size = ni->itype.compressed.block_size, cb_off, err = 0;
+	int i, ip;
+	size_t written = 0;
+	struct address_space *mapping = VFS_I(ni)->i_mapping;
+
+	if (NInoCompressed(ni) && pos + count > ni->allocated_size) {
+		int err;
+		loff_t end = pos + count;
+
+		err = ntfs_attr_expand(ni, end,
+				round_up(end, ni->itype.compressed.block_size));
+		if (err)
+			return err;
+	}
+
+	pages = kmalloc_array(pages_per_cb, sizeof(struct page *), GFP_NOFS);
+	if (!pages)
+		return -ENOMEM;
+
+	while (count) {
+		pgoff_t index;
+		size_t copied, bytes;
+		int off;
+
+		off = pos & (cb_size - 1);
+		bytes = cb_size - off;
+		if (bytes > count)
+			bytes = count;
+
+		cb_off = pos & ~(cb_size - 1);
+		index = cb_off >> PAGE_SHIFT;
+
+		if (unlikely(fault_in_iov_iter_readable(from, bytes))) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		for (i = 0; i < pages_per_cb; i++) {
+			folio = read_mapping_folio(mapping, index + i, NULL);
+			if (IS_ERR(folio)) {
+				for (ip = 0; ip < i; ip++) {
+					folio_unlock(page_folio(pages[ip]));
+					folio_put(page_folio(pages[ip]));
+				}
+				err = PTR_ERR(folio);
+				goto out;
+			}
+
+			folio_lock(folio);
+			pages[i] = folio_page(folio, 0);
+		}
+
+		WARN_ON(!bytes);
+		copied = 0;
+		ip = off >> PAGE_SHIFT;
+		off = offset_in_page(pos);
+
+		for (;;) {
+			size_t cp, tail = PAGE_SIZE - off;
+
+			page = pages[ip];
+			cp = copy_folio_from_iter_atomic(page_folio(page), off,
+					min(tail, bytes), from);
+			flush_dcache_page(page);
+
+			copied += cp;
+			bytes -= cp;
+			if (!bytes || !cp)
+				break;
+
+			if (cp < tail) {
+				off += cp;
+			} else {
+				ip++;
+				off = 0;
+			}
+		}
+
+		err = ntfs_write_cb(ni, pos, pages, pages_per_cb);
+
+		for (i = 0; i < pages_per_cb; i++) {
+			folio = page_folio(pages[i]);
+			if (i < ip) {
+				folio_clear_dirty(folio);
+				folio_mark_uptodate(folio);
+			}
+			folio_unlock(folio);
+			folio_put(folio);
+		}
+
+		if (err)
+			goto out;
+
+		cond_resched();
+		pos += copied;
+		written += copied;
+		count = iov_iter_count(from);
+	}
+
+out:
+	kfree(pages);
+	if (err < 0)
+		written = err;
+
+	return written;
+}
diff --git a/fs/ntfs/debug.c b/fs/ntfs/debug.c
new file mode 100644
index 0000000000000..f241ebc97d374
--- /dev/null
+++ b/fs/ntfs/debug.c
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel debug support.
+ *
+ * Copyright (c) 2001-2004 Anton Altaparmakov
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include "debug.h"
+
+/*
+ * __ntfs_warning - output a warning to the syslog
+ * @function:	name of function outputting the warning
+ * @sb:		super block of mounted ntfs filesystem
+ * @fmt:	warning string containing format specifications
+ * @...:	a variable number of arguments specified in @fmt
+ *
+ * Outputs a warning to the syslog for the mounted ntfs filesystem described
+ * by @sb.
+ *
+ * @fmt and the corresponding @... is printf style format string containing
+ * the warning string and the corresponding format arguments, respectively.
+ *
+ * @function is the name of the function from which __ntfs_warning is being
+ * called.
+ *
+ * Note, you should be using debug.h::ntfs_warning(@sb, @fmt, @...) instead
+ * as this provides the @function parameter automatically.
+ */
+void __ntfs_warning(const char *function, const struct super_block *sb,
+		const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+	int flen = 0;
+
+	if (function)
+		flen = strlen(function);
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+#ifdef DEBUG
+	if (sb)
+		pr_warn("(device %s): %s(): %pV\n",
+			sb->s_id, flen ? function : "", &vaf);
+	else
+		pr_warn("%s(): %pV\n", flen ? function : "", &vaf);
+#else
+	if (sb)
+		pr_warn_ratelimited("(device %s): %s(): %pV\n",
+			sb->s_id, flen ? function : "", &vaf);
+	else
+		pr_warn_ratelimited("%s(): %pV\n", flen ? function : "", &vaf);
+#endif
+	va_end(args);
+}
+
+/*
+ * __ntfs_error - output an error to the syslog
+ * @function:	name of function outputting the error
+ * @sb:		super block of mounted ntfs filesystem
+ * @fmt:	error string containing format specifications
+ * @...:	a variable number of arguments specified in @fmt
+ *
+ * Outputs an error to the syslog for the mounted ntfs filesystem described
+ * by @sb.
+ *
+ * @fmt and the corresponding @... is printf style format string containing
+ * the error string and the corresponding format arguments, respectively.
+ *
+ * @function is the name of the function from which __ntfs_error is being
+ * called.
+ *
+ * Note, you should be using debug.h::ntfs_error(@sb, @fmt, @...) instead
+ * as this provides the @function parameter automatically.
+ */
+void __ntfs_error(const char *function, struct super_block *sb,
+		const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+	int flen = 0;
+
+	if (function)
+		flen = strlen(function);
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+#ifdef DEBUG
+	if (sb)
+		pr_err("(device %s): %s(): %pV\n",
+		       sb->s_id, flen ? function : "", &vaf);
+	else
+		pr_err("%s(): %pV\n", flen ? function : "", &vaf);
+#else
+	if (sb)
+		pr_err_ratelimited("(device %s): %s(): %pV\n",
+		       sb->s_id, flen ? function : "", &vaf);
+	else
+		pr_err_ratelimited("%s(): %pV\n", flen ? function : "", &vaf);
+#endif
+	va_end(args);
+
+	if (sb)
+		ntfs_handle_error(sb);
+}
+
+#ifdef DEBUG
+
+/* If 1, output debug messages, and if 0, don't. */
+int debug_msgs;
+
+void __ntfs_debug(const char *file, int line, const char *function,
+		const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+	int flen = 0;
+
+	if (!debug_msgs)
+		return;
+	if (function)
+		flen = strlen(function);
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+	pr_debug("(%s, %d): %s(): %pV", file, line, flen ? function : "", &vaf);
+	va_end(args);
+}
+
+/* Dump a runlist. Caller has to provide synchronisation for @rl. */
+void ntfs_debug_dump_runlist(const struct runlist_element *rl)
+{
+	int i;
+	const char *lcn_str[5] = { "LCN_DELALLOC     ", "LCN_HOLE         ",
+				   "LCN_RL_NOT_MAPPED", "LCN_ENOENT       ",
+				   "LCN_unknown      " };
+
+	if (!debug_msgs)
+		return;
+	pr_debug("Dumping runlist (values in hex):\n");
+	if (!rl) {
+		pr_debug("Run list not present.\n");
+		return;
+	}
+	pr_debug("VCN              LCN               Run length\n");
+	for (i = 0; ; i++) {
+		s64 lcn = (rl + i)->lcn;
+
+		if (lcn < 0) {
+			int index = -lcn - 1;
+
+			if (index > -LCN_ENOENT - 1)
+				index = 3;
+			pr_debug("%-16Lx %s %-16Lx%s\n",
+					(long long)(rl + i)->vcn, lcn_str[index],
+					(long long)(rl + i)->length,
+					(rl + i)->length ? "" :
+						" (runlist end)");
+		} else
+			pr_debug("%-16Lx %-16Lx  %-16Lx%s\n",
+					(long long)(rl + i)->vcn,
+					(long long)(rl + i)->lcn,
+					(long long)(rl + i)->length,
+					(rl + i)->length ? "" :
+						" (runlist end)");
+		if (!(rl + i)->length)
+			break;
+	}
+}
+
+#endif
diff --git a/fs/ntfs/debug.h b/fs/ntfs/debug.h
new file mode 100644
index 0000000000000..d8831d492d7d6
--- /dev/null
+++ b/fs/ntfs/debug.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * NTFS kernel debug support.
+ *
+ * Copyright (c) 2001-2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_DEBUG_H
+#define _LINUX_NTFS_DEBUG_H
+
+#include <linux/fs.h>
+
+#include "runlist.h"
+
+#ifdef DEBUG
+
+extern int debug_msgs;
+
+extern __printf(4, 5)
+void __ntfs_debug(const char *file, int line, const char *function,
+		  const char *format, ...);
+/*
+ * ntfs_debug - write a debug level message to syslog
+ * @f:		a printf format string containing the message
+ * @...:	the variables to substitute into @f
+ *
+ * ntfs_debug() writes a DEBUG level message to the syslog but only if the
+ * driver was compiled with -DDEBUG. Otherwise, the call turns into a NOP.
+ */
+#define ntfs_debug(f, a...)						\
+	__ntfs_debug(__FILE__, __LINE__, __func__, f, ##a)
+
+void ntfs_debug_dump_runlist(const struct runlist_element *rl);
+
+#else	/* !DEBUG */
+
+#define ntfs_debug(fmt, ...)						\
+do {									\
+	if (0)								\
+		no_printk(fmt, ##__VA_ARGS__);				\
+} while (0)
+
+#define ntfs_debug_dump_runlist(rl)					\
+do {									\
+	if (0)								\
+		(void)rl;						\
+} while (0)
+
+#endif	/* !DEBUG */
+
+extern  __printf(3, 4)
+void __ntfs_warning(const char *function, const struct super_block *sb,
+		    const char *fmt, ...);
+#define ntfs_warning(sb, f, a...)	__ntfs_warning(__func__, sb, f, ##a)
+
+extern  __printf(3, 4)
+void __ntfs_error(const char *function, struct super_block *sb,
+		  const char *fmt, ...);
+#define ntfs_error(sb, f, a...)		__ntfs_error(__func__, sb, f, ##a)
+
+void ntfs_handle_error(struct super_block *sb);
+
+#endif /* _LINUX_NTFS_DEBUG_H */
diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c
new file mode 100644
index 0000000000000..bfa904d2ce665
--- /dev/null
+++ b/fs/ntfs/dir.c
@@ -0,0 +1,1238 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel directory operations.
+ *
+ * Copyright (c) 2001-2007 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/blkdev.h>
+
+#include "dir.h"
+#include "mft.h"
+#include "ntfs.h"
+#include "index.h"
+#include "reparse.h"
+
+#include <linux/filelock.h>
+
+/*
+ * The little endian Unicode string $I30 as a global constant.
+ */
+__le16 I30[5] = { cpu_to_le16('$'), cpu_to_le16('I'),
+		cpu_to_le16('3'),	cpu_to_le16('0'), 0 };
+
+/*
+ * ntfs_lookup_inode_by_name - find an inode in a directory given its name
+ * @dir_ni:	ntfs inode of the directory in which to search for the name
+ * @uname:	Unicode name for which to search in the directory
+ * @uname_len:	length of the name @uname in Unicode characters
+ * @res:	return the found file name if necessary (see below)
+ *
+ * Look for an inode with name @uname in the directory with inode @dir_ni.
+ * ntfs_lookup_inode_by_name() walks the contents of the directory looking for
+ * the Unicode name. If the name is found in the directory, the corresponding
+ * inode number (>= 0) is returned as a mft reference in cpu format, i.e. it
+ * is a 64-bit number containing the sequence number.
+ *
+ * On error, a negative value is returned corresponding to the error code. In
+ * particular if the inode is not found -ENOENT is returned. Note that you
+ * can't just check the return value for being negative, you have to check the
+ * inode number for being negative which you can extract using MREC(return
+ * value).
+ *
+ * Note, @uname_len does not include the (optional) terminating NULL character.
+ *
+ * Note, we look for a case sensitive match first but we also look for a case
+ * insensitive match at the same time. If we find a case insensitive match, we
+ * save that for the case that we don't find an exact match, where we return
+ * the case insensitive match and setup @res (which we allocate!) with the mft
+ * reference, the file name type, length and with a copy of the little endian
+ * Unicode file name itself. If we match a file name which is in the DOS name
+ * space, we only return the mft reference and file name type in @res.
+ * ntfs_lookup() then uses this to find the long file name in the inode itself.
+ * This is to avoid polluting the dcache with short file names. We want them to
+ * work but we don't care for how quickly one can access them. This also fixes
+ * the dcache aliasing issues.
+ *
+ * Locking:  - Caller must hold i_mutex on the directory.
+ *	     - Each page cache page in the index allocation mapping must be
+ *	       locked whilst being accessed otherwise we may find a corrupt
+ *	       page due to it being under ->writepage at the moment which
+ *	       applies the mst protection fixups before writing out and then
+ *	       removes them again after the write is complete after which it
+ *	       unlocks the page.
+ */
+u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname,
+		const int uname_len, struct ntfs_name **res)
+{
+	struct ntfs_volume *vol = dir_ni->vol;
+	struct super_block *sb = vol->sb;
+	struct inode *ia_vi = NULL;
+	struct mft_record *m;
+	struct index_root *ir;
+	struct index_entry *ie;
+	struct index_block *ia;
+	u8 *index_end;
+	u64 mref;
+	struct ntfs_attr_search_ctx *ctx;
+	int err, rc;
+	s64 vcn, old_vcn;
+	struct address_space *ia_mapping;
+	struct folio *folio;
+	u8 *kaddr = NULL;
+	struct ntfs_name *name = NULL;
+
+	/* Get hold of the mft record for the directory. */
+	m = map_mft_record(dir_ni);
+	if (IS_ERR(m)) {
+		ntfs_error(sb, "map_mft_record() failed with error code %ld.",
+				-PTR_ERR(m));
+		return ERR_MREF(PTR_ERR(m));
+	}
+	ctx = ntfs_attr_get_search_ctx(dir_ni, m);
+	if (unlikely(!ctx)) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+	/* Find the index root attribute in the mft record. */
+	err = ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL,
+			0, ctx);
+	if (unlikely(err)) {
+		if (err == -ENOENT) {
+			ntfs_error(sb,
+				"Index root attribute missing in directory inode 0x%llx.",
+				dir_ni->mft_no);
+			err = -EIO;
+		}
+		goto err_out;
+	}
+	/* Get to the index root value (it's been verified in read_inode). */
+	ir = (struct index_root *)((u8 *)ctx->attr +
+			le16_to_cpu(ctx->attr->data.resident.value_offset));
+	index_end = (u8 *)&ir->index + le32_to_cpu(ir->index.index_length);
+	/* The first index entry. */
+	ie = (struct index_entry *)((u8 *)&ir->index +
+			le32_to_cpu(ir->index.entries_offset));
+	/*
+	 * Loop until we exceed valid memory (corruption case) or until we
+	 * reach the last entry.
+	 */
+	for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) {
+		/* Bounds checks. */
+		if ((u8 *)ie < (u8 *)ctx->mrec ||
+		    (u8 *)ie + sizeof(struct index_entry_header) > index_end ||
+		    (u8 *)ie + sizeof(struct index_entry_header) + le16_to_cpu(ie->key_length) >
+				index_end || (u8 *)ie + le16_to_cpu(ie->length) > index_end)
+			goto dir_err_out;
+		/*
+		 * The last entry cannot contain a name. It can however contain
+		 * a pointer to a child node in the B+tree so we just break out.
+		 */
+		if (ie->flags & INDEX_ENTRY_END)
+			break;
+		/* Key length should not be zero if it is not last entry. */
+		if (!ie->key_length)
+			goto dir_err_out;
+		/* Check the consistency of an index entry */
+		if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME,
+				dir_ni->mft_no))
+			goto dir_err_out;
+		/*
+		 * We perform a case sensitive comparison and if that matches
+		 * we are done and return the mft reference of the inode (i.e.
+		 * the inode number together with the sequence number for
+		 * consistency checking). We convert it to cpu format before
+		 * returning.
+		 */
+		if (ntfs_are_names_equal(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length,
+				CASE_SENSITIVE, vol->upcase, vol->upcase_len)) {
+found_it:
+			/*
+			 * We have a perfect match, so we don't need to care
+			 * about having matched imperfectly before, so we can
+			 * free name and set *res to NULL.
+			 * However, if the perfect match is a short file name,
+			 * we need to signal this through *res, so that
+			 * ntfs_lookup() can fix dcache aliasing issues.
+			 * As an optimization we just reuse an existing
+			 * allocation of *res.
+			 */
+			if (ie->key.file_name.file_name_type == FILE_NAME_DOS) {
+				if (!name) {
+					name = kmalloc(sizeof(struct ntfs_name),
+							GFP_NOFS);
+					if (!name) {
+						err = -ENOMEM;
+						goto err_out;
+					}
+				}
+				name->mref = le64_to_cpu(
+						ie->data.dir.indexed_file);
+				name->type = FILE_NAME_DOS;
+				name->len = 0;
+				*res = name;
+			} else {
+				kfree(name);
+				*res = NULL;
+			}
+			mref = le64_to_cpu(ie->data.dir.indexed_file);
+			ntfs_attr_put_search_ctx(ctx);
+			unmap_mft_record(dir_ni);
+			return mref;
+		}
+		/*
+		 * For a case insensitive mount, we also perform a case
+		 * insensitive comparison (provided the file name is not in the
+		 * POSIX namespace). If the comparison matches, and the name is
+		 * in the WIN32 namespace, we cache the filename in *res so
+		 * that the caller, ntfs_lookup(), can work on it. If the
+		 * comparison matches, and the name is in the DOS namespace, we
+		 * only cache the mft reference and the file name type (we set
+		 * the name length to zero for simplicity).
+		 */
+		if ((!NVolCaseSensitive(vol) ||
+		     ie->key.file_name.file_name_type == FILE_NAME_DOS) &&
+		    ntfs_are_names_equal(uname, uname_len,
+					 (__le16 *)&ie->key.file_name.file_name,
+					 ie->key.file_name.file_name_length,
+					 IGNORE_CASE, vol->upcase,
+					 vol->upcase_len)) {
+			int name_size = sizeof(struct ntfs_name);
+			u8 type = ie->key.file_name.file_name_type;
+			u8 len = ie->key.file_name.file_name_length;
+
+			/* Only one case insensitive matching name allowed. */
+			if (name) {
+				ntfs_error(sb,
+					"Found already allocated name in phase 1. Please run chkdsk");
+				goto dir_err_out;
+			}
+
+			if (type != FILE_NAME_DOS)
+				name_size += len * sizeof(__le16);
+			name = kmalloc(name_size, GFP_NOFS);
+			if (!name) {
+				err = -ENOMEM;
+				goto err_out;
+			}
+			name->mref = le64_to_cpu(ie->data.dir.indexed_file);
+			name->type = type;
+			if (type != FILE_NAME_DOS) {
+				name->len = len;
+				memcpy(name->name, ie->key.file_name.file_name,
+						len * sizeof(__le16));
+			} else
+				name->len = 0;
+			*res = name;
+		}
+		/*
+		 * Not a perfect match, need to do full blown collation so we
+		 * know which way in the B+tree we have to go.
+		 */
+		rc = ntfs_collate_names(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length, 1,
+				IGNORE_CASE, vol->upcase, vol->upcase_len);
+		/*
+		 * If uname collates before the name of the current entry, there
+		 * is definitely no such name in this index but we might need to
+		 * descend into the B+tree so we just break out of the loop.
+		 */
+		if (rc == -1)
+			break;
+		/* The names are not equal, continue the search. */
+		if (rc)
+			continue;
+		/*
+		 * Names match with case insensitive comparison, now try the
+		 * case sensitive comparison, which is required for proper
+		 * collation.
+		 */
+		rc = ntfs_collate_names(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length, 1,
+				CASE_SENSITIVE, vol->upcase, vol->upcase_len);
+		if (rc == -1)
+			break;
+		if (rc)
+			continue;
+		/*
+		 * Perfect match, this will never happen as the
+		 * ntfs_are_names_equal() call will have gotten a match but we
+		 * still treat it correctly.
+		 */
+		goto found_it;
+	}
+	/*
+	 * We have finished with this index without success. Check for the
+	 * presence of a child node and if not present return -ENOENT, unless
+	 * we have got a matching name cached in name in which case return the
+	 * mft reference associated with it.
+	 */
+	if (!(ie->flags & INDEX_ENTRY_NODE)) {
+		if (name) {
+			ntfs_attr_put_search_ctx(ctx);
+			unmap_mft_record(dir_ni);
+			return name->mref;
+		}
+		ntfs_debug("Entry not found.");
+		err = -ENOENT;
+		goto err_out;
+	} /* Child node present, descend into it. */
+
+	/* Get the starting vcn of the index_block holding the child node. */
+	vcn = le64_to_cpup((__le64 *)((u8 *)ie + le16_to_cpu(ie->length) - 8));
+
+	/*
+	 * We are done with the index root and the mft record. Release them,
+	 * otherwise we deadlock with read_mapping_folio().
+	 */
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(dir_ni);
+	m = NULL;
+	ctx = NULL;
+
+	ia_vi = ntfs_index_iget(VFS_I(dir_ni), I30, 4);
+	if (IS_ERR(ia_vi)) {
+		err = PTR_ERR(ia_vi);
+		goto err_out;
+	}
+
+	ia_mapping = ia_vi->i_mapping;
+descend_into_child_node:
+	/*
+	 * Convert vcn to index into the index allocation attribute in units
+	 * of PAGE_SIZE and map the page cache page, reading it from
+	 * disk if necessary.
+	 */
+	folio = read_mapping_folio(ia_mapping, vcn <<
+			dir_ni->itype.index.vcn_size_bits >> PAGE_SHIFT, NULL);
+	if (IS_ERR(folio)) {
+		ntfs_error(sb, "Failed to map directory index page, error %ld.",
+				-PTR_ERR(folio));
+		err = PTR_ERR(folio);
+		goto err_out;
+	}
+
+	folio_lock(folio);
+	kaddr = kmalloc(PAGE_SIZE, GFP_NOFS);
+	if (!kaddr) {
+		err = -ENOMEM;
+		folio_unlock(folio);
+		folio_put(folio);
+		goto unm_err_out;
+	}
+
+	memcpy_from_folio(kaddr, folio, 0, PAGE_SIZE);
+	post_read_mst_fixup((struct ntfs_record *)kaddr, PAGE_SIZE);
+	folio_unlock(folio);
+	folio_put(folio);
+fast_descend_into_child_node:
+	/* Get to the index allocation block. */
+	ia = (struct index_block *)(kaddr + ((vcn <<
+			dir_ni->itype.index.vcn_size_bits) & ~PAGE_MASK));
+	/* Bounds checks. */
+	if ((u8 *)ia < kaddr || (u8 *)ia > kaddr + PAGE_SIZE) {
+		ntfs_error(sb,
+			"Out of bounds check failed. Corrupt directory inode 0x%llx or driver bug.",
+			dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	/* Catch multi sector transfer fixup errors. */
+	if (unlikely(!ntfs_is_indx_record(ia->magic))) {
+		ntfs_error(sb,
+			"Directory index record with vcn 0x%llx is corrupt.  Corrupt inode 0x%llx.  Run chkdsk.",
+			vcn, dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	if (le64_to_cpu(ia->index_block_vcn) != vcn) {
+		ntfs_error(sb,
+			"Actual VCN (0x%llx) of index buffer is different from expected VCN (0x%llx). Directory inode 0x%llx is corrupt or driver bug.",
+			le64_to_cpu(ia->index_block_vcn),
+			vcn, dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	if (le32_to_cpu(ia->index.allocated_size) + 0x18 !=
+			dir_ni->itype.index.block_size) {
+		ntfs_error(sb,
+			"Index buffer (VCN 0x%llx) of directory inode 0x%llx has a size (%u) differing from the directory specified size (%u). Directory inode is corrupt or driver bug.",
+			vcn, dir_ni->mft_no,
+			le32_to_cpu(ia->index.allocated_size) + 0x18,
+			dir_ni->itype.index.block_size);
+		goto unm_err_out;
+	}
+	index_end = (u8 *)ia + dir_ni->itype.index.block_size;
+	if (index_end > kaddr + PAGE_SIZE) {
+		ntfs_error(sb,
+			"Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.",
+			vcn, dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length);
+	if (index_end > (u8 *)ia + dir_ni->itype.index.block_size) {
+		ntfs_error(sb,
+			"Size of index buffer (VCN 0x%llx) of directory inode 0x%llx exceeds maximum size.",
+			vcn, dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	/* The first index entry. */
+	ie = (struct index_entry *)((u8 *)&ia->index +
+			le32_to_cpu(ia->index.entries_offset));
+	/*
+	 * Iterate similar to above big loop but applied to index buffer, thus
+	 * loop until we exceed valid memory (corruption case) or until we
+	 * reach the last entry.
+	 */
+	for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) {
+		/* Bounds checks. */
+		if ((u8 *)ie < (u8 *)ia ||
+		    (u8 *)ie + sizeof(struct index_entry_header) > index_end ||
+		    (u8 *)ie + sizeof(struct index_entry_header) + le16_to_cpu(ie->key_length) >
+				index_end || (u8 *)ie + le16_to_cpu(ie->length) > index_end) {
+			ntfs_error(sb, "Index entry out of bounds in directory inode 0x%llx.",
+					dir_ni->mft_no);
+			goto unm_err_out;
+		}
+		/*
+		 * The last entry cannot contain a name. It can however contain
+		 * a pointer to a child node in the B+tree so we just break out.
+		 */
+		if (ie->flags & INDEX_ENTRY_END)
+			break;
+		/* Key length should not be zero if it is not last entry. */
+		if (!ie->key_length)
+			goto unm_err_out;
+		/* Check the consistency of an index entry */
+		if (ntfs_index_entry_inconsistent(NULL, vol, ie, COLLATION_FILE_NAME,
+				dir_ni->mft_no))
+			goto unm_err_out;
+		/*
+		 * We perform a case sensitive comparison and if that matches
+		 * we are done and return the mft reference of the inode (i.e.
+		 * the inode number together with the sequence number for
+		 * consistency checking). We convert it to cpu format before
+		 * returning.
+		 */
+		if (ntfs_are_names_equal(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length,
+				CASE_SENSITIVE, vol->upcase, vol->upcase_len)) {
+found_it2:
+			/*
+			 * We have a perfect match, so we don't need to care
+			 * about having matched imperfectly before, so we can
+			 * free name and set *res to NULL.
+			 * However, if the perfect match is a short file name,
+			 * we need to signal this through *res, so that
+			 * ntfs_lookup() can fix dcache aliasing issues.
+			 * As an optimization we just reuse an existing
+			 * allocation of *res.
+			 */
+			if (ie->key.file_name.file_name_type == FILE_NAME_DOS) {
+				if (!name) {
+					name = kmalloc(sizeof(struct ntfs_name),
+							GFP_NOFS);
+					if (!name) {
+						err = -ENOMEM;
+						goto unm_err_out;
+					}
+				}
+				name->mref = le64_to_cpu(
+						ie->data.dir.indexed_file);
+				name->type = FILE_NAME_DOS;
+				name->len = 0;
+				*res = name;
+			} else {
+				kfree(name);
+				*res = NULL;
+			}
+			mref = le64_to_cpu(ie->data.dir.indexed_file);
+			kfree(kaddr);
+			iput(ia_vi);
+			return mref;
+		}
+		/*
+		 * For a case insensitive mount, we also perform a case
+		 * insensitive comparison (provided the file name is not in the
+		 * POSIX namespace). If the comparison matches, and the name is
+		 * in the WIN32 namespace, we cache the filename in *res so
+		 * that the caller, ntfs_lookup(), can work on it. If the
+		 * comparison matches, and the name is in the DOS namespace, we
+		 * only cache the mft reference and the file name type (we set
+		 * the name length to zero for simplicity).
+		 */
+		if ((!NVolCaseSensitive(vol) ||
+		     ie->key.file_name.file_name_type == FILE_NAME_DOS) &&
+		    ntfs_are_names_equal(uname, uname_len,
+					 (__le16 *)&ie->key.file_name.file_name,
+					 ie->key.file_name.file_name_length,
+					 IGNORE_CASE, vol->upcase,
+					 vol->upcase_len)) {
+			int name_size = sizeof(struct ntfs_name);
+			u8 type = ie->key.file_name.file_name_type;
+			u8 len = ie->key.file_name.file_name_length;
+
+			/* Only one case insensitive matching name allowed. */
+			if (name) {
+				ntfs_error(sb,
+					"Found already allocated name in phase 2. Please run chkdsk");
+				kfree(kaddr);
+				goto dir_err_out;
+			}
+
+			if (type != FILE_NAME_DOS)
+				name_size += len * sizeof(__le16);
+			name = kmalloc(name_size, GFP_NOFS);
+			if (!name) {
+				err = -ENOMEM;
+				goto unm_err_out;
+			}
+			name->mref = le64_to_cpu(ie->data.dir.indexed_file);
+			name->type = type;
+			if (type != FILE_NAME_DOS) {
+				name->len = len;
+				memcpy(name->name, ie->key.file_name.file_name,
+						len * sizeof(__le16));
+			} else
+				name->len = 0;
+			*res = name;
+		}
+		/*
+		 * Not a perfect match, need to do full blown collation so we
+		 * know which way in the B+tree we have to go.
+		 */
+		rc = ntfs_collate_names(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length, 1,
+				IGNORE_CASE, vol->upcase, vol->upcase_len);
+		/*
+		 * If uname collates before the name of the current entry, there
+		 * is definitely no such name in this index but we might need to
+		 * descend into the B+tree so we just break out of the loop.
+		 */
+		if (rc == -1)
+			break;
+		/* The names are not equal, continue the search. */
+		if (rc)
+			continue;
+		/*
+		 * Names match with case insensitive comparison, now try the
+		 * case sensitive comparison, which is required for proper
+		 * collation.
+		 */
+		rc = ntfs_collate_names(uname, uname_len,
+				(__le16 *)&ie->key.file_name.file_name,
+				ie->key.file_name.file_name_length, 1,
+				CASE_SENSITIVE, vol->upcase, vol->upcase_len);
+		if (rc == -1)
+			break;
+		if (rc)
+			continue;
+		/*
+		 * Perfect match, this will never happen as the
+		 * ntfs_are_names_equal() call will have gotten a match but we
+		 * still treat it correctly.
+		 */
+		goto found_it2;
+	}
+	/*
+	 * We have finished with this index buffer without success. Check for
+	 * the presence of a child node.
+	 */
+	if (ie->flags & INDEX_ENTRY_NODE) {
+		if ((ia->index.flags & NODE_MASK) == LEAF_NODE) {
+			ntfs_error(sb,
+				"Index entry with child node found in a leaf node in directory inode 0x%llx.",
+				dir_ni->mft_no);
+			goto unm_err_out;
+		}
+		/* Child node present, descend into it. */
+		old_vcn = vcn;
+		vcn = le64_to_cpup((__le64 *)((u8 *)ie +
+				le16_to_cpu(ie->length) - 8));
+		if (vcn >= 0) {
+			/*
+			 * If vcn is in the same page cache page as old_vcn we
+			 * recycle the mapped page.
+			 */
+			if (ntfs_cluster_to_pidx(vol, old_vcn) ==
+			    ntfs_cluster_to_pidx(vol, vcn))
+				goto fast_descend_into_child_node;
+			kfree(kaddr);
+			kaddr = NULL;
+			goto descend_into_child_node;
+		}
+		ntfs_error(sb, "Negative child node vcn in directory inode 0x%llx.",
+				dir_ni->mft_no);
+		goto unm_err_out;
+	}
+	/*
+	 * No child node present, return -ENOENT, unless we have got a matching
+	 * name cached in name in which case return the mft reference
+	 * associated with it.
+	 */
+	if (name) {
+		kfree(kaddr);
+		iput(ia_vi);
+		return name->mref;
+	}
+	ntfs_debug("Entry not found.");
+	err = -ENOENT;
+unm_err_out:
+	kfree(kaddr);
+err_out:
+	if (!err)
+		err = -EIO;
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	if (m)
+		unmap_mft_record(dir_ni);
+	kfree(name);
+	*res = NULL;
+	if (!IS_ERR_OR_NULL(ia_vi))
+		iput(ia_vi);
+	return ERR_MREF(err);
+dir_err_out:
+	ntfs_error(sb, "Corrupt directory.  Aborting lookup.");
+	goto err_out;
+}
+
+/*
+ * ntfs_filldir - ntfs specific filldir method
+ * @vol:	current ntfs volume
+ * @ndir:	ntfs inode of current directory
+ * @ia_page:	page in which the index allocation buffer @ie is in resides
+ * @ie:		current index entry
+ * @name:	buffer to use for the converted name
+ * @actor:	what to feed the entries to
+ *
+ * Convert the Unicode @name to the loaded NLS and pass it to the @filldir
+ * callback.
+ *
+ * If @ia_page is not NULL it is the locked page containing the index
+ * allocation block containing the index entry @ie.
+ *
+ * Note, we drop (and then reacquire) the page lock on @ia_page across the
+ * @filldir() call otherwise we would deadlock with NFSd when it calls ->lookup
+ * since ntfs_lookup() will lock the same page.  As an optimization, we do not
+ * retake the lock if we are returning a non-zero value as ntfs_readdir()
+ * would need to drop the lock immediately anyway.
+ */
+static inline int ntfs_filldir(struct ntfs_volume *vol,
+		struct ntfs_inode *ndir, struct page *ia_page, struct index_entry *ie,
+		u8 *name, struct dir_context *actor)
+{
+	unsigned long mref;
+	int name_len;
+	unsigned int dt_type;
+	u8 name_type;
+
+	name_type = ie->key.file_name.file_name_type;
+	if (name_type == FILE_NAME_DOS) {
+		ntfs_debug("Skipping DOS name space entry.");
+		return 0;
+	}
+	if (MREF_LE(ie->data.dir.indexed_file) == FILE_root) {
+		ntfs_debug("Skipping root directory self reference entry.");
+		return 0;
+	}
+	if (MREF_LE(ie->data.dir.indexed_file) < FILE_first_user &&
+			!NVolShowSystemFiles(vol)) {
+		ntfs_debug("Skipping system file.");
+		return 0;
+	}
+	if (!NVolShowHiddenFiles(vol) &&
+	    (ie->key.file_name.file_attributes & FILE_ATTR_HIDDEN)) {
+		ntfs_debug("Skipping hidden file.");
+		return 0;
+	}
+
+	name_len = ntfs_ucstonls(vol, (__le16 *)&ie->key.file_name.file_name,
+			ie->key.file_name.file_name_length, &name,
+			NTFS_MAX_NAME_LEN * NLS_MAX_CHARSET_SIZE + 1);
+	if (name_len <= 0) {
+		ntfs_warning(vol->sb, "Skipping unrepresentable inode 0x%llx.",
+				(long long)MREF_LE(ie->data.dir.indexed_file));
+		return 0;
+	}
+
+	mref = MREF_LE(ie->data.dir.indexed_file);
+	if (ie->key.file_name.file_attributes &
+			FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT)
+		dt_type = DT_DIR;
+	else if (ie->key.file_name.file_attributes & FILE_ATTR_REPARSE_POINT)
+		dt_type = ntfs_reparse_tag_dt_types(vol, mref);
+	else
+		dt_type = DT_REG;
+
+	/*
+	 * Drop the page lock otherwise we deadlock with NFS when it calls
+	 * ->lookup since ntfs_lookup() will lock the same page.
+	 */
+	if (ia_page)
+		unlock_page(ia_page);
+	ntfs_debug("Calling filldir for %s with len %i, fpos 0x%llx, inode 0x%lx, DT_%s.",
+		name, name_len, actor->pos, mref, dt_type == DT_DIR ? "DIR" : "REG");
+	if (!dir_emit(actor, name, name_len, mref, dt_type))
+		return 1;
+	/* Relock the page but not if we are aborting ->readdir. */
+	if (ia_page)
+		lock_page(ia_page);
+	return 0;
+}
+
+struct ntfs_file_private {
+	void *key;
+	__le16 key_length;
+	bool end_in_iterate;
+	loff_t curr_pos;
+};
+
+struct ntfs_index_ra {
+	unsigned long start_index;
+	unsigned int count;
+	struct rb_node rb_node;
+};
+
+static void ntfs_insert_rb(struct ntfs_index_ra *nir, struct rb_root *root)
+{
+	struct rb_node **new = &root->rb_node, *parent = NULL;
+	struct ntfs_index_ra *cnir;
+
+	while (*new) {
+		parent = *new;
+		cnir = rb_entry(parent, struct ntfs_index_ra, rb_node);
+		if (nir->start_index < cnir->start_index)
+			new = &parent->rb_left;
+		else if (nir->start_index >= cnir->start_index + cnir->count)
+			new = &parent->rb_right;
+		else {
+			pr_err("nir start index : %ld, count : %d, cnir start_index : %ld, count : %d\n",
+				nir->start_index, nir->count, cnir->start_index, cnir->count);
+			return;
+		}
+	}
+
+	rb_link_node(&nir->rb_node, parent, new);
+	rb_insert_color(&nir->rb_node, root);
+}
+
+static int ntfs_ia_blocks_readahead(struct ntfs_inode *ia_ni, loff_t pos)
+{
+	unsigned long dir_start_index, dir_end_index;
+	struct inode *ia_vi = VFS_I(ia_ni);
+	struct file_ra_state *dir_ra;
+
+	dir_end_index = (i_size_read(ia_vi) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	dir_start_index = (pos + PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+	if (dir_start_index >= dir_end_index)
+		return 0;
+
+	dir_ra = kzalloc(sizeof(*dir_ra), GFP_NOFS);
+	if (!dir_ra)
+		return -ENOMEM;
+
+	file_ra_state_init(dir_ra, ia_vi->i_mapping);
+	dir_end_index = (i_size_read(ia_vi) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	dir_start_index = (pos + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	dir_ra->ra_pages = dir_end_index - dir_start_index;
+	page_cache_sync_readahead(ia_vi->i_mapping, dir_ra, NULL,
+			dir_start_index, dir_end_index - dir_start_index);
+	kfree(dir_ra);
+
+	return 0;
+}
+
+static int ntfs_readdir(struct file *file, struct dir_context *actor)
+{
+	struct inode *vdir = file_inode(file);
+	struct super_block *sb = vdir->i_sb;
+	struct ntfs_inode *ndir = NTFS_I(vdir);
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct ntfs_index_context *ictx = NULL;
+	u8 *name;
+	struct index_root *ir;
+	struct index_entry *next = NULL;
+	struct ntfs_file_private *private = NULL;
+	int err = 0;
+	loff_t ie_pos = 2; /* initialize it with dot and dotdot size */
+	struct ntfs_index_ra *nir = NULL;
+	unsigned long index;
+	struct rb_root ra_root = RB_ROOT;
+	struct file_ra_state *ra;
+
+	ntfs_debug("Entering for inode 0x%llx, fpos 0x%llx.",
+			ndir->mft_no, actor->pos);
+
+	if (file->private_data) {
+		private = file->private_data;
+
+		if (actor->pos != private->curr_pos) {
+			/*
+			 * If actor->pos is different from the previous passed
+			 * one, Discard the private->key and fill dirent buffer
+			 * with linear lookup.
+			 */
+			kfree(private->key);
+			private->key = NULL;
+			private->end_in_iterate = false;
+		} else if (private->end_in_iterate) {
+			kfree(private->key);
+			kfree(file->private_data);
+			file->private_data = NULL;
+			return 0;
+		}
+	}
+
+	/* Emulate . and .. for all directories. */
+	if (!dir_emit_dots(file, actor))
+		return 0;
+
+	/*
+	 * Allocate a buffer to store the current name being processed
+	 * converted to format determined by current NLS.
+	 */
+	name = kmalloc(NTFS_MAX_NAME_LEN * NLS_MAX_CHARSET_SIZE + 1, GFP_NOFS);
+	if (unlikely(!name))
+		return -ENOMEM;
+
+	mutex_lock_nested(&ndir->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+	ictx = ntfs_index_ctx_get(ndir, I30, 4);
+	if (!ictx) {
+		kfree(name);
+		mutex_unlock(&ndir->mrec_lock);
+		return -ENOMEM;
+	}
+
+	ra = kzalloc(sizeof(struct file_ra_state), GFP_NOFS);
+	if (!ra) {
+		kfree(name);
+		ntfs_index_ctx_put(ictx);
+		mutex_unlock(&ndir->mrec_lock);
+		return -ENOMEM;
+	}
+	file_ra_state_init(ra, vol->mft_ino->i_mapping);
+
+	if (private && private->key) {
+		/*
+		 * Find index witk private->key using ntfs_index_lookup()
+		 * instead of linear index lookup.
+		 */
+		err = ntfs_index_lookup(private->key,
+					le16_to_cpu(private->key_length),
+					ictx);
+		if (!err) {
+			next = ictx->entry;
+			/*
+			 * Update ie_pos with private->curr_pos
+			 * to make next d_off of dirent correct.
+			 */
+			ie_pos = private->curr_pos;
+
+			if (actor->pos > vol->mft_record_size && ictx->ia_ni) {
+				err = ntfs_ia_blocks_readahead(ictx->ia_ni, actor->pos);
+				if (err)
+					goto out;
+			}
+
+			goto nextdir;
+		} else {
+			goto out;
+		}
+	} else if (!private) {
+		private = kzalloc(sizeof(struct ntfs_file_private), GFP_KERNEL);
+		if (!private) {
+			err = -ENOMEM;
+			goto out;
+		}
+		file->private_data = private;
+	}
+
+	ctx = ntfs_attr_get_search_ctx(ndir, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	/* Find the index root attribute in the mft record. */
+	if (ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL, 0,
+				ctx)) {
+		ntfs_error(sb, "Index root attribute missing in directory inode %llu",
+				ndir->mft_no);
+		ntfs_attr_put_search_ctx(ctx);
+		err = -ENOMEM;
+		goto out;
+	}
+
+	/* Get to the index root value. */
+	ir = (struct index_root *)((u8 *)ctx->attr +
+			le16_to_cpu(ctx->attr->data.resident.value_offset));
+
+	ictx->ir = ir;
+	ictx->actx = ctx;
+	ictx->parent_vcn[ictx->pindex] = VCN_INDEX_ROOT_PARENT;
+	ictx->is_in_root = true;
+	ictx->parent_pos[ictx->pindex] = 0;
+
+	ictx->block_size = le32_to_cpu(ir->index_block_size);
+	if (ictx->block_size < NTFS_BLOCK_SIZE) {
+		ntfs_error(sb, "Index block size (%d) is smaller than the sector size (%d)",
+				ictx->block_size, NTFS_BLOCK_SIZE);
+		err = -EIO;
+		goto out;
+	}
+
+	if (vol->cluster_size <= ictx->block_size)
+		ictx->vcn_size_bits = vol->cluster_size_bits;
+	else
+		ictx->vcn_size_bits = NTFS_BLOCK_SIZE_BITS;
+
+	/* The first index entry. */
+	next = (struct index_entry *)((u8 *)&ir->index +
+			le32_to_cpu(ir->index.entries_offset));
+
+	if (next->flags & INDEX_ENTRY_NODE) {
+		ictx->ia_ni = ntfs_ia_open(ictx, ictx->idx_ni);
+		if (!ictx->ia_ni) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		err = ntfs_ia_blocks_readahead(ictx->ia_ni, actor->pos);
+		if (err)
+			goto out;
+	}
+
+	if (next->flags & INDEX_ENTRY_NODE) {
+		next = ntfs_index_walk_down(next, ictx);
+		if (!next) {
+			err = -EIO;
+			goto out;
+		}
+	}
+
+	if (next && !(next->flags & INDEX_ENTRY_END))
+		goto nextdir;
+
+	while ((next = ntfs_index_next(next, ictx)) != NULL) {
+nextdir:
+		/* Check the consistency of an index entry */
+		if (ntfs_index_entry_inconsistent(ictx, vol, next, COLLATION_FILE_NAME,
+					ndir->mft_no)) {
+			err = -EIO;
+			goto out;
+		}
+
+		if (ie_pos < actor->pos) {
+			ie_pos += le16_to_cpu(next->length);
+			continue;
+		}
+
+		actor->pos = ie_pos;
+
+		index = ntfs_mft_no_to_pidx(vol,
+				MREF_LE(next->data.dir.indexed_file));
+		if (nir) {
+			struct ntfs_index_ra *cnir;
+			struct rb_node *node = ra_root.rb_node;
+
+			if (nir->start_index <= index &&
+			    index < nir->start_index + nir->count) {
+				/* No behavior */
+				goto filldir;
+			}
+
+			while (node) {
+				cnir = rb_entry(node, struct ntfs_index_ra, rb_node);
+				if (cnir->start_index <= index &&
+				    index < cnir->start_index + cnir->count) {
+					goto filldir;
+				} else if (cnir->start_index + cnir->count == index) {
+					cnir->count++;
+					goto filldir;
+				} else if (!cnir->start_index && cnir->start_index - 1 == index) {
+					cnir->start_index = index;
+					goto filldir;
+				}
+
+				if (index < cnir->start_index)
+					node = node->rb_left;
+				else if (index >= cnir->start_index + cnir->count)
+					node = node->rb_right;
+			}
+
+			if (nir->start_index + nir->count == index) {
+				nir->count++;
+			} else if (!nir->start_index && nir->start_index - 1 == index) {
+				nir->start_index = index;
+			} else if (nir->count > 2) {
+				ntfs_insert_rb(nir, &ra_root);
+				nir = NULL;
+			} else {
+				nir->start_index = index;
+				nir->count = 1;
+			}
+		}
+
+		if (!nir) {
+			nir = kzalloc(sizeof(struct ntfs_index_ra), GFP_KERNEL);
+			if (nir) {
+				nir->start_index = index;
+				nir->count = 1;
+			}
+		}
+
+filldir:
+		/* Submit the name to the filldir callback. */
+		err = ntfs_filldir(vol, ndir, NULL, next, name, actor);
+		if (err) {
+			/*
+			 * Store index key value to file private_data to start
+			 * from current index offset on next round.
+			 */
+			private = file->private_data;
+			kfree(private->key);
+			private->key = kmalloc(le16_to_cpu(next->key_length), GFP_KERNEL);
+			if (!private->key) {
+				err = -ENOMEM;
+				goto out;
+			}
+
+			memcpy(private->key, &next->key.file_name, le16_to_cpu(next->key_length));
+			private->key_length = next->key_length;
+			break;
+		}
+		ie_pos += le16_to_cpu(next->length);
+	}
+
+	if (!err)
+		private->end_in_iterate = true;
+	else
+		err = 0;
+
+	private->curr_pos = actor->pos = ie_pos;
+out:
+	while (!RB_EMPTY_ROOT(&ra_root)) {
+		struct ntfs_index_ra *cnir;
+		struct rb_node *node;
+
+		node = rb_first(&ra_root);
+		cnir = rb_entry(node, struct ntfs_index_ra, rb_node);
+		ra->ra_pages = cnir->count;
+		page_cache_sync_readahead(vol->mft_ino->i_mapping, ra, NULL,
+				cnir->start_index, cnir->count);
+		rb_erase(node, &ra_root);
+		kfree(cnir);
+	}
+
+	if (err) {
+		if (private) {
+			private->curr_pos = actor->pos;
+			private->end_in_iterate = true;
+		}
+		err = 0;
+	}
+	ntfs_index_ctx_put(ictx);
+	kfree(name);
+	kfree(nir);
+	kfree(ra);
+	mutex_unlock(&ndir->mrec_lock);
+	return err;
+}
+
+int ntfs_check_empty_dir(struct ntfs_inode *ni, struct mft_record *ni_mrec)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	int ret = 0;
+
+	if (!(ni_mrec->flags & MFT_RECORD_IS_DIRECTORY))
+		return 0;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ntfs_error(ni->vol->sb, "Failed to get search context");
+		return -ENOMEM;
+	}
+
+	/* Find the index root attribute in the mft record. */
+	ret = ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL,
+				0, ctx);
+	if (ret) {
+		ntfs_error(ni->vol->sb, "Index root attribute missing in directory inode %llu",
+				ni->mft_no);
+		ntfs_attr_put_search_ctx(ctx);
+		return ret;
+	}
+
+	/* Non-empty directory? */
+	if (le32_to_cpu(ctx->attr->data.resident.value_length) !=
+	    sizeof(struct index_root) + sizeof(struct index_entry_header)) {
+		/* Both ENOTEMPTY and EEXIST are ok. We use the more common. */
+		ret = -ENOTEMPTY;
+		ntfs_debug("Directory is not empty\n");
+	}
+
+	ntfs_attr_put_search_ctx(ctx);
+
+	return ret;
+}
+
+/*
+ * ntfs_dir_open - called when an inode is about to be opened
+ * @vi:		inode to be opened
+ * @filp:	file structure describing the inode
+ *
+ * Limit directory size to the page cache limit on architectures where unsigned
+ * long is 32-bits. This is the most we can do for now without overflowing the
+ * page cache page index. Doing it this way means we don't run into problems
+ * because of existing too large directories. It would be better to allow the
+ * user to read the accessible part of the directory but I doubt very much
+ * anyone is going to hit this check on a 32-bit architecture, so there is no
+ * point in adding the extra complexity required to support this.
+ *
+ * On 64-bit architectures, the check is hopefully optimized away by the
+ * compiler.
+ */
+static int ntfs_dir_open(struct inode *vi, struct file *filp)
+{
+	if (sizeof(unsigned long) < 8) {
+		if (i_size_read(vi) > MAX_LFS_FILESIZE)
+			return -EFBIG;
+	}
+	return 0;
+}
+
+static int ntfs_dir_release(struct inode *vi, struct file *filp)
+{
+	if (filp->private_data) {
+		kfree(((struct ntfs_file_private *)filp->private_data)->key);
+		kfree(filp->private_data);
+		filp->private_data = NULL;
+	}
+	return 0;
+}
+
+/*
+ * ntfs_dir_fsync - sync a directory to disk
+ * @filp:	file describing the directory to be synced
+ * @start:	start offset to be synced
+ * @end:	end offset to be synced
+ * @datasync:	if non-zero only flush user data and not metadata
+ *
+ * Data integrity sync of a directory to disk.  Used for fsync, fdatasync, and
+ * msync system calls.  This function is based on file.c::ntfs_file_fsync().
+ *
+ * Write the mft record and all associated extent mft records as well as the
+ * $INDEX_ALLOCATION and $BITMAP attributes and then sync the block device.
+ *
+ * If @datasync is true, we do not wait on the inode(s) to be written out
+ * but we always wait on the page cache pages to be written out.
+ *
+ * Note: In the past @filp could be NULL so we ignore it as we don't need it
+ * anyway.
+ *
+ * Locking: Caller must hold i_mutex on the inode.
+ */
+static int ntfs_dir_fsync(struct file *filp, loff_t start, loff_t end,
+			  int datasync)
+{
+	struct inode *bmp_vi, *vi = filp->f_mapping->host;
+	struct ntfs_volume *vol = NTFS_I(vi)->vol;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_attr_search_ctx *ctx;
+	struct inode *parent_vi, *ia_vi;
+	int err, ret;
+	struct ntfs_attr na;
+
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_CHILD);
+	while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0, ctx))) {
+		struct file_name_attr *fn = (struct file_name_attr *)((u8 *)ctx->attr +
+				le16_to_cpu(ctx->attr->data.resident.value_offset));
+
+		if (MREF_LE(fn->parent_directory) == ni->mft_no)
+			continue;
+
+		parent_vi = ntfs_iget(vi->i_sb, MREF_LE(fn->parent_directory));
+		if (IS_ERR(parent_vi))
+			continue;
+		mutex_lock_nested(&NTFS_I(parent_vi)->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+		ia_vi = ntfs_index_iget(parent_vi, I30, 4);
+		mutex_unlock(&NTFS_I(parent_vi)->mrec_lock);
+		if (IS_ERR(ia_vi)) {
+			iput(parent_vi);
+			continue;
+		}
+		write_inode_now(ia_vi, 1);
+		iput(ia_vi);
+		write_inode_now(parent_vi, 1);
+		iput(parent_vi);
+	}
+	mutex_unlock(&ni->mrec_lock);
+	ntfs_attr_put_search_ctx(ctx);
+
+	err = file_write_and_wait_range(filp, start, end);
+	if (err)
+		return err;
+	inode_lock(vi);
+
+	/* If the bitmap attribute inode is in memory sync it, too. */
+	na.mft_no = vi->i_ino;
+	na.type = AT_BITMAP;
+	na.name = I30;
+	na.name_len = 4;
+	bmp_vi = ilookup5(vi->i_sb, vi->i_ino, ntfs_test_inode, &na);
+	if (bmp_vi) {
+		write_inode_now(bmp_vi, !datasync);
+		iput(bmp_vi);
+	}
+	ret = __ntfs_write_inode(vi, 1);
+
+	write_inode_now(vi, !datasync);
+
+	write_inode_now(vol->mftbmp_ino, 1);
+	down_write(&vol->lcnbmp_lock);
+	write_inode_now(vol->lcnbmp_ino, 1);
+	up_write(&vol->lcnbmp_lock);
+	write_inode_now(vol->mft_ino, 1);
+
+	err = sync_blockdev(vi->i_sb->s_bdev);
+	if (unlikely(err && !ret))
+		ret = err;
+	if (likely(!ret))
+		ntfs_debug("Done.");
+	else
+		ntfs_warning(vi->i_sb,
+			"Failed to f%ssync inode 0x%llx.  Error %u.",
+			datasync ? "data" : "", ni->mft_no, -ret);
+	inode_unlock(vi);
+	return ret;
+}
+
+const struct file_operations ntfs_dir_ops = {
+	.llseek		= generic_file_llseek,	/* Seek inside directory. */
+	.read		= generic_read_dir,	/* Return -EISDIR. */
+	.iterate_shared	= ntfs_readdir,		/* Read directory contents. */
+	.fsync		= ntfs_dir_fsync,	/* Sync a directory to disk. */
+	.open		= ntfs_dir_open,	/* Open directory. */
+	.release	= ntfs_dir_release,
+	.unlocked_ioctl	= ntfs_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= ntfs_compat_ioctl,
+#endif
+	.setlease	= generic_setlease,
+};
diff --git a/fs/ntfs/dir.h b/fs/ntfs/dir.h
new file mode 100644
index 0000000000000..a38c6b0716783
--- /dev/null
+++ b/fs/ntfs/dir.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for directory handling in NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2002-2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_DIR_H
+#define _LINUX_NTFS_DIR_H
+
+#include "inode.h"
+
+/*
+ * ntfs_name is used to return the file name to the caller of
+ * ntfs_lookup_inode_by_name() in order for the caller (namei.c::ntfs_lookup())
+ * to be able to deal with dcache aliasing issues.
+ */
+struct ntfs_name {
+	u64 mref;
+	u8 type;
+	u8 len;
+	__le16 name[];
+} __packed;
+
+/* The little endian Unicode string $I30 as a global constant. */
+extern __le16 I30[5];
+
+u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni,
+		const __le16 *uname, const int uname_len, struct ntfs_name **res);
+int ntfs_check_empty_dir(struct ntfs_inode *ni, struct mft_record *ni_mrec);
+
+#endif /* _LINUX_NTFS_FS_DIR_H */
diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c
new file mode 100644
index 0000000000000..ee99baf9c7d24
--- /dev/null
+++ b/fs/ntfs/ea.c
@@ -0,0 +1,954 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Pocessing of EA's
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ *
+ * Copyright (c) 2014-2021 Jean-Pierre Andre
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/fs.h>
+#include <linux/posix_acl.h>
+#include <linux/posix_acl_xattr.h>
+#include <linux/xattr.h>
+
+#include "layout.h"
+#include "attrib.h"
+#include "index.h"
+#include "dir.h"
+#include "ea.h"
+
+static int ntfs_write_ea(struct ntfs_inode *ni, __le32 type, char *value, s64 ea_off,
+		s64 ea_size, bool need_truncate)
+{
+	struct inode *ea_vi;
+	int err = 0;
+	s64 written;
+
+	ea_vi = ntfs_attr_iget(VFS_I(ni), type, AT_UNNAMED, 0);
+	if (IS_ERR(ea_vi))
+		return PTR_ERR(ea_vi);
+
+	written = ntfs_inode_attr_pwrite(ea_vi, ea_off, ea_size, value, false);
+	if (written != ea_size)
+		err = -EIO;
+	else {
+		struct ntfs_inode *ea_ni = NTFS_I(ea_vi);
+
+		if (need_truncate && ea_ni->data_size > ea_off + ea_size)
+			ntfs_attr_truncate(ea_ni, ea_off + ea_size);
+		mark_mft_record_dirty(ni);
+	}
+
+	iput(ea_vi);
+	return err;
+}
+
+static int ntfs_ea_lookup(char *ea_buf, s64 ea_buf_size, const char *name,
+			  int name_len, s64 *ea_offset, s64 *ea_size)
+{
+	const struct ea_attr *p_ea;
+	size_t actual_size;
+	loff_t offset, p_ea_size;
+	unsigned int next;
+
+	if (ea_buf_size < sizeof(struct ea_attr))
+		goto out;
+
+	offset = 0;
+	do {
+		p_ea = (const struct ea_attr *)&ea_buf[offset];
+		next = le32_to_cpu(p_ea->next_entry_offset);
+		p_ea_size = next ? next : (ea_buf_size - offset);
+
+		if (p_ea_size < sizeof(struct ea_attr) ||
+		    offset + p_ea_size > ea_buf_size)
+			break;
+
+		if ((s64)p_ea->ea_name_length + 1 >
+		    p_ea_size - offsetof(struct ea_attr, ea_name))
+			break;
+
+		actual_size = ALIGN(struct_size(p_ea, ea_name, 1 + p_ea->ea_name_length +
+					le16_to_cpu(p_ea->ea_value_length)), 4);
+		if (actual_size > p_ea_size)
+			break;
+
+		if (p_ea->ea_name_length == name_len &&
+		    !memcmp(p_ea->ea_name, name, name_len)) {
+			*ea_offset = offset;
+			*ea_size = next ? next : actual_size;
+
+			if (ea_buf_size < *ea_offset + *ea_size)
+				goto out;
+
+			return 0;
+		}
+		offset += next;
+	} while (next > 0 && offset < ea_buf_size);
+
+out:
+	return -ENOENT;
+}
+
+/*
+ * Return the existing EA
+ *
+ * The EA_INFORMATION is not examined and the consistency of the
+ * existing EA is not checked.
+ *
+ * If successful, the full attribute is returned unchanged
+ * and its size is returned.
+ * If the designated buffer is too small, the needed size is
+ * returned, and the buffer is left unchanged.
+ * If there is an error, a negative value is returned and errno
+ * is set according to the error.
+ */
+static int ntfs_get_ea(struct inode *inode, const char *name, size_t name_len,
+		void *buffer, size_t size)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	const struct ea_attr *p_ea;
+	char *ea_buf;
+	s64 ea_off, ea_size, all_ea_size, ea_info_size;
+	int err;
+	u32 ea_info_qlen;
+	u16 ea_value_len;
+	struct ea_information *p_ea_info;
+
+	if (!NInoHasEA(ni))
+		return -ENODATA;
+
+	p_ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0,
+			&ea_info_size);
+	if (!p_ea_info || ea_info_size != sizeof(struct ea_information)) {
+		kvfree(p_ea_info);
+		return -ENODATA;
+	}
+
+	ea_info_qlen = le32_to_cpu(p_ea_info->ea_query_length);
+	kvfree(p_ea_info);
+
+	ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &all_ea_size);
+	if (!ea_buf)
+		return -ENODATA;
+
+	if (ea_info_qlen > all_ea_size) {
+		err = -EIO;
+		goto free_ea_buf;
+	}
+
+	err = ntfs_ea_lookup(ea_buf, ea_info_qlen, name, name_len, &ea_off,
+			&ea_size);
+	if (!err) {
+		p_ea = (struct ea_attr *)&ea_buf[ea_off];
+		ea_value_len = le16_to_cpu(p_ea->ea_value_length);
+		if (!buffer) {
+			kvfree(ea_buf);
+			return ea_value_len;
+		}
+
+		if (ea_value_len > size) {
+			err = -ERANGE;
+			goto free_ea_buf;
+		}
+
+		memcpy(buffer, &p_ea->ea_name[p_ea->ea_name_length + 1],
+				ea_value_len);
+		kvfree(ea_buf);
+		return ea_value_len;
+	}
+
+	err = -ENODATA;
+free_ea_buf:
+	kvfree(ea_buf);
+	return err;
+}
+
+static inline int ea_packed_size(const struct ea_attr *p_ea)
+{
+	/*
+	 * 4 bytes for header (flags and lengths) + name length + 1 +
+	 * value length.
+	 */
+	return 5 + p_ea->ea_name_length + le16_to_cpu(p_ea->ea_value_length);
+}
+
+/*
+ * Set a new EA, and set EA_INFORMATION accordingly
+ *
+ * This is roughly the same as ZwSetEaFile() on Windows, however
+ * the "offset to next" of the last EA should not be cleared.
+ *
+ * Consistency of the new EA is first checked.
+ *
+ * EA_INFORMATION is set first, and it is restored to its former
+ * state if setting EA fails.
+ */
+static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
+		const void *value, size_t val_size, int flags,
+		__le16 *packed_ea_size)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct ea_information *p_ea_info = NULL;
+	int ea_packed, err = 0;
+	struct ea_attr *p_ea;
+	u32 ea_info_qsize = 0;
+	char *ea_buf = NULL;
+	size_t new_ea_size = ALIGN(struct_size(p_ea, ea_name, 1 + name_len + val_size), 4);
+	s64 ea_off, ea_info_size, all_ea_size, ea_size;
+
+	if (name_len > 255)
+		return -ENAMETOOLONG;
+
+	if (ntfs_attr_exist(ni, AT_EA_INFORMATION, AT_UNNAMED, 0)) {
+		p_ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0,
+						&ea_info_size);
+		if (!p_ea_info || ea_info_size != sizeof(struct ea_information))
+			goto out;
+
+		ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &all_ea_size);
+		if (!ea_buf) {
+			ea_info_qsize = 0;
+			kvfree(p_ea_info);
+			goto create_ea_info;
+		}
+
+		ea_info_qsize = le32_to_cpu(p_ea_info->ea_query_length);
+	} else {
+create_ea_info:
+		p_ea_info = kzalloc(sizeof(struct ea_information), GFP_NOFS);
+		if (!p_ea_info)
+			return -ENOMEM;
+
+		ea_info_qsize = 0;
+		err = ntfs_attr_add(ni, AT_EA_INFORMATION, AT_UNNAMED, 0,
+				(char *)p_ea_info, sizeof(struct ea_information));
+		if (err)
+			goto out;
+
+		if (ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) {
+			err = ntfs_attr_remove(ni, AT_EA, AT_UNNAMED, 0);
+			if (err)
+				goto out;
+		}
+
+		goto alloc_new_ea;
+	}
+
+	if (ea_info_qsize > all_ea_size) {
+		err = -EIO;
+		goto out;
+	}
+
+	err = ntfs_ea_lookup(ea_buf, ea_info_qsize, name, name_len, &ea_off,
+			&ea_size);
+	if (ea_info_qsize && !err) {
+		if (flags & XATTR_CREATE) {
+			err = -EEXIST;
+			goto out;
+		}
+
+		p_ea = (struct ea_attr *)(ea_buf + ea_off);
+
+		if (val_size &&
+		    le16_to_cpu(p_ea->ea_value_length) == val_size &&
+		    !memcmp(p_ea->ea_name + p_ea->ea_name_length + 1, value,
+			    val_size))
+			goto out;
+
+		le16_add_cpu(&p_ea_info->ea_length, 0 - ea_packed_size(p_ea));
+
+		if (p_ea->flags & NEED_EA)
+			le16_add_cpu(&p_ea_info->need_ea_count, -1);
+
+		memmove((char *)p_ea, (char *)p_ea + ea_size, ea_info_qsize - (ea_off + ea_size));
+		ea_info_qsize -= ea_size;
+		p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize);
+
+		err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0,
+				sizeof(struct ea_information), false);
+		if (err)
+			goto out;
+
+		err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, true);
+		if (err)
+			goto out;
+
+		if ((flags & XATTR_REPLACE) && !val_size) {
+			/* Remove xattr. */
+			goto out;
+		}
+	} else {
+		if (flags & XATTR_REPLACE) {
+			err = -ENODATA;
+			goto out;
+		}
+	}
+	kvfree(ea_buf);
+
+alloc_new_ea:
+	ea_buf = kzalloc(new_ea_size, GFP_NOFS);
+	if (!ea_buf) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	/*
+	 * EA and REPARSE_POINT compatibility not checked any more,
+	 * required by Windows 10, but having both may lead to
+	 * problems with earlier versions.
+	 */
+	p_ea = (struct ea_attr *)ea_buf;
+	memcpy(p_ea->ea_name, name, name_len);
+	p_ea->ea_name_length = name_len;
+	p_ea->ea_name[name_len] = 0;
+	memcpy(p_ea->ea_name + name_len + 1, value, val_size);
+	p_ea->ea_value_length = cpu_to_le16(val_size);
+	p_ea->next_entry_offset = cpu_to_le32(new_ea_size);
+
+	ea_packed = le16_to_cpu(p_ea_info->ea_length) + ea_packed_size(p_ea);
+	p_ea_info->ea_length = cpu_to_le16(ea_packed);
+	p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize + new_ea_size);
+
+	if (ea_packed > 0xffff ||
+	    ntfs_attr_size_bounds_check(ni->vol, AT_EA, new_ea_size)) {
+		err = -EFBIG;
+		goto out;
+	}
+
+	/*
+	 * no EA or EA_INFORMATION : add them
+	 */
+	if (!ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) {
+		err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, (char *)p_ea,
+				new_ea_size);
+		if (err)
+			goto out;
+	} else {
+		err = ntfs_write_ea(ni, AT_EA, (char *)p_ea, ea_info_qsize,
+				new_ea_size, false);
+		if (err)
+			goto out;
+	}
+
+	err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0,
+			sizeof(struct ea_information), false);
+	if (err)
+		goto out;
+
+	if (packed_ea_size)
+		*packed_ea_size = p_ea_info->ea_length;
+	mark_mft_record_dirty(ni);
+out:
+	if (ea_info_qsize > 0)
+		NInoSetHasEA(ni);
+	else
+		NInoClearHasEA(ni);
+
+	kvfree(ea_buf);
+	kvfree(p_ea_info);
+
+	return err;
+}
+
+/*
+ * Check for the presence of an EA "$LXDEV" (used by WSL)
+ * and return its value as a device address
+ */
+int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags)
+{
+	int err;
+	__le32 v;
+
+	if (!(flags & NTFS_VOL_UID)) {
+		/* Load uid to lxuid EA */
+		err = ntfs_get_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v,
+				sizeof(v));
+		if (err < 0)
+			return err;
+		if (err != sizeof(v))
+			return -EIO;
+		i_uid_write(inode, le32_to_cpu(v));
+	}
+
+	if (!(flags & NTFS_VOL_GID)) {
+		/* Load gid to lxgid EA */
+		err = ntfs_get_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v,
+				sizeof(v));
+		if (err < 0)
+			return err;
+		if (err != sizeof(v))
+			return -EIO;
+		i_gid_write(inode, le32_to_cpu(v));
+	}
+
+	/* Load mode to lxmod EA */
+	err = ntfs_get_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v, sizeof(v));
+	if (err == sizeof(v)) {
+		inode->i_mode = le32_to_cpu(v);
+	} else {
+		/* Everyone gets all permissions. */
+		inode->i_mode |= 0777;
+	}
+
+	/* Load mode to lxdev EA */
+	err = ntfs_get_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, &v, sizeof(v));
+	if (err == sizeof(v))
+		*rdevp = le32_to_cpu(v);
+	err = 0;
+
+	return err;
+}
+
+int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size,
+		unsigned int flags)
+{
+	__le32 v;
+	int err;
+
+	if (flags & NTFS_EA_UID) {
+		/* Store uid to lxuid EA */
+		v = cpu_to_le32(i_uid_read(inode));
+		err = ntfs_set_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v,
+				sizeof(v), 0, ea_size);
+		if (err)
+			return err;
+	}
+
+	if (flags & NTFS_EA_GID) {
+		/* Store gid to lxgid EA */
+		v = cpu_to_le32(i_gid_read(inode));
+		err = ntfs_set_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v,
+				sizeof(v), 0, ea_size);
+		if (err)
+			return err;
+	}
+
+	if (flags & NTFS_EA_MODE) {
+		/* Store mode to lxmod EA */
+		v = cpu_to_le32(inode->i_mode);
+		err = ntfs_set_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v,
+				sizeof(v), 0, ea_size);
+		if (err)
+			return err;
+	}
+
+	if (rdev) {
+		v = cpu_to_le32(rdev);
+		err = ntfs_set_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, &v, sizeof(v),
+				0, ea_size);
+	}
+
+	return err;
+}
+
+ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size)
+{
+	struct inode *inode = d_inode(dentry);
+	struct ntfs_inode *ni = NTFS_I(inode);
+	const struct ea_attr *p_ea;
+	s64 offset, ea_buf_size, ea_info_size;
+	s64 ea_size;
+	u32 next;
+	int err = 0;
+	u32 ea_info_qsize;
+	char *ea_buf = NULL;
+	ssize_t ret = 0;
+	struct ea_information *ea_info;
+
+	if (!NInoHasEA(ni))
+		return 0;
+
+	mutex_lock(&NTFS_I(inode)->mrec_lock);
+	ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0,
+			&ea_info_size);
+	if (!ea_info || ea_info_size != sizeof(struct ea_information))
+		goto out;
+
+	ea_info_qsize = le32_to_cpu(ea_info->ea_query_length);
+
+	ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &ea_buf_size);
+	if (!ea_buf)
+		goto out;
+
+	if (ea_info_qsize > ea_buf_size || ea_info_qsize == 0)
+		goto out;
+
+	if (ea_info_qsize < sizeof(struct ea_attr)) {
+		err = -EIO;
+		goto out;
+	}
+
+	offset = 0;
+	do {
+		p_ea = (const struct ea_attr *)&ea_buf[offset];
+		next = le32_to_cpu(p_ea->next_entry_offset);
+		ea_size = next ? next : (ea_info_qsize - offset);
+
+		if (ea_size < sizeof(struct ea_attr) ||
+		    offset + ea_size > ea_info_qsize) {
+			err = -EIO;
+			goto out;
+		}
+
+		if ((int)p_ea->ea_name_length + 1 >
+			ea_size - offsetof(struct ea_attr, ea_name)) {
+			err = -EIO;
+			goto out;
+		}
+
+		if (buffer) {
+			if (ret + p_ea->ea_name_length + 1 > size) {
+				err = -ERANGE;
+				goto out;
+			}
+
+			memcpy(buffer + ret, p_ea->ea_name, p_ea->ea_name_length);
+			buffer[ret + p_ea->ea_name_length] = 0;
+		}
+
+		ret += p_ea->ea_name_length + 1;
+		offset += ea_size;
+	} while (next > 0 && offset < ea_info_qsize);
+
+out:
+	mutex_unlock(&NTFS_I(inode)->mrec_lock);
+	kvfree(ea_info);
+	kvfree(ea_buf);
+
+	return err ? err : ret;
+}
+
+// clang-format off
+#define SYSTEM_DOS_ATTRIB     "system.dos_attrib"
+#define SYSTEM_NTFS_ATTRIB    "system.ntfs_attrib"
+#define SYSTEM_NTFS_ATTRIB_BE "system.ntfs_attrib_be"
+// clang-format on
+
+static int ntfs_getxattr(const struct xattr_handler *handler,
+		struct dentry *unused, struct inode *inode, const char *name,
+		void *buffer, size_t size)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	int err;
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	if (!strcmp(name, SYSTEM_DOS_ATTRIB)) {
+		if (!buffer) {
+			err = sizeof(u8);
+		} else if (size < sizeof(u8)) {
+			err = -ENODATA;
+		} else {
+			err = sizeof(u8);
+			*(u8 *)buffer = (u8)(le32_to_cpu(ni->flags) & 0x3F);
+		}
+		goto out;
+	}
+
+	if (!strcmp(name, SYSTEM_NTFS_ATTRIB) ||
+	    !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) {
+		if (!buffer) {
+			err = sizeof(u32);
+		} else if (size < sizeof(u32)) {
+			err = -ENODATA;
+		} else {
+			err = sizeof(u32);
+			*(u32 *)buffer = le32_to_cpu(ni->flags);
+			if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE))
+				*(__be32 *)buffer = cpu_to_be32(*(u32 *)buffer);
+		}
+		goto out;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	err = ntfs_get_ea(inode, name, strlen(name), buffer, size);
+	mutex_unlock(&ni->mrec_lock);
+
+out:
+	return err;
+}
+
+static int ntfs_new_attr_flags(struct ntfs_inode *ni, __le32 fattr)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	struct mft_record *m;
+	struct attr_record *a;
+	__le16 new_aflags;
+	int mp_size, mp_ofs, name_ofs, arec_size, err;
+
+	m = map_mft_record(ni);
+	if (IS_ERR(m))
+		return PTR_ERR(m);
+
+	ctx = ntfs_attr_get_search_ctx(ni, m);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (err) {
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	a = ctx->attr;
+	new_aflags = ctx->attr->flags;
+
+	if (fattr & FILE_ATTR_SPARSE_FILE)
+		new_aflags |= ATTR_IS_SPARSE;
+	else
+		new_aflags &= ~ATTR_IS_SPARSE;
+
+	if (fattr & FILE_ATTR_COMPRESSED)
+		new_aflags |= ATTR_IS_COMPRESSED;
+	else
+		new_aflags &= ~ATTR_IS_COMPRESSED;
+
+	if (new_aflags == a->flags)
+		return 0;
+
+	if ((new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) ==
+			  (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) {
+		pr_err("file can't be sparsed and compressed\n");
+		err = -EOPNOTSUPP;
+		goto err_out;
+	}
+
+	if (!a->non_resident)
+		goto out;
+
+	if (a->data.non_resident.data_size) {
+		pr_err("Can't change sparsed/compressed for non-empty file\n");
+		err = -EOPNOTSUPP;
+		goto err_out;
+	}
+
+	if (new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED))
+		name_ofs = (offsetof(struct attr_record,
+				     data.non_resident.compressed_size) +
+					sizeof(a->data.non_resident.compressed_size) + 7) & ~7;
+	else
+		name_ofs = (offsetof(struct attr_record,
+				     data.non_resident.compressed_size) + 7) & ~7;
+
+	mp_size = ntfs_get_size_for_mapping_pairs(ni->vol, ni->runlist.rl, 0, -1, -1);
+	if (unlikely(mp_size < 0)) {
+		err = mp_size;
+		ntfs_debug("Failed to get size for mapping pairs array, error code %i.\n", err);
+		goto err_out;
+	}
+
+	mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7;
+	arec_size = (mp_ofs + mp_size + 7) & ~7;
+
+	err = ntfs_attr_record_resize(m, a, arec_size);
+	if (unlikely(err))
+		goto err_out;
+
+	if (new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) {
+		a->data.non_resident.compression_unit = 0;
+		if (new_aflags & ATTR_IS_COMPRESSED || ni->vol->major_ver < 3)
+			a->data.non_resident.compression_unit = 4;
+		a->data.non_resident.compressed_size = 0;
+		ni->itype.compressed.size = 0;
+		if (a->data.non_resident.compression_unit) {
+			ni->itype.compressed.block_size = 1U <<
+				(a->data.non_resident.compression_unit +
+				 ni->vol->cluster_size_bits);
+			ni->itype.compressed.block_size_bits =
+					ffs(ni->itype.compressed.block_size) -
+					1;
+			ni->itype.compressed.block_clusters = 1U <<
+					a->data.non_resident.compression_unit;
+		} else {
+			ni->itype.compressed.block_size = 0;
+			ni->itype.compressed.block_size_bits = 0;
+			ni->itype.compressed.block_clusters = 0;
+		}
+
+		if (new_aflags & ATTR_IS_SPARSE) {
+			NInoSetSparse(ni);
+			ni->flags |= FILE_ATTR_SPARSE_FILE;
+		}
+
+		if (new_aflags & ATTR_IS_COMPRESSED) {
+			NInoSetCompressed(ni);
+			ni->flags |= FILE_ATTR_COMPRESSED;
+		}
+	} else {
+		ni->flags &= ~(FILE_ATTR_SPARSE_FILE | FILE_ATTR_COMPRESSED);
+		a->data.non_resident.compression_unit = 0;
+		NInoClearSparse(ni);
+		NInoClearCompressed(ni);
+	}
+
+	a->name_offset = cpu_to_le16(name_ofs);
+	a->data.non_resident.mapping_pairs_offset = cpu_to_le16(mp_ofs);
+
+out:
+	a->flags = new_aflags;
+	mark_mft_record_dirty(ctx->ntfs_ino);
+err_out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(ni);
+	return err;
+}
+
+static int ntfs_setxattr(const struct xattr_handler *handler,
+		struct mnt_idmap *idmap, struct dentry *unused,
+		struct inode *inode, const char *name, const void *value,
+		size_t size, int flags)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	int err;
+	__le32 fattr;
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	if (!strcmp(name, SYSTEM_DOS_ATTRIB)) {
+		if (sizeof(u8) != size) {
+			err = -EINVAL;
+			goto out;
+		}
+		fattr = cpu_to_le32(*(u8 *)value);
+		goto set_fattr;
+	}
+
+	if (!strcmp(name, SYSTEM_NTFS_ATTRIB) ||
+	    !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) {
+		if (size != sizeof(u32)) {
+			err = -EINVAL;
+			goto out;
+		}
+		if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE))
+			fattr = cpu_to_le32(be32_to_cpu(*(__be32 *)value));
+		else
+			fattr = cpu_to_le32(*(u32 *)value);
+
+		if (S_ISREG(inode->i_mode)) {
+			mutex_lock(&ni->mrec_lock);
+			err = ntfs_new_attr_flags(ni, fattr);
+			mutex_unlock(&ni->mrec_lock);
+			if (err)
+				goto out;
+		}
+
+set_fattr:
+		if (S_ISDIR(inode->i_mode))
+			fattr |= FILE_ATTR_DIRECTORY;
+		else
+			fattr &= ~FILE_ATTR_DIRECTORY;
+
+		if (ni->flags != fattr) {
+			ni->flags = fattr;
+			if (fattr & FILE_ATTR_READONLY)
+				inode->i_mode &= ~0222;
+			else
+				inode->i_mode |= 0222;
+			NInoSetFileNameDirty(ni);
+			mark_inode_dirty(inode);
+		}
+		err = 0;
+		goto out;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	err = ntfs_set_ea(inode, name, strlen(name), value, size, flags, NULL);
+	mutex_unlock(&ni->mrec_lock);
+
+out:
+	inode_set_ctime_current(inode);
+	mark_inode_dirty(inode);
+	return err;
+}
+
+static bool ntfs_xattr_user_list(struct dentry *dentry)
+{
+	return true;
+}
+
+// clang-format off
+static const struct xattr_handler ntfs_other_xattr_handler = {
+	.prefix	= "",
+	.get	= ntfs_getxattr,
+	.set	= ntfs_setxattr,
+	.list	= ntfs_xattr_user_list,
+};
+
+const struct xattr_handler * const ntfs_xattr_handlers[] = {
+	&ntfs_other_xattr_handler,
+	NULL,
+};
+// clang-format on
+
+#ifdef CONFIG_NTFS_FS_POSIX_ACL
+struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+			       int type)
+{
+	struct inode *inode = d_inode(dentry);
+	struct ntfs_inode *ni = NTFS_I(inode);
+	const char *name;
+	size_t name_len;
+	struct posix_acl *acl;
+	int err;
+	void *buf;
+
+	buf = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+
+	/* Possible values of 'type' was already checked above. */
+	if (type == ACL_TYPE_ACCESS) {
+		name = XATTR_NAME_POSIX_ACL_ACCESS;
+		name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1;
+	} else {
+		name = XATTR_NAME_POSIX_ACL_DEFAULT;
+		name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	err = ntfs_get_ea(inode, name, name_len, buf, PATH_MAX);
+	mutex_unlock(&ni->mrec_lock);
+
+	/* Translate extended attribute to acl. */
+	if (err >= 0)
+		acl = posix_acl_from_xattr(&init_user_ns, buf, err);
+	else if (err == -ENODATA)
+		acl = NULL;
+	else
+		acl = ERR_PTR(err);
+
+	if (!IS_ERR(acl))
+		set_cached_acl(inode, type, acl);
+
+	kfree(buf);
+
+	return acl;
+}
+
+static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
+				    struct inode *inode, struct posix_acl *acl,
+				    int type, bool init_acl)
+{
+	const char *name;
+	size_t size, name_len;
+	void *value;
+	int err;
+	int flags;
+	umode_t mode;
+
+	if (S_ISLNK(inode->i_mode))
+		return -EOPNOTSUPP;
+
+	mode = inode->i_mode;
+	switch (type) {
+	case ACL_TYPE_ACCESS:
+		/* Do not change i_mode if we are in init_acl */
+		if (acl && !init_acl) {
+			err = posix_acl_update_mode(idmap, inode, &mode, &acl);
+			if (err)
+				return err;
+		}
+		name = XATTR_NAME_POSIX_ACL_ACCESS;
+		name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1;
+		break;
+
+	case ACL_TYPE_DEFAULT:
+		if (!S_ISDIR(inode->i_mode))
+			return acl ? -EACCES : 0;
+		name = XATTR_NAME_POSIX_ACL_DEFAULT;
+		name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if (!acl) {
+		/* Remove xattr if it can be presented via mode. */
+		size = 0;
+		value = NULL;
+		flags = XATTR_REPLACE;
+	} else {
+		value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_NOFS);
+		if (!value)
+			return -ENOMEM;
+		flags = 0;
+	}
+
+	mutex_lock(&NTFS_I(inode)->mrec_lock);
+	err = ntfs_set_ea(inode, name, name_len, value, size, flags, NULL);
+	mutex_unlock(&NTFS_I(inode)->mrec_lock);
+	if (err == -ENODATA && !size)
+		err = 0; /* Removing non existed xattr. */
+	if (!err) {
+		__le16 ea_size = 0;
+		umode_t old_mode = inode->i_mode;
+
+		inode->i_mode = mode;
+		mutex_lock(&NTFS_I(inode)->mrec_lock);
+		err = ntfs_ea_set_wsl_inode(inode, 0, &ea_size, NTFS_EA_MODE);
+		if (err) {
+			ntfs_set_ea(inode, name, name_len, NULL, 0,
+				    XATTR_REPLACE, NULL);
+			mutex_unlock(&NTFS_I(inode)->mrec_lock);
+			inode->i_mode = old_mode;
+			goto out;
+		}
+		mutex_unlock(&NTFS_I(inode)->mrec_lock);
+
+		set_cached_acl(inode, type, acl);
+		inode_set_ctime_current(inode);
+		mark_inode_dirty(inode);
+	}
+
+out:
+	kfree(value);
+
+	return err;
+}
+
+int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+		 struct posix_acl *acl, int type)
+{
+	return ntfs_set_acl_ex(idmap, d_inode(dentry), acl, type, false);
+}
+
+int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+		  struct inode *dir)
+{
+	struct posix_acl *default_acl, *acl;
+	int err;
+
+	err = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl);
+	if (err)
+		return err;
+
+	if (default_acl) {
+		err = ntfs_set_acl_ex(idmap, inode, default_acl,
+				      ACL_TYPE_DEFAULT, true);
+		posix_acl_release(default_acl);
+	} else {
+		inode->i_default_acl = NULL;
+	}
+
+	if (acl) {
+		if (!err)
+			err = ntfs_set_acl_ex(idmap, inode, acl,
+					      ACL_TYPE_ACCESS, true);
+		posix_acl_release(acl);
+	} else {
+		inode->i_acl = NULL;
+	}
+
+	return err;
+}
+#endif
diff --git a/fs/ntfs/ea.h b/fs/ntfs/ea.h
new file mode 100644
index 0000000000000..1f63bd55e057b
--- /dev/null
+++ b/fs/ntfs/ea.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef _LINUX_NTFS_EA_H
+#define _LINUX_NTFS_EA_H
+
+#define NTFS_EA_UID	BIT(1)
+#define NTFS_EA_GID	BIT(2)
+#define NTFS_EA_MODE	BIT(3)
+
+extern const struct xattr_handler *const ntfs_xattr_handlers[];
+
+int ntfs_ea_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode, dev_t dev);
+int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags);
+int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size,
+		unsigned int flags);
+ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size);
+
+#ifdef CONFIG_NTFS_FS_POSIX_ACL
+struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+			       int type);
+int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+		 struct posix_acl *acl, int type);
+int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+		  struct inode *dir);
+#else
+#define ntfs_get_acl NULL
+#define ntfs_set_acl NULL
+#endif
+
+#endif /* _LINUX_NTFS_EA_H */
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
new file mode 100644
index 0000000000000..e5b897a6c1e15
--- /dev/null
+++ b/fs/ntfs/file.c
@@ -0,0 +1,1161 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel file operations.
+ *
+ * Copyright (c) 2001-2015 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/writeback.h>
+#include <linux/blkdev.h>
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include <linux/uio.h>
+#include <linux/posix_acl.h>
+#include <linux/posix_acl_xattr.h>
+#include <linux/compat.h>
+#include <linux/falloc.h>
+
+#include "lcnalloc.h"
+#include "ntfs.h"
+#include "reparse.h"
+#include "ea.h"
+#include "iomap.h"
+#include "bitmap.h"
+
+#include <linux/filelock.h>
+
+/*
+ * ntfs_file_open - called when an inode is about to be opened
+ * @vi:		inode to be opened
+ * @filp:	file structure describing the inode
+ *
+ * Limit file size to the page cache limit on architectures where unsigned long
+ * is 32-bits. This is the most we can do for now without overflowing the page
+ * cache page index. Doing it this way means we don't run into problems because
+ * of existing too large files. It would be better to allow the user to read
+ * the beginning of the file but I doubt very much anyone is going to hit this
+ * check on a 32-bit architecture, so there is no point in adding the extra
+ * complexity required to support this.
+ *
+ * On 64-bit architectures, the check is hopefully optimized away by the
+ * compiler.
+ *
+ * After the check passes, just call generic_file_open() to do its work.
+ */
+static int ntfs_file_open(struct inode *vi, struct file *filp)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	if (sizeof(unsigned long) < 8) {
+		if (i_size_read(vi) > MAX_LFS_FILESIZE)
+			return -EOVERFLOW;
+	}
+
+	filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
+
+	return generic_file_open(vi, filp);
+}
+
+/*
+ * Trim preallocated space on file release.
+ *
+ * When the preallo_size mount option is set (default 64KB), writes extend
+ * allocated_size and runlist in units of preallocated size to reduce
+ * runlist merge overhead for small writes. This can leave
+ * allocated_size > data_size if not all preallocated space is used.
+ *
+ * We perform the trim here because ->release() is called only when
+ * the file is no longer open. At this point, no further writes can occur,
+ * so it is safe to reclaim the unused preallocated space.
+ *
+ * Returns 0 on success, or negative error on failure.
+ */
+static int ntfs_trim_prealloc(struct inode *vi)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+	struct runlist_element *rl;
+	s64 aligned_data_size;
+	s64 vcn_ds, vcn_tr;
+	ssize_t rc;
+	int err = 0;
+
+	inode_lock(vi);
+	mutex_lock(&ni->mrec_lock);
+	down_write(&ni->runlist.lock);
+
+	aligned_data_size = round_up(ni->data_size, vol->cluster_size);
+	if (aligned_data_size >= ni->allocated_size)
+		goto out_unlock;
+
+	vcn_ds = ntfs_bytes_to_cluster(vol, aligned_data_size);
+	vcn_tr = -1;
+	rc = ni->runlist.count - 2;
+	rl = ni->runlist.rl;
+
+	while (rc >= 0 && rl[rc].lcn == LCN_HOLE && vcn_ds <= rl[rc].vcn) {
+		vcn_tr = rl[rc].vcn;
+		rc--;
+	}
+
+	if (vcn_tr >= 0) {
+		err = ntfs_rl_truncate_nolock(vol, &ni->runlist, vcn_tr);
+		if (err) {
+			kvfree(ni->runlist.rl);
+			ni->runlist.rl = NULL;
+			ntfs_error(vol->sb, "Preallocated block rollback failed");
+		} else {
+			ni->allocated_size = ntfs_cluster_to_bytes(vol, vcn_tr);
+			err = ntfs_attr_update_mapping_pairs(ni, 0);
+			if (err)
+				ntfs_error(vol->sb,
+					   "Failed to rollback mapping pairs for prealloc");
+		}
+	}
+
+out_unlock:
+	up_write(&ni->runlist.lock);
+	mutex_unlock(&ni->mrec_lock);
+	inode_unlock(vi);
+
+	return err;
+}
+
+static int ntfs_file_release(struct inode *vi, struct file *filp)
+{
+	if (!NInoCompressed(NTFS_I(vi)))
+		return ntfs_trim_prealloc(vi);
+
+	return 0;
+}
+
+/*
+ * ntfs_file_fsync - sync a file to disk
+ * @filp:	file to be synced
+ * @start:	start offset to be synced
+ * @end:	end offset to be synced
+ * @datasync:	if non-zero only flush user data and not metadata
+ *
+ * Data integrity sync of a file to disk.  Used for fsync, fdatasync, and msync
+ * system calls.  This function is inspired by fs/buffer.c::file_fsync().
+ *
+ * If @datasync is false, write the mft record and all associated extent mft
+ * records as well as the $DATA attribute and then sync the block device.
+ *
+ * If @datasync is true and the attribute is non-resident, we skip the writing
+ * of the mft record and all associated extent mft records (this might still
+ * happen due to the write_inode_now() call).
+ *
+ * Also, if @datasync is true, we do not wait on the inode to be written out
+ * but we always wait on the page cache pages to be written out.
+ */
+static int ntfs_file_fsync(struct file *filp, loff_t start, loff_t end,
+			   int datasync)
+{
+	struct inode *vi = filp->f_mapping->host;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+	int err, ret = 0;
+	struct inode *parent_vi, *ia_vi;
+	struct ntfs_attr_search_ctx *ctx;
+
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	err = file_write_and_wait_range(filp, start, end);
+	if (err)
+		return err;
+
+	if (!datasync || !NInoNonResident(NTFS_I(vi)))
+		ret = __ntfs_write_inode(vi, 1);
+	write_inode_now(vi, !datasync);
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_CHILD);
+	while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) {
+		if (ctx->attr->type == AT_FILE_NAME) {
+			struct file_name_attr *fn = (struct file_name_attr *)((u8 *)ctx->attr +
+					le16_to_cpu(ctx->attr->data.resident.value_offset));
+
+			parent_vi = ntfs_iget(vi->i_sb, MREF_LE(fn->parent_directory));
+			if (IS_ERR(parent_vi))
+				continue;
+			mutex_lock_nested(&NTFS_I(parent_vi)->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+			ia_vi = ntfs_index_iget(parent_vi, I30, 4);
+			mutex_unlock(&NTFS_I(parent_vi)->mrec_lock);
+			if (IS_ERR(ia_vi)) {
+				iput(parent_vi);
+				continue;
+			}
+			write_inode_now(ia_vi, 1);
+			iput(ia_vi);
+			write_inode_now(parent_vi, 1);
+			iput(parent_vi);
+		} else if (ctx->attr->non_resident) {
+			struct inode *attr_vi;
+			__le16 *name;
+
+			name = (__le16 *)((u8 *)ctx->attr + le16_to_cpu(ctx->attr->name_offset));
+			if (ctx->attr->type == AT_DATA && ctx->attr->name_length == 0)
+				continue;
+
+			attr_vi = ntfs_attr_iget(vi, ctx->attr->type,
+						 name, ctx->attr->name_length);
+			if (IS_ERR(attr_vi))
+				continue;
+			spin_lock(&attr_vi->i_lock);
+			if (inode_state_read_once(attr_vi) & I_DIRTY_PAGES) {
+				spin_unlock(&attr_vi->i_lock);
+				filemap_write_and_wait(attr_vi->i_mapping);
+			} else
+				spin_unlock(&attr_vi->i_lock);
+			iput(attr_vi);
+		}
+	}
+	mutex_unlock(&ni->mrec_lock);
+	ntfs_attr_put_search_ctx(ctx);
+
+	write_inode_now(vol->mftbmp_ino, 1);
+	down_write(&vol->lcnbmp_lock);
+	write_inode_now(vol->lcnbmp_ino, 1);
+	up_write(&vol->lcnbmp_lock);
+	write_inode_now(vol->mft_ino, 1);
+
+	/*
+	 * NOTE: If we were to use mapping->private_list (see ext2 and
+	 * fs/buffer.c) for dirty blocks then we could optimize the below to be
+	 * sync_mapping_buffers(vi->i_mapping).
+	 */
+	err = sync_blockdev(vi->i_sb->s_bdev);
+	if (unlikely(err && !ret))
+		ret = err;
+	if (likely(!ret))
+		ntfs_debug("Done.");
+	else
+		ntfs_warning(vi->i_sb,
+				"Failed to f%ssync inode 0x%llx.  Error %u.",
+				datasync ? "data" : "", ni->mft_no, -ret);
+	if (!ret)
+		blkdev_issue_flush(vi->i_sb->s_bdev);
+	return ret;
+}
+
+static int ntfs_setattr_size(struct inode *vi, struct iattr *attr)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	int err;
+	loff_t old_size = vi->i_size;
+
+	if (NInoCompressed(ni) || NInoEncrypted(ni)) {
+		ntfs_warning(vi->i_sb,
+			"Changes in inode size are not supported yet for %s files, ignoring.",
+			NInoCompressed(ni) ? "compressed" : "encrypted");
+		return -EOPNOTSUPP;
+	}
+
+	err = inode_newsize_ok(vi, attr->ia_size);
+	if (err)
+		return err;
+
+	inode_dio_wait(vi);
+	/* Serialize against page faults */
+	if (NInoNonResident(NTFS_I(vi)) && attr->ia_size < old_size) {
+		err = iomap_truncate_page(vi, attr->ia_size, NULL,
+				&ntfs_read_iomap_ops,
+				&ntfs_iomap_folio_ops, NULL);
+		if (err)
+			return err;
+	}
+
+	truncate_setsize(vi, attr->ia_size);
+	err = ntfs_truncate_vfs(vi, attr->ia_size, old_size);
+	if (err) {
+		i_size_write(vi, old_size);
+		return err;
+	}
+
+	if (NInoNonResident(ni) && attr->ia_size > old_size &&
+	    old_size % PAGE_SIZE != 0) {
+		loff_t len = min_t(loff_t,
+				round_up(old_size, PAGE_SIZE) - old_size,
+				attr->ia_size - old_size);
+		err = iomap_zero_range(vi, old_size, len,
+				NULL, &ntfs_seek_iomap_ops,
+				&ntfs_iomap_folio_ops, NULL);
+	}
+
+	return err;
+}
+
+/*
+ * ntfs_setattr
+ *
+ * Called from notify_change() when an attribute is being changed.
+ *
+ * NOTE: Changes in inode size are not supported yet for compressed or
+ * encrypted files.
+ */
+int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+		 struct iattr *attr)
+{
+	struct inode *vi = d_inode(dentry);
+	int err;
+	unsigned int ia_valid = attr->ia_valid;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	err = setattr_prepare(idmap, dentry, attr);
+	if (err)
+		goto out;
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	if (ia_valid & ATTR_SIZE) {
+		err = ntfs_setattr_size(vi, attr);
+		if (err)
+			goto out;
+
+		ia_valid |= ATTR_MTIME | ATTR_CTIME;
+	}
+
+	setattr_copy(idmap, vi, attr);
+
+	if (vol->sb->s_flags & SB_POSIXACL && !S_ISLNK(vi->i_mode)) {
+		err = posix_acl_chmod(idmap, dentry, vi->i_mode);
+		if (err)
+			goto out;
+	}
+
+	if (0222 & vi->i_mode)
+		ni->flags &= ~FILE_ATTR_READONLY;
+	else
+		ni->flags |= FILE_ATTR_READONLY;
+
+	if (ia_valid & (ATTR_UID | ATTR_GID | ATTR_MODE)) {
+		unsigned int flags = 0;
+
+		if (ia_valid & ATTR_UID)
+			flags |= NTFS_EA_UID;
+		if (ia_valid & ATTR_GID)
+			flags |= NTFS_EA_GID;
+		if (ia_valid & ATTR_MODE)
+			flags |= NTFS_EA_MODE;
+
+		if (S_ISDIR(vi->i_mode))
+			vi->i_mode &= ~vol->dmask;
+		else
+			vi->i_mode &= ~vol->fmask;
+
+		mutex_lock(&ni->mrec_lock);
+		ntfs_ea_set_wsl_inode(vi, 0, NULL, flags);
+		mutex_unlock(&ni->mrec_lock);
+	}
+
+	mark_inode_dirty(vi);
+out:
+	return err;
+}
+
+int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+		struct kstat *stat, unsigned int request_mask,
+		unsigned int query_flags)
+{
+	struct inode *inode = d_backing_inode(path->dentry);
+	struct ntfs_inode *ni = NTFS_I(inode);
+
+	generic_fillattr(idmap, request_mask, inode, stat);
+
+	stat->blksize = NTFS_SB(inode->i_sb)->cluster_size;
+	stat->blocks = (((u64)NTFS_I(inode)->i_dealloc_clusters <<
+			NTFS_SB(inode->i_sb)->cluster_size_bits) >> 9) + inode->i_blocks;
+	stat->result_mask |= STATX_BTIME;
+	stat->btime = NTFS_I(inode)->i_crtime;
+
+	if (NInoCompressed(ni))
+		stat->attributes |= STATX_ATTR_COMPRESSED;
+
+	if (NInoEncrypted(ni))
+		stat->attributes |= STATX_ATTR_ENCRYPTED;
+
+	if (inode->i_flags & S_IMMUTABLE)
+		stat->attributes |= STATX_ATTR_IMMUTABLE;
+
+	if (inode->i_flags & S_APPEND)
+		stat->attributes |= STATX_ATTR_APPEND;
+
+	stat->attributes_mask |= STATX_ATTR_COMPRESSED | STATX_ATTR_ENCRYPTED |
+				 STATX_ATTR_IMMUTABLE | STATX_ATTR_APPEND;
+
+	/*
+	 * If it's a compressed or encrypted file, NTFS currently
+	 * does not support DIO. For normal files, we report the bdev
+	 * logical block size.
+	 */
+	if (request_mask & STATX_DIOALIGN && S_ISREG(inode->i_mode)) {
+		unsigned int align =
+			bdev_logical_block_size(inode->i_sb->s_bdev);
+
+		stat->result_mask |= STATX_DIOALIGN;
+		if (!NInoCompressed(ni) && !NInoEncrypted(ni)) {
+			stat->dio_mem_align = align;
+			stat->dio_offset_align = align;
+		}
+	}
+
+	return 0;
+}
+
+static loff_t ntfs_file_llseek(struct file *file, loff_t offset, int whence)
+{
+	struct inode *inode = file->f_mapping->host;
+
+	switch (whence) {
+	case SEEK_HOLE:
+		inode_lock_shared(inode);
+		offset = iomap_seek_hole(inode, offset, &ntfs_seek_iomap_ops);
+		inode_unlock_shared(inode);
+		break;
+	case SEEK_DATA:
+		inode_lock_shared(inode);
+		offset = iomap_seek_data(inode, offset, &ntfs_seek_iomap_ops);
+		inode_unlock_shared(inode);
+		break;
+	default:
+		return generic_file_llseek_size(file, offset, whence,
+						inode->i_sb->s_maxbytes,
+						i_size_read(inode));
+	}
+	if (offset < 0)
+		return offset;
+	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
+}
+
+static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+{
+	struct inode *vi = file_inode(iocb->ki_filp);
+	struct super_block *sb = vi->i_sb;
+	ssize_t ret;
+
+	if (NVolShutdown(NTFS_SB(sb)))
+		return -EIO;
+
+	if (NInoCompressed(NTFS_I(vi)) && iocb->ki_flags & IOCB_DIRECT)
+		return -EOPNOTSUPP;
+
+	inode_lock_shared(vi);
+
+	if (iocb->ki_flags & IOCB_DIRECT) {
+		size_t count = iov_iter_count(to);
+
+		if ((iocb->ki_pos | count) & (sb->s_blocksize - 1)) {
+			ret = -EINVAL;
+			goto inode_unlock;
+		}
+
+		file_accessed(iocb->ki_filp);
+		ret = iomap_dio_rw(iocb, to, &ntfs_read_iomap_ops, NULL, 0,
+				NULL, 0);
+	} else {
+		ret = generic_file_read_iter(iocb, to);
+	}
+
+inode_unlock:
+	inode_unlock_shared(vi);
+
+	return ret;
+}
+
+static int ntfs_file_write_dio_end_io(struct kiocb *iocb, ssize_t size,
+		int error, unsigned int flags)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+
+	if (error)
+		return error;
+
+	if (size) {
+		if (i_size_read(inode) < iocb->ki_pos + size) {
+			i_size_write(inode, iocb->ki_pos + size);
+			mark_inode_dirty(inode);
+		}
+	}
+
+	return 0;
+}
+
+static const struct iomap_dio_ops ntfs_write_dio_ops = {
+	.end_io			= ntfs_file_write_dio_end_io,
+};
+
+static ssize_t ntfs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
+{
+	ssize_t ret;
+
+	ret = iomap_dio_rw(iocb, from, &ntfs_dio_iomap_ops,
+			&ntfs_write_dio_ops, 0, NULL, 0);
+	if (ret == -ENOTBLK)
+		ret = 0;
+	else if (ret < 0)
+		goto out;
+
+	if (iov_iter_count(from)) {
+		loff_t offset, end;
+		ssize_t written;
+		int ret2;
+
+		offset = iocb->ki_pos;
+		iocb->ki_flags &= ~IOCB_DIRECT;
+		written = iomap_file_buffered_write(iocb, from,
+				&ntfs_write_iomap_ops, &ntfs_iomap_folio_ops,
+				NULL);
+		if (written < 0) {
+			ret = written;
+			goto out;
+		}
+
+		ret += written;
+		end = iocb->ki_pos + written - 1;
+		ret2 = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,
+				offset, end);
+		if (ret2) {
+			ret = -EIO;
+			goto out;
+		}
+		if (!ret2)
+			invalidate_mapping_pages(iocb->ki_filp->f_mapping,
+						 offset >> PAGE_SHIFT,
+						 end >> PAGE_SHIFT);
+	}
+
+out:
+	return ret;
+}
+
+static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
+{
+	struct file *file = iocb->ki_filp;
+	struct inode *vi = file->f_mapping->host;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+	ssize_t ret;
+	ssize_t count;
+	loff_t pos;
+	int err;
+	loff_t old_data_size, old_init_size;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	if (NInoEncrypted(ni)) {
+		ntfs_error(vi->i_sb, "Writing for %s files is not supported yet",
+			   NInoCompressed(ni) ? "Compressed" : "Encrypted");
+		return -EOPNOTSUPP;
+	}
+
+	if (NInoCompressed(ni) && iocb->ki_flags & IOCB_DIRECT)
+		return -EOPNOTSUPP;
+
+	if (iocb->ki_flags & IOCB_NOWAIT) {
+		if (!inode_trylock(vi))
+			return -EAGAIN;
+	} else
+		inode_lock(vi);
+
+	ret = generic_write_checks(iocb, from);
+	if (ret <= 0)
+		goto out_lock;
+
+	err = file_modified(iocb->ki_filp);
+	if (err) {
+		ret = err;
+		goto out_lock;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	pos = iocb->ki_pos;
+	count = ret;
+
+	old_data_size = ni->data_size;
+	old_init_size = ni->initialized_size;
+
+	if (NInoNonResident(ni) && NInoCompressed(ni)) {
+		ret = ntfs_compress_write(ni, pos, count, from);
+		if (ret > 0)
+			iocb->ki_pos += ret;
+		goto out;
+	}
+
+	if (NInoNonResident(ni) && iocb->ki_flags & IOCB_DIRECT)
+		ret = ntfs_dio_write_iter(iocb, from);
+	else
+		ret = iomap_file_buffered_write(iocb, from, &ntfs_write_iomap_ops,
+				&ntfs_iomap_folio_ops, NULL);
+out:
+	if (ret < 0 && ret != -EIOCBQUEUED) {
+		if (ni->initialized_size != old_init_size) {
+			mutex_lock(&ni->mrec_lock);
+			ntfs_attr_set_initialized_size(ni, old_init_size);
+			mutex_unlock(&ni->mrec_lock);
+		}
+		if (ni->data_size != old_data_size) {
+			truncate_setsize(vi, old_data_size);
+			ntfs_attr_truncate(ni, old_data_size);
+		}
+	}
+out_lock:
+	inode_unlock(vi);
+	if (ret > 0)
+		ret = generic_write_sync(iocb, ret);
+	return ret;
+}
+
+static vm_fault_t ntfs_filemap_page_mkwrite(struct vm_fault *vmf)
+{
+	struct inode *inode = file_inode(vmf->vma->vm_file);
+	vm_fault_t ret;
+
+	sb_start_pagefault(inode->i_sb);
+	file_update_time(vmf->vma->vm_file);
+
+	ret = iomap_page_mkwrite(vmf, &ntfs_page_mkwrite_iomap_ops, NULL);
+	sb_end_pagefault(inode->i_sb);
+	return ret;
+}
+
+static const struct vm_operations_struct ntfs_file_vm_ops = {
+	.fault		= filemap_fault,
+	.map_pages	= filemap_map_pages,
+	.page_mkwrite	= ntfs_filemap_page_mkwrite,
+};
+
+static int ntfs_file_mmap_prepare(struct vm_area_desc *desc)
+{
+	struct file *file = desc->file;
+	struct inode *inode = file_inode(file);
+
+	if (NVolShutdown(NTFS_SB(file->f_mapping->host->i_sb)))
+		return -EIO;
+
+	if (NInoCompressed(NTFS_I(inode)))
+		return -EOPNOTSUPP;
+
+	if (vma_desc_test(desc, VMA_WRITE_BIT)) {
+		struct inode *inode = file_inode(file);
+		loff_t from, to;
+		int err;
+
+		from = ((loff_t)desc->pgoff << PAGE_SHIFT);
+		to = min_t(loff_t, i_size_read(inode),
+			   from + desc->end - desc->start);
+
+		if (NTFS_I(inode)->initialized_size < to) {
+			err = ntfs_extend_initialized_size(inode, to, to, false);
+			if (err)
+				return err;
+		}
+	}
+
+
+	file_accessed(file);
+	desc->vm_ops = &ntfs_file_vm_ops;
+	return 0;
+}
+
+static int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
+		u64 start, u64 len)
+{
+	return iomap_fiemap(inode, fieinfo, start, len, &ntfs_read_iomap_ops);
+}
+
+static const char *ntfs_get_link(struct dentry *dentry, struct inode *inode,
+		struct delayed_call *done)
+{
+	if (!NTFS_I(inode)->target)
+		return ERR_PTR(-EINVAL);
+
+	return NTFS_I(inode)->target;
+}
+
+static ssize_t ntfs_file_splice_read(struct file *in, loff_t *ppos,
+		struct pipe_inode_info *pipe, size_t len, unsigned int flags)
+{
+	if (NVolShutdown(NTFS_SB(in->f_mapping->host->i_sb)))
+		return -EIO;
+
+	return filemap_splice_read(in, ppos, pipe, len, flags);
+}
+
+static int ntfs_ioctl_shutdown(struct super_block *sb, unsigned long arg)
+{
+	u32 flags;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (get_user(flags, (__u32 __user *)arg))
+		return -EFAULT;
+
+	return ntfs_force_shutdown(sb, flags);
+}
+
+static int ntfs_ioctl_get_volume_label(struct file *filp, unsigned long arg)
+{
+	struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb);
+	char __user *buf = (char __user *)arg;
+
+	if (!vol->volume_label) {
+		if (copy_to_user(buf, "", 1))
+			return -EFAULT;
+	} else if (copy_to_user(buf, vol->volume_label,
+				MIN(FSLABEL_MAX, strlen(vol->volume_label) + 1)))
+		return -EFAULT;
+	return 0;
+}
+
+static int ntfs_ioctl_set_volume_label(struct file *filp, unsigned long arg)
+{
+	struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb);
+	char *label;
+	int ret;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	label = strndup_user((const char __user *)arg, FSLABEL_MAX);
+	if (IS_ERR(label))
+		return PTR_ERR(label);
+
+	ret = mnt_want_write_file(filp);
+	if (ret)
+		goto out;
+
+	ret = ntfs_write_volume_label(vol, label);
+	mnt_drop_write_file(filp);
+out:
+	kfree(label);
+	return ret;
+}
+
+static int ntfs_ioctl_fitrim(struct ntfs_volume *vol, unsigned long arg)
+{
+	struct fstrim_range __user *user_range;
+	struct fstrim_range range;
+	struct block_device *dev;
+	int err;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	dev = vol->sb->s_bdev;
+	if (!bdev_max_discard_sectors(dev))
+		return -EOPNOTSUPP;
+
+	user_range = (struct fstrim_range __user *)arg;
+	if (copy_from_user(&range, user_range, sizeof(range)))
+		return -EFAULT;
+
+	if (range.len == 0)
+		return -EINVAL;
+
+	if (range.len < vol->cluster_size)
+		return -EINVAL;
+
+	range.minlen = max_t(u32, range.minlen, bdev_discard_granularity(dev));
+
+	err = ntfs_trim_fs(vol, &range);
+	if (err < 0)
+		return err;
+
+	if (copy_to_user(user_range, &range, sizeof(range)))
+		return -EFAULT;
+
+	return 0;
+}
+
+long ntfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	switch (cmd) {
+	case FS_IOC_SHUTDOWN:
+		return ntfs_ioctl_shutdown(file_inode(filp)->i_sb, arg);
+	case FS_IOC_GETFSLABEL:
+		return ntfs_ioctl_get_volume_label(filp, arg);
+	case FS_IOC_SETFSLABEL:
+		return ntfs_ioctl_set_volume_label(filp, arg);
+	case FITRIM:
+		return ntfs_ioctl_fitrim(NTFS_SB(file_inode(filp)->i_sb), arg);
+	default:
+		return -ENOTTY;
+	}
+}
+
+#ifdef CONFIG_COMPAT
+long ntfs_compat_ioctl(struct file *filp, unsigned int cmd,
+		unsigned long arg)
+{
+	return ntfs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
+}
+#endif
+
+static int ntfs_allocate_range(struct ntfs_inode *ni, int mode, loff_t offset,
+		loff_t len)
+{
+	struct inode *vi = VFS_I(ni);
+	struct ntfs_volume *vol = ni->vol;
+	s64 need_space;
+	loff_t old_size, new_size;
+	s64 start_vcn, end_vcn;
+	int err;
+
+	old_size = i_size_read(vi);
+	new_size = max_t(loff_t, old_size, offset + len);
+	start_vcn = ntfs_bytes_to_cluster(vol, offset);
+	end_vcn = ntfs_bytes_to_cluster(vol, offset + len - 1) + 1;
+
+	err = inode_newsize_ok(vi, new_size);
+	if (err)
+		goto out;
+
+	need_space = ntfs_bytes_to_cluster(vol, ni->allocated_size);
+	if (need_space > start_vcn)
+		need_space = end_vcn - need_space;
+	else
+		need_space = end_vcn - start_vcn;
+	if (need_space > 0 &&
+	    need_space > (atomic64_read(&vol->free_clusters) -
+			  atomic64_read(&vol->dirty_clusters))) {
+		err = -ENOSPC;
+		goto out;
+	}
+
+	err = ntfs_attr_fallocate(ni, offset, len,
+			mode & FALLOC_FL_KEEP_SIZE ? true : false);
+
+	if (!(mode & FALLOC_FL_KEEP_SIZE) && new_size != old_size)
+		i_size_write(vi, ni->data_size);
+out:
+	return err;
+}
+
+static int ntfs_punch_hole(struct ntfs_inode *ni, int mode, loff_t offset,
+		loff_t len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct inode *vi = VFS_I(ni);
+	loff_t end_offset;
+	s64 start_vcn, end_vcn;
+	int err = 0;
+
+	loff_t offset_down = round_down(offset, max_t(unsigned int,
+				vol->cluster_size, PAGE_SIZE));
+
+	if (NVolDisableSparse(vol)) {
+		err = -EOPNOTSUPP;
+		goto out;
+	}
+
+	if (offset >= ni->data_size)
+		goto out;
+
+	if (offset + len > ni->data_size)
+		end_offset = ni->data_size;
+	else
+		end_offset = offset + len;
+
+	err = filemap_write_and_wait_range(vi->i_mapping, offset_down, LLONG_MAX);
+	if (err)
+		goto out;
+	truncate_pagecache(vi, offset_down);
+
+	start_vcn = ntfs_bytes_to_cluster(vol, offset);
+	end_vcn = ntfs_bytes_to_cluster(vol, end_offset - 1) + 1;
+
+	if (offset & vol->cluster_size_mask) {
+		loff_t to;
+
+		to = min_t(loff_t, ntfs_cluster_to_bytes(vol, start_vcn + 1),
+				end_offset);
+		err = iomap_zero_range(vi, offset, to - offset, NULL,
+				&ntfs_seek_iomap_ops,
+				&ntfs_iomap_folio_ops, NULL);
+		if (err < 0 || (end_vcn - start_vcn) == 1)
+			goto out;
+		start_vcn++;
+	}
+
+	if (end_offset & vol->cluster_size_mask) {
+		loff_t from;
+
+		from = ntfs_cluster_to_bytes(vol, end_vcn - 1);
+		err = iomap_zero_range(vi, from, end_offset - from, NULL,
+				&ntfs_seek_iomap_ops,
+				&ntfs_iomap_folio_ops, NULL);
+		if (err < 0 || (end_vcn - start_vcn) == 1)
+			goto out;
+		end_vcn--;
+	}
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	err = ntfs_non_resident_attr_punch_hole(ni, start_vcn,
+			end_vcn - start_vcn);
+	mutex_unlock(&ni->mrec_lock);
+out:
+	return err;
+}
+
+static int ntfs_collapse_range(struct ntfs_inode *ni, loff_t offset, loff_t len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct inode *vi = VFS_I(ni);
+	loff_t old_size, new_size;
+	s64 start_vcn, end_vcn;
+	int err;
+
+	loff_t offset_down = round_down(offset,
+			max_t(unsigned long, vol->cluster_size, PAGE_SIZE));
+
+	if ((offset & vol->cluster_size_mask) ||
+	    (len & vol->cluster_size_mask) ||
+	    offset >= ni->allocated_size) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	old_size = i_size_read(vi);
+	start_vcn = ntfs_bytes_to_cluster(vol, offset);
+	end_vcn = ntfs_bytes_to_cluster(vol, offset + len - 1) + 1;
+
+	if (ntfs_cluster_to_bytes(vol, end_vcn) > ni->allocated_size)
+		end_vcn = (round_up(ni->allocated_size - 1,
+			   vol->cluster_size) >> vol->cluster_size_bits) + 1;
+	new_size = old_size - ntfs_cluster_to_bytes(vol, end_vcn - start_vcn);
+	if (new_size < 0)
+		new_size = 0;
+	err = filemap_write_and_wait_range(vi->i_mapping,
+			offset_down, LLONG_MAX);
+	if (err)
+		goto out;
+
+	truncate_pagecache(vi, offset_down);
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	err = ntfs_non_resident_attr_collapse_range(ni, start_vcn,
+			end_vcn - start_vcn);
+	mutex_unlock(&ni->mrec_lock);
+
+	if (new_size != old_size)
+		i_size_write(vi, ni->data_size);
+out:
+	return err;
+}
+
+static int ntfs_insert_range(struct ntfs_inode *ni, loff_t offset, loff_t len)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct inode *vi = VFS_I(ni);
+	loff_t offset_down = round_down(offset,
+			max_t(unsigned long, vol->cluster_size, PAGE_SIZE));
+	loff_t alloc_size, end_offset = offset + len;
+	loff_t old_size, new_size;
+	s64 start_vcn, end_vcn;
+	int err;
+
+	if (NVolDisableSparse(vol)) {
+		err = -EOPNOTSUPP;
+		goto out;
+	}
+
+	if ((offset & vol->cluster_size_mask) ||
+	    (len & vol->cluster_size_mask) ||
+	     offset >= ni->allocated_size) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	old_size = i_size_read(vi);
+	start_vcn = ntfs_bytes_to_cluster(vol, offset);
+	end_vcn = ntfs_bytes_to_cluster(vol, end_offset - 1) + 1;
+
+	new_size = old_size + ntfs_cluster_to_bytes(vol, end_vcn - start_vcn);
+	alloc_size = ni->allocated_size +
+		ntfs_cluster_to_bytes(vol, end_vcn - start_vcn);
+	if (alloc_size < 0) {
+		err = -EFBIG;
+		goto out;
+	}
+	err = inode_newsize_ok(vi, alloc_size);
+	if (err)
+		goto out;
+
+	err = filemap_write_and_wait_range(vi->i_mapping,
+			offset_down, LLONG_MAX);
+	if (err)
+		goto out;
+
+	truncate_pagecache(vi, offset_down);
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	err = ntfs_non_resident_attr_insert_range(ni, start_vcn,
+			end_vcn - start_vcn);
+	mutex_unlock(&ni->mrec_lock);
+
+	if (new_size != old_size)
+		i_size_write(vi, ni->data_size);
+out:
+	return err;
+}
+
+#define NTFS_FALLOC_FL_SUPPORTED					\
+		(FALLOC_FL_ALLOCATE_RANGE | FALLOC_FL_KEEP_SIZE |	\
+		 FALLOC_FL_INSERT_RANGE | FALLOC_FL_PUNCH_HOLE |	\
+		 FALLOC_FL_COLLAPSE_RANGE)
+
+static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
+{
+	struct inode *vi = file_inode(file);
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+	int err = 0;
+	loff_t old_size;
+	bool map_locked = false;
+
+	if (mode & ~(NTFS_FALLOC_FL_SUPPORTED))
+		return -EOPNOTSUPP;
+
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	if ((ni->vol->mft_zone_end - ni->vol->mft_zone_start) == 0)
+		return -ENOSPC;
+
+	if (NInoNonResident(ni) && !NInoFullyMapped(ni)) {
+		down_write(&ni->runlist.lock);
+		err = ntfs_attr_map_whole_runlist(ni);
+		up_write(&ni->runlist.lock);
+		if (err)
+			return err;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY)) {
+		err = ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+		if (err)
+			return err;
+	}
+
+	old_size = i_size_read(vi);
+
+	inode_lock(vi);
+	if (NInoCompressed(ni) || NInoEncrypted(ni)) {
+		err = -EOPNOTSUPP;
+		goto out;
+	}
+
+	inode_dio_wait(vi);
+	if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE |
+		    FALLOC_FL_INSERT_RANGE)) {
+		filemap_invalidate_lock(vi->i_mapping);
+		map_locked = true;
+	}
+
+	switch (mode & FALLOC_FL_MODE_MASK) {
+	case FALLOC_FL_ALLOCATE_RANGE:
+	case FALLOC_FL_KEEP_SIZE:
+		err = ntfs_allocate_range(ni, mode, offset, len);
+		break;
+	case FALLOC_FL_PUNCH_HOLE:
+		err = ntfs_punch_hole(ni, mode, offset, len);
+		break;
+	case FALLOC_FL_COLLAPSE_RANGE:
+		err = ntfs_collapse_range(ni, offset, len);
+		break;
+	case FALLOC_FL_INSERT_RANGE:
+		err = ntfs_insert_range(ni, offset, len);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+	}
+
+	if (err)
+		goto out;
+
+	err = file_modified(file);
+out:
+	if (map_locked)
+		filemap_invalidate_unlock(vi->i_mapping);
+	if (!err) {
+		if (mode == 0 && NInoNonResident(ni) &&
+		    offset > old_size && old_size % PAGE_SIZE != 0) {
+			loff_t len = min_t(loff_t,
+					   round_up(old_size, PAGE_SIZE) - old_size,
+					   offset - old_size);
+			err = iomap_zero_range(vi, old_size, len, NULL,
+					       &ntfs_seek_iomap_ops,
+					       &ntfs_iomap_folio_ops, NULL);
+		}
+		NInoSetFileNameDirty(ni);
+		inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi));
+		mark_inode_dirty(vi);
+	}
+
+	inode_unlock(vi);
+	return err;
+}
+
+const struct file_operations ntfs_file_ops = {
+	.llseek		= ntfs_file_llseek,
+	.read_iter	= ntfs_file_read_iter,
+	.write_iter	= ntfs_file_write_iter,
+	.fsync		= ntfs_file_fsync,
+	.mmap_prepare	= ntfs_file_mmap_prepare,
+	.open		= ntfs_file_open,
+	.release	= ntfs_file_release,
+	.splice_read	= ntfs_file_splice_read,
+	.splice_write	= iter_file_splice_write,
+	.unlocked_ioctl	= ntfs_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= ntfs_compat_ioctl,
+#endif
+	.fallocate	= ntfs_fallocate,
+	.setlease	= generic_setlease,
+};
+
+const struct inode_operations ntfs_file_inode_ops = {
+	.setattr	= ntfs_setattr,
+	.getattr	= ntfs_getattr,
+	.listxattr	= ntfs_listxattr,
+	.get_acl	= ntfs_get_acl,
+	.set_acl	= ntfs_set_acl,
+	.fiemap		= ntfs_fiemap,
+};
+
+const struct inode_operations ntfs_symlink_inode_operations = {
+	.get_link	= ntfs_get_link,
+	.setattr	= ntfs_setattr,
+	.listxattr	= ntfs_listxattr,
+};
+
+const struct inode_operations ntfs_special_inode_operations = {
+	.setattr	= ntfs_setattr,
+	.getattr	= ntfs_getattr,
+	.listxattr	= ntfs_listxattr,
+	.get_acl	= ntfs_get_acl,
+	.set_acl	= ntfs_set_acl,
+};
+
+const struct file_operations ntfs_empty_file_ops = {};
+
+const struct inode_operations ntfs_empty_inode_ops = {};
diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c
new file mode 100644
index 0000000000000..2080f39691378
--- /dev/null
+++ b/fs/ntfs/index.c
@@ -0,0 +1,2117 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel index handling.
+ *
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2004-2005 Richard Russon
+ * Copyright (c) 2005-2006 Yura Pakhuchiy
+ * Copyright (c) 2005-2008 Szabolcs Szakacsits
+ * Copyright (c) 2007-2021 Jean-Pierre Andre
+ */
+
+#include "collate.h"
+#include "index.h"
+#include "ntfs.h"
+#include "attrlist.h"
+
+/*
+ * ntfs_index_entry_inconsistent - Check the consistency of an index entry
+ *
+ * Make sure data and key do not overflow from entry.
+ * As a side effect, an entry with zero length is rejected.
+ * This entry must be a full one (no INDEX_ENTRY_END flag), and its
+ * length must have been checked beforehand to not overflow from the
+ * index record.
+ */
+int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx,
+		struct ntfs_volume *vol, const struct index_entry *ie,
+		__le32 collation_rule, u64 inum)
+{
+	if (icx) {
+		struct index_header *ih;
+		u8 *ie_start, *ie_end;
+
+		if (icx->is_in_root)
+			ih = &icx->ir->index;
+		else
+			ih = &icx->ib->index;
+
+		if ((le32_to_cpu(ih->index_length) > le32_to_cpu(ih->allocated_size)) ||
+				(le32_to_cpu(ih->index_length) > icx->block_size)) {
+			ntfs_error(vol->sb, "%s Index entry(0x%p)'s length is too big.",
+					icx->is_in_root ? "Index root" : "Index block",
+					(u8 *)icx->entry);
+			return -EINVAL;
+		}
+
+		ie_start = (u8 *)ih + le32_to_cpu(ih->entries_offset);
+		ie_end = (u8 *)ih + le32_to_cpu(ih->index_length);
+
+		if (ie_start > (u8 *)ie ||
+		    ie_end <= (u8 *)ie + le16_to_cpu(ie->length) ||
+		    le16_to_cpu(ie->length) > le32_to_cpu(ih->allocated_size) ||
+		    le16_to_cpu(ie->length) > icx->block_size) {
+			ntfs_error(vol->sb, "Index entry(0x%p) is out of range from %s",
+					(u8 *)icx->entry,
+					icx->is_in_root ? "index root" : "index block");
+			return -EIO;
+		}
+	}
+
+	if (ie->key_length &&
+	    ((le16_to_cpu(ie->key_length) + offsetof(struct index_entry, key)) >
+	     le16_to_cpu(ie->length))) {
+		ntfs_error(vol->sb, "Overflow from index entry in inode %lld\n",
+				(long long)inum);
+		return -EIO;
+
+	} else {
+		if (collation_rule == COLLATION_FILE_NAME) {
+			if ((offsetof(struct index_entry, key.file_name.file_name) +
+			     ie->key.file_name.file_name_length	* sizeof(__le16)) >
+					le16_to_cpu(ie->length)) {
+				ntfs_error(vol->sb,
+					"File name overflow from index entry in inode %lld\n",
+					(long long)inum);
+				return -EIO;
+			}
+		} else {
+			if (ie->data.vi.data_length &&
+			    ((le16_to_cpu(ie->data.vi.data_offset) +
+			      le16_to_cpu(ie->data.vi.data_length)) >
+			     le16_to_cpu(ie->length))) {
+				ntfs_error(vol->sb,
+					"Data overflow from index entry in inode %lld\n",
+					(long long)inum);
+				return -EIO;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * ntfs_index_entry_mark_dirty - mark an index entry dirty
+ * @ictx:	ntfs index context describing the index entry
+ *
+ * Mark the index entry described by the index entry context @ictx dirty.
+ *
+ * If the index entry is in the index root attribute, simply mark the inode
+ * containing the index root attribute dirty.  This ensures the mftrecord, and
+ * hence the index root attribute, will be written out to disk later.
+ *
+ * If the index entry is in an index block belonging to the index allocation
+ * attribute, set ib_dirty to true, thus index block will be updated during
+ * ntfs_index_ctx_put.
+ */
+void ntfs_index_entry_mark_dirty(struct ntfs_index_context *ictx)
+{
+	if (ictx->is_in_root)
+		mark_mft_record_dirty(ictx->actx->ntfs_ino);
+	else if (ictx->ib)
+		ictx->ib_dirty = true;
+}
+
+static s64 ntfs_ib_vcn_to_pos(struct ntfs_index_context *icx, s64 vcn)
+{
+	return vcn << icx->vcn_size_bits;
+}
+
+static s64 ntfs_ib_pos_to_vcn(struct ntfs_index_context *icx, s64 pos)
+{
+	return pos >> icx->vcn_size_bits;
+}
+
+static int ntfs_ib_write(struct ntfs_index_context *icx, struct index_block *ib)
+{
+	s64 ret, vcn = le64_to_cpu(ib->index_block_vcn);
+
+	ntfs_debug("vcn: %lld\n", vcn);
+
+	ret = pre_write_mst_fixup((struct ntfs_record *)ib, icx->block_size);
+	if (ret)
+		return -EIO;
+
+	ret = ntfs_inode_attr_pwrite(VFS_I(icx->ia_ni),
+			ntfs_ib_vcn_to_pos(icx, vcn), icx->block_size,
+			(u8 *)ib, icx->sync_write);
+	if (ret != icx->block_size) {
+		ntfs_debug("Failed to write index block %lld, inode %llu",
+				vcn, (unsigned long long)icx->idx_ni->mft_no);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ntfs_icx_ib_write(struct ntfs_index_context *icx)
+{
+	int err;
+
+	err = ntfs_ib_write(icx, icx->ib);
+	if (err)
+		return err;
+
+	icx->ib_dirty = false;
+
+	return 0;
+}
+
+int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx)
+{
+	int ret;
+
+	if (icx->ib_dirty == false)
+		return 0;
+
+	icx->sync_write = true;
+
+	ret = ntfs_ib_write(icx, icx->ib);
+	if (!ret) {
+		kvfree(icx->ib);
+		icx->ib = NULL;
+		icx->ib_dirty = false;
+	} else {
+		post_write_mst_fixup((struct ntfs_record *)icx->ib);
+		icx->sync_write = false;
+	}
+
+	return ret;
+}
+
+/*
+ * ntfs_index_ctx_get - allocate and initialize a new index context
+ * @ni:		ntfs inode with which to initialize the context
+ * @name:	name of the which context describes
+ * @name_len:	length of the index name
+ *
+ * Allocate a new index context, initialize it with @ni and return it.
+ * Return NULL if allocation failed.
+ */
+struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni,
+		__le16 *name, u32 name_len)
+{
+	struct ntfs_index_context *icx;
+
+	ntfs_debug("Entering\n");
+
+	if (!ni)
+		return NULL;
+
+	if (ni->nr_extents == -1)
+		ni = ni->ext.base_ntfs_ino;
+
+	icx = kmem_cache_alloc(ntfs_index_ctx_cache, GFP_NOFS);
+	if (icx)
+		*icx = (struct ntfs_index_context) {
+			.idx_ni = ni,
+			.name = name,
+			.name_len = name_len,
+		};
+	return icx;
+}
+
+static void ntfs_index_ctx_free(struct ntfs_index_context *icx)
+{
+	ntfs_debug("Entering\n");
+
+	if (icx->actx) {
+		ntfs_attr_put_search_ctx(icx->actx);
+		icx->actx = NULL;
+	}
+
+	if (!icx->is_in_root) {
+		if (icx->ib_dirty)
+			ntfs_ib_write(icx, icx->ib);
+		kvfree(icx->ib);
+		icx->ib = NULL;
+	}
+
+	if (icx->ia_ni) {
+		iput(VFS_I(icx->ia_ni));
+		icx->ia_ni = NULL;
+	}
+}
+
+/*
+ * ntfs_index_ctx_put - release an index context
+ * @icx:	index context to free
+ *
+ * Release the index context @icx, releasing all associated resources.
+ */
+void ntfs_index_ctx_put(struct ntfs_index_context *icx)
+{
+	ntfs_index_ctx_free(icx);
+	kmem_cache_free(ntfs_index_ctx_cache, icx);
+}
+
+/*
+ * ntfs_index_ctx_reinit - reinitialize an index context
+ * @icx:	index context to reinitialize
+ *
+ * Reinitialize the index context @icx so it can be used for ntfs_index_lookup.
+ */
+void ntfs_index_ctx_reinit(struct ntfs_index_context *icx)
+{
+	ntfs_debug("Entering\n");
+
+	ntfs_index_ctx_free(icx);
+
+	*icx = (struct ntfs_index_context) {
+		.idx_ni = icx->idx_ni,
+		.name = icx->name,
+		.name_len = icx->name_len,
+	};
+}
+
+static __le64 *ntfs_ie_get_vcn_addr(struct index_entry *ie)
+{
+	return (__le64 *)((u8 *)ie + le16_to_cpu(ie->length) - sizeof(s64));
+}
+
+/*
+ *  Get the subnode vcn to which the index entry refers.
+ */
+static s64 ntfs_ie_get_vcn(struct index_entry *ie)
+{
+	return le64_to_cpup(ntfs_ie_get_vcn_addr(ie));
+}
+
+static struct index_entry *ntfs_ie_get_first(struct index_header *ih)
+{
+	return (struct index_entry *)((u8 *)ih + le32_to_cpu(ih->entries_offset));
+}
+
+static struct index_entry *ntfs_ie_get_next(struct index_entry *ie)
+{
+	return (struct index_entry *)((char *)ie + le16_to_cpu(ie->length));
+}
+
+static u8 *ntfs_ie_get_end(struct index_header *ih)
+{
+	return (u8 *)ih + le32_to_cpu(ih->index_length);
+}
+
+static int ntfs_ie_end(struct index_entry *ie)
+{
+	return ie->flags & INDEX_ENTRY_END || !ie->length;
+}
+
+/*
+ *  Find the last entry in the index block
+ */
+static struct index_entry *ntfs_ie_get_last(struct index_entry *ie, char *ies_end)
+{
+	ntfs_debug("Entering\n");
+
+	while ((char *)ie < ies_end && !ntfs_ie_end(ie))
+		ie = ntfs_ie_get_next(ie);
+
+	return ie;
+}
+
+static struct index_entry *ntfs_ie_get_by_pos(struct index_header *ih, int pos)
+{
+	struct index_entry *ie;
+
+	ntfs_debug("pos: %d\n", pos);
+
+	ie = ntfs_ie_get_first(ih);
+
+	while (pos-- > 0)
+		ie = ntfs_ie_get_next(ie);
+
+	return ie;
+}
+
+static struct index_entry *ntfs_ie_prev(struct index_header *ih, struct index_entry *ie)
+{
+	struct index_entry *ie_prev = NULL;
+	struct index_entry *tmp;
+
+	ntfs_debug("Entering\n");
+
+	tmp = ntfs_ie_get_first(ih);
+
+	while (tmp != ie) {
+		ie_prev = tmp;
+		tmp = ntfs_ie_get_next(tmp);
+	}
+
+	return ie_prev;
+}
+
+static int ntfs_ih_numof_entries(struct index_header *ih)
+{
+	int n;
+	struct index_entry *ie;
+	u8 *end;
+
+	ntfs_debug("Entering\n");
+
+	end = ntfs_ie_get_end(ih);
+	ie = ntfs_ie_get_first(ih);
+	for (n = 0; !ntfs_ie_end(ie) && (u8 *)ie < end; n++)
+		ie = ntfs_ie_get_next(ie);
+	return n;
+}
+
+static int ntfs_ih_one_entry(struct index_header *ih)
+{
+	return (ntfs_ih_numof_entries(ih) == 1);
+}
+
+static int ntfs_ih_zero_entry(struct index_header *ih)
+{
+	return (ntfs_ih_numof_entries(ih) == 0);
+}
+
+static void ntfs_ie_delete(struct index_header *ih, struct index_entry *ie)
+{
+	u32 new_size;
+
+	ntfs_debug("Entering\n");
+
+	new_size = le32_to_cpu(ih->index_length) - le16_to_cpu(ie->length);
+	ih->index_length = cpu_to_le32(new_size);
+	memmove(ie, (u8 *)ie + le16_to_cpu(ie->length),
+			new_size - ((u8 *)ie - (u8 *)ih));
+}
+
+static void ntfs_ie_set_vcn(struct index_entry *ie, s64 vcn)
+{
+	*ntfs_ie_get_vcn_addr(ie) = cpu_to_le64(vcn);
+}
+
+/*
+ *  Insert @ie index entry at @pos entry. Used @ih values should be ok already.
+ */
+static void ntfs_ie_insert(struct index_header *ih, struct index_entry *ie,
+		struct index_entry *pos)
+{
+	int ie_size = le16_to_cpu(ie->length);
+
+	ntfs_debug("Entering\n");
+
+	ih->index_length = cpu_to_le32(le32_to_cpu(ih->index_length) + ie_size);
+	memmove((u8 *)pos + ie_size, pos,
+			le32_to_cpu(ih->index_length) - ((u8 *)pos - (u8 *)ih) - ie_size);
+	memcpy(pos, ie, ie_size);
+}
+
+static struct index_entry *ntfs_ie_dup(struct index_entry *ie)
+{
+	ntfs_debug("Entering\n");
+
+	return kmemdup(ie, le16_to_cpu(ie->length), GFP_NOFS);
+}
+
+static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie)
+{
+	struct index_entry *dup;
+	int size = le16_to_cpu(ie->length);
+
+	ntfs_debug("Entering\n");
+
+	if (ie->flags & INDEX_ENTRY_NODE)
+		size -= sizeof(s64);
+
+	dup = kmemdup(ie, size, GFP_NOFS);
+	if (dup) {
+		dup->flags &= ~INDEX_ENTRY_NODE;
+		dup->length = cpu_to_le16(size);
+	}
+	return dup;
+}
+
+/*
+ * Check the consistency of an index block
+ *
+ * Make sure the index block does not overflow from the index record.
+ * The size of block is assumed to have been checked to be what is
+ * defined in the index root.
+ *
+ * Returns 0 if no error was found -1 otherwise (with errno unchanged)
+ *
+ * |<--->|  offsetof(struct index_block, index)
+ * |     |<--->|  sizeof(struct index_header)
+ * |     |     |
+ * |     |     | seq          index entries         unused
+ * |=====|=====|=====|===========================|==============|
+ * |     |           |                           |              |
+ * |     |<--------->| entries_offset            |              |
+ * |     |<---------------- index_length ------->|              |
+ * |     |<--------------------- allocated_size --------------->|
+ * |<--------------------------- block_size ------------------->|
+ *
+ * size(struct index_header) <= ent_offset < ind_length <= alloc_size < bk_size
+ */
+static int ntfs_index_block_inconsistent(struct ntfs_index_context *icx,
+		struct index_block *ib, s64 vcn)
+{
+	u32 ib_size = (unsigned int)le32_to_cpu(ib->index.allocated_size) +
+		offsetof(struct index_block, index);
+	struct super_block *sb = icx->idx_ni->vol->sb;
+	unsigned long long inum = icx->idx_ni->mft_no;
+
+	ntfs_debug("Entering\n");
+
+	if (!ntfs_is_indx_record(ib->magic)) {
+
+		ntfs_error(sb, "Corrupt index block signature: vcn %lld inode %llu\n",
+				vcn, (unsigned long long)icx->idx_ni->mft_no);
+		return -1;
+	}
+
+	if (le64_to_cpu(ib->index_block_vcn) != vcn) {
+		ntfs_error(sb,
+			"Corrupt index block: s64 (%lld) is different from expected s64 (%lld) in inode %llu\n",
+			(long long)le64_to_cpu(ib->index_block_vcn),
+			vcn, inum);
+		return -1;
+	}
+
+	if (ib_size != icx->block_size) {
+		ntfs_error(sb,
+			"Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n",
+			vcn, inum, ib_size, icx->block_size);
+		return -1;
+	}
+
+	if (le32_to_cpu(ib->index.entries_offset) < sizeof(struct index_header)) {
+		ntfs_error(sb, "Invalid index entry offset in inode %lld\n", inum);
+		return -1;
+	}
+	if (le32_to_cpu(ib->index.index_length) <=
+	    le32_to_cpu(ib->index.entries_offset)) {
+		ntfs_error(sb, "No space for index entries in inode %lld\n", inum);
+		return -1;
+	}
+	if (le32_to_cpu(ib->index.allocated_size) <
+	    le32_to_cpu(ib->index.index_length)) {
+		ntfs_error(sb, "Index entries overflow in inode %lld\n", inum);
+		return -1;
+	}
+
+	return 0;
+}
+
+static struct index_root *ntfs_ir_lookup(struct ntfs_inode *ni, __le16 *name,
+		u32 name_len, struct ntfs_attr_search_ctx **ctx)
+{
+	struct attr_record *a;
+	struct index_root *ir = NULL;
+
+	ntfs_debug("Entering\n");
+	*ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!*ctx) {
+		ntfs_error(ni->vol->sb, "%s, Failed to get search context", __func__);
+		return NULL;
+	}
+
+	if (ntfs_attr_lookup(AT_INDEX_ROOT, name, name_len, CASE_SENSITIVE,
+				0, NULL, 0, *ctx)) {
+		ntfs_error(ni->vol->sb, "Failed to lookup $INDEX_ROOT");
+		goto err_out;
+	}
+
+	a = (*ctx)->attr;
+	if (a->non_resident) {
+		ntfs_error(ni->vol->sb, "Non-resident $INDEX_ROOT detected");
+		goto err_out;
+	}
+
+	ir = (struct index_root *)((char *)a + le16_to_cpu(a->data.resident.value_offset));
+err_out:
+	if (!ir) {
+		ntfs_attr_put_search_ctx(*ctx);
+		*ctx = NULL;
+	}
+	return ir;
+}
+
+static struct index_root *ntfs_ir_lookup2(struct ntfs_inode *ni, __le16 *name, u32 len)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	struct index_root *ir;
+
+	ir = ntfs_ir_lookup(ni, name, len, &ctx);
+	if (ir)
+		ntfs_attr_put_search_ctx(ctx);
+	return ir;
+}
+
+/*
+ * Find a key in the index block.
+ */
+static int ntfs_ie_lookup(const void *key, const u32 key_len,
+		struct ntfs_index_context *icx, struct index_header *ih,
+		s64 *vcn, struct index_entry **ie_out)
+{
+	struct index_entry *ie;
+	u8 *index_end;
+	int rc, item = 0;
+
+	ntfs_debug("Entering\n");
+
+	index_end = ntfs_ie_get_end(ih);
+
+	/*
+	 * Loop until we exceed valid memory (corruption case) or until we
+	 * reach the last entry.
+	 */
+	for (ie = ntfs_ie_get_first(ih); ; ie = ntfs_ie_get_next(ie)) {
+		/* Bounds checks. */
+		if ((u8 *)ie + sizeof(struct index_entry_header) > index_end ||
+				(u8 *)ie + le16_to_cpu(ie->length) > index_end) {
+			ntfs_error(icx->idx_ni->vol->sb,
+					"Index entry out of bounds in inode %llu.\n",
+					(unsigned long long)icx->idx_ni->mft_no);
+			return -ERANGE;
+		}
+
+		/*
+		 * The last entry cannot contain a key.  It can however contain
+		 * a pointer to a child node in the B+tree so we just break out.
+		 */
+		if (ntfs_ie_end(ie))
+			break;
+
+		/*
+		 * Not a perfect match, need to do full blown collation so we
+		 * know which way in the B+tree we have to go.
+		 */
+		rc = ntfs_collate(icx->idx_ni->vol, icx->cr, key, key_len, &ie->key,
+				le16_to_cpu(ie->key_length));
+		if (rc == -EINVAL) {
+			ntfs_error(icx->idx_ni->vol->sb,
+				"Collation error. Perhaps a filename contains invalid characters?\n");
+			return -ERANGE;
+		}
+		/*
+		 * If @key collates before the key of the current entry, there
+		 * is definitely no such key in this index but we might need to
+		 * descend into the B+tree so we just break out of the loop.
+		 */
+		if (rc == -1)
+			break;
+
+		if (!rc) {
+			*ie_out = ie;
+			icx->parent_pos[icx->pindex] = item;
+			return 0;
+		}
+
+		item++;
+	}
+	/*
+	 * We have finished with this index block without success. Check for the
+	 * presence of a child node and if not present return with errno ENOENT,
+	 * otherwise we will keep searching in another index block.
+	 */
+	if (!(ie->flags & INDEX_ENTRY_NODE)) {
+		ntfs_debug("Index entry wasn't found.\n");
+		*ie_out = ie;
+		return -ENOENT;
+	}
+
+	/* Get the starting vcn of the index_block holding the child node. */
+	*vcn = ntfs_ie_get_vcn(ie);
+	if (*vcn < 0) {
+		ntfs_error(icx->idx_ni->vol->sb, "Negative vcn in inode %llu\n",
+				(unsigned long long)icx->idx_ni->mft_no);
+		return -EINVAL;
+	}
+
+	ntfs_debug("Parent entry number %d\n", item);
+	icx->parent_pos[icx->pindex] = item;
+
+	return -EAGAIN;
+}
+
+struct ntfs_inode *ntfs_ia_open(struct ntfs_index_context *icx, struct ntfs_inode *ni)
+{
+	struct inode *ia_vi;
+
+	ia_vi = ntfs_index_iget(VFS_I(ni), icx->name, icx->name_len);
+	if (IS_ERR(ia_vi)) {
+		ntfs_error(icx->idx_ni->vol->sb,
+				"Failed to open index allocation of inode %llu",
+				(unsigned long long)ni->mft_no);
+		return NULL;
+	}
+
+	return NTFS_I(ia_vi);
+}
+
+static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_block *dst)
+{
+	s64 pos, ret;
+
+	ntfs_debug("vcn: %lld\n", vcn);
+
+	pos = ntfs_ib_vcn_to_pos(icx, vcn);
+
+	ret = ntfs_inode_attr_pread(VFS_I(icx->ia_ni), pos, icx->block_size, (u8 *)dst);
+	if (ret != icx->block_size) {
+		if (ret == -1)
+			ntfs_error(icx->idx_ni->vol->sb, "Failed to read index block");
+		else
+			ntfs_error(icx->idx_ni->vol->sb,
+				"Failed to read full index block at %lld\n", pos);
+		return -1;
+	}
+
+	post_read_mst_fixup((struct ntfs_record *)((u8 *)dst), icx->block_size);
+	if (ntfs_index_block_inconsistent(icx, dst, vcn))
+		return -1;
+
+	return 0;
+}
+
+static int ntfs_icx_parent_inc(struct ntfs_index_context *icx)
+{
+	icx->pindex++;
+	if (icx->pindex >= MAX_PARENT_VCN) {
+		ntfs_error(icx->idx_ni->vol->sb, "Index is over %d level deep", MAX_PARENT_VCN);
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+static int ntfs_icx_parent_dec(struct ntfs_index_context *icx)
+{
+	icx->pindex--;
+	if (icx->pindex < 0) {
+		ntfs_error(icx->idx_ni->vol->sb, "Corrupt index pointer (%d)", icx->pindex);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/*
+ * ntfs_index_lookup - find a key in an index and return its index entry
+ * @key:	key for which to search in the index
+ * @key_len:	length of @key in bytes
+ * @icx:	context describing the index and the returned entry
+ *
+ * Before calling ntfs_index_lookup(), @icx must have been obtained from a
+ * call to ntfs_index_ctx_get().
+ *
+ * Look for the @key in the index specified by the index lookup context @icx.
+ * ntfs_index_lookup() walks the contents of the index looking for the @key.
+ *
+ * If the @key is found in the index, 0 is returned and @icx is setup to
+ * describe the index entry containing the matching @key.  @icx->entry is the
+ * index entry and @icx->data and @icx->data_len are the index entry data and
+ * its length in bytes, respectively.
+ *
+ * If the @key is not found in the index, -ENOENT is returned and
+ * @icx is setup to describe the index entry whose key collates immediately
+ * after the search @key, i.e. this is the position in the index at which
+ * an index entry with a key of @key would need to be inserted.
+ *
+ * When finished with the entry and its data, call ntfs_index_ctx_put() to free
+ * the context and other associated resources.
+ *
+ * If the index entry was modified, call ntfs_index_entry_mark_dirty() before
+ * the call to ntfs_index_ctx_put() to ensure that the changes are written
+ * to disk.
+ */
+int ntfs_index_lookup(const void *key, const u32 key_len, struct ntfs_index_context *icx)
+{
+	s64 old_vcn, vcn;
+	struct ntfs_inode *ni = icx->idx_ni;
+	struct super_block *sb = ni->vol->sb;
+	struct index_root *ir;
+	struct index_entry *ie;
+	struct index_block *ib = NULL;
+	int err = 0;
+
+	ntfs_debug("Entering\n");
+
+	if (!key) {
+		ntfs_error(sb, "key: %p  key_len: %d", key, key_len);
+		return -EINVAL;
+	}
+
+	ir = ntfs_ir_lookup(ni, icx->name, icx->name_len, &icx->actx);
+	if (!ir)
+		return -EIO;
+
+	icx->block_size = le32_to_cpu(ir->index_block_size);
+	if (icx->block_size < NTFS_BLOCK_SIZE) {
+		err = -EINVAL;
+		ntfs_error(sb,
+			"Index block size (%d) is smaller than the sector size (%d)",
+			icx->block_size, NTFS_BLOCK_SIZE);
+		goto err_out;
+	}
+
+	if (ni->vol->cluster_size <= icx->block_size)
+		icx->vcn_size_bits = ni->vol->cluster_size_bits;
+	else
+		icx->vcn_size_bits = ni->vol->sector_size_bits;
+
+	icx->cr = ir->collation_rule;
+	if (!ntfs_is_collation_rule_supported(icx->cr)) {
+		err = -EOPNOTSUPP;
+		ntfs_error(sb, "Unknown collation rule 0x%x",
+				(unsigned int)le32_to_cpu(icx->cr));
+		goto err_out;
+	}
+
+	old_vcn = VCN_INDEX_ROOT_PARENT;
+	err = ntfs_ie_lookup(key, key_len, icx, &ir->index, &vcn, &ie);
+	if (err == -ERANGE || err == -EINVAL)
+		goto err_out;
+
+	icx->ir = ir;
+	if (err != -EAGAIN) {
+		icx->is_in_root = true;
+		icx->parent_vcn[icx->pindex] = old_vcn;
+		goto done;
+	}
+
+	/* Child node present, descend into it. */
+	icx->ia_ni = ntfs_ia_open(icx, ni);
+	if (!icx->ia_ni) {
+		err = -ENOENT;
+		goto err_out;
+	}
+
+	ib = kvzalloc(icx->block_size, GFP_NOFS);
+	if (!ib) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+descend_into_child_node:
+	icx->parent_vcn[icx->pindex] = old_vcn;
+	if (ntfs_icx_parent_inc(icx)) {
+		err = -EIO;
+		goto err_out;
+	}
+	old_vcn = vcn;
+
+	ntfs_debug("Descend into node with s64 %lld.\n", vcn);
+
+	if (ntfs_ib_read(icx, vcn, ib)) {
+		err = -EIO;
+		goto err_out;
+	}
+	err = ntfs_ie_lookup(key, key_len, icx, &ib->index, &vcn, &ie);
+	if (err != -EAGAIN) {
+		if (err == -EINVAL || err == -ERANGE)
+			goto err_out;
+
+		icx->is_in_root = false;
+		icx->ib = ib;
+		icx->parent_vcn[icx->pindex] = vcn;
+		goto done;
+	}
+
+	if ((ib->index.flags & NODE_MASK) == LEAF_NODE) {
+		ntfs_error(icx->idx_ni->vol->sb,
+			"Index entry with child node found in a leaf node in inode 0x%llx.\n",
+			(unsigned long long)ni->mft_no);
+		goto err_out;
+	}
+
+	goto descend_into_child_node;
+err_out:
+	if (icx->actx) {
+		ntfs_attr_put_search_ctx(icx->actx);
+		icx->actx = NULL;
+	}
+	kvfree(ib);
+	if (!err)
+		err = -EIO;
+	return err;
+done:
+	icx->entry = ie;
+	icx->data = (u8 *)ie + offsetof(struct index_entry, key);
+	icx->data_len = le16_to_cpu(ie->key_length);
+	ntfs_debug("Done.\n");
+	return err;
+
+}
+
+static struct index_block *ntfs_ib_alloc(s64 ib_vcn, u32 ib_size,
+		u8 node_type)
+{
+	struct index_block *ib;
+	int ih_size = sizeof(struct index_header);
+
+	ntfs_debug("Entering ib_vcn = %lld ib_size = %u\n", ib_vcn, ib_size);
+
+	ib = kvzalloc(ib_size, GFP_NOFS);
+	if (!ib)
+		return NULL;
+
+	ib->magic = magic_INDX;
+	ib->usa_ofs = cpu_to_le16(sizeof(struct index_block));
+	ib->usa_count = cpu_to_le16(ib_size / NTFS_BLOCK_SIZE + 1);
+	/* Set USN to 1 */
+	*(__le16 *)((char *)ib + le16_to_cpu(ib->usa_ofs)) = cpu_to_le16(1);
+	ib->lsn = 0;
+	ib->index_block_vcn = cpu_to_le64(ib_vcn);
+	ib->index.entries_offset = cpu_to_le32((ih_size +
+				le16_to_cpu(ib->usa_count) * 2 + 7) & ~7);
+	ib->index.index_length = 0;
+	ib->index.allocated_size = cpu_to_le32(ib_size -
+			(sizeof(struct index_block) - ih_size));
+	ib->index.flags = node_type;
+
+	return ib;
+}
+
+/*
+ *  Find the median by going through all the entries
+ */
+static struct index_entry *ntfs_ie_get_median(struct index_header *ih)
+{
+	struct index_entry *ie, *ie_start;
+	u8 *ie_end;
+	int i = 0, median;
+
+	ntfs_debug("Entering\n");
+
+	ie = ie_start = ntfs_ie_get_first(ih);
+	ie_end = (u8 *)ntfs_ie_get_end(ih);
+
+	while ((u8 *)ie < ie_end && !ntfs_ie_end(ie)) {
+		ie = ntfs_ie_get_next(ie);
+		i++;
+	}
+	/*
+	 * NOTE: this could be also the entry at the half of the index block.
+	 */
+	median = i / 2 - 1;
+
+	ntfs_debug("Entries: %d  median: %d\n", i, median);
+
+	for (i = 0, ie = ie_start; i <= median; i++)
+		ie = ntfs_ie_get_next(ie);
+
+	return ie;
+}
+
+static u64 ntfs_ibm_vcn_to_pos(struct ntfs_index_context *icx, s64 vcn)
+{
+	u64 pos = ntfs_ib_vcn_to_pos(icx, vcn);
+
+	do_div(pos, icx->block_size);
+	return pos;
+}
+
+static s64 ntfs_ibm_pos_to_vcn(struct ntfs_index_context *icx, s64 pos)
+{
+	return ntfs_ib_pos_to_vcn(icx, pos * icx->block_size);
+}
+
+static int ntfs_ibm_add(struct ntfs_index_context *icx)
+{
+	u8 bmp[8];
+
+	ntfs_debug("Entering\n");
+
+	if (ntfs_attr_exist(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len))
+		return 0;
+	/*
+	 * AT_BITMAP must be at least 8 bytes.
+	 */
+	memset(bmp, 0, sizeof(bmp));
+	if (ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len,
+				bmp, sizeof(bmp))) {
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_BITMAP");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ntfs_ibm_modify(struct ntfs_index_context *icx, s64 vcn, int set)
+{
+	u8 byte;
+	u64 pos = ntfs_ibm_vcn_to_pos(icx, vcn);
+	u32 bpos = pos / 8;
+	u32 bit = 1 << (pos % 8);
+	struct ntfs_inode *bmp_ni;
+	struct inode *bmp_vi;
+	int ret = 0;
+
+	ntfs_debug("%s vcn: %lld\n", set ? "set" : "clear", vcn);
+
+	bmp_vi = ntfs_attr_iget(VFS_I(icx->idx_ni), AT_BITMAP, icx->name, icx->name_len);
+	if (IS_ERR(bmp_vi)) {
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to open $BITMAP attribute");
+		return PTR_ERR(bmp_vi);
+	}
+
+	bmp_ni = NTFS_I(bmp_vi);
+
+	if (set) {
+		if (bmp_ni->data_size < bpos + 1) {
+			ret = ntfs_attr_truncate(bmp_ni, (bmp_ni->data_size + 8) & ~7);
+			if (ret) {
+				ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate AT_BITMAP");
+				goto err;
+			}
+			i_size_write(bmp_vi, (loff_t)bmp_ni->data_size);
+		}
+	}
+
+	if (ntfs_inode_attr_pread(bmp_vi, bpos, 1, &byte) != 1) {
+		ret = -EIO;
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to read $BITMAP");
+		goto err;
+	}
+
+	if (set)
+		byte |= bit;
+	else
+		byte &= ~bit;
+
+	if (ntfs_inode_attr_pwrite(bmp_vi, bpos, 1, &byte, false) != 1) {
+		ret = -EIO;
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to write $Bitmap");
+		goto err;
+	}
+
+err:
+	iput(bmp_vi);
+	return ret;
+}
+
+static int ntfs_ibm_set(struct ntfs_index_context *icx, s64 vcn)
+{
+	return ntfs_ibm_modify(icx, vcn, 1);
+}
+
+static int ntfs_ibm_clear(struct ntfs_index_context *icx, s64 vcn)
+{
+	return ntfs_ibm_modify(icx, vcn, 0);
+}
+
+static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx)
+{
+	u8 *bm;
+	int bit;
+	s64 vcn, byte, size;
+
+	ntfs_debug("Entering\n");
+
+	bm = ntfs_attr_readall(icx->idx_ni, AT_BITMAP,  icx->name, icx->name_len,
+			&size);
+	if (!bm)
+		return (s64)-1;
+
+	for (byte = 0; byte < size; byte++) {
+		if (bm[byte] == 255)
+			continue;
+
+		for (bit = 0; bit < 8; bit++) {
+			if (!(bm[byte] & (1 << bit))) {
+				vcn = ntfs_ibm_pos_to_vcn(icx, byte * 8 + bit);
+				goto out;
+			}
+		}
+	}
+
+	vcn = ntfs_ibm_pos_to_vcn(icx, size * 8);
+out:
+	ntfs_debug("allocated vcn: %lld\n", vcn);
+
+	if (ntfs_ibm_set(icx, vcn))
+		vcn = (s64)-1;
+
+	kvfree(bm);
+	return vcn;
+}
+
+static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn)
+{
+	struct index_block *ib;
+	struct index_entry *ie_last;
+	char *ies_start, *ies_end;
+	int i;
+
+	ntfs_debug("Entering\n");
+
+	ib = ntfs_ib_alloc(ib_vcn, le32_to_cpu(ir->index_block_size), LEAF_NODE);
+	if (!ib)
+		return NULL;
+
+	ies_start = (char *)ntfs_ie_get_first(&ir->index);
+	ies_end   = (char *)ntfs_ie_get_end(&ir->index);
+	ie_last   = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end);
+	/*
+	 * Copy all entries, including the termination entry
+	 * as well, which can never have any data.
+	 */
+	i = (char *)ie_last - ies_start + le16_to_cpu(ie_last->length);
+	memcpy(ntfs_ie_get_first(&ib->index), ies_start, i);
+
+	ib->index.flags = ir->index.flags;
+	ib->index.index_length = cpu_to_le32(i +
+			le32_to_cpu(ib->index.entries_offset));
+	return ib;
+}
+
+static void ntfs_ir_nill(struct index_root *ir)
+{
+	struct index_entry *ie_last;
+	char *ies_start, *ies_end;
+
+	ntfs_debug("Entering\n");
+
+	ies_start = (char *)ntfs_ie_get_first(&ir->index);
+	ies_end   = (char *)ntfs_ie_get_end(&ir->index);
+	ie_last   = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end);
+	/*
+	 * Move the index root termination entry forward
+	 */
+	if ((char *)ie_last > ies_start) {
+		memmove((char *)ntfs_ie_get_first(&ir->index),
+			(char *)ie_last, le16_to_cpu(ie_last->length));
+		ie_last = (struct index_entry *)ies_start;
+	}
+}
+
+static int ntfs_ib_copy_tail(struct ntfs_index_context *icx, struct index_block *src,
+		struct index_entry *median, s64 new_vcn)
+{
+	u8 *ies_end;
+	struct index_entry *ie_head;		/* first entry after the median */
+	int tail_size, ret;
+	struct index_block *dst;
+
+	ntfs_debug("Entering\n");
+
+	dst = ntfs_ib_alloc(new_vcn, icx->block_size,
+			src->index.flags & NODE_MASK);
+	if (!dst)
+		return -ENOMEM;
+
+	ie_head = ntfs_ie_get_next(median);
+
+	ies_end = (u8 *)ntfs_ie_get_end(&src->index);
+	tail_size = ies_end - (u8 *)ie_head;
+	memcpy(ntfs_ie_get_first(&dst->index), ie_head, tail_size);
+
+	dst->index.index_length = cpu_to_le32(tail_size +
+			le32_to_cpu(dst->index.entries_offset));
+	ret = ntfs_ib_write(icx, dst);
+
+	kvfree(dst);
+	return ret;
+}
+
+static int ntfs_ib_cut_tail(struct ntfs_index_context *icx, struct index_block *ib,
+		struct index_entry *ie)
+{
+	char *ies_start, *ies_end;
+	struct index_entry *ie_last;
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	ies_start = (char *)ntfs_ie_get_first(&ib->index);
+	ies_end   = (char *)ntfs_ie_get_end(&ib->index);
+
+	ie_last   = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end);
+	if (ie_last->flags & INDEX_ENTRY_NODE)
+		ntfs_ie_set_vcn(ie_last, ntfs_ie_get_vcn(ie));
+
+	unsafe_memcpy(ie, ie_last, le16_to_cpu(ie_last->length),
+			/* alloc is larger than ie_last->length, see ntfs_ie_get_last() */);
+
+	ib->index.index_length = cpu_to_le32(((char *)ie - ies_start) +
+			le16_to_cpu(ie->length) + le32_to_cpu(ib->index.entries_offset));
+
+	ret = ntfs_ib_write(icx, ib);
+	return ret;
+}
+
+static int ntfs_ia_add(struct ntfs_index_context *icx)
+{
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	ret = ntfs_ibm_add(icx);
+	if (ret)
+		return ret;
+
+	if (!ntfs_attr_exist(icx->idx_ni, AT_INDEX_ALLOCATION, icx->name, icx->name_len)) {
+		ret = ntfs_attr_add(icx->idx_ni, AT_INDEX_ALLOCATION, icx->name,
+					icx->name_len, NULL, 0);
+		if (ret) {
+			ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_INDEX_ALLOCATION");
+			return ret;
+		}
+	}
+
+	icx->ia_ni = ntfs_ia_open(icx, icx->idx_ni);
+	if (!icx->ia_ni)
+		return -ENOENT;
+
+	return 0;
+}
+
+static int ntfs_ir_reparent(struct ntfs_index_context *icx)
+{
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct index_root *ir;
+	struct index_entry *ie;
+	struct index_block *ib = NULL;
+	s64 new_ib_vcn;
+	int ix_root_size;
+	int ret = 0;
+
+	ntfs_debug("Entering\n");
+
+	ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len);
+	if (!ir) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if ((ir->index.flags & NODE_MASK) == SMALL_INDEX) {
+		ret = ntfs_ia_add(icx);
+		if (ret)
+			goto out;
+	}
+
+	new_ib_vcn = ntfs_ibm_get_free(icx);
+	if (new_ib_vcn < 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len);
+	if (!ir) {
+		ret = -ENOENT;
+		goto clear_bmp;
+	}
+
+	ib = ntfs_ir_to_ib(ir, new_ib_vcn);
+	if (ib == NULL) {
+		ret = -EIO;
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to move index root to index block");
+		goto clear_bmp;
+	}
+
+	ret = ntfs_ib_write(icx, ib);
+	if (ret)
+		goto clear_bmp;
+
+retry:
+	ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx);
+	if (!ir) {
+		ret = -ENOENT;
+		goto clear_bmp;
+	}
+
+	ntfs_ir_nill(ir);
+
+	ie = ntfs_ie_get_first(&ir->index);
+	ie->flags |= INDEX_ENTRY_NODE;
+	ie->length = cpu_to_le16(sizeof(struct index_entry_header) + sizeof(s64));
+
+	ir->index.flags = LARGE_INDEX;
+	NInoSetIndexAllocPresent(icx->idx_ni);
+	ir->index.index_length = cpu_to_le32(le32_to_cpu(ir->index.entries_offset) +
+			le16_to_cpu(ie->length));
+	ir->index.allocated_size = ir->index.index_length;
+
+	ix_root_size = sizeof(struct index_root) - sizeof(struct index_header) +
+		le32_to_cpu(ir->index.allocated_size);
+	ret  = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size);
+	if (ret) {
+		/*
+		 * When there is no space to build a non-resident
+		 * index, we may have to move the root to an extent
+		 */
+		if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) {
+			ntfs_attr_put_search_ctx(ctx);
+			ctx = NULL;
+			ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx);
+			if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size -
+					le32_to_cpu(ctx->attr->data.resident.value_length))) {
+				if (ntfs_attrlist_update(ctx->base_ntfs_ino ?
+							 ctx->base_ntfs_ino : ctx->ntfs_ino))
+					goto clear_bmp;
+				ntfs_attr_put_search_ctx(ctx);
+				ctx = NULL;
+				goto retry;
+			}
+		}
+		goto clear_bmp;
+	} else {
+		icx->idx_ni->data_size = icx->idx_ni->initialized_size = ix_root_size;
+		icx->idx_ni->allocated_size = (ix_root_size  + 7) & ~7;
+	}
+	ntfs_ie_set_vcn(ie, new_ib_vcn);
+
+err_out:
+	kvfree(ib);
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+out:
+	return ret;
+clear_bmp:
+	ntfs_ibm_clear(icx, new_ib_vcn);
+	goto err_out;
+}
+
+/*
+ * ntfs_ir_truncate - Truncate index root attribute
+ * @icx: index context
+ * @data_size: new data size for the index root
+ */
+static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size)
+{
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	/*
+	 *  INDEX_ROOT must be resident and its entries can be moved to
+	 *  struct index_block, so ENOSPC isn't a real error.
+	 */
+	ret = ntfs_attr_truncate(icx->idx_ni, data_size + offsetof(struct index_root, index));
+	if (!ret) {
+		i_size_write(VFS_I(icx->idx_ni), icx->idx_ni->initialized_size);
+		icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len);
+		if (!icx->ir)
+			return -ENOENT;
+
+		icx->ir->index.allocated_size = cpu_to_le32(data_size);
+	} else if (ret != -ENOSPC)
+		ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT");
+
+	return ret;
+}
+
+/*
+ * ntfs_ir_make_space - Make more space for the index root attribute
+ * @icx: index context
+ * @data_size: required data size for the index root
+ */
+static int ntfs_ir_make_space(struct ntfs_index_context *icx, int data_size)
+{
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	ret = ntfs_ir_truncate(icx, data_size);
+	if (ret == -ENOSPC) {
+		ret = ntfs_ir_reparent(icx);
+		if (!ret)
+			ret = -EAGAIN;
+		else
+			ntfs_error(icx->idx_ni->vol->sb, "Failed to modify INDEX_ROOT");
+	}
+
+	return ret;
+}
+
+/*
+ * NOTE: 'ie' must be a copy of a real index entry.
+ */
+static int ntfs_ie_add_vcn(struct index_entry **ie)
+{
+	struct index_entry *p, *old = *ie;
+
+	old->length = cpu_to_le16(le16_to_cpu(old->length) + sizeof(s64));
+	p = krealloc(old, le16_to_cpu(old->length), GFP_NOFS);
+	if (!p)
+		return -ENOMEM;
+
+	p->flags |= INDEX_ENTRY_NODE;
+	*ie = p;
+	return 0;
+}
+
+static int ntfs_ih_insert(struct index_header *ih, struct index_entry *orig_ie, s64 new_vcn,
+		int pos)
+{
+	struct index_entry *ie_node, *ie;
+	int ret = 0;
+	s64 old_vcn;
+
+	ntfs_debug("Entering\n");
+	ie = ntfs_ie_dup(orig_ie);
+	if (!ie)
+		return -ENOMEM;
+
+	if (!(ie->flags & INDEX_ENTRY_NODE)) {
+		ret = ntfs_ie_add_vcn(&ie);
+		if (ret)
+			goto out;
+	}
+
+	ie_node = ntfs_ie_get_by_pos(ih, pos);
+	old_vcn = ntfs_ie_get_vcn(ie_node);
+	ntfs_ie_set_vcn(ie_node, new_vcn);
+
+	ntfs_ie_insert(ih, ie, ie_node);
+	ntfs_ie_set_vcn(ie_node, old_vcn);
+out:
+	kfree(ie);
+	return ret;
+}
+
+static s64 ntfs_icx_parent_vcn(struct ntfs_index_context *icx)
+{
+	return icx->parent_vcn[icx->pindex];
+}
+
+static s64 ntfs_icx_parent_pos(struct ntfs_index_context *icx)
+{
+	return icx->parent_pos[icx->pindex];
+}
+
+static int ntfs_ir_insert_median(struct ntfs_index_context *icx, struct index_entry *median,
+		s64 new_vcn)
+{
+	u32 new_size;
+	int ret;
+
+	ntfs_debug("Entering\n");
+
+	icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len);
+	if (!icx->ir)
+		return -ENOENT;
+
+	new_size = le32_to_cpu(icx->ir->index.index_length) +
+		le16_to_cpu(median->length);
+	if (!(median->flags & INDEX_ENTRY_NODE))
+		new_size += sizeof(s64);
+
+	ret = ntfs_ir_make_space(icx, new_size);
+	if (ret)
+		return ret;
+
+	icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len);
+	if (!icx->ir)
+		return -ENOENT;
+
+	return ntfs_ih_insert(&icx->ir->index, median, new_vcn,
+			ntfs_icx_parent_pos(icx));
+}
+
+static int ntfs_ib_split(struct ntfs_index_context *icx, struct index_block *ib);
+
+struct split_info {
+	struct list_head entry;
+	s64 new_vcn;
+	struct index_block *ib;
+};
+
+static int ntfs_ib_insert(struct ntfs_index_context *icx, struct index_entry *ie, s64 new_vcn,
+		struct split_info *si)
+{
+	struct index_block *ib;
+	u32 idx_size, allocated_size;
+	int err;
+	s64 old_vcn;
+
+	ntfs_debug("Entering\n");
+
+	ib = kvzalloc(icx->block_size, GFP_NOFS);
+	if (!ib)
+		return -ENOMEM;
+
+	old_vcn = ntfs_icx_parent_vcn(icx);
+
+	err = ntfs_ib_read(icx, old_vcn, ib);
+	if (err)
+		goto err_out;
+
+	idx_size = le32_to_cpu(ib->index.index_length);
+	allocated_size = le32_to_cpu(ib->index.allocated_size);
+	if (idx_size + le16_to_cpu(ie->length) + sizeof(s64) > allocated_size) {
+		si->ib = ib;
+		si->new_vcn = new_vcn;
+		return -EAGAIN;
+	}
+
+	err = ntfs_ih_insert(&ib->index, ie, new_vcn, ntfs_icx_parent_pos(icx));
+	if (err)
+		goto err_out;
+
+	err = ntfs_ib_write(icx, ib);
+
+err_out:
+	kvfree(ib);
+	return err;
+}
+
+/*
+ * ntfs_ib_split - Split an index block
+ * @icx: index context
+ * @ib: index block to split
+ */
+static int ntfs_ib_split(struct ntfs_index_context *icx, struct index_block *ib)
+{
+	struct index_entry *median;
+	s64 new_vcn;
+	int ret;
+	struct split_info *si;
+	LIST_HEAD(ntfs_cut_tail_list);
+
+	ntfs_debug("Entering\n");
+
+resplit:
+	ret = ntfs_icx_parent_dec(icx);
+	if (ret)
+		goto out;
+
+	median  = ntfs_ie_get_median(&ib->index);
+	new_vcn = ntfs_ibm_get_free(icx);
+	if (new_vcn < 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = ntfs_ib_copy_tail(icx, ib, median, new_vcn);
+	if (ret) {
+		ntfs_ibm_clear(icx, new_vcn);
+		goto out;
+	}
+
+	if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) {
+		ret = ntfs_ir_insert_median(icx, median, new_vcn);
+		if (ret) {
+			ntfs_ibm_clear(icx, new_vcn);
+			goto out;
+		}
+	} else {
+		si = kzalloc(sizeof(struct split_info), GFP_NOFS);
+		if (!si) {
+			ntfs_ibm_clear(icx, new_vcn);
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		ret = ntfs_ib_insert(icx, median, new_vcn, si);
+		if (ret == -EAGAIN) {
+			list_add_tail(&si->entry, &ntfs_cut_tail_list);
+			ib = si->ib;
+			goto resplit;
+		} else if (ret) {
+			kvfree(si->ib);
+			kfree(si);
+			ntfs_ibm_clear(icx, new_vcn);
+			goto out;
+		}
+		kfree(si);
+	}
+
+	ret = ntfs_ib_cut_tail(icx, ib, median);
+
+out:
+	while (!list_empty(&ntfs_cut_tail_list)) {
+		si = list_last_entry(&ntfs_cut_tail_list, struct split_info, entry);
+		ntfs_ibm_clear(icx, si->new_vcn);
+		kvfree(si->ib);
+		list_del(&si->entry);
+		kfree(si);
+		if (!ret)
+			ret = -EAGAIN;
+	}
+
+	return ret;
+}
+
+int ntfs_ie_add(struct ntfs_index_context *icx, struct index_entry *ie)
+{
+	struct index_header *ih;
+	int allocated_size, new_size;
+	int ret;
+
+	while (1) {
+		ret = ntfs_index_lookup(&ie->key, le16_to_cpu(ie->key_length), icx);
+		if (!ret) {
+			ret = -EEXIST;
+			ntfs_error(icx->idx_ni->vol->sb, "Index already have such entry");
+			goto err_out;
+		}
+		if (ret != -ENOENT) {
+			ntfs_error(icx->idx_ni->vol->sb, "Failed to find place for new entry");
+			goto err_out;
+		}
+		ret = 0;
+
+		if (icx->is_in_root)
+			ih = &icx->ir->index;
+		else
+			ih = &icx->ib->index;
+
+		allocated_size = le32_to_cpu(ih->allocated_size);
+		new_size = le32_to_cpu(ih->index_length) + le16_to_cpu(ie->length);
+
+		if (new_size <= allocated_size)
+			break;
+
+		ntfs_debug("index block sizes: allocated: %d  needed: %d\n",
+				allocated_size, new_size);
+
+		if (icx->is_in_root)
+			ret = ntfs_ir_make_space(icx, new_size);
+		else
+			ret = ntfs_ib_split(icx, icx->ib);
+		if (ret && ret != -EAGAIN)
+			goto err_out;
+
+		mark_mft_record_dirty(icx->actx->ntfs_ino);
+		ntfs_index_ctx_reinit(icx);
+	}
+
+	ntfs_ie_insert(ih, ie, icx->entry);
+	ntfs_index_entry_mark_dirty(icx);
+
+err_out:
+	ntfs_debug("%s\n", ret ? "Failed" : "Done");
+	return ret;
+}
+
+/*
+ * ntfs_index_add_filename - add filename to directory index
+ * @ni:		ntfs inode describing directory to which index add filename
+ * @fn:		FILE_NAME attribute to add
+ * @mref:	reference of the inode which @fn describes
+ */
+int ntfs_index_add_filename(struct ntfs_inode *ni, struct file_name_attr *fn, u64 mref)
+{
+	struct index_entry *ie;
+	struct ntfs_index_context *icx;
+	int fn_size, ie_size, err;
+
+	ntfs_debug("Entering\n");
+
+	if (!ni || !fn)
+		return -EINVAL;
+
+	fn_size = (fn->file_name_length * sizeof(__le16)) +
+		sizeof(struct file_name_attr);
+	ie_size = (sizeof(struct index_entry_header) + fn_size + 7) & ~7;
+
+	ie = kzalloc(ie_size, GFP_NOFS);
+	if (!ie)
+		return -ENOMEM;
+
+	ie->data.dir.indexed_file = cpu_to_le64(mref);
+	ie->length	 = cpu_to_le16(ie_size);
+	ie->key_length	 = cpu_to_le16(fn_size);
+
+	unsafe_memcpy(&ie->key, fn, fn_size,
+		      /* "fn_size" was correctly calculated above */);
+
+	icx = ntfs_index_ctx_get(ni, I30, 4);
+	if (!icx) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = ntfs_ie_add(icx, ie);
+	ntfs_index_ctx_put(icx);
+out:
+	kfree(ie);
+	return err;
+}
+
+static int ntfs_ih_takeout(struct ntfs_index_context *icx, struct index_header *ih,
+		struct index_entry *ie, struct index_block *ib)
+{
+	struct index_entry *ie_roam;
+	int freed_space;
+	bool full;
+	int ret = 0;
+
+	ntfs_debug("Entering\n");
+
+	full = ih->index_length == ih->allocated_size;
+	ie_roam = ntfs_ie_dup_novcn(ie);
+	if (!ie_roam)
+		return -ENOMEM;
+
+	ntfs_ie_delete(ih, ie);
+
+	if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) {
+		/*
+		 * Recover the space which may have been freed
+		 * while deleting an entry from root index
+		 */
+		freed_space = le32_to_cpu(ih->allocated_size) -
+			le32_to_cpu(ih->index_length);
+		if (full && (freed_space > 0) && !(freed_space & 7)) {
+			ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length));
+			/* do nothing if truncation fails */
+		}
+
+		mark_mft_record_dirty(icx->actx->ntfs_ino);
+	} else {
+		ret = ntfs_ib_write(icx, ib);
+		if (ret)
+			goto out;
+	}
+
+	ntfs_index_ctx_reinit(icx);
+
+	ret = ntfs_ie_add(icx, ie_roam);
+out:
+	kfree(ie_roam);
+	return ret;
+}
+
+/*
+ *  Used if an empty index block to be deleted has END entry as the parent
+ *  in the INDEX_ROOT which is the only one there.
+ */
+static void ntfs_ir_leafify(struct ntfs_index_context *icx, struct index_header *ih)
+{
+	struct index_entry *ie;
+
+	ntfs_debug("Entering\n");
+
+	ie = ntfs_ie_get_first(ih);
+	ie->flags &= ~INDEX_ENTRY_NODE;
+	ie->length = cpu_to_le16(le16_to_cpu(ie->length) - sizeof(s64));
+
+	ih->index_length = cpu_to_le32(le32_to_cpu(ih->index_length) - sizeof(s64));
+	ih->flags &= ~LARGE_INDEX;
+	NInoClearIndexAllocPresent(icx->idx_ni);
+
+	/* Not fatal error */
+	ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length));
+}
+
+/*
+ *  Used if an empty index block to be deleted has END entry as the parent
+ *  in the INDEX_ROOT which is not the only one there.
+ */
+static int ntfs_ih_reparent_end(struct ntfs_index_context *icx, struct index_header *ih,
+		struct index_block *ib)
+{
+	struct index_entry *ie, *ie_prev;
+
+	ntfs_debug("Entering\n");
+
+	ie = ntfs_ie_get_by_pos(ih, ntfs_icx_parent_pos(icx));
+	ie_prev = ntfs_ie_prev(ih, ie);
+	if (!ie_prev)
+		return -EIO;
+	ntfs_ie_set_vcn(ie, ntfs_ie_get_vcn(ie_prev));
+
+	return ntfs_ih_takeout(icx, ih, ie_prev, ib);
+}
+
+static int ntfs_index_rm_leaf(struct ntfs_index_context *icx)
+{
+	struct index_block *ib = NULL;
+	struct index_header *parent_ih;
+	struct index_entry *ie;
+	int ret;
+
+	ntfs_debug("pindex: %d\n", icx->pindex);
+
+	ret = ntfs_icx_parent_dec(icx);
+	if (ret)
+		return ret;
+
+	ret = ntfs_ibm_clear(icx, icx->parent_vcn[icx->pindex + 1]);
+	if (ret)
+		return ret;
+
+	if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT)
+		parent_ih = &icx->ir->index;
+	else {
+		ib = kvzalloc(icx->block_size, GFP_NOFS);
+		if (!ib)
+			return -ENOMEM;
+
+		ret = ntfs_ib_read(icx, ntfs_icx_parent_vcn(icx), ib);
+		if (ret)
+			goto out;
+
+		parent_ih = &ib->index;
+	}
+
+	ie = ntfs_ie_get_by_pos(parent_ih, ntfs_icx_parent_pos(icx));
+	if (!ntfs_ie_end(ie)) {
+		ret = ntfs_ih_takeout(icx, parent_ih, ie, ib);
+		goto out;
+	}
+
+	if (ntfs_ih_zero_entry(parent_ih)) {
+		if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) {
+			ntfs_ir_leafify(icx, parent_ih);
+			goto out;
+		}
+
+		ret = ntfs_index_rm_leaf(icx);
+		goto out;
+	}
+
+	ret = ntfs_ih_reparent_end(icx, parent_ih, ib);
+out:
+	kvfree(ib);
+	return ret;
+}
+
+static int ntfs_index_rm_node(struct ntfs_index_context *icx)
+{
+	int entry_pos, pindex;
+	s64 vcn;
+	struct index_block *ib = NULL;
+	struct index_entry *ie_succ, *ie, *entry = icx->entry;
+	struct index_header *ih;
+	u32 new_size;
+	int delta, ret;
+
+	ntfs_debug("Entering\n");
+
+	if (!icx->ia_ni) {
+		icx->ia_ni = ntfs_ia_open(icx, icx->idx_ni);
+		if (!icx->ia_ni)
+			return -EINVAL;
+	}
+
+	ib = kvzalloc(icx->block_size, GFP_NOFS);
+	if (!ib)
+		return -ENOMEM;
+
+	ie_succ = ntfs_ie_get_next(icx->entry);
+	entry_pos = icx->parent_pos[icx->pindex]++;
+	pindex = icx->pindex;
+descend:
+	vcn = ntfs_ie_get_vcn(ie_succ);
+	ret = ntfs_ib_read(icx, vcn, ib);
+	if (ret)
+		goto out;
+
+	ie_succ = ntfs_ie_get_first(&ib->index);
+
+	ret = ntfs_icx_parent_inc(icx);
+	if (ret)
+		goto out;
+
+	icx->parent_vcn[icx->pindex] = vcn;
+	icx->parent_pos[icx->pindex] = 0;
+
+	if ((ib->index.flags & NODE_MASK) == INDEX_NODE)
+		goto descend;
+
+	if (ntfs_ih_zero_entry(&ib->index)) {
+		ret = -EIO;
+		ntfs_error(icx->idx_ni->vol->sb, "Empty index block");
+		goto out;
+	}
+
+	ie = ntfs_ie_dup(ie_succ);
+	if (!ie) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = ntfs_ie_add_vcn(&ie);
+	if (ret)
+		goto out2;
+
+	ntfs_ie_set_vcn(ie, ntfs_ie_get_vcn(icx->entry));
+
+	if (icx->is_in_root)
+		ih = &icx->ir->index;
+	else
+		ih = &icx->ib->index;
+
+	delta = le16_to_cpu(ie->length) - le16_to_cpu(icx->entry->length);
+	new_size = le32_to_cpu(ih->index_length) + delta;
+	if (delta > 0) {
+		if (icx->is_in_root) {
+			ret = ntfs_ir_make_space(icx, new_size);
+			if (ret != 0)
+				goto out2;
+
+			ih = &icx->ir->index;
+			entry = ntfs_ie_get_by_pos(ih, entry_pos);
+
+		} else if (new_size > le32_to_cpu(ih->allocated_size)) {
+			icx->pindex = pindex;
+			ret = ntfs_ib_split(icx, icx->ib);
+			if (!ret)
+				ret = -EAGAIN;
+			goto out2;
+		}
+	}
+
+	ntfs_ie_delete(ih, entry);
+	ntfs_ie_insert(ih, ie, entry);
+
+	if (icx->is_in_root)
+		ret = ntfs_ir_truncate(icx, new_size);
+	else
+		ret = ntfs_icx_ib_write(icx);
+	if (ret)
+		goto out2;
+
+	ntfs_ie_delete(&ib->index, ie_succ);
+
+	if (ntfs_ih_zero_entry(&ib->index))
+		ret = ntfs_index_rm_leaf(icx);
+	else
+		ret = ntfs_ib_write(icx, ib);
+
+out2:
+	kfree(ie);
+out:
+	kvfree(ib);
+	return ret;
+}
+
+/*
+ * ntfs_index_rm - remove entry from the index
+ * @icx:	index context describing entry to delete
+ *
+ * Delete entry described by @icx from the index. Index context is always
+ * reinitialized after use of this function, so it can be used for index
+ * lookup once again.
+ */
+int ntfs_index_rm(struct ntfs_index_context *icx)
+{
+	struct index_header *ih;
+	int ret = 0;
+
+	ntfs_debug("Entering\n");
+
+	if (!icx || (!icx->ib && !icx->ir) || ntfs_ie_end(icx->entry)) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+	if (icx->is_in_root)
+		ih = &icx->ir->index;
+	else
+		ih = &icx->ib->index;
+
+	if (icx->entry->flags & INDEX_ENTRY_NODE) {
+		ret = ntfs_index_rm_node(icx);
+		if (ret)
+			goto err_out;
+	} else if (icx->is_in_root || !ntfs_ih_one_entry(ih)) {
+		ntfs_ie_delete(ih, icx->entry);
+
+		if (icx->is_in_root)
+			ret = ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length));
+		else
+			ret = ntfs_icx_ib_write(icx);
+		if (ret)
+			goto err_out;
+	} else {
+		ret = ntfs_index_rm_leaf(icx);
+		if (ret)
+			goto err_out;
+	}
+
+	return 0;
+err_out:
+	return ret;
+}
+
+int ntfs_index_remove(struct ntfs_inode *dir_ni, const void *key, const u32 keylen)
+{
+	int ret = 0;
+	struct ntfs_index_context *icx;
+
+	icx = ntfs_index_ctx_get(dir_ni, I30, 4);
+	if (!icx)
+		return -EINVAL;
+
+	while (1) {
+		ret = ntfs_index_lookup(key, keylen, icx);
+		if (ret)
+			goto err_out;
+
+		ret = ntfs_index_rm(icx);
+		if (ret && ret != -EAGAIN)
+			goto err_out;
+		else if (!ret)
+			break;
+
+		mark_mft_record_dirty(icx->actx->ntfs_ino);
+		ntfs_index_ctx_reinit(icx);
+	}
+
+	mark_mft_record_dirty(icx->actx->ntfs_ino);
+
+	ntfs_index_ctx_put(icx);
+	return 0;
+err_out:
+	ntfs_index_ctx_put(icx);
+	ntfs_error(dir_ni->vol->sb, "Delete failed");
+	return ret;
+}
+
+/*
+ * ntfs_index_walk_down - walk down the index tree (leaf bound)
+ * until there are no subnode in the first index entry returns
+ * the entry at the bottom left in subnode
+ */
+struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_index_context *ictx)
+{
+	struct index_entry *entry;
+	s64 vcn;
+
+	entry = ie;
+	do {
+		vcn = ntfs_ie_get_vcn(entry);
+		if (ictx->is_in_root) {
+			/* down from level zero */
+			ictx->ir = NULL;
+			ictx->ib = kvzalloc(ictx->block_size, GFP_NOFS);
+			ictx->pindex = 1;
+			ictx->is_in_root = false;
+		} else {
+			/* down from non-zero level */
+			ictx->pindex++;
+		}
+
+		ictx->parent_pos[ictx->pindex] = 0;
+		ictx->parent_vcn[ictx->pindex] = vcn;
+		if (!ntfs_ib_read(ictx, vcn, ictx->ib)) {
+			ictx->entry = ntfs_ie_get_first(&ictx->ib->index);
+			entry = ictx->entry;
+		} else
+			entry = NULL;
+	} while (entry && (entry->flags & INDEX_ENTRY_NODE));
+
+	return entry;
+}
+
+/*
+ * ntfs_index_walk_up - walk up the index tree (root bound) until
+ * there is a valid data entry in parent returns the parent entry
+ * or NULL if no more parent.
+ * @ie: current index entry
+ * @ictx: index context
+ */
+static struct index_entry *ntfs_index_walk_up(struct index_entry *ie,
+		struct ntfs_index_context *ictx)
+{
+	struct index_entry *entry = ie;
+	s64 vcn;
+
+	if (ictx->pindex <= 0)
+		return NULL;
+
+	do {
+		ictx->pindex--;
+		if (!ictx->pindex) {
+			/* we have reached the root */
+			kfree(ictx->ib);
+			ictx->ib = NULL;
+			ictx->is_in_root = true;
+			/* a new search context is to be allocated */
+			if (ictx->actx)
+				ntfs_attr_put_search_ctx(ictx->actx);
+			ictx->ir = ntfs_ir_lookup(ictx->idx_ni, ictx->name,
+						  ictx->name_len, &ictx->actx);
+			if (ictx->ir)
+				entry = ntfs_ie_get_by_pos(
+					&ictx->ir->index,
+					ictx->parent_pos[ictx->pindex]);
+			else
+				entry = NULL;
+		} else {
+			/* up into non-root node */
+			vcn = ictx->parent_vcn[ictx->pindex];
+			if (!ntfs_ib_read(ictx, vcn, ictx->ib)) {
+				entry = ntfs_ie_get_by_pos(
+					&ictx->ib->index,
+					ictx->parent_pos[ictx->pindex]);
+			} else
+				entry = NULL;
+		}
+		ictx->entry = entry;
+	} while (entry && (ictx->pindex > 0) &&
+		 (entry->flags & INDEX_ENTRY_END));
+	return entry;
+}
+
+/*
+ * ntfs_index_next - get next entry in an index according to collating sequence.
+ * Returns next entry or NULL if none.
+ *
+ * Sample layout :
+ *
+ *                 +---+---+---+---+---+---+---+---+    n ptrs to subnodes
+ *                 |   |   | 10| 25| 33|   |   |   |    n-1 keys in between
+ *                 +---+---+---+---+---+---+---+---+    no key in last entry
+ *                              | A | A
+ *                              | | | +-------------------------------+
+ *   +--------------------------+ | +-----+                           |
+ *   |                            +--+    |                           |
+ *   V                               |    V                           |
+ * +---+---+---+---+---+---+---+---+ |  +---+---+---+---+---+---+---+---+
+ * | 11| 12| 13| 14| 15| 16| 17|   | |  | 26| 27| 28| 29| 30| 31| 32|   |
+ * +---+---+---+---+---+---+---+---+ |  +---+---+---+---+---+---+---+---+
+ *                               |   |
+ *       +-----------------------+   |
+ *       |                           |
+ *     +---+---+---+---+---+---+---+---+
+ *     | 18| 19| 20| 21| 22| 23| 24|   |
+ *     +---+---+---+---+---+---+---+---+
+ *
+ * @ie: current index entry
+ * @ictx: index context
+ */
+struct index_entry *ntfs_index_next(struct index_entry *ie, struct ntfs_index_context *ictx)
+{
+	struct index_entry *next;
+	__le16 flags;
+
+	/*
+	 * lookup() may have returned an invalid node
+	 * when searching for a partial key
+	 * if this happens, walk up
+	 */
+	if (ie->flags & INDEX_ENTRY_END)
+		next = ntfs_index_walk_up(ie, ictx);
+	else {
+		/*
+		 * get next entry in same node
+		 * there is always one after any entry with data
+		 */
+		next = (struct index_entry *)((char *)ie + le16_to_cpu(ie->length));
+		++ictx->parent_pos[ictx->pindex];
+		flags = next->flags;
+
+		/* walk down if it has a subnode */
+		if (flags & INDEX_ENTRY_NODE) {
+			if (!ictx->ia_ni)
+				ictx->ia_ni = ntfs_ia_open(ictx, ictx->idx_ni);
+
+			next = ntfs_index_walk_down(next, ictx);
+		} else {
+
+			/* walk up it has no subnode, nor data */
+			if (flags & INDEX_ENTRY_END)
+				next = ntfs_index_walk_up(next, ictx);
+		}
+	}
+
+	/* return NULL if stuck at end of a block */
+	if (next && (next->flags & INDEX_ENTRY_END))
+		next = NULL;
+
+	return next;
+}
diff --git a/fs/ntfs/index.h b/fs/ntfs/index.h
new file mode 100644
index 0000000000000..e68d6fabaf9f1
--- /dev/null
+++ b/fs/ntfs/index.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS kernel index handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_INDEX_H
+#define _LINUX_NTFS_INDEX_H
+
+#include <linux/fs.h>
+
+#include "attrib.h"
+#include "mft.h"
+
+#define  VCN_INDEX_ROOT_PARENT  ((s64)-2)
+
+#define MAX_PARENT_VCN	32
+
+/*
+ * @idx_ni:	index inode containing the @entry described by this context
+ * @name:	Unicode name of the indexed attribute
+ *		(usually $I30 for directories)
+ * @name_len:	length of @name in Unicode characters
+ * @entry:	index entry (points into @ir or @ia)
+ * @cr:		creation time of the entry (for sorting/validation)
+ * @data:	index entry data (points into @entry)
+ * @data_len:	length in bytes of @data
+ * @is_in_root:	'true' if @entry is in @ir and 'false' if it is in @ia
+ * @ir:		index root if @is_in_root and NULL otherwise
+ * @actx:	attribute search context if @is_in_root and NULL otherwise
+ * @ib:		index block header (valid when @is_in_root is 'false')
+ * @ia_ni:	index allocation inode (extent inode) for @ia
+ * @parent_pos:	array of parent entry positions in the B-tree nodes
+ * @parent_vcn:	VCNs of parent index blocks in the B-tree traversal
+ * @pindex:	current depth (number of parent nodes) in the traversal
+ *		(maximum is MAX_PARENT_VCN)
+ * @ib_dirty:	true if the current index block (@ia/@ib) was modified
+ * @block_size:	size of index blocks in bytes (from $INDEX_ROOT or $Boot)
+ * @vcn_size_bits: log2(cluster size)
+ * @sync_write:	true if synchronous writeback is requested for this context
+ *
+ * @idx_ni is the index inode this context belongs to.
+ *
+ * @entry is the index entry described by this context.  @data and @data_len
+ * are the index entry data and its length in bytes, respectively.  @data
+ * simply points into @entry.  This is probably what the user is interested in.
+ *
+ * If @is_in_root is 'true', @entry is in the index root attribute @ir described
+ * by the attribute search context @actx and the base inode @base_ni.  @ia and
+ * @page are NULL in this case.
+ *
+ * If @is_in_root is 'false', @entry is in the index allocation attribute and @ia
+ * and @page point to the index allocation block and the mapped, locked page it
+ * is in, respectively.  @ir, @actx and @base_ni are NULL in this case.
+ *
+ * To obtain a context call ntfs_index_ctx_get().
+ *
+ * We use this context to allow ntfs_index_lookup() to return the found index
+ * @entry and its @data without having to allocate a buffer and copy the @entry
+ * and/or its @data into it.
+ *
+ * When finished with the @entry and its @data, call ntfs_index_ctx_put() to
+ * free the context and other associated resources.
+ *
+ * If the index entry was modified, ntfs_index_entry_mark_dirty()
+ * or ntfs_index_entry_write() before the call to ntfs_index_ctx_put() to
+ * ensure that the changes are written to disk.
+ */
+struct ntfs_index_context {
+	struct ntfs_inode *idx_ni;
+	__le16 *name;
+	u32 name_len;
+	struct index_entry *entry;
+	__le32 cr;
+	void *data;
+	u16 data_len;
+	bool is_in_root;
+	struct index_root *ir;
+	struct ntfs_attr_search_ctx *actx;
+	struct index_block *ib;
+	struct ntfs_inode *ia_ni;
+	int parent_pos[MAX_PARENT_VCN];
+	s64 parent_vcn[MAX_PARENT_VCN];
+	int pindex;
+	bool ib_dirty;
+	u32 block_size;
+	u8 vcn_size_bits;
+	bool sync_write;
+};
+
+int ntfs_index_entry_inconsistent(struct ntfs_index_context *icx, struct ntfs_volume *vol,
+		const struct index_entry *ie, __le32 collation_rule, u64 inum);
+struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni, __le16 *name,
+		u32 name_len);
+void ntfs_index_ctx_put(struct ntfs_index_context *ictx);
+int ntfs_index_lookup(const void *key, const u32 key_len,
+		struct ntfs_index_context *ictx);
+
+void ntfs_index_entry_mark_dirty(struct ntfs_index_context *ictx);
+int ntfs_index_add_filename(struct ntfs_inode *ni, struct file_name_attr *fn, u64 mref);
+int ntfs_index_remove(struct ntfs_inode *ni, const void *key, const u32 keylen);
+struct ntfs_inode *ntfs_ia_open(struct ntfs_index_context *icx, struct ntfs_inode *ni);
+struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_index_context *ictx);
+struct index_entry *ntfs_index_next(struct index_entry *ie, struct ntfs_index_context *ictx);
+int ntfs_index_rm(struct ntfs_index_context *icx);
+void ntfs_index_ctx_reinit(struct ntfs_index_context *icx);
+int ntfs_ie_add(struct ntfs_index_context *icx, struct index_entry *ie);
+int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx);
+
+#endif /* _LINUX_NTFS_INDEX_H */
diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c
new file mode 100644
index 0000000000000..314741a403698
--- /dev/null
+++ b/fs/ntfs/inode.c
@@ -0,0 +1,3815 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel inode handling.
+ *
+ * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/writeback.h>
+#include <linux/seq_file.h>
+
+#include "lcnalloc.h"
+#include "time.h"
+#include "ntfs.h"
+#include "index.h"
+#include "attrlist.h"
+#include "reparse.h"
+#include "ea.h"
+#include "attrib.h"
+#include "iomap.h"
+#include "object_id.h"
+
+/*
+ * ntfs_test_inode - compare two (possibly fake) inodes for equality
+ * @vi:		vfs inode which to test
+ * @data:	data which is being tested with
+ *
+ * Compare the ntfs attribute embedded in the ntfs specific part of the vfs
+ * inode @vi for equality with the ntfs attribute @data.
+ *
+ * If searching for the normal file/directory inode, set @na->type to AT_UNUSED.
+ * @na->name and @na->name_len are then ignored.
+ *
+ * Return 1 if the attributes match and 0 if not.
+ *
+ * NOTE: This function runs with the inode_hash_lock spin lock held so it is not
+ * allowed to sleep.
+ */
+int ntfs_test_inode(struct inode *vi, void *data)
+{
+	struct ntfs_attr *na = data;
+	struct ntfs_inode *ni = NTFS_I(vi);
+
+	if (vi->i_ino != na->mft_no)
+		return 0;
+
+	/* If !NInoAttr(ni), @vi is a normal file or directory inode. */
+	if (likely(!NInoAttr(ni))) {
+		/* If not looking for a normal inode this is a mismatch. */
+		if (unlikely(na->type != AT_UNUSED))
+			return 0;
+	} else {
+		/* A fake inode describing an attribute. */
+		if (ni->type != na->type)
+			return 0;
+		if (ni->name_len != na->name_len)
+			return 0;
+		if (na->name_len && memcmp(ni->name, na->name,
+				na->name_len * sizeof(__le16)))
+			return 0;
+		if (!ni->ext.base_ntfs_ino)
+			return 0;
+	}
+
+	/* Match! */
+	return 1;
+}
+
+/*
+ * ntfs_init_locked_inode - initialize an inode
+ * @vi:		vfs inode to initialize
+ * @data:	data which to initialize @vi to
+ *
+ * Initialize the vfs inode @vi with the values from the ntfs attribute @data in
+ * order to enable ntfs_test_inode() to do its work.
+ *
+ * If initializing the normal file/directory inode, set @na->type to AT_UNUSED.
+ * In that case, @na->name and @na->name_len should be set to NULL and 0,
+ * respectively. Although that is not strictly necessary as
+ * ntfs_read_locked_inode() will fill them in later.
+ *
+ * Return 0 on success and error.
+ *
+ * NOTE: This function runs with the inode->i_lock spin lock held so it is not
+ * allowed to sleep. (Hence the GFP_ATOMIC allocation.)
+ */
+static int ntfs_init_locked_inode(struct inode *vi, void *data)
+{
+	struct ntfs_attr *na = data;
+	struct ntfs_inode *ni = NTFS_I(vi);
+
+	vi->i_ino = (unsigned long)na->mft_no;
+
+	if (na->type == AT_INDEX_ALLOCATION)
+		NInoSetMstProtected(ni);
+	else
+		ni->type = na->type;
+
+	ni->name = na->name;
+	ni->name_len = na->name_len;
+	ni->folio = NULL;
+	atomic_set(&ni->count, 1);
+
+	/* If initializing a normal inode, we are done. */
+	if (likely(na->type == AT_UNUSED))
+		return 0;
+
+	/* It is a fake inode. */
+	NInoSetAttr(ni);
+
+	/*
+	 * We have I30 global constant as an optimization as it is the name
+	 * in >99.9% of named attributes! The other <0.1% incur a GFP_ATOMIC
+	 * allocation but that is ok. And most attributes are unnamed anyway,
+	 * thus the fraction of named attributes with name != I30 is actually
+	 * absolutely tiny.
+	 */
+	if (na->name_len && na->name != I30) {
+		unsigned int i;
+
+		i = na->name_len * sizeof(__le16);
+		ni->name = kmalloc(i + sizeof(__le16), GFP_ATOMIC);
+		if (!ni->name)
+			return -ENOMEM;
+		memcpy(ni->name, na->name, i);
+		ni->name[na->name_len] = 0;
+	}
+	return 0;
+}
+
+static int ntfs_read_locked_inode(struct inode *vi);
+static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi);
+static int ntfs_read_locked_index_inode(struct inode *base_vi,
+		struct inode *vi);
+
+/*
+ * ntfs_iget - obtain a struct inode corresponding to a specific normal inode
+ * @sb:		super block of mounted volume
+ * @mft_no:	mft record number / inode number to obtain
+ *
+ * Obtain the struct inode corresponding to a specific normal inode (i.e. a
+ * file or directory).
+ *
+ * If the inode is in the cache, it is just returned with an increased
+ * reference count. Otherwise, a new struct inode is allocated and initialized,
+ * and finally ntfs_read_locked_inode() is called to read in the inode and
+ * fill in the remainder of the inode structure.
+ *
+ * Return the struct inode on success. Check the return value with IS_ERR() and
+ * if true, the function failed and the error code is obtained from PTR_ERR().
+ */
+struct inode *ntfs_iget(struct super_block *sb, u64 mft_no)
+{
+	struct inode *vi;
+	int err;
+	struct ntfs_attr na;
+
+	na.mft_no = mft_no;
+	na.type = AT_UNUSED;
+	na.name = NULL;
+	na.name_len = 0;
+
+	vi = iget5_locked(sb, mft_no, ntfs_test_inode,
+			ntfs_init_locked_inode, &na);
+	if (unlikely(!vi))
+		return ERR_PTR(-ENOMEM);
+
+	err = 0;
+
+	/* If this is a freshly allocated inode, need to read it now. */
+	if (inode_state_read_once(vi) & I_NEW) {
+		err = ntfs_read_locked_inode(vi);
+		unlock_new_inode(vi);
+	}
+	/*
+	 * There is no point in keeping bad inodes around if the failure was
+	 * due to ENOMEM. We want to be able to retry again later.
+	 */
+	if (unlikely(err == -ENOMEM)) {
+		iput(vi);
+		vi = ERR_PTR(err);
+	}
+	return vi;
+}
+
+/*
+ * ntfs_attr_iget - obtain a struct inode corresponding to an attribute
+ * @base_vi:	vfs base inode containing the attribute
+ * @type:	attribute type
+ * @name:	Unicode name of the attribute (NULL if unnamed)
+ * @name_len:	length of @name in Unicode characters (0 if unnamed)
+ *
+ * Obtain the (fake) struct inode corresponding to the attribute specified by
+ * @type, @name, and @name_len, which is present in the base mft record
+ * specified by the vfs inode @base_vi.
+ *
+ * If the attribute inode is in the cache, it is just returned with an
+ * increased reference count. Otherwise, a new struct inode is allocated and
+ * initialized, and finally ntfs_read_locked_attr_inode() is called to read the
+ * attribute and fill in the inode structure.
+ *
+ * Note, for index allocation attributes, you need to use ntfs_index_iget()
+ * instead of ntfs_attr_iget() as working with indices is a lot more complex.
+ *
+ * Return the struct inode of the attribute inode on success. Check the return
+ * value with IS_ERR() and if true, the function failed and the error code is
+ * obtained from PTR_ERR().
+ */
+struct inode *ntfs_attr_iget(struct inode *base_vi, __le32 type,
+		__le16 *name, u32 name_len)
+{
+	struct inode *vi;
+	int err;
+	struct ntfs_attr na;
+
+	/* Make sure no one calls ntfs_attr_iget() for indices. */
+	WARN_ON(type == AT_INDEX_ALLOCATION);
+
+	na.mft_no = base_vi->i_ino;
+	na.type = type;
+	na.name = name;
+	na.name_len = name_len;
+
+	vi = iget5_locked(base_vi->i_sb, na.mft_no, ntfs_test_inode,
+			ntfs_init_locked_inode, &na);
+	if (unlikely(!vi))
+		return ERR_PTR(-ENOMEM);
+	err = 0;
+
+	/* If this is a freshly allocated inode, need to read it now. */
+	if (inode_state_read_once(vi) & I_NEW) {
+		err = ntfs_read_locked_attr_inode(base_vi, vi);
+		unlock_new_inode(vi);
+	}
+	/*
+	 * There is no point in keeping bad attribute inodes around. This also
+	 * simplifies things in that we never need to check for bad attribute
+	 * inodes elsewhere.
+	 */
+	if (unlikely(err)) {
+		iput(vi);
+		vi = ERR_PTR(err);
+	}
+	return vi;
+}
+
+/*
+ * ntfs_index_iget - obtain a struct inode corresponding to an index
+ * @base_vi:	vfs base inode containing the index related attributes
+ * @name:	Unicode name of the index
+ * @name_len:	length of @name in Unicode characters
+ *
+ * Obtain the (fake) struct inode corresponding to the index specified by @name
+ * and @name_len, which is present in the base mft record specified by the vfs
+ * inode @base_vi.
+ *
+ * If the index inode is in the cache, it is just returned with an increased
+ * reference count.  Otherwise, a new struct inode is allocated and
+ * initialized, and finally ntfs_read_locked_index_inode() is called to read
+ * the index related attributes and fill in the inode structure.
+ *
+ * Return the struct inode of the index inode on success. Check the return
+ * value with IS_ERR() and if true, the function failed and the error code is
+ * obtained from PTR_ERR().
+ */
+struct inode *ntfs_index_iget(struct inode *base_vi, __le16 *name,
+		u32 name_len)
+{
+	struct inode *vi;
+	int err;
+	struct ntfs_attr na;
+
+	na.mft_no = base_vi->i_ino;
+	na.type = AT_INDEX_ALLOCATION;
+	na.name = name;
+	na.name_len = name_len;
+
+	vi = iget5_locked(base_vi->i_sb, na.mft_no, ntfs_test_inode,
+			ntfs_init_locked_inode, &na);
+	if (unlikely(!vi))
+		return ERR_PTR(-ENOMEM);
+
+	err = 0;
+
+	/* If this is a freshly allocated inode, need to read it now. */
+	if (inode_state_read_once(vi) & I_NEW) {
+		err = ntfs_read_locked_index_inode(base_vi, vi);
+		unlock_new_inode(vi);
+	}
+	/*
+	 * There is no point in keeping bad index inodes around.  This also
+	 * simplifies things in that we never need to check for bad index
+	 * inodes elsewhere.
+	 */
+	if (unlikely(err)) {
+		iput(vi);
+		vi = ERR_PTR(err);
+	}
+	return vi;
+}
+
+struct inode *ntfs_alloc_big_inode(struct super_block *sb)
+{
+	struct ntfs_inode *ni;
+
+	ntfs_debug("Entering.");
+	ni = alloc_inode_sb(sb, ntfs_big_inode_cache, GFP_NOFS);
+	if (likely(ni != NULL)) {
+		ni->state = 0;
+		ni->type = 0;
+		ni->mft_no = 0;
+		return VFS_I(ni);
+	}
+	ntfs_error(sb, "Allocation of NTFS big inode structure failed.");
+	return NULL;
+}
+
+void ntfs_free_big_inode(struct inode *inode)
+{
+	kmem_cache_free(ntfs_big_inode_cache, NTFS_I(inode));
+}
+
+static int ntfs_non_resident_dealloc_clusters(struct ntfs_inode *ni)
+{
+	struct super_block *sb = ni->vol->sb;
+	struct ntfs_attr_search_ctx *actx;
+	int err = 0;
+
+	actx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!actx)
+		return -ENOMEM;
+	WARN_ON(actx->mrec->link_count != 0);
+
+	/**
+	 * ntfs_truncate_vfs cannot be called in evict() context due
+	 * to some limitations, which are the @ni vfs inode is marked
+	 * with I_FREEING, and etc.
+	 */
+	if (NInoRunlistDirty(ni)) {
+		err = ntfs_cluster_free_from_rl(ni->vol, ni->runlist.rl);
+		if (err)
+			ntfs_error(sb,
+					"Failed to free clusters. Leaving inconsistent metadata.\n");
+	}
+
+	while ((err = ntfs_attrs_walk(actx)) == 0) {
+		if (actx->attr->non_resident &&
+				(!NInoRunlistDirty(ni) || actx->attr->type != AT_DATA)) {
+			struct runlist_element *rl;
+			size_t new_rl_count;
+
+			rl = ntfs_mapping_pairs_decompress(ni->vol, actx->attr, NULL,
+					&new_rl_count);
+			if (IS_ERR(rl)) {
+				err = PTR_ERR(rl);
+				ntfs_error(sb,
+					   "Failed to decompress runlist. Leaving inconsistent metadata.\n");
+				continue;
+			}
+
+			err = ntfs_cluster_free_from_rl(ni->vol, rl);
+			if (err)
+				ntfs_error(sb,
+					   "Failed to free attribute clusters. Leaving inconsistent metadata.\n");
+			kvfree(rl);
+		}
+	}
+
+	ntfs_release_dirty_clusters(ni->vol, ni->i_dealloc_clusters);
+	ntfs_attr_put_search_ctx(actx);
+	return err;
+}
+
+int ntfs_drop_big_inode(struct inode *inode)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+
+	if (!inode_unhashed(inode) && inode_state_read_once(inode) & I_SYNC) {
+		if (ni->type == AT_DATA || ni->type == AT_INDEX_ALLOCATION) {
+			if (!inode->i_nlink) {
+				struct ntfs_inode *ni = NTFS_I(inode);
+
+				if (ni->data_size == 0)
+					return 0;
+
+				/* To avoid evict_inode call simultaneously */
+				atomic_inc(&inode->i_count);
+				spin_unlock(&inode->i_lock);
+
+				truncate_setsize(VFS_I(ni), 0);
+				ntfs_truncate_vfs(VFS_I(ni), 0, 1);
+
+				sb_start_intwrite(inode->i_sb);
+				i_size_write(inode, 0);
+				ni->allocated_size = ni->initialized_size = ni->data_size = 0;
+
+				truncate_inode_pages_final(inode->i_mapping);
+				sb_end_intwrite(inode->i_sb);
+
+				spin_lock(&inode->i_lock);
+				atomic_dec(&inode->i_count);
+			}
+		}
+		return 0;
+	}
+
+	return inode_generic_drop(inode);
+}
+
+static inline struct ntfs_inode *ntfs_alloc_extent_inode(void)
+{
+	struct ntfs_inode *ni;
+
+	ntfs_debug("Entering.");
+	ni = kmem_cache_alloc(ntfs_inode_cache, GFP_NOFS);
+	if (likely(ni != NULL)) {
+		ni->state = 0;
+		return ni;
+	}
+	ntfs_error(NULL, "Allocation of NTFS inode structure failed.");
+	return NULL;
+}
+
+static void ntfs_destroy_extent_inode(struct ntfs_inode *ni)
+{
+	ntfs_debug("Entering.");
+
+	if (!atomic_dec_and_test(&ni->count))
+		WARN_ON(1);
+	if (ni->folio)
+		folio_put(ni->folio);
+	kfree(ni->mrec);
+	kmem_cache_free(ntfs_inode_cache, ni);
+}
+
+static struct lock_class_key attr_inode_mrec_lock_class;
+static struct lock_class_key attr_list_inode_mrec_lock_class;
+
+/*
+ * The attribute runlist lock has separate locking rules from the
+ * normal runlist lock, so split the two lock-classes:
+ */
+static struct lock_class_key attr_list_rl_lock_class;
+
+/*
+ * __ntfs_init_inode - initialize ntfs specific part of an inode
+ * @sb:		super block of mounted volume
+ * @ni:		freshly allocated ntfs inode which to initialize
+ *
+ * Initialize an ntfs inode to defaults.
+ *
+ * NOTE: ni->mft_no, ni->state, ni->type, ni->name, and ni->name_len are left
+ * untouched. Make sure to initialize them elsewhere.
+ */
+void __ntfs_init_inode(struct super_block *sb, struct ntfs_inode *ni)
+{
+	ntfs_debug("Entering.");
+	rwlock_init(&ni->size_lock);
+	ni->initialized_size = ni->allocated_size = 0;
+	ni->seq_no = 0;
+	atomic_set(&ni->count, 1);
+	ni->vol = NTFS_SB(sb);
+	ntfs_init_runlist(&ni->runlist);
+	mutex_init(&ni->mrec_lock);
+	if (ni->type == AT_ATTRIBUTE_LIST) {
+		lockdep_set_class(&ni->mrec_lock,
+				  &attr_list_inode_mrec_lock_class);
+		lockdep_set_class(&ni->runlist.lock,
+				  &attr_list_rl_lock_class);
+	} else if (NInoAttr(ni)) {
+		lockdep_set_class(&ni->mrec_lock,
+				  &attr_inode_mrec_lock_class);
+	}
+
+	ni->folio = NULL;
+	ni->folio_ofs = 0;
+	ni->mrec = NULL;
+	ni->attr_list_size = 0;
+	ni->attr_list = NULL;
+	ni->itype.index.block_size = 0;
+	ni->itype.index.vcn_size = 0;
+	ni->itype.index.collation_rule = 0;
+	ni->itype.index.block_size_bits = 0;
+	ni->itype.index.vcn_size_bits = 0;
+	mutex_init(&ni->extent_lock);
+	ni->nr_extents = 0;
+	ni->ext.base_ntfs_ino = NULL;
+	ni->flags = 0;
+	ni->mft_lcn[0] = LCN_RL_NOT_MAPPED;
+	ni->mft_lcn_count = 0;
+	ni->target = NULL;
+	ni->i_dealloc_clusters = 0;
+}
+
+/*
+ * Extent inodes get MFT-mapped in a nested way, while the base inode
+ * is still mapped. Teach this nesting to the lock validator by creating
+ * a separate class for nested inode's mrec_lock's:
+ */
+static struct lock_class_key extent_inode_mrec_lock_key;
+
+inline struct ntfs_inode *ntfs_new_extent_inode(struct super_block *sb,
+		u64 mft_no)
+{
+	struct ntfs_inode *ni = ntfs_alloc_extent_inode();
+
+	ntfs_debug("Entering.");
+	if (likely(ni != NULL)) {
+		__ntfs_init_inode(sb, ni);
+		lockdep_set_class(&ni->mrec_lock, &extent_inode_mrec_lock_key);
+		ni->mft_no = mft_no;
+		ni->type = AT_UNUSED;
+		ni->name = NULL;
+		ni->name_len = 0;
+	}
+	return ni;
+}
+
+/*
+ * ntfs_is_extended_system_file - check if a file is in the $Extend directory
+ * @ctx:	initialized attribute search context
+ *
+ * Search all file name attributes in the inode described by the attribute
+ * search context @ctx and check if any of the names are in the $Extend system
+ * directory.
+ *
+ * Return values:
+ *	   3: file is $ObjId in $Extend directory
+ *	   2: file is $Reparse in $Extend directory
+ *	   1: file is in $Extend directory
+ *	   0: file is not in $Extend directory
+ *    -errno: failed to determine if the file is in the $Extend directory
+ */
+static int ntfs_is_extended_system_file(struct ntfs_attr_search_ctx *ctx)
+{
+	int nr_links, err;
+
+	/* Restart search. */
+	ntfs_attr_reinit_search_ctx(ctx);
+
+	/* Get number of hard links. */
+	nr_links = le16_to_cpu(ctx->mrec->link_count);
+
+	/* Loop through all hard links. */
+	while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0,
+			ctx))) {
+		struct file_name_attr *file_name_attr;
+		struct attr_record *attr = ctx->attr;
+		u8 *p, *p2;
+
+		nr_links--;
+		/*
+		 * Maximum sanity checking as we are called on an inode that
+		 * we suspect might be corrupt.
+		 */
+		p = (u8 *)attr + le32_to_cpu(attr->length);
+		if (p < (u8 *)ctx->mrec || (u8 *)p > (u8 *)ctx->mrec +
+				le32_to_cpu(ctx->mrec->bytes_in_use)) {
+err_corrupt_attr:
+			ntfs_error(ctx->ntfs_ino->vol->sb,
+					"Corrupt file name attribute. You should run chkdsk.");
+			return -EIO;
+		}
+		if (attr->non_resident) {
+			ntfs_error(ctx->ntfs_ino->vol->sb,
+					"Non-resident file name. You should run chkdsk.");
+			return -EIO;
+		}
+		if (attr->flags) {
+			ntfs_error(ctx->ntfs_ino->vol->sb,
+					"File name with invalid flags. You should run chkdsk.");
+			return -EIO;
+		}
+		if (!(attr->data.resident.flags & RESIDENT_ATTR_IS_INDEXED)) {
+			ntfs_error(ctx->ntfs_ino->vol->sb,
+					"Unindexed file name. You should run chkdsk.");
+			return -EIO;
+		}
+		file_name_attr = (struct file_name_attr *)((u8 *)attr +
+				le16_to_cpu(attr->data.resident.value_offset));
+		p2 = (u8 *)file_name_attr + le32_to_cpu(attr->data.resident.value_length);
+		if (p2 < (u8 *)attr || p2 > p)
+			goto err_corrupt_attr;
+		/* This attribute is ok, but is it in the $Extend directory? */
+		if (MREF_LE(file_name_attr->parent_directory) == FILE_Extend) {
+			unsigned char *s;
+
+			s = ntfs_attr_name_get(ctx->ntfs_ino->vol,
+					file_name_attr->file_name,
+					file_name_attr->file_name_length);
+			if (!s)
+				return 1;
+			if (!strcmp("$Reparse", s)) {
+				ntfs_attr_name_free(&s);
+				return 2; /* it's reparse point file */
+			}
+			if (!strcmp("$ObjId", s)) {
+				ntfs_attr_name_free(&s);
+				return 3; /* it's object id file */
+			}
+			ntfs_attr_name_free(&s);
+			return 1;	/* YES, it's an extended system file. */
+		}
+	}
+	if (unlikely(err != -ENOENT))
+		return err;
+	if (unlikely(nr_links)) {
+		ntfs_error(ctx->ntfs_ino->vol->sb,
+			"Inode hard link count doesn't match number of name attributes. You should run chkdsk.");
+		return -EIO;
+	}
+	return 0;	/* NO, it is not an extended system file. */
+}
+
+static struct lock_class_key ntfs_dir_inval_lock_key;
+
+void ntfs_set_vfs_operations(struct inode *inode, mode_t mode, dev_t dev)
+{
+	if (S_ISDIR(mode)) {
+		if (!NInoAttr(NTFS_I(inode))) {
+			inode->i_op = &ntfs_dir_inode_ops;
+			inode->i_fop = &ntfs_dir_ops;
+		}
+		inode->i_mapping->a_ops = &ntfs_aops;
+		lockdep_set_class(&inode->i_mapping->invalidate_lock,
+				  &ntfs_dir_inval_lock_key);
+	} else if (S_ISLNK(mode)) {
+		inode->i_op = &ntfs_symlink_inode_operations;
+		inode->i_mapping->a_ops = &ntfs_aops;
+	} else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) {
+		inode->i_op = &ntfs_special_inode_operations;
+		init_special_inode(inode, inode->i_mode, dev);
+	} else {
+		if (!NInoAttr(NTFS_I(inode))) {
+			inode->i_op = &ntfs_file_inode_ops;
+			inode->i_fop = &ntfs_file_ops;
+		}
+		if (inode->i_ino == FILE_MFT)
+			inode->i_mapping->a_ops = &ntfs_mft_aops;
+		else
+			inode->i_mapping->a_ops = &ntfs_aops;
+	}
+}
+
+/*
+ * ntfs_read_locked_inode - read an inode from its device
+ * @vi:		inode to read
+ *
+ * ntfs_read_locked_inode() is called from ntfs_iget() to read the inode
+ * described by @vi into memory from the device.
+ *
+ * The only fields in @vi that we need to/can look at when the function is
+ * called are i_sb, pointing to the mounted device's super block, and i_ino,
+ * the number of the inode to load.
+ *
+ * ntfs_read_locked_inode() maps, pins and locks the mft record number i_ino
+ * for reading and sets up the necessary @vi fields as well as initializing
+ * the ntfs inode.
+ *
+ * Q: What locks are held when the function is called?
+ * A: i_state has I_NEW set, hence the inode is locked, also
+ *    i_count is set to 1, so it is not going to go away
+ *    i_flags is set to 0 and we have no business touching it.  Only an ioctl()
+ *    is allowed to write to them. We should of course be honouring them but
+ *    we need to do that using the IS_* macros defined in include/linux/fs.h.
+ *    In any case ntfs_read_locked_inode() has nothing to do with i_flags.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_read_locked_inode(struct inode *vi)
+{
+	struct ntfs_volume *vol = NTFS_SB(vi->i_sb);
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct mft_record *m;
+	struct attr_record *a;
+	struct standard_information *si;
+	struct ntfs_attr_search_ctx *ctx;
+	int err = 0;
+	__le16 *name = I30;
+	unsigned int name_len = 4, flags = 0;
+	int extend_sys = 0;
+	dev_t dev = 0;
+	bool vol_err = true;
+
+	ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no);
+
+	if (uid_valid(vol->uid)) {
+		vi->i_uid = vol->uid;
+		flags |= NTFS_VOL_UID;
+	} else
+		vi->i_uid = GLOBAL_ROOT_UID;
+
+	if (gid_valid(vol->gid)) {
+		vi->i_gid = vol->gid;
+		flags |= NTFS_VOL_GID;
+	} else
+		vi->i_gid = GLOBAL_ROOT_GID;
+
+	vi->i_mode = 0777;
+
+	/*
+	 * Initialize the ntfs specific part of @vi special casing
+	 * FILE_MFT which we need to do at mount time.
+	 */
+	if (vi->i_ino != FILE_MFT)
+		ntfs_init_big_inode(vi);
+
+	m = map_mft_record(ni);
+	if (IS_ERR(m)) {
+		err = PTR_ERR(m);
+		goto err_out;
+	}
+
+	ctx = ntfs_attr_get_search_ctx(ni, m);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto unm_err_out;
+	}
+
+	if (!(m->flags & MFT_RECORD_IN_USE)) {
+		err = -ENOENT;
+		vol_err = false;
+		goto unm_err_out;
+	}
+
+	if (m->base_mft_record) {
+		ntfs_error(vi->i_sb, "Inode is an extent inode!");
+		goto unm_err_out;
+	}
+
+	/* Transfer information from mft record into vfs and ntfs inodes. */
+	vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number);
+
+	if (le16_to_cpu(m->link_count) < 1) {
+		ntfs_error(vi->i_sb, "Inode link count is 0!");
+		goto unm_err_out;
+	}
+	set_nlink(vi, le16_to_cpu(m->link_count));
+
+	/* If read-only, no one gets write permissions. */
+	if (IS_RDONLY(vi))
+		vi->i_mode &= ~0222;
+
+	/*
+	 * Find the standard information attribute in the mft record. At this
+	 * stage we haven't setup the attribute list stuff yet, so this could
+	 * in fact fail if the standard information is in an extent record, but
+	 * I don't think this actually ever happens.
+	 */
+	ntfs_attr_reinit_search_ctx(ctx);
+	err = ntfs_attr_lookup(AT_STANDARD_INFORMATION, NULL, 0, 0, 0, NULL, 0,
+			ctx);
+	if (unlikely(err)) {
+		if (err == -ENOENT)
+			ntfs_error(vi->i_sb, "$STANDARD_INFORMATION attribute is missing.");
+		goto unm_err_out;
+	}
+	a = ctx->attr;
+	/* Get the standard information attribute value. */
+	if ((u8 *)a + le16_to_cpu(a->data.resident.value_offset)
+			+ le32_to_cpu(a->data.resident.value_length) >
+			(u8 *)ctx->mrec + vol->mft_record_size) {
+		ntfs_error(vi->i_sb, "Corrupt standard information attribute in inode.");
+		goto unm_err_out;
+	}
+	si = (struct standard_information *)((u8 *)a +
+			le16_to_cpu(a->data.resident.value_offset));
+
+	/* Transfer information from the standard information into vi. */
+	/*
+	 * Note: The i_?times do not quite map perfectly onto the NTFS times,
+	 * but they are close enough, and in the end it doesn't really matter
+	 * that much...
+	 */
+	/*
+	 * mtime is the last change of the data within the file. Not changed
+	 * when only metadata is changed, e.g. a rename doesn't affect mtime.
+	 */
+	ni->i_crtime = ntfs2utc(si->creation_time);
+
+	inode_set_mtime_to_ts(vi, ntfs2utc(si->last_data_change_time));
+	/*
+	 * ctime is the last change of the metadata of the file. This obviously
+	 * always changes, when mtime is changed. ctime can be changed on its
+	 * own, mtime is then not changed, e.g. when a file is renamed.
+	 */
+	inode_set_ctime_to_ts(vi, ntfs2utc(si->last_mft_change_time));
+	/*
+	 * Last access to the data within the file. Not changed during a rename
+	 * for example but changed whenever the file is written to.
+	 */
+	inode_set_atime_to_ts(vi, ntfs2utc(si->last_access_time));
+	ni->flags = si->file_attributes;
+
+	/* Find the attribute list attribute if present. */
+	ntfs_attr_reinit_search_ctx(ctx);
+	err = ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, 0, 0, NULL, 0, ctx);
+	if (err) {
+		if (unlikely(err != -ENOENT)) {
+			ntfs_error(vi->i_sb, "Failed to lookup attribute list attribute.");
+			goto unm_err_out;
+		}
+	} else {
+		if (vi->i_ino == FILE_MFT)
+			goto skip_attr_list_load;
+		ntfs_debug("Attribute list found in inode 0x%llx.", ni->mft_no);
+		NInoSetAttrList(ni);
+		a = ctx->attr;
+		if (a->flags & ATTR_COMPRESSION_MASK) {
+			ntfs_error(vi->i_sb,
+				"Attribute list attribute is compressed.");
+			goto unm_err_out;
+		}
+		if (a->flags & ATTR_IS_ENCRYPTED ||
+				a->flags & ATTR_IS_SPARSE) {
+			if (a->non_resident) {
+				ntfs_error(vi->i_sb,
+					"Non-resident attribute list attribute is encrypted/sparse.");
+				goto unm_err_out;
+			}
+			ntfs_warning(vi->i_sb,
+				"Resident attribute list attribute in inode 0x%llx is marked encrypted/sparse which is not true.  However, Windows allows this and chkdsk does not detect or correct it so we will just ignore the invalid flags and pretend they are not set.",
+				ni->mft_no);
+		}
+		/* Now allocate memory for the attribute list. */
+		ni->attr_list_size = (u32)ntfs_attr_size(a);
+		if (!ni->attr_list_size) {
+			ntfs_error(vi->i_sb, "Attr_list_size is zero");
+			goto unm_err_out;
+		}
+		ni->attr_list = kvzalloc(ni->attr_list_size, GFP_NOFS);
+		if (!ni->attr_list) {
+			ntfs_error(vi->i_sb,
+				"Not enough memory to allocate buffer for attribute list.");
+			err = -ENOMEM;
+			goto unm_err_out;
+		}
+		if (a->non_resident) {
+			NInoSetAttrListNonResident(ni);
+			if (a->data.non_resident.lowest_vcn) {
+				ntfs_error(vi->i_sb, "Attribute list has non zero lowest_vcn.");
+				goto unm_err_out;
+			}
+
+			/* Now load the attribute list. */
+			err = load_attribute_list(ni, ni->attr_list, ni->attr_list_size);
+			if (err) {
+				ntfs_error(vi->i_sb, "Failed to load attribute list attribute.");
+				goto unm_err_out;
+			}
+		} else /* if (!a->non_resident) */ {
+			if ((u8 *)a + le16_to_cpu(a->data.resident.value_offset)
+					+ le32_to_cpu(
+					a->data.resident.value_length) >
+					(u8 *)ctx->mrec + vol->mft_record_size) {
+				ntfs_error(vi->i_sb, "Corrupt attribute list in inode.");
+				goto unm_err_out;
+			}
+			/* Now copy the attribute list. */
+			memcpy(ni->attr_list, (u8 *)a + le16_to_cpu(
+					a->data.resident.value_offset),
+					le32_to_cpu(
+					a->data.resident.value_length));
+		}
+	}
+skip_attr_list_load:
+	err = ntfs_attr_lookup(AT_EA_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx);
+	if (!err) {
+		NInoSetHasEA(ni);
+		ntfs_ea_get_wsl_inode(vi, &dev, flags);
+	}
+
+	if (m->flags & MFT_RECORD_IS_DIRECTORY) {
+		vi->i_mode |= S_IFDIR;
+		/*
+		 * Apply the directory permissions mask set in the mount
+		 * options.
+		 */
+		vi->i_mode &= ~vol->dmask;
+		/* Things break without this kludge! */
+		if (vi->i_nlink > 1)
+			set_nlink(vi, 1);
+	} else {
+		if (ni->flags & FILE_ATTR_REPARSE_POINT) {
+			unsigned int mode;
+
+			mode = ntfs_make_symlink(ni);
+			if (mode)
+				vi->i_mode |= mode;
+			else {
+				vi->i_mode &= ~S_IFLNK;
+				vi->i_mode |= S_IFREG;
+			}
+		} else
+			vi->i_mode |= S_IFREG;
+		/* Apply the file permissions mask set in the mount options. */
+		vi->i_mode &= ~vol->fmask;
+	}
+
+	/*
+	 * If an attribute list is present we now have the attribute list value
+	 * in ntfs_ino->attr_list and it is ntfs_ino->attr_list_size bytes.
+	 */
+	if (S_ISDIR(vi->i_mode)) {
+		struct index_root *ir;
+		u8 *ir_end, *index_end;
+
+view_index_meta:
+		/* It is a directory, find index root attribute. */
+		ntfs_attr_reinit_search_ctx(ctx);
+		err = ntfs_attr_lookup(AT_INDEX_ROOT, name, name_len, CASE_SENSITIVE,
+				0, NULL, 0, ctx);
+		if (unlikely(err)) {
+			if (err == -ENOENT)
+				ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is missing.");
+			goto unm_err_out;
+		}
+		a = ctx->attr;
+		/* Set up the state. */
+		if (unlikely(a->non_resident)) {
+			ntfs_error(vol->sb,
+				"$INDEX_ROOT attribute is not resident.");
+			goto unm_err_out;
+		}
+		/* Ensure the attribute name is placed before the value. */
+		if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >=
+				le16_to_cpu(a->data.resident.value_offset)))) {
+			ntfs_error(vol->sb,
+				"$INDEX_ROOT attribute name is placed after the attribute value.");
+			goto unm_err_out;
+		}
+		/*
+		 * Compressed/encrypted index root just means that the newly
+		 * created files in that directory should be created compressed/
+		 * encrypted. However index root cannot be both compressed and
+		 * encrypted.
+		 */
+		if (a->flags & ATTR_COMPRESSION_MASK) {
+			NInoSetCompressed(ni);
+			ni->flags |= FILE_ATTR_COMPRESSED;
+		}
+		if (a->flags & ATTR_IS_ENCRYPTED) {
+			if (a->flags & ATTR_COMPRESSION_MASK) {
+				ntfs_error(vi->i_sb, "Found encrypted and compressed attribute.");
+				goto unm_err_out;
+			}
+			NInoSetEncrypted(ni);
+			ni->flags |= FILE_ATTR_ENCRYPTED;
+		}
+		if (a->flags & ATTR_IS_SPARSE) {
+			NInoSetSparse(ni);
+			ni->flags |= FILE_ATTR_SPARSE_FILE;
+		}
+		ir = (struct index_root *)((u8 *)a +
+				le16_to_cpu(a->data.resident.value_offset));
+		ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length);
+		if (ir_end > (u8 *)ctx->mrec + vol->mft_record_size) {
+			ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is corrupt.");
+			goto unm_err_out;
+		}
+		index_end = (u8 *)&ir->index +
+				le32_to_cpu(ir->index.index_length);
+		if (index_end > ir_end) {
+			ntfs_error(vi->i_sb, "Directory index is corrupt.");
+			goto unm_err_out;
+		}
+
+		if (extend_sys) {
+			if (ir->type) {
+				ntfs_error(vi->i_sb, "Indexed attribute is not zero.");
+				goto unm_err_out;
+			}
+		} else {
+			if (ir->type != AT_FILE_NAME) {
+				ntfs_error(vi->i_sb, "Indexed attribute is not $FILE_NAME.");
+				goto unm_err_out;
+			}
+
+			if (ir->collation_rule != COLLATION_FILE_NAME) {
+				ntfs_error(vi->i_sb,
+					"Index collation rule is not COLLATION_FILE_NAME.");
+				goto unm_err_out;
+			}
+		}
+
+		ni->itype.index.collation_rule = ir->collation_rule;
+		ni->itype.index.block_size = le32_to_cpu(ir->index_block_size);
+		if (ni->itype.index.block_size &
+				(ni->itype.index.block_size - 1)) {
+			ntfs_error(vi->i_sb, "Index block size (%u) is not a power of two.",
+					ni->itype.index.block_size);
+			goto unm_err_out;
+		}
+		if (ni->itype.index.block_size > PAGE_SIZE) {
+			ntfs_error(vi->i_sb,
+				"Index block size (%u) > PAGE_SIZE (%ld) is not supported.",
+				ni->itype.index.block_size,
+				PAGE_SIZE);
+			err = -EOPNOTSUPP;
+			goto unm_err_out;
+		}
+		if (ni->itype.index.block_size < NTFS_BLOCK_SIZE) {
+			ntfs_error(vi->i_sb,
+				"Index block size (%u) < NTFS_BLOCK_SIZE (%i) is not supported.",
+				ni->itype.index.block_size,
+				NTFS_BLOCK_SIZE);
+			err = -EOPNOTSUPP;
+			goto unm_err_out;
+		}
+		ni->itype.index.block_size_bits =
+				ffs(ni->itype.index.block_size) - 1;
+		/* Determine the size of a vcn in the directory index. */
+		if (vol->cluster_size <= ni->itype.index.block_size) {
+			ni->itype.index.vcn_size = vol->cluster_size;
+			ni->itype.index.vcn_size_bits = vol->cluster_size_bits;
+		} else {
+			ni->itype.index.vcn_size = vol->sector_size;
+			ni->itype.index.vcn_size_bits = vol->sector_size_bits;
+		}
+
+		/* Setup the index allocation attribute, even if not present. */
+		ni->type = AT_INDEX_ROOT;
+		ni->name = name;
+		ni->name_len = name_len;
+		vi->i_size = ni->initialized_size = ni->data_size =
+			le32_to_cpu(a->data.resident.value_length);
+		ni->allocated_size = (ni->data_size + 7) & ~7;
+		/* We are done with the mft record, so we release it. */
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(ni);
+		m = NULL;
+		ctx = NULL;
+		/* Setup the operations for this inode. */
+		ntfs_set_vfs_operations(vi, S_IFDIR, 0);
+		if (ir->index.flags & LARGE_INDEX)
+			NInoSetIndexAllocPresent(ni);
+	} else {
+		/* It is a file. */
+		ntfs_attr_reinit_search_ctx(ctx);
+
+		/* Setup the data attribute, even if not present. */
+		ni->type = AT_DATA;
+		ni->name = AT_UNNAMED;
+		ni->name_len = 0;
+
+		/* Find first extent of the unnamed data attribute. */
+		err = ntfs_attr_lookup(AT_DATA, NULL, 0, 0, 0, NULL, 0, ctx);
+		if (unlikely(err)) {
+			vi->i_size = ni->initialized_size =
+					ni->allocated_size = 0;
+			if (err != -ENOENT) {
+				ntfs_error(vi->i_sb, "Failed to lookup $DATA attribute.");
+				goto unm_err_out;
+			}
+			/*
+			 * FILE_Secure does not have an unnamed $DATA
+			 * attribute, so we special case it here.
+			 */
+			if (vi->i_ino == FILE_Secure)
+				goto no_data_attr_special_case;
+			/*
+			 * Most if not all the system files in the $Extend
+			 * system directory do not have unnamed data
+			 * attributes so we need to check if the parent
+			 * directory of the file is FILE_Extend and if it is
+			 * ignore this error. To do this we need to get the
+			 * name of this inode from the mft record as the name
+			 * contains the back reference to the parent directory.
+			 */
+			extend_sys = ntfs_is_extended_system_file(ctx);
+			if (extend_sys > 0) {
+				if (m->flags & MFT_RECORD_IS_VIEW_INDEX) {
+					if (extend_sys == 2) {
+						name = reparse_index_name;
+						name_len = 2;
+						goto view_index_meta;
+					} else if (extend_sys == 3) {
+						name = objid_index_name;
+						name_len = 2;
+						goto view_index_meta;
+					}
+				}
+				goto no_data_attr_special_case;
+			}
+
+			err = extend_sys;
+			ntfs_error(vi->i_sb, "$DATA attribute is missing, err : %d", err);
+			goto unm_err_out;
+		}
+		a = ctx->attr;
+		/* Setup the state. */
+		if (a->flags & (ATTR_COMPRESSION_MASK | ATTR_IS_SPARSE)) {
+			if (a->flags & ATTR_COMPRESSION_MASK) {
+				NInoSetCompressed(ni);
+				ni->flags |= FILE_ATTR_COMPRESSED;
+				if (vol->cluster_size > 4096) {
+					ntfs_error(vi->i_sb,
+						"Found compressed data but compression is disabled due to cluster size (%i) > 4kiB.",
+						vol->cluster_size);
+					goto unm_err_out;
+				}
+				if ((a->flags & ATTR_COMPRESSION_MASK)
+						!= ATTR_IS_COMPRESSED) {
+					ntfs_error(vi->i_sb,
+						"Found unknown compression method or corrupt file.");
+					goto unm_err_out;
+				}
+			}
+			if (a->flags & ATTR_IS_SPARSE) {
+				NInoSetSparse(ni);
+				ni->flags |= FILE_ATTR_SPARSE_FILE;
+			}
+		}
+		if (a->flags & ATTR_IS_ENCRYPTED) {
+			if (NInoCompressed(ni)) {
+				ntfs_error(vi->i_sb, "Found encrypted and compressed data.");
+				goto unm_err_out;
+			}
+			NInoSetEncrypted(ni);
+			ni->flags |= FILE_ATTR_ENCRYPTED;
+		}
+		if (a->non_resident) {
+			NInoSetNonResident(ni);
+			if (NInoCompressed(ni) || NInoSparse(ni)) {
+				if (NInoCompressed(ni) &&
+				    a->data.non_resident.compression_unit != 4) {
+					ntfs_error(vi->i_sb,
+						"Found non-standard compression unit (%u instead of 4).  Cannot handle this.",
+						a->data.non_resident.compression_unit);
+					err = -EOPNOTSUPP;
+					goto unm_err_out;
+				}
+
+				if (NInoSparse(ni) &&
+				    a->data.non_resident.compression_unit &&
+				    a->data.non_resident.compression_unit !=
+				     vol->sparse_compression_unit) {
+					ntfs_error(vi->i_sb,
+						   "Found non-standard compression unit (%u instead of 0 or %d).  Cannot handle this.",
+						   a->data.non_resident.compression_unit,
+						   vol->sparse_compression_unit);
+					err = -EOPNOTSUPP;
+					goto unm_err_out;
+				}
+
+
+				if (a->data.non_resident.compression_unit) {
+					ni->itype.compressed.block_size = 1U <<
+							(a->data.non_resident.compression_unit +
+							vol->cluster_size_bits);
+					ni->itype.compressed.block_size_bits =
+							ffs(ni->itype.compressed.block_size) - 1;
+					ni->itype.compressed.block_clusters =
+							1U << a->data.non_resident.compression_unit;
+				} else {
+					ni->itype.compressed.block_size = 0;
+					ni->itype.compressed.block_size_bits =
+							0;
+					ni->itype.compressed.block_clusters =
+							0;
+				}
+				ni->itype.compressed.size = le64_to_cpu(
+						a->data.non_resident.compressed_size);
+			}
+			if (a->data.non_resident.lowest_vcn) {
+				ntfs_error(vi->i_sb,
+					"First extent of $DATA attribute has non zero lowest_vcn.");
+				goto unm_err_out;
+			}
+			vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size);
+			ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size);
+			ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size);
+		} else { /* Resident attribute. */
+			vi->i_size = ni->data_size = ni->initialized_size = le32_to_cpu(
+					a->data.resident.value_length);
+			ni->allocated_size = le32_to_cpu(a->length) -
+					le16_to_cpu(
+					a->data.resident.value_offset);
+			if (vi->i_size > ni->allocated_size) {
+				ntfs_error(vi->i_sb,
+					"Resident data attribute is corrupt (size exceeds allocation).");
+				goto unm_err_out;
+			}
+		}
+no_data_attr_special_case:
+		/* We are done with the mft record, so we release it. */
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(ni);
+		m = NULL;
+		ctx = NULL;
+		/* Setup the operations for this inode. */
+		ntfs_set_vfs_operations(vi, vi->i_mode, dev);
+	}
+
+	if (NVolSysImmutable(vol) && (ni->flags & FILE_ATTR_SYSTEM) &&
+	    !S_ISFIFO(vi->i_mode) && !S_ISSOCK(vi->i_mode) && !S_ISLNK(vi->i_mode))
+		vi->i_flags |= S_IMMUTABLE;
+
+	/*
+	 * The number of 512-byte blocks used on disk (for stat). This is in so
+	 * far inaccurate as it doesn't account for any named streams or other
+	 * special non-resident attributes, but that is how Windows works, too,
+	 * so we are at least consistent with Windows, if not entirely
+	 * consistent with the Linux Way. Doing it the Linux Way would cause a
+	 * significant slowdown as it would involve iterating over all
+	 * attributes in the mft record and adding the allocated/compressed
+	 * sizes of all non-resident attributes present to give us the Linux
+	 * correct size that should go into i_blocks (after division by 512).
+	 */
+	if (S_ISREG(vi->i_mode) && (NInoCompressed(ni) || NInoSparse(ni)))
+		vi->i_blocks = ni->itype.compressed.size >> 9;
+	else
+		vi->i_blocks = ni->allocated_size >> 9;
+
+	ntfs_debug("Done.");
+	return 0;
+unm_err_out:
+	if (!err)
+		err = -EIO;
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	if (m)
+		unmap_mft_record(ni);
+err_out:
+	if (err != -EOPNOTSUPP && err != -ENOMEM && vol_err == true) {
+		ntfs_error(vol->sb,
+			"Failed with error code %i.  Marking corrupt inode 0x%llx as bad.  Run chkdsk.",
+			err, ni->mft_no);
+		NVolSetErrors(vol);
+	}
+	return err;
+}
+
+/*
+ * ntfs_read_locked_attr_inode - read an attribute inode from its base inode
+ * @base_vi:	base inode
+ * @vi:		attribute inode to read
+ *
+ * ntfs_read_locked_attr_inode() is called from ntfs_attr_iget() to read the
+ * attribute inode described by @vi into memory from the base mft record
+ * described by @base_ni.
+ *
+ * ntfs_read_locked_attr_inode() maps, pins and locks the base inode for
+ * reading and looks up the attribute described by @vi before setting up the
+ * necessary fields in @vi as well as initializing the ntfs inode.
+ *
+ * Q: What locks are held when the function is called?
+ * A: i_state has I_NEW set, hence the inode is locked, also
+ *    i_count is set to 1, so it is not going to go away
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Note this cannot be called for AT_INDEX_ALLOCATION.
+ */
+static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi)
+{
+	struct ntfs_volume *vol = NTFS_SB(vi->i_sb);
+	struct ntfs_inode *ni = NTFS_I(vi), *base_ni = NTFS_I(base_vi);
+	struct mft_record *m;
+	struct attr_record *a;
+	struct ntfs_attr_search_ctx *ctx;
+	int err = 0;
+
+	ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no);
+
+	ntfs_init_big_inode(vi);
+
+	/* Just mirror the values from the base inode. */
+	vi->i_uid	= base_vi->i_uid;
+	vi->i_gid	= base_vi->i_gid;
+	set_nlink(vi, base_vi->i_nlink);
+	inode_set_mtime_to_ts(vi, inode_get_mtime(base_vi));
+	inode_set_ctime_to_ts(vi, inode_get_ctime(base_vi));
+	inode_set_atime_to_ts(vi, inode_get_atime(base_vi));
+	vi->i_generation = ni->seq_no = base_ni->seq_no;
+
+	/* Set inode type to zero but preserve permissions. */
+	vi->i_mode	= base_vi->i_mode & ~S_IFMT;
+
+	m = map_mft_record(base_ni);
+	if (IS_ERR(m)) {
+		err = PTR_ERR(m);
+		goto err_out;
+	}
+	ctx = ntfs_attr_get_search_ctx(base_ni, m);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto unm_err_out;
+	}
+	/* Find the attribute. */
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err))
+		goto unm_err_out;
+	a = ctx->attr;
+	if (a->flags & (ATTR_COMPRESSION_MASK | ATTR_IS_SPARSE)) {
+		if (a->flags & ATTR_COMPRESSION_MASK) {
+			NInoSetCompressed(ni);
+			ni->flags |= FILE_ATTR_COMPRESSED;
+			if ((ni->type != AT_DATA) || (ni->type == AT_DATA &&
+					ni->name_len)) {
+				ntfs_error(vi->i_sb,
+					   "Found compressed non-data or named data attribute.");
+				goto unm_err_out;
+			}
+			if (vol->cluster_size > 4096) {
+				ntfs_error(vi->i_sb,
+					"Found compressed attribute but compression is disabled due to cluster size (%i) > 4kiB.",
+					vol->cluster_size);
+				goto unm_err_out;
+			}
+			if ((a->flags & ATTR_COMPRESSION_MASK) !=
+					ATTR_IS_COMPRESSED) {
+				ntfs_error(vi->i_sb, "Found unknown compression method.");
+				goto unm_err_out;
+			}
+		}
+		/*
+		 * The compressed/sparse flag set in an index root just means
+		 * to compress all files.
+		 */
+		if (NInoMstProtected(ni) && ni->type != AT_INDEX_ROOT) {
+			ntfs_error(vi->i_sb,
+				"Found mst protected attribute but the attribute is %s.",
+				NInoCompressed(ni) ? "compressed" : "sparse");
+			goto unm_err_out;
+		}
+		if (a->flags & ATTR_IS_SPARSE) {
+			NInoSetSparse(ni);
+			ni->flags |= FILE_ATTR_SPARSE_FILE;
+		}
+	}
+	if (a->flags & ATTR_IS_ENCRYPTED) {
+		if (NInoCompressed(ni)) {
+			ntfs_error(vi->i_sb, "Found encrypted and compressed data.");
+			goto unm_err_out;
+		}
+		/*
+		 * The encryption flag set in an index root just means to
+		 * encrypt all files.
+		 */
+		if (NInoMstProtected(ni) && ni->type != AT_INDEX_ROOT) {
+			ntfs_error(vi->i_sb,
+				"Found mst protected attribute but the attribute is encrypted.");
+			goto unm_err_out;
+		}
+		if (ni->type != AT_DATA) {
+			ntfs_error(vi->i_sb,
+				"Found encrypted non-data attribute.");
+			goto unm_err_out;
+		}
+		NInoSetEncrypted(ni);
+		ni->flags |= FILE_ATTR_ENCRYPTED;
+	}
+	if (!a->non_resident) {
+		/* Ensure the attribute name is placed before the value. */
+		if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >=
+				le16_to_cpu(a->data.resident.value_offset)))) {
+			ntfs_error(vol->sb,
+				"Attribute name is placed after the attribute value.");
+			goto unm_err_out;
+		}
+		if (NInoMstProtected(ni)) {
+			ntfs_error(vi->i_sb,
+				"Found mst protected attribute but the attribute is resident.");
+			goto unm_err_out;
+		}
+		vi->i_size = ni->initialized_size = ni->data_size = le32_to_cpu(
+				a->data.resident.value_length);
+		ni->allocated_size = le32_to_cpu(a->length) -
+				le16_to_cpu(a->data.resident.value_offset);
+		if (vi->i_size > ni->allocated_size) {
+			ntfs_error(vi->i_sb,
+				"Resident attribute is corrupt (size exceeds allocation).");
+			goto unm_err_out;
+		}
+	} else {
+		NInoSetNonResident(ni);
+		/*
+		 * Ensure the attribute name is placed before the mapping pairs
+		 * array.
+		 */
+		if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >=
+				le16_to_cpu(
+				a->data.non_resident.mapping_pairs_offset)))) {
+			ntfs_error(vol->sb,
+				"Attribute name is placed after the mapping pairs array.");
+			goto unm_err_out;
+		}
+		if (NInoCompressed(ni) || NInoSparse(ni)) {
+			if (NInoCompressed(ni) && a->data.non_resident.compression_unit != 4) {
+				ntfs_error(vi->i_sb,
+					"Found non-standard compression unit (%u instead of 4).  Cannot handle this.",
+					a->data.non_resident.compression_unit);
+				err = -EOPNOTSUPP;
+				goto unm_err_out;
+			}
+			if (a->data.non_resident.compression_unit) {
+				ni->itype.compressed.block_size = 1U <<
+						(a->data.non_resident.compression_unit +
+						vol->cluster_size_bits);
+				ni->itype.compressed.block_size_bits =
+						ffs(ni->itype.compressed.block_size) - 1;
+				ni->itype.compressed.block_clusters = 1U <<
+						a->data.non_resident.compression_unit;
+			} else {
+				ni->itype.compressed.block_size = 0;
+				ni->itype.compressed.block_size_bits = 0;
+				ni->itype.compressed.block_clusters = 0;
+			}
+			ni->itype.compressed.size = le64_to_cpu(
+					a->data.non_resident.compressed_size);
+		}
+		if (a->data.non_resident.lowest_vcn) {
+			ntfs_error(vi->i_sb, "First extent of attribute has non-zero lowest_vcn.");
+			goto unm_err_out;
+		}
+		vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size);
+		ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size);
+		ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size);
+	}
+	vi->i_mapping->a_ops = &ntfs_aops;
+	if ((NInoCompressed(ni) || NInoSparse(ni)) && ni->type != AT_INDEX_ROOT)
+		vi->i_blocks = ni->itype.compressed.size >> 9;
+	else
+		vi->i_blocks = ni->allocated_size >> 9;
+	/*
+	 * Make sure the base inode does not go away and attach it to the
+	 * attribute inode.
+	 */
+	if (!igrab(base_vi)) {
+		err = -ENOENT;
+		goto unm_err_out;
+	}
+	ni->ext.base_ntfs_ino = base_ni;
+	ni->nr_extents = -1;
+
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(base_ni);
+
+	ntfs_debug("Done.");
+	return 0;
+
+unm_err_out:
+	if (!err)
+		err = -EIO;
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(base_ni);
+err_out:
+	if (err != -ENOENT)
+		ntfs_error(vol->sb,
+			"Failed with error code %i while reading attribute inode (mft_no 0x%llx, type 0x%x, name_len %i).  Marking corrupt inode and base inode 0x%llx as bad.  Run chkdsk.",
+			err, ni->mft_no, ni->type, ni->name_len,
+			base_ni->mft_no);
+	if (err != -ENOENT && err != -ENOMEM)
+		NVolSetErrors(vol);
+	return err;
+}
+
+/*
+ * ntfs_read_locked_index_inode - read an index inode from its base inode
+ * @base_vi:	base inode
+ * @vi:		index inode to read
+ *
+ * ntfs_read_locked_index_inode() is called from ntfs_index_iget() to read the
+ * index inode described by @vi into memory from the base mft record described
+ * by @base_ni.
+ *
+ * ntfs_read_locked_index_inode() maps, pins and locks the base inode for
+ * reading and looks up the attributes relating to the index described by @vi
+ * before setting up the necessary fields in @vi as well as initializing the
+ * ntfs inode.
+ *
+ * Note, index inodes are essentially attribute inodes (NInoAttr() is true)
+ * with the attribute type set to AT_INDEX_ALLOCATION.  Apart from that, they
+ * are setup like directory inodes since directories are a special case of
+ * indices ao they need to be treated in much the same way.  Most importantly,
+ * for small indices the index allocation attribute might not actually exist.
+ * However, the index root attribute always exists but this does not need to
+ * have an inode associated with it and this is why we define a new inode type
+ * index.  Also, like for directories, we need to have an attribute inode for
+ * the bitmap attribute corresponding to the index allocation attribute and we
+ * can store this in the appropriate field of the inode, just like we do for
+ * normal directory inodes.
+ *
+ * Q: What locks are held when the function is called?
+ * A: i_state has I_NEW set, hence the inode is locked, also
+ *    i_count is set to 1, so it is not going to go away
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi)
+{
+	loff_t bvi_size;
+	struct ntfs_volume *vol = NTFS_SB(vi->i_sb);
+	struct ntfs_inode *ni = NTFS_I(vi), *base_ni = NTFS_I(base_vi), *bni;
+	struct inode *bvi;
+	struct mft_record *m;
+	struct attr_record *a;
+	struct ntfs_attr_search_ctx *ctx;
+	struct index_root *ir;
+	u8 *ir_end, *index_end;
+	int err = 0;
+
+	ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no);
+	lockdep_assert_held(&base_ni->mrec_lock);
+
+	ntfs_init_big_inode(vi);
+	/* Just mirror the values from the base inode. */
+	vi->i_uid	= base_vi->i_uid;
+	vi->i_gid	= base_vi->i_gid;
+	set_nlink(vi, base_vi->i_nlink);
+	inode_set_mtime_to_ts(vi, inode_get_mtime(base_vi));
+	inode_set_ctime_to_ts(vi, inode_get_ctime(base_vi));
+	inode_set_atime_to_ts(vi, inode_get_atime(base_vi));
+	vi->i_generation = ni->seq_no = base_ni->seq_no;
+	/* Set inode type to zero but preserve permissions. */
+	vi->i_mode	= base_vi->i_mode & ~S_IFMT;
+	/* Map the mft record for the base inode. */
+	m = map_mft_record(base_ni);
+	if (IS_ERR(m)) {
+		err = PTR_ERR(m);
+		goto err_out;
+	}
+	ctx = ntfs_attr_get_search_ctx(base_ni, m);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto unm_err_out;
+	}
+	/* Find the index root attribute. */
+	err = ntfs_attr_lookup(AT_INDEX_ROOT, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err)) {
+		if (err == -ENOENT)
+			ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is missing.");
+		goto unm_err_out;
+	}
+	a = ctx->attr;
+	/* Set up the state. */
+	if (unlikely(a->non_resident)) {
+		ntfs_error(vol->sb, "$INDEX_ROOT attribute is not resident.");
+		goto unm_err_out;
+	}
+	/* Ensure the attribute name is placed before the value. */
+	if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >=
+			le16_to_cpu(a->data.resident.value_offset)))) {
+		ntfs_error(vol->sb,
+			"$INDEX_ROOT attribute name is placed after the attribute value.");
+		goto unm_err_out;
+	}
+
+	ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset));
+	ir_end = (u8 *)ir + le32_to_cpu(a->data.resident.value_length);
+	if (ir_end > (u8 *)ctx->mrec + vol->mft_record_size) {
+		ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is corrupt.");
+		goto unm_err_out;
+	}
+	index_end = (u8 *)&ir->index + le32_to_cpu(ir->index.index_length);
+	if (index_end > ir_end) {
+		ntfs_error(vi->i_sb, "Index is corrupt.");
+		goto unm_err_out;
+	}
+
+	ni->itype.index.collation_rule = ir->collation_rule;
+	ntfs_debug("Index collation rule is 0x%x.",
+			le32_to_cpu(ir->collation_rule));
+	ni->itype.index.block_size = le32_to_cpu(ir->index_block_size);
+	if (!is_power_of_2(ni->itype.index.block_size)) {
+		ntfs_error(vi->i_sb, "Index block size (%u) is not a power of two.",
+				ni->itype.index.block_size);
+		goto unm_err_out;
+	}
+	if (ni->itype.index.block_size > PAGE_SIZE) {
+		ntfs_error(vi->i_sb, "Index block size (%u) > PAGE_SIZE (%ld) is not supported.",
+				ni->itype.index.block_size, PAGE_SIZE);
+		err = -EOPNOTSUPP;
+		goto unm_err_out;
+	}
+	if (ni->itype.index.block_size < NTFS_BLOCK_SIZE) {
+		ntfs_error(vi->i_sb,
+				"Index block size (%u) < NTFS_BLOCK_SIZE (%i) is not supported.",
+				ni->itype.index.block_size, NTFS_BLOCK_SIZE);
+		err = -EOPNOTSUPP;
+		goto unm_err_out;
+	}
+	ni->itype.index.block_size_bits = ffs(ni->itype.index.block_size) - 1;
+	/* Determine the size of a vcn in the index. */
+	if (vol->cluster_size <= ni->itype.index.block_size) {
+		ni->itype.index.vcn_size = vol->cluster_size;
+		ni->itype.index.vcn_size_bits = vol->cluster_size_bits;
+	} else {
+		ni->itype.index.vcn_size = vol->sector_size;
+		ni->itype.index.vcn_size_bits = vol->sector_size_bits;
+	}
+
+	/* Find index allocation attribute. */
+	ntfs_attr_reinit_search_ctx(ctx);
+	err = ntfs_attr_lookup(AT_INDEX_ALLOCATION, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err)) {
+		if (err == -ENOENT) {
+			/* No index allocation. */
+			vi->i_size = ni->initialized_size = ni->allocated_size = 0;
+			/* We are done with the mft record, so we release it. */
+			ntfs_attr_put_search_ctx(ctx);
+			unmap_mft_record(base_ni);
+			m = NULL;
+			ctx = NULL;
+			goto skip_large_index_stuff;
+		} else
+			ntfs_error(vi->i_sb, "Failed to lookup $INDEX_ALLOCATION attribute.");
+		goto unm_err_out;
+	}
+	NInoSetIndexAllocPresent(ni);
+	NInoSetNonResident(ni);
+	ni->type = AT_INDEX_ALLOCATION;
+
+	a = ctx->attr;
+	if (!a->non_resident) {
+		ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is resident.");
+		goto unm_err_out;
+	}
+	/*
+	 * Ensure the attribute name is placed before the mapping pairs array.
+	 */
+	if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >=
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset)))) {
+		ntfs_error(vol->sb,
+			"$INDEX_ALLOCATION attribute name is placed after the mapping pairs array.");
+		goto unm_err_out;
+	}
+	if (a->flags & ATTR_IS_ENCRYPTED) {
+		ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is encrypted.");
+		goto unm_err_out;
+	}
+	if (a->flags & ATTR_IS_SPARSE) {
+		ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is sparse.");
+		goto unm_err_out;
+	}
+	if (a->flags & ATTR_COMPRESSION_MASK) {
+		ntfs_error(vi->i_sb,
+			"$INDEX_ALLOCATION attribute is compressed.");
+		goto unm_err_out;
+	}
+	if (a->data.non_resident.lowest_vcn) {
+		ntfs_error(vi->i_sb,
+			"First extent of $INDEX_ALLOCATION attribute has non zero lowest_vcn.");
+		goto unm_err_out;
+	}
+	vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size);
+	ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size);
+	ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size);
+	/*
+	 * We are done with the mft record, so we release it.  Otherwise
+	 * we would deadlock in ntfs_attr_iget().
+	 */
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(base_ni);
+	m = NULL;
+	ctx = NULL;
+	/* Get the index bitmap attribute inode. */
+	bvi = ntfs_attr_iget(base_vi, AT_BITMAP, ni->name, ni->name_len);
+	if (IS_ERR(bvi)) {
+		ntfs_error(vi->i_sb, "Failed to get bitmap attribute.");
+		err = PTR_ERR(bvi);
+		goto unm_err_out;
+	}
+	bni = NTFS_I(bvi);
+	if (NInoCompressed(bni) || NInoEncrypted(bni) ||
+			NInoSparse(bni)) {
+		ntfs_error(vi->i_sb,
+			"$BITMAP attribute is compressed and/or encrypted and/or sparse.");
+		goto iput_unm_err_out;
+	}
+	/* Consistency check bitmap size vs. index allocation size. */
+	bvi_size = i_size_read(bvi);
+	if ((bvi_size << 3) < (vi->i_size >> ni->itype.index.block_size_bits)) {
+		ntfs_error(vi->i_sb,
+			"Index bitmap too small (0x%llx) for index allocation (0x%llx).",
+			bvi_size << 3, vi->i_size);
+		goto iput_unm_err_out;
+	}
+	iput(bvi);
+skip_large_index_stuff:
+	/* Setup the operations for this index inode. */
+	ntfs_set_vfs_operations(vi, S_IFDIR, 0);
+	vi->i_blocks = ni->allocated_size >> 9;
+	/*
+	 * Make sure the base inode doesn't go away and attach it to the
+	 * index inode.
+	 */
+	if (!igrab(base_vi))
+		goto unm_err_out;
+	ni->ext.base_ntfs_ino = base_ni;
+	ni->nr_extents = -1;
+
+	ntfs_debug("Done.");
+	return 0;
+iput_unm_err_out:
+	iput(bvi);
+unm_err_out:
+	if (!err)
+		err = -EIO;
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	if (m)
+		unmap_mft_record(base_ni);
+err_out:
+	ntfs_error(vi->i_sb,
+		"Failed with error code %i while reading index inode (mft_no 0x%llx, name_len %i.",
+		err, ni->mft_no, ni->name_len);
+	if (err != -EOPNOTSUPP && err != -ENOMEM)
+		NVolSetErrors(vol);
+	return err;
+}
+
+/*
+ * load_attribute_list_mount - load an attribute list into memory
+ * @vol:		ntfs volume from which to read
+ * @rl:			runlist of the attribute list
+ * @al_start:		destination buffer
+ * @size:		size of the destination buffer in bytes
+ * @initialized_size:	initialized size of the attribute list
+ *
+ * Walk the runlist @rl and load all clusters from it copying them into
+ * the linear buffer @al. The maximum number of bytes copied to @al is @size
+ * bytes. Note, @size does not need to be a multiple of the cluster size. If
+ * @initialized_size is less than @size, the region in @al between
+ * @initialized_size and @size will be zeroed and not read from disk.
+ *
+ * Return 0 on success or -errno on error.
+ */
+static int load_attribute_list_mount(struct ntfs_volume *vol,
+		struct runlist_element *rl, u8 *al_start, const s64 size,
+		const s64 initialized_size)
+{
+	s64 lcn;
+	u8 *al = al_start;
+	u8 *al_end = al + initialized_size;
+	struct super_block *sb;
+	int err = 0;
+	loff_t rl_byte_off, rl_byte_len;
+
+	ntfs_debug("Entering.");
+	if (!vol || !rl || !al || size <= 0 || initialized_size < 0 ||
+			initialized_size > size)
+		return -EINVAL;
+	if (!initialized_size) {
+		memset(al, 0, size);
+		return 0;
+	}
+	sb = vol->sb;
+
+	/* Read all clusters specified by the runlist one run at a time. */
+	while (rl->length) {
+		lcn = ntfs_rl_vcn_to_lcn(rl, rl->vcn);
+		ntfs_debug("Reading vcn = 0x%llx, lcn = 0x%llx.",
+				(unsigned long long)rl->vcn,
+				(unsigned long long)lcn);
+		/* The attribute list cannot be sparse. */
+		if (lcn < 0) {
+			ntfs_error(sb, "ntfs_rl_vcn_to_lcn() failed. Cannot read attribute list.");
+			return -EIO;
+		}
+
+		rl_byte_off = ntfs_cluster_to_bytes(vol, lcn);
+		rl_byte_len = ntfs_cluster_to_bytes(vol, rl->length);
+
+		if (al + rl_byte_len > al_end)
+			rl_byte_len = al_end - al;
+
+		err = ntfs_bdev_read(sb->s_bdev, al, rl_byte_off,
+				   round_up(rl_byte_len, SECTOR_SIZE));
+		if (err) {
+			ntfs_error(sb, "Cannot read attribute list.");
+			return -EIO;
+		}
+
+		if (al + rl_byte_len >= al_end) {
+			if (initialized_size < size)
+				goto initialize;
+			goto done;
+		}
+
+		al += rl_byte_len;
+		rl++;
+	}
+	if (initialized_size < size) {
+initialize:
+		memset(al_start + initialized_size, 0, size - initialized_size);
+	}
+done:
+	return err;
+}
+
+/*
+ * The MFT inode has special locking, so teach the lock validator
+ * about this by splitting off the locking rules of the MFT from
+ * the locking rules of other inodes. The MFT inode can never be
+ * accessed from the VFS side (or even internally), only by the
+ * map_mft functions.
+ */
+static struct lock_class_key mft_ni_runlist_lock_key, mft_ni_mrec_lock_key;
+
+/*
+ * ntfs_read_inode_mount - special read_inode for mount time use only
+ * @vi:		inode to read
+ *
+ * Read inode FILE_MFT at mount time, only called with super_block lock
+ * held from within the read_super() code path.
+ *
+ * This function exists because when it is called the page cache for $MFT/$DATA
+ * is not initialized and hence we cannot get at the contents of mft records
+ * by calling map_mft_record*().
+ *
+ * Further it needs to cope with the circular references problem, i.e. cannot
+ * load any attributes other than $ATTRIBUTE_LIST until $DATA is loaded, because
+ * we do not know where the other extent mft records are yet and again, because
+ * we cannot call map_mft_record*() yet.  Obviously this applies only when an
+ * attribute list is actually present in $MFT inode.
+ *
+ * We solve these problems by starting with the $DATA attribute before anything
+ * else and iterating using ntfs_attr_lookup($DATA) over all extents.  As each
+ * extent is found, we ntfs_mapping_pairs_decompress() including the implied
+ * ntfs_runlists_merge().  Each step of the iteration necessarily provides
+ * sufficient information for the next step to complete.
+ *
+ * This should work but there are two possible pit falls (see inline comments
+ * below), but only time will tell if they are real pits or just smoke...
+ */
+int ntfs_read_inode_mount(struct inode *vi)
+{
+	s64 next_vcn, last_vcn, highest_vcn;
+	struct super_block *sb = vi->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct mft_record *m = NULL;
+	struct attr_record *a;
+	struct ntfs_attr_search_ctx *ctx;
+	unsigned int i, nr_blocks;
+	int err;
+	size_t new_rl_count;
+
+	ntfs_debug("Entering.");
+
+	/* Initialize the ntfs specific part of @vi. */
+	ntfs_init_big_inode(vi);
+
+
+	/* Setup the data attribute. It is special as it is mst protected. */
+	NInoSetNonResident(ni);
+	NInoSetMstProtected(ni);
+	NInoSetSparseDisabled(ni);
+	ni->type = AT_DATA;
+	ni->name = AT_UNNAMED;
+	ni->name_len = 0;
+	/*
+	 * This sets up our little cheat allowing us to reuse the async read io
+	 * completion handler for directories.
+	 */
+	ni->itype.index.block_size = vol->mft_record_size;
+	ni->itype.index.block_size_bits = vol->mft_record_size_bits;
+
+	/* Very important! Needed to be able to call map_mft_record*(). */
+	vol->mft_ino = vi;
+
+	/* Allocate enough memory to read the first mft record. */
+	if (vol->mft_record_size > 64 * 1024) {
+		ntfs_error(sb, "Unsupported mft record size %i (max 64kiB).",
+				vol->mft_record_size);
+		goto err_out;
+	}
+
+	i = vol->mft_record_size;
+	if (i < sb->s_blocksize)
+		i = sb->s_blocksize;
+
+	m = kzalloc(i, GFP_NOFS);
+	if (!m) {
+		ntfs_error(sb, "Failed to allocate buffer for $MFT record 0.");
+		goto err_out;
+	}
+
+	/* Determine the first block of the $MFT/$DATA attribute. */
+	nr_blocks = ntfs_bytes_to_sector(vol, vol->mft_record_size);
+	if (!nr_blocks)
+		nr_blocks = 1;
+
+	/* Load $MFT/$DATA's first mft record. */
+	err = ntfs_bdev_read(sb->s_bdev, (char *)m,
+			     ntfs_cluster_to_bytes(vol, vol->mft_lcn), i);
+	if (err) {
+		ntfs_error(sb, "Device read failed.");
+		goto err_out;
+	}
+
+	if (le32_to_cpu(m->bytes_allocated) != vol->mft_record_size) {
+		ntfs_error(sb, "Incorrect mft record size %u in superblock, should be %u.",
+				le32_to_cpu(m->bytes_allocated), vol->mft_record_size);
+		goto err_out;
+	}
+
+	/* Apply the mst fixups. */
+	if (post_read_mst_fixup((struct ntfs_record *)m, vol->mft_record_size)) {
+		ntfs_error(sb, "MST fixup failed. $MFT is corrupt.");
+		goto err_out;
+	}
+
+	if (ntfs_mft_record_check(vol, m, FILE_MFT)) {
+		ntfs_error(sb, "ntfs_mft_record_check failed. $MFT is corrupt.");
+		goto err_out;
+	}
+
+	/* Need this to sanity check attribute list references to $MFT. */
+	vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number);
+
+	/* Provides read_folio() for map_mft_record(). */
+	vi->i_mapping->a_ops = &ntfs_mft_aops;
+
+	ctx = ntfs_attr_get_search_ctx(ni, m);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	/* Find the attribute list attribute if present. */
+	err = ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, 0, 0, NULL, 0, ctx);
+	if (err) {
+		if (unlikely(err != -ENOENT)) {
+			ntfs_error(sb,
+				"Failed to lookup attribute list attribute. You should run chkdsk.");
+			goto put_err_out;
+		}
+	} else /* if (!err) */ {
+		struct attr_list_entry *al_entry, *next_al_entry;
+		u8 *al_end;
+		static const char *es = "  Not allowed.  $MFT is corrupt.  You should run chkdsk.";
+
+		ntfs_debug("Attribute list attribute found in $MFT.");
+		NInoSetAttrList(ni);
+		a = ctx->attr;
+		if (a->flags & ATTR_COMPRESSION_MASK) {
+			ntfs_error(sb,
+				"Attribute list attribute is compressed.%s",
+				es);
+			goto put_err_out;
+		}
+		if (a->flags & ATTR_IS_ENCRYPTED ||
+				a->flags & ATTR_IS_SPARSE) {
+			if (a->non_resident) {
+				ntfs_error(sb,
+					"Non-resident attribute list attribute is encrypted/sparse.%s",
+					es);
+				goto put_err_out;
+			}
+			ntfs_warning(sb,
+				"Resident attribute list attribute in $MFT system file is marked encrypted/sparse which is not true.  However, Windows allows this and chkdsk does not detect or correct it so we will just ignore the invalid flags and pretend they are not set.");
+		}
+		/* Now allocate memory for the attribute list. */
+		ni->attr_list_size = (u32)ntfs_attr_size(a);
+		if (!ni->attr_list_size) {
+			ntfs_error(sb, "Attr_list_size is zero");
+			goto put_err_out;
+		}
+		ni->attr_list = kvzalloc(round_up(ni->attr_list_size, SECTOR_SIZE),
+					 GFP_NOFS);
+		if (!ni->attr_list) {
+			ntfs_error(sb, "Not enough memory to allocate buffer for attribute list.");
+			goto put_err_out;
+		}
+		if (a->non_resident) {
+			struct runlist_element *rl;
+			size_t new_rl_count;
+
+			NInoSetAttrListNonResident(ni);
+			if (a->data.non_resident.lowest_vcn) {
+				ntfs_error(sb,
+					"Attribute list has non zero lowest_vcn. $MFT is corrupt. You should run chkdsk.");
+				goto put_err_out;
+			}
+
+			rl = ntfs_mapping_pairs_decompress(vol, a, NULL, &new_rl_count);
+			if (IS_ERR(rl)) {
+				err = PTR_ERR(rl);
+				ntfs_error(sb,
+					   "Mapping pairs decompression failed with error code %i.",
+					   -err);
+				goto put_err_out;
+			}
+
+			err = load_attribute_list_mount(vol, rl, ni->attr_list, ni->attr_list_size,
+					le64_to_cpu(a->data.non_resident.initialized_size));
+			kvfree(rl);
+			if (err) {
+				ntfs_error(sb,
+					   "Failed to load attribute list with error code %i.",
+					   -err);
+				goto put_err_out;
+			}
+		} else /* if (!ctx.attr->non_resident) */ {
+			if ((u8 *)a + le16_to_cpu(
+					a->data.resident.value_offset) +
+					le32_to_cpu(a->data.resident.value_length) >
+					(u8 *)ctx->mrec + vol->mft_record_size) {
+				ntfs_error(sb, "Corrupt attribute list attribute.");
+				goto put_err_out;
+			}
+			/* Now copy the attribute list. */
+			memcpy(ni->attr_list, (u8 *)a + le16_to_cpu(
+					a->data.resident.value_offset),
+					le32_to_cpu(a->data.resident.value_length));
+		}
+		/* The attribute list is now setup in memory. */
+		al_entry = (struct attr_list_entry *)ni->attr_list;
+		al_end = (u8 *)al_entry + ni->attr_list_size;
+		for (;; al_entry = next_al_entry) {
+			/* Out of bounds check. */
+			if ((u8 *)al_entry < ni->attr_list ||
+					(u8 *)al_entry > al_end)
+				goto em_put_err_out;
+			/* Catch the end of the attribute list. */
+			if ((u8 *)al_entry == al_end)
+				goto em_put_err_out;
+			if (!al_entry->length)
+				goto em_put_err_out;
+			if ((u8 *)al_entry + 6 > al_end ||
+			    (u8 *)al_entry + le16_to_cpu(al_entry->length) > al_end)
+				goto em_put_err_out;
+			next_al_entry = (struct attr_list_entry *)((u8 *)al_entry +
+					le16_to_cpu(al_entry->length));
+			if (le32_to_cpu(al_entry->type) > le32_to_cpu(AT_DATA))
+				goto em_put_err_out;
+			if (al_entry->type != AT_DATA)
+				continue;
+			/* We want an unnamed attribute. */
+			if (al_entry->name_length)
+				goto em_put_err_out;
+			/* Want the first entry, i.e. lowest_vcn == 0. */
+			if (al_entry->lowest_vcn)
+				goto em_put_err_out;
+			/* First entry has to be in the base mft record. */
+			if (MREF_LE(al_entry->mft_reference) != vi->i_ino) {
+				/* MFT references do not match, logic fails. */
+				ntfs_error(sb,
+					"BUG: The first $DATA extent of $MFT is not in the base mft record.");
+				goto put_err_out;
+			} else {
+				/* Sequence numbers must match. */
+				if (MSEQNO_LE(al_entry->mft_reference) !=
+						ni->seq_no)
+					goto em_put_err_out;
+				/* Got it. All is ok. We can stop now. */
+				break;
+			}
+		}
+	}
+
+	ntfs_attr_reinit_search_ctx(ctx);
+
+	/* Now load all attribute extents. */
+	a = NULL;
+	next_vcn = last_vcn = highest_vcn = 0;
+	while (!(err = ntfs_attr_lookup(AT_DATA, NULL, 0, 0, next_vcn, NULL, 0,
+			ctx))) {
+		struct runlist_element *nrl;
+
+		/* Cache the current attribute. */
+		a = ctx->attr;
+		/* $MFT must be non-resident. */
+		if (!a->non_resident) {
+			ntfs_error(sb,
+				"$MFT must be non-resident but a resident extent was found. $MFT is corrupt. Run chkdsk.");
+			goto put_err_out;
+		}
+		/* $MFT must be uncompressed and unencrypted. */
+		if (a->flags & ATTR_COMPRESSION_MASK ||
+				a->flags & ATTR_IS_ENCRYPTED ||
+				a->flags & ATTR_IS_SPARSE) {
+			ntfs_error(sb,
+				"$MFT must be uncompressed, non-sparse, and unencrypted but a compressed/sparse/encrypted extent was found. $MFT is corrupt. Run chkdsk.");
+			goto put_err_out;
+		}
+		/*
+		 * Decompress the mapping pairs array of this extent and merge
+		 * the result into the existing runlist. No need for locking
+		 * as we have exclusive access to the inode at this time and we
+		 * are a mount in progress task, too.
+		 */
+		nrl = ntfs_mapping_pairs_decompress(vol, a, &ni->runlist,
+						    &new_rl_count);
+		if (IS_ERR(nrl)) {
+			ntfs_error(sb,
+				"ntfs_mapping_pairs_decompress() failed with error code %ld.",
+				PTR_ERR(nrl));
+			goto put_err_out;
+		}
+		ni->runlist.rl = nrl;
+		ni->runlist.count = new_rl_count;
+
+		/* Are we in the first extent? */
+		if (!next_vcn) {
+			if (a->data.non_resident.lowest_vcn) {
+				ntfs_error(sb,
+					"First extent of $DATA attribute has non zero lowest_vcn. $MFT is corrupt. You should run chkdsk.");
+				goto put_err_out;
+			}
+			/* Get the last vcn in the $DATA attribute. */
+			last_vcn = ntfs_bytes_to_cluster(vol,
+					le64_to_cpu(a->data.non_resident.allocated_size));
+			/* Fill in the inode size. */
+			vi->i_size = le64_to_cpu(a->data.non_resident.data_size);
+			ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size);
+			ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size);
+			/*
+			 * Verify the number of mft records does not exceed
+			 * 2^32 - 1.
+			 */
+			if ((vi->i_size >> vol->mft_record_size_bits) >=
+					(1ULL << 32)) {
+				ntfs_error(sb, "$MFT is too big! Aborting.");
+				goto put_err_out;
+			}
+			/*
+			 * We have got the first extent of the runlist for
+			 * $MFT which means it is now relatively safe to call
+			 * the normal ntfs_read_inode() function.
+			 * Complete reading the inode, this will actually
+			 * re-read the mft record for $MFT, this time entering
+			 * it into the page cache with which we complete the
+			 * kick start of the volume. It should be safe to do
+			 * this now as the first extent of $MFT/$DATA is
+			 * already known and we would hope that we don't need
+			 * further extents in order to find the other
+			 * attributes belonging to $MFT. Only time will tell if
+			 * this is really the case. If not we will have to play
+			 * magic at this point, possibly duplicating a lot of
+			 * ntfs_read_inode() at this point. We will need to
+			 * ensure we do enough of its work to be able to call
+			 * ntfs_read_inode() on extents of $MFT/$DATA. But lets
+			 * hope this never happens...
+			 */
+			err = ntfs_read_locked_inode(vi);
+			if (err) {
+				ntfs_error(sb, "ntfs_read_inode() of $MFT failed.\n");
+				ntfs_attr_put_search_ctx(ctx);
+				/* Revert to the safe super operations. */
+				kfree(m);
+				return -1;
+			}
+			/*
+			 * Re-initialize some specifics about $MFT's inode as
+			 * ntfs_read_inode() will have set up the default ones.
+			 */
+			/* Set uid and gid to root. */
+			vi->i_uid = GLOBAL_ROOT_UID;
+			vi->i_gid = GLOBAL_ROOT_GID;
+			/* Regular file. No access for anyone. */
+			vi->i_mode = S_IFREG;
+			/* No VFS initiated operations allowed for $MFT. */
+			vi->i_op = &ntfs_empty_inode_ops;
+			vi->i_fop = &ntfs_empty_file_ops;
+		}
+
+		/* Get the lowest vcn for the next extent. */
+		highest_vcn = le64_to_cpu(a->data.non_resident.highest_vcn);
+		next_vcn = highest_vcn + 1;
+
+		/* Only one extent or error, which we catch below. */
+		if (next_vcn <= 0)
+			break;
+
+		/* Avoid endless loops due to corruption. */
+		if (next_vcn < le64_to_cpu(a->data.non_resident.lowest_vcn)) {
+			ntfs_error(sb, "$MFT has corrupt attribute list attribute. Run chkdsk.");
+			goto put_err_out;
+		}
+	}
+	if (err != -ENOENT) {
+		ntfs_error(sb, "Failed to lookup $MFT/$DATA attribute extent. Run chkdsk.\n");
+		goto put_err_out;
+	}
+	if (!a) {
+		ntfs_error(sb, "$MFT/$DATA attribute not found. $MFT is corrupt. Run chkdsk.");
+		goto put_err_out;
+	}
+	if (highest_vcn && highest_vcn != last_vcn - 1) {
+		ntfs_error(sb, "Failed to load the complete runlist for $MFT/$DATA. Run chkdsk.");
+		ntfs_debug("highest_vcn = 0x%llx, last_vcn - 1 = 0x%llx",
+				(unsigned long long)highest_vcn,
+				(unsigned long long)last_vcn - 1);
+		goto put_err_out;
+	}
+	ntfs_attr_put_search_ctx(ctx);
+	ntfs_debug("Done.");
+	kfree(m);
+
+	/*
+	 * Split the locking rules of the MFT inode from the
+	 * locking rules of other inodes:
+	 */
+	lockdep_set_class(&ni->runlist.lock, &mft_ni_runlist_lock_key);
+	lockdep_set_class(&ni->mrec_lock, &mft_ni_mrec_lock_key);
+
+	return 0;
+
+em_put_err_out:
+	ntfs_error(sb,
+		"Couldn't find first extent of $DATA attribute in attribute list. $MFT is corrupt. Run chkdsk.");
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+err_out:
+	ntfs_error(sb, "Failed. Marking inode as bad.");
+	kfree(m);
+	return -1;
+}
+
+static void __ntfs_clear_inode(struct ntfs_inode *ni)
+{
+	/* Free all alocated memory. */
+	if (NInoNonResident(ni) && ni->runlist.rl) {
+		kvfree(ni->runlist.rl);
+		ni->runlist.rl = NULL;
+	}
+
+	if (ni->attr_list) {
+		kvfree(ni->attr_list);
+		ni->attr_list = NULL;
+	}
+
+	if (ni->name_len && ni->name != I30 &&
+	    ni->name != reparse_index_name &&
+	    ni->name != objid_index_name) {
+		WARN_ON(!ni->name);
+		kfree(ni->name);
+	}
+}
+
+void ntfs_clear_extent_inode(struct ntfs_inode *ni)
+{
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+
+	WARN_ON(NInoAttr(ni));
+	WARN_ON(ni->nr_extents != -1);
+
+	__ntfs_clear_inode(ni);
+	ntfs_destroy_extent_inode(ni);
+}
+
+static int ntfs_delete_base_inode(struct ntfs_inode *ni)
+{
+	struct super_block *sb = ni->vol->sb;
+	int err;
+
+	if (NInoAttr(ni) || ni->nr_extents == -1)
+		return 0;
+
+	err = ntfs_non_resident_dealloc_clusters(ni);
+
+	/*
+	 * Deallocate extent mft records and free extent inodes.
+	 * No need to lock as no one else has a reference.
+	 */
+	while (ni->nr_extents) {
+		err = ntfs_mft_record_free(ni->vol, *(ni->ext.extent_ntfs_inos));
+		if (err)
+			ntfs_error(sb,
+				"Failed to free extent MFT record. Leaving inconsistent metadata.\n");
+		ntfs_inode_close(*(ni->ext.extent_ntfs_inos));
+	}
+
+	/* Deallocate base mft record */
+	err = ntfs_mft_record_free(ni->vol, ni);
+	if (err)
+		ntfs_error(sb, "Failed to free base MFT record. Leaving inconsistent metadata.\n");
+	return err;
+}
+
+/*
+ * ntfs_evict_big_inode - clean up the ntfs specific part of an inode
+ * @vi:		vfs inode pending annihilation
+ *
+ * When the VFS is going to remove an inode from memory, ntfs_clear_big_inode()
+ * is called, which deallocates all memory belonging to the NTFS specific part
+ * of the inode and returns.
+ *
+ * If the MFT record is dirty, we commit it before doing anything else.
+ */
+void ntfs_evict_big_inode(struct inode *vi)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+
+	truncate_inode_pages_final(&vi->i_data);
+
+	if (!vi->i_nlink) {
+		if (!NInoAttr(ni)) {
+			/* Never called with extent inodes */
+			WARN_ON(ni->nr_extents == -1);
+			ntfs_delete_base_inode(ni);
+		}
+		goto release;
+	}
+
+	if (NInoDirty(ni)) {
+		/* Committing the inode also commits all extent inodes. */
+		ntfs_commit_inode(vi);
+
+		if (NInoDirty(ni)) {
+			ntfs_debug("Failed to commit dirty inode 0x%llx.  Losing data!",
+				   ni->mft_no);
+			NInoClearAttrListDirty(ni);
+			NInoClearDirty(ni);
+		}
+	}
+
+	/* No need to lock at this stage as no one else has a reference. */
+	if (ni->nr_extents > 0) {
+		int i;
+
+		for (i = 0; i < ni->nr_extents; i++) {
+			if (ni->ext.extent_ntfs_inos[i])
+				ntfs_clear_extent_inode(ni->ext.extent_ntfs_inos[i]);
+		}
+		ni->nr_extents = 0;
+		kvfree(ni->ext.extent_ntfs_inos);
+	}
+
+release:
+	clear_inode(vi);
+	__ntfs_clear_inode(ni);
+
+	if (NInoAttr(ni)) {
+		/* Release the base inode if we are holding it. */
+		if (ni->nr_extents == -1) {
+			iput(VFS_I(ni->ext.base_ntfs_ino));
+			ni->nr_extents = 0;
+			ni->ext.base_ntfs_ino = NULL;
+		}
+	}
+
+	if (!atomic_dec_and_test(&ni->count))
+		WARN_ON(1);
+	if (ni->folio)
+		folio_put(ni->folio);
+	kfree(ni->mrec);
+	kvfree(ni->target);
+}
+
+/*
+ * ntfs_show_options - show mount options in /proc/mounts
+ * @sf:		seq_file in which to write our mount options
+ * @root:	root of the mounted tree whose mount options to display
+ *
+ * Called by the VFS once for each mounted ntfs volume when someone reads
+ * /proc/mounts in order to display the NTFS specific mount options of each
+ * mount. The mount options of fs specified by @root are written to the seq file
+ * @sf and success is returned.
+ */
+int ntfs_show_options(struct seq_file *sf, struct dentry *root)
+{
+	struct ntfs_volume *vol = NTFS_SB(root->d_sb);
+	int i;
+
+	if (uid_valid(vol->uid))
+		seq_printf(sf, ",uid=%i", from_kuid_munged(&init_user_ns, vol->uid));
+	if (gid_valid(vol->gid))
+		seq_printf(sf, ",gid=%i", from_kgid_munged(&init_user_ns, vol->gid));
+	if (vol->fmask == vol->dmask)
+		seq_printf(sf, ",umask=0%o", vol->fmask);
+	else {
+		seq_printf(sf, ",fmask=0%o", vol->fmask);
+		seq_printf(sf, ",dmask=0%o", vol->dmask);
+	}
+	seq_printf(sf, ",iocharset=%s", vol->nls_map->charset);
+	if (NVolCaseSensitive(vol))
+		seq_puts(sf, ",case_sensitive");
+	else
+		seq_puts(sf, ",nocase");
+	if (NVolShowSystemFiles(vol))
+		seq_puts(sf, ",show_sys_files,showmeta");
+	for (i = 0; on_errors_arr[i].val; i++) {
+		if (on_errors_arr[i].val == vol->on_errors)
+			seq_printf(sf, ",errors=%s", on_errors_arr[i].str);
+	}
+	seq_printf(sf, ",mft_zone_multiplier=%i", vol->mft_zone_multiplier);
+	if (NVolSysImmutable(vol))
+		seq_puts(sf, ",sys_immutable");
+	if (!NVolShowHiddenFiles(vol))
+		seq_puts(sf, ",nohidden");
+	if (NVolHideDotFiles(vol))
+		seq_puts(sf, ",hide_dot_files");
+	if (NVolCheckWindowsNames(vol))
+		seq_puts(sf, ",windows_names");
+	if (NVolDiscard(vol))
+		seq_puts(sf, ",discard");
+	if (NVolDisableSparse(vol))
+		seq_puts(sf, ",disable_sparse");
+	if (vol->sb->s_flags & SB_POSIXACL)
+		seq_puts(sf, ",acl");
+	return 0;
+}
+
+int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset,
+				 const loff_t new_size, bool bsync)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	loff_t old_init_size;
+	unsigned long flags;
+	int err;
+
+	read_lock_irqsave(&ni->size_lock, flags);
+	old_init_size = ni->initialized_size;
+	read_unlock_irqrestore(&ni->size_lock, flags);
+
+	if (!NInoNonResident(ni))
+		return -EINVAL;
+	if (old_init_size >= new_size)
+		return 0;
+
+	err = ntfs_attr_map_whole_runlist(ni);
+	if (err)
+		return err;
+
+	if (!NInoCompressed(ni) && old_init_size < offset) {
+		err = iomap_zero_range(vi, old_init_size,
+				       offset - old_init_size,
+				       NULL, &ntfs_seek_iomap_ops,
+				       &ntfs_iomap_folio_ops, NULL);
+		if (err)
+			return err;
+		if (bsync)
+			err = filemap_write_and_wait_range(vi->i_mapping,
+							   old_init_size,
+							   offset - 1);
+	}
+
+
+	mutex_lock(&ni->mrec_lock);
+	err = ntfs_attr_set_initialized_size(ni, new_size);
+	mutex_unlock(&ni->mrec_lock);
+	if (err)
+		truncate_setsize(vi, old_init_size);
+	return err;
+}
+
+int ntfs_truncate_vfs(struct inode *vi, loff_t new_size, loff_t i_size)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	int err;
+
+	mutex_lock(&ni->mrec_lock);
+	err = __ntfs_attr_truncate_vfs(ni, new_size, i_size);
+	mutex_unlock(&ni->mrec_lock);
+	if (err < 0)
+		return err;
+
+	inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi));
+	return 0;
+}
+
+/*
+ * ntfs_inode_sync_standard_information - update standard information attribute
+ * @vi:	inode to update standard information
+ * @m:	mft record
+ *
+ * Return 0 on success or -errno on error.
+ */
+static int ntfs_inode_sync_standard_information(struct inode *vi, struct mft_record *m)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_attr_search_ctx *ctx;
+	struct standard_information *si;
+	__le64 nt;
+	int err = 0;
+	bool modified = false;
+
+	/* Update the access times in the standard information attribute. */
+	ctx = ntfs_attr_get_search_ctx(ni, m);
+	if (unlikely(!ctx))
+		return -ENOMEM;
+	err = ntfs_attr_lookup(AT_STANDARD_INFORMATION, NULL, 0,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err)) {
+		ntfs_attr_put_search_ctx(ctx);
+		return err;
+	}
+	si = (struct standard_information *)((u8 *)ctx->attr +
+			le16_to_cpu(ctx->attr->data.resident.value_offset));
+	if (si->file_attributes != ni->flags) {
+		si->file_attributes = ni->flags;
+		modified = true;
+	}
+
+	/* Update the creation times if they have changed. */
+	nt = utc2ntfs(ni->i_crtime);
+	if (si->creation_time != nt) {
+		ntfs_debug("Updating creation time for inode 0x%llx: old = 0x%llx, new = 0x%llx",
+				ni->mft_no, le64_to_cpu(si->creation_time),
+				le64_to_cpu(nt));
+		si->creation_time = nt;
+		modified = true;
+	}
+
+	/* Update the access times if they have changed. */
+	nt = utc2ntfs(inode_get_mtime(vi));
+	if (si->last_data_change_time != nt) {
+		ntfs_debug("Updating mtime for inode 0x%llx: old = 0x%llx, new = 0x%llx",
+				ni->mft_no, le64_to_cpu(si->last_data_change_time),
+				le64_to_cpu(nt));
+		si->last_data_change_time = nt;
+		modified = true;
+	}
+
+	nt = utc2ntfs(inode_get_ctime(vi));
+	if (si->last_mft_change_time != nt) {
+		ntfs_debug("Updating ctime for inode 0x%llx: old = 0x%llx, new = 0x%llx",
+				ni->mft_no, le64_to_cpu(si->last_mft_change_time),
+				le64_to_cpu(nt));
+		si->last_mft_change_time = nt;
+		modified = true;
+	}
+	nt = utc2ntfs(inode_get_atime(vi));
+	if (si->last_access_time != nt) {
+		ntfs_debug("Updating atime for inode 0x%llx: old = 0x%llx, new = 0x%llx",
+				ni->mft_no,
+				le64_to_cpu(si->last_access_time),
+				le64_to_cpu(nt));
+		si->last_access_time = nt;
+		modified = true;
+	}
+
+	/*
+	 * If we just modified the standard information attribute we need to
+	 * mark the mft record it is in dirty.  We do this manually so that
+	 * mark_inode_dirty() is not called which would redirty the inode and
+	 * hence result in an infinite loop of trying to write the inode.
+	 * There is no need to mark the base inode nor the base mft record
+	 * dirty, since we are going to write this mft record below in any case
+	 * and the base mft record may actually not have been modified so it
+	 * might not need to be written out.
+	 * NOTE: It is not a problem when the inode for $MFT itself is being
+	 * written out as ntfs_mft_mark_dirty() will only set I_DIRTY_PAGES
+	 * on the $MFT inode and hence ntfs_write_inode() will not be
+	 * re-invoked because of it which in turn is ok since the dirtied mft
+	 * record will be cleaned and written out to disk below, i.e. before
+	 * this function returns.
+	 */
+	if (modified)
+		NInoSetDirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+
+	return err;
+}
+
+/*
+ * ntfs_inode_sync_filename - update FILE_NAME attributes
+ * @ni:	ntfs inode to update FILE_NAME attributes
+ *
+ * Update all FILE_NAME attributes for inode @ni in the index.
+ *
+ * Return 0 on success or error.
+ */
+int ntfs_inode_sync_filename(struct ntfs_inode *ni)
+{
+	struct inode *index_vi;
+	struct super_block *sb = VFS_I(ni)->i_sb;
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct ntfs_index_context *ictx;
+	struct ntfs_inode *index_ni;
+	struct file_name_attr *fn;
+	struct file_name_attr *fnx;
+	struct reparse_point *rpp;
+	__le32 reparse_tag;
+	int err = 0;
+	unsigned long flags;
+
+	ntfs_debug("Entering for inode %llu\n", ni->mft_no);
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx)
+		return -ENOMEM;
+
+	/* Collect the reparse tag, if any */
+	reparse_tag = cpu_to_le32(0);
+	if (ni->flags & FILE_ATTR_REPARSE_POINT) {
+		if (!ntfs_attr_lookup(AT_REPARSE_POINT, NULL,
+					0, CASE_SENSITIVE, 0, NULL, 0, ctx)) {
+			rpp = (struct reparse_point *)((u8 *)ctx->attr +
+					le16_to_cpu(ctx->attr->data.resident.value_offset));
+			reparse_tag = rpp->reparse_tag;
+		}
+		ntfs_attr_reinit_search_ctx(ctx);
+	}
+
+	/* Walk through all FILE_NAME attributes and update them. */
+	while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0, ctx))) {
+		fn = (struct file_name_attr *)((u8 *)ctx->attr +
+				le16_to_cpu(ctx->attr->data.resident.value_offset));
+		if (MREF_LE(fn->parent_directory) == ni->mft_no)
+			continue;
+
+		index_vi = ntfs_iget(sb, MREF_LE(fn->parent_directory));
+		if (IS_ERR(index_vi)) {
+			ntfs_error(sb, "Failed to open inode %lld with index",
+					(long long)MREF_LE(fn->parent_directory));
+			continue;
+		}
+
+		index_ni = NTFS_I(index_vi);
+
+		mutex_lock_nested(&index_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+		if (NInoBeingDeleted(ni)) {
+			iput(index_vi);
+			mutex_unlock(&index_ni->mrec_lock);
+			continue;
+		}
+
+		ictx = ntfs_index_ctx_get(index_ni, I30, 4);
+		if (!ictx) {
+			ntfs_error(sb, "Failed to get index ctx, inode %llu",
+					index_ni->mft_no);
+			iput(index_vi);
+			mutex_unlock(&index_ni->mrec_lock);
+			continue;
+		}
+
+		err = ntfs_index_lookup(fn, sizeof(struct file_name_attr), ictx);
+		if (err) {
+			ntfs_debug("Index lookup failed, inode %llu",
+					index_ni->mft_no);
+			ntfs_index_ctx_put(ictx);
+			iput(index_vi);
+			mutex_unlock(&index_ni->mrec_lock);
+			continue;
+		}
+		/* Update flags and file size. */
+		fnx = (struct file_name_attr *)ictx->data;
+		fnx->file_attributes =
+			(fnx->file_attributes & ~FILE_ATTR_VALID_FLAGS) |
+			(ni->flags & FILE_ATTR_VALID_FLAGS);
+		if (ctx->mrec->flags & MFT_RECORD_IS_DIRECTORY)
+			fnx->data_size = fnx->allocated_size = 0;
+		else {
+			read_lock_irqsave(&ni->size_lock, flags);
+			if (NInoSparse(ni) || NInoCompressed(ni))
+				fnx->allocated_size = cpu_to_le64(ni->itype.compressed.size);
+			else
+				fnx->allocated_size = cpu_to_le64(ni->allocated_size);
+			fnx->data_size = cpu_to_le64(ni->data_size);
+
+			/*
+			 * The file name record has also to be fixed if some
+			 * attribute update implied the unnamed data to be
+			 * made non-resident
+			 */
+			fn->allocated_size = fnx->allocated_size;
+			fn->data_size = fnx->data_size;
+			read_unlock_irqrestore(&ni->size_lock, flags);
+		}
+
+		/* update or clear the reparse tag in the index */
+		fnx->type.rp.reparse_point_tag = reparse_tag;
+		fnx->creation_time = fn->creation_time;
+		fnx->last_data_change_time = fn->last_data_change_time;
+		fnx->last_mft_change_time = fn->last_mft_change_time;
+		fnx->last_access_time = fn->last_access_time;
+		ntfs_index_entry_mark_dirty(ictx);
+		ntfs_icx_ib_sync_write(ictx);
+		NInoSetDirty(ctx->ntfs_ino);
+		ntfs_index_ctx_put(ictx);
+		mutex_unlock(&index_ni->mrec_lock);
+		iput(index_vi);
+	}
+	/* Check for real error occurred. */
+	if (err != -ENOENT) {
+		ntfs_error(sb, "Attribute lookup failed, err : %d, inode %llu", err,
+				ni->mft_no);
+	} else
+		err = 0;
+
+	ntfs_attr_put_search_ctx(ctx);
+	return err;
+}
+
+int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni)
+{
+	s64 vcn;
+	struct runlist_element *rl;
+
+	if (ni->mft_lcn[0] != LCN_RL_NOT_MAPPED)
+		return 0;
+
+	vcn = (s64)ni->mft_no << mft_ni->vol->mft_record_size_bits >>
+	      mft_ni->vol->cluster_size_bits;
+
+	rl = mft_ni->runlist.rl;
+	if (!rl) {
+		ntfs_error(mft_ni->vol->sb, "$MFT runlist is not present");
+		return -EIO;
+	}
+
+	/* Seek to element containing target vcn. */
+	while (rl->length && rl[1].vcn <= vcn)
+		rl++;
+	ni->mft_lcn[0] = ntfs_rl_vcn_to_lcn(rl, vcn);
+	ni->mft_lcn_count = 1;
+
+	if (mft_ni->vol->cluster_size < mft_ni->vol->mft_record_size &&
+	    (rl->length - (vcn - rl->vcn)) <= 1) {
+		rl++;
+		ni->mft_lcn[1] = ntfs_rl_vcn_to_lcn(rl, vcn + 1);
+		ni->mft_lcn_count++;
+	}
+	return 0;
+}
+
+/*
+ * __ntfs_write_inode - write out a dirty inode
+ * @vi:		inode to write out
+ * @sync:	if true, write out synchronously
+ *
+ * Write out a dirty inode to disk including any extent inodes if present.
+ *
+ * If @sync is true, commit the inode to disk and wait for io completion.  This
+ * is done using write_mft_record().
+ *
+ * If @sync is false, just schedule the write to happen but do not wait for i/o
+ * completion.
+ *
+ * Return 0 on success and -errno on error.
+ */
+int __ntfs_write_inode(struct inode *vi, int sync)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_inode *mft_ni = NTFS_I(ni->vol->mft_ino);
+	struct mft_record *m;
+	int err = 0;
+	bool need_iput = false;
+
+	ntfs_debug("Entering for %sinode 0x%llx.", NInoAttr(ni) ? "attr " : "",
+			ni->mft_no);
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	/*
+	 * Dirty attribute inodes are written via their real inodes so just
+	 * clean them here.  Access time updates are taken care off when the
+	 * real inode is written.
+	 */
+	if (NInoAttr(ni) || ni->nr_extents == -1) {
+		NInoClearDirty(ni);
+		ntfs_debug("Done.");
+		return 0;
+	}
+
+	/* igrab prevents vi from being evicted while mrec_lock is hold. */
+	if (igrab(vi) != NULL)
+		need_iput = true;
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	/* Map, pin, and lock the mft record belonging to the inode. */
+	m = map_mft_record(ni);
+	if (IS_ERR(m)) {
+		mutex_unlock(&ni->mrec_lock);
+		err = PTR_ERR(m);
+		goto err_out;
+	}
+
+	if (NInoNonResident(ni) && NInoRunlistDirty(ni)) {
+		down_write(&ni->runlist.lock);
+		err = ntfs_attr_update_mapping_pairs(ni, 0);
+		if (!err)
+			NInoClearRunlistDirty(ni);
+		up_write(&ni->runlist.lock);
+	}
+
+	err = ntfs_inode_sync_standard_information(vi, m);
+	if (err)
+		goto unm_err_out;
+
+	/*
+	 * when being umounted and inodes are evicted, write_inode()
+	 * is called with all inodes being marked with I_FREEING.
+	 * then ntfs_inode_sync_filename() waits infinitly because
+	 * of ntfs_iget. This situation happens only where sync_filesysem()
+	 * from umount fails because of a disk unplug and etc.
+	 * the absent of SB_ACTIVE means umounting.
+	 */
+	if ((vi->i_sb->s_flags & SB_ACTIVE) && NInoTestClearFileNameDirty(ni))
+		ntfs_inode_sync_filename(ni);
+
+	/* Now the access times are updated, write the base mft record. */
+	if (NInoDirty(ni)) {
+		down_read(&mft_ni->runlist.lock);
+		err = ntfs_get_block_mft_record(mft_ni, ni);
+		up_read(&mft_ni->runlist.lock);
+		if (err)
+			goto unm_err_out;
+
+		err = write_mft_record(ni, m, sync);
+		if (err)
+			ntfs_error(vi->i_sb, "write_mft_record failed, err : %d\n", err);
+	}
+	unmap_mft_record(ni);
+
+	/* Map any unmapped extent mft records with LCNs. */
+	down_read(&mft_ni->runlist.lock);
+	mutex_lock(&ni->extent_lock);
+	if (ni->nr_extents > 0) {
+		int i;
+
+		for (i = 0; i < ni->nr_extents; i++) {
+			err = ntfs_get_block_mft_record(mft_ni,
+						   ni->ext.extent_ntfs_inos[i]);
+			if (err) {
+				mutex_unlock(&ni->extent_lock);
+				up_read(&mft_ni->runlist.lock);
+				mutex_unlock(&ni->mrec_lock);
+				goto err_out;
+			}
+		}
+	}
+	mutex_unlock(&ni->extent_lock);
+	up_read(&mft_ni->runlist.lock);
+
+	/* Write all attached extent mft records. */
+	mutex_lock(&ni->extent_lock);
+	if (ni->nr_extents > 0) {
+		struct ntfs_inode **extent_nis = ni->ext.extent_ntfs_inos;
+		int i;
+
+		ntfs_debug("Writing %i extent inodes.", ni->nr_extents);
+		for (i = 0; i < ni->nr_extents; i++) {
+			struct ntfs_inode *tni = extent_nis[i];
+
+			if (NInoDirty(tni)) {
+				struct mft_record *tm;
+				int ret;
+
+				mutex_lock(&tni->mrec_lock);
+				tm = map_mft_record(tni);
+				if (IS_ERR(tm)) {
+					mutex_unlock(&tni->mrec_lock);
+					if (!err || err == -ENOMEM)
+						err = PTR_ERR(tm);
+					continue;
+				}
+
+				ret = write_mft_record(tni, tm, sync);
+				unmap_mft_record(tni);
+				mutex_unlock(&tni->mrec_lock);
+
+				if (unlikely(ret)) {
+					if (!err || err == -ENOMEM)
+						err = ret;
+				}
+			}
+		}
+	}
+	mutex_unlock(&ni->extent_lock);
+	mutex_unlock(&ni->mrec_lock);
+
+	if (unlikely(err))
+		goto err_out;
+	if (need_iput)
+		iput(vi);
+	ntfs_debug("Done.");
+	return 0;
+unm_err_out:
+	unmap_mft_record(ni);
+	mutex_unlock(&ni->mrec_lock);
+err_out:
+	if (err == -ENOMEM)
+		mark_inode_dirty(vi);
+	else {
+		ntfs_error(vi->i_sb, "Failed (error %i):  Run chkdsk.", -err);
+		NVolSetErrors(ni->vol);
+	}
+	if (need_iput)
+		iput(vi);
+	return err;
+}
+
+/*
+ * ntfs_extent_inode_open - load an extent inode and attach it to its base
+ * @base_ni:	base ntfs inode
+ * @mref:	mft reference of the extent inode to load (in little endian)
+ *
+ * First check if the extent inode @mref is already attached to the base ntfs
+ * inode @base_ni, and if so, return a pointer to the attached extent inode.
+ *
+ * If the extent inode is not already attached to the base inode, allocate an
+ * ntfs_inode structure and initialize it for the given inode @mref. @mref
+ * specifies the inode number / mft record to read, including the sequence
+ * number, which can be 0 if no sequence number checking is to be performed.
+ *
+ * Then, allocate a buffer for the mft record, read the mft record from the
+ * volume @base_ni->vol, and attach it to the ntfs_inode structure (->mrec).
+ * The mft record is mst deprotected and sanity checked for validity and we
+ * abort if deprotection or checks fail.
+ *
+ * Finally attach the ntfs inode to its base inode @base_ni and return a
+ * pointer to the ntfs_inode structure on success or NULL on error, with errno
+ * set to the error code.
+ *
+ * Note, extent inodes are never closed directly. They are automatically
+ * disposed off by the closing of the base inode.
+ */
+static struct ntfs_inode *ntfs_extent_inode_open(struct ntfs_inode *base_ni,
+		const __le64 mref)
+{
+	u64 mft_no = MREF_LE(mref);
+	struct ntfs_inode *ni = NULL;
+	struct ntfs_inode **extent_nis;
+	int i;
+	struct mft_record *ni_mrec;
+	struct super_block *sb;
+
+	if (!base_ni)
+		return NULL;
+
+	sb = base_ni->vol->sb;
+	ntfs_debug("Opening extent inode %llu (base mft record %llu).\n",
+			mft_no, base_ni->mft_no);
+
+	/* Is the extent inode already open and attached to the base inode? */
+	if (base_ni->nr_extents > 0) {
+		extent_nis = base_ni->ext.extent_ntfs_inos;
+		for (i = 0; i < base_ni->nr_extents; i++) {
+			u16 seq_no;
+
+			ni = extent_nis[i];
+			if (mft_no != ni->mft_no)
+				continue;
+			ni_mrec = map_mft_record(ni);
+			if (IS_ERR(ni_mrec)) {
+				ntfs_error(sb, "failed to map mft record for %llu",
+						ni->mft_no);
+				goto out;
+			}
+			/* Verify the sequence number if given. */
+			seq_no = MSEQNO_LE(mref);
+			if (seq_no &&
+			    seq_no != le16_to_cpu(ni_mrec->sequence_number)) {
+				ntfs_error(sb, "Found stale extent mft reference mft=%llu",
+						ni->mft_no);
+				unmap_mft_record(ni);
+				goto out;
+			}
+			unmap_mft_record(ni);
+			goto out;
+		}
+	}
+	/* Wasn't there, we need to load the extent inode. */
+	ni = ntfs_new_extent_inode(base_ni->vol->sb, mft_no);
+	if (!ni)
+		goto out;
+
+	ni->seq_no = (u16)MSEQNO_LE(mref);
+	ni->nr_extents = -1;
+	ni->ext.base_ntfs_ino = base_ni;
+	/* Attach extent inode to base inode, reallocating memory if needed. */
+	if (!(base_ni->nr_extents & 3)) {
+		i = (base_ni->nr_extents + 4) * sizeof(struct ntfs_inode *);
+
+		extent_nis = kvzalloc(i, GFP_NOFS);
+		if (!extent_nis)
+			goto err_out;
+		if (base_ni->nr_extents) {
+			memcpy(extent_nis, base_ni->ext.extent_ntfs_inos,
+					i - 4 * sizeof(struct ntfs_inode *));
+			kvfree(base_ni->ext.extent_ntfs_inos);
+		}
+		base_ni->ext.extent_ntfs_inos = extent_nis;
+	}
+	base_ni->ext.extent_ntfs_inos[base_ni->nr_extents++] = ni;
+
+out:
+	ntfs_debug("\n");
+	return ni;
+err_out:
+	ntfs_destroy_ext_inode(ni);
+	ni = NULL;
+	goto out;
+}
+
+/*
+ * ntfs_inode_attach_all_extents - attach all extents for target inode
+ * @ni:		opened ntfs inode for which perform attach
+ *
+ * Return 0 on success and error.
+ */
+int ntfs_inode_attach_all_extents(struct ntfs_inode *ni)
+{
+	struct attr_list_entry *ale;
+	u64 prev_attached = 0;
+
+	if (!ni) {
+		ntfs_debug("Invalid arguments.\n");
+		return -EINVAL;
+	}
+
+	if (NInoAttr(ni))
+		ni = ni->ext.base_ntfs_ino;
+
+	ntfs_debug("Entering for inode 0x%llx.\n", ni->mft_no);
+
+	/* Inode haven't got attribute list, thus nothing to attach. */
+	if (!NInoAttrList(ni))
+		return 0;
+
+	if (!ni->attr_list) {
+		ntfs_debug("Corrupt in-memory struct.\n");
+		return -EINVAL;
+	}
+
+	/* Walk through attribute list and attach all extents. */
+	ale = (struct attr_list_entry *)ni->attr_list;
+	while ((u8 *)ale < ni->attr_list + ni->attr_list_size) {
+		if (ni->mft_no != MREF_LE(ale->mft_reference) &&
+				prev_attached != MREF_LE(ale->mft_reference)) {
+			if (!ntfs_extent_inode_open(ni, ale->mft_reference)) {
+				ntfs_debug("Couldn't attach extent inode.\n");
+				return -1;
+			}
+			prev_attached = MREF_LE(ale->mft_reference);
+		}
+		ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length));
+	}
+	return 0;
+}
+
+/*
+ * ntfs_inode_add_attrlist - add attribute list to inode and fill it
+ * @ni: opened ntfs inode to which add attribute list
+ *
+ * Return 0 on success or error.
+ */
+int ntfs_inode_add_attrlist(struct ntfs_inode *ni)
+{
+	int err;
+	struct ntfs_attr_search_ctx *ctx;
+	u8 *al = NULL, *aln;
+	int al_len = 0;
+	struct attr_list_entry *ale = NULL;
+	struct mft_record *ni_mrec;
+	u32 attr_al_len;
+
+	if (!ni)
+		return -EINVAL;
+
+	ntfs_debug("inode %llu\n", ni->mft_no);
+
+	if (NInoAttrList(ni) || ni->nr_extents) {
+		ntfs_error(ni->vol->sb, "Inode already has attribute list");
+		return -EEXIST;
+	}
+
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec))
+		return -EIO;
+
+	/* Form attribute list. */
+	ctx = ntfs_attr_get_search_ctx(ni, ni_mrec);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	/* Walk through all attributes. */
+	while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) {
+		int ale_size;
+
+		if (ctx->attr->type == AT_ATTRIBUTE_LIST) {
+			err = -EIO;
+			ntfs_error(ni->vol->sb, "Attribute list already present");
+			goto put_err_out;
+		}
+
+		ale_size = (sizeof(struct attr_list_entry) + sizeof(__le16) *
+				ctx->attr->name_length + 7) & ~7;
+		al_len += ale_size;
+
+		aln = kvrealloc(al, al_len, GFP_NOFS);
+		if (!aln) {
+			err = -ENOMEM;
+			ntfs_error(ni->vol->sb, "Failed to realloc %d bytes", al_len);
+			goto put_err_out;
+		}
+		ale = (struct attr_list_entry *)(aln + ((u8 *)ale - al));
+		al = aln;
+
+		memset(ale, 0, ale_size);
+
+		/* Add attribute to attribute list. */
+		ale->type = ctx->attr->type;
+		ale->length = cpu_to_le16((sizeof(struct attr_list_entry) +
+					sizeof(__le16) * ctx->attr->name_length + 7) & ~7);
+		ale->name_length = ctx->attr->name_length;
+		ale->name_offset = (u8 *)ale->name - (u8 *)ale;
+		if (ctx->attr->non_resident)
+			ale->lowest_vcn =
+				ctx->attr->data.non_resident.lowest_vcn;
+		else
+			ale->lowest_vcn = 0;
+		ale->mft_reference = MK_LE_MREF(ni->mft_no,
+				le16_to_cpu(ni_mrec->sequence_number));
+		ale->instance = ctx->attr->instance;
+		memcpy(ale->name, (u8 *)ctx->attr +
+				le16_to_cpu(ctx->attr->name_offset),
+				ctx->attr->name_length * sizeof(__le16));
+		ale = (struct attr_list_entry *)(al + al_len);
+	}
+
+	/* Check for real error occurred. */
+	if (err != -ENOENT) {
+		ntfs_error(ni->vol->sb, "%s: Attribute lookup failed, inode %llu",
+				__func__, ni->mft_no);
+		goto put_err_out;
+	}
+
+	/* Set in-memory attribute list. */
+	ni->attr_list = al;
+	ni->attr_list_size = al_len;
+	NInoSetAttrList(ni);
+
+	attr_al_len = offsetof(struct attr_record, data.resident.reserved) + 1 +
+		((al_len + 7) & ~7);
+	/* Free space if there is not enough it for $ATTRIBUTE_LIST. */
+	if (le32_to_cpu(ni_mrec->bytes_allocated) -
+			le32_to_cpu(ni_mrec->bytes_in_use) < attr_al_len) {
+		if (ntfs_inode_free_space(ni, (int)attr_al_len)) {
+			/* Failed to free space. */
+			err = -ENOSPC;
+			ntfs_error(ni->vol->sb, "Failed to free space for attrlist");
+			goto rollback;
+		}
+	}
+
+	/* Add $ATTRIBUTE_LIST to mft record. */
+	err = ntfs_resident_attr_record_add(ni, AT_ATTRIBUTE_LIST, AT_UNNAMED, 0,
+					    NULL, al_len, 0);
+	if (err < 0) {
+		ntfs_error(ni->vol->sb, "Couldn't add $ATTRIBUTE_LIST to MFT");
+		goto rollback;
+	}
+
+	err = ntfs_attrlist_update(ni);
+	if (err < 0)
+		goto remove_attrlist_record;
+
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(ni);
+	return 0;
+
+remove_attrlist_record:
+	/* Prevent ntfs_attr_recorm_rm from freeing attribute list. */
+	ni->attr_list = NULL;
+	NInoClearAttrList(ni);
+	/* Remove $ATTRIBUTE_LIST record. */
+	ntfs_attr_reinit_search_ctx(ctx);
+	if (!ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0,
+				CASE_SENSITIVE, 0, NULL, 0, ctx)) {
+		if (ntfs_attr_record_rm(ctx))
+			ntfs_error(ni->vol->sb, "Rollback failed to remove attrlist");
+	} else {
+		ntfs_error(ni->vol->sb, "Rollback failed to find attrlist");
+	}
+
+	/* Setup back in-memory runlist. */
+	ni->attr_list = al;
+	ni->attr_list_size = al_len;
+	NInoSetAttrList(ni);
+rollback:
+	/*
+	 * Scan attribute list for attributes that placed not in the base MFT
+	 * record and move them to it.
+	 */
+	ntfs_attr_reinit_search_ctx(ctx);
+	ale = (struct attr_list_entry *)al;
+	while ((u8 *)ale < al + al_len) {
+		if (MREF_LE(ale->mft_reference) != ni->mft_no) {
+			if (!ntfs_attr_lookup(ale->type, ale->name,
+						ale->name_length,
+						CASE_SENSITIVE,
+						le64_to_cpu(ale->lowest_vcn),
+						NULL, 0, ctx)) {
+				if (ntfs_attr_record_move_to(ctx, ni))
+					ntfs_error(ni->vol->sb,
+							"Rollback failed to move attribute");
+			} else {
+				ntfs_error(ni->vol->sb, "Rollback failed to find attr");
+			}
+			ntfs_attr_reinit_search_ctx(ctx);
+		}
+		ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length));
+	}
+
+	/* Remove in-memory attribute list. */
+	ni->attr_list = NULL;
+	ni->attr_list_size = 0;
+	NInoClearAttrList(ni);
+	NInoClearAttrListDirty(ni);
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+err_out:
+	kvfree(al);
+	unmap_mft_record(ni);
+	return err;
+}
+
+/*
+ * ntfs_inode_close - close an ntfs inode and free all associated memory
+ * @ni:		ntfs inode to close
+ *
+ * Make sure the ntfs inode @ni is clean.
+ *
+ * If the ntfs inode @ni is a base inode, close all associated extent inodes,
+ * then deallocate all memory attached to it, and finally free the ntfs inode
+ * structure itself.
+ *
+ * If it is an extent inode, we disconnect it from its base inode before we
+ * destroy it.
+ *
+ * It is OK to pass NULL to this function, it is just noop in this case.
+ *
+ * Return 0 on success or error.
+ */
+int ntfs_inode_close(struct ntfs_inode *ni)
+{
+	int err = -1;
+	struct ntfs_inode **tmp_nis;
+	struct ntfs_inode *base_ni;
+	s32 i;
+
+	if (!ni)
+		return 0;
+
+	ntfs_debug("Entering for inode %llu\n", ni->mft_no);
+
+	/* Is this a base inode with mapped extent inodes? */
+	/*
+	 * If the inode is an extent inode, disconnect it from the
+	 * base inode before destroying it.
+	 */
+	base_ni = ni->ext.base_ntfs_ino;
+	tmp_nis = base_ni->ext.extent_ntfs_inos;
+	if (!tmp_nis)
+		goto out;
+	for (i = 0; i < base_ni->nr_extents; ++i) {
+		if (tmp_nis[i] != ni)
+			continue;
+		/* Found it. Disconnect. */
+		memmove(tmp_nis + i, tmp_nis + i + 1,
+				(base_ni->nr_extents - i - 1) *
+				sizeof(struct ntfs_inode *));
+		/* Buffer should be for multiple of four extents. */
+		if ((--base_ni->nr_extents) & 3)
+			break;
+		/*
+		 * ElectricFence is unhappy with realloc(x,0) as free(x)
+		 * thus we explicitly separate these two cases.
+		 */
+		if (base_ni->nr_extents) {
+			/* Resize the memory buffer. */
+			tmp_nis = kvrealloc(tmp_nis, base_ni->nr_extents *
+					sizeof(struct ntfs_inode *), GFP_NOFS);
+			/* Ignore errors, they don't really matter. */
+			if (tmp_nis)
+				base_ni->ext.extent_ntfs_inos = tmp_nis;
+		} else if (tmp_nis) {
+			kvfree(tmp_nis);
+			base_ni->ext.extent_ntfs_inos = NULL;
+		}
+		break;
+	}
+
+out:
+	if (NInoDirty(ni))
+		ntfs_error(ni->vol->sb, "Releasing dirty inode %llu!\n",
+				ni->mft_no);
+	if (NInoAttrList(ni) && ni->attr_list)
+		kvfree(ni->attr_list);
+	ntfs_destroy_ext_inode(ni);
+	err = 0;
+	ntfs_debug("\n");
+	return err;
+}
+
+void ntfs_destroy_ext_inode(struct ntfs_inode *ni)
+{
+	ntfs_debug("Entering.");
+	if (ni == NULL)
+		return;
+
+	ntfs_attr_close(ni);
+
+	if (NInoDirty(ni))
+		ntfs_error(ni->vol->sb, "Releasing dirty ext inode %llu!\n",
+				ni->mft_no);
+	if (NInoAttrList(ni) && ni->attr_list)
+		kvfree(ni->attr_list);
+	kfree(ni->mrec);
+	kmem_cache_free(ntfs_inode_cache, ni);
+}
+
+static struct ntfs_inode *ntfs_inode_base(struct ntfs_inode *ni)
+{
+	if (ni->nr_extents == -1)
+		return ni->ext.base_ntfs_ino;
+	return ni;
+}
+
+static int ntfs_attr_position(__le32 type, struct ntfs_attr_search_ctx *ctx)
+{
+	int err;
+
+	err = ntfs_attr_lookup(type, NULL, 0, CASE_SENSITIVE, 0, NULL,
+				0, ctx);
+	if (err) {
+		__le32 atype;
+
+		if (err != -ENOENT)
+			return err;
+
+		atype = ctx->attr->type;
+		if (atype == AT_END)
+			return -ENOSPC;
+
+		/*
+		 * if ntfs_external_attr_lookup return -ENOENT, ctx->al_entry
+		 * could point to an attribute in an extent mft record, but
+		 * ctx->attr and ctx->ntfs_ino always points to an attibute in
+		 * a base mft record.
+		 */
+		if (ctx->al_entry &&
+		    MREF_LE(ctx->al_entry->mft_reference) != ctx->ntfs_ino->mft_no) {
+			ntfs_attr_reinit_search_ctx(ctx);
+			err = ntfs_attr_lookup(atype, NULL, 0, CASE_SENSITIVE, 0, NULL,
+					       0, ctx);
+			if (err)
+				return err;
+		}
+	}
+	return 0;
+}
+
+/*
+ * ntfs_inode_free_space - free space in the MFT record of inode
+ * @ni:		ntfs inode in which MFT record free space
+ * @size:	amount of space needed to free
+ *
+ * Return 0 on success or error.
+ */
+int ntfs_inode_free_space(struct ntfs_inode *ni, int size)
+{
+	struct ntfs_attr_search_ctx *ctx;
+	int freed, err;
+	struct mft_record *ni_mrec;
+	struct super_block *sb;
+
+	if (!ni || size < 0)
+		return -EINVAL;
+	ntfs_debug("Entering for inode %llu, size %d\n", ni->mft_no, size);
+
+	sb = ni->vol->sb;
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec))
+		return -EIO;
+
+	freed = (le32_to_cpu(ni_mrec->bytes_allocated) -
+			le32_to_cpu(ni_mrec->bytes_in_use));
+
+	unmap_mft_record(ni);
+
+	if (size <= freed)
+		return 0;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		ntfs_error(sb, "%s, Failed to get search context", __func__);
+		return -ENOMEM;
+	}
+
+	/*
+	 * Chkdsk complain if $STANDARD_INFORMATION is not in the base MFT
+	 * record.
+	 *
+	 * Also we can't move $ATTRIBUTE_LIST from base MFT_RECORD, so position
+	 * search context on first attribute after $STANDARD_INFORMATION and
+	 * $ATTRIBUTE_LIST.
+	 *
+	 * Why we reposition instead of simply skip this attributes during
+	 * enumeration? Because in case we have got only in-memory attribute
+	 * list ntfs_attr_lookup will fail when it will try to find
+	 * $ATTRIBUTE_LIST.
+	 */
+	err = ntfs_attr_position(AT_FILE_NAME, ctx);
+	if (err)
+		goto put_err_out;
+
+	while (1) {
+		int record_size;
+
+		/*
+		 * Check whether attribute is from different MFT record. If so,
+		 * find next, because we don't need such.
+		 */
+		while (ctx->ntfs_ino->mft_no != ni->mft_no) {
+retry:
+			err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, CASE_SENSITIVE,
+						0, NULL, 0, ctx);
+			if (err) {
+				if (err != -ENOENT)
+					ntfs_error(sb, "Attr lookup failed #2");
+				else if (ctx->attr->type == AT_END)
+					err = -ENOSPC;
+				else
+					err = 0;
+
+				if (err)
+					goto put_err_out;
+			}
+		}
+
+		if (ntfs_inode_base(ctx->ntfs_ino)->mft_no == FILE_MFT &&
+				ctx->attr->type == AT_DATA)
+			goto retry;
+
+		if (ctx->attr->type == AT_INDEX_ROOT)
+			goto retry;
+
+		record_size = le32_to_cpu(ctx->attr->length);
+
+		/* Move away attribute. */
+		err = ntfs_attr_record_move_away(ctx, 0);
+		if (err) {
+			ntfs_error(sb, "Failed to move out attribute #2");
+			break;
+		}
+		freed += record_size;
+
+		/* Check whether we done. */
+		if (size <= freed) {
+			ntfs_attr_put_search_ctx(ctx);
+			return 0;
+		}
+
+		/*
+		 * Reposition to first attribute after $STANDARD_INFORMATION and
+		 * $ATTRIBUTE_LIST (see comments upwards).
+		 */
+		ntfs_attr_reinit_search_ctx(ctx);
+		err = ntfs_attr_position(AT_FILE_NAME, ctx);
+		if (err)
+			break;
+	}
+put_err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	if (err == -ENOSPC)
+		ntfs_debug("No attributes left that can be moved out.\n");
+	return err;
+}
+
+s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf)
+{
+	struct address_space *mapping = vi->i_mapping;
+	struct folio *folio;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	s64 isize;
+	u32 attr_len, total = 0, offset;
+	pgoff_t index;
+	int err = 0;
+
+	WARN_ON(!NInoAttr(ni));
+	if (!count)
+		return 0;
+
+	mutex_lock(&ni->mrec_lock);
+	isize = i_size_read(vi);
+	if (pos > isize) {
+		mutex_unlock(&ni->mrec_lock);
+		return -EINVAL;
+	}
+	if (pos + count > isize)
+		count = isize - pos;
+
+	if (!NInoNonResident(ni)) {
+		struct ntfs_attr_search_ctx *ctx;
+		u8 *attr;
+
+		ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL);
+		if (!ctx) {
+			ntfs_error(vi->i_sb, "Failed to get attr search ctx");
+			err = -ENOMEM;
+			mutex_unlock(&ni->mrec_lock);
+			goto out;
+		}
+
+		err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+				       0, NULL, 0, ctx);
+		if (err) {
+			ntfs_error(vi->i_sb, "Failed to look up attr %#x", ni->type);
+			ntfs_attr_put_search_ctx(ctx);
+			mutex_unlock(&ni->mrec_lock);
+			goto out;
+		}
+
+		attr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset);
+		memcpy(buf, (u8 *)attr + pos, count);
+		ntfs_attr_put_search_ctx(ctx);
+		mutex_unlock(&ni->mrec_lock);
+		return count;
+	}
+	mutex_unlock(&ni->mrec_lock);
+
+	index = pos >> PAGE_SHIFT;
+	do {
+		/* Update @index and get the next folio. */
+		folio = read_mapping_folio(mapping, index, NULL);
+		if (IS_ERR(folio))
+			break;
+
+		offset = offset_in_folio(folio, pos);
+		attr_len = min_t(size_t, (size_t)count, folio_size(folio) - offset);
+
+		folio_lock(folio);
+		memcpy_from_folio(buf, folio, offset, attr_len);
+		folio_unlock(folio);
+		folio_put(folio);
+
+		total += attr_len;
+		buf += attr_len;
+		pos += attr_len;
+		count -= attr_len;
+		index++;
+	} while (count);
+out:
+	return err ? (s64)err : total;
+}
+
+static inline int ntfs_enlarge_attribute(struct inode *vi, s64 pos, s64 count,
+					 struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct super_block *sb = vi->i_sb;
+	int ret;
+
+	if (pos + count <= ni->initialized_size)
+		return 0;
+
+	if (NInoEncrypted(ni) && NInoNonResident(ni))
+		return -EACCES;
+
+	if (NInoCompressed(ni))
+		return -EOPNOTSUPP;
+
+	if (pos + count > ni->data_size) {
+		if (ntfs_attr_truncate(ni, pos + count)) {
+			ntfs_debug("Failed to truncate attribute");
+			return -1;
+		}
+
+		ntfs_attr_reinit_search_ctx(ctx);
+		ret = ntfs_attr_lookup(ni->type,
+				       ni->name, ni->name_len, CASE_SENSITIVE,
+				       0, NULL, 0, ctx);
+		if (ret) {
+			ntfs_error(sb, "Failed to look up attr %#x", ni->type);
+			return ret;
+		}
+	}
+
+	if (!NInoNonResident(ni)) {
+		if (likely(i_size_read(vi) < ni->data_size))
+			i_size_write(vi, ni->data_size);
+		return 0;
+	}
+
+	if (pos + count > ni->initialized_size) {
+		ctx->attr->data.non_resident.initialized_size = cpu_to_le64(pos + count);
+		mark_mft_record_dirty(ctx->ntfs_ino);
+		ni->initialized_size = pos + count;
+		if (i_size_read(vi) < ni->initialized_size)
+			i_size_write(vi, ni->initialized_size);
+	}
+	return 0;
+}
+
+static s64 __ntfs_inode_resident_attr_pwrite(struct inode *vi,
+					     s64 pos, s64 count, u8 *buf,
+					     struct ntfs_attr_search_ctx *ctx)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct folio *folio;
+	struct address_space *mapping = vi->i_mapping;
+	u8 *addr;
+	int err = 0;
+
+	WARN_ON(NInoNonResident(ni));
+	if (pos + count > PAGE_SIZE) {
+		ntfs_error(vi->i_sb, "Out of write into resident attr %#x", ni->type);
+		return -EINVAL;
+	}
+
+	/* Copy to mft record page */
+	addr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset);
+	memcpy(addr + pos, buf, count);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+
+	/* Keep the first page clean and uptodate */
+	folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS,
+				   mapping_gfp_mask(mapping));
+	if (IS_ERR(folio)) {
+		err = PTR_ERR(folio);
+		ntfs_error(vi->i_sb, "Failed to read a page 0 for attr %#x: %d",
+			   ni->type, err);
+		goto out;
+	}
+	if (!folio_test_uptodate(folio))
+		folio_fill_tail(folio, 0, addr,
+				le32_to_cpu(ctx->attr->data.resident.value_length));
+	else
+		memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, count);
+	folio_mark_uptodate(folio);
+	folio_unlock(folio);
+	folio_put(folio);
+out:
+	return err ? err : count;
+}
+
+static s64 __ntfs_inode_non_resident_attr_pwrite(struct inode *vi,
+						 s64 pos, s64 count, u8 *buf,
+						 struct ntfs_attr_search_ctx *ctx,
+						 bool sync)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct address_space *mapping = vi->i_mapping;
+	struct folio *folio;
+	pgoff_t index;
+	unsigned long offset, length;
+	size_t attr_len;
+	s64 ret = 0, written = 0;
+
+	WARN_ON(!NInoNonResident(ni));
+
+	index = pos >> PAGE_SHIFT;
+	while (count) {
+		if (count == PAGE_SIZE) {
+			folio = __filemap_get_folio(vi->i_mapping, index,
+					FGP_CREAT | FGP_LOCK,
+					mapping_gfp_mask(mapping));
+			if (IS_ERR(folio)) {
+				ret = -ENOMEM;
+				break;
+			}
+		} else {
+			folio = read_mapping_folio(mapping, index, NULL);
+			if (IS_ERR(folio)) {
+				ret = PTR_ERR(folio);
+				ntfs_error(vi->i_sb, "Failed to read a page %lu for attr %#x: %ld",
+						index, ni->type, PTR_ERR(folio));
+				break;
+			}
+
+			folio_lock(folio);
+		}
+
+		if (count == PAGE_SIZE) {
+			offset = 0;
+			attr_len = count;
+		} else {
+			offset = offset_in_folio(folio, pos);
+			attr_len = min_t(size_t, (size_t)count, folio_size(folio) - offset);
+		}
+		memcpy_to_folio(folio, offset, buf, attr_len);
+
+		if (sync) {
+			struct ntfs_volume *vol = ni->vol;
+			s64 lcn, lcn_count;
+			unsigned int lcn_folio_off = 0;
+			struct bio *bio;
+			u64 rl_length = 0;
+			s64 vcn;
+			struct runlist_element *rl;
+
+			lcn_count = max_t(s64, 1, ntfs_bytes_to_cluster(vol, attr_len));
+			vcn = ntfs_pidx_to_cluster(vol, folio->index);
+
+			do {
+				down_write(&ni->runlist.lock);
+				rl = ntfs_attr_vcn_to_rl(ni, vcn, &lcn);
+				if (IS_ERR(rl)) {
+					ret = PTR_ERR(rl);
+					up_write(&ni->runlist.lock);
+					goto err_unlock_folio;
+				}
+
+				rl_length = rl->length - (vcn - rl->vcn);
+				if (rl_length < lcn_count) {
+					lcn_count -= rl_length;
+				} else {
+					rl_length = lcn_count;
+					lcn_count = 0;
+				}
+				up_write(&ni->runlist.lock);
+
+				if (vol->cluster_size_bits > PAGE_SHIFT) {
+					lcn_folio_off = folio->index << PAGE_SHIFT;
+					lcn_folio_off &= vol->cluster_size_mask;
+				}
+
+				bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE,
+						GFP_NOIO);
+				bio->bi_iter.bi_sector =
+					ntfs_bytes_to_sector(vol,
+							ntfs_cluster_to_bytes(vol, lcn) +
+							lcn_folio_off);
+
+				length = min_t(unsigned long,
+					       ntfs_cluster_to_bytes(vol, rl_length),
+					       folio_size(folio));
+				if (!bio_add_folio(bio, folio, length, offset)) {
+					ret = -EIO;
+					bio_put(bio);
+					goto err_unlock_folio;
+				}
+
+				submit_bio_wait(bio);
+				bio_put(bio);
+				vcn += rl_length;
+				offset += length;
+			} while (lcn_count != 0);
+
+			folio_mark_uptodate(folio);
+		} else {
+			folio_mark_uptodate(folio);
+			folio_mark_dirty(folio);
+		}
+err_unlock_folio:
+		folio_unlock(folio);
+		folio_put(folio);
+
+		if (ret)
+			break;
+
+		written += attr_len;
+		buf += attr_len;
+		pos += attr_len;
+		count -= attr_len;
+		index++;
+
+		cond_resched();
+	}
+
+	return ret ? ret : written;
+}
+
+s64 ntfs_inode_attr_pwrite(struct inode *vi, s64 pos, s64 count, u8 *buf, bool sync)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_attr_search_ctx *ctx;
+	s64 ret;
+
+	WARN_ON(!NInoAttr(ni));
+
+	ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL);
+	if (!ctx) {
+		ntfs_error(vi->i_sb, "Failed to get attr search ctx");
+		return -ENOMEM;
+	}
+
+	ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE,
+			       0, NULL, 0, ctx);
+	if (ret) {
+		ntfs_attr_put_search_ctx(ctx);
+		ntfs_error(vi->i_sb, "Failed to look up attr %#x", ni->type);
+		return ret;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	ret = ntfs_enlarge_attribute(vi, pos, count, ctx);
+	mutex_unlock(&ni->mrec_lock);
+	if (ret)
+		goto out;
+
+	if (NInoNonResident(ni))
+		ret = __ntfs_inode_non_resident_attr_pwrite(vi, pos, count, buf, ctx, sync);
+	else
+		ret = __ntfs_inode_resident_attr_pwrite(vi, pos, count, buf, ctx);
+out:
+	ntfs_attr_put_search_ctx(ctx);
+	return ret;
+}
+
+struct folio *ntfs_get_locked_folio(struct address_space *mapping,
+		pgoff_t index, pgoff_t end_index, struct file_ra_state *ra)
+{
+	struct folio *folio;
+
+	folio = filemap_lock_folio(mapping, index);
+	if (IS_ERR(folio)) {
+		if (PTR_ERR(folio) != -ENOENT)
+			return folio;
+
+		page_cache_sync_readahead(mapping, ra, NULL, index,
+				end_index - index);
+		folio = read_mapping_folio(mapping, index, NULL);
+		if (!IS_ERR(folio))
+			folio_lock(folio);
+	}
+
+	return folio;
+}
diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h
new file mode 100644
index 0000000000000..67942b97fac6d
--- /dev/null
+++ b/fs/ntfs/inode.h
@@ -0,0 +1,358 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for inode structures NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2007 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_INODE_H
+#define _LINUX_NTFS_INODE_H
+
+#include "debug.h"
+
+enum ntfs_inode_mutex_lock_class {
+	NTFS_INODE_MUTEX_PARENT,
+	NTFS_INODE_MUTEX_NORMAL,
+	NTFS_INODE_MUTEX_NORMAL_CHILD,
+	NTFS_INODE_MUTEX_PARENT_2,
+	NTFS_INODE_MUTEX_NORMAL_2,
+	NTFS_EXTEND_MUTEX_PARENT,
+	NTFS_EA_MUTEX_NORMAL
+};
+
+/*
+ * The NTFS in-memory inode structure. It is just used as an extension to the
+ * fields already provided in the VFS inode.
+ * @size_lock: Lock serializing access to inode sizes.
+ * @state: NTFS specific flags describing this inode.
+ * @flags: Flags describing the file. (Copy from STANDARD_INFORMATION).
+ * @mft_no: Number of the mft record / inode.
+ * @seq_no: Sequence number of the mft record.
+ * @count: Inode reference count for book keeping.
+ * @vol: Pointer to the ntfs volume of this inode.
+ *
+ * If NInoAttr() is true, the below fields describe the attribute which
+ * this fake inode belongs to. The actual inode of this attribute is
+ * pointed to by base_ntfs_ino and nr_extents is always set to -1 (see
+ * below). For real inodes, we also set the type (AT_DATA for files and
+ * AT_INDEX_ALLOCATION for directories), with the name = NULL and
+ * name_len = 0 for files and name = I30 (global constant) and
+ * name_len = 4 for directories.
+ * @type: Attribute type of this fake inode.
+ * @name: Attribute name of this fake inode.
+ * @name_len: Attribute name length of this fake inode.
+ * @runlist: If state has the NI_NonResident bit set, the runlist of
+ *   the unnamed data attribute (if a file) or of the index allocation
+ *   attribute (directory) or of the attribute described by the fake inode
+ *   (if NInoAttr()). If runlist.rl is NULL, the runlist has not been read
+ *   in yet or has been unmapped. If NI_NonResident is clear, the attribute
+ *   is resident (file and fake inode) or there is no $I30 index allocation
+ *   attribute (small directory). In the latter case runlist.rl is always
+ *   NULL.
+ * @data_size: Copy from the attribute record.
+ * @initialized_size: Copy from the attribute record.
+ * @allocated_size: Copy from the attribute record.
+ * @i_crtime: File Creation time.
+ * @mrec: MFT record
+ * @mrec_lock: Lock for serializing access to the mft record belonging to
+ *   this inode.
+ * @folio: The folio containing the mft record of the inode.
+ * @folio_ofs: Offset into the folio at which the mft record begins.
+ * @mft_lcn: Number containing the mft record.
+ * @mft_lcn_count: Number of clusters per mft record.
+ *
+ * Attribute list support (only for use by the attribute lookup
+ * functions). Setup during read_inode for all inodes with attribute
+ * lists. Only valid if NI_AttrList is set in state.
+ * @attr_list_size: Length of attribute list value in bytes.
+ * @attr_list: Attribute list value itself.
+ *
+ * It is a directory, $MFT, or an index inode.
+ * @block_size: Size of an index block.
+ * @vcn_size: Size of a vcn in this index.
+ * @collation_rule: The collation rule for the index.
+ * @block_size_bits: Log2 of the above.
+ * @vcn_size_bits: Log2 of the above.
+ *
+ * It is a compressed/sparse file/attribute inode.
+ * @size: Copy of compressed_size from $DATA.
+ * @block_size: Size of a compression block (cb).
+ * @block_size_bits: Log2 of the size of a cb.
+ * @block_clusters: Number of clusters per cb.
+ * @extent_lock: Lock for accessing/modifying the below.
+ * @nr_extents: For a base mft record, the number of attached extent inodes
+ *   (0 if none), for extent records and for fake inodes describing an
+ *   attribute this is -1.
+ *
+ * This union is only used if nr_extents != 0.
+ * @extent_ntfs_inos: For nr_extents > 0, array of the ntfs inodes of
+ *   the extent mft records belonging to this base inode which have been
+ *   loaded.
+ * @base_ntfs_ino: For nr_extents == -1, the ntfs inode of the base mft
+ *   record. For fake inodes, the real (base) inode to which the attribute
+ *   belongs.
+ * @i_dealloc_clusters: delayed allocated clusters.
+ * @target: symlink buffer.
+ */
+struct ntfs_inode {
+	rwlock_t size_lock;
+	unsigned long state;
+	__le32 flags;
+	u64 mft_no;
+	u16 seq_no;
+	atomic_t count;
+	struct ntfs_volume *vol;
+	__le32 type;
+	__le16 *name;
+	u32 name_len;
+	struct runlist runlist;
+	s64 data_size;
+	s64 initialized_size;
+	s64 allocated_size;
+	struct timespec64 i_crtime;
+	void *mrec;
+	struct mutex mrec_lock;
+	struct folio *folio;
+	int folio_ofs;
+	s64 mft_lcn[2];
+	unsigned int mft_lcn_count;
+	u32 attr_list_size;
+	u8 *attr_list;
+	union {
+		struct {
+			u32 block_size;
+			u32 vcn_size;
+			__le32 collation_rule;
+			u8 block_size_bits;
+			u8 vcn_size_bits;
+		} index;
+		struct {
+			s64 size;
+			u32 block_size;
+			u8 block_size_bits;
+			u8 block_clusters;
+		} compressed;
+	} itype;
+	struct mutex extent_lock;
+	s32 nr_extents;
+	union {
+		struct ntfs_inode **extent_ntfs_inos;
+		struct ntfs_inode *base_ntfs_ino;
+	} ext;
+	unsigned int i_dealloc_clusters;
+	char *target;
+};
+
+/*
+ * Defined bits for the state field in the ntfs_inode structure.
+ * (f) = files only, (d) = directories only, (a) = attributes/fake inodes only
+ *
+ * NI_Dirty			Mft record needs to be written to disk.
+ * NI_AttrListDirty		Mft record contains an attribute list.
+ * NI_AttrList			Mft record contains an attribute list.
+ * NI_AttrListNonResident	Attribute list is non-resident. Implies
+ *				NI_AttrList is set.
+ * NI_Attr			1: Fake inode for attribute i/o.
+ *				0: Real inode or extent inode.
+ * NI_MstProtected		Attribute is protected by MST fixups.
+ * NI_NonResident		Unnamed data attr is non-resident (f)
+ *				Attribute is non-resident (a).
+ * NI_IndexAllocPresent		$I30 index alloc attr is present (d).
+ * NI_Compressed		Unnamed data attr is compressed (f).
+ *				Create compressed files by default (d).
+ *				Attribute is compressed (a).
+ * NI_Encrypted			Unnamed data attr is encrypted (f).
+ *				Create encrypted files by default (d).
+ *				Attribute is encrypted (a).
+ * NI_Sparse			Unnamed data attr is sparse (f).
+ *				Create sparse files by default (d).
+ *				Attribute is sparse (a).
+ * NI_SparseDisabled		May not create sparse regions.
+ * NI_FullyMapped		Runlist is fully mapped.
+ * NI_FileNameDirty		FILE_NAME attributes need to be updated.
+ * NI_BeingDeleted		ntfs inode is being delated.
+ * NI_BeingCreated		ntfs inode is being created.
+ * NI_HasEA			ntfs inode has EA attribute.
+ * NI_RunlistDirty		runlist need to be updated.
+ */
+enum {
+	NI_Dirty,
+	NI_AttrListDirty,
+	NI_AttrList,
+	NI_AttrListNonResident,
+	NI_Attr,
+	NI_MstProtected,
+	NI_NonResident,
+	NI_IndexAllocPresent,
+	NI_Compressed,
+	NI_Encrypted,
+	NI_Sparse,
+	NI_SparseDisabled,
+	NI_FullyMapped,
+	NI_FileNameDirty,
+	NI_BeingDeleted,
+	NI_BeingCreated,
+	NI_HasEA,
+	NI_RunlistDirty,
+};
+
+/*
+ * NOTE: We should be adding dirty mft records to a list somewhere and they
+ * should be independent of the (ntfs/vfs) inode structure so that an inode can
+ * be removed but the record can be left dirty for syncing later.
+ */
+
+/*
+ * Macro tricks to expand the NInoFoo(), NInoSetFoo(), and NInoClearFoo()
+ * functions.
+ */
+#define NINO_FNS(flag)						\
+static inline int NIno##flag(struct ntfs_inode *ni)		\
+{								\
+	return test_bit(NI_##flag, &(ni)->state);		\
+}								\
+static inline void NInoSet##flag(struct ntfs_inode *ni)		\
+{								\
+	set_bit(NI_##flag, &(ni)->state);			\
+}								\
+static inline void NInoClear##flag(struct ntfs_inode *ni)	\
+{								\
+	clear_bit(NI_##flag, &(ni)->state);			\
+}
+
+/*
+ * As above for NInoTestSetFoo() and NInoTestClearFoo().
+ */
+#define TAS_NINO_FNS(flag)						\
+static inline int NInoTestSet##flag(struct ntfs_inode *ni)		\
+{									\
+	return test_and_set_bit(NI_##flag, &(ni)->state);		\
+}									\
+static inline int NInoTestClear##flag(struct ntfs_inode *ni)		\
+{									\
+	return test_and_clear_bit(NI_##flag, &(ni)->state);		\
+}
+
+/* Emit the ntfs inode bitops functions. */
+NINO_FNS(Dirty)
+TAS_NINO_FNS(Dirty)
+NINO_FNS(AttrList)
+NINO_FNS(AttrListDirty)
+NINO_FNS(AttrListNonResident)
+NINO_FNS(Attr)
+NINO_FNS(MstProtected)
+NINO_FNS(NonResident)
+NINO_FNS(IndexAllocPresent)
+NINO_FNS(Compressed)
+NINO_FNS(Encrypted)
+NINO_FNS(Sparse)
+NINO_FNS(SparseDisabled)
+NINO_FNS(FullyMapped)
+NINO_FNS(FileNameDirty)
+TAS_NINO_FNS(FileNameDirty)
+NINO_FNS(BeingDeleted)
+NINO_FNS(HasEA)
+NINO_FNS(RunlistDirty)
+
+/*
+ * The full structure containing a ntfs_inode and a vfs struct inode. Used for
+ * all real and fake inodes but not for extent inodes which lack the vfs struct
+ * inode.
+ */
+struct big_ntfs_inode {
+	struct ntfs_inode ntfs_inode;
+	struct inode vfs_inode;		/* The vfs inode structure. */
+};
+
+/*
+ * NTFS_I - return the ntfs inode given a vfs inode
+ * @inode:	VFS inode
+ *
+ * NTFS_I() returns the ntfs inode associated with the VFS @inode.
+ */
+static inline struct ntfs_inode *NTFS_I(struct inode *inode)
+{
+	return &container_of(inode, struct big_ntfs_inode, vfs_inode)->ntfs_inode;
+}
+
+static inline struct inode *VFS_I(struct ntfs_inode *ni)
+{
+	return &container_of(ni, struct big_ntfs_inode, ntfs_inode)->vfs_inode;
+}
+
+/*
+ * ntfs_attr - ntfs in memory attribute structure
+ *
+ * This structure exists only to provide a small structure for the
+ * ntfs_{attr_}iget()/ntfs_test_inode()/ntfs_init_locked_inode() mechanism.
+ *
+ * NOTE: Elements are ordered by size to make the structure as compact as
+ * possible on all architectures.
+ */
+struct ntfs_attr {
+	u64 mft_no;
+	__le16 *name;
+	u32 name_len;
+	__le32 type;
+	unsigned long state;
+};
+
+int ntfs_test_inode(struct inode *vi, void *data);
+struct inode *ntfs_iget(struct super_block *sb, u64 mft_no);
+struct inode *ntfs_attr_iget(struct inode *base_vi, __le32 type,
+		__le16 *name, u32 name_len);
+struct inode *ntfs_index_iget(struct inode *base_vi, __le16 *name,
+		u32 name_len);
+struct inode *ntfs_alloc_big_inode(struct super_block *sb);
+void ntfs_free_big_inode(struct inode *inode);
+int ntfs_drop_big_inode(struct inode *inode);
+void ntfs_evict_big_inode(struct inode *vi);
+void __ntfs_init_inode(struct super_block *sb, struct ntfs_inode *ni);
+
+static inline void ntfs_init_big_inode(struct inode *vi)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+
+	ntfs_debug("Entering.");
+	__ntfs_init_inode(vi->i_sb, ni);
+	ni->mft_no = vi->i_ino;
+}
+
+struct ntfs_inode *ntfs_new_extent_inode(struct super_block *sb, u64 mft_no);
+void ntfs_clear_extent_inode(struct ntfs_inode *ni);
+int ntfs_read_inode_mount(struct inode *vi);
+int ntfs_show_options(struct seq_file *sf, struct dentry *root);
+int ntfs_truncate_vfs(struct inode *vi, loff_t new_size, loff_t i_size);
+
+int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+		 struct iattr *attr);
+int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+		struct kstat *stat, unsigned int request_mask,
+		unsigned int query_flags);
+
+int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni);
+int __ntfs_write_inode(struct inode *vi, int sync);
+int ntfs_inode_attach_all_extents(struct ntfs_inode *ni);
+int ntfs_inode_add_attrlist(struct ntfs_inode *ni);
+void ntfs_destroy_ext_inode(struct ntfs_inode *ni);
+int ntfs_inode_free_space(struct ntfs_inode *ni, int size);
+s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf);
+s64 ntfs_inode_attr_pwrite(struct inode *vi, s64 pos, s64 count, u8 *buf,
+		bool sync);
+int ntfs_inode_close(struct ntfs_inode *ni);
+
+static inline void ntfs_commit_inode(struct inode *vi)
+{
+	__ntfs_write_inode(vi, 1);
+}
+
+int ntfs_inode_sync_filename(struct ntfs_inode *ni);
+int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset,
+		const loff_t new_size, bool bsync);
+void ntfs_set_vfs_operations(struct inode *inode, mode_t mode, dev_t dev);
+struct folio *ntfs_get_locked_folio(struct address_space *mapping,
+		pgoff_t index, pgoff_t end_index, struct file_ra_state *ra);
+
+#endif /* _LINUX_NTFS_INODE_H */
diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c
new file mode 100644
index 0000000000000..621645fbbf2ed
--- /dev/null
+++ b/fs/ntfs/iomap.c
@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * iomap callack functions
+ *
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/writeback.h>
+
+#include "attrib.h"
+#include "mft.h"
+#include "ntfs.h"
+#include "iomap.h"
+
+static void ntfs_iomap_put_folio_non_resident(struct inode *inode, loff_t pos,
+					      unsigned int len, struct folio *folio)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	unsigned long sector_size = 1UL << inode->i_blkbits;
+	loff_t start_down, end_up, init;
+
+	start_down = round_down(pos, sector_size);
+	end_up = (pos + len - 1) | (sector_size - 1);
+	init = ni->initialized_size;
+
+	if (init >= start_down && init <= end_up) {
+		if (init < pos) {
+			loff_t offset = offset_in_folio(folio, pos + len);
+
+			if (offset == 0)
+				offset = folio_size(folio);
+			folio_zero_segments(folio,
+					    offset_in_folio(folio, init),
+					    offset_in_folio(folio, pos),
+					    offset,
+					    folio_size(folio));
+
+		} else  {
+			loff_t offset = max_t(loff_t, pos + len, init);
+
+			offset = offset_in_folio(folio, offset);
+			if (offset == 0)
+				offset = folio_size(folio);
+			folio_zero_segment(folio,
+					   offset,
+					   folio_size(folio));
+		}
+	} else if (init <= pos) {
+		loff_t offset = 0, offset2 = offset_in_folio(folio, pos + len);
+
+		if ((init >> folio_shift(folio)) == (pos >> folio_shift(folio)))
+			offset = offset_in_folio(folio, init);
+		if (offset2 == 0)
+			offset2 = folio_size(folio);
+		folio_zero_segments(folio,
+				    offset,
+				    offset_in_folio(folio, pos),
+				    offset2,
+				    folio_size(folio));
+	}
+	folio_unlock(folio);
+	folio_put(folio);
+}
+
+/*
+ * iomap_zero_range is called for an area beyond the initialized size,
+ * garbage values can be read, so zeroing out is needed.
+ */
+static void ntfs_iomap_put_folio(struct inode *inode, loff_t pos,
+		unsigned int len, struct folio *folio)
+{
+	if (NInoNonResident(NTFS_I(inode)))
+		return ntfs_iomap_put_folio_non_resident(inode, pos,
+							 len, folio);
+	folio_unlock(folio);
+	folio_put(folio);
+}
+
+const struct iomap_write_ops ntfs_iomap_folio_ops = {
+	.put_folio = ntfs_iomap_put_folio,
+};
+
+static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, loff_t length,
+		unsigned int flags, struct iomap *iomap)
+{
+	struct ntfs_inode *base_ni, *ni = NTFS_I(inode);
+	struct ntfs_attr_search_ctx *ctx;
+	loff_t i_size;
+	u32 attr_len;
+	int err = 0;
+	char *kattr;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	ctx = ntfs_attr_get_search_ctx(base_ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err))
+		goto out;
+
+	attr_len = le32_to_cpu(ctx->attr->data.resident.value_length);
+	if (unlikely(attr_len > ni->initialized_size))
+		attr_len = ni->initialized_size;
+	i_size = i_size_read(inode);
+
+	if (unlikely(attr_len > i_size)) {
+		/* Race with shrinking truncate. */
+		attr_len = i_size;
+	}
+
+	if (offset >= attr_len) {
+		if (flags & IOMAP_REPORT)
+			err = -ENOENT;
+		else {
+			iomap->type = IOMAP_HOLE;
+			iomap->offset = offset;
+			iomap->length = length;
+		}
+		goto out;
+	}
+
+	kattr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset);
+
+	iomap->inline_data = kmemdup(kattr, attr_len, GFP_KERNEL);
+	if (!iomap->inline_data) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	iomap->type = IOMAP_INLINE;
+	iomap->offset = 0;
+	iomap->length = attr_len;
+
+out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+
+	return err;
+}
+
+/*
+ * ntfs_read_iomap_begin_non_resident - map non-resident NTFS file data
+ * @inode:		inode to map
+ * @offset:		file offset to map
+ * @length:		length of mapping
+ * @flags:		IOMAP flags
+ * @iomap:		iomap structure to fill
+ * @need_unwritten:	true if UNWRITTEN extent type is needed
+ *
+ * Map a range of a non-resident NTFS file to an iomap extent.
+ *
+ * NTFS UNWRITTEN extent handling:
+ * ================================
+ * The concept of an unwritten extent in NTFS is slightly different from
+ * that of other filesystems. NTFS conceptually manages only a single
+ * continuous unwritten region, which is strictly defined based on
+ * initialized_size.
+ *
+ * File offset layout:
+ *   0                        initialized_size                   i_size(EOF)
+ *   |----------#0----------|----------#1----------|----------#2----------|
+ *   | Actual data          | Pre-allocated        | Pre-allocated        |
+ *   | (user written)       | (within initialized) | (initialized ~ EOF)  |
+ *   |----------------------|----------------------|----------------------|
+ *   MAPPED                 MAPPED                 UNWRITTEN (conditionally)
+ *
+ * Region #0: User-written data, initialized and valid.
+ * Region #1: Pre-allocated within initialized_size, must be zero-initialized
+ *            by the filesystem before exposure to userspace.
+ * Region #2: Pre-allocated beyond initialized_size, does not need initialization.
+ *
+ * The @need_unwritten parameter controls whether region #2 is mapped as
+ * IOMAP_UNWRITTEN or IOMAP_MAPPED:
+ * - For seek operations (SEEK_DATA/SEEK_HOLE): IOMAP_MAPPED is needed to
+ *   prevent iomap_seek_data from incorrectly interpreting pre-allocated
+ *   space as a hole. Since NTFS does not support multiple unwritten extents,
+ *   all pre-allocated regions should be treated as data, not holes.
+ * - For zero_range operations: IOMAP_MAPPED is needed to be zeroed out.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static int ntfs_read_iomap_begin_non_resident(struct inode *inode, loff_t offset,
+		loff_t length, unsigned int flags, struct iomap *iomap,
+		bool need_unwritten)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	s64 vcn;
+	s64 lcn;
+	struct runlist_element *rl;
+	struct ntfs_volume *vol = ni->vol;
+	loff_t vcn_ofs;
+	loff_t rl_length;
+
+	vcn = ntfs_bytes_to_cluster(vol, offset);
+	vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset);
+
+	down_write(&ni->runlist.lock);
+	rl = ntfs_attr_vcn_to_rl(ni, vcn, &lcn);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		return PTR_ERR(rl);
+	}
+
+	if (flags & IOMAP_REPORT) {
+		if (lcn < LCN_HOLE) {
+			up_write(&ni->runlist.lock);
+			return -ENOENT;
+		}
+	} else if (lcn < LCN_ENOENT) {
+		up_write(&ni->runlist.lock);
+		return -EINVAL;
+	}
+
+	iomap->bdev = inode->i_sb->s_bdev;
+	iomap->offset = offset;
+
+	if (lcn <= LCN_DELALLOC) {
+		if (lcn == LCN_DELALLOC)
+			iomap->type = IOMAP_DELALLOC;
+		else
+			iomap->type = IOMAP_HOLE;
+		iomap->addr = IOMAP_NULL_ADDR;
+	} else {
+		if (need_unwritten && offset >= ni->initialized_size)
+			iomap->type = IOMAP_UNWRITTEN;
+		else
+			iomap->type = IOMAP_MAPPED;
+		iomap->addr = ntfs_cluster_to_bytes(vol, lcn) + vcn_ofs;
+	}
+
+	rl_length = ntfs_cluster_to_bytes(vol, rl->length - (vcn - rl->vcn));
+
+	if (rl_length == 0 && rl->lcn > LCN_DELALLOC) {
+		ntfs_error(inode->i_sb,
+				"runlist(vcn : %lld, length : %lld, lcn : %lld) is corrupted\n",
+				rl->vcn, rl->length, rl->lcn);
+		up_write(&ni->runlist.lock);
+		return -EIO;
+	}
+
+	if (rl_length && length > rl_length - vcn_ofs)
+		iomap->length = rl_length - vcn_ofs;
+	else
+		iomap->length = length;
+	up_write(&ni->runlist.lock);
+
+	if (!(flags & IOMAP_ZERO) &&
+			iomap->type == IOMAP_MAPPED &&
+			iomap->offset < ni->initialized_size &&
+			iomap->offset + iomap->length > ni->initialized_size) {
+		iomap->length = round_up(ni->initialized_size, 1 << inode->i_blkbits) -
+			iomap->offset;
+	}
+	iomap->flags |= IOMAP_F_MERGED;
+
+	return 0;
+}
+
+static int __ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+		unsigned int flags, struct iomap *iomap, struct iomap *srcmap,
+		bool need_unwritten)
+{
+	if (NInoNonResident(NTFS_I(inode)))
+		return ntfs_read_iomap_begin_non_resident(inode, offset, length,
+				flags, iomap, need_unwritten);
+	return ntfs_read_iomap_begin_resident(inode, offset, length,
+					     flags, iomap);
+}
+
+static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+		unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
+{
+	return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap,
+			srcmap, true);
+}
+
+static int ntfs_read_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+		ssize_t written, unsigned int flags, struct iomap *iomap)
+{
+	if (iomap->type == IOMAP_INLINE)
+		kfree(iomap->inline_data);
+
+	return written;
+}
+
+const struct iomap_ops ntfs_read_iomap_ops = {
+	.iomap_begin = ntfs_read_iomap_begin,
+	.iomap_end = ntfs_read_iomap_end,
+};
+
+/*
+ * Check that the cached iomap still matches the NTFS runlist before
+ * iomap_zero_range() is called. if the runlist changes while iomap is
+ * iterating a cached iomap, iomap_zero_range() may overwrite folios
+ * that have been already written with valid data.
+ */
+static bool ntfs_iomap_valid(struct inode *inode, const struct iomap *iomap)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct runlist_element *rl;
+	s64 vcn, lcn;
+
+	if (!NInoNonResident(ni))
+		return false;
+
+	vcn = iomap->offset >> ni->vol->cluster_size_bits;
+
+	down_read(&ni->runlist.lock);
+	rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn);
+	if (IS_ERR(rl)) {
+		up_read(&ni->runlist.lock);
+		return false;
+	}
+	lcn = ntfs_rl_vcn_to_lcn(rl, vcn);
+	up_read(&ni->runlist.lock);
+	return lcn == LCN_DELALLOC;
+}
+
+static const struct iomap_write_ops ntfs_zero_iomap_folio_ops = {
+	.put_folio = ntfs_iomap_put_folio,
+	.iomap_valid = ntfs_iomap_valid,
+};
+
+static int ntfs_seek_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+		unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
+{
+	return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap,
+			srcmap, false);
+}
+
+static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+		ssize_t written, unsigned int flags, struct iomap *iomap)
+{
+	if ((flags & IOMAP_ZERO) && (iomap->flags & IOMAP_F_STALE))
+		return -EPERM;
+	return written;
+}
+
+static const struct iomap_ops ntfs_zero_read_iomap_ops = {
+	.iomap_begin = ntfs_seek_iomap_begin,
+	.iomap_end = ntfs_zero_read_iomap_end,
+};
+
+const struct iomap_ops ntfs_seek_iomap_ops = {
+	.iomap_begin = ntfs_seek_iomap_begin,
+	.iomap_end = ntfs_read_iomap_end,
+};
+
+int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length)
+{
+	if ((offset | length) & (SECTOR_SIZE - 1))
+		return -EINVAL;
+
+	return  blkdev_issue_zeroout(inode->i_sb->s_bdev,
+				     offset >> SECTOR_SHIFT,
+				     length >> SECTOR_SHIFT,
+				     GFP_NOFS,
+				     BLKDEV_ZERO_NOUNMAP);
+}
+
+static int ntfs_zero_range(struct inode *inode, loff_t offset, loff_t length)
+{
+	return iomap_zero_range(inode,
+				offset, length,
+				NULL,
+				&ntfs_zero_read_iomap_ops,
+				&ntfs_zero_iomap_folio_ops,
+				NULL);
+}
+
+static int ntfs_write_simple_iomap_begin_non_resident(struct inode *inode, loff_t offset,
+						      loff_t length, struct iomap *iomap)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct ntfs_volume *vol = ni->vol;
+	loff_t vcn_ofs, rl_length;
+	struct runlist_element *rl, *rlc;
+	bool is_retry = false;
+	int err;
+	s64 vcn, lcn;
+	s64 max_clu_count =
+		ntfs_bytes_to_cluster(vol, round_up(length, vol->cluster_size));
+
+	vcn = ntfs_bytes_to_cluster(vol, offset);
+	vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset);
+
+	down_read(&ni->runlist.lock);
+	rl = ni->runlist.rl;
+	if (!rl) {
+		up_read(&ni->runlist.lock);
+		err = ntfs_map_runlist(ni, vcn);
+		if (err) {
+			mutex_unlock(&ni->mrec_lock);
+			return -ENOENT;
+		}
+		down_read(&ni->runlist.lock);
+		rl = ni->runlist.rl;
+	}
+	up_read(&ni->runlist.lock);
+
+	down_write(&ni->runlist.lock);
+remap_rl:
+	/* Seek to element containing target vcn. */
+	rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn);
+	if (IS_ERR(rl)) {
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+		return -EIO;
+	}
+	lcn = ntfs_rl_vcn_to_lcn(rl, vcn);
+
+	if (lcn <= LCN_RL_NOT_MAPPED && is_retry == false) {
+		is_retry = true;
+		if (!ntfs_map_runlist_nolock(ni, vcn, NULL)) {
+			rl = ni->runlist.rl;
+			goto remap_rl;
+		}
+	}
+
+	max_clu_count = min(max_clu_count, rl->length - (vcn - rl->vcn));
+	if (max_clu_count == 0) {
+		ntfs_error(inode->i_sb,
+				"runlist(vcn : %lld, length : %lld) is corrupted\n",
+				rl->vcn, rl->length);
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+		return -EIO;
+	}
+
+	iomap->bdev = inode->i_sb->s_bdev;
+	iomap->offset = offset;
+
+	if (lcn <= LCN_DELALLOC) {
+		if (lcn < LCN_DELALLOC) {
+			max_clu_count =
+				ntfs_available_clusters_count(vol, max_clu_count);
+			if (max_clu_count < 0) {
+				err = max_clu_count;
+				up_write(&ni->runlist.lock);
+				mutex_unlock(&ni->mrec_lock);
+				return err;
+			}
+		}
+
+		iomap->type = IOMAP_DELALLOC;
+		iomap->addr = IOMAP_NULL_ADDR;
+
+		if (lcn <= LCN_HOLE) {
+			size_t new_rl_count;
+
+			rlc = kmalloc(sizeof(struct runlist_element) * 2,
+					GFP_NOFS);
+			if (!rlc) {
+				up_write(&ni->runlist.lock);
+				mutex_unlock(&ni->mrec_lock);
+				return -ENOMEM;
+			}
+
+			rlc->vcn = vcn;
+			rlc->lcn = LCN_DELALLOC;
+			rlc->length = max_clu_count;
+
+			rlc[1].vcn = vcn + max_clu_count;
+			rlc[1].lcn = LCN_RL_NOT_MAPPED;
+			rlc[1].length = 0;
+
+			rl = ntfs_runlists_merge(&ni->runlist, rlc, 0,
+					&new_rl_count);
+			if (IS_ERR(rl)) {
+				ntfs_error(vol->sb, "Failed to merge runlists");
+				up_write(&ni->runlist.lock);
+				mutex_unlock(&ni->mrec_lock);
+				kvfree(rlc);
+				return PTR_ERR(rl);
+			}
+
+			ni->runlist.rl = rl;
+			ni->runlist.count = new_rl_count;
+			ni->i_dealloc_clusters += max_clu_count;
+		}
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+
+		if (lcn < LCN_DELALLOC)
+			ntfs_hold_dirty_clusters(vol, max_clu_count);
+
+		rl_length = ntfs_cluster_to_bytes(vol, max_clu_count);
+		if (length > rl_length - vcn_ofs)
+			iomap->length = rl_length - vcn_ofs;
+		else
+			iomap->length = length;
+
+		iomap->flags = IOMAP_F_NEW;
+		if (lcn <= LCN_HOLE) {
+			loff_t end = offset + length;
+
+			if (vcn_ofs || ((vol->cluster_size > iomap->length) &&
+					end < ni->initialized_size)) {
+				loff_t z_start, z_end;
+
+				z_start = vcn << vol->cluster_size_bits;
+				z_end = min_t(loff_t, z_start + vol->cluster_size,
+					      i_size_read(inode));
+				if (z_end > z_start)
+					err = ntfs_zero_range(inode,
+							      z_start,
+							      z_end - z_start);
+			}
+			if ((!err || err == -EPERM) &&
+			    max_clu_count > 1 &&
+			    (iomap->length & vol->cluster_size_mask &&
+			     end < ni->initialized_size)) {
+				loff_t z_start, z_end;
+
+				z_start = (vcn + max_clu_count - 1) <<
+					vol->cluster_size_bits;
+				z_end = min_t(loff_t, z_start + vol->cluster_size,
+					      i_size_read(inode));
+				if (z_end > z_start)
+					err = ntfs_zero_range(inode,
+							      z_start,
+							      z_end - z_start);
+			}
+
+			if (err == -EPERM)
+				err = 0;
+			if (err) {
+				ntfs_release_dirty_clusters(vol, max_clu_count);
+				return err;
+			}
+		}
+	} else {
+		up_write(&ni->runlist.lock);
+		mutex_unlock(&ni->mrec_lock);
+
+		iomap->type = IOMAP_MAPPED;
+		iomap->addr = ntfs_cluster_to_bytes(vol, lcn) + vcn_ofs;
+
+		rl_length = ntfs_cluster_to_bytes(vol, max_clu_count);
+		if (length > rl_length - vcn_ofs)
+			iomap->length = rl_length - vcn_ofs;
+		else
+			iomap->length = length;
+	}
+
+	return 0;
+}
+
+#define NTFS_IOMAP_FLAGS_BEGIN		BIT(1)
+#define NTFS_IOMAP_FLAGS_DIO		BIT(2)
+#define	NTFS_IOMAP_FLAGS_MKWRITE	BIT(3)
+#define	NTFS_IOMAP_FLAGS_WRITEBACK	BIT(4)
+
+static int ntfs_write_da_iomap_begin_non_resident(struct inode *inode,
+		loff_t offset, loff_t length, unsigned int flags,
+		struct iomap *iomap, int ntfs_iomap_flags)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct ntfs_volume *vol = ni->vol;
+	loff_t vcn_ofs, rl_length;
+	s64 vcn, start_lcn, lcn_count;
+	bool balloc = false, update_mp;
+	int err;
+	s64 max_clu_count =
+		ntfs_bytes_to_cluster(vol, round_up(length, vol->cluster_size));
+
+	vcn = ntfs_bytes_to_cluster(vol, offset);
+	vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset);
+
+	update_mp = ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_DIO | NTFS_IOMAP_FLAGS_MKWRITE) ||
+			NInoAttr(ni) || ni->mft_no < FILE_first_user;
+	down_write(&ni->runlist.lock);
+	err = ntfs_attr_map_cluster(ni, vcn, &start_lcn, &lcn_count,
+			max_clu_count, &balloc, update_mp,
+			ntfs_iomap_flags & NTFS_IOMAP_FLAGS_WRITEBACK);
+	up_write(&ni->runlist.lock);
+	mutex_unlock(&ni->mrec_lock);
+	if (err) {
+		ni->i_dealloc_clusters = 0;
+		return err;
+	}
+
+	iomap->bdev = inode->i_sb->s_bdev;
+	iomap->offset = offset;
+
+	rl_length = ntfs_cluster_to_bytes(vol, lcn_count);
+	if (length > rl_length - vcn_ofs)
+		iomap->length = rl_length - vcn_ofs;
+	else
+		iomap->length = length;
+
+	if (start_lcn == LCN_HOLE)
+		iomap->type = IOMAP_HOLE;
+	else
+		iomap->type = IOMAP_MAPPED;
+	if (balloc == true)
+		iomap->flags = IOMAP_F_NEW;
+
+	iomap->addr = ntfs_cluster_to_bytes(vol, start_lcn) + vcn_ofs;
+
+	if (balloc == true) {
+		if (flags & IOMAP_DIRECT ||
+		    ntfs_iomap_flags & NTFS_IOMAP_FLAGS_MKWRITE) {
+			loff_t end = offset + length;
+
+			if (vcn_ofs || ((vol->cluster_size > iomap->length) &&
+					end < ni->initialized_size))
+				err = ntfs_dio_zero_range(inode,
+							  start_lcn <<
+							  vol->cluster_size_bits,
+							  vol->cluster_size);
+			if (!err && lcn_count > 1 &&
+			    (iomap->length & vol->cluster_size_mask &&
+			     end < ni->initialized_size))
+				err = ntfs_dio_zero_range(inode,
+							  (start_lcn + lcn_count - 1) <<
+							  vol->cluster_size_bits,
+							  vol->cluster_size);
+		} else {
+			if (lcn_count > ni->i_dealloc_clusters)
+				ni->i_dealloc_clusters = 0;
+			else
+				ni->i_dealloc_clusters -= lcn_count;
+		}
+		if (err < 0)
+			return err;
+	}
+
+	if (ntfs_iomap_flags & NTFS_IOMAP_FLAGS_MKWRITE &&
+	    iomap->offset + iomap->length > ni->initialized_size) {
+		err = ntfs_attr_set_initialized_size(ni, iomap->offset +
+				iomap->length);
+	}
+
+	return err;
+}
+
+static int ntfs_write_iomap_begin_resident(struct inode *inode, loff_t offset,
+		struct iomap *iomap)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct attr_record *a;
+	struct ntfs_attr_search_ctx *ctx;
+	u32 attr_len;
+	int err = 0;
+	char *kattr;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		goto out;
+	}
+
+	a = ctx->attr;
+	/* The total length of the attribute value. */
+	attr_len = le32_to_cpu(a->data.resident.value_length);
+	kattr = (u8 *)a + le16_to_cpu(a->data.resident.value_offset);
+
+	iomap->inline_data = kmemdup(kattr, attr_len, GFP_KERNEL);
+	if (!iomap->inline_data) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	iomap->type = IOMAP_INLINE;
+	iomap->offset = 0;
+	/* iomap requires there is only one INLINE_DATA extent */
+	iomap->length = attr_len;
+
+out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	mutex_unlock(&ni->mrec_lock);
+	return err;
+}
+
+static int ntfs_write_iomap_begin_non_resident(struct inode *inode, loff_t offset,
+					       loff_t length, unsigned int flags,
+					       struct iomap *iomap, int ntfs_iomap_flags)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+
+	if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) &&
+	    offset + length > ni->initialized_size) {
+		int ret;
+
+		ret = ntfs_extend_initialized_size(inode, offset,
+						   offset + length,
+						   ntfs_iomap_flags &
+						   NTFS_IOMAP_FLAGS_DIO);
+		if (ret < 0)
+			return ret;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	if (ntfs_iomap_flags & NTFS_IOMAP_FLAGS_BEGIN)
+		return  ntfs_write_simple_iomap_begin_non_resident(inode, offset,
+								   length, iomap);
+	else
+		return ntfs_write_da_iomap_begin_non_resident(inode,
+							      offset, length,
+							      flags, iomap,
+							      ntfs_iomap_flags);
+}
+
+static int __ntfs_write_iomap_begin(struct inode *inode, loff_t offset,
+				    loff_t length, unsigned int flags,
+				    struct iomap *iomap, int ntfs_iomap_flags)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	loff_t end = offset + length;
+
+	if (NVolShutdown(ni->vol))
+		return -EIO;
+
+	if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) &&
+	    end > ni->data_size) {
+		struct ntfs_volume *vol = ni->vol;
+		int ret;
+
+		mutex_lock(&ni->mrec_lock);
+		if (end > ni->allocated_size &&
+		    end < ni->allocated_size + vol->preallocated_size)
+			ret = ntfs_attr_expand(ni, end,
+					ni->allocated_size + vol->preallocated_size);
+		else
+			ret = ntfs_attr_expand(ni, end, 0);
+		mutex_unlock(&ni->mrec_lock);
+		if (ret)
+			return ret;
+	}
+
+	if (!NInoNonResident(ni)) {
+		mutex_lock(&ni->mrec_lock);
+		return ntfs_write_iomap_begin_resident(inode, offset, iomap);
+	}
+	return  ntfs_write_iomap_begin_non_resident(inode, offset, length, flags,
+						    iomap, ntfs_iomap_flags);
+}
+
+static int ntfs_write_iomap_begin(struct inode *inode, loff_t offset,
+				  loff_t length, unsigned int flags,
+				  struct iomap *iomap, struct iomap *srcmap)
+{
+	return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap,
+			NTFS_IOMAP_FLAGS_BEGIN);
+}
+
+static int ntfs_write_iomap_end_resident(struct inode *inode, loff_t pos,
+					 loff_t length, ssize_t written,
+					 unsigned int flags, struct iomap *iomap)
+{
+	struct ntfs_inode *ni = NTFS_I(inode);
+	struct ntfs_attr_search_ctx *ctx;
+	u32 attr_len;
+	int err;
+	char *kattr;
+
+	mutex_lock(&ni->mrec_lock);
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		written = -ENOMEM;
+		mutex_unlock(&ni->mrec_lock);
+		return written;
+	}
+
+	err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
+			       CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (err) {
+		if (err == -ENOENT)
+			err = -EIO;
+		written = err;
+		goto err_out;
+	}
+
+	/* The total length of the attribute value. */
+	attr_len = le32_to_cpu(ctx->attr->data.resident.value_length);
+	if (pos >= attr_len || pos + written > attr_len)
+		goto err_out;
+
+	kattr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset);
+	memcpy(kattr + pos, iomap_inline_data(iomap, pos), written);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+err_out:
+	ntfs_attr_put_search_ctx(ctx);
+	kfree(iomap->inline_data);
+	mutex_unlock(&ni->mrec_lock);
+	return written;
+
+}
+
+static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+				ssize_t written, unsigned int flags,
+				struct iomap *iomap)
+{
+	if (iomap->type == IOMAP_INLINE)
+		return ntfs_write_iomap_end_resident(inode, pos, length,
+						     written, flags, iomap);
+	return written;
+}
+
+const struct iomap_ops ntfs_write_iomap_ops = {
+	.iomap_begin		= ntfs_write_iomap_begin,
+	.iomap_end		= ntfs_write_iomap_end,
+};
+
+static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset,
+				  loff_t length, unsigned int flags,
+				  struct iomap *iomap, struct iomap *srcmap)
+{
+	return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap,
+			NTFS_IOMAP_FLAGS_MKWRITE);
+}
+
+const struct iomap_ops ntfs_page_mkwrite_iomap_ops = {
+	.iomap_begin		= ntfs_page_mkwrite_iomap_begin,
+	.iomap_end		= ntfs_write_iomap_end,
+};
+
+static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset,
+				  loff_t length, unsigned int flags,
+				  struct iomap *iomap, struct iomap *srcmap)
+{
+	return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap,
+			NTFS_IOMAP_FLAGS_DIO);
+}
+
+const struct iomap_ops ntfs_dio_iomap_ops = {
+	.iomap_begin		= ntfs_dio_iomap_begin,
+	.iomap_end		= ntfs_write_iomap_end,
+};
+
+static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc,
+		struct folio *folio, u64 offset, unsigned int len, u64 end_pos)
+{
+	if (offset < wpc->iomap.offset ||
+	    offset >= wpc->iomap.offset + wpc->iomap.length) {
+		int error;
+
+		error = __ntfs_write_iomap_begin(wpc->inode, offset,
+				NTFS_I(wpc->inode)->allocated_size - offset,
+				IOMAP_WRITE, &wpc->iomap,
+				NTFS_IOMAP_FLAGS_WRITEBACK);
+		if (error)
+			return error;
+	}
+
+	return iomap_add_to_ioend(wpc, folio, offset, end_pos, len);
+}
+
+const struct iomap_writeback_ops ntfs_writeback_ops = {
+	.writeback_range	= ntfs_writeback_range,
+	.writeback_submit	= iomap_ioend_writeback_submit,
+};
diff --git a/fs/ntfs/iomap.h b/fs/ntfs/iomap.h
new file mode 100644
index 0000000000000..3abc1d493e918
--- /dev/null
+++ b/fs/ntfs/iomap.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_IOMAP_H
+#define _LINUX_NTFS_IOMAP_H
+
+#include <linux/pagemap.h>
+#include <linux/iomap.h>
+
+#include "volume.h"
+#include "inode.h"
+
+extern const struct iomap_ops ntfs_write_iomap_ops;
+extern const struct iomap_ops ntfs_read_iomap_ops;
+extern const struct iomap_ops ntfs_seek_iomap_ops;
+extern const struct iomap_ops ntfs_page_mkwrite_iomap_ops;
+extern const struct iomap_ops ntfs_dio_iomap_ops;
+extern const struct iomap_writeback_ops ntfs_writeback_ops;
+extern const struct iomap_write_ops ntfs_iomap_folio_ops;
+extern int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length);
+#endif /* _LINUX_NTFS_IOMAP_H */
diff --git a/fs/ntfs/layout.h b/fs/ntfs/layout.h
new file mode 100644
index 0000000000000..d94f914e830f3
--- /dev/null
+++ b/fs/ntfs/layout.h
@@ -0,0 +1,2346 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * All NTFS associated on-disk structures.
+ *
+ * Copyright (c) 2001-2005 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ */
+
+#ifndef _LINUX_NTFS_LAYOUT_H
+#define _LINUX_NTFS_LAYOUT_H
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/list.h>
+#include <asm/byteorder.h>
+
+/* The NTFS oem_id "NTFS    " */
+#define magicNTFS	cpu_to_le64(0x202020205346544eULL)
+
+/*
+ * Location of bootsector on partition:
+ *	The standard NTFS_BOOT_SECTOR is on sector 0 of the partition.
+ *	On NT4 and above there is one backup copy of the boot sector to
+ *	be found on the last sector of the partition (not normally accessible
+ *	from within Windows as the bootsector contained number of sectors
+ *	value is one less than the actual value!).
+ *	On versions of NT 3.51 and earlier, the backup copy was located at
+ *	number of sectors/2 (integer divide), i.e. in the middle of the volume.
+ */
+
+/*
+ * BIOS parameter block (bpb) structure.
+ *
+ * @bytes_per_sector:       Size of a sector in bytes (usually 512).
+ *                          Matches the logical sector size of the underlying device.
+ * @sectors_per_cluster:    Size of a cluster in sectors (NTFS cluster size / sector size).
+ * @reserved_sectors:       Number of reserved sectors at the beginning of the volume.
+ *                          Always set to 0 in NTFS.
+ * @fats:                   Number of FAT tables.
+ *                          Always 0 in NTFS (no FAT tables exist).
+ * @root_entries:           Number of entries in the root directory.
+ *                          Always 0 in NTFS.
+ * @sectors:                Total number of sectors on the volume.
+ *                          Always 0 in NTFS (use @large_sectors instead).
+ * @media_type:             Media descriptor byte.
+ *                          0xF8 for hard disk (fixed media) in NTFS.
+ * @sectors_per_fat:        Number of sectors per FAT table.
+ *                          Always 0 in NTFS.
+ * @sectors_per_track:      Number of sectors per track.
+ *                          Irrelevant for NTFS.
+ * @heads:                  Number of heads (CHS geometry).
+ *                          Irrelevant for NTFS.
+ * @hidden_sectors:         Number of hidden sectors before the start of the partition.
+ *                          Always 0 in NTFS boot sector.
+ * @large_sectors:          Total number of sectors on the volume.
+ */
+struct bios_parameter_block {
+	__le16 bytes_per_sector;
+	u8 sectors_per_cluster;
+	__le16 reserved_sectors;
+	u8 fats;
+	__le16 root_entries;
+	__le16 sectors;
+	u8 media_type;
+	__le16 sectors_per_fat;
+	__le16 sectors_per_track;
+	__le16 heads;
+	__le32 hidden_sectors;
+	__le32 large_sectors;
+} __packed;
+
+/*
+ * NTFS boot sector structure.
+ *
+ * @jump:               3-byte jump instruction to boot code (irrelevant for NTFS).
+ *                      Typically 0xEB 0x52 0x90 or similar.
+ * @oem_id:             OEM identifier string (8 bytes).
+ *                      Always "NTFS    " (with trailing spaces) in NTFS volumes.
+ * @bpb:                Legacy BIOS Parameter Block (see struct bios_parameter_block).
+ *                      Mostly zeroed or set to fixed values for NTFS compatibility.
+ * @unused:             4 bytes, reserved/unused.
+ *                      NTFS disk editors show it as:
+ *                        - physical_drive (0x80 for fixed disk)
+ *                        - current_head (0)
+ *                        - extended_boot_signature (0x80 or 0x28)
+ *                        - unused (0)
+ *                      Always zero in practice for NTFS.
+ * @number_of_sectors:  Number of sectors in volume. Gives maximum volume
+ *                      size of 2^63 sectors. Assuming standard sector
+ *                      size of 512 bytes, the maximum byte size is
+ *                      approx. 4.7x10^21 bytes. (-;
+ * @mft_lcn:            Logical cluster number (LCN) of the $MFT data attribute.
+ *                      Location of the Master File Table.
+ * @mftmirr_lcn:        LCN of the $MFTMirr (first 3-4 MFT records copy).
+ *                      Mirror for boot-time recovery.
+ * @clusters_per_mft_record:
+ *                      Size of each MFT record in clusters.
+ * @reserved0:          3 bytes, reserved/zero.
+ * @clusters_per_index_record:
+ *                      Size of each index block/record in clusters.
+ * @reserved1:          3 bytes, reserved/zero.
+ * @volume_serial_number:
+ *                      64-bit volume serial number.
+ *                      Used for identification (irrelevant for NTFS operation).
+ * @checksum:           32-bit checksum of the boot sector (excluding this field).
+ *                      Used to detect boot sector corruption.
+ * @bootstrap:          426 bytes of bootstrap code.
+ *                      Irrelevant for NTFS (contains x86 boot loader stub).
+ * @end_of_sector_marker:
+ *                      2-byte end-of-sector signature.
+ *                      Always 0xAA55 (little-endian magic number).
+ */
+struct ntfs_boot_sector {
+	u8 jump[3];
+	__le64 oem_id;
+	struct bios_parameter_block bpb;
+	u8 unused[4];
+	__le64 number_of_sectors;
+	__le64 mft_lcn;
+	__le64 mftmirr_lcn;
+	s8 clusters_per_mft_record;
+	u8 reserved0[3];
+	s8 clusters_per_index_record;
+	u8 reserved1[3];
+	__le64 volume_serial_number;
+	__le32 checksum;
+	u8 bootstrap[426];
+	__le16 end_of_sector_marker;
+} __packed;
+
+static_assert(sizeof(struct ntfs_boot_sector) == 512);
+
+/*
+ * Magic identifiers present at the beginning of all ntfs record containing
+ * records (like mft records for example).
+ *
+ * magic_FILE:      MFT entry header ("FILE" in ASCII).
+ *                  Used in $MFT/$DATA for all master file table records.
+ * magic_INDX:      Index buffer header ("INDX" in ASCII).
+ *                  Used in $INDEX_ALLOCATION attributes (directories, $I30 indexes).
+ * magic_HOLE:      Hole marker ("HOLE" in ASCII).
+ *                  Introduced in NTFS 3.0+, used for sparse/hole regions in some contexts.
+ * magic_RSTR:      Restart page header ("RSTR" in ASCII).
+ *                  Used in LogFile for restart pages (transaction log recovery).
+ * magic_RCRD:      Log record page header ("RCRD" in ASCII).
+ *                  Used in LogFile for individual log record pages.
+ * magic_CHKD:      Chkdsk modified marker ("CHKD" in ASCII).
+ *                  Set by chkdsk when it modifies a record; indicates repair was done.
+ * magic_BAAD:      Bad record marker ("BAAD" in ASCII).
+ *                  Indicates a multi-sector transfer failure was detected.
+ *                  The record is corrupted/unusable; often set during I/O errors.
+ * magic_empty:     Empty/uninitialized page marker (0xffffffff).
+ *                  Used in LogFile when a page is filled with 0xff bytes
+ *                  and has not yet been initialized. Must be formatted before use.
+ */
+enum {
+	magic_FILE = cpu_to_le32(0x454c4946),
+	magic_INDX = cpu_to_le32(0x58444e49),
+	magic_HOLE = cpu_to_le32(0x454c4f48),
+	magic_RSTR = cpu_to_le32(0x52545352),
+	magic_RCRD = cpu_to_le32(0x44524352),
+	magic_CHKD = cpu_to_le32(0x444b4843),
+	magic_BAAD = cpu_to_le32(0x44414142),
+	magic_empty = cpu_to_le32(0xffffffff)
+};
+
+/*
+ * Generic magic comparison macros. Finally found a use for the ## preprocessor
+ * operator! (-8
+ */
+
+static inline bool __ntfs_is_magic(__le32 x, __le32 r)
+{
+	return (x == r);
+}
+#define ntfs_is_magic(x, m)	__ntfs_is_magic(x, magic_##m)
+
+static inline bool __ntfs_is_magicp(__le32 *p, __le32 r)
+{
+	return (*p == r);
+}
+#define ntfs_is_magicp(p, m)	__ntfs_is_magicp(p, magic_##m)
+
+/*
+ * Specialised magic comparison macros for the NTFS_RECORD_TYPEs defined above.
+ */
+#define ntfs_is_file_record(x)		(ntfs_is_magic(x, FILE))
+#define ntfs_is_file_recordp(p)		(ntfs_is_magicp(p, FILE))
+#define ntfs_is_mft_record(x)		(ntfs_is_file_record(x))
+#define ntfs_is_mft_recordp(p)		(ntfs_is_file_recordp(p))
+#define ntfs_is_indx_record(x)		(ntfs_is_magic(x, INDX))
+#define ntfs_is_indx_recordp(p)		(ntfs_is_magicp(p, INDX))
+#define ntfs_is_hole_record(x)		(ntfs_is_magic(x, HOLE))
+#define ntfs_is_hole_recordp(p)		(ntfs_is_magicp(p, HOLE))
+
+#define ntfs_is_rstr_record(x)		(ntfs_is_magic(x, RSTR))
+#define ntfs_is_rstr_recordp(p)		(ntfs_is_magicp(p, RSTR))
+#define ntfs_is_rcrd_record(x)		(ntfs_is_magic(x, RCRD))
+#define ntfs_is_rcrd_recordp(p)		(ntfs_is_magicp(p, RCRD))
+
+#define ntfs_is_chkd_record(x)		(ntfs_is_magic(x, CHKD))
+#define ntfs_is_chkd_recordp(p)		(ntfs_is_magicp(p, CHKD))
+
+#define ntfs_is_baad_record(x)		(ntfs_is_magic(x, BAAD))
+#define ntfs_is_baad_recordp(p)		(ntfs_is_magicp(p, BAAD))
+
+#define ntfs_is_empty_record(x)		(ntfs_is_magic(x, empty))
+#define ntfs_is_empty_recordp(p)	(ntfs_is_magicp(p, empty))
+
+/*
+ * struct ntfs_record - Common header for all multi-sector protected NTFS records
+ *
+ * @magic:      4-byte magic identifier for the record type and/or status.
+ *              Common values are defined in the magic_* enum (FILE, INDX, RSTR,
+ *              RCRD, CHKD, BAAD, HOLE, empty).
+ *              - "FILE" = MFT record
+ *              - "INDX" = Index allocation block
+ *              - "BAAD" = Record corrupted (multi-sector fixup failed)
+ *              - 0xffffffff = Uninitialized/empty page
+ * @usa_ofs:    Offset (in bytes) from the start of this record to the Update
+ *              Sequence Array (USA).
+ *              The USA is located at record + usa_ofs.
+ * @usa_count:  Number of 16-bit entries in the USA array (including the Update
+ *              Sequence Number itself).
+ *              - Number of fixup locations = usa_count - 1
+ *              - Each fixup location is a 16-bit value in the record that needs
+ *                protection against torn writes.
+ *
+ * The Update Sequence Array (usa) is an array of the __le16 values which belong
+ * to the end of each sector protected by the update sequence record in which
+ * this array is contained. Note that the first entry is the Update Sequence
+ * Number (usn), a cyclic counter of how many times the protected record has
+ * been written to disk. The values 0 and -1 (ie. 0xffff) are not used. All
+ * last le16's of each sector have to be equal to the usn (during reading) or
+ * are set to it (during writing). If they are not, an incomplete multi sector
+ * transfer has occurred when the data was written.
+ * The maximum size for the update sequence array is fixed to:
+ *	maximum size = usa_ofs + (usa_count * 2) = 510 bytes
+ * The 510 bytes comes from the fact that the last __le16 in the array has to
+ * (obviously) finish before the last __le16 of the first 512-byte sector.
+ * This formula can be used as a consistency check in that usa_ofs +
+ * (usa_count * 2) has to be less than or equal to 510.
+ */
+struct ntfs_record {
+	__le32 magic;
+	__le16 usa_ofs;
+	__le16 usa_count;
+} __packed;
+
+/*
+ * System files mft record numbers. All these files are always marked as used
+ * in the bitmap attribute of the mft; presumably in order to avoid accidental
+ * allocation for random other mft records. Also, the sequence number for each
+ * of the system files is always equal to their mft record number and it is
+ * never modified.
+ *
+ * FILE_MFT:        Master File Table (MFT) itself.
+ *                  Data attribute contains all MFT entries;
+ *                  Bitmap attribute tracks which records are in use (bit==1).
+ * FILE_MFTMirr:    MFT mirror: copy of the first four (or more) MFT records
+ *                  in its data attribute.
+ *                  If cluster size > 4 KiB, copies first N records where
+ *                  N = cluster_size / mft_record_size.
+ * FILE_LogFile:    Journaling log (LogFile) in data attribute.
+ *                  Used for transaction logging and recovery.
+ * FILE_Volume:     Volume information and name.
+ *                  Contains $VolumeName (label) and $VolumeInformation
+ *                  (flags, NTFS version). Windows calls this the volume DASD.
+ * FILE_AttrDef:    Attribute definitions array in data attribute.
+ *                  Defines all possible attribute types and their properties.
+ * FILE_root:       Root directory ($Root).
+ *                  The top-level directory of the filesystem.
+ * FILE_Bitmap:     Cluster allocation bitmap ($Bitmap) in data attribute.
+ *                  Tracks free/used clusters (LCNs) on the volume.
+ * FILE_Boot:       Boot sector ($Boot) in data attribute.
+ *                  Always located at cluster 0; contains BPB and NTFS parameters.
+ * FILE_BadClus:    Bad cluster list ($BadClus) in non-resident data attribute.
+ *                  Marks all known bad clusters.
+ * FILE_Secure:     Security descriptors ($Secure).
+ *                  Contains shared $SDS (security descriptors) and two indexes
+ *                  ($SDH, $SII). Introduced in Windows 2000.
+ *                  Before that, it was called $Quota but was unused.
+ * FILE_UpCase:     Uppercase table ($UpCase) in data attribute.
+ *                  Maps all 65536 Unicode characters to their uppercase forms.
+ * FILE_Extend:     System directory ($Extend).
+ *                  Contains additional system files ($ObjId, $Quota, $Reparse,
+ *                  $UsnJrnl, etc.). Introduced in NTFS 3.0 (Windows 2000).
+ * FILE_reserved12: Reserved for future use (MFT records 12–15).
+ * FILE_reserved13: Reserved.
+ * FILE_reserved14: Reserved.
+ * FILE_reserved15: Reserved.
+ * FILE_first_user: First possible user-created file MFT record number.
+ *                  Used as a boundary to distinguish system files from user files.
+ */
+enum {
+	FILE_MFT       = 0,
+	FILE_MFTMirr   = 1,
+	FILE_LogFile   = 2,
+	FILE_Volume    = 3,
+	FILE_AttrDef   = 4,
+	FILE_root      = 5,
+	FILE_Bitmap    = 6,
+	FILE_Boot      = 7,
+	FILE_BadClus   = 8,
+	FILE_Secure    = 9,
+	FILE_UpCase    = 10,
+	FILE_Extend    = 11,
+	FILE_reserved12 = 12,
+	FILE_reserved13 = 13,
+	FILE_reserved14 = 14,
+	FILE_reserved15 = 15,
+	FILE_first_user = 16,
+};
+
+/*
+ * enum - Flags for MFT record header
+ *
+ * These are the so far known MFT_RECORD_* flags (16-bit) which contain
+ * information about the mft record in which they are present.
+ *
+ * MFT_RECORD_IN_USE:        This MFT record is allocated and in use.
+ *                           (bit set = record is valid/used; clear = free)
+ * MFT_RECORD_IS_DIRECTORY:  This MFT record represents a directory.
+ *                           (Used to quickly distinguish files from directories)
+ * MFT_RECORD_IS_4:          Indicates the record is a special "record 4" type.
+ *                           (Rarely used; related to NTFS internal special cases,
+ *                           often for $AttrDef or early system files)
+ * MFT_RECORD_IS_VIEW_INDEX: This MFT record is used as a view index.
+ *                           (Specific to NTFS indexed views or object ID indexes)
+ * MFT_REC_SPACE_FILLER:     Dummy value to force the enum to be 16-bit wide.
+ *                           (Not a real flag; just a sentinel to ensure the type
+ *                           is __le16 and no higher bits are accidentally used)
+ */
+enum {
+	MFT_RECORD_IN_USE		= cpu_to_le16(0x0001),
+	MFT_RECORD_IS_DIRECTORY		= cpu_to_le16(0x0002),
+	MFT_RECORD_IS_4			= cpu_to_le16(0x0004),
+	MFT_RECORD_IS_VIEW_INDEX	= cpu_to_le16(0x0008),
+	MFT_REC_SPACE_FILLER		= cpu_to_le16(0xffff), /*Just to make flags 16-bit.*/
+} __packed;
+
+/*
+ * mft references (aka file references or file record segment references) are
+ * used whenever a structure needs to refer to a record in the mft.
+ *
+ * A reference consists of a 48-bit index into the mft and a 16-bit sequence
+ * number used to detect stale references.
+ *
+ * For error reporting purposes we treat the 48-bit index as a signed quantity.
+ *
+ * The sequence number is a circular counter (skipping 0) describing how many
+ * times the referenced mft record has been (re)used. This has to match the
+ * sequence number of the mft record being referenced, otherwise the reference
+ * is considered stale and removed.
+ *
+ * If the sequence number is zero it is assumed that no sequence number
+ * consistency checking should be performed.
+ */
+
+/*
+ * Define two unpacking macros to get to the reference (MREF) and
+ * sequence number (MSEQNO) respectively.
+ * The _LE versions are to be applied on little endian MFT_REFs.
+ * Note: The _LE versions will return a CPU endian formatted value!
+ */
+#define MFT_REF_MASK_CPU 0x0000ffffffffffffULL
+#define MFT_REF_MASK_LE cpu_to_le64(MFT_REF_MASK_CPU)
+
+#define MK_MREF(m, s)	((u64)(((u64)(s) << 48) |		\
+					((u64)(m) & MFT_REF_MASK_CPU)))
+#define MK_LE_MREF(m, s) cpu_to_le64(MK_MREF(m, s))
+
+#define MREF(x)		((unsigned long)((x) & MFT_REF_MASK_CPU))
+#define MSEQNO(x)	((u16)(((x) >> 48) & 0xffff))
+#define MREF_LE(x)	((unsigned long)(le64_to_cpu(x) & MFT_REF_MASK_CPU))
+#define MREF_INO(x)	((unsigned long)MREF_LE(x))
+#define MSEQNO_LE(x)	((u16)((le64_to_cpu(x) >> 48) & 0xffff))
+
+#define IS_ERR_MREF(x)	(((x) & 0x0000800000000000ULL) ? true : false)
+#define ERR_MREF(x)	((u64)((s64)(x)))
+#define MREF_ERR(x)	((int)((s64)(x)))
+
+/*
+ * struct mft_record - NTFS Master File Table (MFT) record header
+ *
+ * The mft record header present at the beginning of every record in the mft.
+ * This is followed by a sequence of variable length attribute records which
+ * is terminated by an attribute of type AT_END which is a truncated attribute
+ * in that it only consists of the attribute type code AT_END and none of the
+ * other members of the attribute structure are present.
+ *
+ * magic:               Record magic ("FILE" for valid MFT entries).
+ *                      See ntfs_record magic enum for other values.
+ * usa_ofs:             Offset to Update Sequence Array (see ntfs_record).
+ * usa_count:           Number of entries in USA (see ntfs_record).
+ * lsn:                 Log sequence number (LSN) from LogFile.
+ *                      Incremented on every modification to this record.
+ * sequence_number:     Reuse count of this MFT record slot.
+ *                      Incremented (skipping zero) when the file is deleted.
+ *                      Zero means never reused or special case.
+ *                      Part of MFT reference (together with record number).
+ * link_count:          Number of hard links (directory entries) to this file.
+ *                      Only meaningful in base MFT records.
+ *                      When deleting a directory entry:
+ *                        - If link_count == 1, delete the whole file
+ *                        - Else remove only the $FILE_NAME attribute and decrement
+ * attrs_offset:        Byte offset from start of MFT record to first attribute.
+ *                      Must be 8-byte aligned.
+ * flags:               Bit array of MFT_RECORD_* flags (see MFT_RECORD_IN_USE enum).
+ *                      MFT_RECORD_IN_USE cleared when record is freed/deleted.
+ * bytes_in_use:        Number of bytes actually used in this MFT record.
+ *                      Must be 8-byte aligned.
+ *                      Includes header + all attributes + padding.
+ * bytes_allocated:     Total allocated size of this MFT record.
+ *                      Usually equal to MFT record size (1024 bytes or cluster size).
+ * base_mft_record:     MFT reference to the base record.
+ *                      0 for base records.
+ *                      Non-zero for extension records → points to base record
+ *                      containing the $ATTRIBUTE_LIST that describes this extension.
+ * next_attr_instance:  Next attribute instance number to assign.
+ *                      Incremented after each use.
+ *                      Reset to 0 when MFT record is reused.
+ *                      First instance is always 0.
+ * reserved:            Reserved for alignment (NTFS 3.1+).
+ * mft_record_number:   This MFT record's number (index in $MFT).
+ *                      Only present in NTFS 3.1+ (Windows XP and above).
+ */
+struct mft_record {
+	__le32 magic;
+	__le16 usa_ofs;
+	__le16 usa_count;
+
+	__le64 lsn;
+	__le16 sequence_number;
+	__le16 link_count;
+	__le16 attrs_offset;
+	__le16 flags;
+	__le32 bytes_in_use;
+	__le32 bytes_allocated;
+	__le64 base_mft_record;
+	__le16 next_attr_instance;
+	__le16 reserved;
+	__le32 mft_record_number;
+} __packed;
+
+static_assert(sizeof(struct mft_record) == 48);
+
+/**x
+ * struct mft_record_old - Old NTFS MFT record header (pre-NTFS 3.1 / Windows XP)
+ *
+ * This is the older version of the MFT record header used in NTFS versions
+ * prior to 3.1 (Windows XP and later). It lacks the additional fields
+ * @reserved and @mft_record_number that were added in NTFS 3.1+.
+ *
+ * @magic:              Record magic ("FILE" for valid MFT entries).
+ *                      See ntfs_record magic enum for other values.
+ * @usa_ofs:            Offset to Update Sequence Array (see ntfs_record).
+ * @usa_count:          Number of entries in USA (see ntfs_record).
+ * @lsn:                Log sequence number (LSN) from LogFile.
+ *                      Incremented on every modification to this record.
+ * @sequence_number:    Reuse count of this MFT record slot.
+ *                      Incremented (skipping zero) when the file is deleted.
+ *                      Zero means never reused or special case.
+ *                      Part of MFT reference (together with record number).
+ * @link_count:         Number of hard links (directory entries) to this file.
+ *                      Only meaningful in base MFT records.
+ *                      When deleting a directory entry:
+ *                        - If link_count == 1, delete the whole file
+ *                        - Else remove only the $FILE_NAME attribute and decrement
+ * @attrs_offset:       Byte offset from start of MFT record to first attribute.
+ *                      Must be 8-byte aligned.
+ * @flags:              Bit array of MFT_RECORD_* flags (see MFT_RECORD_IN_USE enum).
+ *                      MFT_RECORD_IN_USE cleared when record is freed/deleted.
+ * @bytes_in_use:       Number of bytes actually used in this MFT record.
+ *                      Must be 8-byte aligned.
+ *                      Includes header + all attributes + padding.
+ * @bytes_allocated:    Total allocated size of this MFT record.
+ *                      Usually equal to MFT record size (1024 bytes or cluster size).
+ * @base_mft_record:    MFT reference to the base record.
+ *                      0 for base records.
+ *                      Non-zero for extension records → points to base record
+ *                      containing the $ATTRIBUTE_LIST that describes this extension.
+ * @next_attr_instance: Next attribute instance number to assign.
+ *                      Incremented after each use.
+ *                      Reset to 0 when MFT record is reused.
+ *                      First instance is always 0.
+ */
+struct mft_record_old {
+	__le32 magic;
+	__le16 usa_ofs;
+	__le16 usa_count;
+
+	__le64 lsn;
+	__le16 sequence_number;
+	__le16 link_count;
+	__le16 attrs_offset;
+	__le16 flags;
+	__le32 bytes_in_use;
+	__le32 bytes_allocated;
+	__le64 base_mft_record;
+	__le16 next_attr_instance;
+} __packed;
+
+static_assert(sizeof(struct mft_record_old) == 42);
+
+/*
+ * System defined attributes (32-bit).  Each attribute type has a corresponding
+ * attribute name (Unicode string of maximum 64 character length) as described
+ * by the attribute definitions present in the data attribute of the $AttrDef
+ * system file.  On NTFS 3.0 volumes the names are just as the types are named
+ * in the below defines exchanging AT_ for the dollar sign ($).  If that is not
+ * a revealing choice of symbol I do not know what is... (-;
+ */
+enum {
+	AT_UNUSED			= cpu_to_le32(0),
+	AT_STANDARD_INFORMATION		= cpu_to_le32(0x10),
+	AT_ATTRIBUTE_LIST		= cpu_to_le32(0x20),
+	AT_FILE_NAME			= cpu_to_le32(0x30),
+	AT_OBJECT_ID			= cpu_to_le32(0x40),
+	AT_SECURITY_DESCRIPTOR		= cpu_to_le32(0x50),
+	AT_VOLUME_NAME			= cpu_to_le32(0x60),
+	AT_VOLUME_INFORMATION		= cpu_to_le32(0x70),
+	AT_DATA				= cpu_to_le32(0x80),
+	AT_INDEX_ROOT			= cpu_to_le32(0x90),
+	AT_INDEX_ALLOCATION		= cpu_to_le32(0xa0),
+	AT_BITMAP			= cpu_to_le32(0xb0),
+	AT_REPARSE_POINT		= cpu_to_le32(0xc0),
+	AT_EA_INFORMATION		= cpu_to_le32(0xd0),
+	AT_EA				= cpu_to_le32(0xe0),
+	AT_PROPERTY_SET			= cpu_to_le32(0xf0),
+	AT_LOGGED_UTILITY_STREAM	= cpu_to_le32(0x100),
+	AT_FIRST_USER_DEFINED_ATTRIBUTE	= cpu_to_le32(0x1000),
+	AT_END				= cpu_to_le32(0xffffffff)
+};
+
+/*
+ * The collation rules for sorting views/indexes/etc (32-bit).
+ *
+ * COLLATION_BINARY - Collate by binary compare where the first byte is most
+ *	significant.
+ * COLLATION_UNICODE_STRING - Collate Unicode strings by comparing their binary
+ *	Unicode values, except that when a character can be uppercased, the
+ *	upper case value collates before the lower case one.
+ * COLLATION_FILE_NAME - Collate file names as Unicode strings. The collation
+ *	is done very much like COLLATION_UNICODE_STRING. In fact I have no idea
+ *	what the difference is. Perhaps the difference is that file names
+ *	would treat some special characters in an odd way (see
+ *	unistr.c::ntfs_collate_names() and unistr.c::legal_ansi_char_array[]
+ *	for what I mean but COLLATION_UNICODE_STRING would not give any special
+ *	treatment to any characters at all, but this is speculation.
+ * COLLATION_NTOFS_ULONG - Sorting is done according to ascending __le32 key
+ *	values. E.g. used for $SII index in FILE_Secure, which sorts by
+ *	security_id (le32).
+ * COLLATION_NTOFS_SID - Sorting is done according to ascending SID values.
+ *	E.g. used for $O index in FILE_Extend/$Quota.
+ * COLLATION_NTOFS_SECURITY_HASH - Sorting is done first by ascending hash
+ *	values and second by ascending security_id values. E.g. used for $SDH
+ *	index in FILE_Secure.
+ * COLLATION_NTOFS_ULONGS - Sorting is done according to a sequence of ascending
+ *	__le32 key values. E.g. used for $O index in FILE_Extend/$ObjId, which
+ *	sorts by object_id (16-byte), by splitting up the object_id in four
+ *	__le32 values and using them as individual keys. E.g. take the following
+ *	two security_ids, stored as follows on disk:
+ *		1st: a1 61 65 b7 65 7b d4 11 9e 3d 00 e0 81 10 42 59
+ *		2nd: 38 14 37 d2 d2 f3 d4 11 a5 21 c8 6b 79 b1 97 45
+ *	To compare them, they are split into four __le32 values each, like so:
+ *		1st: 0xb76561a1 0x11d47b65 0xe0003d9e 0x59421081
+ *		2nd: 0xd2371438 0x11d4f3d2 0x6bc821a5 0x4597b179
+ *	Now, it is apparent why the 2nd object_id collates after the 1st: the
+ *	first __le32 value of the 1st object_id is less than the first __le32 of
+ *	the 2nd object_id. If the first __le32 values of both object_ids were
+ *	equal then the second __le32 values would be compared, etc.
+ */
+enum {
+	COLLATION_BINARY		= cpu_to_le32(0x00),
+	COLLATION_FILE_NAME		= cpu_to_le32(0x01),
+	COLLATION_UNICODE_STRING	= cpu_to_le32(0x02),
+	COLLATION_NTOFS_ULONG		= cpu_to_le32(0x10),
+	COLLATION_NTOFS_SID		= cpu_to_le32(0x11),
+	COLLATION_NTOFS_SECURITY_HASH	= cpu_to_le32(0x12),
+	COLLATION_NTOFS_ULONGS		= cpu_to_le32(0x13),
+};
+
+/*
+ * enum - Attribute definition flags
+ *
+ * The flags (32-bit) describing attribute properties in the attribute
+ * definition structure.
+ * The INDEXABLE flag is fairly certainly correct as only the file
+ * name attribute has this flag set and this is the only attribute indexed in
+ * NT4.
+ *
+ * ATTR_DEF_INDEXABLE:      Attribute can be indexed.
+ *                          (Used for creating indexes like $I30, $SDH, etc.)
+ * ATTR_DEF_MULTIPLE:       Attribute type can be present multiple times
+ *                          in the MFT record of an inode.
+ *                          (e.g., multiple $FILE_NAME, $DATA streams)
+ * ATTR_DEF_NOT_ZERO:       Attribute value must contain at least one non-zero byte.
+ *                          (Prevents empty or all-zero values)
+ * ATTR_DEF_INDEXED_UNIQUE: Attribute must be indexed and the value must be unique
+ *                          for this attribute type across all MFT records of an inode.
+ *                          (e.g., security descriptor IDs in $Secure)
+ * ATTR_DEF_NAMED_UNIQUE:   Attribute must be named and the name must be unique
+ *                          for this attribute type across all MFT records of an inode.
+ *                          (e.g., named $DATA streams or alternate data streams)
+ * ATTR_DEF_RESIDENT:       Attribute must be resident (stored in MFT record).
+ *                          (Cannot be non-resident/sparse/compressed)
+ * ATTR_DEF_ALWAYS_LOG:     Always log modifications to this attribute in LogFile,
+ *                          regardless of whether it is resident or non-resident.
+ *                          Without this flag, modifications are logged only if resident.
+ *                          (Used for critical metadata attributes)
+ */
+enum {
+	ATTR_DEF_INDEXABLE	= cpu_to_le32(0x02),
+	ATTR_DEF_MULTIPLE	= cpu_to_le32(0x04),
+	ATTR_DEF_NOT_ZERO	= cpu_to_le32(0x08),
+	ATTR_DEF_INDEXED_UNIQUE	= cpu_to_le32(0x10),
+	ATTR_DEF_NAMED_UNIQUE	= cpu_to_le32(0x20),
+	ATTR_DEF_RESIDENT	= cpu_to_le32(0x40),
+	ATTR_DEF_ALWAYS_LOG	= cpu_to_le32(0x80),
+};
+
+/*
+ * struct attr_def - Attribute definition entry ($AttrDef array)
+ *
+ * The data attribute of FILE_AttrDef contains a sequence of attribute
+ * definitions for the NTFS volume. With this, it is supposed to be safe for an
+ * older NTFS driver to mount a volume containing a newer NTFS version without
+ * damaging it (that's the theory. In practice it's: not damaging it too much).
+ * Entries are sorted by attribute type. The flags describe whether the
+ * attribute can be resident/non-resident and possibly other things, but the
+ * actual bits are unknown.
+ *
+ * @name:           Unicode (UTF-16LE) name of the attribute (e.g. "$DATA", "$FILE_NAME").
+ *                  Zero-terminated string, maximum 0x40 characters (128 bytes).
+ *                  Used for human-readable display and debugging.
+ * @type:           Attribute type code (ATTR_TYPE_* constants).
+ *                  Defines which attribute this entry describes.
+ * @display_rule:   Default display rule (usually 0; rarely used in modern NTFS).
+ *                  Controls how the attribute is displayed in tools (legacy).
+ * @collation_rule: Default collation rule for indexing this attribute.
+ *                  Determines sort order when indexed (e.g. CASE_SENSITIVE, UNICODE).
+ *                  Used in $I30, $SDH, $SII, etc.
+ * @flags:          Bit array of attribute constraints (ATTR_DEF_* flags).
+ *                  See ATTR_DEF_INDEXABLE, ATTR_DEF_MULTIPLE, etc.
+ *                  Defines whether the attribute can be indexed, multiple, resident-only, etc.
+ * @min_size:       Optional minimum size of the attribute value (in bytes).
+ *                  0 means no minimum enforced.
+ * @max_size:       Maximum allowed size of the attribute value (in bytes).
+ */
+struct attr_def {
+	__le16 name[0x40];
+	__le32 type;
+	__le32 display_rule;
+	__le32 collation_rule;
+	__le32 flags;
+	__le64 min_size;
+	__le64 max_size;
+} __packed;
+
+static_assert(sizeof(struct attr_def) == 160);
+
+/*
+ * enum - Attribute flags (16-bit) for non-resident attributes
+ *
+ * ATTR_IS_COMPRESSED:      Attribute is compressed.
+ *                          If set, data is compressed using the method in
+ *                          ATTR_COMPRESSION_MASK.
+ * ATTR_COMPRESSION_MASK:   Mask for compression method.
+ *                          Valid values are defined in NTFS compression types
+ *                          (e.g., 0x02 = LZNT1, etc.).
+ *                          Also serves as the first illegal value for method.
+ * ATTR_IS_ENCRYPTED:       Attribute is encrypted.
+ *                          Data is encrypted using EFS (Encrypting File System).
+ * ATTR_IS_SPARSE:          Attribute is sparse.
+ *                          Contains holes (unallocated regions) that read as zeros.
+ */
+enum {
+	ATTR_IS_COMPRESSED    = cpu_to_le16(0x0001),
+	ATTR_COMPRESSION_MASK = cpu_to_le16(0x00ff),
+	ATTR_IS_ENCRYPTED     = cpu_to_le16(0x4000),
+	ATTR_IS_SPARSE	      = cpu_to_le16(0x8000),
+} __packed;
+
+/*
+ * Attribute compression.
+ *
+ * Only the data attribute is ever compressed in the current ntfs driver in
+ * Windows. Further, compression is only applied when the data attribute is
+ * non-resident. Finally, to use compression, the maximum allowed cluster size
+ * on a volume is 4kib.
+ *
+ * The compression method is based on independently compressing blocks of X
+ * clusters, where X is determined from the compression_unit value found in the
+ * non-resident attribute record header (more precisely: X = 2^compression_unit
+ * clusters). On Windows NT/2k, X always is 16 clusters (compression_unit = 4).
+ *
+ * There are three different cases of how a compression block of X clusters
+ * can be stored:
+ *
+ *   1) The data in the block is all zero (a sparse block):
+ *	  This is stored as a sparse block in the runlist, i.e. the runlist
+ *	  entry has length = X and lcn = -1. The mapping pairs array actually
+ *	  uses a delta_lcn value length of 0, i.e. delta_lcn is not present at
+ *	  all, which is then interpreted by the driver as lcn = -1.
+ *	  NOTE: Even uncompressed files can be sparse on NTFS 3.0 volumes, then
+ *	  the same principles apply as above, except that the length is not
+ *	  restricted to being any particular value.
+ *
+ *   2) The data in the block is not compressed:
+ *	  This happens when compression doesn't reduce the size of the block
+ *	  in clusters. I.e. if compression has a small effect so that the
+ *	  compressed data still occupies X clusters, then the uncompressed data
+ *	  is stored in the block.
+ *	  This case is recognised by the fact that the runlist entry has
+ *	  length = X and lcn >= 0. The mapping pairs array stores this as
+ *	  normal with a run length of X and some specific delta_lcn, i.e.
+ *	  delta_lcn has to be present.
+ *
+ *   3) The data in the block is compressed:
+ *	  The common case. This case is recognised by the fact that the run
+ *	  list entry has length L < X and lcn >= 0. The mapping pairs array
+ *	  stores this as normal with a run length of X and some specific
+ *	  delta_lcn, i.e. delta_lcn has to be present. This runlist entry is
+ *	  immediately followed by a sparse entry with length = X - L and
+ *	  lcn = -1. The latter entry is to make up the vcn counting to the
+ *	  full compression block size X.
+ *
+ * In fact, life is more complicated because adjacent entries of the same type
+ * can be coalesced. This means that one has to keep track of the number of
+ * clusters handled and work on a basis of X clusters at a time being one
+ * block. An example: if length L > X this means that this particular runlist
+ * entry contains a block of length X and part of one or more blocks of length
+ * L - X. Another example: if length L < X, this does not necessarily mean that
+ * the block is compressed as it might be that the lcn changes inside the block
+ * and hence the following runlist entry describes the continuation of the
+ * potentially compressed block. The block would be compressed if the
+ * following runlist entry describes at least X - L sparse clusters, thus
+ * making up the compression block length as described in point 3 above. (Of
+ * course, there can be several runlist entries with small lengths so that the
+ * sparse entry does not follow the first data containing entry with
+ * length < X.)
+ *
+ * NOTE: At the end of the compressed attribute value, there most likely is not
+ * just the right amount of data to make up a compression block, thus this data
+ * is not even attempted to be compressed. It is just stored as is, unless
+ * the number of clusters it occupies is reduced when compressed in which case
+ * it is stored as a compressed compression block, complete with sparse
+ * clusters at the end.
+ */
+
+/*
+ * enum - Flags for resident attributes (8-bit)
+ *
+ * RESIDENT_ATTR_IS_INDEXED: Attribute is referenced in an index.
+ *                           (e.g., part of an index key or entry)
+ *                           Has implications for deletion and modification:
+ *                            - Cannot be freely removed if indexed
+ *                            - Index must be updated when value changes
+ *                            - Used for attributes like $FILE_NAME in directories
+ */
+enum {
+	RESIDENT_ATTR_IS_INDEXED = 0x01,
+} __packed;
+
+/*
+ * struct attr_record - NTFS attribute record header
+ *
+ * Common header for both resident and non-resident attributes.
+ * Always aligned to an 8-byte boundary on disk.
+ * Located at attrs_offset in the MFT record (see struct mft_record).
+ *
+ * @type:           32-bit attribute type (ATTR_TYPE_* constants).
+ *                  Identifies the attribute
+ *                  (e.g. 0x10 = $STANDARD_INFORMATION).
+ * @length:         Total byte size of this attribute record (resident).
+ *                  8-byte aligned; used to locate the next attribute.
+ * @non_resident:   0 = resident attribute
+ *                  1 = non-resident attribute
+ * @name_length:    Number of Unicode characters in the attribute name.
+ *                  0 if unnamed (most system attributes are unnamed).
+ * @name_offset:    Byte offset from start of attribute record to the name.
+ *                  8-byte aligned; when creating, place at end of header.
+ * @flags:          Attribute flags (see ATTR_IS_COMPRESSED,
+ *                  ATTR_IS_ENCRYPTED, etc.).
+ *                  For resident: see RESIDENT_ATTR_* flags.
+ * @instance:       Unique instance number within this MFT record.
+ *                  Incremented via next_attr_instance; unique per record.
+ *
+ * Resident attributes (when @non_resident == 0):
+ * @data.resident.value_length:     Byte size of the attribute value.
+ * @data.resident.value_offset:     Byte offset from start of attribute
+ *                                  record to the value data.
+ *                                  8-byte aligned if name present.
+ * @data.resident.flags:            Resident-specific flags
+ * @data.resident.reserved:         Reserved/alignment to 8 bytes.
+ *
+ * Non-resident attributes (when @non_resident == 1):
+ * @data.non_resident.lowest_vcn:   Lowest valid VCN in this extent.
+ *                                  Usually 0 unless attribute list is used.
+ * @data.non_resident.highest_vcn:  Highest valid VCN in this extent.
+ *                                  -1 for zero-length, 0 for single extent.
+ * @data.non_resident.mapping_pairs_offset:
+ *                                  Byte offset to mapping pairs array
+ *                                  (VCN → LCN mappings).
+ *                                  8-byte aligned when creating.
+ * @data.non_resident.compression_unit:
+ *                                  Log2 of clusters per compression unit.
+ *                                  0 = not compressed.
+ *                                  WinNT4 used 4; sparse files use 0
+ *                                  on XP SP2+.
+ * @data.non_resident.reserved:     5 bytes for 8-byte alignment.
+ * @data.non_resident.allocated_size:
+ *                                  Allocated disk space in bytes.
+ *                                  For compressed: logical allocated size.
+ * @data.non_resident.data_size:    Logical attribute value size in bytes.
+ *                                  Can be larger than allocated_size if
+ *                                  compressed/sparse.
+ * @data.non_resident.initialized_size:
+ *                                  Initialized portion size in bytes.
+ *                                  Usually equals data_size.
+ * @data.non_resident.compressed_size:
+ *                                  Compressed on-disk size in bytes.
+ *                                  Only present when compressed or sparse.
+ *                                  Actual disk usage.
+ */
+struct attr_record {
+	__le32 type;
+	__le32 length;
+	u8 non_resident;
+	u8 name_length;
+	__le16 name_offset;
+	__le16 flags;
+	__le16 instance;
+	union {
+		struct {
+			__le32 value_length;
+			__le16 value_offset;
+			u8 flags;
+			s8 reserved;
+		} __packed resident;
+		struct {
+			__le64 lowest_vcn;
+			__le64 highest_vcn;
+			__le16 mapping_pairs_offset;
+			u8 compression_unit;
+			u8 reserved[5];
+			__le64 allocated_size;
+			__le64 data_size;
+			__le64 initialized_size;
+			__le64 compressed_size;
+		} __packed non_resident;
+	} __packed data;
+} __packed;
+
+/*
+ * enum - NTFS file attribute flags (32-bit)
+ *
+ * File attribute flags (32-bit) appearing in the file_attributes fields of the
+ * STANDARD_INFORMATION attribute of MFT_RECORDs and the FILENAME_ATTR
+ * attributes of MFT_RECORDs and directory index entries.
+ *
+ * All of the below flags appear in the directory index entries but only some
+ * appear in the STANDARD_INFORMATION attribute whilst only some others appear
+ * in the FILENAME_ATTR attribute of MFT_RECORDs.  Unless otherwise stated the
+ * flags appear in all of the above.
+ *
+ * FILE_ATTR_READONLY:         File is read-only.
+ * FILE_ATTR_HIDDEN:           File is hidden (not shown by default).
+ * FILE_ATTR_SYSTEM:           System file (protected by OS).
+ * FILE_ATTR_DIRECTORY:        Directory flag (reserved in NT; use MFT flag instead).
+ * FILE_ATTR_ARCHIVE:          File needs archiving (backup flag).
+ * FILE_ATTR_DEVICE:           Device file (rarely used).
+ * FILE_ATTR_NORMAL:           Normal file (no special attributes).
+ * FILE_ATTR_TEMPORARY:        Temporary file (delete on close).
+ * FILE_ATTR_SPARSE_FILE:      Sparse file (contains holes).
+ * FILE_ATTR_REPARSE_POINT:    Reparse point (junction, symlink, mount point).
+ * FILE_ATTR_COMPRESSED:       File is compressed.
+ * FILE_ATTR_OFFLINE:          File data is offline (not locally available).
+ * FILE_ATTR_NOT_CONTENT_INDEXED:
+ *                              File is excluded from content indexing.
+ * FILE_ATTR_ENCRYPTED:        File is encrypted (EFS).
+ * FILE_ATTR_VALID_FLAGS:      Mask of all valid flags for reading.
+ * FILE_ATTR_VALID_SET_FLAGS:  Mask of flags that can be set by user.
+ * FILE_ATTRIBUTE_RECALL_ON_OPEN:
+ *                              Recall data on open (cloud/HSM related).
+ * FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT:
+ *                              $FILE_NAME has duplicate index entry.
+ * FILE_ATTR_DUP_VIEW_INDEX_PRESENT:
+ *                              Duplicate view index present (object ID, quota, etc.).
+ */
+enum {
+	FILE_ATTR_READONLY		= cpu_to_le32(0x00000001),
+	FILE_ATTR_HIDDEN		= cpu_to_le32(0x00000002),
+	FILE_ATTR_SYSTEM		= cpu_to_le32(0x00000004),
+	/* Old DOS volid. Unused in NT.	= cpu_to_le32(0x00000008), */
+	FILE_ATTR_DIRECTORY		= cpu_to_le32(0x00000010),
+	FILE_ATTR_ARCHIVE		= cpu_to_le32(0x00000020),
+	FILE_ATTR_DEVICE		= cpu_to_le32(0x00000040),
+	FILE_ATTR_NORMAL		= cpu_to_le32(0x00000080),
+
+	FILE_ATTR_TEMPORARY		= cpu_to_le32(0x00000100),
+	FILE_ATTR_SPARSE_FILE		= cpu_to_le32(0x00000200),
+	FILE_ATTR_REPARSE_POINT		= cpu_to_le32(0x00000400),
+	FILE_ATTR_COMPRESSED		= cpu_to_le32(0x00000800),
+
+	FILE_ATTR_OFFLINE		= cpu_to_le32(0x00001000),
+	FILE_ATTR_NOT_CONTENT_INDEXED	= cpu_to_le32(0x00002000),
+	FILE_ATTR_ENCRYPTED		= cpu_to_le32(0x00004000),
+
+	FILE_ATTR_VALID_FLAGS		= cpu_to_le32(0x00007fb7),
+	FILE_ATTR_VALID_SET_FLAGS	= cpu_to_le32(0x000031a7),
+	FILE_ATTRIBUTE_RECALL_ON_OPEN	= cpu_to_le32(0x00040000),
+	FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT	= cpu_to_le32(0x10000000),
+	FILE_ATTR_DUP_VIEW_INDEX_PRESENT	= cpu_to_le32(0x20000000),
+};
+
+/*
+ * NOTE on times in NTFS: All times are in MS standard time format, i.e. they
+ * are the number of 100-nanosecond intervals since 1st January 1601, 00:00:00
+ * universal coordinated time (UTC). (In Linux time starts 1st January 1970,
+ * 00:00:00 UTC and is stored as the number of 1-second intervals since then.)
+ */
+
+/*
+ * struct standard_information - $STANDARD_INFORMATION attribute content
+ *
+ * NOTE: Always resident.
+ * NOTE: Present in all base file records on a volume.
+ * NOTE: There is conflicting information about the meaning of each of the time
+ *	 fields but the meaning as defined below has been verified to be
+ *	 correct by practical experimentation on Windows NT4 SP6a and is hence
+ *	 assumed to be the one and only correct interpretation.
+ *
+ * @creation_time:          File creation time (NTFS timestamp).
+ *                          Updated on filename change(?).
+ * @last_data_change_time:  Last modification time of data streams.
+ * @last_mft_change_time:   Last modification time of this MFT record.
+ * @last_access_time:       Last access time (approximate).
+ *                          Not updated on read-only volumes; can be disabled.
+ * @file_attributes:        File attribute flags (FILE_ATTR_* bits).
+ *
+ * Union (version-specific fields):
+ * @ver.v1.reserved12:      12 bytes reserved/alignment (NTFS 1.2 only).
+ *
+ * @ver.v3 (NTFS 3.x / Windows 2000+):
+ * @maximum_versions:       Max allowed file versions (0 = disabled).
+ * @version_number:         Current version number (0 if disabled).
+ * @class_id:               Class ID (from bidirectional index?).
+ * @owner_id:               Owner ID (maps to $Quota via $Q index).
+ * @security_id:            Security ID (maps to $Secure $SII/$SDS).
+ * @quota_charged:          Quota charge in bytes (0 if quotas disabled).
+ * @usn:                    Last USN from $UsnJrnl (0 if disabled).
+ */
+struct standard_information {
+	__le64 creation_time;
+	__le64 last_data_change_time;
+	__le64 last_mft_change_time;
+	__le64 last_access_time;
+	__le32 file_attributes;
+	union {
+		struct {
+			u8 reserved12[12];
+		} __packed v1;
+		struct {
+			__le32 maximum_versions;
+			__le32 version_number;
+			__le32 class_id;
+			__le32 owner_id;
+			__le32 security_id;
+			__le64 quota_charged;
+			__le64 usn;
+		} __packed v3;
+	} __packed ver;
+} __packed;
+
+/*
+ * struct attr_list_entry - Entry in $ATTRIBUTE_LIST attribute.
+ *
+ * @type:           Attribute type code (ATTR_TYPE_*).
+ * @length:         Byte size of this entry (8-byte aligned).
+ * @name_length:    Unicode char count of attribute name (0 if unnamed).
+ * @name_offset:    Byte offset from start of entry to name (always set).
+ * @lowest_vcn:     Lowest VCN of this attribute extent (usually 0).
+ *                  Signed value; non-zero when attribute spans extents.
+ * @mft_reference:  MFT record reference holding this attribute extent.
+ * @instance:       Attribute instance number (if lowest_vcn == 0); else 0.
+ * @name:           Variable Unicode name (use @name_offset when reading).
+ *
+ * - Can be either resident or non-resident.
+ * - Value consists of a sequence of variable length, 8-byte aligned,
+ * ATTR_LIST_ENTRY records.
+ * - The list is not terminated by anything at all! The only way to know when
+ * the end is reached is to keep track of the current offset and compare it to
+ * the attribute value size.
+ * - The attribute list attribute contains one entry for each attribute of
+ * the file in which the list is located, except for the list attribute
+ * itself. The list is sorted: first by attribute type, second by attribute
+ * name (if present), third by instance number. The extents of one
+ * non-resident attribute (if present) immediately follow after the initial
+ * extent. They are ordered by lowest_vcn and have their instance set to zero.
+ * It is not allowed to have two attributes with all sorting keys equal.
+ * - Further restrictions:
+ *	- If not resident, the vcn to lcn mapping array has to fit inside the
+ *	  base mft record.
+ *	- The attribute list attribute value has a maximum size of 256kb. This
+ *	  is imposed by the Windows cache manager.
+ * - Attribute lists are only used when the attributes of mft record do not
+ * fit inside the mft record despite all attributes (that can be made
+ * non-resident) having been made non-resident. This can happen e.g. when:
+ *	- File has a large number of hard links (lots of file name
+ *	  attributes present).
+ *	- The mapping pairs array of some non-resident attribute becomes so
+ *	  large due to fragmentation that it overflows the mft record.
+ *	- The security descriptor is very complex (not applicable to
+ *	  NTFS 3.0 volumes).
+ *	- There are many named streams.
+ */
+struct attr_list_entry {
+	__le32 type;
+	__le16 length;
+	u8 name_length;
+	u8 name_offset;
+	__le64 lowest_vcn;
+	__le64 mft_reference;
+	__le16 instance;
+	__le16 name[];
+} __packed;
+
+/*
+ * The maximum allowed length for a file name.
+ */
+#define MAXIMUM_FILE_NAME_LENGTH	255
+
+/*
+ * enum - Possible namespaces for filenames in ntfs (8-bit).
+ *
+ * FILE_NAME_POSIX        POSIX namespace (case sensitive, most permissive).
+ *                        Allows all Unicode except '\0' and '/'.
+ *                        WinNT/2k/2003 default utilities ignore case
+ *                        differences. SFU (Services For Unix) enables true
+ *                        case sensitivity.
+ *                        SFU restricts some chars: '"', '/', '<', '>', '\'.
+ * FILE_NAME_WIN32        Standard WinNT/2k long filename namespace
+ *                        (case insensitive).
+ *                        Disallows '\0', '"', '*', '/', ':', '<', '>', '?',
+ *                        '\', '|'. Names cannot end with '.' or space.
+ * FILE_NAME_DOS          DOS 8.3 namespace (uppercase only).
+ *                        Allows 8-bit chars > space except '"', '*', '+',
+ *                        ',', '/', ':', ';', '<', '=', '>', '?', '\'.
+ * FILE_NAME_WIN32_AND_DOS
+ *                        Win32 and DOS names are identical (single record).
+ *                        Value 0x03 indicates both are stored in one entry.
+ */
+enum {
+	FILE_NAME_POSIX		= 0x00,
+	FILE_NAME_WIN32		= 0x01,
+	FILE_NAME_DOS		= 0x02,
+	FILE_NAME_WIN32_AND_DOS	= 0x03,
+} __packed;
+
+/*
+ * struct file_name_attr - $FILE_NAME attribute content
+ *
+ * NOTE: Always resident.
+ * NOTE: All fields, except the parent_directory, are only updated when the
+ *	 filename is changed. Until then, they just become out of sync with
+ *	 reality and the more up to date values are present in the standard
+ *	 information attribute.
+ * NOTE: There is conflicting information about the meaning of each of the time
+ *	 fields but the meaning as defined below has been verified to be
+ *	 correct by practical experimentation on Windows NT4 SP6a and is hence
+ *	 assumed to be the one and only correct interpretation.
+ *
+ * @parent_directory:   MFT reference to parent directory.
+ * @creation_time:      File creation time (NTFS timestamp).
+ * @last_data_change_time:
+ *                      Last data modification time.
+ * @last_mft_change_time:
+ *                      Last MFT record modification time.
+ * @last_access_time:   Last access time (approximate; may not
+ *                      update always).
+ * @allocated_size:     On-disk allocated size for unnamed $DATA.
+ *                      Equals compressed_size if compressed/sparse.
+ *                      0 for directories or no $DATA.
+ *                      Multiple of cluster size.
+ * @data_size:          Logical size of unnamed $DATA.
+ *                      0 for directories or no $DATA.
+ * @file_attributes:    File attribute flags (FILE_ATTR_* bits).
+ * @type.ea.packed_ea_size:
+ *                      Size needed to pack EAs (if present).
+ * @type.ea.reserved:   Alignment padding.
+ * @type.rp.reparse_point_tag:
+ *                      Reparse point type (if reparse point, no EAs).
+ * @file_name_length:   Length of filename in Unicode characters.
+ * @file_name_type:     Namespace (FILE_NAME_POSIX, WIN32, DOS, etc.).
+ * @file_name:          Variable-length Unicode filename.
+ */
+struct file_name_attr {
+	__le64 parent_directory;
+	__le64 creation_time;
+	__le64 last_data_change_time;
+	__le64 last_mft_change_time;
+	__le64 last_access_time;
+	__le64 allocated_size;
+	__le64 data_size;
+	__le32 file_attributes;
+	union {
+		struct {
+			__le16 packed_ea_size;
+			__le16 reserved;
+		} __packed ea;
+		struct {
+			__le32 reparse_point_tag;
+		} __packed rp;
+	} __packed type;
+	u8 file_name_length;
+	u8 file_name_type;
+	__le16 file_name[];
+} __packed;
+
+/*
+ * struct guid - Globally Unique Identifier (GUID) structure
+ *
+ * GUID structures store globally unique identifiers (GUID). A GUID is a
+ * 128-bit value consisting of one group of eight hexadecimal digits, followed
+ * by three groups of four hexadecimal digits each, followed by one group of
+ * twelve hexadecimal digits. GUIDs are Microsoft's implementation of the
+ * distributed computing environment (DCE) universally unique identifier (UUID).
+ * Example of a GUID:
+ *	1F010768-5A73-BC91-0010A52216A7
+ *
+ * @data1:      First 32 bits (first 8 hex digits).
+ * @data2:      Next 16 bits (first group of 4 hex digits).
+ * @data3:      Next 16 bits (second group of 4 hex digits).
+ * @data4:      Final 64 bits (third group of 4 + last 12 hex digits).
+ *              data4[0-1]: third group; data4[2-7]: remaining part.
+ */
+struct guid {
+	__le32 data1;
+	__le16 data2;
+	__le16 data3;
+	u8 data4[8];
+} __packed;
+
+/*
+ * struct object_id_attr - $OBJECT_ID attribute content (NTFS 3.0+)
+ *
+ * NOTE: Always resident.
+ *
+ * @object_id:          Unique 128-bit GUID assigned to the file.
+ *                      Core identifier; always present.
+ *
+ * Optional extended info (union; total value size 16–64 bytes):
+ * @extended_info.birth_volume_id:
+ *                      Birth volume GUID (where file was first created).
+ * @extended_info.birth_object_id:
+ *                      Birth object GUID (original ID before copy/move).
+ * @extended_info.domain_id:
+ *                      Domain GUID (usually zero; reserved).
+ */
+struct object_id_attr {
+	struct guid object_id;
+	union {
+		struct {
+			struct guid birth_volume_id;
+			struct guid birth_object_id;
+			struct guid domain_id;
+		} __packed;
+		u8 extended_info[48];
+	} __packed;
+} __packed;
+
+/*
+ * enum - RIDs (Relative Identifiers) in Windows/NTFS security
+ *
+ * These relative identifiers (RIDs) are used with the above identifier
+ * authorities to make up universal well-known SIDs.
+ *
+ * SECURITY_NULL_RID              S-1-0 (Null authority)
+ * SECURITY_WORLD_RID             S-1-1 (World/Everyone)
+ * SECURITY_LOCAL_RID             S-1-2 (Local)
+ * SECURITY_CREATOR_OWNER_RID     S-1-3-0 (Creator Owner)
+ * SECURITY_CREATOR_GROUP_RID     S-1-3-1 (Creator Group)
+ * SECURITY_CREATOR_OWNER_SERVER_RID S-1-3-2 (Server Creator Owner)
+ * SECURITY_CREATOR_GROUP_SERVER_RID S-1-3-3 (Server Creator Group)
+ * SECURITY_DIALUP_RID            S-1-5-1 (Dialup)
+ * SECURITY_NETWORK_RID           S-1-5-2 (Network)
+ * SECURITY_BATCH_RID             S-1-5-3 (Batch)
+ * SECURITY_INTERACTIVE_RID       S-1-5-4 (Interactive)
+ * SECURITY_SERVICE_RID           S-1-5-6 (Service)
+ * SECURITY_ANONYMOUS_LOGON_RID   S-1-5-7 (Anonymous Logon)
+ * SECURITY_PROXY_RID             S-1-5-8 (Proxy)
+ * SECURITY_ENTERPRISE_CONTROLLERS_RID S-1-5-9 (Enterprise DCs)
+ * SECURITY_SERVER_LOGON_RID      S-1-5-9 (Server Logon alias)
+ * SECURITY_PRINCIPAL_SELF_RID    S-1-5-10 (Self/PrincipalSelf)
+ * SECURITY_AUTHENTICATED_USER_RID S-1-5-11 (Authenticated Users)
+ * SECURITY_RESTRICTED_CODE_RID   S-1-5-12 (Restricted Code)
+ * SECURITY_TERMINAL_SERVER_RID   S-1-5-13 (Terminal Server)
+ * SECURITY_LOGON_IDS_RID         S-1-5-5 (Logon session IDs base)
+ * SECURITY_LOCAL_SYSTEM_RID      S-1-5-18 (Local System)
+ * SECURITY_NT_NON_UNIQUE         S-1-5-21 (NT non-unique authority)
+ * SECURITY_BUILTIN_DOMAIN_RID    S-1-5-32 (Built-in domain)
+ *
+ * Built-in domain relative RIDs (S-1-5-32-...):
+ * Users:
+ * DOMAIN_USER_RID_ADMIN          Administrator
+ * DOMAIN_USER_RID_GUEST          Guest
+ * DOMAIN_USER_RID_KRBTGT         krbtgt (Kerberos ticket-granting)
+ *
+ * Groups:
+ * DOMAIN_GROUP_RID_ADMINS        Administrators
+ * DOMAIN_GROUP_RID_USERS         Users
+ * DOMAIN_GROUP_RID_GUESTS        Guests
+ * DOMAIN_GROUP_RID_COMPUTERS     Computers
+ * DOMAIN_GROUP_RID_CONTROLLERS   Domain Controllers
+ * DOMAIN_GROUP_RID_CERT_ADMINS   Cert Publishers
+ * DOMAIN_GROUP_RID_SCHEMA_ADMINS Schema Admins
+ * DOMAIN_GROUP_RID_ENTERPRISE_ADMINS Enterprise Admins
+ * DOMAIN_GROUP_RID_POLICY_ADMINS Policy Admins (if present)
+ *
+ * Aliases:
+ * DOMAIN_ALIAS_RID_ADMINS        Administrators alias
+ * DOMAIN_ALIAS_RID_USERS         Users alias
+ * DOMAIN_ALIAS_RID_GUESTS        Guests alias
+ * DOMAIN_ALIAS_RID_POWER_USERS   Power Users
+ * DOMAIN_ALIAS_RID_ACCOUNT_OPS   Account Operators
+ * DOMAIN_ALIAS_RID_SYSTEM_OPS    Server Operators
+ * DOMAIN_ALIAS_RID_PRINT_OPS     Print Operators
+ * DOMAIN_ALIAS_RID_BACKUP_OPS    Backup Operators
+ * DOMAIN_ALIAS_RID_REPLICATOR    Replicator
+ * DOMAIN_ALIAS_RID_RAS_SERVERS   RAS Servers
+ * DOMAIN_ALIAS_RID_PREW2KCOMPACCESS Pre-Windows 2000 Compatible Access
+ *
+ * Note: The relative identifier (RID) refers to the portion of a SID, which
+ * identifies a user or group in relation to the authority that issued the SID.
+ * For example, the universal well-known SID Creator Owner ID (S-1-3-0) is
+ * made up of the identifier authority SECURITY_CREATOR_SID_AUTHORITY (3) and
+ * the relative identifier SECURITY_CREATOR_OWNER_RID (0).
+ */
+enum {					/* Identifier authority. */
+	SECURITY_NULL_RID			= 0,	/* S-1-0 */
+	SECURITY_WORLD_RID			= 0,	/* S-1-1 */
+	SECURITY_LOCAL_RID			= 0,	/* S-1-2 */
+
+	SECURITY_CREATOR_OWNER_RID		= 0,	/* S-1-3 */
+	SECURITY_CREATOR_GROUP_RID		= 1,	/* S-1-3 */
+
+	SECURITY_CREATOR_OWNER_SERVER_RID	= 2,	/* S-1-3 */
+	SECURITY_CREATOR_GROUP_SERVER_RID	= 3,	/* S-1-3 */
+
+	SECURITY_DIALUP_RID			= 1,
+	SECURITY_NETWORK_RID			= 2,
+	SECURITY_BATCH_RID			= 3,
+	SECURITY_INTERACTIVE_RID		= 4,
+	SECURITY_SERVICE_RID			= 6,
+	SECURITY_ANONYMOUS_LOGON_RID		= 7,
+	SECURITY_PROXY_RID			= 8,
+	SECURITY_ENTERPRISE_CONTROLLERS_RID	= 9,
+	SECURITY_SERVER_LOGON_RID		= 9,
+	SECURITY_PRINCIPAL_SELF_RID		= 0xa,
+	SECURITY_AUTHENTICATED_USER_RID		= 0xb,
+	SECURITY_RESTRICTED_CODE_RID		= 0xc,
+	SECURITY_TERMINAL_SERVER_RID		= 0xd,
+
+	SECURITY_LOGON_IDS_RID			= 5,
+	SECURITY_LOGON_IDS_RID_COUNT		= 3,
+
+	SECURITY_LOCAL_SYSTEM_RID		= 0x12,
+
+	SECURITY_NT_NON_UNIQUE			= 0x15,
+
+	SECURITY_BUILTIN_DOMAIN_RID		= 0x20,
+
+	/*
+	 * Well-known domain relative sub-authority values (RIDs).
+	 */
+
+	/* Users. */
+	DOMAIN_USER_RID_ADMIN			= 0x1f4,
+	DOMAIN_USER_RID_GUEST			= 0x1f5,
+	DOMAIN_USER_RID_KRBTGT			= 0x1f6,
+
+	/* Groups. */
+	DOMAIN_GROUP_RID_ADMINS			= 0x200,
+	DOMAIN_GROUP_RID_USERS			= 0x201,
+	DOMAIN_GROUP_RID_GUESTS			= 0x202,
+	DOMAIN_GROUP_RID_COMPUTERS		= 0x203,
+	DOMAIN_GROUP_RID_CONTROLLERS		= 0x204,
+	DOMAIN_GROUP_RID_CERT_ADMINS		= 0x205,
+	DOMAIN_GROUP_RID_SCHEMA_ADMINS		= 0x206,
+	DOMAIN_GROUP_RID_ENTERPRISE_ADMINS	= 0x207,
+	DOMAIN_GROUP_RID_POLICY_ADMINS		= 0x208,
+
+	/* Aliases. */
+	DOMAIN_ALIAS_RID_ADMINS			= 0x220,
+	DOMAIN_ALIAS_RID_USERS			= 0x221,
+	DOMAIN_ALIAS_RID_GUESTS			= 0x222,
+	DOMAIN_ALIAS_RID_POWER_USERS		= 0x223,
+
+	DOMAIN_ALIAS_RID_ACCOUNT_OPS		= 0x224,
+	DOMAIN_ALIAS_RID_SYSTEM_OPS		= 0x225,
+	DOMAIN_ALIAS_RID_PRINT_OPS		= 0x226,
+	DOMAIN_ALIAS_RID_BACKUP_OPS		= 0x227,
+
+	DOMAIN_ALIAS_RID_REPLICATOR		= 0x228,
+	DOMAIN_ALIAS_RID_RAS_SERVERS		= 0x229,
+	DOMAIN_ALIAS_RID_PREW2KCOMPACCESS	= 0x22a,
+};
+
+/*
+ * The universal well-known SIDs:
+ *
+ *	NULL_SID			S-1-0-0
+ *	WORLD_SID			S-1-1-0
+ *	LOCAL_SID			S-1-2-0
+ *	CREATOR_OWNER_SID		S-1-3-0
+ *	CREATOR_GROUP_SID		S-1-3-1
+ *	CREATOR_OWNER_SERVER_SID	S-1-3-2
+ *	CREATOR_GROUP_SERVER_SID	S-1-3-3
+ *
+ *	(Non-unique IDs)		S-1-4
+ *
+ * NT well-known SIDs:
+ *
+ *	NT_AUTHORITY_SID	S-1-5
+ *	DIALUP_SID		S-1-5-1
+ *
+ *	NETWORD_SID		S-1-5-2
+ *	BATCH_SID		S-1-5-3
+ *	INTERACTIVE_SID		S-1-5-4
+ *	SERVICE_SID		S-1-5-6
+ *	ANONYMOUS_LOGON_SID	S-1-5-7		(aka null logon session)
+ *	PROXY_SID		S-1-5-8
+ *	SERVER_LOGON_SID	S-1-5-9		(aka domain controller account)
+ *	SELF_SID		S-1-5-10	(self RID)
+ *	AUTHENTICATED_USER_SID	S-1-5-11
+ *	RESTRICTED_CODE_SID	S-1-5-12	(running restricted code)
+ *	TERMINAL_SERVER_SID	S-1-5-13	(running on terminal server)
+ *
+ *	(Logon IDs)		S-1-5-5-X-Y
+ *
+ *	(NT non-unique IDs)	S-1-5-0x15-...
+ *
+ *	(Built-in domain)	S-1-5-0x20
+ */
+
+/*
+ * struct ntfs_sid - Security Identifier (SID) structure
+ *
+ * @revision:            SID revision level (usually 1).
+ * @sub_authority_count: Number of sub-authorities (1 or more).
+ * @identifier_authority:
+ *                       48-bit identifier authority (S-1-x-...).
+ *                       @parts.high_part: high 16 bits.
+ *                       @parts.low_part: low 32 bits.
+ *                       @value: raw 6-byte array.
+ * @sub_authority:       Variable array of 32-bit RIDs.
+ *                       At least one; defines the SID relative to authority.
+ *
+ * The SID structure is a variable-length structure used to uniquely identify
+ * users or groups. SID stands for security identifier.
+ *
+ * The standard textual representation of the SID is of the form:
+ *	S-R-I-S-S...
+ * Where:
+ *    - The first "S" is the literal character 'S' identifying the following
+ *	digits as a SID.
+ *    - R is the revision level of the SID expressed as a sequence of digits
+ *	either in decimal or hexadecimal (if the later, prefixed by "0x").
+ *    - I is the 48-bit identifier_authority, expressed as digits as R above.
+ *    - S... is one or more sub_authority values, expressed as digits as above.
+ *
+ * Example SID; the domain-relative SID of the local Administrators group on
+ * Windows NT/2k:
+ *	S-1-5-32-544
+ * This translates to a SID with:
+ *	revision = 1,
+ *	sub_authority_count = 2,
+ *	identifier_authority = {0,0,0,0,0,5},	// SECURITY_NT_AUTHORITY
+ *	sub_authority[0] = 32,			// SECURITY_BUILTIN_DOMAIN_RID
+ *	sub_authority[1] = 544			// DOMAIN_ALIAS_RID_ADMINS
+ */
+struct ntfs_sid {
+	u8 revision;
+	u8 sub_authority_count;
+	union {
+		struct {
+			u16 high_part;
+			u32 low_part;
+		} __packed parts;
+		u8 value[6];
+	} identifier_authority;
+	__le32 sub_authority[];
+} __packed;
+
+/*
+ * enum - Predefined ACE types (8-bit) for NTFS security descriptors
+ *
+ * ACCESS_MIN_MS_ACE_TYPE:         Minimum MS ACE type (0).
+ * ACCESS_ALLOWED_ACE_TYPE:        Allow access (standard ACE).
+ * ACCESS_DENIED_ACE_TYPE:         Deny access (standard ACE).
+ * SYSTEM_AUDIT_ACE_TYPE:          Audit successful/failed access.
+ * SYSTEM_ALARM_ACE_TYPE:          Alarm on access (not in Win2k+).
+ * ACCESS_MAX_MS_V2_ACE_TYPE:      Max for V2 ACE types.
+ * ACCESS_ALLOWED_COMPOUND_ACE_TYPE:
+ *                                 Compound ACE (legacy).
+ * ACCESS_MAX_MS_V3_ACE_TYPE:      Max for V3 ACE types.
+ * ACCESS_MIN_MS_OBJECT_ACE_TYPE:  Min for object ACE types (Win2k+).
+ * ACCESS_ALLOWED_OBJECT_ACE_TYPE: Allow with object-specific rights.
+ * ACCESS_DENIED_OBJECT_ACE_TYPE:  Deny with object-specific rights.
+ * SYSTEM_AUDIT_OBJECT_ACE_TYPE:   Audit with object-specific rights.
+ * SYSTEM_ALARM_OBJECT_ACE_TYPE:   Alarm with object-specific rights.
+ * ACCESS_MAX_MS_OBJECT_ACE_TYPE:  Max for object ACE types.
+ * ACCESS_MAX_MS_V4_ACE_TYPE:      Max for V4 ACE types.
+ * ACCESS_MAX_MS_ACE_TYPE:         Overall max ACE type (WinNT/2k).
+ */
+enum {
+	ACCESS_MIN_MS_ACE_TYPE			= 0,
+	ACCESS_ALLOWED_ACE_TYPE			= 0,
+	ACCESS_DENIED_ACE_TYPE			= 1,
+	SYSTEM_AUDIT_ACE_TYPE			= 2,
+	SYSTEM_ALARM_ACE_TYPE			= 3,
+	ACCESS_MAX_MS_V2_ACE_TYPE		= 3,
+
+	ACCESS_ALLOWED_COMPOUND_ACE_TYPE	= 4,
+	ACCESS_MAX_MS_V3_ACE_TYPE		= 4,
+	ACCESS_MIN_MS_OBJECT_ACE_TYPE		= 5,
+	ACCESS_ALLOWED_OBJECT_ACE_TYPE		= 5,
+	ACCESS_DENIED_OBJECT_ACE_TYPE		= 6,
+	SYSTEM_AUDIT_OBJECT_ACE_TYPE		= 7,
+	SYSTEM_ALARM_OBJECT_ACE_TYPE		= 8,
+	ACCESS_MAX_MS_OBJECT_ACE_TYPE		= 8,
+
+	ACCESS_MAX_MS_V4_ACE_TYPE		= 8,
+	ACCESS_MAX_MS_ACE_TYPE			= 8,
+} __packed;
+
+/*
+ * enum - ACE inheritance and audit flags (8-bit)
+ *
+ * OBJECT_INHERIT_ACE:         Object inherit (files inherit this ACE).
+ * CONTAINER_INHERIT_ACE:      Container inherit (subdirectories inherit).
+ * NO_PROPAGATE_INHERIT_ACE:   No propagation (stop inheritance after this level).
+ * INHERIT_ONLY_ACE:           Inherit only (not applied to current object).
+ * INHERITED_ACE:              ACE was inherited (Win2k+ only).
+ * VALID_INHERIT_FLAGS:        Mask of all valid inheritance flags (0x1f).
+ * SUCCESSFUL_ACCESS_ACE_FLAG: Audit successful access (system audit ACE).
+ * FAILED_ACCESS_ACE_FLAG:     Audit failed access (system audit ACE).
+ *
+ * SUCCESSFUL_ACCESS_ACE_FLAG is only used with system audit and alarm ACE
+ * types to indicate that a message is generated (in Windows!) for successful
+ * accesses.
+ *
+ * FAILED_ACCESS_ACE_FLAG is only used with system audit and alarm ACE types
+ * to indicate that a message is generated (in Windows!) for failed accesses.
+ */
+enum {
+	OBJECT_INHERIT_ACE		= 0x01,
+	CONTAINER_INHERIT_ACE		= 0x02,
+	NO_PROPAGATE_INHERIT_ACE	= 0x04,
+	INHERIT_ONLY_ACE		= 0x08,
+	INHERITED_ACE			= 0x10,
+	VALID_INHERIT_FLAGS		= 0x1f,
+	SUCCESSFUL_ACCESS_ACE_FLAG	= 0x40,
+	FAILED_ACCESS_ACE_FLAG		= 0x80,
+} __packed;
+
+/*
+ * enum - NTFS access rights masks (32-bit)
+ *
+ * FILE_READ_DATA / FILE_LIST_DIRECTORY:  Read file data / list dir contents.
+ * FILE_WRITE_DATA / FILE_ADD_FILE:       Write file data / create file in dir.
+ * FILE_APPEND_DATA / FILE_ADD_SUBDIRECTORY: Append data / create subdir.
+ * FILE_READ_EA:                          Read extended attributes.
+ * FILE_WRITE_EA:                         Write extended attributes.
+ * FILE_EXECUTE / FILE_TRAVERSE:          Execute file / traverse dir.
+ * FILE_DELETE_CHILD:                     Delete children in dir.
+ * FILE_READ_ATTRIBUTES:                  Read attributes.
+ * FILE_WRITE_ATTRIBUTES:                 Write attributes.
+ *
+ * Standard rights (object-independent):
+ * DELETE:                                Delete object.
+ * READ_CONTROL:                          Read security descriptor/owner.
+ * WRITE_DAC:                             Modify DACL.
+ * WRITE_OWNER:                           Change owner.
+ * SYNCHRONIZE:                           Wait on object signal state.
+ *
+ * Combinations:
+ * STANDARD_RIGHTS_READ / WRITE / EXECUTE: Aliases for READ_CONTROL.
+ * STANDARD_RIGHTS_REQUIRED:              DELETE + READ_CONTROL +
+ *                                        WRITE_DAC + WRITE_OWNER.
+ * STANDARD_RIGHTS_ALL:                   Above + SYNCHRONIZE.
+ *
+ * System/access types:
+ * ACCESS_SYSTEM_SECURITY:                Access system ACL.
+ * MAXIMUM_ALLOWED:                       Maximum allowed access.
+ *
+ * Generic rights (high bits, map to specific/standard):
+ * GENERIC_ALL:                           Full access.
+ * GENERIC_EXECUTE:                       Execute/traverse.
+ * GENERIC_WRITE:                         Write (append, attrs, data, EA, etc.).
+ * GENERIC_READ:                          Read (attrs, data, EA, etc.).
+ *
+ * The specific rights (bits 0 to 15).  These depend on the type of the object
+ * being secured by the ACE.
+ */
+enum {
+	FILE_READ_DATA			= cpu_to_le32(0x00000001),
+	FILE_LIST_DIRECTORY		= cpu_to_le32(0x00000001),
+	FILE_WRITE_DATA			= cpu_to_le32(0x00000002),
+	FILE_ADD_FILE			= cpu_to_le32(0x00000002),
+	FILE_APPEND_DATA		= cpu_to_le32(0x00000004),
+	FILE_ADD_SUBDIRECTORY		= cpu_to_le32(0x00000004),
+	FILE_READ_EA			= cpu_to_le32(0x00000008),
+	FILE_WRITE_EA			= cpu_to_le32(0x00000010),
+	FILE_EXECUTE			= cpu_to_le32(0x00000020),
+	FILE_TRAVERSE			= cpu_to_le32(0x00000020),
+	FILE_DELETE_CHILD		= cpu_to_le32(0x00000040),
+	FILE_READ_ATTRIBUTES		= cpu_to_le32(0x00000080),
+	FILE_WRITE_ATTRIBUTES		= cpu_to_le32(0x00000100),
+	DELETE				= cpu_to_le32(0x00010000),
+	READ_CONTROL			= cpu_to_le32(0x00020000),
+	WRITE_DAC			= cpu_to_le32(0x00040000),
+	WRITE_OWNER			= cpu_to_le32(0x00080000),
+	SYNCHRONIZE			= cpu_to_le32(0x00100000),
+	STANDARD_RIGHTS_READ		= cpu_to_le32(0x00020000),
+	STANDARD_RIGHTS_WRITE		= cpu_to_le32(0x00020000),
+	STANDARD_RIGHTS_EXECUTE		= cpu_to_le32(0x00020000),
+	STANDARD_RIGHTS_REQUIRED	= cpu_to_le32(0x000f0000),
+	STANDARD_RIGHTS_ALL		= cpu_to_le32(0x001f0000),
+	ACCESS_SYSTEM_SECURITY		= cpu_to_le32(0x01000000),
+	MAXIMUM_ALLOWED			= cpu_to_le32(0x02000000),
+	GENERIC_ALL			= cpu_to_le32(0x10000000),
+	GENERIC_EXECUTE			= cpu_to_le32(0x20000000),
+	GENERIC_WRITE			= cpu_to_le32(0x40000000),
+	GENERIC_READ			= cpu_to_le32(0x80000000),
+};
+
+/*
+ * struct ntfs_ace - Access Control Entry (ACE) structure
+ *
+ * @type: ACE type (ACCESS_ALLOWED_ACE_TYPE, ACCESS_DENIED_ACE_TYPE, etc.).
+ * @flags: Inheritance and audit flags (OBJECT_INHERIT_ACE, etc.).
+ * @size: Total byte size of this ACE (header + SID + variable data).
+ * @mask: Access rights mask (FILE_READ_DATA, DELETE, GENERIC_ALL, etc.).
+ * @sid: Security Identifier (SID) this ACE applies to.
+ */
+struct ntfs_ace {
+	u8 type;
+	u8 flags;
+	__le16 size;
+	__le32 mask;
+	struct ntfs_sid sid;
+} __packed;
+
+/*
+ * The object ACE flags (32-bit).
+ */
+enum {
+	ACE_OBJECT_TYPE_PRESENT			= cpu_to_le32(1),
+	ACE_INHERITED_OBJECT_TYPE_PRESENT	= cpu_to_le32(2),
+};
+
+/*
+ * struct ntfs_acl - NTFS Access Control List (ACL) header
+ *
+ * An ACL is an access-control list (ACL).
+ * An ACL starts with an ACL header structure, which specifies the size of
+ * the ACL and the number of ACEs it contains. The ACL header is followed by
+ * zero or more access control entries (ACEs). The ACL as well as each ACE
+ * are aligned on 4-byte boundaries.
+ *
+ * @revision:           ACL revision level (usually 2 or 4).
+ * @alignment1:         Padding/alignment byte (zero).
+ * @size:               Total allocated size in bytes (header + all ACEs +
+ *                      free space).
+ * @ace_count:          Number of ACE entries following the header.
+ * @alignment2:         Padding/alignment (zero).
+ */
+struct ntfs_acl {
+	u8 revision;
+	u8 alignment1;
+	__le16 size;
+	__le16 ace_count;
+	__le16 alignment2;
+} __packed;
+
+static_assert(sizeof(struct ntfs_acl) == 8);
+
+/*
+ * The security descriptor control flags (16-bit).
+ *
+ * SE_OWNER_DEFAULTED - This boolean flag, when set, indicates that the SID
+ *	pointed to by the Owner field was provided by a defaulting mechanism
+ *	rather than explicitly provided by the original provider of the
+ *	security descriptor.  This may affect the treatment of the SID with
+ *	respect to inheritance of an owner.
+ *
+ * SE_GROUP_DEFAULTED - This boolean flag, when set, indicates that the SID in
+ *	the Group field was provided by a defaulting mechanism rather than
+ *	explicitly provided by the original provider of the security
+ *	descriptor.  This may affect the treatment of the SID with respect to
+ *	inheritance of a primary group.
+ *
+ * SE_DACL_PRESENT - This boolean flag, when set, indicates that the security
+ *	descriptor contains a discretionary ACL.  If this flag is set and the
+ *	Dacl field of the SECURITY_DESCRIPTOR is null, then a null ACL is
+ *	explicitly being specified.
+ *
+ * SE_DACL_DEFAULTED - This boolean flag, when set, indicates that the ACL
+ *	pointed to by the Dacl field was provided by a defaulting mechanism
+ *	rather than explicitly provided by the original provider of the
+ *	security descriptor.  This may affect the treatment of the ACL with
+ *	respect to inheritance of an ACL.  This flag is ignored if the
+ *	DaclPresent flag is not set.
+ *
+ * SE_SACL_PRESENT - This boolean flag, when set,  indicates that the security
+ *	descriptor contains a system ACL pointed to by the Sacl field.  If this
+ *	flag is set and the Sacl field of the SECURITY_DESCRIPTOR is null, then
+ *	an empty (but present) ACL is being specified.
+ *
+ * SE_SACL_DEFAULTED - This boolean flag, when set, indicates that the ACL
+ *	pointed to by the Sacl field was provided by a defaulting mechanism
+ *	rather than explicitly provided by the original provider of the
+ *	security descriptor.  This may affect the treatment of the ACL with
+ *	respect to inheritance of an ACL.  This flag is ignored if the
+ *	SaclPresent flag is not set.
+ *
+ * SE_SELF_RELATIVE - This boolean flag, when set, indicates that the security
+ *	descriptor is in self-relative form.  In this form, all fields of the
+ *	security descriptor are contiguous in memory and all pointer fields are
+ *	expressed as offsets from the beginning of the security descriptor.
+ */
+enum {
+	SE_OWNER_DEFAULTED		= cpu_to_le16(0x0001),
+	SE_GROUP_DEFAULTED		= cpu_to_le16(0x0002),
+	SE_DACL_PRESENT			= cpu_to_le16(0x0004),
+	SE_DACL_DEFAULTED		= cpu_to_le16(0x0008),
+
+	SE_SACL_PRESENT			= cpu_to_le16(0x0010),
+	SE_SACL_DEFAULTED		= cpu_to_le16(0x0020),
+
+	SE_DACL_AUTO_INHERIT_REQ	= cpu_to_le16(0x0100),
+	SE_SACL_AUTO_INHERIT_REQ	= cpu_to_le16(0x0200),
+	SE_DACL_AUTO_INHERITED		= cpu_to_le16(0x0400),
+	SE_SACL_AUTO_INHERITED		= cpu_to_le16(0x0800),
+
+	SE_DACL_PROTECTED		= cpu_to_le16(0x1000),
+	SE_SACL_PROTECTED		= cpu_to_le16(0x2000),
+	SE_RM_CONTROL_VALID		= cpu_to_le16(0x4000),
+	SE_SELF_RELATIVE		= cpu_to_le16(0x8000)
+} __packed;
+
+/*
+ * struct security_descriptor_relative - Relative security descriptor
+ *
+ * Self-relative security descriptor. Contains the owner and group SIDs as well
+ * as the sacl and dacl ACLs inside the security descriptor itself.
+ *
+ * @revision:          Security descriptor revision (usually 1).
+ * @alignment:         Padding/alignment byte (zero).
+ * @control:           Control flags (SE_OWNER_DEFAULTED, SE_DACL_PRESENT,
+ *                     SE_SACL_PRESENT, SE_SACL_AUTO_INHERITED, etc.).
+ * @owner:             Byte offset to owner SID (from start of descriptor).
+ *                     0 if no owner SID present.
+ * @group:             Byte offset to primary group SID.
+ *                     0 if no group SID present.
+ * @sacl:              Byte offset to System ACL (SACL).
+ *                     Valid only if SE_SACL_PRESENT in @control.
+ *                     0 means NULL SACL.
+ * @dacl:              Byte offset to Discretionary ACL (DACL).
+ *                     Valid only if SE_DACL_PRESENT in @control.
+ *                     0 means NULL DACL (full access granted).
+ */
+struct security_descriptor_relative {
+	u8 revision;
+	u8 alignment;
+	__le16 control;
+	__le32 owner;
+	__le32 group;
+	__le32 sacl;
+	__le32 dacl;
+} __packed;
+
+static_assert(sizeof(struct security_descriptor_relative) == 20);
+
+/*
+ * On NTFS 3.0+, all security descriptors are stored in FILE_Secure. Only one
+ * referenced instance of each unique security descriptor is stored.
+ *
+ * FILE_Secure contains no unnamed data attribute, i.e. it has zero length. It
+ * does, however, contain two indexes ($SDH and $SII) as well as a named data
+ * stream ($SDS).
+ *
+ * Every unique security descriptor is assigned a unique security identifier
+ * (security_id, not to be confused with a SID). The security_id is unique for
+ * the NTFS volume and is used as an index into the $SII index, which maps
+ * security_ids to the security descriptor's storage location within the $SDS
+ * data attribute. The $SII index is sorted by ascending security_id.
+ *
+ * A simple hash is computed from each security descriptor. This hash is used
+ * as an index into the $SDH index, which maps security descriptor hashes to
+ * the security descriptor's storage location within the $SDS data attribute.
+ * The $SDH index is sorted by security descriptor hash and is stored in a B+
+ * tree. When searching $SDH (with the intent of determining whether or not a
+ * new security descriptor is already present in the $SDS data stream), if a
+ * matching hash is found, but the security descriptors do not match, the
+ * search in the $SDH index is continued, searching for a next matching hash.
+ *
+ * When a precise match is found, the security_id coresponding to the security
+ * descriptor in the $SDS attribute is read from the found $SDH index entry and
+ * is stored in the $STANDARD_INFORMATION attribute of the file/directory to
+ * which the security descriptor is being applied. The $STANDARD_INFORMATION
+ * attribute is present in all base mft records (i.e. in all files and
+ * directories).
+ *
+ * If a match is not found, the security descriptor is assigned a new unique
+ * security_id and is added to the $SDS data attribute. Then, entries
+ * referencing the this security descriptor in the $SDS data attribute are
+ * added to the $SDH and $SII indexes.
+ *
+ * Note: Entries are never deleted from FILE_Secure, even if nothing
+ * references an entry any more.
+ */
+
+/*
+ * struct sii_index_key - Key for $SII index in $Secure file
+ *
+ * The index entry key used in the $SII index. The collation type is
+ * COLLATION_NTOFS_ULONG.
+ *
+ * @security_id:    32-bit security identifier.
+ *                  Unique ID assigned to a security descriptor.
+ */
+struct sii_index_key {
+	__le32 security_id;
+} __packed;
+
+/*
+ * struct sdh_index_key - Key for $SDH index in $Secure file
+ *
+ * The index entry key used in the $SDH index. The keys are sorted first by
+ * hash and then by security_id. The collation rule is
+ * COLLATION_NTOFS_SECURITY_HASH.
+ *
+ * @hash:           32-bit hash of the security descriptor.
+ *                  Used for quick collision checks and indexing.
+ * @security_id:    32-bit security identifier.
+ *                  Unique ID assigned to the descriptor.
+ */
+struct sdh_index_key {
+	__le32 hash;
+	__le32 security_id;
+} __packed;
+
+/*
+ * enum - NTFS volume flags (16-bit)
+ *
+ * These flags are stored in $VolumeInformation attribute.
+ * They indicate volume state and required actions.
+ *
+ * VOLUME_IS_DIRTY:                Volume is dirty (needs chkdsk).
+ * VOLUME_RESIZE_LOG_FILE:         Resize LogFile on next mount.
+ * VOLUME_UPGRADE_ON_MOUNT:        Upgrade volume on mount (old NTFS).
+ * VOLUME_MOUNTED_ON_NT4:          Mounted on NT4 (compatibility flag).
+ * VOLUME_DELETE_USN_UNDERWAY:     USN journal deletion in progress.
+ * VOLUME_REPAIR_OBJECT_ID:        Repair $ObjId on next mount.
+ * VOLUME_CHKDSK_UNDERWAY:         Chkdsk is running.
+ * VOLUME_MODIFIED_BY_CHKDSK:      Modified by chkdsk.
+ * VOLUME_FLAGS_MASK:              Mask of all valid flags (0xc03f).
+ * VOLUME_MUST_MOUNT_RO_MASK:      Flags forcing read-only mount (0xc027).
+ *                                 If any set, mount read-only.
+ */
+enum {
+	VOLUME_IS_DIRTY			= cpu_to_le16(0x0001),
+	VOLUME_RESIZE_LOG_FILE		= cpu_to_le16(0x0002),
+	VOLUME_UPGRADE_ON_MOUNT		= cpu_to_le16(0x0004),
+	VOLUME_MOUNTED_ON_NT4		= cpu_to_le16(0x0008),
+
+	VOLUME_DELETE_USN_UNDERWAY	= cpu_to_le16(0x0010),
+	VOLUME_REPAIR_OBJECT_ID		= cpu_to_le16(0x0020),
+
+	VOLUME_CHKDSK_UNDERWAY		= cpu_to_le16(0x4000),
+	VOLUME_MODIFIED_BY_CHKDSK	= cpu_to_le16(0x8000),
+
+	VOLUME_FLAGS_MASK		= cpu_to_le16(0xc03f),
+
+	VOLUME_MUST_MOUNT_RO_MASK	= cpu_to_le16(0xc027),
+} __packed;
+
+/*
+ * struct volume_information - $VOLUME_INFORMATION (0x70)
+ *
+ * @reserved:       Reserved 64-bit field (currently unused).
+ * @major_ver:      Major NTFS version number (e.g., 3 for NTFS 3.1).
+ * @minor_ver:      Minor NTFS version number (e.g., 1 for NTFS 3.1).
+ * @flags:          Volume flags (VOLUME_IS_DIRTY, VOLUME_CHKDSK_UNDERWAY, etc.).
+ *                  See volume flags enum for details.
+ *
+ * NOTE: Always resident.
+ * NOTE: Present only in FILE_Volume.
+ * NOTE: Windows 2000 uses NTFS 3.0 while Windows NT4 service pack 6a uses
+ *	 NTFS 1.2. I haven't personally seen other values yet.
+ */
+struct volume_information {
+	__le64 reserved;
+	u8 major_ver;
+	u8 minor_ver;
+	__le16 flags;
+} __packed;
+
+/*
+ * enum - Index header flags
+ *
+ * These flags are stored in the index header (INDEX_HEADER.flags) for both
+ * index root ($INDEX_ROOT) and index allocation blocks ($INDEX_ALLOCATION).
+ *
+ * For index root ($INDEX_ROOT attribute):
+ * SMALL_INDEX: Index fits entirely in root attribute (no $INDEX_ALLOCATION).
+ * LARGE_INDEX: Index too large for root; $INDEX_ALLOCATION present.
+ *
+ * For index blocks ($INDEX_ALLOCATION):
+ * LEAF_NODE:   Leaf node (no child nodes; contains actual entries).
+ * INDEX_NODE:  Internal node (indexes other nodes; contains keys/pointers).
+ *
+ * NODE_MASK:   Mask to extract node type bits (0x01).
+ */
+enum {
+	SMALL_INDEX = 0,
+	LARGE_INDEX = 1,
+	LEAF_NODE  = 0,
+	INDEX_NODE = 1,
+	NODE_MASK  = 1,
+} __packed;
+
+/*
+ * struct index_header - Common header for index root and index blocks
+ *
+ * entries_offset:     Byte offset to first INDEX_ENTRY (8-byte aligned).
+ * index_length:       Bytes used by index entries (8-byte aligned).
+ *                     From entries_offset to end of used data.
+ * allocated_size:     Total allocated bytes for this index block.
+ *                     Fixed size in index allocation; dynamic in root.
+ * flags:              Index flags (SMALL_INDEX, LARGE_INDEX, LEAF_NODE, etc.).
+ *                     See INDEX_HEADER_FLAGS enum.
+ * reserved:           3 bytes reserved/padding (zero, 8-byte aligned).
+ *
+ * This is the header for indexes, describing the INDEX_ENTRY records, which
+ * follow the index_header. Together the index header and the index entries
+ * make up a complete index.
+ *
+ * IMPORTANT NOTE: The offset, length and size structure members are counted
+ * relative to the start of the index header structure and not relative to the
+ * start of the index root or index allocation structures themselves.
+ *
+ * For the index root attribute, the above two numbers are always
+ * equal, as the attribute is resident and it is resized as needed. In
+ * the case of the index allocation attribute the attribute is not
+ * resident and hence the allocated_size is a fixed value and must
+ * equal the index_block_size specified by the INDEX_ROOT attribute
+ * corresponding to the INDEX_ALLOCATION attribute this INDEX_BLOCK
+ * belongs to.
+ */
+struct index_header {
+	__le32 entries_offset;
+	__le32 index_length;
+	__le32 allocated_size;
+	u8 flags;
+	u8 reserved[3];
+} __packed;
+
+/*
+ * struct index_root - $INDEX_ROOT attribute (0x90).
+ *
+ * @type:               Indexed attribute type ($FILE_NAME for dirs,
+ *                      0 for view indexes).
+ * @collation_rule:     Collation rule for sorting entries
+ *                      (COLLATION_FILE_NAME for $FILE_NAME).
+ * @index_block_size:   Size of each index block in bytes
+ *                      (in $INDEX_ALLOCATION).
+ * @clusters_per_index_block:
+ *                      Clusters per index block (or log2(bytes)
+ *                      if < cluster).
+ *                      Power of 2; used for encoding block size.
+ * @reserved:           3 bytes reserved/alignment (zero).
+ * @index:              Index header for root entries (entries follow
+ *                      immediately).
+ *
+ * NOTE: Always resident.
+ *
+ * This is followed by a sequence of index entries (INDEX_ENTRY structures)
+ * as described by the index header.
+ *
+ * When a directory is small enough to fit inside the index root then this
+ * is the only attribute describing the directory. When the directory is too
+ * large to fit in the index root, on the other hand, two additional attributes
+ * are present: an index allocation attribute, containing sub-nodes of the B+
+ * directory tree (see below), and a bitmap attribute, describing which virtual
+ * cluster numbers (vcns) in the index allocation attribute are in use by an
+ * index block.
+ *
+ * NOTE: The root directory (FILE_root) contains an entry for itself. Other
+ * directories do not contain entries for themselves, though.
+ */
+struct index_root {
+	__le32 type;
+	__le32 collation_rule;
+	__le32 index_block_size;
+	u8 clusters_per_index_block;
+	u8 reserved[3];
+	struct index_header index;
+} __packed;
+
+/*
+ * struct index_block - Index allocation (0xa0).
+ *
+ * @magic:              Magic value "INDX" (see magic_INDX).
+ * @usa_ofs:            Offset to Update Sequence Array (see ntfs_record).
+ * @usa_count:          Number of USA entries (see ntfs_record).
+ * @lsn:                Log sequence number of last modification.
+ * @index_block_vcn:    VCN of this index block.
+ *                      Units: clusters if cluster_size <= index_block_size;
+ *                      sectors otherwise.
+ * @index:              Index header describing entries in this block.
+ *
+ * When creating the index block, we place the update sequence array at this
+ * offset, i.e. before we start with the index entries. This also makes sense,
+ * otherwise we could run into problems with the update sequence array
+ * containing in itself the last two bytes of a sector which would mean that
+ * multi sector transfer protection wouldn't work. As you can't protect data
+ * by overwriting it since you then can't get it back...
+ * When reading use the data from the ntfs record header.
+ *
+ * NOTE: Always non-resident (doesn't make sense to be resident anyway!).
+ *
+ * This is an array of index blocks. Each index block starts with an
+ * index_block structure containing an index header, followed by a sequence of
+ * index entries (INDEX_ENTRY structures), as described by the struct index_header.
+ */
+struct index_block {
+	__le32 magic;
+	__le16 usa_ofs;
+	__le16 usa_count;
+	__le64 lsn;
+	__le64 index_block_vcn;
+	struct index_header index;
+} __packed;
+
+static_assert(sizeof(struct index_block) == 40);
+
+/*
+ * struct reparse_index_key - Key for $R reparse index in $Extend/$Reparse
+ *
+ * @reparse_tag:    Reparse point type (including flags, REPARSE_TAG_*).
+ * @file_id:        MFT record number of the file with $REPARSE_POINT
+ *                  attribute.
+ *
+ * The system file FILE_Extend/$Reparse contains an index named $R listing
+ * all reparse points on the volume. The index entry keys are as defined
+ * below. Note, that there is no index data associated with the index entries.
+ *
+ * The index entries are sorted by the index key file_id. The collation rule is
+ * COLLATION_NTOFS_ULONGS.
+ */
+struct reparse_index_key {
+	__le32 reparse_tag;
+	__le64 file_id;
+} __packed;
+
+/*
+ * enum - Quota entry flags (32-bit) in $Quota/$Q
+ *
+ * These flags are stored in quota control entries ($Quota file).
+ * They control quota tracking, limits, and state.
+ *
+ * User quota flags (mask 0x00000007):
+ * @QUOTA_FLAG_DEFAULT_LIMITS:   Use default limits.
+ * @QUOTA_FLAG_LIMIT_REACHED:    Quota limit reached.
+ * @QUOTA_FLAG_ID_DELETED:       Quota ID deleted.
+ * @QUOTA_FLAG_USER_MASK:        Mask for user quota flags (0x00000007).
+ *
+ * Default entry flags (owner_id = QUOTA_DEFAULTS_ID):
+ * @QUOTA_FLAG_TRACKING_ENABLED:    Quota tracking enabled.
+ * @QUOTA_FLAG_ENFORCEMENT_ENABLED: Quota enforcement enabled.
+ * @QUOTA_FLAG_TRACKING_REQUESTED:  Tracking requested (pending).
+ * @QUOTA_FLAG_LOG_THRESHOLD:       Log when threshold reached.
+ * @QUOTA_FLAG_LOG_LIMIT:           Log when limit reached.
+ * @QUOTA_FLAG_OUT_OF_DATE:         Quota data out of date.
+ * @QUOTA_FLAG_CORRUPT:             Quota entry corrupt.
+ * @QUOTA_FLAG_PENDING_DELETES:     Pending quota deletes.
+ *
+ */
+enum {
+	QUOTA_FLAG_DEFAULT_LIMITS	= cpu_to_le32(0x00000001),
+	QUOTA_FLAG_LIMIT_REACHED	= cpu_to_le32(0x00000002),
+	QUOTA_FLAG_ID_DELETED		= cpu_to_le32(0x00000004),
+
+	QUOTA_FLAG_USER_MASK		= cpu_to_le32(0x00000007),
+	QUOTA_FLAG_TRACKING_ENABLED	= cpu_to_le32(0x00000010),
+	QUOTA_FLAG_ENFORCEMENT_ENABLED	= cpu_to_le32(0x00000020),
+	QUOTA_FLAG_TRACKING_REQUESTED	= cpu_to_le32(0x00000040),
+	QUOTA_FLAG_LOG_THRESHOLD	= cpu_to_le32(0x00000080),
+
+	QUOTA_FLAG_LOG_LIMIT		= cpu_to_le32(0x00000100),
+	QUOTA_FLAG_OUT_OF_DATE		= cpu_to_le32(0x00000200),
+	QUOTA_FLAG_CORRUPT		= cpu_to_le32(0x00000400),
+	QUOTA_FLAG_PENDING_DELETES	= cpu_to_le32(0x00000800),
+};
+
+/*
+ * struct quota_control_entry - Quota entry in $Quota/$Q
+ *
+ * @version:        Currently 2.
+ * @flags:          Quota flags (QUOTA_FLAG_* bits).
+ * @bytes_used:     Current quota usage in bytes.
+ * @change_time:    Last modification time (NTFS timestamp).
+ * @threshold:      Soft quota limit (-1 = unlimited).
+ * @limit:          Hard quota limit (-1 = unlimited).
+ * @exceeded_time:  Time soft quota has been exceeded.
+ * @sid:            SID of user/object (zero for defaults entry).
+ *
+ * The system file FILE_Extend/$Quota contains two indexes $O and $Q. Quotas
+ * are on a per volume and per user basis.
+ *
+ * The $Q index contains one entry for each existing user_id on the volume. The
+ * index key is the user_id of the user/group owning this quota control entry,
+ * i.e. the key is the owner_id. The user_id of the owner of a file, i.e. the
+ * owner_id, is found in the standard information attribute. The collation rule
+ * for $Q is COLLATION_NTOFS_ULONG.
+ *
+ * The $O index contains one entry for each user/group who has been assigned
+ * a quota on that volume. The index key holds the SID of the user_id the
+ * entry belongs to, i.e. the owner_id. The collation rule for $O is
+ * COLLATION_NTOFS_SID.
+ *
+ * The $O index entry data is the user_id of the user corresponding to the SID.
+ * This user_id is used as an index into $Q to find the quota control entry
+ * associated with the SID.
+ *
+ * The $Q index entry data is the quota control entry and is defined below.
+ */
+struct quota_control_entry {
+	__le32 version;
+	__le32 flags;
+	__le64 bytes_used;
+	__le64 change_time;
+	__le64 threshold;
+	__le64 limit;
+	__le64 exceeded_time;
+	struct ntfs_sid sid;
+} __packed;
+
+/*
+ * Predefined owner_id values (32-bit).
+ */
+enum {
+	QUOTA_INVALID_ID	= cpu_to_le32(0x00000000),
+	QUOTA_DEFAULTS_ID	= cpu_to_le32(0x00000001),
+	QUOTA_FIRST_USER_ID	= cpu_to_le32(0x00000100),
+};
+
+/*
+ * Current constants for quota control entries.
+ */
+enum {
+	/* Current version. */
+	QUOTA_VERSION	= 2,
+};
+
+/*
+ * enum - Index entry flags (16-bit)
+ *
+ * These flags are in INDEX_ENTRY.flags (after key data).
+ * They describe entry type and status in index blocks/root.
+ *
+ * @INDEX_ENTRY_NODE:     Entry points to a sub-node (index block VCN).
+ *                        (Not a leaf entry; internal node reference.)
+ *                        i.e. a reference to an index block in form of
+ *                        a virtual cluster number
+ * @INDEX_ENTRY_END:      Last entry in index block/root.
+ *                        Does not represent a real file; can point to sub-node.
+ * @INDEX_ENTRY_SPACE_FILLER:
+ *                        Dummy value to force enum to 16-bit width.
+ */
+enum {
+	INDEX_ENTRY_NODE = cpu_to_le16(1),
+	INDEX_ENTRY_END  = cpu_to_le16(2),
+	INDEX_ENTRY_SPACE_FILLER = cpu_to_le16(0xffff),
+} __packed;
+
+/*
+ * struct index_entry_header - Common header for all NTFS index entries
+ *
+ * This is the fixed header at the start of every INDEX_ENTRY in index
+ * blocks or index root. It is followed by the variable key, data, and
+ * sub-node VCN.
+ *
+ * Union @data:
+ * - When INDEX_ENTRY_END is not set:
+ *   @data.dir.indexed_file: MFT reference of the file described by
+ *                           this entry. Used in directory indexes ($I30).
+ * - When INDEX_ENTRY_END is set or for view indexes:
+ *   @data.vi.data_offset: Byte offset from end of this header to
+ *                         entry data.
+ *   @data.vi.data_length: Length of data in bytes.
+ *   @data.vi.reservedV:   Reserved (zero).
+ *
+ * @length:         Total byte size of this index entry
+ *                  (multiple of 8 bytes).
+ * @key_length:     Byte size of the key (not multiple of 8 bytes).
+ *                  Key follows the header immediately.
+ * @flags:          Bit field of INDEX_ENTRY_* flags (INDEX_ENTRY_NODE, etc.).
+ * @reserved:       Reserved/padding (zero; align to 8 bytes).
+ */
+struct index_entry_header {
+	union {
+		struct {
+			__le64 indexed_file;
+		} __packed dir;
+		struct {
+			__le16 data_offset;
+			__le16 data_length;
+			__le32 reservedV;
+		} __packed vi;
+	} __packed data;
+	__le16 length;
+	__le16 key_length;
+	__le16 flags;
+	__le16 reserved;
+} __packed;
+
+static_assert(sizeof(struct index_entry_header) == 16);
+
+/*
+ * struct index_entry - NTFS index entry structure
+ *
+ * This is an index entry. A sequence of such entries follows each index_header
+ * structure. Together they make up a complete index. The index follows either
+ * an index root attribute or an index allocation attribute.
+ *
+ * Union @data (valid when INDEX_ENTRY_END not set):
+ * @data.dir.indexed_file: MFT ref of file (for directory indexes).
+ * @data.vi.data_offset:   Offset to data after key.
+ * @data.vi.data_length:   Length of data in bytes.
+ * @data.vi.reservedV:     Reserved (zero).
+ *
+ * Fields:
+ * @length:         Total byte size of entry (multiple of 8 bytes).
+ * @key_length:     Byte size of key (not multiple of 8).
+ * @flags:          INDEX_ENTRY_* flags (NODE, END, etc.).
+ * @reserved:       Reserved/padding (zero).
+ *
+ * Union @key (valid when INDEX_ENTRY_END not set)
+ * The key of the indexed attribute. NOTE: Only present
+ * if INDEX_ENTRY_END bit in flags is not set. NOTE: On
+ * NTFS versions before 3.0 the only valid key is the
+ * struct file_name_attr. On NTFS 3.0+ the following
+ * additional index keys are defined:
+ * @key.file_name:  $FILE_NAME attr (for $I30 directory indexes).
+ * @key.sii:        $SII key (for $Secure $SII index).
+ * @key.sdh:        $SDH key (for $Secure $SDH index).
+ * @key.object_id:  GUID (for $ObjId $O index).
+ * @key.reparse:    Reparse tag + file ID (for $Reparse $R).
+ * @key.sid:        SID (for $Quota $O index).
+ * @key.owner_id:   User ID (for $Quota $Q index).
+ *
+ * The (optional) index data is inserted here when creating.
+ * __le64 vcn;     If INDEX_ENTRY_NODE bit in flags is set, the last
+ *                 eight bytes of this index entry contain the virtual
+ *                 cluster number of the index block that holds the
+ *                 entries immediately preceding the current entry (the
+ *                 vcn references the corresponding cluster in the data
+ *                 of the non-resident index allocation attribute). If
+ *                 the key_length is zero, then the vcn immediately
+ *                 follows the INDEX_ENTRY_HEADER. Regardless of
+ *                 key_length, the address of the 8-byte boundary
+ *                 aligned vcn of INDEX_ENTRY{_HEADER} *ie is given by
+ *                 (char*)ie + le16_to_cpu(ie*)->length) - sizeof(VCN),
+ *                 where sizeof(VCN) can be hardcoded as 8 if wanted.
+ *
+ * NOTE: Before NTFS 3.0 only filename attributes were indexed.
+ */
+struct index_entry {
+	union {
+		struct {
+			__le64 indexed_file;
+		} __packed dir;
+		struct {
+			__le16 data_offset;
+			__le16 data_length;
+			__le32 reservedV;
+		} __packed vi;
+	} __packed data;
+	__le16 length;
+	__le16 key_length;
+	__le16 flags;
+	__le16 reserved;
+	union {
+		struct file_name_attr file_name;
+		struct sii_index_key sii;
+		struct sdh_index_key sdh;
+		struct guid object_id;
+		struct reparse_index_key reparse;
+		struct ntfs_sid sid;
+		__le32 owner_id;
+	} __packed key;
+} __packed;
+
+/*
+ * The reparse point tag defines the type of the reparse point. It also
+ * includes several flags, which further describe the reparse point.
+ *
+ * The reparse point tag is an unsigned 32-bit value divided in three parts:
+ *
+ * 1. The least significant 16 bits (i.e. bits 0 to 15) specify the type of
+ *    the reparse point.
+ * 2. The 12 bits after this (i.e. bits 16 to 27) are reserved for future use.
+ * 3. The most significant four bits are flags describing the reparse point.
+ *    They are defined as follows:
+ *	bit 28: Directory bit. If set, the directory is not a surrogate
+ *		and can be used the usual way.
+ *	bit 29: Name surrogate bit. If set, the filename is an alias for
+ *		another object in the system.
+ *	bit 30: High-latency bit. If set, accessing the first byte of data will
+ *		be slow. (E.g. the data is stored on a tape drive.)
+ *	bit 31: Microsoft bit. If set, the tag is owned by Microsoft. User
+ *		defined tags have to use zero here.
+ * 4. Moreover, on Windows 10 :
+ *	Some flags may be used in bits 12 to 15 to further describe the
+ *	reparse point.
+ */
+enum {
+	IO_REPARSE_TAG_DIRECTORY	= cpu_to_le32(0x10000000),
+	IO_REPARSE_TAG_IS_ALIAS		= cpu_to_le32(0x20000000),
+	IO_REPARSE_TAG_IS_HIGH_LATENCY	= cpu_to_le32(0x40000000),
+	IO_REPARSE_TAG_IS_MICROSOFT	= cpu_to_le32(0x80000000),
+
+	IO_REPARSE_TAG_RESERVED_ZERO	= cpu_to_le32(0x00000000),
+	IO_REPARSE_TAG_RESERVED_ONE	= cpu_to_le32(0x00000001),
+	IO_REPARSE_TAG_RESERVED_RANGE	= cpu_to_le32(0x00000001),
+
+	IO_REPARSE_TAG_CSV		= cpu_to_le32(0x80000009),
+	IO_REPARSE_TAG_DEDUP		= cpu_to_le32(0x80000013),
+	IO_REPARSE_TAG_DFS		= cpu_to_le32(0x8000000A),
+	IO_REPARSE_TAG_DFSR		= cpu_to_le32(0x80000012),
+	IO_REPARSE_TAG_HSM		= cpu_to_le32(0xC0000004),
+	IO_REPARSE_TAG_HSM2		= cpu_to_le32(0x80000006),
+	IO_REPARSE_TAG_MOUNT_POINT	= cpu_to_le32(0xA0000003),
+	IO_REPARSE_TAG_NFS		= cpu_to_le32(0x80000014),
+	IO_REPARSE_TAG_SIS		= cpu_to_le32(0x80000007),
+	IO_REPARSE_TAG_SYMLINK		= cpu_to_le32(0xA000000C),
+	IO_REPARSE_TAG_WIM		= cpu_to_le32(0x80000008),
+	IO_REPARSE_TAG_DFM		= cpu_to_le32(0x80000016),
+	IO_REPARSE_TAG_WOF		= cpu_to_le32(0x80000017),
+	IO_REPARSE_TAG_WCI		= cpu_to_le32(0x80000018),
+	IO_REPARSE_TAG_CLOUD		= cpu_to_le32(0x9000001A),
+	IO_REPARSE_TAG_APPEXECLINK	= cpu_to_le32(0x8000001B),
+	IO_REPARSE_TAG_GVFS		= cpu_to_le32(0x9000001C),
+	IO_REPARSE_TAG_LX_SYMLINK	= cpu_to_le32(0xA000001D),
+	IO_REPARSE_TAG_AF_UNIX		= cpu_to_le32(0x80000023),
+	IO_REPARSE_TAG_LX_FIFO		= cpu_to_le32(0x80000024),
+	IO_REPARSE_TAG_LX_CHR		= cpu_to_le32(0x80000025),
+	IO_REPARSE_TAG_LX_BLK		= cpu_to_le32(0x80000026),
+
+	IO_REPARSE_TAG_VALID_VALUES	= cpu_to_le32(0xf000ffff),
+	IO_REPARSE_PLUGIN_SELECT	= cpu_to_le32(0xffff0fff),
+};
+
+/*
+ * struct reparse_point - $REPARSE_POINT attribute content (0xc0)\
+ *
+ * @reparse_tag:        Reparse point type (with flags; REPARSE_TAG_*).
+ * @reparse_data_length: Byte size of @reparse_data.
+ * @reserved:           Reserved/padding (zero; 8-byte alignment).
+ * @reparse_data:       Variable reparse data (meaning depends on @reparse_tag).
+ *                      - Symbolic link/junction: struct reparse_symlink
+ *                      - Mount point: similar symlink structure
+ *                      - Other tags: vendor-specific or extended data
+ *
+ * NOTE: Can be resident or non-resident.
+ */
+struct reparse_point {
+	__le32 reparse_tag;
+	__le16 reparse_data_length;
+	__le16 reserved;
+	u8 reparse_data[];
+} __packed;
+
+/*
+ * struct ea_information - $EA_INFORMATION attribute content (0xd0)
+ *
+ * @ea_length:      Byte size of packed EAs.
+ * @need_ea_count:  Number of EAs with NEED_EA bit set.
+ * @ea_query_length: Byte size needed to unpack/query EAs via ZwQueryEaFile().
+ *                   (Unpacked format size.)
+ *
+ * NOTE: Always resident. (Is this true???)
+ */
+struct ea_information {
+	__le16 ea_length;
+	__le16 need_ea_count;
+	__le32 ea_query_length;
+} __packed;
+
+/*
+ * enum - Extended attribute flags (8-bit)
+ *
+ * These flags are stored in the EA header of each extended attribute
+ * (in $EA attribute, type 0xe0).
+ *
+ * @NEED_EA:        If set, the file cannot be properly interpreted
+ *                  without understanding its associated EAs.
+ *                  (Critical EA; applications must process it.)
+ */
+enum {
+	NEED_EA	= 0x80
+} __packed;
+
+/*
+ * struct ea_attr - Extended attribute (EA) entry (0xe0)
+ *
+ * @next_entry_offset:  Byte offset to the next EA_ATTR entry.
+ *                      (From start of current entry.)
+ * @flags:              EA flags (NEED_EA = 0x80 if critical).
+ * @ea_name_length:     Length of @ea_name in bytes (excluding '\0').
+ * @ea_value_length:    Byte size of the EA value.
+ * @ea_name:            ASCII name of the EA (zero-terminated).
+ *                      Value immediately follows the name.
+ * u8 ea_value[];       The value of the EA.  Immediately follows the name.
+ *
+ * This is one variable-length record in the $EA attribute value.
+ * The attribute can be resident or non-resident.
+ * Sequence of these entries forms the packed EA list.
+ *
+ * NOTE: Can be resident or non-resident.
+ */
+struct ea_attr {
+	__le32 next_entry_offset;
+	u8 flags;
+	u8 ea_name_length;
+	__le16 ea_value_length;
+	u8 ea_name[];
+} __packed;
+
+#endif /* _LINUX_NTFS_LAYOUT_H */
diff --git a/fs/ntfs/lcnalloc.c b/fs/ntfs/lcnalloc.c
new file mode 100644
index 0000000000000..835a041023a2a
--- /dev/null
+++ b/fs/ntfs/lcnalloc.c
@@ -0,0 +1,1049 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Cluster (de)allocation code.
+ *
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2002-2004 Anton Altaparmakov
+ * Copyright (c) 2004 Yura Pakhuchiy
+ * Copyright (c) 2004-2008 Szabolcs Szakacsits
+ * Copyright (c) 2008-2009 Jean-Pierre Andre
+ */
+
+#include <linux/blkdev.h>
+
+#include "lcnalloc.h"
+#include "bitmap.h"
+#include "ntfs.h"
+
+/*
+ * ntfs_cluster_free_from_rl_nolock - free clusters from runlist
+ * @vol:	mounted ntfs volume on which to free the clusters
+ * @rl:		runlist describing the clusters to free
+ *
+ * Free all the clusters described by the runlist @rl on the volume @vol.  In
+ * the case of an error being returned, at least some of the clusters were not
+ * freed.
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Locking: - The volume lcn bitmap must be locked for writing on entry and is
+ *	      left locked on return.
+ */
+int ntfs_cluster_free_from_rl_nolock(struct ntfs_volume *vol,
+		const struct runlist_element *rl)
+{
+	struct inode *lcnbmp_vi = vol->lcnbmp_ino;
+	int ret = 0;
+	s64 nr_freed = 0;
+
+	ntfs_debug("Entering.");
+	if (!rl)
+		return 0;
+
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	for (; rl->length; rl++) {
+		int err;
+
+		if (rl->lcn < 0)
+			continue;
+		err = ntfs_bitmap_clear_run(lcnbmp_vi, rl->lcn, rl->length);
+		if (unlikely(err && (!ret || ret == -ENOMEM) && ret != err))
+			ret = err;
+		else
+			nr_freed += rl->length;
+	}
+	ntfs_inc_free_clusters(vol, nr_freed);
+	ntfs_debug("Done.");
+	return ret;
+}
+
+static s64 max_empty_bit_range(unsigned char *buf, int size)
+{
+	int i, j, run = 0;
+	int max_range = 0;
+	s64 start_pos = -1;
+
+	ntfs_debug("Entering\n");
+
+	i = 0;
+	while (i < size) {
+		switch (*buf) {
+		case 0:
+			do {
+				buf++;
+				run += 8;
+				i++;
+			} while ((i < size) && !*buf);
+			break;
+		case 255:
+			if (run > max_range) {
+				max_range = run;
+				start_pos = (s64)i * 8 - run;
+			}
+			run = 0;
+			do {
+				buf++;
+				i++;
+			} while ((i < size) && (*buf == 255));
+			break;
+		default:
+			for (j = 0; j < 8; j++) {
+				int bit = *buf & (1 << j);
+
+				if (bit) {
+					if (run > max_range) {
+						max_range = run;
+						start_pos = (s64)i * 8 + (j - run);
+					}
+					run = 0;
+				} else
+					run++;
+			}
+			i++;
+			buf++;
+		}
+	}
+
+	if (run > max_range)
+		start_pos = (s64)i * 8 - run;
+
+	return start_pos;
+}
+
+/*
+ * ntfs_cluster_alloc - allocate clusters on an ntfs volume
+ * @vol:		mounted ntfs volume on which to allocate clusters
+ * @start_vcn:		vcn of the first allocated cluster
+ * @count:		number of clusters to allocate
+ * @start_lcn:		starting lcn at which to allocate the clusters or -1 if none
+ * @zone:		zone from which to allocate (MFT_ZONE or DATA_ZONE)
+ * @is_extension:	if true, the caller is extending an attribute
+ * @is_contig:		if true, require contiguous allocation
+ * @is_dealloc:		if true, the allocation is for deallocation purposes
+ *
+ * Allocate @count clusters preferably starting at cluster @start_lcn or at the
+ * current allocator position if @start_lcn is -1, on the mounted ntfs volume
+ * @vol. @zone is either DATA_ZONE for allocation of normal clusters or
+ * MFT_ZONE for allocation of clusters for the master file table, i.e. the
+ * $MFT/$DATA attribute.
+ *
+ * @start_vcn specifies the vcn of the first allocated cluster.  This makes
+ * merging the resulting runlist with the old runlist easier.
+ *
+ * If @is_extension is 'true', the caller is allocating clusters to extend an
+ * attribute and if it is 'false', the caller is allocating clusters to fill a
+ * hole in an attribute.  Practically the difference is that if @is_extension
+ * is 'true' the returned runlist will be terminated with LCN_ENOENT and if
+ * @is_extension is 'false' the runlist will be terminated with
+ * LCN_RL_NOT_MAPPED.
+ *
+ * You need to check the return value with IS_ERR().  If this is false, the
+ * function was successful and the return value is a runlist describing the
+ * allocated cluster(s).  If IS_ERR() is true, the function failed and
+ * PTR_ERR() gives you the error code.
+ *
+ * Notes on the allocation algorithm
+ * =================================
+ *
+ * There are two data zones.  First is the area between the end of the mft zone
+ * and the end of the volume, and second is the area between the start of the
+ * volume and the start of the mft zone.  On unmodified/standard NTFS 1.x
+ * volumes, the second data zone does not exist due to the mft zone being
+ * expanded to cover the start of the volume in order to reserve space for the
+ * mft bitmap attribute.
+ *
+ * This is not the prettiest function but the complexity stems from the need of
+ * implementing the mft vs data zoned approach and from the fact that we have
+ * access to the lcn bitmap in portions of up to 8192 bytes at a time, so we
+ * need to cope with crossing over boundaries of two buffers.  Further, the
+ * fact that the allocator allows for caller supplied hints as to the location
+ * of where allocation should begin and the fact that the allocator keeps track
+ * of where in the data zones the next natural allocation should occur,
+ * contribute to the complexity of the function.  But it should all be
+ * worthwhile, because this allocator should: 1) be a full implementation of
+ * the MFT zone approach used by Windows NT, 2) cause reduction in
+ * fragmentation, and 3) be speedy in allocations (the code is not optimized
+ * for speed, but the algorithm is, so further speed improvements are probably
+ * possible).
+ *
+ * Locking: - The volume lcn bitmap must be unlocked on entry and is unlocked
+ *	      on return.
+ *	    - This function takes the volume lcn bitmap lock for writing and
+ *	      modifies the bitmap contents.
+ *
+ * Return: Runlist describing the allocated cluster(s) on success, error pointer
+ *         on failure.
+ */
+struct runlist_element *ntfs_cluster_alloc(struct ntfs_volume *vol, const s64 start_vcn,
+		const s64 count, const s64 start_lcn,
+		const int zone,
+		const bool is_extension,
+		const bool is_contig,
+		const bool is_dealloc)
+{
+	s64 zone_start, zone_end, bmp_pos, bmp_initial_pos, last_read_pos, lcn;
+	s64 prev_lcn = 0, prev_run_len = 0, mft_zone_size;
+	s64 clusters, free_clusters;
+	loff_t i_size;
+	struct inode *lcnbmp_vi;
+	struct runlist_element *rl = NULL;
+	struct address_space *mapping;
+	struct folio *folio = NULL;
+	u8 *buf = NULL, *byte;
+	int err = 0, rlpos, rlsize, buf_size, pg_off;
+	u8 pass, done_zones, search_zone, need_writeback = 0, bit;
+	unsigned int memalloc_flags;
+	u8 has_guess, used_zone_pos;
+	pgoff_t index;
+
+	ntfs_debug("Entering for start_vcn 0x%llx, count 0x%llx, start_lcn 0x%llx, zone %s_ZONE.",
+			start_vcn, count, start_lcn,
+			zone == MFT_ZONE ? "MFT" : "DATA");
+
+	lcnbmp_vi = vol->lcnbmp_ino;
+	if (start_vcn < 0 || start_lcn < LCN_HOLE ||
+	    zone < FIRST_ZONE || zone > LAST_ZONE)
+		return ERR_PTR(-EINVAL);
+
+	/* Return NULL if @count is zero. */
+	if (count < 0 || !count)
+		return ERR_PTR(-EINVAL);
+
+	memalloc_flags = memalloc_nofs_save();
+
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+	free_clusters = atomic64_read(&vol->free_clusters);
+
+	/* Take the lcnbmp lock for writing. */
+	down_write(&vol->lcnbmp_lock);
+	if (is_dealloc == false)
+		free_clusters -= atomic64_read(&vol->dirty_clusters);
+
+	if (free_clusters < count) {
+		err = -ENOSPC;
+		goto out_restore;
+	}
+
+	/*
+	 * If no specific @start_lcn was requested, use the current data zone
+	 * position, otherwise use the requested @start_lcn but make sure it
+	 * lies outside the mft zone.  Also set done_zones to 0 (no zones done)
+	 * and pass depending on whether we are starting inside a zone (1) or
+	 * at the beginning of a zone (2).  If requesting from the MFT_ZONE,
+	 * we either start at the current position within the mft zone or at
+	 * the specified position.  If the latter is out of bounds then we start
+	 * at the beginning of the MFT_ZONE.
+	 */
+	done_zones = 0;
+	pass = 1;
+	/*
+	 * zone_start and zone_end are the current search range.  search_zone
+	 * is 1 for mft zone, 2 for data zone 1 (end of mft zone till end of
+	 * volume) and 4 for data zone 2 (start of volume till start of mft
+	 * zone).
+	 */
+	has_guess = 1;
+	zone_start = start_lcn;
+
+	if (zone_start < 0) {
+		if (zone == DATA_ZONE)
+			zone_start = vol->data1_zone_pos;
+		else
+			zone_start = vol->mft_zone_pos;
+		if (!zone_start) {
+			/*
+			 * Zone starts at beginning of volume which means a
+			 * single pass is sufficient.
+			 */
+			pass = 2;
+		}
+		has_guess = 0;
+	}
+
+	used_zone_pos = has_guess ? 0 : 1;
+
+	if (!zone_start || zone_start == vol->mft_zone_start ||
+			zone_start == vol->mft_zone_end)
+		pass = 2;
+
+	if (zone_start < vol->mft_zone_start) {
+		zone_end = vol->mft_zone_start;
+		search_zone = 4;
+		/* Skip searching the mft zone. */
+		done_zones |= 1;
+	} else if (zone_start < vol->mft_zone_end) {
+		zone_end = vol->mft_zone_end;
+		search_zone = 1;
+	} else {
+		zone_end = vol->nr_clusters;
+		search_zone = 2;
+		/* Skip searching the mft zone. */
+		done_zones |= 1;
+	}
+
+	/*
+	 * bmp_pos is the current bit position inside the bitmap.  We use
+	 * bmp_initial_pos to determine whether or not to do a zone switch.
+	 */
+	bmp_pos = bmp_initial_pos = zone_start;
+
+	/* Loop until all clusters are allocated, i.e. clusters == 0. */
+	clusters = count;
+	rlpos = rlsize = 0;
+	mapping = lcnbmp_vi->i_mapping;
+	i_size = i_size_read(lcnbmp_vi);
+	while (1) {
+		ntfs_debug("Start of outer while loop: done_zones 0x%x, search_zone %i, pass %i, zone_start 0x%llx, zone_end 0x%llx, bmp_initial_pos 0x%llx, bmp_pos 0x%llx, rlpos %i, rlsize %i.",
+				done_zones, search_zone, pass,
+				zone_start, zone_end, bmp_initial_pos,
+				bmp_pos, rlpos, rlsize);
+		/* Loop until we run out of free clusters. */
+		last_read_pos = bmp_pos >> 3;
+		ntfs_debug("last_read_pos 0x%llx.", last_read_pos);
+		if (last_read_pos >= i_size) {
+			ntfs_debug("End of attribute reached. Skipping to zone_pass_done.");
+			goto zone_pass_done;
+		}
+		if (likely(folio)) {
+			if (need_writeback) {
+				ntfs_debug("Marking page dirty.");
+				folio_mark_dirty(folio);
+				need_writeback = 0;
+			}
+			folio_unlock(folio);
+			kunmap_local(buf);
+			folio_put(folio);
+			folio = NULL;
+		}
+
+		index = last_read_pos >> PAGE_SHIFT;
+		pg_off = last_read_pos & ~PAGE_MASK;
+		buf_size = PAGE_SIZE - pg_off;
+		if (unlikely(last_read_pos + buf_size > i_size))
+			buf_size = i_size - last_read_pos;
+		buf_size <<= 3;
+		lcn = bmp_pos & 7;
+		bmp_pos &= ~(s64)7;
+
+		if (vol->lcn_empty_bits_per_page[index] == 0)
+			goto next_bmp_pos;
+
+		folio = read_mapping_folio(mapping, index, NULL);
+		if (IS_ERR(folio)) {
+			err = PTR_ERR(folio);
+			ntfs_error(vol->sb, "Failed to map page.");
+			goto out;
+		}
+
+		folio_lock(folio);
+		buf = kmap_local_folio(folio, 0) + pg_off;
+		ntfs_debug("Before inner while loop: buf_size %i, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i.",
+				buf_size, lcn, bmp_pos, need_writeback);
+		while (lcn < buf_size && lcn + bmp_pos < zone_end) {
+			byte = buf + (lcn >> 3);
+			ntfs_debug("In inner while loop: buf_size %i, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i, byte ofs 0x%x, *byte 0x%x.",
+					buf_size, lcn, bmp_pos, need_writeback,
+					(unsigned int)(lcn >> 3),
+					(unsigned int)*byte);
+			bit = 1 << (lcn & 7);
+			ntfs_debug("bit 0x%x.", bit);
+
+			if (has_guess) {
+				if (*byte & bit) {
+					if (is_contig == true && prev_run_len > 0)
+						goto done;
+
+					has_guess = 0;
+					break;
+				}
+			} else {
+				lcn = max_empty_bit_range(buf, buf_size >> 3);
+				if (lcn < 0)
+					break;
+				has_guess = 1;
+				continue;
+			}
+			/*
+			 * Allocate more memory if needed, including space for
+			 * the terminator element.
+			 * kvzalloc() operates on whole pages only.
+			 */
+			if ((rlpos + 2) * sizeof(*rl) > rlsize) {
+				struct runlist_element *rl2;
+
+				ntfs_debug("Reallocating memory.");
+				if (!rl)
+					ntfs_debug("First free bit is at s64 0x%llx.",
+							lcn + bmp_pos);
+				rl2 = kvzalloc(rlsize + PAGE_SIZE, GFP_NOFS);
+				if (unlikely(!rl2)) {
+					err = -ENOMEM;
+					ntfs_error(vol->sb, "Failed to allocate memory.");
+					goto out;
+				}
+				memcpy(rl2, rl, rlsize);
+				kvfree(rl);
+				rl = rl2;
+				rlsize += PAGE_SIZE;
+				ntfs_debug("Reallocated memory, rlsize 0x%x.",
+						rlsize);
+			}
+			/* Allocate the bitmap bit. */
+			*byte |= bit;
+			/* We need to write this bitmap page to disk. */
+			need_writeback = 1;
+			ntfs_debug("*byte 0x%x, need_writeback is set.",
+					(unsigned int)*byte);
+			ntfs_dec_free_clusters(vol, 1);
+			ntfs_set_lcn_empty_bits(vol, index, 1, 1);
+
+			/*
+			 * Coalesce with previous run if adjacent LCNs.
+			 * Otherwise, append a new run.
+			 */
+			ntfs_debug("Adding run (lcn 0x%llx, len 0x%llx), prev_lcn 0x%llx, lcn 0x%llx, bmp_pos 0x%llx, prev_run_len 0x%llx, rlpos %i.",
+					lcn + bmp_pos, 1ULL, prev_lcn,
+					lcn, bmp_pos, prev_run_len, rlpos);
+			if (prev_lcn == lcn + bmp_pos - prev_run_len && rlpos) {
+				ntfs_debug("Coalescing to run (lcn 0x%llx, len 0x%llx).",
+						rl[rlpos - 1].lcn,
+						rl[rlpos - 1].length);
+				rl[rlpos - 1].length = ++prev_run_len;
+				ntfs_debug("Run now (lcn 0x%llx, len 0x%llx), prev_run_len 0x%llx.",
+						rl[rlpos - 1].lcn,
+						rl[rlpos - 1].length,
+						prev_run_len);
+			} else {
+				if (likely(rlpos)) {
+					ntfs_debug("Adding new run, (previous run lcn 0x%llx, len 0x%llx).",
+							rl[rlpos - 1].lcn, rl[rlpos - 1].length);
+					rl[rlpos].vcn = rl[rlpos - 1].vcn +
+							prev_run_len;
+				} else {
+					ntfs_debug("Adding new run, is first run.");
+					rl[rlpos].vcn = start_vcn;
+				}
+				rl[rlpos].lcn = prev_lcn = lcn + bmp_pos;
+				rl[rlpos].length = prev_run_len = 1;
+				rlpos++;
+			}
+			/* Done? */
+			if (!--clusters) {
+				s64 tc;
+done:
+				if (!used_zone_pos)
+					goto out;
+				/*
+				 * Update the current zone position.  Positions
+				 * of already scanned zones have been updated
+				 * during the respective zone switches.
+				 */
+				tc = lcn + bmp_pos + 1;
+				ntfs_debug("Done. Updating current zone position, tc 0x%llx, search_zone %i.",
+						tc, search_zone);
+				switch (search_zone) {
+				case 1:
+					ntfs_debug("Before checks, vol->mft_zone_pos 0x%llx.",
+							vol->mft_zone_pos);
+					if (tc >= vol->mft_zone_end) {
+						vol->mft_zone_pos =
+								vol->mft_lcn;
+						if (!vol->mft_zone_end)
+							vol->mft_zone_pos = 0;
+					} else if ((bmp_initial_pos >=
+							vol->mft_zone_pos ||
+							tc > vol->mft_zone_pos)
+							&& tc >= vol->mft_lcn)
+						vol->mft_zone_pos = tc;
+					ntfs_debug("After checks, vol->mft_zone_pos 0x%llx.",
+							vol->mft_zone_pos);
+					break;
+				case 2:
+					ntfs_debug("Before checks, vol->data1_zone_pos 0x%llx.",
+							vol->data1_zone_pos);
+					if (tc >= vol->nr_clusters)
+						vol->data1_zone_pos =
+							     vol->mft_zone_end;
+					else if ((bmp_initial_pos >=
+						    vol->data1_zone_pos ||
+						    tc > vol->data1_zone_pos)
+						    && tc >= vol->mft_zone_end)
+						vol->data1_zone_pos = tc;
+					ntfs_debug("After checks, vol->data1_zone_pos 0x%llx.",
+							vol->data1_zone_pos);
+					break;
+				case 4:
+					ntfs_debug("Before checks, vol->data2_zone_pos 0x%llx.",
+							vol->data2_zone_pos);
+					if (tc >= vol->mft_zone_start)
+						vol->data2_zone_pos = 0;
+					else if (bmp_initial_pos >=
+						      vol->data2_zone_pos ||
+						      tc > vol->data2_zone_pos)
+						vol->data2_zone_pos = tc;
+					ntfs_debug("After checks, vol->data2_zone_pos 0x%llx.",
+							vol->data2_zone_pos);
+					break;
+				default:
+					WARN_ON(1);
+				}
+				ntfs_debug("Finished.  Going to out.");
+				goto out;
+			}
+			lcn++;
+		}
+
+		if (!used_zone_pos) {
+			used_zone_pos = 1;
+			if (search_zone == 1)
+				zone_start = vol->mft_zone_pos;
+			else if (search_zone == 2)
+				zone_start = vol->data1_zone_pos;
+			else
+				zone_start = vol->data2_zone_pos;
+
+			if (!zone_start || zone_start == vol->mft_zone_start ||
+			    zone_start == vol->mft_zone_end)
+				pass = 2;
+			bmp_pos = zone_start;
+		} else {
+next_bmp_pos:
+			bmp_pos += buf_size;
+		}
+
+		ntfs_debug("After inner while loop: buf_size 0x%x, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i.",
+				buf_size, lcn, bmp_pos, need_writeback);
+		if (bmp_pos < zone_end) {
+			ntfs_debug("Continuing outer while loop, bmp_pos 0x%llx, zone_end 0x%llx.",
+					bmp_pos, zone_end);
+			continue;
+		}
+zone_pass_done:	/* Finished with the current zone pass. */
+		ntfs_debug("At zone_pass_done, pass %i.", pass);
+		if (pass == 1) {
+			/*
+			 * Now do pass 2, scanning the first part of the zone
+			 * we omitted in pass 1.
+			 */
+			pass = 2;
+			zone_end = zone_start;
+			switch (search_zone) {
+			case 1: /* mft_zone */
+				zone_start = vol->mft_zone_start;
+				break;
+			case 2: /* data1_zone */
+				zone_start = vol->mft_zone_end;
+				break;
+			case 4: /* data2_zone */
+				zone_start = 0;
+				break;
+			default:
+				WARN_ON(1);
+			}
+			/* Sanity check. */
+			if (zone_end < zone_start)
+				zone_end = zone_start;
+			bmp_pos = zone_start;
+			ntfs_debug("Continuing outer while loop, pass 2, zone_start 0x%llx, zone_end 0x%llx, bmp_pos 0x%llx.",
+					zone_start, zone_end, bmp_pos);
+			continue;
+		} /* pass == 2 */
+done_zones_check:
+		ntfs_debug("At done_zones_check, search_zone %i, done_zones before 0x%x, done_zones after 0x%x.",
+				search_zone, done_zones,
+				done_zones | search_zone);
+		done_zones |= search_zone;
+		if (done_zones < 7) {
+			ntfs_debug("Switching zone.");
+			/* Now switch to the next zone we haven't done yet. */
+			pass = 1;
+			switch (search_zone) {
+			case 1:
+				ntfs_debug("Switching from mft zone to data1 zone.");
+				/* Update mft zone position. */
+				if (rlpos && used_zone_pos) {
+					s64 tc;
+
+					ntfs_debug("Before checks, vol->mft_zone_pos 0x%llx.",
+							vol->mft_zone_pos);
+					tc = rl[rlpos - 1].lcn +
+							rl[rlpos - 1].length;
+					if (tc >= vol->mft_zone_end) {
+						vol->mft_zone_pos =
+								vol->mft_lcn;
+						if (!vol->mft_zone_end)
+							vol->mft_zone_pos = 0;
+					} else if ((bmp_initial_pos >=
+							vol->mft_zone_pos ||
+							tc > vol->mft_zone_pos)
+							&& tc >= vol->mft_lcn)
+						vol->mft_zone_pos = tc;
+					ntfs_debug("After checks, vol->mft_zone_pos 0x%llx.",
+							vol->mft_zone_pos);
+				}
+				/* Switch from mft zone to data1 zone. */
+switch_to_data1_zone:		search_zone = 2;
+				zone_start = bmp_initial_pos =
+						vol->data1_zone_pos;
+				zone_end = vol->nr_clusters;
+				if (zone_start == vol->mft_zone_end)
+					pass = 2;
+				if (zone_start >= zone_end) {
+					vol->data1_zone_pos = zone_start =
+							vol->mft_zone_end;
+					pass = 2;
+				}
+				break;
+			case 2:
+				ntfs_debug("Switching from data1 zone to data2 zone.");
+				/* Update data1 zone position. */
+				if (rlpos && used_zone_pos) {
+					s64 tc;
+
+					ntfs_debug("Before checks, vol->data1_zone_pos 0x%llx.",
+							vol->data1_zone_pos);
+					tc = rl[rlpos - 1].lcn +
+							rl[rlpos - 1].length;
+					if (tc >= vol->nr_clusters)
+						vol->data1_zone_pos =
+							     vol->mft_zone_end;
+					else if ((bmp_initial_pos >=
+						    vol->data1_zone_pos ||
+						    tc > vol->data1_zone_pos)
+						    && tc >= vol->mft_zone_end)
+						vol->data1_zone_pos = tc;
+					ntfs_debug("After checks, vol->data1_zone_pos 0x%llx.",
+							vol->data1_zone_pos);
+				}
+				/* Switch from data1 zone to data2 zone. */
+				search_zone = 4;
+				zone_start = bmp_initial_pos =
+						vol->data2_zone_pos;
+				zone_end = vol->mft_zone_start;
+				if (!zone_start)
+					pass = 2;
+				if (zone_start >= zone_end) {
+					vol->data2_zone_pos = zone_start =
+							bmp_initial_pos = 0;
+					pass = 2;
+				}
+				break;
+			case 4:
+				ntfs_debug("Switching from data2 zone to data1 zone.");
+				/* Update data2 zone position. */
+				if (rlpos && used_zone_pos) {
+					s64 tc;
+
+					ntfs_debug("Before checks, vol->data2_zone_pos 0x%llx.",
+							vol->data2_zone_pos);
+					tc = rl[rlpos - 1].lcn +
+							rl[rlpos - 1].length;
+					if (tc >= vol->mft_zone_start)
+						vol->data2_zone_pos = 0;
+					else if (bmp_initial_pos >=
+						      vol->data2_zone_pos ||
+						      tc > vol->data2_zone_pos)
+						vol->data2_zone_pos = tc;
+					ntfs_debug("After checks, vol->data2_zone_pos 0x%llx.",
+							vol->data2_zone_pos);
+				}
+				/* Switch from data2 zone to data1 zone. */
+				goto switch_to_data1_zone;
+			default:
+				WARN_ON(1);
+			}
+			ntfs_debug("After zone switch, search_zone %i, pass %i, bmp_initial_pos 0x%llx, zone_start 0x%llx, zone_end 0x%llx.",
+					search_zone, pass,
+					bmp_initial_pos,
+					zone_start,
+					zone_end);
+			bmp_pos = zone_start;
+			if (zone_start == zone_end) {
+				ntfs_debug("Empty zone, going to done_zones_check.");
+				/* Empty zone. Don't bother searching it. */
+				goto done_zones_check;
+			}
+			ntfs_debug("Continuing outer while loop.");
+			continue;
+		} /* done_zones == 7 */
+		ntfs_debug("All zones are finished.");
+		/*
+		 * All zones are finished!  If DATA_ZONE, shrink mft zone.  If
+		 * MFT_ZONE, we have really run out of space.
+		 */
+		mft_zone_size = vol->mft_zone_end - vol->mft_zone_start;
+		ntfs_debug("vol->mft_zone_start 0x%llx, vol->mft_zone_end 0x%llx, mft_zone_size 0x%llx.",
+				vol->mft_zone_start, vol->mft_zone_end,
+				mft_zone_size);
+		if (zone == MFT_ZONE || mft_zone_size <= 0) {
+			ntfs_debug("No free clusters left, going to out.");
+			/* Really no more space left on device. */
+			err = -ENOSPC;
+			goto out;
+		} /* zone == DATA_ZONE && mft_zone_size > 0 */
+		ntfs_debug("Shrinking mft zone.");
+		zone_end = vol->mft_zone_end;
+		mft_zone_size >>= 1;
+		if (mft_zone_size > 0)
+			vol->mft_zone_end = vol->mft_zone_start + mft_zone_size;
+		else /* mft zone and data2 zone no longer exist. */
+			vol->data2_zone_pos = vol->mft_zone_start =
+					vol->mft_zone_end = 0;
+		if (vol->mft_zone_pos >= vol->mft_zone_end) {
+			vol->mft_zone_pos = vol->mft_lcn;
+			if (!vol->mft_zone_end)
+				vol->mft_zone_pos = 0;
+		}
+		bmp_pos = zone_start = bmp_initial_pos =
+				vol->data1_zone_pos = vol->mft_zone_end;
+		search_zone = 2;
+		pass = 2;
+		done_zones &= ~2;
+		ntfs_debug("After shrinking mft zone, mft_zone_size 0x%llx, vol->mft_zone_start 0x%llx, vol->mft_zone_end 0x%llx, vol->mft_zone_pos 0x%llx, search_zone 2, pass 2, dones_zones 0x%x, zone_start 0x%llx, zone_end 0x%llx, vol->data1_zone_pos 0x%llx, continuing outer while loop.",
+				mft_zone_size, vol->mft_zone_start,
+				vol->mft_zone_end, vol->mft_zone_pos,
+				done_zones, zone_start, zone_end,
+				vol->data1_zone_pos);
+	}
+	ntfs_debug("After outer while loop.");
+out:
+	ntfs_debug("At out.");
+	/* Add runlist terminator element. */
+	if (likely(rl)) {
+		rl[rlpos].vcn = rl[rlpos - 1].vcn + rl[rlpos - 1].length;
+		rl[rlpos].lcn = is_extension ? LCN_ENOENT : LCN_RL_NOT_MAPPED;
+		rl[rlpos].length = 0;
+	}
+	if (!IS_ERR_OR_NULL(folio)) {
+		if (need_writeback) {
+			ntfs_debug("Marking page dirty.");
+			folio_mark_dirty(folio);
+			need_writeback = 0;
+		}
+		folio_unlock(folio);
+		kunmap_local(buf);
+		folio_put(folio);
+	}
+	if (likely(!err)) {
+		if (!rl) {
+			err = -EIO;
+			goto out_restore;
+		}
+		if (is_dealloc == true)
+			ntfs_release_dirty_clusters(vol, rl->length);
+		ntfs_debug("Done.");
+		goto out_restore;
+	}
+	if (err != -ENOSPC)
+		ntfs_error(vol->sb,
+			"Failed to allocate clusters, aborting (error %i).",
+			err);
+	if (rl) {
+		int err2;
+
+		if (err == -ENOSPC)
+			ntfs_debug("Not enough space to complete allocation, err -ENOSPC, first free lcn 0x%llx, could allocate up to 0x%llx clusters.",
+					rl[0].lcn, count - clusters);
+		/* Deallocate all allocated clusters. */
+		ntfs_debug("Attempting rollback...");
+		err2 = ntfs_cluster_free_from_rl_nolock(vol, rl);
+		if (err2) {
+			ntfs_error(vol->sb,
+				"Failed to rollback (error %i). Leaving inconsistent metadata! Unmount and run chkdsk.",
+				err2);
+			NVolSetErrors(vol);
+		}
+		/* Free the runlist. */
+		kvfree(rl);
+	} else if (err == -ENOSPC)
+		ntfs_debug("No space left at all, err = -ENOSPC, first free lcn = 0x%llx.",
+				vol->data1_zone_pos);
+	atomic64_set(&vol->dirty_clusters, 0);
+
+out_restore:
+	up_write(&vol->lcnbmp_lock);
+	memalloc_nofs_restore(memalloc_flags);
+
+	return err < 0 ? ERR_PTR(err) : rl;
+}
+
+/*
+ * __ntfs_cluster_free - free clusters on an ntfs volume
+ * @ni:		ntfs inode whose runlist describes the clusters to free
+ * @start_vcn:	vcn in the runlist of @ni at which to start freeing clusters
+ * @count:	number of clusters to free or -1 for all clusters
+ * @ctx:	active attribute search context if present or NULL if not
+ * @is_rollback:	true if this is a rollback operation
+ *
+ * Free @count clusters starting at the cluster @start_vcn in the runlist
+ * described by the vfs inode @ni.
+ *
+ * If @count is -1, all clusters from @start_vcn to the end of the runlist are
+ * deallocated.  Thus, to completely free all clusters in a runlist, use
+ * @start_vcn = 0 and @count = -1.
+ *
+ * If @ctx is specified, it is an active search context of @ni and its base mft
+ * record.  This is needed when __ntfs_cluster_free() encounters unmapped
+ * runlist fragments and allows their mapping.  If you do not have the mft
+ * record mapped, you can specify @ctx as NULL and __ntfs_cluster_free() will
+ * perform the necessary mapping and unmapping.
+ *
+ * Note, __ntfs_cluster_free() saves the state of @ctx on entry and restores it
+ * before returning.  Thus, @ctx will be left pointing to the same attribute on
+ * return as on entry.  However, the actual pointers in @ctx may point to
+ * different memory locations on return, so you must remember to reset any
+ * cached pointers from the @ctx, i.e. after the call to __ntfs_cluster_free(),
+ * you will probably want to do:
+ *	m = ctx->mrec;
+ *	a = ctx->attr;
+ * Assuming you cache ctx->attr in a variable @a of type attr_record * and that
+ * you cache ctx->mrec in a variable @m of type struct mft_record *.
+ *
+ * @is_rollback should always be 'false', it is for internal use to rollback
+ * errors.  You probably want to use ntfs_cluster_free() instead.
+ *
+ * Note, __ntfs_cluster_free() does not modify the runlist, so you have to
+ * remove from the runlist or mark sparse the freed runs later.
+ *
+ * Return the number of deallocated clusters (not counting sparse ones) on
+ * success and -errno on error.
+ *
+ * WARNING: If @ctx is supplied, regardless of whether success or failure is
+ *	    returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx
+ *	    is no longer valid, i.e. you need to either call
+ *	    ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it.
+ *	    In that case PTR_ERR(@ctx->mrec) will give you the error code for
+ *	    why the mapping of the old inode failed.
+ *
+ * Locking: - The runlist described by @ni must be locked for writing on entry
+ *	      and is locked on return.  Note the runlist may be modified when
+ *	      needed runlist fragments need to be mapped.
+ *	    - The volume lcn bitmap must be unlocked on entry and is unlocked
+ *	      on return.
+ *	    - This function takes the volume lcn bitmap lock for writing and
+ *	      modifies the bitmap contents.
+ *	    - If @ctx is NULL, the base mft record of @ni must not be mapped on
+ *	      entry and it will be left unmapped on return.
+ *	    - If @ctx is not NULL, the base mft record must be mapped on entry
+ *	      and it will be left mapped on return.
+ */
+s64 __ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn, s64 count,
+		struct ntfs_attr_search_ctx *ctx, const bool is_rollback)
+{
+	s64 delta, to_free, total_freed, real_freed;
+	struct ntfs_volume *vol;
+	struct inode *lcnbmp_vi;
+	struct runlist_element *rl;
+	int err;
+	unsigned int memalloc_flags;
+
+	ntfs_debug("Entering for i_ino 0x%llx, start_vcn 0x%llx, count 0x%llx.%s",
+			ni->mft_no, start_vcn, count,
+			is_rollback ? " (rollback)" : "");
+	vol = ni->vol;
+	lcnbmp_vi = vol->lcnbmp_ino;
+	if (start_vcn < 0 || count < -1)
+		return -EINVAL;
+
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	/*
+	 * Lock the lcn bitmap for writing but only if not rolling back.  We
+	 * must hold the lock all the way including through rollback otherwise
+	 * rollback is not possible because once we have cleared a bit and
+	 * dropped the lock, anyone could have set the bit again, thus
+	 * allocating the cluster for another use.
+	 */
+	if (likely(!is_rollback)) {
+		memalloc_flags = memalloc_nofs_save();
+		down_write(&vol->lcnbmp_lock);
+	}
+
+	total_freed = real_freed = 0;
+
+	rl = ntfs_attr_find_vcn_nolock(ni, start_vcn, ctx);
+	if (IS_ERR(rl)) {
+		err = PTR_ERR(rl);
+		if (err == -ENOENT) {
+			if (likely(!is_rollback)) {
+				up_write(&vol->lcnbmp_lock);
+				memalloc_nofs_restore(memalloc_flags);
+			}
+			return 0;
+		}
+
+		if (!is_rollback)
+			ntfs_error(vol->sb,
+				"Failed to find first runlist element (error %d), aborting.",
+				err);
+		goto err_out;
+	}
+	if (unlikely(rl->lcn < LCN_HOLE)) {
+		if (!is_rollback)
+			ntfs_error(vol->sb, "First runlist element has invalid lcn, aborting.");
+		err = -EIO;
+		goto err_out;
+	}
+	/* Find the starting cluster inside the run that needs freeing. */
+	delta = start_vcn - rl->vcn;
+
+	/* The number of clusters in this run that need freeing. */
+	to_free = rl->length - delta;
+	if (count >= 0 && to_free > count)
+		to_free = count;
+
+	if (likely(rl->lcn >= 0)) {
+		/* Do the actual freeing of the clusters in this run. */
+		err = ntfs_bitmap_set_bits_in_run(lcnbmp_vi, rl->lcn + delta,
+				to_free, likely(!is_rollback) ? 0 : 1);
+		if (unlikely(err)) {
+			if (!is_rollback)
+				ntfs_error(vol->sb,
+					"Failed to clear first run (error %i), aborting.",
+					err);
+			goto err_out;
+		}
+		/* We have freed @to_free real clusters. */
+		real_freed = to_free;
+	}
+	/* Go to the next run and adjust the number of clusters left to free. */
+	++rl;
+	if (count >= 0)
+		count -= to_free;
+
+	/* Keep track of the total "freed" clusters, including sparse ones. */
+	total_freed = to_free;
+	/*
+	 * Loop over the remaining runs, using @count as a capping value, and
+	 * free them.
+	 */
+	for (; rl->length && count != 0; ++rl) {
+		if (unlikely(rl->lcn < LCN_HOLE)) {
+			s64 vcn;
+
+			/* Attempt to map runlist. */
+			vcn = rl->vcn;
+			rl = ntfs_attr_find_vcn_nolock(ni, vcn, ctx);
+			if (IS_ERR(rl)) {
+				err = PTR_ERR(rl);
+				if (!is_rollback)
+					ntfs_error(vol->sb,
+						"Failed to map runlist fragment or failed to find subsequent runlist element.");
+				goto err_out;
+			}
+			if (unlikely(rl->lcn < LCN_HOLE)) {
+				if (!is_rollback)
+					ntfs_error(vol->sb,
+						"Runlist element has invalid lcn (0x%llx).",
+						rl->lcn);
+				err = -EIO;
+				goto err_out;
+			}
+		}
+		/* The number of clusters in this run that need freeing. */
+		to_free = rl->length;
+		if (count >= 0 && to_free > count)
+			to_free = count;
+
+		if (likely(rl->lcn >= 0)) {
+			/* Do the actual freeing of the clusters in the run. */
+			err = ntfs_bitmap_set_bits_in_run(lcnbmp_vi, rl->lcn,
+					to_free, likely(!is_rollback) ? 0 : 1);
+			if (unlikely(err)) {
+				if (!is_rollback)
+					ntfs_error(vol->sb, "Failed to clear subsequent run.");
+				goto err_out;
+			}
+			/* We have freed @to_free real clusters. */
+			real_freed += to_free;
+		}
+		/* Adjust the number of clusters left to free. */
+		if (count >= 0)
+			count -= to_free;
+
+		/* Update the total done clusters. */
+		total_freed += to_free;
+	}
+	ntfs_inc_free_clusters(vol, real_freed);
+	if (likely(!is_rollback)) {
+		up_write(&vol->lcnbmp_lock);
+		memalloc_nofs_restore(memalloc_flags);
+	}
+
+	WARN_ON(count > 0);
+
+	if (NVolDiscard(vol) && !is_rollback) {
+		s64 total_discarded = 0, rl_off;
+		u32 gran = bdev_discard_granularity(vol->sb->s_bdev);
+
+		rl = ntfs_attr_find_vcn_nolock(ni, start_vcn, ctx);
+		if (IS_ERR(rl))
+			return real_freed;
+		rl_off = start_vcn - rl->vcn;
+		while (rl->length && total_discarded < total_freed) {
+			s64 to_discard = rl->length - rl_off;
+
+			if (to_discard + total_discarded > total_freed)
+				to_discard = total_freed - total_discarded;
+			if (rl->lcn >= 0) {
+				sector_t start_sector, end_sector;
+				int ret;
+
+				start_sector = ALIGN(NTFS_CLU_TO_B(vol, rl->lcn + rl_off),
+						     gran) >> SECTOR_SHIFT;
+				end_sector = ALIGN_DOWN(NTFS_CLU_TO_B(vol,
+							rl->lcn + rl_off + to_discard),
+							gran) >> SECTOR_SHIFT;
+				if (start_sector < end_sector) {
+					ret = blkdev_issue_discard(vol->sb->s_bdev, start_sector,
+								   end_sector - start_sector,
+								   GFP_NOFS);
+					if (ret)
+						break;
+				}
+			}
+
+			total_discarded += to_discard;
+			++rl;
+			rl_off = 0;
+		}
+	}
+
+	/* We are done.  Return the number of actually freed clusters. */
+	ntfs_debug("Done.");
+	return real_freed;
+err_out:
+	if (is_rollback)
+		return err;
+	/* If no real clusters were freed, no need to rollback. */
+	if (!real_freed) {
+		up_write(&vol->lcnbmp_lock);
+		memalloc_nofs_restore(memalloc_flags);
+		return err;
+	}
+	/*
+	 * Attempt to rollback and if that succeeds just return the error code.
+	 * If rollback fails, set the volume errors flag, emit an error
+	 * message, and return the error code.
+	 */
+	delta = __ntfs_cluster_free(ni, start_vcn, total_freed, ctx, true);
+	if (delta < 0) {
+		ntfs_error(vol->sb,
+			"Failed to rollback (error %i).  Leaving inconsistent metadata!  Unmount and run chkdsk.",
+			(int)delta);
+		NVolSetErrors(vol);
+	}
+	ntfs_dec_free_clusters(vol, delta);
+	up_write(&vol->lcnbmp_lock);
+	memalloc_nofs_restore(memalloc_flags);
+	ntfs_error(vol->sb, "Aborting (error %i).", err);
+	return err;
+}
diff --git a/fs/ntfs/lcnalloc.h b/fs/ntfs/lcnalloc.h
new file mode 100644
index 0000000000000..d9ac622e4658c
--- /dev/null
+++ b/fs/ntfs/lcnalloc.h
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Exports for NTFS kernel cluster (de)allocation.
+ *
+ * Copyright (c) 2004-2005 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_LCNALLOC_H
+#define _LINUX_NTFS_LCNALLOC_H
+
+#include <linux/sched/mm.h>
+
+#include "attrib.h"
+
+/*
+ * enum zone_type - Zone identifiers for cluster allocation policy
+ *
+ * FIRST_ZONE		For sanity checking.
+ * MFT_ZONE		Allocate from $MFT zone.
+ * DATA_ZONE		Allocate from $DATA zone.
+ * LAST_ZONE		For sanity checking.
+ */
+enum {
+	FIRST_ZONE	= 0,
+	MFT_ZONE	= 0,
+	DATA_ZONE	= 1,
+	LAST_ZONE	= 1,
+};
+
+struct runlist_element *ntfs_cluster_alloc(struct ntfs_volume *vol,
+		const s64 start_vcn, const s64 count, const s64 start_lcn,
+		const int zone,
+		const bool is_extension,
+		const bool is_contig,
+		const bool is_dealloc);
+s64 __ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn,
+		s64 count, struct ntfs_attr_search_ctx *ctx, const bool is_rollback);
+
+/*
+ * ntfs_cluster_free - free clusters on an ntfs volume
+ * @ni:		ntfs inode whose runlist describes the clusters to free
+ * @start_vcn:	vcn in the runlist of @ni at which to start freeing clusters
+ * @count:	number of clusters to free or -1 for all clusters
+ * @ctx:	active attribute search context if present or NULL if not
+ *
+ * Free @count clusters starting at the cluster @start_vcn in the runlist
+ * described by the ntfs inode @ni.
+ *
+ * If @count is -1, all clusters from @start_vcn to the end of the runlist are
+ * deallocated.  Thus, to completely free all clusters in a runlist, use
+ * @start_vcn = 0 and @count = -1.
+ *
+ * If @ctx is specified, it is an active search context of @ni and its base mft
+ * record.  This is needed when ntfs_cluster_free() encounters unmapped runlist
+ * fragments and allows their mapping.  If you do not have the mft record
+ * mapped, you can specify @ctx as NULL and ntfs_cluster_free() will perform
+ * the necessary mapping and unmapping.
+ *
+ * Note, ntfs_cluster_free() saves the state of @ctx on entry and restores it
+ * before returning.  Thus, @ctx will be left pointing to the same attribute on
+ * return as on entry.  However, the actual pointers in @ctx may point to
+ * different memory locations on return, so you must remember to reset any
+ * cached pointers from the @ctx, i.e. after the call to ntfs_cluster_free(),
+ * you will probably want to do:
+ *	m = ctx->mrec;
+ *	a = ctx->attr;
+ * Assuming you cache ctx->attr in a variable @a of type ATTR_RECORD * and that
+ * you cache ctx->mrec in a variable @m of type MFT_RECORD *.
+ *
+ * Note, ntfs_cluster_free() does not modify the runlist, so you have to remove
+ * from the runlist or mark sparse the freed runs later.
+ *
+ * Return the number of deallocated clusters (not counting sparse ones) on
+ * success and -errno on error.
+ *
+ * WARNING: If @ctx is supplied, regardless of whether success or failure is
+ *	    returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx
+ *	    is no longer valid, i.e. you need to either call
+ *	    ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it.
+ *	    In that case PTR_ERR(@ctx->mrec) will give you the error code for
+ *	    why the mapping of the old inode failed.
+ *
+ * Locking: - The runlist described by @ni must be locked for writing on entry
+ *	      and is locked on return.  Note the runlist may be modified when
+ *	      needed runlist fragments need to be mapped.
+ *	    - The volume lcn bitmap must be unlocked on entry and is unlocked
+ *	      on return.
+ *	    - This function takes the volume lcn bitmap lock for writing and
+ *	      modifies the bitmap contents.
+ *	    - If @ctx is NULL, the base mft record of @ni must not be mapped on
+ *	      entry and it will be left unmapped on return.
+ *	    - If @ctx is not NULL, the base mft record must be mapped on entry
+ *	      and it will be left mapped on return.
+ */
+static inline s64 ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn,
+		s64 count, struct ntfs_attr_search_ctx *ctx)
+{
+	return __ntfs_cluster_free(ni, start_vcn, count, ctx, false);
+}
+
+int ntfs_cluster_free_from_rl_nolock(struct ntfs_volume *vol,
+		const struct runlist_element *rl);
+
+/*
+ * ntfs_cluster_free_from_rl - free clusters from runlist
+ * @vol:	mounted ntfs volume on which to free the clusters
+ * @rl:		runlist describing the clusters to free
+ *
+ * Free all the clusters described by the runlist @rl on the volume @vol.  In
+ * the case of an error being returned, at least some of the clusters were not
+ * freed.
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Locking: - This function takes the volume lcn bitmap lock for writing and
+ *	      modifies the bitmap contents.
+ *	    - The caller must have locked the runlist @rl for reading or
+ *	      writing.
+ */
+static inline int ntfs_cluster_free_from_rl(struct ntfs_volume *vol,
+		const struct runlist_element *rl)
+{
+	int ret;
+	unsigned int memalloc_flags;
+
+	memalloc_flags = memalloc_nofs_save();
+	down_write(&vol->lcnbmp_lock);
+	ret = ntfs_cluster_free_from_rl_nolock(vol, rl);
+	up_write(&vol->lcnbmp_lock);
+	memalloc_nofs_restore(memalloc_flags);
+	return ret;
+}
+
+#endif /* defined _LINUX_NTFS_LCNALLOC_H */
diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c
new file mode 100644
index 0000000000000..3f8d1640f1d50
--- /dev/null
+++ b/fs/ntfs/logfile.c
@@ -0,0 +1,778 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel journal handling.
+ *
+ * Copyright (c) 2002-2007 Anton Altaparmakov
+ */
+
+#include <linux/blkdev.h>
+
+#include "attrib.h"
+#include "logfile.h"
+#include "ntfs.h"
+
+/*
+ * ntfs_check_restart_page_header - check the page header for consistency
+ * @vi:		LogFile inode to which the restart page header belongs
+ * @rp:		restart page header to check
+ * @pos:	position in @vi at which the restart page header resides
+ *
+ * Check the restart page header @rp for consistency and return 'true' if it is
+ * consistent and 'false' otherwise.
+ *
+ * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not
+ * require the full restart page.
+ */
+static bool ntfs_check_restart_page_header(struct inode *vi,
+		struct restart_page_header *rp, s64 pos)
+{
+	u32 logfile_system_page_size, logfile_log_page_size;
+	u16 ra_ofs, usa_count, usa_ofs, usa_end = 0;
+	bool have_usa = true;
+
+	ntfs_debug("Entering.");
+	/*
+	 * If the system or log page sizes are smaller than the ntfs block size
+	 * or either is not a power of 2 we cannot handle this log file.
+	 */
+	logfile_system_page_size = le32_to_cpu(rp->system_page_size);
+	logfile_log_page_size = le32_to_cpu(rp->log_page_size);
+	if (logfile_system_page_size < NTFS_BLOCK_SIZE ||
+			logfile_log_page_size < NTFS_BLOCK_SIZE ||
+			logfile_system_page_size &
+			(logfile_system_page_size - 1) ||
+			!is_power_of_2(logfile_log_page_size)) {
+		ntfs_error(vi->i_sb, "LogFile uses unsupported page size.");
+		return false;
+	}
+	/*
+	 * We must be either at !pos (1st restart page) or at pos = system page
+	 * size (2nd restart page).
+	 */
+	if (pos && pos != logfile_system_page_size) {
+		ntfs_error(vi->i_sb, "Found restart area in incorrect position in LogFile.");
+		return false;
+	}
+	/* We only know how to handle version 1.1. */
+	if (le16_to_cpu(rp->major_ver) != 1 ||
+	    le16_to_cpu(rp->minor_ver) != 1) {
+		ntfs_error(vi->i_sb,
+			"LogFile version %i.%i is not supported.  (This driver supports version 1.1 only.)",
+			(int)le16_to_cpu(rp->major_ver),
+			(int)le16_to_cpu(rp->minor_ver));
+		return false;
+	}
+	/*
+	 * If chkdsk has been run the restart page may not be protected by an
+	 * update sequence array.
+	 */
+	if (ntfs_is_chkd_record(rp->magic) && !le16_to_cpu(rp->usa_count)) {
+		have_usa = false;
+		goto skip_usa_checks;
+	}
+	/* Verify the size of the update sequence array. */
+	usa_count = 1 + (logfile_system_page_size >> NTFS_BLOCK_SIZE_BITS);
+	if (usa_count != le16_to_cpu(rp->usa_count)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart page specifies inconsistent update sequence array count.");
+		return false;
+	}
+	/* Verify the position of the update sequence array. */
+	usa_ofs = le16_to_cpu(rp->usa_ofs);
+	usa_end = usa_ofs + usa_count * sizeof(u16);
+	if (usa_ofs < sizeof(struct restart_page_header) ||
+			usa_end > NTFS_BLOCK_SIZE - sizeof(u16)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart page specifies inconsistent update sequence array offset.");
+		return false;
+	}
+skip_usa_checks:
+	/*
+	 * Verify the position of the restart area.  It must be:
+	 *	- aligned to 8-byte boundary,
+	 *	- after the update sequence array, and
+	 *	- within the system page size.
+	 */
+	ra_ofs = le16_to_cpu(rp->restart_area_offset);
+	if (ra_ofs & 7 || (have_usa ? ra_ofs < usa_end :
+			ra_ofs < sizeof(struct restart_page_header)) ||
+			ra_ofs > logfile_system_page_size) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart page specifies inconsistent restart area offset.");
+		return false;
+	}
+	/*
+	 * Only restart pages modified by chkdsk are allowed to have chkdsk_lsn
+	 * set.
+	 */
+	if (!ntfs_is_chkd_record(rp->magic) && le64_to_cpu(rp->chkdsk_lsn)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart page is not modified by chkdsk but a chkdsk LSN is specified.");
+		return false;
+	}
+	ntfs_debug("Done.");
+	return true;
+}
+
+/*
+ * ntfs_check_restart_area - check the restart area for consistency
+ * @vi:		LogFile inode to which the restart page belongs
+ * @rp:		restart page whose restart area to check
+ *
+ * Check the restart area of the restart page @rp for consistency and return
+ * 'true' if it is consistent and 'false' otherwise.
+ *
+ * This function assumes that the restart page header has already been
+ * consistency checked.
+ *
+ * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not
+ * require the full restart page.
+ */
+static bool ntfs_check_restart_area(struct inode *vi, struct restart_page_header *rp)
+{
+	u64 file_size;
+	struct restart_area *ra;
+	u16 ra_ofs, ra_len, ca_ofs;
+	u8 fs_bits;
+
+	ntfs_debug("Entering.");
+	ra_ofs = le16_to_cpu(rp->restart_area_offset);
+	ra = (struct restart_area *)((u8 *)rp + ra_ofs);
+	/*
+	 * Everything before ra->file_size must be before the first word
+	 * protected by an update sequence number.  This ensures that it is
+	 * safe to access ra->client_array_offset.
+	 */
+	if (ra_ofs + offsetof(struct restart_area, file_size) >
+			NTFS_BLOCK_SIZE - sizeof(u16)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies inconsistent file offset.");
+		return false;
+	}
+	/*
+	 * Now that we can access ra->client_array_offset, make sure everything
+	 * up to the log client array is before the first word protected by an
+	 * update sequence number.  This ensures we can access all of the
+	 * restart area elements safely.  Also, the client array offset must be
+	 * aligned to an 8-byte boundary.
+	 */
+	ca_ofs = le16_to_cpu(ra->client_array_offset);
+	if (((ca_ofs + 7) & ~7) != ca_ofs ||
+			ra_ofs + ca_ofs > NTFS_BLOCK_SIZE - sizeof(u16)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies inconsistent client array offset.");
+		return false;
+	}
+	/*
+	 * The restart area must end within the system page size both when
+	 * calculated manually and as specified by ra->restart_area_length.
+	 * Also, the calculated length must not exceed the specified length.
+	 */
+	ra_len = ca_ofs + le16_to_cpu(ra->log_clients) *
+			sizeof(struct log_client_record);
+	if (ra_ofs + ra_len > le32_to_cpu(rp->system_page_size) ||
+			ra_ofs + le16_to_cpu(ra->restart_area_length) >
+			le32_to_cpu(rp->system_page_size) ||
+			ra_len > le16_to_cpu(ra->restart_area_length)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area is out of bounds of the system page size specified by the restart page header and/or the specified restart area length is inconsistent.");
+		return false;
+	}
+	/*
+	 * The ra->client_free_list and ra->client_in_use_list must be either
+	 * LOGFILE_NO_CLIENT or less than ra->log_clients or they are
+	 * overflowing the client array.
+	 */
+	if ((ra->client_free_list != LOGFILE_NO_CLIENT &&
+			le16_to_cpu(ra->client_free_list) >=
+			le16_to_cpu(ra->log_clients)) ||
+			(ra->client_in_use_list != LOGFILE_NO_CLIENT &&
+			le16_to_cpu(ra->client_in_use_list) >=
+			le16_to_cpu(ra->log_clients))) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies overflowing client free and/or in use lists.");
+		return false;
+	}
+	/*
+	 * Check ra->seq_number_bits against ra->file_size for consistency.
+	 * We cannot just use ffs() because the file size is not a power of 2.
+	 */
+	file_size = le64_to_cpu(ra->file_size);
+	fs_bits = 0;
+	while (file_size) {
+		file_size >>= 1;
+		fs_bits++;
+	}
+	if (le32_to_cpu(ra->seq_number_bits) != 67 - fs_bits) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies inconsistent sequence number bits.");
+		return false;
+	}
+	/* The log record header length must be a multiple of 8. */
+	if (((le16_to_cpu(ra->log_record_header_length) + 7) & ~7) !=
+			le16_to_cpu(ra->log_record_header_length)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies inconsistent log record header length.");
+		return false;
+	}
+	/* Dito for the log page data offset. */
+	if (((le16_to_cpu(ra->log_page_data_offset) + 7) & ~7) !=
+			le16_to_cpu(ra->log_page_data_offset)) {
+		ntfs_error(vi->i_sb,
+			"LogFile restart area specifies inconsistent log page data offset.");
+		return false;
+	}
+	ntfs_debug("Done.");
+	return true;
+}
+
+/*
+ * ntfs_check_log_client_array - check the log client array for consistency
+ * @vi:		LogFile inode to which the restart page belongs
+ * @rp:		restart page whose log client array to check
+ *
+ * Check the log client array of the restart page @rp for consistency and
+ * return 'true' if it is consistent and 'false' otherwise.
+ *
+ * This function assumes that the restart page header and the restart area have
+ * already been consistency checked.
+ *
+ * Unlike ntfs_check_restart_page_header() and ntfs_check_restart_area(), this
+ * function needs @rp->system_page_size bytes in @rp, i.e. it requires the full
+ * restart page and the page must be multi sector transfer deprotected.
+ */
+static bool ntfs_check_log_client_array(struct inode *vi,
+		struct restart_page_header *rp)
+{
+	struct restart_area *ra;
+	struct log_client_record *ca, *cr;
+	u16 nr_clients, idx;
+	bool in_free_list, idx_is_first;
+
+	ntfs_debug("Entering.");
+	ra = (struct restart_area *)((u8 *)rp + le16_to_cpu(rp->restart_area_offset));
+	ca = (struct log_client_record *)((u8 *)ra +
+			le16_to_cpu(ra->client_array_offset));
+	/*
+	 * Check the ra->client_free_list first and then check the
+	 * ra->client_in_use_list.  Check each of the log client records in
+	 * each of the lists and check that the array does not overflow the
+	 * ra->log_clients value.  Also keep track of the number of records
+	 * visited as there cannot be more than ra->log_clients records and
+	 * that way we detect eventual loops in within a list.
+	 */
+	nr_clients = le16_to_cpu(ra->log_clients);
+	idx = le16_to_cpu(ra->client_free_list);
+	in_free_list = true;
+check_list:
+	for (idx_is_first = true; idx != LOGFILE_NO_CLIENT_CPU; nr_clients--,
+			idx = le16_to_cpu(cr->next_client)) {
+		if (!nr_clients || idx >= le16_to_cpu(ra->log_clients))
+			goto err_out;
+		/* Set @cr to the current log client record. */
+		cr = ca + idx;
+		/* The first log client record must not have a prev_client. */
+		if (idx_is_first) {
+			if (cr->prev_client != LOGFILE_NO_CLIENT)
+				goto err_out;
+			idx_is_first = false;
+		}
+	}
+	/* Switch to and check the in use list if we just did the free list. */
+	if (in_free_list) {
+		in_free_list = false;
+		idx = le16_to_cpu(ra->client_in_use_list);
+		goto check_list;
+	}
+	ntfs_debug("Done.");
+	return true;
+err_out:
+	ntfs_error(vi->i_sb, "LogFile log client array is corrupt.");
+	return false;
+}
+
+/*
+ * ntfs_check_and_load_restart_page - check the restart page for consistency
+ * @vi:		LogFile inode to which the restart page belongs
+ * @rp:		restart page to check
+ * @pos:	position in @vi at which the restart page resides
+ * @wrp:	[OUT] copy of the multi sector transfer deprotected restart page
+ * @lsn:	[OUT] set to the current logfile lsn on success
+ *
+ * Check the restart page @rp for consistency and return 0 if it is consistent
+ * and -errno otherwise.  The restart page may have been modified by chkdsk in
+ * which case its magic is CHKD instead of RSTR.
+ *
+ * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not
+ * require the full restart page.
+ *
+ * If @wrp is not NULL, on success, *@wrp will point to a buffer containing a
+ * copy of the complete multi sector transfer deprotected page.  On failure,
+ * *@wrp is undefined.
+ *
+ * Simillarly, if @lsn is not NULL, on success *@lsn will be set to the current
+ * logfile lsn according to this restart page.  On failure, *@lsn is undefined.
+ *
+ * The following error codes are defined:
+ *	-EINVAL	- The restart page is inconsistent.
+ *	-ENOMEM	- Not enough memory to load the restart page.
+ *	-EIO	- Failed to reading from LogFile.
+ */
+static int ntfs_check_and_load_restart_page(struct inode *vi,
+		struct restart_page_header *rp, s64 pos, struct restart_page_header **wrp,
+		s64 *lsn)
+{
+	struct restart_area *ra;
+	struct restart_page_header *trp;
+	int size, err;
+
+	ntfs_debug("Entering.");
+	/* Check the restart page header for consistency. */
+	if (!ntfs_check_restart_page_header(vi, rp, pos)) {
+		/* Error output already done inside the function. */
+		return -EINVAL;
+	}
+	/* Check the restart area for consistency. */
+	if (!ntfs_check_restart_area(vi, rp)) {
+		/* Error output already done inside the function. */
+		return -EINVAL;
+	}
+	ra = (struct restart_area *)((u8 *)rp + le16_to_cpu(rp->restart_area_offset));
+	/*
+	 * Allocate a buffer to store the whole restart page so we can multi
+	 * sector transfer deprotect it.
+	 */
+	trp = kvzalloc(le32_to_cpu(rp->system_page_size), GFP_NOFS);
+	if (!trp) {
+		ntfs_error(vi->i_sb, "Failed to allocate memory for LogFile restart page buffer.");
+		return -ENOMEM;
+	}
+	/*
+	 * Read the whole of the restart page into the buffer.  If it fits
+	 * completely inside @rp, just copy it from there.  Otherwise map all
+	 * the required pages and copy the data from them.
+	 */
+	size = PAGE_SIZE - (pos & ~PAGE_MASK);
+	if (size >= le32_to_cpu(rp->system_page_size)) {
+		memcpy(trp, rp, le32_to_cpu(rp->system_page_size));
+	} else {
+		pgoff_t idx;
+		struct folio *folio;
+		int have_read, to_read;
+
+		/* First copy what we already have in @rp. */
+		memcpy(trp, rp, size);
+		/* Copy the remaining data one page at a time. */
+		have_read = size;
+		to_read = le32_to_cpu(rp->system_page_size) - size;
+		idx = (pos + size) >> PAGE_SHIFT;
+		do {
+			folio = read_mapping_folio(vi->i_mapping, idx, NULL);
+			if (IS_ERR(folio)) {
+				ntfs_error(vi->i_sb, "Error mapping LogFile page (index %lu).",
+						idx);
+				err = PTR_ERR(folio);
+				if (err != -EIO && err != -ENOMEM)
+					err = -EIO;
+				goto err_out;
+			}
+			size = min_t(int, to_read, PAGE_SIZE);
+			memcpy((u8 *)trp + have_read, folio_address(folio), size);
+			folio_put(folio);
+			have_read += size;
+			to_read -= size;
+			idx++;
+		} while (to_read > 0);
+	}
+	/*
+	 * Perform the multi sector transfer deprotection on the buffer if the
+	 * restart page is protected.
+	 */
+	if ((!ntfs_is_chkd_record(trp->magic) || le16_to_cpu(trp->usa_count)) &&
+	    post_read_mst_fixup((struct ntfs_record *)trp, le32_to_cpu(rp->system_page_size))) {
+		/*
+		 * A multi sector transfer error was detected.  We only need to
+		 * abort if the restart page contents exceed the multi sector
+		 * transfer fixup of the first sector.
+		 */
+		if (le16_to_cpu(rp->restart_area_offset) +
+				le16_to_cpu(ra->restart_area_length) >
+				NTFS_BLOCK_SIZE - sizeof(u16)) {
+			ntfs_error(vi->i_sb,
+				"Multi sector transfer error detected in LogFile restart page.");
+			err = -EINVAL;
+			goto err_out;
+		}
+	}
+	/*
+	 * If the restart page is modified by chkdsk or there are no active
+	 * logfile clients, the logfile is consistent.  Otherwise, need to
+	 * check the log client records for consistency, too.
+	 */
+	err = 0;
+	if (ntfs_is_rstr_record(rp->magic) &&
+			ra->client_in_use_list != LOGFILE_NO_CLIENT) {
+		if (!ntfs_check_log_client_array(vi, trp)) {
+			err = -EINVAL;
+			goto err_out;
+		}
+	}
+	if (lsn) {
+		if (ntfs_is_rstr_record(rp->magic))
+			*lsn = le64_to_cpu(ra->current_lsn);
+		else /* if (ntfs_is_chkd_record(rp->magic)) */
+			*lsn = le64_to_cpu(rp->chkdsk_lsn);
+	}
+	ntfs_debug("Done.");
+	if (wrp)
+		*wrp = trp;
+	else {
+err_out:
+		kvfree(trp);
+	}
+	return err;
+}
+
+/*
+ * ntfs_check_logfile - check the journal for consistency
+ * @log_vi:	struct inode of loaded journal LogFile to check
+ * @rp:		[OUT] on success this is a copy of the current restart page
+ *
+ * Check the LogFile journal for consistency and return 'true' if it is
+ * consistent and 'false' if not.  On success, the current restart page is
+ * returned in *@rp.  Caller must call kvfree(*@rp) when finished with it.
+ *
+ * At present we only check the two restart pages and ignore the log record
+ * pages.
+ *
+ * Note that the MstProtected flag is not set on the LogFile inode and hence
+ * when reading pages they are not deprotected.  This is because we do not know
+ * if the LogFile was created on a system with a different page size to ours
+ * yet and mst deprotection would fail if our page size is smaller.
+ */
+bool ntfs_check_logfile(struct inode *log_vi, struct restart_page_header **rp)
+{
+	s64 size, pos;
+	s64 rstr1_lsn, rstr2_lsn;
+	struct ntfs_volume *vol = NTFS_SB(log_vi->i_sb);
+	struct address_space *mapping = log_vi->i_mapping;
+	struct folio *folio = NULL;
+	u8 *kaddr = NULL;
+	struct restart_page_header *rstr1_ph = NULL;
+	struct restart_page_header *rstr2_ph = NULL;
+	int log_page_size, err;
+	bool logfile_is_empty = true;
+	u8 log_page_bits;
+
+	ntfs_debug("Entering.");
+	/* An empty LogFile must have been clean before it got emptied. */
+	if (NVolLogFileEmpty(vol))
+		goto is_empty;
+	size = i_size_read(log_vi);
+	/* Make sure the file doesn't exceed the maximum allowed size. */
+	if (size > MaxLogFileSize)
+		size = MaxLogFileSize;
+	/*
+	 * Truncate size to a multiple of the page cache size or the default
+	 * log page size if the page cache size is between the default log page
+	 * log page size if the page cache size is between the default log page
+	 * size and twice that.
+	 */
+	if (DefaultLogPageSize <= PAGE_SIZE &&
+	    DefaultLogPageSize * 2 <= PAGE_SIZE)
+		log_page_size = DefaultLogPageSize;
+	else
+		log_page_size = PAGE_SIZE;
+	/*
+	 * Use ntfs_ffs() instead of ffs() to enable the compiler to
+	 * optimize log_page_size and log_page_bits into constants.
+	 */
+	log_page_bits = ntfs_ffs(log_page_size) - 1;
+	size &= ~(s64)(log_page_size - 1);
+	/*
+	 * Ensure the log file is big enough to store at least the two restart
+	 * pages and the minimum number of log record pages.
+	 */
+	if (size < log_page_size * 2 || (size - log_page_size * 2) >>
+			log_page_bits < MinLogRecordPages) {
+		ntfs_error(vol->sb, "LogFile is too small.");
+		return false;
+	}
+	/*
+	 * Read through the file looking for a restart page.  Since the restart
+	 * page header is at the beginning of a page we only need to search at
+	 * what could be the beginning of a page (for each page size) rather
+	 * than scanning the whole file byte by byte.  If all potential places
+	 * contain empty and uninitialzed records, the log file can be assumed
+	 * to be empty.
+	 */
+	for (pos = 0; pos < size; pos <<= 1) {
+		pgoff_t idx = pos >> PAGE_SHIFT;
+
+		if (!folio || folio->index != idx) {
+			if (folio) {
+				kunmap_local(kaddr);
+				folio_put(folio);
+			}
+			folio = read_mapping_folio(mapping, idx, NULL);
+			if (IS_ERR(folio)) {
+				ntfs_error(vol->sb, "Error mapping LogFile page (index %lu).",
+						idx);
+				goto err_out;
+			}
+		}
+		kaddr = (u8 *)kmap_local_folio(folio, 0) + (pos & ~PAGE_MASK);
+		/*
+		 * A non-empty block means the logfile is not empty while an
+		 * empty block after a non-empty block has been encountered
+		 * means we are done.
+		 */
+		if (!ntfs_is_empty_recordp((__le32 *)kaddr))
+			logfile_is_empty = false;
+		else if (!logfile_is_empty)
+			break;
+		/*
+		 * A log record page means there cannot be a restart page after
+		 * this so no need to continue searching.
+		 */
+		if (ntfs_is_rcrd_recordp((__le32 *)kaddr))
+			break;
+		/* If not a (modified by chkdsk) restart page, continue. */
+		if (!ntfs_is_rstr_recordp((__le32 *)kaddr) &&
+				!ntfs_is_chkd_recordp((__le32 *)kaddr)) {
+			if (!pos)
+				pos = NTFS_BLOCK_SIZE >> 1;
+			continue;
+		}
+		/*
+		 * Check the (modified by chkdsk) restart page for consistency
+		 * and get a copy of the complete multi sector transfer
+		 * deprotected restart page.
+		 */
+		err = ntfs_check_and_load_restart_page(log_vi,
+				(struct restart_page_header *)kaddr, pos,
+				!rstr1_ph ? &rstr1_ph : &rstr2_ph,
+				!rstr1_ph ? &rstr1_lsn : &rstr2_lsn);
+		if (!err) {
+			/*
+			 * If we have now found the first (modified by chkdsk)
+			 * restart page, continue looking for the second one.
+			 */
+			if (!pos) {
+				pos = NTFS_BLOCK_SIZE >> 1;
+				continue;
+			}
+			/*
+			 * We have now found the second (modified by chkdsk)
+			 * restart page, so we can stop looking.
+			 */
+			break;
+		}
+		/*
+		 * Error output already done inside the function.  Note, we do
+		 * not abort if the restart page was invalid as we might still
+		 * find a valid one further in the file.
+		 */
+		if (err != -EINVAL) {
+			kunmap_local(kaddr);
+			folio_put(folio);
+			goto err_out;
+		}
+		/* Continue looking. */
+		if (!pos)
+			pos = NTFS_BLOCK_SIZE >> 1;
+	}
+	if (folio) {
+		kunmap_local(kaddr);
+		folio_put(folio);
+	}
+	if (logfile_is_empty) {
+		NVolSetLogFileEmpty(vol);
+is_empty:
+		ntfs_debug("Done.  (LogFile is empty.)");
+		return true;
+	}
+	if (!rstr1_ph) {
+		ntfs_error(vol->sb,
+			"Did not find any restart pages in LogFile and it was not empty.");
+		return false;
+	}
+	/* If both restart pages were found, use the more recent one. */
+	if (rstr2_ph) {
+		/*
+		 * If the second restart area is more recent, switch to it.
+		 * Otherwise just throw it away.
+		 */
+		if (rstr2_lsn > rstr1_lsn) {
+			ntfs_debug("Using second restart page as it is more recent.");
+			kvfree(rstr1_ph);
+			rstr1_ph = rstr2_ph;
+			/* rstr1_lsn = rstr2_lsn; */
+		} else {
+			ntfs_debug("Using first restart page as it is more recent.");
+			kvfree(rstr2_ph);
+		}
+		rstr2_ph = NULL;
+	}
+	/* All consistency checks passed. */
+	if (rp)
+		*rp = rstr1_ph;
+	else
+		kvfree(rstr1_ph);
+	ntfs_debug("Done.");
+	return true;
+err_out:
+	if (rstr1_ph)
+		kvfree(rstr1_ph);
+	return false;
+}
+
+/*
+ * ntfs_empty_logfile - empty the contents of the LogFile journal
+ * @log_vi:	struct inode of loaded journal LogFile to empty
+ *
+ * Empty the contents of the LogFile journal @log_vi and return 'true' on
+ * success and 'false' on error.
+ *
+ * This function assumes that the LogFile journal has already been consistency
+ * checked by a call to ntfs_check_logfile() and that ntfs_is_logfile_clean()
+ * has been used to ensure that the LogFile is clean.
+ */
+bool ntfs_empty_logfile(struct inode *log_vi)
+{
+	s64 vcn, end_vcn;
+	struct ntfs_inode *log_ni = NTFS_I(log_vi);
+	struct ntfs_volume *vol = log_ni->vol;
+	struct super_block *sb = vol->sb;
+	struct runlist_element *rl;
+	unsigned long flags;
+	int err;
+	bool should_wait = true;
+	char *empty_buf = NULL;
+	struct file_ra_state *ra = NULL;
+
+	ntfs_debug("Entering.");
+	if (NVolLogFileEmpty(vol)) {
+		ntfs_debug("Done.");
+		return true;
+	}
+
+	/*
+	 * We cannot use ntfs_attr_set() because we may be still in the middle
+	 * of a mount operation.  Thus we do the emptying by hand by first
+	 * zapping the page cache pages for the LogFile/DATA attribute and
+	 * then emptying each of the buffers in each of the clusters specified
+	 * by the runlist by hand.
+	 */
+	vcn = 0;
+	read_lock_irqsave(&log_ni->size_lock, flags);
+	end_vcn = (log_ni->initialized_size + vol->cluster_size_mask) >>
+			vol->cluster_size_bits;
+	read_unlock_irqrestore(&log_ni->size_lock, flags);
+	truncate_inode_pages(log_vi->i_mapping, 0);
+	down_write(&log_ni->runlist.lock);
+	rl = log_ni->runlist.rl;
+	if (unlikely(!rl || vcn < rl->vcn || !rl->length)) {
+map_vcn:
+		err = ntfs_map_runlist_nolock(log_ni, vcn, NULL);
+		if (err) {
+			ntfs_error(sb, "Failed to map runlist fragment (error %d).", -err);
+			goto err;
+		}
+		rl = log_ni->runlist.rl;
+	}
+	/* Seek to the runlist element containing @vcn. */
+	while (rl->length && vcn >= rl[1].vcn)
+		rl++;
+
+	err = -ENOMEM;
+	empty_buf = kvzalloc(vol->cluster_size, GFP_NOFS);
+	if (!empty_buf)
+		goto err;
+
+	memset(empty_buf, 0xff, vol->cluster_size);
+
+	ra = kzalloc(sizeof(*ra), GFP_NOFS);
+	if (!ra)
+		goto err;
+
+	file_ra_state_init(ra, sb->s_bdev->bd_mapping);
+	do {
+		s64 lcn;
+		loff_t start, end;
+		s64 len;
+
+		/*
+		 * If this run is not mapped map it now and start again as the
+		 * runlist will have been updated.
+		 */
+		lcn = rl->lcn;
+		if (unlikely(lcn == LCN_RL_NOT_MAPPED)) {
+			vcn = rl->vcn;
+			kvfree(empty_buf);
+			goto map_vcn;
+		}
+		/* If this run is not valid abort with an error. */
+		if (unlikely(!rl->length || lcn < LCN_HOLE))
+			goto rl_err;
+		/* Skip holes. */
+		if (lcn == LCN_HOLE)
+			continue;
+		start = NTFS_CLU_TO_B(vol, lcn);
+		len = rl->length;
+		if (rl[1].vcn > end_vcn)
+			len = end_vcn - rl->vcn;
+		end = NTFS_CLU_TO_B(vol, lcn + len);
+
+		page_cache_sync_readahead(sb->s_bdev->bd_mapping, ra, NULL,
+			start >> PAGE_SHIFT, (end - start) >> PAGE_SHIFT);
+
+		do {
+			err = ntfs_bdev_write(sb, empty_buf, start,
+						  vol->cluster_size);
+			if (err) {
+				ntfs_error(sb, "ntfs_dev_write failed, err : %d\n", err);
+				goto io_err;
+			}
+
+			/*
+			 * Submit the buffer and wait for i/o to complete but
+			 * only for the first buffer so we do not miss really
+			 * serious i/o errors.  Once the first buffer has
+			 * completed ignore errors afterwards as we can assume
+			 * that if one buffer worked all of them will work.
+			 */
+			if (should_wait) {
+				should_wait = false;
+				err = filemap_write_and_wait_range(sb->s_bdev->bd_mapping,
+						start, start + vol->cluster_size);
+				if (err)
+					goto io_err;
+			}
+			start += vol->cluster_size;
+		} while (start < end);
+	} while ((++rl)->vcn < end_vcn);
+	up_write(&log_ni->runlist.lock);
+	kfree(empty_buf);
+	kfree(ra);
+	truncate_inode_pages(log_vi->i_mapping, 0);
+	/* Set the flag so we do not have to do it again on remount. */
+	NVolSetLogFileEmpty(vol);
+	ntfs_debug("Done.");
+	return true;
+io_err:
+	ntfs_error(sb, "Failed to write buffer.  Unmount and run chkdsk.");
+	goto dirty_err;
+rl_err:
+	ntfs_error(sb, "Runlist is corrupt.  Unmount and run chkdsk.");
+dirty_err:
+	NVolSetErrors(vol);
+	err = -EIO;
+err:
+	kvfree(empty_buf);
+	kfree(ra);
+	up_write(&log_ni->runlist.lock);
+	ntfs_error(sb, "Failed to fill LogFile with 0xff bytes (error %d).",
+			-err);
+	return false;
+}
diff --git a/fs/ntfs/logfile.h b/fs/ntfs/logfile.h
new file mode 100644
index 0000000000000..dadb56fd0b18c
--- /dev/null
+++ b/fs/ntfs/logfile.h
@@ -0,0 +1,245 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS kernel journal (LogFile) handling.
+ *
+ * Copyright (c) 2000-2005 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_LOGFILE_H
+#define _LINUX_NTFS_LOGFILE_H
+
+#include "layout.h"
+
+/*
+ * Journal (LogFile) organization:
+ *
+ * Two restart areas present in the first two pages (restart pages, one restart
+ * area in each page).  When the volume is dismounted they should be identical,
+ * except for the update sequence array which usually has a different update
+ * sequence number.
+ *
+ * These are followed by log records organized in pages headed by a log record
+ * header going up to log file size.  Not all pages contain log records when a
+ * volume is first formatted, but as the volume ages, all records will be used.
+ * When the log file fills up, the records at the beginning are purged (by
+ * modifying the oldest_lsn to a higher value presumably) and writing begins
+ * at the beginning of the file.  Effectively, the log file is viewed as a
+ * circular entity.
+ *
+ * NOTE: Windows NT, 2000, and XP all use log file version 1.1 but they accept
+ * versions <= 1.x, including 0.-1.  (Yes, that is a minus one in there!)  We
+ * probably only want to support 1.1 as this seems to be the current version
+ * and we don't know how that differs from the older versions.  The only
+ * exception is if the journal is clean as marked by the two restart pages
+ * then it doesn't matter whether we are on an earlier version.  We can just
+ * reinitialize the logfile and start again with version 1.1.
+ */
+
+/* Some LogFile related constants. */
+#define MaxLogFileSize		0x100000000ULL
+#define DefaultLogPageSize	4096
+#define MinLogRecordPages	48
+
+/*
+ * Log file restart page header (begins the restart area).
+ *
+ * @magic: The magic is "RSTR".
+ * @usa_ofs: See ntfs_record struct definition in layout.h.  When creating,
+ *   set this to be immediately after this header structure (without any
+ *   alignment).
+ * @usa_count: See ntfs_record struct definition in layout.h.
+ * @chkdsk_lsn: The last log file sequence number found by chkdsk.  Only
+ *   used when the magic is changed to "CHKD".  Otherwise this is zero.
+ * @system_page_size: Byte size of system pages when the log file was
+ *   created, has to be >= 512 and a power of 2.  Use this to calculate
+ *   the required size of the usa (usa_count) and add it to usa_ofs. Then
+ *   verify that the result is less than the value of
+ *   the restart_area_offset.
+ * @log_page_size: Byte size of log file pages, has to be >= 512 and
+ *   a power of 2.  The default is 4096 and is used when the system page
+ *   size is between 4096 and 8192.  Otherwise this is set to the system
+ *   page size instead.
+ * @restart_area_offset: Byte offset from the start of this header to
+ *   the RESTART_AREA. Value has to be aligned to 8-byte boundary.  When
+ *   creating, set this to be after the usa.
+ * @minor_ver: Log file minor version.  Only check if major version is 1.
+ * @major_ver: Log file major version.  We only support version 1.1.
+ */
+struct restart_page_header {
+	__le32 magic;
+	__le16 usa_ofs;
+	__le16 usa_count;
+	__le64 chkdsk_lsn;
+	__le32 system_page_size;
+	__le32 log_page_size;
+	__le16 restart_area_offset;
+	__le16 minor_ver;
+	__le16 major_ver;
+} __packed;
+
+/*
+ * Constant for the log client indices meaning that there are no client records
+ * in this particular client array.  Also inside the client records themselves,
+ * this means that there are no client records preceding or following this one.
+ */
+#define LOGFILE_NO_CLIENT	cpu_to_le16(0xffff)
+#define LOGFILE_NO_CLIENT_CPU	0xffff
+
+/*
+ * These are the so far known RESTART_AREA_* flags (16-bit) which contain
+ * information about the log file in which they are present.
+ * gcc: Force enum bit width to 16.
+ */
+enum {
+	RESTART_VOLUME_IS_CLEAN	= cpu_to_le16(0x0002),
+	RESTART_SPACE_FILLER	= cpu_to_le16(0xffff),
+} __packed;
+
+/*
+ * Log file restart area record.  The offset of this record is found by adding
+ * the offset of the RESTART_PAGE_HEADER to the restart_area_offset value found
+ * in it.  See notes at restart_area_offset above.
+ *
+ * @current_lsn: The current, i.e. last LSN inside the log when
+ *   the restart area was last written.  This happens often but what is
+ *   the interval?  Is it just fixed time or is it every time a check point
+ *   is written or somethine else?  On create set to 0.
+ * @log_clients: Number of log client records in the array of log client
+ *   records which follows this restart area.  Must be 1.
+ * @client_free_list: The index of the first free log client record in
+ *   the array of log client records.  LOGFILE_NO_CLIENT means that there
+ *   are no free log client records in the array.  If != LOGFILE_NO_CLIENT,
+ *   check that log_clients > client_free_list.  On Win2k and presumably
+ *   earlier, on a clean volume this is != LOGFILE_NO_CLIENT, and it should
+ *   be 0, i.e. the first (and only) client record is free and thus
+ *   the logfile is closed and hence clean.  A dirty volume would have left
+ *   the logfile open and hence this would be LOGFILE_NO_CLIENT.  On WinXP
+ *   and presumably later, the logfile is always open, even on clean
+ *   shutdown so this should always be LOGFILE_NO_CLIENT.
+ * @client_in_use_list: The index of the first in-use log client record in
+ *   the array of log client records.  LOGFILE_NO_CLIENT means that there
+ *   are no in-use log client records in the array.
+ *   If != LOGFILE_NO_CLIENT check that log_clients > client_in_use_list.
+ *   On Win2k and presumably earlier, on a clean volume this is
+ *   LOGFILE_NO_CLIENT, i.e. there are no client records in use and thus
+ *   the logfile is closed and hence clean.  A dirty volume would have left
+ *   the logfile open and hence this would be != LOGFILE_NO_CLIENT, and it
+ *   should be 0, i.e. the first (and only) client record is in use.  On
+ *   WinXP and presumably later, the logfile is always open, even on clean
+ *   shutdown so this should always be 0.
+ * @flags: Flags modifying LFS behaviour.  On Win2k and presumably earlier
+ *   this is always 0.  On WinXP and presumably later, if the logfile was
+ *   shutdown cleanly, the second bit, RESTART_VOLUME_IS_CLEAN, is set.
+ *   This bit is cleared when the volume is mounted by WinXP and set when
+ *   the volume is dismounted, thus if the logfile is dirty, this bit is
+ *   clear.  Thus we don't need to check the Windows version to determine
+ *   if the logfile is clean.  Instead if the logfile is closed, we know
+ *   it must be clean.  If it is open and this bit is set, we also know
+ *   it must be clean.  If on the other hand the logfile is open and this
+ *   bit is clear, we can be almost certain that the logfile is dirty.
+ * @seq_number_bits: How many bits to use for the sequence number.  This
+ *   is calculated as 67 - the number of bits required to store the logfile
+ *   size in bytes and this can be used in with the specified file_size as
+ *   a consistency check.
+ * @restart_area_length: Length of the restart area including the client
+ *   array.  Following checks required if version matches.  Otherwise,
+ *   skip them.  restart_area_offset + restart_area_length has to be
+ *   <= system_page_size.  Also, restart_area_length has to be >=
+ *   client_array_offset + (log_clients * sizeof(log client record)).
+ * @client_array_offset: Offset from the start of this record to the first
+ *   log client record if versions are matched.  When creating, set this
+ *   to be after this restart area structure, aligned to 8-bytes boundary.
+ *   If the versions do not match, this is ignored and the offset is
+ *   assumed to be (sizeof(RESTART_AREA) + 7) &  ~7, i.e. rounded up to
+ *   first 8-byte boundary.  Either way, client_array_offset has to be
+ *   aligned to an 8-byte boundary.  Also, restart_area_offset +
+ *   client_array_offset has to be <= 510.  Finally, client_array_offset +
+ *   (log_clients * sizeof(log client record)) has to be <= system_page_size.
+ *   On Win2k and presumably earlier, this is 0x30, i.e. immediately
+ *   following this record.  On WinXP and presumably later, this is 0x40,
+ *   i.e. there are 16 extra bytes between this record and the client
+ *   array.  This probably means that the RESTART_AREA record is actually
+ *   bigger in WinXP and later.
+ * @file_size: Usable byte size of the log file.
+ *   If the restart_area_offset + the offset of the file_size are > 510
+ *   then corruption has occurred.  This is the very first check when
+ *   starting with the restart_area as if it fails it means that some of
+ *   the above values will be corrupted by the multi sector transfer
+ *   protection.  The file_size has to be rounded down to be a multiple
+ *   of the log_page_size in the RESTART_PAGE_HEADER and then it has to be
+ *   at least big enough to store the two restart pages and 48 (0x30) log
+ *   record pages.
+ * @last_lsn_data_length: Length of data of last LSN, not including the log
+ *   record header.  On create set to 0.
+ * @log_record_header_length: Byte size of the log record header.  If the
+ *   version matches then check that the value of log_record_header_length
+ *   is a multiple of 8, i.e. (log_record_header_length + 7) & ~7 ==
+ *   log_record_header_length.  When creating set it to
+ *   sizeof(LOG_RECORD_HEADER), aligned to 8 bytes.
+ * @log_page_data_offset: Offset to the start of data in a log record page.
+ *   Must be a multiple of 8.  On create set it to immediately after
+ *   the update sequence array of the log record page.
+ * @restart_log_open_count: A counter that gets incremented every time
+ *   the logfile is restarted which happens at mount time when the logfile
+ *   is opened. When creating set to a random value.  Win2k sets it to
+ *   the low 32 bits of the current system time in NTFS format (see time.h).
+ * @reserved: Reserved/alignment to 8-byte boundary.
+ */
+struct restart_area {
+	__le64 current_lsn;
+	__le16 log_clients;
+	__le16 client_free_list;
+	__le16 client_in_use_list;
+	__le16 flags;
+	__le32 seq_number_bits;
+	__le16 restart_area_length;
+	__le16 client_array_offset;
+	__le64 file_size;
+	__le32 last_lsn_data_length;
+	__le16 log_record_header_length;
+	__le16 log_page_data_offset;
+	__le32 restart_log_open_count;
+	__le32 reserved;
+} __packed;
+
+/*
+ * Log client record.  The offset of this record is found by adding the offset
+ * of the RESTART_AREA to the client_array_offset value found in it.
+ *
+ * @oldest_lsn: Oldest LSN needed by this client.  On create set to 0.
+ * @client_restart_lsn: LSN at which this client needs to restart
+ *   the volume, i.e. the current position within the log file.
+ *   At present, if clean this should = current_lsn in restart area but it
+ *   probably also = current_lsn when dirty most of the time.
+ *   At create set to 0.
+ * @prev_client: The offset to the previous log client record in the array
+ *   of log client records.  LOGFILE_NO_CLIENT means there is no previous
+ *   client record, i.e. this is the first one.  This is always
+ *   LOGFILE_NO_CLIENT.
+ * @next_client: The offset to the next log client record in the array of
+ *   log client records.  LOGFILE_NO_CLIENT means there are no next client
+ *   records, i.e. this is the last one.  This is always LOGFILE_NO_CLIENT.
+ * @seq_number: On Win2k and presumably earlier, this is set to zero every
+ *   time the logfile is restarted and it is incremented when the logfile
+ *   is closed at dismount time.  Thus it is 0 when dirty and 1 when clean.
+ *   On WinXP and presumably later, this is always 0.
+ * @reserved[6]: Reserved/alignment.
+ * @client_name_length: Length of client name in bytes.  Should always be 8.
+ * @client_name[64]: Name of the client in Unicode. Should always be "NTFS"
+ *   with the remaining bytes set to 0.
+ */
+struct log_client_record {
+	__le64 oldest_lsn;
+	__le64 client_restart_lsn;
+	__le16 prev_client;
+	__le16 next_client;
+	__le16 seq_number;
+	u8 reserved[6];
+	__le32 client_name_length;
+	__le16 client_name[64];
+} __packed;
+
+bool ntfs_check_logfile(struct inode *log_vi,
+		struct restart_page_header **rp);
+bool ntfs_empty_logfile(struct inode *log_vi);
+#endif /* _LINUX_NTFS_LOGFILE_H */
diff --git a/fs/ntfs/malloc.h b/fs/ntfs/malloc.h
new file mode 100644
index 0000000000000..7068425735f14
--- /dev/null
+++ b/fs/ntfs/malloc.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * malloc.h - NTFS kernel memory handling. Part of the Linux-NTFS project.
+ *
+ * Copyright (c) 2001-2005 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_MALLOC_H
+#define _LINUX_NTFS_MALLOC_H
+
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+
+/**
+ * __ntfs_malloc - allocate memory in multiples of pages
+ * @size:	number of bytes to allocate
+ * @gfp_mask:	extra flags for the allocator
+ *
+ * Internal function.  You probably want ntfs_malloc_nofs()...
+ *
+ * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and
+ * returns a pointer to the allocated memory.
+ *
+ * If there was insufficient memory to complete the request, return NULL.
+ * Depending on @gfp_mask the allocation may be guaranteed to succeed.
+ */
+static inline void *__ntfs_malloc(unsigned long size, gfp_t gfp_mask)
+{
+	if (likely(size <= PAGE_SIZE)) {
+		BUG_ON(!size);
+		/* kmalloc() has per-CPU caches so is faster for now. */
+		return kmalloc(PAGE_SIZE, gfp_mask & ~__GFP_HIGHMEM);
+		/* return (void *)__get_free_page(gfp_mask); */
+	}
+	if (likely((size >> PAGE_SHIFT) < totalram_pages()))
+		return __vmalloc(size, gfp_mask);
+	return NULL;
+}
+
+/**
+ * ntfs_malloc_nofs - allocate memory in multiples of pages
+ * @size:	number of bytes to allocate
+ *
+ * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and
+ * returns a pointer to the allocated memory.
+ *
+ * If there was insufficient memory to complete the request, return NULL.
+ */
+static inline void *ntfs_malloc_nofs(unsigned long size)
+{
+	return __ntfs_malloc(size, GFP_NOFS | __GFP_HIGHMEM);
+}
+
+/**
+ * ntfs_malloc_nofs_nofail - allocate memory in multiples of pages
+ * @size:	number of bytes to allocate
+ *
+ * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and
+ * returns a pointer to the allocated memory.
+ *
+ * This function guarantees that the allocation will succeed.  It will sleep
+ * for as long as it takes to complete the allocation.
+ *
+ * If there was insufficient memory to complete the request, return NULL.
+ */
+static inline void *ntfs_malloc_nofs_nofail(unsigned long size)
+{
+	return __ntfs_malloc(size, GFP_NOFS | __GFP_HIGHMEM | __GFP_NOFAIL);
+}
+
+static inline void ntfs_free(void *addr)
+{
+	kvfree(addr);
+}
+
+#endif /* _LINUX_NTFS_MALLOC_H */
diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c
new file mode 100644
index 0000000000000..bf028c1aea26c
--- /dev/null
+++ b/fs/ntfs/mft.c
@@ -0,0 +1,2924 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel mft record operations.
+ * Part of this file is based on code from the NTFS-3G.
+ *
+ * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/writeback.h>
+#include <linux/bio.h>
+#include <linux/iomap.h>
+
+#include "bitmap.h"
+#include "lcnalloc.h"
+#include "mft.h"
+#include "ntfs.h"
+
+/*
+ * ntfs_mft_record_check - Check the consistency of an MFT record
+ *
+ * Make sure its general fields are safe, then examine all its
+ * attributes and apply generic checks to them.
+ *
+ * Returns 0 if the checks are successful. If not, return -EIO.
+ */
+int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
+		u64 mft_no)
+{
+	struct attr_record *a;
+	struct super_block *sb = vol->sb;
+
+	if (!ntfs_is_file_record(m->magic)) {
+		ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n",
+				mft_no, le32_to_cpu(*(__le32 *)m));
+		goto err_out;
+	}
+
+	if (le16_to_cpu(m->usa_ofs) & 0x1 ||
+	    (vol->mft_record_size >> NTFS_BLOCK_SIZE_BITS) + 1 != le16_to_cpu(m->usa_count) ||
+	    le16_to_cpu(m->usa_ofs) + le16_to_cpu(m->usa_count) * 2 > vol->mft_record_size) {
+		ntfs_error(sb, "Record %llu has corrupt fix-up values fields\n",
+				mft_no);
+		goto err_out;
+	}
+
+	if (le32_to_cpu(m->bytes_allocated) != vol->mft_record_size) {
+		ntfs_error(sb, "Record %llu has corrupt allocation size (%u <> %u)\n",
+				mft_no, vol->mft_record_size,
+				le32_to_cpu(m->bytes_allocated));
+		goto err_out;
+	}
+
+	if (le32_to_cpu(m->bytes_in_use) > vol->mft_record_size) {
+		ntfs_error(sb, "Record %llu has corrupt in-use size (%u > %u)\n",
+				mft_no, le32_to_cpu(m->bytes_in_use),
+				vol->mft_record_size);
+		goto err_out;
+	}
+
+	if (le16_to_cpu(m->attrs_offset) & 7) {
+		ntfs_error(sb, "Attributes badly aligned in record %llu\n",
+				mft_no);
+		goto err_out;
+	}
+
+	a = (struct attr_record *)((char *)m + le16_to_cpu(m->attrs_offset));
+	if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) {
+		ntfs_error(sb, "Record %llu is corrupt\n", mft_no);
+		goto err_out;
+	}
+
+	return 0;
+
+err_out:
+	return -EIO;
+}
+
+/*
+ * map_mft_record_folio - map the folio in which a specific mft record resides
+ * @ni:		ntfs inode whose mft record page to map
+ *
+ * This maps the folio in which the mft record of the ntfs inode @ni is
+ * situated.
+ *
+ * This allocates a new buffer (@ni->mrec), copies the MFT record data from
+ * the mapped folio into this buffer, and applies the MST (Multi Sector
+ * Transfer) fixups on the copy.
+ *
+ * The folio is pinned (referenced) in @ni->folio to ensure the data remains
+ * valid in the page cache, but the returned pointer is the allocated copy.
+ *
+ * Return: A pointer to the allocated and fixed-up mft record (@ni->mrec).
+ * The return value needs to be checked with IS_ERR(). If it is true,
+ * PTR_ERR() contains the negative error code.
+ */
+static inline struct mft_record *map_mft_record_folio(struct ntfs_inode *ni)
+{
+	loff_t i_size;
+	struct ntfs_volume *vol = ni->vol;
+	struct inode *mft_vi = vol->mft_ino;
+	struct folio *folio;
+	unsigned long index, end_index;
+	unsigned int ofs;
+
+	WARN_ON(ni->folio);
+	/*
+	 * The index into the page cache and the offset within the page cache
+	 * page of the wanted mft record.
+	 */
+	index = NTFS_MFT_NR_TO_PIDX(vol, ni->mft_no);
+	ofs = NTFS_MFT_NR_TO_POFS(vol, ni->mft_no);
+
+	i_size = i_size_read(mft_vi);
+	/* The maximum valid index into the page cache for $MFT's data. */
+	end_index = i_size >> PAGE_SHIFT;
+
+	/* If the wanted index is out of bounds the mft record doesn't exist. */
+	if (unlikely(index >= end_index)) {
+		if (index > end_index || (i_size & ~PAGE_MASK) < ofs +
+				vol->mft_record_size) {
+			folio = ERR_PTR(-ENOENT);
+			ntfs_error(vol->sb,
+				"Attempt to read mft record 0x%llx, which is beyond the end of the mft. This is probably a bug in the ntfs driver.",
+				ni->mft_no);
+			goto err_out;
+		}
+	}
+
+	/* Read, map, and pin the folio. */
+	folio = read_mapping_folio(mft_vi->i_mapping, index, NULL);
+	if (!IS_ERR(folio)) {
+		u8 *addr;
+
+		ni->mrec = kmalloc(vol->mft_record_size, GFP_NOFS);
+		if (!ni->mrec) {
+			folio_put(folio);
+			folio = ERR_PTR(-ENOMEM);
+			goto err_out;
+		}
+
+		addr = kmap_local_folio(folio, 0);
+		memcpy(ni->mrec, addr + ofs, vol->mft_record_size);
+		post_read_mst_fixup((struct ntfs_record *)ni->mrec, vol->mft_record_size);
+
+		/* Catch multi sector transfer fixup errors. */
+		if (!ntfs_mft_record_check(vol, (struct mft_record *)ni->mrec, ni->mft_no)) {
+			kunmap_local(addr);
+			ni->folio = folio;
+			ni->folio_ofs = ofs;
+			return ni->mrec;
+		}
+		kunmap_local(addr);
+		folio_put(folio);
+		kfree(ni->mrec);
+		ni->mrec = NULL;
+		folio = ERR_PTR(-EIO);
+		NVolSetErrors(vol);
+	}
+err_out:
+	ni->folio = NULL;
+	ni->folio_ofs = 0;
+	return (struct mft_record *)folio;
+}
+
+/*
+ * map_mft_record - map and pin an mft record
+ * @ni:		ntfs inode whose MFT record to map
+ *
+ * This function ensures the MFT record for the given inode is mapped and
+ * accessible.
+ *
+ * It increments the reference count of the ntfs inode. If the record is
+ * already mapped (@ni->folio is set), it returns the cached record
+ * immediately.
+ *
+ * Otherwise, it calls map_mft_record_folio() to read the folio from disk
+ * (if necessary via read_mapping_folio), allocate a buffer, and copy the
+ * record data.
+ *
+ * Return: A pointer to the mft record. You need to check the returned
+ * pointer with IS_ERR().
+ */
+struct mft_record *map_mft_record(struct ntfs_inode *ni)
+{
+	struct mft_record *m;
+
+	if (!ni)
+		return ERR_PTR(-EINVAL);
+
+	ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no);
+
+	/* Make sure the ntfs inode doesn't go away. */
+	atomic_inc(&ni->count);
+
+	if (ni->folio)
+		return (struct mft_record *)ni->mrec;
+
+	m = map_mft_record_folio(ni);
+	if (!IS_ERR(m))
+		return m;
+
+	atomic_dec(&ni->count);
+	ntfs_error(ni->vol->sb, "Failed with error code %lu.", -PTR_ERR(m));
+	return m;
+}
+
+/*
+ * unmap_mft_record - release a reference to a mapped mft record
+ * @ni:		ntfs inode whose MFT record to unmap
+ *
+ * This decrements the reference count of the ntfs inode.
+ *
+ * It releases the caller's hold on the inode. If the reference count indicates
+ * that there are still other users (count > 1), the function returns
+ * immediately, keeping the resources (folio and mrec buffer) pinned for
+ * those users.
+ *
+ * NOTE: If caller has modified the mft record, it is imperative to set the mft
+ * record dirty BEFORE calling unmap_mft_record().
+ */
+void unmap_mft_record(struct ntfs_inode *ni)
+{
+	struct folio *folio;
+
+	if (!ni)
+		return;
+
+	ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no);
+
+	folio = ni->folio;
+	if (atomic_dec_return(&ni->count) > 1)
+		return;
+	WARN_ON(!folio);
+}
+
+/*
+ * map_extent_mft_record - load an extent inode and attach it to its base
+ * @base_ni:	base ntfs inode
+ * @mref:	mft reference of the extent inode to load
+ * @ntfs_ino:	on successful return, pointer to the struct ntfs_inode structure
+ *
+ * Load the extent mft record @mref and attach it to its base inode @base_ni.
+ * Return the mapped extent mft record if IS_ERR(result) is false.  Otherwise
+ * PTR_ERR(result) gives the negative error code.
+ *
+ * On successful return, @ntfs_ino contains a pointer to the ntfs_inode
+ * structure of the mapped extent inode.
+ */
+struct mft_record *map_extent_mft_record(struct ntfs_inode *base_ni, u64 mref,
+		struct ntfs_inode **ntfs_ino)
+{
+	struct mft_record *m;
+	struct ntfs_inode *ni = NULL;
+	struct ntfs_inode **extent_nis = NULL;
+	int i;
+	u64 mft_no = MREF(mref);
+	u16 seq_no = MSEQNO(mref);
+	bool destroy_ni = false;
+
+	ntfs_debug("Mapping extent mft record 0x%llx (base mft record 0x%llx).",
+			mft_no, base_ni->mft_no);
+	/* Make sure the base ntfs inode doesn't go away. */
+	atomic_inc(&base_ni->count);
+	/*
+	 * Check if this extent inode has already been added to the base inode,
+	 * in which case just return it. If not found, add it to the base
+	 * inode before returning it.
+	 */
+retry:
+	mutex_lock(&base_ni->extent_lock);
+	if (base_ni->nr_extents > 0) {
+		extent_nis = base_ni->ext.extent_ntfs_inos;
+		for (i = 0; i < base_ni->nr_extents; i++) {
+			if (mft_no != extent_nis[i]->mft_no)
+				continue;
+			ni = extent_nis[i];
+			/* Make sure the ntfs inode doesn't go away. */
+			atomic_inc(&ni->count);
+			break;
+		}
+	}
+	if (likely(ni != NULL)) {
+		mutex_unlock(&base_ni->extent_lock);
+		atomic_dec(&base_ni->count);
+		/* We found the record; just have to map and return it. */
+		m = map_mft_record(ni);
+		/* map_mft_record() has incremented this on success. */
+		atomic_dec(&ni->count);
+		if (!IS_ERR(m)) {
+			/* Verify the sequence number. */
+			if (likely(le16_to_cpu(m->sequence_number) == seq_no)) {
+				ntfs_debug("Done 1.");
+				*ntfs_ino = ni;
+				return m;
+			}
+			unmap_mft_record(ni);
+			ntfs_error(base_ni->vol->sb,
+					"Found stale extent mft reference! Corrupt filesystem. Run chkdsk.");
+			return ERR_PTR(-EIO);
+		}
+map_err_out:
+		ntfs_error(base_ni->vol->sb,
+				"Failed to map extent mft record, error code %ld.",
+				-PTR_ERR(m));
+		return m;
+	}
+	mutex_unlock(&base_ni->extent_lock);
+
+	/* Record wasn't there. Get a new ntfs inode and initialize it. */
+	ni = ntfs_new_extent_inode(base_ni->vol->sb, mft_no);
+	if (unlikely(!ni)) {
+		atomic_dec(&base_ni->count);
+		return ERR_PTR(-ENOMEM);
+	}
+	ni->vol = base_ni->vol;
+	ni->seq_no = seq_no;
+	ni->nr_extents = -1;
+	ni->ext.base_ntfs_ino = base_ni;
+	/* Now map the record. */
+	m = map_mft_record(ni);
+	if (IS_ERR(m)) {
+		atomic_dec(&base_ni->count);
+		ntfs_clear_extent_inode(ni);
+		goto map_err_out;
+	}
+	/* Verify the sequence number if it is present. */
+	if (seq_no && (le16_to_cpu(m->sequence_number) != seq_no)) {
+		ntfs_error(base_ni->vol->sb,
+				"Found stale extent mft reference! Corrupt filesystem. Run chkdsk.");
+		destroy_ni = true;
+		m = ERR_PTR(-EIO);
+		goto unm_nolock_err_out;
+	}
+
+	mutex_lock(&base_ni->extent_lock);
+	for (i = 0; i < base_ni->nr_extents; i++) {
+		if (mft_no == extent_nis[i]->mft_no) {
+			mutex_unlock(&base_ni->extent_lock);
+			ntfs_clear_extent_inode(ni);
+			goto retry;
+		}
+	}
+	/* Attach extent inode to base inode, reallocating memory if needed. */
+	if (!(base_ni->nr_extents & 3)) {
+		struct ntfs_inode **tmp;
+		int new_size = (base_ni->nr_extents + 4) * sizeof(struct ntfs_inode *);
+
+		tmp = kvzalloc(new_size, GFP_NOFS);
+		if (unlikely(!tmp)) {
+			ntfs_error(base_ni->vol->sb, "Failed to allocate internal buffer.");
+			destroy_ni = true;
+			m = ERR_PTR(-ENOMEM);
+			goto unm_err_out;
+		}
+		if (base_ni->nr_extents) {
+			WARN_ON(!base_ni->ext.extent_ntfs_inos);
+			memcpy(tmp, base_ni->ext.extent_ntfs_inos, new_size -
+					4 * sizeof(struct ntfs_inode *));
+			kvfree(base_ni->ext.extent_ntfs_inos);
+		}
+		base_ni->ext.extent_ntfs_inos = tmp;
+	}
+	base_ni->ext.extent_ntfs_inos[base_ni->nr_extents++] = ni;
+	mutex_unlock(&base_ni->extent_lock);
+	atomic_dec(&base_ni->count);
+	ntfs_debug("Done 2.");
+	*ntfs_ino = ni;
+	return m;
+unm_err_out:
+	mutex_unlock(&base_ni->extent_lock);
+unm_nolock_err_out:
+	unmap_mft_record(ni);
+	atomic_dec(&base_ni->count);
+	/*
+	 * If the extent inode was not attached to the base inode we need to
+	 * release it or we will leak memory.
+	 */
+	if (destroy_ni)
+		ntfs_clear_extent_inode(ni);
+	return m;
+}
+
+/*
+ * __mark_mft_record_dirty - mark the base vfs inode dirty
+ * @ni:		ntfs inode describing the mapped mft record
+ *
+ * Internal function.  Users should call mark_mft_record_dirty() instead.
+ *
+ * This function determines the base ntfs inode (in case @ni is an extent
+ * inode) and marks the corresponding VFS inode dirty.
+ *
+ * NOTE:  We only set I_DIRTY_DATASYNC (and not I_DIRTY_PAGES)
+ * on the base vfs inode, because even though file data may have been modified,
+ * it is dirty in the inode meta data rather than the data page cache of the
+ * inode, and thus there are no data pages that need writing out.  Therefore, a
+ * full mark_inode_dirty() is overkill.  A mark_inode_dirty_sync(), on the
+ * other hand, is not sufficient, because ->write_inode needs to be called even
+ * in case of fdatasync. This needs to happen or the file data would not
+ * necessarily hit the device synchronously, even though the vfs inode has the
+ * O_SYNC flag set.  Also, I_DIRTY_DATASYNC simply "feels" better than just
+ * I_DIRTY_SYNC, since the file data has not actually hit the block device yet,
+ * which is not what I_DIRTY_SYNC on its own would suggest.
+ */
+void __mark_mft_record_dirty(struct ntfs_inode *ni)
+{
+	struct ntfs_inode *base_ni;
+
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+	WARN_ON(NInoAttr(ni));
+	/* Determine the base vfs inode and mark it dirty, too. */
+	if (likely(ni->nr_extents >= 0))
+		base_ni = ni;
+	else
+		base_ni = ni->ext.base_ntfs_ino;
+	__mark_inode_dirty(VFS_I(base_ni), I_DIRTY_DATASYNC);
+}
+
+/*
+ * ntfs_bio_end_io - bio completion callback for MFT record writes
+ *
+ * Decrements the folio reference count that was incremented before
+ * submit_bio(). This prevents a race condition where umount could
+ * evict the inode and release the folio while I/O is still in flight,
+ * potentially causing data corruption or use-after-free.
+ */
+static void ntfs_bio_end_io(struct bio *bio)
+{
+	if (bio->bi_private)
+		folio_put((struct folio *)bio->bi_private);
+	bio_put(bio);
+}
+
+/*
+ * ntfs_sync_mft_mirror - synchronize an mft record to the mft mirror
+ * @vol:	ntfs volume on which the mft record to synchronize resides
+ * @mft_no:	mft record number of mft record to synchronize
+ * @m:		mapped, mst protected (extent) mft record to synchronize
+ *
+ * Write the mapped, mst protected (extent) mft record @m with mft record
+ * number @mft_no to the mft mirror ($MFTMirr) of the ntfs volume @vol.
+ *
+ * On success return 0.  On error return -errno and set the volume errors flag
+ * in the ntfs volume @vol.
+ *
+ * NOTE:  We always perform synchronous i/o.
+ */
+int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
+		struct mft_record *m)
+{
+	u8 *kmirr = NULL;
+	struct folio *folio;
+	unsigned int folio_ofs, lcn_folio_off = 0;
+	int err = 0;
+	struct bio *bio;
+
+	ntfs_debug("Entering for inode 0x%llx.", mft_no);
+
+	if (unlikely(!vol->mftmirr_ino)) {
+		/* This could happen during umount... */
+		err = -EIO;
+		goto err_out;
+	}
+	/* Get the page containing the mirror copy of the mft record @m. */
+	folio = read_mapping_folio(vol->mftmirr_ino->i_mapping,
+			NTFS_MFT_NR_TO_PIDX(vol, mft_no), NULL);
+	if (IS_ERR(folio)) {
+		ntfs_error(vol->sb, "Failed to map mft mirror page.");
+		err = PTR_ERR(folio);
+		goto err_out;
+	}
+
+	folio_lock(folio);
+	folio_clear_uptodate(folio);
+	/* Offset of the mft mirror record inside the page. */
+	folio_ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no);
+	/* The address in the page of the mirror copy of the mft record @m. */
+	kmirr = kmap_local_folio(folio, 0) + folio_ofs;
+	/* Copy the mst protected mft record to the mirror. */
+	memcpy(kmirr, m, vol->mft_record_size);
+
+	if (vol->cluster_size_bits > PAGE_SHIFT) {
+		lcn_folio_off = folio->index << PAGE_SHIFT;
+		lcn_folio_off &= vol->cluster_size_mask;
+	}
+
+	bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOIO);
+	bio->bi_iter.bi_sector =
+		NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) +
+				 lcn_folio_off + folio_ofs);
+
+	if (!bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) {
+		err = -EIO;
+		bio_put(bio);
+		goto unlock_folio;
+	}
+
+	bio->bi_end_io = ntfs_bio_end_io;
+	submit_bio(bio);
+	/* Current state: all buffers are clean, unlocked, and uptodate. */
+	folio_mark_uptodate(folio);
+
+unlock_folio:
+	folio_unlock(folio);
+	kunmap_local(kmirr);
+	folio_put(folio);
+	if (likely(!err)) {
+		ntfs_debug("Done.");
+	} else {
+		ntfs_error(vol->sb, "I/O error while writing mft mirror record 0x%llx!", mft_no);
+err_out:
+		ntfs_error(vol->sb,
+			"Failed to synchronize $MFTMirr (error code %i).  Volume will be left marked dirty on umount.  Run chkdsk on the partition after umounting to correct this.",
+			err);
+		NVolSetErrors(vol);
+	}
+	return err;
+}
+
+/*
+ * write_mft_record_nolock - write out a mapped (extent) mft record
+ * @ni:		ntfs inode describing the mapped (extent) mft record
+ * @m:		mapped (extent) mft record to write
+ * @sync:	if true, wait for i/o completion
+ *
+ * Write the mapped (extent) mft record @m described by the (regular or extent)
+ * ntfs inode @ni to backing store.  If the mft record @m has a counterpart in
+ * the mft mirror, that is also updated.
+ *
+ * We only write the mft record if the ntfs inode @ni is dirty.
+ *
+ * On success, clean the mft record and return 0.
+ * On error (specifically ENOMEM), we redirty the record so it can be retried.
+ * For other errors, we mark the volume with errors.
+ */
+int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int sync)
+{
+	struct ntfs_volume *vol = ni->vol;
+	struct folio *folio = ni->folio;
+	int err = 0, i = 0;
+	u8 *kaddr;
+	struct mft_record *fixup_m;
+	struct bio *bio;
+	unsigned int offset = 0, folio_size;
+
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+
+	WARN_ON(NInoAttr(ni));
+	WARN_ON(!folio_test_locked(folio));
+
+	/*
+	 * If the struct ntfs_inode is clean no need to do anything.  If it is dirty,
+	 * mark it as clean now so that it can be redirtied later on if needed.
+	 * There is no danger of races since the caller is holding the locks
+	 * for the mft record @m and the page it is in.
+	 */
+	if (!NInoTestClearDirty(ni))
+		goto done;
+
+	kaddr = kmap_local_folio(folio, 0);
+	fixup_m = (struct mft_record *)(kaddr + ni->folio_ofs);
+	memcpy(fixup_m, m, vol->mft_record_size);
+
+	/* Apply the mst protection fixups. */
+	err = pre_write_mst_fixup((struct ntfs_record *)fixup_m, vol->mft_record_size);
+	if (err) {
+		ntfs_error(vol->sb, "Failed to apply mst fixups!");
+		goto err_out;
+	}
+
+	folio_size = vol->mft_record_size / ni->mft_lcn_count;
+	while (i < ni->mft_lcn_count) {
+		unsigned int clu_off;
+
+		clu_off = (unsigned int)((s64)ni->mft_no * vol->mft_record_size + offset) &
+			vol->cluster_size_mask;
+
+		bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOIO);
+		bio->bi_iter.bi_sector =
+			NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, ni->mft_lcn[i]) +
+					 clu_off);
+
+		if (!bio_add_folio(bio, folio, folio_size,
+				   ni->folio_ofs + offset)) {
+			err = -EIO;
+			goto put_bio_out;
+		}
+
+		/* Synchronize the mft mirror now if not @sync. */
+		if (!sync && ni->mft_no < vol->mftmirr_size)
+			ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+
+		folio_get(folio);
+		bio->bi_private = folio;
+		bio->bi_end_io = ntfs_bio_end_io;
+		submit_bio(bio);
+		offset += vol->cluster_size;
+		i++;
+	}
+
+	/* If @sync, now synchronize the mft mirror. */
+	if (sync && ni->mft_no < vol->mftmirr_size)
+		ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+	kunmap_local(kaddr);
+	if (unlikely(err)) {
+		/* I/O error during writing.  This is really bad! */
+		ntfs_error(vol->sb,
+			"I/O error while writing mft record 0x%llx!  Marking base inode as bad.  You should unmount the volume and run chkdsk.",
+			ni->mft_no);
+		goto err_out;
+	}
+done:
+	ntfs_debug("Done.");
+	return 0;
+put_bio_out:
+	bio_put(bio);
+err_out:
+	/*
+	 * Current state: all buffers are clean, unlocked, and uptodate.
+	 * The caller should mark the base inode as bad so that no more i/o
+	 * happens.  ->drop_inode() will still be invoked so all extent inodes
+	 * and other allocated memory will be freed.
+	 */
+	if (err == -ENOMEM) {
+		ntfs_error(vol->sb,
+			"Not enough memory to write mft record. Redirtying so the write is retried later.");
+		mark_mft_record_dirty(ni);
+		err = 0;
+	} else
+		NVolSetErrors(vol);
+	return err;
+}
+
+static int ntfs_test_inode_wb(struct inode *vi, u64 ino, void *data)
+{
+	struct ntfs_attr *na = data;
+
+	if (!ntfs_test_inode(vi, na))
+		return 0;
+
+	/*
+	 * Without this, ntfs_write_mst_block() could call iput_final()
+	 * , and ntfs_evict_big_inode() could try to unlink this inode
+	 * and the contex could be blocked infinitly in map_mft_record().
+	 */
+	if (NInoBeingDeleted(NTFS_I(vi))) {
+		na->state = NI_BeingDeleted;
+		return -1;
+	}
+
+	/*
+	 * This condition can prevent ntfs_write_mst_block()
+	 * from applying/undo fixups while ntfs_create() being
+	 * called
+	 */
+	spin_lock(&vi->i_lock);
+	if (inode_state_read_once(vi) & I_CREATING) {
+		spin_unlock(&vi->i_lock);
+		na->state = NI_BeingCreated;
+		return -1;
+	}
+	spin_unlock(&vi->i_lock);
+
+	return igrab(vi) ? 1 : -1;
+}
+
+/*
+ * ntfs_may_write_mft_record - check if an mft record may be written out
+ * @vol:	[IN]  ntfs volume on which the mft record to check resides
+ * @mft_no:	[IN]  mft record number of the mft record to check
+ * @m:		[IN]  mapped mft record to check
+ * @locked_ni:	[OUT] caller has to unlock this ntfs inode if one is returned
+ * @ref_vi:	[OUT] caller has to drop this vfs inode if one is returned
+ *
+ * Check if the mapped (base or extent) mft record @m with mft record number
+ * @mft_no belonging to the ntfs volume @vol may be written out.  If necessary
+ * and possible the ntfs inode of the mft record is locked and the base vfs
+ * inode is pinned.  The locked ntfs inode is then returned in @locked_ni.  The
+ * caller is responsible for unlocking the ntfs inode and unpinning the base
+ * vfs inode.
+ *
+ * To avoid deadlock when the caller holds a folio lock, if the function
+ * returns @ref_vi it defers dropping the vfs inode reference by returning
+ * it in @ref_vi instead of calling iput() directly.  The caller must call
+ * iput() on @ref_vi after releasing the folio lock.
+ *
+ * Return 'true' if the mft record may be written out and 'false' if not.
+ *
+ * The caller has locked the page and cleared the uptodate flag on it which
+ * means that we can safely write out any dirty mft records that do not have
+ * their inodes in icache as determined by find_inode_nowait().
+ *
+ * Here is a description of the tests we perform:
+ *
+ * If the inode is found in icache we know the mft record must be a base mft
+ * record.  If it is dirty, we do not write it and return 'false' as the vfs
+ * inode write paths will result in the access times being updated which would
+ * cause the base mft record to be redirtied and written out again.
+ *
+ * If the inode is in icache and not dirty, we attempt to lock the mft record
+ * and if we find the lock was already taken, it is not safe to write the mft
+ * record and we return 'false'.
+ *
+ * If we manage to obtain the lock we have exclusive access to the mft record,
+ * which also allows us safe writeout of the mft record.  We then set
+ * @locked_ni to the locked ntfs inode and return 'true'.
+ *
+ * Note we cannot just lock the mft record and sleep while waiting for the lock
+ * because this would deadlock due to lock reversal.
+ *
+ * If the inode is not in icache we need to perform further checks.
+ *
+ * If the mft record is not a FILE record or it is a base mft record, we can
+ * safely write it and return 'true'.
+ *
+ * We now know the mft record is an extent mft record.  We check if the inode
+ * corresponding to its base mft record is in icache. If it is not, we cannot
+ * safely determine the state of the extent inode, so we return 'false'.
+ *
+ * We now have the base inode for the extent mft record.  We check if it has an
+ * ntfs inode for the extent mft record attached. If not, it is safe to write
+ * the extent mft record and we return 'true'.
+ *
+ * If the extent inode is attached, we check if it is dirty. If so, we return
+ * 'false' (letting the standard write_inode path handle it).
+ *
+ * If it is not dirty, we attempt to lock the extent mft record. If the lock
+ * was already taken, it is not safe to write and we return 'false'.
+ *
+ * If we manage to obtain the lock we have exclusive access to the extent mft
+ * record. We set @locked_ni to the now locked ntfs inode and return 'true'.
+ */
+static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no,
+		const struct mft_record *m, struct ntfs_inode **locked_ni,
+		struct inode **ref_vi)
+{
+	struct super_block *sb = vol->sb;
+	struct inode *mft_vi = vol->mft_ino;
+	struct inode *vi;
+	struct ntfs_inode *ni, *eni, **extent_nis;
+	int i;
+	struct ntfs_attr na = {0};
+
+	ntfs_debug("Entering for inode 0x%llx.", mft_no);
+	/*
+	 * Normally we do not return a locked inode so set @locked_ni to NULL.
+	 */
+	*locked_ni = NULL;
+	*ref_vi = NULL;
+
+	/*
+	 * Check if the inode corresponding to this mft record is in the VFS
+	 * inode cache and obtain a reference to it if it is.
+	 */
+	ntfs_debug("Looking for inode 0x%llx in icache.", mft_no);
+	na.mft_no = mft_no;
+	na.type = AT_UNUSED;
+	/*
+	 * Optimize inode 0, i.e. $MFT itself, since we have it in memory and
+	 * we get here for it rather often.
+	 */
+	if (!mft_no) {
+		/* Balance the below iput(). */
+		vi = igrab(mft_vi);
+		WARN_ON(vi != mft_vi);
+	} else {
+		/*
+		 * Have to use find_inode_nowait() since ilookup5_nowait()
+		 * waits for inode with I_FREEING, which causes ntfs to deadlock
+		 * when inodes are unlinked concurrently
+		 */
+		vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na);
+		if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated)
+			return false;
+	}
+	if (vi) {
+		ntfs_debug("Base inode 0x%llx is in icache.", mft_no);
+		/* The inode is in icache. */
+		ni = NTFS_I(vi);
+		/* Take a reference to the ntfs inode. */
+		atomic_inc(&ni->count);
+		/* If the inode is dirty, do not write this record. */
+		if (NInoDirty(ni)) {
+			ntfs_debug("Inode 0x%llx is dirty, do not write it.",
+					mft_no);
+			atomic_dec(&ni->count);
+			*ref_vi = vi;
+			return false;
+		}
+		ntfs_debug("Inode 0x%llx is not dirty.", mft_no);
+		/* The inode is not dirty, try to take the mft record lock. */
+		if (unlikely(!mutex_trylock(&ni->mrec_lock))) {
+			ntfs_debug("Mft record 0x%llx is already locked, do not write it.", mft_no);
+			atomic_dec(&ni->count);
+			*ref_vi = vi;
+			return false;
+		}
+		ntfs_debug("Managed to lock mft record 0x%llx, write it.",
+				mft_no);
+		/*
+		 * The write has to occur while we hold the mft record lock so
+		 * return the locked ntfs inode.
+		 */
+		*locked_ni = ni;
+		return true;
+	}
+	ntfs_debug("Inode 0x%llx is not in icache.", mft_no);
+	/* The inode is not in icache. */
+	/* Write the record if it is not a mft record (type "FILE"). */
+	if (!ntfs_is_mft_record(m->magic)) {
+		ntfs_debug("Mft record 0x%llx is not a FILE record, write it.",
+				mft_no);
+		return true;
+	}
+	/* Write the mft record if it is a base inode. */
+	if (!m->base_mft_record) {
+		ntfs_debug("Mft record 0x%llx is a base record, write it.",
+				mft_no);
+		return true;
+	}
+	/*
+	 * This is an extent mft record.  Check if the inode corresponding to
+	 * its base mft record is in icache and obtain a reference to it if it
+	 * is.
+	 */
+	na.mft_no = MREF_LE(m->base_mft_record);
+	na.state = 0;
+	ntfs_debug("Mft record 0x%llx is an extent record.  Looking for base inode 0x%llx in icache.",
+			mft_no, na.mft_no);
+	if (!na.mft_no) {
+		/* Balance the below iput(). */
+		vi = igrab(mft_vi);
+		WARN_ON(vi != mft_vi);
+	} else {
+		vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na);
+		if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated)
+			return false;
+	}
+
+	if (!vi)
+		return false;
+	ntfs_debug("Base inode 0x%llx is in icache.", na.mft_no);
+	/*
+	 * The base inode is in icache.  Check if it has the extent inode
+	 * corresponding to this extent mft record attached.
+	 */
+	ni = NTFS_I(vi);
+	mutex_lock(&ni->extent_lock);
+	if (ni->nr_extents <= 0) {
+		/*
+		 * The base inode has no attached extent inodes, write this
+		 * extent mft record.
+		 */
+		mutex_unlock(&ni->extent_lock);
+		*ref_vi = vi;
+		ntfs_debug("Base inode 0x%llx has no attached extent inodes, write the extent record.",
+				na.mft_no);
+		return true;
+	}
+	/* Iterate over the attached extent inodes. */
+	extent_nis = ni->ext.extent_ntfs_inos;
+	for (eni = NULL, i = 0; i < ni->nr_extents; ++i) {
+		if (mft_no == extent_nis[i]->mft_no) {
+			/*
+			 * Found the extent inode corresponding to this extent
+			 * mft record.
+			 */
+			eni = extent_nis[i];
+			break;
+		}
+	}
+	/*
+	 * If the extent inode was not attached to the base inode, write this
+	 * extent mft record.
+	 */
+	if (!eni) {
+		mutex_unlock(&ni->extent_lock);
+		*ref_vi = vi;
+		ntfs_debug("Extent inode 0x%llx is not attached to its base inode 0x%llx, write the extent record.",
+				mft_no, na.mft_no);
+		return true;
+	}
+	ntfs_debug("Extent inode 0x%llx is attached to its base inode 0x%llx.",
+			mft_no, na.mft_no);
+	/* Take a reference to the extent ntfs inode. */
+	atomic_inc(&eni->count);
+	mutex_unlock(&ni->extent_lock);
+
+	/* if extent inode is dirty, write_inode will write it */
+	if (NInoDirty(eni)) {
+		atomic_dec(&eni->count);
+		*ref_vi = vi;
+		return false;
+	}
+
+	/*
+	 * Found the extent inode coresponding to this extent mft record.
+	 * Try to take the mft record lock.
+	 */
+	if (unlikely(!mutex_trylock(&eni->mrec_lock))) {
+		atomic_dec(&eni->count);
+		*ref_vi = vi;
+		ntfs_debug("Extent mft record 0x%llx is already locked, do not write it.",
+				mft_no);
+		return false;
+	}
+	ntfs_debug("Managed to lock extent mft record 0x%llx, write it.",
+			mft_no);
+	/*
+	 * The write has to occur while we hold the mft record lock so return
+	 * the locked extent ntfs inode.
+	 */
+	*locked_ni = eni;
+	return true;
+}
+
+static const char *es = "  Leaving inconsistent metadata.  Unmount and run chkdsk.";
+
+#define RESERVED_MFT_RECORDS	64
+
+/*
+ * ntfs_mft_bitmap_find_and_alloc_free_rec_nolock - see name
+ * @vol:	volume on which to search for a free mft record
+ * @base_ni:	open base inode if allocating an extent mft record or NULL
+ *
+ * Search for a free mft record in the mft bitmap attribute on the ntfs volume
+ * @vol.
+ *
+ * If @base_ni is NULL start the search at the default allocator position.
+ *
+ * If @base_ni is not NULL start the search at the mft record after the base
+ * mft record @base_ni.
+ *
+ * Return the free mft record on success and -errno on error.  An error code of
+ * -ENOSPC means that there are no free mft records in the currently
+ * initialized mft bitmap.
+ *
+ * Locking: Caller must hold vol->mftbmp_lock for writing.
+ */
+static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vol,
+		struct ntfs_inode *base_ni)
+{
+	s64 pass_end, ll, data_pos, pass_start, ofs, bit;
+	unsigned long flags;
+	struct address_space *mftbmp_mapping;
+	u8 *buf = NULL, *byte;
+	struct folio *folio;
+	unsigned int folio_ofs, size;
+	u8 pass, b;
+
+	ntfs_debug("Searching for free mft record in the currently initialized mft bitmap.");
+	mftbmp_mapping = vol->mftbmp_ino->i_mapping;
+	/*
+	 * Set the end of the pass making sure we do not overflow the mft
+	 * bitmap.
+	 */
+	read_lock_irqsave(&NTFS_I(vol->mft_ino)->size_lock, flags);
+	pass_end = NTFS_I(vol->mft_ino)->allocated_size >>
+			vol->mft_record_size_bits;
+	read_unlock_irqrestore(&NTFS_I(vol->mft_ino)->size_lock, flags);
+	read_lock_irqsave(&NTFS_I(vol->mftbmp_ino)->size_lock, flags);
+	ll = NTFS_I(vol->mftbmp_ino)->initialized_size << 3;
+	read_unlock_irqrestore(&NTFS_I(vol->mftbmp_ino)->size_lock, flags);
+	if (pass_end > ll)
+		pass_end = ll;
+	pass = 1;
+	if (!base_ni)
+		data_pos = vol->mft_data_pos;
+	else
+		data_pos = base_ni->mft_no + 1;
+	if (data_pos < RESERVED_MFT_RECORDS)
+		data_pos = RESERVED_MFT_RECORDS;
+	if (data_pos >= pass_end) {
+		data_pos = RESERVED_MFT_RECORDS;
+		pass = 2;
+		/* This happens on a freshly formatted volume. */
+		if (data_pos >= pass_end)
+			return -ENOSPC;
+	}
+
+	if (base_ni && base_ni->mft_no == FILE_MFT) {
+		data_pos = 0;
+		pass = 2;
+	}
+
+	pass_start = data_pos;
+	ntfs_debug("Starting bitmap search: pass %u, pass_start 0x%llx, pass_end 0x%llx, data_pos 0x%llx.",
+			pass, pass_start, pass_end, data_pos);
+	/* Loop until a free mft record is found. */
+	for (; pass <= 2;) {
+		/* Cap size to pass_end. */
+		ofs = data_pos >> 3;
+		folio_ofs = ofs & ~PAGE_MASK;
+		size = PAGE_SIZE - folio_ofs;
+		ll = ((pass_end + 7) >> 3) - ofs;
+		if (size > ll)
+			size = ll;
+		size <<= 3;
+		/*
+		 * If we are still within the active pass, search the next page
+		 * for a zero bit.
+		 */
+		if (size) {
+			folio = read_mapping_folio(mftbmp_mapping,
+					ofs >> PAGE_SHIFT, NULL);
+			if (IS_ERR(folio)) {
+				ntfs_error(vol->sb, "Failed to read mft bitmap, aborting.");
+				return PTR_ERR(folio);
+			}
+			folio_lock(folio);
+			buf = (u8 *)kmap_local_folio(folio, 0) + folio_ofs;
+			bit = data_pos & 7;
+			data_pos &= ~7ull;
+			ntfs_debug("Before inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx",
+					size, data_pos, bit);
+			for (; bit < size && data_pos + bit < pass_end;
+					bit &= ~7ull, bit += 8) {
+				/*
+				 * If we're extending $MFT and running out of the first
+				 * mft record (base record) then give up searching since
+				 * no guarantee that the found record will be accessible.
+				 */
+				if (base_ni && base_ni->mft_no == FILE_MFT && bit > 400) {
+					folio_unlock(folio);
+					kunmap_local(buf);
+					folio_put(folio);
+					return -ENOSPC;
+				}
+
+				byte = buf + (bit >> 3);
+				if (*byte == 0xff)
+					continue;
+				b = ffz((unsigned long)*byte);
+				if (b < 8 && b >= (bit & 7)) {
+					ll = data_pos + (bit & ~7ull) + b;
+					if (unlikely(ll > (1ll << 32))) {
+						folio_unlock(folio);
+						kunmap_local(buf);
+						folio_put(folio);
+						return -ENOSPC;
+					}
+					*byte |= 1 << b;
+					folio_mark_dirty(folio);
+					folio_unlock(folio);
+					kunmap_local(buf);
+					folio_put(folio);
+					ntfs_debug("Done.  (Found and allocated mft record 0x%llx.)",
+							ll);
+					return ll;
+				}
+			}
+			ntfs_debug("After inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx",
+					size, data_pos, bit);
+			data_pos += size;
+			folio_unlock(folio);
+			kunmap_local(buf);
+			folio_put(folio);
+			/*
+			 * If the end of the pass has not been reached yet,
+			 * continue searching the mft bitmap for a zero bit.
+			 */
+			if (data_pos < pass_end)
+				continue;
+		}
+		/* Do the next pass. */
+		if (++pass == 2) {
+			/*
+			 * Starting the second pass, in which we scan the first
+			 * part of the zone which we omitted earlier.
+			 */
+			pass_end = pass_start;
+			data_pos = pass_start = RESERVED_MFT_RECORDS;
+			ntfs_debug("pass %i, pass_start 0x%llx, pass_end 0x%llx.",
+					pass, pass_start, pass_end);
+			if (data_pos >= pass_end)
+				break;
+		}
+	}
+	/* No free mft records in currently initialized mft bitmap. */
+	ntfs_debug("Done.  (No free mft records left in currently initialized mft bitmap.)");
+	return -ENOSPC;
+}
+
+static int ntfs_mft_attr_extend(struct ntfs_inode *ni)
+{
+	int ret = 0;
+	struct ntfs_inode *base_ni;
+
+	if (NInoAttr(ni))
+		base_ni = ni->ext.base_ntfs_ino;
+	else
+		base_ni = ni;
+
+	if (!NInoAttrList(base_ni)) {
+		ret = ntfs_inode_add_attrlist(base_ni);
+		if (ret) {
+			pr_err("Can not add attrlist\n");
+			goto out;
+		} else {
+			ret = -EAGAIN;
+			goto out;
+		}
+	}
+
+	ret = ntfs_attr_update_mapping_pairs(ni, 0);
+	if (ret)
+		pr_err("MP update failed\n");
+
+out:
+	return ret;
+}
+
+/*
+ * ntfs_mft_bitmap_extend_allocation_nolock - extend mft bitmap by a cluster
+ * @vol:	volume on which to extend the mft bitmap attribute
+ *
+ * Extend the mft bitmap attribute on the ntfs volume @vol by one cluster.
+ *
+ * Note: Only changes allocated_size, i.e. does not touch initialized_size or
+ * data_size.
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Locking: - Caller must hold vol->mftbmp_lock for writing.
+ *	    - This function takes NTFS_I(vol->mftbmp_ino)->runlist.lock for
+ *	      writing and releases it before returning.
+ *	    - This function takes vol->lcnbmp_lock for writing and releases it
+ *	      before returning.
+ */
+static int ntfs_mft_bitmap_extend_allocation_nolock(struct ntfs_volume *vol)
+{
+	s64 lcn;
+	s64 ll;
+	unsigned long flags;
+	struct folio *folio;
+	struct ntfs_inode *mft_ni, *mftbmp_ni;
+	struct runlist_element *rl, *rl2 = NULL;
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct mft_record *mrec;
+	struct attr_record *a = NULL;
+	int ret, mp_size;
+	u32 old_alen = 0;
+	u8 *b, tb;
+	struct {
+		u8 added_cluster:1;
+		u8 added_run:1;
+		u8 mp_rebuilt:1;
+		u8 mp_extended:1;
+	} status = { 0, 0, 0, 0 };
+	size_t new_rl_count;
+
+	ntfs_debug("Extending mft bitmap allocation.");
+	mft_ni = NTFS_I(vol->mft_ino);
+	mftbmp_ni = NTFS_I(vol->mftbmp_ino);
+	/*
+	 * Determine the last lcn of the mft bitmap.  The allocated size of the
+	 * mft bitmap cannot be zero so we are ok to do this.
+	 */
+	down_write(&mftbmp_ni->runlist.lock);
+	read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	ll = mftbmp_ni->allocated_size;
+	read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	rl = ntfs_attr_find_vcn_nolock(mftbmp_ni,
+			NTFS_B_TO_CLU(vol, ll - 1), NULL);
+	if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) {
+		up_write(&mftbmp_ni->runlist.lock);
+		ntfs_error(vol->sb,
+			"Failed to determine last allocated cluster of mft bitmap attribute.");
+		if (!IS_ERR(rl))
+			ret = -EIO;
+		else
+			ret = PTR_ERR(rl);
+		return ret;
+	}
+	lcn = rl->lcn + rl->length;
+	ntfs_debug("Last lcn of mft bitmap attribute is 0x%llx.",
+			(long long)lcn);
+	/*
+	 * Attempt to get the cluster following the last allocated cluster by
+	 * hand as it may be in the MFT zone so the allocator would not give it
+	 * to us.
+	 */
+	ll = lcn >> 3;
+	folio = read_mapping_folio(vol->lcnbmp_ino->i_mapping,
+			ll >> PAGE_SHIFT, NULL);
+	if (IS_ERR(folio)) {
+		up_write(&mftbmp_ni->runlist.lock);
+		ntfs_error(vol->sb, "Failed to read from lcn bitmap.");
+		return PTR_ERR(folio);
+	}
+
+	down_write(&vol->lcnbmp_lock);
+	folio_lock(folio);
+	b = (u8 *)kmap_local_folio(folio, 0) + (ll & ~PAGE_MASK);
+	tb = 1 << (lcn & 7ull);
+	if (*b != 0xff && !(*b & tb)) {
+		/* Next cluster is free, allocate it. */
+		*b |= tb;
+		folio_mark_dirty(folio);
+		folio_unlock(folio);
+		kunmap_local(b);
+		folio_put(folio);
+		up_write(&vol->lcnbmp_lock);
+		/* Update the mft bitmap runlist. */
+		rl->length++;
+		rl[1].vcn++;
+		status.added_cluster = 1;
+		ntfs_debug("Appending one cluster to mft bitmap.");
+	} else {
+		folio_unlock(folio);
+		kunmap_local(b);
+		folio_put(folio);
+		up_write(&vol->lcnbmp_lock);
+		/* Allocate a cluster from the DATA_ZONE. */
+		rl2 = ntfs_cluster_alloc(vol, rl[1].vcn, 1, lcn, DATA_ZONE,
+				true, false, false);
+		if (IS_ERR(rl2)) {
+			up_write(&mftbmp_ni->runlist.lock);
+			ntfs_error(vol->sb,
+					"Failed to allocate a cluster for the mft bitmap.");
+			return PTR_ERR(rl2);
+		}
+		rl = ntfs_runlists_merge(&mftbmp_ni->runlist, rl2, 0, &new_rl_count);
+		if (IS_ERR(rl)) {
+			up_write(&mftbmp_ni->runlist.lock);
+			ntfs_error(vol->sb, "Failed to merge runlists for mft bitmap.");
+			if (ntfs_cluster_free_from_rl(vol, rl2)) {
+				ntfs_error(vol->sb, "Failed to deallocate allocated cluster.%s",
+						es);
+				NVolSetErrors(vol);
+			}
+			kvfree(rl2);
+			return PTR_ERR(rl);
+		}
+		mftbmp_ni->runlist.rl = rl;
+		mftbmp_ni->runlist.count = new_rl_count;
+		status.added_run = 1;
+		ntfs_debug("Adding one run to mft bitmap.");
+		/* Find the last run in the new runlist. */
+		for (; rl[1].length; rl++)
+			;
+	}
+	/*
+	 * Update the attribute record as well.  Note: @rl is the last
+	 * (non-terminator) runlist element of mft bitmap.
+	 */
+	mrec = map_mft_record(mft_ni);
+	if (IS_ERR(mrec)) {
+		ntfs_error(vol->sb, "Failed to map mft record.");
+		ret = PTR_ERR(mrec);
+		goto undo_alloc;
+	}
+	ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
+	if (unlikely(!ctx)) {
+		ntfs_error(vol->sb, "Failed to get search context.");
+		ret = -ENOMEM;
+		goto undo_alloc;
+	}
+	ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
+			mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL,
+			0, ctx);
+	if (unlikely(ret)) {
+		ntfs_error(vol->sb,
+			"Failed to find last attribute extent of mft bitmap attribute.");
+		if (ret == -ENOENT)
+			ret = -EIO;
+		goto undo_alloc;
+	}
+	a = ctx->attr;
+	ll = le64_to_cpu(a->data.non_resident.lowest_vcn);
+	/* Search back for the previous last allocated cluster of mft bitmap. */
+	for (rl2 = rl; rl2 > mftbmp_ni->runlist.rl; rl2--) {
+		if (ll >= rl2->vcn)
+			break;
+	}
+	WARN_ON(ll < rl2->vcn);
+	WARN_ON(ll >= rl2->vcn + rl2->length);
+	/* Get the size for the new mapping pairs array for this extent. */
+	mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1);
+	if (unlikely(mp_size <= 0)) {
+		ntfs_error(vol->sb,
+			"Get size for mapping pairs failed for mft bitmap attribute extent.");
+		ret = mp_size;
+		if (!ret)
+			ret = -EIO;
+		goto undo_alloc;
+	}
+	/* Expand the attribute record if necessary. */
+	old_alen = le32_to_cpu(a->length);
+	ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size +
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset));
+	if (unlikely(ret)) {
+		ret = ntfs_mft_attr_extend(mftbmp_ni);
+		if (!ret)
+			goto extended_ok;
+		if (ret != -EAGAIN)
+			status.mp_extended = 1;
+		goto undo_alloc;
+	}
+	status.mp_rebuilt = 1;
+	/* Generate the mapping pairs array directly into the attr record. */
+	ret = ntfs_mapping_pairs_build(vol, (u8 *)a +
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
+			mp_size, rl2, ll, -1, NULL, NULL, NULL);
+	if (unlikely(ret)) {
+		ntfs_error(vol->sb,
+			"Failed to build mapping pairs array for mft bitmap attribute.");
+		goto undo_alloc;
+	}
+	/* Update the highest_vcn. */
+	a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1);
+	/*
+	 * We now have extended the mft bitmap allocated_size by one cluster.
+	 * Reflect this in the struct ntfs_inode structure and the attribute record.
+	 */
+	if (a->data.non_resident.lowest_vcn) {
+		/*
+		 * We are not in the first attribute extent, switch to it, but
+		 * first ensure the changes will make it to disk later.
+		 */
+		mark_mft_record_dirty(ctx->ntfs_ino);
+extended_ok:
+		ntfs_attr_reinit_search_ctx(ctx);
+		ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
+				mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL,
+				0, ctx);
+		if (unlikely(ret)) {
+			ntfs_error(vol->sb,
+				"Failed to find first attribute extent of mft bitmap attribute.");
+			goto restore_undo_alloc;
+		}
+		a = ctx->attr;
+	}
+
+	write_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	mftbmp_ni->allocated_size += vol->cluster_size;
+	a->data.non_resident.allocated_size =
+			cpu_to_le64(mftbmp_ni->allocated_size);
+	write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	/* Ensure the changes make it to disk. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(mft_ni);
+	up_write(&mftbmp_ni->runlist.lock);
+	ntfs_debug("Done.");
+	return 0;
+
+restore_undo_alloc:
+	ntfs_attr_reinit_search_ctx(ctx);
+	if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
+			mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL,
+			0, ctx)) {
+		ntfs_error(vol->sb,
+			"Failed to find last attribute extent of mft bitmap attribute.%s", es);
+		write_lock_irqsave(&mftbmp_ni->size_lock, flags);
+		mftbmp_ni->allocated_size += vol->cluster_size;
+		write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(mft_ni);
+		up_write(&mftbmp_ni->runlist.lock);
+		/*
+		 * The only thing that is now wrong is ->allocated_size of the
+		 * base attribute extent which chkdsk should be able to fix.
+		 */
+		NVolSetErrors(vol);
+		return ret;
+	}
+	a = ctx->attr;
+	a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 2);
+undo_alloc:
+	if (status.added_cluster) {
+		/* Truncate the last run in the runlist by one cluster. */
+		rl->length--;
+		rl[1].vcn--;
+	} else if (status.added_run) {
+		lcn = rl->lcn;
+		/* Remove the last run from the runlist. */
+		rl->lcn = rl[1].lcn;
+		rl->length = 0;
+		mftbmp_ni->runlist.count--;
+	}
+	/* Deallocate the cluster. */
+	down_write(&vol->lcnbmp_lock);
+	if (ntfs_bitmap_clear_bit(vol->lcnbmp_ino, lcn)) {
+		ntfs_error(vol->sb, "Failed to free allocated cluster.%s", es);
+		NVolSetErrors(vol);
+	} else
+		ntfs_inc_free_clusters(vol, 1);
+	up_write(&vol->lcnbmp_lock);
+	if (status.mp_rebuilt) {
+		if (ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu(
+				a->data.non_resident.mapping_pairs_offset),
+				old_alen - le16_to_cpu(
+				a->data.non_resident.mapping_pairs_offset),
+				rl2, ll, -1, NULL, NULL, NULL)) {
+			ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es);
+			NVolSetErrors(vol);
+		}
+		if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) {
+			ntfs_error(vol->sb, "Failed to restore attribute record.%s", es);
+			NVolSetErrors(vol);
+		}
+		mark_mft_record_dirty(ctx->ntfs_ino);
+	} else if (status.mp_extended && ntfs_attr_update_mapping_pairs(mftbmp_ni, 0)) {
+		ntfs_error(vol->sb, "Failed to restore mapping pairs.%s", es);
+		NVolSetErrors(vol);
+	}
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	if (!IS_ERR(mrec))
+		unmap_mft_record(mft_ni);
+	up_write(&mftbmp_ni->runlist.lock);
+	return ret;
+}
+
+/*
+ * ntfs_mft_bitmap_extend_initialized_nolock - extend mftbmp initialized data
+ * @vol:	volume on which to extend the mft bitmap attribute
+ *
+ * Extend the initialized portion of the mft bitmap attribute on the ntfs
+ * volume @vol by 8 bytes.
+ *
+ * Note:  Only changes initialized_size and data_size, i.e. requires that
+ * allocated_size is big enough to fit the new initialized_size.
+ *
+ * Return 0 on success and -error on error.
+ *
+ * Locking: Caller must hold vol->mftbmp_lock for writing.
+ */
+static int ntfs_mft_bitmap_extend_initialized_nolock(struct ntfs_volume *vol)
+{
+	s64 old_data_size, old_initialized_size;
+	unsigned long flags;
+	struct inode *mftbmp_vi;
+	struct ntfs_inode *mft_ni, *mftbmp_ni;
+	struct ntfs_attr_search_ctx *ctx;
+	struct mft_record *mrec;
+	struct attr_record *a;
+	int ret;
+
+	ntfs_debug("Extending mft bitmap initialized (and data) size.");
+	mft_ni = NTFS_I(vol->mft_ino);
+	mftbmp_vi = vol->mftbmp_ino;
+	mftbmp_ni = NTFS_I(mftbmp_vi);
+	/* Get the attribute record. */
+	mrec = map_mft_record(mft_ni);
+	if (IS_ERR(mrec)) {
+		ntfs_error(vol->sb, "Failed to map mft record.");
+		return PTR_ERR(mrec);
+	}
+	ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
+	if (unlikely(!ctx)) {
+		ntfs_error(vol->sb, "Failed to get search context.");
+		ret = -ENOMEM;
+		goto unm_err_out;
+	}
+	ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
+			mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(ret)) {
+		ntfs_error(vol->sb,
+			"Failed to find first attribute extent of mft bitmap attribute.");
+		if (ret == -ENOENT)
+			ret = -EIO;
+		goto put_err_out;
+	}
+	a = ctx->attr;
+	write_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	old_data_size = i_size_read(mftbmp_vi);
+	old_initialized_size = mftbmp_ni->initialized_size;
+	/*
+	 * We can simply update the initialized_size before filling the space
+	 * with zeroes because the caller is holding the mft bitmap lock for
+	 * writing which ensures that no one else is trying to access the data.
+	 */
+	mftbmp_ni->initialized_size += 8;
+	a->data.non_resident.initialized_size =
+			cpu_to_le64(mftbmp_ni->initialized_size);
+	if (mftbmp_ni->initialized_size > old_data_size) {
+		i_size_write(mftbmp_vi, mftbmp_ni->initialized_size);
+		a->data.non_resident.data_size =
+				cpu_to_le64(mftbmp_ni->initialized_size);
+	}
+	write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	/* Ensure the changes make it to disk. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(mft_ni);
+	/* Initialize the mft bitmap attribute value with zeroes. */
+	ret = ntfs_attr_set(mftbmp_ni, old_initialized_size, 8, 0);
+	if (likely(!ret)) {
+		ntfs_debug("Done.  (Wrote eight initialized bytes to mft bitmap.");
+		ntfs_inc_free_mft_records(vol, 8 * 8);
+		return 0;
+	}
+	ntfs_error(vol->sb, "Failed to write to mft bitmap.");
+	/* Try to recover from the error. */
+	mrec = map_mft_record(mft_ni);
+	if (IS_ERR(mrec)) {
+		ntfs_error(vol->sb, "Failed to map mft record.%s", es);
+		NVolSetErrors(vol);
+		return ret;
+	}
+	ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
+	if (unlikely(!ctx)) {
+		ntfs_error(vol->sb, "Failed to get search context.%s", es);
+		NVolSetErrors(vol);
+		goto unm_err_out;
+	}
+	if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
+			mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx)) {
+		ntfs_error(vol->sb,
+			"Failed to find first attribute extent of mft bitmap attribute.%s", es);
+		NVolSetErrors(vol);
+put_err_out:
+		ntfs_attr_put_search_ctx(ctx);
+unm_err_out:
+		unmap_mft_record(mft_ni);
+		goto err_out;
+	}
+	a = ctx->attr;
+	write_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	mftbmp_ni->initialized_size = old_initialized_size;
+	a->data.non_resident.initialized_size =
+			cpu_to_le64(old_initialized_size);
+	if (i_size_read(mftbmp_vi) != old_data_size) {
+		i_size_write(mftbmp_vi, old_data_size);
+		a->data.non_resident.data_size = cpu_to_le64(old_data_size);
+	}
+	write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(mft_ni);
+#ifdef DEBUG
+	read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	ntfs_debug("Restored status of mftbmp: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+			mftbmp_ni->allocated_size, i_size_read(mftbmp_vi),
+			mftbmp_ni->initialized_size);
+	read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+#endif /* DEBUG */
+err_out:
+	return ret;
+}
+
+/*
+ * ntfs_mft_data_extend_allocation_nolock - extend mft data attribute
+ * @vol:	volume on which to extend the mft data attribute
+ *
+ * Extend the mft data attribute on the ntfs volume @vol by 16 mft records
+ * worth of clusters or if not enough space for this by one mft record worth
+ * of clusters.
+ *
+ * Note:  Only changes allocated_size, i.e. does not touch initialized_size or
+ * data_size.
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Locking: - Caller must hold vol->mftbmp_lock for writing.
+ *	    - This function takes NTFS_I(vol->mft_ino)->runlist.lock for
+ *	      writing and releases it before returning.
+ *	    - This function calls functions which take vol->lcnbmp_lock for
+ *	      writing and release it before returning.
+ */
+static int ntfs_mft_data_extend_allocation_nolock(struct ntfs_volume *vol)
+{
+	s64 lcn;
+	s64 old_last_vcn;
+	s64 min_nr, nr, ll;
+	unsigned long flags;
+	struct ntfs_inode *mft_ni;
+	struct runlist_element *rl, *rl2;
+	struct ntfs_attr_search_ctx *ctx = NULL;
+	struct mft_record *mrec;
+	struct attr_record *a = NULL;
+	int ret, mp_size;
+	u32 old_alen = 0;
+	bool mp_rebuilt = false, mp_extended = false;
+	size_t new_rl_count;
+
+	ntfs_debug("Extending mft data allocation.");
+	mft_ni = NTFS_I(vol->mft_ino);
+	/*
+	 * Determine the preferred allocation location, i.e. the last lcn of
+	 * the mft data attribute.  The allocated size of the mft data
+	 * attribute cannot be zero so we are ok to do this.
+	 */
+	down_write(&mft_ni->runlist.lock);
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	ll = mft_ni->allocated_size;
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	rl = ntfs_attr_find_vcn_nolock(mft_ni,
+			NTFS_B_TO_CLU(vol, ll - 1), NULL);
+	if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) {
+		up_write(&mft_ni->runlist.lock);
+		ntfs_error(vol->sb,
+			"Failed to determine last allocated cluster of mft data attribute.");
+		if (!IS_ERR(rl))
+			ret = -EIO;
+		else
+			ret = PTR_ERR(rl);
+		return ret;
+	}
+	lcn = rl->lcn + rl->length;
+	ntfs_debug("Last lcn of mft data attribute is 0x%llx.", lcn);
+	/* Minimum allocation is one mft record worth of clusters. */
+	min_nr = NTFS_B_TO_CLU(vol, vol->mft_record_size);
+	if (!min_nr)
+		min_nr = 1;
+	/* Want to allocate 16 mft records worth of clusters. */
+	nr = vol->mft_record_size << 4 >> vol->cluster_size_bits;
+	if (!nr)
+		nr = min_nr;
+	/* Ensure we do not go above 2^32-1 mft records. */
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	ll = mft_ni->allocated_size;
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >>
+			vol->mft_record_size_bits >= (1ll << 32))) {
+		nr = min_nr;
+		if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >>
+				vol->mft_record_size_bits >= (1ll << 32))) {
+			ntfs_warning(vol->sb,
+				"Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached.");
+			up_write(&mft_ni->runlist.lock);
+			return -ENOSPC;
+		}
+	}
+	ntfs_debug("Trying mft data allocation with %s cluster count %lli.",
+			nr > min_nr ? "default" : "minimal", (long long)nr);
+	old_last_vcn = rl[1].vcn;
+	/*
+	 * We can release the mft_ni runlist lock, Because this function is
+	 * the only one that expends $MFT data attribute and is called with
+	 * mft_ni->mrec_lock.
+	 * This is required for the lock order, vol->lcnbmp_lock =>
+	 * mft_ni->runlist.lock.
+	 */
+	up_write(&mft_ni->runlist.lock);
+
+	do {
+		rl2 = ntfs_cluster_alloc(vol, old_last_vcn, nr, lcn, MFT_ZONE,
+				true, false, false);
+		if (!IS_ERR(rl2))
+			break;
+		if (PTR_ERR(rl2) != -ENOSPC || nr == min_nr) {
+			ntfs_error(vol->sb,
+				"Failed to allocate the minimal number of clusters (%lli) for the mft data attribute.",
+				nr);
+			return PTR_ERR(rl2);
+		}
+		/*
+		 * There is not enough space to do the allocation, but there
+		 * might be enough space to do a minimal allocation so try that
+		 * before failing.
+		 */
+		nr = min_nr;
+		ntfs_debug("Retrying mft data allocation with minimal cluster count %lli.", nr);
+	} while (1);
+
+	down_write(&mft_ni->runlist.lock);
+	rl = ntfs_runlists_merge(&mft_ni->runlist, rl2, 0, &new_rl_count);
+	if (IS_ERR(rl)) {
+		up_write(&mft_ni->runlist.lock);
+		ntfs_error(vol->sb, "Failed to merge runlists for mft data attribute.");
+		if (ntfs_cluster_free_from_rl(vol, rl2)) {
+			ntfs_error(vol->sb,
+				"Failed to deallocate clusters from the mft data attribute.%s", es);
+			NVolSetErrors(vol);
+		}
+		kvfree(rl2);
+		return PTR_ERR(rl);
+	}
+	mft_ni->runlist.rl = rl;
+	mft_ni->runlist.count = new_rl_count;
+	ntfs_debug("Allocated %lli clusters.", (long long)nr);
+	/* Find the last run in the new runlist. */
+	for (; rl[1].length; rl++)
+		;
+	up_write(&mft_ni->runlist.lock);
+
+	/* Update the attribute record as well. */
+	mrec = map_mft_record(mft_ni);
+	if (IS_ERR(mrec)) {
+		ntfs_error(vol->sb, "Failed to map mft record.");
+		ret = PTR_ERR(mrec);
+		down_write(&mft_ni->runlist.lock);
+		goto undo_alloc;
+	}
+	ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
+	if (unlikely(!ctx)) {
+		ntfs_error(vol->sb, "Failed to get search context.");
+		ret = -ENOMEM;
+		goto undo_alloc;
+	}
+	ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
+			CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx);
+	if (unlikely(ret)) {
+		ntfs_error(vol->sb, "Failed to find last attribute extent of mft data attribute.");
+		if (ret == -ENOENT)
+			ret = -EIO;
+		goto undo_alloc;
+	}
+	a = ctx->attr;
+	ll = le64_to_cpu(a->data.non_resident.lowest_vcn);
+
+	down_write(&mft_ni->runlist.lock);
+	/* Search back for the previous last allocated cluster of mft bitmap. */
+	for (rl2 = rl; rl2 > mft_ni->runlist.rl; rl2--) {
+		if (ll >= rl2->vcn)
+			break;
+	}
+	WARN_ON(ll < rl2->vcn);
+	WARN_ON(ll >= rl2->vcn + rl2->length);
+	/* Get the size for the new mapping pairs array for this extent. */
+	mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1);
+	if (unlikely(mp_size <= 0)) {
+		ntfs_error(vol->sb,
+			"Get size for mapping pairs failed for mft data attribute extent.");
+		ret = mp_size;
+		if (!ret)
+			ret = -EIO;
+		up_write(&mft_ni->runlist.lock);
+		goto undo_alloc;
+	}
+	up_write(&mft_ni->runlist.lock);
+
+	/* Expand the attribute record if necessary. */
+	old_alen = le32_to_cpu(a->length);
+	ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size +
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset));
+	if (unlikely(ret)) {
+		ret = ntfs_mft_attr_extend(mft_ni);
+		if (!ret)
+			goto extended_ok;
+		if (ret != -EAGAIN)
+			mp_extended = true;
+		goto undo_alloc;
+	}
+	mp_rebuilt = true;
+	/* Generate the mapping pairs array directly into the attr record. */
+	ret = ntfs_mapping_pairs_build(vol, (u8 *)a +
+			le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
+			mp_size, rl2, ll, -1, NULL, NULL, NULL);
+	if (unlikely(ret)) {
+		ntfs_error(vol->sb, "Failed to build mapping pairs array of mft data attribute.");
+		goto undo_alloc;
+	}
+	/* Update the highest_vcn. */
+	a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1);
+	/*
+	 * We now have extended the mft data allocated_size by nr clusters.
+	 * Reflect this in the struct ntfs_inode structure and the attribute record.
+	 * @rl is the last (non-terminator) runlist element of mft data
+	 * attribute.
+	 */
+	if (a->data.non_resident.lowest_vcn) {
+		/*
+		 * We are not in the first attribute extent, switch to it, but
+		 * first ensure the changes will make it to disk later.
+		 */
+		mark_mft_record_dirty(ctx->ntfs_ino);
+extended_ok:
+		ntfs_attr_reinit_search_ctx(ctx);
+		ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name,
+				mft_ni->name_len, CASE_SENSITIVE, 0, NULL, 0,
+				ctx);
+		if (unlikely(ret)) {
+			ntfs_error(vol->sb,
+				"Failed to find first attribute extent of mft data attribute.");
+			goto restore_undo_alloc;
+		}
+		a = ctx->attr;
+	}
+
+	write_lock_irqsave(&mft_ni->size_lock, flags);
+	mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr);
+	a->data.non_resident.allocated_size =
+			cpu_to_le64(mft_ni->allocated_size);
+	write_unlock_irqrestore(&mft_ni->size_lock, flags);
+	/* Ensure the changes make it to disk. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(mft_ni);
+	ntfs_debug("Done.");
+	return 0;
+restore_undo_alloc:
+	ntfs_attr_reinit_search_ctx(ctx);
+	if (ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
+			CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx)) {
+		ntfs_error(vol->sb,
+			"Failed to find last attribute extent of mft data attribute.%s", es);
+		write_lock_irqsave(&mft_ni->size_lock, flags);
+		mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr);
+		write_unlock_irqrestore(&mft_ni->size_lock, flags);
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(mft_ni);
+		up_write(&mft_ni->runlist.lock);
+		/*
+		 * The only thing that is now wrong is ->allocated_size of the
+		 * base attribute extent which chkdsk should be able to fix.
+		 */
+		NVolSetErrors(vol);
+		return ret;
+	}
+	ctx->attr->data.non_resident.highest_vcn =
+			cpu_to_le64(old_last_vcn - 1);
+undo_alloc:
+	if (ntfs_cluster_free(mft_ni, old_last_vcn, -1, ctx) < 0) {
+		ntfs_error(vol->sb, "Failed to free clusters from mft data attribute.%s", es);
+		NVolSetErrors(vol);
+	}
+
+	if (ntfs_rl_truncate_nolock(vol, &mft_ni->runlist, old_last_vcn)) {
+		ntfs_error(vol->sb, "Failed to truncate mft data attribute runlist.%s", es);
+		NVolSetErrors(vol);
+	}
+	if (mp_extended && ntfs_attr_update_mapping_pairs(mft_ni, 0)) {
+		ntfs_error(vol->sb, "Failed to restore mapping pairs.%s",
+			   es);
+		NVolSetErrors(vol);
+	}
+	if (ctx) {
+		a = ctx->attr;
+		if (mp_rebuilt && !IS_ERR(ctx->mrec)) {
+			if (ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu(
+				a->data.non_resident.mapping_pairs_offset),
+				old_alen - le16_to_cpu(
+					a->data.non_resident.mapping_pairs_offset),
+				rl2, ll, -1, NULL, NULL, NULL)) {
+				ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es);
+				NVolSetErrors(vol);
+			}
+			if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) {
+				ntfs_error(vol->sb, "Failed to restore attribute record.%s", es);
+				NVolSetErrors(vol);
+			}
+			mark_mft_record_dirty(ctx->ntfs_ino);
+		} else if (IS_ERR(ctx->mrec)) {
+			ntfs_error(vol->sb, "Failed to restore attribute search context.%s", es);
+			NVolSetErrors(vol);
+		}
+		ntfs_attr_put_search_ctx(ctx);
+	}
+	if (!IS_ERR(mrec))
+		unmap_mft_record(mft_ni);
+	return ret;
+}
+
+/*
+ * ntfs_mft_record_layout - layout an mft record into a memory buffer
+ * @vol:	volume to which the mft record will belong
+ * @mft_no:	mft reference specifying the mft record number
+ * @m:		destination buffer of size >= @vol->mft_record_size bytes
+ *
+ * Layout an empty, unused mft record with the mft record number @mft_no into
+ * the buffer @m.  The volume @vol is needed because the mft record structure
+ * was modified in NTFS 3.1 so we need to know which volume version this mft
+ * record will be used on.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_mft_record_layout(const struct ntfs_volume *vol, const s64 mft_no,
+		struct mft_record *m)
+{
+	struct attr_record *a;
+
+	ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no);
+	if (mft_no >= (1ll << 32)) {
+		ntfs_error(vol->sb, "Mft record number 0x%llx exceeds maximum of 2^32.",
+				(long long)mft_no);
+		return -ERANGE;
+	}
+	/* Start by clearing the whole mft record to gives us a clean slate. */
+	memset(m, 0, vol->mft_record_size);
+	/* Aligned to 2-byte boundary. */
+	if (vol->major_ver < 3 || (vol->major_ver == 3 && !vol->minor_ver))
+		m->usa_ofs = cpu_to_le16((sizeof(struct mft_record_old) + 1) & ~1);
+	else {
+		m->usa_ofs = cpu_to_le16((sizeof(struct mft_record) + 1) & ~1);
+		/*
+		 * Set the NTFS 3.1+ specific fields while we know that the
+		 * volume version is 3.1+.
+		 */
+		m->reserved = 0;
+		m->mft_record_number = cpu_to_le32((u32)mft_no);
+	}
+	m->magic = magic_FILE;
+	if (vol->mft_record_size >= NTFS_BLOCK_SIZE)
+		m->usa_count = cpu_to_le16(vol->mft_record_size /
+				NTFS_BLOCK_SIZE + 1);
+	else {
+		m->usa_count = cpu_to_le16(1);
+		ntfs_warning(vol->sb,
+			"Sector size is bigger than mft record size.  Setting usa_count to 1.  If chkdsk reports this as corruption");
+	}
+	/* Set the update sequence number to 1. */
+	*(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = cpu_to_le16(1);
+	m->lsn = 0;
+	m->sequence_number = cpu_to_le16(1);
+	m->link_count = 0;
+	/*
+	 * Place the attributes straight after the update sequence array,
+	 * aligned to 8-byte boundary.
+	 */
+	m->attrs_offset = cpu_to_le16((le16_to_cpu(m->usa_ofs) +
+			(le16_to_cpu(m->usa_count) << 1) + 7) & ~7);
+	m->flags = 0;
+	/*
+	 * Using attrs_offset plus eight bytes (for the termination attribute).
+	 * attrs_offset is already aligned to 8-byte boundary, so no need to
+	 * align again.
+	 */
+	m->bytes_in_use = cpu_to_le32(le16_to_cpu(m->attrs_offset) + 8);
+	m->bytes_allocated = cpu_to_le32(vol->mft_record_size);
+	m->base_mft_record = 0;
+	m->next_attr_instance = 0;
+	/* Add the termination attribute. */
+	a = (struct attr_record *)((u8 *)m + le16_to_cpu(m->attrs_offset));
+	a->type = AT_END;
+	a->length = 0;
+	ntfs_debug("Done.");
+	return 0;
+}
+
+/*
+ * ntfs_mft_record_format - format an mft record on an ntfs volume
+ * @vol:	volume on which to format the mft record
+ * @mft_no:	mft record number to format
+ *
+ * Format the mft record @mft_no in $MFT/$DATA, i.e. lay out an empty, unused
+ * mft record into the appropriate place of the mft data attribute.  This is
+ * used when extending the mft data attribute.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_no)
+{
+	loff_t i_size;
+	struct inode *mft_vi = vol->mft_ino;
+	struct folio *folio;
+	struct mft_record *m;
+	pgoff_t index, end_index;
+	unsigned int ofs;
+	int err;
+
+	ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no);
+	/*
+	 * The index into the page cache and the offset within the page cache
+	 * page of the wanted mft record.
+	 */
+	index = NTFS_MFT_NR_TO_PIDX(vol, mft_no);
+	ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no);
+	/* The maximum valid index into the page cache for $MFT's data. */
+	i_size = i_size_read(mft_vi);
+	end_index = i_size >> PAGE_SHIFT;
+	if (unlikely(index >= end_index)) {
+		if (unlikely(index > end_index ||
+			     ofs + vol->mft_record_size > (i_size & ~PAGE_MASK))) {
+			ntfs_error(vol->sb, "Tried to format non-existing mft record 0x%llx.",
+					(long long)mft_no);
+			return -ENOENT;
+		}
+	}
+
+	/* Read, map, and pin the folio containing the mft record. */
+	folio = read_mapping_folio(mft_vi->i_mapping, index, NULL);
+	if (IS_ERR(folio)) {
+		ntfs_error(vol->sb, "Failed to map page containing mft record to format 0x%llx.",
+				(long long)mft_no);
+		return PTR_ERR(folio);
+	}
+	folio_lock(folio);
+	folio_clear_uptodate(folio);
+	m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs);
+	err = ntfs_mft_record_layout(vol, mft_no, m);
+	if (unlikely(err)) {
+		ntfs_error(vol->sb, "Failed to layout mft record 0x%llx.",
+				(long long)mft_no);
+		folio_mark_uptodate(folio);
+		folio_unlock(folio);
+		kunmap_local(m);
+		folio_put(folio);
+		return err;
+	}
+	pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size);
+	folio_mark_uptodate(folio);
+	/*
+	 * Make sure the mft record is written out to disk.  We could use
+	 * ilookup5() to check if an inode is in icache and so on but this is
+	 * unnecessary as ntfs_writepage() will write the dirty record anyway.
+	 */
+	ntfs_mft_mark_dirty(folio);
+	folio_unlock(folio);
+	kunmap_local(m);
+	folio_put(folio);
+	ntfs_debug("Done.");
+	return 0;
+}
+
+/*
+ * ntfs_mft_record_alloc - allocate an mft record on an ntfs volume
+ * @vol:	[IN]  volume on which to allocate the mft record
+ * @mode:	[IN]  mode if want a file or directory, i.e. base inode or 0
+ * @ni:		[OUT] on success, set to the allocated ntfs inode
+ * @base_ni:	[IN]  open base inode if allocating an extent mft record or NULL
+ * @ni_mrec:	[OUT] on successful return this is the mapped mft record
+ *
+ * Allocate an mft record in $MFT/$DATA of an open ntfs volume @vol.
+ *
+ * If @base_ni is NULL make the mft record a base mft record, i.e. a file or
+ * direvctory inode, and allocate it at the default allocator position.  In
+ * this case @mode is the file mode as given to us by the caller.  We in
+ * particular use @mode to distinguish whether a file or a directory is being
+ * created (S_IFDIR(mode) and S_IFREG(mode), respectively).
+ *
+ * If @base_ni is not NULL make the allocated mft record an extent record,
+ * allocate it starting at the mft record after the base mft record and attach
+ * the allocated and opened ntfs inode to the base inode @base_ni.  In this
+ * case @mode must be 0 as it is meaningless for extent inodes.
+ *
+ * You need to check the return value with IS_ERR().  If false, the function
+ * was successful and the return value is the now opened ntfs inode of the
+ * allocated mft record.  *@mrec is then set to the allocated, mapped, pinned,
+ * and locked mft record.  If IS_ERR() is true, the function failed and the
+ * error code is obtained from PTR_ERR(return value).  *@mrec is undefined in
+ * this case.
+ *
+ * Allocation strategy:
+ *
+ * To find a free mft record, we scan the mft bitmap for a zero bit.  To
+ * optimize this we start scanning at the place specified by @base_ni or if
+ * @base_ni is NULL we start where we last stopped and we perform wrap around
+ * when we reach the end.  Note, we do not try to allocate mft records below
+ * number 64 because numbers 0 to 15 are the defined system files anyway and 16
+ * to 64 are special in that they are used for storing extension mft records
+ * for the $DATA attribute of $MFT.  This is required to avoid the possibility
+ * of creating a runlist with a circular dependency which once written to disk
+ * can never be read in again.  Windows will only use records 16 to 24 for
+ * normal files if the volume is completely out of space.  We never use them
+ * which means that when the volume is really out of space we cannot create any
+ * more files while Windows can still create up to 8 small files.  We can start
+ * doing this at some later time, it does not matter much for now.
+ *
+ * When scanning the mft bitmap, we only search up to the last allocated mft
+ * record.  If there are no free records left in the range 64 to number of
+ * allocated mft records, then we extend the $MFT/$DATA attribute in order to
+ * create free mft records.  We extend the allocated size of $MFT/$DATA by 16
+ * records at a time or one cluster, if cluster size is above 16kiB.  If there
+ * is not sufficient space to do this, we try to extend by a single mft record
+ * or one cluster, if cluster size is above the mft record size.
+ *
+ * No matter how many mft records we allocate, we initialize only the first
+ * allocated mft record, incrementing mft data size and initialized size
+ * accordingly, open an struct ntfs_inode for it and return it to the caller, unless
+ * there are less than 64 mft records, in which case we allocate and initialize
+ * mft records until we reach record 64 which we consider as the first free mft
+ * record for use by normal files.
+ *
+ * If during any stage we overflow the initialized data in the mft bitmap, we
+ * extend the initialized size (and data size) by 8 bytes, allocating another
+ * cluster if required.  The bitmap data size has to be at least equal to the
+ * number of mft records in the mft, but it can be bigger, in which case the
+ * superfluous bits are padded with zeroes.
+ *
+ * Thus, when we return successfully (IS_ERR() is false), we will have:
+ *	- initialized / extended the mft bitmap if necessary,
+ *	- initialized / extended the mft data if necessary,
+ *	- set the bit corresponding to the mft record being allocated in the
+ *	  mft bitmap,
+ *	- opened an struct ntfs_inode for the allocated mft record, and we will have
+ *	- returned the struct ntfs_inode as well as the allocated mapped, pinned, and
+ *	  locked mft record.
+ *
+ * On error, the volume will be left in a consistent state and no record will
+ * be allocated.  If rolling back a partial operation fails, we may leave some
+ * inconsistent metadata in which case we set NVolErrors() so the volume is
+ * left dirty when unmounted.
+ *
+ * Note, this function cannot make use of most of the normal functions, like
+ * for example for attribute resizing, etc, because when the run list overflows
+ * the base mft record and an attribute list is used, it is very important that
+ * the extension mft records used to store the $DATA attribute of $MFT can be
+ * reached without having to read the information contained inside them, as
+ * this would make it impossible to find them in the first place after the
+ * volume is unmounted.  $MFT/$BITMAP probably does not need to follow this
+ * rule because the bitmap is not essential for finding the mft records, but on
+ * the other hand, handling the bitmap in this special way would make life
+ * easier because otherwise there might be circular invocations of functions
+ * when reading the bitmap.
+ */
+int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode,
+			  struct ntfs_inode **ni, struct ntfs_inode *base_ni,
+			  struct mft_record **ni_mrec)
+{
+	s64 ll, bit, old_data_initialized, old_data_size;
+	unsigned long flags;
+	struct folio *folio;
+	struct ntfs_inode *mft_ni, *mftbmp_ni;
+	struct ntfs_attr_search_ctx *ctx;
+	struct mft_record *m = NULL;
+	struct attr_record *a;
+	pgoff_t index;
+	unsigned int ofs;
+	int err;
+	__le16 seq_no, usn;
+	bool record_formatted = false;
+	unsigned int memalloc_flags;
+
+	if (base_ni && *ni)
+		return -EINVAL;
+
+	/* @mode and @base_ni are mutually exclusive. */
+	if (mode && base_ni)
+		return -EINVAL;
+
+	if (base_ni)
+		ntfs_debug("Entering (allocating an extent mft record for base mft record 0x%llx).",
+				(long long)base_ni->mft_no);
+	else
+		ntfs_debug("Entering (allocating a base mft record).");
+
+	memalloc_flags = memalloc_nofs_save();
+
+	mft_ni = NTFS_I(vol->mft_ino);
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		mutex_lock(&mft_ni->mrec_lock);
+	mftbmp_ni = NTFS_I(vol->mftbmp_ino);
+search_free_rec:
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		down_write(&vol->mftbmp_lock);
+	bit = ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(vol, base_ni);
+	if (bit >= 0) {
+		ntfs_debug("Found and allocated free record (#1), bit 0x%llx.",
+				(long long)bit);
+		goto have_alloc_rec;
+	}
+	if (bit != -ENOSPC) {
+		if (!base_ni || base_ni->mft_no != FILE_MFT) {
+			up_write(&vol->mftbmp_lock);
+			mutex_unlock(&mft_ni->mrec_lock);
+		}
+		memalloc_nofs_restore(memalloc_flags);
+		return bit;
+	}
+
+	if (base_ni && base_ni->mft_no == FILE_MFT) {
+		memalloc_nofs_restore(memalloc_flags);
+		return bit;
+	}
+
+	/*
+	 * No free mft records left.  If the mft bitmap already covers more
+	 * than the currently used mft records, the next records are all free,
+	 * so we can simply allocate the first unused mft record.
+	 * Note: We also have to make sure that the mft bitmap at least covers
+	 * the first 24 mft records as they are special and whilst they may not
+	 * be in use, we do not allocate from them.
+	 */
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	ll = mft_ni->initialized_size >> vol->mft_record_size_bits;
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	old_data_initialized = mftbmp_ni->initialized_size;
+	read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	if (old_data_initialized << 3 > ll &&
+	    old_data_initialized > RESERVED_MFT_RECORDS / 8) {
+		bit = ll;
+		if (bit < RESERVED_MFT_RECORDS)
+			bit = RESERVED_MFT_RECORDS;
+		if (unlikely(bit >= (1ll << 32)))
+			goto max_err_out;
+		ntfs_debug("Found free record (#2), bit 0x%llx.",
+				(long long)bit);
+		goto found_free_rec;
+	}
+	/*
+	 * The mft bitmap needs to be expanded until it covers the first unused
+	 * mft record that we can allocate.
+	 * Note: The smallest mft record we allocate is mft record 24.
+	 */
+	bit = old_data_initialized << 3;
+	if (unlikely(bit >= (1ll << 32)))
+		goto max_err_out;
+	read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	old_data_size = mftbmp_ni->allocated_size;
+	ntfs_debug("Status of mftbmp before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+			old_data_size, i_size_read(vol->mftbmp_ino),
+			old_data_initialized);
+	read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+	if (old_data_initialized + 8 > old_data_size) {
+		/* Need to extend bitmap by one more cluster. */
+		ntfs_debug("mftbmp: initialized_size + 8 > allocated_size.");
+		err = ntfs_mft_bitmap_extend_allocation_nolock(vol);
+		if (err == -EAGAIN)
+			err = ntfs_mft_bitmap_extend_allocation_nolock(vol);
+
+		if (unlikely(err)) {
+			if (!base_ni || base_ni->mft_no != FILE_MFT)
+				up_write(&vol->mftbmp_lock);
+			goto err_out;
+		}
+#ifdef DEBUG
+		read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+		ntfs_debug("Status of mftbmp after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+				mftbmp_ni->allocated_size,
+				i_size_read(vol->mftbmp_ino),
+				mftbmp_ni->initialized_size);
+		read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+#endif /* DEBUG */
+	}
+	/*
+	 * We now have sufficient allocated space, extend the initialized_size
+	 * as well as the data_size if necessary and fill the new space with
+	 * zeroes.
+	 */
+	err = ntfs_mft_bitmap_extend_initialized_nolock(vol);
+	if (unlikely(err)) {
+		if (!base_ni || base_ni->mft_no != FILE_MFT)
+			up_write(&vol->mftbmp_lock);
+		goto err_out;
+	}
+#ifdef DEBUG
+	read_lock_irqsave(&mftbmp_ni->size_lock, flags);
+	ntfs_debug("Status of mftbmp after initialized extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+			mftbmp_ni->allocated_size,
+			i_size_read(vol->mftbmp_ino),
+			mftbmp_ni->initialized_size);
+	read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
+#endif /* DEBUG */
+	ntfs_debug("Found free record (#3), bit 0x%llx.", (long long)bit);
+found_free_rec:
+	/* @bit is the found free mft record, allocate it in the mft bitmap. */
+	ntfs_debug("At found_free_rec.");
+	err = ntfs_bitmap_set_bit(vol->mftbmp_ino, bit);
+	if (unlikely(err)) {
+		ntfs_error(vol->sb, "Failed to allocate bit in mft bitmap.");
+		if (!base_ni || base_ni->mft_no != FILE_MFT)
+			up_write(&vol->mftbmp_lock);
+		goto err_out;
+	}
+	ntfs_debug("Set bit 0x%llx in mft bitmap.", (long long)bit);
+have_alloc_rec:
+	/*
+	 * The mft bitmap is now uptodate.  Deal with mft data attribute now.
+	 * Note, we keep hold of the mft bitmap lock for writing until all
+	 * modifications to the mft data attribute are complete, too, as they
+	 * will impact decisions for mft bitmap and mft record allocation done
+	 * by a parallel allocation and if the lock is not maintained a
+	 * parallel allocation could allocate the same mft record as this one.
+	 */
+	ll = (bit + 1) << vol->mft_record_size_bits;
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	old_data_initialized = mft_ni->initialized_size;
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	if (ll <= old_data_initialized) {
+		ntfs_debug("Allocated mft record already initialized.");
+		goto mft_rec_already_initialized;
+	}
+	ntfs_debug("Initializing allocated mft record.");
+	/*
+	 * The mft record is outside the initialized data.  Extend the mft data
+	 * attribute until it covers the allocated record.  The loop is only
+	 * actually traversed more than once when a freshly formatted volume is
+	 * first written to so it optimizes away nicely in the common case.
+	 */
+	if (!base_ni || base_ni->mft_no != FILE_MFT) {
+		read_lock_irqsave(&mft_ni->size_lock, flags);
+		ntfs_debug("Status of mft data before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+				mft_ni->allocated_size, i_size_read(vol->mft_ino),
+				mft_ni->initialized_size);
+		while (ll > mft_ni->allocated_size) {
+			read_unlock_irqrestore(&mft_ni->size_lock, flags);
+			err = ntfs_mft_data_extend_allocation_nolock(vol);
+			if (err == -EAGAIN)
+				err = ntfs_mft_data_extend_allocation_nolock(vol);
+
+			if (unlikely(err)) {
+				ntfs_error(vol->sb, "Failed to extend mft data allocation.");
+				goto undo_mftbmp_alloc_nolock;
+			}
+			read_lock_irqsave(&mft_ni->size_lock, flags);
+			ntfs_debug("Status of mft data after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+					mft_ni->allocated_size, i_size_read(vol->mft_ino),
+					mft_ni->initialized_size);
+		}
+		read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	} else if (ll > mft_ni->allocated_size) {
+		err = -ENOSPC;
+		goto undo_mftbmp_alloc_nolock;
+	}
+	/*
+	 * Extend mft data initialized size (and data size of course) to reach
+	 * the allocated mft record, formatting the mft records allong the way.
+	 * Note: We only modify the struct ntfs_inode structure as that is all that is
+	 * needed by ntfs_mft_record_format().  We will update the attribute
+	 * record itself in one fell swoop later on.
+	 */
+	write_lock_irqsave(&mft_ni->size_lock, flags);
+	old_data_initialized = mft_ni->initialized_size;
+	old_data_size = vol->mft_ino->i_size;
+	while (ll > mft_ni->initialized_size) {
+		s64 new_initialized_size, mft_no;
+
+		new_initialized_size = mft_ni->initialized_size +
+				vol->mft_record_size;
+		mft_no = mft_ni->initialized_size >> vol->mft_record_size_bits;
+		if (new_initialized_size > i_size_read(vol->mft_ino))
+			i_size_write(vol->mft_ino, new_initialized_size);
+		write_unlock_irqrestore(&mft_ni->size_lock, flags);
+		ntfs_debug("Initializing mft record 0x%llx.",
+				(long long)mft_no);
+		err = ntfs_mft_record_format(vol, mft_no);
+		if (unlikely(err)) {
+			ntfs_error(vol->sb, "Failed to format mft record.");
+			goto undo_data_init;
+		}
+		write_lock_irqsave(&mft_ni->size_lock, flags);
+		mft_ni->initialized_size = new_initialized_size;
+	}
+	write_unlock_irqrestore(&mft_ni->size_lock, flags);
+	record_formatted = true;
+	/* Update the mft data attribute record to reflect the new sizes. */
+	m = map_mft_record(mft_ni);
+	if (IS_ERR(m)) {
+		ntfs_error(vol->sb, "Failed to map mft record.");
+		err = PTR_ERR(m);
+		goto undo_data_init;
+	}
+	ctx = ntfs_attr_get_search_ctx(mft_ni, m);
+	if (unlikely(!ctx)) {
+		ntfs_error(vol->sb, "Failed to get search context.");
+		err = -ENOMEM;
+		unmap_mft_record(mft_ni);
+		goto undo_data_init;
+	}
+	err = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
+			CASE_SENSITIVE, 0, NULL, 0, ctx);
+	if (unlikely(err)) {
+		ntfs_error(vol->sb, "Failed to find first attribute extent of mft data attribute.");
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(mft_ni);
+		goto undo_data_init;
+	}
+	a = ctx->attr;
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	a->data.non_resident.initialized_size =
+			cpu_to_le64(mft_ni->initialized_size);
+	a->data.non_resident.data_size =
+			cpu_to_le64(i_size_read(vol->mft_ino));
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+	/* Ensure the changes make it to disk. */
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(mft_ni);
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	ntfs_debug("Status of mft data after mft record initialization: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
+			mft_ni->allocated_size,	i_size_read(vol->mft_ino),
+			mft_ni->initialized_size);
+	WARN_ON(i_size_read(vol->mft_ino) > mft_ni->allocated_size);
+	WARN_ON(mft_ni->initialized_size > i_size_read(vol->mft_ino));
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+mft_rec_already_initialized:
+	/*
+	 * We can finally drop the mft bitmap lock as the mft data attribute
+	 * has been fully updated.  The only disparity left is that the
+	 * allocated mft record still needs to be marked as in use to match the
+	 * set bit in the mft bitmap but this is actually not a problem since
+	 * this mft record is not referenced from anywhere yet and the fact
+	 * that it is allocated in the mft bitmap means that no-one will try to
+	 * allocate it either.
+	 */
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		up_write(&vol->mftbmp_lock);
+	/*
+	 * We now have allocated and initialized the mft record.  Calculate the
+	 * index of and the offset within the page cache page the record is in.
+	 */
+	index = NTFS_MFT_NR_TO_PIDX(vol, bit);
+	ofs = NTFS_MFT_NR_TO_POFS(vol, bit);
+	/* Read, map, and pin the folio containing the mft record. */
+	folio = read_mapping_folio(vol->mft_ino->i_mapping, index, NULL);
+	if (IS_ERR(folio)) {
+		ntfs_error(vol->sb, "Failed to map page containing allocated mft record 0x%llx.",
+				bit);
+		err = PTR_ERR(folio);
+		goto undo_mftbmp_alloc;
+	}
+	folio_lock(folio);
+	folio_clear_uptodate(folio);
+	m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs);
+	/* If we just formatted the mft record no need to do it again. */
+	if (!record_formatted) {
+		/* Sanity check that the mft record is really not in use. */
+		if (ntfs_is_file_record(m->magic) &&
+				(m->flags & MFT_RECORD_IN_USE)) {
+			ntfs_warning(vol->sb,
+				"Mft record 0x%llx was marked free in mft bitmap but is marked used itself. Unmount and run chkdsk.",
+				bit);
+			folio_mark_uptodate(folio);
+			folio_unlock(folio);
+			kunmap_local(m);
+			folio_put(folio);
+			NVolSetErrors(vol);
+			goto search_free_rec;
+		}
+		/*
+		 * We need to (re-)format the mft record, preserving the
+		 * sequence number if it is not zero as well as the update
+		 * sequence number if it is not zero or -1 (0xffff).  This
+		 * means we do not need to care whether or not something went
+		 * wrong with the previous mft record.
+		 */
+		seq_no = m->sequence_number;
+		usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs));
+		err = ntfs_mft_record_layout(vol, bit, m);
+		if (unlikely(err)) {
+			ntfs_error(vol->sb, "Failed to layout allocated mft record 0x%llx.",
+					bit);
+			folio_mark_uptodate(folio);
+			folio_unlock(folio);
+			kunmap_local(m);
+			folio_put(folio);
+			goto undo_mftbmp_alloc;
+		}
+		if (seq_no)
+			m->sequence_number = seq_no;
+		if (usn && le16_to_cpu(usn) != 0xffff)
+			*(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = usn;
+		pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size);
+	}
+	/* Set the mft record itself in use. */
+	m->flags |= MFT_RECORD_IN_USE;
+	if (S_ISDIR(mode))
+		m->flags |= MFT_RECORD_IS_DIRECTORY;
+	folio_mark_uptodate(folio);
+	if (base_ni) {
+		struct mft_record *m_tmp;
+
+		/*
+		 * Setup the base mft record in the extent mft record.  This
+		 * completes initialization of the allocated extent mft record
+		 * and we can simply use it with map_extent_mft_record().
+		 */
+		m->base_mft_record = MK_LE_MREF(base_ni->mft_no,
+				base_ni->seq_no);
+		/*
+		 * Allocate an extent inode structure for the new mft record,
+		 * attach it to the base inode @base_ni and map, pin, and lock
+		 * its, i.e. the allocated, mft record.
+		 */
+		m_tmp = map_extent_mft_record(base_ni,
+					      MK_MREF(bit, le16_to_cpu(m->sequence_number)),
+					      ni);
+		if (IS_ERR(m_tmp)) {
+			ntfs_error(vol->sb, "Failed to map allocated extent mft record 0x%llx.",
+					bit);
+			err = PTR_ERR(m_tmp);
+			/* Set the mft record itself not in use. */
+			m->flags &= cpu_to_le16(
+					~le16_to_cpu(MFT_RECORD_IN_USE));
+			/* Make sure the mft record is written out to disk. */
+			ntfs_mft_mark_dirty(folio);
+			folio_unlock(folio);
+			kunmap_local(m);
+			folio_put(folio);
+			goto undo_mftbmp_alloc;
+		}
+
+		/*
+		 * Make sure the allocated mft record is written out to disk.
+		 * No need to set the inode dirty because the caller is going
+		 * to do that anyway after finishing with the new extent mft
+		 * record (e.g. at a minimum a new attribute will be added to
+		 * the mft record.
+		 */
+		ntfs_mft_mark_dirty(folio);
+		folio_unlock(folio);
+		/*
+		 * Need to unmap the page since map_extent_mft_record() mapped
+		 * it as well so we have it mapped twice at the moment.
+		 */
+		kunmap_local(m);
+		folio_put(folio);
+	} else {
+		/*
+		 * Manually map, pin, and lock the mft record as we already
+		 * have its page mapped and it is very easy to do.
+		 */
+		(*ni)->seq_no = le16_to_cpu(m->sequence_number);
+		/*
+		 * Make sure the allocated mft record is written out to disk.
+		 * NOTE: We do not set the ntfs inode dirty because this would
+		 * fail in ntfs_write_inode() because the inode does not have a
+		 * standard information attribute yet.  Also, there is no need
+		 * to set the inode dirty because the caller is going to do
+		 * that anyway after finishing with the new mft record (e.g. at
+		 * a minimum some new attributes will be added to the mft
+		 * record.
+		 */
+
+		(*ni)->mrec = kmalloc(vol->mft_record_size, GFP_NOFS);
+		if (!(*ni)->mrec) {
+			folio_unlock(folio);
+			kunmap_local(m);
+			folio_put(folio);
+			goto undo_mftbmp_alloc;
+		}
+
+		memcpy((*ni)->mrec, m, vol->mft_record_size);
+		post_read_mst_fixup((struct ntfs_record *)(*ni)->mrec, vol->mft_record_size);
+		ntfs_mft_mark_dirty(folio);
+		folio_unlock(folio);
+		(*ni)->folio = folio;
+		(*ni)->folio_ofs = ofs;
+		atomic_inc(&(*ni)->count);
+		/* Update the default mft allocation position. */
+		vol->mft_data_pos = bit + 1;
+	}
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		mutex_unlock(&mft_ni->mrec_lock);
+	memalloc_nofs_restore(memalloc_flags);
+
+	/*
+	 * Return the opened, allocated inode of the allocated mft record as
+	 * well as the mapped, pinned, and locked mft record.
+	 */
+	ntfs_debug("Returning opened, allocated %sinode 0x%llx.",
+			base_ni ? "extent " : "", bit);
+	(*ni)->mft_no = bit;
+	if (ni_mrec)
+		*ni_mrec = (*ni)->mrec;
+	ntfs_dec_free_mft_records(vol, 1);
+	return 0;
+undo_data_init:
+	write_lock_irqsave(&mft_ni->size_lock, flags);
+	mft_ni->initialized_size = old_data_initialized;
+	i_size_write(vol->mft_ino, old_data_size);
+	write_unlock_irqrestore(&mft_ni->size_lock, flags);
+	goto undo_mftbmp_alloc_nolock;
+undo_mftbmp_alloc:
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		down_write(&vol->mftbmp_lock);
+undo_mftbmp_alloc_nolock:
+	if (ntfs_bitmap_clear_bit(vol->mftbmp_ino, bit)) {
+		ntfs_error(vol->sb, "Failed to clear bit in mft bitmap.%s", es);
+		NVolSetErrors(vol);
+	}
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		up_write(&vol->mftbmp_lock);
+err_out:
+	if (!base_ni || base_ni->mft_no != FILE_MFT)
+		mutex_unlock(&mft_ni->mrec_lock);
+	memalloc_nofs_restore(memalloc_flags);
+	return err;
+max_err_out:
+	ntfs_warning(vol->sb,
+		"Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached.");
+	if (!base_ni || base_ni->mft_no != FILE_MFT) {
+		up_write(&vol->mftbmp_lock);
+		mutex_unlock(&mft_ni->mrec_lock);
+	}
+	memalloc_nofs_restore(memalloc_flags);
+	return -ENOSPC;
+}
+
+/*
+ * ntfs_mft_record_free - free an mft record on an ntfs volume
+ * @vol:	volume on which to free the mft record
+ * @ni:		open ntfs inode of the mft record to free
+ *
+ * Free the mft record of the open inode @ni on the mounted ntfs volume @vol.
+ * Note that this function calls ntfs_inode_close() internally and hence you
+ * cannot use the pointer @ni any more after this function returns success.
+ *
+ * On success return 0 and on error return -1 with errno set to the error code.
+ */
+int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni)
+{
+	u64 mft_no;
+	int err;
+	u16 seq_no;
+	__le16 old_seq_no;
+	struct mft_record *ni_mrec;
+	unsigned int memalloc_flags;
+	struct ntfs_inode *base_ni;
+
+	if (!vol || !ni)
+		return -EINVAL;
+
+	ntfs_debug("Entering for inode 0x%llx.\n", (long long)ni->mft_no);
+
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec))
+		return -EIO;
+
+	/* Cache the mft reference for later. */
+	mft_no = ni->mft_no;
+
+	/* Mark the mft record as not in use. */
+	ni_mrec->flags &= ~MFT_RECORD_IN_USE;
+
+	/* Increment the sequence number, skipping zero, if it is not zero. */
+	old_seq_no = ni_mrec->sequence_number;
+	seq_no = le16_to_cpu(old_seq_no);
+	if (seq_no == 0xffff)
+		seq_no = 1;
+	else if (seq_no)
+		seq_no++;
+	ni_mrec->sequence_number = cpu_to_le16(seq_no);
+
+	down_read(&NTFS_I(vol->mft_ino)->runlist.lock);
+	err = ntfs_get_block_mft_record(NTFS_I(vol->mft_ino), ni);
+	up_read(&NTFS_I(vol->mft_ino)->runlist.lock);
+	if (err) {
+		unmap_mft_record(ni);
+		return err;
+	}
+
+	/*
+	 * Set the ntfs inode dirty and write it out.  We do not need to worry
+	 * about the base inode here since whatever caused the extent mft
+	 * record to be freed is guaranteed to do it already.
+	 */
+	NInoSetDirty(ni);
+	err = write_mft_record(ni, ni_mrec, 0);
+	if (err)
+		goto sync_rollback;
+
+	if (likely(ni->nr_extents >= 0))
+		base_ni = ni;
+	else
+		base_ni = ni->ext.base_ntfs_ino;
+
+	/* Clear the bit in the $MFT/$BITMAP corresponding to this record. */
+	memalloc_flags = memalloc_nofs_save();
+	if (base_ni->mft_no != FILE_MFT)
+		down_write(&vol->mftbmp_lock);
+	err = ntfs_bitmap_clear_bit(vol->mftbmp_ino, mft_no);
+	if (base_ni->mft_no != FILE_MFT)
+		up_write(&vol->mftbmp_lock);
+	memalloc_nofs_restore(memalloc_flags);
+	if (err)
+		goto bitmap_rollback;
+
+	unmap_mft_record(ni);
+	ntfs_inc_free_mft_records(vol, 1);
+	return 0;
+
+	/* Rollback what we did... */
+bitmap_rollback:
+	memalloc_flags = memalloc_nofs_save();
+	if (base_ni->mft_no != FILE_MFT)
+		down_write(&vol->mftbmp_lock);
+	if (ntfs_bitmap_set_bit(vol->mftbmp_ino, mft_no))
+		ntfs_error(vol->sb, "ntfs_bitmap_set_bit failed in bitmap_rollback\n");
+	if (base_ni->mft_no != FILE_MFT)
+		up_write(&vol->mftbmp_lock);
+	memalloc_nofs_restore(memalloc_flags);
+sync_rollback:
+	ntfs_error(vol->sb,
+		"Eeek! Rollback failed in %s. Leaving inconsistent metadata!\n", __func__);
+	ni_mrec->flags |= MFT_RECORD_IN_USE;
+	ni_mrec->sequence_number = old_seq_no;
+	NInoSetDirty(ni);
+	write_mft_record(ni, ni_mrec, 0);
+	unmap_mft_record(ni);
+	return err;
+}
+
+static s64 lcn_from_index(struct ntfs_volume *vol, struct ntfs_inode *ni,
+		unsigned long index)
+{
+	s64 vcn;
+	s64 lcn;
+
+	vcn = ntfs_pidx_to_cluster(vol, index);
+
+	down_read(&ni->runlist.lock);
+	lcn = ntfs_attr_vcn_to_lcn_nolock(ni, vcn, false);
+	up_read(&ni->runlist.lock);
+
+	return lcn;
+}
+
+/*
+ * ntfs_write_mft_block - Write back a folio containing MFT records
+ * @folio:	The folio to write back (contains one or more MFT records)
+ * @wbc:	Writeback control structure
+ *
+ * This function is called as part of the address_space_operations
+ * .writepages implementation for the $MFT inode (or $MFTMirr).
+ * It handles writing one folio (normally 4KiB page) worth of MFT records
+ * to the underlying block device.
+ *
+ * Return: 0 on success, or -errno on error.
+ */
+static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *wbc)
+{
+	struct address_space *mapping = folio->mapping;
+	struct inode *vi = mapping->host;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct ntfs_volume *vol = ni->vol;
+	u8 *kaddr;
+	struct ntfs_inode **locked_nis __free(kfree) = kmalloc_array(PAGE_SIZE / NTFS_BLOCK_SIZE,
+							sizeof(struct ntfs_inode *), GFP_NOFS);
+	int nr_locked_nis = 0, err = 0, mft_ofs, prev_mft_ofs;
+	struct inode **ref_inos __free(kfree) = kmalloc_array(PAGE_SIZE / NTFS_BLOCK_SIZE,
+							      sizeof(struct inode *), GFP_NOFS);
+	int nr_ref_inos = 0;
+	struct bio *bio = NULL;
+	u64 mft_no;
+	struct ntfs_inode *tni;
+	s64 lcn;
+	s64 vcn = ntfs_pidx_to_cluster(vol, folio->index);
+	s64 end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size);
+	unsigned int folio_sz;
+	struct runlist_element *rl;
+	loff_t i_size = i_size_read(vi);
+
+	ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.",
+			ni->mft_no, ni->type, folio->index);
+
+	if (!locked_nis || !ref_inos)
+		return -ENOMEM;
+
+	/* We have to zero every time due to mmap-at-end-of-file. */
+	if (folio->index >= (i_size >> folio_shift(folio)))
+		/* The page straddles i_size. */
+		folio_zero_segment(folio,
+				   offset_in_folio(folio, i_size),
+				   folio_size(folio));
+
+	lcn = lcn_from_index(vol, ni, folio->index);
+	if (lcn <= LCN_HOLE) {
+		folio_start_writeback(folio);
+		folio_unlock(folio);
+		folio_end_writeback(folio);
+		return -EIO;
+	}
+
+	/* Map folio so we can access its contents. */
+	kaddr = kmap_local_folio(folio, 0);
+	/* Clear the page uptodate flag whilst the mst fixups are applied. */
+	folio_clear_uptodate(folio);
+
+	for (mft_ofs = 0; mft_ofs < PAGE_SIZE && vcn < end_vcn;
+	     mft_ofs += vol->mft_record_size) {
+		/* Get the mft record number. */
+		mft_no = (((s64)folio->index << PAGE_SHIFT) + mft_ofs) >>
+			vol->mft_record_size_bits;
+		vcn = ntfs_mft_no_to_cluster(vol, mft_no);
+		/* Check whether to write this mft record. */
+		tni = NULL;
+		if (ntfs_may_write_mft_record(vol, mft_no,
+					(struct mft_record *)(kaddr + mft_ofs),
+					&tni, &ref_inos[nr_ref_inos])) {
+			unsigned int mft_record_off = 0;
+			s64 vcn_off = vcn;
+
+			/*
+			 * Skip $MFT extent mft records and let them being written
+			 * by writeback to avioid deadlocks. the $MFT runlist
+			 * lock must be taken before $MFT extent mrec_lock is taken.
+			 */
+			if (tni && tni->nr_extents < 0 &&
+				tni->ext.base_ntfs_ino == NTFS_I(vol->mft_ino)) {
+				mutex_unlock(&tni->mrec_lock);
+				atomic_dec(&tni->count);
+				iput(vol->mft_ino);
+				continue;
+			}
+
+			/*
+			 * The record should be written.  If a locked ntfs
+			 * inode was returned, add it to the array of locked
+			 * ntfs inodes.
+			 */
+			if (tni)
+				locked_nis[nr_locked_nis++] = tni;
+			else if (ref_inos[nr_ref_inos])
+				nr_ref_inos++;
+
+			if (bio && (mft_ofs != prev_mft_ofs + vol->mft_record_size)) {
+flush_bio:
+				bio->bi_end_io = ntfs_bio_end_io;
+				submit_bio(bio);
+				bio = NULL;
+			}
+
+			if (vol->cluster_size < folio_size(folio)) {
+				down_write(&ni->runlist.lock);
+				rl = ntfs_attr_vcn_to_rl(ni, vcn_off, &lcn);
+				up_write(&ni->runlist.lock);
+				if (IS_ERR(rl) || lcn < 0) {
+					err = -EIO;
+					goto unm_done;
+				}
+
+				if (bio &&
+				   (bio_end_sector(bio) >> (vol->cluster_size_bits - 9)) !=
+				    lcn) {
+					bio->bi_end_io = ntfs_bio_end_io;
+					submit_bio(bio);
+					bio = NULL;
+				}
+			}
+
+			if (!bio) {
+				unsigned int off;
+
+				off = ((mft_no << vol->mft_record_size_bits) +
+				       mft_record_off) & vol->cluster_size_mask;
+
+				bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE,
+						GFP_NOIO);
+				bio->bi_iter.bi_sector =
+					ntfs_bytes_to_sector(vol,
+							ntfs_cluster_to_bytes(vol, lcn) + off);
+			}
+
+			if (vol->cluster_size == NTFS_BLOCK_SIZE &&
+			    (mft_record_off ||
+			     rl->length - (vcn_off - rl->vcn) == 1 ||
+			     mft_ofs + NTFS_BLOCK_SIZE >= PAGE_SIZE))
+				folio_sz = NTFS_BLOCK_SIZE;
+			else
+				folio_sz = vol->mft_record_size;
+			if (!bio_add_folio(bio, folio, folio_sz,
+					   mft_ofs + mft_record_off)) {
+				err = -EIO;
+				bio_put(bio);
+				goto unm_done;
+			}
+			mft_record_off += folio_sz;
+
+			if (mft_record_off != vol->mft_record_size) {
+				vcn_off++;
+				goto flush_bio;
+			}
+			prev_mft_ofs = mft_ofs;
+
+			if (mft_no < vol->mftmirr_size)
+				ntfs_sync_mft_mirror(vol, mft_no,
+						(struct mft_record *)(kaddr + mft_ofs));
+		} else if (ref_inos[nr_ref_inos])
+			nr_ref_inos++;
+	}
+
+	if (bio) {
+		bio->bi_end_io = ntfs_bio_end_io;
+		submit_bio(bio);
+	}
+unm_done:
+	folio_mark_uptodate(folio);
+	kunmap_local(kaddr);
+
+	folio_start_writeback(folio);
+	folio_unlock(folio);
+	folio_end_writeback(folio);
+
+	/* Unlock any locked inodes. */
+	while (nr_locked_nis-- > 0) {
+		struct ntfs_inode *base_tni;
+
+		tni = locked_nis[nr_locked_nis];
+		mutex_unlock(&tni->mrec_lock);
+
+		/* Get the base inode. */
+		mutex_lock(&tni->extent_lock);
+		if (tni->nr_extents >= 0)
+			base_tni = tni;
+		else
+			base_tni = tni->ext.base_ntfs_ino;
+		mutex_unlock(&tni->extent_lock);
+		ntfs_debug("Unlocking %s inode 0x%llx.",
+				tni == base_tni ? "base" : "extent",
+				tni->mft_no);
+		atomic_dec(&tni->count);
+		iput(VFS_I(base_tni));
+	}
+
+	/* Dropping deferred references */
+	while (nr_ref_inos-- > 0) {
+		if (ref_inos[nr_ref_inos])
+			iput(ref_inos[nr_ref_inos]);
+	}
+
+	if (unlikely(err && err != -ENOMEM))
+		NVolSetErrors(vol);
+	if (likely(!err))
+		ntfs_debug("Done.");
+	return err;
+}
+
+/*
+ * ntfs_mft_writepages - Write back dirty folios for the $MFT inode
+ * @mapping:	address space of the $MFT inode
+ * @wbc:	writeback control
+ *
+ * Writeback iterator for MFT records. Iterates over dirty folios and
+ * delegates actual writing to ntfs_write_mft_block() for each folio.
+ * Called from the address_space_operations .writepages vector of the
+ * $MFT inode.
+ *
+ * Returns 0 on success, or the first error encountered.
+ */
+int ntfs_mft_writepages(struct address_space *mapping,
+		struct writeback_control *wbc)
+{
+	struct folio *folio = NULL;
+	int error;
+
+	if (NVolShutdown(NTFS_I(mapping->host)->vol))
+		return -EIO;
+
+	while ((folio = writeback_iter(mapping, wbc, folio, &error)))
+		error = ntfs_write_mft_block(folio, wbc);
+	return error;
+}
+
+void ntfs_mft_mark_dirty(struct folio *folio)
+{
+	iomap_dirty_folio(folio->mapping, folio);
+}
diff --git a/fs/ntfs/mft.h b/fs/ntfs/mft.h
new file mode 100644
index 0000000000000..75a51a98d0f6e
--- /dev/null
+++ b/fs/ntfs/mft.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for mft record handling in NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_MFT_H
+#define _LINUX_NTFS_MFT_H
+
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
+
+#include "inode.h"
+
+struct mft_record *map_mft_record(struct ntfs_inode *ni);
+void unmap_mft_record(struct ntfs_inode *ni);
+struct mft_record *map_extent_mft_record(struct ntfs_inode *base_ni, u64 mref,
+		struct ntfs_inode **ntfs_ino);
+
+static inline void unmap_extent_mft_record(struct ntfs_inode *ni)
+{
+	unmap_mft_record(ni);
+}
+
+void __mark_mft_record_dirty(struct ntfs_inode *ni);
+
+/*
+ * mark_mft_record_dirty - set the mft record and the page containing it dirty
+ * @ni:		ntfs inode describing the mapped mft record
+ *
+ * Set the mapped (extent) mft record of the (base or extent) ntfs inode @ni,
+ * as well as the page containing the mft record, dirty.  Also, mark the base
+ * vfs inode dirty.  This ensures that any changes to the mft record are
+ * written out to disk.
+ *
+ * NOTE:  Do not do anything if the mft record is already marked dirty.
+ */
+static inline void mark_mft_record_dirty(struct ntfs_inode *ni)
+{
+	if (!NInoTestSetDirty(ni))
+		__mark_mft_record_dirty(ni);
+}
+
+int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
+		struct mft_record *m);
+int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int sync);
+
+/*
+ * write_mft_record - write out a mapped (extent) mft record
+ * @ni:		ntfs inode describing the mapped (extent) mft record
+ * @m:		mapped (extent) mft record to write
+ * @sync:	if true, wait for i/o completion
+ *
+ * This is just a wrapper for write_mft_record_nolock() (see mft.c), which
+ * locks the page for the duration of the write.  This ensures that there are
+ * no race conditions between writing the mft record via the dirty inode code
+ * paths and via the page cache write back code paths or between writing
+ * neighbouring mft records residing in the same page.
+ *
+ * Locking the page also serializes us against ->read_folio() if the page is not
+ * uptodate.
+ *
+ * On success, clean the mft record and return 0.  On error, leave the mft
+ * record dirty and return -errno.
+ */
+static inline int write_mft_record(struct ntfs_inode *ni, struct mft_record *m, int sync)
+{
+	struct folio *folio = ni->folio;
+	int err;
+
+	folio_lock(folio);
+	err = write_mft_record_nolock(ni, m, sync);
+	folio_unlock(folio);
+
+	return err;
+}
+
+int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode,
+		struct ntfs_inode **ni, struct ntfs_inode *base_ni,
+		struct mft_record **ni_mrec);
+int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni);
+int ntfs_mft_records_write(const struct ntfs_volume *vol, const u64 mref,
+		const s64 count, struct mft_record *b);
+int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
+			  u64 mft_no);
+int ntfs_mft_writepages(struct address_space *mapping,
+		struct writeback_control *wbc);
+void ntfs_mft_mark_dirty(struct folio *folio);
+
+#endif /* _LINUX_NTFS_MFT_H */
diff --git a/fs/ntfs/mst.c b/fs/ntfs/mst.c
new file mode 100644
index 0000000000000..7f9faad924adf
--- /dev/null
+++ b/fs/ntfs/mst.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS multi sector transfer protection handling code.
+ *
+ * Copyright (c) 2001-2004 Anton Altaparmakov
+ */
+
+#include <linux/ratelimit.h>
+
+#include "ntfs.h"
+
+/*
+ * post_read_mst_fixup - deprotect multi sector transfer protected data
+ * @b:		pointer to the data to deprotect
+ * @size:	size in bytes of @b
+ *
+ * Perform the necessary post read multi sector transfer fixup and detect the
+ * presence of incomplete multi sector transfers. - In that case, overwrite the
+ * magic of the ntfs record header being processed with "BAAD" (in memory only!)
+ * and abort processing.
+ *
+ * Return 0 on success and -EINVAL on error ("BAAD" magic will be present).
+ *
+ * NOTE: We consider the absence / invalidity of an update sequence array to
+ * mean that the structure is not protected at all and hence doesn't need to
+ * be fixed up. Thus, we return success and not failure in this case. This is
+ * in contrast to pre_write_mst_fixup(), see below.
+ */
+int post_read_mst_fixup(struct ntfs_record *b, const u32 size)
+{
+	u16 usa_ofs, usa_count, usn;
+	u16 *usa_pos, *data_pos;
+
+	/* Setup the variables. */
+	usa_ofs = le16_to_cpu(b->usa_ofs);
+	/* Decrement usa_count to get number of fixups. */
+	usa_count = le16_to_cpu(b->usa_count) - 1;
+	/* Size and alignment checks. */
+	if (size & (NTFS_BLOCK_SIZE - 1) || usa_ofs & 1	||
+	    usa_ofs + (usa_count * 2) > size ||
+	    (size >> NTFS_BLOCK_SIZE_BITS) != usa_count)
+		return 0;
+	/* Position of usn in update sequence array. */
+	usa_pos = (u16 *)b + usa_ofs/sizeof(u16);
+	/*
+	 * The update sequence number which has to be equal to each of the
+	 * u16 values before they are fixed up. Note no need to care for
+	 * endianness since we are comparing and moving data for on disk
+	 * structures which means the data is consistent. - If it is
+	 * consistenty the wrong endianness it doesn't make any difference.
+	 */
+	usn = *usa_pos;
+	/*
+	 * Position in protected data of first u16 that needs fixing up.
+	 */
+	data_pos = (u16 *)b + NTFS_BLOCK_SIZE / sizeof(u16) - 1;
+	/*
+	 * Check for incomplete multi sector transfer(s).
+	 */
+	while (usa_count--) {
+		if (*data_pos != usn) {
+			struct mft_record *m = (struct mft_record *)b;
+
+			pr_err_ratelimited("ntfs: Incomplete multi sector transfer detected! (Record magic : 0x%x, mft number : 0x%x, base mft number : 0x%lx, mft in use : %d, data : 0x%x, usn 0x%x)\n",
+					le32_to_cpu(m->magic), le32_to_cpu(m->mft_record_number),
+					MREF_LE(m->base_mft_record), m->flags & MFT_RECORD_IN_USE,
+					*data_pos, usn);
+			/*
+			 * Incomplete multi sector transfer detected! )-:
+			 * Set the magic to "BAAD" and return failure.
+			 * Note that magic_BAAD is already converted to le32.
+			 */
+			b->magic = magic_BAAD;
+			return -EINVAL;
+		}
+		data_pos += NTFS_BLOCK_SIZE / sizeof(u16);
+	}
+	/* Re-setup the variables. */
+	usa_count = le16_to_cpu(b->usa_count) - 1;
+	data_pos = (u16 *)b + NTFS_BLOCK_SIZE / sizeof(u16) - 1;
+	/* Fixup all sectors. */
+	while (usa_count--) {
+		/*
+		 * Increment position in usa and restore original data from
+		 * the usa into the data buffer.
+		 */
+		*data_pos = *(++usa_pos);
+		/* Increment position in data as well. */
+		data_pos += NTFS_BLOCK_SIZE/sizeof(u16);
+	}
+	return 0;
+}
+
+/*
+ * pre_write_mst_fixup - apply multi sector transfer protection
+ * @b:		pointer to the data to protect
+ * @size:	size in bytes of @b
+ *
+ * Perform the necessary pre write multi sector transfer fixup on the data
+ * pointer to by @b of @size.
+ *
+ * Return 0 if fixup applied (success) or -EINVAL if no fixup was performed
+ * (assumed not needed). This is in contrast to post_read_mst_fixup() above.
+ *
+ * NOTE: We consider the absence / invalidity of an update sequence array to
+ * mean that the structure is not subject to protection and hence doesn't need
+ * to be fixed up. This means that you have to create a valid update sequence
+ * array header in the ntfs record before calling this function, otherwise it
+ * will fail (the header needs to contain the position of the update sequence
+ * array together with the number of elements in the array). You also need to
+ * initialise the update sequence number before calling this function
+ * otherwise a random word will be used (whatever was in the record at that
+ * position at that time).
+ */
+int pre_write_mst_fixup(struct ntfs_record *b, const u32 size)
+{
+	__le16 *usa_pos, *data_pos;
+	u16 usa_ofs, usa_count, usn;
+	__le16 le_usn;
+
+	/* Sanity check + only fixup if it makes sense. */
+	if (!b || ntfs_is_baad_record(b->magic) ||
+	    ntfs_is_hole_record(b->magic))
+		return -EINVAL;
+	/* Setup the variables. */
+	usa_ofs = le16_to_cpu(b->usa_ofs);
+	/* Decrement usa_count to get number of fixups. */
+	usa_count = le16_to_cpu(b->usa_count) - 1;
+	/* Size and alignment checks. */
+	if (size & (NTFS_BLOCK_SIZE - 1) || usa_ofs & 1	||
+	    usa_ofs + (usa_count * 2) > size ||
+	    (size >> NTFS_BLOCK_SIZE_BITS) != usa_count)
+		return -EINVAL;
+	/* Position of usn in update sequence array. */
+	usa_pos = (__le16 *)((u8 *)b + usa_ofs);
+	/*
+	 * Cyclically increment the update sequence number
+	 * (skipping 0 and -1, i.e. 0xffff).
+	 */
+	usn = le16_to_cpup(usa_pos) + 1;
+	if (usn == 0xffff || !usn)
+		usn = 1;
+	le_usn = cpu_to_le16(usn);
+	*usa_pos = le_usn;
+	/* Position in data of first u16 that needs fixing up. */
+	data_pos = (__le16 *)b + NTFS_BLOCK_SIZE/sizeof(__le16) - 1;
+	/* Fixup all sectors. */
+	while (usa_count--) {
+		/*
+		 * Increment the position in the usa and save the
+		 * original data from the data buffer into the usa.
+		 */
+		*(++usa_pos) = *data_pos;
+		/* Apply fixup to data. */
+		*data_pos = le_usn;
+		/* Increment position in data as well. */
+		data_pos += NTFS_BLOCK_SIZE / sizeof(__le16);
+	}
+	return 0;
+}
+
+/*
+ * post_write_mst_fixup - fast deprotect multi sector transfer protected data
+ * @b:		pointer to the data to deprotect
+ *
+ * Perform the necessary post write multi sector transfer fixup, not checking
+ * for any errors, because we assume we have just used pre_write_mst_fixup(),
+ * thus the data will be fine or we would never have gotten here.
+ */
+void post_write_mst_fixup(struct ntfs_record *b)
+{
+	__le16 *usa_pos, *data_pos;
+
+	u16 usa_ofs = le16_to_cpu(b->usa_ofs);
+	u16 usa_count = le16_to_cpu(b->usa_count) - 1;
+
+	/* Position of usn in update sequence array. */
+	usa_pos = (__le16 *)b + usa_ofs/sizeof(__le16);
+
+	/* Position in protected data of first u16 that needs fixing up. */
+	data_pos = (__le16 *)b + NTFS_BLOCK_SIZE/sizeof(__le16) - 1;
+
+	/* Fixup all sectors. */
+	while (usa_count--) {
+		/*
+		 * Increment position in usa and restore original data from
+		 * the usa into the data buffer.
+		 */
+		*data_pos = *(++usa_pos);
+
+		/* Increment position in data as well. */
+		data_pos += NTFS_BLOCK_SIZE/sizeof(__le16);
+	}
+}
diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c
new file mode 100644
index 0000000000000..ba42c566940a6
--- /dev/null
+++ b/fs/ntfs/namei.c
@@ -0,0 +1,1692 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel directory inode operations.
+ *
+ * Copyright (c) 2001-2006 Anton Altaparmakov
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/exportfs.h>
+#include <linux/iversion.h>
+
+#include "ntfs.h"
+#include "time.h"
+#include "index.h"
+#include "reparse.h"
+#include "object_id.h"
+#include "ea.h"
+
+static const __le16 aux_name_le[3] = {
+	cpu_to_le16('A'), cpu_to_le16('U'), cpu_to_le16('X')
+};
+
+static const __le16 con_name_le[3] = {
+	cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('N')
+};
+
+static const __le16 com_name_le[3] = {
+	cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M')
+};
+
+static const __le16 lpt_name_le[3] = {
+	cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T')
+};
+
+static const __le16 nul_name_le[3] = {
+	cpu_to_le16('N'), cpu_to_le16('U'), cpu_to_le16('L')
+};
+
+static const __le16 prn_name_le[3] = {
+	cpu_to_le16('P'), cpu_to_le16('R'), cpu_to_le16('N')
+};
+
+static inline int ntfs_check_bad_char(const __le16 *wc, unsigned int wc_len)
+{
+	int i;
+
+	for (i = 0; i < wc_len; i++) {
+		u16 c = le16_to_cpu(wc[i]);
+
+		if (c < 0x0020 ||
+		    c == 0x0022 || c == 0x002A || c == 0x002F ||
+		    c == 0x003A || c == 0x003C || c == 0x003E ||
+		    c == 0x003F || c == 0x005C || c == 0x007C)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ntfs_check_bad_windows_name(struct ntfs_volume *vol,
+				       const __le16 *wc,
+				       unsigned int wc_len)
+{
+	if (ntfs_check_bad_char(wc, wc_len))
+		return -EINVAL;
+
+	if (!NVolCheckWindowsNames(vol))
+		return 0;
+
+	/* Check for trailing space or dot. */
+	if (wc_len > 0 &&
+	    (wc[wc_len - 1] == cpu_to_le16(' ') ||
+	    wc[wc_len - 1] == cpu_to_le16('.')))
+		return -EINVAL;
+
+	if (wc_len == 3 || (wc_len > 3 && wc[3] == cpu_to_le16('.'))) {
+		__le16 *upcase = vol->upcase;
+		u32 size = vol->upcase_len;
+
+		if (ntfs_are_names_equal(wc, 3, aux_name_le, 3, IGNORE_CASE, upcase, size) ||
+		    ntfs_are_names_equal(wc, 3, con_name_le, 3, IGNORE_CASE, upcase, size) ||
+		    ntfs_are_names_equal(wc, 3, nul_name_le, 3, IGNORE_CASE, upcase, size) ||
+		    ntfs_are_names_equal(wc, 3, prn_name_le, 3, IGNORE_CASE, upcase, size))
+			return -EINVAL;
+	}
+
+	if (wc_len == 4 || (wc_len > 4 && wc[4] == cpu_to_le16('.'))) {
+		__le16 *upcase = vol->upcase;
+		u32 size = vol->upcase_len, port;
+
+		if (ntfs_are_names_equal(wc, 3, com_name_le, 3, IGNORE_CASE, upcase, size) ||
+		    ntfs_are_names_equal(wc, 3, lpt_name_le, 3, IGNORE_CASE, upcase, size)) {
+			port = le16_to_cpu(wc[3]);
+			if (port >= '1' && port <= '9')
+				return -EINVAL;
+		}
+	}
+	return 0;
+}
+
+/*
+ * ntfs_lookup - find the inode represented by a dentry in a directory inode
+ * @dir_ino:	directory inode in which to look for the inode
+ * @dent:	dentry representing the inode to look for
+ * @flags:	lookup flags
+ *
+ * In short, ntfs_lookup() looks for the inode represented by the dentry @dent
+ * in the directory inode @dir_ino and if found attaches the inode to the
+ * dentry @dent.
+ *
+ * In more detail, the dentry @dent specifies which inode to look for by
+ * supplying the name of the inode in @dent->d_name.name. ntfs_lookup()
+ * converts the name to Unicode and walks the contents of the directory inode
+ * @dir_ino looking for the converted Unicode name. If the name is found in the
+ * directory, the corresponding inode is loaded by calling ntfs_iget() on its
+ * inode number and the inode is associated with the dentry @dent via a call to
+ * d_splice_alias().
+ *
+ * If the name is not found in the directory, a NULL inode is inserted into the
+ * dentry @dent via a call to d_add(). The dentry is then termed a negative
+ * dentry.
+ *
+ * Only if an actual error occurs, do we return an error via ERR_PTR().
+ *
+ * In order to handle the case insensitivity issues of NTFS with regards to the
+ * dcache and the dcache requiring only one dentry per directory, we deal with
+ * dentry aliases that only differ in case in ->ntfs_lookup() while maintaining
+ * a case sensitive dcache. This means that we get the full benefit of dcache
+ * speed when the file/directory is looked up with the same case as returned by
+ * ->ntfs_readdir() but that a lookup for any other case (or for the short file
+ * name) will not find anything in dcache and will enter ->ntfs_lookup()
+ * instead, where we search the directory for a fully matching file name
+ * (including case) and if that is not found, we search for a file name that
+ * matches with different case and if that has non-POSIX semantics we return
+ * that. We actually do only one search (case sensitive) and keep tabs on
+ * whether we have found a case insensitive match in the process.
+ *
+ * To simplify matters for us, we do not treat the short vs long filenames as
+ * two hard links but instead if the lookup matches a short filename, we
+ * return the dentry for the corresponding long filename instead.
+ *
+ * There are three cases we need to distinguish here:
+ *
+ * 1) @dent perfectly matches (i.e. including case) a directory entry with a
+ *    file name in the WIN32 or POSIX namespaces. In this case
+ *    ntfs_lookup_inode_by_name() will return with name set to NULL and we
+ *    just d_splice_alias() @dent.
+ * 2) @dent matches (not including case) a directory entry with a file name in
+ *    the WIN32 namespace. In this case ntfs_lookup_inode_by_name() will return
+ *    with name set to point to a kmalloc()ed ntfs_name structure containing
+ *    the properly cased little endian Unicode name. We convert the name to the
+ *    current NLS code page, search if a dentry with this name already exists
+ *    and if so return that instead of @dent.  At this point things are
+ *    complicated by the possibility of 'disconnected' dentries due to NFS
+ *    which we deal with appropriately (see the code comments).  The VFS will
+ *    then destroy the old @dent and use the one we returned.  If a dentry is
+ *    not found, we allocate a new one, d_splice_alias() it, and return it as
+ *    above.
+ * 3) @dent matches either perfectly or not (i.e. we don't care about case) a
+ *    directory entry with a file name in the DOS namespace. In this case
+ *    ntfs_lookup_inode_by_name() will return with name set to point to a
+ *    kmalloc()ed ntfs_name structure containing the mft reference (cpu endian)
+ *    of the inode. We use the mft reference to read the inode and to find the
+ *    file name in the WIN32 namespace corresponding to the matched short file
+ *    name. We then convert the name to the current NLS code page, and proceed
+ *    searching for a dentry with this name, etc, as in case 2), above.
+ *
+ * Locking: Caller must hold i_mutex on the directory.
+ */
+static struct dentry *ntfs_lookup(struct inode *dir_ino, struct dentry *dent,
+		unsigned int flags)
+{
+	struct ntfs_volume *vol = NTFS_SB(dir_ino->i_sb);
+	struct inode *dent_inode;
+	__le16 *uname;
+	struct ntfs_name *name = NULL;
+	u64 mref;
+	unsigned long dent_ino;
+	int uname_len;
+
+	ntfs_debug("Looking up %pd in directory inode 0x%llx.",
+			dent, NTFS_I(dir_ino)->mft_no);
+	/* Convert the name of the dentry to Unicode. */
+	uname_len = ntfs_nlstoucs(vol, dent->d_name.name, dent->d_name.len,
+				  &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_debug("Failed to convert name to Unicode.");
+		return ERR_PTR(uname_len);
+	}
+	mutex_lock(&NTFS_I(dir_ino)->mrec_lock);
+	mref = ntfs_lookup_inode_by_name(NTFS_I(dir_ino), uname, uname_len,
+			&name);
+	mutex_unlock(&NTFS_I(dir_ino)->mrec_lock);
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (!IS_ERR_MREF(mref)) {
+		dent_ino = MREF(mref);
+		ntfs_debug("Found inode 0x%lx. Calling ntfs_iget.", dent_ino);
+		dent_inode = ntfs_iget(vol->sb, dent_ino);
+		if (!IS_ERR(dent_inode)) {
+			/* Consistency check. */
+			if (MSEQNO(mref) == NTFS_I(dent_inode)->seq_no ||
+			    dent_ino == FILE_MFT) {
+				/* Perfect WIN32/POSIX match. -- Case 1. */
+				if (!name) {
+					ntfs_debug("Done.  (Case 1.)");
+					return d_splice_alias(dent_inode, dent);
+				}
+				/*
+				 * We are too indented.  Handle imperfect
+				 * matches and short file names further below.
+				 */
+				goto handle_name;
+			}
+			ntfs_error(vol->sb,
+				"Found stale reference to inode 0x%lx (reference sequence number = 0x%x, inode sequence number = 0x%x), returning -EIO. Run chkdsk.",
+				dent_ino, MSEQNO(mref),
+				NTFS_I(dent_inode)->seq_no);
+			iput(dent_inode);
+			dent_inode = ERR_PTR(-EIO);
+		} else
+			ntfs_error(vol->sb, "ntfs_iget(0x%lx) failed with error code %li.",
+					dent_ino, PTR_ERR(dent_inode));
+		kfree(name);
+		/* Return the error code. */
+		return ERR_CAST(dent_inode);
+	}
+	kfree(name);
+	/* It is guaranteed that @name is no longer allocated at this point. */
+	if (MREF_ERR(mref) == -ENOENT) {
+		ntfs_debug("Entry was not found, adding negative dentry.");
+		/* The dcache will handle negative entries. */
+		d_add(dent, NULL);
+		ntfs_debug("Done.");
+		return NULL;
+	}
+	ntfs_error(vol->sb, "ntfs_lookup_ino_by_name() failed with error code %i.",
+			-MREF_ERR(mref));
+	return ERR_PTR(MREF_ERR(mref));
+handle_name:
+	{
+		struct mft_record *m;
+		struct ntfs_attr_search_ctx *ctx;
+		struct ntfs_inode *ni = NTFS_I(dent_inode);
+		int err;
+		struct qstr nls_name;
+
+		nls_name.name = NULL;
+		if (name->type != FILE_NAME_DOS) {			/* Case 2. */
+			ntfs_debug("Case 2.");
+			nls_name.len = (unsigned int)ntfs_ucstonls(vol,
+					(__le16 *)&name->name, name->len,
+					(unsigned char **)&nls_name.name, 0);
+			kfree(name);
+		} else /* if (name->type == FILE_NAME_DOS) */ {		/* Case 3. */
+			struct file_name_attr *fn;
+
+			ntfs_debug("Case 3.");
+			kfree(name);
+
+			/* Find the WIN32 name corresponding to the matched DOS name. */
+			ni = NTFS_I(dent_inode);
+			m = map_mft_record(ni);
+			if (IS_ERR(m)) {
+				err = PTR_ERR(m);
+				m = NULL;
+				ctx = NULL;
+				goto err_out;
+			}
+			ctx = ntfs_attr_get_search_ctx(ni, m);
+			if (unlikely(!ctx)) {
+				err = -ENOMEM;
+				goto err_out;
+			}
+			do {
+				struct attr_record *a;
+				u32 val_len;
+
+				err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0,
+						NULL, 0, ctx);
+				if (unlikely(err)) {
+					ntfs_error(vol->sb,
+						"Inode corrupt: No WIN32 namespace counterpart to DOS file name. Run chkdsk.");
+					if (err == -ENOENT)
+						err = -EIO;
+					goto err_out;
+				}
+				/* Consistency checks. */
+				a = ctx->attr;
+				if (a->non_resident || a->flags)
+					goto eio_err_out;
+				val_len = le32_to_cpu(a->data.resident.value_length);
+				if (le16_to_cpu(a->data.resident.value_offset) +
+						val_len > le32_to_cpu(a->length))
+					goto eio_err_out;
+				fn = (struct file_name_attr *)((u8 *)ctx->attr + le16_to_cpu(
+							ctx->attr->data.resident.value_offset));
+				if ((u32)(fn->file_name_length * sizeof(__le16) +
+							sizeof(struct file_name_attr)) > val_len)
+					goto eio_err_out;
+			} while (fn->file_name_type != FILE_NAME_WIN32);
+
+			/* Convert the found WIN32 name to current NLS code page. */
+			nls_name.len = (unsigned int)ntfs_ucstonls(vol,
+					(__le16 *)&fn->file_name, fn->file_name_length,
+					(unsigned char **)&nls_name.name, 0);
+
+			ntfs_attr_put_search_ctx(ctx);
+			unmap_mft_record(ni);
+		}
+		m = NULL;
+		ctx = NULL;
+
+		/* Check if a conversion error occurred. */
+		if ((int)nls_name.len < 0) {
+			err = (int)nls_name.len;
+			goto err_out;
+		}
+		nls_name.hash = full_name_hash(dent, nls_name.name, nls_name.len);
+
+		dent = d_add_ci(dent, dent_inode, &nls_name);
+		kfree(nls_name.name);
+		return dent;
+
+eio_err_out:
+		ntfs_error(vol->sb, "Illegal file name attribute. Run chkdsk.");
+		err = -EIO;
+err_out:
+		if (ctx)
+			ntfs_attr_put_search_ctx(ctx);
+		if (m)
+			unmap_mft_record(ni);
+		iput(dent_inode);
+		ntfs_error(vol->sb, "Failed, returning error code %i.", err);
+		return ERR_PTR(err);
+	}
+}
+
+static int ntfs_sd_add_everyone(struct ntfs_inode *ni)
+{
+	struct security_descriptor_relative *sd;
+	struct ntfs_acl *acl;
+	struct ntfs_ace *ace;
+	struct ntfs_sid *sid;
+	int ret, sd_len;
+
+	/* Create SECURITY_DESCRIPTOR attribute (everyone has full access). */
+	/*
+	 * Calculate security descriptor length. We have 2 sub-authorities in
+	 * owner and group SIDs, So add 8 bytes to every SID.
+	 */
+	sd_len = sizeof(struct security_descriptor_relative) + 2 *
+		(sizeof(struct ntfs_sid) + 8) + sizeof(struct ntfs_acl) +
+		sizeof(struct ntfs_ace) + 4;
+	sd = kmalloc(sd_len, GFP_NOFS);
+	if (!sd)
+		return -1;
+
+	sd->revision = 1;
+	sd->control = SE_DACL_PRESENT | SE_SELF_RELATIVE;
+
+	sid = (struct ntfs_sid *)((u8 *)sd + sizeof(struct security_descriptor_relative));
+	sid->revision = 1;
+	sid->sub_authority_count = 2;
+	sid->sub_authority[0] = cpu_to_le32(SECURITY_BUILTIN_DOMAIN_RID);
+	sid->sub_authority[1] = cpu_to_le32(DOMAIN_ALIAS_RID_ADMINS);
+	sid->identifier_authority.value[5] = 5;
+	sd->owner = cpu_to_le32((u8 *)sid - (u8 *)sd);
+
+	sid = (struct ntfs_sid *)((u8 *)sid + sizeof(struct ntfs_sid) + 8);
+	sid->revision = 1;
+	sid->sub_authority_count = 2;
+	sid->sub_authority[0] = cpu_to_le32(SECURITY_BUILTIN_DOMAIN_RID);
+	sid->sub_authority[1] = cpu_to_le32(DOMAIN_ALIAS_RID_ADMINS);
+	sid->identifier_authority.value[5] = 5;
+	sd->group = cpu_to_le32((u8 *)sid - (u8 *)sd);
+
+	acl = (struct ntfs_acl *)((u8 *)sid + sizeof(struct ntfs_sid) + 8);
+	acl->revision = 2;
+	acl->size = cpu_to_le16(sizeof(struct ntfs_acl) + sizeof(struct ntfs_ace) + 4);
+	acl->ace_count = cpu_to_le16(1);
+	sd->dacl = cpu_to_le32((u8 *)acl - (u8 *)sd);
+
+	ace = (struct ntfs_ace *)((u8 *)acl + sizeof(struct ntfs_acl));
+	ace->type = ACCESS_ALLOWED_ACE_TYPE;
+	ace->flags = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE;
+	ace->size = cpu_to_le16(sizeof(struct ntfs_ace) + 4);
+	ace->mask = cpu_to_le32(0x1f01ff);
+	ace->sid.revision = 1;
+	ace->sid.sub_authority_count = 1;
+	ace->sid.sub_authority[0] = 0;
+	ace->sid.identifier_authority.value[5] = 1;
+
+	ret = ntfs_attr_add(ni, AT_SECURITY_DESCRIPTOR, AT_UNNAMED, 0, (u8 *)sd,
+			sd_len);
+	if (ret)
+		ntfs_error(ni->vol->sb, "Failed to add SECURITY_DESCRIPTOR\n");
+
+	kfree(sd);
+	return ret;
+}
+
+static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
+		__le16 *name, u8 name_len, mode_t mode, dev_t dev,
+		__le16 *target, int target_len)
+{
+	struct ntfs_inode *dir_ni = NTFS_I(dir);
+	struct ntfs_volume *vol = dir_ni->vol;
+	struct ntfs_inode *ni;
+	bool rollback_data = false, rollback_sd = false, rollback_reparse = false;
+	struct file_name_attr *fn = NULL;
+	struct standard_information *si = NULL;
+	int err = 0, fn_len, si_len;
+	struct inode *vi;
+	struct mft_record *ni_mrec, *dni_mrec;
+	struct super_block *sb = dir_ni->vol->sb;
+	__le64 parent_mft_ref;
+	u64 child_mft_ref;
+	__le16 ea_size;
+
+	vi = new_inode(vol->sb);
+	if (!vi)
+		return ERR_PTR(-ENOMEM);
+
+	ntfs_init_big_inode(vi);
+	ni = NTFS_I(vi);
+	ni->vol = dir_ni->vol;
+	ni->name_len = 0;
+	ni->name = NULL;
+
+	/*
+	 * Set the appropriate mode, attribute type, and name.  For
+	 * directories, also setup the index values to the defaults.
+	 */
+	if (S_ISDIR(mode)) {
+		mode &= ~vol->dmask;
+
+		NInoSetMstProtected(ni);
+		ni->itype.index.block_size = 4096;
+		ni->itype.index.block_size_bits = ntfs_ffs(4096) - 1;
+		ni->itype.index.collation_rule = COLLATION_FILE_NAME;
+		if (vol->cluster_size <= ni->itype.index.block_size) {
+			ni->itype.index.vcn_size = vol->cluster_size;
+			ni->itype.index.vcn_size_bits =
+				vol->cluster_size_bits;
+		} else {
+			ni->itype.index.vcn_size = vol->sector_size;
+			ni->itype.index.vcn_size_bits =
+				vol->sector_size_bits;
+		}
+	} else {
+		mode &= ~vol->fmask;
+	}
+
+	if (IS_RDONLY(vi))
+		mode &= ~0222;
+
+	inode_init_owner(idmap, vi, dir, mode);
+
+	mode = vi->i_mode;
+
+#ifdef CONFIG_NTFS_FS_POSIX_ACL
+	if (!S_ISLNK(mode) && (sb->s_flags & SB_POSIXACL)) {
+		err = ntfs_init_acl(idmap, vi, dir);
+		if (err)
+			goto err_out;
+	} else
+#endif
+	{
+		vi->i_flags |= S_NOSEC;
+	}
+
+	if (uid_valid(vol->uid))
+		vi->i_uid = vol->uid;
+
+	if (gid_valid(vol->gid))
+		vi->i_gid = vol->gid;
+
+	/*
+	 * Set the file size to 0, the ntfs inode sizes are set to 0 by
+	 * the call to ntfs_init_big_inode() below.
+	 */
+	vi->i_size = 0;
+	vi->i_blocks = 0;
+
+	inode_inc_iversion(vi);
+
+	simple_inode_init_ts(vi);
+	ni->i_crtime = inode_get_ctime(vi);
+
+	inode_set_mtime_to_ts(dir, ni->i_crtime);
+	inode_set_ctime_to_ts(dir, ni->i_crtime);
+	mark_inode_dirty(dir);
+
+	err = ntfs_mft_record_alloc(dir_ni->vol, mode, &ni, NULL,
+				    &ni_mrec);
+	if (err) {
+		iput(vi);
+		return ERR_PTR(err);
+	}
+
+	/*
+	 * Prevent iget and writeback from finding this inode.
+	 * Caller must call d_instantiate_new instead of d_instantiate.
+	 */
+	spin_lock(&vi->i_lock);
+	inode_state_set(vi, I_NEW | I_CREATING);
+	spin_unlock(&vi->i_lock);
+
+	/* Add the inode to the inode hash for the superblock. */
+	vi->i_ino = (unsigned long)ni->mft_no;
+	inode_set_iversion(vi, 1);
+	insert_inode_hash(vi);
+
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+	if (NInoBeingDeleted(dir_ni)) {
+		err = -ENOENT;
+		goto err_out;
+	}
+
+	dni_mrec = map_mft_record(dir_ni);
+	if (IS_ERR(dni_mrec)) {
+		ntfs_error(dir_ni->vol->sb, "failed to map mft record for file 0x%llx.\n",
+			   dir_ni->mft_no);
+		err = -EIO;
+		goto err_out;
+	}
+	parent_mft_ref = MK_LE_MREF(dir_ni->mft_no,
+				    le16_to_cpu(dni_mrec->sequence_number));
+	unmap_mft_record(dir_ni);
+
+	/*
+	 * Create STANDARD_INFORMATION attribute. Write STANDARD_INFORMATION
+	 * version 1.2, windows will upgrade it to version 3 if needed.
+	 */
+	si_len = offsetof(struct standard_information, file_attributes) +
+		sizeof(__le32) + 12;
+	si = kzalloc(si_len, GFP_NOFS);
+	if (!si) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	si->creation_time = si->last_data_change_time = utc2ntfs(ni->i_crtime);
+	si->last_mft_change_time = si->last_access_time = si->creation_time;
+
+	if (!S_ISREG(mode) && !S_ISDIR(mode))
+		si->file_attributes = FILE_ATTR_SYSTEM;
+
+	/* Add STANDARD_INFORMATION to inode. */
+	err = ntfs_attr_add(ni, AT_STANDARD_INFORMATION, AT_UNNAMED, 0, (u8 *)si,
+			si_len);
+	if (err) {
+		ntfs_error(sb, "Failed to add STANDARD_INFORMATION attribute.\n");
+		goto err_out;
+	}
+
+	err = ntfs_sd_add_everyone(ni);
+	if (err)
+		goto err_out;
+	rollback_sd = true;
+
+	if (S_ISDIR(mode)) {
+		struct index_root *ir = NULL;
+		struct index_entry *ie;
+		int ir_len, index_len;
+
+		/* Create struct index_root attribute. */
+		index_len = sizeof(struct index_header) + sizeof(struct index_entry_header);
+		ir_len = offsetof(struct index_root, index) + index_len;
+		ir = kzalloc(ir_len, GFP_NOFS);
+		if (!ir) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+		ir->type = AT_FILE_NAME;
+		ir->collation_rule = COLLATION_FILE_NAME;
+		ir->index_block_size = cpu_to_le32(ni->vol->index_record_size);
+		if (ni->vol->cluster_size <= ni->vol->index_record_size)
+			ir->clusters_per_index_block =
+				NTFS_B_TO_CLU(vol, ni->vol->index_record_size);
+		else
+			ir->clusters_per_index_block =
+				ni->vol->index_record_size >> ni->vol->sector_size_bits;
+		ir->index.entries_offset = cpu_to_le32(sizeof(struct index_header));
+		ir->index.index_length = cpu_to_le32(index_len);
+		ir->index.allocated_size = cpu_to_le32(index_len);
+		ie = (struct index_entry *)((u8 *)ir + sizeof(struct index_root));
+		ie->length = cpu_to_le16(sizeof(struct index_entry_header));
+		ie->key_length = 0;
+		ie->flags = INDEX_ENTRY_END;
+
+		/* Add struct index_root attribute to inode. */
+		err = ntfs_attr_add(ni, AT_INDEX_ROOT, I30, 4, (u8 *)ir, ir_len);
+		if (err) {
+			kfree(ir);
+			ntfs_error(vi->i_sb, "Failed to add struct index_root attribute.\n");
+			goto err_out;
+		}
+		kfree(ir);
+		err = ntfs_attr_open(ni, AT_INDEX_ROOT, I30, 4);
+		if (err)
+			goto err_out;
+	} else {
+		/* Add DATA attribute to inode. */
+		err = ntfs_attr_add(ni, AT_DATA, AT_UNNAMED, 0, NULL, 0);
+		if (err) {
+			ntfs_error(dir_ni->vol->sb, "Failed to add DATA attribute.\n");
+			goto err_out;
+		}
+		rollback_data = true;
+
+		err = ntfs_attr_open(ni, AT_DATA, AT_UNNAMED, 0);
+		if (err)
+			goto err_out;
+
+		if (S_ISLNK(mode)) {
+			err = ntfs_reparse_set_wsl_symlink(ni, target, target_len);
+			if (!err)
+				rollback_reparse = true;
+		} else if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode) ||
+			   S_ISFIFO(mode)) {
+			si->file_attributes = FILE_ATTRIBUTE_RECALL_ON_OPEN;
+			ni->flags = FILE_ATTRIBUTE_RECALL_ON_OPEN;
+			err = ntfs_reparse_set_wsl_not_symlink(ni, mode);
+			if (!err)
+				rollback_reparse = true;
+		}
+		if (err)
+			goto err_out;
+	}
+
+	err = ntfs_ea_set_wsl_inode(vi, dev, &ea_size,
+			NTFS_EA_UID | NTFS_EA_GID | NTFS_EA_MODE);
+	if (err)
+		goto err_out;
+
+	/* Create FILE_NAME attribute. */
+	fn_len = sizeof(struct file_name_attr) + name_len * sizeof(__le16);
+	fn = kzalloc(fn_len, GFP_NOFS);
+	if (!fn) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	fn->file_attributes |= ni->flags;
+	fn->parent_directory = parent_mft_ref;
+	fn->file_name_length = name_len;
+	fn->file_name_type = FILE_NAME_POSIX;
+	fn->type.ea.packed_ea_size = ea_size;
+	if (S_ISDIR(mode)) {
+		fn->file_attributes = FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT;
+		fn->allocated_size = fn->data_size = 0;
+	} else {
+		fn->data_size = cpu_to_le64(ni->data_size);
+		fn->allocated_size = cpu_to_le64(ni->allocated_size);
+	}
+	if (!S_ISREG(mode) && !S_ISDIR(mode)) {
+		fn->file_attributes = FILE_ATTR_SYSTEM;
+		if (rollback_reparse)
+			fn->file_attributes |= FILE_ATTR_REPARSE_POINT;
+	}
+	if (NVolHideDotFiles(vol) && name_len > 0 && name[0] == cpu_to_le16('.'))
+		fn->file_attributes |= FILE_ATTR_HIDDEN;
+	fn->creation_time = fn->last_data_change_time = utc2ntfs(ni->i_crtime);
+	fn->last_mft_change_time = fn->last_access_time = fn->creation_time;
+	memcpy(fn->file_name, name, name_len * sizeof(__le16));
+
+	/* Add FILE_NAME attribute to inode. */
+	err = ntfs_attr_add(ni, AT_FILE_NAME, AT_UNNAMED, 0, (u8 *)fn, fn_len);
+	if (err) {
+		ntfs_error(sb, "Failed to add FILE_NAME attribute.\n");
+		goto err_out;
+	}
+
+	child_mft_ref = MK_MREF(ni->mft_no,
+				le16_to_cpu(ni_mrec->sequence_number));
+	/* Set hard links count and directory flag. */
+	ni_mrec->link_count = cpu_to_le16(1);
+	mark_mft_record_dirty(ni);
+
+	/* Add FILE_NAME attribute to index. */
+	err = ntfs_index_add_filename(dir_ni, fn, child_mft_ref);
+	if (err) {
+		ntfs_debug("Failed to add entry to the index");
+		goto err_out;
+	}
+
+	unmap_mft_record(ni);
+	mutex_unlock(&dir_ni->mrec_lock);
+	mutex_unlock(&ni->mrec_lock);
+
+	ni->flags = fn->file_attributes;
+	/* Set the sequence number. */
+	vi->i_generation = ni->seq_no;
+	set_nlink(vi, 1);
+	ntfs_set_vfs_operations(vi, mode, dev);
+
+	/* Done! */
+	kfree(fn);
+	kfree(si);
+	ntfs_debug("Done.\n");
+	return ni;
+
+err_out:
+	if (rollback_sd)
+		ntfs_attr_remove(ni, AT_SECURITY_DESCRIPTOR, AT_UNNAMED, 0);
+
+	if (rollback_data)
+		ntfs_attr_remove(ni, AT_DATA, AT_UNNAMED, 0);
+
+	if (rollback_reparse)
+		ntfs_delete_reparse_index(ni);
+	/*
+	 * Free extent MFT records (should not exist any with current
+	 * ntfs_create implementation, but for any case if something will be
+	 * changed in the future).
+	 */
+	while (ni->nr_extents != 0) {
+		int err2;
+
+		err2 = ntfs_mft_record_free(ni->vol, *(ni->ext.extent_ntfs_inos));
+		if (err2)
+			ntfs_error(sb,
+				"Failed to free extent MFT record. Leaving inconsistent metadata.\n");
+		ntfs_inode_close(*(ni->ext.extent_ntfs_inos));
+	}
+	if (ntfs_mft_record_free(ni->vol, ni))
+		ntfs_error(sb,
+			"Failed to free MFT record. Leaving inconsistent metadata. Run chkdsk.\n");
+	unmap_mft_record(ni);
+	kfree(fn);
+	kfree(si);
+
+	mutex_unlock(&dir_ni->mrec_lock);
+	mutex_unlock(&ni->mrec_lock);
+
+	remove_inode_hash(vi);
+	discard_new_inode(vi);
+	return ERR_PTR(err);
+}
+
+static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
+		struct dentry *dentry, umode_t mode, bool excl)
+{
+	struct ntfs_volume *vol = NTFS_SB(dir->i_sb);
+	struct ntfs_inode *ni;
+	__le16 *uname;
+	int uname_len, err;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len,
+				  &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(vol->sb, "Failed to convert name to unicode.");
+		return uname_len;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname, uname_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return err;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	ni = __ntfs_create(idmap, dir, uname, uname_len, S_IFREG | mode, 0, NULL, 0);
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (IS_ERR(ni))
+		return PTR_ERR(ni);
+
+	d_instantiate_new(dentry, VFS_I(ni));
+
+	return 0;
+}
+
+static int ntfs_check_unlinkable_dir(struct ntfs_attr_search_ctx *ctx, struct file_name_attr *fn)
+{
+	int link_count;
+	int ret;
+	struct ntfs_inode *ni = ctx->base_ntfs_ino ? ctx->base_ntfs_ino : ctx->ntfs_ino;
+	struct mft_record *ni_mrec = ctx->base_mrec ? ctx->base_mrec : ctx->mrec;
+
+	ret = ntfs_check_empty_dir(ni, ni_mrec);
+	if (!ret || ret != -ENOTEMPTY)
+		return ret;
+
+	link_count = le16_to_cpu(ni_mrec->link_count);
+	/*
+	 * Directory is non-empty, so we can unlink only if there is more than
+	 * one "real" hard link, i.e. links aren't different DOS and WIN32 names
+	 */
+	if ((link_count == 1) ||
+	    (link_count == 2 && fn->file_name_type == FILE_NAME_DOS)) {
+		ret = -ENOTEMPTY;
+		ntfs_debug("Non-empty directory without hard links\n");
+		goto no_hardlink;
+	}
+
+	ret = 0;
+no_hardlink:
+	return ret;
+}
+
+static int ntfs_test_inode_attr(struct inode *vi, void *data)
+{
+	struct ntfs_inode *ni = NTFS_I(vi);
+	u64 mft_no = (u64)(uintptr_t)data;
+
+	if (ni->mft_no != mft_no)
+		return 0;
+	if (NInoAttr(ni) || ni->nr_extents == -1)
+		return 1;
+	else
+		return 0;
+}
+
+/*
+ * ntfs_delete - delete file or directory from ntfs volume
+ * @ni:         ntfs inode for object to delte
+ * @dir_ni:     ntfs inode for directory in which delete object
+ * @name:       unicode name of the object to delete
+ * @name_len:   length of the name in unicode characters
+ * @need_lock:  whether mrec lock is needed or not
+ *
+ * Delete the specified name from the directory index @dir_ni and decrement
+ * the link count of the target inode @ni.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_delete(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
+		__le16 *name, u8 name_len, bool need_lock)
+{
+	struct ntfs_attr_search_ctx *actx = NULL;
+	struct file_name_attr *fn = NULL;
+	bool looking_for_dos_name = false, looking_for_win32_name = false;
+	bool case_sensitive_match = true;
+	int err = 0;
+	struct mft_record *ni_mrec;
+	struct super_block *sb;
+	bool link_count_zero = false;
+
+	ntfs_debug("Entering.\n");
+
+	if (need_lock == true) {
+		mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+		mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+	}
+
+	sb = dir_ni->vol->sb;
+
+	if (ni->nr_extents == -1)
+		ni = ni->ext.base_ntfs_ino;
+	if (dir_ni->nr_extents == -1)
+		dir_ni = dir_ni->ext.base_ntfs_ino;
+	/*
+	 * Search for FILE_NAME attribute with such name. If it's in POSIX or
+	 * WIN32_AND_DOS namespace, then simply remove it from index and inode.
+	 * If filename in DOS or in WIN32 namespace, then remove DOS name first,
+	 * only then remove WIN32 name.
+	 */
+	actx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!actx) {
+		ntfs_error(sb, "%s, Failed to get search context", __func__);
+		if (need_lock) {
+			mutex_unlock(&dir_ni->mrec_lock);
+			mutex_unlock(&ni->mrec_lock);
+		}
+		return -ENOMEM;
+	}
+search:
+	while ((err = ntfs_attr_lookup(AT_FILE_NAME, AT_UNNAMED, 0, CASE_SENSITIVE,
+				0, NULL, 0, actx)) == 0) {
+#ifdef DEBUG
+		unsigned char *s;
+#endif
+		bool case_sensitive = IGNORE_CASE;
+
+		fn = (struct file_name_attr *)((u8 *)actx->attr +
+				le16_to_cpu(actx->attr->data.resident.value_offset));
+#ifdef DEBUG
+		s = ntfs_attr_name_get(ni->vol, fn->file_name, fn->file_name_length);
+		ntfs_debug("name: '%s'  type: %d  dos: %d  win32: %d case: %d\n",
+				s, fn->file_name_type,
+				looking_for_dos_name, looking_for_win32_name,
+				case_sensitive_match);
+		ntfs_attr_name_free(&s);
+#endif
+		if (looking_for_dos_name) {
+			if (fn->file_name_type == FILE_NAME_DOS)
+				break;
+			continue;
+		}
+		if (looking_for_win32_name) {
+			if  (fn->file_name_type == FILE_NAME_WIN32)
+				break;
+			continue;
+		}
+
+		/* Ignore hard links from other directories */
+		if (dir_ni->mft_no != MREF_LE(fn->parent_directory)) {
+			ntfs_debug("MFT record numbers don't match (%llu != %lu)\n",
+					dir_ni->mft_no,
+					MREF_LE(fn->parent_directory));
+			continue;
+		}
+
+		if (fn->file_name_type == FILE_NAME_POSIX || case_sensitive_match)
+			case_sensitive = CASE_SENSITIVE;
+
+		if (ntfs_names_are_equal(fn->file_name, fn->file_name_length,
+					name, name_len, case_sensitive,
+					ni->vol->upcase, ni->vol->upcase_len)) {
+			if (fn->file_name_type == FILE_NAME_WIN32) {
+				looking_for_dos_name = true;
+				ntfs_attr_reinit_search_ctx(actx);
+				continue;
+			}
+			if (fn->file_name_type == FILE_NAME_DOS)
+				looking_for_dos_name = true;
+			break;
+		}
+	}
+	if (err) {
+		/*
+		 * If case sensitive search failed, then try once again
+		 * ignoring case.
+		 */
+		if (err == -ENOENT && case_sensitive_match) {
+			case_sensitive_match = false;
+			ntfs_attr_reinit_search_ctx(actx);
+			goto search;
+		}
+		goto err_out;
+	}
+
+	err = ntfs_check_unlinkable_dir(actx, fn);
+	if (err)
+		goto err_out;
+
+	err = ntfs_index_remove(dir_ni, fn, le32_to_cpu(actx->attr->data.resident.value_length));
+	if (err)
+		goto err_out;
+
+	err = ntfs_attr_record_rm(actx);
+	if (err)
+		goto err_out;
+
+	ni_mrec = actx->base_mrec ? actx->base_mrec : actx->mrec;
+	ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) - 1);
+	drop_nlink(VFS_I(ni));
+
+	mark_mft_record_dirty(ni);
+	if (looking_for_dos_name) {
+		looking_for_dos_name = false;
+		looking_for_win32_name = true;
+		ntfs_attr_reinit_search_ctx(actx);
+		goto search;
+	}
+
+	/*
+	 * If hard link count is not equal to zero then we are done. In other
+	 * case there are no reference to this inode left, so we should free all
+	 * non-resident attributes and mark all MFT record as not in use.
+	 */
+	if (ni_mrec->link_count == 0) {
+		NInoSetBeingDeleted(ni);
+		ntfs_delete_reparse_index(ni);
+		ntfs_delete_object_id_index(ni);
+		link_count_zero = true;
+	}
+
+	ntfs_attr_put_search_ctx(actx);
+	if (need_lock == true) {
+		mutex_unlock(&dir_ni->mrec_lock);
+		mutex_unlock(&ni->mrec_lock);
+	}
+
+	/*
+	 * If hard link count is not equal to zero then we are done. In other
+	 * case there are no reference to this inode left, so we should free all
+	 * non-resident attributes and mark all MFT record as not in use.
+	 */
+	if (link_count_zero == true) {
+		struct inode *attr_vi;
+
+		while ((attr_vi = ilookup5(sb, ni->mft_no, ntfs_test_inode_attr,
+					   (void *)(uintptr_t)ni->mft_no)) != NULL) {
+			clear_nlink(attr_vi);
+			iput(attr_vi);
+		}
+	}
+	ntfs_debug("Done.\n");
+	return 0;
+err_out:
+	ntfs_attr_put_search_ctx(actx);
+	if (need_lock) {
+		mutex_unlock(&dir_ni->mrec_lock);
+		mutex_unlock(&ni->mrec_lock);
+	}
+	return err;
+}
+
+static int ntfs_unlink(struct inode *dir, struct dentry *dentry)
+{
+	struct inode *vi = dentry->d_inode;
+	struct super_block *sb = dir->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int err = 0;
+	struct ntfs_inode *ni = NTFS_I(vi);
+	__le16 *uname = NULL;
+	int uname_len;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len,
+				  &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to Unicode.");
+		return -ENOMEM;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname, uname_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return err;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	err = ntfs_delete(ni, NTFS_I(dir), uname, uname_len, true);
+	if (err)
+		goto out;
+
+	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
+	mark_inode_dirty(dir);
+	inode_set_ctime_to_ts(vi, inode_get_ctime(dir));
+	if (vi->i_nlink)
+		mark_inode_dirty(vi);
+out:
+	kmem_cache_free(ntfs_name_cache, uname);
+	return err;
+}
+
+static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+		struct dentry *dentry, umode_t mode)
+{
+	struct super_block *sb = dir->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int err = 0;
+	struct ntfs_inode *ni;
+	__le16 *uname;
+	int uname_len;
+
+	if (NVolShutdown(vol))
+		return ERR_PTR(-EIO);
+
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len,
+				  &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to unicode.");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname, uname_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return ERR_PTR(err);
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	ni = __ntfs_create(idmap, dir, uname, uname_len, S_IFDIR | mode, 0, NULL, 0);
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (IS_ERR(ni)) {
+		err = PTR_ERR(ni);
+		return ERR_PTR(err);
+	}
+
+	d_instantiate_new(dentry, VFS_I(ni));
+	return NULL;
+}
+
+static int ntfs_rmdir(struct inode *dir, struct dentry *dentry)
+{
+	struct inode *vi = dentry->d_inode;
+	struct super_block *sb = dir->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int err = 0;
+	struct ntfs_inode *ni;
+	__le16 *uname = NULL;
+	int uname_len;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	ni = NTFS_I(vi);
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len,
+				  &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to unicode.");
+		return -ENOMEM;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname, uname_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return err;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	err = ntfs_delete(ni, NTFS_I(dir), uname, uname_len, true);
+	if (err)
+		goto out;
+
+	inode_set_mtime_to_ts(vi, inode_set_atime_to_ts(vi, current_time(vi)));
+out:
+	kmem_cache_free(ntfs_name_cache, uname);
+	return err;
+}
+
+/*
+ * __ntfs_link - create hard link for file or directory
+ * @ni:		ntfs inode for object to create hard link
+ * @dir_ni:	ntfs inode for directory in which new link should be placed
+ * @name:	unicode name of the new link
+ * @name_len:	length of the name in unicode characters
+ *
+ * Create a new hard link. This involves adding an entry to the directory
+ * index and adding a new FILE_NAME attribute to the target inode.
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int __ntfs_link(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
+		__le16 *name, u8 name_len)
+{
+	struct super_block *sb;
+	struct inode *vi = VFS_I(ni);
+	struct file_name_attr *fn = NULL;
+	int fn_len, err = 0;
+	struct mft_record *dir_mrec = NULL, *ni_mrec = NULL;
+
+	ntfs_debug("Entering.\n");
+
+	sb = dir_ni->vol->sb;
+	if (NInoBeingDeleted(dir_ni) || NInoBeingDeleted(ni))
+		return -ENOENT;
+
+	ni_mrec = map_mft_record(ni);
+	if (IS_ERR(ni_mrec)) {
+		err = -EIO;
+		goto err_out;
+	}
+
+	if (le16_to_cpu(ni_mrec->link_count) == 0) {
+		err = -ENOENT;
+		goto err_out;
+	}
+
+	/* Create FILE_NAME attribute. */
+	fn_len = sizeof(struct file_name_attr) + name_len * sizeof(__le16);
+
+	fn = kzalloc(fn_len, GFP_NOFS);
+	if (!fn) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	dir_mrec = map_mft_record(dir_ni);
+	if (IS_ERR(dir_mrec)) {
+		err = -EIO;
+		goto err_out;
+	}
+
+	fn->parent_directory = MK_LE_MREF(dir_ni->mft_no,
+			le16_to_cpu(dir_mrec->sequence_number));
+	unmap_mft_record(dir_ni);
+	fn->file_name_length = name_len;
+	fn->file_name_type = FILE_NAME_POSIX;
+	fn->file_attributes = ni->flags;
+	if (ni_mrec->flags & MFT_RECORD_IS_DIRECTORY) {
+		fn->file_attributes |= FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT;
+		fn->allocated_size = fn->data_size = 0;
+	} else {
+		if (NInoSparse(ni) || NInoCompressed(ni))
+			fn->allocated_size =
+				cpu_to_le64(ni->itype.compressed.size);
+		else
+			fn->allocated_size = cpu_to_le64(ni->allocated_size);
+		fn->data_size = cpu_to_le64(ni->data_size);
+	}
+	if (NVolHideDotFiles(dir_ni->vol) && name_len > 0 && name[0] == cpu_to_le16('.'))
+		fn->file_attributes |= FILE_ATTR_HIDDEN;
+
+	fn->creation_time = utc2ntfs(ni->i_crtime);
+	fn->last_data_change_time = utc2ntfs(inode_get_mtime(vi));
+	fn->last_mft_change_time = utc2ntfs(inode_get_ctime(vi));
+	fn->last_access_time = utc2ntfs(inode_get_atime(vi));
+	memcpy(fn->file_name, name, name_len * sizeof(__le16));
+
+	/* Add FILE_NAME attribute to index. */
+	err = ntfs_index_add_filename(dir_ni, fn, MK_MREF(ni->mft_no,
+					le16_to_cpu(ni_mrec->sequence_number)));
+	if (err) {
+		ntfs_error(sb, "Failed to add filename to the index");
+		goto err_out;
+	}
+	/* Add FILE_NAME attribute to inode. */
+	err = ntfs_attr_add(ni, AT_FILE_NAME, AT_UNNAMED, 0, (u8 *)fn, fn_len);
+	if (err) {
+		ntfs_error(sb, "Failed to add FILE_NAME attribute.\n");
+		/* Try to remove just added attribute from index. */
+		if (ntfs_index_remove(dir_ni, fn, fn_len))
+			goto rollback_failed;
+		goto err_out;
+	}
+	/* Increment hard links count. */
+	ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) + 1);
+	inc_nlink(VFS_I(ni));
+
+	/* Done! */
+	mark_mft_record_dirty(ni);
+	kfree(fn);
+	unmap_mft_record(ni);
+
+	ntfs_debug("Done.\n");
+
+	return 0;
+rollback_failed:
+	ntfs_error(sb, "Rollback failed. Leaving inconsistent metadata.\n");
+err_out:
+	kfree(fn);
+	if (!IS_ERR_OR_NULL(ni_mrec))
+		unmap_mft_record(ni);
+	return err;
+}
+
+static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+		struct dentry *old_dentry, struct inode *new_dir,
+		struct dentry *new_dentry, unsigned int flags)
+{
+	struct inode *old_inode, *new_inode = NULL;
+	int err = 0;
+	int is_dir;
+	struct super_block *sb = old_dir->i_sb;
+	__le16 *uname_new = NULL;
+	__le16 *uname_old = NULL;
+	int new_name_len;
+	int old_name_len;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	struct ntfs_inode *old_ni, *new_ni = NULL;
+	struct ntfs_inode *old_dir_ni = NTFS_I(old_dir), *new_dir_ni = NTFS_I(new_dir);
+
+	if (NVolShutdown(old_dir_ni->vol))
+		return -EIO;
+
+	if (flags & (RENAME_EXCHANGE | RENAME_WHITEOUT))
+		return -EINVAL;
+
+	new_name_len = ntfs_nlstoucs(NTFS_I(new_dir)->vol, new_dentry->d_name.name,
+				     new_dentry->d_name.len, &uname_new,
+				     NTFS_MAX_NAME_LEN);
+	if (new_name_len < 0) {
+		if (new_name_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to unicode.");
+		return -ENOMEM;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname_new, new_name_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname_new);
+		return err;
+	}
+
+	old_name_len = ntfs_nlstoucs(NTFS_I(old_dir)->vol, old_dentry->d_name.name,
+				     old_dentry->d_name.len, &uname_old,
+				     NTFS_MAX_NAME_LEN);
+	if (old_name_len < 0) {
+		kmem_cache_free(ntfs_name_cache, uname_new);
+		if (old_name_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to unicode.");
+		return -ENOMEM;
+	}
+
+	old_inode = old_dentry->d_inode;
+	new_inode = new_dentry->d_inode;
+	old_ni = NTFS_I(old_inode);
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	mutex_lock_nested(&old_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+
+	if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni)) {
+		err = -ENOENT;
+		goto unlock_old;
+	}
+
+	is_dir = S_ISDIR(old_inode->i_mode);
+
+	if (new_inode) {
+		new_ni = NTFS_I(new_inode);
+		mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2);
+		if (old_dir != new_dir) {
+			mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2);
+			if (NInoBeingDeleted(new_dir_ni)) {
+				err = -ENOENT;
+				goto err_out;
+			}
+		}
+
+		if (NInoBeingDeleted(new_ni)) {
+			err = -ENOENT;
+			goto err_out;
+		}
+
+		if (is_dir) {
+			struct mft_record *ni_mrec;
+
+			ni_mrec = map_mft_record(NTFS_I(new_inode));
+			if (IS_ERR(ni_mrec)) {
+				err = -EIO;
+				goto err_out;
+			}
+			err = ntfs_check_empty_dir(NTFS_I(new_inode), ni_mrec);
+			unmap_mft_record(NTFS_I(new_inode));
+			if (err)
+				goto err_out;
+		}
+
+		err = ntfs_delete(new_ni, new_dir_ni, uname_new, new_name_len, false);
+		if (err)
+			goto err_out;
+	} else {
+		if (old_dir != new_dir) {
+			mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2);
+			if (NInoBeingDeleted(new_dir_ni)) {
+				err = -ENOENT;
+				goto err_out;
+			}
+		}
+	}
+
+	err = __ntfs_link(old_ni, new_dir_ni, uname_new, new_name_len);
+	if (err)
+		goto err_out;
+
+	err = ntfs_delete(old_ni, old_dir_ni, uname_old, old_name_len, false);
+	if (err) {
+		int err2;
+
+		ntfs_error(sb, "Failed to delete old ntfs inode(%llu) in old dir, err : %d\n",
+				old_ni->mft_no, err);
+		err2 = ntfs_delete(old_ni, new_dir_ni, uname_new, new_name_len, false);
+		if (err2)
+			ntfs_error(sb, "Failed to delete old ntfs inode in new dir, err : %d\n",
+					err2);
+		goto err_out;
+	}
+
+	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
+	mark_inode_dirty(old_inode);
+	mark_inode_dirty(old_dir);
+	if (old_dir != new_dir)
+		mark_inode_dirty(new_dir);
+	if (new_inode)
+		mark_inode_dirty(old_inode);
+
+	inode_inc_iversion(new_dir);
+
+err_out:
+	if (old_dir != new_dir)
+		mutex_unlock(&new_dir_ni->mrec_lock);
+	if (new_inode)
+		mutex_unlock(&new_ni->mrec_lock);
+
+unlock_old:
+	mutex_unlock(&old_dir_ni->mrec_lock);
+	mutex_unlock(&old_ni->mrec_lock);
+	if (uname_new)
+		kmem_cache_free(ntfs_name_cache, uname_new);
+	if (uname_old)
+		kmem_cache_free(ntfs_name_cache, uname_old);
+
+	return err;
+}
+
+static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+		struct dentry *dentry, const char *symname)
+{
+	struct super_block *sb = dir->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	struct inode *vi;
+	int err = 0;
+	struct ntfs_inode *ni;
+	__le16 *usrc;
+	__le16 *utarget;
+	int usrc_len;
+	int utarget_len;
+	int symlen = strlen(symname);
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	usrc_len = ntfs_nlstoucs(vol, dentry->d_name.name,
+				 dentry->d_name.len, &usrc, NTFS_MAX_NAME_LEN);
+	if (usrc_len < 0) {
+		if (usrc_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to Unicode.");
+		err =  -ENOMEM;
+		goto out;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, usrc, usrc_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, usrc);
+		goto out;
+	}
+
+	utarget_len = ntfs_nlstoucs(vol, symname, symlen, &utarget,
+				    PATH_MAX);
+	if (utarget_len < 0) {
+		if (utarget_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert target name to Unicode.");
+		err =  -ENOMEM;
+		kmem_cache_free(ntfs_name_cache, usrc);
+		goto out;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	ni = __ntfs_create(idmap, dir, usrc, usrc_len, S_IFLNK | 0777, 0,
+			utarget, utarget_len);
+	kmem_cache_free(ntfs_name_cache, usrc);
+	kvfree(utarget);
+	if (IS_ERR(ni)) {
+		err = PTR_ERR(ni);
+		goto out;
+	}
+
+	vi = VFS_I(ni);
+	vi->i_size = symlen;
+	d_instantiate_new(dentry, vi);
+out:
+	return err;
+}
+
+static int ntfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+		struct dentry *dentry, umode_t mode, dev_t rdev)
+{
+	struct super_block *sb = dir->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int err = 0;
+	struct ntfs_inode *ni;
+	__le16 *uname = NULL;
+	int uname_len;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name,
+			dentry->d_name.len, &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to Unicode.");
+		return -ENOMEM;
+	}
+
+	err = ntfs_check_bad_windows_name(vol, uname, uname_len);
+	if (err) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return err;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	switch (mode & S_IFMT) {
+	case S_IFCHR:
+	case S_IFBLK:
+		ni = __ntfs_create(idmap, dir, uname, uname_len,
+				mode, rdev, NULL, 0);
+		break;
+	default:
+		ni = __ntfs_create(idmap, dir, uname, uname_len,
+				mode, 0, NULL, 0);
+	}
+
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (IS_ERR(ni)) {
+		err = PTR_ERR(ni);
+		goto out;
+	}
+
+	d_instantiate_new(dentry, VFS_I(ni));
+out:
+	return err;
+}
+
+static int ntfs_link(struct dentry *old_dentry, struct inode *dir,
+		struct dentry *dentry)
+{
+	struct inode *vi = old_dentry->d_inode;
+	struct super_block *sb = vi->i_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	__le16 *uname = NULL;
+	int uname_len;
+	int err;
+	struct ntfs_inode *ni = NTFS_I(vi), *dir_ni = NTFS_I(dir);
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	uname_len = ntfs_nlstoucs(vol, dentry->d_name.name,
+			dentry->d_name.len, &uname, NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		if (uname_len != -ENAMETOOLONG)
+			ntfs_error(sb, "Failed to convert name to unicode.");
+		err = -ENOMEM;
+		goto out;
+	}
+
+	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
+		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
+
+	ihold(vi);
+	mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL);
+	mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
+	err = __ntfs_link(NTFS_I(vi), NTFS_I(dir), uname, uname_len);
+	if (err) {
+		mutex_unlock(&dir_ni->mrec_lock);
+		mutex_unlock(&ni->mrec_lock);
+		iput(vi);
+		pr_err("failed to create link, err = %d\n", err);
+		goto out;
+	}
+
+	inode_inc_iversion(dir);
+	simple_inode_init_ts(dir);
+
+	inode_inc_iversion(vi);
+	simple_inode_init_ts(vi);
+
+	/* timestamp is already written, so mark_inode_dirty() is unneeded. */
+	d_instantiate(dentry, vi);
+	mutex_unlock(&dir_ni->mrec_lock);
+	mutex_unlock(&ni->mrec_lock);
+
+out:
+	kfree(uname);
+	return err;
+}
+
+/*
+ * Inode operations for directories.
+ */
+const struct inode_operations ntfs_dir_inode_ops = {
+	.lookup		= ntfs_lookup,	/* VFS: Lookup directory. */
+	.create		= ntfs_create,
+	.unlink		= ntfs_unlink,
+	.mkdir		= ntfs_mkdir,
+	.rmdir		= ntfs_rmdir,
+	.rename		= ntfs_rename,
+	.get_acl	= ntfs_get_acl,
+	.set_acl	= ntfs_set_acl,
+	.listxattr	= ntfs_listxattr,
+	.setattr	= ntfs_setattr,
+	.getattr	= ntfs_getattr,
+	.symlink	= ntfs_symlink,
+	.mknod		= ntfs_mknod,
+	.link		= ntfs_link,
+};
+
+/*
+ * ntfs_get_parent - find the dentry of the parent of a given directory dentry
+ * @child_dent:		dentry of the directory whose parent directory to find
+ *
+ * Find the dentry for the parent directory of the directory specified by the
+ * dentry @child_dent.  This function is called from
+ * fs/exportfs/expfs.c::find_exported_dentry() which in turn is called from the
+ * default ->decode_fh() which is export_decode_fh() in the same file.
+ *
+ * Note: ntfs_get_parent() is called with @d_inode(child_dent)->i_mutex down.
+ *
+ * Return the dentry of the parent directory on success or the error code on
+ * error (IS_ERR() is true).
+ */
+static struct dentry *ntfs_get_parent(struct dentry *child_dent)
+{
+	struct inode *vi = d_inode(child_dent);
+	struct ntfs_inode *ni = NTFS_I(vi);
+	struct mft_record *mrec;
+	struct ntfs_attr_search_ctx *ctx;
+	struct attr_record *attr;
+	struct file_name_attr *fn;
+	unsigned long parent_ino;
+	int err;
+
+	ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
+	/* Get the mft record of the inode belonging to the child dentry. */
+	mrec = map_mft_record(ni);
+	if (IS_ERR(mrec))
+		return ERR_CAST(mrec);
+	/* Find the first file name attribute in the mft record. */
+	ctx = ntfs_attr_get_search_ctx(ni, mrec);
+	if (unlikely(!ctx)) {
+		unmap_mft_record(ni);
+		return ERR_PTR(-ENOMEM);
+	}
+try_next:
+	err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, CASE_SENSITIVE, 0, NULL,
+			0, ctx);
+	if (unlikely(err)) {
+		ntfs_attr_put_search_ctx(ctx);
+		unmap_mft_record(ni);
+		if (err == -ENOENT)
+			ntfs_error(vi->i_sb,
+				   "Inode 0x%llx does not have a file name attribute.  Run chkdsk.",
+				   ni->mft_no);
+		return ERR_PTR(err);
+	}
+	attr = ctx->attr;
+	if (unlikely(attr->non_resident))
+		goto try_next;
+	fn = (struct file_name_attr *)((u8 *)attr +
+			le16_to_cpu(attr->data.resident.value_offset));
+	if (unlikely((u8 *)fn + le32_to_cpu(attr->data.resident.value_length) >
+	    (u8 *)attr + le32_to_cpu(attr->length)))
+		goto try_next;
+	/* Get the inode number of the parent directory. */
+	parent_ino = MREF_LE(fn->parent_directory);
+	/* Release the search context and the mft record of the child. */
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(ni);
+
+	return d_obtain_alias(ntfs_iget(vi->i_sb, parent_ino));
+}
+
+static struct inode *ntfs_nfs_get_inode(struct super_block *sb,
+		u64 ino, u32 generation)
+{
+	struct inode *inode;
+
+	inode = ntfs_iget(sb, ino);
+	if (!IS_ERR(inode)) {
+		if (inode->i_generation != generation) {
+			iput(inode);
+			inode = ERR_PTR(-ESTALE);
+		}
+	}
+
+	return inode;
+}
+
+static struct dentry *ntfs_fh_to_dentry(struct super_block *sb, struct fid *fid,
+		int fh_len, int fh_type)
+{
+	return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
+				    ntfs_nfs_get_inode);
+}
+
+static struct dentry *ntfs_fh_to_parent(struct super_block *sb, struct fid *fid,
+		int fh_len, int fh_type)
+{
+	return generic_fh_to_parent(sb, fid, fh_len, fh_type,
+				    ntfs_nfs_get_inode);
+}
+
+/*
+ * Export operations allowing NFS exporting of mounted NTFS partitions.
+ */
+const struct export_operations ntfs_export_ops = {
+	.encode_fh = generic_encode_ino32_fh,
+	.get_parent	= ntfs_get_parent,	/* Find the parent of a given directory. */
+	.fh_to_dentry	= ntfs_fh_to_dentry,
+	.fh_to_parent	= ntfs_fh_to_parent,
+};
diff --git a/fs/ntfs/ntfs.h b/fs/ntfs/ntfs.h
new file mode 100644
index 0000000000000..45064dbcc2e47
--- /dev/null
+++ b/fs/ntfs/ntfs.h
@@ -0,0 +1,294 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (C) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_H
+#define _LINUX_NTFS_H
+
+#include <linux/stddef.h>
+#include <linux/kernel.h>
+#include <linux/hex.h>
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/nls.h>
+#include <linux/smp.h>
+#include <linux/pagemap.h>
+#include <linux/uidgid.h>
+
+#include "volume.h"
+#include "layout.h"
+#include "inode.h"
+
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+/*
+ * Default pre-allocation size is optimize runlist merge overhead
+ * with small chunk size.
+ */
+#define NTFS_DEF_PREALLOC_SIZE		65536
+
+/*
+ * The log2 of the standard number of clusters per compression block.
+ * A value of 4 corresponds to 16 clusters (1 << 4), which is the
+ * default chunk size used by NTFS LZNT1 compression.
+ */
+#define STANDARD_COMPRESSION_UNIT	4
+
+/*
+ * The maximum cluster size (4KB) allowed for compression to be enabled.
+ * By design, NTFS does not support compression on volumes where the
+ * cluster size exceeds 4096 bytes.
+ */
+#define MAX_COMPRESSION_CLUSTER_SIZE 4096
+
+#define NTFS_B_TO_CLU(vol, b) ((b) >> (vol)->cluster_size_bits)
+#define NTFS_CLU_TO_B(vol, clu) ((u64)(clu) << (vol)->cluster_size_bits)
+#define NTFS_B_TO_CLU_OFS(vol, clu) ((u64)(clu) & (vol)->cluster_size_mask)
+
+#define NTFS_MFT_NR_TO_CLU(vol, mft_no) (((u64)mft_no << (vol)->mft_record_size_bits) >> \
+					 (vol)->cluster_size_bits)
+#define NTFS_MFT_NR_TO_PIDX(vol, mft_no) (mft_no >> (PAGE_SHIFT - \
+					  (vol)->mft_record_size_bits))
+#define NTFS_MFT_NR_TO_POFS(vol, mft_no) (((u64)mft_no << (vol)->mft_record_size_bits) & \
+					  ~PAGE_MASK)
+
+#define NTFS_PIDX_TO_BLK(vol, idx) (((u64)idx << PAGE_SHIFT) >> \
+				    ((vol)->sb)->s_blocksize_bits)
+#define NTFS_PIDX_TO_CLU(vol, idx) (((u64)idx << PAGE_SHIFT) >> \
+				    (vol)->cluster_size_bits)
+#define NTFS_CLU_TO_PIDX(vol, clu) (((u64)(clu) << (vol)->cluster_size_bits) >> \
+				    PAGE_SHIFT)
+#define NTFS_CLU_TO_POFS(vol, clu) (((u64)(clu) << (vol)->cluster_size_bits) & \
+				    ~PAGE_MASK)
+
+#define NTFS_B_TO_SECTOR(vol, b) ((b) >> ((vol)->sb)->s_blocksize_bits)
+
+enum {
+	NTFS_BLOCK_SIZE		= 512,
+	NTFS_BLOCK_SIZE_BITS	= 9,
+	NTFS_SB_MAGIC		= 0x5346544e,	/* 'NTFS' */
+	NTFS_MAX_NAME_LEN	= 255,
+	NTFS_MAX_LABEL_LEN	= 128,
+};
+
+enum {
+	CASE_SENSITIVE = 0,
+	IGNORE_CASE = 1,
+};
+
+/*
+ * Conversion helpers for NTFS units.
+ */
+
+/* Convert bytes to cluster count */
+static inline u64 ntfs_bytes_to_cluster(const struct ntfs_volume *vol,
+		s64 bytes)
+{
+	return bytes >> vol->cluster_size_bits;
+}
+
+/* Convert cluster count to bytes */
+static inline u64 ntfs_cluster_to_bytes(const struct ntfs_volume *vol,
+		u64 clusters)
+{
+	return clusters << vol->cluster_size_bits;
+}
+
+/* Get the byte offset within a cluster from a linear byte address */
+static inline u64 ntfs_bytes_to_cluster_off(const struct ntfs_volume *vol,
+		u64 bytes)
+{
+	return bytes & vol->cluster_size_mask;
+}
+
+/* Calculate the physical cluster number containing a specific MFT record. */
+static inline u64 ntfs_mft_no_to_cluster(const struct ntfs_volume *vol,
+		unsigned long mft_no)
+{
+	return ((u64)mft_no << vol->mft_record_size_bits) >>
+		vol->cluster_size_bits;
+}
+
+/* Calculate the folio index where the MFT record resides. */
+static inline pgoff_t ntfs_mft_no_to_pidx(const struct ntfs_volume *vol,
+		unsigned long mft_no)
+{
+	return mft_no >> (PAGE_SHIFT - vol->mft_record_size_bits);
+}
+
+/* Calculate the byte offset within a folio for an MFT record. */
+static inline u64 ntfs_mft_no_to_poff(const struct ntfs_volume *vol,
+		unsigned long mft_no)
+{
+	return ((u64)mft_no << vol->mft_record_size_bits) & ~PAGE_MASK;
+}
+
+/* Convert folio index to cluster number. */
+static inline u64 ntfs_pidx_to_cluster(const struct ntfs_volume *vol,
+		pgoff_t idx)
+{
+	return ((u64)idx << PAGE_SHIFT) >> vol->cluster_size_bits;
+}
+
+/* Convert cluster number to folio index. */
+static inline pgoff_t ntfs_cluster_to_pidx(const struct ntfs_volume *vol,
+		u64 clu)
+{
+	return (clu << vol->cluster_size_bits) >> PAGE_SHIFT;
+}
+
+/* Get the byte offset within a folio from a cluster number */
+static inline u64 ntfs_cluster_to_poff(const struct ntfs_volume *vol,
+		u64 clu)
+{
+	return (clu << vol->cluster_size_bits) & ~PAGE_MASK;
+}
+
+/* Convert byte offset to sector (block) number. */
+static inline sector_t ntfs_bytes_to_sector(const struct ntfs_volume *vol,
+		u64 bytes)
+{
+	return bytes >> vol->sb->s_blocksize_bits;
+}
+
+/* Global variables. */
+
+/* Slab caches (from super.c). */
+extern struct kmem_cache *ntfs_name_cache;
+extern struct kmem_cache *ntfs_inode_cache;
+extern struct kmem_cache *ntfs_big_inode_cache;
+extern struct kmem_cache *ntfs_attr_ctx_cache;
+extern struct kmem_cache *ntfs_index_ctx_cache;
+
+/* The various operations structs defined throughout the driver files. */
+extern const struct address_space_operations ntfs_aops;
+extern const struct address_space_operations ntfs_mft_aops;
+
+extern const struct  file_operations ntfs_file_ops;
+extern const struct inode_operations ntfs_file_inode_ops;
+extern const  struct inode_operations ntfs_symlink_inode_operations;
+extern const struct inode_operations ntfs_special_inode_operations;
+
+extern const struct  file_operations ntfs_dir_ops;
+extern const struct inode_operations ntfs_dir_inode_ops;
+
+extern const struct  file_operations ntfs_empty_file_ops;
+extern const struct inode_operations ntfs_empty_inode_ops;
+
+extern const struct export_operations ntfs_export_ops;
+
+/*
+ * NTFS_SB - return the ntfs volume given a vfs super block
+ * @sb:		VFS super block
+ *
+ * NTFS_SB() returns the ntfs volume associated with the VFS super block @sb.
+ */
+static inline struct ntfs_volume *NTFS_SB(struct super_block *sb)
+{
+	return sb->s_fs_info;
+}
+
+/* Declarations of functions and global variables. */
+
+/* From fs/ntfs/compress.c */
+int ntfs_read_compressed_block(struct folio *folio);
+int allocate_compression_buffers(void);
+void free_compression_buffers(void);
+int ntfs_compress_write(struct ntfs_inode *ni, loff_t pos, size_t count,
+		struct iov_iter *from);
+
+/* From fs/ntfs/super.c */
+#define default_upcase_len 0x10000
+extern struct mutex ntfs_lock;
+
+struct option_t {
+	int val;
+	char *str;
+};
+extern const struct option_t on_errors_arr[];
+int ntfs_set_volume_flags(struct ntfs_volume *vol, __le16 flags);
+int ntfs_clear_volume_flags(struct ntfs_volume *vol, __le16 flags);
+int ntfs_write_volume_label(struct ntfs_volume *vol, char *label);
+
+/* From fs/ntfs/mst.c */
+int post_read_mst_fixup(struct ntfs_record *b, const u32 size);
+int pre_write_mst_fixup(struct ntfs_record *b, const u32 size);
+void post_write_mst_fixup(struct ntfs_record *b);
+
+/* From fs/ntfs/unistr.c */
+bool ntfs_are_names_equal(const __le16 *s1, size_t s1_len,
+		const __le16 *s2, size_t s2_len,
+		const u32 ic,
+		const __le16 *upcase, const u32 upcase_size);
+int ntfs_collate_names(const __le16 *name1, const u32 name1_len,
+		const __le16 *name2, const u32 name2_len,
+		const int err_val, const u32 ic,
+		const __le16 *upcase, const u32 upcase_len);
+int ntfs_ucsncmp(const __le16 *s1, const __le16 *s2, size_t n);
+int ntfs_ucsncasecmp(const __le16 *s1, const __le16 *s2, size_t n,
+		const __le16 *upcase, const u32 upcase_size);
+int ntfs_file_compare_values(const struct file_name_attr *file_name_attr1,
+		const struct file_name_attr *file_name_attr2,
+		const int err_val, const u32 ic,
+		const __le16 *upcase, const u32 upcase_len);
+int ntfs_nlstoucs(const struct ntfs_volume *vol, const char *ins,
+		const int ins_len, __le16 **outs, int max_name_len);
+int ntfs_ucstonls(const struct ntfs_volume *vol, const __le16 *ins,
+		const int ins_len, unsigned char **outs, int outs_len);
+__le16 *ntfs_ucsndup(const __le16 *s, u32 maxlen);
+bool ntfs_names_are_equal(const __le16 *s1, size_t s1_len,
+		const __le16 *s2, size_t s2_len,
+		const u32 ic,
+		const __le16 *upcase, const u32 upcase_size);
+int ntfs_force_shutdown(struct super_block *sb, u32 flags);
+long ntfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long ntfs_compat_ioctl(struct file *filp, unsigned int cmd,
+		unsigned long arg);
+#endif
+
+/* From fs/ntfs/upcase.c */
+__le16 *generate_default_upcase(void);
+
+static inline int ntfs_ffs(int x)
+{
+	int r = 1;
+
+	if (!x)
+		return 0;
+	if (!(x & 0xffff)) {
+		x >>= 16;
+		r += 16;
+	}
+	if (!(x & 0xff)) {
+		x >>= 8;
+		r += 8;
+	}
+	if (!(x & 0xf)) {
+		x >>= 4;
+		r += 4;
+	}
+	if (!(x & 3)) {
+		x >>= 2;
+		r += 2;
+	}
+	if (!(x & 1))
+		r += 1;
+	return r;
+}
+
+/* From fs/ntfs/bdev-io.c */
+int ntfs_bdev_read(struct block_device *bdev, char *data, loff_t start, size_t size);
+int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size);
+
+#endif /* _LINUX_NTFS_H */
diff --git a/fs/ntfs/object_id.c b/fs/ntfs/object_id.c
new file mode 100644
index 0000000000000..89002c3759df4
--- /dev/null
+++ b/fs/ntfs/object_id.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Pocessing of object ids
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ *
+ * Copyright (c) 2009-2019 Jean-Pierre Andre
+ * Copyright (c) 2026 LG Electronics Co., Ltd.
+ */
+
+#include "ntfs.h"
+#include "index.h"
+#include "object_id.h"
+
+struct object_id_index_key {
+	union {
+		u32 alignment;
+		struct guid guid;
+	} object_id;
+} __packed;
+
+struct object_id_index_data {
+	__le64 file_id;
+	struct guid birth_volume_id;
+	struct guid birth_object_id;
+	struct guid domain_id;
+} __packed;
+
+/* Index entry in $Extend/$ObjId */
+struct object_id_index {
+	struct index_entry_header header;
+	struct object_id_index_key key;
+	struct object_id_index_data data;
+} __packed;
+
+__le16 objid_index_name[] = {cpu_to_le16('$'), cpu_to_le16('O'), 0};
+
+/*
+ * open_object_id_index - Open the $Extend/$ObjId file and its index
+ * @vol: NTFS volume structure
+ *
+ * Opens the $ObjId system file and retrieves its index context.
+ *
+ * Return: The index context if opened successfully, or NULL if an error
+ *	   occurred.
+ */
+static struct ntfs_index_context *open_object_id_index(struct ntfs_volume *vol)
+{
+	struct inode *dir_vi, *vi;
+	struct ntfs_inode *dir_ni;
+	struct ntfs_index_context *xo = NULL;
+	struct ntfs_name *name = NULL;
+	u64 mref;
+	int uname_len;
+	__le16 *uname;
+
+	uname_len = ntfs_nlstoucs(vol, "$ObjId", 6, &uname,
+			NTFS_MAX_NAME_LEN);
+	if (uname_len < 0)
+		return NULL;
+
+	/* do not use path_name_to inode - could reopen root */
+	dir_vi = ntfs_iget(vol->sb, FILE_Extend);
+	if (IS_ERR(dir_vi)) {
+		kmem_cache_free(ntfs_name_cache, uname);
+		return NULL;
+	}
+	dir_ni = NTFS_I(dir_vi);
+
+	mutex_lock_nested(&dir_ni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT);
+	mref = ntfs_lookup_inode_by_name(dir_ni, uname, uname_len, &name);
+	mutex_unlock(&dir_ni->mrec_lock);
+	kfree(name);
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (IS_ERR_MREF(mref))
+		goto put_dir_vi;
+
+	vi = ntfs_iget(vol->sb, MREF(mref));
+	if (IS_ERR(vi))
+		goto put_dir_vi;
+
+	xo = ntfs_index_ctx_get(NTFS_I(vi), objid_index_name, 2);
+	if (!xo)
+		iput(vi);
+put_dir_vi:
+	iput(dir_vi);
+	return xo;
+}
+
+
+/*
+ * remove_object_id_index - Remove an object id index entry if attribute present
+ * @ni: NTFS inode structure containing the attribute
+ * @xo:	Index context for the object id index
+ *
+ * Reads the existing object ID attribute and removes it from the index.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+static int remove_object_id_index(struct ntfs_inode *ni, struct ntfs_index_context *xo)
+{
+	struct object_id_index_key key = {0};
+	s64 size;
+
+	if (ni->data_size == 0)
+		return -ENODATA;
+
+	/* read the existing object id attribute */
+	size = ntfs_inode_attr_pread(VFS_I(ni), 0, sizeof(struct guid),
+				     (char *)&key);
+	if (size != sizeof(struct guid))
+		return -ENODATA;
+
+	if (!ntfs_index_lookup(&key, sizeof(struct object_id_index_key), xo))
+		return ntfs_index_rm(xo);
+
+	return 0;
+}
+
+/*
+ * ntfs_delete_object_id_index - Delete an object_id index entry
+ * @ni:	NTFS inode structure
+ *
+ * Opens the object ID index and removes the entry corresponding to the inode.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+int ntfs_delete_object_id_index(struct ntfs_inode *ni)
+{
+	struct ntfs_index_context *xo;
+	struct ntfs_inode *xoni;
+	struct inode *attr_vi;
+	int ret = 0;
+
+	attr_vi = ntfs_attr_iget(VFS_I(ni), AT_OBJECT_ID, AT_UNNAMED, 0);
+	if (IS_ERR(attr_vi))
+		return PTR_ERR(attr_vi);
+
+	/*
+	 * read the existing object id and un-index it
+	 */
+	xo = open_object_id_index(ni->vol);
+	if (xo) {
+		xoni = xo->idx_ni;
+		mutex_lock_nested(&xoni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT);
+		ret = remove_object_id_index(NTFS_I(attr_vi), xo);
+		if (!ret) {
+			ntfs_index_entry_mark_dirty(xo);
+			mark_mft_record_dirty(xoni);
+		}
+		ntfs_index_ctx_put(xo);
+		mutex_unlock(&xoni->mrec_lock);
+		iput(VFS_I(xoni));
+	}
+
+	iput(attr_vi);
+	return ret;
+}
diff --git a/fs/ntfs/object_id.h b/fs/ntfs/object_id.h
new file mode 100644
index 0000000000000..e4b8e529d5bc8
--- /dev/null
+++ b/fs/ntfs/object_id.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2008-2021 Jean-Pierre Andre
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_OBJECT_ID_H
+#define _LINUX_NTFS_OBJECT_ID_H
+
+extern __le16 objid_index_name[];
+
+int ntfs_delete_object_id_index(struct ntfs_inode *ni);
+
+#endif /* _LINUX_NTFS_OBJECT_ID_H */
diff --git a/fs/ntfs/quota.c b/fs/ntfs/quota.c
new file mode 100644
index 0000000000000..b443243b58fbf
--- /dev/null
+++ b/fs/ntfs/quota.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel quota ($Quota) handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ */
+
+#include "index.h"
+#include "quota.h"
+#include "debug.h"
+#include "ntfs.h"
+
+/*
+ * ntfs_mark_quotas_out_of_date - mark the quotas out of date on an ntfs volume
+ * @vol:	ntfs volume on which to mark the quotas out of date
+ *
+ * Mark the quotas out of date on the ntfs volume @vol and return 'true' on
+ * success and 'false' on error.
+ */
+bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol)
+{
+	struct ntfs_index_context *ictx;
+	struct quota_control_entry *qce;
+	const __le32 qid = QUOTA_DEFAULTS_ID;
+	int err;
+
+	ntfs_debug("Entering.");
+	if (NVolQuotaOutOfDate(vol))
+		goto done;
+	if (!vol->quota_ino || !vol->quota_q_ino) {
+		ntfs_error(vol->sb, "Quota inodes are not open.");
+		return false;
+	}
+	inode_lock(vol->quota_q_ino);
+	ictx = ntfs_index_ctx_get(NTFS_I(vol->quota_q_ino), I30, 4);
+	if (!ictx) {
+		ntfs_error(vol->sb, "Failed to get index context.");
+		goto err_out;
+	}
+	err = ntfs_index_lookup(&qid, sizeof(qid), ictx);
+	if (err) {
+		if (err == -ENOENT)
+			ntfs_error(vol->sb, "Quota defaults entry is not present.");
+		else
+			ntfs_error(vol->sb, "Lookup of quota defaults entry failed.");
+		goto err_out;
+	}
+	if (ictx->data_len < offsetof(struct quota_control_entry, sid)) {
+		ntfs_error(vol->sb, "Quota defaults entry size is invalid.  Run chkdsk.");
+		goto err_out;
+	}
+	qce = (struct quota_control_entry *)ictx->data;
+	if (le32_to_cpu(qce->version) != QUOTA_VERSION) {
+		ntfs_error(vol->sb,
+			"Quota defaults entry version 0x%x is not supported.",
+			le32_to_cpu(qce->version));
+		goto err_out;
+	}
+	ntfs_debug("Quota defaults flags = 0x%x.", le32_to_cpu(qce->flags));
+	/* If quotas are already marked out of date, no need to do anything. */
+	if (qce->flags & QUOTA_FLAG_OUT_OF_DATE)
+		goto set_done;
+	/*
+	 * If quota tracking is neither requested, nor enabled and there are no
+	 * pending deletes, no need to mark the quotas out of date.
+	 */
+	if (!(qce->flags & (QUOTA_FLAG_TRACKING_ENABLED |
+			QUOTA_FLAG_TRACKING_REQUESTED |
+			QUOTA_FLAG_PENDING_DELETES)))
+		goto set_done;
+	/*
+	 * Set the QUOTA_FLAG_OUT_OF_DATE bit thus marking quotas out of date.
+	 * This is verified on WinXP to be sufficient to cause windows to
+	 * rescan the volume on boot and update all quota entries.
+	 */
+	qce->flags |= QUOTA_FLAG_OUT_OF_DATE;
+	/* Ensure the modified flags are written to disk. */
+	ntfs_index_entry_mark_dirty(ictx);
+set_done:
+	ntfs_index_ctx_put(ictx);
+	inode_unlock(vol->quota_q_ino);
+	/*
+	 * We set the flag so we do not try to mark the quotas out of date
+	 * again on remount.
+	 */
+	NVolSetQuotaOutOfDate(vol);
+done:
+	ntfs_debug("Done.");
+	return true;
+err_out:
+	if (ictx)
+		ntfs_index_ctx_put(ictx);
+	inode_unlock(vol->quota_q_ino);
+	return false;
+}
diff --git a/fs/ntfs/quota.h b/fs/ntfs/quota.h
new file mode 100644
index 0000000000000..4b7322661a326
--- /dev/null
+++ b/fs/ntfs/quota.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for NTFS kernel quota ($Quota) handling.
+ *
+ * Copyright (c) 2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_QUOTA_H
+#define _LINUX_NTFS_QUOTA_H
+
+#include "volume.h"
+
+bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol);
+
+#endif /* _LINUX_NTFS_QUOTA_H */
diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c
new file mode 100644
index 0000000000000..4cddd918defcb
--- /dev/null
+++ b/fs/ntfs/reparse.c
@@ -0,0 +1,573 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Processing of reparse points
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ *
+ * Copyright (c) 2008-2021 Jean-Pierre Andre
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include "ntfs.h"
+#include "layout.h"
+#include "attrib.h"
+#include "inode.h"
+#include "dir.h"
+#include "volume.h"
+#include "mft.h"
+#include "index.h"
+#include "lcnalloc.h"
+#include "reparse.h"
+
+struct wsl_link_reparse_data {
+	__le32	type;
+	char	link[];
+};
+
+/* Index entry in $Extend/$Reparse */
+struct reparse_index {
+	struct index_entry_header header;
+	struct reparse_index_key key;
+	__le32 filling;
+};
+
+__le16 reparse_index_name[] = {cpu_to_le16('$'), cpu_to_le16('R'), 0};
+
+
+/*
+ * Check if the reparse point attribute buffer is valid.
+ * Returns true if valid, false otherwise.
+ */
+static bool ntfs_is_valid_reparse_buffer(struct ntfs_inode *ni,
+		const struct reparse_point *reparse_attr, size_t size)
+{
+	size_t expected;
+
+	if (!ni || !reparse_attr)
+		return false;
+
+	/* Minimum size must cover reparse_point header */
+	if (size < sizeof(struct reparse_point))
+		return false;
+
+	/* Reserved zero tag is invalid */
+	if (reparse_attr->reparse_tag == IO_REPARSE_TAG_RESERVED_ZERO)
+		return false;
+
+	/* Calculate expected total size */
+	expected = sizeof(struct reparse_point) +
+		le16_to_cpu(reparse_attr->reparse_data_length);
+
+	/* Add GUID size for non-Microsoft tags */
+	if (!(reparse_attr->reparse_tag & IO_REPARSE_TAG_IS_MICROSOFT))
+		expected += sizeof(struct guid);
+
+	/* Buffer must exactly match the expected size */
+	return expected == size;
+}
+
+/*
+ * Do some sanity checks on reparse data
+ *
+ * Microsoft reparse points have an 8-byte header whereas
+ * non-Microsoft reparse points have a 24-byte header.  In each case,
+ * 'reparse_data_length' must equal the number of non-header bytes.
+ *
+ * If the reparse data looks like a junction point or symbolic
+ * link, more checks can be done.
+ */
+static bool valid_reparse_data(struct ntfs_inode *ni,
+		const struct reparse_point *reparse_attr, size_t size)
+{
+	const struct wsl_link_reparse_data *wsl_reparse_data =
+		(const struct wsl_link_reparse_data *)reparse_attr->reparse_data;
+	unsigned int data_len = le16_to_cpu(reparse_attr->reparse_data_length);
+
+	if (ntfs_is_valid_reparse_buffer(ni, reparse_attr, size) == false)
+		return false;
+
+	switch (reparse_attr->reparse_tag) {
+	case IO_REPARSE_TAG_LX_SYMLINK:
+		if (data_len <= sizeof(wsl_reparse_data->type) ||
+		    wsl_reparse_data->type != cpu_to_le32(2))
+			return false;
+		break;
+	case IO_REPARSE_TAG_AF_UNIX:
+	case IO_REPARSE_TAG_LX_FIFO:
+	case IO_REPARSE_TAG_LX_CHR:
+	case IO_REPARSE_TAG_LX_BLK:
+		if (data_len || !(ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN))
+			return false;
+	}
+
+	return true;
+}
+
+static unsigned int ntfs_reparse_tag_mode(struct reparse_point *reparse_attr)
+{
+	unsigned int mode = 0;
+
+	switch (reparse_attr->reparse_tag) {
+	case IO_REPARSE_TAG_SYMLINK:
+	case IO_REPARSE_TAG_LX_SYMLINK:
+		mode = S_IFLNK;
+		break;
+	case IO_REPARSE_TAG_AF_UNIX:
+		mode = S_IFSOCK;
+		break;
+	case IO_REPARSE_TAG_LX_FIFO:
+		mode = S_IFIFO;
+		break;
+	case IO_REPARSE_TAG_LX_CHR:
+		mode = S_IFCHR;
+		break;
+	case IO_REPARSE_TAG_LX_BLK:
+		mode = S_IFBLK;
+	}
+
+	return mode;
+}
+
+/*
+ * Get the target for symbolic link
+ */
+unsigned int ntfs_make_symlink(struct ntfs_inode *ni)
+{
+	s64 attr_size = 0;
+	unsigned int lth;
+	struct reparse_point *reparse_attr;
+	struct wsl_link_reparse_data *wsl_link_data;
+	unsigned int mode = 0;
+
+	reparse_attr = ntfs_attr_readall(ni, AT_REPARSE_POINT, NULL, 0,
+					 &attr_size);
+	if (reparse_attr && attr_size &&
+	    valid_reparse_data(ni, reparse_attr, attr_size)) {
+		switch (reparse_attr->reparse_tag) {
+		case IO_REPARSE_TAG_LX_SYMLINK:
+			wsl_link_data =
+				(struct wsl_link_reparse_data *)reparse_attr->reparse_data;
+			if (wsl_link_data->type == cpu_to_le32(2)) {
+				lth = le16_to_cpu(reparse_attr->reparse_data_length) -
+						  sizeof(wsl_link_data->type);
+				ni->target = kvzalloc(lth + 1, GFP_NOFS);
+				if (ni->target) {
+					memcpy(ni->target, wsl_link_data->link, lth);
+					ni->target[lth] = 0;
+					mode = ntfs_reparse_tag_mode(reparse_attr);
+				}
+			}
+			break;
+		default:
+			mode = ntfs_reparse_tag_mode(reparse_attr);
+		}
+	} else
+		ni->flags &= ~FILE_ATTR_REPARSE_POINT;
+
+	if (reparse_attr)
+		kvfree(reparse_attr);
+
+	return mode;
+}
+
+unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mref)
+{
+	s64 attr_size = 0;
+	struct reparse_point *reparse_attr;
+	unsigned int dt_type = DT_UNKNOWN;
+	struct inode *vi;
+
+	vi = ntfs_iget(vol->sb, mref);
+	if (IS_ERR(vi))
+		return PTR_ERR(vi);
+
+	reparse_attr = (struct reparse_point *)ntfs_attr_readall(NTFS_I(vi),
+			AT_REPARSE_POINT, NULL, 0, &attr_size);
+
+	if (reparse_attr && attr_size) {
+		switch (reparse_attr->reparse_tag) {
+		case IO_REPARSE_TAG_SYMLINK:
+		case IO_REPARSE_TAG_LX_SYMLINK:
+			dt_type = DT_LNK;
+			break;
+		case IO_REPARSE_TAG_AF_UNIX:
+			dt_type = DT_SOCK;
+			break;
+		case IO_REPARSE_TAG_LX_FIFO:
+			dt_type = DT_FIFO;
+			break;
+		case IO_REPARSE_TAG_LX_CHR:
+			dt_type = DT_CHR;
+			break;
+		case IO_REPARSE_TAG_LX_BLK:
+			dt_type = DT_BLK;
+		}
+	}
+
+	if (reparse_attr)
+		kvfree(reparse_attr);
+
+	iput(vi);
+	return dt_type;
+}
+
+/*
+ * Set the index for new reparse data
+ */
+static int set_reparse_index(struct ntfs_inode *ni, struct ntfs_index_context *xr,
+		__le32 reparse_tag)
+{
+	struct reparse_index indx;
+	u64 file_id_cpu;
+	__le64 file_id;
+
+	file_id_cpu = MK_MREF(ni->mft_no, ni->seq_no);
+	file_id = cpu_to_le64(file_id_cpu);
+	indx.header.data.vi.data_offset =
+		cpu_to_le16(sizeof(struct index_entry_header) + sizeof(struct reparse_index_key));
+	indx.header.data.vi.data_length = 0;
+	indx.header.data.vi.reservedV = 0;
+	indx.header.length = cpu_to_le16(sizeof(struct reparse_index));
+	indx.header.key_length = cpu_to_le16(sizeof(struct reparse_index_key));
+	indx.header.flags = 0;
+	indx.header.reserved = 0;
+	indx.key.reparse_tag = reparse_tag;
+	/* danger on processors which require proper alignment! */
+	memcpy(&indx.key.file_id, &file_id, 8);
+	indx.filling = 0;
+	ntfs_index_ctx_reinit(xr);
+
+	return ntfs_ie_add(xr, (struct index_entry *)&indx);
+}
+
+/*
+ * Remove a reparse data index entry if attribute present
+ */
+static int remove_reparse_index(struct inode *rp, struct ntfs_index_context *xr,
+				__le32 *preparse_tag)
+{
+	struct reparse_index_key key;
+	u64 file_id_cpu;
+	__le64 file_id;
+	s64 size;
+	struct ntfs_inode *ni = NTFS_I(rp);
+	int err = 0, ret = ni->data_size;
+
+	if (ni->data_size == 0)
+		return 0;
+
+	/* read the existing reparse_tag */
+	size = ntfs_inode_attr_pread(rp, 0, 4, (char *)preparse_tag);
+	if (size != 4)
+		return -ENODATA;
+
+	file_id_cpu = MK_MREF(ni->mft_no, ni->seq_no);
+	file_id = cpu_to_le64(file_id_cpu);
+	key.reparse_tag = *preparse_tag;
+	/* danger on processors which require proper alignment! */
+	memcpy(&key.file_id, &file_id, 8);
+	if (!ntfs_index_lookup(&key, sizeof(struct reparse_index_key), xr)) {
+		err = ntfs_index_rm(xr);
+		if (err)
+			ret = err;
+	}
+	return ret;
+}
+
+/*
+ * Open the $Extend/$Reparse file and its index
+ */
+static struct ntfs_index_context *open_reparse_index(struct ntfs_volume *vol)
+{
+	struct ntfs_index_context *xr = NULL;
+	u64 mref;
+	__le16 *uname;
+	struct ntfs_name *name = NULL;
+	int uname_len;
+	struct inode *vi, *dir_vi;
+
+	/* do not use path_name_to inode - could reopen root */
+	dir_vi = ntfs_iget(vol->sb, FILE_Extend);
+	if (IS_ERR(dir_vi))
+		return NULL;
+
+	uname_len = ntfs_nlstoucs(vol, "$Reparse", 8, &uname,
+				  NTFS_MAX_NAME_LEN);
+	if (uname_len < 0) {
+		iput(dir_vi);
+		return NULL;
+	}
+
+	mutex_lock_nested(&NTFS_I(dir_vi)->mrec_lock, NTFS_EXTEND_MUTEX_PARENT);
+	mref = ntfs_lookup_inode_by_name(NTFS_I(dir_vi), uname, uname_len,
+					 &name);
+	mutex_unlock(&NTFS_I(dir_vi)->mrec_lock);
+	kfree(name);
+	kmem_cache_free(ntfs_name_cache, uname);
+	if (IS_ERR_MREF(mref))
+		goto put_dir_vi;
+
+	vi = ntfs_iget(vol->sb, MREF(mref));
+	if (IS_ERR(vi))
+		goto put_dir_vi;
+
+	xr = ntfs_index_ctx_get(NTFS_I(vi), reparse_index_name, 2);
+	if (!xr)
+		iput(vi);
+put_dir_vi:
+	iput(dir_vi);
+	return xr;
+}
+
+
+/*
+ * Update the reparse data and index
+ *
+ * The reparse data attribute should have been created, and
+ * an existing index is expected if there is an existing value.
+ *
+ */
+static int update_reparse_data(struct ntfs_inode *ni, struct ntfs_index_context *xr,
+		char *value, size_t size)
+{
+	struct inode *rp_inode;
+	int err = 0;
+	s64 written;
+	int oldsize;
+	__le32 reparse_tag;
+	struct ntfs_inode *rp_ni;
+
+	rp_inode = ntfs_attr_iget(VFS_I(ni), AT_REPARSE_POINT, AT_UNNAMED, 0);
+	if (IS_ERR(rp_inode))
+		return -EINVAL;
+	rp_ni = NTFS_I(rp_inode);
+
+	/* remove the existing reparse data */
+	oldsize = remove_reparse_index(rp_inode, xr, &reparse_tag);
+	if (oldsize < 0) {
+		err = oldsize;
+		goto put_rp_inode;
+	}
+
+	/* overwrite value if any */
+	written = ntfs_inode_attr_pwrite(rp_inode, 0, size, value, false);
+	if (written != size) {
+		ntfs_error(ni->vol->sb, "Failed to update reparse data\n");
+		err = -EIO;
+		goto put_rp_inode;
+	}
+
+	if (set_reparse_index(ni, xr, ((const struct reparse_point *)value)->reparse_tag) &&
+	    oldsize > 0) {
+		/*
+		 * If cannot index, try to remove the reparse
+		 * data and log the error. There will be an
+		 * inconsistency if removal fails.
+		 */
+		ntfs_attr_rm(rp_ni);
+		ntfs_error(ni->vol->sb,
+			   "Failed to index reparse data. Possible corruption.\n");
+	}
+
+	mark_mft_record_dirty(ni);
+put_rp_inode:
+	iput(rp_inode);
+
+	return err;
+}
+
+/*
+ * Delete a reparse index entry
+ */
+int ntfs_delete_reparse_index(struct ntfs_inode *ni)
+{
+	struct inode *vi;
+	struct ntfs_index_context *xr;
+	struct ntfs_inode *xrni;
+	__le32 reparse_tag;
+	int err = 0;
+
+	if (!(ni->flags & FILE_ATTR_REPARSE_POINT))
+		return 0;
+
+	vi = ntfs_attr_iget(VFS_I(ni), AT_REPARSE_POINT, AT_UNNAMED, 0);
+	if (IS_ERR(vi))
+		return PTR_ERR(vi);
+
+	/*
+	 * read the existing reparse data (the tag is enough)
+	 * and un-index it
+	 */
+	xr = open_reparse_index(ni->vol);
+	if (xr) {
+		xrni = xr->idx_ni;
+		mutex_lock_nested(&xrni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT);
+		err = remove_reparse_index(vi, xr, &reparse_tag);
+		if (err < 0) {
+			ntfs_index_ctx_put(xr);
+			mutex_unlock(&xrni->mrec_lock);
+			iput(VFS_I(xrni));
+			goto out;
+		}
+		mark_mft_record_dirty(xrni);
+		ntfs_index_ctx_put(xr);
+		mutex_unlock(&xrni->mrec_lock);
+		iput(VFS_I(xrni));
+	}
+
+	ni->flags &= ~FILE_ATTR_REPARSE_POINT;
+	NInoSetFileNameDirty(ni);
+	mark_mft_record_dirty(ni);
+
+out:
+	iput(vi);
+	return err;
+}
+
+/*
+ * Set the reparse data from an extended attribute
+ */
+static int ntfs_set_ntfs_reparse_data(struct ntfs_inode *ni, char *value, size_t size)
+{
+	int err = 0;
+	struct ntfs_inode *xrni;
+	struct ntfs_index_context *xr;
+
+	if (!ni)
+		return -EINVAL;
+
+	/*
+	 * reparse data compatibily with EA is not checked
+	 * any more, it is required by Windows 10, but may
+	 * lead to problems with earlier versions.
+	 */
+	if (valid_reparse_data(ni, (const struct reparse_point *)value, size) == false)
+		return -EINVAL;
+
+	xr = open_reparse_index(ni->vol);
+	if (!xr)
+		return -EINVAL;
+	xrni = xr->idx_ni;
+
+	if (!ntfs_attr_exist(ni, AT_REPARSE_POINT, AT_UNNAMED, 0)) {
+		u8 dummy = 0;
+
+		/*
+		 * no reparse data attribute : add one,
+		 * apparently, this does not feed the new value in
+		 * Note : NTFS version must be >= 3
+		 */
+		if (ni->vol->major_ver < 3) {
+			err = -EOPNOTSUPP;
+			ntfs_index_ctx_put(xr);
+			goto out;
+		}
+
+		err = ntfs_attr_add(ni, AT_REPARSE_POINT, AT_UNNAMED, 0, &dummy, 0);
+		if (err) {
+			ntfs_index_ctx_put(xr);
+			goto out;
+		}
+		ni->flags |= FILE_ATTR_REPARSE_POINT;
+		NInoSetFileNameDirty(ni);
+		mark_mft_record_dirty(ni);
+	}
+
+	/* update value and index */
+	mutex_lock_nested(&xrni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT);
+	err = update_reparse_data(ni, xr, value, size);
+	if (err) {
+		ni->flags &= ~FILE_ATTR_REPARSE_POINT;
+		NInoSetFileNameDirty(ni);
+		mark_mft_record_dirty(ni);
+	}
+	ntfs_index_ctx_put(xr);
+	mutex_unlock(&xrni->mrec_lock);
+
+out:
+	if (!err)
+		mark_mft_record_dirty(xrni);
+	iput(VFS_I(xrni));
+
+	return err;
+}
+
+/*
+ * Set reparse data for a WSL type symlink
+ */
+int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni,
+		const __le16 *target, int target_len)
+{
+	int err = 0;
+	int len;
+	int reparse_len;
+	unsigned char *utarget = NULL;
+	struct reparse_point *reparse;
+	struct wsl_link_reparse_data *data;
+
+	utarget = (char *)NULL;
+	len = ntfs_ucstonls(ni->vol, target, target_len, &utarget, 0);
+	if (len <= 0)
+		return -EINVAL;
+
+	reparse_len = sizeof(struct reparse_point) + sizeof(data->type) + len;
+	reparse = kvzalloc(reparse_len, GFP_NOFS);
+	if (!reparse) {
+		err = -ENOMEM;
+		kvfree(utarget);
+	} else {
+		data = (struct wsl_link_reparse_data *)reparse->reparse_data;
+		reparse->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK;
+		reparse->reparse_data_length =
+			cpu_to_le16(sizeof(data->type) + len);
+		reparse->reserved = 0;
+		data->type = cpu_to_le32(2);
+		memcpy(data->link, utarget, len);
+		err = ntfs_set_ntfs_reparse_data(ni,
+				(char *)reparse, reparse_len);
+		kvfree(reparse);
+		if (!err)
+			ni->target = utarget;
+	}
+	return err;
+}
+
+/*
+ * Set reparse data for a WSL special file other than a symlink
+ * (socket, fifo, character or block device)
+ */
+int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode)
+{
+	int err;
+	int len;
+	int reparse_len;
+	__le32 reparse_tag;
+	struct reparse_point *reparse;
+
+	len = 0;
+	if (S_ISSOCK(mode))
+		reparse_tag = IO_REPARSE_TAG_AF_UNIX;
+	else if (S_ISFIFO(mode))
+		reparse_tag = IO_REPARSE_TAG_LX_FIFO;
+	else if (S_ISCHR(mode))
+		reparse_tag = IO_REPARSE_TAG_LX_CHR;
+	else if (S_ISBLK(mode))
+		reparse_tag = IO_REPARSE_TAG_LX_BLK;
+	else
+		return -EOPNOTSUPP;
+
+	reparse_len = sizeof(struct reparse_point) + len;
+	reparse = kvzalloc(reparse_len, GFP_NOFS);
+	if (!reparse)
+		err = -ENOMEM;
+	else {
+		reparse->reparse_tag = reparse_tag;
+		reparse->reparse_data_length = cpu_to_le16(len);
+		reparse->reserved = cpu_to_le16(0);
+		err = ntfs_set_ntfs_reparse_data(ni, (char *)reparse,
+						 reparse_len);
+		kvfree(reparse);
+	}
+
+	return err;
+}
diff --git a/fs/ntfs/reparse.h b/fs/ntfs/reparse.h
new file mode 100644
index 0000000000000..28da40257f2a4
--- /dev/null
+++ b/fs/ntfs/reparse.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2008-2021 Jean-Pierre Andre
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_REPARSE_H
+#define _LINUX_NTFS_REPARSE_H
+
+extern __le16 reparse_index_name[];
+
+unsigned int ntfs_make_symlink(struct ntfs_inode *ni);
+unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mref);
+int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni,
+			const __le16 *target, int target_len);
+int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode);
+int ntfs_delete_reparse_index(struct ntfs_inode *ni);
+int ntfs_remove_ntfs_reparse_data(struct ntfs_inode *ni);
+
+#endif /* _LINUX_NTFS_REPARSE_H */
diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c
new file mode 100644
index 0000000000000..b213b4976d2b6
--- /dev/null
+++ b/fs/ntfs/runlist.c
@@ -0,0 +1,2066 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS runlist handling code.
+ *
+ * Copyright (c) 2001-2007 Anton Altaparmakov
+ * Copyright (c) 2002-2005 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ *
+ * Part of this file is based on code from the NTFS-3G.
+ * and is copyrighted by the respective authors below:
+ * Copyright (c) 2002-2005 Anton Altaparmakov
+ * Copyright (c) 2002-2005 Richard Russon
+ * Copyright (c) 2002-2008 Szabolcs Szakacsits
+ * Copyright (c) 2004 Yura Pakhuchiy
+ * Copyright (c) 2007-2022 Jean-Pierre Andre
+ */
+
+#include "ntfs.h"
+#include "attrib.h"
+
+/*
+ * ntfs_rl_mm - runlist memmove
+ * @base: base runlist array
+ * @dst: destination index in @base
+ * @src: source index in @base
+ * @size: number of elements to move
+ *
+ * It is up to the caller to serialize access to the runlist @base.
+ */
+static inline void ntfs_rl_mm(struct runlist_element *base, int dst, int src, int size)
+{
+	if (likely((dst != src) && (size > 0)))
+		memmove(base + dst, base + src, size * sizeof(*base));
+}
+
+/*
+ * ntfs_rl_mc - runlist memory copy
+ * @dstbase: destination runlist array
+ * @dst: destination index in @dstbase
+ * @srcbase: source runlist array
+ * @src: source index in @srcbase
+ * @size: number of elements to copy
+ *
+ * It is up to the caller to serialize access to the runlists @dstbase and
+ * @srcbase.
+ */
+static inline void ntfs_rl_mc(struct runlist_element *dstbase, int dst,
+		struct runlist_element *srcbase, int src, int size)
+{
+	if (likely(size > 0))
+		memcpy(dstbase + dst, srcbase + src, size * sizeof(*dstbase));
+}
+
+/*
+ * ntfs_rl_realloc - Reallocate memory for runlists
+ * @rl:		original runlist
+ * @old_size:	number of runlist elements in the original runlist @rl
+ * @new_size:	number of runlist elements we need space for
+ *
+ * As the runlists grow, more memory will be required.  To prevent the
+ * kernel having to allocate and reallocate large numbers of small bits of
+ * memory, this function returns an entire page of memory.
+ *
+ * It is up to the caller to serialize access to the runlist @rl.
+ *
+ * N.B.  If the new allocation doesn't require a different number of pages in
+ *       memory, the function will return the original pointer.
+ *
+ * On success, return a pointer to the newly allocated, or recycled, memory.
+ * On error, return -errno.
+ */
+struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl,
+		int old_size, int new_size)
+{
+	struct runlist_element *new_rl;
+
+	old_size = old_size * sizeof(*rl);
+	new_size = new_size * sizeof(*rl);
+	if (old_size == new_size)
+		return rl;
+
+	new_rl = kvzalloc(new_size, GFP_NOFS);
+	if (unlikely(!new_rl))
+		return ERR_PTR(-ENOMEM);
+
+	if (likely(rl != NULL)) {
+		if (unlikely(old_size > new_size))
+			old_size = new_size;
+		memcpy(new_rl, rl, old_size);
+		kvfree(rl);
+	}
+	return new_rl;
+}
+
+/*
+ * ntfs_rl_realloc_nofail - Reallocate memory for runlists
+ * @rl:		original runlist
+ * @old_size:	number of runlist elements in the original runlist @rl
+ * @new_size:	number of runlist elements we need space for
+ *
+ * As the runlists grow, more memory will be required.  To prevent the
+ * kernel having to allocate and reallocate large numbers of small bits of
+ * memory, this function returns an entire page of memory.
+ *
+ * This function guarantees that the allocation will succeed.  It will sleep
+ * for as long as it takes to complete the allocation.
+ *
+ * It is up to the caller to serialize access to the runlist @rl.
+ *
+ * N.B.  If the new allocation doesn't require a different number of pages in
+ *       memory, the function will return the original pointer.
+ *
+ * On success, return a pointer to the newly allocated, or recycled, memory.
+ * On error, return -errno.
+ */
+static inline struct runlist_element *ntfs_rl_realloc_nofail(struct runlist_element *rl,
+		int old_size, int new_size)
+{
+	struct runlist_element *new_rl;
+
+	old_size = old_size * sizeof(*rl);
+	new_size = new_size * sizeof(*rl);
+	if (old_size == new_size)
+		return rl;
+
+	new_rl = kvmalloc(new_size, GFP_NOFS | __GFP_NOFAIL);
+	if (likely(rl != NULL)) {
+		if (unlikely(old_size > new_size))
+			old_size = new_size;
+		memcpy(new_rl, rl, old_size);
+		kvfree(rl);
+	}
+	return new_rl;
+}
+
+/*
+ * ntfs_are_rl_mergeable - test if two runlists can be joined together
+ * @dst:	original runlist
+ * @src:	new runlist to test for mergeability with @dst
+ *
+ * Test if two runlists can be joined together. For this, their VCNs and LCNs
+ * must be adjacent.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ *
+ * Return: true   Success, the runlists can be merged.
+ *	   false  Failure, the runlists cannot be merged.
+ */
+static inline bool ntfs_are_rl_mergeable(struct runlist_element *dst,
+		struct runlist_element *src)
+{
+	/* We can merge unmapped regions even if they are misaligned. */
+	if ((dst->lcn == LCN_RL_NOT_MAPPED) && (src->lcn == LCN_RL_NOT_MAPPED))
+		return true;
+	/* If the runs are misaligned, we cannot merge them. */
+	if ((dst->vcn + dst->length) != src->vcn)
+		return false;
+	/* If both runs are non-sparse and contiguous, we can merge them. */
+	if ((dst->lcn >= 0) && (src->lcn >= 0) &&
+			((dst->lcn + dst->length) == src->lcn))
+		return true;
+	/* If we are merging two holes, we can merge them. */
+	if ((dst->lcn == LCN_HOLE) && (src->lcn == LCN_HOLE))
+		return true;
+	/* If we are merging two dealloc, we can merge them. */
+	if ((dst->lcn == LCN_DELALLOC) && (src->lcn == LCN_DELALLOC))
+		return true;
+	/* Cannot merge. */
+	return false;
+}
+
+/*
+ * __ntfs_rl_merge - merge two runlists without testing if they can be merged
+ * @dst:	original, destination runlist
+ * @src:	new runlist to merge with @dst
+ *
+ * Merge the two runlists, writing into the destination runlist @dst. The
+ * caller must make sure the runlists can be merged or this will corrupt the
+ * destination runlist.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ */
+static inline void __ntfs_rl_merge(struct runlist_element *dst, struct runlist_element *src)
+{
+	dst->length += src->length;
+}
+
+/*
+ * ntfs_rl_append - append a runlist after a given element
+ * @dst: destination runlist to append to
+ * @dsize: number of elements in @dst
+ * @src: source runlist to append from
+ * @ssize: number of elements in @src
+ * @loc: index in @dst after which to append @src
+ * @new_size: on success, set to the new combined size
+ *
+ * Append the runlist @src after element @loc in @dst.  Merge the right end of
+ * the new runlist, if necessary. Adjust the size of the hole before the
+ * appended runlist.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ *
+ * On success, return a pointer to the new, combined, runlist. Note, both
+ * runlists @dst and @src are deallocated before returning so you cannot use
+ * the pointers for anything any more. (Strictly speaking the returned runlist
+ * may be the same as @dst but this is irrelevant.)
+ *
+ * On error, return -errno. Both runlists are left unmodified.
+ */
+static inline struct runlist_element *ntfs_rl_append(struct runlist_element *dst,
+		int dsize, struct runlist_element *src, int ssize, int loc,
+		size_t *new_size)
+{
+	bool right = false;	/* Right end of @src needs merging. */
+	int marker;		/* End of the inserted runs. */
+
+	/* First, check if the right hand end needs merging. */
+	if ((loc + 1) < dsize)
+		right = ntfs_are_rl_mergeable(src + ssize - 1, dst + loc + 1);
+
+	/* Space required: @dst size + @src size, less one if we merged. */
+	dst = ntfs_rl_realloc(dst, dsize, dsize + ssize - right);
+	if (IS_ERR(dst))
+		return dst;
+
+	*new_size = dsize + ssize - right;
+	/*
+	 * We are guaranteed to succeed from here so can start modifying the
+	 * original runlists.
+	 */
+
+	/* First, merge the right hand end, if necessary. */
+	if (right)
+		__ntfs_rl_merge(src + ssize - 1, dst + loc + 1);
+
+	/* First run after the @src runs that have been inserted. */
+	marker = loc + ssize + 1;
+
+	/* Move the tail of @dst out of the way, then copy in @src. */
+	ntfs_rl_mm(dst, marker, loc + 1 + right, dsize - (loc + 1 + right));
+	ntfs_rl_mc(dst, loc + 1, src, 0, ssize);
+
+	/* Adjust the size of the preceding hole. */
+	dst[loc].length = dst[loc + 1].vcn - dst[loc].vcn;
+
+	/* We may have changed the length of the file, so fix the end marker */
+	if (dst[marker].lcn == LCN_ENOENT)
+		dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length;
+
+	return dst;
+}
+
+/*
+ * ntfs_rl_insert - insert a runlist into another
+ * @dst: destination runlist to insert into
+ * @dsize: number of elements in @dst
+ * @src: source runlist to insert from
+ * @ssize: number of elements in @src
+ * @loc: index in @dst at which to insert @src
+ * @new_size: on success, set to the new combined size
+ *
+ * Insert the runlist @src before element @loc in the runlist @dst. Merge the
+ * left end of the new runlist, if necessary. Adjust the size of the hole
+ * after the inserted runlist.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ *
+ * On success, return a pointer to the new, combined, runlist. Note, both
+ * runlists @dst and @src are deallocated before returning so you cannot use
+ * the pointers for anything any more. (Strictly speaking the returned runlist
+ * may be the same as @dst but this is irrelevant.)
+ *
+ * On error, return -errno. Both runlists are left unmodified.
+ */
+static inline struct runlist_element *ntfs_rl_insert(struct runlist_element *dst,
+		int dsize, struct runlist_element *src, int ssize, int loc,
+		size_t *new_size)
+{
+	bool left = false;	/* Left end of @src needs merging. */
+	bool disc = false;	/* Discontinuity between @dst and @src. */
+	int marker;		/* End of the inserted runs. */
+
+	/*
+	 * disc => Discontinuity between the end of @dst and the start of @src.
+	 *	   This means we might need to insert a "not mapped" run.
+	 */
+	if (loc == 0)
+		disc = (src[0].vcn > 0);
+	else {
+		s64 merged_length;
+
+		left = ntfs_are_rl_mergeable(dst + loc - 1, src);
+
+		merged_length = dst[loc - 1].length;
+		if (left)
+			merged_length += src->length;
+
+		disc = (src[0].vcn > dst[loc - 1].vcn + merged_length);
+	}
+	/*
+	 * Space required: @dst size + @src size, less one if we merged, plus
+	 * one if there was a discontinuity.
+	 */
+	dst = ntfs_rl_realloc(dst, dsize, dsize + ssize - left + disc);
+	if (IS_ERR(dst))
+		return dst;
+
+	*new_size = dsize + ssize - left + disc;
+	/*
+	 * We are guaranteed to succeed from here so can start modifying the
+	 * original runlist.
+	 */
+	if (left)
+		__ntfs_rl_merge(dst + loc - 1, src);
+	/*
+	 * First run after the @src runs that have been inserted.
+	 * Nominally,  @marker equals @loc + @ssize, i.e. location + number of
+	 * runs in @src.  However, if @left, then the first run in @src has
+	 * been merged with one in @dst.  And if @disc, then @dst and @src do
+	 * not meet and we need an extra run to fill the gap.
+	 */
+	marker = loc + ssize - left + disc;
+
+	/* Move the tail of @dst out of the way, then copy in @src. */
+	ntfs_rl_mm(dst, marker, loc, dsize - loc);
+	ntfs_rl_mc(dst, loc + disc, src, left, ssize - left);
+
+	/* Adjust the VCN of the first run after the insertion... */
+	dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length;
+	/* ... and the length. */
+	if (dst[marker].lcn == LCN_HOLE || dst[marker].lcn == LCN_RL_NOT_MAPPED ||
+	    dst[marker].lcn == LCN_DELALLOC)
+		dst[marker].length = dst[marker + 1].vcn - dst[marker].vcn;
+
+	/* Writing beyond the end of the file and there is a discontinuity. */
+	if (disc) {
+		if (loc > 0) {
+			dst[loc].vcn = dst[loc - 1].vcn + dst[loc - 1].length;
+			dst[loc].length = dst[loc + 1].vcn - dst[loc].vcn;
+		} else {
+			dst[loc].vcn = 0;
+			dst[loc].length = dst[loc + 1].vcn;
+		}
+		dst[loc].lcn = LCN_RL_NOT_MAPPED;
+	}
+	return dst;
+}
+
+/*
+ * ntfs_rl_replace - overwrite a runlist element with another runlist
+ * @dst: destination runlist to replace in
+ * @dsize: number of elements in @dst
+ * @src: source runlist to replace with
+ * @ssize: number of elements in @src
+ * @loc: index in @dst to replace
+ * @new_size: on success, set to the new combined size
+ *
+ * Replace the runlist element @dst at @loc with @src. Merge the left and
+ * right ends of the inserted runlist, if necessary.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ *
+ * On success, return a pointer to the new, combined, runlist. Note, both
+ * runlists @dst and @src are deallocated before returning so you cannot use
+ * the pointers for anything any more. (Strictly speaking the returned runlist
+ * may be the same as @dst but this is irrelevant.)
+ *
+ * On error, return -errno. Both runlists are left unmodified.
+ */
+static inline struct runlist_element *ntfs_rl_replace(struct runlist_element *dst,
+		int dsize, struct runlist_element *src, int ssize, int loc,
+		size_t *new_size)
+{
+	int delta;
+	bool left = false;	/* Left end of @src needs merging. */
+	bool right = false;	/* Right end of @src needs merging. */
+	int tail;		/* Start of tail of @dst. */
+	int marker;		/* End of the inserted runs. */
+
+	/* First, see if the left and right ends need merging. */
+	if ((loc + 1) < dsize)
+		right = ntfs_are_rl_mergeable(src + ssize - 1, dst + loc + 1);
+	if (loc > 0)
+		left = ntfs_are_rl_mergeable(dst + loc - 1, src);
+	/*
+	 * Allocate some space.  We will need less if the left, right, or both
+	 * ends get merged.  The -1 accounts for the run being replaced.
+	 */
+	delta = ssize - 1 - left - right;
+	if (delta > 0) {
+		dst = ntfs_rl_realloc(dst, dsize, dsize + delta);
+		if (IS_ERR(dst))
+			return dst;
+	}
+
+	*new_size = dsize + delta;
+	/*
+	 * We are guaranteed to succeed from here so can start modifying the
+	 * original runlists.
+	 */
+
+	/* First, merge the left and right ends, if necessary. */
+	if (right)
+		__ntfs_rl_merge(src + ssize - 1, dst + loc + 1);
+	if (left)
+		__ntfs_rl_merge(dst + loc - 1, src);
+	/*
+	 * Offset of the tail of @dst.  This needs to be moved out of the way
+	 * to make space for the runs to be copied from @src, i.e. the first
+	 * run of the tail of @dst.
+	 * Nominally, @tail equals @loc + 1, i.e. location, skipping the
+	 * replaced run.  However, if @right, then one of @dst's runs is
+	 * already merged into @src.
+	 */
+	tail = loc + right + 1;
+	/*
+	 * First run after the @src runs that have been inserted, i.e. where
+	 * the tail of @dst needs to be moved to.
+	 * Nominally, @marker equals @loc + @ssize, i.e. location + number of
+	 * runs in @src.  However, if @left, then the first run in @src has
+	 * been merged with one in @dst.
+	 */
+	marker = loc + ssize - left;
+
+	/* Move the tail of @dst out of the way, then copy in @src. */
+	ntfs_rl_mm(dst, marker, tail, dsize - tail);
+	ntfs_rl_mc(dst, loc, src, left, ssize - left);
+
+	/* We may have changed the length of the file, so fix the end marker. */
+	if (dsize - tail > 0 && dst[marker].lcn == LCN_ENOENT)
+		dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length;
+	return dst;
+}
+
+/*
+ * ntfs_rl_split - insert a runlist into the centre of a hole
+ * @dst: destination runlist with a hole
+ * @dsize: number of elements in @dst
+ * @src: source runlist to insert
+ * @ssize: number of elements in @src
+ * @loc: index in @dst of the hole to split
+ * @new_size: on success, set to the new combined size
+ *
+ * Split the runlist @dst at @loc into two and insert @new in between the two
+ * fragments. No merging of runlists is necessary. Adjust the size of the
+ * holes either side.
+ *
+ * It is up to the caller to serialize access to the runlists @dst and @src.
+ *
+ * On success, return a pointer to the new, combined, runlist. Note, both
+ * runlists @dst and @src are deallocated before returning so you cannot use
+ * the pointers for anything any more. (Strictly speaking the returned runlist
+ * may be the same as @dst but this is irrelevant.)
+ *
+ * On error, return -errno. Both runlists are left unmodified.
+ */
+static inline struct runlist_element *ntfs_rl_split(struct runlist_element *dst, int dsize,
+		struct runlist_element *src, int ssize, int loc,
+		size_t *new_size)
+{
+	/* Space required: @dst size + @src size + one new hole. */
+	dst = ntfs_rl_realloc(dst, dsize, dsize + ssize + 1);
+	if (IS_ERR(dst))
+		return dst;
+
+	*new_size = dsize + ssize + 1;
+	/*
+	 * We are guaranteed to succeed from here so can start modifying the
+	 * original runlists.
+	 */
+
+	/* Move the tail of @dst out of the way, then copy in @src. */
+	ntfs_rl_mm(dst, loc + 1 + ssize, loc, dsize - loc);
+	ntfs_rl_mc(dst, loc + 1, src, 0, ssize);
+
+	/* Adjust the size of the holes either size of @src. */
+	dst[loc].length		= dst[loc+1].vcn       - dst[loc].vcn;
+	dst[loc+ssize+1].vcn    = dst[loc+ssize].vcn   + dst[loc+ssize].length;
+	dst[loc+ssize+1].length = dst[loc+ssize+2].vcn - dst[loc+ssize+1].vcn;
+
+	return dst;
+}
+
+/*
+ * ntfs_runlists_merge - merge two runlists into one
+ * @d_runlist: destination runlist structure to merge into
+ * @srl: source runlist to merge from
+ * @s_rl_count: number of elements in @srl (0 to auto-detect)
+ * @new_rl_count: on success, set to the new combined runlist size
+ *
+ * First we sanity check the two runlists @srl and @drl to make sure that they
+ * are sensible and can be merged. The runlist @srl must be either after the
+ * runlist @drl or completely within a hole (or unmapped region) in @drl.
+ *
+ * It is up to the caller to serialize access to the runlists @drl and @srl.
+ *
+ * Merging of runlists is necessary in two cases:
+ *   1. When attribute lists are used and a further extent is being mapped.
+ *   2. When new clusters are allocated to fill a hole or extend a file.
+ *
+ * There are four possible ways @srl can be merged. It can:
+ *	- be inserted at the beginning of a hole,
+ *	- split the hole in two and be inserted between the two fragments,
+ *	- be appended at the end of a hole, or it can
+ *	- replace the whole hole.
+ * It can also be appended to the end of the runlist, which is just a variant
+ * of the insert case.
+ *
+ * On success, return a pointer to the new, combined, runlist. Note, both
+ * runlists @drl and @srl are deallocated before returning so you cannot use
+ * the pointers for anything any more. (Strictly speaking the returned runlist
+ * may be the same as @dst but this is irrelevant.)
+ *
+ * On error, return -errno. Both runlists are left unmodified.
+ */
+struct runlist_element *ntfs_runlists_merge(struct runlist *d_runlist,
+				     struct runlist_element *srl, size_t s_rl_count,
+				     size_t *new_rl_count)
+{
+	int di, si;		/* Current index into @[ds]rl. */
+	int sstart;		/* First index with lcn > LCN_RL_NOT_MAPPED. */
+	int dins;		/* Index into @drl at which to insert @srl. */
+	int dend, send;		/* Last index into @[ds]rl. */
+	int dfinal, sfinal;	/* The last index into @[ds]rl with lcn >= LCN_HOLE. */
+	int marker = 0;
+	s64 marker_vcn = 0;
+	struct runlist_element *drl = d_runlist->rl, *rl;
+
+#ifdef DEBUG
+	ntfs_debug("dst:");
+	ntfs_debug_dump_runlist(drl);
+	ntfs_debug("src:");
+	ntfs_debug_dump_runlist(srl);
+#endif
+
+	/* Check for silly calling... */
+	if (unlikely(!srl))
+		return drl;
+	if (IS_ERR(srl) || IS_ERR(drl))
+		return ERR_PTR(-EINVAL);
+
+	if (s_rl_count == 0) {
+		for (; srl[s_rl_count].length; s_rl_count++)
+			;
+		s_rl_count++;
+	}
+
+	/* Check for the case where the first mapping is being done now. */
+	if (unlikely(!drl)) {
+		drl = srl;
+		/* Complete the source runlist if necessary. */
+		if (unlikely(drl[0].vcn)) {
+			/* Scan to the end of the source runlist. */
+			drl = ntfs_rl_realloc(drl, s_rl_count, s_rl_count + 1);
+			if (IS_ERR(drl))
+				return drl;
+			/* Insert start element at the front of the runlist. */
+			ntfs_rl_mm(drl, 1, 0, s_rl_count);
+			drl[0].vcn = 0;
+			drl[0].lcn = LCN_RL_NOT_MAPPED;
+			drl[0].length = drl[1].vcn;
+			s_rl_count++;
+		}
+
+		*new_rl_count = s_rl_count;
+		goto finished;
+	}
+
+	if (d_runlist->count < 1 || s_rl_count < 2)
+		return ERR_PTR(-EINVAL);
+
+	si = di = 0;
+
+	/* Skip any unmapped start element(s) in the source runlist. */
+	while (srl[si].length && srl[si].lcn < LCN_HOLE)
+		si++;
+
+	/* Can't have an entirely unmapped source runlist. */
+	WARN_ON(!srl[si].length);
+
+	/* Record the starting points. */
+	sstart = si;
+
+	/*
+	 * Skip forward in @drl until we reach the position where @srl needs to
+	 * be inserted. If we reach the end of @drl, @srl just needs to be
+	 * appended to @drl.
+	 */
+	rl = __ntfs_attr_find_vcn_nolock(d_runlist, srl[sstart].vcn);
+	if (IS_ERR(rl))
+		di = (int)d_runlist->count - 1;
+	else
+		di = (int)(rl - d_runlist->rl);
+	dins = di;
+
+	/* Sanity check for illegal overlaps. */
+	if ((drl[di].vcn == srl[si].vcn) && (drl[di].lcn >= 0) &&
+			(srl[si].lcn >= 0)) {
+		ntfs_error(NULL, "Run lists overlap. Cannot merge!");
+		return ERR_PTR(-ERANGE);
+	}
+
+	/* Scan to the end of both runlists in order to know their sizes. */
+	send = (int)s_rl_count - 1;
+	dend = (int)d_runlist->count - 1;
+
+	if (srl[send].lcn == LCN_ENOENT)
+		marker_vcn = srl[marker = send].vcn;
+
+	/* Scan to the last element with lcn >= LCN_HOLE. */
+	for (sfinal = send; sfinal >= 0 && srl[sfinal].lcn < LCN_HOLE; sfinal--)
+		;
+	for (dfinal = dend; dfinal >= 0 && drl[dfinal].lcn < LCN_HOLE; dfinal--)
+		;
+
+	{
+	bool start;
+	bool finish;
+	int ds = dend + 1;		/* Number of elements in drl & srl */
+	int ss = sfinal - sstart + 1;
+
+	start  = ((drl[dins].lcn <  LCN_RL_NOT_MAPPED) ||    /* End of file   */
+		  (drl[dins].vcn == srl[sstart].vcn));	     /* Start of hole */
+	finish = ((drl[dins].lcn >= LCN_RL_NOT_MAPPED) &&    /* End of file   */
+		 ((drl[dins].vcn + drl[dins].length) <=      /* End of hole   */
+		  (srl[send - 1].vcn + srl[send - 1].length)));
+
+	/* Or we will lose an end marker. */
+	if (finish && !drl[dins].length)
+		ss++;
+	if (marker && (drl[dins].vcn + drl[dins].length > srl[send - 1].vcn))
+		finish = false;
+
+	if (start) {
+		if (finish)
+			drl = ntfs_rl_replace(drl, ds, srl + sstart, ss, dins, new_rl_count);
+		else
+			drl = ntfs_rl_insert(drl, ds, srl + sstart, ss, dins, new_rl_count);
+	} else {
+		if (finish)
+			drl = ntfs_rl_append(drl, ds, srl + sstart, ss, dins, new_rl_count);
+		else
+			drl = ntfs_rl_split(drl, ds, srl + sstart, ss, dins, new_rl_count);
+	}
+	if (IS_ERR(drl)) {
+		ntfs_error(NULL, "Merge failed.");
+		return drl;
+	}
+	kvfree(srl);
+	if (marker) {
+		ntfs_debug("Triggering marker code.");
+		for (ds = dend; drl[ds].length; ds++)
+			;
+		/* We only need to care if @srl ended after @drl. */
+		if (drl[ds].vcn <= marker_vcn) {
+			int slots = 0;
+
+			if (drl[ds].vcn == marker_vcn) {
+				ntfs_debug("Old marker = 0x%llx, replacing with LCN_ENOENT.",
+						drl[ds].lcn);
+				drl[ds].lcn = LCN_ENOENT;
+				goto finished;
+			}
+			/*
+			 * We need to create an unmapped runlist element in
+			 * @drl or extend an existing one before adding the
+			 * ENOENT terminator.
+			 */
+			if (drl[ds].lcn == LCN_ENOENT) {
+				ds--;
+				slots = 1;
+			}
+			if (drl[ds].lcn != LCN_RL_NOT_MAPPED) {
+				/* Add an unmapped runlist element. */
+				if (!slots) {
+					drl = ntfs_rl_realloc_nofail(drl, ds,
+							ds + 2);
+					slots = 2;
+					*new_rl_count += 2;
+				}
+				ds++;
+				/* Need to set vcn if it isn't set already. */
+				if (slots != 1)
+					drl[ds].vcn = drl[ds - 1].vcn +
+							drl[ds - 1].length;
+				drl[ds].lcn = LCN_RL_NOT_MAPPED;
+				/* We now used up a slot. */
+				slots--;
+			}
+			drl[ds].length = marker_vcn - drl[ds].vcn;
+			/* Finally add the ENOENT terminator. */
+			ds++;
+			if (!slots) {
+				drl = ntfs_rl_realloc_nofail(drl, ds, ds + 1);
+				*new_rl_count += 1;
+			}
+			drl[ds].vcn = marker_vcn;
+			drl[ds].lcn = LCN_ENOENT;
+			drl[ds].length = (s64)0;
+		}
+	}
+	}
+
+finished:
+	/* The merge was completed successfully. */
+	ntfs_debug("Merged runlist:");
+	ntfs_debug_dump_runlist(drl);
+	return drl;
+}
+
+/*
+ * ntfs_mapping_pairs_decompress - convert mapping pairs array to runlist
+ * @vol: ntfs volume
+ * @attr: attribute record whose mapping pairs to decompress
+ * @old_runlist: optional runlist to merge the decompressed runlist into
+ * @new_rl_count: on success, set to the new runlist size
+ *
+ * It is up to the caller to serialize access to the runlist @old_rl.
+ *
+ * Decompress the attribute @attr's mapping pairs array into a runlist. On
+ * success, return the decompressed runlist.
+ *
+ * If @old_rl is not NULL, decompressed runlist is inserted into the
+ * appropriate place in @old_rl and the resultant, combined runlist is
+ * returned. The original @old_rl is deallocated.
+ *
+ * On error, return -errno. @old_rl is left unmodified in that case.
+ */
+struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *vol,
+		const struct attr_record *attr, struct runlist *old_runlist,
+		size_t *new_rl_count)
+{
+	s64 vcn;		/* Current vcn. */
+	s64 lcn;		/* Current lcn. */
+	s64 deltaxcn;		/* Change in [vl]cn. */
+	struct runlist_element *rl, *new_rl;	/* The output runlist. */
+	u8 *buf;		/* Current position in mapping pairs array. */
+	u8 *attr_end;		/* End of attribute. */
+	int rlsize;		/* Size of runlist buffer. */
+	u16 rlpos;		/* Current runlist position in units of struct runlist_elements. */
+	u8 b;			/* Current byte offset in buf. */
+
+#ifdef DEBUG
+	/* Make sure attr exists and is non-resident. */
+	if (!attr || !attr->non_resident) {
+		ntfs_error(vol->sb, "Invalid arguments.");
+		return ERR_PTR(-EINVAL);
+	}
+#endif
+	/* Start at vcn = lowest_vcn and lcn 0. */
+	vcn = le64_to_cpu(attr->data.non_resident.lowest_vcn);
+	lcn = 0;
+	/* Get start of the mapping pairs array. */
+	buf = (u8 *)attr +
+		le16_to_cpu(attr->data.non_resident.mapping_pairs_offset);
+	attr_end = (u8 *)attr + le32_to_cpu(attr->length);
+	if (unlikely(buf < (u8 *)attr || buf > attr_end)) {
+		ntfs_error(vol->sb, "Corrupt attribute.");
+		return ERR_PTR(-EIO);
+	}
+
+	/* Current position in runlist array. */
+	rlpos = 0;
+	/* Allocate first page and set current runlist size to one page. */
+	rl = kvzalloc(rlsize = PAGE_SIZE, GFP_NOFS);
+	if (unlikely(!rl))
+		return ERR_PTR(-ENOMEM);
+	/* Insert unmapped starting element if necessary. */
+	if (vcn) {
+		rl->vcn = 0;
+		rl->lcn = LCN_RL_NOT_MAPPED;
+		rl->length = vcn;
+		rlpos++;
+	}
+	while (buf < attr_end && *buf) {
+		/*
+		 * Allocate more memory if needed, including space for the
+		 * not-mapped and terminator elements. kvzalloc()
+		 * operates on whole pages only.
+		 */
+		if (((rlpos + 3) * sizeof(*rl)) > rlsize) {
+			struct runlist_element *rl2;
+
+			rl2 = kvzalloc(rlsize + PAGE_SIZE, GFP_NOFS);
+			if (unlikely(!rl2)) {
+				kvfree(rl);
+				return ERR_PTR(-ENOMEM);
+			}
+			memcpy(rl2, rl, rlsize);
+			kvfree(rl);
+			rl = rl2;
+			rlsize += PAGE_SIZE;
+		}
+		/* Enter the current vcn into the current runlist element. */
+		rl[rlpos].vcn = vcn;
+		/*
+		 * Get the change in vcn, i.e. the run length in clusters.
+		 * Doing it this way ensures that we signextend negative values.
+		 * A negative run length doesn't make any sense, but hey, I
+		 * didn't make up the NTFS specs and Windows NT4 treats the run
+		 * length as a signed value so that's how it is...
+		 */
+		b = *buf & 0xf;
+		if (b) {
+			if (unlikely(buf + b > attr_end))
+				goto io_error;
+			for (deltaxcn = (s8)buf[b--]; b; b--)
+				deltaxcn = (deltaxcn << 8) + buf[b];
+		} else { /* The length entry is compulsory. */
+			ntfs_error(vol->sb, "Missing length entry in mapping pairs array.");
+			deltaxcn = (s64)-1;
+		}
+		/*
+		 * Assume a negative length to indicate data corruption and
+		 * hence clean-up and return NULL.
+		 */
+		if (unlikely(deltaxcn < 0)) {
+			ntfs_error(vol->sb, "Invalid length in mapping pairs array.");
+			goto err_out;
+		}
+		/*
+		 * Enter the current run length into the current runlist
+		 * element.
+		 */
+		rl[rlpos].length = deltaxcn;
+		/* Increment the current vcn by the current run length. */
+		vcn += deltaxcn;
+		/*
+		 * There might be no lcn change at all, as is the case for
+		 * sparse clusters on NTFS 3.0+, in which case we set the lcn
+		 * to LCN_HOLE.
+		 */
+		if (!(*buf & 0xf0))
+			rl[rlpos].lcn = LCN_HOLE;
+		else {
+			/* Get the lcn change which really can be negative. */
+			u8 b2 = *buf & 0xf;
+
+			b = b2 + ((*buf >> 4) & 0xf);
+			if (buf + b > attr_end)
+				goto io_error;
+			for (deltaxcn = (s8)buf[b--]; b > b2; b--)
+				deltaxcn = (deltaxcn << 8) + buf[b];
+			/* Change the current lcn to its new value. */
+			lcn += deltaxcn;
+#ifdef DEBUG
+			/*
+			 * On NTFS 1.2-, apparently can have lcn == -1 to
+			 * indicate a hole. But we haven't verified ourselves
+			 * whether it is really the lcn or the deltaxcn that is
+			 * -1. So if either is found give us a message so we
+			 * can investigate it further!
+			 */
+			if (vol->major_ver < 3) {
+				if (unlikely(deltaxcn == -1))
+					ntfs_error(vol->sb, "lcn delta == -1");
+				if (unlikely(lcn == -1))
+					ntfs_error(vol->sb, "lcn == -1");
+			}
+#endif
+			/* Check lcn is not below -1. */
+			if (unlikely(lcn < -1)) {
+				ntfs_error(vol->sb, "Invalid s64 < -1 in mapping pairs array.");
+				goto err_out;
+			}
+
+			/* chkdsk accepts zero-sized runs only for holes */
+			if ((lcn != -1) && !rl[rlpos].length) {
+				ntfs_error(vol->sb,
+					   "Invalid zero-sized data run(lcn : %lld).\n",
+					   lcn);
+				goto err_out;
+			}
+
+			/* Enter the current lcn into the runlist element. */
+			rl[rlpos].lcn = lcn;
+		}
+		/* Get to the next runlist element, skipping zero-sized holes */
+		if (rl[rlpos].length)
+			rlpos++;
+		/* Increment the buffer position to the next mapping pair. */
+		buf += (*buf & 0xf) + ((*buf >> 4) & 0xf) + 1;
+	}
+	if (unlikely(buf >= attr_end))
+		goto io_error;
+	/*
+	 * If there is a highest_vcn specified, it must be equal to the final
+	 * vcn in the runlist - 1, or something has gone badly wrong.
+	 */
+	deltaxcn = le64_to_cpu(attr->data.non_resident.highest_vcn);
+	if (unlikely(deltaxcn && vcn - 1 != deltaxcn)) {
+mpa_err:
+		ntfs_error(vol->sb, "Corrupt mapping pairs array in non-resident attribute.");
+		goto err_out;
+	}
+	/* Setup not mapped runlist element if this is the base extent. */
+	if (!attr->data.non_resident.lowest_vcn) {
+		s64 max_cluster;
+
+		max_cluster = ((le64_to_cpu(attr->data.non_resident.allocated_size) +
+				vol->cluster_size - 1) >>
+				vol->cluster_size_bits) - 1;
+		/*
+		 * A highest_vcn of zero means this is a single extent
+		 * attribute so simply terminate the runlist with LCN_ENOENT).
+		 */
+		if (deltaxcn) {
+			/*
+			 * If there is a difference between the highest_vcn and
+			 * the highest cluster, the runlist is either corrupt
+			 * or, more likely, there are more extents following
+			 * this one.
+			 */
+			if (deltaxcn < max_cluster) {
+				ntfs_debug("More extents to follow; deltaxcn = 0x%llx, max_cluster = 0x%llx",
+						deltaxcn, max_cluster);
+				rl[rlpos].vcn = vcn;
+				vcn += rl[rlpos].length = max_cluster -
+						deltaxcn;
+				rl[rlpos].lcn = LCN_RL_NOT_MAPPED;
+				rlpos++;
+			} else if (unlikely(deltaxcn > max_cluster)) {
+				ntfs_error(vol->sb,
+					   "Corrupt attribute. deltaxcn = 0x%llx, max_cluster = 0x%llx",
+					   deltaxcn, max_cluster);
+				goto mpa_err;
+			}
+		}
+		rl[rlpos].lcn = LCN_ENOENT;
+	} else /* Not the base extent. There may be more extents to follow. */
+		rl[rlpos].lcn = LCN_RL_NOT_MAPPED;
+
+	/* Setup terminating runlist element. */
+	rl[rlpos].vcn = vcn;
+	rl[rlpos].length = (s64)0;
+	/* If no existing runlist was specified, we are done. */
+	if (!old_runlist || !old_runlist->rl) {
+		*new_rl_count = rlpos + 1;
+		ntfs_debug("Mapping pairs array successfully decompressed:");
+		ntfs_debug_dump_runlist(rl);
+		return rl;
+	}
+	/* Now combine the new and old runlists checking for overlaps. */
+	new_rl = ntfs_runlists_merge(old_runlist, rl, rlpos + 1, new_rl_count);
+	if (!IS_ERR(new_rl))
+		return new_rl;
+	kvfree(rl);
+	ntfs_error(vol->sb, "Failed to merge runlists.");
+	return new_rl;
+io_error:
+	ntfs_error(vol->sb, "Corrupt attribute.");
+err_out:
+	kvfree(rl);
+	return ERR_PTR(-EIO);
+}
+
+/*
+ * ntfs_rl_vcn_to_lcn - convert a vcn into a lcn given a runlist
+ * @rl:		runlist to use for conversion
+ * @vcn:	vcn to convert
+ *
+ * Convert the virtual cluster number @vcn of an attribute into a logical
+ * cluster number (lcn) of a device using the runlist @rl to map vcns to their
+ * corresponding lcns.
+ *
+ * It is up to the caller to serialize access to the runlist @rl.
+ *
+ * Since lcns must be >= 0, we use negative return codes with special meaning:
+ *
+ * Return code		Meaning / Description
+ * ==================================================
+ *  LCN_HOLE		Hole / not allocated on disk.
+ *  LCN_RL_NOT_MAPPED	This is part of the runlist which has not been
+ *			inserted into the runlist yet.
+ *  LCN_ENOENT		There is no such vcn in the attribute.
+ *
+ * Locking: - The caller must have locked the runlist (for reading or writing).
+ *	    - This function does not touch the lock, nor does it modify the
+ *	      runlist.
+ */
+s64 ntfs_rl_vcn_to_lcn(const struct runlist_element *rl, const s64 vcn)
+{
+	int i;
+
+	/*
+	 * If rl is NULL, assume that we have found an unmapped runlist. The
+	 * caller can then attempt to map it and fail appropriately if
+	 * necessary.
+	 */
+	if (unlikely(!rl))
+		return LCN_RL_NOT_MAPPED;
+
+	/* Catch out of lower bounds vcn. */
+	if (unlikely(vcn < rl[0].vcn))
+		return LCN_ENOENT;
+
+	for (i = 0; likely(rl[i].length); i++) {
+		if (vcn < rl[i+1].vcn) {
+			if (likely(rl[i].lcn >= 0))
+				return rl[i].lcn + (vcn - rl[i].vcn);
+			return rl[i].lcn;
+		}
+	}
+	/*
+	 * The terminator element is setup to the correct value, i.e. one of
+	 * LCN_HOLE, LCN_RL_NOT_MAPPED, or LCN_ENOENT.
+	 */
+	if (likely(rl[i].lcn < 0))
+		return rl[i].lcn;
+	/* Just in case... We could replace this with BUG() some day. */
+	return LCN_ENOENT;
+}
+
+/*
+ * ntfs_rl_find_vcn_nolock - find a vcn in a runlist
+ * @rl:		runlist to search
+ * @vcn:	vcn to find
+ *
+ * Find the virtual cluster number @vcn in the runlist @rl and return the
+ * address of the runlist element containing the @vcn on success.
+ *
+ * Return NULL if @rl is NULL or @vcn is in an unmapped part/out of bounds of
+ * the runlist.
+ *
+ * Locking: The runlist must be locked on entry.
+ */
+struct runlist_element *ntfs_rl_find_vcn_nolock(struct runlist_element *rl, const s64 vcn)
+{
+	if (unlikely(!rl || vcn < rl[0].vcn))
+		return NULL;
+	while (likely(rl->length)) {
+		if (unlikely(vcn < rl[1].vcn)) {
+			if (likely(rl->lcn >= LCN_HOLE))
+				return rl;
+			return NULL;
+		}
+		rl++;
+	}
+	if (likely(rl->lcn == LCN_ENOENT))
+		return rl;
+	return NULL;
+}
+
+/*
+ * ntfs_get_nr_significant_bytes - get number of bytes needed to store a number
+ * @n:		number for which to get the number of bytes for
+ *
+ * Return the number of bytes required to store @n unambiguously as
+ * a signed number.
+ *
+ * This is used in the context of the mapping pairs array to determine how
+ * many bytes will be needed in the array to store a given logical cluster
+ * number (lcn) or a specific run length.
+ *
+ * Return the number of bytes written.  This function cannot fail.
+ */
+static inline int ntfs_get_nr_significant_bytes(const s64 n)
+{
+	s64 l = n;
+	int i;
+	s8 j;
+
+	i = 0;
+	do {
+		l >>= 8;
+		i++;
+	} while (l != 0 && l != -1);
+	j = (n >> 8 * (i - 1)) & 0xff;
+	/* If the sign bit is wrong, we need an extra byte. */
+	if ((n < 0 && j >= 0) || (n > 0 && j < 0))
+		i++;
+	return i;
+}
+
+/*
+ * ntfs_get_size_for_mapping_pairs - get bytes needed for mapping pairs array
+ * @vol: ntfs volume
+ * @rl: runlist to calculate the mapping pairs array size for
+ * @first_vcn: first vcn which to include in the mapping pairs array
+ * @last_vcn: last vcn which to include in the mapping pairs array
+ * @max_mp_size: maximum size to return (0 or less means unlimited)
+ *
+ * Walk the locked runlist @rl and calculate the size in bytes of the mapping
+ * pairs array corresponding to the runlist @rl, starting at vcn @first_vcn and
+ * finishing with vcn @last_vcn.
+ *
+ * A @last_vcn of -1 means end of runlist and in that case the size of the
+ * mapping pairs array corresponding to the runlist starting at vcn @first_vcn
+ * and finishing at the end of the runlist is determined.
+ *
+ * This for example allows us to allocate a buffer of the right size when
+ * building the mapping pairs array.
+ *
+ * If @rl is NULL, just return 1 (for the single terminator byte).
+ *
+ * Return the calculated size in bytes on success.  On error, return -errno.
+ */
+int ntfs_get_size_for_mapping_pairs(const struct ntfs_volume *vol,
+		const struct runlist_element *rl, const s64 first_vcn,
+		const s64 last_vcn, int max_mp_size)
+{
+	s64 prev_lcn;
+	int rls;
+	bool the_end = false;
+
+	if (first_vcn < 0 || last_vcn < -1)
+		return -EINVAL;
+
+	if (last_vcn >= 0 && first_vcn > last_vcn)
+		return -EINVAL;
+
+	if (!rl) {
+		WARN_ON(first_vcn);
+		WARN_ON(last_vcn > 0);
+		return 1;
+	}
+	if (max_mp_size <= 0)
+		max_mp_size = INT_MAX;
+	/* Skip to runlist element containing @first_vcn. */
+	while (rl->length && first_vcn >= rl[1].vcn)
+		rl++;
+	if (unlikely((!rl->length && first_vcn > rl->vcn) ||
+			first_vcn < rl->vcn))
+		return -EINVAL;
+	prev_lcn = 0;
+	/* Always need the termining zero byte. */
+	rls = 1;
+	/* Do the first partial run if present. */
+	if (first_vcn > rl->vcn) {
+		s64 delta, length = rl->length;
+
+		/* We know rl->length != 0 already. */
+		if (unlikely(length < 0 || rl->lcn < LCN_HOLE))
+			goto err_out;
+		/*
+		 * If @stop_vcn is given and finishes inside this run, cap the
+		 * run length.
+		 */
+		if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) {
+			s64 s1 = last_vcn + 1;
+
+			if (unlikely(rl[1].vcn > s1))
+				length = s1 - rl->vcn;
+			the_end = true;
+		}
+		delta = first_vcn - rl->vcn;
+		/* Header byte + length. */
+		rls += 1 + ntfs_get_nr_significant_bytes(length - delta);
+		/*
+		 * If the logical cluster number (lcn) denotes a hole and we
+		 * are on NTFS 3.0+, we don't store it at all, i.e. we need
+		 * zero space.  On earlier NTFS versions we just store the lcn.
+		 * Note: this assumes that on NTFS 1.2-, holes are stored with
+		 * an lcn of -1 and not a delta_lcn of -1 (unless both are -1).
+		 */
+		if (likely(rl->lcn >= 0 || vol->major_ver < 3)) {
+			prev_lcn = rl->lcn;
+			if (likely(rl->lcn >= 0))
+				prev_lcn += delta;
+			/* Change in lcn. */
+			rls += ntfs_get_nr_significant_bytes(prev_lcn);
+		}
+		/* Go to next runlist element. */
+		rl++;
+	}
+	/* Do the full runs. */
+	for (; rl->length && !the_end; rl++) {
+		s64 length = rl->length;
+
+		if (unlikely(length < 0 || rl->lcn < LCN_HOLE))
+			goto err_out;
+		/*
+		 * If @stop_vcn is given and finishes inside this run, cap the
+		 * run length.
+		 */
+		if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) {
+			s64 s1 = last_vcn + 1;
+
+			if (unlikely(rl[1].vcn > s1))
+				length = s1 - rl->vcn;
+			the_end = true;
+		}
+		/* Header byte + length. */
+		rls += 1 + ntfs_get_nr_significant_bytes(length);
+		/*
+		 * If the logical cluster number (lcn) denotes a hole and we
+		 * are on NTFS 3.0+, we don't store it at all, i.e. we need
+		 * zero space.  On earlier NTFS versions we just store the lcn.
+		 * Note: this assumes that on NTFS 1.2-, holes are stored with
+		 * an lcn of -1 and not a delta_lcn of -1 (unless both are -1).
+		 */
+		if (likely(rl->lcn >= 0 || vol->major_ver < 3)) {
+			/* Change in lcn. */
+			rls += ntfs_get_nr_significant_bytes(rl->lcn -
+					prev_lcn);
+			prev_lcn = rl->lcn;
+		}
+
+		if (rls > max_mp_size)
+			break;
+	}
+	return rls;
+err_out:
+	if (rl->lcn == LCN_RL_NOT_MAPPED)
+		rls = -EINVAL;
+	else
+		rls = -EIO;
+	return rls;
+}
+
+/*
+ * ntfs_write_significant_bytes - write the significant bytes of a number
+ * @dst:	destination buffer to write to
+ * @dst_max:	pointer to last byte of destination buffer for bounds checking
+ * @n:		number whose significant bytes to write
+ *
+ * Store in @dst, the minimum bytes of the number @n which are required to
+ * identify @n unambiguously as a signed number, taking care not to exceed
+ * @dest_max, the maximum position within @dst to which we are allowed to
+ * write.
+ *
+ * This is used when building the mapping pairs array of a runlist to compress
+ * a given logical cluster number (lcn) or a specific run length to the minimum
+ * size possible.
+ *
+ * Return the number of bytes written on success.  On error, i.e. the
+ * destination buffer @dst is too small, return -ENOSPC.
+ */
+static inline int ntfs_write_significant_bytes(s8 *dst, const s8 *dst_max,
+		const s64 n)
+{
+	s64 l = n;
+	int i;
+	s8 j;
+
+	i = 0;
+	do {
+		if (unlikely(dst > dst_max))
+			goto err_out;
+		*dst++ = l & 0xffll;
+		l >>= 8;
+		i++;
+	} while (l != 0 && l != -1);
+	j = (n >> 8 * (i - 1)) & 0xff;
+	/* If the sign bit is wrong, we need an extra byte. */
+	if (n < 0 && j >= 0) {
+		if (unlikely(dst > dst_max))
+			goto err_out;
+		i++;
+		*dst = (s8)-1;
+	} else if (n > 0 && j < 0) {
+		if (unlikely(dst > dst_max))
+			goto err_out;
+		i++;
+		*dst = (s8)0;
+	}
+	return i;
+err_out:
+	return -ENOSPC;
+}
+
+/*
+ * ntfs_mapping_pairs_build - build the mapping pairs array from a runlist
+ * @vol: ntfs volume
+ * @dst: destination buffer to build mapping pairs array into
+ * @dst_len: size of @dst in bytes
+ * @rl: runlist to build the mapping pairs array from
+ * @first_vcn: first vcn which to include in the mapping pairs array
+ * @last_vcn: last vcn which to include in the mapping pairs array
+ * @stop_vcn: on return, set to the first vcn outside the destination buffer
+ * @stop_rl: on return, set to the runlist element where encoding stopped
+ * @de_cluster_count: on return, set to the number of clusters encoded
+ *
+ * Create the mapping pairs array from the locked runlist @rl, starting at vcn
+ * @first_vcn and finishing with vcn @last_vcn and save the array in @dst.
+ * @dst_len is the size of @dst in bytes and it should be at least equal to the
+ * value obtained by calling ntfs_get_size_for_mapping_pairs().
+ *
+ * A @last_vcn of -1 means end of runlist and in that case the mapping pairs
+ * array corresponding to the runlist starting at vcn @first_vcn and finishing
+ * at the end of the runlist is created.
+ *
+ * If @rl is NULL, just write a single terminator byte to @dst.
+ *
+ * On success or -ENOSPC error, if @stop_vcn is not NULL, *@stop_vcn is set to
+ * the first vcn outside the destination buffer.  Note that on error, @dst has
+ * been filled with all the mapping pairs that will fit, thus it can be treated
+ * as partial success, in that a new attribute extent needs to be created or
+ * the next extent has to be used and the mapping pairs build has to be
+ * continued with @first_vcn set to *@stop_vcn.
+ *
+ * Return 0 on success and -errno on error.  The following error codes are
+ * defined:
+ *	-EINVAL	- Run list contains unmapped elements.  Make sure to only pass
+ *		  fully mapped runlists to this function.
+ *	-EIO	- The runlist is corrupt.
+ *	-ENOSPC	- The destination buffer is too small.
+ *
+ * Locking: @rl must be locked on entry (either for reading or writing), it
+ *	    remains locked throughout, and is left locked upon return.
+ */
+int ntfs_mapping_pairs_build(const struct ntfs_volume *vol, s8 *dst,
+		const int dst_len, const struct runlist_element *rl,
+		const s64 first_vcn, const s64 last_vcn, s64 *const stop_vcn,
+		struct runlist_element **stop_rl, unsigned int *de_cluster_count)
+{
+	s64 prev_lcn;
+	s8 *dst_max, *dst_next;
+	int err = -ENOSPC;
+	bool the_end = false;
+	s8 len_len, lcn_len;
+	unsigned int de_cnt = 0;
+
+	if (first_vcn < 0 || last_vcn < -1 || dst_len < 1)
+		return -EINVAL;
+	if (last_vcn >= 0 && first_vcn > last_vcn)
+		return -EINVAL;
+
+	if (!rl) {
+		WARN_ON(first_vcn || last_vcn > 0);
+		if (stop_vcn)
+			*stop_vcn = 0;
+		/* Terminator byte. */
+		*dst = 0;
+		return 0;
+	}
+	/* Skip to runlist element containing @first_vcn. */
+	while (rl->length && first_vcn >= rl[1].vcn)
+		rl++;
+	if (unlikely((!rl->length && first_vcn > rl->vcn) ||
+			first_vcn < rl->vcn))
+		return -EINVAL;
+	/*
+	 * @dst_max is used for bounds checking in
+	 * ntfs_write_significant_bytes().
+	 */
+	dst_max = dst + dst_len - 1;
+	prev_lcn = 0;
+	/* Do the first partial run if present. */
+	if (first_vcn > rl->vcn) {
+		s64 delta, length = rl->length;
+
+		/* We know rl->length != 0 already. */
+		if (unlikely(length < 0 || rl->lcn < LCN_HOLE))
+			goto err_out;
+		/*
+		 * If @stop_vcn is given and finishes inside this run, cap the
+		 * run length.
+		 */
+		if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) {
+			s64 s1 = last_vcn + 1;
+
+			if (unlikely(rl[1].vcn > s1))
+				length = s1 - rl->vcn;
+			the_end = true;
+		}
+		delta = first_vcn - rl->vcn;
+		/* Write length. */
+		len_len = ntfs_write_significant_bytes(dst + 1, dst_max,
+				length - delta);
+		if (unlikely(len_len < 0))
+			goto size_err;
+		/*
+		 * If the logical cluster number (lcn) denotes a hole and we
+		 * are on NTFS 3.0+, we don't store it at all, i.e. we need
+		 * zero space.  On earlier NTFS versions we just write the lcn
+		 * change.
+		 */
+		if (likely(rl->lcn >= 0 || vol->major_ver < 3)) {
+			prev_lcn = rl->lcn;
+			if (likely(rl->lcn >= 0))
+				prev_lcn += delta;
+			/* Write change in lcn. */
+			lcn_len = ntfs_write_significant_bytes(dst + 1 +
+					len_len, dst_max, prev_lcn);
+			if (unlikely(lcn_len < 0))
+				goto size_err;
+		} else
+			lcn_len = 0;
+		dst_next = dst + len_len + lcn_len + 1;
+		if (unlikely(dst_next > dst_max))
+			goto size_err;
+		/* Update header byte. */
+		*dst = lcn_len << 4 | len_len;
+		/* Position at next mapping pairs array element. */
+		dst = dst_next;
+		/* Go to next runlist element. */
+		rl++;
+	}
+	/* Do the full runs. */
+	for (; rl->length && !the_end; rl++) {
+		s64 length = rl->length;
+
+		if (unlikely(length < 0 || rl->lcn < LCN_HOLE))
+			goto err_out;
+		/*
+		 * If @stop_vcn is given and finishes inside this run, cap the
+		 * run length.
+		 */
+		if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) {
+			s64 s1 = last_vcn + 1;
+
+			if (unlikely(rl[1].vcn > s1))
+				length = s1 - rl->vcn;
+			the_end = true;
+		}
+		/* Write length. */
+		len_len = ntfs_write_significant_bytes(dst + 1, dst_max,
+				length);
+		if (unlikely(len_len < 0))
+			goto size_err;
+		/*
+		 * If the logical cluster number (lcn) denotes a hole and we
+		 * are on NTFS 3.0+, we don't store it at all, i.e. we need
+		 * zero space.  On earlier NTFS versions we just write the lcn
+		 * change.
+		 */
+		if (likely(rl->lcn >= 0 || vol->major_ver < 3)) {
+			/* Write change in lcn. */
+			lcn_len = ntfs_write_significant_bytes(dst + 1 +
+					len_len, dst_max, rl->lcn - prev_lcn);
+			if (unlikely(lcn_len < 0))
+				goto size_err;
+			prev_lcn = rl->lcn;
+		} else {
+			if (rl->lcn == LCN_DELALLOC)
+				de_cnt += rl->length;
+			lcn_len = 0;
+		}
+		dst_next = dst + len_len + lcn_len + 1;
+		if (unlikely(dst_next > dst_max))
+			goto size_err;
+		/* Update header byte. */
+		*dst = lcn_len << 4 | len_len;
+		/* Position at next mapping pairs array element. */
+		dst = dst_next;
+	}
+	/* Success. */
+	if (de_cluster_count)
+		*de_cluster_count = de_cnt;
+	err = 0;
+size_err:
+	/* Set stop vcn. */
+	if (stop_vcn)
+		*stop_vcn = rl->vcn;
+	if (stop_rl)
+		*stop_rl = (struct runlist_element *)rl;
+	/* Add terminator byte. */
+	*dst = 0;
+	return err;
+err_out:
+	if (rl->lcn == LCN_RL_NOT_MAPPED)
+		err = -EINVAL;
+	else
+		err = -EIO;
+	return err;
+}
+
+/*
+ * ntfs_rl_truncate_nolock - truncate a runlist starting at a specified vcn
+ * @vol:	ntfs volume (needed for error output)
+ * @runlist:	runlist to truncate
+ * @new_length:	the new length of the runlist in VCNs
+ *
+ * Truncate the runlist described by @runlist as well as the memory buffer
+ * holding the runlist elements to a length of @new_length VCNs.
+ *
+ * If @new_length lies within the runlist, the runlist elements with VCNs of
+ * @new_length and above are discarded.  As a special case if @new_length is
+ * zero, the runlist is discarded and set to NULL.
+ *
+ * If @new_length lies beyond the runlist, a sparse runlist element is added to
+ * the end of the runlist @runlist or if the last runlist element is a sparse
+ * one already, this is extended.
+ *
+ * Note, no checking is done for unmapped runlist elements.  It is assumed that
+ * the caller has mapped any elements that need to be mapped already.
+ *
+ * Return 0 on success and -errno on error.
+ *
+ * Locking: The caller must hold @runlist->lock for writing.
+ */
+int ntfs_rl_truncate_nolock(const struct ntfs_volume *vol, struct runlist *const runlist,
+		const s64 new_length)
+{
+	struct runlist_element *rl;
+	int old_size;
+
+	ntfs_debug("Entering for new_length 0x%llx.", (long long)new_length);
+
+	if (!runlist || new_length < 0)
+		return -EINVAL;
+
+	rl = runlist->rl;
+	if (new_length < rl->vcn)
+		return -EINVAL;
+
+	/* Find @new_length in the runlist. */
+	while (likely(rl->length && new_length >= rl[1].vcn))
+		rl++;
+	/*
+	 * If not at the end of the runlist we need to shrink it.
+	 * If at the end of the runlist we need to expand it.
+	 */
+	if (rl->length) {
+		struct runlist_element *trl;
+		bool is_end;
+
+		ntfs_debug("Shrinking runlist.");
+		/* Determine the runlist size. */
+		trl = rl + 1;
+		while (likely(trl->length))
+			trl++;
+		old_size = trl - runlist->rl + 1;
+		/* Truncate the run. */
+		rl->length = new_length - rl->vcn;
+		/*
+		 * If a run was partially truncated, make the following runlist
+		 * element a terminator.
+		 */
+		is_end = false;
+		if (rl->length) {
+			rl++;
+			if (!rl->length)
+				is_end = true;
+			rl->vcn = new_length;
+			rl->length = 0;
+		}
+		rl->lcn = LCN_ENOENT;
+		runlist->count = rl - runlist->rl + 1;
+		/* Reallocate memory if necessary. */
+		if (!is_end) {
+			int new_size = rl - runlist->rl + 1;
+
+			rl = ntfs_rl_realloc(runlist->rl, old_size, new_size);
+			if (IS_ERR(rl))
+				ntfs_warning(vol->sb,
+					"Failed to shrink runlist buffer.  This just wastes a bit of memory temporarily so we ignore it and return success.");
+			else
+				runlist->rl = rl;
+		}
+	} else if (likely(/* !rl->length && */ new_length > rl->vcn)) {
+		ntfs_debug("Expanding runlist.");
+		/*
+		 * If there is a previous runlist element and it is a sparse
+		 * one, extend it.  Otherwise need to add a new, sparse runlist
+		 * element.
+		 */
+		if ((rl > runlist->rl) && ((rl - 1)->lcn == LCN_HOLE))
+			(rl - 1)->length = new_length - (rl - 1)->vcn;
+		else {
+			/* Determine the runlist size. */
+			old_size = rl - runlist->rl + 1;
+			/* Reallocate memory if necessary. */
+			rl = ntfs_rl_realloc(runlist->rl, old_size,
+					old_size + 1);
+			if (IS_ERR(rl)) {
+				ntfs_error(vol->sb, "Failed to expand runlist buffer, aborting.");
+				return PTR_ERR(rl);
+			}
+			runlist->rl = rl;
+			/*
+			 * Set @rl to the same runlist element in the new
+			 * runlist as before in the old runlist.
+			 */
+			rl += old_size - 1;
+			/* Add a new, sparse runlist element. */
+			rl->lcn = LCN_HOLE;
+			rl->length = new_length - rl->vcn;
+			/* Add a new terminator runlist element. */
+			rl++;
+			rl->length = 0;
+			runlist->count = old_size + 1;
+		}
+		rl->vcn = new_length;
+		rl->lcn = LCN_ENOENT;
+	} else /* if (unlikely(!rl->length && new_length == rl->vcn)) */ {
+		/* Runlist already has same size as requested. */
+		rl->lcn = LCN_ENOENT;
+	}
+	ntfs_debug("Done.");
+	return 0;
+}
+
+/*
+ * ntfs_rl_sparse - check whether runlist have sparse regions or not.
+ * @rl:         runlist to check
+ *
+ * Return 1 if have, 0 if not, -errno on error.
+ */
+int ntfs_rl_sparse(struct runlist_element *rl)
+{
+	struct runlist_element *rlc;
+
+	if (!rl)
+		return -EINVAL;
+
+	for (rlc = rl; rlc->length; rlc++)
+		if (rlc->lcn < 0) {
+			if (rlc->lcn != LCN_HOLE && rlc->lcn != LCN_DELALLOC) {
+				pr_err("%s: bad runlist\n", __func__);
+				return -EINVAL;
+			}
+			return 1;
+		}
+	return 0;
+}
+
+/*
+ * ntfs_rl_get_compressed_size - calculate length of non sparse regions
+ * @vol:        ntfs volume (need for cluster size)
+ * @rl:         runlist to calculate for
+ *
+ * Return compressed size or -errno on error.
+ */
+s64 ntfs_rl_get_compressed_size(struct ntfs_volume *vol, struct runlist_element *rl)
+{
+	struct runlist_element *rlc;
+	s64 ret = 0;
+
+	if (!rl)
+		return -EINVAL;
+
+	for (rlc = rl; rlc->length; rlc++) {
+		if (rlc->lcn < 0) {
+			if (rlc->lcn != LCN_HOLE && rlc->lcn != LCN_DELALLOC) {
+				ntfs_error(vol->sb, "%s: bad runlist, rlc->lcn : %lld",
+						__func__, rlc->lcn);
+				return -EINVAL;
+			}
+		} else
+			ret += rlc->length;
+	}
+	return NTFS_CLU_TO_B(vol, ret);
+}
+
+static inline bool ntfs_rle_lcn_contiguous(struct runlist_element *left_rle,
+					   struct runlist_element *right_rle)
+{
+	if (left_rle->lcn > LCN_HOLE &&
+	    left_rle->lcn + left_rle->length == right_rle->lcn)
+		return true;
+	else if (left_rle->lcn == LCN_HOLE && right_rle->lcn == LCN_HOLE)
+		return true;
+	else
+		return false;
+}
+
+static inline bool ntfs_rle_contain(struct runlist_element *rle, s64 vcn)
+{
+	if (rle->length > 0 &&
+	    vcn >= rle->vcn && vcn < rle->vcn + rle->length)
+		return true;
+	else
+		return false;
+}
+
+struct runlist_element *ntfs_rl_insert_range(struct runlist_element *dst_rl, int dst_cnt,
+				      struct runlist_element *src_rl, int src_cnt,
+				      size_t *new_rl_cnt)
+{
+	struct runlist_element *i_rl, *new_rl, *src_rl_origin = src_rl;
+	struct runlist_element dst_rl_split;
+	s64 start_vcn;
+	int new_1st_cnt, new_2nd_cnt, new_3rd_cnt, new_cnt;
+
+	if (!dst_rl || !src_rl || !new_rl_cnt)
+		return ERR_PTR(-EINVAL);
+	if (dst_cnt <= 0 || src_cnt <= 0)
+		return ERR_PTR(-EINVAL);
+	if (!(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT &&
+	      dst_rl[dst_cnt - 1].length == 0) ||
+	    src_rl[src_cnt - 1].lcn < LCN_HOLE)
+		return ERR_PTR(-EINVAL);
+
+	start_vcn = src_rl[0].vcn;
+
+	i_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn);
+	if (!i_rl ||
+	    (i_rl->lcn == LCN_ENOENT && i_rl->vcn != start_vcn) ||
+	    (i_rl->lcn != LCN_ENOENT && !ntfs_rle_contain(i_rl, start_vcn)))
+		return ERR_PTR(-EINVAL);
+
+	new_1st_cnt = (int)(i_rl - dst_rl);
+	if (new_1st_cnt > dst_cnt)
+		return ERR_PTR(-EINVAL);
+	new_3rd_cnt = dst_cnt - new_1st_cnt;
+	if (new_3rd_cnt < 1)
+		return ERR_PTR(-EINVAL);
+
+	if (i_rl[0].vcn != start_vcn) {
+		if (i_rl[0].lcn == LCN_HOLE && src_rl[0].lcn == LCN_HOLE)
+			goto merge_src_rle;
+
+		/* split @i_rl[0] and create @dst_rl_split */
+		dst_rl_split.vcn = i_rl[0].vcn;
+		dst_rl_split.length = start_vcn - i_rl[0].vcn;
+		dst_rl_split.lcn = i_rl[0].lcn;
+
+		i_rl[0].vcn = start_vcn;
+		i_rl[0].length -= dst_rl_split.length;
+		i_rl[0].lcn += dst_rl_split.length;
+	} else {
+		struct runlist_element *dst_rle, *src_rle;
+merge_src_rle:
+
+		/* not split @i_rl[0] */
+		dst_rl_split.lcn = LCN_ENOENT;
+
+		/* merge @src_rl's first run and @i_rl[0]'s left run if possible */
+		dst_rle = &dst_rl[new_1st_cnt - 1];
+		src_rle = &src_rl[0];
+		if (new_1st_cnt > 0 && ntfs_rle_lcn_contiguous(dst_rle, src_rle)) {
+			WARN_ON(dst_rle->vcn + dst_rle->length != src_rle->vcn);
+			dst_rle->length += src_rle->length;
+			src_rl++;
+			src_cnt--;
+		} else {
+			/* merge @src_rl's last run and @i_rl[0]'s right if possible */
+			dst_rle = &dst_rl[new_1st_cnt];
+			src_rle = &src_rl[src_cnt - 1];
+
+			if (ntfs_rle_lcn_contiguous(dst_rle, src_rle)) {
+				dst_rle->length += src_rle->length;
+				src_cnt--;
+			}
+		}
+	}
+
+	new_2nd_cnt = src_cnt;
+	new_cnt = new_1st_cnt + new_2nd_cnt + new_3rd_cnt;
+	new_cnt += dst_rl_split.lcn >= LCN_HOLE ? 1 : 0;
+	new_rl = kvcalloc(new_cnt, sizeof(*new_rl), GFP_NOFS);
+	if (!new_rl)
+		return ERR_PTR(-ENOMEM);
+
+	/* Copy the @dst_rl's first half to @new_rl */
+	ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt);
+	if (dst_rl_split.lcn >= LCN_HOLE) {
+		ntfs_rl_mc(new_rl, new_1st_cnt, &dst_rl_split, 0, 1);
+		new_1st_cnt++;
+	}
+	/* Copy the @src_rl to @new_rl */
+	ntfs_rl_mc(new_rl, new_1st_cnt, src_rl, 0, new_2nd_cnt);
+	/* Copy the @dst_rl's second half to @new_rl */
+	if (new_3rd_cnt >= 1) {
+		struct runlist_element *rl, *rl_3rd;
+		int dst_1st_cnt = dst_rl_split.lcn >= LCN_HOLE ?
+			new_1st_cnt - 1 : new_1st_cnt;
+
+		ntfs_rl_mc(new_rl, new_1st_cnt + new_2nd_cnt,
+			   dst_rl, dst_1st_cnt, new_3rd_cnt);
+		/* Update vcn of the @dst_rl's second half runs to reflect
+		 * appended @src_rl.
+		 */
+		if (new_1st_cnt + new_2nd_cnt == 0) {
+			rl_3rd = &new_rl[new_1st_cnt + new_2nd_cnt + 1];
+			rl = &new_rl[new_1st_cnt + new_2nd_cnt];
+		} else {
+			rl_3rd = &new_rl[new_1st_cnt + new_2nd_cnt];
+			rl = &new_rl[new_1st_cnt + new_2nd_cnt - 1];
+		}
+		do {
+			rl_3rd->vcn = rl->vcn + rl->length;
+			if (rl_3rd->length <= 0)
+				break;
+			rl = rl_3rd;
+			rl_3rd++;
+		} while (1);
+	}
+	*new_rl_cnt = new_1st_cnt + new_2nd_cnt + new_3rd_cnt;
+
+	kvfree(dst_rl);
+	kvfree(src_rl_origin);
+	return new_rl;
+}
+
+struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int dst_cnt,
+				    s64 start_vcn, s64 len,
+				    struct runlist_element **punch_rl,
+				    size_t *new_rl_cnt)
+{
+	struct runlist_element *s_rl, *e_rl, *new_rl, *dst_3rd_rl, hole_rl[1];
+	s64 end_vcn;
+	int new_1st_cnt, dst_3rd_cnt, new_cnt, punch_cnt, merge_cnt;
+	bool begin_split, end_split, one_split_3;
+
+	if (dst_cnt < 2 ||
+	    !(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT &&
+	      dst_rl[dst_cnt - 1].length == 0))
+		return ERR_PTR(-EINVAL);
+
+	end_vcn = min(start_vcn + len - 1,
+		      dst_rl[dst_cnt - 2].vcn + dst_rl[dst_cnt - 2].length - 1);
+
+	s_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn);
+	if (!s_rl ||
+	    s_rl->lcn <= LCN_ENOENT ||
+	    !ntfs_rle_contain(s_rl, start_vcn))
+		return ERR_PTR(-EINVAL);
+
+	begin_split = s_rl->vcn != start_vcn ? true : false;
+
+	e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn);
+	if (!e_rl ||
+	    e_rl->lcn <= LCN_ENOENT ||
+	    !ntfs_rle_contain(e_rl, end_vcn))
+		return ERR_PTR(-EINVAL);
+
+	end_split = e_rl->vcn + e_rl->length - 1 != end_vcn ? true : false;
+
+	/* @s_rl has to be split into left, punched hole, and right */
+	one_split_3 = e_rl == s_rl && begin_split && end_split ? true : false;
+
+	punch_cnt = (int)(e_rl - s_rl) + 1;
+
+	*punch_rl = kvcalloc(punch_cnt + 1, sizeof(struct runlist_element),
+			GFP_NOFS);
+	if (!*punch_rl)
+		return ERR_PTR(-ENOMEM);
+
+	new_cnt = dst_cnt - (int)(e_rl - s_rl + 1) + 3;
+	new_rl = kvcalloc(new_cnt, sizeof(struct runlist_element), GFP_NOFS);
+	if (!new_rl) {
+		kvfree(*punch_rl);
+		*punch_rl = NULL;
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_1st_cnt = (int)(s_rl - dst_rl) + 1;
+	ntfs_rl_mc(*punch_rl, 0, dst_rl, new_1st_cnt - 1, punch_cnt);
+
+	(*punch_rl)[punch_cnt].lcn = LCN_ENOENT;
+	(*punch_rl)[punch_cnt].length = 0;
+
+	if (!begin_split)
+		new_1st_cnt--;
+	dst_3rd_rl = e_rl;
+	dst_3rd_cnt = (int)(&dst_rl[dst_cnt - 1] - e_rl) + 1;
+	if (!end_split) {
+		dst_3rd_rl++;
+		dst_3rd_cnt--;
+	}
+
+	/* Copy the 1st part of @dst_rl into @new_rl */
+	ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt);
+	if (begin_split) {
+		/* the @e_rl has to be splited and copied into the last of @new_rl
+		 * and the first of @punch_rl
+		 */
+		s64 first_cnt = start_vcn - dst_rl[new_1st_cnt - 1].vcn;
+
+		if (new_1st_cnt)
+			new_rl[new_1st_cnt - 1].length = first_cnt;
+
+		(*punch_rl)[0].vcn = start_vcn;
+		(*punch_rl)[0].length -= first_cnt;
+		if ((*punch_rl)[0].lcn > LCN_HOLE)
+			(*punch_rl)[0].lcn += first_cnt;
+	}
+
+	/* Copy a hole into @new_rl */
+	hole_rl[0].vcn = start_vcn;
+	hole_rl[0].length = (s64)len;
+	hole_rl[0].lcn = LCN_HOLE;
+	ntfs_rl_mc(new_rl, new_1st_cnt, hole_rl, 0, 1);
+
+	/* Copy the 3rd part of @dst_rl into @new_rl */
+	ntfs_rl_mc(new_rl, new_1st_cnt + 1, dst_3rd_rl, 0, dst_3rd_cnt);
+	if (end_split) {
+		/* the @e_rl has to be splited and copied into the first of
+		 * @new_rl and the last of @punch_rl
+		 */
+		s64 first_cnt = end_vcn - dst_3rd_rl[0].vcn + 1;
+
+		new_rl[new_1st_cnt + 1].vcn = end_vcn + 1;
+		new_rl[new_1st_cnt + 1].length -= first_cnt;
+		if (new_rl[new_1st_cnt + 1].lcn > LCN_HOLE)
+			new_rl[new_1st_cnt + 1].lcn += first_cnt;
+
+		if (one_split_3)
+			(*punch_rl)[punch_cnt - 1].length -=
+				new_rl[new_1st_cnt + 1].length;
+		else
+			(*punch_rl)[punch_cnt - 1].length = first_cnt;
+	}
+
+	/* Merge left and hole, or hole and right in @new_rl, if left or right
+	 * consists of holes.
+	 */
+	merge_cnt = 0;
+	if (new_1st_cnt > 0 && new_rl[new_1st_cnt - 1].lcn == LCN_HOLE) {
+		/* Merge right and hole */
+		s_rl =  &new_rl[new_1st_cnt - 1];
+		s_rl->length += s_rl[1].length;
+		merge_cnt = 1;
+		/* Merge left and right */
+		if (new_1st_cnt + 1 < new_cnt &&
+		    new_rl[new_1st_cnt + 1].lcn == LCN_HOLE) {
+			s_rl->length += s_rl[2].length;
+			merge_cnt++;
+		}
+	} else if (new_1st_cnt + 1 < new_cnt &&
+		   new_rl[new_1st_cnt + 1].lcn == LCN_HOLE) {
+		/* Merge left and hole */
+		s_rl = &new_rl[new_1st_cnt];
+		s_rl->length += s_rl[1].length;
+		merge_cnt = 1;
+	}
+	if (merge_cnt) {
+		struct runlist_element *d_rl, *src_rl;
+
+		d_rl = s_rl + 1;
+		src_rl = s_rl + 1 + merge_cnt;
+		ntfs_rl_mm(new_rl, (int)(d_rl - new_rl), (int)(src_rl - new_rl),
+			   (int)(&new_rl[new_cnt - 1] - src_rl) + 1);
+	}
+
+	(*punch_rl)[punch_cnt].vcn = (*punch_rl)[punch_cnt - 1].vcn +
+		(*punch_rl)[punch_cnt - 1].length;
+
+	/* punch_cnt elements of dst are replaced with one hole */
+	*new_rl_cnt = dst_cnt - (punch_cnt - (int)begin_split - (int)end_split) +
+		1 - merge_cnt;
+	kvfree(dst_rl);
+	return new_rl;
+}
+
+struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, int dst_cnt,
+					s64 start_vcn, s64 len,
+					struct runlist_element **punch_rl,
+					size_t *new_rl_cnt)
+{
+	struct runlist_element *s_rl, *e_rl, *new_rl, *dst_3rd_rl;
+	s64 end_vcn;
+	int new_1st_cnt, dst_3rd_cnt, new_cnt, punch_cnt, merge_cnt, i;
+	bool begin_split, end_split, one_split_3;
+
+	if (dst_cnt < 2 ||
+	    !(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT &&
+	      dst_rl[dst_cnt - 1].length == 0))
+		return ERR_PTR(-EINVAL);
+
+	end_vcn = min(start_vcn + len - 1,
+			dst_rl[dst_cnt - 1].vcn - 1);
+
+	s_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn);
+	if (!s_rl ||
+	    s_rl->lcn <= LCN_ENOENT ||
+	    !ntfs_rle_contain(s_rl, start_vcn))
+		return ERR_PTR(-EINVAL);
+
+	begin_split = s_rl->vcn != start_vcn ? true : false;
+
+	e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn);
+	if (!e_rl ||
+	    e_rl->lcn <= LCN_ENOENT ||
+	    !ntfs_rle_contain(e_rl, end_vcn))
+		return ERR_PTR(-EINVAL);
+
+	end_split = e_rl->vcn + e_rl->length - 1 != end_vcn ? true : false;
+
+	/* @s_rl has to be split into left, collapsed, and right */
+	one_split_3 = e_rl == s_rl && begin_split && end_split ? true : false;
+
+	punch_cnt = (int)(e_rl - s_rl) + 1;
+	*punch_rl = kvcalloc(punch_cnt + 1, sizeof(struct runlist_element),
+			GFP_NOFS);
+	if (!*punch_rl)
+		return ERR_PTR(-ENOMEM);
+
+	new_cnt = dst_cnt - (int)(e_rl - s_rl + 1) + 3;
+	new_rl = kvcalloc(new_cnt, sizeof(struct runlist_element), GFP_NOFS);
+	if (!new_rl) {
+		kvfree(*punch_rl);
+		*punch_rl = NULL;
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_1st_cnt = (int)(s_rl - dst_rl) + 1;
+	ntfs_rl_mc(*punch_rl, 0, dst_rl, new_1st_cnt - 1, punch_cnt);
+	(*punch_rl)[punch_cnt].lcn = LCN_ENOENT;
+	(*punch_rl)[punch_cnt].length = 0;
+
+	if (!begin_split)
+		new_1st_cnt--;
+	dst_3rd_rl = e_rl;
+	dst_3rd_cnt = (int)(&dst_rl[dst_cnt - 1] - e_rl) + 1;
+	if (!end_split) {
+		dst_3rd_rl++;
+		dst_3rd_cnt--;
+	}
+
+	/* Copy the 1st part of @dst_rl into @new_rl */
+	ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt);
+	if (begin_split) {
+		/* the @e_rl has to be splited and copied into the last of @new_rl
+		 * and the first of @punch_rl
+		 */
+		s64 first_cnt = start_vcn - dst_rl[new_1st_cnt - 1].vcn;
+
+		new_rl[new_1st_cnt - 1].length = first_cnt;
+
+		(*punch_rl)[0].vcn = start_vcn;
+		(*punch_rl)[0].length -= first_cnt;
+		if ((*punch_rl)[0].lcn > LCN_HOLE)
+			(*punch_rl)[0].lcn += first_cnt;
+	}
+
+	/* Copy the 3rd part of @dst_rl into @new_rl */
+	ntfs_rl_mc(new_rl, new_1st_cnt, dst_3rd_rl, 0, dst_3rd_cnt);
+	if (end_split) {
+		/* the @e_rl has to be splited and copied into the first of
+		 * @new_rl and the last of @punch_rl
+		 */
+		s64 first_cnt = end_vcn - dst_3rd_rl[0].vcn + 1;
+
+		new_rl[new_1st_cnt].vcn = end_vcn + 1;
+		new_rl[new_1st_cnt].length -= first_cnt;
+		if (new_rl[new_1st_cnt].lcn > LCN_HOLE)
+			new_rl[new_1st_cnt].lcn += first_cnt;
+
+		if (one_split_3)
+			(*punch_rl)[punch_cnt - 1].length -=
+				new_rl[new_1st_cnt].length;
+		else
+			(*punch_rl)[punch_cnt - 1].length = first_cnt;
+	}
+
+	/* Adjust vcn */
+	if (new_1st_cnt == 0)
+		new_rl[new_1st_cnt].vcn = 0;
+	for (i = new_1st_cnt == 0 ? 1 : new_1st_cnt; new_rl[i].length; i++)
+		new_rl[i].vcn = new_rl[i - 1].vcn + new_rl[i - 1].length;
+	new_rl[i].vcn = new_rl[i - 1].vcn + new_rl[i - 1].length;
+
+	/* Merge left and hole, or hole and right in @new_rl, if left or right
+	 * consists of holes.
+	 */
+	merge_cnt = 0;
+	i = new_1st_cnt == 0 ? 1 : new_1st_cnt;
+	if (ntfs_rle_lcn_contiguous(&new_rl[i - 1], &new_rl[i])) {
+		/* Merge right and left */
+		s_rl =  &new_rl[new_1st_cnt - 1];
+		s_rl->length += s_rl[1].length;
+		merge_cnt = 1;
+	}
+	if (merge_cnt) {
+		struct runlist_element *d_rl, *src_rl;
+
+		d_rl = s_rl + 1;
+		src_rl = s_rl + 1 + merge_cnt;
+		ntfs_rl_mm(new_rl, (int)(d_rl - new_rl), (int)(src_rl - new_rl),
+			   (int)(&new_rl[new_cnt - 1] - src_rl) + 1);
+	}
+
+	(*punch_rl)[punch_cnt].vcn = (*punch_rl)[punch_cnt - 1].vcn +
+		(*punch_rl)[punch_cnt - 1].length;
+
+	/* punch_cnt elements of dst are extracted */
+	*new_rl_cnt = dst_cnt - (punch_cnt - (int)begin_split - (int)end_split) -
+		merge_cnt;
+
+	kvfree(dst_rl);
+	return new_rl;
+}
diff --git a/fs/ntfs/runlist.h b/fs/ntfs/runlist.h
new file mode 100644
index 0000000000000..cc92e09fbf682
--- /dev/null
+++ b/fs/ntfs/runlist.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for runlist handling in NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2005 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_RUNLIST_H
+#define _LINUX_NTFS_RUNLIST_H
+
+#include "volume.h"
+
+/*
+ * runlist_element - in memory vcn to lcn mapping array element
+ * @vcn:	starting vcn of the current array element
+ * @lcn:	starting lcn of the current array element
+ * @length:	length in clusters of the current array element
+ *
+ * The last vcn (in fact the last vcn + 1) is reached when length == 0.
+ *
+ * When lcn == -1 this means that the count vcns starting at vcn are not
+ * physically allocated (i.e. this is a hole / data is sparse).
+ *
+ * In memory vcn to lcn mapping structure element.
+ * @vcn: vcn = Starting virtual cluster number.
+ * @lcn: lcn = Starting logical cluster number.
+ * @length: Run length in clusters.
+ */
+struct runlist_element {
+	s64 vcn;
+	s64 lcn;
+	s64 length;
+};
+
+/*
+ * runlist - in memory vcn to lcn mapping array including a read/write lock
+ * @rl:		pointer to an array of runlist elements
+ * @lock:	read/write spinlock for serializing access to @rl
+ * @rl_hint:	hint/cache pointing to the last accessed runlist element
+ */
+struct runlist {
+	struct runlist_element *rl;
+	struct rw_semaphore lock;
+	size_t count;
+	int rl_hint;
+};
+
+static inline void ntfs_init_runlist(struct runlist *rl)
+{
+	rl->rl = NULL;
+	init_rwsem(&rl->lock);
+	rl->count = 0;
+	rl->rl_hint = -1;
+}
+
+enum {
+	LCN_DELALLOC		= -1,
+	LCN_HOLE		= -2,
+	LCN_RL_NOT_MAPPED	= -3,
+	LCN_ENOENT		= -4,
+	LCN_ENOMEM		= -5,
+	LCN_EIO			= -6,
+	LCN_EINVAL		= -7,
+};
+
+struct runlist_element *ntfs_runlists_merge(struct runlist *d_runlist,
+		struct runlist_element *srl, size_t s_rl_count,
+		size_t *new_rl_count);
+struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *vol,
+		const struct attr_record *attr, struct runlist *old_runlist,
+		size_t *new_rl_count);
+s64 ntfs_rl_vcn_to_lcn(const struct runlist_element *rl, const s64 vcn);
+struct runlist_element *ntfs_rl_find_vcn_nolock(struct runlist_element *rl, const s64 vcn);
+int ntfs_get_size_for_mapping_pairs(const struct ntfs_volume *vol,
+		const struct runlist_element *rl, const s64 first_vcn,
+		const s64 last_vcn, int max_mp_size);
+int ntfs_mapping_pairs_build(const struct ntfs_volume *vol, s8 *dst,
+		const int dst_len, const struct runlist_element *rl,
+		const s64 first_vcn, const s64 last_vcn, s64 *const stop_vcn,
+		struct runlist_element **stop_rl, unsigned int *de_cluster_count);
+int ntfs_rl_truncate_nolock(const struct ntfs_volume *vol,
+		struct runlist *const runlist, const s64 new_length);
+int ntfs_rl_sparse(struct runlist_element *rl);
+s64 ntfs_rl_get_compressed_size(struct ntfs_volume *vol, struct runlist_element *rl);
+struct runlist_element *ntfs_rl_insert_range(struct runlist_element *dst_rl, int dst_cnt,
+		struct runlist_element *src_rl, int src_cnt, size_t *new_cnt);
+struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int dst_cnt,
+		s64 start_vcn, s64 len, struct runlist_element **punch_rl,
+		size_t *new_rl_cnt);
+struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, int dst_cnt,
+		s64 start_vcn, s64 len, struct runlist_element **punch_rl,
+		size_t *new_rl_cnt);
+struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl, int old_size,
+		int new_size);
+#endif /* _LINUX_NTFS_RUNLIST_H */
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
new file mode 100644
index 0000000000000..39a5c3b810012
--- /dev/null
+++ b/fs/ntfs/super.c
@@ -0,0 +1,2771 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS kernel super block handling.
+ *
+ * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc.
+ * Copyright (c) 2001,2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#include <linux/blkdev.h>	/* For bdev_logical_block_size(). */
+#include <linux/backing-dev.h>
+#include <linux/vfs.h>
+#include <linux/fs_struct.h>
+#include <linux/sched/mm.h>
+#include <linux/fs_context.h>
+#include <linux/fs_parser.h>
+
+#include "sysctl.h"
+#include "logfile.h"
+#include "quota.h"
+#include "index.h"
+#include "ntfs.h"
+#include "ea.h"
+#include "volume.h"
+
+/* A global default upcase table and a corresponding reference count. */
+static __le16 *default_upcase;
+static unsigned long ntfs_nr_upcase_users;
+
+static struct workqueue_struct *ntfs_wq;
+
+/* Error constants/strings used in inode.c::ntfs_show_options(). */
+enum {
+	/* One of these must be present, default is ON_ERRORS_CONTINUE. */
+	ON_ERRORS_PANIC = 0x01,
+	ON_ERRORS_REMOUNT_RO = 0x02,
+	ON_ERRORS_CONTINUE = 0x04,
+};
+
+static const struct constant_table ntfs_param_enums[] = {
+	{ "panic",		ON_ERRORS_PANIC },
+	{ "remount-ro",		ON_ERRORS_REMOUNT_RO },
+	{ "continue",		ON_ERRORS_CONTINUE },
+	{}
+};
+
+enum {
+	Opt_uid,
+	Opt_gid,
+	Opt_umask,
+	Opt_dmask,
+	Opt_fmask,
+	Opt_errors,
+	Opt_nls,
+	Opt_charset,
+	Opt_show_sys_files,
+	Opt_show_meta,
+	Opt_case_sensitive,
+	Opt_disable_sparse,
+	Opt_sparse,
+	Opt_mft_zone_multiplier,
+	Opt_preallocated_size,
+	Opt_sys_immutable,
+	Opt_nohidden,
+	Opt_hide_dot_files,
+	Opt_check_windows_names,
+	Opt_acl,
+	Opt_discard,
+	Opt_nocase,
+};
+
+static const struct fs_parameter_spec ntfs_parameters[] = {
+	fsparam_u32("uid",			Opt_uid),
+	fsparam_u32("gid",			Opt_gid),
+	fsparam_u32oct("umask",			Opt_umask),
+	fsparam_u32oct("dmask",			Opt_dmask),
+	fsparam_u32oct("fmask",			Opt_fmask),
+	fsparam_string("nls",			Opt_nls),
+	fsparam_string("iocharset",		Opt_charset),
+	fsparam_enum("errors",			Opt_errors, ntfs_param_enums),
+	fsparam_flag("show_sys_files",		Opt_show_sys_files),
+	fsparam_flag("showmeta",		Opt_show_meta),
+	fsparam_flag("case_sensitive",		Opt_case_sensitive),
+	fsparam_flag("disable_sparse",		Opt_disable_sparse),
+	fsparam_s32("mft_zone_multiplier",	Opt_mft_zone_multiplier),
+	fsparam_u64("preallocated_size",	Opt_preallocated_size),
+	fsparam_flag("sys_immutable",		Opt_sys_immutable),
+	fsparam_flag("nohidden",		Opt_nohidden),
+	fsparam_flag("hide_dot_files",		Opt_hide_dot_files),
+	fsparam_flag("windows_names",		Opt_check_windows_names),
+	fsparam_flag("acl",			Opt_acl),
+	fsparam_flag("discard",			Opt_discard),
+	fsparam_flag("sparse",			Opt_sparse),
+	fsparam_flag("nocase",			Opt_nocase),
+	{}
+};
+
+static int ntfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
+{
+	struct ntfs_volume *vol = fc->s_fs_info;
+	struct fs_parse_result result;
+	int opt;
+
+	opt = fs_parse(fc, ntfs_parameters, param, &result);
+	if (opt < 0)
+		return opt;
+
+	switch (opt) {
+	case Opt_uid:
+		vol->uid = make_kuid(current_user_ns(), result.uint_32);
+		break;
+	case Opt_gid:
+		vol->gid = make_kgid(current_user_ns(), result.uint_32);
+		break;
+	case Opt_umask:
+		vol->fmask = vol->dmask = result.uint_32;
+		break;
+	case Opt_dmask:
+		vol->dmask = result.uint_32;
+		break;
+	case Opt_fmask:
+		vol->fmask = result.uint_32;
+		break;
+	case Opt_errors:
+		vol->on_errors = result.uint_32;
+		break;
+	case Opt_nls:
+	case Opt_charset:
+		if (vol->nls_map)
+			unload_nls(vol->nls_map);
+		vol->nls_map = load_nls(param->string);
+		if (!vol->nls_map) {
+			ntfs_error(vol->sb, "Failed to load NLS table '%s'.",
+				   param->string);
+			return -EINVAL;
+		}
+		break;
+	case Opt_mft_zone_multiplier:
+		if (vol->mft_zone_multiplier && vol->mft_zone_multiplier !=
+				result.int_32) {
+			ntfs_error(vol->sb, "Cannot change mft_zone_multiplier on remount.");
+			return -EINVAL;
+		}
+		if (result.int_32 < 1 || result.int_32 > 4) {
+			ntfs_error(vol->sb,
+				"Invalid mft_zone_multiplier. Using default value, i.e. 1.");
+			vol->mft_zone_multiplier = 1;
+		} else
+			vol->mft_zone_multiplier = result.int_32;
+		break;
+	case Opt_show_sys_files:
+	case Opt_show_meta:
+		if (result.boolean)
+			NVolSetShowSystemFiles(vol);
+		else
+			NVolClearShowSystemFiles(vol);
+		break;
+	case Opt_case_sensitive:
+		if (result.boolean)
+			NVolSetCaseSensitive(vol);
+		else
+			NVolClearCaseSensitive(vol);
+		break;
+	case Opt_nocase:
+		if (result.boolean)
+			NVolClearCaseSensitive(vol);
+		else
+			NVolSetCaseSensitive(vol);
+		break;
+	case Opt_preallocated_size:
+		vol->preallocated_size = (loff_t)result.uint_64;
+		break;
+	case Opt_sys_immutable:
+		if (result.boolean)
+			NVolSetSysImmutable(vol);
+		else
+			NVolClearSysImmutable(vol);
+		break;
+	case Opt_nohidden:
+		if (result.boolean)
+			NVolClearShowHiddenFiles(vol);
+		else
+			NVolSetShowHiddenFiles(vol);
+		break;
+	case Opt_hide_dot_files:
+		if (result.boolean)
+			NVolSetHideDotFiles(vol);
+		else
+			NVolClearHideDotFiles(vol);
+		break;
+	case Opt_check_windows_names:
+		if (result.boolean)
+			NVolSetCheckWindowsNames(vol);
+		else
+			NVolClearCheckWindowsNames(vol);
+		break;
+	case Opt_acl:
+#ifdef CONFIG_NTFS_FS_POSIX_ACL
+		if (result.boolean)
+			fc->sb_flags |= SB_POSIXACL;
+		else
+			fc->sb_flags &= ~SB_POSIXACL;
+		break;
+#else
+		return -EINVAL;
+#endif
+	case Opt_discard:
+		if (result.boolean)
+			NVolSetDiscard(vol);
+		else
+			NVolClearDiscard(vol);
+		break;
+	case Opt_disable_sparse:
+		if (result.boolean)
+			NVolSetDisableSparse(vol);
+		else
+			NVolClearDisableSparse(vol);
+		break;
+	case Opt_sparse:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ntfs_reconfigure(struct fs_context *fc)
+{
+	struct super_block *sb = fc->root->d_sb;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+
+	ntfs_debug("Entering with remount");
+
+	sync_filesystem(sb);
+
+	/*
+	 * For the read-write compiled driver, if we are remounting read-write,
+	 * make sure there are no volume errors and that no unsupported volume
+	 * flags are set.  Also, empty the logfile journal as it would become
+	 * stale as soon as something is written to the volume and mark the
+	 * volume dirty so that chkdsk is run if the volume is not umounted
+	 * cleanly.  Finally, mark the quotas out of date so Windows rescans
+	 * the volume on boot and updates them.
+	 *
+	 * When remounting read-only, mark the volume clean if no volume errors
+	 * have occurred.
+	 */
+	if (sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY)) {
+		static const char *es = ".  Cannot remount read-write.";
+
+		/* Remounting read-write. */
+		if (NVolErrors(vol)) {
+			ntfs_error(sb, "Volume has errors and is read-only%s",
+					es);
+			return -EROFS;
+		}
+		if (vol->vol_flags & VOLUME_IS_DIRTY) {
+			ntfs_error(sb, "Volume is dirty and read-only%s", es);
+			return -EROFS;
+		}
+		if (vol->vol_flags & VOLUME_MODIFIED_BY_CHKDSK) {
+			ntfs_error(sb, "Volume has been modified by chkdsk and is read-only%s", es);
+			return -EROFS;
+		}
+		if (vol->vol_flags & VOLUME_MUST_MOUNT_RO_MASK) {
+			ntfs_error(sb, "Volume has unsupported flags set (0x%x) and is read-only%s",
+					le16_to_cpu(vol->vol_flags), es);
+			return -EROFS;
+		}
+		if (vol->logfile_ino && !ntfs_empty_logfile(vol->logfile_ino)) {
+			ntfs_error(sb, "Failed to empty journal LogFile%s",
+					es);
+			NVolSetErrors(vol);
+			return -EROFS;
+		}
+		if (!ntfs_mark_quotas_out_of_date(vol)) {
+			ntfs_error(sb, "Failed to mark quotas out of date%s",
+					es);
+			NVolSetErrors(vol);
+			return -EROFS;
+		}
+	} else if (!sb_rdonly(sb) && (fc->sb_flags & SB_RDONLY)) {
+		/* Remounting read-only. */
+		if (!NVolErrors(vol)) {
+			if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY))
+				ntfs_warning(sb,
+					"Failed to clear dirty bit in volume information flags.  Run chkdsk.");
+		}
+	}
+
+	ntfs_debug("Done.");
+	return 0;
+}
+
+const struct option_t on_errors_arr[] = {
+	{ ON_ERRORS_PANIC,	"panic" },
+	{ ON_ERRORS_REMOUNT_RO,	"remount-ro", },
+	{ ON_ERRORS_CONTINUE,	"continue", },
+	{ 0,			NULL }
+};
+
+void ntfs_handle_error(struct super_block *sb)
+{
+	struct ntfs_volume *vol = NTFS_SB(sb);
+
+	if (sb_rdonly(sb))
+		return;
+
+	if (vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+		sb->s_flags |= SB_RDONLY;
+		pr_crit("(device %s): Filesystem has been set read-only\n",
+			sb->s_id);
+	} else if (vol->on_errors == ON_ERRORS_PANIC) {
+		panic("ntfs: (device %s): panic from previous error\n",
+		      sb->s_id);
+	} else if (vol->on_errors == ON_ERRORS_CONTINUE) {
+		if (errseq_check(&sb->s_wb_err, vol->wb_err) == -ENODEV) {
+			NVolSetShutdown(vol);
+			vol->wb_err = sb->s_wb_err;
+		}
+	}
+}
+
+/*
+ * ntfs_write_volume_flags - write new flags to the volume information flags
+ * @vol:	ntfs volume on which to modify the flags
+ * @flags:	new flags value for the volume information flags
+ *
+ * Internal function.  You probably want to use ntfs_{set,clear}_volume_flags()
+ * instead (see below).
+ *
+ * Replace the volume information flags on the volume @vol with the value
+ * supplied in @flags.  Note, this overwrites the volume information flags, so
+ * make sure to combine the flags you want to modify with the old flags and use
+ * the result when calling ntfs_write_volume_flags().
+ *
+ * Return 0 on success and -errno on error.
+ */
+static int ntfs_write_volume_flags(struct ntfs_volume *vol, const __le16 flags)
+{
+	struct ntfs_inode *ni = NTFS_I(vol->vol_ino);
+	struct volume_information *vi;
+	struct ntfs_attr_search_ctx *ctx;
+	int err;
+
+	ntfs_debug("Entering, old flags = 0x%x, new flags = 0x%x.",
+			le16_to_cpu(vol->vol_flags), le16_to_cpu(flags));
+	mutex_lock(&ni->mrec_lock);
+	if (vol->vol_flags == flags)
+		goto done;
+
+	ctx = ntfs_attr_get_search_ctx(ni, NULL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto put_unm_err_out;
+	}
+
+	err = ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0,
+			ctx);
+	if (err)
+		goto put_unm_err_out;
+
+	vi = (struct volume_information *)((u8 *)ctx->attr +
+			le16_to_cpu(ctx->attr->data.resident.value_offset));
+	vol->vol_flags = vi->flags = flags;
+	mark_mft_record_dirty(ctx->ntfs_ino);
+	ntfs_attr_put_search_ctx(ctx);
+done:
+	mutex_unlock(&ni->mrec_lock);
+	ntfs_debug("Done.");
+	return 0;
+put_unm_err_out:
+	if (ctx)
+		ntfs_attr_put_search_ctx(ctx);
+	mutex_unlock(&ni->mrec_lock);
+	ntfs_error(vol->sb, "Failed with error code %i.", -err);
+	return err;
+}
+
+/*
+ * ntfs_set_volume_flags - set bits in the volume information flags
+ * @vol:	ntfs volume on which to modify the flags
+ * @flags:	flags to set on the volume
+ *
+ * Set the bits in @flags in the volume information flags on the volume @vol.
+ *
+ * Return 0 on success and -errno on error.
+ */
+int ntfs_set_volume_flags(struct ntfs_volume *vol, __le16 flags)
+{
+	flags &= VOLUME_FLAGS_MASK;
+	return ntfs_write_volume_flags(vol, vol->vol_flags | flags);
+}
+
+/*
+ * ntfs_clear_volume_flags - clear bits in the volume information flags
+ * @vol:	ntfs volume on which to modify the flags
+ * @flags:	flags to clear on the volume
+ *
+ * Clear the bits in @flags in the volume information flags on the volume @vol.
+ *
+ * Return 0 on success and -errno on error.
+ */
+int ntfs_clear_volume_flags(struct ntfs_volume *vol, __le16 flags)
+{
+	flags &= VOLUME_FLAGS_MASK;
+	flags = vol->vol_flags & cpu_to_le16(~le16_to_cpu(flags));
+	return ntfs_write_volume_flags(vol, flags);
+}
+
+int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
+{
+	struct ntfs_inode *vol_ni = NTFS_I(vol->vol_ino);
+	struct ntfs_attr_search_ctx *ctx;
+	__le16 *uname;
+	int uname_len, ret;
+
+	uname_len = ntfs_nlstoucs(vol, label, strlen(label),
+				  &uname, FSLABEL_MAX);
+	if (uname_len < 0) {
+		ntfs_error(vol->sb,
+			"Failed to convert volume label '%s' to Unicode.",
+			label);
+		return uname_len;
+	}
+
+	if (uname_len  > NTFS_MAX_LABEL_LEN) {
+		ntfs_error(vol->sb,
+			   "Volume label is too long (max %d characters).",
+			   NTFS_MAX_LABEL_LEN);
+		kvfree(uname);
+		return -EINVAL;
+	}
+
+	mutex_lock(&vol_ni->mrec_lock);
+	ctx = ntfs_attr_get_search_ctx(vol_ni, NULL);
+	if (!ctx) {
+		ret = -ENOMEM;
+		goto  out;
+	}
+
+	if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0,
+			     ctx))
+		ntfs_attr_record_rm(ctx);
+	ntfs_attr_put_search_ctx(ctx);
+
+	ret = ntfs_resident_attr_record_add(vol_ni, AT_VOLUME_NAME, AT_UNNAMED, 0,
+					    (u8 *)uname, uname_len * sizeof(__le16), 0);
+out:
+	mutex_unlock(&vol_ni->mrec_lock);
+	kvfree(uname);
+	mark_inode_dirty_sync(vol->vol_ino);
+
+	if (ret >= 0) {
+		kfree(vol->volume_label);
+		vol->volume_label = kstrdup(label, GFP_KERNEL);
+		ret = 0;
+	}
+	return ret;
+}
+
+/*
+ * is_boot_sector_ntfs - check whether a boot sector is a valid NTFS boot sector
+ * @sb:		Super block of the device to which @b belongs.
+ * @b:		Boot sector of device @sb to check.
+ * @silent:	If 'true', all output will be silenced.
+ *
+ * is_boot_sector_ntfs() checks whether the boot sector @b is a valid NTFS boot
+ * sector. Returns 'true' if it is valid and 'false' if not.
+ *
+ * @sb is only needed for warning/error output, i.e. it can be NULL when silent
+ * is 'true'.
+ */
+static bool is_boot_sector_ntfs(const struct super_block *sb,
+		const struct ntfs_boot_sector *b, const bool silent)
+{
+	/*
+	 * Check that checksum == sum of u32 values from b to the checksum
+	 * field.  If checksum is zero, no checking is done.  We will work when
+	 * the checksum test fails, since some utilities update the boot sector
+	 * ignoring the checksum which leaves the checksum out-of-date.  We
+	 * report a warning if this is the case.
+	 */
+	if ((void *)b < (void *)&b->checksum && b->checksum && !silent) {
+		__le32 *u;
+		u32 i;
+
+		for (i = 0, u = (__le32 *)b; u < (__le32 *)(&b->checksum); ++u)
+			i += le32_to_cpup(u);
+		if (le32_to_cpu(b->checksum) != i)
+			ntfs_warning(sb, "Invalid boot sector checksum.");
+	}
+	/* Check OEMidentifier is "NTFS    " */
+	if (b->oem_id != magicNTFS)
+		goto not_ntfs;
+	/* Check bytes per sector value is between 256 and 4096. */
+	if (le16_to_cpu(b->bpb.bytes_per_sector) < 0x100 ||
+	    le16_to_cpu(b->bpb.bytes_per_sector) > 0x1000)
+		goto not_ntfs;
+	/*
+	 * Check sectors per cluster value is valid and the cluster size
+	 * is not above the maximum (2MB).
+	 */
+	if (b->bpb.sectors_per_cluster > 0x80 &&
+	    b->bpb.sectors_per_cluster < 0xf4)
+		goto not_ntfs;
+
+	/* Check reserved/unused fields are really zero. */
+	if (le16_to_cpu(b->bpb.reserved_sectors) ||
+			le16_to_cpu(b->bpb.root_entries) ||
+			le16_to_cpu(b->bpb.sectors) ||
+			le16_to_cpu(b->bpb.sectors_per_fat) ||
+			le32_to_cpu(b->bpb.large_sectors) || b->bpb.fats)
+		goto not_ntfs;
+	/* Check clusters per file mft record value is valid. */
+	if ((u8)b->clusters_per_mft_record < 0xe1 ||
+			(u8)b->clusters_per_mft_record > 0xf7)
+		switch (b->clusters_per_mft_record) {
+		case 1: case 2: case 4: case 8: case 16: case 32: case 64:
+			break;
+		default:
+			goto not_ntfs;
+		}
+	/* Check clusters per index block value is valid. */
+	if ((u8)b->clusters_per_index_record < 0xe1 ||
+			(u8)b->clusters_per_index_record > 0xf7)
+		switch (b->clusters_per_index_record) {
+		case 1: case 2: case 4: case 8: case 16: case 32: case 64:
+			break;
+		default:
+			goto not_ntfs;
+		}
+	/*
+	 * Check for valid end of sector marker. We will work without it, but
+	 * many BIOSes will refuse to boot from a bootsector if the magic is
+	 * incorrect, so we emit a warning.
+	 */
+	if (!silent && b->end_of_sector_marker != cpu_to_le16(0xaa55))
+		ntfs_warning(sb, "Invalid end of sector marker.");
+	return true;
+not_ntfs:
+	return false;
+}
+
+/*
+ * read_ntfs_boot_sector - read the NTFS boot sector of a device
+ * @sb:		super block of device to read the boot sector from
+ * @silent:	if true, suppress all output
+ *
+ * Reads the boot sector from the device and validates it.
+ */
+static char *read_ntfs_boot_sector(struct super_block *sb,
+		const int silent)
+{
+	char *boot_sector;
+
+	boot_sector = kzalloc(PAGE_SIZE, GFP_NOFS);
+	if (!boot_sector)
+		return NULL;
+
+	if (ntfs_bdev_read(sb->s_bdev, boot_sector, 0, PAGE_SIZE)) {
+		if (!silent)
+			ntfs_error(sb, "Unable to read primary boot sector.");
+		kfree(boot_sector);
+		return NULL;
+	}
+
+	if (!is_boot_sector_ntfs(sb, (struct ntfs_boot_sector *)boot_sector,
+				 silent)) {
+		if (!silent)
+			ntfs_error(sb, "Primary boot sector is invalid.");
+		kfree(boot_sector);
+		return NULL;
+	}
+
+	return boot_sector;
+}
+
+/*
+ * parse_ntfs_boot_sector - parse the boot sector and store the data in @vol
+ * @vol:	volume structure to initialise with data from boot sector
+ * @b:		boot sector to parse
+ *
+ * Parse the ntfs boot sector @b and store all imporant information therein in
+ * the ntfs super block @vol.  Return 'true' on success and 'false' on error.
+ */
+static bool parse_ntfs_boot_sector(struct ntfs_volume *vol,
+		const struct ntfs_boot_sector *b)
+{
+	unsigned int sectors_per_cluster, sectors_per_cluster_bits, nr_hidden_sects;
+	int clusters_per_mft_record, clusters_per_index_record;
+	s64 ll;
+
+	vol->sector_size = le16_to_cpu(b->bpb.bytes_per_sector);
+	vol->sector_size_bits = ffs(vol->sector_size) - 1;
+	ntfs_debug("vol->sector_size = %i (0x%x)", vol->sector_size,
+			vol->sector_size);
+	ntfs_debug("vol->sector_size_bits = %i (0x%x)", vol->sector_size_bits,
+			vol->sector_size_bits);
+	if (vol->sector_size < vol->sb->s_blocksize) {
+		ntfs_error(vol->sb,
+			"Sector size (%i) is smaller than the device block size (%lu).  This is not supported.",
+			vol->sector_size, vol->sb->s_blocksize);
+		return false;
+	}
+
+	if (b->bpb.sectors_per_cluster >= 0xf4)
+		sectors_per_cluster = 1U << -(s8)b->bpb.sectors_per_cluster;
+	else
+		sectors_per_cluster = b->bpb.sectors_per_cluster;
+	ntfs_debug("sectors_per_cluster = 0x%x", b->bpb.sectors_per_cluster);
+	sectors_per_cluster_bits = ffs(sectors_per_cluster) - 1;
+	ntfs_debug("sectors_per_cluster_bits = 0x%x",
+			sectors_per_cluster_bits);
+	nr_hidden_sects = le32_to_cpu(b->bpb.hidden_sectors);
+	ntfs_debug("number of hidden sectors = 0x%x", nr_hidden_sects);
+	vol->cluster_size = vol->sector_size << sectors_per_cluster_bits;
+	vol->cluster_size_mask = vol->cluster_size - 1;
+	vol->cluster_size_bits = ffs(vol->cluster_size) - 1;
+	ntfs_debug("vol->cluster_size = %i (0x%x)", vol->cluster_size,
+			vol->cluster_size);
+	ntfs_debug("vol->cluster_size_mask = 0x%x", vol->cluster_size_mask);
+	ntfs_debug("vol->cluster_size_bits = %i", vol->cluster_size_bits);
+	if (vol->cluster_size < vol->sector_size) {
+		ntfs_error(vol->sb,
+			"Cluster size (%i) is smaller than the sector size (%i).  This is not supported.",
+			vol->cluster_size, vol->sector_size);
+		return false;
+	}
+	clusters_per_mft_record = b->clusters_per_mft_record;
+	ntfs_debug("clusters_per_mft_record = %i (0x%x)",
+			clusters_per_mft_record, clusters_per_mft_record);
+	if (clusters_per_mft_record > 0)
+		vol->mft_record_size = vol->cluster_size <<
+				(ffs(clusters_per_mft_record) - 1);
+	else
+		/*
+		 * When mft_record_size < cluster_size, clusters_per_mft_record
+		 * = -log2(mft_record_size) bytes. mft_record_size normaly is
+		 * 1024 bytes, which is encoded as 0xF6 (-10 in decimal).
+		 */
+		vol->mft_record_size = 1 << -clusters_per_mft_record;
+	vol->mft_record_size_mask = vol->mft_record_size - 1;
+	vol->mft_record_size_bits = ffs(vol->mft_record_size) - 1;
+	ntfs_debug("vol->mft_record_size = %i (0x%x)", vol->mft_record_size,
+			vol->mft_record_size);
+	ntfs_debug("vol->mft_record_size_mask = 0x%x",
+			vol->mft_record_size_mask);
+	ntfs_debug("vol->mft_record_size_bits = %i (0x%x)",
+			vol->mft_record_size_bits, vol->mft_record_size_bits);
+	/*
+	 * We cannot support mft record sizes above the PAGE_SIZE since
+	 * we store $MFT/$DATA, the table of mft records in the page cache.
+	 */
+	if (vol->mft_record_size > PAGE_SIZE) {
+		ntfs_error(vol->sb,
+			"Mft record size (%i) exceeds the PAGE_SIZE on your system (%lu).  This is not supported.",
+			vol->mft_record_size, PAGE_SIZE);
+		return false;
+	}
+	/* We cannot support mft record sizes below the sector size. */
+	if (vol->mft_record_size < vol->sector_size) {
+		ntfs_warning(vol->sb, "Mft record size (%i) is smaller than the sector size (%i).",
+				vol->mft_record_size, vol->sector_size);
+	}
+	clusters_per_index_record = b->clusters_per_index_record;
+	ntfs_debug("clusters_per_index_record = %i (0x%x)",
+			clusters_per_index_record, clusters_per_index_record);
+	if (clusters_per_index_record > 0)
+		vol->index_record_size = vol->cluster_size <<
+				(ffs(clusters_per_index_record) - 1);
+	else
+		/*
+		 * When index_record_size < cluster_size,
+		 * clusters_per_index_record = -log2(index_record_size) bytes.
+		 * index_record_size normaly equals 4096 bytes, which is
+		 * encoded as 0xF4 (-12 in decimal).
+		 */
+		vol->index_record_size = 1 << -clusters_per_index_record;
+	vol->index_record_size_mask = vol->index_record_size - 1;
+	vol->index_record_size_bits = ffs(vol->index_record_size) - 1;
+	ntfs_debug("vol->index_record_size = %i (0x%x)",
+			vol->index_record_size, vol->index_record_size);
+	ntfs_debug("vol->index_record_size_mask = 0x%x",
+			vol->index_record_size_mask);
+	ntfs_debug("vol->index_record_size_bits = %i (0x%x)",
+			vol->index_record_size_bits,
+			vol->index_record_size_bits);
+	/* We cannot support index record sizes below the sector size. */
+	if (vol->index_record_size < vol->sector_size) {
+		ntfs_error(vol->sb,
+			   "Index record size (%i) is smaller than the sector size (%i).  This is not supported.",
+			   vol->index_record_size, vol->sector_size);
+		return false;
+	}
+	/*
+	 * Get the size of the volume in clusters and check for 64-bit-ness.
+	 * Windows currently only uses 32 bits to save the clusters so we do
+	 * the same as it is much faster on 32-bit CPUs.
+	 */
+	ll = le64_to_cpu(b->number_of_sectors) >> sectors_per_cluster_bits;
+	if ((u64)ll >= 1ULL << 32) {
+		ntfs_error(vol->sb, "Cannot handle 64-bit clusters.");
+		return false;
+	}
+	vol->nr_clusters = ll;
+	ntfs_debug("vol->nr_clusters = 0x%llx", vol->nr_clusters);
+	ll = le64_to_cpu(b->mft_lcn);
+	if (ll >= vol->nr_clusters) {
+		ntfs_error(vol->sb, "MFT LCN (%lli, 0x%llx) is beyond end of volume.  Weird.",
+				ll, ll);
+		return false;
+	}
+	vol->mft_lcn = ll;
+	ntfs_debug("vol->mft_lcn = 0x%llx", vol->mft_lcn);
+	ll = le64_to_cpu(b->mftmirr_lcn);
+	if (ll >= vol->nr_clusters) {
+		ntfs_error(vol->sb, "MFTMirr LCN (%lli, 0x%llx) is beyond end of volume.  Weird.",
+				ll, ll);
+		return false;
+	}
+	vol->mftmirr_lcn = ll;
+	ntfs_debug("vol->mftmirr_lcn = 0x%llx", vol->mftmirr_lcn);
+	/*
+	 * Work out the size of the mft mirror in number of mft records. If the
+	 * cluster size is less than or equal to the size taken by four mft
+	 * records, the mft mirror stores the first four mft records. If the
+	 * cluster size is bigger than the size taken by four mft records, the
+	 * mft mirror contains as many mft records as will fit into one
+	 * cluster.
+	 */
+	if (vol->cluster_size <= (4 << vol->mft_record_size_bits))
+		vol->mftmirr_size = 4;
+	else
+		vol->mftmirr_size = vol->cluster_size >>
+				vol->mft_record_size_bits;
+	ntfs_debug("vol->mftmirr_size = %i", vol->mftmirr_size);
+	vol->serial_no = le64_to_cpu(b->volume_serial_number);
+	ntfs_debug("vol->serial_no = 0x%llx", vol->serial_no);
+
+	vol->sparse_compression_unit = 4;
+	if (vol->cluster_size > 4096) {
+		switch (vol->cluster_size) {
+		case 65536:
+			vol->sparse_compression_unit = 0;
+			break;
+		case 32768:
+			vol->sparse_compression_unit = 1;
+			break;
+		case 16384:
+			vol->sparse_compression_unit = 2;
+			break;
+		case 8192:
+			vol->sparse_compression_unit = 3;
+			break;
+		}
+	}
+
+	return true;
+}
+
+/*
+ * ntfs_setup_allocators - initialize the cluster and mft allocators
+ * @vol:	volume structure for which to setup the allocators
+ *
+ * Setup the cluster (lcn) and mft allocators to the starting values.
+ */
+static void ntfs_setup_allocators(struct ntfs_volume *vol)
+{
+	s64 mft_zone_size, mft_lcn;
+
+	ntfs_debug("vol->mft_zone_multiplier = 0x%x",
+			vol->mft_zone_multiplier);
+	/* Determine the size of the MFT zone. */
+	mft_zone_size = vol->nr_clusters;
+	switch (vol->mft_zone_multiplier) {  /* % of volume size in clusters */
+	case 4:
+		mft_zone_size >>= 1;			/* 50%   */
+		break;
+	case 3:
+		mft_zone_size = (mft_zone_size +
+				(mft_zone_size >> 1)) >> 2;	/* 37.5% */
+		break;
+	case 2:
+		mft_zone_size >>= 2;			/* 25%   */
+		break;
+	/* case 1: */
+	default:
+		mft_zone_size >>= 3;			/* 12.5% */
+		break;
+	}
+	/* Setup the mft zone. */
+	vol->mft_zone_start = vol->mft_zone_pos = vol->mft_lcn;
+	ntfs_debug("vol->mft_zone_pos = 0x%llx", vol->mft_zone_pos);
+	/*
+	 * Calculate the mft_lcn for an unmodified NTFS volume (see mkntfs
+	 * source) and if the actual mft_lcn is in the expected place or even
+	 * further to the front of the volume, extend the mft_zone to cover the
+	 * beginning of the volume as well.  This is in order to protect the
+	 * area reserved for the mft bitmap as well within the mft_zone itself.
+	 * On non-standard volumes we do not protect it as the overhead would
+	 * be higher than the speed increase we would get by doing it.
+	 */
+	mft_lcn = NTFS_B_TO_CLU(vol, 8192 + 2 * vol->cluster_size - 1);
+	if (mft_lcn * vol->cluster_size < 16 * 1024)
+		mft_lcn = (16 * 1024 + vol->cluster_size - 1) >>
+				vol->cluster_size_bits;
+	if (vol->mft_zone_start <= mft_lcn)
+		vol->mft_zone_start = 0;
+	ntfs_debug("vol->mft_zone_start = 0x%llx", vol->mft_zone_start);
+	/*
+	 * Need to cap the mft zone on non-standard volumes so that it does
+	 * not point outside the boundaries of the volume.  We do this by
+	 * halving the zone size until we are inside the volume.
+	 */
+	vol->mft_zone_end = vol->mft_lcn + mft_zone_size;
+	while (vol->mft_zone_end >= vol->nr_clusters) {
+		mft_zone_size >>= 1;
+		vol->mft_zone_end = vol->mft_lcn + mft_zone_size;
+	}
+	ntfs_debug("vol->mft_zone_end = 0x%llx", vol->mft_zone_end);
+	/*
+	 * Set the current position within each data zone to the start of the
+	 * respective zone.
+	 */
+	vol->data1_zone_pos = vol->mft_zone_end;
+	ntfs_debug("vol->data1_zone_pos = 0x%llx", vol->data1_zone_pos);
+	vol->data2_zone_pos = 0;
+	ntfs_debug("vol->data2_zone_pos = 0x%llx", vol->data2_zone_pos);
+
+	/* Set the mft data allocation position to mft record 24. */
+	vol->mft_data_pos = 24;
+	ntfs_debug("vol->mft_data_pos = 0x%llx", vol->mft_data_pos);
+}
+
+static struct lock_class_key mftmirr_runlist_lock_key,
+			     mftmirr_mrec_lock_key;
+/*
+ * load_and_init_mft_mirror - load and setup the mft mirror inode for a volume
+ * @vol:	ntfs super block describing device whose mft mirror to load
+ *
+ * Return 'true' on success or 'false' on error.
+ */
+static bool load_and_init_mft_mirror(struct ntfs_volume *vol)
+{
+	struct inode *tmp_ino;
+	struct ntfs_inode *tmp_ni;
+
+	ntfs_debug("Entering.");
+	/* Get mft mirror inode. */
+	tmp_ino = ntfs_iget(vol->sb, FILE_MFTMirr);
+	if (IS_ERR(tmp_ino)) {
+		if (!IS_ERR(tmp_ino))
+			iput(tmp_ino);
+		/* Caller will display error message. */
+		return false;
+	}
+	lockdep_set_class(&NTFS_I(tmp_ino)->runlist.lock,
+			  &mftmirr_runlist_lock_key);
+	lockdep_set_class(&NTFS_I(tmp_ino)->mrec_lock,
+			  &mftmirr_mrec_lock_key);
+	/*
+	 * Re-initialize some specifics about $MFTMirr's inode as
+	 * ntfs_read_inode() will have set up the default ones.
+	 */
+	/* Set uid and gid to root. */
+	tmp_ino->i_uid = GLOBAL_ROOT_UID;
+	tmp_ino->i_gid = GLOBAL_ROOT_GID;
+	/* Regular file.  No access for anyone. */
+	tmp_ino->i_mode = S_IFREG;
+	/* No VFS initiated operations allowed for $MFTMirr. */
+	tmp_ino->i_op = &ntfs_empty_inode_ops;
+	tmp_ino->i_fop = &ntfs_empty_file_ops;
+	/* Put in our special address space operations. */
+	tmp_ino->i_mapping->a_ops = &ntfs_aops;
+	tmp_ni = NTFS_I(tmp_ino);
+	/* The $MFTMirr, like the $MFT is multi sector transfer protected. */
+	NInoSetMstProtected(tmp_ni);
+	NInoSetSparseDisabled(tmp_ni);
+	/*
+	 * Set up our little cheat allowing us to reuse the async read io
+	 * completion handler for directories.
+	 */
+	tmp_ni->itype.index.block_size = vol->mft_record_size;
+	tmp_ni->itype.index.block_size_bits = vol->mft_record_size_bits;
+	vol->mftmirr_ino = tmp_ino;
+	ntfs_debug("Done.");
+	return true;
+}
+
+/*
+ * check_mft_mirror - compare contents of the mft mirror with the mft
+ * @vol:	ntfs super block describing device whose mft mirror to check
+ *
+ * Return 'true' on success or 'false' on error.
+ *
+ * Note, this function also results in the mft mirror runlist being completely
+ * mapped into memory.  The mft mirror write code requires this and will BUG()
+ * should it find an unmapped runlist element.
+ */
+static bool check_mft_mirror(struct ntfs_volume *vol)
+{
+	struct super_block *sb = vol->sb;
+	struct ntfs_inode *mirr_ni;
+	struct folio *mft_folio = NULL, *mirr_folio = NULL;
+	u8 *kmft = NULL, *kmirr = NULL;
+	struct runlist_element *rl, rl2[2];
+	pgoff_t index;
+	int mrecs_per_page, i;
+
+	ntfs_debug("Entering.");
+	/* Compare contents of $MFT and $MFTMirr. */
+	mrecs_per_page = PAGE_SIZE / vol->mft_record_size;
+	index = i = 0;
+	do {
+		u32 bytes;
+
+		/* Switch pages if necessary. */
+		if (!(i % mrecs_per_page)) {
+			if (index) {
+				kunmap_local(kmirr);
+				folio_put(mirr_folio);
+				kunmap_local(kmft);
+				folio_put(mft_folio);
+			}
+			/* Get the $MFT page. */
+			mft_folio = read_mapping_folio(vol->mft_ino->i_mapping,
+					index, NULL);
+			if (IS_ERR(mft_folio)) {
+				ntfs_error(sb, "Failed to read $MFT.");
+				return false;
+			}
+			kmft = kmap_local_folio(mft_folio, 0);
+			/* Get the $MFTMirr page. */
+			mirr_folio = read_mapping_folio(vol->mftmirr_ino->i_mapping,
+					index, NULL);
+			if (IS_ERR(mirr_folio)) {
+				ntfs_error(sb, "Failed to read $MFTMirr.");
+				goto mft_unmap_out;
+			}
+			kmirr = kmap_local_folio(mirr_folio, 0);
+			++index;
+		}
+
+		/* Do not check the record if it is not in use. */
+		if (((struct mft_record *)kmft)->flags & MFT_RECORD_IN_USE) {
+			/* Make sure the record is ok. */
+			if (ntfs_is_baad_recordp((__le32 *)kmft)) {
+				ntfs_error(sb,
+					"Incomplete multi sector transfer detected in mft record %i.",
+					i);
+mm_unmap_out:
+				kunmap_local(kmirr);
+				folio_put(mirr_folio);
+mft_unmap_out:
+				kunmap_local(kmft);
+				folio_put(mft_folio);
+				return false;
+			}
+		}
+		/* Do not check the mirror record if it is not in use. */
+		if (((struct mft_record *)kmirr)->flags & MFT_RECORD_IN_USE) {
+			if (ntfs_is_baad_recordp((__le32 *)kmirr)) {
+				ntfs_error(sb,
+					"Incomplete multi sector transfer detected in mft mirror record %i.",
+					i);
+				goto mm_unmap_out;
+			}
+		}
+		/* Get the amount of data in the current record. */
+		bytes = le32_to_cpu(((struct mft_record *)kmft)->bytes_in_use);
+		if (bytes < sizeof(struct mft_record_old) ||
+		    bytes > vol->mft_record_size ||
+		    ntfs_is_baad_recordp((__le32 *)kmft)) {
+			bytes = le32_to_cpu(((struct mft_record *)kmirr)->bytes_in_use);
+			if (bytes < sizeof(struct mft_record_old) ||
+			    bytes > vol->mft_record_size ||
+			    ntfs_is_baad_recordp((__le32 *)kmirr))
+				bytes = vol->mft_record_size;
+		}
+		kmft += vol->mft_record_size;
+		kmirr += vol->mft_record_size;
+	} while (++i < vol->mftmirr_size);
+	/* Release the last folios. */
+	kunmap_local(kmirr);
+	folio_put(mirr_folio);
+	kunmap_local(kmft);
+	folio_put(mft_folio);
+
+	/* Construct the mft mirror runlist by hand. */
+	rl2[0].vcn = 0;
+	rl2[0].lcn = vol->mftmirr_lcn;
+	rl2[0].length = NTFS_B_TO_CLU(vol, vol->mftmirr_size * vol->mft_record_size +
+				vol->cluster_size - 1);
+	rl2[1].vcn = rl2[0].length;
+	rl2[1].lcn = LCN_ENOENT;
+	rl2[1].length = 0;
+	/*
+	 * Because we have just read all of the mft mirror, we know we have
+	 * mapped the full runlist for it.
+	 */
+	mirr_ni = NTFS_I(vol->mftmirr_ino);
+	down_read(&mirr_ni->runlist.lock);
+	rl = mirr_ni->runlist.rl;
+	/* Compare the two runlists.  They must be identical. */
+	i = 0;
+	do {
+		if (rl2[i].vcn != rl[i].vcn || rl2[i].lcn != rl[i].lcn ||
+				rl2[i].length != rl[i].length) {
+			ntfs_error(sb, "$MFTMirr location mismatch.  Run chkdsk.");
+			up_read(&mirr_ni->runlist.lock);
+			return false;
+		}
+	} while (rl2[i++].length);
+	up_read(&mirr_ni->runlist.lock);
+	ntfs_debug("Done.");
+	return true;
+}
+
+/*
+ * load_and_check_logfile - load and check the logfile inode for a volume
+ * @vol: ntfs volume to load the logfile for
+ * @rp: on success, set to the restart page header
+ *
+ * Return 0 on success or errno on error.
+ */
+static int load_and_check_logfile(struct ntfs_volume *vol,
+				  struct restart_page_header **rp)
+{
+	struct inode *tmp_ino;
+	int err = 0;
+
+	ntfs_debug("Entering.");
+	tmp_ino = ntfs_iget(vol->sb, FILE_LogFile);
+	if (IS_ERR(tmp_ino)) {
+		if (!IS_ERR(tmp_ino))
+			iput(tmp_ino);
+		/* Caller will display error message. */
+		return -ENOENT;
+	}
+	if (!ntfs_check_logfile(tmp_ino, rp))
+		err = -EINVAL;
+	NInoSetSparseDisabled(NTFS_I(tmp_ino));
+	vol->logfile_ino = tmp_ino;
+	ntfs_debug("Done.");
+	return err;
+}
+
+#define NTFS_HIBERFIL_HEADER_SIZE	4096
+
+/*
+ * check_windows_hibernation_status - check if Windows is suspended on a volume
+ * @vol:	ntfs super block of device to check
+ *
+ * Check if Windows is hibernated on the ntfs volume @vol.  This is done by
+ * looking for the file hiberfil.sys in the root directory of the volume.  If
+ * the file is not present Windows is definitely not suspended.
+ *
+ * If hiberfil.sys exists and is less than 4kiB in size it means Windows is
+ * definitely suspended (this volume is not the system volume).  Caveat:  on a
+ * system with many volumes it is possible that the < 4kiB check is bogus but
+ * for now this should do fine.
+ *
+ * If hiberfil.sys exists and is larger than 4kiB in size, we need to read the
+ * hiberfil header (which is the first 4kiB).  If this begins with "hibr",
+ * Windows is definitely suspended.  If it is completely full of zeroes,
+ * Windows is definitely not hibernated.  Any other case is treated as if
+ * Windows is suspended.  This caters for the above mentioned caveat of a
+ * system with many volumes where no "hibr" magic would be present and there is
+ * no zero header.
+ *
+ * Return 0 if Windows is not hibernated on the volume, >0 if Windows is
+ * hibernated on the volume, and -errno on error.
+ */
+static int check_windows_hibernation_status(struct ntfs_volume *vol)
+{
+	static const __le16 hiberfil[13] = { cpu_to_le16('h'),
+			cpu_to_le16('i'), cpu_to_le16('b'),
+			cpu_to_le16('e'), cpu_to_le16('r'),
+			cpu_to_le16('f'), cpu_to_le16('i'),
+			cpu_to_le16('l'), cpu_to_le16('.'),
+			cpu_to_le16('s'), cpu_to_le16('y'),
+			cpu_to_le16('s'), 0 };
+	u64 mref;
+	struct inode *vi;
+	struct folio *folio;
+	u32 *kaddr, *kend, *start_addr = NULL;
+	struct ntfs_name *name = NULL;
+	int ret = 1;
+
+	ntfs_debug("Entering.");
+	/*
+	 * Find the inode number for the hibernation file by looking up the
+	 * filename hiberfil.sys in the root directory.
+	 */
+	inode_lock(vol->root_ino);
+	mref = ntfs_lookup_inode_by_name(NTFS_I(vol->root_ino), hiberfil, 12,
+			&name);
+	inode_unlock(vol->root_ino);
+	kfree(name);
+	if (IS_ERR_MREF(mref)) {
+		ret = MREF_ERR(mref);
+		/* If the file does not exist, Windows is not hibernated. */
+		if (ret == -ENOENT) {
+			ntfs_debug("hiberfil.sys not present.  Windows is not hibernated on the volume.");
+			return 0;
+		}
+		/* A real error occurred. */
+		ntfs_error(vol->sb, "Failed to find inode number for hiberfil.sys.");
+		return ret;
+	}
+	/* Get the inode. */
+	vi = ntfs_iget(vol->sb, MREF(mref));
+	if (IS_ERR(vi)) {
+		if (!IS_ERR(vi))
+			iput(vi);
+		ntfs_error(vol->sb, "Failed to load hiberfil.sys.");
+		return IS_ERR(vi) ? PTR_ERR(vi) : -EIO;
+	}
+	if (unlikely(i_size_read(vi) < NTFS_HIBERFIL_HEADER_SIZE)) {
+		ntfs_debug("hiberfil.sys is smaller than 4kiB (0x%llx).  Windows is hibernated on the volume.  This is not the system volume.",
+				i_size_read(vi));
+		goto iput_out;
+	}
+
+	folio = read_mapping_folio(vi->i_mapping, 0, NULL);
+	if (IS_ERR(folio)) {
+		ntfs_error(vol->sb, "Failed to read from hiberfil.sys.");
+		ret = PTR_ERR(folio);
+		goto iput_out;
+	}
+	start_addr = (u32 *)kmap_local_folio(folio, 0);
+	kaddr = start_addr;
+	if (*(__le32 *)kaddr == cpu_to_le32(0x72626968)/*'hibr'*/) {
+		ntfs_debug("Magic \"hibr\" found in hiberfil.sys.  Windows is hibernated on the volume.  This is the system volume.");
+		goto unm_iput_out;
+	}
+	kend = kaddr + NTFS_HIBERFIL_HEADER_SIZE/sizeof(*kaddr);
+	do {
+		if (unlikely(*kaddr)) {
+			ntfs_debug("hiberfil.sys is larger than 4kiB (0x%llx), does not contain the \"hibr\" magic, and does not have a zero header.  Windows is hibernated on the volume.  This is not the system volume.",
+					i_size_read(vi));
+			goto unm_iput_out;
+		}
+	} while (++kaddr < kend);
+	ntfs_debug("hiberfil.sys contains a zero header.  Windows is not hibernated on the volume.  This is the system volume.");
+	ret = 0;
+unm_iput_out:
+	kunmap_local(start_addr);
+	folio_put(folio);
+iput_out:
+	iput(vi);
+	return ret;
+}
+
+/*
+ * load_and_init_quota - load and setup the quota file for a volume if present
+ * @vol:	ntfs super block describing device whose quota file to load
+ *
+ * Return 'true' on success or 'false' on error.  If $Quota is not present, we
+ * leave vol->quota_ino as NULL and return success.
+ */
+static bool load_and_init_quota(struct ntfs_volume *vol)
+{
+	static const __le16 Quota[7] = { cpu_to_le16('$'),
+			cpu_to_le16('Q'), cpu_to_le16('u'),
+			cpu_to_le16('o'), cpu_to_le16('t'),
+			cpu_to_le16('a'), 0 };
+	static __le16 Q[3] = { cpu_to_le16('$'),
+			cpu_to_le16('Q'), 0 };
+	struct ntfs_name *name = NULL;
+	u64 mref;
+	struct inode *tmp_ino;
+
+	ntfs_debug("Entering.");
+	/*
+	 * Find the inode number for the quota file by looking up the filename
+	 * $Quota in the extended system files directory $Extend.
+	 */
+	inode_lock(vol->extend_ino);
+	mref = ntfs_lookup_inode_by_name(NTFS_I(vol->extend_ino), Quota, 6,
+			&name);
+	inode_unlock(vol->extend_ino);
+	kfree(name);
+	if (IS_ERR_MREF(mref)) {
+		/*
+		 * If the file does not exist, quotas are disabled and have
+		 * never been enabled on this volume, just return success.
+		 */
+		if (MREF_ERR(mref) == -ENOENT) {
+			ntfs_debug("$Quota not present.  Volume does not have quotas enabled.");
+			/*
+			 * No need to try to set quotas out of date if they are
+			 * not enabled.
+			 */
+			NVolSetQuotaOutOfDate(vol);
+			return true;
+		}
+		/* A real error occurred. */
+		ntfs_error(vol->sb, "Failed to find inode number for $Quota.");
+		return false;
+	}
+	/* Get the inode. */
+	tmp_ino = ntfs_iget(vol->sb, MREF(mref));
+	if (IS_ERR(tmp_ino)) {
+		if (!IS_ERR(tmp_ino))
+			iput(tmp_ino);
+		ntfs_error(vol->sb, "Failed to load $Quota.");
+		return false;
+	}
+	vol->quota_ino = tmp_ino;
+	/* Get the $Q index allocation attribute. */
+	tmp_ino = ntfs_index_iget(vol->quota_ino, Q, 2);
+	if (IS_ERR(tmp_ino)) {
+		ntfs_error(vol->sb, "Failed to load $Quota/$Q index.");
+		return false;
+	}
+	vol->quota_q_ino = tmp_ino;
+	ntfs_debug("Done.");
+	return true;
+}
+
+/*
+ * load_and_init_attrdef - load the attribute definitions table for a volume
+ * @vol:	ntfs super block describing device whose attrdef to load
+ *
+ * Return 'true' on success or 'false' on error.
+ */
+static bool load_and_init_attrdef(struct ntfs_volume *vol)
+{
+	loff_t i_size;
+	struct super_block *sb = vol->sb;
+	struct inode *ino;
+	struct folio *folio;
+	u8 *addr;
+	pgoff_t index, max_index;
+	unsigned int size;
+
+	ntfs_debug("Entering.");
+	/* Read attrdef table and setup vol->attrdef and vol->attrdef_size. */
+	ino = ntfs_iget(sb, FILE_AttrDef);
+	if (IS_ERR(ino)) {
+		if (!IS_ERR(ino))
+			iput(ino);
+		goto failed;
+	}
+	NInoSetSparseDisabled(NTFS_I(ino));
+	/* The size of FILE_AttrDef must be above 0 and fit inside 31 bits. */
+	i_size = i_size_read(ino);
+	if (i_size <= 0 || i_size > 0x7fffffff)
+		goto iput_failed;
+	vol->attrdef = kvzalloc(i_size, GFP_NOFS);
+	if (!vol->attrdef)
+		goto iput_failed;
+	index = 0;
+	max_index = i_size >> PAGE_SHIFT;
+	size = PAGE_SIZE;
+	while (index < max_index) {
+		/* Read the attrdef table and copy it into the linear buffer. */
+read_partial_attrdef_page:
+		folio = read_mapping_folio(ino->i_mapping, index, NULL);
+		if (IS_ERR(folio))
+			goto free_iput_failed;
+		addr = kmap_local_folio(folio, 0);
+		memcpy((u8 *)vol->attrdef + (index++ << PAGE_SHIFT),
+				addr, size);
+		kunmap_local(addr);
+		folio_put(folio);
+	}
+	if (size == PAGE_SIZE) {
+		size = i_size & ~PAGE_MASK;
+		if (size)
+			goto read_partial_attrdef_page;
+	}
+	vol->attrdef_size = i_size;
+	ntfs_debug("Read %llu bytes from $AttrDef.", i_size);
+	iput(ino);
+	return true;
+free_iput_failed:
+	kvfree(vol->attrdef);
+	vol->attrdef = NULL;
+iput_failed:
+	iput(ino);
+failed:
+	ntfs_error(sb, "Failed to initialize attribute definition table.");
+	return false;
+}
+
+/*
+ * load_and_init_upcase - load the upcase table for an ntfs volume
+ * @vol:	ntfs super block describing device whose upcase to load
+ *
+ * Return 'true' on success or 'false' on error.
+ */
+static bool load_and_init_upcase(struct ntfs_volume *vol)
+{
+	loff_t i_size;
+	struct super_block *sb = vol->sb;
+	struct inode *ino;
+	struct folio *folio;
+	u8 *addr;
+	pgoff_t index, max_index;
+	unsigned int size;
+	int i, max;
+
+	ntfs_debug("Entering.");
+	/* Read upcase table and setup vol->upcase and vol->upcase_len. */
+	ino = ntfs_iget(sb, FILE_UpCase);
+	if (IS_ERR(ino)) {
+		if (!IS_ERR(ino))
+			iput(ino);
+		goto upcase_failed;
+	}
+	/*
+	 * The upcase size must not be above 64k Unicode characters, must not
+	 * be zero and must be a multiple of sizeof(__le16).
+	 */
+	i_size = i_size_read(ino);
+	if (!i_size || i_size & (sizeof(__le16) - 1) ||
+			i_size > 64ULL * 1024 * sizeof(__le16))
+		goto iput_upcase_failed;
+	vol->upcase = kvzalloc(i_size, GFP_NOFS);
+	if (!vol->upcase)
+		goto iput_upcase_failed;
+	index = 0;
+	max_index = i_size >> PAGE_SHIFT;
+	size = PAGE_SIZE;
+	while (index < max_index) {
+		/* Read the upcase table and copy it into the linear buffer. */
+read_partial_upcase_page:
+		folio = read_mapping_folio(ino->i_mapping, index, NULL);
+		if (IS_ERR(folio))
+			goto iput_upcase_failed;
+		addr = kmap_local_folio(folio, 0);
+		memcpy((char *)vol->upcase + (index++ << PAGE_SHIFT),
+				addr, size);
+		kunmap_local(addr);
+		folio_put(folio);
+	}
+	if (size == PAGE_SIZE) {
+		size = i_size & ~PAGE_MASK;
+		if (size)
+			goto read_partial_upcase_page;
+	}
+	vol->upcase_len = i_size >> sizeof(unsigned char);
+	ntfs_debug("Read %llu bytes from $UpCase (expected %zu bytes).",
+			i_size, 64 * 1024 * sizeof(__le16));
+	iput(ino);
+	mutex_lock(&ntfs_lock);
+	if (!default_upcase) {
+		ntfs_debug("Using volume specified $UpCase since default is not present.");
+		mutex_unlock(&ntfs_lock);
+		return true;
+	}
+	max = default_upcase_len;
+	if (max > vol->upcase_len)
+		max = vol->upcase_len;
+	for (i = 0; i < max; i++)
+		if (vol->upcase[i] != default_upcase[i])
+			break;
+	if (i == max) {
+		kvfree(vol->upcase);
+		vol->upcase = default_upcase;
+		vol->upcase_len = max;
+		ntfs_nr_upcase_users++;
+		mutex_unlock(&ntfs_lock);
+		ntfs_debug("Volume specified $UpCase matches default. Using default.");
+		return true;
+	}
+	mutex_unlock(&ntfs_lock);
+	ntfs_debug("Using volume specified $UpCase since it does not match the default.");
+	return true;
+iput_upcase_failed:
+	iput(ino);
+	kvfree(vol->upcase);
+	vol->upcase = NULL;
+upcase_failed:
+	mutex_lock(&ntfs_lock);
+	if (default_upcase) {
+		vol->upcase = default_upcase;
+		vol->upcase_len = default_upcase_len;
+		ntfs_nr_upcase_users++;
+		mutex_unlock(&ntfs_lock);
+		ntfs_error(sb, "Failed to load $UpCase from the volume. Using default.");
+		return true;
+	}
+	mutex_unlock(&ntfs_lock);
+	ntfs_error(sb, "Failed to initialize upcase table.");
+	return false;
+}
+
+/*
+ * The lcn and mft bitmap inodes are NTFS-internal inodes with
+ * their own special locking rules:
+ */
+static struct lock_class_key
+	lcnbmp_runlist_lock_key, lcnbmp_mrec_lock_key,
+	mftbmp_runlist_lock_key, mftbmp_mrec_lock_key;
+
+/*
+ * load_system_files - open the system files using normal functions
+ * @vol:	ntfs super block describing device whose system files to load
+ *
+ * Open the system files with normal access functions and complete setting up
+ * the ntfs super block @vol.
+ *
+ * Return 'true' on success or 'false' on error.
+ */
+static bool load_system_files(struct ntfs_volume *vol)
+{
+	struct super_block *sb = vol->sb;
+	struct mft_record *m;
+	struct volume_information *vi;
+	struct ntfs_attr_search_ctx *ctx;
+	struct restart_page_header *rp;
+	int err;
+
+	ntfs_debug("Entering.");
+	/* Get mft mirror inode compare the contents of $MFT and $MFTMirr. */
+	if (!load_and_init_mft_mirror(vol) || !check_mft_mirror(vol)) {
+		/* If a read-write mount, convert it to a read-only mount. */
+		if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+			static const char *es1 = "Failed to load $MFTMirr";
+			static const char *es2 = "$MFTMirr does not match $MFT";
+			static const char *es3 = ".  Run ntfsck and/or chkdsk.";
+
+			sb->s_flags |= SB_RDONLY;
+			ntfs_error(sb, "%s.  Mounting read-only%s",
+					!vol->mftmirr_ino ? es1 : es2, es3);
+		}
+		NVolSetErrors(vol);
+	}
+	/* Get mft bitmap attribute inode. */
+	vol->mftbmp_ino = ntfs_attr_iget(vol->mft_ino, AT_BITMAP, NULL, 0);
+	if (IS_ERR(vol->mftbmp_ino)) {
+		ntfs_error(sb, "Failed to load $MFT/$BITMAP attribute.");
+		goto iput_mirr_err_out;
+	}
+	lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->runlist.lock,
+			   &mftbmp_runlist_lock_key);
+	lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->mrec_lock,
+			   &mftbmp_mrec_lock_key);
+	/* Read upcase table and setup @vol->upcase and @vol->upcase_len. */
+	if (!load_and_init_upcase(vol))
+		goto iput_mftbmp_err_out;
+	/*
+	 * Read attribute definitions table and setup @vol->attrdef and
+	 * @vol->attrdef_size.
+	 */
+	if (!load_and_init_attrdef(vol))
+		goto iput_upcase_err_out;
+	/*
+	 * Get the cluster allocation bitmap inode and verify the size, no
+	 * need for any locking at this stage as we are already running
+	 * exclusively as we are mount in progress task.
+	 */
+	vol->lcnbmp_ino = ntfs_iget(sb, FILE_Bitmap);
+	if (IS_ERR(vol->lcnbmp_ino)) {
+		if (!IS_ERR(vol->lcnbmp_ino))
+			iput(vol->lcnbmp_ino);
+		goto bitmap_failed;
+	}
+	lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->runlist.lock,
+			   &lcnbmp_runlist_lock_key);
+	lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->mrec_lock,
+			   &lcnbmp_mrec_lock_key);
+
+	NInoSetSparseDisabled(NTFS_I(vol->lcnbmp_ino));
+	if ((vol->nr_clusters + 7) >> 3 > i_size_read(vol->lcnbmp_ino)) {
+		iput(vol->lcnbmp_ino);
+bitmap_failed:
+		ntfs_error(sb, "Failed to load $Bitmap.");
+		goto iput_attrdef_err_out;
+	}
+	/*
+	 * Get the volume inode and setup our cache of the volume flags and
+	 * version.
+	 */
+	vol->vol_ino = ntfs_iget(sb, FILE_Volume);
+	if (IS_ERR(vol->vol_ino)) {
+		if (!IS_ERR(vol->vol_ino))
+			iput(vol->vol_ino);
+volume_failed:
+		ntfs_error(sb, "Failed to load $Volume.");
+		goto iput_lcnbmp_err_out;
+	}
+	m = map_mft_record(NTFS_I(vol->vol_ino));
+	if (IS_ERR(m)) {
+iput_volume_failed:
+		iput(vol->vol_ino);
+		goto volume_failed;
+	}
+
+	ctx = ntfs_attr_get_search_ctx(NTFS_I(vol->vol_ino), m);
+	if (!ctx) {
+		ntfs_error(sb, "Failed to get attribute search context.");
+		goto get_ctx_vol_failed;
+	}
+
+	if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, ctx) &&
+	    !ctx->attr->non_resident &&
+	    !(ctx->attr->flags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) &&
+	    le32_to_cpu(ctx->attr->data.resident.value_length) > 0) {
+		err = ntfs_ucstonls(vol, (__le16 *)((u8 *)ctx->attr +
+				    le16_to_cpu(ctx->attr->data.resident.value_offset)),
+				    le32_to_cpu(ctx->attr->data.resident.value_length) / 2,
+				    &vol->volume_label, NTFS_MAX_LABEL_LEN);
+		if (err < 0)
+			vol->volume_label = NULL;
+	}
+
+	if (ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0,
+			ctx) || ctx->attr->non_resident || ctx->attr->flags) {
+err_put_vol:
+		ntfs_attr_put_search_ctx(ctx);
+get_ctx_vol_failed:
+		unmap_mft_record(NTFS_I(vol->vol_ino));
+		goto iput_volume_failed;
+	}
+	vi = (struct volume_information *)((char *)ctx->attr +
+			le16_to_cpu(ctx->attr->data.resident.value_offset));
+	/* Some bounds checks. */
+	if ((u8 *)vi < (u8 *)ctx->attr || (u8 *)vi +
+			le32_to_cpu(ctx->attr->data.resident.value_length) >
+			(u8 *)ctx->attr + le32_to_cpu(ctx->attr->length))
+		goto err_put_vol;
+	/* Copy the volume flags and version to the struct ntfs_volume structure. */
+	vol->vol_flags = vi->flags;
+	vol->major_ver = vi->major_ver;
+	vol->minor_ver = vi->minor_ver;
+	ntfs_attr_put_search_ctx(ctx);
+	unmap_mft_record(NTFS_I(vol->vol_ino));
+	pr_info("volume version %i.%i, dev %s, cluster size %d\n",
+		vol->major_ver, vol->minor_ver, sb->s_id, vol->cluster_size);
+
+	/* Make sure that no unsupported volume flags are set. */
+	if (vol->vol_flags & VOLUME_MUST_MOUNT_RO_MASK) {
+		static const char *es1a = "Volume is dirty";
+		static const char *es1b = "Volume has been modified by chkdsk";
+		static const char *es1c = "Volume has unsupported flags set";
+		static const char *es2a = ".  Run chkdsk and mount in Windows.";
+		static const char *es2b = ".  Mount in Windows.";
+		const char *es1, *es2;
+
+		es2 = es2a;
+		if (vol->vol_flags & VOLUME_IS_DIRTY)
+			es1 = es1a;
+		else if (vol->vol_flags & VOLUME_MODIFIED_BY_CHKDSK) {
+			es1 = es1b;
+			es2 = es2b;
+		} else {
+			es1 = es1c;
+			ntfs_warning(sb, "Unsupported volume flags 0x%x encountered.",
+					(unsigned int)le16_to_cpu(vol->vol_flags));
+		}
+		/* If a read-write mount, convert it to a read-only mount. */
+		if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+			sb->s_flags |= SB_RDONLY;
+			ntfs_error(sb, "%s.  Mounting read-only%s", es1, es2);
+		}
+		/*
+		 * Do not set NVolErrors() because ntfs_remount() re-checks the
+		 * flags which we need to do in case any flags have changed.
+		 */
+	}
+	/*
+	 * Get the inode for the logfile, check it and determine if the volume
+	 * was shutdown cleanly.
+	 */
+	rp = NULL;
+	err = load_and_check_logfile(vol, &rp);
+	if (err) {
+		/* If a read-write mount, convert it to a read-only mount. */
+		if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+			sb->s_flags |= SB_RDONLY;
+			ntfs_error(sb, "Failed to load LogFile. Mounting read-only.");
+		}
+		NVolSetErrors(vol);
+	}
+
+	kvfree(rp);
+	/* Get the root directory inode so we can do path lookups. */
+	vol->root_ino = ntfs_iget(sb, FILE_root);
+	if (IS_ERR(vol->root_ino)) {
+		if (!IS_ERR(vol->root_ino))
+			iput(vol->root_ino);
+		ntfs_error(sb, "Failed to load root directory.");
+		goto iput_logfile_err_out;
+	}
+	/*
+	 * Check if Windows is suspended to disk on the target volume.  If it
+	 * is hibernated, we must not write *anything* to the disk so set
+	 * NVolErrors() without setting the dirty volume flag and mount
+	 * read-only.  This will prevent read-write remounting and it will also
+	 * prevent all writes.
+	 */
+	err = check_windows_hibernation_status(vol);
+	if (unlikely(err)) {
+		static const char *es1a = "Failed to determine if Windows is hibernated";
+		static const char *es1b = "Windows is hibernated";
+		static const char *es2 = ".  Run chkdsk.";
+		const char *es1;
+
+		es1 = err < 0 ? es1a : es1b;
+		/* If a read-write mount, convert it to a read-only mount. */
+		if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+			sb->s_flags |= SB_RDONLY;
+			ntfs_error(sb, "%s.  Mounting read-only%s", es1, es2);
+		}
+		NVolSetErrors(vol);
+	}
+
+	/* If (still) a read-write mount, empty the logfile. */
+	if (!sb_rdonly(sb) &&
+	    vol->logfile_ino && !ntfs_empty_logfile(vol->logfile_ino) &&
+	    vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+		static const char *es1 = "Failed to empty LogFile";
+		static const char *es2 = ".  Mount in Windows.";
+
+		/* Convert to a read-only mount. */
+		ntfs_error(sb, "%s.  Mounting read-only%s", es1, es2);
+		sb->s_flags |= SB_RDONLY;
+		NVolSetErrors(vol);
+	}
+	/* If on NTFS versions before 3.0, we are done. */
+	if (unlikely(vol->major_ver < 3))
+		return true;
+	/* NTFS 3.0+ specific initialization. */
+	/* Get the security descriptors inode. */
+	vol->secure_ino = ntfs_iget(sb, FILE_Secure);
+	if (IS_ERR(vol->secure_ino)) {
+		if (!IS_ERR(vol->secure_ino))
+			iput(vol->secure_ino);
+		ntfs_error(sb, "Failed to load $Secure.");
+		goto iput_root_err_out;
+	}
+	/* Get the extended system files' directory inode. */
+	vol->extend_ino = ntfs_iget(sb, FILE_Extend);
+	if (IS_ERR(vol->extend_ino) ||
+	    !S_ISDIR(vol->extend_ino->i_mode)) {
+		if (!IS_ERR(vol->extend_ino))
+			iput(vol->extend_ino);
+		ntfs_error(sb, "Failed to load $Extend.");
+		goto iput_sec_err_out;
+	}
+	/* Find the quota file, load it if present, and set it up. */
+	if (!load_and_init_quota(vol) &&
+	    vol->on_errors == ON_ERRORS_REMOUNT_RO) {
+		static const char *es1 = "Failed to load $Quota";
+		static const char *es2 = ".  Run chkdsk.";
+
+		sb->s_flags |= SB_RDONLY;
+		ntfs_error(sb, "%s.  Mounting read-only%s", es1, es2);
+		/* This will prevent a read-write remount. */
+		NVolSetErrors(vol);
+	}
+
+	return true;
+
+iput_sec_err_out:
+	iput(vol->secure_ino);
+iput_root_err_out:
+	iput(vol->root_ino);
+iput_logfile_err_out:
+	if (vol->logfile_ino)
+		iput(vol->logfile_ino);
+	iput(vol->vol_ino);
+iput_lcnbmp_err_out:
+	iput(vol->lcnbmp_ino);
+iput_attrdef_err_out:
+	vol->attrdef_size = 0;
+	if (vol->attrdef) {
+		kvfree(vol->attrdef);
+		vol->attrdef = NULL;
+	}
+iput_upcase_err_out:
+	vol->upcase_len = 0;
+	mutex_lock(&ntfs_lock);
+	if (vol->upcase == default_upcase) {
+		ntfs_nr_upcase_users--;
+		vol->upcase = NULL;
+	}
+	mutex_unlock(&ntfs_lock);
+	if (vol->upcase) {
+		kvfree(vol->upcase);
+		vol->upcase = NULL;
+	}
+iput_mftbmp_err_out:
+	iput(vol->mftbmp_ino);
+iput_mirr_err_out:
+	iput(vol->mftmirr_ino);
+	return false;
+}
+
+static void ntfs_volume_free(struct ntfs_volume *vol)
+{
+	/* Throw away the table of attribute definitions. */
+	vol->attrdef_size = 0;
+	if (vol->attrdef) {
+		kvfree(vol->attrdef);
+		vol->attrdef = NULL;
+	}
+	vol->upcase_len = 0;
+	/*
+	 * Destroy the global default upcase table if necessary.  Also decrease
+	 * the number of upcase users if we are a user.
+	 */
+	mutex_lock(&ntfs_lock);
+	if (vol->upcase == default_upcase) {
+		ntfs_nr_upcase_users--;
+		vol->upcase = NULL;
+	}
+
+	if (!ntfs_nr_upcase_users && default_upcase) {
+		kvfree(default_upcase);
+		default_upcase = NULL;
+	}
+
+	free_compression_buffers();
+
+	mutex_unlock(&ntfs_lock);
+	if (vol->upcase) {
+		kvfree(vol->upcase);
+		vol->upcase = NULL;
+	}
+
+	unload_nls(vol->nls_map);
+
+	if (vol->lcn_empty_bits_per_page)
+		kvfree(vol->lcn_empty_bits_per_page);
+	kfree(vol->volume_label);
+	kfree(vol);
+}
+
+/*
+ * ntfs_put_super - called by the vfs to unmount a volume
+ * @sb:		vfs superblock of volume to unmount
+ */
+static void ntfs_put_super(struct super_block *sb)
+{
+	struct ntfs_volume *vol = NTFS_SB(sb);
+
+	pr_info("Entering %s, dev %s\n", __func__, sb->s_id);
+
+	cancel_work_sync(&vol->precalc_work);
+
+	/*
+	 * Commit all inodes while they are still open in case some of them
+	 * cause others to be dirtied.
+	 */
+	ntfs_commit_inode(vol->vol_ino);
+
+	/* NTFS 3.0+ specific. */
+	if (vol->major_ver >= 3) {
+		if (vol->quota_q_ino)
+			ntfs_commit_inode(vol->quota_q_ino);
+		if (vol->quota_ino)
+			ntfs_commit_inode(vol->quota_ino);
+		if (vol->extend_ino)
+			ntfs_commit_inode(vol->extend_ino);
+		if (vol->secure_ino)
+			ntfs_commit_inode(vol->secure_ino);
+	}
+
+	ntfs_commit_inode(vol->root_ino);
+
+	ntfs_commit_inode(vol->lcnbmp_ino);
+
+	/*
+	 * the GFP_NOFS scope is not needed because ntfs_commit_inode
+	 * does nothing
+	 */
+	ntfs_commit_inode(vol->mftbmp_ino);
+
+	if (vol->logfile_ino)
+		ntfs_commit_inode(vol->logfile_ino);
+
+	if (vol->mftmirr_ino)
+		ntfs_commit_inode(vol->mftmirr_ino);
+	ntfs_commit_inode(vol->mft_ino);
+
+	/*
+	 * If a read-write mount and no volume errors have occurred, mark the
+	 * volume clean.  Also, re-commit all affected inodes.
+	 */
+	if (!sb_rdonly(sb)) {
+		if (!NVolErrors(vol)) {
+			if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY))
+				ntfs_warning(sb,
+					"Failed to clear dirty bit in volume information flags.  Run chkdsk.");
+			ntfs_commit_inode(vol->vol_ino);
+			ntfs_commit_inode(vol->root_ino);
+			if (vol->mftmirr_ino)
+				ntfs_commit_inode(vol->mftmirr_ino);
+			ntfs_commit_inode(vol->mft_ino);
+		} else {
+			ntfs_warning(sb,
+				"Volume has errors.  Leaving volume marked dirty.  Run chkdsk.");
+		}
+	}
+
+	iput(vol->vol_ino);
+	vol->vol_ino = NULL;
+
+	/* NTFS 3.0+ specific clean up. */
+	if (vol->major_ver >= 3) {
+		if (vol->quota_q_ino) {
+			iput(vol->quota_q_ino);
+			vol->quota_q_ino = NULL;
+		}
+		if (vol->quota_ino) {
+			iput(vol->quota_ino);
+			vol->quota_ino = NULL;
+		}
+		if (vol->extend_ino) {
+			iput(vol->extend_ino);
+			vol->extend_ino = NULL;
+		}
+		if (vol->secure_ino) {
+			iput(vol->secure_ino);
+			vol->secure_ino = NULL;
+		}
+	}
+
+	iput(vol->root_ino);
+	vol->root_ino = NULL;
+
+	iput(vol->lcnbmp_ino);
+	vol->lcnbmp_ino = NULL;
+
+	iput(vol->mftbmp_ino);
+	vol->mftbmp_ino = NULL;
+
+	if (vol->logfile_ino) {
+		iput(vol->logfile_ino);
+		vol->logfile_ino = NULL;
+	}
+	if (vol->mftmirr_ino) {
+		/* Re-commit the mft mirror and mft just in case. */
+		ntfs_commit_inode(vol->mftmirr_ino);
+		ntfs_commit_inode(vol->mft_ino);
+		iput(vol->mftmirr_ino);
+		vol->mftmirr_ino = NULL;
+	}
+	/*
+	 * We should have no dirty inodes left, due to
+	 * mft.c::ntfs_mft_writepage() cleaning all the dirty pages as
+	 * the underlying mft records are written out and cleaned.
+	 */
+	ntfs_commit_inode(vol->mft_ino);
+	write_inode_now(vol->mft_ino, 1);
+
+	iput(vol->mft_ino);
+	vol->mft_ino = NULL;
+	blkdev_issue_flush(sb->s_bdev);
+
+	ntfs_volume_free(vol);
+}
+
+int ntfs_force_shutdown(struct super_block *sb, u32 flags)
+{
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int ret;
+
+	if (NVolShutdown(vol))
+		return 0;
+
+	switch (flags) {
+	case FS_SHUTDOWN_FLAGS_DEFAULT:
+	case FS_SHUTDOWN_FLAGS_LOGFLUSH:
+		ret = bdev_freeze(sb->s_bdev);
+		if (ret)
+			return ret;
+		bdev_thaw(sb->s_bdev);
+		NVolSetShutdown(vol);
+		break;
+	case FS_SHUTDOWN_FLAGS_NOLOGFLUSH:
+		NVolSetShutdown(vol);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void ntfs_shutdown(struct super_block *sb)
+{
+	ntfs_force_shutdown(sb, FS_SHUTDOWN_FLAGS_NOLOGFLUSH);
+
+}
+
+static int ntfs_sync_fs(struct super_block *sb, int wait)
+{
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int err = 0;
+
+	if (NVolShutdown(vol))
+		return -EIO;
+
+	if (!wait)
+		return 0;
+
+	/* If there are some dirty buffers in the bdev inode */
+	if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) {
+		ntfs_warning(sb, "Failed to clear dirty bit in volume information flags.  Run chkdsk.");
+		err = -EIO;
+	}
+	sync_inodes_sb(sb);
+	sync_blockdev(sb->s_bdev);
+	blkdev_issue_flush(sb->s_bdev);
+	return err;
+}
+
+/*
+ * get_nr_free_clusters - return the number of free clusters on a volume
+ * @vol:	ntfs volume for which to obtain free cluster count
+ *
+ * Calculate the number of free clusters on the mounted NTFS volume @vol. We
+ * actually calculate the number of clusters in use instead because this
+ * allows us to not care about partial pages as these will be just zero filled
+ * and hence not be counted as allocated clusters.
+ *
+ * The only particularity is that clusters beyond the end of the logical ntfs
+ * volume will be marked as allocated to prevent errors which means we have to
+ * discount those at the end. This is important as the cluster bitmap always
+ * has a size in multiples of 8 bytes, i.e. up to 63 clusters could be outside
+ * the logical volume and marked in use when they are not as they do not exist.
+ *
+ * If any pages cannot be read we assume all clusters in the erroring pages are
+ * in use. This means we return an underestimate on errors which is better than
+ * an overestimate.
+ */
+s64 get_nr_free_clusters(struct ntfs_volume *vol)
+{
+	s64 nr_free = vol->nr_clusters;
+	u32 nr_used;
+	struct address_space *mapping = vol->lcnbmp_ino->i_mapping;
+	struct folio *folio;
+	pgoff_t index, max_index;
+	struct file_ra_state *ra;
+
+	ntfs_debug("Entering.");
+	/* Serialize accesses to the cluster bitmap. */
+
+	if (NVolFreeClusterKnown(vol))
+		return atomic64_read(&vol->free_clusters);
+
+	ra = kzalloc(sizeof(*ra), GFP_NOFS);
+	if (!ra)
+		return 0;
+
+	file_ra_state_init(ra, mapping);
+
+	/*
+	 * Convert the number of bits into bytes rounded up, then convert into
+	 * multiples of PAGE_SIZE, rounding up so that if we have one
+	 * full and one partial page max_index = 2.
+	 */
+	max_index = (((vol->nr_clusters + 7) >> 3) + PAGE_SIZE - 1) >>
+			PAGE_SHIFT;
+	/* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */
+	ntfs_debug("Reading $Bitmap, max_index = 0x%lx, max_size = 0x%lx.",
+			max_index, PAGE_SIZE / 4);
+	for (index = 0; index < max_index; index++) {
+		unsigned long *kaddr;
+
+		/*
+		 * Get folio from page cache, getting it from backing store
+		 * if necessary, and increment the use count.
+		 */
+		folio = ntfs_get_locked_folio(mapping, index, max_index, ra);
+
+		/* Ignore pages which errored synchronously. */
+		if (IS_ERR(folio)) {
+			ntfs_debug("Skipping page (index 0x%lx).", index);
+			nr_free -= PAGE_SIZE * 8;
+			vol->lcn_empty_bits_per_page[index] = 0;
+			continue;
+		}
+
+		kaddr = kmap_local_folio(folio, 0);
+		/*
+		 * Subtract the number of set bits. If this
+		 * is the last page and it is partial we don't really care as
+		 * it just means we do a little extra work but it won't affect
+		 * the result as all out of range bytes are set to zero by
+		 * ntfs_readpage().
+		 */
+		nr_used = bitmap_weight(kaddr, PAGE_SIZE * BITS_PER_BYTE);
+		nr_free -= nr_used;
+		vol->lcn_empty_bits_per_page[index] = PAGE_SIZE * BITS_PER_BYTE - nr_used;
+		kunmap_local(kaddr);
+		folio_unlock(folio);
+		folio_put(folio);
+	}
+	ntfs_debug("Finished reading $Bitmap, last index = 0x%lx.", index - 1);
+	/*
+	 * Fixup for eventual bits outside logical ntfs volume (see function
+	 * description above).
+	 */
+	if (vol->nr_clusters & 63)
+		nr_free += 64 - (vol->nr_clusters & 63);
+
+	/* If errors occurred we may well have gone below zero, fix this. */
+	if (nr_free < 0)
+		nr_free = 0;
+	else
+		atomic64_set(&vol->free_clusters, nr_free);
+
+	kfree(ra);
+	NVolSetFreeClusterKnown(vol);
+	wake_up_all(&vol->free_waitq);
+	ntfs_debug("Exiting.");
+	return nr_free;
+}
+
+/*
+ * @nr_clusters is the number of clusters requested for allocation.
+ *
+ * Return the number of clusters available for allocation within
+ * the range of @nr_clusters, which is counts that considered
+ * for delayed allocation.
+ */
+s64 ntfs_available_clusters_count(struct ntfs_volume *vol, s64 nr_clusters)
+{
+	s64 free_clusters;
+
+	/* wait event */
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	free_clusters = atomic64_read(&vol->free_clusters) -
+		atomic64_read(&vol->dirty_clusters);
+	if (free_clusters <= 0)
+		return -ENOSPC;
+	else if (free_clusters < nr_clusters)
+		nr_clusters = free_clusters;
+
+	return nr_clusters;
+}
+
+/*
+ * __get_nr_free_mft_records - return the number of free inodes on a volume
+ * @vol:	ntfs volume for which to obtain free inode count
+ * @nr_free:	number of mft records in filesystem
+ * @max_index:	maximum number of pages containing set bits
+ *
+ * Calculate the number of free mft records (inodes) on the mounted NTFS
+ * volume @vol. We actually calculate the number of mft records in use instead
+ * because this allows us to not care about partial pages as these will be just
+ * zero filled and hence not be counted as allocated mft record.
+ *
+ * If any pages cannot be read we assume all mft records in the erroring pages
+ * are in use. This means we return an underestimate on errors which is better
+ * than an overestimate.
+ *
+ * NOTE: Caller must hold mftbmp_lock rw_semaphore for reading or writing.
+ */
+static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol,
+		s64 nr_free, const pgoff_t max_index)
+{
+	struct address_space *mapping = vol->mftbmp_ino->i_mapping;
+	struct folio *folio;
+	pgoff_t index;
+	struct file_ra_state *ra;
+
+	ntfs_debug("Entering.");
+
+	ra = kzalloc(sizeof(*ra), GFP_NOFS);
+	if (!ra)
+		return 0;
+
+	file_ra_state_init(ra, mapping);
+
+	/* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */
+	ntfs_debug("Reading $MFT/$BITMAP, max_index = 0x%lx, max_size = 0x%lx.",
+			max_index, PAGE_SIZE / 4);
+	for (index = 0; index < max_index; index++) {
+		unsigned long *kaddr;
+
+		/*
+		 * Get folio from page cache, getting it from backing store
+		 * if necessary, and increment the use count.
+		 */
+		folio = ntfs_get_locked_folio(mapping, index, max_index, ra);
+
+		/* Ignore pages which errored synchronously. */
+		if (IS_ERR(folio)) {
+			ntfs_debug("read_mapping_page() error. Skipping page (index 0x%lx).",
+					index);
+			nr_free -= PAGE_SIZE * 8;
+			continue;
+		}
+
+		kaddr = kmap_local_folio(folio, 0);
+		/*
+		 * Subtract the number of set bits. If this
+		 * is the last page and it is partial we don't really care as
+		 * it just means we do a little extra work but it won't affect
+		 * the result as all out of range bytes are set to zero by
+		 * ntfs_readpage().
+		 */
+		nr_free -= bitmap_weight(kaddr,
+					PAGE_SIZE * BITS_PER_BYTE);
+		kunmap_local(kaddr);
+		folio_unlock(folio);
+		folio_put(folio);
+	}
+	ntfs_debug("Finished reading $MFT/$BITMAP, last index = 0x%lx.",
+			index - 1);
+	/* If errors occurred we may well have gone below zero, fix this. */
+	if (nr_free < 0)
+		nr_free = 0;
+	else
+		atomic64_set(&vol->free_mft_records, nr_free);
+
+	kfree(ra);
+	ntfs_debug("Exiting.");
+	return nr_free;
+}
+
+/*
+ * ntfs_statfs - return information about mounted NTFS volume
+ * @dentry:	dentry from mounted volume
+ * @sfs:	statfs structure in which to return the information
+ *
+ * Return information about the mounted NTFS volume @dentry in the statfs structure
+ * pointed to by @sfs (this is initialized with zeros before ntfs_statfs is
+ * called). We interpret the values to be correct of the moment in time at
+ * which we are called. Most values are variable otherwise and this isn't just
+ * the free values but the totals as well. For example we can increase the
+ * total number of file nodes if we run out and we can keep doing this until
+ * there is no more space on the volume left at all.
+ *
+ * Called from vfs_statfs which is used to handle the statfs, fstatfs, and
+ * ustat system calls.
+ *
+ * Return 0 on success or -errno on error.
+ */
+static int ntfs_statfs(struct dentry *dentry, struct kstatfs *sfs)
+{
+	struct super_block *sb = dentry->d_sb;
+	s64 size;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	struct ntfs_inode *mft_ni = NTFS_I(vol->mft_ino);
+	unsigned long flags;
+
+	ntfs_debug("Entering.");
+	/* Type of filesystem. */
+	sfs->f_type   = NTFS_SB_MAGIC;
+	/* Optimal transfer block size. */
+	sfs->f_bsize = vol->cluster_size;
+	/* Fundamental file system block size, used as the unit. */
+	sfs->f_frsize = vol->cluster_size;
+
+	/*
+	 * Total data blocks in filesystem in units of f_bsize and since
+	 * inodes are also stored in data blocs ($MFT is a file) this is just
+	 * the total clusters.
+	 */
+	sfs->f_blocks = vol->nr_clusters;
+
+	/* wait event */
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	/* Free data blocks in filesystem in units of f_bsize. */
+	size = atomic64_read(&vol->free_clusters) -
+		atomic64_read(&vol->dirty_clusters);
+	if (size < 0LL)
+		size = 0LL;
+
+	/* Free blocks avail to non-superuser, same as above on NTFS. */
+	sfs->f_bavail = sfs->f_bfree = size;
+
+	/* Number of inodes in filesystem (at this point in time). */
+	read_lock_irqsave(&mft_ni->size_lock, flags);
+	sfs->f_files = i_size_read(vol->mft_ino) >> vol->mft_record_size_bits;
+	read_unlock_irqrestore(&mft_ni->size_lock, flags);
+
+	/* Free inodes in fs (based on current total count). */
+	sfs->f_ffree = atomic64_read(&vol->free_mft_records);
+
+	/*
+	 * File system id. This is extremely *nix flavour dependent and even
+	 * within Linux itself all fs do their own thing. I interpret this to
+	 * mean a unique id associated with the mounted fs and not the id
+	 * associated with the filesystem driver, the latter is already given
+	 * by the filesystem type in sfs->f_type. Thus we use the 64-bit
+	 * volume serial number splitting it into two 32-bit parts. We enter
+	 * the least significant 32-bits in f_fsid[0] and the most significant
+	 * 32-bits in f_fsid[1].
+	 */
+	sfs->f_fsid = u64_to_fsid(vol->serial_no);
+	/* Maximum length of filenames. */
+	sfs->f_namelen	   = NTFS_MAX_NAME_LEN;
+
+	return 0;
+}
+
+static int ntfs_write_inode(struct inode *vi, struct writeback_control *wbc)
+{
+	return __ntfs_write_inode(vi, wbc->sync_mode == WB_SYNC_ALL);
+}
+
+/*
+ * The complete super operations.
+ */
+static const struct super_operations ntfs_sops = {
+	.alloc_inode	= ntfs_alloc_big_inode,	  /* VFS: Allocate new inode. */
+	.free_inode	= ntfs_free_big_inode, /* VFS: Deallocate inode. */
+	.drop_inode	= ntfs_drop_big_inode,
+	.write_inode	= ntfs_write_inode,	/* VFS: Write dirty inode to disk. */
+	.put_super	= ntfs_put_super,	/* Syscall: umount. */
+	.shutdown	= ntfs_shutdown,
+	.sync_fs	= ntfs_sync_fs,		/* Syscall: sync. */
+	.statfs		= ntfs_statfs,		/* Syscall: statfs */
+	.evict_inode	= ntfs_evict_big_inode,
+	.show_options	= ntfs_show_options,	/* Show mount options in proc. */
+};
+
+static void precalc_free_clusters(struct work_struct *work)
+{
+	struct ntfs_volume *vol = container_of(work, struct ntfs_volume, precalc_work);
+	s64 nr_free;
+
+	nr_free = get_nr_free_clusters(vol);
+
+	ntfs_debug("pre-calculate free clusters(%lld) using workqueue",
+			nr_free);
+}
+
+static struct lock_class_key ntfs_mft_inval_lock_key;
+
+/*
+ * ntfs_fill_super - mount an ntfs filesystem
+ * @sb: super block of the device to mount
+ * @fc: filesystem context containing mount options
+ *
+ * ntfs_fill_super() is called by the VFS to mount the device described by @sb
+ * with the mount otions in @data with the NTFS filesystem.
+ *
+ * If @silent is true, remain silent even if errors are detected. This is used
+ * during bootup, when the kernel tries to mount the root filesystem with all
+ * registered filesystems one after the other until one succeeds. This implies
+ * that all filesystems except the correct one will quite correctly and
+ * expectedly return an error, but nobody wants to see error messages when in
+ * fact this is what is supposed to happen.
+ */
+static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
+{
+	char *boot;
+	struct inode *tmp_ino;
+	int blocksize, result;
+	pgoff_t lcn_bit_pages;
+	struct ntfs_volume *vol = NTFS_SB(sb);
+	int silent = fc->sb_flags & SB_SILENT;
+
+	vol->sb = sb;
+
+	/*
+	 * We do a pretty difficult piece of bootstrap by reading the
+	 * MFT (and other metadata) from disk into memory. We'll only
+	 * release this metadata during umount, so the locking patterns
+	 * observed during bootstrap do not count. So turn off the
+	 * observation of locking patterns (strictly for this context
+	 * only) while mounting NTFS. [The validator is still active
+	 * otherwise, even for this context: it will for example record
+	 * lock class registrations.]
+	 */
+	lockdep_off();
+	ntfs_debug("Entering.");
+
+	if (vol->nls_map && !strcmp(vol->nls_map->charset, "utf8"))
+		vol->nls_utf8 = true;
+	if (NVolDisableSparse(vol))
+		vol->preallocated_size = 0;
+
+	if (NVolDiscard(vol) && !bdev_max_discard_sectors(sb->s_bdev)) {
+		ntfs_warning(
+			sb,
+			"Discard requested but device does not support discard.  Discard disabled.");
+		NVolClearDiscard(vol);
+	}
+
+	/* We support sector sizes up to the PAGE_SIZE. */
+	if (bdev_logical_block_size(sb->s_bdev) > PAGE_SIZE) {
+		if (!silent)
+			ntfs_error(sb,
+				"Device has unsupported sector size (%i).  The maximum supported sector size on this architecture is %lu bytes.",
+				bdev_logical_block_size(sb->s_bdev),
+				PAGE_SIZE);
+		goto err_out_now;
+	}
+
+	/*
+	 * Setup the device access block size to NTFS_BLOCK_SIZE or the hard
+	 * sector size, whichever is bigger.
+	 */
+	blocksize = sb_min_blocksize(sb, NTFS_BLOCK_SIZE);
+	if (blocksize < NTFS_BLOCK_SIZE) {
+		if (!silent)
+			ntfs_error(sb, "Unable to set device block size.");
+		goto err_out_now;
+	}
+
+	ntfs_debug("Set device block size to %i bytes (block size bits %i).",
+			blocksize, sb->s_blocksize_bits);
+	/* Determine the size of the device in units of block_size bytes. */
+	if (!bdev_nr_bytes(sb->s_bdev)) {
+		if (!silent)
+			ntfs_error(sb, "Unable to determine device size.");
+		goto err_out_now;
+	}
+	vol->nr_blocks = bdev_nr_bytes(sb->s_bdev) >>
+			sb->s_blocksize_bits;
+	/* Read the boot sector and return unlocked buffer head to it. */
+	boot = read_ntfs_boot_sector(sb, silent);
+	if (!boot) {
+		if (!silent)
+			ntfs_error(sb, "Not an NTFS volume.");
+		goto err_out_now;
+	}
+	/*
+	 * Extract the data from the boot sector and setup the ntfs volume
+	 * using it.
+	 */
+	result = parse_ntfs_boot_sector(vol, (struct ntfs_boot_sector *)boot);
+	kfree(boot);
+	if (!result) {
+		if (!silent)
+			ntfs_error(sb, "Unsupported NTFS filesystem.");
+		goto err_out_now;
+	}
+
+	if (vol->sector_size > blocksize) {
+		blocksize = sb_set_blocksize(sb, vol->sector_size);
+		if (blocksize != vol->sector_size) {
+			if (!silent)
+				ntfs_error(sb,
+					   "Unable to set device block size to sector size (%i).",
+					   vol->sector_size);
+			goto err_out_now;
+		}
+		vol->nr_blocks = bdev_nr_bytes(sb->s_bdev) >>
+				sb->s_blocksize_bits;
+		ntfs_debug("Changed device block size to %i bytes (block size bits %i) to match volume sector size.",
+				blocksize, sb->s_blocksize_bits);
+	}
+	/* Initialize the cluster and mft allocators. */
+	ntfs_setup_allocators(vol);
+	/* Setup remaining fields in the super block. */
+	sb->s_magic = NTFS_SB_MAGIC;
+	/*
+	 * Ntfs allows 63 bits for the file size, i.e. correct would be:
+	 *	sb->s_maxbytes = ~0ULL >> 1;
+	 * But the kernel uses a long as the page cache page index which on
+	 * 32-bit architectures is only 32-bits. MAX_LFS_FILESIZE is kernel
+	 * defined to the maximum the page cache page index can cope with
+	 * without overflowing the index or to 2^63 - 1, whichever is smaller.
+	 */
+	sb->s_maxbytes = MAX_LFS_FILESIZE;
+	/* Ntfs measures time in 100ns intervals. */
+	sb->s_time_gran = 100;
+
+	sb->s_xattr = ntfs_xattr_handlers;
+	/*
+	 * Now load the metadata required for the page cache and our address
+	 * space operations to function. We do this by setting up a specialised
+	 * read_inode method and then just calling the normal iget() to obtain
+	 * the inode for $MFT which is sufficient to allow our normal inode
+	 * operations and associated address space operations to function.
+	 */
+	sb->s_op = &ntfs_sops;
+	tmp_ino = new_inode(sb);
+	if (!tmp_ino) {
+		if (!silent)
+			ntfs_error(sb, "Failed to load essential metadata.");
+		goto err_out_now;
+	}
+
+	tmp_ino->i_ino = FILE_MFT;
+	insert_inode_hash(tmp_ino);
+	if (ntfs_read_inode_mount(tmp_ino) < 0) {
+		if (!silent)
+			ntfs_error(sb, "Failed to load essential metadata.");
+		goto iput_tmp_ino_err_out_now;
+	}
+	lockdep_set_class(&tmp_ino->i_mapping->invalidate_lock,
+			  &ntfs_mft_inval_lock_key);
+
+	mutex_lock(&ntfs_lock);
+
+	/*
+	 * Generate the global default upcase table if necessary.  Also
+	 * temporarily increment the number of upcase users to avoid race
+	 * conditions with concurrent (u)mounts.
+	 */
+	if (!default_upcase)
+		default_upcase = generate_default_upcase();
+	ntfs_nr_upcase_users++;
+	mutex_unlock(&ntfs_lock);
+
+	lcn_bit_pages = (((vol->nr_clusters + 7) >> 3) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	vol->lcn_empty_bits_per_page = kvmalloc_array(lcn_bit_pages, sizeof(unsigned int),
+						      GFP_KERNEL);
+	if (!vol->lcn_empty_bits_per_page) {
+		ntfs_error(sb,
+			   "Unable to allocate pages for storing LCN empty bit counts\n");
+		goto unl_upcase_iput_tmp_ino_err_out_now;
+	}
+
+	/*
+	 * From now on, ignore @silent parameter. If we fail below this line,
+	 * it will be due to a corrupt fs or a system error, so we report it.
+	 */
+	/*
+	 * Open the system files with normal access functions and complete
+	 * setting up the ntfs super block.
+	 */
+	if (!load_system_files(vol)) {
+		ntfs_error(sb, "Failed to load system files.");
+		goto unl_upcase_iput_tmp_ino_err_out_now;
+	}
+
+	/* We grab a reference, simulating an ntfs_iget(). */
+	ihold(vol->root_ino);
+	sb->s_root = d_make_root(vol->root_ino);
+	if (sb->s_root) {
+		s64 nr_records;
+
+		ntfs_debug("Exiting, status successful.");
+
+		/* Release the default upcase if it has no users. */
+		mutex_lock(&ntfs_lock);
+		if (!--ntfs_nr_upcase_users && default_upcase) {
+			kvfree(default_upcase);
+			default_upcase = NULL;
+		}
+		mutex_unlock(&ntfs_lock);
+		sb->s_export_op = &ntfs_export_ops;
+		lockdep_on();
+
+		nr_records = __get_nr_free_mft_records(vol,
+				i_size_read(vol->mft_ino) >> vol->mft_record_size_bits,
+				((((NTFS_I(vol->mft_ino)->initialized_size >>
+				    vol->mft_record_size_bits) +
+				   7) >> 3) + PAGE_SIZE - 1) >> PAGE_SHIFT);
+		ntfs_debug("Free mft records(%lld)", nr_records);
+
+		init_waitqueue_head(&vol->free_waitq);
+		INIT_WORK(&vol->precalc_work, precalc_free_clusters);
+		queue_work(ntfs_wq, &vol->precalc_work);
+		return 0;
+	}
+	ntfs_error(sb, "Failed to allocate root directory.");
+	/* Clean up after the successful load_system_files() call from above. */
+	iput(vol->vol_ino);
+	vol->vol_ino = NULL;
+	/* NTFS 3.0+ specific clean up. */
+	if (vol->major_ver >= 3) {
+		if (vol->quota_q_ino) {
+			iput(vol->quota_q_ino);
+			vol->quota_q_ino = NULL;
+		}
+		if (vol->quota_ino) {
+			iput(vol->quota_ino);
+			vol->quota_ino = NULL;
+		}
+		if (vol->extend_ino) {
+			iput(vol->extend_ino);
+			vol->extend_ino = NULL;
+		}
+		if (vol->secure_ino) {
+			iput(vol->secure_ino);
+			vol->secure_ino = NULL;
+		}
+	}
+	iput(vol->root_ino);
+	vol->root_ino = NULL;
+	iput(vol->lcnbmp_ino);
+	vol->lcnbmp_ino = NULL;
+	iput(vol->mftbmp_ino);
+	vol->mftbmp_ino = NULL;
+	if (vol->logfile_ino) {
+		iput(vol->logfile_ino);
+		vol->logfile_ino = NULL;
+	}
+	if (vol->mftmirr_ino) {
+		iput(vol->mftmirr_ino);
+		vol->mftmirr_ino = NULL;
+	}
+	/* Throw away the table of attribute definitions. */
+	vol->attrdef_size = 0;
+	if (vol->attrdef) {
+		kvfree(vol->attrdef);
+		vol->attrdef = NULL;
+	}
+	vol->upcase_len = 0;
+	mutex_lock(&ntfs_lock);
+	if (vol->upcase == default_upcase) {
+		ntfs_nr_upcase_users--;
+		vol->upcase = NULL;
+	}
+	mutex_unlock(&ntfs_lock);
+	if (vol->upcase) {
+		kvfree(vol->upcase);
+		vol->upcase = NULL;
+	}
+	if (vol->nls_map) {
+		unload_nls(vol->nls_map);
+		vol->nls_map = NULL;
+	}
+	/* Error exit code path. */
+unl_upcase_iput_tmp_ino_err_out_now:
+	if (vol->lcn_empty_bits_per_page)
+		kvfree(vol->lcn_empty_bits_per_page);
+	/*
+	 * Decrease the number of upcase users and destroy the global default
+	 * upcase table if necessary.
+	 */
+	mutex_lock(&ntfs_lock);
+	if (!--ntfs_nr_upcase_users && default_upcase) {
+		kvfree(default_upcase);
+		default_upcase = NULL;
+	}
+
+	mutex_unlock(&ntfs_lock);
+iput_tmp_ino_err_out_now:
+	iput(tmp_ino);
+	if (vol->mft_ino && vol->mft_ino != tmp_ino)
+		iput(vol->mft_ino);
+	vol->mft_ino = NULL;
+	/* Errors at this stage are irrelevant. */
+err_out_now:
+	sb->s_fs_info = NULL;
+	kfree(vol);
+	ntfs_debug("Failed, returning -EINVAL.");
+	lockdep_on();
+	return -EINVAL;
+}
+
+/*
+ * This is a slab cache to optimize allocations and deallocations of Unicode
+ * strings of the maximum length allowed by NTFS, which is NTFS_MAX_NAME_LEN
+ * (255) Unicode characters + a terminating NULL Unicode character.
+ */
+struct kmem_cache *ntfs_name_cache;
+
+/* Slab caches for efficient allocation/deallocation of inodes. */
+struct kmem_cache *ntfs_inode_cache;
+struct kmem_cache *ntfs_big_inode_cache;
+
+/* Init once constructor for the inode slab cache. */
+static void ntfs_big_inode_init_once(void *foo)
+{
+	struct ntfs_inode *ni = foo;
+
+	inode_init_once(VFS_I(ni));
+}
+
+/*
+ * Slab caches to optimize allocations and deallocations of attribute search
+ * contexts and index contexts, respectively.
+ */
+struct kmem_cache *ntfs_attr_ctx_cache;
+struct kmem_cache *ntfs_index_ctx_cache;
+
+/* Driver wide mutex. */
+DEFINE_MUTEX(ntfs_lock);
+
+static int ntfs_get_tree(struct fs_context *fc)
+{
+	return get_tree_bdev(fc, ntfs_fill_super);
+}
+
+static void ntfs_free_fs_context(struct fs_context *fc)
+{
+	struct ntfs_volume *vol = fc->s_fs_info;
+
+	if (vol)
+		ntfs_volume_free(vol);
+}
+
+static const struct fs_context_operations ntfs_context_ops = {
+	.parse_param	= ntfs_parse_param,
+	.get_tree	= ntfs_get_tree,
+	.free		= ntfs_free_fs_context,
+	.reconfigure	= ntfs_reconfigure,
+};
+
+static int ntfs_init_fs_context(struct fs_context *fc)
+{
+	struct ntfs_volume *vol;
+
+	/* Allocate a new struct ntfs_volume and place it in sb->s_fs_info. */
+	vol = kmalloc(sizeof(struct ntfs_volume), GFP_NOFS);
+	if (!vol)
+		return -ENOMEM;
+
+	/* Initialize struct ntfs_volume structure. */
+	*vol = (struct ntfs_volume) {
+		.uid = INVALID_UID,
+		.gid = INVALID_GID,
+		.fmask = 0,
+		.dmask = 0,
+		.mft_zone_multiplier = 1,
+		.on_errors = ON_ERRORS_CONTINUE,
+		.nls_map = load_nls_default(),
+		.preallocated_size = NTFS_DEF_PREALLOC_SIZE,
+	};
+
+	NVolSetShowHiddenFiles(vol);
+	NVolSetCaseSensitive(vol);
+	init_rwsem(&vol->mftbmp_lock);
+	init_rwsem(&vol->lcnbmp_lock);
+
+	fc->s_fs_info = vol;
+	fc->ops = &ntfs_context_ops;
+	return 0;
+}
+
+static struct file_system_type ntfs_fs_type = {
+	.owner                  = THIS_MODULE,
+	.name                   = "ntfs",
+	.init_fs_context        = ntfs_init_fs_context,
+	.parameters             = ntfs_parameters,
+	.kill_sb                = kill_block_super,
+	.fs_flags               = FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
+};
+MODULE_ALIAS_FS("ntfs");
+
+static int ntfs_workqueue_init(void)
+{
+	ntfs_wq = alloc_workqueue("ntfs-bg-io", 0, 0);
+	if (!ntfs_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+static void ntfs_workqueue_destroy(void)
+{
+	destroy_workqueue(ntfs_wq);
+	ntfs_wq = NULL;
+}
+
+/* Stable names for the slab caches. */
+static const char ntfs_index_ctx_cache_name[] = "ntfs_index_ctx_cache";
+static const char ntfs_attr_ctx_cache_name[] = "ntfs_attr_ctx_cache";
+static const char ntfs_name_cache_name[] = "ntfs_name_cache";
+static const char ntfs_inode_cache_name[] = "ntfs_inode_cache";
+static const char ntfs_big_inode_cache_name[] = "ntfs_big_inode_cache";
+
+static int __init init_ntfs_fs(void)
+{
+	int err = 0;
+
+	err = ntfs_workqueue_init();
+	if (err) {
+		pr_crit("Failed to register workqueue!\n");
+		return err;
+	}
+
+	ntfs_index_ctx_cache = kmem_cache_create(ntfs_index_ctx_cache_name,
+			sizeof(struct ntfs_index_context), 0 /* offset */,
+			SLAB_HWCACHE_ALIGN, NULL /* ctor */);
+	if (!ntfs_index_ctx_cache) {
+		pr_crit("Failed to create %s!\n", ntfs_index_ctx_cache_name);
+		goto ictx_err_out;
+	}
+	ntfs_attr_ctx_cache = kmem_cache_create(ntfs_attr_ctx_cache_name,
+			sizeof(struct ntfs_attr_search_ctx), 0 /* offset */,
+			SLAB_HWCACHE_ALIGN, NULL /* ctor */);
+	if (!ntfs_attr_ctx_cache) {
+		pr_crit("NTFS: Failed to create %s!\n",
+			ntfs_attr_ctx_cache_name);
+		goto actx_err_out;
+	}
+
+	ntfs_name_cache = kmem_cache_create(ntfs_name_cache_name,
+			(NTFS_MAX_NAME_LEN+2) * sizeof(__le16), 0,
+			SLAB_HWCACHE_ALIGN, NULL);
+	if (!ntfs_name_cache) {
+		pr_crit("Failed to create %s!\n", ntfs_name_cache_name);
+		goto name_err_out;
+	}
+
+	ntfs_inode_cache = kmem_cache_create(ntfs_inode_cache_name,
+			sizeof(struct ntfs_inode), 0, SLAB_RECLAIM_ACCOUNT, NULL);
+	if (!ntfs_inode_cache) {
+		pr_crit("Failed to create %s!\n", ntfs_inode_cache_name);
+		goto inode_err_out;
+	}
+
+	ntfs_big_inode_cache = kmem_cache_create(ntfs_big_inode_cache_name,
+			sizeof(struct big_ntfs_inode), 0, SLAB_HWCACHE_ALIGN |
+			SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
+			ntfs_big_inode_init_once);
+	if (!ntfs_big_inode_cache) {
+		pr_crit("Failed to create %s!\n", ntfs_big_inode_cache_name);
+		goto big_inode_err_out;
+	}
+
+	/* Register the ntfs sysctls. */
+	err = ntfs_sysctl(1);
+	if (err) {
+		pr_crit("Failed to register NTFS sysctls!\n");
+		goto sysctl_err_out;
+	}
+
+	err = register_filesystem(&ntfs_fs_type);
+	if (!err) {
+		ntfs_debug("NTFS driver registered successfully.");
+		return 0; /* Success! */
+	}
+	pr_crit("Failed to register NTFS filesystem driver!\n");
+
+	/* Unregister the ntfs sysctls. */
+	ntfs_sysctl(0);
+sysctl_err_out:
+	kmem_cache_destroy(ntfs_big_inode_cache);
+big_inode_err_out:
+	kmem_cache_destroy(ntfs_inode_cache);
+inode_err_out:
+	kmem_cache_destroy(ntfs_name_cache);
+name_err_out:
+	kmem_cache_destroy(ntfs_attr_ctx_cache);
+actx_err_out:
+	kmem_cache_destroy(ntfs_index_ctx_cache);
+ictx_err_out:
+	if (!err) {
+		pr_crit("Aborting NTFS filesystem driver registration...\n");
+		err = -ENOMEM;
+	}
+	return err;
+}
+
+static void __exit exit_ntfs_fs(void)
+{
+	ntfs_debug("Unregistering NTFS driver.");
+
+	unregister_filesystem(&ntfs_fs_type);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
+	kmem_cache_destroy(ntfs_big_inode_cache);
+	kmem_cache_destroy(ntfs_inode_cache);
+	kmem_cache_destroy(ntfs_name_cache);
+	kmem_cache_destroy(ntfs_attr_ctx_cache);
+	kmem_cache_destroy(ntfs_index_ctx_cache);
+	ntfs_workqueue_destroy();
+	/* Unregister the ntfs sysctls. */
+	ntfs_sysctl(0);
+}
+
+module_init(init_ntfs_fs);
+module_exit(exit_ntfs_fs);
+
+MODULE_AUTHOR("Anton Altaparmakov <anton@tuxera.com>"); /* Original read-only NTFS driver */
+MODULE_AUTHOR("Namjae Jeon <linkinjeon@kernel.org>"); /* Add write, iomap and various features */
+MODULE_DESCRIPTION("NTFS read-write filesystem driver");
+MODULE_LICENSE("GPL");
+#ifdef DEBUG
+module_param(debug_msgs, uint, 0);
+MODULE_PARM_DESC(debug_msgs, "Enable debug messages.");
+#endif
diff --git a/fs/ntfs/sysctl.c b/fs/ntfs/sysctl.c
new file mode 100644
index 0000000000000..8c5c1b7cd4839
--- /dev/null
+++ b/fs/ntfs/sysctl.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Code for sysctl handling in NTFS Linux kernel driver.
+ *
+ * Copyright (C) 1997 Martin von Löwis, Régis Duchesne
+ * Copyright (c) 2002-2005 Anton Altaparmakov
+ */
+
+#ifdef DEBUG
+
+#include <linux/module.h>
+
+#ifdef CONFIG_SYSCTL
+
+#include <linux/proc_fs.h>
+#include <linux/sysctl.h>
+
+#include "sysctl.h"
+#include "debug.h"
+
+/* Definition of the ntfs sysctl. */
+static const struct ctl_table ntfs_sysctls[] = {
+	{
+		.procname	= "ntfs-debug",
+		.data		= &debug_msgs,		/* Data pointer and size. */
+		.maxlen		= sizeof(debug_msgs),
+		.mode		= 0644,			/* Mode, proc handler. */
+		.proc_handler	= proc_dointvec
+	},
+};
+
+/* Storage for the sysctls header. */
+static struct ctl_table_header *sysctls_root_table;
+
+/*
+ * ntfs_sysctl - add or remove the debug sysctl
+ * @add:	add (1) or remove (0) the sysctl
+ *
+ * Add or remove the debug sysctl. Return 0 on success or -errno on error.
+ */
+int ntfs_sysctl(int add)
+{
+	if (add) {
+		sysctls_root_table = register_sysctl("fs/ntfs", ntfs_sysctls);
+		if (!sysctls_root_table)
+			return -ENOMEM;
+	} else {
+		unregister_sysctl_table(sysctls_root_table);
+		sysctls_root_table = NULL;
+	}
+	return 0;
+}
+#endif /* CONFIG_SYSCTL */
+#endif /* DEBUG */
diff --git a/fs/ntfs/sysctl.h b/fs/ntfs/sysctl.h
new file mode 100644
index 0000000000000..037127c994bff
--- /dev/null
+++ b/fs/ntfs/sysctl.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for sysctl handling in NTFS Linux kernel driver.
+ *
+ * Copyright (C) 1997 Martin von Löwis, Régis Duchesne
+ * Copyright (c) 2002-2004 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_SYSCTL_H
+#define _LINUX_NTFS_SYSCTL_H
+
+
+#if defined(DEBUG) && defined(CONFIG_SYSCTL)
+
+int ntfs_sysctl(int add);
+
+#else
+
+/* Just return success. */
+static inline int ntfs_sysctl(int add)
+{
+	return 0;
+}
+
+#endif /* DEBUG && CONFIG_SYSCTL */
+#endif /* _LINUX_NTFS_SYSCTL_H */
diff --git a/fs/ntfs/time.h b/fs/ntfs/time.h
new file mode 100644
index 0000000000000..aeed195bdcda6
--- /dev/null
+++ b/fs/ntfs/time.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * NTFS time conversion functions.
+ *
+ * Copyright (c) 2001-2005 Anton Altaparmakov
+ */
+
+#ifndef _LINUX_NTFS_TIME_H
+#define _LINUX_NTFS_TIME_H
+
+#include <linux/time.h>
+#include <asm/div64.h>		/* For do_div(). */
+
+#define NTFS_TIME_OFFSET ((s64)(369 * 365 + 89) * 24 * 3600)
+
+/*
+ * utc2ntfs - convert Linux UTC time to NTFS time
+ * @ts:		Linux UTC time to convert to NTFS time
+ *
+ * Convert the Linux UTC time @ts to its corresponding NTFS time and return
+ * that in little endian format.
+ *
+ * Linux stores time in a struct timespec64 consisting of a time64_t tv_sec
+ * and a long tv_nsec where tv_sec is the number of 1-second intervals since
+ * 1st January 1970, 00:00:00 UTC and tv_nsec is the number of 1-nano-second
+ * intervals since the value of tv_sec.
+ *
+ * NTFS uses Microsoft's standard time format which is stored in a s64 and is
+ * measured as the number of 100-nano-second intervals since 1st January 1601,
+ * 00:00:00 UTC.
+ */
+static inline __le64 utc2ntfs(const struct timespec64 ts)
+{
+	/*
+	 * Convert the seconds to 100ns intervals, add the nano-seconds
+	 * converted to 100ns intervals, and then add the NTFS time offset.
+	 */
+	return cpu_to_le64((u64)(ts.tv_sec + NTFS_TIME_OFFSET) * 10000000 +
+			ts.tv_nsec / 100);
+}
+
+/*
+ * get_current_ntfs_time - get the current time in little endian NTFS format
+ *
+ * Get the current time from the Linux kernel, convert it to its corresponding
+ * NTFS time and return that in little endian format.
+ */
+static inline __le64 get_current_ntfs_time(void)
+{
+	struct timespec64 ts;
+
+	ktime_get_coarse_real_ts64(&ts);
+	return utc2ntfs(ts);
+}
+
+/*
+ * ntfs2utc - convert NTFS time to Linux time
+ * @time:	NTFS time (little endian) to convert to Linux UTC
+ *
+ * Convert the little endian NTFS time @time to its corresponding Linux UTC
+ * time and return that in cpu format.
+ *
+ * Linux stores time in a struct timespec64 consisting of a time64_t tv_sec
+ * and a long tv_nsec where tv_sec is the number of 1-second intervals since
+ * 1st January 1970, 00:00:00 UTC and tv_nsec is the number of 1-nano-second
+ * intervals since the value of tv_sec.
+ *
+ * NTFS uses Microsoft's standard time format which is stored in a s64 and is
+ * measured as the number of 100 nano-second intervals since 1st January 1601,
+ * 00:00:00 UTC.
+ */
+static inline struct timespec64 ntfs2utc(const __le64 time)
+{
+	struct timespec64 ts;
+	s32 t32;
+
+	/* Subtract the NTFS time offset. */
+	s64 t = le64_to_cpu(time) - NTFS_TIME_OFFSET * 10000000;
+	/*
+	 * Convert the time to 1-second intervals and the remainder to
+	 * 1-nano-second intervals.
+	 */
+	ts.tv_sec = div_s64_rem(t, 10000000, &t32);
+	ts.tv_nsec = t32 * 100;
+	return ts;
+}
+#endif /* _LINUX_NTFS_TIME_H */
diff --git a/fs/ntfs/unistr.c b/fs/ntfs/unistr.c
new file mode 100644
index 0000000000000..7f11a2825527f
--- /dev/null
+++ b/fs/ntfs/unistr.c
@@ -0,0 +1,477 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NTFS Unicode string handling.
+ *
+ * Copyright (c) 2001-2006 Anton Altaparmakov
+ */
+
+#include "ntfs.h"
+
+/*
+ * IMPORTANT
+ * =========
+ *
+ * All these routines assume that the Unicode characters are in little endian
+ * encoding inside the strings!!!
+ */
+
+/*
+ * This is used by the name collation functions to quickly determine what
+ * characters are (in)valid.
+ */
+static const u8 legal_ansi_char_array[0x40] = {
+	0x00, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+	0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+
+	0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+	0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
+
+	0x17, 0x07, 0x18, 0x17, 0x17, 0x17, 0x17, 0x17,
+	0x17, 0x17, 0x18, 0x16, 0x16, 0x17, 0x07, 0x00,
+
+	0x17, 0x17, 0x17, 0x17, 0x17, 0x17, 0x17, 0x17,
+	0x17, 0x17, 0x04, 0x16, 0x18, 0x16, 0x18, 0x18,
+};
+
+/*
+ * ntfs_are_names_equal - compare two Unicode names for equality
+ * @s1:			name to compare to @s2
+ * @s1_len:		length in Unicode characters of @s1
+ * @s2:			name to compare to @s1
+ * @s2_len:		length in Unicode characters of @s2
+ * @ic:			ignore case bool
+ * @upcase:		upcase table (only if @ic == IGNORE_CASE)
+ * @upcase_size:	length in Unicode characters of @upcase (if present)
+ *
+ * Compare the names @s1 and @s2 and return 'true' (1) if the names are
+ * identical, or 'false' (0) if they are not identical. If @ic is IGNORE_CASE,
+ * the @upcase table is used to performa a case insensitive comparison.
+ */
+bool ntfs_are_names_equal(const __le16 *s1, size_t s1_len,
+		const __le16 *s2, size_t s2_len, const u32 ic,
+		const __le16 *upcase, const u32 upcase_size)
+{
+	if (s1_len != s2_len)
+		return false;
+	if (ic == CASE_SENSITIVE)
+		return !ntfs_ucsncmp(s1, s2, s1_len);
+	return !ntfs_ucsncasecmp(s1, s2, s1_len, upcase, upcase_size);
+}
+
+/*
+ * ntfs_collate_names - collate two Unicode names
+ * @name1:	first Unicode name to compare
+ * @name1_len:	first Unicode name length
+ * @name2:	second Unicode name to compare
+ * @name2_len:	second Unicode name length
+ * @err_val:	if @name1 contains an invalid character return this value
+ * @ic:		either CASE_SENSITIVE or IGNORE_CASE
+ * @upcase:	upcase table (ignored if @ic is CASE_SENSITIVE)
+ * @upcase_len:	upcase table size (ignored if @ic is CASE_SENSITIVE)
+ *
+ * ntfs_collate_names collates two Unicode names and returns:
+ *
+ *  -1 if the first name collates before the second one,
+ *   0 if the names match,
+ *   1 if the second name collates before the first one, or
+ * @err_val if an invalid character is found in @name1 during the comparison.
+ *
+ * The following characters are considered invalid: '"', '*', '<', '>' and '?'.
+ */
+int ntfs_collate_names(const __le16 *name1, const u32 name1_len,
+		const __le16 *name2, const u32 name2_len,
+		const int err_val, const u32 ic,
+		const __le16 *upcase, const u32 upcase_len)
+{
+	u32 cnt, min_len;
+	u16 c1, c2;
+
+	min_len = name1_len;
+	if (name1_len > name2_len)
+		min_len = name2_len;
+	for (cnt = 0; cnt < min_len; ++cnt) {
+		c1 = le16_to_cpu(*name1++);
+		c2 = le16_to_cpu(*name2++);
+		if (ic) {
+			if (c1 < upcase_len)
+				c1 = le16_to_cpu(upcase[c1]);
+			if (c2 < upcase_len)
+				c2 = le16_to_cpu(upcase[c2]);
+		}
+		if (c1 < 64 && legal_ansi_char_array[c1] & 8)
+			return err_val;
+		if (c1 < c2)
+			return -1;
+		if (c1 > c2)
+			return 1;
+	}
+	if (name1_len < name2_len)
+		return -1;
+	if (name1_len == name2_len)
+		return 0;
+	/* name1_len > name2_len */
+	c1 = le16_to_cpu(*name1);
+	if (c1 < 64 && legal_ansi_char_array[c1] & 8)
+		return err_val;
+	return 1;
+}
+
+/*
+ * ntfs_ucsncmp - compare two little endian Unicode strings
+ * @s1:		first string
+ * @s2:		second string
+ * @n:		maximum unicode characters to compare
+ *
+ * Compare the first @n characters of the Unicode strings @s1 and @s2,
+ * The strings in little endian format and appropriate le16_to_cpu()
+ * conversion is performed on non-little endian machines.
+ *
+ * The function returns an integer less than, equal to, or greater than zero
+ * if @s1 (or the first @n Unicode characters thereof) is found, respectively,
+ * to be less than, to match, or be greater than @s2.
+ */
+int ntfs_ucsncmp(const __le16 *s1, const __le16 *s2, size_t n)
+{
+	u16 c1, c2;
+	size_t i;
+
+	for (i = 0; i < n; ++i) {
+		c1 = le16_to_cpu(s1[i]);
+		c2 = le16_to_cpu(s2[i]);
+		if (c1 < c2)
+			return -1;
+		if (c1 > c2)
+			return 1;
+		if (!c1)
+			break;
+	}
+	return 0;
+}
+
+/*
+ * ntfs_ucsncasecmp - compare two little endian Unicode strings, ignoring case
+ * @s1:			first string
+ * @s2:			second string
+ * @n:			maximum unicode characters to compare
+ * @upcase:		upcase table
+ * @upcase_size:	upcase table size in Unicode characters
+ *
+ * Compare the first @n characters of the Unicode strings @s1 and @s2,
+ * ignoring case. The strings in little endian format and appropriate
+ * le16_to_cpu() conversion is performed on non-little endian machines.
+ *
+ * Each character is uppercased using the @upcase table before the comparison.
+ *
+ * The function returns an integer less than, equal to, or greater than zero
+ * if @s1 (or the first @n Unicode characters thereof) is found, respectively,
+ * to be less than, to match, or be greater than @s2.
+ */
+int ntfs_ucsncasecmp(const __le16 *s1, const __le16 *s2, size_t n,
+		const __le16 *upcase, const u32 upcase_size)
+{
+	size_t i;
+	u16 c1, c2;
+
+	for (i = 0; i < n; ++i) {
+		c1 = le16_to_cpu(s1[i]);
+		if (c1 < upcase_size)
+			c1 = le16_to_cpu(upcase[c1]);
+		c2 = le16_to_cpu(s2[i]);
+		if (c2 < upcase_size)
+			c2 = le16_to_cpu(upcase[c2]);
+		if (c1 < c2)
+			return -1;
+		if (c1 > c2)
+			return 1;
+		if (!c1)
+			break;
+	}
+	return 0;
+}
+
+int ntfs_file_compare_values(const struct file_name_attr *file_name_attr1,
+		const struct file_name_attr *file_name_attr2,
+		const int err_val, const u32 ic,
+		const __le16 *upcase, const u32 upcase_len)
+{
+	return ntfs_collate_names((__le16 *)&file_name_attr1->file_name,
+			file_name_attr1->file_name_length,
+			(__le16 *)&file_name_attr2->file_name,
+			file_name_attr2->file_name_length,
+			err_val, ic, upcase, upcase_len);
+}
+
+/*
+ * ntfs_nlstoucs - convert NLS string to little endian Unicode string
+ * @vol:	ntfs volume which we are working with
+ * @ins:	input NLS string buffer
+ * @ins_len:	length of input string in bytes
+ * @outs:	on return contains the allocated output Unicode string buffer
+ * @max_name_len: maximum number of Unicode characters allowed for the output name
+ *
+ * Convert the input string @ins, which is in whatever format the loaded NLS
+ * map dictates, into a little endian, 2-byte Unicode string.
+ *
+ * This function allocates the string and the caller is responsible for
+ * calling kmem_cache_free(ntfs_name_cache, *@outs); when finished with it.
+ *
+ * On success the function returns the number of Unicode characters written to
+ * the output string *@outs (>= 0), not counting the terminating Unicode NULL
+ * character. *@outs is set to the allocated output string buffer.
+ *
+ * On error, a negative number corresponding to the error code is returned. In
+ * that case the output string is not allocated. Both *@outs and *@outs_len
+ * are then undefined.
+ *
+ * This might look a bit odd due to fast path optimization...
+ */
+int ntfs_nlstoucs(const struct ntfs_volume *vol, const char *ins,
+		const int ins_len, __le16 **outs, int max_name_len)
+{
+	struct nls_table *nls = vol->nls_map;
+	__le16 *ucs;
+	wchar_t wc;
+	int i, o, wc_len;
+
+	/* We do not trust outside sources. */
+	if (likely(ins)) {
+		if (max_name_len > NTFS_MAX_NAME_LEN)
+			ucs = kvmalloc((max_name_len + 2) * sizeof(__le16),
+				       GFP_NOFS | __GFP_ZERO);
+		else
+			ucs = kmem_cache_alloc(ntfs_name_cache, GFP_NOFS);
+		if (likely(ucs)) {
+			if (vol->nls_utf8) {
+				o = utf8s_to_utf16s(ins, ins_len,
+						    UTF16_LITTLE_ENDIAN,
+						    (wchar_t *)ucs,
+						    max_name_len + 2);
+				if (o < 0 || o > max_name_len) {
+					wc_len = o;
+					goto name_err;
+				}
+			} else {
+				for (i = o = 0; i < ins_len; i += wc_len) {
+					wc_len = nls->char2uni(ins + i, ins_len - i,
+							&wc);
+					if (likely(wc_len >= 0 &&
+					    o < max_name_len)) {
+						if (likely(wc)) {
+							ucs[o++] = cpu_to_le16(wc);
+							continue;
+						} /* else if (!wc) */
+						break;
+					}
+
+					goto name_err;
+				}
+			}
+			ucs[o] = 0;
+			*outs = ucs;
+			return o;
+		} /* else if (!ucs) */
+		ntfs_debug("Failed to allocate buffer for converted name from ntfs_name_cache.");
+		return -ENOMEM;
+	} /* else if (!ins) */
+	ntfs_error(vol->sb, "Received NULL pointer.");
+	return -EINVAL;
+name_err:
+	if (max_name_len > NTFS_MAX_NAME_LEN)
+		kvfree(ucs);
+	else
+		kmem_cache_free(ntfs_name_cache, ucs);
+	if (wc_len < 0) {
+		ntfs_debug("Name using character set %s contains characters that cannot be converted to Unicode.",
+				nls->charset);
+		i = -EILSEQ;
+	} else {
+		ntfs_debug("Name is too long (maximum length for a name on NTFS is %d Unicode characters.",
+				max_name_len);
+		i = -ENAMETOOLONG;
+	}
+	return i;
+}
+
+/*
+ * ntfs_ucstonls - convert little endian Unicode string to NLS string
+ * @vol:	ntfs volume which we are working with
+ * @ins:	input Unicode string buffer
+ * @ins_len:	length of input string in Unicode characters
+ * @outs:	on return contains the (allocated) output NLS string buffer
+ * @outs_len:	length of output string buffer in bytes
+ *
+ * Convert the input little endian, 2-byte Unicode string @ins, of length
+ * @ins_len into the string format dictated by the loaded NLS.
+ *
+ * If *@outs is NULL, this function allocates the string and the caller is
+ * responsible for calling kfree(*@outs); when finished with it. In this case
+ * @outs_len is ignored and can be 0.
+ *
+ * On success the function returns the number of bytes written to the output
+ * string *@outs (>= 0), not counting the terminating NULL byte. If the output
+ * string buffer was allocated, *@outs is set to it.
+ *
+ * On error, a negative number corresponding to the error code is returned. In
+ * that case the output string is not allocated. The contents of *@outs are
+ * then undefined.
+ *
+ * This might look a bit odd due to fast path optimization...
+ */
+int ntfs_ucstonls(const struct ntfs_volume *vol, const __le16 *ins,
+		const int ins_len, unsigned char **outs, int outs_len)
+{
+	struct nls_table *nls = vol->nls_map;
+	unsigned char *ns;
+	int i, o, ns_len, wc;
+
+	/* We don't trust outside sources. */
+	if (ins) {
+		ns = *outs;
+		ns_len = outs_len;
+		if (ns && !ns_len) {
+			wc = -ENAMETOOLONG;
+			goto conversion_err;
+		}
+		if (!ns) {
+			ns_len = ins_len * NLS_MAX_CHARSET_SIZE;
+			ns = kmalloc(ns_len + 1, GFP_NOFS);
+			if (!ns)
+				goto mem_err_out;
+		}
+
+		if (vol->nls_utf8) {
+			o = utf16s_to_utf8s((const wchar_t *)ins, ins_len,
+					UTF16_LITTLE_ENDIAN, ns, ns_len);
+			if (o >= ns_len) {
+				wc = -ENAMETOOLONG;
+				goto conversion_err;
+			}
+			goto done;
+		}
+
+		for (i = o = 0; i < ins_len; i++) {
+retry:
+			wc = nls->uni2char(le16_to_cpu(ins[i]), ns + o,
+					ns_len - o);
+			if (wc > 0) {
+				o += wc;
+				continue;
+			} else if (!wc)
+				break;
+			else if (wc == -ENAMETOOLONG && ns != *outs) {
+				unsigned char *tc;
+				/* Grow in multiples of 64 bytes. */
+				tc = kmalloc((ns_len + 64) &
+						~63, GFP_NOFS);
+				if (tc) {
+					memcpy(tc, ns, ns_len);
+					ns_len = ((ns_len + 64) & ~63) - 1;
+					kfree(ns);
+					ns = tc;
+					goto retry;
+				} /* No memory so goto conversion_error; */
+			} /* wc < 0, real error. */
+			goto conversion_err;
+		}
+done:
+		ns[o] = 0;
+		*outs = ns;
+		return o;
+	} /* else (!ins) */
+	ntfs_error(vol->sb, "Received NULL pointer.");
+	return -EINVAL;
+conversion_err:
+	ntfs_error(vol->sb,
+		"Unicode name contains characters that cannot be converted to character set %s.  You might want to try to use the mount option nls=utf8.",
+		nls->charset);
+	if (ns != *outs)
+		kfree(ns);
+	if (wc != -ENAMETOOLONG)
+		wc = -EILSEQ;
+	return wc;
+mem_err_out:
+	ntfs_error(vol->sb, "Failed to allocate name!");
+	return -ENOMEM;
+}
+
+/*
+ * ntfs_ucsnlen - determine the length of a little endian Unicode string
+ * @s:		pointer to Unicode string
+ * @maxlen:	maximum length of string @s
+ *
+ * Return the number of Unicode characters in the little endian Unicode
+ * string @s up to a maximum of maxlen Unicode characters, not including
+ * the terminating (__le16)'\0'. If there is no (__le16)'\0' between @s
+ * and @s + @maxlen, @maxlen is returned.
+ *
+ * This function never looks beyond @s + @maxlen.
+ */
+static u32 ntfs_ucsnlen(const __le16 *s, u32 maxlen)
+{
+	u32 i;
+
+	for (i = 0; i < maxlen; i++) {
+		if (!le16_to_cpu(s[i]))
+			break;
+	}
+	return i;
+}
+
+/*
+ * ntfs_ucsndup - duplicate little endian Unicode string
+ * @s:		pointer to Unicode string
+ * @maxlen:	maximum length of string @s
+ *
+ * Return a pointer to a new little endian Unicode string which is a duplicate
+ * of the string s.  Memory for the new string is obtained with kmalloc,
+ * and can be freed with kfree.
+ *
+ * A maximum of @maxlen Unicode characters are copied and a terminating
+ * (__le16)'\0' little endian Unicode character is added.
+ *
+ * This function never looks beyond @s + @maxlen.
+ *
+ * Return a pointer to the new little endian Unicode string on success and NULL
+ * on failure with errno set to the error code.
+ */
+__le16 *ntfs_ucsndup(const __le16 *s, u32 maxlen)
+{
+	__le16 *dst;
+	u32 len;
+
+	len = ntfs_ucsnlen(s, maxlen);
+	dst = kmalloc((len + 1) * sizeof(__le16), GFP_NOFS);
+	if (dst) {
+		memcpy(dst, s, len * sizeof(__le16));
+		dst[len] = cpu_to_le16(L'\0');
+	}
+	return dst;
+}
+
+/*
+ * ntfs_names_are_equal - compare two Unicode names for equality
+ * @s1:                 name to compare to @s2
+ * @s1_len:             length in Unicode characters of @s1
+ * @s2:                 name to compare to @s1
+ * @s2_len:             length in Unicode characters of @s2
+ * @ic:                 ignore case bool
+ * @upcase:             upcase table (only if @ic == IGNORE_CASE)
+ * @upcase_size:        length in Unicode characters of @upcase (if present)
+ *
+ * Compare the names @s1 and @s2 and return TRUE (1) if the names are
+ * identical, or FALSE (0) if they are not identical. If @ic is IGNORE_CASE,
+ * the @upcase table is used to perform a case insensitive comparison.
+ */
+bool ntfs_names_are_equal(const __le16 *s1, size_t s1_len,
+		const __le16 *s2, size_t s2_len,
+		const u32 ic,
+		const __le16 *upcase, const u32 upcase_size)
+{
+	if (s1_len != s2_len)
+		return false;
+	if (!s1_len)
+		return true;
+	if (ic == CASE_SENSITIVE)
+		return ntfs_ucsncmp(s1, s2, s1_len) ? false : true;
+	return ntfs_ucsncasecmp(s1, s2, s1_len, upcase, upcase_size) ? false : true;
+}
diff --git a/fs/ntfs/upcase.c b/fs/ntfs/upcase.c
new file mode 100644
index 0000000000000..4b954470883f9
--- /dev/null
+++ b/fs/ntfs/upcase.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Generate the full NTFS Unicode upcase table in little endian.
+ *
+ * Copyright (c) 2001 Richard Russon <ntfs@flatcap.org>
+ * Copyright (c) 2001-2006 Anton Altaparmakov
+ */
+
+#include "ntfs.h"
+
+__le16 *generate_default_upcase(void)
+{
+	static const int uc_run_table[][3] = { /* Start, End, Add */
+	{0x0061, 0x007B,  -32}, {0x0451, 0x045D, -80}, {0x1F70, 0x1F72,  74},
+	{0x00E0, 0x00F7,  -32}, {0x045E, 0x0460, -80}, {0x1F72, 0x1F76,  86},
+	{0x00F8, 0x00FF,  -32}, {0x0561, 0x0587, -48}, {0x1F76, 0x1F78, 100},
+	{0x0256, 0x0258, -205}, {0x1F00, 0x1F08,   8}, {0x1F78, 0x1F7A, 128},
+	{0x028A, 0x028C, -217}, {0x1F10, 0x1F16,   8}, {0x1F7A, 0x1F7C, 112},
+	{0x03AC, 0x03AD,  -38}, {0x1F20, 0x1F28,   8}, {0x1F7C, 0x1F7E, 126},
+	{0x03AD, 0x03B0,  -37}, {0x1F30, 0x1F38,   8}, {0x1FB0, 0x1FB2,   8},
+	{0x03B1, 0x03C2,  -32}, {0x1F40, 0x1F46,   8}, {0x1FD0, 0x1FD2,   8},
+	{0x03C2, 0x03C3,  -31}, {0x1F51, 0x1F52,   8}, {0x1FE0, 0x1FE2,   8},
+	{0x03C3, 0x03CC,  -32}, {0x1F53, 0x1F54,   8}, {0x1FE5, 0x1FE6,   7},
+	{0x03CC, 0x03CD,  -64}, {0x1F55, 0x1F56,   8}, {0x2170, 0x2180, -16},
+	{0x03CD, 0x03CF,  -63}, {0x1F57, 0x1F58,   8}, {0x24D0, 0x24EA, -26},
+	{0x0430, 0x0450,  -32}, {0x1F60, 0x1F68,   8}, {0xFF41, 0xFF5B, -32},
+	{0}
+	};
+
+	static const int uc_dup_table[][2] = { /* Start, End */
+	{0x0100, 0x012F}, {0x01A0, 0x01A6}, {0x03E2, 0x03EF}, {0x04CB, 0x04CC},
+	{0x0132, 0x0137}, {0x01B3, 0x01B7}, {0x0460, 0x0481}, {0x04D0, 0x04EB},
+	{0x0139, 0x0149}, {0x01CD, 0x01DD}, {0x0490, 0x04BF}, {0x04EE, 0x04F5},
+	{0x014A, 0x0178}, {0x01DE, 0x01EF}, {0x04BF, 0x04BF}, {0x04F8, 0x04F9},
+	{0x0179, 0x017E}, {0x01F4, 0x01F5}, {0x04C1, 0x04C4}, {0x1E00, 0x1E95},
+	{0x018B, 0x018B}, {0x01FA, 0x0218}, {0x04C7, 0x04C8}, {0x1EA0, 0x1EF9},
+	{0}
+	};
+
+	static const int uc_word_table[][2] = { /* Offset, Value */
+	{0x00FF, 0x0178}, {0x01AD, 0x01AC}, {0x01F3, 0x01F1}, {0x0269, 0x0196},
+	{0x0183, 0x0182}, {0x01B0, 0x01AF}, {0x0253, 0x0181}, {0x026F, 0x019C},
+	{0x0185, 0x0184}, {0x01B9, 0x01B8}, {0x0254, 0x0186}, {0x0272, 0x019D},
+	{0x0188, 0x0187}, {0x01BD, 0x01BC}, {0x0259, 0x018F}, {0x0275, 0x019F},
+	{0x018C, 0x018B}, {0x01C6, 0x01C4}, {0x025B, 0x0190}, {0x0283, 0x01A9},
+	{0x0192, 0x0191}, {0x01C9, 0x01C7}, {0x0260, 0x0193}, {0x0288, 0x01AE},
+	{0x0199, 0x0198}, {0x01CC, 0x01CA}, {0x0263, 0x0194}, {0x0292, 0x01B7},
+	{0x01A8, 0x01A7}, {0x01DD, 0x018E}, {0x0268, 0x0197},
+	{0}
+	};
+
+	int i, r;
+	__le16 *uc;
+
+	uc = kvcalloc(default_upcase_len, sizeof(__le16), GFP_NOFS);
+	if (!uc)
+		return uc;
+	/* Generate the little endian Unicode upcase table used by ntfs. */
+	for (i = 0; i < default_upcase_len; i++)
+		uc[i] = cpu_to_le16(i);
+	for (r = 0; uc_run_table[r][0]; r++)
+		for (i = uc_run_table[r][0]; i < uc_run_table[r][1]; i++)
+			le16_add_cpu(&uc[i], uc_run_table[r][2]);
+	for (r = 0; uc_dup_table[r][0]; r++)
+		for (i = uc_dup_table[r][0]; i < uc_dup_table[r][1]; i += 2)
+			le16_add_cpu(&uc[i + 1], -1);
+	for (r = 0; uc_word_table[r][0]; r++)
+		uc[uc_word_table[r][0]] = cpu_to_le16(uc_word_table[r][1]);
+	return uc;
+}
diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h
new file mode 100644
index 0000000000000..af41427ec622c
--- /dev/null
+++ b/fs/ntfs/volume.h
@@ -0,0 +1,296 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Defines for volume structures in NTFS Linux kernel driver.
+ *
+ * Copyright (c) 2001-2006 Anton Altaparmakov
+ * Copyright (c) 2002 Richard Russon
+ * Copyright (c) 2025 LG Electronics Co., Ltd.
+ */
+
+#ifndef _LINUX_NTFS_VOLUME_H
+#define _LINUX_NTFS_VOLUME_H
+
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/uidgid.h>
+#include <linux/workqueue.h>
+#include <linux/errseq.h>
+
+#include "layout.h"
+
+#define NTFS_VOL_UID	BIT(1)
+#define NTFS_VOL_GID	BIT(2)
+
+/*
+ * The NTFS in memory super block structure.
+ *
+ * @sb: Pointer back to the super_block.
+ * @nr_blocks: Number of sb->s_blocksize bytes sized blocks on the device.
+ * @flags: Miscellaneous flags, see below.
+ * @uid: uid that files will be mounted as.
+ * @gid: gid that files will be mounted as.
+ * @fmask: The mask for file permissions.
+ * @dmask: The mask for directory permissions.
+ * @mft_zone_multiplier: Initial mft zone multiplier.
+ * @on_errors: What to do on filesystem errors.
+ * @wb_err: Writeback error tracking.
+ * @sector_size: in bytes
+ * @sector_size_bits: log2(sector_size)
+ * @cluster_size: in bytes
+ * @cluster_size_mask: cluster_size - 1
+ * @cluster_size_bits: log2(cluster_size)
+ * @mft_record_size: in bytes
+ * @mft_record_size_mask: mft_record_size - 1
+ * @mft_record_size_bits: log2(mft_record_size)
+ * @index_record_size: in bytes
+ * @index_record_size_mask: index_record_size - 1
+ * @index_record_size_bits: log2(index_record_size)
+ * @nr_clusters: Volume size in clusters == number of bits in lcn bitmap.
+ * @mft_lcn: Cluster location of mft data.
+ * @mftmirr_lcn: Cluster location of copy of mft.
+ * @serial_no: The volume serial number.
+ * @upcase_len: Number of entries in upcase[].
+ * @upcase: The upcase table.
+ * @attrdef_size: Size of the attribute definition table in bytes.
+ * @attrdef: Table of attribute definitions. Obtained from FILE_AttrDef.
+ * @mft_data_pos: Mft record number at which to allocate the next mft record.
+ * @mft_zone_start: First cluster of the mft zone.
+ * @mft_zone_end: First cluster beyond the mft zone.
+ * @mft_zone_pos: Current position in the mft zone.
+ * @data1_zone_pos: Current position in the first data zone.
+ * @data2_zone_pos: Current position in the second data zone.
+ * @mft_ino: The VFS inode of $MFT.
+ * @mftbmp_ino: Attribute inode for $MFT/$BITMAP.
+ * @mftbmp_lock: Lock for serializing accesses to the mft record bitmap.
+ * @mftmirr_ino: The VFS inode of $MFTMirr.
+ * @mftmirr_size: Size of mft mirror in mft records.
+ * @logfile_ino: The VFS inode of LogFile.
+ * @lcnbmp_ino: The VFS inode of $Bitmap.
+ * @lcnbmp_lock: Lock for serializing accesses to the cluster bitmap
+ * @vol_ino: The VFS inode of $Volume.
+ * @vol_flags: Volume flags.
+ * @major_ver: Ntfs major version of volume.
+ * @minor_ver: Ntfs minor version of volume.
+ * @volume_label: volume label.
+ * @root_ino: The VFS inode of the root directory.
+ * @secure_ino: The VFS inode of $Secure (NTFS3.0+ only, otherwise NULL).
+ * @extend_ino: The VFS inode of $Extend (NTFS3.0+ only, otherwise NULL).
+ * @quota_ino: The VFS inode of $Quota.
+ * @quota_q_ino: Attribute inode for $Quota/$Q.
+ * @nls_map: NLS (National Language Support) table.
+ * @nls_utf8: NLS table for UTF-8.
+ * @free_waitq: Wait queue for threads waiting for free clusters or MFT records.
+ * @free_clusters: Track the number of free clusters.
+ * @free_mft_records: Track the free mft records.
+ * @dirty_clusters: Number of clusters that are dirty.
+ * @sparse_compression_unit: Size of compression/sparse unit in clusters.
+ * @lcn_empty_bits_per_page: Number of empty bits per page in the LCN bitmap.
+ * @precalc_work: Work structure for background pre-calculation tasks.
+ * @preallocated_size: reallocation size (in bytes).
+ */
+struct ntfs_volume {
+	struct super_block *sb;
+	s64 nr_blocks;
+	unsigned long flags;
+	kuid_t uid;
+	kgid_t gid;
+	umode_t fmask;
+	umode_t dmask;
+	u8 mft_zone_multiplier;
+	u8 on_errors;
+	errseq_t wb_err;
+	u16 sector_size;
+	u8 sector_size_bits;
+	u32 cluster_size;
+	u32 cluster_size_mask;
+	u8 cluster_size_bits;
+	u32 mft_record_size;
+	u32 mft_record_size_mask;
+	u8 mft_record_size_bits;
+	u32 index_record_size;
+	u32 index_record_size_mask;
+	u8 index_record_size_bits;
+	s64 nr_clusters;
+	s64 mft_lcn;
+	s64 mftmirr_lcn;
+	u64 serial_no;
+	u32 upcase_len;
+	__le16 *upcase;
+	s32 attrdef_size;
+	struct attr_def *attrdef;
+	s64 mft_data_pos;
+	s64 mft_zone_start;
+	s64 mft_zone_end;
+	s64 mft_zone_pos;
+	s64 data1_zone_pos;
+	s64 data2_zone_pos;
+	struct inode *mft_ino;
+	struct inode *mftbmp_ino;
+	struct rw_semaphore mftbmp_lock;
+	struct inode *mftmirr_ino;
+	int mftmirr_size;
+	struct inode *logfile_ino;
+	struct inode *lcnbmp_ino;
+	struct rw_semaphore lcnbmp_lock;
+	struct inode *vol_ino;
+	__le16 vol_flags;
+	u8 major_ver;
+	u8 minor_ver;
+	unsigned char *volume_label;
+	struct inode *root_ino;
+	struct inode *secure_ino;
+	struct inode *extend_ino;
+	struct inode *quota_ino;
+	struct inode *quota_q_ino;
+	struct nls_table *nls_map;
+	bool nls_utf8;
+	wait_queue_head_t free_waitq;
+	atomic64_t free_clusters;
+	atomic64_t free_mft_records;
+	atomic64_t dirty_clusters;
+	u8 sparse_compression_unit;
+	unsigned int *lcn_empty_bits_per_page;
+	struct work_struct precalc_work;
+	loff_t preallocated_size;
+};
+
+/*
+ * Defined bits for the flags field in the ntfs_volume structure.
+ *
+ * NV_Errors			Volume has errors, prevent remount rw.
+ * NV_ShowSystemFiles		Return system files in ntfs_readdir().
+ * NV_CaseSensitive		Treat file names as case sensitive and
+ *				create filenames in the POSIX namespace.
+ *				Otherwise be case insensitive but still
+ *				create file names in POSIX namespace.
+ * NV_LogFileEmpty		LogFile journal is empty.
+ * NV_QuotaOutOfDate		Quota is out of date.
+ * NV_UsnJrnlStamped		UsnJrnl has been stamped.
+ * NV_ReadOnly			Volume is mounted read-only.
+ * NV_Compression		Volume supports compression.
+ * NV_FreeClusterKnown		Free cluster count is known and up-to-date.
+ * NV_Shutdown			Volume is in shutdown state
+ * NV_SysImmutable		Protect system files from deletion.
+ * NV_ShowHiddenFiles		Return hidden files in ntfs_readdir().
+ * NV_HideDotFiles		Hide names beginning with a dot (".").
+ * NV_CheckWindowsNames		Refuse creation/rename of files with
+ *				Windows-reserved names (CON, AUX, NUL, COM1,
+ *				LPT1, etc.) or invalid characters.
+ *
+ * NV_Discard			Issue discard/TRIM commands for freed clusters.
+ * NV_DisableSparse		Disable creation of sparse regions.
+ */
+enum {
+	NV_Errors,
+	NV_ShowSystemFiles,
+	NV_CaseSensitive,
+	NV_LogFileEmpty,
+	NV_QuotaOutOfDate,
+	NV_UsnJrnlStamped,
+	NV_ReadOnly,
+	NV_Compression,
+	NV_FreeClusterKnown,
+	NV_Shutdown,
+	NV_SysImmutable,
+	NV_ShowHiddenFiles,
+	NV_HideDotFiles,
+	NV_CheckWindowsNames,
+	NV_Discard,
+	NV_DisableSparse,
+};
+
+/*
+ * Macro tricks to expand the NVolFoo(), NVolSetFoo(), and NVolClearFoo()
+ * functions.
+ */
+#define DEFINE_NVOL_BIT_OPS(flag)					\
+static inline int NVol##flag(struct ntfs_volume *vol)		\
+{								\
+	return test_bit(NV_##flag, &(vol)->flags);		\
+}								\
+static inline void NVolSet##flag(struct ntfs_volume *vol)	\
+{								\
+	set_bit(NV_##flag, &(vol)->flags);			\
+}								\
+static inline void NVolClear##flag(struct ntfs_volume *vol)	\
+{								\
+	clear_bit(NV_##flag, &(vol)->flags);			\
+}
+
+/* Emit the ntfs volume bitops functions. */
+DEFINE_NVOL_BIT_OPS(Errors)
+DEFINE_NVOL_BIT_OPS(ShowSystemFiles)
+DEFINE_NVOL_BIT_OPS(CaseSensitive)
+DEFINE_NVOL_BIT_OPS(LogFileEmpty)
+DEFINE_NVOL_BIT_OPS(QuotaOutOfDate)
+DEFINE_NVOL_BIT_OPS(UsnJrnlStamped)
+DEFINE_NVOL_BIT_OPS(ReadOnly)
+DEFINE_NVOL_BIT_OPS(Compression)
+DEFINE_NVOL_BIT_OPS(FreeClusterKnown)
+DEFINE_NVOL_BIT_OPS(Shutdown)
+DEFINE_NVOL_BIT_OPS(SysImmutable)
+DEFINE_NVOL_BIT_OPS(ShowHiddenFiles)
+DEFINE_NVOL_BIT_OPS(HideDotFiles)
+DEFINE_NVOL_BIT_OPS(CheckWindowsNames)
+DEFINE_NVOL_BIT_OPS(Discard)
+DEFINE_NVOL_BIT_OPS(DisableSparse)
+
+static inline void ntfs_inc_free_clusters(struct ntfs_volume *vol, s64 nr)
+{
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+	atomic64_add(nr, &vol->free_clusters);
+}
+
+static inline void ntfs_dec_free_clusters(struct ntfs_volume *vol, s64 nr)
+{
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+	atomic64_sub(nr, &vol->free_clusters);
+}
+
+static inline void ntfs_inc_free_mft_records(struct ntfs_volume *vol, s64 nr)
+{
+	if (!NVolFreeClusterKnown(vol))
+		return;
+
+	atomic64_add(nr, &vol->free_mft_records);
+}
+
+static inline void ntfs_dec_free_mft_records(struct ntfs_volume *vol, s64 nr)
+{
+	if (!NVolFreeClusterKnown(vol))
+		return;
+
+	atomic64_sub(nr, &vol->free_mft_records);
+}
+
+static inline void ntfs_set_lcn_empty_bits(struct ntfs_volume *vol, unsigned long index,
+		u8 val, unsigned int count)
+{
+	if (!NVolFreeClusterKnown(vol))
+		wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
+
+	if (val)
+		vol->lcn_empty_bits_per_page[index] -= count;
+	else
+		vol->lcn_empty_bits_per_page[index] += count;
+}
+
+static __always_inline void ntfs_hold_dirty_clusters(struct ntfs_volume *vol, s64 nr_clusters)
+{
+	atomic64_add(nr_clusters, &vol->dirty_clusters);
+}
+
+static __always_inline void ntfs_release_dirty_clusters(struct ntfs_volume *vol, s64 nr_clusters)
+{
+	if (atomic64_read(&vol->dirty_clusters) < nr_clusters)
+		atomic64_set(&vol->dirty_clusters, 0);
+	else
+		atomic64_sub(nr_clusters, &vol->dirty_clusters);
+}
+
+s64 ntfs_available_clusters_count(struct ntfs_volume *vol, s64 nr_clusters);
+s64 get_nr_free_clusters(struct ntfs_volume *vol);
+#endif /* _LINUX_NTFS_VOLUME_H */
diff --git a/fs/ntfs3/Kconfig b/fs/ntfs3/Kconfig
index 7bc31d69f680d..876dbc613ae6d 100644
--- a/fs/ntfs3/Kconfig
+++ b/fs/ntfs3/Kconfig
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config NTFS3_FS
 	tristate "NTFS Read-Write file system support"
+	depends on !NTFS_FS || m
 	select BUFFER_HEAD
 	select NLS
 	select LEGACY_DIRECT_IO
@@ -46,12 +47,3 @@ config NTFS3_FS_POSIX_ACL
 	  NOTE: this is linux only feature. Windows will ignore these ACLs.
 
 	  If you don't know what Access Control Lists are, say N.
-
-config NTFS_FS
-	tristate "NTFS file system support"
-	select NTFS3_FS
-	select BUFFER_HEAD
-	select NLS
-	help
-	  This config option is here only for backward compatibility. NTFS
-	  filesystem is now handled by the NTFS3 driver.
diff --git a/fs/ntfs3/Makefile b/fs/ntfs3/Makefile
index 279701b62bbea..53bf2c17ac286 100644
--- a/fs/ntfs3/Makefile
+++ b/fs/ntfs3/Makefile
@@ -3,6 +3,26 @@
 # Makefile for the ntfs3 filesystem support.
 #
 
+# Subset of W=1 warnings
+subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter
+subdir-ccflags-y += -Wmissing-declarations
+subdir-ccflags-y += -Wmissing-format-attribute
+subdir-ccflags-y += -Wmissing-prototypes
+subdir-ccflags-y += -Wold-style-definition
+subdir-ccflags-y += -Wmissing-include-dirs
+condflags := \
+	$(call cc-option, -Wunused-but-set-variable)		\
+	$(call cc-option, -Wunused-const-variable)		\
+	$(call cc-option, -Wpacked-not-aligned)			\
+	$(call cc-option, -Wstringop-truncation)		\
+	$(call cc-option, -Wmaybe-uninitialized)
+subdir-ccflags-y += $(condflags)
+# The following turn off the warnings enabled by -Wextra
+subdir-ccflags-y += -Wno-missing-field-initializers
+subdir-ccflags-y += -Wno-sign-compare
+subdir-ccflags-y += -Wno-type-limits
+subdir-ccflags-y += -Wno-shift-negative-value
+
 # to check robot warnings
 ccflags-y += -Wint-to-pointer-cast \
 	$(call cc-option,-Wunused-but-set-variable,-Wunused-const-variable) \
diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c
index 4652a56ad1051..d99ab086ef6fe 100644
--- a/fs/ntfs3/dir.c
+++ b/fs/ntfs3/dir.c
@@ -676,14 +676,4 @@ const struct file_operations ntfs_dir_operations = {
 #endif
 	.setlease	= generic_setlease,
 };
-
-#if IS_ENABLED(CONFIG_NTFS_FS)
-const struct file_operations ntfs_legacy_dir_operations = {
-	.llseek		= generic_file_llseek,
-	.read		= generic_read_dir,
-	.iterate_shared	= ntfs_readdir,
-	.open		= ntfs_file_open,
-	.setlease	= generic_setlease,
-};
-#endif
 // clang-format on
diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c
index 7eecf1e01f741..f61f6d6b086ee 100644
--- a/fs/ntfs3/file.c
+++ b/fs/ntfs3/file.c
@@ -276,7 +276,7 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc)
 	struct file *file = desc->file;
 	struct inode *inode = file_inode(file);
 	struct ntfs_inode *ni = ntfs_i(inode);
-	const bool rw = vma_desc_test_flags(desc, VMA_WRITE_BIT);
+	const bool rw = vma_desc_test(desc, VMA_WRITE_BIT);
 	int err;
 
 	/* Avoid any operation if inode is bad. */
@@ -1569,15 +1569,4 @@ const struct file_operations ntfs_file_operations = {
 	.release	= ntfs_file_release,
 	.setlease	= generic_setlease,
 };
-
-#if IS_ENABLED(CONFIG_NTFS_FS)
-const struct file_operations ntfs_legacy_file_operations = {
-	.llseek		= generic_file_llseek,
-	.read_iter	= ntfs_file_read_iter,
-	.splice_read	= ntfs_file_splice_read,
-	.open		= ntfs_file_open,
-	.release	= ntfs_file_release,
-	.setlease	= generic_setlease,
-};
-#endif
 // clang-format on
diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c
index 2e901d073fe92..c0b9ca2426abc 100644
--- a/fs/ntfs3/frecord.c
+++ b/fs/ntfs3/frecord.c
@@ -1852,27 +1852,31 @@ enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr,
 	return REPARSE_LINK;
 }
 
-static struct page *ntfs_lock_new_page(struct address_space *mapping,
-				       pgoff_t index, gfp_t gfp)
+static struct folio *ntfs_lock_new_page(struct address_space *mapping,
+					pgoff_t index, gfp_t gfp)
 {
-	struct folio *folio = __filemap_get_folio(
-		mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
-	struct page *page;
+	struct folio *folio = __filemap_get_folio(mapping, index,
+			FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
 
 	if (IS_ERR(folio))
-		return ERR_CAST(folio);
+		return folio;
 
-	if (!folio_test_uptodate(folio))
-		return folio_file_page(folio, index);
+	if (!folio_test_uptodate(folio)) {
+		struct page *page = folio_file_page(folio, index);
+
+		if (IS_ERR(page))
+			return ERR_CAST(page);
+		return page_folio(page);
+	}
 
 	/* Use a temporary page to avoid data corruption */
 	folio_unlock(folio);
 	folio_put(folio);
-	page = alloc_page(gfp);
-	if (!page)
+	folio = folio_alloc(gfp, 0);
+	if (!folio)
 		return ERR_PTR(-ENOMEM);
-	__SetPageLocked(page);
-	return page;
+	__folio_set_locked(folio);
+	return folio;
 }
 
 /*
@@ -1894,6 +1898,7 @@ int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio)
 	u32 i, idx, frame_size, pages_per_frame;
 	gfp_t gfp_mask;
 	struct page *pg;
+	struct folio *f;
 
 	if (vbo >= i_size_read(&ni->vfs_inode)) {
 		folio_zero_range(folio, 0, folio_size(folio));
@@ -1929,12 +1934,12 @@ int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio)
 		if (i == idx)
 			continue;
 
-		pg = ntfs_lock_new_page(mapping, index, gfp_mask);
-		if (IS_ERR(pg)) {
-			err = PTR_ERR(pg);
+		f = ntfs_lock_new_page(mapping, index, gfp_mask);
+		if (IS_ERR(f)) {
+			err = PTR_ERR(f);
 			goto out1;
 		}
-		pages[i] = pg;
+		pages[i] = &f->page;
 	}
 
 	ni_lock(ni);
@@ -2023,18 +2028,18 @@ int ni_decompress_file(struct ntfs_inode *ni)
 		}
 
 		for (i = 0; i < pages_per_frame; i++, index++) {
-			struct page *pg;
+			struct folio *f;
 
-			pg = ntfs_lock_new_page(mapping, index, gfp_mask);
-			if (IS_ERR(pg)) {
+			f = ntfs_lock_new_page(mapping, index, gfp_mask);
+			if (IS_ERR(f)) {
 				while (i--) {
 					unlock_page(pages[i]);
 					put_page(pages[i]);
 				}
-				err = PTR_ERR(pg);
+				err = PTR_ERR(f);
 				goto out;
 			}
-			pages[i] = pg;
+			pages[i] = &f->page;
 		}
 
 		err = ni_read_frame(ni, vbo, pages, pages_per_frame, 1);
diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c
index 97f06c26fe1a6..8b107b6714ce0 100644
--- a/fs/ntfs3/index.c
+++ b/fs/ntfs3/index.c
@@ -714,10 +714,10 @@ static bool fnd_is_empty(struct ntfs_fnd *fnd)
  */
 static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx,
 				  const struct INDEX_HDR *hdr, const void *key,
-				  size_t key_len, const void *ctx, int *diff)
+				  size_t key_len, const void *ctx, int *diff,
+				  NTFS_CMP_FUNC cmp)
 {
 	struct NTFS_DE *e, *found = NULL;
-	NTFS_CMP_FUNC cmp = indx->cmp;
 	int min_idx = 0, mid_idx, max_idx = 0;
 	int diff2;
 	int table_size = 8;
@@ -727,9 +727,6 @@ static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx,
 	u32 total = le32_to_cpu(hdr->total);
 	u16 offs[128];
 
-	if (unlikely(!cmp))
-		return NULL;
-
 fill_table:
 	if (end > total)
 		return NULL;
@@ -800,7 +797,8 @@ static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx,
 static struct NTFS_DE *hdr_insert_de(const struct ntfs_index *indx,
 				     struct INDEX_HDR *hdr,
 				     const struct NTFS_DE *de,
-				     struct NTFS_DE *before, const void *ctx)
+				     struct NTFS_DE *before, const void *ctx,
+				     NTFS_CMP_FUNC cmp)
 {
 	int diff;
 	size_t off = PtrOffset(hdr, before);
@@ -823,7 +821,7 @@ static struct NTFS_DE *hdr_insert_de(const struct ntfs_index *indx,
 	}
 	/* No insert point is applied. Get it manually. */
 	before = hdr_find_e(indx, hdr, de + 1, le16_to_cpu(de->key_size), ctx,
-			    &diff);
+			    &diff, cmp);
 	if (!before)
 		return NULL;
 	off = PtrOffset(hdr, before);
@@ -915,10 +913,6 @@ int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi,
 
 	init_rwsem(&indx->run_lock);
 
-	indx->cmp = get_cmp_func(root);
-	if (!indx->cmp)
-		goto out;
-
 	return 0;
 
 out:
@@ -1141,6 +1135,7 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni,
 	int err;
 	struct NTFS_DE *e;
 	struct indx_node *node;
+	NTFS_CMP_FUNC cmp;
 
 	if (!root)
 		root = indx_get_root(&ni->dir, ni, NULL, NULL);
@@ -1150,10 +1145,16 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni,
 		return -EINVAL;
 	}
 
+	cmp = get_cmp_func(root);
+	if (unlikely(!cmp)) {
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+
 	/* Check cache. */
 	e = fnd->level ? fnd->de[fnd->level - 1] : fnd->root_de;
 	if (e && !de_is_last(e) &&
-	    !(*indx->cmp)(key, key_len, e + 1, le16_to_cpu(e->key_size), ctx)) {
+	    !(*cmp)(key, key_len, e + 1, le16_to_cpu(e->key_size), ctx)) {
 		*entry = e;
 		*diff = 0;
 		return 0;
@@ -1163,7 +1164,7 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni,
 	fnd_clear(fnd);
 
 	/* Lookup entry that is <= to the search value. */
-	e = hdr_find_e(indx, &root->ihdr, key, key_len, ctx, diff);
+	e = hdr_find_e(indx, &root->ihdr, key, key_len, ctx, diff, cmp);
 	if (!e)
 		return -EINVAL;
 
@@ -1183,7 +1184,7 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni,
 
 		/* Lookup entry that is <= to the search value. */
 		e = hdr_find_e(indx, &node->index->ihdr, key, key_len, ctx,
-			       diff);
+			       diff, cmp);
 		if (!e) {
 			put_indx_node(node);
 			return -EINVAL;
@@ -1585,7 +1586,7 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni,
 static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 				 const struct NTFS_DE *new_de,
 				 struct NTFS_DE *root_de, const void *ctx,
-				 struct ntfs_fnd *fnd, bool undo)
+				 struct ntfs_fnd *fnd, bool undo, NTFS_CMP_FUNC cmp)
 {
 	int err = 0;
 	struct NTFS_DE *e, *e0, *re;
@@ -1626,7 +1627,7 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 	if ((undo || asize + ds_root < sbi->max_bytes_per_attr) &&
 	    mi_resize_attr(mi, attr, ds_root)) {
 		hdr->total = cpu_to_le32(hdr_total + ds_root);
-		e = hdr_insert_de(indx, hdr, new_de, root_de, ctx);
+		e = hdr_insert_de(indx, hdr, new_de, root_de, ctx, cmp);
 		WARN_ON(!e);
 		fnd_clear(fnd);
 		fnd->root_de = e;
@@ -1767,7 +1768,7 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 	 * Now root is a parent for new index buffer.
 	 * Insert NewEntry a new buffer.
 	 */
-	e = hdr_insert_de(indx, hdr, new_de, NULL, ctx);
+	e = hdr_insert_de(indx, hdr, new_de, NULL, ctx, cmp);
 	if (!e) {
 		err = -EINVAL;
 		goto out_put_n;
@@ -1797,7 +1798,7 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 static int
 indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
 			struct INDEX_ROOT *root, const struct NTFS_DE *new_de,
-			const void *ctx, int level, struct ntfs_fnd *fnd)
+			const void *ctx, int level, struct ntfs_fnd *fnd, NTFS_CMP_FUNC cmp)
 {
 	int err;
 	const struct NTFS_DE *sp;
@@ -1814,7 +1815,7 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
 
 	/* Try the most easy case. */
 	e = fnd->level - 1 == level ? fnd->de[level] : NULL;
-	e = hdr_insert_de(indx, hdr1, new_de, e, ctx);
+	e = hdr_insert_de(indx, hdr1, new_de, e, ctx, cmp);
 	fnd->de[level] = e;
 	if (e) {
 		/* Just write updated index into disk. */
@@ -1891,12 +1892,12 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
 	 * (depending on sp <=> new_de).
 	 */
 	hdr_insert_de(indx,
-		      (*indx->cmp)(new_de + 1, le16_to_cpu(new_de->key_size),
+		      (*cmp)(new_de + 1, le16_to_cpu(new_de->key_size),
 				   up_e + 1, le16_to_cpu(up_e->key_size),
 				   ctx) < 0 ?
 			      hdr2 :
 			      hdr1,
-		      new_de, NULL, ctx);
+		      new_de, NULL, ctx, cmp);
 
 	indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits);
 
@@ -1911,14 +1912,14 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
 	 */
 	if (!level) {
 		/* Insert in root. */
-		err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0);
+		err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0, cmp);
 	} else {
 		/*
 		 * The target buffer's parent is another index buffer.
 		 * TODO: Remove recursion.
 		 */
 		err = indx_insert_into_buffer(indx, ni, root, up_e, ctx,
-					      level - 1, fnd);
+					      level - 1, fnd, cmp);
 	}
 
 	if (err) {
@@ -1952,6 +1953,7 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 	struct NTFS_DE *e;
 	struct ntfs_fnd *fnd_a = NULL;
 	struct INDEX_ROOT *root;
+	NTFS_CMP_FUNC cmp;
 
 	if (!fnd) {
 		fnd_a = fnd_get();
@@ -1968,6 +1970,12 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 		goto out;
 	}
 
+	cmp = get_cmp_func(root);
+	if (unlikely(!cmp)) {
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+
 	if (fnd_is_empty(fnd)) {
 		/*
 		 * Find the spot the tree where we want to
@@ -1991,13 +1999,13 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 		 * new entry into it.
 		 */
 		err = indx_insert_into_root(indx, ni, new_de, fnd->root_de, ctx,
-					    fnd, undo);
+					    fnd, undo, cmp);
 	} else {
 		/*
 		 * Found a leaf buffer, so we'll insert the new entry into it.
 		 */
 		err = indx_insert_into_buffer(indx, ni, root, new_de, ctx,
-					      fnd->level - 1, fnd);
+					      fnd->level - 1, fnd, cmp);
 	}
 
 	indx->version += 1;
@@ -2291,6 +2299,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 	u32 e_size, root_size, new_root_size;
 	size_t trim_bit;
 	const struct INDEX_NAMES *in;
+	NTFS_CMP_FUNC cmp;
 
 	fnd = fnd_get();
 	if (!fnd) {
@@ -2310,6 +2319,12 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 		goto out;
 	}
 
+	cmp = get_cmp_func(root);
+	if (unlikely(!cmp)) {
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+
 	/* Locate the entry to remove. */
 	err = indx_find(indx, ni, root, key, key_len, ctx, &diff, &e, fnd);
 	if (err)
@@ -2376,9 +2391,9 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 			err = level ? indx_insert_into_buffer(indx, ni, root,
 							      re, ctx,
 							      fnd->level - 1,
-							      fnd) :
+							      fnd, cmp) :
 				      indx_insert_into_root(indx, ni, re, e,
-							    ctx, fnd, 0);
+							    ctx, fnd, 0, cmp);
 			kfree(re);
 
 			if (err)
@@ -2673,6 +2688,7 @@ int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi,
 	struct INDEX_ROOT *root;
 	struct mft_inode *mi;
 	struct ntfs_index *indx = &ni->dir;
+	NTFS_CMP_FUNC cmp;
 
 	fnd = fnd_get();
 	if (!fnd)
@@ -2684,6 +2700,12 @@ int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi,
 		goto out;
 	}
 
+	cmp = get_cmp_func(root);
+	if (unlikely(!cmp)) {
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+
 	/* Find entry in directory. */
 	err = indx_find(indx, ni, root, fname, fname_full_size(fname), sbi,
 			&diff, &e, fnd);
diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c
index 6e65066ebcc1a..997fa8321bdc0 100644
--- a/fs/ntfs3/inode.c
+++ b/fs/ntfs3/inode.c
@@ -432,6 +432,11 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 		ni->mi.dirty = true;
 	}
 
+	if (!links) {
+		err = -EINVAL;
+		goto out;
+	}
+
 	set_nlink(inode, links);
 
 	if (S_ISDIR(mode)) {
@@ -443,9 +448,7 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 		 * Usually a hard links to directories are disabled.
 		 */
 		inode->i_op = &ntfs_dir_inode_operations;
-		inode->i_fop = unlikely(is_legacy_ntfs(sb)) ?
-				       &ntfs_legacy_dir_operations :
-				       &ntfs_dir_operations;
+		inode->i_fop = &ntfs_dir_operations;
 		ni->i_valid = 0;
 	} else if (S_ISLNK(mode)) {
 		ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY;
@@ -455,9 +458,7 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 	} else if (S_ISREG(mode)) {
 		ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY;
 		inode->i_op = &ntfs_file_inode_operations;
-		inode->i_fop = unlikely(is_legacy_ntfs(sb)) ?
-				       &ntfs_legacy_file_operations :
-				       &ntfs_file_operations;
+		inode->i_fop = &ntfs_file_operations;
 		inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr :
 							      &ntfs_aops;
 		if (ino != MFT_REC_MFT)
@@ -605,63 +606,18 @@ static void ntfs_iomap_read_end_io(struct bio *bio)
 	bio_put(bio);
 }
 
-/*
- * Copied from iomap/bio.c.
- */
-static int ntfs_iomap_bio_read_folio_range(const struct iomap_iter *iter,
-					   struct iomap_read_folio_ctx *ctx,
-					   size_t plen)
+static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter,
+		struct iomap_read_folio_ctx *ctx)
 {
-	struct folio *folio = ctx->cur_folio;
-	const struct iomap *iomap = &iter->iomap;
-	loff_t pos = iter->pos;
-	size_t poff = offset_in_folio(folio, pos);
-	loff_t length = iomap_length(iter);
-	sector_t sector;
 	struct bio *bio = ctx->read_ctx;
 
-	sector = iomap_sector(iomap, pos);
-	if (!bio || bio_end_sector(bio) != sector ||
-	    !bio_add_folio(bio, folio, plen, poff)) {
-		gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
-		gfp_t orig_gfp = gfp;
-		unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
-
-		if (bio)
-			submit_bio(bio);
-
-		if (ctx->rac) /* same as readahead_gfp_mask */
-			gfp |= __GFP_NORETRY | __GFP_NOWARN;
-		bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
-				gfp);
-		/*
-		 * If the bio_alloc fails, try it again for a single page to
-		 * avoid having to deal with partial page reads.  This emulates
-		 * what do_mpage_read_folio does.
-		 */
-		if (!bio)
-			bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ, orig_gfp);
-		if (ctx->rac)
-			bio->bi_opf |= REQ_RAHEAD;
-		bio->bi_iter.bi_sector = sector;
-		bio->bi_end_io = ntfs_iomap_read_end_io;
-		bio_add_folio_nofail(bio, folio, plen, poff);
-		ctx->read_ctx = bio;
-	}
-	return 0;
-}
-
-static void ntfs_iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx)
-{
-	struct bio *bio = ctx->read_ctx;
-
-	if (bio)
-		submit_bio(bio);
+	bio->bi_end_io = ntfs_iomap_read_end_io;
+	submit_bio(bio);
 }
 
 static const struct iomap_read_ops ntfs_iomap_bio_read_ops = {
-	.read_folio_range = ntfs_iomap_bio_read_folio_range,
-	.submit_read = ntfs_iomap_bio_submit_read,
+	.read_folio_range	= iomap_bio_read_folio_range,
+	.submit_read		= ntfs_iomap_bio_submit_read,
 };
 
 static int ntfs_read_folio(struct file *file, struct folio *folio)
@@ -822,6 +778,11 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 		return err;
 	}
 
+	if (!clen) {
+		/* broken file? */
+		return -EINVAL;
+	}
+
 	if (lcn == EOF_LCN) {
 		/* request out of file. */
 		if (flags & IOMAP_REPORT) {
@@ -855,11 +816,6 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 		return 0;
 	}
 
-	if (!clen) {
-		/* broken file? */
-		return -EINVAL;
-	}
-
 	iomap->bdev = inode->i_sb->s_bdev;
 	iomap->offset = offset;
 	iomap->length = ((loff_t)clen << cluster_bits) - off;
@@ -1646,9 +1602,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
 
 	if (S_ISDIR(mode)) {
 		inode->i_op = &ntfs_dir_inode_operations;
-		inode->i_fop = unlikely(is_legacy_ntfs(sb)) ?
-				       &ntfs_legacy_dir_operations :
-				       &ntfs_dir_operations;
+		inode->i_fop = &ntfs_dir_operations;
 	} else if (S_ISLNK(mode)) {
 		inode->i_op = &ntfs_link_inode_operations;
 		inode->i_fop = NULL;
@@ -1657,9 +1611,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
 		inode_nohighmem(inode);
 	} else if (S_ISREG(mode)) {
 		inode->i_op = &ntfs_file_inode_operations;
-		inode->i_fop = unlikely(is_legacy_ntfs(sb)) ?
-				       &ntfs_legacy_file_operations :
-				       &ntfs_file_operations;
+		inode->i_fop = &ntfs_file_operations;
 		inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr :
 							      &ntfs_aops;
 		init_rwsem(&ni->file.run_lock);
diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h
index daf5a1f472754..bbf3b6a1dcbee 100644
--- a/fs/ntfs3/ntfs_fs.h
+++ b/fs/ntfs3/ntfs_fs.h
@@ -196,9 +196,6 @@ struct ntfs_index {
 	struct rw_semaphore run_lock;
 	size_t version; /* increment each change */
 
-	/*TODO: Remove 'cmp'. */
-	NTFS_CMP_FUNC cmp;
-
 	u8 index_bits; // log2(root->index_block_size)
 	u8 idx2vbn_bits; // log2(root->index_block_clst)
 	u8 vbn2vbo_bits; // index_block_size < cluster? 9 : cluster_bits
@@ -530,7 +527,6 @@ struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni,
 			   struct ntfs_fnd *fnd);
 bool dir_is_empty(struct inode *dir);
 extern const struct file_operations ntfs_dir_operations;
-extern const struct file_operations ntfs_legacy_dir_operations;
 
 /* Globals from file.c */
 int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
@@ -546,7 +542,6 @@ long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg);
 extern const struct inode_operations ntfs_special_inode_operations;
 extern const struct inode_operations ntfs_file_inode_operations;
 extern const struct file_operations ntfs_file_operations;
-extern const struct file_operations ntfs_legacy_file_operations;
 
 /* Globals from frecord.c */
 void ni_remove_mi(struct ntfs_inode *ni, struct mft_inode *mi);
@@ -1250,13 +1245,4 @@ static inline void le64_sub_cpu(__le64 *var, u64 val)
 	*var = cpu_to_le64(le64_to_cpu(*var) - val);
 }
 
-#if IS_ENABLED(CONFIG_NTFS_FS)
-bool is_legacy_ntfs(struct super_block *sb);
-#else
-static inline bool is_legacy_ntfs(struct super_block *sb)
-{
-	return false;
-}
-#endif
-
 #endif /* _LINUX_NTFS3_NTFS_FS_H */
diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c
index 174a7cb202a08..6588caa036328 100644
--- a/fs/ntfs3/super.c
+++ b/fs/ntfs3/super.c
@@ -153,7 +153,7 @@ void ntfs_inode_printk(struct inode *inode, const char *fmt, ...)
 	vaf.fmt = printk_skip_level(fmt);
 	vaf.va = &args;
 
-	printk("%c%cntfs3(%s): ino=%lx,%s %pV\n", KERN_SOH_ASCII, level,
+	printk("%c%cntfs3(%s): ino=%llx,%s %pV\n", KERN_SOH_ASCII, level,
 	       sb->s_id, inode->i_ino, name ? name : "", &vaf);
 
 	va_end(args);
@@ -434,12 +434,6 @@ static int ntfs_fs_reconfigure(struct fs_context *fc)
 	struct ntfs_mount_options *new_opts = fc->fs_private;
 	int ro_rw;
 
-	/* If ntfs3 is used as legacy ntfs enforce read-only mode. */
-	if (is_legacy_ntfs(sb)) {
-		fc->sb_flags |= SB_RDONLY;
-		goto out;
-	}
-
 	ro_rw = sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY);
 	if (ro_rw && (sbi->flags & NTFS_FLAGS_NEED_REPLAY)) {
 		errorf(fc,
@@ -466,7 +460,6 @@ static int ntfs_fs_reconfigure(struct fs_context *fc)
 		return -EINVAL;
 	}
 
-out:
 	sync_filesystem(sb);
 	swap(sbi->options, fc->fs_private);
 
@@ -1699,8 +1692,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
 
 	ntfs_create_procdir(sb);
 
-	if (is_legacy_ntfs(sb))
-		sb->s_flags |= SB_RDONLY;
 	return 0;
 
 put_inode_out:
@@ -1823,7 +1814,7 @@ static const struct fs_context_operations ntfs_context_ops = {
  * This will called when mount/remount. We will first initialize
  * options so that if remount we can use just that.
  */
-static int __ntfs_init_fs_context(struct fs_context *fc)
+static int ntfs_init_fs_context(struct fs_context *fc)
 {
 	struct ntfs_mount_options *opts;
 	struct ntfs_sb_info *sbi;
@@ -1877,11 +1868,6 @@ static int __ntfs_init_fs_context(struct fs_context *fc)
 	return -ENOMEM;
 }
 
-static int ntfs_init_fs_context(struct fs_context *fc)
-{
-	return __ntfs_init_fs_context(fc);
-}
-
 static void ntfs3_kill_sb(struct super_block *sb)
 {
 	struct ntfs_sb_info *sbi = sb->s_fs_info;
@@ -1903,47 +1889,6 @@ static struct file_system_type ntfs_fs_type = {
 	.fs_flags		= FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
 };
 
-#if IS_ENABLED(CONFIG_NTFS_FS)
-static int ntfs_legacy_init_fs_context(struct fs_context *fc)
-{
-	int ret;
-
-	ret = __ntfs_init_fs_context(fc);
-	/* If ntfs3 is used as legacy ntfs enforce read-only mode. */
-	fc->sb_flags |= SB_RDONLY;
-	return ret;
-}
-
-static struct file_system_type ntfs_legacy_fs_type = {
-	.owner			= THIS_MODULE,
-	.name			= "ntfs",
-	.init_fs_context	= ntfs_legacy_init_fs_context,
-	.parameters		= ntfs_fs_parameters,
-	.kill_sb		= ntfs3_kill_sb,
-	.fs_flags		= FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
-};
-MODULE_ALIAS_FS("ntfs");
-
-static inline void register_as_ntfs_legacy(void)
-{
-	int err = register_filesystem(&ntfs_legacy_fs_type);
-	if (err)
-		pr_warn("ntfs3: Failed to register legacy ntfs filesystem driver: %d\n", err);
-}
-
-static inline void unregister_as_ntfs_legacy(void)
-{
-	unregister_filesystem(&ntfs_legacy_fs_type);
-}
-bool is_legacy_ntfs(struct super_block *sb)
-{
-	return sb->s_type == &ntfs_legacy_fs_type;
-}
-#else
-static inline void register_as_ntfs_legacy(void) {}
-static inline void unregister_as_ntfs_legacy(void) {}
-#endif
-
 // clang-format on
 
 static int __init init_ntfs_fs(void)
@@ -1972,7 +1917,6 @@ static int __init init_ntfs_fs(void)
 		goto out1;
 	}
 
-	register_as_ntfs_legacy();
 	err = register_filesystem(&ntfs_fs_type);
 	if (err)
 		goto out;
@@ -1992,7 +1936,6 @@ static void __exit exit_ntfs_fs(void)
 	rcu_barrier();
 	kmem_cache_destroy(ntfs_inode_cachep);
 	unregister_filesystem(&ntfs_fs_type);
-	unregister_as_ntfs_legacy();
 	ntfs3_exit_bitmap();
 	ntfs_remove_proc_root();
 }
diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c
index 344fd4d95fbc8..d40f5d205bce3 100644
--- a/fs/ocfs2/alloc.c
+++ b/fs/ocfs2/alloc.c
@@ -7318,7 +7318,7 @@ int ocfs2_commit_truncate(struct ocfs2_super *osb,
 		 * to check it up here before changing the tree.
 		*/
 		if (root_el->l_tree_depth && rec->e_int_clusters == 0) {
-			mlog(ML_ERROR, "Inode %lu has an empty "
+			mlog(ML_ERROR, "Inode %llu has an empty "
 				    "extent record, depth %u\n", inode->i_ino,
 				    le16_to_cpu(root_el->l_tree_depth));
 			status = ocfs2_remove_rightmost_empty_extent(osb,
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index 17ba79f443ee7..e9ce52f9de35f 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -137,7 +137,7 @@ int ocfs2_get_block(struct inode *inode, sector_t iblock,
 			      (unsigned long long)iblock, bh_result, create);
 
 	if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SYSTEM_FILE)
-		mlog(ML_NOTICE, "get_block on system inode 0x%p (%lu)\n",
+		mlog(ML_NOTICE, "get_block on system inode 0x%p (%llu)\n",
 		     inode, inode->i_ino);
 
 	if (S_ISLNK(inode->i_mode)) {
@@ -2146,7 +2146,7 @@ static int ocfs2_dio_wr_get_block(struct inode *inode, sector_t iblock,
 	    ((iblock + ((len - 1) >> i_blkbits)) > endblk))
 		len = (endblk - iblock + 1) << i_blkbits;
 
-	mlog(0, "get block of %lu at %llu:%u req %u\n",
+	mlog(0, "get block of %llu at %llu:%u req %u\n",
 			inode->i_ino, pos, len, total_len);
 
 	/*
@@ -2294,8 +2294,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode,
 		goto out;
 	}
 
-	down_write(&oi->ip_alloc_sem);
-
 	/* Delete orphan before acquire i_rwsem. */
 	if (dwc->dw_orphaned) {
 		BUG_ON(dwc->dw_writer_pid != task_pid_nr(current));
@@ -2308,6 +2306,7 @@ static int ocfs2_dio_end_io_write(struct inode *inode,
 			mlog_errno(ret);
 	}
 
+	down_write(&oi->ip_alloc_sem);
 	di = (struct ocfs2_dinode *)di_bh->b_data;
 
 	ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh);
diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c
index 1c8abf2c592ca..b82fe4431eb1f 100644
--- a/fs/ocfs2/dir.c
+++ b/fs/ocfs2/dir.c
@@ -794,7 +794,7 @@ static int ocfs2_dx_dir_lookup_rec(struct inode *inode,
 	if (le16_to_cpu(el->l_count) !=
 	    ocfs2_extent_recs_per_dx_root(inode->i_sb)) {
 		ret = ocfs2_error(inode->i_sb,
-				  "Inode %lu has invalid extent list length %u\n",
+				  "Inode %llu has invalid extent list length %u\n",
 				  inode->i_ino, le16_to_cpu(el->l_count));
 		goto out;
 	}
@@ -812,7 +812,7 @@ static int ocfs2_dx_dir_lookup_rec(struct inode *inode,
 
 		if (el->l_tree_depth) {
 			ret = ocfs2_error(inode->i_sb,
-					  "Inode %lu has non zero tree depth in btree tree block %llu\n",
+					  "Inode %llu has non zero tree depth in btree tree block %llu\n",
 					  inode->i_ino,
 					  (unsigned long long)eb_bh->b_blocknr);
 			goto out;
@@ -821,7 +821,7 @@ static int ocfs2_dx_dir_lookup_rec(struct inode *inode,
 
 	if (le16_to_cpu(el->l_next_free_rec) == 0) {
 		ret = ocfs2_error(inode->i_sb,
-				  "Inode %lu has empty extent list at depth %u\n",
+				  "Inode %llu has empty extent list at depth %u\n",
 				  inode->i_ino,
 				  le16_to_cpu(el->l_tree_depth));
 		goto out;
@@ -839,7 +839,7 @@ static int ocfs2_dx_dir_lookup_rec(struct inode *inode,
 
 	if (!found) {
 		ret = ocfs2_error(inode->i_sb,
-				  "Inode %lu has bad extent record (%u, %u, 0) in btree\n",
+				  "Inode %llu has bad extent record (%u, %u, 0) in btree\n",
 				  inode->i_ino,
 				  le32_to_cpu(rec->e_cpos),
 				  ocfs2_rec_clusters(el, rec));
diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c
index 70ca79e4bdc3f..dc9da9133c8ee 100644
--- a/fs/ocfs2/dlm/dlmdomain.c
+++ b/fs/ocfs2/dlm/dlmdomain.c
@@ -980,6 +980,14 @@ static int dlm_match_regions(struct dlm_ctxt *dlm,
 		goto bail;
 	}
 
+	if (qr->qr_numregions > O2NM_MAX_REGIONS) {
+		mlog(ML_ERROR, "Domain %s: Joining node %d has invalid "
+		     "number of heartbeat regions %u\n",
+		     qr->qr_domain, qr->qr_node, qr->qr_numregions);
+		status = -EINVAL;
+		goto bail;
+	}
+
 	r = remote;
 	for (i = 0; i < qr->qr_numregions; ++i) {
 		mlog(0, "Region %.*s\n", O2HB_MAX_REGION_NAME_LEN, r);
@@ -994,7 +1002,7 @@ static int dlm_match_regions(struct dlm_ctxt *dlm,
 	for (i = 0; i < localnr; ++i) {
 		foundit = 0;
 		r = remote;
-		for (j = 0; j <= qr->qr_numregions; ++j) {
+		for (j = 0; j < qr->qr_numregions; ++j) {
 			if (!memcmp(l, r, O2HB_MAX_REGION_NAME_LEN)) {
 				foundit = 1;
 				break;
diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c
index eb62724bbe9bb..93eff38fdaddb 100644
--- a/fs/ocfs2/dlm/dlmmaster.c
+++ b/fs/ocfs2/dlm/dlmmaster.c
@@ -930,7 +930,6 @@ struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm,
 	if (blocked)
 		goto wait;
 
-	ret = -EINVAL;
 	dlm_node_iter_init(mle->vote_map, &iter);
 	while ((nodenum = dlm_node_iter_next(&iter)) >= 0) {
 		ret = dlm_do_master_request(res, mle, nodenum);
diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c
index 45cce261da65c..5821e33df78fd 100644
--- a/fs/ocfs2/dlmfs/dlmfs.c
+++ b/fs/ocfs2/dlmfs/dlmfs.c
@@ -123,7 +123,7 @@ static int dlmfs_file_open(struct inode *inode,
 	if (S_ISDIR(inode->i_mode))
 		BUG();
 
-	mlog(0, "open called on inode %lu, flags 0x%x\n", inode->i_ino,
+	mlog(0, "open called on inode %llu, flags 0x%x\n", inode->i_ino,
 		file->f_flags);
 
 	status = dlmfs_decode_open_flags(file->f_flags, &level, &flags);
@@ -170,7 +170,7 @@ static int dlmfs_file_release(struct inode *inode,
 	if (S_ISDIR(inode->i_mode))
 		BUG();
 
-	mlog(0, "close called on inode %lu\n", inode->i_ino);
+	mlog(0, "close called on inode %llu\n", inode->i_ino);
 
 	if (fp) {
 		level = fp->fp_lock_level;
@@ -242,7 +242,7 @@ static ssize_t dlmfs_file_write(struct file *filp,
 	int bytes_left;
 	struct inode *inode = file_inode(filp);
 
-	mlog(0, "inode %lu, count = %zu, *ppos = %llu\n",
+	mlog(0, "inode %llu, count = %zu, *ppos = %llu\n",
 		inode->i_ino, count, *ppos);
 
 	if (*ppos >= DLM_LVB_LEN)
@@ -301,7 +301,7 @@ static void dlmfs_evict_inode(struct inode *inode)
 
 	clear_inode(inode);
 
-	mlog(0, "inode %lu\n", inode->i_ino);
+	mlog(0, "inode %llu\n", inode->i_ino);
 
 	ip = DLMFS_I(inode);
 	lockres = &ip->ip_lockres;
@@ -490,7 +490,7 @@ static int dlmfs_unlink(struct inode *dir,
 	int status;
 	struct inode *inode = d_inode(dentry);
 
-	mlog(0, "unlink inode %lu\n", inode->i_ino);
+	mlog(0, "unlink inode %llu\n", inode->i_ino);
 
 	/* if there are no current holders, or none that are waiting
 	 * to acquire a lock, this basically destroys our lockres. */
diff --git a/fs/ocfs2/extent_map.c b/fs/ocfs2/extent_map.c
index d68229422dda3..eb5dcd17d4375 100644
--- a/fs/ocfs2/extent_map.c
+++ b/fs/ocfs2/extent_map.c
@@ -291,7 +291,7 @@ static int ocfs2_last_eb_is_empty(struct inode *inode,
 
 	if (el->l_tree_depth) {
 		ocfs2_error(inode->i_sb,
-			    "Inode %lu has non zero tree depth in leaf block %llu\n",
+			    "Inode %llu has non zero tree depth in leaf block %llu\n",
 			    inode->i_ino,
 			    (unsigned long long)eb_bh->b_blocknr);
 		ret = -EROFS;
@@ -427,7 +427,7 @@ static int ocfs2_get_clusters_nocache(struct inode *inode,
 
 		if (el->l_tree_depth) {
 			ocfs2_error(inode->i_sb,
-				    "Inode %lu has non zero tree depth in leaf block %llu\n",
+				    "Inode %llu has non zero tree depth in leaf block %llu\n",
 				    inode->i_ino,
 				    (unsigned long long)eb_bh->b_blocknr);
 			ret = -EROFS;
@@ -437,7 +437,7 @@ static int ocfs2_get_clusters_nocache(struct inode *inode,
 
 	if (le16_to_cpu(el->l_next_free_rec) > le16_to_cpu(el->l_count)) {
 		ocfs2_error(inode->i_sb,
-			    "Inode %lu has an invalid extent (next_free_rec %u, count %u)\n",
+			    "Inode %llu has an invalid extent (next_free_rec %u, count %u)\n",
 			    inode->i_ino,
 			    le16_to_cpu(el->l_next_free_rec),
 			    le16_to_cpu(el->l_count));
@@ -472,7 +472,7 @@ static int ocfs2_get_clusters_nocache(struct inode *inode,
 
 	if (!rec->e_blkno) {
 		ocfs2_error(inode->i_sb,
-			    "Inode %lu has bad extent record (%u, %u, 0)\n",
+			    "Inode %llu has bad extent record (%u, %u, 0)\n",
 			    inode->i_ino,
 			    le32_to_cpu(rec->e_cpos),
 			    ocfs2_rec_clusters(el, rec));
@@ -561,7 +561,7 @@ int ocfs2_xattr_get_clusters(struct inode *inode, u32 v_cluster,
 
 		if (el->l_tree_depth) {
 			ocfs2_error(inode->i_sb,
-				    "Inode %lu has non zero tree depth in xattr leaf block %llu\n",
+				    "Inode %llu has non zero tree depth in xattr leaf block %llu\n",
 				    inode->i_ino,
 				    (unsigned long long)eb_bh->b_blocknr);
 			ret = -EROFS;
@@ -580,7 +580,7 @@ int ocfs2_xattr_get_clusters(struct inode *inode, u32 v_cluster,
 
 		if (!rec->e_blkno) {
 			ocfs2_error(inode->i_sb,
-				    "Inode %lu has bad extent record (%u, %u, 0) in xattr\n",
+				    "Inode %llu has bad extent record (%u, %u, 0) in xattr\n",
 				    inode->i_ino,
 				    le32_to_cpu(rec->e_cpos),
 				    ocfs2_rec_clusters(el, rec));
diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c
index 03a51662ea8e7..26025ba2656c6 100644
--- a/fs/ocfs2/inode.c
+++ b/fs/ocfs2/inode.c
@@ -1196,7 +1196,7 @@ static void ocfs2_clear_inode(struct inode *inode)
 				inode->i_nlink);
 
 	mlog_bug_on_msg(osb == NULL,
-			"Inode=%lu\n", inode->i_ino);
+			"Inode=%llu\n", inode->i_ino);
 
 	dquot_drop(inode);
 
diff --git a/fs/ocfs2/quota_global.c b/fs/ocfs2/quota_global.c
index e85b1ccf81be0..77b8f0363e942 100644
--- a/fs/ocfs2/quota_global.c
+++ b/fs/ocfs2/quota_global.c
@@ -311,11 +311,25 @@ int ocfs2_lock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex)
 	spin_unlock(&dq_data_lock);
 	if (ex) {
 		inode_lock(oinfo->dqi_gqinode);
-		down_write(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem);
+		if (!down_write_trylock(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem)) {
+			inode_unlock(oinfo->dqi_gqinode);
+			status = -EBUSY;
+			goto bail;
+		}
 	} else {
 		down_read(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem);
 	}
 	return 0;
+
+bail:
+	/* does a similar job as ocfs2_unlock_global_qf */
+	ocfs2_inode_unlock(oinfo->dqi_gqinode, ex);
+	brelse(oinfo->dqi_gqi_bh);
+	spin_lock(&dq_data_lock);
+	if (!--oinfo->dqi_gqi_count)
+		oinfo->dqi_gqi_bh = NULL;
+	spin_unlock(&dq_data_lock);
+	return status;
 }
 
 void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex)
diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c
index c4e0117d89778..12cbb4fccda0d 100644
--- a/fs/ocfs2/quota_local.c
+++ b/fs/ocfs2/quota_local.c
@@ -471,7 +471,7 @@ static int ocfs2_recover_local_quota_file(struct inode *lqinode,
 	qsize_t spacechange, inodechange;
 	unsigned int memalloc;
 
-	trace_ocfs2_recover_local_quota_file((unsigned long)lqinode->i_ino, type);
+	trace_ocfs2_recover_local_quota_file(lqinode->i_ino, type);
 
 	list_for_each_entry_safe(rchunk, next, &(rec->r_list[type]), rc_list) {
 		chunk = rchunk->rc_chunk;
@@ -1224,7 +1224,9 @@ int ocfs2_create_local_dquot(struct dquot *dquot)
 	int status;
 	u64 pcount;
 
-	down_write(&OCFS2_I(lqinode)->ip_alloc_sem);
+	if (!down_write_trylock(&OCFS2_I(lqinode)->ip_alloc_sem))
+		return -EBUSY;
+
 	chunk = ocfs2_find_free_entry(sb, type, &offset);
 	if (!chunk) {
 		chunk = ocfs2_extend_local_quota_file(sb, type, &offset);
diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c
index c1cdececdfa4e..8eee5be4d1ed4 100644
--- a/fs/ocfs2/refcounttree.c
+++ b/fs/ocfs2/refcounttree.c
@@ -31,7 +31,6 @@
 #include <linux/blkdev.h>
 #include <linux/slab.h>
 #include <linux/writeback.h>
-#include <linux/pagevec.h>
 #include <linux/swap.h>
 #include <linux/security.h>
 #include <linux/string.h>
@@ -2341,7 +2340,7 @@ static int ocfs2_mark_extent_refcounted(struct inode *inode,
 					   cpos, len, phys);
 
 	if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) {
-		ret = ocfs2_error(inode->i_sb, "Inode %lu want to use refcount tree, but the feature bit is not set in the super block\n",
+		ret = ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n",
 				  inode->i_ino);
 		goto out;
 	}
@@ -2524,7 +2523,7 @@ int ocfs2_prepare_refcount_change_for_del(struct inode *inode,
 	u64 start_cpos = ocfs2_blocks_to_clusters(inode->i_sb, phys_blkno);
 
 	if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) {
-		ret = ocfs2_error(inode->i_sb, "Inode %lu want to use refcount tree, but the feature bit is not set in the super block\n",
+		ret = ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n",
 				  inode->i_ino);
 		goto out;
 	}
@@ -2650,7 +2649,7 @@ static int ocfs2_refcount_cal_cow_clusters(struct inode *inode,
 
 		if (el->l_tree_depth) {
 			ret = ocfs2_error(inode->i_sb,
-					  "Inode %lu has non zero tree depth in leaf block %llu\n",
+					  "Inode %llu has non zero tree depth in leaf block %llu\n",
 					  inode->i_ino,
 					  (unsigned long long)eb_bh->b_blocknr);
 			goto out;
@@ -2662,7 +2661,7 @@ static int ocfs2_refcount_cal_cow_clusters(struct inode *inode,
 		rec = &el->l_recs[i];
 
 		if (ocfs2_is_empty_extent(rec)) {
-			mlog_bug_on_msg(i != 0, "Inode %lu has empty record in "
+			mlog_bug_on_msg(i != 0, "Inode %llu has empty record in "
 					"index %d\n", inode->i_ino, i);
 			continue;
 		}
@@ -3325,7 +3324,7 @@ static int ocfs2_replace_cow(struct ocfs2_cow_context *context)
 	struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
 
 	if (!ocfs2_refcount_tree(osb)) {
-		return ocfs2_error(inode->i_sb, "Inode %lu want to use refcount tree, but the feature bit is not set in the super block\n",
+		return ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n",
 				   inode->i_ino);
 	}
 
diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c
index 42ee5db362d3e..4d55ad963ac51 100644
--- a/fs/ocfs2/xattr.c
+++ b/fs/ocfs2/xattr.c
@@ -3741,7 +3741,7 @@ static int ocfs2_xattr_get_rec(struct inode *inode,
 
 		if (el->l_tree_depth) {
 			ret = ocfs2_error(inode->i_sb,
-					  "Inode %lu has non zero tree depth in xattr tree block %llu\n",
+					  "Inode %llu has non zero tree depth in xattr tree block %llu\n",
 					  inode->i_ino,
 					  (unsigned long long)eb_bh->b_blocknr);
 			goto out;
@@ -3758,7 +3758,7 @@ static int ocfs2_xattr_get_rec(struct inode *inode,
 	}
 
 	if (!e_blkno) {
-		ret = ocfs2_error(inode->i_sb, "Inode %lu has bad extent record (%u, %u, 0) in xattr\n",
+		ret = ocfs2_error(inode->i_sb, "Inode %llu has bad extent record (%u, %u, 0) in xattr\n",
 				  inode->i_ino,
 				  le32_to_cpu(rec->e_cpos),
 				  ocfs2_rec_clusters(el, rec));
diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c
index 2d4710d0e05e1..9e8a2a9e52298 100644
--- a/fs/orangefs/inode.c
+++ b/fs/orangefs/inode.c
@@ -1062,7 +1062,7 @@ struct inode *orangefs_iget(struct super_block *sb,
 	unlock_new_inode(inode);
 
 	gossip_debug(GOSSIP_INODE_DEBUG,
-		     "iget handle %pU, fsid %d hash %ld i_ino %lu\n",
+		     "iget handle %pU, fsid %d hash %ld i_ino %llu\n",
 		     &ref->khandle,
 		     ref->fs_id,
 		     hash,
diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
index ff3dbd1ca61f2..5fd32ccc134d2 100644
--- a/fs/overlayfs/dir.c
+++ b/fs/overlayfs/dir.c
@@ -66,10 +66,9 @@ void ovl_tempname(char name[OVL_TEMPNAME_SIZE])
 }
 
 static struct dentry *ovl_start_creating_temp(struct ovl_fs *ofs,
-					      struct dentry *workdir)
+					      struct dentry *workdir,
+					      char name[OVL_TEMPNAME_SIZE])
 {
-	char name[OVL_TEMPNAME_SIZE];
-
 	ovl_tempname(name);
 	return start_creating(ovl_upper_mnt_idmap(ofs), workdir,
 			      &QSTR(name));
@@ -81,11 +80,12 @@ static struct dentry *ovl_whiteout(struct ovl_fs *ofs)
 	struct dentry *whiteout, *link;
 	struct dentry *workdir = ofs->workdir;
 	struct inode *wdir = workdir->d_inode;
+	char name[OVL_TEMPNAME_SIZE];
 
 	guard(mutex)(&ofs->whiteout_lock);
 
 	if (!ofs->whiteout) {
-		whiteout = ovl_start_creating_temp(ofs, workdir);
+		whiteout = ovl_start_creating_temp(ofs, workdir, name);
 		if (IS_ERR(whiteout))
 			return whiteout;
 		err = ovl_do_whiteout(ofs, wdir, whiteout);
@@ -97,7 +97,7 @@ static struct dentry *ovl_whiteout(struct ovl_fs *ofs)
 	}
 
 	if (!ofs->no_shared_whiteout) {
-		link = ovl_start_creating_temp(ofs, workdir);
+		link = ovl_start_creating_temp(ofs, workdir, name);
 		if (IS_ERR(link))
 			return link;
 		err = ovl_do_link(ofs, ofs->whiteout, wdir, link);
@@ -159,7 +159,8 @@ int ovl_cleanup_and_whiteout(struct ovl_fs *ofs, struct dentry *dir,
 }
 
 struct dentry *ovl_create_real(struct ovl_fs *ofs, struct dentry *parent,
-			       struct dentry *newdentry, struct ovl_cattr *attr)
+			       struct dentry *newdentry, struct qstr *qname,
+			       struct ovl_cattr *attr)
 {
 	struct inode *dir = parent->d_inode;
 	int err;
@@ -221,19 +222,30 @@ struct dentry *ovl_create_real(struct ovl_fs *ofs, struct dentry *parent,
 		struct dentry *d;
 		/*
 		 * Some filesystems (i.e. casefolded) may return an unhashed
-		 * negative dentry from the ovl_lookup_upper() call before
+		 * negative dentry from the ovl_start_creating_upper() call before
 		 * ovl_create_real().
 		 * In that case, lookup again after making the newdentry
 		 * positive, so ovl_create_upper() always returns a hashed
-		 * positive dentry.
+		 * positive dentry.  We lookup using qname which should be
+		 * the same name as newentry, but is certain not to change.
+		 * As we have to drop the lock before the lookup a race
+		 * could result in a lookup failure.  In that case we return
+		 * an error.
 		 */
-		d = ovl_lookup_upper(ofs, newdentry->d_name.name, parent,
-				     newdentry->d_name.len);
-		dput(newdentry);
-		if (IS_ERR_OR_NULL(d))
+		end_creating_keep(newdentry);
+		d = ovl_start_creating_upper(ofs, parent, qname);
+
+		if (IS_ERR_OR_NULL(d)) {
 			err = d ? PTR_ERR(d) : -ENOENT;
-		else
+		} else if (d->d_inode != newdentry->d_inode) {
+			err = -EIO;
+		} else {
+			dput(newdentry);
 			return d;
+		}
+		end_creating(d);
+		dput(newdentry);
+		return ERR_PTR(err);
 	}
 out:
 	if (err) {
@@ -247,10 +259,12 @@ struct dentry *ovl_create_temp(struct ovl_fs *ofs, struct dentry *workdir,
 			       struct ovl_cattr *attr)
 {
 	struct dentry *ret;
-	ret = ovl_start_creating_temp(ofs, workdir);
+	char name[OVL_TEMPNAME_SIZE];
+
+	ret = ovl_start_creating_temp(ofs, workdir, name);
 	if (IS_ERR(ret))
 		return ret;
-	ret = ovl_create_real(ofs, workdir, ret, attr);
+	ret = ovl_create_real(ofs, workdir, ret, &QSTR(name), attr);
 	return end_creating_keep(ret);
 }
 
@@ -350,14 +364,15 @@ static int ovl_create_upper(struct dentry *dentry, struct inode *inode,
 	struct ovl_fs *ofs = OVL_FS(dentry->d_sb);
 	struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent);
 	struct dentry *newdentry;
+	struct qstr qname = QSTR_LEN(dentry->d_name.name,
+				     dentry->d_name.len);
 	int err;
 
 	newdentry = ovl_start_creating_upper(ofs, upperdir,
-					     &QSTR_LEN(dentry->d_name.name,
-						       dentry->d_name.len));
+					     &qname);
 	if (IS_ERR(newdentry))
 		return PTR_ERR(newdentry);
-	newdentry = ovl_create_real(ofs, upperdir, newdentry, attr);
+	newdentry = ovl_create_real(ofs, upperdir, newdentry, &qname, attr);
 	if (IS_ERR(newdentry))
 		return PTR_ERR(newdentry);
 
@@ -1374,7 +1389,8 @@ static int ovl_create_tmpfile(struct file *file, struct dentry *dentry,
 				return PTR_ERR(cred);
 
 			ovl_path_upper(dentry->d_parent, &realparentpath);
-			realfile = backing_tmpfile_open(&file->f_path, flags, &realparentpath,
+			realfile = backing_tmpfile_open(&file->f_path, file->f_cred,
+							flags, &realparentpath,
 							mode, current_cred());
 			err = PTR_ERR_OR_ZERO(realfile);
 			pr_debug("tmpfile/open(%pd2, 0%o) = %i\n", realparentpath.dentry, mode, err);
diff --git a/fs/overlayfs/export.c b/fs/overlayfs/export.c
index 83f80fdb15674..9aae20c59608a 100644
--- a/fs/overlayfs/export.c
+++ b/fs/overlayfs/export.c
@@ -262,7 +262,7 @@ static int ovl_dentry_to_fid(struct ovl_fs *ofs, struct inode *inode,
 	return err;
 
 fail:
-	pr_warn_ratelimited("failed to encode file handle (ino=%lu, err=%i)\n",
+	pr_warn_ratelimited("failed to encode file handle (ino=%llu, err=%i)\n",
 			    inode->i_ino, err);
 	goto out;
 }
@@ -349,69 +349,64 @@ static struct dentry *ovl_dentry_real_at(struct dentry *dentry, int idx)
 	return NULL;
 }
 
-/*
- * Lookup a child overlay dentry to get a connected overlay dentry whose real
- * dentry is @real. If @real is on upper layer, we lookup a child overlay
- * dentry with the same name as the real dentry. Otherwise, we need to consult
- * index for lookup.
+/**
+ * ovl_lookup_real_one -  Lookup a child overlay dentry to get an overlay dentry whose real dentry is given
+ * @connected: parent overlay dentry
+ * @real: given child real dentry
+ * @layer: layer in which @real exists
+ *
+ *
+ * Lookup a child overlay dentry in @connected with the same name as the @real
+ * dentry.  Then check that the parent of the result is the real dentry for
+ * @connected, and @real is the real dentry for the result.
+ *
+ * Returns:
+ *   %-ECHILD if the parent of @real is no longer the real dentry for @connected.
+ *   %-ESTALE if @real is not the real dentry of the found dentry.
+ *   Otherwise the found dentry is returned.
  */
 static struct dentry *ovl_lookup_real_one(struct dentry *connected,
 					  struct dentry *real,
 					  const struct ovl_layer *layer)
 {
-	struct inode *dir = d_inode(connected);
-	struct dentry *this, *parent = NULL;
+	struct dentry *this;
 	struct name_snapshot name;
 	int err;
 
 	/*
-	 * Lookup child overlay dentry by real name. The dir mutex protects us
-	 * from racing with overlay rename. If the overlay dentry that is above
-	 * real has already been moved to a parent that is not under the
-	 * connected overlay dir, we return -ECHILD and restart the lookup of
-	 * connected real path from the top.
-	 */
-	inode_lock_nested(dir, I_MUTEX_PARENT);
-	err = -ECHILD;
-	parent = dget_parent(real);
-	if (ovl_dentry_real_at(connected, layer->idx) != parent)
-		goto fail;
-
-	/*
-	 * We also need to take a snapshot of real dentry name to protect us
+	 * We need to take a snapshot of real dentry name to protect us
 	 * from racing with underlying layer rename. In this case, we don't
 	 * care about returning ESTALE, only from dereferencing a free name
 	 * pointer because we hold no lock on the real dentry.
 	 */
 	take_dentry_name_snapshot(&name, real);
-	/*
-	 * No idmap handling here: it's an internal lookup.
-	 */
-	this = lookup_noperm(&name.name, connected);
+	this = lookup_noperm_unlocked(&name.name, connected);
 	release_dentry_name_snapshot(&name);
+
+	err = -ECHILD;
+	if (ovl_dentry_real_at(connected, layer->idx) != real->d_parent)
+		goto fail;
+
 	err = PTR_ERR(this);
-	if (IS_ERR(this)) {
+	if (IS_ERR(this))
 		goto fail;
-	} else if (!this || !this->d_inode) {
-		dput(this);
-		err = -ENOENT;
+
+	err = -ENOENT;
+	if (!this || !this->d_inode)
 		goto fail;
-	} else if (ovl_dentry_real_at(this, layer->idx) != real) {
-		dput(this);
-		err = -ESTALE;
+
+	err = -ESTALE;
+	if (ovl_dentry_real_at(this, layer->idx) != real)
 		goto fail;
-	}
 
-out:
-	dput(parent);
-	inode_unlock(dir);
 	return this;
 
 fail:
 	pr_warn_ratelimited("failed to lookup one by real (%pd2, layer=%d, connected=%pd2, err=%i)\n",
 			    real, layer->idx, connected, err);
-	this = ERR_PTR(err);
-	goto out;
+	if (!IS_ERR(this))
+		dput(this);
+	return ERR_PTR(err);
 }
 
 static struct dentry *ovl_lookup_real(struct super_block *sb,
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index 97bed2286030d..767c128407fcc 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -49,6 +49,7 @@ static struct file *ovl_open_realfile(const struct file *file,
 				flags &= ~O_NOATIME;
 
 			realfile = backing_file_open(file_user_path(file),
+						     file_user_cred(file),
 						     flags, realpath, current_cred());
 		}
 	}
diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c
index d8dd4b0529843..ca899fdfaafd2 100644
--- a/fs/overlayfs/namei.c
+++ b/fs/overlayfs/namei.c
@@ -591,7 +591,7 @@ int ovl_verify_origin_xattr(struct ovl_fs *ofs, struct dentry *dentry,
 
 fail:
 	inode = d_inode(real);
-	pr_warn_ratelimited("failed to verify %s (%pd2, ino=%lu, err=%i)\n",
+	pr_warn_ratelimited("failed to verify %s (%pd2, ino=%llu, err=%i)\n",
 			    is_upper ? "upper" : "origin", real,
 			    inode ? inode->i_ino : 0, err);
 	goto out;
@@ -831,7 +831,7 @@ struct dentry *ovl_lookup_index(struct ovl_fs *ofs, struct dentry *upper,
 			index = NULL;
 			goto out;
 		}
-		pr_warn_ratelimited("failed inode index lookup (ino=%lu, key=%.*s, err=%i);\n"
+		pr_warn_ratelimited("failed inode index lookup (ino=%llu, key=%.*s, err=%i);\n"
 				    "overlayfs: mount with '-o index=off' to disable inodes index.\n",
 				    d_inode(origin)->i_ino, name.len, name.name,
 				    err);
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index cad2055ebf185..6fb99c583c31d 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -406,13 +406,6 @@ static inline struct file *ovl_do_tmpfile(struct ovl_fs *ofs,
 	return file;
 }
 
-static inline struct dentry *ovl_lookup_upper(struct ovl_fs *ofs,
-					      const char *name,
-					      struct dentry *base, int len)
-{
-	return lookup_one(ovl_upper_mnt_idmap(ofs), &QSTR_LEN(name, len), base);
-}
-
 static inline struct dentry *ovl_lookup_upper_unlocked(struct ovl_fs *ofs,
 						       const char *name,
 						       struct dentry *base,
@@ -576,8 +569,6 @@ bool ovl_is_inuse(struct dentry *dentry);
 bool ovl_need_index(struct dentry *dentry);
 int ovl_nlink_start(struct dentry *dentry);
 void ovl_nlink_end(struct dentry *dentry);
-int ovl_lock_rename_workdir(struct dentry *workdir, struct dentry *work,
-			    struct dentry *upperdir, struct dentry *upper);
 int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path,
 			     struct ovl_metacopy *data);
 int ovl_set_metacopy_xattr(struct ovl_fs *ofs, struct dentry *d,
@@ -888,6 +879,7 @@ struct ovl_cattr {
 
 struct dentry *ovl_create_real(struct ovl_fs *ofs,
 			       struct dentry *parent, struct dentry *newdentry,
+			       struct qstr *qname,
 			       struct ovl_cattr *attr);
 int ovl_cleanup(struct ovl_fs *ofs, struct dentry *workdir, struct dentry *dentry);
 #define OVL_TEMPNAME_SIZE 20
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index d4c12feec0392..58adefb1c5b8a 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -451,18 +451,13 @@ static int ovl_lower_dir(const char *name, const struct path *path,
 	return 0;
 }
 
-/* Workdir should not be subdir of upperdir and vice versa */
+/*
+ * Workdir should not be subdir of upperdir and vice versa, and
+ * they should not be the same.
+ */
 static bool ovl_workdir_ok(struct dentry *workdir, struct dentry *upperdir)
 {
-	bool ok = false;
-
-	if (workdir != upperdir) {
-		struct dentry *trap = lock_rename(workdir, upperdir);
-		if (!IS_ERR(trap))
-			unlock_rename(workdir, upperdir);
-		ok = (trap == NULL);
-	}
-	return ok;
+	return !is_subdir(workdir, upperdir) && !is_subdir(upperdir, workdir);
 }
 
 static int ovl_setup_trap(struct super_block *sb, struct dentry *dir,
@@ -634,6 +629,7 @@ static struct dentry *ovl_lookup_or_create(struct ovl_fs *ofs,
 	if (!IS_ERR(child)) {
 		if (!child->d_inode)
 			child = ovl_create_real(ofs, parent, child,
+						&QSTR(name),
 						OVL_CATTR(mode));
 		end_creating_keep(child);
 	}
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index 3f1b763a8bb4c..e364351986f04 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -1092,7 +1092,7 @@ static void ovl_cleanup_index(struct dentry *dentry)
 	got_write = true;
 	inode = d_inode(upperdentry);
 	if (!S_ISDIR(inode->i_mode) && inode->i_nlink != 1) {
-		pr_warn_ratelimited("cleanup linked index (%pd2, ino=%lu, nlink=%u)\n",
+		pr_warn_ratelimited("cleanup linked index (%pd2, ino=%llu, nlink=%u)\n",
 				    upperdentry, inode->i_ino, inode->i_nlink);
 		/*
 		 * We either have a bug with persistent union nlink or a lower
@@ -1213,31 +1213,6 @@ void ovl_nlink_end(struct dentry *dentry)
 	ovl_inode_unlock(inode);
 }
 
-int ovl_lock_rename_workdir(struct dentry *workdir, struct dentry *work,
-			    struct dentry *upperdir, struct dentry *upper)
-{
-	struct dentry *trap;
-
-	/* Workdir should not be subdir of upperdir and vice versa */
-	trap = lock_rename(workdir, upperdir);
-	if (IS_ERR(trap))
-		goto err;
-	if (trap)
-		goto err_unlock;
-	if (work && (work->d_parent != workdir || d_unhashed(work)))
-		goto err_unlock;
-	if (upper && (upper->d_parent != upperdir || d_unhashed(upper)))
-		goto err_unlock;
-
-	return 0;
-
-err_unlock:
-	unlock_rename(workdir, upperdir);
-err:
-	pr_err("failed to lock workdir+upperdir\n");
-	return -EIO;
-}
-
 /*
  * err < 0, 0 if no metacopy xattr, metacopy data size if xattr found.
  * an empty xattr returns OVL_METACOPY_MIN_SIZE to distinguish from no xattr value.
diff --git a/fs/pidfs.c b/fs/pidfs.c
index e3825ee246be3..68a9405edfdbc 100644
--- a/fs/pidfs.c
+++ b/fs/pidfs.c
@@ -8,6 +8,8 @@
 #include <linux/mount.h>
 #include <linux/pid.h>
 #include <linux/pidfs.h>
+#include <linux/sched/signal.h>
+#include <linux/signal.h>
 #include <linux/pid_namespace.h>
 #include <linux/poll.h>
 #include <linux/proc_fs.h>
@@ -22,6 +24,7 @@
 #include <net/net_namespace.h>
 #include <linux/coredump.h>
 #include <linux/rhashtable.h>
+#include <linux/llist.h>
 #include <linux/xattr.h>
 #include <linux/cookie.h>
 
@@ -31,7 +34,6 @@
 #define PIDFS_PID_DEAD ERR_PTR(-ESRCH)
 
 static struct kmem_cache *pidfs_attr_cachep __ro_after_init;
-static struct kmem_cache *pidfs_xattr_cachep __ro_after_init;
 
 static struct path pidfs_root_path = {};
 
@@ -46,9 +48,8 @@ enum pidfs_attr_mask_bits {
 	PIDFS_ATTR_BIT_COREDUMP	= 1,
 };
 
-struct pidfs_attr {
+struct pidfs_anon_attr {
 	unsigned long attr_mask;
-	struct simple_xattrs *xattrs;
 	struct /* exit info */ {
 		__u64 cgroupid;
 		__s32 exit_code;
@@ -93,6 +94,13 @@ static const struct rhashtable_params pidfs_ino_ht_params = {
  * inode number and the inode generation number to compare or
  * use file handles.
  */
+struct pidfs_attr {
+	struct simple_xattrs *xattrs;
+	union {
+		struct pidfs_anon_attr;
+		struct llist_node pidfs_llist;
+	};
+};
 
 #if BITS_PER_LONG == 32
 
@@ -178,10 +186,30 @@ void pidfs_remove_pid(struct pid *pid)
 				       pidfs_ino_ht_params);
 }
 
+static LLIST_HEAD(pidfs_free_list);
+
+static void pidfs_free_attr_work(struct work_struct *work)
+{
+	struct pidfs_attr *attr, *next;
+	struct llist_node *head;
+
+	head = llist_del_all(&pidfs_free_list);
+	llist_for_each_entry_safe(attr, next, head, pidfs_llist) {
+		struct simple_xattrs *xattrs = attr->xattrs;
+
+		if (xattrs) {
+			simple_xattrs_free(xattrs, NULL);
+			kfree(xattrs);
+		}
+		kfree(attr);
+	}
+}
+
+static DECLARE_WORK(pidfs_free_work, pidfs_free_attr_work);
+
 void pidfs_free_pid(struct pid *pid)
 {
-	struct pidfs_attr *attr __free(kfree) = no_free_ptr(pid->attr);
-	struct simple_xattrs *xattrs __free(kfree) = NULL;
+	struct pidfs_attr *attr = pid->attr;
 
 	/*
 	 * Any dentry must've been wiped from the pid by now.
@@ -200,9 +228,10 @@ void pidfs_free_pid(struct pid *pid)
 	if (IS_ERR(attr))
 		return;
 
-	xattrs = no_free_ptr(attr->xattrs);
-	if (xattrs)
-		simple_xattrs_free(xattrs, NULL);
+	if (likely(!attr->xattrs))
+		kfree(attr);
+	else if (llist_add(&attr->pidfs_llist, &pidfs_free_list))
+		schedule_work(&pidfs_free_work);
 }
 
 #ifdef CONFIG_PROC_FS
@@ -635,7 +664,28 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	return open_namespace(ns_common);
 }
 
+static int pidfs_file_release(struct inode *inode, struct file *file)
+{
+	struct pid *pid = inode->i_private;
+	struct task_struct *task;
+
+	if (!(file->f_flags & PIDFD_AUTOKILL))
+		return 0;
+
+	guard(rcu)();
+	task = pid_task(pid, PIDTYPE_TGID);
+	if (!task)
+		return 0;
+
+	/* Not available for kthreads or user workers for now. */
+	if (WARN_ON_ONCE(task->flags & (PF_KTHREAD | PF_USER_WORKER)))
+		return 0;
+	do_send_sig_info(SIGKILL, SEND_SIG_PRIV, task, PIDTYPE_TGID);
+	return 0;
+}
+
 static const struct file_operations pidfs_file_operations = {
+	.release	= pidfs_file_release,
 	.poll		= pidfd_poll,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo	= pidfd_show_fdinfo,
@@ -1009,7 +1059,7 @@ static int pidfs_xattr_get(const struct xattr_handler *handler,
 
 	xattrs = READ_ONCE(attr->xattrs);
 	if (!xattrs)
-		return 0;
+		return -ENODATA;
 
 	name = xattr_full_name(handler, suffix);
 	return simple_xattr_get(xattrs, name, value, size);
@@ -1029,22 +1079,16 @@ static int pidfs_xattr_set(const struct xattr_handler *handler,
 	/* Ensure we're the only one to set @attr->xattrs. */
 	WARN_ON_ONCE(!inode_is_locked(inode));
 
-	xattrs = READ_ONCE(attr->xattrs);
-	if (!xattrs) {
-		xattrs = kmem_cache_zalloc(pidfs_xattr_cachep, GFP_KERNEL);
-		if (!xattrs)
-			return -ENOMEM;
-
-		simple_xattrs_init(xattrs);
-		smp_store_release(&pid->attr->xattrs, xattrs);
-	}
+	xattrs = simple_xattrs_lazy_alloc(&attr->xattrs, value, flags);
+	if (IS_ERR_OR_NULL(xattrs))
+		return PTR_ERR(xattrs);
 
 	name = xattr_full_name(handler, suffix);
 	old_xattr = simple_xattr_set(xattrs, name, value, size, flags);
 	if (IS_ERR(old_xattr))
 		return PTR_ERR(old_xattr);
 
-	simple_xattr_free(old_xattr);
+	simple_xattr_free_rcu(old_xattr);
 	return 0;
 }
 
@@ -1091,11 +1135,11 @@ struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags)
 	int ret;
 
 	/*
-	 * Ensure that PIDFD_STALE can be passed as a flag without
-	 * overloading other uapi pidfd flags.
+	 * Ensure that internal pidfd flags don't overlap with each
+	 * other or with uapi pidfd flags.
 	 */
-	BUILD_BUG_ON(PIDFD_STALE == PIDFD_THREAD);
-	BUILD_BUG_ON(PIDFD_STALE == PIDFD_NONBLOCK);
+	BUILD_BUG_ON(hweight32(PIDFD_THREAD | PIDFD_NONBLOCK |
+				PIDFD_STALE | PIDFD_AUTOKILL) != 4);
 
 	ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path);
 	if (ret < 0)
@@ -1106,9 +1150,12 @@ struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags)
 	flags &= ~PIDFD_STALE;
 	flags |= O_RDWR;
 	pidfd_file = dentry_open(&path, flags, current_cred());
-	/* Raise PIDFD_THREAD explicitly as do_dentry_open() strips it. */
+	/*
+	 * Raise PIDFD_THREAD and PIDFD_AUTOKILL explicitly as
+	 * do_dentry_open() strips O_EXCL and O_TRUNC.
+	 */
 	if (!IS_ERR(pidfd_file))
-		pidfd_file->f_flags |= (flags & PIDFD_THREAD);
+		pidfd_file->f_flags |= (flags & (PIDFD_THREAD | PIDFD_AUTOKILL));
 
 	return pidfd_file;
 }
@@ -1122,11 +1169,6 @@ void __init pidfs_init(void)
 					 (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT |
 					  SLAB_ACCOUNT | SLAB_PANIC), NULL);
 
-	pidfs_xattr_cachep = kmem_cache_create("pidfs_xattr_cache",
-					       sizeof(struct simple_xattrs), 0,
-					       (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT |
-						SLAB_ACCOUNT | SLAB_PANIC), NULL);
-
 	pidfs_mnt = kern_mount(&pidfs_type);
 	if (IS_ERR(pidfs_mnt))
 		panic("Failed to mount pidfs pseudo filesystem");
diff --git a/fs/pipe.c b/fs/pipe.c
index b44a756c0b416..9841648c9cf3e 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -873,7 +873,7 @@ static struct vfsmount *pipe_mnt __ro_after_init;
  */
 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
 {
-	return dynamic_dname(buffer, buflen, "pipe:[%lu]",
+	return dynamic_dname(buffer, buflen, "pipe:[%llu]",
 				d_inode(dentry)->i_ino);
 }
 
diff --git a/fs/proc/array.c b/fs/proc/array.c
index f447e734612aa..90fb0c6b5f99f 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -280,7 +280,7 @@ static inline void task_sig(struct seq_file *m, struct task_struct *p)
 		blocked = p->blocked;
 		collect_sigign_sigcatch(p, &ignored, &caught);
 		num_threads = get_nr_threads(p);
-		rcu_read_lock();  /* FIXME: is this correct? */
+		rcu_read_lock();
 		qsize = get_rlimit_value(task_ucounts(p), UCOUNT_RLIMIT_SIGPENDING);
 		rcu_read_unlock();
 		qlim = task_rlimit(p, RLIMIT_SIGPENDING);
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 4c863d17dfb4c..b5496cec888e5 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -1083,14 +1083,20 @@ static ssize_t auxv_read(struct file *file, char __user *buf,
 {
 	struct mm_struct *mm = file->private_data;
 	unsigned int nwords = 0;
+	unsigned long saved_auxv[AT_VECTOR_SIZE];
 
 	if (!mm)
 		return 0;
+
+	spin_lock(&mm->arg_lock);
+	memcpy(saved_auxv, mm->saved_auxv, sizeof(saved_auxv));
+	spin_unlock(&mm->arg_lock);
+
 	do {
 		nwords += 2;
-	} while (mm->saved_auxv[nwords - 2] != 0); /* AT_NULL */
-	return simple_read_from_buffer(buf, count, ppos, mm->saved_auxv,
-				       nwords * sizeof(mm->saved_auxv[0]));
+	} while (saved_auxv[nwords - 2] != 0); /* AT_NULL */
+	return simple_read_from_buffer(buf, count, ppos, saved_auxv,
+				       nwords * sizeof(saved_auxv[0]));
 }
 
 static const struct file_operations proc_auxv_operations = {
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index 9eeccff49b2ab..aae1a83e88468 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c
@@ -54,7 +54,7 @@ static int seq_show(struct seq_file *m, void *v)
 	if (ret)
 		return ret;
 
-	seq_printf(m, "pos:\t%lli\nflags:\t0%o\nmnt_id:\t%i\nino:\t%lu\n",
+	seq_printf(m, "pos:\t%lli\nflags:\t0%o\nmnt_id:\t%i\nino:\t%llu\n",
 		   (long long)file->f_pos, f_flags,
 		   real_mount(file->f_path.mnt)->mnt_id,
 		   file_inode(file)->i_ino);
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
index a458f1e112fdb..549793f447264 100644
--- a/fs/proc/meminfo.c
+++ b/fs/proc/meminfo.c
@@ -126,7 +126,8 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "Committed_AS:   ", committed);
 	seq_printf(m, "VmallocTotal:   %8lu kB\n",
 		   (unsigned long)VMALLOC_TOTAL >> 10);
-	show_val_kb(m, "VmallocUsed:    ", vmalloc_nr_pages());
+	show_val_kb(m, "VmallocUsed:    ",
+		    global_node_page_state(NR_VMALLOC));
 	show_val_kb(m, "VmallocChunk:   ", 0ul);
 	show_val_kb(m, "Percpu:         ", pcpu_nr_pages());
 
diff --git a/fs/proc/self.c b/fs/proc/self.c
index 62d2c0cfe35c5..56adf1c68f7ac 100644
--- a/fs/proc/self.c
+++ b/fs/proc/self.c
@@ -35,11 +35,9 @@ unsigned self_inum __ro_after_init;
 
 int proc_setup_self(struct super_block *s)
 {
-	struct inode *root_inode = d_inode(s->s_root);
 	struct dentry *self;
 	int ret = -ENOMEM;
 
-	inode_lock(root_inode);
 	self = d_alloc_name(s->s_root, "self");
 	if (self) {
 		struct inode *inode = new_inode(s);
@@ -55,7 +53,6 @@ int proc_setup_self(struct super_block *s)
 		}
 		dput(self);
 	}
-	inode_unlock(root_inode);
 
 	if (ret)
 		pr_err("proc_fill_super: can't allocate /proc/self\n");
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index e091931d7ca19..751b9ba160fbf 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -442,7 +442,7 @@ static void get_vma_name(struct vm_area_struct *vma,
 static void show_vma_header_prefix(struct seq_file *m,
 				   unsigned long start, unsigned long end,
 				   vm_flags_t flags, unsigned long long pgoff,
-				   dev_t dev, unsigned long ino)
+				   dev_t dev, u64 ino)
 {
 	seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
 	seq_put_hex_ll(m, NULL, start, 8);
@@ -465,7 +465,7 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
 	const struct path *path;
 	const char *name_fmt, *name;
 	vm_flags_t flags = vma->vm_flags;
-	unsigned long ino = 0;
+	u64 ino = 0;
 	unsigned long long pgoff = 0;
 	unsigned long start, end;
 	dev_t dev = 0;
diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c
index d6113dbe58e05..61ac62c3fd9f8 100644
--- a/fs/proc/thread_self.c
+++ b/fs/proc/thread_self.c
@@ -35,11 +35,9 @@ unsigned thread_self_inum __ro_after_init;
 
 int proc_setup_thread_self(struct super_block *s)
 {
-	struct inode *root_inode = d_inode(s->s_root);
 	struct dentry *thread_self;
 	int ret = -ENOMEM;
 
-	inode_lock(root_inode);
 	thread_self = d_alloc_name(s->s_root, "thread-self");
 	if (thread_self) {
 		struct inode *inode = new_inode(s);
@@ -55,7 +53,6 @@ int proc_setup_thread_self(struct super_block *s)
 		}
 		dput(thread_self);
 	}
-	inode_unlock(root_inode);
 
 	if (ret)
 		pr_err("proc_fill_super: can't allocate /proc/thread-self\n");
diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c
index f188bd900eb2d..44d15436439fd 100644
--- a/fs/proc/vmcore.c
+++ b/fs/proc/vmcore.c
@@ -525,7 +525,7 @@ static int remap_oldmem_pfn_checked(struct vm_area_struct *vma,
 {
 	unsigned long map_size;
 	unsigned long pos_start, pos_end, pos;
-	unsigned long zeropage_pfn = my_zero_pfn(0);
+	unsigned long zeropage_pfn = zero_pfn(0);
 	size_t len = 0;
 
 	pos_start = pfn;
diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c
index 83fa0bb3435ac..d0ca91040591c 100644
--- a/fs/pstore/inode.c
+++ b/fs/pstore/inode.c
@@ -74,9 +74,9 @@ static void *pstore_ftrace_seq_start(struct seq_file *s, loff_t *pos)
 	if (!data)
 		return NULL;
 
-	data->off = ps->total_size % REC_SIZE;
+	data->off = ps->record->size % REC_SIZE;
 	data->off += *pos * REC_SIZE;
-	if (data->off + REC_SIZE > ps->total_size)
+	if (data->off + REC_SIZE > ps->record->size)
 		return NULL;
 
 	return_ptr(data);
@@ -94,7 +94,7 @@ static void *pstore_ftrace_seq_next(struct seq_file *s, void *v, loff_t *pos)
 
 	(*pos)++;
 	data->off += REC_SIZE;
-	if (data->off + REC_SIZE > ps->total_size)
+	if (data->off + REC_SIZE > ps->record->size)
 		return NULL;
 
 	return data;
diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c
index d1f895e57a95a..2eb0d4fa21869 100644
--- a/fs/pstore/ram.c
+++ b/fs/pstore/ram.c
@@ -71,7 +71,7 @@ MODULE_PARM_DESC(max_reason,
 
 static int ramoops_ecc;
 module_param_named(ecc, ramoops_ecc, int, 0400);
-MODULE_PARM_DESC(ramoops_ecc,
+MODULE_PARM_DESC(ecc,
 		"if non-zero, the option enables ECC support and specifies "
 		"ECC buffer size in bytes (1 is a special value, means 16 "
 		"bytes ECC)");
diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c
index ed97494abf60f..05048c6f787aa 100644
--- a/fs/pstore/ram_core.c
+++ b/fs/pstore/ram_core.c
@@ -12,7 +12,6 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
-#include <linux/memblock.h>
 #include <linux/rslib.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
@@ -488,6 +487,10 @@ static void *persistent_ram_iomap(phys_addr_t start, size_t size,
 	else
 		va = ioremap_wc(start, size);
 
+	/* We must release the mem region if ioremap fails. */
+	if (!va)
+		release_mem_region(start, size);
+
 	/*
 	 * Since request_mem_region() and ioremap() are byte-granularity
 	 * there is no need handle anything special like we do when the
diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c
index 8aeb63d397cfd..4deb0eeadbdef 100644
--- a/fs/qnx4/inode.c
+++ b/fs/qnx4/inode.c
@@ -62,7 +62,7 @@ static int qnx4_get_block( struct inode *inode, sector_t iblock, struct buffer_h
 {
 	unsigned long phys;
 
-	QNX4DEBUG((KERN_INFO "qnx4: qnx4_get_block inode=[%ld] iblock=[%ld]\n",inode->i_ino,iblock));
+	QNX4DEBUG((KERN_INFO "qnx4: qnx4_get_block inode=[%llu] iblock=[%ld]\n", inode->i_ino, iblock));
 
 	phys = qnx4_block_map( inode, iblock );
 	if ( phys ) {
@@ -128,7 +128,7 @@ unsigned long qnx4_block_map( struct inode *inode, long iblock )
 			brelse( bh );
 	}
 
-	QNX4DEBUG((KERN_INFO "qnx4: mapping block %ld of inode %ld = %ld\n",iblock,inode->i_ino,block));
+	QNX4DEBUG((KERN_INFO "qnx4: mapping block %ld of inode %llu = %ld\n", iblock, inode->i_ino, block));
 	return block;
 }
 
diff --git a/fs/qnx6/inode.c b/fs/qnx6/inode.c
index c4049bb8bd60d..6de49333acadb 100644
--- a/fs/qnx6/inode.c
+++ b/fs/qnx6/inode.c
@@ -75,7 +75,7 @@ static int qnx6_get_block(struct inode *inode, sector_t iblock,
 {
 	unsigned phys;
 
-	pr_debug("qnx6_get_block inode=[%ld] iblock=[%ld]\n",
+	pr_debug("qnx6_get_block inode=[%llu] iblock=[%ld]\n",
 		 inode->i_ino, (unsigned long)iblock);
 
 	phys = qnx6_block_map(inode, iblock);
diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c
index 0f8e838ece078..2f79bcb89d2e5 100644
--- a/fs/ramfs/file-nommu.c
+++ b/fs/ramfs/file-nommu.c
@@ -14,7 +14,7 @@
 #include <linux/string.h>
 #include <linux/backing-dev.h>
 #include <linux/ramfs.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/mman.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
diff --git a/fs/resctrl/pseudo_lock.c b/fs/resctrl/pseudo_lock.c
index fa3687d69ebd0..d1cb0986006eb 100644
--- a/fs/resctrl/pseudo_lock.c
+++ b/fs/resctrl/pseudo_lock.c
@@ -1044,7 +1044,7 @@ static int pseudo_lock_dev_mmap_prepare(struct vm_area_desc *desc)
 	 * Ensure changes are carried directly to the memory being mapped,
 	 * do not allow copy-on-write mapping.
 	 */
-	if (!vma_desc_test_flags(desc, VMA_SHARED_BIT)) {
+	if (!vma_desc_test(desc, VMA_SHARED_BIT)) {
 		mutex_unlock(&rdtgroup_mutex);
 		return -EINVAL;
 	}
diff --git a/fs/smb/Kconfig b/fs/smb/Kconfig
index 85f7ad5fbc5e7..b4b2cfdc2a6bf 100644
--- a/fs/smb/Kconfig
+++ b/fs/smb/Kconfig
@@ -4,6 +4,7 @@
 
 source "fs/smb/client/Kconfig"
 source "fs/smb/server/Kconfig"
+source "fs/smb/common/smbdirect/Kconfig"
 
 config SMBFS
 	tristate
diff --git a/fs/smb/client/Kconfig b/fs/smb/client/Kconfig
index 17bd368574e94..981b3b5be09b3 100644
--- a/fs/smb/client/Kconfig
+++ b/fs/smb/client/Kconfig
@@ -181,7 +181,9 @@ if CIFS
 
 config CIFS_SMB_DIRECT
 	bool "SMB Direct support"
-	depends on CIFS=m && INFINIBAND && INFINIBAND_ADDR_TRANS || CIFS=y && INFINIBAND=y && INFINIBAND_ADDR_TRANS=y
+	depends on CIFS && INFINIBAND && INFINIBAND_ADDR_TRANS
+	depends on CIFS=m || INFINIBAND=y
+	select SMB_COMMON_SMBDIRECT
 	help
 	  Enables SMB Direct support for SMB 3.0, 3.02 and 3.1.1.
 	  SMB Direct allows transferring SMB packets over RDMA. If unsure,
diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c
index 217444e3e6d01..0691d2a3e04b0 100644
--- a/fs/smb/client/cifs_debug.c
+++ b/fs/smb/client/cifs_debug.c
@@ -23,7 +23,6 @@
 #endif
 #ifdef CONFIG_CIFS_SMB_DIRECT
 #include "smbdirect.h"
-#include "../common/smbdirect/smbdirect_pdu.h"
 #endif
 #include "cifs_swn.h"
 #include "cached_dir.h"
@@ -452,11 +451,6 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
 	c = 0;
 	spin_lock(&cifs_tcp_ses_lock);
 	list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) {
-#ifdef CONFIG_CIFS_SMB_DIRECT
-		struct smbdirect_socket *sc;
-		struct smbdirect_socket_parameters *sp;
-#endif
-
 		/* channel info will be printed as a part of sessions below */
 		if (SERVER_IS_CHAN(server))
 			continue;
@@ -471,66 +465,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
 		seq_printf(m, "\nClientGUID: %pUL", server->client_guid);
 		spin_unlock(&server->srv_lock);
 #ifdef CONFIG_CIFS_SMB_DIRECT
-		if (!server->rdma)
-			goto skip_rdma;
-
-		if (!server->smbd_conn) {
-			seq_printf(m, "\nSMBDirect transport not available");
-			goto skip_rdma;
-		}
-		sc = &server->smbd_conn->socket;
-		sp = &sc->parameters;
-
-		seq_printf(m, "\nSMBDirect protocol version: 0x%x "
-			"transport status: %s (%u)",
-			SMBDIRECT_V1,
-			smbdirect_socket_status_string(sc->status),
-			sc->status);
-		seq_printf(m, "\nConn receive_credit_max: %u "
-			"send_credit_target: %u max_send_size: %u",
-			sp->recv_credit_max,
-			sp->send_credit_target,
-			sp->max_send_size);
-		seq_printf(m, "\nConn max_fragmented_recv_size: %u "
-			"max_fragmented_send_size: %u max_receive_size:%u",
-			sp->max_fragmented_recv_size,
-			sp->max_fragmented_send_size,
-			sp->max_recv_size);
-		seq_printf(m, "\nConn keep_alive_interval: %u "
-			"max_readwrite_size: %u rdma_readwrite_threshold: %u",
-			sp->keepalive_interval_msec * 1000,
-			sp->max_read_write_size,
-			server->rdma_readwrite_threshold);
-		seq_printf(m, "\nDebug count_get_receive_buffer: %llu "
-			"count_put_receive_buffer: %llu count_send_empty: %llu",
-			sc->statistics.get_receive_buffer,
-			sc->statistics.put_receive_buffer,
-			sc->statistics.send_empty);
-		seq_printf(m, "\nRead Queue "
-			"count_enqueue_reassembly_queue: %llu "
-			"count_dequeue_reassembly_queue: %llu "
-			"reassembly_data_length: %u "
-			"reassembly_queue_length: %u",
-			sc->statistics.enqueue_reassembly_queue,
-			sc->statistics.dequeue_reassembly_queue,
-			sc->recv_io.reassembly.data_length,
-			sc->recv_io.reassembly.queue_length);
-		seq_printf(m, "\nCurrent Credits send_credits: %u "
-			"receive_credits: %u receive_credit_target: %u",
-			atomic_read(&sc->send_io.credits.count),
-			atomic_read(&sc->recv_io.credits.count),
-			sc->recv_io.credits.target);
-		seq_printf(m, "\nPending send_pending: %u ",
-			atomic_read(&sc->send_io.pending.count));
-		seq_printf(m, "\nMR responder_resources: %u "
-			"max_frmr_depth: %u mr_type: 0x%x",
-			sp->responder_resources,
-			sp->max_frmr_depth,
-			sc->mr_io.type);
-		seq_printf(m, "\nMR mr_ready_count: %u mr_used_count: %u",
-			atomic_read(&sc->mr_io.ready.count),
-			atomic_read(&sc->mr_io.used.count));
-skip_rdma:
+		smbd_debug_proc_show(server, m);
 #endif
 		seq_printf(m, "\nNumber of credits: %d,%d,%d Dialect 0x%x",
 			server->credits,
diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h
index 7877d327dbb03..709e96e077916 100644
--- a/fs/smb/client/cifsglob.h
+++ b/fs/smb/client/cifsglob.h
@@ -2386,4 +2386,10 @@ static inline int cifs_open_create_options(unsigned int oflags, int opts)
 	return opts;
 }
 
+/*
+ * The number of blocks is not related to (i_size / i_blksize), but instead
+ * 512 byte (2**9) size is required for calculating num blocks.
+ */
+#define CIFS_INO_BLOCKS(size) DIV_ROUND_UP_ULL((u64)(size), 512)
+
 #endif	/* _CIFS_GLOB_H */
diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c
index 3bad2c5c523db..ca1bc67eb23be 100644
--- a/fs/smb/client/connect.c
+++ b/fs/smb/client/connect.c
@@ -20,7 +20,6 @@
 #include <linux/delay.h>
 #include <linux/completion.h>
 #include <linux/kthread.h>
-#include <linux/pagevec.h>
 #include <linux/freezer.h>
 #include <linux/namei.h>
 #include <linux/uuid.h>
@@ -1955,6 +1954,10 @@ static int match_session(struct cifs_ses *ses,
 	case Kerberos:
 		if (!uid_eq(ctx->cred_uid, ses->cred_uid))
 			return 0;
+		if (strncmp(ses->user_name ?: "",
+			    ctx->username ?: "",
+			    CIFS_MAX_USERNAME_LEN))
+			return 0;
 		break;
 	case NTLMv2:
 	case RawNTLMSSP:
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 27f61fe7e4e28..148508e3a82fd 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -15,7 +15,6 @@
 #include <linux/stat.h>
 #include <linux/fcntl.h>
 #include <linux/pagemap.h>
-#include <linux/pagevec.h>
 #include <linux/writeback.h>
 #include <linux/task_io_accounting_ops.h>
 #include <linux/delay.h>
@@ -993,7 +992,6 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry)
 		if (!rc) {
 			netfs_resize_file(&cinode->netfs, 0, true);
 			cifs_setsize(inode, 0);
-			inode->i_blocks = 0;
 		}
 	}
 	if (cfile)
diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c
index 143fa2e665ed4..888f9e35f14b8 100644
--- a/fs/smb/client/inode.c
+++ b/fs/smb/client/inode.c
@@ -219,13 +219,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr,
 	 */
 	if (is_size_safe_to_change(cifs_i, fattr->cf_eof, from_readdir)) {
 		i_size_write(inode, fattr->cf_eof);
-
-		/*
-		 * i_blocks is not related to (i_size / i_blksize),
-		 * but instead 512 byte (2**9) size is required for
-		 * calculating num blocks.
-		 */
-		inode->i_blocks = (512 - 1 + fattr->cf_bytes) >> 9;
+		inode->i_blocks = CIFS_INO_BLOCKS(fattr->cf_bytes);
 	}
 
 	if (S_ISLNK(fattr->cf_mode) && fattr->cf_symlink_target) {
@@ -3015,6 +3009,11 @@ void cifs_setsize(struct inode *inode, loff_t offset)
 {
 	spin_lock(&inode->i_lock);
 	i_size_write(inode, offset);
+	/*
+	 * Until we can query the server for actual allocation size,
+	 * this is best estimate we have for blocks allocated for a file.
+	 */
+	inode->i_blocks = CIFS_INO_BLOCKS(offset);
 	spin_unlock(&inode->i_lock);
 	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
 	truncate_pagecache(inode, offset);
@@ -3087,14 +3086,6 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry,
 	if (rc == 0) {
 		netfs_resize_file(&cifsInode->netfs, size, true);
 		cifs_setsize(inode, size);
-		/*
-		 * i_blocks is not related to (i_size / i_blksize), but instead
-		 * 512 byte (2**9) size is required for calculating num blocks.
-		 * Until we can query the server for actual allocation size,
-		 * this is best estimate we have for blocks allocated for a file
-		 * Number of blocks must be rounded up so size 1 is not 0 blocks
-		 */
-		inode->i_blocks = (512 - 1 + size) >> 9;
 	}
 
 	return rc;
diff --git a/fs/smb/client/smb1transport.c b/fs/smb/client/smb1transport.c
index 38d6d5538b96b..53abb29fe71bd 100644
--- a/fs/smb/client/smb1transport.c
+++ b/fs/smb/client/smb1transport.c
@@ -460,7 +460,7 @@ check_smb_hdr(struct smb_hdr *smb)
 		return 0;
 
 	/*
-	 * Windows NT server returns error resposne (e.g. STATUS_DELETE_PENDING
+	 * Windows NT server returns error response (e.g. STATUS_DELETE_PENDING
 	 * or STATUS_OBJECT_NAME_NOT_FOUND or ERRDOS/ERRbadfile or any other)
 	 * for some TRANS2 requests without the RESPONSE flag set in header.
 	 */
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 98ac4e86bf997..509fcea28a429 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -1497,6 +1497,7 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon,
 {
 	struct smb2_file_network_open_info file_inf;
 	struct inode *inode;
+	u64 asize;
 	int rc;
 
 	rc = __SMB2_close(xid, tcon, cfile->fid.persistent_fid,
@@ -1520,14 +1521,9 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon,
 		inode_set_atime_to_ts(inode,
 				      cifs_NTtimeToUnix(file_inf.LastAccessTime));
 
-	/*
-	 * i_blocks is not related to (i_size / i_blksize),
-	 * but instead 512 byte (2**9) size is required for
-	 * calculating num blocks.
-	 */
-	if (le64_to_cpu(file_inf.AllocationSize) > 4096)
-		inode->i_blocks =
-			(512 - 1 + le64_to_cpu(file_inf.AllocationSize)) >> 9;
+	asize = le64_to_cpu(file_inf.AllocationSize);
+	if (asize > 4096)
+		inode->i_blocks = CIFS_INO_BLOCKS(asize);
 
 	/* End of file and Attributes should not have to be updated on close */
 	spin_unlock(&inode->i_lock);
@@ -2204,14 +2200,6 @@ smb2_duplicate_extents(const unsigned int xid,
 		rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false);
 		if (rc)
 			goto duplicate_extents_out;
-
-		/*
-		 * Although also could set plausible allocation size (i_blocks)
-		 * here in addition to setting the file size, in reflink
-		 * it is likely that the target file is sparse. Its allocation
-		 * size will be queried on next revalidate, but it is important
-		 * to make sure that file's cached size is updated immediately
-		 */
 		netfs_resize_file(netfs_inode(inode), dest_off + len, true);
 		cifs_setsize(inode, dest_off + len);
 	}
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index 5188218c25be4..ef8f16bbfbf56 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -36,7 +36,6 @@
 #include "../common/smb2status.h"
 #include "smb2glob.h"
 #include "cifs_spnego.h"
-#include "../common/smbdirect/smbdirect.h"
 #include "smbdirect.h"
 #include "trace.h"
 #ifdef CONFIG_CIFS_DFS_UPCALL
@@ -4554,9 +4553,7 @@ smb2_new_read_req(void **buf, unsigned int *total_len,
 		req->ReadChannelInfoLength =
 			cpu_to_le16(sizeof(struct smbdirect_buffer_descriptor_v1));
 		v1 = (struct smbdirect_buffer_descriptor_v1 *) &req->Buffer[0];
-		v1->offset = cpu_to_le64(rdata->mr->mr->iova);
-		v1->token = cpu_to_le32(rdata->mr->mr->rkey);
-		v1->length = cpu_to_le32(rdata->mr->mr->length);
+		smbd_mr_fill_buffer_descriptor(rdata->mr, v1);
 
 		*total_len += sizeof(*v1) - 1;
 	}
@@ -5155,9 +5152,7 @@ smb2_async_writev(struct cifs_io_subrequest *wdata)
 		req->WriteChannelInfoLength =
 			cpu_to_le16(sizeof(struct smbdirect_buffer_descriptor_v1));
 		v1 = (struct smbdirect_buffer_descriptor_v1 *) &req->Buffer[0];
-		v1->offset = cpu_to_le64(wdata->mr->mr->iova);
-		v1->token = cpu_to_le32(wdata->mr->mr->rkey);
-		v1->length = cpu_to_le32(wdata->mr->mr->length);
+		smbd_mr_fill_buffer_descriptor(wdata->mr, v1);
 
 		rqst.rq_iov[0].iov_len += sizeof(*v1);
 
diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c
index c79304012b08c..9e67adcdc7d3e 100644
--- a/fs/smb/client/smbdirect.c
+++ b/fs/smb/client/smbdirect.c
@@ -4,60 +4,12 @@
  *
  *   Author(s): Long Li <longli@microsoft.com>
  */
-#include <linux/module.h>
-#include <linux/highmem.h>
-#include <linux/folio_queue.h>
-#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) smbd_disconnect_rdma_connection(__sc)
-#include "../common/smbdirect/smbdirect_pdu.h"
+
 #include "smbdirect.h"
 #include "cifs_debug.h"
 #include "cifsproto.h"
 #include "smb2proto.h"
-
-const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn)
-{
-	struct smbdirect_socket *sc = &conn->socket;
-
-	return &sc->parameters;
-}
-
-static struct smbdirect_recv_io *get_receive_buffer(
-		struct smbdirect_socket *sc);
-static void put_receive_buffer(
-		struct smbdirect_socket *sc,
-		struct smbdirect_recv_io *response);
-static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf);
-static void destroy_receive_buffers(struct smbdirect_socket *sc);
-
-static void enqueue_reassembly(
-		struct smbdirect_socket *sc,
-		struct smbdirect_recv_io *response, int data_length);
-static struct smbdirect_recv_io *_get_first_reassembly(
-		struct smbdirect_socket *sc);
-
-static int smbd_post_send(struct smbdirect_socket *sc,
-			  struct smbdirect_send_batch *batch,
-			  struct smbdirect_send_io *request);
-
-static int smbd_post_recv(
-		struct smbdirect_socket *sc,
-		struct smbdirect_recv_io *response);
-
-static int smbd_post_send_empty(struct smbdirect_socket *sc);
-
-static void destroy_mr_list(struct smbdirect_socket *sc);
-static int allocate_mr_list(struct smbdirect_socket *sc);
-
-struct smb_extract_to_rdma {
-	struct ib_sge		*sge;
-	unsigned int		nr_sge;
-	unsigned int		max_sge;
-	struct ib_device	*device;
-	u32			local_dma_lkey;
-	enum dma_data_direction	direction;
-};
-static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len,
-					struct smb_extract_to_rdma *rdma);
+#include "../common/smbdirect/smbdirect_public.h"
 
 /* Port numbers for SMBD transport */
 #define SMB_PORT	445
@@ -72,21 +24,12 @@ static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len,
 /* The timeout to wait for a keepalive message from peer in seconds */
 #define KEEPALIVE_RECV_TIMEOUT 5
 
-/* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */
-#define SMBD_MIN_RECEIVE_SIZE		128
-#define SMBD_MIN_FRAGMENTED_SIZE	131072
-
 /*
  * Default maximum number of RDMA read/write outstanding on this connection
  * This value is possibly decreased during QP creation on hardware limit
  */
 #define SMBD_CM_RESPONDER_RESOURCES	32
 
-/* Maximum number of retries on data transfer operations */
-#define SMBD_CM_RETRY			6
-/* No need to retry on Receiver Not Ready since SMBD manages credits */
-#define SMBD_CM_RNR_RETRY		0
-
 /*
  * User configurable initial values per SMBD transport connection
  * as defined in [MS-SMBD] 3.1.1.1
@@ -162,6 +105,43 @@ module_param(smbd_logging_level, uint, 0644);
 MODULE_PARM_DESC(smbd_logging_level,
 	"Logging level for SMBD transport, 0 (default): error, 1: info");
 
+static bool smbd_logging_needed(struct smbdirect_socket *sc,
+				void *private_ptr,
+				unsigned int lvl,
+				unsigned int cls)
+{
+#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_LOG_ ##x)
+	BUILD_BUG_SAME(ERR);
+	BUILD_BUG_SAME(INFO);
+#undef BUILD_BUG_SAME
+#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_ ##x)
+	BUILD_BUG_SAME(LOG_OUTGOING);
+	BUILD_BUG_SAME(LOG_INCOMING);
+	BUILD_BUG_SAME(LOG_READ);
+	BUILD_BUG_SAME(LOG_WRITE);
+	BUILD_BUG_SAME(LOG_RDMA_SEND);
+	BUILD_BUG_SAME(LOG_RDMA_RECV);
+	BUILD_BUG_SAME(LOG_KEEP_ALIVE);
+	BUILD_BUG_SAME(LOG_RDMA_EVENT);
+	BUILD_BUG_SAME(LOG_RDMA_MR);
+#undef BUILD_BUG_SAME
+
+	if (lvl <= smbd_logging_level || cls & smbd_logging_class)
+		return true;
+	return false;
+}
+
+static void smbd_logging_vaprintf(struct smbdirect_socket *sc,
+				  const char *func,
+				  unsigned int line,
+				  void *private_ptr,
+				  unsigned int lvl,
+				  unsigned int cls,
+				  struct va_format *vaf)
+{
+	cifs_dbg(VFS, "%s:%u %pV", func, line, vaf);
+}
+
 #define log_rdma(level, class, fmt, args...)				\
 do {									\
 	if (level <= smbd_logging_level || class & smbd_logging_class)	\
@@ -185,2378 +165,289 @@ do {									\
 #define log_rdma_mr(level, fmt, args...) \
 		log_rdma(level, LOG_RDMA_MR, fmt, ##args)
 
-static void smbd_disconnect_wake_up_all(struct smbdirect_socket *sc)
-{
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 */
-	wake_up_all(&sc->status_wait);
-	wake_up_all(&sc->send_io.lcredits.wait_queue);
-	wake_up_all(&sc->send_io.credits.wait_queue);
-	wake_up_all(&sc->send_io.pending.dec_wait_queue);
-	wake_up_all(&sc->send_io.pending.zero_wait_queue);
-	wake_up_all(&sc->recv_io.reassembly.wait_queue);
-	wake_up_all(&sc->mr_io.ready.wait_queue);
-	wake_up_all(&sc->mr_io.cleanup.wait_queue);
-}
-
-static void smbd_disconnect_rdma_work(struct work_struct *work)
+static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
+				    struct smbdirect_send_batch *batch,
+				    struct iov_iter *iter,
+				    u32 remaining_data_length)
 {
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, disconnect_work);
-
-	if (sc->first_error == 0)
-		sc->first_error = -ECONNABORTED;
+	int bytes = 0;
 
 	/*
-	 * make sure this and other work is not queued again
-	 * but here we don't block and avoid
-	 * disable[_delayed]_work_sync()
+	 * smbdirect_connection_send_single_iter() respects the
+	 * negotiated max_send_size, so we need to
+	 * loop until the full iter is posted
 	 */
-	disable_work(&sc->disconnect_work);
-	disable_work(&sc->recv_io.posted.refill_work);
-	disable_work(&sc->mr_io.recovery_work);
-	disable_work(&sc->idle.immediate_work);
-	disable_delayed_work(&sc->idle.timer_work);
-
-	switch (sc->status) {
-	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
-	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
-	case SMBDIRECT_SOCKET_CONNECTED:
-	case SMBDIRECT_SOCKET_ERROR:
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;
-		rdma_disconnect(sc->rdma.cm_id);
-		break;
 
-	case SMBDIRECT_SOCKET_CREATED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
-		/*
-		 * rdma_connect() never reached
-		 * RDMA_CM_EVENT_ESTABLISHED
-		 */
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		break;
+	while (iov_iter_count(iter) > 0) {
+		int rc;
 
-	case SMBDIRECT_SOCKET_DISCONNECTING:
-	case SMBDIRECT_SOCKET_DISCONNECTED:
-	case SMBDIRECT_SOCKET_DESTROYED:
-		break;
+		rc = smbdirect_connection_send_single_iter(sc,
+							   batch,
+							   iter,
+							   0, /* flags */
+							   remaining_data_length);
+		if (rc < 0)
+			return rc;
+		remaining_data_length -= rc;
+		bytes += rc;
 	}
 
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 */
-	smbd_disconnect_wake_up_all(sc);
+	return bytes;
 }
 
-static void smbd_disconnect_rdma_connection(struct smbdirect_socket *sc)
+/*
+ * Destroy the transport and related RDMA and memory resources
+ * Need to go through all the pending counters and make sure on one is using
+ * the transport while it is destroyed
+ */
+void smbd_destroy(struct TCP_Server_Info *server)
 {
-	if (sc->first_error == 0)
-		sc->first_error = -ECONNABORTED;
-
-	/*
-	 * make sure other work (than disconnect_work) is
-	 * not queued again but here we don't block and avoid
-	 * disable[_delayed]_work_sync()
-	 */
-	disable_work(&sc->recv_io.posted.refill_work);
-	disable_work(&sc->mr_io.recovery_work);
-	disable_work(&sc->idle.immediate_work);
-	disable_delayed_work(&sc->idle.timer_work);
-
-	switch (sc->status) {
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
-	case SMBDIRECT_SOCKET_ERROR:
-	case SMBDIRECT_SOCKET_DISCONNECTING:
-	case SMBDIRECT_SOCKET_DISCONNECTED:
-	case SMBDIRECT_SOCKET_DESTROYED:
-		/*
-		 * Keep the current error status
-		 */
-		break;
-
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;
-		break;
+	struct smbd_connection *info = server->smbd_conn;
 
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;
-		break;
+	if (!info) {
+		log_rdma_event(INFO, "rdma session already destroyed\n");
+		return;
+	}
 
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;
-		break;
+	smbdirect_socket_release(info->socket);
 
-	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
-		break;
+	kfree(info);
+	server->smbd_conn = NULL;
+}
 
-	case SMBDIRECT_SOCKET_CREATED:
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		break;
+/*
+ * Reconnect this SMBD connection, called from upper layer
+ * return value: 0 on success, or actual error code
+ */
+int smbd_reconnect(struct TCP_Server_Info *server)
+{
+	log_rdma_event(INFO, "reconnecting rdma session\n");
 
-	case SMBDIRECT_SOCKET_CONNECTED:
-		sc->status = SMBDIRECT_SOCKET_ERROR;
-		break;
+	if (!server->smbd_conn) {
+		log_rdma_event(INFO, "rdma session already destroyed\n");
+		goto create_conn;
 	}
 
 	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
+	 * This is possible if transport is disconnected and we haven't received
+	 * notification from RDMA, but upper layer has detected timeout
 	 */
-	smbd_disconnect_wake_up_all(sc);
+	log_rdma_event(INFO, "disconnecting transport\n");
+	smbd_destroy(server);
 
-	queue_work(sc->workqueue, &sc->disconnect_work);
+create_conn:
+	log_rdma_event(INFO, "creating rdma session\n");
+	server->smbd_conn = smbd_get_connection(
+		server, (struct sockaddr *) &server->dstaddr);
+
+	if (server->smbd_conn) {
+		cifs_dbg(VFS, "RDMA transport re-established\n");
+		trace_smb3_smbd_connect_done(server->hostname, server->conn_id, &server->dstaddr);
+		return 0;
+	}
+	trace_smb3_smbd_connect_err(server->hostname, server->conn_id, &server->dstaddr);
+	return -ENOENT;
 }
 
-/* Upcall from RDMA CM */
-static int smbd_conn_upcall(
-		struct rdma_cm_id *id, struct rdma_cm_event *event)
+/* Create a SMBD connection, called by upper layer */
+static struct smbd_connection *_smbd_get_connection(
+	struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port)
 {
-	struct smbdirect_socket *sc = id->context;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	const char *event_name = rdma_event_msg(event->event);
-	u8 peer_initiator_depth;
-	u8 peer_responder_resources;
-
-	log_rdma_event(INFO, "event=%s status=%d\n",
-		event_name, event->status);
-
-	switch (event->event) {
-	case RDMA_CM_EVENT_ADDR_RESOLVED:
-		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING))
-			break;
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED;
-		wake_up(&sc->status_wait);
-		break;
-
-	case RDMA_CM_EVENT_ROUTE_RESOLVED:
-		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING))
-			break;
-		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;
-		wake_up(&sc->status_wait);
-		break;
-
-	case RDMA_CM_EVENT_ADDR_ERROR:
-		log_rdma_event(ERR, "connecting failed event=%s\n", event_name);
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;
-		smbd_disconnect_rdma_work(&sc->disconnect_work);
-		break;
-
-	case RDMA_CM_EVENT_ROUTE_ERROR:
-		log_rdma_event(ERR, "connecting failed event=%s\n", event_name);
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;
-		smbd_disconnect_rdma_work(&sc->disconnect_work);
-		break;
-
-	case RDMA_CM_EVENT_ESTABLISHED:
-		log_rdma_event(INFO, "connected event=%s\n", event_name);
+	struct net *net = cifs_net_ns(server);
+	struct smbd_connection *info;
+	struct smbdirect_socket *sc;
+	struct smbdirect_socket_parameters init_params = {};
+	struct smbdirect_socket_parameters *sp;
+	__be16 *sport;
+	u64 port_flags = 0;
+	int ret;
 
+	switch (port) {
+	case SMBD_PORT:
 		/*
-		 * Here we work around an inconsistency between
-		 * iWarp and other devices (at least rxe and irdma using RoCEv2)
+		 * only allow iWarp devices
+		 * for port 5445.
 		 */
-		if (rdma_protocol_iwarp(id->device, id->port_num)) {
-			/*
-			 * iWarp devices report the peer's values
-			 * with the perspective of the peer here.
-			 * Tested with siw and irdma (in iwarp mode)
-			 * We need to change to our perspective here,
-			 * so we need to switch the values.
-			 */
-			peer_initiator_depth = event->param.conn.responder_resources;
-			peer_responder_resources = event->param.conn.initiator_depth;
-		} else {
-			/*
-			 * Non iWarp devices report the peer's values
-			 * already changed to our perspective here.
-			 * Tested with rxe and irdma (in roce mode).
-			 */
-			peer_initiator_depth = event->param.conn.initiator_depth;
-			peer_responder_resources = event->param.conn.responder_resources;
-		}
-		if (rdma_protocol_iwarp(id->device, id->port_num) &&
-		    event->param.conn.private_data_len == 8) {
-			/*
-			 * Legacy clients with only iWarp MPA v1 support
-			 * need a private blob in order to negotiate
-			 * the IRD/ORD values.
-			 */
-			const __be32 *ird_ord_hdr = event->param.conn.private_data;
-			u32 ird32 = be32_to_cpu(ird_ord_hdr[0]);
-			u32 ord32 = be32_to_cpu(ird_ord_hdr[1]);
-
-			/*
-			 * cifs.ko sends the legacy IRD/ORD negotiation
-			 * event if iWarp MPA v2 was used.
-			 *
-			 * Here we check that the values match and only
-			 * mark the client as legacy if they don't match.
-			 */
-			if ((u32)event->param.conn.initiator_depth != ird32 ||
-			    (u32)event->param.conn.responder_resources != ord32) {
-				/*
-				 * There are broken clients (old cifs.ko)
-				 * using little endian and also
-				 * struct rdma_conn_param only uses u8
-				 * for initiator_depth and responder_resources,
-				 * so we truncate the value to U8_MAX.
-				 *
-				 * smb_direct_accept_client() will then
-				 * do the real negotiation in order to
-				 * select the minimum between client and
-				 * server.
-				 */
-				ird32 = min_t(u32, ird32, U8_MAX);
-				ord32 = min_t(u32, ord32, U8_MAX);
-
-				sc->rdma.legacy_iwarp = true;
-				peer_initiator_depth = (u8)ird32;
-				peer_responder_resources = (u8)ord32;
-			}
-		}
-
+		port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW;
+		break;
+	case SMB_PORT:
 		/*
-		 * negotiate the value by using the minimum
-		 * between client and server if the client provided
-		 * non 0 values.
+		 * only allow InfiniBand, RoCEv1 or RoCEv2
+		 * devices for port 445.
+		 *
+		 * (Basically don't allow iWarp devices)
 		 */
-		if (peer_initiator_depth != 0)
-			sp->initiator_depth =
-					min_t(u8, sp->initiator_depth,
-					      peer_initiator_depth);
-		if (peer_responder_resources != 0)
-			sp->responder_resources =
-					min_t(u8, sp->responder_resources,
-					      peer_responder_resources);
-
-		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING))
-			break;
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
-		wake_up(&sc->status_wait);
-		break;
-
-	case RDMA_CM_EVENT_CONNECT_ERROR:
-	case RDMA_CM_EVENT_UNREACHABLE:
-	case RDMA_CM_EVENT_REJECTED:
-		log_rdma_event(ERR, "connecting failed event=%s\n", event_name);
-		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;
-		smbd_disconnect_rdma_work(&sc->disconnect_work);
-		break;
-
-	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-	case RDMA_CM_EVENT_DISCONNECTED:
-		/* This happens when we fail the negotiation */
-		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_FAILED) {
-			log_rdma_event(ERR, "event=%s during negotiation\n", event_name);
-		}
-
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		smbd_disconnect_rdma_work(&sc->disconnect_work);
-		break;
-
-	default:
-		log_rdma_event(ERR, "unexpected event=%s status=%d\n",
-			       event_name, event->status);
+		port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB;
 		break;
 	}
 
-	return 0;
-}
+	/*
+	 * Create the initial parameters
+	 */
+	sp = &init_params;
+	sp->flags = port_flags;
+	sp->resolve_addr_timeout_msec = RDMA_RESOLVE_TIMEOUT;
+	sp->resolve_route_timeout_msec = RDMA_RESOLVE_TIMEOUT;
+	sp->rdma_connect_timeout_msec = RDMA_RESOLVE_TIMEOUT;
+	sp->negotiate_timeout_msec = SMBD_NEGOTIATE_TIMEOUT * 1000;
+	sp->initiator_depth = 1;
+	sp->responder_resources = SMBD_CM_RESPONDER_RESOURCES;
+	sp->recv_credit_max = smbd_receive_credit_max;
+	sp->send_credit_target = smbd_send_credit_target;
+	sp->max_send_size = smbd_max_send_size;
+	sp->max_fragmented_recv_size = smbd_max_fragmented_recv_size;
+	sp->max_recv_size = smbd_max_receive_size;
+	sp->max_frmr_depth = smbd_max_frmr_depth;
+	sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000;
+	sp->keepalive_timeout_msec = KEEPALIVE_RECV_TIMEOUT * 1000;
 
-/* Upcall from RDMA QP */
-static void
-smbd_qp_async_error_upcall(struct ib_event *event, void *context)
-{
-	struct smbdirect_socket *sc = context;
+	info = kzalloc_obj(*info);
+	if (!info)
+		return NULL;
+	ret = smbdirect_socket_create_kern(net, &sc);
+	if (ret)
+		goto socket_init_failed;
+	smbdirect_socket_set_logging(sc, NULL, smbd_logging_needed, smbd_logging_vaprintf);
+	ret = smbdirect_socket_set_initial_parameters(sc, sp);
+	if (ret)
+		goto set_params_failed;
+	ret = smbdirect_socket_set_kernel_settings(sc, IB_POLL_SOFTIRQ, GFP_KERNEL);
+	if (ret)
+		goto set_settings_failed;
 
-	log_rdma_event(ERR, "%s on device %s socket %p\n",
-		ib_event_msg(event->event), event->device->name, sc);
+	if (dstaddr->sa_family == AF_INET6)
+		sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port;
+	else
+		sport = &((struct sockaddr_in *)dstaddr)->sin_port;
 
-	switch (event->event) {
-	case IB_EVENT_CQ_ERR:
-	case IB_EVENT_QP_FATAL:
-		smbd_disconnect_rdma_connection(sc);
-		break;
+	*sport = htons(port);
 
-	default:
-		break;
+	ret = smbdirect_connect_sync(sc, dstaddr);
+	if (ret) {
+		log_rdma_event(ERR, "connect to %pISpsfc failed: %1pe\n",
+			       dstaddr, ERR_PTR(ret));
+		goto connect_failed;
 	}
-}
-
-static inline void *smbdirect_send_io_payload(struct smbdirect_send_io *request)
-{
-	return (void *)request->packet;
-}
-
-static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *response)
-{
-	return (void *)response->packet;
-}
 
-static struct smbdirect_send_io *smbd_alloc_send_io(struct smbdirect_socket *sc)
-{
-	struct smbdirect_send_io *msg;
-
-	msg = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL);
-	if (!msg)
-		return ERR_PTR(-ENOMEM);
-	msg->socket = sc;
-	INIT_LIST_HEAD(&msg->sibling_list);
-	msg->num_sge = 0;
+	info->socket = sc;
+	return info;
 
-	return msg;
+connect_failed:
+set_settings_failed:
+set_params_failed:
+	smbdirect_socket_release(sc);
+socket_init_failed:
+	kfree(info);
+	return NULL;
 }
 
-static void smbd_free_send_io(struct smbdirect_send_io *msg)
+const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn)
 {
-	struct smbdirect_socket *sc = msg->socket;
-	size_t i;
-
-	/*
-	 * The list needs to be empty!
-	 * The caller should take care of it.
-	 */
-	WARN_ON_ONCE(!list_empty(&msg->sibling_list));
+	if (unlikely(!conn->socket)) {
+		static const struct smbdirect_socket_parameters zero_params;
 
-	/*
-	 * Note we call ib_dma_unmap_page(), even if some sges are mapped using
-	 * ib_dma_map_single().
-	 *
-	 * The difference between _single() and _page() only matters for the
-	 * ib_dma_map_*() case.
-	 *
-	 * For the ib_dma_unmap_*() case it does not matter as both take the
-	 * dma_addr_t and dma_unmap_single_attrs() is just an alias to
-	 * dma_unmap_page_attrs().
-	 */
-	for (i = 0; i < msg->num_sge; i++)
-		ib_dma_unmap_page(sc->ib.dev,
-				  msg->sge[i].addr,
-				  msg->sge[i].length,
-				  DMA_TO_DEVICE);
+		return &zero_params;
+	}
 
-	mempool_free(msg, sc->send_io.mem.pool);
+	return smbdirect_socket_get_current_parameters(conn->socket);
 }
 
-/* Called when a RDMA send is done */
-static void send_done(struct ib_cq *cq, struct ib_wc *wc)
+struct smbd_connection *smbd_get_connection(
+	struct TCP_Server_Info *server, struct sockaddr *dstaddr)
 {
-	struct smbdirect_send_io *request =
-		container_of(wc->wr_cqe, struct smbdirect_send_io, cqe);
-	struct smbdirect_socket *sc = request->socket;
-	struct smbdirect_send_io *sibling, *next;
-	int lcredits = 0;
-
-	log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%s\n",
-		request, ib_wc_status_msg(wc->status));
-
-	if (unlikely(!(request->wr.send_flags & IB_SEND_SIGNALED))) {
-		/*
-		 * This happens when smbdirect_send_io is a sibling
-		 * before the final message, it is signaled on
-		 * error anyway, so we need to skip
-		 * smbdirect_connection_free_send_io here,
-		 * otherwise is will destroy the memory
-		 * of the siblings too, which will cause
-		 * use after free problems for the others
-		 * triggered from ib_drain_qp().
-		 */
-		if (wc->status != IB_WC_SUCCESS)
-			goto skip_free;
+	struct smbd_connection *ret;
+	const struct smbdirect_socket_parameters *sp;
+	int port = SMBD_PORT;
 
-		/*
-		 * This should not happen!
-		 * But we better just close the
-		 * connection...
-		 */
-		log_rdma_send(ERR,
-			"unexpected send completion wc->status=%s (%d) wc->opcode=%d\n",
-			ib_wc_status_msg(wc->status), wc->status, wc->opcode);
-		smbd_disconnect_rdma_connection(sc);
-		return;
-	}
+try_again:
+	ret = _smbd_get_connection(server, dstaddr, port);
 
-	/*
-	 * Free possible siblings and then the main send_io
-	 */
-	list_for_each_entry_safe(sibling, next, &request->sibling_list, sibling_list) {
-		list_del_init(&sibling->sibling_list);
-		smbd_free_send_io(sibling);
-		lcredits += 1;
-	}
-	/* Note this frees wc->wr_cqe, but not wc */
-	smbd_free_send_io(request);
-	lcredits += 1;
-
-	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
-skip_free:
-		if (wc->status != IB_WC_WR_FLUSH_ERR)
-			log_rdma_send(ERR, "wc->status=%s wc->opcode=%d\n",
-				ib_wc_status_msg(wc->status), wc->opcode);
-		smbd_disconnect_rdma_connection(sc);
-		return;
+	/* Try SMB_PORT if SMBD_PORT doesn't work */
+	if (!ret && port == SMBD_PORT) {
+		port = SMB_PORT;
+		goto try_again;
 	}
+	if (!ret)
+		return NULL;
 
-	atomic_add(lcredits, &sc->send_io.lcredits.count);
-	wake_up(&sc->send_io.lcredits.wait_queue);
+	sp = smbd_get_parameters(ret);
 
-	if (atomic_dec_and_test(&sc->send_io.pending.count))
-		wake_up(&sc->send_io.pending.zero_wait_queue);
+	server->rdma_readwrite_threshold =
+		rdma_readwrite_threshold > sp->max_fragmented_send_size ?
+		sp->max_fragmented_send_size :
+		rdma_readwrite_threshold;
 
-	wake_up(&sc->send_io.pending.dec_wait_queue);
+	return ret;
 }
 
-static void dump_smbdirect_negotiate_resp(struct smbdirect_negotiate_resp *resp)
+/*
+ * Receive data from the transport's receive reassembly queue
+ * All the incoming data packets are placed in reassembly queue
+ * iter: the buffer to read data into
+ * size: the length of data to read
+ * return value: actual data read
+ *
+ * Note: this implementation copies the data from reassembly queue to receive
+ * buffers used by upper layer. This is not the optimal code path. A better way
+ * to do it is to not have upper layer allocate its receive buffers but rather
+ * borrow the buffer from reassembly queue, and return it after data is
+ * consumed. But this will require more changes to upper layer code, and also
+ * need to consider packet boundaries while they still being reassembled.
+ */
+int smbd_recv(struct smbd_connection *info, struct msghdr *msg)
 {
-	log_rdma_event(INFO, "resp message min_version %u max_version %u negotiated_version %u credits_requested %u credits_granted %u status %u max_readwrite_size %u preferred_send_size %u max_receive_size %u max_fragmented_size %u\n",
-		       resp->min_version, resp->max_version,
-		       resp->negotiated_version, resp->credits_requested,
-		       resp->credits_granted, resp->status,
-		       resp->max_readwrite_size, resp->preferred_send_size,
-		       resp->max_receive_size, resp->max_fragmented_size);
+	struct smbdirect_socket *sc = info->socket;
+
+	if (!smbdirect_connection_is_connected(sc))
+		return -ENOTCONN;
+
+	return smbdirect_connection_recvmsg(sc, msg, 0);
 }
 
 /*
- * Process a negotiation response message, according to [MS-SMBD]3.1.5.7
- * response, packet_length: the negotiation response message
- * return value: true if negotiation is a success, false if failed
+ * Send data to transport
+ * Each rqst is transported as a SMBDirect payload
+ * rqst: the data to write
+ * return value: 0 if successfully write, otherwise error code
  */
-static bool process_negotiation_response(
-		struct smbdirect_recv_io *response, int packet_length)
+int smbd_send(struct TCP_Server_Info *server,
+	int num_rqst, struct smb_rqst *rqst_array)
 {
-	struct smbdirect_socket *sc = response->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_negotiate_resp *packet = smbdirect_recv_io_payload(response);
-
-	if (packet_length < sizeof(struct smbdirect_negotiate_resp)) {
-		log_rdma_event(ERR,
-			"error: packet_length=%d\n", packet_length);
-		return false;
-	}
+	struct smbd_connection *info = server->smbd_conn;
+	struct smbdirect_socket *sc = info->socket;
+	const struct smbdirect_socket_parameters *sp = smbd_get_parameters(info);
+	struct smb_rqst *rqst;
+	struct iov_iter iter;
+	struct smbdirect_send_batch_storage bstorage;
+	struct smbdirect_send_batch *batch;
+	unsigned int remaining_data_length, klen;
+	int rc, i, rqst_idx;
+	int error = 0;
 
-	if (le16_to_cpu(packet->negotiated_version) != SMBDIRECT_V1) {
-		log_rdma_event(ERR, "error: negotiated_version=%x\n",
-			le16_to_cpu(packet->negotiated_version));
-		return false;
-	}
+	if (!smbdirect_connection_is_connected(sc))
+		return -EAGAIN;
 
-	if (packet->credits_requested == 0) {
-		log_rdma_event(ERR, "error: credits_requested==0\n");
-		return false;
-	}
-	sc->recv_io.credits.target = le16_to_cpu(packet->credits_requested);
-	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);
+	/*
+	 * Add in the page array if there is one. The caller needs to set
+	 * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and
+	 * ends at page boundary
+	 */
+	remaining_data_length = 0;
+	for (i = 0; i < num_rqst; i++)
+		remaining_data_length += smb_rqst_len(server, &rqst_array[i]);
 
-	if (packet->credits_granted == 0) {
-		log_rdma_event(ERR, "error: credits_granted==0\n");
-		return false;
+	if (unlikely(remaining_data_length > sp->max_fragmented_send_size)) {
+		/* assertion: payload never exceeds negotiated maximum */
+		log_write(ERR, "payload size %d > max size %d\n",
+			remaining_data_length, sp->max_fragmented_send_size);
+		return -EINVAL;
 	}
-	atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target);
-	atomic_set(&sc->send_io.credits.count, le16_to_cpu(packet->credits_granted));
 
-	if (le32_to_cpu(packet->preferred_send_size) > sp->max_recv_size) {
-		log_rdma_event(ERR, "error: preferred_send_size=%d\n",
-			le32_to_cpu(packet->preferred_send_size));
-		return false;
-	}
-	sp->max_recv_size = le32_to_cpu(packet->preferred_send_size);
-
-	if (le32_to_cpu(packet->max_receive_size) < SMBD_MIN_RECEIVE_SIZE) {
-		log_rdma_event(ERR, "error: max_receive_size=%d\n",
-			le32_to_cpu(packet->max_receive_size));
-		return false;
-	}
-	sp->max_send_size = min_t(u32, sp->max_send_size,
-				  le32_to_cpu(packet->max_receive_size));
-
-	if (le32_to_cpu(packet->max_fragmented_size) <
-			SMBD_MIN_FRAGMENTED_SIZE) {
-		log_rdma_event(ERR, "error: max_fragmented_size=%d\n",
-			le32_to_cpu(packet->max_fragmented_size));
-		return false;
-	}
-	sp->max_fragmented_send_size =
-		le32_to_cpu(packet->max_fragmented_size);
-
-
-	sp->max_read_write_size = min_t(u32,
-			le32_to_cpu(packet->max_readwrite_size),
-			sp->max_frmr_depth * PAGE_SIZE);
-	sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE;
-
-	atomic_set(&sc->send_io.bcredits.count, 1);
-	sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;
-	return true;
-}
-
-static void smbd_post_send_credits(struct work_struct *work)
-{
-	int rc;
-	struct smbdirect_recv_io *response;
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, recv_io.posted.refill_work);
-	int posted = 0;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
-		return;
-	}
-
-	if (sc->recv_io.credits.target >
-		atomic_read(&sc->recv_io.credits.count)) {
-		while (true) {
-			response = get_receive_buffer(sc);
-			if (!response)
-				break;
-
-			response->first_segment = false;
-			rc = smbd_post_recv(sc, response);
-			if (rc) {
-				log_rdma_recv(ERR,
-					"post_recv failed rc=%d\n", rc);
-				put_receive_buffer(sc, response);
-				break;
-			}
-
-			atomic_inc(&sc->recv_io.posted.count);
-			posted += 1;
-		}
-	}
-
-	atomic_add(posted, &sc->recv_io.credits.available);
-
-	/*
-	 * If the last send credit is waiting for credits
-	 * it can grant we need to wake it up
-	 */
-	if (posted &&
-	    atomic_read(&sc->send_io.bcredits.count) == 0 &&
-	    atomic_read(&sc->send_io.credits.count) == 0)
-		wake_up(&sc->send_io.credits.wait_queue);
-
-	/* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */
-	if (atomic_read(&sc->recv_io.credits.count) <
-		sc->recv_io.credits.target - 1) {
-		log_keep_alive(INFO, "schedule send of an empty message\n");
-		queue_work(sc->workqueue, &sc->idle.immediate_work);
-	}
-}
-
-/* Called from softirq, when recv is done */
-static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	struct smbdirect_data_transfer *data_transfer;
-	struct smbdirect_recv_io *response =
-		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
-	struct smbdirect_socket *sc = response->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int current_recv_credits;
-	u16 old_recv_credit_target;
-	u32 data_offset = 0;
-	u32 data_length = 0;
-	u32 remaining_data_length = 0;
-	bool negotiate_done = false;
-
-	log_rdma_recv(INFO,
-		      "response=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n",
-		      response, sc->recv_io.expected,
-		      ib_wc_status_msg(wc->status), wc->opcode,
-		      wc->byte_len, wc->pkey_index);
-
-	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
-		if (wc->status != IB_WC_WR_FLUSH_ERR)
-			log_rdma_recv(ERR, "wc->status=%s opcode=%d\n",
-				ib_wc_status_msg(wc->status), wc->opcode);
-		goto error;
-	}
-
-	ib_dma_sync_single_for_cpu(
-		wc->qp->device,
-		response->sge.addr,
-		response->sge.length,
-		DMA_FROM_DEVICE);
-
-	/*
-	 * Reset timer to the keepalive interval in
-	 * order to trigger our next keepalive message.
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->keepalive_interval_msec));
-
-	switch (sc->recv_io.expected) {
-	/* SMBD negotiation response */
-	case SMBDIRECT_EXPECT_NEGOTIATE_REP:
-		dump_smbdirect_negotiate_resp(smbdirect_recv_io_payload(response));
-		sc->recv_io.reassembly.full_packet_received = true;
-		negotiate_done =
-			process_negotiation_response(response, wc->byte_len);
-		put_receive_buffer(sc, response);
-		if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING))
-			negotiate_done = false;
-		if (!negotiate_done) {
-			sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
-			smbd_disconnect_rdma_connection(sc);
-		} else {
-			sc->status = SMBDIRECT_SOCKET_CONNECTED;
-			wake_up(&sc->status_wait);
-		}
-
-		return;
-
-	/* SMBD data transfer packet */
-	case SMBDIRECT_EXPECT_DATA_TRANSFER:
-		data_transfer = smbdirect_recv_io_payload(response);
-
-		if (wc->byte_len <
-		    offsetof(struct smbdirect_data_transfer, padding))
-			goto error;
-
-		remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length);
-		data_offset = le32_to_cpu(data_transfer->data_offset);
-		data_length = le32_to_cpu(data_transfer->data_length);
-		if (wc->byte_len < data_offset ||
-		    (u64)wc->byte_len < (u64)data_offset + data_length)
-			goto error;
-
-		if (remaining_data_length > sp->max_fragmented_recv_size ||
-		    data_length > sp->max_fragmented_recv_size ||
-		    (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size)
-			goto error;
-
-		if (data_length) {
-			if (sc->recv_io.reassembly.full_packet_received)
-				response->first_segment = true;
-
-			if (le32_to_cpu(data_transfer->remaining_data_length))
-				sc->recv_io.reassembly.full_packet_received = false;
-			else
-				sc->recv_io.reassembly.full_packet_received = true;
-		}
-
-		atomic_dec(&sc->recv_io.posted.count);
-		current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count);
-
-		old_recv_credit_target = sc->recv_io.credits.target;
-		sc->recv_io.credits.target =
-			le16_to_cpu(data_transfer->credits_requested);
-		sc->recv_io.credits.target =
-			min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);
-		sc->recv_io.credits.target =
-			max_t(u16, sc->recv_io.credits.target, 1);
-		if (le16_to_cpu(data_transfer->credits_granted)) {
-			atomic_add(le16_to_cpu(data_transfer->credits_granted),
-				&sc->send_io.credits.count);
-			/*
-			 * We have new send credits granted from remote peer
-			 * If any sender is waiting for credits, unblock it
-			 */
-			wake_up(&sc->send_io.credits.wait_queue);
-		}
-
-		log_incoming(INFO, "data flags %d data_offset %d data_length %d remaining_data_length %d\n",
-			     le16_to_cpu(data_transfer->flags),
-			     le32_to_cpu(data_transfer->data_offset),
-			     le32_to_cpu(data_transfer->data_length),
-			     le32_to_cpu(data_transfer->remaining_data_length));
-
-		/* Send an immediate response right away if requested */
-		if (le16_to_cpu(data_transfer->flags) &
-				SMBDIRECT_FLAG_RESPONSE_REQUESTED) {
-			log_keep_alive(INFO, "schedule send of immediate response\n");
-			queue_work(sc->workqueue, &sc->idle.immediate_work);
-		}
-
-		/*
-		 * If this is a packet with data playload place the data in
-		 * reassembly queue and wake up the reading thread
-		 */
-		if (data_length) {
-			if (current_recv_credits <= (sc->recv_io.credits.target / 4) ||
-			    sc->recv_io.credits.target > old_recv_credit_target)
-				queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-
-			enqueue_reassembly(sc, response, data_length);
-			wake_up(&sc->recv_io.reassembly.wait_queue);
-		} else
-			put_receive_buffer(sc, response);
-
-		return;
-
-	case SMBDIRECT_EXPECT_NEGOTIATE_REQ:
-		/* Only server... */
-		break;
-	}
-
-	/*
-	 * This is an internal error!
-	 */
-	log_rdma_recv(ERR, "unexpected response type=%d\n", sc->recv_io.expected);
-	WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER);
-error:
-	put_receive_buffer(sc, response);
-	smbd_disconnect_rdma_connection(sc);
-}
-
-static struct rdma_cm_id *smbd_create_id(
-		struct smbdirect_socket *sc,
-		struct sockaddr *dstaddr, int port)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct rdma_cm_id *id;
-	u8 node_type = RDMA_NODE_UNSPECIFIED;
-	int rc;
-	__be16 *sport;
-
-	id = rdma_create_id(&init_net, smbd_conn_upcall, sc,
-		RDMA_PS_TCP, IB_QPT_RC);
-	if (IS_ERR(id)) {
-		rc = PTR_ERR(id);
-		log_rdma_event(ERR, "rdma_create_id() failed %i\n", rc);
-		return id;
-	}
-
-	switch (port) {
-	case SMBD_PORT:
-		/*
-		 * only allow iWarp devices
-		 * for port 5445.
-		 */
-		node_type = RDMA_NODE_RNIC;
-		break;
-	case SMB_PORT:
-		/*
-		 * only allow InfiniBand, RoCEv1 or RoCEv2
-		 * devices for port 445.
-		 *
-		 * (Basically don't allow iWarp devices)
-		 */
-		node_type = RDMA_NODE_IB_CA;
-		break;
-	}
-	rc = rdma_restrict_node_type(id, node_type);
-	if (rc) {
-		log_rdma_event(ERR, "rdma_restrict_node_type(%u) failed %i\n",
-			       node_type, rc);
-		goto out;
-	}
-
-	if (dstaddr->sa_family == AF_INET6)
-		sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port;
-	else
-		sport = &((struct sockaddr_in *)dstaddr)->sin_port;
-
-	*sport = htons(port);
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED);
-	sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING;
-	rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)dstaddr,
-		sp->resolve_addr_timeout_msec);
-	if (rc) {
-		log_rdma_event(ERR, "rdma_resolve_addr() failed %i\n", rc);
-		goto out;
-	}
-	rc = wait_event_interruptible_timeout(
-		sc->status_wait,
-		sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING,
-		msecs_to_jiffies(sp->resolve_addr_timeout_msec));
-	/* e.g. if interrupted returns -ERESTARTSYS */
-	if (rc < 0) {
-		log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc);
-		goto out;
-	}
-	if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING) {
-		rc = -ETIMEDOUT;
-		log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc);
-		goto out;
-	}
-	if (sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED) {
-		rc = -EHOSTUNREACH;
-		log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc);
-		goto out;
-	}
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED);
-	sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING;
-	rc = rdma_resolve_route(id, sp->resolve_route_timeout_msec);
-	if (rc) {
-		log_rdma_event(ERR, "rdma_resolve_route() failed %i\n", rc);
-		goto out;
-	}
-	rc = wait_event_interruptible_timeout(
-		sc->status_wait,
-		sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING,
-		msecs_to_jiffies(sp->resolve_route_timeout_msec));
-	/* e.g. if interrupted returns -ERESTARTSYS */
-	if (rc < 0)  {
-		log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc);
-		goto out;
-	}
-	if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING) {
-		rc = -ETIMEDOUT;
-		log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc);
-		goto out;
-	}
-	if (sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED) {
-		rc = -ENETUNREACH;
-		log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc);
-		goto out;
-	}
-
-	return id;
-
-out:
-	rdma_destroy_id(id);
-	return ERR_PTR(rc);
-}
-
-/*
- * Test if FRWR (Fast Registration Work Requests) is supported on the device
- * This implementation requires FRWR on RDMA read/write
- * return value: true if it is supported
- */
-static bool frwr_is_supported(struct ib_device_attr *attrs)
-{
-	if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
-		return false;
-	if (attrs->max_fast_reg_page_list_len == 0)
-		return false;
-	return true;
-}
-
-static int smbd_ia_open(
-		struct smbdirect_socket *sc,
-		struct sockaddr *dstaddr, int port)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int rc;
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);
-	sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED;
-
-	sc->rdma.cm_id = smbd_create_id(sc, dstaddr, port);
-	if (IS_ERR(sc->rdma.cm_id)) {
-		rc = PTR_ERR(sc->rdma.cm_id);
-		goto out1;
-	}
-	sc->ib.dev = sc->rdma.cm_id->device;
-
-	if (!frwr_is_supported(&sc->ib.dev->attrs)) {
-		log_rdma_event(ERR, "Fast Registration Work Requests (FRWR) is not supported\n");
-		log_rdma_event(ERR, "Device capability flags = %llx max_fast_reg_page_list_len = %u\n",
-			       sc->ib.dev->attrs.device_cap_flags,
-			       sc->ib.dev->attrs.max_fast_reg_page_list_len);
-		rc = -EPROTONOSUPPORT;
-		goto out2;
-	}
-	sp->max_frmr_depth = min_t(u32,
-		sp->max_frmr_depth,
-		sc->ib.dev->attrs.max_fast_reg_page_list_len);
-	sc->mr_io.type = IB_MR_TYPE_MEM_REG;
-	if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)
-		sc->mr_io.type = IB_MR_TYPE_SG_GAPS;
-
-	return 0;
-
-out2:
-	rdma_destroy_id(sc->rdma.cm_id);
-	sc->rdma.cm_id = NULL;
-
-out1:
-	return rc;
-}
-
-/*
- * Send a negotiation request message to the peer
- * The negotiation procedure is in [MS-SMBD] 3.1.5.2 and 3.1.5.3
- * After negotiation, the transport is connected and ready for
- * carrying upper layer SMB payload
- */
-static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int rc;
-	struct smbdirect_send_io *request;
-	struct smbdirect_negotiate_req *packet;
-
-	request = smbd_alloc_send_io(sc);
-	if (IS_ERR(request))
-		return PTR_ERR(request);
-
-	packet = smbdirect_send_io_payload(request);
-	packet->min_version = cpu_to_le16(SMBDIRECT_V1);
-	packet->max_version = cpu_to_le16(SMBDIRECT_V1);
-	packet->reserved = 0;
-	packet->credits_requested = cpu_to_le16(sp->send_credit_target);
-	packet->preferred_send_size = cpu_to_le32(sp->max_send_size);
-	packet->max_receive_size = cpu_to_le32(sp->max_recv_size);
-	packet->max_fragmented_size =
-		cpu_to_le32(sp->max_fragmented_recv_size);
-
-	request->sge[0].addr = ib_dma_map_single(
-				sc->ib.dev, (void *)packet,
-				sizeof(*packet), DMA_TO_DEVICE);
-	if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) {
-		rc = -EIO;
-		goto dma_mapping_failed;
-	}
-	request->num_sge = 1;
-
-	request->sge[0].length = sizeof(*packet);
-	request->sge[0].lkey = sc->ib.pd->local_dma_lkey;
-
-	rc = smbd_post_send(sc, NULL, request);
-	if (!rc)
-		return 0;
-
-	if (rc == -EAGAIN)
-		rc = -EIO;
-
-dma_mapping_failed:
-	smbd_free_send_io(request);
-	return rc;
-}
-
-/*
- * Extend the credits to remote peer
- * This implements [MS-SMBD] 3.1.5.9
- * The idea is that we should extend credits to remote peer as quickly as
- * it's allowed, to maintain data flow. We allocate as much receive
- * buffer as possible, and extend the receive credits to remote peer
- * return value: the new credtis being granted.
- */
-static int manage_credits_prior_sending(struct smbdirect_socket *sc)
-{
-	int missing;
-	int available;
-	int new_credits;
-
-	if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target)
-		return 0;
-
-	missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count);
-	available = atomic_xchg(&sc->recv_io.credits.available, 0);
-	new_credits = (u16)min3(U16_MAX, missing, available);
-	if (new_credits <= 0) {
-		/*
-		 * If credits are available, but not granted
-		 * we need to re-add them again.
-		 */
-		if (available)
-			atomic_add(available, &sc->recv_io.credits.available);
-		return 0;
-	}
-
-	if (new_credits < available) {
-		/*
-		 * Readd the remaining available again.
-		 */
-		available -= new_credits;
-		atomic_add(available, &sc->recv_io.credits.available);
-	}
-
-	/*
-	 * Remember we granted the credits
-	 */
-	atomic_add(new_credits, &sc->recv_io.credits.count);
-	return new_credits;
-}
-
-/*
- * Check if we need to send a KEEP_ALIVE message
- * The idle connection timer triggers a KEEP_ALIVE message when expires
- * SMBDIRECT_FLAG_RESPONSE_REQUESTED is set in the message flag to have peer send
- * back a response.
- * return value:
- * 1 if SMBDIRECT_FLAG_RESPONSE_REQUESTED needs to be set
- * 0: otherwise
- */
-static int manage_keep_alive_before_sending(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-
-	if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) {
-		sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT;
-		/*
-		 * Now use the keepalive timeout (instead of keepalive interval)
-		 * in order to wait for a response
-		 */
-		mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-				 msecs_to_jiffies(sp->keepalive_timeout_msec));
-		return 1;
-	}
-	return 0;
-}
-
-static int smbd_ib_post_send(struct smbdirect_socket *sc,
-			     struct ib_send_wr *wr)
-{
-	int ret;
-
-	atomic_inc(&sc->send_io.pending.count);
-	ret = ib_post_send(sc->ib.qp, wr, NULL);
-	if (ret) {
-		pr_err("failed to post send: %d\n", ret);
-		smbd_disconnect_rdma_connection(sc);
-		ret = -EAGAIN;
-	}
-	return ret;
-}
-
-/* Post the send request */
-static int smbd_post_send(struct smbdirect_socket *sc,
-			  struct smbdirect_send_batch *batch,
-			  struct smbdirect_send_io *request)
-{
-	int i;
-
-	for (i = 0; i < request->num_sge; i++) {
-		log_rdma_send(INFO,
-			"rdma_request sge[%d] addr=0x%llx length=%u\n",
-			i, request->sge[i].addr, request->sge[i].length);
-		ib_dma_sync_single_for_device(
-			sc->ib.dev,
-			request->sge[i].addr,
-			request->sge[i].length,
-			DMA_TO_DEVICE);
-	}
-
-	request->cqe.done = send_done;
-	request->wr.next = NULL;
-	request->wr.sg_list = request->sge;
-	request->wr.num_sge = request->num_sge;
-	request->wr.opcode = IB_WR_SEND;
-
-	if (batch) {
-		request->wr.wr_cqe = NULL;
-		request->wr.send_flags = 0;
-		if (!list_empty(&batch->msg_list)) {
-			struct smbdirect_send_io *last;
-
-			last = list_last_entry(&batch->msg_list,
-					       struct smbdirect_send_io,
-					       sibling_list);
-			last->wr.next = &request->wr;
-		}
-		list_add_tail(&request->sibling_list, &batch->msg_list);
-		batch->wr_cnt++;
-		return 0;
-	}
-
-	request->wr.wr_cqe = &request->cqe;
-	request->wr.send_flags = IB_SEND_SIGNALED;
-	return smbd_ib_post_send(sc, &request->wr);
-}
-
-static void smbd_send_batch_init(struct smbdirect_send_batch *batch,
-				 bool need_invalidate_rkey,
-				 unsigned int remote_key)
-{
-	INIT_LIST_HEAD(&batch->msg_list);
-	batch->wr_cnt = 0;
-	batch->need_invalidate_rkey = need_invalidate_rkey;
-	batch->remote_key = remote_key;
-	batch->credit = 0;
-}
-
-static int smbd_send_batch_flush(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch,
-				 bool is_last)
-{
-	struct smbdirect_send_io *first, *last;
-	int ret = 0;
-
-	if (list_empty(&batch->msg_list))
-		goto release_credit;
-
-	first = list_first_entry(&batch->msg_list,
-				 struct smbdirect_send_io,
-				 sibling_list);
-	last = list_last_entry(&batch->msg_list,
-			       struct smbdirect_send_io,
-			       sibling_list);
-
-	if (batch->need_invalidate_rkey) {
-		first->wr.opcode = IB_WR_SEND_WITH_INV;
-		first->wr.ex.invalidate_rkey = batch->remote_key;
-		batch->need_invalidate_rkey = false;
-		batch->remote_key = 0;
-	}
-
-	last->wr.send_flags = IB_SEND_SIGNALED;
-	last->wr.wr_cqe = &last->cqe;
-
-	/*
-	 * Remove last from batch->msg_list
-	 * and splice the rest of batch->msg_list
-	 * to last->sibling_list.
-	 *
-	 * batch->msg_list is a valid empty list
-	 * at the end.
-	 */
-	list_del_init(&last->sibling_list);
-	list_splice_tail_init(&batch->msg_list, &last->sibling_list);
-	batch->wr_cnt = 0;
-
-	ret = smbd_ib_post_send(sc, &first->wr);
-	if (ret) {
-		struct smbdirect_send_io *sibling, *next;
-
-		list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
-			list_del_init(&sibling->sibling_list);
-			smbd_free_send_io(sibling);
-		}
-		smbd_free_send_io(last);
-	}
-
-release_credit:
-	if (is_last && !ret && batch->credit) {
-		atomic_add(batch->credit, &sc->send_io.bcredits.count);
-		batch->credit = 0;
-		wake_up(&sc->send_io.bcredits.wait_queue);
-	}
-
-	return ret;
-}
-
-static int wait_for_credits(struct smbdirect_socket *sc,
-			    wait_queue_head_t *waitq, atomic_t *total_credits,
-			    int needed)
-{
-	int ret;
-
-	do {
-		if (atomic_sub_return(needed, total_credits) >= 0)
-			return 0;
-
-		atomic_add(needed, total_credits);
-		ret = wait_event_interruptible(*waitq,
-					       atomic_read(total_credits) >= needed ||
-					       sc->status != SMBDIRECT_SOCKET_CONNECTED);
-
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			return -ENOTCONN;
-		else if (ret < 0)
-			return ret;
-	} while (true);
-}
-
-static int wait_for_send_bcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	int ret;
-
-	if (batch->credit)
-		return 0;
-
-	ret = wait_for_credits(sc,
-			       &sc->send_io.bcredits.wait_queue,
-			       &sc->send_io.bcredits.count,
-			       1);
-	if (ret)
-		return ret;
-
-	batch->credit = 1;
-	return 0;
-}
-
-static int wait_for_send_lcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {
-		int ret;
-
-		ret = smbd_send_batch_flush(sc, batch, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc,
-				&sc->send_io.lcredits.wait_queue,
-				&sc->send_io.lcredits.count,
-				1);
-}
-
-static int wait_for_send_credits(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	if (batch &&
-	    (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
-		int ret;
-
-		ret = smbd_send_batch_flush(sc, batch, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc,
-				&sc->send_io.credits.wait_queue,
-				&sc->send_io.credits.count,
-				1);
-}
-
-static int smbd_post_send_iter(struct smbdirect_socket *sc,
-			       struct smbdirect_send_batch *batch,
-			       struct iov_iter *iter,
-			       int *_remaining_data_length)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int rc;
-	int header_length;
-	int data_length;
-	struct smbdirect_send_io *request;
-	struct smbdirect_data_transfer *packet;
-	int new_credits = 0;
-	struct smbdirect_send_batch _batch;
-
-	if (!batch) {
-		smbd_send_batch_init(&_batch, false, 0);
-		batch = &_batch;
-	}
-
-	rc = wait_for_send_bcredit(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_bcredit\n");
-		rc = -EAGAIN;
-		goto err_wait_bcredit;
-	}
-
-	rc = wait_for_send_lcredit(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_lcredit\n");
-		rc = -EAGAIN;
-		goto err_wait_lcredit;
-	}
-
-	rc = wait_for_send_credits(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_credit\n");
-		rc = -EAGAIN;
-		goto err_wait_credit;
-	}
-
-	new_credits = manage_credits_prior_sending(sc);
-	if (new_credits == 0 &&
-	    atomic_read(&sc->send_io.credits.count) == 0 &&
-	    atomic_read(&sc->recv_io.credits.count) == 0) {
-		queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-		rc = wait_event_interruptible(sc->send_io.credits.wait_queue,
-					      atomic_read(&sc->send_io.credits.count) >= 1 ||
-					      atomic_read(&sc->recv_io.credits.available) >= 1 ||
-					      sc->status != SMBDIRECT_SOCKET_CONNECTED);
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			rc = -ENOTCONN;
-		if (rc < 0) {
-			log_outgoing(ERR, "disconnected not sending on last credit\n");
-			rc = -EAGAIN;
-			goto err_wait_credit;
-		}
-
-		new_credits = manage_credits_prior_sending(sc);
-	}
-
-	request = smbd_alloc_send_io(sc);
-	if (IS_ERR(request)) {
-		rc = PTR_ERR(request);
-		goto err_alloc;
-	}
-
-	memset(request->sge, 0, sizeof(request->sge));
-
-	/* Map the packet to DMA */
-	header_length = sizeof(struct smbdirect_data_transfer);
-	/* If this is a packet without payload, don't send padding */
-	if (!iter)
-		header_length = offsetof(struct smbdirect_data_transfer, padding);
-
-	packet = smbdirect_send_io_payload(request);
-	request->sge[0].addr = ib_dma_map_single(sc->ib.dev,
-						 (void *)packet,
-						 header_length,
-						 DMA_TO_DEVICE);
-	if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) {
-		rc = -EIO;
-		goto err_dma;
-	}
-
-	request->sge[0].length = header_length;
-	request->sge[0].lkey = sc->ib.pd->local_dma_lkey;
-	request->num_sge = 1;
-
-	/* Fill in the data payload to find out how much data we can add */
-	if (iter) {
-		struct smb_extract_to_rdma extract = {
-			.nr_sge		= request->num_sge,
-			.max_sge	= SMBDIRECT_SEND_IO_MAX_SGE,
-			.sge		= request->sge,
-			.device		= sc->ib.dev,
-			.local_dma_lkey	= sc->ib.pd->local_dma_lkey,
-			.direction	= DMA_TO_DEVICE,
-		};
-		size_t payload_len = umin(*_remaining_data_length,
-					  sp->max_send_size - sizeof(*packet));
-
-		rc = smb_extract_iter_to_rdma(iter, payload_len,
-					      &extract);
-		if (rc < 0)
-			goto err_dma;
-		data_length = rc;
-		request->num_sge = extract.nr_sge;
-		*_remaining_data_length -= data_length;
-	} else {
-		data_length = 0;
-	}
-
-	/* Fill in the packet header */
-	packet->credits_requested = cpu_to_le16(sp->send_credit_target);
-	packet->credits_granted = cpu_to_le16(new_credits);
-
-	packet->flags = 0;
-	if (manage_keep_alive_before_sending(sc))
-		packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
-
-	packet->reserved = 0;
-	if (!data_length)
-		packet->data_offset = 0;
-	else
-		packet->data_offset = cpu_to_le32(24);
-	packet->data_length = cpu_to_le32(data_length);
-	packet->remaining_data_length = cpu_to_le32(*_remaining_data_length);
-	packet->padding = 0;
-
-	log_outgoing(INFO, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
-		     le16_to_cpu(packet->credits_requested),
-		     le16_to_cpu(packet->credits_granted),
-		     le32_to_cpu(packet->data_offset),
-		     le32_to_cpu(packet->data_length),
-		     le32_to_cpu(packet->remaining_data_length));
-
-	rc = smbd_post_send(sc, batch, request);
-	if (!rc) {
-		if (batch != &_batch)
-			return 0;
-
-		rc = smbd_send_batch_flush(sc, batch, true);
-		if (!rc)
-			return 0;
-	}
-
-err_dma:
-	smbd_free_send_io(request);
-
-err_alloc:
-	atomic_inc(&sc->send_io.credits.count);
-	wake_up(&sc->send_io.credits.wait_queue);
-
-err_wait_credit:
-	atomic_inc(&sc->send_io.lcredits.count);
-	wake_up(&sc->send_io.lcredits.wait_queue);
-
-err_wait_lcredit:
-	atomic_add(batch->credit, &sc->send_io.bcredits.count);
-	batch->credit = 0;
-	wake_up(&sc->send_io.bcredits.wait_queue);
-
-err_wait_bcredit:
-	return rc;
-}
-
-/*
- * Send an empty message
- * Empty message is used to extend credits to peer to for keep live
- * while there is no upper layer payload to send at the time
- */
-static int smbd_post_send_empty(struct smbdirect_socket *sc)
-{
-	int remaining_data_length = 0;
-
-	sc->statistics.send_empty++;
-	return smbd_post_send_iter(sc, NULL, NULL, &remaining_data_length);
-}
-
-static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
-				    struct smbdirect_send_batch *batch,
-				    struct iov_iter *iter,
-				    int *_remaining_data_length)
-{
-	int rc = 0;
-
-	/*
-	 * smbd_post_send_iter() respects the
-	 * negotiated max_send_size, so we need to
-	 * loop until the full iter is posted
-	 */
-
-	while (iov_iter_count(iter) > 0) {
-		rc = smbd_post_send_iter(sc, batch, iter, _remaining_data_length);
-		if (rc < 0)
-			break;
-	}
-
-	return rc;
-}
-
-/*
- * Post a receive request to the transport
- * The remote peer can only send data when a receive request is posted
- * The interaction is controlled by send/receive credit system
- */
-static int smbd_post_recv(
-		struct smbdirect_socket *sc, struct smbdirect_recv_io *response)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct ib_recv_wr recv_wr;
-	int rc = -EIO;
-
-	response->sge.addr = ib_dma_map_single(
-				sc->ib.dev, response->packet,
-				sp->max_recv_size, DMA_FROM_DEVICE);
-	if (ib_dma_mapping_error(sc->ib.dev, response->sge.addr))
-		return rc;
-
-	response->sge.length = sp->max_recv_size;
-	response->sge.lkey = sc->ib.pd->local_dma_lkey;
-
-	response->cqe.done = recv_done;
-
-	recv_wr.wr_cqe = &response->cqe;
-	recv_wr.next = NULL;
-	recv_wr.sg_list = &response->sge;
-	recv_wr.num_sge = 1;
-
-	rc = ib_post_recv(sc->ib.qp, &recv_wr, NULL);
-	if (rc) {
-		ib_dma_unmap_single(sc->ib.dev, response->sge.addr,
-				    response->sge.length, DMA_FROM_DEVICE);
-		response->sge.length = 0;
-		smbd_disconnect_rdma_connection(sc);
-		log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc);
-	}
-
-	return rc;
-}
-
-/* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */
-static int smbd_negotiate(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int rc;
-	struct smbdirect_recv_io *response = get_receive_buffer(sc);
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED);
-	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
-
-	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP;
-	rc = smbd_post_recv(sc, response);
-	log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n",
-		       rc, response->sge.addr,
-		       response->sge.length, response->sge.lkey);
-	if (rc) {
-		put_receive_buffer(sc, response);
-		return rc;
-	}
-
-	rc = smbd_post_send_negotiate_req(sc);
-	if (rc)
-		return rc;
-
-	rc = wait_event_interruptible_timeout(
-		sc->status_wait,
-		sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING,
-		msecs_to_jiffies(sp->negotiate_timeout_msec));
-	log_rdma_event(INFO, "wait_event_interruptible_timeout rc=%d\n", rc);
-
-	if (sc->status == SMBDIRECT_SOCKET_CONNECTED)
-		return 0;
-
-	if (rc == 0)
-		rc = -ETIMEDOUT;
-	else if (rc == -ERESTARTSYS)
-		rc = -EINTR;
-	else
-		rc = -ENOTCONN;
-
-	return rc;
-}
-
-/*
- * Implement Connection.FragmentReassemblyBuffer defined in [MS-SMBD] 3.1.1.1
- * This is a queue for reassembling upper layer payload and present to upper
- * layer. All the inncoming payload go to the reassembly queue, regardless of
- * if reassembly is required. The uuper layer code reads from the queue for all
- * incoming payloads.
- * Put a received packet to the reassembly queue
- * response: the packet received
- * data_length: the size of payload in this packet
- */
-static void enqueue_reassembly(
-	struct smbdirect_socket *sc,
-	struct smbdirect_recv_io *response,
-	int data_length)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-	list_add_tail(&response->list, &sc->recv_io.reassembly.list);
-	sc->recv_io.reassembly.queue_length++;
-	/*
-	 * Make sure reassembly_data_length is updated after list and
-	 * reassembly_queue_length are updated. On the dequeue side
-	 * reassembly_data_length is checked without a lock to determine
-	 * if reassembly_queue_length and list is up to date
-	 */
-	virt_wmb();
-	sc->recv_io.reassembly.data_length += data_length;
-	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-	sc->statistics.enqueue_reassembly_queue++;
-}
-
-/*
- * Get the first entry at the front of reassembly queue
- * Caller is responsible for locking
- * return value: the first entry if any, NULL if queue is empty
- */
-static struct smbdirect_recv_io *_get_first_reassembly(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *ret = NULL;
-
-	if (!list_empty(&sc->recv_io.reassembly.list)) {
-		ret = list_first_entry(
-			&sc->recv_io.reassembly.list,
-			struct smbdirect_recv_io, list);
-	}
-	return ret;
-}
-
-/*
- * Get a receive buffer
- * For each remote send, we need to post a receive. The receive buffers are
- * pre-allocated in advance.
- * return value: the receive buffer, NULL if none is available
- */
-static struct smbdirect_recv_io *get_receive_buffer(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *ret = NULL;
-	unsigned long flags;
-
-	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
-	if (!list_empty(&sc->recv_io.free.list)) {
-		ret = list_first_entry(
-			&sc->recv_io.free.list,
-			struct smbdirect_recv_io, list);
-		list_del(&ret->list);
-		sc->statistics.get_receive_buffer++;
-	}
-	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
-
-	return ret;
-}
-
-/*
- * Return a receive buffer
- * Upon returning of a receive buffer, we can post new receive and extend
- * more receive credits to remote peer. This is done immediately after a
- * receive buffer is returned.
- */
-static void put_receive_buffer(
-	struct smbdirect_socket *sc, struct smbdirect_recv_io *response)
-{
-	unsigned long flags;
-
-	if (likely(response->sge.length != 0)) {
-		ib_dma_unmap_single(sc->ib.dev,
-				    response->sge.addr,
-				    response->sge.length,
-				    DMA_FROM_DEVICE);
-		response->sge.length = 0;
-	}
-
-	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
-	list_add_tail(&response->list, &sc->recv_io.free.list);
-	sc->statistics.put_receive_buffer++;
-	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
-
-	queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-}
-
-/* Preallocate all receive buffer on transport establishment */
-static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf)
-{
-	struct smbdirect_recv_io *response;
-	int i;
-
-	for (i = 0; i < num_buf; i++) {
-		response = mempool_alloc(sc->recv_io.mem.pool, GFP_KERNEL);
-		if (!response)
-			goto allocate_failed;
-
-		response->socket = sc;
-		response->sge.length = 0;
-		list_add_tail(&response->list, &sc->recv_io.free.list);
-	}
-
-	return 0;
-
-allocate_failed:
-	while (!list_empty(&sc->recv_io.free.list)) {
-		response = list_first_entry(
-				&sc->recv_io.free.list,
-				struct smbdirect_recv_io, list);
-		list_del(&response->list);
-
-		mempool_free(response, sc->recv_io.mem.pool);
-	}
-	return -ENOMEM;
-}
-
-static void destroy_receive_buffers(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *response;
-
-	while ((response = get_receive_buffer(sc)))
-		mempool_free(response, sc->recv_io.mem.pool);
-}
-
-static void send_immediate_empty_message(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, idle.immediate_work);
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return;
-
-	log_keep_alive(INFO, "send an empty message\n");
-	smbd_post_send_empty(sc);
-}
-
-/* Implement idle connection timer [MS-SMBD] 3.1.6.2 */
-static void idle_connection_timer(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, idle.timer_work.work);
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-
-	if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) {
-		log_keep_alive(ERR,
-			"error status sc->idle.keepalive=%d\n",
-			sc->idle.keepalive);
-		smbd_disconnect_rdma_connection(sc);
-		return;
-	}
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return;
-
-	/*
-	 * Now use the keepalive timeout (instead of keepalive interval)
-	 * in order to wait for a response
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->keepalive_timeout_msec));
-	log_keep_alive(INFO, "schedule send of empty idle message\n");
-	queue_work(sc->workqueue, &sc->idle.immediate_work);
-}
-
-/*
- * Destroy the transport and related RDMA and memory resources
- * Need to go through all the pending counters and make sure on one is using
- * the transport while it is destroyed
- */
-void smbd_destroy(struct TCP_Server_Info *server)
-{
-	struct smbd_connection *info = server->smbd_conn;
-	struct smbdirect_socket *sc;
-	struct smbdirect_recv_io *response;
-	unsigned long flags;
-
-	if (!info) {
-		log_rdma_event(INFO, "rdma session already destroyed\n");
-		return;
-	}
-	sc = &info->socket;
-
-	log_rdma_event(INFO, "cancelling and disable disconnect_work\n");
-	disable_work_sync(&sc->disconnect_work);
-
-	log_rdma_event(INFO, "destroying rdma session\n");
-	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING)
-		smbd_disconnect_rdma_work(&sc->disconnect_work);
-	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) {
-		log_rdma_event(INFO, "wait for transport being disconnected\n");
-		wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED);
-		log_rdma_event(INFO, "waited for transport being disconnected\n");
-	}
-
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 *
-	 * Most likely this was already called via
-	 * smbd_disconnect_rdma_work(), but call it again...
-	 */
-	smbd_disconnect_wake_up_all(sc);
-
-	log_rdma_event(INFO, "cancelling recv_io.posted.refill_work\n");
-	disable_work_sync(&sc->recv_io.posted.refill_work);
-
-	log_rdma_event(INFO, "destroying qp\n");
-	ib_drain_qp(sc->ib.qp);
-	rdma_destroy_qp(sc->rdma.cm_id);
-	sc->ib.qp = NULL;
-
-	log_rdma_event(INFO, "cancelling idle timer\n");
-	disable_delayed_work_sync(&sc->idle.timer_work);
-	log_rdma_event(INFO, "cancelling send immediate work\n");
-	disable_work_sync(&sc->idle.immediate_work);
-
-	/* It's not possible for upper layer to get to reassembly */
-	log_rdma_event(INFO, "drain the reassembly queue\n");
-	do {
-		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-		response = _get_first_reassembly(sc);
-		if (response) {
-			list_del(&response->list);
-			spin_unlock_irqrestore(
-				&sc->recv_io.reassembly.lock, flags);
-			put_receive_buffer(sc, response);
-		} else
-			spin_unlock_irqrestore(
-				&sc->recv_io.reassembly.lock, flags);
-	} while (response);
-	sc->recv_io.reassembly.data_length = 0;
-
-	log_rdma_event(INFO, "free receive buffers\n");
-	destroy_receive_buffers(sc);
-
-	log_rdma_event(INFO, "freeing mr list\n");
-	destroy_mr_list(sc);
-
-	ib_free_cq(sc->ib.send_cq);
-	ib_free_cq(sc->ib.recv_cq);
-	ib_dealloc_pd(sc->ib.pd);
-	rdma_destroy_id(sc->rdma.cm_id);
-
-	/* free mempools */
-	mempool_destroy(sc->send_io.mem.pool);
-	kmem_cache_destroy(sc->send_io.mem.cache);
-
-	mempool_destroy(sc->recv_io.mem.pool);
-	kmem_cache_destroy(sc->recv_io.mem.cache);
-
-	sc->status = SMBDIRECT_SOCKET_DESTROYED;
-
-	destroy_workqueue(sc->workqueue);
-	log_rdma_event(INFO,  "rdma session destroyed\n");
-	kfree(info);
-	server->smbd_conn = NULL;
-}
-
-/*
- * Reconnect this SMBD connection, called from upper layer
- * return value: 0 on success, or actual error code
- */
-int smbd_reconnect(struct TCP_Server_Info *server)
-{
-	log_rdma_event(INFO, "reconnecting rdma session\n");
-
-	if (!server->smbd_conn) {
-		log_rdma_event(INFO, "rdma session already destroyed\n");
-		goto create_conn;
-	}
-
-	/*
-	 * This is possible if transport is disconnected and we haven't received
-	 * notification from RDMA, but upper layer has detected timeout
-	 */
-	if (server->smbd_conn->socket.status == SMBDIRECT_SOCKET_CONNECTED) {
-		log_rdma_event(INFO, "disconnecting transport\n");
-		smbd_destroy(server);
-	}
-
-create_conn:
-	log_rdma_event(INFO, "creating rdma session\n");
-	server->smbd_conn = smbd_get_connection(
-		server, (struct sockaddr *) &server->dstaddr);
-
-	if (server->smbd_conn) {
-		cifs_dbg(VFS, "RDMA transport re-established\n");
-		trace_smb3_smbd_connect_done(server->hostname, server->conn_id, &server->dstaddr);
-		return 0;
-	}
-	trace_smb3_smbd_connect_err(server->hostname, server->conn_id, &server->dstaddr);
-	return -ENOENT;
-}
-
-static void destroy_caches(struct smbdirect_socket *sc)
-{
-	destroy_receive_buffers(sc);
-	mempool_destroy(sc->recv_io.mem.pool);
-	kmem_cache_destroy(sc->recv_io.mem.cache);
-	mempool_destroy(sc->send_io.mem.pool);
-	kmem_cache_destroy(sc->send_io.mem.cache);
-}
-
-#define MAX_NAME_LEN	80
-static int allocate_caches(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	char name[MAX_NAME_LEN];
-	int rc;
-
-	if (WARN_ON_ONCE(sp->max_recv_size < sizeof(struct smbdirect_data_transfer)))
-		return -ENOMEM;
-
-	scnprintf(name, MAX_NAME_LEN, "smbdirect_send_io_%p", sc);
-	sc->send_io.mem.cache =
-		kmem_cache_create(
-			name,
-			sizeof(struct smbdirect_send_io) +
-				sizeof(struct smbdirect_data_transfer),
-			0, SLAB_HWCACHE_ALIGN, NULL);
-	if (!sc->send_io.mem.cache)
-		return -ENOMEM;
-
-	sc->send_io.mem.pool =
-		mempool_create(sp->send_credit_target, mempool_alloc_slab,
-			mempool_free_slab, sc->send_io.mem.cache);
-	if (!sc->send_io.mem.pool)
-		goto out1;
-
-	scnprintf(name, MAX_NAME_LEN, "smbdirect_recv_io_%p", sc);
-
-	struct kmem_cache_args response_args = {
-		.align		= __alignof__(struct smbdirect_recv_io),
-		.useroffset	= (offsetof(struct smbdirect_recv_io, packet) +
-				   sizeof(struct smbdirect_data_transfer)),
-		.usersize	= sp->max_recv_size - sizeof(struct smbdirect_data_transfer),
-	};
-	sc->recv_io.mem.cache =
-		kmem_cache_create(name,
-				  sizeof(struct smbdirect_recv_io) + sp->max_recv_size,
-				  &response_args, SLAB_HWCACHE_ALIGN);
-	if (!sc->recv_io.mem.cache)
-		goto out2;
-
-	sc->recv_io.mem.pool =
-		mempool_create(sp->recv_credit_max, mempool_alloc_slab,
-		       mempool_free_slab, sc->recv_io.mem.cache);
-	if (!sc->recv_io.mem.pool)
-		goto out3;
-
-	rc = allocate_receive_buffers(sc, sp->recv_credit_max);
-	if (rc) {
-		log_rdma_event(ERR, "failed to allocate receive buffers\n");
-		goto out4;
-	}
-
-	return 0;
-
-out4:
-	mempool_destroy(sc->recv_io.mem.pool);
-out3:
-	kmem_cache_destroy(sc->recv_io.mem.cache);
-out2:
-	mempool_destroy(sc->send_io.mem.pool);
-out1:
-	kmem_cache_destroy(sc->send_io.mem.cache);
-	return -ENOMEM;
-}
-
-/* Create a SMBD connection, called by upper layer */
-static struct smbd_connection *_smbd_get_connection(
-	struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port)
-{
-	int rc;
-	struct smbd_connection *info;
-	struct smbdirect_socket *sc;
-	struct smbdirect_socket_parameters *sp;
-	struct rdma_conn_param conn_param;
-	struct ib_qp_cap qp_cap;
-	struct ib_qp_init_attr qp_attr;
-	struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr;
-	struct ib_port_immutable port_immutable;
-	__be32 ird_ord_hdr[2];
-	char wq_name[80];
-	struct workqueue_struct *workqueue;
-
-	info = kzalloc_obj(struct smbd_connection);
-	if (!info)
-		return NULL;
-	sc = &info->socket;
-	scnprintf(wq_name, ARRAY_SIZE(wq_name), "smbd_%p", sc);
-	workqueue = create_workqueue(wq_name);
-	if (!workqueue)
-		goto create_wq_failed;
-	smbdirect_socket_init(sc);
-	sc->workqueue = workqueue;
-	sp = &sc->parameters;
-
-	INIT_WORK(&sc->disconnect_work, smbd_disconnect_rdma_work);
-
-	sp->resolve_addr_timeout_msec = RDMA_RESOLVE_TIMEOUT;
-	sp->resolve_route_timeout_msec = RDMA_RESOLVE_TIMEOUT;
-	sp->rdma_connect_timeout_msec = RDMA_RESOLVE_TIMEOUT;
-	sp->negotiate_timeout_msec = SMBD_NEGOTIATE_TIMEOUT * 1000;
-	sp->initiator_depth = 1;
-	sp->responder_resources = SMBD_CM_RESPONDER_RESOURCES;
-	sp->recv_credit_max = smbd_receive_credit_max;
-	sp->send_credit_target = smbd_send_credit_target;
-	sp->max_send_size = smbd_max_send_size;
-	sp->max_fragmented_recv_size = smbd_max_fragmented_recv_size;
-	sp->max_recv_size = smbd_max_receive_size;
-	sp->max_frmr_depth = smbd_max_frmr_depth;
-	sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000;
-	sp->keepalive_timeout_msec = KEEPALIVE_RECV_TIMEOUT * 1000;
-
-	rc = smbd_ia_open(sc, dstaddr, port);
-	if (rc) {
-		log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc);
-		goto create_id_failed;
-	}
-
-	if (sp->send_credit_target > sc->ib.dev->attrs.max_cqe ||
-	    sp->send_credit_target > sc->ib.dev->attrs.max_qp_wr) {
-		log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
-			       sp->send_credit_target,
-			       sc->ib.dev->attrs.max_cqe,
-			       sc->ib.dev->attrs.max_qp_wr);
-		goto config_failed;
-	}
-
-	if (sp->recv_credit_max > sc->ib.dev->attrs.max_cqe ||
-	    sp->recv_credit_max > sc->ib.dev->attrs.max_qp_wr) {
-		log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
-			       sp->recv_credit_max,
-			       sc->ib.dev->attrs.max_cqe,
-			       sc->ib.dev->attrs.max_qp_wr);
-		goto config_failed;
-	}
-
-	if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE ||
-	    sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) {
-		log_rdma_event(ERR,
-			"device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
-			IB_DEVICE_NAME_MAX,
-			sc->ib.dev->name,
-			sc->ib.dev->attrs.max_send_sge,
-			sc->ib.dev->attrs.max_recv_sge);
-		goto config_failed;
-	}
-
-	sp->responder_resources =
-		min_t(u8, sp->responder_resources,
-		      sc->ib.dev->attrs.max_qp_rd_atom);
-	log_rdma_mr(INFO, "responder_resources=%d\n",
-		sp->responder_resources);
-
-	/*
-	 * We use allocate sp->responder_resources * 2 MRs
-	 * and each MR needs WRs for REG and INV, so
-	 * we use '* 4'.
-	 *
-	 * +1 for ib_drain_qp()
-	 */
-	memset(&qp_cap, 0, sizeof(qp_cap));
-	qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1;
-	qp_cap.max_recv_wr = sp->recv_credit_max + 1;
-	qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
-	qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
-
-	sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
-	if (IS_ERR(sc->ib.pd)) {
-		rc = PTR_ERR(sc->ib.pd);
-		sc->ib.pd = NULL;
-		log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc);
-		goto alloc_pd_failed;
-	}
-
-	sc->ib.send_cq =
-		ib_alloc_cq_any(sc->ib.dev, sc,
-				qp_cap.max_send_wr, IB_POLL_SOFTIRQ);
-	if (IS_ERR(sc->ib.send_cq)) {
-		sc->ib.send_cq = NULL;
-		goto alloc_cq_failed;
-	}
-
-	sc->ib.recv_cq =
-		ib_alloc_cq_any(sc->ib.dev, sc,
-				qp_cap.max_recv_wr, IB_POLL_SOFTIRQ);
-	if (IS_ERR(sc->ib.recv_cq)) {
-		sc->ib.recv_cq = NULL;
-		goto alloc_cq_failed;
-	}
-
-	memset(&qp_attr, 0, sizeof(qp_attr));
-	qp_attr.event_handler = smbd_qp_async_error_upcall;
-	qp_attr.qp_context = sc;
-	qp_attr.cap = qp_cap;
-	qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
-	qp_attr.qp_type = IB_QPT_RC;
-	qp_attr.send_cq = sc->ib.send_cq;
-	qp_attr.recv_cq = sc->ib.recv_cq;
-	qp_attr.port_num = ~0;
-
-	rc = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);
-	if (rc) {
-		log_rdma_event(ERR, "rdma_create_qp failed %i\n", rc);
-		goto create_qp_failed;
-	}
-	sc->ib.qp = sc->rdma.cm_id->qp;
-
-	memset(&conn_param, 0, sizeof(conn_param));
-	conn_param.initiator_depth = sp->initiator_depth;
-	conn_param.responder_resources = sp->responder_resources;
-
-	/* Need to send IRD/ORD in private data for iWARP */
-	sc->ib.dev->ops.get_port_immutable(
-		sc->ib.dev, sc->rdma.cm_id->port_num, &port_immutable);
-	if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
-		ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources);
-		ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth);
-		conn_param.private_data = ird_ord_hdr;
-		conn_param.private_data_len = sizeof(ird_ord_hdr);
-	} else {
-		conn_param.private_data = NULL;
-		conn_param.private_data_len = 0;
-	}
-
-	conn_param.retry_count = SMBD_CM_RETRY;
-	conn_param.rnr_retry_count = SMBD_CM_RNR_RETRY;
-	conn_param.flow_control = 0;
-
-	log_rdma_event(INFO, "connecting to IP %pI4 port %d\n",
-		&addr_in->sin_addr, port);
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED);
-	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;
-	rc = rdma_connect(sc->rdma.cm_id, &conn_param);
-	if (rc) {
-		log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc);
-		goto rdma_connect_failed;
-	}
-
-	wait_event_interruptible_timeout(
-		sc->status_wait,
-		sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING,
-		msecs_to_jiffies(sp->rdma_connect_timeout_msec));
-
-	if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) {
-		log_rdma_event(ERR, "rdma_connect failed port=%d\n", port);
-		goto rdma_connect_failed;
-	}
-
-	log_rdma_event(INFO, "rdma_connect connected\n");
-
-	rc = allocate_caches(sc);
-	if (rc) {
-		log_rdma_event(ERR, "cache allocation failed\n");
-		goto allocate_cache_failed;
-	}
-
-	INIT_WORK(&sc->idle.immediate_work, send_immediate_empty_message);
-	INIT_DELAYED_WORK(&sc->idle.timer_work, idle_connection_timer);
-	/*
-	 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
-	 * so that the timer will cause a disconnect.
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->negotiate_timeout_msec));
-
-	INIT_WORK(&sc->recv_io.posted.refill_work, smbd_post_send_credits);
-
-	rc = smbd_negotiate(sc);
-	if (rc) {
-		log_rdma_event(ERR, "smbd_negotiate rc=%d\n", rc);
-		goto negotiation_failed;
-	}
-
-	rc = allocate_mr_list(sc);
-	if (rc) {
-		log_rdma_mr(ERR, "memory registration allocation failed\n");
-		goto allocate_mr_failed;
-	}
-
-	return info;
-
-allocate_mr_failed:
-	/* At this point, need to a full transport shutdown */
-	server->smbd_conn = info;
-	smbd_destroy(server);
-	return NULL;
-
-negotiation_failed:
-	disable_delayed_work_sync(&sc->idle.timer_work);
-	destroy_caches(sc);
-	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
-	rdma_disconnect(sc->rdma.cm_id);
-	wait_event(sc->status_wait,
-		sc->status == SMBDIRECT_SOCKET_DISCONNECTED);
-
-allocate_cache_failed:
-rdma_connect_failed:
-	rdma_destroy_qp(sc->rdma.cm_id);
-
-create_qp_failed:
-alloc_cq_failed:
-	if (sc->ib.send_cq)
-		ib_free_cq(sc->ib.send_cq);
-	if (sc->ib.recv_cq)
-		ib_free_cq(sc->ib.recv_cq);
-
-	ib_dealloc_pd(sc->ib.pd);
-
-alloc_pd_failed:
-config_failed:
-	rdma_destroy_id(sc->rdma.cm_id);
-
-create_id_failed:
-	destroy_workqueue(sc->workqueue);
-create_wq_failed:
-	kfree(info);
-	return NULL;
-}
-
-struct smbd_connection *smbd_get_connection(
-	struct TCP_Server_Info *server, struct sockaddr *dstaddr)
-{
-	struct smbd_connection *ret;
-	const struct smbdirect_socket_parameters *sp;
-	int port = SMBD_PORT;
-
-try_again:
-	ret = _smbd_get_connection(server, dstaddr, port);
-
-	/* Try SMB_PORT if SMBD_PORT doesn't work */
-	if (!ret && port == SMBD_PORT) {
-		port = SMB_PORT;
-		goto try_again;
-	}
-	if (!ret)
-		return NULL;
-
-	sp = &ret->socket.parameters;
-
-	server->rdma_readwrite_threshold =
-		rdma_readwrite_threshold > sp->max_fragmented_send_size ?
-		sp->max_fragmented_send_size :
-		rdma_readwrite_threshold;
-
-	return ret;
-}
-
-/*
- * Receive data from the transport's receive reassembly queue
- * All the incoming data packets are placed in reassembly queue
- * iter: the buffer to read data into
- * size: the length of data to read
- * return value: actual data read
- *
- * Note: this implementation copies the data from reassembly queue to receive
- * buffers used by upper layer. This is not the optimal code path. A better way
- * to do it is to not have upper layer allocate its receive buffers but rather
- * borrow the buffer from reassembly queue, and return it after data is
- * consumed. But this will require more changes to upper layer code, and also
- * need to consider packet boundaries while they still being reassembled.
- */
-int smbd_recv(struct smbd_connection *info, struct msghdr *msg)
-{
-	struct smbdirect_socket *sc = &info->socket;
-	struct smbdirect_recv_io *response;
-	struct smbdirect_data_transfer *data_transfer;
-	size_t size = iov_iter_count(&msg->msg_iter);
-	int to_copy, to_read, data_read, offset;
-	u32 data_length, remaining_data_length, data_offset;
-	int rc;
-
-	if (WARN_ON_ONCE(iov_iter_rw(&msg->msg_iter) == WRITE))
-		return -EINVAL; /* It's a bug in upper layer to get there */
-
-again:
-	/*
-	 * No need to hold the reassembly queue lock all the time as we are
-	 * the only one reading from the front of the queue. The transport
-	 * may add more entries to the back of the queue at the same time
-	 */
-	log_read(INFO, "size=%zd sc->recv_io.reassembly.data_length=%d\n", size,
-		sc->recv_io.reassembly.data_length);
-	if (sc->recv_io.reassembly.data_length >= size) {
-		int queue_length;
-		int queue_removed = 0;
-		unsigned long flags;
-
-		/*
-		 * Need to make sure reassembly_data_length is read before
-		 * reading reassembly_queue_length and calling
-		 * _get_first_reassembly. This call is lock free
-		 * as we never read at the end of the queue which are being
-		 * updated in SOFTIRQ as more data is received
-		 */
-		virt_rmb();
-		queue_length = sc->recv_io.reassembly.queue_length;
-		data_read = 0;
-		to_read = size;
-		offset = sc->recv_io.reassembly.first_entry_offset;
-		while (data_read < size) {
-			response = _get_first_reassembly(sc);
-			data_transfer = smbdirect_recv_io_payload(response);
-			data_length = le32_to_cpu(data_transfer->data_length);
-			remaining_data_length =
-				le32_to_cpu(
-					data_transfer->remaining_data_length);
-			data_offset = le32_to_cpu(data_transfer->data_offset);
-
-			/*
-			 * The upper layer expects RFC1002 length at the
-			 * beginning of the payload. Return it to indicate
-			 * the total length of the packet. This minimize the
-			 * change to upper layer packet processing logic. This
-			 * will be eventually remove when an intermediate
-			 * transport layer is added
-			 */
-			if (response->first_segment && size == 4) {
-				unsigned int rfc1002_len =
-					data_length + remaining_data_length;
-				__be32 rfc1002_hdr = cpu_to_be32(rfc1002_len);
-				if (copy_to_iter(&rfc1002_hdr, sizeof(rfc1002_hdr),
-						 &msg->msg_iter) != sizeof(rfc1002_hdr))
-					return -EFAULT;
-				data_read = 4;
-				response->first_segment = false;
-				log_read(INFO, "returning rfc1002 length %d\n",
-					rfc1002_len);
-				goto read_rfc1002_done;
-			}
-
-			to_copy = min_t(int, data_length - offset, to_read);
-			if (copy_to_iter((char *)data_transfer + data_offset + offset,
-					 to_copy, &msg->msg_iter) != to_copy)
-				return -EFAULT;
-
-			/* move on to the next buffer? */
-			if (to_copy == data_length - offset) {
-				queue_length--;
-				/*
-				 * No need to lock if we are not at the
-				 * end of the queue
-				 */
-				if (queue_length)
-					list_del(&response->list);
-				else {
-					spin_lock_irqsave(
-						&sc->recv_io.reassembly.lock, flags);
-					list_del(&response->list);
-					spin_unlock_irqrestore(
-						&sc->recv_io.reassembly.lock, flags);
-				}
-				queue_removed++;
-				sc->statistics.dequeue_reassembly_queue++;
-				put_receive_buffer(sc, response);
-				offset = 0;
-				log_read(INFO, "put_receive_buffer offset=0\n");
-			} else
-				offset += to_copy;
-
-			to_read -= to_copy;
-			data_read += to_copy;
-
-			log_read(INFO, "_get_first_reassembly memcpy %d bytes data_transfer_length-offset=%d after that to_read=%d data_read=%d offset=%d\n",
-				 to_copy, data_length - offset,
-				 to_read, data_read, offset);
-		}
-
-		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-		sc->recv_io.reassembly.data_length -= data_read;
-		sc->recv_io.reassembly.queue_length -= queue_removed;
-		spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-
-		sc->recv_io.reassembly.first_entry_offset = offset;
-		log_read(INFO, "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
-			 data_read, sc->recv_io.reassembly.data_length,
-			 sc->recv_io.reassembly.first_entry_offset);
-read_rfc1002_done:
-		return data_read;
-	}
-
-	log_read(INFO, "wait_event on more data\n");
-	rc = wait_event_interruptible(
-		sc->recv_io.reassembly.wait_queue,
-		sc->recv_io.reassembly.data_length >= size ||
-			sc->status != SMBDIRECT_SOCKET_CONNECTED);
-	/* Don't return any data if interrupted */
-	if (rc)
-		return rc;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
-		log_read(ERR, "disconnected\n");
-		return -ECONNABORTED;
-	}
-
-	goto again;
-}
-
-/*
- * Send data to transport
- * Each rqst is transported as a SMBDirect payload
- * rqst: the data to write
- * return value: 0 if successfully write, otherwise error code
- */
-int smbd_send(struct TCP_Server_Info *server,
-	int num_rqst, struct smb_rqst *rqst_array)
-{
-	struct smbd_connection *info = server->smbd_conn;
-	struct smbdirect_socket *sc = &info->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smb_rqst *rqst;
-	struct iov_iter iter;
-	struct smbdirect_send_batch batch;
-	unsigned int remaining_data_length, klen;
-	int rc, i, rqst_idx;
-	int error = 0;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return -EAGAIN;
-
-	/*
-	 * Add in the page array if there is one. The caller needs to set
-	 * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and
-	 * ends at page boundary
-	 */
-	remaining_data_length = 0;
-	for (i = 0; i < num_rqst; i++)
-		remaining_data_length += smb_rqst_len(server, &rqst_array[i]);
-
-	if (unlikely(remaining_data_length > sp->max_fragmented_send_size)) {
-		/* assertion: payload never exceeds negotiated maximum */
-		log_write(ERR, "payload size %d > max size %d\n",
-			remaining_data_length, sp->max_fragmented_send_size);
-		return -EINVAL;
-	}
-
-	log_write(INFO, "num_rqst=%d total length=%u\n",
-			num_rqst, remaining_data_length);
+	log_write(INFO, "num_rqst=%d total length=%u\n",
+			num_rqst, remaining_data_length);
 
 	rqst_idx = 0;
-	smbd_send_batch_init(&batch, false, 0);
+	batch = smbdirect_init_send_batch_storage(&bstorage, false, 0);
 	do {
 		rqst = &rqst_array[rqst_idx];
 
@@ -2575,25 +466,27 @@ int smbd_send(struct TCP_Server_Info *server,
 			klen += rqst->rq_iov[i].iov_len;
 		iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen);
 
-		rc = smbd_post_send_full_iter(sc, &batch, &iter, &remaining_data_length);
+		rc = smbd_post_send_full_iter(sc, batch, &iter, remaining_data_length);
 		if (rc < 0) {
 			error = rc;
 			break;
 		}
+		remaining_data_length -= rc;
 
 		if (iov_iter_count(&rqst->rq_iter) > 0) {
 			/* And then the data pages if there are any */
-			rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter,
-						      &remaining_data_length);
+			rc = smbd_post_send_full_iter(sc, batch, &rqst->rq_iter,
+						      remaining_data_length);
 			if (rc < 0) {
 				error = rc;
 				break;
 			}
+			remaining_data_length -= rc;
 		}
 
 	} while (++rqst_idx < num_rqst);
 
-	rc = smbd_send_batch_flush(sc, &batch, true);
+	rc = smbdirect_connection_send_batch_flush(sc, batch, true);
 	if (unlikely(!rc && error))
 		rc = error;
 
@@ -2604,298 +497,15 @@ int smbd_send(struct TCP_Server_Info *server,
 	 * that means all the I/Os have been out and we are good to return
 	 */
 
-	wait_event(sc->send_io.pending.zero_wait_queue,
-		atomic_read(&sc->send_io.pending.count) == 0 ||
-		sc->status != SMBDIRECT_SOCKET_CONNECTED);
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0)
-		rc = -EAGAIN;
-
-	return rc;
-}
-
-static void register_mr_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	struct smbdirect_mr_io *mr =
-		container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe);
-	struct smbdirect_socket *sc = mr->socket;
-
-	if (wc->status) {
-		log_rdma_mr(ERR, "status=%d\n", wc->status);
-		smbd_disconnect_rdma_connection(sc);
-	}
-}
-
-/*
- * The work queue function that recovers MRs
- * We need to call ib_dereg_mr() and ib_alloc_mr() before this MR can be used
- * again. Both calls are slow, so finish them in a workqueue. This will not
- * block I/O path.
- * There is one workqueue that recovers MRs, there is no need to lock as the
- * I/O requests calling smbd_register_mr will never update the links in the
- * mr_list.
- */
-static void smbd_mr_recovery_work(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, mr_io.recovery_work);
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_mr_io *smbdirect_mr;
-	int rc;
-
-	list_for_each_entry(smbdirect_mr, &sc->mr_io.all.list, list) {
-		if (smbdirect_mr->state == SMBDIRECT_MR_ERROR) {
-
-			/* recover this MR entry */
-			rc = ib_dereg_mr(smbdirect_mr->mr);
-			if (rc) {
-				log_rdma_mr(ERR,
-					"ib_dereg_mr failed rc=%x\n",
-					rc);
-				smbd_disconnect_rdma_connection(sc);
-				continue;
-			}
-
-			smbdirect_mr->mr = ib_alloc_mr(
-				sc->ib.pd, sc->mr_io.type,
-				sp->max_frmr_depth);
-			if (IS_ERR(smbdirect_mr->mr)) {
-				log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n",
-					    sc->mr_io.type,
-					    sp->max_frmr_depth);
-				smbd_disconnect_rdma_connection(sc);
-				continue;
-			}
-		} else
-			/* This MR is being used, don't recover it */
-			continue;
-
-		smbdirect_mr->state = SMBDIRECT_MR_READY;
-
-		/* smbdirect_mr->state is updated by this function
-		 * and is read and updated by I/O issuing CPUs trying
-		 * to get a MR, the call to atomic_inc_return
-		 * implicates a memory barrier and guarantees this
-		 * value is updated before waking up any calls to
-		 * get_mr() from the I/O issuing CPUs
-		 */
-		if (atomic_inc_return(&sc->mr_io.ready.count) == 1)
-			wake_up(&sc->mr_io.ready.wait_queue);
-	}
-}
-
-static void smbd_mr_disable_locked(struct smbdirect_mr_io *mr)
-{
-	struct smbdirect_socket *sc = mr->socket;
-
-	lockdep_assert_held(&mr->mutex);
-
-	if (mr->state == SMBDIRECT_MR_DISABLED)
-		return;
-
-	if (mr->mr)
-		ib_dereg_mr(mr->mr);
-	if (mr->sgt.nents)
-		ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
-	kfree(mr->sgt.sgl);
-
-	mr->mr = NULL;
-	mr->sgt.sgl = NULL;
-	mr->sgt.nents = 0;
-
-	mr->state = SMBDIRECT_MR_DISABLED;
-}
-
-static void smbd_mr_free_locked(struct kref *kref)
-{
-	struct smbdirect_mr_io *mr =
-		container_of(kref, struct smbdirect_mr_io, kref);
-
-	lockdep_assert_held(&mr->mutex);
-
-	/*
-	 * smbd_mr_disable_locked() should already be called!
-	 */
-	if (WARN_ON_ONCE(mr->state != SMBDIRECT_MR_DISABLED))
-		smbd_mr_disable_locked(mr);
-
-	mutex_unlock(&mr->mutex);
-	mutex_destroy(&mr->mutex);
-	kfree(mr);
-}
-
-static void destroy_mr_list(struct smbdirect_socket *sc)
-{
-	struct smbdirect_mr_io *mr, *tmp;
-	LIST_HEAD(all_list);
-	unsigned long flags;
-
-	disable_work_sync(&sc->mr_io.recovery_work);
-
-	spin_lock_irqsave(&sc->mr_io.all.lock, flags);
-	list_splice_tail_init(&sc->mr_io.all.list, &all_list);
-	spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
+	error = rc;
+	rc = smbdirect_connection_send_wait_zero_pending(sc);
+	if (unlikely(rc && !error))
+		error = -EAGAIN;
 
-	list_for_each_entry_safe(mr, tmp, &all_list, list) {
-		mutex_lock(&mr->mutex);
-
-		smbd_mr_disable_locked(mr);
-		list_del(&mr->list);
-		mr->socket = NULL;
-
-		/*
-		 * No kref_put_mutex() as it's already locked.
-		 *
-		 * If smbd_mr_free_locked() is called
-		 * and the mutex is unlocked and mr is gone,
-		 * in that case kref_put() returned 1.
-		 *
-		 * If kref_put() returned 0 we know that
-		 * smbd_mr_free_locked() didn't
-		 * run. Not by us nor by anyone else, as we
-		 * still hold the mutex, so we need to unlock.
-		 *
-		 * If the mr is still registered it will
-		 * be dangling (detached from the connection
-		 * waiting for smbd_deregister_mr() to be
-		 * called in order to free the memory.
-		 */
-		if (!kref_put(&mr->kref, smbd_mr_free_locked))
-			mutex_unlock(&mr->mutex);
-	}
-}
-
-/*
- * Allocate MRs used for RDMA read/write
- * The number of MRs will not exceed hardware capability in responder_resources
- * All MRs are kept in mr_list. The MR can be recovered after it's used
- * Recovery is done in smbd_mr_recovery_work. The content of list entry changes
- * as MRs are used and recovered for I/O, but the list links will not change
- */
-static int allocate_mr_list(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_mr_io *mr;
-	int ret;
-	u32 i;
-
-	if (sp->responder_resources == 0) {
-		log_rdma_mr(ERR, "responder_resources negotiated as 0\n");
-		return -EINVAL;
-	}
-
-	/* Allocate more MRs (2x) than hardware responder_resources */
-	for (i = 0; i < sp->responder_resources * 2; i++) {
-		mr = kzalloc_obj(*mr);
-		if (!mr) {
-			ret = -ENOMEM;
-			goto kzalloc_mr_failed;
-		}
-
-		kref_init(&mr->kref);
-		mutex_init(&mr->mutex);
-
-		mr->mr = ib_alloc_mr(sc->ib.pd,
-				     sc->mr_io.type,
-				     sp->max_frmr_depth);
-		if (IS_ERR(mr->mr)) {
-			ret = PTR_ERR(mr->mr);
-			log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n",
-				    sc->mr_io.type, sp->max_frmr_depth);
-			goto ib_alloc_mr_failed;
-		}
-
-		mr->sgt.sgl = kzalloc_objs(struct scatterlist,
-					   sp->max_frmr_depth);
-		if (!mr->sgt.sgl) {
-			ret = -ENOMEM;
-			log_rdma_mr(ERR, "failed to allocate sgl\n");
-			goto kcalloc_sgl_failed;
-		}
-		mr->state = SMBDIRECT_MR_READY;
-		mr->socket = sc;
-
-		list_add_tail(&mr->list, &sc->mr_io.all.list);
-		atomic_inc(&sc->mr_io.ready.count);
-	}
-
-	INIT_WORK(&sc->mr_io.recovery_work, smbd_mr_recovery_work);
+	if (unlikely(error))
+		return error;
 
 	return 0;
-
-kcalloc_sgl_failed:
-	ib_dereg_mr(mr->mr);
-ib_alloc_mr_failed:
-	mutex_destroy(&mr->mutex);
-	kfree(mr);
-kzalloc_mr_failed:
-	destroy_mr_list(sc);
-	return ret;
-}
-
-/*
- * Get a MR from mr_list. This function waits until there is at least one
- * MR available in the list. It may access the list while the
- * smbd_mr_recovery_work is recovering the MR list. This doesn't need a lock
- * as they never modify the same places. However, there may be several CPUs
- * issuing I/O trying to get MR at the same time, mr_list_lock is used to
- * protect this situation.
- */
-static struct smbdirect_mr_io *get_mr(struct smbdirect_socket *sc)
-{
-	struct smbdirect_mr_io *ret;
-	unsigned long flags;
-	int rc;
-again:
-	rc = wait_event_interruptible(sc->mr_io.ready.wait_queue,
-		atomic_read(&sc->mr_io.ready.count) ||
-		sc->status != SMBDIRECT_SOCKET_CONNECTED);
-	if (rc) {
-		log_rdma_mr(ERR, "wait_event_interruptible rc=%x\n", rc);
-		return NULL;
-	}
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
-		log_rdma_mr(ERR, "sc->status=%x\n", sc->status);
-		return NULL;
-	}
-
-	spin_lock_irqsave(&sc->mr_io.all.lock, flags);
-	list_for_each_entry(ret, &sc->mr_io.all.list, list) {
-		if (ret->state == SMBDIRECT_MR_READY) {
-			ret->state = SMBDIRECT_MR_REGISTERED;
-			kref_get(&ret->kref);
-			spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
-			atomic_dec(&sc->mr_io.ready.count);
-			atomic_inc(&sc->mr_io.used.count);
-			return ret;
-		}
-	}
-
-	spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
-	/*
-	 * It is possible that we could fail to get MR because other processes may
-	 * try to acquire a MR at the same time. If this is the case, retry it.
-	 */
-	goto again;
-}
-
-/*
- * Transcribe the pages from an iterator into an MR scatterlist.
- */
-static int smbd_iter_to_mr(struct iov_iter *iter,
-			   struct sg_table *sgt,
-			   unsigned int max_sg)
-{
-	int ret;
-
-	memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist));
-
-	ret = extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0);
-	WARN_ON(ret < 0);
-	if (sgt->nents > 0)
-		sg_mark_end(&sgt->sgl[sgt->nents - 1]);
-	return ret;
 }
 
 /*
@@ -2909,132 +519,18 @@ struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info,
 				 struct iov_iter *iter,
 				 bool writing, bool need_invalidate)
 {
-	struct smbdirect_socket *sc = &info->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_mr_io *mr;
-	int rc, num_pages;
-	struct ib_reg_wr *reg_wr;
-
-	num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1);
-	if (num_pages > sp->max_frmr_depth) {
-		log_rdma_mr(ERR, "num_pages=%d max_frmr_depth=%d\n",
-			num_pages, sp->max_frmr_depth);
-		WARN_ON_ONCE(1);
-		return NULL;
-	}
+	struct smbdirect_socket *sc = info->socket;
 
-	mr = get_mr(sc);
-	if (!mr) {
-		log_rdma_mr(ERR, "get_mr returning NULL\n");
+	if (!smbdirect_connection_is_connected(sc))
 		return NULL;
-	}
-
-	mutex_lock(&mr->mutex);
-
-	mr->dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
-	mr->need_invalidate = need_invalidate;
-	mr->sgt.nents = 0;
-	mr->sgt.orig_nents = 0;
-
-	log_rdma_mr(INFO, "num_pages=0x%x count=0x%zx depth=%u\n",
-		    num_pages, iov_iter_count(iter), sp->max_frmr_depth);
-	smbd_iter_to_mr(iter, &mr->sgt, sp->max_frmr_depth);
-
-	rc = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
-	if (!rc) {
-		log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x rc=%x\n",
-			    num_pages, mr->dir, rc);
-		goto dma_map_error;
-	}
-
-	rc = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE);
-	if (rc != mr->sgt.nents) {
-		log_rdma_mr(ERR,
-			    "ib_map_mr_sg failed rc = %d nents = %x\n",
-			    rc, mr->sgt.nents);
-		goto map_mr_error;
-	}
-
-	ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey));
-	reg_wr = &mr->wr;
-	reg_wr->wr.opcode = IB_WR_REG_MR;
-	mr->cqe.done = register_mr_done;
-	reg_wr->wr.wr_cqe = &mr->cqe;
-	reg_wr->wr.num_sge = 0;
-	reg_wr->wr.send_flags = IB_SEND_SIGNALED;
-	reg_wr->mr = mr->mr;
-	reg_wr->key = mr->mr->rkey;
-	reg_wr->access = writing ?
-			IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
-			IB_ACCESS_REMOTE_READ;
 
-	/*
-	 * There is no need for waiting for complemtion on ib_post_send
-	 * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution
-	 * on the next ib_post_send when we actually send I/O to remote peer
-	 */
-	rc = ib_post_send(sc->ib.qp, &reg_wr->wr, NULL);
-	if (!rc) {
-		/*
-		 * get_mr() gave us a reference
-		 * via kref_get(&mr->kref), we keep that and let
-		 * the caller use smbd_deregister_mr()
-		 * to remove it again.
-		 */
-		mutex_unlock(&mr->mutex);
-		return mr;
-	}
-
-	log_rdma_mr(ERR, "ib_post_send failed rc=%x reg_wr->key=%x\n",
-		rc, reg_wr->key);
-
-	/* If all failed, attempt to recover this MR by setting it SMBDIRECT_MR_ERROR*/
-map_mr_error:
-	ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
-
-dma_map_error:
-	mr->sgt.nents = 0;
-	mr->state = SMBDIRECT_MR_ERROR;
-	if (atomic_dec_and_test(&sc->mr_io.used.count))
-		wake_up(&sc->mr_io.cleanup.wait_queue);
-
-	smbd_disconnect_rdma_connection(sc);
-
-	/*
-	 * get_mr() gave us a reference
-	 * via kref_get(&mr->kref), we need to remove it again
-	 * on error.
-	 *
-	 * No kref_put_mutex() as it's already locked.
-	 *
-	 * If smbd_mr_free_locked() is called
-	 * and the mutex is unlocked and mr is gone,
-	 * in that case kref_put() returned 1.
-	 *
-	 * If kref_put() returned 0 we know that
-	 * smbd_mr_free_locked() didn't
-	 * run. Not by us nor by anyone else, as we
-	 * still hold the mutex, so we need to unlock.
-	 */
-	if (!kref_put(&mr->kref, smbd_mr_free_locked))
-		mutex_unlock(&mr->mutex);
-
-	return NULL;
+	return smbdirect_connection_register_mr_io(sc, iter, writing, need_invalidate);
 }
 
-static void local_inv_done(struct ib_cq *cq, struct ib_wc *wc)
+void smbd_mr_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
+				    struct smbdirect_buffer_descriptor_v1 *v1)
 {
-	struct smbdirect_mr_io *smbdirect_mr;
-	struct ib_cqe *cqe;
-
-	cqe = wc->wr_cqe;
-	smbdirect_mr = container_of(cqe, struct smbdirect_mr_io, cqe);
-	smbdirect_mr->state = SMBDIRECT_MR_INVALIDATED;
-	if (wc->status != IB_WC_SUCCESS) {
-		log_rdma_mr(ERR, "invalidate failed status=%x\n", wc->status);
-		smbdirect_mr->state = SMBDIRECT_MR_ERROR;
-	}
-	complete(&smbdirect_mr->invalidate_done);
+	smbdirect_mr_io_fill_buffer_descriptor(mr, v1);
 }
 
 /*
@@ -3045,300 +541,20 @@ static void local_inv_done(struct ib_cq *cq, struct ib_wc *wc)
  */
 void smbd_deregister_mr(struct smbdirect_mr_io *mr)
 {
-	struct smbdirect_socket *sc = mr->socket;
-
-	mutex_lock(&mr->mutex);
-	if (mr->state == SMBDIRECT_MR_DISABLED)
-		goto put_kref;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
-		smbd_mr_disable_locked(mr);
-		goto put_kref;
-	}
-
-	if (mr->need_invalidate) {
-		struct ib_send_wr *wr = &mr->inv_wr;
-		int rc;
-
-		/* Need to finish local invalidation before returning */
-		wr->opcode = IB_WR_LOCAL_INV;
-		mr->cqe.done = local_inv_done;
-		wr->wr_cqe = &mr->cqe;
-		wr->num_sge = 0;
-		wr->ex.invalidate_rkey = mr->mr->rkey;
-		wr->send_flags = IB_SEND_SIGNALED;
-
-		init_completion(&mr->invalidate_done);
-		rc = ib_post_send(sc->ib.qp, wr, NULL);
-		if (rc) {
-			log_rdma_mr(ERR, "ib_post_send failed rc=%x\n", rc);
-			smbd_mr_disable_locked(mr);
-			smbd_disconnect_rdma_connection(sc);
-			goto done;
-		}
-		wait_for_completion(&mr->invalidate_done);
-		mr->need_invalidate = false;
-	} else
-		/*
-		 * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED
-		 * and defer to mr_recovery_work to recover the MR for next use
-		 */
-		mr->state = SMBDIRECT_MR_INVALIDATED;
-
-	if (mr->sgt.nents) {
-		ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
-		mr->sgt.nents = 0;
-	}
-
-	if (mr->state == SMBDIRECT_MR_INVALIDATED) {
-		mr->state = SMBDIRECT_MR_READY;
-		if (atomic_inc_return(&sc->mr_io.ready.count) == 1)
-			wake_up(&sc->mr_io.ready.wait_queue);
-	} else
-		/*
-		 * Schedule the work to do MR recovery for future I/Os MR
-		 * recovery is slow and don't want it to block current I/O
-		 */
-		queue_work(sc->workqueue, &sc->mr_io.recovery_work);
-
-done:
-	if (atomic_dec_and_test(&sc->mr_io.used.count))
-		wake_up(&sc->mr_io.cleanup.wait_queue);
-
-put_kref:
-	/*
-	 * No kref_put_mutex() as it's already locked.
-	 *
-	 * If smbd_mr_free_locked() is called
-	 * and the mutex is unlocked and mr is gone,
-	 * in that case kref_put() returned 1.
-	 *
-	 * If kref_put() returned 0 we know that
-	 * smbd_mr_free_locked() didn't
-	 * run. Not by us nor by anyone else, as we
-	 * still hold the mutex, so we need to unlock
-	 * and keep the mr in SMBDIRECT_MR_READY or
-	 * SMBDIRECT_MR_ERROR state.
-	 */
-	if (!kref_put(&mr->kref, smbd_mr_free_locked))
-		mutex_unlock(&mr->mutex);
-}
-
-static bool smb_set_sge(struct smb_extract_to_rdma *rdma,
-			struct page *lowest_page, size_t off, size_t len)
-{
-	struct ib_sge *sge = &rdma->sge[rdma->nr_sge];
-	u64 addr;
-
-	addr = ib_dma_map_page(rdma->device, lowest_page,
-			       off, len, rdma->direction);
-	if (ib_dma_mapping_error(rdma->device, addr))
-		return false;
-
-	sge->addr   = addr;
-	sge->length = len;
-	sge->lkey   = rdma->local_dma_lkey;
-	rdma->nr_sge++;
-	return true;
-}
-
-/*
- * Extract page fragments from a BVEC-class iterator and add them to an RDMA
- * element list.  The pages are not pinned.
- */
-static ssize_t smb_extract_bvec_to_rdma(struct iov_iter *iter,
-					struct smb_extract_to_rdma *rdma,
-					ssize_t maxsize)
-{
-	const struct bio_vec *bv = iter->bvec;
-	unsigned long start = iter->iov_offset;
-	unsigned int i;
-	ssize_t ret = 0;
-
-	for (i = 0; i < iter->nr_segs; i++) {
-		size_t off, len;
-
-		len = bv[i].bv_len;
-		if (start >= len) {
-			start -= len;
-			continue;
-		}
-
-		len = min_t(size_t, maxsize, len - start);
-		off = bv[i].bv_offset + start;
-
-		if (!smb_set_sge(rdma, bv[i].bv_page, off, len))
-			return -EIO;
-
-		ret += len;
-		maxsize -= len;
-		if (rdma->nr_sge >= rdma->max_sge || maxsize <= 0)
-			break;
-		start = 0;
-	}
-
-	if (ret > 0)
-		iov_iter_advance(iter, ret);
-	return ret;
-}
-
-/*
- * Extract fragments from a KVEC-class iterator and add them to an RDMA list.
- * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers.
- * The pages are not pinned.
- */
-static ssize_t smb_extract_kvec_to_rdma(struct iov_iter *iter,
-					struct smb_extract_to_rdma *rdma,
-					ssize_t maxsize)
-{
-	const struct kvec *kv = iter->kvec;
-	unsigned long start = iter->iov_offset;
-	unsigned int i;
-	ssize_t ret = 0;
-
-	for (i = 0; i < iter->nr_segs; i++) {
-		struct page *page;
-		unsigned long kaddr;
-		size_t off, len, seg;
-
-		len = kv[i].iov_len;
-		if (start >= len) {
-			start -= len;
-			continue;
-		}
-
-		kaddr = (unsigned long)kv[i].iov_base + start;
-		off = kaddr & ~PAGE_MASK;
-		len = min_t(size_t, maxsize, len - start);
-		kaddr &= PAGE_MASK;
-
-		maxsize -= len;
-		do {
-			seg = min_t(size_t, len, PAGE_SIZE - off);
-
-			if (is_vmalloc_or_module_addr((void *)kaddr))
-				page = vmalloc_to_page((void *)kaddr);
-			else
-				page = virt_to_page((void *)kaddr);
-
-			if (!smb_set_sge(rdma, page, off, seg))
-				return -EIO;
-
-			ret += seg;
-			len -= seg;
-			kaddr += PAGE_SIZE;
-			off = 0;
-		} while (len > 0 && rdma->nr_sge < rdma->max_sge);
-
-		if (rdma->nr_sge >= rdma->max_sge || maxsize <= 0)
-			break;
-		start = 0;
-	}
-
-	if (ret > 0)
-		iov_iter_advance(iter, ret);
-	return ret;
-}
-
-/*
- * Extract folio fragments from a FOLIOQ-class iterator and add them to an RDMA
- * list.  The folios are not pinned.
- */
-static ssize_t smb_extract_folioq_to_rdma(struct iov_iter *iter,
-					  struct smb_extract_to_rdma *rdma,
-					  ssize_t maxsize)
-{
-	const struct folio_queue *folioq = iter->folioq;
-	unsigned int slot = iter->folioq_slot;
-	ssize_t ret = 0;
-	size_t offset = iter->iov_offset;
-
-	BUG_ON(!folioq);
-
-	if (slot >= folioq_nr_slots(folioq)) {
-		folioq = folioq->next;
-		if (WARN_ON_ONCE(!folioq))
-			return -EIO;
-		slot = 0;
-	}
-
-	do {
-		struct folio *folio = folioq_folio(folioq, slot);
-		size_t fsize = folioq_folio_size(folioq, slot);
-
-		if (offset < fsize) {
-			size_t part = umin(maxsize, fsize - offset);
-
-			if (!smb_set_sge(rdma, folio_page(folio, 0), offset, part))
-				return -EIO;
-
-			offset += part;
-			ret += part;
-			maxsize -= part;
-		}
-
-		if (offset >= fsize) {
-			offset = 0;
-			slot++;
-			if (slot >= folioq_nr_slots(folioq)) {
-				if (!folioq->next) {
-					WARN_ON_ONCE(ret < iter->count);
-					break;
-				}
-				folioq = folioq->next;
-				slot = 0;
-			}
-		}
-	} while (rdma->nr_sge < rdma->max_sge && maxsize > 0);
-
-	iter->folioq = folioq;
-	iter->folioq_slot = slot;
-	iter->iov_offset = offset;
-	iter->count -= ret;
-	return ret;
+	smbdirect_connection_deregister_mr_io(mr);
 }
 
-/*
- * Extract page fragments from up to the given amount of the source iterator
- * and build up an RDMA list that refers to all of those bits.  The RDMA list
- * is appended to, up to the maximum number of elements set in the parameter
- * block.
- *
- * The extracted page fragments are not pinned or ref'd in any way; if an
- * IOVEC/UBUF-type iterator is to be used, it should be converted to a
- * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some
- * way.
- */
-static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len,
-					struct smb_extract_to_rdma *rdma)
+void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m)
 {
-	ssize_t ret;
-	int before = rdma->nr_sge;
-
-	switch (iov_iter_type(iter)) {
-	case ITER_BVEC:
-		ret = smb_extract_bvec_to_rdma(iter, rdma, len);
-		break;
-	case ITER_KVEC:
-		ret = smb_extract_kvec_to_rdma(iter, rdma, len);
-		break;
-	case ITER_FOLIOQ:
-		ret = smb_extract_folioq_to_rdma(iter, rdma, len);
-		break;
-	default:
-		WARN_ON_ONCE(1);
-		return -EIO;
-	}
-
-	if (ret < 0) {
-		while (rdma->nr_sge > before) {
-			struct ib_sge *sge = &rdma->sge[rdma->nr_sge--];
+	if (!server->rdma)
+		return;
 
-			ib_dma_unmap_single(rdma->device, sge->addr, sge->length,
-					    rdma->direction);
-			sge->addr = 0;
-		}
+	if (!server->smbd_conn) {
+		seq_puts(m, "\nSMBDirect transport not available");
+		return;
 	}
 
-	return ret;
+	smbdirect_connection_legacy_debug_proc_show(server->smbd_conn->socket,
+						    server->rdma_readwrite_threshold,
+						    m);
 }
diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h
index 577d37dbeb8a3..0017d5b2de449 100644
--- a/fs/smb/client/smbdirect.h
+++ b/fs/smb/client/smbdirect.h
@@ -11,12 +11,8 @@
 #define cifs_rdma_enabled(server)	((server)->rdma)
 
 #include "cifsglob.h"
-#include <rdma/ib_verbs.h>
-#include <rdma/rdma_cm.h>
-#include <linux/mempool.h>
 
 #include "../common/smbdirect/smbdirect.h"
-#include "../common/smbdirect/smbdirect_socket.h"
 
 extern int rdma_readwrite_threshold;
 extern int smbd_max_frmr_depth;
@@ -27,17 +23,8 @@ extern int smbd_max_send_size;
 extern int smbd_send_credit_target;
 extern int smbd_receive_credit_max;
 
-/*
- * The context for the SMBDirect transport
- * Everything related to the transport is here. It has several logical parts
- * 1. RDMA related structures
- * 2. SMBDirect connection parameters
- * 3. Memory registrations
- * 4. Receive and reassembly queues for data receive path
- * 5. mempools for allocating packets
- */
 struct smbd_connection {
-	struct smbdirect_socket socket;
+	struct smbdirect_socket *socket;
 };
 
 /* Create a SMBDirect session */
@@ -60,8 +47,12 @@ int smbd_send(struct TCP_Server_Info *server,
 struct smbdirect_mr_io *smbd_register_mr(
 	struct smbd_connection *info, struct iov_iter *iter,
 	bool writing, bool need_invalidate);
+void smbd_mr_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
+				    struct smbdirect_buffer_descriptor_v1 *v1);
 void smbd_deregister_mr(struct smbdirect_mr_io *mr);
 
+void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m);
+
 #else
 #define cifs_rdma_enabled(server)	0
 struct smbd_connection {};
diff --git a/fs/smb/common/Makefile b/fs/smb/common/Makefile
index 9e0730a385fb1..e6ee65c31b5dc 100644
--- a/fs/smb/common/Makefile
+++ b/fs/smb/common/Makefile
@@ -4,3 +4,4 @@
 #
 
 obj-$(CONFIG_SMBFS) += cifs_md4.o
+obj-$(CONFIG_SMB_COMMON_SMBDIRECT) += smbdirect/
diff --git a/fs/smb/common/smbdirect/Kconfig b/fs/smb/common/smbdirect/Kconfig
new file mode 100644
index 0000000000000..d8d8cde238608
--- /dev/null
+++ b/fs/smb/common/smbdirect/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# smbdirect configuration
+
+config SMB_COMMON_SMBDIRECT
+	def_tristate n
+	depends on INFINIBAND && INFINIBAND_ADDR_TRANS
+	depends on m || INFINIBAND != m
+	select SG_POOL
diff --git a/fs/smb/common/smbdirect/Makefile b/fs/smb/common/smbdirect/Makefile
new file mode 100644
index 0000000000000..423f533e1002f
--- /dev/null
+++ b/fs/smb/common/smbdirect/Makefile
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Makefile for smbdirect support
+#
+
+obj-$(CONFIG_SMB_COMMON_SMBDIRECT) += smbdirect.o
+
+smbdirect-y := \
+	smbdirect_socket.o	\
+	smbdirect_connection.o	\
+	smbdirect_mr.o		\
+	smbdirect_rw.o		\
+	smbdirect_debug.o	\
+	smbdirect_connect.o	\
+	smbdirect_listen.o	\
+	smbdirect_accept.o	\
+	smbdirect_devices.o	\
+	smbdirect_main.o
diff --git a/fs/smb/common/smbdirect/smbdirect.h b/fs/smb/common/smbdirect/smbdirect.h
index 05cc6a9d0ccd3..e70a21eb57f01 100644
--- a/fs/smb/common/smbdirect/smbdirect.h
+++ b/fs/smb/common/smbdirect/smbdirect.h
@@ -7,6 +7,8 @@
 #ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__
 #define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__
 
+#include <linux/types.h>
+
 /* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */
 struct smbdirect_buffer_descriptor_v1 {
 	__le64 offset;
@@ -23,6 +25,9 @@ struct smbdirect_buffer_descriptor_v1 {
  * Some values are important for the upper layer.
  */
 struct smbdirect_socket_parameters {
+	__u64 flags;
+#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1)
+#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2)
 	__u32 resolve_addr_timeout_msec;
 	__u32 resolve_route_timeout_msec;
 	__u32 rdma_connect_timeout_msec;
@@ -41,4 +46,8 @@ struct smbdirect_socket_parameters {
 	__u32 keepalive_timeout_msec;
 } __packed;
 
+#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \
+		SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \
+		SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW)
+
 #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ */
diff --git a/fs/smb/common/smbdirect/smbdirect_accept.c b/fs/smb/common/smbdirect/smbdirect_accept.c
new file mode 100644
index 0000000000000..d6d5e6a3f5de7
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_accept.c
@@ -0,0 +1,857 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (C) 2018, LG Electronics.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+#include <net/sock.h>
+#include "../../common/smb2status.h"
+
+static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id,
+					       struct rdma_cm_event *event);
+static int smbdirect_accept_init_params(struct smbdirect_socket *sc);
+static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc);
+static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc);
+
+int smbdirect_accept_connect_request(struct smbdirect_socket *sc,
+				     const struct rdma_conn_param *param)
+{
+	struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io;
+	u8 peer_initiator_depth;
+	u8 peer_responder_resources;
+	struct rdma_conn_param conn_param;
+	__be32 ird_ord_hdr[2];
+	int ret;
+
+	if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED))
+		return -EINVAL;
+
+	/*
+	 * First set what the we as server are able to support
+	 */
+	sp->initiator_depth = min_t(u8, sp->initiator_depth,
+				    sc->ib.dev->attrs.max_qp_rd_atom);
+
+	peer_initiator_depth = param->initiator_depth;
+	peer_responder_resources = param->responder_resources;
+	smbdirect_connection_negotiate_rdma_resources(sc,
+						      peer_initiator_depth,
+						      peer_responder_resources,
+						      param);
+
+	ret = smbdirect_accept_init_params(sc);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_accept_init_params() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto init_params_failed;
+	}
+
+	ret = smbdirect_connection_create_qp(sc);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_create_qp() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto create_qp_failed;
+	}
+
+	ret = smbdirect_connection_create_mem_pools(sc);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_create_mem_pools() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto create_mem_failed;
+	}
+
+	recv_io = smbdirect_connection_get_recv_io(sc);
+	if (WARN_ON_ONCE(!recv_io)) {
+		ret = -EINVAL;
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_get_recv_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto get_recv_io_failed;
+	}
+	recv_io->cqe.done = smbdirect_accept_negotiate_recv_done;
+
+	/*
+	 * Now post the recv_io buffer in order to get
+	 * the negotiate request
+	 */
+	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ;
+	ret = smbdirect_connection_post_recv_io(recv_io);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_post_recv_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto post_recv_io_failed;
+	}
+	/*
+	 * From here recv_io is known to the RDMA QP and needs ib_drain_qp and
+	 * smbdirect_accept_negotiate_recv_done to cleanup...
+	 */
+	recv_io = NULL;
+
+	/* already checked with SMBDIRECT_CHECK_STATUS_WARN above */
+	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);
+	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;
+
+	/*
+	 * We already negotiated sp->initiator_depth
+	 * and sp->responder_resources above.
+	 */
+	memset(&conn_param, 0, sizeof(conn_param));
+	conn_param.initiator_depth = sp->initiator_depth;
+	conn_param.responder_resources = sp->responder_resources;
+
+	if (sc->rdma.legacy_iwarp) {
+		ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources);
+		ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth);
+		conn_param.private_data = ird_ord_hdr;
+		conn_param.private_data_len = sizeof(ird_ord_hdr);
+	} else {
+		conn_param.private_data = NULL;
+		conn_param.private_data_len = 0;
+	}
+	conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY;
+	conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY;
+	conn_param.flow_control = 0;
+
+	/* explicitly set above */
+	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED);
+	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;
+	sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED;
+	sc->rdma.cm_id->event_handler = smbdirect_accept_rdma_event_handler;
+	ret = rdma_accept(sc->rdma.cm_id, &conn_param);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"rdma_accept() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto rdma_accept_failed;
+	}
+
+	/*
+	 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
+	 * so that the timer will cause a disconnect.
+	 */
+	INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work);
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->negotiate_timeout_msec));
+
+	return 0;
+
+rdma_accept_failed:
+	/*
+	 * smbdirect_connection_destroy_qp() calls ib_drain_qp(),
+	 * so that smbdirect_accept_negotiate_recv_done() will
+	 * call smbdirect_connection_put_recv_io()
+	 */
+post_recv_io_failed:
+	if (recv_io)
+		smbdirect_connection_put_recv_io(recv_io);
+get_recv_io_failed:
+	smbdirect_connection_destroy_mem_pools(sc);
+create_mem_failed:
+	smbdirect_connection_destroy_qp(sc);
+create_qp_failed:
+init_params_failed:
+	return ret;
+}
+
+static int smbdirect_accept_init_params(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	int max_send_sges;
+	unsigned int maxpages;
+
+	/* need 3 more sge. because a SMB_DIRECT header, SMB2 header,
+	 * SMB2 response could be mapped.
+	 */
+	max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3;
+	if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) {
+		pr_err("max_send_size %d is too large\n", sp->max_send_size);
+		return -EINVAL;
+	}
+
+	/*
+	 * There is only a single batch credit
+	 */
+	atomic_set(&sc->send_io.bcredits.count, 1);
+
+	/*
+	 * Initialize the local credits to post
+	 * IB_WR_SEND[_WITH_INV].
+	 */
+	atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target);
+
+	if (sp->max_read_write_size) {
+		maxpages = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE);
+		sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev,
+							  sc->rdma.cm_id->port_num,
+							  maxpages);
+		sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max);
+		/* add one extra in order to handle unaligned pages */
+		sc->rw_io.credits.max += 1;
+	}
+
+	sc->recv_io.credits.target = 1;
+
+	atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max);
+
+	return 0;
+}
+
+static void smbdirect_accept_negotiate_recv_work(struct work_struct *work);
+
+static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_recv_io *recv_io =
+		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
+	struct smbdirect_socket *sc = recv_io->socket;
+	unsigned long flags;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		goto error;
+	}
+
+	smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO,
+		"smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n",
+		ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+
+	/*
+	 * This is an internal error!
+	 */
+	if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ))
+		goto error;
+
+	/*
+	 * Don't reset timer to the keepalive interval in
+	 * this will be done in smbdirect_accept_direct_negotiate_recv_work.
+	 */
+
+	ib_dma_sync_single_for_cpu(sc->ib.dev,
+				   recv_io->sge.addr,
+				   recv_io->sge.length,
+				   DMA_FROM_DEVICE);
+
+	/*
+	 * Only remember recv_io if it has enough bytes,
+	 * this gives smbdirect_accept_negotiate_recv_work enough
+	 * information in order to disconnect if it was not
+	 * valid.
+	 */
+	sc->recv_io.reassembly.full_packet_received = true;
+	if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req))
+		smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0);
+	else
+		smbdirect_connection_put_recv_io(recv_io);
+
+	/*
+	 * Some drivers (at least mlx5_ib and irdma) might post a
+	 * recv completion before RDMA_CM_EVENT_ESTABLISHED,
+	 * we need to adjust our expectation in that case.
+	 *
+	 * So we defer further processing of the negotiation
+	 * to smbdirect_accept_negotiate_recv_work().
+	 *
+	 * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED
+	 * we queue the work directly otherwise
+	 * smbdirect_accept_rdma_event_handler() will do it, when
+	 * RDMA_CM_EVENT_ESTABLISHED arrived.
+	 */
+	spin_lock_irqsave(&sc->connect.lock, flags);
+	if (!sc->first_error) {
+		INIT_WORK(&sc->connect.work, smbdirect_accept_negotiate_recv_work);
+		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)
+			queue_work(sc->workqueues.accept, &sc->connect.work);
+	}
+	spin_unlock_irqrestore(&sc->connect.lock, flags);
+
+	return;
+
+error:
+	/*
+	 * recv_io.posted.refill_work is still disabled,
+	 * so smbdirect_connection_put_recv_io() won't
+	 * start it.
+	 */
+	smbdirect_connection_put_recv_io(recv_io);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+}
+
+static void smbdirect_accept_negotiate_recv_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, connect.work);
+	struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io;
+	struct smbdirect_negotiate_req *nreq;
+	unsigned long flags;
+	u16 min_version;
+	u16 max_version;
+	u16 credits_requested;
+	u32 preferred_send_size;
+	u32 max_receive_size;
+	u32 max_fragmented_size;
+	u32 ntstatus;
+
+	if (sc->first_error)
+		return;
+
+	/*
+	 * make sure we won't start again...
+	 */
+	disable_work(work);
+
+	/*
+	 * Reset timer to the keepalive interval in
+	 * order to trigger our next keepalive message.
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->keepalive_interval_msec));
+
+	/*
+	 * If smbdirect_accept_negotiate_recv_done() detected an
+	 * invalid request we want to disconnect.
+	 */
+	recv_io = smbdirect_connection_reassembly_first_recv_io(sc);
+	if (!recv_io) {
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
+	sc->recv_io.reassembly.queue_length--;
+	list_del(&recv_io->list);
+	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+	smbdirect_connection_put_recv_io(recv_io);
+
+	if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED))
+		return;
+	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
+
+	/*
+	 * Note recv_io is already part of the free list,
+	 * as we just called smbdirect_connection_put_recv_io(),
+	 * but it won't be reused before we call
+	 * smbdirect_connection_recv_io_refill() below.
+	 */
+
+	nreq = (struct smbdirect_negotiate_req *)recv_io->packet;
+	min_version = le16_to_cpu(nreq->min_version);
+	max_version = le16_to_cpu(nreq->max_version);
+	credits_requested = le16_to_cpu(nreq->credits_requested);
+	preferred_send_size = le32_to_cpu(nreq->preferred_send_size);
+	max_receive_size = le32_to_cpu(nreq->max_receive_size);
+	max_fragmented_size = le32_to_cpu(nreq->max_fragmented_size);
+
+	smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO,
+		"ReqIn: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n",
+		"MinVersion=0x",
+		le16_to_cpu(nreq->min_version),
+		"MaxVersion=0x",
+		le16_to_cpu(nreq->max_version),
+		"CreditsRequested=",
+		le16_to_cpu(nreq->credits_requested),
+		"PreferredSendSize=",
+		le32_to_cpu(nreq->preferred_send_size),
+		"MaxRecvSize=",
+		le32_to_cpu(nreq->max_receive_size),
+		"MaxFragmentedSize=",
+		le32_to_cpu(nreq->max_fragmented_size));
+
+	if (!(min_version <= SMBDIRECT_V1 && max_version >= SMBDIRECT_V1)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: min_version=0x%x max_version=0x%x\n",
+			min_version, max_version);
+		ntstatus = le32_to_cpu(STATUS_NOT_SUPPORTED);
+		goto not_supported;
+	}
+
+	if (credits_requested == 0) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: credits_requested == 0\n");
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: max_receive_size=%u < %u\n",
+			max_receive_size,
+			SMBDIRECT_MIN_RECEIVE_SIZE);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: max_fragmented_size=%u < %u\n",
+			max_fragmented_size,
+			SMBDIRECT_MIN_FRAGMENTED_SIZE);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used.
+	 */
+	sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size);
+	sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE);
+
+	/*
+	 * The maximum fragmented upper-layer payload receive size supported
+	 *
+	 * Assume max_payload_per_credit is
+	 * smb_direct_receive_credit_max - 24 = 1340
+	 *
+	 * The maximum number would be
+	 * smb_direct_receive_credit_max * max_payload_per_credit
+	 *
+	 *                       1340 * 255 = 341700 (0x536C4)
+	 *
+	 * The minimum value from the spec is 131072 (0x20000)
+	 *
+	 * For now we use the logic we used in ksmbd before:
+	 *                 (1364 * 255) / 2 = 173910 (0x2A756)
+	 *
+	 * We need to adjust this here in case the peer
+	 * lowered sp->max_recv_size.
+	 *
+	 * TODO: instead of adjusting max_fragmented_recv_size
+	 * we should adjust the number of available buffers,
+	 * but for now we keep the logic as it was used
+	 * in ksmbd before.
+	 */
+	sp->max_fragmented_recv_size = (sp->recv_credit_max * sp->max_recv_size) / 2;
+
+	/*
+	 * We take the value from the peer, which is checked to be higher than 0,
+	 * but we limit it to the max value we support in order to have
+	 * the main logic simpler.
+	 */
+	sc->recv_io.credits.target = credits_requested;
+	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target,
+					   sp->recv_credit_max);
+
+	/*
+	 * Note nreq->max_receive_size was already checked against
+	 * SMBDIRECT_MIN_RECEIVE_SIZE above.
+	 */
+	sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size);
+
+	/*
+	 * Note nreq->max_fragmented_size was already checked against
+	 * SMBDIRECT_MIN_FRAGMENTED_SIZE above.
+	 */
+	sp->max_fragmented_send_size = max_fragmented_size;
+
+	if (sc->accept.listener) {
+		struct smbdirect_socket *lsc = sc->accept.listener;
+		unsigned long flags;
+
+		spin_lock_irqsave(&lsc->listen.lock, flags);
+		list_del(&sc->accept.list);
+		list_add_tail(&sc->accept.list, &lsc->listen.ready);
+		wake_up(&lsc->listen.wait_queue);
+		spin_unlock_irqrestore(&lsc->listen.lock, flags);
+
+		/*
+		 * smbdirect_socket_accept() will call
+		 * smbdirect_accept_negotiate_finish(nsc, 0);
+		 *
+		 * So that we don't send the negotiation
+		 * response that grants credits to the peer
+		 * before the socket is accepted by the
+		 * application.
+		 */
+		return;
+	}
+
+	ntstatus = le32_to_cpu(STATUS_SUCCESS);
+
+not_supported:
+	smbdirect_accept_negotiate_finish(sc, ntstatus);
+}
+
+void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io;
+	struct smbdirect_send_io *send_io;
+	struct smbdirect_negotiate_resp *nrep;
+	int posted;
+	u16 new_credits;
+	int ret;
+
+	if (ntstatus)
+		goto not_supported;
+
+	/*
+	 * Prepare for receiving data_transfer messages
+	 */
+	sc->recv_io.reassembly.full_packet_received = true;
+	sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;
+	list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
+		recv_io->cqe.done = smbdirect_connection_recv_io_done;
+	recv_io = NULL;
+
+	/*
+	 * We should at least post 1 smbdirect_recv_io!
+	 */
+	posted = smbdirect_connection_recv_io_refill(sc);
+	if (posted < 1) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_recv_io_refill() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(posted));
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * The response will grant credits for all posted
+	 * smbdirect_recv_io messages.
+	 */
+	new_credits = smbdirect_connection_grant_recv_credits(sc);
+
+not_supported:
+	send_io = smbdirect_connection_alloc_send_io(sc);
+	if (IS_ERR(send_io)) {
+		ret = PTR_ERR(send_io);
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_alloc_send_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_socket_schedule_cleanup(sc, ret);
+		return;
+	}
+	send_io->cqe.done = smbdirect_accept_negotiate_send_done;
+
+	nrep = (struct smbdirect_negotiate_resp *)send_io->packet;
+	nrep->min_version = cpu_to_le16(SMBDIRECT_V1);
+	nrep->max_version = cpu_to_le16(SMBDIRECT_V1);
+	if (ntstatus == 0) {
+		nrep->negotiated_version = cpu_to_le16(SMBDIRECT_V1);
+		nrep->reserved = 0;
+		nrep->credits_requested = cpu_to_le16(sp->send_credit_target);
+		nrep->credits_granted = cpu_to_le16(new_credits);
+		nrep->status = cpu_to_le32(ntstatus);
+		nrep->max_readwrite_size = cpu_to_le32(sp->max_read_write_size);
+		nrep->preferred_send_size = cpu_to_le32(sp->max_send_size);
+		nrep->max_receive_size = cpu_to_le32(sp->max_recv_size);
+		nrep->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size);
+	} else {
+		nrep->negotiated_version = 0;
+		nrep->reserved = 0;
+		nrep->credits_requested = 0;
+		nrep->credits_granted = 0;
+		nrep->status = cpu_to_le32(ntstatus);
+		nrep->max_readwrite_size = 0;
+		nrep->preferred_send_size = 0;
+		nrep->max_receive_size = 0;
+		nrep->max_fragmented_size = 0;
+	}
+
+	smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO,
+		"RepOut: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n",
+		"MinVersion=0x",
+		le16_to_cpu(nrep->min_version),
+		"MaxVersion=0x",
+		le16_to_cpu(nrep->max_version),
+		"NegotiatedVersion=0x",
+		le16_to_cpu(nrep->negotiated_version),
+		"CreditsRequested=",
+		le16_to_cpu(nrep->credits_requested),
+		"CreditsGranted=",
+		le16_to_cpu(nrep->credits_granted),
+		"Status=0x",
+		le32_to_cpu(nrep->status),
+		"MaxReadWriteSize=",
+		le32_to_cpu(nrep->max_readwrite_size),
+		"PreferredSendSize=",
+		le32_to_cpu(nrep->preferred_send_size),
+		"MaxRecvSize=",
+		le32_to_cpu(nrep->max_receive_size),
+		"MaxFragmentedSize=",
+		le32_to_cpu(nrep->max_fragmented_size));
+
+	send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev,
+						 nrep,
+						 sizeof(*nrep),
+						 DMA_TO_DEVICE);
+	ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"ib_dma_mapping_error() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_connection_free_send_io(send_io);
+		smbdirect_socket_schedule_cleanup(sc, ret);
+		return;
+	}
+
+	send_io->sge[0].length = sizeof(*nrep);
+	send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey;
+	send_io->num_sge = 1;
+
+	ib_dma_sync_single_for_device(sc->ib.dev,
+				      send_io->sge[0].addr,
+				      send_io->sge[0].length,
+				      DMA_TO_DEVICE);
+
+	send_io->wr.next = NULL;
+	send_io->wr.wr_cqe = &send_io->cqe;
+	send_io->wr.sg_list = send_io->sge;
+	send_io->wr.num_sge = send_io->num_sge;
+	send_io->wr.opcode = IB_WR_SEND;
+	send_io->wr.send_flags = IB_SEND_SIGNALED;
+
+	ret = smbdirect_connection_post_send_wr(sc, &send_io->wr);
+	if (ret) {
+		/* if we reach here, post send failed */
+		smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_post_send_wr() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		/*
+		 * Note smbdirect_connection_free_send_io()
+		 * does ib_dma_unmap_page()
+		 */
+		smbdirect_connection_free_send_io(send_io);
+		smbdirect_socket_schedule_cleanup(sc, ret);
+		return;
+	}
+
+	/*
+	 * smbdirect_accept_negotiate_send_done
+	 * will do all remaining work...
+	 */
+}
+
+static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_send_io *send_io =
+		container_of(wc->wr_cqe, struct smbdirect_send_io, cqe);
+	struct smbdirect_socket *sc = send_io->socket;
+	struct smbdirect_negotiate_resp *nrep;
+	u32 ntstatus;
+
+	smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO,
+		"smbdirect_send_io completed. status='%s (%d)', opcode=%d\n",
+		ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+
+	nrep = (struct smbdirect_negotiate_resp *)send_io->packet;
+	ntstatus = le32_to_cpu(nrep->status);
+
+	/* Note this frees wc->wr_cqe, but not wc */
+	smbdirect_connection_free_send_io(send_io);
+	atomic_dec(&sc->send_io.pending.count);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * If we send a smbdirect_negotiate_resp without NT_STATUS_OK (0)
+	 * we need to disconnect now.
+	 *
+	 * Otherwise smbdirect_connection_negotiation_done()
+	 * will setup all required things and wake up
+	 * the waiter.
+	 */
+	if (ntstatus)
+		smbdirect_socket_schedule_cleanup(sc, -EOPNOTSUPP);
+	else
+		smbdirect_connection_negotiation_done(sc);
+}
+
+static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id,
+					       struct rdma_cm_event *event)
+{
+	struct smbdirect_socket *sc = id->context;
+	unsigned long flags;
+
+	/*
+	 * cma_cm_event_handler() has
+	 * lockdep_assert_held(&id_priv->handler_mutex);
+	 *
+	 * Mutexes are not allowed in interrupts,
+	 * and we rely on not being in an interrupt here,
+	 * as we might sleep.
+	 *
+	 * We didn't timeout so we cancel our idle timer,
+	 * it will be scheduled again if needed.
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	if (event->status || event->event != sc->rdma.expected_event) {
+		int ret = -ECONNABORTED;
+
+		if (event->event == RDMA_CM_EVENT_REJECTED)
+			ret = -ECONNREFUSED;
+		if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+			ret = -ENETDOWN;
+		if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+			ret = event->status;
+
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			rdma_event_msg(sc->rdma.expected_event),
+			rdma_event_msg(event->event),
+			event->status,
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+
+		smbdirect_socket_schedule_cleanup(sc, ret);
+		return 0;
+	}
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"%s (first_error=%1pe) event=%s\n",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+		rdma_event_msg(event->event));
+
+	if (sc->first_error)
+		return 0;
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ESTABLISHED:
+		smbdirect_connection_rdma_established(sc);
+
+		/*
+		 * Some drivers (at least mlx5_ib and irdma) might post a
+		 * recv completion before RDMA_CM_EVENT_ESTABLISHED,
+		 * we need to adjust our expectation in that case.
+		 *
+		 * If smbdirect_accept_negotiate_recv_done was called first
+		 * it initialized sc->connect.work only for us to
+		 * start, so that we turned into
+		 * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before
+		 * smbdirect_accept_negotiate_recv_work() runs.
+		 *
+		 * If smbdirect_accept_negotiate_recv_done didn't happen
+		 * yet. sc->connect.work is still be disabled and
+		 * queue_work() is a no-op.
+		 */
+		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING))
+			return 0;
+		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
+		spin_lock_irqsave(&sc->connect.lock, flags);
+		if (!sc->first_error)
+			queue_work(sc->workqueues.accept, &sc->connect.work);
+		spin_unlock_irqrestore(&sc->connect.lock, flags);
+
+		/*
+		 * wait for smbdirect_accept_negotiate_recv_done()
+		 * to get the negotiate request.
+		 */
+		return 0;
+
+	default:
+		break;
+	}
+
+	/*
+	 * This is an internal error
+	 */
+	WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+	return 0;
+}
+
+static long smbdirect_socket_wait_for_accept(struct smbdirect_socket *lsc, long timeo)
+{
+	long ret;
+
+	ret = wait_event_interruptible_timeout(lsc->listen.wait_queue,
+					       !list_empty_careful(&lsc->listen.ready) ||
+					       lsc->status != SMBDIRECT_SOCKET_LISTENING ||
+					       lsc->first_error,
+					       timeo);
+	if (lsc->status != SMBDIRECT_SOCKET_LISTENING)
+		return -EINVAL;
+	if (lsc->first_error)
+		return lsc->first_error;
+	if (!ret)
+		ret = -ETIMEDOUT;
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc,
+						 long timeo,
+						 struct proto_accept_arg *arg)
+{
+	struct smbdirect_socket *nsc;
+	unsigned long flags;
+
+	if (lsc->status != SMBDIRECT_SOCKET_LISTENING) {
+		arg->err = -EINVAL;
+		return NULL;
+	}
+
+	if (lsc->first_error) {
+		arg->err = lsc->first_error;
+		return NULL;
+	}
+
+	if (list_empty_careful(&lsc->listen.ready)) {
+		int ret;
+
+		if (timeo == 0) {
+			arg->err = -EAGAIN;
+			return NULL;
+		}
+
+		ret = smbdirect_socket_wait_for_accept(lsc, timeo);
+		if (ret) {
+			arg->err = ret;
+			return NULL;
+		}
+	}
+
+	spin_lock_irqsave(&lsc->listen.lock, flags);
+	nsc = list_first_entry_or_null(&lsc->listen.ready,
+				       struct smbdirect_socket,
+				       accept.list);
+	if (nsc) {
+		nsc->accept.listener = NULL;
+		list_del_init_careful(&nsc->accept.list);
+		arg->is_empty = list_empty_careful(&lsc->listen.ready);
+	}
+	spin_unlock_irqrestore(&lsc->listen.lock, flags);
+	if (!nsc) {
+		arg->err = -EAGAIN;
+		return NULL;
+	}
+
+	/*
+	 * We did not send the negotiation response
+	 * yet, so we did not grant any credits to the client,
+	 * so it didn't grant any credits to us.
+	 *
+	 * The caller expects a connected socket
+	 * now as there are no credits anyway.
+	 *
+	 * Then we send the negotiation response in
+	 * order to grant credits to the peer.
+	 */
+	nsc->status = SMBDIRECT_SOCKET_CONNECTED;
+	smbdirect_accept_negotiate_finish(nsc, 0);
+
+	return nsc;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_accept);
diff --git a/fs/smb/common/smbdirect/smbdirect_connect.c b/fs/smb/common/smbdirect/smbdirect_connect.c
new file mode 100644
index 0000000000000..2b54f79dba432
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_connect.c
@@ -0,0 +1,925 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (c) 2012,2016,2017,2025 Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+#include "../../common/smb2status.h"
+
+static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc);
+static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc,
+					  const struct sockaddr *src,
+					  const struct sockaddr *dst);
+static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id,
+						struct rdma_cm_event *event);
+static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc);
+static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc);
+static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc);
+
+int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst)
+{
+	const struct sockaddr *src = NULL;
+	union {
+		struct sockaddr sa;
+		struct sockaddr_storage ss;
+	} src_addr = {
+		.sa = {
+			.sa_family = AF_UNSPEC,
+		},
+	};
+	int ret;
+
+	if (sc->first_error)
+		return -ENOTCONN;
+
+	if (sc->status != SMBDIRECT_SOCKET_CREATED)
+		return -EALREADY;
+
+	if (WARN_ON_ONCE(!sc->rdma.cm_id))
+		return -EINVAL;
+
+	src_addr.ss = sc->rdma.cm_id->route.addr.src_addr;
+	if (src_addr.sa.sa_family != AF_UNSPEC)
+		src = &src_addr.sa;
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"connect: src: %pISpsfc dst: %pISpsfc\n",
+		src, dst);
+
+	ret = smbdirect_connect_setup_connection(sc);
+	if (ret)
+		return ret;
+
+	ret = smbdirect_connect_resolve_addr(sc, src, dst);
+	if (ret)
+		return ret;
+
+	/*
+	 * The rest happens async via smbdirect_connect_rdma_event_handler()
+	 * the caller will decide to wait or not.
+	 */
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect);
+
+static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc)
+{
+	rdma_lock_handler(sc->rdma.cm_id);
+	sc->rdma.cm_id->event_handler = smbdirect_connect_rdma_event_handler;
+	rdma_unlock_handler(sc->rdma.cm_id);
+
+	if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED))
+		return -EINVAL;
+	sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED;
+
+	return 0;
+}
+
+static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc,
+					  const struct sockaddr *src,
+					  const struct sockaddr *dst)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct sockaddr *src_addr = NULL;
+	struct sockaddr *dst_addr = NULL;
+	int ret;
+
+	src_addr = (struct sockaddr *)src;
+	if (src_addr && src_addr->sa_family == AF_UNSPEC)
+		src_addr = NULL;
+	dst_addr = (struct sockaddr *)dst;
+
+	if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED))
+		return -EINVAL;
+	sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING;
+	sc->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED;
+	ret = rdma_resolve_addr(sc->rdma.cm_id, src_addr, dst_addr,
+				sp->resolve_addr_timeout_msec);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"rdma_resolve_addr() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smbdirect_connect_resolve_route(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	int ret;
+
+	if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED))
+		return sc->first_error;
+	sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING;
+	sc->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+	ret = rdma_resolve_route(sc->rdma.cm_id, sp->resolve_route_timeout_msec);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"rdma_resolve_route() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc)
+{
+	struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct rdma_conn_param conn_param;
+	__be32 ird_ord_hdr[2];
+	int ret;
+
+	sc->ib.dev = sc->rdma.cm_id->device;
+
+	if (!smbdirect_frwr_is_supported(&sc->ib.dev->attrs)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"Fast Registration Work Requests (FRWR) is not supported device %.*s\n",
+			IB_DEVICE_NAME_MAX,
+			sc->ib.dev->name);
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"Device capability flags = %llx max_fast_reg_page_list_len = %u\n",
+			sc->ib.dev->attrs.device_cap_flags,
+			sc->ib.dev->attrs.max_fast_reg_page_list_len);
+		return -EPROTONOSUPPORT;
+	}
+
+	if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB &&
+	    !rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n",
+			IB_DEVICE_NAME_MAX,
+			sc->ib.dev->name,
+			rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num),
+			&sc->rdma.cm_id->route.addr.src_addr,
+			&sc->rdma.cm_id->route.addr.dst_addr);
+		return -EPROTONOSUPPORT;
+	}
+	if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW &&
+	    !rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n",
+			IB_DEVICE_NAME_MAX,
+			sc->ib.dev->name,
+			rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num),
+			&sc->rdma.cm_id->route.addr.src_addr,
+			&sc->rdma.cm_id->route.addr.dst_addr);
+		return -EPROTONOSUPPORT;
+	}
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"rdma connect: device: %.*s local: %pISpsfc remote: %pISpsfc\n",
+		IB_DEVICE_NAME_MAX,
+		sc->ib.dev->name,
+		&sc->rdma.cm_id->route.addr.src_addr,
+		&sc->rdma.cm_id->route.addr.dst_addr);
+
+	sp->max_frmr_depth = min_t(u32, sp->max_frmr_depth,
+				   sc->ib.dev->attrs.max_fast_reg_page_list_len);
+	sc->mr_io.type = IB_MR_TYPE_MEM_REG;
+	if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)
+		sc->mr_io.type = IB_MR_TYPE_SG_GAPS;
+
+	sp->responder_resources = min_t(u8, sp->responder_resources,
+					sc->ib.dev->attrs.max_qp_rd_atom);
+	smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO,
+		"responder_resources=%d\n",
+		sp->responder_resources);
+
+	ret = smbdirect_connection_create_qp(sc);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_create_qp() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	memset(&conn_param, 0, sizeof(conn_param));
+	conn_param.initiator_depth = sp->initiator_depth;
+	conn_param.responder_resources = sp->responder_resources;
+
+	/* Need to send IRD/ORD in private data for iWARP */
+	if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) {
+		ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources);
+		ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth);
+		conn_param.private_data = ird_ord_hdr;
+		conn_param.private_data_len = sizeof(ird_ord_hdr);
+	} else {
+		conn_param.private_data = NULL;
+		conn_param.private_data_len = 0;
+	}
+
+	conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY;
+	conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY;
+	conn_param.flow_control = 0;
+
+	if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED))
+		return sc->first_error;
+	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;
+	sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED;
+	ret = rdma_connect_locked(sc->rdma.cm_id, &conn_param);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"rdma_connect_locked() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	/*
+	 * start with the rdma connect timeout and SMBDIRECT_KEEPALIVE_PENDING
+	 * so that the timer will cause a disconnect.
+	 */
+	INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work);
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->rdma_connect_timeout_msec));
+
+	return 0;
+}
+
+static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id,
+						struct rdma_cm_event *event)
+{
+	struct smbdirect_socket *sc = id->context;
+	u8 peer_initiator_depth;
+	u8 peer_responder_resources;
+	int ret;
+
+	/*
+	 * cma_cm_event_handler() has
+	 * lockdep_assert_held(&id_priv->handler_mutex);
+	 *
+	 * Mutexes are not allowed in interrupts,
+	 * and we rely on not being in an interrupt here,
+	 * as we might sleep.
+	 *
+	 * We didn't timeout so we cancel our idle timer,
+	 * it will be scheduled again if needed.
+	 */
+	WARN_ON_ONCE(in_interrupt());
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	cancel_delayed_work_sync(&sc->idle.timer_work);
+
+	if (event->status || event->event != sc->rdma.expected_event) {
+		int lvl = SMBDIRECT_LOG_ERR;
+
+		ret = -ECONNABORTED;
+
+		if (event->event == RDMA_CM_EVENT_REJECTED)
+			ret = -ECONNREFUSED;
+		if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+			ret = -ENETDOWN;
+		if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+			ret = event->status;
+
+		if (ret == -ENODEV)
+			lvl = SMBDIRECT_LOG_INFO;
+
+		smbdirect_log_rdma_event(sc, lvl,
+			"%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			rdma_event_msg(sc->rdma.expected_event),
+			rdma_event_msg(event->event),
+			event->status,
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+
+		smbdirect_socket_schedule_cleanup_lvl(sc,
+						      lvl,
+						      ret);
+		return 0;
+	}
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"%s (first_error=%1pe) event=%s\n",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+		rdma_event_msg(event->event));
+
+	if (sc->first_error)
+		return 0;
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING))
+			return 0;
+		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED;
+
+		ret = smbdirect_connect_resolve_route(sc);
+		if (ret)
+			smbdirect_socket_schedule_cleanup(sc, ret);
+		return 0;
+
+	case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING))
+			return 0;
+		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;
+
+		ret = smbdirect_connect_rdma_connect(sc);
+		if (ret)
+			smbdirect_socket_schedule_cleanup(sc, ret);
+		return 0;
+
+	case RDMA_CM_EVENT_ESTABLISHED:
+		smbdirect_connection_rdma_established(sc);
+
+		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING))
+			return 0;
+		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
+
+		/*
+		 * Here we work around an inconsistency between
+		 * iWarp and other devices (at least rxe and irdma using RoCEv2)
+		 */
+		if (rdma_protocol_iwarp(id->device, id->port_num)) {
+			/*
+			 * iWarp devices report the peer's values
+			 * with the perspective of the peer here.
+			 * Tested with siw and irdma (in iwarp mode)
+			 * We need to change to our perspective here,
+			 * so we need to switch the values.
+			 */
+			peer_initiator_depth = event->param.conn.responder_resources;
+			peer_responder_resources = event->param.conn.initiator_depth;
+		} else {
+			/*
+			 * Non iWarp devices report the peer's values
+			 * already changed to our perspective here.
+			 * Tested with rxe and irdma (in roce mode).
+			 */
+			peer_initiator_depth = event->param.conn.initiator_depth;
+			peer_responder_resources = event->param.conn.responder_resources;
+		}
+		smbdirect_connection_negotiate_rdma_resources(sc,
+							      peer_initiator_depth,
+							      peer_responder_resources,
+							      &event->param.conn);
+
+		ret = smbdirect_connect_negotiate_start(sc);
+		if (ret)
+			smbdirect_socket_schedule_cleanup(sc, ret);
+		return 0;
+
+	default:
+		break;
+	}
+
+	/*
+	 * This is an internal error
+	 */
+	WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+	return 0;
+}
+
+static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io = NULL;
+	struct smbdirect_send_io *send_io = NULL;
+	struct smbdirect_negotiate_req *nreq = NULL;
+	int ret;
+
+	if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED))
+		return sc->first_error;
+	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
+
+	ret = smbdirect_connection_create_mem_pools(sc);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_create_mem_pools() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto create_mem_pools_failed;
+	}
+
+	/*
+	 * There is only a single batch credit
+	 */
+	atomic_set(&sc->send_io.bcredits.count, 1);
+
+	/*
+	 * Initialize the local credits to post
+	 * IB_WR_SEND[_WITH_INV].
+	 */
+	atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target);
+
+	recv_io = smbdirect_connection_get_recv_io(sc);
+	if (WARN_ON_ONCE(!recv_io)) {
+		ret = -EINVAL;
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_get_recv_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto get_recv_io_failed;
+	}
+	recv_io->cqe.done = smbdirect_connect_negotiate_recv_done;
+
+	send_io = smbdirect_connection_alloc_send_io(sc);
+	if (IS_ERR(send_io)) {
+		ret = PTR_ERR(send_io);
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_alloc_send_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto alloc_send_io_failed;
+	}
+	send_io->cqe.done = smbdirect_connect_negotiate_send_done;
+
+	nreq = (struct smbdirect_negotiate_req *)send_io->packet;
+	nreq->min_version = cpu_to_le16(SMBDIRECT_V1);
+	nreq->max_version = cpu_to_le16(SMBDIRECT_V1);
+	nreq->reserved = 0;
+	nreq->credits_requested = cpu_to_le16(sp->send_credit_target);
+	nreq->preferred_send_size = cpu_to_le32(sp->max_send_size);
+	nreq->max_receive_size = cpu_to_le32(sp->max_recv_size);
+	nreq->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size);
+
+	smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO,
+		"ReqOut: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n",
+		"MinVersion=0x",
+		le16_to_cpu(nreq->min_version),
+		"MaxVersion=0x",
+		le16_to_cpu(nreq->max_version),
+		"CreditsRequested=",
+		le16_to_cpu(nreq->credits_requested),
+		"PreferredSendSize=",
+		le32_to_cpu(nreq->preferred_send_size),
+		"MaxRecvSize=",
+		le32_to_cpu(nreq->max_receive_size),
+		"MaxFragmentedSize=",
+		le32_to_cpu(nreq->max_fragmented_size));
+
+	send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev,
+						 nreq,
+						 sizeof(*nreq),
+						 DMA_TO_DEVICE);
+	ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"ib_dma_mapping_error() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto dma_mapping_failed;
+	}
+
+	send_io->sge[0].length = sizeof(*nreq);
+	send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey;
+	send_io->num_sge = 1;
+
+	ib_dma_sync_single_for_device(sc->ib.dev,
+				      send_io->sge[0].addr,
+				      send_io->sge[0].length,
+				      DMA_TO_DEVICE);
+
+	smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO,
+		"sge addr=0x%llx length=%u lkey=0x%x\n",
+		send_io->sge[0].addr,
+		send_io->sge[0].length,
+		send_io->sge[0].lkey);
+
+	/*
+	 * Now post the recv_io buffer in order to get
+	 * the negotiate response
+	 */
+	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP;
+	ret = smbdirect_connection_post_recv_io(recv_io);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_post_recv_io() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto post_recv_io_failed;
+	}
+
+	send_io->wr.next = NULL;
+	send_io->wr.wr_cqe = &send_io->cqe;
+	send_io->wr.sg_list = send_io->sge;
+	send_io->wr.num_sge = send_io->num_sge;
+	send_io->wr.opcode = IB_WR_SEND;
+	send_io->wr.send_flags = IB_SEND_SIGNALED;
+
+	ret = smbdirect_connection_post_send_wr(sc, &send_io->wr);
+	if (ret) {
+		/* if we reach here, post send failed */
+		smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_post_send_wr() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto post_send_wr_failed;
+	}
+
+	/*
+	 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
+	 * so that the timer will cause a disconnect.
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->negotiate_timeout_msec));
+
+	return 0;
+
+post_send_wr_failed:
+	/*
+	 * ib_dma_unmap_single is called in
+	 * smbdirect_connection_free_send_io()
+	 */
+	smbdirect_connection_free_send_io(send_io);
+	/*
+	 * recv_io is given to the rdma layer,
+	 * we should not put it even on error
+	 * nor call smbdirect_connection_destroy_mem_pools()
+	 * it will be cleaned up during disconnect.
+	 */
+	return ret;
+
+post_recv_io_failed:
+	/*
+	 * ib_dma_unmap_single is called in
+	 * smbdirect_connection_free_send_io()
+	 */
+dma_mapping_failed:
+	smbdirect_connection_free_send_io(send_io);
+
+alloc_send_io_failed:
+	smbdirect_connection_put_recv_io(recv_io);
+
+get_recv_io_failed:
+	smbdirect_connection_destroy_mem_pools(sc);
+
+create_mem_pools_failed:
+	return ret;
+}
+
+static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_send_io *send_io =
+		container_of(wc->wr_cqe, struct smbdirect_send_io, cqe);
+	struct smbdirect_socket *sc = send_io->socket;
+
+	smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO,
+		"smbdirect_send_io completed. status='%s (%d)', opcode=%d\n",
+		ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+
+	/* Note this frees wc->wr_cqe, but not wc */
+	smbdirect_connection_free_send_io(send_io);
+	atomic_dec(&sc->send_io.pending.count);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+}
+
+static void smbdirect_connect_negotiate_recv_work(struct work_struct *work);
+
+static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_recv_io *recv_io =
+		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
+	struct smbdirect_socket *sc = recv_io->socket;
+	unsigned long flags;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		goto error;
+	}
+
+	smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO,
+		"smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n",
+		ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+
+	/*
+	 * This is an internal error!
+	 */
+	if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REP))
+		goto error;
+
+	/*
+	 * Don't reset timer to the keepalive interval in
+	 * this will be done in smbdirect_accept_direct_negotiate_recv_work.
+	 */
+
+	ib_dma_sync_single_for_cpu(sc->ib.dev,
+				   recv_io->sge.addr,
+				   recv_io->sge.length,
+				   DMA_FROM_DEVICE);
+
+	/*
+	 * Only remember recv_io if it has enough bytes,
+	 * this gives smbdirect_accept_negotiate_recv_work enough
+	 * information in order to disconnect if it was not
+	 * valid.
+	 */
+	sc->recv_io.reassembly.full_packet_received = true;
+	if (wc->byte_len >= sizeof(struct smbdirect_negotiate_resp))
+		smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0);
+	else
+		smbdirect_connection_put_recv_io(recv_io);
+
+	/*
+	 * We continue via the workqueue as we may have
+	 * complex work that might sleep.
+	 *
+	 * So we defer further processing of the negotiation
+	 * to smbdirect_connect_negotiate_recv_work().
+	 */
+	spin_lock_irqsave(&sc->connect.lock, flags);
+	if (!sc->first_error) {
+		INIT_WORK(&sc->connect.work, smbdirect_connect_negotiate_recv_work);
+		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)
+			queue_work(sc->workqueues.connect, &sc->connect.work);
+	}
+	spin_unlock_irqrestore(&sc->connect.lock, flags);
+
+	return;
+
+error:
+	/*
+	 * recv_io.posted.refill_work is still disabled,
+	 * so smbdirect_connection_put_recv_io() won't
+	 * start it.
+	 */
+	smbdirect_connection_put_recv_io(recv_io);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+}
+
+static void smbdirect_connect_negotiate_recv_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, connect.work);
+	struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io;
+	struct smbdirect_negotiate_resp *nrep;
+	unsigned long flags;
+	u16 negotiated_version;
+	u16 credits_requested;
+	u16 credits_granted;
+	u32 status;
+	u32 max_readwrite_size;
+	u32 preferred_send_size;
+	u32 max_receive_size;
+	u32 max_fragmented_size;
+	int posted;
+	int ret;
+
+	if (sc->first_error)
+		return;
+
+	/*
+	 * make sure we won't start again...
+	 */
+	disable_work(work);
+
+	/*
+	 * Reset timer to the keepalive interval in
+	 * order to trigger our next keepalive message.
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->keepalive_interval_msec));
+
+	/*
+	 * If smbdirect_connect_negotiate_recv_done() detected an
+	 * invalid request we want to disconnect.
+	 */
+	recv_io = smbdirect_connection_reassembly_first_recv_io(sc);
+	if (!recv_io) {
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
+	sc->recv_io.reassembly.queue_length--;
+	list_del(&recv_io->list);
+	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+	smbdirect_connection_put_recv_io(recv_io);
+
+	if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING))
+		return;
+
+	/*
+	 * Note recv_io is already part of the free list,
+	 * as we just called smbdirect_connection_put_recv_io(),
+	 * but it won't be reused before we call
+	 * smbdirect_connection_recv_io_refill() below.
+	 */
+
+	nrep = (struct smbdirect_negotiate_resp *)recv_io->packet;
+	negotiated_version = le16_to_cpu(nrep->negotiated_version);
+	credits_requested = le16_to_cpu(nrep->credits_requested);
+	credits_granted = le16_to_cpu(nrep->credits_granted);
+	status = le32_to_cpu(nrep->status);
+	max_readwrite_size = le32_to_cpu(nrep->max_readwrite_size);
+	preferred_send_size = le32_to_cpu(nrep->preferred_send_size);
+	max_receive_size = le32_to_cpu(nrep->max_receive_size);
+	max_fragmented_size = le32_to_cpu(nrep->max_fragmented_size);
+
+	smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO,
+		"RepIn: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n",
+		"MinVersion=0x",
+		le16_to_cpu(nrep->min_version),
+		"MaxVersion=0x",
+		le16_to_cpu(nrep->max_version),
+		"NegotiatedVersion=0x",
+		le16_to_cpu(nrep->negotiated_version),
+		"CreditsRequested=",
+		le16_to_cpu(nrep->credits_requested),
+		"CreditsGranted=",
+		le16_to_cpu(nrep->credits_granted),
+		"Status=0x",
+		le32_to_cpu(nrep->status),
+		"MaxReadWriteSize=",
+		le32_to_cpu(nrep->max_readwrite_size),
+		"PreferredSendSize=",
+		le32_to_cpu(nrep->preferred_send_size),
+		"MaxRecvSize=",
+		le32_to_cpu(nrep->max_receive_size),
+		"MaxFragmentedSize=",
+		le32_to_cpu(nrep->max_fragmented_size));
+
+	if (negotiated_version != SMBDIRECT_V1) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: negotiated_version=0x%x\n",
+			negotiated_version);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED);
+		return;
+	}
+
+	if (status != le32_to_cpu(STATUS_SUCCESS)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: status=0x%x != 0x0\n",
+			status);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED);
+		return;
+	}
+
+	if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: max_receive_size=%u < %u\n",
+			max_receive_size,
+			SMBDIRECT_MIN_RECEIVE_SIZE);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: max_fragmented_size=%u < %u\n",
+			max_fragmented_size,
+			SMBDIRECT_MIN_FRAGMENTED_SIZE);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (credits_granted == 0) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: credits_granted == 0\n");
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (credits_requested == 0) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: credits_requested == 0\n");
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	if (preferred_send_size > sp->max_recv_size) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: preferred_send_size=%u > max_recv_size=%u\n",
+			preferred_send_size,
+			sp->max_recv_size);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * We take the value from the peer, which is checked to be higher than 0,
+	 * but we limit it to the max value we support in order to have
+	 * the main logic simpler.
+	 */
+	sc->recv_io.credits.target = credits_requested;
+	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target,
+					   sp->recv_credit_max);
+
+	/*
+	 * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used.
+	 */
+	sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size);
+	sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE);
+
+	/*
+	 * We already sent our sp->max_fragmented_recv_size
+	 * to the peer, so we can't lower it here any more.
+	 *
+	 * TODO: but if the peer lowered sp->max_recv_size
+	 * we will have to adjust our number of buffers.
+	 *
+	 * But for now we keep it as the cifs.ko code
+	 * worked before.
+	 */
+
+	/*
+	 * Note nrep->max_receive_size was already checked against
+	 * SMBDIRECT_MIN_RECEIVE_SIZE above.
+	 */
+	sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size);
+
+	/*
+	 * Make sure the resulting max_frmr_depth is at least 1,
+	 * which means max_read_write_size needs to be at least PAGE_SIZE.
+	 */
+	sp->max_read_write_size = min_t(u32, sp->max_frmr_depth * PAGE_SIZE,
+					max_readwrite_size);
+	if (sp->max_read_write_size < PAGE_SIZE) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: max_readwrite_size=%u < PAGE_SIZE(%lu)\n",
+			max_readwrite_size,
+			PAGE_SIZE);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+	sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE;
+
+	/*
+	 * Note nrep->credits_granted was already checked against 0 above.
+	 */
+	atomic_set(&sc->send_io.credits.count, credits_granted);
+
+	/*
+	 * Note nrep->max_fragmented_size was already checked against
+	 * SMBDIRECT_MIN_FRAGMENTED_SIZE above.
+	 */
+	sp->max_fragmented_send_size = max_fragmented_size;
+
+	ret = smbdirect_connection_create_mr_list(sc);
+	if (ret) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_create_mr_list() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_socket_schedule_cleanup(sc, ret);
+		return;
+	}
+
+	/*
+	 * Prepare for receiving data_transfer messages
+	 */
+	sc->recv_io.reassembly.full_packet_received = true;
+	sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;
+	list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
+		recv_io->cqe.done = smbdirect_connection_recv_io_done;
+	recv_io = NULL;
+
+	/*
+	 * We should at least post 1 smbdirect_recv_io!
+	 */
+	posted = smbdirect_connection_recv_io_refill(sc);
+	if (posted < 1) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_recv_io_refill() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * smbdirect_connection_negotiation_done()
+	 * will setup all required things and wake up
+	 * the waiter.
+	 */
+	smbdirect_connection_negotiation_done(sc);
+}
+
+int smbdirect_connect_sync(struct smbdirect_socket *sc,
+			   const struct sockaddr *dst)
+{
+	int ret;
+
+	ret = smbdirect_connect(sc, dst);
+	if (ret) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connect(%pISpsfc) failed %1pe\n",
+			dst, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = smbdirect_connection_wait_for_connected(sc);
+	if (ret) {
+		int lvl = SMBDIRECT_LOG_ERR;
+
+		if (ret == -ENODEV)
+			lvl = SMBDIRECT_LOG_INFO;
+
+		smbdirect_log_rdma_event(sc, lvl,
+			"wait for smbdirect_connect(%pISpsfc) failed %1pe\n",
+			dst, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect_sync);
diff --git a/fs/smb/common/smbdirect/smbdirect_connection.c b/fs/smb/common/smbdirect/smbdirect_connection.c
new file mode 100644
index 0000000000000..1e946f78e9357
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_connection.c
@@ -0,0 +1,2175 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+#include <linux/folio_queue.h>
+
+struct smbdirect_map_sges {
+	struct ib_sge *sge;
+	size_t num_sge;
+	size_t max_sge;
+	struct ib_device *device;
+	u32 local_dma_lkey;
+	enum dma_data_direction direction;
+};
+
+static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len,
+					    struct smbdirect_map_sges *state);
+
+static void smbdirect_connection_recv_io_refill_work(struct work_struct *work);
+static void smbdirect_connection_send_immediate_work(struct work_struct *work);
+
+static void smbdirect_connection_qp_event_handler(struct ib_event *event, void *context)
+{
+	struct smbdirect_socket *sc = context;
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+		"%s on device %.*s socket %p (cm_id=%p) status %s first_error %1pe\n",
+		ib_event_msg(event->event),
+		IB_DEVICE_NAME_MAX,
+		event->device->name,
+		sc, sc->rdma.cm_id,
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+
+	switch (event->event) {
+	case IB_EVENT_CQ_ERR:
+	case IB_EVENT_QP_FATAL:
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static int smbdirect_connection_rdma_event_handler(struct rdma_cm_id *id,
+						   struct rdma_cm_event *event)
+{
+	struct smbdirect_socket *sc = id->context;
+	int ret = -ECONNRESET;
+
+	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+		ret = -ENETDOWN;
+	if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+		ret = event->status;
+
+	/*
+	 * cma_cm_event_handler() has
+	 * lockdep_assert_held(&id_priv->handler_mutex);
+	 *
+	 * Mutexes are not allowed in interrupts,
+	 * and we rely on not being in an interrupt here.
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	if (event->event != sc->rdma.expected_event) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			rdma_event_msg(sc->rdma.expected_event),
+			rdma_event_msg(event->event),
+			event->status,
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+
+		/*
+		 * If we get RDMA_CM_EVENT_DEVICE_REMOVAL,
+		 * we should change to SMBDIRECT_SOCKET_DISCONNECTED,
+		 * so that rdma_disconnect() is avoided later via
+		 * smbdirect_socket_schedule_cleanup[_status]() =>
+		 * smbdirect_socket_cleanup_work().
+		 *
+		 * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING,
+		 * but never ever get RDMA_CM_EVENT_DISCONNECTED and
+		 * never reach SMBDIRECT_SOCKET_DISCONNECTED.
+		 */
+		if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+			smbdirect_socket_schedule_cleanup_status(sc,
+								 SMBDIRECT_LOG_ERR,
+								 ret,
+								 SMBDIRECT_SOCKET_DISCONNECTED);
+		else
+			smbdirect_socket_schedule_cleanup(sc, ret);
+		if (sc->ib.qp)
+			ib_drain_qp(sc->ib.qp);
+		return 0;
+	}
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"%s (first_error=%1pe) event=%s\n",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+		rdma_event_msg(event->event));
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_DISCONNECTED:
+		/*
+		 * We need to change to SMBDIRECT_SOCKET_DISCONNECTED,
+		 * so that rdma_disconnect() is avoided later via
+		 * smbdirect_socket_schedule_cleanup_status() =>
+		 * smbdirect_socket_cleanup_work().
+		 *
+		 * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING,
+		 * but never ever get RDMA_CM_EVENT_DISCONNECTED and
+		 * never reach SMBDIRECT_SOCKET_DISCONNECTED.
+		 *
+		 * This is also a normal disconnect so
+		 * SMBDIRECT_LOG_INFO should be good enough
+		 * and avoids spamming the default logs.
+		 */
+		smbdirect_socket_schedule_cleanup_status(sc,
+							 SMBDIRECT_LOG_INFO,
+							 ret,
+							 SMBDIRECT_SOCKET_DISCONNECTED);
+		if (sc->ib.qp)
+			ib_drain_qp(sc->ib.qp);
+		return 0;
+
+	default:
+		break;
+	}
+
+	/*
+	 * This is an internal error, should be handled above via
+	 * event->event != sc->rdma.expected_event already.
+	 */
+	WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_DISCONNECTED);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+	return 0;
+}
+
+void smbdirect_connection_rdma_established(struct smbdirect_socket *sc)
+{
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"rdma established: device: %.*s local: %pISpsfc remote: %pISpsfc\n",
+		IB_DEVICE_NAME_MAX,
+		sc->ib.dev->name,
+		&sc->rdma.cm_id->route.addr.src_addr,
+		&sc->rdma.cm_id->route.addr.dst_addr);
+
+	sc->rdma.cm_id->event_handler = smbdirect_connection_rdma_event_handler;
+	sc->rdma.expected_event = RDMA_CM_EVENT_DISCONNECTED;
+}
+
+void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc)
+{
+	if (unlikely(sc->first_error))
+		return;
+
+	if (sc->status == SMBDIRECT_SOCKET_CONNECTED)
+		/*
+		 * This is the accept case where
+		 * smbdirect_socket_accept() already sets
+		 * SMBDIRECT_SOCKET_CONNECTED
+		 */
+		goto done;
+
+	if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) {
+		/*
+		 * Something went wrong...
+		 */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"status=%s first_error=%1pe local: %pISpsfc remote: %pISpsfc\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			&sc->rdma.cm_id->route.addr.src_addr,
+			&sc->rdma.cm_id->route.addr.dst_addr);
+		return;
+	}
+
+	/*
+	 * We are done, so we can wake up the waiter.
+	 */
+	WARN_ONCE(sc->status == SMBDIRECT_SOCKET_CONNECTED,
+		  "status=%s first_error=%1pe",
+		  smbdirect_socket_status_string(sc->status),
+		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+	sc->status = SMBDIRECT_SOCKET_CONNECTED;
+
+	/*
+	 * We need to setup the refill and send immediate work
+	 * in order to get a working connection.
+	 */
+done:
+	INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work);
+	INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work);
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"negotiated: local: %pISpsfc remote: %pISpsfc\n",
+		&sc->rdma.cm_id->route.addr.src_addr,
+		&sc->rdma.cm_id->route.addr.dst_addr);
+
+	wake_up(&sc->status_wait);
+}
+
+static u32 smbdirect_rdma_rw_send_wrs(struct ib_device *dev,
+				      const struct ib_qp_init_attr *attr)
+{
+	/*
+	 * This could be split out of rdma_rw_init_qp()
+	 * and be a helper function next to rdma_rw_mr_factor()
+	 *
+	 * We can't check unlikely(rdma_rw_force_mr) here,
+	 * but that is most likely 0 anyway.
+	 */
+	u32 factor;
+
+	WARN_ON_ONCE(attr->port_num == 0);
+
+	/*
+	 * Each context needs at least one RDMA READ or WRITE WR.
+	 *
+	 * For some hardware we might need more, eventually we should ask the
+	 * HCA driver for a multiplier here.
+	 */
+	factor = 1;
+
+	/*
+	 * If the device needs MRs to perform RDMA READ or WRITE operations,
+	 * we'll need two additional MRs for the registrations and the
+	 * invalidation.
+	 */
+	if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd)
+		factor += 2;	/* inv + reg */
+
+	return factor * attr->cap.max_rdma_ctxs;
+}
+
+int smbdirect_connection_create_qp(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct ib_qp_init_attr qp_attr;
+	struct ib_qp_cap qp_cap;
+	u32 rdma_send_wr;
+	u32 max_send_wr;
+	int ret;
+
+	/*
+	 * Note that {rdma,ib}_create_qp() will call
+	 * rdma_rw_init_qp() if max_rdma_ctxs is not 0.
+	 * It will adjust max_send_wr to the required
+	 * number of additional WRs for the RDMA RW operations.
+	 * It will cap max_send_wr to the device limit.
+	 *
+	 * We use allocate sp->responder_resources * 2 MRs
+	 * and each MR needs WRs for REG and INV, so
+	 * we use '* 4'.
+	 *
+	 * +1 for ib_drain_qp()
+	 */
+	memset(&qp_cap, 0, sizeof(qp_cap));
+	qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1;
+	qp_cap.max_recv_wr = sp->recv_credit_max + 1;
+	qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
+	qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
+	qp_cap.max_inline_data = 0;
+	qp_cap.max_rdma_ctxs = sc->rw_io.credits.max;
+
+	/*
+	 * Find out the number of max_send_wr
+	 * after rdma_rw_init_qp() adjusted it.
+	 *
+	 * We only do it on a temporary variable,
+	 * as rdma_create_qp() will trigger
+	 * rdma_rw_init_qp() again.
+	 */
+	memset(&qp_attr, 0, sizeof(qp_attr));
+	qp_attr.cap = qp_cap;
+	qp_attr.port_num = sc->rdma.cm_id->port_num;
+	rdma_send_wr = smbdirect_rdma_rw_send_wrs(sc->ib.dev, &qp_attr);
+	max_send_wr = qp_cap.max_send_wr + rdma_send_wr;
+
+	if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe ||
+	    qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) {
+		pr_err("Possible CQE overrun: max_send_wr %d\n",
+		       qp_cap.max_send_wr);
+		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		       IB_DEVICE_NAME_MAX,
+		       sc->ib.dev->name,
+		       sc->ib.dev->attrs.max_cqe,
+		       sc->ib.dev->attrs.max_qp_wr);
+		pr_err("consider lowering send_credit_target = %d\n",
+		       sp->send_credit_target);
+		return -EINVAL;
+	}
+
+	if (qp_cap.max_rdma_ctxs &&
+	    (max_send_wr >= sc->ib.dev->attrs.max_cqe ||
+	     max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) {
+		pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n",
+		       rdma_send_wr, qp_cap.max_send_wr, max_send_wr);
+		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		       IB_DEVICE_NAME_MAX,
+		       sc->ib.dev->name,
+		       sc->ib.dev->attrs.max_cqe,
+		       sc->ib.dev->attrs.max_qp_wr);
+		pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n",
+		       sp->send_credit_target, qp_cap.max_rdma_ctxs);
+		return -EINVAL;
+	}
+
+	if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe ||
+	    qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) {
+		pr_err("Possible CQE overrun: max_recv_wr %d\n",
+		       qp_cap.max_recv_wr);
+		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		       IB_DEVICE_NAME_MAX,
+		       sc->ib.dev->name,
+		       sc->ib.dev->attrs.max_cqe,
+		       sc->ib.dev->attrs.max_qp_wr);
+		pr_err("consider lowering receive_credit_max = %d\n",
+		       sp->recv_credit_max);
+		return -EINVAL;
+	}
+
+	if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge ||
+	    qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) {
+		pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
+		       IB_DEVICE_NAME_MAX,
+		       sc->ib.dev->name,
+		       sc->ib.dev->attrs.max_send_sge,
+		       sc->ib.dev->attrs.max_recv_sge);
+		return -EINVAL;
+	}
+
+	sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
+	if (IS_ERR(sc->ib.pd)) {
+		pr_err("Can't create RDMA PD: %1pe\n", sc->ib.pd);
+		ret = PTR_ERR(sc->ib.pd);
+		sc->ib.pd = NULL;
+		return ret;
+	}
+
+	sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc,
+					 max_send_wr,
+					 sc->ib.poll_ctx);
+	if (IS_ERR(sc->ib.send_cq)) {
+		pr_err("Can't create RDMA send CQ: %1pe\n", sc->ib.send_cq);
+		ret = PTR_ERR(sc->ib.send_cq);
+		sc->ib.send_cq = NULL;
+		goto err;
+	}
+
+	sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc,
+					 qp_cap.max_recv_wr,
+					 sc->ib.poll_ctx);
+	if (IS_ERR(sc->ib.recv_cq)) {
+		pr_err("Can't create RDMA recv CQ: %1pe\n", sc->ib.recv_cq);
+		ret = PTR_ERR(sc->ib.recv_cq);
+		sc->ib.recv_cq = NULL;
+		goto err;
+	}
+
+	/*
+	 * We reset completely here!
+	 * As the above use was just temporary
+	 * to calc max_send_wr and rdma_send_wr.
+	 *
+	 * rdma_create_qp() will trigger rdma_rw_init_qp()
+	 * again if max_rdma_ctxs is not 0.
+	 */
+	memset(&qp_attr, 0, sizeof(qp_attr));
+	qp_attr.event_handler = smbdirect_connection_qp_event_handler;
+	qp_attr.qp_context = sc;
+	qp_attr.cap = qp_cap;
+	qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
+	qp_attr.qp_type = IB_QPT_RC;
+	qp_attr.send_cq = sc->ib.send_cq;
+	qp_attr.recv_cq = sc->ib.recv_cq;
+	qp_attr.port_num = ~0;
+
+	ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);
+	if (ret) {
+		pr_err("Can't create RDMA QP: %1pe\n",
+		       SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto err;
+	}
+	sc->ib.qp = sc->rdma.cm_id->qp;
+
+	return 0;
+err:
+	smbdirect_connection_destroy_qp(sc);
+	return ret;
+}
+
+void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc)
+{
+	if (sc->ib.qp) {
+		ib_drain_qp(sc->ib.qp);
+		sc->ib.qp = NULL;
+		rdma_destroy_qp(sc->rdma.cm_id);
+	}
+	if (sc->ib.recv_cq) {
+		ib_destroy_cq(sc->ib.recv_cq);
+		sc->ib.recv_cq = NULL;
+	}
+	if (sc->ib.send_cq) {
+		ib_destroy_cq(sc->ib.send_cq);
+		sc->ib.send_cq = NULL;
+	}
+	if (sc->ib.pd) {
+		ib_dealloc_pd(sc->ib.pd);
+		sc->ib.pd = NULL;
+	}
+}
+
+int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	char name[80];
+	size_t i;
+
+	/*
+	 * We use sizeof(struct smbdirect_negotiate_resp) for the
+	 * payload size as it is larger as
+	 * sizeof(struct smbdirect_data_transfer).
+	 *
+	 * This will fit client and server usage for now.
+	 */
+	snprintf(name, sizeof(name), "smbdirect_send_io_cache_%p", sc);
+	struct kmem_cache_args send_io_args = {
+		.align		= __alignof__(struct smbdirect_send_io),
+	};
+	sc->send_io.mem.cache = kmem_cache_create(name,
+						  sizeof(struct smbdirect_send_io) +
+						  sizeof(struct smbdirect_negotiate_resp),
+						  &send_io_args,
+						  SLAB_HWCACHE_ALIGN);
+	if (!sc->send_io.mem.cache)
+		goto err;
+
+	sc->send_io.mem.pool = mempool_create_slab_pool(sp->send_credit_target,
+							sc->send_io.mem.cache);
+	if (!sc->send_io.mem.pool)
+		goto err;
+
+	/*
+	 * A payload size of sp->max_recv_size should fit
+	 * any message.
+	 *
+	 * For smbdirect_data_transfer messages the whole
+	 * buffer might be exposed to userspace
+	 * (currently on the client side...)
+	 * The documentation says data_offset = 0 would be
+	 * strange but valid.
+	 */
+	snprintf(name, sizeof(name), "smbdirect_recv_io_cache_%p", sc);
+	struct kmem_cache_args recv_io_args = {
+		.align		= __alignof__(struct smbdirect_recv_io),
+		.useroffset	= sizeof(struct smbdirect_recv_io),
+		.usersize	= sp->max_recv_size,
+	};
+	sc->recv_io.mem.cache = kmem_cache_create(name,
+						  sizeof(struct smbdirect_recv_io) +
+						  sp->max_recv_size,
+						  &recv_io_args,
+						  SLAB_HWCACHE_ALIGN);
+	if (!sc->recv_io.mem.cache)
+		goto err;
+
+	sc->recv_io.mem.pool = mempool_create_slab_pool(sp->recv_credit_max,
+							sc->recv_io.mem.cache);
+	if (!sc->recv_io.mem.pool)
+		goto err;
+
+	for (i = 0; i < sp->recv_credit_max; i++) {
+		struct smbdirect_recv_io *recv_io;
+
+		recv_io = mempool_alloc(sc->recv_io.mem.pool,
+					sc->recv_io.mem.gfp_mask);
+		if (!recv_io)
+			goto err;
+		recv_io->socket = sc;
+		recv_io->sge.length = 0;
+		list_add_tail(&recv_io->list, &sc->recv_io.free.list);
+	}
+
+	return 0;
+err:
+	smbdirect_connection_destroy_mem_pools(sc);
+	return -ENOMEM;
+}
+
+void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc)
+{
+	struct smbdirect_recv_io *recv_io, *next_io;
+
+	list_for_each_entry_safe(recv_io, next_io, &sc->recv_io.free.list, list) {
+		list_del(&recv_io->list);
+		mempool_free(recv_io, sc->recv_io.mem.pool);
+	}
+
+	/*
+	 * Note mempool_destroy() and kmem_cache_destroy()
+	 * work fine with a NULL pointer
+	 */
+
+	mempool_destroy(sc->recv_io.mem.pool);
+	sc->recv_io.mem.pool = NULL;
+
+	kmem_cache_destroy(sc->recv_io.mem.cache);
+	sc->recv_io.mem.cache = NULL;
+
+	mempool_destroy(sc->send_io.mem.pool);
+	sc->send_io.mem.pool = NULL;
+
+	kmem_cache_destroy(sc->send_io.mem.cache);
+	sc->send_io.mem.cache = NULL;
+}
+
+struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc)
+{
+	struct smbdirect_send_io *msg;
+
+	msg = mempool_alloc(sc->send_io.mem.pool, sc->send_io.mem.gfp_mask);
+	if (!msg)
+		return ERR_PTR(-ENOMEM);
+	msg->socket = sc;
+	INIT_LIST_HEAD(&msg->sibling_list);
+	msg->num_sge = 0;
+
+	return msg;
+}
+
+void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg)
+{
+	struct smbdirect_socket *sc = msg->socket;
+	size_t i;
+
+	/*
+	 * The list needs to be empty!
+	 * The caller should take care of it.
+	 */
+	WARN_ON_ONCE(!list_empty(&msg->sibling_list));
+
+	/*
+	 * Note we call ib_dma_unmap_page(), even if some sges are mapped using
+	 * ib_dma_map_single().
+	 *
+	 * The difference between _single() and _page() only matters for the
+	 * ib_dma_map_*() case.
+	 *
+	 * For the ib_dma_unmap_*() case it does not matter as both take the
+	 * dma_addr_t and dma_unmap_single_attrs() is just an alias to
+	 * dma_unmap_page_attrs().
+	 */
+	for (i = 0; i < msg->num_sge; i++)
+		ib_dma_unmap_page(sc->ib.dev,
+				  msg->sge[i].addr,
+				  msg->sge[i].length,
+				  DMA_TO_DEVICE);
+
+	mempool_free(msg, sc->send_io.mem.pool);
+}
+
+struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc)
+{
+	struct smbdirect_recv_io *msg = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
+	if (likely(!sc->first_error))
+		msg = list_first_entry_or_null(&sc->recv_io.free.list,
+					       struct smbdirect_recv_io,
+					       list);
+	if (likely(msg)) {
+		list_del(&msg->list);
+		sc->statistics.get_receive_buffer++;
+	}
+	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
+
+	return msg;
+}
+
+void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg)
+{
+	struct smbdirect_socket *sc = msg->socket;
+	unsigned long flags;
+
+	if (likely(msg->sge.length != 0)) {
+		ib_dma_unmap_single(sc->ib.dev,
+				    msg->sge.addr,
+				    msg->sge.length,
+				    DMA_FROM_DEVICE);
+		msg->sge.length = 0;
+	}
+
+	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
+	list_add_tail(&msg->list, &sc->recv_io.free.list);
+	sc->statistics.put_receive_buffer++;
+	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
+
+	queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
+}
+
+void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc,
+						    struct smbdirect_recv_io *msg,
+						    u32 data_length)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
+	list_add_tail(&msg->list, &sc->recv_io.reassembly.list);
+	sc->recv_io.reassembly.queue_length++;
+	/*
+	 * Make sure reassembly_data_length is updated after list and
+	 * reassembly_queue_length are updated. On the dequeue side
+	 * reassembly_data_length is checked without a lock to determine
+	 * if reassembly_queue_length and list is up to date
+	 */
+	virt_wmb();
+	sc->recv_io.reassembly.data_length += data_length;
+	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+	sc->statistics.enqueue_reassembly_queue++;
+}
+
+struct smbdirect_recv_io *
+smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc)
+{
+	struct smbdirect_recv_io *msg;
+
+	msg = list_first_entry_or_null(&sc->recv_io.reassembly.list,
+				       struct smbdirect_recv_io,
+				       list);
+
+	return msg;
+}
+
+void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc,
+						   u8 peer_initiator_depth,
+						   u8 peer_responder_resources,
+						   const struct rdma_conn_param *param)
+{
+	struct smbdirect_socket_parameters *sp = &sc->parameters;
+
+	if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num) &&
+	    param->private_data_len == 8) {
+		/*
+		 * Legacy clients with only iWarp MPA v1 support
+		 * need a private blob in order to negotiate
+		 * the IRD/ORD values.
+		 */
+		const __be32 *ird_ord_hdr = param->private_data;
+		u32 ird32 = be32_to_cpu(ird_ord_hdr[0]);
+		u32 ord32 = be32_to_cpu(ird_ord_hdr[1]);
+
+		/*
+		 * cifs.ko sends the legacy IRD/ORD negotiation
+		 * event if iWarp MPA v2 was used.
+		 *
+		 * Here we check that the values match and only
+		 * mark the client as legacy if they don't match.
+		 */
+		if ((u32)param->initiator_depth != ird32 ||
+		    (u32)param->responder_resources != ord32) {
+			/*
+			 * There are broken clients (old cifs.ko)
+			 * using little endian and also
+			 * struct rdma_conn_param only uses u8
+			 * for initiator_depth and responder_resources,
+			 * so we truncate the value to U8_MAX.
+			 *
+			 * smb_direct_accept_client() will then
+			 * do the real negotiation in order to
+			 * select the minimum between client and
+			 * server.
+			 */
+			ird32 = min_t(u32, ird32, U8_MAX);
+			ord32 = min_t(u32, ord32, U8_MAX);
+
+			sc->rdma.legacy_iwarp = true;
+			peer_initiator_depth = (u8)ird32;
+			peer_responder_resources = (u8)ord32;
+		}
+	}
+
+	/*
+	 * negotiate the value by using the minimum
+	 * between client and server if the client provided
+	 * non 0 values.
+	 */
+	if (peer_initiator_depth != 0)
+		sp->initiator_depth = min_t(u8, sp->initiator_depth,
+					    peer_initiator_depth);
+	if (peer_responder_resources != 0)
+		sp->responder_resources = min_t(u8, sp->responder_resources,
+						peer_responder_resources);
+}
+
+bool smbdirect_connection_is_connected(struct smbdirect_socket *sc)
+{
+	if (unlikely(!sc || sc->first_error || sc->status != SMBDIRECT_SOCKET_CONNECTED))
+		return false;
+	return true;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_is_connected);
+
+int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	union {
+		struct sockaddr sa;
+		struct sockaddr_storage ss;
+	} src_addr, dst_addr;
+	const struct sockaddr *src = NULL;
+	const struct sockaddr *dst = NULL;
+	char _devname[IB_DEVICE_NAME_MAX] = { 0, };
+	const char *devname = NULL;
+	int ret;
+
+	if (sc->rdma.cm_id) {
+		src_addr.ss = sc->rdma.cm_id->route.addr.src_addr;
+		if (src_addr.sa.sa_family != AF_UNSPEC)
+			src = &src_addr.sa;
+		dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr;
+		if (dst_addr.sa.sa_family != AF_UNSPEC)
+			dst = &dst_addr.sa;
+
+		if (sc->ib.dev) {
+			memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX);
+			devname = _devname;
+		}
+	}
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"waiting for connection: device: %.*s local: %pISpsfc remote: %pISpsfc\n",
+		IB_DEVICE_NAME_MAX, devname, src, dst);
+
+	ret = wait_event_interruptible_timeout(sc->status_wait,
+					       sc->status == SMBDIRECT_SOCKET_CONNECTED ||
+					       sc->first_error,
+					       msecs_to_jiffies(sp->negotiate_timeout_msec));
+	if (sc->rdma.cm_id) {
+		/*
+		 * Maybe src and dev are updated in the meantime.
+		 */
+		src_addr.ss = sc->rdma.cm_id->route.addr.src_addr;
+		if (src_addr.sa.sa_family != AF_UNSPEC)
+			src = &src_addr.sa;
+		dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr;
+		if (dst_addr.sa.sa_family != AF_UNSPEC)
+			dst = &dst_addr.sa;
+
+		if (sc->ib.dev) {
+			memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX);
+			devname = _devname;
+		}
+	}
+	if (ret == 0)
+		ret = -ETIMEDOUT;
+	if (ret < 0)
+		smbdirect_socket_schedule_cleanup(sc, ret);
+	if (sc->first_error) {
+		int lvl = SMBDIRECT_LOG_ERR;
+
+		ret = sc->first_error;
+		if (ret == -ENODEV)
+			lvl = SMBDIRECT_LOG_INFO;
+
+		smbdirect_log_rdma_event(sc, lvl,
+			"connection failed %1pe device: %.*s local: %pISpsfc remote: %pISpsfc\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret),
+			IB_DEVICE_NAME_MAX, devname, src, dst);
+		return ret;
+	}
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_wait_for_connected);
+
+void smbdirect_connection_idle_timer_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, idle.timer_work.work);
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+
+	if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) {
+		smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_ERR,
+			"%s => timeout sc->idle.keepalive=%s\n",
+			smbdirect_socket_status_string(sc->status),
+			sc->idle.keepalive == SMBDIRECT_KEEPALIVE_SENT ?
+			"SENT" : "PENDING");
+		smbdirect_socket_schedule_cleanup(sc, -ETIMEDOUT);
+		return;
+	}
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
+		return;
+
+	/*
+	 * Now use the keepalive timeout (instead of keepalive interval)
+	 * in order to wait for a response
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->keepalive_timeout_msec));
+	smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
+		"schedule send of empty idle message\n");
+	queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
+}
+
+u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc)
+{
+	int missing;
+	int available;
+	int new_credits;
+
+	if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target)
+		return 0;
+
+	missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count);
+	available = atomic_xchg(&sc->recv_io.credits.available, 0);
+	new_credits = min3((int)U16_MAX, missing, available);
+	if (new_credits <= 0) {
+		/*
+		 * If credits are available, but not granted
+		 * we need to re-add them again.
+		 */
+		if (available)
+			atomic_add(available, &sc->recv_io.credits.available);
+		return 0;
+	}
+
+	if (new_credits < available) {
+		/*
+		 * Readd the remaining available again.
+		 */
+		available -= new_credits;
+		atomic_add(available, &sc->recv_io.credits.available);
+	}
+
+	/*
+	 * Remember we granted the credits
+	 */
+	atomic_add(new_credits, &sc->recv_io.credits.count);
+	return new_credits;
+}
+
+static bool smbdirect_connection_request_keep_alive(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+
+	if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) {
+		sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT;
+		/*
+		 * Now use the keepalive timeout (instead of keepalive interval)
+		 * in order to wait for a response
+		 */
+		mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+				 msecs_to_jiffies(sp->keepalive_timeout_msec));
+		return true;
+	}
+
+	return false;
+}
+
+int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc,
+				      struct ib_send_wr *wr)
+{
+	int ret;
+
+	if (unlikely(sc->first_error))
+		return sc->first_error;
+
+	atomic_inc(&sc->send_io.pending.count);
+	ret = ib_post_send(sc->ib.qp, wr, NULL);
+	if (ret) {
+		atomic_dec(&sc->send_io.pending.count);
+		smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+			"ib_post_send() failed %1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_socket_schedule_cleanup(sc, ret);
+	}
+
+	return ret;
+}
+
+static void smbdirect_connection_send_batch_init(struct smbdirect_send_batch *batch,
+						 bool need_invalidate_rkey,
+						 unsigned int remote_key)
+{
+	INIT_LIST_HEAD(&batch->msg_list);
+	batch->wr_cnt = 0;
+	batch->need_invalidate_rkey = need_invalidate_rkey;
+	batch->remote_key = remote_key;
+	batch->credit = 0;
+}
+
+int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc,
+					  struct smbdirect_send_batch *batch,
+					  bool is_last)
+{
+	struct smbdirect_send_io *first, *last;
+	int ret = 0;
+
+	if (list_empty(&batch->msg_list))
+		goto release_credit;
+
+	first = list_first_entry(&batch->msg_list,
+				 struct smbdirect_send_io,
+				 sibling_list);
+	last = list_last_entry(&batch->msg_list,
+			       struct smbdirect_send_io,
+			       sibling_list);
+
+	if (batch->need_invalidate_rkey) {
+		first->wr.opcode = IB_WR_SEND_WITH_INV;
+		first->wr.ex.invalidate_rkey = batch->remote_key;
+		batch->need_invalidate_rkey = false;
+		batch->remote_key = 0;
+	}
+
+	last->wr.send_flags = IB_SEND_SIGNALED;
+	last->wr.wr_cqe = &last->cqe;
+
+	/*
+	 * Remove last from send_ctx->msg_list
+	 * and splice the rest of send_ctx->msg_list
+	 * to last->sibling_list.
+	 *
+	 * send_ctx->msg_list is a valid empty list
+	 * at the end.
+	 */
+	list_del_init(&last->sibling_list);
+	list_splice_tail_init(&batch->msg_list, &last->sibling_list);
+	batch->wr_cnt = 0;
+
+	ret = smbdirect_connection_post_send_wr(sc, &first->wr);
+	if (ret) {
+		struct smbdirect_send_io *sibling, *next;
+
+		list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
+			list_del_init(&sibling->sibling_list);
+			smbdirect_connection_free_send_io(sibling);
+		}
+		smbdirect_connection_free_send_io(last);
+	}
+
+release_credit:
+	if (is_last && !ret && batch->credit) {
+		atomic_add(batch->credit, &sc->send_io.bcredits.count);
+		batch->credit = 0;
+		wake_up(&sc->send_io.bcredits.wait_queue);
+	}
+
+	return ret;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_batch_flush);
+
+struct smbdirect_send_batch *
+smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage,
+				  bool need_invalidate_rkey,
+				  unsigned int remote_key)
+{
+	struct smbdirect_send_batch *batch = (struct smbdirect_send_batch *)storage;
+
+	memset(storage, 0, sizeof(*storage));
+	BUILD_BUG_ON(sizeof(*batch) > sizeof(*storage));
+
+	smbdirect_connection_send_batch_init(batch,
+					     need_invalidate_rkey,
+					     remote_key);
+
+	return batch;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_init_send_batch_storage);
+
+static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc,
+						      struct smbdirect_send_batch *batch)
+{
+	int ret;
+
+	if (batch->credit)
+		return 0;
+
+	ret = smbdirect_socket_wait_for_credits(sc,
+						 SMBDIRECT_SOCKET_CONNECTED,
+						 -ENOTCONN,
+						 &sc->send_io.bcredits.wait_queue,
+						 &sc->send_io.bcredits.count,
+						 1);
+	if (ret)
+		return ret;
+
+	batch->credit = 1;
+	return 0;
+}
+
+static int smbdirect_connection_wait_for_send_lcredit(struct smbdirect_socket *sc,
+						      struct smbdirect_send_batch *batch)
+{
+	if (batch && atomic_read(&sc->send_io.lcredits.count) <= 1) {
+		int ret;
+
+		ret = smbdirect_connection_send_batch_flush(sc, batch, false);
+		if (ret)
+			return ret;
+	}
+
+	return smbdirect_socket_wait_for_credits(sc,
+						 SMBDIRECT_SOCKET_CONNECTED,
+						 -ENOTCONN,
+						 &sc->send_io.lcredits.wait_queue,
+						 &sc->send_io.lcredits.count,
+						 1);
+}
+
+static int smbdirect_connection_wait_for_send_credits(struct smbdirect_socket *sc,
+						      struct smbdirect_send_batch *batch)
+{
+	if (batch && (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
+		int ret;
+
+		ret = smbdirect_connection_send_batch_flush(sc, batch, false);
+		if (ret)
+			return ret;
+	}
+
+	return smbdirect_socket_wait_for_credits(sc,
+						 SMBDIRECT_SOCKET_CONNECTED,
+						 -ENOTCONN,
+						 &sc->send_io.credits.wait_queue,
+						 &sc->send_io.credits.count,
+						 1);
+}
+
+static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc);
+
+static int smbdirect_connection_post_send_io(struct smbdirect_socket *sc,
+					     struct smbdirect_send_batch *batch,
+					     struct smbdirect_send_io *msg)
+{
+	int i;
+
+	for (i = 0; i < msg->num_sge; i++)
+		ib_dma_sync_single_for_device(sc->ib.dev,
+					      msg->sge[i].addr, msg->sge[i].length,
+					      DMA_TO_DEVICE);
+
+	msg->cqe.done = smbdirect_connection_send_io_done;
+	msg->wr.wr_cqe = &msg->cqe;
+	msg->wr.opcode = IB_WR_SEND;
+	msg->wr.sg_list = &msg->sge[0];
+	msg->wr.num_sge = msg->num_sge;
+	msg->wr.next = NULL;
+
+	if (batch) {
+		msg->wr.send_flags = 0;
+		if (!list_empty(&batch->msg_list)) {
+			struct smbdirect_send_io *last;
+
+			last = list_last_entry(&batch->msg_list,
+					       struct smbdirect_send_io,
+					       sibling_list);
+			last->wr.next = &msg->wr;
+		}
+		list_add_tail(&msg->sibling_list, &batch->msg_list);
+		batch->wr_cnt++;
+		return 0;
+	}
+
+	msg->wr.send_flags = IB_SEND_SIGNALED;
+	return smbdirect_connection_post_send_wr(sc, &msg->wr);
+}
+
+int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc,
+					  struct smbdirect_send_batch *batch,
+					  struct iov_iter *iter,
+					  unsigned int flags,
+					  u32 remaining_data_length)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_send_batch _batch;
+	struct smbdirect_send_io *msg;
+	struct smbdirect_data_transfer *packet;
+	size_t header_length;
+	u16 new_credits = 0;
+	u32 data_length = 0;
+	int ret;
+
+	if (WARN_ON_ONCE(flags))
+		return -EINVAL; /* no flags support for now */
+
+	if (iter) {
+		if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE))
+			return -EINVAL; /* It's a bug in upper layer to get there */
+
+		header_length = sizeof(struct smbdirect_data_transfer);
+		if (WARN_ON_ONCE(remaining_data_length == 0 ||
+				 iov_iter_count(iter) > remaining_data_length))
+			return -EINVAL;
+	} else {
+		/* If this is a packet without payload, don't send padding */
+		header_length = offsetof(struct smbdirect_data_transfer, padding);
+		if (WARN_ON_ONCE(remaining_data_length))
+			return -EINVAL;
+	}
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_log_write(sc, SMBDIRECT_LOG_ERR,
+			"status=%s first_error=%1pe => %1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN));
+		return -ENOTCONN;
+	}
+
+	if (!batch) {
+		smbdirect_connection_send_batch_init(&_batch, false, 0);
+		batch = &_batch;
+	}
+
+	ret = smbdirect_connection_wait_for_send_bcredit(sc, batch);
+	if (ret)
+		goto bcredit_failed;
+
+	ret = smbdirect_connection_wait_for_send_lcredit(sc, batch);
+	if (ret)
+		goto lcredit_failed;
+
+	ret = smbdirect_connection_wait_for_send_credits(sc, batch);
+	if (ret)
+		goto credit_failed;
+
+	new_credits = smbdirect_connection_grant_recv_credits(sc);
+	if (new_credits == 0 &&
+	    atomic_read(&sc->send_io.credits.count) == 0 &&
+	    atomic_read(&sc->recv_io.credits.count) == 0) {
+		/*
+		 * queue the refill work in order to
+		 * get some new recv credits we can grant to
+		 * the peer.
+		 */
+		queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
+
+		/*
+		 * wait until either the refill work or the peer
+		 * granted new credits
+		 */
+		ret = wait_event_interruptible(sc->send_io.credits.wait_queue,
+					       atomic_read(&sc->send_io.credits.count) >= 1 ||
+					       atomic_read(&sc->recv_io.credits.available) >= 1 ||
+					       sc->status != SMBDIRECT_SOCKET_CONNECTED);
+		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
+			ret = -ENOTCONN;
+		if (ret < 0)
+			goto credit_failed;
+
+		new_credits = smbdirect_connection_grant_recv_credits(sc);
+	}
+
+	msg = smbdirect_connection_alloc_send_io(sc);
+	if (IS_ERR(msg)) {
+		ret = PTR_ERR(msg);
+		goto alloc_failed;
+	}
+
+	/* Map the packet to DMA */
+	msg->sge[0].addr = ib_dma_map_single(sc->ib.dev,
+					     msg->packet,
+					     header_length,
+					     DMA_TO_DEVICE);
+	ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr);
+	if (ret)
+		goto err;
+
+	msg->sge[0].length = header_length;
+	msg->sge[0].lkey = sc->ib.pd->local_dma_lkey;
+	msg->num_sge = 1;
+
+	if (iter) {
+		struct smbdirect_map_sges extract = {
+			.num_sge	= msg->num_sge,
+			.max_sge	= ARRAY_SIZE(msg->sge),
+			.sge		= msg->sge,
+			.device		= sc->ib.dev,
+			.local_dma_lkey	= sc->ib.pd->local_dma_lkey,
+			.direction	= DMA_TO_DEVICE,
+		};
+		size_t payload_len = umin(iov_iter_count(iter),
+					  sp->max_send_size - sizeof(*packet));
+
+		ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract);
+		if (ret < 0)
+			goto err;
+		data_length = ret;
+		remaining_data_length -= data_length;
+		msg->num_sge = extract.num_sge;
+	}
+
+	/* Fill in the packet header */
+	packet = (struct smbdirect_data_transfer *)msg->packet;
+	packet->credits_requested = cpu_to_le16(sp->send_credit_target);
+	packet->credits_granted = cpu_to_le16(new_credits);
+
+	packet->flags = 0;
+	if (smbdirect_connection_request_keep_alive(sc))
+		packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
+
+	packet->reserved = 0;
+	if (!data_length)
+		packet->data_offset = 0;
+	else
+		packet->data_offset = cpu_to_le32(24);
+	packet->data_length = cpu_to_le32(data_length);
+	packet->remaining_data_length = cpu_to_le32(remaining_data_length);
+	packet->padding = 0;
+
+	smbdirect_log_outgoing(sc, SMBDIRECT_LOG_INFO,
+		"DataOut: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n",
+		"CreditsRequested",
+		le16_to_cpu(packet->credits_requested),
+		"CreditsGranted",
+		le16_to_cpu(packet->credits_granted),
+		"Flags",
+		le16_to_cpu(packet->flags),
+		"RemainingDataLength",
+		le32_to_cpu(packet->remaining_data_length),
+		"DataOffset",
+		le32_to_cpu(packet->data_offset),
+		"DataLength",
+		le32_to_cpu(packet->data_length));
+
+	ret = smbdirect_connection_post_send_io(sc, batch, msg);
+	if (ret)
+		goto err;
+
+	if (batch == &_batch) {
+		ret = smbdirect_connection_send_batch_flush(sc, batch, true);
+		if (ret)
+			goto err;
+	}
+
+	return data_length;
+err:
+	smbdirect_connection_free_send_io(msg);
+alloc_failed:
+	atomic_inc(&sc->send_io.credits.count);
+credit_failed:
+	atomic_inc(&sc->send_io.lcredits.count);
+lcredit_failed:
+	atomic_add(batch->credit, &sc->send_io.bcredits.count);
+	batch->credit = 0;
+bcredit_failed:
+	return ret;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_single_iter);
+
+int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc)
+{
+	/*
+	 * As an optimization, we don't wait for individual I/O to finish
+	 * before sending the next one.
+	 * Send them all and wait for pending send count to get to 0
+	 * that means all the I/Os have been out and we are good to return
+	 */
+
+	wait_event(sc->send_io.pending.zero_wait_queue,
+		   atomic_read(&sc->send_io.pending.count) == 0 ||
+		   sc->status != SMBDIRECT_SOCKET_CONNECTED);
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_log_write(sc, SMBDIRECT_LOG_ERR,
+			"status=%s first_error=%1pe => %1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN));
+		return -ENOTCONN;
+	}
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_wait_zero_pending);
+
+int smbdirect_connection_send_iter(struct smbdirect_socket *sc,
+				   struct iov_iter *iter,
+				   unsigned int flags,
+				   bool need_invalidate,
+				   unsigned int remote_key)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_send_batch batch;
+	int total_count = iov_iter_count(iter);
+	int ret;
+	int error = 0;
+	__be32 hdr;
+
+	if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags))
+		return -EINVAL; /* no flags support for now */
+
+	if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE))
+		return -EINVAL; /* It's a bug in upper layer to get there */
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_log_write(sc, SMBDIRECT_LOG_INFO,
+			"status=%s first_error=%1pe => %1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN));
+		return -ENOTCONN;
+	}
+
+	/*
+	 * For now we expect the iter to have the full
+	 * message, including a 4 byte length header.
+	 */
+	if (iov_iter_count(iter) <= 4)
+		return -EINVAL;
+	if (!copy_from_iter_full(&hdr, sizeof(hdr), iter))
+		return -EFAULT;
+	if (iov_iter_count(iter) != be32_to_cpu(hdr))
+		return -EINVAL;
+
+	/*
+	 * The size must fit into the negotiated
+	 * fragmented send size.
+	 */
+	if (iov_iter_count(iter) > sp->max_fragmented_send_size)
+		return -EMSGSIZE;
+
+	smbdirect_log_write(sc, SMBDIRECT_LOG_INFO,
+		"Sending (RDMA): length=%zu\n",
+		iov_iter_count(iter));
+
+	smbdirect_connection_send_batch_init(&batch, need_invalidate, remote_key);
+	while (iov_iter_count(iter)) {
+		ret = smbdirect_connection_send_single_iter(sc,
+							    &batch,
+							    iter,
+							    flags,
+							    iov_iter_count(iter));
+		if (unlikely(ret < 0)) {
+			error = ret;
+			break;
+		}
+	}
+
+	ret = smbdirect_connection_send_batch_flush(sc, &batch, true);
+	if (unlikely(ret && !error))
+		error = ret;
+
+	/*
+	 * As an optimization, we don't wait for individual I/O to finish
+	 * before sending the next one.
+	 * Send them all and wait for pending send count to get to 0
+	 * that means all the I/Os have been out and we are good to return
+	 */
+
+	ret = smbdirect_connection_send_wait_zero_pending(sc);
+	if (unlikely(ret && !error))
+		error = ret;
+
+	if (unlikely(error))
+		return error;
+
+	return total_count;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_iter);
+
+static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_send_io *msg =
+		container_of(wc->wr_cqe, struct smbdirect_send_io, cqe);
+	struct smbdirect_socket *sc = msg->socket;
+	struct smbdirect_send_io *sibling, *next;
+	int lcredits = 0;
+
+	smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO,
+		"smbdirect_send_io completed. status='%s (%d)', opcode=%d\n",
+		ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+
+	if (unlikely(!(msg->wr.send_flags & IB_SEND_SIGNALED))) {
+		/*
+		 * This happens when smbdirect_send_io is a sibling
+		 * before the final message, it is signaled on
+		 * error anyway, so we need to skip
+		 * smbdirect_connection_free_send_io here,
+		 * otherwise is will destroy the memory
+		 * of the siblings too, which will cause
+		 * use after free problems for the others
+		 * triggered from ib_drain_qp().
+		 */
+		if (wc->status != IB_WC_SUCCESS)
+			goto skip_free;
+
+		/*
+		 * This should not happen!
+		 * But we better just close the
+		 * connection...
+		 */
+		smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+			"unexpected send completion wc->status=%s (%d) wc->opcode=%d\n",
+			ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	/*
+	 * Free possible siblings and then the main send_io
+	 */
+	list_for_each_entry_safe(sibling, next, &msg->sibling_list, sibling_list) {
+		list_del_init(&sibling->sibling_list);
+		smbdirect_connection_free_send_io(sibling);
+		lcredits += 1;
+	}
+	/* Note this frees wc->wr_cqe, but not wc */
+	smbdirect_connection_free_send_io(msg);
+	lcredits += 1;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) {
+skip_free:
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+		return;
+	}
+
+	atomic_add(lcredits, &sc->send_io.lcredits.count);
+	wake_up(&sc->send_io.lcredits.wait_queue);
+
+	if (atomic_dec_and_test(&sc->send_io.pending.count))
+		wake_up(&sc->send_io.pending.zero_wait_queue);
+}
+
+static void smbdirect_connection_send_immediate_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, idle.immediate_work);
+	int ret;
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
+		return;
+
+	smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
+		"send an empty message\n");
+	sc->statistics.send_empty++;
+	ret = smbdirect_connection_send_single_iter(sc, NULL, NULL, 0, 0);
+	if (ret < 0) {
+		smbdirect_log_write(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_send_single_iter ret=%1pe\n",
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+		smbdirect_socket_schedule_cleanup(sc, ret);
+	}
+}
+
+int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg)
+{
+	struct smbdirect_socket *sc = msg->socket;
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct ib_recv_wr recv_wr = {
+		.wr_cqe = &msg->cqe,
+		.sg_list = &msg->sge,
+		.num_sge = 1,
+	};
+	int ret;
+
+	if (unlikely(sc->first_error))
+		return sc->first_error;
+
+	msg->sge.addr = ib_dma_map_single(sc->ib.dev,
+					  msg->packet,
+					  sp->max_recv_size,
+					  DMA_FROM_DEVICE);
+	ret = ib_dma_mapping_error(sc->ib.dev, msg->sge.addr);
+	if (ret)
+		return ret;
+
+	msg->sge.length = sp->max_recv_size;
+	msg->sge.lkey = sc->ib.pd->local_dma_lkey;
+
+	ret = ib_post_recv(sc->ib.qp, &recv_wr, NULL);
+	if (ret) {
+		smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR,
+			"ib_post_recv failed ret=%d (%1pe)\n",
+			ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		ib_dma_unmap_single(sc->ib.dev,
+				    msg->sge.addr,
+				    msg->sge.length,
+				    DMA_FROM_DEVICE);
+		msg->sge.length = 0;
+		smbdirect_socket_schedule_cleanup(sc, ret);
+	}
+
+	return ret;
+}
+
+void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_recv_io *recv_io =
+		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
+	struct smbdirect_socket *sc = recv_io->socket;
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_data_transfer *data_transfer;
+	int current_recv_credits;
+	u16 old_recv_credit_target;
+	u16 credits_requested;
+	u16 credits_granted;
+	u16 flags;
+	u32 data_offset;
+	u32 data_length;
+	u32 remaining_data_length;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR,
+				"wc->status=%s (%d) wc->opcode=%d\n",
+				ib_wc_status_msg(wc->status), wc->status, wc->opcode);
+		goto error;
+	}
+
+	smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO,
+		"recv_io=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n",
+		recv_io, sc->recv_io.expected,
+		ib_wc_status_msg(wc->status), wc->opcode,
+		wc->byte_len, wc->pkey_index);
+
+	/*
+	 * Reset timer to the keepalive interval in
+	 * order to trigger our next keepalive message.
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->keepalive_interval_msec));
+
+	ib_dma_sync_single_for_cpu(sc->ib.dev,
+				   recv_io->sge.addr,
+				   recv_io->sge.length,
+				   DMA_FROM_DEVICE);
+
+	if (unlikely(wc->byte_len <
+	    offsetof(struct smbdirect_data_transfer, padding))) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"wc->byte_len=%u < %zu\n",
+			wc->byte_len,
+			offsetof(struct smbdirect_data_transfer, padding));
+		goto error;
+	}
+
+	data_transfer = (struct smbdirect_data_transfer *)recv_io->packet;
+	credits_requested = le16_to_cpu(data_transfer->credits_requested);
+	credits_granted = le16_to_cpu(data_transfer->credits_granted);
+	flags = le16_to_cpu(data_transfer->flags);
+	remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length);
+	data_offset = le32_to_cpu(data_transfer->data_offset);
+	data_length = le32_to_cpu(data_transfer->data_length);
+
+	smbdirect_log_incoming(sc, SMBDIRECT_LOG_INFO,
+		"DataIn: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n",
+		"CreditsRequested",
+		credits_requested,
+		"CreditsGranted",
+		credits_granted,
+		"Flags",
+		flags,
+		"RemainingDataLength",
+		remaining_data_length,
+		"DataOffset",
+		data_offset,
+		"DataLength",
+		data_length);
+
+	if (unlikely(credits_requested == 0)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: credits_requested == 0\n");
+		goto error;
+	}
+
+	if (unlikely(data_offset % 8 != 0)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"invalid: data_offset=%u (0x%x) not aligned to 8\n",
+			data_offset, data_offset);
+		goto error;
+	}
+
+	if (unlikely(wc->byte_len < data_offset ||
+	    (u64)wc->byte_len < (u64)data_offset + data_length)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"wc->byte_len=%u < date_offset=%u + data_length=%u\n",
+			wc->byte_len, data_offset, data_length);
+		goto error;
+	}
+
+	if (unlikely(remaining_data_length > sp->max_fragmented_recv_size ||
+	    data_length > sp->max_fragmented_recv_size ||
+	    (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size)) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"remaining_data_length=%u + data_length=%u > max_fragmented=%u\n",
+			remaining_data_length, data_length, sp->max_fragmented_recv_size);
+		goto error;
+	}
+
+	if (data_length) {
+		if (sc->recv_io.reassembly.full_packet_received)
+			recv_io->first_segment = true;
+
+		if (remaining_data_length)
+			sc->recv_io.reassembly.full_packet_received = false;
+		else
+			sc->recv_io.reassembly.full_packet_received = true;
+	}
+
+	atomic_dec(&sc->recv_io.posted.count);
+	current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count);
+
+	/*
+	 * We take the value from the peer, which is checked to be higher than 0,
+	 * but we limit it to the max value we support in order to have
+	 * the main logic simpler.
+	 */
+	old_recv_credit_target = sc->recv_io.credits.target;
+	sc->recv_io.credits.target = credits_requested;
+	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target,
+					   sp->recv_credit_max);
+	if (credits_granted) {
+		atomic_add(credits_granted, &sc->send_io.credits.count);
+		/*
+		 * We have new send credits granted from remote peer
+		 * If any sender is waiting for credits, unblock it
+		 */
+		wake_up(&sc->send_io.credits.wait_queue);
+	}
+
+	/* Send an immediate response right away if requested */
+	if (flags & SMBDIRECT_FLAG_RESPONSE_REQUESTED) {
+		smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
+			"schedule send of immediate response\n");
+		queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
+	}
+
+	/*
+	 * If this is a packet with data playload place the data in
+	 * reassembly queue and wake up the reading thread
+	 */
+	if (data_length) {
+		if (current_recv_credits <= (sc->recv_io.credits.target / 4) ||
+		    sc->recv_io.credits.target > old_recv_credit_target)
+			queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
+
+		smbdirect_connection_reassembly_append_recv_io(sc, recv_io, data_length);
+		wake_up(&sc->recv_io.reassembly.wait_queue);
+	} else
+		smbdirect_connection_put_recv_io(recv_io);
+
+	return;
+
+error:
+	/*
+	 * Make sure smbdirect_connection_put_recv_io() does not
+	 * start recv_io.posted.refill_work.
+	 */
+	disable_work(&sc->recv_io.posted.refill_work);
+	smbdirect_connection_put_recv_io(recv_io);
+	smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+}
+
+int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc)
+{
+	int missing;
+	int posted = 0;
+
+	if (unlikely(sc->first_error))
+		return sc->first_error;
+
+	/*
+	 * Find out how much smbdirect_recv_io buffers we should post.
+	 *
+	 * Note that sc->recv_io.credits.target is the value
+	 * from the peer and it can in theory change over time,
+	 * but it is forced to be at least 1 and at max
+	 * sp->recv_credit_max.
+	 *
+	 * So it can happen that missing will be lower than 0,
+	 * which means the peer has recently lowered its desired
+	 * target, while be already granted a higher number of credits.
+	 *
+	 * Note 'posted' is the number of smbdirect_recv_io buffers
+	 * posted within this function, while sc->recv_io.posted.count
+	 * is the overall value of posted smbdirect_recv_io buffers.
+	 *
+	 * We try to post as much buffers as missing, but
+	 * this is limited if a lot of smbdirect_recv_io buffers
+	 * are still in the sc->recv_io.reassembly.list instead of
+	 * the sc->recv_io.free.list.
+	 *
+	 */
+	missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.posted.count);
+	while (posted < missing) {
+		struct smbdirect_recv_io *recv_io;
+		int ret;
+
+		/*
+		 * It's ok if smbdirect_connection_get_recv_io()
+		 * returns NULL, it means smbdirect_recv_io structures
+		 * are still be in the reassembly.list.
+		 */
+		recv_io = smbdirect_connection_get_recv_io(sc);
+		if (!recv_io)
+			break;
+
+		recv_io->first_segment = false;
+
+		ret = smbdirect_connection_post_recv_io(recv_io);
+		if (ret) {
+			smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR,
+				"smbdirect_connection_post_recv_io failed rc=%d (%1pe)\n",
+				ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+			smbdirect_connection_put_recv_io(recv_io);
+			return ret;
+		}
+
+		atomic_inc(&sc->recv_io.posted.count);
+		posted += 1;
+	}
+
+	/* If nothing was posted we're done */
+	if (posted == 0)
+		return 0;
+
+	atomic_add(posted, &sc->recv_io.credits.available);
+
+	/*
+	 * If the last send credit is waiting for credits
+	 * it can grant we need to wake it up
+	 */
+	if (atomic_read(&sc->send_io.bcredits.count) == 0 &&
+	    atomic_read(&sc->send_io.credits.count) == 0)
+		wake_up(&sc->send_io.credits.wait_queue);
+
+	/*
+	 * If we posted at least one smbdirect_recv_io buffer,
+	 * we need to inform the peer about it and grant
+	 * additional credits.
+	 *
+	 * However there is one case where we don't want to
+	 * do that.
+	 *
+	 * If only a single credit was missing before
+	 * reaching the requested target, we should not
+	 * post an immediate send, as that would cause
+	 * endless ping pong once a keep alive exchange
+	 * is started.
+	 *
+	 * However if sc->recv_io.credits.target is only 1,
+	 * the peer has no credit left and we need to
+	 * grant the credit anyway.
+	 */
+	if (missing == 1 && sc->recv_io.credits.target != 1)
+		return 0;
+
+	return posted;
+}
+
+static void smbdirect_connection_recv_io_refill_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, recv_io.posted.refill_work);
+	int posted;
+
+	posted = smbdirect_connection_recv_io_refill(sc);
+	if (unlikely(posted < 0)) {
+		smbdirect_socket_schedule_cleanup(sc, posted);
+		return;
+	}
+	if (posted > 0) {
+		smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
+			"schedule send of an empty message\n");
+		queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
+	}
+}
+
+int smbdirect_connection_recvmsg(struct smbdirect_socket *sc,
+				 struct msghdr *msg,
+				 unsigned int flags)
+{
+	struct smbdirect_recv_io *response;
+	struct smbdirect_data_transfer *data_transfer;
+	size_t size = iov_iter_count(&msg->msg_iter);
+	int to_copy, to_read, data_read, offset;
+	u32 data_length, remaining_data_length, data_offset;
+	int ret;
+
+	if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags))
+		return -EINVAL; /* no flags support for now */
+
+	if (WARN_ON_ONCE(iov_iter_rw(&msg->msg_iter) != ITER_DEST))
+		return -EINVAL; /* It's a bug in upper layer to get there */
+
+again:
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+			"status=%s first_error=%1pe => %1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+			SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN));
+		return -ENOTCONN;
+	}
+
+	/*
+	 * No need to hold the reassembly queue lock all the time as we are
+	 * the only one reading from the front of the queue. The transport
+	 * may add more entries to the back of the queue at the same time
+	 */
+	smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+		"size=%zd sc->recv_io.reassembly.data_length=%d\n",
+		size, sc->recv_io.reassembly.data_length);
+	if (sc->recv_io.reassembly.data_length >= size) {
+		int queue_length;
+		int queue_removed = 0;
+		unsigned long flags;
+
+		/*
+		 * Need to make sure reassembly_data_length is read before
+		 * reading reassembly_queue_length and calling
+		 * smbdirect_connection_reassembly_first_recv_io. This call is lock free
+		 * as we never read at the end of the queue which are being
+		 * updated in SOFTIRQ as more data is received
+		 */
+		virt_rmb();
+		queue_length = sc->recv_io.reassembly.queue_length;
+		data_read = 0;
+		to_read = size;
+		offset = sc->recv_io.reassembly.first_entry_offset;
+		while (data_read < size) {
+			response = smbdirect_connection_reassembly_first_recv_io(sc);
+			data_transfer = (void *)response->packet;
+			data_length = le32_to_cpu(data_transfer->data_length);
+			remaining_data_length =
+				le32_to_cpu(
+					data_transfer->remaining_data_length);
+			data_offset = le32_to_cpu(data_transfer->data_offset);
+
+			/*
+			 * The upper layer expects RFC1002 length at the
+			 * beginning of the payload. Return it to indicate
+			 * the total length of the packet. This minimize the
+			 * change to upper layer packet processing logic. This
+			 * will be eventually remove when an intermediate
+			 * transport layer is added
+			 */
+			if (response->first_segment && size == 4) {
+				unsigned int rfc1002_len =
+					data_length + remaining_data_length;
+				__be32 rfc1002_hdr = cpu_to_be32(rfc1002_len);
+
+				if (copy_to_iter(&rfc1002_hdr, sizeof(rfc1002_hdr),
+						 &msg->msg_iter) != sizeof(rfc1002_hdr))
+					return -EFAULT;
+				data_read = 4;
+				response->first_segment = false;
+				smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+					"returning rfc1002 length %d\n",
+					rfc1002_len);
+				goto read_rfc1002_done;
+			}
+
+			to_copy = min_t(int, data_length - offset, to_read);
+			if (copy_to_iter((u8 *)data_transfer + data_offset + offset,
+					 to_copy, &msg->msg_iter) != to_copy)
+				return -EFAULT;
+
+			/* move on to the next buffer? */
+			if (to_copy == data_length - offset) {
+				queue_length--;
+				/*
+				 * No need to lock if we are not at the
+				 * end of the queue
+				 */
+				if (queue_length)
+					list_del(&response->list);
+				else {
+					spin_lock_irqsave(
+						&sc->recv_io.reassembly.lock, flags);
+					list_del(&response->list);
+					spin_unlock_irqrestore(
+						&sc->recv_io.reassembly.lock, flags);
+				}
+				queue_removed++;
+				sc->statistics.dequeue_reassembly_queue++;
+				smbdirect_connection_put_recv_io(response);
+				offset = 0;
+				smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+					"smbdirect_connection_put_recv_io offset=0\n");
+			} else
+				offset += to_copy;
+
+			to_read -= to_copy;
+			data_read += to_copy;
+
+			smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+				 "memcpy %d bytes len-ofs=%u => todo=%u done=%u ofs=%u\n",
+				 to_copy, data_length - offset,
+				 to_read, data_read, offset);
+		}
+
+		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
+		sc->recv_io.reassembly.data_length -= data_read;
+		sc->recv_io.reassembly.queue_length -= queue_removed;
+		spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+
+		sc->recv_io.reassembly.first_entry_offset = offset;
+		smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+			 "returning data_read=%d reassembly_length=%d first_ofs=%u\n",
+			 data_read, sc->recv_io.reassembly.data_length,
+			 sc->recv_io.reassembly.first_entry_offset);
+read_rfc1002_done:
+		return data_read;
+	}
+
+	smbdirect_log_read(sc, SMBDIRECT_LOG_INFO,
+		"wait_event on more data\n");
+	ret = wait_event_interruptible(sc->recv_io.reassembly.wait_queue,
+				       sc->recv_io.reassembly.data_length >= size ||
+				       sc->status != SMBDIRECT_SOCKET_CONNECTED);
+	/* Don't return any data if interrupted */
+	if (ret)
+		return ret;
+
+	goto again;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_recvmsg);
+
+static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state,
+					   struct page *page, size_t off, size_t len)
+{
+	struct ib_sge *sge;
+	u64 addr;
+
+	if (state->num_sge >= state->max_sge)
+		return false;
+
+	addr = ib_dma_map_page(state->device, page,
+			       off, len, state->direction);
+	if (ib_dma_mapping_error(state->device, addr))
+		return false;
+
+	sge = &state->sge[state->num_sge++];
+	sge->addr   = addr;
+	sge->length = len;
+	sge->lkey   = state->local_dma_lkey;
+
+	return true;
+}
+
+/*
+ * Extract page fragments from a BVEC-class iterator and add them to an ib_sge
+ * list.  The pages are not pinned.
+ */
+static ssize_t smbdirect_map_sges_from_bvec(struct iov_iter *iter,
+					    struct smbdirect_map_sges *state,
+					    ssize_t maxsize)
+{
+	const struct bio_vec *bv = iter->bvec;
+	unsigned long start = iter->iov_offset;
+	unsigned int i;
+	ssize_t ret = 0;
+
+	for (i = 0; i < iter->nr_segs; i++) {
+		size_t off, len;
+		bool ok;
+
+		len = bv[i].bv_len;
+		if (start >= len) {
+			start -= len;
+			continue;
+		}
+
+		len = min_t(size_t, maxsize, len - start);
+		off = bv[i].bv_offset + start;
+
+		ok = smbdirect_map_sges_single_page(state,
+						    bv[i].bv_page,
+						    off,
+						    len);
+		if (!ok)
+			return -EIO;
+
+		ret += len;
+		maxsize -= len;
+		if (state->num_sge >= state->max_sge || maxsize <= 0)
+			break;
+		start = 0;
+	}
+
+	if (ret > 0)
+		iov_iter_advance(iter, ret);
+	return ret;
+}
+
+/*
+ * Extract fragments from a KVEC-class iterator and add them to an ib_sge list.
+ * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers.
+ * The pages are not pinned.
+ */
+static ssize_t smbdirect_map_sges_from_kvec(struct iov_iter *iter,
+					    struct smbdirect_map_sges *state,
+					    ssize_t maxsize)
+{
+	const struct kvec *kv = iter->kvec;
+	unsigned long start = iter->iov_offset;
+	unsigned int i;
+	ssize_t ret = 0;
+
+	for (i = 0; i < iter->nr_segs; i++) {
+		struct page *page;
+		unsigned long kaddr;
+		size_t off, len, seg;
+
+		len = kv[i].iov_len;
+		if (start >= len) {
+			start -= len;
+			continue;
+		}
+
+		kaddr = (unsigned long)kv[i].iov_base + start;
+		off = kaddr & ~PAGE_MASK;
+		len = min_t(size_t, maxsize, len - start);
+		kaddr &= PAGE_MASK;
+
+		maxsize -= len;
+		do {
+			bool ok;
+
+			seg = min_t(size_t, len, PAGE_SIZE - off);
+
+			if (is_vmalloc_or_module_addr((void *)kaddr))
+				page = vmalloc_to_page((void *)kaddr);
+			else
+				page = virt_to_page((void *)kaddr);
+
+			ok = smbdirect_map_sges_single_page(state, page, off, seg);
+			if (!ok)
+				return -EIO;
+
+			ret += seg;
+			len -= seg;
+			kaddr += PAGE_SIZE;
+			off = 0;
+		} while (len > 0 && state->num_sge < state->max_sge);
+
+		if (state->num_sge >= state->max_sge || maxsize <= 0)
+			break;
+		start = 0;
+	}
+
+	if (ret > 0)
+		iov_iter_advance(iter, ret);
+	return ret;
+}
+
+/*
+ * Extract folio fragments from a FOLIOQ-class iterator and add them to an
+ * ib_sge list.  The folios are not pinned.
+ */
+static ssize_t smbdirect_map_sges_from_folioq(struct iov_iter *iter,
+					      struct smbdirect_map_sges *state,
+					      ssize_t maxsize)
+{
+	const struct folio_queue *folioq = iter->folioq;
+	unsigned int slot = iter->folioq_slot;
+	ssize_t ret = 0;
+	size_t offset = iter->iov_offset;
+
+	if (WARN_ON_ONCE(!folioq))
+		return -EIO;
+
+	if (slot >= folioq_nr_slots(folioq)) {
+		folioq = folioq->next;
+		if (WARN_ON_ONCE(!folioq))
+			return -EIO;
+		slot = 0;
+	}
+
+	do {
+		struct folio *folio = folioq_folio(folioq, slot);
+		size_t fsize = folioq_folio_size(folioq, slot);
+
+		if (offset < fsize) {
+			size_t part = umin(maxsize, fsize - offset);
+			bool ok;
+
+			ok = smbdirect_map_sges_single_page(state,
+							    folio_page(folio, 0),
+							    offset,
+							    part);
+			if (!ok)
+				return -EIO;
+
+			offset += part;
+			ret += part;
+			maxsize -= part;
+		}
+
+		if (offset >= fsize) {
+			offset = 0;
+			slot++;
+			if (slot >= folioq_nr_slots(folioq)) {
+				if (!folioq->next) {
+					WARN_ON_ONCE(ret < iter->count);
+					break;
+				}
+				folioq = folioq->next;
+				slot = 0;
+			}
+		}
+	} while (state->num_sge < state->max_sge && maxsize > 0);
+
+	iter->folioq = folioq;
+	iter->folioq_slot = slot;
+	iter->iov_offset = offset;
+	iter->count -= ret;
+	return ret;
+}
+
+/*
+ * Extract page fragments from up to the given amount of the source iterator
+ * and build up an ib_sge list that refers to all of those bits.  The ib_sge list
+ * is appended to, up to the maximum number of elements set in the parameter
+ * block.
+ *
+ * The extracted page fragments are not pinned or ref'd in any way; if an
+ * IOVEC/UBUF-type iterator is to be used, it should be converted to a
+ * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some
+ * way.
+ */
+static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len,
+					    struct smbdirect_map_sges *state)
+{
+	ssize_t ret;
+	size_t before = state->num_sge;
+
+	if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE))
+		return -EIO;
+
+	switch (iov_iter_type(iter)) {
+	case ITER_BVEC:
+		ret = smbdirect_map_sges_from_bvec(iter, state, len);
+		break;
+	case ITER_KVEC:
+		ret = smbdirect_map_sges_from_kvec(iter, state, len);
+		break;
+	case ITER_FOLIOQ:
+		ret = smbdirect_map_sges_from_folioq(iter, state, len);
+		break;
+	default:
+		WARN_ONCE(1, "iov_iter_type[%u]\n", iov_iter_type(iter));
+		return -EIO;
+	}
+
+	if (ret < 0) {
+		while (state->num_sge > before) {
+			struct ib_sge *sge = &state->sge[state->num_sge--];
+
+			ib_dma_unmap_page(state->device,
+					  sge->addr,
+					  sge->length,
+					  state->direction);
+		}
+	}
+
+	return ret;
+}
diff --git a/fs/smb/common/smbdirect/smbdirect_debug.c b/fs/smb/common/smbdirect/smbdirect_debug.c
new file mode 100644
index 0000000000000..d8664fd7f71a1
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_debug.c
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+#include <linux/seq_file.h>
+
+void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
+						 unsigned int rdma_readwrite_threshold,
+						 struct seq_file *m)
+{
+	const struct smbdirect_socket_parameters *sp;
+
+	if (!sc)
+		return;
+	sp = &sc->parameters;
+
+	seq_puts(m, "\n");
+	seq_printf(m, "SMBDirect protocol version: 0x%x ",
+		   SMBDIRECT_V1);
+	seq_printf(m, "transport status: %s (%u)",
+		   smbdirect_socket_status_string(sc->status),
+		   sc->status);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Conn receive_credit_max: %u ",
+		   sp->recv_credit_max);
+	seq_printf(m, "send_credit_target: %u max_send_size: %u",
+		   sp->send_credit_target,
+		   sp->max_send_size);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Conn max_fragmented_recv_size: %u ",
+		   sp->max_fragmented_recv_size);
+	seq_printf(m, "max_fragmented_send_size: %u max_receive_size:%u",
+		   sp->max_fragmented_send_size,
+		   sp->max_recv_size);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Conn keep_alive_interval: %u ",
+		   sp->keepalive_interval_msec * 1000);
+	seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u",
+		   sp->max_read_write_size,
+		   rdma_readwrite_threshold);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Debug count_get_receive_buffer: %llu ",
+		   sc->statistics.get_receive_buffer);
+	seq_printf(m, "count_put_receive_buffer: %llu count_send_empty: %llu",
+		   sc->statistics.put_receive_buffer,
+		   sc->statistics.send_empty);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Read Queue count_enqueue_reassembly_queue: %llu ",
+		   sc->statistics.enqueue_reassembly_queue);
+	seq_printf(m, "count_dequeue_reassembly_queue: %llu ",
+		   sc->statistics.dequeue_reassembly_queue);
+	seq_printf(m, "reassembly_data_length: %u ",
+		   sc->recv_io.reassembly.data_length);
+	seq_printf(m, "reassembly_queue_length: %u",
+		   sc->recv_io.reassembly.queue_length);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Current Credits send_credits: %u ",
+		   atomic_read(&sc->send_io.credits.count));
+	seq_printf(m, "receive_credits: %u receive_credit_target: %u",
+		   atomic_read(&sc->recv_io.credits.count),
+		   sc->recv_io.credits.target);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "Pending send_pending: %u ",
+		   atomic_read(&sc->send_io.pending.count));
+
+	seq_puts(m, "\n");
+	seq_printf(m, "MR responder_resources: %u ",
+		   sp->responder_resources);
+	seq_printf(m, "max_frmr_depth: %u mr_type: 0x%x",
+		   sp->max_frmr_depth,
+		   sc->mr_io.type);
+
+	seq_puts(m, "\n");
+	seq_printf(m, "MR mr_ready_count: %u mr_used_count: %u",
+		   atomic_read(&sc->mr_io.ready.count),
+		   atomic_read(&sc->mr_io.used.count));
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_legacy_debug_proc_show);
diff --git a/fs/smb/common/smbdirect/smbdirect_devices.c b/fs/smb/common/smbdirect/smbdirect_devices.c
new file mode 100644
index 0000000000000..aaab99e9c0454
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_devices.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (C) 2018, LG Electronics.
+ *   Copyright (c) 2025 Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+
+static u8 smbdirect_ib_device_rdma_capable_node_type(struct ib_device *ib_dev)
+{
+	if (!smbdirect_frwr_is_supported(&ib_dev->attrs))
+		return RDMA_NODE_UNSPECIFIED;
+
+	switch (ib_dev->node_type) {
+	case RDMA_NODE_IB_CA: /* Infiniband, RoCE v1 and v2 */
+	case RDMA_NODE_RNIC:  /* iWarp */
+		return ib_dev->node_type;
+	}
+
+	return RDMA_NODE_UNSPECIFIED;
+}
+
+static int smbdirect_ib_client_add(struct ib_device *ib_dev)
+{
+	u8 node_type = smbdirect_ib_device_rdma_capable_node_type(ib_dev);
+	struct smbdirect_device *sdev;
+	const char *node_str;
+	const char *action;
+	u32 pidx;
+
+	switch (node_type) {
+	case RDMA_NODE_IB_CA:
+		node_str = "IB_CA";
+		action = "added";
+		break;
+	case RDMA_NODE_RNIC:
+		node_str = "RNIC";
+		action = "added";
+		break;
+	case RDMA_NODE_UNSPECIFIED:
+		node_str = "UNSPECIFIED";
+		action = "ignored";
+		break;
+	default:
+		node_str = "UNKNOWN";
+		action = "ignored";
+		node_type = RDMA_NODE_UNSPECIFIED;
+		break;
+	}
+
+	pr_info("ib_dev[%.*s]: %s: %s %s=%u %s=0x%llx %s=0x%llx %s=0x%llx\n",
+		IB_DEVICE_NAME_MAX,
+		ib_dev->name,
+		action,
+		node_str,
+		"max_fast_reg_page_list_len",
+		ib_dev->attrs.max_fast_reg_page_list_len,
+		"device_cap_flags",
+		ib_dev->attrs.device_cap_flags,
+		"kernel_cap_flags",
+		ib_dev->attrs.kernel_cap_flags,
+		"page_size_cap",
+		ib_dev->attrs.page_size_cap);
+
+	if (node_type == RDMA_NODE_UNSPECIFIED)
+		return 0;
+
+	pr_info("ib_dev[%.*s]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u\n",
+		IB_DEVICE_NAME_MAX,
+		ib_dev->name,
+		"num_ports",
+		rdma_end_port(ib_dev),
+		"max_qp_rd_atom",
+		ib_dev->attrs.max_qp_rd_atom,
+		"max_qp_init_rd_atom",
+		ib_dev->attrs.max_qp_init_rd_atom,
+		"max_sgl_rd",
+		ib_dev->attrs.max_sgl_rd,
+		"max_sge_rd",
+		ib_dev->attrs.max_sge_rd,
+		"max_cqe",
+		ib_dev->attrs.max_cqe,
+		"max_qp_wr",
+		ib_dev->attrs.max_qp_wr,
+		"max_send_sge",
+		ib_dev->attrs.max_send_sge,
+		"max_recv_sge",
+		ib_dev->attrs.max_recv_sge);
+
+	rdma_for_each_port(ib_dev, pidx) {
+		const struct ib_port_immutable *ib_pi =
+			ib_port_immutable_read(ib_dev, pidx);
+		u32 core_cap_flags = ib_pi ? ib_pi->core_cap_flags : 0;
+
+		pr_info("ib_dev[%.*s]PORT[%u]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=0x%x\n",
+			IB_DEVICE_NAME_MAX,
+			ib_dev->name,
+			pidx,
+			"iwarp",
+			rdma_protocol_iwarp(ib_dev, pidx),
+			"ib",
+			rdma_protocol_ib(ib_dev, pidx),
+			"roce",
+			rdma_protocol_roce(ib_dev, pidx),
+			"v1",
+			rdma_protocol_roce_eth_encap(ib_dev, pidx),
+			"v2",
+			rdma_protocol_roce_udp_encap(ib_dev, pidx),
+			"core_cap_flags",
+			core_cap_flags);
+	}
+
+	sdev = kzalloc_obj(*sdev);
+	if (!sdev)
+		return -ENOMEM;
+	sdev->ib_dev = ib_dev;
+	snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s",
+		 IB_DEVICE_NAME_MAX, ib_dev->name);
+
+	write_lock(&smbdirect_globals.devices.lock);
+	list_add(&sdev->list, &smbdirect_globals.devices.list);
+	write_unlock(&smbdirect_globals.devices.lock);
+
+	return 0;
+}
+
+static void smbdirect_ib_client_remove(struct ib_device *ib_dev, void *client_data)
+{
+	struct smbdirect_device *sdev, *tmp;
+
+	write_lock(&smbdirect_globals.devices.lock);
+	list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) {
+		if (sdev->ib_dev == ib_dev) {
+			list_del(&sdev->list);
+			pr_info("ib_dev[%.*s] removed\n",
+				IB_DEVICE_NAME_MAX, sdev->ib_name);
+			kfree(sdev);
+			break;
+		}
+	}
+	write_unlock(&smbdirect_globals.devices.lock);
+}
+
+static void smbdirect_ib_client_rename(struct ib_device *ib_dev, void *client_data)
+{
+	struct smbdirect_device *sdev;
+
+	write_lock(&smbdirect_globals.devices.lock);
+	list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) {
+		if (sdev->ib_dev == ib_dev) {
+			pr_info("ib_dev[%.*s] renamed to [%.*s]\n",
+				IB_DEVICE_NAME_MAX, sdev->ib_name,
+				IB_DEVICE_NAME_MAX, ib_dev->name);
+			snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s",
+				 IB_DEVICE_NAME_MAX, ib_dev->name);
+			break;
+		}
+	}
+	write_unlock(&smbdirect_globals.devices.lock);
+}
+
+static struct ib_client smbdirect_ib_client = {
+	.name	= "smbdirect_ib_client",
+	.add	= smbdirect_ib_client_add,
+	.remove	= smbdirect_ib_client_remove,
+	.rename	= smbdirect_ib_client_rename,
+};
+
+static u8 smbdirect_netdev_find_rdma_capable_node_type(struct net_device *netdev)
+{
+	struct smbdirect_device *sdev;
+	u8 node_type = RDMA_NODE_UNSPECIFIED;
+
+	read_lock(&smbdirect_globals.devices.lock);
+	list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) {
+		u32 pi;
+
+		rdma_for_each_port(sdev->ib_dev, pi) {
+			struct net_device *ndev;
+
+			ndev = ib_device_get_netdev(sdev->ib_dev, pi);
+			if (!ndev)
+				continue;
+
+			if (ndev == netdev) {
+				dev_put(ndev);
+				node_type = sdev->ib_dev->node_type;
+				goto out;
+			}
+			dev_put(ndev);
+		}
+	}
+out:
+	read_unlock(&smbdirect_globals.devices.lock);
+
+	if (node_type == RDMA_NODE_UNSPECIFIED) {
+		struct ib_device *ibdev;
+
+		ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
+		if (ibdev) {
+			node_type = smbdirect_ib_device_rdma_capable_node_type(ibdev);
+			ib_device_put(ibdev);
+		}
+	}
+
+	return node_type;
+}
+
+/*
+ * Returns RDMA_NODE_UNSPECIFIED when the netdev has
+ * no support for smbdirect capable rdma.
+ *
+ * Otherwise RDMA_NODE_RNIC is returned for iwarp devices
+ * and RDMA_NODE_IB_CA or Infiniband and RoCE (v1 and v2)
+ */
+u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev)
+{
+	struct net_device *lower_dev;
+	struct list_head *iter;
+	u8 node_type = RDMA_NODE_UNSPECIFIED;
+
+	node_type = smbdirect_netdev_find_rdma_capable_node_type(netdev);
+	if (node_type != RDMA_NODE_UNSPECIFIED)
+		return node_type;
+
+	/* check if netdev is bridge or VLAN */
+	if (netif_is_bridge_master(netdev) || netdev->priv_flags & IFF_802_1Q_VLAN)
+		netdev_for_each_lower_dev(netdev, lower_dev, iter) {
+			node_type = smbdirect_netdev_find_rdma_capable_node_type(lower_dev);
+			if (node_type != RDMA_NODE_UNSPECIFIED)
+				return node_type;
+		}
+
+	/* check if netdev is IPoIB safely without layer violation */
+	if (netdev->type == ARPHRD_INFINIBAND)
+		return RDMA_NODE_IB_CA;
+
+	return RDMA_NODE_UNSPECIFIED;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_netdev_rdma_capable_node_type);
+
+__init int smbdirect_devices_init(void)
+{
+	int ret;
+
+	rwlock_init(&smbdirect_globals.devices.lock);
+	INIT_LIST_HEAD(&smbdirect_globals.devices.list);
+
+	ret = ib_register_client(&smbdirect_ib_client);
+	if (ret) {
+		pr_crit("failed to ib_register_client: %d %1pe\n",
+			ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+__exit void smbdirect_devices_exit(void)
+{
+	struct smbdirect_device *sdev, *tmp;
+
+	/*
+	 * On exist we just cleanup so that
+	 * smbdirect_ib_client_remove() won't
+	 * print removals of devices.
+	 */
+	write_lock(&smbdirect_globals.devices.lock);
+	list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) {
+		list_del(&sdev->list);
+		kfree(sdev);
+	}
+	write_unlock(&smbdirect_globals.devices.lock);
+
+	ib_unregister_client(&smbdirect_ib_client);
+}
diff --git a/fs/smb/common/smbdirect/smbdirect_internal.h b/fs/smb/common/smbdirect/smbdirect_internal.h
new file mode 100644
index 0000000000000..30a1b86436574
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_internal.h
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__
+#define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "smbdirect.h"
+#include "smbdirect_pdu.h"
+#include "smbdirect_public.h"
+
+#include <linux/mutex.h>
+
+struct smbdirect_module_state {
+	struct mutex mutex;
+
+	struct {
+		struct workqueue_struct *accept;
+		struct workqueue_struct *connect;
+		struct workqueue_struct *idle;
+		struct workqueue_struct *refill;
+		struct workqueue_struct *immediate;
+		struct workqueue_struct *cleanup;
+	} workqueues;
+
+	struct {
+		rwlock_t lock;
+		struct list_head list;
+	} devices;
+};
+
+extern struct smbdirect_module_state smbdirect_globals;
+
+#include "smbdirect_socket.h"
+
+struct smbdirect_device {
+	struct list_head list;
+	struct ib_device *ib_dev;
+	/*
+	 * copy of ib_dev->name,
+	 * in order to print renames
+	 */
+	char ib_name[IB_DEVICE_NAME_MAX];
+};
+
+int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc);
+
+int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc);
+
+void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc,
+					 const char *macro_name,
+					 unsigned int lvl,
+					 const char *func,
+					 unsigned int line,
+					 int error,
+					 enum smbdirect_socket_status *force_status);
+#define smbdirect_socket_schedule_cleanup(__sc, __error) \
+	__smbdirect_socket_schedule_cleanup(__sc, \
+		"smbdirect_socket_schedule_cleanup", SMBDIRECT_LOG_ERR, \
+		__func__, __LINE__, __error, NULL)
+#define smbdirect_socket_schedule_cleanup_lvl(__sc, __lvl, __error) \
+	__smbdirect_socket_schedule_cleanup(__sc, \
+		"smbdirect_socket_schedule_cleanup_lvl", __lvl, \
+		__func__, __LINE__, __error, NULL)
+#define smbdirect_socket_schedule_cleanup_status(__sc, __lvl, __error, __status) do { \
+	enum smbdirect_socket_status __force_status = __status; \
+	__smbdirect_socket_schedule_cleanup(__sc, \
+		"smbdirect_socket_schedule_cleanup_status", __lvl, \
+		__func__, __LINE__, __error, &__force_status); \
+} while (0)
+
+void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc);
+
+int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc,
+				      enum smbdirect_socket_status expected_status,
+				      int unexpected_errno,
+				      wait_queue_head_t *waitq,
+				      atomic_t *total_credits,
+				      int needed);
+
+void smbdirect_connection_rdma_established(struct smbdirect_socket *sc);
+
+void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc);
+
+int smbdirect_connection_create_qp(struct smbdirect_socket *sc);
+
+void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc);
+
+int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc);
+
+void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc);
+
+struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc);
+
+void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg);
+
+struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc);
+
+void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg);
+
+void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc,
+						    struct smbdirect_recv_io *msg,
+						    u32 data_length);
+
+struct smbdirect_recv_io *
+smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc);
+
+void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc,
+						   u8 peer_initiator_depth,
+						   u8 peer_responder_resources,
+						   const struct rdma_conn_param *param);
+
+void smbdirect_connection_idle_timer_work(struct work_struct *work);
+
+u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc);
+
+int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc,
+				      struct ib_send_wr *wr);
+
+int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg);
+
+void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc);
+
+int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc);
+
+int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc);
+
+void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc);
+
+int smbdirect_accept_connect_request(struct smbdirect_socket *sc,
+				     const struct rdma_conn_param *param);
+
+void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus);
+
+__init int smbdirect_devices_init(void);
+__exit void smbdirect_devices_exit(void);
+
+#endif /* __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ */
diff --git a/fs/smb/common/smbdirect/smbdirect_listen.c b/fs/smb/common/smbdirect/smbdirect_listen.c
new file mode 100644
index 0000000000000..05c7902e7020c
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_listen.c
@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (C) 2018, LG Electronics.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+
+static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *id,
+					       struct rdma_cm_event *event);
+
+int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog)
+{
+	int ret;
+
+	if (backlog < 0)
+		return -EINVAL;
+	if (!backlog)
+		backlog = 1; /* use 1 as default for now */
+
+	if (sc->first_error)
+		return -EINVAL;
+
+	if (sc->status != SMBDIRECT_SOCKET_CREATED)
+		return -EINVAL;
+
+	if (WARN_ON_ONCE(!sc->rdma.cm_id))
+		return -EINVAL;
+
+	if (sc->rdma.cm_id->device)
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"try to listen on addr: %pISpsfc dev: %.*s\n",
+			&sc->rdma.cm_id->route.addr.src_addr,
+			IB_DEVICE_NAME_MAX,
+			sc->rdma.cm_id->device->name);
+	else
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"try to listen on addr: %pISpsfc\n",
+			&sc->rdma.cm_id->route.addr.src_addr);
+
+	/* already checked above */
+	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);
+	sc->status = SMBDIRECT_SOCKET_LISTENING;
+	sc->rdma.expected_event = RDMA_CM_EVENT_CONNECT_REQUEST;
+	rdma_lock_handler(sc->rdma.cm_id);
+	sc->rdma.cm_id->event_handler = smbdirect_listen_rdma_event_handler;
+	rdma_unlock_handler(sc->rdma.cm_id);
+
+	ret = rdma_listen(sc->rdma.cm_id, backlog);
+	if (ret) {
+		sc->first_error = ret;
+		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
+		if (sc->rdma.cm_id->device)
+			smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+				"listening failed %1pe on addr: %pISpsfc dev: %.*s\n",
+				SMBDIRECT_DEBUG_ERR_PTR(ret),
+				&sc->rdma.cm_id->route.addr.src_addr,
+				IB_DEVICE_NAME_MAX,
+				sc->rdma.cm_id->device->name);
+		else
+			smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+				"listening failed %1pe on addr: %pISpsfc\n",
+				SMBDIRECT_DEBUG_ERR_PTR(ret),
+				&sc->rdma.cm_id->route.addr.src_addr);
+		return ret;
+	}
+
+	/*
+	 * This is a value > 0, checked above,
+	 * so we are able to use sc->listen.backlog == -1,
+	 * as indication that the socket was never
+	 * a listener.
+	 */
+	sc->listen.backlog = backlog;
+
+	if (sc->rdma.cm_id->device)
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"listening on addr: %pISpsfc dev: %.*s\n",
+			&sc->rdma.cm_id->route.addr.src_addr,
+			IB_DEVICE_NAME_MAX,
+			sc->rdma.cm_id->device->name);
+	else
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"listening on addr: %pISpsfc\n",
+			&sc->rdma.cm_id->route.addr.src_addr);
+
+	/*
+	 * The rest happens async via smbdirect_listen_rdma_event_handler()
+	 */
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_listen);
+
+static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id,
+					    struct rdma_cm_event *event)
+{
+	int ret = -ESTALE;
+
+	/*
+	 * This should be replaced before any real work
+	 * starts! So it should never be called!
+	 */
+
+	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+		ret = -ENETDOWN;
+	if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+		ret = event->status;
+	WARN_ONCE(1,
+		  "%s should not be called! event=%s status=%d => ret=%1pe\n",
+		  __func__,
+		  rdma_event_msg(event->event),
+		  event->status,
+		  SMBDIRECT_DEBUG_ERR_PTR(ret));
+	return -ESTALE;
+}
+
+static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc,
+					    struct rdma_cm_id *new_id,
+					    const struct rdma_cm_event *event);
+
+static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *new_id,
+					       struct rdma_cm_event *event)
+{
+	struct smbdirect_socket *lsc = new_id->context;
+	int ret;
+
+	if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
+		new_id->context = NULL;
+		new_id->event_handler = smbdirect_new_rdma_event_handler;
+	} else
+		new_id = NULL;
+
+	/*
+	 * cma_cm_event_handler() has
+	 * lockdep_assert_held(&id_priv->handler_mutex);
+	 *
+	 * Mutexes are not allowed in interrupts,
+	 * and we rely on not being in an interrupt here,
+	 * as we might sleep.
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	if (event->status || event->event != lsc->rdma.expected_event) {
+		ret = -ECONNABORTED;
+
+		if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+			ret = -ENETDOWN;
+		if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+			ret = event->status;
+
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n",
+			smbdirect_socket_status_string(lsc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error),
+			rdma_event_msg(lsc->rdma.expected_event),
+			rdma_event_msg(event->event),
+			event->status,
+			SMBDIRECT_DEBUG_ERR_PTR(ret));
+
+		/*
+		 * In case of error return it and let the caller
+		 * destroy new_id
+		 */
+		smbdirect_socket_schedule_cleanup(lsc, ret);
+		return new_id ? ret : 0;
+	}
+
+	smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_INFO,
+		"%s (first_error=%1pe) event=%s\n",
+		smbdirect_socket_status_string(lsc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error),
+		rdma_event_msg(event->event));
+
+	/*
+	 * In case of error return it and let the caller
+	 * destroy new_id
+	 */
+	if (lsc->first_error)
+		return new_id ? lsc->first_error : 0;
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_CONNECT_REQUEST:
+		WARN_ON_ONCE(lsc->status != SMBDIRECT_SOCKET_LISTENING);
+
+		/*
+		 * In case of error return it and let the caller
+		 * destroy new_id
+		 */
+		ret = smbdirect_listen_connect_request(lsc, new_id, event);
+		if (ret)
+			return ret;
+		return 0;
+
+	default:
+		break;
+	}
+
+	/*
+	 * This is an internal error
+	 */
+	WARN_ON_ONCE(lsc->rdma.expected_event != RDMA_CM_EVENT_CONNECT_REQUEST);
+	smbdirect_socket_schedule_cleanup(lsc, -EINVAL);
+	return 0;
+}
+
+static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc,
+					    struct rdma_cm_id *new_id,
+					    const struct rdma_cm_event *event)
+{
+	const struct smbdirect_socket_parameters *lsp = &lsc->parameters;
+	struct smbdirect_socket *nsc;
+	unsigned long flags;
+	size_t backlog = max_t(size_t, 1, lsc->listen.backlog);
+	size_t psockets;
+	size_t rsockets;
+	int ret;
+
+	if (!smbdirect_frwr_is_supported(&new_id->device->attrs)) {
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"Fast Registration Work Requests (FRWR) is not supported device %.*s\n",
+			IB_DEVICE_NAME_MAX,
+			new_id->device->name);
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"Device capability flags = %llx max_fast_reg_page_list_len = %u\n",
+			new_id->device->attrs.device_cap_flags,
+			new_id->device->attrs.max_fast_reg_page_list_len);
+		return -EPROTONOSUPPORT;
+	}
+
+	if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB &&
+	    !rdma_ib_or_roce(new_id->device, new_id->port_num)) {
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n",
+			IB_DEVICE_NAME_MAX,
+			new_id->device->name,
+			rdma_protocol_iwarp(new_id->device, new_id->port_num),
+			&new_id->route.addr.src_addr,
+			&new_id->route.addr.dst_addr);
+		return -EPROTONOSUPPORT;
+	}
+	if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW &&
+	    !rdma_protocol_iwarp(new_id->device, new_id->port_num)) {
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n",
+			IB_DEVICE_NAME_MAX,
+			new_id->device->name,
+			rdma_ib_or_roce(new_id->device, new_id->port_num),
+			&new_id->route.addr.src_addr,
+			&new_id->route.addr.dst_addr);
+		return -EPROTONOSUPPORT;
+	}
+
+	spin_lock_irqsave(&lsc->listen.lock, flags);
+	psockets = list_count_nodes(&lsc->listen.pending);
+	rsockets = list_count_nodes(&lsc->listen.ready);
+	spin_unlock_irqrestore(&lsc->listen.lock, flags);
+
+	if (psockets > backlog ||
+	    rsockets > backlog ||
+	    (psockets + rsockets) > backlog) {
+		smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR,
+			"Backlog[%d][%zu] full pending[%zu] ready[%zu]\n",
+			lsc->listen.backlog, backlog, psockets, rsockets);
+		return -EBUSY;
+	}
+
+	ret = smbdirect_socket_create_accepting(new_id, &nsc);
+	if (ret)
+		goto socket_init_failed;
+
+	nsc->logging = lsc->logging;
+	ret = smbdirect_socket_set_initial_parameters(nsc, &lsc->parameters);
+	if (ret)
+		goto set_params_failed;
+	ret = smbdirect_socket_set_kernel_settings(nsc,
+						   lsc->ib.poll_ctx,
+						   lsc->send_io.mem.gfp_mask);
+	if (ret)
+		goto set_settings_failed;
+
+	spin_lock_irqsave(&lsc->listen.lock, flags);
+	list_add_tail(&nsc->accept.list, &lsc->listen.pending);
+	nsc->accept.listener = lsc;
+	spin_unlock_irqrestore(&lsc->listen.lock, flags);
+
+	ret = smbdirect_accept_connect_request(nsc, &event->param.conn);
+	if (ret)
+		goto accept_connect_failed;
+
+	return 0;
+
+accept_connect_failed:
+	spin_lock_irqsave(&lsc->listen.lock, flags);
+	list_del_init(&nsc->accept.list);
+	nsc->accept.listener = NULL;
+	spin_unlock_irqrestore(&lsc->listen.lock, flags);
+set_settings_failed:
+set_params_failed:
+	/*
+	 * The caller will destroy new_id
+	 */
+	nsc->ib.dev = NULL;
+	nsc->rdma.cm_id = NULL;
+	smbdirect_socket_release(nsc);
+socket_init_failed:
+	return ret;
+}
diff --git a/fs/smb/common/smbdirect/smbdirect_main.c b/fs/smb/common/smbdirect/smbdirect_main.c
new file mode 100644
index 0000000000000..fe6e8d93c34c6
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_main.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+#include <linux/module.h>
+
+struct smbdirect_module_state smbdirect_globals = {
+	.mutex = __MUTEX_INITIALIZER(smbdirect_globals.mutex),
+};
+
+static __init int smbdirect_module_init(void)
+{
+	int ret = -ENOMEM;
+
+	pr_notice("subsystem loading...\n");
+	mutex_lock(&smbdirect_globals.mutex);
+
+	smbdirect_globals.workqueues.accept = alloc_workqueue("smbdirect-accept",
+							      WQ_SYSFS |
+							      WQ_PERCPU |
+							      WQ_POWER_EFFICIENT,
+							      0);
+	if (smbdirect_globals.workqueues.accept == NULL)
+		goto alloc_accept_wq_failed;
+
+	smbdirect_globals.workqueues.connect = alloc_workqueue("smbdirect-connect",
+							       WQ_SYSFS |
+							       WQ_PERCPU |
+							       WQ_POWER_EFFICIENT,
+							       0);
+	if (smbdirect_globals.workqueues.connect == NULL)
+		goto alloc_connect_wq_failed;
+
+	smbdirect_globals.workqueues.idle = alloc_workqueue("smbdirect-idle",
+							    WQ_SYSFS |
+							    WQ_PERCPU |
+							    WQ_POWER_EFFICIENT,
+							    0);
+	if (smbdirect_globals.workqueues.idle == NULL)
+		goto alloc_idle_wq_failed;
+
+	smbdirect_globals.workqueues.refill = alloc_workqueue("smbdirect-refill",
+							      WQ_HIGHPRI |
+							      WQ_SYSFS |
+							      WQ_PERCPU |
+							      WQ_POWER_EFFICIENT,
+							      0);
+	if (smbdirect_globals.workqueues.refill == NULL)
+		goto alloc_refill_wq_failed;
+
+	smbdirect_globals.workqueues.immediate = alloc_workqueue("smbdirect-immediate",
+								 WQ_HIGHPRI |
+								 WQ_SYSFS |
+								 WQ_PERCPU |
+								 WQ_POWER_EFFICIENT,
+								 0);
+	if (smbdirect_globals.workqueues.immediate == NULL)
+		goto alloc_immediate_wq_failed;
+
+	smbdirect_globals.workqueues.cleanup = alloc_workqueue("smbdirect-cleanup",
+							       WQ_MEM_RECLAIM |
+							       WQ_HIGHPRI |
+							       WQ_SYSFS |
+							       WQ_PERCPU |
+							       WQ_POWER_EFFICIENT,
+							       0);
+	if (smbdirect_globals.workqueues.cleanup == NULL)
+		goto alloc_cleanup_wq_failed;
+
+	ret = smbdirect_devices_init();
+	if (ret)
+		goto devices_init_failed;
+
+	mutex_unlock(&smbdirect_globals.mutex);
+	pr_notice("subsystem loaded\n");
+	return 0;
+
+devices_init_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.cleanup);
+alloc_cleanup_wq_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.immediate);
+alloc_immediate_wq_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.refill);
+alloc_refill_wq_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.idle);
+alloc_idle_wq_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.connect);
+alloc_connect_wq_failed:
+	destroy_workqueue(smbdirect_globals.workqueues.accept);
+alloc_accept_wq_failed:
+	mutex_unlock(&smbdirect_globals.mutex);
+	pr_crit("failed to loaded: %d (%1pe)\n",
+		ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+	return ret;
+}
+
+static __exit void smbdirect_module_exit(void)
+{
+	pr_notice("subsystem unloading...\n");
+	mutex_lock(&smbdirect_globals.mutex);
+
+	smbdirect_devices_exit();
+
+	destroy_workqueue(smbdirect_globals.workqueues.accept);
+	destroy_workqueue(smbdirect_globals.workqueues.connect);
+	destroy_workqueue(smbdirect_globals.workqueues.idle);
+	destroy_workqueue(smbdirect_globals.workqueues.refill);
+	destroy_workqueue(smbdirect_globals.workqueues.immediate);
+	destroy_workqueue(smbdirect_globals.workqueues.cleanup);
+
+	mutex_unlock(&smbdirect_globals.mutex);
+	pr_notice("subsystem unloaded\n");
+}
+
+module_init(smbdirect_module_init);
+module_exit(smbdirect_module_exit);
+
+MODULE_DESCRIPTION("smbdirect subsystem");
+MODULE_LICENSE("GPL");
diff --git a/fs/smb/common/smbdirect/smbdirect_mr.c b/fs/smb/common/smbdirect/smbdirect_mr.c
new file mode 100644
index 0000000000000..4873a2c1066fc
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_mr.c
@@ -0,0 +1,485 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+
+/*
+ * Allocate MRs used for RDMA read/write
+ * The number of MRs will not exceed hardware capability in responder_resources
+ * All MRs are kept in mr_list. The MR can be recovered after it's used
+ * Recovery is done in smbd_mr_recovery_work. The content of list entry changes
+ * as MRs are used and recovered for I/O, but the list links will not change
+ */
+int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_mr_io *mr;
+	int ret;
+	u32 i;
+
+	if (sp->responder_resources == 0) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"responder_resources negotiated as 0\n");
+		return -EINVAL;
+	}
+
+	/* Allocate more MRs (2x) than hardware responder_resources */
+	for (i = 0; i < sp->responder_resources * 2; i++) {
+		mr = kzalloc_obj(*mr);
+		if (!mr) {
+			ret = -ENOMEM;
+			goto kzalloc_mr_failed;
+		}
+
+		kref_init(&mr->kref);
+		mutex_init(&mr->mutex);
+
+		mr->mr = ib_alloc_mr(sc->ib.pd,
+				     sc->mr_io.type,
+				     sp->max_frmr_depth);
+		if (IS_ERR(mr->mr)) {
+			ret = PTR_ERR(mr->mr);
+			smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+				"ib_alloc_mr failed ret=%d (%1pe) type=0x%x max_frmr_depth=%u\n",
+				ret, SMBDIRECT_DEBUG_ERR_PTR(ret),
+				sc->mr_io.type, sp->max_frmr_depth);
+			goto ib_alloc_mr_failed;
+		}
+		mr->sgt.sgl = kzalloc_objs(struct scatterlist, sp->max_frmr_depth);
+		if (!mr->sgt.sgl) {
+			ret = -ENOMEM;
+			smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+				"failed to allocate sgl, max_frmr_depth=%u\n",
+				sp->max_frmr_depth);
+			goto kcalloc_sgl_failed;
+		}
+		mr->state = SMBDIRECT_MR_READY;
+		mr->socket = sc;
+
+		list_add_tail(&mr->list, &sc->mr_io.all.list);
+		atomic_inc(&sc->mr_io.ready.count);
+	}
+
+	return 0;
+
+kcalloc_sgl_failed:
+	ib_dereg_mr(mr->mr);
+ib_alloc_mr_failed:
+	mutex_destroy(&mr->mutex);
+	kfree(mr);
+kzalloc_mr_failed:
+	smbdirect_connection_destroy_mr_list(sc);
+	return ret;
+}
+
+static void smbdirect_mr_io_disable_locked(struct smbdirect_mr_io *mr)
+{
+	struct smbdirect_socket *sc = mr->socket;
+
+	lockdep_assert_held(&mr->mutex);
+
+	if (mr->state == SMBDIRECT_MR_DISABLED)
+		return;
+
+	if (mr->mr)
+		ib_dereg_mr(mr->mr);
+	if (mr->sgt.nents)
+		ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
+	kfree(mr->sgt.sgl);
+
+	mr->mr = NULL;
+	mr->sgt.sgl = NULL;
+	mr->sgt.nents = 0;
+
+	mr->state = SMBDIRECT_MR_DISABLED;
+}
+
+static void smbdirect_mr_io_free_locked(struct kref *kref)
+{
+	struct smbdirect_mr_io *mr =
+		container_of(kref, struct smbdirect_mr_io, kref);
+
+	lockdep_assert_held(&mr->mutex);
+
+	/*
+	 * smbdirect_mr_io_disable_locked() should already be called!
+	 */
+	if (WARN_ON_ONCE(mr->state != SMBDIRECT_MR_DISABLED))
+		smbdirect_mr_io_disable_locked(mr);
+
+	mutex_unlock(&mr->mutex);
+	mutex_destroy(&mr->mutex);
+	kfree(mr);
+}
+
+void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc)
+{
+	struct smbdirect_mr_io *mr, *tmp;
+	LIST_HEAD(all_list);
+	unsigned long flags;
+
+	spin_lock_irqsave(&sc->mr_io.all.lock, flags);
+	list_splice_tail_init(&sc->mr_io.all.list, &all_list);
+	spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
+
+	list_for_each_entry_safe(mr, tmp, &all_list, list) {
+		mutex_lock(&mr->mutex);
+
+		smbdirect_mr_io_disable_locked(mr);
+		list_del(&mr->list);
+		mr->socket = NULL;
+
+		/*
+		 * No kref_put_mutex() as it's already locked.
+		 *
+		 * If smbdirect_mr_io_free_locked() is called
+		 * and the mutex is unlocked and mr is gone,
+		 * in that case kref_put() returned 1.
+		 *
+		 * If kref_put() returned 0 we know that
+		 * smbdirect_mr_io_free_locked() didn't
+		 * run. Not by us nor by anyone else, as we
+		 * still hold the mutex, so we need to unlock.
+		 *
+		 * If the mr is still registered it will
+		 * be dangling (detached from the connection
+		 * waiting for smbd_deregister_mr() to be
+		 * called in order to free the memory.
+		 */
+		if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked))
+			mutex_unlock(&mr->mutex);
+	}
+}
+
+/*
+ * Get a MR from mr_list. This function waits until there is at least one MR
+ * available in the list. There may be several CPUs issuing I/O trying to get MR
+ * at the same time, mr_list_lock is used to protect this situation.
+ */
+static struct smbdirect_mr_io *
+smbdirect_connection_get_mr_io(struct smbdirect_socket *sc)
+{
+	struct smbdirect_mr_io *mr;
+	unsigned long flags;
+	int ret;
+
+again:
+	ret = wait_event_interruptible(sc->mr_io.ready.wait_queue,
+				       atomic_read(&sc->mr_io.ready.count) ||
+				       sc->status != SMBDIRECT_SOCKET_CONNECTED);
+	if (ret) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"wait_event_interruptible ret=%d (%1pe)\n",
+			ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return NULL;
+	}
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"sc->status=%s sc->first_error=%1pe\n",
+			smbdirect_socket_status_string(sc->status),
+			SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+		return NULL;
+	}
+
+	spin_lock_irqsave(&sc->mr_io.all.lock, flags);
+	list_for_each_entry(mr, &sc->mr_io.all.list, list) {
+		if (mr->state == SMBDIRECT_MR_READY) {
+			mr->state = SMBDIRECT_MR_REGISTERED;
+			kref_get(&mr->kref);
+			spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
+			atomic_dec(&sc->mr_io.ready.count);
+			atomic_inc(&sc->mr_io.used.count);
+			return mr;
+		}
+	}
+
+	spin_unlock_irqrestore(&sc->mr_io.all.lock, flags);
+	/*
+	 * It is possible that we could fail to get MR because other processes may
+	 * try to acquire a MR at the same time. If this is the case, retry it.
+	 */
+	goto again;
+}
+
+static void smbdirect_connection_mr_io_register_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_mr_io *mr =
+		container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe);
+	struct smbdirect_socket *sc = mr->socket;
+
+	if (wc->status != IB_WC_SUCCESS) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"wc->status=%s opcode=%d\n",
+			ib_wc_status_msg(wc->status), wc->opcode);
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+	}
+}
+
+static void smbdirect_connection_mr_io_local_inv_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_mr_io *mr =
+		container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe);
+	struct smbdirect_socket *sc = mr->socket;
+
+	mr->state = SMBDIRECT_MR_INVALIDATED;
+	if (wc->status != IB_WC_SUCCESS) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"invalidate failed status=%s\n",
+			ib_wc_status_msg(wc->status));
+		smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED);
+	}
+	complete(&mr->invalidate_done);
+}
+
+/*
+ * Transcribe the pages from an iterator into an MR scatterlist.
+ */
+static int smbdirect_iter_to_sgt(struct iov_iter *iter,
+				 struct sg_table *sgt,
+				 unsigned int max_sg)
+{
+	int ret;
+
+	memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist));
+
+	ret = extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0);
+	WARN_ON(ret < 0);
+	if (sgt->nents > 0)
+		sg_mark_end(&sgt->sgl[sgt->nents - 1]);
+
+	return ret;
+}
+
+/*
+ * Register memory for RDMA read/write
+ * iter: the buffer to register memory with
+ * writing: true if this is a RDMA write (SMB read), false for RDMA read
+ * need_invalidate: true if this MR needs to be locally invalidated after I/O
+ * return value: the MR registered, NULL if failed.
+ */
+struct smbdirect_mr_io *
+smbdirect_connection_register_mr_io(struct smbdirect_socket *sc,
+				    struct iov_iter *iter,
+				    bool writing,
+				    bool need_invalidate)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_mr_io *mr;
+	int ret, num_pages;
+	struct ib_reg_wr *reg_wr;
+
+	num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1);
+	if (num_pages > sp->max_frmr_depth) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"num_pages=%d max_frmr_depth=%d\n",
+			num_pages, sp->max_frmr_depth);
+		WARN_ON_ONCE(1);
+		return NULL;
+	}
+
+	mr = smbdirect_connection_get_mr_io(sc);
+	if (!mr) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"smbdirect_connection_get_mr_io returning NULL\n");
+		return NULL;
+	}
+
+	mutex_lock(&mr->mutex);
+
+	mr->dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+	mr->need_invalidate = need_invalidate;
+	mr->sgt.nents = 0;
+	mr->sgt.orig_nents = 0;
+
+	smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO,
+		"num_pages=%u count=%zu depth=%u\n",
+		num_pages, iov_iter_count(iter), sp->max_frmr_depth);
+	smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth);
+
+	ret = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
+	if (!ret) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"ib_dma_map_sg num_pages=%u dir=%x ret=%d (%1pe)\n",
+			num_pages, mr->dir, ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		goto dma_map_error;
+	}
+
+	ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE);
+	if (ret != mr->sgt.nents) {
+		smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+			"ib_map_mr_sg failed ret = %d nents = %u\n",
+			ret, mr->sgt.nents);
+		goto map_mr_error;
+	}
+
+	ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey));
+	reg_wr = &mr->wr;
+	reg_wr->wr.opcode = IB_WR_REG_MR;
+	mr->cqe.done = smbdirect_connection_mr_io_register_done;
+	reg_wr->wr.wr_cqe = &mr->cqe;
+	reg_wr->wr.num_sge = 0;
+	reg_wr->wr.send_flags = IB_SEND_SIGNALED;
+	reg_wr->mr = mr->mr;
+	reg_wr->key = mr->mr->rkey;
+	reg_wr->access = writing ?
+			IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
+			IB_ACCESS_REMOTE_READ;
+
+	/*
+	 * There is no need for waiting for complemtion on ib_post_send
+	 * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution
+	 * on the next ib_post_send when we actually send I/O to remote peer
+	 */
+	ret = ib_post_send(sc->ib.qp, &reg_wr->wr, NULL);
+	if (!ret) {
+		/*
+		 * smbdirect_connection_get_mr_io() gave us a reference
+		 * via kref_get(&mr->kref), we keep that and let
+		 * the caller use smbdirect_connection_deregister_mr_io()
+		 * to remove it again.
+		 */
+		mutex_unlock(&mr->mutex);
+		return mr;
+	}
+
+	smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+		"ib_post_send failed ret=%d (%1pe) reg_wr->key=0x%x\n",
+		ret, SMBDIRECT_DEBUG_ERR_PTR(ret), reg_wr->key);
+
+map_mr_error:
+	ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
+
+dma_map_error:
+	mr->sgt.nents = 0;
+	mr->state = SMBDIRECT_MR_ERROR;
+	atomic_dec(&sc->mr_io.used.count);
+
+	smbdirect_socket_schedule_cleanup(sc, ret);
+
+	/*
+	 * smbdirect_connection_get_mr_io() gave us a reference
+	 * via kref_get(&mr->kref), we need to remove it again
+	 * on error.
+	 *
+	 * No kref_put_mutex() as it's already locked.
+	 *
+	 * If smbdirect_mr_io_free_locked() is called
+	 * and the mutex is unlocked and mr is gone,
+	 * in that case kref_put() returned 1.
+	 *
+	 * If kref_put() returned 0 we know that
+	 * smbdirect_mr_io_free_locked() didn't
+	 * run. Not by us nor by anyone else, as we
+	 * still hold the mutex, so we need to unlock.
+	 */
+	if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked))
+		mutex_unlock(&mr->mutex);
+	return NULL;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_register_mr_io);
+
+void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
+					    struct smbdirect_buffer_descriptor_v1 *v1)
+{
+	mutex_lock(&mr->mutex);
+	if (mr->state == SMBDIRECT_MR_REGISTERED) {
+		v1->offset = cpu_to_le64(mr->mr->iova);
+		v1->token = cpu_to_le32(mr->mr->rkey);
+		v1->length = cpu_to_le32(mr->mr->length);
+	} else {
+		v1->offset = cpu_to_le64(U64_MAX);
+		v1->token = cpu_to_le32(U32_MAX);
+		v1->length = cpu_to_le32(U32_MAX);
+	}
+	mutex_unlock(&mr->mutex);
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_mr_io_fill_buffer_descriptor);
+
+/*
+ * Deregister a MR after I/O is done
+ * This function may wait if remote invalidation is not used
+ * and we have to locally invalidate the buffer to prevent data is being
+ * modified by remote peer after upper layer consumes it
+ */
+void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr)
+{
+	struct smbdirect_socket *sc = mr->socket;
+	int ret = 0;
+
+	mutex_lock(&mr->mutex);
+	if (mr->state == SMBDIRECT_MR_DISABLED)
+		goto put_kref;
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
+		smbdirect_mr_io_disable_locked(mr);
+		goto put_kref;
+	}
+
+	if (mr->need_invalidate) {
+		struct ib_send_wr *wr = &mr->inv_wr;
+
+		/* Need to finish local invalidation before returning */
+		wr->opcode = IB_WR_LOCAL_INV;
+		mr->cqe.done = smbdirect_connection_mr_io_local_inv_done;
+		wr->wr_cqe = &mr->cqe;
+		wr->num_sge = 0;
+		wr->ex.invalidate_rkey = mr->mr->rkey;
+		wr->send_flags = IB_SEND_SIGNALED;
+
+		init_completion(&mr->invalidate_done);
+		ret = ib_post_send(sc->ib.qp, wr, NULL);
+		if (ret) {
+			smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
+				"ib_post_send failed ret=%d (%1pe)\n",
+				ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+			smbdirect_mr_io_disable_locked(mr);
+			smbdirect_socket_schedule_cleanup(sc, ret);
+			goto done;
+		}
+		wait_for_completion(&mr->invalidate_done);
+		mr->need_invalidate = false;
+	} else
+		/*
+		 * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED
+		 * and defer to mr_recovery_work to recover the MR for next use
+		 */
+		mr->state = SMBDIRECT_MR_INVALIDATED;
+
+	if (mr->sgt.nents) {
+		ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
+		mr->sgt.nents = 0;
+	}
+
+	WARN_ONCE(mr->state != SMBDIRECT_MR_INVALIDATED,
+		  "mr->state[%u] != SMBDIRECT_MR_INVALIDATED[%u]\n",
+		  mr->state, SMBDIRECT_MR_INVALIDATED);
+	mr->state = SMBDIRECT_MR_READY;
+	if (atomic_inc_return(&sc->mr_io.ready.count) == 1)
+		wake_up(&sc->mr_io.ready.wait_queue);
+
+done:
+	atomic_dec(&sc->mr_io.used.count);
+
+put_kref:
+	/*
+	 * No kref_put_mutex() as it's already locked.
+	 *
+	 * If smbdirect_mr_io_free_locked() is called
+	 * and the mutex is unlocked and mr is gone,
+	 * in that case kref_put() returned 1.
+	 *
+	 * If kref_put() returned 0 we know that
+	 * smbdirect_mr_io_free_locked() didn't
+	 * run. Not by us nor by anyone else, as we
+	 * still hold the mutex, so we need to unlock
+	 * and keep the mr in SMBDIRECT_MR_READY or
+	 * SMBDIRECT_MR_ERROR state.
+	 */
+	if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked))
+		mutex_unlock(&mr->mutex);
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_deregister_mr_io);
diff --git a/fs/smb/common/smbdirect/smbdirect_pdu.h b/fs/smb/common/smbdirect/smbdirect_pdu.h
index ae9fdb05ce231..7693ba3378739 100644
--- a/fs/smb/common/smbdirect/smbdirect_pdu.h
+++ b/fs/smb/common/smbdirect/smbdirect_pdu.h
@@ -8,6 +8,10 @@
 
 #define SMBDIRECT_V1 0x0100
 
+/* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */
+#define SMBDIRECT_MIN_RECEIVE_SIZE		128
+#define SMBDIRECT_MIN_FRAGMENTED_SIZE		131072
+
 /* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
 struct smbdirect_negotiate_req {
 	__le16 min_version;
diff --git a/fs/smb/common/smbdirect/smbdirect_public.h b/fs/smb/common/smbdirect/smbdirect_public.h
new file mode 100644
index 0000000000000..50088155e7c37
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_public.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *   Copyright (C) 2025, Stefan Metzmacher
+ */
+
+#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__
+#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__
+
+struct smbdirect_buffer_descriptor_v1;
+struct smbdirect_socket_parameters;
+
+struct smbdirect_socket;
+struct smbdirect_send_batch;
+struct smbdirect_mr_io;
+
+#define __SMBDIRECT_EXPORT_SYMBOL__(__sym) EXPORT_SYMBOL_FOR_MODULES(__sym, "cifs,ksmbd")
+
+#include <rdma/rw.h>
+
+u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev);
+
+bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs);
+
+int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc);
+
+int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc);
+
+int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc,
+					    const struct smbdirect_socket_parameters *sp);
+
+const struct smbdirect_socket_parameters *
+smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc);
+
+int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc,
+					 enum ib_poll_context poll_ctx,
+					 gfp_t gfp_mask);
+
+#define SMBDIRECT_LOG_ERR		0x0
+#define SMBDIRECT_LOG_INFO		0x1
+
+#define SMBDIRECT_LOG_OUTGOING			0x1
+#define SMBDIRECT_LOG_INCOMING			0x2
+#define SMBDIRECT_LOG_READ			0x4
+#define SMBDIRECT_LOG_WRITE			0x8
+#define SMBDIRECT_LOG_RDMA_SEND			0x10
+#define SMBDIRECT_LOG_RDMA_RECV			0x20
+#define SMBDIRECT_LOG_KEEP_ALIVE		0x40
+#define SMBDIRECT_LOG_RDMA_EVENT		0x80
+#define SMBDIRECT_LOG_RDMA_MR			0x100
+#define SMBDIRECT_LOG_RDMA_RW			0x200
+#define SMBDIRECT_LOG_NEGOTIATE			0x400
+void smbdirect_socket_set_logging(struct smbdirect_socket *sc,
+				  void *private_ptr,
+				  bool (*needed)(struct smbdirect_socket *sc,
+						 void *private_ptr,
+						 unsigned int lvl,
+						 unsigned int cls),
+				  void (*vaprintf)(struct smbdirect_socket *sc,
+						   const char *func,
+						   unsigned int line,
+						   void *private_ptr,
+						   unsigned int lvl,
+						   unsigned int cls,
+						   struct va_format *vaf));
+
+bool smbdirect_connection_is_connected(struct smbdirect_socket *sc);
+
+int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc);
+
+int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr);
+
+void smbdirect_socket_shutdown(struct smbdirect_socket *sc);
+
+void smbdirect_socket_release(struct smbdirect_socket *sc);
+
+int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc,
+					  struct smbdirect_send_batch *batch,
+					  bool is_last);
+
+/*
+ * This is only temporary and only needed
+ * as long as the client still requires
+ * to use smbdirect_connection_send_single_iter()
+ */
+struct smbdirect_send_batch_storage {
+	union {
+		struct list_head __msg_list;
+		__aligned_u64 __space[5];
+	};
+};
+
+struct smbdirect_send_batch *
+smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage,
+				  bool need_invalidate_rkey,
+				  unsigned int remote_key);
+
+int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc,
+					  struct smbdirect_send_batch *batch,
+					  struct iov_iter *iter,
+					  unsigned int flags,
+					  u32 remaining_data_length);
+
+int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc);
+
+int smbdirect_connection_send_iter(struct smbdirect_socket *sc,
+				   struct iov_iter *iter,
+				   unsigned int flags,
+				   bool need_invalidate,
+				   unsigned int remote_key);
+
+int smbdirect_connection_recvmsg(struct smbdirect_socket *sc,
+				 struct msghdr *msg,
+				 unsigned int flags);
+
+int smbdirect_connect(struct smbdirect_socket *sc,
+		      const struct sockaddr *dst);
+
+int smbdirect_connect_sync(struct smbdirect_socket *sc,
+			   const struct sockaddr *dst);
+
+int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog);
+
+struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc,
+						 long timeo,
+						 struct proto_accept_arg *arg);
+
+int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc,
+				   void *buf, size_t buf_len,
+				   struct smbdirect_buffer_descriptor_v1 *desc,
+				   size_t desc_len,
+				   bool is_read);
+
+struct smbdirect_mr_io *
+smbdirect_connection_register_mr_io(struct smbdirect_socket *sc,
+				    struct iov_iter *iter,
+				    bool writing,
+				    bool need_invalidate);
+
+void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
+					    struct smbdirect_buffer_descriptor_v1 *v1);
+
+void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr);
+
+void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
+						 unsigned int rdma_readwrite_threshold,
+						 struct seq_file *m);
+
+#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ */
diff --git a/fs/smb/common/smbdirect/smbdirect_rw.c b/fs/smb/common/smbdirect/smbdirect_rw.c
new file mode 100644
index 0000000000000..3b2eb8c48efcd
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_rw.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (C) 2018, LG Electronics.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+
+static int smbdirect_connection_wait_for_rw_credits(struct smbdirect_socket *sc,
+						    int credits)
+{
+	return smbdirect_socket_wait_for_credits(sc,
+						 SMBDIRECT_SOCKET_CONNECTED,
+						 -ENOTCONN,
+						 &sc->rw_io.credits.wait_queue,
+						 &sc->rw_io.credits.count,
+						 credits);
+}
+
+static int smbdirect_connection_calc_rw_credits(struct smbdirect_socket *sc,
+						const void *buf,
+						size_t len)
+{
+	return DIV_ROUND_UP(smbdirect_get_buf_page_count(buf, len),
+			    sc->rw_io.credits.num_pages);
+}
+
+static int smbdirect_connection_rdma_get_sg_list(void *buf,
+						 size_t size,
+						 struct scatterlist *sg_list,
+						 size_t nentries)
+{
+	bool high = is_vmalloc_addr(buf);
+	struct page *page;
+	size_t offset, len;
+	int i = 0;
+
+	if (size == 0 || nentries < smbdirect_get_buf_page_count(buf, size))
+		return -EINVAL;
+
+	offset = offset_in_page(buf);
+	buf -= offset;
+	while (size > 0) {
+		len = min_t(size_t, PAGE_SIZE - offset, size);
+		if (high)
+			page = vmalloc_to_page(buf);
+		else
+			page = kmap_to_page(buf);
+
+		if (!sg_list)
+			return -EINVAL;
+		sg_set_page(sg_list, page, len, offset);
+		sg_list = sg_next(sg_list);
+
+		buf += PAGE_SIZE;
+		size -= len;
+		offset = 0;
+		i++;
+	}
+
+	return i;
+}
+
+static void smbdirect_connection_rw_io_free(struct smbdirect_rw_io *msg,
+					    enum dma_data_direction dir)
+{
+	struct smbdirect_socket *sc = msg->socket;
+
+	rdma_rw_ctx_destroy(&msg->rdma_ctx,
+			    sc->ib.qp,
+			    sc->ib.qp->port,
+			    msg->sgt.sgl,
+			    msg->sgt.nents,
+			    dir);
+	sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
+	kfree(msg);
+}
+
+static void smbdirect_connection_rdma_rw_done(struct ib_cq *cq, struct ib_wc *wc,
+					      enum dma_data_direction dir)
+{
+	struct smbdirect_rw_io *msg =
+		container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe);
+	struct smbdirect_socket *sc = msg->socket;
+
+	if (wc->status != IB_WC_SUCCESS) {
+		msg->error = -EIO;
+		pr_err("read/write error. opcode = %d, status = %s(%d)\n",
+		       wc->opcode, ib_wc_status_msg(wc->status), wc->status);
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			smbdirect_socket_schedule_cleanup(sc, msg->error);
+	}
+
+	complete(msg->completion);
+}
+
+static void smbdirect_connection_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	smbdirect_connection_rdma_rw_done(cq, wc, DMA_FROM_DEVICE);
+}
+
+static void smbdirect_connection_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	smbdirect_connection_rdma_rw_done(cq, wc, DMA_TO_DEVICE);
+}
+
+int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc,
+				   void *buf, size_t buf_len,
+				   struct smbdirect_buffer_descriptor_v1 *desc,
+				   size_t desc_len,
+				   bool is_read)
+{
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	enum dma_data_direction direction = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+	struct smbdirect_rw_io *msg, *next_msg;
+	size_t i;
+	int ret;
+	DECLARE_COMPLETION_ONSTACK(completion);
+	struct ib_send_wr *first_wr;
+	LIST_HEAD(msg_list);
+	u8 *desc_buf;
+	int credits_needed;
+	size_t desc_buf_len, desc_num = 0;
+
+	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
+		return -ENOTCONN;
+
+	if (buf_len > sp->max_read_write_size)
+		return -EINVAL;
+
+	/* calculate needed credits */
+	credits_needed = 0;
+	desc_buf = buf;
+	for (i = 0; i < desc_len / sizeof(*desc); i++) {
+		if (!buf_len)
+			break;
+
+		desc_buf_len = le32_to_cpu(desc[i].length);
+		if (!desc_buf_len)
+			return -EINVAL;
+
+		if (desc_buf_len > buf_len) {
+			desc_buf_len = buf_len;
+			desc[i].length = cpu_to_le32(desc_buf_len);
+			buf_len = 0;
+		}
+
+		credits_needed += smbdirect_connection_calc_rw_credits(sc,
+								       desc_buf,
+								       desc_buf_len);
+		desc_buf += desc_buf_len;
+		buf_len -= desc_buf_len;
+		desc_num++;
+	}
+
+	smbdirect_log_rdma_rw(sc, SMBDIRECT_LOG_INFO,
+		"RDMA %s, len %zu, needed credits %d\n",
+		str_read_write(is_read), buf_len, credits_needed);
+
+	ret = smbdirect_connection_wait_for_rw_credits(sc, credits_needed);
+	if (ret < 0)
+		return ret;
+
+	/* build rdma_rw_ctx for each descriptor */
+	desc_buf = buf;
+	for (i = 0; i < desc_num; i++) {
+		size_t page_count;
+
+		msg = kzalloc_flex(*msg, sg_list, SG_CHUNK_SIZE,
+				   sc->rw_io.mem.gfp_mask);
+		if (!msg) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		desc_buf_len = le32_to_cpu(desc[i].length);
+		page_count = smbdirect_get_buf_page_count(desc_buf, desc_buf_len);
+
+		msg->socket = sc;
+		msg->cqe.done = is_read ?
+			smbdirect_connection_rdma_read_done :
+			smbdirect_connection_rdma_write_done;
+		msg->completion = &completion;
+
+		msg->sgt.sgl = &msg->sg_list[0];
+		ret = sg_alloc_table_chained(&msg->sgt,
+					     page_count,
+					     msg->sg_list,
+					     SG_CHUNK_SIZE);
+		if (ret) {
+			ret = -ENOMEM;
+			goto free_msg;
+		}
+
+		ret = smbdirect_connection_rdma_get_sg_list(desc_buf,
+							    desc_buf_len,
+							    msg->sgt.sgl,
+							    msg->sgt.orig_nents);
+		if (ret < 0)
+			goto free_table;
+
+		ret = rdma_rw_ctx_init(&msg->rdma_ctx,
+				       sc->ib.qp,
+				       sc->ib.qp->port,
+				       msg->sgt.sgl,
+				       page_count,
+				       0,
+				       le64_to_cpu(desc[i].offset),
+				       le32_to_cpu(desc[i].token),
+				       direction);
+		if (ret < 0) {
+			pr_err("failed to init rdma_rw_ctx: %d\n", ret);
+			goto free_table;
+		}
+
+		list_add_tail(&msg->list, &msg_list);
+		desc_buf += desc_buf_len;
+	}
+
+	/* concatenate work requests of rdma_rw_ctxs */
+	first_wr = NULL;
+	list_for_each_entry_reverse(msg, &msg_list, list) {
+		first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx,
+					   sc->ib.qp,
+					   sc->ib.qp->port,
+					   &msg->cqe,
+					   first_wr);
+	}
+
+	ret = ib_post_send(sc->ib.qp, first_wr, NULL);
+	if (ret) {
+		pr_err("failed to post send wr for RDMA R/W: %d\n", ret);
+		goto out;
+	}
+
+	msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list);
+	wait_for_completion(&completion);
+	ret = msg->error;
+out:
+	list_for_each_entry_safe(msg, next_msg, &msg_list, list) {
+		list_del(&msg->list);
+		smbdirect_connection_rw_io_free(msg, direction);
+	}
+	atomic_add(credits_needed, &sc->rw_io.credits.count);
+	wake_up(&sc->rw_io.credits.wait_queue);
+	return ret;
+
+free_table:
+	sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
+free_msg:
+	kfree(msg);
+	goto out;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_rdma_xmit);
diff --git a/fs/smb/common/smbdirect/smbdirect_socket.c b/fs/smb/common/smbdirect/smbdirect_socket.c
new file mode 100644
index 0000000000000..1e3b361728cc9
--- /dev/null
+++ b/fs/smb/common/smbdirect/smbdirect_socket.c
@@ -0,0 +1,724 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *   Copyright (C) 2017, Microsoft Corporation.
+ *   Copyright (c) 2025, Stefan Metzmacher
+ */
+
+#include "smbdirect_internal.h"
+
+bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs)
+{
+	/*
+	 * Test if FRWR (Fast Registration Work Requests) is supported on the
+	 * device This implementation requires FRWR on RDMA read/write return
+	 * value: true if it is supported
+	 */
+
+	if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
+		return false;
+	if (attrs->max_fast_reg_page_list_len == 0)
+		return false;
+	return true;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_frwr_is_supported);
+
+static void smbdirect_socket_cleanup_work(struct work_struct *work);
+
+static int smbdirect_socket_rdma_event_handler(struct rdma_cm_id *id,
+					       struct rdma_cm_event *event)
+{
+	struct smbdirect_socket *sc = id->context;
+	int ret = -ESTALE;
+
+	/*
+	 * This should be replaced before any real work
+	 * starts! So it should never be called!
+	 */
+
+	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+		ret = -ENETDOWN;
+	if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status)))
+		ret = event->status;
+	pr_err("%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),
+		rdma_event_msg(sc->rdma.expected_event),
+		rdma_event_msg(event->event),
+		event->status,
+		SMBDIRECT_DEBUG_ERR_PTR(ret));
+	WARN_ONCE(1, "%s should not be called!\n", __func__);
+	sc->rdma.cm_id = NULL;
+	return -ESTALE;
+}
+
+int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc)
+{
+	struct rdma_cm_id *id;
+	int ret;
+
+	smbdirect_socket_init(sc);
+
+	id = rdma_create_id(net,
+			    smbdirect_socket_rdma_event_handler,
+			    sc,
+			    RDMA_PS_TCP,
+			    IB_QPT_RC);
+	if (IS_ERR(id)) {
+		pr_err("%s: rdma_create_id() failed %1pe\n", __func__, id);
+		return PTR_ERR(id);
+	}
+
+	ret = rdma_set_afonly(id, 1);
+	if (ret) {
+		rdma_destroy_id(id);
+		pr_err("%s: rdma_set_afonly() failed %1pe\n",
+		       __func__, SMBDIRECT_DEBUG_ERR_PTR(ret));
+		return ret;
+	}
+
+	sc->rdma.cm_id = id;
+
+	INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);
+
+	return 0;
+}
+
+int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc)
+{
+	struct smbdirect_socket *sc;
+	int ret;
+
+	ret = -ENOMEM;
+	sc = kzalloc_obj(*sc);
+	if (!sc)
+		goto alloc_failed;
+
+	ret = smbdirect_socket_init_new(net, sc);
+	if (ret)
+		goto init_failed;
+
+	kref_init(&sc->refs.destroy);
+
+	*_sc = sc;
+	return 0;
+
+init_failed:
+	kfree(sc);
+alloc_failed:
+	return ret;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_kern);
+
+int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc)
+{
+	smbdirect_socket_init(sc);
+
+	sc->rdma.cm_id = id;
+	sc->rdma.cm_id->context = sc;
+	sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler;
+
+	sc->ib.dev = sc->rdma.cm_id->device;
+
+	INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);
+
+	return 0;
+}
+
+int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc)
+{
+	struct smbdirect_socket *sc;
+	int ret;
+
+	ret = -ENOMEM;
+	sc = kzalloc_obj(*sc);
+	if (!sc)
+		goto alloc_failed;
+
+	ret = smbdirect_socket_init_accepting(id, sc);
+	if (ret)
+		goto init_failed;
+
+	kref_init(&sc->refs.destroy);
+
+	*_sc = sc;
+	return 0;
+
+init_failed:
+	kfree(sc);
+alloc_failed:
+	return ret;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_accepting);
+
+int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc,
+					    const struct smbdirect_socket_parameters *sp)
+{
+	/*
+	 * This is only allowed before connect or accept
+	 */
+	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED,
+		  "status=%s first_error=%1pe",
+		  smbdirect_socket_status_string(sc->status),
+		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+	if (sc->status != SMBDIRECT_SOCKET_CREATED)
+		return -EINVAL;
+
+	if (sp->flags & ~SMBDIRECT_FLAG_PORT_RANGE_MASK)
+		return -EINVAL;
+
+	if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB &&
+	    sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW)
+		return -EINVAL;
+	else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB)
+		rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA);
+	else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW)
+		rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC);
+
+	/*
+	 * Make a copy of the callers parameters
+	 * from here we only work on the copy
+	 *
+	 * TODO: do we want consistency checking?
+	 */
+	sc->parameters = *sp;
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_initial_parameters);
+
+const struct smbdirect_socket_parameters *
+smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc)
+{
+	return &sc->parameters;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_get_current_parameters);
+
+int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc,
+					 enum ib_poll_context poll_ctx,
+					 gfp_t gfp_mask)
+{
+	/*
+	 * This is only allowed before connect or accept
+	 */
+	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED,
+		  "status=%s first_error=%1pe",
+		  smbdirect_socket_status_string(sc->status),
+		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+	if (sc->status != SMBDIRECT_SOCKET_CREATED)
+		return -EINVAL;
+
+	sc->ib.poll_ctx = poll_ctx;
+
+	sc->send_io.mem.gfp_mask = gfp_mask;
+	sc->recv_io.mem.gfp_mask = gfp_mask;
+	sc->rw_io.mem.gfp_mask = gfp_mask;
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_kernel_settings);
+
+void smbdirect_socket_set_logging(struct smbdirect_socket *sc,
+				  void *private_ptr,
+				  bool (*needed)(struct smbdirect_socket *sc,
+						 void *private_ptr,
+						 unsigned int lvl,
+						 unsigned int cls),
+				  void (*vaprintf)(struct smbdirect_socket *sc,
+						   const char *func,
+						   unsigned int line,
+						   void *private_ptr,
+						   unsigned int lvl,
+						   unsigned int cls,
+						   struct va_format *vaf))
+{
+	sc->logging.private_ptr = private_ptr;
+	sc->logging.needed = needed;
+	sc->logging.vaprintf = vaprintf;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_logging);
+
+static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc)
+{
+	/*
+	 * Wake up all waiters in all wait queues
+	 * in order to notice the broken connection.
+	 */
+	wake_up_all(&sc->status_wait);
+	wake_up_all(&sc->listen.wait_queue);
+	wake_up_all(&sc->send_io.bcredits.wait_queue);
+	wake_up_all(&sc->send_io.lcredits.wait_queue);
+	wake_up_all(&sc->send_io.credits.wait_queue);
+	wake_up_all(&sc->send_io.pending.zero_wait_queue);
+	wake_up_all(&sc->recv_io.reassembly.wait_queue);
+	wake_up_all(&sc->rw_io.credits.wait_queue);
+	wake_up_all(&sc->mr_io.ready.wait_queue);
+}
+
+void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc,
+					 const char *macro_name,
+					 unsigned int lvl,
+					 const char *func,
+					 unsigned int line,
+					 int error,
+					 enum smbdirect_socket_status *force_status)
+{
+	struct smbdirect_socket *psc, *tsc;
+	unsigned long flags;
+	bool was_first = false;
+
+	if (!sc->first_error) {
+		___smbdirect_log_generic(sc, func, line,
+			lvl,
+			SMBDIRECT_LOG_RDMA_EVENT,
+			"%s(%1pe%s%s) called from %s in line=%u status=%s\n",
+			macro_name,
+			SMBDIRECT_DEBUG_ERR_PTR(error),
+			force_status ? ", " : "",
+			force_status ? smbdirect_socket_status_string(*force_status) : "",
+			func, line,
+			smbdirect_socket_status_string(sc->status));
+		if (error)
+			sc->first_error = error;
+		else
+			sc->first_error = -ECONNABORTED;
+		was_first = true;
+	}
+
+	/*
+	 * make sure other work (than disconnect_work)
+	 * is not queued again but here we don't block and avoid
+	 * disable[_delayed]_work_sync()
+	 */
+	disable_work(&sc->connect.work);
+	disable_work(&sc->recv_io.posted.refill_work);
+	disable_work(&sc->idle.immediate_work);
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	disable_delayed_work(&sc->idle.timer_work);
+
+	/*
+	 * In case we were a listener we need to
+	 * disconnect all pending and ready sockets
+	 *
+	 * First we move ready sockets to pending again.
+	 */
+	spin_lock_irqsave(&sc->listen.lock, flags);
+	list_splice_init(&sc->listen.ready, &sc->listen.pending);
+	list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list)
+		smbdirect_socket_schedule_cleanup(psc, sc->first_error);
+	spin_unlock_irqrestore(&sc->listen.lock, flags);
+
+	switch (sc->status) {
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
+	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
+	case SMBDIRECT_SOCKET_ERROR:
+	case SMBDIRECT_SOCKET_DISCONNECTING:
+	case SMBDIRECT_SOCKET_DISCONNECTED:
+	case SMBDIRECT_SOCKET_DESTROYED:
+		/*
+		 * Keep the current error status
+		 */
+		break;
+
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
+		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;
+		break;
+
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
+		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;
+		break;
+
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
+		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;
+		break;
+
+	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
+	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
+		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
+		break;
+
+	case SMBDIRECT_SOCKET_CREATED:
+	case SMBDIRECT_SOCKET_LISTENING:
+		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
+		break;
+
+	case SMBDIRECT_SOCKET_CONNECTED:
+		sc->status = SMBDIRECT_SOCKET_ERROR;
+		break;
+	}
+
+	if (force_status && (was_first || *force_status > sc->status))
+		sc->status = *force_status;
+
+	/*
+	 * Wake up all waiters in all wait queues
+	 * in order to notice the broken connection.
+	 */
+	smbdirect_socket_wake_up_all(sc);
+
+	queue_work(sc->workqueues.cleanup, &sc->disconnect_work);
+}
+
+static void smbdirect_socket_cleanup_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, disconnect_work);
+	struct smbdirect_socket *psc, *tsc;
+	unsigned long flags;
+
+	/*
+	 * This should not never be called in an interrupt!
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	if (!sc->first_error) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,
+			"%s called with first_error==0\n",
+			smbdirect_socket_status_string(sc->status));
+
+		sc->first_error = -ECONNABORTED;
+	}
+
+	/*
+	 * make sure this and other work is not queued again
+	 * but here we don't block and avoid
+	 * disable[_delayed]_work_sync()
+	 */
+	disable_work(&sc->disconnect_work);
+	disable_work(&sc->connect.work);
+	disable_work(&sc->recv_io.posted.refill_work);
+	disable_work(&sc->idle.immediate_work);
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	disable_delayed_work(&sc->idle.timer_work);
+
+	/*
+	 * In case we were a listener we need to
+	 * disconnect all pending and ready sockets
+	 *
+	 * First we move ready sockets to pending again.
+	 */
+	spin_lock_irqsave(&sc->listen.lock, flags);
+	list_splice_init(&sc->listen.ready, &sc->listen.pending);
+	list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list)
+		smbdirect_socket_schedule_cleanup(psc, sc->first_error);
+	spin_unlock_irqrestore(&sc->listen.lock, flags);
+
+	switch (sc->status) {
+	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
+	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
+	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
+	case SMBDIRECT_SOCKET_CONNECTED:
+	case SMBDIRECT_SOCKET_ERROR:
+		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;
+		/*
+		 * Make sure we hold the callback lock
+		 * im order to coordinate with the
+		 * rdma_event handlers, typically
+		 * smbdirect_connection_rdma_event_handler(),
+		 * and smbdirect_socket_destroy().
+		 *
+		 * So that the order of ib_drain_qp()
+		 * and rdma_disconnect() is controlled
+		 * by the mutex.
+		 */
+		rdma_lock_handler(sc->rdma.cm_id);
+		rdma_disconnect(sc->rdma.cm_id);
+		rdma_unlock_handler(sc->rdma.cm_id);
+		break;
+
+	case SMBDIRECT_SOCKET_CREATED:
+	case SMBDIRECT_SOCKET_LISTENING:
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
+	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
+	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
+	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
+		/*
+		 * rdma_{accept,connect}() never reached
+		 * RDMA_CM_EVENT_ESTABLISHED
+		 */
+		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
+		break;
+
+	case SMBDIRECT_SOCKET_DISCONNECTING:
+	case SMBDIRECT_SOCKET_DISCONNECTED:
+	case SMBDIRECT_SOCKET_DESTROYED:
+		break;
+	}
+
+	/*
+	 * Wake up all waiters in all wait queues
+	 * in order to notice the broken connection.
+	 */
+	smbdirect_socket_wake_up_all(sc);
+}
+
+static void smbdirect_socket_destroy(struct smbdirect_socket *sc)
+{
+	struct smbdirect_socket *psc, *tsc;
+	size_t psockets;
+	struct smbdirect_recv_io *recv_io;
+	struct smbdirect_recv_io *recv_tmp;
+	LIST_HEAD(all_list);
+	unsigned long flags;
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"status=%s first_error=%1pe",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+
+	/*
+	 * This should not never be called in an interrupt!
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	if (sc->status == SMBDIRECT_SOCKET_DESTROYED)
+		return;
+
+	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_DISCONNECTED,
+		  "status=%s first_error=%1pe",
+		  smbdirect_socket_status_string(sc->status),
+		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+
+	/*
+	 * The listener should clear this before we reach this
+	 */
+	WARN_ONCE(sc->accept.listener,
+		  "status=%s first_error=%1pe",
+		  smbdirect_socket_status_string(sc->status),
+		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+
+	/*
+	 * Wake up all waiters in all wait queues
+	 * in order to notice the broken connection.
+	 *
+	 * Most likely this was already called via
+	 * smbdirect_socket_cleanup_work(), but call it again...
+	 */
+	smbdirect_socket_wake_up_all(sc);
+
+	disable_work_sync(&sc->disconnect_work);
+	disable_work_sync(&sc->connect.work);
+	disable_work_sync(&sc->recv_io.posted.refill_work);
+	disable_work_sync(&sc->idle.immediate_work);
+	disable_delayed_work_sync(&sc->idle.timer_work);
+
+	if (sc->rdma.cm_id)
+		rdma_lock_handler(sc->rdma.cm_id);
+
+	if (sc->ib.qp) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"drain qp\n");
+		ib_drain_qp(sc->ib.qp);
+	}
+
+	/*
+	 * In case we were a listener we need to
+	 * disconnect all pending and ready sockets
+	 *
+	 * We move ready sockets to pending again.
+	 */
+	spin_lock_irqsave(&sc->listen.lock, flags);
+	list_splice_tail_init(&sc->listen.ready, &all_list);
+	list_splice_tail_init(&sc->listen.pending, &all_list);
+	spin_unlock_irqrestore(&sc->listen.lock, flags);
+	psockets = list_count_nodes(&all_list);
+	if (sc->listen.backlog != -1) /* was a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"release %zu pending sockets\n", psockets);
+	list_for_each_entry_safe(psc, tsc, &all_list, accept.list) {
+		list_del_init(&psc->accept.list);
+		psc->accept.listener = NULL;
+		smbdirect_socket_release(psc);
+	}
+	if (sc->listen.backlog != -1) /* was a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"released %zu pending sockets\n", psockets);
+	INIT_LIST_HEAD(&all_list);
+
+	/* It's not possible for upper layer to get to reassembly */
+	if (sc->listen.backlog == -1) /* was not a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"drain the reassembly queue\n");
+	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
+	list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list);
+	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+	list_for_each_entry_safe(recv_io, recv_tmp, &all_list, list)
+		smbdirect_connection_put_recv_io(recv_io);
+	sc->recv_io.reassembly.data_length = 0;
+
+	if (sc->listen.backlog == -1) /* was not a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"freeing mr list\n");
+	smbdirect_connection_destroy_mr_list(sc);
+
+	if (sc->listen.backlog == -1) /* was not a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"destroying qp\n");
+	smbdirect_connection_destroy_qp(sc);
+	if (sc->rdma.cm_id) {
+		rdma_unlock_handler(sc->rdma.cm_id);
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"destroying cm_id\n");
+		rdma_destroy_id(sc->rdma.cm_id);
+		sc->rdma.cm_id = NULL;
+	}
+
+	if (sc->listen.backlog == -1) /* was not a listener */
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"destroying mem pools\n");
+	smbdirect_connection_destroy_mem_pools(sc);
+
+	sc->status = SMBDIRECT_SOCKET_DESTROYED;
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"rdma session destroyed\n");
+}
+
+void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc)
+{
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"status=%s first_error=%1pe",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+
+	/*
+	 * This should not never be called in an interrupt!
+	 */
+	WARN_ON_ONCE(in_interrupt());
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"cancelling and disable disconnect_work\n");
+	disable_work_sync(&sc->disconnect_work);
+
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"destroying rdma session\n");
+	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) {
+		/*
+		 * SMBDIRECT_LOG_INFO is enough here
+		 * as this is the typical case where
+		 * we terminate the connection ourself.
+		 */
+		smbdirect_socket_schedule_cleanup_lvl(sc,
+						      SMBDIRECT_LOG_INFO,
+						      -ESHUTDOWN);
+		smbdirect_socket_cleanup_work(&sc->disconnect_work);
+	}
+	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) {
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"wait for transport being disconnected\n");
+		wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED);
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+			"waited for transport being disconnected\n");
+	}
+
+	/*
+	 * Once we reached SMBDIRECT_SOCKET_DISCONNECTED,
+	 * we should call smbdirect_socket_destroy()
+	 */
+	smbdirect_socket_destroy(sc);
+	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,
+		"status=%s first_error=%1pe",
+		smbdirect_socket_status_string(sc->status),
+		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
+}
+
+int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr)
+{
+	int ret;
+
+	if (sc->status != SMBDIRECT_SOCKET_CREATED)
+		return -EINVAL;
+
+	ret = rdma_bind_addr(sc->rdma.cm_id, addr);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_bind);
+
+void smbdirect_socket_shutdown(struct smbdirect_socket *sc)
+{
+	smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN);
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_shutdown);
+
+static void smbdirect_socket_release_disconnect(struct kref *kref)
+{
+	struct smbdirect_socket *sc =
+		container_of(kref, struct smbdirect_socket, refs.disconnect);
+
+	/*
+	 * For now do a sync disconnect/destroy
+	 */
+	smbdirect_socket_destroy_sync(sc);
+}
+
+static void smbdirect_socket_release_destroy(struct kref *kref)
+{
+	struct smbdirect_socket *sc =
+		container_of(kref, struct smbdirect_socket, refs.destroy);
+
+	/*
+	 * Do a sync disconnect/destroy...
+	 * hopefully a no-op, as it should be already
+	 * in DESTROYED state, before we free the memory.
+	 */
+	smbdirect_socket_destroy_sync(sc);
+	kfree(sc);
+}
+
+void smbdirect_socket_release(struct smbdirect_socket *sc)
+{
+	/*
+	 * We expect only 1 disconnect reference
+	 * and if it is already 0, it's a use after free!
+	 */
+	WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1);
+	WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect));
+
+	/*
+	 * This may not trigger smbdirect_socket_release_destroy(),
+	 * if struct smbdirect_socket is embedded in another structure
+	 * indicated by REFCOUNT_MAX.
+	 */
+	kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy);
+}
+__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_release);
+
+int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc,
+				      enum smbdirect_socket_status expected_status,
+				      int unexpected_errno,
+				      wait_queue_head_t *waitq,
+				      atomic_t *total_credits,
+				      int needed)
+{
+	int ret;
+
+	if (WARN_ON_ONCE(needed < 0))
+		return -EINVAL;
+
+	do {
+		if (atomic_sub_return(needed, total_credits) >= 0)
+			return 0;
+
+		atomic_add(needed, total_credits);
+		ret = wait_event_interruptible(*waitq,
+					       atomic_read(total_credits) >= needed ||
+					       sc->status != expected_status);
+
+		if (sc->status != expected_status)
+			return unexpected_errno;
+		else if (ret < 0)
+			return ret;
+	} while (true);
+}
diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h
index 95265192bb01b..c09eddd8ad16d 100644
--- a/fs/smb/common/smbdirect/smbdirect_socket.h
+++ b/fs/smb/common/smbdirect/smbdirect_socket.h
@@ -6,10 +6,18 @@
 #ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__
 #define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__
 
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/kref.h>
+#include <linux/mempool.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/completion.h>
 #include <rdma/rw.h>
 
 enum smbdirect_socket_status {
 	SMBDIRECT_SOCKET_CREATED,
+	SMBDIRECT_SOCKET_LISTENING,
 	SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED,
 	SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING,
 	SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED,
@@ -35,6 +43,8 @@ const char *smbdirect_socket_status_string(enum smbdirect_socket_status status)
 	switch (status) {
 	case SMBDIRECT_SOCKET_CREATED:
 		return "CREATED";
+	case SMBDIRECT_SOCKET_LISTENING:
+		return "LISTENING";
 	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
 		return "RESOLVE_ADDR_NEEDED";
 	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
@@ -99,18 +109,59 @@ struct smbdirect_socket {
 	int first_error;
 
 	/*
-	 * This points to the workqueue to
+	 * This points to the workqueues to
 	 * be used for this socket.
-	 * It can be per socket (on the client)
-	 * or point to a global workqueue (on the server)
 	 */
-	struct workqueue_struct *workqueue;
+	struct {
+		struct workqueue_struct *accept;
+		struct workqueue_struct *connect;
+		struct workqueue_struct *idle;
+		struct workqueue_struct *refill;
+		struct workqueue_struct *immediate;
+		struct workqueue_struct *cleanup;
+	} workqueues;
 
 	struct work_struct disconnect_work;
 
+	/*
+	 * The reference counts.
+	 */
+	struct {
+		/*
+		 * This holds the references by the
+		 * frontend, typically the smb layer.
+		 *
+		 * It is typically 1 and a disconnect
+		 * will happen if it reaches 0.
+		 */
+		struct kref disconnect;
+
+		/*
+		 * This holds the reference by the
+		 * backend, the code that manages
+		 * the lifetime of the whole
+		 * struct smbdirect_socket,
+		 * if this reaches 0 it can will
+		 * be freed.
+		 *
+		 * Can be REFCOUNT_MAX is part
+		 * of another structure.
+		 *
+		 * This is equal or higher than
+		 * the disconnect refcount.
+		 */
+		struct kref destroy;
+	} refs;
+
 	/* RDMA related */
 	struct {
 		struct rdma_cm_id *cm_id;
+		/*
+		 * The expected event in our current
+		 * cm_id->event_handler, all other events
+		 * are treated as an error.
+		 */
+		enum rdma_cm_event_type expected_event;
 		/*
 		 * This is for iWarp MPA v1
 		 */
@@ -120,6 +171,7 @@ struct smbdirect_socket {
 	/* IB verbs related */
 	struct {
 		struct ib_pd *pd;
+		enum ib_poll_context poll_ctx;
 		struct ib_cq *send_cq;
 		struct ib_cq *recv_cq;
 
@@ -149,6 +201,35 @@ struct smbdirect_socket {
 		struct delayed_work timer_work;
 	} idle;
 
+	/*
+	 * The state for listen sockets
+	 */
+	struct {
+		spinlock_t lock;
+		struct list_head pending;
+		struct list_head ready;
+		wait_queue_head_t wait_queue;
+		/*
+		 * This starts as -1 and a value != -1
+		 * means this socket was in LISTENING state
+		 * before. Note the valid backlog can
+		 * only be > 0.
+		 */
+		int backlog;
+	} listen;
+
+	/*
+	 * The state for sockets waiting
+	 * for accept, either still waiting
+	 * for the negotiation to finish
+	 * or already ready with a usable
+	 * connection.
+	 */
+	struct {
+		struct smbdirect_socket *listener;
+		struct list_head list;
+	} accept;
+
 	/*
 	 * The state for posted send buffers
 	 */
@@ -158,8 +239,9 @@ struct smbdirect_socket {
 		 * smbdirect_send_io buffers
 		 */
 		struct {
-			struct kmem_cache	*cache;
-			mempool_t		*pool;
+			struct kmem_cache *cache;
+			mempool_t *pool;
+			gfp_t gfp_mask;
 		} mem;
 
 		/*
@@ -194,10 +276,6 @@ struct smbdirect_socket {
 		 */
 		struct {
 			atomic_t count;
-			/*
-			 * woken when count is decremented
-			 */
-			wait_queue_head_t dec_wait_queue;
 			/*
 			 * woken when count reached zero
 			 */
@@ -223,8 +301,9 @@ struct smbdirect_socket {
 		 * smbdirect_recv_io buffers
 		 */
 		struct {
-			struct kmem_cache	*cache;
-			mempool_t		*pool;
+			struct kmem_cache *cache;
+			mempool_t *pool;
+			gfp_t gfp_mask;
 		} mem;
 
 		/*
@@ -310,19 +389,20 @@ struct smbdirect_socket {
 		struct {
 			atomic_t count;
 		} used;
-
-		struct work_struct recovery_work;
-
-		/* Used by transport to wait until all MRs are returned */
-		struct {
-			wait_queue_head_t wait_queue;
-		} cleanup;
 	} mr_io;
 
 	/*
 	 * The state for RDMA read/write requests on the server
 	 */
 	struct {
+		/*
+		 * Memory hints for
+		 * smbdirect_rw_io structs
+		 */
+		struct {
+			gfp_t gfp_mask;
+		} mem;
+
 		/*
 		 * The credit state for the send side
 		 */
@@ -350,6 +430,21 @@ struct smbdirect_socket {
 		u64 dequeue_reassembly_queue;
 		u64 send_empty;
 	} statistics;
+
+	struct {
+		void *private_ptr;
+		bool (*needed)(struct smbdirect_socket *sc,
+			       void *private_ptr,
+			       unsigned int lvl,
+			       unsigned int cls);
+		void (*vaprintf)(struct smbdirect_socket *sc,
+				 const char *func,
+				 unsigned int line,
+				 void *private_ptr,
+				 unsigned int lvl,
+				 unsigned int cls,
+				 struct va_format *vaf);
+	} logging;
 };
 
 static void __smbdirect_socket_disabled_work(struct work_struct *work)
@@ -360,6 +455,100 @@ static void __smbdirect_socket_disabled_work(struct work_struct *work)
 	WARN_ON_ONCE(1);
 }
 
+static bool __smbdirect_log_needed(struct smbdirect_socket *sc,
+				   void *private_ptr,
+				   unsigned int lvl,
+				   unsigned int cls)
+{
+	/*
+	 * Should never be called, the caller should
+	 * set it's own functions.
+	 */
+	WARN_ON_ONCE(1);
+	return false;
+}
+
+static void __smbdirect_log_vaprintf(struct smbdirect_socket *sc,
+				     const char *func,
+				     unsigned int line,
+				     void *private_ptr,
+				     unsigned int lvl,
+				     unsigned int cls,
+				     struct va_format *vaf)
+{
+	/*
+	 * Should never be called, the caller should
+	 * set it's own functions.
+	 */
+	WARN_ON_ONCE(1);
+}
+
+__printf(6, 7)
+static void __smbdirect_log_printf(struct smbdirect_socket *sc,
+				   const char *func,
+				   unsigned int line,
+				   unsigned int lvl,
+				   unsigned int cls,
+				   const char *fmt,
+				   ...);
+__maybe_unused
+static void __smbdirect_log_printf(struct smbdirect_socket *sc,
+				   const char *func,
+				   unsigned int line,
+				   unsigned int lvl,
+				   unsigned int cls,
+				   const char *fmt,
+				   ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+
+	vaf.fmt = fmt;
+	vaf.va = &args;
+
+	sc->logging.vaprintf(sc,
+			     func,
+			     line,
+			     sc->logging.private_ptr,
+			     lvl,
+			     cls,
+			     &vaf);
+	va_end(args);
+}
+
+#define ___smbdirect_log_generic(sc, func, line, lvl, cls, fmt, args...) do {	\
+	if (sc->logging.needed(sc, sc->logging.private_ptr, lvl, cls)) {	\
+		__smbdirect_log_printf(sc, func, line, lvl, cls, fmt, ##args);	\
+	}									\
+} while (0)
+#define __smbdirect_log_generic(sc, lvl, cls, fmt, args...) \
+	___smbdirect_log_generic(sc, __func__, __LINE__, lvl, cls, fmt, ##args)
+
+#define smbdirect_log_outgoing(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_OUTGOING, fmt, ##args)
+#define smbdirect_log_incoming(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_INCOMING, fmt, ##args)
+#define smbdirect_log_read(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_READ, fmt, ##args)
+#define smbdirect_log_write(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_WRITE, fmt, ##args)
+#define smbdirect_log_rdma_send(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_SEND, fmt, ##args)
+#define smbdirect_log_rdma_recv(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RECV, fmt, ##args)
+#define smbdirect_log_keep_alive(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_KEEP_ALIVE, fmt, ##args)
+#define smbdirect_log_rdma_event(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_EVENT, fmt, ##args)
+#define smbdirect_log_rdma_mr(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_MR, fmt, ##args)
+#define smbdirect_log_rdma_rw(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RW, fmt, ##args)
+#define smbdirect_log_negotiate(sc, lvl, fmt, args...) \
+		__smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_NEGOTIATE, fmt, ##args)
+
 static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 {
 	/*
@@ -370,9 +559,23 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 
 	init_waitqueue_head(&sc->status_wait);
 
+	sc->workqueues.accept = smbdirect_globals.workqueues.accept;
+	sc->workqueues.connect = smbdirect_globals.workqueues.connect;
+	sc->workqueues.idle = smbdirect_globals.workqueues.idle;
+	sc->workqueues.refill = smbdirect_globals.workqueues.refill;
+	sc->workqueues.immediate = smbdirect_globals.workqueues.immediate;
+	sc->workqueues.cleanup = smbdirect_globals.workqueues.cleanup;
+
 	INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work);
 	disable_work_sync(&sc->disconnect_work);
 
+	kref_init(&sc->refs.disconnect);
+	sc->refs.destroy = (struct kref) KREF_INIT(REFCOUNT_MAX);
+
+	sc->rdma.expected_event = RDMA_CM_EVENT_INTERNAL;
+
+	sc->ib.poll_ctx = IB_POLL_UNBOUND_WORKQUEUE;
+
 	spin_lock_init(&sc->connect.lock);
 	INIT_WORK(&sc->connect.work, __smbdirect_socket_disabled_work);
 	disable_work_sync(&sc->connect.work);
@@ -382,6 +585,16 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 	INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work);
 	disable_delayed_work_sync(&sc->idle.timer_work);
 
+	spin_lock_init(&sc->listen.lock);
+	INIT_LIST_HEAD(&sc->listen.pending);
+	INIT_LIST_HEAD(&sc->listen.ready);
+	sc->listen.backlog = -1; /* not a listener */
+	init_waitqueue_head(&sc->listen.wait_queue);
+
+	INIT_LIST_HEAD(&sc->accept.list);
+
+	sc->send_io.mem.gfp_mask = GFP_KERNEL;
+
 	atomic_set(&sc->send_io.bcredits.count, 0);
 	init_waitqueue_head(&sc->send_io.bcredits.wait_queue);
 
@@ -392,9 +605,10 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 	init_waitqueue_head(&sc->send_io.credits.wait_queue);
 
 	atomic_set(&sc->send_io.pending.count, 0);
-	init_waitqueue_head(&sc->send_io.pending.dec_wait_queue);
 	init_waitqueue_head(&sc->send_io.pending.zero_wait_queue);
 
+	sc->recv_io.mem.gfp_mask = GFP_KERNEL;
+
 	INIT_LIST_HEAD(&sc->recv_io.free.list);
 	spin_lock_init(&sc->recv_io.free.lock);
 
@@ -409,6 +623,7 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 	spin_lock_init(&sc->recv_io.reassembly.lock);
 	init_waitqueue_head(&sc->recv_io.reassembly.wait_queue);
 
+	sc->rw_io.mem.gfp_mask = GFP_KERNEL;
 	atomic_set(&sc->rw_io.credits.count, 0);
 	init_waitqueue_head(&sc->rw_io.credits.wait_queue);
 
@@ -417,9 +632,10 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 	atomic_set(&sc->mr_io.ready.count, 0);
 	init_waitqueue_head(&sc->mr_io.ready.wait_queue);
 	atomic_set(&sc->mr_io.used.count, 0);
-	INIT_WORK(&sc->mr_io.recovery_work, __smbdirect_socket_disabled_work);
-	disable_work_sync(&sc->mr_io.recovery_work);
-	init_waitqueue_head(&sc->mr_io.cleanup.wait_queue);
+
+	sc->logging.private_ptr = NULL;
+	sc->logging.needed = __smbdirect_log_needed;
+	sc->logging.vaprintf = __smbdirect_log_vaprintf;
 }
 
 #define __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, __error_cmd, __unexpected_cmd) ({ \
@@ -436,7 +652,6 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 
 #define __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, __unexpected_cmd) \
 	__SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, \
-	, \
 	{ \
 		const struct sockaddr_storage *__src = NULL; \
 		const struct sockaddr_storage *__dst = NULL; \
@@ -444,6 +659,26 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 			__src = &(__sc)->rdma.cm_id->route.addr.src_addr; \
 			__dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \
 		} \
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, \
+			"expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \
+			smbdirect_socket_status_string(__expected_status), \
+			smbdirect_socket_status_string((__sc)->status), \
+			SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \
+			__src, __dst); \
+	}, \
+	{ \
+		const struct sockaddr_storage *__src = NULL; \
+		const struct sockaddr_storage *__dst = NULL; \
+		if ((__sc)->rdma.cm_id) { \
+			__src = &(__sc)->rdma.cm_id->route.addr.src_addr; \
+			__dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \
+		} \
+		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, \
+			"expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \
+			smbdirect_socket_status_string(__expected_status), \
+			smbdirect_socket_status_string((__sc)->status), \
+			SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \
+			__src, __dst); \
 		WARN_ONCE(1, \
 			"expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \
 			smbdirect_socket_status_string(__expected_status), \
@@ -456,6 +691,11 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 #define SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status) \
 	__SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, /* nothing */)
 
+#ifndef __SMBDIRECT_SOCKET_DISCONNECT
+#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) \
+		smbdirect_socket_schedule_cleanup(__sc, -ECONNABORTED)
+#endif /* ! __SMBDIRECT_SOCKET_DISCONNECT */
+
 #define SMBDIRECT_CHECK_STATUS_DISCONNECT(__sc, __expected_status) \
 	__SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, \
 		__SMBDIRECT_SOCKET_DISCONNECT(__sc);)
@@ -574,4 +814,19 @@ struct smbdirect_rw_io {
 	struct scatterlist sg_list[];
 };
 
+static inline size_t smbdirect_get_buf_page_count(const void *buf, size_t size)
+{
+	return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
+		(uintptr_t)buf / PAGE_SIZE;
+}
+
+/*
+ * Maximum number of retries on data transfer operations
+ */
+#define SMBDIRECT_RDMA_CM_RETRY 6
+/*
+ * No need to retry on Receiver Not Ready since SMB_DIRECT manages credits
+ */
+#define SMBDIRECT_RDMA_CM_RNR_RETRY 0
+
 #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */
diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig
index 12594879cb640..5c77c5552ed8a 100644
--- a/fs/smb/server/Kconfig
+++ b/fs/smb/server/Kconfig
@@ -48,8 +48,9 @@ if SMB_SERVER
 
 config SMB_SERVER_SMBDIRECT
 	bool "Support for SMB Direct protocol"
-	depends on SMB_SERVER=m && INFINIBAND && INFINIBAND_ADDR_TRANS || SMB_SERVER=y && INFINIBAND=y && INFINIBAND_ADDR_TRANS=y
-	select SG_POOL
+	depends on SMB_SERVER && INFINIBAND && INFINIBAND_ADDR_TRANS
+	depends on SMB_SERVER=m || INFINIBAND=y
+	select SMB_COMMON_SMBDIRECT
 	default n
 
 	help
diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c
index 1bb2081c492c3..31392d245c309 100644
--- a/fs/smb/server/connection.c
+++ b/fs/smb/server/connection.c
@@ -375,9 +375,6 @@ int ksmbd_conn_handler_loop(void *p)
 	mutex_init(&conn->srv_mutex);
 	__module_get(THIS_MODULE);
 
-	if (t->ops->prepare && t->ops->prepare(t))
-		goto out;
-
 	max_req = server_conf.max_inflight_req;
 	conn->last_active = jiffies;
 	set_freezable();
@@ -469,7 +466,6 @@ int ksmbd_conn_handler_loop(void *p)
 		}
 	}
 
-out:
 	ksmbd_conn_set_releasing(conn);
 	/* Wait till all reference dropped to the Server object*/
 	ksmbd_debug(CONN, "Wait for all pending requests(%d)\n", atomic_read(&conn->r_count));
@@ -565,6 +561,5 @@ void ksmbd_conn_transport_destroy(void)
 	ksmbd_tcp_destroy();
 	ksmbd_rdma_stop_listening();
 	stop_sessions();
-	ksmbd_rdma_destroy();
 	mutex_unlock(&init_lock);
 }
diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h
index 1e2587036bca6..ae21a1bd4c70e 100644
--- a/fs/smb/server/connection.h
+++ b/fs/smb/server/connection.h
@@ -127,7 +127,6 @@ struct ksmbd_conn_ops {
 };
 
 struct ksmbd_transport_ops {
-	int (*prepare)(struct ksmbd_transport *t);
 	void (*disconnect)(struct ksmbd_transport *t);
 	void (*shutdown)(struct ksmbd_transport *t);
 	int (*read)(struct ksmbd_transport *t, char *buf,
diff --git a/fs/smb/server/mgmt/tree_connect.c b/fs/smb/server/mgmt/tree_connect.c
index a72d7e42a6c27..58e5b8592da46 100644
--- a/fs/smb/server/mgmt/tree_connect.c
+++ b/fs/smb/server/mgmt/tree_connect.c
@@ -102,8 +102,10 @@ ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name)
 
 void ksmbd_tree_connect_put(struct ksmbd_tree_connect *tcon)
 {
-	if (atomic_dec_and_test(&tcon->refcount))
+	if (atomic_dec_and_test(&tcon->refcount)) {
+		ksmbd_share_config_put(tcon->share_conf);
 		kfree(tcon);
+	}
 }
 
 static int __ksmbd_tree_conn_disconnect(struct ksmbd_session *sess,
@@ -113,10 +115,11 @@ static int __ksmbd_tree_conn_disconnect(struct ksmbd_session *sess,
 
 	ret = ksmbd_ipc_tree_disconnect_request(sess->id, tree_conn->id);
 	ksmbd_release_tree_conn_id(sess, tree_conn->id);
-	ksmbd_share_config_put(tree_conn->share_conf);
 	ksmbd_counter_dec(KSMBD_COUNTER_TREE_CONNS);
-	if (atomic_dec_and_test(&tree_conn->refcount))
+	if (atomic_dec_and_test(&tree_conn->refcount)) {
+		ksmbd_share_config_put(tree_conn->share_conf);
 		kfree(tree_conn);
+	}
 	return ret;
 }
 
diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c
index 393a4ae47cc1d..9b2bb8764a805 100644
--- a/fs/smb/server/oplock.c
+++ b/fs/smb/server/oplock.c
@@ -82,11 +82,19 @@ static void lease_del_list(struct oplock_info *opinfo)
 	spin_unlock(&lb->lb_lock);
 }
 
-static void lb_add(struct lease_table *lb)
+static struct lease_table *alloc_lease_table(struct oplock_info *opinfo)
 {
-	write_lock(&lease_list_lock);
-	list_add(&lb->l_entry, &lease_table_list);
-	write_unlock(&lease_list_lock);
+	struct lease_table *lb;
+
+	lb = kmalloc_obj(struct lease_table, KSMBD_DEFAULT_GFP);
+	if (!lb)
+		return NULL;
+
+	memcpy(lb->client_guid, opinfo->conn->ClientGUID,
+	       SMB2_CLIENT_GUID_SIZE);
+	INIT_LIST_HEAD(&lb->lease_list);
+	spin_lock_init(&lb->lb_lock);
+	return lb;
 }
 
 static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx)
@@ -1042,34 +1050,27 @@ static void copy_lease(struct oplock_info *op1, struct oplock_info *op2)
 	lease2->version = lease1->version;
 }
 
-static int add_lease_global_list(struct oplock_info *opinfo)
+static void add_lease_global_list(struct oplock_info *opinfo,
+				  struct lease_table *new_lb)
 {
 	struct lease_table *lb;
 
-	read_lock(&lease_list_lock);
+	write_lock(&lease_list_lock);
 	list_for_each_entry(lb, &lease_table_list, l_entry) {
 		if (!memcmp(lb->client_guid, opinfo->conn->ClientGUID,
 			    SMB2_CLIENT_GUID_SIZE)) {
 			opinfo->o_lease->l_lb = lb;
 			lease_add_list(opinfo);
-			read_unlock(&lease_list_lock);
-			return 0;
+			write_unlock(&lease_list_lock);
+			kfree(new_lb);
+			return;
 		}
 	}
-	read_unlock(&lease_list_lock);
 
-	lb = kmalloc_obj(struct lease_table, KSMBD_DEFAULT_GFP);
-	if (!lb)
-		return -ENOMEM;
-
-	memcpy(lb->client_guid, opinfo->conn->ClientGUID,
-	       SMB2_CLIENT_GUID_SIZE);
-	INIT_LIST_HEAD(&lb->lease_list);
-	spin_lock_init(&lb->lb_lock);
-	opinfo->o_lease->l_lb = lb;
+	opinfo->o_lease->l_lb = new_lb;
 	lease_add_list(opinfo);
-	lb_add(lb);
-	return 0;
+	list_add(&new_lb->l_entry, &lease_table_list);
+	write_unlock(&lease_list_lock);
 }
 
 static void set_oplock_level(struct oplock_info *opinfo, int level,
@@ -1189,6 +1190,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid,
 	int err = 0;
 	struct oplock_info *opinfo = NULL, *prev_opinfo = NULL;
 	struct ksmbd_inode *ci = fp->f_ci;
+	struct lease_table *new_lb = NULL;
 	bool prev_op_has_lease;
 	__le32 prev_op_state = 0;
 
@@ -1291,21 +1293,37 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid,
 	set_oplock_level(opinfo, req_op_level, lctx);
 
 out:
-	opinfo_count_inc(fp);
-	opinfo_add(opinfo, fp);
-
+	/*
+	 * Set o_fp before any publication so that concurrent readers
+	 * (e.g. find_same_lease_key() on the lease list) that
+	 * dereference opinfo->o_fp don't hit a NULL pointer.
+	 *
+	 * Keep the original publication order so concurrent opens can
+	 * still observe the in-flight grant via ci->m_op_list, but make
+	 * everything after opinfo_add() no-fail by preallocating any new
+	 * lease_table first.
+	 */
+	opinfo->o_fp = fp;
 	if (opinfo->is_lease) {
-		err = add_lease_global_list(opinfo);
-		if (err)
+		new_lb = alloc_lease_table(opinfo);
+		if (!new_lb) {
+			err = -ENOMEM;
 			goto err_out;
+		}
 	}
 
+	opinfo_count_inc(fp);
+	opinfo_add(opinfo, fp);
+
+	if (opinfo->is_lease)
+		add_lease_global_list(opinfo, new_lb);
+
 	rcu_assign_pointer(fp->f_opinfo, opinfo);
-	opinfo->o_fp = fp;
 
 	return 0;
 err_out:
-	__free_opinfo(opinfo);
+	kfree(new_lb);
+	opinfo_put(opinfo);
 	return err;
 }
 
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 9f7ff7491e9a8..c7019be421821 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -24,7 +24,6 @@
 #include "asn1.h"
 #include "connection.h"
 #include "transport_ipc.h"
-#include "../common/smbdirect/smbdirect.h"
 #include "transport_rdma.h"
 #include "vfs.h"
 #include "vfs_cache.h"
@@ -126,6 +125,8 @@ int smb2_get_ksmbd_tcon(struct ksmbd_work *work)
 			pr_err("The first operation in the compound does not have tcon\n");
 			return -EINVAL;
 		}
+		if (work->tcon->t_state != TREE_CONNECTED)
+			return -ENOENT;
 		if (tree_id != UINT_MAX && work->tcon->id != tree_id) {
 			pr_err("tree id(%u) is different with id(%u) in first operation\n",
 					tree_id, work->tcon->id);
@@ -1937,8 +1938,14 @@ int smb2_sess_setup(struct ksmbd_work *work)
 			if (sess->user && sess->user->flags & KSMBD_USER_FLAG_DELAY_SESSION)
 				try_delay = true;
 
-			sess->last_active = jiffies;
-			sess->state = SMB2_SESSION_EXPIRED;
+			/*
+			 * For binding requests, session belongs to another
+			 * connection. Do not expire it.
+			 */
+			if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) {
+				sess->last_active = jiffies;
+				sess->state = SMB2_SESSION_EXPIRED;
+			}
 			ksmbd_user_session_put(sess);
 			work->sess = NULL;
 			if (try_delay) {
@@ -1948,6 +1955,7 @@ int smb2_sess_setup(struct ksmbd_work *work)
 			}
 		}
 		smb2_set_err_rsp(work);
+		conn->binding = false;
 	} else {
 		unsigned int iov_len;
 
@@ -2828,7 +2836,11 @@ static int parse_durable_handle_context(struct ksmbd_work *work,
 						goto out;
 					}
 
-					dh_info->fp->conn = conn;
+					if (dh_info->fp->conn) {
+						ksmbd_put_durable_fd(dh_info->fp);
+						err = -EBADF;
+						goto out;
+					}
 					dh_info->reconnected = true;
 					goto out;
 				}
@@ -5452,7 +5464,6 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 				    struct smb2_query_info_req *req,
 				    struct smb2_query_info_rsp *rsp)
 {
-	struct ksmbd_session *sess = work->sess;
 	struct ksmbd_conn *conn = work->conn;
 	struct ksmbd_share_config *share = work->tcon->share_conf;
 	int fsinfoclass = 0;
@@ -5589,10 +5600,11 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 
 		info = (struct object_id_info *)(rsp->Buffer);
 
-		if (!user_guest(sess->user))
-			memcpy(info->objid, user_passkey(sess->user), 16);
+		if (path.mnt->mnt_sb->s_uuid_len == 16)
+			memcpy(info->objid, path.mnt->mnt_sb->s_uuid.b,
+					path.mnt->mnt_sb->s_uuid_len);
 		else
-			memset(info->objid, 0, 16);
+			memcpy(info->objid, &stfs.f_fsid, sizeof(stfs.f_fsid));
 
 		info->extended_info.magic = cpu_to_le32(EXTENDED_INFO_MAGIC);
 		info->extended_info.version = cpu_to_le32(1);
@@ -7579,14 +7591,15 @@ int smb2_lock(struct ksmbd_work *work)
 		rc = vfs_lock_file(filp, smb_lock->cmd, flock, NULL);
 skip:
 		if (smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) {
+			locks_free_lock(flock);
+			kfree(smb_lock);
 			if (!rc) {
 				ksmbd_debug(SMB, "File unlocked\n");
 			} else if (rc == -ENOENT) {
 				rsp->hdr.Status = STATUS_NOT_LOCKED;
+				err = rc;
 				goto out;
 			}
-			locks_free_lock(flock);
-			kfree(smb_lock);
 		} else {
 			if (rc == FILE_LOCK_DEFERRED) {
 				void **argv;
@@ -7655,6 +7668,9 @@ int smb2_lock(struct ksmbd_work *work)
 				spin_unlock(&work->conn->llist_lock);
 				ksmbd_debug(SMB, "successful in taking lock\n");
 			} else {
+				locks_free_lock(flock);
+				kfree(smb_lock);
+				err = rc;
 				goto out;
 			}
 		}
@@ -7685,13 +7701,17 @@ int smb2_lock(struct ksmbd_work *work)
 		struct file_lock *rlock = NULL;
 
 		rlock = smb_flock_init(filp);
-		rlock->c.flc_type = F_UNLCK;
-		rlock->fl_start = smb_lock->start;
-		rlock->fl_end = smb_lock->end;
+		if (rlock) {
+			rlock->c.flc_type = F_UNLCK;
+			rlock->fl_start = smb_lock->start;
+			rlock->fl_end = smb_lock->end;
 
-		rc = vfs_lock_file(filp, F_SETLK, rlock, NULL);
-		if (rc)
-			pr_err("rollback unlock fail : %d\n", rc);
+			rc = vfs_lock_file(filp, F_SETLK, rlock, NULL);
+			if (rc)
+				pr_err("rollback unlock fail : %d\n", rc);
+		} else {
+			pr_err("rollback unlock alloc failed\n");
+		}
 
 		list_del(&smb_lock->llist);
 		spin_lock(&work->conn->llist_lock);
@@ -7701,7 +7721,8 @@ int smb2_lock(struct ksmbd_work *work)
 		spin_unlock(&work->conn->llist_lock);
 
 		locks_free_lock(smb_lock->fl);
-		locks_free_lock(rlock);
+		if (rlock)
+			locks_free_lock(rlock);
 		kfree(smb_lock);
 	}
 out2:
diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c
index 188572491d53f..706a2c8979488 100644
--- a/fs/smb/server/transport_rdma.c
+++ b/fs/smb/server/transport_rdma.c
@@ -11,30 +11,19 @@
 
 #include <linux/kthread.h>
 #include <linux/list.h>
-#include <linux/mempool.h>
-#include <linux/highmem.h>
-#include <linux/scatterlist.h>
 #include <linux/string_choices.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/rdma_cm.h>
-#include <rdma/rw.h>
-
-#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) smb_direct_disconnect_rdma_connection(__sc)
 
 #include "glob.h"
 #include "connection.h"
 #include "smb_common.h"
 #include "../common/smb2status.h"
-#include "../common/smbdirect/smbdirect.h"
-#include "../common/smbdirect/smbdirect_pdu.h"
-#include "../common/smbdirect/smbdirect_socket.h"
 #include "transport_rdma.h"
+#include "../common/smbdirect/smbdirect_public.h"
+
 
 #define SMB_DIRECT_PORT_IWARP		5445
 #define SMB_DIRECT_PORT_INFINIBAND	445
 
-#define SMB_DIRECT_VERSION_LE		cpu_to_le16(SMBDIRECT_V1)
-
 /* SMB_DIRECT negotiation timeout (for the server) in seconds */
 #define SMB_DIRECT_NEGOTIATE_TIMEOUT		5
 
@@ -50,11 +39,6 @@
  */
 #define SMB_DIRECT_CM_INITIATOR_DEPTH		8
 
-/* Maximum number of retries on data transfer operations */
-#define SMB_DIRECT_CM_RETRY			6
-/* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
-#define SMB_DIRECT_CM_RNR_RETRY		0
-
 /*
  * User configurable initial values per SMB_DIRECT transport connection
  * as defined in [MS-SMBD] 3.1.1.1
@@ -93,27 +77,79 @@ static int smb_direct_max_receive_size = 1364;
 
 static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE;
 
-static LIST_HEAD(smb_direct_device_list);
-static DEFINE_RWLOCK(smb_direct_device_lock);
-
-struct smb_direct_device {
-	struct ib_device	*ib_dev;
-	struct list_head	list;
-};
-
 static struct smb_direct_listener {
 	int			port;
-	struct rdma_cm_id	*cm_id;
-} smb_direct_ib_listener, smb_direct_iw_listener;
 
-static struct workqueue_struct *smb_direct_wq;
+	struct task_struct	*thread;
+
+	struct smbdirect_socket *socket;
+} smb_direct_ib_listener, smb_direct_iw_listener;
 
 struct smb_direct_transport {
 	struct ksmbd_transport	transport;
 
-	struct smbdirect_socket socket;
+	struct smbdirect_socket *socket;
 };
 
+static bool smb_direct_logging_needed(struct smbdirect_socket *sc,
+				      void *private_ptr,
+				      unsigned int lvl,
+				      unsigned int cls)
+{
+	if (lvl <= SMBDIRECT_LOG_ERR)
+		return true;
+
+	if (lvl > SMBDIRECT_LOG_INFO)
+		return false;
+
+	switch (cls) {
+	/*
+	 * These were more or less also logged before
+	 * the move to common code.
+	 *
+	 * SMBDIRECT_LOG_RDMA_MR was not used, but
+	 * that's client only code and we should
+	 * notice if it's used on the server...
+	 */
+	case SMBDIRECT_LOG_RDMA_EVENT:
+	case SMBDIRECT_LOG_RDMA_SEND:
+	case SMBDIRECT_LOG_RDMA_RECV:
+	case SMBDIRECT_LOG_WRITE:
+	case SMBDIRECT_LOG_READ:
+	case SMBDIRECT_LOG_NEGOTIATE:
+	case SMBDIRECT_LOG_OUTGOING:
+	case SMBDIRECT_LOG_RDMA_RW:
+	case SMBDIRECT_LOG_RDMA_MR:
+		return true;
+	/*
+	 * These were not logged before the move
+	 * to common code.
+	 */
+	case SMBDIRECT_LOG_KEEP_ALIVE:
+	case SMBDIRECT_LOG_INCOMING:
+		return false;
+	}
+
+	/*
+	 * Log all unknown messages
+	 */
+	return true;
+}
+
+static void smb_direct_logging_vaprintf(struct smbdirect_socket *sc,
+					const char *func,
+					unsigned int line,
+					void *private_ptr,
+					unsigned int lvl,
+					unsigned int cls,
+					struct va_format *vaf)
+{
+	if (lvl <= SMBDIRECT_LOG_ERR)
+		pr_err("%pV", vaf);
+	else
+		ksmbd_debug(RDMA, "%pV", vaf);
+}
+
 #define KSMBD_TRANS(t) (&(t)->transport)
 #define SMBD_TRANS(t)	(container_of(t, \
 				struct smb_direct_transport, transport))
@@ -129,299 +165,263 @@ void init_smbd_max_io_size(unsigned int sz)
 unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt)
 {
 	struct smb_direct_transport *t;
-	struct smbdirect_socket *sc;
-	struct smbdirect_socket_parameters *sp;
+	const struct smbdirect_socket_parameters *sp;
 
 	if (kt->ops != &ksmbd_smb_direct_transport_ops)
 		return 0;
 
 	t = SMBD_TRANS(kt);
-	sc = &t->socket;
-	sp = &sc->parameters;
+	sp = smbdirect_socket_get_current_parameters(t->socket);
 
 	return sp->max_read_write_size;
 }
 
-static inline int get_buf_page_count(void *buf, int size)
+static struct smb_direct_transport *alloc_transport(struct smbdirect_socket *sc)
 {
-	return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
-		(uintptr_t)buf / PAGE_SIZE;
-}
+	struct smb_direct_transport *t;
+	struct ksmbd_conn *conn;
 
-static void smb_direct_destroy_pools(struct smbdirect_socket *sc);
-static void smb_direct_post_recv_credits(struct work_struct *work);
-static int smb_direct_post_send_data(struct smbdirect_socket *sc,
-				     struct smbdirect_send_batch *send_ctx,
-				     struct kvec *iov, int niov,
-				     int remaining_data_length);
+	t = kzalloc_obj(*t, KSMBD_DEFAULT_GFP);
+	if (!t)
+		return NULL;
+	t->socket = sc;
 
-static inline void
-*smbdirect_recv_io_payload(struct smbdirect_recv_io *recvmsg)
-{
-	return (void *)recvmsg->packet;
-}
+	conn = ksmbd_conn_alloc();
+	if (!conn)
+		goto conn_alloc_failed;
 
-static struct
-smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *recvmsg = NULL;
-	unsigned long flags;
-
-	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
-	if (!list_empty(&sc->recv_io.free.list)) {
-		recvmsg = list_first_entry(&sc->recv_io.free.list,
-					   struct smbdirect_recv_io,
-					   list);
-		list_del(&recvmsg->list);
-	}
-	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
-	return recvmsg;
-}
+	down_write(&conn_list_lock);
+	hash_add(conn_list, &conn->hlist, 0);
+	up_write(&conn_list_lock);
 
-static void put_recvmsg(struct smbdirect_socket *sc,
-			struct smbdirect_recv_io *recvmsg)
-{
-	unsigned long flags;
-
-	if (likely(recvmsg->sge.length != 0)) {
-		ib_dma_unmap_single(sc->ib.dev,
-				    recvmsg->sge.addr,
-				    recvmsg->sge.length,
-				    DMA_FROM_DEVICE);
-		recvmsg->sge.length = 0;
-	}
+	conn->transport = KSMBD_TRANS(t);
+	KSMBD_TRANS(t)->conn = conn;
+	KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
 
-	spin_lock_irqsave(&sc->recv_io.free.lock, flags);
-	list_add(&recvmsg->list, &sc->recv_io.free.list);
-	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
+	return t;
 
-	queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
+conn_alloc_failed:
+	kfree(t);
+	return NULL;
 }
 
-static void enqueue_reassembly(struct smbdirect_socket *sc,
-			       struct smbdirect_recv_io *recvmsg,
-			       int data_length)
+static void smb_direct_free_transport(struct ksmbd_transport *kt)
 {
-	unsigned long flags;
+	struct smb_direct_transport *t = SMBD_TRANS(kt);
 
-	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-	list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list);
-	sc->recv_io.reassembly.queue_length++;
-	/*
-	 * Make sure reassembly_data_length is updated after list and
-	 * reassembly_queue_length are updated. On the dequeue side
-	 * reassembly_data_length is checked without a lock to determine
-	 * if reassembly_queue_length and list is up to date
-	 */
-	virt_wmb();
-	sc->recv_io.reassembly.data_length += data_length;
-	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
+	smbdirect_socket_release(t->socket);
+	kfree(t);
 }
 
-static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc)
+static void free_transport(struct smb_direct_transport *t)
 {
-	if (!list_empty(&sc->recv_io.reassembly.list))
-		return list_first_entry(&sc->recv_io.reassembly.list,
-				struct smbdirect_recv_io, list);
-	else
-		return NULL;
+	smbdirect_socket_shutdown(t->socket);
+	ksmbd_conn_free(KSMBD_TRANS(t)->conn);
 }
 
-static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc)
+static int smb_direct_read(struct ksmbd_transport *t, char *buf,
+			   unsigned int size, int unused)
 {
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 */
-	wake_up_all(&sc->status_wait);
-	wake_up_all(&sc->send_io.bcredits.wait_queue);
-	wake_up_all(&sc->send_io.lcredits.wait_queue);
-	wake_up_all(&sc->send_io.credits.wait_queue);
-	wake_up_all(&sc->send_io.pending.zero_wait_queue);
-	wake_up_all(&sc->recv_io.reassembly.wait_queue);
-	wake_up_all(&sc->rw_io.credits.wait_queue);
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
+	struct msghdr msg = { .msg_flags = 0, };
+	struct kvec iov = {
+		.iov_base = buf,
+		.iov_len = size,
+	};
+	int ret;
+
+	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, size);
+
+	ret = smbdirect_connection_recvmsg(sc, &msg, 0);
+	if (ret == -ERESTARTSYS)
+		ret = -EINTR;
+	return ret;
 }
 
-static void smb_direct_disconnect_rdma_work(struct work_struct *work)
+static int smb_direct_writev(struct ksmbd_transport *t,
+			     struct kvec *iov, int niovs, int buflen,
+			     bool need_invalidate, unsigned int remote_key)
 {
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, disconnect_work);
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
+	struct iov_iter iter;
 
-	if (sc->first_error == 0)
-		sc->first_error = -ECONNABORTED;
+	iov_iter_kvec(&iter, ITER_SOURCE, iov, niovs, buflen);
 
-	/*
-	 * make sure this and other work is not queued again
-	 * but here we don't block and avoid
-	 * disable[_delayed]_work_sync()
-	 */
-	disable_work(&sc->disconnect_work);
-	disable_work(&sc->connect.work);
-	disable_work(&sc->recv_io.posted.refill_work);
-	disable_delayed_work(&sc->idle.timer_work);
-	disable_work(&sc->idle.immediate_work);
-
-	switch (sc->status) {
-	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
-	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
-	case SMBDIRECT_SOCKET_CONNECTED:
-	case SMBDIRECT_SOCKET_ERROR:
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;
-		rdma_disconnect(sc->rdma.cm_id);
-		break;
+	return smbdirect_connection_send_iter(sc, &iter, 0,
+					      need_invalidate, remote_key);
+}
 
-	case SMBDIRECT_SOCKET_CREATED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
-		/*
-		 * rdma_accept() never reached
-		 * RDMA_CM_EVENT_ESTABLISHED
-		 */
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		break;
+static int smb_direct_rdma_write(struct ksmbd_transport *t,
+				 void *buf, unsigned int buflen,
+				 struct smbdirect_buffer_descriptor_v1 *desc,
+				 unsigned int desc_len)
+{
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
 
-	case SMBDIRECT_SOCKET_DISCONNECTING:
-	case SMBDIRECT_SOCKET_DISCONNECTED:
-	case SMBDIRECT_SOCKET_DESTROYED:
-		break;
-	}
+	return smbdirect_connection_rdma_xmit(sc, buf, buflen,
+					      desc, desc_len, false);
+}
 
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 */
-	smb_direct_disconnect_wake_up_all(sc);
+static int smb_direct_rdma_read(struct ksmbd_transport *t,
+				void *buf, unsigned int buflen,
+				struct smbdirect_buffer_descriptor_v1 *desc,
+				unsigned int desc_len)
+{
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
+
+	return smbdirect_connection_rdma_xmit(sc, buf, buflen,
+					      desc, desc_len, true);
 }
 
-static void
-smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc)
+static void smb_direct_disconnect(struct ksmbd_transport *t)
 {
-	if (sc->first_error == 0)
-		sc->first_error = -ECONNABORTED;
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
 
-	/*
-	 * make sure other work (than disconnect_work) is
-	 * not queued again but here we don't block and avoid
-	 * disable[_delayed]_work_sync()
-	 */
-	disable_work(&sc->connect.work);
-	disable_work(&sc->recv_io.posted.refill_work);
-	disable_work(&sc->idle.immediate_work);
-	disable_delayed_work(&sc->idle.timer_work);
-
-	switch (sc->status) {
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
-	case SMBDIRECT_SOCKET_ERROR:
-	case SMBDIRECT_SOCKET_DISCONNECTING:
-	case SMBDIRECT_SOCKET_DISCONNECTED:
-	case SMBDIRECT_SOCKET_DESTROYED:
-		/*
-		 * Keep the current error status
-		 */
-		break;
+	ksmbd_debug(RDMA, "Disconnecting sc=%p\n", sc);
 
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;
-		break;
+	free_transport(st);
+}
 
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
-	case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;
-		break;
+static void smb_direct_shutdown(struct ksmbd_transport *t)
+{
+	struct smb_direct_transport *st = SMBD_TRANS(t);
+	struct smbdirect_socket *sc = st->socket;
 
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
-	case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;
-		break;
+	ksmbd_debug(RDMA, "smb-direct shutdown sc=%p\n", sc);
 
-	case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
-	case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
-		break;
+	smbdirect_socket_shutdown(sc);
+}
 
-	case SMBDIRECT_SOCKET_CREATED:
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		break;
+static int smb_direct_new_connection(struct smb_direct_listener *listener,
+				     struct smbdirect_socket *client_sc)
+{
+	struct smb_direct_transport *t;
+	struct task_struct *handler;
+	int ret;
 
-	case SMBDIRECT_SOCKET_CONNECTED:
-		sc->status = SMBDIRECT_SOCKET_ERROR;
-		break;
+	t = alloc_transport(client_sc);
+	if (!t) {
+		smbdirect_socket_release(client_sc);
+		return -ENOMEM;
 	}
 
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 */
-	smb_direct_disconnect_wake_up_all(sc);
+	handler = kthread_run(ksmbd_conn_handler_loop,
+			      KSMBD_TRANS(t)->conn, "ksmbd:r%u",
+			      listener->port);
+	if (IS_ERR(handler)) {
+		ret = PTR_ERR(handler);
+		pr_err("Can't start thread\n");
+		goto out_err;
+	}
 
-	queue_work(sc->workqueue, &sc->disconnect_work);
+	return 0;
+out_err:
+	free_transport(t);
+	return ret;
 }
 
-static void smb_direct_send_immediate_work(struct work_struct *work)
+static int smb_direct_listener_kthread_fn(void *p)
 {
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, idle.immediate_work);
+	struct smb_direct_listener *listener = (struct smb_direct_listener *)p;
+	struct smbdirect_socket *client_sc = NULL;
+
+	while (!kthread_should_stop()) {
+		struct proto_accept_arg arg = { .err = -EINVAL, };
+		long timeo = MAX_SCHEDULE_TIMEOUT;
 
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return;
+		if (!listener->socket)
+			break;
+		client_sc = smbdirect_socket_accept(listener->socket, timeo, &arg);
+		if (!client_sc && arg.err == -EINVAL)
+			break;
+		if (!client_sc)
+			continue;
+
+		ksmbd_debug(CONN, "connect success: accepted new connection\n");
+		smb_direct_new_connection(listener, client_sc);
+	}
 
-	smb_direct_post_send_data(sc, NULL, NULL, 0, 0);
+	ksmbd_debug(CONN, "releasing socket\n");
+	return 0;
 }
 
-static void smb_direct_idle_connection_timer(struct work_struct *work)
+static void smb_direct_listener_destroy(struct smb_direct_listener *listener)
 {
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, idle.timer_work.work);
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
+	int ret;
 
-	if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) {
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
+	if (listener->socket)
+		smbdirect_socket_shutdown(listener->socket);
+
+	if (listener->thread) {
+		ret = kthread_stop(listener->thread);
+		if (ret)
+			pr_err("failed to stop forker thread\n");
+		listener->thread = NULL;
 	}
 
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return;
+	if (listener->socket) {
+		smbdirect_socket_release(listener->socket);
+		listener->socket = NULL;
+	}
 
-	/*
-	 * Now use the keepalive timeout (instead of keepalive interval)
-	 * in order to wait for a response
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->keepalive_timeout_msec));
-	queue_work(sc->workqueue, &sc->idle.immediate_work);
+	listener->port = 0;
 }
 
-static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
+static int smb_direct_listen(struct smb_direct_listener *listener,
+			     int port)
 {
-	struct smb_direct_transport *t;
-	struct smbdirect_socket *sc;
+	struct net *net = current->nsproxy->net_ns;
+	struct task_struct *kthread;
+	struct sockaddr_in sin = {
+		.sin_family		= AF_INET,
+		.sin_addr.s_addr	= htonl(INADDR_ANY),
+		.sin_port		= htons(port),
+	};
+	struct smbdirect_socket_parameters init_params = {};
 	struct smbdirect_socket_parameters *sp;
-	struct ksmbd_conn *conn;
-
-	t = kzalloc_obj(*t, KSMBD_DEFAULT_GFP);
-	if (!t)
-		return NULL;
-	sc = &t->socket;
-	smbdirect_socket_init(sc);
-	sp = &sc->parameters;
+	struct smbdirect_socket *sc;
+	u64 port_flags = 0;
+	int ret;
 
-	sc->workqueue = smb_direct_wq;
+	switch (port) {
+	case SMB_DIRECT_PORT_IWARP:
+		/*
+		 * only allow iWarp devices
+		 * for port 5445.
+		 */
+		port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW;
+		break;
+	case SMB_DIRECT_PORT_INFINIBAND:
+		/*
+		 * only allow InfiniBand, RoCEv1 or RoCEv2
+		 * devices for port 445.
+		 *
+		 * (Basically don't allow iWarp devices)
+		 */
+		port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB;
+		break;
+	default:
+		pr_err("unsupported smbdirect port=%d!\n", port);
+		return -ENODEV;
+	}
 
-	INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work);
+	ret = smbdirect_socket_create_kern(net, &sc);
+	if (ret) {
+		pr_err("smbdirect_socket_create_kern() failed: %d %1pe\n",
+		       ret, ERR_PTR(ret));
+		return ret;
+	}
 
+	/*
+	 * Create the initial parameters
+	 */
+	sp = &init_params;
+	sp->flags |= port_flags;
 	sp->negotiate_timeout_msec = SMB_DIRECT_NEGOTIATE_TIMEOUT * 1000;
 	sp->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH;
 	sp->responder_resources = 1;
@@ -434,2612 +434,104 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
 	sp->keepalive_interval_msec = SMB_DIRECT_KEEPALIVE_SEND_INTERVAL * 1000;
 	sp->keepalive_timeout_msec = SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT * 1000;
 
-	sc->rdma.cm_id = cm_id;
-	cm_id->context = sc;
-
-	sc->ib.dev = sc->rdma.cm_id->device;
+	smbdirect_socket_set_logging(sc, NULL,
+				     smb_direct_logging_needed,
+				     smb_direct_logging_vaprintf);
+	ret = smbdirect_socket_set_initial_parameters(sc, sp);
+	if (ret) {
+		pr_err("Failed smbdirect_socket_set_initial_parameters(): %d %1pe\n",
+		       ret, ERR_PTR(ret));
+		goto err;
+	}
+	ret = smbdirect_socket_set_kernel_settings(sc, IB_POLL_WORKQUEUE, KSMBD_DEFAULT_GFP);
+	if (ret) {
+		pr_err("Failed smbdirect_socket_set_kernel_settings(): %d %1pe\n",
+		       ret, ERR_PTR(ret));
+		goto err;
+	}
 
-	INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer);
+	ret = smbdirect_socket_bind(sc, (struct sockaddr *)&sin);
+	if (ret) {
+		pr_err("smbdirect_socket_bind() failed: %d %1pe\n",
+		       ret, ERR_PTR(ret));
+		goto err;
+	}
 
-	conn = ksmbd_conn_alloc();
-	if (!conn)
+	ret = smbdirect_socket_listen(sc, 10);
+	if (ret) {
+		pr_err("Port[%d] smbdirect_socket_listen() failed: %d %1pe\n",
+		       port, ret, ERR_PTR(ret));
 		goto err;
+	}
 
-	down_write(&conn_list_lock);
-	hash_add(conn_list, &conn->hlist, 0);
-	up_write(&conn_list_lock);
+	listener->port = port;
+	listener->socket = sc;
+
+	kthread = kthread_run(smb_direct_listener_kthread_fn,
+			      listener,
+			      "ksmbd-smbdirect-listener-%u", port);
+	if (IS_ERR(kthread)) {
+		ret = PTR_ERR(kthread);
+		pr_err("Can't start ksmbd listen kthread: %d %1pe\n",
+		       ret, ERR_PTR(ret));
+		goto err;
+	}
 
-	conn->transport = KSMBD_TRANS(t);
-	KSMBD_TRANS(t)->conn = conn;
-	KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
-	return t;
+	listener->thread = kthread;
+	return 0;
 err:
-	kfree(t);
-	return NULL;
-}
-
-static void smb_direct_free_transport(struct ksmbd_transport *kt)
-{
-	kfree(SMBD_TRANS(kt));
+	smb_direct_listener_destroy(listener);
+	return ret;
 }
 
-static void free_transport(struct smb_direct_transport *t)
+int ksmbd_rdma_init(void)
 {
-	struct smbdirect_socket *sc = &t->socket;
-	struct smbdirect_recv_io *recvmsg;
-
-	disable_work_sync(&sc->disconnect_work);
-	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING)
-		smb_direct_disconnect_rdma_work(&sc->disconnect_work);
-	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED)
-		wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED);
+	int ret;
 
-	/*
-	 * Wake up all waiters in all wait queues
-	 * in order to notice the broken connection.
-	 *
-	 * Most likely this was already called via
-	 * smb_direct_disconnect_rdma_work(), but call it again...
-	 */
-	smb_direct_disconnect_wake_up_all(sc);
+	smb_direct_ib_listener = smb_direct_iw_listener = (struct smb_direct_listener) {
+		.socket = NULL,
+	};
 
-	disable_work_sync(&sc->connect.work);
-	disable_work_sync(&sc->recv_io.posted.refill_work);
-	disable_delayed_work_sync(&sc->idle.timer_work);
-	disable_work_sync(&sc->idle.immediate_work);
+	ret = smb_direct_listen(&smb_direct_ib_listener,
+				SMB_DIRECT_PORT_INFINIBAND);
+	if (ret) {
+		pr_err("Can't listen on InfiniBand/RoCEv1/RoCEv2: %d\n", ret);
+		goto err;
+	}
 
-	if (sc->rdma.cm_id)
-		rdma_lock_handler(sc->rdma.cm_id);
+	ksmbd_debug(RDMA, "InfiniBand/RoCEv1/RoCEv2 RDMA listener. socket=%p\n",
+		    smb_direct_ib_listener.socket);
 
-	if (sc->ib.qp) {
-		ib_drain_qp(sc->ib.qp);
-		sc->ib.qp = NULL;
-		rdma_destroy_qp(sc->rdma.cm_id);
+	ret = smb_direct_listen(&smb_direct_iw_listener,
+				SMB_DIRECT_PORT_IWARP);
+	if (ret) {
+		pr_err("Can't listen on iWarp: %d\n", ret);
+		goto err;
 	}
 
-	ksmbd_debug(RDMA, "drain the reassembly queue\n");
-	do {
-		unsigned long flags;
-
-		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-		recvmsg = get_first_reassembly(sc);
-		if (recvmsg) {
-			list_del(&recvmsg->list);
-			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-			put_recvmsg(sc, recvmsg);
-		} else {
-			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-		}
-	} while (recvmsg);
-	sc->recv_io.reassembly.data_length = 0;
-
-	if (sc->ib.send_cq)
-		ib_free_cq(sc->ib.send_cq);
-	if (sc->ib.recv_cq)
-		ib_free_cq(sc->ib.recv_cq);
-	if (sc->ib.pd)
-		ib_dealloc_pd(sc->ib.pd);
-	if (sc->rdma.cm_id) {
-		rdma_unlock_handler(sc->rdma.cm_id);
-		rdma_destroy_id(sc->rdma.cm_id);
-	}
+	ksmbd_debug(RDMA, "iWarp RDMA listener. socket=%p\n",
+		    smb_direct_iw_listener.socket);
 
-	smb_direct_destroy_pools(sc);
-	ksmbd_conn_free(KSMBD_TRANS(t)->conn);
+	return 0;
+err:
+	ksmbd_rdma_stop_listening();
+	return ret;
 }
 
-static struct smbdirect_send_io
-*smb_direct_alloc_sendmsg(struct smbdirect_socket *sc)
+void ksmbd_rdma_stop_listening(void)
 {
-	struct smbdirect_send_io *msg;
-
-	msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP);
-	if (!msg)
-		return ERR_PTR(-ENOMEM);
-	msg->socket = sc;
-	INIT_LIST_HEAD(&msg->sibling_list);
-	msg->num_sge = 0;
-	return msg;
+	smb_direct_listener_destroy(&smb_direct_ib_listener);
+	smb_direct_listener_destroy(&smb_direct_iw_listener);
 }
 
-static void smb_direct_free_sendmsg(struct smbdirect_socket *sc,
-				    struct smbdirect_send_io *msg)
+bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
 {
-	int i;
-
-	/*
-	 * The list needs to be empty!
-	 * The caller should take care of it.
-	 */
-	WARN_ON_ONCE(!list_empty(&msg->sibling_list));
-
-	if (msg->num_sge > 0) {
-		ib_dma_unmap_single(sc->ib.dev,
-				    msg->sge[0].addr, msg->sge[0].length,
-				    DMA_TO_DEVICE);
-		for (i = 1; i < msg->num_sge; i++)
-			ib_dma_unmap_page(sc->ib.dev,
-					  msg->sge[i].addr, msg->sge[i].length,
-					  DMA_TO_DEVICE);
-	}
-	mempool_free(msg, sc->send_io.mem.pool);
-}
-
-static int smb_direct_check_recvmsg(struct smbdirect_recv_io *recvmsg)
-{
-	struct smbdirect_socket *sc = recvmsg->socket;
-
-	switch (sc->recv_io.expected) {
-	case SMBDIRECT_EXPECT_DATA_TRANSFER: {
-		struct smbdirect_data_transfer *req =
-			(struct smbdirect_data_transfer *)recvmsg->packet;
-		struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
-				+ le32_to_cpu(req->data_offset));
-		ksmbd_debug(RDMA,
-			    "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
-			    le16_to_cpu(req->credits_granted),
-			    le16_to_cpu(req->credits_requested),
-			    req->data_length, req->remaining_data_length,
-			    hdr->ProtocolId, hdr->Command);
-		return 0;
-	}
-	case SMBDIRECT_EXPECT_NEGOTIATE_REQ: {
-		struct smbdirect_negotiate_req *req =
-			(struct smbdirect_negotiate_req *)recvmsg->packet;
-		ksmbd_debug(RDMA,
-			    "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
-			    le16_to_cpu(req->min_version),
-			    le16_to_cpu(req->max_version),
-			    le16_to_cpu(req->credits_requested),
-			    le32_to_cpu(req->preferred_send_size),
-			    le32_to_cpu(req->max_receive_size),
-			    le32_to_cpu(req->max_fragmented_size));
-		if (le16_to_cpu(req->min_version) > 0x0100 ||
-		    le16_to_cpu(req->max_version) < 0x0100)
-			return -EOPNOTSUPP;
-		if (le16_to_cpu(req->credits_requested) <= 0 ||
-		    le32_to_cpu(req->max_receive_size) <= 128 ||
-		    le32_to_cpu(req->max_fragmented_size) <=
-					128 * 1024)
-			return -ECONNABORTED;
-
-		return 0;
-	}
-	case SMBDIRECT_EXPECT_NEGOTIATE_REP:
-		/* client only */
-		break;
-	}
-
-	/* This is an internal error */
-	return -EINVAL;
-}
-
-static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	struct smbdirect_recv_io *recvmsg;
-	struct smbdirect_socket *sc;
-	struct smbdirect_socket_parameters *sp;
-
-	recvmsg = container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
-	sc = recvmsg->socket;
-	sp = &sc->parameters;
-
-	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
-		put_recvmsg(sc, recvmsg);
-		if (wc->status != IB_WC_WR_FLUSH_ERR) {
-			pr_err("Recv error. status='%s (%d)' opcode=%d\n",
-			       ib_wc_status_msg(wc->status), wc->status,
-			       wc->opcode);
-			smb_direct_disconnect_rdma_connection(sc);
-		}
-		return;
-	}
-
-	ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
-		    ib_wc_status_msg(wc->status), wc->status,
-		    wc->opcode);
-
-	ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
-				   recvmsg->sge.length, DMA_FROM_DEVICE);
-
-	/*
-	 * Reset timer to the keepalive interval in
-	 * order to trigger our next keepalive message.
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->keepalive_interval_msec));
-
-	switch (sc->recv_io.expected) {
-	case SMBDIRECT_EXPECT_NEGOTIATE_REQ:
-		/* see smb_direct_negotiate_recv_done */
-		break;
-	case SMBDIRECT_EXPECT_DATA_TRANSFER: {
-		struct smbdirect_data_transfer *data_transfer =
-			(struct smbdirect_data_transfer *)recvmsg->packet;
-		u32 remaining_data_length, data_offset, data_length;
-		int current_recv_credits;
-		u16 old_recv_credit_target;
-
-		if (wc->byte_len <
-		    offsetof(struct smbdirect_data_transfer, padding)) {
-			put_recvmsg(sc, recvmsg);
-			smb_direct_disconnect_rdma_connection(sc);
-			return;
-		}
-
-		remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length);
-		data_length = le32_to_cpu(data_transfer->data_length);
-		data_offset = le32_to_cpu(data_transfer->data_offset);
-		if (wc->byte_len < data_offset ||
-		    wc->byte_len < (u64)data_offset + data_length) {
-			put_recvmsg(sc, recvmsg);
-			smb_direct_disconnect_rdma_connection(sc);
-			return;
-		}
-		if (remaining_data_length > sp->max_fragmented_recv_size ||
-		    data_length > sp->max_fragmented_recv_size ||
-		    (u64)remaining_data_length + (u64)data_length >
-		    (u64)sp->max_fragmented_recv_size) {
-			put_recvmsg(sc, recvmsg);
-			smb_direct_disconnect_rdma_connection(sc);
-			return;
-		}
-
-		if (data_length) {
-			if (sc->recv_io.reassembly.full_packet_received)
-				recvmsg->first_segment = true;
-
-			if (le32_to_cpu(data_transfer->remaining_data_length))
-				sc->recv_io.reassembly.full_packet_received = false;
-			else
-				sc->recv_io.reassembly.full_packet_received = true;
-		}
-
-		atomic_dec(&sc->recv_io.posted.count);
-		current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count);
-
-		old_recv_credit_target = sc->recv_io.credits.target;
-		sc->recv_io.credits.target =
-				le16_to_cpu(data_transfer->credits_requested);
-		sc->recv_io.credits.target =
-			min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);
-		sc->recv_io.credits.target =
-			max_t(u16, sc->recv_io.credits.target, 1);
-		atomic_add(le16_to_cpu(data_transfer->credits_granted),
-			   &sc->send_io.credits.count);
-
-		if (le16_to_cpu(data_transfer->flags) &
-		    SMBDIRECT_FLAG_RESPONSE_REQUESTED)
-			queue_work(sc->workqueue, &sc->idle.immediate_work);
-
-		if (atomic_read(&sc->send_io.credits.count) > 0)
-			wake_up(&sc->send_io.credits.wait_queue);
-
-		if (data_length) {
-			if (current_recv_credits <= (sc->recv_io.credits.target / 4) ||
-			    sc->recv_io.credits.target > old_recv_credit_target)
-				queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-
-			enqueue_reassembly(sc, recvmsg, (int)data_length);
-			wake_up(&sc->recv_io.reassembly.wait_queue);
-		} else
-			put_recvmsg(sc, recvmsg);
-
-		return;
-	}
-	case SMBDIRECT_EXPECT_NEGOTIATE_REP:
-		/* client only */
-		break;
-	}
-
-	/*
-	 * This is an internal error!
-	 */
-	WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER);
-	put_recvmsg(sc, recvmsg);
-	smb_direct_disconnect_rdma_connection(sc);
-}
-
-static void smb_direct_negotiate_recv_work(struct work_struct *work);
-
-static void smb_direct_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	struct smbdirect_recv_io *recv_io =
-		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
-	struct smbdirect_socket *sc = recv_io->socket;
-	unsigned long flags;
-
-	/*
-	 * reset the common recv_done for later reuse.
-	 */
-	recv_io->cqe.done = recv_done;
-
-	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
-		put_recvmsg(sc, recv_io);
-		if (wc->status != IB_WC_WR_FLUSH_ERR) {
-			pr_err("Negotiate Recv error. status='%s (%d)' opcode=%d\n",
-			       ib_wc_status_msg(wc->status), wc->status,
-			       wc->opcode);
-			smb_direct_disconnect_rdma_connection(sc);
-		}
-		return;
-	}
-
-	ksmbd_debug(RDMA, "Negotiate Recv completed. status='%s (%d)', opcode=%d\n",
-		    ib_wc_status_msg(wc->status), wc->status,
-		    wc->opcode);
-
-	ib_dma_sync_single_for_cpu(sc->ib.dev,
-				   recv_io->sge.addr,
-				   recv_io->sge.length,
-				   DMA_FROM_DEVICE);
-
-	/*
-	 * This is an internal error!
-	 */
-	if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) {
-		put_recvmsg(sc, recv_io);
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
-	}
-
-	/*
-	 * Don't reset timer to the keepalive interval in
-	 * this will be done in smb_direct_negotiate_recv_work.
-	 */
-
-	/*
-	 * Only remember the recv_io if it has enough bytes,
-	 * this gives smb_direct_negotiate_recv_work enough
-	 * information in order to disconnect if it was not
-	 * valid.
-	 */
-	sc->recv_io.reassembly.full_packet_received = true;
-	if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req))
-		enqueue_reassembly(sc, recv_io, 0);
-	else
-		put_recvmsg(sc, recv_io);
-
-	/*
-	 * Some drivers (at least mlx5_ib and irdma in roce mode)
-	 * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED,
-	 * we need to adjust our expectation in that case.
-	 *
-	 * So we defer further processing of the negotiation
-	 * to smb_direct_negotiate_recv_work().
-	 *
-	 * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED
-	 * we queue the work directly otherwise
-	 * smb_direct_cm_handler() will do it, when
-	 * RDMA_CM_EVENT_ESTABLISHED arrived.
-	 */
-	spin_lock_irqsave(&sc->connect.lock, flags);
-	if (!sc->first_error) {
-		INIT_WORK(&sc->connect.work, smb_direct_negotiate_recv_work);
-		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)
-			queue_work(sc->workqueue, &sc->connect.work);
-	}
-	spin_unlock_irqrestore(&sc->connect.lock, flags);
-}
-
-static void smb_direct_negotiate_recv_work(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, connect.work);
-	const struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_recv_io *recv_io;
-
-	if (sc->first_error)
-		return;
-
-	ksmbd_debug(RDMA, "Negotiate Recv Work running\n");
-
-	/*
-	 * Reset timer to the keepalive interval in
-	 * order to trigger our next keepalive message.
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->keepalive_interval_msec));
-
-	/*
-	 * If smb_direct_negotiate_recv_done() detected an
-	 * invalid request we want to disconnect.
-	 */
-	recv_io = get_first_reassembly(sc);
-	if (!recv_io) {
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
-	}
-
-	if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) {
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
-	}
-	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
-	wake_up(&sc->status_wait);
-}
-
-static int smb_direct_post_recv(struct smbdirect_socket *sc,
-				struct smbdirect_recv_io *recvmsg)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct ib_recv_wr wr;
-	int ret;
-
-	recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev,
-					      recvmsg->packet,
-					      sp->max_recv_size,
-					      DMA_FROM_DEVICE);
-	ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr);
-	if (ret)
-		return ret;
-	recvmsg->sge.length = sp->max_recv_size;
-	recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey;
-
-	wr.wr_cqe = &recvmsg->cqe;
-	wr.next = NULL;
-	wr.sg_list = &recvmsg->sge;
-	wr.num_sge = 1;
-
-	ret = ib_post_recv(sc->ib.qp, &wr, NULL);
-	if (ret) {
-		pr_err("Can't post recv: %d\n", ret);
-		ib_dma_unmap_single(sc->ib.dev,
-				    recvmsg->sge.addr, recvmsg->sge.length,
-				    DMA_FROM_DEVICE);
-		recvmsg->sge.length = 0;
-		smb_direct_disconnect_rdma_connection(sc);
-		return ret;
-	}
-	return ret;
-}
-
-static int smb_direct_read(struct ksmbd_transport *t, char *buf,
-			   unsigned int size, int unused)
-{
-	struct smbdirect_recv_io *recvmsg;
-	struct smbdirect_data_transfer *data_transfer;
-	int to_copy, to_read, data_read, offset;
-	u32 data_length, remaining_data_length, data_offset;
-	int rc;
-	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = &st->socket;
-
-again:
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {
-		pr_err("disconnected\n");
-		return -ENOTCONN;
-	}
-
-	/*
-	 * No need to hold the reassembly queue lock all the time as we are
-	 * the only one reading from the front of the queue. The transport
-	 * may add more entries to the back of the queue at the same time
-	 */
-	if (sc->recv_io.reassembly.data_length >= size) {
-		int queue_length;
-		int queue_removed = 0;
-		unsigned long flags;
-
-		/*
-		 * Need to make sure reassembly_data_length is read before
-		 * reading reassembly_queue_length and calling
-		 * get_first_reassembly. This call is lock free
-		 * as we never read at the end of the queue which are being
-		 * updated in SOFTIRQ as more data is received
-		 */
-		virt_rmb();
-		queue_length = sc->recv_io.reassembly.queue_length;
-		data_read = 0;
-		to_read = size;
-		offset = sc->recv_io.reassembly.first_entry_offset;
-		while (data_read < size) {
-			recvmsg = get_first_reassembly(sc);
-			data_transfer = smbdirect_recv_io_payload(recvmsg);
-			data_length = le32_to_cpu(data_transfer->data_length);
-			remaining_data_length =
-				le32_to_cpu(data_transfer->remaining_data_length);
-			data_offset = le32_to_cpu(data_transfer->data_offset);
-
-			/*
-			 * The upper layer expects RFC1002 length at the
-			 * beginning of the payload. Return it to indicate
-			 * the total length of the packet. This minimize the
-			 * change to upper layer packet processing logic. This
-			 * will be eventually remove when an intermediate
-			 * transport layer is added
-			 */
-			if (recvmsg->first_segment && size == 4) {
-				unsigned int rfc1002_len =
-					data_length + remaining_data_length;
-				*((__be32 *)buf) = cpu_to_be32(rfc1002_len);
-				data_read = 4;
-				recvmsg->first_segment = false;
-				ksmbd_debug(RDMA,
-					    "returning rfc1002 length %d\n",
-					    rfc1002_len);
-				goto read_rfc1002_done;
-			}
-
-			to_copy = min_t(int, data_length - offset, to_read);
-			memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
-			       to_copy);
-
-			/* move on to the next buffer? */
-			if (to_copy == data_length - offset) {
-				queue_length--;
-				/*
-				 * No need to lock if we are not at the
-				 * end of the queue
-				 */
-				if (queue_length) {
-					list_del(&recvmsg->list);
-				} else {
-					spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-					list_del(&recvmsg->list);
-					spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-				}
-				queue_removed++;
-				put_recvmsg(sc, recvmsg);
-				offset = 0;
-			} else {
-				offset += to_copy;
-			}
-
-			to_read -= to_copy;
-			data_read += to_copy;
-		}
-
-		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-		sc->recv_io.reassembly.data_length -= data_read;
-		sc->recv_io.reassembly.queue_length -= queue_removed;
-		spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-
-		sc->recv_io.reassembly.first_entry_offset = offset;
-		ksmbd_debug(RDMA,
-			    "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
-			    data_read, sc->recv_io.reassembly.data_length,
-			    sc->recv_io.reassembly.first_entry_offset);
-read_rfc1002_done:
-		return data_read;
-	}
-
-	ksmbd_debug(RDMA, "wait_event on more data\n");
-	rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue,
-				      sc->recv_io.reassembly.data_length >= size ||
-				       sc->status != SMBDIRECT_SOCKET_CONNECTED);
-	if (rc)
-		return -EINTR;
-
-	goto again;
-}
-
-static void smb_direct_post_recv_credits(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, recv_io.posted.refill_work);
-	struct smbdirect_recv_io *recvmsg;
-	int credits = 0;
-	int ret;
-
-	if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) {
-		while (true) {
-			recvmsg = get_free_recvmsg(sc);
-			if (!recvmsg)
-				break;
-
-			recvmsg->first_segment = false;
-
-			ret = smb_direct_post_recv(sc, recvmsg);
-			if (ret) {
-				pr_err("Can't post recv: %d\n", ret);
-				put_recvmsg(sc, recvmsg);
-				break;
-			}
-			credits++;
-
-			atomic_inc(&sc->recv_io.posted.count);
-		}
-	}
-
-	atomic_add(credits, &sc->recv_io.credits.available);
-
-	/*
-	 * If the last send credit is waiting for credits
-	 * it can grant we need to wake it up
-	 */
-	if (credits &&
-	    atomic_read(&sc->send_io.bcredits.count) == 0 &&
-	    atomic_read(&sc->send_io.credits.count) == 0)
-		wake_up(&sc->send_io.credits.wait_queue);
-
-	if (credits)
-		queue_work(sc->workqueue, &sc->idle.immediate_work);
-}
-
-static void send_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	struct smbdirect_send_io *sendmsg, *sibling, *next;
-	struct smbdirect_socket *sc;
-	int lcredits = 0;
-
-	sendmsg = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe);
-	sc = sendmsg->socket;
-
-	ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
-		    ib_wc_status_msg(wc->status), wc->status,
-		    wc->opcode);
-
-	if (unlikely(!(sendmsg->wr.send_flags & IB_SEND_SIGNALED))) {
-		/*
-		 * This happens when smbdirect_send_io is a sibling
-		 * before the final message, it is signaled on
-		 * error anyway, so we need to skip
-		 * smbdirect_connection_free_send_io here,
-		 * otherwise is will destroy the memory
-		 * of the siblings too, which will cause
-		 * use after free problems for the others
-		 * triggered from ib_drain_qp().
-		 */
-		if (wc->status != IB_WC_SUCCESS)
-			goto skip_free;
-
-		/*
-		 * This should not happen!
-		 * But we better just close the
-		 * connection...
-		 */
-		pr_err("unexpected send completion wc->status=%s (%d) wc->opcode=%d\n",
-		       ib_wc_status_msg(wc->status), wc->status, wc->opcode);
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
-	}
-
-	/*
-	 * Free possible siblings and then the main send_io
-	 */
-	list_for_each_entry_safe(sibling, next, &sendmsg->sibling_list, sibling_list) {
-		list_del_init(&sibling->sibling_list);
-		smb_direct_free_sendmsg(sc, sibling);
-		lcredits += 1;
-	}
-	/* Note this frees wc->wr_cqe, but not wc */
-	smb_direct_free_sendmsg(sc, sendmsg);
-	lcredits += 1;
-
-	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
-skip_free:
-		pr_err("Send error. status='%s (%d)', opcode=%d\n",
-		       ib_wc_status_msg(wc->status), wc->status,
-		       wc->opcode);
-		smb_direct_disconnect_rdma_connection(sc);
-		return;
-	}
-
-	atomic_add(lcredits, &sc->send_io.lcredits.count);
-	wake_up(&sc->send_io.lcredits.wait_queue);
-
-	if (atomic_dec_and_test(&sc->send_io.pending.count))
-		wake_up(&sc->send_io.pending.zero_wait_queue);
-}
-
-static int manage_credits_prior_sending(struct smbdirect_socket *sc)
-{
-	int missing;
-	int available;
-	int new_credits;
-
-	if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target)
-		return 0;
-
-	missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count);
-	available = atomic_xchg(&sc->recv_io.credits.available, 0);
-	new_credits = (u16)min3(U16_MAX, missing, available);
-	if (new_credits <= 0) {
-		/*
-		 * If credits are available, but not granted
-		 * we need to re-add them again.
-		 */
-		if (available)
-			atomic_add(available, &sc->recv_io.credits.available);
-		return 0;
-	}
-
-	if (new_credits < available) {
-		/*
-		 * Readd the remaining available again.
-		 */
-		available -= new_credits;
-		atomic_add(available, &sc->recv_io.credits.available);
-	}
-
-	/*
-	 * Remember we granted the credits
-	 */
-	atomic_add(new_credits, &sc->recv_io.credits.count);
-	return new_credits;
-}
-
-static int manage_keep_alive_before_sending(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-
-	if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) {
-		sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT;
-		/*
-		 * Now use the keepalive timeout (instead of keepalive interval)
-		 * in order to wait for a response
-		 */
-		mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-				 msecs_to_jiffies(sp->keepalive_timeout_msec));
-		return 1;
-	}
-	return 0;
-}
-
-static int smb_direct_post_send(struct smbdirect_socket *sc,
-				struct ib_send_wr *wr)
-{
-	int ret;
-
-	atomic_inc(&sc->send_io.pending.count);
-	ret = ib_post_send(sc->ib.qp, wr, NULL);
-	if (ret) {
-		pr_err("failed to post send: %d\n", ret);
-		smb_direct_disconnect_rdma_connection(sc);
-	}
-	return ret;
-}
-
-static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx,
-				     bool need_invalidate_rkey,
-				     unsigned int remote_key)
-{
-	INIT_LIST_HEAD(&send_ctx->msg_list);
-	send_ctx->wr_cnt = 0;
-	send_ctx->need_invalidate_rkey = need_invalidate_rkey;
-	send_ctx->remote_key = remote_key;
-	send_ctx->credit = 0;
-}
-
-static int smb_direct_flush_send_list(struct smbdirect_socket *sc,
-				      struct smbdirect_send_batch *send_ctx,
-				      bool is_last)
-{
-	struct smbdirect_send_io *first, *last;
-	int ret = 0;
-
-	if (list_empty(&send_ctx->msg_list))
-		goto release_credit;
-
-	first = list_first_entry(&send_ctx->msg_list,
-				 struct smbdirect_send_io,
-				 sibling_list);
-	last = list_last_entry(&send_ctx->msg_list,
-			       struct smbdirect_send_io,
-			       sibling_list);
-
-	if (send_ctx->need_invalidate_rkey) {
-		first->wr.opcode = IB_WR_SEND_WITH_INV;
-		first->wr.ex.invalidate_rkey = send_ctx->remote_key;
-		send_ctx->need_invalidate_rkey = false;
-		send_ctx->remote_key = 0;
-	}
-
-	last->wr.send_flags = IB_SEND_SIGNALED;
-	last->wr.wr_cqe = &last->cqe;
-
-	/*
-	 * Remove last from send_ctx->msg_list
-	 * and splice the rest of send_ctx->msg_list
-	 * to last->sibling_list.
-	 *
-	 * send_ctx->msg_list is a valid empty list
-	 * at the end.
-	 */
-	list_del_init(&last->sibling_list);
-	list_splice_tail_init(&send_ctx->msg_list, &last->sibling_list);
-	send_ctx->wr_cnt = 0;
-
-	ret = smb_direct_post_send(sc, &first->wr);
-	if (ret) {
-		struct smbdirect_send_io *sibling, *next;
-
-		list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
-			list_del_init(&sibling->sibling_list);
-			smb_direct_free_sendmsg(sc, sibling);
-		}
-		smb_direct_free_sendmsg(sc, last);
-	}
-
-release_credit:
-	if (is_last && !ret && send_ctx->credit) {
-		atomic_add(send_ctx->credit, &sc->send_io.bcredits.count);
-		send_ctx->credit = 0;
-		wake_up(&sc->send_io.bcredits.wait_queue);
-	}
-
-	return ret;
-}
-
-static int wait_for_credits(struct smbdirect_socket *sc,
-			    wait_queue_head_t *waitq, atomic_t *total_credits,
-			    int needed)
-{
-	int ret;
-
-	do {
-		if (atomic_sub_return(needed, total_credits) >= 0)
-			return 0;
-
-		atomic_add(needed, total_credits);
-		ret = wait_event_interruptible(*waitq,
-					       atomic_read(total_credits) >= needed ||
-					       sc->status != SMBDIRECT_SOCKET_CONNECTED);
-
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			return -ENOTCONN;
-		else if (ret < 0)
-			return ret;
-	} while (true);
-}
-
-static int wait_for_send_bcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *send_ctx)
-{
-	int ret;
-
-	if (send_ctx->credit)
-		return 0;
-
-	ret = wait_for_credits(sc,
-			       &sc->send_io.bcredits.wait_queue,
-			       &sc->send_io.bcredits.count,
-			       1);
-	if (ret)
-		return ret;
-
-	send_ctx->credit = 1;
-	return 0;
-}
-
-static int wait_for_send_lcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *send_ctx)
-{
-	if (send_ctx && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {
-		int ret;
-
-		ret = smb_direct_flush_send_list(sc, send_ctx, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc,
-				&sc->send_io.lcredits.wait_queue,
-				&sc->send_io.lcredits.count,
-				1);
-}
-
-static int wait_for_send_credits(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *send_ctx)
-{
-	int ret;
-
-	if (send_ctx &&
-	    (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
-		ret = smb_direct_flush_send_list(sc, send_ctx, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1);
-}
-
-static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits)
-{
-	return wait_for_credits(sc,
-				&sc->rw_io.credits.wait_queue,
-				&sc->rw_io.credits.count,
-				credits);
-}
-
-static int calc_rw_credits(struct smbdirect_socket *sc,
-			   char *buf, unsigned int len)
-{
-	return DIV_ROUND_UP(get_buf_page_count(buf, len),
-			    sc->rw_io.credits.num_pages);
-}
-
-static int smb_direct_create_header(struct smbdirect_socket *sc,
-				    int size, int remaining_data_length,
-				    int new_credits,
-				    struct smbdirect_send_io **sendmsg_out)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_send_io *sendmsg;
-	struct smbdirect_data_transfer *packet;
-	int header_length;
-	int ret;
-
-	sendmsg = smb_direct_alloc_sendmsg(sc);
-	if (IS_ERR(sendmsg))
-		return PTR_ERR(sendmsg);
-
-	/* Fill in the packet header */
-	packet = (struct smbdirect_data_transfer *)sendmsg->packet;
-	packet->credits_requested = cpu_to_le16(sp->send_credit_target);
-	packet->credits_granted = cpu_to_le16(new_credits);
-
-	packet->flags = 0;
-	if (manage_keep_alive_before_sending(sc))
-		packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
-
-	packet->reserved = 0;
-	if (!size)
-		packet->data_offset = 0;
-	else
-		packet->data_offset = cpu_to_le32(24);
-	packet->data_length = cpu_to_le32(size);
-	packet->remaining_data_length = cpu_to_le32(remaining_data_length);
-	packet->padding = 0;
-
-	ksmbd_debug(RDMA,
-		    "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
-		    le16_to_cpu(packet->credits_requested),
-		    le16_to_cpu(packet->credits_granted),
-		    le32_to_cpu(packet->data_offset),
-		    le32_to_cpu(packet->data_length),
-		    le32_to_cpu(packet->remaining_data_length));
-
-	/* Map the packet to DMA */
-	header_length = sizeof(struct smbdirect_data_transfer);
-	/* If this is a packet without payload, don't send padding */
-	if (!size)
-		header_length =
-			offsetof(struct smbdirect_data_transfer, padding);
-
-	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,
-						 (void *)packet,
-						 header_length,
-						 DMA_TO_DEVICE);
-	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);
-	if (ret) {
-		smb_direct_free_sendmsg(sc, sendmsg);
-		return ret;
-	}
-
-	sendmsg->num_sge = 1;
-	sendmsg->sge[0].length = header_length;
-	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;
-
-	*sendmsg_out = sendmsg;
-	return 0;
-}
-
-static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
-{
-	bool high = is_vmalloc_addr(buf);
-	struct page *page;
-	int offset, len;
-	int i = 0;
-
-	if (size <= 0 || nentries < get_buf_page_count(buf, size))
-		return -EINVAL;
-
-	offset = offset_in_page(buf);
-	buf -= offset;
-	while (size > 0) {
-		len = min_t(int, PAGE_SIZE - offset, size);
-		if (high)
-			page = vmalloc_to_page(buf);
-		else
-			page = kmap_to_page(buf);
-
-		if (!sg_list)
-			return -EINVAL;
-		sg_set_page(sg_list, page, len, offset);
-		sg_list = sg_next(sg_list);
-
-		buf += PAGE_SIZE;
-		size -= len;
-		offset = 0;
-		i++;
-	}
-	return i;
-}
-
-static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
-			      struct scatterlist *sg_list, int nentries,
-			      enum dma_data_direction dir, int *npages)
-{
-	*npages = get_sg_list(buf, size, sg_list, nentries);
-	if (*npages < 0)
-		return -EINVAL;
-	return ib_dma_map_sg(device, sg_list, *npages, dir);
-}
-
-static int post_sendmsg(struct smbdirect_socket *sc,
-			struct smbdirect_send_batch *send_ctx,
-			struct smbdirect_send_io *msg)
-{
-	int i;
-
-	for (i = 0; i < msg->num_sge; i++)
-		ib_dma_sync_single_for_device(sc->ib.dev,
-					      msg->sge[i].addr, msg->sge[i].length,
-					      DMA_TO_DEVICE);
-
-	msg->cqe.done = send_done;
-	msg->wr.opcode = IB_WR_SEND;
-	msg->wr.sg_list = &msg->sge[0];
-	msg->wr.num_sge = msg->num_sge;
-	msg->wr.next = NULL;
-
-	if (send_ctx) {
-		msg->wr.wr_cqe = NULL;
-		msg->wr.send_flags = 0;
-		if (!list_empty(&send_ctx->msg_list)) {
-			struct smbdirect_send_io *last;
-
-			last = list_last_entry(&send_ctx->msg_list,
-					       struct smbdirect_send_io,
-					       sibling_list);
-			last->wr.next = &msg->wr;
-		}
-		list_add_tail(&msg->sibling_list, &send_ctx->msg_list);
-		send_ctx->wr_cnt++;
-		return 0;
-	}
-
-	msg->wr.wr_cqe = &msg->cqe;
-	msg->wr.send_flags = IB_SEND_SIGNALED;
-	return smb_direct_post_send(sc, &msg->wr);
-}
-
-static int smb_direct_post_send_data(struct smbdirect_socket *sc,
-				     struct smbdirect_send_batch *send_ctx,
-				     struct kvec *iov, int niov,
-				     int remaining_data_length)
-{
-	int i, j, ret;
-	struct smbdirect_send_io *msg;
-	int data_length;
-	struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1];
-	struct smbdirect_send_batch _send_ctx;
-	int new_credits;
-
-	if (!send_ctx) {
-		smb_direct_send_ctx_init(&_send_ctx, false, 0);
-		send_ctx = &_send_ctx;
-	}
-
-	ret = wait_for_send_bcredit(sc, send_ctx);
-	if (ret)
-		goto bcredit_failed;
-
-	ret = wait_for_send_lcredit(sc, send_ctx);
-	if (ret)
-		goto lcredit_failed;
-
-	ret = wait_for_send_credits(sc, send_ctx);
-	if (ret)
-		goto credit_failed;
-
-	new_credits = manage_credits_prior_sending(sc);
-	if (new_credits == 0 &&
-	    atomic_read(&sc->send_io.credits.count) == 0 &&
-	    atomic_read(&sc->recv_io.credits.count) == 0) {
-		queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-		ret = wait_event_interruptible(sc->send_io.credits.wait_queue,
-					       atomic_read(&sc->send_io.credits.count) >= 1 ||
-					       atomic_read(&sc->recv_io.credits.available) >= 1 ||
-					       sc->status != SMBDIRECT_SOCKET_CONNECTED);
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			ret = -ENOTCONN;
-		if (ret < 0)
-			goto credit_failed;
-
-		new_credits = manage_credits_prior_sending(sc);
-	}
-
-	data_length = 0;
-	for (i = 0; i < niov; i++)
-		data_length += iov[i].iov_len;
-
-	ret = smb_direct_create_header(sc, data_length, remaining_data_length,
-				       new_credits, &msg);
-	if (ret)
-		goto header_failed;
-
-	for (i = 0; i < niov; i++) {
-		struct ib_sge *sge;
-		int sg_cnt;
-		int npages;
-
-		sg_init_table(sg, SMBDIRECT_SEND_IO_MAX_SGE - 1);
-		sg_cnt = get_mapped_sg_list(sc->ib.dev,
-					    iov[i].iov_base, iov[i].iov_len,
-					    sg, SMBDIRECT_SEND_IO_MAX_SGE - 1,
-					    DMA_TO_DEVICE, &npages);
-		if (sg_cnt <= 0) {
-			pr_err("failed to map buffer\n");
-			ret = -ENOMEM;
-			goto err;
-		} else if (sg_cnt + msg->num_sge > SMBDIRECT_SEND_IO_MAX_SGE) {
-			pr_err("buffer not fitted into sges\n");
-			ret = -E2BIG;
-			ib_dma_unmap_sg(sc->ib.dev, sg, npages,
-					DMA_TO_DEVICE);
-			goto err;
-		}
-
-		for (j = 0; j < sg_cnt; j++) {
-			sge = &msg->sge[msg->num_sge];
-			sge->addr = sg_dma_address(&sg[j]);
-			sge->length = sg_dma_len(&sg[j]);
-			sge->lkey  = sc->ib.pd->local_dma_lkey;
-			msg->num_sge++;
-		}
-	}
-
-	ret = post_sendmsg(sc, send_ctx, msg);
-	if (ret)
-		goto err;
-
-	if (send_ctx == &_send_ctx) {
-		ret = smb_direct_flush_send_list(sc, send_ctx, true);
-		if (ret)
-			goto err;
-	}
-
-	return 0;
-err:
-	smb_direct_free_sendmsg(sc, msg);
-header_failed:
-	atomic_inc(&sc->send_io.credits.count);
-credit_failed:
-	atomic_inc(&sc->send_io.lcredits.count);
-lcredit_failed:
-	atomic_add(send_ctx->credit, &sc->send_io.bcredits.count);
-	send_ctx->credit = 0;
-bcredit_failed:
-	return ret;
-}
-
-static int smb_direct_writev(struct ksmbd_transport *t,
-			     struct kvec *iov, int niovs, int buflen,
-			     bool need_invalidate, unsigned int remote_key)
-{
-	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = &st->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	size_t remaining_data_length;
-	size_t iov_idx;
-	size_t iov_ofs;
-	size_t max_iov_size = sp->max_send_size -
-			sizeof(struct smbdirect_data_transfer);
-	int ret;
-	struct smbdirect_send_batch send_ctx;
-	int error = 0;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return -ENOTCONN;
-
-	//FIXME: skip RFC1002 header..
-	if (WARN_ON_ONCE(niovs <= 1 || iov[0].iov_len != 4))
-		return -EINVAL;
-	buflen -= 4;
-	iov_idx = 1;
-	iov_ofs = 0;
-
-	remaining_data_length = buflen;
-	ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
-
-	smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key);
-	while (remaining_data_length) {
-		struct kvec vecs[SMBDIRECT_SEND_IO_MAX_SGE - 1]; /* minus smbdirect hdr */
-		size_t possible_bytes = max_iov_size;
-		size_t possible_vecs;
-		size_t bytes = 0;
-		size_t nvecs = 0;
-
-		/*
-		 * For the last message remaining_data_length should be
-		 * have been 0 already!
-		 */
-		if (WARN_ON_ONCE(iov_idx >= niovs)) {
-			error = -EINVAL;
-			goto done;
-		}
-
-		/*
-		 * We have 2 factors which limit the arguments we pass
-		 * to smb_direct_post_send_data():
-		 *
-		 * 1. The number of supported sges for the send,
-		 *    while one is reserved for the smbdirect header.
-		 *    And we currently need one SGE per page.
-		 * 2. The number of negotiated payload bytes per send.
-		 */
-		possible_vecs = min_t(size_t, ARRAY_SIZE(vecs), niovs - iov_idx);
-
-		while (iov_idx < niovs && possible_vecs && possible_bytes) {
-			struct kvec *v = &vecs[nvecs];
-			int page_count;
-
-			v->iov_base = ((u8 *)iov[iov_idx].iov_base) + iov_ofs;
-			v->iov_len = min_t(size_t,
-					   iov[iov_idx].iov_len - iov_ofs,
-					   possible_bytes);
-			page_count = get_buf_page_count(v->iov_base, v->iov_len);
-			if (page_count > possible_vecs) {
-				/*
-				 * If the number of pages in the buffer
-				 * is to much (because we currently require
-				 * one SGE per page), we need to limit the
-				 * length.
-				 *
-				 * We know possible_vecs is at least 1,
-				 * so we always keep the first page.
-				 *
-				 * We need to calculate the number extra
-				 * pages (epages) we can also keep.
-				 *
-				 * We calculate the number of bytes in the
-				 * first page (fplen), this should never be
-				 * larger than v->iov_len because page_count is
-				 * at least 2, but adding a limitation feels
-				 * better.
-				 *
-				 * Then we calculate the number of bytes (elen)
-				 * we can keep for the extra pages.
-				 */
-				size_t epages = possible_vecs - 1;
-				size_t fpofs = offset_in_page(v->iov_base);
-				size_t fplen = min_t(size_t, PAGE_SIZE - fpofs, v->iov_len);
-				size_t elen = min_t(size_t, v->iov_len - fplen, epages*PAGE_SIZE);
-
-				v->iov_len = fplen + elen;
-				page_count = get_buf_page_count(v->iov_base, v->iov_len);
-				if (WARN_ON_ONCE(page_count > possible_vecs)) {
-					/*
-					 * Something went wrong in the above
-					 * logic...
-					 */
-					error = -EINVAL;
-					goto done;
-				}
-			}
-			possible_vecs -= page_count;
-			nvecs += 1;
-			possible_bytes -= v->iov_len;
-			bytes += v->iov_len;
-
-			iov_ofs += v->iov_len;
-			if (iov_ofs >= iov[iov_idx].iov_len) {
-				iov_idx += 1;
-				iov_ofs = 0;
-			}
-		}
-
-		remaining_data_length -= bytes;
-
-		ret = smb_direct_post_send_data(sc, &send_ctx,
-						vecs, nvecs,
-						remaining_data_length);
-		if (unlikely(ret)) {
-			error = ret;
-			goto done;
-		}
-	}
-
-done:
-	ret = smb_direct_flush_send_list(sc, &send_ctx, true);
-	if (unlikely(!ret && error))
-		ret = error;
-
-	/*
-	 * As an optimization, we don't wait for individual I/O to finish
-	 * before sending the next one.
-	 * Send them all and wait for pending send count to get to 0
-	 * that means all the I/Os have been out and we are good to return
-	 */
-
-	wait_event(sc->send_io.pending.zero_wait_queue,
-		   atomic_read(&sc->send_io.pending.count) == 0 ||
-		   sc->status != SMBDIRECT_SOCKET_CONNECTED);
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0)
-		ret = -ENOTCONN;
-
-	return ret;
-}
-
-static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t,
-					struct smbdirect_rw_io *msg,
-					enum dma_data_direction dir)
-{
-	struct smbdirect_socket *sc = &t->socket;
-
-	rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,
-			    msg->sgt.sgl, msg->sgt.nents, dir);
-	sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
-	kfree(msg);
-}
-
-static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
-			    enum dma_data_direction dir)
-{
-	struct smbdirect_rw_io *msg =
-		container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe);
-	struct smbdirect_socket *sc = msg->socket;
-
-	if (wc->status != IB_WC_SUCCESS) {
-		msg->error = -EIO;
-		pr_err("read/write error. opcode = %d, status = %s(%d)\n",
-		       wc->opcode, ib_wc_status_msg(wc->status), wc->status);
-		if (wc->status != IB_WC_WR_FLUSH_ERR)
-			smb_direct_disconnect_rdma_connection(sc);
-	}
-
-	complete(msg->completion);
-}
-
-static void read_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	read_write_done(cq, wc, DMA_FROM_DEVICE);
-}
-
-static void write_done(struct ib_cq *cq, struct ib_wc *wc)
-{
-	read_write_done(cq, wc, DMA_TO_DEVICE);
-}
-
-static int smb_direct_rdma_xmit(struct smb_direct_transport *t,
-				void *buf, int buf_len,
-				struct smbdirect_buffer_descriptor_v1 *desc,
-				unsigned int desc_len,
-				bool is_read)
-{
-	struct smbdirect_socket *sc = &t->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_rw_io *msg, *next_msg;
-	int i, ret;
-	DECLARE_COMPLETION_ONSTACK(completion);
-	struct ib_send_wr *first_wr;
-	LIST_HEAD(msg_list);
-	char *desc_buf;
-	int credits_needed;
-	unsigned int desc_buf_len, desc_num = 0;
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return -ENOTCONN;
-
-	if (buf_len > sp->max_read_write_size)
-		return -EINVAL;
-
-	/* calculate needed credits */
-	credits_needed = 0;
-	desc_buf = buf;
-	for (i = 0; i < desc_len / sizeof(*desc); i++) {
-		if (!buf_len)
-			break;
-
-		desc_buf_len = le32_to_cpu(desc[i].length);
-		if (!desc_buf_len)
-			return -EINVAL;
-
-		if (desc_buf_len > buf_len) {
-			desc_buf_len = buf_len;
-			desc[i].length = cpu_to_le32(desc_buf_len);
-			buf_len = 0;
-		}
-
-		credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len);
-		desc_buf += desc_buf_len;
-		buf_len -= desc_buf_len;
-		desc_num++;
-	}
-
-	ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n",
-		    str_read_write(is_read), buf_len, credits_needed);
-
-	ret = wait_for_rw_credits(sc, credits_needed);
-	if (ret < 0)
-		return ret;
-
-	/* build rdma_rw_ctx for each descriptor */
-	desc_buf = buf;
-	for (i = 0; i < desc_num; i++) {
-		msg = kzalloc_flex(*msg, sg_list, SG_CHUNK_SIZE,
-				   KSMBD_DEFAULT_GFP);
-		if (!msg) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		desc_buf_len = le32_to_cpu(desc[i].length);
-
-		msg->socket = sc;
-		msg->cqe.done = is_read ? read_done : write_done;
-		msg->completion = &completion;
-
-		msg->sgt.sgl = &msg->sg_list[0];
-		ret = sg_alloc_table_chained(&msg->sgt,
-					     get_buf_page_count(desc_buf, desc_buf_len),
-					     msg->sg_list, SG_CHUNK_SIZE);
-		if (ret) {
-			ret = -ENOMEM;
-			goto free_msg;
-		}
-
-		ret = get_sg_list(desc_buf, desc_buf_len,
-				  msg->sgt.sgl, msg->sgt.orig_nents);
-		if (ret < 0)
-			goto free_table;
-
-		ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,
-				       msg->sgt.sgl,
-				       get_buf_page_count(desc_buf, desc_buf_len),
-				       0,
-				       le64_to_cpu(desc[i].offset),
-				       le32_to_cpu(desc[i].token),
-				       is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
-		if (ret < 0) {
-			pr_err("failed to init rdma_rw_ctx: %d\n", ret);
-			goto free_table;
-		}
-
-		list_add_tail(&msg->list, &msg_list);
-		desc_buf += desc_buf_len;
-	}
-
-	/* concatenate work requests of rdma_rw_ctxs */
-	first_wr = NULL;
-	list_for_each_entry_reverse(msg, &msg_list, list) {
-		first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,
-					   &msg->cqe, first_wr);
-	}
-
-	ret = ib_post_send(sc->ib.qp, first_wr, NULL);
-	if (ret) {
-		pr_err("failed to post send wr for RDMA R/W: %d\n", ret);
-		goto out;
-	}
-
-	msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list);
-	wait_for_completion(&completion);
-	ret = msg->error;
-out:
-	list_for_each_entry_safe(msg, next_msg, &msg_list, list) {
-		list_del(&msg->list);
-		smb_direct_free_rdma_rw_msg(t, msg,
-					    is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	}
-	atomic_add(credits_needed, &sc->rw_io.credits.count);
-	wake_up(&sc->rw_io.credits.wait_queue);
-	return ret;
-
-free_table:
-	sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
-free_msg:
-	kfree(msg);
-	goto out;
-}
-
-static int smb_direct_rdma_write(struct ksmbd_transport *t,
-				 void *buf, unsigned int buflen,
-				 struct smbdirect_buffer_descriptor_v1 *desc,
-				 unsigned int desc_len)
-{
-	return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen,
-				    desc, desc_len, false);
-}
-
-static int smb_direct_rdma_read(struct ksmbd_transport *t,
-				void *buf, unsigned int buflen,
-				struct smbdirect_buffer_descriptor_v1 *desc,
-				unsigned int desc_len)
-{
-	return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen,
-				    desc, desc_len, true);
-}
-
-static void smb_direct_disconnect(struct ksmbd_transport *t)
-{
-	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = &st->socket;
-
-	ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id);
-
-	free_transport(st);
-}
-
-static void smb_direct_shutdown(struct ksmbd_transport *t)
-{
-	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = &st->socket;
-
-	ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id);
-
-	smb_direct_disconnect_rdma_work(&sc->disconnect_work);
-}
-
-static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
-				 struct rdma_cm_event *event)
-{
-	struct smbdirect_socket *sc = cm_id->context;
-	unsigned long flags;
-
-	ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
-		    cm_id, rdma_event_msg(event->event), event->event);
-
-	switch (event->event) {
-	case RDMA_CM_EVENT_ESTABLISHED: {
-		/*
-		 * Some drivers (at least mlx5_ib and irdma in roce mode)
-		 * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED,
-		 * we need to adjust our expectation in that case.
-		 *
-		 * If smb_direct_negotiate_recv_done was called first
-		 * it initialized sc->connect.work only for us to
-		 * start, so that we turned into
-		 * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before
-		 * smb_direct_negotiate_recv_work() runs.
-		 *
-		 * If smb_direct_negotiate_recv_done didn't happen
-		 * yet. sc->connect.work is still be disabled and
-		 * queue_work() is a no-op.
-		 */
-		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING))
-			break;
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
-		spin_lock_irqsave(&sc->connect.lock, flags);
-		if (!sc->first_error)
-			queue_work(sc->workqueue, &sc->connect.work);
-		spin_unlock_irqrestore(&sc->connect.lock, flags);
-		wake_up(&sc->status_wait);
-		break;
-	}
-	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-	case RDMA_CM_EVENT_DISCONNECTED: {
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		smb_direct_disconnect_rdma_work(&sc->disconnect_work);
-		if (sc->ib.qp)
-			ib_drain_qp(sc->ib.qp);
-		break;
-	}
-	case RDMA_CM_EVENT_CONNECT_ERROR: {
-		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;
-		smb_direct_disconnect_rdma_work(&sc->disconnect_work);
-		break;
-	}
-	default:
-		pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
-		       cm_id, rdma_event_msg(event->event),
-		       event->event);
-		break;
-	}
-	return 0;
-}
-
-static void smb_direct_qpair_handler(struct ib_event *event, void *context)
-{
-	struct smbdirect_socket *sc = context;
-
-	ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
-		    sc->rdma.cm_id, ib_event_msg(event->event), event->event);
-
-	switch (event->event) {
-	case IB_EVENT_CQ_ERR:
-	case IB_EVENT_QP_FATAL:
-		smb_direct_disconnect_rdma_connection(sc);
-		break;
-	default:
-		break;
-	}
-}
-
-static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc,
-					      int failed)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_send_io *sendmsg;
-	struct smbdirect_negotiate_resp *resp;
-	int ret;
-
-	sendmsg = smb_direct_alloc_sendmsg(sc);
-	if (IS_ERR(sendmsg))
-		return -ENOMEM;
-
-	resp = (struct smbdirect_negotiate_resp *)sendmsg->packet;
-	if (failed) {
-		memset(resp, 0, sizeof(*resp));
-		resp->min_version = SMB_DIRECT_VERSION_LE;
-		resp->max_version = SMB_DIRECT_VERSION_LE;
-		resp->status = STATUS_NOT_SUPPORTED;
-
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;
-	} else {
-		resp->status = STATUS_SUCCESS;
-		resp->min_version = SMB_DIRECT_VERSION_LE;
-		resp->max_version = SMB_DIRECT_VERSION_LE;
-		resp->negotiated_version = SMB_DIRECT_VERSION_LE;
-		resp->reserved = 0;
-		resp->credits_requested =
-				cpu_to_le16(sp->send_credit_target);
-		resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc));
-		resp->max_readwrite_size = cpu_to_le32(sp->max_read_write_size);
-		resp->preferred_send_size = cpu_to_le32(sp->max_send_size);
-		resp->max_receive_size = cpu_to_le32(sp->max_recv_size);
-		resp->max_fragmented_size =
-				cpu_to_le32(sp->max_fragmented_recv_size);
-
-		atomic_set(&sc->send_io.bcredits.count, 1);
-		sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;
-		sc->status = SMBDIRECT_SOCKET_CONNECTED;
-	}
-
-	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,
-						 (void *)resp, sizeof(*resp),
-						 DMA_TO_DEVICE);
-	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);
-	if (ret) {
-		smb_direct_free_sendmsg(sc, sendmsg);
-		return ret;
-	}
-
-	sendmsg->num_sge = 1;
-	sendmsg->sge[0].length = sizeof(*resp);
-	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;
-
-	ret = post_sendmsg(sc, NULL, sendmsg);
-	if (ret) {
-		smb_direct_free_sendmsg(sc, sendmsg);
-		return ret;
-	}
-
-	wait_event(sc->send_io.pending.zero_wait_queue,
-		   atomic_read(&sc->send_io.pending.count) == 0 ||
-		   sc->status != SMBDIRECT_SOCKET_CONNECTED);
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return -ENOTCONN;
-
-	return 0;
-}
-
-static int smb_direct_accept_client(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct rdma_conn_param conn_param;
-	__be32 ird_ord_hdr[2];
-	int ret;
-
-	/*
-	 * smb_direct_handle_connect_request()
-	 * already negotiated sp->initiator_depth
-	 * and sp->responder_resources
-	 */
-	memset(&conn_param, 0, sizeof(conn_param));
-	conn_param.initiator_depth = sp->initiator_depth;
-	conn_param.responder_resources = sp->responder_resources;
-
-	if (sc->rdma.legacy_iwarp) {
-		ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources);
-		ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth);
-		conn_param.private_data = ird_ord_hdr;
-		conn_param.private_data_len = sizeof(ird_ord_hdr);
-	} else {
-		conn_param.private_data = NULL;
-		conn_param.private_data_len = 0;
-	}
-	conn_param.retry_count = SMB_DIRECT_CM_RETRY;
-	conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
-	conn_param.flow_control = 0;
-
-	/*
-	 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
-	 * so that the timer will cause a disconnect.
-	 */
-	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
-			 msecs_to_jiffies(sp->negotiate_timeout_msec));
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED);
-	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;
-	ret = rdma_accept(sc->rdma.cm_id, &conn_param);
-	if (ret) {
-		pr_err("error at rdma_accept: %d\n", ret);
-		return ret;
-	}
-	return 0;
-}
-
-static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *recvmsg;
-	bool recv_posted = false;
-	int ret;
-
-	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);
-	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;
-
-	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ;
-
-	recvmsg = get_free_recvmsg(sc);
-	if (!recvmsg)
-		return -ENOMEM;
-	recvmsg->cqe.done = smb_direct_negotiate_recv_done;
-
-	ret = smb_direct_post_recv(sc, recvmsg);
-	if (ret) {
-		pr_err("Can't post recv: %d\n", ret);
-		goto out_err;
-	}
-	recv_posted = true;
-
-	ret = smb_direct_accept_client(sc);
-	if (ret) {
-		pr_err("Can't accept client\n");
-		goto out_err;
-	}
-
-	return 0;
-out_err:
-	/*
-	 * If the recv was never posted, return it to the free list.
-	 * If it was posted, leave it alone so disconnect teardown can
-	 * drain the QP and complete it (flush) and the completion path
-	 * will unmap it exactly once.
-	 */
-	if (!recv_posted)
-		put_recvmsg(sc, recvmsg);
-	return ret;
-}
-
-static int smb_direct_init_params(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int max_send_sges;
-	unsigned int maxpages;
-
-	/* need 3 more sge. because a SMB_DIRECT header, SMB2 header,
-	 * SMB2 response could be mapped.
-	 */
-	max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3;
-	if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) {
-		pr_err("max_send_size %d is too large\n", sp->max_send_size);
-		return -EINVAL;
-	}
-
-	atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target);
-
-	maxpages = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE);
-	sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev,
-						  sc->rdma.cm_id->port_num,
-						  maxpages);
-	sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max);
-	/* add one extra in order to handle unaligned pages */
-	sc->rw_io.credits.max += 1;
-
-	sc->recv_io.credits.target = 1;
-
-	atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max);
-
-	return 0;
-}
-
-static void smb_direct_destroy_pools(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *recvmsg;
-
-	while ((recvmsg = get_free_recvmsg(sc)))
-		mempool_free(recvmsg, sc->recv_io.mem.pool);
-
-	mempool_destroy(sc->recv_io.mem.pool);
-	sc->recv_io.mem.pool = NULL;
-
-	kmem_cache_destroy(sc->recv_io.mem.cache);
-	sc->recv_io.mem.cache = NULL;
-
-	mempool_destroy(sc->send_io.mem.pool);
-	sc->send_io.mem.pool = NULL;
-
-	kmem_cache_destroy(sc->send_io.mem.cache);
-	sc->send_io.mem.cache = NULL;
-}
-
-static int smb_direct_create_pools(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	char name[80];
-	int i;
-	struct smbdirect_recv_io *recvmsg;
-
-	snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc);
-	sc->send_io.mem.cache = kmem_cache_create(name,
-					     sizeof(struct smbdirect_send_io) +
-					      sizeof(struct smbdirect_negotiate_resp),
-					     0, SLAB_HWCACHE_ALIGN, NULL);
-	if (!sc->send_io.mem.cache)
-		return -ENOMEM;
-
-	sc->send_io.mem.pool = mempool_create(sp->send_credit_target,
-					    mempool_alloc_slab, mempool_free_slab,
-					    sc->send_io.mem.cache);
-	if (!sc->send_io.mem.pool)
-		goto err;
-
-	snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc);
-	sc->recv_io.mem.cache = kmem_cache_create(name,
-					     sizeof(struct smbdirect_recv_io) +
-					     sp->max_recv_size,
-					     0, SLAB_HWCACHE_ALIGN, NULL);
-	if (!sc->recv_io.mem.cache)
-		goto err;
-
-	sc->recv_io.mem.pool =
-		mempool_create(sp->recv_credit_max, mempool_alloc_slab,
-			       mempool_free_slab, sc->recv_io.mem.cache);
-	if (!sc->recv_io.mem.pool)
-		goto err;
-
-	for (i = 0; i < sp->recv_credit_max; i++) {
-		recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP);
-		if (!recvmsg)
-			goto err;
-		recvmsg->socket = sc;
-		recvmsg->sge.length = 0;
-		list_add(&recvmsg->list, &sc->recv_io.free.list);
-	}
-
-	return 0;
-err:
-	smb_direct_destroy_pools(sc);
-	return -ENOMEM;
-}
-
-static u32 smb_direct_rdma_rw_send_wrs(struct ib_device *dev, const struct ib_qp_init_attr *attr)
-{
-	/*
-	 * This could be split out of rdma_rw_init_qp()
-	 * and be a helper function next to rdma_rw_mr_factor()
-	 *
-	 * We can't check unlikely(rdma_rw_force_mr) here,
-	 * but that is most likely 0 anyway.
-	 */
-	u32 factor;
-
-	WARN_ON_ONCE(attr->port_num == 0);
-
-	/*
-	 * Each context needs at least one RDMA READ or WRITE WR.
-	 *
-	 * For some hardware we might need more, eventually we should ask the
-	 * HCA driver for a multiplier here.
-	 */
-	factor = 1;
-
-	/*
-	 * If the device needs MRs to perform RDMA READ or WRITE operations,
-	 * we'll need two additional MRs for the registrations and the
-	 * invalidation.
-	 */
-	if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd)
-		factor += 2;	/* inv + reg */
-
-	return factor * attr->cap.max_rdma_ctxs;
-}
-
-static int smb_direct_create_qpair(struct smbdirect_socket *sc)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int ret;
-	struct ib_qp_cap qp_cap;
-	struct ib_qp_init_attr qp_attr;
-	u32 max_send_wr;
-	u32 rdma_send_wr;
-
-	/*
-	 * Note that {rdma,ib}_create_qp() will call
-	 * rdma_rw_init_qp() if cap->max_rdma_ctxs is not 0.
-	 * It will adjust cap->max_send_wr to the required
-	 * number of additional WRs for the RDMA RW operations.
-	 * It will cap cap->max_send_wr to the device limit.
-	 *
-	 * +1 for ib_drain_qp
-	 */
-	qp_cap.max_send_wr = sp->send_credit_target + 1;
-	qp_cap.max_recv_wr = sp->recv_credit_max + 1;
-	qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
-	qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
-	qp_cap.max_inline_data = 0;
-	qp_cap.max_rdma_ctxs = sc->rw_io.credits.max;
-
-	/*
-	 * Find out the number of max_send_wr
-	 * after rdma_rw_init_qp() adjusted it.
-	 *
-	 * We only do it on a temporary variable,
-	 * as rdma_create_qp() will trigger
-	 * rdma_rw_init_qp() again.
-	 */
-	memset(&qp_attr, 0, sizeof(qp_attr));
-	qp_attr.cap = qp_cap;
-	qp_attr.port_num = sc->rdma.cm_id->port_num;
-	rdma_send_wr = smb_direct_rdma_rw_send_wrs(sc->ib.dev, &qp_attr);
-	max_send_wr = qp_cap.max_send_wr + rdma_send_wr;
-
-	if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe ||
-	    qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) {
-		pr_err("Possible CQE overrun: max_send_wr %d\n",
-		       qp_cap.max_send_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
-		       IB_DEVICE_NAME_MAX,
-		       sc->ib.dev->name,
-		       sc->ib.dev->attrs.max_cqe,
-		       sc->ib.dev->attrs.max_qp_wr);
-		pr_err("consider lowering send_credit_target = %d\n",
-		       sp->send_credit_target);
-		return -EINVAL;
-	}
-
-	if (qp_cap.max_rdma_ctxs &&
-	    (max_send_wr >= sc->ib.dev->attrs.max_cqe ||
-	     max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) {
-		pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n",
-		       rdma_send_wr, qp_cap.max_send_wr, max_send_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
-		       IB_DEVICE_NAME_MAX,
-		       sc->ib.dev->name,
-		       sc->ib.dev->attrs.max_cqe,
-		       sc->ib.dev->attrs.max_qp_wr);
-		pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n",
-		       sp->send_credit_target, qp_cap.max_rdma_ctxs);
-		return -EINVAL;
-	}
-
-	if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe ||
-	    qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) {
-		pr_err("Possible CQE overrun: max_recv_wr %d\n",
-		       qp_cap.max_recv_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
-		       IB_DEVICE_NAME_MAX,
-		       sc->ib.dev->name,
-		       sc->ib.dev->attrs.max_cqe,
-		       sc->ib.dev->attrs.max_qp_wr);
-		pr_err("consider lowering receive_credit_max = %d\n",
-		       sp->recv_credit_max);
-		return -EINVAL;
-	}
-
-	if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge ||
-	    qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) {
-		pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
-		       IB_DEVICE_NAME_MAX,
-		       sc->ib.dev->name,
-		       sc->ib.dev->attrs.max_send_sge,
-		       sc->ib.dev->attrs.max_recv_sge);
-		return -EINVAL;
-	}
-
-	sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
-	if (IS_ERR(sc->ib.pd)) {
-		pr_err("Can't create RDMA PD\n");
-		ret = PTR_ERR(sc->ib.pd);
-		sc->ib.pd = NULL;
-		return ret;
-	}
-
-	sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc,
-					 max_send_wr,
-					 IB_POLL_WORKQUEUE);
-	if (IS_ERR(sc->ib.send_cq)) {
-		pr_err("Can't create RDMA send CQ\n");
-		ret = PTR_ERR(sc->ib.send_cq);
-		sc->ib.send_cq = NULL;
-		goto err;
-	}
-
-	sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc,
-					 qp_cap.max_recv_wr,
-					 IB_POLL_WORKQUEUE);
-	if (IS_ERR(sc->ib.recv_cq)) {
-		pr_err("Can't create RDMA recv CQ\n");
-		ret = PTR_ERR(sc->ib.recv_cq);
-		sc->ib.recv_cq = NULL;
-		goto err;
-	}
-
-	/*
-	 * We reset completely here!
-	 * As the above use was just temporary
-	 * to calc max_send_wr and rdma_send_wr.
-	 *
-	 * rdma_create_qp() will trigger rdma_rw_init_qp()
-	 * again if max_rdma_ctxs is not 0.
-	 */
-	memset(&qp_attr, 0, sizeof(qp_attr));
-	qp_attr.event_handler = smb_direct_qpair_handler;
-	qp_attr.qp_context = sc;
-	qp_attr.cap = qp_cap;
-	qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
-	qp_attr.qp_type = IB_QPT_RC;
-	qp_attr.send_cq = sc->ib.send_cq;
-	qp_attr.recv_cq = sc->ib.recv_cq;
-	qp_attr.port_num = ~0;
-
-	ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);
-	if (ret) {
-		pr_err("Can't create RDMA QP: %d\n", ret);
-		goto err;
-	}
-
-	sc->ib.qp = sc->rdma.cm_id->qp;
-	sc->rdma.cm_id->event_handler = smb_direct_cm_handler;
-
-	return 0;
-err:
-	if (sc->ib.qp) {
-		sc->ib.qp = NULL;
-		rdma_destroy_qp(sc->rdma.cm_id);
-	}
-	if (sc->ib.recv_cq) {
-		ib_destroy_cq(sc->ib.recv_cq);
-		sc->ib.recv_cq = NULL;
-	}
-	if (sc->ib.send_cq) {
-		ib_destroy_cq(sc->ib.send_cq);
-		sc->ib.send_cq = NULL;
-	}
-	if (sc->ib.pd) {
-		ib_dealloc_pd(sc->ib.pd);
-		sc->ib.pd = NULL;
-	}
-	return ret;
-}
-
-static int smb_direct_prepare(struct ksmbd_transport *t)
-{
-	struct smb_direct_transport *st = SMBD_TRANS(t);
-	struct smbdirect_socket *sc = &st->socket;
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	struct smbdirect_recv_io *recvmsg;
-	struct smbdirect_negotiate_req *req;
-	unsigned long flags;
-	int ret;
-
-	/*
-	 * We are waiting to pass the following states:
-	 *
-	 * SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED
-	 * SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING
-	 * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED
-	 *
-	 * To finally get to SMBDIRECT_SOCKET_NEGOTIATE_RUNNING
-	 * in order to continue below.
-	 *
-	 * Everything else is unexpected and an error.
-	 */
-	ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n");
-	ret = wait_event_interruptible_timeout(sc->status_wait,
-					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED &&
-					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING &&
-					sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED,
-					msecs_to_jiffies(sp->negotiate_timeout_msec));
-	if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)
-		return ret < 0 ? ret : -ETIMEDOUT;
-
-	recvmsg = get_first_reassembly(sc);
-	if (!recvmsg)
-		return -ECONNABORTED;
-
-	ret = smb_direct_check_recvmsg(recvmsg);
-	if (ret)
-		goto put;
-
-	req = (struct smbdirect_negotiate_req *)recvmsg->packet;
-	sp->max_recv_size = min_t(u32, sp->max_recv_size,
-				  le32_to_cpu(req->preferred_send_size));
-	sp->max_send_size = min_t(u32, sp->max_send_size,
-				  le32_to_cpu(req->max_receive_size));
-	sp->max_fragmented_send_size =
-		le32_to_cpu(req->max_fragmented_size);
-	/*
-	 * The maximum fragmented upper-layer payload receive size supported
-	 *
-	 * Assume max_payload_per_credit is
-	 * smb_direct_receive_credit_max - 24 = 1340
-	 *
-	 * The maximum number would be
-	 * smb_direct_receive_credit_max * max_payload_per_credit
-	 *
-	 *                       1340 * 255 = 341700 (0x536C4)
-	 *
-	 * The minimum value from the spec is 131072 (0x20000)
-	 *
-	 * For now we use the logic we used before:
-	 *                 (1364 * 255) / 2 = 173910 (0x2A756)
-	 *
-	 * We need to adjust this here in case the peer
-	 * lowered sp->max_recv_size.
-	 *
-	 * TODO: instead of adjusting max_fragmented_recv_size
-	 * we should adjust the number of available buffers,
-	 * but for now we keep the current logic.
-	 */
-	sp->max_fragmented_recv_size =
-		(sp->recv_credit_max * sp->max_recv_size) / 2;
-	sc->recv_io.credits.target = le16_to_cpu(req->credits_requested);
-	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);
-	sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1);
-
-put:
-	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);
-	sc->recv_io.reassembly.queue_length--;
-	list_del(&recvmsg->list);
-	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);
-	put_recvmsg(sc, recvmsg);
-
-	if (ret == -ECONNABORTED)
-		return ret;
-
-	if (ret)
-		goto respond;
-
-	/*
-	 * We negotiated with success, so we need to refill the recv queue.
-	 * We do that with sc->idle.immediate_work still being disabled
-	 * via smbdirect_socket_init(), so that queue_work(sc->workqueue,
-	 * &sc->idle.immediate_work) in smb_direct_post_recv_credits()
-	 * is a no-op.
-	 *
-	 * The message that grants the credits to the client is
-	 * the negotiate response.
-	 */
-	INIT_WORK(&sc->recv_io.posted.refill_work, smb_direct_post_recv_credits);
-	smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work);
-	if (unlikely(sc->first_error))
-		return sc->first_error;
-	INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work);
-
-respond:
-	ret = smb_direct_send_negotiate_response(sc, ret);
-
-	return ret;
-}
-
-static int smb_direct_connect(struct smbdirect_socket *sc)
-{
-	struct smbdirect_recv_io *recv_io;
-	int ret;
-
-	ret = smb_direct_init_params(sc);
-	if (ret) {
-		pr_err("Can't configure RDMA parameters\n");
-		return ret;
-	}
-
-	ret = smb_direct_create_pools(sc);
-	if (ret) {
-		pr_err("Can't init RDMA pool: %d\n", ret);
-		return ret;
-	}
-
-	list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
-		recv_io->cqe.done = recv_done;
-
-	ret = smb_direct_create_qpair(sc);
-	if (ret) {
-		pr_err("Can't accept RDMA client: %d\n", ret);
-		return ret;
-	}
-
-	ret = smb_direct_prepare_negotiation(sc);
-	if (ret) {
-		pr_err("Can't negotiate: %d\n", ret);
-		return ret;
-	}
-	return 0;
-}
-
-static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)
-{
-	if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
-		return false;
-	if (attrs->max_fast_reg_page_list_len == 0)
-		return false;
-	return true;
-}
-
-static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id,
-					     struct rdma_cm_event *event)
-{
-	struct smb_direct_listener *listener = new_cm_id->context;
-	struct smb_direct_transport *t;
-	struct smbdirect_socket *sc;
-	struct smbdirect_socket_parameters *sp;
-	struct task_struct *handler;
-	u8 peer_initiator_depth;
-	u8 peer_responder_resources;
-	int ret;
-
-	if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) {
-		ksmbd_debug(RDMA,
-			    "Fast Registration Work Requests is not supported. device capabilities=%llx\n",
-			    new_cm_id->device->attrs.device_cap_flags);
-		return -EPROTONOSUPPORT;
-	}
-
-	t = alloc_transport(new_cm_id);
-	if (!t)
-		return -ENOMEM;
-	sc = &t->socket;
-	sp = &sc->parameters;
-
-	peer_initiator_depth = event->param.conn.initiator_depth;
-	peer_responder_resources = event->param.conn.responder_resources;
-	if (rdma_protocol_iwarp(new_cm_id->device, new_cm_id->port_num) &&
-	    event->param.conn.private_data_len == 8) {
-		/*
-		 * Legacy clients with only iWarp MPA v1 support
-		 * need a private blob in order to negotiate
-		 * the IRD/ORD values.
-		 */
-		const __be32 *ird_ord_hdr = event->param.conn.private_data;
-		u32 ird32 = be32_to_cpu(ird_ord_hdr[0]);
-		u32 ord32 = be32_to_cpu(ird_ord_hdr[1]);
-
-		/*
-		 * cifs.ko sends the legacy IRD/ORD negotiation
-		 * event if iWarp MPA v2 was used.
-		 *
-		 * Here we check that the values match and only
-		 * mark the client as legacy if they don't match.
-		 */
-		if ((u32)event->param.conn.initiator_depth != ird32 ||
-		    (u32)event->param.conn.responder_resources != ord32) {
-			/*
-			 * There are broken clients (old cifs.ko)
-			 * using little endian and also
-			 * struct rdma_conn_param only uses u8
-			 * for initiator_depth and responder_resources,
-			 * so we truncate the value to U8_MAX.
-			 *
-			 * smb_direct_accept_client() will then
-			 * do the real negotiation in order to
-			 * select the minimum between client and
-			 * server.
-			 */
-			ird32 = min_t(u32, ird32, U8_MAX);
-			ord32 = min_t(u32, ord32, U8_MAX);
-
-			sc->rdma.legacy_iwarp = true;
-			peer_initiator_depth = (u8)ird32;
-			peer_responder_resources = (u8)ord32;
-		}
-	}
-
-	/*
-	 * First set what the we as server are able to support
-	 */
-	sp->initiator_depth = min_t(u8, sp->initiator_depth,
-				   new_cm_id->device->attrs.max_qp_rd_atom);
-
-	/*
-	 * negotiate the value by using the minimum
-	 * between client and server if the client provided
-	 * non 0 values.
-	 */
-	if (peer_initiator_depth != 0)
-		sp->initiator_depth = min_t(u8, sp->initiator_depth,
-					   peer_initiator_depth);
-	if (peer_responder_resources != 0)
-		sp->responder_resources = min_t(u8, sp->responder_resources,
-					       peer_responder_resources);
-
-	ret = smb_direct_connect(sc);
-	if (ret)
-		goto out_err;
-
-	handler = kthread_run(ksmbd_conn_handler_loop,
-			      KSMBD_TRANS(t)->conn, "ksmbd:r%u",
-			      listener->port);
-	if (IS_ERR(handler)) {
-		ret = PTR_ERR(handler);
-		pr_err("Can't start thread\n");
-		goto out_err;
-	}
-
-	return 0;
-out_err:
-	free_transport(t);
-	return ret;
-}
-
-static int smb_direct_listen_handler(struct rdma_cm_id *cm_id,
-				     struct rdma_cm_event *event)
-{
-	switch (event->event) {
-	case RDMA_CM_EVENT_CONNECT_REQUEST: {
-		int ret = smb_direct_handle_connect_request(cm_id, event);
-
-		if (ret) {
-			pr_err("Can't create transport: %d\n", ret);
-			return ret;
-		}
-
-		ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n",
-			    cm_id);
-		break;
-	}
-	default:
-		pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n",
-		       cm_id, rdma_event_msg(event->event), event->event);
-		break;
-	}
-	return 0;
-}
-
-static int smb_direct_listen(struct smb_direct_listener *listener,
-			     int port)
-{
-	int ret;
-	struct rdma_cm_id *cm_id;
-	u8 node_type = RDMA_NODE_UNSPECIFIED;
-	struct sockaddr_in sin = {
-		.sin_family		= AF_INET,
-		.sin_addr.s_addr	= htonl(INADDR_ANY),
-		.sin_port		= htons(port),
-	};
-
-	switch (port) {
-	case SMB_DIRECT_PORT_IWARP:
-		/*
-		 * only allow iWarp devices
-		 * for port 5445.
-		 */
-		node_type = RDMA_NODE_RNIC;
-		break;
-	case SMB_DIRECT_PORT_INFINIBAND:
-		/*
-		 * only allow InfiniBand, RoCEv1 or RoCEv2
-		 * devices for port 445.
-		 *
-		 * (Basically don't allow iWarp devices)
-		 */
-		node_type = RDMA_NODE_IB_CA;
-		break;
-	default:
-		pr_err("unsupported smbdirect port=%d!\n", port);
-		return -ENODEV;
-	}
-
-	cm_id = rdma_create_id(&init_net, smb_direct_listen_handler,
-			       listener, RDMA_PS_TCP, IB_QPT_RC);
-	if (IS_ERR(cm_id)) {
-		pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id));
-		return PTR_ERR(cm_id);
-	}
-
-	ret = rdma_restrict_node_type(cm_id, node_type);
-	if (ret) {
-		pr_err("rdma_restrict_node_type(%u) failed %d\n",
-		       node_type, ret);
-		goto err;
-	}
-
-	ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
-	if (ret) {
-		pr_err("Can't bind: %d\n", ret);
-		goto err;
-	}
-
-	ret = rdma_listen(cm_id, 10);
-	if (ret) {
-		pr_err("Can't listen: %d\n", ret);
-		goto err;
-	}
-
-	listener->port = port;
-	listener->cm_id = cm_id;
-
-	return 0;
-err:
-	listener->port = 0;
-	listener->cm_id = NULL;
-	rdma_destroy_id(cm_id);
-	return ret;
-}
-
-static int smb_direct_ib_client_add(struct ib_device *ib_dev)
-{
-	struct smb_direct_device *smb_dev;
-
-	if (!rdma_frwr_is_supported(&ib_dev->attrs))
-		return 0;
-
-	smb_dev = kzalloc_obj(*smb_dev, KSMBD_DEFAULT_GFP);
-	if (!smb_dev)
-		return -ENOMEM;
-	smb_dev->ib_dev = ib_dev;
-
-	write_lock(&smb_direct_device_lock);
-	list_add(&smb_dev->list, &smb_direct_device_list);
-	write_unlock(&smb_direct_device_lock);
-
-	ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name);
-	return 0;
-}
-
-static void smb_direct_ib_client_remove(struct ib_device *ib_dev,
-					void *client_data)
-{
-	struct smb_direct_device *smb_dev, *tmp;
-
-	write_lock(&smb_direct_device_lock);
-	list_for_each_entry_safe(smb_dev, tmp, &smb_direct_device_list, list) {
-		if (smb_dev->ib_dev == ib_dev) {
-			list_del(&smb_dev->list);
-			kfree(smb_dev);
-			break;
-		}
-	}
-	write_unlock(&smb_direct_device_lock);
-}
-
-static struct ib_client smb_direct_ib_client = {
-	.name	= "ksmbd_smb_direct_ib",
-	.add	= smb_direct_ib_client_add,
-	.remove	= smb_direct_ib_client_remove,
-};
-
-int ksmbd_rdma_init(void)
-{
-	int ret;
-
-	smb_direct_ib_listener = smb_direct_iw_listener = (struct smb_direct_listener) {
-		.cm_id = NULL,
-	};
-
-	ret = ib_register_client(&smb_direct_ib_client);
-	if (ret) {
-		pr_err("failed to ib_register_client\n");
-		return ret;
-	}
-
-	/* When a client is running out of send credits, the credits are
-	 * granted by the server's sending a packet using this queue.
-	 * This avoids the situation that a clients cannot send packets
-	 * for lack of credits
-	 */
-	smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq",
-					WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU,
-					0);
-	if (!smb_direct_wq) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	ret = smb_direct_listen(&smb_direct_ib_listener,
-				SMB_DIRECT_PORT_INFINIBAND);
-	if (ret) {
-		pr_err("Can't listen on InfiniBand/RoCEv1/RoCEv2: %d\n", ret);
-		goto err;
-	}
-
-	ksmbd_debug(RDMA, "InfiniBand/RoCEv1/RoCEv2 RDMA listener. cm_id=%p\n",
-		    smb_direct_ib_listener.cm_id);
-
-	ret = smb_direct_listen(&smb_direct_iw_listener,
-				SMB_DIRECT_PORT_IWARP);
-	if (ret) {
-		pr_err("Can't listen on iWarp: %d\n", ret);
-		goto err;
-	}
-
-	ksmbd_debug(RDMA, "iWarp RDMA listener. cm_id=%p\n",
-		    smb_direct_iw_listener.cm_id);
-
-	return 0;
-err:
-	ksmbd_rdma_stop_listening();
-	ksmbd_rdma_destroy();
-	return ret;
-}
-
-void ksmbd_rdma_stop_listening(void)
-{
-	if (!smb_direct_ib_listener.cm_id && !smb_direct_iw_listener.cm_id)
-		return;
-
-	ib_unregister_client(&smb_direct_ib_client);
-
-	if (smb_direct_ib_listener.cm_id)
-		rdma_destroy_id(smb_direct_ib_listener.cm_id);
-	if (smb_direct_iw_listener.cm_id)
-		rdma_destroy_id(smb_direct_iw_listener.cm_id);
-
-	smb_direct_ib_listener = smb_direct_iw_listener = (struct smb_direct_listener) {
-		.cm_id = NULL,
-	};
-}
-
-void ksmbd_rdma_destroy(void)
-{
-	if (smb_direct_wq) {
-		destroy_workqueue(smb_direct_wq);
-		smb_direct_wq = NULL;
-	}
-}
-
-static bool ksmbd_find_rdma_capable_netdev(struct net_device *netdev)
-{
-	struct smb_direct_device *smb_dev;
-	int i;
-	bool rdma_capable = false;
-
-	read_lock(&smb_direct_device_lock);
-	list_for_each_entry(smb_dev, &smb_direct_device_list, list) {
-		for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) {
-			struct net_device *ndev;
-
-			ndev = ib_device_get_netdev(smb_dev->ib_dev, i + 1);
-			if (!ndev)
-				continue;
-
-			if (ndev == netdev) {
-				dev_put(ndev);
-				rdma_capable = true;
-				goto out;
-			}
-			dev_put(ndev);
-		}
-	}
-out:
-	read_unlock(&smb_direct_device_lock);
-
-	if (rdma_capable == false) {
-		struct ib_device *ibdev;
-
-		ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
-		if (ibdev) {
-			rdma_capable = rdma_frwr_is_supported(&ibdev->attrs);
-			ib_device_put(ibdev);
-		}
-	}
-
-	ksmbd_debug(RDMA, "netdev(%s) rdma capable : %s\n",
-		    netdev->name, str_true_false(rdma_capable));
-
-	return rdma_capable;
-}
-
-bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
-{
-	struct net_device *lower_dev;
-	struct list_head *iter;
-
-	if (ksmbd_find_rdma_capable_netdev(netdev))
-		return true;
-
-	/* check if netdev is bridge or VLAN */
-	if (netif_is_bridge_master(netdev) ||
-	    netdev->priv_flags & IFF_802_1Q_VLAN)
-		netdev_for_each_lower_dev(netdev, lower_dev, iter)
-			if (ksmbd_find_rdma_capable_netdev(lower_dev))
-				return true;
-
-	/* check if netdev is IPoIB safely without layer violation */
-	if (netdev->type == ARPHRD_INFINIBAND)
-		return true;
+	u8 node_type = smbdirect_netdev_rdma_capable_node_type(netdev);
 
-	return false;
+	return node_type != RDMA_NODE_UNSPECIFIED;
 }
 
 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
-	.prepare	= smb_direct_prepare,
 	.disconnect	= smb_direct_disconnect,
 	.shutdown	= smb_direct_shutdown,
 	.writev		= smb_direct_writev,
diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h
index 3f93c6a9f7e4a..05352dc47f952 100644
--- a/fs/smb/server/transport_rdma.h
+++ b/fs/smb/server/transport_rdma.h
@@ -14,17 +14,17 @@
 #ifdef CONFIG_SMB_SERVER_SMBDIRECT
 int ksmbd_rdma_init(void);
 void ksmbd_rdma_stop_listening(void);
-void ksmbd_rdma_destroy(void);
 bool ksmbd_rdma_capable_netdev(struct net_device *netdev);
 void init_smbd_max_io_size(unsigned int sz);
 unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt);
 #else
 static inline int ksmbd_rdma_init(void) { return 0; }
 static inline void ksmbd_rdma_stop_listening(void) { }
-static inline void ksmbd_rdma_destroy(void) { }
 static inline bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { return false; }
 static inline void init_smbd_max_io_size(unsigned int sz) { }
 static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; }
 #endif
 
+#include "../common/smbdirect/smbdirect.h"
+
 #endif /* __KSMBD_TRANSPORT_RDMA_H__ */
diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c
index 0a4ff3ec9c8cd..e6232ae81f738 100644
--- a/fs/squashfs/xz_wrapper.c
+++ b/fs/squashfs/xz_wrapper.c
@@ -58,9 +58,9 @@ static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk,
 		opts->dict_size = le32_to_cpu(comp_opts->dictionary_size);
 
 		/* the dictionary size should be 2^n or 2^n+2^(n+1) */
-		n = ffs(opts->dict_size) - 1;
-		if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) +
-						(1 << (n + 1))) {
+		n = ffs(opts->dict_size);
+		if (n-- == 0 || (opts->dict_size != (1 << n) &&
+				opts->dict_size != (1 << n) + (1 << (n + 1)))) {
 			err = -EIO;
 			goto out;
 		}
diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c
index e1e639f515a0f..b3edae0578c07 100644
--- a/fs/sysfs/group.c
+++ b/fs/sysfs/group.c
@@ -217,7 +217,7 @@ int sysfs_create_group(struct kobject *kobj,
 EXPORT_SYMBOL_GPL(sysfs_create_group);
 
 static int internal_create_groups(struct kobject *kobj, int update,
-				  const struct attribute_group **groups)
+				  const struct attribute_group *const *groups)
 {
 	int error = 0;
 	int i;
@@ -250,7 +250,7 @@ static int internal_create_groups(struct kobject *kobj, int update,
  * Returns 0 on success or error code from sysfs_create_group on failure.
  */
 int sysfs_create_groups(struct kobject *kobj,
-			const struct attribute_group **groups)
+			const struct attribute_group *const *groups)
 {
 	return internal_create_groups(kobj, 0, groups);
 }
@@ -268,7 +268,7 @@ EXPORT_SYMBOL_GPL(sysfs_create_groups);
  * Returns 0 on success or error code from sysfs_update_group on failure.
  */
 int sysfs_update_groups(struct kobject *kobj,
-			const struct attribute_group **groups)
+			const struct attribute_group *const *groups)
 {
 	return internal_create_groups(kobj, 1, groups);
 }
@@ -342,7 +342,7 @@ EXPORT_SYMBOL_GPL(sysfs_remove_group);
  * If groups is not NULL, remove the specified groups from the kobject.
  */
 void sysfs_remove_groups(struct kobject *kobj,
-			 const struct attribute_group **groups)
+			 const struct attribute_group *const *groups)
 {
 	int i;
 
@@ -613,7 +613,7 @@ EXPORT_SYMBOL_GPL(sysfs_group_change_owner);
  * Returns 0 on success or error code on failure.
  */
 int sysfs_groups_change_owner(struct kobject *kobj,
-			      const struct attribute_group **groups,
+			      const struct attribute_group *const *groups,
 			      kuid_t kuid, kgid_t kgid)
 {
 	int error = 0, i;
diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c
index f412d1a0f6bba..1c32cac098cf5 100644
--- a/fs/tests/exec_kunit.c
+++ b/fs/tests/exec_kunit.c
@@ -94,9 +94,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = {
 	{ { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3 + sizeof(void *)),
 	    .argc = 0, .envc = 0 },
 	  .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) },
-	{ { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 *  + sizeof(void *)),
-	    .argc = 0, .envc = 0 },
-	  .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) },
 	{ { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM,
 	    .argc = 0, .envc = 0 },
 	  .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) },
diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c
index 51c00c8fa1755..edfdf139cb026 100644
--- a/fs/tracefs/inode.c
+++ b/fs/tracefs/inode.c
@@ -664,6 +664,7 @@ struct dentry *tracefs_create_file(const char *name, umode_t mode,
 	fsnotify_create(d_inode(dentry->d_parent), dentry);
 	return tracefs_end_creating(dentry);
 }
+EXPORT_SYMBOL_GPL(tracefs_create_file);
 
 static struct dentry *__create_dir(const char *name, struct dentry *parent,
 				   const struct inode_operations *ops)
diff --git a/fs/ubifs/debug.c b/fs/ubifs/debug.c
index 160c16aa7b6e7..5794de5a9069f 100644
--- a/fs/ubifs/debug.c
+++ b/fs/ubifs/debug.c
@@ -230,7 +230,7 @@ void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
 	int count = 2;
 
 	pr_err("Dump in-memory inode:");
-	pr_err("\tinode          %lu\n", inode->i_ino);
+	pr_err("\tinode          %llu\n", inode->i_ino);
 	pr_err("\tsize           %llu\n",
 	       (unsigned long long)i_size_read(inode));
 	pr_err("\tnlink          %u\n", inode->i_nlink);
@@ -1101,7 +1101,7 @@ int dbg_check_synced_i_size(const struct ubifs_info *c, struct inode *inode)
 	if (ui->ui_size != ui->synced_i_size && !ui->dirty) {
 		ubifs_err(c, "ui_size is %lld, synced_i_size is %lld, but inode is clean",
 			  ui->ui_size, ui->synced_i_size);
-		ubifs_err(c, "i_ino %lu, i_mode %#x, i_size %lld", inode->i_ino,
+		ubifs_err(c, "i_ino %llu, i_mode %#x, i_size %lld", inode->i_ino,
 			  inode->i_mode, i_size_read(inode));
 		dump_stack();
 		err = -EINVAL;
@@ -1163,7 +1163,7 @@ int dbg_check_dir(struct ubifs_info *c, const struct inode *dir)
 	kfree(pdent);
 
 	if (i_size_read(dir) != size) {
-		ubifs_err(c, "directory inode %lu has size %llu, but calculated size is %llu",
+		ubifs_err(c, "directory inode %llu has size %llu, but calculated size is %llu",
 			  dir->i_ino, (unsigned long long)i_size_read(dir),
 			  (unsigned long long)size);
 		ubifs_dump_inode(c, dir);
@@ -1171,7 +1171,7 @@ int dbg_check_dir(struct ubifs_info *c, const struct inode *dir)
 		return -EINVAL;
 	}
 	if (dir->i_nlink != nlink) {
-		ubifs_err(c, "directory inode %lu has nlink %u, but calculated nlink is %u",
+		ubifs_err(c, "directory inode %llu has nlink %u, but calculated nlink is %u",
 			  dir->i_ino, dir->i_nlink, nlink);
 		ubifs_dump_inode(c, dir);
 		dump_stack();
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index 4c9f57f3b2adb..86d41e077e4d6 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -223,7 +223,7 @@ static struct dentry *ubifs_lookup(struct inode *dir, struct dentry *dentry,
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
 	struct fscrypt_name nm;
 
-	dbg_gen("'%pd' in dir ino %lu", dentry, dir->i_ino);
+	dbg_gen("'%pd' in dir ino %llu", dentry, dir->i_ino);
 
 	err = fscrypt_prepare_lookup(dir, dentry, &nm);
 	if (err == -ENOENT)
@@ -281,7 +281,7 @@ static struct dentry *ubifs_lookup(struct inode *dir, struct dentry *dentry,
 	if (IS_ENCRYPTED(dir) &&
 	    (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
 	    !fscrypt_has_permitted_context(dir, inode)) {
-		ubifs_warn(c, "Inconsistent encryption contexts: %lu/%lu",
+		ubifs_warn(c, "Inconsistent encryption contexts: %llu/%llu",
 			   dir->i_ino, inode->i_ino);
 		iput(inode);
 		inode = ERR_PTR(-EPERM);
@@ -318,7 +318,7 @@ static int ubifs_create(struct mnt_idmap *idmap, struct inode *dir,
 	 * parent directory inode.
 	 */
 
-	dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
+	dbg_gen("dent '%pd', mode %#hx in dir ino %llu",
 		dentry, mode, dir->i_ino);
 
 	err = ubifs_budget_space(c, &req);
@@ -386,7 +386,7 @@ static struct inode *create_whiteout(struct inode *dir, struct dentry *dentry)
 	 * atomically.
 	 */
 
-	dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
+	dbg_gen("dent '%pd', mode %#hx in dir ino %llu",
 		dentry, mode, dir->i_ino);
 
 	inode = ubifs_new_inode(c, dir, mode, false);
@@ -460,7 +460,7 @@ static int ubifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	 * be released via writeback.
 	 */
 
-	dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
+	dbg_gen("dent '%pd', mode %#hx in dir ino %llu",
 		dentry, mode, dir->i_ino);
 
 	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
@@ -589,7 +589,7 @@ static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 	bool encrypted = IS_ENCRYPTED(dir);
 	struct ubifs_dir_data *data = file->private_data;
 
-	dbg_gen("dir ino %lu, f_pos %#llx", dir->i_ino, ctx->pos);
+	dbg_gen("dir ino %llu, f_pos %#llx", dir->i_ino, ctx->pos);
 
 	if (ctx->pos > UBIFS_S_KEY_HASH_MASK || ctx->pos == 2)
 		/*
@@ -764,7 +764,7 @@ static int ubifs_link(struct dentry *old_dentry, struct inode *dir,
 	 * changing the parent inode.
 	 */
 
-	dbg_gen("dent '%pd' to ino %lu (nlink %d) in dir ino %lu",
+	dbg_gen("dent '%pd' to ino %llu (nlink %d) in dir ino %llu",
 		dentry, inode->i_ino,
 		inode->i_nlink, dir->i_ino);
 	ubifs_assert(c, inode_is_locked(dir));
@@ -836,7 +836,7 @@ static int ubifs_unlink(struct inode *dir, struct dentry *dentry)
 	 * deletions.
 	 */
 
-	dbg_gen("dent '%pd' from ino %lu (nlink %d) in dir ino %lu",
+	dbg_gen("dent '%pd' from ino %llu (nlink %d) in dir ino %llu",
 		dentry, inode->i_ino,
 		inode->i_nlink, dir->i_ino);
 
@@ -941,7 +941,7 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 	 * because we have extra space reserved for deletions.
 	 */
 
-	dbg_gen("directory '%pd', ino %lu in dir ino %lu", dentry,
+	dbg_gen("directory '%pd', ino %llu in dir ino %llu", dentry,
 		inode->i_ino, dir->i_ino);
 	ubifs_assert(c, inode_is_locked(dir));
 	ubifs_assert(c, inode_is_locked(inode));
@@ -1018,7 +1018,7 @@ static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	 * directory inode.
 	 */
 
-	dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
+	dbg_gen("dent '%pd', mode %#hx in dir ino %llu",
 		dentry, mode, dir->i_ino);
 
 	err = ubifs_budget_space(c, &req);
@@ -1096,7 +1096,7 @@ static int ubifs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	 * directory inode.
 	 */
 
-	dbg_gen("dent '%pd' in dir ino %lu", dentry, dir->i_ino);
+	dbg_gen("dent '%pd' in dir ino %llu", dentry, dir->i_ino);
 
 	if (S_ISBLK(mode) || S_ISCHR(mode)) {
 		dev = kmalloc_obj(union ubifs_dev_desc, GFP_NOFS);
@@ -1183,7 +1183,7 @@ static int ubifs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 					.dirtied_ino = 1 };
 	struct fscrypt_name nm;
 
-	dbg_gen("dent '%pd', target '%s' in dir ino %lu", dentry,
+	dbg_gen("dent '%pd', target '%s' in dir ino %llu", dentry,
 		symname, dir->i_ino);
 
 	err = fscrypt_prepare_symlink(dir, symname, len, UBIFS_MAX_INO_DATA,
@@ -1349,7 +1349,7 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
 	 *   ino_req: marks the target inode as dirty and does not write it.
 	 */
 
-	dbg_gen("dent '%pd' ino %lu in dir ino %lu to dent '%pd' in dir ino %lu flags 0x%x",
+	dbg_gen("dent '%pd' ino %llu in dir ino %llu to dent '%pd' in dir ino %llu flags 0x%x",
 		old_dentry, old_inode->i_ino, old_dir->i_ino,
 		new_dentry, new_dir->i_ino, flags);
 
@@ -1597,7 +1597,7 @@ static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry,
 	 * parent directory inodes.
 	 */
 
-	dbg_gen("dent '%pd' ino %lu in dir ino %lu exchange dent '%pd' ino %lu in dir ino %lu",
+	dbg_gen("dent '%pd' ino %llu in dir ino %llu exchange dent '%pd' ino %llu in dir ino %llu",
 		old_dentry, fst_inode->i_ino, old_dir->i_ino,
 		new_dentry, snd_inode->i_ino, new_dir->i_ino);
 
diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
index cd04755e792a7..e73c28b12f97f 100644
--- a/fs/ubifs/file.c
+++ b/fs/ubifs/file.c
@@ -90,7 +90,7 @@ static int read_block(struct inode *inode, struct folio *folio, size_t offset,
 	return 0;
 
 dump:
-	ubifs_err(c, "bad data node (block %u, inode %lu)",
+	ubifs_err(c, "bad data node (block %u, inode %llu)",
 		  block, inode->i_ino);
 	ubifs_dump_node(c, dn, UBIFS_MAX_DATA_NODE_SZ);
 	return -EINVAL;
@@ -106,7 +106,7 @@ static int do_readpage(struct folio *folio)
 	loff_t i_size = i_size_read(inode);
 	size_t offset = 0;
 
-	dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",
+	dbg_gen("ino %llu, pg %lu, i_size %lld, flags %#lx",
 		inode->i_ino, folio->index, i_size, folio->flags.f);
 	ubifs_assert(c, !folio_test_checked(folio));
 	ubifs_assert(c, !folio->private);
@@ -162,7 +162,7 @@ static int do_readpage(struct folio *folio)
 			dbg_gen("hole");
 			err = 0;
 		} else {
-			ubifs_err(c, "cannot read page %lu of inode %lu, error %d",
+			ubifs_err(c, "cannot read page %lu of inode %llu, error %d",
 				  folio->index, inode->i_ino, err);
 		}
 	}
@@ -212,7 +212,7 @@ static int write_begin_slow(struct address_space *mapping,
 	int err, appending = !!(pos + len > inode->i_size);
 	struct folio *folio;
 
-	dbg_gen("ino %lu, pos %llu, len %u, i_size %lld",
+	dbg_gen("ino %llu, pos %llu, len %u, i_size %lld",
 		inode->i_ino, pos, len, inode->i_size);
 
 	/*
@@ -526,7 +526,7 @@ static int ubifs_write_end(const struct kiocb *iocb,
 	loff_t end_pos = pos + len;
 	int appending = !!(end_pos > inode->i_size);
 
-	dbg_gen("ino %lu, pos %llu, pg %lu, len %u, copied %d, i_size %lld",
+	dbg_gen("ino %llu, pos %llu, pg %lu, len %u, copied %d, i_size %lld",
 		inode->i_ino, pos, folio->index, len, copied, inode->i_size);
 
 	if (unlikely(copied < len && !folio_test_uptodate(folio))) {
@@ -599,7 +599,7 @@ static int populate_page(struct ubifs_info *c, struct folio *folio,
 	size_t offset = 0;
 	pgoff_t end_index;
 
-	dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx",
+	dbg_gen("ino %llu, pg %lu, i_size %lld, flags %#lx",
 		inode->i_ino, folio->index, i_size, folio->flags.f);
 
 	end_index = (i_size - 1) >> PAGE_SHIFT;
@@ -680,7 +680,7 @@ static int populate_page(struct ubifs_info *c, struct folio *folio,
 	return 0;
 
 out_err:
-	ubifs_err(c, "bad data node (block %u, inode %lu)",
+	ubifs_err(c, "bad data node (block %u, inode %llu)",
 		  page_block, inode->i_ino);
 	return -EINVAL;
 }
@@ -913,7 +913,7 @@ static int do_writepage(struct folio *folio, size_t len)
 	}
 	if (err) {
 		mapping_set_error(folio->mapping, err);
-		ubifs_err(c, "cannot write folio %lu of inode %lu, error %d",
+		ubifs_err(c, "cannot write folio %lu of inode %llu, error %d",
 			  folio->index, inode->i_ino, err);
 		ubifs_ro_mode(c, err);
 	}
@@ -987,7 +987,7 @@ static int ubifs_writepage(struct folio *folio, struct writeback_control *wbc)
 	loff_t i_size =  i_size_read(inode), synced_i_size;
 	int err, len = folio_size(folio);
 
-	dbg_gen("ino %lu, pg %lu, pg flags %#lx",
+	dbg_gen("ino %llu, pg %lu, pg flags %#lx",
 		inode->i_ino, folio->index, folio->flags.f);
 	ubifs_assert(c, folio->private != NULL);
 
@@ -1106,7 +1106,7 @@ static int do_truncation(struct ubifs_info *c, struct inode *inode,
 	int offset = new_size & (UBIFS_BLOCK_SIZE - 1), budgeted = 1;
 	struct ubifs_inode *ui = ubifs_inode(inode);
 
-	dbg_gen("ino %lu, size %lld -> %lld", inode->i_ino, old_size, new_size);
+	dbg_gen("ino %llu, size %lld -> %lld", inode->i_ino, old_size, new_size);
 	memset(&req, 0, sizeof(struct ubifs_budget_req));
 
 	/*
@@ -1258,7 +1258,7 @@ int ubifs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	struct inode *inode = d_inode(dentry);
 	struct ubifs_info *c = inode->i_sb->s_fs_info;
 
-	dbg_gen("ino %lu, mode %#x, ia_valid %#x",
+	dbg_gen("ino %llu, mode %#x, ia_valid %#x",
 		inode->i_ino, inode->i_mode, attr->ia_valid);
 	err = setattr_prepare(&nop_mnt_idmap, dentry, attr);
 	if (err)
@@ -1308,7 +1308,7 @@ int ubifs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	struct ubifs_info *c = inode->i_sb->s_fs_info;
 	int err;
 
-	dbg_gen("syncing inode %lu", inode->i_ino);
+	dbg_gen("syncing inode %llu", inode->i_ino);
 
 	if (c->ro_mount)
 		/*
@@ -1495,7 +1495,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf)
 	struct ubifs_budget_req req = { .new_page = 1 };
 	int err, update_time;
 
-	dbg_gen("ino %lu, pg %lu, i_size %lld",	inode->i_ino, folio->index,
+	dbg_gen("ino %llu, pg %lu, i_size %lld",	inode->i_ino, folio->index,
 		i_size_read(inode));
 	ubifs_assert(c, !c->ro_media && !c->ro_mount);
 
@@ -1531,7 +1531,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf)
 	err = ubifs_budget_space(c, &req);
 	if (unlikely(err)) {
 		if (err == -ENOSPC)
-			ubifs_warn(c, "out of space for mmapped file (inode number %lu)",
+			ubifs_warn(c, "out of space for mmapped file (inode number %llu)",
 				   inode->i_ino);
 		return VM_FAULT_SIGBUS;
 	}
diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c
index e28ab4395e5ca..40a95a2fad500 100644
--- a/fs/ubifs/journal.c
+++ b/fs/ubifs/journal.c
@@ -982,7 +982,7 @@ int ubifs_jnl_write_inode(struct ubifs_info *c, const struct inode *inode)
 	int kill_xattrs = ui->xattr_cnt && last_reference;
 	u8 hash[UBIFS_HASH_ARR_SZ];
 
-	dbg_jnl("ino %lu, nlink %u", inode->i_ino, inode->i_nlink);
+	dbg_jnl("ino %llu, nlink %u", inode->i_ino, inode->i_nlink);
 
 	if (kill_xattrs && ui->xattr_cnt > ubifs_xattr_max_cnt(c)) {
 		ubifs_err(c, "Cannot delete inode, it has too many xattrs!");
@@ -1743,7 +1743,7 @@ int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode,
 			int dn_len = le32_to_cpu(dn->size);
 
 			if (dn_len <= 0 || dn_len > UBIFS_BLOCK_SIZE) {
-				ubifs_err(c, "bad data node (block %u, inode %lu)",
+				ubifs_err(c, "bad data node (block %u, inode %llu)",
 					  blk, inode->i_ino);
 				ubifs_dump_node(c, dn, dn_size);
 				err = -EUCLEAN;
@@ -1987,7 +1987,7 @@ int ubifs_jnl_change_xattr(struct ubifs_info *c, const struct inode *inode,
 	u8 hash_host[UBIFS_HASH_ARR_SZ];
 	u8 hash[UBIFS_HASH_ARR_SZ];
 
-	dbg_jnl("ino %lu, ino %lu", host->i_ino, inode->i_ino);
+	dbg_jnl("ino %llu, ino %llu", host->i_ino, inode->i_ino);
 	ubifs_assert(c, inode->i_nlink > 0);
 	ubifs_assert(c, mutex_is_locked(&host_ui->ui_mutex));
 
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 03bf924756ca0..9a77d8b64ffa7 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -92,7 +92,7 @@ static int validate_inode(struct ubifs_info *c, const struct inode *inode)
 		return 5;
 
 	if (!ubifs_compr_present(c, ui->compr_type)) {
-		ubifs_warn(c, "inode %lu uses '%s' compression, but it was not compiled in",
+		ubifs_warn(c, "inode %llu uses '%s' compression, but it was not compiled in",
 			   inode->i_ino, ubifs_compr_name(c, ui->compr_type));
 	}
 
@@ -248,14 +248,14 @@ struct inode *ubifs_iget(struct super_block *sb, unsigned long inum)
 	return inode;
 
 out_invalid:
-	ubifs_err(c, "inode %lu validation failed, error %d", inode->i_ino, err);
+	ubifs_err(c, "inode %llu validation failed, error %d", inode->i_ino, err);
 	ubifs_dump_node(c, ino, UBIFS_MAX_INO_NODE_SZ);
 	ubifs_dump_inode(c, inode);
 	err = -EINVAL;
 out_ino:
 	kfree(ino);
 out:
-	ubifs_err(c, "failed to read inode %lu, error %d", inode->i_ino, err);
+	ubifs_err(c, "failed to read inode %llu, error %d", inode->i_ino, err);
 	iget_failed(inode);
 	return ERR_PTR(err);
 }
@@ -316,12 +316,12 @@ static int ubifs_write_inode(struct inode *inode, struct writeback_control *wbc)
 	 * As an optimization, do not write orphan inodes to the media just
 	 * because this is not needed.
 	 */
-	dbg_gen("inode %lu, mode %#x, nlink %u",
+	dbg_gen("inode %llu, mode %#x, nlink %u",
 		inode->i_ino, (int)inode->i_mode, inode->i_nlink);
 	if (inode->i_nlink) {
 		err = ubifs_jnl_write_inode(c, inode);
 		if (err)
-			ubifs_err(c, "can't write inode %lu, error %d",
+			ubifs_err(c, "can't write inode %llu, error %d",
 				  inode->i_ino, err);
 		else
 			err = dbg_check_inode_size(c, inode, ui->ui_size);
@@ -357,7 +357,7 @@ static void ubifs_evict_inode(struct inode *inode)
 		 */
 		goto out;
 
-	dbg_gen("inode %lu, mode %#x", inode->i_ino, (int)inode->i_mode);
+	dbg_gen("inode %llu, mode %#x", inode->i_ino, (int)inode->i_mode);
 	ubifs_assert(c, !icount_read(inode));
 
 	truncate_inode_pages_final(&inode->i_data);
@@ -375,7 +375,7 @@ static void ubifs_evict_inode(struct inode *inode)
 		 * Worst case we have a lost orphan inode wasting space, so a
 		 * simple error message is OK here.
 		 */
-		ubifs_err(c, "can't delete inode %lu, error %d",
+		ubifs_err(c, "can't delete inode %llu, error %d",
 			  inode->i_ino, err);
 
 out:
@@ -399,7 +399,7 @@ static void ubifs_dirty_inode(struct inode *inode, int flags)
 	ubifs_assert(c, mutex_is_locked(&ui->ui_mutex));
 	if (!ui->dirty) {
 		ui->dirty = 1;
-		dbg_gen("inode %lu",  inode->i_ino);
+		dbg_gen("inode %llu",  inode->i_ino);
 	}
 }
 
diff --git a/fs/ubifs/tnc.c b/fs/ubifs/tnc.c
index 694b08d27d7d9..c9d8935f66780 100644
--- a/fs/ubifs/tnc.c
+++ b/fs/ubifs/tnc.c
@@ -3561,8 +3561,8 @@ int dbg_check_inode_size(struct ubifs_info *c, const struct inode *inode,
 
 out_dump:
 	block = key_block(c, key);
-	ubifs_err(c, "inode %lu has size %lld, but there are data at offset %lld",
-		  (unsigned long)inode->i_ino, size,
+	ubifs_err(c, "inode %llu has size %lld, but there are data at offset %lld",
+		  inode->i_ino, size,
 		  ((loff_t)block) << UBIFS_BLOCK_SHIFT);
 	mutex_unlock(&c->tnc_mutex);
 	ubifs_dump_inode(c, inode);
diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c
index c21a0c2b3e907..b5a9ab9d8a10a 100644
--- a/fs/ubifs/xattr.c
+++ b/fs/ubifs/xattr.c
@@ -76,7 +76,7 @@ static int create_xattr(struct ubifs_info *c, struct inode *host,
 				.dirtied_ino_d = ALIGN(host_ui->data_len, 8) };
 
 	if (host_ui->xattr_cnt >= ubifs_xattr_max_cnt(c)) {
-		ubifs_err(c, "inode %lu already has too many xattrs (%d), cannot create more",
+		ubifs_err(c, "inode %llu already has too many xattrs (%d), cannot create more",
 			  host->i_ino, host_ui->xattr_cnt);
 		return -ENOSPC;
 	}
@@ -88,7 +88,7 @@ static int create_xattr(struct ubifs_info *c, struct inode *host,
 	 */
 	names_len = host_ui->xattr_names + host_ui->xattr_cnt + fname_len(nm) + 1;
 	if (names_len > XATTR_LIST_MAX) {
-		ubifs_err(c, "cannot add one more xattr name to inode %lu, total names length would become %d, max. is %d",
+		ubifs_err(c, "cannot add one more xattr name to inode %llu, total names length would become %d, max. is %d",
 			  host->i_ino, names_len, XATTR_LIST_MAX);
 		return -ENOSPC;
 	}
@@ -390,7 +390,7 @@ ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 	int err, len, written = 0;
 	struct fscrypt_name nm = {0};
 
-	dbg_gen("ino %lu ('%pd'), buffer size %zd", host->i_ino,
+	dbg_gen("ino %llu ('%pd'), buffer size %zd", host->i_ino,
 		dentry, size);
 
 	down_read(&host_ui->xattr_sem);
@@ -498,7 +498,7 @@ int ubifs_purge_xattrs(struct inode *host)
 	if (ubifs_inode(host)->xattr_cnt <= ubifs_xattr_max_cnt(c))
 		return 0;
 
-	ubifs_warn(c, "inode %lu has too many xattrs, doing a non-atomic deletion",
+	ubifs_warn(c, "inode %llu has too many xattrs, doing a non-atomic deletion",
 		   host->i_ino);
 
 	down_write(&ubifs_inode(host)->xattr_sem);
@@ -641,7 +641,7 @@ int ubifs_init_security(struct inode *dentry, struct inode *inode,
 					   &init_xattrs, NULL);
 	if (err) {
 		struct ubifs_info *c = dentry->i_sb->s_fs_info;
-		ubifs_err(c, "cannot initialize security for inode %lu, error %d",
+		ubifs_err(c, "cannot initialize security for inode %llu, error %d",
 			  inode->i_ino, err);
 	}
 	return err;
@@ -652,7 +652,7 @@ static int xattr_get(const struct xattr_handler *handler,
 			   struct dentry *dentry, struct inode *inode,
 			   const char *name, void *buffer, size_t size)
 {
-	dbg_gen("xattr '%s', ino %lu ('%pd'), buf size %zd", name,
+	dbg_gen("xattr '%s', ino %llu ('%pd'), buf size %zd", name,
 		inode->i_ino, dentry, size);
 
 	name = xattr_full_name(handler, name);
@@ -665,7 +665,7 @@ static int xattr_set(const struct xattr_handler *handler,
 			   const char *name, const void *value,
 			   size_t size, int flags)
 {
-	dbg_gen("xattr '%s', host ino %lu ('%pd'), size %zd",
+	dbg_gen("xattr '%s', host ino %llu ('%pd'), size %zd",
 		name, inode->i_ino, dentry, size);
 
 	name = xattr_full_name(handler, name);
diff --git a/fs/udf/directory.c b/fs/udf/directory.c
index 632453aa38934..f5c81e13eacb1 100644
--- a/fs/udf/directory.c
+++ b/fs/udf/directory.c
@@ -22,7 +22,7 @@ static int udf_verify_fi(struct udf_fileident_iter *iter)
 
 	if (iter->fi.descTag.tagIdent != cpu_to_le16(TAG_IDENT_FID)) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has entry at pos %llu with incorrect tag %x\n",
+			"directory (ino %llu) has entry at pos %llu with incorrect tag %x\n",
 			iter->dir->i_ino, (unsigned long long)iter->pos,
 			le16_to_cpu(iter->fi.descTag.tagIdent));
 		return -EFSCORRUPTED;
@@ -30,7 +30,7 @@ static int udf_verify_fi(struct udf_fileident_iter *iter)
 	len = udf_dir_entry_len(&iter->fi);
 	if (le16_to_cpu(iter->fi.lengthOfImpUse) & 3) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has entry at pos %llu with unaligned length of impUse field\n",
+			"directory (ino %llu) has entry at pos %llu with unaligned length of impUse field\n",
 			iter->dir->i_ino, (unsigned long long)iter->pos);
 		return -EFSCORRUPTED;
 	}
@@ -41,20 +41,20 @@ static int udf_verify_fi(struct udf_fileident_iter *iter)
 	 */
 	if (len > 1 << iter->dir->i_blkbits) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has too big (%u) entry at pos %llu\n",
+			"directory (ino %llu) has too big (%u) entry at pos %llu\n",
 			iter->dir->i_ino, len, (unsigned long long)iter->pos);
 		return -EFSCORRUPTED;
 	}
 	if (iter->pos + len > iter->dir->i_size) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has entry past directory size at pos %llu\n",
+			"directory (ino %llu) has entry past directory size at pos %llu\n",
 			iter->dir->i_ino, (unsigned long long)iter->pos);
 		return -EFSCORRUPTED;
 	}
 	if (udf_dir_entry_len(&iter->fi) !=
 	    sizeof(struct tag) + le16_to_cpu(iter->fi.descTag.descCRCLength)) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has entry where CRC length (%u) does not match entry length (%u)\n",
+			"directory (ino %llu) has entry where CRC length (%u) does not match entry length (%u)\n",
 			iter->dir->i_ino,
 			(unsigned)le16_to_cpu(iter->fi.descTag.descCRCLength),
 			(unsigned)(udf_dir_entry_len(&iter->fi) -
@@ -78,7 +78,7 @@ static int udf_copy_fi(struct udf_fileident_iter *iter)
 	}
 	if (iter->dir->i_size < iter->pos + sizeof(struct fileIdentDesc)) {
 		udf_err(iter->dir->i_sb,
-			"directory (ino %lu) has entry straddling EOF\n",
+			"directory (ino %llu) has entry straddling EOF\n",
 			iter->dir->i_ino);
 		return -EFSCORRUPTED;
 	}
@@ -184,7 +184,7 @@ static int udf_fiiter_advance_blk(struct udf_fileident_iter *iter)
 			return 0;
 		}
 		udf_err(iter->dir->i_sb,
-			"extent after position %llu not allocated in directory (ino %lu)\n",
+			"extent after position %llu not allocated in directory (ino %llu)\n",
 			(unsigned long long)iter->pos, iter->dir->i_ino);
 		return -EFSCORRUPTED;
 	}
@@ -272,7 +272,7 @@ int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir,
 		if (pos == dir->i_size)
 			return 0;
 		udf_err(dir->i_sb,
-			"position %llu not allocated in directory (ino %lu)\n",
+			"position %llu not allocated in directory (ino %llu)\n",
 			(unsigned long long)pos, dir->i_ino);
 		err = -EFSCORRUPTED;
 		goto out;
@@ -483,7 +483,7 @@ int udf_fiiter_append_blk(struct udf_fileident_iter *iter)
 		   &iter->loffset, &etype);
 	if (err <= 0 || etype != (EXT_RECORDED_ALLOCATED >> 30)) {
 		udf_err(iter->dir->i_sb,
-			"block %llu not allocated in directory (ino %lu)\n",
+			"block %llu not allocated in directory (ino %llu)\n",
 			(unsigned long long)block, iter->dir->i_ino);
 		return -EFSCORRUPTED;
 	}
diff --git a/fs/udf/file.c b/fs/udf/file.c
index 32ae7cfd72c54..b043fe10e5d60 100644
--- a/fs/udf/file.c
+++ b/fs/udf/file.c
@@ -133,7 +133,7 @@ long udf_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	int result;
 
 	if (file_permission(filp, MAY_READ) != 0) {
-		udf_debug("no permission to access inode %lu\n", inode->i_ino);
+		udf_debug("no permission to access inode %llu\n", inode->i_ino);
 		return -EPERM;
 	}
 
diff --git a/fs/udf/inode.c b/fs/udf/inode.c
index 7fae8002344a0..5508eeec7a83c 100644
--- a/fs/udf/inode.c
+++ b/fs/udf/inode.c
@@ -147,7 +147,7 @@ void udf_evict_inode(struct inode *inode)
 		if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB &&
 		    inode->i_size != iinfo->i_lenExtents) {
 			udf_warn(inode->i_sb,
-				 "Inode %lu (mode %o) has inode size %llu different from extent length %llu. Filesystem need not be standards compliant.\n",
+				 "Inode %llu (mode %o) has inode size %llu different from extent length %llu. Filesystem need not be standards compliant.\n",
 				 inode->i_ino, inode->i_mode,
 				 (unsigned long long)inode->i_size,
 				 (unsigned long long)iinfo->i_lenExtents);
@@ -734,7 +734,7 @@ static int inode_getblk(struct inode *inode, struct udf_map_rq *map)
 	sector_t offset = 0;
 	int8_t etype, tmpetype;
 	struct udf_inode_info *iinfo = UDF_I(inode);
-	udf_pblk_t goal = 0, pgoal = iinfo->i_location.logicalBlockNum;
+	udf_pblk_t goal = 0, pgoal = 0;
 	int lastblock = 0;
 	bool isBeyondEOF = false;
 	int ret = 0;
@@ -893,11 +893,10 @@ static int inode_getblk(struct inode *inode, struct udf_map_rq *map)
 	else { /* otherwise, allocate a new block */
 		if (iinfo->i_next_alloc_block == map->lblk)
 			goal = iinfo->i_next_alloc_goal;
-
-		if (!goal) {
-			if (!(goal = pgoal)) /* XXX: what was intended here? */
-				goal = iinfo->i_location.logicalBlockNum + 1;
-		}
+		if (!goal)
+			goal = pgoal;
+		if (!goal)
+			goal = iinfo->i_location.logicalBlockNum + 1;
 
 		newblocknum = udf_new_block(inode->i_sb, inode,
 				iinfo->i_location.partitionReferenceNum,
@@ -1386,13 +1385,13 @@ static int udf_read_inode(struct inode *inode, bool hidden_inode)
 	 */
 	bh = udf_read_ptagged(inode->i_sb, iloc, 0, &ident);
 	if (!bh) {
-		udf_err(inode->i_sb, "(ino %lu) failed !bh\n", inode->i_ino);
+		udf_err(inode->i_sb, "(ino %llu) failed !bh\n", inode->i_ino);
 		return -EIO;
 	}
 
 	if (ident != TAG_IDENT_FE && ident != TAG_IDENT_EFE &&
 	    ident != TAG_IDENT_USE) {
-		udf_err(inode->i_sb, "(ino %lu) failed ident=%u\n",
+		udf_err(inode->i_sb, "(ino %llu) failed ident=%u\n",
 			inode->i_ino, ident);
 		goto out;
 	}
@@ -1641,7 +1640,7 @@ static int udf_read_inode(struct inode *inode, bool hidden_inode)
 		udf_debug("METADATA BITMAP FILE-----\n");
 		break;
 	default:
-		udf_err(inode->i_sb, "(ino %lu) failed unknown file type=%u\n",
+		udf_err(inode->i_sb, "(ino %llu) failed unknown file type=%u\n",
 			inode->i_ino, fe->icbTag.fileType);
 		goto out;
 	}
@@ -1942,7 +1941,7 @@ static int udf_update_inode(struct inode *inode, int do_sync)
 	if (do_sync) {
 		sync_dirty_buffer(bh);
 		if (buffer_write_io_error(bh)) {
-			udf_warn(inode->i_sb, "IO error syncing udf inode [%08lx]\n",
+			udf_warn(inode->i_sb, "IO error syncing udf inode [%08llx]\n",
 				 inode->i_ino);
 			err = -EIO;
 		}
@@ -2224,7 +2223,7 @@ int udf_next_aext(struct inode *inode, struct extent_position *epos,
 
 		if (++indirections > UDF_MAX_INDIR_EXTS) {
 			udf_err(inode->i_sb,
-				"too many indirect extents in inode %lu\n",
+				"too many indirect extents in inode %llu\n",
 				inode->i_ino);
 			return -EFSCORRUPTED;
 		}
diff --git a/fs/udf/namei.c b/fs/udf/namei.c
index 5f2e9a892bffa..ccafcaa968098 100644
--- a/fs/udf/namei.c
+++ b/fs/udf/namei.c
@@ -550,7 +550,7 @@ static int udf_unlink(struct inode *dir, struct dentry *dentry)
 		goto end_unlink;
 
 	if (!inode->i_nlink) {
-		udf_debug("Deleting nonexistent file (%lu), %u\n",
+		udf_debug("Deleting nonexistent file (%llu), %u\n",
 			  inode->i_ino, inode->i_nlink);
 		set_nlink(inode, 1);
 	}
@@ -809,7 +809,7 @@ static int udf_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 					       &diriter);
 		if (retval == -ENOENT) {
 			udf_err(old_inode->i_sb,
-				"directory (ino %lu) has no '..' entry\n",
+				"directory (ino %llu) has no '..' entry\n",
 				old_inode->i_ino);
 			retval = -EFSCORRUPTED;
 		}
@@ -821,7 +821,7 @@ static int udf_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 				old_dir->i_ino) {
 			retval = -EFSCORRUPTED;
 			udf_err(old_inode->i_sb,
-				"directory (ino %lu) has parent entry pointing to another inode (%lu != %u)\n",
+				"directory (ino %llu) has parent entry pointing to another inode (%llu != %u)\n",
 				old_inode->i_ino, old_dir->i_ino,
 				udf_get_lb_pblock(old_inode->i_sb, &tloc, 0));
 			goto out_oiter;
@@ -869,7 +869,7 @@ static int udf_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	retval = udf_fiiter_find_entry(old_dir, &old_dentry->d_name, &oiter);
 	if (retval) {
 		udf_err(old_dir->i_sb,
-			"failed to find renamed entry again in directory (ino %lu)\n",
+			"failed to find renamed entry again in directory (ino %llu)\n",
 			old_dir->i_ino);
 	} else {
 		udf_fiiter_delete_entry(&oiter);
diff --git a/fs/udf/super.c b/fs/udf/super.c
index 27f463fd1d89e..e6414488c49a5 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -1166,7 +1166,7 @@ static int udf_fill_partdesc_info(struct super_block *sb,
 		}
 		map->s_uspace.s_table = inode;
 		map->s_partition_flags |= UDF_PART_FLAG_UNALLOC_TABLE;
-		udf_debug("unallocSpaceTable (part %d) @ %lu\n",
+		udf_debug("unallocSpaceTable (part %d) @ %llu\n",
 			  p_index, map->s_uspace.s_table->i_ino);
 	}
 
@@ -1694,8 +1694,9 @@ static struct udf_vds_record *handle_partition_descriptor(
 			return &(data->part_descs_loc[i].rec);
 	if (data->num_part_descs >= data->size_part_descs) {
 		struct part_desc_seq_scan_data *new_loc;
-		unsigned int new_size = ALIGN(partnum, PART_DESC_ALLOC_STEP);
+		unsigned int new_size;
 
+		new_size = data->num_part_descs + PART_DESC_ALLOC_STEP;
 		new_loc = kzalloc_objs(*new_loc, new_size);
 		if (!new_loc)
 			return ERR_PTR(-ENOMEM);
@@ -1705,6 +1706,7 @@ static struct udf_vds_record *handle_partition_descriptor(
 		data->part_descs_loc = new_loc;
 		data->size_part_descs = new_size;
 	}
+	data->part_descs_loc[data->num_part_descs].partnum = partnum;
 	return &(data->part_descs_loc[data->num_part_descs++].rec);
 }
 
@@ -2318,7 +2320,7 @@ static int udf_fill_super(struct super_block *sb, struct fs_context *fc)
 
 error_out:
 	iput(sbi->s_vat_inode);
-	unload_nls(uopt->nls_map);
+	unload_nls(sbi->s_nls_map);
 	if (lvid_open)
 		udf_close_lvid(sb);
 	brelse(sbi->s_lvid_bh);
diff --git a/fs/ufs/balloc.c b/fs/ufs/balloc.c
index 194ed3ab945e3..628edfde3a9fd 100644
--- a/fs/ufs/balloc.c
+++ b/fs/ufs/balloc.c
@@ -245,7 +245,7 @@ static void ufs_change_blocknr(struct inode *inode, sector_t beg,
 	sector_t end, i;
 	struct buffer_head *head, *bh;
 
-	UFSD("ENTER, ino %lu, count %u, oldb %llu, newb %llu\n",
+	UFSD("ENTER, ino %llu, count %u, oldb %llu, newb %llu\n",
 	      inode->i_ino, count,
 	     (unsigned long long)oldb, (unsigned long long)newb);
 
@@ -340,7 +340,7 @@ u64 ufs_new_fragments(struct inode *inode, void *p, u64 fragment,
 	unsigned cgno, oldcount, newcount;
 	u64 tmp, request, result;
 	
-	UFSD("ENTER, ino %lu, fragment %llu, goal %llu, count %u\n",
+	UFSD("ENTER, ino %llu, fragment %llu, goal %llu, count %u\n",
 	     inode->i_ino, (unsigned long long)fragment,
 	     (unsigned long long)goal, count);
 	
@@ -583,7 +583,7 @@ static u64 ufs_alloc_fragments(struct inode *inode, unsigned cgno,
 	unsigned oldcg, i, j, k, allocsize;
 	u64 result;
 	
-	UFSD("ENTER, ino %lu, cgno %u, goal %llu, count %u\n",
+	UFSD("ENTER, ino %llu, cgno %u, goal %llu, count %u\n",
 	     inode->i_ino, cgno, (unsigned long long)goal, count);
 
 	sb = inode->i_sb;
diff --git a/fs/ufs/dir.c b/fs/ufs/dir.c
index 43f1578ab8666..f10a50f7e78b8 100644
--- a/fs/ufs/dir.c
+++ b/fs/ufs/dir.c
@@ -150,7 +150,7 @@ static bool ufs_check_folio(struct folio *folio, char *kaddr)
 
 Ebadsize:
 	ufs_error(sb, __func__,
-		  "size of directory #%lu is not a multiple of chunk size",
+		  "size of directory #%llu is not a multiple of chunk size",
 		  dir->i_ino
 	);
 	goto fail;
@@ -169,7 +169,7 @@ static bool ufs_check_folio(struct folio *folio, char *kaddr)
 Einumber:
 	error = "inode out of bounds";
 bad_entry:
-	ufs_error(sb, __func__, "bad entry in directory #%lu: %s - "
+	ufs_error(sb, __func__, "bad entry in directory #%llu: %s - "
 		   "offset=%llu, rec_len=%d, name_len=%d",
 		   dir->i_ino, error, folio_pos(folio) + offs,
 		   rec_len, ufs_get_de_namlen(sb, p));
@@ -177,7 +177,7 @@ static bool ufs_check_folio(struct folio *folio, char *kaddr)
 Eend:
 	p = (struct ufs_dir_entry *)(kaddr + offs);
 	ufs_error(sb, __func__,
-		   "entry in directory #%lu spans the page boundary"
+		   "entry in directory #%llu spans the page boundary"
 		   "offset=%llu",
 		   dir->i_ino, folio_pos(folio) + offs);
 fail:
@@ -258,7 +258,7 @@ struct ufs_dir_entry *ufs_find_entry(struct inode *dir, const struct qstr *qstr,
 	struct ufs_inode_info *ui = UFS_I(dir);
 	struct ufs_dir_entry *de;
 
-	UFSD("ENTER, dir_ino %lu, name %s, namlen %u\n", dir->i_ino, name, namelen);
+	UFSD("ENTER, dir_ino %llu, name %s, namlen %u\n", dir->i_ino, name, namelen);
 
 	if (npages == 0 || namelen > UFS_MAXNAMLEN)
 		goto out;
@@ -434,7 +434,7 @@ ufs_readdir(struct file *file, struct dir_context *ctx)
 
 		if (IS_ERR(kaddr)) {
 			ufs_error(sb, __func__,
-				  "bad page in #%lu",
+				  "bad page in #%llu",
 				  inode->i_ino);
 			ctx->pos += PAGE_SIZE - offset;
 			return PTR_ERR(kaddr);
diff --git a/fs/ufs/ialloc.c b/fs/ufs/ialloc.c
index 73531827ecee1..8e51f4630d186 100644
--- a/fs/ufs/ialloc.c
+++ b/fs/ufs/ialloc.c
@@ -63,7 +63,7 @@ void ufs_free_inode (struct inode * inode)
 	int is_directory;
 	unsigned ino, cg, bit;
 	
-	UFSD("ENTER, ino %lu\n", inode->i_ino);
+	UFSD("ENTER, ino %llu\n", inode->i_ino);
 
 	sb = inode->i_sb;
 	uspi = UFS_SB(sb)->s_uspi;
@@ -317,7 +317,7 @@ struct inode *ufs_new_inode(struct inode *dir, umode_t mode)
 		bh = sb_bread(sb, uspi->s_sbbase + ufs_inotofsba(inode->i_ino));
 		if (!bh) {
 			ufs_warning(sb, "ufs_read_inode",
-				    "unable to read inode %lu\n",
+				    "unable to read inode %llu\n",
 				    inode->i_ino);
 			err = -EIO;
 			goto fail_remove_inode;
@@ -336,7 +336,7 @@ struct inode *ufs_new_inode(struct inode *dir, umode_t mode)
 	}
 	mutex_unlock(&sbi->s_lock);
 
-	UFSD("allocating inode %lu\n", inode->i_ino);
+	UFSD("allocating inode %llu\n", inode->i_ino);
 	UFSD("EXIT\n");
 	return inode;
 
diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c
index e2b0a35de2a70..2a8728c879796 100644
--- a/fs/ufs/inode.c
+++ b/fs/ufs/inode.c
@@ -400,7 +400,7 @@ static int ufs_getfrag_block(struct inode *inode, sector_t fragment, struct buff
 
 	mutex_lock(&UFS_I(inode)->truncate_mutex);
 
-	UFSD("ENTER, ino %lu, fragment %llu\n", inode->i_ino, (unsigned long long)fragment);
+	UFSD("ENTER, ino %llu, fragment %llu\n", inode->i_ino, (unsigned long long)fragment);
 	if (unlikely(!depth)) {
 		ufs_warning(sb, "ufs_get_block", "block > big");
 		err = -EIO;
@@ -595,7 +595,7 @@ static int ufs2_read_inode(struct inode *inode, struct ufs2_inode *ufs2_inode)
 	struct super_block *sb = inode->i_sb;
 	umode_t mode;
 
-	UFSD("Reading ufs2 inode, ino %lu\n", inode->i_ino);
+	UFSD("Reading ufs2 inode, ino %llu\n", inode->i_ino);
 	/*
 	 * Copy data to the in-core inode.
 	 */
@@ -662,7 +662,7 @@ struct inode *ufs_iget(struct super_block *sb, unsigned long ino)
 
 	bh = sb_bread(sb, uspi->s_sbbase + ufs_inotofsba(inode->i_ino));
 	if (!bh) {
-		ufs_warning(sb, "ufs_read_inode", "unable to read inode %lu\n",
+		ufs_warning(sb, "ufs_read_inode", "unable to read inode %llu\n",
 			    inode->i_ino);
 		goto bad_inode;
 	}
@@ -793,17 +793,17 @@ static int ufs_update_inode(struct inode * inode, int do_sync)
 	struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi;
 	struct buffer_head * bh;
 
-	UFSD("ENTER, ino %lu\n", inode->i_ino);
+	UFSD("ENTER, ino %llu\n", inode->i_ino);
 
 	if (inode->i_ino < UFS_ROOTINO ||
 	    inode->i_ino > (uspi->s_ncg * uspi->s_ipg)) {
-		ufs_warning (sb, "ufs_read_inode", "bad inode number (%lu)\n", inode->i_ino);
+		ufs_warning (sb, "ufs_read_inode", "bad inode number (%llu)\n", inode->i_ino);
 		return -1;
 	}
 
 	bh = sb_bread(sb, ufs_inotofsba(inode->i_ino));
 	if (!bh) {
-		ufs_warning (sb, "ufs_read_inode", "unable to read inode %lu\n", inode->i_ino);
+		ufs_warning (sb, "ufs_read_inode", "unable to read inode %llu\n", inode->i_ino);
 		return -1;
 	}
 	if (uspi->fs_magic == UFS2_MAGIC) {
@@ -891,7 +891,7 @@ static void ufs_trunc_direct(struct inode *inode)
 	unsigned int old_tail, new_tail;
 	struct to_free ctx = {.inode = inode};
 
-	UFSD("ENTER: ino %lu\n", inode->i_ino);
+	UFSD("ENTER: ino %llu\n", inode->i_ino);
 
 	new_frags = DIRECT_FRAGMENT;
 	// new_frags = first fragment past the new EOF
@@ -956,7 +956,7 @@ static void ufs_trunc_direct(struct inode *inode)
 		}
 	}
 done:
-	UFSD("EXIT: ino %lu\n", inode->i_ino);
+	UFSD("EXIT: ino %llu\n", inode->i_ino);
 }
 
 static void free_full_branch(struct inode *inode, u64 ind_block, int depth)
@@ -1169,7 +1169,7 @@ static int ufs_truncate(struct inode *inode, loff_t size)
 {
 	int err = 0;
 
-	UFSD("ENTER: ino %lu, i_size: %llu, old_i_size: %llu\n",
+	UFSD("ENTER: ino %llu, i_size: %llu, old_i_size: %llu\n",
 	     inode->i_ino, (unsigned long long)size,
 	     (unsigned long long)i_size_read(inode));
 
diff --git a/fs/ufs/ufs_fs.h b/fs/ufs/ufs_fs.h
index 0905f9a16b915..b8dc354ae90f4 100644
--- a/fs/ufs/ufs_fs.h
+++ b/fs/ufs/ufs_fs.h
@@ -226,10 +226,10 @@ typedef __u16 __bitwise __fs16;
  *     inode number to cylinder group number.
  *     inode number to file system block address.
  */
-#define	ufs_inotocg(x)		((x) / uspi->s_ipg)
-#define	ufs_inotocgoff(x)	((x) % uspi->s_ipg)
+#define	ufs_inotocg(x)		((unsigned int)(x) / uspi->s_ipg)
+#define	ufs_inotocgoff(x)	((unsigned int)(x) % uspi->s_ipg)
 #define	ufs_inotofsba(x)	(((u64)ufs_cgimin(ufs_inotocg(x))) + ufs_inotocgoff(x) / uspi->s_inopf)
-#define	ufs_inotofsbo(x)	((x) % uspi->s_inopf)
+#define	ufs_inotofsbo(x)	((unsigned int)(x) % uspi->s_inopf)
 
 /*
  * Compute the cylinder and rotational position of a cyl block addr.
diff --git a/fs/ufs/util.c b/fs/ufs/util.c
index 034b1d82c355c..dff6f74618def 100644
--- a/fs/ufs/util.c
+++ b/fs/ufs/util.c
@@ -203,7 +203,7 @@ struct folio *ufs_get_locked_folio(struct address_space *mapping,
 		folio = read_mapping_folio(mapping, index, NULL);
 
 		if (IS_ERR(folio)) {
-			printk(KERN_ERR "ufs_change_blocknr: read_mapping_folio error: ino %lu, index: %lu\n",
+			printk(KERN_ERR "ufs_change_blocknr: read_mapping_folio error: ino %llu, index: %lu\n",
 			       mapping->host->i_ino, index);
 			return folio;
 		}
diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c
index 543c60c124615..8c4b8e27e9562 100644
--- a/fs/unicode/utf8-core.c
+++ b/fs/unicode/utf8-core.c
@@ -138,28 +138,6 @@ int utf8_casefold_hash(const struct unicode_map *um, const void *salt,
 }
 EXPORT_SYMBOL(utf8_casefold_hash);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen)
-{
-	struct utf8cursor cur;
-	ssize_t nlen = 0;
-
-	if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0)
-		return -EINVAL;
-
-	for (nlen = 0; nlen < dlen; nlen++) {
-		int c = utf8byte(&cur);
-
-		dest[nlen] = c;
-		if (!c)
-			return nlen;
-		if (c == -1)
-			break;
-	}
-	return -EINVAL;
-}
-EXPORT_SYMBOL(utf8_normalize);
-
 static const struct utf8data *find_table_version(const struct utf8data *table,
 		size_t nr_entries, unsigned int version)
 {
diff --git a/fs/verity/init.c b/fs/verity/init.c
index d652066085838..3aa55dec88fc9 100644
--- a/fs/verity/init.c
+++ b/fs/verity/init.c
@@ -50,7 +50,7 @@ void fsverity_msg(const struct inode *inode, const char *level,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 	if (inode)
-		printk("%sfs-verity (%s, inode %lu): %pV\n",
+		printk("%sfs-verity (%s, inode %llu): %pV\n",
 		       level, inode->i_sb->s_id, inode->i_ino, &vaf);
 	else
 		printk("%sfs-verity: %pV\n", level, &vaf);
diff --git a/fs/xattr.c b/fs/xattr.c
index 3e49e612e1bad..09ecbaaa16608 100644
--- a/fs/xattr.c
+++ b/fs/xattr.c
@@ -22,6 +22,7 @@
 #include <linux/audit.h>
 #include <linux/vmalloc.h>
 #include <linux/posix_acl_xattr.h>
+#include <linux/rhashtable.h>
 
 #include <linux/uaccess.h>
 
@@ -105,6 +106,13 @@ int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode)
 	return 0;
 }
 
+static inline int xattr_permission_error(int mask)
+{
+	if (mask & MAY_WRITE)
+		return -EPERM;
+	return -ENODATA;
+}
+
 /*
  * Check permissions for extended attribute access.  This is a bit complicated
  * because different namespaces have very different rules.
@@ -134,7 +142,7 @@ xattr_permission(struct mnt_idmap *idmap, struct inode *inode,
 	 */
 	if (!strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) {
 		if (!capable(CAP_SYS_ADMIN))
-			return (mask & MAY_WRITE) ? -EPERM : -ENODATA;
+			return xattr_permission_error(mask);
 		return 0;
 	}
 
@@ -144,12 +152,22 @@ xattr_permission(struct mnt_idmap *idmap, struct inode *inode,
 	 * privileged users can write attributes.
 	 */
 	if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) {
-		if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
-			return (mask & MAY_WRITE) ? -EPERM : -ENODATA;
-		if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) &&
-		    (mask & MAY_WRITE) &&
-		    !inode_owner_or_capable(idmap, inode))
+		switch (inode->i_mode & S_IFMT) {
+		case S_IFREG:
+			break;
+		case S_IFDIR:
+			if (!(inode->i_mode & S_ISVTX))
+				break;
+			if (!(mask & MAY_WRITE))
+				break;
+			if (inode_owner_or_capable(idmap, inode))
+				break;
 			return -EPERM;
+		case S_IFSOCK:
+			break;
+		default:
+			return xattr_permission_error(mask);
+		}
 	}
 
 	return inode_permission(idmap, inode, mask);
@@ -1197,6 +1215,27 @@ void simple_xattr_free(struct simple_xattr *xattr)
 	kvfree(xattr);
 }
 
+static void simple_xattr_rcu_free(struct rcu_head *head)
+{
+	struct simple_xattr *xattr = container_of(head, struct simple_xattr, rcu);
+
+	simple_xattr_free(xattr);
+}
+
+/**
+ * simple_xattr_free_rcu - free an xattr object with RCU delay
+ * @xattr: the xattr object
+ *
+ * Free the xattr object after an RCU grace period. This must be used when
+ * the xattr was removed from a data structure that concurrent RCU readers
+ * may still be traversing. Can handle @xattr being NULL.
+ */
+void simple_xattr_free_rcu(struct simple_xattr *xattr)
+{
+	if (xattr)
+		call_rcu(&xattr->rcu, simple_xattr_rcu_free);
+}
+
 /**
  * simple_xattr_alloc - allocate new xattr object
  * @value: value of the xattr object
@@ -1205,65 +1244,58 @@ void simple_xattr_free(struct simple_xattr *xattr)
  * Allocate a new xattr object and initialize respective members. The caller is
  * responsible for handling the name of the xattr.
  *
- * Return: On success a new xattr object is returned. On failure NULL is
- * returned.
+ * Return: New xattr object on success, NULL if @value is NULL, ERR_PTR on
+ * failure.
  */
 struct simple_xattr *simple_xattr_alloc(const void *value, size_t size)
 {
 	struct simple_xattr *new_xattr;
 	size_t len;
 
+	if (!value)
+		return NULL;
+
 	/* wrap around? */
 	len = sizeof(*new_xattr) + size;
 	if (len < sizeof(*new_xattr))
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	new_xattr = kvmalloc(len, GFP_KERNEL_ACCOUNT);
 	if (!new_xattr)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	new_xattr->size = size;
 	memcpy(new_xattr->value, value, size);
 	return new_xattr;
 }
 
-/**
- * rbtree_simple_xattr_cmp - compare xattr name with current rbtree xattr entry
- * @key: xattr name
- * @node: current node
- *
- * Compare the xattr name with the xattr name attached to @node in the rbtree.
- *
- * Return: Negative value if continuing left, positive if continuing right, 0
- * if the xattr attached to @node matches @key.
- */
-static int rbtree_simple_xattr_cmp(const void *key, const struct rb_node *node)
+static u32 simple_xattr_hashfn(const void *data, u32 len, u32 seed)
 {
-	const char *xattr_name = key;
-	const struct simple_xattr *xattr;
+	const char *name = data;
+	return jhash(name, strlen(name), seed);
+}
 
-	xattr = rb_entry(node, struct simple_xattr, rb_node);
-	return strcmp(xattr->name, xattr_name);
+static u32 simple_xattr_obj_hashfn(const void *obj, u32 len, u32 seed)
+{
+	const struct simple_xattr *xattr = obj;
+	return jhash(xattr->name, strlen(xattr->name), seed);
 }
 
-/**
- * rbtree_simple_xattr_node_cmp - compare two xattr rbtree nodes
- * @new_node: new node
- * @node: current node
- *
- * Compare the xattr attached to @new_node with the xattr attached to @node.
- *
- * Return: Negative value if continuing left, positive if continuing right, 0
- * if the xattr attached to @new_node matches the xattr attached to @node.
- */
-static int rbtree_simple_xattr_node_cmp(struct rb_node *new_node,
-					const struct rb_node *node)
+static int simple_xattr_obj_cmpfn(struct rhashtable_compare_arg *arg,
+				   const void *obj)
 {
-	struct simple_xattr *xattr;
-	xattr = rb_entry(new_node, struct simple_xattr, rb_node);
-	return rbtree_simple_xattr_cmp(xattr->name, node);
+	const struct simple_xattr *xattr = obj;
+	return strcmp(xattr->name, arg->key);
 }
 
+static const struct rhashtable_params simple_xattr_params = {
+	.head_offset    = offsetof(struct simple_xattr, hash_node),
+	.hashfn         = simple_xattr_hashfn,
+	.obj_hashfn     = simple_xattr_obj_hashfn,
+	.obj_cmpfn      = simple_xattr_obj_cmpfn,
+	.automatic_shrinking = true,
+};
+
 /**
  * simple_xattr_get - get an xattr object
  * @xattrs: the header of the xattr object
@@ -1282,14 +1314,12 @@ static int rbtree_simple_xattr_node_cmp(struct rb_node *new_node,
 int simple_xattr_get(struct simple_xattrs *xattrs, const char *name,
 		     void *buffer, size_t size)
 {
-	struct simple_xattr *xattr = NULL;
-	struct rb_node *rbp;
+	struct simple_xattr *xattr;
 	int ret = -ENODATA;
 
-	read_lock(&xattrs->lock);
-	rbp = rb_find(name, &xattrs->rb_root, rbtree_simple_xattr_cmp);
-	if (rbp) {
-		xattr = rb_entry(rbp, struct simple_xattr, rb_node);
+	guard(rcu)();
+	xattr = rhashtable_lookup(&xattrs->ht, name, simple_xattr_params);
+	if (xattr) {
 		ret = xattr->size;
 		if (buffer) {
 			if (size < xattr->size)
@@ -1298,7 +1328,6 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name,
 				memcpy(buffer, xattr->value, xattr->size);
 		}
 	}
-	read_unlock(&xattrs->lock);
 	return ret;
 }
 
@@ -1325,6 +1354,11 @@ int simple_xattr_get(struct simple_xattrs *xattrs, const char *name,
  * nothing if XATTR_CREATE is specified in @flags or @flags is zero. For
  * XATTR_REPLACE we fail as mentioned above.
  *
+ * Note: Callers must externally serialize writes. All current callers hold
+ * the inode lock for write operations. The lookup->replace/remove sequence
+ * is not atomic with respect to the rhashtable's per-bucket locking, but
+ * is safe because writes are serialized by the caller.
+ *
  * Return: On success, the removed or replaced xattr is returned, to be freed
  * by the caller; or NULL if none. On failure a negative error code is returned.
  */
@@ -1332,64 +1366,57 @@ struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs,
 				      const char *name, const void *value,
 				      size_t size, int flags)
 {
-	struct simple_xattr *old_xattr = NULL, *new_xattr = NULL;
-	struct rb_node *parent = NULL, **rbp;
-	int err = 0, ret;
+	struct simple_xattr *old_xattr = NULL;
+	int err;
 
-	/* value == NULL means remove */
-	if (value) {
-		new_xattr = simple_xattr_alloc(value, size);
-		if (!new_xattr)
-			return ERR_PTR(-ENOMEM);
+	CLASS(simple_xattr, new_xattr)(value, size);
+	if (IS_ERR(new_xattr))
+		return new_xattr;
 
+	if (new_xattr) {
 		new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
-		if (!new_xattr->name) {
-			simple_xattr_free(new_xattr);
+		if (!new_xattr->name)
 			return ERR_PTR(-ENOMEM);
-		}
 	}
 
-	write_lock(&xattrs->lock);
-	rbp = &xattrs->rb_root.rb_node;
-	while (*rbp) {
-		parent = *rbp;
-		ret = rbtree_simple_xattr_cmp(name, *rbp);
-		if (ret < 0)
-			rbp = &(*rbp)->rb_left;
-		else if (ret > 0)
-			rbp = &(*rbp)->rb_right;
-		else
-			old_xattr = rb_entry(*rbp, struct simple_xattr, rb_node);
-		if (old_xattr)
-			break;
-	}
+	/* Lookup is safe without RCU here since writes are serialized. */
+	old_xattr = rhashtable_lookup_fast(&xattrs->ht, name,
+					   simple_xattr_params);
 
 	if (old_xattr) {
 		/* Fail if XATTR_CREATE is requested and the xattr exists. */
-		if (flags & XATTR_CREATE) {
-			err = -EEXIST;
-			goto out_unlock;
-		}
+		if (flags & XATTR_CREATE)
+			return ERR_PTR(-EEXIST);
 
-		if (new_xattr)
-			rb_replace_node(&old_xattr->rb_node,
-					&new_xattr->rb_node, &xattrs->rb_root);
-		else
-			rb_erase(&old_xattr->rb_node, &xattrs->rb_root);
+		if (new_xattr) {
+			err = rhashtable_replace_fast(&xattrs->ht,
+						      &old_xattr->hash_node,
+						      &new_xattr->hash_node,
+						      simple_xattr_params);
+			if (err)
+				return ERR_PTR(err);
+		} else {
+			err = rhashtable_remove_fast(&xattrs->ht,
+						     &old_xattr->hash_node,
+						     simple_xattr_params);
+			if (err)
+				return ERR_PTR(err);
+		}
 	} else {
 		/* Fail if XATTR_REPLACE is requested but no xattr is found. */
-		if (flags & XATTR_REPLACE) {
-			err = -ENODATA;
-			goto out_unlock;
-		}
+		if (flags & XATTR_REPLACE)
+			return ERR_PTR(-ENODATA);
 
 		/*
 		 * If XATTR_CREATE or no flags are specified together with a
 		 * new value simply insert it.
 		 */
 		if (new_xattr) {
-			rb_link_node(&new_xattr->rb_node, parent, rbp);
-			rb_insert_color(&new_xattr->rb_node, &xattrs->rb_root);
+			err = rhashtable_insert_fast(&xattrs->ht,
+						     &new_xattr->hash_node,
+						     simple_xattr_params);
+			if (err)
+				return ERR_PTR(err);
 		}
 
 		/*
@@ -1398,12 +1425,73 @@ struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs,
 		 */
 	}
 
-out_unlock:
-	write_unlock(&xattrs->lock);
-	if (!err)
-		return old_xattr;
-	simple_xattr_free(new_xattr);
-	return ERR_PTR(err);
+	retain_and_null_ptr(new_xattr);
+	return old_xattr;
+}
+
+static inline void simple_xattr_limits_dec(struct simple_xattr_limits *limits,
+					   size_t size)
+{
+	atomic_sub(size, &limits->xattr_size);
+	atomic_dec(&limits->nr_xattrs);
+}
+
+static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits,
+					  size_t size)
+{
+	if (atomic_inc_return(&limits->nr_xattrs) > SIMPLE_XATTR_MAX_NR) {
+		atomic_dec(&limits->nr_xattrs);
+		return -ENOSPC;
+	}
+
+	if (atomic_add_return(size, &limits->xattr_size) <= SIMPLE_XATTR_MAX_SIZE)
+		return 0;
+
+	simple_xattr_limits_dec(limits, size);
+	return -ENOSPC;
+}
+
+/**
+ * simple_xattr_set_limited - set an xattr with per-inode user.* limits
+ * @xattrs: the header of the xattr object
+ * @limits: per-inode limit counters for user.* xattrs
+ * @name: the name of the xattr to set or remove
+ * @value: the value to store (NULL to remove)
+ * @size: the size of @value
+ * @flags: XATTR_CREATE, XATTR_REPLACE, or 0
+ *
+ * Like simple_xattr_set(), but enforces per-inode count and total value size
+ * limits for user.* xattrs. Uses speculative pre-increment of the atomic
+ * counters to avoid races without requiring external locks.
+ *
+ * Return: On success zero is returned. On failure a negative error code is
+ * returned.
+ */
+int simple_xattr_set_limited(struct simple_xattrs *xattrs,
+			     struct simple_xattr_limits *limits,
+			     const char *name, const void *value,
+			     size_t size, int flags)
+{
+	struct simple_xattr *old_xattr;
+	int ret;
+
+	if (value) {
+		ret = simple_xattr_limits_inc(limits, size);
+		if (ret)
+			return ret;
+	}
+
+	old_xattr = simple_xattr_set(xattrs, name, value, size, flags);
+	if (IS_ERR(old_xattr)) {
+		if (value)
+			simple_xattr_limits_dec(limits, size);
+		return PTR_ERR(old_xattr);
+	}
+	if (old_xattr) {
+		simple_xattr_limits_dec(limits, old_xattr->size);
+		simple_xattr_free_rcu(old_xattr);
+	}
+	return 0;
 }
 
 static bool xattr_is_trusted(const char *name)
@@ -1443,8 +1531,8 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
 			  char *buffer, size_t size)
 {
 	bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN);
+	struct rhashtable_iter iter;
 	struct simple_xattr *xattr;
-	struct rb_node *rbp;
 	ssize_t remaining_size = size;
 	int err = 0;
 
@@ -1464,9 +1552,19 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
 	remaining_size -= err;
 	err = 0;
 
-	read_lock(&xattrs->lock);
-	for (rbp = rb_first(&xattrs->rb_root); rbp; rbp = rb_next(rbp)) {
-		xattr = rb_entry(rbp, struct simple_xattr, rb_node);
+	if (!xattrs)
+		return size - remaining_size;
+
+	rhashtable_walk_enter(&xattrs->ht, &iter);
+	rhashtable_walk_start(&iter);
+
+	while ((xattr = rhashtable_walk_next(&iter)) != NULL) {
+		if (IS_ERR(xattr)) {
+			if (PTR_ERR(xattr) == -EAGAIN)
+				continue;
+			err = PTR_ERR(xattr);
+			break;
+		}
 
 		/* skip "trusted." attributes for unprivileged callers */
 		if (!trusted && xattr_is_trusted(xattr->name))
@@ -1480,25 +1578,11 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
 		if (err)
 			break;
 	}
-	read_unlock(&xattrs->lock);
 
-	return err ? err : size - remaining_size;
-}
+	rhashtable_walk_stop(&iter);
+	rhashtable_walk_exit(&iter);
 
-/**
- * rbtree_simple_xattr_less - compare two xattr rbtree nodes
- * @new_node: new node
- * @node: current node
- *
- * Compare the xattr attached to @new_node with the xattr attached to @node.
- * Note that this function technically tolerates duplicate entries.
- *
- * Return: True if insertion point in the rbtree is found.
- */
-static bool rbtree_simple_xattr_less(struct rb_node *new_node,
-				     const struct rb_node *node)
-{
-	return rbtree_simple_xattr_node_cmp(new_node, node) < 0;
+	return err ? err : size - remaining_size;
 }
 
 /**
@@ -1509,25 +1593,100 @@ static bool rbtree_simple_xattr_less(struct rb_node *new_node,
  * Add an xattr object to @xattrs. This assumes no replacement or removal
  * of matching xattrs is wanted. Should only be called during inode
  * initialization when a few distinct initial xattrs are supposed to be set.
+ *
+ * Return: On success zero is returned. On failure a negative error code is
+ * returned.
  */
-void simple_xattr_add(struct simple_xattrs *xattrs,
-		      struct simple_xattr *new_xattr)
+int simple_xattr_add(struct simple_xattrs *xattrs,
+		     struct simple_xattr *new_xattr)
 {
-	write_lock(&xattrs->lock);
-	rb_add(&new_xattr->rb_node, &xattrs->rb_root, rbtree_simple_xattr_less);
-	write_unlock(&xattrs->lock);
+	return rhashtable_insert_fast(&xattrs->ht, &new_xattr->hash_node,
+				      simple_xattr_params);
 }
 
 /**
  * simple_xattrs_init - initialize new xattr header
  * @xattrs: header to initialize
  *
- * Initialize relevant fields of a an xattr header.
+ * Initialize the rhashtable used to store xattr objects.
+ *
+ * Return: On success zero is returned. On failure a negative error code is
+ * returned.
+ */
+int simple_xattrs_init(struct simple_xattrs *xattrs)
+{
+	return rhashtable_init(&xattrs->ht, &simple_xattr_params);
+}
+
+/**
+ * simple_xattrs_alloc - allocate and initialize a new xattr header
+ *
+ * Dynamically allocate a simple_xattrs header and initialize the
+ * underlying rhashtable. This is intended for consumers that want
+ * to lazily allocate xattr storage only when the first xattr is set,
+ * avoiding the per-inode rhashtable overhead when no xattrs are used.
+ *
+ * Return: On success a new simple_xattrs is returned. On failure an
+ * ERR_PTR is returned.
+ */
+struct simple_xattrs *simple_xattrs_alloc(void)
+{
+	struct simple_xattrs *xattrs __free(kfree) = NULL;
+	int ret;
+
+	xattrs = kzalloc(sizeof(*xattrs), GFP_KERNEL);
+	if (!xattrs)
+		return ERR_PTR(-ENOMEM);
+
+	ret = simple_xattrs_init(xattrs);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return no_free_ptr(xattrs);
+}
+
+/**
+ * simple_xattrs_lazy_alloc - get or allocate xattrs for a set operation
+ * @xattrsp: pointer to the xattrs pointer (may point to NULL)
+ * @value: value being set (NULL means remove)
+ * @flags: xattr set flags
+ *
+ * For lazily-allocated xattrs on the write path. If no xattrs exist yet
+ * and this is a remove operation, returns the appropriate result without
+ * allocating. Otherwise ensures xattrs is allocated and published with
+ * store-release semantics.
+ *
+ * Return: On success a valid pointer to the xattrs is returned. On
+ * failure or early-exit an ERR_PTR or NULL is returned. Callers should
+ * check with IS_ERR_OR_NULL() and propagate with PTR_ERR() which
+ * correctly returns 0 for the NULL no-op case.
  */
-void simple_xattrs_init(struct simple_xattrs *xattrs)
+struct simple_xattrs *simple_xattrs_lazy_alloc(struct simple_xattrs **xattrsp,
+					       const void *value, int flags)
 {
-	xattrs->rb_root = RB_ROOT;
-	rwlock_init(&xattrs->lock);
+	struct simple_xattrs *xattrs;
+
+	xattrs = READ_ONCE(*xattrsp);
+	if (xattrs)
+		return xattrs;
+
+	if (!value)
+		return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL;
+
+	xattrs = simple_xattrs_alloc();
+	if (!IS_ERR(xattrs))
+		smp_store_release(xattrsp, xattrs);
+	return xattrs;
+}
+
+static void simple_xattr_ht_free(void *ptr, void *arg)
+{
+	struct simple_xattr *xattr = ptr;
+	size_t *freed_space = arg;
+
+	if (freed_space)
+		*freed_space += simple_xattr_space(xattr->name, xattr->size);
+	simple_xattr_free(xattr);
 }
 
 /**
@@ -1540,22 +1699,10 @@ void simple_xattrs_init(struct simple_xattrs *xattrs)
  */
 void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space)
 {
-	struct rb_node *rbp;
+	might_sleep();
 
 	if (freed_space)
 		*freed_space = 0;
-	rbp = rb_first(&xattrs->rb_root);
-	while (rbp) {
-		struct simple_xattr *xattr;
-		struct rb_node *rbp_next;
-
-		rbp_next = rb_next(rbp);
-		xattr = rb_entry(rbp, struct simple_xattr, rb_node);
-		rb_erase(&xattr->rb_node, &xattrs->rb_root);
-		if (freed_space)
-			*freed_space += simple_xattr_space(xattr->name,
-							   xattr->size);
-		simple_xattr_free(xattr);
-		rbp = rbp_next;
-	}
+	rhashtable_free_and_destroy(&xattrs->ht, simple_xattr_ht_free,
+				    freed_space);
 }
diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h
index 8244305949deb..67fd9c75ac3fb 100644
--- a/fs/xfs/libxfs/xfs_attr.h
+++ b/fs/xfs/libxfs/xfs_attr.h
@@ -55,7 +55,8 @@ struct xfs_attr_list_context {
 	struct xfs_trans	*tp;
 	struct xfs_inode	*dp;		/* inode */
 	struct xfs_attrlist_cursor_kern cursor;	/* position in list */
-	void			*buffer;	/* output buffer */
+	/* output buffer */
+	void			*buffer __counted_by_ptr(bufsize);
 
 	/*
 	 * Abort attribute list iteration if non-zero.  Can be used to pass
diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h
index d165de607d179..185f09f327c00 100644
--- a/fs/xfs/libxfs/xfs_fs.h
+++ b/fs/xfs/libxfs/xfs_fs.h
@@ -995,7 +995,8 @@ struct xfs_rtgroup_geometry {
 	__u32 rg_sick;		/* o: sick things in ag */
 	__u32 rg_checked;	/* o: checked metadata in ag */
 	__u32 rg_flags;		/* i/o: flags for this ag */
-	__u32 rg_reserved[27];	/* o: zero */
+	__u32 rg_writepointer;  /* o: write pointer block offset for zoned */
+	__u32 rg_reserved[26];	/* o: zero */
 };
 #define XFS_RTGROUP_GEOM_SICK_SUPER	(1U << 0)  /* superblock */
 #define XFS_RTGROUP_GEOM_SICK_BITMAP	(1U << 1)  /* rtbitmap */
@@ -1003,6 +1004,8 @@ struct xfs_rtgroup_geometry {
 #define XFS_RTGROUP_GEOM_SICK_RMAPBT	(1U << 3)  /* reverse mappings */
 #define XFS_RTGROUP_GEOM_SICK_REFCNTBT	(1U << 4)  /* reference counts */
 
+#define XFS_RTGROUP_GEOM_WRITEPOINTER  (1U << 0)  /* write pointer */
+
 /* Health monitor event domains */
 
 /* affects the whole fs */
diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c
index 1d25bd5b892e0..222812fe202c2 100644
--- a/fs/xfs/scrub/quota.c
+++ b/fs/xfs/scrub/quota.c
@@ -171,8 +171,10 @@ xchk_quota_item(
 
 	error = xchk_quota_item_bmap(sc, dq, offset);
 	xchk_iunlock(sc, XFS_ILOCK_SHARED);
-	if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error))
+	if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) {
+		mutex_unlock(&dq->q_qlock);
 		return error;
+	}
 
 	/*
 	 * Warn if the hard limits are larger than the fs.
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 76678814f46fe..f279055fcea03 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -22,6 +22,7 @@
 #include "xfs_icache.h"
 #include "xfs_zone_alloc.h"
 #include "xfs_rtgroup.h"
+#include <linux/bio-integrity.h>
 
 struct xfs_writepage_ctx {
 	struct iomap_writepage_ctx ctx;
@@ -661,6 +662,8 @@ xfs_zoned_writeback_submit(
 		bio_endio(&ioend->io_bio);
 		return error;
 	}
+	if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
+		fs_bio_integrity_generate(&ioend->io_bio);
 	xfs_zone_alloc_and_submit(ioend, &XFS_ZWPC(wpc)->open_zone);
 	return 0;
 }
@@ -741,12 +744,45 @@ xfs_vm_bmap(
 	return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
 }
 
+static void
+xfs_bio_submit_read(
+	const struct iomap_iter		*iter,
+	struct iomap_read_folio_ctx	*ctx)
+{
+	struct bio			*bio = ctx->read_ctx;
+
+	/* defer read completions to the ioend workqueue */
+	iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0);
+	bio->bi_end_io = xfs_end_bio;
+	submit_bio(bio);
+}
+
+static const struct iomap_read_ops xfs_iomap_read_ops = {
+	.read_folio_range	= iomap_bio_read_folio_range,
+	.submit_read		= xfs_bio_submit_read,
+	.bio_set		= &iomap_ioend_bioset,
+};
+
+static inline const struct iomap_read_ops *
+xfs_get_iomap_read_ops(
+	const struct address_space	*mapping)
+{
+	struct xfs_inode		*ip = XFS_I(mapping->host);
+
+	if (bdev_has_integrity_csum(xfs_inode_buftarg(ip)->bt_bdev))
+		return &xfs_iomap_read_ops;
+	return &iomap_bio_read_ops;
+}
+
 STATIC int
 xfs_vm_read_folio(
-	struct file		*unused,
-	struct folio		*folio)
+	struct file			*file,
+	struct folio			*folio)
 {
-	iomap_bio_read_folio(folio, &xfs_read_iomap_ops);
+	struct iomap_read_folio_ctx	ctx = { .cur_folio = folio };
+
+	ctx.ops = xfs_get_iomap_read_ops(folio->mapping);
+	iomap_read_folio(&xfs_read_iomap_ops, &ctx, NULL);
 	return 0;
 }
 
@@ -754,7 +790,10 @@ STATIC void
 xfs_vm_readahead(
 	struct readahead_control	*rac)
 {
-	iomap_bio_readahead(rac, &xfs_read_iomap_ops);
+	struct iomap_read_folio_ctx	ctx = { .rac = rac };
+
+	ctx.ops = xfs_get_iomap_read_ops(rac->mapping),
+	iomap_readahead(&xfs_read_iomap_ops, &ctx, NULL);
 }
 
 static int
diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c
index 491e2a7053a39..65a0e69c3d083 100644
--- a/fs/xfs/xfs_dquot_item.c
+++ b/fs/xfs/xfs_dquot_item.c
@@ -125,6 +125,7 @@ xfs_qm_dquot_logitem_push(
 	struct xfs_dq_logitem	*qlip = DQUOT_ITEM(lip);
 	struct xfs_dquot	*dqp = qlip->qli_dquot;
 	struct xfs_buf		*bp;
+	struct xfs_ail		*ailp = lip->li_ailp;
 	uint			rval = XFS_ITEM_SUCCESS;
 	int			error;
 
@@ -153,7 +154,7 @@ xfs_qm_dquot_logitem_push(
 		goto out_unlock;
 	}
 
-	spin_unlock(&lip->li_ailp->ail_lock);
+	spin_unlock(&ailp->ail_lock);
 
 	error = xfs_dquot_use_attached_buf(dqp, &bp);
 	if (error == -EAGAIN) {
@@ -172,9 +173,13 @@ xfs_qm_dquot_logitem_push(
 			rval = XFS_ITEM_FLUSHING;
 	}
 	xfs_buf_relse(bp);
+	/*
+	 * The buffer no longer protects the log item from reclaim, so
+	 * do not reference lip after this point.
+	 */
 
 out_relock_ail:
-	spin_lock(&lip->li_ailp->ail_lock);
+	spin_lock(&ailp->ail_lock);
 out_unlock:
 	mutex_unlock(&dqp->q_qlock);
 	return rval;
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index 6246f34df9fd1..7918968e1d623 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -560,6 +560,72 @@ xfs_zoned_write_space_reserve(
 			flags, ac);
 }
 
+/*
+ * We need to lock the test/set EOF update as we can be racing with
+ * other IO completions here to update the EOF. Failing to serialise
+ * here can result in EOF moving backwards and Bad Things Happen when
+ * that occurs.
+ *
+ * As IO completion only ever extends EOF, we can do an unlocked check
+ * here to avoid taking the spinlock. If we land within the current EOF,
+ * then we do not need to do an extending update at all, and we don't
+ * need to take the lock to check this. If we race with an update moving
+ * EOF, then we'll either still be beyond EOF and need to take the lock,
+ * or we'll be within EOF and we don't need to take it at all.
+ */
+static int
+xfs_dio_endio_set_isize(
+	struct inode		*inode,
+	loff_t			offset,
+	ssize_t			size)
+{
+	struct xfs_inode	*ip = XFS_I(inode);
+
+	if (offset + size <= i_size_read(inode))
+		return 0;
+
+	spin_lock(&ip->i_flags_lock);
+	if (offset + size <= i_size_read(inode)) {
+		spin_unlock(&ip->i_flags_lock);
+		return 0;
+	}
+
+	i_size_write(inode, offset + size);
+	spin_unlock(&ip->i_flags_lock);
+
+	return xfs_setfilesize(ip, offset, size);
+}
+
+static int
+xfs_zoned_dio_write_end_io(
+	struct kiocb		*iocb,
+	ssize_t			size,
+	int			error,
+	unsigned		flags)
+{
+	struct inode		*inode = file_inode(iocb->ki_filp);
+	struct xfs_inode	*ip = XFS_I(inode);
+	unsigned int		nofs_flag;
+
+	ASSERT(!(flags & (IOMAP_DIO_UNWRITTEN | IOMAP_DIO_COW)));
+
+	trace_xfs_end_io_direct_write(ip, iocb->ki_pos, size);
+
+	if (xfs_is_shutdown(ip->i_mount))
+		return -EIO;
+
+	if (error || !size)
+		return error;
+
+	XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
+
+	nofs_flag = memalloc_nofs_save();
+	error = xfs_dio_endio_set_isize(inode, iocb->ki_pos, size);
+	memalloc_nofs_restore(nofs_flag);
+
+	return error;
+}
+
 static int
 xfs_dio_write_end_io(
 	struct kiocb		*iocb,
@@ -572,8 +638,7 @@ xfs_dio_write_end_io(
 	loff_t			offset = iocb->ki_pos;
 	unsigned int		nofs_flag;
 
-	ASSERT(!xfs_is_zoned_inode(ip) ||
-	       !(flags & (IOMAP_DIO_UNWRITTEN | IOMAP_DIO_COW)));
+	ASSERT(!xfs_is_zoned_inode(ip));
 
 	trace_xfs_end_io_direct_write(ip, offset, size);
 
@@ -623,30 +688,8 @@ xfs_dio_write_end_io(
 	 * with the on-disk inode size being outside the in-core inode size. We
 	 * have no other method of updating EOF for AIO, so always do it here
 	 * if necessary.
-	 *
-	 * We need to lock the test/set EOF update as we can be racing with
-	 * other IO completions here to update the EOF. Failing to serialise
-	 * here can result in EOF moving backwards and Bad Things Happen when
-	 * that occurs.
-	 *
-	 * As IO completion only ever extends EOF, we can do an unlocked check
-	 * here to avoid taking the spinlock. If we land within the current EOF,
-	 * then we do not need to do an extending update at all, and we don't
-	 * need to take the lock to check this. If we race with an update moving
-	 * EOF, then we'll either still be beyond EOF and need to take the lock,
-	 * or we'll be within EOF and we don't need to take it at all.
 	 */
-	if (offset + size <= i_size_read(inode))
-		goto out;
-
-	spin_lock(&ip->i_flags_lock);
-	if (offset + size > i_size_read(inode)) {
-		i_size_write(inode, offset + size);
-		spin_unlock(&ip->i_flags_lock);
-		error = xfs_setfilesize(ip, offset, size);
-	} else {
-		spin_unlock(&ip->i_flags_lock);
-	}
+	error = xfs_dio_endio_set_isize(inode, offset, size);
 
 out:
 	memalloc_nofs_restore(nofs_flag);
@@ -688,7 +731,7 @@ xfs_dio_zoned_submit_io(
 static const struct iomap_dio_ops xfs_dio_zoned_write_ops = {
 	.bio_set	= &iomap_ioend_bioset,
 	.submit_io	= xfs_dio_zoned_submit_io,
-	.end_io		= xfs_dio_write_end_io,
+	.end_io		= xfs_zoned_dio_write_end_io,
 };
 
 /*
diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c
index d1291ca152394..2b8617ae7ec24 100644
--- a/fs/xfs/xfs_handle.c
+++ b/fs/xfs/xfs_handle.c
@@ -443,8 +443,8 @@ xfs_ioc_attr_list(
 	context.dp = dp;
 	context.resynch = 1;
 	context.attr_filter = xfs_attr_filter(flags);
-	context.buffer = buffer;
 	context.bufsize = round_down(bufsize, sizeof(uint32_t));
+	context.buffer = buffer;
 	context.firstu = context.bufsize;
 	context.put_listent = xfs_ioc_attr_put_listent;
 
diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c
index 8913036b8024b..4ae81eed0442c 100644
--- a/fs/xfs/xfs_inode_item.c
+++ b/fs/xfs/xfs_inode_item.c
@@ -746,6 +746,7 @@ xfs_inode_item_push(
 	struct xfs_inode_log_item *iip = INODE_ITEM(lip);
 	struct xfs_inode	*ip = iip->ili_inode;
 	struct xfs_buf		*bp = lip->li_buf;
+	struct xfs_ail		*ailp = lip->li_ailp;
 	uint			rval = XFS_ITEM_SUCCESS;
 	int			error;
 
@@ -771,7 +772,7 @@ xfs_inode_item_push(
 	if (!xfs_buf_trylock(bp))
 		return XFS_ITEM_LOCKED;
 
-	spin_unlock(&lip->li_ailp->ail_lock);
+	spin_unlock(&ailp->ail_lock);
 
 	/*
 	 * We need to hold a reference for flushing the cluster buffer as it may
@@ -795,7 +796,11 @@ xfs_inode_item_push(
 		rval = XFS_ITEM_LOCKED;
 	}
 
-	spin_lock(&lip->li_ailp->ail_lock);
+	/*
+	 * The buffer no longer protects the log item from reclaim, so
+	 * do not reference lip after this point.
+	 */
+	spin_lock(&ailp->ail_lock);
 	return rval;
 }
 
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index facffdc8dca83..46e234863644f 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -37,12 +37,15 @@
 #include "xfs_ioctl.h"
 #include "xfs_xattr.h"
 #include "xfs_rtbitmap.h"
+#include "xfs_rtrmap_btree.h"
 #include "xfs_file.h"
 #include "xfs_exchrange.h"
 #include "xfs_handle.h"
 #include "xfs_rtgroup.h"
 #include "xfs_healthmon.h"
 #include "xfs_verify_media.h"
+#include "xfs_zone_priv.h"
+#include "xfs_zone_alloc.h"
 
 #include <linux/mount.h>
 #include <linux/fileattr.h>
@@ -413,6 +416,7 @@ xfs_ioc_rtgroup_geometry(
 {
 	struct xfs_rtgroup	*rtg;
 	struct xfs_rtgroup_geometry rgeo;
+	xfs_rgblock_t		highest_rgbno;
 	int			error;
 
 	if (copy_from_user(&rgeo, arg, sizeof(rgeo)))
@@ -433,6 +437,21 @@ xfs_ioc_rtgroup_geometry(
 	if (error)
 		return error;
 
+	if (xfs_has_zoned(mp)) {
+		xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
+		if (rtg->rtg_open_zone) {
+			rgeo.rg_writepointer = rtg->rtg_open_zone->oz_allocated;
+		} else {
+			highest_rgbno = xfs_rtrmap_highest_rgbno(rtg);
+			if (highest_rgbno == NULLRGBLOCK)
+				rgeo.rg_writepointer = 0;
+			else
+				rgeo.rg_writepointer = highest_rgbno + 1;
+		}
+		xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP);
+		rgeo.rg_flags |= XFS_RTGROUP_GEOM_WRITEPOINTER;
+	}
+
 	if (copy_to_user(arg, &rgeo, sizeof(rgeo)))
 		return -EFAULT;
 	return 0;
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
index be86d43044df8..9c2f12d5fec9d 100644
--- a/fs/xfs/xfs_iomap.c
+++ b/fs/xfs/xfs_iomap.c
@@ -143,11 +143,14 @@ xfs_bmbt_to_iomap(
 	}
 	iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
 	iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
-	if (mapping_flags & IOMAP_DAX)
+	iomap->flags = iomap_flags;
+	if (mapping_flags & IOMAP_DAX) {
 		iomap->dax_dev = target->bt_daxdev;
-	else
+	} else {
 		iomap->bdev = target->bt_bdev;
-	iomap->flags = iomap_flags;
+		if (bdev_has_integrity_csum(iomap->bdev))
+			iomap->flags |= IOMAP_F_INTEGRITY;
+	}
 
 	/*
 	 * If the inode is dirty for datasync purposes, let iomap know so it
diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
index 9c295abd0a0ac..ef1ea8a1238c6 100644
--- a/fs/xfs/xfs_mount.c
+++ b/fs/xfs/xfs_mount.c
@@ -608,8 +608,9 @@ xfs_unmount_check(
  * have been retrying in the background.  This will prevent never-ending
  * retries in AIL pushing from hanging the unmount.
  *
- * Finally, we can push the AIL to clean all the remaining dirty objects, then
- * reclaim the remaining inodes that are still in memory at this point in time.
+ * Stop inodegc and background reclaim before pushing the AIL so that they
+ * are not running while the AIL is being flushed. Then push the AIL to
+ * clean all the remaining dirty objects and reclaim the remaining inodes.
  */
 static void
 xfs_unmount_flush_inodes(
@@ -621,9 +622,9 @@ xfs_unmount_flush_inodes(
 
 	xfs_set_unmounting(mp);
 
-	xfs_ail_push_all_sync(mp->m_ail);
 	xfs_inodegc_stop(mp);
 	cancel_delayed_work_sync(&mp->m_reclaim_work);
+	xfs_ail_push_all_sync(mp->m_ail);
 	xfs_reclaim_inodes(mp);
 	xfs_health_unmount(mp);
 	xfs_healthmon_unmount(mp);
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 813e5a9f57eb7..0e994b3f768f9 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -56,6 +56,7 @@
 #include <linux/tracepoint.h>
 
 struct xfs_agf;
+struct xfs_ail;
 struct xfs_alloc_arg;
 struct xfs_attr_list_context;
 struct xfs_buf_log_item;
@@ -1650,16 +1651,43 @@ TRACE_EVENT(xfs_log_force,
 DEFINE_EVENT(xfs_log_item_class, name, \
 	TP_PROTO(struct xfs_log_item *lip), \
 	TP_ARGS(lip))
-DEFINE_LOG_ITEM_EVENT(xfs_ail_push);
-DEFINE_LOG_ITEM_EVENT(xfs_ail_pinned);
-DEFINE_LOG_ITEM_EVENT(xfs_ail_locked);
-DEFINE_LOG_ITEM_EVENT(xfs_ail_flushing);
 DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_mark);
 DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_skip);
 DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_unpin);
 DEFINE_LOG_ITEM_EVENT(xlog_ail_insert_abort);
 DEFINE_LOG_ITEM_EVENT(xfs_trans_free_abort);
 
+DECLARE_EVENT_CLASS(xfs_ail_push_class,
+	TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn),
+	TP_ARGS(ailp, type, flags, lsn),
+	TP_STRUCT__entry(
+		__field(dev_t, dev)
+		__field(uint, type)
+		__field(unsigned long, flags)
+		__field(xfs_lsn_t, lsn)
+	),
+	TP_fast_assign(
+		__entry->dev = ailp->ail_log->l_mp->m_super->s_dev;
+		__entry->type = type;
+		__entry->flags = flags;
+		__entry->lsn = lsn;
+	),
+	TP_printk("dev %d:%d lsn %d/%d type %s flags %s",
+		  MAJOR(__entry->dev), MINOR(__entry->dev),
+		  CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn),
+		  __print_symbolic(__entry->type, XFS_LI_TYPE_DESC),
+		  __print_flags(__entry->flags, "|", XFS_LI_FLAGS))
+)
+
+#define DEFINE_AIL_PUSH_EVENT(name) \
+DEFINE_EVENT(xfs_ail_push_class, name, \
+	TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), \
+	TP_ARGS(ailp, type, flags, lsn))
+DEFINE_AIL_PUSH_EVENT(xfs_ail_push);
+DEFINE_AIL_PUSH_EVENT(xfs_ail_pinned);
+DEFINE_AIL_PUSH_EVENT(xfs_ail_locked);
+DEFINE_AIL_PUSH_EVENT(xfs_ail_flushing);
+
 DECLARE_EVENT_CLASS(xfs_ail_class,
 	TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn),
 	TP_ARGS(lip, old_lsn, new_lsn),
diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c
index 923729af42067..99a9bf3762b7e 100644
--- a/fs/xfs/xfs_trans_ail.c
+++ b/fs/xfs/xfs_trans_ail.c
@@ -365,6 +365,12 @@ xfsaild_resubmit_item(
 	return XFS_ITEM_SUCCESS;
 }
 
+/*
+ * Push a single log item from the AIL.
+ *
+ * @lip may have been released and freed by the time this function returns,
+ * so callers must not dereference the log item afterwards.
+ */
 static inline uint
 xfsaild_push_item(
 	struct xfs_ail		*ailp,
@@ -458,6 +464,74 @@ xfs_ail_calc_push_target(
 	return target_lsn;
 }
 
+static void
+xfsaild_process_logitem(
+	struct xfs_ail		*ailp,
+	struct xfs_log_item	*lip,
+	int			*stuck,
+	int			*flushing)
+{
+	struct xfs_mount	*mp = ailp->ail_log->l_mp;
+	uint			type = lip->li_type;
+	unsigned long		flags = lip->li_flags;
+	xfs_lsn_t		item_lsn = lip->li_lsn;
+	int			lock_result;
+
+	/*
+	 * Note that iop_push may unlock and reacquire the AIL lock. We
+	 * rely on the AIL cursor implementation to be able to deal with
+	 * the dropped lock.
+	 *
+	 * The log item may have been freed by the push, so it must not
+	 * be accessed or dereferenced below this line.
+	 */
+	lock_result = xfsaild_push_item(ailp, lip);
+	switch (lock_result) {
+	case XFS_ITEM_SUCCESS:
+		XFS_STATS_INC(mp, xs_push_ail_success);
+		trace_xfs_ail_push(ailp, type, flags, item_lsn);
+
+		ailp->ail_last_pushed_lsn = item_lsn;
+		break;
+
+	case XFS_ITEM_FLUSHING:
+		/*
+		 * The item or its backing buffer is already being
+		 * flushed.  The typical reason for that is that an
+		 * inode buffer is locked because we already pushed the
+		 * updates to it as part of inode clustering.
+		 *
+		 * We do not want to stop flushing just because lots
+		 * of items are already being flushed, but we need to
+		 * re-try the flushing relatively soon if most of the
+		 * AIL is being flushed.
+		 */
+		XFS_STATS_INC(mp, xs_push_ail_flushing);
+		trace_xfs_ail_flushing(ailp, type, flags, item_lsn);
+
+		(*flushing)++;
+		ailp->ail_last_pushed_lsn = item_lsn;
+		break;
+
+	case XFS_ITEM_PINNED:
+		XFS_STATS_INC(mp, xs_push_ail_pinned);
+		trace_xfs_ail_pinned(ailp, type, flags, item_lsn);
+
+		(*stuck)++;
+		ailp->ail_log_flush++;
+		break;
+	case XFS_ITEM_LOCKED:
+		XFS_STATS_INC(mp, xs_push_ail_locked);
+		trace_xfs_ail_locked(ailp, type, flags, item_lsn);
+
+		(*stuck)++;
+		break;
+	default:
+		ASSERT(0);
+		break;
+	}
+}
+
 static long
 xfsaild_push(
 	struct xfs_ail		*ailp)
@@ -505,62 +579,11 @@ xfsaild_push(
 
 	lsn = lip->li_lsn;
 	while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) {
-		int	lock_result;
 
 		if (test_bit(XFS_LI_FLUSHING, &lip->li_flags))
 			goto next_item;
 
-		/*
-		 * Note that iop_push may unlock and reacquire the AIL lock.  We
-		 * rely on the AIL cursor implementation to be able to deal with
-		 * the dropped lock.
-		 */
-		lock_result = xfsaild_push_item(ailp, lip);
-		switch (lock_result) {
-		case XFS_ITEM_SUCCESS:
-			XFS_STATS_INC(mp, xs_push_ail_success);
-			trace_xfs_ail_push(lip);
-
-			ailp->ail_last_pushed_lsn = lsn;
-			break;
-
-		case XFS_ITEM_FLUSHING:
-			/*
-			 * The item or its backing buffer is already being
-			 * flushed.  The typical reason for that is that an
-			 * inode buffer is locked because we already pushed the
-			 * updates to it as part of inode clustering.
-			 *
-			 * We do not want to stop flushing just because lots
-			 * of items are already being flushed, but we need to
-			 * re-try the flushing relatively soon if most of the
-			 * AIL is being flushed.
-			 */
-			XFS_STATS_INC(mp, xs_push_ail_flushing);
-			trace_xfs_ail_flushing(lip);
-
-			flushing++;
-			ailp->ail_last_pushed_lsn = lsn;
-			break;
-
-		case XFS_ITEM_PINNED:
-			XFS_STATS_INC(mp, xs_push_ail_pinned);
-			trace_xfs_ail_pinned(lip);
-
-			stuck++;
-			ailp->ail_log_flush++;
-			break;
-		case XFS_ITEM_LOCKED:
-			XFS_STATS_INC(mp, xs_push_ail_locked);
-			trace_xfs_ail_locked(lip);
-
-			stuck++;
-			break;
-		default:
-			ASSERT(0);
-			break;
-		}
-
+		xfsaild_process_logitem(ailp, lip, &stuck, &flushing);
 		count++;
 
 		/*
diff --git a/fs/xfs/xfs_verify_media.c b/fs/xfs/xfs_verify_media.c
index 8bbd4ec567f8a..5ead3976d5115 100644
--- a/fs/xfs/xfs_verify_media.c
+++ b/fs/xfs/xfs_verify_media.c
@@ -183,10 +183,9 @@ xfs_verify_iosize(
 			min_not_zero(SZ_1M, me->me_max_io_size);
 
 	BUILD_BUG_ON(BBSHIFT != SECTOR_SHIFT);
-	ASSERT(BBTOB(bbcount) >= bdev_logical_block_size(btp->bt_bdev));
+	ASSERT(BBTOB(bbcount) >= btp->bt_logical_sectorsize);
 
-	return clamp(iosize, bdev_logical_block_size(btp->bt_bdev),
-			BBTOB(bbcount));
+	return clamp(iosize, btp->bt_logical_sectorsize, BBTOB(bbcount));
 }
 
 /* Allocate as much memory as we can get for verification buffer. */
@@ -218,8 +217,8 @@ xfs_verify_media_error(
 	unsigned int		bio_bbcount,
 	blk_status_t		bio_status)
 {
-	trace_xfs_verify_media_error(mp, me, btp->bt_bdev->bd_dev, daddr,
-			bio_bbcount, bio_status);
+	trace_xfs_verify_media_error(mp, me, btp->bt_dev, daddr, bio_bbcount,
+			bio_status);
 
 	/*
 	 * Pass any error, I/O or otherwise, up to the caller if we didn't
@@ -280,7 +279,7 @@ xfs_verify_media(
 		btp = mp->m_ddev_targp;
 		break;
 	case XFS_DEV_LOG:
-		if (mp->m_logdev_targp->bt_bdev != mp->m_ddev_targp->bt_bdev)
+		if (mp->m_logdev_targp != mp->m_ddev_targp)
 			btp = mp->m_logdev_targp;
 		break;
 	case XFS_DEV_RT:
@@ -299,7 +298,7 @@ xfs_verify_media(
 
 	/* start and end have to be aligned to the lba size */
 	if (!IS_ALIGNED(BBTOB(me->me_start_daddr | me->me_end_daddr),
-			bdev_logical_block_size(btp->bt_bdev)))
+			btp->bt_logical_sectorsize))
 		return -EINVAL;
 
 	/*
@@ -331,8 +330,7 @@ xfs_verify_media(
 	if (!folio)
 		return -ENOMEM;
 
-	trace_xfs_verify_media(mp, me, btp->bt_bdev->bd_dev, daddr, bbcount,
-			folio);
+	trace_xfs_verify_media(mp, me, btp->bt_dev, daddr, bbcount, folio);
 
 	bio = bio_alloc(btp->bt_bdev, 1, REQ_OP_READ, GFP_KERNEL);
 	if (!bio) {
@@ -400,7 +398,7 @@ xfs_verify_media(
 	 * an operational error.
 	 */
 	me->me_start_daddr = daddr;
-	trace_xfs_verify_media_end(mp, me, btp->bt_bdev->bd_dev);
+	trace_xfs_verify_media_end(mp, me, btp->bt_dev);
 	return 0;
 }
 
diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c
index a735f16d9cd8c..544213067d599 100644
--- a/fs/xfs/xfs_xattr.c
+++ b/fs/xfs/xfs_xattr.c
@@ -332,8 +332,8 @@ xfs_vn_listxattr(
 	memset(&context, 0, sizeof(context));
 	context.dp = XFS_I(inode);
 	context.resynch = 1;
-	context.buffer = size ? data : NULL;
 	context.bufsize = size;
+	context.buffer = size ? data : NULL;
 	context.firstu = context.bufsize;
 	context.put_listent = xfs_xattr_put_listent;
 
diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c
index e3d19b6dc64a8..06e2cb79030e2 100644
--- a/fs/xfs/xfs_zone_alloc.c
+++ b/fs/xfs/xfs_zone_alloc.c
@@ -189,27 +189,16 @@ xfs_open_zone_mark_full(
 		xfs_zone_account_reclaimable(rtg, rtg_blocks(rtg) - used);
 }
 
-static void
-xfs_zone_record_blocks(
-	struct xfs_trans	*tp,
+static inline void
+xfs_zone_inc_written(
 	struct xfs_open_zone	*oz,
-	xfs_fsblock_t		fsbno,
 	xfs_filblks_t		len)
 {
-	struct xfs_mount	*mp = tp->t_mountp;
-	struct xfs_rtgroup	*rtg = oz->oz_rtg;
-	struct xfs_inode	*rmapip = rtg_rmap(rtg);
-
-	trace_xfs_zone_record_blocks(oz, xfs_rtb_to_rgbno(mp, fsbno), len);
+	xfs_assert_ilocked(rtg_rmap(oz->oz_rtg), XFS_ILOCK_EXCL);
 
-	xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
-	xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP);
-	rmapip->i_used_blocks += len;
-	ASSERT(rmapip->i_used_blocks <= rtg_blocks(rtg));
 	oz->oz_written += len;
-	if (oz->oz_written == rtg_blocks(rtg))
+	if (oz->oz_written == rtg_blocks(oz->oz_rtg))
 		xfs_open_zone_mark_full(oz);
-	xfs_trans_log_inode(tp, rmapip, XFS_ILOG_CORE);
 }
 
 /*
@@ -227,9 +216,7 @@ xfs_zone_skip_blocks(
 	trace_xfs_zone_skip_blocks(oz, 0, len);
 
 	xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
-	oz->oz_written += len;
-	if (oz->oz_written == rtg_blocks(rtg))
-		xfs_open_zone_mark_full(oz);
+	xfs_zone_inc_written(oz, len);
 	xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP);
 
 	xfs_add_frextents(rtg_mount(rtg), len);
@@ -244,6 +231,8 @@ xfs_zoned_map_extent(
 	xfs_fsblock_t		old_startblock)
 {
 	struct xfs_bmbt_irec	data;
+	struct xfs_rtgroup	*rtg = oz->oz_rtg;
+	struct xfs_inode	*rmapip = rtg_rmap(rtg);
 	int			nmaps = 1;
 	int			error;
 
@@ -302,7 +291,15 @@ xfs_zoned_map_extent(
 		}
 	}
 
-	xfs_zone_record_blocks(tp, oz, new->br_startblock, new->br_blockcount);
+	trace_xfs_zone_record_blocks(oz,
+		xfs_rtb_to_rgbno(tp->t_mountp, new->br_startblock),
+		new->br_blockcount);
+	xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
+	xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP);
+	rmapip->i_used_blocks += new->br_blockcount;
+	ASSERT(rmapip->i_used_blocks <= rtg_blocks(rtg));
+	xfs_zone_inc_written(oz, new->br_blockcount);
+	xfs_trans_log_inode(tp, rmapip, XFS_ILOG_CORE);
 
 	/* Map the new blocks into the data fork. */
 	xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, new);
@@ -681,10 +678,11 @@ xfs_select_zone_nowait(
 	if (oz)
 		goto out_unlock;
 
-	if (pack_tight)
+	if (pack_tight) {
 		oz = xfs_select_open_zone_mru(zi, write_hint);
-	if (oz)
-		goto out_unlock;
+		if (oz)
+			goto out_unlock;
+	}
 
 	/*
 	 * See if we can open a new zone and use that so that data for different
@@ -695,7 +693,7 @@ xfs_select_zone_nowait(
 		goto out_unlock;
 
 	/*
-	 * Try to find an zone that is an ok match to colocate data with.
+	 * Try to find a zone that is an ok match to colocate data with.
 	 */
 	oz = xfs_select_open_zone_lru(zi, write_hint, XFS_ZONE_ALLOC_OK);
 	if (oz)
diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c
index 309f700985243..0ff710fa0ee70 100644
--- a/fs/xfs/xfs_zone_gc.c
+++ b/fs/xfs/xfs_zone_gc.c
@@ -170,25 +170,37 @@ xfs_zoned_need_gc(
 	s64			available, free, threshold;
 	s32			remainder;
 
+	/* If we have no reclaimable blocks, running GC is useless. */
 	if (!xfs_zoned_have_reclaimable(mp->m_zone_info))
 		return false;
 
+	/*
+	 * In order to avoid file fragmentation as much as possible, we should
+	 * make sure that we can open enough zones. So trigger GC if the number
+	 * of blocks immediately available for writes is lower than the total
+	 * number of blocks from all possible open zones.
+	 */
 	available = xfs_estimate_freecounter(mp, XC_FREE_RTAVAILABLE);
-
 	if (available <
 	    xfs_rtgs_to_rfsbs(mp, mp->m_max_open_zones - XFS_OPEN_GC_ZONES))
 		return true;
 
-	free = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS);
+	/*
+	 * For cases where the user wants to be more aggressive with GC,
+	 * the sysfs attribute zonegc_low_space may be set to a non zero value,
+	 * to indicate that GC should try to maintain at least zonegc_low_space
+	 * percent of the free space to be directly available for writing. Check
+	 * this here.
+	 */
+	if (!mp->m_zonegc_low_space)
+		return false;
 
+	free = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS);
 	threshold = div_s64_rem(free, 100, &remainder);
 	threshold = threshold * mp->m_zonegc_low_space +
 		    remainder * div_s64(mp->m_zonegc_low_space, 100);
 
-	if (available < threshold)
-		return true;
-
-	return false;
+	return available < threshold;
 }
 
 static struct xfs_zone_gc_data *
diff --git a/fs/xfs/xfs_zone_info.c b/fs/xfs/xfs_zone_info.c
index 53eabbc3334c0..a2af440116544 100644
--- a/fs/xfs/xfs_zone_info.c
+++ b/fs/xfs/xfs_zone_info.c
@@ -90,9 +90,14 @@ xfs_zoned_show_stats(
 	seq_printf(m, "\tRT GC required: %d\n",
 		xfs_zoned_need_gc(mp));
 
+	seq_printf(m, "\ttotal number of zones: %u\n",
+		mp->m_sb.sb_rgcount);
 	seq_printf(m, "\tfree zones: %d\n", atomic_read(&zi->zi_nr_free_zones));
-	seq_puts(m, "\topen zones:\n");
+
 	spin_lock(&zi->zi_open_zones_lock);
+	seq_printf(m, "\tnumber of open zones: %u / %u\n",
+		zi->zi_nr_open_zones, mp->m_max_open_zones);
+	seq_puts(m, "\topen zones:\n");
 	list_for_each_entry(oz, &zi->zi_open_zones, oz_entry)
 		xfs_show_open_zone(m, oz);
 	if (zi->zi_open_gc_zone) {
diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c
index 8a7161fc49e5c..5ada33f70bb47 100644
--- a/fs/zonefs/file.c
+++ b/fs/zonefs/file.c
@@ -333,8 +333,7 @@ static int zonefs_file_mmap_prepare(struct vm_area_desc *desc)
 	 * ordering between msync() and page cache writeback.
 	 */
 	if (zonefs_inode_is_seq(file_inode(file)) &&
-	    vma_desc_test_flags(desc, VMA_SHARED_BIT) &&
-	    vma_desc_test_flags(desc, VMA_MAYWRITE_BIT))
+	    vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT))
 		return -EINVAL;
 
 	file_accessed(file);
diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
index e83b2ec5e49f8..9b646cb5335d0 100644
--- a/fs/zonefs/super.c
+++ b/fs/zonefs/super.c
@@ -297,7 +297,7 @@ static void zonefs_handle_io_error(struct inode *inode, struct blk_zone *zone,
 	 */
 	if (isize != data_size)
 		zonefs_warn(sb,
-			    "inode %lu: invalid size %lld (should be %lld)\n",
+			    "inode %llu: invalid size %lld (should be %lld)\n",
 			    inode->i_ino, isize, data_size);
 
 	/*
@@ -308,7 +308,7 @@ static void zonefs_handle_io_error(struct inode *inode, struct blk_zone *zone,
 	 */
 	if ((z->z_flags & ZONEFS_ZONE_OFFLINE) ||
 	    (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZOL)) {
-		zonefs_warn(sb, "inode %lu: read/write access disabled\n",
+		zonefs_warn(sb, "inode %llu: read/write access disabled\n",
 			    inode->i_ino);
 		if (!(z->z_flags & ZONEFS_ZONE_OFFLINE))
 			z->z_flags |= ZONEFS_ZONE_OFFLINE;
@@ -316,7 +316,7 @@ static void zonefs_handle_io_error(struct inode *inode, struct blk_zone *zone,
 		data_size = 0;
 	} else if ((z->z_flags & ZONEFS_ZONE_READONLY) ||
 		   (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZRO)) {
-		zonefs_warn(sb, "inode %lu: write access disabled\n",
+		zonefs_warn(sb, "inode %llu: write access disabled\n",
 			    inode->i_ino);
 		if (!(z->z_flags & ZONEFS_ZONE_READONLY))
 			z->z_flags |= ZONEFS_ZONE_READONLY;
@@ -402,7 +402,7 @@ void __zonefs_io_error(struct inode *inode, bool write)
 	memalloc_noio_restore(noio_flag);
 
 	if (ret != 1) {
-		zonefs_err(sb, "Get inode %lu zone information failed %d\n",
+		zonefs_err(sb, "Get inode %llu zone information failed %d\n",
 			   inode->i_ino, ret);
 		zonefs_warn(sb, "remounting filesystem read-only\n");
 		sb->s_flags |= SB_RDONLY;
diff --git a/fs/zonefs/trace.h b/fs/zonefs/trace.h
index 9969db3a9c7dc..7315a72ca2f6f 100644
--- a/fs/zonefs/trace.h
+++ b/fs/zonefs/trace.h
@@ -25,7 +25,7 @@ TRACE_EVENT(zonefs_zone_mgmt,
 	    TP_ARGS(sb, z, op),
 	    TP_STRUCT__entry(
 			     __field(dev_t, dev)
-			     __field(ino_t, ino)
+			     __field(u64, ino)
 			     __field(enum req_op, op)
 			     __field(sector_t, sector)
 			     __field(sector_t, nr_sectors)
@@ -38,8 +38,8 @@ TRACE_EVENT(zonefs_zone_mgmt,
 			   __entry->sector = z->z_sector;
 			   __entry->nr_sectors = z->z_size >> SECTOR_SHIFT;
 	    ),
-	    TP_printk("bdev=(%d,%d), ino=%lu op=%s, sector=%llu, nr_sectors=%llu",
-		      show_dev(__entry->dev), (unsigned long)__entry->ino,
+	    TP_printk("bdev=(%d,%d), ino=%llu op=%s, sector=%llu, nr_sectors=%llu",
+		      show_dev(__entry->dev), __entry->ino,
 		      blk_op_str(__entry->op), __entry->sector,
 		      __entry->nr_sectors
 	    )
@@ -50,7 +50,7 @@ TRACE_EVENT(zonefs_file_dio_append,
 	    TP_ARGS(inode, size, ret),
 	    TP_STRUCT__entry(
 			     __field(dev_t, dev)
-			     __field(ino_t, ino)
+			     __field(u64, ino)
 			     __field(sector_t, sector)
 			     __field(ssize_t, size)
 			     __field(loff_t, wpoffset)
@@ -65,8 +65,8 @@ TRACE_EVENT(zonefs_file_dio_append,
 				zonefs_inode_zone(inode)->z_wpoffset;
 			   __entry->ret = ret;
 	    ),
-	    TP_printk("bdev=(%d, %d), ino=%lu, sector=%llu, size=%zu, wpoffset=%llu, ret=%zu",
-		      show_dev(__entry->dev), (unsigned long)__entry->ino,
+	    TP_printk("bdev=(%d, %d), ino=%llu, sector=%llu, size=%zu, wpoffset=%llu, ret=%zu",
+		      show_dev(__entry->dev), __entry->ino,
 		      __entry->sector, __entry->size, __entry->wpoffset,
 		      __entry->ret
 	    )
@@ -77,7 +77,7 @@ TRACE_EVENT(zonefs_iomap_begin,
 	    TP_ARGS(inode, iomap),
 	    TP_STRUCT__entry(
 			     __field(dev_t, dev)
-			     __field(ino_t, ino)
+			     __field(u64, ino)
 			     __field(u64, addr)
 			     __field(loff_t, offset)
 			     __field(u64, length)
@@ -89,8 +89,8 @@ TRACE_EVENT(zonefs_iomap_begin,
 			   __entry->offset = iomap->offset;
 			   __entry->length = iomap->length;
 	    ),
-	    TP_printk("bdev=(%d,%d), ino=%lu, addr=%llu, offset=%llu, length=%llu",
-		      show_dev(__entry->dev), (unsigned long)__entry->ino,
+	    TP_printk("bdev=(%d,%d), ino=%llu, addr=%llu, offset=%llu, length=%llu",
+		      show_dev(__entry->dev), __entry->ino,
 		      __entry->addr, __entry->offset, __entry->length
 	    )
 );
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index aad1a95e6863d..b701b5f972cb4 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -613,6 +613,8 @@ struct acpi_bus_event {
 	u32 data;
 };
 
+#define ACPI_AC_CLASS	"ac_adapter"
+
 extern struct kobject *acpi_kobj;
 extern int acpi_bus_generate_netlink_event(const char*, const char*, u8, int);
 void acpi_bus_private_data_handler(acpi_handle, void *);
@@ -625,7 +627,8 @@ int acpi_dev_install_notify_handler(struct acpi_device *adev,
 void acpi_dev_remove_notify_handler(struct acpi_device *adev,
 				    u32 handler_type,
 				    acpi_notify_handler handler);
-extern int acpi_notifier_call_chain(struct acpi_device *, u32, u32);
+extern int acpi_notifier_call_chain(const char *device_class,
+				    const char *bus_id, u32 type, u32 data);
 extern int register_acpi_notifier(struct notifier_block *);
 extern int unregister_acpi_notifier(struct notifier_block *);
 
@@ -760,8 +763,6 @@ int acpi_disable_wakeup_device_power(struct acpi_device *dev);
 #ifdef CONFIG_X86
 bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *status);
 bool acpi_quirk_skip_acpi_ac_and_battery(void);
-int acpi_install_cmos_rtc_space_handler(acpi_handle handle);
-void acpi_remove_cmos_rtc_space_handler(acpi_handle handle);
 int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip);
 #else
 static inline bool acpi_device_override_status(struct acpi_device *adev,
@@ -773,13 +774,6 @@ static inline bool acpi_quirk_skip_acpi_ac_and_battery(void)
 {
 	return false;
 }
-static inline int acpi_install_cmos_rtc_space_handler(acpi_handle handle)
-{
-	return 1;
-}
-static inline void acpi_remove_cmos_rtc_space_handler(acpi_handle handle)
-{
-}
 static inline int
 acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip)
 {
diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h
index 4d644f03098e3..d8e405becdc31 100644
--- a/include/acpi/cppc_acpi.h
+++ b/include/acpi/cppc_acpi.h
@@ -42,6 +42,11 @@
 #define CPPC_EPP_PERFORMANCE_PREF		0x00
 #define CPPC_EPP_ENERGY_EFFICIENCY_PREF		0xFF
 
+#define CPPC_PERF_LIMITED_DESIRED_EXCURSION	BIT(0)
+#define CPPC_PERF_LIMITED_MINIMUM_EXCURSION	BIT(1)
+#define CPPC_PERF_LIMITED_MASK		(CPPC_PERF_LIMITED_DESIRED_EXCURSION | \
+					 CPPC_PERF_LIMITED_MINIMUM_EXCURSION)
+
 /* Each register has the folowing format. */
 struct cpc_reg {
 	u8 descriptor;
@@ -116,6 +121,7 @@ struct cppc_perf_caps {
 	u32 guaranteed_perf;
 	u32 highest_perf;
 	u32 nominal_perf;
+	u32 reference_perf;
 	u32 lowest_perf;
 	u32 lowest_nonlinear_perf;
 	u32 lowest_freq;
@@ -133,7 +139,6 @@ struct cppc_perf_ctrls {
 struct cppc_perf_fb_ctrs {
 	u64 reference;
 	u64 delivered;
-	u64 reference_perf;
 	u64 wraparound_time;
 };
 
@@ -151,6 +156,7 @@ extern int cppc_get_desired_perf(int cpunum, u64 *desired_perf);
 extern int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf);
 extern int cppc_get_highest_perf(int cpunum, u64 *highest_perf);
 extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs);
+extern int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls);
 extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls);
 extern int cppc_set_enable(int cpu, bool enable);
 extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps);
@@ -173,6 +179,8 @@ extern int cppc_get_auto_act_window(int cpu, u64 *auto_act_window);
 extern int cppc_set_auto_act_window(int cpu, u64 auto_act_window);
 extern int cppc_get_auto_sel(int cpu, bool *enable);
 extern int cppc_set_auto_sel(int cpu, bool enable);
+extern int cppc_get_perf_limited(int cpu, u64 *perf_limited);
+extern int cppc_set_perf_limited(int cpu, u64 bits_to_clear);
 extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf);
 extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator);
 extern int amd_detect_prefcore(bool *detected);
@@ -193,6 +201,10 @@ static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_
 {
 	return -EOPNOTSUPP;
 }
+static inline int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
+{
+	return -EOPNOTSUPP;
+}
 static inline int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 {
 	return -EOPNOTSUPP;
@@ -265,6 +277,14 @@ static inline int cppc_set_auto_sel(int cpu, bool enable)
 {
 	return -EOPNOTSUPP;
 }
+static inline int cppc_get_perf_limited(int cpu, u64 *perf_limited)
+{
+	return -EOPNOTSUPP;
+}
+static inline int cppc_set_perf_limited(int cpu, u64 bits_to_clear)
+{
+	return -EOPNOTSUPP;
+}
 static inline int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf)
 {
 	return -ENODEV;
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index 7146a8e9e9c25..554be224ce766 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -14,8 +14,6 @@
 
 #include <asm/acpi.h>
 
-#define ACPI_PROCESSOR_CLASS		"processor"
-#define ACPI_PROCESSOR_DEVICE_NAME	"Processor"
 #define ACPI_PROCESSOR_DEVICE_HID	"ACPI0007"
 #define ACPI_PROCESSOR_CONTAINER_HID	"ACPI0010"
 
diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild
index 9aff61e7b8f27..2c53a1e0b7604 100644
--- a/include/asm-generic/Kbuild
+++ b/include/asm-generic/Kbuild
@@ -65,4 +65,3 @@ mandatory-y += vermagic.h
 mandatory-y += vga.h
 mandatory-y += video.h
 mandatory-y += word-at-a-time.h
-mandatory-y += xor.h
diff --git a/include/asm-generic/bitsperlong.h b/include/asm-generic/bitsperlong.h
index 1023e2a4bd372..90e8aeebfd2f4 100644
--- a/include/asm-generic/bitsperlong.h
+++ b/include/asm-generic/bitsperlong.h
@@ -19,6 +19,15 @@
 #error Inconsistent word size. Check asm/bitsperlong.h
 #endif
 
+#if __CHAR_BIT__ * __SIZEOF_LONG__ != __BITS_PER_LONG
+#error Inconsistent word size. Check asm/bitsperlong.h
+#endif
+
+#ifndef __ASSEMBLER__
+_Static_assert(sizeof(long) * 8 == __BITS_PER_LONG,
+	       "Inconsistent word size. Check asm/bitsperlong.h");
+#endif
+
 #ifndef BITS_PER_LONG_LONG
 #define BITS_PER_LONG_LONG 64
 #endif
diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h
index a14f4bdafddae..4948e5d4e9d99 100644
--- a/include/asm-generic/codetag.lds.h
+++ b/include/asm-generic/codetag.lds.h
@@ -18,7 +18,7 @@
 	IF_MEM_ALLOC_PROFILING(SECTION_WITH_BOUNDARIES(alloc_tags))
 
 #define MOD_SEPARATE_CODETAG_SECTION(_name)	\
-	.codetag.##_name : {			\
+	.codetag.##_name 0 : {			\
 		SECTION_WITH_BOUNDARIES(_name)	\
 	}
 
diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index 2a19215baae53..fbbcfd801cd0a 100644
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -25,7 +25,9 @@
  *			  argument and comparison of the previous
  *			  futex value with another constant.
  *
- * @encoded_op:	encoded operation to execute
+ * @op:		operation to execute
+ * @oparg:	argument of the operation
+ * @oval:	previous value at @uaddr on successful return
  * @uaddr:	pointer to user space address
  *
  * Return:
diff --git a/include/asm-generic/thread_info_tif.h b/include/asm-generic/thread_info_tif.h
index da1610a78f921..528e6fc7efe9e 100644
--- a/include/asm-generic/thread_info_tif.h
+++ b/include/asm-generic/thread_info_tif.h
@@ -41,11 +41,14 @@
 #define _TIF_PATCH_PENDING	BIT(TIF_PATCH_PENDING)
 
 #ifdef HAVE_TIF_RESTORE_SIGMASK
-# define TIF_RESTORE_SIGMASK	10	// Restore signal mask in do_signal() */
+# define TIF_RESTORE_SIGMASK	10	// Restore signal mask in do_signal()
 # define _TIF_RESTORE_SIGMASK	BIT(TIF_RESTORE_SIGMASK)
 #endif
 
 #define TIF_RSEQ		11	// Run RSEQ fast path
 #define _TIF_RSEQ		BIT(TIF_RSEQ)
 
+#define TIF_HRTIMER_REARM	12       // re-arm the timer
+#define _TIF_HRTIMER_REARM	BIT(TIF_HRTIMER_REARM)
+
 #endif /* _ASM_GENERIC_THREAD_INFO_TIF_H_ */
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 4aeac0c3d3f03..bdcc2778ac64f 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -251,6 +251,8 @@ static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
 
 void tlb_remove_table_sync_one(void);
 
+void tlb_remove_table_sync_rcu(void);
+
 #else
 
 #ifdef tlb_needs_table_invalidate
@@ -259,6 +261,8 @@ void tlb_remove_table_sync_one(void);
 
 static inline void tlb_remove_table_sync_one(void) { }
 
+static inline void tlb_remove_table_sync_rcu(void) { }
+
 #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
 
 
diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h
deleted file mode 100644
index 44509d48fca21..0000000000000
--- a/include/asm-generic/xor.h
+++ /dev/null
@@ -1,738 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm-generic/xor.h
- *
- * Generic optimized RAID-5 checksumming functions.
- */
-
-#include <linux/prefetch.h>
-
-static void
-xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		p1[0] ^= p2[0];
-		p1[1] ^= p2[1];
-		p1[2] ^= p2[2];
-		p1[3] ^= p2[3];
-		p1[4] ^= p2[4];
-		p1[5] ^= p2[5];
-		p1[6] ^= p2[6];
-		p1[7] ^= p2[7];
-		p1 += 8;
-		p2 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		p1[0] ^= p2[0] ^ p3[0];
-		p1[1] ^= p2[1] ^ p3[1];
-		p1[2] ^= p2[2] ^ p3[2];
-		p1[3] ^= p2[3] ^ p3[3];
-		p1[4] ^= p2[4] ^ p3[4];
-		p1[5] ^= p2[5] ^ p3[5];
-		p1[6] ^= p2[6] ^ p3[6];
-		p1[7] ^= p2[7] ^ p3[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3,
-	    const unsigned long * __restrict p4)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
-		p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
-		p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
-		p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
-		p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
-		p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
-		p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
-		p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
-	    const unsigned long * __restrict p2,
-	    const unsigned long * __restrict p3,
-	    const unsigned long * __restrict p4,
-	    const unsigned long * __restrict p5)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
-		p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
-		p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
-		p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
-		p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
-		p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
-		p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
-		p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-		p5 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3,
-	     const unsigned long * __restrict p4)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		d0 ^= p4[0];
-		d1 ^= p4[1];
-		d2 ^= p4[2];
-		d3 ^= p4[3];
-		d4 ^= p4[4];
-		d5 ^= p4[5];
-		d6 ^= p4[6];
-		d7 ^= p4[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
-	     const unsigned long * __restrict p2,
-	     const unsigned long * __restrict p3,
-	     const unsigned long * __restrict p4,
-	     const unsigned long * __restrict p5)
-{
-	long lines = bytes / (sizeof (long)) / 8;
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		d0 ^= p4[0];
-		d1 ^= p4[1];
-		d2 ^= p4[2];
-		d3 ^= p4[3];
-		d4 ^= p4[4];
-		d5 ^= p4[5];
-		d6 ^= p4[6];
-		d7 ^= p4[7];
-		d0 ^= p5[0];
-		d1 ^= p5[1];
-		d2 ^= p5[2];
-		d3 ^= p5[3];
-		d4 ^= p5[4];
-		d5 ^= p5[5];
-		d6 ^= p5[6];
-		d7 ^= p5[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-		p5 += 8;
-	} while (--lines > 0);
-}
-
-static void
-xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-	prefetchw(p1);
-	prefetch(p2);
-
-	do {
-		prefetchw(p1+8);
-		prefetch(p2+8);
- once_more:
-		p1[0] ^= p2[0];
-		p1[1] ^= p2[1];
-		p1[2] ^= p2[2];
-		p1[3] ^= p2[3];
-		p1[4] ^= p2[4];
-		p1[5] ^= p2[5];
-		p1[6] ^= p2[6];
-		p1[7] ^= p2[7];
-		p1 += 8;
-		p2 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-
-	do {
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
- once_more:
-		p1[0] ^= p2[0] ^ p3[0];
-		p1[1] ^= p2[1] ^ p3[1];
-		p1[2] ^= p2[2] ^ p3[2];
-		p1[3] ^= p2[3] ^ p3[3];
-		p1[4] ^= p2[4] ^ p3[4];
-		p1[5] ^= p2[5] ^ p3[5];
-		p1[6] ^= p2[6] ^ p3[6];
-		p1[7] ^= p2[7] ^ p3[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3,
-	      const unsigned long * __restrict p4)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-	prefetch(p4);
-
-	do {
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
-		prefetch(p4+8);
- once_more:
-		p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
-		p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
-		p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
-		p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
-		p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
-		p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
-		p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
-		p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
-	      const unsigned long * __restrict p2,
-	      const unsigned long * __restrict p3,
-	      const unsigned long * __restrict p4,
-	      const unsigned long * __restrict p5)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-	prefetch(p4);
-	prefetch(p5);
-
-	do {
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
-		prefetch(p4+8);
-		prefetch(p5+8);
- once_more:
-		p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
-		p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
-		p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
-		p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
-		p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
-		p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
-		p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
-		p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-		p5 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-		prefetchw(p1+8);
-		prefetch(p2+8);
- once_more:
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
- once_more:
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-	prefetch(p4);
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
-		prefetch(p4+8);
- once_more:
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		d0 ^= p4[0];
-		d1 ^= p4[1];
-		d2 ^= p4[2];
-		d3 ^= p4[3];
-		d4 ^= p4[4];
-		d5 ^= p4[5];
-		d6 ^= p4[6];
-		d7 ^= p4[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static void
-xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
-	       const unsigned long * __restrict p2,
-	       const unsigned long * __restrict p3,
-	       const unsigned long * __restrict p4,
-	       const unsigned long * __restrict p5)
-{
-	long lines = bytes / (sizeof (long)) / 8 - 1;
-
-	prefetchw(p1);
-	prefetch(p2);
-	prefetch(p3);
-	prefetch(p4);
-	prefetch(p5);
-
-	do {
-		register long d0, d1, d2, d3, d4, d5, d6, d7;
-
-		prefetchw(p1+8);
-		prefetch(p2+8);
-		prefetch(p3+8);
-		prefetch(p4+8);
-		prefetch(p5+8);
- once_more:
-		d0 = p1[0];	/* Pull the stuff into registers	*/
-		d1 = p1[1];	/*  ... in bursts, if possible.		*/
-		d2 = p1[2];
-		d3 = p1[3];
-		d4 = p1[4];
-		d5 = p1[5];
-		d6 = p1[6];
-		d7 = p1[7];
-		d0 ^= p2[0];
-		d1 ^= p2[1];
-		d2 ^= p2[2];
-		d3 ^= p2[3];
-		d4 ^= p2[4];
-		d5 ^= p2[5];
-		d6 ^= p2[6];
-		d7 ^= p2[7];
-		d0 ^= p3[0];
-		d1 ^= p3[1];
-		d2 ^= p3[2];
-		d3 ^= p3[3];
-		d4 ^= p3[4];
-		d5 ^= p3[5];
-		d6 ^= p3[6];
-		d7 ^= p3[7];
-		d0 ^= p4[0];
-		d1 ^= p4[1];
-		d2 ^= p4[2];
-		d3 ^= p4[3];
-		d4 ^= p4[4];
-		d5 ^= p4[5];
-		d6 ^= p4[6];
-		d7 ^= p4[7];
-		d0 ^= p5[0];
-		d1 ^= p5[1];
-		d2 ^= p5[2];
-		d3 ^= p5[3];
-		d4 ^= p5[4];
-		d5 ^= p5[5];
-		d6 ^= p5[6];
-		d7 ^= p5[7];
-		p1[0] = d0;	/* Store the result (in bursts)		*/
-		p1[1] = d1;
-		p1[2] = d2;
-		p1[3] = d3;
-		p1[4] = d4;
-		p1[5] = d5;
-		p1[6] = d6;
-		p1[7] = d7;
-		p1 += 8;
-		p2 += 8;
-		p3 += 8;
-		p4 += 8;
-		p5 += 8;
-	} while (--lines > 0);
-	if (lines == 0)
-		goto once_more;
-}
-
-static struct xor_block_template xor_block_8regs = {
-	.name = "8regs",
-	.do_2 = xor_8regs_2,
-	.do_3 = xor_8regs_3,
-	.do_4 = xor_8regs_4,
-	.do_5 = xor_8regs_5,
-};
-
-static struct xor_block_template xor_block_32regs = {
-	.name = "32regs",
-	.do_2 = xor_32regs_2,
-	.do_3 = xor_32regs_3,
-	.do_4 = xor_32regs_4,
-	.do_5 = xor_32regs_5,
-};
-
-static struct xor_block_template xor_block_8regs_p __maybe_unused = {
-	.name = "8regs_prefetch",
-	.do_2 = xor_8regs_p_2,
-	.do_3 = xor_8regs_p_3,
-	.do_4 = xor_8regs_p_4,
-	.do_5 = xor_8regs_p_5,
-};
-
-static struct xor_block_template xor_block_32regs_p __maybe_unused = {
-	.name = "32regs_prefetch",
-	.do_2 = xor_32regs_p_2,
-	.do_3 = xor_32regs_p_3,
-	.do_4 = xor_32regs_p_4,
-	.do_5 = xor_32regs_p_5,
-};
-
-#define XOR_TRY_TEMPLATES			\
-	do {					\
-		xor_speed(&xor_block_8regs);	\
-		xor_speed(&xor_block_8regs_p);	\
-		xor_speed(&xor_block_32regs);	\
-		xor_speed(&xor_block_32regs_p);	\
-	} while (0)
diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h
index 9eacb9fa375d7..5d5358dfab734 100644
--- a/include/crypto/acompress.h
+++ b/include/crypto/acompress.h
@@ -240,9 +240,10 @@ static inline const char *crypto_acomp_driver_name(struct crypto_acomp *tfm)
 }
 
 /**
- * acomp_request_alloc() -- allocates asynchronous (de)compression request
+ * acomp_request_alloc_extra() -- allocates asynchronous (de)compression request
  *
  * @tfm:	ACOMPRESS tfm handle allocated with crypto_alloc_acomp()
+ * @extra:	amount of extra memory
  * @gfp:	gfp to pass to kzalloc (defaults to GFP_KERNEL)
  *
  * Return:	allocated handle in case of success or NULL in case of an error
@@ -318,7 +319,7 @@ static inline void acomp_request_free(struct acomp_req *req)
  *
  * @req:	request that the callback will be set for
  * @flgs:	specify for instance if the operation may backlog
- * @cmlp:	callback which will be called
+ * @cmpl:	callback which will be called
  * @data:	private data used by the caller
  */
 static inline void acomp_request_set_callback(struct acomp_req *req,
diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h
new file mode 100644
index 0000000000000..e61df108b926d
--- /dev/null
+++ b/include/crypto/aes-cbc-macs.h
@@ -0,0 +1,154 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Support for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC
+ *
+ * Copyright 2026 Google LLC
+ */
+#ifndef _CRYPTO_AES_CBC_MACS_H
+#define _CRYPTO_AES_CBC_MACS_H
+
+#include <crypto/aes.h>
+
+/**
+ * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC
+ * @aes: The AES key for cipher block chaining
+ * @k_final: Finalization subkeys for the final block.
+ *	     k_final[0] (CMAC K1, XCBC-MAC K2) is used if it's a full block.
+ *	     k_final[1] (CMAC K2, XCBC-MAC K3) is used if it's a partial block.
+ */
+struct aes_cmac_key {
+	struct aes_enckey aes;
+	union {
+		u8 b[AES_BLOCK_SIZE];
+		__be64 w[2];
+	} k_final[2];
+};
+
+/**
+ * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value
+ * @key: Pointer to the key struct.  A pointer is used rather than a copy of the
+ *	 struct, since the key struct size may be large.  It is assumed that the
+ *	 key lives at least as long as the context.
+ * @partial_len: Number of bytes that have been XOR'ed into @h since the last
+ *		 AES encryption.  This is 0 if no data has been processed yet,
+ *		 or between 1 and AES_BLOCK_SIZE inclusive otherwise.
+ * @h: The current chaining value
+ */
+struct aes_cmac_ctx {
+	const struct aes_cmac_key *key;
+	size_t partial_len;
+	u8 h[AES_BLOCK_SIZE];
+};
+
+/**
+ * aes_cmac_preparekey() - Prepare a key for AES-CMAC
+ * @key: (output) The key struct to initialize
+ * @in_key: The raw AES key
+ * @key_len: Length of the raw key in bytes.  The supported values are
+ *	     AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256.
+ *
+ * Context: Any context.
+ * Return: 0 on success or -EINVAL if the given key length is invalid.  No other
+ *	   errors are possible, so callers that always pass a valid key length
+ *	   don't need to check for errors.
+ */
+int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key,
+			size_t key_len);
+
+/**
+ * aes_xcbcmac_preparekey() - Prepare a key for AES-XCBC-MAC
+ * @key: (output) The key struct to initialize
+ * @in_key: The raw key.  As per the AES-XCBC-MAC specification (RFC 3566), this
+ *	    is 128 bits, matching the internal use of AES-128.
+ *
+ * AES-XCBC-MAC and AES-CMAC are the same except for the key preparation.  After
+ * that step, AES-XCBC-MAC is supported via the aes_cmac_* functions.
+ *
+ * New users should use AES-CMAC instead of AES-XCBC-MAC.
+ *
+ * Context: Any context.
+ */
+void aes_xcbcmac_preparekey(struct aes_cmac_key *key,
+			    const u8 in_key[at_least AES_KEYSIZE_128]);
+
+/**
+ * aes_cmac_init() - Start computing an AES-CMAC or AES-XCBC-MAC value
+ * @ctx: (output) The context to initialize
+ * @key: The key to use.  Note that a pointer to the key is saved in the
+ *	 context, so the key must live at least as long as the context.
+ *
+ * This supports both AES-CMAC and AES-XCBC-MAC.  Which one is done depends on
+ * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called.
+ */
+static inline void aes_cmac_init(struct aes_cmac_ctx *ctx,
+				 const struct aes_cmac_key *key)
+{
+	*ctx = (struct aes_cmac_ctx){ .key = key };
+}
+
+/**
+ * aes_cmac_update() - Update an AES-CMAC or AES-XCBC-MAC context with more data
+ * @ctx: The context to update; must have been initialized
+ * @data: The message data
+ * @data_len: The data length in bytes.  Doesn't need to be block-aligned.
+ *
+ * This can be called any number of times.
+ *
+ * Context: Any context.
+ */
+void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len);
+
+/**
+ * aes_cmac_final() - Finish computing an AES-CMAC or AES-XCBC-MAC value
+ * @ctx: The context to finalize; must have been initialized
+ * @out: (output) The resulting MAC
+ *
+ * After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+ *
+ * Context: Any context.
+ */
+void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[at_least AES_BLOCK_SIZE]);
+
+/**
+ * aes_cmac() - Compute AES-CMAC or AES-XCBC-MAC in one shot
+ * @key: The key to use
+ * @data: The message data
+ * @data_len: The data length in bytes
+ * @out: (output) The resulting AES-CMAC or AES-XCBC-MAC value
+ *
+ * This supports both AES-CMAC and AES-XCBC-MAC.  Which one is done depends on
+ * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called.
+ *
+ * Context: Any context.
+ */
+static inline void aes_cmac(const struct aes_cmac_key *key, const u8 *data,
+			    size_t data_len, u8 out[at_least AES_BLOCK_SIZE])
+{
+	struct aes_cmac_ctx ctx;
+
+	aes_cmac_init(&ctx, key);
+	aes_cmac_update(&ctx, data, data_len);
+	aes_cmac_final(&ctx, out);
+}
+
+/*
+ * AES-CBC-MAC support.  This is provided only for use by the implementation of
+ * AES-CCM.  It should have no other users.  Warning: unlike AES-CMAC and
+ * AES-XCBC-MAC, AES-CBC-MAC isn't a secure MAC for variable-length messages.
+ */
+struct aes_cbcmac_ctx {
+	const struct aes_enckey *key;
+	size_t partial_len;
+	u8 h[AES_BLOCK_SIZE];
+};
+static inline void aes_cbcmac_init(struct aes_cbcmac_ctx *ctx,
+				   const struct aes_enckey *key)
+{
+	*ctx = (struct aes_cbcmac_ctx){ .key = key };
+}
+void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data,
+		       size_t data_len);
+void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx,
+		      u8 out[at_least AES_BLOCK_SIZE]);
+
+#endif /* _CRYPTO_AES_CBC_MACS_H */
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index cbf1cc96db521..3feb4105c2a20 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -167,6 +167,72 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 #ifdef CONFIG_ARM64
 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		     unsigned int key_len);
+asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks);
+asmlinkage void neon_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks);
+asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks, u8 iv[]);
+asmlinkage void neon_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int blocks, u8 iv[]);
+asmlinkage void neon_aes_cbc_cts_encrypt(u8 out[], u8 const in[],
+					 u32 const rk[], int rounds, int bytes,
+					 u8 const iv[]);
+asmlinkage void neon_aes_cbc_cts_decrypt(u8 out[], u8 const in[],
+					 u32 const rk[], int rounds, int bytes,
+					 u8 const iv[]);
+asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				     int rounds, int bytes, u8 ctr[]);
+asmlinkage void neon_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				      int rounds, int bytes, u8 ctr[],
+				      int byte_ctr);
+asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+				     int rounds, int bytes, u32 const rk2[],
+				     u8 iv[], int first);
+asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+				     int rounds, int bytes, u32 const rk2[],
+				     u8 iv[], int first);
+asmlinkage void neon_aes_essiv_cbc_encrypt(u8 out[], u8 const in[],
+					   u32 const rk1[], int rounds,
+					   int blocks, u8 iv[],
+					   u32 const rk2[]);
+asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[],
+					   u32 const rk1[], int rounds,
+					   int blocks, u8 iv[],
+					   u32 const rk2[]);
+
+asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks);
+asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks);
+asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks, u8 iv[]);
+asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int blocks, u8 iv[]);
+asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				       int rounds, int bytes, u8 const iv[]);
+asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+				       int rounds, int bytes, u8 const iv[]);
+asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				   int rounds, int bytes, u8 ctr[]);
+asmlinkage void ce_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
+				    int rounds, int bytes, u8 ctr[],
+				    int byte_ctr);
+asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+				   int rounds, int bytes, u32 const rk2[],
+				   u8 iv[], int first);
+asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+				   int rounds, int bytes, u32 const rk2[],
+				   u8 iv[], int first);
+asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[],
+					 u32 const rk1[], int rounds,
+					 int blocks, u8 iv[], u32 const rk2[]);
+asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[],
+					 u32 const rk1[], int rounds,
+					 int blocks, u8 iv[], u32 const rk2[]);
+asmlinkage size_t ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+				    size_t blocks, u8 dg[], int enc_before,
+				    int enc_after);
 #elif defined(CONFIG_PPC)
 void ppc_expand_key_128(u32 *key_enc, const u8 *key);
 void ppc_expand_key_192(u32 *key_enc, const u8 *key);
diff --git a/include/crypto/des.h b/include/crypto/des.h
index 7812b4331ae4a..73eec617f4809 100644
--- a/include/crypto/des.h
+++ b/include/crypto/des.h
@@ -34,9 +34,9 @@ void des3_ede_decrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src);
  * des_expand_key - Expand a DES input key into a key schedule
  * @ctx: the key schedule
  * @key: buffer containing the input key
- * @len: size of the buffer contents
+ * @keylen: size of the buffer contents
  *
- * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if
+ * Returns: 0 on success, -EINVAL if the input key is rejected and -ENOKEY if
  * the key is accepted but has been found to be weak.
  */
 int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen);
@@ -45,9 +45,9 @@ int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen);
  * des3_ede_expand_key - Expand a triple DES input key into a key schedule
  * @ctx: the key schedule
  * @key: buffer containing the input key
- * @len: size of the buffer contents
+ * @keylen: size of the buffer contents
  *
- * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if
+ * Returns: 0 on success, -EINVAL if the input key is rejected and -ENOKEY if
  * the key is accepted but has been found to be weak. Note that weak keys will
  * be rejected (and -EINVAL will be returned) when running in FIPS mode.
  */
diff --git a/include/crypto/internal/blockhash.h b/include/crypto/internal/blockhash.h
deleted file mode 100644
index 52d9d4c82493d..0000000000000
--- a/include/crypto/internal/blockhash.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Handle partial blocks for block hash.
- *
- * Copyright (c) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
- * Copyright (c) 2025 Herbert Xu <herbert@gondor.apana.org.au>
- */
-
-#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H
-#define _CRYPTO_INTERNAL_BLOCKHASH_H
-
-#include <linux/string.h>
-#include <linux/types.h>
-
-#define BLOCK_HASH_UPDATE_BASE(block_fn, state, src, nbytes, bs, dv,	\
-			       buf, buflen)				\
-	({								\
-		typeof(block_fn) *_block_fn = &(block_fn);		\
-		typeof(state + 0) _state = (state);			\
-		unsigned int _buflen = (buflen);			\
-		size_t _nbytes = (nbytes);				\
-		unsigned int _bs = (bs);				\
-		const u8 *_src = (src);					\
-		u8 *_buf = (buf);					\
-		while ((_buflen + _nbytes) >= _bs) {			\
-			const u8 *data = _src;				\
-			size_t len = _nbytes;				\
-			size_t blocks;					\
-			int remain;					\
-			if (_buflen) {					\
-				remain = _bs - _buflen;			\
-				memcpy(_buf + _buflen, _src, remain);	\
-				data = _buf;				\
-				len = _bs;				\
-			}						\
-			remain = len % bs;				\
-			blocks = (len - remain) / (dv);			\
-			(*_block_fn)(_state, data, blocks);		\
-			_src += len - remain - _buflen;			\
-			_nbytes -= len - remain - _buflen;		\
-			_buflen = 0;					\
-		}							\
-		memcpy(_buf + _buflen, _src, _nbytes);			\
-		_buflen += _nbytes;					\
-	})
-
-#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \
-	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen)
-#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \
-	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen)
-
-#endif	/* _CRYPTO_INTERNAL_BLOCKHASH_H */
diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h
index 57cd752421414..a4b48d76f53ad 100644
--- a/include/crypto/internal/ecc.h
+++ b/include/crypto/internal/ecc.h
@@ -72,8 +72,8 @@ static inline void ecc_swap_digits(const void *in, u64 *out, unsigned int ndigit
 /**
  * ecc_digits_from_bytes() - Create ndigits-sized digits array from byte array
  * @in:       Input byte array
- * @nbytes    Size of input byte array
- * @out       Output digits array
+ * @nbytes:   Size of input byte array
+ * @out:      Output digits array
  * @ndigits:  Number of digits to create from byte array
  *
  * The first byte in the input byte array is expected to hold the most
@@ -90,7 +90,7 @@ void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes,
  * @private_key:	private key to be used for the given curve
  * @private_key_len:	private key length
  *
- * Returns 0 if the key is acceptable, a negative value otherwise
+ * Returns: 0 if the key is acceptable, a negative value otherwise
  */
 int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
 		     const u64 *private_key, unsigned int private_key_len);
@@ -104,7 +104,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
  * @ndigits:		curve number of digits
  * @private_key:	buffer for storing the generated private key
  *
- * Returns 0 if the private key was generated successfully, a negative value
+ * Returns: 0 if the private key was generated successfully, a negative value
  * if an error occurred.
  */
 int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits,
@@ -118,7 +118,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits,
  * @private_key:	pregenerated private key for the given curve
  * @public_key:		buffer for storing the generated public key
  *
- * Returns 0 if the public key was generated successfully, a negative value
+ * Returns: 0 if the public key was generated successfully, a negative value
  * if an error occurred.
  */
 int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
@@ -136,7 +136,7 @@ int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
  * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
  * before using it for symmetric encryption or HMAC.
  *
- * Returns 0 if the shared secret was generated successfully, a negative value
+ * Returns: 0 if the shared secret was generated successfully, a negative value
  * if an error occurred.
  */
 int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
@@ -179,6 +179,8 @@ int ecc_is_pubkey_valid_full(const struct ecc_curve *curve,
  *
  * @vli:		vli to check.
  * @ndigits:		length of the @vli
+ *
+ * Returns: %true if vli == 0, %false otherwise.
  */
 bool vli_is_zero(const u64 *vli, unsigned int ndigits);
 
@@ -189,7 +191,7 @@ bool vli_is_zero(const u64 *vli, unsigned int ndigits);
  * @right:		vli
  * @ndigits:		length of both vlis
  *
- * Returns sign of @left - @right, i.e. -1 if @left < @right,
+ * Returns: sign of @left - @right, i.e. -1 if @left < @right,
  * 0 if @left == @right, 1 if @left > @right.
  */
 int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits);
@@ -199,7 +201,7 @@ int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits);
  *
  * @result:		where to write result
  * @left:		vli
- * @right		vli
+ * @right:		vli
  * @ndigits:		length of all vlis
  *
  * Note: can modify in-place.
@@ -263,7 +265,7 @@ void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right,
 unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits);
 
 /**
- * ecc_aloc_point() - Allocate ECC point.
+ * ecc_alloc_point() - Allocate ECC point.
  *
  * @ndigits:		Length of vlis in u64 qwords.
  *
@@ -281,7 +283,7 @@ void ecc_free_point(struct ecc_point *p);
 /**
  * ecc_point_is_zero() - Check if point is zero.
  *
- * @p:			Point to check for zero.
+ * @point:		Point to check for zero.
  *
  * Return: true if point is the point at infinity, false otherwise.
  */
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
new file mode 100644
index 0000000000000..fa72691546205
--- /dev/null
+++ b/include/cxl/cxl.h
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2020 Intel Corporation. */
+/* Copyright(c) 2026 Advanced Micro Devices, Inc. */
+
+#ifndef __CXL_CXL_H__
+#define __CXL_CXL_H__
+
+#include <linux/node.h>
+#include <linux/ioport.h>
+#include <cxl/mailbox.h>
+
+/**
+ * enum cxl_devtype - delineate type-2 from a generic type-3 device
+ * @CXL_DEVTYPE_DEVMEM: Vendor specific CXL Type-2 device implementing HDM-D or
+ *			 HDM-DB, no requirement that this device implements a
+ *			 mailbox, or other memory-device-standard manageability
+ *			 flows.
+ * @CXL_DEVTYPE_CLASSMEM: Common class definition of a CXL Type-3 device with
+ *			   HDM-H and class-mandatory memory device registers
+ */
+enum cxl_devtype {
+	CXL_DEVTYPE_DEVMEM,
+	CXL_DEVTYPE_CLASSMEM,
+};
+
+struct device;
+
+/*
+ * Using struct_group() allows for per register-block-type helper routines,
+ * without requiring block-type agnostic code to include the prefix.
+ */
+struct cxl_regs {
+	/*
+	 * Common set of CXL Component register block base pointers
+	 * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure
+	 * @ras: CXL 2.0 8.2.5.9 CXL RAS Capability Structure
+	 */
+	struct_group_tagged(cxl_component_regs, component,
+		void __iomem *hdm_decoder;
+		void __iomem *ras;
+	);
+	/*
+	 * Common set of CXL Device register block base pointers
+	 * @status: CXL 2.0 8.2.8.3 Device Status Registers
+	 * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers
+	 * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers
+	 */
+	struct_group_tagged(cxl_device_regs, device_regs,
+		void __iomem *status, *mbox, *memdev;
+	);
+
+	struct_group_tagged(cxl_pmu_regs, pmu_regs,
+		void __iomem *pmu;
+	);
+
+	/*
+	 * RCH downstream port specific RAS register
+	 * @aer: CXL 3.0 8.2.1.1 RCH Downstream Port RCRB
+	 */
+	struct_group_tagged(cxl_rch_regs, rch_regs,
+		void __iomem *dport_aer;
+	);
+
+	/*
+	 * RCD upstream port specific PCIe cap register
+	 * @pcie_cap: CXL 3.0 8.2.1.2 RCD Upstream Port RCRB
+	 */
+	struct_group_tagged(cxl_rcd_regs, rcd_regs,
+		void __iomem *rcd_pcie_cap;
+	);
+};
+
+struct cxl_reg_map {
+	bool valid;
+	int id;
+	unsigned long offset;
+	unsigned long size;
+};
+
+struct cxl_component_reg_map {
+	struct cxl_reg_map hdm_decoder;
+	struct cxl_reg_map ras;
+};
+
+struct cxl_device_reg_map {
+	struct cxl_reg_map status;
+	struct cxl_reg_map mbox;
+	struct cxl_reg_map memdev;
+};
+
+struct cxl_pmu_reg_map {
+	struct cxl_reg_map pmu;
+};
+
+/**
+ * struct cxl_register_map - DVSEC harvested register block mapping parameters
+ * @host: device for devm operations and logging
+ * @base: virtual base of the register-block-BAR + @block_offset
+ * @resource: physical resource base of the register block
+ * @max_size: maximum mapping size to perform register search
+ * @reg_type: see enum cxl_regloc_type
+ * @component_map: cxl_reg_map for component registers
+ * @device_map: cxl_reg_maps for device registers
+ * @pmu_map: cxl_reg_maps for CXL Performance Monitoring Units
+ */
+struct cxl_register_map {
+	struct device *host;
+	void __iomem *base;
+	resource_size_t resource;
+	resource_size_t max_size;
+	u8 reg_type;
+	union {
+		struct cxl_component_reg_map component_map;
+		struct cxl_device_reg_map device_map;
+		struct cxl_pmu_reg_map pmu_map;
+	};
+};
+
+/**
+ * struct cxl_dpa_perf - DPA performance property entry
+ * @dpa_range: range for DPA address
+ * @coord: QoS performance data (i.e. latency, bandwidth)
+ * @cdat_coord: raw QoS performance data from CDAT
+ * @qos_class: QoS Class cookies
+ */
+struct cxl_dpa_perf {
+	struct range dpa_range;
+	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
+	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
+	int qos_class;
+};
+
+enum cxl_partition_mode {
+	CXL_PARTMODE_RAM,
+	CXL_PARTMODE_PMEM,
+};
+
+/**
+ * struct cxl_dpa_partition - DPA partition descriptor
+ * @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res)
+ * @perf: performance attributes of the partition from CDAT
+ * @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic...
+ */
+struct cxl_dpa_partition {
+	struct resource res;
+	struct cxl_dpa_perf perf;
+	enum cxl_partition_mode mode;
+};
+
+#define CXL_NR_PARTITIONS_MAX 2
+
+/**
+ * struct cxl_dev_state - The driver device state
+ *
+ * cxl_dev_state represents the CXL driver/device state.  It provides an
+ * interface to mailbox commands as well as some cached data about the device.
+ * Currently only memory devices are represented.
+ *
+ * @dev: The device associated with this CXL state
+ * @cxlmd: The device representing the CXL.mem capabilities of @dev
+ * @reg_map: component and ras register mapping parameters
+ * @regs: Parsed register blocks
+ * @cxl_dvsec: Offset to the PCIe device DVSEC
+ * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH)
+ * @media_ready: Indicate whether the device media is usable
+ * @dpa_res: Overall DPA resource tree for the device
+ * @part: DPA partition array
+ * @nr_partitions: Number of DPA partitions
+ * @serial: PCIe Device Serial Number
+ * @type: Generic Memory Class device or Vendor Specific Memory device
+ * @cxl_mbox: CXL mailbox context
+ * @cxlfs: CXL features context
+ */
+struct cxl_dev_state {
+	/* public for Type2 drivers */
+	struct device *dev;
+	struct cxl_memdev *cxlmd;
+
+	/* private for Type2 drivers */
+	struct cxl_register_map reg_map;
+	struct cxl_device_regs regs;
+	int cxl_dvsec;
+	bool rcd;
+	bool media_ready;
+	struct resource dpa_res;
+	struct cxl_dpa_partition part[CXL_NR_PARTITIONS_MAX];
+	unsigned int nr_partitions;
+	u64 serial;
+	enum cxl_devtype type;
+	struct cxl_mailbox cxl_mbox;
+#ifdef CONFIG_CXL_FEATURES
+	struct cxl_features_state *cxlfs;
+#endif
+};
+
+struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
+						 enum cxl_devtype type,
+						 u64 serial, u16 dvsec,
+						 size_t size, bool has_mbox);
+
+/**
+ * cxl_dev_state_create - safely create and cast a cxl dev state embedded in a
+ * driver specific struct.
+ *
+ * @parent: device behind the request
+ * @type: CXL device type
+ * @serial: device identification
+ * @dvsec: dvsec capability offset
+ * @drv_struct: driver struct embedding a cxl_dev_state struct
+ * @member: name of the struct cxl_dev_state member in drv_struct
+ * @mbox: true if mailbox supported
+ *
+ * Returns a pointer to the drv_struct allocated and embedding a cxl_dev_state
+ * struct initialized.
+ *
+ * Introduced for Type2 driver support.
+ */
+#define devm_cxl_dev_state_create(parent, type, serial, dvsec, drv_struct, member, mbox)	\
+	({										\
+		static_assert(__same_type(struct cxl_dev_state,				\
+			      ((drv_struct *)NULL)->member));				\
+		static_assert(offsetof(drv_struct, member) == 0);			\
+		(drv_struct *)_devm_cxl_dev_state_create(parent, type, serial, dvsec,	\
+						      sizeof(drv_struct), mbox);	\
+	})
+#endif /* __CXL_CXL_H__ */
diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h
index d05df49fd8846..25363541e69d5 100644
--- a/include/drm/bridge/dw_dp.h
+++ b/include/drm/bridge/dw_dp.h
@@ -11,8 +11,15 @@
 struct drm_encoder;
 struct dw_dp;
 
+enum {
+	DW_DP_MP_SINGLE_PIXEL,
+	DW_DP_MP_DUAL_PIXEL,
+	DW_DP_MP_QUAD_PIXEL,
+};
+
 struct dw_dp_plat_data {
 	u32 max_link_rate;
+	u8 pixel_mode;
 };
 
 struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 178f8f62c80fc..f03cd199aee73 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -261,6 +261,19 @@ struct drm_private_state;
  * drm_atomic_get_private_obj_state().
  */
 struct drm_private_state_funcs {
+	/**
+	 * @atomic_create_state:
+	 *
+	 * Allocates a pristine, initialized, state for the private
+	 * object and returns it.
+	 *
+	 * RETURNS:
+	 *
+	 * A new, pristine, private state instance or an error pointer
+	 * on failure.
+	 */
+	struct drm_private_state *(*atomic_create_state)(struct drm_private_obj *obj);
+
 	/**
 	 * @atomic_duplicate_state:
 	 *
@@ -723,10 +736,9 @@ struct drm_connector_state * __must_check
 drm_atomic_get_connector_state(struct drm_atomic_state *state,
 			       struct drm_connector *connector);
 
-void drm_atomic_private_obj_init(struct drm_device *dev,
-				 struct drm_private_obj *obj,
-				 struct drm_private_state *state,
-				 const struct drm_private_state_funcs *funcs);
+int drm_atomic_private_obj_init(struct drm_device *dev,
+				struct drm_private_obj *obj,
+				const struct drm_private_state_funcs *funcs);
 void drm_atomic_private_obj_fini(struct drm_private_obj *obj);
 
 struct drm_private_state * __must_check
diff --git a/include/drm/drm_atomic_state_helper.h b/include/drm/drm_atomic_state_helper.h
index b9740edb26586..900672c6ea90b 100644
--- a/include/drm/drm_atomic_state_helper.h
+++ b/include/drm/drm_atomic_state_helper.h
@@ -84,6 +84,9 @@ void
 __drm_atomic_helper_connector_destroy_state(struct drm_connector_state *state);
 void drm_atomic_helper_connector_destroy_state(struct drm_connector *connector,
 					  struct drm_connector_state *state);
+
+void __drm_atomic_helper_private_obj_create_state(struct drm_private_obj *obj,
+						  struct drm_private_state *state);
 void __drm_atomic_helper_private_obj_duplicate_state(struct drm_private_obj *obj,
 						     struct drm_private_state *state);
 
diff --git a/include/drm/drm_blend.h b/include/drm/drm_blend.h
index 88bdfec3bd884..c7e888767c81c 100644
--- a/include/drm/drm_blend.h
+++ b/include/drm/drm_blend.h
@@ -31,8 +31,9 @@
 #define DRM_MODE_BLEND_COVERAGE		1
 #define DRM_MODE_BLEND_PIXEL_NONE	2
 
-struct drm_device;
 struct drm_atomic_state;
+struct drm_crtc;
+struct drm_device;
 struct drm_plane;
 
 static inline bool drm_rotation_90_or_270(unsigned int rotation)
@@ -58,4 +59,5 @@ int drm_atomic_normalize_zpos(struct drm_device *dev,
 			      struct drm_atomic_state *state);
 int drm_plane_create_blend_mode_property(struct drm_plane *plane,
 					 unsigned int supported_modes);
+void drm_crtc_attach_background_color_property(struct drm_crtc *crtc);
 #endif
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 4f19f7064ee31..66ab89cf48aaf 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -1290,6 +1290,7 @@ void drm_bridge_unplug(struct drm_bridge *bridge);
 
 struct drm_bridge *drm_bridge_get(struct drm_bridge *bridge);
 void drm_bridge_put(struct drm_bridge *bridge);
+void drm_bridge_clear_and_put(struct drm_bridge **bridge_pp);
 
 /* Cleanup action for use with __free() */
 DEFINE_FREE(drm_bridge_put, struct drm_bridge *, if (_T) drm_bridge_put(_T))
diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h
index b909fa8f810a0..3054369bebffb 100644
--- a/include/drm/drm_buddy.h
+++ b/include/drm/drm_buddy.h
@@ -6,166 +6,13 @@
 #ifndef __DRM_BUDDY_H__
 #define __DRM_BUDDY_H__
 
-#include <linux/bitops.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/sched.h>
-#include <linux/rbtree.h>
+#include <linux/gpu_buddy.h>
 
 struct drm_printer;
 
-#define DRM_BUDDY_RANGE_ALLOCATION		BIT(0)
-#define DRM_BUDDY_TOPDOWN_ALLOCATION		BIT(1)
-#define DRM_BUDDY_CONTIGUOUS_ALLOCATION		BIT(2)
-#define DRM_BUDDY_CLEAR_ALLOCATION		BIT(3)
-#define DRM_BUDDY_CLEARED			BIT(4)
-#define DRM_BUDDY_TRIM_DISABLE			BIT(5)
-
-struct drm_buddy_block {
-#define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
-#define DRM_BUDDY_HEADER_STATE  GENMASK_ULL(11, 10)
-#define   DRM_BUDDY_ALLOCATED	   (1 << 10)
-#define   DRM_BUDDY_FREE	   (2 << 10)
-#define   DRM_BUDDY_SPLIT	   (3 << 10)
-#define DRM_BUDDY_HEADER_CLEAR  GENMASK_ULL(9, 9)
-/* Free to be used, if needed in the future */
-#define DRM_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6)
-#define DRM_BUDDY_HEADER_ORDER  GENMASK_ULL(5, 0)
-	u64 header;
-
-	struct drm_buddy_block *left;
-	struct drm_buddy_block *right;
-	struct drm_buddy_block *parent;
-
-	void *private; /* owned by creator */
-
-	/*
-	 * While the block is allocated by the user through drm_buddy_alloc*,
-	 * the user has ownership of the link, for example to maintain within
-	 * a list, if so desired. As soon as the block is freed with
-	 * drm_buddy_free* ownership is given back to the mm.
-	 */
-	union {
-		struct rb_node rb;
-		struct list_head link;
-	};
-
-	struct list_head tmp_link;
-};
-
-/* Order-zero must be at least SZ_4K */
-#define DRM_BUDDY_MAX_ORDER (63 - 12)
-
-/*
- * Binary Buddy System.
- *
- * Locking should be handled by the user, a simple mutex around
- * drm_buddy_alloc* and drm_buddy_free* should suffice.
- */
-struct drm_buddy {
-	/* Maintain a free list for each order. */
-	struct rb_root **free_trees;
-
-	/*
-	 * Maintain explicit binary tree(s) to track the allocation of the
-	 * address space. This gives us a simple way of finding a buddy block
-	 * and performing the potentially recursive merge step when freeing a
-	 * block.  Nodes are either allocated or free, in which case they will
-	 * also exist on the respective free list.
-	 */
-	struct drm_buddy_block **roots;
-
-	/*
-	 * Anything from here is public, and remains static for the lifetime of
-	 * the mm. Everything above is considered do-not-touch.
-	 */
-	unsigned int n_roots;
-	unsigned int max_order;
-
-	/* Must be at least SZ_4K */
-	u64 chunk_size;
-	u64 size;
-	u64 avail;
-	u64 clear_avail;
-};
-
-static inline u64
-drm_buddy_block_offset(const struct drm_buddy_block *block)
-{
-	return block->header & DRM_BUDDY_HEADER_OFFSET;
-}
-
-static inline unsigned int
-drm_buddy_block_order(struct drm_buddy_block *block)
-{
-	return block->header & DRM_BUDDY_HEADER_ORDER;
-}
-
-static inline unsigned int
-drm_buddy_block_state(struct drm_buddy_block *block)
-{
-	return block->header & DRM_BUDDY_HEADER_STATE;
-}
-
-static inline bool
-drm_buddy_block_is_allocated(struct drm_buddy_block *block)
-{
-	return drm_buddy_block_state(block) == DRM_BUDDY_ALLOCATED;
-}
-
-static inline bool
-drm_buddy_block_is_clear(struct drm_buddy_block *block)
-{
-	return block->header & DRM_BUDDY_HEADER_CLEAR;
-}
-
-static inline bool
-drm_buddy_block_is_free(struct drm_buddy_block *block)
-{
-	return drm_buddy_block_state(block) == DRM_BUDDY_FREE;
-}
-
-static inline bool
-drm_buddy_block_is_split(struct drm_buddy_block *block)
-{
-	return drm_buddy_block_state(block) == DRM_BUDDY_SPLIT;
-}
-
-static inline u64
-drm_buddy_block_size(struct drm_buddy *mm,
-		     struct drm_buddy_block *block)
-{
-	return mm->chunk_size << drm_buddy_block_order(block);
-}
-
-int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size);
-
-void drm_buddy_fini(struct drm_buddy *mm);
-
-struct drm_buddy_block *
-drm_get_buddy(struct drm_buddy_block *block);
-
-int drm_buddy_alloc_blocks(struct drm_buddy *mm,
-			   u64 start, u64 end, u64 size,
-			   u64 min_page_size,
-			   struct list_head *blocks,
-			   unsigned long flags);
-
-int drm_buddy_block_trim(struct drm_buddy *mm,
-			 u64 *start,
-			 u64 new_size,
-			 struct list_head *blocks);
-
-void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear);
-
-void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block);
-
-void drm_buddy_free_list(struct drm_buddy *mm,
-			 struct list_head *objects,
-			 unsigned int flags);
-
-void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p);
-void drm_buddy_block_print(struct drm_buddy *mm,
-			   struct drm_buddy_block *block,
+/* DRM-specific GPU Buddy Allocator print helpers */
+void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p);
+void drm_buddy_block_print(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block,
 			   struct drm_printer *p);
 #endif
diff --git a/include/drm/drm_client.h b/include/drm/drm_client.h
index c972a8a3385b3..49a21f3dcb36f 100644
--- a/include/drm/drm_client.h
+++ b/include/drm/drm_client.h
@@ -195,6 +195,9 @@ struct drm_client_buffer {
 	struct drm_framebuffer *fb;
 };
 
+struct drm_client_buffer *
+drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height,
+			 u32 format, u32 handle, u32 pitch);
 struct drm_client_buffer *
 drm_client_buffer_create_dumb(struct drm_client_dev *client, u32 width, u32 height, u32 format);
 void drm_client_buffer_delete(struct drm_client_buffer *buffer);
diff --git a/include/drm/drm_colorop.h b/include/drm/drm_colorop.h
index a3a32f9f918c7..bd082854ca74c 100644
--- a/include/drm/drm_colorop.h
+++ b/include/drm/drm_colorop.h
@@ -187,6 +187,19 @@ struct drm_colorop_state {
 	struct drm_atomic_state *state;
 };
 
+/**
+ * struct drm_colorop_funcs - driver colorop control functions
+ */
+struct drm_colorop_funcs {
+	/**
+	 * @destroy:
+	 *
+	 * Clean up colorop resources. This is called at driver unload time
+	 * through drm_mode_config_cleanup()
+	 */
+	void (*destroy)(struct drm_colorop *colorop);
+};
+
 /**
  * struct drm_colorop - DRM color operation control structure
  *
@@ -362,6 +375,8 @@ struct drm_colorop {
 	 */
 	struct drm_property *next_property;
 
+	/** @funcs: colorop control functions */
+	const struct drm_colorop_funcs *funcs;
 };
 
 #define obj_to_colorop(x) container_of(x, struct drm_colorop, base)
@@ -390,17 +405,22 @@ void drm_colorop_pipeline_destroy(struct drm_device *dev);
 void drm_colorop_cleanup(struct drm_colorop *colorop);
 
 int drm_plane_colorop_curve_1d_init(struct drm_device *dev, struct drm_colorop *colorop,
-				    struct drm_plane *plane, u64 supported_tfs, uint32_t flags);
+				    struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				    u64 supported_tfs, uint32_t flags);
 int drm_plane_colorop_curve_1d_lut_init(struct drm_device *dev, struct drm_colorop *colorop,
-					struct drm_plane *plane, uint32_t lut_size,
+					struct drm_plane *plane,
+					const struct drm_colorop_funcs *funcs,
+					uint32_t lut_size,
 					enum drm_colorop_lut1d_interpolation_type interpolation,
 					uint32_t flags);
 int drm_plane_colorop_ctm_3x4_init(struct drm_device *dev, struct drm_colorop *colorop,
-				   struct drm_plane *plane, uint32_t flags);
+				   struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				   uint32_t flags);
 int drm_plane_colorop_mult_init(struct drm_device *dev, struct drm_colorop *colorop,
-				struct drm_plane *plane, uint32_t flags);
+				struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+				uint32_t flags);
 int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *colorop,
-				 struct drm_plane *plane,
+				 struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
 				 uint32_t lut_size,
 				 enum drm_colorop_lut3d_interpolation_type interpolation,
 				 uint32_t flags);
@@ -420,6 +440,8 @@ void drm_colorop_atomic_destroy_state(struct drm_colorop *colorop,
  */
 void drm_colorop_reset(struct drm_colorop *colorop);
 
+void drm_colorop_destroy(struct drm_colorop *colorop);
+
 /**
  * drm_colorop_index - find the index of a registered colorop
  * @colorop: colorop to find index for
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index 7eaec37ae1c73..c18be8c19de0a 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -2493,6 +2493,7 @@ int drm_connector_attach_scaling_mode_property(struct drm_connector *connector,
 					       u32 scaling_mode_mask);
 int drm_connector_attach_vrr_capable_property(
 		struct drm_connector *connector);
+void drm_connector_attach_panel_type_property(struct drm_connector *connector);
 int drm_connector_attach_broadcast_rgb_property(struct drm_connector *connector);
 int drm_connector_attach_colorspace_property(struct drm_connector *connector);
 int drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector);
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 66278ffeebd68..312fc1e745d22 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -274,6 +274,18 @@ struct drm_crtc_state {
 	 */
 	struct drm_property_blob *gamma_lut;
 
+	/**
+	 * @background_color:
+	 *
+	 * RGB value representing the CRTC's background color.  The background
+	 * color (aka "canvas color") of a CRTC is the color that will be used
+	 * for pixels not covered by a plane, or covered by transparent pixels
+	 * of a plane.  The value here should be built using DRM_ARGB64_PREP*()
+	 * helpers, while the individual color components can be extracted with
+	 * desired precision via the DRM_ARGB64_GET*() macros.
+	 */
+	u64 background_color;
+
 	/**
 	 * @target_vblank:
 	 *
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index 05cca77b7249c..80c81fdb04a2b 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -271,111 +271,16 @@ int drm_fb_helper_ioctl(struct fb_info *info, unsigned int cmd,
 
 int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper);
 int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper);
+bool drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
+			     const struct drm_gem_object *obj);
 #else
-static inline void drm_fb_helper_prepare(struct drm_device *dev,
-					 struct drm_fb_helper *helper,
-					 unsigned int preferred_bpp,
-					 const struct drm_fb_helper_funcs *funcs)
+static inline bool
+drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
+			const struct drm_gem_object *obj)
 {
+	return false;
 }
 
-static inline void drm_fb_helper_unprepare(struct drm_fb_helper *fb_helper)
-{
-}
-
-static inline int drm_fb_helper_init(struct drm_device *dev,
-		       struct drm_fb_helper *helper)
-{
-	/* So drivers can use it to free the struct */
-	helper->dev = dev;
-	dev->fb_helper = helper;
-
-	return 0;
-}
-
-static inline void drm_fb_helper_fini(struct drm_fb_helper *helper)
-{
-	if (helper && helper->dev)
-		helper->dev->fb_helper = NULL;
-}
-
-static inline int drm_fb_helper_blank(int blank, struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_pan_display(struct fb_var_screeninfo *var,
-					    struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_set_par(struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
-					  struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int
-drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
-{
-	return 0;
-}
-
-static inline void drm_fb_helper_unregister_info(struct drm_fb_helper *fb_helper)
-{
-}
-
-static inline void
-drm_fb_helper_fill_info(struct fb_info *info,
-			struct drm_fb_helper *fb_helper,
-			struct drm_fb_helper_surface_size *sizes)
-{
-}
-
-static inline int drm_fb_helper_setcmap(struct fb_cmap *cmap,
-					struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_ioctl(struct fb_info *info, unsigned int cmd,
-				      unsigned long arg)
-{
-	return 0;
-}
-
-#ifdef CONFIG_FB_DEFERRED_IO
-static inline void drm_fb_helper_deferred_io(struct fb_info *info,
-					     struct list_head *pagelist)
-{
-}
-#endif
-
-static inline void drm_fb_helper_set_suspend(struct drm_fb_helper *fb_helper,
-					     bool suspend)
-{
-}
-
-static inline void
-drm_fb_helper_set_suspend_unlocked(struct drm_fb_helper *fb_helper, bool suspend)
-{
-}
-
-static inline int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper)
-{
-	return 0;
-}
 #endif
 
 #endif
diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
index 895fb820dba07..687c0ee163d25 100644
--- a/include/drm/drm_mode_config.h
+++ b/include/drm/drm_mode_config.h
@@ -600,6 +600,10 @@ struct drm_mode_config {
 	 * multiple CRTCs.
 	 */
 	struct drm_property *tile_property;
+	/**
+	 * @panel_type_property: Default connector property for panel type
+	 */
+	struct drm_property *panel_type_property;
 	/**
 	 * @link_status_property: Default connector property for link status
 	 * of a connector
@@ -832,6 +836,11 @@ struct drm_mode_config {
 	 * gamma LUT as supported by the driver (read-only).
 	 */
 	struct drm_property *gamma_lut_size_property;
+	/**
+	 * @background_color_property: Optional CRTC property to set the
+	 * background color.
+	 */
+	struct drm_property *background_color_property;
 
 	/**
 	 * @suggested_x_property: Optional connector property with a hint for
diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h
index c848f578e3daa..75e6ca58922d5 100644
--- a/include/drm/drm_pagemap.h
+++ b/include/drm/drm_pagemap.h
@@ -4,6 +4,7 @@
 
 #include <linux/dma-direction.h>
 #include <linux/hmm.h>
+#include <linux/memremap.h>
 #include <linux/types.h>
 
 #define NR_PAGES(order) (1U << (order))
@@ -367,6 +368,26 @@ void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim
 
 int drm_pagemap_reinit(struct drm_pagemap *dpagemap);
 
+/**
+ * drm_pagemap_page_zone_device_data() - Page to zone_device_data
+ * @page: Pointer to the page
+ *
+ * Return: Page's zone_device_data
+ */
+static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page)
+{
+	struct folio *folio = page_folio(page);
+
+	return folio_zone_device_data(folio);
+}
+
+#else
+
+static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page)
+{
+	return NULL;
+}
+
 #endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */
 
 #endif
diff --git a/include/drm/drm_ras.h b/include/drm/drm_ras.h
new file mode 100644
index 0000000000000..5d50209e51db4
--- /dev/null
+++ b/include/drm/drm_ras.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __DRM_RAS_H__
+#define __DRM_RAS_H__
+
+#include <uapi/drm/drm_ras.h>
+
+/**
+ * struct drm_ras_node - A DRM RAS Node
+ */
+struct drm_ras_node {
+	/** @id: Unique identifier for the node. Dynamically assigned. */
+	u32 id;
+	/**
+	 * @device_name: Human-readable name of the device. Given by the driver.
+	 */
+	const char *device_name;
+	/** @node_name: Human-readable name of the node. Given by the driver. */
+	const char *node_name;
+	/** @type: Type of the node (enum drm_ras_node_type). */
+	enum drm_ras_node_type type;
+
+	/* Error-Counter Related Callback and Variables */
+
+	/** @error_counter_range: Range of valid Error IDs for this node. */
+	struct {
+		/** @first: First valid Error ID. */
+		u32 first;
+		/** @last: Last valid Error ID. Mandatory entry. */
+		u32 last;
+	} error_counter_range;
+
+	/**
+	 * @query_error_counter:
+	 *
+	 * This callback is used by drm-ras to query a specific error counter.
+	 * Used for input check and to iterate all error counters in a node.
+	 *
+	 * Driver should expect query_error_counter() to be called with
+	 * error_id from `error_counter_range.first` to
+	 * `error_counter_range.last`.
+	 *
+	 * The @query_error_counter is a mandatory callback for
+	 * error_counter_node.
+	 *
+	 * Returns: 0 on success,
+	 *          -ENOENT when error_id is not supported as an indication that
+	 *                  drm_ras should silently skip this entry. Used for
+	 *                  supporting non-contiguous error ranges.
+	 *                  Driver is responsible for maintaining the list of
+	 *                  supported error IDs in the range of first to last.
+	 *          Other negative values on errors that should terminate the
+	 *          netlink query.
+	 */
+	int (*query_error_counter)(struct drm_ras_node *node, u32 error_id,
+				   const char **name, u32 *val);
+
+	/** @priv: Driver private data */
+	void *priv;
+};
+
+struct drm_device;
+
+#if IS_ENABLED(CONFIG_DRM_RAS)
+int drm_ras_node_register(struct drm_ras_node *node);
+void drm_ras_node_unregister(struct drm_ras_node *node);
+#else
+static inline int drm_ras_node_register(struct drm_ras_node *node) { return 0; }
+static inline void drm_ras_node_unregister(struct drm_ras_node *node) { }
+#endif
+
+#endif
diff --git a/include/drm/drm_ras_genl_family.h b/include/drm/drm_ras_genl_family.h
new file mode 100644
index 0000000000000..910fb3943a752
--- /dev/null
+++ b/include/drm/drm_ras_genl_family.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __DRM_RAS_GENL_FAMILY_H__
+#define __DRM_RAS_GENL_FAMILY_H__
+
+#if IS_ENABLED(CONFIG_DRM_RAS)
+int drm_ras_genl_family_register(void);
+void drm_ras_genl_family_unregister(void);
+#else
+static inline int drm_ras_genl_family_register(void) { return 0; }
+static inline void drm_ras_genl_family_unregister(void) { }
+#endif
+
+#endif
diff --git a/include/drm/drm_suballoc.h b/include/drm/drm_suballoc.h
index 7ba72a81a808f..29befdda35d23 100644
--- a/include/drm/drm_suballoc.h
+++ b/include/drm/drm_suballoc.h
@@ -53,6 +53,12 @@ void drm_suballoc_manager_init(struct drm_suballoc_manager *sa_manager,
 
 void drm_suballoc_manager_fini(struct drm_suballoc_manager *sa_manager);
 
+struct drm_suballoc *drm_suballoc_alloc(gfp_t gfp);
+
+int drm_suballoc_insert(struct drm_suballoc_manager *sa_manager,
+			struct drm_suballoc *sa, size_t size, bool intr,
+			size_t align);
+
 struct drm_suballoc *
 drm_suballoc_new(struct drm_suballoc_manager *sa_manager, size_t size,
 		 gfp_t gfp, bool intr, size_t align);
diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h
index ce946859a3a97..97ec94a2e7493 100644
--- a/include/drm/intel/display_parent_interface.h
+++ b/include/drm/intel/display_parent_interface.h
@@ -9,19 +9,66 @@
 struct dma_fence;
 struct drm_crtc;
 struct drm_device;
+struct drm_file;
 struct drm_framebuffer;
 struct drm_gem_object;
+struct drm_mode_fb_cmd2;
 struct drm_plane_state;
 struct drm_scanout_buffer;
 struct i915_vma;
+struct intel_dpt;
+struct intel_dsb_buffer;
+struct intel_frontbuffer;
 struct intel_hdcp_gsc_context;
 struct intel_initial_plane_config;
 struct intel_panic;
 struct intel_stolen_node;
 struct ref_tracker;
+struct seq_file;
+struct vm_area_struct;
 
 /* Keep struct definitions sorted */
 
+struct intel_display_bo_interface {
+	bool (*is_tiled)(struct drm_gem_object *obj); /* Optional */
+	bool (*is_userptr)(struct drm_gem_object *obj); /* Optional */
+	bool (*is_shmem)(struct drm_gem_object *obj); /* Optional */
+	bool (*is_protected)(struct drm_gem_object *obj);
+	int (*key_check)(struct drm_gem_object *obj);
+	int (*fb_mmap)(struct drm_gem_object *obj, struct vm_area_struct *vma);
+	int (*read_from_page)(struct drm_gem_object *obj, u64 offset, void *dst, int size);
+	void (*describe)(struct seq_file *m, struct drm_gem_object *obj); /* Optional */
+	int (*framebuffer_init)(struct drm_gem_object *obj, struct drm_mode_fb_cmd2 *mode_cmd);
+	void (*framebuffer_fini)(struct drm_gem_object *obj);
+	struct drm_gem_object *(*framebuffer_lookup)(struct drm_device *drm,
+						     struct drm_file *filp,
+						     const struct drm_mode_fb_cmd2 *user_mode_cmd);
+};
+
+struct intel_display_dpt_interface {
+	struct intel_dpt *(*create)(struct drm_gem_object *obj, size_t size);
+	void (*destroy)(struct intel_dpt *dpt);
+	void (*suspend)(struct intel_dpt *dpt);
+	void (*resume)(struct intel_dpt *dpt);
+};
+
+struct intel_display_dsb_interface {
+	u32 (*ggtt_offset)(struct intel_dsb_buffer *dsb_buf);
+	void (*write)(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val);
+	u32 (*read)(struct intel_dsb_buffer *dsb_buf, u32 idx);
+	void (*fill)(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size);
+	struct intel_dsb_buffer *(*create)(struct drm_device *drm, size_t size);
+	void (*cleanup)(struct intel_dsb_buffer *dsb_buf);
+	void (*flush_map)(struct intel_dsb_buffer *dsb_buf);
+};
+
+struct intel_display_frontbuffer_interface {
+	struct intel_frontbuffer *(*get)(struct drm_gem_object *obj);
+	void (*ref)(struct intel_frontbuffer *front);
+	void (*put)(struct intel_frontbuffer *front);
+	void (*flush_for_display)(struct intel_frontbuffer *front);
+};
+
 struct intel_display_hdcp_interface {
 	ssize_t (*gsc_msg_send)(struct intel_hdcp_gsc_context *gsc_context,
 				void *msg_in, size_t msg_in_len,
@@ -44,6 +91,35 @@ struct intel_display_irq_interface {
 	void (*synchronize)(struct drm_device *drm);
 };
 
+struct intel_display_overlay_interface {
+	bool (*is_active)(struct drm_device *drm);
+
+	int (*overlay_on)(struct drm_device *drm,
+			  u32 frontbuffer_bits);
+	int (*overlay_continue)(struct drm_device *drm,
+				struct i915_vma *vma,
+				bool load_polyphase_filter);
+	int (*overlay_off)(struct drm_device *drm);
+	int (*recover_from_interrupt)(struct drm_device *drm);
+	int (*release_old_vid)(struct drm_device *drm);
+
+	void (*reset)(struct drm_device *drm);
+
+	struct i915_vma *(*pin_fb)(struct drm_device *drm,
+				   struct drm_gem_object *obj,
+				   u32 *offset);
+	void (*unpin_fb)(struct drm_device *drm,
+			 struct i915_vma *vma);
+
+	struct drm_gem_object *(*obj_lookup)(struct drm_device *drm,
+					     struct drm_file *filp,
+					     u32 handle);
+
+	void __iomem *(*setup)(struct drm_device *drm,
+			       bool needs_physical);
+	void (*cleanup)(struct drm_device *drm);
+};
+
 struct intel_display_panic_interface {
 	struct intel_panic *(*alloc)(void);
 	int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb);
@@ -55,6 +131,13 @@ struct intel_display_pc8_interface {
 	void (*unblock)(struct drm_device *drm);
 };
 
+struct intel_display_pcode_interface {
+	int (*read)(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1);
+	int (*write)(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms);
+	int (*request)(struct drm_device *drm, u32 mbox, u32 request,
+		       u32 reply_mask, u32 reply, int timeout_base_ms);
+};
+
 struct intel_display_rpm_interface {
 	struct ref_tracker *(*get)(const struct drm_device *drm);
 	struct ref_tracker *(*get_raw)(const struct drm_device *drm);
@@ -93,6 +176,10 @@ struct intel_display_stolen_interface {
 	void (*node_free)(const struct intel_stolen_node *node);
 };
 
+struct intel_display_vma_interface {
+	int (*fence_id)(const struct i915_vma *vma);
+};
+
 /**
  * struct intel_display_parent_interface - services parent driver provides to display
  *
@@ -106,6 +193,18 @@ struct intel_display_stolen_interface {
  * check the optional pointers.
  */
 struct intel_display_parent_interface {
+	/** @bo: BO interface */
+	const struct intel_display_bo_interface *bo;
+
+	/** @dpt: DPT interface. Optional. */
+	const struct intel_display_dpt_interface *dpt;
+
+	/** @dsb: DSB buffer interface */
+	const struct intel_display_dsb_interface *dsb;
+
+	/** @frontbuffer: Frontbuffer interface */
+	const struct intel_display_frontbuffer_interface *frontbuffer;
+
 	/** @hdcp: HDCP GSC interface */
 	const struct intel_display_hdcp_interface *hdcp;
 
@@ -118,9 +217,15 @@ struct intel_display_parent_interface {
 	/** @panic: Panic interface */
 	const struct intel_display_panic_interface *panic;
 
+	/** @overlay: Overlay. Optional. */
+	const struct intel_display_overlay_interface *overlay;
+
 	/** @pc8: PC8 interface. Optional. */
 	const struct intel_display_pc8_interface *pc8;
 
+	/** @pcode: Pcode interface */
+	const struct intel_display_pcode_interface *pcode;
+
 	/** @rpm: Runtime PM functions */
 	const struct intel_display_rpm_interface *rpm;
 
@@ -130,6 +235,9 @@ struct intel_display_parent_interface {
 	/** @stolen: Stolen memory. */
 	const struct intel_display_stolen_interface *stolen;
 
+	/** @vma: VMA interface. Optional. */
+	const struct intel_display_vma_interface *vma;
+
 	/* Generic independent functions */
 	struct {
 		/** @fence_priority_display: Set display priority. Optional. */
diff --git a/include/drm/intel/i915_drm.h b/include/drm/intel/i915_drm.h
index adff685384844..1fdaabed14702 100644
--- a/include/drm/intel/i915_drm.h
+++ b/include/drm/intel/i915_drm.h
@@ -39,46 +39,46 @@ bool i915_gpu_turbo_disable(void);
 extern struct resource intel_graphics_stolen_res;
 
 /*
- * The Bridge device's PCI config space has information about the
- * fb aperture size and the amount of pre-reserved memory.
- * This is all handled in the intel-gtt.ko module. i915.ko only
- * cares about the vga bit for the vga arbiter.
+ * The bridge device's (device 0) PCI config space has information
+ * about the fb aperture size and the amount of pre-reserved memory.
  */
-#define INTEL_GMCH_CTRL		0x52
-#define INTEL_GMCH_VGA_DISABLE  (1 << 1)
+
+/* device 2 has a read-only mirror */
 #define SNB_GMCH_CTRL		0x50
-#define    SNB_GMCH_GGMS_SHIFT	8 /* GTT Graphics Memory Size */
-#define    SNB_GMCH_GGMS_MASK	0x3
-#define    SNB_GMCH_GMS_SHIFT   3 /* Graphics Mode Select */
-#define    SNB_GMCH_GMS_MASK    0x1f
-#define    BDW_GMCH_GGMS_SHIFT	6
-#define    BDW_GMCH_GGMS_MASK	0x3
-#define    BDW_GMCH_GMS_SHIFT   8
-#define    BDW_GMCH_GMS_MASK    0xff
+#define   SNB_GMCH_GGMS_SHIFT	8 /* GTT Graphics Memory Size */
+#define   SNB_GMCH_GGMS_MASK	0x3
+#define   SNB_GMCH_GMS_SHIFT	3 /* Graphics Mode Select */
+#define   SNB_GMCH_GMS_MASK	0x1f
+#define   BDW_GMCH_GGMS_SHIFT	6
+#define   BDW_GMCH_GGMS_MASK	0x3
+#define   BDW_GMCH_GMS_SHIFT	8
+#define   BDW_GMCH_GMS_MASK	0xff
 
+/* device 2 has a read-only mirror from i85x/i865 onwards */
 #define I830_GMCH_CTRL			0x52
+#define   I830_GMCH_GMS_MASK		(0x7 << 4)
+#define   I830_GMCH_GMS_LOCAL		(0x1 << 4)
+#define   I830_GMCH_GMS_STOLEN_512	(0x2 << 4)
+#define   I830_GMCH_GMS_STOLEN_1024	(0x3 << 4)
+#define   I830_GMCH_GMS_STOLEN_8192	(0x4 << 4)
+#define   I855_GMCH_GMS_MASK		(0xF << 4)
+#define   I855_GMCH_GMS_STOLEN_0M	(0x0 << 4)
+#define   I855_GMCH_GMS_STOLEN_1M	(0x1 << 4)
+#define   I855_GMCH_GMS_STOLEN_4M	(0x2 << 4)
+#define   I855_GMCH_GMS_STOLEN_8M	(0x3 << 4)
+#define   I855_GMCH_GMS_STOLEN_16M	(0x4 << 4)
+#define   I855_GMCH_GMS_STOLEN_32M	(0x5 << 4)
+#define   I915_GMCH_GMS_STOLEN_48M	(0x6 << 4)
+#define   I915_GMCH_GMS_STOLEN_64M	(0x7 << 4)
+#define   G33_GMCH_GMS_STOLEN_128M	(0x8 << 4)
+#define   G33_GMCH_GMS_STOLEN_256M	(0x9 << 4)
+#define   INTEL_GMCH_GMS_STOLEN_96M	(0xa << 4)
+#define   INTEL_GMCH_GMS_STOLEN_160M	(0xb << 4)
+#define   INTEL_GMCH_GMS_STOLEN_224M	(0xc << 4)
+#define   INTEL_GMCH_GMS_STOLEN_352M	(0xd << 4)
 
-#define I830_GMCH_GMS_MASK		0x70
-#define I830_GMCH_GMS_LOCAL		0x10
-#define I830_GMCH_GMS_STOLEN_512	0x20
-#define I830_GMCH_GMS_STOLEN_1024	0x30
-#define I830_GMCH_GMS_STOLEN_8192	0x40
-
-#define I855_GMCH_GMS_MASK		0xF0
-#define I855_GMCH_GMS_STOLEN_0M		0x0
-#define I855_GMCH_GMS_STOLEN_1M		(0x1 << 4)
-#define I855_GMCH_GMS_STOLEN_4M		(0x2 << 4)
-#define I855_GMCH_GMS_STOLEN_8M		(0x3 << 4)
-#define I855_GMCH_GMS_STOLEN_16M	(0x4 << 4)
-#define I855_GMCH_GMS_STOLEN_32M	(0x5 << 4)
-#define I915_GMCH_GMS_STOLEN_48M	(0x6 << 4)
-#define I915_GMCH_GMS_STOLEN_64M	(0x7 << 4)
-#define G33_GMCH_GMS_STOLEN_128M	(0x8 << 4)
-#define G33_GMCH_GMS_STOLEN_256M	(0x9 << 4)
-#define INTEL_GMCH_GMS_STOLEN_96M	(0xa << 4)
-#define INTEL_GMCH_GMS_STOLEN_160M	(0xb << 4)
-#define INTEL_GMCH_GMS_STOLEN_224M	(0xc << 4)
-#define INTEL_GMCH_GMS_STOLEN_352M	(0xd << 4)
+/* valid for both I830_GMCH_CTRL and SNB_GMCH_CTRL */
+#define   INTEL_GMCH_VGA_DISABLE  (1 << 1)
 
 #define I830_DRB3		0x63
 #define I85X_DRB3		0x43
@@ -87,12 +87,12 @@ extern struct resource intel_graphics_stolen_res;
 #define I830_ESMRAMC		0x91
 #define I845_ESMRAMC		0x9e
 #define I85X_ESMRAMC		0x61
-#define    TSEG_ENABLE		(1 << 0)
-#define    I830_TSEG_SIZE_512K	(0 << 1)
-#define    I830_TSEG_SIZE_1M	(1 << 1)
-#define    I845_TSEG_SIZE_MASK	(3 << 1)
-#define    I845_TSEG_SIZE_512K	(2 << 1)
-#define    I845_TSEG_SIZE_1M	(3 << 1)
+#define   TSEG_ENABLE		(1 << 0)
+#define   I830_TSEG_SIZE_512K	(0 << 1)
+#define   I830_TSEG_SIZE_1M	(1 << 1)
+#define   I845_TSEG_SIZE_MASK	(3 << 1)
+#define   I845_TSEG_SIZE_512K	(2 << 1)
+#define   I845_TSEG_SIZE_1M	(3 << 1)
 
 #define INTEL_BSM		0x5c
 #define INTEL_GEN11_BSM_DW0	0xc0
diff --git a/include/drm/intel/intel_gmd_interrupt_regs.h b/include/drm/intel/intel_gmd_interrupt_regs.h
new file mode 100644
index 0000000000000..ce66c4151e769
--- /dev/null
+++ b/include/drm/intel/intel_gmd_interrupt_regs.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _INTEL_GMD_INTERRUPT_REGS_H_
+#define _INTEL_GMD_INTERRUPT_REGS_H_
+
+#define I915_PM_INTERRUPT				(1 << 31)
+#define I915_ISP_INTERRUPT				(1 << 22)
+#define I915_LPE_PIPE_B_INTERRUPT			(1 << 21)
+#define I915_LPE_PIPE_A_INTERRUPT			(1 << 20)
+#define I915_MIPIC_INTERRUPT				(1 << 19)
+#define I915_MIPIA_INTERRUPT				(1 << 18)
+#define I915_PIPE_CONTROL_NOTIFY_INTERRUPT		(1 << 18)
+#define I915_DISPLAY_PORT_INTERRUPT			(1 << 17)
+#define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT		(1 << 16)
+#define I915_MASTER_ERROR_INTERRUPT			(1 << 15)
+#define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT		(1 << 14)
+#define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT	(1 << 14) /* p-state */
+#define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT		(1 << 13)
+#define I915_HWB_OOM_INTERRUPT				(1 << 13)
+#define I915_LPE_PIPE_C_INTERRUPT			(1 << 12)
+#define I915_SYNC_STATUS_INTERRUPT			(1 << 12)
+#define I915_MISC_INTERRUPT				(1 << 11)
+#define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT	(1 << 11)
+#define I915_DISPLAY_PIPE_C_VBLANK_INTERRUPT		(1 << 10)
+#define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT	(1 << 10)
+#define I915_DISPLAY_PIPE_C_EVENT_INTERRUPT		(1 << 9)
+#define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT	(1 << 9)
+#define I915_DISPLAY_PIPE_C_DPBM_INTERRUPT		(1 << 8)
+#define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT	(1 << 8)
+#define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT		(1 << 7)
+#define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT		(1 << 6)
+#define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT		(1 << 5)
+#define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT		(1 << 4)
+#define I915_DISPLAY_PIPE_A_DPBM_INTERRUPT		(1 << 3)
+#define I915_DISPLAY_PIPE_B_DPBM_INTERRUPT		(1 << 2)
+#define I915_DEBUG_INTERRUPT				(1 << 2)
+#define I915_WINVALID_INTERRUPT				(1 << 1)
+#define I915_USER_INTERRUPT				(1 << 1)
+#define I915_ASLE_INTERRUPT				(1 << 0)
+#define I915_BSD_USER_INTERRUPT				(1 << 25)
+
+#define GEN8_MASTER_IRQ			_MMIO(0x44200)
+#define  GEN8_MASTER_IRQ_CONTROL	(1 << 31)
+#define  GEN8_PCU_IRQ			(1 << 30)
+#define  GEN8_DE_PCH_IRQ		(1 << 23)
+#define  GEN8_DE_MISC_IRQ		(1 << 22)
+#define  GEN8_DE_PORT_IRQ		(1 << 20)
+#define  GEN8_DE_PIPE_C_IRQ		(1 << 18)
+#define  GEN8_DE_PIPE_B_IRQ		(1 << 17)
+#define  GEN8_DE_PIPE_A_IRQ		(1 << 16)
+#define  GEN8_DE_PIPE_IRQ(pipe)		(1 << (16 + (pipe)))
+#define  GEN8_GT_VECS_IRQ		(1 << 6)
+#define  GEN8_GT_GUC_IRQ		(1 << 5)
+#define  GEN8_GT_PM_IRQ			(1 << 4)
+#define  GEN8_GT_VCS1_IRQ		(1 << 3) /* NB: VCS2 in bspec! */
+#define  GEN8_GT_VCS0_IRQ		(1 << 2) /* NB: VCS1 in bpsec! */
+#define  GEN8_GT_BCS_IRQ		(1 << 1)
+#define  GEN8_GT_RCS_IRQ		(1 << 0)
+
+#define GEN11_GU_MISC_ISR	_MMIO(0x444f0)
+#define GEN11_GU_MISC_IMR	_MMIO(0x444f4)
+#define GEN11_GU_MISC_IIR	_MMIO(0x444f8)
+#define GEN11_GU_MISC_IER	_MMIO(0x444fc)
+#define  GEN11_GU_MISC_GSE	(1 << 27)
+
+#define GEN11_GU_MISC_IRQ_REGS		I915_IRQ_REGS(GEN11_GU_MISC_IMR, \
+						      GEN11_GU_MISC_IER, \
+						      GEN11_GU_MISC_IIR)
+
+#define GEN11_GFX_MSTR_IRQ		_MMIO(0x190010)
+#define  GEN11_MASTER_IRQ		(1 << 31)
+#define  GEN11_PCU_IRQ			(1 << 30)
+#define  GEN11_GU_MISC_IRQ		(1 << 29)
+#define  GEN11_DISPLAY_IRQ		(1 << 16)
+#define  GEN11_GT_DW_IRQ(x)		(1 << (x))
+#define  GEN11_GT_DW1_IRQ		(1 << 1)
+#define  GEN11_GT_DW0_IRQ		(1 << 0)
+
+#define SCPD0		_MMIO(0x209c) /* 915+ only */
+#define  SCPD_FBC_IGNORE_3D			(1 << 6)
+#define  CSTATE_RENDER_CLOCK_GATE_DISABLE	(1 << 5)
+
+#define VLV_IIR_RW	_MMIO(VLV_DISPLAY_BASE + 0x2084)
+#define VLV_IER		_MMIO(VLV_DISPLAY_BASE + 0x20a0)
+#define VLV_IIR		_MMIO(VLV_DISPLAY_BASE + 0x20a4)
+#define VLV_IMR		_MMIO(VLV_DISPLAY_BASE + 0x20a8)
+#define VLV_ISR		_MMIO(VLV_DISPLAY_BASE + 0x20ac)
+#define VLV_PCBR	_MMIO(VLV_DISPLAY_BASE + 0x2120)
+#define VLV_PCBR_ADDR_SHIFT	12
+
+#endif
diff --git a/include/drm/intel/intel_gmd_misc_regs.h b/include/drm/intel/intel_gmd_misc_regs.h
new file mode 100644
index 0000000000000..763d7711f21c8
--- /dev/null
+++ b/include/drm/intel/intel_gmd_misc_regs.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _INTEL_GMD_MISC_REGS_H_
+#define _INTEL_GMD_MISC_REGS_H_
+
+#define DISP_ARB_CTL	_MMIO(0x45000)
+#define   DISP_FBC_MEMORY_WAKE		REG_BIT(31)
+#define   DISP_TILE_SURFACE_SWIZZLING	REG_BIT(13)
+#define   DISP_FBC_WM_DIS		REG_BIT(15)
+
+#define INSTPM	        _MMIO(0x20c0)
+#define   INSTPM_SELF_EN (1 << 12) /* 915GM only */
+#define   INSTPM_AGPBUSY_INT_EN (1 << 11) /* gen3: when disabled, pending interrupts
+					will not assert AGPBUSY# and will only
+					be delivered when out of C3. */
+#define   INSTPM_FORCE_ORDERING				(1 << 7) /* GEN6+ */
+#define   INSTPM_TLB_INVALIDATE	(1 << 9)
+#define   INSTPM_SYNC_FLUSH	(1 << 5)
+
+#endif
diff --git a/include/drm/intel/intel_pcode_regs.h b/include/drm/intel/intel_pcode_regs.h
new file mode 100644
index 0000000000000..db989ee7c4884
--- /dev/null
+++ b/include/drm/intel/intel_pcode_regs.h
@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _INTEL_PCODE_REGS_H_
+#define _INTEL_PCODE_REGS_H_
+
+#define GEN6_PCODE_MAILBOX			_MMIO(0x138124)
+#define   GEN6_PCODE_READY			(1 << 31)
+#define   GEN6_PCODE_MB_PARAM2			REG_GENMASK(23, 16)
+#define   GEN6_PCODE_MB_PARAM1			REG_GENMASK(15, 8)
+#define   GEN6_PCODE_MB_COMMAND			REG_GENMASK(7, 0)
+#define   GEN6_PCODE_ERROR_MASK			0xFF
+#define     GEN6_PCODE_SUCCESS			0x0
+#define     GEN6_PCODE_ILLEGAL_CMD		0x1
+#define     GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x2
+#define     GEN6_PCODE_TIMEOUT			0x3
+#define     GEN6_PCODE_UNIMPLEMENTED_CMD	0xFF
+#define     GEN7_PCODE_TIMEOUT			0x2
+#define     GEN7_PCODE_ILLEGAL_DATA		0x3
+#define     GEN11_PCODE_ILLEGAL_SUBCOMMAND	0x4
+#define     GEN11_PCODE_LOCKED			0x6
+#define     GEN11_PCODE_REJECTED		0x11
+#define     GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10
+#define   GEN6_PCODE_WRITE_RC6VIDS		0x4
+#define   GEN6_PCODE_READ_RC6VIDS		0x5
+#define     GEN6_ENCODE_RC6_VID(mv)		(((mv) - 245) / 5)
+#define     GEN6_DECODE_RC6_VID(vids)		(((vids) * 5) + 245)
+#define   BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ	0x18
+#define   GEN9_PCODE_READ_MEM_LATENCY		0x6
+#define     GEN9_MEM_LATENCY_LEVEL_3_7_MASK	REG_GENMASK(31, 24)
+#define     GEN9_MEM_LATENCY_LEVEL_2_6_MASK	REG_GENMASK(23, 16)
+#define     GEN9_MEM_LATENCY_LEVEL_1_5_MASK	REG_GENMASK(15, 8)
+#define     GEN9_MEM_LATENCY_LEVEL_0_4_MASK	REG_GENMASK(7, 0)
+#define   SKL_PCODE_LOAD_HDCP_KEYS		0x5
+#define   SKL_PCODE_CDCLK_CONTROL		0x7
+#define     SKL_CDCLK_PREPARE_FOR_CHANGE	0x3
+#define     SKL_CDCLK_READY_FOR_CHANGE		0x1
+#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE	0x8
+#define   GEN6_PCODE_READ_MIN_FREQ_TABLE	0x9
+#define   GEN6_READ_OC_PARAMS			0xc
+#define   ICL_PCODE_MEM_SUBSYSYSTEM_INFO	0xd
+#define     ICL_PCODE_MEM_SS_READ_GLOBAL_INFO	(0x0 << 8)
+#define     ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point)	(((point) << 16) | (0x1 << 8))
+#define     ADL_PCODE_MEM_SS_READ_PSF_GV_INFO	((0) | (0x2 << 8))
+#define   DISPLAY_TO_PCODE_CDCLK_MAX		0x28D
+#define   DISPLAY_TO_PCODE_VOLTAGE_MASK		REG_GENMASK(1, 0)
+#define	  DISPLAY_TO_PCODE_VOLTAGE_MAX		DISPLAY_TO_PCODE_VOLTAGE_MASK
+#define   DISPLAY_TO_PCODE_CDCLK_VALID		REG_BIT(27)
+#define   DISPLAY_TO_PCODE_PIPE_COUNT_VALID	REG_BIT(31)
+#define   DISPLAY_TO_PCODE_CDCLK_MASK		REG_GENMASK(25, 16)
+#define   DISPLAY_TO_PCODE_PIPE_COUNT_MASK	REG_GENMASK(30, 28)
+#define   DISPLAY_TO_PCODE_CDCLK(x)		REG_FIELD_PREP(DISPLAY_TO_PCODE_CDCLK_MASK, (x))
+#define   DISPLAY_TO_PCODE_PIPE_COUNT(x)	REG_FIELD_PREP(DISPLAY_TO_PCODE_PIPE_COUNT_MASK, (x))
+#define   DISPLAY_TO_PCODE_VOLTAGE(x)		REG_FIELD_PREP(DISPLAY_TO_PCODE_VOLTAGE_MASK, (x))
+#define   DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, num_pipes, voltage_level) \
+		((DISPLAY_TO_PCODE_CDCLK(cdclk)) | \
+		(DISPLAY_TO_PCODE_PIPE_COUNT(num_pipes)) | \
+		(DISPLAY_TO_PCODE_VOLTAGE(voltage_level)))
+#define   ICL_PCODE_SAGV_DE_MEM_SS_CONFIG	0xe
+#define     ICL_PCODE_REP_QGV_MASK		REG_GENMASK(1, 0)
+#define     ICL_PCODE_REP_QGV_SAFE		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 0)
+#define     ICL_PCODE_REP_QGV_POLL		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 1)
+#define     ICL_PCODE_REP_QGV_REJECTED		REG_FIELD_PREP(ICL_PCODE_REP_QGV_MASK, 2)
+#define     ADLS_PCODE_REP_PSF_MASK		REG_GENMASK(3, 2)
+#define     ADLS_PCODE_REP_PSF_SAFE		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 0)
+#define     ADLS_PCODE_REP_PSF_POLL		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 1)
+#define     ADLS_PCODE_REP_PSF_REJECTED		REG_FIELD_PREP(ADLS_PCODE_REP_PSF_MASK, 2)
+#define     ICL_PCODE_REQ_QGV_PT_MASK		REG_GENMASK(7, 0)
+#define     ICL_PCODE_REQ_QGV_PT(x)		REG_FIELD_PREP(ICL_PCODE_REQ_QGV_PT_MASK, (x))
+#define     ADLS_PCODE_REQ_PSF_PT_MASK		REG_GENMASK(10, 8)
+#define     ADLS_PCODE_REQ_PSF_PT(x)		REG_FIELD_PREP(ADLS_PCODE_REQ_PSF_PT_MASK, (x))
+#define   GEN6_PCODE_READ_D_COMP		0x10
+#define   GEN6_PCODE_WRITE_D_COMP		0x11
+#define   ICL_PCODE_EXIT_TCCOLD			0x12
+#define   HSW_PCODE_DE_WRITE_FREQ_REQ		0x17
+#define   DISPLAY_IPS_CONTROL			0x19
+#define   TGL_PCODE_TCCOLD			0x26
+#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED	REG_BIT(0)
+#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ	0
+#define     TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ	REG_BIT(0)
+/* See also IPS_CTL */
+#define     IPS_PCODE_CONTROL			(1 << 30)
+#define   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL	0x1A
+#define   GEN9_PCODE_SAGV_CONTROL		0x21
+#define     GEN9_SAGV_DISABLE			0x0
+#define     GEN9_SAGV_IS_DISABLED		0x1
+#define     GEN9_SAGV_ENABLE			0x3
+#define   DG1_PCODE_STATUS			0x7E
+#define     DG1_UNCORE_GET_INIT_STATUS		0x0
+#define     DG1_UNCORE_INIT_STATUS_COMPLETE	0x1
+#define   PCODE_POWER_SETUP			0x7C
+#define     POWER_SETUP_SUBCOMMAND_READ_I1	0x4
+#define     POWER_SETUP_SUBCOMMAND_WRITE_I1	0x5
+#define	    POWER_SETUP_I1_WATTS		REG_BIT(31)
+#define	    POWER_SETUP_I1_SHIFT		6	/* 10.6 fixed point format */
+#define	    POWER_SETUP_I1_DATA_MASK		REG_GENMASK(15, 0)
+#define     POWER_SETUP_SUBCOMMAND_G8_ENABLE	0x6
+#define GEN12_PCODE_READ_SAGV_BLOCK_TIME_US	0x23
+#define   XEHP_PCODE_FREQUENCY_CONFIG		0x6e	/* pvc */
+/* XEHP_PCODE_FREQUENCY_CONFIG sub-commands (param1) */
+#define     PCODE_MBOX_FC_SC_READ_FUSED_P0	0x0
+#define     PCODE_MBOX_FC_SC_READ_FUSED_PN	0x1
+/* PCODE_MBOX_DOMAIN_* - mailbox domain IDs */
+/*   XEHP_PCODE_FREQUENCY_CONFIG param2 */
+#define     PCODE_MBOX_DOMAIN_NONE		0x0
+#define     PCODE_MBOX_DOMAIN_MEDIAFF		0x3
+
+#endif
diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h
index 52520e684ab19..33b91cb2e6845 100644
--- a/include/drm/intel/pciids.h
+++ b/include/drm/intel/pciids.h
@@ -900,4 +900,16 @@
 #define INTEL_CRI_IDS(MACRO__, ...) \
 	MACRO__(0x674C, ## __VA_ARGS__)
 
+/* NVL-P */
+#define INTEL_NVLP_IDS(MACRO__, ...) \
+	MACRO__(0xD750, ## __VA_ARGS__), \
+	MACRO__(0xD751, ## __VA_ARGS__), \
+	MACRO__(0xD752, ## __VA_ARGS__), \
+	MACRO__(0xD753, ## __VA_ARGS__), \
+	MACRO__(0XD754, ## __VA_ARGS__), \
+	MACRO__(0XD755, ## __VA_ARGS__), \
+	MACRO__(0XD756, ## __VA_ARGS__), \
+	MACRO__(0XD757, ## __VA_ARGS__), \
+	MACRO__(0xD75F, ## __VA_ARGS__)
+
 #endif /* __PCIIDS_H__ */
diff --git a/include/drm/intel/pick.h b/include/drm/intel/pick.h
new file mode 100644
index 0000000000000..d976fab8f2705
--- /dev/null
+++ b/include/drm/intel/pick.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _PICK_H_
+#define _PICK_H_
+
+/*
+ * Given the first two numbers __a and __b of arbitrarily many evenly spaced
+ * numbers, pick the 0-based __index'th value.
+ *
+ * Always prefer this over _PICK() if the numbers are evenly spaced.
+ */
+#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a)))
+
+/*
+ * Like _PICK_EVEN(), but supports 2 ranges of evenly spaced address offsets.
+ * @__c_index corresponds to the index in which the second range starts to be
+ * used. Using math interval notation, the first range is used for indexes [ 0,
+ * @__c_index), while the second range is used for [ @__c_index, ... ). Example:
+ *
+ * #define _FOO_A			0xf000
+ * #define _FOO_B			0xf004
+ * #define _FOO_C			0xf008
+ * #define _SUPER_FOO_A			0xa000
+ * #define _SUPER_FOO_B			0xa100
+ * #define FOO(x)			_MMIO(_PICK_EVEN_2RANGES(x, 3,		\
+ *					      _FOO_A, _FOO_B,			\
+ *					      _SUPER_FOO_A, _SUPER_FOO_B))
+ *
+ * This expands to:
+ *	0: 0xf000,
+ *	1: 0xf004,
+ *	2: 0xf008,
+ *	3: 0xa000,
+ *	4: 0xa100,
+ *	5: 0xa200,
+ *	...
+ */
+#define _PICK_EVEN_2RANGES(__index, __c_index, __a, __b, __c, __d)		\
+	(BUILD_BUG_ON_ZERO(!__is_constexpr(__c_index)) +			\
+	 ((__index) < (__c_index) ? _PICK_EVEN(__index, __a, __b) :		\
+				   _PICK_EVEN((__index) - (__c_index), __c, __d)))
+
+/*
+ * Given the arbitrary numbers in varargs, pick the 0-based __index'th number.
+ *
+ * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced.
+ */
+#define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index])
+
+#endif
diff --git a/include/drm/intel/reg_bits.h b/include/drm/intel/reg_bits.h
new file mode 100644
index 0000000000000..2a9066e1d808c
--- /dev/null
+++ b/include/drm/intel/reg_bits.h
@@ -0,0 +1,139 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef _REG_BITS_H_
+#define _REG_BITS_H_
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+
+/*
+ * Wrappers over the generic fixed width BIT_U*() and GENMASK_U*()
+ * implementations, for compatibility reasons with previous implementation.
+ */
+#define REG_GENMASK(high, low)		GENMASK_U32(high, low)
+#define REG_GENMASK64(high, low)	GENMASK_U64(high, low)
+#define REG_GENMASK16(high, low)	GENMASK_U16(high, low)
+#define REG_GENMASK8(high, low)		GENMASK_U8(high, low)
+
+#define REG_BIT(n)			BIT_U32(n)
+#define REG_BIT64(n)			BIT_U64(n)
+#define REG_BIT16(n)			BIT_U16(n)
+#define REG_BIT8(n)			BIT_U8(n)
+
+/*
+ * Local integer constant expression version of is_power_of_2().
+ */
+#define IS_POWER_OF_2(__x)		((__x) && (((__x) & ((__x) - 1)) == 0))
+
+/**
+ * REG_FIELD_PREP8() - Prepare a u8 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to put in the field
+ *
+ * Local copy of FIELD_PREP() to generate an integer constant expression, force
+ * u8 and for consistency with REG_FIELD_GET8(), REG_BIT8() and REG_GENMASK8().
+ *
+ * @return: @__val masked and shifted into the field defined by @__mask.
+ */
+#define REG_FIELD_PREP8(__mask, __val)                                          \
+	((u8)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +      \
+	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +             \
+	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U8_MAX) +          \
+	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
+	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
+
+/**
+ * REG_FIELD_PREP16() - Prepare a u16 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to put in the field
+ *
+ * Local copy of FIELD_PREP16() to generate an integer constant
+ * expression, force u8 and for consistency with
+ * REG_FIELD_GET16(), REG_BIT16() and REG_GENMASK16().
+ *
+ * @return: @__val masked and shifted into the field defined by @__mask.
+ */
+#define REG_FIELD_PREP16(__mask, __val)                                          \
+	((u16)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +      \
+	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +             \
+	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U16_MAX) +          \
+	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
+	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
+
+/**
+ * REG_FIELD_PREP() - Prepare a u32 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to put in the field
+ *
+ * Local copy of FIELD_PREP() to generate an integer constant expression, force
+ * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK().
+ *
+ * @return: @__val masked and shifted into the field defined by @__mask.
+ */
+#define REG_FIELD_PREP(__mask, __val)						\
+	((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +	\
+	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +		\
+	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) +		\
+	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
+	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
+
+/**
+ * REG_FIELD_GET8() - Extract a u8 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to extract the bitfield value from
+ *
+ * Local wrapper for FIELD_GET() to force u8 and for consistency with
+ * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
+ *
+ * @return: Masked and shifted value of the field defined by @__mask in @__val.
+ */
+#define REG_FIELD_GET8(__mask, __val)   ((u8)FIELD_GET(__mask, __val))
+
+/**
+ * REG_FIELD_GET() - Extract a u32 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to extract the bitfield value from
+ *
+ * Local wrapper for FIELD_GET() to force u32 and for consistency with
+ * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
+ *
+ * @return: Masked and shifted value of the field defined by @__mask in @__val.
+ */
+#define REG_FIELD_GET(__mask, __val)	((u32)FIELD_GET(__mask, __val))
+
+/**
+ * REG_FIELD_GET64() - Extract a u64 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to extract the bitfield value from
+ *
+ * Local wrapper for FIELD_GET() to force u64 and for consistency with
+ * REG_GENMASK64().
+ *
+ * @return: Masked and shifted value of the field defined by @__mask in @__val.
+ */
+#define REG_FIELD_GET64(__mask, __val)	((u64)FIELD_GET(__mask, __val))
+
+/**
+ * REG_FIELD_MAX() - produce the maximum value representable by a field
+ * @__mask: shifted mask defining the field's length and position
+ *
+ * Local wrapper for FIELD_MAX() to return the maximum bit value that can
+ * be held in the field specified by @_mask, cast to u32 for consistency
+ * with other macros.
+ */
+#define REG_FIELD_MAX(__mask)	((u32)FIELD_MAX(__mask))
+
+#define REG_MASKED_FIELD(mask, value) \
+	(BUILD_BUG_ON_ZERO(__builtin_choose_expr(__builtin_constant_p(mask), (mask) & 0xffff0000, 0)) + \
+	 BUILD_BUG_ON_ZERO(__builtin_choose_expr(__builtin_constant_p(value), (value) & 0xffff0000, 0)) + \
+	 BUILD_BUG_ON_ZERO(__builtin_choose_expr(__builtin_constant_p(mask) && __builtin_constant_p(value), (value) & ~(mask), 0)) + \
+	 ((mask) << 16 | (value)))
+
+#define REG_MASKED_FIELD_ENABLE(a) \
+	(__builtin_choose_expr(__builtin_constant_p(a), REG_MASKED_FIELD((a), (a)), ({ typeof(a) _a = (a); REG_MASKED_FIELD(_a, _a); })))
+
+#define REG_MASKED_FIELD_DISABLE(a) \
+	(REG_MASKED_FIELD((a), 0))
+
+#endif
diff --git a/include/drm/intel/step.h b/include/drm/intel/step.h
new file mode 100644
index 0000000000000..4de7520109bc5
--- /dev/null
+++ b/include/drm/intel/step.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __STEP_H__
+#define __STEP_H__
+
+#define STEP_ENUM_VAL(name)  STEP_##name,
+
+#define STEP_NAME_LIST(func)		\
+	func(A0)			\
+	func(A1)			\
+	func(A2)			\
+	func(A3)			\
+	func(B0)			\
+	func(B1)			\
+	func(B2)			\
+	func(B3)			\
+	func(C0)			\
+	func(C1)			\
+	func(C2)			\
+	func(C3)			\
+	func(D0)			\
+	func(D1)			\
+	func(D2)			\
+	func(D3)			\
+	func(E0)			\
+	func(E1)			\
+	func(E2)			\
+	func(E3)			\
+	func(F0)			\
+	func(F1)			\
+	func(F2)			\
+	func(F3)			\
+	func(G0)			\
+	func(G1)			\
+	func(G2)			\
+	func(G3)			\
+	func(H0)			\
+	func(H1)			\
+	func(H2)			\
+	func(H3)			\
+	func(I0)			\
+	func(I1)			\
+	func(I2)			\
+	func(I3)			\
+	func(J0)			\
+	func(J1)			\
+	func(J2)			\
+	func(J3)
+
+/*
+ * Symbolic steppings that do not match the hardware. These are valid both as gt
+ * and display steppings as symbolic names.
+ */
+enum intel_step {
+	STEP_NONE = 0,
+	STEP_NAME_LIST(STEP_ENUM_VAL)
+	STEP_FUTURE,
+	STEP_FOREVER,
+};
+
+#endif /* __STEP_H__ */
diff --git a/include/drm/ttm/ttm_backup.h b/include/drm/ttm/ttm_backup.h
index c33cba111171f..29b9c855af779 100644
--- a/include/drm/ttm/ttm_backup.h
+++ b/include/drm/ttm/ttm_backup.h
@@ -56,7 +56,7 @@ ttm_backup_page_ptr_to_handle(const struct page *page)
 void ttm_backup_drop(struct file *backup, pgoff_t handle);
 
 int ttm_backup_copy_page(struct file *backup, struct page *dst,
-			 pgoff_t handle, bool intr);
+			 pgoff_t handle, bool intr, gfp_t additional_gfp);
 
 s64
 ttm_backup_backup_page(struct file *backup, struct page *page,
diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h
index bca3a8849d47d..8310bc3d55f90 100644
--- a/include/drm/ttm/ttm_bo.h
+++ b/include/drm/ttm/ttm_bo.h
@@ -167,24 +167,34 @@ struct ttm_bo_kmap_obj {
 /**
  * struct ttm_operation_ctx
  *
- * @interruptible: Sleep interruptible if sleeping.
- * @no_wait_gpu: Return immediately if the GPU is busy.
- * @gfp_retry_mayfail: Set the __GFP_RETRY_MAYFAIL when allocation pages.
- * @allow_res_evict: Allow eviction of reserved BOs. Can be used when multiple
- * BOs share the same reservation object.
- * faults. Should only be used by TTM internally.
- * @resv: Reservation object to allow reserved evictions with.
- * @bytes_moved: Statistics on how many bytes have been moved.
- *
  * Context for TTM operations like changing buffer placement or general memory
  * allocation.
  */
 struct ttm_operation_ctx {
+	/** @interruptible: Sleep interruptible if sleeping. */
 	bool interruptible;
+	/** @no_wait_gpu: Return immediately if the GPU is busy. */
 	bool no_wait_gpu;
+	/**
+	 * @gfp_retry_mayfail: Use __GFP_RETRY_MAYFAIL | __GFP_NOWARN
+	 * when allocation pages. This is to avoid invoking the OOM
+	 * killer when populating a buffer object, in order to
+	 * forward the error for it to be dealt with.
+	 */
 	bool gfp_retry_mayfail;
+	/**
+	 * @allow_res_evict: Allow eviction of reserved BOs. Can be used
+	 * when multiple BOs share the same reservation object @resv.
+	 */
 	bool allow_res_evict;
+	/**
+	 * @resv: Reservation object to be used together with
+	 * @allow_res_evict.
+	 */
 	struct dma_resv *resv;
+	/**
+	 * @bytes_moved: Statistics on how many bytes have been moved.
+	 */
 	uint64_t bytes_moved;
 };
 
diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h
index 8776844e0eeb3..0316b85747d94 100644
--- a/include/dt-bindings/arm/qcom,ids.h
+++ b/include/dt-bindings/arm/qcom,ids.h
@@ -245,6 +245,7 @@
 #define QCOM_ID_IPQ5000			503
 #define QCOM_ID_IPQ0509			504
 #define QCOM_ID_IPQ0518			505
+#define QCOM_ID_SM7450			506
 #define QCOM_ID_SM6375			507
 #define QCOM_ID_IPQ9514			510
 #define QCOM_ID_IPQ9550			511
@@ -264,6 +265,7 @@
 #define QCOM_ID_QRU1000			539
 #define QCOM_ID_SM8475_2		540
 #define QCOM_ID_QDU1000			545
+#define QCOM_ID_SM7450P			547
 #define QCOM_ID_X1E80100		555
 #define QCOM_ID_SM8650			557
 #define QCOM_ID_SM4450			568
@@ -294,6 +296,13 @@
 #define QCOM_ID_QCS8275			675
 #define QCOM_ID_QCS9075			676
 #define QCOM_ID_QCS615			680
+#define QCOM_ID_CQ7790M			731
+#define QCOM_ID_CQ7790S			732
+#define QCOM_ID_IPQ5200			765
+#define QCOM_ID_IPQ5210			766
+#define QCOM_ID_QCF2200			767
+#define QCOM_ID_QCF3200			768
+#define QCOM_ID_QCF3210			769
 
 /*
  * The board type and revision information, used by Qualcomm bootloaders and
diff --git a/include/dt-bindings/clock/axis,artpec9-clk.h b/include/dt-bindings/clock/axis,artpec9-clk.h
new file mode 100644
index 0000000000000..c6787be8d6864
--- /dev/null
+++ b/include/dt-bindings/clock/axis,artpec9-clk.h
@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd.
+ *             https://www.samsung.com
+ * Copyright (c) 2025  Axis Communications AB.
+ *             https://www.axis.com
+ *
+ * Device Tree binding constants for ARTPEC-9 clock controller.
+ */
+
+#ifndef _DT_BINDINGS_CLOCK_ARTPEC9_H
+#define _DT_BINDINGS_CLOCK_ARTPEC9_H
+
+/* CMU_CMU */
+#define CLK_FOUT_SHARED0_PLL						1
+#define CLK_DOUT_SHARED0_DIV2						2
+#define CLK_DOUT_SHARED0_DIV3						3
+#define CLK_DOUT_SHARED0_DIV4						4
+#define CLK_FOUT_SHARED1_PLL						5
+#define CLK_DOUT_SHARED1_DIV2						6
+#define CLK_DOUT_SHARED1_DIV3						7
+#define CLK_DOUT_SHARED1_DIV4						8
+#define CLK_FOUT_AUDIO_PLL						9
+#define CLK_DOUT_CMU_ADD						10
+#define CLK_DOUT_CMU_BUS						11
+#define CLK_DOUT_CMU_CDC_CORE						12
+#define CLK_DOUT_CMU_CORE_MAIN						13
+#define CLK_DOUT_CMU_CPUCL_SWITCH					14
+#define CLK_DOUT_CMU_DLP_CORE						15
+#define CLK_DOUT_CMU_FSYS0_BUS						16
+#define CLK_DOUT_CMU_FSYS0_IP						17
+#define CLK_DOUT_CMU_FSYS1_BUS						18
+#define CLK_DOUT_CMU_FSYS1_SCAN0					19
+#define CLK_DOUT_CMU_FSYS1_SCAN1					20
+#define CLK_DOUT_CMU_GPU_3D						21
+#define CLK_DOUT_CMU_GPU_2D						22
+#define CLK_DOUT_CMU_IMEM_ACLK						23
+#define CLK_DOUT_CMU_IMEM_CA5						24
+#define CLK_DOUT_CMU_IMEM_JPEG						25
+#define CLK_DOUT_CMU_IMEM_SSS						26
+#define CLK_DOUT_CMU_IPA_CORE						27
+#define CLK_DOUT_CMU_LCPU						28
+#define CLK_DOUT_CMU_MIF_SWITCH						29
+#define CLK_DOUT_CMU_MIF_BUSP						30
+#define CLK_DOUT_CMU_PERI_DISP						31
+#define CLK_DOUT_CMU_PERI_IP						32
+#define CLK_DOUT_CMU_RSP_CORE						33
+#define CLK_DOUT_CMU_TRFM						34
+#define CLK_DOUT_CMU_VIO_CORE_L						35
+#define CLK_DOUT_CMU_VIO_CORE						36
+#define CLK_DOUT_CMU_VIP0						37
+#define CLK_DOUT_CMU_VIP1						38
+#define CLK_DOUT_CMU_VPP_CORE						39
+#define CLK_DOUT_CMU_VIO_AUDIO						40
+
+/* CMU_BUS */
+#define CLK_MOUT_BUS_ACLK_USER						1
+
+/* CMU_CORE */
+#define CLK_MOUT_CORE_ACLK_USER						1
+
+/* CMU_CPUCL */
+#define CLK_FOUT_CPUCL_PLL0						1
+#define CLK_MOUT_CPUCL_PLL0						2
+#define CLK_FOUT_CPUCL_PLL1						3
+#define CLK_MOUT_CPUCL_PLL_SCU						4
+#define CLK_MOUT_CPUCL_SWITCH_SCU_USER					5
+#define CLK_MOUT_CPUCL_SWITCH_USER					6
+#define CLK_DOUT_CPUCL_CPU						7
+#define CLK_DOUT_CPUCL_CLUSTER_PERIPHCLK				8
+#define CLK_DOUT_CPUCL_CLUSTER_GICCLK					9
+#define CLK_DOUT_CPUCL_CLUSTER_PCLK					10
+#define CLK_DOUT_CPUCL_CMUREF						11
+#define CLK_DOUT_CPUCL_CLUSTER_ATCLK					12
+#define CLK_DOUT_CPUCL_CLUSTER_SCU					13
+#define CLK_DOUT_CPUCL_DBG						14
+#define CLK_GOUT_CPUCL_SHORTSTOP					15
+#define CLK_GOUT_CPUCL_CLUSTER_CPU					16
+#define CLK_GOUT_CPUCL_CSSYS_IPCLKPORT_ATCLK				17
+#define CLK_GOUT_CPUCL_CSSYS_IPCLKPORT_PCLKDBG				18
+
+/* CMU_FSYS0 */
+#define CLK_MOUT_FSYS0_BUS_USER						1
+#define CLK_MOUT_FSYS0_IP_USER						2
+#define CLK_MOUT_FSYS0_MAIN_USER					3
+#define CLK_DOUT_FSYS0_125						4
+#define CLK_DOUT_FSYS0_ADC						5
+#define CLK_DOUT_FSYS0_BUS_300						6
+#define CLK_DOUT_FSYS0_EQOS0						7
+#define CLK_DOUT_FSYS0_EQOS1						8
+#define CLK_DOUT_FSYS0_MMC_CARD0					9
+#define CLK_DOUT_FSYS0_MMC_CARD1					10
+#define CLK_DOUT_FSYS0_MMC_CARD2					11
+#define CLK_DOUT_FSYS0_QSPI						12
+#define CLK_DOUT_FSYS0_SFMC_NAND					13
+#define CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_ACLK_I			14
+#define CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_CLK_CSR_I			15
+#define CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_I_RGMII_PHASE_CLK_250	16
+#define CLK_GOUT_FSYS0_EQOS_TOP0_IPCLKPORT_I_RGMII_TXCLK		17
+#define CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_I_RGMII_PHASE_CLK_250	18
+#define CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_I_RGMII_TXCLK		19
+#define CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_ACLK_I			20
+#define CLK_GOUT_FSYS0_EQOS_TOP1_IPCLKPORT_CLK_CSR_I			21
+#define CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_APB_S_PCLK			22
+#define CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_CORE_CLK			23
+#define CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_DMA_CLK				24
+#define CLK_GOUT_FSYS0_I3C0_IPCLKPORT_I_HDR_TX_CLK			25
+#define CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_APB_S_PCLK			26
+#define CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_CORE_CLK			27
+#define CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_DMA_CLK				28
+#define CLK_GOUT_FSYS0_I3C1_IPCLKPORT_I_HDR_TX_CLK			29
+#define CLK_GOUT_FSYS0_MMC0_IPCLKPORT_SDCLKIN				30
+#define CLK_GOUT_FSYS0_MMC1_IPCLKPORT_SDCLKIN				31
+#define CLK_GOUT_FSYS0_MMC2_IPCLKPORT_SDCLKIN				32
+#define CLK_GOUT_FSYS0_QSPI_IPCLKPORT_HCLK				33
+#define CLK_GOUT_FSYS0_QSPI_IPCLKPORT_SSI_CLK				34
+#define CLK_GOUT_FSYS0_SFMC_IPCLKPORT_I_ACLK_NAND			35
+#define CLK_GOUT_FSYS0_I2C0_IPCLKPORT_I_PCLK				36
+#define CLK_GOUT_FSYS0_I2C1_IPCLKPORT_I_PCLK				37
+#define CLK_GOUT_FSYS0_MMC0_IPCLKPORT_I_ACLK				38
+#define CLK_GOUT_FSYS0_MMC1_IPCLKPORT_I_ACLK				39
+#define CLK_GOUT_FSYS0_MMC2_IPCLKPORT_I_ACLK				40
+#define CLK_GOUT_FSYS0_PWM_IPCLKPORT_I_PCLK_S0				41
+
+/* CMU_FSYS1 */
+#define CLK_FOUT_FSYS1_PLL						1
+#define CLK_MOUT_FSYS1_SCAN0_USER					2
+#define CLK_MOUT_FSYS1_SCAN1_USER					3
+#define CLK_MOUT_FSYS1_BUS_USER						4
+#define CLK_DOUT_FSYS1_200						5
+#define CLK_DOUT_FSYS1_BUS_300						6
+#define CLK_DOUT_FSYS1_OTP_MEM						7
+#define CLK_DOUT_FSYS1_PCIE_PHY_REFCLK_SYSPLL				8
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_100		9
+#define CLK_GOUT_FSYS1_UART0_PCLK					10
+#define CLK_GOUT_FSYS1_UART0_SCLK_UART					11
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_PHY_APB2CR_PCLK_300		12
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_DBI_ACLK_SOC		13
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_MSTR_ACLK_SOC		14
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X1_SLV_ACLK_SOC		15
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_DBI_ACLK_SOC		16
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_MSTR_ACLK_SOC		17
+#define CLK_GOUT_FSYS1_IPCLKPORT_PCIE_SUB_CON_X2_SLV_ACLK_SOC		18
+#define CLK_GOUT_FSYS1_USB20DRD_IPCLKPORT_ACLK_PHYCTRL_20		19
+#define CLK_GOUT_FSYS1_USB20DRD_IPCLKPORT_BUS_CLK_EARLY			20
+#define CLK_GOUT_FSYS1_XHB_AHBBR_FSYS1_IPCLKPORT_CLK			21
+#define CLK_GOUT_FSYS1_XHB_USB_IPCLKPORT_CLK				22
+
+/* CMU_IMEM */
+#define CLK_MOUT_IMEM_ACLK_USER						1
+#define CLK_MOUT_IMEM_CA5_USER						2
+#define CLK_MOUT_IMEM_SSS_USER						3
+#define CLK_MOUT_IMEM_JPEG_USER						4
+#define CLK_DOUT_IMEM_PCLK						5
+#define CLK_GOUT_IMEM_CA5_0_IPCLKPORT_ATCLK				6
+#define CLK_GOUT_IMEM_CA5_0_IPCLKPORT_CLKIN				7
+#define CLK_GOUT_IMEM_CA5_0_IPCLKPORT_PCLK_DBG				8
+#define CLK_GOUT_IMEM_CA5_1_IPCLKPORT_ATCLK				9
+#define CLK_GOUT_IMEM_CA5_1_IPCLKPORT_CLKIN				10
+#define CLK_GOUT_IMEM_CA5_1_IPCLKPORT_PCLK_DBG				11
+#define CLK_GOUT_IMEM_MCT0_PCLK						12
+#define CLK_GOUT_IMEM_MCT1_PCLK						13
+#define CLK_GOUT_IMEM_MCT2_PCLK						14
+#define CLK_GOUT_IMEM_MCT3_PCLK						15
+#define CLK_GOUT_IMEM_PCLK_TMU0_APBIF					16
+
+/* CMU_PERI */
+#define CLK_MOUT_PERI_IP_USER						1
+#define CLK_MOUT_PERI_DISP_USER						2
+#define CLK_DOUT_PERI_125						3
+#define CLK_DOUT_PERI_PCLK						4
+#define CLK_DOUT_PERI_SPI						5
+#define CLK_DOUT_PERI_UART1						6
+#define CLK_DOUT_PERI_UART2						7
+#define CLK_GOUT_PERI_DMA4DSIM_IPCLKPORT_CLK_APB_CLK			8
+#define CLK_GOUT_PERI_DMA4DSIM_IPCLKPORT_CLK_AXI_CLK			9
+#define CLK_GOUT_PERI_I3C2_IPCLKPORT_I_APB_S_PCLK			10
+#define CLK_GOUT_PERI_I3C2_IPCLKPORT_I_CORE_CLK				11
+#define CLK_GOUT_PERI_I3C2_IPCLKPORT_I_DMA_CLK				12
+#define CLK_GOUT_PERI_I3C2_IPCLKPORT_I_HDR_TX_CLK			13
+#define CLK_GOUT_PERI_I3C3_IPCLKPORT_I_APB_S_PCLK			14
+#define CLK_GOUT_PERI_I3C3_IPCLKPORT_I_CORE_CLK				15
+#define CLK_GOUT_PERI_I3C3_IPCLKPORT_I_DMA_CLK				16
+#define CLK_GOUT_PERI_I3C3_IPCLKPORT_I_HDR_TX_CLK			17
+#define CLK_GOUT_PERI_APB_ASYNC_DSIM_IPCLKPORT_PCLKS			18
+#define CLK_GOUT_PERI_I2C2_IPCLKPORT_I_PCLK				19
+#define CLK_GOUT_PERI_I2C3_IPCLKPORT_I_PCLK				20
+#define CLK_GOUT_PERI_SPI0_PCLK						21
+#define CLK_GOUT_PERI_SPI0_SCLK_SPI					22
+#define CLK_GOUT_PERI_UART1_PCLK					23
+#define CLK_GOUT_PERI_UART1_SCLK_UART					24
+#define CLK_GOUT_PERI_UART2_PCLK					25
+#define CLK_GOUT_PERI_UART2_SCLK_UART					26
+
+#endif /* _DT_BINDINGS_CLOCK_ARTPEC9_H */
diff --git a/include/dt-bindings/clock/qcom,dispcc-sm6125.h b/include/dt-bindings/clock/qcom,dispcc-sm6125.h
index 4ff974f4fcc35..f58b85d2c8145 100644
--- a/include/dt-bindings/clock/qcom,dispcc-sm6125.h
+++ b/include/dt-bindings/clock/qcom,dispcc-sm6125.h
@@ -6,6 +6,7 @@
 #ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6125_H
 #define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6125_H
 
+/* Clocks */
 #define DISP_CC_PLL0			0
 #define DISP_CC_MDSS_AHB_CLK		1
 #define DISP_CC_MDSS_AHB_CLK_SRC	2
@@ -35,7 +36,10 @@
 #define DISP_CC_MDSS_VSYNC_CLK_SRC	26
 #define DISP_CC_XO_CLK			27
 
-/* DISP_CC GDSCR */
+/* Resets */
+#define DISP_CC_MDSS_CORE_BCR			0
+
+/* GDSCs */
 #define MDSS_GDSC			0
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,eliza-gcc.h b/include/dt-bindings/clock/qcom,eliza-gcc.h
new file mode 100644
index 0000000000000..4d27b329ae99f
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-gcc.h
@@ -0,0 +1,210 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_GCC_ELIZA_H
+
+/* GCC clocks */
+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
+#define GCC_AGGRE_UFS_PHY_AXI_CLK				1
+#define GCC_AGGRE_USB3_PRIM_AXI_CLK				2
+#define GCC_BOOT_ROM_AHB_CLK					3
+#define GCC_CAM_BIST_MCLK_AHB_CLK				4
+#define GCC_CAMERA_AHB_CLK					5
+#define GCC_CAMERA_HF_AXI_CLK					6
+#define GCC_CAMERA_SF_AXI_CLK					7
+#define GCC_CAMERA_XO_CLK					8
+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				9
+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				10
+#define GCC_CNOC_PCIE_SF_AXI_CLK				11
+#define GCC_DDRSS_GPU_AXI_CLK					12
+#define GCC_DDRSS_PCIE_SF_QTB_CLK				13
+#define GCC_DISP_AHB_CLK					14
+#define GCC_DISP_HF_AXI_CLK					15
+#define GCC_GP1_CLK						16
+#define GCC_GP1_CLK_SRC						17
+#define GCC_GP2_CLK						18
+#define GCC_GP2_CLK_SRC						19
+#define GCC_GP3_CLK						20
+#define GCC_GP3_CLK_SRC						21
+#define GCC_GPLL0						22
+#define GCC_GPLL0_OUT_EVEN					23
+#define GCC_GPLL4						24
+#define GCC_GPLL7						25
+#define GCC_GPLL8						26
+#define GCC_GPLL9						27
+#define GCC_GPU_CFG_AHB_CLK					28
+#define GCC_GPU_GEMNOC_GFX_CLK					29
+#define GCC_GPU_GPLL0_CPH_CLK_SRC				30
+#define GCC_GPU_GPLL0_DIV_CPH_CLK_SRC				31
+#define GCC_GPU_SMMU_VOTE_CLK					32
+#define GCC_MMU_TCU_VOTE_CLK					33
+#define GCC_PCIE_0_AUX_CLK					34
+#define GCC_PCIE_0_AUX_CLK_SRC					35
+#define GCC_PCIE_0_CFG_AHB_CLK					36
+#define GCC_PCIE_0_MSTR_AXI_CLK					37
+#define GCC_PCIE_0_PHY_RCHNG_CLK				38
+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				39
+#define GCC_PCIE_0_PIPE_CLK					40
+#define GCC_PCIE_0_PIPE_CLK_SRC					41
+#define GCC_PCIE_0_PIPE_DIV2_CLK				42
+#define GCC_PCIE_0_PIPE_DIV2_CLK_SRC				43
+#define GCC_PCIE_0_SLV_AXI_CLK					44
+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				45
+#define GCC_PCIE_1_AUX_CLK					46
+#define GCC_PCIE_1_AUX_CLK_SRC					47
+#define GCC_PCIE_1_CFG_AHB_CLK					48
+#define GCC_PCIE_1_MSTR_AXI_CLK					49
+#define GCC_PCIE_1_PHY_RCHNG_CLK				50
+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				51
+#define GCC_PCIE_1_PIPE_CLK					52
+#define GCC_PCIE_1_PIPE_CLK_SRC					53
+#define GCC_PCIE_1_PIPE_DIV2_CLK				54
+#define GCC_PCIE_1_PIPE_DIV2_CLK_SRC				55
+#define GCC_PCIE_1_SLV_AXI_CLK					56
+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				57
+#define GCC_PCIE_RSCC_CFG_AHB_CLK				58
+#define GCC_PCIE_RSCC_XO_CLK					59
+#define GCC_PDM2_CLK						60
+#define GCC_PDM2_CLK_SRC					61
+#define GCC_PDM_AHB_CLK						62
+#define GCC_PDM_XO4_CLK						63
+#define GCC_QMIP_CAMERA_CMD_AHB_CLK				64
+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				65
+#define GCC_QMIP_CAMERA_RT_AHB_CLK				66
+#define GCC_QMIP_GPU_AHB_CLK					67
+#define GCC_QMIP_PCIE_AHB_CLK					68
+#define GCC_QMIP_VIDEO_V_CPU_AHB_CLK				69
+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				70
+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				71
+#define GCC_QUPV3_WRAP1_CORE_CLK				72
+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				73
+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			74
+#define GCC_QUPV3_WRAP1_S0_CLK					75
+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				76
+#define GCC_QUPV3_WRAP1_S1_CLK					77
+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				78
+#define GCC_QUPV3_WRAP1_S2_CLK					79
+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				80
+#define GCC_QUPV3_WRAP1_S3_CLK					81
+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				82
+#define GCC_QUPV3_WRAP1_S4_CLK					83
+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				84
+#define GCC_QUPV3_WRAP1_S5_CLK					85
+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				86
+#define GCC_QUPV3_WRAP1_S6_CLK					87
+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				88
+#define GCC_QUPV3_WRAP1_S7_CLK					89
+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				90
+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				91
+#define GCC_QUPV3_WRAP2_CORE_CLK				92
+#define GCC_QUPV3_WRAP2_S0_CLK					93
+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				94
+#define GCC_QUPV3_WRAP2_S1_CLK					95
+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				96
+#define GCC_QUPV3_WRAP2_S2_CLK					97
+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				98
+#define GCC_QUPV3_WRAP2_S3_CLK					99
+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				100
+#define GCC_QUPV3_WRAP2_S4_CLK					101
+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				102
+#define GCC_QUPV3_WRAP2_S5_CLK					103
+#define GCC_QUPV3_WRAP2_S5_CLK_SRC				104
+#define GCC_QUPV3_WRAP2_S6_CLK					105
+#define GCC_QUPV3_WRAP2_S6_CLK_SRC				106
+#define GCC_QUPV3_WRAP2_S7_CLK					107
+#define GCC_QUPV3_WRAP2_S7_CLK_SRC				108
+#define GCC_QUPV3_WRAP_1_M_AHB_CLK				109
+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				110
+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				111
+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				112
+#define GCC_SDCC1_AHB_CLK					113
+#define GCC_SDCC1_APPS_CLK					114
+#define GCC_SDCC1_APPS_CLK_SRC					115
+#define GCC_SDCC1_ICE_CORE_CLK					116
+#define GCC_SDCC1_ICE_CORE_CLK_SRC				117
+#define GCC_SDCC2_AHB_CLK					118
+#define GCC_SDCC2_APPS_CLK					119
+#define GCC_SDCC2_APPS_CLK_SRC					120
+#define GCC_UFS_PHY_AHB_CLK					121
+#define GCC_UFS_PHY_AXI_CLK					122
+#define GCC_UFS_PHY_AXI_CLK_SRC					123
+#define GCC_UFS_PHY_ICE_CORE_CLK				124
+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				125
+#define GCC_UFS_PHY_PHY_AUX_CLK					126
+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				127
+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				128
+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				129
+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				130
+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				131
+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				132
+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				133
+#define GCC_UFS_PHY_UNIPRO_CORE_CLK				134
+#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC				135
+#define GCC_USB30_PRIM_ATB_CLK					136
+#define GCC_USB30_PRIM_MASTER_CLK				137
+#define GCC_USB30_PRIM_MASTER_CLK_SRC				138
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				139
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			140
+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		141
+#define GCC_USB30_PRIM_SLEEP_CLK				142
+#define GCC_USB3_PRIM_PHY_AUX_CLK				143
+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				144
+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				145
+#define GCC_USB3_PRIM_PHY_PIPE_CLK				146
+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				147
+#define GCC_VIDEO_AHB_CLK					148
+#define GCC_VIDEO_AXI0_CLK					149
+#define GCC_VIDEO_AXI1_CLK					150
+#define GCC_VIDEO_XO_CLK					151
+
+/* GCC power domains */
+#define GCC_PCIE_0_GDSC						0
+#define GCC_PCIE_0_PHY_GDSC					1
+#define GCC_PCIE_1_GDSC						2
+#define GCC_PCIE_1_PHY_GDSC					3
+#define GCC_UFS_MEM_PHY_GDSC					4
+#define GCC_UFS_PHY_GDSC					5
+#define GCC_USB30_PRIM_GDSC					6
+#define GCC_USB3_PHY_GDSC					7
+
+/* GCC resets */
+#define GCC_CAMERA_BCR						0
+#define GCC_DISPLAY_BCR						1
+#define GCC_GPU_BCR						2
+#define GCC_PCIE_0_BCR						3
+#define GCC_PCIE_0_LINK_DOWN_BCR				4
+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				5
+#define GCC_PCIE_0_PHY_BCR					6
+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			7
+#define GCC_PCIE_1_BCR						8
+#define GCC_PCIE_1_LINK_DOWN_BCR				9
+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				10
+#define GCC_PCIE_1_PHY_BCR					11
+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			12
+#define GCC_PCIE_PHY_BCR					13
+#define GCC_PCIE_PHY_CFG_AHB_BCR				14
+#define GCC_PCIE_PHY_COM_BCR					15
+#define GCC_PCIE_RSCC_BCR					16
+#define GCC_PDM_BCR						17
+#define GCC_QUPV3_WRAPPER_1_BCR					18
+#define GCC_QUPV3_WRAPPER_2_BCR					19
+#define GCC_QUSB2PHY_PRIM_BCR					20
+#define GCC_QUSB2PHY_SEC_BCR					21
+#define GCC_SDCC1_BCR						22
+#define GCC_SDCC2_BCR						23
+#define GCC_UFS_PHY_BCR						24
+#define GCC_USB30_PRIM_BCR					25
+#define GCC_USB3_DP_PHY_PRIM_BCR				26
+#define GCC_USB3_DP_PHY_SEC_BCR					27
+#define GCC_USB3_PHY_PRIM_BCR					28
+#define GCC_USB3_PHY_SEC_BCR					29
+#define GCC_USB3PHY_PHY_PRIM_BCR				30
+#define GCC_USB3PHY_PHY_SEC_BCR					31
+#define GCC_VIDEO_AXI0_CLK_ARES					32
+#define GCC_VIDEO_AXI1_CLK_ARES					33
+#define GCC_VIDEO_BCR						34
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,eliza-tcsr.h b/include/dt-bindings/clock/qcom,eliza-tcsr.h
new file mode 100644
index 0000000000000..aeb5e2b1a47b4
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-tcsr.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_ELIZA_H
+
+/* TCSR_CC clocks */
+#define TCSR_HDMI_CLKREF_EN				0
+#define TCSR_PCIE_0_CLKREF_EN				1
+#define TCSR_PCIE_1_CLKREF_EN				2
+#define TCSR_UFS_CLKREF_EN				3
+#define TCSR_USB2_CLKREF_EN				4
+#define TCSR_USB3_CLKREF_EN				5
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,gcc-sc8180x.h b/include/dt-bindings/clock/qcom,gcc-sc8180x.h
index b9d8438a15ffb..9ed7b794aacc4 100644
--- a/include/dt-bindings/clock/qcom,gcc-sc8180x.h
+++ b/include/dt-bindings/clock/qcom,gcc-sc8180x.h
@@ -322,5 +322,10 @@
 #define USB30_MP_GDSC						8
 #define USB30_PRIM_GDSC						9
 #define USB30_SEC_GDSC						10
+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC		11
+#define HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC		12
+#define HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC		13
+#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC			14
+#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC			15
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,glymur-gcc.h b/include/dt-bindings/clock/qcom,glymur-gcc.h
index 10c12b8c51c34..6907653c79927 100644
--- a/include/dt-bindings/clock/qcom,glymur-gcc.h
+++ b/include/dt-bindings/clock/qcom,glymur-gcc.h
@@ -574,5 +574,6 @@
 #define GCC_VIDEO_AXI0_CLK_ARES					89
 #define GCC_VIDEO_AXI1_CLK_ARES					90
 #define GCC_VIDEO_BCR						91
+#define GCC_VIDEO_AXI0C_CLK_ARES				92
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,glymur-gpucc.h b/include/dt-bindings/clock/qcom,glymur-gpucc.h
new file mode 100644
index 0000000000000..35f5abb848fd0
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,glymur-gpucc.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) 2025, Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_GLYMUR_H
+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_GLYMUR_H
+
+/* GPU_CC clocks */
+#define GPU_CC_AHB_CLK						0
+#define GPU_CC_CB_CLK						1
+#define GPU_CC_CX_ACCU_SHIFT_CLK				2
+#define GPU_CC_CX_FF_CLK					3
+#define GPU_CC_CX_GMU_CLK					4
+#define GPU_CC_CXO_AON_CLK					5
+#define GPU_CC_CXO_CLK						6
+#define GPU_CC_DEMET_CLK					7
+#define GPU_CC_DPM_CLK						8
+#define GPU_CC_FF_CLK_SRC					9
+#define GPU_CC_FREQ_MEASURE_CLK					10
+#define GPU_CC_GMU_CLK_SRC					11
+#define GPU_CC_GPU_SMMU_VOTE_CLK				12
+#define GPU_CC_GX_ACCU_SHIFT_CLK				13
+#define GPU_CC_GX_ACD_AHB_FF_CLK				14
+#define GPU_CC_GX_AHB_FF_CLK					15
+#define GPU_CC_GX_GMU_CLK					16
+#define GPU_CC_GX_RCG_AHB_FF_CLK				17
+#define GPU_CC_HUB_AON_CLK					18
+#define GPU_CC_HUB_CLK_SRC					19
+#define GPU_CC_HUB_CX_INT_CLK					20
+#define GPU_CC_HUB_DIV_CLK_SRC					21
+#define GPU_CC_MEMNOC_GFX_CLK					22
+#define GPU_CC_PLL0						23
+#define GPU_CC_PLL0_OUT_EVEN					24
+#define GPU_CC_RSCC_HUB_AON_CLK					25
+#define GPU_CC_RSCC_XO_AON_CLK					26
+#define GPU_CC_SLEEP_CLK					27
+
+/* GPU_CC power domains */
+#define GPU_CC_CX_GDSC						0
+
+/* GPU_CC resets */
+#define GPU_CC_CB_BCR						0
+#define GPU_CC_CX_BCR						1
+#define GPU_CC_FAST_HUB_BCR					2
+#define GPU_CC_FF_BCR						3
+#define GPU_CC_GMU_BCR						4
+#define GPU_CC_GX_BCR						5
+#define GPU_CC_XO_BCR						6
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,glymur-videocc.h b/include/dt-bindings/clock/qcom,glymur-videocc.h
new file mode 100644
index 0000000000000..98c0debef8fa9
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,glymur-videocc.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_GLYMUR_H
+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_GLYMUR_H
+
+/* VIDEO_CC clocks */
+#define VIDEO_CC_AHB_CLK					0
+#define VIDEO_CC_AHB_CLK_SRC					1
+#define VIDEO_CC_MVS0_CLK					2
+#define VIDEO_CC_MVS0_CLK_SRC					3
+#define VIDEO_CC_MVS0_DIV_CLK_SRC				4
+#define VIDEO_CC_MVS0_FREERUN_CLK				5
+#define VIDEO_CC_MVS0_SHIFT_CLK					6
+#define VIDEO_CC_MVS0C_CLK					7
+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				8
+#define VIDEO_CC_MVS0C_FREERUN_CLK				9
+#define VIDEO_CC_MVS0C_SHIFT_CLK				10
+#define VIDEO_CC_MVS1_CLK					11
+#define VIDEO_CC_MVS1_DIV_CLK_SRC				12
+#define VIDEO_CC_MVS1_FREERUN_CLK				13
+#define VIDEO_CC_MVS1_SHIFT_CLK					14
+#define VIDEO_CC_PLL0						15
+#define VIDEO_CC_SLEEP_CLK					16
+#define VIDEO_CC_SLEEP_CLK_SRC					17
+#define VIDEO_CC_XO_CLK						18
+#define VIDEO_CC_XO_CLK_SRC					19
+
+/* VIDEO_CC power domains */
+#define VIDEO_CC_MVS0_GDSC					0
+#define VIDEO_CC_MVS0C_GDSC					1
+#define VIDEO_CC_MVS1_GDSC					2
+
+/* VIDEO_CC resets */
+#define VIDEO_CC_INTERFACE_BCR					0
+#define VIDEO_CC_MVS0_BCR					1
+#define VIDEO_CC_MVS0C_BCR					2
+#define VIDEO_CC_MVS0C_FREERUN_CLK_ARES				3
+#define VIDEO_CC_MVS0_FREERUN_CLK_ARES				4
+#define VIDEO_CC_MVS1_FREERUN_CLK_ARES				5
+#define VIDEO_CC_XO_CLK_ARES					6
+#define VIDEO_CC_MVS1_BCR					7
+#endif
diff --git a/include/dt-bindings/clock/qcom,ipq5210-gcc.h b/include/dt-bindings/clock/qcom,ipq5210-gcc.h
new file mode 100644
index 0000000000000..84116f34ee4d0
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,ipq5210-gcc.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ5210_H
+#define _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ5210_H
+
+#define GCC_ADSS_PWM_CLK					0
+#define GCC_ADSS_PWM_CLK_SRC					1
+#define GCC_CMN_12GPLL_AHB_CLK					2
+#define GCC_CMN_12GPLL_SYS_CLK					3
+#define GCC_CNOC_LPASS_CFG_CLK					4
+#define GCC_CNOC_PCIE0_1LANE_S_CLK				5
+#define GCC_CNOC_PCIE1_2LANE_S_CLK				6
+#define GCC_CNOC_USB_CLK					7
+#define GCC_GEPHY_SYS_CLK					8
+#define GCC_LPASS_AXIM_CLK_SRC					9
+#define GCC_LPASS_CORE_AXIM_CLK					10
+#define GCC_LPASS_SWAY_CLK					11
+#define GCC_LPASS_SWAY_CLK_SRC					12
+#define GCC_MDIO_AHB_CLK					13
+#define GCC_MDIO_GEPHY_AHB_CLK					14
+#define GCC_NSS_TS_CLK						15
+#define GCC_NSS_TS_CLK_SRC					16
+#define GCC_NSSCC_CLK						17
+#define GCC_NSSCFG_CLK						18
+#define GCC_NSSNOC_ATB_CLK					19
+#define GCC_NSSNOC_MEMNOC_1_CLK					20
+#define GCC_NSSNOC_MEMNOC_BFDCD_CLK_SRC				21
+#define GCC_NSSNOC_MEMNOC_CLK					22
+#define GCC_NSSNOC_MEMNOC_DIV_CLK_SRC				23
+#define GCC_NSSNOC_NSSCC_CLK					24
+#define GCC_NSSNOC_PCNOC_1_CLK					25
+#define GCC_NSSNOC_QOSGEN_REF_CLK				26
+#define GCC_NSSNOC_SNOC_1_CLK					27
+#define GCC_NSSNOC_SNOC_CLK					28
+#define GCC_NSSNOC_TIMEOUT_REF_CLK				29
+#define GCC_NSSNOC_XO_DCD_CLK					30
+#define GCC_PCIE0_AHB_CLK					31
+#define GCC_PCIE0_AUX_CLK					32
+#define GCC_PCIE0_AXI_M_CLK					33
+#define GCC_PCIE0_AXI_M_CLK_SRC					34
+#define GCC_PCIE0_AXI_S_BRIDGE_CLK				35
+#define GCC_PCIE0_AXI_S_CLK					36
+#define GCC_PCIE0_AXI_S_CLK_SRC					37
+#define GCC_PCIE0_PIPE_CLK					38
+#define GCC_PCIE0_PIPE_CLK_SRC					39
+#define GCC_PCIE0_RCHNG_CLK					40
+#define GCC_PCIE0_RCHNG_CLK_SRC					41
+#define GCC_PCIE1_AHB_CLK					42
+#define GCC_PCIE1_AUX_CLK					43
+#define GCC_PCIE1_AXI_M_CLK					44
+#define GCC_PCIE1_AXI_M_CLK_SRC					45
+#define GCC_PCIE1_AXI_S_BRIDGE_CLK				46
+#define GCC_PCIE1_AXI_S_CLK					47
+#define GCC_PCIE1_AXI_S_CLK_SRC					48
+#define GCC_PCIE1_PIPE_CLK					49
+#define GCC_PCIE1_PIPE_CLK_SRC					50
+#define GCC_PCIE1_RCHNG_CLK					51
+#define GCC_PCIE1_RCHNG_CLK_SRC					52
+#define GCC_PCIE_AUX_CLK_SRC					53
+#define GCC_PCNOC_BFDCD_CLK_SRC					54
+#define GCC_PON_APB_CLK						55
+#define GCC_PON_TM_CLK						56
+#define GCC_PON_TM2X_CLK					57
+#define GCC_PON_TM2X_CLK_SRC					58
+#define GCC_QDSS_AT_CLK						59
+#define GCC_QDSS_AT_CLK_SRC					60
+#define GCC_QDSS_DAP_CLK					61
+#define GCC_QDSS_TSCTR_CLK_SRC					62
+#define GCC_QPIC_AHB_CLK					63
+#define GCC_QPIC_CLK						64
+#define GCC_QPIC_CLK_SRC					65
+#define GCC_QPIC_IO_MACRO_CLK					66
+#define GCC_QPIC_IO_MACRO_CLK_SRC				67
+#define GCC_QRNG_AHB_CLK					68
+#define GCC_QUPV3_AHB_MST_CLK					69
+#define GCC_QUPV3_AHB_SLV_CLK					70
+#define GCC_QUPV3_WRAP_SE0_CLK					71
+#define GCC_QUPV3_WRAP_SE0_CLK_SRC				72
+#define GCC_QUPV3_WRAP_SE1_CLK					73
+#define GCC_QUPV3_WRAP_SE1_CLK_SRC				74
+#define GCC_QUPV3_WRAP_SE2_CLK					75
+#define GCC_QUPV3_WRAP_SE2_CLK_SRC				76
+#define GCC_QUPV3_WRAP_SE3_CLK					77
+#define GCC_QUPV3_WRAP_SE3_CLK_SRC				78
+#define GCC_QUPV3_WRAP_SE4_CLK					79
+#define GCC_QUPV3_WRAP_SE4_CLK_SRC				80
+#define GCC_QUPV3_WRAP_SE5_CLK					81
+#define GCC_QUPV3_WRAP_SE5_CLK_SRC				82
+#define GCC_SDCC1_AHB_CLK					83
+#define GCC_SDCC1_APPS_CLK					84
+#define GCC_SDCC1_APPS_CLK_SRC					85
+#define GCC_SDCC1_ICE_CORE_CLK					86
+#define GCC_SDCC1_ICE_CORE_CLK_SRC				87
+#define GCC_SLEEP_CLK_SRC					88
+#define GCC_SNOC_LPASS_CLK					89
+#define GCC_SNOC_PCIE0_AXI_M_CLK				90
+#define GCC_SNOC_PCIE1_AXI_M_CLK				91
+#define GCC_SYSTEM_NOC_BFDCD_CLK_SRC				92
+#define GCC_UNIPHY0_AHB_CLK					93
+#define GCC_UNIPHY0_SYS_CLK					94
+#define GCC_UNIPHY1_AHB_CLK					95
+#define GCC_UNIPHY1_SYS_CLK					96
+#define GCC_UNIPHY2_AHB_CLK					97
+#define GCC_UNIPHY2_SYS_CLK					98
+#define GCC_UNIPHY_SYS_CLK_SRC					99
+#define GCC_USB0_AUX_CLK					100
+#define GCC_USB0_AUX_CLK_SRC					101
+#define GCC_USB0_MASTER_CLK					102
+#define GCC_USB0_MASTER_CLK_SRC					103
+#define GCC_USB0_MOCK_UTMI_CLK					104
+#define GCC_USB0_MOCK_UTMI_CLK_SRC				105
+#define GCC_USB0_MOCK_UTMI_DIV_CLK_SRC				106
+#define GCC_USB0_PHY_CFG_AHB_CLK				107
+#define GCC_USB0_PIPE_CLK					108
+#define GCC_USB0_PIPE_CLK_SRC					109
+#define GCC_USB0_SLEEP_CLK					110
+#define GCC_XO_CLK_SRC						111
+#define GPLL0_MAIN						112
+#define GPLL0							113
+#define GPLL2_MAIN						114
+#define GPLL2							115
+#define GPLL4_MAIN						116
+#endif
diff --git a/include/dt-bindings/clock/qcom,sm6115-dispcc.h b/include/dt-bindings/clock/qcom,sm6115-dispcc.h
index d1a6c45b50290..ab8d312ade370 100644
--- a/include/dt-bindings/clock/qcom,sm6115-dispcc.h
+++ b/include/dt-bindings/clock/qcom,sm6115-dispcc.h
@@ -6,7 +6,7 @@
 #ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H
 #define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H
 
-/* DISP_CC clocks */
+/* Clocks */
 #define DISP_CC_PLL0			0
 #define DISP_CC_PLL0_OUT_MAIN		1
 #define DISP_CC_MDSS_AHB_CLK		2
@@ -30,7 +30,10 @@
 #define DISP_CC_SLEEP_CLK		20
 #define DISP_CC_SLEEP_CLK_SRC		21
 
-/* DISP_CC GDSCR */
+/* Resets */
+#define DISP_CC_MDSS_CORE_BCR			0
+
+/* GDSCs */
 #define MDSS_GDSC			0
 
 #endif
diff --git a/include/dt-bindings/clock/rockchip,rv1103b-cru.h b/include/dt-bindings/clock/rockchip,rv1103b-cru.h
new file mode 100644
index 0000000000000..35afdee7e961c
--- /dev/null
+++ b/include/dt-bindings/clock/rockchip,rv1103b-cru.h
@@ -0,0 +1,220 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
+/*
+ * Copyright (c) 2024 Rockchip Electronics Co. Ltd.
+ * Author: Elaine Zhang <zhangqing@rock-chips.com>
+ */
+
+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RV1103B_H
+#define _DT_BINDINGS_CLK_ROCKCHIP_RV1103B_H
+
+#define PLL_GPLL		0
+#define ARMCLK			1
+#define PLL_DPLL		2
+#define XIN_OSC0_HALF		3
+#define CLK_GPLL_DIV24		4
+#define CLK_GPLL_DIV12		5
+#define CLK_GPLL_DIV6		6
+#define CLK_GPLL_DIV4		7
+#define CLK_GPLL_DIV3		8
+#define CLK_GPLL_DIV2P5		9
+#define CLK_GPLL_DIV2		10
+#define CLK_UART0_SRC		11
+#define CLK_UART1_SRC		12
+#define CLK_UART2_SRC		13
+#define CLK_UART0_FRAC		14
+#define CLK_UART1_FRAC		15
+#define CLK_UART2_FRAC		16
+#define CLK_SAI_SRC		17
+#define CLK_SAI_FRAC		18
+#define LSCLK_NPU_SRC		19
+#define CLK_NPU_SRC		20
+#define ACLK_VEPU_SRC		21
+#define CLK_VEPU_SRC		22
+#define ACLK_VI_SRC		23
+#define CLK_ISP_SRC		24
+#define DCLK_VICAP		25
+#define CCLK_EMMC		26
+#define CCLK_SDMMC0		27
+#define SCLK_SFC_2X		28
+#define LSCLK_PERI_SRC		29
+#define ACLK_PERI_SRC		30
+#define HCLK_HPMCU		31
+#define SCLK_UART0		32
+#define SCLK_UART1		33
+#define SCLK_UART2		34
+#define CLK_I2C_PMU		35
+#define CLK_I2C_PERI		36
+#define CLK_SPI0		37
+#define CLK_PWM0_SRC		38
+#define CLK_PWM1		39
+#define CLK_PWM2		40
+#define DCLK_DECOM_SRC		41
+#define CCLK_SDMMC1		42
+#define CLK_CORE_CRYPTO		43
+#define CLK_PKA_CRYPTO		44
+#define CLK_CORE_RGA		45
+#define MCLK_SAI_SRC		46
+#define CLK_FREQ_PWM0_SRC	47
+#define CLK_COUNTER_PWM0_SRC	48
+#define PCLK_TOP_ROOT		49
+#define CLK_REF_MIPI0		50
+#define CLK_MIPI0_OUT2IO	51
+#define CLK_REF_MIPI1		52
+#define CLK_MIPI1_OUT2IO	53
+#define MCLK_SAI_OUT2IO		54
+#define ACLK_NPU_ROOT		55
+#define HCLK_RKNN		56
+#define ACLK_RKNN		57
+#define LSCLK_VEPU_ROOT		58
+#define HCLK_VEPU		59
+#define ACLK_VEPU		60
+#define CLK_CORE_VEPU		61
+#define PCLK_IOC_VCCIO3		62
+#define PCLK_ACODEC		63
+#define PCLK_USBPHY		64
+#define LSCLK_VI_100M		65
+#define LSCLK_VI_ROOT		66
+#define HCLK_ISP		67
+#define ACLK_ISP		68
+#define CLK_CORE_ISP		69
+#define ACLK_VICAP		70
+#define HCLK_VICAP		71
+#define ISP0CLK_VICAP		72
+#define PCLK_CSI2HOST0		73
+#define PCLK_CSI2HOST1		74
+#define HCLK_EMMC		75
+#define HCLK_SFC		76
+#define HCLK_SFC_XIP		77
+#define HCLK_SDMMC0		78
+#define PCLK_CSIPHY		79
+#define PCLK_GPIO1		80
+#define DBCLK_GPIO1		81
+#define PCLK_IOC_VCCIO47	82
+#define LSCLK_DDR_ROOT		83
+#define CLK_TIMER_DDRMON	84
+#define LSCLK_PMU_ROOT		85
+#define PCLK_PMU		86
+#define XIN_RC_DIV		87
+#define CLK_32K			88
+#define PCLK_PMU_GPIO0		89
+#define DBCLK_PMU_GPIO0		90
+#define CLK_DDR_FAIL_SAFE	91
+#define PCLK_PMU_HP_TIMER	92
+#define CLK_PMU_32K_HP_TIMER	93
+#define PCLK_PWM0		94
+#define CLK_PWM0		95
+#define CLK_OSC_PWM0		96
+#define CLK_RC_PWM0		97
+#define CLK_FREQ_PWM0		98
+#define CLK_COUNTER_PWM0	99
+#define PCLK_I2C0		100
+#define CLK_I2C0		101
+#define PCLK_UART0		102
+#define PCLK_IOC_PMUIO0		103
+#define CLK_REFOUT		104
+#define CLK_PREROLL		105
+#define CLK_PREROLL_32K		106
+#define CLK_LPMCU_PMU		107
+#define PCLK_SPI2AHB		108
+#define HCLK_SPI2AHB		109
+#define SCLK_SPI2AHB		110
+#define PCLK_WDT_LPMCU		111
+#define TCLK_WDT_LPMCU		112
+#define HCLK_SFC_PMU1		113
+#define HCLK_SFC_XIP_PMU1	114
+#define SCLK_SFC_2X_PMU1	115
+#define CLK_LPMCU		116
+#define CLK_LPMCU_RTC		117
+#define PCLK_LPMCU_MAILBOX	118
+#define PCLK_IOC_PMUIO1		119
+#define PCLK_CRU_PMU1		120
+#define PCLK_PERI_ROOT		121
+#define PCLK_RTC_ROOT		122
+#define CLK_TIMER_ROOT		123
+#define PCLK_TIMER		124
+#define CLK_TIMER0		125
+#define CLK_TIMER1		126
+#define CLK_TIMER2		127
+#define CLK_TIMER3		128
+#define CLK_TIMER4		129
+#define CLK_TIMER5		130
+#define PCLK_STIMER		131
+#define CLK_STIMER0		132
+#define CLK_STIMER1		133
+#define PCLK_WDT_NS		134
+#define TCLK_WDT_NS		135
+#define PCLK_WDT_S		136
+#define TCLK_WDT_S		137
+#define PCLK_WDT_HPMCU		138
+#define TCLK_WDT_HPMCU		139
+#define PCLK_I2C1		140
+#define CLK_I2C1		141
+#define PCLK_I2C2		142
+#define CLK_I2C2		143
+#define PCLK_I2C3		144
+#define CLK_I2C3		145
+#define PCLK_I2C4		146
+#define CLK_I2C4		147
+#define PCLK_SPI0		148
+#define PCLK_PWM1		149
+#define CLK_OSC_PWM1		150
+#define PCLK_PWM2		151
+#define CLK_OSC_PWM2		152
+#define PCLK_UART2		153
+#define PCLK_UART1		154
+#define ACLK_RKDMA		155
+#define PCLK_TSADC		156
+#define CLK_TSADC		157
+#define CLK_TSADC_TSEN		158
+#define PCLK_SARADC		159
+#define CLK_SARADC		160
+#define PCLK_GPIO2		161
+#define DBCLK_GPIO2		162
+#define PCLK_IOC_VCCIO6		163
+#define ACLK_USBOTG		164
+#define CLK_REF_USBOTG		165
+#define HCLK_SDMMC1		166
+#define HCLK_SAI		167
+#define MCLK_SAI		168
+#define ACLK_CRYPTO		169
+#define HCLK_CRYPTO		170
+#define HCLK_RK_RNG_NS		171
+#define HCLK_RK_RNG_S		172
+#define PCLK_OTPC_NS		173
+#define CLK_OTPC_ROOT_NS	174
+#define CLK_SBPI_OTPC_NS	175
+#define CLK_USER_OTPC_NS	176
+#define PCLK_OTPC_S		177
+#define CLK_OTPC_ROOT_S		178
+#define CLK_SBPI_OTPC_S		179
+#define CLK_USER_OTPC_S		180
+#define CLK_OTPC_ARB		181
+#define PCLK_OTP_MASK		182
+#define HCLK_RGA		183
+#define ACLK_RGA		184
+#define ACLK_MAC		185
+#define PCLK_MAC		186
+#define CLK_MACPHY		187
+#define ACLK_SPINLOCK		188
+#define HCLK_CACHE		189
+#define PCLK_HPMCU_MAILBOX	190
+#define PCLK_HPMCU_INTMUX	191
+#define CLK_HPMCU		192
+#define CLK_HPMCU_RTC		193
+#define DCLK_DECOM		194
+#define ACLK_DECOM		195
+#define PCLK_DECOM		196
+#define ACLK_SYS_SRAM		197
+#define PCLK_DMA2DDR		198
+#define ACLK_DMA2DDR		199
+#define PCLK_DCF		200
+#define ACLK_DCF		201
+#define MCLK_ACODEC_TX		202
+#define SCLK_UART0_SRC		203
+#define SCLK_UART1_SRC		204
+#define SCLK_UART2_SRC		205
+#define XIN_RC_SRC		206
+#define CLK_UTMI_USBOTG		207
+#define CLK_REF_USBPHY		208
+
+#endif // _DT_BINDINGS_CLK_ROCKCHIP_RV1103B_H
diff --git a/include/dt-bindings/clock/samsung,exynosautov920.h b/include/dt-bindings/clock/samsung,exynosautov920.h
index 06dec27a8c770..f2628c220b228 100644
--- a/include/dt-bindings/clock/samsung,exynosautov920.h
+++ b/include/dt-bindings/clock/samsung,exynosautov920.h
@@ -309,4 +309,10 @@
 #define CLK_MOUT_MFD_NOC_USER           1
 #define CLK_DOUT_MFD_NOCP               2
 
+/* CMU_G3D */
+#define FOUT_PLL_G3D                    1
+#define CLK_MOUT_G3D_NOC                2
+#define CLK_MOUT_G3D_SWITCH_USER        3
+#define CLK_MOUT_G3D_NOCP_USER          4
+
 #endif /* _DT_BINDINGS_CLOCK_EXYNOSAUTOV920_H */
diff --git a/include/dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h b/include/dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h
new file mode 100644
index 0000000000000..c1c875e016f8e
--- /dev/null
+++ b/include/dt-bindings/clock/tenstorrent,atlantis-prcm-rcpu.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Tenstorrent Atlantis PRCM Clock and Reset Indices
+ *
+ * Copyright (c) 2026 Tenstorrent
+ */
+
+#ifndef _DT_BINDINGS_ATLANTIS_PRCM_RCPU_H
+#define _DT_BINDINGS_ATLANTIS_PRCM_RCPU_H
+
+/*
+ * RCPU Domain Clock IDs
+ */
+#define CLK_RCPU_PLL		0
+#define CLK_RCPU_ROOT		1
+#define CLK_RCPU_DIV2		2
+#define CLK_RCPU_DIV4		3
+#define CLK_RCPU_RTC		4
+#define CLK_SMNDMA0_ACLK	5
+#define CLK_SMNDMA1_ACLK	6
+#define CLK_WDT0_PCLK		7
+#define CLK_WDT1_PCLK		8
+#define CLK_TIMER_PCLK		9
+#define CLK_PVTC_PCLK		10
+#define CLK_PMU_PCLK		11
+#define CLK_MAILBOX_HCLK	12
+#define CLK_SEC_SPACC_HCLK	13
+#define CLK_SEC_OTP_HCLK	14
+#define CLK_TRNG_PCLK		15
+#define CLK_SEC_CRC_HCLK	16
+#define CLK_SMN_HCLK		17
+#define CLK_AHB0_HCLK		18
+#define CLK_SMN_PCLK		19
+#define CLK_SMN_CLK		20
+#define CLK_SCRATCHPAD_CLK	21
+#define CLK_RCPU_CORE_CLK	22
+#define CLK_RCPU_ROM_CLK	23
+#define CLK_OTP_LOAD_CLK	24
+#define CLK_NOC_PLL		25
+#define CLK_NOCC_CLK		26
+#define CLK_NOCC_DIV2		27
+#define CLK_NOCC_DIV4		28
+#define CLK_NOCC_RTC		29
+#define CLK_NOCC_CAN		30
+#define CLK_QSPI_SCLK		31
+#define CLK_QSPI_HCLK		32
+#define CLK_I2C0_PCLK		33
+#define CLK_I2C1_PCLK		34
+#define CLK_I2C2_PCLK		35
+#define CLK_I2C3_PCLK		36
+#define CLK_I2C4_PCLK		37
+#define CLK_UART0_PCLK		38
+#define CLK_UART1_PCLK		39
+#define CLK_UART2_PCLK		40
+#define CLK_UART3_PCLK		41
+#define CLK_UART4_PCLK		42
+#define CLK_SPI0_PCLK		43
+#define CLK_SPI1_PCLK		44
+#define CLK_SPI2_PCLK		45
+#define CLK_SPI3_PCLK		46
+#define CLK_GPIO_PCLK		47
+#define CLK_CAN0_HCLK		48
+#define CLK_CAN0_CLK		49
+#define CLK_CAN1_HCLK		50
+#define CLK_CAN1_CLK		51
+#define CLK_CAN0_TIMER_CLK	52
+#define CLK_CAN1_TIMER_CLK	53
+
+/* RCPU domain reset */
+#define RST_SMNDMA0		0
+#define RST_SMNDMA1		1
+#define RST_WDT0		2
+#define RST_WDT1		3
+#define RST_TMR			4
+#define RST_PVTC		5
+#define RST_PMU			6
+#define RST_MAILBOX		7
+#define RST_SPACC		8
+#define RST_OTP			9
+#define RST_TRNG		10
+#define RST_CRC			11
+#define RST_QSPI		12
+#define RST_I2C0		13
+#define RST_I2C1		14
+#define RST_I2C2		15
+#define RST_I2C3		16
+#define RST_I2C4		17
+#define RST_UART0		18
+#define RST_UART1		19
+#define RST_UART2		20
+#define RST_UART3		21
+#define RST_UART4		22
+#define RST_SPI0		23
+#define RST_SPI1		24
+#define RST_SPI2		25
+#define RST_SPI3		26
+#define RST_GPIO		27
+#define RST_CAN0		28
+#define RST_CAN1		29
+#define RST_I2S0		30
+#define RST_I2S1		31
+
+#endif /* _DT_BINDINGS_ATLANTIS_PRCM_RCPU_H */
diff --git a/include/dt-bindings/clock/vf610-clock.h b/include/dt-bindings/clock/vf610-clock.h
index 373644e467479..5d94bd561a2e2 100644
--- a/include/dt-bindings/clock/vf610-clock.h
+++ b/include/dt-bindings/clock/vf610-clock.h
@@ -197,6 +197,10 @@
 #define VF610_CLK_TCON1			188
 #define VF610_CLK_CAAM			189
 #define VF610_CLK_CRC			190
-#define VF610_CLK_END			191
+#define VF610_CLK_ESW			191
+#define VF610_CLK_ESW_MAC_TAB0		192
+#define VF610_CLK_ESW_MAC_TAB1		193
+#define VF610_CLK_ESW_MAC_TAB2		194
+#define VF610_CLK_ESW_MAC_TAB3		195
 
 #endif /* __DT_BINDINGS_CLOCK_VF610_H */
diff --git a/include/dt-bindings/interconnect/qcom,eliza-rpmh.h b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h
new file mode 100644
index 0000000000000..95db2fe647de5
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_ELIZA_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_ELIZA_H
+
+#define MASTER_QSPI_0				0
+#define MASTER_QUP_1				1
+#define MASTER_UFS_MEM				2
+#define MASTER_USB3_0				3
+#define SLAVE_A1NOC_SNOC			4
+
+#define MASTER_QUP_2				0
+#define MASTER_CRYPTO				1
+#define MASTER_IPA				2
+#define MASTER_SOCCP_AGGR_NOC			3
+#define MASTER_QDSS_ETR				4
+#define MASTER_QDSS_ETR_1			5
+#define MASTER_SDCC_1				6
+#define MASTER_SDCC_2				7
+#define SLAVE_A2NOC_SNOC			8
+
+#define MASTER_QUP_CORE_1			0
+#define MASTER_QUP_CORE_2			1
+#define SLAVE_QUP_CORE_1			2
+#define SLAVE_QUP_CORE_2			3
+
+#define MASTER_CNOC_CFG				0
+#define SLAVE_AHB2PHY_SOUTH			1
+#define SLAVE_AHB2PHY_NORTH			2
+#define SLAVE_CAMERA_CFG			3
+#define SLAVE_CLK_CTL				4
+#define SLAVE_CRYPTO_0_CFG			5
+#define SLAVE_DISPLAY_CFG			6
+#define SLAVE_GFX3D_CFG				7
+#define SLAVE_I3C_IBI0_CFG			8
+#define SLAVE_I3C_IBI1_CFG			9
+#define SLAVE_IMEM_CFG				10
+#define SLAVE_CNOC_MSS				11
+#define SLAVE_PCIE_0_CFG			12
+#define SLAVE_PRNG				13
+#define SLAVE_QDSS_CFG				14
+#define SLAVE_QSPI_0				15
+#define SLAVE_QUP_1				16
+#define SLAVE_QUP_2				17
+#define SLAVE_SDCC_2				18
+#define SLAVE_TCSR				19
+#define SLAVE_TLMM				20
+#define SLAVE_UFS_MEM_CFG			21
+#define SLAVE_USB3_0				22
+#define SLAVE_VENUS_CFG				23
+#define SLAVE_VSENSE_CTRL_CFG			24
+#define SLAVE_CNOC_MNOC_HF_CFG			25
+#define SLAVE_CNOC_MNOC_SF_CFG			26
+#define SLAVE_PCIE_ANOC_CFG			27
+#define SLAVE_QDSS_STM				28
+#define SLAVE_TCU				29
+
+#define MASTER_GEM_NOC_CNOC			0
+#define MASTER_GEM_NOC_PCIE_SNOC		1
+#define SLAVE_AOSS				2
+#define SLAVE_IPA_CFG				3
+#define SLAVE_IPC_ROUTER_CFG			4
+#define SLAVE_SOCCP				5
+#define SLAVE_TME_CFG				6
+#define SLAVE_APPSS				7
+#define SLAVE_CNOC_CFG				8
+#define SLAVE_DDRSS_CFG				9
+#define SLAVE_BOOT_IMEM				10
+#define SLAVE_IMEM				11
+#define SLAVE_BOOT_IMEM_2			12
+#define SLAVE_SERVICE_CNOC			13
+#define SLAVE_PCIE_0				14
+#define SLAVE_PCIE_1				15
+
+#define MASTER_GPU_TCU				0
+#define MASTER_SYS_TCU				1
+#define MASTER_APPSS_PROC			2
+#define MASTER_GFX3D				3
+#define MASTER_LPASS_GEM_NOC			4
+#define MASTER_MSS_PROC				5
+#define MASTER_MNOC_HF_MEM_NOC			6
+#define MASTER_MNOC_SF_MEM_NOC			7
+#define MASTER_COMPUTE_NOC			8
+#define MASTER_ANOC_PCIE_GEM_NOC		9
+#define MASTER_SNOC_SF_MEM_NOC			10
+#define MASTER_WLAN_Q6				11
+#define MASTER_GIC				12
+#define SLAVE_GEM_NOC_CNOC			13
+#define SLAVE_LLCC				14
+#define SLAVE_MEM_NOC_PCIE_SNOC			15
+
+#define MASTER_LPIAON_NOC			0
+#define SLAVE_LPASS_GEM_NOC			1
+
+#define MASTER_LPASS_LPINOC			0
+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		1
+
+#define MASTER_LPASS_PROC			0
+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
+
+#define MASTER_LLCC				0
+#define SLAVE_EBI1				1
+
+#define MASTER_CAMNOC_NRT_ICP_SF		0
+#define MASTER_CAMNOC_RT_CDM_SF			1
+#define MASTER_CAMNOC_SF			2
+#define MASTER_VIDEO_MVP			3
+#define MASTER_VIDEO_V_PROC			4
+#define MASTER_CNOC_MNOC_SF_CFG			5
+#define MASTER_CAMNOC_HF			6
+#define MASTER_MDP				7
+#define MASTER_CNOC_MNOC_HF_CFG			8
+#define SLAVE_MNOC_SF_MEM_NOC			9
+#define SLAVE_SERVICE_MNOC_SF			10
+#define SLAVE_MNOC_HF_MEM_NOC			11
+#define SLAVE_SERVICE_MNOC_HF			12
+
+#define MASTER_CDSP_PROC			0
+#define SLAVE_CDSP_MEM_NOC			1
+
+#define MASTER_PCIE_ANOC_CFG			0
+#define MASTER_PCIE_0				1
+#define MASTER_PCIE_1				2
+#define SLAVE_ANOC_PCIE_GEM_NOC			3
+#define SLAVE_SERVICE_PCIE_ANOC			4
+
+#define MASTER_A1NOC_SNOC			0
+#define MASTER_A2NOC_SNOC			1
+#define MASTER_CNOC_SNOC			2
+#define MASTER_NSINOC_SNOC			3
+#define SLAVE_SNOC_GEM_NOC_SF			4
+
+#endif
diff --git a/include/dt-bindings/power/allwinner,sun60i-a733-pck-600.h b/include/dt-bindings/power/allwinner,sun60i-a733-pck-600.h
new file mode 100644
index 0000000000000..cf476a005b553
--- /dev/null
+++ b/include/dt-bindings/power/allwinner,sun60i-a733-pck-600.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+
+#ifndef _DT_BINDINGS_POWER_SUN60I_A733_PCK600_H_
+#define _DT_BINDINGS_POWER_SUN60I_A733_PCK600_H_
+
+#define PD_VI			0
+#define PD_DE_SYS		1
+#define PD_VE_DEC		2
+#define PD_VE_ENC		3
+#define PD_NPU			4
+#define PD_GPU_TOP		5
+#define PD_GPU_CORE		6
+#define PD_PCIE			7
+#define PD_USB2			8
+#define PD_VO			9
+#define PD_VO1			10
+
+#endif /* _DT_BINDINGS_POWER_SUN60I_A733_PCK600_H_ */
diff --git a/include/dt-bindings/power/marvell,pxa1908-power.h b/include/dt-bindings/power/marvell,pxa1908-power.h
index 19b088351af13..173f47e0e69de 100644
--- a/include/dt-bindings/power/marvell,pxa1908-power.h
+++ b/include/dt-bindings/power/marvell,pxa1908-power.h
@@ -13,5 +13,6 @@
 #define PXA1908_POWER_DOMAIN_GPU2D	2
 #define PXA1908_POWER_DOMAIN_DSI	3
 #define PXA1908_POWER_DOMAIN_ISP	4
+#define PXA1908_POWER_DOMAIN_AUDIO	5
 
 #endif
diff --git a/include/dt-bindings/power/mediatek,mt8189-power.h b/include/dt-bindings/power/mediatek,mt8189-power.h
new file mode 100644
index 0000000000000..70a8c21134572
--- /dev/null
+++ b/include/dt-bindings/power/mediatek,mt8189-power.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) 2025 MediaTek Inc.
+ * Author: Qiqi Wang <qiqi.wang@mediatek.com>
+ */
+
+#ifndef _DT_BINDINGS_POWER_MT8189_POWER_H
+#define _DT_BINDINGS_POWER_MT8189_POWER_H
+
+/* SPM */
+#define MT8189_POWER_DOMAIN_CONN			0
+#define MT8189_POWER_DOMAIN_AUDIO			1
+#define MT8189_POWER_DOMAIN_ADSP_TOP_DORMANT		2
+#define MT8189_POWER_DOMAIN_ADSP_INFRA			3
+#define MT8189_POWER_DOMAIN_ADSP_AO			4
+#define MT8189_POWER_DOMAIN_MM_INFRA			5
+#define MT8189_POWER_DOMAIN_ISP_IMG1			6
+#define MT8189_POWER_DOMAIN_ISP_IMG2			7
+#define MT8189_POWER_DOMAIN_ISP_IPE			8
+#define MT8189_POWER_DOMAIN_VDE0			9
+#define MT8189_POWER_DOMAIN_VEN0			10
+#define MT8189_POWER_DOMAIN_CAM_MAIN			11
+#define MT8189_POWER_DOMAIN_CAM_SUBA			12
+#define MT8189_POWER_DOMAIN_CAM_SUBB			13
+#define MT8189_POWER_DOMAIN_MDP0			14
+#define MT8189_POWER_DOMAIN_DISP			15
+#define MT8189_POWER_DOMAIN_DP_TX			16
+#define MT8189_POWER_DOMAIN_CSI_RX			17
+#define MT8189_POWER_DOMAIN_SSUSB			18
+#define MT8189_POWER_DOMAIN_MFG0			19
+#define MT8189_POWER_DOMAIN_MFG1			20
+#define MT8189_POWER_DOMAIN_MFG2			21
+#define MT8189_POWER_DOMAIN_MFG3			22
+#define MT8189_POWER_DOMAIN_EDP_TX_DORMANT		23
+#define MT8189_POWER_DOMAIN_PCIE			24
+#define MT8189_POWER_DOMAIN_PCIE_PHY			25
+
+#endif /* _DT_BINDINGS_POWER_MT8189_POWER_H */
diff --git a/include/dt-bindings/power/mt7622-power.h b/include/dt-bindings/power/mt7622-power.h
index ffad81ad3d464..ec244989eeba1 100644
--- a/include/dt-bindings/power/mt7622-power.h
+++ b/include/dt-bindings/power/mt7622-power.h
@@ -10,5 +10,6 @@
 #define MT7622_POWER_DOMAIN_HIF0	1
 #define MT7622_POWER_DOMAIN_HIF1	2
 #define MT7622_POWER_DOMAIN_WB		3
+#define MT7622_POWER_DOMAIN_AUDIO	4
 
 #endif /* _DT_BINDINGS_POWER_MT7622_POWER_H */
diff --git a/include/dt-bindings/reset/cix,sky1-s5-system-control.h b/include/dt-bindings/reset/cix,sky1-s5-system-control.h
new file mode 100644
index 0000000000000..808bbcbe0c988
--- /dev/null
+++ b/include/dt-bindings/reset/cix,sky1-s5-system-control.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/* Author: Jerry Zhu <jerry.zhu@cixtech.com> */
+#ifndef DT_BINDING_RESET_CIX_SKY1_S5_SYSTEM_CONTROL_H
+#define DT_BINDING_RESET_CIX_SKY1_S5_SYSTEM_CONTROL_H
+
+/* reset for csu_pm */
+#define SKY1_CSU_PM_RESET_N		0
+#define SKY1_SENSORFUSION_RESET_N	1
+#define SKY1_SENSORFUSION_NOC_RESET_N	2
+
+/* reset group0 for s0 domain modules */
+#define SKY1_DDRC_RESET_N		3
+#define SKY1_GIC_RESET_N		4
+#define SKY1_CI700_RESET_N		5
+#define SKY1_SYS_NI700_RESET_N		6
+#define SKY1_MM_NI700_RESET_N		7
+#define SKY1_PCIE_NI700_RESET_N		8
+#define SKY1_GPU_RESET_N		9
+#define SKY1_NPUTOP_RESET_N		10
+#define SKY1_NPUCORE0_RESET_N		11
+#define SKY1_NPUCORE1_RESET_N		12
+#define SKY1_NPUCORE2_RESET_N		13
+#define SKY1_VPU_RESET_N		14
+#define SKY1_ISP_SRESET_N		15
+#define SKY1_ISP_ARESET_N		16
+#define SKY1_ISP_HRESET_N		17
+#define SKY1_ISP_GDCRESET_N		18
+#define SKY1_DPU_RESET0_N		19
+#define SKY1_DPU_RESET1_N		20
+#define SKY1_DPU_RESET2_N		21
+#define SKY1_DPU_RESET3_N		22
+#define SKY1_DPU_RESET4_N		23
+#define SKY1_DP_RESET0_N		24
+#define SKY1_DP_RESET1_N		25
+#define SKY1_DP_RESET2_N		26
+#define SKY1_DP_RESET3_N		27
+#define SKY1_DP_RESET4_N		28
+#define SKY1_DP_PHY_RST_N		29
+
+/* reset group1 for s0 domain modules */
+#define SKY1_AUDIO_HIFI5_RESET_N	30
+#define SKY1_AUDIO_HIFI5_NOC_RESET_N	31
+#define SKY1_CSIDPHY_PRST0_N		32
+#define SKY1_CSIDPHY_CMNRST0_N		33
+#define SKY1_CSI0_RST_N			34
+#define SKY1_CSIDPHY_PRST1_N		35
+#define SKY1_CSIDPHY_CMNRST1_N		36
+#define SKY1_CSI1_RST_N			37
+#define SKY1_CSI2_RST_N			38
+#define SKY1_CSI3_RST_N			39
+#define SKY1_CSIBRDGE0_RST_N		40
+#define SKY1_CSIBRDGE1_RST_N		41
+#define SKY1_CSIBRDGE2_RST_N		42
+#define SKY1_CSIBRDGE3_RST_N		43
+#define SKY1_GMAC0_RST_N		44
+#define SKY1_GMAC1_RST_N		45
+#define SKY1_PCIE0_RESET_N		46
+#define SKY1_PCIE1_RESET_N		47
+#define SKY1_PCIE2_RESET_N		48
+#define SKY1_PCIE3_RESET_N		49
+#define SKY1_PCIE4_RESET_N		50
+
+/* reset group1 for usb phys */
+#define SKY1_USB_DP_PHY0_PRST_N		51
+#define SKY1_USB_DP_PHY1_PRST_N		52
+#define SKY1_USB_DP_PHY2_PRST_N		53
+#define SKY1_USB_DP_PHY3_PRST_N		54
+#define SKY1_USB_DP_PHY0_RST_N		55
+#define SKY1_USB_DP_PHY1_RST_N		56
+#define SKY1_USB_DP_PHY2_RST_N		57
+#define SKY1_USB_DP_PHY3_RST_N		58
+#define SKY1_USBPHY_SS_PST_N		59
+#define SKY1_USBPHY_SS_RST_N		60
+#define SKY1_USBPHY_HS0_PRST_N		61
+#define SKY1_USBPHY_HS1_PRST_N		62
+#define SKY1_USBPHY_HS2_PRST_N		63
+#define SKY1_USBPHY_HS3_PRST_N		64
+#define SKY1_USBPHY_HS4_PRST_N		65
+#define SKY1_USBPHY_HS5_PRST_N		66
+#define SKY1_USBPHY_HS6_PRST_N		67
+#define SKY1_USBPHY_HS7_PRST_N		68
+#define SKY1_USBPHY_HS8_PRST_N		69
+#define SKY1_USBPHY_HS9_PRST_N		70
+
+/* reset group1 for usb controllers */
+#define SKY1_USBC_SS0_PRST_N		71
+#define SKY1_USBC_SS1_PRST_N		72
+#define SKY1_USBC_SS2_PRST_N		73
+#define SKY1_USBC_SS3_PRST_N		74
+#define SKY1_USBC_SS4_PRST_N		75
+#define SKY1_USBC_SS5_PRST_N		76
+#define SKY1_USBC_SS0_RST_N		77
+#define SKY1_USBC_SS1_RST_N		78
+#define SKY1_USBC_SS2_RST_N		79
+#define SKY1_USBC_SS3_RST_N		80
+#define SKY1_USBC_SS4_RST_N		81
+#define SKY1_USBC_SS5_RST_N		82
+#define SKY1_USBC_HS0_PRST_N		83
+#define SKY1_USBC_HS1_PRST_N		84
+#define SKY1_USBC_HS2_PRST_N		85
+#define SKY1_USBC_HS3_PRST_N		86
+#define SKY1_USBC_HS0_RST_N		87
+#define SKY1_USBC_HS1_RST_N		88
+#define SKY1_USBC_HS2_RST_N		89
+#define SKY1_USBC_HS3_RST_N		90
+
+/* reset group0 for rcsu */
+#define SKY1_AUDIO_RCSU_RESET_N			91
+#define SKY1_CI700_RCSU_RESET_N			92
+#define SKY1_CSI_RCSU0_RESET_N			93
+#define SKY1_CSI_RCSU1_RESET_N			94
+#define SKY1_CSU_PM_RCSU_RESET_N		95
+#define SKY1_DDR_BROADCAST_RCSU_RESET_N		96
+#define SKY1_DDR_CTRL_RCSU_0_RESET_N		97
+#define SKY1_DDR_CTRL_RCSU_1_RESET_N		98
+#define SKY1_DDR_CTRL_RCSU_2_RESET_N		99
+#define SKY1_DDR_CTRL_RCSU_3_RESET_N		100
+#define SKY1_DDR_TZC400_RCSU_0_RESET_N		101
+#define SKY1_DDR_TZC400_RCSU_1_RESET_N		102
+#define SKY1_DDR_TZC400_RCSU_2_RESET_N		103
+#define SKY1_DDR_TZC400_RCSU_3_RESET_N		104
+#define SKY1_DP0_RCSU_RESET_N			105
+#define SKY1_DP1_RCSU_RESET_N			106
+#define SKY1_DP2_RCSU_RESET_N			107
+#define SKY1_DP3_RCSU_RESET_N			108
+#define SKY1_DP4_RCSU_RESET_N			109
+#define SKY1_DPU0_RCSU_RESET_N			110
+#define SKY1_DPU1_RCSU_RESET_N			111
+#define SKY1_DPU2_RCSU_RESET_N			112
+#define SKY1_DPU3_RCSU_RESET_N			113
+#define SKY1_DPU4_RCSU_RESET_N			114
+#define SKY1_DSU_RCSU_RESET_N			115
+#define SKY1_FCH_RCSU_RESET_N			116
+#define SKY1_GICD_RCSU_RESET_N			117
+#define SKY1_GMAC_RCSU_RESET_N			118
+#define SKY1_GPU_RCSU_RESET_N			119
+#define SKY1_ISP_RCSU0_RESET_N			120
+#define SKY1_ISP_RCSU1_RESET_N			121
+#define SKY1_NI700_MMHUB_RCSU_RESET_N		122
+
+/* reset group1 for rcsu */
+#define SKY1_NPU_RCSU_RESET_N			123
+#define SKY1_NI700_PCIE_RCSU_RESET_N		124
+#define SKY1_PCIE_X421_RCSU_RESET_N		125
+#define SKY1_PCIE_X8_RCSU_RESET_N		126
+#define SKY1_SF_RCSU_RESET_N			127
+#define SKY1_RCSU_SMMU_MMHUB_RESET_N		128
+#define SKY1_RCSU_SMMU_PCIEHUB_RESET_N		129
+#define SKY1_RCSU_SYSHUB_RESET_N		130
+#define SKY1_NI700_SMN_RCSU_RESET_N		131
+#define SKY1_NI700_SYSHUB_RCSU_RESET_N		132
+#define SKY1_RCSU_USB2_HOST0_RESET_N		133
+#define SKY1_RCSU_USB2_HOST1_RESET_N		134
+#define SKY1_RCSU_USB2_HOST2_RESET_N		135
+#define SKY1_RCSU_USB2_HOST3_RESET_N		136
+#define SKY1_RCSU_USB3_TYPEA_DRD_RESET_N	137
+#define SKY1_RCSU_USB3_TYPEC_DRD_RESET_N	138
+#define SKY1_RCSU_USB3_TYPEC_HOST0_RESET_N	139
+#define SKY1_RCSU_USB3_TYPEC_HOST1_RESET_N	140
+#define SKY1_RCSU_USB3_TYPEC_HOST2_RESET_N	141
+#define SKY1_VPU_RCSU_RESET_N			142
+
+#endif
diff --git a/include/dt-bindings/reset/cix,sky1-system-control.h b/include/dt-bindings/reset/cix,sky1-system-control.h
new file mode 100644
index 0000000000000..7a16fc4ef3b54
--- /dev/null
+++ b/include/dt-bindings/reset/cix,sky1-system-control.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/* Author: Jerry Zhu <jerry.zhu@cixtech.com> */
+#ifndef DT_BINDING_RESET_CIX_SKY1_SYSTEM_CONTROL_H
+#define DT_BINDING_RESET_CIX_SKY1_SYSTEM_CONTROL_H
+
+/* func reset for sky1 fch */
+#define SW_I3C0_RST_FUNC_G_N	0
+#define SW_I3C0_RST_FUNC_I_N	1
+#define SW_I3C1_RST_FUNC_G_N	2
+#define SW_I3C1_RST_FUNC_I_N	3
+#define SW_UART0_RST_FUNC_N	4
+#define SW_UART1_RST_FUNC_N	5
+#define SW_UART2_RST_FUNC_N	6
+#define SW_UART3_RST_FUNC_N	7
+#define SW_TIMER_RST_FUNC_N	8
+
+/* apb reset for sky1 fch */
+#define SW_I3C0_RST_APB_N	9
+#define SW_I3C1_RST_APB_N	10
+#define SW_DMA_RST_AXI_N	11
+#define SW_UART0_RST_APB_N	12
+#define SW_UART1_RST_APB_N	13
+#define SW_UART2_RST_APB_N	14
+#define SW_UART3_RST_APB_N	15
+#define SW_SPI0_RST_APB_N	16
+#define SW_SPI1_RST_APB_N	17
+#define SW_I2C0_RST_APB_N	18
+#define SW_I2C1_RST_APB_N	19
+#define SW_I2C2_RST_APB_N	20
+#define SW_I2C3_RST_APB_N	21
+#define SW_I2C4_RST_APB_N	22
+#define SW_I2C5_RST_APB_N	23
+#define SW_I2C6_RST_APB_N	24
+#define SW_I2C7_RST_APB_N	25
+#define SW_GPIO_RST_APB_N	26
+
+/* fch rst for xspi */
+#define SW_XSPI_REG_RST_N	27
+#define SW_XSPI_SYS_RST_N	28
+
+#endif
diff --git a/include/dt-bindings/reset/qcom,ipq5210-gcc.h b/include/dt-bindings/reset/qcom,ipq5210-gcc.h
new file mode 100644
index 0000000000000..09890a09087c3
--- /dev/null
+++ b/include/dt-bindings/reset/qcom,ipq5210-gcc.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_RESET_IPQ_GCC_IPQ5210_H
+#define _DT_BINDINGS_RESET_IPQ_GCC_IPQ5210_H
+
+#define GCC_ADSS_BCR						0
+#define GCC_ADSS_PWM_ARES					1
+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR			2
+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_GPLL0_ARES		3
+#define GCC_APSS_AHB_ARES					4
+#define GCC_APSS_ATB_ARES					5
+#define GCC_APSS_AXI_ARES					6
+#define GCC_APSS_TS_ARES					7
+#define GCC_BOOT_ROM_AHB_ARES					8
+#define GCC_BOOT_ROM_BCR					9
+#define GCC_GEPHY_BCR						10
+#define GCC_GEPHY_SYS_ARES					11
+#define GCC_GP1_ARES						12
+#define GCC_GP2_ARES						13
+#define GCC_GP3_ARES						14
+#define GCC_MDIO_AHB_ARES					15
+#define GCC_MDIO_BCR						16
+#define GCC_MDIO_GEPHY_AHB_ARES					17
+#define GCC_NSS_BCR						18
+#define GCC_NSS_TS_ARES						19
+#define GCC_NSSCC_ARES						20
+#define GCC_NSSCFG_ARES						21
+#define GCC_NSSNOC_ATB_ARES					22
+#define GCC_NSSNOC_MEMNOC_1_ARES				23
+#define GCC_NSSNOC_MEMNOC_ARES					24
+#define GCC_NSSNOC_NSSCC_ARES					25
+#define GCC_NSSNOC_PCNOC_1_ARES					26
+#define GCC_NSSNOC_QOSGEN_REF_ARES				27
+#define GCC_NSSNOC_SNOC_1_ARES					28
+#define GCC_NSSNOC_SNOC_ARES					29
+#define GCC_NSSNOC_TIMEOUT_REF_ARES				30
+#define GCC_NSSNOC_XO_DCD_ARES					31
+#define GCC_PCIE0_AHB_ARES					32
+#define GCC_PCIE0_AUX_ARES					33
+#define GCC_PCIE0_AXI_M_ARES					34
+#define GCC_PCIE0_AXI_S_BRIDGE_ARES				35
+#define GCC_PCIE0_AXI_S_ARES					36
+#define GCC_PCIE0_BCR						37
+#define GCC_PCIE0_LINK_DOWN_BCR					38
+#define GCC_PCIE0_PHY_BCR					39
+#define GCC_PCIE0_PIPE_ARES					40
+#define GCC_PCIE0PHY_PHY_BCR					41
+#define GCC_PCIE1_AHB_ARES					42
+#define GCC_PCIE1_AUX_ARES					43
+#define GCC_PCIE1_AXI_M_ARES					44
+#define GCC_PCIE1_AXI_S_BRIDGE_ARES				45
+#define GCC_PCIE1_AXI_S_ARES					46
+#define GCC_PCIE1_BCR						47
+#define GCC_PCIE1_LINK_DOWN_BCR					48
+#define GCC_PCIE1_PHY_BCR					49
+#define GCC_PCIE1_PIPE_ARES					50
+#define GCC_PCIE1PHY_PHY_BCR					51
+#define GCC_QRNG_AHB_ARES					52
+#define GCC_QRNG_BCR						53
+#define GCC_QUPV3_2X_CORE_ARES					54
+#define GCC_QUPV3_AHB_MST_ARES					55
+#define GCC_QUPV3_AHB_SLV_ARES					56
+#define GCC_QUPV3_BCR						57
+#define GCC_QUPV3_CORE_ARES					58
+#define GCC_QUPV3_WRAP_SE0_ARES					59
+#define GCC_QUPV3_WRAP_SE0_BCR					60
+#define GCC_QUPV3_WRAP_SE1_ARES					61
+#define GCC_QUPV3_WRAP_SE1_BCR					62
+#define GCC_QUPV3_WRAP_SE2_ARES					63
+#define GCC_QUPV3_WRAP_SE2_BCR					64
+#define GCC_QUPV3_WRAP_SE3_ARES					65
+#define GCC_QUPV3_WRAP_SE3_BCR					66
+#define GCC_QUPV3_WRAP_SE4_ARES					67
+#define GCC_QUPV3_WRAP_SE4_BCR					68
+#define GCC_QUPV3_WRAP_SE5_ARES					69
+#define GCC_QUPV3_WRAP_SE5_BCR					70
+#define GCC_QUSB2_0_PHY_BCR					71
+#define GCC_SDCC1_AHB_ARES					72
+#define GCC_SDCC1_APPS_ARES					73
+#define GCC_SDCC1_ICE_CORE_ARES					74
+#define GCC_SDCC_BCR						75
+#define GCC_TLMM_AHB_ARES					76
+#define GCC_TLMM_ARES						77
+#define GCC_TLMM_BCR						78
+#define GCC_UNIPHY0_AHB_ARES					79
+#define GCC_UNIPHY0_BCR						80
+#define GCC_UNIPHY0_SYS_ARES					81
+#define GCC_UNIPHY1_AHB_ARES					82
+#define GCC_UNIPHY1_BCR						83
+#define GCC_UNIPHY1_SYS_ARES					84
+#define GCC_UNIPHY2_AHB_ARES					85
+#define GCC_UNIPHY2_BCR						86
+#define GCC_UNIPHY2_SYS_ARES					87
+#define GCC_USB0_AUX_ARES					88
+#define GCC_USB0_MASTER_ARES					89
+#define GCC_USB0_MOCK_UTMI_ARES					90
+#define GCC_USB0_PHY_BCR					91
+#define GCC_USB0_PHY_CFG_AHB_ARES				92
+#define GCC_USB0_PIPE_ARES					93
+#define GCC_USB0_SLEEP_ARES					94
+#define GCC_USB3PHY_0_PHY_BCR					95
+#define GCC_USB_BCR						96
+#define GCC_PCIE0_PIPE_RESET					97
+#define GCC_PCIE0_CORE_STICKY_RESET				98
+#define GCC_PCIE0_AXI_S_STICKY_RESET				99
+#define GCC_PCIE0_AXI_S_RESET					100
+#define GCC_PCIE0_AXI_M_STICKY_RESET				101
+#define GCC_PCIE0_AXI_M_RESET					102
+#define GCC_PCIE0_AUX_RESET					103
+#define GCC_PCIE0_AHB_RESET					104
+#define GCC_PCIE1_PIPE_RESET					105
+#define GCC_PCIE1_CORE_STICKY_RESET				106
+#define GCC_PCIE1_AXI_S_STICKY_RESET				107
+#define GCC_PCIE1_AXI_S_RESET					108
+#define GCC_PCIE1_AXI_M_STICKY_RESET				109
+#define GCC_PCIE1_AXI_M_RESET					110
+#define GCC_PCIE1_AUX_RESET					111
+#define GCC_PCIE1_AHB_RESET					112
+#define GCC_UNIPHY0_XPCS_ARES					113
+#define GCC_UNIPHY1_XPCS_ARES					114
+#define GCC_UNIPHY2_XPCS_ARES					115
+#define GCC_QDSS_BCR						116
+
+#endif
diff --git a/include/dt-bindings/usb/pd.h b/include/dt-bindings/usb/pd.h
index e6526b1381745..1e64a1f563f9b 100644
--- a/include/dt-bindings/usb/pd.h
+++ b/include/dt-bindings/usb/pd.h
@@ -60,6 +60,7 @@
 	 PDO_VAR_MAX_VOLT(max_mv) | PDO_VAR_MAX_CURR(max_ma))
 
 #define APDO_TYPE_PPS		0
+#define APDO_TYPE_SPR_AVS	2
 
 #define PDO_APDO_TYPE_SHIFT	28	/* Only valid value currently is 0x0 - PPS */
 #define PDO_APDO_TYPE_MASK	0x3
@@ -85,6 +86,23 @@
 	 PDO_PPS_APDO_MIN_VOLT(min_mv) | PDO_PPS_APDO_MAX_VOLT(max_mv) |	\
 	 PDO_PPS_APDO_MAX_CURR(max_ma))
 
+#define PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR_SHIFT	10	/* 10mA units */
+#define PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR_SHIFT	0	/* 10mA units */
+#define PDO_SPR_AVS_APDO_MAX_CURR_MASK			0x3ff
+
+#define PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR(max_cur_9v_to_15v_ma)		\
+	((((max_cur_9v_to_15v_ma) / 10) & PDO_SPR_AVS_APDO_MAX_CURR_MASK) <<	\
+	PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR_SHIFT)
+
+#define PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR(max_cur_15v_to_20v_ma)		\
+	((((max_cur_15v_to_20v_ma) / 10) & PDO_SPR_AVS_APDO_MAX_CURR_MASK) <<	\
+	PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR_SHIFT)
+
+#define PDO_SPR_AVS_SNK_APDO(max_cur_9v_to_15v_ma, max_cur_15v_to_20v_ma)	\
+	(PDO_TYPE(PDO_TYPE_APDO) | PDO_APDO_TYPE(APDO_TYPE_SPR_AVS) |		\
+	PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR(max_cur_9v_to_15v_ma) |		\
+	PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR(max_cur_15v_to_20v_ma))
+
  /*
   * Based on "Table 6-14 Fixed Supply PDO - Sink" of "USB Power Delivery Specification Revision 3.0,
   * Version 1.2"
@@ -465,4 +483,22 @@
 	 | ((vbm) & 0x3) << 15 | (curr) << 14 | ((vbi) & 0x3f) << 7	\
 	 | ((gi) & 0x3f) << 1 | (ct))
 
+/*
+ * Sink Load Characteristics
+ * -------------------------
+ *  <15>    :: Can tolerate vbus voltage droop
+ *  <11:14> :: Duty cycle in 5% increments when bits 4:0 are non-zero
+ *  <10:5>  :: Overload period in 20ms when bits 4:0 are non-zero
+ *  <4:0>   :: Percent overload in 10% increments. Values higher than 25 are
+ *             clipped to 250%
+ */
+#define SINK_LOAD_CHAR(vdroop, duty_cycle, period, percent_ol)		\
+	(((vdroop) & 0x1) << 15 | ((duty_cycle) & 0xf) << 11 |		\
+	 ((period) & 0x3f) << 5 | ((percent_ol) & 0x1f))
+
+/* Compliance */
+#define COMPLIANCE_LPS		(1 << 0)
+#define COMPLIANCE_PS1		(1 << 1)
+#define COMPLIANCE_PS2		(1 << 2)
+
 #endif /* __DT_POWER_DELIVERY_H */
diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h
index 056ef7b6b3600..1823a290a7b7a 100644
--- a/include/hyperv/hvgdk_mini.h
+++ b/include/hyperv/hvgdk_mini.h
@@ -477,7 +477,6 @@ union hv_vp_assist_msr_contents {	 /* HV_REGISTER_VP_ASSIST_PAGE */
 #define HVCALL_NOTIFY_PARTITION_EVENT                   0x0087
 #define HVCALL_ENTER_SLEEP_STATE			0x0084
 #define HVCALL_NOTIFY_PORT_RING_EMPTY			0x008b
-#define HVCALL_SCRUB_PARTITION				0x008d
 #define HVCALL_REGISTER_INTERCEPT_RESULT		0x0091
 #define HVCALL_ASSERT_VIRTUAL_INTERRUPT			0x0094
 #define HVCALL_CREATE_PORT				0x0095
@@ -1121,6 +1120,8 @@ enum hv_register_name {
 	HV_X64_REGISTER_MSR_MTRR_FIX4KF8000	= 0x0008007A,
 
 	HV_X64_REGISTER_REG_PAGE	= 0x0009001C,
+#elif defined(CONFIG_ARM64)
+	HV_ARM64_REGISTER_SINT_RESERVED_INTERRUPT_ID	= 0x00070001,
 #endif
 };
 
diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h
index 7310841f45121..a7754e0a2ef41 100644
--- a/include/kvm/arm_arch_timer.h
+++ b/include/kvm/arm_arch_timer.h
@@ -10,6 +10,8 @@
 #include <linux/clocksource.h>
 #include <linux/hrtimer.h>
 
+#include <linux/irqchip/arm-gic-v5.h>
+
 enum kvm_arch_timers {
 	TIMER_PTIMER,
 	TIMER_VTIMER,
@@ -47,7 +49,7 @@ struct arch_timer_vm_data {
 	u64	poffset;
 
 	/* The PPI for each timer, global to the VM */
-	u8	ppi[NR_KVM_TIMERS];
+	u32	ppi[NR_KVM_TIMERS];
 };
 
 struct arch_timer_context {
@@ -74,6 +76,9 @@ struct arch_timer_context {
 
 	/* Duplicated state from arch_timer.c for convenience */
 	u32				host_timer_irq;
+
+	/* Is this a direct timer? */
+	bool				direct;
 };
 
 struct timer_map {
@@ -130,6 +135,10 @@ void kvm_timer_init_vhe(void);
 #define timer_vm_data(ctx)		(&(timer_context_to_vcpu(ctx)->kvm->arch.timer_data))
 #define timer_irq(ctx)			(timer_vm_data(ctx)->ppi[arch_timer_ctx_index(ctx)])
 
+#define get_vgic_ppi(k, i) (((k)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V5) ? \
+			    (i) : (FIELD_PREP(GICV5_HWIRQ_ID, i) |	\
+				   FIELD_PREP(GICV5_HWIRQ_TYPE, GICV5_HWIRQ_TYPE_PPI)))
+
 u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
 			      enum kvm_arch_timers tmr,
 			      enum kvm_arch_timer_regs treg);
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 96754b51b4116..0a36a3d5c8944 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -12,6 +12,9 @@
 
 #define KVM_ARMV8_PMU_MAX_COUNTERS	32
 
+/* PPI #23 - architecturally specified for GICv5 */
+#define KVM_ARMV8_PMU_GICV5_IRQ		0x20000017
+
 #if IS_ENABLED(CONFIG_HW_PERF_EVENTS) && IS_ENABLED(CONFIG_KVM)
 struct kvm_pmc {
 	u8 idx;	/* index into the pmu->pmc array */
@@ -38,7 +41,7 @@ struct arm_pmu_entry {
 };
 
 bool kvm_supports_guest_pmuv3(void);
-#define kvm_arm_pmu_irq_initialized(v)	((v)->arch.pmu.irq_num >= VGIC_NR_SGIS)
+#define kvm_arm_pmu_irq_initialized(v)	((v)->arch.pmu.irq_num != 0)
 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx);
 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val);
 void kvm_pmu_set_counter_value_user(struct kvm_vcpu *vcpu, u64 select_idx, u64 val);
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index f2eafc65bbf4c..1388dc6028a9a 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -19,7 +19,9 @@
 #include <linux/jump_label.h>
 
 #include <linux/irqchip/arm-gic-v4.h>
+#include <linux/irqchip/arm-gic-v5.h>
 
+#define VGIC_V5_MAX_CPUS	512
 #define VGIC_V3_MAX_CPUS	512
 #define VGIC_V2_MAX_CPUS	8
 #define VGIC_NR_IRQS_LEGACY     256
@@ -31,9 +33,96 @@
 #define VGIC_MIN_LPI		8192
 #define KVM_IRQCHIP_NUM_PINS	(1020 - 32)
 
-#define irq_is_ppi(irq) ((irq) >= VGIC_NR_SGIS && (irq) < VGIC_NR_PRIVATE_IRQS)
-#define irq_is_spi(irq) ((irq) >= VGIC_NR_PRIVATE_IRQS && \
-			 (irq) <= VGIC_MAX_SPI)
+/*
+ * GICv5 supports 128 PPIs, but only the first 64 are architected. We only
+ * support the timers and PMU in KVM, both of which are architected. Rather than
+ * handling twice the state, we instead opt to only support the architected set
+ * in KVM for now. At a future stage, this can be bumped up to 128, if required.
+ */
+#define VGIC_V5_NR_PRIVATE_IRQS	64
+
+#define is_v5_type(t, i)	(FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t))
+
+#define __irq_is_sgi(t, i)						\
+	({								\
+		bool __ret;						\
+									\
+		switch (t) {						\
+		case KVM_DEV_TYPE_ARM_VGIC_V5:				\
+			__ret = false;					\
+			break;						\
+		default:						\
+			__ret  = (i) < VGIC_NR_SGIS;			\
+		}							\
+									\
+		__ret;							\
+	})
+
+#define __irq_is_ppi(t, i)						\
+	({								\
+		bool __ret;						\
+									\
+		switch (t) {						\
+		case KVM_DEV_TYPE_ARM_VGIC_V5:				\
+			__ret = is_v5_type(GICV5_HWIRQ_TYPE_PPI, (i));	\
+			break;						\
+		default:						\
+			__ret  = (i) >= VGIC_NR_SGIS;			\
+			__ret &= (i) < VGIC_NR_PRIVATE_IRQS;		\
+		}							\
+									\
+		__ret;							\
+	})
+
+#define __irq_is_spi(t, i)						\
+	({								\
+		bool __ret;						\
+									\
+		switch (t) {						\
+		case KVM_DEV_TYPE_ARM_VGIC_V5:				\
+			__ret = is_v5_type(GICV5_HWIRQ_TYPE_SPI, (i));	\
+			break;						\
+		default:						\
+			__ret  = (i) <= VGIC_MAX_SPI;			\
+			__ret &= (i) >= VGIC_NR_PRIVATE_IRQS;		\
+		}							\
+									\
+		__ret;							\
+	})
+
+#define __irq_is_lpi(t, i)						\
+	({								\
+		bool __ret;						\
+									\
+		switch (t) {						\
+		case KVM_DEV_TYPE_ARM_VGIC_V5:				\
+			__ret = is_v5_type(GICV5_HWIRQ_TYPE_LPI, (i));	\
+			break;						\
+		default:						\
+			__ret  = (i) >= 8192;				\
+		}							\
+									\
+		__ret;							\
+	})
+
+#define irq_is_sgi(k, i) __irq_is_sgi((k)->arch.vgic.vgic_model, i)
+#define irq_is_ppi(k, i) __irq_is_ppi((k)->arch.vgic.vgic_model, i)
+#define irq_is_spi(k, i) __irq_is_spi((k)->arch.vgic.vgic_model, i)
+#define irq_is_lpi(k, i) __irq_is_lpi((k)->arch.vgic.vgic_model, i)
+
+#define irq_is_private(k, i) (irq_is_ppi(k, i) || irq_is_sgi(k, i))
+
+#define vgic_v5_get_hwirq_id(x) FIELD_GET(GICV5_HWIRQ_ID, (x))
+#define vgic_v5_set_hwirq_id(x) FIELD_PREP(GICV5_HWIRQ_ID, (x))
+
+#define __vgic_v5_set_type(t) (FIELD_PREP(GICV5_HWIRQ_TYPE, GICV5_HWIRQ_TYPE_##t))
+#define vgic_v5_make_ppi(x) (__vgic_v5_set_type(PPI) | vgic_v5_set_hwirq_id(x))
+#define vgic_v5_make_spi(x) (__vgic_v5_set_type(SPI) | vgic_v5_set_hwirq_id(x))
+#define vgic_v5_make_lpi(x) (__vgic_v5_set_type(LPI) | vgic_v5_set_hwirq_id(x))
+
+#define __vgic_is_v(k, v) ((k)->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V##v)
+#define vgic_is_v3(k) (__vgic_is_v(k, 3))
+#define vgic_is_v5(k) (__vgic_is_v(k, 5))
 
 enum vgic_type {
 	VGIC_V2,		/* Good ol' GICv2 */
@@ -101,6 +190,8 @@ enum vgic_irq_config {
 	VGIC_CONFIG_LEVEL
 };
 
+struct vgic_irq;
+
 /*
  * Per-irq ops overriding some common behavious.
  *
@@ -119,6 +210,19 @@ struct irq_ops {
 	 * peaking into the physical GIC.
 	 */
 	bool (*get_input_level)(int vintid);
+
+	/*
+	 * Function pointer to override the queuing of an IRQ.
+	 */
+	bool (*queue_irq_unlock)(struct kvm *kvm, struct vgic_irq *irq,
+				unsigned long flags) __releases(&irq->irq_lock);
+
+	/*
+	 * Callback function pointer to either enable or disable direct
+	 * injection for a mapped interrupt.
+	 */
+	void (*set_direct_injection)(struct kvm_vcpu *vcpu,
+				     struct vgic_irq *irq, bool direct);
 };
 
 struct vgic_irq {
@@ -238,6 +342,26 @@ struct vgic_redist_region {
 	struct list_head list;
 };
 
+struct vgic_v5_vm {
+	/*
+	 * We only expose a subset of PPIs to the guest. This subset is a
+	 * combination of the PPIs that are actually implemented and what we
+	 * actually choose to expose.
+	 */
+	DECLARE_BITMAP(vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+
+	/* A mask of the PPIs that are exposed for userspace to drive. */
+	DECLARE_BITMAP(userspace_ppis, VGIC_V5_NR_PRIVATE_IRQS);
+
+	/*
+	 * The HMR itself is handled by the hardware, but we still need to have
+	 * a mask that we can use when merging in pending state (only the state
+	 * of Edge PPIs is merged back in from the guest an the HMR provides a
+	 * convenient way to do that).
+	 */
+	DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS);
+};
+
 struct vgic_dist {
 	bool			in_kernel;
 	bool			ready;
@@ -310,6 +434,11 @@ struct vgic_dist {
 	 * else.
 	 */
 	struct its_vm		its_vm;
+
+	/*
+	 * GICv5 per-VM data.
+	 */
+	struct vgic_v5_vm	gicv5_vm;
 };
 
 struct vgic_v2_cpu_if {
@@ -340,11 +469,40 @@ struct vgic_v3_cpu_if {
 	unsigned int used_lrs;
 };
 
+struct vgic_v5_cpu_if {
+	u64	vgic_apr;
+	u64	vgic_vmcr;
+
+	/* PPI register state */
+	DECLARE_BITMAP(vgic_ppi_dvir, VGIC_V5_NR_PRIVATE_IRQS);
+	DECLARE_BITMAP(vgic_ppi_activer, VGIC_V5_NR_PRIVATE_IRQS);
+	DECLARE_BITMAP(vgic_ppi_enabler, VGIC_V5_NR_PRIVATE_IRQS);
+	/* We have one byte (of which 5 bits are used) per PPI for priority */
+	u64	vgic_ppi_priorityr[VGIC_V5_NR_PRIVATE_IRQS / 8];
+
+	/*
+	 * The ICSR is re-used across host and guest, and hence it needs to be
+	 * saved/restored. Only one copy is required as the host should block
+	 * preemption between executing GIC CDRCFG and acccessing the
+	 * ICC_ICSR_EL1. A guest, of course, can never guarantee this, and hence
+	 * it is the hyp's responsibility to keep the state constistent.
+	 */
+	u64	vgic_icsr;
+
+	struct gicv5_vpe gicv5_vpe;
+};
+
+/* What PPI capabilities does a GICv5 host have */
+struct vgic_v5_ppi_caps {
+	DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+};
+
 struct vgic_cpu {
 	/* CPU vif control registers for world switch */
 	union {
 		struct vgic_v2_cpu_if	vgic_v2;
 		struct vgic_v3_cpu_if	vgic_v3;
+		struct vgic_v5_cpu_if	vgic_v5;
 	};
 
 	struct vgic_irq *private_irqs;
@@ -392,13 +550,17 @@ int kvm_vgic_create(struct kvm *kvm, u32 type);
 void kvm_vgic_destroy(struct kvm *kvm);
 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
 int kvm_vgic_map_resources(struct kvm *kvm);
+void kvm_vgic_finalize_idregs(struct kvm *kvm);
 int kvm_vgic_hyp_init(void);
 void kvm_vgic_init_cpu_hardware(void);
 
 int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 			unsigned int intid, bool level, void *owner);
+void kvm_vgic_set_irq_ops(struct kvm_vcpu *vcpu, u32 vintid,
+			  struct irq_ops *ops);
+void kvm_vgic_clear_irq_ops(struct kvm_vcpu *vcpu, u32 vintid);
 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
-			  u32 vintid, struct irq_ops *ops);
+			  u32 vintid);
 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid);
 int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid);
 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid);
@@ -414,8 +576,20 @@ u64 vgic_v3_get_misr(struct kvm_vcpu *vcpu);
 
 #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
 #define vgic_initialized(k)	((k)->arch.vgic.initialized)
-#define vgic_valid_spi(k, i)	(((i) >= VGIC_NR_PRIVATE_IRQS) && \
-			((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
+#define vgic_valid_spi(k, i)						\
+	({								\
+		bool __ret = irq_is_spi(k, i);				\
+									\
+		switch ((k)->arch.vgic.vgic_model) {			\
+		case KVM_DEV_TYPE_ARM_VGIC_V5:				\
+			__ret &= FIELD_GET(GICV5_HWIRQ_ID, i) < (k)->arch.vgic.nr_spis; \
+			break;						\
+		default:						\
+			__ret &= (i) < ((k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS); \
+		}							\
+									\
+		__ret;							\
+	})
 
 bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
@@ -455,6 +629,11 @@ int vgic_v4_load(struct kvm_vcpu *vcpu);
 void vgic_v4_commit(struct kvm_vcpu *vcpu);
 int vgic_v4_put(struct kvm_vcpu *vcpu);
 
+int vgic_v5_finalize_ppi_state(struct kvm *kvm);
+bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
+				  unsigned long flags);
+void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi);
+
 bool vgic_state_is_nested(struct kvm_vcpu *vcpu);
 
 /* CPU HP callbacks */
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 4d2f0bed7a06d..5ecdcdaf31aa3 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -791,6 +791,14 @@ const char *acpi_get_subsystem_id(acpi_handle handle);
 int acpi_mrrm_max_mem_region(void);
 #endif
 
+#define ACPI_CMOS_RTC_IDS	\
+	{ "PNP0B00", },		\
+	{ "PNP0B01", },		\
+	{ "PNP0B02", },		\
+	{ "", }
+
+extern bool cmos_rtc_platform_device_present;
+
 #else	/* !CONFIG_ACPI */
 
 #define acpi_disabled 1
@@ -1116,6 +1124,8 @@ static inline int acpi_mrrm_max_mem_region(void)
 	return 1;
 }
 
+#define cmos_rtc_platform_device_present	false
+
 #endif	/* !CONFIG_ACPI */
 
 #ifdef CONFIG_ACPI_HMAT
diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h
index 70807c679f1ab..82a32526df645 100644
--- a/include/linux/atmdev.h
+++ b/include/linux/atmdev.h
@@ -309,17 +309,19 @@ struct atm_ioctl {
 
 /**
  * register_atm_ioctl - register handler for ioctl operations
+ * @ioctl: ioctl handler to register
  *
  * Special (non-device) handlers of ioctl's should
  * register here. If you're a normal device, you should
  * set .ioctl in your atmdev_ops instead.
  */
-void register_atm_ioctl(struct atm_ioctl *);
+void register_atm_ioctl(struct atm_ioctl *ioctl);
 
 /**
  * deregister_atm_ioctl - remove the ioctl handler
+ * @ioctl: ioctl handler to deregister
  */
-void deregister_atm_ioctl(struct atm_ioctl *);
+void deregister_atm_ioctl(struct atm_ioctl *ioctl);
 
 
 /* register_atmdevice_notifier - register atm_dev notify events
diff --git a/include/linux/audit.h b/include/linux/audit.h
index b642b5faca654..803b0183d98dd 100644
--- a/include/linux/audit.h
+++ b/include/linux/audit.h
@@ -15,7 +15,16 @@
 #include <uapi/linux/audit.h>
 #include <uapi/linux/fanotify.h>
 
-#define AUDIT_INO_UNSET ((unsigned long)-1)
+#define AUDIT_STATUS_ALL (AUDIT_STATUS_ENABLED | \
+			  AUDIT_STATUS_FAILURE | \
+			  AUDIT_STATUS_PID | \
+			  AUDIT_STATUS_RATE_LIMIT | \
+			  AUDIT_STATUS_BACKLOG_LIMIT | \
+			  AUDIT_STATUS_BACKLOG_WAIT_TIME | \
+			  AUDIT_STATUS_LOST | \
+			  AUDIT_STATUS_BACKLOG_WAIT_TIME_ACTUAL)
+
+#define AUDIT_INO_UNSET ((u64)-1)
 #define AUDIT_DEV_UNSET ((dev_t)-1)
 
 struct audit_sig_info {
diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h
index 4086afd0cc6b9..bc09b55e36822 100644
--- a/include/linux/auxiliary_bus.h
+++ b/include/linux/auxiliary_bus.h
@@ -271,6 +271,8 @@ struct auxiliary_device *__devm_auxiliary_device_create(struct device *dev,
 	__devm_auxiliary_device_create(dev, KBUILD_MODNAME, devname,  \
 				       platform_data, 0)
 
+bool dev_is_auxiliary(struct device *dev);
+
 /**
  * module_auxiliary_driver() - Helper macro for registering an auxiliary driver
  * @__auxiliary_driver: auxiliary driver struct
diff --git a/include/linux/auxvec.h b/include/linux/auxvec.h
index 407f7005e6d60..8bcb9b7262628 100644
--- a/include/linux/auxvec.h
+++ b/include/linux/auxvec.h
@@ -4,6 +4,6 @@
 
 #include <uapi/linux/auxvec.h>
 
-#define AT_VECTOR_SIZE_BASE 22 /* NEW_AUX_ENT entries in auxiliary table */
+#define AT_VECTOR_SIZE_BASE 24 /* NEW_AUX_ENT entries in auxiliary table */
   /* number of "#define AT_.*" above, minus {AT_NULL, AT_IGNORE, AT_NOTELF} */
 #endif /* _LINUX_AUXVEC_H */
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 0c8342747cab1..5b7d12b40d5e7 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -136,6 +136,19 @@ static inline bool mapping_can_writeback(struct address_space *mapping)
 	return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK;
 }
 
+/* Must not be used by file systems that support cgroup writeback */
+static inline int bdi_wb_dirty_exceeded(struct backing_dev_info *bdi)
+{
+	return bdi->wb.dirty_exceeded;
+}
+
+/* Must not be used by file systems that support cgroup writeback */
+static inline void bdi_wb_stat_mod(struct inode *inode, enum wb_stat_item item,
+				   s64 amount)
+{
+	wb_stat_mod(&inode_to_bdi(inode)->wb, item, amount);
+}
+
 #ifdef CONFIG_CGROUP_WRITEBACK
 
 struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi,
diff --git a/include/linux/backing-file.h b/include/linux/backing-file.h
index 1476a6ed1bfd7..8afba93f3ce07 100644
--- a/include/linux/backing-file.h
+++ b/include/linux/backing-file.h
@@ -9,19 +9,42 @@
 #define _LINUX_BACKING_FILE_H
 
 #include <linux/file.h>
-#include <linux/uio.h>
 #include <linux/fs.h>
 
+/*
+ * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file
+ * stored in ->vm_file is a backing file whose f_inode is on the underlying
+ * filesystem.
+ *
+ * LSM can use file_user_path_file() to store context related to the user path
+ * that was opened and mmaped.
+ */
+const struct file *backing_file_user_path_file(const struct file *f);
+
+static inline const struct file *file_user_path_file(const struct file *f)
+{
+	if (f && unlikely(f->f_mode & FMODE_BACKING))
+		return backing_file_user_path_file(f);
+	return f;
+}
+
+static inline const struct cred *file_user_cred(const struct file *f)
+{
+	return file_user_path_file(f)->f_cred;
+}
+
 struct backing_file_ctx {
 	const struct cred *cred;
 	void (*accessed)(struct file *file);
 	void (*end_write)(struct kiocb *iocb, ssize_t);
 };
 
-struct file *backing_file_open(const struct path *user_path, int flags,
+struct file *backing_file_open(const struct path *user_path,
+			       const struct cred *user_cred, int flags,
 			       const struct path *real_path,
 			       const struct cred *cred);
-struct file *backing_tmpfile_open(const struct path *user_path, int flags,
+struct file *backing_tmpfile_open(const struct path *user_path,
+				  const struct cred *user_cred, int flags,
 				  const struct path *real_parentpath,
 				  umode_t mode, const struct cred *cred);
 ssize_t backing_file_read_iter(struct file *file, struct iov_iter *iter,
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 65abd5ab8836c..94a3dcf9b1d25 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -2,6 +2,7 @@
 #ifndef _LINUX_BINFMTS_H
 #define _LINUX_BINFMTS_H
 
+#include <linux/elf.h>
 #include <linux/sched.h>
 #include <linux/unistd.h>
 #include <asm/exec.h>
@@ -67,6 +68,16 @@ struct linux_binprm {
 	unsigned long exec;
 
 	struct rlimit rlim_stack; /* Saved RLIMIT_STACK used during exec. */
+	unsigned long hwcap;
+#ifdef ELF_HWCAP2
+	unsigned long hwcap2;
+#endif
+#ifdef ELF_HWCAP3
+	unsigned long hwcap3;
+#endif
+#ifdef ELF_HWCAP4
+	unsigned long hwcap4;
+#endif
 
 	char buf[BINPRM_BUF_SIZE];
 } __randomize_layout;
diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h
index 21e4652dcfd2e..af5178434ec61 100644
--- a/include/linux/bio-integrity.h
+++ b/include/linux/bio-integrity.h
@@ -78,7 +78,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len,
 int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter);
 int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta);
 void bio_integrity_unmap_user(struct bio *bio);
-bool bio_integrity_prep(struct bio *bio);
+void bio_integrity_prep(struct bio *bio, unsigned int action);
 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done);
 void bio_integrity_trim(struct bio *bio);
 int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask);
@@ -104,9 +104,8 @@ static inline void bio_integrity_unmap_user(struct bio *bio)
 {
 }
 
-static inline bool bio_integrity_prep(struct bio *bio)
+static inline void bio_integrity_prep(struct bio *bio, unsigned int action)
 {
-	return true;
 }
 
 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
@@ -144,5 +143,12 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
 
 void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer);
 void bio_integrity_free_buf(struct bio_integrity_payload *bip);
+void bio_integrity_setup_default(struct bio *bio);
+
+unsigned int fs_bio_integrity_alloc(struct bio *bio);
+void fs_bio_integrity_free(struct bio *bio);
+void fs_bio_integrity_generate(struct bio *bio);
+int fs_bio_integrity_verify(struct bio *bio, sector_t sector,
+		unsigned int size);
 
 #endif /* _LINUX_BIO_INTEGRITY_H */
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 36a3f2275ecd0..d34e041002be1 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio)
 	return page_folio(bio_first_page_all(bio));
 }
 
+static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
+{
+	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+	return &bio->bi_io_vec[bio->bi_vcnt - 1];
+}
+
 /**
  * struct folio_iter - State for iterating all folios in a bio.
  * @folio: The current folio we're iterating.  NULL after the last folio.
@@ -350,8 +356,7 @@ extern void bioset_exit(struct bio_set *);
 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
 
 struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
-			     blk_opf_t opf, gfp_t gfp_mask,
-			     struct bio_set *bs);
+			     blk_opf_t opf, gfp_t gfp, struct bio_set *bs);
 struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
 extern void bio_put(struct bio *);
 
@@ -474,7 +479,7 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty);
 extern void bio_set_pages_dirty(struct bio *bio);
 extern void bio_check_pages_dirty(struct bio *bio);
 
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter);
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen);
 void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
 
 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index b0395e4ccf903..af2a3962255c0 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -57,6 +57,7 @@ struct device;
  *  bitmap_weight(src, nbits)                   Hamming Weight: number set bits
  *  bitmap_weight_and(src1, src2, nbits)        Hamming Weight of and'ed bitmap
  *  bitmap_weight_andnot(src1, src2, nbits)     Hamming Weight of andnot'ed bitmap
+ *  bitmap_weight_from(src, start, end)         Hamming Weight starting from @start
  *  bitmap_set(dst, pos, nbits)                 Set specified bit area
  *  bitmap_clear(dst, pos, nbits)               Clear specified bit area
  *  bitmap_find_next_zero_area(buf, len, pos, n, mask)  Find bit free area
@@ -479,6 +480,38 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1,
 	return __bitmap_weight_andnot(src1, src2, nbits);
 }
 
+/**
+ * bitmap_weight_from - Hamming weight for a memory region
+ * @bitmap: The base address
+ * @start: The bitnumber to starts weighting
+ * @end: the bitmap size in bits
+ *
+ * Returns the number of set bits in the region. If @start >= @end,
+ * the result is undefined.
+ */
+static __always_inline
+unsigned long bitmap_weight_from(const unsigned long *bitmap,
+				   unsigned int start, unsigned int end)
+{
+	unsigned long w;
+
+	if (unlikely(start >= end))
+		return end;
+
+	if (small_const_nbits(end))
+		return hweight_long(*bitmap & GENMASK(end - 1, start));
+
+	bitmap += start / BITS_PER_LONG;
+	/* Opencode round_down() to not include math.h */
+	end -= start & ~(BITS_PER_LONG - 1);
+	start %= BITS_PER_LONG;
+	w = bitmap_weight(bitmap, end);
+	if (start)
+		w -= hweight_long(*bitmap & BITMAP_LAST_WORD_MASK(start));
+
+	return w;
+}
+
 static __always_inline
 void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
 {
diff --git a/include/linux/bitops.h b/include/linux/bitops.h
index ea7898cc59039..1fe46703792f6 100644
--- a/include/linux/bitops.h
+++ b/include/linux/bitops.h
@@ -230,7 +230,7 @@ static inline int get_count_order_long(unsigned long l)
 
 /**
  * parity8 - get the parity of an u8 value
- * @value: the value to be examined
+ * @val: the value to be examined
  *
  * Determine the parity of the u8 argument.
  *
diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h
index c15b1ac627650..b1b530613c342 100644
--- a/include/linux/blk-integrity.h
+++ b/include/linux/blk-integrity.h
@@ -8,17 +8,13 @@
 
 struct request;
 
-/*
- * Maximum contiguous integrity buffer allocation.
- */
-#define BLK_INTEGRITY_MAX_SIZE		SZ_2M
-
 enum blk_integrity_flags {
 	BLK_INTEGRITY_NOVERIFY		= 1 << 0,
 	BLK_INTEGRITY_NOGENERATE	= 1 << 1,
 	BLK_INTEGRITY_DEVICE_CAPABLE	= 1 << 2,
 	BLK_INTEGRITY_REF_TAG		= 1 << 3,
 	BLK_INTEGRITY_STACKED		= 1 << 4,
+	BLK_SPLIT_INTERVAL_CAPABLE	= 1 << 5,
 };
 
 const char *blk_integrity_profile_name(struct blk_integrity *bi);
@@ -180,4 +176,27 @@ static inline struct bio_vec rq_integrity_vec(struct request *rq)
 }
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
+enum bio_integrity_action {
+	BI_ACT_BUFFER		= (1u << 0),	/* allocate buffer */
+	BI_ACT_CHECK		= (1u << 1),	/* generate / verify PI */
+	BI_ACT_ZERO		= (1u << 2),	/* zero buffer */
+};
+
+/**
+ * bio_integrity_action - return the integrity action needed for a bio
+ * @bio:	bio to operate on
+ *
+ * Returns the mask of integrity actions (BI_ACT_*) that need to be performed
+ * for @bio.
+ */
+unsigned int __bio_integrity_action(struct bio *bio);
+static inline unsigned int bio_integrity_action(struct bio *bio)
+{
+	if (!blk_get_integrity(bio->bi_bdev->bd_disk))
+		return 0;
+	if (bio_integrity(bio))
+		return 0;
+	return __bio_integrity_action(bio);
+}
+
 #endif /* _LINUX_BLK_INTEGRITY_H */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d463b9b5a0a59..b8e7f42aee719 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -13,6 +13,7 @@
 #include <linux/minmax.h>
 #include <linux/timer.h>
 #include <linux/workqueue.h>
+#include <linux/completion.h>
 #include <linux/wait.h>
 #include <linux/bio.h>
 #include <linux/gfp.h>
@@ -200,10 +201,14 @@ struct gendisk {
 	u8 __rcu		*zones_cond;
 	unsigned int		zone_wplugs_hash_bits;
 	atomic_t		nr_zone_wplugs;
-	spinlock_t		zone_wplugs_lock;
+	spinlock_t		zone_wplugs_hash_lock;
 	struct mempool		*zone_wplugs_pool;
 	struct hlist_head	*zone_wplugs_hash;
 	struct workqueue_struct *zone_wplugs_wq;
+	spinlock_t		zone_wplugs_list_lock;
+	struct list_head	zone_wplugs_list;
+	struct task_struct	*zone_wplugs_worker;
+	struct completion	zone_wplugs_worker_bio_done;
 #endif /* CONFIG_BLK_DEV_ZONED */
 
 #if IS_ENABLED(CONFIG_CDROM)
@@ -502,7 +507,7 @@ struct request_queue {
 
 	/* hw dispatch queues */
 	unsigned int		nr_hw_queues;
-	struct blk_mq_hw_ctx * __rcu *queue_hw_ctx;
+	struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues);
 
 	struct percpu_ref	q_usage_counter;
 	struct lock_class_key	io_lock_cls_key;
@@ -668,6 +673,7 @@ enum {
 	QUEUE_FLAG_NO_ELV_SWITCH,	/* can't switch elevator any more */
 	QUEUE_FLAG_QOS_ENABLED,		/* qos is enabled */
 	QUEUE_FLAG_BIO_ISSUE_TIME,	/* record bio->issue_time_ns */
+	QUEUE_FLAG_ZONED_QD1_WRITES,	/* Limit zoned devices writes to QD=1 */
 	QUEUE_FLAG_MAX
 };
 
@@ -707,6 +713,8 @@ void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
 	test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags)
 #define blk_queue_no_elv_switch(q)	\
 	test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags)
+#define blk_queue_zoned_qd1_writes(q)	\
+	test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags)
 
 extern void blk_set_pm_only(struct request_queue *q);
 extern void blk_clear_pm_only(struct request_queue *q);
@@ -1467,24 +1475,23 @@ static inline bool bdev_rot(struct block_device *bdev)
 	return blk_queue_rot(bdev_get_queue(bdev));
 }
 
-static inline bool bdev_nonrot(struct block_device *bdev)
+static inline bool bdev_synchronous(struct block_device *bdev)
 {
-	return !bdev_rot(bdev);
+	return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
 }
 
-static inline bool bdev_synchronous(struct block_device *bdev)
+static inline bool bdev_has_integrity_csum(struct block_device *bdev)
 {
-	return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
+	struct queue_limits *lim = bdev_limits(bdev);
+
+	return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
+		lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE;
 }
 
 static inline bool bdev_stable_writes(struct block_device *bdev)
 {
-	struct request_queue *q = bdev_get_queue(bdev);
-
-	if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
-	    q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE)
-		return true;
-	return q->limits.features & BLK_FEAT_STABLE_WRITES;
+	return bdev_has_integrity_csum(bdev) ||
+		(bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES);
 }
 
 static inline bool blk_queue_write_cache(struct request_queue *q)
@@ -1877,6 +1884,24 @@ static inline int bio_split_rw_at(struct bio *bio,
 	return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
 }
 
+/*
+ * Maximum contiguous integrity buffer allocation.
+ */
+#define BLK_INTEGRITY_MAX_SIZE		SZ_2M
+
+/*
+ * Maximum size of I/O that needs a block layer integrity buffer.  Limited
+ * by the number of intervals for which we can fit the integrity buffer into
+ * the buffer size.  Because the buffer is a single segment it is also limited
+ * by the maximum segment size.
+ */
+static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
+{
+	return min_t(unsigned int, lim->max_segment_size,
+		(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
+			lim->integrity.interval_exp);
+}
+
 #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
 
 #endif /* _LINUX_BLKDEV_H */
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ef8e45a362d96..090aa26d1c984 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -38,10 +38,9 @@ struct bpf_reg_state {
 	/* Ordering of fields matters.  See states_equal() */
 	enum bpf_reg_type type;
 	/*
-	 * Fixed part of pointer offset, pointer types only.
-	 * Or constant delta between "linked" scalars with the same ID.
+	 * Constant delta between "linked" scalars with the same ID.
 	 */
-	s32 off;
+	s32 delta;
 	union {
 		/* valid when type == PTR_TO_PACKET */
 		int range;
@@ -146,9 +145,9 @@ struct bpf_reg_state {
 	 * Upper bit of ID is used to remember relationship between "linked"
 	 * registers. Example:
 	 * r1 = r2;    both will have r1->id == r2->id == N
-	 * r1 += 10;   r1->id == N | BPF_ADD_CONST and r1->off == 10
+	 * r1 += 10;   r1->id == N | BPF_ADD_CONST and r1->delta == 10
 	 * r3 = r2;    both will have r3->id == r2->id == N
-	 * w3 += 10;   r3->id == N | BPF_ADD_CONST32 and r3->off == 10
+	 * w3 += 10;   r3->id == N | BPF_ADD_CONST32 and r3->delta == 10
 	 */
 #define BPF_ADD_CONST64 (1U << 31)
 #define BPF_ADD_CONST32 (1U << 30)
@@ -266,6 +265,7 @@ struct bpf_reference_state {
 struct bpf_retval_range {
 	s32 minval;
 	s32 maxval;
+	bool return_32bit;
 };
 
 /* state of the program:
diff --git a/include/linux/bsg.h b/include/linux/bsg.h
index ee2df73edf83f..162730bfc2d8d 100644
--- a/include/linux/bsg.h
+++ b/include/linux/bsg.h
@@ -7,13 +7,17 @@
 struct bsg_device;
 struct device;
 struct request_queue;
+struct io_uring_cmd;
 
 typedef int (bsg_sg_io_fn)(struct request_queue *, struct sg_io_v4 *hdr,
 		bool open_for_write, unsigned int timeout);
 
+typedef int (bsg_uring_cmd_fn)(struct request_queue *q, struct io_uring_cmd *ioucmd,
+			       unsigned int issue_flags, bool open_for_write);
+
 struct bsg_device *bsg_register_queue(struct request_queue *q,
 		struct device *parent, const char *name,
-		bsg_sg_io_fn *sg_io_fn);
+		bsg_sg_io_fn *sg_io_fn, bsg_uring_cmd_fn *uring_cmd_fn);
 void bsg_unregister_queue(struct bsg_device *bcd);
 
 #endif /* _LINUX_BSG_H */
diff --git a/include/linux/build_bug.h b/include/linux/build_bug.h
index 2cfbb4c65c784..d3dc5dc5f916f 100644
--- a/include/linux/build_bug.h
+++ b/include/linux/build_bug.h
@@ -32,7 +32,8 @@
 /**
  * BUILD_BUG_ON_MSG - break compile if a condition is true & emit supplied
  *		      error message.
- * @condition: the condition which the compiler should know is false.
+ * @cond: the condition which the compiler should know is false.
+ * @msg: build-time error message
  *
  * See BUILD_BUG_ON for description.
  */
@@ -60,6 +61,7 @@
 
 /**
  * static_assert - check integer constant expression at build time
+ * @expr: expression to be checked
  *
  * static_assert() is a wrapper for the C11 _Static_assert, with a
  * little macro magic to make the message optional (defaulting to the
diff --git a/include/linux/bus/stm32_firewall.h b/include/linux/bus/stm32_firewall.h
new file mode 100644
index 0000000000000..e5fac85fe3460
--- /dev/null
+++ b/include/linux/bus/stm32_firewall.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023, STMicroelectronics - All Rights Reserved
+ */
+
+#ifndef _STM32_FIREWALL_H
+#define _STM32_FIREWALL_H
+
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+/**
+ * STM32_PERIPHERAL_FIREWALL:		This type of firewall protects peripherals
+ * STM32_MEMORY_FIREWALL:		This type of firewall protects memories/subsets of memory
+ *					zones
+ * STM32_NOTYPE_FIREWALL:		Undefined firewall type
+ */
+
+#define STM32_PERIPHERAL_FIREWALL	BIT(1)
+#define STM32_MEMORY_FIREWALL		BIT(2)
+#define STM32_NOTYPE_FIREWALL		BIT(3)
+
+/**
+ * struct stm32_firewall_controller - Information on firewall controller supplying services
+ *
+ * @name:			Name of the firewall controller
+ * @dev:			Device reference of the firewall controller
+ * @mmio:			Base address of the firewall controller
+ * @entry:			List entry of the firewall controller list
+ * @type:			Type of firewall
+ * @max_entries:		Number of entries covered by the firewall
+ * @grant_access:		Callback used to grant access for a device access against a
+ *				firewall controller
+ * @release_access:		Callback used to release resources taken by a device when access was
+ *				granted
+ * @grant_memory_range_access:	Callback used to grant access for a device to a given memory region
+ */
+struct stm32_firewall_controller {
+	const char *name;
+	struct device *dev;
+	void __iomem *mmio;
+	struct list_head entry;
+	unsigned int type;
+	unsigned int max_entries;
+
+	int (*grant_access)(struct stm32_firewall_controller *ctrl, u32 id);
+	void (*release_access)(struct stm32_firewall_controller *ctrl, u32 id);
+	int (*grant_memory_range_access)(struct stm32_firewall_controller *ctrl, phys_addr_t paddr,
+					 size_t size);
+};
+
+/**
+ * stm32_firewall_controller_register - Register a firewall controller to the STM32 firewall
+ *					framework
+ * @firewall_controller:	Firewall controller to register
+ *
+ * Returns 0 in case of success or -ENODEV if no controller was given.
+ */
+int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller);
+
+/**
+ * stm32_firewall_controller_unregister - Unregister a firewall controller from the STM32
+ *					  firewall framework
+ * @firewall_controller:	Firewall controller to unregister
+ */
+void stm32_firewall_controller_unregister(struct stm32_firewall_controller *firewall_controller);
+
+/**
+ * stm32_firewall_populate_bus - Populate device tree nodes that have a correct firewall
+ *				 configuration. This is used at boot-time only, as a sanity check
+ *				 between device tree and firewalls hardware configurations to
+ *				 prevent a kernel crash when a device driver is not granted access
+ *
+ * @firewall_controller:	Firewall controller which nodes will be populated or not
+ *
+ * Returns 0 in case of success or appropriate errno code if error occurred.
+ */
+int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller);
+
+#endif /* _STM32_FIREWALL_H */
diff --git a/include/linux/bus/stm32_firewall_device.h b/include/linux/bus/stm32_firewall_device.h
index eaa7a3f544507..6c878f3ca86f2 100644
--- a/include/linux/bus/stm32_firewall_device.h
+++ b/include/linux/bus/stm32_firewall_device.h
@@ -112,6 +112,25 @@ int stm32_firewall_grant_access_by_id(struct stm32_firewall *firewall, u32 subsy
  */
 void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id);
 
+/**
+ * stm32_firewall_get_grant_all_access - Allocate and get all the firewall(s) associated to given
+ *					 device. Then, try to grant access rights for each element.
+ *					 This function is basically a helper function that wraps
+ *					 both stm32_firewall_get_firewall() and
+ *					 stm32_firewall_grant_access() on all firewall references of
+ *					 a device along with the allocation of the array.
+ *					 Realease access using stm32_firewall_release_access* APIs
+ *					 when done.
+ *
+ * @dev:			Device performing the checks
+ * @firewall:			Pointer to the array of firewall references to be allocated
+ * @nb_firewall:		Number of allocated elements in @firewall
+ *
+ * Returns 0 on success, or appropriate errno code if error occurred.
+ */
+int stm32_firewall_get_grant_all_access(struct device *dev, struct stm32_firewall **firewall,
+					int *nb_firewall);
+
 #else /* CONFIG_STM32_FIREWALL */
 
 static inline int stm32_firewall_get_firewall(struct device_node *np,
@@ -141,5 +160,12 @@ static inline void stm32_firewall_release_access_by_id(struct stm32_firewall *fi
 {
 }
 
+static inline int stm32_firewall_get_grant_all_access(struct device *dev,
+						      struct stm32_firewall **firewall,
+						      int *nb_firewall)
+{
+	return -ENODEV;
+}
+
 #endif /* CONFIG_STM32_FIREWALL */
 #endif /* STM32_FIREWALL_DEVICE_H */
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index bb92f5c169ca2..57ed0426608e8 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -17,6 +17,7 @@
 #include <linux/refcount.h>
 #include <linux/percpu-refcount.h>
 #include <linux/percpu-rwsem.h>
+#include <linux/sched.h>
 #include <linux/u64_stats_sync.h>
 #include <linux/workqueue.h>
 #include <linux/bpf-cgroup-defs.h>
@@ -167,6 +168,7 @@ struct cgroup_file {
 	struct kernfs_node *kn;
 	unsigned long notified_at;
 	struct timer_list notify_timer;
+	spinlock_t lock;
 };
 
 /*
@@ -624,6 +626,9 @@ struct cgroup {
 #ifdef CONFIG_BPF_SYSCALL
 	struct bpf_local_storage __rcu  *bpf_cgrp_storage;
 #endif
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct scx_sched __rcu *scx_sched;
+#endif
 
 	/* All ancestors including self */
 	union {
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index bc892e3b37eea..e52160e85af4b 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -42,6 +42,14 @@ struct kernel_clone_args;
 
 #ifdef CONFIG_CGROUPS
 
+/*
+ * To avoid confusing the compiler (and generating warnings) with code
+ * that attempts to access what would be a 0-element array (i.e. sized
+ * to a potentially empty array when CGROUP_SUBSYS_COUNT == 0), this
+ * constant expression can be added.
+ */
+#define CGROUP_HAS_SUBSYS_CONFIG	(CGROUP_SUBSYS_COUNT > 0)
+
 enum css_task_iter_flags {
 	CSS_TASK_ITER_PROCS    = (1U << 0),  /* walk only threadgroup leaders */
 	CSS_TASK_ITER_THREADED = (1U << 1),  /* walk all threaded css_sets in the domain */
@@ -76,6 +84,7 @@ enum cgroup_lifetime_events {
 extern struct file_system_type cgroup_fs_type;
 extern struct cgroup_root cgrp_dfl_root;
 extern struct css_set init_css_set;
+extern struct mutex cgroup_mutex;
 extern spinlock_t css_set_lock;
 extern struct blocking_notifier_head cgroup_lifetime_notifier;
 
@@ -103,6 +112,8 @@ extern struct blocking_notifier_head cgroup_lifetime_notifier;
 #define cgroup_subsys_on_dfl(ss)						\
 	static_branch_likely(&ss ## _on_dfl_key)
 
+bool cgroup_on_dfl(const struct cgroup *cgrp);
+
 bool css_has_online_children(struct cgroup_subsys_state *css);
 struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss);
 struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup,
@@ -274,6 +285,32 @@ void css_task_iter_end(struct css_task_iter *it);
 	for ((pos) = css_next_descendant_post(NULL, (css)); (pos);	\
 	     (pos) = css_next_descendant_post((pos), (css)))
 
+/* iterate over child cgrps, lock should be held throughout iteration */
+#define cgroup_for_each_live_child(child, cgrp)				\
+	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
+		if (({ lockdep_assert_held(&cgroup_mutex);		\
+		       cgroup_is_dead(child); }))			\
+			;						\
+		else
+
+/* walk live descendants in pre order */
+#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp)		\
+	css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL))	\
+		if (({ lockdep_assert_held(&cgroup_mutex);		\
+		       (dsct) = (d_css)->cgroup;			\
+		       cgroup_is_dead(dsct); }))			\
+			;						\
+		else
+
+/* walk live descendants in postorder */
+#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp)		\
+	css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL))	\
+		if (({ lockdep_assert_held(&cgroup_mutex);		\
+		       (dsct) = (d_css)->cgroup;			\
+		       cgroup_is_dead(dsct); }))			\
+			;						\
+		else
+
 /**
  * cgroup_taskset_for_each - iterate cgroup_taskset
  * @task: the loop cursor
@@ -336,6 +373,27 @@ static inline u64 cgroup_id(const struct cgroup *cgrp)
 	return cgrp->kn->id;
 }
 
+/**
+ * cgroup_css - obtain a cgroup's css for the specified subsystem
+ * @cgrp: the cgroup of interest
+ * @ss: the subsystem of interest (%NULL returns @cgrp->self)
+ *
+ * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
+ * function must be called either under cgroup_mutex or rcu_read_lock() and
+ * the caller is responsible for pinning the returned css if it wants to
+ * keep accessing it outside the said locks.  This function may return
+ * %NULL if @cgrp doesn't have @subsys_id enabled.
+ */
+static inline struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
+						     struct cgroup_subsys *ss)
+{
+	if (CGROUP_HAS_SUBSYS_CONFIG && ss)
+		return rcu_dereference_check(cgrp->subsys[ss->id],
+					lockdep_is_held(&cgroup_mutex));
+	else
+		return &cgrp->self;
+}
+
 /**
  * css_is_dying - test whether the specified css is dying
  * @css: target css
@@ -372,6 +430,11 @@ static inline bool css_is_self(struct cgroup_subsys_state *css)
 	return false;
 }
 
+static inline bool cgroup_is_dead(const struct cgroup *cgrp)
+{
+	return !(cgrp->self.flags & CSS_ONLINE);
+}
+
 static inline void cgroup_get(struct cgroup *cgrp)
 {
 	css_get(&cgrp->self);
@@ -387,8 +450,6 @@ static inline void cgroup_put(struct cgroup *cgrp)
 	css_put(&cgrp->self);
 }
 
-extern struct mutex cgroup_mutex;
-
 static inline void cgroup_lock(void)
 {
 	mutex_lock(&cgroup_mutex);
diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
index dbc4162921e9b..ea95ca4bc11c2 100644
--- a/include/linux/cleanup.h
+++ b/include/linux/cleanup.h
@@ -286,15 +286,18 @@ static __always_inline _type class_##_name##_constructor(_init_args)	\
 	__no_context_analysis						\
 { _type t = _init; return t; }
 
-#define EXTEND_CLASS(_name, ext, _init, _init_args...)			\
-typedef lock_##_name##_t lock_##_name##ext##_t;			\
+#define EXTEND_CLASS_COND(_name, ext, _cond, _init, _init_args...)	\
+typedef lock_##_name##_t lock_##_name##ext##_t;				\
 typedef class_##_name##_t class_##_name##ext##_t;			\
-static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *p) \
-{ class_##_name##_destructor(p); }					\
+static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *_T) \
+{ if (_cond) return; class_##_name##_destructor(_T); }			\
 static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
 	__no_context_analysis \
 { class_##_name##_t t = _init; return t; }
 
+#define EXTEND_CLASS(_name, ext, _init, _init_args...)			\
+	EXTEND_CLASS_COND(_name, ext, 0, _init, _init_args)
+
 #define CLASS(_name, var)						\
 	class_##_name##_t var __cleanup(class_##_name##_destructor) =	\
 		class_##_name##_constructor
@@ -394,12 +397,12 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond
 	__DEFINE_GUARD_LOCK_PTR(_name, _T)
 
 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \
-	DEFINE_CLASS(_name, _type, if (!__GUARD_IS_ERR(_T)) { _unlock; }, ({ _lock; _T; }), _type _T); \
+	DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
 	DEFINE_CLASS_IS_GUARD(_name)
 
 #define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \
 	__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \
-	EXTEND_CLASS(_name, _ext, \
+	EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(*_T), \
 		     ({ void *_t = _T; int _RET = (_lock); if (_T && !(_cond)) _t = ERR_PTR(_RET); _t; }), \
 		     class_##_name##_t _T) \
 	static __always_inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
@@ -488,7 +491,7 @@ typedef struct {							\
 static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \
 	__no_context_analysis						\
 {									\
-	if (!__GUARD_IS_ERR(_T->lock)) { _unlock; }			\
+	if (_T->lock) { _unlock; }					\
 }									\
 									\
 __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock)
@@ -565,7 +568,7 @@ __DEFINE_LOCK_GUARD_0(_name, _lock)
 
 #define DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _cond)		\
 	__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true);		\
-	EXTEND_CLASS(_name, _ext,					\
+	EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(_T->lock),	\
 		     ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\
 		        int _RET = (_lock);                             \
 		        if (_T->lock && !(_cond)) _T->lock = ERR_PTR(_RET);\
diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h
index b0df28ddd394b..92d90220c0d49 100644
--- a/include/linux/clockchips.h
+++ b/include/linux/clockchips.h
@@ -43,9 +43,9 @@ enum clock_event_state {
 /*
  * Clock event features
  */
-# define CLOCK_EVT_FEAT_PERIODIC	0x000001
-# define CLOCK_EVT_FEAT_ONESHOT		0x000002
-# define CLOCK_EVT_FEAT_KTIME		0x000004
+# define CLOCK_EVT_FEAT_PERIODIC		0x000001
+# define CLOCK_EVT_FEAT_ONESHOT			0x000002
+# define CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED	0x000004
 
 /*
  * x86(64) specific (mis)features:
@@ -101,6 +101,7 @@ struct clock_event_device {
 	void			(*event_handler)(struct clock_event_device *);
 	int			(*set_next_event)(unsigned long evt, struct clock_event_device *);
 	int			(*set_next_ktime)(ktime_t expires, struct clock_event_device *);
+	void			(*set_next_coupled)(u64 cycles, struct clock_event_device *);
 	ktime_t			next_event;
 	u64			max_delta_ns;
 	u64			min_delta_ns;
@@ -108,6 +109,7 @@ struct clock_event_device {
 	u32			shift;
 	enum clock_event_state	state_use_accessors;
 	unsigned int		features;
+	enum clocksource_ids	cs_id;
 	unsigned long		retries;
 
 	int			(*set_state_periodic)(struct clock_event_device *);
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index 65b7c41471c39..691d408fc0bb7 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -25,8 +25,7 @@ struct clocksource_base;
 struct clocksource;
 struct module;
 
-#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \
-    defined(CONFIG_GENERIC_GETTIMEOFDAY)
+#if defined(CONFIG_GENERIC_GETTIMEOFDAY)
 #include <asm/clocksource.h>
 #endif
 
@@ -106,9 +105,6 @@ struct clocksource {
 	u64			max_idle_ns;
 	u32			maxadj;
 	u32			uncertainty_margin;
-#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA
-	struct arch_clocksource_data archdata;
-#endif
 	u64			max_cycles;
 	u64			max_raw_delta;
 	const char		*name;
@@ -149,6 +145,9 @@ struct clocksource {
 #define CLOCK_SOURCE_SUSPEND_NONSTOP		0x80
 #define CLOCK_SOURCE_RESELECT			0x100
 #define CLOCK_SOURCE_VERIFY_PERCPU		0x200
+#define CLOCK_SOURCE_CAN_INLINE_READ		0x400
+#define CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT	0x800
+
 /* simplify initialization of mask field */
 #define CLOCKSOURCE_MASK(bits) GENMASK_ULL((bits) - 1, 0)
 
diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h
index 00c074a2ccb09..a9317571e6aff 100644
--- a/include/linux/compiler-context-analysis.h
+++ b/include/linux/compiler-context-analysis.h
@@ -320,6 +320,38 @@ static inline void _context_unsafe_alias(void **p) { }
  */
 #define __releases(...)		__releases_ctx_lock(__VA_ARGS__)
 
+/*
+ * Clang's analysis does not care precisely about the value, only that it is
+ * either zero or non-zero. So the __cond_acquires() interface might be
+ * misleading if we say that @ret is the value returned if acquired. Instead,
+ * provide symbolic variants which we translate.
+ */
+#define __cond_acquires_impl_not_true(x, ...)     __try_acquires##__VA_ARGS__##_ctx_lock(0, x)
+#define __cond_acquires_impl_not_false(x, ...)    __try_acquires##__VA_ARGS__##_ctx_lock(1, x)
+#define __cond_acquires_impl_not_nonzero(x, ...)  __try_acquires##__VA_ARGS__##_ctx_lock(0, x)
+#define __cond_acquires_impl_not_0(x, ...)        __try_acquires##__VA_ARGS__##_ctx_lock(1, x)
+#define __cond_acquires_impl_not_nonnull(x, ...)  __try_acquires##__VA_ARGS__##_ctx_lock(0, x)
+#define __cond_acquires_impl_not_NULL(x, ...)     __try_acquires##__VA_ARGS__##_ctx_lock(1, x)
+
+/**
+ * __cond_releases() - function attribute, function conditionally
+ *                     releases a context lock exclusively
+ * @ret: abstract value returned by function if context lock releases
+ * @x: context lock instance pointer
+ *
+ * Function attribute declaring that the function conditionally releases the
+ * given context lock instance @x exclusively. The associated context(s) must
+ * be active on entry. The function return value @ret denotes when the context
+ * lock is released.
+ *
+ * @ret may be one of: true, false, nonzero, 0, nonnull, NULL.
+ *
+ * NOTE: clang does not have a native attribute for this; instead implement
+ *       it as an unconditional release and a conditional acquire for the
+ *       inverted condition -- which is semantically equivalent.
+ */
+#define __cond_releases(ret, x) __releases(x) __cond_acquires_impl_not_##ret(x)
+
 /**
  * __acquire() - function to acquire context lock exclusively
  * @x: context lock instance pointer
diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h
index 13b35637bd5ae..6ce498b31855f 100644
--- a/include/linux/console_struct.h
+++ b/include/linux/console_struct.h
@@ -13,8 +13,9 @@
 #ifndef _LINUX_CONSOLE_STRUCT_H
 #define _LINUX_CONSOLE_STRUCT_H
 
-#include <linux/wait.h>
+#include <linux/math.h>
 #include <linux/vt.h>
+#include <linux/wait.h>
 #include <linux/workqueue.h>
 
 struct uni_pagedict;
@@ -58,6 +59,58 @@ struct vc_state {
 	bool		reverse;
 };
 
+/**
+ * struct vc_font - Describes a font
+ * @width: The width of a single glyph in bits
+ * @height: The height of a single glyph in scanlines
+ * @charcount: The number of glyphs in the font
+ * @data: The raw font data
+ *
+ * Font data is organized as an array of glyphs. Each glyph is a bitmap with
+ * set bits indicating the foreground color. Unset bits indicate background
+ * color. The fields @width and @height store a single glyph's number of
+ * horizontal bits and vertical scanlines. If width is not a multiple of 8,
+ * there are trailing bits to fill up the byte. These bits should not be drawn.
+ *
+ * The field @data points to the first glyph's first byte. The value @charcount
+ * gives the number of glyphs in the font. There are no empty scanlines between
+ * two adjacent glyphs.
+ */
+struct vc_font {
+	unsigned int width;
+	unsigned int height;
+	unsigned int charcount;
+	const unsigned char *data;
+};
+
+/**
+ * vc_font_pitch - Calculates the number of bytes between two adjacent scanlines
+ * @font: The VC font
+ *
+ * Returns:
+ * The number of bytes between two adjacent scanlines in the font data
+ */
+static inline unsigned int vc_font_pitch(const struct vc_font *font)
+{
+	return DIV_ROUND_UP(font->width, 8);
+}
+
+/**
+ * vc_font_size - Calculates the size of the font data in bytes
+ * @font: The VC font
+ *
+ * vc_font_size() calculates the number of bytes of font data in the
+ * font specified by @font. The function calculates the size from the
+ * font parameters.
+ *
+ * Returns:
+ * The size of the font data in bytes.
+ */
+static inline unsigned int vc_font_size(const struct vc_font *font)
+{
+	return font->height * vc_font_pitch(font) * font->charcount;
+}
+
 /*
  * Example: vc_data of a console that was scrolled 3 lines down.
  *
@@ -120,9 +173,9 @@ struct vc_data {
 	unsigned short	vc_complement_mask;	/* [#] Xor mask for mouse pointer */
 	unsigned short	vc_s_complement_mask;	/* Saved mouse pointer mask */
 	unsigned long	vc_pos;			/* Cursor address */
-	/* fonts */	
+	/* fonts */
 	unsigned short	vc_hi_font_mask;	/* [#] Attribute set for upper 256 chars of font or 0 if not supported */
-	struct console_font vc_font;		/* Current VC font set */
+	struct vc_font vc_font;			/* Current VC font set */
 	unsigned short	vc_video_erase_char;	/* Background erase character */
 	/* VT terminal data */
 	unsigned int	vc_state;		/* Escape sequence parser state */
@@ -160,6 +213,7 @@ struct vc_data {
 	struct uni_pagedict **uni_pagedict_loc; /* [!] Location of uni_pagedict variable for this console */
 	u32 **vc_uni_lines;			/* unicode screen content */
 	u16		*vc_saved_screen;
+	u32		**vc_saved_uni_lines;
 	unsigned int	vc_saved_cols;
 	unsigned int	vc_saved_rows;
 	/* additional information is in vt_kern.h */
diff --git a/include/linux/coreboot.h b/include/linux/coreboot.h
new file mode 100644
index 0000000000000..5d40ca7a1d890
--- /dev/null
+++ b/include/linux/coreboot.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * coreboot.h
+ *
+ * Coreboot device and driver interfaces.
+ *
+ * Copyright 2014 Gerd Hoffmann <kraxel@redhat.com>
+ * Copyright 2017 Google Inc.
+ * Copyright 2017 Samuel Holland <samuel@sholland.org>
+ */
+
+#ifndef _LINUX_COREBOOT_H
+#define _LINUX_COREBOOT_H
+
+#include <linux/compiler_attributes.h>
+#include <linux/stddef.h>
+#include <linux/types.h>
+
+typedef __aligned(4) u64 cb_u64;
+
+/* List of coreboot entry structures that is used */
+
+#define CB_TAG_FRAMEBUFFER 0x12
+#define LB_TAG_CBMEM_ENTRY 0x31
+
+/* Generic */
+struct coreboot_table_entry {
+	u32 tag;
+	u32 size;
+};
+
+/* Points to a CBMEM entry */
+struct lb_cbmem_ref {
+	u32 tag;
+	u32 size;
+
+	cb_u64 cbmem_addr;
+};
+
+/* Corresponds to LB_TAG_CBMEM_ENTRY */
+struct lb_cbmem_entry {
+	u32 tag;
+	u32 size;
+
+	cb_u64 address;
+	u32 entry_size;
+	u32 id;
+};
+
+#define LB_FRAMEBUFFER_ORIENTATION_NORMAL	0
+#define LB_FRAMEBUFFER_ORIENTATION_BOTTOM_UP	1
+#define LB_FRAMEBUFFER_ORIENTATION_LEFT_UP	2
+#define LB_FRAMEBUFFER_ORIENTATION_RIGHT_UP	3
+
+/* Describes framebuffer setup by coreboot */
+struct lb_framebuffer {
+	u32 tag;
+	u32 size;
+
+	cb_u64 physical_address;
+	u32 x_resolution;
+	u32 y_resolution;
+	u32 bytes_per_line;
+	u8  bits_per_pixel;
+	u8  red_mask_pos;
+	u8  red_mask_size;
+	u8  green_mask_pos;
+	u8  green_mask_size;
+	u8  blue_mask_pos;
+	u8  blue_mask_size;
+	u8  reserved_mask_pos;
+	u8  reserved_mask_size;
+	u8  orientation;
+};
+
+/*
+ * True if the coreboot-provided data is large enough to hold information
+ * on the linear framebuffer. False otherwise.
+ */
+#define LB_FRAMEBUFFER_HAS_LFB(__fb) \
+	((__fb)->size >= offsetofend(struct lb_framebuffer, reserved_mask_size))
+
+/*
+ * True if the coreboot-provided data is large enough to hold information
+ * on the display orientation. False otherwise.
+ */
+#define LB_FRAMEBUFFER_HAS_ORIENTATION(__fb) \
+	((__fb)->size >= offsetofend(struct lb_framebuffer, orientation))
+
+#endif /* _LINUX_COREBOOT_H */
diff --git a/include/linux/coresight.h b/include/linux/coresight.h
index 2b48be97fcd0d..2131febebee93 100644
--- a/include/linux/coresight.h
+++ b/include/linux/coresight.h
@@ -306,24 +306,19 @@ struct coresight_device {
  * coresight_dev_list - Mapping for devices to "name" index for device
  * names.
  *
+ * @node:		Node on the global device index list.
  * @nr_idx:		Number of entries already allocated.
  * @pfx:		Prefix pattern for device name.
  * @fwnode_list:	Array of fwnode_handles associated with each allocated
  *			index, upto nr_idx entries.
  */
 struct coresight_dev_list {
+	struct list_head	node;
 	int			nr_idx;
-	const char		*pfx;
+	char			*pfx;
 	struct fwnode_handle	**fwnode_list;
 };
 
-#define DEFINE_CORESIGHT_DEVLIST(var, dev_pfx)				\
-static struct coresight_dev_list (var) = {				\
-						.pfx = dev_pfx,		\
-						.nr_idx = 0,		\
-						.fwnode_list = NULL,	\
-}
-
 #define to_coresight_device(d) container_of(d, struct coresight_device, dev)
 
 /**
@@ -663,8 +658,7 @@ void coresight_clear_self_claim_tag(struct csdev_access *csa);
 void coresight_clear_self_claim_tag_unlocked(struct csdev_access *csa);
 void coresight_disclaim_device(struct coresight_device *csdev);
 void coresight_disclaim_device_unlocked(struct coresight_device *csdev);
-char *coresight_alloc_device_name(struct coresight_dev_list *devs,
-					 struct device *dev);
+char *coresight_alloc_device_name(const char *prefix, struct device *dev);
 
 bool coresight_loses_context_with_cpu(struct device *dev);
 
diff --git a/include/linux/count_zeros.h b/include/linux/count_zeros.h
index 5b8ff5ac660da..5034a30b5c7c7 100644
--- a/include/linux/count_zeros.h
+++ b/include/linux/count_zeros.h
@@ -10,6 +10,8 @@
 
 #include <asm/bitops.h>
 
+#define COUNT_TRAILING_ZEROS_0 (-1)
+
 /**
  * count_leading_zeros - Count the number of zeros from the MSB back
  * @x: The value
@@ -18,7 +20,7 @@
  *
  * If the MSB of @x is set, the result is 0.
  * If only the LSB of @x is set, then the result is BITS_PER_LONG-1.
- * If @x is 0 then the result is COUNT_LEADING_ZEROS_0.
+ * If @x is 0 then the result is BITS_PER_LONG.
  */
 static inline int count_leading_zeros(unsigned long x)
 {
@@ -28,8 +30,6 @@ static inline int count_leading_zeros(unsigned long x)
 		return BITS_PER_LONG - fls64(x);
 }
 
-#define COUNT_LEADING_ZEROS_0 BITS_PER_LONG
-
 /**
  * count_trailing_zeros - Count the number of zeros from the LSB forwards
  * @x: The value
@@ -42,12 +42,7 @@ static inline int count_leading_zeros(unsigned long x)
  */
 static inline int count_trailing_zeros(unsigned long x)
 {
-#define COUNT_TRAILING_ZEROS_0 (-1)
-
-	if (sizeof(x) == 4)
-		return ffs(x);
-	else
-		return (x != 0) ? __ffs(x) : COUNT_TRAILING_ZEROS_0;
+	return (x != 0) ? __ffs(x) : COUNT_TRAILING_ZEROS_0;
 }
 
 #endif /* _LINUX_BITOPS_COUNT_ZEROS_H_ */
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index cc894fc389710..8ca2bcb3d7aee 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -232,7 +232,7 @@ static inline bool policy_is_inactive(struct cpufreq_policy *policy)
 
 static inline bool policy_is_shared(struct cpufreq_policy *policy)
 {
-	return cpumask_weight(policy->cpus) > 1;
+	return cpumask_nth(1, policy->cpus) < nr_cpumask_bits;
 }
 
 #ifdef CONFIG_CPU_FREQ
diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
index d35726d6a415b..c1dee3f971a91 100644
--- a/include/linux/crash_core.h
+++ b/include/linux/crash_core.h
@@ -34,13 +34,6 @@ static inline void arch_kexec_protect_crashkres(void) { }
 static inline void arch_kexec_unprotect_crashkres(void) { }
 #endif
 
-#ifdef CONFIG_CRASH_DM_CRYPT
-int crash_load_dm_crypt_keys(struct kimage *image);
-ssize_t dm_crypt_keys_read(char *buf, size_t count, u64 *ppos);
-#else
-static inline int crash_load_dm_crypt_keys(struct kimage *image) {return 0; }
-#endif
-
 #ifndef arch_crash_handle_hotplug_event
 static inline void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) { }
 #endif
@@ -96,4 +89,11 @@ static inline void crash_save_cpu(struct pt_regs *regs, int cpu) {};
 static inline int kimage_crash_copy_vmcoreinfo(struct kimage *image) { return 0; };
 #endif /* CONFIG_CRASH_DUMP*/
 
+#ifdef CONFIG_CRASH_DM_CRYPT
+int crash_load_dm_crypt_keys(struct kimage *image);
+ssize_t dm_crypt_keys_read(char *buf, size_t count, u64 *ppos);
+#else
+static inline int crash_load_dm_crypt_keys(struct kimage *image) {return 0; }
+#endif
+
 #endif /* LINUX_CRASH_CORE_H */
diff --git a/include/linux/cred.h b/include/linux/cred.h
index ed1609d78cd7b..c6676265a9859 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -33,12 +33,14 @@ struct group_info {
 
 /**
  * get_group_info - Get a reference to a group info structure
- * @group_info: The group info to reference
+ * @gi: The group info to reference
  *
  * This gets a reference to a set of supplementary groups.
  *
  * If the caller is accessing a task's credentials, they must hold the RCU read
  * lock when reading.
+ *
+ * Returns: @gi
  */
 static inline struct group_info *get_group_info(struct group_info *gi)
 {
@@ -209,6 +211,8 @@ DEFINE_CLASS(override_creds,
  * usage count.  The purpose of this is to attempt to catch at compile time the
  * accidental alteration of a set of credentials that should be considered
  * immutable.
+ *
+ * Returns: @cred when the references are acquired, NULL otherwise.
  */
 static inline const struct cred *get_cred_many(const struct cred *cred, int nr)
 {
@@ -246,8 +250,8 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
 }
 
 /**
- * put_cred - Release a reference to a set of credentials
- * @cred: The credentials to release
+ * put_cred_many - Release a reference to a set of credentials
+ * @_cred: The credentials to release
  * @nr: Number of references to release
  *
  * Release a reference to a set of credentials, deleting them when the last ref
diff --git a/include/linux/damon.h b/include/linux/damon.h
index a4fea23da8576..d9a3babbafc16 100644
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -55,6 +55,8 @@ struct damon_size_range {
  * @list:		List head for siblings.
  * @age:		Age of this region.
  *
+ * For any use case, @ar should be non-zero positive size.
+ *
  * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be
  * increased for every &damon_attrs->sample_interval if an access to the region
  * during the last sampling interval is found.  The update of this field should
@@ -213,12 +215,23 @@ struct damos_quota_goal {
 	struct list_head list;
 };
 
+/**
+ * enum damos_quota_goal_tuner - Goal-based quota tuning logic.
+ * @DAMOS_QUOTA_GOAL_TUNER_CONSIST:	Aim long term consistent quota.
+ * @DAMOS_QUOTA_GOAL_TUNER_TEMPORAL:	Aim zero quota asap.
+ */
+enum damos_quota_goal_tuner {
+	DAMOS_QUOTA_GOAL_TUNER_CONSIST,
+	DAMOS_QUOTA_GOAL_TUNER_TEMPORAL,
+};
+
 /**
  * struct damos_quota - Controls the aggressiveness of the given scheme.
  * @reset_interval:	Charge reset interval in milliseconds.
  * @ms:			Maximum milliseconds that the scheme can use.
  * @sz:			Maximum bytes of memory that the action can be applied.
  * @goals:		Head of quota tuning goals (&damos_quota_goal) list.
+ * @goal_tuner:		Goal-based @esz tuning algorithm to use.
  * @esz:		Effective size quota in bytes.
  *
  * @weight_sz:		Weight of the region's size for prioritization.
@@ -260,6 +273,7 @@ struct damos_quota {
 	unsigned long ms;
 	unsigned long sz;
 	struct list_head goals;
+	enum damos_quota_goal_tuner goal_tuner;
 	unsigned long esz;
 
 	unsigned int weight_sz;
@@ -647,8 +661,7 @@ struct damon_operations {
 	void (*prepare_access_checks)(struct damon_ctx *context);
 	unsigned int (*check_accesses)(struct damon_ctx *context);
 	int (*get_scheme_score)(struct damon_ctx *context,
-			struct damon_target *t, struct damon_region *r,
-			struct damos *scheme);
+			struct damon_region *r, struct damos *scheme);
 	unsigned long (*apply_scheme)(struct damon_ctx *context,
 			struct damon_target *t, struct damon_region *r,
 			struct damos *scheme, unsigned long *sz_filter_passed);
@@ -810,6 +823,12 @@ struct damon_ctx {
 	struct damos_walk_control *walk_control;
 	struct mutex walk_control_lock;
 
+	/*
+	 * indicate if this may be corrupted.  Currentonly this is set only for
+	 * damon_commit_ctx() failure.
+	 */
+	bool maybe_corrupted;
+
 	/* Working thread of the given DAMON context */
 	struct task_struct *kdamond;
 	/* Protects @kdamond field access */
@@ -975,6 +994,7 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control);
 
 int damon_set_region_biggest_system_ram_default(struct damon_target *t,
 				unsigned long *start, unsigned long *end,
+				unsigned long addr_unit,
 				unsigned long min_region_sz);
 
 #endif	/* CONFIG_DAMON */
diff --git a/include/linux/dax.h b/include/linux/dax.h
index bf103f317cac0..10a7cc79aea5b 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -69,7 +69,7 @@ static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc,
 					    const struct inode *inode,
 					    struct dax_device *dax_dev)
 {
-	if (!vma_desc_test_flags(desc, VMA_SYNC_BIT))
+	if (!vma_desc_test(desc, VMA_SYNC_BIT))
 		return true;
 	if (!IS_DAX(inode))
 		return false;
@@ -115,7 +115,7 @@ static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc,
 					    const struct inode *inode,
 					    struct dax_device *dax_dev)
 {
-	return !vma_desc_test_flags(desc, VMA_SYNC_BIT);
+	return !vma_desc_test(desc, VMA_SYNC_BIT);
 }
 static inline size_t dax_recovery_write(struct dax_device *dax_dev,
 		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
diff --git a/include/linux/device.h b/include/linux/device.h
index 0be95294b6e61..67cec9ec0cd0c 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -189,6 +189,22 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
 #define DEVICE_ATTR_ADMIN_RW(_name) \
 	struct device_attribute dev_attr_##_name = __ATTR_RW_MODE(_name, 0600)
 
+/**
+ * DEVICE_ATTR_RW_NAMED - Define a read-write device attribute with a sysfs name
+ * that differs from the function name.
+ * @_name: Attribute function preface
+ * @_attrname: Attribute name as it wil be exposed in the sysfs.
+ *
+ * Like DEVICE_ATTR_RW(), but allows for reusing names under separate paths in
+ * the same driver.
+ */
+#define DEVICE_ATTR_RW_NAMED(_name, _attrname)                            \
+	struct device_attribute dev_attr_##_name = {                      \
+		.attr = { .name = _attrname, .mode = 0644 }, \
+		.show = _name##_show,                                     \
+		.store = _name##_store,                                   \
+	}
+
 /**
  * DEVICE_ATTR_RO - Define a readable device attribute.
  * @_name: Attribute name.
@@ -207,6 +223,21 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
 #define DEVICE_ATTR_ADMIN_RO(_name) \
 	struct device_attribute dev_attr_##_name = __ATTR_RO_MODE(_name, 0400)
 
+/**
+ * DEVICE_ATTR_RO_NAMED - Define a read-only device attribute with a sysfs name
+ * that differs from the function name.
+ * @_name: Attribute function preface
+ * @_attrname: Attribute name as it wil be exposed in the sysfs.
+ *
+ * Like DEVICE_ATTR_RO(), but allows for reusing names under separate paths in
+ * the same driver.
+ */
+#define DEVICE_ATTR_RO_NAMED(_name, _attrname)                            \
+	struct device_attribute dev_attr_##_name = {                      \
+		.attr = { .name = _attrname, .mode = 0444 }, \
+		.show = _name##_show,                                     \
+	}
+
 /**
  * DEVICE_ATTR_WO - Define an admin-only writable device attribute.
  * @_name: Attribute name.
@@ -216,6 +247,21 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
 #define DEVICE_ATTR_WO(_name) \
 	struct device_attribute dev_attr_##_name = __ATTR_WO(_name)
 
+/**
+ * DEVICE_ATTR_WO_NAMED - Define a read-only device attribute with a sysfs name
+ * that differs from the function name.
+ * @_name: Attribute function preface
+ * @_attrname: Attribute name as it wil be exposed in the sysfs.
+ *
+ * Like DEVICE_ATTR_WO(), but allows for reusing names under separate paths in
+ * the same driver.
+ */
+#define DEVICE_ATTR_WO_NAMED(_name, _attrname)                            \
+	struct device_attribute dev_attr_##_name = {                      \
+		.attr = { .name = _attrname, .mode = 0200 }, \
+		.store = _name##_store,                                   \
+	}
+
 /**
  * DEVICE_ULONG_ATTR - Define a device attribute backed by an unsigned long.
  * @_name: Attribute name.
@@ -483,6 +529,8 @@ struct device_physical_location {
  * 		on.  This shrinks the "Board Support Packages" (BSPs) and
  * 		minimizes board-specific #ifdefs in drivers.
  * @driver_data: Private pointer for driver specific info.
+ * @driver_override: Driver name to force a match.  Do not touch directly; use
+ *		     device_set_driver_override() instead.
  * @links:	Links to suppliers and consumers of this device.
  * @power:	For device power management.
  *		See Documentation/driver-api/pm/devices.rst for details.
@@ -576,6 +624,10 @@ struct device {
 					   core doesn't touch it */
 	void		*driver_data;	/* Driver data, set and get with
 					   dev_set_drvdata/dev_get_drvdata */
+	struct {
+		const char	*name;
+		spinlock_t	lock;
+	} driver_override;
 	struct mutex		mutex;	/* mutex to synchronize calls to
 					 * its driver.
 					 */
@@ -701,6 +753,54 @@ struct device_link {
 
 #define kobj_to_dev(__kobj)	container_of_const(__kobj, struct device, kobj)
 
+int __device_set_driver_override(struct device *dev, const char *s, size_t len);
+
+/**
+ * device_set_driver_override() - Helper to set or clear driver override.
+ * @dev: Device to change
+ * @s: NUL-terminated string, new driver name to force a match, pass empty
+ *     string to clear it ("" or "\n", where the latter is only for sysfs
+ *     interface).
+ *
+ * Helper to set or clear driver override of a device.
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+static inline int device_set_driver_override(struct device *dev, const char *s)
+{
+	return __device_set_driver_override(dev, s, s ? strlen(s) : 0);
+}
+
+/**
+ * device_has_driver_override() - Check if a driver override has been set.
+ * @dev: device to check
+ *
+ * Returns true if a driver override has been set for this device.
+ */
+static inline bool device_has_driver_override(struct device *dev)
+{
+	guard(spinlock)(&dev->driver_override.lock);
+	return !!dev->driver_override.name;
+}
+
+/**
+ * device_match_driver_override() - Match a driver against the device's driver_override.
+ * @dev: device to check
+ * @drv: driver to match against
+ *
+ * Returns > 0 if a driver override is set and matches the given driver, 0 if a
+ * driver override is set but does not match, or < 0 if a driver override is not
+ * set at all.
+ */
+static inline int device_match_driver_override(struct device *dev,
+					       const struct device_driver *drv)
+{
+	guard(spinlock)(&dev->driver_override.lock);
+	if (dev->driver_override.name)
+		return !strcmp(dev->driver_override.name, drv->name);
+	return -1;
+}
+
 /**
  * device_iommu_mapped - Returns true when the device DMA is translated
  *			 by an IOMMU
@@ -911,6 +1011,7 @@ static inline void device_unlock(struct device *dev)
 }
 
 DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
+DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0)
 
 static inline void device_lock_assert(struct device *dev)
 {
@@ -1131,9 +1232,9 @@ device_create_with_groups(const struct class *cls, struct device *parent, dev_t
 void device_destroy(const struct class *cls, dev_t devt);
 
 int __must_check device_add_groups(struct device *dev,
-				   const struct attribute_group **groups);
+				   const struct attribute_group *const *groups);
 void device_remove_groups(struct device *dev,
-			  const struct attribute_group **groups);
+			  const struct attribute_group *const *groups);
 
 static inline int __must_check device_add_group(struct device *dev,
 					const struct attribute_group *grp)
diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h
index 63de5f053c331..c1b463cd6464d 100644
--- a/include/linux/device/bus.h
+++ b/include/linux/device/bus.h
@@ -65,6 +65,9 @@ struct fwnode_handle;
  *			this bus.
  * @pm:		Power management operations of this bus, callback the specific
  *		device driver's pm-ops.
+ * @driver_override:	Set to true if this bus supports the driver_override
+ *			mechanism, which allows userspace to force a specific
+ *			driver to bind to a device via a sysfs attribute.
  * @need_parent_lock:	When probing or removing a device on this bus, the
  *			device core should lock the device's parent.
  *
@@ -106,6 +109,7 @@ struct bus_type {
 
 	const struct dev_pm_ops *pm;
 
+	bool driver_override;
 	bool need_parent_lock;
 };
 
diff --git a/include/linux/device/class.h b/include/linux/device/class.h
index 65880e60c7208..2079239a5aa5b 100644
--- a/include/linux/device/class.h
+++ b/include/linux/device/class.h
@@ -50,8 +50,8 @@ struct fwnode_handle;
 struct class {
 	const char		*name;
 
-	const struct attribute_group	**class_groups;
-	const struct attribute_group	**dev_groups;
+	const struct attribute_group	*const *class_groups;
+	const struct attribute_group	*const *dev_groups;
 
 	int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env);
 	char *(*devnode)(const struct device *dev, umode_t *mode);
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 133b9e637b558..166933b82e27a 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -407,7 +407,7 @@ struct dma_buf {
 	 *   through the device.
 	 *
 	 * - Dynamic importers should set fences for any access that they can't
-	 *   disable immediately from their &dma_buf_attach_ops.move_notify
+	 *   disable immediately from their &dma_buf_attach_ops.invalidate_mappings
 	 *   callback.
 	 *
 	 * IMPORTANT:
@@ -446,7 +446,7 @@ struct dma_buf_attach_ops {
 	bool allow_peer2peer;
 
 	/**
-	 * @move_notify: [optional] notification that the DMA-buf is moving
+	 * @invalidate_mappings: [optional] notification that the DMA-buf is moving
 	 *
 	 * If this callback is provided the framework can avoid pinning the
 	 * backing store while mappings exists.
@@ -456,14 +456,10 @@ struct dma_buf_attach_ops {
 	 * called with this lock held as well. This makes sure that no mapping
 	 * is created concurrently with an ongoing move operation.
 	 *
-	 * Mappings stay valid and are not directly affected by this callback.
-	 * But the DMA-buf can now be in a different physical location, so all
-	 * mappings should be destroyed and re-created as soon as possible.
-	 *
-	 * New mappings can be created after this callback returns, and will
-	 * point to the new location of the DMA-buf.
+	 * See the kdoc for dma_buf_invalidate_mappings() for details on the
+	 * required behavior.
 	 */
-	void (*move_notify)(struct dma_buf_attachment *attach);
+	void (*invalidate_mappings)(struct dma_buf_attachment *attach);
 };
 
 /**
@@ -578,7 +574,8 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *,
 					enum dma_data_direction);
 void dma_buf_unmap_attachment(struct dma_buf_attachment *, struct sg_table *,
 				enum dma_data_direction);
-void dma_buf_move_notify(struct dma_buf *dma_buf);
+void dma_buf_invalidate_mappings(struct dma_buf *dma_buf);
+bool dma_buf_attach_revocable(struct dma_buf_attachment *attach);
 int dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
 			     enum dma_data_direction dir);
 int dma_buf_end_cpu_access(struct dma_buf *dma_buf,
diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h
index 079b3dec0a16b..370b3d2bba371 100644
--- a/include/linux/dma-fence-array.h
+++ b/include/linux/dma-fence-array.h
@@ -38,7 +38,6 @@ struct dma_fence_array_cb {
 struct dma_fence_array {
 	struct dma_fence base;
 
-	spinlock_t lock;
 	unsigned num_fences;
 	atomic_t num_pending;
 	struct dma_fence **fences;
diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h
index 5cd3ba53b4a16..df3beadf15157 100644
--- a/include/linux/dma-fence-chain.h
+++ b/include/linux/dma-fence-chain.h
@@ -46,7 +46,6 @@ struct dma_fence_chain {
 		 */
 		struct irq_work work;
 	};
-	spinlock_t lock;
 };
 
 
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index d4c92fd350924..3dc93f068bf61 100644
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -34,7 +34,8 @@ struct seq_file;
  * @ops: dma_fence_ops associated with this fence
  * @rcu: used for releasing fence with kfree_rcu
  * @cb_list: list of all callbacks to call
- * @lock: spin_lock_irqsave used for locking
+ * @extern_lock: external spin_lock_irqsave used for locking (deprecated)
+ * @inline_lock: alternative internal spin_lock_irqsave used for locking
  * @context: execution context this fence belongs to, returned by
  *           dma_fence_context_alloc()
  * @seqno: the sequence number of this fence inside the execution context,
@@ -48,6 +49,8 @@ struct seq_file;
  * atomic ops (bit_*), so taking the spinlock will not be needed most
  * of the time.
  *
+ * DMA_FENCE_FLAG_INITIALIZED_BIT - fence was initialized
+ * DMA_FENCE_FLAG_INLINE_LOCK_BIT - use inline spinlock instead of external one
  * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled
  * DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling
  * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called
@@ -65,8 +68,11 @@ struct seq_file;
  * been completed, or never called at all.
  */
 struct dma_fence {
-	spinlock_t *lock;
-	const struct dma_fence_ops *ops;
+	union {
+		spinlock_t *extern_lock;
+		spinlock_t inline_lock;
+	};
+	const struct dma_fence_ops __rcu *ops;
 	/*
 	 * We clear the callback list on kref_put so that by the time we
 	 * release the fence it is unused. No one should be adding to the
@@ -98,6 +104,8 @@ struct dma_fence {
 };
 
 enum dma_fence_flag_bits {
+	DMA_FENCE_FLAG_INITIALIZED_BIT,
+	DMA_FENCE_FLAG_INLINE_LOCK_BIT,
 	DMA_FENCE_FLAG_SEQNO64_BIT,
 	DMA_FENCE_FLAG_SIGNALED_BIT,
 	DMA_FENCE_FLAG_TIMESTAMP_BIT,
@@ -218,6 +226,10 @@ struct dma_fence_ops {
 	 * timed out. Can also return other error values on custom implementations,
 	 * which should be treated as if the fence is signaled. For example a hardware
 	 * lockup could be reported like that.
+	 *
+	 * Implementing this callback prevents the fence from detaching after
+	 * signaling and so it is necessary for the module providing the
+	 * dma_fence_ops to stay loaded as long as the dma_fence exists.
 	 */
 	signed long (*wait)(struct dma_fence *fence,
 			    bool intr, signed long timeout);
@@ -229,6 +241,13 @@ struct dma_fence_ops {
 	 * Can be called from irq context.  This callback is optional. If it is
 	 * NULL, then dma_fence_free() is instead called as the default
 	 * implementation.
+	 *
+	 * Implementing this callback prevents the fence from detaching after
+	 * signaling and so it is necessary for the module providing the
+	 * dma_fence_ops to stay loaded as long as the dma_fence exists.
+	 *
+	 * If the callback is implemented the memory backing the dma_fence
+	 * object must be freed RCU safe.
 	 */
 	void (*release)(struct dma_fence *fence);
 
@@ -263,6 +282,19 @@ void dma_fence_release(struct kref *kref);
 void dma_fence_free(struct dma_fence *fence);
 void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq);
 
+/**
+ * dma_fence_was_initialized - test if fence was initialized
+ * @fence: fence to test
+ *
+ * Return: True if fence was ever initialized, false otherwise. Works correctly
+ * only when memory backing the fence structure is zero initialized on
+ * allocation.
+ */
+static inline bool dma_fence_was_initialized(struct dma_fence *fence)
+{
+	return fence && test_bit(DMA_FENCE_FLAG_INITIALIZED_BIT, &fence->flags);
+}
+
 /**
  * dma_fence_put - decreases refcount of the fence
  * @fence: fence to reduce refcount of
@@ -351,6 +383,45 @@ dma_fence_get_rcu_safe(struct dma_fence __rcu **fencep)
 	} while (1);
 }
 
+/**
+ * dma_fence_spinlock - return pointer to the spinlock protecting the fence
+ * @fence: the fence to get the lock from
+ *
+ * Return either the pointer to the embedded or the external spin lock.
+ */
+static inline spinlock_t *dma_fence_spinlock(struct dma_fence *fence)
+{
+	return test_bit(DMA_FENCE_FLAG_INLINE_LOCK_BIT, &fence->flags) ?
+		&fence->inline_lock : fence->extern_lock;
+}
+
+/**
+ * dma_fence_lock_irqsave - irqsave lock the fence
+ * @fence: the fence to lock
+ * @flags: where to store the CPU flags.
+ *
+ * Lock the fence, preventing it from changing to the signaled state.
+ */
+#define dma_fence_lock_irqsave(fence, flags)	\
+	spin_lock_irqsave(dma_fence_spinlock(fence), flags)
+
+/**
+ * dma_fence_unlock_irqrestore - unlock the fence and irqrestore
+ * @fence: the fence to unlock
+ * @flags the CPU flags to restore
+ *
+ * Unlock the fence, allowing it to change it's state to signaled again.
+ */
+#define dma_fence_unlock_irqrestore(fence, flags)	\
+	spin_unlock_irqrestore(dma_fence_spinlock(fence), flags)
+
+/**
+ * dma_fence_assert_held - lockdep assertion that fence is locked
+ * @fence: the fence which should be locked
+ */
+#define dma_fence_assert_held(fence)	\
+	lockdep_assert_held(dma_fence_spinlock(fence));
+
 #ifdef CONFIG_LOCKDEP
 bool dma_fence_begin_signalling(void);
 void dma_fence_end_signalling(bool cookie);
@@ -439,13 +510,19 @@ dma_fence_test_signaled_flag(struct dma_fence *fence)
 static inline bool
 dma_fence_is_signaled_locked(struct dma_fence *fence)
 {
+	const struct dma_fence_ops *ops;
+
 	if (dma_fence_test_signaled_flag(fence))
 		return true;
 
-	if (fence->ops->signaled && fence->ops->signaled(fence)) {
+	rcu_read_lock();
+	ops = rcu_dereference(fence->ops);
+	if (ops && ops->signaled && ops->signaled(fence)) {
+		rcu_read_unlock();
 		dma_fence_signal_locked(fence);
 		return true;
 	}
+	rcu_read_unlock();
 
 	return false;
 }
@@ -469,13 +546,19 @@ dma_fence_is_signaled_locked(struct dma_fence *fence)
 static inline bool
 dma_fence_is_signaled(struct dma_fence *fence)
 {
+	const struct dma_fence_ops *ops;
+
 	if (dma_fence_test_signaled_flag(fence))
 		return true;
 
-	if (fence->ops->signaled && fence->ops->signaled(fence)) {
+	rcu_read_lock();
+	ops = rcu_dereference(fence->ops);
+	if (ops && ops->signaled && ops->signaled(fence)) {
+		rcu_read_unlock();
 		dma_fence_signal(fence);
 		return true;
 	}
+	rcu_read_unlock();
 
 	return false;
 }
@@ -680,7 +763,7 @@ extern const struct dma_fence_ops dma_fence_chain_ops;
  */
 static inline bool dma_fence_is_array(struct dma_fence *fence)
 {
-	return fence->ops == &dma_fence_array_ops;
+	return rcu_access_pointer(fence->ops) == &dma_fence_array_ops;
 }
 
 /**
@@ -691,7 +774,7 @@ static inline bool dma_fence_is_array(struct dma_fence *fence)
  */
 static inline bool dma_fence_is_chain(struct dma_fence *fence)
 {
-	return fence->ops == &dma_fence_chain_ops;
+	return rcu_access_pointer(fence->ops) == &dma_fence_chain_ops;
 }
 
 /**
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 60b63756df821..8a07df5a9ef67 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -361,6 +361,12 @@ static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 }
 #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */
 
+#ifndef CONFIG_ARCH_HAS_BATCHED_DMA_SYNC
+static inline void arch_sync_dma_flush(void)
+{
+}
+#endif
+
 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
 void arch_sync_dma_for_cpu_all(void);
 #else
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 29973baa05816..482b919f040f7 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -80,11 +80,18 @@
 #define DMA_ATTR_MMIO		(1UL << 10)
 
 /*
- * DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline
- * overlapping this buffer while it is mapped for DMA. All mappings sharing
- * a cacheline must have this attribute for this to be considered safe.
+ * DMA_ATTR_DEBUGGING_IGNORE_CACHELINES: Indicates the CPU cache line can be
+ * overlapped. All mappings sharing a cacheline must have this attribute for
+ * this to be considered safe.
  */
-#define DMA_ATTR_CPU_CACHE_CLEAN	(1UL << 11)
+#define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES	(1UL << 11)
+
+/*
+ * DMA_ATTR_REQUIRE_COHERENT: Indicates that DMA coherency is required.
+ * All mappings that carry this attribute can't work with SWIOTLB and cache
+ * flushing.
+ */
+#define DMA_ATTR_REQUIRE_COHERENT	(1UL << 12)
 
 /*
  * A dma_addr_t can hold any valid DMA or bus address for the platform.  It can
diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index 270b5458aecf3..1fafd5b0e3153 100644
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -73,6 +73,8 @@ enum dw_edma_chip_flags {
  * @ll_region_rd:	 DMA descriptor link list memory for read channel
  * @dt_region_wr:	 DMA data memory for write channel
  * @dt_region_rd:	 DMA data memory for read channel
+ * @db_irq:		 Virtual IRQ dedicated to interrupt emulation
+ * @db_offset:		 Offset from DMA register base
  * @mf:			 DMA register map format
  * @dw:			 struct dw_edma that is filled by dw_edma_probe()
  */
@@ -94,9 +96,14 @@ struct dw_edma_chip {
 	struct dw_edma_region	dt_region_wr[EDMA_MAX_WR_CH];
 	struct dw_edma_region	dt_region_rd[EDMA_MAX_RD_CH];
 
+	/* interrupt emulation */
+	int			db_irq;
+	resource_size_t		db_offset;
+
 	enum dw_edma_map_format	mf;
 
 	struct dw_edma		*dw;
+	bool			cfg_non_ll;
 };
 
 /* Export to the platform drivers */
diff --git a/include/linux/dma/qcom-gpi-dma.h b/include/linux/dma/qcom-gpi-dma.h
index 6680dd1a43c68..332be28427e47 100644
--- a/include/linux/dma/qcom-gpi-dma.h
+++ b/include/linux/dma/qcom-gpi-dma.h
@@ -8,6 +8,9 @@
 
 /**
  * enum spi_transfer_cmd - spi transfer commands
+ * @SPI_TX: SPI peripheral TX command
+ * @SPI_RX: SPI peripheral RX command
+ * @SPI_DUPLEX: SPI peripheral Duplex command
  */
 enum spi_transfer_cmd {
 	SPI_TX = 1,
@@ -64,7 +67,7 @@ enum i2c_op {
  * @set_config: set peripheral config
  * @rx_len: receive length for buffer
  * @op: i2c cmd
- * @muli-msg: is part of multi i2c r-w msgs
+ * @multi_msg: is part of multi i2c r-w msgs
  */
 struct gpi_i2c_config {
 	u8 set_config;
diff --git a/include/linux/dma/ti-cppi5.h b/include/linux/dma/ti-cppi5.h
index c53c0f6e3b1a6..3fe19b75ddf76 100644
--- a/include/linux/dma/ti-cppi5.h
+++ b/include/linux/dma/ti-cppi5.h
@@ -16,8 +16,8 @@
  * struct cppi5_desc_hdr_t - Descriptor header, present in all types of
  *			     descriptors
  * @pkt_info0:		Packet info word 0 (n/a in Buffer desc)
- * @pkt_info0:		Packet info word 1 (n/a in Buffer desc)
- * @pkt_info0:		Packet info word 2 (n/a in Buffer desc)
+ * @pkt_info1:		Packet info word 1 (n/a in Buffer desc)
+ * @pkt_info2:		Packet info word 2 (n/a in Buffer desc)
  * @src_dst_tag:	Packet info word 3 (n/a in Buffer desc)
  */
 struct cppi5_desc_hdr_t {
@@ -35,7 +35,7 @@ struct cppi5_desc_hdr_t {
  * @buf_info1:		word 8: Buffer valid data length
  * @org_buf_len:	word 9: Original buffer length
  * @org_buf_ptr:	word 10/11: Original buffer pointer
- * @epib[0]:		Extended Packet Info Data (optional, 4 words), and/or
+ * @epib:		Extended Packet Info Data (optional, 4 words), and/or
  *			Protocol Specific Data (optional, 0-128 bytes in
  *			multiples of 4), and/or
  *			Other Software Data (0-N bytes, optional)
@@ -132,7 +132,7 @@ struct cppi5_desc_epib_t {
 /**
  * struct cppi5_monolithic_desc_t - Monolithic-mode packet descriptor
  * @hdr:		Descriptor header
- * @epib[0]:		Extended Packet Info Data (optional, 4 words), and/or
+ * @epib:		Extended Packet Info Data (optional, 4 words), and/or
  *			Protocol Specific Data (optional, 0-128 bytes in
  *			multiples of 4), and/or
  *			Other Software Data (0-N bytes, optional)
@@ -179,7 +179,7 @@ static inline void cppi5_desc_dump(void *desc, u32 size)
  * cppi5_desc_is_tdcm - check if the paddr indicates Teardown Complete Message
  * @paddr: Physical address of the packet popped from the ring
  *
- * Returns true if the address indicates TDCM
+ * Returns: true if the address indicates TDCM
  */
 static inline bool cppi5_desc_is_tdcm(dma_addr_t paddr)
 {
@@ -190,7 +190,7 @@ static inline bool cppi5_desc_is_tdcm(dma_addr_t paddr)
  * cppi5_desc_get_type - get descriptor type
  * @desc_hdr: packet descriptor/TR header
  *
- * Returns descriptor type:
+ * Returns: descriptor type:
  * CPPI5_INFO0_DESC_TYPE_VAL_HOST
  * CPPI5_INFO0_DESC_TYPE_VAL_MONO
  * CPPI5_INFO0_DESC_TYPE_VAL_TR
@@ -205,7 +205,7 @@ static inline u32 cppi5_desc_get_type(struct cppi5_desc_hdr_t *desc_hdr)
  * cppi5_desc_get_errflags - get Error Flags from Desc
  * @desc_hdr: packet/TR descriptor header
  *
- * Returns Error Flags from Packet/TR Descriptor
+ * Returns: Error Flags from Packet/TR Descriptor
  */
 static inline u32 cppi5_desc_get_errflags(struct cppi5_desc_hdr_t *desc_hdr)
 {
@@ -307,7 +307,7 @@ static inline void cppi5_desc_set_tags_ids(struct cppi5_desc_hdr_t *desc_hdr,
  * @psdata_size: PSDATA size
  * @sw_data_size: SWDATA size
  *
- * Returns required Host Packet Descriptor size
+ * Returns: required Host Packet Descriptor size
  * 0 - if PSDATA > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE
  */
 static inline u32 cppi5_hdesc_calc_size(bool epib, u32 psdata_size,
@@ -381,6 +381,8 @@ cppi5_hdesc_update_psdata_size(struct cppi5_host_desc_t *desc, u32 psdata_size)
 /**
  * cppi5_hdesc_get_psdata_size - get PSdata size in bytes
  * @desc: Host packet descriptor
+ *
+ * Returns: PSdata size in bytes
  */
 static inline u32 cppi5_hdesc_get_psdata_size(struct cppi5_host_desc_t *desc)
 {
@@ -398,7 +400,7 @@ static inline u32 cppi5_hdesc_get_psdata_size(struct cppi5_host_desc_t *desc)
  * cppi5_hdesc_get_pktlen - get Packet Length from HDesc
  * @desc: Host packet descriptor
  *
- * Returns Packet Length from Host Packet Descriptor
+ * Returns: Packet Length from Host Packet Descriptor
  */
 static inline u32 cppi5_hdesc_get_pktlen(struct cppi5_host_desc_t *desc)
 {
@@ -408,6 +410,7 @@ static inline u32 cppi5_hdesc_get_pktlen(struct cppi5_host_desc_t *desc)
 /**
  * cppi5_hdesc_set_pktlen - set Packet Length in HDesc
  * @desc: Host packet descriptor
+ * @pkt_len: Packet length to set
  */
 static inline void cppi5_hdesc_set_pktlen(struct cppi5_host_desc_t *desc,
 					  u32 pkt_len)
@@ -420,7 +423,7 @@ static inline void cppi5_hdesc_set_pktlen(struct cppi5_host_desc_t *desc,
  * cppi5_hdesc_get_psflags - get Protocol Specific Flags from HDesc
  * @desc: Host packet descriptor
  *
- * Returns Protocol Specific Flags from Host Packet Descriptor
+ * Returns: Protocol Specific Flags from Host Packet Descriptor
  */
 static inline u32 cppi5_hdesc_get_psflags(struct cppi5_host_desc_t *desc)
 {
@@ -431,6 +434,7 @@ static inline u32 cppi5_hdesc_get_psflags(struct cppi5_host_desc_t *desc)
 /**
  * cppi5_hdesc_set_psflags - set Protocol Specific Flags in HDesc
  * @desc: Host packet descriptor
+ * @ps_flags: Protocol Specific flags to set
  */
 static inline void cppi5_hdesc_set_psflags(struct cppi5_host_desc_t *desc,
 					   u32 ps_flags)
@@ -442,8 +446,10 @@ static inline void cppi5_hdesc_set_psflags(struct cppi5_host_desc_t *desc,
 }
 
 /**
- * cppi5_hdesc_get_errflags - get Packet Type from HDesc
+ * cppi5_hdesc_get_pkttype - get Packet Type from HDesc
  * @desc: Host packet descriptor
+ *
+ * Returns: Packet type
  */
 static inline u32 cppi5_hdesc_get_pkttype(struct cppi5_host_desc_t *desc)
 {
@@ -452,7 +458,7 @@ static inline u32 cppi5_hdesc_get_pkttype(struct cppi5_host_desc_t *desc)
 }
 
 /**
- * cppi5_hdesc_get_errflags - set Packet Type in HDesc
+ * cppi5_hdesc_set_pkttype - set Packet Type in HDesc
  * @desc: Host packet descriptor
  * @pkt_type: Packet Type
  */
@@ -501,7 +507,7 @@ static inline void cppi5_hdesc_reset_to_original(struct cppi5_host_desc_t *desc)
 /**
  * cppi5_hdesc_link_hbdesc - link Host Buffer Descriptor to HDesc
  * @desc: Host Packet Descriptor
- * @buf_desc: Host Buffer Descriptor physical address
+ * @hbuf_desc: Host Buffer Descriptor physical address
  *
  * add and link Host Buffer Descriptor to HDesc
  */
@@ -527,7 +533,7 @@ static inline void cppi5_hdesc_reset_hbdesc(struct cppi5_host_desc_t *desc)
  * cppi5_hdesc_epib_present -  check if EPIB present
  * @desc_hdr: packet descriptor/TR header
  *
- * Returns true if EPIB present in the packet
+ * Returns: true if EPIB present in the packet
  */
 static inline bool cppi5_hdesc_epib_present(struct cppi5_desc_hdr_t *desc_hdr)
 {
@@ -538,7 +544,7 @@ static inline bool cppi5_hdesc_epib_present(struct cppi5_desc_hdr_t *desc_hdr)
  * cppi5_hdesc_get_psdata -  Get pointer on PSDATA
  * @desc: Host packet descriptor
  *
- * Returns pointer on PSDATA in HDesc.
+ * Returns: pointer on PSDATA in HDesc.
  * NULL - if ps_data placed at the start of data buffer.
  */
 static inline void *cppi5_hdesc_get_psdata(struct cppi5_host_desc_t *desc)
@@ -568,7 +574,7 @@ static inline void *cppi5_hdesc_get_psdata(struct cppi5_host_desc_t *desc)
  * cppi5_hdesc_get_swdata -  Get pointer on swdata
  * @desc: Host packet descriptor
  *
- * Returns pointer on SWDATA in HDesc.
+ * Returns: pointer on SWDATA in HDesc.
  * NOTE. It's caller responsibility to be sure hdesc actually has swdata.
  */
 static inline void *cppi5_hdesc_get_swdata(struct cppi5_host_desc_t *desc)
@@ -648,6 +654,7 @@ enum cppi5_tr_types {
 	CPPI5_TR_TYPE11,
 	/* type12-14: Reserved */
 	CPPI5_TR_TYPE15 = 15,
+	/* private: */
 	CPPI5_TR_TYPE_MAX
 };
 
@@ -673,6 +680,7 @@ enum cppi5_tr_event_size {
 	CPPI5_TR_EVENT_SIZE_ICNT1_DEC,
 	CPPI5_TR_EVENT_SIZE_ICNT2_DEC,
 	CPPI5_TR_EVENT_SIZE_ICNT3_DEC,
+	/* private: */
 	CPPI5_TR_EVENT_SIZE_MAX
 };
 
@@ -690,6 +698,7 @@ enum cppi5_tr_trigger {
 	CPPI5_TR_TRIGGER_GLOBAL0,
 	CPPI5_TR_TRIGGER_GLOBAL1,
 	CPPI5_TR_TRIGGER_LOCAL_EVENT,
+	/* private: */
 	CPPI5_TR_TRIGGER_MAX
 };
 
@@ -711,6 +720,7 @@ enum cppi5_tr_trigger_type {
 	CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
 	CPPI5_TR_TRIGGER_TYPE_ICNT3_DEC,
 	CPPI5_TR_TRIGGER_TYPE_ALL,
+	/* private: */
 	CPPI5_TR_TRIGGER_TYPE_MAX
 };
 
@@ -815,7 +825,7 @@ struct cppi5_tr_type3_t {
  *			destination
  * @dicnt1:		Total loop iteration count for level 1 for destination
  * @dicnt2:		Total loop iteration count for level 2 for destination
- * @sicnt3:		Total loop iteration count for level 3 (outermost) for
+ * @dicnt3:		Total loop iteration count for level 3 (outermost) for
  *			destination
  */
 struct cppi5_tr_type15_t {
@@ -887,6 +897,7 @@ enum cppi5_tr_resp_status_type {
 	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_ERR,
 	CPPI5_TR_RESPONSE_STATUS_TRANSFER_EXCEPTION,
 	CPPI5_TR_RESPONSE_STATUS__TEARDOWN_FLUSH,
+	/* private: */
 	CPPI5_TR_RESPONSE_STATUS_MAX
 };
 
@@ -903,6 +914,7 @@ enum cppi5_tr_resp_status_submission {
 	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_ICNT0,
 	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_FIFO_FULL,
 	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_OWN,
+	/* private: */
 	CPPI5_TR_RESPONSE_STATUS_SUBMISSION_MAX
 };
 
@@ -931,6 +943,7 @@ enum cppi5_tr_resp_status_unsupported {
 	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_DFMT,
 	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_SECTR,
 	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_AMODE_SPECIFIC,
+	/* private: */
 	CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_MAX
 };
 
@@ -939,7 +952,7 @@ enum cppi5_tr_resp_status_unsupported {
  * @tr_count: number of TR records
  * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128]
  *
- * Returns required TR Descriptor size
+ * Returns: required TR Descriptor size
  */
 static inline size_t cppi5_trdesc_calc_size(u32 tr_count, u32 tr_size)
 {
@@ -955,7 +968,7 @@ static inline size_t cppi5_trdesc_calc_size(u32 tr_count, u32 tr_size)
 
 /**
  * cppi5_trdesc_init - Init TR Descriptor
- * @desc: TR Descriptor
+ * @desc_hdr: TR Descriptor
  * @tr_count: number of TR records
  * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128]
  * @reload_idx: Absolute index to jump to on the 2nd and following passes
@@ -1044,7 +1057,7 @@ static inline void cppi5_tr_set_trigger(cppi5_tr_flags_t *flags,
 }
 
 /**
- * cppi5_tr_cflag_set - Update the Configuration specific flags
+ * cppi5_tr_csf_set - Update the Configuration specific flags
  * @flags: Pointer to the TR's flags
  * @csf: Configuration specific flags
  *
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 99efe2b9b4ea9..b3d251c9734e9 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -996,7 +996,8 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
  * @vecs: The array of DMA vectors that should be transferred
  * @nents: The number of DMA vectors in the array
  * @dir: Specifies the direction of the data transfer
- * @flags: DMA engine flags
+ * @flags: DMA engine flags - DMA_PREP_REPEAT can be used to mark a cyclic
+ *         DMA transfer
  */
 static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
 	struct dma_chan *chan, const struct dma_vec *vecs, size_t nents,
diff --git a/include/linux/drbd.h b/include/linux/drbd.h
index 5468a2399d483..ed408088a2825 100644
--- a/include/linux/drbd.h
+++ b/include/linux/drbd.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
   drbd.h
   Kernel module for 2.6.x Kernels
@@ -9,10 +9,10 @@
   Copyright (C) 2001-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2001-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
 
-
 */
 #ifndef DRBD_H
 #define DRBD_H
+
 #include <asm/types.h>
 
 #ifdef __KERNEL__
@@ -44,8 +44,7 @@ enum drbd_io_error_p {
 	EP_DETACH
 };
 
-enum drbd_fencing_p {
-	FP_NOT_AVAIL = -1, /* Not a policy */
+enum drbd_fencing_policy {
 	FP_DONT_CARE = 0,
 	FP_RESOURCE,
 	FP_STONITH
@@ -68,7 +67,9 @@ enum drbd_after_sb_p {
 	ASB_CONSENSUS,
 	ASB_DISCARD_SECONDARY,
 	ASB_CALL_HELPER,
-	ASB_VIOLENTLY
+	ASB_VIOLENTLY,
+	ASB_RETRY_CONNECT,
+	ASB_AUTO_DISCARD,
 };
 
 enum drbd_on_no_data {
@@ -76,6 +77,16 @@ enum drbd_on_no_data {
 	OND_SUSPEND_IO
 };
 
+enum drbd_on_no_quorum {
+	ONQ_IO_ERROR = OND_IO_ERROR,
+	ONQ_SUSPEND_IO = OND_SUSPEND_IO
+};
+
+enum drbd_on_susp_primary_outdated {
+	SPO_DISCONNECT,
+	SPO_FORCE_SECONDARY,
+};
+
 enum drbd_on_congestion {
 	OC_BLOCK,
 	OC_PULL_AHEAD,
@@ -96,6 +107,11 @@ enum drbd_read_balancing {
 	RB_1M_STRIPING,
 };
 
+/* Windows km/dderror.h has that a 0L */
+#ifdef NO_ERROR
+#undef NO_ERROR
+#endif
+
 /* KEEP the order, do not delete or insert. Only append. */
 enum drbd_ret_code {
 	ERR_CODE_BASE		= 100,
@@ -162,6 +178,12 @@ enum drbd_ret_code {
 	ERR_MD_LAYOUT_TOO_SMALL = 168,
 	ERR_MD_LAYOUT_NO_FIT    = 169,
 	ERR_IMPLICIT_SHRINK     = 170,
+	ERR_INVALID_PEER_NODE_ID = 171,
+	ERR_CREATE_TRANSPORT    = 172,
+	ERR_LOCAL_AND_PEER_ADDR = 173,
+	ERR_ALREADY_EXISTS 	= 174,
+	ERR_APV_TOO_LOW         = 175,
+
 	/* insert new ones above this line */
 	AFTER_LAST_ERR_CODE
 };
@@ -178,17 +200,17 @@ enum drbd_role {
 };
 
 /* The order of these constants is important.
- * The lower ones (<C_WF_REPORT_PARAMS) indicate
+ * The lower ones (< C_CONNECTED) indicate
  * that there is no socket!
- * >=C_WF_REPORT_PARAMS ==> There is a socket
+ * >= C_CONNECTED ==> There is a socket
  */
-enum drbd_conns {
+enum drbd_conn_state {
 	C_STANDALONE,
-	C_DISCONNECTING,  /* Temporal state on the way to StandAlone. */
+	C_DISCONNECTING,  /* Temporary state on the way to C_STANDALONE. */
 	C_UNCONNECTED,    /* >= C_UNCONNECTED -> inc_net() succeeds */
 
-	/* These temporal states are all used on the way
-	 * from >= C_CONNECTED to Unconnected.
+	/* These temporary states are used on the way
+	 * from C_CONNECTED to C_UNCONNECTED.
 	 * The 'disconnect reason' states
 	 * I do not allow to change between them. */
 	C_TIMEOUT,
@@ -197,35 +219,44 @@ enum drbd_conns {
 	C_PROTOCOL_ERROR,
 	C_TEAR_DOWN,
 
-	C_WF_CONNECTION,
-	C_WF_REPORT_PARAMS, /* we have a socket */
-	C_CONNECTED,      /* we have introduced each other */
-	C_STARTING_SYNC_S,  /* starting full sync by admin request. */
-	C_STARTING_SYNC_T,  /* starting full sync by admin request. */
-	C_WF_BITMAP_S,
-	C_WF_BITMAP_T,
-	C_WF_SYNC_UUID,
+	C_CONNECTING,
+
+	C_CONNECTED, /* we have a socket */
+
+	C_MASK = 31,
+};
+
+enum drbd_repl_state {
+	L_NEGOTIATING = C_CONNECTED, /* used for peer_device->negotiation_result only */
+	L_OFF = C_CONNECTED,
+
+	L_ESTABLISHED,      /* we have introduced each other */
+	L_STARTING_SYNC_S,  /* starting full sync by admin request. */
+	L_STARTING_SYNC_T,  /* starting full sync by admin request. */
+	L_WF_BITMAP_S,
+	L_WF_BITMAP_T,
+	L_WF_SYNC_UUID,
 
 	/* All SyncStates are tested with this comparison
-	 * xx >= C_SYNC_SOURCE && xx <= C_PAUSED_SYNC_T */
-	C_SYNC_SOURCE,
-	C_SYNC_TARGET,
-	C_VERIFY_S,
-	C_VERIFY_T,
-	C_PAUSED_SYNC_S,
-	C_PAUSED_SYNC_T,
-
-	C_AHEAD,
-	C_BEHIND,
-
-	C_MASK = 31
+	 * xx >= L_SYNC_SOURCE && xx <= L_PAUSED_SYNC_T */
+	L_SYNC_SOURCE,
+	L_SYNC_TARGET,
+	L_VERIFY_S,
+	L_VERIFY_T,
+	L_PAUSED_SYNC_S,
+	L_PAUSED_SYNC_T,
+
+	L_AHEAD,
+	L_BEHIND,
+	L_NEG_NO_RESULT = L_BEHIND,  /* used for peer_device->negotiation_result only */
 };
 
 enum drbd_disk_state {
 	D_DISKLESS,
 	D_ATTACHING,      /* In the process of reading the meta-data */
+	D_DETACHING,      /* Added in protocol version 110 */
 	D_FAILED,         /* Becomes D_DISKLESS as soon as we told it the peer */
-			  /* when >= D_FAILED it is legal to access mdev->ldev */
+			  /* when >= D_FAILED it is legal to access device->ldev */
 	D_NEGOTIATING,    /* Late attaching state, we need to talk to the peer */
 	D_INCONSISTENT,
 	D_OUTDATED,
@@ -257,9 +288,11 @@ union drbd_state {
 		unsigned user_isp:1 ;
 		unsigned susp_nod:1 ; /* IO suspended because no data */
 		unsigned susp_fen:1 ; /* IO suspended because fence peer handler runs*/
-		unsigned _pad:9;   /* 0	 unused */
+		unsigned quorum:1;
+		unsigned _pad:8;   /* 0	 unused */
 #elif defined(__BIG_ENDIAN_BITFIELD)
-		unsigned _pad:9;
+		unsigned _pad:8;
+		unsigned quorum:1;
 		unsigned susp_fen:1 ;
 		unsigned susp_nod:1 ;
 		unsigned user_isp:1 ;
@@ -297,29 +330,48 @@ enum drbd_state_rv {
 	SS_DEVICE_IN_USE = -12,
 	SS_NO_NET_CONFIG = -13,
 	SS_NO_VERIFY_ALG = -14,       /* drbd-8.2 only */
-	SS_NEED_CONNECTION = -15,    /* drbd-8.2 only */
+	SS_NEED_CONNECTION = -15,
 	SS_LOWER_THAN_OUTDATED = -16,
-	SS_NOT_SUPPORTED = -17,      /* drbd-8.2 only */
+	SS_NOT_SUPPORTED = -17,
 	SS_IN_TRANSIENT_STATE = -18,  /* Retry after the next state change */
 	SS_CONCURRENT_ST_CHG = -19,   /* Concurrent cluster side state change! */
 	SS_O_VOL_PEER_PRI = -20,
-	SS_OUTDATE_WO_CONN = -21,
-	SS_AFTER_LAST_ERROR = -22,    /* Keep this at bottom */
+	SS_INTERRUPTED = -21,	/* interrupted in stable_state_change() */
+	SS_PRIMARY_READER = -22,
+	SS_TIMEOUT = -23,
+	SS_WEAKLY_CONNECTED = -24,
+	SS_NO_QUORUM = -25,
+	SS_ATTACH_NO_BITMAP = -26,
+	SS_HANDSHAKE_DISCONNECT = -27,
+	SS_HANDSHAKE_RETRY = -28,
+	SS_AFTER_LAST_ERROR = -29,    /* Keep this at bottom */
 };
 
 #define SHARED_SECRET_MAX 64
 
-#define MDF_CONSISTENT		(1 << 0)
-#define MDF_PRIMARY_IND		(1 << 1)
-#define MDF_CONNECTED_IND	(1 << 2)
-#define MDF_FULL_SYNC		(1 << 3)
-#define MDF_WAS_UP_TO_DATE	(1 << 4)
-#define MDF_PEER_OUT_DATED	(1 << 5)
-#define MDF_CRASHED_PRIMARY	(1 << 6)
-#define MDF_AL_CLEAN		(1 << 7)
-#define MDF_AL_DISABLED		(1 << 8)
+enum mdf_flag {
+	MDF_CONSISTENT =	1 << 0,
+	MDF_PRIMARY_IND =	1 << 1,
+	MDF_WAS_UP_TO_DATE =	1 << 4,
+	MDF_CRASHED_PRIMARY =	1 << 6,
+	MDF_AL_CLEAN =		1 << 7,
+	MDF_AL_DISABLED =       1 << 8,
+	MDF_PRIMARY_LOST_QUORUM = 1 << 9,
+	MDF_HAVE_QUORUM =       1 << 10,
+};
+
+enum mdf_peer_flag {
+	MDF_PEER_CONNECTED =	1 << 0,
+	MDF_PEER_OUTDATED =	1 << 1,
+	MDF_PEER_FENCING =	1 << 2,
+	MDF_PEER_FULL_SYNC =	1 << 3,
+	MDF_PEER_DEVICE_SEEN =	1 << 4,
+	MDF_NODE_EXISTS =       1 << 16,
+	MDF_HAVE_BITMAP =       1 << 31,  /* For in core use; no meaning when persistet */
+};
 
-#define MAX_PEERS 32
+#define DRBD_PEERS_MAX 32
+#define DRBD_NODE_ID_MAX DRBD_PEERS_MAX
 
 enum drbd_uuid_index {
 	UI_CURRENT,
@@ -331,7 +383,8 @@ enum drbd_uuid_index {
 	UI_EXTENDED_SIZE   /* Everything. */
 };
 
-#define HISTORY_UUIDS MAX_PEERS
+#define HISTORY_UUIDS_V08 (UI_HISTORY_END - UI_HISTORY_START + 1)
+#define HISTORY_UUIDS DRBD_PEERS_MAX
 
 enum drbd_timeout_flag {
 	UT_DEFAULT      = 0,
@@ -339,6 +392,16 @@ enum drbd_timeout_flag {
 	UT_PEER_OUTDATED = 2,
 };
 
+#define UUID_JUST_CREATED ((__u64)4)
+#define UUID_PRIMARY ((__u64)1)
+
+enum write_ordering_e {
+	WO_NONE,
+	WO_DRAIN_IO,
+	WO_BDEV_FLUSH,
+	WO_BIO_BARRIER
+};
+
 enum drbd_notification_type {
 	NOTIFY_EXISTS,
 	NOTIFY_CREATE,
@@ -346,11 +409,13 @@ enum drbd_notification_type {
 	NOTIFY_DESTROY,
 	NOTIFY_CALL,
 	NOTIFY_RESPONSE,
+	NOTIFY_RENAME,
 
 	NOTIFY_CONTINUES = 0x8000,
 	NOTIFY_FLAGS = NOTIFY_CONTINUES,
 };
 
+/* These values are part of the ABI! */
 enum drbd_peer_state {
 	P_INCONSISTENT = 3,
 	P_OUTDATED = 4,
@@ -359,15 +424,6 @@ enum drbd_peer_state {
 	P_FENCING = 7,
 };
 
-#define UUID_JUST_CREATED ((__u64)4)
-
-enum write_ordering_e {
-	WO_NONE,
-	WO_DRAIN_IO,
-	WO_BDEV_FLUSH,
-	WO_BIO_BARRIER
-};
-
 /* magic numbers used in meta data and network packets */
 #define DRBD_MAGIC 0x83740267
 #define DRBD_MAGIC_BIG 0x835a
@@ -376,17 +432,23 @@ enum write_ordering_e {
 #define DRBD_MD_MAGIC_07   (DRBD_MAGIC+3)
 #define DRBD_MD_MAGIC_08   (DRBD_MAGIC+4)
 #define DRBD_MD_MAGIC_84_UNCLEAN	(DRBD_MAGIC+5)
-
-
-/* how I came up with this magic?
- * base64 decode "actlog==" ;) */
-#define DRBD_AL_MAGIC 0x69cb65a2
+#define DRBD_MD_MAGIC_09   (DRBD_MAGIC+6)
 
 /* these are of type "int" */
 #define DRBD_MD_INDEX_INTERNAL -1
 #define DRBD_MD_INDEX_FLEX_EXT -2
 #define DRBD_MD_INDEX_FLEX_INT -3
 
-#define DRBD_CPU_MASK_SIZE 32
+/*
+ * This is the maximum string length accepted by drbdadm.
+ * It allows a full mask for up to 908 CPUs.
+ */
+#define DRBD_CPU_MASK_SIZE 256
+
+#define DRBD_MAX_BIO_SIZE (1U << 20)
+
+#define QOU_OFF 0
+#define QOU_MAJORITY 1024
+#define QOU_ALL 1025
 
 #endif
diff --git a/include/linux/drbd_config.h b/include/linux/drbd_config.h
index d215365c6bb1b..e6383dfcf4f67 100644
--- a/include/linux/drbd_config.h
+++ b/include/linux/drbd_config.h
@@ -1,16 +1,35 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * drbd_config.h
- * DRBD's compile time configuration.
- */
+  drbd_config.h
+  DRBD's compile time configuration.
+*/
 
 #ifndef DRBD_CONFIG_H
 #define DRBD_CONFIG_H
 
-extern const char *drbd_buildtag(void);
+#include "drbd_protocol.h"
+
+const char *drbd_buildtag(void);
+
+#define REL_VERSION "9.3.0"
+#define PRO_VERSION_MIN 118 /* 9.0.26 */
+#define PRO_VERSION_MAX 123
+#define PRO_FEATURES (DRBD_FF_TRIM | DRBD_FF_THIN_RESYNC | DRBD_FF_WSAME | DRBD_FF_WZEROES | \
+		      DRBD_FF_RESYNC_DAGTAG | \
+		      DRBD_FF_2PC_V2 | DRBD_FF_RS_SKIP_UUID | \
+		      DRBD_FF_RESYNC_WITHOUT_REPLICATION)
 
-#define REL_VERSION "8.4.11"
-#define PRO_VERSION_MIN 86
-#define PRO_VERSION_MAX 101
+#define PRO_VERSION_8_MIN 86
+#define PRO_VERSION_8_MAX 101
+
+/* We support two ranges of DRBD protocol version:
+ *  86-101: accepted DRBD 8 protocol versions as "rolling upgrade" path
+ * 102-109: never defined
+ * 110-117: _rejected_ because of bugs in the backward compat path
+ *	in more recent DRBD versions.  That is 9.0.0 to 9.0.25 inclusive.
+ *	"Rolling" upgrade path for those versions:
+ *	first upgrade to 9.0.latest, then connect to 9.1/9.2 or later.
+ * 118-PRO_VERSION_MAX: accepted DRBD 9 protocol versions.
+ */
 
 #endif
diff --git a/include/linux/drbd_genl.h b/include/linux/drbd_genl.h
index 53f44b8cd75f3..75e671a3c5d16 100644
--- a/include/linux/drbd_genl.h
+++ b/include/linux/drbd_genl.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * General overview:
  * full generic netlink message:
@@ -68,7 +68,7 @@
  *	genl_magic_func.h
  *		generates an entry in the static genl_ops array,
  *		and static register/unregister functions to
- *		genl_register_family().
+ *		genl_register_family_with_ops().
  *
  *	flags and handler:
  *		GENL_op_init( .doit = x, .dumpit = y, .flags = something)
@@ -96,10 +96,12 @@ GENL_struct(DRBD_NLA_CFG_REPLY, 1, drbd_cfg_reply,
  * and/or the replication group (aka resource) name,
  * and the volume id within the resource. */
 GENL_struct(DRBD_NLA_CFG_CONTEXT, 2, drbd_cfg_context,
+	__u32_field(6, DRBD_GENLA_F_MANDATORY,	ctx_peer_node_id)
 	__u32_field(1, DRBD_GENLA_F_MANDATORY,	ctx_volume)
 	__str_field(2, DRBD_GENLA_F_MANDATORY,	ctx_resource_name, 128)
 	__bin_field(3, DRBD_GENLA_F_MANDATORY,	ctx_my_addr, 128)
 	__bin_field(4, DRBD_GENLA_F_MANDATORY,	ctx_peer_addr, 128)
+	__str_field_def(5, 0, ctx_conn_name, SHARED_SECRET_MAX)
 )
 
 GENL_struct(DRBD_NLA_DISK_CONF, 3, disk_conf,
@@ -109,37 +111,45 @@ GENL_struct(DRBD_NLA_DISK_CONF, 3, disk_conf,
 
 	/* use the resize command to try and change the disk_size */
 	__u64_field(4, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT,	disk_size)
-	/* we could change the max_bio_bvecs,
-	 * but it won't propagate through the stack */
-	__u32_field(5, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT,	max_bio_bvecs)
+	/*__u32_field(5, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT,	max_bio_bvecs)*/
 
 	__u32_field_def(6, DRBD_GENLA_F_MANDATORY,	on_io_error, DRBD_ON_IO_ERROR_DEF)
-	__u32_field_def(7, DRBD_GENLA_F_MANDATORY,	fencing, DRBD_FENCING_DEF)
+	/*__u32_field_def(7, DRBD_GENLA_F_MANDATORY,	fencing_policy, DRBD_FENCING_DEF)*/
 
-	__u32_field_def(8,	DRBD_GENLA_F_MANDATORY,	resync_rate, DRBD_RESYNC_RATE_DEF)
 	__s32_field_def(9,	DRBD_GENLA_F_MANDATORY,	resync_after, DRBD_MINOR_NUMBER_DEF)
 	__u32_field_def(10,	DRBD_GENLA_F_MANDATORY,	al_extents, DRBD_AL_EXTENTS_DEF)
-	__u32_field_def(11,	DRBD_GENLA_F_MANDATORY,	c_plan_ahead, DRBD_C_PLAN_AHEAD_DEF)
-	__u32_field_def(12,	DRBD_GENLA_F_MANDATORY,	c_delay_target, DRBD_C_DELAY_TARGET_DEF)
-	__u32_field_def(13,	DRBD_GENLA_F_MANDATORY,	c_fill_target, DRBD_C_FILL_TARGET_DEF)
-	__u32_field_def(14,	DRBD_GENLA_F_MANDATORY,	c_max_rate, DRBD_C_MAX_RATE_DEF)
-	__u32_field_def(15,	DRBD_GENLA_F_MANDATORY,	c_min_rate, DRBD_C_MIN_RATE_DEF)
-	__u32_field_def(20,     DRBD_GENLA_F_MANDATORY, disk_timeout, DRBD_DISK_TIMEOUT_DEF)
-	__u32_field_def(21,     0 /* OPTIONAL */,       read_balancing, DRBD_READ_BALANCING_DEF)
-	__u32_field_def(25,     0 /* OPTIONAL */,       rs_discard_granularity, DRBD_RS_DISCARD_GRANULARITY_DEF)
 
 	__flg_field_def(16, DRBD_GENLA_F_MANDATORY,	disk_barrier, DRBD_DISK_BARRIER_DEF)
 	__flg_field_def(17, DRBD_GENLA_F_MANDATORY,	disk_flushes, DRBD_DISK_FLUSHES_DEF)
 	__flg_field_def(18, DRBD_GENLA_F_MANDATORY,	disk_drain, DRBD_DISK_DRAIN_DEF)
 	__flg_field_def(19, DRBD_GENLA_F_MANDATORY,	md_flushes, DRBD_MD_FLUSHES_DEF)
+	__u32_field_def(20,	DRBD_GENLA_F_MANDATORY,	disk_timeout, DRBD_DISK_TIMEOUT_DEF)
+	__u32_field_def(21, DRBD_GENLA_F_MANDATORY,     read_balancing, DRBD_READ_BALANCING_DEF)
+	__u32_field_def(22,	DRBD_GENLA_F_MANDATORY,	unplug_watermark, DRBD_UNPLUG_WATERMARK_DEF)
+	__u32_field_def(25, 0 /* OPTIONAL */,           rs_discard_granularity, DRBD_RS_DISCARD_GRANULARITY_DEF)
 	__flg_field_def(23,     0 /* OPTIONAL */,	al_updates, DRBD_AL_UPDATES_DEF)
-	__flg_field_def(24,     0 /* OPTIONAL */,	discard_zeroes_if_aligned, DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF)
+	__flg_field_def(24,     0 /* OPTIONAL */,       discard_zeroes_if_aligned, DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF)
 	__flg_field_def(26,     0 /* OPTIONAL */,	disable_write_same, DRBD_DISABLE_WRITE_SAME_DEF)
+	__flg_field_def(27, 0 /* OPTIONAL */, 		d_bitmap, DRBD_BITMAP_DEF)
 )
 
 GENL_struct(DRBD_NLA_RESOURCE_OPTS, 4, res_opts,
 	__str_field_def(1,	DRBD_GENLA_F_MANDATORY,	cpu_mask,       DRBD_CPU_MASK_SIZE)
 	__u32_field_def(2,	DRBD_GENLA_F_MANDATORY,	on_no_data, DRBD_ON_NO_DATA_DEF)
+	__flg_field_def(3,	DRBD_GENLA_F_MANDATORY,	auto_promote, DRBD_AUTO_PROMOTE_DEF)
+	__u32_field(4,		DRBD_F_REQUIRED | DRBD_F_INVARIANT,	node_id)
+	__u32_field_def(5,	DRBD_GENLA_F_MANDATORY,	peer_ack_window, DRBD_PEER_ACK_WINDOW_DEF)
+	__u32_field_def(6,	DRBD_GENLA_F_MANDATORY,	twopc_timeout, DRBD_TWOPC_TIMEOUT_DEF)
+	__u32_field_def(7,	DRBD_GENLA_F_MANDATORY, twopc_retry_timeout, DRBD_TWOPC_RETRY_TIMEOUT_DEF)
+	__u32_field_def(8,	0 /* OPTIONAL */,	peer_ack_delay, DRBD_PEER_ACK_DELAY_DEF)
+	__u32_field_def(9,	0 /* OPTIONAL */,	auto_promote_timeout, DRBD_AUTO_PROMOTE_TIMEOUT_DEF)
+	__u32_field_def(10,	0 /* OPTIONAL */,	nr_requests, DRBD_NR_REQUESTS_DEF)
+	__s32_field_def(11,	0 /* OPTIONAL */,	quorum, DRBD_QUORUM_DEF)
+	__u32_field_def(12,     0 /* OPTIONAL */,	on_no_quorum, DRBD_ON_NO_QUORUM_DEF)
+	__s32_field_def(13,	0 /* OPTIONAL */,	quorum_min_redundancy, DRBD_QUORUM_DEF)
+	__u32_field_def(14,	0 /* OPTIONAL */,	on_susp_primary_outdated, DRBD_ON_SUSP_PRI_OUTD_DEF)
+	__flg_field_def(15, 0, drbd8_compat_mode, DRBD_DRBD8_COMPAT_MODE_DEF) /* invisible by drbdsetup show */
+	__flg_field_def(16,	0 /* OPTIONAL */,	explicit_drbd8_compat, DRBD_DRBD8_COMPAT_MODE_DEF)
 )
 
 GENL_struct(DRBD_NLA_NET_CONF, 5, net_conf,
@@ -157,9 +167,7 @@ GENL_struct(DRBD_NLA_NET_CONF, 5, net_conf,
 	__u32_field_def(11,	DRBD_GENLA_F_MANDATORY,	sndbuf_size, DRBD_SNDBUF_SIZE_DEF)
 	__u32_field_def(12,	DRBD_GENLA_F_MANDATORY,	rcvbuf_size, DRBD_RCVBUF_SIZE_DEF)
 	__u32_field_def(13,	DRBD_GENLA_F_MANDATORY,	ko_count, DRBD_KO_COUNT_DEF)
-	__u32_field_def(14,	DRBD_GENLA_F_MANDATORY,	max_buffers, DRBD_MAX_BUFFERS_DEF)
 	__u32_field_def(15,	DRBD_GENLA_F_MANDATORY,	max_epoch_size, DRBD_MAX_EPOCH_SIZE_DEF)
-	__u32_field_def(16,	DRBD_GENLA_F_MANDATORY,	unplug_watermark, DRBD_UNPLUG_WATERMARK_DEF)
 	__u32_field_def(17,	DRBD_GENLA_F_MANDATORY,	after_sb_0p, DRBD_AFTER_SB_0P_DEF)
 	__u32_field_def(18,	DRBD_GENLA_F_MANDATORY,	after_sb_1p, DRBD_AFTER_SB_1P_DEF)
 	__u32_field_def(19,	DRBD_GENLA_F_MANDATORY,	after_sb_2p, DRBD_AFTER_SB_2P_DEF)
@@ -168,20 +176,29 @@ GENL_struct(DRBD_NLA_NET_CONF, 5, net_conf,
 	__u32_field_def(22,	DRBD_GENLA_F_MANDATORY,	cong_fill, DRBD_CONG_FILL_DEF)
 	__u32_field_def(23,	DRBD_GENLA_F_MANDATORY,	cong_extents, DRBD_CONG_EXTENTS_DEF)
 	__flg_field_def(24, DRBD_GENLA_F_MANDATORY,	two_primaries, DRBD_ALLOW_TWO_PRIMARIES_DEF)
-	__flg_field(25, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT,	discard_my_data)
 	__flg_field_def(26, DRBD_GENLA_F_MANDATORY,	tcp_cork, DRBD_TCP_CORK_DEF)
 	__flg_field_def(27, DRBD_GENLA_F_MANDATORY,	always_asbp, DRBD_ALWAYS_ASBP_DEF)
-	__flg_field(28, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT,	tentative)
 	__flg_field_def(29,	DRBD_GENLA_F_MANDATORY,	use_rle, DRBD_USE_RLE_DEF)
-	/* 9: __u32_field_def(30,	DRBD_GENLA_F_MANDATORY,	fencing_policy, DRBD_FENCING_DEF) */
-	/* 9: __str_field_def(31,     DRBD_GENLA_F_MANDATORY, name, SHARED_SECRET_MAX) */
-	/* 9: __u32_field(32,         DRBD_F_REQUIRED | DRBD_F_INVARIANT,     peer_node_id) */
+	__u32_field_def(30,	DRBD_GENLA_F_MANDATORY,	fencing_policy, DRBD_FENCING_DEF)
+	__str_field_def(31,	DRBD_GENLA_F_MANDATORY, name, SHARED_SECRET_MAX)
+	/* moved into ctx_peer_node_id: __u32_field(32,		DRBD_F_REQUIRED | DRBD_F_INVARIANT,	peer_node_id) */
 	__flg_field_def(33, 0 /* OPTIONAL */,	csums_after_crash_only, DRBD_CSUMS_AFTER_CRASH_ONLY_DEF)
 	__u32_field_def(34, 0 /* OPTIONAL */, sock_check_timeo, DRBD_SOCKET_CHECK_TIMEO_DEF)
+	__str_field_def(35, DRBD_F_INVARIANT, transport_name, SHARED_SECRET_MAX)
+	__u32_field_def(36, 0 /* OPTIONAL */, max_buffers, DRBD_MAX_BUFFERS_DEF)
+	__flg_field_def(37, 0 /* OPTIONAL */, allow_remote_read, DRBD_ALLOW_REMOTE_READ_DEF)
+	__flg_field_def(38, 0 /* OPTIONAL */, tls, DRBD_TLS_DEF)
+	__s32_field_def(39, 0 /* OPTIONAL */, tls_privkey, DRBD_TLS_PRIVKEY_DEF)
+	__s32_field_def(40, 0 /* OPTIONAL */, tls_certificate, DRBD_TLS_CERTIFICATE_DEF)
+	__s32_field_def(41, 0 /* OPTIONAL */, tls_keyring, DRBD_TLS_KEYRING_DEF)
+	__flg_field_def(42, DRBD_F_INVARIANT, load_balance_paths, DRBD_LOAD_BALANCE_PATHS_DEF)
+	__u32_field_def(43, 0 /* OPTIONAL */, rdma_ctrl_rcvbuf_size, DRBD_RDMA_CTRL_RCVBUF_SIZE_DEF)
+	__u32_field_def(44, 0 /* OPTIONAL */, rdma_ctrl_sndbuf_size, DRBD_RDMA_CTRL_SNDBUF_SIZE_DEF)
+
 )
 
 GENL_struct(DRBD_NLA_SET_ROLE_PARMS, 6, set_role_parms,
-	__flg_field(1, DRBD_GENLA_F_MANDATORY,	assume_uptodate)
+	__flg_field(1, DRBD_GENLA_F_MANDATORY,	force)
 )
 
 GENL_struct(DRBD_NLA_RESIZE_PARMS, 7, resize_parms,
@@ -192,46 +209,6 @@ GENL_struct(DRBD_NLA_RESIZE_PARMS, 7, resize_parms,
 	__u32_field_def(5, 0 /* OPTIONAL */, al_stripe_size, DRBD_AL_STRIPE_SIZE_DEF)
 )
 
-GENL_struct(DRBD_NLA_STATE_INFO, 8, state_info,
-	/* the reason of the broadcast,
-	 * if this is an event triggered broadcast. */
-	__u32_field(1, DRBD_GENLA_F_MANDATORY,	sib_reason)
-	__u32_field(2, DRBD_F_REQUIRED,	current_state)
-	__u64_field(3, DRBD_GENLA_F_MANDATORY,	capacity)
-	__u64_field(4, DRBD_GENLA_F_MANDATORY,	ed_uuid)
-
-	/* These are for broadcast from after state change work.
-	 * prev_state and new_state are from the moment the state change took
-	 * place, new_state is not neccessarily the same as current_state,
-	 * there may have been more state changes since.  Which will be
-	 * broadcasted soon, in their respective after state change work.  */
-	__u32_field(5, DRBD_GENLA_F_MANDATORY,	prev_state)
-	__u32_field(6, DRBD_GENLA_F_MANDATORY,	new_state)
-
-	/* if we have a local disk: */
-	__bin_field(7, DRBD_GENLA_F_MANDATORY,	uuids, (UI_SIZE*sizeof(__u64)))
-	__u32_field(8, DRBD_GENLA_F_MANDATORY,	disk_flags)
-	__u64_field(9, DRBD_GENLA_F_MANDATORY,	bits_total)
-	__u64_field(10, DRBD_GENLA_F_MANDATORY,	bits_oos)
-	/* and in case resync or online verify is active */
-	__u64_field(11, DRBD_GENLA_F_MANDATORY,	bits_rs_total)
-	__u64_field(12, DRBD_GENLA_F_MANDATORY,	bits_rs_failed)
-
-	/* for pre and post notifications of helper execution */
-	__str_field(13, DRBD_GENLA_F_MANDATORY,	helper, 32)
-	__u32_field(14, DRBD_GENLA_F_MANDATORY,	helper_exit_code)
-
-	__u64_field(15,                      0, send_cnt)
-	__u64_field(16,                      0, recv_cnt)
-	__u64_field(17,                      0, read_cnt)
-	__u64_field(18,                      0, writ_cnt)
-	__u64_field(19,                      0, al_writ_cnt)
-	__u64_field(20,                      0, bm_writ_cnt)
-	__u32_field(21,                      0, ap_bio_cnt)
-	__u32_field(22,                      0, ap_pending_cnt)
-	__u32_field(23,                      0, rs_pending_cnt)
-)
-
 GENL_struct(DRBD_NLA_START_OV_PARMS, 9, start_ov_parms,
 	__u64_field(1, DRBD_GENLA_F_MANDATORY,	ov_start_sector)
 	__u64_field(2, DRBD_GENLA_F_MANDATORY,	ov_stop_sector)
@@ -239,6 +216,7 @@ GENL_struct(DRBD_NLA_START_OV_PARMS, 9, start_ov_parms,
 
 GENL_struct(DRBD_NLA_NEW_C_UUID_PARMS, 10, new_c_uuid_parms,
 	__flg_field(1, DRBD_GENLA_F_MANDATORY, clear_bm)
+	__flg_field(2, DRBD_GENLA_F_MANDATORY, force_resync)
 )
 
 GENL_struct(DRBD_NLA_TIMEOUT_PARMS, 11, timeout_parms,
@@ -251,6 +229,13 @@ GENL_struct(DRBD_NLA_DISCONNECT_PARMS, 12, disconnect_parms,
 
 GENL_struct(DRBD_NLA_DETACH_PARMS, 13, detach_parms,
 	__flg_field(1, DRBD_GENLA_F_MANDATORY,	force_detach)
+	__flg_field_def(2, 0 /* OPTIONAL */, intentional_diskless_detach, DRBD_DISK_DISKLESS_DEF)
+)
+
+GENL_struct(DRBD_NLA_DEVICE_CONF, 14, device_conf,
+	__u32_field_def(1, DRBD_F_INVARIANT,	max_bio_size, DRBD_MAX_BIO_SIZE_DEF)
+	__flg_field_def(2, 0 /* OPTIONAL */, intentional_diskless, DRBD_DISK_DISKLESS_DEF)
+	__u32_field_def(3, 0 /* OPTIONAL */, block_size, DRBD_BLOCK_SIZE_DEF)
 )
 
 GENL_struct(DRBD_NLA_RESOURCE_INFO, 15, resource_info,
@@ -258,11 +243,16 @@ GENL_struct(DRBD_NLA_RESOURCE_INFO, 15, resource_info,
 	__flg_field(2, 0, res_susp)
 	__flg_field(3, 0, res_susp_nod)
 	__flg_field(4, 0, res_susp_fen)
-	/* __flg_field(5, 0, res_weak) */
+	__flg_field(5, 0, res_susp_quorum)
+	__flg_field(6, 0, res_fail_io)
 )
 
 GENL_struct(DRBD_NLA_DEVICE_INFO, 16, device_info,
 	__u32_field(1, 0, dev_disk_state)
+	__flg_field(2, 0, is_intentional_diskless)
+	__flg_field(3, 0, dev_has_quorum)
+	__flg_field(5, 0, dev_is_open)
+	__str_field(4, 0, backing_dev_path, 128)
 )
 
 GENL_struct(DRBD_NLA_CONNECTION_INFO, 17, connection_info,
@@ -276,6 +266,7 @@ GENL_struct(DRBD_NLA_PEER_DEVICE_INFO, 18, peer_device_info,
 	__u32_field(3, 0, peer_resync_susp_user)
 	__u32_field(4, 0, peer_resync_susp_peer)
 	__u32_field(5, 0, peer_resync_susp_dependency)
+	__flg_field(6, 0, peer_is_intentional_diskless)
 )
 
 GENL_struct(DRBD_NLA_RESOURCE_STATISTICS, 19, resource_statistics,
@@ -301,6 +292,8 @@ GENL_struct(DRBD_NLA_DEVICE_STATISTICS, 20, device_statistics,
 
 GENL_struct(DRBD_NLA_CONNECTION_STATISTICS, 21, connection_statistics,
 	__flg_field(1, 0, conn_congested)
+	__u64_field(2, 0, ap_in_flight) /* sectors */
+	__u64_field(3, 0, rs_in_flight) /* sectors */
 )
 
 GENL_struct(DRBD_NLA_PEER_DEVICE_STATISTICS, 22, peer_device_statistics,
@@ -312,6 +305,27 @@ GENL_struct(DRBD_NLA_PEER_DEVICE_STATISTICS, 22, peer_device_statistics,
 	__u64_field(6, 0, peer_dev_resync_failed)  /* sectors */
 	__u64_field(7, 0, peer_dev_bitmap_uuid)
 	__u32_field(9, 0, peer_dev_flags)
+	/* you need the peer_repl_state from peer_device_info
+	 * to properly interpret these stats for "progress"
+	 * of syncer/verify */
+	__u64_field(10, 0, peer_dev_rs_total)	/* sectors */
+	__u64_field(11, 0, peer_dev_ov_start_sector)
+	__u64_field(12, 0, peer_dev_ov_stop_sector)
+	__u64_field(13, 0, peer_dev_ov_position) /* sectors */
+	__u64_field(14, 0, peer_dev_ov_left)	/* sectors */
+	__u64_field(15, 0, peer_dev_ov_skipped)	/* sectors */
+	__u64_field(16, 0, peer_dev_rs_same_csum)
+	__u64_field(17, 0, peer_dev_rs_dt_start_ms)
+	__u64_field(18, 0, peer_dev_rs_paused_ms)
+	/* resync progress marks for "resync speed" guestimation */
+	__u64_field(19, 0, peer_dev_rs_dt0_ms)
+	__u64_field(20, 0, peer_dev_rs_db0_sectors)
+	__u64_field(21, 0, peer_dev_rs_dt1_ms)
+	__u64_field(22, 0, peer_dev_rs_db1_sectors)
+	__u32_field(23, 0, peer_dev_rs_c_sync_rate)
+	/* events may not be sent for every change of the UUID flags, however
+	 * UUID_FLAG_STABLE can be trusted */
+	__u64_field(24, 0, peer_dev_uuid_flags)
 )
 
 GENL_struct(DRBD_NLA_NOTIFICATION_HEADER, 23, drbd_notification_header,
@@ -323,38 +337,67 @@ GENL_struct(DRBD_NLA_HELPER, 24, drbd_helper_info,
 	__u32_field(2, DRBD_GENLA_F_MANDATORY, helper_status)
 )
 
-/*
- * Notifications and commands (genlmsghdr->cmd)
- */
-GENL_mc_group(events)
+GENL_struct(DRBD_NLA_INVALIDATE_PARMS, 25, invalidate_parms,
+	__s32_field_def(1, DRBD_GENLA_F_MANDATORY, sync_from_peer_node_id, DRBD_SYNC_FROM_NID_DEF)
+	__flg_field_def(2, DRBD_GENLA_F_MANDATORY, reset_bitmap, DRBD_INVALIDATE_RESET_BITMAP_DEF)
+)
 
-	/* kernel -> userspace announcement of changes */
-GENL_notification(
-	DRBD_EVENT, 1, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_STATE_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_GENLA_F_MANDATORY)
-	GENL_tla_expected(DRBD_NLA_DISK_CONF, DRBD_GENLA_F_MANDATORY)
-	GENL_tla_expected(DRBD_NLA_SYNCER_CONF, DRBD_GENLA_F_MANDATORY)
+GENL_struct(DRBD_NLA_FORGET_PEER_PARMS, 26, forget_peer_parms,
+	__s32_field_def(1, DRBD_GENLA_F_MANDATORY, forget_peer_node_id, DRBD_SYNC_FROM_NID_DEF)
 )
 
-	/* query kernel for specific or all info */
-GENL_op(
-	DRBD_ADM_GET_STATUS, 2,
-	GENL_op_init(
-		.doit = drbd_adm_get_status,
-		.dumpit = drbd_adm_get_status_all,
-		/* anyone may ask for the status,
-		 * it is broadcasted anyways */
-	),
-	/* To select the object .doit.
-	 * Or a subset of objects in .dumpit. */
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+GENL_struct(DRBD_NLA_PEER_DEVICE_OPTS, 27, peer_device_conf,
+	__u32_field_def(1,	DRBD_GENLA_F_MANDATORY,	resync_rate, DRBD_RESYNC_RATE_DEF)
+	__u32_field_def(2,	DRBD_GENLA_F_MANDATORY,	c_plan_ahead, DRBD_C_PLAN_AHEAD_DEF)
+	__u32_field_def(3,	DRBD_GENLA_F_MANDATORY,	c_delay_target, DRBD_C_DELAY_TARGET_DEF)
+	__u32_field_def(4,	DRBD_GENLA_F_MANDATORY,	c_fill_target, DRBD_C_FILL_TARGET_DEF)
+	__u32_field_def(5,	DRBD_GENLA_F_MANDATORY,	c_max_rate, DRBD_C_MAX_RATE_DEF)
+	__u32_field_def(6,	DRBD_GENLA_F_MANDATORY,	c_min_rate, DRBD_C_MIN_RATE_DEF)
+	__flg_field_def(7, 0 /* OPTIONAL */, bitmap, DRBD_BITMAP_DEF)
+#if (PRO_FEATURES & DRBD_FF_RESYNC_WITHOUT_REPLICATION) || !defined(__KERNEL__)
+	__flg_field_def(8, 0 /* OPTIONAL */, resync_without_replication, DRBD_RESYNC_WITHOUT_REPLICATION_DEF)
+#endif
+)
+
+GENL_struct(DRBD_NLA_PATH_PARMS, 28, path_parms,
+	__bin_field(1, DRBD_GENLA_F_MANDATORY,	my_addr, 128)
+	__bin_field(2, DRBD_GENLA_F_MANDATORY,	peer_addr, 128)
+)
+
+GENL_struct(DRBD_NLA_CONNECT_PARMS, 29, connect_parms,
+	__flg_field_def(1,	DRBD_GENLA_F_MANDATORY,	tentative, 0)
+	__flg_field_def(2,	DRBD_GENLA_F_MANDATORY,	discard_my_data, 0)
+)
+
+GENL_struct(DRBD_NLA_PATH_INFO, 30, drbd_path_info,
+	__flg_field(1, 0, path_established)
 )
 
+GENL_struct(DRBD_NLA_RENAME_RESOURCE_PARMS, 31, rename_resource_parms,
+	__str_field(1, DRBD_GENLA_F_MANDATORY, new_resource_name, 128)
+)
+
+GENL_struct(DRBD_NLA_RENAME_RESOURCE_INFO, 32, rename_resource_info,
+	__str_field(1, DRBD_GENLA_F_MANDATORY, res_new_name, 128)
+)
+
+GENL_struct(DRBD_NLA_INVAL_PEER_PARAMS, 33, invalidate_peer_parms,
+	__flg_field_def(1, DRBD_GENLA_F_MANDATORY, p_reset_bitmap, DRBD_INVALIDATE_RESET_BITMAP_DEF)
+)
+
+GENL_struct(DRBD_NLA_SUSPEND_IO_PARAMS, 34, suspend_io_parms,
+	__flg_field_def(1, DRBD_GENLA_F_MANDATORY, bdev_freeze, DRBD_SUSPEND_IO_BDEV_FREEZE_DEF)
+)
+
+/*
+ * Notifications and commands (genlmsghdr->cmd)
+ */
+GENL_mc_group(events)
+
 	/* add DRBD minor devices as volumes to resources */
 GENL_op(DRBD_ADM_NEW_MINOR, 5, GENL_doit(drbd_adm_new_minor),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_DEVICE_CONF, DRBD_GENLA_F_MANDATORY))
 GENL_op(DRBD_ADM_DEL_MINOR, 6, GENL_doit(drbd_adm_del_minor),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
 
@@ -370,11 +413,29 @@ GENL_op(DRBD_ADM_RESOURCE_OPTS, 9,
 	GENL_tla_expected(DRBD_NLA_RESOURCE_OPTS, DRBD_GENLA_F_MANDATORY)
 )
 
-GENL_op(
-	DRBD_ADM_CONNECT, 10,
-	GENL_doit(drbd_adm_connect),
+GENL_op(DRBD_ADM_NEW_PEER, 44, GENL_doit(drbd_adm_new_peer),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_GENLA_F_MANDATORY)
+)
+
+GENL_op(DRBD_ADM_NEW_PATH, 45, GENL_doit(drbd_adm_new_path),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_PATH_PARMS, DRBD_F_REQUIRED)
+)
+
+GENL_op(DRBD_ADM_DEL_PEER, 46, GENL_doit(drbd_adm_del_peer),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_DISCONNECT_PARMS, DRBD_GENLA_F_MANDATORY)
+)
+
+GENL_op(DRBD_ADM_DEL_PATH, 47, GENL_doit(drbd_adm_del_path),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_PATH_PARMS, DRBD_F_REQUIRED)
+)
+
+GENL_op(DRBD_ADM_CONNECT, 10, GENL_doit(drbd_adm_connect),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_CONNECT_PARMS, DRBD_GENLA_F_MANDATORY)
 )
 
 GENL_op(
@@ -385,7 +446,9 @@ GENL_op(
 )
 
 GENL_op(DRBD_ADM_DISCONNECT, 11, GENL_doit(drbd_adm_disconnect),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_DISCONNECT_PARMS, DRBD_GENLA_F_MANDATORY)
+)
 
 GENL_op(DRBD_ADM_ATTACH, 12,
 	GENL_doit(drbd_adm_attach),
@@ -438,15 +501,22 @@ GENL_op(DRBD_ADM_DETACH,	18, GENL_doit(drbd_adm_detach),
 	GENL_tla_expected(DRBD_NLA_DETACH_PARMS, DRBD_GENLA_F_MANDATORY))
 
 GENL_op(DRBD_ADM_INVALIDATE,	19, GENL_doit(drbd_adm_invalidate),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_INVALIDATE_PARMS, DRBD_F_REQUIRED))
+
 GENL_op(DRBD_ADM_INVAL_PEER,	20, GENL_doit(drbd_adm_invalidate_peer),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_INVAL_PEER_PARAMS, 0 /* OPTIONAL */))
+
 GENL_op(DRBD_ADM_PAUSE_SYNC,	21, GENL_doit(drbd_adm_pause_sync),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
 GENL_op(DRBD_ADM_RESUME_SYNC,	22, GENL_doit(drbd_adm_resume_sync),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+
 GENL_op(DRBD_ADM_SUSPEND_IO,	23, GENL_doit(drbd_adm_suspend_io),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_SUSPEND_IO_PARAMS, 0 /* OPTIONAL */))
+
 GENL_op(DRBD_ADM_RESUME_IO,	24, GENL_doit(drbd_adm_resume_io),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
 GENL_op(DRBD_ADM_OUTDATE,	25, GENL_doit(drbd_adm_outdate),
@@ -457,39 +527,47 @@ GENL_op(DRBD_ADM_DOWN,		27, GENL_doit(drbd_adm_down),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
 
 GENL_op(DRBD_ADM_GET_RESOURCES, 30,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_resources,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, DRBD_GENLA_F_MANDATORY))
+	GENL_op_init(
+		.dumpit = drbd_adm_dump_resources,
+	),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, DRBD_GENLA_F_MANDATORY))
 
 GENL_op(DRBD_ADM_GET_DEVICES, 31,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_devices,
-		 .done = drbd_adm_dump_devices_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_DEVICE_INFO, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY))
+	GENL_op_init(
+		.dumpit = drbd_adm_dump_devices,
+		.done = drbd_adm_dump_devices_done,
+	),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_DEVICE_INFO, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY))
 
 GENL_op(DRBD_ADM_GET_CONNECTIONS, 32,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_connections,
-		 .done = drbd_adm_dump_connections_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_GENLA_F_MANDATORY))
+	GENL_op_init(
+		.dumpit = drbd_adm_dump_connections,
+		.done = drbd_adm_dump_connections_done,
+	),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_GENLA_F_MANDATORY))
 
 GENL_op(DRBD_ADM_GET_PEER_DEVICES, 33,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_peer_devices,
-		 .done = drbd_adm_dump_peer_devices_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, DRBD_GENLA_F_MANDATORY)
-	 GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY))
+	GENL_op_init(
+		.dumpit = drbd_adm_dump_peer_devices,
+		.done = drbd_adm_dump_peer_devices_done,
+	),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY))
+
+GENL_op(DRBD_ADM_GET_PATHS, 50,
+	GENL_op_init(
+		.dumpit = drbd_adm_dump_paths,
+		.done = drbd_adm_dump_paths_done,
+	),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)
+	GENL_tla_expected(DRBD_NLA_PATH_INFO, DRBD_GENLA_F_MANDATORY))
 
 GENL_notification(
 	DRBD_RESOURCE_STATE, 34, events,
@@ -509,6 +587,7 @@ GENL_notification(
 	DRBD_CONNECTION_STATE, 36, events,
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
 	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_PATH_PARMS, DRBD_GENLA_F_MANDATORY)
 	GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_F_REQUIRED)
 	GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_F_REQUIRED))
 
@@ -522,7 +601,8 @@ GENL_notification(
 GENL_op(
 	DRBD_ADM_GET_INITIAL_STATE, 38,
 	GENL_op_init(
-	        .dumpit = drbd_adm_get_initial_state,
+		.dumpit = drbd_adm_get_initial_state,
+		.done = drbd_adm_get_initial_state_done,
 	),
 	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY))
 
@@ -534,3 +614,21 @@ GENL_notification(
 GENL_notification(
 	DRBD_INITIAL_STATE_DONE, 41, events,
 	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED))
+
+GENL_op(DRBD_ADM_FORGET_PEER,		42, GENL_doit(drbd_adm_forget_peer),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_FORGET_PEER_PARMS, DRBD_F_REQUIRED))
+
+GENL_op(DRBD_ADM_CHG_PEER_DEVICE_OPTS, 43,
+	GENL_doit(drbd_adm_peer_device_opts),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_PEER_DEVICE_OPTS, DRBD_F_REQUIRED))
+
+GENL_op(DRBD_ADM_RENAME_RESOURCE,		49, GENL_doit(drbd_adm_rename_resource),
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_RENAME_RESOURCE_PARMS, DRBD_F_REQUIRED))
+
+GENL_notification(
+	DRBD_PATH_STATE, 48, events,
+	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
+	GENL_tla_expected(DRBD_NLA_PATH_INFO, DRBD_F_REQUIRED))
diff --git a/include/linux/drbd_genl_api.h b/include/linux/drbd_genl_api.h
index 70682c0580270..c89b0afa6f9c6 100644
--- a/include/linux/drbd_genl_api.h
+++ b/include/linux/drbd_genl_api.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: GPL-2.0-only */
 #ifndef DRBD_GENL_STRUCT_H
 #define DRBD_GENL_STRUCT_H
 
@@ -13,12 +13,6 @@
  *     is used instead.
  * @flags: possible operation modifiers (relevant only for user->kernel):
  *     DRBD_GENL_F_SET_DEFAULTS
- * @volume:
- *     When creating a new minor (adding it to a resource), the resource needs
- *     to know which volume number within the resource this is supposed to be.
- *     The volume number corresponds to the same volume number on the remote side,
- *     whereas the minor number on the remote side may be different
- *     (union with flags).
  * @ret_code: kernel->userland unicast cfg reply return code (union with flags);
  */
 struct drbd_genlmsghdr {
@@ -34,20 +28,13 @@ enum {
 	DRBD_GENL_F_SET_DEFAULTS = 1,
 };
 
-enum drbd_state_info_bcast_reason {
-	SIB_GET_STATUS_REPLY = 1,
-	SIB_STATE_CHANGE = 2,
-	SIB_HELPER_PRE = 3,
-	SIB_HELPER_POST = 4,
-	SIB_SYNC_PROGRESS = 5,
-};
-
 /* hack around predefined gcc/cpp "linux=1",
  * we cannot possibly include <1/drbd_genl.h> */
 #undef linux
 
 #include <linux/drbd.h>
-#define GENL_MAGIC_VERSION	1
+#include <linux/drbd_config.h>
+#define GENL_MAGIC_VERSION	2
 #define GENL_MAGIC_FAMILY	drbd
 #define GENL_MAGIC_FAMILY_HDRSZ	sizeof(struct drbd_genlmsghdr)
 #define GENL_MAGIC_INCLUDE_FILE <linux/drbd_genl.h>
diff --git a/include/linux/drbd_limits.h b/include/linux/drbd_limits.h
index 5b042fb427e97..ed38f94d43c66 100644
--- a/include/linux/drbd_limits.h
+++ b/include/linux/drbd_limits.h
@@ -64,7 +64,7 @@
 #define DRBD_DISK_TIMEOUT_DEF 0U    /* disabled */
 #define DRBD_DISK_TIMEOUT_SCALE '1'
 
-  /* active connection retries when C_WF_CONNECTION */
+  /* active connection retries when C_CONNECTING */
 #define DRBD_CONNECT_INT_MIN 1U
 #define DRBD_CONNECT_INT_MAX 120U
 #define DRBD_CONNECT_INT_DEF 10U   /* seconds */
@@ -88,14 +88,13 @@
 #define DRBD_MAX_EPOCH_SIZE_DEF 2048U
 #define DRBD_MAX_EPOCH_SIZE_SCALE '1'
 
-  /* I don't think that a tcp send buffer of more than 10M is useful */
 #define DRBD_SNDBUF_SIZE_MIN  0U
-#define DRBD_SNDBUF_SIZE_MAX  (10U<<20)
+#define DRBD_SNDBUF_SIZE_MAX  (128U<<20)
 #define DRBD_SNDBUF_SIZE_DEF  0U
 #define DRBD_SNDBUF_SIZE_SCALE '1'
 
 #define DRBD_RCVBUF_SIZE_MIN  0U
-#define DRBD_RCVBUF_SIZE_MAX  (10U<<20)
+#define DRBD_RCVBUF_SIZE_MAX  (128U<<20)
 #define DRBD_RCVBUF_SIZE_DEF  0U
 #define DRBD_RCVBUF_SIZE_SCALE '1'
 
@@ -117,16 +116,19 @@
 #define DRBD_KO_COUNT_MAX  200U
 #define DRBD_KO_COUNT_DEF  7U
 #define DRBD_KO_COUNT_SCALE '1'
+
+#define DRBD_ALLOW_REMOTE_READ_DEF 1U
 /* } */
 
 /* syncer { */
   /* FIXME allow rate to be zero? */
 #define DRBD_RESYNC_RATE_MIN 1U
 /* channel bonding 10 GbE, or other hardware */
-#define DRBD_RESYNC_RATE_MAX (4 << 20)
+#define DRBD_RESYNC_RATE_MAX (8U << 20)
 #define DRBD_RESYNC_RATE_DEF 250U
 #define DRBD_RESYNC_RATE_SCALE 'k'  /* kilobytes */
 
+  /* less than 67 would hit performance unnecessarily. */
 #define DRBD_AL_EXTENTS_MIN  67U
   /* we use u16 as "slot number", (u16)~0 is "FREE".
    * If you use >= 292 kB on-disk ring buffer,
@@ -182,7 +184,7 @@
 #define DRBD_C_FILL_TARGET_DEF 100U /* Try to place 50KiB in socket send buffer during resync */
 #define DRBD_C_FILL_TARGET_SCALE 's'  /* sectors */
 
-#define DRBD_C_MAX_RATE_MIN     250U
+#define DRBD_C_MAX_RATE_MIN     0U
 #define DRBD_C_MAX_RATE_MAX     (4U << 20)
 #define DRBD_C_MAX_RATE_DEF     102400U
 #define DRBD_C_MAX_RATE_SCALE	'k'  /* kilobytes */
@@ -207,10 +209,11 @@
 #define DRBD_DISK_BARRIER_DEF	0U
 #define DRBD_DISK_FLUSHES_DEF	1U
 #define DRBD_DISK_DRAIN_DEF	1U
+#define DRBD_DISK_DISKLESS_DEF	0U
 #define DRBD_MD_FLUSHES_DEF	1U
 #define DRBD_TCP_CORK_DEF	1U
 #define DRBD_AL_UPDATES_DEF     1U
-
+#define DRBD_INVALIDATE_RESET_BITMAP_DEF 1U
 /* We used to ignore the discard_zeroes_data setting.
  * To not change established (and expected) behaviour,
  * by default assume that, for discard_zeroes_data=0,
@@ -227,6 +230,52 @@
 #define DRBD_ALWAYS_ASBP_DEF	0U
 #define DRBD_USE_RLE_DEF	1U
 #define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0U
+#define DRBD_AUTO_PROMOTE_DEF	1U
+#define DRBD_BITMAP_DEF         1U
+#define DRBD_RESYNC_WITHOUT_REPLICATION_DEF 1U
+
+#define DRBD_NR_REQUESTS_MIN	4U
+#define DRBD_NR_REQUESTS_DEF	8000U
+#define DRBD_NR_REQUESTS_MAX	-1U
+#define DRBD_NR_REQUESTS_SCALE	'1'
+
+#define DRBD_MAX_BIO_SIZE_DEF	DRBD_MAX_BIO_SIZE
+#define DRBD_MAX_BIO_SIZE_MIN	(1U << 9)
+#define DRBD_MAX_BIO_SIZE_MAX	DRBD_MAX_BIO_SIZE
+#define DRBD_MAX_BIO_SIZE_SCALE '1'
+
+#define DRBD_NODE_ID_DEF		0U
+#define DRBD_NODE_ID_MIN		0U
+#ifndef DRBD_NODE_ID_MAX /* Is also defined in drbd.h */
+#define DRBD_NODE_ID_MAX		DRBD_PEERS_MAX
+#endif
+#define DRBD_NODE_ID_SCALE		'1'
+
+#define DRBD_PEER_ACK_WINDOW_DEF	4096U   /* 2 MiByte */
+#define DRBD_PEER_ACK_WINDOW_MIN	2048U   /* 1 MiByte */
+#define DRBD_PEER_ACK_WINDOW_MAX	204800U /* 100 MiByte */
+#define DRBD_PEER_ACK_WINDOW_SCALE 's' /* sectors*/
+
+#define DRBD_PEER_ACK_DELAY_DEF	100U    /* 100ms */
+#define DRBD_PEER_ACK_DELAY_MIN 1U
+#define DRBD_PEER_ACK_DELAY_MAX 10000U  /* 10 seconds */
+#define DRBD_PEER_ACK_DELAY_SCALE '1' /* milliseconds */
+
+/* Two-phase commit timeout (1/10 seconds). */
+#define DRBD_TWOPC_TIMEOUT_MIN	50U
+#define DRBD_TWOPC_TIMEOUT_MAX	600U
+#define DRBD_TWOPC_TIMEOUT_DEF	300U
+#define DRBD_TWOPC_TIMEOUT_SCALE '1'
+
+#define DRBD_TWOPC_RETRY_TIMEOUT_MIN 1U
+#define DRBD_TWOPC_RETRY_TIMEOUT_MAX 50U
+#define DRBD_TWOPC_RETRY_TIMEOUT_DEF 1U
+#define DRBD_TWOPC_RETRY_TIMEOUT_SCALE '1'
+
+#define DRBD_SYNC_FROM_NID_DEF -1
+#define DRBD_SYNC_FROM_NID_MIN -1
+#define DRBD_SYNC_FROM_NID_MAX DRBD_PEERS_MAX
+#define DRBD_SYNC_FROM_NID_SCALE '1'
 
 #define DRBD_AL_STRIPES_MIN     1U
 #define DRBD_AL_STRIPES_MAX     1024U
@@ -243,9 +292,51 @@
 #define DRBD_SOCKET_CHECK_TIMEO_DEF 0U
 #define DRBD_SOCKET_CHECK_TIMEO_SCALE '1'
 
+/* Auto promote timeout (1/10 seconds). */
+#define DRBD_AUTO_PROMOTE_TIMEOUT_MIN 0U
+#define DRBD_AUTO_PROMOTE_TIMEOUT_MAX 600U
+#define DRBD_AUTO_PROMOTE_TIMEOUT_DEF 20U
+#define DRBD_AUTO_PROMOTE_TIMEOUT_SCALE '1'
+
 #define DRBD_RS_DISCARD_GRANULARITY_MIN 0U
 #define DRBD_RS_DISCARD_GRANULARITY_MAX (1U<<20)  /* 1MiByte */
 #define DRBD_RS_DISCARD_GRANULARITY_DEF 0U     /* disabled by default */
 #define DRBD_RS_DISCARD_GRANULARITY_SCALE '1' /* bytes */
 
+#define DRBD_QUORUM_MIN 0U
+#define DRBD_QUORUM_MAX QOU_ALL /* Note: user visible min/max different */
+#define DRBD_QUORUM_DEF QOU_OFF /* kernel min/max includes symbolic values */
+#define DRBD_QUORUM_SCALE '1' /* nodes */
+
+#define DRBD_BLOCK_SIZE_MIN 512
+#define DRBD_BLOCK_SIZE_MAX 4096
+#define DRBD_BLOCK_SIZE_DEF 512
+#define DRBD_BLOCK_SIZE_SCALE '1' /* Bytes */
+
+/* By default freeze IO, if set error all IOs as quick as possible */
+#define DRBD_ON_NO_QUORUM_DEF ONQ_SUSPEND_IO
+
+#define DRBD_ON_SUSP_PRI_OUTD_DEF SPO_DISCONNECT
+#define DRBD_DRBD8_COMPAT_MODE_DEF 0U
+
+#define DRBD_TLS_DEF 0U /* disabled by default */
+#define DRBD_TLS_PRIVKEY_DEF 0 /* disabled by default */
+#define DRBD_TLS_CERTIFICATE_DEF 0 /* disabled by default */
+#define DRBD_TLS_KEYRING_DEF 0 /* disabled by default */
+
+#define DRBD_LOAD_BALANCE_PATHS_DEF 0U
+
+#define DRBD_RDMA_CTRL_RCVBUF_SIZE_MIN  0U
+#define DRBD_RDMA_CTRL_RCVBUF_SIZE_MAX  (10U<<20)
+#define DRBD_RDMA_CTRL_RCVBUF_SIZE_DEF 0
+#define DRBD_RDMA_CTRL_RCVBUF_SIZE_SCALE '1'
+
+#define DRBD_RDMA_CTRL_SNDBUF_SIZE_MIN  0U
+#define DRBD_RDMA_CTRL_SNDBUF_SIZE_MAX  (10U<<20)
+#define DRBD_RDMA_CTRL_SNDBUF_SIZE_DEF 0
+#define DRBD_RDMA_CTRL_SNDBUF_SIZE_SCALE '1'
+
+/* Enable bdev_freeze/lockfs by default */
+#define DRBD_SUSPEND_IO_BDEV_FREEZE_DEF 1U
+
 #endif
diff --git a/include/linux/drbd_protocol.h b/include/linux/drbd_protocol.h
new file mode 100644
index 0000000000000..3dee1586596b1
--- /dev/null
+++ b/include/linux/drbd_protocol.h
@@ -0,0 +1,721 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __DRBD_PROTOCOL_H
+#define __DRBD_PROTOCOL_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#define u64 uint64_t
+#endif
+#include <linux/drbd.h>
+
+
+enum drbd_packet {
+	/* receiver (data socket) */
+	P_DATA		      = 0x00,
+	P_DATA_REPLY	      = 0x01, /* Response to P_DATA_REQUEST */
+	P_RS_DATA_REPLY	      = 0x02, /* Response to P_RS_DATA_REQUEST */
+	P_BARRIER	      = 0x03,
+	P_BITMAP	      = 0x04,
+	P_BECOME_SYNC_TARGET  = 0x05,
+	P_BECOME_SYNC_SOURCE  = 0x06,
+	P_UNPLUG_REMOTE	      = 0x07, /* Used at various times to hint the peer */
+	P_DATA_REQUEST	      = 0x08, /* Used to ask for a data block */
+	P_RS_DATA_REQUEST     = 0x09, /* Used to ask for a data block for resync */
+	P_SYNC_PARAM	      = 0x0a,
+	P_PROTOCOL	      = 0x0b,
+	P_UUIDS		      = 0x0c,
+	P_SIZES		      = 0x0d,
+	P_STATE		      = 0x0e,
+	P_SYNC_UUID	      = 0x0f,
+	P_AUTH_CHALLENGE      = 0x10,
+	P_AUTH_RESPONSE	      = 0x11,
+	P_STATE_CHG_REQ	      = 0x12,
+
+	/* asender (meta socket */
+	P_PING		      = 0x13,
+	P_PING_ACK	      = 0x14,
+	P_RECV_ACK	      = 0x15, /* Used in protocol B */
+	P_WRITE_ACK	      = 0x16, /* Used in protocol C */
+	P_RS_WRITE_ACK	      = 0x17, /* Write ack for resync reply. */
+	P_SUPERSEDED	      = 0x18, /* Used in proto C, two-primaries conflict detection */
+	P_NEG_ACK	      = 0x19, /* Sent if local disk is unusable */
+	P_NEG_DREPLY	      = 0x1a, /* Local disk is broken... */
+	P_NEG_RS_DREPLY	      = 0x1b, /* Local disk is broken... */
+	P_BARRIER_ACK	      = 0x1c,
+	P_STATE_CHG_REPLY     = 0x1d,
+
+	/* "new" commands, no longer fitting into the ordering scheme above */
+
+	P_OV_REQUEST	      = 0x1e, /* data socket */
+	P_OV_REPLY	      = 0x1f,
+	P_OV_RESULT	      = 0x20, /* meta sock: Protocol < 122 version of P_OV_RESULT_ID */
+	P_CSUM_RS_REQUEST     = 0x21, /* data socket */
+	P_RS_IS_IN_SYNC	      = 0x22, /* meta socket */
+	P_SYNC_PARAM89	      = 0x23, /* data socket, protocol version 89 replacement for P_SYNC_PARAM */
+	P_COMPRESSED_BITMAP   = 0x24, /* compressed or otherwise encoded bitmap transfer */
+	/* P_CKPT_FENCE_REQ      = 0x25, * currently reserved for protocol D */
+	/* P_CKPT_DISABLE_REQ    = 0x26, * currently reserved for protocol D */
+	P_DELAY_PROBE         = 0x27, /* is used on BOTH sockets */
+	P_OUT_OF_SYNC         = 0x28, /* Mark as out of sync (Outrunning), data socket */
+	P_RS_CANCEL           = 0x29, /* meta: Used to cancel RS_DATA_REQUEST packet by SyncSource */
+	P_CONN_ST_CHG_REQ     = 0x2a, /* data sock: state change request */
+	P_CONN_ST_CHG_REPLY   = 0x2b, /* meta sock: state change reply */
+	P_RETRY_WRITE	      = 0x2c, /* Protocol C: retry conflicting write request */
+	P_PROTOCOL_UPDATE     = 0x2d, /* data sock: is used in established connections */
+	P_TWOPC_PREPARE       = 0x2e, /* data sock: prepare state change */
+	P_TWOPC_ABORT         = 0x2f, /* data sock: abort state change */
+
+	P_DAGTAG	      = 0x30, /* data sock: set the current dagtag */
+
+	/* REQ_DISCARD. We used "discard" in different contexts before,
+	 * which is why I chose TRIM here, to disambiguate. */
+	P_TRIM                = 0x31,
+
+	/* Only use these two if both support FF_THIN_RESYNC */
+	P_RS_THIN_REQ         = 0x32, /* Request a block for resync or reply P_RS_DEALLOCATED */
+	P_RS_DEALLOCATED      = 0x33, /* Protocol < 122 version of P_RS_DEALLOCATED_ID */
+
+	/* REQ_WRITE_SAME.
+	 * On a receiving side without REQ_WRITE_SAME,
+	 * we may fall back to an opencoded loop instead. */
+	P_WSAME               = 0x34,
+	P_TWOPC_PREP_RSZ      = 0x35, /* PREPARE a 2PC resize operation*/
+	P_ZEROES              = 0x36, /* data sock: zero-out, WRITE_ZEROES */
+
+	/* place new packets for both 8.4 and 9 here,
+	 * place new packets for 9-only in the next gap. */
+
+	P_PEER_ACK            = 0x40, /* meta sock: tell which nodes have acked a request */
+	P_PEERS_IN_SYNC       = 0x41, /* data sock: Mark area as in sync */
+
+	P_UUIDS110	      = 0x42, /* data socket */
+	P_PEER_DAGTAG         = 0x43, /* data socket, used to trigger reconciliation resync */
+	P_CURRENT_UUID	      = 0x44, /* data socket */
+
+	P_TWOPC_YES           = 0x45, /* meta sock: allow two-phase commit */
+	P_TWOPC_NO            = 0x46, /* meta sock: reject two-phase commit */
+	P_TWOPC_COMMIT        = 0x47, /* data sock: commit state change */
+	P_TWOPC_RETRY         = 0x48, /* meta sock: retry two-phase commit */
+
+	P_CONFIRM_STABLE      = 0x49, /* meta sock: similar to an unsolicited partial barrier ack */
+	P_RS_CANCEL_AHEAD     = 0x4a, /* protocol version 115,
+		 * meta: cancel RS_DATA_REQUEST packet if already Ahead again,
+		 *       tell peer to stop sending resync requests... */
+	P_DISCONNECT          = 0x4b, /* data sock: Disconnect and stop connection attempts */
+
+	P_RS_DAGTAG_REQ       = 0x4c, /* data sock: Request a block for resync, with dagtag dependency */
+	P_RS_CSUM_DAGTAG_REQ  = 0x4d, /* data sock: Request a block for resync if checksum differs, with dagtag dependency */
+	P_RS_THIN_DAGTAG_REQ  = 0x4e, /* data sock: Request a block for resync or reply P_RS_DEALLOCATED, with dagtag dependency */
+	P_OV_DAGTAG_REQ       = 0x4f, /* data sock: Request a checksum for online verify, with dagtag dependency */
+	P_OV_DAGTAG_REPLY     = 0x50, /* data sock: Reply with a checksum for online verify, with dagtag dependency */
+
+	P_WRITE_ACK_IN_SYNC   = 0x51, /* meta sock: Application write ack setting bits in sync. */
+	P_RS_NEG_ACK          = 0x52, /* meta sock: Local disk is unusable writing resync reply. */
+	P_OV_RESULT_ID        = 0x53, /* meta sock: Online verify result with block ID. */
+	P_RS_DEALLOCATED_ID   = 0x54, /* data sock: Contains only zeros on sync source node. */
+
+	P_FLUSH_REQUESTS      = 0x55, /* data sock: Flush prior requests then send ack and/or forward */
+	P_FLUSH_FORWARD       = 0x56, /* meta sock: Send ack after sending P_OUT_OF_SYNC for prior P_PEER_ACK */
+	P_FLUSH_REQUESTS_ACK  = 0x57, /* data sock: Response to initiator of P_FLUSH_REQUESTS */
+	P_ENABLE_REPLICATION_NEXT = 0x58, /* data sock: whether to start replication on next resync start */
+	P_ENABLE_REPLICATION  = 0x59, /* data sock: enable or disable replication during resync */
+
+	P_MAY_IGNORE	      = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */
+
+	/* special command ids for handshake */
+
+	P_INITIAL_META	      = 0xfff1, /* First Packet on the MetaSock */
+	P_INITIAL_DATA	      = 0xfff2, /* First Packet on the Socket */
+
+	P_CONNECTION_FEATURES = 0xfffe	/* FIXED for the next century! */
+};
+
+
+/* This is the layout for a packet on the wire.
+ * The byteorder is the network byte order.
+ *     (except block_id and barrier fields.
+ *	these are pointers to local structs
+ *	and have no relevance for the partner,
+ *	which just echoes them as received.)
+ *
+ * NOTE that the payload starts at a long aligned offset,
+ * regardless of 32 or 64 bit arch!
+ */
+struct p_header80 {
+	uint32_t magic;
+	uint16_t command;
+	uint16_t length;	/* bytes of data after this header */
+} __packed;
+
+/* Header for big packets, Used for data packets exceeding 64kB */
+struct p_header95 {
+	uint16_t magic;	/* use DRBD_MAGIC_BIG here */
+	uint16_t command;
+	uint32_t length;
+} __packed;
+
+struct p_header100 {
+	uint32_t magic;
+	uint16_t volume;
+	uint16_t command;
+	uint32_t length;
+	uint32_t pad;
+} __packed;
+
+/* These defines must not be changed without changing the protocol version.
+ * New defines may only be introduced together with protocol version bump or
+ * new protocol feature flags.
+ */
+#define DP_HARDBARRIER	      1 /* no longer used */
+#define DP_RW_SYNC	      2 /* equals REQ_SYNC    */
+#define DP_MAY_SET_IN_SYNC    4
+#define DP_UNPLUG             8 /* equals REQ_UNPLUG (compat) */
+#define DP_FUA               16 /* equals REQ_FUA     */
+#define DP_FLUSH             32 /* equals REQ_PREFLUSH   */
+#define DP_DISCARD           64 /* equals REQ_DISCARD */
+#define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */
+#define DP_SEND_WRITE_ACK   256 /* This is a proto C write request */
+#define DP_WSAME            512 /* equiv. REQ_WRITE_SAME */
+#define DP_ZEROES          1024 /* equiv. REQ_OP_WRITE_ZEROES */
+
+/* possible combinations:
+ * REQ_OP_WRITE_ZEROES:  DP_DISCARD | DP_ZEROES
+ * REQ_OP_WRITE_ZEROES + REQ_NOUNMAP: DP_ZEROES
+ */
+
+struct p_data {
+	uint64_t sector;    /* 64 bits sector number */
+	uint64_t block_id;  /* to identify the request in protocol B&C */
+	uint32_t seq_num;
+	uint32_t dp_flags;
+} __packed;
+
+struct p_trim {
+	struct p_data p_data;
+	uint32_t size;	/* == bio->bi_size */
+} __packed;
+
+struct p_wsame {
+	struct p_data p_data;
+	uint32_t size;     /* == bio->bi_size */
+} __packed;
+
+/*
+ * struct p_block_ack shared by commands:
+ *   P_RECV_ACK (proto B)
+ *   P_WRITE_ACK (proto C),
+ *   P_WRITE_ACK_IN_SYNC,
+ *   P_SUPERSEDED (proto C, two-primaries conflict detection)
+ *   P_RS_WRITE_ACK
+ *   P_NEG_ACK
+ *   P_NEG_DREPLY
+ *   P_NEG_RS_DREPLY
+ *   P_RS_NEG_ACK
+ *   P_OV_RESULT
+ *   P_RS_IS_IN_SYNC
+ *   P_RS_CANCEL
+ *   P_RS_DEALLOCATED_ID
+ *   P_RS_CANCEL_AHEAD
+ */
+struct p_block_ack {
+	uint64_t sector;
+	uint64_t block_id;
+	uint32_t blksize;
+	uint32_t seq_num;
+} __packed;
+
+/* For P_OV_RESULT_ID. */
+struct p_ov_result {
+	uint64_t sector;
+	uint64_t block_id;
+	uint32_t blksize;
+	uint32_t seq_num;
+	uint32_t result;
+	uint32_t pad;
+} __packed;
+
+enum ov_result {
+	OV_RESULT_SKIP = 0,
+	OV_RESULT_IN_SYNC = 1,
+	OV_RESULT_OUT_OF_SYNC = 2,
+};
+
+struct p_block_req_common {
+	uint64_t sector;
+	uint64_t block_id;
+	uint32_t blksize;
+} __packed;
+
+/*
+ * struct p_block_req shared by commands:
+ *   P_DATA_REQUEST
+ *   P_RS_DATA_REQUEST
+ *   P_OV_REQUEST
+ *   P_OV_REPLY
+ *   P_CSUM_RS_REQUEST
+ *   P_RS_THIN_REQ
+ */
+struct p_block_req {
+	/* Allow fields to be addressed directly or via req_common. */
+	union {
+		struct {
+			uint64_t sector;
+			uint64_t block_id;
+			uint32_t blksize;
+		} __packed;
+		struct p_block_req_common req_common;
+	};
+	uint32_t pad;	/* to multiple of 8 Byte */
+} __packed;
+
+/*
+ * struct p_rs_req shared by commands:
+ *   P_RS_DAGTAG_REQ
+ *   P_RS_CSUM_DAGTAG_REQ
+ *   P_RS_THIN_DAGTAG_REQ
+ *   P_OV_DAGTAG_REQ
+ *   P_OV_DAGTAG_REPLY
+ */
+struct p_rs_req {
+	struct p_block_req_common req_common;
+	uint32_t dagtag_node_id;
+	uint64_t dagtag;
+} __packed;
+
+/* supports TRIM/DISCARD on the "wire" protocol */
+#define DRBD_FF_TRIM 1
+
+/* Detect all-zeros during resync, and rather TRIM/UNMAP/DISCARD those blocks
+ * instead of fully allocate a supposedly thin volume on initial resync */
+#define DRBD_FF_THIN_RESYNC 2
+
+/* supports REQ_WRITE_SAME on the "wire" protocol.
+ * Note: this flag is overloaded,
+ * its presence also
+ *   - indicates support for 128 MiB "batch bios",
+ *     max discard size of 128 MiB
+ *     instead of 4M before that.
+ *   - indicates that we exchange additional settings in p_sizes
+ *     drbd_send_sizes()/receive_sizes()
+ */
+#define DRBD_FF_WSAME 4
+
+/* supports REQ_OP_WRITE_ZEROES on the "wire" protocol.
+ *
+ * We used to map that to "discard" on the sending side, and if we cannot
+ * guarantee that discard zeroes data, the receiving side would map discard
+ * back to zero-out.
+ *
+ * With the introduction of REQ_OP_WRITE_ZEROES,
+ * we started to use that for both WRITE_ZEROES and DISCARDS,
+ * hoping that WRITE_ZEROES would "do what we want",
+ * UNMAP if possible, zero-out the rest.
+ *
+ * The example scenario is some LVM "thin" backend.
+ *
+ * While an un-allocated block on dm-thin reads as zeroes, on a dm-thin
+ * with "skip_block_zeroing=true", after a partial block write allocated
+ * that block, that same block may well map "undefined old garbage" from
+ * the backends on LBAs that have not yet been written to.
+ *
+ * If we cannot distinguish between zero-out and discard on the receiving
+ * side, to avoid "undefined old garbage" to pop up randomly at later times
+ * on supposedly zero-initialized blocks, we'd need to map all discards to
+ * zero-out on the receiving side.  But that would potentially do a full
+ * alloc on thinly provisioned backends, even when the expectation was to
+ * unmap/trim/discard/de-allocate.
+ *
+ * We need to distinguish on the protocol level, whether we need to guarantee
+ * zeroes (and thus use zero-out, potentially doing the mentioned full-alloc),
+ * or if we want to put the emphasis on discard, and only do a "best effort
+ * zeroing" (by "discarding" blocks aligned to discard-granularity, and zeroing
+ * only potential unaligned head and tail clippings), to at least *try* to
+ * avoid "false positives" in an online-verify later, hoping that someone
+ * set skip_block_zeroing=false.
+ */
+#define DRBD_FF_WZEROES 8
+
+/* Supports synchronization of application and resync IO using data generation
+ * tags (dagtags). See Documentation/application-resync-synchronization.rst for
+ * details.
+ */
+#define DRBD_FF_RESYNC_DAGTAG 16
+
+/* V2 of p_twopc_request has a 32 bit flag field and the two fields for node
+ * Ids are reduced to 8 bit instead of 32 bits.
+ *
+ * The flag TWOPC_HAS_RECHABLE indicates that in the commit phase
+ * (P_TWOPC_COMMIT) the reachable_nodes mask is set.
+ *
+ * The old behavior sends the primary_nodes mask, mask, and val in
+ * phase 2 (P_TWOPC_COMMIT), where mask and val are the same values as
+ * in phase 1 (P_TWOPC_PREPARE).
+ */
+#define DRBD_FF_2PC_V2 32
+
+/* Starting with drbd-9.1.15, a node with a backing disk sends the new
+ * current-uuid also to diskless nodes when the initial resync is
+ * skipped.
+ *
+ * The peer needs to know about this detail to apply the necessary
+ * strictness regarding downgrading its view of the partner's disk
+ * state.
+ */
+#define DRBD_FF_RS_SKIP_UUID 64
+
+/* Support for resync_without_replication.
+ */
+#define DRBD_FF_RESYNC_WITHOUT_REPLICATION 128
+
+/* Support for bitmap block size != 4k. If you connect peers with
+ * different bitmap block sizes, the resync becomes more
+ * interesting, and we need to communicate the bitmap block size.
+ */
+#define DRBD_FF_BM_BLOCK_SHIFT 256
+
+struct p_connection_features {
+	uint32_t protocol_min;
+	uint32_t feature_flags;
+	uint32_t protocol_max;
+	uint32_t sender_node_id;
+	uint32_t receiver_node_id;
+
+	/* should be more than enough for future enhancements
+	 * for now, feature_flags and the reserved array shall be zero.
+	 */
+
+	uint32_t _pad;
+	uint64_t reserved[6];
+} __packed;
+
+struct p_barrier {
+	uint32_t barrier;	/* barrier number _handle_ only */
+	uint32_t pad;	/* to multiple of 8 Byte */
+} __packed;
+
+struct p_barrier_ack {
+	uint32_t barrier;
+	uint32_t set_size;
+} __packed;
+
+struct p_confirm_stable {
+	uint64_t oldest_block_id;
+	uint64_t youngest_block_id;
+	uint32_t set_size;
+	uint32_t pad; /* to multiple of 8 Byte */
+} __packed;
+
+struct p_rs_param {
+	uint32_t resync_rate;
+
+	/* Since protocol version 88 and higher. */
+	char verify_alg[];
+} __packed;
+
+struct p_rs_param_89 {
+	uint32_t resync_rate;
+	/* protocol version 89: */
+	char verify_alg[SHARED_SECRET_MAX];
+	char csums_alg[SHARED_SECRET_MAX];
+} __packed;
+
+struct p_rs_param_95 {
+	uint32_t resync_rate;
+	char verify_alg[SHARED_SECRET_MAX];
+	char csums_alg[SHARED_SECRET_MAX];
+	uint32_t c_plan_ahead;
+	uint32_t c_delay_target;
+	uint32_t c_fill_target;
+	uint32_t c_max_rate;
+} __packed;
+
+enum drbd_conn_flags {
+	CF_DISCARD_MY_DATA = 1,
+	CF_DRY_RUN = 2,
+};
+
+struct p_protocol {
+	uint32_t protocol;
+	uint32_t after_sb_0p;
+	uint32_t after_sb_1p;
+	uint32_t after_sb_2p;
+	uint32_t conn_flags;
+	uint32_t two_primaries;
+
+	/* Since protocol version 87 and higher. */
+	char integrity_alg[];
+
+} __packed;
+
+#define UUID_FLAG_DISCARD_MY_DATA     ((u64)1 << 0)
+#define UUID_FLAG_CRASHED_PRIMARY     ((u64)1 << 1)
+#define UUID_FLAG_INCONSISTENT        ((u64)1 << 2)
+#define UUID_FLAG_SKIP_INITIAL_SYNC   ((u64)1 << 3)
+
+#define UUID_FLAG_MASK_COMPAT_84 \
+	(UUID_FLAG_DISCARD_MY_DATA|\
+	 UUID_FLAG_CRASHED_PRIMARY|\
+	 UUID_FLAG_INCONSISTENT|\
+	 UUID_FLAG_SKIP_INITIAL_SYNC)
+
+#define UUID_FLAG_NEW_DATAGEN         ((u64)1 << 4)
+#define UUID_FLAG_STABLE              ((u64)1 << 5)
+#define UUID_FLAG_GOT_STABLE          ((u64)1 << 6) /* send UUIDs */
+#define UUID_FLAG_RESYNC              ((u64)1 << 7) /* compare UUIDs and eventually start resync */
+#define UUID_FLAG_RECONNECT           ((u64)1 << 8)
+#define UUID_FLAG_DISKLESS_PRIMARY    ((u64)1 << 9) /* Use with UUID_FLAG_RESYNC if a diskless primary is the reason */
+#define UUID_FLAG_PRIMARY_LOST_QUORUM ((u64)1 << 10)
+#define UUID_FLAG_SYNC_TARGET         ((u64)1 << 11) /* currently L_SYNC_TARGET to some peer */
+#define UUID_FLAG_HAS_UNALLOC         ((u64)1 << 12) /* highest byte contains index of not allocated bitmap uuid */
+
+#define UUID_FLAG_UNALLOC_SHIFT       56
+#define UUID_FLAG_UNALLOC_MASK        ((u64)0xff << UUID_FLAG_UNALLOC_SHIFT)
+
+struct p_uuids {
+	uint64_t current_uuid;
+	uint64_t bitmap_uuid;
+	uint64_t history_uuids[HISTORY_UUIDS_V08];
+	uint64_t dirty_bits;
+	uint64_t uuid_flags;
+} __packed;
+
+struct p_uuids110 {
+	uint64_t current_uuid;
+	uint64_t dirty_bits;
+	uint64_t uuid_flags;
+	uint64_t node_mask; /* weak_nodes when UUID_FLAG_NEW_DATAGEN is set ;
+			       authoritative nodes when UUID_FLAG_STABLE not set */
+
+	uint64_t bitmap_uuids_mask; /* non zero bitmap UUIDS for these nodes */
+	uint64_t other_uuids[]; /* the first hweight(bitmap_uuids_mask) slots carry bitmap uuids.
+				    The node with the lowest node_id first.
+				    The remaining slots carry history uuids */
+} __packed;
+
+struct p_current_uuid {
+	uint64_t uuid;
+	uint64_t weak_nodes;
+} __packed;
+
+struct p_uuid {
+	uint64_t uuid;
+} __packed;
+
+/* optional queue_limits if (agreed_features & DRBD_FF_WSAME)
+ * see also struct queue_limits, as of late 2015 */
+struct o_qlim {
+	/* we don't need it yet, but we may as well communicate it now */
+	uint32_t physical_block_size;
+
+	/* so the original in struct queue_limits is unsigned short,
+	 * but I'd have to put in padding anyways. */
+	uint32_t logical_block_size;
+
+	/* One incoming bio becomes one DRBD request,
+	 * which may be translated to several bio on the receiving side.
+	 * We don't need to communicate chunk/boundary/segment ... limits.
+	 */
+
+	/* various IO hints may be useful with "diskless client" setups */
+	uint32_t alignment_offset;
+	uint32_t io_min;
+	uint32_t io_opt;
+
+	/* We may need to communicate integrity stuff at some point,
+	 * but let's not get ahead of ourselves. */
+
+	/* Backend discard capabilities.
+	 * Receiving side uses "blkdev_issue_discard()", no need to communicate
+	 * more specifics.  If the backend cannot do discards, the DRBD peer
+	 * may fall back to blkdev_issue_zeroout().
+	 */
+	uint8_t discard_enabled;
+	uint8_t discard_zeroes_data;
+	uint8_t write_same_capable;
+
+	/* Bitmap block shift relative to 4k. If peers have differnt bitmap
+	 * granularity, any resync related request needs to be aligned to the
+	 * larger granularity: we can not clear partial bits.
+	 * 0 to 8 to represent 4k to 1M.
+	 * If DRBD_FF_BM_BLOCK_SHIFT is agreed on.
+	 */
+	uint8_t bm_block_shift_minus_12;
+} __packed;
+
+struct p_sizes {
+	uint64_t d_size;  /* size of disk */
+	uint64_t u_size;  /* user requested size */
+	uint64_t c_size;  /* current exported size */
+	uint32_t max_bio_size;  /* Maximal size of a BIO */
+	uint16_t queue_order_type;  /* not yet implemented in DRBD*/
+	uint16_t dds_flags; /* use enum dds_flags here. */
+
+	/* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */
+	struct o_qlim qlim[];
+} __packed;
+
+struct p_state {
+	uint32_t state;
+} __packed;
+
+struct p_req_state {
+	uint32_t mask;
+	uint32_t val;
+} __packed;
+
+struct p_req_state_reply {
+	uint32_t retcode;
+} __packed;
+
+struct p_twopc_request {
+	uint32_t tid;  /* transaction identifier */
+	union {
+		struct { /* when DRBD_FF_2PC_V2 is set */
+			uint32_t flags;
+			uint16_t _pad;
+			int8_t  s8_initiator_node_id;  /* initiator of the transaction */
+			int8_t  s8_target_node_id;  /* target of the transaction (or -1) */
+		};
+		struct { /* original packet version */
+			uint32_t u32_initiator_node_id;  /* initiator of the transaction */
+			uint32_t u32_target_node_id;  /* target of the transaction (or -1) */
+		};
+	};
+	uint64_t nodes_to_reach;
+	union {
+		union { /* TWOPC_STATE_CHANGE */
+			struct {    /* P_TWOPC_PREPARE */
+				uint64_t _compat_pad;
+				uint32_t mask;
+				uint32_t val;
+			};
+			struct { /* P_TWOPC_COMMIT */
+				uint64_t primary_nodes;
+				uint64_t reachable_nodes; /* when TWOPC_HAS_RECHABLE flag is set */
+			};
+		};
+		union {	 /* TWOPC_RESIZE */
+			struct {    /* P_TWOPC_PREP_RSZ */
+				uint64_t user_size;
+				uint16_t dds_flags;
+			};
+			struct {    /* P_TWOPC_COMMIT	*/
+				uint64_t diskful_primary_nodes;
+				uint64_t exposed_size;
+			};
+		};
+	};
+} __packed;
+
+#define TWOPC_HAS_FLAGS     0x80000000 /* For packet dissectors */
+#define TWOPC_HAS_REACHABLE 0x40000000 /* The reachable_nodes field is valid */
+#define TWOPC_PRI_INCAPABLE 0x20000000 /* The primary has no access to data */
+
+struct p_twopc_reply {
+	uint32_t tid;  /* transaction identifier */
+	uint32_t initiator_node_id;  /* initiator of the transaction */
+	uint64_t reachable_nodes;
+
+	union {
+		struct { /* TWOPC_STATE_CHANGE */
+			uint64_t primary_nodes;
+			uint64_t weak_nodes;
+		};
+		struct { /* TWOPC_RESIZE */
+			uint64_t diskful_primary_nodes;
+			uint64_t max_possible_size;
+		};
+	};
+} __packed;
+
+struct p_drbd06_param {
+	uint64_t size;
+	uint32_t state;
+	uint32_t blksize;
+	uint32_t protocol;
+	uint32_t version;
+	uint32_t gen_cnt[5];
+	uint32_t bit_map_gen[5];
+} __packed;
+
+struct p_block_desc {
+	uint64_t sector;
+	uint32_t blksize;
+	uint32_t pad;	/* to multiple of 8 Byte */
+} __packed;
+
+/* Valid values for the encoding field.
+ * Bump proto version when changing this. */
+enum drbd_bitmap_code {
+	/* RLE_VLI_Bytes = 0,
+	 * and other bit variants had been defined during
+	 * algorithm evaluation. */
+	RLE_VLI_Bits = 2,
+};
+
+struct p_compressed_bm {
+	/* (encoding & 0x0f): actual encoding, see enum drbd_bitmap_code
+	 * (encoding & 0x80): polarity (set/unset) of first runlength
+	 * ((encoding >> 4) & 0x07): pad_bits, number of trailing zero bits
+	 * used to pad up to head.length bytes
+	 */
+	uint8_t encoding;
+
+	uint8_t code[];
+} __packed;
+
+struct p_delay_probe93 {
+	uint32_t seq_num; /* sequence number to match the two probe packets */
+	uint32_t offset;  /* usecs the probe got sent after the reference time point */
+} __packed;
+
+struct p_dagtag {
+	uint64_t dagtag;
+} __packed;
+
+struct p_peer_ack {
+	uint64_t mask;
+	uint64_t dagtag;
+} __packed;
+
+struct p_peer_block_desc {
+	uint64_t sector;
+	uint64_t mask;
+	uint32_t size;
+	uint32_t pad;	/* to multiple of 8 Byte */
+} __packed;
+
+struct p_peer_dagtag {
+	uint64_t dagtag;
+	uint32_t node_id;
+} __packed;
+
+struct p_flush_requests {
+	uint64_t flush_sequence;
+} __packed;
+
+struct p_flush_forward {
+	uint64_t flush_sequence;
+	uint32_t initiator_node_id;
+} __packed;
+
+struct p_flush_ack {
+	uint64_t flush_sequence;
+	uint32_t primary_node_id;
+} __packed;
+
+struct p_enable_replication {
+	uint8_t enable;
+	uint8_t _pad1;
+	uint16_t _pad2;
+} __packed;
+
+/*
+ * Bitmap packets need to fit within a single page on the sender and receiver,
+ * so we are limited to 4 KiB (and not to PAGE_SIZE, which can be bigger).
+ */
+#define DRBD_SOCKET_BUFFER_SIZE 4096
+
+#endif  /* __DRBD_PROTOCOL_H */
diff --git a/include/linux/efi.h b/include/linux/efi.h
index 664898d09ff57..72e76ec546415 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -832,27 +832,6 @@ extern int __init parse_efi_signature_list(
 	const void *data, size_t size,
 	efi_element_handler_t (*get_handler_for_guid)(const efi_guid_t *));
 
-/**
- * efi_range_is_wc - check the WC bit on an address range
- * @start: starting kvirt address
- * @len: length of range
- *
- * Consult the EFI memory map and make sure it's ok to set this range WC.
- * Returns true or false.
- */
-static inline int efi_range_is_wc(unsigned long start, unsigned long len)
-{
-	unsigned long i;
-
-	for (i = 0; i < len; i += (1UL << EFI_PAGE_SHIFT)) {
-		unsigned long paddr = __pa(start + i);
-		if (!(efi_mem_attributes(paddr) & EFI_MEMORY_WC))
-			return 0;
-	}
-	/* The range checked out */
-	return 1;
-}
-
 /*
  * We play games with efi_enabled so that the compiler will, if
  * possible, remove EFI-related code altogether.
diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h
index e7497f8046442..c909a8ba22e83 100644
--- a/include/linux/energy_model.h
+++ b/include/linux/energy_model.h
@@ -248,7 +248,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
 	struct em_perf_state *ps;
 	int i;
 
-	WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n");
+	lockdep_assert(rcu_read_lock_any_held());
 
 	if (!sum_util)
 		return 0;
@@ -267,7 +267,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd,
 	 * Find the lowest performance state of the Energy Model above the
 	 * requested performance.
 	 */
-	em_table = rcu_dereference(pd->em_table);
+	em_table = rcu_dereference_all(pd->em_table);
 	i = em_pd_get_efficient_state(em_table->state, pd, max_util);
 	ps = &em_table->state[i];
 
diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 9a1eacf35d370..df8f88f63a706 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -42,7 +42,8 @@ extern const struct header_ops eth_header_ops;
 
 int eth_header(struct sk_buff *skb, struct net_device *dev, unsigned short type,
 	       const void *daddr, const void *saddr, unsigned len);
-int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr);
+int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev,
+		     unsigned char *haddr);
 int eth_header_cache(const struct neighbour *neigh, struct hh_cache *hh,
 		     __be16 type);
 void eth_header_cache_update(struct hh_cache *hh, const struct net_device *dev,
diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h
index 798abec67a1b0..656d465bcd069 100644
--- a/include/linux/ethtool.h
+++ b/include/linux/ethtool.h
@@ -332,6 +332,8 @@ struct kernel_ethtool_coalesce {
 	u32 tx_aggr_max_bytes;
 	u32 tx_aggr_max_frames;
 	u32 tx_aggr_time_usecs;
+	u32 rx_cqe_frames;
+	u32 rx_cqe_nsecs;
 };
 
 /**
@@ -380,7 +382,9 @@ bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
 #define ETHTOOL_COALESCE_TX_AGGR_TIME_USECS	BIT(26)
 #define ETHTOOL_COALESCE_RX_PROFILE		BIT(27)
 #define ETHTOOL_COALESCE_TX_PROFILE		BIT(28)
-#define ETHTOOL_COALESCE_ALL_PARAMS		GENMASK(28, 0)
+#define ETHTOOL_COALESCE_RX_CQE_FRAMES		BIT(29)
+#define ETHTOOL_COALESCE_RX_CQE_NSECS		BIT(30)
+#define ETHTOOL_COALESCE_ALL_PARAMS		GENMASK(30, 0)
 
 #define ETHTOOL_COALESCE_USECS						\
 	(ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_TX_USECS)
@@ -512,12 +516,14 @@ struct ethtool_eth_ctrl_stats {
  *
  *	Equivalent to `30.3.4.3 aPAUSEMACCtrlFramesReceived`
  *	from the standard.
+ * @tx_pause_storm_events: TX pause storm event count (see ethtool.yaml).
  */
 struct ethtool_pause_stats {
 	enum ethtool_mac_stats_src src;
 	struct_group(stats,
 		u64 tx_pause_frames;
 		u64 rx_pause_frames;
+		u64 tx_pause_storm_events;
 	);
 };
 
diff --git a/include/linux/evm.h b/include/linux/evm.h
index ddece4a6b25d1..913f4573b2038 100644
--- a/include/linux/evm.h
+++ b/include/linux/evm.h
@@ -18,6 +18,8 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry,
 					     const char *xattr_name,
 					     void *xattr_value,
 					     size_t xattr_value_len);
+int evm_fix_hmac(struct dentry *dentry, const char *xattr_name,
+		 const char *xattr_value, size_t xattr_value_len);
 int evm_inode_init_security(struct inode *inode, struct inode *dir,
 			    const struct qstr *qstr, struct xattr *xattrs,
 			    int *xattr_count);
@@ -51,6 +53,12 @@ static inline enum integrity_status evm_verifyxattr(struct dentry *dentry,
 {
 	return INTEGRITY_UNKNOWN;
 }
+
+static inline int evm_fix_hmac(struct dentry *dentry, const char *xattr_name,
+			       const char *xattr_value, size_t xattr_value_len)
+{
+	return -EOPNOTSUPP;
+}
 #endif
 
 static inline int evm_inode_init_security(struct inode *inode, struct inode *dir,
diff --git a/include/linux/fb.h b/include/linux/fb.h
index 6d4a58084fd5f..5178a33c752c6 100644
--- a/include/linux/fb.h
+++ b/include/linux/fb.h
@@ -217,14 +217,12 @@ struct fb_deferred_io {
 	/* delay between mkwrite and deferred handler */
 	unsigned long delay;
 	bool sort_pagereflist; /* sort pagelist by offset */
-	int open_count; /* number of opened files; protected by fb_info lock */
-	struct mutex lock; /* mutex that protects the pageref list */
-	struct list_head pagereflist; /* list of pagerefs for touched pages */
-	struct address_space *mapping; /* page cache object for fb device */
 	/* callback */
 	struct page *(*get_page)(struct fb_info *info, unsigned long offset);
 	void (*deferred_io)(struct fb_info *info, struct list_head *pagelist);
 };
+
+struct fb_deferred_io_state;
 #endif
 
 /*
@@ -484,9 +482,8 @@ struct fb_info {
 
 #ifdef CONFIG_FB_DEFERRED_IO
 	struct delayed_work deferred_work;
-	unsigned long npagerefs;
-	struct fb_deferred_io_pageref *pagerefs;
 	struct fb_deferred_io *fbdefio;
+	struct fb_deferred_io_state *fbdefio_state;
 #endif
 
 	const struct fb_ops *fbops;
@@ -605,9 +602,9 @@ extern int register_framebuffer(struct fb_info *fb_info);
 extern void unregister_framebuffer(struct fb_info *fb_info);
 extern int devm_register_framebuffer(struct device *dev, struct fb_info *fb_info);
 extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size);
-extern void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 idx,
-				u32 height, u32 shift_high, u32 shift_low, u32 mod);
-extern void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 s_pitch, u32 height);
+void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 idx, u32 height,
+			     u32 shift_high, u32 shift_low, u32 mod);
+void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 s_pitch, u32 height);
 extern void fb_set_suspend(struct fb_info *info, int state);
 extern int fb_get_color_depth(struct fb_var_screeninfo *var,
 			      struct fb_fix_screeninfo *fix);
@@ -633,8 +630,8 @@ static inline struct device *dev_of_fbinfo(const struct fb_info *info)
 #endif
 }
 
-static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch,
-					   u8 *src, u32 s_pitch, u32 height)
+static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 s_pitch,
+					   u32 height)
 {
 	u32 i, j;
 
diff --git a/include/linux/file_ref.h b/include/linux/file_ref.h
index 31551e4cb8f34..c7512ce70f9c4 100644
--- a/include/linux/file_ref.h
+++ b/include/linux/file_ref.h
@@ -54,11 +54,11 @@ typedef struct {
 /**
  * file_ref_init - Initialize a file reference count
  * @ref: Pointer to the reference count
- * @cnt: The initial reference count typically '1'
+ * @cnt: The initial reference count typically FILE_REF_ONEREF
  */
 static inline void file_ref_init(file_ref_t *ref, unsigned long cnt)
 {
-	atomic_long_set(&ref->refcnt, cnt - 1);
+	atomic_long_set(&ref->refcnt, cnt);
 }
 
 bool __file_ref_put(file_ref_t *ref, unsigned long cnt);
diff --git a/include/linux/filelock.h b/include/linux/filelock.h
index d2c9740e26a8e..5f0a2fb314506 100644
--- a/include/linux/filelock.h
+++ b/include/linux/filelock.h
@@ -50,6 +50,7 @@ struct lease_manager_operations {
 	void (*lm_setup)(struct file_lease *, void **);
 	bool (*lm_breaker_owns_lease)(struct file_lease *);
 	int (*lm_open_conflict)(struct file *, int);
+	bool (*lm_breaker_timedout)(struct file_lease *fl);
 };
 
 struct lock_manager {
diff --git a/include/linux/find.h b/include/linux/find.h
index 9d720ad92bc15..6c2be8ca615df 100644
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -22,8 +22,6 @@ extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long si
 unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n);
 unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
 				unsigned long size, unsigned long n);
-unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-					unsigned long size, unsigned long n);
 unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
 					const unsigned long *addr3, unsigned long size,
 					unsigned long n);
diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h
index 0ec1cdc5585d2..4e3baa5570687 100644
--- a/include/linux/firmware/cirrus/cs_dsp.h
+++ b/include/linux/firmware/cirrus/cs_dsp.h
@@ -179,6 +179,7 @@ struct cs_dsp {
 
 	bool booted;
 	bool running;
+	bool hibernating;
 
 	struct list_head ctl_list;
 
@@ -354,4 +355,6 @@ int cs_dsp_chunk_write(struct cs_dsp_chunk *ch, int nbits, u32 val);
 int cs_dsp_chunk_flush(struct cs_dsp_chunk *ch);
 int cs_dsp_chunk_read(struct cs_dsp_chunk *ch, int nbits);
 
+void cs_dsp_hibernate(struct cs_dsp *dsp, bool hibernating);
+
 #endif
diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h
index 2091da965a5ad..13f17dc4443b8 100644
--- a/include/linux/firmware/samsung/exynos-acpm-protocol.h
+++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h
@@ -14,30 +14,24 @@ struct acpm_handle;
 struct device_node;
 
 struct acpm_dvfs_ops {
-	int (*set_rate)(const struct acpm_handle *handle,
-			unsigned int acpm_chan_id, unsigned int clk_id,
-			unsigned long rate);
-	unsigned long (*get_rate)(const struct acpm_handle *handle,
+	int (*set_rate)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			unsigned int clk_id, unsigned long rate);
+	unsigned long (*get_rate)(struct acpm_handle *handle,
 				  unsigned int acpm_chan_id,
 				  unsigned int clk_id);
 };
 
 struct acpm_pmic_ops {
-	int (*read_reg)(const struct acpm_handle *handle,
-			unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-			u8 *buf);
-	int (*bulk_read)(const struct acpm_handle *handle,
-			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-			 u8 count, u8 *buf);
-	int (*write_reg)(const struct acpm_handle *handle,
-			 unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-			 u8 value);
-	int (*bulk_write)(const struct acpm_handle *handle,
-			  unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-			  u8 count, const u8 *buf);
-	int (*update_reg)(const struct acpm_handle *handle,
-			  unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan,
-			  u8 value, u8 mask);
+	int (*read_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			u8 type, u8 reg, u8 chan, u8 *buf);
+	int (*bulk_read)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			 u8 type, u8 reg, u8 chan, u8 count, u8 *buf);
+	int (*write_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			 u8 type, u8 reg, u8 chan, u8 value);
+	int (*bulk_write)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			  u8 type, u8 reg, u8 chan, u8 count, const u8 *buf);
+	int (*update_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id,
+			  u8 type, u8 reg, u8 chan, u8 value, u8 mask);
 };
 
 struct acpm_ops {
@@ -56,12 +50,12 @@ struct acpm_handle {
 struct device;
 
 #if IS_ENABLED(CONFIG_EXYNOS_ACPM_PROTOCOL)
-const struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
-						struct device_node *np);
+struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
+					  struct device_node *np);
 #else
 
-static inline const struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
-							      struct device_node *np)
+static inline struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
+							struct device_node *np)
 {
 	return NULL;
 }
diff --git a/include/linux/folio_batch.h b/include/linux/folio_batch.h
new file mode 100644
index 0000000000000..b45946adc50bb
--- /dev/null
+++ b/include/linux/folio_batch.h
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * include/linux/folio_batch.h
+ *
+ * In many places it is efficient to batch an operation up against multiple
+ * folios.  A folio_batch is a container which is used for that.
+ */
+
+#ifndef _LINUX_FOLIO_BATCH_H
+#define _LINUX_FOLIO_BATCH_H
+
+#include <linux/types.h>
+
+/* 31 pointers + header align the folio_batch structure to a power of two */
+#define FOLIO_BATCH_SIZE	31
+
+struct folio;
+
+/**
+ * struct folio_batch - A collection of folios.
+ *
+ * The folio_batch is used to amortise the cost of retrieving and
+ * operating on a set of folios.  The order of folios in the batch may be
+ * significant (eg delete_from_page_cache_batch()).  Some users of the
+ * folio_batch store "exceptional" entries in it which can be removed
+ * by calling folio_batch_remove_exceptionals().
+ */
+struct folio_batch {
+	unsigned char nr;
+	unsigned char i;
+	bool percpu_pvec_drained;
+	struct folio *folios[FOLIO_BATCH_SIZE];
+};
+
+/**
+ * folio_batch_init() - Initialise a batch of folios
+ * @fbatch: The folio batch.
+ *
+ * A freshly initialised folio_batch contains zero folios.
+ */
+static inline void folio_batch_init(struct folio_batch *fbatch)
+{
+	fbatch->nr = 0;
+	fbatch->i = 0;
+	fbatch->percpu_pvec_drained = false;
+}
+
+static inline void folio_batch_reinit(struct folio_batch *fbatch)
+{
+	fbatch->nr = 0;
+	fbatch->i = 0;
+}
+
+static inline unsigned int folio_batch_count(const struct folio_batch *fbatch)
+{
+	return fbatch->nr;
+}
+
+static inline unsigned int folio_batch_space(const struct folio_batch *fbatch)
+{
+	return FOLIO_BATCH_SIZE - fbatch->nr;
+}
+
+/**
+ * folio_batch_add() - Add a folio to a batch.
+ * @fbatch: The folio batch.
+ * @folio: The folio to add.
+ *
+ * The folio is added to the end of the batch.
+ * The batch must have previously been initialised using folio_batch_init().
+ *
+ * Return: The number of slots still available.
+ */
+static inline unsigned folio_batch_add(struct folio_batch *fbatch,
+		struct folio *folio)
+{
+	fbatch->folios[fbatch->nr++] = folio;
+	return folio_batch_space(fbatch);
+}
+
+/**
+ * folio_batch_next - Return the next folio to process.
+ * @fbatch: The folio batch being processed.
+ *
+ * Use this function to implement a queue of folios.
+ *
+ * Return: The next folio in the queue, or NULL if the queue is empty.
+ */
+static inline struct folio *folio_batch_next(struct folio_batch *fbatch)
+{
+	if (fbatch->i == fbatch->nr)
+		return NULL;
+	return fbatch->folios[fbatch->i++];
+}
+
+void __folio_batch_release(struct folio_batch *fbatch);
+
+static inline void folio_batch_release(struct folio_batch *fbatch)
+{
+	if (folio_batch_count(fbatch))
+		__folio_batch_release(fbatch);
+}
+
+void folio_batch_remove_exceptionals(struct folio_batch *fbatch);
+#endif /* _LINUX_FOLIO_BATCH_H */
diff --git a/include/linux/folio_queue.h b/include/linux/folio_queue.h
index adab609c972e1..f6d5f1f127c9f 100644
--- a/include/linux/folio_queue.h
+++ b/include/linux/folio_queue.h
@@ -14,7 +14,7 @@
 #ifndef _LINUX_FOLIO_QUEUE_H
 #define _LINUX_FOLIO_QUEUE_H
 
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/mm.h>
 
 /*
@@ -29,12 +29,12 @@
  */
 struct folio_queue {
 	struct folio_batch	vec;		/* Folios in the queue segment */
-	u8			orders[PAGEVEC_SIZE]; /* Order of each folio */
+	u8			orders[FOLIO_BATCH_SIZE]; /* Order of each folio */
 	struct folio_queue	*next;		/* Next queue segment or NULL */
 	struct folio_queue	*prev;		/* Previous queue segment of NULL */
 	unsigned long		marks;		/* 1-bit mark per folio */
 	unsigned long		marks2;		/* Second 1-bit mark per folio */
-#if PAGEVEC_SIZE > BITS_PER_LONG
+#if FOLIO_BATCH_SIZE > BITS_PER_LONG
 #error marks is not big enough
 #endif
 	unsigned int		rreq_id;
@@ -70,7 +70,7 @@ static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
  */
 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
 {
-	return PAGEVEC_SIZE;
+	return FOLIO_BATCH_SIZE;
 }
 
 /**
diff --git a/include/linux/font.h b/include/linux/font.h
index fd8625cd76b25..5401f07dd6ce8 100644
--- a/include/linux/font.h
+++ b/include/linux/font.h
@@ -13,45 +13,70 @@
 
 #include <linux/types.h>
 
+struct console_font;
+
+/*
+ * font_data_t and helpers
+ */
+
+/**
+ * font_data_t - Raw font data
+ *
+ * Values of type font_data_t store a pointer to raw font data. The format
+ * is monochrome. Each bit sets a pixel of a stored glyph. Font data does
+ * not store geometry information for the individual glyphs. Users of the
+ * font have to store glyph size, pitch and character count separately.
+ *
+ * Font data in font_data_t is not equivalent to raw u8. Each pointer stores
+ * an additional hidden header before the font data. The layout is
+ *
+ * +------+-----------------------------+
+ * | -16  |  CRC32 Checksum (optional)  |
+ * | -12  |  <Unused>                   |
+ * |  -8  |  Number of data bytes       |
+ * |  -4  |  Reference count            |
+ * +------+-----------------------------+
+ * |   0  |  Data buffer                |
+ * |  ... |                             |
+ * +------+-----------------------------+
+ *
+ * Use helpers to access font_data_t. Use font_data_buf() to get the stored data.
+ */
+typedef const unsigned char font_data_t;
+
+/**
+ * font_data_buf() - Returns the font data as raw bytes
+ * @fd: The font data
+ *
+ * Returns:
+ * The raw font data. The provided buffer is read-only.
+ */
+static inline const unsigned char *font_data_buf(font_data_t *fd)
+{
+	return (const unsigned char *)fd;
+}
+
+font_data_t *font_data_import(const struct console_font *font, unsigned int vpitch,
+			      u32 (*calc_csum)(u32, const void *, size_t));
+void font_data_get(font_data_t *fd);
+bool font_data_put(font_data_t *fd);
+unsigned int font_data_size(font_data_t *fd);
+bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs);
+int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch);
+
+/*
+ * Font description
+ */
+
 struct font_desc {
     int idx;
     const char *name;
     unsigned int width, height;
     unsigned int charcount;
-    const void *data;
+    font_data_t *data;
     int pref;
 };
 
-#define VGA8x8_IDX	0
-#define VGA8x16_IDX	1
-#define PEARL8x8_IDX	2
-#define VGA6x11_IDX	3
-#define FONT7x14_IDX	4
-#define	FONT10x18_IDX	5
-#define SUN8x16_IDX	6
-#define SUN12x22_IDX	7
-#define ACORN8x8_IDX	8
-#define	MINI4x6_IDX	9
-#define FONT6x10_IDX	10
-#define TER16x32_IDX	11
-#define FONT6x8_IDX	12
-#define TER10x18_IDX	13
-
-extern const struct font_desc	font_vga_8x8,
-			font_vga_8x16,
-			font_pearl_8x8,
-			font_vga_6x11,
-			font_7x14,
-			font_10x18,
-			font_sun_8x16,
-			font_sun_12x22,
-			font_acorn_8x8,
-			font_mini_4x6,
-			font_6x10,
-			font_ter_16x32,
-			font_6x8,
-			font_ter_10x18;
-
 /* Find a font with a specific name */
 
 extern const struct font_desc *find_font(const char *name);
@@ -65,17 +90,23 @@ extern const struct font_desc *get_default_font(int xres, int yres,
 /* Max. length for the name of a predefined font */
 #define MAX_FONT_NAME	32
 
-/* Extra word getters */
-#define REFCOUNT(fd)	(((int *)(fd))[-1])
-#define FNTSIZE(fd)	(((int *)(fd))[-2])
-#define FNTCHARCNT(fd)	(((int *)(fd))[-3])
-#define FNTSUM(fd)	(((int *)(fd))[-4])
-
-#define FONT_EXTRA_WORDS 4
+/*
+ * Built-in fonts
+ */
 
-struct font_data {
-	unsigned int extra[FONT_EXTRA_WORDS];
-	const unsigned char data[];
-} __packed;
+extern const struct font_desc font_10x18;
+extern const struct font_desc font_6x10;
+extern const struct font_desc font_6x8;
+extern const struct font_desc font_7x14;
+extern const struct font_desc font_acorn_8x8;
+extern const struct font_desc font_mini_4x6;
+extern const struct font_desc font_pearl_8x8;
+extern const struct font_desc font_sun_12x22;
+extern const struct font_desc font_sun_8x16;
+extern const struct font_desc font_ter_10x18;
+extern const struct font_desc font_ter_16x32;
+extern const struct font_desc font_vga_6x11;
+extern const struct font_desc font_vga_8x16;
+extern const struct font_desc font_vga_8x8;
 
 #endif /* _VIDEO_FONT_H */
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 8b3dd145b25ec..8afbe2ef26864 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -783,7 +783,7 @@ struct inode {
 #endif
 
 	/* Stat data, not accessed from path walking */
-	unsigned long		i_ino;
+	u64			i_ino;
 	/*
 	 * Filesystems may only read i_nlink directly.  They shall use the
 	 * following functions for modification:
@@ -2058,16 +2058,24 @@ static inline bool can_mmap_file(struct file *file)
 	return true;
 }
 
-int __compat_vma_mmap(const struct file_operations *f_op,
-		struct file *file, struct vm_area_struct *vma);
+void compat_set_desc_from_vma(struct vm_area_desc *desc, const struct file *file,
+			      const struct vm_area_struct *vma);
+int __compat_vma_mmap(struct vm_area_desc *desc, struct vm_area_struct *vma);
 int compat_vma_mmap(struct file *file, struct vm_area_struct *vma);
+int __vma_check_mmap_hook(struct vm_area_struct *vma);
 
 static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	int err;
+
 	if (file->f_op->mmap_prepare)
 		return compat_vma_mmap(file, vma);
 
-	return file->f_op->mmap(file, vma);
+	err = file->f_op->mmap(file, vma);
+	if (err)
+		return err;
+
+	return __vma_check_mmap_hook(vma);
 }
 
 static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc)
@@ -2912,65 +2920,63 @@ static inline bool name_contains_dotdot(const char *name)
 #include <linux/err.h>
 
 /* needed for stackable file system support */
-extern loff_t default_llseek(struct file *file, loff_t offset, int whence);
+loff_t default_llseek(struct file *file, loff_t offset, int whence);
 
-extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence);
+loff_t vfs_llseek(struct file *file, loff_t offset, int whence);
 
-extern int inode_init_always_gfp(struct super_block *, struct inode *, gfp_t);
+int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp);
 static inline int inode_init_always(struct super_block *sb, struct inode *inode)
 {
 	return inode_init_always_gfp(sb, inode, GFP_NOFS);
 }
 
-extern void inode_init_once(struct inode *);
-extern void address_space_init_once(struct address_space *mapping);
-extern struct inode * igrab(struct inode *);
-extern ino_t iunique(struct super_block *, ino_t);
-extern int inode_needs_sync(struct inode *inode);
-extern int inode_just_drop(struct inode *inode);
+void inode_init_once(struct inode *inode);
+void address_space_init_once(struct address_space *mapping);
+struct inode *igrab(struct inode *inode);
+ino_t iunique(struct super_block *sb, ino_t max_reserved);
+int inode_needs_sync(struct inode *inode);
+int inode_just_drop(struct inode *inode);
 static inline int inode_generic_drop(struct inode *inode)
 {
 	return !inode->i_nlink || inode_unhashed(inode);
 }
-extern void d_mark_dontcache(struct inode *inode);
+void d_mark_dontcache(struct inode *inode);
 
-extern struct inode *ilookup5_nowait(struct super_block *sb,
-		unsigned long hashval, int (*test)(struct inode *, void *),
-		void *data, bool *isnew);
-extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
-		int (*test)(struct inode *, void *), void *data);
-extern struct inode *ilookup(struct super_block *sb, unsigned long ino);
+struct inode *ilookup5_nowait(struct super_block *sb, u64 hashval,
+			      int (*test)(struct inode *, void *), void *data,
+			      bool *isnew);
+struct inode *ilookup5(struct super_block *sb, u64 hashval,
+		       int (*test)(struct inode *, void *), void *data);
+struct inode *ilookup(struct super_block *sb, u64 ino);
 
-extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
-		int (*test)(struct inode *, void *),
-		int (*set)(struct inode *, void *),
-		void *data);
-struct inode *iget5_locked(struct super_block *, unsigned long,
+struct inode *inode_insert5(struct inode *inode, u64 hashval,
+			    int (*test)(struct inode *, void *),
+			    int (*set)(struct inode *, void *), void *data);
+struct inode *iget5_locked(struct super_block *sb, u64 hashval,
 			   int (*test)(struct inode *, void *),
-			   int (*set)(struct inode *, void *), void *);
-struct inode *iget5_locked_rcu(struct super_block *, unsigned long,
+			   int (*set)(struct inode *, void *), void *data);
+struct inode *iget5_locked_rcu(struct super_block *sb, u64 hashval,
 			       int (*test)(struct inode *, void *),
-			       int (*set)(struct inode *, void *), void *);
-extern struct inode * iget_locked(struct super_block *, unsigned long);
-extern struct inode *find_inode_nowait(struct super_block *,
-				       unsigned long,
-				       int (*match)(struct inode *,
-						    unsigned long, void *),
-				       void *data);
-extern struct inode *find_inode_rcu(struct super_block *, unsigned long,
-				    int (*)(struct inode *, void *), void *);
-extern struct inode *find_inode_by_ino_rcu(struct super_block *, unsigned long);
-extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *);
-extern int insert_inode_locked(struct inode *);
+			       int (*set)(struct inode *, void *), void *data);
+struct inode *iget_locked(struct super_block *sb, u64 ino);
+struct inode *find_inode_nowait(struct super_block *sb, u64 hashval,
+				int (*match)(struct inode *, u64, void *),
+				void *data);
+struct inode *find_inode_rcu(struct super_block *sb, u64 hashval,
+			     int (*test)(struct inode *, void *), void *data);
+struct inode *find_inode_by_ino_rcu(struct super_block *sb, u64 ino);
+int insert_inode_locked4(struct inode *inode, u64 hashval,
+			 int (*test)(struct inode *, void *), void *data);
+int insert_inode_locked(struct inode *inode);
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
-extern void lockdep_annotate_inode_mutex_key(struct inode *inode);
+void lockdep_annotate_inode_mutex_key(struct inode *inode);
 #else
 static inline void lockdep_annotate_inode_mutex_key(struct inode *inode) { };
 #endif
-extern void unlock_new_inode(struct inode *);
-extern void discard_new_inode(struct inode *);
-extern unsigned int get_next_ino(void);
-extern void evict_inodes(struct super_block *sb);
+void unlock_new_inode(struct inode *inode);
+void discard_new_inode(struct inode *inode);
+unsigned int get_next_ino(void);
+void evict_inodes(struct super_block *sb);
 void dump_mapping(const struct address_space *);
 
 /*
@@ -3015,21 +3021,21 @@ int setattr_should_drop_sgid(struct mnt_idmap *idmap,
  */
 #define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp)
 
-extern void __insert_inode_hash(struct inode *, unsigned long hashval);
+void __insert_inode_hash(struct inode *inode, u64 hashval);
 static inline void insert_inode_hash(struct inode *inode)
 {
 	__insert_inode_hash(inode, inode->i_ino);
 }
 
-extern void __remove_inode_hash(struct inode *);
+void __remove_inode_hash(struct inode *inode);
 static inline void remove_inode_hash(struct inode *inode)
 {
 	if (!inode_unhashed(inode) && !hlist_fake(&inode->i_hash))
 		__remove_inode_hash(inode);
 }
 
-extern void inode_sb_list_add(struct inode *inode);
-extern void inode_lru_list_add(struct inode *inode);
+void inode_sb_list_add(struct inode *inode);
+void inode_lru_list_add(struct inode *inode);
 
 int generic_file_mmap(struct file *, struct vm_area_struct *);
 int generic_file_mmap_prepare(struct vm_area_desc *desc);
diff --git a/include/linux/fs_parser.h b/include/linux/fs_parser.h
index 5e8a3b5460331..98b83708f92b5 100644
--- a/include/linux/fs_parser.h
+++ b/include/linux/fs_parser.h
@@ -27,8 +27,8 @@ typedef int fs_param_type(struct p_log *,
  * The type of parameter expected.
  */
 fs_param_type fs_param_is_bool, fs_param_is_u32, fs_param_is_s32, fs_param_is_u64,
-	fs_param_is_enum, fs_param_is_string, fs_param_is_blob, fs_param_is_blockdev,
-	fs_param_is_path, fs_param_is_fd, fs_param_is_uid, fs_param_is_gid,
+	fs_param_is_enum, fs_param_is_string, fs_param_is_blockdev,
+	fs_param_is_fd, fs_param_is_uid, fs_param_is_gid,
 	fs_param_is_file_or_string;
 
 /*
@@ -84,8 +84,6 @@ extern int fs_lookup_param(struct fs_context *fc,
 
 extern int lookup_constant(const struct constant_table tbl[], const char *name, int not_found);
 
-extern const struct constant_table bool_names[];
-
 #ifdef CONFIG_VALIDATE_FS_PARSER
 extern bool fs_validate_description(const char *name,
 				    const struct fs_parameter_spec *desc);
@@ -127,9 +125,7 @@ static inline bool fs_validate_description(const char *name,
 #define fsparam_enum(NAME, OPT, array)	__fsparam(fs_param_is_enum, NAME, OPT, 0, array)
 #define fsparam_string(NAME, OPT) \
 				__fsparam(fs_param_is_string, NAME, OPT, 0, NULL)
-#define fsparam_blob(NAME, OPT)	__fsparam(fs_param_is_blob, NAME, OPT, 0, NULL)
 #define fsparam_bdev(NAME, OPT)	__fsparam(fs_param_is_blockdev, NAME, OPT, 0, NULL)
-#define fsparam_path(NAME, OPT)	__fsparam(fs_param_is_path, NAME, OPT, 0, NULL)
 #define fsparam_fd(NAME, OPT)	__fsparam(fs_param_is_fd, NAME, OPT, 0, NULL)
 #define fsparam_file_or_string(NAME, OPT) \
 				__fsparam(fs_param_is_file_or_string, NAME, OPT, 0, NULL)
diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h
index 0070764b790a3..b63003cec25fa 100644
--- a/include/linux/fs_struct.h
+++ b/include/linux/fs_struct.h
@@ -8,10 +8,11 @@
 #include <linux/seqlock.h>
 
 struct fs_struct {
-	int users;
 	seqlock_t seq;
+	int users;
 	int umask;
 	int in_exec;
+	int pwd_refs;	/* A pool of extra pwd references */
 	struct path root, pwd;
 } __randomize_layout;
 
@@ -22,6 +23,7 @@ extern void set_fs_root(struct fs_struct *, const struct path *);
 extern void set_fs_pwd(struct fs_struct *, const struct path *);
 extern struct fs_struct *copy_fs_struct(struct fs_struct *);
 extern void free_fs_struct(struct fs_struct *);
+extern void drain_fs_pwd_pool(struct fs_struct *);
 extern int unshare_fs_struct(void);
 
 static inline void get_fs_root(struct fs_struct *fs, struct path *root)
@@ -40,6 +42,45 @@ static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd)
 	read_sequnlock_excl(&fs->seq);
 }
 
+/* Borrow a pwd reference from the pool. Caller must hold fs->seq. */
+static inline void get_fs_pwd_pool_locked(struct fs_struct *fs, struct path *pwd)
+{
+	*pwd = fs->pwd;
+	if (fs->pwd_refs)
+		fs->pwd_refs--;
+	else
+		path_get(pwd);
+}
+
+/*
+ * Acquire a pwd reference from the pwd_refs pool, if available.
+ *
+ * Uses read_seqlock_excl() (writer spinlock without sequence bump) rather
+ * than write_seqlock() because modifying pwd_refs does not change the path
+ * values that lockless seq readers care about. Bumping the sequence counter
+ * would force unnecessary retries in concurrent get_fs_pwd()/get_fs_root()
+ * callers.
+ */
+static inline void get_fs_pwd_pool(struct fs_struct *fs, struct path *pwd)
+{
+	read_seqlock_excl(&fs->seq);
+	get_fs_pwd_pool_locked(fs, pwd);
+	read_sequnlock_excl(&fs->seq);
+}
+
+/* Release a pwd reference back to the pwd_refs pool, if appropriate. */
+static inline void put_fs_pwd_pool(struct fs_struct *fs, struct path *pwd)
+{
+	read_seqlock_excl(&fs->seq);
+	if (path_equal(&fs->pwd, pwd)) {
+		fs->pwd_refs++;
+		pwd = NULL;
+	}
+	read_sequnlock_excl(&fs->seq);
+	if (pwd)
+		path_put(pwd);
+}
+
 extern bool current_chrooted(void);
 
 static inline int current_umask(void)
diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
index 516aba5b858b5..54712ec61ffb7 100644
--- a/include/linux/fscrypt.h
+++ b/include/linux/fscrypt.h
@@ -450,8 +450,8 @@ u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name);
 
 /* bio.c */
 bool fscrypt_decrypt_bio(struct bio *bio);
-int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
-			  sector_t pblk, unsigned int len);
+int fscrypt_zeroout_range(const struct inode *inode, loff_t pos,
+			  sector_t sector, u64 len);
 
 /* hooks.c */
 int fscrypt_file_open(struct inode *inode, struct file *filp);
@@ -755,8 +755,8 @@ static inline bool fscrypt_decrypt_bio(struct bio *bio)
 	return true;
 }
 
-static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
-					sector_t pblk, unsigned int len)
+static inline int fscrypt_zeroout_range(const struct inode *inode, loff_t pos,
+					sector_t sector, u64 len)
 {
 	return -EOPNOTSUPP;
 }
@@ -865,19 +865,11 @@ static inline void fscrypt_set_ops(struct super_block *sb,
 
 bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode);
 
-void fscrypt_set_bio_crypt_ctx(struct bio *bio,
-			       const struct inode *inode, u64 first_lblk,
-			       gfp_t gfp_mask);
-
-void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio,
-				  const struct buffer_head *first_bh,
-				  gfp_t gfp_mask);
+void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
+			       loff_t pos, gfp_t gfp_mask);
 
 bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
-			   u64 next_lblk);
-
-bool fscrypt_mergeable_bio_bh(struct bio *bio,
-			      const struct buffer_head *next_bh);
+			   loff_t pos);
 
 bool fscrypt_dio_supported(struct inode *inode);
 
@@ -892,22 +884,11 @@ static inline bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
 
 static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio,
 					     const struct inode *inode,
-					     u64 first_lblk, gfp_t gfp_mask) { }
-
-static inline void fscrypt_set_bio_crypt_ctx_bh(
-					 struct bio *bio,
-					 const struct buffer_head *first_bh,
-					 gfp_t gfp_mask) { }
+					     loff_t pos, gfp_t gfp_mask) { }
 
 static inline bool fscrypt_mergeable_bio(struct bio *bio,
 					 const struct inode *inode,
-					 u64 next_lblk)
-{
-	return true;
-}
-
-static inline bool fscrypt_mergeable_bio_bh(struct bio *bio,
-					    const struct buffer_head *next_bh)
+					 loff_t pos)
 {
 	return true;
 }
diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h
index 897d6211c1635..e9522233f9b57 100644
--- a/include/linux/fsl/mc.h
+++ b/include/linux/fsl/mc.h
@@ -361,9 +361,11 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct fsl_mc_command *cmd);
 
 #ifdef CONFIG_FSL_MC_BUS
 #define dev_is_fsl_mc(_dev) ((_dev)->bus == &fsl_mc_bus_type)
+u32 fsl_mc_get_msi_id(struct device *dev);
 #else
 /* If fsl-mc bus is not present device cannot belong to fsl-mc bus */
 #define dev_is_fsl_mc(_dev) (0)
+#define fsl_mc_get_msi_id(_dev)	(0)
 #endif
 
 /* Macro to check if a device is a container device */
@@ -423,10 +425,6 @@ int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev,
 
 void fsl_mc_object_free(struct fsl_mc_device *mc_adev);
 
-struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
-						struct msi_domain_info *info,
-						struct irq_domain *parent);
-
 int __must_check fsl_mc_allocate_irqs(struct fsl_mc_device *mc_dev);
 
 void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev);
diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h
index 6a9a1acb5aad3..fc5d0b5edadc0 100644
--- a/include/linux/generic_pt/common.h
+++ b/include/linux/generic_pt/common.h
@@ -175,6 +175,22 @@ enum {
 	PT_FEAT_VTDSS_FORCE_WRITEABLE,
 };
 
+struct pt_riscv_32 {
+	struct pt_common common;
+};
+
+struct pt_riscv_64 {
+	struct pt_common common;
+};
+
+enum {
+	/*
+	 * Support the 64k contiguous page size following the Svnapot extension.
+	 */
+	PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START,
+
+};
+
 struct pt_x86_64 {
 	struct pt_common common;
 };
diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
index 9eefbb74efd08..dd0edd02a48a2 100644
--- a/include/linux/generic_pt/iommu.h
+++ b/include/linux/generic_pt/iommu.h
@@ -66,6 +66,13 @@ struct pt_iommu {
 	struct device *iommu_device;
 };
 
+static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain)
+{
+	if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt)
+		return NULL;
+	return container_of(domain, struct pt_iommu, domain);
+}
+
 /**
  * struct pt_iommu_info - Details about the IOMMU page table
  *
@@ -80,6 +87,56 @@ struct pt_iommu_info {
 };
 
 struct pt_iommu_ops {
+	/**
+	 * @map_range: Install translation for an IOVA range
+	 * @iommu_table: Table to manipulate
+	 * @iova: IO virtual address to start
+	 * @paddr: Physical/Output address to start
+	 * @len: Length of the range starting from @iova
+	 * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO
+	 * @gfp: GFP flags for any memory allocations
+	 *
+	 * The range starting at IOVA will have paddr installed into it. The
+	 * rage is automatically segmented into optimally sized table entries,
+	 * and can have any valid alignment.
+	 *
+	 * On error the caller will probably want to invoke unmap on the range
+	 * from iova up to the amount indicated by @mapped to return the table
+	 * back to an unchanged state.
+	 *
+	 * Context: The caller must hold a write range lock that includes
+	 * the whole range.
+	 *
+	 * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA
+	 * that were mapped are added to @mapped, @mapped is not zerod first.
+	 */
+	int (*map_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+			 phys_addr_t paddr, dma_addr_t len, unsigned int prot,
+			 gfp_t gfp, size_t *mapped);
+
+	/**
+	 * @unmap_range: Make a range of IOVA empty/not present
+	 * @iommu_table: Table to manipulate
+	 * @iova: IO virtual address to start
+	 * @len: Length of the range starting from @iova
+	 * @iotlb_gather: Gather struct that must be flushed on return
+	 *
+	 * unmap_range() will remove a translation created by map_range(). It
+	 * cannot subdivide a mapping created by map_range(), so it should be
+	 * called with IOVA ranges that match those passed to map_pages. The
+	 * IOVA range can aggregate contiguous map_range() calls so long as no
+	 * individual range is split.
+	 *
+	 * Context: The caller must hold a write range lock that includes
+	 * the whole range.
+	 *
+	 * Returns: Number of bytes of VA unmapped. iova + res will be the
+	 * point unmapping stopped.
+	 */
+	size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+			      dma_addr_t len,
+			      struct iommu_iotlb_gather *iotlb_gather);
+
 	/**
 	 * @set_dirty: Make the iova write dirty
 	 * @iommu_table: Table to manipulate
@@ -194,14 +251,6 @@ struct pt_iommu_cfg {
 #define IOMMU_PROTOTYPES(fmt)                                                  \
 	phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \
 						  dma_addr_t iova);            \
-	int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain,            \
-				       unsigned long iova, phys_addr_t paddr,  \
-				       size_t pgsize, size_t pgcount,          \
-				       int prot, gfp_t gfp, size_t *mapped);   \
-	size_t pt_iommu_##fmt##_unmap_pages(                                   \
-		struct iommu_domain *domain, unsigned long iova,               \
-		size_t pgsize, size_t pgcount,                                 \
-		struct iommu_iotlb_gather *iotlb_gather);                      \
 	int pt_iommu_##fmt##_read_and_clear_dirty(                             \
 		struct iommu_domain *domain, unsigned long iova, size_t size,  \
 		unsigned long flags, struct iommu_dirty_bitmap *dirty);        \
@@ -222,9 +271,7 @@ struct pt_iommu_cfg {
  * iommu_pt
  */
 #define IOMMU_PT_DOMAIN_OPS(fmt)                        \
-	.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \
-	.map_pages = &pt_iommu_##fmt##_map_pages,       \
-	.unmap_pages = &pt_iommu_##fmt##_unmap_pages
+	.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys
 #define IOMMU_PT_DIRTY_OPS(fmt) \
 	.read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty
 
@@ -275,6 +322,17 @@ struct pt_iommu_vtdss_hw_info {
 
 IOMMU_FORMAT(vtdss, vtdss_pt);
 
+struct pt_iommu_riscv_64_cfg {
+	struct pt_iommu_cfg common;
+};
+
+struct pt_iommu_riscv_64_hw_info {
+	u64 ppn;
+	u8 fsc_iosatp_mode;
+};
+
+IOMMU_FORMAT(riscv_64, riscv_64pt);
+
 struct pt_iommu_x86_64_cfg {
 	struct pt_iommu_cfg common;
 	/* 4 is a 57 bit 5 level table */
diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h
index d4da060b75322..db462b860d183 100644
--- a/include/linux/genl_magic_func.h
+++ b/include/linux/genl_magic_func.h
@@ -130,41 +130,53 @@ static void dprint_array(const char *dir, int nla_type,
  *									{{{2
  */
 
-/* processing of generic netlink messages is serialized.
- * use one static buffer for parsing of nested attributes */
-static struct nlattr *nested_attr_tb[128];
-
 #undef GENL_struct
 #define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-/* *_from_attrs functions are static, but potentially unused */		\
 static int __ ## s_name ## _from_attrs(struct s_name *s,		\
+		struct nlattr ***ret_nested_attribute_table,		\
 		struct genl_info *info, bool exclude_invariants)	\
 {									\
 	const int maxtype = ARRAY_SIZE(s_name ## _nl_policy)-1;		\
 	struct nlattr *tla = info->attrs[tag_number];			\
-	struct nlattr **ntb = nested_attr_tb;				\
+	struct nlattr **ntb;						\
 	struct nlattr *nla;						\
-	int err;							\
-	BUILD_BUG_ON(ARRAY_SIZE(s_name ## _nl_policy) > ARRAY_SIZE(nested_attr_tb));	\
+	int err = 0;							\
+	if (ret_nested_attribute_table)					\
+		*ret_nested_attribute_table = NULL;			\
 	if (!tla)							\
 		return -ENOMSG;						\
+	ntb = kcalloc(ARRAY_SIZE(s_name ## _nl_policy), sizeof(*ntb), GFP_KERNEL); \
+	if (!ntb)							\
+		return -ENOMEM;						\
 	DPRINT_TLA(#s_name, "<=-", #tag_name);				\
 	err = drbd_nla_parse_nested(ntb, maxtype, tla, s_name ## _nl_policy);	\
 	if (err)							\
-		return err;						\
+		goto out;						\
 									\
 	s_fields							\
-	return 0;							\
+ out:									\
+	if (!err && ret_nested_attribute_table)				\
+		*ret_nested_attribute_table = ntb;			\
+	else								\
+		kfree(ntb);						\
+	return err;							\
 }					__attribute__((unused))		\
 static int s_name ## _from_attrs(struct s_name *s,			\
 						struct genl_info *info)	\
 {									\
-	return __ ## s_name ## _from_attrs(s, info, false);		\
+	return __ ## s_name ## _from_attrs(s, NULL, info, false);	\
+}					__attribute__((unused))		\
+static int s_name ## _ntb_from_attrs(					\
+			struct nlattr ***ret_nested_attribute_table,	\
+						struct genl_info *info)	\
+{									\
+	return __ ## s_name ## _from_attrs(NULL,			\
+			ret_nested_attribute_table, info, false);	\
 }					__attribute__((unused))		\
 static int s_name ## _from_attrs_for_change(struct s_name *s,		\
 						struct genl_info *info)	\
 {									\
-	return __ ## s_name ## _from_attrs(s, info, true);		\
+	return __ ## s_name ## _from_attrs(s, NULL, info, true);	\
 }					__attribute__((unused))		\
 
 #define __assign(attr_nr, attr_flag, name, nla_type, type, assignment...)	\
@@ -172,7 +184,8 @@ static int s_name ## _from_attrs_for_change(struct s_name *s,		\
 		if (nla) {						\
 			if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) {		\
 				pr_info("<< must not change invariant attr: %s\n", #name);	\
-				return -EEXIST;				\
+				err = -EEXIST;				\
+				goto out;				\
 			}						\
 			assignment;					\
 		} else if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) {		\
@@ -180,7 +193,8 @@ static int s_name ## _from_attrs_for_change(struct s_name *s,		\
 			/* which was expected */			\
 		} else if ((attr_flag) & DRBD_F_REQUIRED) {		\
 			pr_info("<< missing attr: %s\n", #name);	\
-			return -ENOMSG;					\
+			err = -ENOMSG;					\
+			goto out;					\
 		}
 
 #undef __field
@@ -271,12 +285,12 @@ enum CONCATENATE(GENL_MAGIC_FAMILY, group_ids) {
 #undef GENL_mc_group
 #define GENL_mc_group(group)						\
 static int CONCATENATE(GENL_MAGIC_FAMILY, _genl_multicast_ ## group)(	\
-	struct sk_buff *skb, gfp_t flags)				\
+	struct sk_buff *skb)						\
 {									\
 	unsigned int group_id =						\
 		CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group);		\
-	return genlmsg_multicast(&ZZZ_genl_family, skb, 0,		\
-				 group_id, flags);			\
+	return genlmsg_multicast_allns(&ZZZ_genl_family, skb, 0,	\
+				 group_id);				\
 }
 
 #include GENL_MAGIC_INCLUDE_FILE
@@ -298,6 +312,8 @@ static struct genl_family ZZZ_genl_family __ro_after_init = {
 	.resv_start_op = 42, /* drbd is currently the only user */
 	.n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps),
 	.module = THIS_MODULE,
+	.netnsok = false,
+	.parallel_ops = true,
 };
 
 int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void)
diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h
index 0d84085829186..3efb5cb1e1d16 100644
--- a/include/linux/gpio/consumer.h
+++ b/include/linux/gpio/consumer.h
@@ -6,6 +6,8 @@
 #include <linux/err.h>
 #include <linux/types.h>
 
+#include "defs.h"
+
 struct acpi_device;
 struct device;
 struct fwnode_handle;
diff --git a/include/linux/gpio/defs.h b/include/linux/gpio/defs.h
new file mode 100644
index 0000000000000..b69fd7c041b28
--- /dev/null
+++ b/include/linux/gpio/defs.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __LINUX_GPIO_DEFS_H
+#define __LINUX_GPIO_DEFS_H
+
+#define GPIO_LINE_DIRECTION_IN		1
+#define GPIO_LINE_DIRECTION_OUT		0
+
+#endif /* __LINUX_GPIO_DEFS_H */
diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
index fabe2baf7b509..17511434ed077 100644
--- a/include/linux/gpio/driver.h
+++ b/include/linux/gpio/driver.h
@@ -20,6 +20,8 @@
 #include <asm/msi.h>
 #endif
 
+#include "defs.h"
+
 struct device;
 struct irq_chip;
 struct irq_data;
@@ -42,9 +44,6 @@ union gpio_irq_fwspec {
 #endif
 };
 
-#define GPIO_LINE_DIRECTION_IN	1
-#define GPIO_LINE_DIRECTION_OUT	0
-
 /**
  * struct gpio_irq_chip - GPIO interrupt controller
  */
@@ -344,11 +343,17 @@ struct gpio_irq_chip {
  * @direction_output: configures signal "offset" as output, returns 0 on
  *	success or a negative error number. This can be omitted on input-only
  *	or output-only gpio chips.
- * @get: returns value for signal "offset", 0=low, 1=high, or negative error
+ * @get: returns value for signal "offset", 0=low, 1=high, or negative error.
+ *	the low and high values are defined as physical low on the line
+ *	in/out to the connector such as a physical pad, pin or rail. The GPIO
+ *	library has internal logic to handle lines that are active low, such
+ *	as indicated by overstrike or #name in a schematic, and the driver
+ *	should not try to second-guess the logic value of a line.
  * @get_multiple: reads values for multiple signals defined by "mask" and
  *	stores them in "bits", returns 0 on success or negative error
  * @set: assigns output value for signal "offset", returns 0 on success or
- *       negative error value
+ *	negative error value. The output value follows the same semantic
+ *	rules as for @get.
  * @set_multiple: assigns output values for multiple signals defined by
  *                "mask", returns 0 on success or negative error value
  * @set_config: optional hook for all kinds of settings. Uses the same
diff --git a/include/linux/gpio/generic.h b/include/linux/gpio/generic.h
index ff566dc9c3cbe..de43c06c83ef7 100644
--- a/include/linux/gpio/generic.h
+++ b/include/linux/gpio/generic.h
@@ -3,9 +3,15 @@
 #ifndef __LINUX_GPIO_GENERIC_H
 #define __LINUX_GPIO_GENERIC_H
 
+#include <linux/bits.h>
+#include <linux/bug.h>
 #include <linux/cleanup.h>
-#include <linux/gpio/driver.h>
+#include <linux/container_of.h>
+#include <linux/errno.h>
 #include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include <linux/gpio/driver.h>
 
 struct device;
 
diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h
index 592a774a53cdf..8061b98263611 100644
--- a/include/linux/gpio/gpio-nomadik.h
+++ b/include/linux/gpio/gpio-nomadik.h
@@ -114,8 +114,7 @@ struct nmk_gpio_chip {
 }
 
 /**
- * enum prcm_gpiocr_reg_index
- * Used to reference an PRCM GPIOCR register address.
+ * enum prcm_gpiocr_reg_index - Used to reference a PRCM GPIOCR register address.
  */
 enum prcm_gpiocr_reg_index {
 	PRCM_IDX_GPIOCR1,
@@ -123,8 +122,7 @@ enum prcm_gpiocr_reg_index {
 	PRCM_IDX_GPIOCR3
 };
 /**
- * enum prcm_gpiocr_altcx_index
- * Used to reference an Other alternate-C function.
+ * enum prcm_gpiocr_altcx_index - Used to reference an Other alternate-C function.
  */
 enum prcm_gpiocr_altcx_index {
 	PRCM_IDX_GPIOCR_ALTC1,
@@ -135,7 +133,7 @@ enum prcm_gpiocr_altcx_index {
 };
 
 /**
- * struct prcm_gpio_altcx - Other alternate-C function
+ * struct prcm_gpiocr_altcx - Other alternate-C function
  * @used: other alternate-C function availability
  * @reg_index: PRCM GPIOCR register index used to control the function
  * @control_bit: PRCM GPIOCR bit used to control the function
@@ -147,7 +145,7 @@ struct prcm_gpiocr_altcx {
 } __packed;
 
 /**
- * struct prcm_gpio_altcx_pin_desc - Other alternate-C pin
+ * struct prcm_gpiocr_altcx_pin_desc - Other alternate-C pin
  * @pin: The pin number
  * @altcx: array of other alternate-C[1-4] functions
  */
@@ -193,7 +191,7 @@ struct nmk_pingroup {
  *		numbering.
  * @npins:	The number of entries in @pins.
  * @functions:	The functions supported on this SoC.
- * @nfunction:	The number of entries in @functions.
+ * @nfunctions:	The number of entries in @functions.
  * @groups:	An array describing all pin groups the pin SoC supports.
  * @ngroups:	The number of entries in @groups.
  * @altcx_pins:	The pins that support Other alternate-C function on this SoC
diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h
index 44e5f162973eb..5eb88f5d0630f 100644
--- a/include/linux/gpio/machine.h
+++ b/include/linux/gpio/machine.h
@@ -46,23 +46,6 @@ struct gpiod_lookup_table {
 	struct gpiod_lookup table[];
 };
 
-/**
- * struct gpiod_hog - GPIO line hog table
- * @chip_label: name of the chip the GPIO belongs to
- * @chip_hwnum: hardware number (i.e. relative to the chip) of the GPIO
- * @line_name: consumer name for the hogged line
- * @lflags: bitmask of gpio_lookup_flags GPIO_* values
- * @dflags: GPIO flags used to specify the direction and value
- */
-struct gpiod_hog {
-	struct list_head list;
-	const char *chip_label;
-	u16 chip_hwnum;
-	const char *line_name;
-	unsigned long lflags;
-	int dflags;
-};
-
 /*
  * Helper for lookup tables with just one single lookup for a device.
  */
@@ -95,24 +78,10 @@ static struct gpiod_lookup_table _name = {				\
 	.flags = _flags,                                                  \
 }
 
-/*
- * Simple definition of a single GPIO hog in an array.
- */
-#define GPIO_HOG(_chip_label, _chip_hwnum, _line_name, _lflags, _dflags)  \
-(struct gpiod_hog) {                                                      \
-	.chip_label = _chip_label,                                        \
-	.chip_hwnum = _chip_hwnum,                                        \
-	.line_name = _line_name,                                          \
-	.lflags = _lflags,                                                \
-	.dflags = _dflags,                                                \
-}
-
 #ifdef CONFIG_GPIOLIB
 void gpiod_add_lookup_table(struct gpiod_lookup_table *table);
 void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n);
 void gpiod_remove_lookup_table(struct gpiod_lookup_table *table);
-void gpiod_add_hogs(struct gpiod_hog *hogs);
-void gpiod_remove_hogs(struct gpiod_hog *hogs);
 #else /* ! CONFIG_GPIOLIB */
 static inline
 void gpiod_add_lookup_table(struct gpiod_lookup_table *table) {}
@@ -120,8 +89,6 @@ static inline
 void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n) {}
 static inline
 void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) {}
-static inline void gpiod_add_hogs(struct gpiod_hog *hogs) {}
-static inline void gpiod_remove_hogs(struct gpiod_hog *hogs) {}
 #endif /* CONFIG_GPIOLIB */
 
 #endif /* __LINUX_GPIO_MACHINE_H */
diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h
new file mode 100644
index 0000000000000..5fa917ba5450b
--- /dev/null
+++ b/include/linux/gpu_buddy.h
@@ -0,0 +1,241 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+
+#ifndef __GPU_BUDDY_H__
+#define __GPU_BUDDY_H__
+
+#include <linux/bitops.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/rbtree.h>
+#include <linux/rbtree_augmented.h>
+
+/**
+ * GPU_BUDDY_RANGE_ALLOCATION - Allocate within a specific address range
+ *
+ * When set, allocation is restricted to the range [start, end) specified
+ * in gpu_buddy_alloc_blocks(). Without this flag, start/end are ignored
+ * and allocation can use any free space.
+ */
+#define GPU_BUDDY_RANGE_ALLOCATION		BIT(0)
+
+/**
+ * GPU_BUDDY_TOPDOWN_ALLOCATION - Allocate from top of address space
+ *
+ * Allocate starting from high addresses and working down. Useful for
+ * separating different allocation types (e.g., kernel vs userspace)
+ * to reduce fragmentation.
+ */
+#define GPU_BUDDY_TOPDOWN_ALLOCATION		BIT(1)
+
+/**
+ * GPU_BUDDY_CONTIGUOUS_ALLOCATION - Require physically contiguous blocks
+ *
+ * The allocation must be satisfied with a single contiguous block.
+ * If the requested size cannot be allocated contiguously, the
+ * allocation fails with -ENOSPC.
+ */
+#define GPU_BUDDY_CONTIGUOUS_ALLOCATION		BIT(2)
+
+/**
+ * GPU_BUDDY_CLEAR_ALLOCATION - Prefer pre-cleared (zeroed) memory
+ *
+ * Attempt to allocate from the clear tree first. If insufficient clear
+ * memory is available, falls back to dirty memory. Useful when the
+ * caller needs zeroed memory and wants to avoid GPU clear operations.
+ */
+#define GPU_BUDDY_CLEAR_ALLOCATION		BIT(3)
+
+/**
+ * GPU_BUDDY_CLEARED - Mark returned blocks as cleared
+ *
+ * Used with gpu_buddy_free_list() to indicate that the memory being
+ * freed has been cleared (zeroed). The blocks will be placed in the
+ * clear tree for future GPU_BUDDY_CLEAR_ALLOCATION requests.
+ */
+#define GPU_BUDDY_CLEARED			BIT(4)
+
+/**
+ * GPU_BUDDY_TRIM_DISABLE - Disable automatic block trimming
+ *
+ * By default, if an allocation is smaller than the allocated block,
+ * excess memory is trimmed and returned to the free pool. This flag
+ * disables trimming, keeping the full power-of-two block size.
+ */
+#define GPU_BUDDY_TRIM_DISABLE			BIT(5)
+
+enum gpu_buddy_free_tree {
+	GPU_BUDDY_CLEAR_TREE = 0,
+	GPU_BUDDY_DIRTY_TREE,
+	GPU_BUDDY_MAX_FREE_TREES,
+};
+
+#define for_each_free_tree(tree) \
+	for ((tree) = 0; (tree) < GPU_BUDDY_MAX_FREE_TREES; (tree)++)
+
+/**
+ * struct gpu_buddy_block - Block within a buddy allocator
+ *
+ * Each block in the buddy allocator is represented by this structure.
+ * Blocks are organized in a binary tree where each parent block can be
+ * split into two children (left and right buddies). The allocator manages
+ * blocks at various orders (power-of-2 sizes) from chunk_size up to the
+ * largest contiguous region.
+ *
+ * @private: Private data owned by the allocator user (e.g., driver-specific data)
+ * @link: List node for user ownership while block is allocated
+ */
+struct gpu_buddy_block {
+/* private: */
+	/*
+	 * Header bit layout:
+	 * - Bits 63:12: block offset within the address space
+	 * - Bits 11:10: state (ALLOCATED, FREE, or SPLIT)
+	 * - Bit 9: clear bit (1 if memory is zeroed)
+	 * - Bits 8:6: reserved
+	 * - Bits 5:0: order (log2 of size relative to chunk_size)
+	 */
+#define GPU_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
+#define GPU_BUDDY_HEADER_STATE  GENMASK_ULL(11, 10)
+#define   GPU_BUDDY_ALLOCATED	   (1 << 10)
+#define   GPU_BUDDY_FREE	   (2 << 10)
+#define   GPU_BUDDY_SPLIT	   (3 << 10)
+#define GPU_BUDDY_HEADER_CLEAR  GENMASK_ULL(9, 9)
+/* Free to be used, if needed in the future */
+#define GPU_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6)
+#define GPU_BUDDY_HEADER_ORDER  GENMASK_ULL(5, 0)
+	u64 header;
+
+	struct gpu_buddy_block *left;
+	struct gpu_buddy_block *right;
+	struct gpu_buddy_block *parent;
+/* public: */
+	void *private; /* owned by creator */
+
+	/*
+	 * While the block is allocated by the user through gpu_buddy_alloc*,
+	 * the user has ownership of the link, for example to maintain within
+	 * a list, if so desired. As soon as the block is freed with
+	 * gpu_buddy_free* ownership is given back to the mm.
+	 */
+	union {
+/* private: */
+		struct rb_node rb;
+/* public: */
+		struct list_head link;
+	};
+/* private: */
+	struct list_head tmp_link;
+	unsigned int subtree_max_alignment;
+};
+
+/* Order-zero must be at least SZ_4K */
+#define GPU_BUDDY_MAX_ORDER (63 - 12)
+
+/**
+ * struct gpu_buddy - GPU binary buddy allocator
+ *
+ * The buddy allocator provides efficient power-of-two memory allocation
+ * with fast allocation and free operations. It is commonly used for GPU
+ * memory management where allocations can be split into power-of-two
+ * block sizes.
+ *
+ * Locking should be handled by the user; a simple mutex around
+ * gpu_buddy_alloc_blocks() and gpu_buddy_free_block()/gpu_buddy_free_list()
+ * should suffice.
+ *
+ * @n_roots: Number of root blocks in the roots array.
+ * @max_order: Maximum block order (log2 of largest block size / chunk_size).
+ * @chunk_size: Minimum allocation granularity in bytes. Must be at least SZ_4K.
+ * @size: Total size of the address space managed by this allocator in bytes.
+ * @avail: Total free space currently available for allocation in bytes.
+ * @clear_avail: Free space available in the clear tree (zeroed memory) in bytes.
+ *               This is a subset of @avail.
+ */
+struct gpu_buddy {
+/* private: */
+	/*
+	 * Array of red-black trees for free block management.
+	 * Indexed as free_trees[clear/dirty][order] where:
+	 * - Index 0 (GPU_BUDDY_CLEAR_TREE): blocks with zeroed content
+	 * - Index 1 (GPU_BUDDY_DIRTY_TREE): blocks with unknown content
+	 * Each tree holds free blocks of the corresponding order.
+	 */
+	struct rb_root **free_trees;
+	/*
+	 * Array of root blocks representing the top-level blocks of the
+	 * binary tree(s). Multiple roots exist when the total size is not
+	 * a power of two, with each root being the largest power-of-two
+	 * that fits in the remaining space.
+	 */
+	struct gpu_buddy_block **roots;
+/* public: */
+	unsigned int n_roots;
+	unsigned int max_order;
+	u64 chunk_size;
+	u64 size;
+	u64 avail;
+	u64 clear_avail;
+};
+
+static inline u64
+gpu_buddy_block_offset(const struct gpu_buddy_block *block)
+{
+	return block->header & GPU_BUDDY_HEADER_OFFSET;
+}
+
+static inline unsigned int
+gpu_buddy_block_order(struct gpu_buddy_block *block)
+{
+	return block->header & GPU_BUDDY_HEADER_ORDER;
+}
+
+static inline bool
+gpu_buddy_block_is_free(struct gpu_buddy_block *block)
+{
+	return (block->header & GPU_BUDDY_HEADER_STATE) == GPU_BUDDY_FREE;
+}
+
+static inline bool
+gpu_buddy_block_is_clear(struct gpu_buddy_block *block)
+{
+	return block->header & GPU_BUDDY_HEADER_CLEAR;
+}
+
+static inline u64
+gpu_buddy_block_size(struct gpu_buddy *mm,
+		     struct gpu_buddy_block *block)
+{
+	return mm->chunk_size << gpu_buddy_block_order(block);
+}
+
+int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size);
+
+void gpu_buddy_fini(struct gpu_buddy *mm);
+
+int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
+			   u64 start, u64 end, u64 size,
+			   u64 min_page_size,
+			   struct list_head *blocks,
+			   unsigned long flags);
+
+int gpu_buddy_block_trim(struct gpu_buddy *mm,
+			 u64 *start,
+			 u64 new_size,
+			 struct list_head *blocks);
+
+void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear);
+
+void gpu_buddy_free_block(struct gpu_buddy *mm, struct gpu_buddy_block *block);
+
+void gpu_buddy_free_list(struct gpu_buddy *mm,
+			 struct list_head *objects,
+			 unsigned int flags);
+
+void gpu_buddy_print(struct gpu_buddy *mm);
+void gpu_buddy_block_print(struct gpu_buddy *mm,
+			   struct gpu_buddy_block *block);
+#endif
diff --git a/include/linux/hid.h b/include/linux/hid.h
index 2990b9f94cb57..442a80d79e899 100644
--- a/include/linux/hid.h
+++ b/include/linux/hid.h
@@ -634,6 +634,38 @@ enum hid_battery_status {
 	HID_BATTERY_REPORTED,		/* Device sent unsolicited battery strength report */
 };
 
+/**
+ * struct hid_battery - represents a single battery power supply
+ * @dev: pointer to the parent hid_device
+ * @ps: the power supply instance
+ * @min: minimum battery value from HID descriptor
+ * @max: maximum battery value from HID descriptor
+ * @report_type: HID report type (input/feature)
+ * @report_id: HID report ID for this battery
+ * @charge_status: current charging status
+ * @status: battery reporting status
+ * @capacity: current battery capacity (0-100)
+ * @avoid_query: if true, avoid querying battery (e.g., for stylus)
+ * @present: if true, battery is present (may be dynamic)
+ * @ratelimit_time: rate limiting for battery reports
+ * @list: list node for linking into hid_device's battery list
+ */
+struct hid_battery {
+	struct hid_device *dev;
+	struct power_supply *ps;
+	__s32 min;
+	__s32 max;
+	__s32 report_type;
+	__s32 report_id;
+	__s32 charge_status;
+	enum hid_battery_status status;
+	__s32 capacity;
+	bool avoid_query;
+	bool present;
+	ktime_t ratelimit_time;
+	struct list_head list;
+};
+
 struct hid_driver;
 struct hid_ll_driver;
 
@@ -670,19 +702,10 @@ struct hid_device {
 #ifdef CONFIG_HID_BATTERY_STRENGTH
 	/*
 	 * Power supply information for HID devices which report
-	 * battery strength. power_supply was successfully registered if
-	 * battery is non-NULL.
+	 * battery strength. Each battery is tracked separately in the
+	 * batteries list.
 	 */
-	struct power_supply *battery;
-	__s32 battery_capacity;
-	__s32 battery_min;
-	__s32 battery_max;
-	__s32 battery_report_type;
-	__s32 battery_report_id;
-	__s32 battery_charge_status;
-	enum hid_battery_status battery_status;
-	bool battery_avoid_query;
-	ktime_t battery_ratelimit_time;
+	struct list_head batteries;
 #endif
 
 	unsigned long status;						/* see STAT flags above */
@@ -698,6 +721,7 @@ struct hid_device {
 	char name[128];							/* Device name */
 	char phys[64];							/* Device physical location */
 	char uniq[64];							/* Device unique identifier (serial #) */
+	u64 firmware_version;						/* Firmware version */
 
 	void *driver_data;
 
@@ -743,6 +767,15 @@ static inline void hid_set_drvdata(struct hid_device *hdev, void *data)
 	dev_set_drvdata(&hdev->dev, data);
 }
 
+#ifdef CONFIG_HID_BATTERY_STRENGTH
+static inline struct hid_battery *hid_get_battery(struct hid_device *hdev)
+{
+	if (list_empty(&hdev->batteries))
+		return NULL;
+	return list_first_entry(&hdev->batteries, struct hid_battery, list);
+}
+#endif
+
 #define HID_GLOBAL_STACK_SIZE 4
 #define HID_COLLECTION_STACK_SIZE 4
 
diff --git a/include/linux/host1x.h b/include/linux/host1x.h
index 5e7a63143a4ac..1f5f55917d1cd 100644
--- a/include/linux/host1x.h
+++ b/include/linux/host1x.h
@@ -453,16 +453,6 @@ void host1x_client_unregister(struct host1x_client *client);
 int host1x_client_suspend(struct host1x_client *client);
 int host1x_client_resume(struct host1x_client *client);
 
-struct tegra_mipi_device;
-
-struct tegra_mipi_device *tegra_mipi_request(struct device *device,
-					     struct device_node *np);
-void tegra_mipi_free(struct tegra_mipi_device *device);
-int tegra_mipi_enable(struct tegra_mipi_device *device);
-int tegra_mipi_disable(struct tegra_mipi_device *device);
-int tegra_mipi_start_calibration(struct tegra_mipi_device *device);
-int tegra_mipi_finish_calibration(struct tegra_mipi_device *device);
-
 /* host1x memory contexts */
 
 struct host1x_memory_context {
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 74adbd4e70032..9ced498fefaa2 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -13,6 +13,7 @@
 #define _LINUX_HRTIMER_H
 
 #include <linux/hrtimer_defs.h>
+#include <linux/hrtimer_rearm.h>
 #include <linux/hrtimer_types.h>
 #include <linux/init.h>
 #include <linux/list.h>
@@ -31,6 +32,13 @@
  *				  soft irq context
  * HRTIMER_MODE_HARD		- Timer callback function will be executed in
  *				  hard irq context even on PREEMPT_RT.
+ * HRTIMER_MODE_LAZY_REARM	- Avoid reprogramming if the timer was the
+ *				  first expiring timer and is moved into the
+ *				  future. Special mode for the HRTICK timer to
+ *				  avoid extensive reprogramming of the hardware,
+ *				  which is expensive in virtual machines. Risks
+ *				  a pointless expiry, but that's better than
+ *				  reprogramming on every context switch,
  */
 enum hrtimer_mode {
 	HRTIMER_MODE_ABS	= 0x00,
@@ -38,6 +46,7 @@ enum hrtimer_mode {
 	HRTIMER_MODE_PINNED	= 0x02,
 	HRTIMER_MODE_SOFT	= 0x04,
 	HRTIMER_MODE_HARD	= 0x08,
+	HRTIMER_MODE_LAZY_REARM	= 0x10,
 
 	HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
 	HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
@@ -55,33 +64,6 @@ enum hrtimer_mode {
 	HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
 };
 
-/*
- * Values to track state of the timer
- *
- * Possible states:
- *
- * 0x00		inactive
- * 0x01		enqueued into rbtree
- *
- * The callback state is not part of the timer->state because clearing it would
- * mean touching the timer after the callback, this makes it impossible to free
- * the timer from the callback function.
- *
- * Therefore we track the callback state in:
- *
- *	timer->base->cpu_base->running == timer
- *
- * On SMP it is possible to have a "callback function running and enqueued"
- * status. It happens for example when a posix timer expired and the callback
- * queued a signal. Between dropping the lock which protects the posix timer
- * and reacquiring the base lock of the hrtimer, another CPU can deliver the
- * signal and rearm the timer.
- *
- * All state transitions are protected by cpu_base->lock.
- */
-#define HRTIMER_STATE_INACTIVE	0x00
-#define HRTIMER_STATE_ENQUEUED	0x01
-
 /**
  * struct hrtimer_sleeper - simple sleeper structure
  * @timer:	embedded timer structure
@@ -134,11 +116,6 @@ static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
 	return timer->_softexpires;
 }
 
-static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
-{
-	return ktime_to_ns(timer->node.expires);
-}
-
 ktime_t hrtimer_cb_get_time(const struct hrtimer *timer);
 
 static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
@@ -146,24 +123,23 @@ static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
 	return ktime_sub(timer->node.expires, hrtimer_cb_get_time(timer));
 }
 
-static inline int hrtimer_is_hres_active(struct hrtimer *timer)
-{
-	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
-		timer->base->cpu_base->hres_active : 0;
-}
-
 #ifdef CONFIG_HIGH_RES_TIMERS
+extern unsigned int hrtimer_resolution;
 struct clock_event_device;
 
 extern void hrtimer_interrupt(struct clock_event_device *dev);
 
-extern unsigned int hrtimer_resolution;
+extern struct static_key_false hrtimer_highres_enabled_key;
 
-#else
+static inline bool hrtimer_highres_enabled(void)
+{
+	return static_branch_likely(&hrtimer_highres_enabled_key);
+}
 
+#else  /* CONFIG_HIGH_RES_TIMERS */
 #define hrtimer_resolution	(unsigned int)LOW_RES_NSEC
-
-#endif
+static inline bool hrtimer_highres_enabled(void) { return false; }
+#endif  /* !CONFIG_HIGH_RES_TIMERS */
 
 static inline ktime_t
 __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
@@ -293,8 +269,8 @@ extern bool hrtimer_active(const struct hrtimer *timer);
  */
 static inline bool hrtimer_is_queued(struct hrtimer *timer)
 {
-	/* The READ_ONCE pairs with the update functions of timer->state */
-	return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
+	/* The READ_ONCE pairs with the update functions of timer->is_queued */
+	return READ_ONCE(timer->is_queued);
 }
 
 /*
diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h
index 02b010df65709..52ed9e46ff136 100644
--- a/include/linux/hrtimer_defs.h
+++ b/include/linux/hrtimer_defs.h
@@ -19,21 +19,23 @@
  *			timer to a base on another cpu.
  * @clockid:		clock id for per_cpu support
  * @seq:		seqcount around __run_hrtimer
+ * @expires_next:	Absolute time of the next event in this clock base
  * @running:		pointer to the currently running hrtimer
  * @active:		red black tree root node for the active timers
  * @offset:		offset of this clock to the monotonic base
  */
 struct hrtimer_clock_base {
-	struct hrtimer_cpu_base	*cpu_base;
-	unsigned int		index;
-	clockid_t		clockid;
-	seqcount_raw_spinlock_t	seq;
-	struct hrtimer		*running;
-	struct timerqueue_head	active;
-	ktime_t			offset;
+	struct hrtimer_cpu_base		*cpu_base;
+	const unsigned int		index;
+	const clockid_t			clockid;
+	seqcount_raw_spinlock_t		seq;
+	ktime_t				expires_next;
+	struct hrtimer			*running;
+	struct timerqueue_linked_head	active;
+	ktime_t				offset;
 } __hrtimer_clock_base_align;
 
-enum  hrtimer_base_type {
+enum hrtimer_base_type {
 	HRTIMER_BASE_MONOTONIC,
 	HRTIMER_BASE_REALTIME,
 	HRTIMER_BASE_BOOTTIME,
@@ -42,37 +44,36 @@ enum  hrtimer_base_type {
 	HRTIMER_BASE_REALTIME_SOFT,
 	HRTIMER_BASE_BOOTTIME_SOFT,
 	HRTIMER_BASE_TAI_SOFT,
-	HRTIMER_MAX_CLOCK_BASES,
+	HRTIMER_MAX_CLOCK_BASES
 };
 
 /**
  * struct hrtimer_cpu_base - the per cpu clock bases
- * @lock:		lock protecting the base and associated clock bases
- *			and timers
- * @cpu:		cpu number
- * @active_bases:	Bitfield to mark bases with active timers
- * @clock_was_set_seq:	Sequence counter of clock was set events
- * @hres_active:	State of high resolution mode
- * @in_hrtirq:		hrtimer_interrupt() is currently executing
- * @hang_detected:	The last hrtimer interrupt detected a hang
- * @softirq_activated:	displays, if the softirq is raised - update of softirq
- *			related settings is not required then.
- * @nr_events:		Total number of hrtimer interrupt events
- * @nr_retries:		Total number of hrtimer interrupt retries
- * @nr_hangs:		Total number of hrtimer interrupt hangs
- * @max_hang_time:	Maximum time spent in hrtimer_interrupt
- * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
- *			 expired
- * @online:		CPU is online from an hrtimers point of view
- * @timer_waiters:	A hrtimer_cancel() invocation waits for the timer
- *			callback to finish.
- * @expires_next:	absolute time of the next event, is required for remote
- *			hrtimer enqueue; it is the total first expiry time (hard
- *			and soft hrtimer are taken into account)
- * @next_timer:		Pointer to the first expiring timer
- * @softirq_expires_next: Time to check, if soft queues needs also to be expired
- * @softirq_next_timer: Pointer to the first expiring softirq based timer
- * @clock_base:		array of clock bases for this cpu
+ * @lock:			lock protecting the base and associated clock bases and timers
+ * @cpu:			cpu number
+ * @active_bases:		Bitfield to mark bases with active timers
+ * @clock_was_set_seq:		Sequence counter of clock was set events
+ * @hres_active:		State of high resolution mode
+ * @deferred_rearm:		A deferred rearm is pending
+ * @deferred_needs_update:	The deferred rearm must re-evaluate the first timer
+ * @hang_detected:		The last hrtimer interrupt detected a hang
+ * @softirq_activated:		displays, if the softirq is raised - update of softirq
+ *				related settings is not required then.
+ * @nr_events:			Total number of hrtimer interrupt events
+ * @nr_retries:			Total number of hrtimer interrupt retries
+ * @nr_hangs:			Total number of hrtimer interrupt hangs
+ * @max_hang_time:		Maximum time spent in hrtimer_interrupt
+ * @softirq_expiry_lock:	Lock which is taken while softirq based hrtimer are expired
+ * @online:			CPU is online from an hrtimers point of view
+ * @timer_waiters:		A hrtimer_cancel() waiters for the timer callback to finish.
+ * @expires_next:		Absolute time of the next event, is required for remote
+ *				hrtimer enqueue; it is the total first expiry time (hard
+ *				and soft hrtimer are taken into account)
+ * @next_timer:			Pointer to the first expiring timer
+ * @softirq_expires_next:	Time to check, if soft queues needs also to be expired
+ * @softirq_next_timer:		Pointer to the first expiring softirq based timer
+ * @deferred_expires_next:	Cached expires next value for deferred rearm
+ * @clock_base:			Array of clock bases for this cpu
  *
  * Note: next_timer is just an optimization for __remove_hrtimer().
  *	 Do not dereference the pointer because it is not reliable on
@@ -83,11 +84,12 @@ struct hrtimer_cpu_base {
 	unsigned int			cpu;
 	unsigned int			active_bases;
 	unsigned int			clock_was_set_seq;
-	unsigned int			hres_active		: 1,
-					in_hrtirq		: 1,
-					hang_detected		: 1,
-					softirq_activated       : 1,
-					online			: 1;
+	bool				hres_active;
+	bool				deferred_rearm;
+	bool				deferred_needs_update;
+	bool				hang_detected;
+	bool				softirq_activated;
+	bool				online;
 #ifdef CONFIG_HIGH_RES_TIMERS
 	unsigned int			nr_events;
 	unsigned short			nr_retries;
@@ -102,6 +104,7 @@ struct hrtimer_cpu_base {
 	struct hrtimer			*next_timer;
 	ktime_t				softirq_expires_next;
 	struct hrtimer			*softirq_next_timer;
+	ktime_t				deferred_expires_next;
 	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
 	call_single_data_t		csd;
 } ____cacheline_aligned;
diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h
new file mode 100644
index 0000000000000..a6f2e5d5e1c7d
--- /dev/null
+++ b/include/linux/hrtimer_rearm.h
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef _LINUX_HRTIMER_REARM_H
+#define _LINUX_HRTIMER_REARM_H
+
+#ifdef CONFIG_HRTIMER_REARM_DEFERRED
+#include <linux/thread_info.h>
+
+void __hrtimer_rearm_deferred(void);
+
+/*
+ * This is purely CPU local, so check the TIF bit first to avoid the overhead of
+ * the atomic test_and_clear_bit() operation for the common case where the bit
+ * is not set.
+ */
+static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)
+{
+	lockdep_assert_irqs_disabled();
+
+	if (unlikely(tif_work & _TIF_HRTIMER_REARM)) {
+		clear_thread_flag(TIF_HRTIMER_REARM);
+		return true;
+	}
+	return false;
+}
+
+#define TIF_REARM_MASK	(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM)
+
+/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */
+static __always_inline bool
+hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask)
+{
+	/* Help the compiler to optimize the function out for syscall returns */
+	if (!(tif_mask & _TIF_HRTIMER_REARM))
+		return false;
+	/*
+	 * Rearm the timer if none of the resched flags is set before going into
+	 * the loop which re-enables interrupts.
+	 */
+	if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) {
+		clear_thread_flag(TIF_HRTIMER_REARM);
+		__hrtimer_rearm_deferred();
+		/* Don't go into the loop if HRTIMER_REARM was the only flag */
+		*tif_work &= ~TIF_HRTIMER_REARM;
+		return !*tif_work;
+	}
+	return false;
+}
+
+/* Invoked from the time slice extension decision function */
+static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work)
+{
+	if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work))
+		__hrtimer_rearm_deferred();
+}
+
+/*
+ * This is to be called on all irqentry_exit() paths that will enable
+ * interrupts.
+ */
+static __always_inline void hrtimer_rearm_deferred(void)
+{
+	hrtimer_rearm_deferred_tif(read_thread_flags());
+}
+
+/*
+ * Invoked from the scheduler on entry to __schedule() so it can defer
+ * rearming after the load balancing callbacks which might change hrtick.
+ */
+static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void)
+{
+	return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags());
+}
+
+#else  /* CONFIG_HRTIMER_REARM_DEFERRED */
+static __always_inline void __hrtimer_rearm_deferred(void) { }
+static __always_inline void hrtimer_rearm_deferred(void) { }
+static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { }
+static __always_inline bool
+hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; }
+static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; }
+#endif  /* !CONFIG_HRTIMER_REARM_DEFERRED */
+
+#endif
diff --git a/include/linux/hrtimer_types.h b/include/linux/hrtimer_types.h
index 8fbbb6bdf7a18..b5dacc8271a4a 100644
--- a/include/linux/hrtimer_types.h
+++ b/include/linux/hrtimer_types.h
@@ -17,7 +17,7 @@ enum hrtimer_restart {
 
 /**
  * struct hrtimer - the basic hrtimer structure
- * @node:	timerqueue node, which also manages node.expires,
+ * @node:	Linked timerqueue node, which also manages node.expires,
  *		the absolute expiry time in the hrtimers internal
  *		representation. The time is related to the clock on
  *		which the timer is based. Is setup by adding
@@ -28,23 +28,26 @@ enum hrtimer_restart {
  *		was armed.
  * @function:	timer expiry callback function
  * @base:	pointer to the timer base (per cpu and per clock)
- * @state:	state information (See bit values above)
+ * @is_queued:	Indicates whether a timer is enqueued or not
  * @is_rel:	Set if the timer was armed relative
  * @is_soft:	Set if hrtimer will be expired in soft interrupt context.
  * @is_hard:	Set if hrtimer will be expired in hard interrupt context
  *		even on RT.
+ * @is_lazy:	Set if the timer is frequently rearmed to avoid updates
+ *		of the clock event device
  *
  * The hrtimer structure must be initialized by hrtimer_setup()
  */
 struct hrtimer {
-	struct timerqueue_node		node;
+	struct timerqueue_linked_node	node;
+	struct hrtimer_clock_base	*base;
+	bool				is_queued;
+	bool				is_rel;
+	bool				is_soft;
+	bool				is_hard;
+	bool				is_lazy;
 	ktime_t				_softexpires;
 	enum hrtimer_restart		(*__private function)(struct hrtimer *);
-	struct hrtimer_clock_base	*base;
-	u8				state;
-	u8				is_rel;
-	u8				is_soft;
-	u8				is_hard;
 };
 
 #endif /* _LINUX_HRTIMER_TYPES_H */
diff --git a/include/linux/hsi/hsi.h b/include/linux/hsi/hsi.h
index 6ca92bff02c68..ea6bef9b6012e 100644
--- a/include/linux/hsi/hsi.h
+++ b/include/linux/hsi/hsi.h
@@ -271,7 +271,7 @@ struct hsi_controller {
 	struct module		*owner;
 	unsigned int		id;
 	unsigned int		num_ports;
-	struct hsi_port		**port;
+	struct hsi_port		*port[] __counted_by(num_ports);
 };
 
 #define to_hsi_controller(dev) container_of(dev, struct hsi_controller, device)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index a4d9f964dfdea..bd7f0e1d80945 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -771,6 +771,11 @@ static inline bool pmd_is_huge(pmd_t pmd)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
+static inline bool is_pmd_order(unsigned int order)
+{
+	return order == HPAGE_PMD_ORDER;
+}
+
 static inline int split_folio_to_list_to_order(struct folio *folio,
 		struct list_head *list, int new_order)
 {
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 65910437be1ca..9c098a02a09e1 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -777,10 +777,6 @@ static inline unsigned long huge_page_size(const struct hstate *h)
 	return (unsigned long)PAGE_SIZE << h->order;
 }
 
-extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
-
-extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
-
 static inline unsigned long huge_page_mask(struct hstate *h)
 {
 	return h->mask;
@@ -796,6 +792,23 @@ static inline unsigned huge_page_shift(struct hstate *h)
 	return h->order + PAGE_SHIFT;
 }
 
+/**
+ * hugetlb_linear_page_index() - linear_page_index() but in hugetlb
+ *				 page size granularity.
+ * @vma: the hugetlb VMA
+ * @address: the virtual address within the VMA
+ *
+ * Return: the page offset within the mapping in huge page units.
+ */
+static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma,
+		unsigned long address)
+{
+	struct hstate *h = hstate_vma(vma);
+
+	return ((address - vma->vm_start) >> huge_page_shift(h)) +
+		(vma->vm_pgoff >> huge_page_order(h));
+}
+
 static inline bool order_is_gigantic(unsigned int order)
 {
 	return order > MAX_PAGE_ORDER;
@@ -1177,16 +1190,6 @@ static inline unsigned long huge_page_mask(struct hstate *h)
 	return PAGE_MASK;
 }
 
-static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
-{
-	return PAGE_SIZE;
-}
-
-static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
-{
-	return PAGE_SIZE;
-}
-
 static inline unsigned int huge_page_order(struct hstate *h)
 {
 	return 0;
diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h
index 593f5d4e108b2..565b473fd1353 100644
--- a/include/linux/hugetlb_inline.h
+++ b/include/linux/hugetlb_inline.h
@@ -13,7 +13,7 @@ static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
 
 static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
 {
-	return vma_flags_test(flags, VMA_HUGETLB_BIT);
+	return vma_flags_test_any(flags, VMA_HUGETLB_BIT);
 }
 
 #else
@@ -30,7 +30,7 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
 
 #endif
 
-static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
+static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma)
 {
 	return is_vm_hugetlb_flags(vma->vm_flags);
 }
diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h
index dfc516c1c7193..3a721b1853a46 100644
--- a/include/linux/hyperv.h
+++ b/include/linux/hyperv.h
@@ -1015,8 +1015,8 @@ struct vmbus_channel {
 	/* The max size of a packet on this channel */
 	u32 max_pkt_size;
 
-	/* function to mmap ring buffer memory to the channel's sysfs ring attribute */
-	int (*mmap_ring_buffer)(struct vmbus_channel *channel, struct vm_area_struct *vma);
+	/* function to mmap_prepare ring buffer memory to the channel's sysfs ring attribute */
+	int (*mmap_prepare_ring_buffer)(struct vmbus_channel *channel, struct vm_area_desc *desc);
 
 	/* boolean to control visibility of sysfs for ring buffer */
 	bool ring_sysfs_visible;
diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h
index f8e9f5d36d2a2..a97b1d01f3acf 100644
--- a/include/linux/ieee80211-eht.h
+++ b/include/linux/ieee80211-eht.h
@@ -251,8 +251,8 @@ struct ieee80211_eht_operation_info {
 #define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF		0x40
 #define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK	0x07
 
-#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ			0x08
-#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ		0x30
+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ			0x10
+#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ		0x20
 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ		0x40
 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK			0x78
 #define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP		0x80
diff --git a/include/linux/ieee80211-ht.h b/include/linux/ieee80211-ht.h
index 21bbf470540f6..7612b72f9c7cb 100644
--- a/include/linux/ieee80211-ht.h
+++ b/include/linux/ieee80211-ht.h
@@ -281,6 +281,9 @@ enum ieee80211_back_actioncode {
 	WLAN_ACTION_ADDBA_REQ = 0,
 	WLAN_ACTION_ADDBA_RESP = 1,
 	WLAN_ACTION_DELBA = 2,
+	WLAN_ACTION_NDP_ADDBA_REQ = 128,
+	WLAN_ACTION_NDP_ADDBA_RESP = 129,
+	WLAN_ACTION_NDP_DELBA = 130,
 };
 
 /* BACK (block-ack) parties */
diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h
index 132acced7d798..d199f3ebdba0a 100644
--- a/include/linux/ieee80211-uhr.h
+++ b/include/linux/ieee80211-uhr.h
@@ -12,8 +12,8 @@
 
 #define IEEE80211_UHR_OPER_PARAMS_DPS_ENA		0x0001
 #define IEEE80211_UHR_OPER_PARAMS_NPCA_ENA		0x0002
-#define IEEE80211_UHR_OPER_PARAMS_DBE_ENA		0x0004
-#define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA		0x0008
+#define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA		0x0004
+#define IEEE80211_UHR_OPER_PARAMS_DBE_ENA		0x0008
 
 struct ieee80211_uhr_operation {
 	__le16 params;
@@ -29,11 +29,216 @@ struct ieee80211_uhr_operation {
 #define IEEE80211_UHR_NPCA_PARAMS_MOPLEN		0x00400000
 #define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES	0x00800000
 
+/**
+ * struct ieee80211_uhr_npca_info - npca operation information
+ *
+ * This structure is the "NPCA Operation Parameters field format" of "UHR
+ * Operation Element" fields as described in P802.11bn_D1.3
+ * subclause 9.4.2.353. See Figure 9-aa4.
+ *
+ * Refer to IEEE80211_UHR_NPCA*
+ * @params:
+ *	NPCA Primary Channel - NPCA primary channel
+ *	NPCA_Min Duration Threshold - Minimum duration of inter-BSS activity
+ *	NPCA Switching Delay -
+ *		Time needed by an NPCA AP to switch from the
+ *		BSS primary channel to the NPCA primary channel
+ *		in the unit of 4 µs.
+ *	NPCA Switching Back Delay -
+ *		Time to switch from the NPCA primary channel
+ *		to the BSS primary channel in the unit of 4 µs.
+ *	NPCA Initial QSRC -
+ *		Initialize the EDCAF QSRC[AC] variables
+ *		when an NPCA STA in the BSS
+ *		switches to NPCA operation.
+ *	NPCA MOPLEN -
+ *		Indicates which conditions can be used to
+ *		initiate an NPCA operation,
+ *		1 -> both PHYLEN NPCA operation and MOPLEN
+ *		NPCA operation are
+ *		permitted in the BSS
+ *		0 -> only PHYLEN NPCA operation is allowed in the BSS.
+ *	NPCA Disabled Subchannel Bitmap Present -
+ *		Indicates whether the NPCA Disabled Subchannel
+ *		Bitmap field is present. A 1 in this field indicates that
+ *		the NPCA Disabled Subchannel Bitmap field is present
+ * @dis_subch_bmap:
+ *		A bit in the bitmap that lies within the BSS bandwidth is set
+ *		to 1 to indicate that the corresponding 20 MHz subchannel is
+ *		punctured and is set to 0 to indicate that the corresponding
+ *		20 MHz subchannel is not punctured. A bit in the bitmap that
+ *		falls outside of the BSS bandwidth is reserved. This field is
+ *		present when the value of the NPCA Disabled Subchannel Bitmap
+ *		Field Present field is equal to 1, and not present, otherwise
+ */
 struct ieee80211_uhr_npca_info {
 	__le32 params;
 	__le16 dis_subch_bmap[];
 } __packed;
 
+#define IEEE80211_UHR_DPS_PADDING_DELAY			0x0000003F
+#define IEEE80211_UHR_DPS_TRANSITION_DELAY		0x00003F00
+#define IEEE80211_UHR_DPS_ICF_REQUIRED			0x00010000
+#define IEEE80211_UHR_DPS_PARAMETERIZED_FLAG		0x00020000
+#define IEEE80211_UHR_DPS_LC_MODE_BW			0x001C0000
+#define IEEE80211_UHR_DPS_LC_MODE_NSS			0x01E00000
+#define IEEE80211_UHR_DPS_LC_MODE_MCS			0x1E000000
+#define IEEE80211_UHR_DPS_MOBILE_AP_DPS_STATIC_HCM	0x20000000
+
+/**
+ * struct ieee80211_uhr_dps_info - DPS operation information
+ *
+ * This structure is the "DPS Operation Parameter field" of "UHR
+ * Operation Element" fields as described in P802.11bn_D1.3
+ * subclause 9.4.1.87. See Figure 9-207u.
+ *
+ * Refer to IEEE80211_UHR_DPS*
+ * @params:
+ *	DPS Padding Delay -
+ *		Indicates the minimum MAC padding
+ *		duration that is required by a DPS STA
+ *		in an ICF to cause the STA to transition
+ *		from the lower capability mode to the
+ *		higher capability mode. The DPS Padding
+ *		Delay field is in units of 4 µs.
+ *	DPS Transition Delay -
+ *		Indicates the amount of time required by a
+ *		DPS STA to transition from the higher
+ *		capability mode to the lower capability
+ *		mode. The DPS Transition Delay field is in
+ *		units of 4 µs.
+ *	ICF Required -
+ *		Indicates when the DPS assisting STA needs
+ *		to transmit an ICF frame to the peer DPS STA
+ *		before performing the frame exchanges with
+ *		the peer DPS STA in a TXOP.
+ *			1 -> indicates that the transmission of the
+ *			ICF frame to the peer DPS STA prior to
+ *			any frame exchange is needed.
+ *			0 -> ICF transmission before the frame
+ *			exchanges with the peer DPS STA is only
+ *			needed if the frame exchange is performed
+ *			in the HC mode.
+ *	Parameterized Flag -
+ *		0 -> indicates that only 20 MHz, 1 SS,
+ *		non-HT PPDU format with the data
+ *		rate of 6, 12, and 24 Mb/s as the
+ *		default mode are supported by the
+ *		DPS STA in the LC mode
+ *		1 -> indicates that a bandwidth up to the
+ *		bandwidth indicated in the LC Mode
+ *		Bandwidth field, a number of spatial
+ *		streams up to the NSS indicated in
+ *		the LC Mode Nss field, and an MCS up
+ *		to the MCS indicated in the LC Mode
+ *		MCS fields are supported by the DPS
+ *		STA in the LC mode as the
+ *		parameterized mode.
+ *	LC Mode Bandwidth -
+ *		Indicates the maximum bandwidth supported
+ *		by the STA in the LC mode.
+ *	LC Mode NSS -
+ *		Indicates the maximum number of the spatial
+ *		streams supported by the STA in the LC mode.
+ *	LC Mode MCS -
+ *		Indicates the highest MCS supported by the STA
+ *		in the LC mode.
+ *	Mobile AP DPS Static HCM -
+ *		1 -> indicates that it will remain in the DPS high
+ *		capability mode until the next TBTT on that
+ *		link.
+ *		0 -> otherwise.
+ */
+struct ieee80211_uhr_dps_info {
+	__le32 params;
+} __packed;
+
+#define IEEE80211_UHR_DBE_OPER_BANDWIDTH			0x07
+#define IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES	0x08
+
+/**
+ * enum ieee80211_uhr_dbe_oper_bw - DBE Operational Bandwidth
+ *
+ * Encoding for the DBE Operational Bandwidth field in the UHR Operation
+ * element (DBE Operation Parameters).
+ *
+ * @IEEE80211_UHR_DBE_OPER_BW_40: 40 MHz operational DBE bandwidth
+ * @IEEE80211_UHR_DBE_OPER_BW_80: 80 MHz operational DBE bandwidth
+ * @IEEE80211_UHR_DBE_OPER_BW_160: 160 MHz operational DBE bandwidth
+ * @IEEE80211_UHR_DBE_OPER_BW_320_1: 320-1 MHz operational DBE bandwidth
+ * @IEEE80211_UHR_DBE_OPER_BW_320_2: 320-2 MHz operational DBE bandwidth
+ */
+enum ieee80211_uhr_dbe_oper_bw {
+	IEEE80211_UHR_DBE_OPER_BW_40 = 1,
+	IEEE80211_UHR_DBE_OPER_BW_80 = 2,
+	IEEE80211_UHR_DBE_OPER_BW_160 = 3,
+	IEEE80211_UHR_DBE_OPER_BW_320_1 = 4,
+	IEEE80211_UHR_DBE_OPER_BW_320_2 = 5,
+};
+
+/**
+ * struct ieee80211_uhr_dbe_info - DBE operation information
+ *
+ * This structure is the "DBE Operation Parameters field" of
+ * "UHR Operation Element" fields as described in P802.11bn_D1.3
+ * subclause 9.4.2.353. See Figure 9-aa6.
+ *
+ * Refer to IEEE80211_UHR_DBE_OPER*
+ * @params:
+ *	B0-B2 - DBE Operational Bandwidth field, see
+ *	"enum ieee80211_uhr_dbe_oper_bw" for values.
+ *	Value 0 is reserved.
+ *	Value 1 indicates 40 MHz operational DBE bandwidth.
+ *	Value 2 indicates 80 MHz operational DBE bandwidth.
+ *	Value 3 indicates 160 MHz operational DBE bandwidth.
+ *	Value 4 indicates 320-1 MHz operational DBE bandwidth.
+ *	Value 5 indicates 320-2 MHz operational DBE bandwidth.
+ *	Values 6 to 7 are reserved.
+ *	B3 - DBE Disabled Subchannel Bitmap Present.
+ * @dis_subch_bmap: DBE Disabled Subchannel Bitmap field is set to indicate
+ *	disabled 20 MHz subchannels within the DBE Bandwidth.
+ */
+struct ieee80211_uhr_dbe_info {
+	u8 params;
+	__le16 dis_subch_bmap[];
+} __packed;
+
+#define IEEE80211_UHR_P_EDCA_ECWMIN		0x0F
+#define IEEE80211_UHR_P_EDCA_ECWMAX		0xF0
+#define IEEE80211_UHR_P_EDCA_AIFSN		0x000F
+#define IEEE80211_UHR_P_EDCA_CW_DS		0x0030
+#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD	0x01C0
+#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD	0x0600
+
+/**
+ * struct ieee80211_uhr_p_edca_info - P-EDCA operation information
+ *
+ * This structure is the "P-EDCA Operation Parameters field" of
+ * "UHR Operation Element" fields as described in P802.11bn_D1.3
+ * subclause 9.4.2.353. See Figure 9-aa5.
+ *
+ * Refer to IEEE80211_UHR_P_EDCA*
+ * @p_edca_ec: P-EDCA ECWmin and ECWmax.
+ *	These fields indicate the CWmin and CWmax values used by a
+ *	P-EDCA STA during P-EDCA contention.
+ * @params: AIFSN, CW DS, PSRC threshold, and QSRC threshold.
+ *	- The AIFSN field indicates the AIFSN value used by a P-EDCA STA
+ *	  during P-EDCA contention.
+ *	- The CW DS field indicates the value used for randomization of the
+ *	  transmission slot of the DS-CTS frame. The value 3 is reserved.
+ *	  The value 0 indicates that randomization is not enabled.
+ *	- The P-EDCA PSRC threshold field indicates the maximum number of
+ *	  allowed consecutive DS-CTS transmissions. The value 0 and values
+ *	  greater than 4 are reserved.
+ *	- The P-EDCA QSRC threshold field indicates the value of the
+ *	  QSRC[AC_VO] counter required to start P-EDCA contention. The
+ *	  value 0 is reserved.
+ */
+struct ieee80211_uhr_p_edca_info {
+	u8 p_edca_ec;
+	__le16 params;
+} __packed;
+
 static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len,
 					      bool beacon)
 {
@@ -47,19 +252,52 @@ static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len,
 	if (beacon)
 		return true;
 
-	/* FIXME: DPS, DBE, P-EDCA (consider order, also relative to NPCA) */
+	/* DPS Operation Parameters (fixed 4 bytes) */
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_ENA)) {
+		needed += sizeof(struct ieee80211_uhr_dps_info);
+		if (len < needed)
+			return false;
+	}
 
+	/* NPCA Operation Parameters (fixed 4 bytes + optional 2 bytes) */
 	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA)) {
 		const struct ieee80211_uhr_npca_info *npca =
-			(const void *)oper->variable;
+			(const void *)(data + needed);
 
 		needed += sizeof(*npca);
-
 		if (len < needed)
 			return false;
 
-		if (npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES))
+		if (npca->params &
+		    cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES)) {
 			needed += sizeof(npca->dis_subch_bmap[0]);
+			if (len < needed)
+				return false;
+		}
+	}
+
+	/* P-EDCA Operation Parameters (fixed 3 bytes) */
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA)) {
+		needed += sizeof(struct ieee80211_uhr_p_edca_info);
+		if (len < needed)
+			return false;
+	}
+
+	/* DBE Operation Parameters (fixed 1 byte + optional 2 bytes) */
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_ENA)) {
+		const struct ieee80211_uhr_dbe_info *dbe =
+			(const void *)(data + needed);
+
+		needed += sizeof(*dbe);
+		if (len < needed)
+			return false;
+
+		if (dbe->params &
+		    IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES) {
+			needed += sizeof(dbe->dis_subch_bmap[0]);
+			if (len < needed)
+				return false;
+		}
 	}
 
 	return len >= needed;
@@ -72,12 +310,15 @@ static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len,
 static inline const struct ieee80211_uhr_npca_info *
 ieee80211_uhr_npca_info(const struct ieee80211_uhr_operation *oper)
 {
+	const u8 *pos = oper->variable;
+
 	if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA)))
 		return NULL;
 
-	/* FIXME: DPS */
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_ENA))
+		pos += sizeof(struct ieee80211_uhr_dps_info);
 
-	return (const void *)oper->variable;
+	return (const void *)pos;
 }
 
 static inline const __le16 *
@@ -131,6 +372,24 @@ ieee80211_uhr_npca_dis_subch_bitmap(const struct ieee80211_uhr_operation *oper)
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES	0x08
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES	0x10
 
+/**
+ * enum ieee80211_uhr_dbe_max_supported_bw - DBE Maximum Supported Bandwidth
+ *
+ * As per spec P802.11bn_D1.3 "Table 9-bb5—Encoding of the DBE Maximum
+ * Supported Bandwidth field".
+ *
+ * @IEEE80211_UHR_DBE_MAX_BW_40: Indicates 40 MHz DBE max supported bw
+ * @IEEE80211_UHR_DBE_MAX_BW_80: Indicates 80 MHz DBE max supported bw
+ * @IEEE80211_UHR_DBE_MAX_BW_160: Indicates 160 MHz DBE max supported bw
+ * @IEEE80211_UHR_DBE_MAX_BW_320: Indicates 320 MHz DBE max supported bw
+ */
+enum ieee80211_uhr_dbe_max_supported_bw {
+	IEEE80211_UHR_DBE_MAX_BW_40 = 1,
+	IEEE80211_UHR_DBE_MAX_BW_80 = 2,
+	IEEE80211_UHR_DBE_MAX_BW_160 = 3,
+	IEEE80211_UHR_DBE_MAX_BW_320 = 4,
+};
+
 struct ieee80211_uhr_cap_mac {
 	u8 mac_cap[5];
 } __packed;
diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
index 0aa2fb8f88de4..52db36120314e 100644
--- a/include/linux/ieee80211.h
+++ b/include/linux/ieee80211.h
@@ -1046,31 +1046,28 @@ struct ieee80211_mgmt {
 		} __packed probe_resp;
 		struct {
 			u8 category;
+			u8 action_code;
 			union {
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 status_code;
 					u8 variable[];
 				} __packed wme_action;
 				struct{
-					u8 action_code;
+					u8 no_fixed_fields[0];
 					u8 variable[];
 				} __packed chan_switch;
 				struct{
-					u8 action_code;
 					struct ieee80211_ext_chansw_ie data;
 					u8 variable[];
 				} __packed ext_chan_switch;
 				struct{
-					u8 action_code;
 					u8 dialog_token;
 					u8 element_id;
 					u8 length;
 					struct ieee80211_msrment_ie msr_elem;
 				} __packed measurement;
 				struct{
-					u8 action_code;
 					u8 dialog_token;
 					__le16 capab;
 					__le16 timeout;
@@ -1079,7 +1076,6 @@ struct ieee80211_mgmt {
 					u8 variable[];
 				} __packed addba_req;
 				struct{
-					u8 action_code;
 					u8 dialog_token;
 					__le16 status;
 					__le16 capab;
@@ -1088,54 +1084,45 @@ struct ieee80211_mgmt {
 					u8 variable[];
 				} __packed addba_resp;
 				struct{
-					u8 action_code;
 					__le16 params;
 					__le16 reason_code;
 				} __packed delba;
 				struct {
-					u8 action_code;
+					u8 no_fixed_fields[0];
 					u8 variable[];
 				} __packed self_prot;
 				struct{
-					u8 action_code;
+					u8 no_fixed_fields[0];
 					u8 variable[];
 				} __packed mesh_action;
 				struct {
-					u8 action;
 					u8 trans_id[WLAN_SA_QUERY_TR_ID_LEN];
 				} __packed sa_query;
 				struct {
-					u8 action;
 					u8 smps_control;
 				} __packed ht_smps;
 				struct {
-					u8 action_code;
 					u8 chanwidth;
 				} __packed ht_notify_cw;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					__le16 capability;
 					u8 variable[];
 				} __packed tdls_discover_resp;
 				struct {
-					u8 action_code;
 					u8 operating_mode;
 				} __packed vht_opmode_notif;
 				struct {
-					u8 action_code;
 					u8 membership[WLAN_MEMBERSHIP_LEN];
 					u8 position[WLAN_USER_POSITION_LEN];
 				} __packed vht_group_notif;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 tpc_elem_id;
 					u8 tpc_elem_length;
 					struct ieee80211_tpc_report_ie tpc;
 				} __packed tpc_report;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 follow_up;
 					u8 tod[6];
@@ -1145,11 +1132,10 @@ struct ieee80211_mgmt {
 					u8 variable[];
 				} __packed ftm;
 				struct {
-					u8 action_code;
+					u8 no_fixed_fields[0];
 					u8 variable[];
 				} __packed s1g;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 follow_up;
 					u32 tod;
@@ -1158,41 +1144,37 @@ struct ieee80211_mgmt {
 					u8 max_toa_error;
 				} __packed wnm_timing_msr;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 variable[];
 				} __packed ttlm_req;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					__le16 status_code;
 					u8 variable[];
 				} __packed ttlm_res;
 				struct {
-					u8 action_code;
+					u8 no_fixed_fields[0];
+					/* no variable fields either */
 				} __packed ttlm_tear_down;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 variable[];
 				} __packed ml_reconf_req;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 count;
 					u8 variable[];
 				} __packed ml_reconf_resp;
 				struct {
-					u8 action_code;
+					u8 no_fixed_fields[0];
 					u8 variable[];
 				} __packed epcs;
 				struct {
-					u8 action_code;
 					u8 dialog_token;
 					u8 control;
 					u8 variable[];
 				} __packed eml_omn;
-			} u;
+			};
 		} __packed action;
 		DECLARE_FLEX_ARRAY(u8, body); /* Generic frame body */
 	} u;
@@ -1210,8 +1192,7 @@ struct ieee80211_mgmt {
 
 #define BSS_MEMBERSHIP_SELECTOR_MIN	BSS_MEMBERSHIP_SELECTOR_UHR_PHY
 
-/* mgmt header + 1 byte category code */
-#define IEEE80211_MIN_ACTION_SIZE offsetof(struct ieee80211_mgmt, u.action.u)
+#define IEEE80211_MIN_ACTION_SIZE(type)	offsetofend(struct ieee80211_mgmt, u.action.type)
 
 
 /* Management MIC information element (IEEE 802.11w) for CMAC */
@@ -1358,6 +1339,7 @@ struct ieee80211_tdls_data {
 #define WLAN_AUTH_FILS_SK 4
 #define WLAN_AUTH_FILS_SK_PFS 5
 #define WLAN_AUTH_FILS_PK 6
+#define WLAN_AUTH_IEEE8021X 8
 #define WLAN_AUTH_EPPKE 9
 #define WLAN_AUTH_LEAP 128
 
@@ -1500,6 +1482,8 @@ enum ieee80211_statuscode {
 	WLAN_STATUS_REJECT_DSE_BAND = 96,
 	WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL = 99,
 	WLAN_STATUS_DENIED_DUE_TO_SPECTRUM_MANAGEMENT = 103,
+	/* 802.11ah */
+	WLAN_STATUS_REJECTED_NDP_BLOCK_ACK_SUGGESTED = 109,
 	/* 802.11ai */
 	WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 112,
 	WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 113,
@@ -1507,6 +1491,7 @@ enum ieee80211_statuscode {
 	WLAN_STATUS_SAE_PK = 127,
 	WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133,
 	WLAN_STATUS_PREF_TID_TO_LINK_MAPPING_SUGGESTED = 134,
+	WLAN_STATUS_8021X_AUTH_SUCCESS = 153,
 };
 
 
@@ -2389,7 +2374,7 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb)
 	if (!ieee80211_is_action(fc))
 		return false;
 
-	if (skb->len < offsetofend(typeof(*mgmt), u.action.u.ftm.action_code))
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(action_code))
 		return true;
 
 	/* action frame - additionally check for non-bufferable FTM */
@@ -2398,8 +2383,8 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb)
 	    mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
 		return true;
 
-	if (mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_REQUEST ||
-	    mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_RESPONSE)
+	if (mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_REQUEST ||
+	    mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE)
 		return false;
 
 	return true;
@@ -2449,7 +2434,7 @@ static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr)
  */
 static inline bool ieee80211_is_robust_mgmt_frame(struct sk_buff *skb)
 {
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE)
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
 		return false;
 	return _ieee80211_is_robust_mgmt_frame((void *)skb->data);
 }
@@ -2465,7 +2450,7 @@ static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr,
 {
 	struct ieee80211_mgmt *mgmt = (void *)hdr;
 
-	if (len < IEEE80211_MIN_ACTION_SIZE)
+	if (len < IEEE80211_MIN_ACTION_SIZE(category))
 		return false;
 	if (!ieee80211_is_action(hdr->frame_control))
 		return false;
@@ -2483,13 +2468,14 @@ static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr,
 static inline bool
 ieee80211_is_protected_dual_of_public_action(struct sk_buff *skb)
 {
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
 	u8 action;
 
 	if (!ieee80211_is_public_action((void *)skb->data, skb->len) ||
-	    skb->len < IEEE80211_MIN_ACTION_SIZE + 1)
+	    skb->len < IEEE80211_MIN_ACTION_SIZE(action_code))
 		return false;
 
-	action = *(u8 *)(skb->data + IEEE80211_MIN_ACTION_SIZE);
+	action = mgmt->u.action.action_code;
 
 	return action != WLAN_PUB_ACTION_20_40_BSS_COEX &&
 		action != WLAN_PUB_ACTION_DSE_REG_LOC_ANN &&
@@ -2528,7 +2514,7 @@ static inline bool _ieee80211_is_group_privacy_action(struct ieee80211_hdr *hdr)
  */
 static inline bool ieee80211_is_group_privacy_action(struct sk_buff *skb)
 {
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE)
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
 		return false;
 	return _ieee80211_is_group_privacy_action((void *)skb->data);
 }
@@ -2624,8 +2610,7 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb)
 	if (!ieee80211_is_action(mgmt->frame_control))
 		return false;
 
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE +
-		       sizeof(mgmt->u.action.u.tpc_report))
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(tpc_report))
 		return false;
 
 	/*
@@ -2644,12 +2629,11 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb)
 		return false;
 
 	/* both spectrum mgmt and link measurement have same action code */
-	if (mgmt->u.action.u.tpc_report.action_code !=
-	    WLAN_ACTION_SPCT_TPC_RPRT)
+	if (mgmt->u.action.action_code != WLAN_ACTION_SPCT_TPC_RPRT)
 		return false;
 
-	if (mgmt->u.action.u.tpc_report.tpc_elem_id != WLAN_EID_TPC_REPORT ||
-	    mgmt->u.action.u.tpc_report.tpc_elem_length !=
+	if (mgmt->u.action.tpc_report.tpc_elem_id != WLAN_EID_TPC_REPORT ||
+	    mgmt->u.action.tpc_report.tpc_elem_length !=
 	    sizeof(struct ieee80211_tpc_report_ie))
 		return false;
 
@@ -2665,16 +2649,15 @@ static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb)
 {
 	struct ieee80211_mgmt *mgmt = (void *)skb->data;
 
-	if (skb->len < IEEE80211_MIN_ACTION_SIZE)
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(wnm_timing_msr))
 		return false;
 
 	if (!ieee80211_is_action(mgmt->frame_control))
 		return false;
 
 	if (mgmt->u.action.category == WLAN_CATEGORY_WNM_UNPROTECTED &&
-	    mgmt->u.action.u.wnm_timing_msr.action_code ==
-		WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE &&
-	    skb->len >= offsetofend(typeof(*mgmt), u.action.u.wnm_timing_msr))
+	    mgmt->u.action.action_code ==
+			WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE)
 		return true;
 
 	return false;
@@ -2689,15 +2672,13 @@ static inline bool ieee80211_is_ftm(struct sk_buff *skb)
 {
 	struct ieee80211_mgmt *mgmt = (void *)skb->data;
 
-	if (!ieee80211_is_public_action((void *)mgmt, skb->len))
+	if (skb->len < IEEE80211_MIN_ACTION_SIZE(ftm))
 		return false;
 
-	if (mgmt->u.action.u.ftm.action_code ==
-		WLAN_PUB_ACTION_FTM_RESPONSE &&
-	    skb->len >= offsetofend(typeof(*mgmt), u.action.u.ftm))
-		return true;
+	if (!ieee80211_is_public_action((void *)mgmt, skb->len))
+		return false;
 
-	return false;
+	return mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE;
 }
 
 struct element {
diff --git a/include/linux/if_ether.h b/include/linux/if_ether.h
index 61b7335aa037c..ca9afa824aa4f 100644
--- a/include/linux/if_ether.h
+++ b/include/linux/if_ether.h
@@ -40,7 +40,8 @@ static inline struct ethhdr *inner_eth_hdr(const struct sk_buff *skb)
 	return (struct ethhdr *)skb_inner_mac_header(skb);
 }
 
-int eth_header_parse(const struct sk_buff *skb, unsigned char *haddr);
+int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev,
+		     unsigned char *haddr);
 
 extern ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len);
 
diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h
index db45d6f1c4f49..8bbf676c2a851 100644
--- a/include/linux/if_pppox.h
+++ b/include/linux/if_pppox.h
@@ -25,8 +25,6 @@ struct pppoe_opt {
 	struct net_device      *dev;	  /* device associated with socket*/
 	int			ifindex;  /* ifindex of device associated with socket */
 	struct pppoe_addr	pa;	  /* what this socket is bound to*/
-	struct sockaddr_pppox	relay;	  /* what socket data will be
-					     relayed to (PPPoE relaying) */
 	struct work_struct      padt_work;/* Work item for handling PADT */
 };
 
@@ -53,7 +51,6 @@ struct pppox_sock {
 #define pppoe_dev	proto.pppoe.dev
 #define pppoe_ifindex	proto.pppoe.ifindex
 #define pppoe_pa	proto.pppoe.pa
-#define pppoe_relay	proto.pppoe.relay
 
 static inline struct pppox_sock *pppox_sk(struct sock *sk)
 {
@@ -80,14 +77,11 @@ extern void pppox_unbind_sock(struct sock *sk);/* delete ppp-channel binding */
 extern int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);
 extern int pppox_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);
 
-#define PPPOEIOCSFWD32    _IOW(0xB1 ,0, compat_size_t)
-
 /* PPPoX socket states */
 enum {
     PPPOX_NONE		= 0,  /* initial state */
     PPPOX_CONNECTED	= 1,  /* connection established ==TCP_ESTABLISHED */
     PPPOX_BOUND		= 2,  /* bound to ppp device */
-    PPPOX_RELAY		= 4,  /* forwarding is enabled */
     PPPOX_DEAD		= 16  /* dead, useless, please clean me up!*/
 };
 
diff --git a/include/linux/iio/adc/qcom-adc5-gen3-common.h b/include/linux/iio/adc/qcom-adc5-gen3-common.h
new file mode 100644
index 0000000000000..6303eaa6640be
--- /dev/null
+++ b/include/linux/iio/adc/qcom-adc5-gen3-common.h
@@ -0,0 +1,211 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * Code used in the main and auxiliary Qualcomm PMIC voltage ADCs
+ * of type ADC5 Gen3.
+ */
+
+#ifndef QCOM_ADC5_GEN3_COMMON_H
+#define QCOM_ADC5_GEN3_COMMON_H
+
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/iio/adc/qcom-vadc-common.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#define ADC5_GEN3_HS				0x45
+#define ADC5_GEN3_HS_BUSY			BIT(7)
+#define ADC5_GEN3_HS_READY			BIT(0)
+
+#define ADC5_GEN3_STATUS1			0x46
+#define ADC5_GEN3_STATUS1_CONV_FAULT		BIT(7)
+#define ADC5_GEN3_STATUS1_THR_CROSS		BIT(6)
+#define ADC5_GEN3_STATUS1_EOC			BIT(0)
+
+#define ADC5_GEN3_TM_EN_STS			0x47
+#define ADC5_GEN3_TM_HIGH_STS			0x48
+#define ADC5_GEN3_TM_LOW_STS			0x49
+
+#define ADC5_GEN3_EOC_STS			0x4a
+#define ADC5_GEN3_EOC_CHAN_0			BIT(0)
+
+#define ADC5_GEN3_EOC_CLR			0x4b
+#define ADC5_GEN3_TM_HIGH_STS_CLR		0x4c
+#define ADC5_GEN3_TM_LOW_STS_CLR		0x4d
+#define ADC5_GEN3_CONV_ERR_CLR			0x4e
+#define ADC5_GEN3_CONV_ERR_CLR_REQ		BIT(0)
+
+#define ADC5_GEN3_SID				0x4f
+#define ADC5_GEN3_SID_MASK			GENMASK(3, 0)
+
+#define ADC5_GEN3_PERPH_CH			0x50
+#define ADC5_GEN3_CHAN_CONV_REQ			BIT(7)
+
+#define ADC5_GEN3_TIMER_SEL			0x51
+#define ADC5_GEN3_TIME_IMMEDIATE		0x1
+
+#define ADC5_GEN3_DIG_PARAM			0x52
+#define ADC5_GEN3_DIG_PARAM_CAL_SEL_MASK	GENMASK(5, 4)
+#define ADC5_GEN3_DIG_PARAM_DEC_RATIO_SEL_MASK	GENMASK(3, 2)
+
+#define ADC5_GEN3_FAST_AVG			0x53
+#define ADC5_GEN3_FAST_AVG_CTL_EN		BIT(7)
+#define ADC5_GEN3_FAST_AVG_CTL_SAMPLES_MASK	GENMASK(2, 0)
+
+#define ADC5_GEN3_ADC_CH_SEL_CTL		0x54
+#define ADC5_GEN3_DELAY_CTL			0x55
+#define ADC5_GEN3_HW_SETTLE_DELAY_MASK		GENMASK(3, 0)
+
+#define ADC5_GEN3_CH_EN				0x56
+#define ADC5_GEN3_HIGH_THR_INT_EN		BIT(1)
+#define ADC5_GEN3_LOW_THR_INT_EN		BIT(0)
+
+#define ADC5_GEN3_LOW_THR0			0x57
+#define ADC5_GEN3_LOW_THR1			0x58
+#define ADC5_GEN3_HIGH_THR0			0x59
+#define ADC5_GEN3_HIGH_THR1			0x5a
+
+#define ADC5_GEN3_CH_DATA0(channel)	(0x5c + (channel) * 2)
+#define ADC5_GEN3_CH_DATA1(channel)	(0x5d + (channel) * 2)
+
+#define ADC5_GEN3_CONV_REQ			0xe5
+#define ADC5_GEN3_CONV_REQ_REQ			BIT(0)
+
+#define ADC5_GEN3_VIRTUAL_SID_MASK		GENMASK(15, 8)
+#define ADC5_GEN3_CHANNEL_MASK			GENMASK(7, 0)
+#define ADC5_GEN3_V_CHAN(x)		\
+	(FIELD_PREP(ADC5_GEN3_VIRTUAL_SID_MASK, (x).sid) | (x).channel)
+
+/* ADC channels for PMIC5 Gen3 */
+#define ADC5_GEN3_REF_GND			0x00
+#define ADC5_GEN3_1P25VREF			0x01
+#define ADC5_GEN3_DIE_TEMP			0x03
+#define ADC5_GEN3_USB_SNS_V_16			0x11
+#define ADC5_GEN3_VIN_DIV16_MUX			0x12
+#define ADC5_GEN3_VPH_PWR			0x8e
+#define ADC5_GEN3_VBAT_SNS_QBG			0x8f
+/* 100k pull-up channels */
+#define ADC5_GEN3_AMUX1_THM_100K_PU		0x44
+#define ADC5_GEN3_AMUX2_THM_100K_PU		0x45
+#define ADC5_GEN3_AMUX3_THM_100K_PU		0x46
+#define ADC5_GEN3_AMUX4_THM_100K_PU		0x47
+#define ADC5_GEN3_AMUX5_THM_100K_PU		0x48
+#define ADC5_GEN3_AMUX6_THM_100K_PU		0x49
+#define ADC5_GEN3_AMUX1_GPIO_100K_PU		0x4a
+#define ADC5_GEN3_AMUX2_GPIO_100K_PU		0x4b
+#define ADC5_GEN3_AMUX3_GPIO_100K_PU		0x4c
+#define ADC5_GEN3_AMUX4_GPIO_100K_PU		0x4d
+
+#define ADC5_MAX_CHANNEL			0xc0
+
+enum adc5_cal_method {
+	ADC5_NO_CAL = 0,
+	ADC5_RATIOMETRIC_CAL,
+	ADC5_ABSOLUTE_CAL,
+};
+
+enum adc5_time_select {
+	MEAS_INT_DISABLE = 0,
+	MEAS_INT_IMMEDIATE,
+	MEAS_INT_50MS,
+	MEAS_INT_100MS,
+	MEAS_INT_1S,
+	MEAS_INT_NONE,
+};
+
+/**
+ * struct adc5_sdam_data - data per SDAM allocated for adc usage
+ * @base_addr: base address for the ADC SDAM peripheral.
+ * @irq_name: ADC IRQ name.
+ * @irq: ADC IRQ number.
+ */
+struct adc5_sdam_data {
+	u16 base_addr;
+	const char *irq_name;
+	int irq;
+};
+
+/**
+ * struct adc5_device_data - Top-level ADC device data
+ * @regmap: ADC peripheral register map field.
+ * @base: array of SDAM data.
+ * @num_sdams: number of ADC SDAM peripherals.
+ */
+struct adc5_device_data {
+	struct regmap *regmap;
+	struct adc5_sdam_data *base;
+	int num_sdams;
+};
+
+/**
+ * struct adc5_channel_common_prop - ADC channel properties (common to ADC and TM).
+ * @channel: channel number, refer to the channel list.
+ * @cal_method: calibration method.
+ * @decimation: sampling rate supported for the channel.
+ * @sid: ID of PMIC owning the channel.
+ * @label: Channel name used in device tree.
+ * @prescale: channel scaling performed on the input signal.
+ * @hw_settle_time_us: the time between AMUX being configured and the
+ *	start of conversion in uS.
+ * @avg_samples: ability to provide single result from the ADC
+ *	that is an average of multiple measurements.
+ * @scale_fn_type: Represents the scaling function to convert voltage
+ *	physical units desired by the client for the channel.
+ */
+struct adc5_channel_common_prop {
+	unsigned int channel;
+	enum adc5_cal_method cal_method;
+	unsigned int decimation;
+	unsigned int sid;
+	const char *label;
+	unsigned int prescale;
+	unsigned int hw_settle_time_us;
+	unsigned int avg_samples;
+	enum vadc_scale_fn_type scale_fn_type;
+};
+
+/**
+ * struct tm5_aux_dev_wrapper - wrapper structure around TM auxiliary device
+ * @aux_dev: TM auxiliary device structure.
+ * @dev_data: Top-level ADC device data.
+ * @tm_props: Array of common ADC channel properties for TM channels.
+ * @n_tm_channels: number of TM channels.
+ */
+struct tm5_aux_dev_wrapper {
+	struct auxiliary_device aux_dev;
+	struct adc5_device_data *dev_data;
+	struct adc5_channel_common_prop *tm_props;
+	unsigned int n_tm_channels;
+};
+
+int adc5_gen3_read(struct adc5_device_data *adc, unsigned int sdam_index,
+		   u16 offset, u8 *data, int len);
+
+int adc5_gen3_write(struct adc5_device_data *adc, unsigned int sdam_index,
+		    u16 offset, u8 *data, int len);
+
+int adc5_gen3_poll_wait_hs(struct adc5_device_data *adc,
+			   unsigned int sdam_index);
+
+void adc5_gen3_update_dig_param(struct adc5_channel_common_prop *prop,
+				u8 *data);
+
+int adc5_gen3_status_clear(struct adc5_device_data *adc,
+			   int sdam_index, u16 offset, u8 *val, int len);
+
+void adc5_gen3_mutex_lock(struct device *dev);
+void adc5_gen3_mutex_unlock(struct device *dev);
+int adc5_gen3_get_scaled_reading(struct device *dev,
+				 struct adc5_channel_common_prop *common_props,
+				 int *val);
+int adc5_gen3_therm_code_to_temp(struct device *dev,
+				 struct adc5_channel_common_prop *common_props,
+				 u16 code, int *val);
+void adc5_gen3_register_tm_event_notifier(struct device *dev,
+					  void (*handler)(struct auxiliary_device *));
+
+#endif /* QCOM_ADC5_GEN3_COMMON_H */
diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h
index 7f815f3fed6ae..4d15c2a9802c1 100644
--- a/include/linux/iio/backend.h
+++ b/include/linux/iio/backend.h
@@ -84,6 +84,27 @@ enum iio_backend_filter_type {
 	IIO_BACKEND_FILTER_TYPE_MAX
 };
 
+/**
+ * enum iio_backend_capabilities - Backend capabilities
+ * Backend capabilities can be used by frontends to check if a given
+ * functionality is supported by the backend. This is useful for frontend
+ * devices which are expected to work with alternative backend
+ * implementations. Capabilities are loosely coupled with operations,
+ * meaning that a capability requires certain operations to be implemented
+ * by the backend. A capability might be mapped to a single operation or
+ * multiple operations.
+ *
+ * @IIO_BACKEND_CAP_CALIBRATION: Backend supports digital interface
+ * calibration. Calibration procedure is device specific.
+ * @IIO_BACKEND_CAP_BUFFER: Support for IIO buffer interface.
+ * @IIO_BACKEND_CAP_ENABLE: Backend can be explicitly enabled/disabled.
+ */
+enum iio_backend_capabilities {
+	IIO_BACKEND_CAP_CALIBRATION = BIT(0),
+	IIO_BACKEND_CAP_BUFFER = BIT(1),
+	IIO_BACKEND_CAP_ENABLE = BIT(2),
+};
+
 /**
  * struct iio_backend_ops - operations structure for an iio_backend
  * @enable: Enable backend.
@@ -179,10 +200,12 @@ struct iio_backend_ops {
  * struct iio_backend_info - info structure for an iio_backend
  * @name: Backend name.
  * @ops: Backend operations.
+ * @caps: Backend capabilities. (bitmask of enum iio_backend_capabilities).
  */
 struct iio_backend_info {
 	const char *name;
 	const struct iio_backend_ops *ops;
+	u32 caps;
 };
 
 int iio_backend_chan_enable(struct iio_backend *back, unsigned int chan);
@@ -235,6 +258,7 @@ int iio_backend_read_raw(struct iio_backend *back,
 			 long mask);
 int iio_backend_extend_chan_spec(struct iio_backend *back,
 				 struct iio_chan_spec *chan);
+bool iio_backend_has_caps(struct iio_backend *back, u32 caps);
 void *iio_backend_get_priv(const struct iio_backend *conv);
 struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name);
 struct iio_backend *devm_iio_backend_fwnode_get(struct device *dev,
diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h
index a9ecff191bd9d..2c91b7659ce93 100644
--- a/include/linux/iio/iio.h
+++ b/include/linux/iio/iio.h
@@ -931,6 +931,18 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev)
 #define IIO_DECLARE_DMA_BUFFER_WITH_TS(type, name, count) \
 	__IIO_DECLARE_BUFFER_WITH_TS(type, name, count) __aligned(IIO_DMA_MINALIGN)
 
+/**
+ * IIO_DECLARE_QUATERNION() - Declare a quaternion element
+ * @type: element type of the individual vectors
+ * @name: identifier name
+ *
+ * Quaternions are a vector composed of 4 elements (W, X, Y, Z). Use this macro
+ * to declare a quaternion element in a struct to ensure proper alignment in
+ * an IIO buffer.
+ */
+#define IIO_DECLARE_QUATERNION(type, name) \
+	type name[4] __aligned(sizeof(type) * 4)
+
 struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv);
 
 /* The information at the returned address is guaranteed to be cacheline aligned */
diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h
index 34eebad12d2c2..4e3099defc1d3 100644
--- a/include/linux/iio/types.h
+++ b/include/linux/iio/types.h
@@ -21,6 +21,7 @@ enum iio_event_info {
 	IIO_EV_INFO_TAP2_MIN_DELAY,
 	IIO_EV_INFO_RUNNING_PERIOD,
 	IIO_EV_INFO_RUNNING_COUNT,
+	IIO_EV_INFO_SCALE,
 };
 
 #define IIO_VAL_INT 1
diff --git a/include/linux/ima.h b/include/linux/ima.h
index abf8923f8fc51..8e08baf16c2fd 100644
--- a/include/linux/ima.h
+++ b/include/linux/ima.h
@@ -11,6 +11,7 @@
 #include <linux/fs.h>
 #include <linux/security.h>
 #include <linux/kexec.h>
+#include <linux/secure_boot.h>
 #include <crypto/hash_info.h>
 struct linux_binprm;
 
@@ -73,14 +74,8 @@ int ima_validate_range(phys_addr_t phys, size_t size);
 #endif
 
 #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT
-extern bool arch_ima_get_secureboot(void);
 extern const char * const *arch_get_ima_policy(void);
 #else
-static inline bool arch_ima_get_secureboot(void)
-{
-	return false;
-}
-
 static inline const char * const *arch_get_ima_policy(void)
 {
 	return NULL;
diff --git a/include/linux/input.h b/include/linux/input.h
index 7d7cb0593a63e..06ca62328db1e 100644
--- a/include/linux/input.h
+++ b/include/linux/input.h
@@ -517,6 +517,10 @@ INPUT_GENERATE_ABS_ACCESSORS(res, resolution)
 int input_scancode_to_scalar(const struct input_keymap_entry *ke,
 			     unsigned int *scancode);
 
+int input_default_setkeycode(struct input_dev *dev,
+			     const struct input_keymap_entry *ke,
+			     unsigned int *old_keycode);
+
 int input_get_keycode(struct input_dev *dev, struct input_keymap_entry *ke);
 int input_set_keycode(struct input_dev *dev,
 		      const struct input_keymap_entry *ke);
diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h
index fa1f328d67120..6d694099a3ad3 100644
--- a/include/linux/intel_rapl.h
+++ b/include/linux/intel_rapl.h
@@ -128,6 +128,16 @@ struct reg_action {
 	int err;
 };
 
+struct rapl_defaults {
+	u8 floor_freq_reg_addr;
+	int (*check_unit)(struct rapl_domain *rd);
+	void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
+	u64 (*compute_time_window)(struct rapl_domain *rd, u64 val, bool to_raw);
+	unsigned int dram_domain_energy_unit;
+	unsigned int psys_domain_energy_unit;
+	bool spr_psys_bits;
+};
+
 /**
  * struct rapl_if_priv: private data for different RAPL interfaces
  * @control_type:		Each RAPL interface must have its own powercap
@@ -142,7 +152,7 @@ struct reg_action {
  *				registers.
  * @write_raw:			Callback for writing RAPL interface specific
  *				registers.
- * @defaults:			internal pointer to interface default settings
+ * @defaults:			pointer to default settings
  * @rpi:			internal pointer to interface primitive info
  */
 struct rapl_if_priv {
@@ -154,7 +164,7 @@ struct rapl_if_priv {
 	int limits[RAPL_DOMAIN_MAX];
 	int (*read_raw)(int id, struct reg_action *ra, bool pmu_ctx);
 	int (*write_raw)(int id, struct reg_action *ra);
-	void *defaults;
+	const struct rapl_defaults *defaults;
 	void *rpi;
 };
 
@@ -211,6 +221,9 @@ void rapl_remove_package_cpuslocked(struct rapl_package *rp);
 struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu);
 struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu);
 void rapl_remove_package(struct rapl_package *rp);
+int rapl_default_check_unit(struct rapl_domain *rd);
+void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode);
+u64 rapl_default_compute_time_window(struct rapl_domain *rd, u64 value, bool to_raw);
 
 #ifdef CONFIG_PERF_EVENTS
 int rapl_package_add_pmu(struct rapl_package *rp);
diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h
index 1a0f357c24271..1fe5665a9d02a 100644
--- a/include/linux/intel_vsec.h
+++ b/include/linux/intel_vsec.h
@@ -29,9 +29,15 @@
 #define INTEL_DVSEC_TABLE_OFFSET(x)	((x) & GENMASK(31, 3))
 #define TABLE_OFFSET_SHIFT		3
 
+struct device;
 struct pci_dev;
 struct resource;
 
+enum intel_vsec_disc_source {
+	INTEL_VSEC_DISC_PCI,	/* PCI, default */
+	INTEL_VSEC_DISC_ACPI,	/* ACPI */
+};
+
 enum intel_vsec_id {
 	VSEC_ID_TELEMETRY	= 2,
 	VSEC_ID_WATCHER		= 3,
@@ -82,14 +88,14 @@ enum intel_vsec_quirks {
  * struct pmt_callbacks - Callback infrastructure for PMT devices
  * @read_telem: when specified, called by client driver to access PMT
  * data (instead of direct copy).
- * * pdev:  PCI device reference for the callback's use
+ * * dev:   device reference for the callback's use
  * * guid:  ID of data to acccss
  * * data:  buffer for the data to be copied
  * * off:   offset into the requested buffer
  * * count: size of buffer
  */
 struct pmt_callbacks {
-	int (*read_telem)(struct pci_dev *pdev, u32 guid, u64 *data, loff_t off, u32 count);
+	int (*read_telem)(struct device *dev, u32 guid, u64 *data, loff_t off, u32 count);
 };
 
 struct vsec_feature_dependency {
@@ -102,6 +108,10 @@ struct vsec_feature_dependency {
  * @parent:    parent device in the auxbus chain
  * @headers:   list of headers to define the PMT client devices to create
  * @deps:      array of feature dependencies
+ * @acpi_disc: ACPI discovery tables, each entry is two QWORDs
+ *             in little-endian format as defined by the PMT ACPI spec.
+ *             Valid only when @provider == INTEL_VSEC_DISC_ACPI.
+ * @src:       source of discovery table data
  * @priv_data: private data, usable by parent devices, currently a callback
  * @caps:      bitmask of PMT capabilities for the given headers
  * @quirks:    bitmask of VSEC device quirks
@@ -112,6 +122,8 @@ struct intel_vsec_platform_info {
 	struct device *parent;
 	struct intel_vsec_header **headers;
 	const struct vsec_feature_dependency *deps;
+	u32 (*acpi_disc)[4];
+	enum intel_vsec_disc_source src;
 	void *priv_data;
 	unsigned long caps;
 	unsigned long quirks;
@@ -122,8 +134,13 @@ struct intel_vsec_platform_info {
 /**
  * struct intel_vsec_device - Auxbus specific device information
  * @auxdev:        auxbus device struct for auxbus access
- * @pcidev:        pci device associated with the device
- * @resource:      any resources shared by the parent
+ * @dev:           struct device associated with the device
+ * @resource:      PCI discovery resources (BAR windows), one per discovery
+ *                 instance. Valid only when @src == INTEL_VSEC_DISC_PCI
+ * @acpi_disc:     ACPI discovery tables, each entry is two QWORDs
+ *                 in little-endian format as defined by the PMT ACPI spec.
+ *                 Valid only when @src == INTEL_VSEC_DISC_ACPI.
+ * @src:           source of discovery table data
  * @ida:           id reference
  * @num_resources: number of resources
  * @id:            xarray id
@@ -135,8 +152,10 @@ struct intel_vsec_platform_info {
  */
 struct intel_vsec_device {
 	struct auxiliary_device auxdev;
-	struct pci_dev *pcidev;
+	struct device *dev;
 	struct resource *resource;
+	u32 (*acpi_disc)[4];
+	enum intel_vsec_disc_source src;
 	struct ida *ida;
 	int num_resources;
 	int id; /* xa */
@@ -184,7 +203,7 @@ struct pmt_feature_group {
 	struct telemetry_region	regions[];
 };
 
-int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent,
+int intel_vsec_add_aux(struct device *parent,
 		       struct intel_vsec_device *intel_vsec_dev,
 		       const char *name);
 
@@ -199,14 +218,14 @@ static inline struct intel_vsec_device *auxdev_to_ivdev(struct auxiliary_device
 }
 
 #if IS_ENABLED(CONFIG_INTEL_VSEC)
-int intel_vsec_register(struct pci_dev *pdev,
-			 struct intel_vsec_platform_info *info);
+int intel_vsec_register(struct device *dev,
+			const struct intel_vsec_platform_info *info);
 int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info,
 			   struct intel_vsec_device *vsec_dev);
 struct oobmsm_plat_info *intel_vsec_get_mapping(struct pci_dev *pdev);
 #else
-static inline int intel_vsec_register(struct pci_dev *pdev,
-				       struct intel_vsec_platform_info *info)
+static inline int intel_vsec_register(struct device *dev,
+				      const struct intel_vsec_platform_info *info)
 {
 	return -ENODEV;
 }
diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 7a1516011ccf7..e19872e37e067 100644
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -53,7 +53,7 @@ struct iommu_flush_ops {
  *                 tables.
  * @ias:           Input address (iova) size, in bits.
  * @oas:           Output address (paddr) size, in bits.
- * @coherent_walk  A flag to indicate whether or not page table walks made
+ * @coherent_walk: A flag to indicate whether or not page table walks made
  *                 by the IOMMU are coherent with the CPU caches.
  * @tlb:           TLB management callbacks for this set of tables.
  * @iommu_dev:     The device representing the DMA configuration for the
@@ -136,6 +136,7 @@ struct io_pgtable_cfg {
 	void (*free)(void *cookie, void *pages, size_t size);
 
 	/* Low-level data specific to the table format */
+	/* private: */
 	union {
 		struct {
 			u64	ttbr;
@@ -203,6 +204,9 @@ struct arm_lpae_io_pgtable_walk_data {
  * @unmap_pages:  Unmap a range of virtually contiguous pages of the same size.
  * @iova_to_phys: Translate iova to physical address.
  * @pgtable_walk: (optional) Perform a page table walk for a given iova.
+ * @read_and_clear_dirty: Record dirty info per IOVA. If an IOVA is dirty,
+ *			  clear its dirty state from the PTE unless the
+ *			  IOMMU_DIRTY_NO_CLEAR flag is passed in.
  *
  * These functions map directly onto the iommu_ops member functions with
  * the same names.
@@ -231,7 +235,9 @@ struct io_pgtable_ops {
  *          the configuration actually provided by the allocator (e.g. the
  *          pgsize_bitmap may be restricted).
  * @cookie: An opaque token provided by the IOMMU driver and passed back to
- *          the callback routines in cfg->tlb.
+ *          the callback routines.
+ *
+ * Returns: Pointer to the &struct io_pgtable_ops for this set of page tables.
  */
 struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
 					    struct io_pgtable_cfg *cfg,
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index dd1420bfcb735..244392026c6d5 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -8,6 +8,9 @@
 #include <linux/llist.h>
 #include <uapi/linux/io_uring.h>
 
+struct iou_loop_params;
+struct io_uring_bpf_ops;
+
 enum {
 	/*
 	 * A hint to not wake right away but delay until there are enough of
@@ -41,6 +44,8 @@ enum io_uring_cmd_flags {
 	IO_URING_F_COMPAT		= (1 << 12),
 };
 
+struct iou_loop_params;
+
 struct io_wq_work_node {
 	struct io_wq_work_node *next;
 };
@@ -268,24 +273,30 @@ struct io_alloc_cache {
 	unsigned int		init_clear;
 };
 
+enum {
+	IO_RING_F_DRAIN_NEXT		= BIT(0),
+	IO_RING_F_OP_RESTRICTED		= BIT(1),
+	IO_RING_F_REG_RESTRICTED	= BIT(2),
+	IO_RING_F_OFF_TIMEOUT_USED	= BIT(3),
+	IO_RING_F_DRAIN_ACTIVE		= BIT(4),
+	IO_RING_F_HAS_EVFD		= BIT(5),
+	/* all CQEs should be posted only by the submitter task */
+	IO_RING_F_TASK_COMPLETE		= BIT(6),
+	IO_RING_F_LOCKLESS_CQ		= BIT(7),
+	IO_RING_F_SYSCALL_IOPOLL	= BIT(8),
+	IO_RING_F_POLL_ACTIVATED	= BIT(9),
+	IO_RING_F_DRAIN_DISABLED	= BIT(10),
+	IO_RING_F_COMPAT		= BIT(11),
+	IO_RING_F_IOWQ_LIMITS_SET	= BIT(12),
+};
+
 struct io_ring_ctx {
 	/* const or read-mostly hot data */
 	struct {
+		/* ring setup flags */
 		unsigned int		flags;
-		unsigned int		drain_next: 1;
-		unsigned int		op_restricted: 1;
-		unsigned int		reg_restricted: 1;
-		unsigned int		off_timeout_used: 1;
-		unsigned int		drain_active: 1;
-		unsigned int		has_evfd: 1;
-		/* all CQEs should be posted only by the submitter task */
-		unsigned int		task_complete: 1;
-		unsigned int		lockless_cq: 1;
-		unsigned int		syscall_iopoll: 1;
-		unsigned int		poll_activated: 1;
-		unsigned int		drain_disabled: 1;
-		unsigned int		compat: 1;
-		unsigned int		iowq_limits_set : 1;
+		/* internal state flags IO_RING_F_* flags , mostly read-only */
+		unsigned int		int_flags;
 
 		struct task_struct	*submitter_task;
 		struct io_rings		*rings;
@@ -355,6 +366,9 @@ struct io_ring_ctx {
 		struct io_alloc_cache	rw_cache;
 		struct io_alloc_cache	cmd_cache;
 
+		int (*loop_step)(struct io_ring_ctx *ctx,
+				 struct iou_loop_params *);
+
 		/*
 		 * Any cancelable uring_cmd is added to this list in
 		 * ->uring_cmd() by io_uring_cmd_insert_cancelable()
@@ -477,6 +491,8 @@ struct io_ring_ctx {
 	DECLARE_HASHTABLE(napi_ht, 4);
 #endif
 
+	struct io_uring_bpf_ops		*bpf_ops;
+
 	/*
 	 * Protection for resize vs mmap races - both the mmap and resize
 	 * side will need to grab this lock, to prevent either side from
@@ -541,9 +557,11 @@ enum {
 	REQ_F_BL_NO_RECYCLE_BIT,
 	REQ_F_BUFFERS_COMMIT_BIT,
 	REQ_F_BUF_NODE_BIT,
+	REQ_F_BUF_MORE_BIT,
 	REQ_F_HAS_METADATA_BIT,
 	REQ_F_IMPORT_BUFFER_BIT,
 	REQ_F_SQE_COPIED_BIT,
+	REQ_F_IOPOLL_BIT,
 
 	/* not a real bit, just to check we're not overflowing the space */
 	__REQ_F_LAST_BIT,
@@ -626,6 +644,8 @@ enum {
 	REQ_F_BUFFERS_COMMIT	= IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
 	/* buf node is valid */
 	REQ_F_BUF_NODE		= IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
+	/* incremental buffer consumption, more space available */
+	REQ_F_BUF_MORE		= IO_REQ_FLAG(REQ_F_BUF_MORE_BIT),
 	/* request has read/write metadata assigned */
 	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
 	/*
@@ -635,6 +655,8 @@ enum {
 	REQ_F_IMPORT_BUFFER	= IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT),
 	/* ->sqe_copy() has been called, if necessary */
 	REQ_F_SQE_COPIED	= IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT),
+	/* request must be iopolled to completion (set in ->issue()) */
+	REQ_F_IOPOLL		= IO_REQ_FLAG(REQ_F_IOPOLL_BIT),
 };
 
 struct io_tw_req {
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 99b7209dabd77..2c5685adf3a97 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -9,7 +9,7 @@
 #include <linux/types.h>
 #include <linux/mm_types.h>
 #include <linux/blkdev.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 
 struct address_space;
 struct fiemap_extent_info;
@@ -65,6 +65,8 @@ struct vm_fault;
  *
  * IOMAP_F_ATOMIC_BIO indicates that (write) I/O will be issued as an atomic
  * bio, i.e. set REQ_ATOMIC.
+ *
+ * IOMAP_F_INTEGRITY indicates that the filesystems handles integrity metadata.
  */
 #define IOMAP_F_NEW		(1U << 0)
 #define IOMAP_F_DIRTY		(1U << 1)
@@ -79,6 +81,11 @@ struct vm_fault;
 #define IOMAP_F_BOUNDARY	(1U << 6)
 #define IOMAP_F_ANON_WRITE	(1U << 7)
 #define IOMAP_F_ATOMIC_BIO	(1U << 8)
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+#define IOMAP_F_INTEGRITY	(1U << 9)
+#else
+#define IOMAP_F_INTEGRITY	0
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
 
 /*
  * Flag reserved for file system specific usage
@@ -493,6 +500,7 @@ struct iomap_read_folio_ctx {
 	struct folio		*cur_folio;
 	struct readahead_control *rac;
 	void			*read_ctx;
+	loff_t			read_ctx_file_offset;
 };
 
 struct iomap_read_ops {
@@ -512,7 +520,14 @@ struct iomap_read_ops {
 	 *
 	 * This is optional.
 	 */
-	void (*submit_read)(struct iomap_read_folio_ctx *ctx);
+	void (*submit_read)(const struct iomap_iter *iter,
+			struct iomap_read_folio_ctx *ctx);
+
+	/*
+	 * Optional, allows filesystem to specify own bio_set, so new bio's
+	 * can be allocated from the provided bio_set.
+	 */
+	struct bio_set *bio_set;
 };
 
 /*
@@ -598,6 +613,9 @@ int iomap_swapfile_activate(struct swap_info_struct *sis,
 extern struct bio_set iomap_ioend_bioset;
 
 #ifdef CONFIG_BLOCK
+int iomap_bio_read_folio_range(const struct iomap_iter *iter,
+		struct iomap_read_folio_ctx *ctx, size_t plen);
+
 extern const struct iomap_read_ops iomap_bio_read_ops;
 
 static inline void iomap_bio_read_folio(struct folio *folio,
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 54b8b48c762e8..a904821ed169d 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -223,6 +223,7 @@ enum iommu_domain_cookie_type {
 struct iommu_domain {
 	unsigned type;
 	enum iommu_domain_cookie_type cookie_type;
+	bool is_iommupt;
 	const struct iommu_domain_ops *ops;
 	const struct iommu_dirty_ops *dirty_ops;
 	const struct iommu_ops *owner; /* Whose domain_alloc we came from */
@@ -271,6 +272,8 @@ enum iommu_cap {
 	 */
 	IOMMU_CAP_DEFERRED_FLUSH,
 	IOMMU_CAP_DIRTY_TRACKING,	/* IOMMU supports dirty tracking */
+	/* ATS is supported and may be enabled for this device */
+	IOMMU_CAP_PCI_ATS_SUPPORTED,
 };
 
 /* These are the possible reserved region types */
diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h
index bdd2e0652bc30..53edd69acb9bd 100644
--- a/include/linux/iopoll.h
+++ b/include/linux/iopoll.h
@@ -159,7 +159,7 @@
  *
  * This macro does not rely on timekeeping.  Hence it is safe to call even when
  * timekeeping is suspended, at the expense of an underestimation of wall clock
- * time, which is rather minimal with a non-zero delay_us.
+ * time, which is rather minimal with a non-zero @delay_us.
  *
  * When available, you'll probably want to use one of the specialized
  * macros defined below rather than this macro directly.
@@ -167,9 +167,9 @@
  * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
  * case, the last read value at @args is stored in @val.
  */
-#define read_poll_timeout_atomic(op, val, cond, sleep_us, timeout_us, \
-				 sleep_before_read, args...) \
-	poll_timeout_us_atomic((val) = op(args), cond, sleep_us, timeout_us, sleep_before_read)
+#define read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \
+				 delay_before_read, args...) \
+	poll_timeout_us_atomic((val) = op(args), cond, delay_us, timeout_us, delay_before_read)
 
 /**
  * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index d26d1b1bcbfb9..b976946b3cdbb 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -3,6 +3,7 @@
 #define __LINUX_IRQENTRYCOMMON_H
 
 #include <linux/context_tracking.h>
+#include <linux/hrtimer_rearm.h>
 #include <linux/kmsan.h>
 #include <linux/rseq_entry.h>
 #include <linux/static_call_types.h>
@@ -33,6 +34,14 @@
 	 _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | _TIF_RSEQ |		\
 	 ARCH_EXIT_TO_USER_MODE_WORK)
 
+#ifdef CONFIG_HRTIMER_REARM_DEFERRED
+# define EXIT_TO_USER_MODE_WORK_SYSCALL	(EXIT_TO_USER_MODE_WORK)
+# define EXIT_TO_USER_MODE_WORK_IRQ	(EXIT_TO_USER_MODE_WORK | _TIF_HRTIMER_REARM)
+#else
+# define EXIT_TO_USER_MODE_WORK_SYSCALL	(EXIT_TO_USER_MODE_WORK)
+# define EXIT_TO_USER_MODE_WORK_IRQ	(EXIT_TO_USER_MODE_WORK)
+#endif
+
 /**
  * arch_enter_from_user_mode - Architecture specific sanity check for user mode regs
  * @regs:	Pointer to currents pt_regs
@@ -203,6 +212,7 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work
 /**
  * __exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required
  * @regs:	Pointer to pt_regs on entry stack
+ * @work_mask:	Which TIF bits need to be evaluated
  *
  * 1) check that interrupts are disabled
  * 2) call tick_nohz_user_enter_prepare()
@@ -212,7 +222,8 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work
  *
  * Don't invoke directly, use the syscall/irqentry_ prefixed variants below
  */
-static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs)
+static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs,
+							const unsigned long work_mask)
 {
 	unsigned long ti_work;
 
@@ -222,8 +233,10 @@ static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs)
 	tick_nohz_user_enter_prepare();
 
 	ti_work = read_thread_flags();
-	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
-		ti_work = exit_to_user_mode_loop(regs, ti_work);
+	if (unlikely(ti_work & work_mask)) {
+		if (!hrtimer_rearm_deferred_user_irq(&ti_work, work_mask))
+			ti_work = exit_to_user_mode_loop(regs, ti_work);
+	}
 
 	arch_exit_to_user_mode_prepare(regs, ti_work);
 }
@@ -239,7 +252,7 @@ static __always_inline void __exit_to_user_mode_validate(void)
 /* Temporary workaround to keep ARM64 alive */
 static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *regs)
 {
-	__exit_to_user_mode_prepare(regs);
+	__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK);
 	rseq_exit_to_user_mode_legacy();
 	__exit_to_user_mode_validate();
 }
@@ -253,7 +266,7 @@ static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *reg
  */
 static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
 {
-	__exit_to_user_mode_prepare(regs);
+	__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_SYSCALL);
 	rseq_syscall_exit_to_user_mode();
 	__exit_to_user_mode_validate();
 }
@@ -267,7 +280,7 @@ static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *re
  */
 static __always_inline void irqentry_exit_to_user_mode_prepare(struct pt_regs *regs)
 {
-	__exit_to_user_mode_prepare(regs);
+	__exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_IRQ);
 	rseq_irqentry_exit_to_user_mode();
 	__exit_to_user_mode_validate();
 }
diff --git a/include/linux/irq.h b/include/linux/irq.h
index 951acbdb9f845..efa514ee562fc 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -35,6 +35,10 @@ enum irqchip_irq_state;
  *
  * Bits 0-7 are the same as the IRQF_* bits in linux/interrupt.h
  *
+ * Note that the first 6 definitions are shadowed by C preprocessor definitions
+ * in include/dt-bindings/interrupt-controller/irq.h.  This is not an issue, as
+ * the actual values must be the same, due to being part of the stable DT ABI.
+ *
  * IRQ_TYPE_NONE		- default, unspecified type
  * IRQ_TYPE_EDGE_RISING		- rising edge triggered
  * IRQ_TYPE_EDGE_FALLING	- falling edge triggered
diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h
index b78488df6c989..40d2fce682940 100644
--- a/include/linux/irqchip/arm-gic-v5.h
+++ b/include/linux/irqchip/arm-gic-v5.h
@@ -24,6 +24,28 @@
 #define GICV5_HWIRQ_TYPE_LPI		UL(0x2)
 #define GICV5_HWIRQ_TYPE_SPI		UL(0x3)
 
+/*
+ * Architected PPIs
+ */
+#define GICV5_ARCH_PPI_S_DB_PPI		0x0
+#define GICV5_ARCH_PPI_RL_DB_PPI	0x1
+#define GICV5_ARCH_PPI_NS_DB_PPI	0x2
+#define GICV5_ARCH_PPI_SW_PPI		0x3
+#define GICV5_ARCH_PPI_HACDBSIRQ	0xf
+#define GICV5_ARCH_PPI_CNTHVS		0x13
+#define GICV5_ARCH_PPI_CNTHPS		0x14
+#define GICV5_ARCH_PPI_PMBIRQ		0x15
+#define GICV5_ARCH_PPI_COMMIRQ		0x16
+#define GICV5_ARCH_PPI_PMUIRQ		0x17
+#define GICV5_ARCH_PPI_CTIIRQ		0x18
+#define GICV5_ARCH_PPI_GICMNT		0x19
+#define GICV5_ARCH_PPI_CNTHP		0x1a
+#define GICV5_ARCH_PPI_CNTV		0x1b
+#define GICV5_ARCH_PPI_CNTHV		0x1c
+#define GICV5_ARCH_PPI_CNTPS		0x1d
+#define GICV5_ARCH_PPI_CNTP		0x1e
+#define GICV5_ARCH_PPI_TRBIRQ		0x1f
+
 /*
  * Tables attributes
  */
@@ -365,6 +387,11 @@ int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type);
 int gicv5_irs_iste_alloc(u32 lpi);
 void gicv5_irs_syncr(void);
 
+/* Embedded in kvm.arch */
+struct gicv5_vpe {
+	bool			resident;
+};
+
 struct gicv5_its_devtab_cfg {
 	union {
 		struct {
diff --git a/include/linux/irqdomain_defs.h b/include/linux/irqdomain_defs.h
index 36653e2ee1c92..3a03bdfeeee93 100644
--- a/include/linux/irqdomain_defs.h
+++ b/include/linux/irqdomain_defs.h
@@ -17,7 +17,6 @@ enum irq_domain_bus_token {
 	DOMAIN_BUS_PLATFORM_MSI,
 	DOMAIN_BUS_NEXUS,
 	DOMAIN_BUS_IPI,
-	DOMAIN_BUS_FSL_MC_MSI,
 	DOMAIN_BUS_TI_SCI_INTA_MSI,
 	DOMAIN_BUS_WAKEUP,
 	DOMAIN_BUS_VMD_MSI,
diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h
index d1c3d4941854a..bbd57061802cd 100644
--- a/include/linux/jiffies.h
+++ b/include/linux/jiffies.h
@@ -67,10 +67,6 @@ extern void register_refined_jiffies(long clock_tick_rate);
 /* USER_TICK_USEC is the time between ticks in usec assuming fake USER_HZ */
 #define USER_TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ)
 
-#ifndef __jiffy_arch_data
-#define __jiffy_arch_data
-#endif
-
 /*
  * The 64-bit value is not atomic on 32-bit systems - you MUST NOT read it
  * without sampling the sequence number in jiffies_lock.
@@ -83,7 +79,7 @@ extern void register_refined_jiffies(long clock_tick_rate);
  * See arch/ARCH/kernel/vmlinux.lds.S
  */
 extern u64 __cacheline_aligned_in_smp jiffies_64;
-extern unsigned long volatile __cacheline_aligned_in_smp __jiffy_arch_data jiffies;
+extern unsigned long volatile __cacheline_aligned_in_smp jiffies;
 
 #if (BITS_PER_LONG < 64)
 u64 get_jiffies_64(void);
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index fdb79dd1ebd8c..b9c7b0ebf7b9d 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -87,13 +87,6 @@ struct static_key {
 	atomic_t enabled;
 #ifdef CONFIG_JUMP_LABEL
 /*
- * Note:
- *   To make anonymous unions work with old compilers, the static
- *   initialization of them requires brackets. This creates a dependency
- *   on the order of the struct with the initializers. If any fields
- *   are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
- *   to be modified.
- *
  * bit 0 => 1 if key is initially true
  *	    0 if initially false
  * bit 1 => 1 if points to struct static_key_mod
@@ -238,19 +231,12 @@ extern void static_key_enable_cpuslocked(struct static_key *key);
 extern void static_key_disable_cpuslocked(struct static_key *key);
 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
 
-/*
- * We should be using ATOMIC_INIT() for initializing .enabled, but
- * the inclusion of atomic.h is problematic for inclusion of jump_label.h
- * in 'low-level' headers. Thus, we are initializing .enabled with a
- * raw value, but have added a BUILD_BUG_ON() to catch any issues in
- * jump_label_init() see: kernel/jump_label.c.
- */
 #define STATIC_KEY_INIT_TRUE					\
-	{ .enabled = { 1 },					\
-	  { .type = JUMP_TYPE_TRUE } }
+	{ .enabled = ATOMIC_INIT(1),				\
+	  .type = JUMP_TYPE_TRUE }
 #define STATIC_KEY_INIT_FALSE					\
-	{ .enabled = { 0 },					\
-	  { .type = JUMP_TYPE_FALSE } }
+	{ .enabled = ATOMIC_INIT(0),				\
+	  .type = JUMP_TYPE_FALSE }
 
 #else  /* !CONFIG_JUMP_LABEL */
 
diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h
index d5dd54c53ace6..7d4c9dca06c87 100644
--- a/include/linux/kallsyms.h
+++ b/include/linux/kallsyms.h
@@ -16,10 +16,15 @@
 #include <asm/sections.h>
 
 #define KSYM_NAME_LEN 512
+
+/* Extra space for " (path/to/file.c:12345)" suffix when lineinfo is enabled */
+#define KSYM_LINEINFO_LEN (IS_ENABLED(CONFIG_KALLSYMS_LINEINFO) ? 128 : 0)
+
 #define KSYM_SYMBOL_LEN (sizeof("%s+%#lx/%#lx [%s %s]") + \
 			(KSYM_NAME_LEN - 1) + \
 			2*(BITS_PER_LONG*3/10) + (MODULE_NAME_LEN - 1) + \
-			(BUILD_ID_SIZE_MAX * 2) + 1)
+			(BUILD_ID_SIZE_MAX * 2) + 1 + \
+			KSYM_LINEINFO_LEN)
 
 struct cred;
 struct module;
@@ -96,6 +101,9 @@ extern int sprint_backtrace_build_id(char *buffer, unsigned long address);
 
 int lookup_symbol_name(unsigned long addr, char *symname);
 
+bool kallsyms_lookup_lineinfo(unsigned long addr,
+			      const char **file, unsigned int *line);
+
 #else /* !CONFIG_KALLSYMS */
 
 static inline unsigned long kallsyms_lookup_name(const char *name)
@@ -164,6 +172,13 @@ static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long)
 {
 	return -EOPNOTSUPP;
 }
+
+static inline bool kallsyms_lookup_lineinfo(unsigned long addr,
+					    const char **file,
+					    unsigned int *line)
+{
+	return false;
+}
 #endif /*CONFIG_KALLSYMS*/
 
 static inline void print_ip_sym(const char *loglvl, unsigned long ip)
diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index 338a1921a50ad..bf233bde68c7e 100644
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -352,8 +352,8 @@ bool __kasan_mempool_poison_object(void *ptr, unsigned long ip);
  * kasan_mempool_unpoison_object().
  *
  * This function operates on all slab allocations including large kmalloc
- * allocations (the ones returned by kmalloc_large() or by kmalloc() with the
- * size > KMALLOC_MAX_SIZE).
+ * allocations (i.e. the ones backed directly by the buddy allocator rather
+ * than kmalloc slab caches).
  *
  * Return: true if the allocation can be safely reused; false otherwise.
  */
@@ -381,8 +381,8 @@ void __kasan_mempool_unpoison_object(void *ptr, size_t size, unsigned long ip);
  * original tags based on the pointer value.
  *
  * This function operates on all slab allocations including large kmalloc
- * allocations (the ones returned by kmalloc_large() or by kmalloc() with the
- * size > KMALLOC_MAX_SIZE).
+ * allocations (i.e. the ones backed directly by the buddy allocator rather
+ * than kmalloc slab caches).
  */
 static __always_inline void kasan_mempool_unpoison_object(void *ptr,
 							  size_t size)
diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h
index b5a5f32fdfd1a..d8f57f0af5e48 100644
--- a/include/linux/kernfs.h
+++ b/include/linux/kernfs.h
@@ -99,8 +99,6 @@ enum kernfs_node_type {
 
 #define KERNFS_TYPE_MASK		0x000f
 #define KERNFS_FLAG_MASK		~KERNFS_TYPE_MASK
-#define KERNFS_MAX_USER_XATTRS		128
-#define KERNFS_USER_XATTR_SIZE_LIMIT	(128 << 10)
 
 enum kernfs_node_flag {
 	KERNFS_ACTIVATED	= 0x0010,
diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h
index 2201a0d2c159a..6b7d8ef550f98 100644
--- a/include/linux/kho/abi/kexec_handover.h
+++ b/include/linux/kho/abi/kexec_handover.h
@@ -10,8 +10,13 @@
 #ifndef _LINUX_KHO_ABI_KEXEC_HANDOVER_H
 #define _LINUX_KHO_ABI_KEXEC_HANDOVER_H
 
+#include <linux/bits.h>
+#include <linux/log2.h>
+#include <linux/math.h>
 #include <linux/types.h>
 
+#include <asm/page.h>
+
 /**
  * DOC: Kexec Handover ABI
  *
@@ -29,32 +34,32 @@
  * compatibility is only guaranteed for kernels supporting the same ABI version.
  *
  * FDT Structure Overview:
- *   The FDT serves as a central registry for physical
- *   addresses of preserved data structures and sub-FDTs. The first kernel
- *   populates this FDT with references to memory regions and other FDTs that
- *   need to persist across the kexec transition. The subsequent kernel then
- *   parses this FDT to locate and restore the preserved data.::
+ *   The FDT serves as a central registry for physical addresses of preserved
+ *   data structures. The first kernel populates this FDT with references to
+ *   memory regions and other metadata that need to persist across the kexec
+ *   transition. The subsequent kernel then parses this FDT to locate and
+ *   restore the preserved data.::
  *
  *     / {
- *         compatible = "kho-v1";
+ *         compatible = "kho-v2";
  *
  *         preserved-memory-map = <0x...>;
  *
  *         <subnode-name-1> {
- *             fdt = <0x...>;
+ *             preserved-data = <0x...>;
  *         };
  *
  *         <subnode-name-2> {
- *             fdt = <0x...>;
+ *             preserved-data = <0x...>;
  *         };
  *               ... ...
  *         <subnode-name-N> {
- *             fdt = <0x...>;
+ *             preserved-data = <0x...>;
  *         };
  *     };
  *
  *   Root KHO Node (/):
- *     - compatible: "kho-v1"
+ *     - compatible: "kho-v2"
  *
  *       Indentifies the overall KHO ABI version.
  *
@@ -69,20 +74,20 @@
  *     is provided by the subsystem that uses KHO for preserving its
  *     data.
  *
- *     - fdt: u64
+ *     - preserved-data: u64
  *
- *       Physical address pointing to a subnode FDT blob that is also
+ *       Physical address pointing to a subnode data blob that is also
  *       being preserved.
  */
 
 /* The compatible string for the KHO FDT root node. */
-#define KHO_FDT_COMPATIBLE "kho-v1"
+#define KHO_FDT_COMPATIBLE "kho-v2"
 
 /* The FDT property for the preserved memory map. */
 #define KHO_FDT_MEMORY_MAP_PROP_NAME "preserved-memory-map"
 
-/* The FDT property for sub-FDTs. */
-#define KHO_FDT_SUB_TREE_PROP_NAME "fdt"
+/* The FDT property for preserved data blobs. */
+#define KHO_FDT_SUB_TREE_PROP_NAME "preserved-data"
 
 /**
  * DOC: Kexec Handover ABI for vmalloc Preservation
@@ -160,4 +165,113 @@ struct kho_vmalloc {
 	unsigned short order;
 };
 
+/**
+ * DOC: KHO persistent memory tracker
+ *
+ * KHO tracks preserved memory using a radix tree data structure. Each node of
+ * the tree is exactly a single page. The leaf nodes are bitmaps where each set
+ * bit is a preserved page of any order. The intermediate nodes are tables of
+ * physical addresses that point to a lower level node.
+ *
+ * The tree hierarchy is shown below::
+ *
+ *   root
+ *   +-------------------+
+ *   |     Level 5       | (struct kho_radix_node)
+ *   +-------------------+
+ *     |
+ *     v
+ *   +-------------------+
+ *   |     Level 4       | (struct kho_radix_node)
+ *   +-------------------+
+ *     |
+ *     | ... (intermediate levels)
+ *     |
+ *     v
+ *   +-------------------+
+ *   |      Level 0      | (struct kho_radix_leaf)
+ *   +-------------------+
+ *
+ * The tree is traversed using a key that encodes the page's physical address
+ * (pa) and its order into a single unsigned long value. The encoded key value
+ * is composed of two parts: the 'order bit' in the upper part and the
+ * 'shifted physical address' in the lower part.::
+ *
+ *   +------------+-----------------------------+--------------------------+
+ *   | Page Order | Order Bit                   | Shifted Physical Address |
+ *   +------------+-----------------------------+--------------------------+
+ *   | 0          | ...000100 ... (at bit 52)   | pa >> (PAGE_SHIFT + 0)   |
+ *   | 1          | ...000010 ... (at bit 51)   | pa >> (PAGE_SHIFT + 1)   |
+ *   | 2          | ...000001 ... (at bit 50)   | pa >> (PAGE_SHIFT + 2)   |
+ *   | ...        | ...                         | ...                      |
+ *   +------------+-----------------------------+--------------------------+
+ *
+ * Shifted Physical Address:
+ * The 'shifted physical address' is the physical address normalized for its
+ * order. It effectively represents the PFN shifted right by the order.
+ *
+ * Order Bit:
+ * The 'order bit' encodes the page order by setting a single bit at a
+ * specific position. The position of this bit itself represents the order.
+ *
+ * For instance, on a 64-bit system with 4KB pages (PAGE_SHIFT = 12), the
+ * maximum range for the shifted physical address (for order 0) is 52 bits
+ * (64 - 12). This address occupies bits [0-51]. For order 0, the order bit is
+ * set at position 52.
+ *
+ * The following diagram illustrates how the encoded key value is split into
+ * indices for the tree levels, with PAGE_SIZE of 4KB::
+ *
+ *        63:60   59:51    50:42    41:33    32:24    23:15         14:0
+ *   +---------+--------+--------+--------+--------+--------+-----------------+
+ *   |    0    |  Lv 5  |  Lv 4  |  Lv 3  |  Lv 2  |  Lv 1  |  Lv 0 (bitmap)  |
+ *   +---------+--------+--------+--------+--------+--------+-----------------+
+ *
+ * The radix tree stores pages of all orders in a single 6-level hierarchy. It
+ * efficiently shares higher tree levels, especially due to common zero top
+ * address bits, allowing a single, efficient algorithm to manage all
+ * pages. This bitmap approach also offers memory efficiency; for example, a
+ * 512KB bitmap can cover a 16GB memory range for 0-order pages with PAGE_SIZE =
+ * 4KB.
+ *
+ * The data structures defined here are part of the KHO ABI. Any modification
+ * to these structures that breaks backward compatibility must be accompanied by
+ * an update to the "compatible" string. This ensures that a newer kernel can
+ * correctly interpret the data passed by an older kernel.
+ */
+
+/*
+ * Defines constants for the KHO radix tree structure, used to track preserved
+ * memory. These constants govern the indexing, sizing, and depth of the tree.
+ */
+enum kho_radix_consts {
+	/*
+	 * The bit position of the order bit (and also the length of the
+	 * shifted physical address) for an order-0 page.
+	 */
+	KHO_ORDER_0_LOG2 = 64 - PAGE_SHIFT,
+
+	/* Size of the table in kho_radix_node, in log2 */
+	KHO_TABLE_SIZE_LOG2 = const_ilog2(PAGE_SIZE / sizeof(phys_addr_t)),
+
+	/* Number of bits in the kho_radix_leaf bitmap, in log2 */
+	KHO_BITMAP_SIZE_LOG2 = PAGE_SHIFT + const_ilog2(BITS_PER_BYTE),
+
+	/*
+	 * The total tree depth is the number of intermediate levels
+	 * and 1 bitmap level.
+	 */
+	KHO_TREE_MAX_DEPTH =
+		DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2,
+			     KHO_TABLE_SIZE_LOG2) + 1,
+};
+
+struct kho_radix_node {
+	u64 table[1 << KHO_TABLE_SIZE_LOG2];
+};
+
+struct kho_radix_leaf {
+	DECLARE_BITMAP(bitmap, 1 << KHO_BITMAP_SIZE_LOG2);
+};
+
 #endif	/* _LINUX_KHO_ABI_KEXEC_HANDOVER_H */
diff --git a/include/linux/kho/abi/memfd.h b/include/linux/kho/abi/memfd.h
index 68cb6303b846e..08b10fea2afc8 100644
--- a/include/linux/kho/abi/memfd.h
+++ b/include/linux/kho/abi/memfd.h
@@ -56,10 +56,24 @@ struct memfd_luo_folio_ser {
 	u64 index;
 } __packed;
 
+/*
+ * The set of seals this version supports preserving. If support for any new
+ * seals is needed, add it here and bump version.
+ */
+#define MEMFD_LUO_ALL_SEALS (F_SEAL_SEAL | \
+			     F_SEAL_SHRINK | \
+			     F_SEAL_GROW | \
+			     F_SEAL_WRITE | \
+			     F_SEAL_FUTURE_WRITE | \
+			     F_SEAL_EXEC)
+
 /**
  * struct memfd_luo_ser - Main serialization structure for a memfd.
  * @pos:       The file's current position (f_pos).
  * @size:      The total size of the file in bytes (i_size).
+ * @seals:     The seals present on the memfd. The seals are uABI so it is safe
+ *             to directly use them in the ABI.
+ * @flags:     Flags for the file. Unused flag bits must be set to 0.
  * @nr_folios: Number of folios in the folios array.
  * @folios:    KHO vmalloc descriptor pointing to the array of
  *             struct memfd_luo_folio_ser.
@@ -67,11 +81,13 @@ struct memfd_luo_folio_ser {
 struct memfd_luo_ser {
 	u64 pos;
 	u64 size;
+	u32 seals;
+	u32 flags;
 	u64 nr_folios;
 	struct kho_vmalloc folios;
 } __packed;
 
 /* The compatibility string for memfd file handler */
-#define MEMFD_LUO_FH_COMPATIBLE	"memfd-v1"
+#define MEMFD_LUO_FH_COMPATIBLE	"memfd-v2"
 
 #endif /* _LINUX_KHO_ABI_MEMFD_H */
diff --git a/include/linux/kho_radix_tree.h b/include/linux/kho_radix_tree.h
new file mode 100644
index 0000000000000..84e918b96e53d
--- /dev/null
+++ b/include/linux/kho_radix_tree.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _LINUX_KHO_RADIX_TREE_H
+#define _LINUX_KHO_RADIX_TREE_H
+
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/mutex_types.h>
+#include <linux/types.h>
+
+/**
+ * DOC: Kexec Handover Radix Tree
+ *
+ * This is a radix tree implementation for tracking physical memory pages
+ * across kexec transitions. It was developed for the KHO mechanism but is
+ * designed for broader use by any subsystem that needs to preserve pages.
+ *
+ * The radix tree is a multi-level tree where leaf nodes are bitmaps
+ * representing individual pages. To allow pages of different sizes (orders)
+ * to be stored efficiently in a single tree, it uses a unique key encoding
+ * scheme. Each key is an unsigned long that combines a page's physical
+ * address and its order.
+ *
+ * Client code is responsible for allocating the root node of the tree,
+ * initializing the mutex lock, and managing its lifecycle. It must use the
+ * tree data structures defined in the KHO ABI,
+ * `include/linux/kho/abi/kexec_handover.h`.
+ */
+
+struct kho_radix_node;
+
+struct kho_radix_tree {
+	struct kho_radix_node *root;
+	struct mutex lock; /* protects the tree's structure and root pointer */
+};
+
+typedef int (*kho_radix_tree_walk_callback_t)(phys_addr_t phys,
+					      unsigned int order);
+
+#ifdef CONFIG_KEXEC_HANDOVER
+
+int kho_radix_add_page(struct kho_radix_tree *tree, unsigned long pfn,
+		       unsigned int order);
+
+void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
+			unsigned int order);
+
+int kho_radix_walk_tree(struct kho_radix_tree *tree,
+			kho_radix_tree_walk_callback_t cb);
+
+#else  /* #ifdef CONFIG_KEXEC_HANDOVER */
+
+static inline int kho_radix_add_page(struct kho_radix_tree *tree, long pfn,
+				     unsigned int order)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void kho_radix_del_page(struct kho_radix_tree *tree,
+				      unsigned long pfn, unsigned int order) { }
+
+static inline int kho_radix_walk_tree(struct kho_radix_tree *tree,
+				      kho_radix_tree_walk_callback_t cb)
+{
+	return -EOPNOTSUPP;
+}
+
+#endif /* #ifdef CONFIG_KEXEC_HANDOVER */
+
+#endif	/* _LINUX_KHO_RADIX_TREE_H */
diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index c982694c987b9..d39d0d5483a23 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -17,8 +17,8 @@
 #ifdef CONFIG_KSM
 int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
 		unsigned long end, int advice, vm_flags_t *vm_flags);
-vm_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file,
-			 vm_flags_t vm_flags);
+vma_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file,
+			  vma_flags_t vma_flags);
 int ksm_enable_merge_any(struct mm_struct *mm);
 int ksm_disable_merge_any(struct mm_struct *mm);
 int ksm_disable(struct mm_struct *mm);
@@ -103,10 +103,10 @@ bool ksm_process_mergeable(struct mm_struct *mm);
 
 #else  /* !CONFIG_KSM */
 
-static inline vm_flags_t ksm_vma_flags(struct mm_struct *mm,
-		const struct file *file, vm_flags_t vm_flags)
+static inline vma_flags_t ksm_vma_flags(struct mm_struct *mm,
+		const struct file *file, vma_flags_t vma_flags)
 {
-	return vm_flags;
+	return vma_flags;
 }
 
 static inline int ksm_disable(struct mm_struct *mm)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 6b76e7a6f4c22..d2ac8777c7661 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -318,7 +318,8 @@ static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop)
 struct kvm_mmio_fragment {
 	gpa_t gpa;
 	void *data;
-	unsigned len;
+	u64 val;
+	unsigned int len;
 };
 
 struct kvm_vcpu {
@@ -1627,6 +1628,13 @@ static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {}
 #endif
 
 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
+/*
+ * kvm_arch_shutdown() is invoked immediately prior to forcefully disabling
+ * hardware virtualization on all CPUs via IPI function calls (in preparation
+ * for shutdown or reboot), e.g. to allow arch code to prepare for disabling
+ * virtualization while KVM may be actively running vCPUs.
+ */
+void kvm_arch_shutdown(void);
 /*
  * kvm_arch_{enable,disable}_virtualization() are called on one CPU, under
  * kvm_usage_lock, immediately after/before 0=>1 and 1=>0 transitions of
@@ -2300,7 +2308,6 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
 
 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
 extern bool enable_virt_at_load;
-extern bool kvm_rebooting;
 #endif
 
 extern unsigned int halt_poll_ns;
@@ -2366,6 +2373,7 @@ void kvm_unregister_device_ops(u32 type);
 extern struct kvm_device_ops kvm_mpic_ops;
 extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
 extern struct kvm_device_ops kvm_arm_vgic_v3_ops;
+extern struct kvm_device_ops kvm_arm_vgic_v5_ops;
 
 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
 
@@ -2594,12 +2602,4 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
 				    struct kvm_pre_fault_memory *range);
 #endif
 
-#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
-int kvm_enable_virtualization(void);
-void kvm_disable_virtualization(void);
-#else
-static inline int kvm_enable_virtualization(void) { return 0; }
-static inline void kvm_disable_virtualization(void) { }
-#endif
-
 #endif
diff --git a/include/linux/leafops.h b/include/linux/leafops.h
index a9ff94b744f22..dd4130b7cb7f5 100644
--- a/include/linux/leafops.h
+++ b/include/linux/leafops.h
@@ -363,6 +363,19 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry)
 	return swp_offset(entry) & SWP_PFN_MASK;
 }
 
+static inline void softleaf_migration_entry_check(softleaf_t entry,
+			struct folio *folio)
+{
+	/* See __split_folio_to_order() comment */
+	smp_rmb();
+
+	/*
+	 * Any use of migration entries may only occur while the
+	 * corresponding page is locked
+	 */
+	VM_WARN_ON_ONCE(!folio_test_locked(folio));
+}
+
 /**
  * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry.
  * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
@@ -374,11 +387,8 @@ static inline struct page *softleaf_to_page(softleaf_t entry)
 	struct page *page = pfn_to_page(softleaf_to_pfn(entry));
 
 	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
-	/*
-	 * Any use of migration entries may only occur while the
-	 * corresponding page is locked
-	 */
-	VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page));
+	if (softleaf_is_migration(entry))
+		softleaf_migration_entry_check(entry, page_folio(page));
 
 	return page;
 }
@@ -394,12 +404,8 @@ static inline struct folio *softleaf_to_folio(softleaf_t entry)
 	struct folio *folio = pfn_folio(softleaf_to_pfn(entry));
 
 	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
-	/*
-	 * Any use of migration entries may only occur while the
-	 * corresponding folio is locked.
-	 */
-	VM_WARN_ON_ONCE(softleaf_is_migration(entry) &&
-			!folio_test_locked(folio));
+	if (softleaf_is_migration(entry))
+		softleaf_migration_entry_check(entry, folio);
 
 	return folio;
 }
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 00346ce3af5e0..5c085ef4eda7b 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -1205,7 +1205,6 @@ extern unsigned int ata_do_dev_read_id(struct ata_device *dev,
 				       struct ata_taskfile *tf, __le16 *id);
 extern void ata_qc_complete(struct ata_queued_cmd *qc);
 extern u64 ata_qc_get_active(struct ata_port *ap);
-extern void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd);
 extern int ata_std_bios_param(struct scsi_device *sdev,
 			      struct gendisk *unused,
 			      sector_t capacity, int geom[]);
@@ -1226,7 +1225,8 @@ extern int ata_ncq_prio_enable(struct ata_port *ap, struct scsi_device *sdev,
 extern struct ata_device *ata_dev_pair(struct ata_device *adev);
 int ata_set_mode(struct ata_link *link, struct ata_device **r_failed_dev);
 extern void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap);
-extern void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, struct list_head *eh_q);
+int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
+			       struct list_head *eh_q);
 
 /*
  * SATA specific code - drivers/ata/libata-sata.c
diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h
index dd11fdc76a5f2..636175432d622 100644
--- a/include/linux/liveupdate.h
+++ b/include/linux/liveupdate.h
@@ -12,6 +12,7 @@
 #include <linux/kho/abi/luo.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
+#include <linux/rwsem.h>
 #include <linux/types.h>
 #include <uapi/linux/liveupdate.h>
 
@@ -107,6 +108,8 @@ struct liveupdate_file_handler {
 	struct list_head __private list;
 	/* A list of FLB dependencies. */
 	struct list_head __private flb_list;
+	/* Protects flb_list */
+	struct rw_semaphore __private flb_lock;
 };
 
 /**
@@ -228,12 +231,12 @@ bool liveupdate_enabled(void);
 int liveupdate_reboot(void);
 
 int liveupdate_register_file_handler(struct liveupdate_file_handler *fh);
-int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh);
+void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh);
 
 int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 			    struct liveupdate_flb *flb);
-int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
-			      struct liveupdate_flb *flb);
+void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+			       struct liveupdate_flb *flb);
 
 int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp);
 int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp);
@@ -255,9 +258,8 @@ static inline int liveupdate_register_file_handler(struct liveupdate_file_handle
 	return -EOPNOTSUPP;
 }
 
-static inline int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
+static inline void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
 {
-	return -EOPNOTSUPP;
 }
 
 static inline int liveupdate_register_flb(struct liveupdate_file_handler *fh,
@@ -266,10 +268,9 @@ static inline int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 	return -EOPNOTSUPP;
 }
 
-static inline int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
-					    struct liveupdate_flb *flb)
+static inline void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+					     struct liveupdate_flb *flb)
 {
-	return -EOPNOTSUPP;
 }
 
 static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb,
diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h
index eff711bf973f5..234be7f12c15e 100644
--- a/include/linux/local_lock_internal.h
+++ b/include/linux/local_lock_internal.h
@@ -315,7 +315,7 @@ do {								\
 
 #endif /* CONFIG_PREEMPT_RT */
 
-#if defined(WARN_CONTEXT_ANALYSIS)
+#if defined(WARN_CONTEXT_ANALYSIS) && !defined(__CHECKER__)
 /*
  * Because the compiler only knows about the base per-CPU variable, use this
  * helper function to make the compiler think we lock/unlock the @base variable,
diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h
index c53c81242e727..b614e0deea726 100644
--- a/include/linux/lockd/bind.h
+++ b/include/linux/lockd/bind.h
@@ -10,27 +10,20 @@
 #ifndef LINUX_LOCKD_BIND_H
 #define LINUX_LOCKD_BIND_H
 
-#include <linux/lockd/nlm.h>
-/* need xdr-encoded error codes too, so... */
-#include <linux/lockd/xdr.h>
-#ifdef CONFIG_LOCKD_V4
-#include <linux/lockd/xdr4.h>
-#endif
-
-/* Dummy declarations */
+struct file_lock;
+struct nfs_fh;
 struct svc_rqst;
 struct rpc_task;
 struct rpc_clnt;
+struct super_block;
 
 /*
  * This is the set of functions for lockd->nfsd communication
  */
 struct nlmsvc_binding {
-	__be32			(*fopen)(struct svc_rqst *,
-						struct nfs_fh *,
-						struct file **,
-						int mode);
-	void			(*fclose)(struct file *);
+	int		(*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f,
+				 struct file **filp, int flags);
+	void		(*fclose)(struct file *filp);
 };
 
 extern const struct nlmsvc_binding *nlmsvc_ops;
@@ -58,6 +51,7 @@ struct nlmclnt_initdata {
 extern struct nlm_host *nlmclnt_init(const struct nlmclnt_initdata *nlm_init);
 extern void	nlmclnt_done(struct nlm_host *host);
 extern struct rpc_clnt *nlmclnt_rpc_clnt(struct nlm_host *host);
+extern void	nlmclnt_shutdown_rpc_clnt(struct nlm_host *host);
 
 /*
  * NLM client operations provide a means to modify RPC processing of NLM
@@ -82,4 +76,10 @@ extern int	nlmclnt_proc(struct nlm_host *host, int cmd, struct file_lock *fl, vo
 extern int	lockd_up(struct net *net, const struct cred *cred);
 extern void	lockd_down(struct net *net);
 
+/*
+ * Cluster failover support
+ */
+int nlmsvc_unlock_all_by_sb(struct super_block *sb);
+int nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr);
+
 #endif /* LINUX_LOCKD_BIND_H */
diff --git a/include/linux/lockd/debug.h b/include/linux/lockd/debug.h
deleted file mode 100644
index eede2ab5246f4..0000000000000
--- a/include/linux/lockd/debug.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/debug.h
- *
- * Debugging stuff.
- *
- * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LINUX_LOCKD_DEBUG_H
-#define LINUX_LOCKD_DEBUG_H
-
-#include <linux/sunrpc/debug.h>
-
-/*
- * Enable lockd debugging.
- * Requires RPC_DEBUG.
- */
-#undef ifdebug
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
-# define ifdebug(flag)		if (unlikely(nlm_debug & NLMDBG_##flag))
-#else
-# define ifdebug(flag)		if (0)
-#endif
-
-/*
- * Debug flags
- */
-#define NLMDBG_SVC		0x0001
-#define NLMDBG_CLIENT		0x0002
-#define NLMDBG_CLNTLOCK		0x0004
-#define NLMDBG_SVCLOCK		0x0008
-#define NLMDBG_MONITOR		0x0010
-#define NLMDBG_CLNTSUBS		0x0020
-#define NLMDBG_SVCSUBS		0x0040
-#define NLMDBG_HOSTCACHE	0x0080
-#define NLMDBG_XDR		0x0100
-#define NLMDBG_ALL		0x7fff
-
-#endif /* LINUX_LOCKD_DEBUG_H */
diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h
deleted file mode 100644
index 330e38776bb20..0000000000000
--- a/include/linux/lockd/lockd.h
+++ /dev/null
@@ -1,395 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/lockd.h
- *
- * General-purpose lockd include file.
- *
- * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LINUX_LOCKD_LOCKD_H
-#define LINUX_LOCKD_LOCKD_H
-
-/* XXX: a lot of this should really be under fs/lockd. */
-
-#include <linux/exportfs.h>
-#include <linux/in.h>
-#include <linux/in6.h>
-#include <net/ipv6.h>
-#include <linux/fs.h>
-#include <linux/kref.h>
-#include <linux/refcount.h>
-#include <linux/utsname.h>
-#include <linux/lockd/bind.h>
-#include <linux/lockd/xdr.h>
-#ifdef CONFIG_LOCKD_V4
-#include <linux/lockd/xdr4.h>
-#endif
-#include <linux/lockd/debug.h>
-#include <linux/sunrpc/svc.h>
-
-/*
- * Version string
- */
-#define LOCKD_VERSION		"0.5"
-
-/*
- * Default timeout for RPC calls (seconds)
- */
-#define LOCKD_DFLT_TIMEO	10
-
-/*
- * Lockd host handle (used both by the client and server personality).
- */
-struct nlm_host {
-	struct hlist_node	h_hash;		/* doubly linked list */
-	struct sockaddr_storage	h_addr;		/* peer address */
-	size_t			h_addrlen;
-	struct sockaddr_storage	h_srcaddr;	/* our address (optional) */
-	size_t			h_srcaddrlen;
-	struct rpc_clnt		*h_rpcclnt;	/* RPC client to talk to peer */
-	char			*h_name;		/* remote hostname */
-	u32			h_version;	/* interface version */
-	unsigned short		h_proto;	/* transport proto */
-	unsigned short		h_reclaiming : 1,
-				h_server     : 1, /* server side, not client side */
-				h_noresvport : 1,
-				h_inuse      : 1;
-	wait_queue_head_t	h_gracewait;	/* wait while reclaiming */
-	struct rw_semaphore	h_rwsem;	/* Reboot recovery lock */
-	u32			h_state;	/* pseudo-state counter */
-	u32			h_nsmstate;	/* true remote NSM state */
-	u32			h_pidcount;	/* Pseudopids */
-	refcount_t		h_count;	/* reference count */
-	struct mutex		h_mutex;	/* mutex for pmap binding */
-	unsigned long		h_nextrebind;	/* next portmap call */
-	unsigned long		h_expires;	/* eligible for GC */
-	struct list_head	h_lockowners;	/* Lockowners for the client */
-	spinlock_t		h_lock;
-	struct list_head	h_granted;	/* Locks in GRANTED state */
-	struct list_head	h_reclaim;	/* Locks in RECLAIM state */
-	struct nsm_handle	*h_nsmhandle;	/* NSM status handle */
-	char			*h_addrbuf;	/* address eyecatcher */
-	struct net		*net;		/* host net */
-	const struct cred	*h_cred;
-	char			nodename[UNX_MAXNODENAME + 1];
-	const struct nlmclnt_operations	*h_nlmclnt_ops;	/* Callback ops for NLM users */
-};
-
-/*
- * The largest string sm_addrbuf should hold is a full-size IPv6 address
- * (no "::" anywhere) with a scope ID.  The buffer size is computed to
- * hold eight groups of colon-separated four-hex-digit numbers, a
- * percent sign, a scope id (at most 32 bits, in decimal), and NUL.
- */
-#define NSM_ADDRBUF		((8 * 4 + 7) + (1 + 10) + 1)
-
-struct nsm_handle {
-	struct list_head	sm_link;
-	refcount_t		sm_count;
-	char			*sm_mon_name;
-	char			*sm_name;
-	struct sockaddr_storage	sm_addr;
-	size_t			sm_addrlen;
-	unsigned int		sm_monitored : 1,
-				sm_sticky : 1;	/* don't unmonitor */
-	struct nsm_private	sm_priv;
-	char			sm_addrbuf[NSM_ADDRBUF];
-};
-
-/*
- * Rigorous type checking on sockaddr type conversions
- */
-static inline struct sockaddr *nlm_addr(const struct nlm_host *host)
-{
-	return (struct sockaddr *)&host->h_addr;
-}
-
-static inline struct sockaddr *nlm_srcaddr(const struct nlm_host *host)
-{
-	return (struct sockaddr *)&host->h_srcaddr;
-}
-
-/*
- * Map an fl_owner_t into a unique 32-bit "pid"
- */
-struct nlm_lockowner {
-	struct list_head list;
-	refcount_t count;
-
-	struct nlm_host *host;
-	fl_owner_t owner;
-	uint32_t pid;
-};
-
-/*
- * This is the representation of a blocked client lock.
- */
-struct nlm_wait {
-	struct list_head	b_list;		/* linked list */
-	wait_queue_head_t	b_wait;		/* where to wait on */
-	struct nlm_host		*b_host;
-	struct file_lock	*b_lock;	/* local file lock */
-	__be32			b_status;	/* grant callback status */
-};
-
-/*
- * Memory chunk for NLM client RPC request.
- */
-#define NLMCLNT_OHSIZE		((__NEW_UTS_LEN) + 10u)
-struct nlm_rqst {
-	refcount_t		a_count;
-	unsigned int		a_flags;	/* initial RPC task flags */
-	struct nlm_host *	a_host;		/* host handle */
-	struct nlm_args		a_args;		/* arguments */
-	struct nlm_res		a_res;		/* result */
-	struct nlm_block *	a_block;
-	unsigned int		a_retries;	/* Retry count */
-	u8			a_owner[NLMCLNT_OHSIZE];
-	void *	a_callback_data; /* sent to nlmclnt_operations callbacks */
-};
-
-/*
- * This struct describes a file held open by lockd on behalf of
- * an NFS client.
- */
-struct nlm_file {
-	struct hlist_node	f_list;		/* linked list */
-	struct nfs_fh		f_handle;	/* NFS file handle */
-	struct file *		f_file[2];	/* VFS file pointers,
-						   indexed by O_ flags */
-	struct nlm_share *	f_shares;	/* DOS shares */
-	struct list_head	f_blocks;	/* blocked locks */
-	unsigned int		f_locks;	/* guesstimate # of locks */
-	unsigned int		f_count;	/* reference count */
-	struct mutex		f_mutex;	/* avoid concurrent access */
-};
-
-/*
- * This is a server block (i.e. a lock requested by some client which
- * couldn't be granted because of a conflicting lock).
- */
-#define NLM_NEVER		(~(unsigned long) 0)
-/* timeout on non-blocking call: */
-#define NLM_TIMEOUT		(7 * HZ)
-
-struct nlm_block {
-	struct kref		b_count;	/* Reference count */
-	struct list_head	b_list;		/* linked list of all blocks */
-	struct list_head	b_flist;	/* linked list (per file) */
-	struct nlm_rqst	*	b_call;		/* RPC args & callback info */
-	struct svc_serv *	b_daemon;	/* NLM service */
-	struct nlm_host *	b_host;		/* host handle for RPC clnt */
-	unsigned long		b_when;		/* next re-xmit */
-	unsigned int		b_id;		/* block id */
-	unsigned char		b_granted;	/* VFS granted lock */
-	struct nlm_file *	b_file;		/* file in question */
-	struct cache_req *	b_cache_req;	/* deferred request handling */
-	struct cache_deferred_req * b_deferred_req;
-	unsigned int		b_flags;	/* block flags */
-#define B_QUEUED		1	/* lock queued */
-#define B_GOT_CALLBACK		2	/* got lock or conflicting lock */
-#define B_TIMED_OUT		4	/* filesystem too slow to respond */
-};
-
-/*
- * Global variables
- */
-extern const struct rpc_program	nlm_program;
-extern const struct svc_procedure nlmsvc_procedures[24];
-#ifdef CONFIG_LOCKD_V4
-extern const struct svc_procedure nlmsvc_procedures4[24];
-#endif
-extern int			nlmsvc_grace_period;
-extern unsigned long		nlm_timeout;
-extern bool			nsm_use_hostnames;
-extern u32			nsm_local_state;
-
-extern struct timer_list	nlmsvc_retry;
-
-/*
- * Lockd client functions
- */
-struct nlm_rqst * nlm_alloc_call(struct nlm_host *host);
-int		  nlm_async_call(struct nlm_rqst *, u32, const struct rpc_call_ops *);
-int		  nlm_async_reply(struct nlm_rqst *, u32, const struct rpc_call_ops *);
-void		  nlmclnt_release_call(struct nlm_rqst *);
-void		  nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host,
-					struct file_lock *fl);
-void		  nlmclnt_queue_block(struct nlm_wait *block);
-__be32		  nlmclnt_dequeue_block(struct nlm_wait *block);
-int		  nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout);
-__be32		  nlmclnt_grant(const struct sockaddr *addr,
-				const struct nlm_lock *lock);
-void		  nlmclnt_recovery(struct nlm_host *);
-int		  nlmclnt_reclaim(struct nlm_host *, struct file_lock *,
-				  struct nlm_rqst *);
-void		  nlmclnt_next_cookie(struct nlm_cookie *);
-
-/*
- * Host cache
- */
-struct nlm_host  *nlmclnt_lookup_host(const struct sockaddr *sap,
-					const size_t salen,
-					const unsigned short protocol,
-					const u32 version,
-					const char *hostname,
-					int noresvport,
-					struct net *net,
-					const struct cred *cred);
-void		  nlmclnt_release_host(struct nlm_host *);
-struct nlm_host  *nlmsvc_lookup_host(const struct svc_rqst *rqstp,
-					const char *hostname,
-					const size_t hostname_len);
-void		  nlmsvc_release_host(struct nlm_host *);
-struct rpc_clnt * nlm_bind_host(struct nlm_host *);
-void		  nlm_rebind_host(struct nlm_host *);
-struct nlm_host * nlm_get_host(struct nlm_host *);
-void		  nlm_shutdown_hosts(void);
-void		  nlm_shutdown_hosts_net(struct net *net);
-void		  nlm_host_rebooted(const struct net *net,
-					const struct nlm_reboot *);
-
-/*
- * Host monitoring
- */
-int		  nsm_monitor(const struct nlm_host *host);
-void		  nsm_unmonitor(const struct nlm_host *host);
-
-struct nsm_handle *nsm_get_handle(const struct net *net,
-					const struct sockaddr *sap,
-					const size_t salen,
-					const char *hostname,
-					const size_t hostname_len);
-struct nsm_handle *nsm_reboot_lookup(const struct net *net,
-					const struct nlm_reboot *info);
-void		  nsm_release(struct nsm_handle *nsm);
-
-/*
- * This is used in garbage collection and resource reclaim
- * A return value != 0 means destroy the lock/block/share
- */
-typedef int	  (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref);
-
-/*
- * Server-side lock handling
- */
-int		  lock_to_openmode(struct file_lock *);
-__be32		  nlmsvc_lock(struct svc_rqst *, struct nlm_file *,
-			      struct nlm_host *, struct nlm_lock *, int,
-			      struct nlm_cookie *, int);
-__be32		  nlmsvc_unlock(struct net *net, struct nlm_file *, struct nlm_lock *);
-__be32		  nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file,
-			struct nlm_host *host, struct nlm_lock *lock,
-			struct nlm_lock *conflock);
-__be32		  nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct nlm_lock *);
-void		  nlmsvc_retry_blocked(struct svc_rqst *rqstp);
-void		  nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *,
-					nlm_host_match_fn_t match);
-void		  nlmsvc_grant_reply(struct nlm_cookie *, __be32);
-void		  nlmsvc_release_call(struct nlm_rqst *);
-void		  nlmsvc_locks_init_private(struct file_lock *, struct nlm_host *, pid_t);
-
-/*
- * File handling for the server personality
- */
-__be32		  nlm_lookup_file(struct svc_rqst *, struct nlm_file **,
-					struct nlm_lock *);
-void		  nlm_release_file(struct nlm_file *);
-void		  nlmsvc_put_lockowner(struct nlm_lockowner *);
-void		  nlmsvc_release_lockowner(struct nlm_lock *);
-void		  nlmsvc_mark_resources(struct net *);
-void		  nlmsvc_free_host_resources(struct nlm_host *);
-void		  nlmsvc_invalidate_all(void);
-
-/*
- * Cluster failover support
- */
-int           nlmsvc_unlock_all_by_sb(struct super_block *sb);
-int           nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr);
-
-static inline struct file *nlmsvc_file_file(const struct nlm_file *file)
-{
-	return file->f_file[O_RDONLY] ?
-	       file->f_file[O_RDONLY] : file->f_file[O_WRONLY];
-}
-
-static inline struct inode *nlmsvc_file_inode(struct nlm_file *file)
-{
-	return file_inode(nlmsvc_file_file(file));
-}
-
-static inline bool
-nlmsvc_file_cannot_lock(const struct nlm_file *file)
-{
-	return exportfs_cannot_lock(nlmsvc_file_file(file)->f_path.dentry->d_sb->s_export_op);
-}
-
-static inline int __nlm_privileged_request4(const struct sockaddr *sap)
-{
-	const struct sockaddr_in *sin = (struct sockaddr_in *)sap;
-
-	if (ntohs(sin->sin_port) > 1023)
-		return 0;
-
-	return ipv4_is_loopback(sin->sin_addr.s_addr);
-}
-
-#if IS_ENABLED(CONFIG_IPV6)
-static inline int __nlm_privileged_request6(const struct sockaddr *sap)
-{
-	const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap;
-
-	if (ntohs(sin6->sin6_port) > 1023)
-		return 0;
-
-	if (ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_MAPPED)
-		return ipv4_is_loopback(sin6->sin6_addr.s6_addr32[3]);
-
-	return ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LOOPBACK;
-}
-#else	/* IS_ENABLED(CONFIG_IPV6) */
-static inline int __nlm_privileged_request6(const struct sockaddr *sap)
-{
-	return 0;
-}
-#endif	/* IS_ENABLED(CONFIG_IPV6) */
-
-/*
- * Ensure incoming requests are from local privileged callers.
- *
- * Return TRUE if sender is local and is connecting via a privileged port;
- * otherwise return FALSE.
- */
-static inline int nlm_privileged_requester(const struct svc_rqst *rqstp)
-{
-	const struct sockaddr *sap = svc_addr(rqstp);
-
-	switch (sap->sa_family) {
-	case AF_INET:
-		return __nlm_privileged_request4(sap);
-	case AF_INET6:
-		return __nlm_privileged_request6(sap);
-	default:
-		return 0;
-	}
-}
-
-/*
- * Compare two NLM locks.
- * When the second lock is of type F_UNLCK, this acts like a wildcard.
- */
-static inline int nlm_compare_locks(const struct file_lock *fl1,
-				    const struct file_lock *fl2)
-{
-	return file_inode(fl1->c.flc_file) == file_inode(fl2->c.flc_file)
-	     && fl1->c.flc_pid   == fl2->c.flc_pid
-	     && fl1->c.flc_owner == fl2->c.flc_owner
-	     && fl1->fl_start == fl2->fl_start
-	     && fl1->fl_end   == fl2->fl_end
-	     &&(fl1->c.flc_type  == fl2->c.flc_type || fl2->c.flc_type == F_UNLCK);
-}
-
-extern const struct lock_manager_operations nlmsvc_lock_operations;
-
-#endif /* LINUX_LOCKD_LOCKD_H */
diff --git a/include/linux/lockd/nlm.h b/include/linux/lockd/nlm.h
deleted file mode 100644
index 6e343ef760dc3..0000000000000
--- a/include/linux/lockd/nlm.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/nlm.h
- *
- * Declarations for the Network Lock Manager protocol.
- *
- * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LINUX_LOCKD_NLM_H
-#define LINUX_LOCKD_NLM_H
-
-
-/* Maximum file offset in file_lock.fl_end */
-# define NLM_OFFSET_MAX		((s32) 0x7fffffff)
-# define NLM4_OFFSET_MAX	((s64) ((~(u64)0) >> 1))
-
-/* Return states for NLM */
-enum {
-	NLM_LCK_GRANTED			= 0,
-	NLM_LCK_DENIED			= 1,
-	NLM_LCK_DENIED_NOLOCKS		= 2,
-	NLM_LCK_BLOCKED			= 3,
-	NLM_LCK_DENIED_GRACE_PERIOD	= 4,
-#ifdef CONFIG_LOCKD_V4
-	NLM_DEADLCK			= 5,
-	NLM_ROFS			= 6,
-	NLM_STALE_FH			= 7,
-	NLM_FBIG			= 8,
-	NLM_FAILED			= 9,
-#endif
-};
-
-#define NLM_PROGRAM		100021
-
-#define NLMPROC_NULL		0
-#define NLMPROC_TEST		1
-#define NLMPROC_LOCK		2
-#define NLMPROC_CANCEL		3
-#define NLMPROC_UNLOCK		4
-#define NLMPROC_GRANTED		5
-#define NLMPROC_TEST_MSG	6
-#define NLMPROC_LOCK_MSG	7
-#define NLMPROC_CANCEL_MSG	8
-#define NLMPROC_UNLOCK_MSG	9
-#define NLMPROC_GRANTED_MSG	10
-#define NLMPROC_TEST_RES	11
-#define NLMPROC_LOCK_RES	12
-#define NLMPROC_CANCEL_RES	13
-#define NLMPROC_UNLOCK_RES	14
-#define NLMPROC_GRANTED_RES	15
-#define NLMPROC_NSM_NOTIFY	16		/* statd callback */
-#define NLMPROC_SHARE		20
-#define NLMPROC_UNSHARE		21
-#define NLMPROC_NM_LOCK		22
-#define NLMPROC_FREE_ALL	23
-
-#endif /* LINUX_LOCKD_NLM_H */
diff --git a/include/linux/lockd/share.h b/include/linux/lockd/share.h
deleted file mode 100644
index 1f18a9faf6452..0000000000000
--- a/include/linux/lockd/share.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/share.h
- *
- * DOS share management for lockd.
- *
- * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LINUX_LOCKD_SHARE_H
-#define LINUX_LOCKD_SHARE_H
-
-/*
- * DOS share for a specific file
- */
-struct nlm_share {
-	struct nlm_share *	s_next;		/* linked list */
-	struct nlm_host *	s_host;		/* client host */
-	struct nlm_file *	s_file;		/* shared file */
-	struct xdr_netobj	s_owner;	/* owner handle */
-	u32			s_access;	/* access mode */
-	u32			s_mode;		/* deny mode */
-};
-
-__be32	nlmsvc_share_file(struct nlm_host *, struct nlm_file *,
-					       struct nlm_args *);
-__be32	nlmsvc_unshare_file(struct nlm_host *, struct nlm_file *,
-					       struct nlm_args *);
-void	nlmsvc_traverse_shares(struct nlm_host *, struct nlm_file *,
-					       nlm_host_match_fn_t);
-
-#endif /* LINUX_LOCKD_SHARE_H */
diff --git a/include/linux/lockd/xdr.h b/include/linux/lockd/xdr.h
deleted file mode 100644
index 17d53165d9f24..0000000000000
--- a/include/linux/lockd/xdr.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/xdr.h
- *
- * XDR types for the NLM protocol
- *
- * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LOCKD_XDR_H
-#define LOCKD_XDR_H
-
-#include <linux/fs.h>
-#include <linux/filelock.h>
-#include <linux/nfs.h>
-#include <linux/sunrpc/xdr.h>
-
-#define SM_MAXSTRLEN		1024
-#define SM_PRIV_SIZE		16
-
-struct nsm_private {
-	unsigned char		data[SM_PRIV_SIZE];
-};
-
-struct svc_rqst;
-
-#define NLM_MAXCOOKIELEN    	32
-#define NLM_MAXSTRLEN		1024
-
-#define	nlm_granted		cpu_to_be32(NLM_LCK_GRANTED)
-#define	nlm_lck_denied		cpu_to_be32(NLM_LCK_DENIED)
-#define	nlm_lck_denied_nolocks	cpu_to_be32(NLM_LCK_DENIED_NOLOCKS)
-#define	nlm_lck_blocked		cpu_to_be32(NLM_LCK_BLOCKED)
-#define	nlm_lck_denied_grace_period	cpu_to_be32(NLM_LCK_DENIED_GRACE_PERIOD)
-
-#define nlm_drop_reply		cpu_to_be32(30000)
-
-/* Lock info passed via NLM */
-struct nlm_lock {
-	char *			caller;
-	unsigned int		len; 	/* length of "caller" */
-	struct nfs_fh		fh;
-	struct xdr_netobj	oh;
-	u32			svid;
-	u64			lock_start;
-	u64			lock_len;
-	struct file_lock	fl;
-};
-
-/*
- *	NLM cookies. Technically they can be 1K, but Linux only uses 8 bytes.
- *	FreeBSD uses 16, Apple Mac OS X 10.3 uses 20. Therefore we set it to
- *	32 bytes.
- */
-
-struct nlm_cookie
-{
-	unsigned char data[NLM_MAXCOOKIELEN];
-	unsigned int len;
-};
-
-/*
- * Generic lockd arguments for all but sm_notify
- */
-struct nlm_args {
-	struct nlm_cookie	cookie;
-	struct nlm_lock		lock;
-	u32			block;
-	u32			reclaim;
-	u32			state;
-	u32			monitor;
-	u32			fsm_access;
-	u32			fsm_mode;
-};
-
-/*
- * Generic lockd result
- */
-struct nlm_res {
-	struct nlm_cookie	cookie;
-	__be32			status;
-	struct nlm_lock		lock;
-};
-
-/*
- * statd callback when client has rebooted
- */
-struct nlm_reboot {
-	char			*mon;
-	unsigned int		len;
-	u32			state;
-	struct nsm_private	priv;
-};
-
-/*
- * Contents of statd callback when monitored host rebooted
- */
-#define NLMSVC_XDRSIZE		sizeof(struct nlm_args)
-
-bool	nlmsvc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-
-bool	nlmsvc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlmsvc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-
-#endif /* LOCKD_XDR_H */
diff --git a/include/linux/lockd/xdr4.h b/include/linux/lockd/xdr4.h
deleted file mode 100644
index 72831e35dca32..0000000000000
--- a/include/linux/lockd/xdr4.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * linux/include/linux/lockd/xdr4.h
- *
- * XDR types for the NLM protocol
- *
- * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de>
- */
-
-#ifndef LOCKD_XDR4_H
-#define LOCKD_XDR4_H
-
-#include <linux/fs.h>
-#include <linux/nfs.h>
-#include <linux/sunrpc/xdr.h>
-#include <linux/lockd/xdr.h>
-
-/* error codes new to NLMv4 */
-#define	nlm4_deadlock		cpu_to_be32(NLM_DEADLCK)
-#define	nlm4_rofs		cpu_to_be32(NLM_ROFS)
-#define	nlm4_stale_fh		cpu_to_be32(NLM_STALE_FH)
-#define	nlm4_fbig		cpu_to_be32(NLM_FBIG)
-#define	nlm4_failed		cpu_to_be32(NLM_FAILED)
-
-void	nlm4svc_set_file_lock_range(struct file_lock *fl, u64 off, u64 len);
-bool	nlm4svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-
-bool	nlm4svc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-bool	nlm4svc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr);
-
-extern const struct rpc_version nlm_version4;
-
-#endif /* LOCKD_XDR4_H */
diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h
index 382c56a97bba1..584db296e43b2 100644
--- a/include/linux/lsm_audit.h
+++ b/include/linux/lsm_audit.h
@@ -94,7 +94,7 @@ struct common_audit_data {
 #endif
 		char *kmod_name;
 		struct lsm_ioctlop_audit *op;
-		struct file *file;
+		const struct file *file;
 		struct lsm_ibpkey_audit *ibpkey;
 		struct lsm_ibendport_audit *ibendport;
 		int reason;
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 8c42b4bde09c0..4150c50a04820 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -198,6 +198,8 @@ LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd,
 LSM_HOOK(int, 0, mmap_addr, unsigned long addr)
 LSM_HOOK(int, 0, mmap_file, struct file *file, unsigned long reqprot,
 	 unsigned long prot, unsigned long flags)
+LSM_HOOK(int, 0, mmap_backing_file, struct vm_area_struct *vma,
+	 struct file *backing_file, struct file *user_file)
 LSM_HOOK(int, 0, file_mprotect, struct vm_area_struct *vma,
 	 unsigned long reqprot, unsigned long prot)
 LSM_HOOK(int, 0, file_lock, struct file *file, unsigned int cmd)
diff --git a/include/linux/mailbox_client.h b/include/linux/mailbox_client.h
index c6eea9afb943d..e5997120f45cf 100644
--- a/include/linux/mailbox_client.h
+++ b/include/linux/mailbox_client.h
@@ -45,6 +45,7 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg);
 int mbox_flush(struct mbox_chan *chan, unsigned long timeout);
 void mbox_client_txdone(struct mbox_chan *chan, int r); /* atomic */
 bool mbox_client_peek_data(struct mbox_chan *chan); /* atomic */
+unsigned int mbox_chan_tx_slots_available(struct mbox_chan *chan); /* atomic */
 void mbox_free_channel(struct mbox_chan *chan); /* may sleep */
 
 #endif /* __MAILBOX_CLIENT_H */
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 7b8aad47121e7..0c464eade1d66 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -139,6 +139,7 @@ enum maple_type {
 	maple_leaf_64,
 	maple_range_64,
 	maple_arange_64,
+	maple_copy,
 };
 
 enum store_type {
@@ -154,6 +155,46 @@ enum store_type {
 	wr_slot_store,
 };
 
+struct maple_copy {
+	/*
+	 * min, max, and pivots are values
+	 * start, end, split are indexes into arrays
+	 * data is a size
+	 */
+
+	struct {
+		struct maple_node *node;
+		unsigned long max;
+		enum maple_type mt;
+	} dst[3];
+	struct {
+		struct maple_node *node;
+		unsigned long max;
+		unsigned char start;
+		unsigned char end;
+		enum maple_type mt;
+	} src[4];
+	/* Simulated node */
+	void __rcu *slot[3];
+	unsigned long gap[3];
+	unsigned long min;
+	union {
+		unsigned long pivot[3];
+		struct {
+			void *_pad[2];
+			unsigned long max;
+		};
+	};
+	unsigned char end;
+
+	/*Avoid passing these around */
+	unsigned char s_count;
+	unsigned char d_count;
+	unsigned char split;
+	unsigned char data;
+	unsigned char height;
+};
+
 /**
  * DOC: Maple tree flags
  *
@@ -299,6 +340,7 @@ struct maple_node {
 		};
 		struct maple_range_64 mr64;
 		struct maple_arange_64 ma64;
+		struct maple_copy cp;
 	};
 };
 
diff --git a/include/linux/math.h b/include/linux/math.h
index 6dc1d1d32fbc8..1e8fb3efbc8ce 100644
--- a/include/linux/math.h
+++ b/include/linux/math.h
@@ -89,23 +89,7 @@
 }							\
 )
 
-/*
- * Divide positive or negative dividend by positive or negative divisor
- * and round to closest integer. Result is undefined for negative
- * divisors if the dividend variable type is unsigned and for negative
- * dividends if the divisor variable type is unsigned.
- */
-#define DIV_ROUND_CLOSEST(x, divisor)(			\
-{							\
-	typeof(x) __x = x;				\
-	typeof(divisor) __d = divisor;			\
-	(((typeof(x))-1) > 0 ||				\
-	 ((typeof(divisor))-1) > 0 ||			\
-	 (((__x) > 0) == ((__d) > 0))) ?		\
-		(((__x) + ((__d) / 2)) / (__d)) :	\
-		(((__x) - ((__d) / 2)) / (__d));	\
-}							\
-)
+#define DIV_ROUND_CLOSEST __KERNEL_DIV_ROUND_CLOSEST
 /*
  * Same as above but for u64 dividends. divisor must be a 32-bit
  * number.
diff --git a/include/linux/mdio-gpio.h b/include/linux/mdio-gpio.h
deleted file mode 100644
index cea443a672cb4..0000000000000
--- a/include/linux/mdio-gpio.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_MDIO_GPIO_H
-#define __LINUX_MDIO_GPIO_H
-
-#define MDIO_GPIO_MDC	0
-#define MDIO_GPIO_MDIO	1
-#define MDIO_GPIO_MDO	2
-
-#endif
diff --git a/include/linux/mdio.h b/include/linux/mdio.h
index 5d1203b9af205..f4f9d96094488 100644
--- a/include/linux/mdio.h
+++ b/include/linux/mdio.h
@@ -688,8 +688,6 @@ static inline int mdiodev_c45_write(struct mdio_device *mdiodev, u32 devad,
 				 val);
 }
 
-int mdiobus_register_device(struct mdio_device *mdiodev);
-int mdiobus_unregister_device(struct mdio_device *mdiodev);
 bool mdiobus_is_registered_device(struct mii_bus *bus, int addr);
 struct phy_device *mdiobus_get_phy(struct mii_bus *bus, int addr);
 
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 70b685a85bf4c..0861589695298 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -35,10 +35,10 @@ enum memcg_stat_item {
 	MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
 	MEMCG_SOCK,
 	MEMCG_PERCPU_B,
-	MEMCG_VMALLOC,
 	MEMCG_KMEM,
 	MEMCG_ZSWAP_B,
 	MEMCG_ZSWAPPED,
+	MEMCG_ZSWAP_INCOMP,
 	MEMCG_NR_STAT,
 };
 
@@ -115,6 +115,16 @@ struct mem_cgroup_per_node {
 	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
 	struct mem_cgroup_reclaim_iter	iter;
 
+	/*
+	 * objcg is wiped out as a part of the objcg repaprenting process.
+	 * orig_objcg preserves a pointer (and a reference) to the original
+	 * objcg until the end of live of memcg.
+	 */
+	struct obj_cgroup __rcu	*objcg;
+	struct obj_cgroup	*orig_objcg;
+	/* list of inherited objcgs, protected by objcg_lock */
+	struct list_head objcg_list;
+
 #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
 	/* slab stats for nmi context */
 	atomic_t		slab_reclaimable;
@@ -179,6 +189,7 @@ struct obj_cgroup {
 		struct list_head list; /* protected by objcg_lock */
 		struct rcu_head rcu;
 	};
+	bool is_root;
 };
 
 /*
@@ -257,15 +268,6 @@ struct mem_cgroup {
 	seqlock_t		socket_pressure_seqlock;
 #endif
 	int kmemcg_id;
-	/*
-	 * memcg->objcg is wiped out as a part of the objcg repaprenting
-	 * process. memcg->orig_objcg preserves a pointer (and a reference)
-	 * to the original objcg until the end of live of memcg.
-	 */
-	struct obj_cgroup __rcu	*objcg;
-	struct obj_cgroup	*orig_objcg;
-	/* list of inherited objcgs, protected by objcg_lock */
-	struct list_head objcg_list;
 
 	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
 
@@ -367,9 +369,6 @@ enum objext_flags {
 #define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1)
 
 #ifdef CONFIG_MEMCG
-
-static inline bool folio_memcg_kmem(struct folio *folio);
-
 /*
  * After the initialization objcg->memcg is always pointing at
  * a valid memcg, but can be atomically swapped to the parent memcg.
@@ -383,43 +382,19 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
 }
 
 /*
- * __folio_memcg - Get the memory cgroup associated with a non-kmem folio
- * @folio: Pointer to the folio.
- *
- * Returns a pointer to the memory cgroup associated with the folio,
- * or NULL. This function assumes that the folio is known to have a
- * proper memory cgroup pointer. It's not safe to call this function
- * against some type of folios, e.g. slab folios or ex-slab folios or
- * kmem folios.
- */
-static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
-{
-	unsigned long memcg_data = folio->memcg_data;
-
-	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
-	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
-	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);
-
-	return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
-}
-
-/*
- * __folio_objcg - get the object cgroup associated with a kmem folio.
+ * folio_objcg - get the object cgroup associated with a folio.
  * @folio: Pointer to the folio.
  *
  * Returns a pointer to the object cgroup associated with the folio,
  * or NULL. This function assumes that the folio is known to have a
- * proper object cgroup pointer. It's not safe to call this function
- * against some type of folios, e.g. slab folios or ex-slab folios or
- * LRU folios.
+ * proper object cgroup pointer.
  */
-static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
+static inline struct obj_cgroup *folio_objcg(struct folio *folio)
 {
 	unsigned long memcg_data = folio->memcg_data;
 
 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
-	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);
 
 	return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
 }
@@ -433,21 +408,30 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
  * proper memory cgroup pointer. It's not safe to call this function
  * against some type of folios, e.g. slab folios or ex-slab folios.
  *
- * For a non-kmem folio any of the following ensures folio and memcg binding
- * stability:
+ * For a folio any of the following ensures folio and objcg binding stability:
  *
  * - the folio lock
  * - LRU isolation
  * - exclusive reference
  *
- * For a kmem folio a caller should hold an rcu read lock to protect memcg
- * associated with a kmem folio from being released.
+ * Based on the stable binding of folio and objcg, for a folio any of the
+ * following ensures folio and memcg binding stability:
+ *
+ * - cgroup_mutex
+ * - the lruvec lock
+ *
+ * If the caller only want to ensure that the page counters of memcg are
+ * updated correctly, ensure that the binding stability of folio and objcg
+ * is sufficient.
+ *
+ * Note: The caller should hold an rcu read lock or cgroup_mutex to protect
+ * memcg associated with a folio from being released.
  */
 static inline struct mem_cgroup *folio_memcg(struct folio *folio)
 {
-	if (folio_memcg_kmem(folio))
-		return obj_cgroup_memcg(__folio_objcg(folio));
-	return __folio_memcg(folio);
+	struct obj_cgroup *objcg = folio_objcg(folio);
+
+	return objcg ? obj_cgroup_memcg(objcg) : NULL;
 }
 
 /*
@@ -471,15 +455,10 @@ static inline bool folio_memcg_charged(struct folio *folio)
  * has an associated memory cgroup pointer or an object cgroups vector or
  * an object cgroup.
  *
- * For a non-kmem folio any of the following ensures folio and memcg binding
- * stability:
+ * The page and objcg or memcg binding rules can refer to folio_memcg().
  *
- * - the folio lock
- * - LRU isolation
- * - exclusive reference
- *
- * For a kmem folio a caller should hold an rcu read lock to protect memcg
- * associated with a kmem folio from being released.
+ * A caller should hold an rcu read lock to protect memcg associated with a
+ * page from being released.
  */
 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
 {
@@ -488,18 +467,14 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
 	 * for slabs, READ_ONCE() should be used here.
 	 */
 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);
+	struct obj_cgroup *objcg;
 
 	if (memcg_data & MEMCG_DATA_OBJEXTS)
 		return NULL;
 
-	if (memcg_data & MEMCG_DATA_KMEM) {
-		struct obj_cgroup *objcg;
+	objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
 
-		objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
-		return obj_cgroup_memcg(objcg);
-	}
-
-	return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
+	return objcg ? obj_cgroup_memcg(objcg) : NULL;
 }
 
 static inline struct mem_cgroup *page_memcg_check(struct page *page)
@@ -548,6 +523,11 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
 	return (memcg == root_mem_cgroup);
 }
 
+static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg)
+{
+	return objcg->is_root;
+}
+
 static inline bool mem_cgroup_disabled(void)
 {
 	return !cgroup_subsys_enabled(memory_cgrp_subsys);
@@ -735,7 +715,15 @@ static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
  * folio_lruvec - return lruvec for isolating/putting an LRU folio
  * @folio: Pointer to the folio.
  *
- * This function relies on folio->mem_cgroup being stable.
+ * Call with rcu_read_lock() held to ensure the lifetime of the returned lruvec.
+ * Note that this alone will NOT guarantee the stability of the folio->lruvec
+ * association; the folio can be reparented to an ancestor if this races with
+ * cgroup deletion.
+ *
+ * Use folio_lruvec_lock() to ensure both lifetime and stability of the binding.
+ * Once a lruvec is locked, folio_lruvec() can be called on other folios, and
+ * their binding is stable if the returned lruvec matches the one the caller has
+ * locked. Useful for lock batching.
  */
 static inline struct lruvec *folio_lruvec(struct folio *folio)
 {
@@ -758,15 +746,6 @@ struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
 						unsigned long *flags);
 
-#ifdef CONFIG_DEBUG_VM
-void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
-#else
-static inline
-void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
-{
-}
-#endif
-
 static inline
 struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
 	return css ? container_of(css, struct mem_cgroup, css) : NULL;
@@ -774,23 +753,26 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
 
 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
 {
+	if (obj_cgroup_is_root(objcg))
+		return true;
 	return percpu_ref_tryget(&objcg->refcnt);
 }
 
-static inline void obj_cgroup_get(struct obj_cgroup *objcg)
+static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
+				       unsigned long nr)
 {
-	percpu_ref_get(&objcg->refcnt);
+	if (!obj_cgroup_is_root(objcg))
+		percpu_ref_get_many(&objcg->refcnt, nr);
 }
 
-static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
-				       unsigned long nr)
+static inline void obj_cgroup_get(struct obj_cgroup *objcg)
 {
-	percpu_ref_get_many(&objcg->refcnt, nr);
+	obj_cgroup_get_many(objcg, 1);
 }
 
 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
 {
-	if (objcg)
+	if (objcg && !obj_cgroup_is_root(objcg))
 		percpu_ref_put(&objcg->refcnt);
 }
 
@@ -885,7 +867,7 @@ static inline bool mm_match_cgroup(struct mm_struct *mm,
 	return match;
 }
 
-struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
+struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio);
 ino_t page_cgroup_ino(struct page *page);
 
 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
@@ -966,10 +948,15 @@ void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
 static inline void count_memcg_folio_events(struct folio *folio,
 		enum vm_event_item idx, unsigned long nr)
 {
-	struct mem_cgroup *memcg = folio_memcg(folio);
+	struct mem_cgroup *memcg;
 
-	if (memcg)
-		count_memcg_events(memcg, idx, nr);
+	if (!folio_memcg_charged(folio))
+		return;
+
+	rcu_read_lock();
+	memcg = folio_memcg(folio);
+	count_memcg_events(memcg, idx, nr);
+	rcu_read_unlock();
 }
 
 static inline void count_memcg_events_mm(struct mm_struct *mm,
@@ -1087,6 +1074,11 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
 	return true;
 }
 
+static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg)
+{
+	return true;
+}
+
 static inline bool mem_cgroup_disabled(void)
 {
 	return true;
@@ -1179,11 +1171,6 @@ static inline struct lruvec *folio_lruvec(struct folio *folio)
 	return &pgdat->__lruvec;
 }
 
-static inline
-void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
-{
-}
-
 static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
 {
 	return NULL;
@@ -1242,6 +1229,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 
+	rcu_read_lock();
 	spin_lock(&pgdat->__lruvec.lru_lock);
 	return &pgdat->__lruvec;
 }
@@ -1250,6 +1238,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 
+	rcu_read_lock();
 	spin_lock_irq(&pgdat->__lruvec.lru_lock);
 	return &pgdat->__lruvec;
 }
@@ -1259,6 +1248,7 @@ static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 
+	rcu_read_lock();
 	spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
 	return &pgdat->__lruvec;
 }
@@ -1479,20 +1469,28 @@ static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
 	return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
 }
 
-static inline void unlock_page_lruvec(struct lruvec *lruvec)
+static inline void lruvec_lock_irq(struct lruvec *lruvec)
+{
+	rcu_read_lock();
+	spin_lock_irq(&lruvec->lru_lock);
+}
+
+static inline void lruvec_unlock(struct lruvec *lruvec)
 {
 	spin_unlock(&lruvec->lru_lock);
+	rcu_read_unlock();
 }
 
-static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
+static inline void lruvec_unlock_irq(struct lruvec *lruvec)
 {
 	spin_unlock_irq(&lruvec->lru_lock);
+	rcu_read_unlock();
 }
 
-static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
-		unsigned long flags)
+static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long flags)
 {
 	spin_unlock_irqrestore(&lruvec->lru_lock, flags);
+	rcu_read_unlock();
 }
 
 /* Test requires a stable folio->memcg binding, see folio_memcg() */
@@ -1511,7 +1509,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
 		if (folio_matches_lruvec(folio, locked_lruvec))
 			return locked_lruvec;
 
-		unlock_page_lruvec_irq(locked_lruvec);
+		lruvec_unlock_irq(locked_lruvec);
 	}
 
 	return folio_lruvec_lock_irq(folio);
@@ -1525,7 +1523,7 @@ static inline void folio_lruvec_relock_irqsave(struct folio *folio,
 		if (folio_matches_lruvec(folio, *lruvecp))
 			return;
 
-		unlock_page_lruvec_irqrestore(*lruvecp, *flags);
+		lruvec_unlock_irqrestore(*lruvecp, *flags);
 	}
 
 	*lruvecp = folio_lruvec_lock_irqsave(folio, flags);
@@ -1549,9 +1547,14 @@ static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
 	if (mem_cgroup_disabled())
 		return;
 
+	if (!folio_memcg_charged(folio))
+		return;
+
+	rcu_read_lock();
 	memcg = folio_memcg(folio);
-	if (unlikely(memcg && &memcg->css != wb->memcg_css))
+	if (unlikely(&memcg->css != wb->memcg_css))
 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);
+	rcu_read_unlock();
 }
 
 void mem_cgroup_flush_foreign(struct bdi_writeback *wb);
diff --git a/include/linux/memfd.h b/include/linux/memfd.h
index c328a7b356d0e..b4fda09dab9f6 100644
--- a/include/linux/memfd.h
+++ b/include/linux/memfd.h
@@ -18,6 +18,8 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx);
  */
 int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr);
 struct file *memfd_alloc_file(const char *name, unsigned int flags);
+int memfd_get_seals(struct file *file);
+int memfd_add_seals(struct file *file, unsigned int seals);
 #else
 static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a)
 {
@@ -37,6 +39,16 @@ static inline struct file *memfd_alloc_file(const char *name, unsigned int flags
 {
 	return ERR_PTR(-EINVAL);
 }
+
+static inline int memfd_get_seals(struct file *file)
+{
+	return -EINVAL;
+}
+
+static inline int memfd_add_seals(struct file *file, unsigned int seals)
+{
+	return -EINVAL;
+}
 #endif
 
 #endif /* __LINUX_MEMFD_H */
diff --git a/include/linux/memory.h b/include/linux/memory.h
index faeaa921e55b2..5bb5599c6b2bf 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -19,6 +19,7 @@
 #include <linux/node.h>
 #include <linux/compiler.h>
 #include <linux/mutex.h>
+#include <linux/memory_hotplug.h>
 
 #define MIN_MEMORY_BLOCK_SIZE     (1UL << SECTION_SIZE_BITS)
 
@@ -77,7 +78,7 @@ enum memory_block_state {
 struct memory_block {
 	unsigned long start_section_nr;
 	enum memory_block_state state;	/* serialized by the dev->lock */
-	int online_type;		/* for passing data to online routine */
+	enum mmop online_type;	/* for passing data to online routine */
 	int nid;			/* NID for this memory block */
 	/*
 	 * The single zone of this memory block if all PFNs of this memory block
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index f2f16cdd73ee1..e77ef3d7ff733 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -16,11 +16,8 @@ struct resource;
 struct vmem_altmap;
 struct dev_pagemap;
 
-#ifdef CONFIG_MEMORY_HOTPLUG
-struct page *pfn_to_online_page(unsigned long pfn);
-
 /* Types for control the zone type of onlined and offlined memory */
-enum {
+enum mmop {
 	/* Offline the memory. */
 	MMOP_OFFLINE = 0,
 	/* Online the memory. Zone depends, see default_zone_for_pfn(). */
@@ -31,6 +28,9 @@ enum {
 	MMOP_ONLINE_MOVABLE,
 };
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+struct page *pfn_to_online_page(unsigned long pfn);
+
 /* Flags for add_memory() and friends to specify memory hotplug details. */
 typedef int __bitwise mhp_t;
 
@@ -286,8 +286,8 @@ static inline void __remove_memory(u64 start, u64 size) {}
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 /* Default online_type (MMOP_*) when new memory blocks are added. */
-extern int mhp_get_default_online_type(void);
-extern void mhp_set_default_online_type(int online_type);
+extern enum mmop mhp_get_default_online_type(void);
+extern void mhp_set_default_online_type(enum mmop online_type);
 extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat);
 extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
 extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
@@ -310,8 +310,8 @@ extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
 				  struct vmem_altmap *altmap);
 extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,
 					  unsigned long pnum);
-extern struct zone *zone_for_pfn_range(int online_type, int nid,
-		struct memory_group *group, unsigned long start_pfn,
+extern struct zone *zone_for_pfn_range(enum mmop online_type,
+		int nid, struct memory_group *group, unsigned long start_pfn,
 		unsigned long nr_pages);
 extern int arch_create_linear_mapping(int nid, u64 start, u64 size,
 				      struct mhp_params *params);
diff --git a/include/linux/mfd/arizona/pdata.h b/include/linux/mfd/arizona/pdata.h
index f72e6d4b14a78..d465dcd8c90aa 100644
--- a/include/linux/mfd/arizona/pdata.h
+++ b/include/linux/mfd/arizona/pdata.h
@@ -117,11 +117,6 @@ struct arizona_pdata {
 	/** Check for line output with HPDET method */
 	bool hpdet_acc_id_line;
 
-#ifdef CONFIG_GPIOLIB_LEGACY
-	/** GPIO used for mic isolation with HPDET */
-	int hpdet_id_gpio;
-#endif
-
 	/** Channel to use for headphone detection */
 	unsigned int hpdet_channel;
 
@@ -131,11 +126,6 @@ struct arizona_pdata {
 	/** Extra debounce timeout used during initial mic detection (ms) */
 	unsigned int micd_detect_debounce;
 
-#ifdef CONFIG_GPIOLIB_LEGACY
-	/** GPIO for mic detection polarity */
-	int micd_pol_gpio;
-#endif
-
 	/** Mic detect ramp rate */
 	unsigned int micd_bias_start_time;
 
diff --git a/include/linux/mfd/atmel-hlcdc.h b/include/linux/mfd/atmel-hlcdc.h
index 80d675a03b395..07c2081867fd6 100644
--- a/include/linux/mfd/atmel-hlcdc.h
+++ b/include/linux/mfd/atmel-hlcdc.h
@@ -75,6 +75,7 @@
  */
 struct atmel_hlcdc {
 	struct regmap *regmap;
+	struct clk *lvds_pll_clk;
 	struct clk *periph_clk;
 	struct clk *sys_clk;
 	struct clk *slow_clk;
diff --git a/include/linux/mfd/cgbc.h b/include/linux/mfd/cgbc.h
index badbec4c70337..91f501e76c8fb 100644
--- a/include/linux/mfd/cgbc.h
+++ b/include/linux/mfd/cgbc.h
@@ -26,8 +26,8 @@ struct cgbc_version {
  * @io_cmd:		Pointer to the command IO memory
  * @session:		Session id returned by the Board Controller
  * @dev:		Pointer to kernel device structure
- * @cgbc_version:	Board Controller version structure
- * @mutex:		Board Controller mutex
+ * @version:		Board Controller version structure
+ * @lock:		Board Controller mutex
  */
 struct cgbc_device_data {
 	void __iomem		*io_session;
diff --git a/include/linux/mfd/cs42l43-regs.h b/include/linux/mfd/cs42l43-regs.h
index c39a49269cb7d..68831f113589d 100644
--- a/include/linux/mfd/cs42l43-regs.h
+++ b/include/linux/mfd/cs42l43-regs.h
@@ -1181,4 +1181,80 @@
 /* CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_REG */
 #define CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_DISABLE_VAL		0xF05AA50F
 
+/* CS42L43B VARIANT REGISTERS */
+#define CS42L43B_DEVID_VAL					0x0042A43B
+
+#define CS42L43B_DECIM_VOL_CTRL_CH1_CH2				0x00008280
+#define CS42L43B_DECIM_VOL_CTRL_CH3_CH4				0x00008284
+
+#define CS42L43B_DECIM_VOL_CTRL_CH5_CH6				0x00008290
+#define CS42L43B_DECIM_VOL_CTRL_UPDATE				0x0000829C
+
+#define CS42L43B_DECIM_HPF_WNF_CTRL5				0x000082A0
+#define CS42L43B_DECIM_HPF_WNF_CTRL6				0x000082A4
+
+#define CS42L43B_SWIRE_DP3_CH3_INPUT				0x0000C320
+#define CS42L43B_SWIRE_DP3_CH4_INPUT				0x0000C330
+#define CS42L43B_SWIRE_DP4_CH3_INPUT				0x0000C340
+#define CS42L43B_SWIRE_DP4_CH4_INPUT				0x0000C350
+
+#define CS42L43B_ISRC1DEC3_INPUT1				0x0000C780
+#define CS42L43B_ISRC1DEC4_INPUT1				0x0000C790
+#define CS42L43B_ISRC2DEC3_INPUT1				0x0000C7A0
+#define CS42L43B_ISRC2DEC4_INPUT1				0x0000C7B0
+
+#define CS42L43B_FW_MISSION_CTRL_NEED_CONFIGS			0x00117E00
+#define CS42L43B_FW_MISSION_CTRL_HAVE_CONFIGS			0x00117E04
+#define CS42L43B_FW_MISSION_CTRL_PATCH_START_ADDR_REG		0x00117E08
+#define CS42L43B_FW_MISSION_CTRL_MM_CTRL_SELECTION		0x00117E0C
+#define CS42L43B_FW_MISSION_CTRL_MM_MCU_CFG_REG			0x00117E10
+
+#define CS42L43B_MCU_SW_REV					0x00117314
+#define CS42L43B_PATCH_START_ADDR				0x00117318
+#define CS42L43B_CONFIG_SELECTION				0x0011731C
+#define CS42L43B_NEED_CONFIGS					0x00117320
+#define CS42L43B_BOOT_STATUS					0x00117330
+
+#define CS42L43B_FW_MISSION_CTRL_NEED_CONFIGS			0x00117E00
+#define CS42L43B_FW_MISSION_CTRL_HAVE_CONFIGS			0x00117E04
+#define CS42L43B_FW_MISSION_CTRL_PATCH_START_ADDR_REG		0x00117E08
+#define CS42L43B_FW_MISSION_CTRL_MM_CTRL_SELECTION		0x00117E0C
+#define CS42L43B_FW_MISSION_CTRL_MM_MCU_CFG_REG			0x00117E10
+
+#define CS42L43B_MCU_RAM_MAX					0x00117FFF
+
+/* CS42L43B_DECIM_DECIM_VOL_CTRL_CH5_CH6 */
+#define CS42L43B_DECIM6_MUTE_MASK				0x80000000
+#define CS42L43B_DECIM6_MUTE_SHIFT				31
+#define CS42L43B_DECIM6_VOL_MASK				0x3FC00000
+#define CS42L43B_DECIM6_VOL_SHIFT				22
+#define CS42L43B_DECIM6_PATH1_VOL_FALL_RATE_MASK		0x00380000
+#define CS42L43B_DECIM6_PATH1_VOL_FALL_RATE_SHIFT		19
+#define CS42L43B_DECIM6_PATH1_VOL_RISE_RATE_MASK		0x00070000
+#define CS42L43B_DECIM6_PATH1_VOL_RISE_RATE_SHIFT		16
+#define CS42L43B_DECIM5_MUTE_MASK				0x00008000
+#define CS42L43B_DECIM5_MUTE_SHIFT				15
+#define CS42L43B_DECIM5_VOL_MASK				0x00003FC0
+#define CS42L43B_DECIM5_VOL_SHIFT				6
+#define CS42L43B_DECIM5_PATH1_VOL_FALL_RATE_MASK		0x00000038
+#define CS42L43B_DECIM5_PATH1_VOL_FALL_RATE_SHIFT		3
+#define CS42L43B_DECIM5_PATH1_VOL_RISE_RATE_MASK		0x00000007
+#define CS42L43B_DECIM5_PATH1_VOL_RISE_RATE_SHIFT		0
+
+/* CS42L43B_DECIM_VOL_CTRL_UPDATE */
+#define CS42L43B_DECIM6_PATH1_VOL_TRIG_MASK			0x00000800
+#define CS42L43B_DECIM6_PATH1_VOL_TRIG_SHIFT			11
+#define CS42L43B_DECIM5_PATH1_VOL_TRIG_MASK			0x00000100
+#define CS42L43B_DECIM5_PATH1_VOL_TRIG_SHIFT			8
+#define CS42L43B_DECIM4_VOL_UPDATE_MASK				0x00000020
+#define CS42L43B_DECIM4_VOL_UPDATE_SHIFT			5
+
+/* CS42L43_ISRC1_CTRL..CS42L43_ISRC2_CTRL */
+#define CS42L43B_ISRC_DEC4_EN_MASK				0x00000008
+#define CS42L43B_ISRC_DEC4_EN_SHIFT				3
+#define CS42L43B_ISRC_DEC4_EN_WIDTH				1
+#define CS42L43B_ISRC_DEC3_EN_MASK				0x00000004
+#define CS42L43B_ISRC_DEC3_EN_SHIFT				2
+#define CS42L43B_ISRC_DEC3_EN_WIDTH				1
+
 #endif /* CS42L43_CORE_REGS_H */
diff --git a/include/linux/mfd/cs42l43.h b/include/linux/mfd/cs42l43.h
index 2239d8585e785..ff0f7e365a19f 100644
--- a/include/linux/mfd/cs42l43.h
+++ b/include/linux/mfd/cs42l43.h
@@ -98,6 +98,7 @@ struct cs42l43 {
 	bool sdw_pll_active;
 	bool attached;
 	bool hw_lock;
+	long variant_id;
 };
 
 #endif /* CS42L43_CORE_EXT_H */
diff --git a/include/linux/mfd/kempld.h b/include/linux/mfd/kempld.h
index 643c096b93ace..5d75071eaaea0 100644
--- a/include/linux/mfd/kempld.h
+++ b/include/linux/mfd/kempld.h
@@ -37,6 +37,7 @@
 #define KEMPLD_SPEC_GET_MINOR(x)	(x & 0x0f)
 #define KEMPLD_SPEC_GET_MAJOR(x)	((x >> 4) & 0x0f)
 #define KEMPLD_IRQ_GPIO			0x35
+#define KEMPLD_IRQ_GPIO_MASK		0x0f
 #define KEMPLD_IRQ_I2C			0x36
 #define KEMPLD_CFG			0x37
 #define KEMPLD_CFG_GPIO_I2C_MUX		(1 << 0)
@@ -97,10 +98,10 @@ struct kempld_device_data {
 /**
  * struct kempld_platform_data - PLD hardware configuration structure
  * @pld_clock:			PLD clock frequency
- * @gpio_base			GPIO base pin number
+ * @gpio_base:			GPIO base pin number
  * @ioresource:			IO addresses of the PLD
- * @get_mutex:			PLD specific get_mutex callback
- * @release_mutex:		PLD specific release_mutex callback
+ * @get_hardware_mutex:		PLD specific get_mutex callback
+ * @release_hardware_mutex:	PLD specific release_mutex callback
  * @get_info:			PLD specific get_info callback
  * @register_cells:		PLD specific register_cells callback
  */
diff --git a/include/linux/mfd/lpc_ich.h b/include/linux/mfd/lpc_ich.h
index 1fbda1f8967db..1819aa743c5c9 100644
--- a/include/linux/mfd/lpc_ich.h
+++ b/include/linux/mfd/lpc_ich.h
@@ -37,4 +37,6 @@ struct lpc_ich_info {
 	u8 use_gpio;
 };
 
+extern const struct software_node lpc_ich_gpio_swnode;
+
 #endif
diff --git a/include/linux/mfd/mt6397/core.h b/include/linux/mfd/mt6397/core.h
index b774c3a4bb62e..340fc72e22aa6 100644
--- a/include/linux/mfd/mt6397/core.h
+++ b/include/linux/mfd/mt6397/core.h
@@ -12,9 +12,9 @@
 
 enum chip_id {
 	MT6323_CHIP_ID = 0x23,
-	MT6328_CHIP_ID = 0x30,
-	MT6331_CHIP_ID = 0x20,
-	MT6332_CHIP_ID = 0x20,
+	MT6328_CHIP_ID = 0x28,
+	MT6331_CHIP_ID = 0x31,
+	MT6332_CHIP_ID = 0x32,
 	MT6357_CHIP_ID = 0x57,
 	MT6358_CHIP_ID = 0x58,
 	MT6359_CHIP_ID = 0x59,
diff --git a/include/linux/mfd/rsmu.h b/include/linux/mfd/rsmu.h
index 0379aa2074287..2f27386a7122a 100644
--- a/include/linux/mfd/rsmu.h
+++ b/include/linux/mfd/rsmu.h
@@ -19,7 +19,6 @@ enum rsmu_type {
 };
 
 /**
- *
  * struct rsmu_ddata - device data structure for sub devices.
  *
  * @dev:    i2c/spi device.
diff --git a/include/linux/mfd/si476x-core.h b/include/linux/mfd/si476x-core.h
index dd95c37ca1344..e913b2cdf77d5 100644
--- a/include/linux/mfd/si476x-core.h
+++ b/include/linux/mfd/si476x-core.h
@@ -77,6 +77,7 @@ enum si476x_power_state {
  * underlying "core" device which all the MFD cell-devices use.
  *
  * @client: Actual I2C client used to transfer commands to the chip.
+ * @regmap: Regmap for accessing the device registers
  * @chip_id: Last digit of the chip model(E.g. "1" for SI4761)
  * @cells: MFD cell devices created by this driver.
  * @cmd_lock: Mutex used to serialize all the requests to the core
@@ -100,16 +101,18 @@ enum si476x_power_state {
  * @stc: Similar to @cts, but for the STC bit of the status value.
  * @power_up_parameters: Parameters used as argument for POWER_UP
  * command when the device is started.
- * @state: Current power state of the device.
- * @supplues: Structure containing handles to all power supplies used
+ * @power_state: Current power state of the device.
+ * @supplies: Structure containing handles to all power supplies used
  * by the device (NULL ones are ignored).
  * @gpio_reset: GPIO pin connectet to the RSTB pin of the chip.
  * @pinmux: Chip's configurable pins configuration.
  * @diversity_mode: Chips role when functioning in diversity mode.
+ * @is_alive: Chip is initialized and active.
  * @status_monitor: Polling worker used in polling use case scenarion
  * (when IRQ is not avalible).
  * @revision: Chip's running firmware revision number(Used for correct
  * command set support).
+ * @rds_fifo_depth: RDS FIFO size: 20 for IRQ mode or 5 for polling mode.
  */
 
 struct si476x_core {
@@ -166,6 +169,7 @@ static inline struct si476x_core *i2c_mfd_cell_to_core(struct device *dev)
 /**
  * si476x_core_lock() - lock the core device to get an exclusive access
  * to it.
+ * @core: Core device structure
  */
 static inline void si476x_core_lock(struct si476x_core *core)
 {
@@ -175,6 +179,7 @@ static inline void si476x_core_lock(struct si476x_core *core)
 /**
  * si476x_core_unlock() - unlock the core device to relinquish an
  * exclusive access to it.
+ * @core: Core device structure
  */
 static inline void si476x_core_unlock(struct si476x_core *core)
 {
@@ -246,9 +251,10 @@ static inline int si476x_to_v4l2(struct si476x_core *core, u16 freq)
  * struct si476x_func_info - structure containing result of the
  * FUNC_INFO command.
  *
+ * @firmware: Firmware version numbers.
  * @firmware.major: Firmware major number.
  * @firmware.minor[...]: Firmware minor numbers.
- * @patch_id:
+ * @patch_id: Firmware patch level.
  * @func: Mode tuner is working in.
  */
 struct si476x_func_info {
@@ -318,8 +324,9 @@ enum si476x_smoothmetrics {
  * @tp: Current channel's TP flag.
  * @pty: Current channel's PTY code.
  * @pi: Current channel's PI code.
- * @rdsfifoused: Number of blocks remaining in the RDS FIFO (0 if
- * empty).
+ * @rdsfifoused: Number of blocks remaining in the RDS FIFO (0 if empty).
+ * @ble:
+ * @rds: RDS data descriptor
  */
 struct si476x_rds_status_report {
 	bool rdstpptyint, rdspiint, rdssyncint, rdsfifoint;
diff --git a/include/linux/mhi.h b/include/linux/mhi.h
index 88ccb3e14f481..fb3ba639f4f87 100644
--- a/include/linux/mhi.h
+++ b/include/linux/mhi.h
@@ -85,17 +85,33 @@ enum mhi_ch_type {
 	MHI_CH_TYPE_INBOUND_COALESCED = 3,
 };
 
+/**
+ * struct mhi_buf - MHI Buffer description
+ * @buf: Virtual address of the buffer
+ * @name: Buffer label. For offload channel, configurations name must be:
+ *        ECA - Event context array data
+ *        CCA - Channel context array data
+ * @dma_addr: IOMMU address of the buffer
+ * @len: # of bytes
+ */
+struct mhi_buf {
+	void *buf;
+	const char *name;
+	dma_addr_t dma_addr;
+	size_t len;
+};
+
 /**
  * struct image_info - Firmware and RDDM table
  * @mhi_buf: Buffer for firmware and RDDM table
  * @entries: # of entries in table
  */
 struct image_info {
-	struct mhi_buf *mhi_buf;
 	/* private: from internal.h */
 	struct bhi_vec_entry *bhi_vec;
 	/* public: */
 	u32 entries;
+	struct mhi_buf mhi_buf[] __counted_by(entries);
 };
 
 /**
@@ -488,22 +504,6 @@ struct mhi_result {
 	int transaction_status;
 };
 
-/**
- * struct mhi_buf - MHI Buffer description
- * @buf: Virtual address of the buffer
- * @name: Buffer label. For offload channel, configurations name must be:
- *        ECA - Event context array data
- *        CCA - Channel context array data
- * @dma_addr: IOMMU address of the buffer
- * @len: # of bytes
- */
-struct mhi_buf {
-	void *buf;
-	const char *name;
-	dma_addr_t dma_addr;
-	size_t len;
-};
-
 /**
  * struct mhi_driver - Structure representing a MHI client driver
  * @probe: CB function for client driver probe function
diff --git a/include/linux/micrel_phy.h b/include/linux/micrel_phy.h
index ca691641788b8..9c6f9817383f3 100644
--- a/include/linux/micrel_phy.h
+++ b/include/linux/micrel_phy.h
@@ -33,6 +33,7 @@
 #define PHY_ID_LAN8804		0x00221670
 #define PHY_ID_LAN8841		0x00221650
 #define PHY_ID_LAN8842		0x002216C0
+#define PHY_ID_LAN9645X		0x002216D0
 
 #define PHY_ID_KSZ886X		0x00221430
 #define PHY_ID_KSZ8863		0x00221435
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index b37fe39cef273..07a25f2642921 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -293,6 +293,7 @@ enum {
 	MLX5_UMR_INLINE			= (1 << 7),
 };
 
+#define MLX5_UMR_ALIGN (2048)
 #define MLX5_UMR_FLEX_ALIGNMENT 0x40
 #define MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_mtt))
 #define MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_klm))
@@ -1259,6 +1260,7 @@ enum mlx5_cap_type {
 	MLX5_CAP_PORT_SELECTION = 0x25,
 	MLX5_CAP_ADV_VIRTUALIZATION = 0x26,
 	MLX5_CAP_ADV_RDMA = 0x28,
+	MLX5_CAP_TLP_EMULATION = 0x2a,
 	/* NUM OF CAP Types */
 	MLX5_CAP_NUM
 };
@@ -1481,6 +1483,14 @@ enum mlx5_qcam_feature_groups {
 	MLX5_GET64(virtio_emulation_cap, \
 		(mdev)->caps.hca[MLX5_CAP_VDPA_EMULATION]->cur, cap)
 
+#define MLX5_CAP_DEV_TLP_EMULATION(mdev, cap)\
+	MLX5_GET(tlp_dev_emu_capabilities, \
+		(mdev)->caps.hca[MLX5_CAP_TLP_EMULATION]->cur, cap)
+
+#define MLX5_CAP64_DEV_TLP_EMULATION(mdev, cap)\
+	MLX5_GET64(tlp_dev_emu_capabilities, \
+		(mdev)->caps.hca[MLX5_CAP_TLP_EMULATION]->cur, cap)
+
 #define MLX5_CAP_IPSEC(mdev, cap)\
 	MLX5_GET(ipsec_cap, (mdev)->caps.hca[MLX5_CAP_IPSEC]->cur, cap)
 
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 04dcd09f7517b..51b03a063becf 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -705,23 +705,12 @@ struct mlx5_st;
 
 enum {
 	MLX5_PROF_MASK_QP_SIZE		= (u64)1 << 0,
-	MLX5_PROF_MASK_MR_CACHE		= (u64)1 << 1,
-};
-
-enum {
-	MKEY_CACHE_LAST_STD_ENTRY = 20,
-	MLX5_IMR_KSM_CACHE_ENTRY,
-	MAX_MKEY_CACHE_ENTRIES
 };
 
 struct mlx5_profile {
 	u64	mask;
 	u8	log_max_qp;
 	u8	num_cmd_caches;
-	struct {
-		int	size;
-		int	limit;
-	} mr_cache[MAX_MKEY_CACHE_ENTRIES];
 };
 
 struct mlx5_hca_cap {
@@ -798,6 +787,7 @@ struct mlx5_core_dev {
 	enum mlx5_wc_state wc_state;
 	/* sync write combining state */
 	struct mutex wc_state_lock;
+	struct devlink *shd;
 };
 
 struct mlx5_db {
diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h
index 9cadb1d5e6df0..d8f3b7ef319e1 100644
--- a/include/linux/mlx5/fs.h
+++ b/include/linux/mlx5/fs.h
@@ -55,6 +55,7 @@ enum mlx5_flow_destination_type {
 	MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM,
 	MLX5_FLOW_DESTINATION_TYPE_RANGE,
 	MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE,
+	MLX5_FLOW_DESTINATION_TYPE_VHCA_RX,
 };
 
 enum {
@@ -189,6 +190,9 @@ struct mlx5_flow_destination {
 		u32			ft_num;
 		struct mlx5_flow_table	*ft;
 		struct mlx5_fc          *counter;
+		struct {
+			u16		id;
+		} vhca;
 		struct {
 			u16		num;
 			u16		vhca_id;
@@ -248,9 +252,9 @@ mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
 struct mlx5_flow_table *
 mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
 			     struct mlx5_flow_table_attr *ft_attr, u16 vport);
-struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
-					       struct mlx5_flow_namespace *ns,
-					       int prio, u32 level);
+struct mlx5_flow_table *
+mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
+				 struct mlx5_flow_table_attr *ft_attr);
 int mlx5_destroy_flow_table(struct mlx5_flow_table *ft);
 
 /* inbox should be set with the following values:
diff --git a/include/linux/mlx5/lag.h b/include/linux/mlx5/lag.h
new file mode 100644
index 0000000000000..ab9f754664e53
--- /dev/null
+++ b/include/linux/mlx5/lag.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_LAG_API_H__
+#define __MLX5_LAG_API_H__
+
+#include <linux/types.h>
+
+struct mlx5_core_dev;
+struct mlx5_flow_table;
+struct mlx5_flow_table_attr;
+
+int mlx5_lag_demux_init(struct mlx5_core_dev *dev,
+			struct mlx5_flow_table_attr *ft_attr);
+void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev);
+int mlx5_lag_demux_rule_add(struct mlx5_core_dev *dev, u16 vport_num,
+			    int vport_index);
+void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int vport_index);
+int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev);
+
+#endif /* __MLX5_LAG_API_H__ */
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 775cb0c56865f..8fa4fb3d36cfb 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -469,7 +469,8 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
 	u8	   table_miss_action_domain[0x1];
 	u8         termination_table[0x1];
 	u8         reformat_and_fwd_to_table[0x1];
-	u8         reserved_at_1a[0x2];
+	u8         forward_vhca_rx[0x1];
+	u8         reserved_at_1b[0x1];
 	u8         ipsec_encrypt[0x1];
 	u8         ipsec_decrypt[0x1];
 	u8         sw_owner_v2[0x1];
@@ -1389,6 +1390,26 @@ struct mlx5_ifc_virtio_emulation_cap_bits {
 	u8         reserved_at_1c0[0x640];
 };
 
+struct mlx5_ifc_tlp_dev_emu_capabilities_bits {
+	u8         reserved_at_0[0x20];
+
+	u8         reserved_at_20[0x13];
+	u8         log_tlp_rsp_gw_page_stride[0x5];
+	u8         reserved_at_38[0x8];
+
+	u8         reserved_at_40[0xc0];
+
+	u8         reserved_at_100[0xc];
+	u8         tlp_rsp_gw_num_pages[0x4];
+	u8         reserved_at_110[0x10];
+
+	u8         reserved_at_120[0xa0];
+
+	u8         tlp_rsp_gw_pages_bar_offset[0x40];
+
+	u8         reserved_at_200[0x600];
+};
+
 enum {
 	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE     = 0x0,
 	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES    = 0x2,
@@ -1961,7 +1982,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         log_max_rqt[0x5];
 	u8         reserved_at_390[0x3];
 	u8         log_max_rqt_size[0x5];
-	u8         reserved_at_398[0x1];
+	u8         tlp_device_emulation_manager[0x1];
 	u8	   vnic_env_cnt_bar_uar_access[0x1];
 	u8	   vnic_env_cnt_odp_page_fault[0x1];
 	u8         log_max_tis_per_sq[0x5];
@@ -1992,12 +2013,14 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         disable_local_lb_mc[0x1];
 	u8         log_min_hairpin_wq_data_sz[0x5];
 	u8         reserved_at_3e8[0x1];
-	u8         silent_mode[0x1];
+	u8         silent_mode_set[0x1];
 	u8         vhca_state[0x1];
 	u8         log_max_vlan_list[0x5];
 	u8         reserved_at_3f0[0x3];
 	u8         log_max_current_mc_list[0x5];
-	u8         reserved_at_3f8[0x3];
+	u8         reserved_at_3f8[0x1];
+	u8         silent_mode_query[0x1];
+	u8         reserved_at_3fa[0x1];
 	u8         log_max_current_uc_list[0x5];
 
 	u8         general_obj_types[0x40];
@@ -2259,6 +2282,7 @@ enum mlx5_ifc_flow_destination_type {
 	MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT        = 0x0,
 	MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE   = 0x1,
 	MLX5_IFC_FLOW_DESTINATION_TYPE_TIR          = 0x2,
+	MLX5_IFC_FLOW_DESTINATION_TYPE_VHCA_RX	    = 0x4,
 	MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6,
 	MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK       = 0x8,
 	MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE   = 0xA,
@@ -3830,6 +3854,7 @@ union mlx5_ifc_hca_cap_union_bits {
 	struct mlx5_ifc_tls_cap_bits tls_cap;
 	struct mlx5_ifc_device_mem_cap_bits device_mem_cap;
 	struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap;
+	struct mlx5_ifc_tlp_dev_emu_capabilities_bits tlp_dev_emu_capabilities;
 	struct mlx5_ifc_macsec_cap_bits macsec_cap;
 	struct mlx5_ifc_crypto_cap_bits crypto_cap;
 	struct mlx5_ifc_ipsec_cap_bits ipsec_cap;
@@ -6244,7 +6269,9 @@ struct mlx5_ifc_query_l2_table_entry_out_bits {
 
 	u8         reserved_at_40[0xa0];
 
-	u8         reserved_at_e0[0x13];
+	u8         reserved_at_e0[0x11];
+	u8         silent_mode[0x1];
+	u8         reserved_at_f2[0x1];
 	u8         vlan_valid[0x1];
 	u8         vlan[0xc];
 
@@ -6260,7 +6287,10 @@ struct mlx5_ifc_query_l2_table_entry_in_bits {
 	u8         reserved_at_20[0x10];
 	u8         op_mod[0x10];
 
-	u8         reserved_at_40[0x60];
+	u8         reserved_at_40[0x40];
+
+	u8         silent_mode_query[0x1];
+	u8         reserved_at_81[0x1f];
 
 	u8         reserved_at_a0[0x8];
 	u8         table_index[0x18];
@@ -10824,7 +10854,9 @@ struct mlx5_ifc_pcam_enhanced_features_bits {
 	u8         fec_200G_per_lane_in_pplm[0x1];
 	u8         reserved_at_1e[0x2a];
 	u8         fec_100G_per_lane_in_pplm[0x1];
-	u8         reserved_at_49[0xa];
+	u8         reserved_at_49[0x2];
+	u8         shp_pbmc_pbsr_support[0x1];
+	u8         reserved_at_4c[0x7];
 	u8	   buffer_ownership[0x1];
 	u8	   resereved_at_54[0x14];
 	u8         fec_50G_per_lane_in_pplm[0x1];
@@ -12069,8 +12101,9 @@ struct mlx5_ifc_pbmc_reg_bits {
 	u8         port_buffer_size[0x10];
 
 	struct mlx5_ifc_bufferx_reg_bits buffer[10];
+	struct mlx5_ifc_bufferx_reg_bits shared_headroom_pool;
 
-	u8         reserved_at_2e0[0x80];
+	u8         reserved_at_320[0x40];
 };
 
 struct mlx5_ifc_sbpr_reg_bits {
diff --git a/include/linux/mm.h b/include/linux/mm.h
index abb4963c1f064..8aadf115278e1 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -27,7 +27,6 @@
 #include <linux/page-flags.h>
 #include <linux/page_ref.h>
 #include <linux/overflow.h>
-#include <linux/sizes.h>
 #include <linux/sched.h>
 #include <linux/pgtable.h>
 #include <linux/kasan.h>
@@ -208,8 +207,6 @@ static inline void __mm_zero_struct_page(struct page *page)
 #define MAPCOUNT_ELF_CORE_MARGIN	(5)
 #define DEFAULT_MAX_MAP_COUNT	(USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
 
-extern int sysctl_max_map_count;
-
 extern unsigned long sysctl_user_reserve_kbytes;
 extern unsigned long sysctl_admin_reserve_kbytes;
 
@@ -349,9 +346,9 @@ enum {
 	 * if KVM does not lock down the memory type.
 	 */
 	DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39),
-#ifdef CONFIG_PPC32
+#if defined(CONFIG_PPC32)
 	DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1),
-#else
+#elif defined(CONFIG_64BIT)
 	DECLARE_VMA_BIT(DROPPABLE, 40),
 #endif
 	DECLARE_VMA_BIT(UFFD_MINOR, 41),
@@ -466,8 +463,10 @@ enum {
 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \
 	defined(CONFIG_RISCV_USER_CFI)
 #define VM_SHADOW_STACK	INIT_VM_FLAG(SHADOW_STACK)
+#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)
 #else
 #define VM_SHADOW_STACK	VM_NONE
+#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)
 #endif
 #if defined(CONFIG_PPC64)
 #define VM_SAO		INIT_VM_FLAG(SAO)
@@ -506,32 +505,41 @@ enum {
 #endif
 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
 #define VM_DROPPABLE		INIT_VM_FLAG(DROPPABLE)
+#define VMA_DROPPABLE		mk_vma_flags(VMA_DROPPABLE_BIT)
 #else
 #define VM_DROPPABLE		VM_NONE
+#define VMA_DROPPABLE		EMPTY_VMA_FLAGS
 #endif
 
 /* Bits set in the VMA until the stack is in its final location */
 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
 
-#define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0)
+#define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \
+		       VMA_EXEC_BIT : VMA_READ_BIT)
 
 /* Common data flag combinations */
-#define VM_DATA_FLAGS_TSK_EXEC	(VM_READ | VM_WRITE | TASK_EXEC | \
-				 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
-#define VM_DATA_FLAGS_NON_EXEC	(VM_READ | VM_WRITE | VM_MAYREAD | \
-				 VM_MAYWRITE | VM_MAYEXEC)
-#define VM_DATA_FLAGS_EXEC	(VM_READ | VM_WRITE | VM_EXEC | \
-				 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
-
-#ifndef VM_DATA_DEFAULT_FLAGS		/* arch can override this */
-#define VM_DATA_DEFAULT_FLAGS  VM_DATA_FLAGS_EXEC
+#define VMA_DATA_FLAGS_TSK_EXEC	mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \
+		TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT,	  \
+		VMA_MAYEXEC_BIT)
+#define VMA_DATA_FLAGS_NON_EXEC	mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \
+		VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT)
+#define VMA_DATA_FLAGS_EXEC	mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \
+		VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT,	  \
+		VMA_MAYEXEC_BIT)
+
+#ifndef VMA_DATA_DEFAULT_FLAGS		/* arch can override this */
+#define VMA_DATA_DEFAULT_FLAGS  VMA_DATA_FLAGS_EXEC
 #endif
 
-#ifndef VM_STACK_DEFAULT_FLAGS		/* arch can override this */
-#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
+#ifndef VMA_STACK_DEFAULT_FLAGS		/* arch can override this */
+#define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS
 #endif
 
-#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
+#define VMA_STACK_FLAGS	append_vma_flags(VMA_STACK_DEFAULT_FLAGS,	\
+		VMA_STACK_BIT, VMA_ACCOUNT_BIT)
+
+/* Temporary until VMA flags conversion complete. */
+#define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS)
 
 #ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS
 #define VM_SEALED_SYSMAP	VM_SEALED
@@ -539,16 +547,18 @@ enum {
 #define VM_SEALED_SYSMAP	VM_NONE
 #endif
 
-#define VM_STACK_FLAGS	(VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
-
 /* VMA basic access permission flags */
 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
+#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
 
 /*
  * Special vmas that are non-mergable, non-mlock()able.
  */
 #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
 
+#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \
+				       VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)
+
 /*
  * Physically remapped pages are special. Tell the
  * rest of the world about it:
@@ -574,6 +584,8 @@ enum {
 /* This mask represents all the VMA flag bits used by mlock */
 #define VM_LOCKED_MASK	(VM_LOCKED | VM_LOCKONFAULT)
 
+#define VMA_LOCKED_MASK	mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT)
+
 /* These flags can be updated atomically via VMA/mmap read lock. */
 #define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
 
@@ -588,27 +600,32 @@ enum {
  * possesses it but the other does not, the merged VMA should nonetheless have
  * applied to it:
  *
- *   VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its
- *                  references cleared via /proc/$pid/clear_refs, any merged VMA
- *                  should be considered soft-dirty also as it operates at a VMA
- *                  granularity.
+ *   VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its
+ *                       references cleared via /proc/$pid/clear_refs, any
+ *                       merged VMA should be considered soft-dirty also as it
+ *                       operates at a VMA granularity.
  *
- * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that
- *                  mapped page tables may contain metadata not described by the
- *                  VMA and thus any merged VMA may also contain this metadata,
- *                  and thus we must make this flag sticky.
+ * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies
+ *                       that mapped page tables may contain metadata not
+ *                       described by the VMA and thus any merged VMA may also
+ *                       contain this metadata, and thus we must make this flag
+ *                       sticky.
  */
-#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
+#ifdef CONFIG_MEM_SOFT_DIRTY
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
+#else
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
+#endif
 
 /*
  * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one
  * of these flags and the other not does not preclude a merge.
  *
- *    VM_STICKY - When merging VMAs, VMA flags must match, unless they are
- *                'sticky'. If any sticky flags exist in either VMA, we simply
- *                set all of them on the merged VMA.
+ *    VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they
+ *                       are 'sticky'. If any sticky flags exist in either VMA,
+ *                       we simply set all of them on the merged VMA.
  */
-#define VM_IGNORE_MERGE VM_STICKY
+#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS
 
 /*
  * Flags which should result in page tables being copied on fork. These are
@@ -741,21 +758,45 @@ struct vm_fault {
 					 */
 };
 
+struct vm_uffd_ops;
+
 /*
  * These are the virtual MM functions - opening of an area, closing and
  * unmapping it (needed to keep files on disk up-to-date etc), pointer
  * to the functions called when a no-page or a wp-page exception occurs.
  */
 struct vm_operations_struct {
-	void (*open)(struct vm_area_struct * area);
+	/**
+	 * @open: Called when a VMA is remapped, split or forked. Not called
+	 * upon first mapping a VMA.
+	 * Context: User context.  May sleep.  Caller holds mmap_lock.
+	 */
+	void (*open)(struct vm_area_struct *vma);
 	/**
 	 * @close: Called when the VMA is being removed from the MM.
 	 * Context: User context.  May sleep.  Caller holds mmap_lock.
 	 */
-	void (*close)(struct vm_area_struct * area);
+	void (*close)(struct vm_area_struct *vma);
+	/**
+	 * @mapped: Called when the VMA is first mapped in the MM. Not called if
+	 * the new VMA is merged with an adjacent VMA.
+	 *
+	 * The @vm_private_data field is an output field allowing the user to
+	 * modify vma->vm_private_data as necessary.
+	 *
+	 * ONLY valid if set from f_op->mmap_prepare. Will result in an error if
+	 * set from f_op->mmap.
+	 *
+	 * Returns %0 on success, or an error otherwise. On error, the VMA will
+	 * be unmapped.
+	 *
+	 * Context: User context.  May sleep.  Caller holds mmap_lock.
+	 */
+	int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff,
+		      const struct file *file, void **vm_private_data);
 	/* Called any time before splitting to check if it's allowed */
-	int (*may_split)(struct vm_area_struct *area, unsigned long addr);
-	int (*mremap)(struct vm_area_struct *area);
+	int (*may_split)(struct vm_area_struct *vma, unsigned long addr);
+	int (*mremap)(struct vm_area_struct *vma);
 	/*
 	 * Called by mprotect() to make driver-specific permission
 	 * checks before mprotect() is finalised.   The VMA must not
@@ -767,7 +808,7 @@ struct vm_operations_struct {
 	vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order);
 	vm_fault_t (*map_pages)(struct vm_fault *vmf,
 			pgoff_t start_pgoff, pgoff_t end_pgoff);
-	unsigned long (*pagesize)(struct vm_area_struct * area);
+	unsigned long (*pagesize)(struct vm_area_struct *vma);
 
 	/* notification that a previously read-only page is about to become
 	 * writable, if an error is returned it will cause a SIGBUS */
@@ -826,6 +867,9 @@ struct vm_operations_struct {
 	struct page *(*find_normal_page)(struct vm_area_struct *vma,
 					 unsigned long addr);
 #endif /* CONFIG_FIND_NORMAL_PAGE */
+#ifdef CONFIG_USERFAULTFD
+	const struct vm_uffd_ops *uffd_ops;
+#endif
 };
 
 #ifdef CONFIG_NUMA_BALANCING
@@ -991,7 +1035,8 @@ static inline void vm_flags_mod(struct vm_area_struct *vma,
 	__vm_flags_mod(vma, set, clear);
 }
 
-static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit)
+static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma,
+		vma_flag_t bit)
 {
 	const vm_flags_t mask = BIT((__force int)bit);
 
@@ -1006,7 +1051,8 @@ static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_
  * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific
  * valid flags are allowed to do this.
  */
-static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
+static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma,
+		vma_flag_t bit)
 {
 	unsigned long *bitmap = vma->flags.__vma_flags;
 
@@ -1022,7 +1068,8 @@ static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bi
  * This is necessarily racey, so callers must ensure that serialisation is
  * achieved through some other means, or that races are permissible.
  */
-static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
+static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma,
+		vma_flag_t bit)
 {
 	if (__vma_atomic_valid_flag(vma, bit))
 		return test_bit((__force int)bit, &vma->vm_flags);
@@ -1031,21 +1078,21 @@ static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t b
 }
 
 /* Set an individual VMA flag in flags, non-atomically. */
-static inline void vma_flag_set(vma_flags_t *flags, vma_flag_t bit)
+static __always_inline void vma_flags_set_flag(vma_flags_t *flags,
+		vma_flag_t bit)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 
 	__set_bit((__force int)bit, bitmap);
 }
 
-static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
+static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags,
+		size_t count, const vma_flag_t *bits)
 {
-	vma_flags_t flags;
 	int i;
 
-	vma_flags_clear_all(&flags);
 	for (i = 0; i < count; i++)
-		vma_flag_set(&flags, bits[i]);
+		vma_flags_set_flag(&flags, bits[i]);
 	return flags;
 }
 
@@ -1054,16 +1101,73 @@ static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
  * vma_flags_t bitmap value. E.g.:
  *
  * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT,
- * 		VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
+ *              VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
  *
  * The compiler cleverly optimises away all of the work and this ends up being
  * equivalent to aggregating the values manually.
  */
-#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
-					 (const vma_flag_t []){__VA_ARGS__})
+#define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS,			\
+		COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
+
+/*
+ * Helper macro which acts like mk_vma_flags, only appending to a copy of the
+ * specified flags rather than establishing new flags. E.g.:
+ *
+ * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT,
+ *              VMA_ACCOUNT_BIT);
+ */
+#define append_vma_flags(flags, ...) __mk_vma_flags(flags,			\
+		COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
+
+/* Calculates the number of set bits in the specified VMA flags. */
+static __always_inline int vma_flags_count(const vma_flags_t *flags)
+{
+	const unsigned long *bitmap = flags->__vma_flags;
+
+	return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS);
+}
+
+/*
+ * Test whether a specific VMA flag is set, e.g.:
+ *
+ * if (vma_flags_test(flags, VMA_READ_BIT)) { ... }
+ */
+static __always_inline bool vma_flags_test(const vma_flags_t *flags,
+		vma_flag_t bit)
+{
+	const unsigned long *bitmap = flags->__vma_flags;
+
+	return test_bit((__force int)bit, bitmap);
+}
+
+/*
+ * Obtain a set of VMA flags which contain the overlapping flags contained
+ * within flags and to_and.
+ */
+static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags,
+						      vma_flags_t to_and)
+{
+	vma_flags_t dst;
+	unsigned long *bitmap_dst = dst.__vma_flags;
+	const unsigned long *bitmap = flags->__vma_flags;
+	const unsigned long *bitmap_to_and = to_and.__vma_flags;
+
+	bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS);
+	return dst;
+}
+
+/*
+ * Obtain a set of VMA flags which contains the specified overlapping flags,
+ * e.g.:
+ *
+ * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT,
+ *                                        VMA_MAY_READ_BIT);
+ */
+#define vma_flags_and(flags, ...)				\
+	vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__))
 
 /*  Test each of to_test flags in flags, non-atomically. */
-static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
+static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags,
 		vma_flags_t to_test)
 {
 	const unsigned long *bitmap = flags->__vma_flags;
@@ -1075,10 +1179,10 @@ static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
 /*
  * Test whether any specified VMA flag is set, e.g.:
  *
- * if (vma_flags_test(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
+ * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
  */
-#define vma_flags_test(flags, ...) \
-	vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
+#define vma_flags_test_any(flags, ...) \
+	vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__))
 
 /* Test that ALL of the to_test flags are set, non-atomically. */
 static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
@@ -1098,8 +1202,29 @@ static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
 #define vma_flags_test_all(flags, ...) \
 	vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
 
+/*
+ * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set
+ * (returning false if flagmask has no flags set).
+ *
+ * This is defined to make the semantics clearer when testing an optionally
+ * defined VMA flags mask, e.g.:
+ *
+ * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... }
+ *
+ * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS
+ * otherwise.
+ */
+static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags,
+		vma_flags_t flagmask)
+{
+	VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1);
+
+	return vma_flags_test_any_mask(flags, flagmask);
+}
+
 /* Set each of the to_set flags in flags, non-atomically. */
-static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
+static __always_inline void vma_flags_set_mask(vma_flags_t *flags,
+		vma_flags_t to_set)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 	const unsigned long *bitmap_to_set = to_set.__vma_flags;
@@ -1116,7 +1241,8 @@ static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t t
 	vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
 
 /* Clear all of the to-clear flags in flags, non-atomically. */
-static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
+static __always_inline void vma_flags_clear_mask(vma_flags_t *flags,
+		vma_flags_t to_clear)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 	const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
@@ -1132,14 +1258,86 @@ static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t
 #define vma_flags_clear(flags, ...) \
 	vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
 
+/*
+ * Obtain a VMA flags value containing those flags that are present in flags or
+ * flags_other but not in both.
+ */
+static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags,
+		const vma_flags_t *flags_other)
+{
+	vma_flags_t dst;
+	const unsigned long *bitmap_other = flags_other->__vma_flags;
+	const unsigned long *bitmap = flags->__vma_flags;
+	unsigned long *bitmap_dst = dst.__vma_flags;
+
+	bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS);
+	return dst;
+}
+
+/* Determine if flags and flags_other have precisely the same flags set. */
+static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags,
+						const vma_flags_t *flags_other)
+{
+	const unsigned long *bitmap = flags->__vma_flags;
+	const unsigned long *bitmap_other = flags_other->__vma_flags;
+
+	return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);
+}
+
+/* Determine if flags and flags_other have precisely the same flags set.  */
+static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags,
+						vma_flags_t flags_other)
+{
+	const unsigned long *bitmap = flags->__vma_flags;
+	const unsigned long *bitmap_other = flags_other.__vma_flags;
+
+	return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);
+}
+
+/*
+ * Helper macro to determine if only the specific flags are set, e.g.:
+ *
+ * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... }
+ */
+#define vma_flags_same(flags, ...) \
+	vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
+
+/*
+ * Test whether a specific flag in the VMA is set, e.g.:
+ *
+ * if (vma_test(vma, VMA_READ_BIT)) { ... }
+ */
+static __always_inline bool vma_test(const struct vm_area_struct *vma,
+		vma_flag_t bit)
+{
+	return vma_flags_test(&vma->flags, bit);
+}
+
+/* Helper to test any VMA flags in a VMA . */
+static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma,
+		vma_flags_t flags)
+{
+	return vma_flags_test_any_mask(&vma->flags, flags);
+}
+
+/*
+ * Helper macro for testing whether any VMA flags are set in a VMA,
+ * e.g.:
+ *
+ * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT,
+ *		VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
+ */
+#define vma_test_any(vma, ...) \
+	vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__))
+
 /*
  * Helper to test that ALL specified flags are set in a VMA.
  *
  * Note: appropriate locks must be held, this function does not acquire them for
  * you.
  */
-static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
-					   vma_flags_t flags)
+static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma,
+		vma_flags_t flags)
 {
 	return vma_flags_test_all_mask(&vma->flags, flags);
 }
@@ -1147,10 +1345,28 @@ static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
 /*
  * Helper macro for checking that ALL specified flags are set in a VMA, e.g.:
  *
- * if (vma_test_all_flags(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... }
+ * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... }
  */
-#define vma_test_all_flags(vma, ...) \
-	vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
+#define vma_test_all(vma, ...) \
+	vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__))
+
+/*
+ * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set
+ * (returning false if flagmask has no flags set).
+ *
+ * This is useful when a flag needs to be either defined or not depending upon
+ * kernel configuration, e.g.:
+ *
+ * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... }
+ *
+ * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS
+ * otherwise.
+ */
+static __always_inline bool
+vma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask)
+{
+	return vma_flags_test_single_mask(&vma->flags, flagmask);
+}
 
 /*
  * Helper to set all VMA flags in a VMA.
@@ -1158,8 +1374,8 @@ static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
  * Note: appropriate locks must be held, this function does not acquire them for
  * you.
  */
-static inline void vma_set_flags_mask(struct vm_area_struct *vma,
-				      vma_flags_t flags)
+static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma,
+		vma_flags_t flags)
 {
 	vma_flags_set_mask(&vma->flags, flags);
 }
@@ -1176,26 +1392,69 @@ static inline void vma_set_flags_mask(struct vm_area_struct *vma,
 #define vma_set_flags(vma, ...) \
 	vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
 
-/* Helper to test all VMA flags in a VMA descriptor. */
-static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
-					    vma_flags_t flags)
+/* Helper to clear all VMA flags in a VMA. */
+static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma,
+		vma_flags_t flags)
 {
-	return vma_flags_test_mask(&desc->vma_flags, flags);
+	vma_flags_clear_mask(&vma->flags, flags);
 }
 
 /*
- * Helper macro for testing VMA flags for an input pointer to a struct
- * vm_area_desc object describing a proposed VMA, e.g.:
+ * Helper macro for clearing VMA flags, e.g.:
  *
- * if (vma_desc_test_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT,
+ * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
+ * 		VMA_DONTDUMP_BIT);
+ */
+#define vma_clear_flags(vma, ...) \
+	vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
+
+/*
+ * Test whether a specific VMA flag is set in a VMA descriptor, e.g.:
+ *
+ * if (vma_desc_test(desc, VMA_READ_BIT)) { ... }
+ */
+static __always_inline bool vma_desc_test(const struct vm_area_desc *desc,
+		vma_flag_t bit)
+{
+	return vma_flags_test(&desc->vma_flags, bit);
+}
+
+/* Helper to test any VMA flags in a VMA descriptor. */
+static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc,
+		vma_flags_t flags)
+{
+	return vma_flags_test_any_mask(&desc->vma_flags, flags);
+}
+
+/*
+ * Helper macro for testing whether any VMA flags are set in a VMA descriptor,
+ * e.g.:
+ *
+ * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT,
  *		VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
  */
-#define vma_desc_test_flags(desc, ...) \
-	vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
+#define vma_desc_test_any(desc, ...) \
+	vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__))
+
+/* Helper to test all VMA flags in a VMA descriptor. */
+static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc,
+		vma_flags_t flags)
+{
+	return vma_flags_test_all_mask(&desc->vma_flags, flags);
+}
+
+/*
+ * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor,
+ * e.g.:
+ *
+ * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
+ */
+#define vma_desc_test_all(desc, ...) \
+	vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__))
 
 /* Helper to set all VMA flags in a VMA descriptor. */
-static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
-					   vma_flags_t flags)
+static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
+		vma_flags_t flags)
 {
 	vma_flags_set_mask(&desc->vma_flags, flags);
 }
@@ -1205,14 +1464,14 @@ static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
  * vm_area_desc object describing a proposed VMA, e.g.:
  *
  * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
- * 		VMA_DONTDUMP_BIT);
+ *              VMA_DONTDUMP_BIT);
  */
 #define vma_desc_set_flags(desc, ...) \
 	vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
 
 /* Helper to clear all VMA flags in a VMA descriptor. */
-static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
-					     vma_flags_t flags)
+static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
+		vma_flags_t flags)
 {
 	vma_flags_clear_mask(&desc->vma_flags, flags);
 }
@@ -1292,12 +1551,6 @@ static inline bool vma_is_accessible(const struct vm_area_struct *vma)
 	return vma->vm_flags & VM_ACCESS_FLAGS;
 }
 
-static inline bool is_shared_maywrite_vm_flags(vm_flags_t vm_flags)
-{
-	return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
-		(VM_SHARED | VM_MAYWRITE);
-}
-
 static inline bool is_shared_maywrite(const vma_flags_t *flags)
 {
 	return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT);
@@ -1308,6 +1561,28 @@ static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma)
 	return is_shared_maywrite(&vma->flags);
 }
 
+/**
+ * vma_kernel_pagesize - Default page size granularity for this VMA.
+ * @vma: The user mapping.
+ *
+ * The kernel page size specifies in which granularity VMA modifications
+ * can be performed. Folios in this VMA will be aligned to, and at least
+ * the size of the number of bytes returned by this function.
+ *
+ * The default kernel page size is not affected by Transparent Huge Pages
+ * being in effect.
+ *
+ * Return: The default page size granularity for this VMA.
+ */
+static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
+{
+	if (unlikely(vma->vm_ops && vma->vm_ops->pagesize))
+		return vma->vm_ops->pagesize(vma);
+	return PAGE_SIZE;
+}
+
+unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
+
 static inline
 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max)
 {
@@ -1507,7 +1782,7 @@ static inline int folio_put_testzero(struct folio *folio)
  */
 static inline bool get_page_unless_zero(struct page *page)
 {
-	return page_ref_add_unless(page, 1, 0);
+	return page_ref_add_unless_zero(page, 1);
 }
 
 static inline struct folio *folio_get_nontail_page(struct page *page)
@@ -1957,7 +2232,7 @@ static inline bool is_nommu_shared_mapping(vm_flags_t flags)
 
 static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags)
 {
-	return vma_flags_test(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT);
+	return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT);
 }
 #endif
 
@@ -2479,36 +2754,6 @@ static inline unsigned long folio_nr_pages(const struct folio *folio)
 	return folio_large_nr_pages(folio);
 }
 
-#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS)
-/*
- * We don't expect any folios that exceed buddy sizes (and consequently
- * memory sections).
- */
-#define MAX_FOLIO_ORDER		MAX_PAGE_ORDER
-#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
-/*
- * Only pages within a single memory section are guaranteed to be
- * contiguous. By limiting folios to a single memory section, all folio
- * pages are guaranteed to be contiguous.
- */
-#define MAX_FOLIO_ORDER		PFN_SECTION_SHIFT
-#elif defined(CONFIG_HUGETLB_PAGE)
-/*
- * There is no real limit on the folio size. We limit them to the maximum we
- * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect
- * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit.
- */
-#define MAX_FOLIO_ORDER		get_order(IS_ENABLED(CONFIG_64BIT) ? SZ_16G : SZ_1G)
-#else
-/*
- * Without hugetlb, gigantic folios that are bigger than a single PUD are
- * currently impossible.
- */
-#define MAX_FOLIO_ORDER		PUD_ORDER
-#endif
-
-#define MAX_FOLIO_NR_PAGES	(1UL << MAX_FOLIO_ORDER)
-
 /*
  * compound_nr() returns the number of pages in this potentially compound
  * page.  compound_nr() can be called on a tail page, and is defined to
@@ -2667,7 +2912,7 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio)
  * The caller must add any reference (e.g., from folio_try_get()) it might be
  * holding itself to the result.
  *
- * Returns the expected folio refcount.
+ * Returns: the expected folio refcount.
  */
 static inline int folio_expected_ref_count(const struct folio *folio)
 {
@@ -2798,8 +3043,9 @@ extern void pagefault_out_of_memory(void);
  */
 struct zap_details {
 	struct folio *single_folio;	/* Locked folio to be unmapped */
-	bool even_cows;			/* Zap COWed private pages too? */
+	bool skip_cows;			/* Do not zap COWed private pages */
 	bool reclaim_pt;		/* Need reclaim page tables? */
+	bool reaping;			/* Reaping, do not block. */
 	zap_flags_t zap_flags;		/* Extra flags for zapping */
 };
 
@@ -2832,14 +3078,17 @@ struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr,
 		pud_t pud);
 
-void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
+void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address,
 		  unsigned long size);
-void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
-			   unsigned long size, struct zap_details *details);
-static inline void zap_vma_pages(struct vm_area_struct *vma)
+void zap_vma_range(struct vm_area_struct *vma, unsigned long address,
+			   unsigned long size);
+/**
+ * zap_vma - zap all page table entries in a vma
+ * @vma: The vma to zap.
+ */
+static inline void zap_vma(struct vm_area_struct *vma)
 {
-	zap_page_range_single(vma, vma->vm_start,
-			      vma->vm_end - vma->vm_start, NULL);
+	zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start);
 }
 struct mmu_notifier_range;
 
@@ -3057,38 +3306,47 @@ static inline bool get_user_page_fast_only(unsigned long addr,
 {
 	return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
 }
+
+static inline struct percpu_counter_tree_level_item *get_rss_stat_items(struct mm_struct *mm)
+{
+	unsigned long ptr = (unsigned long)mm;
+
+	ptr += offsetof(struct mm_struct, flexible_array);
+	return (struct percpu_counter_tree_level_item *)ptr;
+}
+
 /*
  * per-process(per-mm_struct) statistics.
  */
 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
 {
-	return percpu_counter_read_positive(&mm->rss_stat[member]);
+	return percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member]);
 }
 
 static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member)
 {
-	return percpu_counter_sum_positive(&mm->rss_stat[member]);
+	return percpu_counter_tree_precise_sum_positive(&mm->rss_stat[member]);
 }
 
 void mm_trace_rss_stat(struct mm_struct *mm, int member);
 
 static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
 {
-	percpu_counter_add(&mm->rss_stat[member], value);
+	percpu_counter_tree_add(&mm->rss_stat[member], value);
 
 	mm_trace_rss_stat(mm, member);
 }
 
 static inline void inc_mm_counter(struct mm_struct *mm, int member)
 {
-	percpu_counter_inc(&mm->rss_stat[member]);
+	percpu_counter_tree_add(&mm->rss_stat[member], 1);
 
 	mm_trace_rss_stat(mm, member);
 }
 
 static inline void dec_mm_counter(struct mm_struct *mm, int member)
 {
-	percpu_counter_dec(&mm->rss_stat[member]);
+	percpu_counter_tree_add(&mm->rss_stat[member], -1);
 
 	mm_trace_rss_stat(mm, member);
 }
@@ -3847,7 +4105,6 @@ extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
 extern struct file *get_mm_exe_file(struct mm_struct *mm);
 extern struct file *get_task_exe_file(struct task_struct *task);
 
-extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
 extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
 
 extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
@@ -3898,11 +4155,13 @@ static inline void mm_populate(unsigned long addr, unsigned long len) {}
 #endif
 
 /* This takes the mm semaphore itself */
-extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long);
-extern int vm_munmap(unsigned long, size_t);
-extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
-        unsigned long, unsigned long,
-        unsigned long, unsigned long);
+int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec);
+int vm_munmap(unsigned long start, size_t len);
+unsigned long __must_check vm_mmap(struct file *file, unsigned long addr,
+		unsigned long len, unsigned long prot,
+		unsigned long flag, unsigned long offset);
+unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr,
+		unsigned long len, unsigned long flags);
 
 struct vm_unmapped_area_info {
 #define VM_UNMAPPED_AREA_TOPDOWN 1
@@ -3999,6 +4258,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma)
 	return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 }
 
+static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma)
+{
+	return vma->vm_pgoff + vma_pages(vma) - 1;
+}
+
 static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
 {
 	return desc->end - desc->start;
@@ -4073,15 +4337,76 @@ static inline void mmap_action_ioremap(struct vm_area_desc *desc,
  * @start_pfn: The first PFN in the range to remap.
  */
 static inline void mmap_action_ioremap_full(struct vm_area_desc *desc,
-					  unsigned long start_pfn)
+					    unsigned long start_pfn)
 {
 	mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc));
 }
 
-void mmap_action_prepare(struct mmap_action *action,
-			 struct vm_area_desc *desc);
-int mmap_action_complete(struct mmap_action *action,
-			 struct vm_area_struct *vma);
+/**
+ * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the
+ * physical range in [start_phys_addr, start_phys_addr + size) should be I/O
+ * remapped.
+ * @desc: The VMA descriptor for the VMA requiring remap.
+ * @start_phys_addr: Start of the physical memory to be mapped.
+ * @size: Size of the area to map.
+ *
+ * NOTE: Some drivers might want to tweak desc->page_prot for purposes of
+ * write-combine or similar.
+ */
+static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc,
+					      phys_addr_t start_phys_addr,
+					      unsigned long size)
+{
+	struct mmap_action *action = &desc->action;
+
+	action->simple_ioremap.start_phys_addr = start_phys_addr;
+	action->simple_ioremap.size = size;
+	action->type = MMAP_SIMPLE_IO_REMAP;
+}
+
+/**
+ * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that
+ * @num kernel pages contained in the @pages array should be mapped to userland
+ * starting at virtual address @start.
+ * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped.
+ * @start: The virtual address from which to map them.
+ * @pages: An array of struct page pointers describing the memory to map.
+ * @nr_pages: The number of entries in the @pages aray.
+ */
+static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc,
+		unsigned long start, struct page **pages,
+		unsigned long nr_pages)
+{
+	struct mmap_action *action = &desc->action;
+
+	action->type = MMAP_MAP_KERNEL_PAGES;
+	action->map_kernel.start = start;
+	action->map_kernel.pages = pages;
+	action->map_kernel.nr_pages = nr_pages;
+	action->map_kernel.pgoff = desc->pgoff;
+}
+
+/**
+ * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that
+ * kernel pages contained in the @pages array should be mapped to userland
+ * from @desc->start to @desc->end.
+ * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped.
+ * @pages: An array of struct page pointers describing the memory to map.
+ *
+ * The caller must ensure that @pages contains sufficient entries to cover the
+ * entire range described by @desc.
+ */
+static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc,
+		struct page **pages)
+{
+	mmap_action_map_kernel_pages(desc, desc->start, pages,
+				     vma_desc_pages(desc));
+}
+
+int mmap_action_prepare(struct vm_area_desc *desc);
+int mmap_action_complete(struct vm_area_struct *vma,
+			 struct mmap_action *action,
+			 bool rmap_lock_held);
 
 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
@@ -4095,20 +4420,81 @@ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
 	return vma;
 }
 
+/**
+ * range_is_subset - Is the specified inner range a subset of the outer range?
+ * @outer_start: The start of the outer range.
+ * @outer_end: The exclusive end of the outer range.
+ * @inner_start: The start of the inner range.
+ * @inner_end: The exclusive end of the inner range.
+ *
+ * Returns: %true if [inner_start, inner_end) is a subset of [outer_start,
+ * outer_end), otherwise %false.
+ */
+static inline bool range_is_subset(unsigned long outer_start,
+				   unsigned long outer_end,
+				   unsigned long inner_start,
+				   unsigned long inner_end)
+{
+	return outer_start <= inner_start && inner_end <= outer_end;
+}
+
+/**
+ * range_in_vma - is the specified [@start, @end) range a subset of the VMA?
+ * @vma: The VMA against which we want to check [@start, @end).
+ * @start: The start of the range we wish to check.
+ * @end: The exclusive end of the range we wish to check.
+ *
+ * Returns: %true if [@start, @end) is a subset of [@vma->vm_start,
+ * @vma->vm_end), %false otherwise.
+ */
 static inline bool range_in_vma(const struct vm_area_struct *vma,
 				unsigned long start, unsigned long end)
 {
-	return (vma && vma->vm_start <= start && end <= vma->vm_end);
+	if (!vma)
+		return false;
+
+	return range_is_subset(vma->vm_start, vma->vm_end, start, end);
+}
+
+/**
+ * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA
+ * described by @desc, a VMA descriptor?
+ * @desc: The VMA descriptor against which we want to check [@start, @end).
+ * @start: The start of the range we wish to check.
+ * @end: The exclusive end of the range we wish to check.
+ *
+ * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end),
+ * %false otherwise.
+ */
+static inline bool range_in_vma_desc(const struct vm_area_desc *desc,
+				     unsigned long start, unsigned long end)
+{
+	if (!desc)
+		return false;
+
+	return range_is_subset(desc->start, desc->end, start, end);
 }
 
 #ifdef CONFIG_MMU
 pgprot_t vm_get_page_prot(vm_flags_t vm_flags);
+
+static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags)
+{
+	const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
+
+	return vm_get_page_prot(vm_flags);
+}
+
 void vma_set_page_prot(struct vm_area_struct *vma);
 #else
 static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
 {
 	return __pgprot(0);
 }
+static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags)
+{
+	return __pgprot(0);
+}
 static inline void vma_set_page_prot(struct vm_area_struct *vma)
 {
 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
@@ -4130,6 +4516,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
 			struct page **pages, unsigned long *num);
+int map_kernel_pages_prepare(struct vm_area_desc *desc);
+int map_kernel_pages_complete(struct vm_area_struct *vma,
+			      struct mmap_action *action);
 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
 				unsigned long num);
 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
@@ -4486,13 +4875,6 @@ unsigned long section_map_size(void);
 struct page * __populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap);
-pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
-p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
-pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
-pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
-pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
-			    struct vmem_altmap *altmap, unsigned long ptpfn,
-			    unsigned long flags);
 void *vmemmap_alloc_block(unsigned long size, int node);
 struct vmem_altmap;
 void *vmemmap_alloc_block_buf(unsigned long size, int node,
@@ -4508,10 +4890,9 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
 			       int node, struct vmem_altmap *altmap);
 int vmemmap_populate(unsigned long start, unsigned long end, int node,
 		struct vmem_altmap *altmap);
-int vmemmap_populate_hvo(unsigned long start, unsigned long end, int node,
+int vmemmap_populate_hvo(unsigned long start, unsigned long end,
+			 unsigned int order, struct zone *zone,
 			 unsigned long headsize);
-int vmemmap_undo_hvo(unsigned long start, unsigned long end, int node,
-		     unsigned long headsize);
 void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
 			  unsigned long headsize);
 void vmemmap_populate_print_last(void);
@@ -4817,10 +5198,9 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
  * DMA mapping IDs for page_pool
  *
  * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and
- * stashes it in the upper bits of page->pp_magic. We always want to be able to
- * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP
- * pages can have arbitrary kernel pointers stored in the same field as pp_magic
- * (since it overlaps with page->lru.next), so we must ensure that we cannot
+ * stashes it in the upper bits of page->pp_magic. Non-PP pages can have
+ * arbitrary kernel pointers stored in the same field as pp_magic (since
+ * it overlaps with page->lru.next), so we must ensure that we cannot
  * mistake a valid kernel pointer with any of the values we write into this
  * field.
  *
@@ -4855,26 +5235,6 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
 #define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \
 				  PP_DMA_INDEX_SHIFT)
 
-/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is
- * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for
- * the head page of compound page and bit 1 for pfmemalloc page, as well as the
- * bits used for the DMA index. page_is_pfmemalloc() is checked in
- * __page_pool_put_page() to avoid recycling the pfmemalloc page.
- */
-#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
-
-#ifdef CONFIG_PAGE_POOL
-static inline bool page_pool_page_is_pp(const struct page *page)
-{
-	return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;
-}
-#else
-static inline bool page_pool_page_is_pp(const struct page *page)
-{
-	return false;
-}
-#endif
-
 #define PAGE_SNAPSHOT_FAITHFUL (1 << 0)
 #define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)
 #define PAGE_SNAPSHOT_PG_IDLE  (1 << 2)
@@ -4894,4 +5254,8 @@ static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps)
 
 void snapshot_page(struct page_snapshot *ps, const struct page *page);
 
+void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte,
+		struct vm_area_struct *vma, unsigned long addr,
+		bool uffd_wp);
+
 #endif /* _LINUX_MM_H */
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index fa2d6ba811b53..ad50688d89dba 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -342,6 +342,8 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
 {
 	enum lru_list lru = folio_lru_list(folio);
 
+	VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
+
 	if (lru_gen_add_folio(lruvec, folio, false))
 		return;
 
@@ -356,6 +358,8 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
 {
 	enum lru_list lru = folio_lru_list(folio);
 
+	VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
+
 	if (lru_gen_add_folio(lruvec, folio, true))
 		return;
 
@@ -370,6 +374,8 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
 {
 	enum lru_list lru = folio_lru_list(folio);
 
+	VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
+
 	if (lru_gen_del_folio(lruvec, folio, false))
 		return;
 
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 3cc8ae7228860..8665d0dae21f5 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -18,7 +18,7 @@
 #include <linux/page-flags-layout.h>
 #include <linux/workqueue.h>
 #include <linux/seqlock.h>
-#include <linux/percpu_counter.h>
+#include <linux/percpu_counter_tree.h>
 #include <linux/types.h>
 #include <linux/rseq_types.h>
 #include <linux/bitmap.h>
@@ -126,14 +126,14 @@ struct page {
 			atomic_long_t pp_ref_count;
 		};
 		struct {	/* Tail pages of compound page */
-			unsigned long compound_head;	/* Bit zero is set */
+			unsigned long compound_info;	/* Bit zero is set */
 		};
 		struct {	/* ZONE_DEVICE pages */
 			/*
-			 * The first word is used for compound_head or folio
+			 * The first word is used for compound_info or folio
 			 * pgmap
 			 */
-			void *_unused_pgmap_compound_head;
+			void *_unused_pgmap_compound_info;
 			void *zone_device_data;
 			/*
 			 * ZONE_DEVICE private pages are counted as being
@@ -409,7 +409,7 @@ struct folio {
 	/* private: avoid cluttering the output */
 				/* For the Unevictable "LRU list" slot */
 				struct {
-					/* Avoid compound_head */
+					/* Avoid compound_info */
 					void *__filler;
 	/* public: */
 					unsigned int mlock_count;
@@ -510,7 +510,7 @@ struct folio {
 FOLIO_MATCH(flags, flags);
 FOLIO_MATCH(lru, lru);
 FOLIO_MATCH(mapping, mapping);
-FOLIO_MATCH(compound_head, lru);
+FOLIO_MATCH(compound_info, lru);
 FOLIO_MATCH(__folio_index, index);
 FOLIO_MATCH(private, private);
 FOLIO_MATCH(_mapcount, _mapcount);
@@ -529,7 +529,7 @@ FOLIO_MATCH(_last_cpupid, _last_cpupid);
 	static_assert(offsetof(struct folio, fl) ==			\
 			offsetof(struct page, pg) + sizeof(struct page))
 FOLIO_MATCH(flags, _flags_1);
-FOLIO_MATCH(compound_head, _head_1);
+FOLIO_MATCH(compound_info, _head_1);
 FOLIO_MATCH(_mapcount, _mapcount_1);
 FOLIO_MATCH(_refcount, _refcount_1);
 #undef FOLIO_MATCH
@@ -537,13 +537,13 @@ FOLIO_MATCH(_refcount, _refcount_1);
 	static_assert(offsetof(struct folio, fl) ==			\
 			offsetof(struct page, pg) + 2 * sizeof(struct page))
 FOLIO_MATCH(flags, _flags_2);
-FOLIO_MATCH(compound_head, _head_2);
+FOLIO_MATCH(compound_info, _head_2);
 #undef FOLIO_MATCH
 #define FOLIO_MATCH(pg, fl)						\
 	static_assert(offsetof(struct folio, fl) ==			\
 			offsetof(struct page, pg) + 3 * sizeof(struct page))
 FOLIO_MATCH(flags, _flags_3);
-FOLIO_MATCH(compound_head, _head_3);
+FOLIO_MATCH(compound_info, _head_3);
 #undef FOLIO_MATCH
 
 /**
@@ -609,8 +609,8 @@ struct ptdesc {
 #define TABLE_MATCH(pg, pt)						\
 	static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt))
 TABLE_MATCH(flags, pt_flags);
-TABLE_MATCH(compound_head, pt_list);
-TABLE_MATCH(compound_head, _pt_pad_1);
+TABLE_MATCH(compound_info, pt_list);
+TABLE_MATCH(compound_info, _pt_pad_1);
 TABLE_MATCH(mapping, __page_mapping);
 TABLE_MATCH(__folio_index, pt_index);
 TABLE_MATCH(rcu_head, pt_rcu_head);
@@ -814,6 +814,8 @@ enum mmap_action_type {
 	MMAP_NOTHING,		/* Mapping is complete, no further action. */
 	MMAP_REMAP_PFN,		/* Remap PFN range. */
 	MMAP_IO_REMAP_PFN,	/* I/O remap PFN range. */
+	MMAP_SIMPLE_IO_REMAP,	/* I/O remap with guardrails. */
+	MMAP_MAP_KERNEL_PAGES,	/* Map kernel page range from array. */
 };
 
 /*
@@ -822,13 +824,22 @@ enum mmap_action_type {
  */
 struct mmap_action {
 	union {
-		/* Remap range. */
 		struct {
 			unsigned long start;
 			unsigned long start_pfn;
 			unsigned long size;
 			pgprot_t pgprot;
 		} remap;
+		struct {
+			phys_addr_t start_phys_addr;
+			unsigned long size;
+		} simple_ioremap;
+		struct {
+			unsigned long start;
+			struct page **pages;
+			unsigned long nr_pages;
+			pgoff_t pgoff;
+		} map_kernel;
 	};
 	enum mmap_action_type type;
 
@@ -870,6 +881,14 @@ typedef struct {
 
 #define EMPTY_VMA_FLAGS ((vma_flags_t){ })
 
+/* Are no flags set in the specified VMA flags? */
+static __always_inline bool vma_flags_empty(const vma_flags_t *flags)
+{
+	const unsigned long *bitmap = flags->__vma_flags;
+
+	return bitmap_empty(bitmap, NUM_VMA_FLAG_BITS);
+}
+
 /*
  * Describes a VMA that is about to be mmap()'ed. Drivers may choose to
  * manipulate mutable fields which will cause those fields to be updated in the
@@ -879,8 +898,8 @@ typedef struct {
  */
 struct vm_area_desc {
 	/* Immutable state. */
-	const struct mm_struct *const mm;
-	struct file *const file; /* May vary from vm_file in stacked callers. */
+	struct mm_struct *mm;
+	struct file *file; /* May vary from vm_file in stacked callers. */
 	unsigned long start;
 	unsigned long end;
 
@@ -1056,31 +1075,59 @@ struct vm_area_struct {
 } __randomize_layout;
 
 /* Clears all bits in the VMA flags bitmap, non-atomically. */
-static inline void vma_flags_clear_all(vma_flags_t *flags)
+static __always_inline void vma_flags_clear_all(vma_flags_t *flags)
 {
 	bitmap_zero(flags->__vma_flags, NUM_VMA_FLAG_BITS);
 }
 
+/*
+ * Helper function which converts a vma_flags_t value to a legacy vm_flags_t
+ * value. This is only valid if the input flags value can be expressed in a
+ * system word.
+ *
+ * Will be removed once the conversion to VMA flags is complete.
+ */
+static __always_inline vm_flags_t vma_flags_to_legacy(vma_flags_t flags)
+{
+	return (vm_flags_t)flags.__vma_flags[0];
+}
+
 /*
  * Copy value to the first system word of VMA flags, non-atomically.
  *
  * IMPORTANT: This does not overwrite bytes past the first system word. The
  * caller must account for this.
  */
-static inline void vma_flags_overwrite_word(vma_flags_t *flags, unsigned long value)
+static __always_inline void vma_flags_overwrite_word(vma_flags_t *flags,
+		unsigned long value)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 
 	bitmap[0] = value;
 }
 
+/*
+ * Helper function which converts a legacy vm_flags_t value to a vma_flags_t
+ * value.
+ *
+ * Will be removed once the conversion to VMA flags is complete.
+ */
+static __always_inline vma_flags_t legacy_to_vma_flags(vm_flags_t flags)
+{
+	vma_flags_t ret = EMPTY_VMA_FLAGS;
+
+	vma_flags_overwrite_word(&ret, flags);
+	return ret;
+}
+
 /*
  * Copy value to the first system word of VMA flags ONCE, non-atomically.
  *
  * IMPORTANT: This does not overwrite bytes past the first system word. The
  * caller must account for this.
  */
-static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned long value)
+static __always_inline void vma_flags_overwrite_word_once(vma_flags_t *flags,
+		unsigned long value)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 
@@ -1088,7 +1135,8 @@ static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned lo
 }
 
 /* Update the first system word of VMA flags setting bits, non-atomically. */
-static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value)
+static __always_inline void vma_flags_set_word(vma_flags_t *flags,
+		unsigned long value)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 
@@ -1096,7 +1144,8 @@ static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value)
 }
 
 /* Update the first system word of VMA flags clearing bits, non-atomically. */
-static inline void vma_flags_clear_word(vma_flags_t *flags, unsigned long value)
+static __always_inline void vma_flags_clear_word(vma_flags_t *flags,
+		unsigned long value)
 {
 	unsigned long *bitmap = flags->__vma_flags;
 
@@ -1118,6 +1167,19 @@ typedef struct {
 	DECLARE_BITMAP(__mm_flags, NUM_MM_FLAG_BITS);
 } __private mm_flags_t;
 
+/*
+ * The alignment of the mm_struct flexible array is based on the largest
+ * alignment of its content:
+ * __alignof__(struct percpu_counter_tree_level_item) provides a
+ * cacheline aligned alignment on SMP systems, else alignment on
+ * unsigned long on UP systems.
+ */
+#ifdef CONFIG_SMP
+# define __mm_struct_flexible_array_aligned	__aligned(__alignof__(struct percpu_counter_tree_level_item))
+#else
+# define __mm_struct_flexible_array_aligned	__aligned(__alignof__(unsigned long))
+#endif
+
 struct kioctx_table;
 struct iommu_mm_data;
 struct mm_struct {
@@ -1241,7 +1303,11 @@ struct mm_struct {
 		unsigned long data_vm;	   /* VM_WRITE & ~VM_SHARED & ~VM_STACK */
 		unsigned long exec_vm;	   /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
 		unsigned long stack_vm;	   /* VM_STACK */
-		vm_flags_t def_flags;
+		union {
+			/* Temporary while VMA flags are being converted. */
+			vm_flags_t def_flags;
+			vma_flags_t def_vma_flags;
+		};
 
 		/**
 		 * @write_protect_seq: Locked when any thread is write
@@ -1255,7 +1321,6 @@ struct mm_struct {
 		unsigned long start_code, end_code, start_data, end_data;
 		unsigned long start_brk, brk, start_stack;
 		unsigned long arg_start, arg_end, env_start, env_end;
-
 		unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
 
 #ifdef CONFIG_ARCH_HAS_ELF_CORE_EFLAGS
@@ -1263,7 +1328,7 @@ struct mm_struct {
 		unsigned long saved_e_flags;
 #endif
 
-		struct percpu_counter rss_stat[NR_MM_COUNTERS];
+		struct percpu_counter_tree rss_stat[NR_MM_COUNTERS];
 
 		struct linux_binfmt *binfmt;
 
@@ -1374,10 +1439,13 @@ struct mm_struct {
 	} __randomize_layout;
 
 	/*
-	 * The mm_cpumask needs to be at the end of mm_struct, because it
-	 * is dynamically sized based on nr_cpu_ids.
+	 * The rss hierarchical counter items, mm_cpumask, and mm_cid
+	 * masks need to be at the end of mm_struct, because they are
+	 * dynamically sized based on nr_cpu_ids.
+	 * The content of the flexible array needs to be placed in
+	 * decreasing alignment requirement order.
 	 */
-	char flexible_array[] __aligned(__alignof__(unsigned long));
+	char flexible_array[] __mm_struct_flexible_array_aligned;
 };
 
 /* Copy value to the first system word of mm flags, non-atomically. */
@@ -1414,24 +1482,30 @@ static inline void __mm_flags_set_mask_bits_word(struct mm_struct *mm,
 			 MT_FLAGS_USE_RCU)
 extern struct mm_struct init_mm;
 
-#define MM_STRUCT_FLEXIBLE_ARRAY_INIT				\
-{								\
-	[0 ... sizeof(cpumask_t) + MM_CID_STATIC_SIZE - 1] = 0	\
+#define MM_STRUCT_FLEXIBLE_ARRAY_INIT									\
+{													\
+	[0 ... (PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE * NR_MM_COUNTERS) + sizeof(cpumask_t) + MM_CID_STATIC_SIZE - 1] = 0	\
 }
 
-/* Pointer magic because the dynamic array size confuses some compilers. */
-static inline void mm_init_cpumask(struct mm_struct *mm)
+static inline size_t get_rss_stat_items_size(void)
 {
-	unsigned long cpu_bitmap = (unsigned long)mm;
-
-	cpu_bitmap += offsetof(struct mm_struct, flexible_array);
-	cpumask_clear((struct cpumask *)cpu_bitmap);
+	return percpu_counter_tree_items_size() * NR_MM_COUNTERS;
 }
 
 /* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
 static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
 {
-	return (struct cpumask *)&mm->flexible_array;
+	unsigned long ptr = (unsigned long)mm;
+
+	ptr += offsetof(struct mm_struct, flexible_array);
+	/* Skip RSS stats counters. */
+	ptr += get_rss_stat_items_size();
+	return (struct cpumask *)ptr;
+}
+
+static inline void mm_init_cpumask(struct mm_struct *mm)
+{
+	cpumask_clear((struct cpumask *)mm_cpumask(mm));
 }
 
 #ifdef CONFIG_LRU_GEN
@@ -1523,6 +1597,8 @@ static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
 	unsigned long bitmap = (unsigned long)mm;
 
 	bitmap += offsetof(struct mm_struct, flexible_array);
+	/* Skip RSS stats counters. */
+	bitmap += get_rss_stat_items_size();
 	/* Skip cpu_bitmap */
 	bitmap += cpumask_size();
 	return (struct cpumask *)bitmap;
@@ -1919,6 +1995,8 @@ enum {
 #define MMF_TOPDOWN		31	/* mm searches top down by default */
 #define MMF_TOPDOWN_MASK	BIT(MMF_TOPDOWN)
 
+#define MMF_USER_HWCAP		32	/* user-defined HWCAPs */
+
 #define MMF_INIT_LEGACY_MASK	(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
 				 MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\
 				 MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK)
diff --git a/include/linux/mman.h b/include/linux/mman.h
index 0ba8a7e8b90ae..389521594c69c 100644
--- a/include/linux/mman.h
+++ b/include/linux/mman.h
@@ -170,53 +170,4 @@ static inline bool arch_memory_deny_write_exec_supported(void)
 }
 #define arch_memory_deny_write_exec_supported arch_memory_deny_write_exec_supported
 #endif
-
-/*
- * Denies creating a writable executable mapping or gaining executable permissions.
- *
- * This denies the following:
- *
- * 	a)	mmap(PROT_WRITE | PROT_EXEC)
- *
- *	b)	mmap(PROT_WRITE)
- *		mprotect(PROT_EXEC)
- *
- *	c)	mmap(PROT_WRITE)
- *		mprotect(PROT_READ)
- *		mprotect(PROT_EXEC)
- *
- * But allows the following:
- *
- *	d)	mmap(PROT_READ | PROT_EXEC)
- *		mmap(PROT_READ | PROT_EXEC | PROT_BTI)
- *
- * This is only applicable if the user has set the Memory-Deny-Write-Execute
- * (MDWE) protection mask for the current process.
- *
- * @old specifies the VMA flags the VMA originally possessed, and @new the ones
- * we propose to set.
- *
- * Return: false if proposed change is OK, true if not ok and should be denied.
- */
-static inline bool map_deny_write_exec(unsigned long old, unsigned long new)
-{
-	/* If MDWE is disabled, we have nothing to deny. */
-	if (!mm_flags_test(MMF_HAS_MDWE, current->mm))
-		return false;
-
-	/* If the new VMA is not executable, we have nothing to deny. */
-	if (!(new & VM_EXEC))
-		return false;
-
-	/* Under MDWE we do not accept newly writably executable VMAs... */
-	if (new & VM_WRITE)
-		return true;
-
-	/* ...nor previously non-executable VMAs becoming executable. */
-	if (!(old & VM_EXEC))
-		return true;
-
-	return false;
-}
-
 #endif /* _LINUX_MMAN_H */
diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h
index e9e964c20e530..9dc4750296af9 100644
--- a/include/linux/mmc/card.h
+++ b/include/linux/mmc/card.h
@@ -329,6 +329,8 @@ struct mmc_card {
 #define MMC_QUIRK_BROKEN_CACHE_FLUSH	(1<<16)	/* Don't flush cache until the write has occurred */
 #define MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY	(1<<17) /* Disable broken SD poweroff notify support */
 #define MMC_QUIRK_NO_UHS_DDR50_TUNING	(1<<18) /* Disable DDR50 tuning */
+#define MMC_QUIRK_BROKEN_MDT    (1<<19) /* Wrong manufacturing year */
+#define MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME	(1<<20) /* Secure erase/trim time is fixed regardless of size */
 
 	bool			written_flag;	/* Indicates eMMC has been written since power on */
 	bool			reenable_cmdq;	/* Re-enable Command Queue */
diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h
index 673cbdf434533..0685dd717e855 100644
--- a/include/linux/mmc/sdio_ids.h
+++ b/include/linux/mmc/sdio_ids.h
@@ -111,11 +111,15 @@
 #define SDIO_VENDOR_ID_MEDIATEK			0x037a
 #define SDIO_DEVICE_ID_MEDIATEK_MT7663		0x7663
 #define SDIO_DEVICE_ID_MEDIATEK_MT7668		0x7668
+#define SDIO_DEVICE_ID_MEDIATEK_MT7902		0x790a
 #define SDIO_DEVICE_ID_MEDIATEK_MT7961		0x7961
 
 #define SDIO_VENDOR_ID_MICROCHIP_WILC		0x0296
 #define SDIO_DEVICE_ID_MICROCHIP_WILC1000	0x5347
 
+#define SDIO_VENDOR_ID_NXP			0x0471
+#define SDIO_DEVICE_ID_NXP_IW61X		0x0205
+
 #define SDIO_VENDOR_ID_REALTEK			0x024c
 #define SDIO_DEVICE_ID_REALTEK_RTW8723BS	0xb723
 #define SDIO_DEVICE_ID_REALTEK_RTW8821BS	0xb821
diff --git a/include/linux/mmdebug.h b/include/linux/mmdebug.h
index ab60ffba08f5f..a167c5aa525ef 100644
--- a/include/linux/mmdebug.h
+++ b/include/linux/mmdebug.h
@@ -60,7 +60,7 @@ void vma_iter_dump_tree(const struct vma_iterator *vmi);
 	static bool __section(".data..once") __warned;			\
 	int __ret_warn_once = !!(cond);					\
 									\
-	if (unlikely(__ret_warn_once && !__warned)) {			\
+	if (unlikely(__ret_warn_once) && unlikely(!__warned)) {		\
 		dump_page(page, "VM_WARN_ON_ONCE_PAGE(" __stringify(cond)")");\
 		__warned = true;					\
 		WARN_ON(1);						\
@@ -80,7 +80,7 @@ void vma_iter_dump_tree(const struct vma_iterator *vmi);
 	static bool __section(".data..once") __warned;			\
 	int __ret_warn_once = !!(cond);					\
 									\
-	if (unlikely(__ret_warn_once && !__warned)) {			\
+	if (unlikely(__ret_warn_once) && unlikely(!__warned)) {		\
 		dump_page(&folio->page, "VM_WARN_ON_ONCE_FOLIO(" __stringify(cond)")");\
 		__warned = true;					\
 		WARN_ON(1);						\
@@ -91,7 +91,7 @@ void vma_iter_dump_tree(const struct vma_iterator *vmi);
 	static bool __section(".data..once") __warned;			\
 	int __ret_warn_once = !!(cond);					\
 									\
-	if (unlikely(__ret_warn_once && !__warned)) {			\
+	if (unlikely(__ret_warn_once) && unlikely(!__warned)) {		\
 		dump_mm(mm);						\
 		__warned = true;					\
 		WARN_ON(1);						\
@@ -102,7 +102,7 @@ void vma_iter_dump_tree(const struct vma_iterator *vmi);
 	static bool __section(".data..once") __warned;			\
 	int __ret_warn_once = !!(cond);					\
 									\
-	if (unlikely(__ret_warn_once && !__warned)) {			\
+	if (unlikely(__ret_warn_once) && unlikely(!__warned)) {		\
 		dump_vma(vma);						\
 		__warned = true;					\
 		WARN_ON(1);						\
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 8450e18a87c26..c5cd095f36e9e 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -233,16 +233,58 @@ struct mmu_notifier {
 	unsigned int users;
 };
 
+/**
+ * struct mmu_interval_notifier_finish - mmu_interval_notifier two-pass abstraction
+ * @link: Lockless list link for the notifiers pending pass list
+ * @notifier: The mmu_interval_notifier for which the finish pass is called.
+ *
+ * Allocate, typically using GFP_NOWAIT in the interval notifier's start pass.
+ * Note that with a large number of notifiers implementing two passes,
+ * allocation with GFP_NOWAIT will become increasingly likely to fail, so consider
+ * implementing a small pool instead of using kmalloc() allocations.
+ *
+ * If the implementation needs to pass data between the start and the finish passes,
+ * the recommended way is to embed struct mmu_interval_notifier_finish into a larger
+ * structure that also contains the data needed to be shared. Keep in mind that
+ * a notifier callback can be invoked in parallel, and each invocation needs its
+ * own struct mmu_interval_notifier_finish.
+ *
+ * If allocation fails, then the &mmu_interval_notifier_ops->invalidate_start op
+ * needs to implements the full notifier functionality. Please refer to its
+ * documentation.
+ */
+struct mmu_interval_notifier_finish {
+	struct llist_node link;
+	struct mmu_interval_notifier *notifier;
+};
+
 /**
  * struct mmu_interval_notifier_ops - callback for range notification
  * @invalidate: Upon return the caller must stop using any SPTEs within this
  *              range. This function can sleep. Return false only if sleeping
  *              was required but mmu_notifier_range_blockable(range) is false.
+ * @invalidate_start: Similar to @invalidate, but intended for two-pass notifier
+ *                    callbacks where the call to @invalidate_start is the first
+ *                    pass and any struct mmu_interval_notifier_finish pointer
+ *                    returned in the @finish parameter describes the finish pass.
+ *                    If *@finish is %NULL on return, then no final pass will be
+ *                    called, and @invalidate_start needs to implement the full
+ *                    notifier, behaving like @invalidate. The value of *@finish
+ *                    is guaranteed to be %NULL at function entry.
+ * @invalidate_finish: Called as the second pass for any notifier that returned
+ *                     a non-NULL *@finish from @invalidate_start. The @finish
+ *                     pointer passed here is the same one returned by
+ *                     @invalidate_start.
  */
 struct mmu_interval_notifier_ops {
 	bool (*invalidate)(struct mmu_interval_notifier *interval_sub,
 			   const struct mmu_notifier_range *range,
 			   unsigned long cur_seq);
+	bool (*invalidate_start)(struct mmu_interval_notifier *interval_sub,
+				 const struct mmu_notifier_range *range,
+				 unsigned long cur_seq,
+				 struct mmu_interval_notifier_finish **finish);
+	void (*invalidate_finish)(struct mmu_interval_notifier_finish *finish);
 };
 
 struct mmu_interval_notifier {
@@ -516,55 +558,6 @@ static inline void mmu_notifier_range_init_owner(
 	range->owner = owner;
 }
 
-#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)	\
-({									\
-	int __young;							\
-	struct vm_area_struct *___vma = __vma;				\
-	unsigned long ___address = __address;				\
-	unsigned int ___nr = __nr;					\
-	__young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr);	\
-	__young |= mmu_notifier_clear_flush_young(___vma->vm_mm,	\
-						  ___address,		\
-						  ___address +		\
-						  ___nr * PAGE_SIZE);	\
-	__young;							\
-})
-
-#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp)		\
-({									\
-	int __young;							\
-	struct vm_area_struct *___vma = __vma;				\
-	unsigned long ___address = __address;				\
-	__young = pmdp_clear_flush_young(___vma, ___address, __pmdp);	\
-	__young |= mmu_notifier_clear_flush_young(___vma->vm_mm,	\
-						  ___address,		\
-						  ___address +		\
-							PMD_SIZE);	\
-	__young;							\
-})
-
-#define ptep_clear_young_notify(__vma, __address, __ptep)		\
-({									\
-	int __young;							\
-	struct vm_area_struct *___vma = __vma;				\
-	unsigned long ___address = __address;				\
-	__young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
-	__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,	\
-					    ___address + PAGE_SIZE);	\
-	__young;							\
-})
-
-#define pmdp_clear_young_notify(__vma, __address, __pmdp)		\
-({									\
-	int __young;							\
-	struct vm_area_struct *___vma = __vma;				\
-	unsigned long ___address = __address;				\
-	__young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
-	__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address,	\
-					    ___address + PMD_SIZE);	\
-	__young;							\
-})
-
 #else /* CONFIG_MMU_NOTIFIER */
 
 struct mmu_notifier_range {
@@ -652,11 +645,6 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
 
 #define mmu_notifier_range_update_to_read_only(r) false
 
-#define clear_flush_young_ptes_notify clear_flush_young_ptes
-#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
-#define ptep_clear_young_notify ptep_test_and_clear_young
-#define pmdp_clear_young_notify pmdp_test_and_clear_young
-
 static inline void mmu_notifier_synchronize(void)
 {
 }
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 3e51190a55e4c..7bd0134c241ce 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -23,6 +23,7 @@
 #include <linux/page-flags.h>
 #include <linux/local_lock.h>
 #include <linux/zswap.h>
+#include <linux/sizes.h>
 #include <asm/page.h>
 
 /* Free memory management - zoned buddy allocator.  */
@@ -61,6 +62,59 @@
  */
 #define PAGE_ALLOC_COSTLY_ORDER 3
 
+#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS)
+/*
+ * We don't expect any folios that exceed buddy sizes (and consequently
+ * memory sections).
+ */
+#define MAX_FOLIO_ORDER		MAX_PAGE_ORDER
+#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
+/*
+ * Only pages within a single memory section are guaranteed to be
+ * contiguous. By limiting folios to a single memory section, all folio
+ * pages are guaranteed to be contiguous.
+ */
+#define MAX_FOLIO_ORDER		PFN_SECTION_SHIFT
+#elif defined(CONFIG_HUGETLB_PAGE)
+/*
+ * There is no real limit on the folio size. We limit them to the maximum we
+ * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect
+ * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit.
+ */
+#ifdef CONFIG_64BIT
+#define MAX_FOLIO_ORDER		(ilog2(SZ_16G) - PAGE_SHIFT)
+#else
+#define MAX_FOLIO_ORDER		(ilog2(SZ_1G) - PAGE_SHIFT)
+#endif
+#else
+/*
+ * Without hugetlb, gigantic folios that are bigger than a single PUD are
+ * currently impossible.
+ */
+#define MAX_FOLIO_ORDER		(PUD_SHIFT - PAGE_SHIFT)
+#endif
+
+#define MAX_FOLIO_NR_PAGES	(1UL << MAX_FOLIO_ORDER)
+
+/*
+ * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to
+ * be naturally aligned with regard to the folio size.
+ *
+ * HVO which is only active if the size of struct page is a power of 2.
+ */
+#define MAX_FOLIO_VMEMMAP_ALIGN \
+	(IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \
+	 is_power_of_2(sizeof(struct page)) ? \
+	 MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
+
+/*
+ * vmemmap optimization (like HVO) is only possible for page orders that fill
+ * two or more pages with struct pages.
+ */
+#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page)))
+#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1)
+#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0)
+
 enum migratetype {
 	MIGRATE_UNMOVABLE,
 	MIGRATE_MOVABLE,
@@ -220,6 +274,7 @@ enum node_stat_item {
 	NR_KERNEL_MISC_RECLAIMABLE,	/* reclaimable non-slab kernel pages */
 	NR_FOLL_PIN_ACQUIRED,	/* via: pin_user_page(), gup flag: FOLL_PIN */
 	NR_FOLL_PIN_RELEASED,	/* pages returned via unpin_user_page() */
+	NR_VMALLOC,
 	NR_KERNEL_STACK_KB,	/* measured in KiB */
 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
 	NR_KERNEL_SCS_KB,	/* measured in KiB */
@@ -255,6 +310,19 @@ enum node_stat_item {
 	PGDEMOTE_DIRECT,
 	PGDEMOTE_KHUGEPAGED,
 	PGDEMOTE_PROACTIVE,
+	PGSTEAL_KSWAPD,
+	PGSTEAL_DIRECT,
+	PGSTEAL_KHUGEPAGED,
+	PGSTEAL_PROACTIVE,
+	PGSTEAL_ANON,
+	PGSTEAL_FILE,
+	PGSCAN_KSWAPD,
+	PGSCAN_DIRECT,
+	PGSCAN_KHUGEPAGED,
+	PGSCAN_PROACTIVE,
+	PGSCAN_ANON,
+	PGSCAN_FILE,
+	PGREFILL,
 #ifdef CONFIG_HUGETLB_PAGE
 	NR_HUGETLB,
 #endif
@@ -616,7 +684,7 @@ struct lru_gen_memcg {
 
 void lru_gen_init_pgdat(struct pglist_data *pgdat);
 void lru_gen_init_lruvec(struct lruvec *lruvec);
-bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
+bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
 
 void lru_gen_init_memcg(struct mem_cgroup *memcg);
 void lru_gen_exit_memcg(struct mem_cgroup *memcg);
@@ -624,6 +692,9 @@ void lru_gen_online_memcg(struct mem_cgroup *memcg);
 void lru_gen_offline_memcg(struct mem_cgroup *memcg);
 void lru_gen_release_memcg(struct mem_cgroup *memcg);
 void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid);
+void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid);
+bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid);
+void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
 
 #else /* !CONFIG_LRU_GEN */
 
@@ -635,7 +706,8 @@ static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
 {
 }
 
-static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw,
+		unsigned int nr)
 {
 	return false;
 }
@@ -664,6 +736,20 @@ static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
 {
 }
 
+static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid)
+{
+}
+
+static inline bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid)
+{
+	return true;
+}
+
+static inline
+void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
+{
+}
+
 #endif /* CONFIG_LRU_GEN */
 
 struct lruvec {
@@ -952,12 +1038,12 @@ struct zone {
 	 * Locking rules:
 	 *
 	 * zone_start_pfn and spanned_pages are protected by span_seqlock.
-	 * It is a seqlock because it has to be read outside of zone->lock,
+	 * It is a seqlock because it has to be read outside of zone_lock,
 	 * and it is done in the main allocator path.  But, it is written
 	 * quite infrequently.
 	 *
-	 * The span_seq lock is declared along with zone->lock because it is
-	 * frequently read in proximity to zone->lock.  It's good to
+	 * The span_seq lock is declared along with zone_lock because it is
+	 * frequently read in proximity to zone_lock.  It's good to
 	 * give them a chance of being in the same cacheline.
 	 *
 	 * Write access to present_pages at runtime should be protected by
@@ -980,7 +1066,7 @@ struct zone {
 	/*
 	 * Number of isolated pageblock. It is used to solve incorrect
 	 * freepage counting problem due to racy retrieving migratetype
-	 * of pageblock. Protected by zone->lock.
+	 * of pageblock. Protected by zone_lock.
 	 */
 	unsigned long		nr_isolate_pageblock;
 #endif
@@ -1009,8 +1095,11 @@ struct zone {
 	/* zone flags, see below */
 	unsigned long		flags;
 
-	/* Primarily protects free_area */
-	spinlock_t		lock;
+	/*
+	 * Primarily protects free_area. Should be accessed via zone_lock_*
+	 * helpers.
+	 */
+	spinlock_t		_lock;
 
 	/* Pages to be freed when next trylock succeeds */
 	struct llist_head	trylock_free_pages;
@@ -1057,6 +1146,9 @@ struct zone {
 	/* Zone statistics */
 	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
 	atomic_long_t		vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+	struct page *vmemmap_tails[NR_VMEMMAP_TAILS];
+#endif
 } ____cacheline_internodealigned_in_smp;
 
 enum pgdat_flags {
@@ -1411,7 +1503,7 @@ typedef struct pglist_data {
 	 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG
 	 * or CONFIG_DEFERRED_STRUCT_PAGE_INIT.
 	 *
-	 * Nests above zone->lock and zone->span_seqlock
+	 * Nests above zone_lock and zone->span_seqlock
 	 */
 	spinlock_t node_size_lock;
 #endif
diff --git a/include/linux/mmzone_lock.h b/include/linux/mmzone_lock.h
new file mode 100644
index 0000000000000..6bd8b026029f5
--- /dev/null
+++ b/include/linux/mmzone_lock.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_MMZONE_LOCK_H
+#define _LINUX_MMZONE_LOCK_H
+
+#include <linux/mmzone.h>
+#include <linux/spinlock.h>
+#include <linux/tracepoint-defs.h>
+
+DECLARE_TRACEPOINT(zone_lock_start_locking);
+DECLARE_TRACEPOINT(zone_lock_acquire_returned);
+DECLARE_TRACEPOINT(zone_lock_released);
+
+#ifdef CONFIG_TRACING
+
+void __zone_lock_do_trace_start_locking(struct zone *zone);
+void __zone_lock_do_trace_acquire_returned(struct zone *zone, bool success);
+void __zone_lock_do_trace_released(struct zone *zone);
+
+static inline void __zone_lock_trace_start_locking(struct zone *zone)
+{
+	if (tracepoint_enabled(zone_lock_start_locking))
+		__zone_lock_do_trace_start_locking(zone);
+}
+
+static inline void __zone_lock_trace_acquire_returned(struct zone *zone,
+						      bool success)
+{
+	if (tracepoint_enabled(zone_lock_acquire_returned))
+		__zone_lock_do_trace_acquire_returned(zone, success);
+}
+
+static inline void __zone_lock_trace_released(struct zone *zone)
+{
+	if (tracepoint_enabled(zone_lock_released))
+		__zone_lock_do_trace_released(zone);
+}
+
+#else /* !CONFIG_TRACING */
+
+static inline void __zone_lock_trace_start_locking(struct zone *zone)
+{
+}
+
+static inline void __zone_lock_trace_acquire_returned(struct zone *zone,
+						      bool success)
+{
+}
+
+static inline void __zone_lock_trace_released(struct zone *zone)
+{
+}
+
+#endif /* CONFIG_TRACING */
+
+static inline void zone_lock_init(struct zone *zone)
+{
+	spin_lock_init(&zone->_lock);
+}
+
+#define zone_lock_irqsave(zone, flags)				\
+do {								\
+	bool success = true;					\
+								\
+	__zone_lock_trace_start_locking(zone);			\
+	spin_lock_irqsave(&(zone)->_lock, flags);		\
+	__zone_lock_trace_acquire_returned(zone, success);	\
+} while (0)
+
+#define zone_trylock_irqsave(zone, flags)			\
+({								\
+	bool success;						\
+								\
+	__zone_lock_trace_start_locking(zone);			\
+	success = spin_trylock_irqsave(&(zone)->_lock, flags);	\
+	__zone_lock_trace_acquire_returned(zone, success);	\
+	success;						\
+})
+
+static inline void zone_unlock_irqrestore(struct zone *zone, unsigned long flags)
+{
+	__zone_lock_trace_released(zone);
+	spin_unlock_irqrestore(&zone->_lock, flags);
+}
+
+static inline void zone_lock_irq(struct zone *zone)
+{
+	bool success = true;
+
+	__zone_lock_trace_start_locking(zone);
+	spin_lock_irq(&zone->_lock);
+	__zone_lock_trace_acquire_returned(zone, success);
+}
+
+static inline void zone_unlock_irq(struct zone *zone)
+{
+	__zone_lock_trace_released(zone);
+	spin_unlock_irq(&zone->_lock);
+}
+
+#endif /* _LINUX_MMZONE_LOCK_H */
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index 5b1725fe97076..23ff24080dfd9 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -691,6 +691,7 @@ struct x86_cpu_id {
 	__u16 feature;	/* bit index */
 	/* Solely for kernel-internal use: DO NOT EXPORT to userspace! */
 	__u16 flags;
+	__u8  platform_mask;
 	__u8  type;
 	kernel_ulong_t driver_data;
 };
@@ -702,6 +703,7 @@ struct x86_cpu_id {
 #define X86_STEPPING_ANY 0
 #define X86_STEP_MIN 0
 #define X86_STEP_MAX 0xf
+#define X86_PLATFORM_ANY 0x0
 #define X86_FEATURE_ANY 0	/* Same as FPU, you can't test for that */
 #define X86_CPU_TYPE_ANY 0
 
diff --git a/include/linux/mod_lineinfo.h b/include/linux/mod_lineinfo.h
new file mode 100644
index 0000000000000..364e5d81fe5bb
--- /dev/null
+++ b/include/linux/mod_lineinfo.h
@@ -0,0 +1,243 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * mod_lineinfo.h - Binary format for per-module source line information
+ *
+ * This header defines the layout of the .mod_lineinfo section embedded
+ * in loadable kernel modules.  It is dual-use: included from both the
+ * kernel and the userspace gen_lineinfo tool.
+ *
+ * Section layout (all values in target-native endianness):
+ *
+ *   struct mod_lineinfo_header
+ *   u32 block_addrs[num_blocks]    -- first addr per block, for binary search
+ *   u32 block_offsets[num_blocks]  -- byte offset into compressed data stream
+ *   u8  data[data_size]            -- LEB128 delta-compressed entries
+ *   u32 file_offsets[num_files]    -- byte offset into filenames[]
+ *   char filenames[filenames_size] -- concatenated NUL-terminated strings
+ *
+ * Each sub-array is located by an explicit (offset, size) pair in the
+ * header, similar to a flattened devicetree.  This makes bounds checking
+ * straightforward: validate offset + size <= section_size for each array.
+ *
+ * Compressed stream format (per block of LINEINFO_BLOCK_ENTRIES entries):
+ *   Entry 0: file_id (ULEB128), line (ULEB128)
+ *            addr is in block_addrs[]
+ *   Entry 1..N: addr_delta (ULEB128),
+ *               file_id_delta (SLEB128),
+ *               line_delta (SLEB128)
+ */
+#ifndef _LINUX_MOD_LINEINFO_H
+#define _LINUX_MOD_LINEINFO_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint8_t  u8;
+#endif
+
+#define LINEINFO_BLOCK_ENTRIES 64
+
+struct mod_lineinfo_header {
+	u32 num_entries;
+	u32 num_blocks;
+	u32 num_files;
+	u32 blocks_offset;	/* offset to block_addrs[] from section start */
+	u32 blocks_size;	/* bytes: num_blocks * 2 * sizeof(u32) */
+	u32 data_offset;	/* offset to compressed stream */
+	u32 data_size;		/* bytes of compressed data */
+	u32 files_offset;	/* offset to file_offsets[] */
+	u32 files_size;		/* bytes: num_files * sizeof(u32) */
+	u32 filenames_offset;
+	u32 filenames_size;
+	u32 reserved;		/* must be 0 */
+};
+
+/*
+ * Descriptor for a lineinfo table, used by the shared lookup function.
+ * Callers populate this from either linker globals (vmlinux) or a
+ * validated mod_lineinfo_header (modules).
+ */
+struct lineinfo_table {
+	const u32 *blk_addrs;
+	const u32 *blk_offsets;
+	const u8  *data;
+	u32 data_size;
+	const u32 *file_offsets;
+	const char *filenames;
+	u32 num_entries;
+	u32 num_blocks;
+	u32 num_files;
+	u32 filenames_size;
+};
+
+/*
+ * Read a ULEB128 varint from a byte stream.
+ * Returns the decoded value and advances *pos past the encoded bytes.
+ * If *pos would exceed 'end', returns 0 and sets *pos = end (safe for
+ * NMI/panic context: no crash, just a missed annotation).
+ */
+static inline u32 lineinfo_read_uleb128(const u8 *data, u32 *pos, u32 end)
+{
+	u32 result = 0;
+	unsigned int shift = 0;
+
+	while (*pos < end) {
+		u8 byte = data[*pos];
+		(*pos)++;
+		result |= (u32)(byte & 0x7f) << shift;
+		if (!(byte & 0x80))
+			return result;
+		shift += 7;
+		if (shift >= 32) {
+			/* Malformed: skip remaining continuation bytes */
+			while (*pos < end && (data[*pos] & 0x80))
+				(*pos)++;
+			if (*pos < end)
+				(*pos)++;
+			return result;
+		}
+	}
+	return result;
+}
+
+/* Read an SLEB128 varint. Same safety guarantees as above. */
+static inline int32_t lineinfo_read_sleb128(const u8 *data, u32 *pos, u32 end)
+{
+	int32_t result = 0;
+	unsigned int shift = 0;
+	u8 byte = 0;
+
+	while (*pos < end) {
+		byte = data[*pos];
+		(*pos)++;
+		result |= (int32_t)(byte & 0x7f) << shift;
+		shift += 7;
+		if (!(byte & 0x80))
+			break;
+		if (shift >= 32) {
+			while (*pos < end && (data[*pos] & 0x80))
+				(*pos)++;
+			if (*pos < end)
+				(*pos)++;
+			return result;
+		}
+	}
+
+	/* Sign-extend if the high bit of the last byte was set */
+	if (shift < 32 && (byte & 0x40))
+		result |= -(1 << shift);
+
+	return result;
+}
+
+/*
+ * Search a lineinfo table for the source file and line corresponding to a
+ * given offset (from _text for vmlinux, from .text base for modules).
+ *
+ * Safe for NMI and panic context: no locks, no allocations, all state on stack.
+ * Returns true and sets @file and @line on success; false on any failure.
+ */
+static inline bool lineinfo_search(const struct lineinfo_table *tbl,
+				   unsigned int offset,
+				   const char **file, unsigned int *line)
+{
+	unsigned int low, high, mid, block;
+	unsigned int cur_addr, cur_file_id, cur_line;
+	unsigned int best_file_id = 0, best_line = 0;
+	unsigned int block_entries, data_end;
+	bool found = false;
+	u32 pos;
+
+	if (!tbl->num_entries || !tbl->num_blocks)
+		return false;
+
+	/* Binary search on blk_addrs[] to find the right block */
+	low = 0;
+	high = tbl->num_blocks;
+	while (low < high) {
+		mid = low + (high - low) / 2;
+		if (tbl->blk_addrs[mid] <= offset)
+			low = mid + 1;
+		else
+			high = mid;
+	}
+
+	if (low == 0)
+		return false;
+	block = low - 1;
+
+	/* How many entries in this block? */
+	block_entries = LINEINFO_BLOCK_ENTRIES;
+	if (block == tbl->num_blocks - 1) {
+		unsigned int remaining = tbl->num_entries -
+					block * LINEINFO_BLOCK_ENTRIES;
+
+		if (remaining < block_entries)
+			block_entries = remaining;
+	}
+
+	/* Determine end of this block's data in the compressed stream */
+	if (block + 1 < tbl->num_blocks)
+		data_end = tbl->blk_offsets[block + 1];
+	else
+		data_end = tbl->data_size;
+
+	/* Clamp data_end to actual data size */
+	if (data_end > tbl->data_size)
+		data_end = tbl->data_size;
+
+	/* Decode entry 0: addr from blk_addrs, file_id and line from stream */
+	pos = tbl->blk_offsets[block];
+	if (pos >= data_end)
+		return false;
+
+	cur_addr = tbl->blk_addrs[block];
+	cur_file_id = lineinfo_read_uleb128(tbl->data, &pos, data_end);
+	cur_line = lineinfo_read_uleb128(tbl->data, &pos, data_end);
+
+	/* Check entry 0 */
+	if (cur_addr <= offset) {
+		best_file_id = cur_file_id;
+		best_line = cur_line;
+		found = true;
+	}
+
+	/* Decode entries 1..N */
+	for (unsigned int i = 1; i < block_entries; i++) {
+		unsigned int addr_delta;
+		int32_t file_delta, line_delta;
+
+		addr_delta = lineinfo_read_uleb128(tbl->data, &pos, data_end);
+		file_delta = lineinfo_read_sleb128(tbl->data, &pos, data_end);
+		line_delta = lineinfo_read_sleb128(tbl->data, &pos, data_end);
+
+		cur_addr += addr_delta;
+		cur_file_id = (unsigned int)((int32_t)cur_file_id + file_delta);
+		cur_line = (unsigned int)((int32_t)cur_line + line_delta);
+
+		if (cur_addr > offset)
+			break;
+
+		best_file_id = cur_file_id;
+		best_line = cur_line;
+		found = true;
+	}
+
+	if (!found)
+		return false;
+
+	if (best_file_id >= tbl->num_files)
+		return false;
+
+	if (tbl->file_offsets[best_file_id] >= tbl->filenames_size)
+		return false;
+
+	*file = &tbl->filenames[tbl->file_offsets[best_file_id]];
+	*line = best_line;
+	return true;
+}
+
+#endif /* _LINUX_MOD_LINEINFO_H */
diff --git a/include/linux/module.h b/include/linux/module.h
index 14f391b186c6d..0c7024fd0ab44 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -413,6 +413,7 @@ struct module {
 	struct module_attribute *modinfo_attrs;
 	const char *version;
 	const char *srcversion;
+	const char *imported_namespaces;
 	struct kobject *holders_dir;
 
 	/* Exported symbols */
@@ -508,6 +509,8 @@ struct module {
 	void *btf_data;
 	void *btf_base_data;
 #endif
+	void *lineinfo_data;		/* .mod_lineinfo section in MOD_RODATA */
+	unsigned int lineinfo_data_size;
 #ifdef CONFIG_JUMP_LABEL
 	struct jump_entry *jump_entries;
 	unsigned int num_jump_entries;
@@ -1021,6 +1024,9 @@ static inline unsigned long find_kallsyms_symbol_value(struct module *mod,
 
 #endif  /* CONFIG_MODULES && CONFIG_KALLSYMS */
 
+bool module_lookup_lineinfo(struct module *mod, unsigned long addr,
+			    const char **file, unsigned int *line);
+
 /* Define __free(module_put) macro for struct module *. */
 DEFINE_FREE(module_put, struct module *, if (_T) module_put(_T))
 
diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h
index 7d22d4c4ea2e7..075f28585074f 100644
--- a/include/linux/moduleparam.h
+++ b/include/linux/moduleparam.h
@@ -317,8 +317,8 @@ struct kparam_array
 			    name, &__param_ops_##name, arg, perm, -1, 0)
 
 #ifdef CONFIG_SYSFS
-extern void kernel_param_lock(struct module *mod);
-extern void kernel_param_unlock(struct module *mod);
+void kernel_param_lock(struct module *mod);
+void kernel_param_unlock(struct module *mod);
 #else
 static inline void kernel_param_lock(struct module *mod)
 {
@@ -398,7 +398,7 @@ static inline void kernel_param_unlock(struct module *mod)
  * Returns: true if the two parameter names are equal.
  * Dashes (-) are considered equal to underscores (_).
  */
-extern bool parameq(const char *name1, const char *name2);
+bool parameq(const char *name1, const char *name2);
 
 /**
  * parameqn - checks if two parameter names match
@@ -412,28 +412,23 @@ extern bool parameq(const char *name1, const char *name2);
  * are equal.
  * Dashes (-) are considered equal to underscores (_).
  */
-extern bool parameqn(const char *name1, const char *name2, size_t n);
+bool parameqn(const char *name1, const char *name2, size_t n);
 
 typedef int (*parse_unknown_fn)(char *param, char *val, const char *doing, void *arg);
 
 /* Called on module insert or kernel boot */
-extern char *parse_args(const char *name,
-		      char *args,
-		      const struct kernel_param *params,
-		      unsigned num,
-		      s16 level_min,
-		      s16 level_max,
-		      void *arg, parse_unknown_fn unknown);
+char *parse_args(const char *doing,
+		 char *args,
+		 const struct kernel_param *params,
+		 unsigned int num,
+		 s16 min_level,
+		 s16 max_level,
+		 void *arg, parse_unknown_fn unknown);
 
 /* Called by module remove. */
-#ifdef CONFIG_SYSFS
-extern void destroy_params(const struct kernel_param *params, unsigned num);
-#else
-static inline void destroy_params(const struct kernel_param *params,
-				  unsigned num)
-{
-}
-#endif /* !CONFIG_SYSFS */
+#ifdef CONFIG_MODULES
+void module_destroy_params(const struct kernel_param *params, unsigned int num);
+#endif
 
 /* All the helper functions */
 /* The macros to do compile-time type checking stolen from Jakub
@@ -442,78 +437,77 @@ static inline void destroy_params(const struct kernel_param *params,
 	static inline type __always_unused *__check_##name(void) { return(p); }
 
 extern const struct kernel_param_ops param_ops_byte;
-extern int param_set_byte(const char *val, const struct kernel_param *kp);
-extern int param_get_byte(char *buffer, const struct kernel_param *kp);
+int param_set_byte(const char *val, const struct kernel_param *kp);
+int param_get_byte(char *buffer, const struct kernel_param *kp);
 #define param_check_byte(name, p) __param_check(name, p, unsigned char)
 
 extern const struct kernel_param_ops param_ops_short;
-extern int param_set_short(const char *val, const struct kernel_param *kp);
-extern int param_get_short(char *buffer, const struct kernel_param *kp);
+int param_set_short(const char *val, const struct kernel_param *kp);
+int param_get_short(char *buffer, const struct kernel_param *kp);
 #define param_check_short(name, p) __param_check(name, p, short)
 
 extern const struct kernel_param_ops param_ops_ushort;
-extern int param_set_ushort(const char *val, const struct kernel_param *kp);
-extern int param_get_ushort(char *buffer, const struct kernel_param *kp);
+int param_set_ushort(const char *val, const struct kernel_param *kp);
+int param_get_ushort(char *buffer, const struct kernel_param *kp);
 #define param_check_ushort(name, p) __param_check(name, p, unsigned short)
 
 extern const struct kernel_param_ops param_ops_int;
-extern int param_set_int(const char *val, const struct kernel_param *kp);
-extern int param_get_int(char *buffer, const struct kernel_param *kp);
+int param_set_int(const char *val, const struct kernel_param *kp);
+int param_get_int(char *buffer, const struct kernel_param *kp);
 #define param_check_int(name, p) __param_check(name, p, int)
 
 extern const struct kernel_param_ops param_ops_uint;
-extern int param_set_uint(const char *val, const struct kernel_param *kp);
-extern int param_get_uint(char *buffer, const struct kernel_param *kp);
+int param_set_uint(const char *val, const struct kernel_param *kp);
+int param_get_uint(char *buffer, const struct kernel_param *kp);
 int param_set_uint_minmax(const char *val, const struct kernel_param *kp,
 		unsigned int min, unsigned int max);
 #define param_check_uint(name, p) __param_check(name, p, unsigned int)
 
 extern const struct kernel_param_ops param_ops_long;
-extern int param_set_long(const char *val, const struct kernel_param *kp);
-extern int param_get_long(char *buffer, const struct kernel_param *kp);
+int param_set_long(const char *val, const struct kernel_param *kp);
+int param_get_long(char *buffer, const struct kernel_param *kp);
 #define param_check_long(name, p) __param_check(name, p, long)
 
 extern const struct kernel_param_ops param_ops_ulong;
-extern int param_set_ulong(const char *val, const struct kernel_param *kp);
-extern int param_get_ulong(char *buffer, const struct kernel_param *kp);
+int param_set_ulong(const char *val, const struct kernel_param *kp);
+int param_get_ulong(char *buffer, const struct kernel_param *kp);
 #define param_check_ulong(name, p) __param_check(name, p, unsigned long)
 
 extern const struct kernel_param_ops param_ops_ullong;
-extern int param_set_ullong(const char *val, const struct kernel_param *kp);
-extern int param_get_ullong(char *buffer, const struct kernel_param *kp);
+int param_set_ullong(const char *val, const struct kernel_param *kp);
+int param_get_ullong(char *buffer, const struct kernel_param *kp);
 #define param_check_ullong(name, p) __param_check(name, p, unsigned long long)
 
 extern const struct kernel_param_ops param_ops_hexint;
-extern int param_set_hexint(const char *val, const struct kernel_param *kp);
-extern int param_get_hexint(char *buffer, const struct kernel_param *kp);
+int param_set_hexint(const char *val, const struct kernel_param *kp);
+int param_get_hexint(char *buffer, const struct kernel_param *kp);
 #define param_check_hexint(name, p) param_check_uint(name, p)
 
 extern const struct kernel_param_ops param_ops_charp;
-extern int param_set_charp(const char *val, const struct kernel_param *kp);
-extern int param_get_charp(char *buffer, const struct kernel_param *kp);
-extern void param_free_charp(void *arg);
+int param_set_charp(const char *val, const struct kernel_param *kp);
+int param_get_charp(char *buffer, const struct kernel_param *kp);
+void param_free_charp(void *arg);
 #define param_check_charp(name, p) __param_check(name, p, char *)
 
 /* We used to allow int as well as bool.  We're taking that away! */
 extern const struct kernel_param_ops param_ops_bool;
-extern int param_set_bool(const char *val, const struct kernel_param *kp);
-extern int param_get_bool(char *buffer, const struct kernel_param *kp);
+int param_set_bool(const char *val, const struct kernel_param *kp);
+int param_get_bool(char *buffer, const struct kernel_param *kp);
 #define param_check_bool(name, p) __param_check(name, p, bool)
 
 extern const struct kernel_param_ops param_ops_bool_enable_only;
-extern int param_set_bool_enable_only(const char *val,
-				      const struct kernel_param *kp);
+int param_set_bool_enable_only(const char *val, const struct kernel_param *kp);
 /* getter is the same as for the regular bool */
 #define param_check_bool_enable_only param_check_bool
 
 extern const struct kernel_param_ops param_ops_invbool;
-extern int param_set_invbool(const char *val, const struct kernel_param *kp);
-extern int param_get_invbool(char *buffer, const struct kernel_param *kp);
+int param_set_invbool(const char *val, const struct kernel_param *kp);
+int param_get_invbool(char *buffer, const struct kernel_param *kp);
 #define param_check_invbool(name, p) __param_check(name, p, bool)
 
 /* An int, which can only be set like a bool (though it shows as an int). */
 extern const struct kernel_param_ops param_ops_bint;
-extern int param_set_bint(const char *val, const struct kernel_param *kp);
+int param_set_bint(const char *val, const struct kernel_param *kp);
 #define param_get_bint param_get_int
 #define param_check_bint param_check_int
 
@@ -620,19 +614,19 @@ enum hwparam_type {
 extern const struct kernel_param_ops param_array_ops;
 
 extern const struct kernel_param_ops param_ops_string;
-extern int param_set_copystring(const char *val, const struct kernel_param *);
-extern int param_get_string(char *buffer, const struct kernel_param *kp);
+int param_set_copystring(const char *val, const struct kernel_param *kp);
+int param_get_string(char *buffer, const struct kernel_param *kp);
 
 /* for exporting parameters in /sys/module/.../parameters */
 
 struct module;
 
 #if defined(CONFIG_SYSFS) && defined(CONFIG_MODULES)
-extern int module_param_sysfs_setup(struct module *mod,
-				    const struct kernel_param *kparam,
-				    unsigned int num_params);
+int module_param_sysfs_setup(struct module *mod,
+			     const struct kernel_param *kparam,
+			     unsigned int num_params);
 
-extern void module_param_sysfs_remove(struct module *mod);
+void module_param_sysfs_remove(struct module *mod);
 #else
 static inline int module_param_sysfs_setup(struct module *mod,
 			     const struct kernel_param *kparam,
diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h
index 0075f6e5c3da9..cf3374580f744 100644
--- a/include/linux/mroute_base.h
+++ b/include/linux/mroute_base.h
@@ -76,7 +76,7 @@ static inline int mr_call_vif_notifiers(struct net *net,
 					struct vif_device *vif,
 					struct net_device *vif_dev,
 					unsigned short vif_index, u32 tb_id,
-					unsigned int *ipmr_seq)
+					atomic_t *ipmr_seq)
 {
 	struct vif_entry_notifier_info info = {
 		.info = {
@@ -89,7 +89,7 @@ static inline int mr_call_vif_notifiers(struct net *net,
 	};
 
 	ASSERT_RTNL();
-	(*ipmr_seq)++;
+	atomic_inc(ipmr_seq);
 	return call_fib_notifiers(net, event_type, &info.info);
 }
 
@@ -198,7 +198,7 @@ static inline int mr_call_mfc_notifiers(struct net *net,
 					unsigned short family,
 					enum fib_event_type event_type,
 					struct mr_mfc *mfc, u32 tb_id,
-					unsigned int *ipmr_seq)
+					atomic_t *ipmr_seq)
 {
 	struct mfc_entry_notifier_info info = {
 		.info = {
@@ -208,8 +208,7 @@ static inline int mr_call_mfc_notifiers(struct net *net,
 		.tb_id = tb_id
 	};
 
-	ASSERT_RTNL();
-	(*ipmr_seq)++;
+	atomic_inc(ipmr_seq);
 	return call_fib_notifiers(net, event_type, &info.info);
 }
 
diff --git a/include/linux/mtd/concat.h b/include/linux/mtd/concat.h
index d6f653e074263..f8d4d6ac1fc1d 100644
--- a/include/linux/mtd/concat.h
+++ b/include/linux/mtd/concat.h
@@ -9,6 +9,18 @@
 #define MTD_CONCAT_H
 
 
+/*
+ * Our storage structure:
+ * Subdev points to an array of pointers to struct mtd_info objects
+ * which is allocated along with this structure
+ *
+ */
+struct mtd_concat {
+	struct mtd_info mtd;
+	int num_subdev;
+	struct mtd_info *subdev[];
+};
+
 struct mtd_info *mtd_concat_create(
     struct mtd_info *subdev[],  /* subdevices to concatenate */
     int num_devs,               /* number of subdevices      */
@@ -16,5 +28,54 @@ struct mtd_info *mtd_concat_create(
 
 void mtd_concat_destroy(struct mtd_info *mtd);
 
-#endif
+/**
+ * mtd_virt_concat_node_create - Create a component for concatenation
+ *
+ * Returns a positive number representing the no. of devices found for
+ * concatenation, or a negative error code.
+ *
+ * List all the devices for concatenations found in DT and create a
+ * component for concatenation.
+ */
+int mtd_virt_concat_node_create(void);
+
+/**
+ * mtd_virt_concat_add - add mtd_info object to the list of subdevices for concatenation
+ * @mtd: pointer to new MTD device info structure
+ *
+ * Returns true if the mtd_info object is added successfully else returns false.
+ *
+ * The mtd_info object is added to the list of subdevices for concatenation.
+ * It returns true if a match is found, and false if all subdevices have
+ * already been added or if the mtd_info object does not match any of the
+ * intended MTD devices.
+ */
+bool mtd_virt_concat_add(struct mtd_info *mtd);
 
+/**
+ * mtd_virt_concat_create_join - Create and register the concatenated MTD device
+ *
+ * Returns 0 on succes, or a negative error code.
+ *
+ * Creates and registers the concatenated MTD device
+ */
+int mtd_virt_concat_create_join(void);
+
+/**
+ * mtd_virt_concat_destroy - Remove the concat that includes a specific mtd device
+ *                           as one of its components.
+ * @mtd: pointer to MTD device info structure.
+ *
+ * Returns 0 on succes, or a negative error code.
+ *
+ * If the mtd_info object is part of a concatenated device, all other MTD devices
+ * within that concat are registered individually. The concatenated device is then
+ * removed, along with its concatenation component.
+ *
+ */
+int mtd_virt_concat_destroy(struct mtd_info *mtd);
+
+void mtd_virt_concat_destroy_joins(void);
+void mtd_virt_concat_destroy_items(void);
+
+#endif
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index 6a024cf1c53ac..58abd306ebe39 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -477,8 +477,9 @@ struct spinand_ecc_info {
 	const struct mtd_ooblayout_ops *ooblayout;
 };
 
-#define SPINAND_HAS_QE_BIT		BIT(0)
-#define SPINAND_HAS_CR_FEAT_BIT		BIT(1)
+/* SPI NAND flags */
+#define SPINAND_HAS_QE_BIT				BIT(0)
+#define SPINAND_HAS_CR_FEAT_BIT				BIT(1)
 #define SPINAND_HAS_PROG_PLANE_SELECT_BIT		BIT(2)
 #define SPINAND_HAS_READ_PLANE_SELECT_BIT		BIT(3)
 #define SPINAND_NO_RAW_ACCESS				BIT(4)
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index ecaa0440f6ec4..734048c02f4f5 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -79,7 +79,7 @@ do {									\
 #define __MUTEX_INITIALIZER(lockname) \
 		{ .owner = ATOMIC_LONG_INIT(0) \
 		, .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
-		, .wait_list = LIST_HEAD_INIT(lockname.wait_list) \
+		, .first_waiter = NULL \
 		__DEBUG_MUTEX_INITIALIZER(lockname) \
 		__DEP_MAP_MUTEX_INITIALIZER(lockname) }
 
@@ -87,12 +87,12 @@ do {									\
 	struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
-void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key);
+void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key);
 
 static inline void __mutex_init(struct mutex *lock, const char *name,
 				struct lock_class_key *key)
 {
-	mutex_init_lockep(lock, name, key);
+	mutex_init_lockdep(lock, name, key);
 }
 #else
 extern void mutex_init_generic(struct mutex *lock);
@@ -146,7 +146,7 @@ static inline void __mutex_init(struct mutex *lock, const char *name,
 {
 	mutex_rt_init_generic(lock);
 }
-#endif /* !CONFIG_LOCKDEP */
+#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
 #endif /* CONFIG_PREEMPT_RT */
 
 #ifdef CONFIG_DEBUG_MUTEXES
@@ -183,7 +183,7 @@ static inline int __must_check __devm_mutex_init(struct device *dev, struct mute
  */
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass) __acquires(lock);
-extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
+extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) __acquires(lock);
 extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock,
 					unsigned int subclass) __cond_acquires(0, lock);
 extern int __must_check _mutex_lock_killable(struct mutex *lock,
@@ -253,6 +253,7 @@ extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) __cond_a
 DEFINE_LOCK_GUARD_1(mutex, struct mutex, mutex_lock(_T->lock), mutex_unlock(_T->lock))
 DEFINE_LOCK_GUARD_1_COND(mutex, _try, mutex_trylock(_T->lock))
 DEFINE_LOCK_GUARD_1_COND(mutex, _intr, mutex_lock_interruptible(_T->lock), _RET == 0)
+DEFINE_LOCK_GUARD_1_COND(mutex, _kill, mutex_lock_killable(_T->lock), _RET == 0)
 DEFINE_LOCK_GUARD_1(mutex_init, struct mutex, mutex_init(_T->lock), /* */)
 
 DECLARE_LOCK_GUARD_1_ATTRS(mutex,	__acquires(_T), __releases(*(struct mutex **)_T))
@@ -261,6 +262,8 @@ DECLARE_LOCK_GUARD_1_ATTRS(mutex_try,	__acquires(_T), __releases(*(struct mutex
 #define class_mutex_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_try, _T)
 DECLARE_LOCK_GUARD_1_ATTRS(mutex_intr,	__acquires(_T), __releases(*(struct mutex **)_T))
 #define class_mutex_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_intr, _T)
+DECLARE_LOCK_GUARD_1_ATTRS(mutex_kill,	__acquires(_T), __releases(*(struct mutex **)_T))
+#define class_mutex_kill_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_kill, _T)
 DECLARE_LOCK_GUARD_1_ATTRS(mutex_init,	__acquires(_T), __releases(*(struct mutex **)_T))
 #define class_mutex_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_init, _T)
 
diff --git a/include/linux/mutex_types.h b/include/linux/mutex_types.h
index 80975935ec488..24ed599fdda88 100644
--- a/include/linux/mutex_types.h
+++ b/include/linux/mutex_types.h
@@ -44,7 +44,7 @@ context_lock_struct(mutex) {
 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
 	struct optimistic_spin_queue osq; /* Spinner MCS lock */
 #endif
-	struct list_head	wait_list;
+	struct mutex_waiter	*first_waiter __guarded_by(&wait_lock);
 #ifdef CONFIG_DEBUG_MUTEXES
 	void			*magic;
 #endif
diff --git a/include/linux/mux/consumer.h b/include/linux/mux/consumer.h
index 2e25c838f8312..a961861a503b3 100644
--- a/include/linux/mux/consumer.h
+++ b/include/linux/mux/consumer.h
@@ -16,6 +16,8 @@ struct device;
 struct mux_control;
 struct mux_state;
 
+#if IS_ENABLED(CONFIG_MULTIPLEXER)
+
 unsigned int mux_control_states(struct mux_control *mux);
 int __must_check mux_control_select_delay(struct mux_control *mux,
 					  unsigned int state,
@@ -54,11 +56,109 @@ int mux_control_deselect(struct mux_control *mux);
 int mux_state_deselect(struct mux_state *mstate);
 
 struct mux_control *mux_control_get(struct device *dev, const char *mux_name);
+struct mux_control *mux_control_get_optional(struct device *dev, const char *mux_name);
 void mux_control_put(struct mux_control *mux);
 
-struct mux_control *devm_mux_control_get(struct device *dev,
-					 const char *mux_name);
-struct mux_state *devm_mux_state_get(struct device *dev,
-				     const char *mux_name);
+struct mux_control *devm_mux_control_get(struct device *dev, const char *mux_name);
+struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name);
+struct mux_state *devm_mux_state_get_optional(struct device *dev, const char *mux_name);
+struct mux_state *devm_mux_state_get_selected(struct device *dev, const char *mux_name);
+struct mux_state *devm_mux_state_get_optional_selected(struct device *dev, const char *mux_name);
+
+#else
+
+static inline unsigned int mux_control_states(struct mux_control *mux)
+{
+	return 0;
+}
+static inline int __must_check mux_control_select_delay(struct mux_control *mux,
+							unsigned int state, unsigned int delay_us)
+{
+	return -EOPNOTSUPP;
+}
+static inline int __must_check mux_state_select_delay(struct mux_state *mstate,
+						      unsigned int delay_us)
+{
+	return -EOPNOTSUPP;
+}
+static inline int __must_check mux_control_try_select_delay(struct mux_control *mux,
+							    unsigned int state,
+							    unsigned int delay_us)
+{
+	return -EOPNOTSUPP;
+}
+static inline int __must_check mux_state_try_select_delay(struct mux_state *mstate,
+							  unsigned int delay_us)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int __must_check mux_control_select(struct mux_control *mux,
+						  unsigned int state)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int __must_check mux_state_select(struct mux_state *mstate)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int __must_check mux_control_try_select(struct mux_control *mux,
+						      unsigned int state)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int __must_check mux_state_try_select(struct mux_state *mstate)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mux_control_deselect(struct mux_control *mux)
+{
+	return -EOPNOTSUPP;
+}
+static inline int mux_state_deselect(struct mux_state *mstate)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+static inline struct mux_control *mux_control_get_optional(struct device *dev,
+							   const char *mux_name)
+{
+	return NULL;
+}
+static inline void mux_control_put(struct mux_control *mux) {}
+
+static inline struct mux_control *devm_mux_control_get(struct device *dev, const char *mux_name)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+static inline struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+static inline struct mux_state *devm_mux_state_get_optional(struct device *dev,
+							    const char *mux_name)
+{
+	return NULL;
+}
+static inline struct mux_state *devm_mux_state_get_selected(struct device *dev,
+							    const char *mux_name)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+static inline struct mux_state *devm_mux_state_get_optional_selected(struct device *dev,
+								     const char *mux_name)
+{
+	return NULL;
+}
+
+#endif /* CONFIG_MULTIPLEXER */
 
 #endif /* _LINUX_MUX_CONSUMER_H */
diff --git a/include/linux/namei.h b/include/linux/namei.h
index 58600cf234bcf..2ad6dd9987b90 100644
--- a/include/linux/namei.h
+++ b/include/linux/namei.h
@@ -54,9 +54,6 @@ extern int path_pts(struct path *path);
 
 extern int user_path_at(int, const char __user *, unsigned, struct path *);
 
-struct dentry *lookup_one_qstr_excl(const struct qstr *name,
-				    struct dentry *base,
-				    unsigned int flags);
 extern int kern_path(const char *, unsigned, struct path *);
 struct dentry *kern_path_parent(const char *name, struct path *parent);
 
@@ -168,9 +165,6 @@ extern int follow_down_one(struct path *);
 extern int follow_down(struct path *path, unsigned int flags);
 extern int follow_up(struct path *);
 
-extern struct dentry *lock_rename(struct dentry *, struct dentry *);
-extern struct dentry *lock_rename_child(struct dentry *, struct dentry *);
-extern void unlock_rename(struct dentry *, struct dentry *);
 int start_renaming(struct renamedata *rd, int lookup_flags,
 		   struct qstr *old_last, struct qstr *new_last);
 int start_renaming_dentry(struct renamedata *rd, int lookup_flags,
diff --git a/include/linux/net.h b/include/linux/net.h
index f58b38ab37f8a..a8e818de95b33 100644
--- a/include/linux/net.h
+++ b/include/linux/net.h
@@ -304,6 +304,8 @@ do {									\
 
 #define net_get_random_once(buf, nbytes)			\
 	get_random_once((buf), (nbytes))
+#define net_get_random_sleepable_once(buf, nbytes)		\
+	get_random_sleepable_once((buf), (nbytes))
 
 /*
  * E.g. XFS meta- & log-data is in slab pages, or bcache meta
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index ae269a2e7f4dd..8527a9c713685 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -311,7 +311,9 @@ struct header_ops {
 	int	(*create) (struct sk_buff *skb, struct net_device *dev,
 			   unsigned short type, const void *daddr,
 			   const void *saddr, unsigned int len);
-	int	(*parse)(const struct sk_buff *skb, unsigned char *haddr);
+	int	(*parse)(const struct sk_buff *skb,
+			 const struct net_device *dev,
+			 unsigned char *haddr);
 	int	(*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
 	void	(*cache_update)(struct hh_cache *hh,
 				const struct net_device *dev,
@@ -2102,6 +2104,8 @@ enum netdev_reg_state {
  *
  *	FIXME: cleanup struct net_device such that network protocol info
  *	moves out.
+ *
+ *	@netdev_trace_buffer_list: Linked list for debugging refcount leak.
  */
 
 struct net_device {
@@ -2155,6 +2159,7 @@ struct net_device {
 	unsigned long		state;
 	unsigned int		flags;
 	unsigned short		hard_header_len;
+	enum netdev_stat_type	pcpu_stat_type:8;
 	netdev_features_t	features;
 	struct inet6_dev __rcu	*ip6_ptr;
 	__cacheline_group_end(net_device_read_txrx);
@@ -2257,6 +2262,9 @@ struct net_device {
 #if IS_ENABLED(CONFIG_TLS_DEVICE)
 	const struct tlsdev_ops *tlsdev_ops;
 #endif
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	struct list_head	netdev_trace_buffer_list;
+#endif
 
 	unsigned int		operstate;
 	unsigned char		link_mode;
@@ -2404,8 +2412,6 @@ struct net_device {
 	void				*ml_priv;
 	enum netdev_ml_priv_type	ml_priv_type;
 
-	enum netdev_stat_type		pcpu_stat_type:8;
-
 #if IS_ENABLED(CONFIG_GARP)
 	struct garp_port __rcu	*garp_port;
 #endif
@@ -3185,6 +3191,7 @@ enum netdev_cmd {
 	NETDEV_OFFLOAD_XSTATS_REPORT_USED,
 	NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
 	NETDEV_XDP_FEAT_CHANGE,
+	NETDEV_DEBUG_UNREGISTER,
 };
 const char *netdev_cmd_to_name(enum netdev_cmd cmd);
 
@@ -3446,7 +3453,7 @@ static inline int dev_parse_header(const struct sk_buff *skb,
 
 	if (!dev->header_ops || !dev->header_ops->parse)
 		return 0;
-	return dev->header_ops->parse(skb, haddr);
+	return dev->header_ops->parse(skb, dev, haddr);
 }
 
 static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
@@ -4405,9 +4412,15 @@ static inline bool dev_nit_active(const struct net_device *dev)
 
 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
 
+void save_netdev_trace_buffer(struct net_device *dev, int delta);
+int trim_netdev_trace(unsigned long *entries, int nr_entries);
+
 static inline void __dev_put(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, -1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_dec(*dev->pcpu_refcnt);
 #else
@@ -4419,6 +4432,9 @@ static inline void __dev_put(struct net_device *dev)
 static inline void __dev_hold(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, 1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_inc(*dev->pcpu_refcnt);
 #else
diff --git a/include/linux/nmi.h b/include/linux/nmi.h
index 207156f2143c5..bc1162895f355 100644
--- a/include/linux/nmi.h
+++ b/include/linux/nmi.h
@@ -21,6 +21,7 @@ void lockup_detector_soft_poweroff(void);
 extern int watchdog_user_enabled;
 extern int watchdog_thresh;
 extern unsigned long watchdog_enabled;
+extern int watchdog_hardlockup_miss_thresh;
 
 extern struct cpumask watchdog_cpumask;
 extern unsigned long *watchdog_cpumask_bits;
diff --git a/include/linux/of.h b/include/linux/of.h
index be6ec4916adf5..959786f8f1966 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -410,7 +410,7 @@ extern int of_alias_get_id(const struct device_node *np, const char *stem);
 extern int of_alias_get_highest_id(const char *stem);
 
 bool of_machine_compatible_match(const char *const *compats);
-bool of_machine_device_match(const struct of_device_id *matches);
+const struct of_device_id *of_machine_get_match(const struct of_device_id *matches);
 const void *of_machine_get_match_data(const struct of_device_id *matches);
 
 /**
@@ -426,6 +426,9 @@ static inline bool of_machine_is_compatible(const char *compat)
 	return of_machine_compatible_match(compats);
 }
 
+int of_machine_read_compatible(const char **compatible, unsigned int index);
+int of_machine_read_model(const char **model);
+
 extern int of_add_property(struct device_node *np, struct property *prop);
 extern int of_remove_property(struct device_node *np, struct property *prop);
 extern int of_update_property(struct device_node *np, struct property *newprop);
@@ -851,6 +854,17 @@ static inline int of_machine_is_compatible(const char *compat)
 	return 0;
 }
 
+static inline int of_machine_read_compatible(const char **compatible,
+					     unsigned int index)
+{
+	return -ENOSYS;
+}
+
+static inline int of_machine_read_model(const char **model)
+{
+	return -ENOSYS;
+}
+
 static inline int of_add_property(struct device_node *np, struct property *prop)
 {
 	return 0;
@@ -866,9 +880,9 @@ static inline bool of_machine_compatible_match(const char *const *compats)
 	return false;
 }
 
-static inline bool of_machine_device_match(const struct of_device_id *matches)
+static inline const struct of_device_id *of_machine_get_match(const struct of_device_id *matches)
 {
-	return false;
+	return NULL;
 }
 
 static inline const void *
@@ -976,6 +990,11 @@ static inline int of_numa_init(void)
 }
 #endif
 
+static inline bool of_machine_device_match(const struct of_device_id *matches)
+{
+	return of_machine_get_match(matches) != NULL;
+}
+
 static inline struct device_node *of_find_matching_node(
 	struct device_node *from,
 	const struct of_device_id *matches)
diff --git a/include/linux/of_dma.h b/include/linux/of_dma.h
index fd706cdf255c6..16b08234d03b3 100644
--- a/include/linux/of_dma.h
+++ b/include/linux/of_dma.h
@@ -38,6 +38,26 @@ extern int of_dma_controller_register(struct device_node *np,
 		void *data);
 extern void of_dma_controller_free(struct device_node *np);
 
+static void __of_dma_controller_free(void *np)
+{
+	of_dma_controller_free(np);
+}
+
+static inline int
+devm_of_dma_controller_register(struct device *dev, struct device_node *np,
+				struct dma_chan *(*of_dma_xlate)
+				(struct of_phandle_args *, struct of_dma *),
+				void *data)
+{
+	int ret;
+
+	ret = of_dma_controller_register(np, of_dma_xlate, data);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, __of_dma_controller_free, np);
+}
+
 extern int of_dma_router_register(struct device_node *np,
 		void *(*of_dma_route_allocate)
 		(struct of_phandle_args *, struct of_dma *),
@@ -64,6 +84,15 @@ static inline void of_dma_controller_free(struct device_node *np)
 {
 }
 
+static inline int
+devm_of_dma_controller_register(struct device *dev, struct device_node *np,
+				struct dma_chan *(*of_dma_xlate)
+				(struct of_phandle_args *, struct of_dma *),
+				void *data)
+{
+	return -ENODEV;
+}
+
 static inline int of_dma_router_register(struct device_node *np,
 		void *(*of_dma_route_allocate)
 		(struct of_phandle_args *, struct of_dma *),
diff --git a/include/linux/of_gpio.h b/include/linux/of_gpio.h
deleted file mode 100644
index d0f66a5e1b2a7..0000000000000
--- a/include/linux/of_gpio.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * OF helpers for the GPIO API
- *
- * Copyright (c) 2007-2008  MontaVista Software, Inc.
- *
- * Author: Anton Vorontsov <avorontsov@ru.mvista.com>
- */
-
-#ifndef __LINUX_OF_GPIO_H
-#define __LINUX_OF_GPIO_H
-
-#include <linux/compiler.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio.h>		/* FIXME: Shouldn't be here */
-#include <linux/of.h>
-
-struct device_node;
-
-#ifdef CONFIG_OF_GPIO
-
-extern int of_get_named_gpio(const struct device_node *np,
-			     const char *list_name, int index);
-
-#else /* CONFIG_OF_GPIO */
-
-#include <linux/errno.h>
-
-/* Drivers may not strictly depend on the GPIO support, so let them link. */
-static inline int of_get_named_gpio(const struct device_node *np,
-                                   const char *propname, int index)
-{
-	return -ENOSYS;
-}
-
-#endif /* CONFIG_OF_GPIO */
-
-#endif /* __LINUX_OF_GPIO_H */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index f7a0e4af0c734..0e03d816e8b9d 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -198,97 +198,91 @@ enum pageflags {
 
 #ifndef __GENERATING_BOUNDS_H
 
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
-DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
-
 /*
- * Return the real head page struct iff the @page is a fake head page, otherwise
- * return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
+ * For tail pages, if the size of struct page is power-of-2 ->compound_info
+ * encodes the mask that converts the address of the tail page address to
+ * the head page address.
+ *
+ * Otherwise, ->compound_info has direct pointer to head pages.
  */
-static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
+static __always_inline bool compound_info_has_mask(void)
 {
-	if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
-		return page;
-
 	/*
-	 * Only addresses aligned with PAGE_SIZE of struct page may be fake head
-	 * struct page. The alignment check aims to avoid access the fields (
-	 * e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
-	 * cold cacheline in some cases.
+	 * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it
+	 * makes a difference.
+	 *
+	 * The approach with mask would work in the wider set of conditions,
+	 * but it requires validating that struct pages are naturally aligned
+	 * for all orders up to the MAX_FOLIO_ORDER, which can be tricky.
 	 */
-	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
-	    test_bit(PG_head, &page->flags.f)) {
-		/*
-		 * We can safely access the field of the @page[1] with PG_head
-		 * because the @page is a compound page composed with at least
-		 * two contiguous pages.
-		 */
-		unsigned long head = READ_ONCE(page[1].compound_head);
-
-		if (likely(head & 1))
-			return (const struct page *)(head - 1);
-	}
-	return page;
+	if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP))
+		return false;
+
+	return is_power_of_2(sizeof(struct page));
 }
 
-static __always_inline bool page_count_writable(const struct page *page, int u)
+static __always_inline unsigned long _compound_head(const struct page *page)
 {
-	if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
-		return true;
+	unsigned long info = READ_ONCE(page->compound_info);
+	unsigned long mask;
+
+	if (!compound_info_has_mask()) {
+		/* Bit 0 encodes PageTail() */
+		if (info & 1)
+			return info - 1;
+
+		return (unsigned long)page;
+	}
 
 	/*
-	 * The refcount check is ordered before the fake-head check to prevent
-	 * the following race:
-	 *   CPU 1 (HVO)                     CPU 2 (speculative PFN walker)
-	 *
-	 *   page_ref_freeze()
-	 *   synchronize_rcu()
-	 *                                   rcu_read_lock()
-	 *                                   page_is_fake_head() is false
-	 *   vmemmap_remap_pte()
-	 *   XXX: struct page[] becomes r/o
+	 * If compound_info_has_mask() is true the rest of the info encodes
+	 * the mask that converts the address of the tail page to the head page.
 	 *
-	 *   page_ref_unfreeze()
-	 *                                   page_ref_count() is not zero
+	 * No need to clear bit 0 in the mask as 'page' always has it clear.
 	 *
-	 *                                   atomic_add_unless(&page->_refcount)
-	 *                                   XXX: try to modify r/o struct page[]
-	 *
-	 * The refcount check also prevents modification attempts to other (r/o)
-	 * tail pages that are not fake heads.
+	 * Let's do it in a branchless manner.
 	 */
-	if (atomic_read_acquire(&page->_refcount) == u)
-		return false;
 
-	return page_fixed_fake_head(page) == page;
-}
-#else
-static inline const struct page *page_fixed_fake_head(const struct page *page)
-{
-	return page;
-}
+	/* Non-tail: -1UL, Tail: 0 */
+	mask = (info & 1) - 1;
 
-static inline bool page_count_writable(const struct page *page, int u)
-{
-	return true;
-}
-#endif
+	/* Non-tail: -1UL, Tail: info */
+	mask |= info;
 
-static __always_inline int page_is_fake_head(const struct page *page)
-{
-	return page_fixed_fake_head(page) != page;
+	return (unsigned long)page & mask;
 }
 
-static __always_inline unsigned long _compound_head(const struct page *page)
+#define compound_head(page)	((typeof(page))_compound_head(page))
+
+static __always_inline void set_compound_head(struct page *tail,
+		const struct page *head, unsigned int order)
 {
-	unsigned long head = READ_ONCE(page->compound_head);
+	unsigned int shift;
+	unsigned long mask;
+
+	if (!compound_info_has_mask()) {
+		WRITE_ONCE(tail->compound_info, (unsigned long)head | 1);
+		return;
+	}
+
+	/*
+	 * If the size of struct page is power-of-2, bits [shift:0] of the
+	 * virtual address of compound head are zero.
+	 *
+	 * Calculate mask that can be applied to the virtual address of
+	 * the tail page to get address of the head page.
+	 */
+	shift = order + order_base_2(sizeof(struct page));
+	mask = GENMASK(BITS_PER_LONG - 1, shift);
 
-	if (unlikely(head & 1))
-		return head - 1;
-	return (unsigned long)page_fixed_fake_head(page);
+	/* Bit 0 encodes PageTail() */
+	WRITE_ONCE(tail->compound_info, mask | 1);
 }
 
-#define compound_head(page)	((typeof(page))_compound_head(page))
+static __always_inline void clear_compound_head(struct page *page)
+{
+	WRITE_ONCE(page->compound_info, 0);
+}
 
 /**
  * page_folio - Converts from page to folio.
@@ -320,13 +314,13 @@ static __always_inline unsigned long _compound_head(const struct page *page)
 
 static __always_inline int PageTail(const struct page *page)
 {
-	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
+	return READ_ONCE(page->compound_info) & 1;
 }
 
 static __always_inline int PageCompound(const struct page *page)
 {
 	return test_bit(PG_head, &page->flags.f) ||
-	       READ_ONCE(page->compound_head) & 1;
+	       READ_ONCE(page->compound_info) & 1;
 }
 
 #define	PAGE_POISON_PATTERN	-1l
@@ -348,7 +342,7 @@ static const unsigned long *const_folio_flags(const struct folio *folio,
 {
 	const struct page *page = &folio->page;
 
-	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
+	VM_BUG_ON_PGFLAGS(page->compound_info & 1, page);
 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
 	return &page[n].flags.f;
 }
@@ -357,7 +351,7 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n)
 {
 	struct page *page = &folio->page;
 
-	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
+	VM_BUG_ON_PGFLAGS(page->compound_info & 1, page);
 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
 	return &page[n].flags.f;
 }
@@ -724,6 +718,11 @@ static __always_inline bool folio_test_anon(const struct folio *folio)
 	return ((unsigned long)folio->mapping & FOLIO_MAPPING_ANON) != 0;
 }
 
+static __always_inline bool folio_test_lazyfree(const struct folio *folio)
+{
+	return folio_test_anon(folio) && !folio_test_swapbacked(folio);
+}
+
 static __always_inline bool PageAnonNotKsm(const struct page *page)
 {
 	unsigned long flags = (unsigned long)page_folio(page)->mapping;
@@ -847,7 +846,7 @@ static __always_inline bool folio_test_head(const struct folio *folio)
 static __always_inline int PageHead(const struct page *page)
 {
 	PF_POISONED_CHECK(page);
-	return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
+	return test_bit(PG_head, &page->flags.f);
 }
 
 __SETPAGEFLAG(Head, head, PF_ANY)
@@ -865,16 +864,6 @@ static inline bool folio_test_large(const struct folio *folio)
 	return folio_test_head(folio);
 }
 
-static __always_inline void set_compound_head(struct page *page, struct page *head)
-{
-	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
-}
-
-static __always_inline void clear_compound_head(struct page *page)
-{
-	WRITE_ONCE(page->compound_head, 0);
-}
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline void ClearPageCompound(struct page *page)
 {
@@ -934,6 +923,7 @@ enum pagetype {
 	PGTY_zsmalloc		= 0xf6,
 	PGTY_unaccepted		= 0xf7,
 	PGTY_large_kmalloc	= 0xf8,
+	PGTY_netpp		= 0xf9,
 
 	PGTY_mapcount_underflow = 0xff
 };
@@ -1066,6 +1056,11 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
 PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
 
+/*
+ * Marks page_pool allocated pages.
+ */
+PAGE_TYPE_OPS(Netpp, netpp, netpp)
+
 /**
  * PageHuge - Determine if the page belongs to hugetlbfs
  * @page: The page to test.
diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h
index 544150d1d5fd7..94d3f0e71c06d 100644
--- a/include/linux/page_ref.h
+++ b/include/linux/page_ref.h
@@ -228,24 +228,18 @@ static inline int folio_ref_dec_return(struct folio *folio)
 	return page_ref_dec_return(&folio->page);
 }
 
-static inline bool page_ref_add_unless(struct page *page, int nr, int u)
+static inline bool page_ref_add_unless_zero(struct page *page, int nr)
 {
-	bool ret = false;
-
-	rcu_read_lock();
-	/* avoid writing to the vmemmap area being remapped */
-	if (page_count_writable(page, u))
-		ret = atomic_add_unless(&page->_refcount, nr, u);
-	rcu_read_unlock();
+	bool ret = atomic_add_unless(&page->_refcount, nr, 0);
 
 	if (page_ref_tracepoint_active(page_ref_mod_unless))
 		__page_ref_mod_unless(page, nr, ret);
 	return ret;
 }
 
-static inline bool folio_ref_add_unless(struct folio *folio, int nr, int u)
+static inline bool folio_ref_add_unless_zero(struct folio *folio, int nr)
 {
-	return page_ref_add_unless(&folio->page, nr, u);
+	return page_ref_add_unless_zero(&folio->page, nr);
 }
 
 /**
@@ -261,12 +255,12 @@ static inline bool folio_ref_add_unless(struct folio *folio, int nr, int u)
  */
 static inline bool folio_try_get(struct folio *folio)
 {
-	return folio_ref_add_unless(folio, 1, 0);
+	return folio_ref_add_unless_zero(folio, 1);
 }
 
 static inline bool folio_ref_try_add(struct folio *folio, int count)
 {
-	return folio_ref_add_unless(folio, count, 0);
+	return folio_ref_add_unless_zero(folio, count);
 }
 
 static inline int page_ref_freeze(struct page *page, int count)
diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h
index fe648dfa3a7ca..9d4ca5c218a0f 100644
--- a/include/linux/page_reporting.h
+++ b/include/linux/page_reporting.h
@@ -7,6 +7,7 @@
 
 /* This value should always be a power of 2, see page_reporting_cycle() */
 #define PAGE_REPORTING_CAPACITY		32
+#define PAGE_REPORTING_ORDER_UNSPECIFIED	-1
 
 struct page_reporting_dev_info {
 	/* function that alters pages to make them "reported" */
diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h
deleted file mode 100644
index 63be5a4516276..0000000000000
--- a/include/linux/pagevec.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * include/linux/pagevec.h
- *
- * In many places it is efficient to batch an operation up against multiple
- * folios.  A folio_batch is a container which is used for that.
- */
-
-#ifndef _LINUX_PAGEVEC_H
-#define _LINUX_PAGEVEC_H
-
-#include <linux/types.h>
-
-/* 31 pointers + header align the folio_batch structure to a power of two */
-#define PAGEVEC_SIZE	31
-
-struct folio;
-
-/**
- * struct folio_batch - A collection of folios.
- *
- * The folio_batch is used to amortise the cost of retrieving and
- * operating on a set of folios.  The order of folios in the batch may be
- * significant (eg delete_from_page_cache_batch()).  Some users of the
- * folio_batch store "exceptional" entries in it which can be removed
- * by calling folio_batch_remove_exceptionals().
- */
-struct folio_batch {
-	unsigned char nr;
-	unsigned char i;
-	bool percpu_pvec_drained;
-	struct folio *folios[PAGEVEC_SIZE];
-};
-
-/**
- * folio_batch_init() - Initialise a batch of folios
- * @fbatch: The folio batch.
- *
- * A freshly initialised folio_batch contains zero folios.
- */
-static inline void folio_batch_init(struct folio_batch *fbatch)
-{
-	fbatch->nr = 0;
-	fbatch->i = 0;
-	fbatch->percpu_pvec_drained = false;
-}
-
-static inline void folio_batch_reinit(struct folio_batch *fbatch)
-{
-	fbatch->nr = 0;
-	fbatch->i = 0;
-}
-
-static inline unsigned int folio_batch_count(const struct folio_batch *fbatch)
-{
-	return fbatch->nr;
-}
-
-static inline unsigned int folio_batch_space(const struct folio_batch *fbatch)
-{
-	return PAGEVEC_SIZE - fbatch->nr;
-}
-
-/**
- * folio_batch_add() - Add a folio to a batch.
- * @fbatch: The folio batch.
- * @folio: The folio to add.
- *
- * The folio is added to the end of the batch.
- * The batch must have previously been initialised using folio_batch_init().
- *
- * Return: The number of slots still available.
- */
-static inline unsigned folio_batch_add(struct folio_batch *fbatch,
-		struct folio *folio)
-{
-	fbatch->folios[fbatch->nr++] = folio;
-	return folio_batch_space(fbatch);
-}
-
-/**
- * folio_batch_next - Return the next folio to process.
- * @fbatch: The folio batch being processed.
- *
- * Use this function to implement a queue of folios.
- *
- * Return: The next folio in the queue, or NULL if the queue is empty.
- */
-static inline struct folio *folio_batch_next(struct folio_batch *fbatch)
-{
-	if (fbatch->i == fbatch->nr)
-		return NULL;
-	return fbatch->folios[fbatch->i++];
-}
-
-void __folio_batch_release(struct folio_batch *pvec);
-
-static inline void folio_batch_release(struct folio_batch *fbatch)
-{
-	if (folio_batch_count(fbatch))
-		__folio_batch_release(fbatch);
-}
-
-void folio_batch_remove_exceptionals(struct folio_batch *fbatch);
-#endif /* _LINUX_PAGEVEC_H */
diff --git a/include/linux/pagewalk.h b/include/linux/pagewalk.h
index 88e18615dd726..b41d7265c01bc 100644
--- a/include/linux/pagewalk.h
+++ b/include/linux/pagewalk.h
@@ -148,14 +148,8 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index,
 
 typedef int __bitwise folio_walk_flags_t;
 
-/*
- * Walk migration entries as well. Careful: a large folio might get split
- * concurrently.
- */
-#define FW_MIGRATION			((__force folio_walk_flags_t)BIT(0))
-
 /* Walk shared zeropages (small + huge) as well. */
-#define FW_ZEROPAGE			((__force folio_walk_flags_t)BIT(1))
+#define FW_ZEROPAGE			((__force folio_walk_flags_t)BIT(0))
 
 enum folio_walk_level {
 	FW_LEVEL_PTE,
diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
index c021c7af175fd..334c2b7578d00 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -191,13 +191,45 @@ struct pci_epc {
  * @BAR_RESIZABLE: The BAR implements the PCI-SIG Resizable BAR Capability.
  *		   NOTE: An EPC driver can currently only set a single supported
  *		   size.
- * @BAR_RESERVED: The BAR should not be touched by an EPF driver.
+ * @BAR_RESERVED: Used for HW-backed BARs (e.g. MSI-X table, DMA regs). The BAR
+ *		  should not be disabled by an EPC driver. The BAR should not be
+ *		  reprogrammed by an EPF driver. An EPF driver is allowed to
+ *		  disable the BAR if absolutely necessary. (However, right now
+ *		  there is no EPC operation to disable a BAR that has not been
+ *		  programmed using pci_epc_set_bar().)
+ * @BAR_DISABLED: The BAR should be disabled by an EPC driver. The BAR will be
+ *		  unavailable to an EPF driver.
  */
 enum pci_epc_bar_type {
 	BAR_PROGRAMMABLE = 0,
 	BAR_FIXED,
 	BAR_RESIZABLE,
 	BAR_RESERVED,
+	BAR_DISABLED,
+};
+
+/**
+ * enum pci_epc_bar_rsvd_region_type - type of a fixed subregion behind a BAR
+ * @PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO: Integrated DMA controller MMIO window
+ *
+ * BARs marked BAR_RESERVED are owned by the SoC/EPC hardware and must not be
+ * reprogrammed by EPF drivers. Some of them still expose fixed subregions that
+ * EPFs may want to reference (e.g. embedded doorbell fallback).
+ */
+enum pci_epc_bar_rsvd_region_type {
+	PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO = 0,
+};
+
+/**
+ * struct pci_epc_bar_rsvd_region - fixed subregion behind a BAR
+ * @type: reserved region type
+ * @offset: offset within the BAR aperture
+ * @size: size of the reserved region
+ */
+struct pci_epc_bar_rsvd_region {
+	enum pci_epc_bar_rsvd_region_type	type;
+	resource_size_t				offset;
+	resource_size_t				size;
 };
 
 /**
@@ -206,18 +238,16 @@ enum pci_epc_bar_type {
  * @fixed_size: the fixed size, only applicable if type is BAR_FIXED_MASK.
  * @only_64bit: if true, an EPF driver is not allowed to choose if this BAR
  *		should be configured as 32-bit or 64-bit, the EPF driver must
- *		configure this BAR as 64-bit. Additionally, the BAR succeeding
- *		this BAR must be set to type BAR_RESERVED.
- *
- *		only_64bit should not be set on a BAR of type BAR_RESERVED.
- *		(If BARx is a 64-bit BAR that an EPF driver is not allowed to
- *		touch, then both BARx and BARx+1 must be set to type
- *		BAR_RESERVED.)
+ *		configure this BAR as 64-bit.
+ * @nr_rsvd_regions: number of fixed subregions described for BAR_RESERVED
+ * @rsvd_regions: fixed subregions behind BAR_RESERVED
  */
 struct pci_epc_bar_desc {
 	enum pci_epc_bar_type type;
 	u64 fixed_size;
 	bool only_64bit;
+	u8 nr_rsvd_regions;
+	const struct pci_epc_bar_rsvd_region *rsvd_regions;
 };
 
 /**
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 1c270f1d51230..8861eeb4381db 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -72,12 +72,20 @@
 /* return bus from PCI devid = ((u16)bus_number) << 8) | devfn */
 #define PCI_BUS_NUM(x) (((x) >> 8) & 0xff)
 
+/*
+ * PCI_SLOT_ALL_DEVICES indicates a slot that covers all devices on the bus.
+ * Used for PCIe hotplug where the physical slot is the entire secondary bus,
+ * and, if ARI Forwarding is enabled, functions may appear to be on multiple
+ * devices.
+ */
+#define PCI_SLOT_ALL_DEVICES	0xfe
+
 /* pci_slot represents a physical slot */
 struct pci_slot {
 	struct pci_bus		*bus;		/* Bus this slot is on */
 	struct list_head	list;		/* Node in list of slots */
 	struct hotplug_slot	*hotplug;	/* Hotplug info (move here) */
-	unsigned char		number;		/* PCI_SLOT(pci_dev->devfn) */
+	unsigned char		number;		/* Device nr, or PCI_SLOT_ALL_DEVICES */
 	struct kobject		kobj;
 };
 
@@ -518,7 +526,7 @@ struct pci_dev {
 	unsigned int	ptm_root:1;
 	unsigned int	ptm_responder:1;
 	unsigned int	ptm_requester:1;
-	unsigned int	ptm_enabled:1;
+	atomic_t	ptm_enable_cnt;
 	u8		ptm_granularity;
 #endif
 #ifdef CONFIG_PCI_MSI
@@ -1975,11 +1983,11 @@ struct pci_ptm_debugfs {
 };
 
 #ifdef CONFIG_PCIE_PTM
-int pci_enable_ptm(struct pci_dev *dev, u8 *granularity);
+int pci_enable_ptm(struct pci_dev *dev);
 void pci_disable_ptm(struct pci_dev *dev);
 bool pcie_ptm_enabled(struct pci_dev *dev);
 #else
-static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
+static inline int pci_enable_ptm(struct pci_dev *dev)
 { return -EINVAL; }
 static inline void pci_disable_ptm(struct pci_dev *dev) { }
 static inline bool pcie_ptm_enabled(struct pci_dev *dev)
diff --git a/include/linux/percpu_counter_tree.h b/include/linux/percpu_counter_tree.h
new file mode 100644
index 0000000000000..828c763edd4a7
--- /dev/null
+++ b/include/linux/percpu_counter_tree.h
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR MIT */
+/* SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
+
+#ifndef _PERCPU_COUNTER_TREE_H
+#define _PERCPU_COUNTER_TREE_H
+
+#include <linux/preempt.h>
+#include <linux/atomic.h>
+#include <linux/percpu.h>
+
+#ifdef CONFIG_SMP
+
+#if NR_CPUS == (1U << 0)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	0
+#elif NR_CPUS <= (1U << 1)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	1
+#elif NR_CPUS <= (1U << 2)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	3
+#elif NR_CPUS <= (1U << 3)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	7
+#elif NR_CPUS <= (1U << 4)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	7
+#elif NR_CPUS <= (1U << 5)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	11
+#elif NR_CPUS <= (1U << 6)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	21
+#elif NR_CPUS <= (1U << 7)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	21
+#elif NR_CPUS <= (1U << 8)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	37
+#elif NR_CPUS <= (1U << 9)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	73
+#elif NR_CPUS <= (1U << 10)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	149
+#elif NR_CPUS <= (1U << 11)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	293
+#elif NR_CPUS <= (1U << 12)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	585
+#elif NR_CPUS <= (1U << 13)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	1173
+#elif NR_CPUS <= (1U << 14)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	2341
+#elif NR_CPUS <= (1U << 15)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	4681
+#elif NR_CPUS <= (1U << 16)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	4681
+#elif NR_CPUS <= (1U << 17)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	8777
+#elif NR_CPUS <= (1U << 18)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	17481
+#elif NR_CPUS <= (1U << 19)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	34953
+#elif NR_CPUS <= (1U << 20)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS	69905
+#else
+# error "Unsupported number of CPUs."
+#endif
+
+struct percpu_counter_tree_level_item {
+	atomic_long_t count;		/*
+					 * Count the number of carry for this tree item.
+					 * The carry counter is kept at the order of the
+					 * carry accounted for at this tree level.
+					 */
+} ____cacheline_aligned_in_smp;
+
+#define PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE	\
+	(PERCPU_COUNTER_TREE_STATIC_NR_ITEMS * sizeof(struct percpu_counter_tree_level_item))
+
+struct percpu_counter_tree {
+	/* Fast-path fields. */
+	unsigned long __percpu *level0;	/* Pointer to per-CPU split counters (tree level 0). */
+	unsigned long level0_bit_mask;	/* Bit mask to apply to detect carry propagation from tree level 0. */
+	union {
+		unsigned long *i;	/* Approximate sum for single-CPU topology. */
+		atomic_long_t *a;	/* Approximate sum for SMP topology.  */
+	} approx_sum;
+	long bias;			/* Bias to apply to counter precise and approximate values. */
+
+	/* Slow-path fields. */
+	struct percpu_counter_tree_level_item *items;	/* Array of tree items for levels 1 to N. */
+	unsigned long batch_size;	/*
+					 * The batch size is the increment step at level 0 which
+					 * triggers a carry propagation. The batch size is required
+					 * to be greater than 1, and a power of 2.
+					 */
+	/*
+	 * The tree approximate sum is guaranteed to be within this accuracy range:
+	 * (precise_sum - approx_accuracy_range.under) <= approx_sum <= (precise_sum + approx_accuracy_range.over).
+	 * This accuracy is derived from the hardware topology and the tree batch_size.
+	 * The "under" accuracy is larger than the "over" accuracy because the negative range of a
+	 * two's complement signed integer is one unit larger than the positive range. This delta
+	 * is summed for each tree item, which leads to a significantly larger "under" accuracy range
+	 * compared to the "over" accuracy range.
+	 */
+	struct {
+		unsigned long under;
+		unsigned long over;
+	} approx_accuracy_range;
+};
+
+size_t percpu_counter_tree_items_size(void);
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned long batch_size, gfp_t gfp_flags);
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned long batch_size, gfp_t gfp_flags);
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters);
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter);
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, long inc);
+long percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter);
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, long v);
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, long v);
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, long v);
+int percpu_counter_tree_subsystem_init(void);
+
+/**
+ * percpu_counter_tree_approximate_sum() - Return approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the approximate sum is fast, but it is only accurate within
+ * the bounds delimited by percpu_counter_tree_approximate_accuracy_range().
+ * This is meant to be used when speed is preferred over accuracy.
+ *
+ * Return: The current approximate counter sum.
+ */
+static inline
+long percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+	unsigned long v;
+
+	if (!counter->level0_bit_mask)
+		v = READ_ONCE(*counter->approx_sum.i);
+	else
+		v = atomic_long_read(counter->approx_sum.a);
+	return (long) (v + (unsigned long)READ_ONCE(counter->bias));
+}
+
+/**
+ * percpu_counter_tree_approximate_accuracy_range - Query the accuracy range for a counter tree.
+ * @counter: Counter to query.
+ * @under: Pointer to a variable to be incremented of the approximation
+ *         accuracy range below the precise sum.
+ * @over: Pointer to a variable to be incremented of the approximation
+ *        accuracy range above the precise sum.
+ *
+ * Query the accuracy range limits for the counter.
+ * Because of two's complement binary representation, the "under" range is typically
+ * slightly larger than the "over" range.
+ * Those values are derived from the hardware topology and the counter tree batch size.
+ * They are invariant for a given counter tree.
+ * Using this function should not be typically required, see the following functions instead:
+ * * percpu_counter_tree_approximate_compare(),
+ * * percpu_counter_tree_approximate_compare_value(),
+ * * percpu_counter_tree_precise_compare(),
+ * * percpu_counter_tree_precise_compare_value().
+ */
+static inline
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+						    unsigned long *under, unsigned long *over)
+{
+	*under += counter->approx_accuracy_range.under;
+	*over += counter->approx_accuracy_range.over;
+}
+
+#else	/* !CONFIG_SMP */
+
+#define PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE	0
+
+struct percpu_counter_tree_level_item;
+
+struct percpu_counter_tree {
+	atomic_long_t count;
+};
+
+static inline
+size_t percpu_counter_tree_items_size(void)
+{
+	return 0;
+}
+
+static inline
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned long batch_size, gfp_t gfp_flags)
+{
+	for (unsigned int i = 0; i < nr_counters; i++)
+		atomic_long_set(&counters[i].count, 0);
+	return 0;
+}
+
+static inline
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned long batch_size, gfp_t gfp_flags)
+{
+	return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+
+static inline
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters)
+{
+}
+
+static inline
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+}
+
+static inline
+long percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+	return atomic_long_read(&counter->count);
+}
+
+static inline
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	long count_a = percpu_counter_tree_precise_sum(a),
+	     count_b = percpu_counter_tree_precise_sum(b);
+
+	if (count_a == count_b)
+		return 0;
+	if (count_a < count_b)
+		return -1;
+	return 1;
+}
+
+static inline
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, long v)
+{
+	long count = percpu_counter_tree_precise_sum(counter);
+
+	if (count == v)
+		return 0;
+	if (count < v)
+		return -1;
+	return 1;
+}
+
+static inline
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	return percpu_counter_tree_precise_compare(a, b);
+}
+
+static inline
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, long v)
+{
+	return percpu_counter_tree_precise_compare_value(counter, v);
+}
+
+static inline
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, long v)
+{
+	atomic_long_set(&counter->count, v);
+}
+
+static inline
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+						    unsigned long *under, unsigned long *over)
+{
+}
+
+static inline
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, long inc)
+{
+	atomic_long_add(inc, &counter->count);
+}
+
+static inline
+long percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_precise_sum(counter);
+}
+
+static inline
+int percpu_counter_tree_subsystem_init(void)
+{
+	return 0;
+}
+
+#endif	/* CONFIG_SMP */
+
+/**
+ * percpu_counter_tree_approximate_sum_positive() - Return a positive approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return an approximate counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive approximate counter sum.
+ */
+static inline
+long percpu_counter_tree_approximate_sum_positive(struct percpu_counter_tree *counter)
+{
+	long v = percpu_counter_tree_approximate_sum(counter);
+	return v > 0 ? v : 0;
+}
+
+/**
+ * percpu_counter_tree_precise_sum_positive() - Return a positive precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return a precise counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive precise counter sum.
+ */
+static inline
+long percpu_counter_tree_precise_sum_positive(struct percpu_counter_tree *counter)
+{
+	long v = percpu_counter_tree_precise_sum(counter);
+	return v > 0 ? v : 0;
+}
+
+/**
+ * percpu_counter_tree_approximate_min_max_range() - Return the approximation min and max precise values.
+ * @approx_sum: Approximated sum.
+ * @under: Tree accuracy range (under).
+ * @over: Tree accuracy range (over).
+ * @precise_min: Minimum possible value for precise sum (output).
+ * @precise_max: Maximum possible value for precise sum (output).
+ *
+ * Calculate the minimum and maximum precise values for a given
+ * approximation and (under, over) accuracy range.
+ *
+ * The range of the approximation as a function of the precise sum is expressed as:
+ *
+ *   approx_sum >= precise_sum - approx_accuracy_range.under
+ *   approx_sum <= precise_sum + approx_accuracy_range.over
+ *
+ * Therefore, the range of the precise sum as a function of the approximation is expressed as:
+ *
+ *   precise_sum <= approx_sum + approx_accuracy_range.under
+ *   precise_sum >= approx_sum - approx_accuracy_range.over
+ */
+static inline
+void percpu_counter_tree_approximate_min_max_range(long approx_sum, unsigned long under, unsigned long over,
+						   long *precise_min, long *precise_max)
+{
+	*precise_min = approx_sum - over;
+	*precise_max = approx_sum + under;
+}
+
+/**
+ * percpu_counter_tree_approximate_min_max() - Return the tree approximation, min and max possible precise values.
+ * @counter: The counter to sum.
+ * @approx_sum: Approximate sum (output).
+ * @precise_min: Minimum possible value for precise sum (output).
+ * @precise_max: Maximum possible value for precise sum (output).
+ *
+ * Return the approximate sum, minimum and maximum precise values for
+ * a counter.
+ */
+static inline
+void percpu_counter_tree_approximate_min_max(struct percpu_counter_tree *counter,
+					     long *approx_sum, long *precise_min, long *precise_max)
+{
+	unsigned long under = 0, over = 0;
+	long v = percpu_counter_tree_approximate_sum(counter);
+
+	percpu_counter_tree_approximate_accuracy_range(counter, &under, &over);
+	percpu_counter_tree_approximate_min_max_range(v, under, over, precise_min, precise_max);
+	*approx_sum = v;
+}
+
+#endif  /* _PERCPU_COUNTER_TREE_H */
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index a50df42a893fb..17d961c612fc9 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -1103,6 +1103,43 @@ static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
 }
 #endif
 
+#ifndef test_and_clear_young_ptes
+/**
+ * test_and_clear_young_ptes - Mark PTEs that map consecutive pages of the same
+ *			       folio as old
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_test_and_clear_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ *
+ * Returns: whether any PTE was young.
+ */
+static inline int test_and_clear_young_ptes(struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+	int young = 0;
+
+	for (;;) {
+		young |= ptep_test_and_clear_young(vma, addr, ptep);
+		if (--nr == 0)
+			break;
+		ptep++;
+		addr += PAGE_SIZE;
+	}
+
+	return young;
+}
+#endif
+
 /*
  * On some architectures hardware does not set page access bit when accessing
  * memory page, it is responsibility of software setting this bit. It brings
@@ -1917,41 +1954,56 @@ static inline void pfnmap_setup_cachemode_pfn(unsigned long pfn, pgprot_t *prot)
 	pfnmap_setup_cachemode(pfn, PAGE_SIZE, prot);
 }
 
-#ifdef CONFIG_MMU
+/*
+ * ZERO_PAGE() is global shared page(s) that is always zero. It is used for
+ * zero-mapped memory areas, CoW etc.
+ *
+ * On architectures that __HAVE_COLOR_ZERO_PAGE there are several such pages
+ * for different ranges in the virtual address space.
+ *
+ * zero_page_pfn identifies the first (or the only) pfn for these pages.
+ *
+ * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in
+ * empty_zero_page in BSS.
+ */
+void arch_setup_zero_pages(void);
+
 #ifdef __HAVE_COLOR_ZERO_PAGE
 static inline int is_zero_pfn(unsigned long pfn)
 {
-	extern unsigned long zero_pfn;
-	unsigned long offset_from_zero_pfn = pfn - zero_pfn;
+	extern unsigned long zero_page_pfn;
+	unsigned long offset_from_zero_pfn = pfn - zero_page_pfn;
+
 	return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
 }
 
-#define my_zero_pfn(addr)	page_to_pfn(ZERO_PAGE(addr))
+#define zero_pfn(addr)	page_to_pfn(ZERO_PAGE(addr))
 
 #else
 static inline int is_zero_pfn(unsigned long pfn)
 {
-	extern unsigned long zero_pfn;
-	return pfn == zero_pfn;
-}
+	extern unsigned long zero_page_pfn;
 
-static inline unsigned long my_zero_pfn(unsigned long addr)
-{
-	extern unsigned long zero_pfn;
-	return zero_pfn;
+	return pfn == zero_page_pfn;
 }
-#endif
-#else
-static inline int is_zero_pfn(unsigned long pfn)
+
+static inline unsigned long zero_pfn(unsigned long addr)
 {
-	return 0;
+	extern unsigned long zero_page_pfn;
+
+	return zero_page_pfn;
 }
 
-static inline unsigned long my_zero_pfn(unsigned long addr)
+extern uint8_t empty_zero_page[PAGE_SIZE];
+extern struct page *__zero_page;
+
+static inline struct page *_zero_page(unsigned long addr)
 {
-	return 0;
+	return __zero_page;
 }
-#endif /* CONFIG_MMU */
+#define ZERO_PAGE(vaddr) _zero_page(vaddr)
+
+#endif /* __HAVE_COLOR_ZERO_PAGE */
 
 #ifdef CONFIG_MMU
 
@@ -1989,7 +2041,7 @@ static inline int pud_trans_unstable(pud_t *pud)
 {
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
 	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
-	pud_t pudval = READ_ONCE(*pud);
+	pud_t pudval = pudp_get(pud);
 
 	if (pud_none(pudval) || pud_trans_huge(pudval))
 		return 1;
diff --git a/include/linux/phy.h b/include/linux/phy.h
index 6f9979a26892d..5de4b172cd0ba 100644
--- a/include/linux/phy.h
+++ b/include/linux/phy.h
@@ -2152,8 +2152,6 @@ int phy_suspend(struct phy_device *phydev);
 int phy_resume(struct phy_device *phydev);
 int __phy_resume(struct phy_device *phydev);
 int phy_loopback(struct phy_device *phydev, bool enable, int speed);
-struct phy_device *phy_attach(struct net_device *dev, const char *bus_id,
-			      phy_interface_t interface);
 struct phy_device *phy_find_next(struct mii_bus *bus, struct phy_device *pos);
 int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 		      u32 flags, phy_interface_t interface);
@@ -2448,9 +2446,6 @@ int __phy_hwtstamp_set(struct phy_device *phydev,
 
 struct phy_port *phy_get_sfp_port(struct phy_device *phydev);
 
-extern const struct bus_type mdio_bus_type;
-extern const struct class mdio_bus_class;
-
 /**
  * phy_module_driver() - Helper macro for registering PHY drivers
  * @__phy_drivers: array of PHY drivers to register
diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h
index 89277808ea614..531dc3e9b3f75 100644
--- a/include/linux/pinctrl/pinconf-generic.h
+++ b/include/linux/pinctrl/pinconf-generic.h
@@ -115,7 +115,7 @@ struct pinctrl_map;
  * @PIN_CONFIG_SKEW_DELAY_INPUT_PS: if the pin has independent values for the
  *	programmable skew rate (on inputs) and latch delay (on outputs), then
  *	this parameter specifies the clock skew only. The argument is in ps.
- * @PIN_CONFIG_SKEW_DELAY_OUPUT_PS: if the pin has independent values for the
+ * @PIN_CONFIG_SKEW_DELAY_OUTPUT_PS: if the pin has independent values for the
  *	programmable skew rate (on inputs) and latch delay (on outputs), then
  *	this parameter specifies the latch delay only. The argument is in ps.
  * @PIN_CONFIG_SLEEP_HARDWARE_STATE: indicate this is sleep related state.
diff --git a/include/linux/platform_data/dma-mcf-edma.h b/include/linux/platform_data/dma-mcf-edma.h
index d718ccfa3421d..0b31af66a1acd 100644
--- a/include/linux/platform_data/dma-mcf-edma.h
+++ b/include/linux/platform_data/dma-mcf-edma.h
@@ -26,8 +26,9 @@ bool mcf_edma_filter_fn(struct dma_chan *chan, void *param);
 /**
  * struct mcf_edma_platform_data - platform specific data for eDMA engine
  *
- * @ver			The eDMA module version.
- * @dma_channels	The number of eDMA channels.
+ * @dma_channels:	The number of eDMA channels.
+ * @slave_map:		Slave device map
+ * @slavecnt:		Number of entries in @slave_map
  */
 struct mcf_edma_platform_data {
 	int dma_channels;
diff --git a/include/linux/platform_data/mdio-gpio.h b/include/linux/platform_data/mdio-gpio.h
deleted file mode 100644
index 13874fa6e7679..0000000000000
--- a/include/linux/platform_data/mdio-gpio.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * MDIO-GPIO bus platform data structure
- */
-
-#ifndef __LINUX_MDIO_GPIO_PDATA_H
-#define __LINUX_MDIO_GPIO_PDATA_H
-
-struct mdio_gpio_platform_data {
-	u32 phy_mask;
-	u32 phy_ignore_ta_mask;
-};
-
-#endif /* __LINUX_MDIO_GPIO_PDATA_H */
diff --git a/include/linux/platform_data/tsl2772.h b/include/linux/platform_data/tsl2772.h
index f8ade15a35e27..f042e82b39c31 100644
--- a/include/linux/platform_data/tsl2772.h
+++ b/include/linux/platform_data/tsl2772.h
@@ -61,9 +61,9 @@ struct tsl2772_lux {
  *  @prox_pulse_count:      Number if proximity emitter pulses.
  *  @prox_max_samples_cal:  The number of samples that are taken when performing
  *                          a proximity calibration.
- *  @prox_diode             Which diode(s) to use for driving the external
+ *  @prox_diode:            Which diode(s) to use for driving the external
  *                          LED(s) for proximity sensing.
- *  @prox_power             The amount of power to use for the external LED(s).
+ *  @prox_power:            The amount of power to use for the external LED(s).
  */
 struct tsl2772_settings {
 	int als_time;
diff --git a/include/linux/platform_data/voltage-omap.h b/include/linux/platform_data/voltage-omap.h
index 6d74e507dbd24..2b48f2b0135db 100644
--- a/include/linux/platform_data/voltage-omap.h
+++ b/include/linux/platform_data/voltage-omap.h
@@ -10,14 +10,14 @@
 
 /**
  * struct omap_volt_data - Omap voltage specific data.
- * @voltage_nominal:	The possible voltage value in uV
+ * @volt_nominal:	The possible voltage value in uV
  * @sr_efuse_offs:	The offset of the efuse register(from system
  *			control module base address) from where to read
  *			the n-target value for the smartreflex module.
  * @sr_errminlimit:	Error min limit value for smartreflex. This value
  *			differs at differnet opp and thus is linked
  *			with voltage.
- * @vp_errorgain:	Error gain value for the voltage processor. This
+ * @vp_errgain:		Error gain value for the voltage processor. This
  *			field also differs according to the voltage/opp.
  */
 struct omap_volt_data {
diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h
index 813da101b5bf8..975400a472e30 100644
--- a/include/linux/platform_device.h
+++ b/include/linux/platform_device.h
@@ -31,11 +31,6 @@ struct platform_device {
 	struct resource	*resource;
 
 	const struct platform_device_id	*id_entry;
-	/*
-	 * Driver name to force a match.  Do not set directly, because core
-	 * frees it.  Use driver_set_override() to set or clear it.
-	 */
-	const char *driver_override;
 
 	/* MFD cell pointer */
 	struct mfd_cell *mfd_cell;
@@ -118,22 +113,58 @@ extern int platform_get_irq_byname_optional(struct platform_device *dev,
 					    const char *name);
 extern int platform_add_devices(struct platform_device **, int);
 
+/**
+ * struct platform_device_info - set of parameters for creating a platform device
+ * @parent: parent device for the new platform device.
+ * @fwnode: firmware node associated with the device.
+ * @of_node_reused: indicates that device tree node associated with the device
+ *	is shared with another device, typically its ancestor. Setting this to
+ *	%true prevents the device from being matched via the OF match table,
+ *	and stops the device core from automatically binding pinctrl
+ *	configuration to avoid disrupting the other device.
+ * @name: name of the device.
+ * @id: instance ID of the device. Use %PLATFORM_DEVID_NONE if there is only
+ *	one instance of the device, or %PLATFORM_DEVID_AUTO to let the
+ *	kernel automatically assign a unique instance ID.
+ * @res: set of resources to attach to the device.
+ * @num_res: number of entries in @res.
+ * @data: device-specific data for this platform device.
+ * @size_data: size of device-specific data.
+ * @dma_mask: DMA mask for the device.
+ * @swnode: a secondary software node to be attached to the device. The node
+ *	will be automatically registered and its lifetime tied to the platform
+ *	device if it is not registered yet.
+ * @properties: a set of software properties for the device. If provided,
+ *	a managed software node will be automatically created and
+ *	assigned to the device. The properties array must be terminated
+ *	with a sentinel entry. Specifying both @properties and @swnode is not
+ *	allowed.
+ *
+ * This structure is used to hold information needed to create and register
+ * a platform device using platform_device_register_full().
+ *
+ * platform_device_register_full() makes deep copies of @name, @res, @data and
+ * @properties, so the caller does not need to keep them after registration.
+ * If the registration is performed during initialization, these can be marked
+ * as __initconst.
+ */
 struct platform_device_info {
-		struct device *parent;
-		struct fwnode_handle *fwnode;
-		bool of_node_reused;
+	struct device *parent;
+	struct fwnode_handle *fwnode;
+	bool of_node_reused;
 
-		const char *name;
-		int id;
+	const char *name;
+	int id;
 
-		const struct resource *res;
-		unsigned int num_res;
+	const struct resource *res;
+	unsigned int num_res;
 
-		const void *data;
-		size_t size_data;
-		u64 dma_mask;
+	const void *data;
+	size_t size_data;
+	u64 dma_mask;
 
-		const struct property_entry *properties;
+	const struct software_node *swnode;
+	const struct property_entry *properties;
 };
 extern struct platform_device *platform_device_register_full(
 		const struct platform_device_info *pdevinfo);
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index 93ba0143ca476..b299dc0128d65 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -49,8 +49,8 @@
 
 struct dev_pm_domain_attach_data {
 	const char * const *pd_names;
-	const u32 num_pd_names;
-	const u32 pd_flags;
+	u32 num_pd_names;
+	u32 pd_flags;
 };
 
 struct dev_pm_domain_list {
@@ -183,6 +183,7 @@ struct genpd_power_state {
 	u64 rejected;
 	u64 above;
 	u64 below;
+	u64 usage_s2idle;
 	struct fwnode_handle *fwnode;
 	u64 idle_time;
 	void *data;
diff --git a/include/linux/power/max17042_battery.h b/include/linux/power/max17042_battery.h
index c417abd2ab703..d5b08313cf113 100644
--- a/include/linux/power/max17042_battery.h
+++ b/include/linux/power/max17042_battery.h
@@ -17,6 +17,7 @@
 #define MAX17042_DEFAULT_VMAX		(4500) /* LiHV cell max */
 #define MAX17042_DEFAULT_TEMP_MIN	(0)    /* For sys without temp sensor */
 #define MAX17042_DEFAULT_TEMP_MAX	(700)  /* 70 degrees Celcius */
+#define MAX17042_DEFAULT_TASK_PERIOD	(5760)
 
 /* Consider RepCap which is less then 10 units below FullCAP full */
 #define MAX17042_FULL_THRESHOLD		10
@@ -105,7 +106,7 @@ enum max17042_register {
 
 	MAX17042_OCV		= 0xEE,
 
-	MAX17042_OCVInternal	= 0xFB,  /* MAX17055 VFOCV */
+	MAX17042_OCVInternal	= 0xFB, /* MAX17055/77759 VFOCV */
 
 	MAX17042_VFSOC		= 0xFF,
 };
@@ -156,7 +157,7 @@ enum max17055_register {
 	MAX17055_AtAvCap	= 0xDF,
 };
 
-/* Registers specific to max17047/50/55 */
+/* Registers specific to max17047/50/55/77759 */
 enum max17047_register {
 	MAX17047_QRTbl00	= 0x12,
 	MAX17047_FullSOCThr	= 0x13,
@@ -167,12 +168,32 @@ enum max17047_register {
 	MAX17047_QRTbl30	= 0x42,
 };
 
+enum max77759_register {
+	MAX77759_AvgTA0		= 0x26,
+	MAX77759_AtTTF		= 0x33,
+	MAX77759_Tconvert	= 0x34,
+	MAX77759_AvgCurrent0	= 0x3B,
+	MAX77759_THMHOT		= 0x40,
+	MAX77759_CTESample	= 0x41,
+	MAX77759_ISys		= 0x43,
+	MAX77759_AvgVCell0	= 0x44,
+	MAX77759_RlxSOC		= 0x47,
+	MAX77759_AvgISys	= 0x4B,
+	MAX77759_QH0		= 0x4C,
+	MAX77759_MixAtFull	= 0x4F,
+	MAX77759_VSys		= 0xB1,
+	MAX77759_TAlrtTh2	= 0xB2,
+	MAX77759_VByp		= 0xB3,
+	MAX77759_IIn		= 0xD0,
+};
+
 enum max170xx_chip_type {
 	MAXIM_DEVICE_TYPE_UNKNOWN	= 0,
 	MAXIM_DEVICE_TYPE_MAX17042,
 	MAXIM_DEVICE_TYPE_MAX17047,
 	MAXIM_DEVICE_TYPE_MAX17050,
 	MAXIM_DEVICE_TYPE_MAX17055,
+	MAXIM_DEVICE_TYPE_MAX77759,
 
 	MAXIM_DEVICE_TYPE_NUM
 };
diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h
index 360ffdf272dab..7a5e4c3242a01 100644
--- a/include/linux/power_supply.h
+++ b/include/linux/power_supply.h
@@ -210,6 +210,9 @@ enum power_supply_usb_type {
 	POWER_SUPPLY_USB_TYPE_PD,		/* Power Delivery Port */
 	POWER_SUPPLY_USB_TYPE_PD_DRP,		/* PD Dual Role Port */
 	POWER_SUPPLY_USB_TYPE_PD_PPS,		/* PD Programmable Power Supply */
+	/* PD Standard Power Range Adjustable Voltage Supply */
+	POWER_SUPPLY_USB_TYPE_PD_SPR_AVS,
+	POWER_SUPPLY_USB_TYPE_PD_PPS_SPR_AVS,	/* Supports both PD PPS + SPR AVS */
 	POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID,	/* Apple Charging Method */
 };
 
diff --git a/include/linux/powercap.h b/include/linux/powercap.h
index 3d557bbcd2c71..603419db924ca 100644
--- a/include/linux/powercap.h
+++ b/include/linux/powercap.h
@@ -238,7 +238,7 @@ static inline void *powercap_get_zone_data(struct powercap_zone *power_zone)
 *			Advantage of this parameter is that client can embed
 *			this data in its data structures and allocate in a
 *			single call, preventing multiple allocations.
-* @control_type_name:	The Name of this control_type, which will be shown
+* @name:		The Name of this control_type, which will be shown
 *			in the sysfs Interface.
 * @ops:			Callbacks for control type. This parameter is optional.
 *
@@ -277,7 +277,7 @@ int powercap_unregister_control_type(struct powercap_control_type *instance);
 * @name:	A name for this zone.
 * @parent:	A pointer to the parent power zone instance if any or NULL
 * @ops:		Pointer to zone operation callback structure.
-* @no_constraints: Number of constraints for this zone
+* @nr_constraints: Number of constraints for this zone
 * @const_ops:	Pointer to constraint callback structure
 *
 * Register a power zone under a given control type. A power zone must register
diff --git a/include/linux/ppp_channel.h b/include/linux/ppp_channel.h
index f73fbea0dbc23..2f63e9a6cc889 100644
--- a/include/linux/ppp_channel.h
+++ b/include/linux/ppp_channel.h
@@ -55,7 +55,7 @@ extern void ppp_input(struct ppp_channel *, struct sk_buff *);
 
 /* Called by the channel when an input error occurs, indicating
    that we may have missed a packet. */
-extern void ppp_input_error(struct ppp_channel *, int code);
+extern void ppp_input_error(struct ppp_channel *);
 
 /* Attach a channel to a given PPP unit in specified net. */
 extern int ppp_register_net_channel(struct net *, struct ppp_channel *);
@@ -72,7 +72,9 @@ extern int ppp_channel_index(struct ppp_channel *);
 /* Get the unit number associated with a channel, or -1 if none */
 extern int ppp_unit_number(struct ppp_channel *);
 
-/* Get the device name associated with a channel, or NULL if none */
+/* Get the device name associated with a channel, or NULL if none.
+ * Caller must hold RCU read lock.
+ */
 extern char *ppp_dev_name(struct ppp_channel *);
 
 /*
diff --git a/include/linux/raid/xor.h b/include/linux/raid/xor.h
index 51b811b623224..870558c9d36ee 100644
--- a/include/linux/raid/xor.h
+++ b/include/linux/raid/xor.h
@@ -2,29 +2,6 @@
 #ifndef _XOR_H
 #define _XOR_H
 
-#define MAX_XOR_BLOCKS 4
+void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes);
 
-extern void xor_blocks(unsigned int count, unsigned int bytes,
-	void *dest, void **srcs);
-
-struct xor_block_template {
-        struct xor_block_template *next;
-        const char *name;
-        int speed;
-	void (*do_2)(unsigned long, unsigned long * __restrict,
-		     const unsigned long * __restrict);
-	void (*do_3)(unsigned long, unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict);
-	void (*do_4)(unsigned long, unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict);
-	void (*do_5)(unsigned long, unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict,
-		     const unsigned long * __restrict);
-};
-
-#endif
+#endif /* _XOR_H */
diff --git a/include/linux/random.h b/include/linux/random.h
index 8a8064dc3970b..54401dd53f689 100644
--- a/include/linux/random.h
+++ b/include/linux/random.h
@@ -50,6 +50,7 @@ static inline unsigned long get_random_long(void)
 #endif
 }
 
+u32 __limit_random_u32_below(u32 ceil, u32 rand);
 u32 __get_random_u32_below(u32 ceil);
 
 /*
diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h
index 4091e978aef24..48acdc3889dd4 100644
--- a/include/linux/rbtree.h
+++ b/include/linux/rbtree.h
@@ -35,10 +35,15 @@
 #define RB_CLEAR_NODE(node)  \
 	((node)->__rb_parent_color = (unsigned long)(node))
 
+#define RB_EMPTY_LINKED_NODE(lnode)  RB_EMPTY_NODE(&(lnode)->node)
+#define RB_CLEAR_LINKED_NODE(lnode)  ({					\
+	RB_CLEAR_NODE(&(lnode)->node);					\
+	(lnode)->prev = (lnode)->next = NULL;				\
+})
 
 extern void rb_insert_color(struct rb_node *, struct rb_root *);
 extern void rb_erase(struct rb_node *, struct rb_root *);
-
+extern bool rb_erase_linked(struct rb_node_linked *, struct rb_root_linked *);
 
 /* Find logical next and previous nodes in a tree */
 extern struct rb_node *rb_next(const struct rb_node *);
@@ -213,15 +218,10 @@ rb_add_cached(struct rb_node *node, struct rb_root_cached *tree,
 	return leftmost ? node : NULL;
 }
 
-/**
- * rb_add() - insert @node into @tree
- * @node: node to insert
- * @tree: tree to insert @node into
- * @less: operator defining the (partial) node order
- */
 static __always_inline void
-rb_add(struct rb_node *node, struct rb_root *tree,
-       bool (*less)(struct rb_node *, const struct rb_node *))
+__rb_add(struct rb_node *node, struct rb_root *tree,
+	 bool (*less)(struct rb_node *, const struct rb_node *),
+	 void (*linkop)(struct rb_node *, struct rb_node *, struct rb_node **))
 {
 	struct rb_node **link = &tree->rb_node;
 	struct rb_node *parent = NULL;
@@ -234,10 +234,73 @@ rb_add(struct rb_node *node, struct rb_root *tree,
 			link = &parent->rb_right;
 	}
 
+	linkop(node, parent, link);
 	rb_link_node(node, parent, link);
 	rb_insert_color(node, tree);
 }
 
+#define __node_2_linked_node(_n) \
+	rb_entry((_n), struct rb_node_linked, node)
+
+static inline void
+rb_link_linked_node(struct rb_node *node, struct rb_node *parent, struct rb_node **link)
+{
+	if (!parent)
+		return;
+
+	struct rb_node_linked *nnew = __node_2_linked_node(node);
+	struct rb_node_linked *npar = __node_2_linked_node(parent);
+
+	if (link == &parent->rb_left) {
+		nnew->prev = npar->prev;
+		nnew->next = npar;
+		npar->prev = nnew;
+		if (nnew->prev)
+			nnew->prev->next = nnew;
+	} else {
+		nnew->next = npar->next;
+		nnew->prev = npar;
+		npar->next = nnew;
+		if (nnew->next)
+			nnew->next->prev = nnew;
+	}
+}
+
+/**
+ * rb_add_linked() - insert @node into the leftmost linked tree @tree
+ * @node: node to insert
+ * @tree: linked tree to insert @node into
+ * @less: operator defining the (partial) node order
+ *
+ * Returns @true when @node is the new leftmost, @false otherwise.
+ */
+static __always_inline bool
+rb_add_linked(struct rb_node_linked *node, struct rb_root_linked *tree,
+	      bool (*less)(struct rb_node *, const struct rb_node *))
+{
+	__rb_add(&node->node, &tree->rb_root, less, rb_link_linked_node);
+	if (!node->prev)
+		tree->rb_leftmost = node;
+	return !node->prev;
+}
+
+/* Empty linkop function which is optimized away by the compiler */
+static __always_inline void
+rb_link_noop(struct rb_node *n, struct rb_node *p, struct rb_node **l) { }
+
+/**
+ * rb_add() - insert @node into @tree
+ * @node: node to insert
+ * @tree: tree to insert @node into
+ * @less: operator defining the (partial) node order
+ */
+static __always_inline void
+rb_add(struct rb_node *node, struct rb_root *tree,
+       bool (*less)(struct rb_node *, const struct rb_node *))
+{
+	__rb_add(node, tree, less, rb_link_noop);
+}
+
 /**
  * rb_find_add_cached() - find equivalent @node in @tree, or add @node
  * @node: node to look-for / insert
diff --git a/include/linux/rbtree_types.h b/include/linux/rbtree_types.h
index 45b6ecde3665a..3c7ae53e81397 100644
--- a/include/linux/rbtree_types.h
+++ b/include/linux/rbtree_types.h
@@ -9,6 +9,12 @@ struct rb_node {
 } __attribute__((aligned(sizeof(long))));
 /* The alignment might seem pointless, but allegedly CRIS needs it */
 
+struct rb_node_linked {
+	struct rb_node		node;
+	struct rb_node_linked	*prev;
+	struct rb_node_linked	*next;
+};
+
 struct rb_root {
 	struct rb_node *rb_node;
 };
@@ -28,7 +34,17 @@ struct rb_root_cached {
 	struct rb_node *rb_leftmost;
 };
 
+/*
+ * Leftmost tree with links. This would allow a trivial rb_rightmost update,
+ * but that has been omitted due to the lack of users.
+ */
+struct rb_root_linked {
+	struct rb_root		rb_root;
+	struct rb_node_linked	*rb_leftmost;
+};
+
 #define RB_ROOT (struct rb_root) { NULL, }
 #define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL }
+#define RB_ROOT_LINKED (struct rb_root_linked) { {NULL, }, NULL }
 
 #endif
diff --git a/include/linux/rculist_bl.h b/include/linux/rculist_bl.h
index 0b952d06eb0b0..36363b876e53d 100644
--- a/include/linux/rculist_bl.h
+++ b/include/linux/rculist_bl.h
@@ -8,21 +8,31 @@
 #include <linux/list_bl.h>
 #include <linux/rcupdate.h>
 
+/* return the first ptr or next element in an RCU protected list */
+#define hlist_bl_first_rcu(head)	\
+	(*((struct hlist_bl_node __rcu **)(&(head)->first)))
+#define hlist_bl_next_rcu(node)	\
+	(*((struct hlist_bl_node __rcu **)(&(node)->next)))
+
 static inline void hlist_bl_set_first_rcu(struct hlist_bl_head *h,
 					struct hlist_bl_node *n)
 {
 	LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
 	LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) !=
 							LIST_BL_LOCKMASK);
-	rcu_assign_pointer(h->first,
+	rcu_assign_pointer(hlist_bl_first_rcu(h),
 		(struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK));
 }
 
-static inline struct hlist_bl_node *hlist_bl_first_rcu(struct hlist_bl_head *h)
-{
-	return (struct hlist_bl_node *)
-		((unsigned long)rcu_dereference_check(h->first, hlist_bl_is_locked(h)) & ~LIST_BL_LOCKMASK);
-}
+#define hlist_bl_first_rcu_dereference(head)				\
+({									\
+	struct hlist_bl_head *__head = (head);				\
+									\
+	(struct hlist_bl_node *)					\
+	((unsigned long)rcu_dereference_check(hlist_bl_first_rcu(__head), \
+					      hlist_bl_is_locked(__head)) & \
+					      ~LIST_BL_LOCKMASK);	\
+})
 
 /**
  * hlist_bl_del_rcu - deletes entry from hash list without re-initialization
@@ -73,7 +83,7 @@ static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n,
 {
 	struct hlist_bl_node *first;
 
-	/* don't need hlist_bl_first_rcu because we're under lock */
+	/* don't need hlist_bl_first_rcu* because we're under lock */
 	first = hlist_bl_first(h);
 
 	n->next = first;
@@ -93,9 +103,30 @@ static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n,
  *
  */
 #define hlist_bl_for_each_entry_rcu(tpos, pos, head, member)		\
-	for (pos = hlist_bl_first_rcu(head);				\
+	for (pos = hlist_bl_first_rcu_dereference(head);		\
 		pos &&							\
 		({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \
-		pos = rcu_dereference_raw(pos->next))
+		pos = rcu_dereference_raw(hlist_bl_next_rcu(pos)))
+
+/**
+ * hlist_bl_for_each_entry_continue_rcu - continue iteration over list of given
+ *   type
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct hlist_bl_node to use as a loop cursor.
+ * @member:	the name of the hlist_bl_node within the struct.
+ *
+ * Continue to iterate over list of given type, continuing after
+ * the current position which must have been in the list when the RCU read
+ * lock was taken.
+ * This would typically require either that you obtained the node from a
+ * previous walk of the list in the same RCU read-side critical section, or
+ * that you held some sort of non-RCU reference (such as a reference count)
+ * to keep the node alive *and* in the list.
+ */
+#define hlist_bl_for_each_entry_continue_rcu(tpos, pos, member)		\
+	for (pos = rcu_dereference_raw(hlist_bl_next_rcu(&(tpos)->member)); \
+	     pos &&							\
+	     ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \
+	     pos = rcu_dereference_raw(hlist_bl_next_rcu(pos)))
 
 #endif
diff --git a/include/linux/regmap.h b/include/linux/regmap.h
index caff2240bdab7..f1c5cb63c171a 100644
--- a/include/linux/regmap.h
+++ b/include/linux/regmap.h
@@ -10,15 +10,16 @@
  * Author: Mark Brown <broonie@opensource.wolfsonmicro.com>
  */
 
-#include <linux/list.h>
-#include <linux/rbtree.h>
-#include <linux/ktime.h>
+#include <linux/bug.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/err.h>
-#include <linux/bug.h>
-#include <linux/lockdep.h>
-#include <linux/iopoll.h>
 #include <linux/fwnode.h>
+#include <linux/iopoll.h>
+#include <linux/ktime.h>
+#include <linux/list.h>
+#include <linux/lockdep.h>
+#include <linux/rbtree.h>
 
 struct module;
 struct clk;
@@ -1460,6 +1461,8 @@ struct regmap_field *regmap_field_alloc(struct regmap *regmap,
 		struct reg_field reg_field);
 void regmap_field_free(struct regmap_field *field);
 
+DEFINE_FREE(regmap_field, struct regmap_field *, if (_T) regmap_field_free(_T))
+
 struct regmap_field *devm_regmap_field_alloc(struct device *dev,
 		struct regmap *regmap, struct reg_field reg_field);
 void devm_regmap_field_free(struct device *dev,	struct regmap_field *field);
diff --git a/include/linux/reset-controller.h b/include/linux/reset-controller.h
index 46514cb1b9e06..52a5a4e81f184 100644
--- a/include/linux/reset-controller.h
+++ b/include/linux/reset-controller.h
@@ -3,7 +3,10 @@
 #define _LINUX_RESET_CONTROLLER_H_
 
 #include <linux/list.h>
+#include <linux/mutex.h>
 
+struct fwnode_handle;
+struct fwnode_reference_args;
 struct reset_controller_dev;
 
 /**
@@ -35,14 +38,16 @@ struct of_phandle_args;
  * @reset_control_head: head of internal list of requested reset controls
  * @dev: corresponding driver model device struct
  * @of_node: corresponding device tree node as phandle target
- * @of_args: for reset-gpios controllers: corresponding phandle args with
- *           of_node and GPIO number complementing of_node; either this or
- *           of_node should be present
  * @of_reset_n_cells: number of cells in reset line specifiers
  * @of_xlate: translation function to translate from specifier as found in the
- *            device tree to id as given to the reset control ops, defaults
- *            to :c:func:`of_reset_simple_xlate`.
+ *            device tree to id as given to the reset control ops
+ * @fwnode: firmware node associated with this device
+ * @fwnode_reset_n_cells: number of cells in reset line specifiers
+ * @fwnode_xlate: translation function to translate from firmware specifier to
+ *                id as given to the reset control ops, defaults to
+ *                :c:func:`fwnode_reset_simple_xlate`
  * @nr_resets: number of reset controls in this reset controller device
+ * @lock: protects the reset control list from concurrent access
  */
 struct reset_controller_dev {
 	const struct reset_control_ops *ops;
@@ -51,11 +56,15 @@ struct reset_controller_dev {
 	struct list_head reset_control_head;
 	struct device *dev;
 	struct device_node *of_node;
-	const struct of_phandle_args *of_args;
 	int of_reset_n_cells;
 	int (*of_xlate)(struct reset_controller_dev *rcdev,
 			const struct of_phandle_args *reset_spec);
+	struct fwnode_handle *fwnode;
+	int fwnode_reset_n_cells;
+	int (*fwnode_xlate)(struct reset_controller_dev *rcdev,
+			    const struct fwnode_reference_args *reset_spec);
 	unsigned int nr_resets;
+	struct mutex lock;
 };
 
 #if IS_ENABLED(CONFIG_RESET_CONTROLLER)
diff --git a/include/linux/reset.h b/include/linux/reset.h
index 44f9e3415f92c..9c391cf0c8229 100644
--- a/include/linux/reset.h
+++ b/include/linux/reset.h
@@ -5,10 +5,12 @@
 #include <linux/bits.h>
 #include <linux/err.h>
 #include <linux/errno.h>
+#include <linux/of.h>
 #include <linux/types.h>
 
 struct device;
 struct device_node;
+struct fwnode_handle;
 struct reset_control;
 
 /**
@@ -84,7 +86,7 @@ int reset_control_bulk_deassert(int num_rstcs, struct reset_control_bulk_data *r
 int reset_control_bulk_acquire(int num_rstcs, struct reset_control_bulk_data *rstcs);
 void reset_control_bulk_release(int num_rstcs, struct reset_control_bulk_data *rstcs);
 
-struct reset_control *__of_reset_control_get(struct device_node *node,
+struct reset_control *__fwnode_reset_control_get(struct fwnode_handle *fwnode,
 				     const char *id, int index, enum reset_control_flags flags);
 struct reset_control *__reset_control_get(struct device *dev, const char *id,
 					  int index, enum reset_control_flags flags);
@@ -103,7 +105,8 @@ int __devm_reset_control_bulk_get(struct device *dev, int num_rstcs,
 
 struct reset_control *devm_reset_control_array_get(struct device *dev,
 						   enum reset_control_flags flags);
-struct reset_control *of_reset_control_array_get(struct device_node *np, enum reset_control_flags);
+struct reset_control *fwnode_reset_control_array_get(struct fwnode_handle *fwnode,
+						     enum reset_control_flags);
 
 int reset_control_get_count(struct device *dev);
 
@@ -152,8 +155,8 @@ static inline int __device_reset(struct device *dev, bool optional)
 	return optional ? 0 : -ENOTSUPP;
 }
 
-static inline struct reset_control *__of_reset_control_get(
-					struct device_node *node,
+static inline struct reset_control *__fwnode_reset_control_get(
+					struct fwnode_handle *fwnode,
 					const char *id, int index, enum reset_control_flags flags)
 {
 	bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL;
@@ -242,7 +245,7 @@ devm_reset_control_array_get(struct device *dev, enum reset_control_flags flags)
 }
 
 static inline struct reset_control *
-of_reset_control_array_get(struct device_node *np, enum reset_control_flags flags)
+fwnode_reset_control_array_get(struct fwnode_handle *fwnode, enum reset_control_flags flags)
 {
 	bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL;
 
@@ -500,7 +503,8 @@ reset_control_bulk_get_optional_shared(struct device *dev, int num_rstcs,
 static inline struct reset_control *of_reset_control_get_exclusive(
 				struct device_node *node, const char *id)
 {
-	return __of_reset_control_get(node, id, 0, RESET_CONTROL_EXCLUSIVE);
+	return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0,
+					  RESET_CONTROL_EXCLUSIVE);
 }
 
 /**
@@ -520,7 +524,8 @@ static inline struct reset_control *of_reset_control_get_exclusive(
 static inline struct reset_control *of_reset_control_get_optional_exclusive(
 				struct device_node *node, const char *id)
 {
-	return __of_reset_control_get(node, id, 0, RESET_CONTROL_OPTIONAL_EXCLUSIVE);
+	return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0,
+					  RESET_CONTROL_OPTIONAL_EXCLUSIVE);
 }
 
 /**
@@ -545,7 +550,8 @@ static inline struct reset_control *of_reset_control_get_optional_exclusive(
 static inline struct reset_control *of_reset_control_get_shared(
 				struct device_node *node, const char *id)
 {
-	return __of_reset_control_get(node, id, 0, RESET_CONTROL_SHARED);
+	return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0,
+					  RESET_CONTROL_SHARED);
 }
 
 /**
@@ -562,7 +568,8 @@ static inline struct reset_control *of_reset_control_get_shared(
 static inline struct reset_control *of_reset_control_get_exclusive_by_index(
 					struct device_node *node, int index)
 {
-	return __of_reset_control_get(node, NULL, index, RESET_CONTROL_EXCLUSIVE);
+	return __fwnode_reset_control_get(of_fwnode_handle(node), NULL, index,
+					  RESET_CONTROL_EXCLUSIVE);
 }
 
 /**
@@ -590,7 +597,8 @@ static inline struct reset_control *of_reset_control_get_exclusive_by_index(
 static inline struct reset_control *of_reset_control_get_shared_by_index(
 					struct device_node *node, int index)
 {
-	return __of_reset_control_get(node, NULL, index, RESET_CONTROL_SHARED);
+	return __fwnode_reset_control_get(of_fwnode_handle(node), NULL, index,
+					  RESET_CONTROL_SHARED);
 }
 
 /**
@@ -1032,30 +1040,35 @@ devm_reset_control_array_get_optional_shared(struct device *dev)
 static inline struct reset_control *
 of_reset_control_array_get_exclusive(struct device_node *node)
 {
-	return of_reset_control_array_get(node, RESET_CONTROL_EXCLUSIVE);
+	return fwnode_reset_control_array_get(of_fwnode_handle(node),
+					      RESET_CONTROL_EXCLUSIVE);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_exclusive_released(struct device_node *node)
 {
-	return of_reset_control_array_get(node, RESET_CONTROL_EXCLUSIVE_RELEASED);
+	return fwnode_reset_control_array_get(of_fwnode_handle(node),
+					      RESET_CONTROL_EXCLUSIVE_RELEASED);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_shared(struct device_node *node)
 {
-	return of_reset_control_array_get(node, RESET_CONTROL_SHARED);
+	return fwnode_reset_control_array_get(of_fwnode_handle(node),
+					      RESET_CONTROL_SHARED);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_optional_exclusive(struct device_node *node)
 {
-	return of_reset_control_array_get(node, RESET_CONTROL_OPTIONAL_EXCLUSIVE);
+	return fwnode_reset_control_array_get(of_fwnode_handle(node),
+					      RESET_CONTROL_OPTIONAL_EXCLUSIVE);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_optional_shared(struct device_node *node)
 {
-	return of_reset_control_array_get(node, RESET_CONTROL_OPTIONAL_SHARED);
+	return fwnode_reset_control_array_get(of_fwnode_handle(node),
+					      RESET_CONTROL_OPTIONAL_SHARED);
 }
 #endif
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index 133ccb39137ab..0480509a63393 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -129,10 +129,10 @@ static __always_inline unsigned int rht_key_get_hash(struct rhashtable *ht,
 	unsigned int hash;
 
 	/* params must be equal to ht->p if it isn't constant. */
-	if (!__builtin_constant_p(params.key_len))
+	if (!__builtin_constant_p(params.key_len)) {
 		hash = ht->p.hashfn(key, ht->key_len, hash_rnd);
-	else if (params.key_len) {
-		unsigned int key_len = params.key_len;
+	} else {
+		unsigned int key_len = params.key_len ? : ht->p.key_len;
 
 		if (params.hashfn)
 			hash = params.hashfn(key, key_len, hash_rnd);
@@ -140,13 +140,6 @@ static __always_inline unsigned int rht_key_get_hash(struct rhashtable *ht,
 			hash = jhash(key, key_len, hash_rnd);
 		else
 			hash = jhash2(key, key_len / sizeof(u32), hash_rnd);
-	} else {
-		unsigned int key_len = ht->p.key_len;
-
-		if (params.hashfn)
-			hash = params.hashfn(key, key_len, hash_rnd);
-		else
-			hash = jhash(key, key_len, hash_rnd);
 	}
 
 	return hash;
diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h
index d862fa610270b..994f52b343440 100644
--- a/include/linux/ring_buffer.h
+++ b/include/linux/ring_buffer.h
@@ -251,4 +251,62 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu,
 void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu);
 int ring_buffer_unmap(struct trace_buffer *buffer, int cpu);
 int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu);
+
+struct ring_buffer_desc {
+	int		cpu;
+	unsigned int	nr_page_va; /* excludes the meta page */
+	unsigned long	meta_va;
+	unsigned long	page_va[] __counted_by(nr_page_va);
+};
+
+struct trace_buffer_desc {
+	int		nr_cpus;
+	size_t		struct_len;
+	char		__data[]; /* list of ring_buffer_desc */
+};
+
+static inline struct ring_buffer_desc *__next_ring_buffer_desc(struct ring_buffer_desc *desc)
+{
+	size_t len = struct_size(desc, page_va, desc->nr_page_va);
+
+	return (struct ring_buffer_desc *)((void *)desc + len);
+}
+
+static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buffer_desc *desc)
+{
+	return (struct ring_buffer_desc *)(&desc->__data[0]);
+}
+
+static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus)
+{
+	unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1;
+	struct ring_buffer_desc *rbdesc;
+
+	return size_add(offsetof(struct trace_buffer_desc, __data),
+			size_mul(nr_cpus, struct_size(rbdesc, page_va, nr_pages)));
+}
+
+#define for_each_ring_buffer_desc(__pdesc, __cpu, __trace_pdesc)		\
+	for (__pdesc = __first_ring_buffer_desc(__trace_pdesc), __cpu = 0;	\
+	     (__cpu) < (__trace_pdesc)->nr_cpus;				\
+	     (__cpu)++, __pdesc = __next_ring_buffer_desc(__pdesc))
+
+struct ring_buffer_remote {
+	struct trace_buffer_desc	*desc;
+	int				(*swap_reader_page)(unsigned int cpu, void *priv);
+	int				(*reset)(unsigned int cpu, void *priv);
+	void				*priv;
+};
+
+int ring_buffer_poll_remote(struct trace_buffer *buffer, int cpu);
+
+struct trace_buffer *
+__ring_buffer_alloc_remote(struct ring_buffer_remote *remote,
+			   struct lock_class_key *key);
+
+#define ring_buffer_alloc_remote(remote)			\
+({								\
+	static struct lock_class_key __key;			\
+	__ring_buffer_alloc_remote(remote, &__key);		\
+})
 #endif /* _LINUX_RING_BUFFER_H */
diff --git a/include/linux/ring_buffer_types.h b/include/linux/ring_buffer_types.h
new file mode 100644
index 0000000000000..54577021a49d5
--- /dev/null
+++ b/include/linux/ring_buffer_types.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_RING_BUFFER_TYPES_H
+#define _LINUX_RING_BUFFER_TYPES_H
+
+#include <asm/local.h>
+
+#define TS_SHIFT        27
+#define TS_MASK         ((1ULL << TS_SHIFT) - 1)
+#define TS_DELTA_TEST   (~TS_MASK)
+
+/*
+ * We need to fit the time_stamp delta into 27 bits.
+ */
+static inline bool test_time_stamp(u64 delta)
+{
+	return !!(delta & TS_DELTA_TEST);
+}
+
+#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
+
+#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
+#define RB_ALIGNMENT		4U
+#define RB_MAX_SMALL_DATA	(RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
+#define RB_EVNT_MIN_SIZE	8U	/* two 32bit words */
+
+#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS
+# define RB_FORCE_8BYTE_ALIGNMENT	0
+# define RB_ARCH_ALIGNMENT		RB_ALIGNMENT
+#else
+# define RB_FORCE_8BYTE_ALIGNMENT	1
+# define RB_ARCH_ALIGNMENT		8U
+#endif
+
+#define RB_ALIGN_DATA		__aligned(RB_ARCH_ALIGNMENT)
+
+struct buffer_data_page {
+	u64		 time_stamp;	/* page time stamp */
+	local_t		 commit;	/* write committed index */
+	unsigned char	 data[] RB_ALIGN_DATA;	/* data of buffer page */
+};
+#endif
diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
index c6831c93cd6ee..f11ebd34f8b95 100644
--- a/include/linux/rseq_entry.h
+++ b/include/linux/rseq_entry.h
@@ -40,6 +40,7 @@ DECLARE_PER_CPU(struct rseq_stats, rseq_stats);
 #endif /* !CONFIG_RSEQ_STATS */
 
 #ifdef CONFIG_RSEQ
+#include <linux/hrtimer_rearm.h>
 #include <linux/jump_label.h>
 #include <linux/rseq.h>
 #include <linux/sched/signal.h>
@@ -110,7 +111,7 @@ static __always_inline void rseq_slice_clear_grant(struct task_struct *t)
 	t->rseq.slice.state.granted = false;
 }
 
-static __always_inline bool rseq_grant_slice_extension(bool work_pending)
+static __always_inline bool __rseq_grant_slice_extension(bool work_pending)
 {
 	struct task_struct *curr = current;
 	struct rseq_slice_ctrl usr_ctrl;
@@ -215,11 +216,20 @@ static __always_inline bool rseq_grant_slice_extension(bool work_pending)
 	return false;
 }
 
+static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask)
+{
+	if (unlikely(__rseq_grant_slice_extension(ti_work & mask))) {
+		hrtimer_rearm_deferred_tif(ti_work);
+		return true;
+	}
+	return false;
+}
+
 #else /* CONFIG_RSEQ_SLICE_EXTENSION */
 static __always_inline bool rseq_slice_extension_enabled(void) { return false; }
 static __always_inline bool rseq_arm_slice_extension_timer(void) { return false; }
 static __always_inline void rseq_slice_clear_grant(struct task_struct *t) { }
-static __always_inline bool rseq_grant_slice_extension(bool work_pending) { return false; }
+static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
 #endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
 
 bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
@@ -778,7 +788,7 @@ static inline void rseq_syscall_exit_to_user_mode(void) { }
 static inline void rseq_irqentry_exit_to_user_mode(void) { }
 static inline void rseq_exit_to_user_mode_legacy(void) { }
 static inline void rseq_debug_syscall_return(struct pt_regs *regs) { }
-static inline bool rseq_grant_slice_extension(bool work_pending) { return false; }
+static inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
 #endif /* !CONFIG_RSEQ */
 
 #endif /* _LINUX_RSEQ_ENTRY_H */
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index ede4c6bf6f226..78e7e588817cf 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -22,8 +22,8 @@ extern int max_lock_depth;
 
 struct rt_mutex_base {
 	raw_spinlock_t		wait_lock;
-	struct rb_root_cached   waiters;
-	struct task_struct	*owner;
+	struct rb_root_cached   waiters __guarded_by(&wait_lock);
+	struct task_struct	*owner  __guarded_by(&wait_lock);
 };
 
 #define __RT_MUTEX_BASE_INITIALIZER(rtbasename)				\
@@ -41,7 +41,7 @@ struct rt_mutex_base {
  */
 static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock)
 {
-	return READ_ONCE(lock->owner) != NULL;
+	return data_race(READ_ONCE(lock->owner) != NULL);
 }
 
 #ifdef CONFIG_RT_MUTEXES
@@ -49,7 +49,7 @@ static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock)
 
 static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock)
 {
-	unsigned long owner = (unsigned long) READ_ONCE(lock->owner);
+	unsigned long owner = (unsigned long) data_race(READ_ONCE(lock->owner));
 
 	return (struct task_struct *) (owner & ~RT_MUTEX_HAS_WAITERS);
 }
diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h
index 3390d21c95dd1..4e67cd934d8fd 100644
--- a/include/linux/rwlock.h
+++ b/include/linux/rwlock.h
@@ -30,17 +30,27 @@ do {								\
 
 #ifdef CONFIG_DEBUG_SPINLOCK
  extern void do_raw_read_lock(rwlock_t *lock) __acquires_shared(lock);
- extern int do_raw_read_trylock(rwlock_t *lock);
+ extern int do_raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock);
  extern void do_raw_read_unlock(rwlock_t *lock) __releases_shared(lock);
  extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock);
- extern int do_raw_write_trylock(rwlock_t *lock);
+extern int do_raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock);
  extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock);
 #else
 # define do_raw_read_lock(rwlock)	do {__acquire_shared(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0)
-# define do_raw_read_trylock(rwlock)	arch_read_trylock(&(rwlock)->raw_lock)
+static inline int do_raw_read_trylock(rwlock_t *rwlock)
+	__cond_acquires_shared(true, rwlock)
+	__no_context_analysis
+{
+	return arch_read_trylock(&(rwlock)->raw_lock);
+}
 # define do_raw_read_unlock(rwlock)	do {arch_read_unlock(&(rwlock)->raw_lock); __release_shared(lock); } while (0)
 # define do_raw_write_lock(rwlock)	do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0)
-# define do_raw_write_trylock(rwlock)	arch_write_trylock(&(rwlock)->raw_lock)
+static inline int do_raw_write_trylock(rwlock_t *rwlock)
+	__cond_acquires(true, rwlock)
+	__no_context_analysis
+{
+	return arch_write_trylock(&(rwlock)->raw_lock);
+}
 # define do_raw_write_unlock(rwlock)	do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0)
 #endif
 
diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 61a852609eab4..9e02a5f28cd1d 100644
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -23,7 +23,7 @@ void __lockfunc _raw_write_lock_bh(rwlock_t *lock)	__acquires(lock);
 void __lockfunc _raw_read_lock_irq(rwlock_t *lock)	__acquires_shared(lock);
 void __lockfunc _raw_write_lock_irq(rwlock_t *lock)	__acquires(lock);
 unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
-							__acquires(lock);
+							__acquires_shared(lock);
 unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
 							__acquires(lock);
 int __lockfunc _raw_read_trylock(rwlock_t *lock)	__cond_acquires_shared(true, lock);
@@ -36,7 +36,7 @@ void __lockfunc _raw_read_unlock_irq(rwlock_t *lock)	__releases_shared(lock);
 void __lockfunc _raw_write_unlock_irq(rwlock_t *lock)	__releases(lock);
 void __lockfunc
 _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
-							__releases(lock);
+							__releases_shared(lock);
 void __lockfunc
 _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 							__releases(lock);
@@ -116,6 +116,7 @@ _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 #endif
 
 static inline int __raw_read_trylock(rwlock_t *lock)
+	__cond_acquires_shared(true, lock)
 {
 	preempt_disable();
 	if (do_raw_read_trylock(lock)) {
@@ -127,6 +128,7 @@ static inline int __raw_read_trylock(rwlock_t *lock)
 }
 
 static inline int __raw_write_trylock(rwlock_t *lock)
+	__cond_acquires(true, lock)
 {
 	preempt_disable();
 	if (do_raw_write_trylock(lock)) {
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 9bf1d93d3d7ba..6a1a7bae5f811 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -57,7 +57,7 @@ context_lock_struct(rw_semaphore) {
 	struct optimistic_spin_queue osq; /* spinner MCS lock */
 #endif
 	raw_spinlock_t wait_lock;
-	struct list_head wait_list;
+	struct rwsem_waiter *first_waiter __guarded_by(&wait_lock);
 #ifdef CONFIG_DEBUG_RWSEMS
 	void *magic;
 #endif
@@ -106,7 +106,7 @@ static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *
 	  .owner = ATOMIC_LONG_INIT(0),				\
 	  __RWSEM_OPT_INIT(name)				\
 	  .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
-	  .wait_list = LIST_HEAD_INIT((name).wait_list),	\
+	  .first_waiter = NULL,					\
 	  __RWSEM_DEBUG_INIT(name)				\
 	  __RWSEM_DEP_MAP_INIT(name) }
 
@@ -129,9 +129,9 @@ do {								\
  * rwsem to see if somebody from an incompatible type is wanting access to the
  * lock.
  */
-static inline int rwsem_is_contended(struct rw_semaphore *sem)
+static inline bool rwsem_is_contended(struct rw_semaphore *sem)
 {
-	return !list_empty(&sem->wait_list);
+	return data_race(sem->first_waiter != NULL);
 }
 
 #if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5a5d3dbc9cdf3..0d27775546f8c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1533,7 +1533,7 @@ struct task_struct {
 	/* Used by memcontrol for targeted memcg charge: */
 	struct mem_cgroup		*active_memcg;
 
-	/* Cache for current->cgroups->memcg->objcg lookups: */
+	/* Cache for current->cgroups->memcg->nodeinfo[nid]->objcg lookups: */
 	struct obj_cgroup		*objcg;
 #endif
 
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index bcb962d5ee7d8..602dc83cab367 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -62,6 +62,16 @@ enum scx_dsq_id_flags {
 	SCX_DSQ_LOCAL_CPU_MASK	= 0xffffffffLLU,
 };
 
+struct scx_deferred_reenq_user {
+	struct list_head	node;
+	u64			flags;
+};
+
+struct scx_dsq_pcpu {
+	struct scx_dispatch_q	*dsq;
+	struct scx_deferred_reenq_user deferred_reenq_user;
+};
+
 /*
  * A dispatch queue (DSQ) can be either a FIFO or p->scx.dsq_vtime ordered
  * queue. A built-in DSQ is always a FIFO. The built-in local DSQs are used to
@@ -78,30 +88,58 @@ struct scx_dispatch_q {
 	u64			id;
 	struct rhash_head	hash_node;
 	struct llist_node	free_node;
+	struct scx_sched	*sched;
+	struct scx_dsq_pcpu __percpu *pcpu;
 	struct rcu_head		rcu;
 };
 
-/* scx_entity.flags */
+/* sched_ext_entity.flags */
 enum scx_ent_flags {
 	SCX_TASK_QUEUED		= 1 << 0, /* on ext runqueue */
+	SCX_TASK_IN_CUSTODY	= 1 << 1, /* in custody, needs ops.dequeue() when leaving */
 	SCX_TASK_RESET_RUNNABLE_AT = 1 << 2, /* runnable_at should be reset */
 	SCX_TASK_DEQD_FOR_SLEEP	= 1 << 3, /* last dequeue was for SLEEP */
+	SCX_TASK_SUB_INIT	= 1 << 4, /* task being initialized for a sub sched */
+	SCX_TASK_IMMED		= 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
 
-	SCX_TASK_STATE_SHIFT	= 8,	  /* bit 8 and 9 are used to carry scx_task_state */
+	/*
+	 * Bits 8 and 9 are used to carry task state:
+	 *
+	 * NONE		ops.init_task() not called yet
+	 * INIT		ops.init_task() succeeded, but task can be cancelled
+	 * READY	fully initialized, but not in sched_ext
+	 * ENABLED	fully initialized and in sched_ext
+	 */
+	SCX_TASK_STATE_SHIFT	= 8,	  /* bits 8 and 9 are used to carry task state */
 	SCX_TASK_STATE_BITS	= 2,
 	SCX_TASK_STATE_MASK	= ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT,
 
-	SCX_TASK_CURSOR		= 1 << 31, /* iteration cursor, not a task */
-};
+	SCX_TASK_NONE		= 0 << SCX_TASK_STATE_SHIFT,
+	SCX_TASK_INIT		= 1 << SCX_TASK_STATE_SHIFT,
+	SCX_TASK_READY		= 2 << SCX_TASK_STATE_SHIFT,
+	SCX_TASK_ENABLED	= 3 << SCX_TASK_STATE_SHIFT,
+
+	/*
+	 * Bits 12 and 13 are used to carry reenqueue reason. In addition to
+	 * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
+	 * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
+	 *
+	 * NONE		not being reenqueued
+	 * KFUNC	reenqueued by scx_bpf_dsq_reenq() and friends
+	 * IMMED	reenqueued due to failed ENQ_IMMED
+	 * PREEMPTED	preempted while running
+	 */
+	SCX_TASK_REENQ_REASON_SHIFT = 12,
+	SCX_TASK_REENQ_REASON_BITS = 2,
+	SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
 
-/* scx_entity.flags & SCX_TASK_STATE_MASK */
-enum scx_task_state {
-	SCX_TASK_NONE,		/* ops.init_task() not called yet */
-	SCX_TASK_INIT,		/* ops.init_task() succeeded, but task can be cancelled */
-	SCX_TASK_READY,		/* fully initialized, but not in sched_ext */
-	SCX_TASK_ENABLED,	/* fully initialized and in sched_ext */
+	SCX_TASK_REENQ_NONE	= 0 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_KFUNC	= 1 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_IMMED	= 2 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
 
-	SCX_TASK_NR_STATES,
+	/* iteration cursor, not a task */
+	SCX_TASK_CURSOR		= 1 << 31,
 };
 
 /* scx_entity.dsq_flags */
@@ -149,19 +187,31 @@ struct scx_dsq_list_node {
 	u32			priv;		/* can be used by iter cursor */
 };
 
-#define INIT_DSQ_LIST_CURSOR(__node, __flags, __priv)				\
+#define INIT_DSQ_LIST_CURSOR(__cursor, __dsq, __flags)				\
 	(struct scx_dsq_list_node) {						\
-		.node = LIST_HEAD_INIT((__node).node),				\
+		.node = LIST_HEAD_INIT((__cursor).node),			\
 		.flags = SCX_DSQ_LNODE_ITER_CURSOR | (__flags),			\
-		.priv = (__priv),						\
+		.priv = READ_ONCE((__dsq)->seq),				\
 	}
 
+struct scx_sched;
+
 /*
  * The following is embedded in task_struct and contains all fields necessary
  * for a task to be scheduled by SCX.
  */
 struct sched_ext_entity {
+#ifdef CONFIG_CGROUPS
+	/*
+	 * Associated scx_sched. Updated either during fork or while holding
+	 * both p->pi_lock and rq lock.
+	 */
+	struct scx_sched __rcu	*sched;
+#endif
 	struct scx_dispatch_q	*dsq;
+	atomic_long_t		ops_state;
+	u64			ddsp_dsq_id;
+	u64			ddsp_enq_flags;
 	struct scx_dsq_list_node dsq_list;	/* dispatch order */
 	struct rb_node		dsq_priq;	/* p->scx.dsq_vtime order */
 	u32			dsq_seq;
@@ -173,7 +223,6 @@ struct sched_ext_entity {
 	s32			selected_cpu;
 	u32			kf_mask;	/* see scx_kf_mask above */
 	struct task_struct	*kf_tasks[2];	/* see SCX_CALL_OP_TASK() */
-	atomic_long_t		ops_state;
 
 	struct list_head	runnable_node;	/* rq->scx.runnable_list */
 	unsigned long		runnable_at;
@@ -181,8 +230,6 @@ struct sched_ext_entity {
 #ifdef CONFIG_SCHED_CORE
 	u64			core_sched_at;	/* see scx_prio_less() */
 #endif
-	u64			ddsp_dsq_id;
-	u64			ddsp_enq_flags;
 
 	/* BPF scheduler modifiable fields */
 
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index a22248aebcf90..584ae88b435e9 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -132,6 +132,7 @@ struct signal_struct {
 	 */
 	unsigned int		is_child_subreaper:1;
 	unsigned int		has_child_subreaper:1;
+	unsigned int		autoreap:1;
 
 #ifdef CONFIG_POSIX_TIMERS
 
@@ -739,7 +740,7 @@ static inline int thread_group_empty(struct task_struct *p)
 
 extern struct sighand_struct *lock_task_sighand(struct task_struct *task,
 						unsigned long *flags)
-	__acquires(&task->sighand->siglock);
+	__cond_acquires(nonnull, &task->sighand->siglock);
 
 static inline void unlock_task_sighand(struct task_struct *task,
 						unsigned long *flags)
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 45c0022b91ced..51c29581f15eb 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -95,6 +95,7 @@ struct sched_domain {
 	unsigned int newidle_call;
 	unsigned int newidle_success;
 	unsigned int newidle_ratio;
+	u64 newidle_stamp;
 	u64 max_newidle_lb_cost;
 	unsigned long last_decay_max_lb_cost;
 
@@ -171,7 +172,6 @@ typedef int (*sched_domain_flags_f)(void);
 
 struct sd_data {
 	struct sched_domain *__percpu *sd;
-	struct sched_domain_shared *__percpu *sds;
 	struct sched_group *__percpu *sg;
 	struct sched_group_capacity *__percpu *sgc;
 };
diff --git a/include/linux/scmi_imx_protocol.h b/include/linux/scmi_imx_protocol.h
index 2407d7693b6ba..ab867463c08cc 100644
--- a/include/linux/scmi_imx_protocol.h
+++ b/include/linux/scmi_imx_protocol.h
@@ -52,6 +52,17 @@ struct scmi_imx_misc_ctrl_notify_report {
 	unsigned int		flags;
 };
 
+
+#define MISC_EXT_INFO_LEN_MAX	4
+struct scmi_imx_misc_reset_reason {
+	bool valid:1;
+	bool orig_valid:1;
+	bool err_valid:1;
+	u32 reason;
+	u32 origin;
+	u32 errid;
+};
+
 struct scmi_imx_misc_proto_ops {
 	int (*misc_ctrl_set)(const struct scmi_protocol_handle *ph, u32 id,
 			     u32 num, u32 *val);
@@ -61,6 +72,9 @@ struct scmi_imx_misc_proto_ops {
 				    u32 ctrl_id, u32 evt_id, u32 flags);
 	int (*misc_syslog)(const struct scmi_protocol_handle *ph, u16 *size,
 			   void *array);
+	int (*misc_reset_reason)(const struct scmi_protocol_handle *ph,
+				 bool system, struct scmi_imx_misc_reset_reason *boot_r,
+				 struct scmi_imx_misc_reset_reason *shut_r, u32 *extinfo);
 };
 
 /* See LMM_ATTRIBUTES in imx95.rst */
diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h
index aafaac1496b06..c710107c2120a 100644
--- a/include/linux/scmi_protocol.h
+++ b/include/linux/scmi_protocol.h
@@ -15,7 +15,6 @@
 
 #define SCMI_MAX_STR_SIZE		64
 #define SCMI_SHORT_NAME_MAX_SIZE	16
-#define SCMI_MAX_NUM_RATES		16
 
 /**
  * struct scmi_revision_info - version information structure
@@ -41,6 +40,12 @@ struct scmi_revision_info {
 	char sub_vendor_id[SCMI_SHORT_NAME_MAX_SIZE];
 };
 
+struct scmi_clock_rates {
+	bool rate_discrete;
+	unsigned int num_rates;
+	u64 *rates;
+};
+
 struct scmi_clock_info {
 	char name[SCMI_MAX_STR_SIZE];
 	unsigned int enable_latency;
@@ -51,17 +56,8 @@ struct scmi_clock_info {
 	bool rate_ctrl_forbidden;
 	bool parent_ctrl_forbidden;
 	bool extended_config;
-	union {
-		struct {
-			int num_rates;
-			u64 rates[SCMI_MAX_NUM_RATES];
-		} list;
-		struct {
-			u64 min_rate;
-			u64 max_rate;
-			u64 step_size;
-		} range;
-	};
+	u64 min_rate;
+	u64 max_rate;
 	int num_parents;
 	u32 *parents;
 };
@@ -91,6 +87,11 @@ enum scmi_clock_oem_config {
  * @info_get: get the information of the specified clock
  * @rate_get: request the current clock rate of a clock
  * @rate_set: set the clock rate of a clock
+ * @determine_rate: determine the effective rate that can be supported by a
+ *		    clock calculating the closest allowed rate.
+ *		    Note that @rate is an input/output parameter used both to
+ *		    describe the requested rate and report the closest match
+ * @all_rates_get: get the list of all available rates for the specified clock.
  * @enable: enables the specified clock
  * @disable: disables the specified clock
  * @state_get: get the status of the specified clock
@@ -108,6 +109,10 @@ struct scmi_clk_proto_ops {
 			u64 *rate);
 	int (*rate_set)(const struct scmi_protocol_handle *ph, u32 clk_id,
 			u64 rate);
+	int (*determine_rate)(const struct scmi_protocol_handle *ph, u32 clk_id,
+			      unsigned long *rate);
+	const struct scmi_clock_rates __must_check *(*all_rates_get)
+		(const struct scmi_protocol_handle *ph, u32 clk_id);
 	int (*enable)(const struct scmi_protocol_handle *ph, u32 clk_id,
 		      bool atomic);
 	int (*disable)(const struct scmi_protocol_handle *ph, u32 clk_id,
diff --git a/include/linux/secure_boot.h b/include/linux/secure_boot.h
new file mode 100644
index 0000000000000..d17e923515672
--- /dev/null
+++ b/include/linux/secure_boot.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved.
+ *
+ * Author: Coiby Xu <coxu@redhat.com>
+ */
+
+#ifndef _LINUX_SECURE_BOOT_H
+#define _LINUX_SECURE_BOOT_H
+
+#include <linux/types.h>
+
+#ifdef CONFIG_HAVE_ARCH_GET_SECUREBOOT
+/*
+ * Returns true if the platform secure boot is enabled.
+ * Returns false if disabled or not supported.
+ */
+bool arch_get_secureboot(void);
+#else
+static inline bool arch_get_secureboot(void) { return false; }
+#endif
+
+#endif /* _LINUX_SECURE_BOOT_H */
diff --git a/include/linux/security.h b/include/linux/security.h
index 83a646d72f6f8..4017361d8cba9 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -476,6 +476,9 @@ int security_file_ioctl_compat(struct file *file, unsigned int cmd,
 			       unsigned long arg);
 int security_mmap_file(struct file *file, unsigned long prot,
 			unsigned long flags);
+int security_mmap_backing_file(struct vm_area_struct *vma,
+			       struct file *backing_file,
+			       struct file *user_file);
 int security_mmap_addr(unsigned long addr);
 int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
 			   unsigned long prot);
@@ -1159,6 +1162,13 @@ static inline int security_mmap_file(struct file *file, unsigned long prot,
 	return 0;
 }
 
+static inline int security_mmap_backing_file(struct vm_area_struct *vma,
+					     struct file *backing_file,
+					     struct file *user_file)
+{
+	return 0;
+}
+
 static inline int security_mmap_addr(unsigned long addr)
 {
 	return cap_mmap_addr(addr);
diff --git a/include/linux/sed-opal.h b/include/linux/sed-opal.h
index 80f33a93f9443..aa006edb612bc 100644
--- a/include/linux/sed-opal.h
+++ b/include/linux/sed-opal.h
@@ -53,6 +53,10 @@ static inline bool is_sed_ioctl(unsigned int cmd)
 	case IOC_OPAL_DISCOVERY:
 	case IOC_OPAL_REVERT_LSP:
 	case IOC_OPAL_SET_SID_PW:
+	case IOC_OPAL_REACTIVATE_LSP:
+	case IOC_OPAL_LR_SET_START_LEN:
+	case IOC_OPAL_ENABLE_DISABLE_LR:
+	case IOC_OPAL_GET_SUM_STATUS:
 		return true;
 	}
 	return false;
diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 89706157e6223..a4c8651ef0217 100644
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -15,7 +15,7 @@
 struct semaphore {
 	raw_spinlock_t		lock;
 	unsigned int		count;
-	struct list_head	wait_list;
+	struct semaphore_waiter *first_waiter;
 
 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
 	unsigned long		last_holder;
@@ -33,7 +33,7 @@ struct semaphore {
 {									\
 	.lock		= __RAW_SPIN_LOCK_UNLOCKED((name).lock),	\
 	.count		= n,						\
-	.wait_list	= LIST_HEAD_INIT((name).wait_list)		\
+	.first_waiter	= NULL						\
 	__LAST_HOLDER_SEMAPHORE_INITIALIZER				\
 }
 
diff --git a/include/linux/serdev.h b/include/linux/serdev.h
index 5654c58eb73c0..090c93c08045d 100644
--- a/include/linux/serdev.h
+++ b/include/linux/serdev.h
@@ -37,8 +37,8 @@ struct serdev_device_ops {
  * @nr:		Device number on serdev bus.
  * @ctrl:	serdev controller managing this device.
  * @ops:	Device operations.
- * @write_comp	Completion used by serdev_device_write() internally
- * @write_lock	Lock to serialize access when writing data
+ * @write_comp:	Completion used by serdev_device_write() internally
+ * @write_lock:	Lock to serialize access when writing data
  */
 struct serdev_device {
 	struct device dev;
@@ -60,6 +60,7 @@ static inline struct serdev_device *to_serdev_device(struct device *d)
  *		structure.
  * @probe:	binds this driver to a serdev device.
  * @remove:	unbinds this driver from the serdev device.
+ * @shutdown:	shut down this serdev device.
  */
 struct serdev_device_driver {
 	struct device_driver driver;
@@ -129,7 +130,7 @@ static inline void serdev_device_set_drvdata(struct serdev_device *serdev, void
 
 /**
  * serdev_device_put() - decrement serdev device refcount
- * @serdev	serdev device.
+ * @serdev:	serdev device.
  */
 static inline void serdev_device_put(struct serdev_device *serdev)
 {
@@ -157,7 +158,7 @@ static inline void serdev_controller_set_drvdata(struct serdev_controller *ctrl,
 
 /**
  * serdev_controller_put() - decrement controller refcount
- * @ctrl	serdev controller.
+ * @ctrl:	serdev controller.
  */
 static inline void serdev_controller_put(struct serdev_controller *ctrl)
 {
diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h
index 01efdce0fda07..a95b2d143d248 100644
--- a/include/linux/serial_8250.h
+++ b/include/linux/serial_8250.h
@@ -195,6 +195,7 @@ void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl);
 void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud,
 			       unsigned int quot);
 int fsl8250_handle_irq(struct uart_port *port);
+void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir);
 int serial8250_handle_irq(struct uart_port *port, unsigned int iir);
 u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr);
 void serial8250_read_char(struct uart_8250_port *up, u16 lsr);
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index a8273b32e041b..93a0ba872ebe0 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -48,7 +48,7 @@ struct shmem_inode_info {
 	};
 	struct timespec64	i_crtime;	/* file creation time */
 	struct shared_policy	policy;		/* NUMA memory alloc policy */
-	struct simple_xattrs	xattrs;		/* list of xattrs */
+	struct simple_xattrs	*xattrs;	/* list of xattrs */
 	pgoff_t			fallocend;	/* highest fallocate endindex */
 	unsigned int		fsflags;	/* for FS_IOC_[SG]ETFLAGS */
 	atomic_t		stop_eviction;	/* hold when working on inode */
@@ -221,20 +221,6 @@ static inline pgoff_t shmem_fallocend(struct inode *inode, pgoff_t eof)
 
 extern bool shmem_charge(struct inode *inode, long pages);
 
-#ifdef CONFIG_USERFAULTFD
-#ifdef CONFIG_SHMEM
-extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
-				  struct vm_area_struct *dst_vma,
-				  unsigned long dst_addr,
-				  unsigned long src_addr,
-				  uffd_flags_t flags,
-				  struct folio **foliop);
-#else /* !CONFIG_SHMEM */
-#define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \
-			       src_addr, flags, foliop) ({ BUG(); 0; })
-#endif /* CONFIG_SHMEM */
-#endif /* CONFIG_USERFAULTFD */
-
 /*
  * Used space is stored as unsigned 64-bit value in bytes but
  * quota core supports only signed 64-bit values so use that
diff --git a/include/linux/simple_ring_buffer.h b/include/linux/simple_ring_buffer.h
new file mode 100644
index 0000000000000..21aec556293eb
--- /dev/null
+++ b/include/linux/simple_ring_buffer.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_SIMPLE_RING_BUFFER_H
+#define _LINUX_SIMPLE_RING_BUFFER_H
+
+#include <linux/list.h>
+#include <linux/ring_buffer.h>
+#include <linux/ring_buffer_types.h>
+#include <linux/types.h>
+
+/*
+ * Ideally those struct would stay private but the caller needs to know
+ * the allocation size for simple_ring_buffer_init().
+ */
+struct simple_buffer_page {
+	struct list_head	link;
+	struct buffer_data_page	*page;
+	u64			entries;
+	u32			write;
+	u32			id;
+};
+
+struct simple_rb_per_cpu {
+	struct simple_buffer_page	*tail_page;
+	struct simple_buffer_page	*reader_page;
+	struct simple_buffer_page	*head_page;
+	struct simple_buffer_page	*bpages;
+	struct trace_buffer_meta	*meta;
+	u32				nr_pages;
+
+#define SIMPLE_RB_UNAVAILABLE	0
+#define SIMPLE_RB_READY		1
+#define SIMPLE_RB_WRITING	2
+	u32				status;
+
+	u64				last_overrun;
+	u64				write_stamp;
+
+	struct simple_rb_cbs		*cbs;
+};
+
+int simple_ring_buffer_init(struct simple_rb_per_cpu *cpu_buffer, struct simple_buffer_page *bpages,
+			    const struct ring_buffer_desc *desc);
+
+void simple_ring_buffer_unload(struct simple_rb_per_cpu *cpu_buffer);
+
+void *simple_ring_buffer_reserve(struct simple_rb_per_cpu *cpu_buffer, unsigned long length,
+				 u64 timestamp);
+
+void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer);
+
+int simple_ring_buffer_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable);
+
+int simple_ring_buffer_reset(struct simple_rb_per_cpu *cpu_buffer);
+
+int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer);
+
+int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer,
+			       struct simple_buffer_page *bpages,
+			       const struct ring_buffer_desc *desc,
+			       void *(*load_page)(unsigned long va),
+			       void (*unload_page)(void *va));
+
+void simple_ring_buffer_unload_mm(struct simple_rb_per_cpu *cpu_buffer,
+				  void (*unload_page)(void *));
+#endif
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index daa4e4944ce3f..9cc98f850f1d7 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -2682,8 +2682,17 @@ static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
 	shinfo->nr_frags = i + 1;
 }
 
-void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
-			    int off, int size, unsigned int truesize);
+static inline void skb_add_rx_frag_netmem(struct sk_buff *skb, int i,
+					  netmem_ref netmem, int off,
+					  int size, unsigned int truesize)
+{
+	DEBUG_NET_WARN_ON_ONCE(size > truesize);
+
+	skb_fill_netmem_desc(skb, i, netmem, off, size);
+	skb->len += size;
+	skb->data_len += size;
+	skb->truesize += truesize;
+}
 
 static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
 				   struct page *page, int off, int size,
diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h
index 8243ab3a12a8d..227125d843183 100644
--- a/include/linux/soc/qcom/llcc-qcom.h
+++ b/include/linux/soc/qcom/llcc-qcom.h
@@ -91,10 +91,12 @@
  * struct llcc_slice_desc - Cache slice descriptor
  * @slice_id: llcc slice id
  * @slice_size: Size allocated for the llcc slice
+ * @refcount: Atomic counter to track activate/deactivate calls
  */
 struct llcc_slice_desc {
 	u32 slice_id;
 	size_t slice_size;
+	refcount_t refcount;
 };
 
 /**
@@ -152,11 +154,10 @@ struct llcc_edac_reg_offset {
  * @edac_reg_offset: Offset of the LLCC EDAC registers
  * @lock: mutex associated with each slice
  * @cfg_size: size of the config data table
- * @max_slices: max slices as read from device tree
  * @num_banks: Number of llcc banks
- * @bitmap: Bit map to track the active slice ids
  * @ecc_irq: interrupt for llcc cache error detection and reporting
  * @ecc_irq_configured: 'True' if firmware has already configured the irq propagation
+ * @desc: Array pointer of pre-allocated LLCC slice descriptors
  * @version: Indicates the LLCC version
  */
 struct llcc_drv_data {
@@ -167,12 +168,11 @@ struct llcc_drv_data {
 	const struct llcc_edac_reg_offset *edac_reg_offset;
 	struct mutex lock;
 	u32 cfg_size;
-	u32 max_slices;
 	u32 num_banks;
-	unsigned long *bitmap;
 	int ecc_irq;
 	bool ecc_irq_configured;
 	u32 version;
+	struct llcc_slice_desc *desc;
 };
 
 #if IS_ENABLED(CONFIG_QCOM_LLCC)
diff --git a/include/linux/soc/qcom/pdr.h b/include/linux/soc/qcom/pdr.h
index 83a8ea612e69a..2b7691e47c2a9 100644
--- a/include/linux/soc/qcom/pdr.h
+++ b/include/linux/soc/qcom/pdr.h
@@ -5,6 +5,7 @@
 #include <linux/soc/qcom/qmi.h>
 
 #define SERVREG_NAME_LENGTH	64
+#define SERVREG_PFR_LENGTH	256
 
 struct pdr_service;
 struct pdr_handle;
diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h
index 291cdc7ef49ce..b9dcb437a0be5 100644
--- a/include/linux/soc/qcom/qmi.h
+++ b/include/linux/soc/qcom/qmi.h
@@ -92,6 +92,18 @@ struct qmi_elem_info {
 #define QMI_ERR_INCOMPATIBLE_STATE_V01		90
 #define QMI_ERR_NOT_SUPPORTED_V01		94
 
+/*
+ * Enumerate the IDs of the QMI services
+ */
+#define QMI_SERVICE_ID_TEST		0x0f	/*   15 */
+#define QMI_SERVICE_ID_SSCTL		0x2b	/*   43 */
+#define QMI_SERVICE_ID_IPA		0x31	/*   49 */
+#define QMI_SERVICE_ID_SERVREG_LOC	0x40	/*   64 */
+#define QMI_SERVICE_ID_SERVREG_NOTIF	0x42	/*   66 */
+#define QMI_SERVICE_ID_WLFW		0x45	/*   69 */
+#define QMI_SERVICE_ID_SLIMBUS		0x301	/*  769 */
+#define QMI_SERVICE_ID_USB_AUDIO_STREAM 0x41d	/* 1053 */
+
 /**
  * struct qmi_response_type_v01 - common response header (decoded)
  * @result:	result of the transaction
diff --git a/include/linux/socket.h b/include/linux/socket.h
index ec715ad4bf25f..ec4a0a0257939 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -415,7 +415,7 @@ struct __kernel_timespec;
 struct old_timespec32;
 
 struct scm_timestamping_internal {
-	struct timespec64 ts[3];
+	ktime_t ts[3];
 };
 
 extern void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_internal *tss);
diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h
index f462717acf206..6147eb1fb210d 100644
--- a/include/linux/soundwire/sdw.h
+++ b/include/linux/soundwire/sdw.h
@@ -532,7 +532,7 @@ struct sdw_slave_intr_status {
 };
 
 /**
- * sdw_reg_bank - SoundWire register banks
+ * enum sdw_reg_bank - SoundWire register banks
  * @SDW_BANK0: Soundwire register bank 0
  * @SDW_BANK1: Soundwire register bank 1
  */
@@ -751,7 +751,7 @@ struct sdw_port_params {
  * struct sdw_transport_params: Data Port Transport Parameters
  *
  * @blk_grp_ctrl_valid: Port implements block group control
- * @num: Port number
+ * @port_num: Port number
  * @blk_grp_ctrl: Block group control value
  * @sample_interval: Sample interval
  * @offset1: Blockoffset of the payload data
@@ -782,7 +782,7 @@ struct sdw_transport_params {
 /**
  * struct sdw_enable_ch: Enable/disable Data Port channel
  *
- * @num: Port number
+ * @port_num: Port number
  * @ch_mask: Active channel mask
  * @enable: Enable (true) /disable (false) channel
  */
@@ -885,7 +885,7 @@ void sdw_bus_master_delete(struct sdw_bus *bus);
 void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay);
 
 /**
- * sdw_port_config: Master or Slave Port configuration
+ * struct sdw_port_config: Master or Slave Port configuration
  *
  * @num: Port number
  * @ch_mask: channels mask for port
@@ -896,7 +896,7 @@ struct sdw_port_config {
 };
 
 /**
- * sdw_stream_config: Master or Slave stream configuration
+ * struct sdw_stream_config: Master or Slave stream configuration
  *
  * @frame_rate: Audio frame rate of the stream, in Hz
  * @ch_count: Channel count of the stream
@@ -913,7 +913,7 @@ struct sdw_stream_config {
 };
 
 /**
- * sdw_stream_state: Stream states
+ * enum sdw_stream_state: Stream states
  *
  * @SDW_STREAM_ALLOCATED: New stream allocated.
  * @SDW_STREAM_CONFIGURED: Stream configured
@@ -934,7 +934,7 @@ enum sdw_stream_state {
 };
 
 /**
- * sdw_stream_params: Stream parameters
+ * struct sdw_stream_params: Stream parameters
  *
  * @rate: Sampling frequency, in Hz
  * @ch_count: Number of channels
@@ -947,7 +947,7 @@ struct sdw_stream_params {
 };
 
 /**
- * sdw_stream_runtime: Runtime stream parameters
+ * struct sdw_stream_runtime: Runtime stream parameters
  *
  * @name: SoundWire stream name
  * @params: Stream parameters
@@ -983,7 +983,7 @@ struct sdw_stream_runtime {
  * @defer_msg: Defer message
  * @params: Current bus parameters
  * @stream_refcount: number of streams currently using this bus
- * @btp_stream_refcount: number of BTP streams currently using this bus (should
+ * @bpt_stream_refcount: number of BTP streams currently using this bus (should
  * be zero or one, multiple streams per link is not supported).
  * @bpt_stream: pointer stored to handle BTP streams.
  * @ops: Master callback ops
diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h
index fe31773d52108..470360a2723cf 100644
--- a/include/linux/soundwire/sdw_amd.h
+++ b/include/linux/soundwire/sdw_amd.h
@@ -66,8 +66,10 @@ struct sdw_amd_dai_runtime {
  * @status: peripheral devices status array
  * @num_din_ports: number of input ports
  * @num_dout_ports: number of output ports
+ * @max_ports: total number of input ports and output ports
  * @cols_index: Column index in frame shape
  * @rows_index: Rows index in frame shape
+ * @port_offset_map: dynamic array to map port block offset
  * @instance: SoundWire manager instance
  * @quirks: SoundWire manager quirks
  * @wake_en_mask: wake enable mask per SoundWire manager
@@ -92,10 +94,12 @@ struct amd_sdw_manager {
 
 	int num_din_ports;
 	int num_dout_ports;
+	int max_ports;
 
 	int cols_index;
 	int rows_index;
 
+	int *port_offset_map;
 	u32 instance;
 	u32 quirks;
 	u32 wake_en_mask;
diff --git a/include/linux/spi/spi-mem.h b/include/linux/spi/spi-mem.h
index 5774e554c0f02..37f709784350f 100644
--- a/include/linux/spi/spi-mem.h
+++ b/include/linux/spi/spi-mem.h
@@ -130,11 +130,13 @@ enum spi_mem_data_dir {
 
 /**
  * struct spi_mem_op - describes a SPI memory operation
+ * @cmd: the complete command
  * @cmd.nbytes: number of opcode bytes (only 1 or 2 are valid). The opcode is
  *		sent MSB-first.
  * @cmd.buswidth: number of IO lines used to transmit the command
  * @cmd.opcode: operation opcode
  * @cmd.dtr: whether the command opcode should be sent in DTR mode or not
+ * @addr: the address attributes
  * @addr.nbytes: number of address bytes to send. Can be zero if the operation
  *		 does not need to send an address
  * @addr.buswidth: number of IO lines used to transmit the address cycles
@@ -143,10 +145,12 @@ enum spi_mem_data_dir {
  *	      Note that only @addr.nbytes are taken into account in this
  *	      address value, so users should make sure the value fits in the
  *	      assigned number of bytes.
+ * @dummy: data for dummy operation
  * @dummy.nbytes: number of dummy bytes to send after an opcode or address. Can
  *		  be zero if the operation does not require dummy bytes
  * @dummy.buswidth: number of IO lanes used to transmit the dummy bytes
  * @dummy.dtr: whether the dummy bytes should be sent in DTR mode or not
+ * @data: the data attributes
  * @data.buswidth: number of IO lanes used to send/receive the data
  * @data.dtr: whether the data should be sent in DTR mode or not
  * @data.ecc: whether error correction is required or not
@@ -273,7 +277,7 @@ struct spi_mem {
 };
 
 /**
- * struct spi_mem_set_drvdata() - attach driver private data to a SPI mem
+ * spi_mem_set_drvdata() - attach driver private data to a SPI mem
  *				  device
  * @mem: memory device
  * @data: data to attach to the memory device
@@ -284,7 +288,7 @@ static inline void spi_mem_set_drvdata(struct spi_mem *mem, void *data)
 }
 
 /**
- * struct spi_mem_get_drvdata() - get driver private data attached to a SPI mem
+ * spi_mem_get_drvdata() - get driver private data attached to a SPI mem
  *				  device
  * @mem: memory device
  *
diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h
index af7cfee7b8f60..1c9aab627583c 100644
--- a/include/linux/spi/spi.h
+++ b/include/linux/spi/spi.h
@@ -387,6 +387,7 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv)
 }
 
 extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 chip_select);
+extern struct spi_device *devm_spi_new_ancillary_device(struct spi_device *spi, u8 chip_select);
 
 /* Use a define to avoid include chaining to get THIS_MODULE */
 #define spi_register_driver(driver) \
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index e1e2f144af9b4..241277cd34cf3 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -178,7 +178,7 @@ do {									\
 
 #ifdef CONFIG_DEBUG_SPINLOCK
  extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
- extern int do_raw_spin_trylock(raw_spinlock_t *lock);
+ extern int do_raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock);
  extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
 #else
 static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
@@ -189,6 +189,7 @@ static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
 }
 
 static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
+	__cond_acquires(true, lock)
 {
 	int ret = arch_spin_trylock(&(lock)->raw_lock);
 
diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h
index 1e84e71ca495e..3a50976471d71 100644
--- a/include/linux/spinlock_up.h
+++ b/include/linux/spinlock_up.h
@@ -48,16 +48,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 	lock->slock = 1;
 }
 
-/*
- * Read-write spinlocks. No debug version.
- */
-#define arch_read_lock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_write_lock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_read_trylock(lock)	({ barrier(); (void)(lock); 1; })
-#define arch_write_trylock(lock)	({ barrier(); (void)(lock); 1; })
-#define arch_read_unlock(lock)		do { barrier(); (void)(lock); } while (0)
-#define arch_write_unlock(lock)	do { barrier(); (void)(lock); } while (0)
-
 #else /* DEBUG_SPINLOCK */
 #define arch_spin_is_locked(lock)	((void)(lock), 0)
 /* for sched/core.c and kernel_lock.c: */
@@ -68,4 +58,14 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)
 
 #define arch_spin_is_contended(lock)	(((void)(lock), 0))
 
+/*
+ * Read-write spinlocks. No debug version.
+ */
+#define arch_read_lock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_write_lock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_read_trylock(lock)	({ barrier(); (void)(lock); 1; })
+#define arch_write_trylock(lock)	({ barrier(); (void)(lock); 1; })
+#define arch_read_unlock(lock)		do { barrier(); (void)(lock); } while (0)
+#define arch_write_unlock(lock)	do { barrier(); (void)(lock); } while (0)
+
 #endif /* __LINUX_SPINLOCK_UP_H */
diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
index 32352a216567e..72febd246bdbf 100644
--- a/include/linux/stmmac.h
+++ b/include/linux/stmmac.h
@@ -84,61 +84,82 @@ struct stmmac_priv;
 /* Platfrom data for platform device structure's platform_data field */
 
 struct stmmac_mdio_bus_data {
-	unsigned int phy_mask;
-	unsigned int pcs_mask;
-	unsigned int default_an_inband;
+	u32 phy_mask;
+	u32 pcs_mask;
 	int *irqs;
 	int probed_phy_irq;
 	bool needs_reset;
+	bool default_an_inband;
 };
 
 struct stmmac_dma_cfg {
+	/* pbl: programmable burst limit
+	 * txpbl: transmit programmable burst limit
+	 * rxpbl: receive programmable burst limit
+	 * If txpbl or rxpbl are zero, the value of pbl will be substituted.
+	 * Range 0 - 63.
+	 */
 	int pbl;
 	int txpbl;
 	int rxpbl;
+	/* pblx8: multiplies pbl, txpbl, rxpbl by a factor of 8 for dwmac >=
+	 * 3.50a, or a factor of 4 for previous versions.
+	 */
 	bool pblx8;
-	int fixed_burst;
-	int mixed_burst;
+	/* fixed_burst:
+	 *  when set, AXI bursts defined by axi_blen_regval are permitted.
+	 *   AHB uses SINGLE, INCR4, INCR8 or INCR16 during burst transfers.
+	 *  when clear, AXI and AHB use SINGLE or INCR bursts.
+	 */
+	bool fixed_burst;
+	/* mixed_burst:
+	 *  when set and fixed_burst is clear, AHB uses INCR for bursts > 16
+	 *  and SINGLE or INCRx for bursts <= 16.
+	 */
+	bool mixed_burst;
+	/* aal: address aligned bursts for AHB and AXI master interface */
 	bool aal;
+	bool dche;
 	bool eame;
+	/* multi_msi_en: stmmac core internal */
 	bool multi_msi_en;
-	bool dche;
+	/* atds: stmmac core internal */
 	bool atds;
 };
 
 #define AXI_BLEN	7
 struct stmmac_axi {
-	bool axi_lpi_en;
-	bool axi_xit_frm;
 	u32 axi_wr_osr_lmt;
 	u32 axi_rd_osr_lmt;
-	bool axi_kbbe;
 	u32 axi_blen_regval;
+	bool axi_lpi_en;
+	bool axi_xit_frm;
+	bool axi_kbbe;
 	bool axi_fb;
 	bool axi_mb;
 	bool axi_rb;
 };
 
 struct stmmac_rxq_cfg {
-	u8 mode_to_use;
 	u32 chan;
+	u32 prio;
+	u8 mode_to_use;
 	u8 pkt_route;
 	bool use_prio;
-	u32 prio;
 };
 
 struct stmmac_txq_cfg {
 	u32 weight;
-	bool coe_unsupported;
-	u8 mode_to_use;
 	/* Credit Base Shaper parameters */
 	u32 send_slope;
 	u32 idle_slope;
 	u32 high_credit;
 	u32 low_credit;
-	bool use_prio;
 	u32 prio;
 	int tbs_en;
+	bool use_prio;
+	bool coe_unsupported;
+	u8 mode_to_use;
 };
 
 struct stmmac_safety_feature_cfg {
@@ -225,28 +246,27 @@ struct plat_stmmacenet_data {
 	phy_interface_t phy_interface;
 	struct stmmac_mdio_bus_data *mdio_bus_data;
 	struct device_node *phy_node;
-	struct fwnode_handle *port_node;
 	struct device_node *mdio_node;
 	struct stmmac_dma_cfg *dma_cfg;
 	struct stmmac_safety_feature_cfg *safety_feat_cfg;
 	int clk_csr;
-	int enh_desc;
-	int tx_coe;
+	bool enh_desc;
+	bool tx_coe;
+	bool bugged_jumbo;
+	bool pmt;
+	bool force_sf_dma_mode;
+	bool force_thresh_dma_mode;
+	bool riwt_off;
 	int rx_coe;
-	int bugged_jumbo;
-	int pmt;
-	int force_sf_dma_mode;
-	int force_thresh_dma_mode;
-	int riwt_off;
 	int max_speed;
 	int maxmtu;
 	int multicast_filter_bins;
 	int unicast_filter_entries;
 	int tx_fifo_size;
 	int rx_fifo_size;
-	u32 host_dma_width;
-	u32 rx_queues_to_use;
-	u32 tx_queues_to_use;
+	u8 host_dma_width;
+	u8 rx_queues_to_use;
+	u8 tx_queues_to_use;
 	u8 rx_sched_algorithm;
 	u8 tx_sched_algorithm;
 	struct stmmac_rxq_cfg rx_queues_cfg[MTL_MAX_RX_QUEUES];
@@ -256,7 +276,8 @@ struct plat_stmmacenet_data {
 	int (*set_phy_intf_sel)(void *priv, u8 phy_intf_sel);
 	int (*set_clk_tx_rate)(void *priv, struct clk *clk_tx_i,
 			       phy_interface_t interface, int speed);
-	void (*fix_mac_speed)(void *priv, int speed, unsigned int mode);
+	void (*fix_mac_speed)(void *priv, phy_interface_t interface,
+			      int speed, unsigned int mode);
 	int (*fix_soc_reset)(struct stmmac_priv *priv);
 	int (*serdes_powerup)(struct net_device *ndev, void *priv);
 	void (*serdes_powerdown)(struct net_device *ndev, void *priv);
@@ -279,10 +300,41 @@ struct plat_stmmacenet_data {
 	struct phylink_pcs *(*select_pcs)(struct stmmac_priv *priv,
 					  phy_interface_t interface);
 	void *bsp_priv;
+
+	/* stmmac clocks:
+	 *  stmmac_clk: CSR clock (which can be hclk_i, clk_csr_i, aclk_i,
+	 *    or clk_app_i depending on GMAC configuration). This clock
+	 *    generates the MDC clock.
+	 *
+	 *  pclk: introduced for Imagination Technologies Pistachio board -
+	 *    see 5f9755d26fbf ("stmmac: Add an optional register interface
+	 *    clock"). This is probably used for cases where separate clocks
+	 *    are provided for the host interface and register interface. In
+	 *    this case, as the MDC clock is derived from stmmac_clk, pclk
+	 *    can only really be the "application clock" for the "host
+	 *    interface" and not the "register interface" aka CSR clock as
+	 *    it is never used when determining the divider for the MDC
+	 *    clock.
+	 *
+	 *  clk_ptp_ref: optional PTP reference clock (clk_ptp_ref_i). When
+	 *    present, this clock increments the timestamp value. Otherwise,
+	 *    the rate of stmmac_clk will be used.
+	 *
+	 *  clk_tx_i: MAC transmit clock, which will be 2.5MHz for 10M,
+	 *    25MHz for 100M, or 125MHz for 1G irrespective of the interface
+	 *    mode. For the DWMAC PHY interface modes:
+	 *
+	 *    GMII/MII	PHY's transmit clock for 10M (2.5MHz) or 100M (25MHz),
+	 *		or 125MHz local clock for 1G mode
+	 *    RMII	50MHz RMII clock divided by 2 or 20.
+	 *    RGMII	125MHz local clock divided by 1, 5, or 50.
+	 *    SGMII	125MHz SerDes clock divided by 1, 5, or 50.
+	 *    TBI/RTBI	125MHz SerDes clock
+	 */
 	struct clk *stmmac_clk;
 	struct clk *pclk;
 	struct clk *clk_ptp_ref;
-	struct clk *clk_tx_i;		/* clk_tx_i to MAC core */
+	struct clk *clk_tx_i;
 	unsigned long clk_ptp_rate;
 	unsigned long clk_ref_rate;
 	struct clk_bulk_data *clks;
@@ -296,7 +348,7 @@ struct plat_stmmacenet_data {
 	int rss_en;
 	int mac_port_sel_speed;
 	u8 vlan_fail_q;
-	struct pci_dev *pdev;
+	bool provide_bus_info;
 	int int_snapshot_num;
 	int msi_mac_vec;
 	int msi_wol_vec;
@@ -306,5 +358,6 @@ struct plat_stmmacenet_data {
 	int msi_tx_base_vec;
 	const struct dwmac4_addrs *dwmac4_addrs;
 	unsigned int flags;
+	struct stmmac_dma_cfg __dma_cfg;
 };
 #endif
diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h
index 72820503514cc..01011113d2263 100644
--- a/include/linux/stop_machine.h
+++ b/include/linux/stop_machine.h
@@ -99,7 +99,7 @@ static inline void print_stop_info(const char *log_lvl, struct task_struct *task
  * stop_machine: freeze the machine on all CPUs and run this function
  * @fn: the function to run
  * @data: the data ptr to pass to @fn()
- * @cpus: the cpus to run @fn() on (NULL = run on each online CPU)
+ * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU)
  *
  * Description: This causes a thread to be scheduled on every CPU, which
  * will run with interrupts disabled.  Each CPU specified by @cpus will
@@ -133,7 +133,7 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus);
  * stop_machine_cpuslocked: freeze the machine on all CPUs and run this function
  * @fn: the function to run
  * @data: the data ptr to pass to @fn()
- * @cpus: the cpus to run @fn() on (NULL = run on each online CPU)
+ * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU)
  *
  * Same as above.  Avoids nested calls to cpus_read_lock().
  *
diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h
index eb4bd62df3190..ab61bed2f7afc 100644
--- a/include/linux/sunrpc/debug.h
+++ b/include/linux/sunrpc/debug.h
@@ -38,6 +38,8 @@ extern unsigned int		nlm_debug;
 do {									\
 	ifdebug(fac)							\
 		__sunrpc_printk(fmt, ##__VA_ARGS__);			\
+	else								\
+		no_printk(fmt, ##__VA_ARGS__);				\
 } while (0)
 
 # define dfprintk_rcu(fac, fmt, ...)					\
@@ -46,15 +48,15 @@ do {									\
 		rcu_read_lock();					\
 		__sunrpc_printk(fmt, ##__VA_ARGS__);			\
 		rcu_read_unlock();					\
+	} else {							\
+		no_printk(fmt, ##__VA_ARGS__);				\
 	}								\
 } while (0)
 
-# define RPC_IFDEBUG(x)		x
 #else
 # define ifdebug(fac)		if (0)
-# define dfprintk(fac, fmt, ...)	do {} while (0)
-# define dfprintk_rcu(fac, fmt, ...)	do {} while (0)
-# define RPC_IFDEBUG(x)
+# define dfprintk(fac, fmt, ...)	no_printk(fmt, ##__VA_ARGS__)
+# define dfprintk_rcu(fac, fmt, ...)	no_printk(fmt, ##__VA_ARGS__)
 #endif
 
 /*
diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
index ccba79ebf8932..0dbdf3722537f 100644
--- a/include/linux/sunrpc/sched.h
+++ b/include/linux/sunrpc/sched.h
@@ -95,10 +95,7 @@ struct rpc_task {
 	int			tk_rpc_status;	/* Result of last RPC operation */
 	unsigned short		tk_flags;	/* misc flags */
 	unsigned short		tk_timeouts;	/* maj timeouts */
-
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS)
 	unsigned short		tk_pid;		/* debugging aid */
-#endif
 	unsigned char		tk_priority : 2,/* Task priority */
 				tk_garb_retry : 2,
 				tk_cred_retry : 2;
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 4dc14c7a711b0..992d5c9df6804 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -20,7 +20,7 @@
 #include <linux/lwq.h>
 #include <linux/wait.h>
 #include <linux/mm.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 #include <linux/kthread.h>
 
 /*
@@ -175,6 +175,9 @@ static inline unsigned long svc_serv_maxpages(const struct svc_serv *serv)
 /*
  * The context of a single thread, including the request currently being
  * processed.
+ *
+ * RPC programs are free to use rq_private to stash thread-local information.
+ * The sunrpc layer will not access it.
  */
 struct svc_rqst {
 	struct list_head	rq_all;		/* all threads list */
@@ -215,7 +218,6 @@ struct svc_rqst {
 	u32			rq_vers;	/* program version */
 	u32			rq_proc;	/* procedure number */
 	u32			rq_prot;	/* IP protocol */
-	int			rq_cachetype;	/* catering to nfsd */
 	unsigned long		rq_flags;	/* flags field */
 	ktime_t			rq_qtime;	/* enqueue time */
 
@@ -251,7 +253,7 @@ struct svc_rqst {
 	unsigned long		bc_to_initval;
 	unsigned int		bc_to_retries;
 	unsigned int		rq_status_counter; /* RPC processing counter */
-	void			**rq_lease_breaker; /* The v4 client breaking a lease */
+	void			*rq_private;	/* For use by the service thread */
 };
 
 /* bits for rq_flags */
diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h
new file mode 100644
index 0000000000000..e95e8f105624d
--- /dev/null
+++ b/include/linux/sunrpc/xdrgen/nlm4.h
@@ -0,0 +1,233 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Generated by xdrgen. Manual edits will be lost. */
+/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */
+/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */
+
+#ifndef _LINUX_XDRGEN_NLM4_DEF_H
+#define _LINUX_XDRGEN_NLM4_DEF_H
+
+#include <linux/types.h>
+#include <linux/sunrpc/xdrgen/_defs.h>
+
+enum { LM_MAXSTRLEN = 1024 };
+
+enum { LM_MAXNAMELEN = 1025 };
+
+enum { MAXNETOBJ_SZ = 1024 };
+
+typedef opaque netobj;
+
+enum fsh4_mode {
+	fsm_DN = 0,
+	fsm_DR = 1,
+	fsm_DW = 2,
+	fsm_DRW = 3,
+};
+
+typedef enum fsh4_mode fsh4_mode;
+
+enum fsh4_access {
+	fsa_NONE = 0,
+	fsa_R = 1,
+	fsa_W = 2,
+	fsa_RW = 3,
+};
+
+typedef enum fsh4_access fsh4_access;
+
+enum { SM_MAXSTRLEN = 1024 };
+
+typedef u64 uint64;
+
+typedef s64 int64;
+
+typedef u32 uint32;
+
+typedef s32 int32;
+
+enum nlm4_stats {
+	NLM4_GRANTED = 0,
+	NLM4_DENIED = 1,
+	NLM4_DENIED_NOLOCKS = 2,
+	NLM4_BLOCKED = 3,
+	NLM4_DENIED_GRACE_PERIOD = 4,
+	NLM4_DEADLCK = 5,
+	NLM4_ROFS = 6,
+	NLM4_STALE_FH = 7,
+	NLM4_FBIG = 8,
+	NLM4_FAILED = 9,
+};
+
+typedef __be32 nlm4_stats;
+
+struct nlm4_holder {
+	bool exclusive;
+	int32 svid;
+	netobj oh;
+	uint64 l_offset;
+	uint64 l_len;
+};
+
+struct nlm4_testrply {
+	nlm4_stats stat;
+	union {
+		struct nlm4_holder holder;
+	} u;
+};
+
+struct nlm4_stat {
+	nlm4_stats stat;
+};
+
+struct nlm4_res {
+	netobj cookie;
+	struct nlm4_stat stat;
+};
+
+struct nlm4_testres {
+	netobj cookie;
+	struct nlm4_testrply stat;
+};
+
+struct nlm4_lock {
+	string caller_name;
+	netobj fh;
+	netobj oh;
+	int32 svid;
+	uint64 l_offset;
+	uint64 l_len;
+};
+
+struct nlm4_lockargs {
+	netobj cookie;
+	bool block;
+	bool exclusive;
+	struct nlm4_lock alock;
+	bool reclaim;
+	int32 state;
+};
+
+struct nlm4_cancargs {
+	netobj cookie;
+	bool block;
+	bool exclusive;
+	struct nlm4_lock alock;
+};
+
+struct nlm4_testargs {
+	netobj cookie;
+	bool exclusive;
+	struct nlm4_lock alock;
+};
+
+struct nlm4_unlockargs {
+	netobj cookie;
+	struct nlm4_lock alock;
+};
+
+struct nlm4_share {
+	string caller_name;
+	netobj fh;
+	netobj oh;
+	fsh4_mode mode;
+	fsh4_access access;
+};
+
+struct nlm4_shareargs {
+	netobj cookie;
+	struct nlm4_share share;
+	bool reclaim;
+};
+
+struct nlm4_shareres {
+	netobj cookie;
+	nlm4_stats stat;
+	int32 sequence;
+};
+
+struct nlm4_notify {
+	string name;
+	int32 state;
+};
+
+enum { SM_PRIV_SIZE = 16 };
+
+struct nlm4_notifyargs {
+	struct nlm4_notify notify;
+	u8 private[SM_PRIV_SIZE];
+};
+
+enum {
+	NLMPROC4_NULL = 0,
+	NLMPROC4_TEST = 1,
+	NLMPROC4_LOCK = 2,
+	NLMPROC4_CANCEL = 3,
+	NLMPROC4_UNLOCK = 4,
+	NLMPROC4_GRANTED = 5,
+	NLMPROC4_TEST_MSG = 6,
+	NLMPROC4_LOCK_MSG = 7,
+	NLMPROC4_CANCEL_MSG = 8,
+	NLMPROC4_UNLOCK_MSG = 9,
+	NLMPROC4_GRANTED_MSG = 10,
+	NLMPROC4_TEST_RES = 11,
+	NLMPROC4_LOCK_RES = 12,
+	NLMPROC4_CANCEL_RES = 13,
+	NLMPROC4_UNLOCK_RES = 14,
+	NLMPROC4_GRANTED_RES = 15,
+	NLMPROC4_SM_NOTIFY = 16,
+	NLMPROC4_SHARE = 20,
+	NLMPROC4_UNSHARE = 21,
+	NLMPROC4_NM_LOCK = 22,
+	NLMPROC4_FREE_ALL = 23,
+};
+
+#ifndef NLM4_PROG
+#define NLM4_PROG (100021)
+#endif
+
+#define NLM4_netobj_sz                  (XDR_unsigned_int + XDR_QUADLEN(MAXNETOBJ_SZ))
+#define NLM4_fsh4_mode_sz               (XDR_int)
+#define NLM4_fsh4_access_sz             (XDR_int)
+#define NLM4_uint64_sz                  \
+	(XDR_unsigned_hyper)
+#define NLM4_int64_sz                   \
+	(XDR_hyper)
+#define NLM4_uint32_sz                  \
+	(XDR_unsigned_long)
+#define NLM4_int32_sz                   \
+	(XDR_long)
+#define NLM4_nlm4_stats_sz              (XDR_int)
+#define NLM4_nlm4_holder_sz             \
+	(XDR_bool + NLM4_int32_sz + NLM4_netobj_sz + NLM4_uint64_sz + NLM4_uint64_sz)
+#define NLM4_nlm4_testrply_sz           \
+	(NLM4_nlm4_stats_sz + NLM4_nlm4_holder_sz)
+#define NLM4_nlm4_stat_sz               \
+	(NLM4_nlm4_stats_sz)
+#define NLM4_nlm4_res_sz                \
+	(NLM4_netobj_sz + NLM4_nlm4_stat_sz)
+#define NLM4_nlm4_testres_sz            \
+	(NLM4_netobj_sz + NLM4_nlm4_testrply_sz)
+#define NLM4_nlm4_lock_sz               \
+	(XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_int32_sz + NLM4_uint64_sz + NLM4_uint64_sz)
+#define NLM4_nlm4_lockargs_sz           \
+	(NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz + XDR_bool + NLM4_int32_sz)
+#define NLM4_nlm4_cancargs_sz           \
+	(NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz)
+#define NLM4_nlm4_testargs_sz           \
+	(NLM4_netobj_sz + XDR_bool + NLM4_nlm4_lock_sz)
+#define NLM4_nlm4_unlockargs_sz         \
+	(NLM4_netobj_sz + NLM4_nlm4_lock_sz)
+#define NLM4_nlm4_share_sz              \
+	(XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_fsh4_mode_sz + NLM4_fsh4_access_sz)
+#define NLM4_nlm4_shareargs_sz          \
+	(NLM4_netobj_sz + NLM4_nlm4_share_sz + XDR_bool)
+#define NLM4_nlm4_shareres_sz           \
+	(NLM4_netobj_sz + NLM4_nlm4_stats_sz + NLM4_int32_sz)
+#define NLM4_nlm4_notify_sz             \
+	(XDR_unsigned_int + XDR_QUADLEN(LM_MAXNAMELEN) + NLM4_int32_sz)
+#define NLM4_nlm4_notifyargs_sz         \
+	(NLM4_nlm4_notify_sz + XDR_QUADLEN(SM_PRIV_SIZE))
+#define NLM4_MAX_ARGS_SZ                \
+	(NLM4_nlm4_lockargs_sz)
+
+#endif /* _LINUX_XDRGEN_NLM4_DEF_H */
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 62fc7499b4089..7a09df6977a5f 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -20,8 +20,6 @@ struct notifier_block;
 
 struct bio;
 
-struct pagevec;
-
 #define SWAP_FLAG_PREFER	0x8000	/* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK	0x7fff
 #define SWAP_FLAG_DISCARD	0x10000 /* enable discard for swap */
@@ -208,7 +206,6 @@ enum {
 	SWP_DISCARDABLE = (1 << 2),	/* blkdev support discard */
 	SWP_DISCARDING	= (1 << 3),	/* now discarding a free cluster */
 	SWP_SOLIDSTATE	= (1 << 4),	/* blkdev seeks are cheap */
-	SWP_CONTINUED	= (1 << 5),	/* swap_map has count continuation */
 	SWP_BLKDEV	= (1 << 6),	/* its a block device */
 	SWP_ACTIVATED	= (1 << 7),	/* set after swap_activate success */
 	SWP_FS_OPS	= (1 << 8),	/* swapfile operations go through fs */
@@ -223,16 +220,6 @@ enum {
 #define SWAP_CLUSTER_MAX_SKIPPED (SWAP_CLUSTER_MAX << 10)
 #define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
 
-/* Bit flag in swap_map */
-#define COUNT_CONTINUED	0x80	/* Flag swap_map continuation for full count */
-
-/* Special value in first swap_map */
-#define SWAP_MAP_MAX	0x3e	/* Max count */
-#define SWAP_MAP_BAD	0x3f	/* Note page is bad */
-
-/* Special value in each swap_map continuation */
-#define SWAP_CONT_MAX	0x7f	/* Max count */
-
 /*
  * The first page in the swap file is the swap header, which is always marked
  * bad to prevent it from being allocated as an entry. This also prevents the
@@ -264,8 +251,7 @@ struct swap_info_struct {
 	signed short	prio;		/* swap priority of this type */
 	struct plist_node list;		/* entry in swap_active_head */
 	signed char	type;		/* strange name for an index */
-	unsigned int	max;		/* extent of the swap_map */
-	unsigned char *swap_map;	/* vmalloc'ed array of usage counts */
+	unsigned int	max;		/* size of this swap device */
 	unsigned long *zeromap;		/* kvmalloc'ed bitmap to track zero pages */
 	struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
 	struct list_head free_clusters; /* free clusters list */
@@ -284,18 +270,14 @@ struct swap_info_struct {
 	struct completion comp;		/* seldom referenced */
 	spinlock_t lock;		/*
 					 * protect map scan related fields like
-					 * swap_map, inuse_pages and all cluster
-					 * lists. other fields are only changed
+					 * inuse_pages and all cluster lists.
+					 * Other fields are only changed
 					 * at swapon/swapoff, so are protected
 					 * by swap_lock. changing flags need
 					 * hold this lock and swap_lock. If
 					 * both locks need hold, hold swap_lock
 					 * first.
 					 */
-	spinlock_t cont_lock;		/*
-					 * protect swap count continuation page
-					 * list.
-					 */
 	struct work_struct discard_work; /* discard worker */
 	struct work_struct reclaim_work; /* reclaim worker */
 	struct list_head discard_clusters; /* discard clusters list */
@@ -328,8 +310,7 @@ extern unsigned long totalreserve_pages;
 
 /* linux/mm/swap.c */
 void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
-		unsigned int nr_io, unsigned int nr_rotated)
-		__releases(lruvec->lru_lock);
+		unsigned int nr_io, unsigned int nr_rotated);
 void lru_note_cost_refault(struct folio *);
 void folio_add_lru(struct folio *);
 void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
@@ -371,6 +352,7 @@ extern void swap_setup(void);
 extern unsigned long zone_reclaimable_pages(struct zone *zone);
 extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 					gfp_t gfp_mask, nodemask_t *mask);
+unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
 
 #define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
 #define MEMCG_RECLAIM_PROACTIVE (1 << 2)
@@ -451,7 +433,6 @@ static inline long get_nr_swap_pages(void)
 }
 
 extern void si_swapinfo(struct sysinfo *);
-extern int add_swap_count_continuation(swp_entry_t, gfp_t);
 int swap_type_of(dev_t device, sector_t offset);
 int find_first_swap(dev_t *device);
 extern unsigned int count_swap_pages(int, int);
@@ -517,11 +498,6 @@ static inline void free_swap_cache(struct folio *folio)
 {
 }
 
-static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask)
-{
-	return 0;
-}
-
 static inline int swap_dup_entry_direct(swp_entry_t ent)
 {
 	return 0;
@@ -571,6 +547,8 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
 
 	return READ_ONCE(memcg->swappiness);
 }
+
+void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
 #else
 static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
 {
@@ -635,5 +613,24 @@ static inline bool mem_cgroup_swap_full(struct folio *folio)
 }
 #endif
 
+/* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to
+ * and including the specified highidx
+ * @zone: The current zone in the iterator
+ * @pgdat: The pgdat which node_zones are being iterated
+ * @idx: The index variable
+ * @highidx: The index of the highest zone to return
+ *
+ * This macro iterates through all managed zones up to and including the specified highidx.
+ * The zone iterator enters an invalid state after macro call and must be reinitialized
+ * before it can be used again.
+ */
+#define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx)	\
+	for ((idx) = 0, (zone) = (pgdat)->node_zones;		\
+	    (idx) <= (highidx);					\
+	    (idx)++, (zone)++)					\
+		if (!managed_zone(zone))			\
+			continue;				\
+		else
+
 #endif /* __KERNEL__*/
 #endif /* _LINUX_SWAP_H */
diff --git a/include/linux/sys_soc.h b/include/linux/sys_soc.h
index d9b3cf0f410c8..f19f5cec18e28 100644
--- a/include/linux/sys_soc.h
+++ b/include/linux/sys_soc.h
@@ -37,6 +37,16 @@ void soc_device_unregister(struct soc_device *soc_dev);
  */
 struct device *soc_device_to_device(struct soc_device *soc);
 
+/**
+ * soc_attr_read_machine - retrieve the machine model and store it in
+ *                         the soc_device_attribute structure
+ * @soc_dev_attr: SoC attribute structure to store the model in
+ *
+ * Returns:
+ * 0 on success, negative error number on failure.
+ */
+int soc_attr_read_machine(struct soc_device_attribute *soc_dev_attr);
+
 #ifdef CONFIG_SOC_BUS
 const struct soc_device_attribute *soc_device_match(
 	const struct soc_device_attribute *matches);
diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h
index 99b775f3ff469..9777e9445dd51 100644
--- a/include/linux/sysfs.h
+++ b/include/linux/sysfs.h
@@ -445,15 +445,15 @@ void sysfs_delete_link(struct kobject *dir, struct kobject *targ,
 int __must_check sysfs_create_group(struct kobject *kobj,
 				    const struct attribute_group *grp);
 int __must_check sysfs_create_groups(struct kobject *kobj,
-				     const struct attribute_group **groups);
+				     const struct attribute_group *const *groups);
 int __must_check sysfs_update_groups(struct kobject *kobj,
-				     const struct attribute_group **groups);
+				     const struct attribute_group *const *groups);
 int sysfs_update_group(struct kobject *kobj,
 		       const struct attribute_group *grp);
 void sysfs_remove_group(struct kobject *kobj,
 			const struct attribute_group *grp);
 void sysfs_remove_groups(struct kobject *kobj,
-			 const struct attribute_group **groups);
+			 const struct attribute_group *const *groups);
 int sysfs_add_file_to_group(struct kobject *kobj,
 			const struct attribute *attr, const char *group);
 void sysfs_remove_file_from_group(struct kobject *kobj,
@@ -486,7 +486,7 @@ int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t kgid);
 int sysfs_link_change_owner(struct kobject *kobj, struct kobject *targ,
 			    const char *name, kuid_t kuid, kgid_t kgid);
 int sysfs_groups_change_owner(struct kobject *kobj,
-			      const struct attribute_group **groups,
+			      const struct attribute_group *const *groups,
 			      kuid_t kuid, kgid_t kgid);
 int sysfs_group_change_owner(struct kobject *kobj,
 			     const struct attribute_group *groups, kuid_t kuid,
@@ -629,13 +629,13 @@ static inline int sysfs_create_group(struct kobject *kobj,
 }
 
 static inline int sysfs_create_groups(struct kobject *kobj,
-				      const struct attribute_group **groups)
+				      const struct attribute_group *const *groups)
 {
 	return 0;
 }
 
 static inline int sysfs_update_groups(struct kobject *kobj,
-				      const struct attribute_group **groups)
+				      const struct attribute_group *const *groups)
 {
 	return 0;
 }
@@ -652,7 +652,7 @@ static inline void sysfs_remove_group(struct kobject *kobj,
 }
 
 static inline void sysfs_remove_groups(struct kobject *kobj,
-				       const struct attribute_group **groups)
+				       const struct attribute_group *const *groups)
 {
 }
 
@@ -733,7 +733,7 @@ static inline int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t k
 }
 
 static inline int sysfs_groups_change_owner(struct kobject *kobj,
-			  const struct attribute_group **groups,
+			  const struct attribute_group *const *groups,
 			  kuid_t kuid, kgid_t kgid)
 {
 	return 0;
diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index f72eef31fa23c..6982f10e826b4 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -228,8 +228,7 @@ struct tcp_sock {
 	u32	sacked_out;	/* SACK'd packets			*/
 	u16	tcp_header_len;	/* Bytes of tcp header to send		*/
 	u8	scaling_ratio;	/* see tcp_win_from_space() */
-	u8	chrono_type : 2,	/* current chronograph type */
-		repair      : 1,
+	u8	repair      : 1,
 		tcp_usec_ts : 1, /* TSval values in usec */
 		is_sack_reneg:1,    /* in recovery from loss with SACK reneg? */
 		is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */
@@ -264,6 +263,7 @@ struct tcp_sock {
 				 * total number of data bytes sent.
 				 */
 	u32	snd_sml;	/* Last byte of the most recently transmitted small packet */
+	u8	chrono_type;	/* current chronograph type */
 	u32	chrono_start;	/* Start time in jiffies of a TCP chrono */
 	u32	chrono_stat[3];	/* Time in jiffies for chrono_stat stats */
 	u32	write_seq;	/* Tail(+1) of data held in tcp send buffer */
@@ -316,6 +316,9 @@ struct tcp_sock {
 					*/
 	u32	app_limited;	/* limited until "delivered" reaches this val */
 	u32	rcv_wnd;	/* Current receiver window		*/
+	u32	rcv_mwnd_seq;	/* Maximum window sequence number (RFC 7323,
+				 * section 2.4, receiver requirements)
+				 */
 	u32	rcv_tstamp;	/* timestamp of last received ACK (for keepalives) */
 /*
  *      Options received (usually on last packet, some only on SYN packets).
@@ -548,6 +551,13 @@ enum tsq_flags {
 	TCPF_ACK_DEFERRED		= BIT(TCP_ACK_DEFERRED),
 };
 
+/* Flags of interest for tcp_release_cb() */
+#define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |		\
+			  TCPF_WRITE_TIMER_DEFERRED |	\
+			  TCPF_DELACK_TIMER_DEFERRED |	\
+			  TCPF_MTU_REDUCED_DEFERRED |	\
+			  TCPF_ACK_DEFERRED)
+
 #define tcp_sk(ptr) container_of_const(ptr, struct tcp_sock, inet_conn.icsk_inet.sk)
 
 /* Variant of tcp_sk() upgrading a const sock to a read/write tcp socket.
diff --git a/include/linux/tee_core.h b/include/linux/tee_core.h
index ee5f0bd41f43a..f993d5118edda 100644
--- a/include/linux/tee_core.h
+++ b/include/linux/tee_core.h
@@ -50,7 +50,7 @@ enum tee_dma_heap_id {
  * @dev:	embedded basic device structure
  * @cdev:	embedded cdev
  * @num_users:	number of active users of this device
- * @c_no_user:	completion used when unregistering the device
+ * @c_no_users:	completion used when unregistering the device
  * @mutex:	mutex protecting @num_users and @idr
  * @idr:	register of user space shared memory objects allocated or
  *		registered on this device
@@ -132,6 +132,7 @@ struct tee_driver_ops {
 /* Size for TEE revision string buffer used by get_tee_revision(). */
 #define TEE_REVISION_STR_SIZE	128
 
+#define TEE_DESC_PRIVILEGED	0x1
 /**
  * struct tee_desc - Describes the TEE driver to the subsystem
  * @name:	name of driver
@@ -139,7 +140,6 @@ struct tee_driver_ops {
  * @owner:	module providing the driver
  * @flags:	Extra properties of driver, defined by TEE_DESC_* below
  */
-#define TEE_DESC_PRIVILEGED	0x1
 struct tee_desc {
 	const char *name;
 	const struct tee_driver_ops *ops;
@@ -187,7 +187,7 @@ struct tee_protmem_pool_ops {
  * Allocates a new struct tee_device instance. The device is
  * removed by tee_device_unregister().
  *
- * @returns a pointer to a 'struct tee_device' or an ERR_PTR on failure
+ * @returns: a pointer to a 'struct tee_device' or an ERR_PTR on failure
  */
 struct tee_device *tee_device_alloc(const struct tee_desc *teedesc,
 				    struct device *dev,
@@ -201,7 +201,7 @@ struct tee_device *tee_device_alloc(const struct tee_desc *teedesc,
  * tee_device_unregister() need to be called to remove the @teedev if
  * this function fails.
  *
- * @returns < 0 on failure
+ * @returns: < 0 on failure
  */
 int tee_device_register(struct tee_device *teedev);
 
@@ -254,14 +254,14 @@ void tee_device_set_dev_groups(struct tee_device *teedev,
  * tee_session_calc_client_uuid() - Calculates client UUID for session
  * @uuid:		Resulting UUID
  * @connection_method:	Connection method for session (TEE_IOCTL_LOGIN_*)
- * @connectuon_data:	Connection data for opening session
+ * @connection_data:	Connection data for opening session
  *
  * Based on connection method calculates UUIDv5 based client UUID.
  *
  * For group based logins verifies that calling process has specified
  * credentials.
  *
- * @return < 0 on failure
+ * @returns: < 0 on failure
  */
 int tee_session_calc_client_uuid(uuid_t *uuid, u32 connection_method,
 				 const u8 connection_data[TEE_IOCTL_UUID_LEN]);
@@ -295,7 +295,7 @@ struct tee_shm_pool_ops {
  * @paddr:	Physical address of start of pool
  * @size:	Size in bytes of the pool
  *
- * @returns pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure.
+ * @returns: pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure.
  */
 struct tee_shm_pool *tee_shm_pool_alloc_res_mem(unsigned long vaddr,
 						phys_addr_t paddr, size_t size,
@@ -318,14 +318,16 @@ static inline void tee_shm_pool_free(struct tee_shm_pool *pool)
  * @paddr:	Physical address of start of pool
  * @size:	Size in bytes of the pool
  *
- * @returns pointer to a 'struct tee_protmem_pool' or an ERR_PTR on failure.
+ * @returns: pointer to a 'struct tee_protmem_pool' or an ERR_PTR on failure.
  */
 struct tee_protmem_pool *tee_protmem_static_pool_alloc(phys_addr_t paddr,
 						       size_t size);
 
 /**
  * tee_get_drvdata() - Return driver_data pointer
- * @returns the driver_data pointer supplied to tee_register().
+ * @teedev: Pointer to the tee_device
+ *
+ * @returns: the driver_data pointer supplied to tee_register().
  */
 void *tee_get_drvdata(struct tee_device *teedev);
 
@@ -334,7 +336,7 @@ void *tee_get_drvdata(struct tee_device *teedev);
  *                            TEE driver
  * @ctx:	The TEE context for shared memory allocation
  * @size:	Shared memory allocation size
- * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
+ * @returns: a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
  */
 struct tee_shm *tee_shm_alloc_priv_buf(struct tee_context *ctx, size_t size);
 
@@ -354,7 +356,7 @@ void tee_dyn_shm_free_helper(struct tee_shm *shm,
 /**
  * tee_shm_is_dynamic() - Check if shared memory object is of the dynamic kind
  * @shm:	Shared memory handle
- * @returns true if object is dynamic shared memory
+ * @returns: true if object is dynamic shared memory
  */
 static inline bool tee_shm_is_dynamic(struct tee_shm *shm)
 {
@@ -370,7 +372,7 @@ void tee_shm_put(struct tee_shm *shm);
 /**
  * tee_shm_get_id() - Get id of a shared memory object
  * @shm:	Shared memory handle
- * @returns id
+ * @returns: id
  */
 static inline int tee_shm_get_id(struct tee_shm *shm)
 {
@@ -382,7 +384,7 @@ static inline int tee_shm_get_id(struct tee_shm *shm)
  * count
  * @ctx:	Context owning the shared memory
  * @id:		Id of shared memory object
- * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
+ * @returns: a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
  */
 struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id);
 
@@ -402,7 +404,7 @@ static inline bool tee_param_is_memref(struct tee_param *param)
  * teedev_open() - Open a struct tee_device
  * @teedev:	Device to open
  *
- * @return a pointer to struct tee_context on success or an ERR_PTR on failure.
+ * @returns: pointer to struct tee_context on success or an ERR_PTR on failure.
  */
 struct tee_context *teedev_open(struct tee_device *teedev);
 
diff --git a/include/linux/tegra-mipi-cal.h b/include/linux/tegra-mipi-cal.h
new file mode 100644
index 0000000000000..2a540b50f65da
--- /dev/null
+++ b/include/linux/tegra-mipi-cal.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __TEGRA_MIPI_CAL_H_
+#define __TEGRA_MIPI_CAL_H_
+
+struct tegra_mipi_device {
+	const struct tegra_mipi_ops *ops;
+	struct platform_device *pdev;
+	unsigned long pads;
+};
+
+/**
+ * Operations for Tegra MIPI calibration device
+ */
+struct tegra_mipi_ops {
+	/**
+	 * @enable:
+	 *
+	 * Enable MIPI calibration device
+	 */
+	int (*enable)(struct tegra_mipi_device *device);
+
+	/**
+	 * @disable:
+	 *
+	 * Disable MIPI calibration device
+	 */
+	int (*disable)(struct tegra_mipi_device *device);
+
+	/**
+	 * @start_calibration:
+	 *
+	 * Start MIPI calibration
+	 */
+	int (*start_calibration)(struct tegra_mipi_device *device);
+
+	/**
+	 * @finish_calibration:
+	 *
+	 * Finish MIPI calibration
+	 */
+	int (*finish_calibration)(struct tegra_mipi_device *device);
+};
+
+int devm_tegra_mipi_add_provider(struct device *device, struct device_node *np,
+				 const struct tegra_mipi_ops *ops);
+
+struct tegra_mipi_device *tegra_mipi_request(struct device *device,
+					     struct device_node *np);
+void tegra_mipi_free(struct tegra_mipi_device *device);
+
+int tegra_mipi_enable(struct tegra_mipi_device *device);
+int tegra_mipi_disable(struct tegra_mipi_device *device);
+int tegra_mipi_start_calibration(struct tegra_mipi_device *device);
+int tegra_mipi_finish_calibration(struct tegra_mipi_device *device);
+
+#endif /* __TEGRA_MIPI_CAL_H_ */
diff --git a/include/linux/timb_gpio.h b/include/linux/timb_gpio.h
index 3faf5a6bb13e3..74f5e73bf6db6 100644
--- a/include/linux/timb_gpio.h
+++ b/include/linux/timb_gpio.h
@@ -9,10 +9,10 @@
 
 /**
  * struct timbgpio_platform_data - Platform data of the Timberdale GPIO driver
- * @gpio_base		The number of the first GPIO pin, set to -1 for
+ * @gpio_base:		The number of the first GPIO pin, set to -1 for
  *			dynamic number allocation.
- * @nr_pins		Number of pins that is supported by the hardware (1-32)
- * @irq_base		If IRQ is supported by the hardware, this is the base
+ * @nr_pins:		Number of pins that is supported by the hardware (1-32)
+ * @irq_base:		If IRQ is supported by the hardware, this is the base
  *			number of IRQ:s. One IRQ per pin will be used. Set to
  *			-1 if IRQ:s is not supported.
  */
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index b8ae89ea28abf..e36d11e33e0c1 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -72,6 +72,10 @@ struct tk_read_base {
  * @id:				The timekeeper ID
  * @tkr_raw:			The readout base structure for CLOCK_MONOTONIC_RAW
  * @raw_sec:			CLOCK_MONOTONIC_RAW  time in seconds
+ * @cs_id:			The ID of the current clocksource
+ * @cs_ns_to_cyc_mult:		Multiplicator for nanoseconds to cycles conversion
+ * @cs_ns_to_cyc_shift:		Shift value for nanoseconds to cycles conversion
+ * @cs_ns_to_cyc_maxns:		Maximum nanoseconds to cyles conversion range
  * @clock_was_set_seq:		The sequence number of clock was set events
  * @cs_was_changed_seq:		The sequence number of clocksource change events
  * @clock_valid:		Indicator for valid clock
@@ -159,6 +163,10 @@ struct timekeeper {
 	u64			raw_sec;
 
 	/* Cachline 3 and 4 (timekeeping internal variables): */
+	enum clocksource_ids	cs_id;
+	u32			cs_ns_to_cyc_mult;
+	u32			cs_ns_to_cyc_shift;
+	u64			cs_ns_to_cyc_maxns;
 	unsigned int		clock_was_set_seq;
 	u8			cs_was_changed_seq;
 	u8			clock_valid;
diff --git a/include/linux/timerqueue.h b/include/linux/timerqueue.h
index d306d9dd22073..7d0aaa7665803 100644
--- a/include/linux/timerqueue.h
+++ b/include/linux/timerqueue.h
@@ -5,12 +5,11 @@
 #include <linux/rbtree.h>
 #include <linux/timerqueue_types.h>
 
-extern bool timerqueue_add(struct timerqueue_head *head,
-			   struct timerqueue_node *node);
-extern bool timerqueue_del(struct timerqueue_head *head,
-			   struct timerqueue_node *node);
-extern struct timerqueue_node *timerqueue_iterate_next(
-						struct timerqueue_node *node);
+bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node);
+bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node);
+struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node);
+
+bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node);
 
 /**
  * timerqueue_getnext - Returns the timer with the earliest expiration time
@@ -19,8 +18,7 @@ extern struct timerqueue_node *timerqueue_iterate_next(
  *
  * Returns a pointer to the timer node that has the earliest expiration time.
  */
-static inline
-struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
+static inline struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
 {
 	struct rb_node *leftmost = rb_first_cached(&head->rb_root);
 
@@ -41,4 +39,46 @@ static inline void timerqueue_init_head(struct timerqueue_head *head)
 {
 	head->rb_root = RB_ROOT_CACHED;
 }
+
+/* Timer queues with linked nodes */
+
+static __always_inline
+struct timerqueue_linked_node *timerqueue_linked_first(struct timerqueue_linked_head *head)
+{
+	return rb_entry_safe(head->rb_root.rb_leftmost, struct timerqueue_linked_node, node);
+}
+
+static __always_inline
+struct timerqueue_linked_node *timerqueue_linked_next(struct timerqueue_linked_node *node)
+{
+	return rb_entry_safe(node->node.next, struct timerqueue_linked_node, node);
+}
+
+static __always_inline
+struct timerqueue_linked_node *timerqueue_linked_prev(struct timerqueue_linked_node *node)
+{
+	return rb_entry_safe(node->node.prev, struct timerqueue_linked_node, node);
+}
+
+static __always_inline
+bool timerqueue_linked_del(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node)
+{
+	return rb_erase_linked(&node->node, &head->rb_root);
+}
+
+static __always_inline void timerqueue_linked_init(struct timerqueue_linked_node *node)
+{
+	RB_CLEAR_LINKED_NODE(&node->node);
+}
+
+static __always_inline bool timerqueue_linked_node_queued(struct timerqueue_linked_node *node)
+{
+	return !RB_EMPTY_LINKED_NODE(&node->node);
+}
+
+static __always_inline void timerqueue_linked_init_head(struct timerqueue_linked_head *head)
+{
+	head->rb_root = RB_ROOT_LINKED;
+}
+
 #endif /* _LINUX_TIMERQUEUE_H */
diff --git a/include/linux/timerqueue_types.h b/include/linux/timerqueue_types.h
index dc298d0923e3b..be2218b147c4b 100644
--- a/include/linux/timerqueue_types.h
+++ b/include/linux/timerqueue_types.h
@@ -6,12 +6,21 @@
 #include <linux/types.h>
 
 struct timerqueue_node {
-	struct rb_node node;
-	ktime_t expires;
+	struct rb_node		node;
+	ktime_t			expires;
 };
 
 struct timerqueue_head {
-	struct rb_root_cached rb_root;
+	struct rb_root_cached	rb_root;
+};
+
+struct timerqueue_linked_node {
+	struct rb_node_linked		node;
+	ktime_t				expires;
+};
+
+struct timerqueue_linked_head {
+	struct rb_root_linked		rb_root;
 };
 
 #endif /* _LINUX_TIMERQUEUE_TYPES_H */
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 37eb2f0f3dd8e..40a43a4c7caf4 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -22,20 +22,23 @@ union bpf_attr;
 
 const char *trace_print_flags_seq(struct trace_seq *p, const char *delim,
 				  unsigned long flags,
-				  const struct trace_print_flags *flag_array);
+				  const struct trace_print_flags *flag_array,
+				  size_t flag_array_size);
 
 const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
-				    const struct trace_print_flags *symbol_array);
+				    const struct trace_print_flags *symbol_array,
+				    size_t symbol_array_size);
 
 #if BITS_PER_LONG == 32
 const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
 		      unsigned long long flags,
-		      const struct trace_print_flags_u64 *flag_array);
+		      const struct trace_print_flags_u64 *flag_array,
+		      size_t flag_array_size);
 
 const char *trace_print_symbols_seq_u64(struct trace_seq *p,
 					unsigned long long val,
-					const struct trace_print_flags_u64
-								 *symbol_array);
+					const struct trace_print_flags_u64 *symbol_array,
+					size_t symbol_array_size);
 #endif
 
 struct trace_iterator;
diff --git a/include/linux/trace_remote.h b/include/linux/trace_remote.h
new file mode 100644
index 0000000000000..fcd1d46ea466f
--- /dev/null
+++ b/include/linux/trace_remote.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _LINUX_TRACE_REMOTE_H
+#define _LINUX_TRACE_REMOTE_H
+
+#include <linux/dcache.h>
+#include <linux/ring_buffer.h>
+#include <linux/trace_remote_event.h>
+
+/**
+ * struct trace_remote_callbacks - Callbacks used by Tracefs to control the remote
+ * @init:		Called once the remote has been registered. Allows the
+ *			caller to extend the Tracefs remote directory
+ * @load_trace_buffer:  Called before Tracefs accesses the trace buffer for the first
+ *			time. Must return a &trace_buffer_desc
+ *			(most likely filled with trace_remote_alloc_buffer())
+ * @unload_trace_buffer:
+ *			Called once Tracefs has no use for the trace buffer
+ *			(most likely call trace_remote_free_buffer())
+ * @enable_tracing:	Called on Tracefs tracing_on. It is expected from the
+ *			remote to allow writing.
+ * @swap_reader_page:	Called when Tracefs consumes a new page from a
+ *			ring-buffer. It is expected from the remote to isolate a
+ * @reset:		Called on `echo 0 > trace`. It is expected from the
+ *			remote to reset all ring-buffer pages.
+ *			new reader-page from the @cpu ring-buffer.
+ * @enable_event:	Called on events/event_name/enable. It is expected from
+ *			the remote to allow the writing event @id.
+ */
+struct trace_remote_callbacks {
+	int	(*init)(struct dentry *d, void *priv);
+	struct trace_buffer_desc *(*load_trace_buffer)(unsigned long size, void *priv);
+	void	(*unload_trace_buffer)(struct trace_buffer_desc *desc, void *priv);
+	int	(*enable_tracing)(bool enable, void *priv);
+	int	(*swap_reader_page)(unsigned int cpu, void *priv);
+	int	(*reset)(unsigned int cpu, void *priv);
+	int	(*enable_event)(unsigned short id, bool enable, void *priv);
+};
+
+int trace_remote_register(const char *name, struct trace_remote_callbacks *cbs, void *priv,
+			  struct remote_event *events, size_t nr_events);
+
+int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size,
+			      const struct cpumask *cpumask);
+
+void trace_remote_free_buffer(struct trace_buffer_desc *desc);
+
+#endif
diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h
new file mode 100644
index 0000000000000..c8ae1e1f5e721
--- /dev/null
+++ b/include/linux/trace_remote_event.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _LINUX_TRACE_REMOTE_EVENTS_H
+#define _LINUX_TRACE_REMOTE_EVENTS_H
+
+struct trace_remote;
+struct trace_event_fields;
+struct trace_seq;
+
+struct remote_event_hdr {
+	unsigned short	id;
+};
+
+#define REMOTE_EVENT_NAME_MAX 30
+struct remote_event {
+	char				name[REMOTE_EVENT_NAME_MAX];
+	unsigned short			id;
+	bool				enabled;
+	struct trace_remote		*remote;
+	struct trace_event_fields	*fields;
+	char				*print_fmt;
+	void				(*print)(void *evt, struct trace_seq *seq);
+};
+
+#define RE_STRUCT(__args...) __args
+#define re_field(__type, __field) __type __field;
+
+#define REMOTE_EVENT_FORMAT(__name, __struct)	\
+	struct remote_event_format_##__name {	\
+		struct remote_event_hdr hdr;	\
+		__struct			\
+	}
+#endif
diff --git a/include/linux/tty_buffer.h b/include/linux/tty_buffer.h
index 31125e3be3c55..48adcb0e8ff39 100644
--- a/include/linux/tty_buffer.h
+++ b/include/linux/tty_buffer.h
@@ -34,6 +34,7 @@ static inline u8 *flag_buf_ptr(struct tty_buffer *b, unsigned int ofs)
 
 struct tty_bufhead {
 	struct tty_buffer *head;	/* Queue head */
+	struct workqueue_struct *flip_wq;
 	struct work_struct work;
 	struct mutex	   lock;
 	atomic_t	   priority;
diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h
index 188ee9b768eb6..1f2896e56e779 100644
--- a/include/linux/tty_driver.h
+++ b/include/linux/tty_driver.h
@@ -69,6 +69,10 @@ struct serial_struct;
  *	Do not create numbered ``/dev`` nodes. For example, create
  *	``/dev/ttyprintk`` and not ``/dev/ttyprintk0``. Applicable only when a
  *	driver for a single tty device is being allocated.
+ *
+ * @TTY_DRIVER_NO_WORKQUEUE:
+ *	Do not create workqueue when tty_register_driver(). Whenever set, flip
+ *	buffer workqueue can be set by tty_port_link_wq() for every port.
  */
 enum tty_driver_flag {
 	TTY_DRIVER_INSTALLED		= BIT(0),
@@ -79,6 +83,7 @@ enum tty_driver_flag {
 	TTY_DRIVER_HARDWARE_BREAK	= BIT(5),
 	TTY_DRIVER_DYNAMIC_ALLOC	= BIT(6),
 	TTY_DRIVER_UNNUMBERED_NODE	= BIT(7),
+	TTY_DRIVER_NO_WORKQUEUE		= BIT(8),
 };
 
 enum tty_driver_type {
@@ -506,6 +511,7 @@ struct tty_operations {
  * @flags: tty driver flags (%TTY_DRIVER_)
  * @proc_entry: proc fs entry, used internally
  * @other: driver of the linked tty; only used for the PTY driver
+ * @flip_wq: workqueue to queue flip buffer work on
  * @ttys: array of active &struct tty_struct, set by tty_standard_install()
  * @ports: array of &struct tty_port; can be set during initialization by
  *	   tty_port_link_device() and similar
@@ -539,6 +545,7 @@ struct tty_driver {
 	unsigned long	flags;
 	struct proc_dir_entry *proc_entry;
 	struct tty_driver *other;
+	struct workqueue_struct *flip_wq;
 
 	/*
 	 * Pointer to the tty data structures
diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h
index c5cccc3fc1e8d..d227a58e3e490 100644
--- a/include/linux/tty_ldisc.h
+++ b/include/linux/tty_ldisc.h
@@ -266,7 +266,7 @@ struct tty_ldisc_ops {
 };
 
 struct tty_ldisc {
-	struct tty_ldisc_ops *ops;
+	const struct tty_ldisc_ops *ops;
 	struct tty_struct *tty;
 };
 
@@ -281,8 +281,8 @@ struct tty_ldisc *tty_ldisc_ref_wait(struct tty_struct *);
 
 void tty_ldisc_flush(struct tty_struct *tty);
 
-int tty_register_ldisc(struct tty_ldisc_ops *new_ldisc);
-void tty_unregister_ldisc(struct tty_ldisc_ops *ldisc);
+int tty_register_ldisc(const struct tty_ldisc_ops *new_ldisc);
+void tty_unregister_ldisc(const struct tty_ldisc_ops *ldisc);
 int tty_set_ldisc(struct tty_struct *tty, int disc);
 
 #endif /* _LINUX_TTY_LDISC_H */
diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h
index 660c254f1efe5..d2a7882c0b58b 100644
--- a/include/linux/tty_port.h
+++ b/include/linux/tty_port.h
@@ -138,6 +138,7 @@ struct tty_port {
 					   kernel */
 
 void tty_port_init(struct tty_port *port);
+void tty_port_link_wq(struct tty_port *port, struct workqueue_struct *flip_wq);
 void tty_port_link_device(struct tty_port *port, struct tty_driver *driver,
 		unsigned index);
 struct device *tty_port_register_device(struct tty_port *port,
@@ -165,6 +166,18 @@ static inline struct tty_port *tty_port_get(struct tty_port *port)
 	return NULL;
 }
 
+/*
+ * Never overwrite the workqueue set by tty_port_link_wq().
+ * No effect when %TTY_DRIVER_NO_WORKQUEUE is set, as driver->flip_wq is
+ * %NULL.
+ */
+static inline void tty_port_link_driver_wq(struct tty_port *port,
+					   struct tty_driver *driver)
+{
+	if (!port->buf.flip_wq)
+		tty_port_link_wq(port, driver->flip_wq);
+}
+
 /* If the cts flow control is enabled, return true. */
 static inline bool tty_port_cts_enabled(const struct tty_port *port)
 {
diff --git a/include/linux/types.h b/include/linux/types.h
index 7e71d260763c7..608050dbca6a7 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -239,7 +239,7 @@ struct ustat {
  *
  * This guarantee is important for few reasons:
  *  - future call_rcu_lazy() will make use of lower bits in the pointer;
- *  - the structure shares storage space in struct page with @compound_head,
+ *  - the structure shares storage space in struct page with @compound_info,
  *    which encode PageTail() in bit 0. The guarantee is needed to avoid
  *    false-positive PageTail().
  */
diff --git a/include/linux/udp.h b/include/linux/udp.h
index 1cbf6b4d3aab3..ce56ebcee5cbc 100644
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -40,8 +40,6 @@ enum {
 	UDP_FLAGS_ACCEPT_FRAGLIST,
 	UDP_FLAGS_ACCEPT_L4,
 	UDP_FLAGS_ENCAP_ENABLED, /* This socket enabled encap */
-	UDP_FLAGS_UDPLITE_SEND_CC, /* set via udplite setsockopt */
-	UDP_FLAGS_UDPLITE_RECV_CC, /* set via udplite setsockopt */
 };
 
 /* per NUMA structure for lockless producer usage. */
@@ -74,11 +72,7 @@ struct udp_sock {
 	 */
 	__u16		 len;		/* total length of pending frames */
 	__u16		 gso_size;
-	/*
-	 * Fields specific to UDP-Lite.
-	 */
-	__u16		 pcslen;
-	__u16		 pcrlen;
+
 	/*
 	 * For encapsulation sockets.
 	 */
@@ -236,8 +230,6 @@ static inline void udp_allow_gso(struct sock *sk)
 	hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
 #endif
 
-#define IS_UDPLITE(__sk) (unlikely(__sk->sk_protocol == IPPROTO_UDPLITE))
-
 static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
 {
 #if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h
index 334641e20fb12..02eaac47ac444 100644
--- a/include/linux/uio_driver.h
+++ b/include/linux/uio_driver.h
@@ -97,7 +97,7 @@ struct uio_device {
  * @irq_flags:		flags for request_irq()
  * @priv:		optional private data
  * @handler:		the device's irq handler
- * @mmap:		mmap operation for this uio device
+ * @mmap_prepare:	mmap_prepare operation for this uio device
  * @open:		open operation for this uio device
  * @release:		release operation for this uio device
  * @irqcontrol:		disable/enable irqs when 0/1 is written to /dev/uioX
@@ -112,7 +112,7 @@ struct uio_info {
 	unsigned long		irq_flags;
 	void			*priv;
 	irqreturn_t (*handler)(int irq, struct uio_info *dev_info);
-	int (*mmap)(struct uio_info *info, struct vm_area_struct *vma);
+	int (*mmap_prepare)(struct uio_info *info, struct vm_area_desc *desc);
 	int (*open)(struct uio_info *info, struct inode *inode);
 	int (*release)(struct uio_info *info, struct inode *inode);
 	int (*irqcontrol)(struct uio_info *info, s32 irq_on);
diff --git a/include/linux/unicode.h b/include/linux/unicode.h
index 5e6b212a2aeda..64fa44fe180c4 100644
--- a/include/linux/unicode.h
+++ b/include/linux/unicode.h
@@ -66,9 +66,6 @@ int utf8_strncasecmp_folded(const struct unicode_map *um,
 			    const struct qstr *cf,
 			    const struct qstr *s1);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen);
-
 int utf8_casefold(const struct unicode_map *um, const struct qstr *str,
 		  unsigned char *dest, size_t dlen);
 
diff --git a/include/linux/units.h b/include/linux/units.h
index 80d57c50b9e3b..c6d78988613a0 100644
--- a/include/linux/units.h
+++ b/include/linux/units.h
@@ -57,6 +57,9 @@
 #define MICROWATT_PER_MILLIWATT	1000UL
 #define MICROWATT_PER_WATT	1000000UL
 
+#define MICROJOULE_PER_JOULE	1000000UL
+#define NANOJOULE_PER_JOULE	1000000000UL
+
 #define BYTES_PER_KBIT		(KILO / BITS_PER_BYTE)
 #define BYTES_PER_MBIT		(MEGA / BITS_PER_BYTE)
 #define BYTES_PER_GBIT		(GIGA / BITS_PER_BYTE)
diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h
index 4ac082a631738..97ef37a1ff4ac 100644
--- a/include/linux/usb/cdc_ncm.h
+++ b/include/linux/usb/cdc_ncm.h
@@ -118,8 +118,8 @@ struct cdc_ncm_ctx {
 
 	u32 timer_interval;
 	u32 max_ndp_size;
-	u8 is_ndp16;
-	u8 filtering_supported;
+	bool is_ndp16;
+	bool filtering_supported;
 	union {
 		struct usb_cdc_ncm_ndp16 *delayed_ndp16;
 		struct usb_cdc_ncm_ndp32 *delayed_ndp32;
diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h
index 6ccd1b2af993e..337a5485af7c7 100644
--- a/include/linux/usb/pd.h
+++ b/include/linux/usb/pd.h
@@ -34,7 +34,8 @@ enum pd_ctrl_msg_type {
 	PD_CTRL_FR_SWAP = 19,
 	PD_CTRL_GET_PPS_STATUS = 20,
 	PD_CTRL_GET_COUNTRY_CODES = 21,
-	/* 22-23 Reserved */
+	PD_CTRL_GET_SINK_CAP_EXT = 22,
+	/* 23 Reserved */
 	PD_CTRL_GET_REVISION = 24,
 	/* 25-31 Reserved */
 };
@@ -72,7 +73,8 @@ enum pd_ext_msg_type {
 	PD_EXT_PPS_STATUS = 12,
 	PD_EXT_COUNTRY_INFO = 13,
 	PD_EXT_COUNTRY_CODES = 14,
-	/* 15-31 Reserved */
+	PD_EXT_SINK_CAP_EXT = 15,
+	/* 16-31 Reserved */
 };
 
 #define PD_REV10	0x0
@@ -205,6 +207,72 @@ struct pd_message {
 	};
 } __packed;
 
+/*
+ * count_chunked_data_objs - Helper to calculate number of Data Objects on a 4
+ *   byte boundary.
+ * @size: Size of data block for extended message. Should *not* include extended
+ *   header size.
+ */
+static inline u8 count_chunked_data_objs(u32 size)
+{
+	size += offsetof(struct pd_chunked_ext_message_data, data);
+	return ((size / 4) + (size % 4 ? 1 : 0));
+}
+
+/* Sink Caps Extended Data Block Version */
+#define SKEDB_VER_1_0				1
+
+/* Sink Caps Extended Sink Modes */
+#define SINK_MODE_PPS		BIT(0)
+#define SINK_MODE_VBUS		BIT(1)
+#define SINK_MODE_AC_SUPPLY	BIT(2)
+#define SINK_MODE_BATT		BIT(3)
+#define SINK_MODE_BATT_UL	BIT(4) /* Unlimited battery power supply */
+#define SINK_MODE_AVS		BIT(5)
+
+/**
+ * struct sink_caps_ext_msg - Sink extended capability PD message
+ * @vid: Vendor ID
+ * @pid: Product ID
+ * @xid: Value assigned by USB-IF for product
+ * @fw: Firmware version
+ * @hw: Hardware version
+ * @skedb_ver: Sink Caps Extended Data Block (SKEDB) Version
+ * @load_step: Indicates the load step slew rate.
+ * @load_char: Sink overload characteristics
+ * @compliance: Types of sources the sink has been tested & certified on
+ * @touch_temp: Indicates the IEC standard to which the touch temperature
+ *              conforms to (if applicable).
+ * @batt_info: Indicates number batteries and hot swappable ports
+ * @modes: Charging caps & power sources supported
+ * @spr_min_pdp: Sink Minimum PDP for SPR mode
+ * @spr_op_pdp: Sink Operational PDP for SPR mode
+ * @spr_max_pdp: Sink Maximum PDP for SPR mode
+ * @epr_min_pdp: Sink Minimum PDP for EPR mode
+ * @epr_op_pdp: Sink Operational PDP for EPR mode
+ * @epr_max_pdp: Sink Maximum PDP for EPR mode
+ */
+struct sink_caps_ext_msg {
+	__le16 vid;
+	__le16 pid;
+	__le32 xid;
+	u8 fw;
+	u8 hw;
+	u8 skedb_ver;
+	u8 load_step;
+	__le16 load_char;
+	u8 compliance;
+	u8 touch_temp;
+	u8 batt_info;
+	u8 modes;
+	u8 spr_min_pdp;
+	u8 spr_op_pdp;
+	u8 spr_max_pdp;
+	u8 epr_min_pdp;
+	u8 epr_op_pdp;
+	u8 epr_max_pdp;
+} __packed;
+
 /* PDO: Power Data Object */
 #define PDO_MAX_OBJECTS		7
 
@@ -329,6 +397,32 @@ enum pd_apdo_type {
 #define PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR	GENMASK(19, 10)	/* 10mA unit */
 #define PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR	GENMASK(9, 0)	/* 10mA unit */
 
+/* SPR AVS has two different current ranges 9V - 15V, 15V - 20V */
+#define SPR_AVS_TIER1_MIN_VOLT_MV              9000
+#define SPR_AVS_TIER1_MAX_VOLT_MV              15000
+#define SPR_AVS_TIER2_MAX_VOLT_MV              20000
+
+#define SPR_AVS_AVS_SMALL_STEP_V	1
+/* vAvsStep - 100mv */
+#define SPR_AVS_VOLT_MV_STEP		100
+/* SPR AVS RDO Operating Current is in 50mA step */
+#define RDO_SPR_AVS_CURR_MA_STEP	50
+/* SPR AVS RDO Output voltage is in 25mV step */
+#define RDO_SPR_AVS_OUT_VOLT_MV_STEP	25
+
+#define RDO_SPR_AVS_VOLT	GENMASK(20, 9)
+#define RDO_SPR_AVS_CURR	GENMASK(6, 0)
+
+#define RDO_SPR_AVS_OUT_VOLT(mv)					\
+	FIELD_PREP(RDO_SPR_AVS_VOLT, ((mv) / RDO_SPR_AVS_OUT_VOLT_MV_STEP))
+
+#define RDO_SPR_AVS_OP_CURR(ma)						\
+	FIELD_PREP(RDO_SPR_AVS_CURR, ((ma) / RDO_SPR_AVS_CURR_MA_STEP))
+
+#define RDO_AVS(idx, out_mv, op_ma, flags)				\
+	(RDO_OBJ(idx) | (flags) |					\
+	 RDO_SPR_AVS_OUT_VOLT(out_mv) | RDO_SPR_AVS_OP_CURR(op_ma))
+
 static inline enum pd_pdo_type pdo_type(u32 pdo)
 {
 	return (pdo >> PDO_TYPE_SHIFT) & PDO_TYPE_MASK;
@@ -339,6 +433,11 @@ static inline unsigned int pdo_fixed_voltage(u32 pdo)
 	return ((pdo >> PDO_FIXED_VOLT_SHIFT) & PDO_VOLT_MASK) * 50;
 }
 
+static inline unsigned int pdo_fixed_current(u32 pdo)
+{
+	return ((pdo >> PDO_FIXED_CURR_SHIFT) & PDO_CURR_MASK) * 10;
+}
+
 static inline unsigned int pdo_min_voltage(u32 pdo)
 {
 	return ((pdo >> PDO_VAR_MIN_VOLT_SHIFT) & PDO_VOLT_MASK) * 50;
@@ -582,6 +681,11 @@ static inline unsigned int rdo_max_power(u32 rdo)
 
 #define PD_P_SNK_STDBY_MW	2500	/* 2500 mW */
 
+#define PD_I_SNK_STBY_MA		500	/* 500 mA */
+
+#define PD_T_AVS_SRC_TRANS_SMALL	50	/* 50 ms */
+#define PD_T_AVS_SRC_TRANS_LARGE	700	/* 700 ms */
+
 #if IS_ENABLED(CONFIG_TYPEC)
 
 struct usb_power_delivery;
diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h
index b22e659f81ba5..93079450bba01 100644
--- a/include/linux/usb/tcpm.h
+++ b/include/linux/usb/tcpm.h
@@ -31,7 +31,7 @@ enum typec_cc_polarity {
 /* Time to wait for TCPC to complete transmit */
 #define PD_T_TCPC_TX_TIMEOUT	100		/* in ms	*/
 #define PD_ROLE_SWAP_TIMEOUT	(MSEC_PER_SEC * 10)
-#define PD_PPS_CTRL_TIMEOUT	(MSEC_PER_SEC * 10)
+#define PD_AUG_PSY_CTRL_TIMEOUT	(MSEC_PER_SEC * 10)
 
 enum tcpm_transmit_status {
 	TCPC_TX_SUCCESS = 0,
diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h
index 0513d333b7977..b90cc5cfff8de 100644
--- a/include/linux/usb/typec_altmode.h
+++ b/include/linux/usb/typec_altmode.h
@@ -26,6 +26,9 @@ struct typec_altmode_ops;
  * @mode: Index of the Mode
  * @vdo: VDO returned by Discover Modes USB PD command
  * @active: Tells has the mode been entered or not
+ * @priority: Priority used by the automatic alternate mode selection process
+ * @mode_selection: Whether entry to this alternate mode is managed by the
+ * automatic alternate mode selection process or by the specific driver
  * @desc: Optional human readable description of the mode
  * @ops: Operations vector from the driver
  * @cable_ops: Cable operations vector from the driver.
diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index fd5f427654973..3bd2003328dcb 100644
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -23,6 +23,9 @@
 /* The set of all possible UFFD-related VM flags. */
 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
 
+#define __VMA_UFFD_FLAGS mk_vma_flags(VMA_UFFD_MISSING_BIT, VMA_UFFD_WP_BIT, \
+				      VMA_UFFD_MINOR_BIT)
+
 /*
  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
  * new flags, since they might collide with O_* ones. We want
@@ -80,6 +83,39 @@ struct userfaultfd_ctx {
 
 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
 
+/* VMA userfaultfd operations */
+struct vm_uffd_ops {
+	/* Checks if a VMA can support userfaultfd */
+	bool (*can_userfault)(struct vm_area_struct *vma, vm_flags_t vm_flags);
+	/*
+	 * Called to resolve UFFDIO_CONTINUE request.
+	 * Should return the folio found at pgoff in the VMA's pagecache if it
+	 * exists or ERR_PTR otherwise.
+	 * The returned folio is locked and with reference held.
+	 */
+	struct folio *(*get_folio_noalloc)(struct inode *inode, pgoff_t pgoff);
+	/*
+	 * Called during resolution of UFFDIO_COPY request.
+	 * Should allocate and return a folio or NULL if allocation
+	 * fails.
+	 */
+	struct folio *(*alloc_folio)(struct vm_area_struct *vma,
+				     unsigned long addr);
+	/*
+	 * Called during resolution of UFFDIO_COPY request.
+	 * Should lock the folio and add it to VMA's page cache.
+	 * Returns 0 on success, error code on failure.
+	 */
+	int (*filemap_add)(struct folio *folio, struct vm_area_struct *vma,
+			 unsigned long addr);
+	/*
+	 * Called during resolution of UFFDIO_COPY request on the error
+	 * handling path.
+	 * Should revert the operation of ->filemap_add().
+	 */
+	void (*filemap_remove)(struct folio *folio, struct vm_area_struct *vma);
+};
+
 /* A combined operation mode + behavior flags. */
 typedef unsigned int __bitwise uffd_flags_t;
 
@@ -111,11 +147,6 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at
 /* Flags controlling behavior. These behavior changes are mode-independent. */
 #define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0)
 
-extern int mfill_atomic_install_pte(pmd_t *dst_pmd,
-				    struct vm_area_struct *dst_vma,
-				    unsigned long dst_addr, struct page *page,
-				    bool newly_allocated, uffd_flags_t flags);
-
 extern ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start,
 				 unsigned long src_start, unsigned long len,
 				 uffd_flags_t flags);
@@ -208,39 +239,8 @@ static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 	return vma->vm_flags & __VM_UFFD_FLAGS;
 }
 
-static inline bool vma_can_userfault(struct vm_area_struct *vma,
-				     vm_flags_t vm_flags,
-				     bool wp_async)
-{
-	vm_flags &= __VM_UFFD_FLAGS;
-
-	if (vma->vm_flags & VM_DROPPABLE)
-		return false;
-
-	if ((vm_flags & VM_UFFD_MINOR) &&
-	    (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma)))
-		return false;
-
-	/*
-	 * If wp async enabled, and WP is the only mode enabled, allow any
-	 * memory type.
-	 */
-	if (wp_async && (vm_flags == VM_UFFD_WP))
-		return true;
-
-	/*
-	 * If user requested uffd-wp but not enabled pte markers for
-	 * uffd-wp, then shmem & hugetlbfs are not supported but only
-	 * anonymous.
-	 */
-	if (!uffd_supports_wp_marker() && (vm_flags & VM_UFFD_WP) &&
-	    !vma_is_anonymous(vma))
-		return false;
-
-	/* By default, allow any of anon|shmem|hugetlb */
-	return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
-	    vma_is_shmem(vma);
-}
+bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
+		       bool wp_async);
 
 static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma)
 {
diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
index a91fa24b06e09..0b530428db711 100644
--- a/include/linux/vdso_datastore.h
+++ b/include/linux/vdso_datastore.h
@@ -2,9 +2,15 @@
 #ifndef _LINUX_VDSO_DATASTORE_H
 #define _LINUX_VDSO_DATASTORE_H
 
+#ifdef CONFIG_HAVE_GENERIC_VDSO
 #include <linux/mm_types.h>
 
 extern const struct vm_special_mapping vdso_vvar_mapping;
 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
 
+void __init vdso_setup_data_pages(void);
+#else /* !CONFIG_HAVE_GENERIC_VDSO */
+static inline void vdso_setup_data_pages(void) { }
+#endif /* CONFIG_HAVE_GENERIC_VDSO */
+
 #endif /* _LINUX_VDSO_DATASTORE_H */
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 22a139f82d75f..03fe95f5a0201 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -38,21 +38,8 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
 		PGFAULT, PGMAJFAULT,
 		PGLAZYFREED,
-		PGREFILL,
 		PGREUSE,
-		PGSTEAL_KSWAPD,
-		PGSTEAL_DIRECT,
-		PGSTEAL_KHUGEPAGED,
-		PGSTEAL_PROACTIVE,
-		PGSCAN_KSWAPD,
-		PGSCAN_DIRECT,
-		PGSCAN_KHUGEPAGED,
-		PGSCAN_PROACTIVE,
 		PGSCAN_DIRECT_THROTTLE,
-		PGSCAN_ANON,
-		PGSCAN_FILE,
-		PGSTEAL_ANON,
-		PGSTEAL_FILE,
 #ifdef CONFIG_NUMA
 		PGSCAN_ZONE_RECLAIM_SUCCESS,
 		PGSCAN_ZONE_RECLAIM_FAILED,
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index e8e94f90d6866..3b02c0c6b3718 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -286,8 +286,6 @@ int unregister_vmap_purge_notifier(struct notifier_block *nb);
 #ifdef CONFIG_MMU
 #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START)
 
-unsigned long vmalloc_nr_pages(void);
-
 int vm_area_map_pages(struct vm_struct *area, unsigned long start,
 		      unsigned long end, struct page **pages);
 void vm_area_unmap_pages(struct vm_struct *area, unsigned long start,
@@ -304,7 +302,6 @@ static inline void set_vm_flush_reset_perms(void *addr)
 #else  /* !CONFIG_MMU */
 #define VMALLOC_TOTAL 0UL
 
-static inline unsigned long vmalloc_nr_pages(void) { return 0; }
 static inline void set_vm_flush_reset_perms(void *addr) {}
 #endif /* CONFIG_MMU */
 
diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h
index 9e29d79fc790a..ace7379d627d2 100644
--- a/include/linux/wait_bit.h
+++ b/include/linux/wait_bit.h
@@ -406,7 +406,7 @@ do {									\
 			  schedule())
 
 /**
- * wait_var_event_killable - wait for a variable to be updated and notified
+ * wait_var_event_interruptible - wait for a variable to be updated and notified
  * @var: the address of variable being waited on
  * @condition: the condition to wait for
  *
@@ -492,7 +492,7 @@ do {									\
  * wait_var_event_mutex - wait for a variable to be updated under a mutex
  * @var: the address of the variable being waited on
  * @condition: condition to wait for
- * @mutex: the mutex which protects updates to the variable
+ * @lock: the mutex which protects updates to the variable
  *
  * Wait for a condition which can only be reliably tested while holding
  * a mutex.  The variables assessed in the condition will normal be
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index a4749f56398fd..9f971912c6be2 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -440,6 +440,9 @@ enum wq_consts {
  * system_long_wq is similar to system_percpu_wq but may host long running
  * works.  Queue flushing might take relatively long.
  *
+ * system_dfl_long_wq is similar to system_dfl_wq but it may host long running
+ * works.
+ *
  * system_dfl_wq is unbound workqueue.  Workers are not bound to
  * any specific CPU, not concurrency managed, and all queued works are
  * executed immediately as long as max_active limit is not reached and
@@ -468,6 +471,7 @@ extern struct workqueue_struct *system_power_efficient_wq;
 extern struct workqueue_struct *system_freezable_power_efficient_wq;
 extern struct workqueue_struct *system_bh_wq;
 extern struct workqueue_struct *system_bh_highpri_wq;
+extern struct workqueue_struct *system_dfl_long_wq;
 
 void workqueue_softirq_action(bool highpri);
 void workqueue_softirq_dead(unsigned int cpu);
@@ -512,6 +516,26 @@ __printf(1, 4) struct workqueue_struct *
 alloc_workqueue_noprof(const char *fmt, unsigned int flags, int max_active, ...);
 #define alloc_workqueue(...)	alloc_hooks(alloc_workqueue_noprof(__VA_ARGS__))
 
+/**
+ * devm_alloc_workqueue - Resource-managed allocate a workqueue
+ * @dev: Device to allocate workqueue for
+ * @fmt: printf format for the name of the workqueue
+ * @flags: WQ_* flags
+ * @max_active: max in-flight work items, 0 for default
+ * @...: args for @fmt
+ *
+ * Resource managed workqueue, see alloc_workqueue() for details.
+ *
+ * The workqueue will be automatically destroyed on driver detach.  Typically
+ * this should be used in drivers already relying on devm interafaces.
+ *
+ * RETURNS:
+ * Pointer to the allocated workqueue on success, %NULL on failure.
+ */
+__printf(2, 5) struct workqueue_struct *
+devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags,
+		     int max_active, ...);
+
 #ifdef CONFIG_LOCKDEP
 /**
  * alloc_workqueue_lockdep_map - allocate a workqueue with user-defined lockdep_map
@@ -568,6 +592,8 @@ alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, int max_active,
  */
 #define alloc_ordered_workqueue(fmt, flags, args...)			\
 	alloc_workqueue(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args)
+#define devm_alloc_ordered_workqueue(dev, fmt, flags, args...)		\
+	devm_alloc_workqueue(dev, fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args)
 
 #define create_workqueue(name)						\
 	alloc_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_PERCPU, 1, (name))
@@ -712,14 +738,14 @@ static inline bool schedule_work_on(int cpu, struct work_struct *work)
 }
 
 /**
- * schedule_work - put work task in global workqueue
+ * schedule_work - put work task in per-CPU workqueue
  * @work: job to be done
  *
- * Returns %false if @work was already on the kernel-global workqueue and
+ * Returns %false if @work was already on the system per-CPU workqueue and
  * %true otherwise.
  *
- * This puts a job in the kernel-global workqueue if it was not already
- * queued and leaves it in the same position on the kernel-global
+ * This puts a job in the system per-CPU workqueue if it was not already
+ * queued and leaves it in the same position on the system per-CPU
  * workqueue otherwise.
  *
  * Shares the same memory-ordering properties of queue_work(), cf. the
@@ -783,6 +809,8 @@ extern void __warn_flushing_systemwide_wq(void)
 	     _wq == system_highpri_wq) ||				\
 	    (__builtin_constant_p(_wq == system_long_wq) &&		\
 	     _wq == system_long_wq) ||					\
+	    (__builtin_constant_p(_wq == system_dfl_long_wq) &&		\
+	     _wq == system_dfl_long_wq) ||					\
 	    (__builtin_constant_p(_wq == system_dfl_wq) &&		\
 	     _wq == system_dfl_wq) ||				\
 	    (__builtin_constant_p(_wq == system_freezable_wq) &&	\
@@ -796,12 +824,12 @@ extern void __warn_flushing_systemwide_wq(void)
 })
 
 /**
- * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
+ * schedule_delayed_work_on - queue work in per-CPU workqueue on CPU after delay
  * @cpu: cpu to use
  * @dwork: job to be done
  * @delay: number of jiffies to wait
  *
- * After waiting for a given time this puts a job in the kernel-global
+ * After waiting for a given time this puts a job in the system per-CPU
  * workqueue on the specified CPU.
  */
 static inline bool schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
@@ -811,11 +839,11 @@ static inline bool schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
 }
 
 /**
- * schedule_delayed_work - put work task in global workqueue after delay
+ * schedule_delayed_work - put work task in per-CPU workqueue after delay
  * @dwork: job to be done
  * @delay: number of jiffies to wait or 0 for immediate execution
  *
- * After waiting for a given time this puts a job in the kernel-global
+ * After waiting for a given time this puts a job in the system per-CPU
  * workqueue.
  */
 static inline bool schedule_delayed_work(struct delayed_work *dwork,
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index e530112c4b3a6..62552a2ce5b9e 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -11,7 +11,7 @@
 #include <linux/flex_proportions.h>
 #include <linux/backing-dev-defs.h>
 #include <linux/blk_types.h>
-#include <linux/pagevec.h>
+#include <linux/folio_batch.h>
 
 struct bio;
 
diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h
index 85b1fff02fde9..0c95ead5a2977 100644
--- a/include/linux/ww_mutex.h
+++ b/include/linux/ww_mutex.h
@@ -181,7 +181,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx,
  * data structures.
  */
 static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
-	__releases(ctx) __acquires_shared(ctx) __no_context_analysis
+	__must_hold(ctx)
 {
 #ifdef DEBUG_WW_MUTEXES
 	lockdep_assert_held(ctx);
@@ -199,7 +199,7 @@ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
  * mutexes have been released with ww_mutex_unlock.
  */
 static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx)
-	__releases_shared(ctx) __no_context_analysis
+	__releases(ctx) __no_context_analysis
 {
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	mutex_release(&ctx->first_lock_dep_map, _THIS_IP_);
diff --git a/include/linux/xattr.h b/include/linux/xattr.h
index 296b5ee5c979e..8b6601367eae8 100644
--- a/include/linux/xattr.h
+++ b/include/linux/xattr.h
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/spinlock.h>
 #include <linux/mm.h>
+#include <linux/rhashtable-types.h>
 #include <linux/user_namespace.h>
 #include <uapi/linux/xattr.h>
 
@@ -106,31 +107,65 @@ static inline const char *xattr_prefix(const struct xattr_handler *handler)
 }
 
 struct simple_xattrs {
-	struct rb_root rb_root;
-	rwlock_t lock;
+	struct rhashtable ht;
 };
 
 struct simple_xattr {
-	struct rb_node rb_node;
+	struct rhash_head hash_node;
+	struct rcu_head rcu;
 	char *name;
 	size_t size;
 	char value[] __counted_by(size);
 };
 
-void simple_xattrs_init(struct simple_xattrs *xattrs);
+#define SIMPLE_XATTR_MAX_NR		128
+#define SIMPLE_XATTR_MAX_SIZE		(128 << 10)
+
+struct simple_xattr_limits {
+	atomic_t	nr_xattrs;	/* current user.* xattr count */
+	atomic_t	xattr_size;	/* current total user.* value bytes */
+};
+
+static inline void simple_xattr_limits_init(struct simple_xattr_limits *limits)
+{
+	atomic_set(&limits->nr_xattrs, 0);
+	atomic_set(&limits->xattr_size, 0);
+}
+
+int simple_xattrs_init(struct simple_xattrs *xattrs);
+struct simple_xattrs *simple_xattrs_alloc(void);
+struct simple_xattrs *simple_xattrs_lazy_alloc(struct simple_xattrs **xattrsp,
+					       const void *value, int flags);
 void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space);
 size_t simple_xattr_space(const char *name, size_t size);
 struct simple_xattr *simple_xattr_alloc(const void *value, size_t size);
 void simple_xattr_free(struct simple_xattr *xattr);
+void simple_xattr_free_rcu(struct simple_xattr *xattr);
 int simple_xattr_get(struct simple_xattrs *xattrs, const char *name,
 		     void *buffer, size_t size);
 struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs,
 				      const char *name, const void *value,
 				      size_t size, int flags);
+int simple_xattr_set_limited(struct simple_xattrs *xattrs,
+			     struct simple_xattr_limits *limits,
+			     const char *name, const void *value,
+			     size_t size, int flags);
 ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
 			  char *buffer, size_t size);
-void simple_xattr_add(struct simple_xattrs *xattrs,
-		      struct simple_xattr *new_xattr);
+int simple_xattr_add(struct simple_xattrs *xattrs,
+		     struct simple_xattr *new_xattr);
 int xattr_list_one(char **buffer, ssize_t *remaining_size, const char *name);
 
+DEFINE_CLASS(simple_xattr,
+	     struct simple_xattr *,
+	     if (!IS_ERR_OR_NULL(_T)) simple_xattr_free(_T),
+	     simple_xattr_alloc(value, size),
+	     const void *value, size_t size)
+
+DEFINE_CLASS(simple_xattrs,
+            struct simple_xattrs *,
+            if (!IS_ERR_OR_NULL(_T)) { simple_xattrs_free(_T, NULL); kfree(_T); },
+            simple_xattrs_alloc(),
+            void)
+
 #endif	/* _LINUX_XATTR_H */
diff --git a/include/media/rc-core.h b/include/media/rc-core.h
index 35c7a0546f02e..d37fffc5dc3c2 100644
--- a/include/media/rc-core.h
+++ b/include/media/rc-core.h
@@ -81,7 +81,6 @@ struct lirc_fh {
 /**
  * struct rc_dev - represents a remote control device
  * @dev: driver model's view of this device
- * @managed_alloc: devm_rc_allocate_device was used to create rc_dev
  * @registered: set to true by rc_register_device(), false by
  *	rc_unregister_device
  * @idle: used to keep track of RX state
@@ -156,7 +155,6 @@ struct lirc_fh {
  */
 struct rc_dev {
 	struct device			dev;
-	bool				managed_alloc;
 	bool				registered;
 	bool				idle;
 	bool				encode_wakeup;
@@ -303,7 +301,7 @@ struct ir_raw_event {
 
 #define US_TO_NS(usec)		((usec) * 1000)
 #define MS_TO_US(msec)		((msec) * 1000)
-#define IR_MAX_DURATION		MS_TO_US(500)
+#define IR_MAX_DURATION		MS_TO_US(1000)
 #define IR_DEFAULT_TIMEOUT	MS_TO_US(125)
 #define IR_MAX_TIMEOUT		LIRC_VALUE_MASK
 
diff --git a/include/media/v4l2-subdev.h b/include/media/v4l2-subdev.h
index a37d9a847196b..e754ed3421c5c 100644
--- a/include/media/v4l2-subdev.h
+++ b/include/media/v4l2-subdev.h
@@ -1722,6 +1722,28 @@ int v4l2_subdev_disable_streams(struct v4l2_subdev *sd, u32 pad,
  */
 int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable);
 
+/**
+ * v4l2_subdev_get_frame_desc_passthrough() - Helper to implement the subdev
+ *	v4l2_get_frame_desc operation in simple passthrough cases
+ * @sd: The subdevice
+ * @pad: The source pad index
+ * @fd: The mbus frame desc
+ *
+ * Subdevice drivers that only pass through the streams can use this helper
+ * to implement the &v4l2_subdev_pad_ops.v4l2_get_frame_desc operation.
+ *
+ * The helper will call get_frame_desc on the subdevice's sources, create a new
+ * frame desc which contains only the streams on the given source pad. The data
+ * for each frame desc entry is copied directly from the data provided from the
+ * calls to the subdevice's sources, with the exception of the 'stream' field
+ * which is set according to the subdevice's routing table.
+ *
+ * Return: 0 on success, or a negative error code otherwise.
+ */
+int v4l2_subdev_get_frame_desc_passthrough(struct v4l2_subdev *sd,
+					   unsigned int pad,
+					   struct v4l2_mbus_frame_desc *fd);
+
 #endif /* CONFIG_VIDEO_V4L2_SUBDEV_API */
 
 #endif /* CONFIG_MEDIA_CONTROLLER */
diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h
index 533d8e75f7bbc..4e40063adab47 100644
--- a/include/net/af_vsock.h
+++ b/include/net/af_vsock.h
@@ -179,6 +179,15 @@ struct vsock_transport {
 	/* Addressing. */
 	u32 (*get_local_cid)(void);
 
+	/* Check if this transport serves a specific remote CID.
+	 * For H2G transports: return true if the CID belongs to a registered
+	 * guest. If not implemented, all CIDs > VMADDR_CID_HOST go to H2G.
+	 * For G2H transports: return true if the transport can reach arbitrary
+	 * CIDs via the hypervisor (i.e. supports the fallback overlay). VMCI
+	 * does not implement this as it only serves CIDs 0 and 2.
+	 */
+	bool (*has_remote_cid)(struct vsock_sock *vsk, u32 remote_cid);
+
 	/* Read a single skb */
 	int (*read_skb)(struct vsock_sock *, skb_read_actor_t);
 
diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h
index 010f1a8fd15f8..5172afee54943 100644
--- a/include/net/bluetooth/l2cap.h
+++ b/include/net/bluetooth/l2cap.h
@@ -658,6 +658,7 @@ struct l2cap_conn {
 	struct sk_buff		*rx_skb;
 	__u32			rx_len;
 	struct ida		tx_ida;
+	__u8			tx_ident;
 
 	struct sk_buff_head	pending_rx;
 	struct work_struct	pending_rx_work;
diff --git a/include/net/bonding.h b/include/net/bonding.h
index 395c6e281c5fc..d3520ecfa7f0a 100644
--- a/include/net/bonding.h
+++ b/include/net/bonding.h
@@ -91,22 +91,22 @@
 			    NETIF_F_GSO_ESP)
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
-extern atomic_t netpoll_block_tx;
+DECLARE_STATIC_KEY_FALSE(netpoll_block_tx);
 
 static inline void block_netpoll_tx(void)
 {
-	atomic_inc(&netpoll_block_tx);
+	static_branch_inc(&netpoll_block_tx);
 }
 
 static inline void unblock_netpoll_tx(void)
 {
-	atomic_dec(&netpoll_block_tx);
+	static_branch_dec(&netpoll_block_tx);
 }
 
 static inline int is_netpoll_tx_blocked(struct net_device *dev)
 {
-	if (unlikely(netpoll_tx_running(dev)))
-		return atomic_read(&netpoll_block_tx);
+	if (static_branch_unlikely(&netpoll_block_tx))
+		return netpoll_tx_running(dev);
 	return 0;
 }
 #else
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index fc01de19c7981..8cd870ece3519 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -190,6 +190,8 @@ enum ieee80211_channel_flags {
  *	on this channel.
  * @dfs_state_entered: timestamp (jiffies) when the dfs state was entered.
  * @dfs_cac_ms: DFS CAC time in milliseconds, this is valid for DFS channels.
+ * @cac_start_time: timestamp (CLOCK_BOOTTIME, nanoseconds) when CAC was
+ *	started on this channel. Zero when CAC is not in progress.
  * @psd: power spectral density (in dBm)
  */
 struct ieee80211_channel {
@@ -207,6 +209,7 @@ struct ieee80211_channel {
 	enum nl80211_dfs_state dfs_state;
 	unsigned long dfs_state_entered;
 	unsigned int dfs_cac_ms;
+	u64 cac_start_time;
 	s8 psd;
 };
 
@@ -4020,7 +4023,6 @@ struct cfg80211_nan_band_config {
  *	(i.e. BIT(NL80211_BAND_2GHZ)).
  * @cluster_id: cluster ID used for NAN synchronization. This is a MAC address
  *	that can take a value from 50-6F-9A-01-00-00 to 50-6F-9A-01-FF-FF.
- *	If NULL, the device will pick a random Cluster ID.
  * @scan_period: period (in seconds) between NAN scans.
  * @scan_dwell_time: dwell time (in milliseconds) for NAN scans.
  * @discovery_beacon_interval: interval (in TUs) for discovery beacons.
@@ -4036,7 +4038,7 @@ struct cfg80211_nan_band_config {
 struct cfg80211_nan_conf {
 	u8 master_pref;
 	u8 bands;
-	const u8 *cluster_id;
+	u8 cluster_id[ETH_ALEN] __aligned(2);
 	u16 scan_period;
 	u16 scan_dwell_time;
 	u8 discovery_beacon_interval;
@@ -4922,24 +4924,24 @@ struct cfg80211_ops {
 				 struct wireless_dev *wdev,
 				 unsigned int link_id);
 
-	int	(*add_key)(struct wiphy *wiphy, struct net_device *netdev,
+	int	(*add_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   int link_id, u8 key_index, bool pairwise,
 			   const u8 *mac_addr, struct key_params *params);
-	int	(*get_key)(struct wiphy *wiphy, struct net_device *netdev,
+	int	(*get_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   int link_id, u8 key_index, bool pairwise,
 			   const u8 *mac_addr, void *cookie,
 			   void (*callback)(void *cookie, struct key_params*));
-	int	(*del_key)(struct wiphy *wiphy, struct net_device *netdev,
+	int	(*del_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			   int link_id, u8 key_index, bool pairwise,
 			   const u8 *mac_addr);
 	int	(*set_default_key)(struct wiphy *wiphy,
 				   struct net_device *netdev, int link_id,
 				   u8 key_index, bool unicast, bool multicast);
 	int	(*set_default_mgmt_key)(struct wiphy *wiphy,
-					struct net_device *netdev, int link_id,
+					struct wireless_dev *wdev, int link_id,
 					u8 key_index);
 	int	(*set_default_beacon_key)(struct wiphy *wiphy,
-					  struct net_device *netdev,
+					  struct wireless_dev *wdev,
 					  int link_id,
 					  u8 key_index);
 
@@ -4951,17 +4953,17 @@ struct cfg80211_ops {
 			   unsigned int link_id);
 
 
-	int	(*add_station)(struct wiphy *wiphy, struct net_device *dev,
+	int	(*add_station)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       const u8 *mac,
 			       struct station_parameters *params);
-	int	(*del_station)(struct wiphy *wiphy, struct net_device *dev,
+	int	(*del_station)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       struct station_del_parameters *params);
-	int	(*change_station)(struct wiphy *wiphy, struct net_device *dev,
+	int	(*change_station)(struct wiphy *wiphy, struct wireless_dev *wdev,
 				  const u8 *mac,
 				  struct station_parameters *params);
-	int	(*get_station)(struct wiphy *wiphy, struct net_device *dev,
+	int	(*get_station)(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       const u8 *mac, struct station_info *sinfo);
-	int	(*dump_station)(struct wiphy *wiphy, struct net_device *dev,
+	int	(*dump_station)(struct wiphy *wiphy, struct wireless_dev *wdev,
 				int idx, u8 *mac, struct station_info *sinfo);
 
 	int	(*add_mpath)(struct wiphy *wiphy, struct net_device *dev,
@@ -8962,35 +8964,35 @@ static inline void cfg80211_sinfo_release_content(struct station_info *sinfo)
 /**
  * cfg80211_new_sta - notify userspace about station
  *
- * @dev: the netdev
+ * @wdev: the wireless device
  * @mac_addr: the station's address
  * @sinfo: the station information
  * @gfp: allocation flags
  */
-void cfg80211_new_sta(struct net_device *dev, const u8 *mac_addr,
+void cfg80211_new_sta(struct wireless_dev *wdev, const u8 *mac_addr,
 		      struct station_info *sinfo, gfp_t gfp);
 
 /**
  * cfg80211_del_sta_sinfo - notify userspace about deletion of a station
- * @dev: the netdev
+ * @wdev: the wireless device
  * @mac_addr: the station's address. For MLD station, MLD address is used.
  * @sinfo: the station information/statistics
  * @gfp: allocation flags
  */
-void cfg80211_del_sta_sinfo(struct net_device *dev, const u8 *mac_addr,
+void cfg80211_del_sta_sinfo(struct wireless_dev *wdev, const u8 *mac_addr,
 			    struct station_info *sinfo, gfp_t gfp);
 
 /**
  * cfg80211_del_sta - notify userspace about deletion of a station
  *
- * @dev: the netdev
+ * @wdev: the wireless device
  * @mac_addr: the station's address. For MLD station, MLD address is used.
  * @gfp: allocation flags
  */
-static inline void cfg80211_del_sta(struct net_device *dev,
+static inline void cfg80211_del_sta(struct wireless_dev *wdev,
 				    const u8 *mac_addr, gfp_t gfp)
 {
-	cfg80211_del_sta_sinfo(dev, mac_addr, NULL, gfp);
+	cfg80211_del_sta_sinfo(wdev, mac_addr, NULL, gfp);
 }
 
 /**
@@ -10472,4 +10474,27 @@ cfg80211_s1g_get_primary_sibling(struct wiphy *wiphy,
 	return ieee80211_get_channel_khz(wiphy, sibling_1mhz_khz);
 }
 
+
+/**
+ * cfg80211_incumbent_signal_notify - Notify userspace of incumbent signal detection
+ * @wiphy: the wiphy to use
+ * @chandef: channel definition in which the interference was detected
+ * @signal_interference_bitmap: bitmap indicating interference across 20 MHz segments
+ * @gfp: allocation context for message creation and multicast; pass GFP_ATOMIC
+ *	if called from atomic context (e.g. firmware event handler), otherwise
+ *	GFP_KERNEL
+ *
+ * Use this function to notify userspace when an incumbent signal is detected on
+ * the operating channel in the 6 GHz band. The notification includes the
+ * current channel definition and a bitmap representing interference across
+ * the operating bandwidth. Each bit in the bitmap corresponds to a 20 MHz
+ * segment, with the lowest bit representing the lowest frequency segment.
+ * Punctured sub-channels are included in the bitmap structure but are always
+ * set to zero since interference detection is not performed on them.
+ */
+void cfg80211_incumbent_signal_notify(struct wiphy *wiphy,
+				      const struct cfg80211_chan_def *chandef,
+				      u32 signal_interference_bitmap,
+				      gfp_t gfp);
+
 #endif /* __NET_CFG80211_H */
diff --git a/include/net/devlink.h b/include/net/devlink.h
index cb839e0435a17..3038af6ec017e 100644
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -1611,6 +1611,9 @@ struct devlink_ops {
 void *devlink_priv(struct devlink *devlink);
 struct devlink *priv_to_devlink(void *priv);
 struct device *devlink_to_dev(const struct devlink *devlink);
+const char *devlink_bus_name(const struct devlink *devlink);
+const char *devlink_dev_name(const struct devlink *devlink);
+const char *devlink_dev_driver_name(const struct devlink *devlink);
 
 /* Devlink instance explicit locking */
 void devl_lock(struct devlink *devlink);
@@ -1644,6 +1647,13 @@ void devlink_register(struct devlink *devlink);
 void devlink_unregister(struct devlink *devlink);
 void devlink_free(struct devlink *devlink);
 
+struct devlink *devlink_shd_get(const char *id,
+				const struct devlink_ops *ops,
+				size_t priv_size,
+				const struct device_driver *driver);
+void devlink_shd_put(struct devlink *devlink);
+void *devlink_shd_get_priv(struct devlink *devlink);
+
 /**
  * struct devlink_port_ops - Port operations
  * @port_split: Callback used to split the port into multiple ones.
diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h
index a7b7abd66e215..de61dd5dbfd9d 100644
--- a/include/net/dropreason-core.h
+++ b/include/net/dropreason-core.h
@@ -68,12 +68,6 @@
 	FN(SECURITY_HOOK)		\
 	FN(QDISC_DROP)			\
 	FN(QDISC_BURST_DROP)		\
-	FN(QDISC_OVERLIMIT)		\
-	FN(QDISC_CONGESTED)		\
-	FN(CAKE_FLOOD)			\
-	FN(FQ_BAND_LIMIT)		\
-	FN(FQ_HORIZON_LIMIT)		\
-	FN(FQ_FLOW_LIMIT)		\
 	FN(CPU_BACKLOG)			\
 	FN(XDP)				\
 	FN(TC_INGRESS)			\
@@ -127,9 +121,9 @@
 	FN(CANFD_RX_INVALID_FRAME)	\
 	FN(CANXL_RX_INVALID_FRAME)	\
 	FN(PFMEMALLOC)	\
-	FN(DUALPI2_STEP_DROP)		\
 	FN(PSP_INPUT)			\
 	FN(PSP_OUTPUT)			\
+	FN(RECURSION_LIMIT)		\
 	FNe(MAX)
 
 /**
@@ -371,8 +365,10 @@ enum skb_drop_reason {
 	/** @SKB_DROP_REASON_SECURITY_HOOK: dropped due to security HOOK */
 	SKB_DROP_REASON_SECURITY_HOOK,
 	/**
-	 * @SKB_DROP_REASON_QDISC_DROP: dropped by qdisc when packet outputting (
-	 * failed to enqueue to current qdisc)
+	 * @SKB_DROP_REASON_QDISC_DROP: dropped by qdisc during enqueue or
+	 * dequeue. More specific drop reasons are available via the
+	 * qdisc:qdisc_drop tracepoint, which also provides qdisc handle
+	 * and name for identifying the source.
 	 */
 	SKB_DROP_REASON_QDISC_DROP,
 	/**
@@ -380,36 +376,6 @@ enum skb_drop_reason {
 	 * limit is hit.
 	 */
 	SKB_DROP_REASON_QDISC_BURST_DROP,
-	/**
-	 * @SKB_DROP_REASON_QDISC_OVERLIMIT: dropped by qdisc when a qdisc
-	 * instance exceeds its total buffer size limit.
-	 */
-	SKB_DROP_REASON_QDISC_OVERLIMIT,
-	/**
-	 * @SKB_DROP_REASON_QDISC_CONGESTED: dropped by a qdisc AQM algorithm
-	 * due to congestion.
-	 */
-	SKB_DROP_REASON_QDISC_CONGESTED,
-	/**
-	 * @SKB_DROP_REASON_CAKE_FLOOD: dropped by the flood protection part of
-	 * CAKE qdisc AQM algorithm (BLUE).
-	 */
-	SKB_DROP_REASON_CAKE_FLOOD,
-	/**
-	 * @SKB_DROP_REASON_FQ_BAND_LIMIT: dropped by fq qdisc when per band
-	 * limit is reached.
-	 */
-	SKB_DROP_REASON_FQ_BAND_LIMIT,
-	/**
-	 * @SKB_DROP_REASON_FQ_HORIZON_LIMIT: dropped by fq qdisc when packet
-	 * timestamp is too far in the future.
-	 */
-	SKB_DROP_REASON_FQ_HORIZON_LIMIT,
-	/**
-	 * @SKB_DROP_REASON_FQ_FLOW_LIMIT: dropped by fq qdisc when a flow
-	 * exceeds its limits.
-	 */
-	SKB_DROP_REASON_FQ_FLOW_LIMIT,
 	/**
 	 * @SKB_DROP_REASON_CPU_BACKLOG: failed to enqueue the skb to the per CPU
 	 * backlog queue. This can be caused by backlog queue full (see
@@ -613,15 +579,12 @@ enum skb_drop_reason {
 	 * reached a path or socket not eligible for use of memory reserves
 	 */
 	SKB_DROP_REASON_PFMEMALLOC,
-	/**
-	 * @SKB_DROP_REASON_DUALPI2_STEP_DROP: dropped by the step drop
-	 * threshold of DualPI2 qdisc.
-	 */
-	SKB_DROP_REASON_DUALPI2_STEP_DROP,
 	/** @SKB_DROP_REASON_PSP_INPUT: PSP input checks failed */
 	SKB_DROP_REASON_PSP_INPUT,
 	/** @SKB_DROP_REASON_PSP_OUTPUT: PSP output checks failed */
 	SKB_DROP_REASON_PSP_OUTPUT,
+	/** @SKB_DROP_REASON_RECURSION_LIMIT: Dead loop on virtual device. */
+	SKB_DROP_REASON_RECURSION_LIMIT,
 	/**
 	 * @SKB_DROP_REASON_MAX: the maximum of core drop reasons, which
 	 * shouldn't be used as a real 'reason' - only for tracing code gen
diff --git a/include/net/dropreason-qdisc.h b/include/net/dropreason-qdisc.h
new file mode 100644
index 0000000000000..fb151cd31751b
--- /dev/null
+++ b/include/net/dropreason-qdisc.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef _LINUX_DROPREASON_QDISC_H
+#define _LINUX_DROPREASON_QDISC_H
+#include <net/dropreason.h>
+
+#define DEFINE_QDISC_DROP_REASON(FN, FNe)	\
+	FN(UNSPEC)			\
+	FN(GENERIC)			\
+	FN(OVERLIMIT)			\
+	FN(CONGESTED)			\
+	FN(MAXFLOWS)			\
+	FN(FLOOD_PROTECTION)		\
+	FN(BAND_LIMIT)		\
+	FN(HORIZON_LIMIT)		\
+	FN(FLOW_LIMIT)			\
+	FN(L4S_STEP_NON_ECN)		\
+	FNe(MAX)
+
+#undef FN
+#undef FNe
+#define FN(reason)	QDISC_DROP_##reason,
+#define FNe(reason)	QDISC_DROP_##reason
+
+/**
+ * enum qdisc_drop_reason - reason why a qdisc dropped a packet
+ *
+ * Qdisc-specific drop reasons for packet drops that occur within the
+ * traffic control (TC) queueing discipline layer. These reasons provide
+ * detailed diagnostics about why packets were dropped by various qdisc
+ * algorithms, enabling fine-grained monitoring and troubleshooting of
+ * queue behavior.
+ */
+enum qdisc_drop_reason {
+	/**
+	 * @QDISC_DROP_UNSPEC: unspecified/invalid qdisc drop reason.
+	 * Value 0 serves as analogous to SKB_NOT_DROPPED_YET for enum skb_drop_reason.
+	 * Used for catching zero-initialized drop_reason fields.
+	 */
+	QDISC_DROP_UNSPEC = 0,
+	/**
+	 * @__QDISC_DROP_REASON: subsystem base value for qdisc drop reasons
+	 */
+	__QDISC_DROP_REASON = SKB_DROP_REASON_SUBSYS_QDISC <<
+				SKB_DROP_REASON_SUBSYS_SHIFT,
+	/**
+	 * @QDISC_DROP_GENERIC: generic/default qdisc drop, used when no
+	 * more specific reason applies
+	 */
+	QDISC_DROP_GENERIC,
+	/**
+	 * @QDISC_DROP_OVERLIMIT: packet dropped because the qdisc queue
+	 * length exceeded its configured limit (sch->limit). This typically
+	 * indicates the queue is full and cannot accept more packets.
+	 */
+	QDISC_DROP_OVERLIMIT,
+	/**
+	 * @QDISC_DROP_CONGESTED: packet dropped due to active congestion
+	 * control algorithms (e.g., CoDel, PIE, RED) detecting network
+	 * congestion. The qdisc proactively dropped the packet to signal
+	 * congestion to the sender and prevent bufferbloat.
+	 */
+	QDISC_DROP_CONGESTED,
+	/**
+	 * @QDISC_DROP_MAXFLOWS: packet dropped because the qdisc's flow
+	 * tracking table is full and no free slots are available to allocate
+	 * for a new flow. This indicates flow table exhaustion in flow-based
+	 * qdiscs that maintain per-flow state (e.g., SFQ).
+	 */
+	QDISC_DROP_MAXFLOWS,
+	/**
+	 * @QDISC_DROP_FLOOD_PROTECTION: packet dropped by flood protection
+	 * mechanism detecting unresponsive flows (potential DoS/flood).
+	 * Used by qdiscs implementing probabilistic drop algorithms like
+	 * BLUE (e.g., CAKE's Cobalt AQM).
+	 */
+	QDISC_DROP_FLOOD_PROTECTION,
+	/**
+	 * @QDISC_DROP_BAND_LIMIT: packet dropped because the priority band's
+	 * limit was reached. Used by qdiscs with priority bands that have
+	 * per-band packet limits (e.g., FQ).
+	 */
+	QDISC_DROP_BAND_LIMIT,
+	/**
+	 * @QDISC_DROP_HORIZON_LIMIT: packet dropped because its timestamp
+	 * is too far in the future (beyond the configured horizon).
+	 * Used by qdiscs with time-based scheduling (e.g., FQ).
+	 */
+	QDISC_DROP_HORIZON_LIMIT,
+	/**
+	 * @QDISC_DROP_FLOW_LIMIT: packet dropped because an individual flow
+	 * exceeded its per-flow packet/depth limit. Used by FQ and SFQ qdiscs
+	 * to enforce per-flow fairness and prevent a single flow from
+	 * monopolizing queue resources.
+	 */
+	QDISC_DROP_FLOW_LIMIT,
+	/**
+	 * @QDISC_DROP_L4S_STEP_NON_ECN: DualPI2 qdisc dropped a non-ECN-capable
+	 * packet because the L4S queue delay exceeded the step threshold.
+	 * Since the packet cannot be ECN-marked, it must be dropped to signal
+	 * congestion. See RFC 9332 for the DualQ Coupled AQM step mechanism.
+	 */
+	QDISC_DROP_L4S_STEP_NON_ECN,
+	/**
+	 * @QDISC_DROP_MAX: the maximum of qdisc drop reasons, which
+	 * shouldn't be used as a real 'reason' - only for tracing code gen
+	 */
+	QDISC_DROP_MAX,
+};
+
+#undef FN
+#undef FNe
+
+#endif
diff --git a/include/net/dropreason.h b/include/net/dropreason.h
index 7d3b1a2a6feca..1df60645fb278 100644
--- a/include/net/dropreason.h
+++ b/include/net/dropreason.h
@@ -23,6 +23,12 @@ enum skb_drop_reason_subsys {
 	 */
 	SKB_DROP_REASON_SUBSYS_OPENVSWITCH,
 
+	/**
+	 * @SKB_DROP_REASON_SUBSYS_QDISC: TC qdisc drop reasons,
+	 * see include/net/dropreason-qdisc.h
+	 */
+	SKB_DROP_REASON_SUBSYS_QDISC,
+
 	/** @SKB_DROP_REASON_SUBSYS_NUM: number of subsystems defined */
 	SKB_DROP_REASON_SUBSYS_NUM
 };
diff --git a/include/net/dst.h b/include/net/dst.h
index 307073eae7f83..10b73fb45117c 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -95,8 +95,19 @@ struct dst_entry {
 #ifdef CONFIG_64BIT
 	struct lwtunnel_state   *lwtstate;
 #endif
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	atomic_t dst_trace_seq;
+#endif
 };
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+void save_dst_trace_buffer(struct dst_entry *dst, int delta);
+void dump_dst_trace_buffer(const struct net_device *dev);
+#else
+static inline void save_dst_trace_buffer(struct dst_entry *dst, int delta) { };
+static inline void dump_dst_trace_buffer(const struct net_device *dev) { };
+#endif
+
 struct dst_metrics {
 	u32		metrics[RTAX_MAX];
 	refcount_t	refcnt;
@@ -244,7 +255,10 @@ static inline void dst_hold(struct dst_entry *dst)
 	 * the placement of __rcuref in struct dst_entry
 	 */
 	BUILD_BUG_ON(offsetof(struct dst_entry, __rcuref) & 63);
-	WARN_ON(!rcuref_get(&dst->__rcuref));
+	if (!rcuref_get(&dst->__rcuref))
+		WARN_ON(1);
+	else
+		save_dst_trace_buffer(dst, 1);
 }
 
 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
@@ -308,7 +322,11 @@ static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb
  */
 static inline bool dst_hold_safe(struct dst_entry *dst)
 {
-	return rcuref_get(&dst->__rcuref);
+	const bool ret = rcuref_get(&dst->__rcuref);
+
+	if (ret)
+		save_dst_trace_buffer(dst, 1);
+	return ret;
 }
 
 /**
diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index 6632b1aa75848..62534d1f3c707 100644
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -6,6 +6,9 @@
 #include <linux/types.h>
 #include <linux/netdevice.h>
 #include <net/protocol.h>
+#ifdef CONFIG_RPS
+#include <net/rps-types.h>
+#endif
 
 struct skb_defer_node {
 	struct llist_head	defer_list;
@@ -33,7 +36,7 @@ struct net_hotdata {
 	struct kmem_cache	*skbuff_fclone_cache;
 	struct kmem_cache	*skb_small_head_cache;
 #ifdef CONFIG_RPS
-	struct rps_sock_flow_table __rcu *rps_sock_flow_table;
+	rps_tag_ptr		rps_sock_flow_table;
 	u32			rps_cpu_mask;
 #endif
 	struct skb_defer_node __percpu *skb_defer_nodes;
diff --git a/include/net/inet6_connection_sock.h b/include/net/inet6_connection_sock.h
index ece8dabd209a5..b814e1acc5121 100644
--- a/include/net/inet6_connection_sock.h
+++ b/include/net/inet6_connection_sock.h
@@ -18,6 +18,9 @@ struct sk_buff;
 struct sock;
 struct sockaddr;
 
+struct dst_entry *inet6_csk_route_socket(struct sock *sk,
+					 struct flowi6 *fl6);
+
 struct dst_entry *inet6_csk_route_req(const struct sock *sk,
 				      struct dst_entry *dst,
 				      struct flowi6 *fl6,
@@ -25,5 +28,4 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk,
 
 int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
 
-struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu);
 #endif /* _INET6_CONNECTION_SOCK_H */
diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h
index c16de5b7963fd..2cc5d416bbb5a 100644
--- a/include/net/inet6_hashtables.h
+++ b/include/net/inet6_hashtables.h
@@ -24,6 +24,8 @@
 
 struct inet_hashinfo;
 
+void inet6_init_ehash_secret(void);
+
 static inline unsigned int __inet6_ehashfn(const u32 lhash,
 				    const u16 lport,
 				    const u32 fhash,
diff --git a/include/net/inet_common.h b/include/net/inet_common.h
index 5dd2bf24449ef..3d747896be300 100644
--- a/include/net/inet_common.h
+++ b/include/net/inet_common.h
@@ -59,8 +59,7 @@ int inet_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);
 int inet_ctl_sock_create(struct sock **sk, unsigned short family,
 			 unsigned short type, unsigned char protocol,
 			 struct net *net);
-int inet_recv_error(struct sock *sk, struct msghdr *msg, int len,
-		    int *addr_len);
+int inet_recv_error(struct sock *sk, struct msghdr *msg, int len);
 
 struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb);
 int inet_gro_complete(struct sk_buff *skb, int nhoff);
diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index 5cb3056d6ddc7..433c2df23076d 100644
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -34,7 +34,7 @@ struct tcp_congestion_ops;
  */
 struct inet_connection_sock_af_ops {
 	int	    (*queue_xmit)(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
-	void	    (*send_check)(struct sock *sk, struct sk_buff *skb);
+	u16	    net_header_len;
 	int	    (*rebuild_header)(struct sock *sk);
 	void	    (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb);
 	int	    (*conn_request)(struct sock *sk, struct sk_buff *skb);
@@ -45,7 +45,6 @@ struct inet_connection_sock_af_ops {
 				      bool *own_req,
 				      void (*opt_child_init)(struct sock *newsk,
 							     const struct sock *sk));
-	u16	    net_header_len;
 	int	    (*setsockopt)(struct sock *sk, int level, int optname,
 				  sockptr_t optval, unsigned int optlen);
 	int	    (*getsockopt)(struct sock *sk, int level, int optname,
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index 5a979dcab5383..1a47e7ed371a9 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -286,7 +286,6 @@ void inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,
 			 unsigned long numentries, int scale,
 			 unsigned long low_limit,
 			 unsigned long high_limit);
-int inet_hashinfo2_init_mod(struct inet_hashinfo *h);
 
 bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk);
 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk,
diff --git a/include/net/ip.h b/include/net/ip.h
index 7f9abd457e018..f39a3787fedd5 100644
--- a/include/net/ip.h
+++ b/include/net/ip.h
@@ -812,7 +812,7 @@ int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
 int ip_ra_control(struct sock *sk, unsigned char on,
 		  void (*destructor)(struct sock *));
 
-int ip_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len);
+int ip_recv_error(struct sock *sk, struct msghdr *msg, int len);
 void ip_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port,
 		   u32 info, u8 *payload);
 void ip_local_error(struct sock *sk, int err, __be32 daddr, __be16 dport,
diff --git a/include/net/ip6_checksum.h b/include/net/ip6_checksum.h
index c8a96b8882772..6677b3cc39726 100644
--- a/include/net/ip6_checksum.h
+++ b/include/net/ip6_checksum.h
@@ -82,6 +82,4 @@ static inline __sum16 udp_v6_check(int len,
 void udp6_set_csum(bool nocheck, struct sk_buff *skb,
 		   const struct in6_addr *saddr,
 		   const struct in6_addr *daddr, int len);
-
-int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto);
 #endif
diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h
index a55f9bf95fe39..0c8eeb6abe7ac 100644
--- a/include/net/ip6_route.h
+++ b/include/net/ip6_route.h
@@ -252,15 +252,22 @@ static inline bool ipv6_unicast_destination(const struct sk_buff *skb)
 	return rt->rt6i_flags & RTF_LOCAL;
 }
 
+static inline bool __ipv6_anycast_destination(const struct rt6key *rt6i_dst,
+					      u32 rt6i_flags,
+					      const struct in6_addr *daddr)
+{
+	return rt6i_flags & RTF_ANYCAST ||
+	       (rt6i_dst->plen < 127 &&
+	       !(rt6i_flags & (RTF_GATEWAY | RTF_NONEXTHOP)) &&
+	       ipv6_addr_equal(&rt6i_dst->addr, daddr));
+}
+
 static inline bool ipv6_anycast_destination(const struct dst_entry *dst,
 					    const struct in6_addr *daddr)
 {
 	const struct rt6_info *rt = dst_rt6_info(dst);
 
-	return rt->rt6i_flags & RTF_ANYCAST ||
-		(rt->rt6i_dst.plen < 127 &&
-		 !(rt->rt6i_flags & (RTF_GATEWAY | RTF_NONEXTHOP)) &&
-		 ipv6_addr_equal(&rt->rt6i_dst.addr, daddr));
+	return __ipv6_anycast_destination(&rt->rt6i_dst, rt->rt6i_flags, daddr);
 }
 
 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
index 359b595f1df93..b99805ee2fd14 100644
--- a/include/net/ip6_tunnel.h
+++ b/include/net/ip6_tunnel.h
@@ -162,7 +162,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
 					     dev->name);
 			DEV_STATS_INC(dev, tx_errors);
 		}
-		kfree_skb(skb);
+		kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT);
 		return;
 	}
 
diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
index 80662f8120803..1f577a4f8ce9b 100644
--- a/include/net/ip_tunnels.h
+++ b/include/net/ip_tunnels.h
@@ -665,13 +665,29 @@ static inline int iptunnel_pull_offloads(struct sk_buff *skb)
 static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len)
 {
 	if (pkt_len > 0) {
-		struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats);
-
-		u64_stats_update_begin(&tstats->syncp);
-		u64_stats_add(&tstats->tx_bytes, pkt_len);
-		u64_stats_inc(&tstats->tx_packets);
-		u64_stats_update_end(&tstats->syncp);
-		put_cpu_ptr(tstats);
+		if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) {
+			struct pcpu_dstats *dstats = get_cpu_ptr(dev->dstats);
+
+			u64_stats_update_begin(&dstats->syncp);
+			u64_stats_add(&dstats->tx_bytes, pkt_len);
+			u64_stats_inc(&dstats->tx_packets);
+			u64_stats_update_end(&dstats->syncp);
+			put_cpu_ptr(dstats);
+			return;
+		}
+		if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
+			struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats);
+
+			u64_stats_update_begin(&tstats->syncp);
+			u64_stats_add(&tstats->tx_bytes, pkt_len);
+			u64_stats_inc(&tstats->tx_packets);
+			u64_stats_update_end(&tstats->syncp);
+			put_cpu_ptr(tstats);
+			return;
+		}
+		pr_err_once("iptunnel_xmit_stats pcpu_stat_type=%d\n",
+			    dev->pcpu_stat_type);
+		WARN_ON_ONCE(1);
 		return;
 	}
 
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h
index 29a36709e7f35..72d325c813132 100644
--- a/include/net/ip_vs.h
+++ b/include/net/ip_vs.h
@@ -11,6 +11,7 @@
 #include <asm/types.h>                  /* for __uXX types */
 
 #include <linux/list.h>                 /* for struct list_head */
+#include <linux/rculist_bl.h>           /* for struct hlist_bl_head */
 #include <linux/spinlock.h>             /* for struct rwlock_t */
 #include <linux/atomic.h>               /* for struct atomic_t */
 #include <linux/refcount.h>             /* for struct refcount_t */
@@ -30,10 +31,23 @@
 #endif
 #include <net/net_namespace.h>		/* Netw namespace */
 #include <linux/sched/isolation.h>
+#include <linux/siphash.h>
 
 #define IP_VS_HDR_INVERSE	1
 #define IP_VS_HDR_ICMP		2
 
+/* conn_tab limits (as per Kconfig) */
+#define IP_VS_CONN_TAB_MIN_BITS	8
+#if BITS_PER_LONG > 32
+#define IP_VS_CONN_TAB_MAX_BITS	27
+#else
+#define IP_VS_CONN_TAB_MAX_BITS	20
+#endif
+
+/* svc_table limits */
+#define IP_VS_SVC_TAB_MIN_BITS	4
+#define IP_VS_SVC_TAB_MAX_BITS	20
+
 /* Generic access of ipvs struct */
 static inline struct netns_ipvs *net_ipvs(struct net* net)
 {
@@ -43,8 +57,6 @@ static inline struct netns_ipvs *net_ipvs(struct net* net)
 /* Connections' size value needed by ip_vs_ctl.c */
 extern int ip_vs_conn_tab_size;
 
-extern struct mutex __ip_vs_mutex;
-
 struct ip_vs_iphdr {
 	int hdr_flags;	/* ipvs flags */
 	__u32 off;	/* Where IP or IPv4 header starts */
@@ -265,6 +277,29 @@ static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len,
 			pr_err(msg, ##__VA_ARGS__);			\
 	} while (0)
 
+struct ip_vs_aligned_lock {
+	spinlock_t	l;	/* Protect buckets */
+} ____cacheline_aligned_in_smp;
+
+/* For arrays per family */
+enum {
+	IP_VS_AF_INET,
+	IP_VS_AF_INET6,
+	IP_VS_AF_MAX
+};
+
+static inline int ip_vs_af_index(int af)
+{
+	return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET;
+}
+
+/* work_flags */
+enum {
+	IP_VS_WORK_SVC_RESIZE,		/* Schedule svc_resize_work */
+	IP_VS_WORK_SVC_NORESIZE,	/* Stopping svc_resize_work */
+	IP_VS_WORK_CONN_RESIZE,		/* Schedule conn_resize_work */
+};
+
 /* The port number of FTP service (in network order). */
 #define FTPPORT  cpu_to_be16(21)
 #define FTPDATA  cpu_to_be16(20)
@@ -466,6 +501,198 @@ struct ip_vs_est_kt_data {
 	int			est_row;	/* estimated row */
 };
 
+/* IPVS resizable hash tables */
+struct ip_vs_rht {
+	struct hlist_bl_head		*buckets;
+	struct ip_vs_rht __rcu		*new_tbl; /* New/Same table	*/
+	seqcount_t			*seqc;	/* Protects moves	*/
+	struct ip_vs_aligned_lock	*lock;	/* Protect seqc		*/
+	int				mask;	/* Buckets mask		*/
+	int				size;	/* Buckets		*/
+	int				seqc_mask; /* seqc mask		*/
+	int				lock_mask; /* lock mask		*/
+	u32				table_id;
+	int				u_thresh; /* upper threshold	*/
+	int				l_thresh; /* lower threshold	*/
+	int				lfactor;  /* Load Factor (shift)*/
+	int				bits;	/* size = 1 << bits	*/
+	siphash_key_t			hash_key;
+	struct rcu_head			rcu_head;
+};
+
+/**
+ * ip_vs_rht_for_each_table() - Walk the hash tables
+ * @table:	struct ip_vs_rht __rcu *table
+ * @t:		current table, used as cursor, struct ip_vs_rht *var
+ * @p:		previous table, temp struct ip_vs_rht *var
+ *
+ * Walk tables assuming others can not change the installed tables
+ */
+#define ip_vs_rht_for_each_table(table, t, p)				\
+	for (p = NULL, t = rcu_dereference_protected(table, 1);		\
+	     t != p;							\
+	     p = t, t = rcu_dereference_protected(t->new_tbl, 1))
+
+/**
+ * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock
+ * @table:	struct ip_vs_rht __rcu *table
+ * @t:		current table, used as cursor, struct ip_vs_rht *var
+ * @p:		previous table, temp struct ip_vs_rht *var
+ *
+ * We usually search in one table and also in second table on resizing
+ */
+#define ip_vs_rht_for_each_table_rcu(table, t, p)			\
+	for (p = NULL, t = rcu_dereference(table);			\
+	     t != p;							\
+	     p = t, t = rcu_dereference(t->new_tbl))
+
+/**
+ * ip_vs_rht_for_each_bucket() - Walk all table buckets
+ * @t:		current table, used as cursor, struct ip_vs_rht *var
+ * @bucket:	bucket index, used as cursor, u32 var
+ * @head:	bucket address, used as cursor, struct hlist_bl_head *var
+ */
+#define ip_vs_rht_for_each_bucket(t, bucket, head)			\
+	for (bucket = 0, head = (t)->buckets;				\
+	     bucket < t->size; bucket++, head++)
+
+/**
+ * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved
+ * @t:		current table, used as cursor, struct ip_vs_rht *var
+ * @bucket:	index of current bucket or hash key
+ * @sc:		temp seqcount_t *var
+ * @seq:	temp unsigned int var for sequence count
+ * @retry:	temp int var
+ */
+#define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry)		\
+	for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask];	\
+	     retry && ({ seq = read_seqcount_begin(sc); 1; });		\
+	     retry = read_seqcount_retry(sc, seq))
+
+/**
+ * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables
+ *
+ * Variables for ip_vs_rht_walk_buckets_rcu
+ */
+#define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU()				\
+	struct ip_vs_rht *_t, *_p;					\
+	unsigned int _seq;						\
+	seqcount_t *_sc;						\
+	u32 _bucket;							\
+	int _retry
+/**
+ * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock
+ * @table:	struct ip_vs_rht __rcu *table
+ * @head:	bucket address, used as cursor, struct hlist_bl_head *var
+ *
+ * Can be used while others add/delete/move entries
+ * Not suitable if duplicates are not desired
+ * Possible cases for reader that uses cond_resched_rcu() in the loop:
+ * - new table can not be installed, no need to repeat
+ * - new table can be installed => check and repeat if new table is
+ * installed, needed for !PREEMPT_RCU
+ */
+#define ip_vs_rht_walk_buckets_rcu(table, head)				\
+	ip_vs_rht_for_each_table_rcu(table, _t, _p)			\
+		ip_vs_rht_for_each_bucket(_t, _bucket, head)		\
+			ip_vs_rht_for_bucket_retry(_t, _bucket, _sc,	\
+						   _seq, _retry)
+
+/**
+ * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables
+ *
+ * Variables for ip_vs_rht_walk_bucket_rcu
+ */
+#define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU()				\
+	unsigned int _seq;						\
+	seqcount_t *_sc;						\
+	int _retry
+/**
+ * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock
+ * @t:		current table, struct ip_vs_rht *var
+ * @bucket:	index of current bucket or hash key
+ * @head:	bucket address, used as cursor, struct hlist_bl_head *var
+ *
+ * Can be used while others add/delete/move entries
+ * Not suitable if duplicates are not desired
+ * Possible cases for reader that uses cond_resched_rcu() in the loop:
+ * - new table can not be installed, no need to repeat
+ * - new table can be installed => check and repeat if new table is
+ * installed, needed for !PREEMPT_RCU
+ */
+#define ip_vs_rht_walk_bucket_rcu(t, bucket, head)			\
+	if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; }))	\
+		{}							\
+	else								\
+		ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry)
+
+/**
+ * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables
+ *
+ * Variables for ip_vs_rht_walk_buckets_safe_rcu
+ */
+#define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU()			\
+	struct ip_vs_rht *_t, *_p;					\
+	u32 _bucket
+/**
+ * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock
+ * @table:	struct ip_vs_rht __rcu *table
+ * @head:	bucket address, used as cursor, struct hlist_bl_head *var
+ *
+ * Can be used while others add/delete entries but moving is disabled
+ * Using cond_resched_rcu() should be safe if tables do not change
+ */
+#define ip_vs_rht_walk_buckets_safe_rcu(table, head)			\
+	ip_vs_rht_for_each_table_rcu(table, _t, _p)			\
+		ip_vs_rht_for_each_bucket(_t, _bucket, head)
+
+/**
+ * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables
+ *
+ * Variables for ip_vs_rht_walk_buckets
+ */
+#define DECLARE_IP_VS_RHT_WALK_BUCKETS()				\
+	struct ip_vs_rht *_t, *_p;					\
+	u32 _bucket
+
+/**
+ * ip_vs_rht_walk_buckets() - Walk all buckets
+ * @table:	struct ip_vs_rht __rcu *table
+ * @head:	bucket address, used as cursor, struct hlist_bl_head *var
+ *
+ * Use if others can not add/delete/move entries
+ */
+#define ip_vs_rht_walk_buckets(table, head)				\
+	ip_vs_rht_for_each_table(table, _t, _p)				\
+		ip_vs_rht_for_each_bucket(_t, _bucket, head)
+
+/* Entries can be in one of two tables, so we flip bit when new table is
+ * created and store it as highest bit in hash keys
+ */
+#define IP_VS_RHT_TABLE_ID_MASK	BIT(31)
+
+/* Check if hash key is from this table */
+static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key)
+{
+	return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK);
+}
+
+/* Build per-table hash key from hash value */
+static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash)
+{
+	return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK);
+}
+
+void ip_vs_rht_free(struct ip_vs_rht *t);
+void ip_vs_rht_rcu_free(struct rcu_head *head);
+struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks);
+int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n,
+			   int lfactor, int min_bits, int max_bits);
+void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor,
+			      int min_bits, int max_bits);
+u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af,
+			 const union nf_inet_addr *addr, u32 v1, u32 v2);
+
 struct dst_entry;
 struct iphdr;
 struct ip_vs_conn;
@@ -559,50 +786,48 @@ struct ip_vs_conn_param {
 	__u8				pe_data_len;
 };
 
+/* Hash node in conn_tab */
+struct ip_vs_conn_hnode {
+	struct hlist_bl_node	node;		/* node in conn_tab */
+	u32			hash_key;	/* Key for the hash table */
+	u8			dir;		/* 0=out->in, 1=in->out */
+} __packed;
+
 /* IP_VS structure allocated for each dynamically scheduled connection */
 struct ip_vs_conn {
-	struct hlist_node	c_list;         /* hashed list heads */
-	/* Protocol, addresses and port numbers */
+	/* Cacheline for hash table nodes - rarely modified */
+
+	struct ip_vs_conn_hnode	hn0;		/* Original direction */
+	u8			af;		/* address family */
 	__be16                  cport;
+	struct ip_vs_conn_hnode	hn1;		/* Reply direction */
+	u8			daf;		/* Address family of the dest */
 	__be16                  dport;
-	__be16                  vport;
-	u16			af;		/* address family */
-	union nf_inet_addr      caddr;          /* client address */
-	union nf_inet_addr      vaddr;          /* virtual address */
-	union nf_inet_addr      daddr;          /* destination address */
+	struct ip_vs_dest       *dest;          /* real server */
+	atomic_t                n_control;      /* Number of controlled ones */
 	volatile __u32          flags;          /* status flags */
-	__u16                   protocol;       /* Which protocol (TCP/UDP) */
-	__u16			daf;		/* Address family of the dest */
-	struct netns_ipvs	*ipvs;
-
-	/* counter and timer */
-	refcount_t		refcnt;		/* reference count */
-	struct timer_list	timer;		/* Expiration timer */
-	volatile unsigned long	timeout;	/* timeout */
+	/* 44/64 */
 
-	/* Flags and state transition */
-	spinlock_t              lock;           /* lock for state transition */
+	struct ip_vs_conn       *control;       /* Master control connection */
+	const struct ip_vs_pe	*pe;
+	char			*pe_data;
+	__u8			pe_data_len;
 	volatile __u16          state;          /* state info */
 	volatile __u16          old_state;      /* old state, to be used for
 						 * state transition triggered
 						 * synchronization
 						 */
-	__u32			fwmark;		/* Fire wall mark from skb */
-	unsigned long		sync_endtime;	/* jiffies + sent_retries */
+	/* 2-byte hole */
+	/* 64/96 */
 
-	/* Control members */
-	struct ip_vs_conn       *control;       /* Master control connection */
-	atomic_t                n_control;      /* Number of controlled ones */
-	struct ip_vs_dest       *dest;          /* real server */
-	atomic_t                in_pkts;        /* incoming packet counter */
+	union nf_inet_addr      caddr;          /* client address */
+	union nf_inet_addr      vaddr;          /* virtual address */
+	/* 96/128 */
 
-	/* Packet transmitter for different forwarding methods.  If it
-	 * mangles the packet, it must return NF_DROP or better NF_STOLEN,
-	 * otherwise this must be changed to a sk_buff **.
-	 * NF_ACCEPT can be returned when destination is local.
-	 */
-	int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp,
-			   struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
+	union nf_inet_addr      daddr;          /* destination address */
+	__u32			fwmark;		/* Fire wall mark from skb */
+	__be16                  vport;
+	__u16                   protocol;       /* Which protocol (TCP/UDP) */
 
 	/* Note: we can group the following members into a structure,
 	 * in order to save more space, and the following members are
@@ -610,14 +835,31 @@ struct ip_vs_conn {
 	 */
 	struct ip_vs_app        *app;           /* bound ip_vs_app object */
 	void                    *app_data;      /* Application private data */
+	/* 128/168 */
 	struct_group(sync_conn_opt,
 		struct ip_vs_seq  in_seq;       /* incoming seq. struct */
 		struct ip_vs_seq  out_seq;      /* outgoing seq. struct */
 	);
+	/* 152/192 */
 
-	const struct ip_vs_pe	*pe;
-	char			*pe_data;
-	__u8			pe_data_len;
+	struct timer_list	timer;		/* Expiration timer */
+	volatile unsigned long	timeout;	/* timeout */
+	spinlock_t              lock;           /* lock for state transition */
+	refcount_t		refcnt;		/* reference count */
+	atomic_t                in_pkts;        /* incoming packet counter */
+	/* 64-bit: 4-byte gap */
+
+	/* 188/256 */
+	unsigned long		sync_endtime;	/* jiffies + sent_retries */
+	struct netns_ipvs	*ipvs;
+
+	/* Packet transmitter for different forwarding methods.  If it
+	 * mangles the packet, it must return NF_DROP or better NF_STOLEN,
+	 * otherwise this must be changed to a sk_buff **.
+	 * NF_ACCEPT can be returned when destination is local.
+	 */
+	int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp,
+			   struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
 
 	struct rcu_head		rcu_head;
 };
@@ -673,15 +915,15 @@ struct ip_vs_dest_user_kern {
  * forwarding entries.
  */
 struct ip_vs_service {
-	struct hlist_node	s_list;   /* for normal service table */
-	struct hlist_node	f_list;   /* for fwmark-based service table */
-	atomic_t		refcnt;   /* reference counter */
-
+	struct hlist_bl_node	s_list;   /* node in service table */
+	u32			hash_key; /* Key for the hash table */
 	u16			af;       /* address family */
 	__u16			protocol; /* which protocol (TCP/UDP) */
+
 	union nf_inet_addr	addr;	  /* IP address for virtual service */
-	__be16			port;	  /* port number for the service */
 	__u32                   fwmark;   /* firewall mark of the service */
+	atomic_t		refcnt;   /* reference counter */
+	__be16			port;	  /* port number for the service */
 	unsigned int		flags;	  /* service status flags */
 	unsigned int		timeout;  /* persistent timeout in ticks */
 	__be32			netmask;  /* grouping granularity, mask/plen */
@@ -791,8 +1033,8 @@ struct ip_vs_pe {
 	int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb);
 	bool (*ct_match)(const struct ip_vs_conn_param *p,
 			 struct ip_vs_conn *ct);
-	u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, u32 initval,
-			   bool inverse);
+	u32 (*hashkey_raw)(const struct ip_vs_conn_param *p,
+			   struct ip_vs_rht *t, bool inverse);
 	int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf);
 	/* create connections for real-server outgoing packets */
 	struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc,
@@ -931,21 +1173,27 @@ struct netns_ipvs {
 #endif
 	/* ip_vs_conn */
 	atomic_t		conn_count;      /* connection counter */
+	atomic_t		no_cport_conns[IP_VS_AF_MAX];
+	struct delayed_work	conn_resize_work;/* resize conn_tab */
 
 	/* ip_vs_ctl */
 	struct ip_vs_stats_rcu	*tot_stats;      /* Statistics & est. */
 
-	int			num_services;    /* no of virtual services */
-	int			num_services6;   /* IPv6 virtual services */
-
 	/* Trash for destinations */
 	struct list_head	dest_trash;
 	spinlock_t		dest_trash_lock;
 	struct timer_list	dest_trash_timer; /* expiration timer */
+	struct mutex		service_mutex;    /* service reconfig */
+	struct rw_semaphore	svc_resize_sem;   /* svc_table resizing */
+	struct delayed_work	svc_resize_work;  /* resize svc_table */
+	atomic_t		svc_table_changes;/* ++ on new table */
 	/* Service counters */
-	atomic_t		ftpsvc_counter;
-	atomic_t		nullsvc_counter;
-	atomic_t		conn_out_counter;
+	atomic_t		num_services[IP_VS_AF_MAX];   /* Services */
+	atomic_t		fwm_services[IP_VS_AF_MAX];   /* Services */
+	atomic_t		nonfwm_services[IP_VS_AF_MAX];/* Services */
+	atomic_t		ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */
+	atomic_t		nullsvc_counter[IP_VS_AF_MAX];/* Zero port */
+	atomic_t		conn_out_counter[IP_VS_AF_MAX];/* out conn */
 
 #ifdef CONFIG_SYSCTL
 	/* delayed work for expiring no dest connections */
@@ -956,6 +1204,7 @@ struct netns_ipvs {
 	int			drop_counter;
 	int			old_secure_tcp;
 	atomic_t		dropentry;
+	s8			dropentry_counters[8];
 	/* locks in ctl.c */
 	spinlock_t		dropentry_lock;  /* drop entry handling */
 	spinlock_t		droppacket_lock; /* drop packet handling */
@@ -1002,6 +1251,8 @@ struct netns_ipvs {
 	int			sysctl_est_nice;	/* kthread nice */
 	int			est_stopped;		/* stop tasks */
 #endif
+	int			sysctl_conn_lfactor;
+	int			sysctl_svc_lfactor;
 
 	/* ip_vs_lblc */
 	int			sysctl_lblc_expiration;
@@ -1011,6 +1262,7 @@ struct netns_ipvs {
 	int			sysctl_lblcr_expiration;
 	struct ctl_table_header	*lblcr_ctl_header;
 	struct ctl_table	*lblcr_ctl_table;
+	unsigned long		work_flags;	/* IP_VS_WORK_* flags */
 	/* ip_vs_est */
 	struct delayed_work	est_reload_work;/* Reload kthread tasks */
 	struct mutex		est_mutex;	/* protect kthread tasks */
@@ -1041,6 +1293,10 @@ struct netns_ipvs {
 	 */
 	unsigned int		mixed_address_family_dests;
 	unsigned int		hooks_afmask;	/* &1=AF_INET, &2=AF_INET6 */
+
+	struct ip_vs_rht __rcu	*svc_table;	/* Services */
+	struct ip_vs_rht __rcu	*conn_tab;	/* Connections */
+	atomic_t		conn_tab_changes;/* ++ on new table */
 };
 
 #define DEFAULT_SYNC_THRESHOLD	3
@@ -1290,6 +1546,24 @@ static inline int sysctl_est_nice(struct netns_ipvs *ipvs)
 
 #endif
 
+/* Get load factor to map conn_count/u_thresh to t->size */
+static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs)
+{
+	return READ_ONCE(ipvs->sysctl_conn_lfactor);
+}
+
+/* Get load factor to map num_services/u_thresh to t->size
+ * Smaller value decreases u_thresh to reduce collisions but increases
+ * the table size
+ * Returns factor where:
+ * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load
+ * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load
+ */
+static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs)
+{
+	return READ_ONCE(ipvs->sysctl_svc_lfactor);
+}
+
 /* IPVS core functions
  * (from ip_vs_core.c)
  */
@@ -1363,6 +1637,23 @@ static inline void __ip_vs_conn_put(struct ip_vs_conn *cp)
 }
 void ip_vs_conn_put(struct ip_vs_conn *cp);
 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport);
+int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t,
+			    int lfactor);
+struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets,
+				       int lfactor);
+
+static inline struct ip_vs_conn *
+ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn)
+{
+	return container_of(hn, struct ip_vs_conn, hn0);
+}
+
+static inline struct ip_vs_conn *
+ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn)
+{
+	return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) :
+			 container_of(hn, struct ip_vs_conn, hn0);
+}
 
 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af,
 				  const union nf_inet_addr *daddr,
@@ -1716,6 +2007,13 @@ static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp)
 	return fwd;
 }
 
+/* Check if connection uses double hashing */
+static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp)
+{
+	return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ &&
+	       !(cp->flags & IP_VS_CONN_F_TEMPLATE);
+}
+
 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
 		    struct ip_vs_conn *cp, int dir);
 
diff --git a/include/net/ipv6.h b/include/net/ipv6.h
index 53c5056508be5..0958cc5c6ec38 100644
--- a/include/net/ipv6.h
+++ b/include/net/ipv6.h
@@ -1129,10 +1129,8 @@ int ip6_datagram_connect_v6_only(struct sock *sk, struct sockaddr_unsized *addr,
 int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr);
 void ip6_datagram_release_cb(struct sock *sk);
 
-int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len,
-		    int *addr_len);
-int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len,
-		     int *addr_len);
+int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len);
+int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len);
 void ipv6_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port,
 		     u32 info, u8 *payload);
 void ipv6_local_error(struct sock *sk, int err, struct flowi6 *fl6, u32 info);
@@ -1181,8 +1179,6 @@ int tcp6_proc_init(struct net *net);
 void tcp6_proc_exit(struct net *net);
 int udp6_proc_init(struct net *net);
 void udp6_proc_exit(struct net *net);
-int udplite6_proc_init(void);
-void udplite6_proc_exit(void);
 int ipv6_misc_proc_init(void);
 void ipv6_misc_proc_exit(void);
 int snmp6_register_dev(struct inet6_dev *idev);
diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h
index d3013e721b144..907681cecde8e 100644
--- a/include/net/ipv6_stubs.h
+++ b/include/net/ipv6_stubs.h
@@ -83,10 +83,9 @@ struct ipv6_bpf_stub {
 	int (*inet6_bind)(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len,
 			  u32 flags);
 	struct sock *(*udp6_lib_lookup)(const struct net *net,
-				     const struct in6_addr *saddr, __be16 sport,
-				     const struct in6_addr *daddr, __be16 dport,
-				     int dif, int sdif, struct udp_table *tbl,
-				     struct sk_buff *skb);
+					const struct in6_addr *saddr, __be16 sport,
+					const struct in6_addr *daddr, __be16 dport,
+					int dif, int sdif, struct sk_buff *skb);
 	int (*ipv6_setsockopt)(struct sock *sk, int level, int optname,
 			       sockptr_t optval, unsigned int optlen);
 	int (*ipv6_getsockopt)(struct sock *sk, int level, int optname,
diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index 7f9d96939a4ea..72395895dc0e2 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -2913,6 +2913,9 @@ struct ieee80211_txq {
  *	HW flag so drivers can opt in according to their own control, e.g. in
  *	testing.
  *
+ * @IEEE80211_HW_SUPPORTS_NDP_BLOCKACK: HW can transmit/receive S1G NDP
+ *	BlockAck frames.
+ *
  * @NUM_IEEE80211_HW_FLAGS: number of hardware flags, used for sizing arrays
  */
 enum ieee80211_hw_flags {
@@ -2973,6 +2976,7 @@ enum ieee80211_hw_flags {
 	IEEE80211_HW_DISALLOW_PUNCTURING,
 	IEEE80211_HW_HANDLES_QUIET_CSA,
 	IEEE80211_HW_STRICT,
+	IEEE80211_HW_SUPPORTS_NDP_BLOCKACK,
 
 	/* keep last, obviously */
 	NUM_IEEE80211_HW_FLAGS
@@ -7407,7 +7411,9 @@ void ieee80211_report_wowlan_wakeup(struct ieee80211_vif *vif,
  * @band: the band to transmit on
  * @sta: optional pointer to get the station to send the frame to
  *
- * Return: %true if the skb was prepared, %false otherwise
+ * Return: %true if the skb was prepared, %false otherwise.
+ * On failure, the skb is freed by this function; callers must not
+ * free it again.
  *
  * Note: must be called under RCU lock
  */
@@ -7766,19 +7772,22 @@ u32 ieee80211_calc_tx_airtime(struct ieee80211_hw *hw,
  * ieee80211_get_fils_discovery_tmpl - Get FILS discovery template.
  * @hw: pointer obtained from ieee80211_alloc_hw().
  * @vif: &struct ieee80211_vif pointer from the add_interface callback.
+ * @link_id: valid link_id during MLO or 0 for non-MLO.
  *
  * The driver is responsible for freeing the returned skb.
  *
  * Return: FILS discovery template. %NULL on error.
  */
 struct sk_buff *ieee80211_get_fils_discovery_tmpl(struct ieee80211_hw *hw,
-						  struct ieee80211_vif *vif);
+						  struct ieee80211_vif *vif,
+						  unsigned int link_id);
 
 /**
  * ieee80211_get_unsol_bcast_probe_resp_tmpl - Get unsolicited broadcast
  *	probe response template.
  * @hw: pointer obtained from ieee80211_alloc_hw().
  * @vif: &struct ieee80211_vif pointer from the add_interface callback.
+ * @link_id: valid link_id during MLO or 0 for non-MLO.
  *
  * The driver is responsible for freeing the returned skb.
  *
@@ -7786,7 +7795,8 @@ struct sk_buff *ieee80211_get_fils_discovery_tmpl(struct ieee80211_hw *hw,
  */
 struct sk_buff *
 ieee80211_get_unsol_bcast_probe_resp_tmpl(struct ieee80211_hw *hw,
-					  struct ieee80211_vif *vif);
+					  struct ieee80211_vif *vif,
+					  unsigned int link_id);
 
 /**
  * ieee80211_obss_color_collision_notify - notify userland about a BSS color
@@ -7962,4 +7972,11 @@ int ieee80211_emulate_switch_vif_chanctx(struct ieee80211_hw *hw,
  * Return: %true iff the vif is a NAN interface and NAN is started
  */
 bool ieee80211_vif_nan_started(struct ieee80211_vif *vif);
+
+/**
+ * ieee80211_encrypt_tx_skb - Encrypt the transmit skb
+ * @skb: the skb
+ * Return: 0 if success and non-zero on error
+ */
+int ieee80211_encrypt_tx_skb(struct sk_buff *skb);
 #endif /* MAC80211_H */
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 766f4fb25e266..7fe3a1b61b2df 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -215,6 +215,12 @@ enum gdma_page_type {
 
 #define GDMA_INVALID_DMA_REGION 0
 
+struct mana_serv_work {
+	struct work_struct serv_work;
+	struct pci_dev *pdev;
+	enum gdma_eqe_type type;
+};
+
 struct gdma_mem_info {
 	struct device *dev;
 
@@ -386,6 +392,7 @@ struct gdma_irq_context {
 
 enum gdma_context_flags {
 	GC_PROBE_SUCCEEDED	= 0,
+	GC_IN_SERVICE		= 1,
 };
 
 struct gdma_context {
@@ -411,14 +418,15 @@ struct gdma_context {
 	u32			test_event_eq_id;
 
 	bool			is_pf;
-	bool			in_service;
 
 	phys_addr_t		bar0_pa;
 	void __iomem		*bar0_va;
+	resource_size_t		bar0_size;
 	void __iomem		*shm_base;
 	void __iomem		*db_page_base;
 	phys_addr_t		phys_db_page_base;
-	u32 db_page_size;
+	u64 db_page_off;
+	u64 db_page_size;
 	int                     numa_node;
 
 	/* Shared memory chanenl (used to bootstrap HWC) */
@@ -473,6 +481,8 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
 
 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
 
+int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type);
+
 struct gdma_wqe {
 	u32 reserved	:24;
 	u32 last_vbytes	:8;
@@ -615,6 +625,9 @@ enum {
 /* Driver can handle hardware recovery events during probe */
 #define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22)
 
+/* Driver supports self recovery on Hardware Channel timeouts */
+#define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -628,7 +641,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \
 	 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \
 	 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \
-	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY)
+	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
+	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index a078af283bddb..3336688fed5e9 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -61,8 +61,11 @@ enum TRI_STATE {
 
 #define MAX_PORTS_IN_MANA_DEV 256
 
+/* Maximum number of packets per coalesced CQE */
+#define MANA_RXCOMP_OOB_NUM_PPI 4
+
 /* Update this count whenever the respective structures are changed */
-#define MANA_STATS_RX_COUNT 5
+#define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
 #define MANA_STATS_TX_COUNT 11
 
 #define MANA_RX_FRAG_ALIGNMENT 64
@@ -73,6 +76,8 @@ struct mana_stats_rx {
 	u64 xdp_drop;
 	u64 xdp_tx;
 	u64 xdp_redirect;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	struct u64_stats_sync syncp;
 };
 
@@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
 	u32 pkt_hash;
 }; /* HW DATA */
 
-#define MANA_RXCOMP_OOB_NUM_PPI 4
-
 /* Receive completion OOB */
 struct mana_rxcomp_oob {
 	struct mana_cqe_header cqe_hdr;
@@ -378,7 +381,6 @@ struct mana_ethtool_stats {
 	u64 tx_cqe_err;
 	u64 tx_cqe_unknown_type;
 	u64 tx_linear_pkt_cnt;
-	u64 rx_coalesced_err;
 	u64 rx_cqe_unknown_type;
 };
 
@@ -557,6 +559,9 @@ struct mana_port_context {
 	bool port_is_up;
 	bool port_st_save; /* Saved port state */
 
+	u8 cqe_coalescing_enable;
+	u32 cqe_coalescing_timeout_ns;
+
 	struct mana_ethtool_stats eth_stats;
 
 	struct mana_ethtool_phy_stats phy_stats;
@@ -902,6 +907,10 @@ struct mana_cfg_rx_steer_req_v2 {
 
 struct mana_cfg_rx_steer_resp {
 	struct gdma_resp_hdr hdr;
+
+	/* V2 */
+	u32 cqe_coalescing_timeout_ns;
+	u32 reserved1;
 }; /* HW DATA */
 
 /* Register HW vPort */
diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h
index cfa72c4853876..08f81329fc11d 100644
--- a/include/net/netdev_rx_queue.h
+++ b/include/net/netdev_rx_queue.h
@@ -8,13 +8,14 @@
 #include <net/xdp.h>
 #include <net/page_pool/types.h>
 #include <net/netdev_queues.h>
+#include <net/rps-types.h>
 
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
 	struct xdp_rxq_info		xdp_rxq;
 #ifdef CONFIG_RPS
 	struct rps_map __rcu		*rps_map;
-	struct rps_dev_flow_table __rcu	*rps_flow_table;
+	rps_tag_ptr			rps_flow_table;
 #endif
 	struct kobject			kobj;
 	const struct attribute_group	**groups;
diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index e2d2bfc1f9893..66cedcfa338e6 100644
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -122,17 +122,6 @@ struct nft_regs {
 	};
 };
 
-struct nft_regs_track {
-	struct {
-		const struct nft_expr		*selector;
-		const struct nft_expr		*bitwise;
-		u8				num_reg;
-	} regs[NFT_REG32_NUM];
-
-	const struct nft_expr			*cur;
-	const struct nft_expr			*last;
-};
-
 /* Store/load an u8, u16 or u64 integer to/from the u32 data register.
  *
  * Note, when using concatenations, register allocation happens at 32-bit
@@ -277,8 +266,6 @@ struct nft_userdata {
 	unsigned char		data[];
 };
 
-#define NFT_SET_ELEM_INTERNAL_LAST	0x1
-
 /* placeholder structure for opaque set element backend representation. */
 struct nft_elem_priv { };
 
@@ -288,7 +275,6 @@ struct nft_elem_priv { };
  *	@key: element key
  *	@key_end: closing element key
  *	@data: element data
- * 	@flags: flags
  *	@priv: element private data and extensions
  */
 struct nft_set_elem {
@@ -304,7 +290,6 @@ struct nft_set_elem {
 		u32		buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
 		struct nft_data val;
 	} data;
-	u32			flags;
 	struct nft_elem_priv	*priv;
 };
 
@@ -429,8 +414,6 @@ int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp);
 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr);
 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
 		  const struct nft_expr *expr, bool reset);
-bool nft_expr_reduce_bitwise(struct nft_regs_track *track,
-			     const struct nft_expr *expr);
 
 struct nft_set_ext;
 
@@ -878,6 +861,8 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
 					u64 timeout, u64 expiration, gfp_t gfp);
 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
 			    struct nft_expr *expr_array[]);
+void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
+			       struct nft_set_elem_expr *elem_expr);
 void nft_set_elem_destroy(const struct nft_set *set,
 			  const struct nft_elem_priv *elem_priv,
 			  bool destroy_expr);
@@ -943,7 +928,6 @@ struct nft_offload_ctx;
  *	@destroy_clone: destruction clone function
  *	@dump: function to dump parameters
  *	@validate: validate expression, called during loop detection
- *	@reduce: reduce expression
  *	@gc: garbage collection expression
  *	@offload: hardware offload expression
  *	@offload_action: function to report true/false to allocate one slot or not in the flow
@@ -977,8 +961,6 @@ struct nft_expr_ops {
 						bool reset);
 	int				(*validate)(const struct nft_ctx *ctx,
 						    const struct nft_expr *expr);
-	bool				(*reduce)(struct nft_regs_track *track,
-						  const struct nft_expr *expr);
 	bool				(*gc)(struct net *net,
 					      const struct nft_expr *expr);
 	int				(*offload)(struct nft_offload_ctx *ctx,
@@ -1961,20 +1943,4 @@ static inline u64 nft_net_tstamp(const struct net *net)
 	return nft_pernet(net)->tstamp;
 }
 
-#define __NFT_REDUCE_READONLY	1UL
-#define NFT_REDUCE_READONLY	(void *)__NFT_REDUCE_READONLY
-
-void nft_reg_track_update(struct nft_regs_track *track,
-			  const struct nft_expr *expr, u8 dreg, u8 len);
-void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len);
-void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg);
-
-static inline bool nft_reg_track_cmp(struct nft_regs_track *track,
-				     const struct nft_expr *expr, u8 dreg)
-{
-	return track->regs[dreg].selector &&
-	       track->regs[dreg].selector->ops == expr->ops &&
-	       track->regs[dreg].num_reg == 0;
-}
-
 #endif /* _NET_NF_TABLES_H */
diff --git a/include/net/netfilter/nft_fib.h b/include/net/netfilter/nft_fib.h
index 7370fba844efc..e0422456f27bc 100644
--- a/include/net/netfilter/nft_fib.h
+++ b/include/net/netfilter/nft_fib.h
@@ -66,6 +66,4 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs,
 void nft_fib_store_result(void *reg, const struct nft_fib *priv,
 			  const struct net_device *dev);
 
-bool nft_fib_reduce(struct nft_regs_track *track,
-		    const struct nft_expr *expr);
 #endif
diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h
index d602263590fed..f74e63290603d 100644
--- a/include/net/netfilter/nft_meta.h
+++ b/include/net/netfilter/nft_meta.h
@@ -43,9 +43,6 @@ void nft_meta_set_destroy(const struct nft_ctx *ctx,
 int nft_meta_set_validate(const struct nft_ctx *ctx,
 			  const struct nft_expr *expr);
 
-bool nft_meta_get_reduce(struct nft_regs_track *track,
-			 const struct nft_expr *expr);
-
 struct nft_inner_tun_ctx;
 void nft_meta_inner_eval(const struct nft_expr *expr,
 			 struct nft_regs *regs, const struct nft_pktinfo *pkt,
diff --git a/include/net/netmem.h b/include/net/netmem.h
index a96b3e5e5574c..507b74c9f52d7 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -93,27 +93,21 @@ enum net_iov_type {
  *		supported.
  */
 struct net_iov {
-	union {
-		struct netmem_desc desc;
-
-		/* XXX: The following part should be removed once all
-		 * the references to them are converted so as to be
-		 * accessed via netmem_desc e.g. niov->desc.pp instead
-		 * of niov->pp.
-		 */
-		struct {
-			unsigned long _flags;
-			unsigned long pp_magic;
-			struct page_pool *pp;
-			unsigned long _pp_mapping_pad;
-			unsigned long dma_addr;
-			atomic_long_t pp_ref_count;
-		};
-	};
-	struct net_iov_area *owner;
+	struct netmem_desc desc;
+	unsigned int page_type;
 	enum net_iov_type type;
+	struct net_iov_area *owner;
 };
 
+/* Make sure 'the offset of page_type in struct page == the offset of
+ * type in struct net_iov'.
+ */
+#define NET_IOV_ASSERT_OFFSET(pg, iov)			\
+	static_assert(offsetof(struct page, pg) ==	\
+		      offsetof(struct net_iov, iov))
+NET_IOV_ASSERT_OFFSET(page_type, page_type);
+#undef NET_IOV_ASSERT_OFFSET
+
 struct net_iov_area {
 	/* Array of net_iovs for this area. */
 	struct net_iov *niovs;
@@ -123,26 +117,6 @@ struct net_iov_area {
 	unsigned long base_virtual;
 };
 
-/* net_iov is union'ed with struct netmem_desc mirroring struct page, so
- * the page_pool can access these fields without worrying whether the
- * underlying fields are accessed via netmem_desc or directly via
- * net_iov, until all the references to them are converted so as to be
- * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp.
- *
- * The non-net stack fields of struct page are private to the mm stack
- * and must never be mirrored to net_iov.
- */
-#define NET_IOV_ASSERT_OFFSET(desc, iov)                    \
-	static_assert(offsetof(struct netmem_desc, desc) == \
-		      offsetof(struct net_iov, iov))
-NET_IOV_ASSERT_OFFSET(_flags, _flags);
-NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic);
-NET_IOV_ASSERT_OFFSET(pp, pp);
-NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
-NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr);
-NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
-#undef NET_IOV_ASSERT_OFFSET
-
 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov)
 {
 	return niov->owner;
@@ -256,7 +230,7 @@ static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
  */
 #define pp_page_to_nmdesc(p)						\
 ({									\
-	DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p));		\
+	DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p));				\
 	__pp_page_to_nmdesc(p);						\
 })
 
diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index 8e971c7bf1646..80ccd4dda8e0f 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -74,6 +74,7 @@ struct netns_ipv4 {
 
 	/* TXRX readonly hotpath cache lines */
 	__cacheline_group_begin(netns_ipv4_read_txrx);
+	u8		sysctl_tcp_shrink_window;
 	__cacheline_group_end(netns_ipv4_read_txrx);
 
 	/* RX readonly hotpath cache line */
@@ -122,7 +123,6 @@ struct netns_ipv4 {
 #endif
 	bool			fib_has_custom_local_routes;
 	bool			fib_offload_disabled;
-	u8			sysctl_tcp_shrink_window;
 #ifdef CONFIG_IP_ROUTE_CLASSID
 	atomic_t		fib_num_tclassid_users;
 #endif
@@ -166,6 +166,7 @@ struct netns_ipv4 {
 	u8 sysctl_ip_autobind_reuse;
 	/* Shall we try to damage output packets if routing dev changes? */
 	u8 sysctl_ip_dynaddr;
+	u32 sysctl_ip_local_port_step_width;
 #ifdef CONFIG_NET_L3_MASTER_DEV
 	u8 sysctl_raw_l3mdev_accept;
 #endif
@@ -279,6 +280,9 @@ struct netns_ipv4 {
 	struct list_head	mr_tables;
 	struct fib_rules_ops	*mr_rules_ops;
 #endif
+	struct fib_notifier_ops	*ipmr_notifier_ops;
+	atomic_t		ipmr_seq;
+	struct mutex		mfc_mutex;
 #endif
 #ifdef CONFIG_IP_ROUTE_MULTIPATH
 	struct sysctl_fib_multipath_hash_seed sysctl_fib_multipath_hash_seed;
@@ -290,9 +294,6 @@ struct netns_ipv4 {
 	struct fib_notifier_ops	*notifier_ops;
 	unsigned int	fib_seq;	/* writes protected by rtnl_mutex */
 
-	struct fib_notifier_ops	*ipmr_notifier_ops;
-	unsigned int	ipmr_seq;	/* protected by rtnl_mutex */
-
 	atomic_t	rt_genid;
 	siphash_key_t	ip_id_key;
 	struct hlist_head	*inet_addr_lst;
diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h
index 34bdb1308e8ff..499e4288170fc 100644
--- a/include/net/netns/ipv6.h
+++ b/include/net/netns/ipv6.h
@@ -118,7 +118,7 @@ struct netns_ipv6 {
 	struct seg6_pernet_data *seg6_data;
 	struct fib_notifier_ops	*notifier_ops;
 	struct fib_notifier_ops	*ip6mr_notifier_ops;
-	unsigned int ipmr_seq; /* protected by rtnl_mutex */
+	atomic_t		ipmr_seq;
 	struct {
 		struct hlist_head head;
 		spinlock_t	lock;
diff --git a/include/net/netns/mib.h b/include/net/netns/mib.h
index 7e373664b1e7b..dce05f8e6a336 100644
--- a/include/net/netns/mib.h
+++ b/include/net/netns/mib.h
@@ -28,11 +28,6 @@ struct netns_mib {
 	DEFINE_SNMP_STAT(struct mptcp_mib, mptcp_statistics);
 #endif
 
-	DEFINE_SNMP_STAT(struct udp_mib, udplite_statistics);
-#if IS_ENABLED(CONFIG_IPV6)
-	DEFINE_SNMP_STAT(struct udp_mib, udplite_stats_in6);
-#endif
-
 	DEFINE_SNMP_STAT(struct icmp_mib, icmp_statistics);
 	DEFINE_SNMP_STAT_ATOMIC(struct icmpmsg_mib, icmpmsg_statistics);
 #if IS_ENABLED(CONFIG_IPV6)
diff --git a/include/net/netns/vsock.h b/include/net/netns/vsock.h
index dc8cbe45f406e..7f84aad92f571 100644
--- a/include/net/netns/vsock.h
+++ b/include/net/netns/vsock.h
@@ -20,5 +20,7 @@ struct netns_vsock {
 
 	/* 0 = unlocked, 1 = locked to global, 2 = locked to local */
 	int child_ns_mode_locked;
+
+	int g2h_fallback;
 };
 #endif /* __NET_NET_NAMESPACE_VSOCK_H */
diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h
index 23dd647fe0248..b73983a17e088 100644
--- a/include/net/netns/xfrm.h
+++ b/include/net/netns/xfrm.h
@@ -59,7 +59,7 @@ struct netns_xfrm {
 	struct list_head	inexact_bins;
 
 
-	struct sock		*nlsk;
+	struct sock		__rcu *nlsk;
 	struct sock		*nlsk_stash;
 
 	u32			sysctl_aevent_etime;
diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
index cdd95477af7a2..03da138722f58 100644
--- a/include/net/page_pool/types.h
+++ b/include/net/page_pool/types.h
@@ -44,6 +44,8 @@
  * use-case.  The NAPI budget is 64 packets.  After a NAPI poll the RX
  * ring is usually refilled and the max consumed elements will be 64,
  * thus a natural max size of objects needed in the cache.
+ * The refill watermark is set to 64 for 4KB pages,
+ * and scales to balance its size in bytes across page sizes.
  *
  * Keeping room for more objects, is due to XDP_DROP use-case.  As
  * XDP_DROP allows the opportunity to recycle objects directly into
@@ -51,8 +53,15 @@
  * cache is already full (or partly full) then the XDP_DROP recycles
  * would have to take a slower code path.
  */
-#define PP_ALLOC_CACHE_SIZE	128
+#if PAGE_SIZE >= SZ_64K
+#define PP_ALLOC_CACHE_REFILL	4
+#elif PAGE_SIZE >= SZ_16K
+#define PP_ALLOC_CACHE_REFILL	16
+#else
 #define PP_ALLOC_CACHE_REFILL	64
+#endif
+
+#define PP_ALLOC_CACHE_SIZE	(PP_ALLOC_CACHE_REFILL * 2)
 struct pp_alloc_cache {
 	u32 count;
 	netmem_ref cache[PP_ALLOC_CACHE_SIZE];
diff --git a/include/net/ping.h b/include/net/ping.h
index 05bfd594a64c0..bcbdb5a136e3d 100644
--- a/include/net/ping.h
+++ b/include/net/ping.h
@@ -20,8 +20,7 @@
 
 /* Compatibility glue so we can support IPv6 when it's compiled as a module */
 struct pingv6_ops {
-	int (*ipv6_recv_error)(struct sock *sk, struct msghdr *msg, int len,
-			       int *addr_len);
+	int (*ipv6_recv_error)(struct sock *sk, struct msghdr *msg, int len);
 	void (*ip6_datagram_recv_common_ctl)(struct sock *sk,
 					     struct msghdr *msg,
 					     struct sk_buff *skb);
@@ -64,7 +63,7 @@ int  ping_getfrag(void *from, char *to, int offset, int fraglen, int odd,
 		  struct sk_buff *);
 
 int  ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-		  int flags, int *addr_len);
+		  int flags);
 int  ping_common_sendmsg(int family, struct msghdr *msg, size_t len,
 			 void *user_icmph, size_t icmph_len);
 int  ping_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
diff --git a/include/net/rps-types.h b/include/net/rps-types.h
new file mode 100644
index 0000000000000..6b90a66866c1f
--- /dev/null
+++ b/include/net/rps-types.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _NET_RPS_TYPES_H
+#define _NET_RPS_TYPES_H
+
+/* Define a rps_tag_ptr:
+ * Low order 5 bits are used to store the ilog2(size) of an RPS table.
+ */
+typedef unsigned long rps_tag_ptr;
+
+static inline u8 rps_tag_to_log(rps_tag_ptr tag_ptr)
+{
+	return tag_ptr & 31U;
+}
+
+static inline u32 rps_tag_to_mask(rps_tag_ptr tag_ptr)
+{
+	return (1U << rps_tag_to_log(tag_ptr)) - 1;
+}
+
+static inline void *rps_tag_to_table(rps_tag_ptr tag_ptr)
+{
+	return (void *)(tag_ptr & ~31UL);
+}
+#endif /* _NET_RPS_TYPES_H */
diff --git a/include/net/rps.h b/include/net/rps.h
index f1794cd2e7fb3..e33c6a2fa8bbc 100644
--- a/include/net/rps.h
+++ b/include/net/rps.h
@@ -8,6 +8,7 @@
 #include <net/hotdata.h>
 
 #ifdef CONFIG_RPS
+#include <net/rps-types.h>
 
 extern struct static_key_false rps_needed;
 extern struct static_key_false rfs_needed;
@@ -38,17 +39,6 @@ struct rps_dev_flow {
 };
 #define RPS_NO_FILTER 0xffff
 
-/*
- * The rps_dev_flow_table structure contains a table of flow mappings.
- */
-struct rps_dev_flow_table {
-	u8			log;
-	struct rcu_head		rcu;
-	struct rps_dev_flow	flows[];
-};
-#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
-    ((_num) * sizeof(struct rps_dev_flow)))
-
 /*
  * The rps_sock_flow_table contains mappings of flows to the last CPU
  * on which they were processed by the application (set in recvmsg).
@@ -60,41 +50,38 @@ struct rps_dev_flow_table {
  * meaning we use 32-6=26 bits for the hash.
  */
 struct rps_sock_flow_table {
-	struct rcu_head	rcu;
-	u32		mask;
-
-	u32		ents[] ____cacheline_aligned_in_smp;
+	u32	ent;
 };
-#define	RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
 
 #define RPS_NO_CPU 0xffff
 
-static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
-					u32 hash)
+static inline void rps_record_sock_flow(rps_tag_ptr tag_ptr, u32 hash)
 {
-	unsigned int index = hash & table->mask;
+	unsigned int index = hash & rps_tag_to_mask(tag_ptr);
 	u32 val = hash & ~net_hotdata.rps_cpu_mask;
+	struct rps_sock_flow_table *table;
 
 	/* We only give a hint, preemption can change CPU under us */
 	val |= raw_smp_processor_id();
 
+	table = rps_tag_to_table(tag_ptr);
 	/* The following WRITE_ONCE() is paired with the READ_ONCE()
 	 * here, and another one in get_rps_cpu().
 	 */
-	if (READ_ONCE(table->ents[index]) != val)
-		WRITE_ONCE(table->ents[index], val);
+	if (READ_ONCE(table[index].ent) != val)
+		WRITE_ONCE(table[index].ent, val);
 }
 
 static inline void _sock_rps_record_flow_hash(__u32 hash)
 {
-	struct rps_sock_flow_table *sock_flow_table;
+	rps_tag_ptr tag_ptr;
 
 	if (!hash)
 		return;
 	rcu_read_lock();
-	sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
-	if (sock_flow_table)
-		rps_record_sock_flow(sock_flow_table, hash);
+	tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
+	if (tag_ptr)
+		rps_record_sock_flow(tag_ptr, hash);
 	rcu_read_unlock();
 }
 
@@ -121,6 +108,7 @@ static inline void _sock_rps_record_flow(const struct sock *sk)
 static inline void _sock_rps_delete_flow(const struct sock *sk)
 {
 	struct rps_sock_flow_table *table;
+	rps_tag_ptr tag_ptr;
 	u32 hash, index;
 
 	hash = READ_ONCE(sk->sk_rxhash);
@@ -128,11 +116,12 @@ static inline void _sock_rps_delete_flow(const struct sock *sk)
 		return;
 
 	rcu_read_lock();
-	table = rcu_dereference(net_hotdata.rps_sock_flow_table);
-	if (table) {
-		index = hash & table->mask;
-		if (READ_ONCE(table->ents[index]) != RPS_NO_CPU)
-			WRITE_ONCE(table->ents[index], RPS_NO_CPU);
+	tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
+	if (tag_ptr) {
+		index = hash & rps_tag_to_mask(tag_ptr);
+		table = rps_tag_to_table(tag_ptr);
+		if (READ_ONCE(table[index].ent) != RPS_NO_CPU)
+			WRITE_ONCE(table[index].ent, RPS_NO_CPU);
 	}
 	rcu_read_unlock();
 }
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index d5d55cb21686d..5af262ec4bbd2 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -20,12 +20,15 @@
 #include <net/rtnetlink.h>
 #include <net/flow_offload.h>
 #include <linux/xarray.h>
+#include <net/dropreason-qdisc.h>
 
 struct Qdisc_ops;
 struct qdisc_walker;
 struct tcf_walker;
 struct module;
 struct bpf_flow_keys;
+struct Qdisc;
+struct netdev_queue;
 
 struct qdisc_rate_table {
 	struct tc_ratespec rate;
@@ -707,8 +710,8 @@ void dev_qdisc_change_real_num_tx(struct net_device *dev,
 void dev_init_scheduler(struct net_device *dev);
 void dev_shutdown(struct net_device *dev);
 void dev_activate(struct net_device *dev);
-void dev_deactivate(struct net_device *dev);
-void dev_deactivate_many(struct list_head *head);
+void dev_deactivate(struct net_device *dev, bool reset_needed);
+void dev_deactivate_many(struct list_head *head, bool reset_needed);
 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
 			      struct Qdisc *qdisc);
 void qdisc_reset(struct Qdisc *qdisc);
@@ -716,6 +719,34 @@ void qdisc_destroy(struct Qdisc *qdisc);
 void qdisc_put(struct Qdisc *qdisc);
 void qdisc_put_unlocked(struct Qdisc *qdisc);
 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
+
+static inline void dev_reset_queue(struct net_device *dev,
+				   struct netdev_queue *dev_queue,
+				   void *_unused)
+{
+	struct Qdisc *qdisc;
+	bool nolock;
+
+	qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
+	if (!qdisc)
+		return;
+
+	nolock = qdisc->flags & TCQ_F_NOLOCK;
+
+	if (nolock)
+		spin_lock_bh(&qdisc->seqlock);
+	spin_lock_bh(qdisc_lock(qdisc));
+
+	qdisc_reset(qdisc);
+
+	spin_unlock_bh(qdisc_lock(qdisc));
+	if (nolock) {
+		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
+		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
+		spin_unlock_bh(&qdisc->seqlock);
+	}
+}
+
 #ifdef CONFIG_NET_SCHED
 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
 			      void *type_data);
@@ -1116,36 +1147,50 @@ static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
 	return cb;
 }
 
+/* TC classifier accessors - use enum skb_drop_reason */
 static inline enum skb_drop_reason
 tcf_get_drop_reason(const struct sk_buff *skb)
 {
-	return tc_skb_cb(skb)->drop_reason;
+	return (enum skb_drop_reason)tc_skb_cb(skb)->drop_reason;
 }
 
 static inline void tcf_set_drop_reason(const struct sk_buff *skb,
 				       enum skb_drop_reason reason)
 {
-	tc_skb_cb(skb)->drop_reason = reason;
+	tc_skb_cb(skb)->drop_reason = (enum qdisc_drop_reason)reason;
 }
 
-static inline void tcf_kfree_skb_list(struct sk_buff *skb)
+/* Qdisc accessors - use enum qdisc_drop_reason */
+static inline enum qdisc_drop_reason
+tcf_get_qdisc_drop_reason(const struct sk_buff *skb)
 {
-	while (unlikely(skb)) {
-		struct sk_buff *next = skb->next;
+	return tc_skb_cb(skb)->drop_reason;
+}
 
-		prefetch(next);
-		kfree_skb_reason(skb, tcf_get_drop_reason(skb));
-		skb = next;
-	}
+static inline void tcf_set_qdisc_drop_reason(const struct sk_buff *skb,
+					     enum qdisc_drop_reason reason)
+{
+	tc_skb_cb(skb)->drop_reason = reason;
+}
+
+void __tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q,
+			  struct netdev_queue *txq, struct net_device *dev);
+
+static inline void tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q,
+				      struct netdev_queue *txq,
+				      struct net_device *dev)
+{
+	if (unlikely(skb))
+		__tcf_kfree_skb_list(skb, q, txq, dev);
 }
 
 static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb,
-				      enum skb_drop_reason reason)
+				      enum qdisc_drop_reason reason)
 {
 	DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS));
 	DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK);
 
-	tcf_set_drop_reason(skb, reason);
+	tcf_set_qdisc_drop_reason(skb, reason);
 	skb->next = q->to_free;
 	q->to_free = skb;
 }
@@ -1322,9 +1367,9 @@ static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,
 
 static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch,
 				    struct sk_buff **to_free,
-				    enum skb_drop_reason reason)
+				    enum qdisc_drop_reason reason)
 {
-	tcf_set_drop_reason(skb, reason);
+	tcf_set_qdisc_drop_reason(skb, reason);
 	return qdisc_drop(skb, sch, to_free);
 }
 
@@ -1429,6 +1474,11 @@ void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
 void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
 				struct tcf_block *block);
 
+static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp)
+{
+	return !!miniqp->p_miniq;
+}
+
 void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
 
 int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb));
diff --git a/include/net/sock.h b/include/net/sock.h
index 6c9a83016e955..6fb40ab03da16 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -81,8 +81,13 @@
  * mini-semaphore synchronizes multiple users amongst themselves.
  */
 typedef struct {
-	spinlock_t		slock;
-	int			owned;
+	union {
+		struct slock_owned {
+			int		owned;
+			spinlock_t	slock;
+		};
+		long	combined;
+	};
 	wait_queue_head_t	wq;
 	/*
 	 * We express the mutex-alike socket_lock semantics
@@ -121,14 +126,14 @@ typedef __u64 __bitwise __addrpair;
  *	@skc_bypass_prot_mem: bypass the per-protocol memory accounting for skb
  *	@skc_bound_dev_if: bound device index if != 0
  *	@skc_bind_node: bind hash linkage for various protocol lookup tables
- *	@skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
+ *	@skc_portaddr_node: second hash linkage for UDP
  *	@skc_prot: protocol handlers inside a network family
  *	@skc_net: reference to the network namespace of this socket
  *	@skc_v6_daddr: IPV6 destination address
  *	@skc_v6_rcv_saddr: IPV6 source address
  *	@skc_cookie: socket's cookie value
  *	@skc_node: main hash linkage for various protocol lookup tables
- *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
+ *	@skc_nulls_node: main hash linkage for TCP
  *	@skc_tx_queue_mapping: tx queue number for this connection
  *	@skc_rx_queue_mapping: rx queue number for this connection
  *	@skc_flags: place holder for sk_flags
@@ -537,7 +542,7 @@ struct sock {
 	rwlock_t		sk_callback_lock;
 	u32			sk_ack_backlog;
 	u32			sk_max_ack_backlog;
-	unsigned long		sk_ino;
+	u64			sk_ino;
 	spinlock_t		sk_peer_lock;
 	int			sk_bind_phc;
 	struct pid		*sk_peer_pid;
@@ -1316,7 +1321,7 @@ struct proto {
 	int			(*sendmsg)(struct sock *sk, struct msghdr *msg,
 					   size_t len);
 	int			(*recvmsg)(struct sock *sk, struct msghdr *msg,
-					   size_t len, int flags, int *addr_len);
+					   size_t len, int flags);
 	void			(*splice_eof)(struct socket *sock);
 	int			(*bind)(struct sock *sk,
 					struct sockaddr_unsized *addr, int addr_len);
@@ -1387,7 +1392,6 @@ struct proto {
 
 	union {
 		struct inet_hashinfo	*hashinfo;
-		struct udp_table	*udp_table;
 		struct raw_hashinfo	*raw_hash;
 		struct smc_hashinfo	*smc_hash;
 	} h;
@@ -1709,7 +1713,6 @@ static inline void lock_sock(struct sock *sk)
 	lock_sock_nested(sk, 0);
 }
 
-void __lock_sock(struct sock *sk);
 void __release_sock(struct sock *sk);
 void release_sock(struct sock *sk);
 
@@ -2140,7 +2143,7 @@ static inline void sock_graft(struct sock *sk, struct socket *parent)
 	write_unlock_bh(&sk->sk_callback_lock);
 }
 
-static inline unsigned long sock_i_ino(const struct sock *sk)
+static inline u64 sock_i_ino(const struct sock *sk)
 {
 	/* Paired with WRITE_ONCE() in sock_graft() and sock_orphan() */
 	return READ_ONCE(sk->sk_ino);
@@ -2193,8 +2196,12 @@ sk_dst_get(const struct sock *sk)
 
 	rcu_read_lock();
 	dst = rcu_dereference(sk->sk_dst_cache);
-	if (dst && !rcuref_get(&dst->__rcuref))
-		dst = NULL;
+	if (dst) {
+		if (!rcuref_get(&dst->__rcuref))
+			dst = NULL;
+		else
+			save_dst_trace_buffer(dst, 1);
+	}
 	rcu_read_unlock();
 	return dst;
 }
diff --git a/include/net/switchdev.h b/include/net/switchdev.h
index 8346b0d29542c..ee500706496b0 100644
--- a/include/net/switchdev.h
+++ b/include/net/switchdev.h
@@ -15,6 +15,7 @@
 #define SWITCHDEV_F_NO_RECURSE		BIT(0)
 #define SWITCHDEV_F_SKIP_EOPNOTSUPP	BIT(1)
 #define SWITCHDEV_F_DEFER		BIT(2)
+#define SWITCHDEV_F_NO_FOREIGN		BIT(3)
 
 enum switchdev_attr_id {
 	SWITCHDEV_ATTR_ID_UNDEFINED,
diff --git a/include/net/tc_wrapper.h b/include/net/tc_wrapper.h
index ffe58a02537c3..4ebb053bb0dd6 100644
--- a/include/net/tc_wrapper.h
+++ b/include/net/tc_wrapper.h
@@ -12,7 +12,8 @@
 
 #define TC_INDIRECT_SCOPE
 
-extern struct static_key_false tc_skip_wrapper;
+extern struct static_key_false tc_skip_wrapper_act;
+extern struct static_key_false tc_skip_wrapper_cls;
 
 /* TC Actions */
 #ifdef CONFIG_NET_CLS_ACT
@@ -46,7 +47,7 @@ TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
 			   struct tcf_result *res)
 {
-	if (static_branch_likely(&tc_skip_wrapper))
+	if (static_branch_likely(&tc_skip_wrapper_act))
 		goto skip;
 
 #if IS_BUILTIN(CONFIG_NET_ACT_GACT)
@@ -153,7 +154,7 @@ TC_INDIRECT_FILTER_DECLARE(u32_classify);
 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
 				struct tcf_result *res)
 {
-	if (static_branch_likely(&tc_skip_wrapper))
+	if (static_branch_likely(&tc_skip_wrapper_cls))
 		goto skip;
 
 #if IS_BUILTIN(CONFIG_NET_CLS_BPF)
@@ -202,8 +203,44 @@ static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
 static inline void tc_wrapper_init(void)
 {
 #ifdef CONFIG_X86
-	if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
-		static_branch_enable(&tc_skip_wrapper);
+	int cnt_cls = IS_BUILTIN(CONFIG_NET_CLS_BPF) +
+		IS_BUILTIN(CONFIG_NET_CLS_U32)  +
+		IS_BUILTIN(CONFIG_NET_CLS_FLOWER) +
+		IS_BUILTIN(CONFIG_NET_CLS_FW) +
+		IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) +
+		IS_BUILTIN(CONFIG_NET_CLS_BASIC) +
+		IS_BUILTIN(CONFIG_NET_CLS_CGROUP) +
+		IS_BUILTIN(CONFIG_NET_CLS_FLOW) +
+		IS_BUILTIN(CONFIG_NET_CLS_ROUTE4);
+
+	int cnt_act = IS_BUILTIN(CONFIG_NET_ACT_GACT) +
+		IS_BUILTIN(CONFIG_NET_ACT_MIRRED) +
+		IS_BUILTIN(CONFIG_NET_ACT_PEDIT) +
+		IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) +
+		IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) +
+		IS_BUILTIN(CONFIG_NET_ACT_POLICE) +
+		IS_BUILTIN(CONFIG_NET_ACT_BPF) +
+		IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) +
+		IS_BUILTIN(CONFIG_NET_ACT_CSUM) +
+		IS_BUILTIN(CONFIG_NET_ACT_CT) +
+		IS_BUILTIN(CONFIG_NET_ACT_CTINFO) +
+		IS_BUILTIN(CONFIG_NET_ACT_GATE) +
+		IS_BUILTIN(CONFIG_NET_ACT_MPLS) +
+		IS_BUILTIN(CONFIG_NET_ACT_NAT) +
+		IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) +
+		IS_BUILTIN(CONFIG_NET_ACT_VLAN) +
+		IS_BUILTIN(CONFIG_NET_ACT_IFE) +
+		IS_BUILTIN(CONFIG_NET_ACT_SIMP) +
+		IS_BUILTIN(CONFIG_NET_ACT_SAMPLE);
+
+	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE))
+		return;
+
+	if (cnt_cls > 1)
+		static_branch_enable(&tc_skip_wrapper_cls);
+
+	if (cnt_act > 1)
+		static_branch_enable(&tc_skip_wrapper_act);
 #endif
 }
 
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 978eea2d5df04..f87bdacb5a699 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -363,7 +363,6 @@ int tcp_v4_err(struct sk_buff *skb, u32);
 
 void tcp_shutdown(struct sock *sk, int how);
 
-int tcp_v4_early_demux(struct sk_buff *skb);
 int tcp_v4_rcv(struct sk_buff *skb);
 
 void tcp_remove_empty_skb(struct sock *sk);
@@ -376,7 +375,21 @@ int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
 int tcp_wmem_schedule(struct sock *sk, int copy);
 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
 	      int size_goal);
+
 void tcp_release_cb(struct sock *sk);
+
+static inline bool tcp_release_cb_cond(struct sock *sk)
+{
+#ifdef CONFIG_INET
+	if (likely(sk->sk_prot->release_cb == tcp_release_cb)) {
+		if (unlikely(smp_load_acquire(&sk->sk_tsq_flags) & TCP_DEFERRED_ALL))
+			tcp_release_cb(sk);
+		return true;
+	}
+#endif
+	return false;
+}
+
 void tcp_wfree(struct sk_buff *skb);
 void tcp_write_timer_handler(struct sock *sk);
 void tcp_delack_timer_handler(struct sock *sk);
@@ -501,11 +514,18 @@ void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
 void tcp_set_keepalive(struct sock *sk, int val);
 void tcp_syn_ack_timeout(const struct request_sock *req);
 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-		int flags, int *addr_len);
+		int flags);
 int tcp_set_rcvlowat(struct sock *sk, int val);
 int tcp_set_window_clamp(struct sock *sk, int val);
-void tcp_update_recv_tstamps(struct sk_buff *skb,
-			     struct scm_timestamping_internal *tss);
+
+static inline void
+tcp_update_recv_tstamps(struct sk_buff *skb,
+			struct scm_timestamping_internal *tss)
+{
+	tss->ts[0] = skb->tstamp;
+	tss->ts[2] = skb_hwtstamps(skb)->hwtstamp;
+}
+
 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
 			struct scm_timestamping_internal *tss);
 void tcp_data_ready(struct sock *sk);
@@ -532,7 +552,6 @@ u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
  *	TCP v4 functions exported for the inet6 API
  */
 
-void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
 void tcp_v4_mtu_reduced(struct sock *sk);
 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
@@ -915,6 +934,28 @@ static inline u32 tcp_receive_window(const struct tcp_sock *tp)
 	return (u32) win;
 }
 
+/* Compute the maximum receive window we ever advertised.
+ * Rcv_nxt can be after the window if our peer push more data
+ * than the offered window.
+ */
+static inline u32 tcp_max_receive_window(const struct tcp_sock *tp)
+{
+	s32 win = tp->rcv_mwnd_seq - tp->rcv_nxt;
+
+	if (win < 0)
+		win = 0;
+	return (u32) win;
+}
+
+/* Check if we need to update the maximum receive window sequence number */
+static inline void tcp_update_max_rcv_wnd_seq(struct tcp_sock *tp)
+{
+	u32 wre = tp->rcv_wup + tp->rcv_wnd;
+
+	if (after(wre, tp->rcv_mwnd_seq))
+		tp->rcv_mwnd_seq = wre;
+}
+
 /* Choose a new window, without checks for shrinking, and without
  * scaling applied to the result.  The caller does these things
  * if necessary.  This is a "raw" window selection.
@@ -1135,9 +1176,7 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb)
 
 extern const struct inet_connection_sock_af_ops ipv6_specific;
 
-INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
-void tcp_v6_early_demux(struct sk_buff *skb);
 
 #endif
 
@@ -2156,7 +2195,30 @@ enum tcp_chrono {
 	__TCP_CHRONO_MAX,
 };
 
-void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
+static inline void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
+{
+	const u32 now = tcp_jiffies32;
+	enum tcp_chrono old = tp->chrono_type;
+
+	if (old > TCP_CHRONO_UNSPEC)
+		tp->chrono_stat[old - 1] += now - tp->chrono_start;
+	tp->chrono_start = now;
+	tp->chrono_type = new;
+}
+
+static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
+{
+	struct tcp_sock *tp = tcp_sk(sk);
+
+	/* If there are multiple conditions worthy of tracking in a
+	 * chronograph then the highest priority enum takes precedence
+	 * over the other conditions. So that if something "more interesting"
+	 * starts happening, stop the previous chrono and start a new one.
+	 */
+	if (type > tp->chrono_type)
+		tcp_chrono_set(tp, type);
+}
+
 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
 
 /* This helper is needed, because skb->tcp_tsorted_anchor uses
@@ -2385,7 +2447,15 @@ void tcp_gro_complete(struct sk_buff *skb);
 static inline void tcp_gro_complete(struct sk_buff *skb) { }
 #endif
 
-void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
+static inline void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr,
+				       __be32 daddr)
+{
+	struct tcphdr *th = tcp_hdr(skb);
+
+	th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
+	skb->csum_start = skb_transport_header(skb) - skb->head;
+	skb->csum_offset = offsetof(struct tcphdr, check);
+}
 
 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
 {
diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h
index 1a97e3f32029a..c0a421fe0c2a0 100644
--- a/include/net/transp_v6.h
+++ b/include/net/transp_v6.h
@@ -8,7 +8,6 @@
 /* IPv6 transport protocols */
 extern struct proto rawv6_prot;
 extern struct proto udpv6_prot;
-extern struct proto udplitev6_prot;
 extern struct proto tcpv6_prot;
 extern struct proto pingv6_prot;
 
@@ -28,8 +27,6 @@ int rawv6_init(void);
 void rawv6_exit(void);
 int udpv6_init(void);
 void udpv6_exit(void);
-int udplitev6_init(void);
-void udplitev6_exit(void);
 int tcpv6_init(void);
 void tcpv6_exit(void);
 
diff --git a/include/net/udp.h b/include/net/udp.h
index da68702ddf6ed..8262e2b215b4e 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -29,13 +29,12 @@
 #include <linux/seq_file.h>
 #include <linux/poll.h>
 #include <linux/indirect_call_wrapper.h>
+#include <linux/math.h>
 
 /**
- *	struct udp_skb_cb  -  UDP(-Lite) private variables
+ *	struct udp_skb_cb  -  UDP private variables
  *
  *	@header:      private variables used by IPv4/IPv6
- *	@cscov:       checksum coverage length (UDP-Lite only)
- *	@partial_cov: if set indicates partial csum coverage
  */
 struct udp_skb_cb {
 	union {
@@ -44,8 +43,6 @@ struct udp_skb_cb {
 		struct inet6_skb_parm	h6;
 #endif
 	} header;
-	__u16		cscov;
-	__u8		partial_cov;
 };
 #define UDP_SKB_CB(__skb)	((struct udp_skb_cb *)((__skb)->cb))
 
@@ -104,7 +101,7 @@ struct udp_table {
 	unsigned int		log;
 };
 extern struct udp_table udp_table;
-void udp_table_init(struct udp_table *, const char *);
+
 static inline struct udp_hslot *udp_hashslot(struct udp_table *table,
 					     const struct net *net,
 					     unsigned int num)
@@ -215,13 +212,11 @@ extern int sysctl_udp_wmem_min;
 struct sk_buff;
 
 /*
- *	Generic checksumming routines for UDP(-Lite) v4 and v6
+ *	Generic checksumming routines for UDP v4 and v6
  */
 static inline __sum16 __udp_lib_checksum_complete(struct sk_buff *skb)
 {
-	return (UDP_SKB_CB(skb)->cscov == skb->len ?
-		__skb_checksum_complete(skb) :
-		__skb_checksum_complete_head(skb, UDP_SKB_CB(skb)->cscov));
+	return __skb_checksum_complete(skb);
 }
 
 static inline int udp_lib_checksum_complete(struct sk_buff *skb)
@@ -272,7 +267,6 @@ static inline void udp_csum_pull_header(struct sk_buff *skb)
 		skb->csum = csum_partial(skb->data, sizeof(struct udphdr),
 					 skb->csum);
 	skb_pull_rcsum(skb, sizeof(struct udphdr));
-	UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr);
 }
 
 typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport,
@@ -281,6 +275,10 @@ typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport,
 void udp_v6_early_demux(struct sk_buff *skb);
 INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *));
 
+int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len);
+INDIRECT_CALLABLE_DECLARE(int udpv6_recvmsg(struct sock *sk, struct msghdr *msg,
+					    size_t len, int flags));
+
 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
 				  netdev_features_t features, bool is_ipv6);
 
@@ -307,7 +305,7 @@ static inline void udp_drops_inc(struct sock *sk)
 	numa_drop_add(&udp_sk(sk)->drop_counters, 1);
 }
 
-/* hash routines shared between UDPv4/6 and UDP-Litev4/6 */
+/* hash routines shared between UDPv4/6 */
 static inline int udp_lib_hash(struct sock *sk)
 {
 	BUG();
@@ -376,7 +374,7 @@ static inline __be16 udp_flow_src_port(struct net *net, struct sk_buff *skb,
 	 */
 	hash ^= hash << 16;
 
-	return htons((((u64) hash * (max - min)) >> 32) + min);
+	return htons(reciprocal_scale(hash, max - min + 1) + min);
 }
 
 static inline int udp_rqueue_get(struct sock *sk)
@@ -415,6 +413,8 @@ bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst);
 int udp_err(struct sk_buff *, u32);
 int udp_abort(struct sock *sk, int err);
 int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len);
+INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *sk, struct msghdr *msg,
+					  size_t len, int flags));
 void udp_splice_eof(struct socket *sock);
 int udp_push_pending_frames(struct sock *sk);
 void udp_flush_pending_frames(struct sock *sk);
@@ -422,7 +422,6 @@ int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size);
 void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst);
 int udp_rcv(struct sk_buff *skb);
 int udp_ioctl(struct sock *sk, int cmd, int *karg);
-int udp_init_sock(struct sock *sk);
 int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len);
 int __udp_disconnect(struct sock *sk, int flags);
 int udp_disconnect(struct sock *sk, int flags);
@@ -438,9 +437,8 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
 struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
 			     __be32 daddr, __be16 dport, int dif);
 struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
-			       __be16 sport,
-			       __be32 daddr, __be16 dport, int dif, int sdif,
-			       struct udp_table *tbl, struct sk_buff *skb);
+			       __be16 sport, __be32 daddr, __be16 dport,
+			       int dif, int sdif, struct sk_buff *skb);
 struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
 				 __be16 sport, __be16 dport);
 struct sock *udp6_lib_lookup(const struct net *net,
@@ -450,8 +448,7 @@ struct sock *udp6_lib_lookup(const struct net *net,
 struct sock *__udp6_lib_lookup(const struct net *net,
 			       const struct in6_addr *saddr, __be16 sport,
 			       const struct in6_addr *daddr, __be16 dport,
-			       int dif, int sdif, struct udp_table *tbl,
-			       struct sk_buff *skb);
+			       int dif, int sdif, struct sk_buff *skb);
 struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
 				 __be16 sport, __be16 dport);
 int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
@@ -523,38 +520,28 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off,
 }
 
 /*
- * 	SNMP statistics for UDP and UDP-Lite
+ *	SNMP statistics for UDP
  */
-#define UDP_INC_STATS(net, field, is_udplite)		      do { \
-	if (unlikely(is_udplite)) SNMP_INC_STATS((net)->mib.udplite_statistics, field);	\
-	else		SNMP_INC_STATS((net)->mib.udp_statistics, field);  }  while(0)
-#define __UDP_INC_STATS(net, field, is_udplite) 	      do { \
-	if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_statistics, field);	\
-	else		__SNMP_INC_STATS((net)->mib.udp_statistics, field);    }  while(0)
-
-#define __UDP6_INC_STATS(net, field, is_udplite)	    do { \
-	if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);	\
-	else		__SNMP_INC_STATS((net)->mib.udp_stats_in6, field);  \
-} while(0)
-#define UDP6_INC_STATS(net, field, __lite)		    do { \
-	if (unlikely(__lite)) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);	\
-	else	    SNMP_INC_STATS((net)->mib.udp_stats_in6, field);      \
-} while(0)
+#define __UDP_INC_STATS(net, field)				\
+	__SNMP_INC_STATS((net)->mib.udp_statistics, field)
+#define UDP_INC_STATS(net, field)				\
+	SNMP_INC_STATS((net)->mib.udp_statistics, field)
+#define __UDP6_INC_STATS(net, field)				\
+	__SNMP_INC_STATS((net)->mib.udp_stats_in6, field)
+#define UDP6_INC_STATS(net, field)				\
+	SNMP_INC_STATS((net)->mib.udp_stats_in6, field)
 
 #if IS_ENABLED(CONFIG_IPV6)
-#define __UDPX_MIB(sk, ipv4)						\
-({									\
-	ipv4 ? (IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics :	\
-				 sock_net(sk)->mib.udp_statistics) :	\
-		(IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_stats_in6 :	\
-				 sock_net(sk)->mib.udp_stats_in6);	\
-})
+#define __UDPX_MIB(sk, ipv4)					\
+	({							\
+		ipv4 ? sock_net(sk)->mib.udp_statistics :	\
+			sock_net(sk)->mib.udp_stats_in6;	\
+	})
 #else
-#define __UDPX_MIB(sk, ipv4)						\
-({									\
-	IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics :		\
-			 sock_net(sk)->mib.udp_statistics;		\
-})
+#define __UDPX_MIB(sk, ipv4)					\
+	({							\
+		sock_net(sk)->mib.udp_statistics;		\
+	})
 #endif
 
 #define __UDPX_INC_STATS(sk, field) \
@@ -563,7 +550,6 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off,
 #ifdef CONFIG_PROC_FS
 struct udp_seq_afinfo {
 	sa_family_t			family;
-	struct udp_table		*udp_table;
 };
 
 struct udp_iter_state {
@@ -575,9 +561,6 @@ void *udp_seq_start(struct seq_file *seq, loff_t *pos);
 void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
 void udp_seq_stop(struct seq_file *seq, void *v);
 
-extern const struct seq_operations udp_seq_ops;
-extern const struct seq_operations udp6_seq_ops;
-
 int udp4_proc_init(void);
 void udp4_proc_exit(void);
 #endif /* CONFIG_PROC_FS */
@@ -648,9 +631,6 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk,
 
 static inline void udp_post_segment_fix_csum(struct sk_buff *skb)
 {
-	/* UDP-lite can't land here - no GRO */
-	WARN_ON_ONCE(UDP_SKB_CB(skb)->partial_cov);
-
 	/* UDP packets generated with UDP_SEGMENT and traversing:
 	 *
 	 * UDP tunnel(xmit) -> veth (segmentation) -> veth (gro) -> UDP tunnel (rx)
@@ -664,7 +644,6 @@ static inline void udp_post_segment_fix_csum(struct sk_buff *skb)
 	 * a valid csum after the segmentation.
 	 * Additionally fixup the UDP CB.
 	 */
-	UDP_SKB_CB(skb)->cscov = skb->len;
 	if (skb->ip_summed == CHECKSUM_NONE && !skb->csum_valid)
 		skb->csum_valid = 1;
 }
diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h
index d9c6d04bb3b58..fc1fc43345b5d 100644
--- a/include/net/udp_tunnel.h
+++ b/include/net/udp_tunnel.h
@@ -52,7 +52,7 @@ int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
 static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
 				   struct socket **sockp)
 {
-	return 0;
+	return -EPFNOSUPPORT;
 }
 #endif
 
diff --git a/include/net/udplite.h b/include/net/udplite.h
deleted file mode 100644
index 786919d29f8de..0000000000000
--- a/include/net/udplite.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- *	Definitions for the UDP-Lite (RFC 3828) code.
- */
-#ifndef _UDPLITE_H
-#define _UDPLITE_H
-
-#include <net/ip6_checksum.h>
-#include <net/udp.h>
-
-/* UDP-Lite socket options */
-#define UDPLITE_SEND_CSCOV   10 /* sender partial coverage (as sent)      */
-#define UDPLITE_RECV_CSCOV   11 /* receiver partial coverage (threshold ) */
-
-extern struct proto 		udplite_prot;
-extern struct udp_table		udplite_table;
-
-/*
- *	Checksum computation is all in software, hence simpler getfrag.
- */
-static __inline__ int udplite_getfrag(void *from, char *to, int  offset,
-				      int len, int odd, struct sk_buff *skb)
-{
-	struct msghdr *msg = from;
-	return copy_from_iter_full(to, len, &msg->msg_iter) ? 0 : -EFAULT;
-}
-
-/*
- * 	Checksumming routines
- */
-static inline int udplite_checksum_init(struct sk_buff *skb, struct udphdr *uh)
-{
-	u16 cscov;
-
-        /* In UDPv4 a zero checksum means that the transmitter generated no
-         * checksum. UDP-Lite (like IPv6) mandates checksums, hence packets
-         * with a zero checksum field are illegal.                            */
-	if (uh->check == 0) {
-		net_dbg_ratelimited("UDPLite: zeroed checksum field\n");
-		return 1;
-	}
-
-	cscov = ntohs(uh->len);
-
-	if (cscov == 0)		 /* Indicates that full coverage is required. */
-		;
-	else if (cscov < 8  || cscov > skb->len) {
-		/*
-		 * Coverage length violates RFC 3828: log and discard silently.
-		 */
-		net_dbg_ratelimited("UDPLite: bad csum coverage %d/%d\n",
-				    cscov, skb->len);
-		return 1;
-
-	} else if (cscov < skb->len) {
-        	UDP_SKB_CB(skb)->partial_cov = 1;
-		UDP_SKB_CB(skb)->cscov = cscov;
-		if (skb->ip_summed == CHECKSUM_COMPLETE)
-			skb->ip_summed = CHECKSUM_NONE;
-		skb->csum_valid = 0;
-        }
-
-	return 0;
-}
-
-/* Fast-path computation of checksum. Socket may not be locked. */
-static inline __wsum udplite_csum(struct sk_buff *skb)
-{
-	const int off = skb_transport_offset(skb);
-	const struct sock *sk = skb->sk;
-	int len = skb->len - off;
-
-	if (udp_test_bit(UDPLITE_SEND_CC, sk)) {
-		u16 pcslen = READ_ONCE(udp_sk(sk)->pcslen);
-
-		if (pcslen < len) {
-			if (pcslen > 0)
-				len = pcslen;
-			udp_hdr(skb)->len = htons(pcslen);
-		}
-	}
-	skb->ip_summed = CHECKSUM_NONE;     /* no HW support for checksumming */
-
-	return skb_checksum(skb, off, len, 0);
-}
-
-void udplite4_register(void);
-#endif	/* _UDPLITE_H */
diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index 0b1abdb99c9ea..ccb3b350001f2 100644
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -174,13 +174,6 @@ static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,
 	dma_sync_single_for_device(pool->dev, dma, size, DMA_BIDIRECTIONAL);
 }
 
-/* Masks for xdp_umem_page flags.
- * The low 12-bits of the addr will be 0 since this is the page address, so we
- * can use them for flags.
- */
-#define XSK_NEXT_PG_CONTIG_SHIFT 0
-#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
-
 static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
 						 u64 addr, u32 len)
 {
diff --git a/include/rdma/frmr_pools.h b/include/rdma/frmr_pools.h
new file mode 100644
index 0000000000000..af1b88801fa46
--- /dev/null
+++ b/include/rdma/frmr_pools.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+ *
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#ifndef FRMR_POOLS_H
+#define FRMR_POOLS_H
+
+#include <linux/types.h>
+#include <asm/page.h>
+
+struct ib_device;
+struct ib_mr;
+
+struct ib_frmr_key {
+	u64 vendor_key;
+	/* A pool with non-zero kernel_vendor_key is a kernel-only pool. */
+	u64 kernel_vendor_key;
+	size_t num_dma_blocks;
+	int access_flags;
+	u8 ats:1;
+};
+
+struct ib_frmr_pool_ops {
+	int (*create_frmrs)(struct ib_device *device, struct ib_frmr_key *key,
+			    u32 *handles, u32 count);
+	void (*destroy_frmrs)(struct ib_device *device, u32 *handles,
+			      u32 count);
+	int (*build_key)(struct ib_device *device, const struct ib_frmr_key *in,
+			 struct ib_frmr_key *out);
+};
+
+int ib_frmr_pools_init(struct ib_device *device,
+		       const struct ib_frmr_pool_ops *pool_ops);
+void ib_frmr_pools_cleanup(struct ib_device *device);
+int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr);
+int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr);
+
+#endif /* FRMR_POOLS_H */
diff --git a/include/rdma/ib_cache.h b/include/rdma/ib_cache.h
index 2bf09b594d10f..eed46d966e40a 100644
--- a/include/rdma/ib_cache.h
+++ b/include/rdma/ib_cache.h
@@ -34,7 +34,7 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);
 
 /**
  * ib_get_cached_pkey - Returns a cached PKey table entry
- * @device: The device to query.
+ * @device_handle: The device to query.
  * @port_num: The port number of the device to query.
  * @index: The index into the cached PKey table to query.
  * @pkey: The PKey value found at the specified index.
@@ -80,7 +80,7 @@ int ib_get_cached_lmc(struct ib_device *device,
  * ib_get_cached_port_state - Returns a cached port state table entry
  * @device: The device to query.
  * @port_num: The port number of the device to query.
- * @port_state: port_state for the specified port for that device.
+ * @port_active: port_state for the specified port for that device.
  *
  * ib_get_cached_port_state() fetches the specified port_state table entry stored in
  * the local software cache.
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 0a8e092c0ea87..38414281a686b 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -7,13 +7,9 @@
 #ifndef IB_UMEM_H
 #define IB_UMEM_H
 
-#include <linux/list.h>
 #include <linux/scatterlist.h>
-#include <linux/workqueue.h>
-#include <rdma/ib_verbs.h>
 
-struct ib_ucontext;
-struct ib_umem_odp;
+struct ib_device;
 struct dma_buf_attach_ops;
 
 struct ib_umem {
@@ -36,6 +32,7 @@ struct ib_umem_dmabuf {
 	struct scatterlist *last_sg;
 	unsigned long first_sg_offset;
 	unsigned long last_sg_trim;
+	void (*pinned_revoke)(void *priv);
 	void *private;
 	u8 pinned : 1;
 	u8 revoked : 1;
@@ -75,37 +72,6 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem)
 {
 	return ib_umem_num_dma_blocks(umem, PAGE_SIZE);
 }
-
-static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,
-						struct ib_umem *umem,
-						unsigned long pgsz)
-{
-	__rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl,
-				umem->sgt_append.sgt.nents, pgsz);
-	biter->__sg_advance = ib_umem_offset(umem) & ~(pgsz - 1);
-	biter->__sg_numblocks = ib_umem_num_dma_blocks(umem, pgsz);
-}
-
-static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter)
-{
-	return __rdma_block_iter_next(biter) && biter->__sg_numblocks--;
-}
-
-/**
- * rdma_umem_for_each_dma_block - iterate over contiguous DMA blocks of the umem
- * @umem: umem to iterate over
- * @pgsz: Page size to split the list into
- *
- * pgsz must be <= PAGE_SIZE or computed by ib_umem_find_best_pgsz(). The
- * returned DMA blocks will be aligned to pgsz and span the range:
- * ALIGN_DOWN(umem->address, pgsz) to ALIGN(umem->address + umem->length, pgsz)
- *
- * Performs exactly ib_umem_num_dma_blocks() iterations.
- */
-#define rdma_umem_for_each_dma_block(umem, biter, pgsz)                        \
-	for (__rdma_umem_block_iter_start(biter, umem, pgsz);                  \
-	     __rdma_umem_block_iter_next(biter);)
-
 #ifdef CONFIG_INFINIBAND_USER_MEM
 
 struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
@@ -121,7 +87,7 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
  * ib_umem_find_best_pgoff - Find best HW page size
  *
  * @umem: umem struct
- * @pgsz_bitmap bitmap of HW supported page sizes
+ * @pgsz_bitmap: bitmap of HW supported page sizes
  * @pgoff_bitmask: Mask of bits that can be represented with an offset
  *
  * This is very similar to ib_umem_find_best_pgsz() except instead of accepting
@@ -134,6 +100,9 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
  *
  * If the pgoff_bitmask requires either alignment in the low bit or an
  * unavailable page size for the high bits, this function returns 0.
+ *
+ * Returns: best HW page size for the parameters or 0 if none available
+ *   for the given parameters.
  */
 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,
 						    unsigned long pgsz_bitmap,
@@ -169,6 +138,12 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
 						 size_t size, int fd,
 						 int access);
 struct ib_umem_dmabuf *
+ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device,
+					     unsigned long offset, size_t size,
+					     int fd, int access);
+void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf,
+				      void (*revoke)(void *priv), void *priv);
+struct ib_umem_dmabuf *
 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
 					  struct device *dma_device,
 					  unsigned long offset, size_t size,
@@ -176,6 +151,8 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf);
 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf);
 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf);
+void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf);
+void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf);
 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf);
 
 #else /* CONFIG_INFINIBAND_USER_MEM */
@@ -221,6 +198,18 @@ ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset,
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
+static inline struct ib_umem_dmabuf *
+ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device,
+					     unsigned long offset, size_t size,
+					     int fd, int access)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
+static inline void
+ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf,
+				 void (*revoke)(void *priv), void *priv) {}
+
 static inline struct ib_umem_dmabuf *
 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,
 					  struct device *dma_device,
@@ -236,6 +225,8 @@ static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
 }
 static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { }
 static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { }
+static inline void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf) {}
+static inline void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf) {}
 static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {}
 
 #endif /* CONFIG_INFINIBAND_USER_MEM */
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 3f3827e1c711d..57b81ca0fabdc 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -44,6 +44,7 @@
 #include <uapi/rdma/rdma_user_ioctl.h>
 #include <uapi/rdma/ib_user_ioctl_verbs.h>
 #include <linux/pci-tph.h>
+#include <rdma/frmr_pools.h>
 #include <linux/dma-buf.h>
 
 #define IB_FW_VERSION_NAME_MAX	ETHTOOL_FWVERS_LEN
@@ -1576,6 +1577,93 @@ struct ib_uobject {
 	const struct uverbs_api_object *uapi_object;
 };
 
+/**
+ * struct ib_udata - Driver request/response data from userspace
+ * @inbuf: Pointer to request data from userspace
+ * @outbuf: Pointer to response buffer in userspace
+ * @inlen: Length of request data
+ * @outlen: Length of response buffer
+ *
+ * struct ib_udata is used to hold the driver data request and response
+ * structures defined in the uapi. They follow these rules for forwards and
+ * backwards compatibility:
+ *
+ * 1) Userspace can provide a longer request so long as the trailing part the
+ *    kernel doesn't understand is all zeros.
+ *
+ *    This provides a degree of safety if userspace wrongly tries to use a new
+ *    feature the kernel does not understand with some non-zero value.
+ *
+ *    It allows a simpler rdma-core implementation because the library can
+ *    simply always use the latest structs for the request, even if they are
+ *    bigger. It simply has to avoid using the new members if they are not
+ *    supported/required.
+ *
+ * 2) Userspace can provide a shorter request; the kernel will zero-pad it out
+ *    to fill the storage. The newer kernel should understand that older
+ *    userspace will provide 0 to new fields. The kernel has three options to
+ *    enable new request fields:
+ *
+ *    - Input comp_mask that says the field is supported
+ *    - Look for non-zero values
+ *    - Check if the udata->inlen size covers the field
+ *
+ *    This also corrects any bugs related to not filling in request structures
+ *    as the new helper always fully writes to the struct.
+ *
+ * 3) Userspace can provide a shorter or longer response struct. If shorter,
+ *    the kernel reply is truncated. The kernel should be designed to not write
+ *    to new reply fields unless userspace has affirmatively requested them.
+ *
+ *    If the user buffer is longer, the kernel will zero-fill it.
+ *
+ *    Userspace has three options to enable new response fields:
+ *
+ *    - Output comp_mask that says the field is supported
+ *    - Look for non-zero values
+ *    - Infer the output must be valid because the request contents demand it
+ *      and old kernels will fail the request
+ *
+ * The following helper functions implement these semantics:
+ *
+ * ib_copy_validate_udata_in() - Checks the minimum length, and zero trailing::
+ *
+ *     struct driver_create_cq_req req;
+ *     int err;
+ *
+ *     err = ib_copy_validate_udata_in(udata, req, end_member);
+ *     if (err)
+ *         return err;
+ *
+ * The third argument specifies the last member of the struct in the first
+ * kernel version that introduced it, establishing the minimum required size.
+ *
+ * ib_copy_validate_udata_in_cm() - The above but also validate a
+ * comp_mask member only has supported bits set::
+ *
+ *     err = ib_copy_validate_udata_in_cm(udata, req, first_version_last_member,
+ *                                        DRIVER_CREATE_CQ_MASK_FEATURE_A |
+ *                                        DRIVER_CREATE_CQ_MASK_FEATURE_B);
+ *
+ * ib_respond_udata() - Implements the response rules::
+ *
+ *     struct driver_create_cq_resp resp = {};
+ *
+ *     resp.some_field = value;
+ *     return ib_respond_udata(udata, resp);
+ *
+ * ib_is_udata_in_empty() - Used instead of ib_copy_validate_udata_in() if the
+ * driver does not have a request structure::
+ *
+ *     ret = ib_is_udata_in_empty(udata);
+ *     if (ret)
+ *         return ret;
+ *
+ * Similarly ib_respond_empty_udata() is used instead of ib_respond_udata() if
+ * the driver does not have a response structure::
+ *
+ *    return ib_respond_empty_udata(udata);
+ */
 struct ib_udata {
 	const void __user *inbuf;
 	void __user *outbuf;
@@ -1650,6 +1738,7 @@ struct ib_cq {
 	u8 interrupt:1;
 	u8 shared:1;
 	unsigned int comp_vector;
+	struct ib_umem *umem;
 
 	/*
 	 * Implementation details of the RDMA core, don't use in drivers:
@@ -1904,6 +1993,11 @@ struct ib_mr {
 	struct ib_dm      *dm;
 	struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
 	struct ib_dmah *dmah;
+	struct {
+		struct ib_frmr_pool *pool;
+		struct ib_frmr_key key;
+		u32 handle;
+	} frmr;
 	/*
 	 * Implementation details of the RDMA core, don't use in drivers:
 	 */
@@ -2267,7 +2361,6 @@ struct ib_port_data {
 
 /* rdma netdev type - specifies protocol type */
 enum rdma_netdev_t {
-	RDMA_NETDEV_OPA_VNIC,
 	RDMA_NETDEV_IPOIB,
 };
 
@@ -2281,11 +2374,6 @@ struct rdma_netdev {
 	u32		   port_num;
 	int                mtu;
 
-	/*
-	 * cleanup function must be specified.
-	 * FIXME: This is only used for OPA_VNIC and that usage should be
-	 * removed too.
-	 */
 	void (*free_rdma_netdev)(struct net_device *netdev);
 
 	/* control functions */
@@ -2387,6 +2475,12 @@ struct ib_device_ops {
 	enum rdma_driver_id driver_id;
 	u32 uverbs_abi_ver;
 	unsigned int uverbs_no_driver_id_binding:1;
+	/*
+	 * Indicates the driver checks every op accepting a udata for the
+	 * correct size on input and always handles the output using the udata
+	 * helpers.
+	 */
+	unsigned int uverbs_robust_udata:1;
 
 	/*
 	 * NOTE: New drivers should not make use of device_group; instead new
@@ -2419,8 +2513,6 @@ struct ib_device_ops {
 	int (*modify_device)(struct ib_device *device, int device_modify_mask,
 			     struct ib_device_modify *device_modify);
 	void (*get_dev_fw_str)(struct ib_device *device, char *str);
-	const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
-						     int comp_vector);
 	int (*query_port)(struct ib_device *device, u32 port_num,
 			  struct ib_port_attr *port_attr);
 	int (*query_port_speed)(struct ib_device *device, u32 port_num,
@@ -2537,9 +2629,8 @@ struct ib_device_ops {
 	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
 	int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
 			 struct uverbs_attr_bundle *attrs);
-	int (*create_cq_umem)(struct ib_cq *cq,
+	int (*create_user_cq)(struct ib_cq *cq,
 			      const struct ib_cq_init_attr *attr,
-			      struct ib_umem *umem,
 			      struct uverbs_attr_bundle *attrs);
 	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
@@ -2907,6 +2998,8 @@ struct ib_device {
 	struct list_head subdev_list;
 
 	enum rdma_nl_name_assign_type name_assign_type;
+
+	struct ib_frmr_pools *frmr_pools;
 };
 
 static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,
@@ -2959,22 +3052,6 @@ struct ib_client {
 	u8 no_kverbs_req:1;
 };
 
-/*
- * IB block DMA iterator
- *
- * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
- * to a HW supported page size.
- */
-struct ib_block_iter {
-	/* internal states */
-	struct scatterlist *__sg;	/* sg holding the current aligned block */
-	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */
-	size_t __sg_numblocks;		/* ib_umem_num_dma_blocks() */
-	unsigned int __sg_nents;	/* number of SG entries */
-	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */
-	unsigned int __pg_bit;		/* alignment of current block */
-};
-
 struct ib_device *_ib_alloc_device(size_t size, struct net *net);
 #define ib_alloc_device(drv_struct, member)                                    \
 	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
@@ -3003,38 +3080,6 @@ void ib_unregister_device_queued(struct ib_device *ib_dev);
 int ib_register_client   (struct ib_client *client);
 void ib_unregister_client(struct ib_client *client);
 
-void __rdma_block_iter_start(struct ib_block_iter *biter,
-			     struct scatterlist *sglist,
-			     unsigned int nents,
-			     unsigned long pgsz);
-bool __rdma_block_iter_next(struct ib_block_iter *biter);
-
-/**
- * rdma_block_iter_dma_address - get the aligned dma address of the current
- * block held by the block iterator.
- * @biter: block iterator holding the memory block
- */
-static inline dma_addr_t
-rdma_block_iter_dma_address(struct ib_block_iter *biter)
-{
-	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
-}
-
-/**
- * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
- * @sglist: sglist to iterate over
- * @biter: block iterator holding the memory block
- * @nents: maximum number of sg entries to iterate over
- * @pgsz: best HW supported page size to use
- *
- * Callers may use rdma_block_iter_dma_address() to get each
- * blocks aligned DMA address.
- */
-#define rdma_for_each_block(sglist, biter, nents, pgsz)		\
-	for (__rdma_block_iter_start(biter, sglist, nents,	\
-				     pgsz);			\
-	     __rdma_block_iter_next(biter);)
-
 /**
  * ib_get_client_data - Get IB client context
  * @device:Device to get context for
@@ -4874,27 +4919,6 @@ static inline __be16 ib_lid_be16(u32 lid)
 	return cpu_to_be16((u16)lid);
 }
 
-/**
- * ib_get_vector_affinity - Get the affinity mappings of a given completion
- *   vector
- * @device:         the rdma device
- * @comp_vector:    index of completion vector
- *
- * Returns NULL on failure, otherwise a corresponding cpu map of the
- * completion vector (returns all-cpus map if the device driver doesn't
- * implement get_vector_affinity).
- */
-static inline const struct cpumask *
-ib_get_vector_affinity(struct ib_device *device, int comp_vector)
-{
-	if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
-	    !device->ops.get_vector_affinity)
-		return NULL;
-
-	return device->ops.get_vector_affinity(device, comp_vector);
-
-}
-
 /**
  * rdma_roce_rescan_device - Rescan all of the network devices in the system
  * and add their gids, as needed, to the relevant RoCE devices.
diff --git a/include/rdma/iter.h b/include/rdma/iter.h
new file mode 100644
index 0000000000000..19d64ef04ba9b
--- /dev/null
+++ b/include/rdma/iter.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. */
+
+#ifndef _RDMA_ITER_H_
+#define _RDMA_ITER_H_
+
+#include <linux/scatterlist.h>
+#include <rdma/ib_umem.h>
+
+/**
+ * IB block DMA iterator
+ *
+ * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
+ * to a HW supported page size.
+ */
+struct ib_block_iter {
+	/* internal states */
+	struct scatterlist *__sg;	/* sg holding the current aligned block */
+	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */
+	size_t __sg_numblocks;		/* ib_umem_num_dma_blocks() */
+	unsigned int __sg_nents;	/* number of SG entries */
+	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */
+	unsigned int __pg_bit;		/* alignment of current block */
+};
+
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist,
+			     unsigned int nents,
+			     unsigned long pgsz);
+bool __rdma_block_iter_next(struct ib_block_iter *biter);
+
+/**
+ * rdma_block_iter_dma_address - get the aligned dma address of the current
+ * block held by the block iterator.
+ * @biter: block iterator holding the memory block
+ */
+static inline dma_addr_t
+rdma_block_iter_dma_address(struct ib_block_iter *biter)
+{
+	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
+}
+
+/**
+ * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
+ * @sglist: sglist to iterate over
+ * @biter: block iterator holding the memory block
+ * @nents: maximum number of sg entries to iterate over
+ * @pgsz: best HW supported page size to use
+ *
+ * Callers may use rdma_block_iter_dma_address() to get each
+ * blocks aligned DMA address.
+ */
+#define rdma_for_each_block(sglist, biter, nents, pgsz)		\
+	for (__rdma_block_iter_start(biter, sglist, nents,	\
+				     pgsz);			\
+	     __rdma_block_iter_next(biter);)
+
+static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,
+						struct ib_umem *umem,
+						unsigned long pgsz)
+{
+	__rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl,
+				umem->sgt_append.sgt.nents, pgsz);
+	biter->__sg_advance = ib_umem_offset(umem) & ~(pgsz - 1);
+	biter->__sg_numblocks = ib_umem_num_dma_blocks(umem, pgsz);
+}
+
+static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter)
+{
+	return __rdma_block_iter_next(biter) && biter->__sg_numblocks--;
+}
+
+/**
+ * rdma_umem_for_each_dma_block - iterate over contiguous DMA blocks of the umem
+ * @umem: umem to iterate over
+ * @pgsz: Page size to split the list into
+ *
+ * pgsz must be <= PAGE_SIZE or computed by ib_umem_find_best_pgsz(). The
+ * returned DMA blocks will be aligned to pgsz and span the range:
+ * ALIGN_DOWN(umem->address, pgsz) to ALIGN(umem->address + umem->length, pgsz)
+ *
+ * Performs exactly ib_umem_num_dma_blocks() iterations.
+ */
+#define rdma_umem_for_each_dma_block(umem, biter, pgsz)                        \
+	for (__rdma_umem_block_iter_start(biter, umem, pgsz);                  \
+	     __rdma_umem_block_iter_next(biter);)
+
+#endif /* _RDMA_ITER_H_ */
diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h
index 2b22f153ef636..57b33edd9ce74 100644
--- a/include/rdma/iw_cm.h
+++ b/include/rdma/iw_cm.h
@@ -33,8 +33,8 @@ struct iw_cm_event {
 };
 
 /**
- * iw_cm_handler - Function to be called by the IW CM when delivering events
- * to the client.
+ * typedef iw_cm_handler - Function to be called by the IW CM when delivering
+ * events to the client.
  *
  * @cm_id: The IW CM identifier associated with the event.
  * @event: Pointer to the event structure.
@@ -43,9 +43,9 @@ typedef int (*iw_cm_handler)(struct iw_cm_id *cm_id,
 			     struct iw_cm_event *event);
 
 /**
- * iw_event_handler - Function called by the provider when delivering provider
- * events to the IW CM.  Returns either 0 indicating the event was processed
- * or -errno if the event could not be processed.
+ * typedef iw_event_handler - Function called by the provider when delivering
+ * provider events to the IW CM.  Returns either 0 indicating the event was
+ * processed or -errno if the event could not be processed.
  *
  * @cm_id: The IW CM identifier associated with the event.
  * @event: Pointer to the event structure.
@@ -97,7 +97,7 @@ enum iw_flags {
  * iw_create_cm_id - Create an IW CM identifier.
  *
  * @device: The IB device on which to create the IW CM identier.
- * @event_handler: User callback invoked to report events associated with the
+ * @cm_handler: User callback invoked to report events associated with the
  *   returned IW CM identifier.
  * @context: User specified context associated with the id.
  */
@@ -147,7 +147,7 @@ int iw_cm_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param);
  * iw_cm_reject - Reject an incoming connection request.
  *
  * @cm_id: Connection identifier associated with the request.
- * @private_daa: Pointer to data to deliver to the remote peer as part of the
+ * @private_data: Pointer to data to deliver to the remote peer as part of the
  *   reject message.
  * @private_data_len: The number of bytes in the private_data parameter.
  *
diff --git a/include/rdma/opa_port_info.h b/include/rdma/opa_port_info.h
index 73bcac90a0486..fb66d3a1dfa9d 100644
--- a/include/rdma/opa_port_info.h
+++ b/include/rdma/opa_port_info.h
@@ -93,9 +93,11 @@
 #define OPA_LINKINIT_QUARANTINED                (9 << 4)
 #define OPA_LINKINIT_INSUFIC_CAPABILITY         (10 << 4)
 
-#define OPA_LINK_SPEED_NOP              0x0000  /*  Reserved (1-5 Gbps) */
-#define OPA_LINK_SPEED_12_5G            0x0001  /*  12.5 Gbps */
-#define OPA_LINK_SPEED_25G              0x0002  /*  25.78125?  Gbps (EDR) */
+#define OPA_LINK_SPEED_NOP           0x0000  /* no change */
+#define OPA_LINK_SPEED_12_5G         0x0001  /* 12.5 Gbps */
+#define OPA_LINK_SPEED_25G           0x0002  /* 25.78125 Gbps */
+#define OPA_LINK_SPEED_50G           0x0004  /* 53.125 Gbps */
+#define OPA_LINK_SPEED_100G          0x0008  /* 106.25 Gbps */
 
 #define OPA_LINK_WIDTH_1X            0x0001
 #define OPA_LINK_WIDTH_2X            0x0002
diff --git a/include/rdma/opa_vnic.h b/include/rdma/opa_vnic.h
deleted file mode 100644
index d297f084001a5..0000000000000
--- a/include/rdma/opa_vnic.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
-/*
- * Copyright(c) 2017 - 2020 Intel Corporation.
- */
-
-#ifndef _OPA_VNIC_H
-#define _OPA_VNIC_H
-
-/*
- * This file contains Intel Omni-Path (OPA) Virtual Network Interface
- * Controller (VNIC) specific declarations.
- */
-
-#include <rdma/ib_verbs.h>
-
-/* 16 header bytes + 2 reserved bytes */
-#define OPA_VNIC_L2_HDR_LEN   (16 + 2)
-
-#define OPA_VNIC_L4_HDR_LEN   2
-
-#define OPA_VNIC_HDR_LEN      (OPA_VNIC_L2_HDR_LEN + \
-			       OPA_VNIC_L4_HDR_LEN)
-
-#define OPA_VNIC_L4_ETHR  0x78
-
-#define OPA_VNIC_ICRC_LEN   4
-#define OPA_VNIC_TAIL_LEN   1
-#define OPA_VNIC_ICRC_TAIL_LEN  (OPA_VNIC_ICRC_LEN + OPA_VNIC_TAIL_LEN)
-
-#define OPA_VNIC_SKB_MDATA_LEN         4
-#define OPA_VNIC_SKB_MDATA_ENCAP_ERR   0x1
-
-/* opa vnic rdma netdev's private data structure */
-struct opa_vnic_rdma_netdev {
-	struct rdma_netdev rn;  /* keep this first */
-	/* followed by device private data */
-	char *dev_priv[];
-};
-
-static inline void *opa_vnic_priv(const struct net_device *dev)
-{
-	struct rdma_netdev *rn = netdev_priv(dev);
-
-	return rn->clnt_priv;
-}
-
-static inline void *opa_vnic_dev_priv(const struct net_device *dev)
-{
-	struct opa_vnic_rdma_netdev *oparn = netdev_priv(dev);
-
-	return oparn->dev_priv;
-}
-
-/* opa_vnic skb meta data structure */
-struct opa_vnic_skb_mdata {
-	u8 vl;
-	u8 entropy;
-	u8 flags;
-	u8 rsvd;
-} __packed;
-
-/* OPA VNIC group statistics */
-struct opa_vnic_grp_stats {
-	u64 unicast;
-	u64 mcastbcast;
-	u64 untagged;
-	u64 vlan;
-	u64 s_64;
-	u64 s_65_127;
-	u64 s_128_255;
-	u64 s_256_511;
-	u64 s_512_1023;
-	u64 s_1024_1518;
-	u64 s_1519_max;
-};
-
-struct opa_vnic_stats {
-	/* standard netdev statistics */
-	struct rtnl_link_stats64 netstats;
-
-	/* OPA VNIC statistics */
-	struct opa_vnic_grp_stats tx_grp;
-	struct opa_vnic_grp_stats rx_grp;
-	u64 tx_dlid_zero;
-	u64 tx_drop_state;
-	u64 rx_drop_state;
-	u64 rx_runt;
-	u64 rx_oversize;
-};
-
-static inline bool rdma_cap_opa_vnic(struct ib_device *device)
-{
-	return !!(device->attrs.kernel_cap_flags & IBK_RDMA_NETDEV_OPA);
-}
-
-#endif /* _OPA_VNIC_H */
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index c429d6ddb1292..7d8de561f71b5 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -149,6 +149,7 @@ struct rvt_driver_params {
 /* User context */
 struct rvt_ucontext {
 	struct ib_ucontext ibucontext;
+	void *priv;
 };
 
 /* Protection domain */
@@ -359,6 +360,15 @@ struct rvt_driver_provided {
 
 	/* Get and return CPU to pin CQ processing thread */
 	int (*comp_vect_cpu_lookup)(struct rvt_dev_info *rdi, int comp_vect);
+
+	/* allocate a ucontext */
+	int (*alloc_ucontext)(struct ib_ucontext *uctx, struct ib_udata *udata);
+
+	/* deallocate a ucontext */
+	void (*dealloc_ucontext)(struct ib_ucontext *context);
+
+	/* driver mmap */
+	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
 };
 
 struct rvt_dev_info {
diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h
index 8a9bcf77daceb..451f99e3717d8 100644
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -87,11 +87,11 @@ struct rdma_restrack_entry {
 	 * query stage.
 	 */
 	u8			no_track : 1;
-	/*
+	/**
 	 * @kref: Protect destroy of the resource
 	 */
 	struct kref		kref;
-	/*
+	/**
 	 * @comp: Signal that all consumers of resource are completed their work
 	 */
 	struct completion	comp;
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index e6c0de227fad3..e2af17da3e32c 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -667,6 +667,8 @@ rdma_udata_to_uverbs_attr_bundle(struct ib_udata *udata)
 	(udata ? container_of(rdma_udata_to_uverbs_attr_bundle(udata)->context, \
 			      drv_dev_struct, member) : (drv_dev_struct *)NULL)
 
+struct ib_device *rdma_udata_to_dev(struct ib_udata *udata);
+
 #define IS_UVERBS_COPY_ERR(_ret)		((_ret) && (_ret) != -ENOENT)
 
 static inline const struct uverbs_attr *uverbs_attr_get(const struct uverbs_attr_bundle *attrs_bundle,
@@ -895,6 +897,10 @@ int _uverbs_get_const_unsigned(u64 *to,
 			       size_t idx, u64 upper_bound, u64 *def_val);
 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
 				  size_t idx, const void *from, size_t size);
+
+int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
+			       size_t kernel_size, size_t minimum_size);
+int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len);
 #else
 static inline int
 uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
@@ -951,6 +957,19 @@ _uverbs_get_const_unsigned(u64 *to,
 {
 	return -EINVAL;
 }
+
+static inline int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
+					     size_t kernel_size,
+					     size_t minimum_size)
+{
+	return -EINVAL;
+}
+
+static inline int _ib_respond_udata(struct ib_udata *udata, const void *src,
+				    size_t len)
+{
+	return -EINVAL;
+}
 #endif
 
 #define uverbs_get_const_signed(_to, _attrs_bundle, _idx)                      \
@@ -1016,4 +1035,86 @@ uverbs_get_raw_fd(int *to, const struct uverbs_attr_bundle *attrs_bundle,
 	return uverbs_get_const_signed(to, attrs_bundle, idx);
 }
 
+/**
+ * ib_copy_validate_udata_in - Copy and validate that the request structure is
+ *                             compatible with this kernel
+ * @_udata: The system calls ib_udata struct
+ * @_req: The name of an on-stack structure that holds the driver data
+ * @_end_member: The member in the struct that is the original end of struct
+ *               from the first kernel to introduce it.
+ *
+ * Check that the udata input request struct is properly formed for this kernel.
+ * Then copy it into req
+ */
+#define ib_copy_validate_udata_in(_udata, _req, _end_member)      \
+	_ib_copy_validate_udata_in(_udata, &(_req), sizeof(_req), \
+				   offsetofend(typeof(_req), _end_member))
+
+int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
+				    u64 valid_cm);
+
+/**
+ * ib_copy_validate_udata_in_cm - Copy the req structure and check the comp_mask
+ * @_udata: The system calls ib_udata struct
+ * @_req: The name of an on-stack structure that holds the driver data
+ * @_end_member: The member in the struct that is the original end of struct
+ *               from the first kernel to introduce it.
+ * @_valid_cm: A bitmask of bits permitted in the comp_mask_field.
+ *
+ * Check that the udata input request struct is properly formed for this kernel.
+ * Then copy it into req
+ */
+#define ib_copy_validate_udata_in_cm(_udata, _req, _end_member, _valid_cm)    \
+	({                                                                    \
+		typeof((_req).comp_mask) __valid_cm = _valid_cm;              \
+		int ret =                                                     \
+			ib_copy_validate_udata_in(_udata, _req, _end_member); \
+		if (!ret && ((_req).comp_mask & ~__valid_cm))                 \
+			ret = _ib_copy_validate_udata_cm_fail(                \
+				_udata, (_req).comp_mask, __valid_cm);        \
+		ret;                                                          \
+	})
+
+/**
+ * ib_is_udata_in_empty - Check if the udata input buffer is all zeros
+ * @udata: The system calls ib_udata struct
+ *
+ * This should be used if the driver does not currently define a driver data
+ * struct. Returns 0 if the buffer is empty or all zeros, -EOPNOTSUPP if
+ * non-zero data is present, or a negative error code on failure.
+ */
+static inline int ib_is_udata_in_empty(struct ib_udata *udata)
+{
+	if (!udata || udata->inlen == 0)
+		return 0;
+	return _ib_copy_validate_udata_in(udata, NULL, 0, 0);
+}
+
+/**
+ * ib_respond_udata - Copy a driver data response to userspace
+ * @_udata: The system calls ib_udata struct
+ * @_rep: Kernel buffer containing the response driver data on the stack
+ *
+ * Copy driver data response structures back to userspace in a way that
+ * is forwards and backwards compatible. Longer kernel structs are truncated,
+ * userspace has made some kind of error if it needed the truncated information.
+ * Shorter structs are zero padded.
+ */
+#define ib_respond_udata(_udata, _rep) \
+	_ib_respond_udata(_udata, &(_rep), sizeof(_rep))
+
+/**
+ * ib_respond_empty_udata - Zero fill the response buffer to userspace
+ * @_udata: The system calls ib_udata struct
+ *
+ * Used when there is no driver response data to return. Provides forward
+ * compatability by zeroing any buffer the user may have provided.
+ */
+static inline int ib_respond_empty_udata(struct ib_udata *udata)
+{
+	if (udata && udata->outlen && clear_user(udata->outbuf, udata->outlen))
+		return -EFAULT;
+	return 0;
+}
+
 #endif
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index d32f5841f4f85..9c2a7bbe5891e 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -571,6 +571,7 @@ void scsi_put_internal_cmd(struct scsi_cmnd *scmd);
 extern void sdev_disable_disk_events(struct scsi_device *sdev);
 extern void sdev_enable_disk_events(struct scsi_device *sdev);
 extern int scsi_vpd_lun_id(struct scsi_device *, char *, size_t);
+extern int scsi_vpd_lun_serial(struct scsi_device *, char *, size_t);
 extern int scsi_vpd_tpg_id(struct scsi_device *, int *);
 
 #ifdef CONFIG_PM
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index f6e12565a81de..7e2011830ba4b 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -660,6 +660,10 @@ struct Scsi_Host {
 	 */
 	unsigned nr_hw_queues;
 	unsigned nr_maps;
+
+	/* Asynchronous scan in progress */
+	bool async_scan __guarded_by(&scan_mutex);
+
 	unsigned active_mode:2;
 
 	/*
@@ -678,9 +682,6 @@ struct Scsi_Host {
 	/* Task mgmt function in progress */
 	unsigned tmf_in_progress:1;
 
-	/* Asynchronous scan in progress */
-	unsigned async_scan:1;
-
 	/* Don't resume host in EH */
 	unsigned eh_noresume:1;
 
diff --git a/include/soc/tegra/bpmp-abi.h b/include/soc/tegra/bpmp-abi.h
index dc0789c20333a..39bb3f87e28d4 100644
--- a/include/soc/tegra/bpmp-abi.h
+++ b/include/soc/tegra/bpmp-abi.h
@@ -1020,9 +1020,9 @@ struct cmd_i2c_xfer_request {
 	/**
 	 * @brief Tegra PWR_I2C bus identifier
 	 *
-	 * @cond (bpmp_t234 || bpmp_t239 || bpmp_t194)
+	 * @cond (bpmp_t234 || bpmp_t238 || bpmp_t194)
 	 * Must be set to 5.
-	 * @endcond (bpmp_t234 || bpmp_t239 || bpmp_t194)
+	 * @endcond (bpmp_t234 || bpmp_t238 || bpmp_t194)
 	 * @cond bpmp_th500
 	 * Must be set to 1.
 	 * @endcond bpmp_th500
@@ -2108,7 +2108,7 @@ struct mrq_emc_dvfs_latency_response {
  * @brief Query EMC HUB frequencies
  *
  * * Platforms: T234 onwards
- * @cond (bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: N/A
@@ -2139,7 +2139,7 @@ struct mrq_emc_dvfs_emchub_response {
 } BPMP_ABI_PACKED;
 
 /** @} EMC */
-/** @endcond (bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2147,7 +2147,7 @@ struct mrq_emc_dvfs_emchub_response {
  * @brief Set EMC display RFL handshake mode of operations
  *
  * * Platforms: T234 onwards
- * @cond (bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_emc_disp_rfl_request
@@ -2184,7 +2184,7 @@ struct mrq_emc_disp_rfl_request {
 } BPMP_ABI_PACKED;
 
 /** @} EMC */
-/** @endcond (bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2192,7 +2192,7 @@ struct mrq_emc_disp_rfl_request {
  * @brief bwmgr requests
  *
  * * Platforms: T234 onwards
- * @cond (bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_bwmgr_request
@@ -2299,7 +2299,7 @@ struct mrq_bwmgr_response {
 } BPMP_ABI_PACKED;
 
 /** @} BWMGR */
-/** @endcond (bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2307,7 +2307,7 @@ struct mrq_bwmgr_response {
  * @brief bpmp-integrated bwmgr requests
  *
  * * Platforms: T234 onwards
- * @cond (bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_bwmgr_int_request
@@ -2445,7 +2445,7 @@ struct mrq_bwmgr_int_response {
 } BPMP_ABI_PACKED;
 
 /** @} BWMGR_INT */
-/** @endcond (bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2453,7 +2453,7 @@ struct mrq_bwmgr_int_response {
  * @brief ISO client requests
  *
  * * Platforms: T234 onwards
- * @cond (bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_iso_client_request
@@ -2636,7 +2636,7 @@ struct mrq_iso_client_response {
 } BPMP_ABI_PACKED;
 
 /** @} ISO_CLIENT */
-/** @endcond (bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2644,7 +2644,7 @@ struct mrq_iso_client_response {
  * @brief CPU freq. limits in ndiv
  *
  * * Platforms: T194 onwards
- * @cond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_cpu_ndiv_limits_request
@@ -2678,7 +2678,7 @@ struct mrq_cpu_ndiv_limits_response {
 } BPMP_ABI_PACKED;
 
 /** @} CPU */
-/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2911,7 +2911,7 @@ union mrq_ringbuf_console_bpmp_to_host_response {
  * @brief Set a strap value controlled by BPMP
  *
  * * Platforms: T194 onwards
- * @cond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_strap_request
@@ -2949,7 +2949,7 @@ struct mrq_strap_request {
 } BPMP_ABI_PACKED;
 
 /** @} Strap */
-/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -2957,7 +2957,7 @@ struct mrq_strap_request {
  * @brief Perform a UPHY operation
  *
  * * Platforms: T194 onwards
- * @cond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_uphy_request
@@ -2997,23 +2997,23 @@ struct cmd_uphy_margin_status_response {
 } BPMP_ABI_PACKED;
 
 struct cmd_uphy_ep_controller_pll_init_request {
-	/** @brief EP controller number, T194 valid: 0, 4, 5; T234 valid: 5, 6, 7, 10; T239 valid: 0 */
+	/** @brief EP controller number, T194 valid: 0, 4, 5; T234 valid: 5, 6, 7, 10; T238 valid: 0 */
 	uint8_t ep_controller;
 } BPMP_ABI_PACKED;
 
 struct cmd_uphy_pcie_controller_state_request {
-	/** @brief PCIE controller number, T194 valid: 0-4; T234 valid: 0-10; T239 valid: 0-3 */
+	/** @brief PCIE controller number, T194 valid: 0-4; T234 valid: 0-10; T238 valid: 0-3 */
 	uint8_t pcie_controller;
 	uint8_t enable;
 } BPMP_ABI_PACKED;
 
 struct cmd_uphy_ep_controller_pll_off_request {
-	/** @brief EP controller number, T194 valid: 0, 4, 5; T234 valid: 5, 6, 7, 10; T239 valid: 0 */
+	/** @brief EP controller number, T194 valid: 0, 4, 5; T234 valid: 5, 6, 7, 10; T238 valid: 0 */
 	uint8_t ep_controller;
 } BPMP_ABI_PACKED;
 
 struct cmd_uphy_display_port_init_request {
-	/** @brief DisplayPort link rate, T239 valid: 1620, 2700, 5400, 8100, 2160, 2430, 3240, 4320, 6750 */
+	/** @brief DisplayPort link rate, T238 valid: 1620, 2700, 5400, 8100, 2160, 2430, 3240, 4320, 6750 */
 	uint16_t link_rate;
 	/** @brief 1: lane 0; 2: lane 1; 3: lane 0 and 1 */
 	uint16_t lanes_bitmap;
@@ -3084,7 +3084,7 @@ struct mrq_uphy_response {
 } BPMP_ABI_PACKED;
 
 /** @} UPHY */
-/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500) */
+/** @endcond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500) */
 
 /**
  * @ingroup MRQ_Codes
@@ -3092,12 +3092,12 @@ struct mrq_uphy_response {
  * @brief Perform a frequency monitor configuration operations
  *
  * * Platforms: T194 onwards
- * @cond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @cond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500)
  * * Initiators: CCPLEX
  * * Targets: BPMP
  * * Request Payload: @ref mrq_fmon_request
  * * Response Payload: @ref mrq_fmon_response
- * @endcond (bpmp_t194 || bpmp_t234 || bpmp_t239 || bpmp_th500)
+ * @endcond (bpmp_t194 || bpmp_t234 || bpmp_t238 || bpmp_th500)
  *
  * @addtogroup FMON
  * @{
diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h
index 6ee4c59db6201..e6da035d1306d 100644
--- a/include/soc/tegra/mc.h
+++ b/include/soc/tegra/mc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2014 NVIDIA Corporation
+ * Copyright (C) 2014-2026 NVIDIA Corporation
  */
 
 #ifndef __SOC_TEGRA_MC_H__
@@ -10,10 +10,11 @@
 #include <linux/debugfs.h>
 #include <linux/err.h>
 #include <linux/interconnect-provider.h>
+#include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/reset-controller.h>
-#include <linux/types.h>
 #include <linux/tegra-icc.h>
+#include <linux/types.h>
 
 struct clk;
 struct device;
@@ -164,10 +165,31 @@ struct tegra_mc_ops {
 	int (*probe)(struct tegra_mc *mc);
 	void (*remove)(struct tegra_mc *mc);
 	int (*resume)(struct tegra_mc *mc);
-	irqreturn_t (*handle_irq)(int irq, void *data);
 	int (*probe_device)(struct tegra_mc *mc, struct device *dev);
 };
 
+struct tegra_mc_regs {
+	unsigned int cfg_channel_enable;
+	unsigned int err_status;
+	unsigned int err_add;
+	unsigned int err_add_hi;
+	unsigned int err_vpr_status;
+	unsigned int err_vpr_add;
+	unsigned int err_sec_status;
+	unsigned int err_sec_add;
+	unsigned int err_mts_status;
+	unsigned int err_mts_add;
+	unsigned int err_gen_co_status;
+	unsigned int err_gen_co_add;
+	unsigned int err_route_status;
+	unsigned int err_route_add;
+};
+
+struct tegra_mc_intmask {
+	u32 reg;
+	u32 mask;
+};
+
 struct tegra_mc_soc {
 	const struct tegra_mc_client *clients;
 	unsigned int num_clients;
@@ -185,7 +207,6 @@ struct tegra_mc_soc {
 
 	const struct tegra_smmu_soc *smmu;
 
-	u32 intmask;
 	u32 ch_intmask;
 	u32 global_intstatus_channel_shift;
 	bool has_addr_hi_reg;
@@ -196,6 +217,14 @@ struct tegra_mc_soc {
 
 	const struct tegra_mc_icc_ops *icc_ops;
 	const struct tegra_mc_ops *ops;
+	const struct tegra_mc_regs *regs;
+
+	const irq_handler_t *handle_irq;
+	unsigned int num_interrupts;
+	unsigned int mc_addr_hi_mask;
+	unsigned int mc_err_status_type_mask;
+	const struct tegra_mc_intmask *intmasks;
+	unsigned int num_intmasks;
 };
 
 struct tegra_mc {
@@ -206,7 +235,6 @@ struct tegra_mc {
 	void __iomem *bcast_ch_regs;
 	void __iomem **ch_regs;
 	struct clk *clk;
-	int irq;
 
 	const struct tegra_mc_soc *soc;
 	unsigned long tick;
@@ -256,4 +284,6 @@ tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
 }
 #endif
 
+extern const struct tegra_mc_regs tegra20_mc_regs;
+
 #endif /* __SOC_TEGRA_MC_H__ */
diff --git a/include/sound/core.h b/include/sound/core.h
index 64327e9711222..4093ec82a0a12 100644
--- a/include/sound/core.h
+++ b/include/sound/core.h
@@ -133,6 +133,9 @@ struct snd_card {
 #ifdef CONFIG_SND_DEBUG
 	struct dentry *debugfs_root;    /* debugfs root for card */
 #endif
+#ifdef CONFIG_SND_CTL_DEBUG
+	struct snd_ctl_elem_value *value_buf; /* buffer for kctl->put() verification */
+#endif
 
 #ifdef CONFIG_PM
 	unsigned int power_state;	/* power state */
diff --git a/include/sound/cs35l56.h b/include/sound/cs35l56.h
index 28f9f5940ab6e..7ca25487030a1 100644
--- a/include/sound/cs35l56.h
+++ b/include/sound/cs35l56.h
@@ -32,9 +32,6 @@ struct snd_ctl_elem_value;
 #define CS35L56_UPDATE_REGS				0x0002A0C
 #define CS35L56_REFCLK_INPUT				0x0002C04
 #define CS35L56_GLOBAL_SAMPLE_RATE			0x0002C0C
-#define CS35L56_OTP_MEM_53				0x00300D4
-#define CS35L56_OTP_MEM_54				0x00300D8
-#define CS35L56_OTP_MEM_55				0x00300DC
 #define CS35L56_ASP1_ENABLES1				0x0004800
 #define CS35L56_ASP1_CONTROL1				0x0004804
 #define CS35L56_ASP1_CONTROL2				0x0004808
@@ -86,6 +83,9 @@ struct snd_ctl_elem_value;
 #define CS35L56_DIE_STS1				0x0017040
 #define CS35L56_DIE_STS2				0x0017044
 #define CS35L56_DSP_RESTRICT_STS1			0x00190F0
+#define CS35L56_OTP_MEM_53				0x00300D4
+#define CS35L56_OTP_MEM_54				0x00300D8
+#define CS35L56_OTP_MEM_55				0x00300DC
 #define CS35L56_DSP1_XMEM_PACKED_0			0x2000000
 #define CS35L56_DSP1_XMEM_PACKED_6143			0x2005FFC
 #define CS35L56_DSP1_XMEM_UNPACKED32_0			0x2400000
diff --git a/include/sound/pcm.h b/include/sound/pcm.h
index a7860c047503a..76fc33dce537d 100644
--- a/include/sound/pcm.h
+++ b/include/sound/pcm.h
@@ -729,6 +729,10 @@ static inline void __snd_pcm_set_state(struct snd_pcm_runtime *runtime,
 	runtime->status->state = state; /* copy for mmap */
 }
 
+void snd_pcm_set_state(struct snd_pcm_substream *substream,
+		       snd_pcm_state_t state);
+snd_pcm_state_t snd_pcm_get_state(struct snd_pcm_substream *substream);
+
 /**
  * bytes_to_samples - Unit conversion of the size from bytes to samples
  * @runtime: PCM runtime instance
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index 69a9c9c4d0e9c..915e6ae5f68d7 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -54,6 +54,11 @@ struct prop_nums {
 	int platforms;
 };
 
+enum simple_util_sysclk_order {
+	SIMPLE_SYSCLK_ORDER_CODEC_FIRST = 0,
+	SIMPLE_SYSCLK_ORDER_CPU_FIRST,
+};
+
 struct simple_util_priv {
 	struct snd_soc_card snd_card;
 	struct simple_dai_props {
@@ -63,6 +68,7 @@ struct simple_util_priv {
 		struct snd_soc_codec_conf *codec_conf;
 		struct prop_nums num;
 		unsigned int mclk_fs;
+		enum simple_util_sysclk_order sysclk_order;
 	} *dai_props;
 	struct simple_util_jack hp_jack;
 	struct simple_util_jack mic_jack;
diff --git a/include/sound/soc-component.h b/include/sound/soc-component.h
index 2a2b74b24a609..60f73c4b0bbbb 100644
--- a/include/sound/soc-component.h
+++ b/include/sound/soc-component.h
@@ -86,10 +86,10 @@ struct snd_soc_component_driver {
 		     unsigned int reg, unsigned int val);
 
 	/* pcm creation and destruction */
-	int (*pcm_construct)(struct snd_soc_component *component,
-			     struct snd_soc_pcm_runtime *rtd);
-	void (*pcm_destruct)(struct snd_soc_component *component,
-			     struct snd_pcm *pcm);
+	int (*pcm_new)(struct snd_soc_component *component,
+		       struct snd_soc_pcm_runtime *rtd);
+	void (*pcm_free)(struct snd_soc_component *component,
+			 struct snd_pcm *pcm);
 
 	/* component wide operations */
 	int (*set_sysclk)(struct snd_soc_component *component,
diff --git a/include/sound/soc-dai.h b/include/sound/soc-dai.h
index 224396927aef3..6a42812bba8ca 100644
--- a/include/sound/soc-dai.h
+++ b/include/sound/soc-dai.h
@@ -52,6 +52,21 @@ struct snd_compr_stream;
 #define SND_SOC_POSSIBLE_DAIFMT_AC97		(1 << SND_SOC_DAI_FORMAT_AC97)
 #define SND_SOC_POSSIBLE_DAIFMT_PDM		(1 << SND_SOC_DAI_FORMAT_PDM)
 
+/*
+ * DAI TDM slot idle modes
+ *
+ * Describes a CODEC/CPU's behaviour when not actively receiving or
+ * transmitting on a given TDM slot. NONE is undefined behaviour.
+ * Add new modes to the end.
+ */
+#define SND_SOC_DAI_TDM_IDLE_NONE	0
+#define SND_SOC_DAI_TDM_IDLE_OFF	1
+#define SND_SOC_DAI_TDM_IDLE_ZERO	2
+#define SND_SOC_DAI_TDM_IDLE_PULLDOWN	3
+#define SND_SOC_DAI_TDM_IDLE_HIZ	4
+#define SND_SOC_DAI_TDM_IDLE_PULLUP	5
+#define SND_SOC_DAI_TDM_IDLE_DRIVE_HIGH	6
+
 /*
  * DAI Clock gating.
  *
@@ -181,6 +196,10 @@ int snd_soc_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt);
 int snd_soc_dai_set_tdm_slot(struct snd_soc_dai *dai,
 	unsigned int tx_mask, unsigned int rx_mask, int slots, int slot_width);
 
+int snd_soc_dai_set_tdm_idle(struct snd_soc_dai *dai,
+			     unsigned int tx_mask, unsigned int rx_mask,
+			     int tx_mode, int rx_mode);
+
 int snd_soc_dai_set_channel_map(struct snd_soc_dai *dai,
 	unsigned int tx_num, const unsigned int *tx_slot,
 	unsigned int rx_num, const unsigned int *rx_slot);
@@ -297,6 +316,9 @@ struct snd_soc_dai_ops {
 	int (*set_tdm_slot)(struct snd_soc_dai *dai,
 		unsigned int tx_mask, unsigned int rx_mask,
 		int slots, int slot_width);
+	int (*set_tdm_idle)(struct snd_soc_dai *dai,
+			    unsigned int tx_mask, unsigned int rx_mask,
+			    int tx_mode, int rx_mode);
 	int (*set_channel_map)(struct snd_soc_dai *dai,
 		unsigned int tx_num, const unsigned int *tx_slot,
 		unsigned int rx_num, const unsigned int *rx_slot);
diff --git a/include/sound/soc-dapm.h b/include/sound/soc-dapm.h
index 49f0fe05db01f..4f8fb7622a139 100644
--- a/include/sound/soc-dapm.h
+++ b/include/sound/soc-dapm.h
@@ -424,6 +424,7 @@ enum snd_soc_dapm_type {
 	snd_soc_dapm_input = 0,		/* input pin */
 	snd_soc_dapm_output,		/* output pin */
 	snd_soc_dapm_mux,		/* selects 1 analog signal from many inputs */
+	snd_soc_dapm_mux_named_ctl,	/* mux with named controls */
 	snd_soc_dapm_demux,		/* connects the input to one of multiple outputs */
 	snd_soc_dapm_mixer,		/* mixes several analog signals together */
 	snd_soc_dapm_mixer_named_ctl,	/* mixer with named controls */
diff --git a/include/sound/soc.h b/include/sound/soc.h
index 7d8376c8e1bed..a30f95ff7d86d 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -422,11 +422,6 @@ struct snd_soc_jack_pin;
 #include <sound/soc-dpcm.h>
 #include <sound/soc-topology.h>
 
-enum snd_soc_pcm_subclass {
-	SND_SOC_PCM_CLASS_PCM	= 0,
-	SND_SOC_PCM_CLASS_BE	= 1,
-};
-
 int snd_soc_register_card(struct snd_soc_card *card);
 void snd_soc_unregister_card(struct snd_soc_card *card);
 int devm_snd_soc_register_card(struct device *dev, struct snd_soc_card *card);
@@ -465,6 +460,7 @@ struct snd_soc_component *snd_soc_lookup_component_nolocked(struct device *dev,
 							    const char *driver_name);
 struct snd_soc_component *snd_soc_lookup_component(struct device *dev,
 						   const char *driver_name);
+struct snd_soc_component *snd_soc_lookup_component_by_name(const char *component_name);
 
 int soc_new_pcm(struct snd_soc_pcm_runtime *rtd);
 #ifdef CONFIG_SND_SOC_COMPRESS
@@ -999,7 +995,6 @@ struct snd_soc_card {
 
 	/* Mutex for PCM operations */
 	struct mutex pcm_mutex;
-	enum snd_soc_pcm_subclass pcm_subclass;
 
 	int (*probe)(struct snd_soc_card *card);
 	int (*late_probe)(struct snd_soc_card *card);
@@ -1239,7 +1234,6 @@ struct soc_mixer_control {
 	unsigned int sign_bit;
 	unsigned int invert:1;
 	unsigned int autodisable:1;
-	unsigned int sdca_q78:1;
 #ifdef CONFIG_SND_SOC_TOPOLOGY
 	struct snd_soc_dobj dobj;
 #endif
@@ -1412,6 +1406,9 @@ struct snd_soc_dai *snd_soc_find_dai(
 struct snd_soc_dai *snd_soc_find_dai_with_mutex(
 	const struct snd_soc_dai_link_component *dlc);
 
+void soc_pcm_set_dai_params(struct snd_soc_dai *dai,
+			    struct snd_pcm_hw_params *params);
+
 #include <sound/soc-dai.h>
 
 static inline
@@ -1517,7 +1514,7 @@ static inline void _snd_soc_dapm_mutex_assert_held_d(struct snd_soc_dapm_context
  */
 static inline void _snd_soc_dpcm_mutex_lock_c(struct snd_soc_card *card)
 {
-	mutex_lock_nested(&card->pcm_mutex, card->pcm_subclass);
+	mutex_lock(&card->pcm_mutex);
 }
 
 static inline void _snd_soc_dpcm_mutex_unlock_c(struct snd_soc_card *card)
diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h
index 227347c8f0b33..98531e500cbb3 100644
--- a/include/sound/soc_sdw_utils.h
+++ b/include/sound/soc_sdw_utils.h
@@ -259,6 +259,8 @@ int asoc_sdw_cs42l43_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_so
 int asoc_sdw_cs42l43_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 int asoc_sdw_cs42l45_hs_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 int asoc_sdw_cs42l45_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_cs47l47_hs_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_cs47l47_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 int asoc_sdw_cs_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 int asoc_sdw_maxim_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 /* TI */
diff --git a/include/sound/timer.h b/include/sound/timer.h
index 760e132cc0cd2..83bafe70cf335 100644
--- a/include/sound/timer.h
+++ b/include/sound/timer.h
@@ -102,6 +102,7 @@ struct snd_timer_instance {
 	unsigned int slave_id;
 	struct list_head open_list;
 	struct list_head active_list;
+	struct list_head master_list;
 	struct list_head ack_list;
 	struct list_head slave_list_head;
 	struct list_head slave_active_head;
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index b62d5fcce950d..9a0e9f9e1ec44 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -111,6 +111,15 @@
 /* Peripheral Device Text Identification Information */
 #define PD_TEXT_ID_INFO_LEN			256
 
+enum target_compl_type {
+	/* Use the fabric driver's default completion type */
+	TARGET_FABRIC_DEFAULT_COMPL,
+	/* Complete from the backend calling context */
+	TARGET_DIRECT_COMPL,
+	/* Defer completion to the LIO workqueue */
+	TARGET_QUEUE_COMPL,
+};
+
 enum target_submit_type {
 	/* Use the fabric driver's default submission type */
 	TARGET_FABRIC_DEFAULT_SUBMIT,
@@ -741,6 +750,7 @@ struct se_dev_attrib {
 	u32		atomic_granularity;
 	u32		atomic_max_with_boundary;
 	u32		atomic_max_boundary;
+	u8		complete_type;
 	u8		submit_type;
 	struct se_device *da_dev;
 	struct config_group da_group;
diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h
index 3378ff9ee271c..e9039e73d058a 100644
--- a/include/target/target_core_fabric.h
+++ b/include/target/target_core_fabric.h
@@ -118,15 +118,21 @@ struct target_core_fabric_ops {
 	 * its entirety before a command is aborted.
 	 */
 	unsigned int write_pending_must_be_called:1;
+	/*
+	 * Set this if the driver does not require calling queue_data_in
+	 * queue_status and check_stop_free from a worker thread when
+	 * completing successful commands.
+	 */
+	unsigned int direct_compl_supp:1;
 	/*
 	 * Set this if the driver supports submitting commands to the backend
 	 * from target_submit/target_submit_cmd.
 	 */
 	unsigned int direct_submit_supp:1;
-	/*
-	 * Set this to a target_submit_type value.
-	 */
+	/* Set this to a target_submit_type value. */
 	u8 default_submit_type;
+	/* Set this to the target_compl_type value. */
+	u8 default_compl_type;
 };
 
 int target_register_template(const struct target_core_fabric_ops *fo);
diff --git a/include/trace/define_remote_events.h b/include/trace/define_remote_events.h
new file mode 100644
index 0000000000000..676e803dc144d
--- /dev/null
+++ b/include/trace/define_remote_events.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/trace_events.h>
+#include <linux/trace_remote_event.h>
+#include <linux/trace_seq.h>
+#include <linux/stringify.h>
+
+#define REMOTE_EVENT_INCLUDE(__file) __stringify(../../__file)
+
+#ifdef REMOTE_EVENT_SECTION
+# define __REMOTE_EVENT_SECTION(__name) __used __section(REMOTE_EVENT_SECTION"."#__name)
+#else
+# define __REMOTE_EVENT_SECTION(__name)
+#endif
+
+#define REMOTE_PRINTK_COUNT_ARGS(__args...) \
+	__COUNT_ARGS(, ##__args, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0)
+
+#define __remote_printk0()								\
+	trace_seq_putc(seq, '\n')
+
+#define __remote_printk1(__fmt)								\
+	trace_seq_puts(seq, " " __fmt "\n")						\
+
+#define __remote_printk2(__fmt, __args...)						\
+do {											\
+	trace_seq_putc(seq, ' ');							\
+	trace_seq_printf(seq, __fmt, __args);						\
+	trace_seq_putc(seq, '\n');							\
+} while (0)
+
+/* Apply the appropriate trace_seq sequence according to the number of arguments */
+#define remote_printk(__args...)							\
+	CONCATENATE(__remote_printk, REMOTE_PRINTK_COUNT_ARGS(__args))(__args)
+
+#define RE_PRINTK(__args...) __args
+
+#define REMOTE_EVENT(__name, __id, __struct, __printk)					\
+	REMOTE_EVENT_FORMAT(__name, __struct);						\
+	static void remote_event_print_##__name(void *evt, struct trace_seq *seq)	\
+	{										\
+		struct remote_event_format_##__name __maybe_unused *__entry = evt;	\
+		trace_seq_puts(seq, #__name);						\
+		remote_printk(__printk);						\
+	}
+#include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE)
+
+#undef REMOTE_EVENT
+#undef RE_PRINTK
+#undef re_field
+#define re_field(__type, __field)							\
+	{										\
+		.type = #__type, .name = #__field,					\
+		.size = sizeof(__type), .align = __alignof__(__type),			\
+		.is_signed = is_signed_type(__type),					\
+	},
+#define __entry REC
+#define RE_PRINTK(__fmt, __args...) "\"" __fmt "\", " __stringify(__args)
+#define REMOTE_EVENT(__name, __id, __struct, __printk)					\
+	static struct trace_event_fields remote_event_fields_##__name[] = {		\
+		__struct								\
+		{}									\
+	};										\
+	static char remote_event_print_fmt_##__name[] = __printk;			\
+	static struct remote_event __REMOTE_EVENT_SECTION(__name)			\
+	remote_event_##__name = {							\
+		.name		= #__name,						\
+		.id		= __id,							\
+		.fields		= remote_event_fields_##__name,				\
+		.print_fmt	= remote_event_print_fmt_##__name,			\
+		.print		= remote_event_print_##__name,				\
+	}
+#include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE)
diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h
index 125bdc166bfed..abf1b73ee60fc 100644
--- a/include/trace/events/btrfs.h
+++ b/include/trace/events/btrfs.h
@@ -1110,6 +1110,30 @@ TRACE_EVENT(btrfs_cow_block,
 		  __entry->cow_level)
 );
 
+TRACE_EVENT(btrfs_search_slot_restart,
+
+	TP_PROTO(const struct btrfs_root *root, int level,
+		 const char *reason),
+
+	TP_ARGS(root, level, reason),
+
+	TP_STRUCT__entry_btrfs(
+		__field(	u64,	root_objectid		)
+		__field(	int,	level			)
+		__string(	reason,	reason			)
+	),
+
+	TP_fast_assign_btrfs(root->fs_info,
+		__entry->root_objectid	= btrfs_root_id(root);
+		__entry->level		= level;
+		__assign_str(reason);
+	),
+
+	TP_printk_btrfs("root=%llu(%s) level=%d reason=%s",
+		  show_root_type(__entry->root_objectid),
+		  __entry->level, __get_str(reason))
+);
+
 TRACE_EVENT(btrfs_space_reservation,
 
 	TP_PROTO(const struct btrfs_fs_info *fs_info, const char *type, u64 val,
diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h
index a743b2a35ea70..6e3b1424eea4d 100644
--- a/include/trace/events/cachefiles.h
+++ b/include/trace/events/cachefiles.h
@@ -249,10 +249,10 @@ TRACE_EVENT(cachefiles_lookup,
 	    TP_ARGS(obj, dir, de),
 
 	    TP_STRUCT__entry(
+		    __field(u64,			dino)
+		    __field(u64,			ino)
 		    __field(unsigned int,		obj)
 		    __field(short,			error)
-		    __field(unsigned long,		dino)
-		    __field(unsigned long,		ino)
 			     ),
 
 	    TP_fast_assign(
@@ -263,7 +263,7 @@ TRACE_EVENT(cachefiles_lookup,
 		    __entry->error	= IS_ERR(de) ? PTR_ERR(de) : 0;
 			   ),
 
-	    TP_printk("o=%08x dB=%lx B=%lx e=%d",
+	    TP_printk("o=%08x dB=%llx B=%llx e=%d",
 		      __entry->obj, __entry->dino, __entry->ino, __entry->error)
 	    );
 
@@ -578,8 +578,8 @@ TRACE_EVENT(cachefiles_mark_active,
 
 	    /* Note that obj may be NULL */
 	    TP_STRUCT__entry(
+		    __field(u64,			inode)
 		    __field(unsigned int,		obj)
-		    __field(ino_t,			inode)
 			     ),
 
 	    TP_fast_assign(
@@ -587,7 +587,7 @@ TRACE_EVENT(cachefiles_mark_active,
 		    __entry->inode	= inode->i_ino;
 			   ),
 
-	    TP_printk("o=%08x B=%lx",
+	    TP_printk("o=%08x B=%llx",
 		      __entry->obj, __entry->inode)
 	    );
 
@@ -599,8 +599,8 @@ TRACE_EVENT(cachefiles_mark_failed,
 
 	    /* Note that obj may be NULL */
 	    TP_STRUCT__entry(
+		    __field(u64,			inode)
 		    __field(unsigned int,		obj)
-		    __field(ino_t,			inode)
 			     ),
 
 	    TP_fast_assign(
@@ -608,7 +608,7 @@ TRACE_EVENT(cachefiles_mark_failed,
 		    __entry->inode	= inode->i_ino;
 			   ),
 
-	    TP_printk("o=%08x B=%lx",
+	    TP_printk("o=%08x B=%llx",
 		      __entry->obj, __entry->inode)
 	    );
 
@@ -620,8 +620,8 @@ TRACE_EVENT(cachefiles_mark_inactive,
 
 	    /* Note that obj may be NULL */
 	    TP_STRUCT__entry(
+		    __field(u64,			inode)
 		    __field(unsigned int,		obj)
-		    __field(ino_t,			inode)
 			     ),
 
 	    TP_fast_assign(
@@ -629,7 +629,7 @@ TRACE_EVENT(cachefiles_mark_inactive,
 		    __entry->inode	= inode->i_ino;
 			   ),
 
-	    TP_printk("o=%08x B=%lx",
+	    TP_printk("o=%08x B=%llx",
 		      __entry->obj, __entry->inode)
 	    );
 
diff --git a/include/trace/events/devlink.h b/include/trace/events/devlink.h
index f241e204fe6b9..4f8edf77dfbe8 100644
--- a/include/trace/events/devlink.h
+++ b/include/trace/events/devlink.h
@@ -21,9 +21,9 @@ TRACE_EVENT(devlink_hwmsg,
 	TP_ARGS(devlink, incoming, type, buf, len),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__field(bool, incoming)
 		__field(unsigned long, type)
 		__dynamic_array(u8, buf, len)
@@ -55,9 +55,9 @@ TRACE_EVENT(devlink_hwerr,
 	TP_ARGS(devlink, err, msg),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__field(int, err)
 		__string(msg, msg)
 		),
@@ -85,9 +85,9 @@ TRACE_EVENT(devlink_health_report,
 	TP_ARGS(devlink, reporter_name, msg),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__string(reporter_name, reporter_name)
 		__string(msg, msg)
 	),
@@ -116,9 +116,9 @@ TRACE_EVENT(devlink_health_recover_aborted,
 	TP_ARGS(devlink, reporter_name, health_state, time_since_last_recover),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__string(reporter_name, reporter_name)
 		__field(bool, health_state)
 		__field(u64, time_since_last_recover)
@@ -150,9 +150,9 @@ TRACE_EVENT(devlink_health_reporter_state_update,
 	TP_ARGS(devlink, reporter_name, new_state),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__string(reporter_name, reporter_name)
 		__field(u8, new_state)
 	),
@@ -181,9 +181,9 @@ TRACE_EVENT(devlink_trap_report,
 	TP_ARGS(devlink, skb, metadata),
 
 	TP_STRUCT__entry(
-		__string(bus_name, devlink_to_dev(devlink)->bus->name)
-		__string(dev_name, dev_name(devlink_to_dev(devlink)))
-		__string(driver_name, devlink_to_dev(devlink)->driver->name)
+		__string(bus_name, devlink_bus_name(devlink))
+		__string(dev_name, devlink_dev_name(devlink))
+		__string(driver_name, devlink_dev_driver_name(devlink))
 		__string(trap_name, metadata->trap_name)
 		__string(trap_group_name, metadata->trap_group_name)
 		__array(char, input_dev_name, IFNAMSIZ)
diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h
index 33e99e792f1aa..63597b0044247 100644
--- a/include/trace/events/dma.h
+++ b/include/trace/events/dma.h
@@ -32,7 +32,9 @@ TRACE_DEFINE_ENUM(DMA_NONE);
 		{ DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \
 		{ DMA_ATTR_NO_WARN, "NO_WARN" }, \
 		{ DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \
-		{ DMA_ATTR_MMIO, "MMIO" })
+		{ DMA_ATTR_MMIO, "MMIO" }, \
+		{ DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \
+		{ DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" })
 
 DECLARE_EVENT_CLASS(dma_map,
 	TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
diff --git a/include/trace/events/dma_fence.h b/include/trace/events/dma_fence.h
index 4814a65b68dcb..3abba45c0601a 100644
--- a/include/trace/events/dma_fence.h
+++ b/include/trace/events/dma_fence.h
@@ -9,37 +9,12 @@
 
 struct dma_fence;
 
-DECLARE_EVENT_CLASS(dma_fence,
-
-	TP_PROTO(struct dma_fence *fence),
-
-	TP_ARGS(fence),
-
-	TP_STRUCT__entry(
-		__string(driver, dma_fence_driver_name(fence))
-		__string(timeline, dma_fence_timeline_name(fence))
-		__field(unsigned int, context)
-		__field(unsigned int, seqno)
-	),
-
-	TP_fast_assign(
-		__assign_str(driver);
-		__assign_str(timeline);
-		__entry->context = fence->context;
-		__entry->seqno = fence->seqno;
-	),
-
-	TP_printk("driver=%s timeline=%s context=%u seqno=%u",
-		  __get_str(driver), __get_str(timeline), __entry->context,
-		  __entry->seqno)
-);
-
 /*
  * Safe only for call sites which are guaranteed to not race with fence
  * signaling,holding the fence->lock and having checked for not signaled, or the
  * signaling path itself.
  */
-DECLARE_EVENT_CLASS(dma_fence_unsignaled,
+DECLARE_EVENT_CLASS(dma_fence,
 
 	TP_PROTO(struct dma_fence *fence),
 
@@ -64,14 +39,14 @@ DECLARE_EVENT_CLASS(dma_fence_unsignaled,
 		  __entry->seqno)
 );
 
-DEFINE_EVENT(dma_fence_unsignaled, dma_fence_emit,
+DEFINE_EVENT(dma_fence, dma_fence_emit,
 
 	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence_unsignaled, dma_fence_init,
+DEFINE_EVENT(dma_fence, dma_fence_init,
 
 	TP_PROTO(struct dma_fence *fence),
 
@@ -85,14 +60,14 @@ DEFINE_EVENT(dma_fence, dma_fence_destroy,
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence_unsignaled, dma_fence_enable_signal,
+DEFINE_EVENT(dma_fence, dma_fence_enable_signal,
 
 	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence_unsignaled, dma_fence_signaled,
+DEFINE_EVENT(dma_fence, dma_fence_signaled,
 
 	TP_PROTO(struct dma_fence *fence),
 
diff --git a/include/trace/events/ext4.h b/include/trace/events/ext4.h
index a3e8fe414df85..f493642cf1219 100644
--- a/include/trace/events/ext4.h
+++ b/include/trace/events/ext4.h
@@ -138,14 +138,14 @@ TRACE_DEFINE_ENUM(CR_ANY_FREE);
 			 { CR_ANY_FREE, "CR_ANY_FREE" })
 
 TRACE_EVENT(ext4_other_inode_update_time,
-	TP_PROTO(struct inode *inode, ino_t orig_ino),
+	TP_PROTO(struct inode *inode, u64 orig_ino),
 
 	TP_ARGS(inode, orig_ino),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	u64,	orig_ino		)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
-		__field(	ino_t,	orig_ino		)
 		__field(	uid_t,	uid			)
 		__field(	gid_t,	gid			)
 		__field(	__u16, mode			)
@@ -160,10 +160,10 @@ TRACE_EVENT(ext4_other_inode_update_time,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d orig_ino %lu ino %lu mode 0%o uid %u gid %u",
+	TP_printk("dev %d,%d orig_ino %llu ino %llu mode 0%o uid %u gid %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->orig_ino,
-		  (unsigned long) __entry->ino, __entry->mode,
+		  __entry->orig_ino,
+		  __entry->ino, __entry->mode,
 		  __entry->uid, __entry->gid)
 );
 
@@ -173,11 +173,11 @@ TRACE_EVENT(ext4_free_inode,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	__u64, blocks			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	uid_t,	uid			)
 		__field(	gid_t,	gid			)
-		__field(	__u64, blocks			)
 		__field(	__u16, mode			)
 	),
 
@@ -190,9 +190,9 @@ TRACE_EVENT(ext4_free_inode,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o uid %u gid %u blocks %llu",
+	TP_printk("dev %d,%d ino %llu mode 0%o uid %u gid %u blocks %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->mode,
+		  __entry->ino, __entry->mode,
 		  __entry->uid, __entry->gid, __entry->blocks)
 );
 
@@ -202,8 +202,8 @@ TRACE_EVENT(ext4_request_inode,
 	TP_ARGS(dir, mode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	dir			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	dir			)
 		__field(	__u16, mode			)
 	),
 
@@ -213,9 +213,9 @@ TRACE_EVENT(ext4_request_inode,
 		__entry->mode	= mode;
 	),
 
-	TP_printk("dev %d,%d dir %lu mode 0%o",
+	TP_printk("dev %d,%d dir %llu mode 0%o",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->dir, __entry->mode)
+		  __entry->dir, __entry->mode)
 );
 
 TRACE_EVENT(ext4_allocate_inode,
@@ -224,9 +224,9 @@ TRACE_EVENT(ext4_allocate_inode,
 	TP_ARGS(inode, dir, mode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	u64,	dir			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
-		__field(	ino_t,	dir			)
 		__field(	__u16,	mode			)
 	),
 
@@ -237,10 +237,10 @@ TRACE_EVENT(ext4_allocate_inode,
 		__entry->mode	= mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu dir %lu mode 0%o",
+	TP_printk("dev %d,%d ino %llu dir %llu mode 0%o",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
-		  (unsigned long) __entry->dir, __entry->mode)
+		  __entry->ino,
+		  __entry->dir, __entry->mode)
 );
 
 TRACE_EVENT(ext4_evict_inode,
@@ -249,8 +249,8 @@ TRACE_EVENT(ext4_evict_inode,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	int,	nlink			)
 	),
 
@@ -260,9 +260,9 @@ TRACE_EVENT(ext4_evict_inode,
 		__entry->nlink	= inode->i_nlink;
 	),
 
-	TP_printk("dev %d,%d ino %lu nlink %d",
+	TP_printk("dev %d,%d ino %llu nlink %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->nlink)
+		  __entry->ino, __entry->nlink)
 );
 
 TRACE_EVENT(ext4_drop_inode,
@@ -271,8 +271,8 @@ TRACE_EVENT(ext4_drop_inode,
 	TP_ARGS(inode, drop),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	int,	drop			)
 	),
 
@@ -282,9 +282,9 @@ TRACE_EVENT(ext4_drop_inode,
 		__entry->drop	= drop;
 	),
 
-	TP_printk("dev %d,%d ino %lu drop %d",
+	TP_printk("dev %d,%d ino %llu drop %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->drop)
+		  __entry->ino, __entry->drop)
 );
 
 TRACE_EVENT(ext4_nfs_commit_metadata,
@@ -293,8 +293,8 @@ TRACE_EVENT(ext4_nfs_commit_metadata,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 	),
 
 	TP_fast_assign(
@@ -302,9 +302,9 @@ TRACE_EVENT(ext4_nfs_commit_metadata,
 		__entry->ino	= inode->i_ino;
 	),
 
-	TP_printk("dev %d,%d ino %lu",
+	TP_printk("dev %d,%d ino %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino)
+		  __entry->ino)
 );
 
 TRACE_EVENT(ext4_mark_inode_dirty,
@@ -313,9 +313,9 @@ TRACE_EVENT(ext4_mark_inode_dirty,
 	TP_ARGS(inode, IP),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(unsigned long,	ip			)
+		__field(	dev_t,	dev			)
 	),
 
 	TP_fast_assign(
@@ -324,9 +324,9 @@ TRACE_EVENT(ext4_mark_inode_dirty,
 		__entry->ip	= IP;
 	),
 
-	TP_printk("dev %d,%d ino %lu caller %pS",
+	TP_printk("dev %d,%d ino %llu caller %pS",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, (void *)__entry->ip)
+		  __entry->ino, (void *)__entry->ip)
 );
 
 TRACE_EVENT(ext4_begin_ordered_truncate,
@@ -335,9 +335,9 @@ TRACE_EVENT(ext4_begin_ordered_truncate,
 	TP_ARGS(inode, new_size),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	new_size		)
+		__field(	dev_t,	dev			)
 	),
 
 	TP_fast_assign(
@@ -346,9 +346,9 @@ TRACE_EVENT(ext4_begin_ordered_truncate,
 		__entry->new_size	= new_size;
 	),
 
-	TP_printk("dev %d,%d ino %lu new_size %lld",
+	TP_printk("dev %d,%d ino %llu new_size %lld",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->new_size)
 );
 
@@ -359,9 +359,9 @@ DECLARE_EVENT_CLASS(ext4__write_begin,
 	TP_ARGS(inode, pos, len),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	pos			)
+		__field(	dev_t,	dev			)
 		__field(	unsigned int, len		)
 	),
 
@@ -372,9 +372,9 @@ DECLARE_EVENT_CLASS(ext4__write_begin,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev %d,%d ino %lu pos %lld len %u",
+	TP_printk("dev %d,%d ino %llu pos %lld len %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->pos, __entry->len)
 );
 
@@ -399,9 +399,9 @@ DECLARE_EVENT_CLASS(ext4__write_end,
 	TP_ARGS(inode, pos, len, copied),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	pos			)
+		__field(	dev_t,	dev			)
 		__field(	unsigned int, len		)
 		__field(	unsigned int, copied		)
 	),
@@ -414,9 +414,9 @@ DECLARE_EVENT_CLASS(ext4__write_end,
 		__entry->copied	= copied;
 	),
 
-	TP_printk("dev %d,%d ino %lu pos %lld len %u copied %u",
+	TP_printk("dev %d,%d ino %llu pos %lld len %u copied %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->pos, __entry->len, __entry->copied)
 );
 
@@ -450,13 +450,13 @@ TRACE_EVENT(ext4_writepages,
 	TP_ARGS(inode, wbc),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
-		__field(	long,	nr_to_write		)
-		__field(	long,	pages_skipped		)
+		__field(	u64,	ino			)
 		__field(	loff_t,	range_start		)
 		__field(	loff_t,	range_end		)
+		__field(	long,	nr_to_write		)
+		__field(	long,	pages_skipped		)
 		__field(       pgoff_t,	writeback_index		)
+		__field(	dev_t,	dev			)
 		__field(	int,	sync_mode		)
 		__field(	char,	for_kupdate		)
 		__field(	char,	range_cyclic		)
@@ -475,11 +475,11 @@ TRACE_EVENT(ext4_writepages,
 		__entry->range_cyclic	= wbc->range_cyclic;
 	),
 
-	TP_printk("dev %d,%d ino %lu nr_to_write %ld pages_skipped %ld "
+	TP_printk("dev %d,%d ino %llu nr_to_write %ld pages_skipped %ld "
 		  "range_start %lld range_end %lld sync_mode %d "
 		  "for_kupdate %d range_cyclic %d writeback_index %lu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->nr_to_write,
+		  __entry->ino, __entry->nr_to_write,
 		  __entry->pages_skipped, __entry->range_start,
 		  __entry->range_end, __entry->sync_mode,
 		  __entry->for_kupdate, __entry->range_cyclic,
@@ -493,11 +493,11 @@ TRACE_EVENT(ext4_da_write_folios_start,
 	TP_ARGS(inode, start_pos, next_pos, wbc),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(       loff_t,	start_pos		)
 		__field(       loff_t,	next_pos		)
 		__field(	 long,	nr_to_write		)
+		__field(	dev_t,	dev			)
 		__field(	  int,	sync_mode		)
 	),
 
@@ -510,9 +510,9 @@ TRACE_EVENT(ext4_da_write_folios_start,
 		__entry->sync_mode	= wbc->sync_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld sync_mode %d",
+	TP_printk("dev %d,%d ino %llu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld sync_mode %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->start_pos, __entry->next_pos,
+		  __entry->ino, __entry->start_pos, __entry->next_pos,
 		  __entry->nr_to_write, __entry->sync_mode)
 );
 
@@ -523,11 +523,11 @@ TRACE_EVENT(ext4_da_write_folios_end,
 	TP_ARGS(inode, start_pos, next_pos, wbc, ret),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(       loff_t,	start_pos		)
 		__field(       loff_t,	next_pos		)
 		__field(	 long,	nr_to_write		)
+		__field(	dev_t,	dev			)
 		__field(	  int,	ret			)
 	),
 
@@ -540,9 +540,9 @@ TRACE_EVENT(ext4_da_write_folios_end,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld ret %d",
+	TP_printk("dev %d,%d ino %llu start_pos 0x%llx next_pos 0x%llx nr_to_write %ld ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->start_pos, __entry->next_pos,
+		  __entry->ino, __entry->start_pos, __entry->next_pos,
 		  __entry->nr_to_write, __entry->ret)
 );
 
@@ -552,9 +552,9 @@ TRACE_EVENT(ext4_da_write_pages_extent,
 	TP_ARGS(inode, map),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	lblk			)
+		__field(	dev_t,	dev			)
 		__field(	__u32,	len			)
 		__field(	__u32,	flags			)
 	),
@@ -567,9 +567,9 @@ TRACE_EVENT(ext4_da_write_pages_extent,
 		__entry->flags		= map->m_flags;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %llu len %u flags %s",
+	TP_printk("dev %d,%d ino %llu lblk %llu len %u flags %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->lblk, __entry->len,
+		  __entry->ino, __entry->lblk, __entry->len,
 		  show_mflags(__entry->flags))
 );
 
@@ -580,12 +580,12 @@ TRACE_EVENT(ext4_writepages_result,
 	TP_ARGS(inode, wbc, ret, pages_written),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	long,	pages_skipped		)
+		__field(       pgoff_t,	writeback_index		)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	int,	ret			)
 		__field(	int,	pages_written		)
-		__field(	long,	pages_skipped		)
-		__field(       pgoff_t,	writeback_index		)
 		__field(	int,	sync_mode		)
 	),
 
@@ -599,10 +599,10 @@ TRACE_EVENT(ext4_writepages_result,
 		__entry->sync_mode	= wbc->sync_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu ret %d pages_written %d pages_skipped %ld "
+	TP_printk("dev %d,%d ino %llu ret %d pages_written %d pages_skipped %ld "
 		  "sync_mode %d writeback_index %lu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->ret,
+		  __entry->ino, __entry->ret,
 		  __entry->pages_written, __entry->pages_skipped,
 		  __entry->sync_mode,
 		  (unsigned long) __entry->writeback_index)
@@ -614,9 +614,9 @@ DECLARE_EVENT_CLASS(ext4__folio_op,
 	TP_ARGS(inode, folio),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	pgoff_t, index			)
+		__field(	dev_t,	dev			)
 
 	),
 
@@ -626,9 +626,9 @@ DECLARE_EVENT_CLASS(ext4__folio_op,
 		__entry->index	= folio->index;
 	),
 
-	TP_printk("dev %d,%d ino %lu folio_index %lu",
+	TP_printk("dev %d,%d ino %llu folio_index %lu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned long) __entry->index)
 );
 
@@ -652,11 +652,11 @@ DECLARE_EVENT_CLASS(ext4_invalidate_folio_op,
 	TP_ARGS(folio, offset, length),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	pgoff_t, index			)
 		__field(	size_t, offset			)
 		__field(	size_t, length			)
+		__field(	dev_t,	dev			)
 	),
 
 	TP_fast_assign(
@@ -667,9 +667,9 @@ DECLARE_EVENT_CLASS(ext4_invalidate_folio_op,
 		__entry->length	= length;
 	),
 
-	TP_printk("dev %d,%d ino %lu folio_index %lu offset %zu length %zu",
+	TP_printk("dev %d,%d ino %llu folio_index %lu offset %zu length %zu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned long) __entry->index,
 		  __entry->offset, __entry->length)
 );
@@ -717,10 +717,10 @@ DECLARE_EVENT_CLASS(ext4__mb_new_pa,
 	TP_ARGS(ac, pa),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	pa_pstart		)
 		__field(	__u64,	pa_lstart		)
+		__field(	dev_t,	dev			)
 		__field(	__u32,	pa_len			)
 
 	),
@@ -733,9 +733,9 @@ DECLARE_EVENT_CLASS(ext4__mb_new_pa,
 		__entry->pa_len		= pa->pa_len;
 	),
 
-	TP_printk("dev %d,%d ino %lu pstart %llu len %u lstart %llu",
+	TP_printk("dev %d,%d ino %llu pstart %llu len %u lstart %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->pa_pstart, __entry->pa_len, __entry->pa_lstart)
 );
 
@@ -762,9 +762,9 @@ TRACE_EVENT(ext4_mb_release_inode_pa,
 	TP_ARGS(pa, block, count),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	block			)
+		__field(	dev_t,	dev			)
 		__field(	__u32,	count			)
 
 	),
@@ -776,9 +776,9 @@ TRACE_EVENT(ext4_mb_release_inode_pa,
 		__entry->count		= count;
 	),
 
-	TP_printk("dev %d,%d ino %lu block %llu count %u",
+	TP_printk("dev %d,%d ino %llu block %llu count %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->block, __entry->count)
 );
 
@@ -811,8 +811,8 @@ TRACE_EVENT(ext4_discard_preallocations,
 	TP_ARGS(inode, len),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	unsigned int,	len		)
 
 	),
@@ -823,9 +823,9 @@ TRACE_EVENT(ext4_discard_preallocations,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev %d,%d ino %lu len: %u",
+	TP_printk("dev %d,%d ino %llu len: %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->len)
+		  __entry->ino, __entry->len)
 );
 
 TRACE_EVENT(ext4_mb_discard_preallocations,
@@ -855,15 +855,15 @@ TRACE_EVENT(ext4_request_blocks,
 	TP_ARGS(ar),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	__u64,	goal			)
+		__field(	__u64,	pleft			)
+		__field(	__u64,	pright			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	unsigned int, len		)
 		__field(	__u32,  logical			)
 		__field(	__u32,	lleft			)
 		__field(	__u32,	lright			)
-		__field(	__u64,	goal			)
-		__field(	__u64,	pleft			)
-		__field(	__u64,	pright			)
 		__field(	unsigned int, flags		)
 	),
 
@@ -880,10 +880,10 @@ TRACE_EVENT(ext4_request_blocks,
 		__entry->flags	= ar->flags;
 	),
 
-	TP_printk("dev %d,%d ino %lu flags %s len %u lblk %u goal %llu "
+	TP_printk("dev %d,%d ino %llu flags %s len %u lblk %u goal %llu "
 		  "lleft %u lright %u pleft %llu pright %llu ",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, show_mballoc_flags(__entry->flags),
+		  __entry->ino, show_mballoc_flags(__entry->flags),
 		  __entry->len, __entry->logical, __entry->goal,
 		  __entry->lleft, __entry->lright, __entry->pleft,
 		  __entry->pright)
@@ -895,16 +895,16 @@ TRACE_EVENT(ext4_allocate_blocks,
 	TP_ARGS(ar, block),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	block			)
+		__field(	__u64,	goal			)
+		__field(	__u64,	pleft			)
+		__field(	__u64,	pright			)
+		__field(	dev_t,	dev			)
 		__field(	unsigned int, len		)
 		__field(	__u32,  logical			)
 		__field(	__u32,	lleft			)
 		__field(	__u32,	lright			)
-		__field(	__u64,	goal			)
-		__field(	__u64,	pleft			)
-		__field(	__u64,	pright			)
 		__field(	unsigned int, flags		)
 	),
 
@@ -922,10 +922,10 @@ TRACE_EVENT(ext4_allocate_blocks,
 		__entry->flags	= ar->flags;
 	),
 
-	TP_printk("dev %d,%d ino %lu flags %s len %u block %llu lblk %u "
+	TP_printk("dev %d,%d ino %llu flags %s len %u block %llu lblk %u "
 		  "goal %llu lleft %u lright %u pleft %llu pright %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, show_mballoc_flags(__entry->flags),
+		  __entry->ino, show_mballoc_flags(__entry->flags),
 		  __entry->len, __entry->block, __entry->logical,
 		  __entry->goal,  __entry->lleft, __entry->lright,
 		  __entry->pleft, __entry->pright)
@@ -938,10 +938,10 @@ TRACE_EVENT(ext4_free_blocks,
 	TP_ARGS(inode, block, count, flags),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	block			)
 		__field(	unsigned long,	count		)
+		__field(	dev_t,	dev			)
 		__field(	int,	flags			)
 		__field(	__u16,	mode			)
 	),
@@ -955,9 +955,9 @@ TRACE_EVENT(ext4_free_blocks,
 		__entry->mode		= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o block %llu count %lu flags %s",
+	TP_printk("dev %d,%d ino %llu mode 0%o block %llu count %lu flags %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->mode, __entry->block, __entry->count,
 		  show_free_flags(__entry->flags))
 );
@@ -968,9 +968,9 @@ TRACE_EVENT(ext4_sync_file_enter,
 	TP_ARGS(file, datasync),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
+		__field(	u64,	parent			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
-		__field(	ino_t,	parent			)
 		__field(	int,	datasync		)
 	),
 
@@ -983,10 +983,10 @@ TRACE_EVENT(ext4_sync_file_enter,
 		__entry->parent		= d_inode(dentry->d_parent)->i_ino;
 	),
 
-	TP_printk("dev %d,%d ino %lu parent %lu datasync %d ",
+	TP_printk("dev %d,%d ino %llu parent %llu datasync %d ",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
-		  (unsigned long) __entry->parent, __entry->datasync)
+		  __entry->ino,
+		  __entry->parent, __entry->datasync)
 );
 
 TRACE_EVENT(ext4_sync_file_exit,
@@ -995,8 +995,8 @@ TRACE_EVENT(ext4_sync_file_exit,
 	TP_ARGS(inode, ret),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	int,	ret			)
 	),
 
@@ -1006,9 +1006,9 @@ TRACE_EVENT(ext4_sync_file_exit,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu ret %d",
+	TP_printk("dev %d,%d ino %llu ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->ret)
 );
 
@@ -1039,8 +1039,8 @@ TRACE_EVENT(ext4_alloc_da_blocks,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field( unsigned int,	data_blocks		)
 	),
 
@@ -1050,9 +1050,9 @@ TRACE_EVENT(ext4_alloc_da_blocks,
 		__entry->data_blocks = EXT4_I(inode)->i_reserved_data_blocks;
 	),
 
-	TP_printk("dev %d,%d ino %lu reserved_data_blocks %u",
+	TP_printk("dev %d,%d ino %llu reserved_data_blocks %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->data_blocks)
 );
 
@@ -1062,8 +1062,8 @@ TRACE_EVENT(ext4_mballoc_alloc,
 	TP_ARGS(ac),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	__u32, 	orig_logical		)
 		__field(	  int,	orig_start		)
 		__field(	__u32, 	orig_group		)
@@ -1107,11 +1107,11 @@ TRACE_EVENT(ext4_mballoc_alloc,
 		__entry->cr		= ac->ac_criteria;
 	),
 
-	TP_printk("dev %d,%d inode %lu orig %u/%d/%u@%u goal %u/%d/%u@%u "
+	TP_printk("dev %d,%d inode %llu orig %u/%d/%u@%u goal %u/%d/%u@%u "
 		  "result %u/%d/%u@%u blks %u grps %u cr %s flags %s "
 		  "tail %u broken %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->orig_group, __entry->orig_start,
 		  __entry->orig_len, __entry->orig_logical,
 		  __entry->goal_group, __entry->goal_start,
@@ -1129,8 +1129,8 @@ TRACE_EVENT(ext4_mballoc_prealloc,
 	TP_ARGS(ac),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	__u32, 	orig_logical		)
 		__field(	  int,	orig_start		)
 		__field(	__u32, 	orig_group		)
@@ -1154,9 +1154,9 @@ TRACE_EVENT(ext4_mballoc_prealloc,
 		__entry->result_len	= ac->ac_b_ex.fe_len;
 	),
 
-	TP_printk("dev %d,%d inode %lu orig %u/%d/%u@%u result %u/%d/%u@%u",
+	TP_printk("dev %d,%d inode %llu orig %u/%d/%u@%u result %u/%d/%u@%u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->orig_group, __entry->orig_start,
 		  __entry->orig_len, __entry->orig_logical,
 		  __entry->result_group, __entry->result_start,
@@ -1173,8 +1173,8 @@ DECLARE_EVENT_CLASS(ext4__mballoc,
 	TP_ARGS(sb, inode, group, start, len),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	  int,	result_start		)
 		__field(	__u32, 	result_group		)
 		__field(	  int,	result_len		)
@@ -1188,9 +1188,9 @@ DECLARE_EVENT_CLASS(ext4__mballoc,
 		__entry->result_len	= len;
 	),
 
-	TP_printk("dev %d,%d inode %lu extent %u/%d/%d ",
+	TP_printk("dev %d,%d inode %llu extent %u/%d/%d ",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->result_group, __entry->result_start,
 		  __entry->result_len)
 );
@@ -1223,9 +1223,9 @@ TRACE_EVENT(ext4_forget,
 	TP_ARGS(inode, is_metadata, block),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	block			)
+		__field(	dev_t,	dev			)
 		__field(	int,	is_metadata		)
 		__field(	__u16,	mode			)
 	),
@@ -1238,9 +1238,9 @@ TRACE_EVENT(ext4_forget,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o is_metadata %d block %llu",
+	TP_printk("dev %d,%d ino %llu mode 0%o is_metadata %d block %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->mode, __entry->is_metadata, __entry->block)
 );
 
@@ -1250,9 +1250,9 @@ TRACE_EVENT(ext4_da_update_reserve_space,
 	TP_ARGS(inode, used_blocks, quota_claim),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	i_blocks		)
+		__field(	dev_t,	dev			)
 		__field(	int,	used_blocks		)
 		__field(	int,	reserved_data_blocks	)
 		__field(	int,	quota_claim		)
@@ -1270,10 +1270,10 @@ TRACE_EVENT(ext4_da_update_reserve_space,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o i_blocks %llu used_blocks %d "
+	TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu used_blocks %d "
 		  "reserved_data_blocks %d quota_claim %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->mode, __entry->i_blocks,
 		  __entry->used_blocks, __entry->reserved_data_blocks,
 		  __entry->quota_claim)
@@ -1285,9 +1285,9 @@ TRACE_EVENT(ext4_da_reserve_space,
 	TP_ARGS(inode, nr_resv),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	i_blocks		)
+		__field(	dev_t,	dev			)
 		__field(	int,	reserve_blocks		)
 		__field(	int,	reserved_data_blocks	)
 		__field(	__u16,  mode			)
@@ -1302,10 +1302,10 @@ TRACE_EVENT(ext4_da_reserve_space,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o i_blocks %llu reserve_blocks %d"
+	TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu reserve_blocks %d"
 		  "reserved_data_blocks %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->mode, __entry->i_blocks,
 		  __entry->reserve_blocks, __entry->reserved_data_blocks)
 );
@@ -1316,9 +1316,9 @@ TRACE_EVENT(ext4_da_release_space,
 	TP_ARGS(inode, freed_blocks),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	__u64,	i_blocks		)
+		__field(	dev_t,	dev			)
 		__field(	int,	freed_blocks		)
 		__field(	int,	reserved_data_blocks	)
 		__field(	__u16,  mode			)
@@ -1333,10 +1333,10 @@ TRACE_EVENT(ext4_da_release_space,
 		__entry->mode	= inode->i_mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o i_blocks %llu freed_blocks %d "
+	TP_printk("dev %d,%d ino %llu mode 0%o i_blocks %llu freed_blocks %d "
 		  "reserved_data_blocks %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->mode, __entry->i_blocks,
 		  __entry->freed_blocks, __entry->reserved_data_blocks)
 );
@@ -1412,10 +1412,10 @@ DECLARE_EVENT_CLASS(ext4__fallocate_mode,
 	TP_ARGS(inode, offset, len, mode),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	offset			)
 		__field(	loff_t, len			)
+		__field(	dev_t,	dev			)
 		__field(	int,	mode			)
 	),
 
@@ -1427,9 +1427,9 @@ DECLARE_EVENT_CLASS(ext4__fallocate_mode,
 		__entry->mode	= mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu offset %lld len %lld mode %s",
+	TP_printk("dev %d,%d ino %llu offset %lld len %lld mode %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->offset, __entry->len,
 		  show_falloc_mode(__entry->mode))
 );
@@ -1462,9 +1462,9 @@ TRACE_EVENT(ext4_fallocate_exit,
 	TP_ARGS(inode, offset, max_blocks, ret),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	pos			)
+		__field(	dev_t,	dev			)
 		__field(	unsigned int,	blocks		)
 		__field(	int, 	ret			)
 	),
@@ -1477,9 +1477,9 @@ TRACE_EVENT(ext4_fallocate_exit,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu pos %lld blocks %u ret %d",
+	TP_printk("dev %d,%d ino %llu pos %lld blocks %u ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->pos, __entry->blocks,
 		  __entry->ret)
 );
@@ -1490,10 +1490,10 @@ TRACE_EVENT(ext4_unlink_enter,
 	TP_ARGS(parent, dentry),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
-		__field(	ino_t,	parent			)
+		__field(	u64,	ino			)
+		__field(	u64,	parent			)
 		__field(	loff_t,	size			)
+		__field(	dev_t,	dev			)
 	),
 
 	TP_fast_assign(
@@ -1503,10 +1503,10 @@ TRACE_EVENT(ext4_unlink_enter,
 		__entry->size		= d_inode(dentry)->i_size;
 	),
 
-	TP_printk("dev %d,%d ino %lu size %lld parent %lu",
+	TP_printk("dev %d,%d ino %llu size %lld parent %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->size,
-		  (unsigned long) __entry->parent)
+		  __entry->ino, __entry->size,
+		  __entry->parent)
 );
 
 TRACE_EVENT(ext4_unlink_exit,
@@ -1515,8 +1515,8 @@ TRACE_EVENT(ext4_unlink_exit,
 	TP_ARGS(dentry, ret),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino			)
 		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
 		__field(	int,	ret			)
 	),
 
@@ -1526,9 +1526,9 @@ TRACE_EVENT(ext4_unlink_exit,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu ret %d",
+	TP_printk("dev %d,%d ino %llu ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->ret)
 );
 
@@ -1538,9 +1538,9 @@ DECLARE_EVENT_CLASS(ext4__truncate,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
+		__field(	u64,		ino		)
 		__field(	__u64,		blocks		)
+		__field(	dev_t,		dev		)
 	),
 
 	TP_fast_assign(
@@ -1549,9 +1549,9 @@ DECLARE_EVENT_CLASS(ext4__truncate,
 		__entry->blocks	= inode->i_blocks;
 	),
 
-	TP_printk("dev %d,%d ino %lu blocks %llu",
+	TP_printk("dev %d,%d ino %llu blocks %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->blocks)
+		  __entry->ino, __entry->blocks)
 );
 
 DEFINE_EVENT(ext4__truncate, ext4_truncate_enter,
@@ -1576,13 +1576,13 @@ TRACE_EVENT(ext4_ext_convert_to_initialized_enter,
 	TP_ARGS(inode, map, ux),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino	)
+		__field(	ext4_fsblk_t,	u_pblk	)
 		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
 		__field(	ext4_lblk_t,	m_lblk	)
 		__field(	unsigned,	m_len	)
 		__field(	ext4_lblk_t,	u_lblk	)
 		__field(	unsigned,	u_len	)
-		__field(	ext4_fsblk_t,	u_pblk	)
 	),
 
 	TP_fast_assign(
@@ -1595,10 +1595,10 @@ TRACE_EVENT(ext4_ext_convert_to_initialized_enter,
 		__entry->u_pblk		= ext4_ext_pblock(ux);
 	),
 
-	TP_printk("dev %d,%d ino %lu m_lblk %u m_len %u u_lblk %u u_len %u "
+	TP_printk("dev %d,%d ino %llu m_lblk %u m_len %u u_lblk %u u_len %u "
 		  "u_pblk %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->m_lblk, __entry->m_len,
 		  __entry->u_lblk, __entry->u_len, __entry->u_pblk)
 );
@@ -1614,16 +1614,16 @@ TRACE_EVENT(ext4_ext_convert_to_initialized_fastpath,
 	TP_ARGS(inode, map, ux, ix),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino	)
+		__field(	ext4_fsblk_t,	u_pblk	)
+		__field(	ext4_fsblk_t,	i_pblk	)
 		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
 		__field(	ext4_lblk_t,	m_lblk	)
 		__field(	unsigned,	m_len	)
 		__field(	ext4_lblk_t,	u_lblk	)
 		__field(	unsigned,	u_len	)
-		__field(	ext4_fsblk_t,	u_pblk	)
 		__field(	ext4_lblk_t,	i_lblk	)
 		__field(	unsigned,	i_len	)
-		__field(	ext4_fsblk_t,	i_pblk	)
 	),
 
 	TP_fast_assign(
@@ -1639,11 +1639,11 @@ TRACE_EVENT(ext4_ext_convert_to_initialized_fastpath,
 		__entry->i_pblk		= ext4_ext_pblock(ix);
 	),
 
-	TP_printk("dev %d,%d ino %lu m_lblk %u m_len %u "
+	TP_printk("dev %d,%d ino %llu m_lblk %u m_len %u "
 		  "u_lblk %u u_len %u u_pblk %llu "
 		  "i_lblk %u i_len %u i_pblk %llu ",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->m_lblk, __entry->m_len,
 		  __entry->u_lblk, __entry->u_len, __entry->u_pblk,
 		  __entry->i_lblk, __entry->i_len, __entry->i_pblk)
@@ -1656,8 +1656,8 @@ DECLARE_EVENT_CLASS(ext4__map_blocks_enter,
 	TP_ARGS(inode, lblk, len, flags),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	unsigned int,	len		)
 		__field(	unsigned int,	flags		)
@@ -1671,9 +1671,9 @@ DECLARE_EVENT_CLASS(ext4__map_blocks_enter,
 		__entry->flags	= flags;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %u len %u flags %s",
+	TP_printk("dev %d,%d ino %llu lblk %u len %u flags %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->len, show_map_flags(__entry->flags))
 );
 
@@ -1698,10 +1698,10 @@ DECLARE_EVENT_CLASS(ext4__map_blocks_exit,
 	TP_ARGS(inode, flags, map, ret),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	ext4_fsblk_t,	pblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	unsigned int,	flags		)
-		__field(	ext4_fsblk_t,	pblk		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	unsigned int,	len		)
 		__field(	unsigned int,	mflags		)
@@ -1719,10 +1719,10 @@ DECLARE_EVENT_CLASS(ext4__map_blocks_exit,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu flags %s lblk %u pblk %llu len %u "
+	TP_printk("dev %d,%d ino %llu flags %s lblk %u pblk %llu len %u "
 		  "mflags %s ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  show_map_flags(__entry->flags), __entry->lblk, __entry->pblk,
 		  __entry->len, show_mflags(__entry->mflags), __entry->ret)
 );
@@ -1747,9 +1747,9 @@ TRACE_EVENT(ext4_ext_load_extent,
 	TP_ARGS(inode, lblk, pblk),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
+		__field(	u64,		ino		)
 		__field(	ext4_fsblk_t,	pblk		)
+		__field(	dev_t,		dev		)
 		__field(	ext4_lblk_t,	lblk		)
 	),
 
@@ -1760,9 +1760,9 @@ TRACE_EVENT(ext4_ext_load_extent,
 		__entry->lblk	= lblk;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %u pblk %llu",
+	TP_printk("dev %d,%d ino %llu lblk %u pblk %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->pblk)
 );
 
@@ -1772,8 +1772,8 @@ TRACE_EVENT(ext4_load_inode,
 	TP_ARGS(sb, ino),
 
 	TP_STRUCT__entry(
+		__field(	u64,	ino		)
 		__field(	dev_t,	dev		)
-		__field(	ino_t,	ino		)
 	),
 
 	TP_fast_assign(
@@ -1781,9 +1781,9 @@ TRACE_EVENT(ext4_load_inode,
 		__entry->ino		= ino;
 	),
 
-	TP_printk("dev %d,%d ino %ld",
+	TP_printk("dev %d,%d ino %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino)
+		  __entry->ino)
 );
 
 TRACE_EVENT(ext4_journal_start_sb,
@@ -1823,7 +1823,7 @@ TRACE_EVENT(ext4_journal_start_inode,
 	TP_ARGS(inode, blocks, rsv_blocks, revoke_creds, type, IP),
 
 	TP_STRUCT__entry(
-		__field(	unsigned long,	ino		)
+		__field(	u64,		ino		)
 		__field(	dev_t,		dev		)
 		__field(	unsigned long,	ip		)
 		__field(	int,		blocks		)
@@ -1843,10 +1843,10 @@ TRACE_EVENT(ext4_journal_start_inode,
 	),
 
 	TP_printk("dev %d,%d blocks %d, rsv_blocks %d, revoke_creds %d,"
-		  " type %d, ino %lu, caller %pS", MAJOR(__entry->dev),
+		  " type %d, ino %llu, caller %pS", MAJOR(__entry->dev),
 		  MINOR(__entry->dev), __entry->blocks, __entry->rsv_blocks,
-		  __entry->revoke_creds, __entry->type, __entry->ino,
-		  (void *)__entry->ip)
+		  __entry->revoke_creds, __entry->type,
+		  __entry->ino, (void *)__entry->ip)
 );
 
 TRACE_EVENT(ext4_journal_start_reserved,
@@ -1927,14 +1927,14 @@ TRACE_EVENT(ext4_ext_handle_unwritten_extents,
 	TP_ARGS(inode, map, flags, allocated, newblock),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	ext4_fsblk_t,	pblk		)
+		__field(	ext4_fsblk_t,	newblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	int,		flags		)
 		__field(	ext4_lblk_t,	lblk		)
-		__field(	ext4_fsblk_t,	pblk		)
 		__field(	unsigned int,	len		)
 		__field(	unsigned int,	allocated	)
-		__field(	ext4_fsblk_t,	newblk		)
 	),
 
 	TP_fast_assign(
@@ -1948,10 +1948,10 @@ TRACE_EVENT(ext4_ext_handle_unwritten_extents,
 		__entry->newblk		= newblock;
 	),
 
-	TP_printk("dev %d,%d ino %lu m_lblk %u m_pblk %llu m_len %u flags %s "
+	TP_printk("dev %d,%d ino %llu m_lblk %u m_pblk %llu m_len %u flags %s "
 		  "allocated %d newblock %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->lblk, (unsigned long long) __entry->pblk,
 		  __entry->len, show_map_flags(__entry->flags),
 		  (unsigned int) __entry->allocated,
@@ -1994,9 +1994,9 @@ TRACE_EVENT(ext4_ext_show_extent,
 	TP_ARGS(inode, lblk, pblk, len),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
+		__field(	u64,		ino	)
 		__field(	ext4_fsblk_t,	pblk	)
+		__field(	dev_t,		dev	)
 		__field(	ext4_lblk_t,	lblk	)
 		__field(	unsigned short,	len	)
 	),
@@ -2009,9 +2009,9 @@ TRACE_EVENT(ext4_ext_show_extent,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %u pblk %llu len %u",
+	TP_printk("dev %d,%d ino %llu lblk %u pblk %llu len %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->lblk,
 		  (unsigned long long) __entry->pblk,
 		  (unsigned short) __entry->len)
@@ -2025,14 +2025,14 @@ TRACE_EVENT(ext4_remove_blocks,
 	TP_ARGS(inode, ex, from, to, pc),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino	)
+		__field(	ext4_fsblk_t,	ee_pblk	)
+		__field(	ext4_fsblk_t,	pc_pclu	)
 		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
 		__field(	ext4_lblk_t,	from	)
 		__field(	ext4_lblk_t,	to	)
-		__field(	ext4_fsblk_t,	ee_pblk	)
 		__field(	ext4_lblk_t,	ee_lblk	)
 		__field(	unsigned short,	ee_len	)
-		__field(	ext4_fsblk_t,	pc_pclu	)
 		__field(	ext4_lblk_t,	pc_lblk	)
 		__field(	int,		pc_state)
 	),
@@ -2050,10 +2050,10 @@ TRACE_EVENT(ext4_remove_blocks,
 		__entry->pc_state	= pc->state;
 	),
 
-	TP_printk("dev %d,%d ino %lu extent [%u(%llu), %u]"
+	TP_printk("dev %d,%d ino %llu extent [%u(%llu), %u]"
 		  "from %u to %u partial [pclu %lld lblk %u state %d]",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->ee_lblk,
 		  (unsigned long long) __entry->ee_pblk,
 		  (unsigned short) __entry->ee_len,
@@ -2072,13 +2072,13 @@ TRACE_EVENT(ext4_ext_rm_leaf,
 	TP_ARGS(inode, start, ex, pc),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino	)
+		__field(	ext4_fsblk_t,	ee_pblk	)
+		__field(	ext4_fsblk_t,	pc_pclu	)
 		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
 		__field(	ext4_lblk_t,	start	)
 		__field(	ext4_lblk_t,	ee_lblk	)
-		__field(	ext4_fsblk_t,	ee_pblk	)
 		__field(	short,		ee_len	)
-		__field(	ext4_fsblk_t,	pc_pclu	)
 		__field(	ext4_lblk_t,	pc_lblk	)
 		__field(	int,		pc_state)
 	),
@@ -2095,10 +2095,10 @@ TRACE_EVENT(ext4_ext_rm_leaf,
 		__entry->pc_state	= pc->state;
 	),
 
-	TP_printk("dev %d,%d ino %lu start_lblk %u last_extent [%u(%llu), %u]"
+	TP_printk("dev %d,%d ino %llu start_lblk %u last_extent [%u(%llu), %u]"
 		  "partial [pclu %lld lblk %u state %d]",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->start,
 		  (unsigned) __entry->ee_lblk,
 		  (unsigned long long) __entry->ee_pblk,
@@ -2114,9 +2114,9 @@ TRACE_EVENT(ext4_ext_rm_idx,
 	TP_ARGS(inode, pblk),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
+		__field(	u64,		ino	)
 		__field(	ext4_fsblk_t,	pblk	)
+		__field(	dev_t,		dev	)
 	),
 
 	TP_fast_assign(
@@ -2125,9 +2125,9 @@ TRACE_EVENT(ext4_ext_rm_idx,
 		__entry->pblk	= pblk;
 	),
 
-	TP_printk("dev %d,%d ino %lu index_pblk %llu",
+	TP_printk("dev %d,%d ino %llu index_pblk %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned long long) __entry->pblk)
 );
 
@@ -2138,8 +2138,8 @@ TRACE_EVENT(ext4_ext_remove_space,
 	TP_ARGS(inode, start, end, depth),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino	)
 		__field(	dev_t,		dev	)
-		__field(	ino_t,		ino	)
 		__field(	ext4_lblk_t,	start	)
 		__field(	ext4_lblk_t,	end	)
 		__field(	int,		depth	)
@@ -2153,9 +2153,9 @@ TRACE_EVENT(ext4_ext_remove_space,
 		__entry->depth	= depth;
 	),
 
-	TP_printk("dev %d,%d ino %lu since %u end %u depth %d",
+	TP_printk("dev %d,%d ino %llu since %u end %u depth %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->start,
 		  (unsigned) __entry->end,
 		  __entry->depth)
@@ -2168,12 +2168,12 @@ TRACE_EVENT(ext4_ext_remove_space_done,
 	TP_ARGS(inode, start, end, depth, pc, eh_entries),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	ext4_fsblk_t,	pc_pclu		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	start		)
 		__field(	ext4_lblk_t,	end		)
 		__field(	int,		depth		)
-		__field(	ext4_fsblk_t,	pc_pclu		)
 		__field(	ext4_lblk_t,	pc_lblk		)
 		__field(	int,		pc_state	)
 		__field(	unsigned short,	eh_entries	)
@@ -2191,11 +2191,11 @@ TRACE_EVENT(ext4_ext_remove_space_done,
 		__entry->eh_entries	= le16_to_cpu(eh_entries);
 	),
 
-	TP_printk("dev %d,%d ino %lu since %u end %u depth %d "
+	TP_printk("dev %d,%d ino %llu since %u end %u depth %d "
 		  "partial [pclu %lld lblk %u state %d] "
 		  "remaining_entries %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  (unsigned) __entry->start,
 		  (unsigned) __entry->end,
 		  __entry->depth,
@@ -2211,13 +2211,13 @@ DECLARE_EVENT_CLASS(ext4__es_extent,
 	TP_ARGS(inode, es),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	u64,		seq		)
+		__field(	ext4_fsblk_t,	pblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	ext4_lblk_t,	len		)
-		__field(	ext4_fsblk_t,	pblk		)
 		__field(	char,		status		)
-		__field(	u64,		seq		)
 	),
 
 	TP_fast_assign(
@@ -2230,9 +2230,9 @@ DECLARE_EVENT_CLASS(ext4__es_extent,
 		__entry->seq	= EXT4_I(inode)->i_es_seq;
 	),
 
-	TP_printk("dev %d,%d ino %lu es [%u/%u) mapped %llu status %s seq %llu",
+	TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s seq %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->len,
 		  __entry->pblk, show_extent_status(__entry->status),
 		  __entry->seq)
@@ -2256,11 +2256,11 @@ TRACE_EVENT(ext4_es_remove_extent,
 	TP_ARGS(inode, lblk, len),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(	loff_t,	lblk			)
 		__field(	loff_t,	len			)
 		__field(	u64,	seq			)
+		__field(	dev_t,	dev			)
 	),
 
 	TP_fast_assign(
@@ -2271,9 +2271,9 @@ TRACE_EVENT(ext4_es_remove_extent,
 		__entry->seq	= EXT4_I(inode)->i_es_seq;
 	),
 
-	TP_printk("dev %d,%d ino %lu es [%lld/%lld) seq %llu",
+	TP_printk("dev %d,%d ino %llu es [%lld/%lld) seq %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->len, __entry->seq)
 );
 
@@ -2283,8 +2283,8 @@ TRACE_EVENT(ext4_es_find_extent_range_enter,
 	TP_ARGS(inode, lblk),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 	),
 
@@ -2294,9 +2294,9 @@ TRACE_EVENT(ext4_es_find_extent_range_enter,
 		__entry->lblk	= lblk;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %u",
+	TP_printk("dev %d,%d ino %llu lblk %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->lblk)
+		  __entry->ino, __entry->lblk)
 );
 
 TRACE_EVENT(ext4_es_find_extent_range_exit,
@@ -2305,11 +2305,11 @@ TRACE_EVENT(ext4_es_find_extent_range_exit,
 	TP_ARGS(inode, es),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	ext4_fsblk_t,	pblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	ext4_lblk_t,	len		)
-		__field(	ext4_fsblk_t,	pblk		)
 		__field(	char, status	)
 	),
 
@@ -2322,9 +2322,9 @@ TRACE_EVENT(ext4_es_find_extent_range_exit,
 		__entry->status	= ext4_es_status(es);
 	),
 
-	TP_printk("dev %d,%d ino %lu es [%u/%u) mapped %llu status %s",
+	TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->len,
 		  __entry->pblk, show_extent_status(__entry->status))
 );
@@ -2335,8 +2335,8 @@ TRACE_EVENT(ext4_es_lookup_extent_enter,
 	TP_ARGS(inode, lblk),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 	),
 
@@ -2346,9 +2346,9 @@ TRACE_EVENT(ext4_es_lookup_extent_enter,
 		__entry->lblk	= lblk;
 	),
 
-	TP_printk("dev %d,%d ino %lu lblk %u",
+	TP_printk("dev %d,%d ino %llu lblk %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->lblk)
+		  __entry->ino, __entry->lblk)
 );
 
 TRACE_EVENT(ext4_es_lookup_extent_exit,
@@ -2358,13 +2358,13 @@ TRACE_EVENT(ext4_es_lookup_extent_exit,
 	TP_ARGS(inode, es, found),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	ext4_fsblk_t,	pblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	ext4_lblk_t,	len		)
-		__field(	ext4_fsblk_t,	pblk		)
-		__field(	char,		status		)
 		__field(	int,		found		)
+		__field(	char,		status		)
 	),
 
 	TP_fast_assign(
@@ -2377,9 +2377,9 @@ TRACE_EVENT(ext4_es_lookup_extent_exit,
 		__entry->found	= found;
 	),
 
-	TP_printk("dev %d,%d ino %lu found %d [%u/%u) %llu %s",
+	TP_printk("dev %d,%d ino %llu found %d [%u/%u) %llu %s",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino, __entry->found,
+		  __entry->ino, __entry->found,
 		  __entry->lblk, __entry->len,
 		  __entry->found ? __entry->pblk : 0,
 		  show_extent_status(__entry->found ? __entry->status : 0))
@@ -2447,10 +2447,10 @@ TRACE_EVENT(ext4_collapse_range,
 	TP_ARGS(inode, offset, len),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	offset)
 		__field(loff_t, len)
+		__field(dev_t,	dev)
 	),
 
 	TP_fast_assign(
@@ -2460,9 +2460,9 @@ TRACE_EVENT(ext4_collapse_range,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev %d,%d ino %lu offset %lld len %lld",
+	TP_printk("dev %d,%d ino %llu offset %lld len %lld",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->offset, __entry->len)
 );
 
@@ -2472,10 +2472,10 @@ TRACE_EVENT(ext4_insert_range,
 	TP_ARGS(inode, offset, len),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	offset)
 		__field(loff_t, len)
+		__field(dev_t,	dev)
 	),
 
 	TP_fast_assign(
@@ -2485,9 +2485,9 @@ TRACE_EVENT(ext4_insert_range,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev %d,%d ino %lu offset %lld len %lld",
+	TP_printk("dev %d,%d ino %llu offset %lld len %lld",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->offset, __entry->len)
 );
 
@@ -2526,15 +2526,15 @@ TRACE_EVENT(ext4_es_insert_delayed_extent,
 	TP_ARGS(inode, es, lclu_allocated, end_allocated),
 
 	TP_STRUCT__entry(
+		__field(	u64,		ino		)
+		__field(	u64,		seq		)
+		__field(	ext4_fsblk_t,	pblk		)
 		__field(	dev_t,		dev		)
-		__field(	ino_t,		ino		)
 		__field(	ext4_lblk_t,	lblk		)
 		__field(	ext4_lblk_t,	len		)
-		__field(	ext4_fsblk_t,	pblk		)
 		__field(	char,		status		)
 		__field(	bool,		lclu_allocated	)
 		__field(	bool,		end_allocated	)
-		__field(	u64,		seq		)
 	),
 
 	TP_fast_assign(
@@ -2549,9 +2549,9 @@ TRACE_EVENT(ext4_es_insert_delayed_extent,
 		__entry->seq		= EXT4_I(inode)->i_es_seq;
 	),
 
-	TP_printk("dev %d,%d ino %lu es [%u/%u) mapped %llu status %s allocated %d %d seq %llu",
+	TP_printk("dev %d,%d ino %llu es [%u/%u) mapped %llu status %s allocated %d %d seq %llu",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->ino,
+		  __entry->ino,
 		  __entry->lblk, __entry->len,
 		  __entry->pblk, show_extent_status(__entry->status),
 		  __entry->lclu_allocated, __entry->end_allocated,
@@ -2875,9 +2875,9 @@ DECLARE_EVENT_CLASS(ext4_fc_track_dentry,
 	TP_ARGS(handle, inode, dentry, ret),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
 		__field(dev_t, dev)
 		__field(tid_t, t_tid)
-		__field(ino_t, i_ino)
 		__field(tid_t, i_sync_tid)
 		__field(int, error)
 	),
@@ -2892,7 +2892,7 @@ DECLARE_EVENT_CLASS(ext4_fc_track_dentry,
 		__entry->error = ret;
 	),
 
-	TP_printk("dev %d,%d, t_tid %u, ino %lu, i_sync_tid %u, error %d",
+	TP_printk("dev %d,%d, t_tid %u, ino %llu, i_sync_tid %u, error %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
 		  __entry->error
@@ -2916,9 +2916,9 @@ TRACE_EVENT(ext4_fc_track_inode,
 	TP_ARGS(handle, inode, ret),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
 		__field(dev_t, dev)
 		__field(tid_t, t_tid)
-		__field(ino_t, i_ino)
 		__field(tid_t, i_sync_tid)
 		__field(int, error)
 	),
@@ -2933,7 +2933,7 @@ TRACE_EVENT(ext4_fc_track_inode,
 		__entry->error = ret;
 	),
 
-	TP_printk("dev %d:%d, t_tid %u, inode %lu, i_sync_tid %u, error %d",
+	TP_printk("dev %d:%d, t_tid %u, inode %llu, i_sync_tid %u, error %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
 		  __entry->error)
@@ -2946,12 +2946,12 @@ TRACE_EVENT(ext4_fc_track_range,
 	TP_ARGS(handle, inode, start, end, ret),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
+		__field(long, start)
+		__field(long, end)
 		__field(dev_t, dev)
 		__field(tid_t, t_tid)
-		__field(ino_t, i_ino)
 		__field(tid_t, i_sync_tid)
-		__field(long, start)
-		__field(long, end)
 		__field(int, error)
 	),
 
@@ -2967,7 +2967,7 @@ TRACE_EVENT(ext4_fc_track_range,
 		__entry->error = ret;
 	),
 
-	TP_printk("dev %d:%d, t_tid %u, inode %lu, i_sync_tid %u, error %d, start %ld, end %ld",
+	TP_printk("dev %d:%d, t_tid %u, inode %llu, i_sync_tid %u, error %d, start %ld, end %ld",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->t_tid, __entry->i_ino, __entry->i_sync_tid,
 		  __entry->error, __entry->start, __entry->end)
@@ -3029,11 +3029,11 @@ TRACE_EVENT(ext4_move_extent_enter,
 	TP_ARGS(orig_inode, orig_map, donor_inode, donor_lblk),
 
 	TP_STRUCT__entry(
+		__field(u64, orig_ino)
+		__field(u64, donor_ino)
 		__field(dev_t, dev)
-		__field(ino_t, orig_ino)
 		__field(ext4_lblk_t, orig_lblk)
 		__field(unsigned int, orig_flags)
-		__field(ino_t, donor_ino)
 		__field(ext4_lblk_t, donor_lblk)
 		__field(unsigned int, len)
 	),
@@ -3048,11 +3048,11 @@ TRACE_EVENT(ext4_move_extent_enter,
 		__entry->len		= orig_map->m_len;
 	),
 
-	TP_printk("dev %d,%d origin ino %lu lblk %u flags %s donor ino %lu lblk %u len %u",
+	TP_printk("dev %d,%d origin ino %llu lblk %u flags %s donor ino %llu lblk %u len %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->orig_ino,  __entry->orig_lblk,
+		  __entry->orig_ino,  __entry->orig_lblk,
 		  show_mflags(__entry->orig_flags),
-		  (unsigned long) __entry->donor_ino,  __entry->donor_lblk,
+		  __entry->donor_ino,  __entry->donor_lblk,
 		  __entry->len)
 );
 
@@ -3065,13 +3065,13 @@ TRACE_EVENT(ext4_move_extent_exit,
 		move_len, move_type, ret),
 
 	TP_STRUCT__entry(
+		__field(u64, orig_ino)
+		__field(u64, donor_ino)
+		__field(u64, move_len)
 		__field(dev_t, dev)
-		__field(ino_t, orig_ino)
 		__field(ext4_lblk_t, orig_lblk)
-		__field(ino_t, donor_ino)
 		__field(ext4_lblk_t, donor_lblk)
 		__field(unsigned int, m_len)
-		__field(u64, move_len)
 		__field(int, move_type)
 		__field(int, ret)
 	),
@@ -3088,10 +3088,10 @@ TRACE_EVENT(ext4_move_extent_exit,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev %d,%d origin ino %lu lblk %u donor ino %lu lblk %u m_len %u, move_len %llu type %d ret %d",
+	TP_printk("dev %d,%d origin ino %llu lblk %u donor ino %llu lblk %u m_len %u, move_len %llu type %d ret %d",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long) __entry->orig_ino,  __entry->orig_lblk,
-		  (unsigned long) __entry->donor_ino,  __entry->donor_lblk,
+		  __entry->orig_ino,  __entry->orig_lblk,
+		  __entry->donor_ino,  __entry->donor_lblk,
 		  __entry->m_len, __entry->move_len, __entry->move_type,
 		  __entry->ret)
 );
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index 9364e67755620..05a46908acd94 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -9,7 +9,7 @@
 #include <uapi/linux/f2fs.h>
 
 #define show_dev(dev)		MAJOR(dev), MINOR(dev)
-#define show_dev_ino(entry)	show_dev(entry->dev), (unsigned long)entry->ino
+#define show_dev_ino(entry)	show_dev(entry->dev), (unsigned long long)entry->ino
 
 TRACE_DEFINE_ENUM(NODE);
 TRACE_DEFINE_ENUM(DATA);
@@ -206,13 +206,13 @@ DECLARE_EVENT_CLASS(f2fs__inode,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
+		__field(u64,	pino)
+		__field(loff_t,	size)
+		__field(blkcnt_t, blocks)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
-		__field(ino_t,	pino)
 		__field(umode_t, mode)
-		__field(loff_t,	size)
 		__field(unsigned int, nlink)
-		__field(blkcnt_t, blocks)
 		__field(__u8,	advise)
 	),
 
@@ -227,10 +227,10 @@ DECLARE_EVENT_CLASS(f2fs__inode,
 		__entry->advise	= F2FS_I(inode)->i_advise;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pino = %lu, i_mode = 0x%hx, "
+	TP_printk("dev = (%d,%d), ino = %llu, pino = %llu, i_mode = 0x%hx, "
 		"i_size = %lld, i_nlink = %u, i_blocks = %llu, i_advise = 0x%x",
 		show_dev_ino(__entry),
-		(unsigned long)__entry->pino,
+		__entry->pino,
 		__entry->mode,
 		__entry->size,
 		(unsigned int)__entry->nlink,
@@ -245,8 +245,8 @@ DECLARE_EVENT_CLASS(f2fs__inode_exit,
 	TP_ARGS(inode, ret),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(umode_t, mode)
 		__field(int,	ret)
 	),
@@ -258,7 +258,7 @@ DECLARE_EVENT_CLASS(f2fs__inode_exit,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, type: %s, mode = 0%o, ret = %d",
+	TP_printk("dev = (%d,%d), ino = %llu, type: %s, mode = 0%o, ret = %d",
 		show_dev_ino(__entry),
 		show_inode_type(__entry->mode & S_IFMT),
 		__entry->mode & S_ALL_PERM,
@@ -279,8 +279,8 @@ TRACE_EVENT(f2fs_sync_file_exit,
 	TP_ARGS(inode, cp_reason, datasync, ret),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(int,	cp_reason)
 		__field(int,	datasync)
 		__field(int,	ret)
@@ -294,7 +294,7 @@ TRACE_EVENT(f2fs_sync_file_exit,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, cp_reason: %s, "
+	TP_printk("dev = (%d,%d), ino = %llu, cp_reason: %s, "
 		"datasync = %d, ret = %d",
 		show_dev_ino(__entry),
 		show_fsync_cpreason(__entry->cp_reason),
@@ -361,10 +361,10 @@ TRACE_EVENT(f2fs_unlink_enter,
 	TP_ARGS(dir, dentry),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	size)
 		__field(blkcnt_t, blocks)
+		__field(dev_t,	dev)
 		__string(name,  dentry->d_name.name)
 	),
 
@@ -376,7 +376,7 @@ TRACE_EVENT(f2fs_unlink_enter,
 		__assign_str(name);
 	),
 
-	TP_printk("dev = (%d,%d), dir ino = %lu, i_size = %lld, "
+	TP_printk("dev = (%d,%d), dir ino = %llu, i_size = %lld, "
 		"i_blocks = %llu, name = %s",
 		show_dev_ino(__entry),
 		__entry->size,
@@ -412,8 +412,8 @@ TRACE_EVENT(f2fs_truncate_data_blocks_range,
 	TP_ARGS(inode, nid,  ofs, free),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(nid_t,	nid)
 		__field(unsigned int,	ofs)
 		__field(int,	free)
@@ -427,7 +427,7 @@ TRACE_EVENT(f2fs_truncate_data_blocks_range,
 		__entry->free	= free;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, nid = %u, offset = %u, freed = %d",
+	TP_printk("dev = (%d,%d), ino = %llu, nid = %u, offset = %u, freed = %d",
 		show_dev_ino(__entry),
 		(unsigned int)__entry->nid,
 		__entry->ofs,
@@ -441,11 +441,11 @@ DECLARE_EVENT_CLASS(f2fs__truncate_op,
 	TP_ARGS(inode, from),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	size)
 		__field(blkcnt_t, blocks)
 		__field(u64,	from)
+		__field(dev_t,	dev)
 	),
 
 	TP_fast_assign(
@@ -456,7 +456,7 @@ DECLARE_EVENT_CLASS(f2fs__truncate_op,
 		__entry->from	= from;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, i_size = %lld, i_blocks = %llu, "
+	TP_printk("dev = (%d,%d), ino = %llu, i_size = %lld, i_blocks = %llu, "
 		"start file offset = %llu",
 		show_dev_ino(__entry),
 		__entry->size,
@@ -499,8 +499,8 @@ DECLARE_EVENT_CLASS(f2fs__truncate_node,
 	TP_ARGS(inode, nid, blk_addr),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(nid_t,	nid)
 		__field(block_t,	blk_addr)
 	),
@@ -512,7 +512,7 @@ DECLARE_EVENT_CLASS(f2fs__truncate_node,
 		__entry->blk_addr	= blk_addr;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, nid = %u, block_address = 0x%llx",
+	TP_printk("dev = (%d,%d), ino = %llu, nid = %u, block_address = 0x%llx",
 		show_dev_ino(__entry),
 		(unsigned int)__entry->nid,
 		(unsigned long long)__entry->blk_addr)
@@ -546,8 +546,8 @@ TRACE_EVENT(f2fs_truncate_partial_nodes,
 	TP_ARGS(inode, nid, depth, err),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__array(nid_t,	nid, 3)
 		__field(int,	depth)
 		__field(int,	err)
@@ -563,7 +563,7 @@ TRACE_EVENT(f2fs_truncate_partial_nodes,
 		__entry->err	= err;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, "
 		"nid[0] = %u, nid[1] = %u, nid[2] = %u, depth = %d, err = %d",
 		show_dev_ino(__entry),
 		(unsigned int)__entry->nid[0],
@@ -581,11 +581,11 @@ TRACE_EVENT(f2fs_file_write_iter,
 	TP_ARGS(inode, offset, length, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, offset)
 		__field(size_t, length)
 		__field(ssize_t, ret)
+		__field(dev_t,	dev)
 	),
 
 	TP_fast_assign(
@@ -596,7 +596,7 @@ TRACE_EVENT(f2fs_file_write_iter,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, "
 		"offset = %lld, length = %zu, written(err) = %zd",
 		show_dev_ino(__entry),
 		__entry->offset,
@@ -611,11 +611,11 @@ TRACE_EVENT(f2fs_fadvise,
 	TP_ARGS(inode, offset, len, advice),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t, size)
 		__field(loff_t,	offset)
 		__field(loff_t,	len)
+		__field(dev_t,	dev)
 		__field(int,	advice)
 	),
 
@@ -628,7 +628,7 @@ TRACE_EVENT(f2fs_fadvise,
 		__entry->advice	= advice;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, i_size = %lld offset:%llu, len:%llu, advise:%d",
+	TP_printk("dev = (%d,%d), ino = %llu, i_size = %lld offset:%llu, len:%llu, advise:%d",
 		show_dev_ino(__entry),
 		(unsigned long long)__entry->size,
 		__entry->offset,
@@ -643,8 +643,8 @@ TRACE_EVENT(f2fs_map_blocks,
 	TP_ARGS(inode, map, flag, ret),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(block_t,	m_lblk)
 		__field(block_t,	m_pblk)
 		__field(unsigned int,	m_len)
@@ -670,7 +670,7 @@ TRACE_EVENT(f2fs_map_blocks,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, file offset = %llu, "
+	TP_printk("dev = (%d,%d), ino = %llu, file offset = %llu, "
 		"start blkaddr = 0x%llx, len = 0x%llx, flags = %u, "
 		"seg_type = %d, may_create = %d, multidevice = %d, "
 		"flag = %d, err = %d",
@@ -885,8 +885,8 @@ TRACE_EVENT(f2fs_lookup_start,
 	TP_ARGS(dir, dentry, flags),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__string(name,	dentry->d_name.name)
 		__field(unsigned int, flags)
 	),
@@ -898,7 +898,7 @@ TRACE_EVENT(f2fs_lookup_start,
 		__entry->flags	= flags;
 	),
 
-	TP_printk("dev = (%d,%d), pino = %lu, name:%s, flags:%u",
+	TP_printk("dev = (%d,%d), pino = %llu, name:%s, flags:%u",
 		show_dev_ino(__entry),
 		__get_str(name),
 		__entry->flags)
@@ -912,8 +912,8 @@ TRACE_EVENT(f2fs_lookup_end,
 	TP_ARGS(dir, dentry, ino, err),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__string(name,	dentry->d_name.name)
 		__field(nid_t,	cino)
 		__field(int,	err)
@@ -927,7 +927,7 @@ TRACE_EVENT(f2fs_lookup_end,
 		__entry->err	= err;
 	),
 
-	TP_printk("dev = (%d,%d), pino = %lu, name:%s, ino:%u, err:%d",
+	TP_printk("dev = (%d,%d), pino = %llu, name:%s, ino:%u, err:%d",
 		show_dev_ino(__entry),
 		__get_str(name),
 		__entry->cino,
@@ -943,10 +943,10 @@ TRACE_EVENT(f2fs_rename_start,
 	TP_ARGS(old_dir, old_dentry, new_dir, new_dentry, flags),
 
 	TP_STRUCT__entry(
+		__field(u64,		ino)
+		__field(u64,		new_pino)
 		__field(dev_t,		dev)
-		__field(ino_t,		ino)
 		__string(old_name,	old_dentry->d_name.name)
-		__field(ino_t,		new_pino)
 		__string(new_name,	new_dentry->d_name.name)
 		__field(unsigned int,	flags)
 	),
@@ -960,8 +960,8 @@ TRACE_EVENT(f2fs_rename_start,
 		__entry->flags		= flags;
 	),
 
-	TP_printk("dev = (%d,%d), old_dir = %lu, old_name: %s, "
-		"new_dir = %lu, new_name: %s, flags = %u",
+	TP_printk("dev = (%d,%d), old_dir = %llu, old_name: %s, "
+		"new_dir = %llu, new_name: %s, flags = %u",
 		show_dev_ino(__entry),
 		__get_str(old_name),
 		__entry->new_pino,
@@ -977,8 +977,8 @@ TRACE_EVENT(f2fs_rename_end,
 	TP_ARGS(old_dentry, new_dentry, flags, ret),
 
 	TP_STRUCT__entry(
+		__field(u64,		ino)
 		__field(dev_t,		dev)
-		__field(ino_t,		ino)
 		__string(old_name,	old_dentry->d_name.name)
 		__string(new_name,	new_dentry->d_name.name)
 		__field(unsigned int,	flags)
@@ -994,7 +994,7 @@ TRACE_EVENT(f2fs_rename_end,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, old_name: %s, "
+	TP_printk("dev = (%d,%d), ino = %llu, old_name: %s, "
 		"new_name: %s, flags = %u, ret = %d",
 		show_dev_ino(__entry),
 		__get_str(old_name),
@@ -1010,10 +1010,10 @@ TRACE_EVENT(f2fs_readdir,
 	TP_ARGS(dir, start_pos, end_pos, err),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	start)
 		__field(loff_t,	end)
+		__field(dev_t,	dev)
 		__field(int,	err)
 	),
 
@@ -1025,7 +1025,7 @@ TRACE_EVENT(f2fs_readdir,
 		__entry->err	= err;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, start_pos:%llu, end_pos:%llu, err:%d",
+	TP_printk("dev = (%d,%d), ino = %llu, start_pos:%llu, end_pos:%llu, err:%d",
 		show_dev_ino(__entry),
 		__entry->start,
 		__entry->end,
@@ -1040,13 +1040,13 @@ TRACE_EVENT(f2fs_fallocate,
 	TP_ARGS(inode, mode, offset, len, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
-		__field(int,	mode)
+		__field(u64,	ino)
 		__field(loff_t,	offset)
 		__field(loff_t,	len)
 		__field(loff_t, size)
 		__field(blkcnt_t, blocks)
+		__field(dev_t,	dev)
+		__field(int,	mode)
 		__field(int,	ret)
 	),
 
@@ -1061,7 +1061,7 @@ TRACE_EVENT(f2fs_fallocate,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, mode = %x, offset = %lld, "
+	TP_printk("dev = (%d,%d), ino = %llu, mode = %x, offset = %lld, "
 		"len = %lld,  i_size = %lld, i_blocks = %llu, ret = %d",
 		show_dev_ino(__entry),
 		__entry->mode,
@@ -1079,12 +1079,12 @@ TRACE_EVENT(f2fs_direct_IO_enter,
 	TP_ARGS(inode, iocb, len, rw),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	ki_pos)
+		__field(unsigned long,	len)
+		__field(dev_t,	dev)
 		__field(int,	ki_flags)
 		__field(u16,	ki_ioprio)
-		__field(unsigned long,	len)
 		__field(int,	rw)
 	),
 
@@ -1098,7 +1098,7 @@ TRACE_EVENT(f2fs_direct_IO_enter,
 		__entry->rw		= rw;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu pos = %lld len = %lu ki_flags = %x ki_ioprio = %x rw = %d",
+	TP_printk("dev = (%d,%d), ino = %llu pos = %lld len = %lu ki_flags = %x ki_ioprio = %x rw = %d",
 		show_dev_ino(__entry),
 		__entry->ki_pos,
 		__entry->len,
@@ -1115,10 +1115,10 @@ TRACE_EVENT(f2fs_direct_IO_exit,
 	TP_ARGS(inode, offset, len, rw, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	pos)
 		__field(unsigned long,	len)
+		__field(dev_t,	dev)
 		__field(int,	rw)
 		__field(int,	ret)
 	),
@@ -1132,7 +1132,7 @@ TRACE_EVENT(f2fs_direct_IO_exit,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu pos = %lld len = %lu "
+	TP_printk("dev = (%d,%d), ino = %llu pos = %lld len = %lu "
 		"rw = %d ret = %d",
 		show_dev_ino(__entry),
 		__entry->pos,
@@ -1176,9 +1176,9 @@ DECLARE_EVENT_CLASS(f2fs__submit_folio_bio,
 	TP_ARGS(folio, fio),
 
 	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(pgoff_t, index)
+		__field(dev_t, dev)
 		__field(block_t, old_blkaddr)
 		__field(block_t, new_blkaddr)
 		__field(enum req_op, op)
@@ -1199,7 +1199,7 @@ DECLARE_EVENT_CLASS(f2fs__submit_folio_bio,
 		__entry->type		= fio->type;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, folio_index = 0x%lx, "
+	TP_printk("dev = (%d,%d), ino = %llu, folio_index = 0x%lx, "
 		"oldaddr = 0x%llx, newaddr = 0x%llx, rw = %s(%s), type = %s_%s",
 		show_dev_ino(__entry),
 		(unsigned long)__entry->index,
@@ -1306,9 +1306,9 @@ TRACE_EVENT(f2fs_write_begin,
 	TP_ARGS(inode, pos, len),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	pos)
+		__field(dev_t,	dev)
 		__field(unsigned int, len)
 	),
 
@@ -1319,7 +1319,7 @@ TRACE_EVENT(f2fs_write_begin,
 		__entry->len	= len;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pos = %llu, len = %u",
+	TP_printk("dev = (%d,%d), ino = %llu, pos = %llu, len = %u",
 		show_dev_ino(__entry),
 		(unsigned long long)__entry->pos,
 		__entry->len)
@@ -1333,9 +1333,9 @@ TRACE_EVENT(f2fs_write_end,
 	TP_ARGS(inode, pos, len, copied),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	pos)
+		__field(dev_t,	dev)
 		__field(unsigned int, len)
 		__field(unsigned int, copied)
 	),
@@ -1348,7 +1348,7 @@ TRACE_EVENT(f2fs_write_end,
 		__entry->copied	= copied;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pos = %llu, len = %u, copied = %u",
+	TP_printk("dev = (%d,%d), ino = %llu, pos = %llu, len = %u, copied = %u",
 		show_dev_ino(__entry),
 		(unsigned long long)__entry->pos,
 		__entry->len,
@@ -1362,12 +1362,12 @@ DECLARE_EVENT_CLASS(f2fs__folio,
 	TP_ARGS(folio, type),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
+		__field(pgoff_t, index)
+		__field(pgoff_t, nrpages)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(int, type)
 		__field(int, dir)
-		__field(pgoff_t, index)
-		__field(pgoff_t, nrpages)
 		__field(int, dirty)
 		__field(int, uptodate)
 	),
@@ -1383,7 +1383,7 @@ DECLARE_EVENT_CLASS(f2fs__folio,
 		__entry->uptodate = folio_test_uptodate(folio);
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, %s, %s, index = %lu, nr_pages = %lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, %s, %s, index = %lu, nr_pages = %lu, "
 		"dirty = %d, uptodate = %d",
 		show_dev_ino(__entry),
 		show_block_type(__entry->type),
@@ -1437,10 +1437,10 @@ TRACE_EVENT(f2fs_replace_atomic_write_block,
 	TP_ARGS(inode, cow_inode, index, old_addr, new_addr, recovery),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
-		__field(ino_t,	cow_ino)
+		__field(u64,	ino)
+		__field(u64,	cow_ino)
 		__field(pgoff_t, index)
+		__field(dev_t,	dev)
 		__field(block_t, old_addr)
 		__field(block_t, new_addr)
 		__field(bool, recovery)
@@ -1456,7 +1456,7 @@ TRACE_EVENT(f2fs_replace_atomic_write_block,
 		__entry->recovery	= recovery;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, cow_ino = %lu, index = %lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, cow_ino = %llu, index = %lu, "
 			"old_addr = 0x%llx, new_addr = 0x%llx, recovery = %d",
 		show_dev_ino(__entry),
 		__entry->cow_ino,
@@ -1474,10 +1474,10 @@ DECLARE_EVENT_CLASS(f2fs_mmap,
 	TP_ARGS(inode, index, flags, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(pgoff_t, index)
 		__field(vm_flags_t, flags)
+		__field(dev_t,	dev)
 		__field(vm_fault_t, ret)
 	),
 
@@ -1489,7 +1489,7 @@ DECLARE_EVENT_CLASS(f2fs_mmap,
 		__entry->ret	= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, index = %lu, flags: %s, ret: %s",
+	TP_printk("dev = (%d,%d), ino = %llu, index = %lu, flags: %s, ret: %s",
 		show_dev_ino(__entry),
 		(unsigned long)__entry->index,
 		__print_flags(__entry->flags, "|", FAULT_FLAG_TRACE),
@@ -1519,15 +1519,15 @@ TRACE_EVENT(f2fs_writepages,
 	TP_ARGS(inode, wbc, type),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
-		__field(int,	type)
-		__field(int,	dir)
-		__field(long,	nr_to_write)
-		__field(long,	pages_skipped)
+		__field(u64,	ino)
 		__field(loff_t,	range_start)
 		__field(loff_t,	range_end)
+		__field(long,	nr_to_write)
+		__field(long,	pages_skipped)
 		__field(pgoff_t, writeback_index)
+		__field(dev_t,	dev)
+		__field(int,	type)
+		__field(int,	dir)
 		__field(int,	sync_mode)
 		__field(char,	for_kupdate)
 		__field(char,	for_background)
@@ -1554,7 +1554,7 @@ TRACE_EVENT(f2fs_writepages,
 		__entry->for_sync	= wbc->for_sync;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, %s, %s, nr_to_write %ld, "
+	TP_printk("dev = (%d,%d), ino = %llu, %s, %s, nr_to_write %ld, "
 		"skipped %ld, start %lld, end %lld, wb_idx %lu, sync_mode %d, "
 		"kupdate %u background %u tagged %u cyclic %u sync %u",
 		show_dev_ino(__entry),
@@ -1580,9 +1580,9 @@ TRACE_EVENT(f2fs_readpages,
 	TP_ARGS(inode, start, nrpage),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(pgoff_t,	start)
+		__field(dev_t,	dev)
 		__field(unsigned int,	nrpage)
 	),
 
@@ -1593,7 +1593,7 @@ TRACE_EVENT(f2fs_readpages,
 		__entry->nrpage	= nrpage;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, start = %lu nrpage = %u",
+	TP_printk("dev = (%d,%d), ino = %llu, start = %lu nrpage = %u",
 		show_dev_ino(__entry),
 		(unsigned long)__entry->start,
 		__entry->nrpage)
@@ -1738,8 +1738,8 @@ TRACE_EVENT(f2fs_lookup_extent_tree_start,
 	TP_ARGS(inode, pgofs, type),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, pgofs)
 		__field(enum extent_type, type)
 	),
@@ -1751,7 +1751,7 @@ TRACE_EVENT(f2fs_lookup_extent_tree_start,
 		__entry->type = type;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, type = %s",
+	TP_printk("dev = (%d,%d), ino = %llu, pgofs = %u, type = %s",
 		show_dev_ino(__entry),
 		__entry->pgofs,
 		show_extent_type(__entry->type))
@@ -1767,8 +1767,8 @@ TRACE_EVENT_CONDITION(f2fs_lookup_read_extent_tree_end,
 	TP_CONDITION(ei),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, pgofs)
 		__field(unsigned int, fofs)
 		__field(unsigned int, len)
@@ -1784,7 +1784,7 @@ TRACE_EVENT_CONDITION(f2fs_lookup_read_extent_tree_end,
 		__entry->blk = ei->blk;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, "
+	TP_printk("dev = (%d,%d), ino = %llu, pgofs = %u, "
 		"read_ext_info(fofs: %u, len: %u, blk: %u)",
 		show_dev_ino(__entry),
 		__entry->pgofs,
@@ -1803,13 +1803,13 @@ TRACE_EVENT_CONDITION(f2fs_lookup_age_extent_tree_end,
 	TP_CONDITION(ei),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
+		__field(unsigned long long, age)
+		__field(unsigned long long, blocks)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, pgofs)
 		__field(unsigned int, fofs)
 		__field(unsigned int, len)
-		__field(unsigned long long, age)
-		__field(unsigned long long, blocks)
 	),
 
 	TP_fast_assign(
@@ -1822,7 +1822,7 @@ TRACE_EVENT_CONDITION(f2fs_lookup_age_extent_tree_end,
 		__entry->blocks = ei->last_blocks;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, "
+	TP_printk("dev = (%d,%d), ino = %llu, pgofs = %u, "
 		"age_ext_info(fofs: %u, len: %u, age: %llu, blocks: %llu)",
 		show_dev_ino(__entry),
 		__entry->pgofs,
@@ -1841,8 +1841,8 @@ TRACE_EVENT(f2fs_update_read_extent_tree_range,
 	TP_ARGS(inode, pgofs, len, blkaddr, c_len),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, pgofs)
 		__field(u32, blk)
 		__field(unsigned int, len)
@@ -1858,7 +1858,7 @@ TRACE_EVENT(f2fs_update_read_extent_tree_range,
 		__entry->c_len = c_len;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, "
+	TP_printk("dev = (%d,%d), ino = %llu, pgofs = %u, "
 				"len = %u, blkaddr = %u, c_len = %u",
 		show_dev_ino(__entry),
 		__entry->pgofs,
@@ -1876,12 +1876,12 @@ TRACE_EVENT(f2fs_update_age_extent_tree_range,
 	TP_ARGS(inode, pgofs, len, age, last_blks),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
+		__field(unsigned long long, age)
+		__field(unsigned long long, blocks)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, pgofs)
 		__field(unsigned int, len)
-		__field(unsigned long long, age)
-		__field(unsigned long long, blocks)
 	),
 
 	TP_fast_assign(
@@ -1893,7 +1893,7 @@ TRACE_EVENT(f2fs_update_age_extent_tree_range,
 		__entry->blocks = last_blks;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, "
+	TP_printk("dev = (%d,%d), ino = %llu, pgofs = %u, "
 				"len = %u, age = %llu, blocks = %llu",
 		show_dev_ino(__entry),
 		__entry->pgofs,
@@ -1938,8 +1938,8 @@ TRACE_EVENT(f2fs_destroy_extent_tree,
 	TP_ARGS(inode, node_cnt, type),
 
 	TP_STRUCT__entry(
+		__field(u64,	ino)
 		__field(dev_t,	dev)
-		__field(ino_t,	ino)
 		__field(unsigned int, node_cnt)
 		__field(enum extent_type, type)
 	),
@@ -1951,7 +1951,7 @@ TRACE_EVENT(f2fs_destroy_extent_tree,
 		__entry->type = type;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, destroyed: node_cnt = %u, type = %s",
+	TP_printk("dev = (%d,%d), ino = %llu, destroyed: node_cnt = %u, type = %s",
 		show_dev_ino(__entry),
 		__entry->node_cnt,
 		show_extent_type(__entry->type))
@@ -2027,9 +2027,9 @@ DECLARE_EVENT_CLASS(f2fs_zip_start,
 	TP_ARGS(inode, cluster_idx, cluster_size, algtype),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(pgoff_t, idx)
+		__field(dev_t,	dev)
 		__field(unsigned int, size)
 		__field(unsigned int, algtype)
 	),
@@ -2042,7 +2042,7 @@ DECLARE_EVENT_CLASS(f2fs_zip_start,
 		__entry->algtype = algtype;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, cluster_idx:%lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, cluster_idx:%lu, "
 		"cluster_size = %u, algorithm = %s",
 		show_dev_ino(__entry),
 		__entry->idx,
@@ -2058,9 +2058,9 @@ DECLARE_EVENT_CLASS(f2fs_zip_end,
 	TP_ARGS(inode, cluster_idx, compressed_size, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,	dev)
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(pgoff_t, idx)
+		__field(dev_t,	dev)
 		__field(unsigned int, size)
 		__field(unsigned int, ret)
 	),
@@ -2073,7 +2073,7 @@ DECLARE_EVENT_CLASS(f2fs_zip_end,
 		__entry->ret = ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, cluster_idx:%lu, "
+	TP_printk("dev = (%d,%d), ino = %llu, cluster_idx:%lu, "
 		"compressed_size = %u, ret = %d",
 		show_dev_ino(__entry),
 		__entry->idx,
@@ -2311,10 +2311,10 @@ TRACE_EVENT(f2fs_bmap,
 	TP_ARGS(inode, lblock, pblock),
 
 	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(sector_t, lblock)
 		__field(sector_t, pblock)
+		__field(dev_t, dev)
 	),
 
 	TP_fast_assign(
@@ -2324,7 +2324,7 @@ TRACE_EVENT(f2fs_bmap,
 		__entry->pblock		= pblock;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, lblock:%lld, pblock:%lld",
+	TP_printk("dev = (%d,%d), ino = %llu, lblock:%lld, pblock:%lld",
 		show_dev_ino(__entry),
 		(unsigned long long)__entry->lblock,
 		(unsigned long long)__entry->pblock)
@@ -2338,11 +2338,11 @@ TRACE_EVENT(f2fs_fiemap,
 	TP_ARGS(inode, lblock, pblock, len, flags, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(sector_t, lblock)
 		__field(sector_t, pblock)
 		__field(unsigned long long, len)
+		__field(dev_t, dev)
 		__field(unsigned int, flags)
 		__field(int, ret)
 	),
@@ -2357,7 +2357,7 @@ TRACE_EVENT(f2fs_fiemap,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, lblock:%lld, pblock:%lld, "
+	TP_printk("dev = (%d,%d), ino = %llu, lblock:%lld, pblock:%lld, "
 		"len:%llu, flags:%u, ret:%d",
 		show_dev_ino(__entry),
 		(unsigned long long)__entry->lblock,
@@ -2375,13 +2375,13 @@ DECLARE_EVENT_CLASS(f2fs__rw_start,
 	TP_ARGS(inode, offset, bytes, pid, pathname, command),
 
 	TP_STRUCT__entry(
-		__string(pathbuf, pathname)
+		__field(u64, ino)
 		__field(loff_t, offset)
-		__field(int, bytes)
 		__field(loff_t, i_size)
+		__string(pathbuf, pathname)
 		__string(cmdline, command)
 		__field(pid_t, pid)
-		__field(ino_t, ino)
+		__field(int, bytes)
 	),
 
 	TP_fast_assign(
@@ -2402,10 +2402,10 @@ DECLARE_EVENT_CLASS(f2fs__rw_start,
 	),
 
 	TP_printk("entry_name %s, offset %llu, bytes %d, cmdline %s,"
-		" pid %d, i_size %llu, ino %lu",
+		" pid %d, i_size %llu, ino %llu",
 		__get_str(pathbuf), __entry->offset, __entry->bytes,
 		__get_str(cmdline), __entry->pid, __entry->i_size,
-		(unsigned long) __entry->ino)
+		__entry->ino)
 );
 
 DECLARE_EVENT_CLASS(f2fs__rw_end,
@@ -2415,7 +2415,7 @@ DECLARE_EVENT_CLASS(f2fs__rw_end,
 	TP_ARGS(inode, offset, bytes),
 
 	TP_STRUCT__entry(
-		__field(ino_t,	ino)
+		__field(u64,	ino)
 		__field(loff_t,	offset)
 		__field(int,	bytes)
 	),
@@ -2426,8 +2426,8 @@ DECLARE_EVENT_CLASS(f2fs__rw_end,
 		__entry->bytes		= bytes;
 	),
 
-	TP_printk("ino %lu, offset %llu, bytes %d",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu, offset %llu, bytes %d",
+		__entry->ino,
 		__entry->offset, __entry->bytes)
 );
 
diff --git a/include/trace/events/filelock.h b/include/trace/events/filelock.h
index 370016c38a5bb..1167748862449 100644
--- a/include/trace/events/filelock.h
+++ b/include/trace/events/filelock.h
@@ -42,10 +42,10 @@ TRACE_EVENT(locks_get_lock_context,
 	TP_ARGS(inode, type, ctx),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
+		__field(struct file_lock_context *, ctx)
 		__field(dev_t, s_dev)
 		__field(unsigned char, type)
-		__field(struct file_lock_context *, ctx)
 	),
 
 	TP_fast_assign(
@@ -55,7 +55,7 @@ TRACE_EVENT(locks_get_lock_context,
 		__entry->ctx = ctx;
 	),
 
-	TP_printk("dev=0x%x:0x%x ino=0x%lx type=%s ctx=%p",
+	TP_printk("dev=0x%x:0x%x ino=0x%llx type=%s ctx=%p",
 		  MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 		  __entry->i_ino, show_fl_type(__entry->type), __entry->ctx)
 );
@@ -66,16 +66,16 @@ DECLARE_EVENT_CLASS(filelock_lock,
 	TP_ARGS(inode, fl, ret),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
+		__field(loff_t, fl_start)
+		__field(loff_t, fl_end)
 		__field(struct file_lock *, fl)
-		__field(unsigned long, i_ino)
-		__field(dev_t, s_dev)
 		__field(struct file_lock_core *, blocker)
 		__field(fl_owner_t, owner)
+		__field(dev_t, s_dev)
 		__field(unsigned int, pid)
 		__field(unsigned int, flags)
 		__field(unsigned char, type)
-		__field(loff_t, fl_start)
-		__field(loff_t, fl_end)
 		__field(int, ret)
 	),
 
@@ -93,7 +93,7 @@ DECLARE_EVENT_CLASS(filelock_lock,
 		__entry->ret = ret;
 	),
 
-	TP_printk("fl=%p dev=0x%x:0x%x ino=0x%lx fl_blocker=%p fl_owner=%p fl_pid=%u fl_flags=%s fl_type=%s fl_start=%lld fl_end=%lld ret=%d",
+	TP_printk("fl=%p dev=0x%x:0x%x ino=0x%llx fl_blocker=%p fl_owner=%p fl_pid=%u fl_flags=%s fl_type=%s fl_start=%lld fl_end=%lld ret=%d",
 		__entry->fl, MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 		__entry->i_ino, __entry->blocker, __entry->owner,
 		__entry->pid, show_fl_flags(__entry->flags),
@@ -123,15 +123,15 @@ DECLARE_EVENT_CLASS(filelock_lease,
 	TP_ARGS(inode, fl),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
 		__field(struct file_lease *, fl)
-		__field(unsigned long, i_ino)
-		__field(dev_t, s_dev)
 		__field(struct file_lock_core *, blocker)
 		__field(fl_owner_t, owner)
-		__field(unsigned int, flags)
-		__field(unsigned char, type)
 		__field(unsigned long, break_time)
 		__field(unsigned long, downgrade_time)
+		__field(dev_t, s_dev)
+		__field(unsigned int, flags)
+		__field(unsigned char, type)
 	),
 
 	TP_fast_assign(
@@ -146,7 +146,7 @@ DECLARE_EVENT_CLASS(filelock_lease,
 		__entry->downgrade_time = fl ? fl->fl_downgrade_time : 0;
 	),
 
-	TP_printk("fl=%p dev=0x%x:0x%x ino=0x%lx fl_blocker=%p fl_owner=%p fl_flags=%s fl_type=%s fl_break_time=%lu fl_downgrade_time=%lu",
+	TP_printk("fl=%p dev=0x%x:0x%x ino=0x%llx fl_blocker=%p fl_owner=%p fl_flags=%s fl_type=%s fl_break_time=%lu fl_downgrade_time=%lu",
 		__entry->fl, MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 		__entry->i_ino, __entry->blocker, __entry->owner,
 		show_fl_flags(__entry->flags),
@@ -175,12 +175,12 @@ TRACE_EVENT(generic_add_lease,
 	TP_ARGS(inode, fl),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
+		__field(fl_owner_t, owner)
+		__field(dev_t, s_dev)
 		__field(int, wcount)
 		__field(int, rcount)
 		__field(int, icount)
-		__field(dev_t, s_dev)
-		__field(fl_owner_t, owner)
 		__field(unsigned int, flags)
 		__field(unsigned char, type)
 	),
@@ -196,7 +196,7 @@ TRACE_EVENT(generic_add_lease,
 		__entry->type = fl->c.flc_type;
 	),
 
-	TP_printk("dev=0x%x:0x%x ino=0x%lx wcount=%d rcount=%d icount=%d fl_owner=%p fl_flags=%s fl_type=%s",
+	TP_printk("dev=0x%x:0x%x ino=0x%llx wcount=%d rcount=%d icount=%d fl_owner=%p fl_flags=%s fl_type=%s",
 		MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 		__entry->i_ino, __entry->wcount, __entry->rcount,
 		__entry->icount, __entry->owner,
diff --git a/include/trace/events/filemap.h b/include/trace/events/filemap.h
index f48fe637bfd25..4dcf8e9e2e0d8 100644
--- a/include/trace/events/filemap.h
+++ b/include/trace/events/filemap.h
@@ -20,8 +20,8 @@ DECLARE_EVENT_CLASS(mm_filemap_op_page_cache,
 	TP_ARGS(folio),
 
 	TP_STRUCT__entry(
+		__field(u64, i_ino)
 		__field(unsigned long, pfn)
-		__field(unsigned long, i_ino)
 		__field(unsigned long, index)
 		__field(dev_t, s_dev)
 		__field(unsigned char, order)
@@ -38,7 +38,7 @@ DECLARE_EVENT_CLASS(mm_filemap_op_page_cache,
 		__entry->order = folio_order(folio);
 	),
 
-	TP_printk("dev %d:%d ino %lx pfn=0x%lx ofs=%lu order=%u",
+	TP_printk("dev %d:%d ino %llx pfn=0x%lx ofs=%lu order=%u",
 		MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 		__entry->i_ino,
 		__entry->pfn,
@@ -67,7 +67,7 @@ DECLARE_EVENT_CLASS(mm_filemap_op_page_cache_range,
 	TP_ARGS(mapping, index, last_index),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
 		__field(dev_t, s_dev)
 		__field(unsigned long, index)
 		__field(unsigned long, last_index)
@@ -85,7 +85,7 @@ DECLARE_EVENT_CLASS(mm_filemap_op_page_cache_range,
 	),
 
 	TP_printk(
-		"dev=%d:%d ino=%lx ofs=%lld-%lld",
+		"dev=%d:%d ino=%llx ofs=%lld-%lld",
 		MAJOR(__entry->s_dev),
 		MINOR(__entry->s_dev), __entry->i_ino,
 		((loff_t)__entry->index) << PAGE_SHIFT,
@@ -117,7 +117,7 @@ TRACE_EVENT(mm_filemap_fault,
 	TP_ARGS(mapping, index),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
 		__field(dev_t, s_dev)
 		__field(unsigned long, index)
 	),
@@ -133,7 +133,7 @@ TRACE_EVENT(mm_filemap_fault,
 	),
 
 	TP_printk(
-		"dev=%d:%d ino=%lx ofs=%lld",
+		"dev=%d:%d ino=%llx ofs=%lld",
 		MAJOR(__entry->s_dev),
 		MINOR(__entry->s_dev), __entry->i_ino,
 		((loff_t)__entry->index) << PAGE_SHIFT
@@ -146,7 +146,7 @@ TRACE_EVENT(filemap_set_wb_err,
 		TP_ARGS(mapping, eseq),
 
 		TP_STRUCT__entry(
-			__field(unsigned long, i_ino)
+			__field(u64, i_ino)
 			__field(dev_t, s_dev)
 			__field(errseq_t, errseq)
 		),
@@ -160,7 +160,7 @@ TRACE_EVENT(filemap_set_wb_err,
 				__entry->s_dev = mapping->host->i_rdev;
 		),
 
-		TP_printk("dev=%d:%d ino=0x%lx errseq=0x%x",
+		TP_printk("dev=%d:%d ino=0x%llx errseq=0x%x",
 			MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
 			__entry->i_ino, __entry->errseq)
 );
@@ -171,8 +171,8 @@ TRACE_EVENT(file_check_and_advance_wb_err,
 		TP_ARGS(file, old),
 
 		TP_STRUCT__entry(
+			__field(u64, i_ino)
 			__field(struct file *, file)
-			__field(unsigned long, i_ino)
 			__field(dev_t, s_dev)
 			__field(errseq_t, old)
 			__field(errseq_t, new)
@@ -191,7 +191,7 @@ TRACE_EVENT(file_check_and_advance_wb_err,
 			__entry->new = file->f_wb_err;
 		),
 
-		TP_printk("file=%p dev=%d:%d ino=0x%lx old=0x%x new=0x%x",
+		TP_printk("file=%p dev=%d:%d ino=0x%llx old=0x%x new=0x%x",
 			__entry->file, MAJOR(__entry->s_dev),
 			MINOR(__entry->s_dev), __entry->i_ino, __entry->old,
 			__entry->new)
diff --git a/include/trace/events/fs_dax.h b/include/trace/events/fs_dax.h
index 50ebc1290ab06..11121baa8ece7 100644
--- a/include/trace/events/fs_dax.h
+++ b/include/trace/events/fs_dax.h
@@ -12,7 +12,7 @@ DECLARE_EVENT_CLASS(dax_pmd_fault_class,
 		pgoff_t max_pgoff, int result),
 	TP_ARGS(inode, vmf, max_pgoff, result),
 	TP_STRUCT__entry(
-		__field(unsigned long, ino)
+		__field(u64, ino)
 		__field(unsigned long, vm_start)
 		__field(unsigned long, vm_end)
 		__field(vm_flags_t, vm_flags)
@@ -35,7 +35,7 @@ DECLARE_EVENT_CLASS(dax_pmd_fault_class,
 		__entry->max_pgoff = max_pgoff;
 		__entry->result = result;
 	),
-	TP_printk("dev %d:%d ino %#lx %s %s address %#lx vm_start "
+	TP_printk("dev %d:%d ino %#llx %s %s address %#lx vm_start "
 			"%#lx vm_end %#lx pgoff %#lx max_pgoff %#lx %s",
 		MAJOR(__entry->dev),
 		MINOR(__entry->dev),
@@ -66,7 +66,7 @@ DECLARE_EVENT_CLASS(dax_pmd_load_hole_class,
 		void *radix_entry),
 	TP_ARGS(inode, vmf, zero_folio, radix_entry),
 	TP_STRUCT__entry(
-		__field(unsigned long, ino)
+		__field(u64, ino)
 		__field(vm_flags_t, vm_flags)
 		__field(unsigned long, address)
 		__field(struct folio *, zero_folio)
@@ -81,7 +81,7 @@ DECLARE_EVENT_CLASS(dax_pmd_load_hole_class,
 		__entry->zero_folio = zero_folio;
 		__entry->radix_entry = radix_entry;
 	),
-	TP_printk("dev %d:%d ino %#lx %s address %#lx zero_folio %p "
+	TP_printk("dev %d:%d ino %#llx %s address %#lx zero_folio %p "
 			"radix_entry %#lx",
 		MAJOR(__entry->dev),
 		MINOR(__entry->dev),
@@ -106,7 +106,7 @@ DECLARE_EVENT_CLASS(dax_pte_fault_class,
 	TP_PROTO(struct inode *inode, struct vm_fault *vmf, int result),
 	TP_ARGS(inode, vmf, result),
 	TP_STRUCT__entry(
-		__field(unsigned long, ino)
+		__field(u64, ino)
 		__field(vm_flags_t, vm_flags)
 		__field(unsigned long, address)
 		__field(pgoff_t, pgoff)
@@ -123,7 +123,7 @@ DECLARE_EVENT_CLASS(dax_pte_fault_class,
 		__entry->pgoff = vmf->pgoff;
 		__entry->result = result;
 	),
-	TP_printk("dev %d:%d ino %#lx %s %s address %#lx pgoff %#lx %s",
+	TP_printk("dev %d:%d ino %#llx %s %s address %#lx pgoff %#lx %s",
 		MAJOR(__entry->dev),
 		MINOR(__entry->dev),
 		__entry->ino,
@@ -150,7 +150,7 @@ DECLARE_EVENT_CLASS(dax_writeback_range_class,
 	TP_PROTO(struct inode *inode, pgoff_t start_index, pgoff_t end_index),
 	TP_ARGS(inode, start_index, end_index),
 	TP_STRUCT__entry(
-		__field(unsigned long, ino)
+		__field(u64, ino)
 		__field(pgoff_t, start_index)
 		__field(pgoff_t, end_index)
 		__field(dev_t, dev)
@@ -161,7 +161,7 @@ DECLARE_EVENT_CLASS(dax_writeback_range_class,
 		__entry->start_index = start_index;
 		__entry->end_index = end_index;
 	),
-	TP_printk("dev %d:%d ino %#lx pgoff %#lx-%#lx",
+	TP_printk("dev %d:%d ino %#llx pgoff %#lx-%#lx",
 		MAJOR(__entry->dev),
 		MINOR(__entry->dev),
 		__entry->ino,
@@ -182,7 +182,7 @@ TRACE_EVENT(dax_writeback_one,
 	TP_PROTO(struct inode *inode, pgoff_t pgoff, pgoff_t pglen),
 	TP_ARGS(inode, pgoff, pglen),
 	TP_STRUCT__entry(
-		__field(unsigned long, ino)
+		__field(u64, ino)
 		__field(pgoff_t, pgoff)
 		__field(pgoff_t, pglen)
 		__field(dev_t, dev)
@@ -193,7 +193,7 @@ TRACE_EVENT(dax_writeback_one,
 		__entry->pgoff = pgoff;
 		__entry->pglen = pglen;
 	),
-	TP_printk("dev %d:%d ino %#lx pgoff %#lx pglen %#lx",
+	TP_printk("dev %d:%d ino %#llx pgoff %#lx pglen %#lx",
 		MAJOR(__entry->dev),
 		MINOR(__entry->dev),
 		__entry->ino,
diff --git a/include/trace/events/fsverity.h b/include/trace/events/fsverity.h
index a8c52f21cbd5e..4477c17e05748 100644
--- a/include/trace/events/fsverity.h
+++ b/include/trace/events/fsverity.h
@@ -16,7 +16,7 @@ TRACE_EVENT(fsverity_enable,
 		 const struct merkle_tree_params *params),
 	TP_ARGS(inode, params),
 	TP_STRUCT__entry(
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(u64, data_size)
 		__field(u64, tree_size)
 		__field(unsigned int, merkle_block)
@@ -29,8 +29,8 @@ TRACE_EVENT(fsverity_enable,
 		__entry->merkle_block = params->block_size;
 		__entry->num_levels = params->num_levels;
 	),
-	TP_printk("ino %lu data_size %llu tree_size %llu merkle_block %u levels %u",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu data_size %llu tree_size %llu merkle_block %u levels %u",
+		__entry->ino,
 		__entry->data_size,
 		__entry->tree_size,
 		__entry->merkle_block,
@@ -42,7 +42,7 @@ TRACE_EVENT(fsverity_tree_done,
 		 const struct merkle_tree_params *params),
 	TP_ARGS(inode, vi, params),
 	TP_STRUCT__entry(
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(u64, data_size)
 		__field(u64, tree_size)
 		__field(unsigned int, merkle_block)
@@ -59,8 +59,8 @@ TRACE_EVENT(fsverity_tree_done,
 		memcpy(__get_dynamic_array(root_hash), vi->root_hash, __get_dynamic_array_len(root_hash));
 		memcpy(__get_dynamic_array(file_digest), vi->file_digest, __get_dynamic_array_len(file_digest));
 	),
-	TP_printk("ino %lu data_size %llu tree_size %lld merkle_block %u levels %u root_hash %s digest %s",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu data_size %llu tree_size %lld merkle_block %u levels %u root_hash %s digest %s",
+		__entry->ino,
 		__entry->data_size,
 		__entry->tree_size,
 		__entry->merkle_block,
@@ -75,7 +75,7 @@ TRACE_EVENT(fsverity_verify_data_block,
 		 u64 data_pos),
 	TP_ARGS(inode, params, data_pos),
 	TP_STRUCT__entry(
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(u64, data_pos)
 		__field(unsigned int, merkle_block)
 	),
@@ -84,8 +84,8 @@ TRACE_EVENT(fsverity_verify_data_block,
 		__entry->data_pos = data_pos;
 		__entry->merkle_block = params->block_size;
 	),
-	TP_printk("ino %lu data_pos %llu merkle_block %u",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu data_pos %llu merkle_block %u",
+		__entry->ino,
 		__entry->data_pos,
 		__entry->merkle_block)
 );
@@ -96,7 +96,7 @@ TRACE_EVENT(fsverity_merkle_hit,
 		 unsigned int hidx),
 	TP_ARGS(inode, data_pos, hblock_idx, level, hidx),
 	TP_STRUCT__entry(
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(u64, data_pos)
 		__field(unsigned long, hblock_idx)
 		__field(unsigned int, level)
@@ -109,8 +109,8 @@ TRACE_EVENT(fsverity_merkle_hit,
 		__entry->level = level;
 		__entry->hidx = hidx;
 	),
-	TP_printk("ino %lu data_pos %llu hblock_idx %lu level %u hidx %u",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu data_pos %llu hblock_idx %lu level %u hidx %u",
+		__entry->ino,
 		__entry->data_pos,
 		__entry->hblock_idx,
 		__entry->level,
@@ -122,7 +122,7 @@ TRACE_EVENT(fsverity_verify_merkle_block,
 		 unsigned int level, unsigned int hidx),
 	TP_ARGS(inode, hblock_idx, level, hidx),
 	TP_STRUCT__entry(
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(unsigned long, hblock_idx)
 		__field(unsigned int, level)
 		__field(unsigned int, hidx)
@@ -133,8 +133,8 @@ TRACE_EVENT(fsverity_verify_merkle_block,
 		__entry->level = level;
 		__entry->hidx = hidx;
 	),
-	TP_printk("ino %lu hblock_idx %lu level %u hidx %u",
-		(unsigned long) __entry->ino,
+	TP_printk("ino %llu hblock_idx %lu level %u hidx %u",
+		__entry->ino,
 		__entry->hblock_idx,
 		__entry->level,
 		__entry->hidx)
diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index 4e41bff318888..bcdc57eea2701 100644
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -25,6 +25,7 @@
 	EM( SCAN_PAGE_LRU,		"page_not_in_lru")		\
 	EM( SCAN_PAGE_LOCK,		"page_locked")			\
 	EM( SCAN_PAGE_ANON,		"page_not_anon")		\
+	EM( SCAN_PAGE_LAZYFREE,		"page_lazyfree")		\
 	EM( SCAN_PAGE_COMPOUND,		"page_compound")		\
 	EM( SCAN_ANY_PROCESS,		"no_process_for_page")		\
 	EM( SCAN_VMA_NULL,		"vma_null")			\
@@ -237,5 +238,30 @@ TRACE_EVENT(mm_khugepaged_collapse_file,
 		__print_symbolic(__entry->result, SCAN_STATUS))
 );
 
+TRACE_EVENT(mm_khugepaged_scan,
+
+	TP_PROTO(struct mm_struct *mm, unsigned int progress,
+		 bool full_scan_finished),
+
+	TP_ARGS(mm, progress, full_scan_finished),
+
+	TP_STRUCT__entry(
+		__field(struct mm_struct *, mm)
+		__field(unsigned int, progress)
+		__field(bool, full_scan_finished)
+	),
+
+	TP_fast_assign(
+		__entry->mm = mm;
+		__entry->progress = progress;
+		__entry->full_scan_finished = full_scan_finished;
+	),
+
+	TP_printk("mm=%p, progress=%u, full_scan_finished=%d",
+		__entry->mm,
+		__entry->progress,
+		__entry->full_scan_finished)
+);
+
 #endif /* __HUGE_MEMORY_H */
 #include <trace/define_trace.h>
diff --git a/include/trace/events/hugetlbfs.h b/include/trace/events/hugetlbfs.h
index 59605dfaeeb43..8ba72c1d4f4d8 100644
--- a/include/trace/events/hugetlbfs.h
+++ b/include/trace/events/hugetlbfs.h
@@ -14,9 +14,9 @@ TRACE_EVENT(hugetlbfs_alloc_inode,
 	TP_ARGS(inode, dir, mode),
 
 	TP_STRUCT__entry(
+		__field(u64,		ino)
+		__field(u64,		dir)
 		__field(dev_t,		dev)
-		__field(ino_t,		ino)
-		__field(ino_t,		dir)
 		__field(__u16,		mode)
 	),
 
@@ -27,10 +27,10 @@ TRACE_EVENT(hugetlbfs_alloc_inode,
 		__entry->mode		= mode;
 	),
 
-	TP_printk("dev %d,%d ino %lu dir %lu mode 0%o",
+	TP_printk("dev %d,%d ino %llu dir %llu mode 0%o",
 		MAJOR(__entry->dev), MINOR(__entry->dev),
-		(unsigned long) __entry->ino,
-		(unsigned long) __entry->dir, __entry->mode)
+		__entry->ino,
+		__entry->dir, __entry->mode)
 );
 
 DECLARE_EVENT_CLASS(hugetlbfs__inode,
@@ -40,13 +40,13 @@ DECLARE_EVENT_CLASS(hugetlbfs__inode,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
-		__field(dev_t,		dev)
-		__field(ino_t,		ino)
-		__field(__u16,		mode)
+		__field(u64,		ino)
 		__field(loff_t,		size)
+		__field(blkcnt_t,	blocks)
+		__field(dev_t,		dev)
 		__field(unsigned int,	nlink)
 		__field(unsigned int,	seals)
-		__field(blkcnt_t,	blocks)
+		__field(__u16,		mode)
 	),
 
 	TP_fast_assign(
@@ -59,8 +59,8 @@ DECLARE_EVENT_CLASS(hugetlbfs__inode,
 		__entry->blocks		= inode->i_blocks;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o size %lld nlink %u seals %u blocks %llu",
-		MAJOR(__entry->dev), MINOR(__entry->dev), (unsigned long) __entry->ino,
+	TP_printk("dev %d,%d ino %llu mode 0%o size %lld nlink %u seals %u blocks %llu",
+		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino,
 		__entry->mode, __entry->size, __entry->nlink, __entry->seals,
 		(unsigned long long)__entry->blocks)
 );
@@ -87,14 +87,14 @@ TRACE_EVENT(hugetlbfs_setattr,
 	TP_ARGS(inode, dentry, attr),
 
 	TP_STRUCT__entry(
+		__field(u64,		ino)
+		__field(loff_t,		old_size)
+		__field(loff_t,		ia_size)
 		__field(dev_t,		dev)
-		__field(ino_t,		ino)
 		__field(unsigned int,	d_len)
 		__string(d_name,	dentry->d_name.name)
 		__field(unsigned int,	ia_valid)
 		__field(unsigned int,	ia_mode)
-		__field(loff_t,		old_size)
-		__field(loff_t,		ia_size)
 	),
 
 	TP_fast_assign(
@@ -108,8 +108,8 @@ TRACE_EVENT(hugetlbfs_setattr,
 		__entry->ia_size	= attr->ia_size;
 	),
 
-	TP_printk("dev %d,%d ino %lu name %.*s valid %#x mode 0%o old_size %lld size %lld",
-		MAJOR(__entry->dev), MINOR(__entry->dev), (unsigned long)__entry->ino,
+	TP_printk("dev %d,%d ino %llu name %.*s valid %#x mode 0%o old_size %lld size %lld",
+		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino,
 		__entry->d_len, __get_str(d_name), __entry->ia_valid, __entry->ia_mode,
 		__entry->old_size, __entry->ia_size)
 );
@@ -122,12 +122,12 @@ TRACE_EVENT(hugetlbfs_fallocate,
 	TP_ARGS(inode, mode, offset, len, ret),
 
 	TP_STRUCT__entry(
-		__field(dev_t,		dev)
-		__field(ino_t,		ino)
-		__field(int,		mode)
+		__field(u64,		ino)
 		__field(loff_t,		offset)
 		__field(loff_t,		len)
 		__field(loff_t,		size)
+		__field(dev_t,		dev)
+		__field(int,		mode)
 		__field(int,		ret)
 	),
 
@@ -141,9 +141,9 @@ TRACE_EVENT(hugetlbfs_fallocate,
 		__entry->ret		= ret;
 	),
 
-	TP_printk("dev %d,%d ino %lu mode 0%o offset %lld len %lld size %lld ret %d",
+	TP_printk("dev %d,%d ino %llu mode 0%o offset %lld len %lld size %lld ret %d",
 		MAJOR(__entry->dev), MINOR(__entry->dev),
-		(unsigned long)__entry->ino, __entry->mode,
+		__entry->ino, __entry->mode,
 		(unsigned long long)__entry->offset,
 		(unsigned long long)__entry->len,
 		(unsigned long long)__entry->size,
diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h
index cd7920c81f85b..290ccb9fd25dd 100644
--- a/include/trace/events/kmem.h
+++ b/include/trace/events/kmem.h
@@ -448,7 +448,7 @@ TRACE_EVENT(rss_stat,
 		 */
 		__entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD);
 		__entry->member = member;
-		__entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member])
+		__entry->size = (percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member])
 							    << PAGE_SHIFT);
 	),
 
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index 2d366be46a1c3..8add6d35e648b 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -300,7 +300,7 @@ TRACE_EVENT(netfs_read,
 		    __field(loff_t,			start)
 		    __field(size_t,			len)
 		    __field(enum netfs_read_trace,	what)
-		    __field(unsigned int,		netfs_inode)
+		    __field(u64,			netfs_inode)
 			     ),
 
 	    TP_fast_assign(
@@ -313,7 +313,7 @@ TRACE_EVENT(netfs_read,
 		    __entry->netfs_inode = rreq->inode->i_ino;
 			   ),
 
-	    TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx",
+	    TP_printk("R=%08x %s c=%08x ni=%llx s=%llx l=%zx sz=%llx",
 		      __entry->rreq,
 		      __print_symbolic(__entry->what, netfs_read_traces),
 		      __entry->cookie,
@@ -486,7 +486,7 @@ TRACE_EVENT(netfs_folio,
 	    TP_ARGS(folio, why),
 
 	    TP_STRUCT__entry(
-		    __field(ino_t,			ino)
+		    __field(u64,			ino)
 		    __field(pgoff_t,			index)
 		    __field(unsigned int,		nr)
 		    __field(enum netfs_folio_trace,	why)
@@ -500,7 +500,7 @@ TRACE_EVENT(netfs_folio,
 		    __entry->nr = folio_nr_pages(folio);
 			   ),
 
-	    TP_printk("i=%05lx ix=%05lx-%05lx %s",
+	    TP_printk("i=%05llx ix=%05lx-%05lx %s",
 		      __entry->ino, __entry->index, __entry->index + __entry->nr - 1,
 		      __print_symbolic(__entry->why, netfs_folio_traces))
 	    );
diff --git a/include/trace/events/nilfs2.h b/include/trace/events/nilfs2.h
index 8880c11733dd3..49efbd209585b 100644
--- a/include/trace/events/nilfs2.h
+++ b/include/trace/events/nilfs2.h
@@ -165,14 +165,14 @@ TRACE_EVENT(nilfs2_segment_usage_freed,
 
 TRACE_EVENT(nilfs2_mdt_insert_new_block,
 	    TP_PROTO(struct inode *inode,
-		     unsigned long ino,
+		     u64 ino,
 		     unsigned long block),
 
 	    TP_ARGS(inode, ino, block),
 
 	    TP_STRUCT__entry(
+		    __field(u64, ino)
 		    __field(struct inode *, inode)
-		    __field(unsigned long, ino)
 		    __field(unsigned long, block)
 	    ),
 
@@ -182,7 +182,7 @@ TRACE_EVENT(nilfs2_mdt_insert_new_block,
 		    __entry->block = block;
 		    ),
 
-	    TP_printk("inode = %p ino = %lu block = %lu",
+	    TP_printk("inode = %p ino = %llu block = %lu",
 		      __entry->inode,
 		      __entry->ino,
 		      __entry->block)
@@ -190,15 +190,15 @@ TRACE_EVENT(nilfs2_mdt_insert_new_block,
 
 TRACE_EVENT(nilfs2_mdt_submit_block,
 	    TP_PROTO(struct inode *inode,
-		     unsigned long ino,
+		     u64 ino,
 		     unsigned long blkoff,
 		     enum req_op mode),
 
 	    TP_ARGS(inode, ino, blkoff, mode),
 
 	    TP_STRUCT__entry(
+		    __field(u64, ino)
 		    __field(struct inode *, inode)
-		    __field(unsigned long, ino)
 		    __field(unsigned long, blkoff)
 		    /*
 		     * Use field_struct() to avoid is_signed_type() on the
@@ -214,7 +214,7 @@ TRACE_EVENT(nilfs2_mdt_submit_block,
 		    __entry->mode = mode;
 		    ),
 
-	    TP_printk("inode = %p ino = %lu blkoff = %lu mode = %x",
+	    TP_printk("inode = %p ino = %llu blkoff = %lu mode = %x",
 		      __entry->inode,
 		      __entry->ino,
 		      __entry->blkoff,
diff --git a/include/trace/events/qdisc.h b/include/trace/events/qdisc.h
index ff33f41a9db70..d8a5c26774708 100644
--- a/include/trace/events/qdisc.h
+++ b/include/trace/events/qdisc.h
@@ -74,6 +74,57 @@ TRACE_EVENT(qdisc_enqueue,
 		  __entry->ifindex, __entry->handle, __entry->parent, __entry->skbaddr)
 );
 
+#undef FN
+#undef FNe
+#define FN(reason)	TRACE_DEFINE_ENUM(QDISC_DROP_##reason);
+#define FNe(reason)	TRACE_DEFINE_ENUM(QDISC_DROP_##reason);
+DEFINE_QDISC_DROP_REASON(FN, FNe)
+
+#undef FN
+#undef FNe
+#define FN(reason)	{ QDISC_DROP_##reason, #reason },
+#define FNe(reason)	{ QDISC_DROP_##reason, #reason }
+
+TRACE_EVENT(qdisc_drop,
+
+	TP_PROTO(struct Qdisc *qdisc, const struct netdev_queue *txq,
+		 struct net_device *dev, struct sk_buff *skb,
+		 enum qdisc_drop_reason reason),
+
+	TP_ARGS(qdisc, txq, dev, skb, reason),
+
+	TP_STRUCT__entry(
+		__field(struct Qdisc *, qdisc)
+		__field(const struct netdev_queue *, txq)
+		__field(void *,	skbaddr)
+		__field(int, ifindex)
+		__field(u32, handle)
+		__field(u32, parent)
+		__field(enum qdisc_drop_reason, reason)
+		__string(kind, qdisc->ops->id)
+	),
+
+	TP_fast_assign(
+		__entry->qdisc = qdisc;
+		__entry->txq	 = txq;
+		__entry->skbaddr = skb;
+		__entry->ifindex = dev ? dev->ifindex : 0;
+		__entry->handle	 = qdisc->handle;
+		__entry->parent	 = qdisc->parent;
+		__entry->reason	 = reason;
+		__assign_str(kind);
+	),
+
+	TP_printk("drop ifindex=%d kind=%s handle=0x%X parent=0x%X skbaddr=%p reason=%s",
+		  __entry->ifindex, __get_str(kind), __entry->handle,
+		  __entry->parent, __entry->skbaddr,
+		  __print_symbolic(__entry->reason,
+				   DEFINE_QDISC_DROP_REASON(FN, FNe)))
+);
+
+#undef FN
+#undef FNe
+
 TRACE_EVENT(qdisc_reset,
 
 	TP_PROTO(struct Qdisc *q),
diff --git a/include/trace/events/readahead.h b/include/trace/events/readahead.h
index 0997ac5eceab1..087f171e2b02d 100644
--- a/include/trace/events/readahead.h
+++ b/include/trace/events/readahead.h
@@ -18,7 +18,7 @@ TRACE_EVENT(page_cache_ra_unbounded,
 	TP_ARGS(inode, index, nr_to_read, lookahead_size),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
 		__field(dev_t, s_dev)
 		__field(pgoff_t, index)
 		__field(unsigned long, nr_to_read)
@@ -34,7 +34,7 @@ TRACE_EVENT(page_cache_ra_unbounded,
 	),
 
 	TP_printk(
-		"dev=%d:%d ino=%lx index=%lu nr_to_read=%lu lookahead_size=%lu",
+		"dev=%d:%d ino=%llx index=%lu nr_to_read=%lu lookahead_size=%lu",
 		MAJOR(__entry->s_dev), MINOR(__entry->s_dev), __entry->i_ino,
 		__entry->index, __entry->nr_to_read, __entry->lookahead_size
 	)
@@ -46,7 +46,7 @@ TRACE_EVENT(page_cache_ra_order,
 	TP_ARGS(inode, index, ra),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
+		__field(u64, i_ino)
 		__field(dev_t, s_dev)
 		__field(pgoff_t, index)
 		__field(unsigned int, order)
@@ -66,7 +66,7 @@ TRACE_EVENT(page_cache_ra_order,
 	),
 
 	TP_printk(
-		"dev=%d:%d ino=%lx index=%lu order=%u size=%u async_size=%u ra_pages=%u",
+		"dev=%d:%d ino=%llx index=%lu order=%u size=%u async_size=%u ra_pages=%u",
 		MAJOR(__entry->s_dev), MINOR(__entry->s_dev), __entry->i_ino,
 		__entry->index, __entry->order, __entry->size,
 		__entry->async_size, __entry->ra_pages
@@ -80,16 +80,16 @@ DECLARE_EVENT_CLASS(page_cache_ra_op,
 	TP_ARGS(inode, index, ra, req_count),
 
 	TP_STRUCT__entry(
-		__field(unsigned long, i_ino)
-		__field(dev_t, s_dev)
+		__field(u64, i_ino)
+		__field(loff_t, prev_pos)
 		__field(pgoff_t, index)
+		__field(unsigned long, req_count)
+		__field(dev_t, s_dev)
 		__field(unsigned int, order)
 		__field(unsigned int, size)
 		__field(unsigned int, async_size)
 		__field(unsigned int, ra_pages)
 		__field(unsigned int, mmap_miss)
-		__field(loff_t, prev_pos)
-		__field(unsigned long, req_count)
 	),
 
 	TP_fast_assign(
@@ -106,7 +106,7 @@ DECLARE_EVENT_CLASS(page_cache_ra_op,
 	),
 
 	TP_printk(
-		"dev=%d:%d ino=%lx index=%lu req_count=%lu order=%u size=%u async_size=%u ra_pages=%u mmap_miss=%u prev_pos=%lld",
+		"dev=%d:%d ino=%llx index=%lu req_count=%lu order=%u size=%u async_size=%u ra_pages=%u mmap_miss=%u prev_pos=%lld",
 		MAJOR(__entry->s_dev), MINOR(__entry->s_dev), __entry->i_ino,
 		__entry->index, __entry->req_count, __entry->order,
 		__entry->size, __entry->async_size, __entry->ra_pages,
diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h
index 750ecce569306..ff855197880de 100644
--- a/include/trace/events/sunrpc.h
+++ b/include/trace/events/sunrpc.h
@@ -1933,7 +1933,9 @@ TRACE_EVENT(svc_stats_latency,
 	svc_xprt_flag(CONG_CTRL)					\
 	svc_xprt_flag(HANDSHAKE)					\
 	svc_xprt_flag(TLS_SESSION)					\
-	svc_xprt_flag_end(PEER_AUTH)
+	svc_xprt_flag(PEER_AUTH)					\
+	svc_xprt_flag(PEER_VALID)					\
+	svc_xprt_flag_end(RPCB_UNREG)
 
 #undef svc_xprt_flag
 #undef svc_xprt_flag_end
diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h
index 1641ae3e6ca06..07cbb9836b91e 100644
--- a/include/trace/events/timer.h
+++ b/include/trace/events/timer.h
@@ -218,12 +218,13 @@ TRACE_EVENT(hrtimer_setup,
  * hrtimer_start - called when the hrtimer is started
  * @hrtimer:	pointer to struct hrtimer
  * @mode:	the hrtimers mode
+ * @was_armed:	Was armed when hrtimer_start*() was invoked
  */
 TRACE_EVENT(hrtimer_start,
 
-	TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode),
+	TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed),
 
-	TP_ARGS(hrtimer, mode),
+	TP_ARGS(hrtimer, mode, was_armed),
 
 	TP_STRUCT__entry(
 		__field( void *,	hrtimer		)
@@ -231,6 +232,7 @@ TRACE_EVENT(hrtimer_start,
 		__field( s64,		expires		)
 		__field( s64,		softexpires	)
 		__field( enum hrtimer_mode,	mode	)
+		__field( bool,		was_armed	)
 	),
 
 	TP_fast_assign(
@@ -239,26 +241,26 @@ TRACE_EVENT(hrtimer_start,
 		__entry->expires	= hrtimer_get_expires(hrtimer);
 		__entry->softexpires	= hrtimer_get_softexpires(hrtimer);
 		__entry->mode		= mode;
+		__entry->was_armed	= was_armed;
 	),
 
 	TP_printk("hrtimer=%p function=%ps expires=%llu softexpires=%llu "
-		  "mode=%s", __entry->hrtimer, __entry->function,
+		  "mode=%s was_armed=%d", __entry->hrtimer, __entry->function,
 		  (unsigned long long) __entry->expires,
 		  (unsigned long long) __entry->softexpires,
-		  decode_hrtimer_mode(__entry->mode))
+		  decode_hrtimer_mode(__entry->mode), __entry->was_armed)
 );
 
 /**
  * hrtimer_expire_entry - called immediately before the hrtimer callback
  * @hrtimer:	pointer to struct hrtimer
- * @now:	pointer to variable which contains current time of the
- *		timers base.
+ * @now:	variable which contains current time of the timers base.
  *
  * Allows to determine the timer latency.
  */
 TRACE_EVENT(hrtimer_expire_entry,
 
-	TP_PROTO(struct hrtimer *hrtimer, ktime_t *now),
+	TP_PROTO(struct hrtimer *hrtimer, ktime_t now),
 
 	TP_ARGS(hrtimer, now),
 
@@ -270,7 +272,7 @@ TRACE_EVENT(hrtimer_expire_entry,
 
 	TP_fast_assign(
 		__entry->hrtimer	= hrtimer;
-		__entry->now		= *now;
+		__entry->now		= now;
 		__entry->function	= ACCESS_PRIVATE(hrtimer, function);
 	),
 
@@ -321,6 +323,30 @@ DEFINE_EVENT(hrtimer_class, hrtimer_cancel,
 	TP_ARGS(hrtimer)
 );
 
+/**
+ * hrtimer_rearm - Invoked when the clockevent device is rearmed
+ * @next_event:	The next expiry time (CLOCK_MONOTONIC)
+ */
+TRACE_EVENT(hrtimer_rearm,
+
+	TP_PROTO(ktime_t next_event, bool deferred),
+
+	TP_ARGS(next_event, deferred),
+
+	TP_STRUCT__entry(
+		__field( s64,		next_event	)
+		__field( bool,		deferred	)
+	),
+
+	TP_fast_assign(
+		__entry->next_event	= next_event;
+		__entry->deferred	= deferred;
+	),
+
+	TP_printk("next_event=%llu deferred=%d",
+		  (unsigned long long) __entry->next_event, __entry->deferred)
+);
+
 /**
  * itimer_state - called when itimer is started or canceled
  * @which:	name of the interval timer
diff --git a/include/trace/events/timestamp.h b/include/trace/events/timestamp.h
index c9e5ec9300548..d6503612dddf9 100644
--- a/include/trace/events/timestamp.h
+++ b/include/trace/events/timestamp.h
@@ -18,9 +18,9 @@ DECLARE_EVENT_CLASS(ctime,
 	TP_ARGS(inode, ctime),
 
 	TP_STRUCT__entry(
-		__field(dev_t,		dev)
-		__field(ino_t,		ino)
+		__field(u64,		ino)
 		__field(time64_t,	ctime_s)
+		__field(dev_t,		dev)
 		__field(u32,		ctime_ns)
 		__field(u32,		gen)
 	),
@@ -33,7 +33,7 @@ DECLARE_EVENT_CLASS(ctime,
 		__entry->ctime_ns	= ctime->tv_nsec;
 	),
 
-	TP_printk("ino=%d:%d:%ld:%u ctime=%lld.%u",
+	TP_printk("ino=%d:%d:%llu:%u ctime=%lld.%u",
 		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->gen,
 		__entry->ctime_s, __entry->ctime_ns
 	)
@@ -58,8 +58,8 @@ TRACE_EVENT(ctime_ns_xchg,
 	TP_ARGS(inode, old, new, cur),
 
 	TP_STRUCT__entry(
+		__field(u64,		ino)
 		__field(dev_t,		dev)
-		__field(ino_t,		ino)
 		__field(u32,		gen)
 		__field(u32,		old)
 		__field(u32,		new)
@@ -75,7 +75,7 @@ TRACE_EVENT(ctime_ns_xchg,
 		__entry->cur		= cur;
 	),
 
-	TP_printk("ino=%d:%d:%ld:%u old=%u:%s new=%u cur=%u:%s",
+	TP_printk("ino=%d:%d:%llu:%u old=%u:%s new=%u cur=%u:%s",
 		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->gen,
 		__entry->old & ~I_CTIME_QUERIED,
 		__print_flags(__entry->old & I_CTIME_QUERIED, "|", CTIME_QUERIED_FLAGS),
@@ -93,10 +93,10 @@ TRACE_EVENT(fill_mg_cmtime,
 	TP_ARGS(inode, ctime, mtime),
 
 	TP_STRUCT__entry(
-		__field(dev_t,		dev)
-		__field(ino_t,		ino)
+		__field(u64,		ino)
 		__field(time64_t,	ctime_s)
 		__field(time64_t,	mtime_s)
+		__field(dev_t,		dev)
 		__field(u32,		ctime_ns)
 		__field(u32,		mtime_ns)
 		__field(u32,		gen)
@@ -112,7 +112,7 @@ TRACE_EVENT(fill_mg_cmtime,
 		__entry->mtime_ns	= mtime->tv_nsec;
 	),
 
-	TP_printk("ino=%d:%d:%ld:%u ctime=%lld.%u mtime=%lld.%u",
+	TP_printk("ino=%d:%d:%llu:%u ctime=%lld.%u mtime=%lld.%u",
 		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->gen,
 		__entry->ctime_s, __entry->ctime_ns,
 		__entry->mtime_s, __entry->mtime_ns
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index ea58e4656abf7..4445a8d9218de 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -124,85 +124,96 @@ TRACE_EVENT(mm_vmscan_wakeup_kswapd,
 
 DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_begin_template,
 
-	TP_PROTO(int order, gfp_t gfp_flags),
+	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
 
-	TP_ARGS(order, gfp_flags),
+	TP_ARGS(gfp_flags, order, memcg),
 
 	TP_STRUCT__entry(
-		__field(	int,	order		)
 		__field(	unsigned long,	gfp_flags	)
+		__field(	u64,	memcg_id	)
+		__field(	int,	order		)
 	),
 
 	TP_fast_assign(
-		__entry->order		= order;
 		__entry->gfp_flags	= (__force unsigned long)gfp_flags;
+		__entry->order		= order;
+		__entry->memcg_id	= mem_cgroup_id(memcg);
 	),
 
-	TP_printk("order=%d gfp_flags=%s",
+	TP_printk("order=%d gfp_flags=%s pid=%d memcg_id=%llu %s",
 		__entry->order,
-		show_gfp_flags(__entry->gfp_flags))
+		show_gfp_flags(__entry->gfp_flags),
+		__entry->ent.pid,
+		__entry->memcg_id,
+		__event_in_irq() ? "(in-irq)" : "")
 );
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_direct_reclaim_begin,
 
-	TP_PROTO(int order, gfp_t gfp_flags),
+	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
 
-	TP_ARGS(order, gfp_flags)
+	TP_ARGS(gfp_flags, order, memcg)
 );
 
 #ifdef CONFIG_MEMCG
 DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_reclaim_begin,
 
-	TP_PROTO(int order, gfp_t gfp_flags),
+	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
 
-	TP_ARGS(order, gfp_flags)
+	TP_ARGS(gfp_flags, order, memcg)
 );
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_softlimit_reclaim_begin,
 
-	TP_PROTO(int order, gfp_t gfp_flags),
+	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
 
-	TP_ARGS(order, gfp_flags)
+	TP_ARGS(gfp_flags, order, memcg)
 );
 #endif /* CONFIG_MEMCG */
 
 DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_end_template,
 
-	TP_PROTO(unsigned long nr_reclaimed),
+	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
 
-	TP_ARGS(nr_reclaimed),
+	TP_ARGS(nr_reclaimed, memcg),
 
 	TP_STRUCT__entry(
 		__field(	unsigned long,	nr_reclaimed	)
+		__field(	u64,	memcg_id	)
 	),
 
 	TP_fast_assign(
 		__entry->nr_reclaimed	= nr_reclaimed;
+		__entry->memcg_id	= mem_cgroup_id(memcg);
 	),
 
-	TP_printk("nr_reclaimed=%lu", __entry->nr_reclaimed)
+	TP_printk("nr_reclaimed=%lu pid=%d memcg_id=%llu %s",
+		__entry->nr_reclaimed,
+		__entry->ent.pid,
+		__entry->memcg_id,
+		__event_in_irq() ? "(in-irq)" : "")
 );
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_direct_reclaim_end,
 
-	TP_PROTO(unsigned long nr_reclaimed),
+	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
 
-	TP_ARGS(nr_reclaimed)
+	TP_ARGS(nr_reclaimed, memcg)
 );
 
 #ifdef CONFIG_MEMCG
 DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_reclaim_end,
 
-	TP_PROTO(unsigned long nr_reclaimed),
+	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
 
-	TP_ARGS(nr_reclaimed)
+	TP_ARGS(nr_reclaimed, memcg)
 );
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_softlimit_reclaim_end,
 
-	TP_PROTO(unsigned long nr_reclaimed),
+	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
 
-	TP_ARGS(nr_reclaimed)
+	TP_ARGS(nr_reclaimed, memcg)
 );
 #endif /* CONFIG_MEMCG */
 
@@ -210,82 +221,92 @@ TRACE_EVENT(mm_shrink_slab_start,
 	TP_PROTO(struct shrinker *shr, struct shrink_control *sc,
 		long nr_objects_to_shrink, unsigned long cache_items,
 		unsigned long long delta, unsigned long total_scan,
-		int priority),
+		int priority, struct mem_cgroup *memcg),
 
 	TP_ARGS(shr, sc, nr_objects_to_shrink, cache_items, delta, total_scan,
-		priority),
+		priority, memcg),
 
 	TP_STRUCT__entry(
 		__field(struct shrinker *, shr)
 		__field(void *, shrink)
-		__field(int, nid)
 		__field(long, nr_objects_to_shrink)
 		__field(unsigned long, gfp_flags)
 		__field(unsigned long, cache_items)
 		__field(unsigned long long, delta)
 		__field(unsigned long, total_scan)
 		__field(int, priority)
+		__field(int, nid)
+		__field(u64, memcg_id)
 	),
 
 	TP_fast_assign(
 		__entry->shr = shr;
 		__entry->shrink = shr->scan_objects;
-		__entry->nid = sc->nid;
 		__entry->nr_objects_to_shrink = nr_objects_to_shrink;
 		__entry->gfp_flags = (__force unsigned long)sc->gfp_mask;
 		__entry->cache_items = cache_items;
 		__entry->delta = delta;
 		__entry->total_scan = total_scan;
 		__entry->priority = priority;
+		__entry->nid = sc->nid;
+		__entry->memcg_id = mem_cgroup_id(memcg);
 	),
 
-	TP_printk("%pS %p: nid: %d objects to shrink %ld gfp_flags %s cache items %ld delta %lld total_scan %ld priority %d",
+	TP_printk("%pS %p: nid: %d pid: %d memcg_id: %llu objects to shrink %ld gfp_flags %s cache items %ld delta %lld total_scan %ld priority %d %s",
 		__entry->shrink,
 		__entry->shr,
 		__entry->nid,
+		__entry->ent.pid,
+		__entry->memcg_id,
 		__entry->nr_objects_to_shrink,
 		show_gfp_flags(__entry->gfp_flags),
 		__entry->cache_items,
 		__entry->delta,
 		__entry->total_scan,
-		__entry->priority)
+		__entry->priority,
+		__event_in_irq() ? "(in-irq)" : "")
 );
 
 TRACE_EVENT(mm_shrink_slab_end,
 	TP_PROTO(struct shrinker *shr, int nid, int shrinker_retval,
-		long unused_scan_cnt, long new_scan_cnt, long total_scan),
+		long unused_scan_cnt, long new_scan_cnt, long total_scan, struct mem_cgroup *memcg),
 
 	TP_ARGS(shr, nid, shrinker_retval, unused_scan_cnt, new_scan_cnt,
-		total_scan),
+		total_scan, memcg),
 
 	TP_STRUCT__entry(
 		__field(struct shrinker *, shr)
-		__field(int, nid)
 		__field(void *, shrink)
 		__field(long, unused_scan)
 		__field(long, new_scan)
-		__field(int, retval)
 		__field(long, total_scan)
+		__field(int, nid)
+		__field(int, retval)
+		__field(u64, memcg_id)
 	),
 
 	TP_fast_assign(
 		__entry->shr = shr;
-		__entry->nid = nid;
 		__entry->shrink = shr->scan_objects;
 		__entry->unused_scan = unused_scan_cnt;
 		__entry->new_scan = new_scan_cnt;
-		__entry->retval = shrinker_retval;
 		__entry->total_scan = total_scan;
+		__entry->nid = nid;
+		__entry->retval = shrinker_retval;
+		__entry->memcg_id = mem_cgroup_id(memcg);
 	),
 
-	TP_printk("%pS %p: nid: %d unused scan count %ld new scan count %ld total_scan %ld last shrinker return val %d",
+	TP_printk("%pS %p: nid: %d pid: %d memcg_id: %llu unused scan count %ld new scan count %ld total_scan %ld last shrinker return val %d %s",
 		__entry->shrink,
 		__entry->shr,
 		__entry->nid,
+		__entry->ent.pid,
+		__entry->memcg_id,
 		__entry->unused_scan,
 		__entry->new_scan,
 		__entry->total_scan,
-		__entry->retval)
+		__entry->retval,
+		__event_in_irq() ? "(in-irq)" : "")
 );
 
 TRACE_EVENT(mm_vmscan_lru_isolate,
@@ -514,9 +535,9 @@ TRACE_EVENT(mm_vmscan_node_reclaim_begin,
 
 DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_node_reclaim_end,
 
-	TP_PROTO(unsigned long nr_reclaimed),
+	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
 
-	TP_ARGS(nr_reclaimed)
+	TP_ARGS(nr_reclaimed, memcg)
 );
 
 TRACE_EVENT(mm_vmscan_throttled,
diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h
index 4d3d8c8f3a1bc..bdac0d685a984 100644
--- a/include/trace/events/writeback.h
+++ b/include/trace/events/writeback.h
@@ -67,7 +67,7 @@ DECLARE_EVENT_CLASS(writeback_folio_template,
 
 	TP_STRUCT__entry (
 		__array(char, name, 32)
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(pgoff_t, index)
 	),
 
@@ -79,9 +79,9 @@ DECLARE_EVENT_CLASS(writeback_folio_template,
 		__entry->index = folio->index;
 	),
 
-	TP_printk("bdi %s: ino=%lu index=%lu",
+	TP_printk("bdi %s: ino=%llu index=%lu",
 		__entry->name,
-		(unsigned long)__entry->ino,
+		__entry->ino,
 		__entry->index
 	)
 );
@@ -108,7 +108,7 @@ DECLARE_EVENT_CLASS(writeback_dirty_inode_template,
 
 	TP_STRUCT__entry (
 		__array(char, name, 32)
-		__field(ino_t, ino)
+		__field(u64, ino)
 		__field(unsigned long, state)
 		__field(unsigned long, flags)
 	),
@@ -123,9 +123,9 @@ DECLARE_EVENT_CLASS(writeback_dirty_inode_template,
 		__entry->flags		= flags;
 	),
 
-	TP_printk("bdi %s: ino=%lu state=%s flags=%s",
+	TP_printk("bdi %s: ino=%llu state=%s flags=%s",
 		__entry->name,
-		(unsigned long)__entry->ino,
+		__entry->ino,
 		show_inode_state(__entry->state),
 		show_inode_state(__entry->flags)
 	)
@@ -155,12 +155,12 @@ DEFINE_EVENT(writeback_dirty_inode_template, writeback_dirty_inode,
 #ifdef CREATE_TRACE_POINTS
 #ifdef CONFIG_CGROUP_WRITEBACK
 
-static inline ino_t __trace_wb_assign_cgroup(struct bdi_writeback *wb)
+static inline u64 __trace_wb_assign_cgroup(struct bdi_writeback *wb)
 {
 	return cgroup_ino(wb->memcg_css->cgroup);
 }
 
-static inline ino_t __trace_wbc_assign_cgroup(struct writeback_control *wbc)
+static inline u64 __trace_wbc_assign_cgroup(struct writeback_control *wbc)
 {
 	if (wbc->wb)
 		return __trace_wb_assign_cgroup(wbc->wb);
@@ -169,12 +169,12 @@ static inline ino_t __trace_wbc_assign_cgroup(struct writeback_control *wbc)
 }
 #else	/* CONFIG_CGROUP_WRITEBACK */
 
-static inline ino_t __trace_wb_assign_cgroup(struct bdi_writeback *wb)
+static inline u64 __trace_wb_assign_cgroup(struct bdi_writeback *wb)
 {
 	return 1;
 }
 
-static inline ino_t __trace_wbc_assign_cgroup(struct writeback_control *wbc)
+static inline u64 __trace_wbc_assign_cgroup(struct writeback_control *wbc)
 {
 	return 1;
 }
@@ -192,8 +192,8 @@ TRACE_EVENT(inode_foreign_history,
 
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
-		__field(ino_t,		ino)
-		__field(ino_t,		cgroup_ino)
+		__field(u64,		ino)
+		__field(u64,		cgroup_ino)
 		__field(unsigned int,	history)
 	),
 
@@ -204,10 +204,10 @@ TRACE_EVENT(inode_foreign_history,
 		__entry->history	= history;
 	),
 
-	TP_printk("bdi %s: ino=%lu cgroup_ino=%lu history=0x%x",
+	TP_printk("bdi %s: ino=%llu cgroup_ino=%llu history=0x%x",
 		__entry->name,
-		(unsigned long)__entry->ino,
-		(unsigned long)__entry->cgroup_ino,
+		__entry->ino,
+		__entry->cgroup_ino,
 		__entry->history
 	)
 );
@@ -221,8 +221,8 @@ TRACE_EVENT(inode_switch_wbs_queue,
 
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
-		__field(ino_t,		old_cgroup_ino)
-		__field(ino_t,		new_cgroup_ino)
+		__field(u64,		old_cgroup_ino)
+		__field(u64,		new_cgroup_ino)
 		__field(unsigned int,	count)
 	),
 
@@ -233,10 +233,10 @@ TRACE_EVENT(inode_switch_wbs_queue,
 		__entry->count		= count;
 	),
 
-	TP_printk("bdi %s: old_cgroup_ino=%lu new_cgroup_ino=%lu count=%u",
+	TP_printk("bdi %s: old_cgroup_ino=%llu new_cgroup_ino=%llu count=%u",
 		__entry->name,
-		(unsigned long)__entry->old_cgroup_ino,
-		(unsigned long)__entry->new_cgroup_ino,
+		__entry->old_cgroup_ino,
+		__entry->new_cgroup_ino,
 		__entry->count
 	)
 );
@@ -250,9 +250,9 @@ TRACE_EVENT(inode_switch_wbs,
 
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
-		__field(ino_t,		ino)
-		__field(ino_t,		old_cgroup_ino)
-		__field(ino_t,		new_cgroup_ino)
+		__field(u64,		ino)
+		__field(u64,		old_cgroup_ino)
+		__field(u64,		new_cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -262,11 +262,11 @@ TRACE_EVENT(inode_switch_wbs,
 		__entry->new_cgroup_ino	= __trace_wb_assign_cgroup(new_wb);
 	),
 
-	TP_printk("bdi %s: ino=%lu old_cgroup_ino=%lu new_cgroup_ino=%lu",
+	TP_printk("bdi %s: ino=%llu old_cgroup_ino=%llu new_cgroup_ino=%llu",
 		__entry->name,
-		(unsigned long)__entry->ino,
-		(unsigned long)__entry->old_cgroup_ino,
-		(unsigned long)__entry->new_cgroup_ino
+		__entry->ino,
+		__entry->old_cgroup_ino,
+		__entry->new_cgroup_ino
 	)
 );
 
@@ -279,10 +279,10 @@ TRACE_EVENT(track_foreign_dirty,
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
 		__field(u64,		bdi_id)
-		__field(ino_t,		ino)
+		__field(u64,		ino)
+		__field(u64,		cgroup_ino)
+		__field(u64,		page_cgroup_ino)
 		__field(unsigned int,	memcg_id)
-		__field(ino_t,		cgroup_ino)
-		__field(ino_t,		page_cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -294,16 +294,19 @@ TRACE_EVENT(track_foreign_dirty,
 		__entry->ino		= inode ? inode->i_ino : 0;
 		__entry->memcg_id	= wb->memcg_css->id;
 		__entry->cgroup_ino	= __trace_wb_assign_cgroup(wb);
+
+		rcu_read_lock();
 		__entry->page_cgroup_ino = cgroup_ino(folio_memcg(folio)->css.cgroup);
+		rcu_read_unlock();
 	),
 
-	TP_printk("bdi %s[%llu]: ino=%lu memcg_id=%u cgroup_ino=%lu page_cgroup_ino=%lu",
+	TP_printk("bdi %s[%llu]: ino=%llu memcg_id=%u cgroup_ino=%llu page_cgroup_ino=%llu",
 		__entry->name,
 		__entry->bdi_id,
-		(unsigned long)__entry->ino,
+		__entry->ino,
 		__entry->memcg_id,
-		(unsigned long)__entry->cgroup_ino,
-		(unsigned long)__entry->page_cgroup_ino
+		__entry->cgroup_ino,
+		__entry->page_cgroup_ino
 	)
 );
 
@@ -316,7 +319,7 @@ TRACE_EVENT(flush_foreign,
 
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
-		__field(ino_t,		cgroup_ino)
+		__field(u64,		cgroup_ino)
 		__field(unsigned int,	frn_bdi_id)
 		__field(unsigned int,	frn_memcg_id)
 	),
@@ -328,9 +331,9 @@ TRACE_EVENT(flush_foreign,
 		__entry->frn_memcg_id	= frn_memcg_id;
 	),
 
-	TP_printk("bdi %s: cgroup_ino=%lu frn_bdi_id=%u frn_memcg_id=%u",
+	TP_printk("bdi %s: cgroup_ino=%llu frn_bdi_id=%u frn_memcg_id=%u",
 		__entry->name,
-		(unsigned long)__entry->cgroup_ino,
+		__entry->cgroup_ino,
 		__entry->frn_bdi_id,
 		__entry->frn_memcg_id
 	)
@@ -345,9 +348,9 @@ DECLARE_EVENT_CLASS(writeback_write_inode_template,
 
 	TP_STRUCT__entry (
 		__array(char, name, 32)
-		__field(ino_t, ino)
+		__field(u64, ino)
+		__field(u64, cgroup_ino)
 		__field(int, sync_mode)
-		__field(ino_t, cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -358,11 +361,11 @@ DECLARE_EVENT_CLASS(writeback_write_inode_template,
 		__entry->cgroup_ino	= __trace_wbc_assign_cgroup(wbc);
 	),
 
-	TP_printk("bdi %s: ino=%lu sync_mode=%d cgroup_ino=%lu",
+	TP_printk("bdi %s: ino=%llu sync_mode=%d cgroup_ino=%llu",
 		__entry->name,
-		(unsigned long)__entry->ino,
+		__entry->ino,
 		__entry->sync_mode,
-		(unsigned long)__entry->cgroup_ino
+		__entry->cgroup_ino
 	)
 );
 
@@ -385,6 +388,7 @@ DECLARE_EVENT_CLASS(writeback_work_class,
 	TP_ARGS(wb, work),
 	TP_STRUCT__entry(
 		__array(char, name, 32)
+		__field(u64, cgroup_ino)
 		__field(long, nr_pages)
 		__field(dev_t, sb_dev)
 		__field(int, sync_mode)
@@ -392,7 +396,6 @@ DECLARE_EVENT_CLASS(writeback_work_class,
 		__field(int, range_cyclic)
 		__field(int, for_background)
 		__field(int, reason)
-		__field(ino_t, cgroup_ino)
 	),
 	TP_fast_assign(
 		strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
@@ -406,7 +409,7 @@ DECLARE_EVENT_CLASS(writeback_work_class,
 		__entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
 	),
 	TP_printk("bdi %s: sb_dev %d:%d nr_pages=%ld sync_mode=%d "
-		  "kupdate=%d range_cyclic=%d background=%d reason=%s cgroup_ino=%lu",
+		  "kupdate=%d range_cyclic=%d background=%d reason=%s cgroup_ino=%llu",
 		  __entry->name,
 		  MAJOR(__entry->sb_dev), MINOR(__entry->sb_dev),
 		  __entry->nr_pages,
@@ -415,7 +418,7 @@ DECLARE_EVENT_CLASS(writeback_work_class,
 		  __entry->range_cyclic,
 		  __entry->for_background,
 		  __print_symbolic(__entry->reason, WB_WORK_REASON),
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	)
 );
 #define DEFINE_WRITEBACK_WORK_EVENT(name) \
@@ -445,15 +448,15 @@ DECLARE_EVENT_CLASS(writeback_class,
 	TP_ARGS(wb),
 	TP_STRUCT__entry(
 		__array(char, name, 32)
-		__field(ino_t, cgroup_ino)
+		__field(u64, cgroup_ino)
 	),
 	TP_fast_assign(
 		strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
 		__entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
 	),
-	TP_printk("bdi %s: cgroup_ino=%lu",
+	TP_printk("bdi %s: cgroup_ino=%llu",
 		  __entry->name,
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	)
 );
 #define DEFINE_WRITEBACK_EVENT(name) \
@@ -482,15 +485,15 @@ DECLARE_EVENT_CLASS(wbc_class,
 	TP_ARGS(wbc, bdi),
 	TP_STRUCT__entry(
 		__array(char, name, 32)
+		__field(u64, cgroup_ino)
 		__field(long, nr_to_write)
 		__field(long, pages_skipped)
+		__field(long, range_start)
+		__field(long, range_end)
 		__field(int, sync_mode)
 		__field(int, for_kupdate)
 		__field(int, for_background)
 		__field(int, range_cyclic)
-		__field(long, range_start)
-		__field(long, range_end)
-		__field(ino_t, cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -507,7 +510,7 @@ DECLARE_EVENT_CLASS(wbc_class,
 	),
 
 	TP_printk("bdi %s: towrt=%ld skip=%ld mode=%d kupd=%d bgrd=%d "
-		"cyclic=%d start=0x%lx end=0x%lx cgroup_ino=%lu",
+		"cyclic=%d start=0x%lx end=0x%lx cgroup_ino=%llu",
 		__entry->name,
 		__entry->nr_to_write,
 		__entry->pages_skipped,
@@ -517,7 +520,7 @@ DECLARE_EVENT_CLASS(wbc_class,
 		__entry->range_cyclic,
 		__entry->range_start,
 		__entry->range_end,
-		(unsigned long)__entry->cgroup_ino
+		__entry->cgroup_ino
 	)
 )
 
@@ -535,11 +538,11 @@ TRACE_EVENT(writeback_queue_io,
 	TP_ARGS(wb, work, dirtied_before, moved),
 	TP_STRUCT__entry(
 		__array(char,		name, 32)
+		__field(u64,		cgroup_ino)
 		__field(unsigned long,	older)
 		__field(long,		age)
 		__field(int,		moved)
 		__field(int,		reason)
-		__field(ino_t,		cgroup_ino)
 	),
 	TP_fast_assign(
 		strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
@@ -549,13 +552,13 @@ TRACE_EVENT(writeback_queue_io,
 		__entry->reason	= work->reason;
 		__entry->cgroup_ino	= __trace_wb_assign_cgroup(wb);
 	),
-	TP_printk("bdi %s: older=%lu age=%ld enqueue=%d reason=%s cgroup_ino=%lu",
+	TP_printk("bdi %s: older=%lu age=%ld enqueue=%d reason=%s cgroup_ino=%llu",
 		__entry->name,
 		__entry->older,	/* dirtied_before in jiffies */
 		__entry->age,	/* dirtied_before in relative milliseconds */
 		__entry->moved,
 		__print_symbolic(__entry->reason, WB_WORK_REASON),
-		(unsigned long)__entry->cgroup_ino
+		__entry->cgroup_ino
 	)
 );
 
@@ -614,13 +617,13 @@ TRACE_EVENT(bdi_dirty_ratelimit,
 
 	TP_STRUCT__entry(
 		__array(char,		bdi, 32)
+		__field(u64,		cgroup_ino)
 		__field(unsigned long,	write_bw)
 		__field(unsigned long,	avg_write_bw)
 		__field(unsigned long,	dirty_rate)
 		__field(unsigned long,	dirty_ratelimit)
 		__field(unsigned long,	task_ratelimit)
 		__field(unsigned long,	balanced_dirty_ratelimit)
-		__field(ino_t,		cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -638,7 +641,7 @@ TRACE_EVENT(bdi_dirty_ratelimit,
 	TP_printk("bdi %s: "
 		  "write_bw=%lu awrite_bw=%lu dirty_rate=%lu "
 		  "dirty_ratelimit=%lu task_ratelimit=%lu "
-		  "balanced_dirty_ratelimit=%lu cgroup_ino=%lu",
+		  "balanced_dirty_ratelimit=%lu cgroup_ino=%llu",
 		  __entry->bdi,
 		  __entry->write_bw,		/* write bandwidth */
 		  __entry->avg_write_bw,	/* avg write bandwidth */
@@ -646,7 +649,7 @@ TRACE_EVENT(bdi_dirty_ratelimit,
 		  __entry->dirty_ratelimit,	/* base ratelimit */
 		  __entry->task_ratelimit, /* ratelimit with position control */
 		  __entry->balanced_dirty_ratelimit, /* the balanced ratelimit */
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	)
 );
 
@@ -667,6 +670,7 @@ TRACE_EVENT(balance_dirty_pages,
 
 	TP_STRUCT__entry(
 		__array(	 char,	bdi, 32)
+		__field(u64,		cgroup_ino)
 		__field(unsigned long,	limit)
 		__field(unsigned long,	setpoint)
 		__field(unsigned long,	dirty)
@@ -674,13 +678,12 @@ TRACE_EVENT(balance_dirty_pages,
 		__field(unsigned long,	wb_dirty)
 		__field(unsigned long,	dirty_ratelimit)
 		__field(unsigned long,	task_ratelimit)
-		__field(unsigned int,	dirtied)
-		__field(unsigned int,	dirtied_pause)
 		__field(unsigned long,	paused)
 		__field(	 long,	pause)
 		__field(unsigned long,	period)
 		__field(	 long,	think)
-		__field(ino_t,		cgroup_ino)
+		__field(unsigned int,	dirtied)
+		__field(unsigned int,	dirtied_pause)
 	),
 
 	TP_fast_assign(
@@ -711,7 +714,7 @@ TRACE_EVENT(balance_dirty_pages,
 		  "wb_setpoint=%lu wb_dirty=%lu "
 		  "dirty_ratelimit=%lu task_ratelimit=%lu "
 		  "dirtied=%u dirtied_pause=%u "
-		  "paused=%lu pause=%ld period=%lu think=%ld cgroup_ino=%lu",
+		  "paused=%lu pause=%ld period=%lu think=%ld cgroup_ino=%llu",
 		  __entry->bdi,
 		  __entry->limit,
 		  __entry->setpoint,
@@ -726,7 +729,7 @@ TRACE_EVENT(balance_dirty_pages,
 		  __entry->pause,	/* ms */
 		  __entry->period,	/* ms */
 		  __entry->think,	/* ms */
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	  )
 );
 
@@ -737,10 +740,10 @@ TRACE_EVENT(writeback_sb_inodes_requeue,
 
 	TP_STRUCT__entry(
 		__array(char, name, 32)
-		__field(ino_t, ino)
+		__field(u64, ino)
+		__field(u64, cgroup_ino)
 		__field(unsigned long, state)
 		__field(unsigned long, dirtied_when)
-		__field(ino_t, cgroup_ino)
 	),
 
 	TP_fast_assign(
@@ -752,13 +755,13 @@ TRACE_EVENT(writeback_sb_inodes_requeue,
 		__entry->cgroup_ino	= __trace_wb_assign_cgroup(inode_to_wb(inode));
 	),
 
-	TP_printk("bdi %s: ino=%lu state=%s dirtied_when=%lu age=%lu cgroup_ino=%lu",
+	TP_printk("bdi %s: ino=%llu state=%s dirtied_when=%lu age=%lu cgroup_ino=%llu",
 		  __entry->name,
-		  (unsigned long)__entry->ino,
+		  __entry->ino,
 		  show_inode_state(__entry->state),
 		  __entry->dirtied_when,
 		  (jiffies - __entry->dirtied_when) / HZ,
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	)
 );
 
@@ -773,13 +776,13 @@ DECLARE_EVENT_CLASS(writeback_single_inode_template,
 
 	TP_STRUCT__entry(
 		__array(char, name, 32)
-		__field(ino_t, ino)
+		__field(u64, ino)
+		__field(u64, cgroup_ino)
 		__field(unsigned long, state)
 		__field(unsigned long, dirtied_when)
 		__field(unsigned long, writeback_index)
-		__field(long, nr_to_write)
 		__field(unsigned long, wrote)
-		__field(ino_t, cgroup_ino)
+		__field(long, nr_to_write)
 	),
 
 	TP_fast_assign(
@@ -794,17 +797,17 @@ DECLARE_EVENT_CLASS(writeback_single_inode_template,
 		__entry->cgroup_ino	= __trace_wbc_assign_cgroup(wbc);
 	),
 
-	TP_printk("bdi %s: ino=%lu state=%s dirtied_when=%lu age=%lu "
-		  "index=%lu to_write=%ld wrote=%lu cgroup_ino=%lu",
+	TP_printk("bdi %s: ino=%llu state=%s dirtied_when=%lu age=%lu "
+		  "index=%lu to_write=%ld wrote=%lu cgroup_ino=%llu",
 		  __entry->name,
-		  (unsigned long)__entry->ino,
+		  __entry->ino,
 		  show_inode_state(__entry->state),
 		  __entry->dirtied_when,
 		  (jiffies - __entry->dirtied_when) / HZ,
 		  __entry->writeback_index,
 		  __entry->nr_to_write,
 		  __entry->wrote,
-		  (unsigned long)__entry->cgroup_ino
+		  __entry->cgroup_ino
 	)
 );
 
@@ -828,11 +831,11 @@ DECLARE_EVENT_CLASS(writeback_inode_template,
 	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
-		__field(	dev_t,	dev			)
-		__field(	ino_t,	ino			)
+		__field(	u64,	ino			)
 		__field(unsigned long,	state			)
-		__field(	__u16, mode			)
 		__field(unsigned long, dirtied_when		)
+		__field(	dev_t,	dev			)
+		__field(	__u16, mode			)
 	),
 
 	TP_fast_assign(
@@ -843,9 +846,9 @@ DECLARE_EVENT_CLASS(writeback_inode_template,
 		__entry->dirtied_when = inode->dirtied_when;
 	),
 
-	TP_printk("dev %d,%d ino %lu dirtied %lu state %s mode 0%o",
+	TP_printk("dev %d,%d ino %llu dirtied %lu state %s mode 0%o",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  (unsigned long)__entry->ino, __entry->dirtied_when,
+		  __entry->ino, __entry->dirtied_when,
 		  show_inode_state(__entry->state), __entry->mode)
 );
 
diff --git a/include/trace/events/zone_lock.h b/include/trace/events/zone_lock.h
new file mode 100644
index 0000000000000..3df82a8c01603
--- /dev/null
+++ b/include/trace/events/zone_lock.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM zone_lock
+
+#if !defined(_TRACE_ZONE_LOCK_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_ZONE_LOCK_H
+
+#include <linux/tracepoint.h>
+#include <linux/types.h>
+
+struct zone;
+
+DECLARE_EVENT_CLASS(zone_lock,
+
+	TP_PROTO(struct zone *zone),
+
+	TP_ARGS(zone),
+
+	TP_STRUCT__entry(
+		__field(struct zone *, zone)
+	),
+
+	TP_fast_assign(
+		__entry->zone = zone;
+	),
+
+	TP_printk("zone=%p", __entry->zone)
+);
+
+#define DEFINE_ZONE_LOCK_EVENT(name)			\
+	DEFINE_EVENT(zone_lock, name,			\
+		TP_PROTO(struct zone *zone),		\
+		TP_ARGS(zone))
+
+DEFINE_ZONE_LOCK_EVENT(zone_lock_start_locking);
+DEFINE_ZONE_LOCK_EVENT(zone_lock_released);
+
+TRACE_EVENT(zone_lock_acquire_returned,
+
+	TP_PROTO(struct zone *zone, bool success),
+
+	TP_ARGS(zone, success),
+
+	TP_STRUCT__entry(
+		__field(struct zone *, zone)
+		__field(bool, success)
+	),
+
+	TP_fast_assign(
+		__entry->zone = zone;
+		__entry->success = success;
+	),
+
+	TP_printk(
+		"zone=%p success=%s",
+		__entry->zone,
+		__entry->success ? "true" : "false"
+	)
+);
+
+#endif /* _TRACE_ZONE_LOCK_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/stages/stage3_trace_output.h b/include/trace/stages/stage3_trace_output.h
index fce85ea2df1c8..181b813357812 100644
--- a/include/trace/stages/stage3_trace_output.h
+++ b/include/trace/stages/stage3_trace_output.h
@@ -64,36 +64,36 @@
 #define __get_rel_sockaddr(field)	((struct sockaddr *)__get_rel_dynamic_array(field))
 
 #undef __print_flags
-#define __print_flags(flag, delim, flag_array...)			\
-	({								\
-		static const struct trace_print_flags __flags[] =	\
-			{ flag_array, { -1, NULL }};			\
-		trace_print_flags_seq(p, delim, flag, __flags);	\
+#define __print_flags(flag, delim, flag_array...)					\
+	({										\
+		static const struct trace_print_flags __flags[] =			\
+			{ flag_array };							\
+		trace_print_flags_seq(p, delim, flag, __flags, ARRAY_SIZE(__flags));	\
 	})
 
 #undef __print_symbolic
-#define __print_symbolic(value, symbol_array...)			\
-	({								\
-		static const struct trace_print_flags symbols[] =	\
-			{ symbol_array, { -1, NULL }};			\
-		trace_print_symbols_seq(p, value, symbols);		\
+#define __print_symbolic(value, symbol_array...)					\
+	({										\
+		static const struct trace_print_flags symbols[] =			\
+			{ symbol_array };						\
+		trace_print_symbols_seq(p, value, symbols, ARRAY_SIZE(symbols));	\
 	})
 
 #undef __print_flags_u64
 #undef __print_symbolic_u64
 #if BITS_PER_LONG == 32
-#define __print_flags_u64(flag, delim, flag_array...)			\
-	({								\
-		static const struct trace_print_flags_u64 __flags[] =	\
-			{ flag_array, { -1, NULL } };			\
-		trace_print_flags_seq_u64(p, delim, flag, __flags);	\
+#define __print_flags_u64(flag, delim, flag_array...)						\
+	({											\
+		static const struct trace_print_flags_u64 __flags[] =				\
+			{ flag_array };								\
+		trace_print_flags_seq_u64(p, delim, flag, __flags, ARRAY_SIZE(__flags));	\
 	})
 
-#define __print_symbolic_u64(value, symbol_array...)			\
-	({								\
-		static const struct trace_print_flags_u64 symbols[] =	\
-			{ symbol_array, { -1, NULL } };			\
-		trace_print_symbols_seq_u64(p, value, symbols);	\
+#define __print_symbolic_u64(value, symbol_array...)					\
+	({										\
+		static const struct trace_print_flags_u64 symbols[] =			\
+			{ symbol_array };						\
+		trace_print_symbols_seq_u64(p, value, symbols, ARRAY_SIZE(symbols));	\
 	})
 #else
 #define __print_flags_u64(flag, delim, flag_array...)			\
@@ -150,3 +150,11 @@
 
 #undef __get_buf
 #define __get_buf(len)		trace_seq_acquire(p, (len))
+
+#undef __event_in_hardirq
+#undef __event_in_softirq
+#undef __event_in_irq
+
+#define __event_in_hardirq()	(__entry->ent.flags & TRACE_FLAG_HARDIRQ)
+#define __event_in_softirq()	(__entry->ent.flags & TRACE_FLAG_SOFTIRQ)
+#define __event_in_irq()	(__entry->ent.flags & (TRACE_FLAG_HARDIRQ | TRACE_FLAG_SOFTIRQ))
diff --git a/include/trace/stages/stage7_class_define.h b/include/trace/stages/stage7_class_define.h
index fcd564a590f43..47008897a7956 100644
--- a/include/trace/stages/stage7_class_define.h
+++ b/include/trace/stages/stage7_class_define.h
@@ -26,6 +26,25 @@
 #undef __print_hex_dump
 #undef __get_buf
 
+#undef __event_in_hardirq
+#undef __event_in_softirq
+#undef __event_in_irq
+
+/*
+ * The TRACE_FLAG_* are enums. Instead of using TRACE_DEFINE_ENUM(),
+ * use their hardcoded values. These values are parsed by user space
+ * tooling elsewhere so they will never change.
+ *
+ * See "enum trace_flag_type" in linux/trace_events.h:
+ *   TRACE_FLAG_HARDIRQ
+ *   TRACE_FLAG_SOFTIRQ
+ */
+
+/* This is what is displayed in the format files */
+#define __event_in_hardirq()	(REC->common_flags & 0x8)
+#define __event_in_softirq()	(REC->common_flags & 0x10)
+#define __event_in_irq()	(REC->common_flags & 0x18)
+
 /*
  * The below is not executed in the kernel. It is only what is
  * displayed in the print format for userspace to parse.
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index ebbd861ef0bc6..9f3090db2f163 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -479,7 +479,9 @@ struct drm_amdgpu_userq_signal {
 	 * @num_syncobj_handles: A count that represents the number of syncobj handles in
 	 * @syncobj_handles.
 	 */
-	__u64	num_syncobj_handles;
+	__u16	num_syncobj_handles;
+	__u16	pad0;
+	__u32	pad1;
 	/**
 	 * @bo_read_handles: The list of BO handles that the submitted user queue job
 	 * is using for read only. This will update BO fences in the kernel.
@@ -563,7 +565,8 @@ struct drm_amdgpu_userq_wait {
 	 * @num_syncobj_handles: A count that represents the number of syncobj handles in
 	 * @syncobj_handles.
 	 */
-	__u32	num_syncobj_handles;
+	__u16	num_syncobj_handles;
+	__u16	pad0;
 	/**
 	 * @num_bo_read_handles: A count that represents the number of read BO handles in
 	 * @bo_read_handles.
diff --git a/include/uapi/drm/amdxdna_accel.h b/include/uapi/drm/amdxdna_accel.h
index 9c44db2b3dcd4..5bd13f4435f55 100644
--- a/include/uapi/drm/amdxdna_accel.h
+++ b/include/uapi/drm/amdxdna_accel.h
@@ -353,7 +353,8 @@ struct amdxdna_drm_query_clock_metadata {
 };
 
 enum amdxdna_sensor_type {
-	AMDXDNA_SENSOR_TYPE_POWER
+	AMDXDNA_SENSOR_TYPE_POWER,
+	AMDXDNA_SENSOR_TYPE_COLUMN_UTILIZATION
 };
 
 /**
diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h
index c89aede3cb120..ac66fa93b5a3e 100644
--- a/include/uapi/drm/drm_fourcc.h
+++ b/include/uapi/drm/drm_fourcc.h
@@ -1422,6 +1422,22 @@ drm_fourcc_canonicalize_nvidia_format_mod(__u64 modifier)
 #define DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED \
 	DRM_FORMAT_MOD_ARM_CODE(DRM_FORMAT_MOD_ARM_TYPE_MISC, 1ULL)
 
+/*
+ * ARM 64k interleaved modifier
+ *
+ * This is used by ARM Mali v10+ GPUs. With this modifier, the plane is divided
+ * into 64k byte 1:1 or 2:1 -sided tiles. The 64k tiles are laid out linearly.
+ * Each 64k tile is divided into blocks of 16x16 texel blocks, which are
+ * themselves laid out linearly within a 64k tile. Then within each 16x16
+ * block, texel blocks are laid out according to U order, similar to
+ * 16X16_BLOCK_U_INTERLEAVED.
+ *
+ * Note that unlike 16X16_BLOCK_U_INTERLEAVED, the layout does not change
+ * depending on whether a format is compressed or not.
+ */
+#define DRM_FORMAT_MOD_ARM_INTERLEAVED_64K \
+	DRM_FORMAT_MOD_ARM_CODE(DRM_FORMAT_MOD_ARM_TYPE_MISC, 2ULL)
+
 /*
  * Allwinner tiled modifier
  *
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index cbbbfc1dfe2b8..a4bdc4bd11bc1 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -27,6 +27,9 @@
 #ifndef _DRM_MODE_H
 #define _DRM_MODE_H
 
+#include <linux/bits.h>
+#include <linux/const.h>
+
 #include "drm.h"
 
 #if defined(__cplusplus)
@@ -166,6 +169,10 @@ extern "C" {
 #define DRM_MODE_LINK_STATUS_GOOD	0
 #define DRM_MODE_LINK_STATUS_BAD	1
 
+/* Panel type property */
+#define DRM_MODE_PANEL_TYPE_UNKNOWN	0
+#define DRM_MODE_PANEL_TYPE_OLED	1
+
 /*
  * DRM_MODE_ROTATE_<degrees>
  *
@@ -1545,6 +1552,83 @@ struct drm_mode_closefb {
 	__u32 pad;
 };
 
+/*
+ * Put 16-bit ARGB values into a standard 64-bit representation that can be
+ * used for ioctl parameters, inter-driver communication, etc.
+ *
+ * If the component values being provided contain less than 16 bits of
+ * precision, use a conversion ratio to get a better color approximation.
+ * The ratio is computed as (2^16 - 1) / (2^bpc - 1), where bpc and 16 are
+ * the input and output precision, respectively.
+ * Also note bpc must be greater than 0.
+ */
+#define __DRM_ARGB64_PREP(c, shift)					\
+	(((__u64)(c) & __GENMASK(15, 0)) << (shift))
+
+#define __DRM_ARGB64_PREP_BPC(c, shift, bpc)				\
+({									\
+	__u16 mask = __GENMASK((bpc) - 1, 0);				\
+	__u16 conv = __KERNEL_DIV_ROUND_CLOSEST((mask & (c)) *		\
+						__GENMASK(15, 0), mask);\
+	__DRM_ARGB64_PREP(conv, shift);					\
+})
+
+#define DRM_ARGB64_PREP(alpha, red, green, blue)			\
+(									\
+	__DRM_ARGB64_PREP(alpha, 48) |					\
+	__DRM_ARGB64_PREP(red,   32) |					\
+	__DRM_ARGB64_PREP(green, 16) |					\
+	__DRM_ARGB64_PREP(blue,   0)					\
+)
+
+#define DRM_ARGB64_PREP_BPC(alpha, red, green, blue, bpc)		\
+({									\
+	__typeof__(bpc) __bpc = bpc;					\
+	__DRM_ARGB64_PREP_BPC(alpha, 48, __bpc) |			\
+	__DRM_ARGB64_PREP_BPC(red,   32, __bpc) |			\
+	__DRM_ARGB64_PREP_BPC(green, 16, __bpc) |			\
+	__DRM_ARGB64_PREP_BPC(blue,   0, __bpc);			\
+})
+
+/*
+ * Extract the specified color component from a standard 64-bit ARGB value.
+ *
+ * If the requested precision is less than 16 bits, make use of a conversion
+ * ratio calculated as (2^bpc - 1) / (2^16 - 1), where bpc and 16 are the
+ * output and input precision, respectively.
+ *
+ * If speed is more important than accuracy, use DRM_ARGB64_GET*_BPCS()
+ * instead of DRM_ARGB64_GET*_BPC() in order to replace the expensive
+ * division with a simple bit right-shift operation.
+ */
+#define __DRM_ARGB64_GET(c, shift)					\
+	((__u16)(((__u64)(c) >> (shift)) & __GENMASK(15, 0)))
+
+#define __DRM_ARGB64_GET_BPC(c, shift, bpc)				\
+({									\
+	__u16 comp = __DRM_ARGB64_GET(c, shift);			\
+	__KERNEL_DIV_ROUND_CLOSEST(comp * __GENMASK((bpc) - 1, 0),	\
+				   __GENMASK(15, 0));			\
+})
+
+#define __DRM_ARGB64_GET_BPCS(c, shift, bpc)				\
+	(__DRM_ARGB64_GET(c, shift) >> (16 - (bpc)))
+
+#define DRM_ARGB64_GETA(c)		__DRM_ARGB64_GET(c, 48)
+#define DRM_ARGB64_GETR(c)		__DRM_ARGB64_GET(c, 32)
+#define DRM_ARGB64_GETG(c)		__DRM_ARGB64_GET(c, 16)
+#define DRM_ARGB64_GETB(c)		__DRM_ARGB64_GET(c, 0)
+
+#define DRM_ARGB64_GETA_BPC(c, bpc)	__DRM_ARGB64_GET_BPC(c, 48, bpc)
+#define DRM_ARGB64_GETR_BPC(c, bpc)	__DRM_ARGB64_GET_BPC(c, 32, bpc)
+#define DRM_ARGB64_GETG_BPC(c, bpc)	__DRM_ARGB64_GET_BPC(c, 16, bpc)
+#define DRM_ARGB64_GETB_BPC(c, bpc)	__DRM_ARGB64_GET_BPC(c, 0, bpc)
+
+#define DRM_ARGB64_GETA_BPCS(c, bpc)	__DRM_ARGB64_GET_BPCS(c, 48, bpc)
+#define DRM_ARGB64_GETR_BPCS(c, bpc)	__DRM_ARGB64_GET_BPCS(c, 32, bpc)
+#define DRM_ARGB64_GETG_BPCS(c, bpc)	__DRM_ARGB64_GET_BPCS(c, 16, bpc)
+#define DRM_ARGB64_GETB_BPCS(c, bpc)	__DRM_ARGB64_GET_BPCS(c, 0, bpc)
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/drm/drm_ras.h b/include/uapi/drm/drm_ras.h
new file mode 100644
index 0000000000000..5f40fa5b869d0
--- /dev/null
+++ b/include/uapi/drm/drm_ras.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/drm_ras.yaml */
+/* YNL-GEN uapi header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _UAPI_LINUX_DRM_RAS_H
+#define _UAPI_LINUX_DRM_RAS_H
+
+#define DRM_RAS_FAMILY_NAME	"drm-ras"
+#define DRM_RAS_FAMILY_VERSION	1
+
+/*
+ * Type of the node. Currently, only error-counter nodes are supported, which
+ * expose reliability counters for a hardware/software component.
+ */
+enum drm_ras_node_type {
+	DRM_RAS_NODE_TYPE_ERROR_COUNTER = 1,
+};
+
+enum {
+	DRM_RAS_A_NODE_ATTRS_NODE_ID = 1,
+	DRM_RAS_A_NODE_ATTRS_DEVICE_NAME,
+	DRM_RAS_A_NODE_ATTRS_NODE_NAME,
+	DRM_RAS_A_NODE_ATTRS_NODE_TYPE,
+
+	__DRM_RAS_A_NODE_ATTRS_MAX,
+	DRM_RAS_A_NODE_ATTRS_MAX = (__DRM_RAS_A_NODE_ATTRS_MAX - 1)
+};
+
+enum {
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID = 1,
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_NAME,
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_VALUE,
+
+	__DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX,
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX = (__DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX - 1)
+};
+
+enum {
+	DRM_RAS_CMD_LIST_NODES = 1,
+	DRM_RAS_CMD_GET_ERROR_COUNTER,
+
+	__DRM_RAS_CMD_MAX,
+	DRM_RAS_CMD_MAX = (__DRM_RAS_CMD_MAX - 1)
+};
+
+#endif /* _UAPI_LINUX_DRM_RAS_H */
diff --git a/include/uapi/drm/nouveau_drm.h b/include/uapi/drm/nouveau_drm.h
index dd87f8f30793d..1fa82fa6af386 100644
--- a/include/uapi/drm/nouveau_drm.h
+++ b/include/uapi/drm/nouveau_drm.h
@@ -432,6 +432,69 @@ struct drm_nouveau_exec {
 	__u64 push_ptr;
 };
 
+struct drm_nouveau_get_zcull_info {
+	/**
+	 * @width_align_pixels: required alignment for region widths, in pixels
+	 * (typically #TPC's * 16).
+	 */
+	__u32 width_align_pixels;
+	/**
+	 * @height_align_pixels: required alignment for region heights, in
+	 * pixels (typically 32).
+	 */
+	__u32 height_align_pixels;
+	/**
+	 * @pixel_squares_by_aliquots: the pixel area covered by an aliquot
+	 * (typically #Zcull_banks * 16 * 16).
+	 */
+	__u32 pixel_squares_by_aliquots;
+	/**
+	 * @aliquot_total: the total aliquot pool available in hardware
+	 */
+	__u32 aliquot_total;
+	/**
+	 * @zcull_region_byte_multiplier: the size of an aliquot in bytes, which
+	 * is used for save/restore operations on a region
+	 */
+	__u32 zcull_region_byte_multiplier;
+	/**
+	 * @zcull_region_header_size: the region header size in bytes, which is
+	 * used for save/restore operations on a region
+	 */
+	__u32 zcull_region_header_size;
+	/**
+	 * @zcull_subregion_header_size: the subregion header size in bytes,
+	 * which is used for save/restore operations on a region
+	 */
+	__u32 zcull_subregion_header_size;
+	/**
+	 * @subregion_count: the total number of subregions the hardware
+	 * supports
+	 */
+	__u32 subregion_count;
+	/**
+	 * @subregion_width_align_pixels: required alignment for subregion
+	 * widths, in pixels (typically #TPC's * 16).
+	 */
+	__u32 subregion_width_align_pixels;
+	/**
+	 * @subregion_height_align_pixels: required alignment for subregion
+	 * heights, in pixels
+	 */
+	__u32 subregion_height_align_pixels;
+
+	/**
+	 * @ctxsw_size: the size, in bytes, of a zcull context switching region.
+	 * Will be zero if the kernel does not support zcull context switching.
+	 */
+	__u32 ctxsw_size;
+	/**
+	 * @ctxsw_align: the alignment, in bytes, of a zcull context switching
+	 * region
+	 */
+	__u32 ctxsw_align;
+};
+
 #define DRM_NOUVEAU_GETPARAM           0x00
 #define DRM_NOUVEAU_SETPARAM           0x01 /* deprecated */
 #define DRM_NOUVEAU_CHANNEL_ALLOC      0x02
@@ -445,6 +508,7 @@ struct drm_nouveau_exec {
 #define DRM_NOUVEAU_VM_INIT            0x10
 #define DRM_NOUVEAU_VM_BIND            0x11
 #define DRM_NOUVEAU_EXEC               0x12
+#define DRM_NOUVEAU_GET_ZCULL_INFO     0x13
 #define DRM_NOUVEAU_GEM_NEW            0x40
 #define DRM_NOUVEAU_GEM_PUSHBUF        0x41
 #define DRM_NOUVEAU_GEM_CPU_PREP       0x42
@@ -513,6 +577,8 @@ struct drm_nouveau_svm_bind {
 #define DRM_IOCTL_NOUVEAU_VM_INIT            DRM_IOWR(DRM_COMMAND_BASE + DRM_NOUVEAU_VM_INIT, struct drm_nouveau_vm_init)
 #define DRM_IOCTL_NOUVEAU_VM_BIND            DRM_IOWR(DRM_COMMAND_BASE + DRM_NOUVEAU_VM_BIND, struct drm_nouveau_vm_bind)
 #define DRM_IOCTL_NOUVEAU_EXEC               DRM_IOWR(DRM_COMMAND_BASE + DRM_NOUVEAU_EXEC, struct drm_nouveau_exec)
+
+#define DRM_IOCTL_NOUVEAU_GET_ZCULL_INFO     DRM_IOR (DRM_COMMAND_BASE + DRM_NOUVEAU_GET_ZCULL_INFO, struct drm_nouveau_get_zcull_info)
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 077e66a682e29..f8b2afb20540f 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -335,10 +335,6 @@ struct drm_xe_mem_region {
 	__u64 total_size;
 	/**
 	 * @used: Estimate of the memory used in bytes for this region.
-	 *
-	 * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable
-	 * accounting.  Without this the value here will always equal
-	 * zero.
 	 */
 	__u64 used;
 	/**
@@ -363,9 +359,7 @@ struct drm_xe_mem_region {
 	 * @cpu_visible_used: Estimate of CPU visible memory used, in
 	 * bytes.
 	 *
-	 * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable
-	 * accounting. Without this the value here will always equal
-	 * zero.  Note this is only currently tracked for
+	 * Note this is only currently tracked for
 	 * DRM_XE_MEM_REGION_CLASS_VRAM regions (for other types the value
 	 * here will always be zero).
 	 */
@@ -412,6 +406,9 @@ struct drm_xe_query_mem_regions {
  *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT - Flag is set if the
  *      device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION.
  *      This is exposed only on Xe2+.
+ *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_DISABLE_STATE_CACHE_PERF_FIX - Flag is set
+ *      if a queue can be creaed with
+ *      %DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX
  *  - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment
  *    required by this device, typically SZ_4K or SZ_64K
  *  - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
@@ -431,6 +428,7 @@ struct drm_xe_query_config {
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY	(1 << 1)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR	(1 << 2)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT (1 << 3)
+	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_DISABLE_STATE_CACHE_PERF_FIX	(1 << 4)
 #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT		2
 #define DRM_XE_QUERY_CONFIG_VA_BITS			3
 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY	4
@@ -975,6 +973,11 @@ struct drm_xe_gem_mmap_offset {
  *    demand when accessed, and also allows per-VM overcommit of memory.
  *    The xe driver internally uses recoverable pagefaults to implement
  *    this.
+ *  - %DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT - Requires also
+ *    DRM_XE_VM_CREATE_FLAG_FAULT_MODE. This disallows per-VM overcommit
+ *    but only during a &DRM_IOCTL_XE_VM_BIND operation with the
+ *    %DRM_XE_VM_BIND_FLAG_IMMEDIATE flag set. This may be useful for
+ *    user-space naively probing the amount of available memory.
  */
 struct drm_xe_vm_create {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -983,6 +986,7 @@ struct drm_xe_vm_create {
 #define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE	(1 << 0)
 #define DRM_XE_VM_CREATE_FLAG_LR_MODE	        (1 << 1)
 #define DRM_XE_VM_CREATE_FLAG_FAULT_MODE	(1 << 2)
+#define DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT  (1 << 3)
 	/** @flags: Flags */
 	__u32 flags;
 
@@ -1053,6 +1057,13 @@ struct drm_xe_vm_destroy {
  *    not invoke autoreset. Neither will stack variables going out of scope.
  *    Therefore it's recommended to always explicitly reset the madvises when
  *    freeing the memory backing a region used in a &DRM_IOCTL_XE_MADVISE call.
+ *  - %DRM_XE_VM_BIND_FLAG_DECOMPRESS - Request on-device decompression for a MAP.
+ *    When set on a MAP bind operation, request the driver schedule an on-device
+ *    in-place decompression (via the migrate/resolve path) for the GPU mapping
+ *    created by this bind. Only valid for DRM_XE_VM_BIND_OP_MAP; usage on
+ *    other ops is rejected. The bind's pat_index must select the device's
+ *    "no-compression" PAT. Only meaningful for VRAM-backed BOs on devices that
+ *    support Flat CCS and the required HW generation XE2+.
  *
  * The @prefetch_mem_region_instance for %DRM_XE_VM_BIND_OP_PREFETCH can also be:
  *  - %DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC, which ensures prefetching occurs in
@@ -1160,6 +1171,7 @@ struct drm_xe_vm_bind_op {
 #define DRM_XE_VM_BIND_FLAG_CHECK_PXP	(1 << 4)
 #define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR	(1 << 5)
 #define DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET	(1 << 6)
+#define DRM_XE_VM_BIND_FLAG_DECOMPRESS (1 << 7)
 	/** @flags: Bind flags */
 	__u32 flags;
 
@@ -1285,6 +1297,9 @@ struct drm_xe_vm_bind {
  *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue
  *    priority within the multi-queue group. Current valid priority values are 0–2
  *    (default is 1), with higher values indicating higher priority.
+ *  - %DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX - Set the queue to
+ *    enable render color cache keying on BTP+BTI instead of just BTI
+ *    (only valid for render queues).
  *
  * The example below shows how to use @drm_xe_exec_queue_create to create
  * a simple exec_queue (no parallel submission) of class
@@ -1329,6 +1344,7 @@ struct drm_xe_exec_queue_create {
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP		4
 #define     DRM_XE_MULTI_GROUP_CREATE				(1ull << 63)
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY	5
+#define   DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX	6
 	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
 
@@ -2357,6 +2373,85 @@ struct drm_xe_exec_queue_set_property {
 	__u64 reserved[2];
 };
 
+/**
+ * DOC: Xe DRM RAS
+ *
+ * The enums and strings defined below map to the attributes of the DRM RAS Netlink Interface.
+ * Refer to Documentation/netlink/specs/drm_ras.yaml for complete interface specification.
+ *
+ * Node Registration
+ * =================
+ *
+ * The driver registers DRM RAS nodes for each error severity level.
+ * enum drm_xe_ras_error_severity defines the node-id, while DRM_XE_RAS_ERROR_SEVERITY_NAMES maps
+ * node-id to node-name.
+ *
+ * Error Classification
+ * ====================
+ *
+ * Each node contains a list of error counters. Each error is identified by a error-id and
+ * an error-name. enum drm_xe_ras_error_component defines the error-id, while
+ * DRM_XE_RAS_ERROR_COMPONENT_NAMES maps error-id to error-name.
+ *
+ * User Interface
+ * ==============
+ *
+ * To retrieve error values of a error counter, userspace applications should
+ * follow the below steps:
+ *
+ * 1. Use command LIST_NODES to enumerate all available nodes
+ * 2. Select node by node-id or node-name
+ * 3. Use command GET_ERROR_COUNTERS to list errors of specific node
+ * 4. Query specific error values using either error-id or error-name
+ *
+ * .. code-block:: C
+ *
+ *	// Lookup tables for ID-to-name resolution
+ *	static const char *nodes[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;
+ *	static const char *errors[] = DRM_XE_RAS_ERROR_COMPONENT_NAMES;
+ *
+ */
+
+/**
+ * enum drm_xe_ras_error_severity - DRM RAS error severity.
+ */
+enum drm_xe_ras_error_severity {
+	/** @DRM_XE_RAS_ERR_SEV_CORRECTABLE: Correctable Error */
+	DRM_XE_RAS_ERR_SEV_CORRECTABLE = 0,
+	/** @DRM_XE_RAS_ERR_SEV_UNCORRECTABLE: Uncorrectable Error */
+	DRM_XE_RAS_ERR_SEV_UNCORRECTABLE,
+	/** @DRM_XE_RAS_ERR_SEV_MAX: Max severity */
+	DRM_XE_RAS_ERR_SEV_MAX /* non-ABI */
+};
+
+/**
+ * enum drm_xe_ras_error_component - DRM RAS error component.
+ */
+enum drm_xe_ras_error_component {
+	/** @DRM_XE_RAS_ERR_COMP_CORE_COMPUTE: Core Compute Error */
+	DRM_XE_RAS_ERR_COMP_CORE_COMPUTE = 1,
+	/** @DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: SoC Internal Error */
+	DRM_XE_RAS_ERR_COMP_SOC_INTERNAL,
+	/** @DRM_XE_RAS_ERR_COMP_MAX: Max Error */
+	DRM_XE_RAS_ERR_COMP_MAX	/* non-ABI */
+};
+
+/*
+ * Error severity to name mapping.
+ */
+#define DRM_XE_RAS_ERROR_SEVERITY_NAMES {				\
+	[DRM_XE_RAS_ERR_SEV_CORRECTABLE] = "correctable-errors",	\
+	[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE] = "uncorrectable-errors",	\
+}
+
+/*
+ * Error component to name mapping.
+ */
+#define DRM_XE_RAS_ERROR_COMPONENT_NAMES {				\
+	[DRM_XE_RAS_ERR_COMP_CORE_COMPUTE] = "core-compute",		\
+	[DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal"		\
+}
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h
index 14a1c1fe013ac..e8f5ce677df73 100644
--- a/include/uapi/linux/audit.h
+++ b/include/uapi/linux/audit.h
@@ -350,7 +350,7 @@ enum {
 #define AUDIT_STATUS_ENABLED			0x0001
 #define AUDIT_STATUS_FAILURE			0x0002
 #define AUDIT_STATUS_PID			0x0004
-#define AUDIT_STATUS_RATE_LIMIT		0x0008
+#define AUDIT_STATUS_RATE_LIMIT			0x0008
 #define AUDIT_STATUS_BACKLOG_LIMIT		0x0010
 #define AUDIT_STATUS_BACKLOG_WAIT_TIME		0x0020
 #define AUDIT_STATUS_LOST			0x0040
@@ -386,8 +386,8 @@ enum {
  * These bits disambiguate different calling conventions that share an
  * ELF machine type, bitness, and endianness
  */
-#define __AUDIT_ARCH_CONVENTION_MASK 0x30000000
-#define __AUDIT_ARCH_CONVENTION_MIPS64_N32 0x20000000
+#define __AUDIT_ARCH_CONVENTION_MASK		0x30000000
+#define __AUDIT_ARCH_CONVENTION_MIPS64_N32	0x20000000
 
 /* distinguish syscall tables */
 #define __AUDIT_ARCH_64BIT 0x80000000
@@ -508,7 +508,7 @@ struct audit_tty_status {
 	__u32		log_passwd;	/* 1 = enabled, 0 = disabled */
 };
 
-#define AUDIT_UID_UNSET (unsigned int)-1
+#define AUDIT_UID_UNSET ((unsigned int)-1)
 #define AUDIT_SID_UNSET ((unsigned int)-1)
 
 /* audit_rule_data supports filter rules with both integer and string
diff --git a/include/uapi/linux/bsg.h b/include/uapi/linux/bsg.h
index cd6302def5ed7..6cff77f5b8571 100644
--- a/include/uapi/linux/bsg.h
+++ b/include/uapi/linux/bsg.h
@@ -2,6 +2,9 @@
 #ifndef _UAPIBSG_H
 #define _UAPIBSG_H
 
+#ifdef __KERNEL__
+#include <linux/build_bug.h>
+#endif /* __KERNEL__ */
 #include <linux/types.h>
 
 #define BSG_PROTOCOL_SCSI		0
@@ -63,5 +66,77 @@ struct sg_io_v4 {
 	__u32 padding;
 };
 
+struct bsg_uring_cmd {
+	__u64 request;		/* [i], [*i] command descriptor address */
+	__u32 request_len;	/* [i] command descriptor length in bytes */
+	__u32 protocol;		/* [i] protocol type (BSG_PROTOCOL_*) */
+	__u32 subprotocol;	/* [i] subprotocol type (BSG_SUB_PROTOCOL_*) */
+	__u32 max_response_len;	/* [i] response buffer size in bytes */
+
+	__u64 response;		/* [i], [*o] response data address */
+	__u64 dout_xferp;	/* [i], [*i] */
+	__u32 dout_xfer_len;	/* [i] bytes to be transferred to device */
+	__u32 dout_iovec_count;	/* [i] 0 -> "flat" dout transfer else
+				 * dout_xferp points to array of iovec
+				 */
+	__u64 din_xferp;	/* [i], [*o] */
+	__u32 din_xfer_len;	/* [i] bytes to be transferred from device */
+	__u32 din_iovec_count;	/* [i] 0 -> "flat" din transfer */
+
+	__u32 timeout_ms;	/* [i] timeout in milliseconds */
+	__u8  reserved[12];	/* reserved for future extension */
+};
+
+#ifdef __KERNEL__
+/* Must match IORING_OP_URING_CMD payload size (e.g. SQE128). */
+static_assert(sizeof(struct bsg_uring_cmd) == 80);
+#endif /* __KERNEL__ */
+
+
+/*
+ * SCSI BSG io_uring completion (res2, 64-bit)
+ *
+ * When using BSG_PROTOCOL_SCSI + BSG_SUB_PROTOCOL_SCSI_CMD with
+ * IORING_OP_URING_CMD, the completion queue entry (CQE) contains:
+ *   - result: errno (0 on success)
+ *   - res2: packed SCSI status
+ *
+ * res2 bit layout:
+ *   [0..7]   device_status  (SCSI status byte, e.g. CHECK_CONDITION)
+ *   [8..15]  driver_status  (e.g. DRIVER_SENSE when sense data is valid)
+ *   [16..23] host_status    (e.g. DID_OK, DID_TIME_OUT)
+ *   [24..31] sense_len_wr   (bytes of sense data written to response buffer)
+ *   [32..63] resid_len      (residual transfer length)
+ */
+static inline __u8 bsg_scsi_res2_device_status(__u64 res2)
+{
+	return res2 & 0xff;
+}
+static inline __u8 bsg_scsi_res2_driver_status(__u64 res2)
+{
+	return res2 >> 8;
+}
+static inline __u8 bsg_scsi_res2_host_status(__u64 res2)
+{
+	return res2 >> 16;
+}
+static inline __u8 bsg_scsi_res2_sense_len(__u64 res2)
+{
+	return res2 >> 24;
+}
+static inline __u32 bsg_scsi_res2_resid_len(__u64 res2)
+{
+	return res2 >> 32;
+}
+static inline __u64 bsg_scsi_res2_build(__u8 device_status, __u8 driver_status,
+					__u8 host_status, __u8 sense_len_wr,
+					__u32 resid_len)
+{
+	return ((__u64)(__u32)(resid_len) << 32) |
+		((__u64)sense_len_wr << 24) |
+		((__u64)host_status << 16) |
+		((__u64)driver_status << 8) |
+		(__u64)device_status;
+}
 
 #endif /* _UAPIBSG_H */
diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h
index f7843e6bb978d..cc3b9f7dccafa 100644
--- a/include/uapi/linux/btrfs_tree.h
+++ b/include/uapi/linux/btrfs_tree.h
@@ -1245,7 +1245,8 @@ struct btrfs_free_space_info {
 	__le32 flags;
 } __attribute__ ((__packed__));
 
-#define BTRFS_FREE_SPACE_USING_BITMAPS (1ULL << 0)
+#define BTRFS_FREE_SPACE_USING_BITMAPS	(1UL << 0)
+#define BTRFS_FREE_SPACE_FLAGS_MASK	(BTRFS_FREE_SPACE_USING_BITMAPS)
 
 #define BTRFS_QGROUP_LEVEL_SHIFT		48
 static inline __u16 btrfs_qgroup_level(__u64 qgroupid)
diff --git a/include/uapi/linux/const.h b/include/uapi/linux/const.h
index b8f629ef135f3..565f309b9df8b 100644
--- a/include/uapi/linux/const.h
+++ b/include/uapi/linux/const.h
@@ -50,4 +50,22 @@
 
 #define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
 
+/*
+ * Divide positive or negative dividend by positive or negative divisor
+ * and round to closest integer. Result is undefined for negative
+ * divisors if the dividend variable type is unsigned and for negative
+ * dividends if the divisor variable type is unsigned.
+ */
+#define __KERNEL_DIV_ROUND_CLOSEST(x, divisor)		\
+({							\
+	__typeof__(x) __x = x;				\
+	__typeof__(divisor) __d = divisor;		\
+							\
+	(((__typeof__(x))-1) > 0 ||			\
+	 ((__typeof__(divisor))-1) > 0 ||		\
+	 (((__x) > 0) == ((__d) > 0))) ?		\
+		(((__x) + ((__d) / 2)) / (__d)) :	\
+		(((__x) - ((__d) / 2)) / (__d));	\
+})
+
 #endif /* _UAPI_LINUX_CONST_H */
diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h
index e7d6b6d134706..7de2d8cc862f8 100644
--- a/include/uapi/linux/devlink.h
+++ b/include/uapi/linux/devlink.h
@@ -19,6 +19,8 @@
 #define DEVLINK_GENL_VERSION 0x1
 #define DEVLINK_GENL_MCGRP_CONFIG_NAME "config"
 
+#define DEVLINK_INDEX_BUS_NAME "devlink_index"
+
 enum devlink_command {
 	/* don't change the order or add anything between, this is ABI! */
 	DEVLINK_CMD_UNSPEC,
@@ -642,6 +644,8 @@ enum devlink_attr {
 	DEVLINK_ATTR_PARAM_VALUE_DEFAULT,	/* dynamic */
 	DEVLINK_ATTR_PARAM_RESET_DEFAULT,	/* flag */
 
+	DEVLINK_ATTR_INDEX,			/* uint */
+
 	/* Add new attributes above here, update the spec in
 	 * Documentation/netlink/specs/devlink.yaml and re-generate
 	 * net/devlink/netlink_gen.c.
diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h
index b74b805085539..1cdfb8341df24 100644
--- a/include/uapi/linux/ethtool.h
+++ b/include/uapi/linux/ethtool.h
@@ -225,7 +225,7 @@ enum tunable_id {
 	ETHTOOL_ID_UNSPEC,
 	ETHTOOL_RX_COPYBREAK,
 	ETHTOOL_TX_COPYBREAK,
-	ETHTOOL_PFC_PREVENTION_TOUT, /* timeout in msecs */
+	ETHTOOL_PFC_PREVENTION_TOUT, /* both pause and pfc, see man ethtool */
 	ETHTOOL_TX_COPYBREAK_BUF_SIZE,
 	/*
 	 * Add your fresh new tunable attribute above and remember to update
diff --git a/include/uapi/linux/ethtool_netlink_generated.h b/include/uapi/linux/ethtool_netlink_generated.h
index 556a0c834df53..8134baf7860f3 100644
--- a/include/uapi/linux/ethtool_netlink_generated.h
+++ b/include/uapi/linux/ethtool_netlink_generated.h
@@ -371,6 +371,8 @@ enum {
 	ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS,
 	ETHTOOL_A_COALESCE_RX_PROFILE,
 	ETHTOOL_A_COALESCE_TX_PROFILE,
+	ETHTOOL_A_COALESCE_RX_CQE_FRAMES,
+	ETHTOOL_A_COALESCE_RX_CQE_NSECS,
 
 	__ETHTOOL_A_COALESCE_CNT,
 	ETHTOOL_A_COALESCE_MAX = (__ETHTOOL_A_COALESCE_CNT - 1)
@@ -381,6 +383,7 @@ enum {
 	ETHTOOL_A_PAUSE_STAT_PAD,
 	ETHTOOL_A_PAUSE_STAT_TX_FRAMES,
 	ETHTOOL_A_PAUSE_STAT_RX_FRAMES,
+	ETHTOOL_A_PAUSE_STAT_TX_PAUSE_STORM_EVENTS,
 
 	__ETHTOOL_A_PAUSE_STAT_CNT,
 	ETHTOOL_A_PAUSE_STAT_MAX = (__ETHTOOL_A_PAUSE_STAT_CNT - 1)
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index 70b2b661f42cb..13f71202845e8 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -657,4 +657,16 @@ struct procmap_query {
 	__u64 build_id_addr;		/* in */
 };
 
+/*
+ * Shutdown the filesystem.
+ */
+#define FS_IOC_SHUTDOWN _IOR('X', 125, __u32)
+
+/*
+ * Flags for FS_IOC_SHUTDOWN
+ */
+#define FS_SHUTDOWN_FLAGS_DEFAULT	0x0
+#define FS_SHUTDOWN_FLAGS_LOGFLUSH	0x1	/* flush log but not data*/
+#define FS_SHUTDOWN_FLAGS_NOLOGFLUSH	0x2	/* don't flush log nor data */
+
 #endif /* _UAPI_LINUX_FS_H */
diff --git a/include/uapi/linux/fsl_hypervisor.h b/include/uapi/linux/fsl_hypervisor.h
index 1e237fba951fe..ab4388441e80c 100644
--- a/include/uapi/linux/fsl_hypervisor.h
+++ b/include/uapi/linux/fsl_hypervisor.h
@@ -114,9 +114,9 @@ struct fsl_hv_ioctl_stop {
  * @target: the partition ID of the target partition, or -1 for this
  *          partition
  * @reserved: reserved, must be set to 0
- * @local_addr: user-space virtual address of a buffer in the local
+ * @local_vaddr: user-space virtual address of a buffer in the local
  *              partition
- * @remote_addr: guest physical address of a buffer in the
+ * @remote_paddr: guest physical address of a buffer in the
  *           remote partition
  * @count: the number of bytes to copy.  Both the local and remote
  *         buffers must be at least 'count' bytes long
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index e9b5f79e1ee17..83a96c56b8cad 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -1568,6 +1568,8 @@ enum {
 	IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE,
 	IFLA_BOND_SLAVE_PRIO,
 	IFLA_BOND_SLAVE_ACTOR_PORT_PRIO,
+	IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE,
+	IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE,
 	__IFLA_BOND_SLAVE_MAX,
 };
 
diff --git a/include/uapi/linux/if_pppox.h b/include/uapi/linux/if_pppox.h
index 9abd80dcc46f9..7ae044d71fb72 100644
--- a/include/uapi/linux/if_pppox.h
+++ b/include/uapi/linux/if_pppox.h
@@ -103,16 +103,6 @@ struct sockaddr_pppol2tpv3in6 {
 	struct pppol2tpv3in6_addr pppol2tp;
 } __packed;
 
-/*********************************************************************
- *
- * ioctl interface for defining forwarding of connections
- *
- ********************************************************************/
-
-#define PPPOEIOCSFWD	_IOW(0xB1 ,0, size_t)
-#define PPPOEIOCDFWD	_IO(0xB1 ,1)
-/*#define PPPOEIOCGFWD	_IOWR(0xB1,2, size_t)*/
-
 /* Codes to identify message types */
 #define PADI_CODE	0x09
 #define PADO_CODE	0x07
@@ -122,7 +112,9 @@ struct sockaddr_pppol2tpv3in6 {
 struct pppoe_tag {
 	__be16 tag_type;
 	__be16 tag_len;
+#ifndef __KERNEL__
 	char tag_data[];
+#endif
 } __attribute__ ((packed));
 
 /* Tag identifiers */
@@ -150,7 +142,9 @@ struct pppoe_hdr {
 	__u8 code;
 	__be16 sid;
 	__be16 length;
+#ifndef __KERNEL__
 	struct pppoe_tag tag[];
+#endif
 } __packed;
 
 /* Length of entire PPPoE + PPP header */
diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h
index 86bb2e8b17c9f..21f0d735fbae2 100644
--- a/include/uapi/linux/inet_diag.h
+++ b/include/uapi/linux/inet_diag.h
@@ -129,6 +129,15 @@ struct inet_diag_msg {
 	__u32	idiag_inode;
 };
 
+enum {
+	IDIAG_TIMER_OFF,
+	IDIAG_TIMER_ON,
+	IDIAG_TIMER_KEEPALIVE,
+	IDIAG_TIMER_TIMEWAIT,
+	IDIAG_TIMER_PROBE0,
+	IDIAG_TIMER_DELACK,
+};
+
 /* Extensions */
 
 enum {
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 1ff16141c8a5b..17ac1b7854405 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -10,6 +10,8 @@
 
 #include <linux/fs.h>
 #include <linux/types.h>
+#include <linux/io_uring/zcrx.h>
+
 /*
  * this file is shared with liburing and that has to autodetect
  * if linux/time_types.h is available or not, it can
@@ -341,6 +343,10 @@ enum io_uring_op {
 
 /*
  * sqe->timeout_flags
+ *
+ * IORING_TIMEOUT_IMMEDIATE_ARG:	If set, sqe->addr stores the timeout
+ *					value in nanoseconds instead of
+ *					pointing to a timespec.
  */
 #define IORING_TIMEOUT_ABS		(1U << 0)
 #define IORING_TIMEOUT_UPDATE		(1U << 1)
@@ -349,6 +355,7 @@ enum io_uring_op {
 #define IORING_LINK_TIMEOUT_UPDATE	(1U << 4)
 #define IORING_TIMEOUT_ETIME_SUCCESS	(1U << 5)
 #define IORING_TIMEOUT_MULTISHOT	(1U << 6)
+#define IORING_TIMEOUT_IMMEDIATE_ARG	(1U << 7)
 #define IORING_TIMEOUT_CLOCK_MASK	(IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
 #define IORING_TIMEOUT_UPDATE_MASK	(IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
 /*
@@ -1050,100 +1057,6 @@ struct io_timespec {
 	__u64		tv_nsec;
 };
 
-/* Zero copy receive refill queue entry */
-struct io_uring_zcrx_rqe {
-	__u64	off;
-	__u32	len;
-	__u32	__pad;
-};
-
-struct io_uring_zcrx_cqe {
-	__u64	off;
-	__u64	__pad;
-};
-
-/* The bit from which area id is encoded into offsets */
-#define IORING_ZCRX_AREA_SHIFT	48
-#define IORING_ZCRX_AREA_MASK	(~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1))
-
-struct io_uring_zcrx_offsets {
-	__u32	head;
-	__u32	tail;
-	__u32	rqes;
-	__u32	__resv2;
-	__u64	__resv[2];
-};
-
-enum io_uring_zcrx_area_flags {
-	IORING_ZCRX_AREA_DMABUF		= 1,
-};
-
-struct io_uring_zcrx_area_reg {
-	__u64	addr;
-	__u64	len;
-	__u64	rq_area_token;
-	__u32	flags;
-	__u32	dmabuf_fd;
-	__u64	__resv2[2];
-};
-
-enum zcrx_reg_flags {
-	ZCRX_REG_IMPORT	= 1,
-};
-
-enum zcrx_features {
-	/*
-	 * The user can ask for the desired rx page size by passing the
-	 * value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
-	 */
-	ZCRX_FEATURE_RX_PAGE_SIZE	= 1 << 0,
-};
-
-/*
- * Argument for IORING_REGISTER_ZCRX_IFQ
- */
-struct io_uring_zcrx_ifq_reg {
-	__u32	if_idx;
-	__u32	if_rxq;
-	__u32	rq_entries;
-	__u32	flags;
-
-	__u64	area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
-	__u64	region_ptr; /* struct io_uring_region_desc * */
-
-	struct io_uring_zcrx_offsets offsets;
-	__u32	zcrx_id;
-	__u32	rx_buf_len;
-	__u64	__resv[3];
-};
-
-enum zcrx_ctrl_op {
-	ZCRX_CTRL_FLUSH_RQ,
-	ZCRX_CTRL_EXPORT,
-
-	__ZCRX_CTRL_LAST,
-};
-
-struct zcrx_ctrl_flush_rq {
-	__u64		__resv[6];
-};
-
-struct zcrx_ctrl_export {
-	__u32		zcrx_fd;
-	__u32 		__resv1[11];
-};
-
-struct zcrx_ctrl {
-	__u32	zcrx_id;
-	__u32	op; /* see enum zcrx_ctrl_op */
-	__u64	__resv[2];
-
-	union {
-		struct zcrx_ctrl_export		zc_export;
-		struct zcrx_ctrl_flush_rq	zc_flush;
-	};
-};
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h
new file mode 100644
index 0000000000000..3163a4b8aeb02
--- /dev/null
+++ b/include/uapi/linux/io_uring/zcrx.h
@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
+/*
+ * Header file for the io_uring zerocopy receive (zcrx) interface.
+ *
+ * Copyright (C) 2026 Pavel Begunkov
+ * Copyright (C) 2026 David Wei
+ * Copyright (C) Meta Platforms, Inc.
+ */
+#ifndef LINUX_IO_ZCRX_H
+#define LINUX_IO_ZCRX_H
+
+#include <linux/types.h>
+
+/* Zero copy receive refill queue entry */
+struct io_uring_zcrx_rqe {
+	__u64	off;
+	__u32	len;
+	__u32	__pad;
+};
+
+struct io_uring_zcrx_cqe {
+	__u64	off;
+	__u64	__pad;
+};
+
+/* The bit from which area id is encoded into offsets */
+#define IORING_ZCRX_AREA_SHIFT	48
+#define IORING_ZCRX_AREA_MASK	(~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1))
+
+struct io_uring_zcrx_offsets {
+	__u32	head;
+	__u32	tail;
+	__u32	rqes;
+	__u32	__resv2;
+	__u64	__resv[2];
+};
+
+enum io_uring_zcrx_area_flags {
+	IORING_ZCRX_AREA_DMABUF		= 1,
+};
+
+struct io_uring_zcrx_area_reg {
+	__u64	addr;
+	__u64	len;
+	__u64	rq_area_token;
+	__u32	flags;
+	__u32	dmabuf_fd;
+	__u64	__resv2[2];
+};
+
+enum zcrx_reg_flags {
+	ZCRX_REG_IMPORT	= 1,
+};
+
+enum zcrx_features {
+	/*
+	 * The user can ask for the desired rx page size by passing the
+	 * value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
+	 */
+	ZCRX_FEATURE_RX_PAGE_SIZE	= 1 << 0,
+};
+
+/*
+ * Argument for IORING_REGISTER_ZCRX_IFQ
+ */
+struct io_uring_zcrx_ifq_reg {
+	__u32	if_idx;
+	__u32	if_rxq;
+	__u32	rq_entries;
+	__u32	flags;
+
+	__u64	area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
+	__u64	region_ptr; /* struct io_uring_region_desc * */
+
+	struct io_uring_zcrx_offsets offsets;
+	__u32	zcrx_id;
+	__u32	rx_buf_len;
+	__u64	__resv[3];
+};
+
+enum zcrx_ctrl_op {
+	ZCRX_CTRL_FLUSH_RQ,
+	ZCRX_CTRL_EXPORT,
+
+	__ZCRX_CTRL_LAST,
+};
+
+struct zcrx_ctrl_flush_rq {
+	__u64		__resv[6];
+};
+
+struct zcrx_ctrl_export {
+	__u32		zcrx_fd;
+	__u32 		__resv1[11];
+};
+
+struct zcrx_ctrl {
+	__u32	zcrx_id;
+	__u32	op; /* see enum zcrx_ctrl_op */
+	__u64	__resv[2];
+
+	union {
+		struct zcrx_ctrl_export		zc_export;
+		struct zcrx_ctrl_flush_rq	zc_flush;
+	};
+};
+
+#endif /* LINUX_IO_ZCRX_H */
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index 1dafbc552d37d..507ee9bcba01f 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -695,11 +695,15 @@ enum iommu_hw_info_type {
  * @IOMMU_HW_CAP_PCI_PASID_PRIV: Privileged Mode Supported, user ignores it
  *                               when the struct
  *                               iommu_hw_info::out_max_pasid_log2 is zero.
+ * @IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED: ATS is not supported or cannot be used
+ *                                      on this device (absence implies ATS
+ *                                      may be enabled)
  */
 enum iommufd_hw_capabilities {
 	IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0,
 	IOMMU_HW_CAP_PCI_PASID_EXEC = 1 << 1,
 	IOMMU_HW_CAP_PCI_PASID_PRIV = 1 << 2,
+	IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED = 1 << 3,
 };
 
 /**
diff --git a/include/uapi/linux/keyboard.h b/include/uapi/linux/keyboard.h
index 36d230cedf124..48ecb0cefb453 100644
--- a/include/uapi/linux/keyboard.h
+++ b/include/uapi/linux/keyboard.h
@@ -41,6 +41,7 @@
 #define KT_SLOCK	12
 #define KT_DEAD2	13
 #define KT_BRL		14
+#define KT_CSI		15	/* CSI sequences with modifier support */
 
 #define K(t,v)		(((t)<<8)|(v))
 #define KTYP(x)		((x) >> 8)
@@ -461,5 +462,33 @@
 
 #define NR_BRL		11
 
+/* KT_CSI keys: value is the CSI parameter number for ESC [ <value> ~ */
+#define K_CSI_HOME	K(KT_CSI, 1)	/* ESC [ 1 ~ */
+#define K_CSI_INSERT	K(KT_CSI, 2)	/* ESC [ 2 ~ */
+#define K_CSI_DELETE	K(KT_CSI, 3)	/* ESC [ 3 ~ */
+#define K_CSI_END	K(KT_CSI, 4)	/* ESC [ 4 ~ */
+#define K_CSI_PGUP	K(KT_CSI, 5)	/* ESC [ 5 ~ */
+#define K_CSI_PGDN	K(KT_CSI, 6)	/* ESC [ 6 ~ */
+#define K_CSI_F1	K(KT_CSI, 11)	/* ESC [ 11 ~ */
+#define K_CSI_F2	K(KT_CSI, 12)	/* ESC [ 12 ~ */
+#define K_CSI_F3	K(KT_CSI, 13)	/* ESC [ 13 ~ */
+#define K_CSI_F4	K(KT_CSI, 14)	/* ESC [ 14 ~ */
+#define K_CSI_F5	K(KT_CSI, 15)	/* ESC [ 15 ~ */
+#define K_CSI_F6	K(KT_CSI, 17)	/* ESC [ 17 ~ */
+#define K_CSI_F7	K(KT_CSI, 18)	/* ESC [ 18 ~ */
+#define K_CSI_F8	K(KT_CSI, 19)	/* ESC [ 19 ~ */
+#define K_CSI_F9	K(KT_CSI, 20)	/* ESC [ 20 ~ */
+#define K_CSI_F10	K(KT_CSI, 21)	/* ESC [ 21 ~ */
+#define K_CSI_F11	K(KT_CSI, 23)	/* ESC [ 23 ~ */
+#define K_CSI_F12	K(KT_CSI, 24)	/* ESC [ 24 ~ */
+#define K_CSI_F13	K(KT_CSI, 25)	/* ESC [ 25 ~ */
+#define K_CSI_F14	K(KT_CSI, 26)	/* ESC [ 26 ~ */
+#define K_CSI_F15	K(KT_CSI, 28)	/* ESC [ 28 ~ */
+#define K_CSI_F16	K(KT_CSI, 29)	/* ESC [ 29 ~ */
+#define K_CSI_F17	K(KT_CSI, 31)	/* ESC [ 31 ~ */
+#define K_CSI_F18	K(KT_CSI, 32)	/* ESC [ 32 ~ */
+#define K_CSI_F19	K(KT_CSI, 33)	/* ESC [ 33 ~ */
+#define K_CSI_F20	K(KT_CSI, 34)	/* ESC [ 34 ~ */
+
 #define MAX_DIACR	256
 #endif /* _UAPI__LINUX_KEYBOARD_H */
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 80364d4dbebb0..c3d5fe8c8815b 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -11,6 +11,7 @@
 #include <linux/const.h>
 #include <linux/types.h>
 #include <linux/compiler.h>
+#include <linux/stddef.h>
 #include <linux/ioctl.h>
 #include <asm/kvm.h>
 
@@ -542,7 +543,7 @@ struct kvm_coalesced_mmio {
 
 struct kvm_coalesced_mmio_ring {
 	__u32 first, last;
-	struct kvm_coalesced_mmio coalesced_mmio[];
+	__DECLARE_FLEX_ARRAY(struct kvm_coalesced_mmio, coalesced_mmio);
 };
 
 #define KVM_COALESCED_MMIO_MAX \
@@ -592,7 +593,7 @@ struct kvm_clear_dirty_log {
 /* for KVM_SET_SIGNAL_MASK */
 struct kvm_signal_mask {
 	__u32 len;
-	__u8  sigset[];
+	__DECLARE_FLEX_ARRAY(__u8, sigset);
 };
 
 /* for KVM_TPR_ACCESS_REPORTING */
@@ -1051,7 +1052,7 @@ struct kvm_irq_routing_entry {
 struct kvm_irq_routing {
 	__u32 nr;
 	__u32 flags;
-	struct kvm_irq_routing_entry entries[];
+	__DECLARE_FLEX_ARRAY(struct kvm_irq_routing_entry, entries);
 };
 
 #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)
@@ -1142,7 +1143,7 @@ struct kvm_dirty_tlb {
 
 struct kvm_reg_list {
 	__u64 n; /* number of regs */
-	__u64 reg[];
+	__DECLARE_FLEX_ARRAY(__u64, reg);
 };
 
 struct kvm_one_reg {
@@ -1224,6 +1225,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_LOONGARCH_EIOINTC	KVM_DEV_TYPE_LOONGARCH_EIOINTC
 	KVM_DEV_TYPE_LOONGARCH_PCHPIC,
 #define KVM_DEV_TYPE_LOONGARCH_PCHPIC	KVM_DEV_TYPE_LOONGARCH_PCHPIC
+	KVM_DEV_TYPE_ARM_VGIC_V5,
+#define KVM_DEV_TYPE_ARM_VGIC_V5	KVM_DEV_TYPE_ARM_VGIC_V5
 
 	KVM_DEV_TYPE_MAX,
 
@@ -1608,7 +1611,7 @@ struct kvm_stats_desc {
 #ifdef __KERNEL__
 	char name[KVM_STATS_NAME_SIZE];
 #else
-	char name[];
+	__DECLARE_FLEX_ARRAY(char, name);
 #endif
 };
 
diff --git a/include/uapi/linux/map_benchmark.h b/include/uapi/linux/map_benchmark.h
index c2d91088a40dd..4b17829a9f17a 100644
--- a/include/uapi/linux/map_benchmark.h
+++ b/include/uapi/linux/map_benchmark.h
@@ -17,6 +17,12 @@
 #define DMA_MAP_TO_DEVICE       1
 #define DMA_MAP_FROM_DEVICE     2
 
+enum {
+	DMA_MAP_BENCH_SINGLE_MODE,
+	DMA_MAP_BENCH_SG_MODE,
+	DMA_MAP_BENCH_MODE_MAX
+};
+
 struct map_benchmark {
 	__u64 avg_map_100ns; /* average map latency in 100ns */
 	__u64 map_stddev; /* standard deviation of map latency */
@@ -28,8 +34,11 @@ struct map_benchmark {
 	__u32 dma_bits; /* DMA addressing capability */
 	__u32 dma_dir; /* DMA data direction */
 	__u32 dma_trans_ns; /* time for DMA transmission in ns */
-	__u32 granule;  /* how many PAGE_SIZE will do map/unmap once a time */
-	__u8 expansion[76]; /* For future use */
+	__u32 granule;  /* - SINGLE_MODE: number of pages mapped/unmapped per operation
+			 * - SG_MODE: number of scatterlist entries (each maps one page)
+			 */
+	__u8 map_mode;  /* the mode of dma map */
+	__u8 expansion[75]; /* For future use */
 };
 
 #endif /* _UAPI_DMA_BENCHMARK_H */
diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h
index d9d86598d100c..2204708dbf7a5 100644
--- a/include/uapi/linux/mount.h
+++ b/include/uapi/linux/mount.h
@@ -110,6 +110,7 @@ enum fsconfig_command {
  * fsmount() flags.
  */
 #define FSMOUNT_CLOEXEC		0x00000001
+#define FSMOUNT_NAMESPACE	0x00000002	/* Create the mount in a new mount namespace */
 
 /*
  * Mount attributes.
diff --git a/include/uapi/linux/nfc.h b/include/uapi/linux/nfc.h
index 2f5b4be252619..82805eee43574 100644
--- a/include/uapi/linux/nfc.h
+++ b/include/uapi/linux/nfc.h
@@ -55,7 +55,7 @@
  *	(it sends %NFC_ATTR_DEVICE_INDEX)
  * @NFC_EVENT_TM_ACTIVATED: event emitted when the adapter is activated in
  *      target mode.
- * @NFC_EVENT_DEVICE_DEACTIVATED: event emitted when the adapter is deactivated
+ * @NFC_EVENT_TM_DEACTIVATED: event emitted when the adapter is deactivated
  *      from target mode.
  * @NFC_CMD_LLC_GET_PARAMS: request LTO, RW, and MIUX parameters for a device
  * @NFC_CMD_LLC_SET_PARAMS: set one or more of LTO, RW, and MIUX parameters for
@@ -156,7 +156,7 @@ enum nfc_commands {
  * @NFC_ATTR_SE_INDEX: Secure element index
  * @NFC_ATTR_SE_TYPE: Secure element type (UICC or EMBEDDED)
  * @NFC_ATTR_FIRMWARE_DOWNLOAD_STATUS: Firmware download operation status
- * @NFC_ATTR_APDU: Secure element APDU
+ * @NFC_ATTR_SE_APDU: Secure element APDU
  * @NFC_ATTR_TARGET_ISO15693_DSFID: ISO 15693 Data Storage Format Identifier
  * @NFC_ATTR_TARGET_ISO15693_UID: ISO 15693 Unique Identifier
  * @NFC_ATTR_SE_PARAMS: Parameters data from an evt_transaction
@@ -291,7 +291,7 @@ struct sockaddr_nfc_llcp {
 
 #define NFC_HEADER_SIZE 1
 
-/**
+/*
  * Pseudo-header info for raw socket packets
  * First byte is the adapter index
  * Second byte contains flags
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index b63f718509060..67d7640239883 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -1361,6 +1361,12 @@
  *	user space that the NAN new cluster has been joined. The cluster ID is
  *	indicated by %NL80211_ATTR_MAC.
  *
+ * @NL80211_CMD_INCUMBENT_SIGNAL_DETECT: Once any incumbent signal is detected
+ *	on the operating channel in 6 GHz band, userspace is notified with the
+ *	signal interference bitmap using
+ *	%NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP. The current channel
+ *	definition is also sent.
+ *
  * @NL80211_CMD_MAX: highest used command number
  * @__NL80211_CMD_AFTER_LAST: internal use
  */
@@ -1624,6 +1630,8 @@ enum nl80211_commands {
 	NL80211_CMD_NAN_NEXT_DW_NOTIFICATION,
 	NL80211_CMD_NAN_CLUSTER_JOINED,
 
+	NL80211_CMD_INCUMBENT_SIGNAL_DETECT,
+
 	/* add new commands above here */
 
 	/* used to define NL80211_CMD_MAX below */
@@ -2984,6 +2992,19 @@ enum nl80211_commands {
  *	this feature during association. This is a flag attribute.
  *	Currently only supported in mac80211 drivers.
  *
+ * @NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP: u32 attribute specifying
+ *	the signal interference bitmap detected on the operating bandwidth for
+ *	%NL80211_CMD_INCUMBENT_SIGNAL_DETECT. Each bit represents a 20 MHz
+ *	segment, lowest bit corresponds to the lowest 20 MHz segment, in the
+ *	operating bandwidth where the interference is detected. Punctured
+ *	sub-channels are included in the bitmap structure; however, since
+ *	interference detection is not performed on these sub-channels, their
+ *	corresponding bits are consistently set to zero.
+ *
+ * @NL80211_ATTR_UHR_OPERATION: Full UHR Operation element, as it appears in
+ *	association response etc., since it's abridged in the beacon. Used
+ *	for START_AP etc.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -3557,6 +3578,10 @@ enum nl80211_attrs {
 	NL80211_ATTR_UHR_CAPABILITY,
 	NL80211_ATTR_DISABLE_UHR,
 
+	NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP,
+
+	NL80211_ATTR_UHR_OPERATION,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -4480,6 +4505,10 @@ enum nl80211_wmm_rule {
  *	as a non-primary subchannel. Only applicable to S1G channels.
  * @NL80211_FREQUENCY_ATTR_NO_UHR: UHR operation is not allowed on this channel
  *	in current regulatory domain.
+ * @NL80211_FREQUENCY_ATTR_CAC_START_TIME: Channel Availability Check (CAC)
+ *	start time (CLOCK_BOOTTIME, nanoseconds). Only present when CAC is
+ *	currently in progress on this channel.
+ * @NL80211_FREQUENCY_ATTR_PAD: attribute used for padding for 64-bit alignment
  * @NL80211_FREQUENCY_ATTR_MAX: highest frequency attribute number
  *	currently defined
  * @__NL80211_FREQUENCY_ATTR_AFTER_LAST: internal use
@@ -4530,6 +4559,8 @@ enum nl80211_frequency_attr {
 	NL80211_FREQUENCY_ATTR_NO_16MHZ,
 	NL80211_FREQUENCY_ATTR_S1G_NO_PRIMARY,
 	NL80211_FREQUENCY_ATTR_NO_UHR,
+	NL80211_FREQUENCY_ATTR_CAC_START_TIME,
+	NL80211_FREQUENCY_ATTR_PAD,
 
 	/* keep last */
 	__NL80211_FREQUENCY_ATTR_AFTER_LAST,
@@ -5466,6 +5497,8 @@ enum nl80211_bss_status {
  * @NL80211_AUTHTYPE_FILS_SK_PFS: Fast Initial Link Setup shared key with PFS
  * @NL80211_AUTHTYPE_FILS_PK: Fast Initial Link Setup public key
  * @NL80211_AUTHTYPE_EPPKE: Enhanced Privacy Protection Key Exchange
+ * @NL80211_AUTHTYPE_IEEE8021X: IEEE 802.1X authentication utilizing
+ *	Authentication frames
  * @__NL80211_AUTHTYPE_NUM: internal
  * @NL80211_AUTHTYPE_MAX: maximum valid auth algorithm
  * @NL80211_AUTHTYPE_AUTOMATIC: determine automatically (if necessary by
@@ -5482,6 +5515,7 @@ enum nl80211_auth_type {
 	NL80211_AUTHTYPE_FILS_SK_PFS,
 	NL80211_AUTHTYPE_FILS_PK,
 	NL80211_AUTHTYPE_EPPKE,
+	NL80211_AUTHTYPE_IEEE8021X,
 
 	/* keep last */
 	__NL80211_AUTHTYPE_NUM,
@@ -6795,6 +6829,11 @@ enum nl80211_feature_flags {
  *	frames in both non‑AP STA and AP mode as specified in
  *	"IEEE P802.11bi/D3.0, 12.16.6".
  *
+ * @NL80211_EXT_FEATURE_IEEE8021X_AUTH: Driver supports IEEE 802.1X
+ *	authentication utilizing Authentication frames with user space SME
+ *	(NL80211_CMD_AUTHENTICATE) in non-AP STA mode, as specified in
+ *	"IEEE P802.11bi/D4.0, 12.16.5".
+ *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
  */
@@ -6873,6 +6912,7 @@ enum nl80211_ext_feature_index {
 	NL80211_EXT_FEATURE_BEACON_RATE_EHT,
 	NL80211_EXT_FEATURE_EPPKE,
 	NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION,
+	NL80211_EXT_FEATURE_IEEE8021X_AUTH,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
diff --git a/include/uapi/linux/openvswitch.h b/include/uapi/linux/openvswitch.h
index 3092c2c6f1d24..aa2acdbda8f89 100644
--- a/include/uapi/linux/openvswitch.h
+++ b/include/uapi/linux/openvswitch.h
@@ -70,12 +70,15 @@ enum ovs_datapath_cmd {
  * set on the datapath port (for OVS_ACTION_ATTR_MISS).  Only valid on
  * %OVS_DP_CMD_NEW requests. A value of zero indicates that upcalls should
  * not be sent.
+ * @OVS_DP_ATTR_MASKS_CACHE_SIZE: Number of the entries in the flow table
+ * masks cache.
  * @OVS_DP_ATTR_PER_CPU_PIDS: Per-cpu array of PIDs for upcalls when
  * OVS_DP_F_DISPATCH_UPCALL_PER_CPU feature is set.
  * @OVS_DP_ATTR_STATS: Statistics about packets that have passed through the
  * datapath.  Always present in notifications.
  * @OVS_DP_ATTR_MEGAFLOW_STATS: Statistics about mega flow masks usage for the
  * datapath. Always present in notifications.
+ * @OVS_DP_ATTR_USER_FEATURES: OVS_DP_F_* flags.
  * @OVS_DP_ATTR_IFINDEX: Interface index for a new datapath netdev. Only
  * valid for %OVS_DP_CMD_NEW requests.
  *
@@ -83,18 +86,23 @@ enum ovs_datapath_cmd {
  * payload for %OVS_DP_* commands.
  */
 enum ovs_datapath_attr {
+	/* private: */
 	OVS_DP_ATTR_UNSPEC,
+	/* public: */
 	OVS_DP_ATTR_NAME,		/* name of dp_ifindex netdev */
 	OVS_DP_ATTR_UPCALL_PID,		/* Netlink PID to receive upcalls */
 	OVS_DP_ATTR_STATS,		/* struct ovs_dp_stats */
 	OVS_DP_ATTR_MEGAFLOW_STATS,	/* struct ovs_dp_megaflow_stats */
 	OVS_DP_ATTR_USER_FEATURES,	/* OVS_DP_F_*  */
+	/* private: */
 	OVS_DP_ATTR_PAD,
+	/* public: */
 	OVS_DP_ATTR_MASKS_CACHE_SIZE,
 	OVS_DP_ATTR_PER_CPU_PIDS,   /* Netlink PIDS to receive upcalls in
 				     * per-cpu dispatch mode
 				     */
 	OVS_DP_ATTR_IFINDEX,
+	/* private: */
 	__OVS_DP_ATTR_MAX
 };
 
@@ -181,6 +189,7 @@ enum ovs_packet_cmd {
  * %OVS_USERSPACE_ATTR_EGRESS_TUN_PORT attribute, which is sent only if the
  * output port is actually a tunnel port. Contains the output tunnel key
  * extracted from the packet as nested %OVS_TUNNEL_KEY_ATTR_* attributes.
+ * @OVS_PACKET_ATTR_PROBE: Packet operation is a feature probe.
  * @OVS_PACKET_ATTR_MRU: Present for an %OVS_PACKET_CMD_ACTION and
  * @OVS_PACKET_ATTR_LEN: Packet size before truncation.
  * %OVS_PACKET_ATTR_USERSPACE action specify the Maximum received fragment
@@ -196,21 +205,26 @@ enum ovs_packet_cmd {
  * payload for %OVS_PACKET_* commands.
  */
 enum ovs_packet_attr {
+	/* private: */
 	OVS_PACKET_ATTR_UNSPEC,
+	/* public: */
 	OVS_PACKET_ATTR_PACKET,      /* Packet data. */
 	OVS_PACKET_ATTR_KEY,         /* Nested OVS_KEY_ATTR_* attributes. */
 	OVS_PACKET_ATTR_ACTIONS,     /* Nested OVS_ACTION_ATTR_* attributes. */
 	OVS_PACKET_ATTR_USERDATA,    /* OVS_ACTION_ATTR_USERSPACE arg. */
 	OVS_PACKET_ATTR_EGRESS_TUN_KEY,  /* Nested OVS_TUNNEL_KEY_ATTR_*
 					    attributes. */
+	/* private: */
 	OVS_PACKET_ATTR_UNUSED1,
 	OVS_PACKET_ATTR_UNUSED2,
+	/* public: */
 	OVS_PACKET_ATTR_PROBE,      /* Packet operation is a feature probe,
 				       error logging should be suppressed. */
 	OVS_PACKET_ATTR_MRU,	    /* Maximum received IP fragment size. */
 	OVS_PACKET_ATTR_LEN,	    /* Packet size before truncation. */
 	OVS_PACKET_ATTR_HASH,	    /* Packet hash. */
 	OVS_PACKET_ATTR_UPCALL_PID, /* u32 Netlink PID. */
+	/* private: */
 	__OVS_PACKET_ATTR_MAX
 };
 
@@ -257,6 +271,11 @@ enum ovs_vport_type {
  * upcalls should not be sent.
  * @OVS_VPORT_ATTR_STATS: A &struct ovs_vport_stats giving statistics for
  * packets sent or received through the vport.
+ * @OVS_VPORT_ATTR_IFINDEX: Provides the ifindex of a vport, or sets the desired
+ * ifindex while creating a new vport with type %OVS_VPORT_TYPE_INTERNAL.
+ * @OVS_VPORT_ATTR_NETNSID: Provides the netns id of the vport if it's not local.
+ * @OVS_VPORT_ATTR_UPCALL_STATS: Provides upcall statistics for a vport.
+ * Contains nested %OVS_VPORT_UPCALL_ATTR_* attributes.
  *
  * These attributes follow the &struct ovs_header within the Generic Netlink
  * payload for %OVS_VPORT_* commands.
@@ -272,7 +291,9 @@ enum ovs_vport_type {
  * ovs_header plus %OVS_VPORT_ATTR_PORT_NO determine the vport.
  */
 enum ovs_vport_attr {
+	/* private: */
 	OVS_VPORT_ATTR_UNSPEC,
+	/* public: */
 	OVS_VPORT_ATTR_PORT_NO,	/* u32 port number within datapath */
 	OVS_VPORT_ATTR_TYPE,	/* u32 OVS_VPORT_TYPE_* constant. */
 	OVS_VPORT_ATTR_NAME,	/* string name, up to IFNAMSIZ bytes long */
@@ -280,23 +301,27 @@ enum ovs_vport_attr {
 	OVS_VPORT_ATTR_UPCALL_PID, /* array of u32 Netlink socket PIDs for */
 				/* receiving upcalls */
 	OVS_VPORT_ATTR_STATS,	/* struct ovs_vport_stats */
+	/* private: */
 	OVS_VPORT_ATTR_PAD,
+	/* public: */
 	OVS_VPORT_ATTR_IFINDEX,
 	OVS_VPORT_ATTR_NETNSID,
 	OVS_VPORT_ATTR_UPCALL_STATS,
+	/* private: */
 	__OVS_VPORT_ATTR_MAX
 };
 
 #define OVS_VPORT_ATTR_MAX (__OVS_VPORT_ATTR_MAX - 1)
 
 /**
- * enum ovs_vport_upcall_attr - attributes for %OVS_VPORT_UPCALL* commands
- * @OVS_VPORT_UPCALL_SUCCESS: 64-bit upcall success packets.
- * @OVS_VPORT_UPCALL_FAIL: 64-bit upcall fail packets.
+ * enum ovs_vport_upcall_attr - attributes for %OVS_VPORT_ATTR_UPCALL_STATS
+ * @OVS_VPORT_UPCALL_ATTR_SUCCESS: 64-bit upcall success packets.
+ * @OVS_VPORT_UPCALL_ATTR_FAIL: 64-bit upcall fail packets.
  */
 enum ovs_vport_upcall_attr {
 	OVS_VPORT_UPCALL_ATTR_SUCCESS,
 	OVS_VPORT_UPCALL_ATTR_FAIL,
+	/* private: */
 	__OVS_VPORT_UPCALL_ATTR_MAX
 };
 
@@ -431,6 +456,7 @@ enum ovs_frag_type {
 	OVS_FRAG_TYPE_NONE,
 	OVS_FRAG_TYPE_FIRST,
 	OVS_FRAG_TYPE_LATER,
+	/* private: */
 	__OVS_FRAG_TYPE_MAX
 };
 
@@ -604,6 +630,8 @@ struct ovs_nsh_key_md1 {
  * a wildcarded match. Omitting attribute is treated as wildcarding all
  * corresponding fields. Optional for all requests. If not present,
  * all flow key bits are exact match bits.
+ * @OVS_FLOW_ATTR_PROBE: Flow operation is a feature probe, error logging
+ * should be suppressed.
  * @OVS_FLOW_ATTR_UFID: A value between 1-16 octets specifying a unique
  * identifier for the flow. Causes the flow to be indexed by this value rather
  * than the value of the %OVS_FLOW_ATTR_KEY attribute. Optional for all
@@ -617,7 +645,9 @@ struct ovs_nsh_key_md1 {
  * payload for %OVS_FLOW_* commands.
  */
 enum ovs_flow_attr {
+	/* private: */
 	OVS_FLOW_ATTR_UNSPEC,
+	/* public: */
 	OVS_FLOW_ATTR_KEY,       /* Sequence of OVS_KEY_ATTR_* attributes. */
 	OVS_FLOW_ATTR_ACTIONS,   /* Nested OVS_ACTION_ATTR_* attributes. */
 	OVS_FLOW_ATTR_STATS,     /* struct ovs_flow_stats. */
@@ -629,13 +659,14 @@ enum ovs_flow_attr {
 				  * logging should be suppressed. */
 	OVS_FLOW_ATTR_UFID,      /* Variable length unique flow identifier. */
 	OVS_FLOW_ATTR_UFID_FLAGS,/* u32 of OVS_UFID_F_*. */
+	/* private: */
 	OVS_FLOW_ATTR_PAD,
 	__OVS_FLOW_ATTR_MAX
 };
 
 #define OVS_FLOW_ATTR_MAX (__OVS_FLOW_ATTR_MAX - 1)
 
-/**
+/*
  * Omit attributes for notifications.
  *
  * If a datapath request contains an %OVS_UFID_F_OMIT_* flag, then the datapath
@@ -653,17 +684,23 @@ enum ovs_flow_attr {
  * fractions of packets.
  * @OVS_SAMPLE_ATTR_ACTIONS: Set of actions to execute in sampling event.
  * Actions are passed as nested attributes.
+ * @OVS_SAMPLE_ATTR_ARG: For in-kernel use, passing &struct sample_arg
+ * derived from other attributes.
  *
  * Executes the specified actions with the given probability on a per-packet
  * basis. Nested actions will be able to access the probability value of the
  * parent @OVS_ACTION_ATTR_SAMPLE.
  */
 enum ovs_sample_attr {
+	/* private: */
 	OVS_SAMPLE_ATTR_UNSPEC,
+	/* public: */
 	OVS_SAMPLE_ATTR_PROBABILITY, /* u32 number */
 	OVS_SAMPLE_ATTR_ACTIONS,     /* Nested OVS_ACTION_ATTR_* attributes. */
+	/* private: */
 	__OVS_SAMPLE_ATTR_MAX,
 
+	/* public: */
 #ifdef __KERNEL__
 	OVS_SAMPLE_ATTR_ARG          /* struct sample_arg  */
 #endif
@@ -693,12 +730,15 @@ struct sample_arg {
  * @OVS_USERSPACE_ATTR_ACTIONS: If present, send actions with upcall.
  */
 enum ovs_userspace_attr {
+	/* private: */
 	OVS_USERSPACE_ATTR_UNSPEC,
+	/* public: */
 	OVS_USERSPACE_ATTR_PID,	      /* u32 Netlink PID to receive upcalls. */
 	OVS_USERSPACE_ATTR_USERDATA,  /* Optional user-specified cookie. */
 	OVS_USERSPACE_ATTR_EGRESS_TUN_PORT,  /* Optional, u32 output port
 					      * to get tunnel info. */
 	OVS_USERSPACE_ATTR_ACTIONS,   /* Optional flag to get actions. */
+	/* private: */
 	__OVS_USERSPACE_ATTR_MAX
 };
 
@@ -819,7 +859,9 @@ struct ovs_action_hash {
  * @OVS_CT_ATTR_TIMEOUT: Variable length string defining conntrack timeout.
  */
 enum ovs_ct_attr {
+	/* private: */
 	OVS_CT_ATTR_UNSPEC,
+	/* public: */
 	OVS_CT_ATTR_COMMIT,     /* No argument, commits connection. */
 	OVS_CT_ATTR_ZONE,       /* u16 zone id. */
 	OVS_CT_ATTR_MARK,       /* mark to associate with this connection. */
@@ -831,6 +873,7 @@ enum ovs_ct_attr {
 	OVS_CT_ATTR_EVENTMASK,  /* u32 mask of IPCT_* events. */
 	OVS_CT_ATTR_TIMEOUT,	/* Associate timeout with this connection for
 				 * fine-grain timeout tuning. */
+	/* private: */
 	__OVS_CT_ATTR_MAX
 };
 
@@ -859,7 +902,9 @@ enum ovs_ct_attr {
  * @OVS_NAT_ATTR_PROTO_RANDOM: Flag for fully randomized L4 port mapping
  */
 enum ovs_nat_attr {
+	/* private: */
 	OVS_NAT_ATTR_UNSPEC,
+	/* public: */
 	OVS_NAT_ATTR_SRC,
 	OVS_NAT_ATTR_DST,
 	OVS_NAT_ATTR_IP_MIN,
@@ -869,38 +914,44 @@ enum ovs_nat_attr {
 	OVS_NAT_ATTR_PERSISTENT,
 	OVS_NAT_ATTR_PROTO_HASH,
 	OVS_NAT_ATTR_PROTO_RANDOM,
+	/* private: */
 	__OVS_NAT_ATTR_MAX,
 };
 
 #define OVS_NAT_ATTR_MAX (__OVS_NAT_ATTR_MAX - 1)
 
-/*
+/**
  * struct ovs_action_push_eth - %OVS_ACTION_ATTR_PUSH_ETH action argument.
  * @addresses: Source and destination MAC addresses.
- * @eth_type: Ethernet type
  */
 struct ovs_action_push_eth {
 	struct ovs_key_ethernet addresses;
 };
 
-/*
+/**
  * enum ovs_check_pkt_len_attr - Attributes for %OVS_ACTION_ATTR_CHECK_PKT_LEN.
  *
  * @OVS_CHECK_PKT_LEN_ATTR_PKT_LEN: u16 Packet length to check for.
  * @OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER: Nested OVS_ACTION_ATTR_*
  * actions to apply if the packer length is greater than the specified
  * length in the attr - OVS_CHECK_PKT_LEN_ATTR_PKT_LEN.
- * @OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL - Nested OVS_ACTION_ATTR_*
+ * @OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL: Nested OVS_ACTION_ATTR_*
  * actions to apply if the packer length is lesser or equal to the specified
  * length in the attr - OVS_CHECK_PKT_LEN_ATTR_PKT_LEN.
+ * @OVS_CHECK_PKT_LEN_ATTR_ARG: For in-kernel use, passing &struct
+ * check_pkt_len_arg derived from other attributes.
  */
 enum ovs_check_pkt_len_attr {
+	/* private: */
 	OVS_CHECK_PKT_LEN_ATTR_UNSPEC,
+	/* public: */
 	OVS_CHECK_PKT_LEN_ATTR_PKT_LEN,
 	OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER,
 	OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL,
+	/* private: */
 	__OVS_CHECK_PKT_LEN_ATTR_MAX,
 
+	/* public: */
 #ifdef __KERNEL__
 	OVS_CHECK_PKT_LEN_ATTR_ARG          /* struct check_pkt_len_arg  */
 #endif
@@ -968,6 +1019,9 @@ enum ovs_psample_attr {
  * from the packet.
  * @OVS_ACTION_ATTR_SAMPLE: Probabilitically executes actions, as specified in
  * the nested %OVS_SAMPLE_ATTR_* attributes.
+ * @OVS_ACTION_ATTR_RECIRC: Recirculate the clone of the packet through the
+ * datapath with the new id (u32 recirc_id).
+ * @OVS_ACTION_ATTR_HASH: Compute the packet hash, using &struct ovs_action_hash.
  * @OVS_ACTION_ATTR_PUSH_MPLS: Push a new MPLS label stack entry onto the
  * top of the packets MPLS label stack.  Set the ethertype of the
  * encapsulating frame to either %ETH_P_MPLS_UC or %ETH_P_MPLS_MC to
@@ -997,6 +1051,8 @@ enum ovs_psample_attr {
  * start of the packet or at the start of the l3 header depending on the value
  * of l3 tunnel flag in the tun_flags field of OVS_ACTION_ATTR_ADD_MPLS
  * argument.
+ * @OVS_ACTION_ATTR_DEC_TTL: Decrement TTL or hop limit of the packet. Execute
+ * nested %OVS_DEC_TTL_ATTR_* actions if the value is less or equal to 1.
  * @OVS_ACTION_ATTR_DROP: Explicit drop action.
  * @OVS_ACTION_ATTR_PSAMPLE: Send a sample of the packet to external observers
  * via psample.
@@ -1010,7 +1066,9 @@ enum ovs_psample_attr {
  */
 
 enum ovs_action_attr {
+	/* private: */
 	OVS_ACTION_ATTR_UNSPEC,
+	/* public: */
 	OVS_ACTION_ATTR_OUTPUT,	      /* u32 port number. */
 	OVS_ACTION_ATTR_USERSPACE,    /* Nested OVS_USERSPACE_ATTR_*. */
 	OVS_ACTION_ATTR_SET,          /* One nested OVS_KEY_ATTR_*. */
@@ -1040,9 +1098,11 @@ enum ovs_action_attr {
 	OVS_ACTION_ATTR_DROP,         /* u32 error code. */
 	OVS_ACTION_ATTR_PSAMPLE,      /* Nested OVS_PSAMPLE_ATTR_*. */
 
+	/* private: */
 	__OVS_ACTION_ATTR_MAX,	      /* Nothing past this will be accepted
 				       * from userspace. */
 
+	/* public: */
 #ifdef __KERNEL__
 	OVS_ACTION_ATTR_SET_TO_MASKED, /* Kernel module internal masked
 					* set action converted from
diff --git a/include/uapi/linux/ovpn.h b/include/uapi/linux/ovpn.h
index 959b41def61f6..06690090a1a95 100644
--- a/include/uapi/linux/ovpn.h
+++ b/include/uapi/linux/ovpn.h
@@ -55,6 +55,7 @@ enum {
 	OVPN_A_PEER_LINK_TX_BYTES,
 	OVPN_A_PEER_LINK_RX_PACKETS,
 	OVPN_A_PEER_LINK_TX_PACKETS,
+	OVPN_A_PEER_TX_ID,
 
 	__OVPN_A_PEER_MAX,
 	OVPN_A_PEER_MAX = (__OVPN_A_PEER_MAX - 1)
@@ -100,6 +101,7 @@ enum {
 	OVPN_CMD_KEY_SWAP,
 	OVPN_CMD_KEY_SWAP_NTF,
 	OVPN_CMD_KEY_DEL,
+	OVPN_CMD_PEER_FLOAT_NTF,
 
 	__OVPN_CMD_MAX,
 	OVPN_CMD_MAX = (__OVPN_CMD_MAX - 1)
diff --git a/include/uapi/linux/pidfd.h b/include/uapi/linux/pidfd.h
index ea9a6811fc768..9281956a9f32b 100644
--- a/include/uapi/linux/pidfd.h
+++ b/include/uapi/linux/pidfd.h
@@ -13,6 +13,7 @@
 #ifdef __KERNEL__
 #include <linux/sched.h>
 #define PIDFD_STALE CLONE_PIDFD
+#define PIDFD_AUTOKILL O_TRUNC
 #endif
 
 /* Flags for pidfd_send_signal(). */
diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h
index 2b5b042eb73b6..52dae70b058b6 100644
--- a/include/uapi/linux/psp-sev.h
+++ b/include/uapi/linux/psp-sev.h
@@ -277,7 +277,7 @@ struct sev_user_data_snp_wrapped_vlek_hashstick {
  * struct sev_issue_cmd - SEV ioctl parameters
  *
  * @cmd: SEV commands to execute
- * @opaque: pointer to the command structure
+ * @data: pointer to the command structure
  * @error: SEV FW return code on failure
  */
 struct sev_issue_cmd {
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
index 359a14cc76a40..c15966e307b86 100644
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -36,6 +36,9 @@
 /* Flags for the clone3() syscall. */
 #define CLONE_CLEAR_SIGHAND 0x100000000ULL /* Clear any signal handler and reset to SIG_DFL. */
 #define CLONE_INTO_CGROUP 0x200000000ULL /* Clone into a specific cgroup given the right permissions. */
+#define CLONE_AUTOREAP 0x400000000ULL /* Auto-reap child on exit. */
+#define CLONE_PIDFD_AUTOKILL 0x800000000ULL /* Kill child when clone pidfd closes. */
+#define CLONE_NNP 0x1000000000ULL /* Set no_new_privs on child. */
 
 /*
  * cloning flags intersect with CSIGNAL so can be used with unshare and clone3
@@ -146,4 +149,7 @@ struct clone_args {
 			 SCHED_FLAG_KEEP_ALL		| \
 			 SCHED_FLAG_UTIL_CLAMP)
 
+/* Only for sched_getattr() own flag param, if task is SCHED_DEADLINE */
+#define SCHED_GETATTR_FLAG_DL_DYNAMIC	0x01
+
 #endif /* _UAPI_LINUX_SCHED_H */
diff --git a/include/uapi/linux/sed-opal.h b/include/uapi/linux/sed-opal.h
index 9025dd5a4f0ff..9830298ec51cc 100644
--- a/include/uapi/linux/sed-opal.h
+++ b/include/uapi/linux/sed-opal.h
@@ -74,6 +74,19 @@ struct opal_lr_act {
 	__u8 align[2]; /* Align to 8 byte boundary */
 };
 
+struct opal_lr_react {
+	struct opal_key key;
+	struct opal_key new_admin_key; /* Set new Admin1 PIN if key_len is > 0 */
+	__u8 num_lrs; /*
+		       * Configure selected ranges (from lr[]) in SUM.
+		       * If num_lrs > 0 the 'entire_table' must be 0
+		       */
+	__u8 lr[OPAL_MAX_LRS];
+	__u8 range_policy; /* Set RangeStartRangeLengthPolicy parameter */
+	__u8 entire_table; /* Set all locking objects in SUM */
+	__u8 align[4]; /* Align to 8 byte boundary */
+};
+
 struct opal_session_info {
 	__u32 sum;
 	__u32 who;
@@ -98,6 +111,18 @@ struct opal_lr_status {
 	__u8  align[4];
 };
 
+struct opal_sum_ranges {
+	/*
+	 * Initiate Admin1 session if key_len > 0,
+	 * use Anybody session otherwise.
+	 */
+	struct opal_key key;
+	__u8 num_lrs;
+	__u8 lr[OPAL_MAX_LRS];
+	__u8 range_policy;
+	__u8 align[5]; /* Align to 8 byte boundary */
+};
+
 struct opal_lock_unlock {
 	struct opal_session_info session;
 	__u32 l_state;
@@ -216,5 +241,9 @@ struct opal_revert_lsp {
 #define IOC_OPAL_DISCOVERY          _IOW('p', 239, struct opal_discovery)
 #define IOC_OPAL_REVERT_LSP         _IOW('p', 240, struct opal_revert_lsp)
 #define IOC_OPAL_SET_SID_PW         _IOW('p', 241, struct opal_new_pw)
+#define IOC_OPAL_REACTIVATE_LSP     _IOW('p', 242, struct opal_lr_react)
+#define IOC_OPAL_LR_SET_START_LEN   _IOW('p', 243, struct opal_user_lr_setup)
+#define IOC_OPAL_ENABLE_DISABLE_LR  _IOW('p', 244, struct opal_user_lr_setup)
+#define IOC_OPAL_GET_SUM_STATUS     _IOW('p', 245, struct opal_sum_ranges)
 
 #endif /* _UAPI_SED_OPAL_H */
diff --git a/include/uapi/linux/trace_mmap.h b/include/uapi/linux/trace_mmap.h
index c102ef35d11e4..e8185889a1c82 100644
--- a/include/uapi/linux/trace_mmap.h
+++ b/include/uapi/linux/trace_mmap.h
@@ -17,8 +17,8 @@
  * @entries:		Number of entries in the ring-buffer.
  * @overrun:		Number of entries lost in the ring-buffer.
  * @read:		Number of entries that have been read.
- * @Reserved1:		Internal use only.
- * @Reserved2:		Internal use only.
+ * @pages_lost:		Number of pages overwritten by the writer.
+ * @pages_touched:	Number of pages written by the writer.
  */
 struct trace_buffer_meta {
 	__u32		meta_page_size;
@@ -39,8 +39,8 @@ struct trace_buffer_meta {
 	__u64	overrun;
 	__u64	read;
 
-	__u64	Reserved1;
-	__u64	Reserved2;
+	__u64	pages_lost;
+	__u64	pages_touched;
 };
 
 #define TRACE_MMAP_IOCTL_GET_READER		_IO('R', 0x20)
diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h
index edca3e430305a..877fb02df8fba 100644
--- a/include/uapi/linux/udp.h
+++ b/include/uapi/linux/udp.h
@@ -29,6 +29,8 @@ struct udphdr {
 
 /* UDP socket options */
 #define UDP_CORK	1	/* Never send partially complete segments */
+/* Deprecated, reserved for UDPLITE_SEND_CSCOV 10 */
+/* Deprecated, reserved for UDPLITE_RECV_CSCOV 11 */
 #define UDP_ENCAP	100	/* Set the socket to accept encapsulated packets */
 #define UDP_NO_CHECK6_TX 101	/* Disable sending checksum for UDP6X */
 #define UDP_NO_CHECK6_RX 102	/* Disable accepting checksum for UDP6 */
diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
index f24edf1c75eb3..40955eaba32e6 100644
--- a/include/uapi/rdma/bnxt_re-abi.h
+++ b/include/uapi/rdma/bnxt_re-abi.h
@@ -102,12 +102,17 @@ struct bnxt_re_pd_resp {
 struct bnxt_re_cq_req {
 	__aligned_u64 cq_va;
 	__aligned_u64 cq_handle;
+	__aligned_u64 comp_mask;
 };
 
-enum bnxt_re_cq_mask {
+enum bnxt_re_resp_cq_mask {
 	BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT = 0x1,
 };
 
+enum bnxt_re_req_cq_mask {
+	BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE = 0x1,
+};
+
 struct bnxt_re_cq_resp {
 	__u32 cqid;
 	__u32 tail;
@@ -163,6 +168,8 @@ enum bnxt_re_objects {
 	BNXT_RE_OBJECT_ALLOC_PAGE = (1U << UVERBS_ID_NS_SHIFT),
 	BNXT_RE_OBJECT_NOTIFY_DRV,
 	BNXT_RE_OBJECT_GET_TOGGLE_MEM,
+	BNXT_RE_OBJECT_DBR,
+	BNXT_RE_OBJECT_DEFAULT_DBR,
 };
 
 enum bnxt_re_alloc_page_type {
@@ -231,4 +238,31 @@ struct bnxt_re_packet_pacing_caps {
 struct bnxt_re_query_device_ex_resp {
 	struct bnxt_re_packet_pacing_caps packet_pacing_caps;
 };
+
+struct bnxt_re_db_region {
+	__u32 dpi;
+	__u32 reserved;
+	__aligned_u64 umdbr;
+};
+
+enum bnxt_re_obj_dbr_alloc_attrs {
+	BNXT_RE_ALLOC_DBR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_ALLOC_DBR_ATTR,
+	BNXT_RE_ALLOC_DBR_OFFSET,
+};
+
+enum bnxt_re_obj_dbr_free_attrs {
+	BNXT_RE_FREE_DBR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_obj_default_dbr_attrs {
+	BNXT_RE_DEFAULT_DBR_ATTR = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum bnxt_re_obj_dpi_methods {
+	BNXT_RE_METHOD_DBR_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
+	BNXT_RE_METHOD_DBR_FREE,
+	BNXT_RE_METHOD_GET_DEFAULT_DBR,
+};
+
 #endif /* __BNXT_RE_UVERBS_ABI_H__*/
diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h
index 98b71b9979f8d..13225b038124b 100644
--- a/include/uapi/rdma/efa-abi.h
+++ b/include/uapi/rdma/efa-abi.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef EFA_ABI_USER_H
@@ -44,7 +44,8 @@ struct efa_ibv_alloc_ucontext_resp {
 	__u32 max_llq_size; /* bytes */
 	__u16 max_tx_batch; /* units of 64 bytes */
 	__u16 min_sq_wr;
-	__u8 reserved_a0[4];
+	__u16 inline_buf_size_ex;
+	__u8 reserved_b0[2];
 };
 
 struct efa_ibv_alloc_pd_resp {
diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h
index 89e6a3f13191b..90c5cd8e7753c 100644
--- a/include/uapi/rdma/ib_user_ioctl_verbs.h
+++ b/include/uapi/rdma/ib_user_ioctl_verbs.h
@@ -46,6 +46,7 @@
 
 enum ib_uverbs_core_support {
 	IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS = 1 << 0,
+	IB_UVERBS_CORE_SUPPORT_ROBUST_UDATA = 1 << 1,
 };
 
 enum ib_uverbs_access_flags {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 18f9fe0702132..01a2a050e4689 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -139,6 +139,7 @@ enum mlx5_ib_var_alloc_attrs {
 	MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
 	MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
 	MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
+	MLX5_IB_ATTR_VAR_OBJ_ALLOC_FLAGS,
 };
 
 enum mlx5_ib_var_obj_destroy_attrs {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_verbs.h b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
index 8f86e79d78a5f..ef295b38a1cfb 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_verbs.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
@@ -100,6 +100,10 @@ enum mlx5_ib_uapi_query_port_flags {
 	MLX5_IB_UAPI_QUERY_PORT_ESW_OWNER_VHCA_ID	= 1 << 5,
 };
 
+enum mlx5_ib_uapi_var_alloc_flags {
+	MLX5_IB_UAPI_VAR_ALLOC_FLAG_TLP = 1 << 0,
+};
+
 struct mlx5_ib_uapi_reg {
 	__u32 value;
 	__u32 mask;
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index f41f0228fcd0e..aac9782ddc09a 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -308,6 +308,10 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_MONITOR,
 
+	RDMA_NLDEV_CMD_FRMR_POOLS_GET, /* can dump */
+
+	RDMA_NLDEV_CMD_FRMR_POOLS_SET,
+
 	RDMA_NLDEV_NUM_OPS
 };
 
@@ -582,6 +586,24 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_SYS_ATTR_MONITOR_MODE,	/* u8 */
 
 	RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED,	/* u8 */
+
+	/*
+	 * FRMR Pools attributes
+	 */
+	RDMA_NLDEV_ATTR_FRMR_POOLS,		/* nested table */
+	RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY,	/* nested table */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY,		/* nested table */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS,	/* u8 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS,	/* u32 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY,	/* u64 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS,	/* u64 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES,	/* u32 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE,	/* u64 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE,	/* u64 */
+	RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD,	/* u32 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES,	/* u32 */
+	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY,	/* u64 */
+
 	/*
 	 * Always the end
 	 */
diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 8563b66489764..cb6f1537a3f36 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -690,6 +690,12 @@ enum ufshcd_quirks {
 	 * because it causes link startup to become unreliable.
 	 */
 	UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE		= 1 << 26,
+
+	/*
+	 * On some platforms, the VCC regulator has a slow ramp-up time. Add a
+	 * delay after enabling VCC to ensure it's stable.
+	 */
+	UFSHCD_QUIRK_VCC_ON_DELAY			= 1 << 27,
 };
 
 enum ufshcd_caps {
@@ -943,6 +949,11 @@ enum ufshcd_mcq_opr {
  * @critical_health_count: count of critical health exceptions
  * @dev_lvl_exception_count: count of device level exceptions since last reset
  * @dev_lvl_exception_id: vendor specific information about the device level exception event.
+ * @dme_qos_notification: Bitfield of pending DME Quality of Service (QoS)
+ *	events. Bits[3:1] reflect the corresponding bits of UIC DME Error Code
+ *	field within the Host Controller's UECDME register. Bit[0] is a flag
+ *	indicating that the DME QoS Monitor has been reset by the host.
+ * @dme_qos_sysfs_handle: handle for 'dme_qos_notification' sysfs entry
  * @rpmbs: list of OP-TEE RPMB devices (one per RPMB region)
  */
 struct ufs_hba {
@@ -1116,6 +1127,10 @@ struct ufs_hba {
 	int critical_health_count;
 	atomic_t dev_lvl_exception_count;
 	u64 dev_lvl_exception_id;
+
+	atomic_t dme_qos_notification;
+	struct kernfs_node *dme_qos_sysfs_handle;
+
 	u32 vcc_off_delay_us;
 	struct list_head rpmbs;
 };
diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h
index 806fdaf52bd9d..9f0fdd850e54e 100644
--- a/include/ufs/ufshci.h
+++ b/include/ufs/ufshci.h
@@ -115,6 +115,7 @@ enum {
 enum {
 	REG_CQIS		= 0x0,
 	REG_CQIE		= 0x4,
+	REG_MCQIACR		= 0x8,
 };
 
 enum {
@@ -188,6 +189,7 @@ static inline u32 ufshci_version(u32 major, u32 minor)
 #define SYSTEM_BUS_FATAL_ERROR			0x20000
 #define CRYPTO_ENGINE_FATAL_ERROR		0x40000
 #define MCQ_CQ_EVENT_STATUS			0x100000
+#define MCQ_IAG_EVENT_STATUS			0x200000
 
 #define UFSHCD_UIC_HIBERN8_MASK	(UIC_HIBERNATE_ENTER |\
 				UIC_HIBERNATE_EXIT)
@@ -271,6 +273,7 @@ enum {
 /* UECDME - Host UIC Error Code DME 48h */
 #define UIC_DME_ERROR			0x80000000
 #define UIC_DME_ERROR_CODE_MASK		0x1
+#define UIC_DME_QOS_MASK		0xE
 
 /* UTRIACR - Interrupt Aggregation control register - 0x4Ch */
 #define INT_AGGR_TIMEOUT_VAL_MASK		0xFF
diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h
index 23c39b96190fd..5977723fb3b5d 100644
--- a/include/vdso/datapage.h
+++ b/include/vdso/datapage.h
@@ -4,24 +4,16 @@
 
 #ifndef __ASSEMBLY__
 
-#include <linux/compiler.h>
+#include <linux/types.h>
+
 #include <uapi/linux/bits.h>
 #include <uapi/linux/time.h>
-#include <uapi/linux/types.h>
-#include <uapi/asm-generic/errno-base.h>
 
 #include <vdso/align.h>
 #include <vdso/bits.h>
 #include <vdso/cache.h>
-#include <vdso/clocksource.h>
-#include <vdso/ktime.h>
-#include <vdso/limits.h>
-#include <vdso/math64.h>
 #include <vdso/page.h>
-#include <vdso/processor.h>
 #include <vdso/time.h>
-#include <vdso/time32.h>
-#include <vdso/time64.h>
 
 #ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA
 #include <asm/vdso/time_data.h>
@@ -80,8 +72,8 @@ struct vdso_timestamp {
  * @mask:		clocksource mask
  * @mult:		clocksource multiplier
  * @shift:		clocksource shift
- * @basetime[clock_id]:	basetime per clock_id
- * @offset[clock_id]:	time namespace offset per clock_id
+ * @basetime:		basetime per clock_id
+ * @offset:		time namespace offset per clock_id
  *
  * See also struct vdso_time_data for basic access and ordering information as
  * struct vdso_clock is used there.
@@ -184,17 +176,6 @@ enum vdso_pages {
 	VDSO_NR_PAGES
 };
 
-/*
- * The generic vDSO implementation requires that gettimeofday.h
- * provides:
- * - __arch_get_hw_counter(): to get the hw counter based on the
- *   clock_mode.
- * - gettimeofday_fallback(): fallback for gettimeofday.
- * - clock_gettime_fallback(): fallback for clock_gettime.
- * - clock_getres_fallback(): fallback for clock_getres.
- */
-#include <asm/vdso/gettimeofday.h>
-
 #else /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_VDSO_GETRANDOM
diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h
index 1a5ee9d9052c3..a3bf4f1c0d37d 100644
--- a/include/vdso/helpers.h
+++ b/include/vdso/helpers.h
@@ -6,6 +6,13 @@
 
 #include <asm/barrier.h>
 #include <vdso/datapage.h>
+#include <vdso/processor.h>
+#include <vdso/clocksource.h>
+
+static __always_inline bool vdso_is_timens_clock(const struct vdso_clock *vc)
+{
+	return IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS;
+}
 
 static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
 {
@@ -18,6 +25,28 @@ static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
 	return seq;
 }
 
+/*
+ * Variant of vdso_read_begin() to handle VDSO_CLOCKMODE_TIMENS.
+ *
+ * Time namespace enabled tasks have a special VVAR page installed which has
+ * vc->seq set to 1 and vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non
+ * time namespace affected tasks this does not affect performance because if
+ * vc->seq is odd, i.e. a concurrent update is in progress the extra check for
+ * vc->clock_mode is just a few extra instructions while spin waiting for
+ * vc->seq to become even again.
+ */
+static __always_inline bool vdso_read_begin_timens(const struct vdso_clock *vc, u32 *seq)
+{
+	while (unlikely((*seq = READ_ONCE(vc->seq)) & 1)) {
+		if (vdso_is_timens_clock(vc))
+			return true;
+		cpu_relax();
+	}
+	smp_rmb();
+
+	return false;
+}
+
 static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
 					   u32 start)
 {
@@ -25,7 +54,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
 
 	smp_rmb();
 	seq = READ_ONCE(vc->seq);
-	return seq != start;
+	return unlikely(seq != start);
 }
 
 static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc)
diff --git a/include/video/vga.h b/include/video/vga.h
index 468764d6727ae..2f13c371800b3 100644
--- a/include/video/vga.h
+++ b/include/video/vga.h
@@ -46,6 +46,7 @@
 #define VGA_MIS_R   	0x3CC	/* Misc Output Read Register */
 #define VGA_MIS_W   	0x3C2	/* Misc Output Write Register */
 #define VGA_FTC_R	0x3CA	/* Feature Control Read Register */
+#define VGA_IS0_R	0x3C2	/* Input Status Register 0 */
 #define VGA_IS1_RC  	0x3DA	/* Input Status Register 1 - color emulation */
 #define VGA_IS1_RM  	0x3BA	/* Input Status Register 1 - mono emulation */
 #define VGA_PEL_D   	0x3C9	/* PEL Data Register */
diff --git a/init/Kconfig b/init/Kconfig
index 444ce811ea674..938fbe6a91e15 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -960,6 +960,11 @@ config CC_IMPLICIT_FALLTHROUGH
 	default "-Wimplicit-fallthrough=5" if CC_IS_GCC && $(cc-option,-Wimplicit-fallthrough=5)
 	default "-Wimplicit-fallthrough" if CC_IS_CLANG && $(cc-option,-Wunreachable-code-fallthrough)
 
+config CC_MS_EXTENSIONS
+	string
+	default "-fms-anonymous-structs" if $(cc-option,-fms-anonymous-structs)
+	default "-fms-extensions"
+
 # Currently, disable gcc-10+ array-bounds globally.
 # It's still broken in gcc-13, so no upper bound yet.
 config GCC10_NO_ARRAY_BOUNDS
@@ -1176,6 +1181,10 @@ config EXT_GROUP_SCHED
 
 endif #CGROUP_SCHED
 
+config EXT_SUB_SCHED
+        def_bool y
+        depends on SCHED_CLASS_EXT
+
 config SCHED_MM_CID
 	def_bool y
 	depends on SMP && RSEQ
@@ -2050,6 +2059,41 @@ config KALLSYMS_ALL
 
 	  Say N unless you really need all symbols, or kernel live patching.
 
+config KALLSYMS_LINEINFO
+	bool "Embed source file:line information in stack traces"
+	depends on KALLSYMS && DEBUG_INFO
+	help
+	  Embeds an address-to-source-line mapping table in the kernel
+	  image so that stack traces directly include file:line information,
+	  similar to what scripts/decode_stacktrace.sh provides but without
+	  needing external tools or a vmlinux with debug info at runtime.
+
+	  When enabled, stack traces will look like:
+
+	    kmem_cache_alloc_noprof+0x60/0x630 (mm/slub.c:3456)
+	    anon_vma_clone+0x2ed/0xcf0 (mm/rmap.c:412)
+
+	  This requires elfutils (libdw-dev/elfutils-devel) on the build host.
+	  Adds approximately 10-15MB to a typical kernel image (~2-3 bytes
+	  per entry after delta compression, ~4.6M entries for a typical
+	  config).
+
+	  If unsure, say N.
+
+config KALLSYMS_LINEINFO_MODULES
+	bool "Embed source file:line information in module stack traces"
+	depends on KALLSYMS_LINEINFO && MODULES
+	help
+	  Extends KALLSYMS_LINEINFO to loadable kernel modules.  Each .ko
+	  gets a lineinfo table generated from its DWARF data at build time,
+	  so stack traces from module code include (file.c:123) annotations.
+
+	  Requires elfutils (libdw-dev/elfutils-devel) on the build host.
+	  Increases .ko sizes by approximately 2-3 bytes per DWARF line
+	  entry after delta compression.
+
+	  If unsure, say N.
+
 # end of the "standard kernel features (expert users)" menu
 
 config ARCH_HAS_MEMBARRIER_CALLBACKS
diff --git a/init/main.c b/init/main.c
index 1cb395dd94e43..453ac9dff2da0 100644
--- a/init/main.c
+++ b/init/main.c
@@ -105,6 +105,8 @@
 #include <linux/ptdump.h>
 #include <linux/time_namespace.h>
 #include <linux/unaligned.h>
+#include <linux/percpu_counter_tree.h>
+#include <linux/vdso_datastore.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -1067,6 +1069,7 @@ void start_kernel(void)
 	vfs_caches_init_early();
 	sort_main_extable();
 	trap_init();
+	percpu_counter_tree_subsystem_init();
 	mm_core_init();
 	maple_tree_init();
 	poking_init();
@@ -1119,6 +1122,7 @@ void start_kernel(void)
 	srcu_init();
 	hrtimers_init();
 	softirq_init();
+	vdso_setup_data_pages();
 	timekeeping_init();
 	time_init();
 
diff --git a/io_uring/Kconfig b/io_uring/Kconfig
index a7ae23cf10357..a283d9e53787f 100644
--- a/io_uring/Kconfig
+++ b/io_uring/Kconfig
@@ -14,3 +14,8 @@ config IO_URING_BPF
 	def_bool y
 	depends on BPF
 	depends on NET
+
+config IO_URING_BPF_OPS
+	def_bool y
+	depends on IO_URING
+	depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF
diff --git a/io_uring/Makefile b/io_uring/Makefile
index 931f9156132a0..c54e328d14104 100644
--- a/io_uring/Makefile
+++ b/io_uring/Makefile
@@ -14,7 +14,7 @@ obj-$(CONFIG_IO_URING)		+= io_uring.o opdef.o kbuf.o rsrc.o notif.o \
 					advise.o openclose.o statx.o timeout.o \
 					cancel.o waitid.o register.o \
 					truncate.o memmap.o alloc_cache.o \
-					query.o
+					query.o loop.o
 
 obj-$(CONFIG_IO_URING_ZCRX)	+= zcrx.o
 obj-$(CONFIG_IO_WQ)		+= io-wq.o
@@ -25,3 +25,4 @@ obj-$(CONFIG_NET) += net.o cmd_net.o
 obj-$(CONFIG_PROC_FS) += fdinfo.o
 obj-$(CONFIG_IO_URING_MOCK_FILE) += mock_file.o
 obj-$(CONFIG_IO_URING_BPF) += bpf_filter.o
+obj-$(CONFIG_IO_URING_BPF_OPS) += bpf-ops.o
diff --git a/io_uring/bpf-ops.c b/io_uring/bpf-ops.c
new file mode 100644
index 0000000000000..e4b244337aa98
--- /dev/null
+++ b/io_uring/bpf-ops.c
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/mutex.h>
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+
+#include "io_uring.h"
+#include "register.h"
+#include "loop.h"
+#include "memmap.h"
+#include "bpf-ops.h"
+
+static DEFINE_MUTEX(io_bpf_ctrl_mutex);
+static const struct btf_type *loop_params_type;
+
+__bpf_kfunc_start_defs();
+
+__bpf_kfunc int bpf_io_uring_submit_sqes(struct io_ring_ctx *ctx, u32 nr)
+{
+	return io_submit_sqes(ctx, nr);
+}
+
+__bpf_kfunc
+__u8 *bpf_io_uring_get_region(struct io_ring_ctx *ctx, __u32 region_id,
+			      const size_t rdwr_buf_size)
+{
+	struct io_mapped_region *r;
+
+	lockdep_assert_held(&ctx->uring_lock);
+
+	switch (region_id) {
+	case IOU_REGION_MEM:
+		r = &ctx->param_region;
+		break;
+	case IOU_REGION_CQ:
+		r = &ctx->ring_region;
+		break;
+	case IOU_REGION_SQ:
+		r = &ctx->sq_region;
+		break;
+	default:
+		return NULL;
+	}
+
+	if (unlikely(rdwr_buf_size > io_region_size(r)))
+		return NULL;
+	return io_region_get_ptr(r);
+}
+
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(io_uring_kfunc_set)
+BTF_ID_FLAGS(func, bpf_io_uring_submit_sqes, KF_SLEEPABLE);
+BTF_ID_FLAGS(func, bpf_io_uring_get_region, KF_RET_NULL);
+BTF_KFUNCS_END(io_uring_kfunc_set)
+
+static const struct btf_kfunc_id_set bpf_io_uring_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set = &io_uring_kfunc_set,
+};
+
+static int io_bpf_ops__loop_step(struct io_ring_ctx *ctx,
+				 struct iou_loop_params *lp)
+{
+	return IOU_LOOP_STOP;
+}
+
+static struct io_uring_bpf_ops io_bpf_ops_stubs = {
+	.loop_step = io_bpf_ops__loop_step,
+};
+
+static bool bpf_io_is_valid_access(int off, int size,
+				    enum bpf_access_type type,
+				    const struct bpf_prog *prog,
+				    struct bpf_insn_access_aux *info)
+{
+	if (type != BPF_READ)
+		return false;
+	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
+		return false;
+	if (off % size != 0)
+		return false;
+
+	return btf_ctx_access(off, size, type, prog, info);
+}
+
+static int bpf_io_btf_struct_access(struct bpf_verifier_log *log,
+				    const struct bpf_reg_state *reg, int off,
+				    int size)
+{
+	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
+
+	if (t == loop_params_type) {
+		if (off + size <= offsetofend(struct iou_loop_params, cq_wait_idx))
+			return SCALAR_VALUE;
+	}
+
+	return -EACCES;
+}
+
+static const struct bpf_verifier_ops bpf_io_verifier_ops = {
+	.get_func_proto = bpf_base_func_proto,
+	.is_valid_access = bpf_io_is_valid_access,
+	.btf_struct_access = bpf_io_btf_struct_access,
+};
+
+static const struct btf_type *
+io_lookup_struct_type(struct btf *btf, const char *name)
+{
+	s32 type_id;
+
+	type_id = btf_find_by_name_kind(btf, name, BTF_KIND_STRUCT);
+	if (type_id < 0)
+		return NULL;
+	return btf_type_by_id(btf, type_id);
+}
+
+static int bpf_io_init(struct btf *btf)
+{
+	int ret;
+
+	loop_params_type = io_lookup_struct_type(btf, "iou_loop_params");
+	if (!loop_params_type) {
+		pr_err("io_uring: Failed to locate iou_loop_params\n");
+		return -EINVAL;
+	}
+
+	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
+					&bpf_io_uring_kfunc_set);
+	if (ret) {
+		pr_err("io_uring: Failed to register kfuncs (%d)\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int bpf_io_check_member(const struct btf_type *t,
+				const struct btf_member *member,
+				const struct bpf_prog *prog)
+{
+	return 0;
+}
+
+static int bpf_io_init_member(const struct btf_type *t,
+			       const struct btf_member *member,
+			       void *kdata, const void *udata)
+{
+	u32 moff = __btf_member_bit_offset(t, member) / 8;
+	const struct io_uring_bpf_ops *uops = udata;
+	struct io_uring_bpf_ops *ops = kdata;
+
+	switch (moff) {
+	case offsetof(struct io_uring_bpf_ops, ring_fd):
+		ops->ring_fd = uops->ring_fd;
+		return 1;
+	}
+	return 0;
+}
+
+static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)
+{
+	if (ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL))
+		return -EOPNOTSUPP;
+	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
+		return -EOPNOTSUPP;
+
+	if (ctx->bpf_ops)
+		return -EBUSY;
+	if (WARN_ON_ONCE(!ops->loop_step))
+		return -EINVAL;
+
+	ops->priv = ctx;
+	ctx->bpf_ops = ops;
+	ctx->loop_step = ops->loop_step;
+	return 0;
+}
+
+static int bpf_io_reg(void *kdata, struct bpf_link *link)
+{
+	struct io_uring_bpf_ops *ops = kdata;
+	struct io_ring_ctx *ctx;
+	struct file *file;
+	int ret = -EBUSY;
+
+	file = io_uring_register_get_file(ops->ring_fd, false);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+	ctx = file->private_data;
+
+	scoped_guard(mutex, &io_bpf_ctrl_mutex) {
+		guard(mutex)(&ctx->uring_lock);
+		ret = io_install_bpf(ctx, ops);
+	}
+
+	fput(file);
+	return ret;
+}
+
+static void io_eject_bpf(struct io_ring_ctx *ctx)
+{
+	struct io_uring_bpf_ops *ops = ctx->bpf_ops;
+
+	if (WARN_ON_ONCE(!ops))
+		return;
+	if (WARN_ON_ONCE(ops->priv != ctx))
+		return;
+
+	ops->priv = NULL;
+	ctx->bpf_ops = NULL;
+	ctx->loop_step = NULL;
+}
+
+static void bpf_io_unreg(void *kdata, struct bpf_link *link)
+{
+	struct io_uring_bpf_ops *ops = kdata;
+	struct io_ring_ctx *ctx;
+
+	guard(mutex)(&io_bpf_ctrl_mutex);
+	ctx = ops->priv;
+	if (ctx) {
+		guard(mutex)(&ctx->uring_lock);
+		if (WARN_ON_ONCE(ctx->bpf_ops != ops))
+			return;
+
+		io_eject_bpf(ctx);
+	}
+}
+
+void io_unregister_bpf_ops(struct io_ring_ctx *ctx)
+{
+	/*
+	 * ->bpf_ops is write protected by io_bpf_ctrl_mutex and uring_lock,
+	 * and read protected by either. Try to avoid taking the global lock
+	 * for rings that never had any bpf installed.
+	 */
+	scoped_guard(mutex, &ctx->uring_lock) {
+		if (!ctx->bpf_ops)
+			return;
+	}
+
+	guard(mutex)(&io_bpf_ctrl_mutex);
+	guard(mutex)(&ctx->uring_lock);
+	if (ctx->bpf_ops)
+		io_eject_bpf(ctx);
+}
+
+static struct bpf_struct_ops bpf_ring_ops = {
+	.verifier_ops = &bpf_io_verifier_ops,
+	.reg = bpf_io_reg,
+	.unreg = bpf_io_unreg,
+	.check_member = bpf_io_check_member,
+	.init_member = bpf_io_init_member,
+	.init = bpf_io_init,
+	.cfi_stubs = &io_bpf_ops_stubs,
+	.name = "io_uring_bpf_ops",
+	.owner = THIS_MODULE,
+};
+
+static int __init io_uring_bpf_init(void)
+{
+	int ret;
+
+	ret = register_bpf_struct_ops(&bpf_ring_ops, io_uring_bpf_ops);
+	if (ret) {
+		pr_err("io_uring: Failed to register struct_ops (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+__initcall(io_uring_bpf_init);
diff --git a/io_uring/bpf-ops.h b/io_uring/bpf-ops.h
new file mode 100644
index 0000000000000..b39b3fd3acdab
--- /dev/null
+++ b/io_uring/bpf-ops.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef IOU_BPF_OPS_H
+#define IOU_BPF_OPS_H
+
+#include <linux/io_uring_types.h>
+
+enum {
+	IOU_REGION_MEM,
+	IOU_REGION_CQ,
+	IOU_REGION_SQ,
+};
+
+struct io_uring_bpf_ops {
+	int (*loop_step)(struct io_ring_ctx *ctx, struct iou_loop_params *lp);
+
+	__u32 ring_fd;
+	void *priv;
+};
+
+#ifdef CONFIG_IO_URING_BPF_OPS
+void io_unregister_bpf_ops(struct io_ring_ctx *ctx);
+#else
+static inline void io_unregister_bpf_ops(struct io_ring_ctx *ctx)
+{
+}
+#endif
+
+#endif /* IOU_BPF_OPS_H */
diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c
index 125a81c520a6c..7cd411fc4f33e 100644
--- a/io_uring/cmd_net.c
+++ b/io_uring/cmd_net.c
@@ -7,6 +7,21 @@
 #include "uring_cmd.h"
 #include "io_uring.h"
 
+static int io_uring_cmd_get_sock_ioctl(struct socket *sock, int op)
+{
+	struct sock *sk = sock->sk;
+	struct proto *prot = READ_ONCE(sk->sk_prot);
+	int ret, arg = 0;
+
+	if (!prot || !prot->ioctl)
+		return -EOPNOTSUPP;
+
+	ret = prot->ioctl(sk, op, &arg);
+	if (ret)
+		return ret;
+	return arg;
+}
+
 static inline int io_uring_cmd_getsockopt(struct socket *sock,
 					  struct io_uring_cmd *cmd,
 					  unsigned int issue_flags)
@@ -156,27 +171,12 @@ static int io_uring_cmd_getsockname(struct socket *sock,
 int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags)
 {
 	struct socket *sock = cmd->file->private_data;
-	struct sock *sk = sock->sk;
-	struct proto *prot = READ_ONCE(sk->sk_prot);
-	int ret, arg = 0;
 
 	switch (cmd->cmd_op) {
 	case SOCKET_URING_OP_SIOCINQ:
-		if (!prot || !prot->ioctl)
-			return -EOPNOTSUPP;
-
-		ret = prot->ioctl(sk, SIOCINQ, &arg);
-		if (ret)
-			return ret;
-		return arg;
+		return io_uring_cmd_get_sock_ioctl(sock, SIOCINQ);
 	case SOCKET_URING_OP_SIOCOUTQ:
-		if (!prot || !prot->ioctl)
-			return -EOPNOTSUPP;
-
-		ret = prot->ioctl(sk, SIOCOUTQ, &arg);
-		if (ret)
-			return ret;
-		return arg;
+		return io_uring_cmd_get_sock_ioctl(sock, SIOCOUTQ);
 	case SOCKET_URING_OP_GETSOCKOPT:
 		return io_uring_cmd_getsockopt(sock, cmd, issue_flags);
 	case SOCKET_URING_OP_SETSOCKOPT:
diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c
index 7482a7dc6b385..3da028500f76a 100644
--- a/io_uring/eventfd.c
+++ b/io_uring/eventfd.c
@@ -148,7 +148,7 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
 	spin_unlock(&ctx->completion_lock);
 
 	ev_fd->eventfd_async = eventfd_async;
-	ctx->has_evfd = true;
+	ctx->int_flags |= IO_RING_F_HAS_EVFD;
 	refcount_set(&ev_fd->refs, 1);
 	atomic_set(&ev_fd->ops, 0);
 	rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
@@ -162,7 +162,7 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx)
 	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
 					lockdep_is_held(&ctx->uring_lock));
 	if (ev_fd) {
-		ctx->has_evfd = false;
+		ctx->int_flags &= ~IO_RING_F_HAS_EVFD;
 		rcu_assign_pointer(ctx->io_ev_fd, NULL);
 		io_eventfd_put(ev_fd);
 		return 0;
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 9a37035e76c02..6eaa21e094698 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -87,6 +87,7 @@
 #include "msg_ring.h"
 #include "memmap.h"
 #include "zcrx.h"
+#include "bpf-ops.h"
 
 #include "timeout.h"
 #include "poll.h"
@@ -95,6 +96,7 @@
 #include "eventfd.h"
 #include "wait.h"
 #include "bpf_filter.h"
+#include "loop.h"
 
 #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
 			  IOSQE_IO_HARDLINK | IOSQE_ASYNC)
@@ -356,7 +358,6 @@ static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
 static void io_prep_async_work(struct io_kiocb *req)
 {
 	const struct io_issue_def *def = &io_issue_defs[req->opcode];
-	struct io_ring_ctx *ctx = req->ctx;
 
 	if (!(req->flags & REQ_F_CREDS)) {
 		req->flags |= REQ_F_CREDS;
@@ -378,7 +379,7 @@ static void io_prep_async_work(struct io_kiocb *req)
 		if (should_hash && (req->file->f_flags & O_DIRECT) &&
 		    (req->file->f_op->fop_flags & FOP_DIO_PARALLEL_WRITE))
 			should_hash = false;
-		if (should_hash || (ctx->flags & IORING_SETUP_IOPOLL))
+		if (should_hash || (req->flags & REQ_F_IOPOLL))
 			io_wq_hash_work(&req->work, file_inode(req->file));
 	} else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
 		if (def->unbound_nonreg_file)
@@ -477,17 +478,17 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
 
 void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
 {
-	if (ctx->poll_activated)
+	if (ctx->int_flags & IO_RING_F_POLL_ACTIVATED)
 		io_poll_wq_wake(ctx);
-	if (ctx->off_timeout_used)
+	if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
 		io_flush_timeouts(ctx);
-	if (ctx->has_evfd)
+	if (ctx->int_flags & IO_RING_F_HAS_EVFD)
 		io_eventfd_signal(ctx, true);
 }
 
 static inline void __io_cq_lock(struct io_ring_ctx *ctx)
 {
-	if (!ctx->lockless_cq)
+	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
 		spin_lock(&ctx->completion_lock);
 }
 
@@ -500,11 +501,11 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
 {
 	io_commit_cqring(ctx);
-	if (!ctx->task_complete) {
-		if (!ctx->lockless_cq)
+	if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) {
+		if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
 			spin_unlock(&ctx->completion_lock);
 		/* IOPOLL rings only need to wake up if it's also SQPOLL */
-		if (!ctx->syscall_iopoll)
+		if (!(ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL))
 			io_cqring_wake(ctx);
 	}
 	io_commit_cqring_flush(ctx);
@@ -589,6 +590,11 @@ void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
 	mutex_unlock(&ctx->uring_lock);
 }
 
+void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx)
+{
+	__io_cqring_overflow_flush(ctx, false);
+}
+
 /* must to be called somewhat shortly after putting a request */
 static inline void io_put_task(struct io_kiocb *req)
 {
@@ -830,7 +836,7 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 {
 	lockdep_assert_held(&ctx->uring_lock);
-	lockdep_assert(ctx->lockless_cq);
+	lockdep_assert(ctx->int_flags & IO_RING_F_LOCKLESS_CQ);
 
 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
@@ -860,7 +866,7 @@ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)
 	lockdep_assert(!io_wq_current_is_worker());
 	lockdep_assert_held(&ctx->uring_lock);
 
-	if (!ctx->lockless_cq) {
+	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
 		spin_lock(&ctx->completion_lock);
 		posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
 		spin_unlock(&ctx->completion_lock);
@@ -885,7 +891,7 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])
 	lockdep_assert_held(&ctx->uring_lock);
 
 	cqe[0].user_data = req->cqe.user_data;
-	if (!ctx->lockless_cq) {
+	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
 		spin_lock(&ctx->completion_lock);
 		posted = io_fill_cqe_aux32(ctx, cqe);
 		spin_unlock(&ctx->completion_lock);
@@ -913,7 +919,7 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 	 * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
 	 * the submitter task context, IOPOLL protects with uring_lock.
 	 */
-	if (ctx->lockless_cq || (req->flags & REQ_F_REISSUE)) {
+	if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) {
 defer_complete:
 		req->io_task_work.func = io_req_task_complete;
 		io_req_task_work_add(req);
@@ -1067,12 +1073,14 @@ void io_queue_next(struct io_kiocb *req)
 
 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
 {
+	struct io_ring_ctx *ctx = req->ctx;
+
 	if (req->file_node) {
-		io_put_rsrc_node(req->ctx, req->file_node);
+		io_put_rsrc_node(ctx, req->file_node);
 		req->file_node = NULL;
 	}
 	if (req->flags & REQ_F_BUF_NODE)
-		io_put_rsrc_node(req->ctx, req->buf_node);
+		io_put_rsrc_node(ctx, req->buf_node);
 }
 
 static void io_free_batch_list(struct io_ring_ctx *ctx,
@@ -1135,7 +1143,7 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 		 */
 		if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
 		    unlikely(!io_fill_cqe_req(ctx, req))) {
-			if (ctx->lockless_cq)
+			if (ctx->int_flags & IO_RING_F_LOCKLESS_CQ)
 				io_cqe_overflow(ctx, &req->cqe, &req->big_cqe);
 			else
 				io_cqe_overflow_locked(ctx, &req->cqe, &req->big_cqe);
@@ -1148,7 +1156,7 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 		INIT_WQ_LIST(&state->compl_reqs);
 	}
 
-	if (unlikely(ctx->drain_active))
+	if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
 		io_queue_deferred(ctx);
 
 	ctx->submit_state.cq_flush = false;
@@ -1187,7 +1195,6 @@ __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
 
 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
 {
-	unsigned int nr_events = 0;
 	unsigned long check_cq;
 
 	min_events = min(min_events, ctx->cq_entries);
@@ -1230,8 +1237,6 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
 		 * very same mutex.
 		 */
 		if (list_empty(&ctx->iopoll_list) || io_task_work_pending(ctx)) {
-			u32 tail = ctx->cached_cq_tail;
-
 			(void) io_run_local_work_locked(ctx, min_events);
 
 			if (task_work_pending(current) || list_empty(&ctx->iopoll_list)) {
@@ -1240,7 +1245,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
 				mutex_lock(&ctx->uring_lock);
 			}
 			/* some requests don't go through iopoll_list */
-			if (tail != ctx->cached_cq_tail || list_empty(&ctx->iopoll_list))
+			if (list_empty(&ctx->iopoll_list))
 				break;
 		}
 		ret = io_do_iopoll(ctx, !min_events);
@@ -1251,9 +1256,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
 			return -EINTR;
 		if (need_resched())
 			break;
-
-		nr_events += ret;
-	} while (nr_events < min_events);
+	} while (io_cqring_events(ctx) < min_events);
 
 	return 0;
 }
@@ -1344,7 +1347,7 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	list_add_tail(&de->list, &ctx->defer_list);
 	io_queue_deferred(ctx);
 	if (!drain && list_empty(&ctx->defer_list))
-		ctx->drain_active = false;
+		ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
 }
 
 static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
@@ -1418,8 +1421,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
 	if (ret == IOU_ISSUE_SKIP_COMPLETE) {
 		ret = 0;
 
-		/* If the op doesn't have a file, we're not polling for it */
-		if ((req->ctx->flags & IORING_SETUP_IOPOLL) && def->iopoll_queue)
+		if (req->flags & REQ_F_IOPOLL)
 			io_iopoll_req_issued(req, issue_flags);
 	}
 	return ret;
@@ -1435,7 +1437,7 @@ int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
 	io_tw_lock(req->ctx, tw);
 
 	WARN_ON_ONCE(!req->file);
-	if (WARN_ON_ONCE(req->ctx->flags & IORING_SETUP_IOPOLL))
+	if (WARN_ON_ONCE(req->flags & REQ_F_IOPOLL))
 		return -EFAULT;
 
 	ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]);
@@ -1533,7 +1535,7 @@ void io_wq_submit_work(struct io_wq_work *work)
 		 * wait for request slots on the block side.
 		 */
 		if (!needs_poll) {
-			if (!(req->ctx->flags & IORING_SETUP_IOPOLL))
+			if (!(req->flags & REQ_F_IOPOLL))
 				break;
 			if (io_wq_worker_stopped())
 				break;
@@ -1655,7 +1657,7 @@ static void io_queue_sqe_fallback(struct io_kiocb *req)
 	} else {
 		/* can't fail with IO_URING_F_INLINE */
 		io_req_sqe_copy(req, IO_URING_F_INLINE);
-		if (unlikely(req->ctx->drain_active))
+		if (unlikely(req->ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
 			io_drain_req(req);
 		else
 			io_queue_iowq(req);
@@ -1671,7 +1673,7 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx,
 					struct io_kiocb *req,
 					unsigned int sqe_flags)
 {
-	if (!ctx->op_restricted)
+	if (!(ctx->int_flags & IO_RING_F_OP_RESTRICTED))
 		return true;
 	if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
 		return false;
@@ -1691,7 +1693,7 @@ static void io_init_drain(struct io_ring_ctx *ctx)
 {
 	struct io_kiocb *head = ctx->submit_state.link.head;
 
-	ctx->drain_active = true;
+	ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
 	if (head) {
 		/*
 		 * If we need to drain a request in the middle of a link, drain
@@ -1701,7 +1703,7 @@ static void io_init_drain(struct io_ring_ctx *ctx)
 		 * link.
 		 */
 		head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
-		ctx->drain_next = true;
+		ctx->int_flags |= IO_RING_F_DRAIN_NEXT;
 	}
 }
 
@@ -1767,23 +1769,23 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 			req->buf_index = READ_ONCE(sqe->buf_group);
 		}
 		if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
-			ctx->drain_disabled = true;
+			ctx->int_flags |= IO_RING_F_DRAIN_DISABLED;
 		if (sqe_flags & IOSQE_IO_DRAIN) {
-			if (ctx->drain_disabled)
+			if (ctx->int_flags & IO_RING_F_DRAIN_DISABLED)
 				return io_init_fail_req(req, -EOPNOTSUPP);
 			io_init_drain(ctx);
 		}
 	}
-	if (unlikely(ctx->op_restricted || ctx->drain_active || ctx->drain_next)) {
+	if (unlikely(ctx->int_flags & (IO_RING_F_OP_RESTRICTED | IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT))) {
 		if (!io_check_restriction(ctx, req, sqe_flags))
 			return io_init_fail_req(req, -EACCES);
 		/* knock it to the slow queue path, will be drained there */
-		if (ctx->drain_active)
+		if (ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)
 			req->flags |= REQ_F_FORCE_ASYNC;
 		/* if there is no link, we're at "next" request and need to drain */
-		if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
-			ctx->drain_next = false;
-			ctx->drain_active = true;
+		if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT) && !ctx->submit_state.link.head) {
+			ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT;
+			ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
 			req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
 		}
 	}
@@ -2148,6 +2150,7 @@ static __cold void io_req_caches_free(struct io_ring_ctx *ctx)
 
 static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
+	io_unregister_bpf_ops(ctx);
 	io_sq_thread_finish(ctx);
 
 	mutex_lock(&ctx->uring_lock);
@@ -2204,7 +2207,7 @@ static __cold void io_activate_pollwq_cb(struct callback_head *cb)
 					       poll_wq_task_work);
 
 	mutex_lock(&ctx->uring_lock);
-	ctx->poll_activated = true;
+	ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
 	mutex_unlock(&ctx->uring_lock);
 
 	/*
@@ -2219,9 +2222,9 @@ __cold void io_activate_pollwq(struct io_ring_ctx *ctx)
 {
 	spin_lock(&ctx->completion_lock);
 	/* already activated or in progress */
-	if (ctx->poll_activated || ctx->poll_wq_task_work.func)
+	if ((ctx->int_flags & IO_RING_F_POLL_ACTIVATED) || ctx->poll_wq_task_work.func)
 		goto out;
-	if (WARN_ON_ONCE(!ctx->task_complete))
+	if (WARN_ON_ONCE(!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)))
 		goto out;
 	if (!ctx->submitter_task)
 		goto out;
@@ -2242,7 +2245,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)
 	struct io_ring_ctx *ctx = file->private_data;
 	__poll_t mask = 0;
 
-	if (unlikely(!ctx->poll_activated))
+	if (unlikely(!(data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED)))
 		io_activate_pollwq(ctx);
 	/*
 	 * provides mb() which pairs with barrier from wq_has_sleeper
@@ -2578,6 +2581,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 	if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED))
 		goto out;
 
+	if (io_has_loop_ops(ctx)) {
+		ret = io_run_loop(ctx);
+		goto out;
+	}
+
 	/*
 	 * For SQ polling, the thread will do all submissions and completions.
 	 * Just return the requested submit count, and wake the thread if
@@ -2607,7 +2615,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 			goto out;
 		}
 		if (flags & IORING_ENTER_GETEVENTS) {
-			if (ctx->syscall_iopoll)
+			if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL)
 				goto iopoll_locked;
 			/*
 			 * Ignore errors, we'll soon call io_cqring_wait() and
@@ -2622,7 +2630,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 	if (flags & IORING_ENTER_GETEVENTS) {
 		int ret2;
 
-		if (ctx->syscall_iopoll) {
+		if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL) {
 			/*
 			 * We disallow the app entering submit/complete with
 			 * polling, but we still need to lock the ring to
@@ -2923,9 +2931,9 @@ static void io_ctx_restriction_clone(struct io_ring_ctx *ctx,
 	if (dst->bpf_filters)
 		WRITE_ONCE(ctx->bpf_filters, dst->bpf_filters->filters);
 	if (dst->op_registered)
-		ctx->op_restricted = 1;
+		ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
 	if (dst->reg_registered)
-		ctx->reg_restricted = 1;
+		ctx->int_flags |= IO_RING_F_REG_RESTRICTED;
 }
 
 static __cold int io_uring_create(struct io_ctx_config *config)
@@ -2952,17 +2960,18 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 
 	if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
 	    !(ctx->flags & IORING_SETUP_IOPOLL))
-		ctx->task_complete = true;
+		ctx->int_flags |= IO_RING_F_TASK_COMPLETE;
 
-	if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))
-		ctx->lockless_cq = true;
+	if ((ctx->int_flags & IO_RING_F_TASK_COMPLETE) ||
+	    (ctx->flags & IORING_SETUP_IOPOLL))
+		ctx->int_flags |= IO_RING_F_LOCKLESS_CQ;
 
 	/*
 	 * lazy poll_wq activation relies on ->task_complete for synchronisation
 	 * purposes, see io_activate_pollwq()
 	 */
-	if (!ctx->task_complete)
-		ctx->poll_activated = true;
+	if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE))
+		ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
 
 	/*
 	 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
@@ -2972,9 +2981,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 	 */
 	if (ctx->flags & IORING_SETUP_IOPOLL &&
 	    !(ctx->flags & IORING_SETUP_SQPOLL))
-		ctx->syscall_iopoll = 1;
+		ctx->int_flags |= IO_RING_F_SYSCALL_IOPOLL;
 
-	ctx->compat = in_compat_syscall();
+	if (in_compat_syscall())
+		ctx->int_flags |= IO_RING_F_COMPAT;
 	if (!ns_capable_noaudit(&init_user_ns, CAP_IPC_LOCK))
 		ctx->user = get_uid(current_user());
 
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 0fa844faf2871..91cf67b5d85bc 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -211,7 +211,7 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
 
 	if (ctx->flags & IORING_SETUP_IOPOLL) {
 		lockdep_assert_held(&ctx->uring_lock);
-	} else if (!ctx->task_complete) {
+	} else if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) {
 		lockdep_assert_held(&ctx->completion_lock);
 	} else if (ctx->submitter_task) {
 		/*
@@ -228,7 +228,7 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
 
 static inline bool io_is_compat(struct io_ring_ctx *ctx)
 {
-	return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->compat);
+	return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->int_flags & IO_RING_F_COMPAT);
 }
 
 static inline void io_submit_flush_completions(struct io_ring_ctx *ctx)
@@ -470,10 +470,12 @@ static inline void io_req_complete_defer(struct io_kiocb *req)
 	wq_list_add_tail(&req->comp_list, &state->compl_reqs);
 }
 
+#define SHOULD_FLUSH_MASK	(IO_RING_F_OFF_TIMEOUT_USED | \
+				 IO_RING_F_HAS_EVFD | IO_RING_F_POLL_ACTIVATED)
+
 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
 {
-	if (unlikely(ctx->off_timeout_used ||
-		     ctx->has_evfd || ctx->poll_activated))
+	if (unlikely(data_race(ctx->int_flags) & SHOULD_FLUSH_MASK))
 		__io_commit_cqring_flush(ctx);
 }
 
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 26813b0f1dfd1..8da2ff7981708 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -34,6 +34,10 @@ struct io_provide_buf {
 
 static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len)
 {
+	/* No data consumed, return false early to avoid consuming the buffer */
+	if (!len)
+		return false;
+
 	while (len) {
 		struct io_uring_buf *buf;
 		u32 buf_len, this_len;
@@ -212,7 +216,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,
 	sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr));
 
 	if (io_should_commit(req, issue_flags)) {
-		io_kbuf_commit(req, sel.buf_list, *len, 1);
+		if (!io_kbuf_commit(req, sel.buf_list, *len, 1))
+			req->flags |= REQ_F_BUF_MORE;
 		sel.buf_list = NULL;
 	}
 	return sel;
@@ -225,7 +230,7 @@ struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
 	struct io_br_sel sel = { };
 	struct io_buffer_list *bl;
 
-	io_ring_submit_lock(req->ctx, issue_flags);
+	io_ring_submit_lock(ctx, issue_flags);
 
 	bl = io_buffer_get_list(ctx, buf_group);
 	if (likely(bl)) {
@@ -234,7 +239,7 @@ struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
 		else
 			sel.addr = io_provided_buffer_select(req, len, bl);
 	}
-	io_ring_submit_unlock(req->ctx, issue_flags);
+	io_ring_submit_unlock(ctx, issue_flags);
 	return sel;
 }
 
@@ -345,7 +350,8 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
 		 */
 		if (ret > 0) {
 			req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE;
-			io_kbuf_commit(req, sel->buf_list, arg->out_len, ret);
+			if (!io_kbuf_commit(req, sel->buf_list, arg->out_len, ret))
+				req->flags |= REQ_F_BUF_MORE;
 		}
 	} else {
 		ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs);
@@ -391,8 +397,10 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req,
 
 	if (bl)
 		ret = io_kbuf_commit(req, bl, len, nr);
+	if (ret && (req->flags & REQ_F_BUF_MORE))
+		ret = false;
 
-	req->flags &= ~REQ_F_BUFFER_RING;
+	req->flags &= ~(REQ_F_BUFFER_RING | REQ_F_BUF_MORE);
 	return ret;
 }
 
diff --git a/io_uring/loop.c b/io_uring/loop.c
new file mode 100644
index 0000000000000..31843cc3e4510
--- /dev/null
+++ b/io_uring/loop.c
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include "io_uring.h"
+#include "wait.h"
+#include "loop.h"
+
+static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
+				  const struct iou_loop_params *lp)
+{
+	return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
+}
+
+static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait)
+{
+	atomic_set(&ctx->cq_wait_nr, nr_wait);
+	set_current_state(TASK_INTERRUPTIBLE);
+}
+
+static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
+{
+	__set_current_state(TASK_RUNNING);
+	atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
+}
+
+static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
+			 unsigned nr_wait)
+{
+	io_loop_wait_start(ctx, nr_wait);
+
+	if (unlikely(io_local_work_pending(ctx) ||
+		     io_loop_nr_cqes(ctx, lp) <= 0) ||
+		     READ_ONCE(ctx->check_cq)) {
+		io_loop_wait_finish(ctx);
+		return;
+	}
+
+	mutex_unlock(&ctx->uring_lock);
+	schedule();
+	io_loop_wait_finish(ctx);
+	mutex_lock(&ctx->uring_lock);
+}
+
+static int __io_run_loop(struct io_ring_ctx *ctx)
+{
+	struct iou_loop_params lp = {};
+
+	while (true) {
+		int nr_wait, step_res;
+
+		if (unlikely(!ctx->loop_step))
+			return -EFAULT;
+
+		step_res = ctx->loop_step(ctx, &lp);
+		if (step_res == IOU_LOOP_STOP)
+			break;
+		if (step_res != IOU_LOOP_CONTINUE)
+			return -EINVAL;
+
+		nr_wait = io_loop_nr_cqes(ctx, &lp);
+		if (nr_wait > 0)
+			io_loop_wait(ctx, &lp, nr_wait);
+		else
+			nr_wait = 0;
+
+		if (task_work_pending(current)) {
+			mutex_unlock(&ctx->uring_lock);
+			io_run_task_work();
+			mutex_lock(&ctx->uring_lock);
+		}
+		if (unlikely(task_sigpending(current)))
+			return -EINTR;
+		io_run_local_work_locked(ctx, nr_wait);
+
+		if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
+			io_cqring_overflow_flush_locked(ctx);
+	}
+
+	return 0;
+}
+
+int io_run_loop(struct io_ring_ctx *ctx)
+{
+	int ret;
+
+	if (!io_allowed_run_tw(ctx))
+		return -EEXIST;
+
+	mutex_lock(&ctx->uring_lock);
+	ret = __io_run_loop(ctx);
+	mutex_unlock(&ctx->uring_lock);
+	return ret;
+}
diff --git a/io_uring/loop.h b/io_uring/loop.h
new file mode 100644
index 0000000000000..d7718b9ce61ee
--- /dev/null
+++ b/io_uring/loop.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef IOU_LOOP_H
+#define IOU_LOOP_H
+
+#include <linux/io_uring_types.h>
+
+struct iou_loop_params {
+	/*
+	 * The CQE index to wait for. Only serves as a hint and can still be
+	 * woken up earlier.
+	 */
+	__u32			cq_wait_idx;
+};
+
+enum {
+	IOU_LOOP_CONTINUE = 0,
+	IOU_LOOP_STOP,
+};
+
+static inline bool io_has_loop_ops(struct io_ring_ctx *ctx)
+{
+	return data_race(ctx->loop_step);
+}
+
+int io_run_loop(struct io_ring_ctx *ctx);
+
+#endif
diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 57ad0085869a8..3ff9098573dbf 100644
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -67,7 +67,7 @@ void io_msg_ring_cleanup(struct io_kiocb *req)
 
 static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
 {
-	return target_ctx->task_complete;
+	return target_ctx->int_flags & IO_RING_F_TASK_COMPLETE;
 }
 
 static void io_msg_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
diff --git a/io_uring/net.c b/io_uring/net.c
index d27adbe3f20be..b3f73883a24cf 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -375,10 +375,13 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 		kmsg->msg.msg_namelen = addr_len;
 	}
 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
-		if (sr->flags & IORING_SEND_VECTORIZED)
-			return -EINVAL;
-		req->flags |= REQ_F_IMPORT_BUFFER;
-		return 0;
+		if (!(sr->flags & IORING_SEND_VECTORIZED)) {
+			req->flags |= REQ_F_IMPORT_BUFFER;
+			return 0;
+		}
+
+		kmsg->msg.msg_iter.nr_segs = sr->len;
+		return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len);
 	}
 	if (req->flags & REQ_F_BUFFER_SELECT)
 		return 0;
@@ -396,6 +399,7 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe
 	struct user_msghdr msg;
 	int ret;
 
+	sr->flags |= IORING_SEND_VECTORIZED;
 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
 	ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL);
 	if (unlikely(ret))
@@ -1329,11 +1333,12 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_async_msghdr *iomsg;
 	struct io_kiocb *notif;
+	u64 user_data;
 	int ret;
 
 	zc->done_io = 0;
 
-	if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
+	if (unlikely(READ_ONCE(sqe->__pad2[0])))
 		return -EINVAL;
 	/* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
 	if (req->flags & REQ_F_CQE_SKIP)
@@ -1342,7 +1347,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	notif = zc->notif = io_alloc_notif(ctx);
 	if (!notif)
 		return -ENOMEM;
-	notif->cqe.user_data = req->cqe.user_data;
+	user_data = READ_ONCE(sqe->addr3);
+	if (!user_data)
+		user_data = req->cqe.user_data;
+
+	notif->cqe.user_data = user_data;
 	notif->cqe.res = 0;
 	notif->cqe.flags = IORING_CQE_F_NOTIF;
 	req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY;
@@ -1366,7 +1375,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	if (zc->msg_flags & MSG_DONTWAIT)
 		req->flags |= REQ_F_NOWAIT;
 
-	if (io_is_compat(req->ctx))
+	if (io_is_compat(ctx))
 		zc->msg_flags |= MSG_CMSG_COMPAT;
 
 	iomsg = io_msg_alloc_async(req);
@@ -1441,22 +1450,39 @@ static int io_sg_from_iter(struct sk_buff *skb,
 	return ret;
 }
 
-static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
+static int io_send_zc_import(struct io_kiocb *req,
+			     struct io_async_msghdr *kmsg,
+			     unsigned int issue_flags)
 {
 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
-	struct io_async_msghdr *kmsg = req->async_data;
+	struct io_kiocb *notif = sr->notif;
+	int ret;
 
 	WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF));
 
-	sr->notif->buf_index = req->buf_index;
-	return io_import_reg_buf(sr->notif, &kmsg->msg.msg_iter,
-				(u64)(uintptr_t)sr->buf, sr->len,
-				ITER_SOURCE, issue_flags);
+	notif->buf_index = req->buf_index;
+
+	if (!(sr->flags & IORING_SEND_VECTORIZED)) {
+		ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter,
+					(u64)(uintptr_t)sr->buf, sr->len,
+					ITER_SOURCE, issue_flags);
+	} else {
+		unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
+
+		ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,
+					notif, &kmsg->vec, uvec_segs,
+					issue_flags);
+	}
+
+	if (unlikely(ret))
+		return ret;
+	req->flags &= ~REQ_F_IMPORT_BUFFER;
+	return 0;
 }
 
-int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
+int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
 {
-	struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
+	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
 	struct io_async_msghdr *kmsg = req->async_data;
 	struct socket *sock;
 	unsigned msg_flags;
@@ -1467,106 +1493,38 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
 		return -ENOTSOCK;
 	if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
 		return -EOPNOTSUPP;
-
 	if (!(req->flags & REQ_F_POLLED) &&
-	    (zc->flags & IORING_RECVSEND_POLL_FIRST))
+	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
 		return -EAGAIN;
 
 	if (req->flags & REQ_F_IMPORT_BUFFER) {
-		req->flags &= ~REQ_F_IMPORT_BUFFER;
-		ret = io_send_zc_import(req, issue_flags);
+		ret = io_send_zc_import(req, kmsg, issue_flags);
 		if (unlikely(ret))
 			return ret;
 	}
 
-	msg_flags = zc->msg_flags;
+	msg_flags = sr->msg_flags;
 	if (issue_flags & IO_URING_F_NONBLOCK)
 		msg_flags |= MSG_DONTWAIT;
 	if (msg_flags & MSG_WAITALL)
 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
-	msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
-
-	kmsg->msg.msg_flags = msg_flags;
-	kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
-	ret = sock_sendmsg(sock, &kmsg->msg);
-
-	if (unlikely(ret < min_ret)) {
-		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
-			return -EAGAIN;
-
-		if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
-			zc->done_io += ret;
-			return -EAGAIN;
-		}
-		if (ret == -ERESTARTSYS)
-			ret = -EINTR;
-		req_set_fail(req);
-	}
-
-	if (ret >= 0)
-		ret += zc->done_io;
-	else if (zc->done_io)
-		ret = zc->done_io;
-
-	/*
-	 * If we're in io-wq we can't rely on tw ordering guarantees, defer
-	 * flushing notif to io_send_zc_cleanup()
-	 */
-	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
-		io_notif_flush(zc->notif);
-		zc->notif = NULL;
-		io_req_msg_cleanup(req, 0);
-	}
-	io_req_set_res(req, ret, IORING_CQE_F_MORE);
-	return IOU_COMPLETE;
-}
 
-int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
-{
-	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
-	struct io_async_msghdr *kmsg = req->async_data;
-	struct socket *sock;
-	unsigned flags;
-	int ret, min_ret = 0;
-
-	if (req->flags & REQ_F_IMPORT_BUFFER) {
-		unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
-		int ret;
+	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
 
-		sr->notif->buf_index = req->buf_index;
-		ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,
-					sr->notif, &kmsg->vec, uvec_segs,
-					issue_flags);
-		if (unlikely(ret))
-			return ret;
-		req->flags &= ~REQ_F_IMPORT_BUFFER;
+	if (req->opcode == IORING_OP_SEND_ZC) {
+		msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
+		kmsg->msg.msg_flags = msg_flags;
+		ret = sock_sendmsg(sock, &kmsg->msg);
+	} else {
+		kmsg->msg.msg_control_user = sr->msg_control;
+		ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
 	}
 
-	sock = sock_from_file(req->file);
-	if (unlikely(!sock))
-		return -ENOTSOCK;
-	if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
-		return -EOPNOTSUPP;
-
-	if (!(req->flags & REQ_F_POLLED) &&
-	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
-		return -EAGAIN;
-
-	flags = sr->msg_flags;
-	if (issue_flags & IO_URING_F_NONBLOCK)
-		flags |= MSG_DONTWAIT;
-	if (flags & MSG_WAITALL)
-		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
-
-	kmsg->msg.msg_control_user = sr->msg_control;
-	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
-	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
-
 	if (unlikely(ret < min_ret)) {
 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
 			return -EAGAIN;
 
-		if (ret > 0 && io_net_retry(sock, flags)) {
+		if (ret > 0 && io_net_retry(sock, sr->msg_flags)) {
 			sr->done_io += ret;
 			return -EAGAIN;
 		}
diff --git a/io_uring/net.h b/io_uring/net.h
index a862960a3bb99..d4d1ddce50e3a 100644
--- a/io_uring/net.h
+++ b/io_uring/net.h
@@ -50,7 +50,6 @@ void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_connect(struct io_kiocb *req, unsigned int issue_flags);
 
-int io_send_zc(struct io_kiocb *req, unsigned int issue_flags);
 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags);
 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 void io_send_zc_cleanup(struct io_kiocb *req);
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index 91a23baf415e8..c3ef52b708113 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -67,7 +67,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.vectored		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_readv,
@@ -82,7 +81,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.vectored		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_writev,
@@ -102,7 +100,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_read_fixed,
 		.issue			= io_read_fixed,
@@ -116,7 +113,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_write_fixed,
 		.issue			= io_write_fixed,
@@ -250,7 +246,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_read,
 		.issue			= io_read,
@@ -264,7 +259,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_write,
 		.issue			= io_write,
@@ -423,7 +417,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.needs_file		= 1,
 		.plug			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.async_size		= sizeof(struct io_async_cmd),
 		.prep			= io_uring_cmd_prep,
 		.issue			= io_uring_cmd,
@@ -437,7 +430,7 @@ const struct io_issue_def io_issue_defs[] = {
 #if defined(CONFIG_NET)
 		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_send_zc_prep,
-		.issue			= io_send_zc,
+		.issue			= io_sendmsg_zc,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
@@ -556,7 +549,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.vectored		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_readv_fixed,
@@ -571,7 +563,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.audit_skip		= 1,
 		.ioprio			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.vectored		= 1,
 		.async_size		= sizeof(struct io_async_rw),
 		.prep			= io_prep_writev_fixed,
@@ -593,7 +584,6 @@ const struct io_issue_def io_issue_defs[] = {
 		.needs_file		= 1,
 		.plug			= 1,
 		.iopoll			= 1,
-		.iopoll_queue		= 1,
 		.is_128			= 1,
 		.async_size		= sizeof(struct io_async_cmd),
 		.prep			= io_uring_cmd_prep,
diff --git a/io_uring/opdef.h b/io_uring/opdef.h
index faf3955dce8b9..667f981e63b05 100644
--- a/io_uring/opdef.h
+++ b/io_uring/opdef.h
@@ -25,8 +25,6 @@ struct io_issue_def {
 	unsigned		poll_exclusive : 1;
 	/* skip auditing */
 	unsigned		audit_skip : 1;
-	/* have to be put into the iopoll list */
-	unsigned		iopoll_queue : 1;
 	/* vectored opcode, set if 1) vectored, and 2) handler needs to know */
 	unsigned		vectored : 1;
 	/* set to 1 if this opcode uses 128b sqes in a mixed sq */
diff --git a/io_uring/poll.c b/io_uring/poll.c
index b671b84657d9d..74eef78841596 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -272,12 +272,15 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 				atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
 				v &= ~IO_POLL_RETRY_FLAG;
 			}
+			v &= IO_POLL_REF_MASK;
 		}
 
 		/* the mask was stashed in __io_poll_execute */
 		if (!req->cqe.res) {
-			struct poll_table_struct pt = { ._key = req->apoll_events };
-			req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;
+			__poll_t events = req->apoll_events;
+			struct poll_table_struct pt = { ._key = events };
+
+			req->cqe.res = vfs_poll(req->file, &pt) & events;
 			/*
 			 * We got woken with a mask, but someone else got to
 			 * it first. The above vfs_poll() doesn't add us back
@@ -286,7 +289,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 			 */
 			if (unlikely(!req->cqe.res)) {
 				/* Multishot armed need not reissue */
-				if (!(req->apoll_events & EPOLLONESHOT))
+				if (!(events & EPOLLONESHOT))
 					continue;
 				return IOU_POLL_REISSUE;
 			}
@@ -304,8 +307,13 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 				return IOU_POLL_REMOVE_POLL_USE_RES;
 			}
 		} else {
-			int ret = io_poll_issue(req, tw);
+			int ret;
+
+			/* multiple refs and HUP, ensure we loop once more */
+			if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1)
+				v--;
 
+			ret = io_poll_issue(req, tw);
 			if (ret == IOU_COMPLETE)
 				return IOU_POLL_REMOVE_POLL_USE_RES;
 			else if (ret == IOU_REQUEUE)
@@ -321,7 +329,6 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 		 * Release all references, retry if someone tried to restart
 		 * task_work while we were executing it.
 		 */
-		v &= IO_POLL_REF_MASK;
 	} while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK);
 
 	io_napi_add(req);
diff --git a/io_uring/query.c b/io_uring/query.c
index 63cc30c9803d1..c1704d088374c 100644
--- a/io_uring/query.c
+++ b/io_uring/query.c
@@ -34,12 +34,12 @@ static ssize_t io_query_zcrx(union io_query_data *data)
 {
 	struct io_uring_query_zcrx *e = &data->zcrx;
 
-	e->register_flags = ZCRX_REG_IMPORT;
+	e->register_flags = ZCRX_SUPPORTED_REG_FLAGS;
 	e->area_flags = IORING_ZCRX_AREA_DMABUF;
 	e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
 	e->rq_hdr_size = sizeof(struct io_uring);
 	e->rq_hdr_alignment = L1_CACHE_BYTES;
-	e->features = ZCRX_FEATURE_RX_PAGE_SIZE;
+	e->features = ZCRX_FEATURES;
 	e->__resv2 = 0;
 	return sizeof(*e);
 }
diff --git a/io_uring/register.c b/io_uring/register.c
index 0148735f7711e..489a6feaf2288 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -184,9 +184,9 @@ static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
 		return ret;
 	}
 	if (ctx->restrictions.op_registered)
-		ctx->op_restricted = 1;
+		ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
 	if (ctx->restrictions.reg_registered)
-		ctx->reg_restricted = 1;
+		ctx->int_flags |= IO_RING_F_REG_RESTRICTED;
 	return 0;
 }
 
@@ -384,7 +384,7 @@ static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
 	for (i = 0; i < ARRAY_SIZE(new_count); i++)
 		if (new_count[i])
 			ctx->iowq_limits[i] = new_count[i];
-	ctx->iowq_limits_set = true;
+	ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET;
 
 	if (tctx && tctx->io_wq) {
 		ret = io_wq_max_workers(tctx->io_wq, new_count);
@@ -725,7 +725,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
 	if (ctx->submitter_task && ctx->submitter_task != current)
 		return -EEXIST;
 
-	if (ctx->reg_restricted && !(ctx->flags & IORING_SETUP_R_DISABLED)) {
+	if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && !(ctx->flags & IORING_SETUP_R_DISABLED)) {
 		opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
 		if (!test_bit(opcode, ctx->restrictions.register_op))
 			return -EACCES;
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 4fa59bf89bba1..52554ed89b11c 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -295,7 +295,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		u64 tag = 0;
 
 		uvec = u64_to_user_ptr(user_data);
-		iov = iovec_from_user(uvec, 1, 1, &fast_iov, ctx->compat);
+		iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
 		if (IS_ERR(iov)) {
 			err = PTR_ERR(iov);
 			break;
@@ -319,7 +319,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		i = array_index_nospec(up->offset + done, ctx->buf_table.nr);
 		io_reset_rsrc_node(ctx, &ctx->buf_table, i);
 		ctx->buf_table.nodes[i] = node;
-		if (ctx->compat)
+		if (io_is_compat(ctx))
 			user_data += sizeof(struct compat_iovec);
 		else
 			user_data += sizeof(struct iovec);
@@ -883,12 +883,12 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
 
 		if (arg) {
 			uvec = (struct iovec __user *) arg;
-			iov = iovec_from_user(uvec, 1, 1, &fast_iov, ctx->compat);
+			iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
 			if (IS_ERR(iov)) {
 				ret = PTR_ERR(iov);
 				break;
 			}
-			if (ctx->compat)
+			if (io_is_compat(ctx))
 				arg += sizeof(struct compat_iovec);
 			else
 				arg += sizeof(struct iovec);
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 1a5f262734e8e..046f76a71b9c1 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -504,7 +504,7 @@ static bool io_rw_should_reissue(struct io_kiocb *req)
 	if (!S_ISBLK(mode) && !S_ISREG(mode))
 		return false;
 	if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
-	    !(ctx->flags & IORING_SETUP_IOPOLL)))
+	    !(req->flags & REQ_F_IOPOLL)))
 		return false;
 	/*
 	 * If ref is dying, we might be running poll reap from the exit work.
@@ -640,7 +640,7 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
 		}
 	}
 
-	if (req->ctx->flags & IORING_SETUP_IOPOLL)
+	if (req->flags & REQ_F_IOPOLL)
 		io_complete_rw_iopoll(&rw->kiocb, ret);
 	else
 		io_complete_rw(&rw->kiocb, ret);
@@ -654,7 +654,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,
 
 	if (ret >= 0 && req->flags & REQ_F_CUR_POS)
 		req->file->f_pos = rw->kiocb.ki_pos;
-	if (ret >= 0 && !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
+	if (ret >= 0 && !(req->flags & REQ_F_IOPOLL)) {
 		u32 cflags = 0;
 
 		__io_complete_rw_common(req, ret);
@@ -876,6 +876,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type)
 	if (ctx->flags & IORING_SETUP_IOPOLL) {
 		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)
 			return -EOPNOTSUPP;
+		req->flags |= REQ_F_IOPOLL;
 		kiocb->private = NULL;
 		kiocb->ki_flags |= IOCB_HIPRI;
 		req->iopoll_completed = 0;
@@ -899,7 +900,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type)
 		 * We have a union of meta fields with wpq used for buffered-io
 		 * in io_async_rw, so fail it here.
 		 */
-		if (!(req->file->f_flags & O_DIRECT))
+		if (!(file->f_flags & O_DIRECT))
 			return -EOPNOTSUPP;
 		kiocb->ki_flags |= IOCB_HAS_METADATA;
 		kiocb->private = &io->meta;
@@ -963,7 +964,7 @@ static int __io_read(struct io_kiocb *req, struct io_br_sel *sel,
 		if (io_file_can_poll(req))
 			return -EAGAIN;
 		/* IOPOLL retry should happen for io-wq threads */
-		if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
+		if (!force_nonblock && !(req->flags & REQ_F_IOPOLL))
 			goto done;
 		/* no retry on NONBLOCK nor RWF_NOWAIT */
 		if (req->flags & REQ_F_NOWAIT)
@@ -1188,7 +1189,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)
 		goto done;
 	if (!force_nonblock || ret2 != -EAGAIN) {
 		/* IOPOLL retry should happen for io-wq threads */
-		if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL))
+		if (ret2 == -EAGAIN && (req->flags & REQ_F_IOPOLL))
 			goto ret_eagain;
 
 		if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) {
diff --git a/io_uring/tctx.c b/io_uring/tctx.c
index 7cbcb82aedfb8..143de8e990eb3 100644
--- a/io_uring/tctx.c
+++ b/io_uring/tctx.c
@@ -121,7 +121,7 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
 			return ret;
 
 		tctx = current->io_uring;
-		if (ctx->iowq_limits_set) {
+		if (ctx->int_flags & IO_RING_F_IOWQ_LIMITS_SET) {
 			unsigned int limits[2] = { ctx->iowq_limits[0],
 						   ctx->iowq_limits[1], };
 
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index cb61d4862fc65..579fdddac71ae 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -30,11 +30,30 @@ struct io_timeout_rem {
 	u64				addr;
 
 	/* timeout update */
-	struct timespec64		ts;
+	ktime_t				time;
 	u32				flags;
 	bool				ltimeout;
 };
 
+static int io_parse_user_time(ktime_t *time, u64 arg, unsigned flags)
+{
+	struct timespec64 ts;
+
+	if (flags & IORING_TIMEOUT_IMMEDIATE_ARG) {
+		*time = ns_to_ktime(arg);
+		if (*time < 0)
+			return -EINVAL;
+		return 0;
+	}
+
+	if (get_timespec64(&ts, u64_to_user_ptr(arg)))
+		return -EFAULT;
+	if (ts.tv_sec < 0 || ts.tv_nsec < 0)
+		return -EINVAL;
+	*time = timespec64_to_ktime(ts);
+	return 0;
+}
+
 static struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
 						   struct io_kiocb *link);
 
@@ -80,7 +99,7 @@ static void io_timeout_complete(struct io_tw_req tw_req, io_tw_token_t tw)
 			/* re-arm timer */
 			raw_spin_lock_irq(&ctx->timeout_lock);
 			list_add(&timeout->list, ctx->timeout_list.prev);
-			hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
+			hrtimer_start(&data->timer, data->time, data->mode);
 			raw_spin_unlock_irq(&ctx->timeout_lock);
 			return;
 		}
@@ -395,7 +414,7 @@ static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
 }
 
 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
-				    struct timespec64 *ts, enum hrtimer_mode mode)
+				    ktime_t ts, enum hrtimer_mode mode)
 	__must_hold(&ctx->timeout_lock)
 {
 	struct io_timeout_data *io;
@@ -417,12 +436,12 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
 	if (hrtimer_try_to_cancel(&io->timer) == -1)
 		return -EALREADY;
 	hrtimer_setup(&io->timer, io_link_timeout_fn, io_timeout_get_clock(io), mode);
-	hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
+	hrtimer_start(&io->timer, ts, mode);
 	return 0;
 }
 
 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
-			     struct timespec64 *ts, enum hrtimer_mode mode)
+			     ktime_t time, enum hrtimer_mode mode)
 	__must_hold(&ctx->timeout_lock)
 {
 	struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };
@@ -435,20 +454,23 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
 
 	timeout->off = 0; /* noseq */
 	data = req->async_data;
-	data->ts = *ts;
+	data->time = time;
 
 	list_add_tail(&timeout->list, &ctx->timeout_list);
 	hrtimer_setup(&data->timer, io_timeout_fn, io_timeout_get_clock(data), mode);
-	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), mode);
+	hrtimer_start(&data->timer, data->time, mode);
 	return 0;
 }
 
 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
+	int ret;
 
 	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
 		return -EINVAL;
+	if (sqe->addr3 || sqe->__pad2[0])
+		return -EINVAL;
 	if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
 		return -EINVAL;
 
@@ -460,12 +482,13 @@ int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 			return -EINVAL;
 		if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
 			tr->ltimeout = true;
-		if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
-			return -EINVAL;
-		if (get_timespec64(&tr->ts, u64_to_user_ptr(READ_ONCE(sqe->addr2))))
-			return -EFAULT;
-		if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
+		if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK |
+				  IORING_TIMEOUT_ABS |
+				  IORING_TIMEOUT_IMMEDIATE_ARG))
 			return -EINVAL;
+		ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2), tr->flags);
+		if (ret)
+			return ret;
 	} else if (tr->flags) {
 		/* timeout removal doesn't support flags */
 		return -EINVAL;
@@ -500,9 +523,9 @@ int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
 
 		raw_spin_lock_irq(&ctx->timeout_lock);
 		if (tr->ltimeout)
-			ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
+			ret = io_linked_timeout_update(ctx, tr->addr, tr->time, mode);
 		else
-			ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
+			ret = io_timeout_update(ctx, tr->addr, tr->time, mode);
 		raw_spin_unlock_irq(&ctx->timeout_lock);
 	}
 
@@ -520,7 +543,10 @@ static int __io_timeout_prep(struct io_kiocb *req,
 	struct io_timeout_data *data;
 	unsigned flags;
 	u32 off = READ_ONCE(sqe->off);
+	int ret;
 
+	if (sqe->addr3 || sqe->__pad2[0])
+		return -EINVAL;
 	if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
 		return -EINVAL;
 	if (off && is_timeout_link)
@@ -528,7 +554,8 @@ static int __io_timeout_prep(struct io_kiocb *req,
 	flags = READ_ONCE(sqe->timeout_flags);
 	if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
 		      IORING_TIMEOUT_ETIME_SUCCESS |
-		      IORING_TIMEOUT_MULTISHOT))
+		      IORING_TIMEOUT_MULTISHOT |
+		      IORING_TIMEOUT_IMMEDIATE_ARG))
 		return -EINVAL;
 	/* more than one clock specified is invalid, obviously */
 	if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
@@ -539,8 +566,8 @@ static int __io_timeout_prep(struct io_kiocb *req,
 
 	INIT_LIST_HEAD(&timeout->list);
 	timeout->off = off;
-	if (unlikely(off && !req->ctx->off_timeout_used))
-		req->ctx->off_timeout_used = true;
+	if (unlikely(off && !(req->ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)))
+		req->ctx->int_flags |= IO_RING_F_OFF_TIMEOUT_USED;
 	/*
 	 * for multishot reqs w/ fixed nr of repeats, repeats tracks the
 	 * remaining nr
@@ -557,11 +584,9 @@ static int __io_timeout_prep(struct io_kiocb *req,
 	data->req = req;
 	data->flags = flags;
 
-	if (get_timespec64(&data->ts, u64_to_user_ptr(READ_ONCE(sqe->addr))))
-		return -EFAULT;
-
-	if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
-		return -EINVAL;
+	ret = io_parse_user_time(&data->time, READ_ONCE(sqe->addr), flags);
+	if (ret)
+		return ret;
 
 	data->mode = io_translate_timeout_mode(flags);
 
@@ -637,7 +662,7 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
 	}
 add:
 	list_add(&timeout->list, entry);
-	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
+	hrtimer_start(&data->timer, data->time, data->mode);
 	raw_spin_unlock_irq(&ctx->timeout_lock);
 	return IOU_ISSUE_SKIP_COMPLETE;
 }
@@ -655,8 +680,7 @@ void io_queue_linked_timeout(struct io_kiocb *req)
 	if (timeout->head) {
 		struct io_timeout_data *data = req->async_data;
 
-		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
-				data->mode);
+		hrtimer_start(&data->timer, data->time, data->mode);
 		list_add_tail(&timeout->list, &ctx->ltimeout_list);
 	}
 	raw_spin_unlock_irq(&ctx->timeout_lock);
diff --git a/io_uring/timeout.h b/io_uring/timeout.h
index 2b7c9ad729925..1620f94dd45a8 100644
--- a/io_uring/timeout.h
+++ b/io_uring/timeout.h
@@ -3,7 +3,7 @@
 struct io_timeout_data {
 	struct io_kiocb			*req;
 	struct hrtimer			timer;
-	struct timespec64		ts;
+	ktime_t				time;
 	enum hrtimer_mode		mode;
 	u32				flags;
 };
diff --git a/io_uring/tw.c b/io_uring/tw.c
index 2f2b4ac4b1267..fdff81eebc95c 100644
--- a/io_uring/tw.c
+++ b/io_uring/tw.c
@@ -222,7 +222,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
 
 	if (!head) {
 		io_ctx_mark_taskrun(ctx);
-		if (ctx->has_evfd)
+		if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)
 			io_eventfd_signal(ctx, false);
 	}
 
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index ee7b49f47cb55..7b25dcd9d05fc 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -110,7 +110,7 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
 	 * because iopoll completion data overlaps with the hash_node used
 	 * for tracking.
 	 */
-	if (ctx->flags & IORING_SETUP_IOPOLL)
+	if (req->flags & REQ_F_IOPOLL)
 		return;
 
 	if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
@@ -167,7 +167,7 @@ void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
 		io_req_set_cqe32_extra(req, res2, 0);
 	}
 	io_req_uring_cleanup(req, issue_flags);
-	if (req->ctx->flags & IORING_SETUP_IOPOLL) {
+	if (req->flags & REQ_F_IOPOLL) {
 		/* order with io_iopoll_req_issued() checking ->iopoll_complete */
 		smp_store_release(&req->iopoll_completed, 1);
 	} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
@@ -257,9 +257,8 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
 		issue_flags |= IO_URING_F_CQE32;
 	if (io_is_compat(ctx))
 		issue_flags |= IO_URING_F_COMPAT;
-	if (ctx->flags & IORING_SETUP_IOPOLL) {
-		if (!file->f_op->uring_cmd_iopoll)
-			return -EOPNOTSUPP;
+	if (ctx->flags & IORING_SETUP_IOPOLL && file->f_op->uring_cmd_iopoll) {
+		req->flags |= REQ_F_IOPOLL;
 		issue_flags |= IO_URING_F_IOPOLL;
 		req->iopoll_completed = 0;
 		if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) {
diff --git a/io_uring/wait.h b/io_uring/wait.h
index 5e236f74e1aff..037e512dd80c1 100644
--- a/io_uring/wait.h
+++ b/io_uring/wait.h
@@ -25,6 +25,7 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
 		   struct ext_arg *ext_arg);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx);
+void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx);
 
 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
 {
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 62d693287457f..73fa827597710 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -778,11 +778,13 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
 		return -EFAULT;
 	if (!mem_is_zero(&reg.__resv, sizeof(reg.__resv)) || reg.zcrx_id)
 		return -EINVAL;
+	if (reg.flags & ~ZCRX_SUPPORTED_REG_FLAGS)
+		return -EINVAL;
 	if (reg.flags & ZCRX_REG_IMPORT)
 		return import_zcrx(ctx, arg, &reg);
 	if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd)))
 		return -EFAULT;
-	if (reg.if_rxq == -1 || !reg.rq_entries || reg.flags)
+	if (reg.if_rxq == -1 || !reg.rq_entries)
 		return -EINVAL;
 	if (reg.rq_entries > IO_RQ_MAX_ENTRIES) {
 		if (!(ctx->flags & IORING_SETUP_CLAMP))
diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 32ab95b2cb811..0ddcf0ee88615 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -8,6 +8,9 @@
 #include <net/page_pool/types.h>
 #include <net/net_trackers.h>
 
+#define ZCRX_SUPPORTED_REG_FLAGS	(ZCRX_REG_IMPORT)
+#define ZCRX_FEATURES			(ZCRX_FEATURE_RX_PAGE_SIZE)
+
 struct io_zcrx_mem {
 	unsigned long			size;
 	bool				is_dmabuf;
diff --git a/kernel/audit.c b/kernel/audit.c
index 5a02160565249..e1d489bc2dff9 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -355,8 +355,8 @@ void audit_panic(const char *message)
 
 static inline int audit_rate_check(void)
 {
-	static unsigned long	last_check = 0;
-	static int		messages   = 0;
+	static unsigned long	last_check;
+	static int		messages;
 	static DEFINE_SPINLOCK(lock);
 	unsigned long		flags;
 	unsigned long		now;
@@ -391,7 +391,7 @@ static inline int audit_rate_check(void)
 */
 void audit_log_lost(const char *message)
 {
-	static unsigned long	last_msg = 0;
+	static unsigned long	last_msg;
 	static DEFINE_SPINLOCK(lock);
 	unsigned long		flags;
 	unsigned long		now;
@@ -1295,6 +1295,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
 		memset(&s, 0, sizeof(s));
 		/* guard against past and future API changes */
 		memcpy(&s, data, min_t(size_t, sizeof(s), data_len));
+		if (s.mask & ~AUDIT_STATUS_ALL)
+			return -EINVAL;
 		if (s.mask & AUDIT_STATUS_ENABLED) {
 			err = audit_set_enabled(s.enabled);
 			if (err < 0)
diff --git a/kernel/audit.h b/kernel/audit.h
index 7c401729e21bb..ac81fa02bcd75 100644
--- a/kernel/audit.h
+++ b/kernel/audit.h
@@ -76,7 +76,7 @@ struct audit_names {
 	int			name_len;	/* number of chars to log */
 	bool			hidden;		/* don't log this record */
 
-	unsigned long		ino;
+	u64			ino;
 	dev_t			dev;
 	umode_t			mode;
 	kuid_t			uid;
@@ -225,9 +225,9 @@ extern int auditd_test_task(struct task_struct *task);
 #define AUDIT_INODE_BUCKETS	32
 extern struct list_head audit_inode_hash[AUDIT_INODE_BUCKETS];
 
-static inline int audit_hash_ino(u32 ino)
+static inline int audit_hash_ino(u64 ino)
 {
-	return (ino & (AUDIT_INODE_BUCKETS-1));
+	return ((u32)ino & (AUDIT_INODE_BUCKETS-1));
 }
 
 /* Indicates that audit should log the full pathname. */
@@ -277,16 +277,15 @@ extern int audit_to_watch(struct audit_krule *krule, char *path, int len,
 extern int audit_add_watch(struct audit_krule *krule, struct list_head **list);
 extern void audit_remove_watch_rule(struct audit_krule *krule);
 extern char *audit_watch_path(struct audit_watch *watch);
-extern int audit_watch_compare(struct audit_watch *watch, unsigned long ino,
-			       dev_t dev);
+extern int audit_watch_compare(struct audit_watch *watch, u64 ino, dev_t dev);
 
 extern struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule,
 						    char *pathname, int len);
 extern char *audit_mark_path(struct audit_fsnotify_mark *mark);
 extern void audit_remove_mark(struct audit_fsnotify_mark *audit_mark);
 extern void audit_remove_mark_rule(struct audit_krule *krule);
-extern int audit_mark_compare(struct audit_fsnotify_mark *mark,
-			      unsigned long ino, dev_t dev);
+extern int audit_mark_compare(struct audit_fsnotify_mark *mark, u64 ino,
+			      dev_t dev);
 extern int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old);
 extern int audit_exe_compare(struct task_struct *tsk,
 			     struct audit_fsnotify_mark *mark);
diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c
index a4401f6510608..711454f9f7242 100644
--- a/kernel/audit_fsnotify.c
+++ b/kernel/audit_fsnotify.c
@@ -25,7 +25,7 @@
  */
 struct audit_fsnotify_mark {
 	dev_t dev;		/* associated superblock device */
-	unsigned long ino;	/* associated inode number */
+	u64 ino;		/* associated inode number */
 	char *path;		/* insertion path */
 	struct fsnotify_mark mark; /* fsnotify mark on the inode */
 	struct audit_krule *rule;
@@ -57,7 +57,7 @@ char *audit_mark_path(struct audit_fsnotify_mark *mark)
 	return mark->path;
 }
 
-int audit_mark_compare(struct audit_fsnotify_mark *mark, unsigned long ino, dev_t dev)
+int audit_mark_compare(struct audit_fsnotify_mark *mark, u64 ino, dev_t dev)
 {
 	if (mark->ino == AUDIT_INO_UNSET)
 		return 0;
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index 096faac2435ce..33577f0f54eff 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -37,7 +37,7 @@ struct audit_watch {
 	refcount_t		count;	/* reference count */
 	dev_t			dev;	/* associated superblock device */
 	char			*path;	/* insertion path */
-	unsigned long		ino;	/* associated inode number */
+	u64			ino;	/* associated inode number */
 	struct audit_parent	*parent; /* associated parent */
 	struct list_head	wlist;	/* entry in parent->watches list */
 	struct list_head	rules;	/* anchor for krule->rlist */
@@ -125,7 +125,7 @@ char *audit_watch_path(struct audit_watch *watch)
 	return watch->path;
 }
 
-int audit_watch_compare(struct audit_watch *watch, unsigned long ino, dev_t dev)
+int audit_watch_compare(struct audit_watch *watch, u64 ino, dev_t dev)
 {
 	return (watch->ino != AUDIT_INO_UNSET) &&
 		(watch->ino == ino) &&
@@ -244,7 +244,7 @@ static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watc
 /* Update inode info in audit rules based on filesystem event. */
 static void audit_update_watch(struct audit_parent *parent,
 			       const struct qstr *dname, dev_t dev,
-			       unsigned long ino, unsigned invalidating)
+			       u64 ino, unsigned invalidating)
 {
 	struct audit_watch *owatch, *nwatch, *nextw;
 	struct audit_krule *r, *nextr;
@@ -285,7 +285,7 @@ static void audit_update_watch(struct audit_parent *parent,
 				list_del(&oentry->rule.list);
 				audit_panic("error updating watch, removing");
 			} else {
-				int h = audit_hash_ino((u32)ino);
+				int h = audit_hash_ino(ino);
 
 				/*
 				 * nentry->rule.watch == oentry->rule.watch so
@@ -439,7 +439,7 @@ int audit_add_watch(struct audit_krule *krule, struct list_head **list)
 
 	audit_add_to_parent(krule, parent);
 
-	h = audit_hash_ino((u32)watch->ino);
+	h = audit_hash_ino(watch->ino);
 	*list = &audit_inode_hash[h];
 error:
 	path_put(&parent_path);
@@ -527,7 +527,7 @@ int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old)
 int audit_exe_compare(struct task_struct *tsk, struct audit_fsnotify_mark *mark)
 {
 	struct file *exe_file;
-	unsigned long ino;
+	u64 ino;
 	dev_t dev;
 
 	/* only do exe filtering if we are recording @current events/records */
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 6e3abbf08e3d0..093425123f6c7 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -303,8 +303,7 @@ static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data *
 	return ERR_PTR(err);
 }
 
-static u32 audit_ops[] =
-{
+static u32 audit_ops[] = {
 	[Audit_equal] = AUDIT_EQUAL,
 	[Audit_not_equal] = AUDIT_NOT_EQUAL,
 	[Audit_bitmask] = AUDIT_BIT_MASK,
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index f6af6a8f68c4f..f4a42d4f5bd9b 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -886,7 +886,7 @@ static int audit_filter_inode_name(struct task_struct *tsk,
 				   struct audit_names *n,
 				   struct audit_context *ctx)
 {
-	int h = audit_hash_ino((u32)n->ino);
+	int h = audit_hash_ino(n->ino);
 	struct list_head *list = &audit_inode_hash[h];
 
 	return __audit_filter_op(tsk, ctx, list, n, ctx->major);
@@ -931,6 +931,9 @@ static inline void audit_free_names(struct audit_context *context)
 {
 	struct audit_names *n, *next;
 
+	if (!context->name_count)
+		return;	/* audit_alloc_name() has not been called */
+
 	list_for_each_entry_safe(n, next, &context->names_list, list) {
 		list_del(&n->list);
 		if (n->name)
@@ -939,7 +942,7 @@ static inline void audit_free_names(struct audit_context *context)
 			kfree(n);
 	}
 	context->name_count = 0;
-	path_put(&context->pwd);
+	put_fs_pwd_pool(current->fs, &context->pwd);
 	context->pwd.dentry = NULL;
 	context->pwd.mnt = NULL;
 }
@@ -1534,7 +1537,7 @@ static void audit_log_name(struct audit_context *context, struct audit_names *n,
 		audit_log_format(ab, " name=(null)");
 
 	if (n->ino != AUDIT_INO_UNSET)
-		audit_log_format(ab, " inode=%lu dev=%02x:%02x mode=%#ho ouid=%u ogid=%u rdev=%02x:%02x",
+		audit_log_format(ab, " inode=%llu dev=%02x:%02x mode=%#ho ouid=%u ogid=%u rdev=%02x:%02x",
 				 n->ino,
 				 MAJOR(n->dev),
 				 MINOR(n->dev),
@@ -2165,7 +2168,7 @@ static struct audit_names *audit_alloc_name(struct audit_context *context,
 
 	context->name_count++;
 	if (!context->pwd.dentry)
-		get_fs_pwd(current->fs, &context->pwd);
+		get_fs_pwd_pool(current->fs, &context->pwd);
 	return aname;
 }
 
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index f355cf1c1a169..08d008cc471e6 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -656,8 +656,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
 	guard(mutex)(&arena->lock);
 	/* iterate link list under lock */
 	list_for_each_entry(vml, &arena->vma_list, head)
-		zap_page_range_single(vml->vma, uaddr,
-				      PAGE_SIZE * page_cnt, NULL);
+		zap_vma_range(vml->vma, uaddr, PAGE_SIZE * page_cnt);
 }
 
 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable)
diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 9c96a4477f81a..cad17ca8552f8 100644
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -393,6 +393,9 @@ int bpf_selem_unlink(struct bpf_local_storage_elem *selem)
 	unsigned long flags;
 	int err;
 
+	if (in_nmi())
+		return -EOPNOTSUPP;
+
 	if (unlikely(!selem_linked_to_storage_lockless(selem)))
 		/* selem has already been unlinked from sk */
 		return 0;
@@ -494,6 +497,14 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem,
 			}
 			raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
 		}
+		/*
+		 * Highly unlikely scenario: memory leak
+		 *
+		 * When destroy() fails to acqurire local_storage->lock and initializes
+		 * selem->local_storage to NULL before any racing map_free() sees the same
+		 * selem, no one will free the local storage.
+		 */
+		WARN_ON_ONCE(err && !in_map_free);
 		if (!err || !in_map_free)
 			RCU_INIT_POINTER(selem->local_storage, NULL);
 	}
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 4872d2a6c42d3..c5876c91face3 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -1787,7 +1787,16 @@ static void btf_free_id(struct btf *btf)
 	 * of the _bh() version.
 	 */
 	spin_lock_irqsave(&btf_idr_lock, flags);
-	idr_remove(&btf_idr, btf->id);
+	if (btf->id) {
+		idr_remove(&btf_idr, btf->id);
+		/*
+		 * Clear the id here to make this function idempotent, since it will get
+		 * called a couple of times for module BTFs: on module unload, and then
+		 * the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use
+		 * 0 as sentinel value.
+		 */
+		WRITE_ONCE(btf->id, 0);
+	}
 	spin_unlock_irqrestore(&btf_idr_lock, flags);
 }
 
@@ -7836,15 +7845,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 			tname, nargs, MAX_BPF_FUNC_REG_ARGS);
 		return -EINVAL;
 	}
-	/* check that function returns int, exception cb also requires this */
+	/* check that function is void or returns int, exception cb also requires this */
 	t = btf_type_by_id(btf, t->type);
 	while (btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (!btf_type_is_int(t) && !btf_is_any_enum(t)) {
+	if (!btf_type_is_void(t) && !btf_type_is_int(t) && !btf_is_any_enum(t)) {
 		if (!is_global)
 			return -EINVAL;
 		bpf_log(log,
-			"Global function %s() doesn't return scalar. Only those are supported.\n",
+			"Global function %s() return value not void or scalar. "
+			"Only those are supported.\n",
 			tname);
 		return -EINVAL;
 	}
@@ -8115,7 +8125,7 @@ static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp)
 {
 	const struct btf *btf = filp->private_data;
 
-	seq_printf(m, "btf_id:\t%u\n", btf->id);
+	seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id));
 }
 #endif
 
@@ -8197,7 +8207,7 @@ int btf_get_info_by_fd(const struct btf *btf,
 	if (copy_from_user(&info, uinfo, info_copy))
 		return -EFAULT;
 
-	info.id = btf->id;
+	info.id = READ_ONCE(btf->id);
 	ubtf = u64_to_user_ptr(info.btf);
 	btf_copy = min_t(u32, btf->data_size, info.btf_size);
 	if (copy_to_user(ubtf, btf->data, btf_copy))
@@ -8260,7 +8270,7 @@ int btf_get_fd_by_id(u32 id)
 
 u32 btf_obj_id(const struct btf *btf)
 {
-	return btf->id;
+	return READ_ONCE(btf->id);
 }
 
 bool btf_is_kernel(const struct btf *btf)
@@ -8382,6 +8392,13 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op,
 			if (btf_mod->module != module)
 				continue;
 
+			/*
+			 * For modules, we do the freeing of BTF IDR as soon as
+			 * module goes away to disable BTF discovery, since the
+			 * btf_try_get_module() on such BTFs will fail. This may
+			 * be called again on btf_put(), but it's ok to do so.
+			 */
+			btf_free_id(btf_mod->btf);
 			list_del(&btf_mod->list);
 			if (btf_mod->sysfs_attr)
 				sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 3ece2da55625c..ed3f6fe646d5d 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1422,6 +1422,27 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
 		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
 		*to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off);
 		break;
+
+	case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
+	case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
+	case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
+	case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
+		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^
+				      from->imm);
+		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
+		/*
+		 * Cannot use BPF_STX_MEM() macro here as it
+		 * hardcodes BPF_MEM mode, losing PROBE_MEM32
+		 * and breaking arena addressing in the JIT.
+		 */
+		*to++ = (struct bpf_insn) {
+			.code  = BPF_STX | BPF_PROBE_MEM32 |
+				 BPF_SIZE(from->code),
+			.dst_reg = from->dst_reg,
+			.src_reg = BPF_REG_AX,
+			.off   = from->off,
+		};
+		break;
 	}
 out:
 	return to - to_buff;
@@ -2060,12 +2081,12 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
 		if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT))
 			goto out;
 
-		tail_call_cnt++;
-
 		prog = READ_ONCE(array->ptrs[index]);
 		if (!prog)
 			goto out;
 
+		tail_call_cnt++;
+
 		/* ARG1 at this point is guaranteed to point to CTX from
 		 * the verifier side due to the fact that the tail call is
 		 * handled like a helper, that is, bpf_tail_call_proto,
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index 32b43cb9061bb..5e59ab896f058 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -223,7 +223,10 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
 			}
 			break;
 		default:
-			bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
+			bpf_warn_invalid_xdp_action(xdpf->dev_rx, rcpu->prog, act);
+			fallthrough;
+		case XDP_ABORTED:
+			trace_xdp_exception(xdpf->dev_rx, rcpu->prog, act);
 			fallthrough;
 		case XDP_DROP:
 			xdp_return_frame(xdpf);
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 6eb6c82ed2ee1..cb6d242bd0936 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -4578,6 +4578,9 @@ BTF_ID_FLAGS(func, bpf_key_put, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_verify_pkcs7_signature, KF_SLEEPABLE)
 #endif
 #endif
+#ifdef CONFIG_S390
+BTF_ID_FLAGS(func, bpf_get_lowcore)
+#endif
 BTF_KFUNCS_END(generic_btf_ids)
 
 static const struct btf_kfunc_id_set generic_kfunc_set = {
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index a0c3b35de2ce6..37d72b0521920 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -581,6 +581,8 @@ int tnum_strn(char *str, size_t size, struct tnum a)
 	if (a.mask == 0) {
 		if (is_unum_decimal(a.value))
 			return snprintf(str, size, "%llu", a.value);
+		if (is_snum_decimal(a.value))
+			return snprintf(str, size, "%lld", a.value);
 		else
 			return snprintf(str, size, "%#llx", a.value);
 	}
@@ -692,7 +694,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
 		if (state->frameno != reg->frameno)
 			verbose(env, "[%d]", reg->frameno);
 		if (tnum_is_const(reg->var_off)) {
-			verbose_snum(env, reg->var_off.value + reg->off);
+			verbose_snum(env, reg->var_off.value + reg->delta);
 			return;
 		}
 	}
@@ -702,7 +704,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
 	if (reg->id)
 		verbose_a("id=%d", reg->id & ~BPF_ADD_CONST);
 	if (reg->id & BPF_ADD_CONST)
-		verbose(env, "%+d", reg->off);
+		verbose(env, "%+d", reg->delta);
 	if (reg->ref_obj_id)
 		verbose_a("ref_obj_id=%d", reg->ref_obj_id);
 	if (type_is_non_owning_ref(reg->type))
@@ -714,9 +716,9 @@ static void print_reg_state(struct bpf_verifier_env *env,
 			  reg->map_ptr->key_size,
 			  reg->map_ptr->value_size);
 	}
-	if (t != SCALAR_VALUE && reg->off) {
+	if (t != SCALAR_VALUE && reg->delta) {
 		verbose_a("off=");
-		verbose_snum(env, reg->off);
+		verbose_snum(env, reg->delta);
 	}
 	if (type_is_pkt_pointer(t)) {
 		verbose_a("r=");
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 159b25f8269dd..99613da1af2af 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -444,6 +444,29 @@ static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
 }
 
+static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+{
+	const struct btf_type *type, *func, *func_proto;
+	const struct btf *btf = env->prog->aux->btf;
+	u32 btf_id;
+
+	btf_id = env->prog->aux->func_info[subprog].type_id;
+
+	func = btf_type_by_id(btf, btf_id);
+	if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id))
+		return false;
+
+	func_proto = btf_type_by_id(btf, func->type);
+	if (!func_proto)
+		return false;
+
+	type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
+	if (!type)
+		return false;
+
+	return btf_type_is_void(type);
+}
+
 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
 {
 	struct bpf_func_info *info;
@@ -654,7 +677,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s
 		return -EINVAL;
 	}
 
-	off = reg->off + reg->var_off.value;
+	off = reg->var_off.value;
 	if (off % BPF_REG_SIZE) {
 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
 		return -EINVAL;
@@ -2281,11 +2304,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
 				struct btf_field_graph_root *ds_head)
 {
-	__mark_reg_known_zero(&regs[regno]);
+	__mark_reg_known(&regs[regno], ds_head->node_offset);
 	regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
 	regs[regno].btf = ds_head->btf;
 	regs[regno].btf_id = ds_head->value_btf_id;
-	regs[regno].off = ds_head->node_offset;
 }
 
 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
@@ -2316,7 +2338,6 @@ static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
 	 */
 	return reg->type == which &&
 	       reg->id == 0 &&
-	       reg->off == 0 &&
 	       tnum_equals_const(reg->var_off, 0);
 }
 
@@ -2427,7 +2448,7 @@ static void __update_reg_bounds(struct bpf_reg_state *reg)
 }
 
 /* Uses signed min/max values to inform unsigned, and vice-versa */
-static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
+static void deduce_bounds_32_from_64(struct bpf_reg_state *reg)
 {
 	/* If upper 32 bits of u64/s64 range don't change, we can use lower 32
 	 * bits to improve our u32/s32 boundaries.
@@ -2497,6 +2518,10 @@ static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
 		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
 		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
 	}
+}
+
+static void deduce_bounds_32_from_32(struct bpf_reg_state *reg)
+{
 	/* if u32 range forms a valid s32 range (due to matching sign bit),
 	 * try to learn from that
 	 */
@@ -2538,7 +2563,7 @@ static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
 	}
 }
 
-static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
+static void deduce_bounds_64_from_64(struct bpf_reg_state *reg)
 {
 	/* If u64 range forms a valid s64 range (due to matching sign bit),
 	 * try to learn from that. Let's do a bit of ASCII art to see when
@@ -2673,7 +2698,7 @@ static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
 	}
 }
 
-static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
+static void deduce_bounds_64_from_32(struct bpf_reg_state *reg)
 {
 	/* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
 	 * values on both sides of 64-bit range in hope to have tighter range.
@@ -2742,9 +2767,10 @@ static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
 
 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
 {
-	__reg32_deduce_bounds(reg);
-	__reg64_deduce_bounds(reg);
-	__reg_deduce_mixed_bounds(reg);
+	deduce_bounds_64_from_64(reg);
+	deduce_bounds_32_from_64(reg);
+	deduce_bounds_32_from_32(reg);
+	deduce_bounds_64_from_32(reg);
 }
 
 /* Attempts to improve var_off based on unsigned min/max information */
@@ -2767,7 +2793,6 @@ static void reg_bounds_sync(struct bpf_reg_state *reg)
 	/* We might have learned something about the sign bit. */
 	__reg_deduce_bounds(reg);
 	__reg_deduce_bounds(reg);
-	__reg_deduce_bounds(reg);
 	/* We might have learned some bits from the bounds. */
 	__reg_bound_offset(reg);
 	/* Intersecting with the old var_off might have improved our bounds
@@ -2979,7 +3004,11 @@ static void init_reg_state(struct bpf_verifier_env *env,
 
 static struct bpf_retval_range retval_range(s32 minval, s32 maxval)
 {
-	return (struct bpf_retval_range){ minval, maxval };
+	/*
+	 * return_32bit is set to false by default and set explicitly
+	 * by the caller when necessary.
+	 */
+	return (struct bpf_retval_range){ minval, maxval, false };
 }
 
 #define BPF_MAIN_FUNC (-1)
@@ -5151,7 +5180,7 @@ static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
 		 * Cleared it, since multiple rX += const are not supported.
 		 */
 		src_reg->id = 0;
-		src_reg->off = 0;
+		src_reg->delta = 0;
 	}
 
 	if (!src_reg->id && !tnum_is_const(src_reg->var_off))
@@ -5356,7 +5385,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
  * tracks the effects of the write, considering that each stack slot in the
  * dynamic range is potentially written to.
  *
- * 'off' includes 'regno->off'.
  * 'value_regno' can be -1, meaning that an unknown value is being written to
  * the stack.
  *
@@ -5778,7 +5806,6 @@ static int check_stack_read(struct bpf_verifier_env *env,
  * check_stack_write_var_off.
  *
  * 'ptr_regno' is the register used as a pointer into the stack.
- * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
  * 'value_regno' is the register whose value we're writing to the stack. It can
  * be -1, meaning that we're not writing from a register.
  *
@@ -5815,14 +5842,14 @@ static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
 	u32 cap = bpf_map_flags_to_cap(map);
 
 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
-		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
-			map->value_size, off, size);
+		verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n",
+			map->value_size, reg->smin_value + off, size);
 		return -EACCES;
 	}
 
 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
-		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
-			map->value_size, off, size);
+		verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n",
+			map->value_size, reg->smin_value + off, size);
 		return -EACCES;
 	}
 
@@ -5929,24 +5956,24 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env,
 	 * is only allowed in its original, unmodified form.
 	 */
 
-	if (reg->off < 0) {
-		verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
-			reg_type_str(env, reg->type), regno, reg->off);
+	if (!tnum_is_const(reg->var_off)) {
+		char tn_buf[48];
+
+		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
+		verbose(env, "variable %s access var_off=%s disallowed\n",
+			reg_type_str(env, reg->type), tn_buf);
 		return -EACCES;
 	}
 
-	if (!fixed_off_ok && reg->off) {
-		verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
-			reg_type_str(env, reg->type), regno, reg->off);
+	if (reg->smin_value < 0) {
+		verbose(env, "negative offset %s ptr R%d off=%lld disallowed\n",
+			reg_type_str(env, reg->type), regno, reg->var_off.value);
 		return -EACCES;
 	}
 
-	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
-		char tn_buf[48];
-
-		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-		verbose(env, "variable %s access var_off=%s disallowed\n",
-			reg_type_str(env, reg->type), tn_buf);
+	if (!fixed_off_ok && reg->var_off.value != 0) {
+		verbose(env, "dereference of modified %s ptr R%d off=%lld disallowed\n",
+			reg_type_str(env, reg->type), regno, reg->var_off.value);
 		return -EACCES;
 	}
 
@@ -5988,14 +6015,14 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
 	/* For ref_ptr case, release function check should ensure we get one
 	 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
 	 * normal store of unreferenced kptr, we must ensure var_off is zero.
-	 * Since ref_ptr cannot be accessed directly by BPF insns, checks for
-	 * reg->off and reg->ref_obj_id are not needed here.
+	 * Since ref_ptr cannot be accessed directly by BPF insns, check for
+	 * reg->ref_obj_id is not needed here.
 	 */
 	if (__check_ptr_off_reg(env, reg, regno, true))
 		return -EACCES;
 
 	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
-	 * we also need to take into account the reg->off.
+	 * we also need to take into account the reg->var_off.
 	 *
 	 * We want to support cases like:
 	 *
@@ -6006,19 +6033,19 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
 	 *
 	 * struct foo *v;
 	 * v = func();	      // PTR_TO_BTF_ID
-	 * val->foo = v;      // reg->off is zero, btf and btf_id match type
-	 * val->bar = &v->br; // reg->off is still zero, but we need to retry with
+	 * val->foo = v;      // reg->var_off is zero, btf and btf_id match type
+	 * val->bar = &v->br; // reg->var_off is still zero, but we need to retry with
 	 *                    // first member type of struct after comparison fails
-	 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
+	 * val->baz = &v->bz; // reg->var_off is non-zero, so struct needs to be walked
 	 *                    // to match type
 	 *
-	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
+	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->var_off
 	 * is zero. We must also ensure that btf_struct_ids_match does not walk
 	 * the struct to match type against first member of struct, i.e. reject
 	 * second case from above. Hence, when type is BPF_KPTR_REF, we set
 	 * strict mode to true for type match.
 	 */
-	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
+	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value,
 				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
 				  kptr_field->type != BPF_KPTR_UNREF))
 		goto bad_type;
@@ -6327,27 +6354,14 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
 	struct bpf_reg_state *reg = reg_state(env, regno);
 	int err;
 
-	/* We may have added a variable offset to the packet pointer; but any
-	 * reg->range we have comes after that.  We are only checking the fixed
-	 * offset.
-	 */
-
-	/* We don't allow negative numbers, because we aren't tracking enough
-	 * detail to prove they're safe.
-	 */
-	if (reg->smin_value < 0) {
-		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
-			regno);
-		return -EACCES;
+	if (reg->range < 0) {
+		verbose(env, "R%d offset is outside of the packet\n", regno);
+		return -EINVAL;
 	}
 
-	err = reg->range < 0 ? -EINVAL :
-	      __check_mem_access(env, regno, off, size, reg->range,
-				 zero_size_allowed);
-	if (err) {
-		verbose(env, "R%d offset is outside of the packet\n", regno);
+	err = check_mem_region_access(env, regno, off, size, reg->range, zero_size_allowed);
+	if (err)
 		return err;
-	}
 
 	/* __check_mem_access has made sure "off + size - 1" is within u16.
 	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
@@ -6359,7 +6373,7 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
 		max_t(u32, env->prog->aux->max_pkt_offset,
 		      off + reg->umax_value + size - 1);
 
-	return err;
+	return 0;
 }
 
 /* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
@@ -6576,14 +6590,14 @@ static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
 	 */
 	ip_align = 2;
 
-	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
+	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + off));
 	if (!tnum_is_aligned(reg_off, size)) {
 		char tn_buf[48];
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 		verbose(env,
-			"misaligned packet access off %d+%s+%d+%d size %d\n",
-			ip_align, tn_buf, reg->off, off, size);
+			"misaligned packet access off %d+%s+%d size %d\n",
+			ip_align, tn_buf, off, size);
 		return -EACCES;
 	}
 
@@ -6601,13 +6615,13 @@ static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
 	if (!strict || size == 1)
 		return 0;
 
-	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
+	reg_off = tnum_add(reg->var_off, tnum_const(off));
 	if (!tnum_is_aligned(reg_off, size)) {
 		char tn_buf[48];
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
-			pointer_desc, tn_buf, reg->off, off, size);
+		verbose(env, "misaligned %saccess off %s+%d size %d\n",
+			pointer_desc, tn_buf, off, size);
 		return -EACCES;
 	}
 
@@ -6710,22 +6724,30 @@ static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
 	return round_up(max_t(u32, stack_depth, 1), 32);
 }
 
+/* temporary state used for call frame depth calculation */
+struct bpf_subprog_call_depth_info {
+	int ret_insn; /* caller instruction where we return to. */
+	int caller; /* caller subprogram idx */
+	int frame; /* # of consecutive static call stack frames on top of stack */
+};
+
 /* starting from main bpf function walk all instructions of the function
  * and recursively walk all callees that given function can call.
  * Ignore jump and exit insns.
- * Since recursion is prevented by check_cfg() this algorithm
- * only needs a local stack of MAX_CALL_FRAMES to remember callsites
  */
 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
+					 struct bpf_subprog_call_depth_info *dinfo,
 					 bool priv_stack_supported)
 {
 	struct bpf_subprog_info *subprog = env->subprog_info;
 	struct bpf_insn *insn = env->prog->insnsi;
 	int depth = 0, frame = 0, i, subprog_end, subprog_depth;
 	bool tail_call_reachable = false;
-	int ret_insn[MAX_CALL_FRAMES];
-	int ret_prog[MAX_CALL_FRAMES];
-	int j;
+	int total;
+	int tmp;
+
+	/* no caller idx */
+	dinfo[idx].caller = -1;
 
 	i = subprog[idx].start;
 	if (!priv_stack_supported)
@@ -6777,8 +6799,12 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	} else {
 		depth += subprog_depth;
 		if (depth > MAX_BPF_STACK) {
+			total = 0;
+			for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller)
+				total++;
+
 			verbose(env, "combined stack size of %d calls is %d. Too large\n",
-				frame + 1, depth);
+				total, depth);
 			return -EACCES;
 		}
 	}
@@ -6792,10 +6818,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 
 			if (!is_bpf_throw_kfunc(insn + i))
 				continue;
-			if (subprog[idx].is_cb)
-				err = true;
-			for (int c = 0; c < frame && !err; c++) {
-				if (subprog[ret_prog[c]].is_cb) {
+			for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
+				if (subprog[tmp].is_cb) {
 					err = true;
 					break;
 				}
@@ -6811,8 +6835,6 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
 			continue;
 		/* remember insn and function to return to */
-		ret_insn[frame] = i + 1;
-		ret_prog[frame] = idx;
 
 		/* find the callee */
 		next_insn = i + insn[i].imm + 1;
@@ -6832,7 +6854,16 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 				return -EINVAL;
 			}
 		}
+
+		/* store caller info for after we return from callee */
+		dinfo[idx].frame = frame;
+		dinfo[idx].ret_insn = i + 1;
+
+		/* push caller idx into callee's dinfo */
+		dinfo[sidx].caller = idx;
+
 		i = next_insn;
+
 		idx = sidx;
 		if (!priv_stack_supported)
 			subprog[idx].priv_stack_mode = NO_PRIV_STACK;
@@ -6840,7 +6871,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 		if (subprog[idx].has_tail_call)
 			tail_call_reachable = true;
 
-		frame++;
+		frame = subprog_is_global(env, idx) ? 0 : frame + 1;
 		if (frame >= MAX_CALL_FRAMES) {
 			verbose(env, "the call stack of %d frames is too deep !\n",
 				frame);
@@ -6854,12 +6885,12 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
 	 */
 	if (tail_call_reachable)
-		for (j = 0; j < frame; j++) {
-			if (subprog[ret_prog[j]].is_exception_cb) {
+		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
+			if (subprog[tmp].is_exception_cb) {
 				verbose(env, "cannot tail call within exception cb\n");
 				return -EINVAL;
 			}
-			subprog[ret_prog[j]].tail_call_reachable = true;
+			subprog[tmp].tail_call_reachable = true;
 		}
 	if (subprog[0].tail_call_reachable)
 		env->prog->aux->tail_call_reachable = true;
@@ -6867,23 +6898,33 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	/* end of for() loop means the last insn of the 'subprog'
 	 * was reached. Doesn't matter whether it was JA or EXIT
 	 */
-	if (frame == 0)
+	if (frame == 0 && dinfo[idx].caller < 0)
 		return 0;
 	if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
 		depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
-	frame--;
-	i = ret_insn[frame];
-	idx = ret_prog[frame];
+
+	/* pop caller idx from callee */
+	idx = dinfo[idx].caller;
+
+	/* retrieve caller state from its frame */
+	frame = dinfo[idx].frame;
+	i = dinfo[idx].ret_insn;
+
 	goto continue_func;
 }
 
 static int check_max_stack_depth(struct bpf_verifier_env *env)
 {
 	enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
+	struct bpf_subprog_call_depth_info *dinfo;
 	struct bpf_subprog_info *si = env->subprog_info;
 	bool priv_stack_supported;
 	int ret;
 
+	dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT);
+	if (!dinfo)
+		return -ENOMEM;
+
 	for (int i = 0; i < env->subprog_cnt; i++) {
 		if (si[i].has_tail_call) {
 			priv_stack_mode = NO_PRIV_STACK;
@@ -6905,9 +6946,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
 	for (int i = env->subprog_cnt - 1; i >= 0; i--) {
 		if (!i || si[i].is_async_cb) {
 			priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
-			ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
-			if (ret < 0)
+			ret = check_max_stack_depth_subprog(env, i, dinfo,
+					priv_stack_supported);
+			if (ret < 0) {
+				kvfree(dinfo);
 				return ret;
+			}
 		}
 	}
 
@@ -6918,6 +6962,8 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
 		}
 	}
 
+	kvfree(dinfo);
+
 	return 0;
 }
 
@@ -6945,7 +6991,7 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
 			regno, buf_info, off, size);
 		return -EACCES;
 	}
-	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
+	if (!tnum_is_const(reg->var_off)) {
 		char tn_buf[48];
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
@@ -6968,8 +7014,8 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env,
 	if (err)
 		return err;
 
-	if (off + size > env->prog->aux->max_tp_access)
-		env->prog->aux->max_tp_access = off + size;
+	env->prog->aux->max_tp_access = max(reg->var_off.value + off + size,
+					    env->prog->aux->max_tp_access);
 
 	return 0;
 }
@@ -6987,8 +7033,7 @@ static int check_buffer_access(struct bpf_verifier_env *env,
 	if (err)
 		return err;
 
-	if (off + size > *max_access)
-		*max_access = off + size;
+	*max_access = max(reg->var_off.value + off + size, *max_access);
 
 	return 0;
 }
@@ -7381,13 +7426,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 			tname);
 		return -EINVAL;
 	}
-	if (off < 0) {
-		verbose(env,
-			"R%d is ptr_%s invalid negative access: off=%d\n",
-			regno, tname, off);
-		return -EACCES;
-	}
-	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
+
+	if (!tnum_is_const(reg->var_off)) {
 		char tn_buf[48];
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
@@ -7397,6 +7437,15 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
+	off += reg->var_off.value;
+
+	if (off < 0) {
+		verbose(env,
+			"R%d is ptr_%s invalid negative access: off=%d\n",
+			regno, tname, off);
+		return -EACCES;
+	}
+
 	if (reg->type & MEM_USER) {
 		verbose(env,
 			"R%d is ptr_%s access user memory: off=%d\n",
@@ -7643,8 +7692,8 @@ static int check_stack_access_within_bounds(
 
 	if (err) {
 		if (tnum_is_const(reg->var_off)) {
-			verbose(env, "invalid%s stack R%d off=%d size=%d\n",
-				err_extra, regno, off, access_size);
+			verbose(env, "invalid%s stack R%d off=%lld size=%d\n",
+				err_extra, regno, min_off, access_size);
 		} else {
 			char tn_buf[48];
 
@@ -7690,14 +7739,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 	if (size < 0)
 		return size;
 
-	/* alignment checks will add in reg->off themselves */
 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
 	if (err)
 		return err;
 
-	/* for access checks, reg->off is just part of off */
-	off += reg->off;
-
 	if (reg->type == PTR_TO_MAP_KEY) {
 		if (t == BPF_WRITE) {
 			verbose(env, "write to change key R%d not allowed\n", regno);
@@ -7794,6 +7839,11 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
 			mark_reg_unknown(env, regs, value_regno);
 	} else if (reg->type == PTR_TO_CTX) {
+		/*
+		 * Program types that don't rewrite ctx accesses can safely
+		 * dereference ctx pointers with fixed offsets.
+		 */
+		bool fixed_off_ok = !env->ops->convert_ctx_access;
 		struct bpf_retval_range range;
 		struct bpf_insn_access_aux info = {
 			.reg_type = SCALAR_VALUE,
@@ -7807,10 +7857,16 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 			return -EACCES;
 		}
 
-		err = check_ptr_off_reg(env, reg, regno);
+		err = __check_ptr_off_reg(env, reg, regno, fixed_off_ok);
 		if (err < 0)
 			return err;
 
+		/*
+		 * Fold the register's constant offset into the insn offset so
+		 * that is_valid_access() sees the true effective offset.
+		 */
+		if (fixed_off_ok)
+			off += reg->var_off.value;
 		err = check_ctx_access(env, insn_idx, off, size, t, &info);
 		if (err)
 			verbose_linfo(env, insn_idx, "; ");
@@ -8176,8 +8232,6 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
  * on the access type and privileges, that all elements of the stack are
  * initialized.
  *
- * 'off' includes 'regno->off', but not its dynamic part (if any).
- *
  * All registers that have been spilled on the stack in the slots within the
  * read offsets are marked as read.
  */
@@ -8338,7 +8392,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 	switch (base_type(reg->type)) {
 	case PTR_TO_PACKET:
 	case PTR_TO_PACKET_META:
-		return check_packet_access(env, regno, reg->off, access_size,
+		return check_packet_access(env, regno, 0, access_size,
 					   zero_size_allowed);
 	case PTR_TO_MAP_KEY:
 		if (access_type == BPF_WRITE) {
@@ -8346,12 +8400,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 				reg_type_str(env, reg->type));
 			return -EACCES;
 		}
-		return check_mem_region_access(env, regno, reg->off, access_size,
+		return check_mem_region_access(env, regno, 0, access_size,
 					       reg->map_ptr->key_size, false);
 	case PTR_TO_MAP_VALUE:
-		if (check_map_access_type(env, regno, reg->off, access_size, access_type))
+		if (check_map_access_type(env, regno, 0, access_size, access_type))
 			return -EACCES;
-		return check_map_access(env, regno, reg->off, access_size,
+		return check_map_access(env, regno, 0, access_size,
 					zero_size_allowed, ACCESS_HELPER);
 	case PTR_TO_MEM:
 		if (type_is_rdonly_mem(reg->type)) {
@@ -8361,7 +8415,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 				return -EACCES;
 			}
 		}
-		return check_mem_region_access(env, regno, reg->off,
+		return check_mem_region_access(env, regno, 0,
 					       access_size, reg->mem_size,
 					       zero_size_allowed);
 	case PTR_TO_BUF:
@@ -8376,16 +8430,16 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 		} else {
 			max_access = &env->prog->aux->max_rdwr_access;
 		}
-		return check_buffer_access(env, reg, regno, reg->off,
+		return check_buffer_access(env, reg, regno, 0,
 					   access_size, zero_size_allowed,
 					   max_access);
 	case PTR_TO_STACK:
 		return check_stack_range_initialized(
 				env,
-				regno, reg->off, access_size,
+				regno, 0, access_size,
 				zero_size_allowed, access_type, meta);
 	case PTR_TO_BTF_ID:
-		return check_ptr_to_btf_access(env, regs, regno, reg->off,
+		return check_ptr_to_btf_access(env, regs, regno, 0,
 					       access_size, BPF_READ, -1);
 	case PTR_TO_CTX:
 		/* in case the function doesn't know how to access the context,
@@ -8597,9 +8651,9 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)
 		return -EINVAL;
 	}
 	spin_lock_off = is_res_lock ? rec->res_spin_lock_off : rec->spin_lock_off;
-	if (spin_lock_off != val + reg->off) {
+	if (spin_lock_off != val) {
 		verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n",
-			val + reg->off, lock_str, spin_lock_off);
+			val, lock_str, spin_lock_off);
 		return -EINVAL;
 	}
 	if (is_lock) {
@@ -8714,9 +8768,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,
 		verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);
 		return -EINVAL;
 	}
-	if (field_off != val + reg->off) {
+	if (field_off != val) {
 		verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n",
-			val + reg->off, struct_name, field_off);
+			val, struct_name, field_off);
 		return -EINVAL;
 	}
 	if (map_desc->ptr) {
@@ -8784,7 +8838,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno,
 		return -EINVAL;
 	}
 
-	kptr_off = reg->off + reg->var_off.value;
+	kptr_off = reg->var_off.value;
 	kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR);
 	if (!kptr_field) {
 		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
@@ -9383,7 +9437,9 @@ static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE }
 static const struct bpf_reg_types kptr_xchg_dest_types = {
 	.types = {
 		PTR_TO_MAP_VALUE,
-		PTR_TO_BTF_ID | MEM_ALLOC
+		PTR_TO_BTF_ID | MEM_ALLOC,
+		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF,
+		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU,
 	}
 };
 static const struct bpf_reg_types dynptr_types = {
@@ -9427,7 +9483,7 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 	struct bpf_reg_state *reg = reg_state(env, regno);
 	enum bpf_reg_type expected, type = reg->type;
 	const struct bpf_reg_types *compatible;
-	int i, j;
+	int i, j, err;
 
 	compatible = compatible_reg_types[base_type(arg_type)];
 	if (!compatible) {
@@ -9530,8 +9586,12 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 				return -EACCES;
 			}
 
-			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
-						  btf_vmlinux, *arg_btf_id,
+			err = __check_ptr_off_reg(env, reg, regno, true);
+			if (err)
+				return err;
+
+			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,
+						  reg->var_off.value, btf_vmlinux, *arg_btf_id,
 						  strict_type_match)) {
 				verbose(env, "R%d is of type %s but %s is expected\n",
 					regno, btf_type_name(reg->btf, reg->btf_id),
@@ -9543,6 +9603,8 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 	}
 	case PTR_TO_BTF_ID | MEM_ALLOC:
 	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
+	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
+	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
 		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
 		    meta->func_id != BPF_FUNC_kptr_xchg) {
 			verifier_bug(env, "unimplemented handling of MEM_ALLOC");
@@ -9609,12 +9671,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env,
 		 * because fixed_off_ok is false, but checking here allows us
 		 * to give the user a better error message.
 		 */
-		if (reg->off) {
+		if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) {
 			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
 				regno);
 			return -EINVAL;
 		}
-		return __check_ptr_off_reg(env, reg, regno, false);
 	}
 
 	switch (type) {
@@ -9711,7 +9772,7 @@ static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
 	if (reg->type == CONST_PTR_TO_DYNPTR)
 		return reg->dynptr.type;
 
-	spi = __get_spi(reg->off);
+	spi = __get_spi(reg->var_off.value);
 	if (spi < 0) {
 		verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
 		return BPF_DYNPTR_TYPE_INVALID;
@@ -9752,13 +9813,13 @@ static int check_reg_const_str(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
-	err = check_map_access(env, regno, reg->off,
-			       map->value_size - reg->off, false,
+	err = check_map_access(env, regno, 0,
+			       map->value_size - reg->var_off.value, false,
 			       ACCESS_HELPER);
 	if (err)
 		return err;
 
-	map_off = reg->off + reg->var_off.value;
+	map_off = reg->var_off.value;
 	err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
 	if (err) {
 		verbose(env, "direct value access on string failed\n");
@@ -9795,7 +9856,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env,
 	if (!tnum_is_const(key->var_off))
 		return -EOPNOTSUPP;
 
-	stack_off = key->off + key->var_off.value;
+	stack_off = key->var_off.value;
 	slot = -stack_off - 1;
 	spi = slot / BPF_REG_SIZE;
 	off = slot % BPF_REG_SIZE;
@@ -10911,9 +10972,11 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		subprog_aux(env, subprog)->called = true;
 		clear_caller_saved_regs(env, caller->regs);
 
-		/* All global functions return a 64-bit SCALAR_VALUE */
-		mark_reg_unknown(env, caller->regs, BPF_REG_0);
-		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
+		/* All non-void global functions return a 64-bit SCALAR_VALUE. */
+		if (!subprog_returns_void(env, subprog)) {
+			mark_reg_unknown(env, caller->regs, BPF_REG_0);
+			caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
+		}
 
 		/* continue with next insn after call */
 		return 0;
@@ -11127,7 +11190,8 @@ static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
 	 */
 	struct btf_field *field;
 
-	field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off,
+	field = reg_find_field_offset(&caller->regs[BPF_REG_1],
+				      caller->regs[BPF_REG_1].var_off.value,
 				      BPF_RB_ROOT);
 	if (!field || !field->graph_root.value_btf_id)
 		return -EFAULT;
@@ -11200,10 +11264,9 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
 	return is_rbtree_lock_required_kfunc(kfunc_btf_id);
 }
 
-static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg,
-				bool return_32bit)
+static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
 {
-	if (return_32bit)
+	if (range.return_32bit)
 		return range.minval <= reg->s32_min_value && reg->s32_max_value <= range.maxval;
 	else
 		return range.minval <= reg->smin_value && reg->smax_value <= range.maxval;
@@ -11247,7 +11310,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 			return err;
 
 		/* enforce R0 return value range, and bpf_callback_t returns 64bit */
-		if (!retval_range_within(callee->callback_ret_range, r0, false)) {
+		if (!retval_range_within(callee->callback_ret_range, r0)) {
 			verbose_invalid_scalar(env, r0, callee->callback_ret_range,
 					       "At callback return", "R0");
 			return -EINVAL;
@@ -11503,7 +11566,7 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
 	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
 	 * and map_direct_value_addr is set.
 	 */
-	fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
+	fmt_map_off = fmt_reg->var_off.value;
 	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
 						  fmt_map_off);
 	if (err) {
@@ -12809,13 +12872,12 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
 	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
 		strict_type_match = true;
 
-	WARN_ON_ONCE(is_kfunc_release(meta) &&
-		     (reg->off || !tnum_is_const(reg->var_off) ||
-		      reg->var_off.value));
+	WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off));
 
 	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
 	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
-	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match);
+	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value,
+					   meta->btf, ref_id, strict_type_match);
 	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
 	 * actually use it -- it must cast to the underlying type. So we allow
 	 * caller to pass in the underlying type.
@@ -13187,7 +13249,7 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
 	}
 
 	rec = reg_btf_record(reg);
-	head_off = reg->off + reg->var_off.value;
+	head_off = reg->var_off.value;
 	field = btf_record_find(rec, head_off, head_field_type);
 	if (!field) {
 		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
@@ -13254,7 +13316,7 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
 		return -EINVAL;
 	}
 
-	node_off = reg->off + reg->var_off.value;
+	node_off = reg->var_off.value;
 	field = reg_find_field_offset(reg, node_off, node_field_type);
 	if (!field) {
 		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
@@ -14282,7 +14344,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	    meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
 	    meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
 		release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
-		insn_aux->insert_off = regs[BPF_REG_2].off;
+		insn_aux->insert_off = regs[BPF_REG_2].var_off.value;
 		insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
 		err = ref_convert_owning_non_owning(env, release_ref_obj_id);
 		if (err) {
@@ -14480,9 +14542,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	return 0;
 }
 
-static bool check_reg_sane_offset(struct bpf_verifier_env *env,
-				  const struct bpf_reg_state *reg,
-				  enum bpf_reg_type type)
+static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
+					 const struct bpf_reg_state *reg,
+					 enum bpf_reg_type type)
 {
 	bool known = tnum_is_const(reg->var_off);
 	s64 val = reg->var_off.value;
@@ -14494,12 +14556,6 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env,
 		return false;
 	}
 
-	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
-		verbose(env, "%s pointer offset %d is not allowed\n",
-			reg_type_str(env, type), reg->off);
-		return false;
-	}
-
 	if (smin == S64_MIN) {
 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
 			reg_type_str(env, type));
@@ -14515,6 +14571,29 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env,
 	return true;
 }
 
+static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env,
+				      const struct bpf_reg_state *reg,
+				      enum bpf_reg_type type)
+{
+	bool known = tnum_is_const(reg->var_off);
+	s64 val = reg->var_off.value;
+	s64 smin = reg->smin_value;
+
+	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
+		verbose(env, "%s pointer offset %lld is not allowed\n",
+			reg_type_str(env, type), val);
+		return false;
+	}
+
+	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
+		verbose(env, "%s pointer offset %lld is not allowed\n",
+			reg_type_str(env, type), smin);
+		return false;
+	}
+
+	return true;
+}
+
 enum {
 	REASON_BOUNDS	= -1,
 	REASON_TYPE	= -2,
@@ -14536,13 +14615,11 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
 		 * currently prohibited for unprivileged.
 		 */
 		max = MAX_BPF_STACK + mask_to_left;
-		ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
+		ptr_limit = -ptr_reg->var_off.value;
 		break;
 	case PTR_TO_MAP_VALUE:
 		max = ptr_reg->map_ptr->value_size;
-		ptr_limit = (mask_to_left ?
-			     ptr_reg->smin_value :
-			     ptr_reg->umax_value) + ptr_reg->off;
+		ptr_limit = mask_to_left ? ptr_reg->smin_value : ptr_reg->umax_value;
 		break;
 	default:
 		return REASON_TYPE;
@@ -14773,9 +14850,6 @@ static int sanitize_err(struct bpf_verifier_env *env,
  * Variable offset is prohibited for unprivileged mode for simplicity since it
  * requires corresponding support in Spectre masking for stack ALU.  See also
  * retrieve_ptr_limit().
- *
- *
- * 'off' includes 'reg->off'.
  */
 static int check_stack_access_for_ptr_arithmetic(
 				struct bpf_verifier_env *env,
@@ -14816,11 +14890,11 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env,
 	switch (dst_reg->type) {
 	case PTR_TO_STACK:
 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
-					dst_reg->off + dst_reg->var_off.value))
+							  dst_reg->var_off.value))
 			return -EACCES;
 		break;
 	case PTR_TO_MAP_VALUE:
-		if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
+		if (check_map_access(env, dst, 0, 1, false, ACCESS_HELPER)) {
 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
 				"prohibited for !root\n", dst);
 			return -EACCES;
@@ -14928,8 +15002,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 	dst_reg->type = ptr_reg->type;
 	dst_reg->id = ptr_reg->id;
 
-	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
-	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
+	if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) ||
+	    !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type))
 		return -EINVAL;
 
 	/* pointer types do not carry 32-bit bounds at the moment. */
@@ -14944,23 +15018,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 
 	switch (opcode) {
 	case BPF_ADD:
-		/* We can take a fixed offset as long as it doesn't overflow
-		 * the s32 'off' field
-		 */
-		if (known && (ptr_reg->off + smin_val ==
-			      (s64)(s32)(ptr_reg->off + smin_val))) {
-			/* pointer += K.  Accumulate it into fixed offset */
-			dst_reg->smin_value = smin_ptr;
-			dst_reg->smax_value = smax_ptr;
-			dst_reg->umin_value = umin_ptr;
-			dst_reg->umax_value = umax_ptr;
-			dst_reg->var_off = ptr_reg->var_off;
-			dst_reg->off = ptr_reg->off + smin_val;
-			dst_reg->raw = ptr_reg->raw;
-			break;
-		}
-		/* A new variable offset is created.  Note that off_reg->off
-		 * == 0, since it's a scalar.
+		/*
 		 * dst_reg gets the pointer type and since some positive
 		 * integer value was added to the pointer, give it a new 'id'
 		 * if it's a PTR_TO_PACKET.
@@ -14979,9 +15037,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 			dst_reg->umax_value = U64_MAX;
 		}
 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
-		dst_reg->off = ptr_reg->off;
 		dst_reg->raw = ptr_reg->raw;
-		if (reg_is_pkt_pointer(ptr_reg)) {
+		if (!known && reg_is_pkt_pointer(ptr_reg)) {
 			dst_reg->id = ++env->id_gen;
 			/* something was added to pkt_ptr, set range to zero */
 			memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
@@ -15003,19 +15060,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 				dst);
 			return -EACCES;
 		}
-		if (known && (ptr_reg->off - smin_val ==
-			      (s64)(s32)(ptr_reg->off - smin_val))) {
-			/* pointer -= K.  Subtract it from fixed offset */
-			dst_reg->smin_value = smin_ptr;
-			dst_reg->smax_value = smax_ptr;
-			dst_reg->umin_value = umin_ptr;
-			dst_reg->umax_value = umax_ptr;
-			dst_reg->var_off = ptr_reg->var_off;
-			dst_reg->id = ptr_reg->id;
-			dst_reg->off = ptr_reg->off - smin_val;
-			dst_reg->raw = ptr_reg->raw;
-			break;
-		}
 		/* A new variable offset is created.  If the subtrahend is known
 		 * nonnegative, then any reg->range we had before is still good.
 		 */
@@ -15035,9 +15079,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 			dst_reg->umax_value = umax_ptr - umin_val;
 		}
 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
-		dst_reg->off = ptr_reg->off;
 		dst_reg->raw = ptr_reg->raw;
-		if (reg_is_pkt_pointer(ptr_reg)) {
+		if (!known && reg_is_pkt_pointer(ptr_reg)) {
 			dst_reg->id = ++env->id_gen;
 			/* something was added to pkt_ptr, set range to zero */
 			if (smin_val < 0)
@@ -15058,7 +15101,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
-	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
+	if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type))
 		return -EINVAL;
 	reg_bounds_sync(dst_reg);
 	bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
@@ -15910,6 +15953,13 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins
 	/* Apply bswap if alu64 or switch between big-endian and little-endian machines */
 	bool need_bswap = alu64 || (to_le == is_big_endian);
 
+	/*
+	 * If the register is mutated, manually reset its scalar ID to break
+	 * any existing ties and avoid incorrect bounds propagation.
+	 */
+	if (need_bswap || insn->imm == 16 || insn->imm == 32)
+		dst_reg->id = 0;
+
 	if (need_bswap) {
 		if (insn->imm == 16)
 			dst_reg->var_off = tnum_bswap16(dst_reg->var_off);
@@ -16309,14 +16359,14 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 			 * we cannot accumulate another val into rx->off.
 			 */
 clear_id:
-			dst_reg->off = 0;
+			dst_reg->delta = 0;
 			dst_reg->id = 0;
 		} else {
 			if (alu32)
 				dst_reg->id |= BPF_ADD_CONST32;
 			else
 				dst_reg->id |= BPF_ADD_CONST64;
-			dst_reg->off = off;
+			dst_reg->delta = off;
 		}
 	} else {
 		/*
@@ -16581,19 +16631,17 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	struct bpf_reg_state *reg;
 	int new_range;
 
-	if (dst_reg->off < 0 ||
-	    (dst_reg->off == 0 && range_right_open))
+	if (dst_reg->umax_value == 0 && range_right_open)
 		/* This doesn't give us any range */
 		return;
 
-	if (dst_reg->umax_value > MAX_PACKET_OFF ||
-	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
+	if (dst_reg->umax_value > MAX_PACKET_OFF)
 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
 		 * than pkt_end, but that's because it's also less than pkt.
 		 */
 		return;
 
-	new_range = dst_reg->off;
+	new_range = dst_reg->umax_value;
 	if (range_right_open)
 		new_range++;
 
@@ -16642,7 +16690,7 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	/* If our ids match, then we must have the same max_value.  And we
 	 * don't care about the other reg's fixed offset, since if it's too big
 	 * the range won't allow anything.
-	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
+	 * dst_reg->umax_value is known < MAX_PACKET_OFF, therefore it fits in a u16.
 	 */
 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
 		if (reg->type == type && reg->id == dst_reg->id)
@@ -17168,29 +17216,24 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state,
 {
 	if (type_may_be_null(reg->type) && reg->id == id &&
 	    (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
-		/* Old offset (both fixed and variable parts) should have been
-		 * known-zero, because we don't allow pointer arithmetic on
-		 * pointers that might be NULL. If we see this happening, don't
-		 * convert the register.
+		/* Old offset should have been known-zero, because we don't
+		 * allow pointer arithmetic on pointers that might be NULL.
+		 * If we see this happening, don't convert the register.
 		 *
 		 * But in some cases, some helpers that return local kptrs
-		 * advance offset for the returned pointer. In those cases, it
-		 * is fine to expect to see reg->off.
+		 * advance offset for the returned pointer. In those cases,
+		 * it is fine to expect to see reg->var_off.
 		 */
-		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0)))
-			return;
 		if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
-		    WARN_ON_ONCE(reg->off))
+		    WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0)))
 			return;
-
 		if (is_null) {
-			reg->type = SCALAR_VALUE;
 			/* We don't need id and ref_obj_id from this point
 			 * onwards anymore, thus we should better reset it,
 			 * so that state pruning has chances to take effect.
 			 */
-			reg->id = 0;
-			reg->ref_obj_id = 0;
+			__mark_reg_known_zero(reg);
+			reg->type = SCALAR_VALUE;
 
 			return;
 		}
@@ -17409,18 +17452,18 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s
 		if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
 			continue;
 		if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
-		    reg->off == known_reg->off) {
+		    reg->delta == known_reg->delta) {
 			s32 saved_subreg_def = reg->subreg_def;
 
 			copy_register_state(reg, known_reg);
 			reg->subreg_def = saved_subreg_def;
 		} else {
 			s32 saved_subreg_def = reg->subreg_def;
-			s32 saved_off = reg->off;
+			s32 saved_off = reg->delta;
 			u32 saved_id = reg->id;
 
 			fake_reg.type = SCALAR_VALUE;
-			__mark_reg_known(&fake_reg, (s64)reg->off - (s64)known_reg->off);
+			__mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta);
 
 			/* reg = known_reg; reg += delta */
 			copy_register_state(reg, known_reg);
@@ -17428,7 +17471,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s
 			 * Must preserve off, id and subreg_def flag,
 			 * otherwise another sync_linked_regs() will be incorrect.
 			 */
-			reg->off = saved_off;
+			reg->delta = saved_off;
 			reg->id = saved_id;
 			reg->subreg_def = saved_subreg_def;
 
@@ -17678,12 +17721,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	}
 
 	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
+	 * Also does the same detection for a register whose the value is
+	 * known to be 0.
 	 * NOTE: these optimizations below are related with pointer comparison
 	 *       which will never be JMP32.
 	 */
-	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
-	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
-	    type_may_be_null(dst_reg->type)) {
+	if (!is_jmp32 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
+	    type_may_be_null(dst_reg->type) &&
+	    ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) ||
+	     (BPF_SRC(insn->code) == BPF_X && register_is_null(src_reg)))) {
 		/* Mark all identical registers in each branch as either
 		 * safe or unknown depending R == 0 or R != 0 conditional.
 		 */
@@ -17777,22 +17823,24 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
 	}
 
 	map = env->used_maps[aux->map_index];
-	dst_reg->map_ptr = map;
 
 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
 	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
 		if (map->map_type == BPF_MAP_TYPE_ARENA) {
 			__mark_reg_unknown(env, dst_reg);
+			dst_reg->map_ptr = map;
 			return 0;
 		}
+		__mark_reg_known(dst_reg, aux->map_off);
 		dst_reg->type = PTR_TO_MAP_VALUE;
-		dst_reg->off = aux->map_off;
+		dst_reg->map_ptr = map;
 		WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY &&
 			     map->max_entries != 1);
 		/* We want reg->id to be same (0) as map_value is not distinct */
 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
 		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
 		dst_reg->type = CONST_PTR_TO_MAP;
+		dst_reg->map_ptr = map;
 	} else {
 		verifier_bug(env, "unexpected src reg value for ldimm64");
 		return -EFAULT;
@@ -17898,104 +17946,39 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
 	return 0;
 }
 
-static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
+
+static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range)
 {
-	const char *exit_ctx = "At program exit";
-	struct tnum enforce_attach_type_range = tnum_unknown;
-	const struct bpf_prog *prog = env->prog;
-	struct bpf_reg_state *reg = reg_state(env, regno);
-	struct bpf_retval_range range = retval_range(0, 1);
 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
-	int err;
-	struct bpf_func_state *frame = env->cur_state->frame[0];
-	const bool is_subprog = frame->subprogno;
-	bool return_32bit = false;
-	const struct btf_type *reg_type, *ret_type = NULL;
-
-	/* LSM and struct_ops func-ptr's return type could be "void" */
-	if (!is_subprog || frame->in_exception_callback_fn) {
-		switch (prog_type) {
-		case BPF_PROG_TYPE_LSM:
-			if (prog->expected_attach_type == BPF_LSM_CGROUP)
-				/* See below, can be 0 or 0-1 depending on hook. */
-				break;
-			if (!prog->aux->attach_func_proto->type)
-				return 0;
-			break;
-		case BPF_PROG_TYPE_STRUCT_OPS:
-			if (!prog->aux->attach_func_proto->type)
-				return 0;
 
-			if (frame->in_exception_callback_fn)
-				break;
+	/* Default return value range. */
+	*range = retval_range(0, 1);
 
-			/* Allow a struct_ops program to return a referenced kptr if it
-			 * matches the operator's return type and is in its unmodified
-			 * form. A scalar zero (i.e., a null pointer) is also allowed.
-			 */
-			reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
-			ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
-							prog->aux->attach_func_proto->type,
-							NULL);
-			if (ret_type && ret_type == reg_type && reg->ref_obj_id)
-				return __check_ptr_off_reg(env, reg, regno, false);
+	switch (prog_type) {
+	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
+		switch (env->prog->expected_attach_type) {
+		case BPF_CGROUP_UDP4_RECVMSG:
+		case BPF_CGROUP_UDP6_RECVMSG:
+		case BPF_CGROUP_UNIX_RECVMSG:
+		case BPF_CGROUP_INET4_GETPEERNAME:
+		case BPF_CGROUP_INET6_GETPEERNAME:
+		case BPF_CGROUP_UNIX_GETPEERNAME:
+		case BPF_CGROUP_INET4_GETSOCKNAME:
+		case BPF_CGROUP_INET6_GETSOCKNAME:
+		case BPF_CGROUP_UNIX_GETSOCKNAME:
+			*range = retval_range(1, 1);
+			break;
+		case BPF_CGROUP_INET4_BIND:
+		case BPF_CGROUP_INET6_BIND:
+			*range = retval_range(0, 3);
 			break;
 		default:
 			break;
 		}
-	}
-
-	/* eBPF calling convention is such that R0 is used
-	 * to return the value from eBPF program.
-	 * Make sure that it's readable at this time
-	 * of bpf_exit, which means that program wrote
-	 * something into it earlier
-	 */
-	err = check_reg_arg(env, regno, SRC_OP);
-	if (err)
-		return err;
-
-	if (is_pointer_value(env, regno)) {
-		verbose(env, "R%d leaks addr as return value\n", regno);
-		return -EACCES;
-	}
-
-	if (frame->in_async_callback_fn) {
-		exit_ctx = "At async callback return";
-		range = frame->callback_ret_range;
-		goto enforce_retval;
-	}
-
-	if (is_subprog && !frame->in_exception_callback_fn) {
-		if (reg->type != SCALAR_VALUE) {
-			verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
-				regno, reg_type_str(env, reg->type));
-			return -EINVAL;
-		}
-		return 0;
-	}
-
-	switch (prog_type) {
-	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
-		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
-		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
-		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||
-		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
-		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
-		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||
-		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
-		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||
-		    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)
-			range = retval_range(1, 1);
-		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
-		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
-			range = retval_range(0, 3);
 		break;
 	case BPF_PROG_TYPE_CGROUP_SKB:
-		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
-			range = retval_range(0, 3);
-			enforce_attach_type_range = tnum_range(2, 3);
-		}
+		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS)
+			*range = retval_range(0, 3);
 		break;
 	case BPF_PROG_TYPE_CGROUP_SOCK:
 	case BPF_PROG_TYPE_SOCK_OPS:
@@ -18005,72 +17988,164 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
 		break;
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 		if (!env->prog->aux->attach_btf_id)
-			return 0;
-		range = retval_range(0, 0);
+			return false;
+		*range = retval_range(0, 0);
 		break;
 	case BPF_PROG_TYPE_TRACING:
 		switch (env->prog->expected_attach_type) {
 		case BPF_TRACE_FENTRY:
 		case BPF_TRACE_FEXIT:
 		case BPF_TRACE_FSESSION:
-			range = retval_range(0, 0);
+			*range = retval_range(0, 0);
 			break;
 		case BPF_TRACE_RAW_TP:
 		case BPF_MODIFY_RETURN:
-			return 0;
+			return false;
 		case BPF_TRACE_ITER:
-			break;
 		default:
-			return -ENOTSUPP;
+			break;
 		}
 		break;
 	case BPF_PROG_TYPE_KPROBE:
 		switch (env->prog->expected_attach_type) {
 		case BPF_TRACE_KPROBE_SESSION:
 		case BPF_TRACE_UPROBE_SESSION:
-			range = retval_range(0, 1);
 			break;
 		default:
-			return 0;
+			return false;
 		}
 		break;
 	case BPF_PROG_TYPE_SK_LOOKUP:
-		range = retval_range(SK_DROP, SK_PASS);
+		*range = retval_range(SK_DROP, SK_PASS);
 		break;
 
 	case BPF_PROG_TYPE_LSM:
 		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
 			/* no range found, any return value is allowed */
-			if (!get_func_retval_range(env->prog, &range))
-				return 0;
+			if (!get_func_retval_range(env->prog, range))
+				return false;
 			/* no restricted range, any return value is allowed */
-			if (range.minval == S32_MIN && range.maxval == S32_MAX)
-				return 0;
-			return_32bit = true;
+			if (range->minval == S32_MIN && range->maxval == S32_MAX)
+				return false;
+			range->return_32bit = true;
 		} else if (!env->prog->aux->attach_func_proto->type) {
 			/* Make sure programs that attach to void
 			 * hooks don't try to modify return value.
 			 */
-			range = retval_range(1, 1);
+			*range = retval_range(1, 1);
 		}
 		break;
 
 	case BPF_PROG_TYPE_NETFILTER:
-		range = retval_range(NF_DROP, NF_ACCEPT);
+		*range = retval_range(NF_DROP, NF_ACCEPT);
 		break;
 	case BPF_PROG_TYPE_STRUCT_OPS:
-		if (!ret_type)
-			return 0;
-		range = retval_range(0, 0);
+		*range = retval_range(0, 0);
 		break;
 	case BPF_PROG_TYPE_EXT:
 		/* freplace program can return anything as its return value
 		 * depends on the to-be-replaced kernel func or bpf program.
 		 */
 	default:
+		return false;
+	}
+
+	/* Continue calculating. */
+
+	return true;
+}
+
+static bool program_returns_void(struct bpf_verifier_env *env)
+{
+	const struct bpf_prog *prog = env->prog;
+	enum bpf_prog_type prog_type = prog->type;
+
+	switch (prog_type) {
+	case BPF_PROG_TYPE_LSM:
+		/* See return_retval_range, for BPF_LSM_CGROUP can be 0 or 0-1 depending on hook. */
+		if (prog->expected_attach_type != BPF_LSM_CGROUP &&
+		    !prog->aux->attach_func_proto->type)
+			return true;
+		break;
+	case BPF_PROG_TYPE_STRUCT_OPS:
+		if (!prog->aux->attach_func_proto->type)
+			return true;
+		break;
+	case BPF_PROG_TYPE_EXT:
+		/*
+		 * If the actual program is an extension, let it
+		 * return void - attaching will succeed only if the
+		 * program being replaced also returns void, and since
+		 * it has passed verification its actual type doesn't matter.
+		 */
+		if (subprog_returns_void(env, 0))
+			return true;
+		break;
+	default:
+		break;
+	}
+	return false;
+}
+
+static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
+{
+	const char *exit_ctx = "At program exit";
+	struct tnum enforce_attach_type_range = tnum_unknown;
+	const struct bpf_prog *prog = env->prog;
+	struct bpf_reg_state *reg = reg_state(env, regno);
+	struct bpf_retval_range range = retval_range(0, 1);
+	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
+	struct bpf_func_state *frame = env->cur_state->frame[0];
+	const struct btf_type *reg_type, *ret_type = NULL;
+	int err;
+
+	/* LSM and struct_ops func-ptr's return type could be "void" */
+	if (!frame->in_async_callback_fn && program_returns_void(env))
 		return 0;
+
+	if (prog_type == BPF_PROG_TYPE_STRUCT_OPS) {
+		/* Allow a struct_ops program to return a referenced kptr if it
+		 * matches the operator's return type and is in its unmodified
+		 * form. A scalar zero (i.e., a null pointer) is also allowed.
+		 */
+		reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
+		ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
+						prog->aux->attach_func_proto->type,
+						NULL);
+		if (ret_type && ret_type == reg_type && reg->ref_obj_id)
+			return __check_ptr_off_reg(env, reg, regno, false);
+	}
+
+	/* eBPF calling convention is such that R0 is used
+	 * to return the value from eBPF program.
+	 * Make sure that it's readable at this time
+	 * of bpf_exit, which means that program wrote
+	 * something into it earlier
+	 */
+	err = check_reg_arg(env, regno, SRC_OP);
+	if (err)
+		return err;
+
+	if (is_pointer_value(env, regno)) {
+		verbose(env, "R%d leaks addr as return value\n", regno);
+		return -EACCES;
+	}
+
+	if (frame->in_async_callback_fn) {
+		exit_ctx = "At async callback return";
+		range = frame->callback_ret_range;
+		goto enforce_retval;
 	}
 
+	if (prog_type == BPF_PROG_TYPE_STRUCT_OPS && !ret_type)
+		return 0;
+
+	if (prog_type == BPF_PROG_TYPE_CGROUP_SKB && (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS))
+		enforce_attach_type_range = tnum_range(2, 3);
+
+	if (!return_retval_range(env, &range))
+		return 0;
+
 enforce_retval:
 	if (reg->type != SCALAR_VALUE) {
 		verbose(env, "%s the register R%d is not a known value (%s)\n",
@@ -18082,10 +18157,9 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
 	if (err)
 		return err;
 
-	if (!retval_range_within(range, reg, return_32bit)) {
+	if (!retval_range_within(range, reg)) {
 		verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);
-		if (!is_subprog &&
-		    prog->expected_attach_type == BPF_LSM_CGROUP &&
+		if (prog->expected_attach_type == BPF_LSM_CGROUP &&
 		    prog_type == BPF_PROG_TYPE_LSM &&
 		    !prog->aux->attach_func_proto->type)
 			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
@@ -18098,6 +18172,33 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
 	return 0;
 }
 
+static int check_global_subprog_return_code(struct bpf_verifier_env *env)
+{
+	struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
+	struct bpf_func_state *cur_frame = cur_func(env);
+	int err;
+
+	if (subprog_returns_void(env, cur_frame->subprogno))
+		return 0;
+
+	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
+	if (err)
+		return err;
+
+	if (is_pointer_value(env, BPF_REG_0)) {
+		verbose(env, "R%d leaks addr as return value\n", BPF_REG_0);
+		return -EACCES;
+	}
+
+	if (reg->type != SCALAR_VALUE) {
+		verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
+			reg_type_str(env, reg->type));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
 {
 	struct bpf_subprog_info *subprog;
@@ -19734,7 +19835,7 @@ static void clear_singular_ids(struct bpf_verifier_env *env,
 			continue;
 		if (idset_cnt_get(idset, reg->id & ~BPF_ADD_CONST) == 1) {
 			reg->id = 0;
-			reg->off = 0;
+			reg->delta = 0;
 		}
 	}));
 }
@@ -19871,7 +19972,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
 			return false;
 
 		/* Both have offset linkage: offsets must match */
-		if ((rold->id & BPF_ADD_CONST) && rold->off != rcur->off)
+		if ((rold->id & BPF_ADD_CONST) && rold->delta != rcur->delta)
 			return false;
 
 		if (!check_scalar_ids(rold->id, rcur->id, idmap))
@@ -19901,11 +20002,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
 		 */
 		if (rold->range > rcur->range)
 			return false;
-		/* If the offsets don't match, we can't trust our alignment;
-		 * nor can we be sure that we won't fall out of range.
-		 */
-		if (rold->off != rcur->off)
-			return false;
 		/* id relations must be preserved */
 		if (!check_ids(rold->id, rcur->id, idmap))
 			return false;
@@ -19921,8 +20017,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
 		return true;
 	case PTR_TO_INSN:
 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
-			rold->off == rcur->off && range_within(rold, rcur) &&
-			tnum_in(rold->var_off, rcur->var_off);
+		       range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off);
 	default:
 		return regs_exact(rold, rcur, idmap);
 	}
@@ -20535,7 +20630,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 					 * so we can assume valid iter and reg state,
 					 * no need for extra (re-)validations
 					 */
-					spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
+					spi = __get_spi(iter_reg->var_off.value);
 					iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
 					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
 						loop = true;
@@ -20897,6 +20992,8 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
 				 bool *do_print_state,
 				 bool exception_exit)
 {
+	struct bpf_func_state *cur_frame = cur_func(env);
+
 	/* We must do check_reference_leak here before
 	 * prepare_func_exit to handle the case when
 	 * state->curframe > 0, it may be a callback function,
@@ -20930,7 +21027,21 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
 		return 0;
 	}
 
-	err = check_return_code(env, BPF_REG_0, "R0");
+	/*
+	 * Return from a regular global subprogram differs from return
+	 * from the main program or async/exception callback.
+	 * Main program exit implies return code restrictions
+	 * that depend on program type.
+	 * Exit from exception callback is equivalent to main program exit.
+	 * Exit from async callback implies return code restrictions
+	 * that depend on async scheduling mechanism.
+	 */
+	if (cur_frame->subprogno &&
+	    !cur_frame->in_async_callback_fn &&
+	    !cur_frame->in_exception_callback_fn)
+		err = check_global_subprog_return_code(env);
+	else
+		err = check_return_code(env, BPF_REG_0, "R0");
 	if (err)
 		return err;
 	return PROCESS_BPF_EXIT;
@@ -20942,19 +21053,16 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
 				       u32 *pmin_index, u32 *pmax_index)
 {
 	struct bpf_reg_state *reg = reg_state(env, regno);
-	u64 min_index, max_index;
+	u64 min_index = reg->umin_value;
+	u64 max_index = reg->umax_value;
 	const u32 size = 8;
 
-	if (check_add_overflow(reg->umin_value, reg->off, &min_index) ||
-		(min_index > (u64) U32_MAX * size)) {
-		verbose(env, "the sum of R%u umin_value %llu and off %u is too big\n",
-			     regno, reg->umin_value, reg->off);
+	if (min_index > (u64) U32_MAX * size) {
+		verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg->umin_value);
 		return -ERANGE;
 	}
-	if (check_add_overflow(reg->umax_value, reg->off, &max_index) ||
-		(max_index > (u64) U32_MAX * size)) {
-		verbose(env, "the sum of R%u umax_value %llu and off %u is too big\n",
-			     regno, reg->umax_value, reg->off);
+	if (max_index > (u64) U32_MAX * size) {
+		verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg->umax_value);
 		return -ERANGE;
 	}
 
@@ -21109,7 +21217,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 
 			if (env->cur_state->active_locks) {
 				if ((insn->src_reg == BPF_REG_0 &&
-				     insn->imm != BPF_FUNC_spin_unlock) ||
+				     insn->imm != BPF_FUNC_spin_unlock &&
+				     insn->imm != BPF_FUNC_kptr_xchg) ||
 				    (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
 				     (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) {
 					verbose(env,
@@ -24571,10 +24680,18 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 
 		if (subprog_is_exc_cb(env, subprog)) {
 			state->frame[0]->in_exception_callback_fn = true;
-			/* We have already ensured that the callback returns an integer, just
-			 * like all global subprogs. We need to determine it only has a single
-			 * scalar argument.
+
+			/*
+			 * Global functions are scalar or void, make sure
+			 * we return a scalar.
 			 */
+			if (subprog_returns_void(env, subprog)) {
+				verbose(env, "exception cb cannot return void\n");
+				ret = -EINVAL;
+				goto out;
+			}
+
+			/* Also ensure the callback only has a single scalar argument. */
 			if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) {
 				verbose(env, "exception cb only supports single integer argument\n");
 				ret = -EINVAL;
@@ -24881,14 +24998,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
 }
 #define SECURITY_PREFIX "security_"
 
-static int check_attach_modify_return(unsigned long addr, const char *func_name)
-{
-	if (within_error_injection_list(addr) ||
-	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
-		return 0;
-
-	return -EINVAL;
-}
+#ifdef CONFIG_FUNCTION_ERROR_INJECTION
 
 /* list of non-sleepable functions that are otherwise on
  * ALLOW_ERROR_INJECTION list
@@ -24911,6 +25021,75 @@ static int check_non_sleepable_error_inject(u32 btf_id)
 	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
 }
 
+static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name)
+{
+	/* fentry/fexit/fmod_ret progs can be sleepable if they are
+	 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
+	 */
+	if (!check_non_sleepable_error_inject(btf_id) &&
+	    within_error_injection_list(addr))
+		return 0;
+
+	return -EINVAL;
+}
+
+static int check_attach_modify_return(unsigned long addr, const char *func_name)
+{
+	if (within_error_injection_list(addr) ||
+	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
+		return 0;
+
+	return -EINVAL;
+}
+
+#else
+
+/* Unfortunately, the arch-specific prefixes are hard-coded in arch syscall code
+ * so we need to hard-code them, too. Ftrace has arch_syscall_match_sym_name()
+ * but that just compares two concrete function names.
+ */
+static bool has_arch_syscall_prefix(const char *func_name)
+{
+#if defined(__x86_64__)
+	return !strncmp(func_name, "__x64_", 6);
+#elif defined(__i386__)
+	return !strncmp(func_name, "__ia32_", 7);
+#elif defined(__s390x__)
+	return !strncmp(func_name, "__s390x_", 8);
+#elif defined(__aarch64__)
+	return !strncmp(func_name, "__arm64_", 8);
+#elif defined(__riscv)
+	return !strncmp(func_name, "__riscv_", 8);
+#elif defined(__powerpc__) || defined(__powerpc64__)
+	return !strncmp(func_name, "sys_", 4);
+#elif defined(__loongarch__)
+	return !strncmp(func_name, "sys_", 4);
+#else
+	return false;
+#endif
+}
+
+/* Without error injection, allow sleepable and fmod_ret progs on syscalls. */
+
+static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name)
+{
+	if (has_arch_syscall_prefix(func_name))
+		return 0;
+
+	return -EINVAL;
+}
+
+static int check_attach_modify_return(unsigned long addr, const char *func_name)
+{
+	if (has_arch_syscall_prefix(func_name) ||
+	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
+		return 0;
+
+	return -EINVAL;
+}
+
+#endif /* CONFIG_FUNCTION_ERROR_INJECTION */
+
 int bpf_check_attach_target(struct bpf_verifier_log *log,
 			    const struct bpf_prog *prog,
 			    const struct bpf_prog *tgt_prog,
@@ -25190,12 +25369,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 			ret = -EINVAL;
 			switch (prog->type) {
 			case BPF_PROG_TYPE_TRACING:
-
-				/* fentry/fexit/fmod_ret progs can be sleepable if they are
-				 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
-				 */
-				if (!check_non_sleepable_error_inject(btf_id) &&
-				    within_error_injection_list(addr))
+				if (!check_attach_sleepable(btf_id, addr, tname))
 					ret = 0;
 				/* fentry/fexit/fmod_ret progs can also be sleepable if they are
 				 * in the fmodret id set with the KF_SLEEPABLE flag.
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index 3bfe37693d68d..58797123b752f 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -184,11 +184,6 @@ extern bool cgrp_dfl_visible;
 	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&		\
 	     (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
 
-static inline bool cgroup_is_dead(const struct cgroup *cgrp)
-{
-	return !(cgrp->self.flags & CSS_ONLINE);
-}
-
 static inline bool notify_on_release(const struct cgroup *cgrp)
 {
 	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
@@ -222,7 +217,6 @@ static inline void get_css_set(struct css_set *cset)
 }
 
 bool cgroup_ssid_enabled(int ssid);
-bool cgroup_on_dfl(const struct cgroup *cgrp);
 
 struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root);
 struct cgroup *task_cgroup_from_root(struct task_struct *task,
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 01fc2a93f3ef2..71d536cc5e6cf 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -68,14 +68,6 @@
 /* let's not notify more than 100 times per second */
 #define CGROUP_FILE_NOTIFY_MIN_INTV	DIV_ROUND_UP(HZ, 100)
 
-/*
- * To avoid confusing the compiler (and generating warnings) with code
- * that attempts to access what would be a 0-element array (i.e. sized
- * to a potentially empty array when CGROUP_SUBSYS_COUNT == 0), this
- * constant expression can be added.
- */
-#define CGROUP_HAS_SUBSYS_CONFIG	(CGROUP_SUBSYS_COUNT > 0)
-
 /*
  * cgroup_mutex is the master lock.  Any modification to cgroup or its
  * hierarchy must be performed while holding it.
@@ -107,12 +99,6 @@ static bool cgroup_debug __read_mostly;
  */
 static DEFINE_SPINLOCK(cgroup_idr_lock);
 
-/*
- * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
- * against file removal/re-creation across css hiding.
- */
-static DEFINE_SPINLOCK(cgroup_file_kn_lock);
-
 DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem);
 
 #define cgroup_assert_mutex_or_rcu_locked()				\
@@ -509,27 +495,6 @@ static u32 cgroup_ss_mask(struct cgroup *cgrp)
 	return cgrp->root->subsys_mask;
 }
 
-/**
- * cgroup_css - obtain a cgroup's css for the specified subsystem
- * @cgrp: the cgroup of interest
- * @ss: the subsystem of interest (%NULL returns @cgrp->self)
- *
- * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
- * function must be called either under cgroup_mutex or rcu_read_lock() and
- * the caller is responsible for pinning the returned css if it wants to
- * keep accessing it outside the said locks.  This function may return
- * %NULL if @cgrp doesn't have @subsys_id enabled.
- */
-static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
-					      struct cgroup_subsys *ss)
-{
-	if (CGROUP_HAS_SUBSYS_CONFIG && ss)
-		return rcu_dereference_check(cgrp->subsys[ss->id],
-					lockdep_is_held(&cgroup_mutex));
-	else
-		return &cgrp->self;
-}
-
 /**
  * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss
  * @cgrp: the cgroup of interest
@@ -741,32 +706,6 @@ EXPORT_SYMBOL_GPL(of_css);
 	}								\
 } while (false)
 
-/* iterate over child cgrps, lock should be held throughout iteration */
-#define cgroup_for_each_live_child(child, cgrp)				\
-	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
-		if (({ lockdep_assert_held(&cgroup_mutex);		\
-		       cgroup_is_dead(child); }))			\
-			;						\
-		else
-
-/* walk live descendants in pre order */
-#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp)		\
-	css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL))	\
-		if (({ lockdep_assert_held(&cgroup_mutex);		\
-		       (dsct) = (d_css)->cgroup;			\
-		       cgroup_is_dead(dsct); }))			\
-			;						\
-		else
-
-/* walk live descendants in postorder */
-#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp)		\
-	css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL))	\
-		if (({ lockdep_assert_held(&cgroup_mutex);		\
-		       (dsct) = (d_css)->cgroup;			\
-		       cgroup_is_dead(dsct); }))			\
-			;						\
-		else
-
 /*
  * The default css_set - used by init and its children prior to any
  * hierarchies being mounted. It contains a pointer to the root state
@@ -1748,9 +1687,9 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
 		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
 		struct cgroup_file *cfile = (void *)css + cft->file_offset;
 
-		spin_lock_irq(&cgroup_file_kn_lock);
-		cfile->kn = NULL;
-		spin_unlock_irq(&cgroup_file_kn_lock);
+		spin_lock_irq(&cfile->lock);
+		WRITE_ONCE(cfile->kn, NULL);
+		spin_unlock_irq(&cfile->lock);
 
 		timer_delete_sync(&cfile->notify_timer);
 	}
@@ -4428,10 +4367,8 @@ static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
 		struct cgroup_file *cfile = (void *)css + cft->file_offset;
 
 		timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0);
-
-		spin_lock_irq(&cgroup_file_kn_lock);
+		spin_lock_init(&cfile->lock);
 		cfile->kn = kn;
-		spin_unlock_irq(&cgroup_file_kn_lock);
 	}
 
 	return 0;
@@ -4686,21 +4623,32 @@ int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
  */
 void cgroup_file_notify(struct cgroup_file *cfile)
 {
-	unsigned long flags;
+	unsigned long flags, last, next;
+	struct kernfs_node *kn = NULL;
+
+	if (!READ_ONCE(cfile->kn))
+		return;
+
+	last = READ_ONCE(cfile->notified_at);
+	next = last + CGROUP_FILE_NOTIFY_MIN_INTV;
+	if (time_in_range(jiffies, last, next)) {
+		timer_reduce(&cfile->notify_timer, next);
+		if (timer_pending(&cfile->notify_timer))
+			return;
+	}
 
-	spin_lock_irqsave(&cgroup_file_kn_lock, flags);
+	spin_lock_irqsave(&cfile->lock, flags);
 	if (cfile->kn) {
-		unsigned long last = cfile->notified_at;
-		unsigned long next = last + CGROUP_FILE_NOTIFY_MIN_INTV;
+		kn = cfile->kn;
+		kernfs_get(kn);
+		WRITE_ONCE(cfile->notified_at, jiffies);
+	}
+	spin_unlock_irqrestore(&cfile->lock, flags);
 
-		if (time_in_range(jiffies, last, next)) {
-			timer_reduce(&cfile->notify_timer, next);
-		} else {
-			kernfs_notify(cfile->kn);
-			cfile->notified_at = jiffies;
-		}
+	if (kn) {
+		kernfs_notify(kn);
+		kernfs_put(kn);
 	}
-	spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
 }
 EXPORT_SYMBOL_GPL(cgroup_file_notify);
 
@@ -4713,10 +4661,10 @@ void cgroup_file_show(struct cgroup_file *cfile, bool show)
 {
 	struct kernfs_node *kn;
 
-	spin_lock_irq(&cgroup_file_kn_lock);
+	spin_lock_irq(&cfile->lock);
 	kn = cfile->kn;
 	kernfs_get(kn);
-	spin_unlock_irq(&cgroup_file_kn_lock);
+	spin_unlock_irq(&cfile->lock);
 
 	if (kn)
 		kernfs_show(kn, show);
@@ -6050,8 +5998,9 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
  */
 static void css_killed_work_fn(struct work_struct *work)
 {
-	struct cgroup_subsys_state *css =
-		container_of(work, struct cgroup_subsys_state, destroy_work);
+	struct cgroup_subsys_state *css;
+
+	css = container_of(to_rcu_work(work), struct cgroup_subsys_state, destroy_rwork);
 
 	cgroup_lock();
 
@@ -6072,8 +6021,8 @@ static void css_killed_ref_fn(struct percpu_ref *ref)
 		container_of(ref, struct cgroup_subsys_state, refcnt);
 
 	if (atomic_dec_and_test(&css->online_cnt)) {
-		INIT_WORK(&css->destroy_work, css_killed_work_fn);
-		queue_work(cgroup_offline_wq, &css->destroy_work);
+		INIT_RCU_WORK(&css->destroy_rwork, css_killed_work_fn);
+		queue_rcu_work(cgroup_offline_wq, &css->destroy_rwork);
 	}
 }
 
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 2c1a3791e4101..4f21fc3b108b8 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -27,8 +27,6 @@
 #include <asm/page.h>
 #include <asm/sections.h>
 
-#include <crypto/sha1.h>
-
 #include "kallsyms_internal.h"
 #include "kexec_internal.h"
 
diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c
index 1f4067fbdb943..cb875ddb6ba68 100644
--- a/kernel/crash_dump_dm_crypt.c
+++ b/kernel/crash_dump_dm_crypt.c
@@ -6,6 +6,7 @@
 #include <linux/cc_platform.h>
 #include <linux/configfs.h>
 #include <linux/module.h>
+#include <linux/sysfs.h>
 
 #define KEY_NUM_MAX 128	/* maximum dm crypt keys */
 #define KEY_SIZE_MAX 256	/* maximum dm crypt key size */
@@ -115,7 +116,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 
 	addr = dm_crypt_keys_addr;
 	dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr);
-	if (key_count < 0 || key_count > KEY_NUM_MAX) {
+	if (key_count > KEY_NUM_MAX) {
 		kexec_dprintk("Failed to read the number of dm-crypt keys\n");
 		return -1;
 	}
@@ -139,7 +140,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 	return 0;
 }
 
-static int read_key_from_user_keying(struct dm_crypt_key *dm_key)
+static int read_key_from_user_keyring(struct dm_crypt_key *dm_key)
 {
 	const struct user_key_payload *ukp;
 	struct key *key;
@@ -168,8 +169,8 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key)
 
 	memcpy(dm_key->data, ukp->data, ukp->datalen);
 	dm_key->key_size = ukp->datalen;
-	kexec_dprintk("Get dm crypt key (size=%u) %s: %8ph\n", dm_key->key_size,
-		      dm_key->key_desc, dm_key->data);
+	kexec_dprintk("Get dm crypt key (size=%u) %s\n", dm_key->key_size,
+		      dm_key->key_desc);
 
 out:
 	up_read(&key->sem);
@@ -189,7 +190,7 @@ static inline struct config_key *to_config_key(struct config_item *item)
 
 static ssize_t config_key_description_show(struct config_item *item, char *page)
 {
-	return sprintf(page, "%s\n", to_config_key(item)->description);
+	return sysfs_emit(page, "%s\n", to_config_key(item)->description);
 }
 
 static ssize_t config_key_description_store(struct config_item *item,
@@ -265,7 +266,7 @@ static struct config_item *config_keys_make_item(struct config_group *group,
 
 static ssize_t config_keys_count_show(struct config_item *item, char *page)
 {
-	return sprintf(page, "%d\n", key_count);
+	return sysfs_emit(page, "%d\n", key_count);
 }
 
 CONFIGFS_ATTR_RO(config_keys_, count);
@@ -274,7 +275,7 @@ static bool is_dm_key_reused;
 
 static ssize_t config_keys_reuse_show(struct config_item *item, char *page)
 {
-	return sprintf(page, "%d\n", is_dm_key_reused);
+	return sysfs_emit(page, "%d\n", is_dm_key_reused);
 }
 
 static ssize_t config_keys_reuse_store(struct config_item *item,
@@ -321,7 +322,7 @@ static bool restore;
 
 static ssize_t config_keys_restore_show(struct config_item *item, char *page)
 {
-	return sprintf(page, "%d\n", restore);
+	return sysfs_emit(page, "%d\n", restore);
 }
 
 static ssize_t config_keys_restore_store(struct config_item *item,
@@ -387,7 +388,7 @@ static int build_keys_header(void)
 
 		strscpy(keys_header->keys[i].key_desc, key->description,
 			KEY_DESC_MAX_LEN);
-		r = read_key_from_user_keying(&keys_header->keys[i]);
+		r = read_key_from_user_keyring(&keys_header->keys[i]);
 		if (r != 0) {
 			kexec_dprintk("Failed to read key %s\n",
 				      keys_header->keys[i].key_desc);
@@ -414,14 +415,16 @@ int crash_load_dm_crypt_keys(struct kimage *image)
 
 	if (key_count <= 0) {
 		kexec_dprintk("No dm-crypt keys\n");
-		return -ENOENT;
+		return 0;
 	}
 
 	if (!is_dm_key_reused) {
 		image->dm_crypt_keys_addr = 0;
 		r = build_keys_header();
-		if (r)
+		if (r) {
+			pr_err("Failed to build dm-crypt keys header, ret=%d\n", r);
 			return r;
+		}
 	}
 
 	kbuf.buffer = keys_header;
@@ -432,6 +435,7 @@ int crash_load_dm_crypt_keys(struct kimage *image)
 	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 	r = kexec_add_buffer(&kbuf);
 	if (r) {
+		pr_err("Failed to call kexec_add_buffer, ret=%d\n", r);
 		kvfree((void *)kbuf.buffer);
 		return r;
 	}
diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c
index 62e60e0223cff..eee37a11380ce 100644
--- a/kernel/crash_reserve.c
+++ b/kernel/crash_reserve.c
@@ -20,8 +20,6 @@
 #include <asm/page.h>
 #include <asm/sections.h>
 
-#include <crypto/sha1.h>
-
 #include "kallsyms_internal.h"
 #include "kexec_internal.h"
 
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index 159900736f25b..bfef21b4a9ae7 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -72,6 +72,9 @@ config ARCH_HAS_DMA_PREP_COHERENT
 config ARCH_HAS_FORCE_DMA_UNENCRYPTED
 	bool
 
+config ARCH_HAS_BATCHED_DMA_SYNC
+	bool
+
 #
 # Select this option if the architecture assumes DMA devices are coherent
 # by default.
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 86f87e43438c3..0677918f06a80 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -453,7 +453,7 @@ static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
 	return overlap;
 }
 
-static void active_cacheline_inc_overlap(phys_addr_t cln)
+static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean)
 {
 	int overlap = active_cacheline_read_overlap(cln);
 
@@ -462,7 +462,7 @@ static void active_cacheline_inc_overlap(phys_addr_t cln)
 	/* If we overflowed the overlap counter then we're potentially
 	 * leaking dma-mappings.
 	 */
-	WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
+	WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
 		  pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
 		  ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
 }
@@ -495,7 +495,7 @@ static int active_cacheline_insert(struct dma_debug_entry *entry,
 	if (rc == -EEXIST) {
 		struct dma_debug_entry *existing;
 
-		active_cacheline_inc_overlap(cln);
+		active_cacheline_inc_overlap(cln, entry->is_cache_clean);
 		existing = radix_tree_lookup(&dma_active_cacheline, cln);
 		/* A lookup failure here after we got -EEXIST is unexpected. */
 		WARN_ON(!existing);
@@ -601,7 +601,8 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs)
 	unsigned long flags;
 	int rc;
 
-	entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN);
+	entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
+					 DMA_ATTR_REQUIRE_COHERENT);
 
 	bucket = get_hash_bucket(entry, &flags);
 	hash_bucket_add(bucket, entry);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 8f43a930716d4..ec887f4437418 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -406,6 +406,8 @@ void dma_direct_sync_sg_for_device(struct device *dev,
 			arch_sync_dma_for_device(paddr, sg->length,
 					dir);
 	}
+	if (!dev_is_dma_coherent(dev))
+		arch_sync_dma_flush();
 }
 #endif
 
@@ -427,8 +429,10 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
 		swiotlb_sync_single_for_cpu(dev, paddr, sg->length, dir);
 	}
 
-	if (!dev_is_dma_coherent(dev))
+	if (!dev_is_dma_coherent(dev)) {
+		arch_sync_dma_flush();
 		arch_sync_dma_for_cpu_all();
+	}
 }
 
 /*
@@ -440,14 +444,19 @@ void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
 {
 	struct scatterlist *sg;
 	int i;
+	bool need_sync = false;
 
 	for_each_sg(sgl,  sg, nents, i) {
-		if (sg_dma_is_bus_address(sg))
+		if (sg_dma_is_bus_address(sg)) {
 			sg_dma_unmark_bus_address(sg);
-		else
+		} else {
+			need_sync = true;
 			dma_direct_unmap_phys(dev, sg->dma_address,
-					      sg_dma_len(sg), dir, attrs);
+					      sg_dma_len(sg), dir, attrs, false);
+		}
 	}
+	if (need_sync && !dev_is_dma_coherent(dev))
+		arch_sync_dma_flush();
 }
 #endif
 
@@ -457,6 +466,7 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 	struct pci_p2pdma_map_state p2pdma_state = {};
 	struct scatterlist *sg;
 	int i, ret;
+	bool need_sync = false;
 
 	for_each_sg(sgl, sg, nents, i) {
 		switch (pci_p2pdma_state(&p2pdma_state, dev, sg_page(sg))) {
@@ -468,8 +478,9 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 			 */
 			break;
 		case PCI_P2PDMA_MAP_NONE:
+			need_sync = true;
 			sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg),
-					sg->length, dir, attrs);
+					sg->length, dir, attrs, false);
 			if (sg->dma_address == DMA_MAPPING_ERROR) {
 				ret = -EIO;
 				goto out_unmap;
@@ -488,6 +499,8 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 		sg_dma_len(sg) = sg->length;
 	}
 
+	if (need_sync && !dev_is_dma_coherent(dev))
+		arch_sync_dma_flush();
 	return nents;
 
 out_unmap:
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index e89f175e9c2d0..b86ff65496fcd 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -60,17 +60,22 @@ static inline void dma_direct_sync_single_for_device(struct device *dev,
 
 	swiotlb_sync_single_for_device(dev, paddr, size, dir);
 
-	if (!dev_is_dma_coherent(dev))
+	if (!dev_is_dma_coherent(dev)) {
 		arch_sync_dma_for_device(paddr, size, dir);
+		arch_sync_dma_flush();
+	}
 }
 
 static inline void dma_direct_sync_single_for_cpu(struct device *dev,
-		dma_addr_t addr, size_t size, enum dma_data_direction dir)
+		dma_addr_t addr, size_t size, enum dma_data_direction dir,
+		bool flush)
 {
 	phys_addr_t paddr = dma_to_phys(dev, addr);
 
 	if (!dev_is_dma_coherent(dev)) {
 		arch_sync_dma_for_cpu(paddr, size, dir);
+		if (flush)
+			arch_sync_dma_flush();
 		arch_sync_dma_for_cpu_all();
 	}
 
@@ -79,12 +84,12 @@ static inline void dma_direct_sync_single_for_cpu(struct device *dev,
 
 static inline dma_addr_t dma_direct_map_phys(struct device *dev,
 		phys_addr_t phys, size_t size, enum dma_data_direction dir,
-		unsigned long attrs)
+		unsigned long attrs, bool flush)
 {
 	dma_addr_t dma_addr;
 
 	if (is_swiotlb_force_bounce(dev)) {
-		if (attrs & DMA_ATTR_MMIO)
+		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
 			return DMA_MAPPING_ERROR;
 
 		return swiotlb_map(dev, phys, size, dir, attrs);
@@ -98,7 +103,8 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
 		dma_addr = phys_to_dma(dev, phys);
 		if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||
 		    dma_kmalloc_needs_bounce(dev, size, dir)) {
-			if (is_swiotlb_active(dev))
+			if (is_swiotlb_active(dev) &&
+			    !(attrs & DMA_ATTR_REQUIRE_COHERENT))
 				return swiotlb_map(dev, phys, size, dir, attrs);
 
 			goto err_overflow;
@@ -106,8 +112,11 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
 	}
 
 	if (!dev_is_dma_coherent(dev) &&
-	    !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))
+	    !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
 		arch_sync_dma_for_device(phys, size, dir);
+		if (flush)
+			arch_sync_dma_flush();
+	}
 	return dma_addr;
 
 err_overflow:
@@ -119,17 +128,18 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
 }
 
 static inline void dma_direct_unmap_phys(struct device *dev, dma_addr_t addr,
-		size_t size, enum dma_data_direction dir, unsigned long attrs)
+		size_t size, enum dma_data_direction dir, unsigned long attrs,
+		bool flush)
 {
 	phys_addr_t phys;
 
-	if (attrs & DMA_ATTR_MMIO)
+	if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
 		/* nothing to do: uncached and no swiotlb */
 		return;
 
 	phys = dma_to_phys(dev, addr);
 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
-		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
+		dma_direct_sync_single_for_cpu(dev, addr, size, dir, flush);
 
 	swiotlb_tbl_unmap_single(dev, phys, size, dir,
 					 attrs | DMA_ATTR_SKIP_CPU_SYNC);
diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c
index 0f33b3ea7daf8..29eeb5fdf199c 100644
--- a/kernel/dma/map_benchmark.c
+++ b/kernel/dma/map_benchmark.c
@@ -5,6 +5,7 @@
 
 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/device.h>
@@ -15,6 +16,7 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/timekeeping.h>
 #include <uapi/linux/map_benchmark.h>
@@ -31,17 +33,219 @@ struct map_benchmark_data {
 	atomic64_t loops;
 };
 
+struct map_benchmark_ops {
+	void *(*prepare)(struct map_benchmark_data *map);
+	void (*unprepare)(void *mparam);
+	void (*initialize_data)(void *mparam);
+	int (*do_map)(void *mparam);
+	void (*do_unmap)(void *mparam);
+};
+
+struct dma_single_map_param {
+	struct device *dev;
+	dma_addr_t addr;
+	void *xbuf;
+	u32 npages;
+	u32 dma_dir;
+};
+
+static void *dma_single_map_benchmark_prepare(struct map_benchmark_data *map)
+{
+	struct dma_single_map_param *params __free(kfree) = kzalloc(sizeof(*params),
+								    GFP_KERNEL);
+	if (!params)
+		return NULL;
+
+	params->npages = map->bparam.granule;
+	params->dma_dir = map->bparam.dma_dir;
+	params->dev = map->dev;
+	params->xbuf = alloc_pages_exact(params->npages * PAGE_SIZE, GFP_KERNEL);
+	if (!params->xbuf)
+		return NULL;
+
+	return_ptr(params);
+}
+
+static void dma_single_map_benchmark_unprepare(void *mparam)
+{
+	struct dma_single_map_param *params = mparam;
+
+	free_pages_exact(params->xbuf, params->npages * PAGE_SIZE);
+	kfree(params);
+}
+
+static void dma_single_map_benchmark_initialize_data(void *mparam)
+{
+	struct dma_single_map_param *params = mparam;
+
+	/*
+	 * for a non-coherent device, if we don't stain them in the
+	 * cache, this will give an underestimate of the real-world
+	 * overhead of BIDIRECTIONAL or TO_DEVICE mappings;
+	 * 66 means everything goes well! 66 is lucky.
+	 */
+	if (params->dma_dir != DMA_FROM_DEVICE)
+		memset(params->xbuf, 0x66, params->npages * PAGE_SIZE);
+}
+
+static int dma_single_map_benchmark_do_map(void *mparam)
+{
+	struct dma_single_map_param *params = mparam;
+
+	params->addr = dma_map_single(params->dev, params->xbuf,
+				      params->npages * PAGE_SIZE, params->dma_dir);
+	if (unlikely(dma_mapping_error(params->dev, params->addr))) {
+		pr_err("dma_map_single failed on %s\n", dev_name(params->dev));
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void dma_single_map_benchmark_do_unmap(void *mparam)
+{
+	struct dma_single_map_param *params = mparam;
+
+	dma_unmap_single(params->dev, params->addr,
+			 params->npages * PAGE_SIZE, params->dma_dir);
+}
+
+static struct map_benchmark_ops dma_single_map_benchmark_ops = {
+	.prepare = dma_single_map_benchmark_prepare,
+	.unprepare = dma_single_map_benchmark_unprepare,
+	.initialize_data = dma_single_map_benchmark_initialize_data,
+	.do_map = dma_single_map_benchmark_do_map,
+	.do_unmap = dma_single_map_benchmark_do_unmap,
+};
+
+struct dma_sg_map_param {
+	struct sg_table sgt;
+	struct device *dev;
+	void **buf;
+	u32 npages;
+	u32 dma_dir;
+};
+
+static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map)
+{
+	struct scatterlist *sg;
+	int i;
+
+	struct dma_sg_map_param *params = kzalloc(sizeof(*params), GFP_KERNEL);
+
+	if (!params)
+		return NULL;
+	/*
+	 * Set the number of scatterlist entries based on the granule.
+	 * In SG mode, 'granule' represents the number of scatterlist entries.
+	 * Each scatterlist entry corresponds to a single page.
+	 */
+	params->npages = map->bparam.granule;
+	params->dma_dir = map->bparam.dma_dir;
+	params->dev = map->dev;
+	params->buf = kmalloc_array(params->npages, sizeof(*params->buf),
+				    GFP_KERNEL);
+	if (!params->buf)
+		goto out;
+
+	if (sg_alloc_table(&params->sgt, params->npages, GFP_KERNEL))
+		goto free_buf;
+
+	for_each_sgtable_sg(&params->sgt, sg, i) {
+		params->buf[i] = (void *)__get_free_page(GFP_KERNEL);
+		if (!params->buf[i])
+			goto free_page;
+
+		sg_set_buf(sg, params->buf[i], PAGE_SIZE);
+	}
+
+	return params;
+
+free_page:
+	while (i-- > 0)
+		free_page((unsigned long)params->buf[i]);
+
+	sg_free_table(&params->sgt);
+free_buf:
+	kfree(params->buf);
+out:
+	kfree(params);
+	return NULL;
+}
+
+static void dma_sg_map_benchmark_unprepare(void *mparam)
+{
+	struct dma_sg_map_param *params = mparam;
+	int i;
+
+	for (i = 0; i < params->npages; i++)
+		free_page((unsigned long)params->buf[i]);
+
+	sg_free_table(&params->sgt);
+
+	kfree(params->buf);
+	kfree(params);
+}
+
+static void dma_sg_map_benchmark_initialize_data(void *mparam)
+{
+	struct dma_sg_map_param *params = mparam;
+	struct scatterlist *sg;
+	int i = 0;
+
+	if (params->dma_dir == DMA_FROM_DEVICE)
+		return;
+
+	for_each_sgtable_sg(&params->sgt, sg, i)
+		memset(params->buf[i], 0x66, PAGE_SIZE);
+}
+
+static int dma_sg_map_benchmark_do_map(void *mparam)
+{
+	struct dma_sg_map_param *params = mparam;
+	int ret = 0;
+
+	int sg_mapped = dma_map_sg(params->dev, params->sgt.sgl,
+				   params->npages, params->dma_dir);
+	if (!sg_mapped) {
+		pr_err("dma_map_sg failed on %s\n", dev_name(params->dev));
+		ret = -ENOMEM;
+	}
+
+	return ret;
+}
+
+static void dma_sg_map_benchmark_do_unmap(void *mparam)
+{
+	struct dma_sg_map_param *params = mparam;
+
+	dma_unmap_sg(params->dev, params->sgt.sgl, params->npages,
+		     params->dma_dir);
+}
+
+static struct map_benchmark_ops dma_sg_map_benchmark_ops = {
+	.prepare = dma_sg_map_benchmark_prepare,
+	.unprepare = dma_sg_map_benchmark_unprepare,
+	.initialize_data = dma_sg_map_benchmark_initialize_data,
+	.do_map = dma_sg_map_benchmark_do_map,
+	.do_unmap = dma_sg_map_benchmark_do_unmap,
+};
+
+static struct map_benchmark_ops *dma_map_benchmark_ops[DMA_MAP_BENCH_MODE_MAX] = {
+	[DMA_MAP_BENCH_SINGLE_MODE] = &dma_single_map_benchmark_ops,
+	[DMA_MAP_BENCH_SG_MODE] = &dma_sg_map_benchmark_ops,
+};
+
 static int map_benchmark_thread(void *data)
 {
-	void *buf;
-	dma_addr_t dma_addr;
 	struct map_benchmark_data *map = data;
-	int npages = map->bparam.granule;
-	u64 size = npages * PAGE_SIZE;
+	__u8 map_mode = map->bparam.map_mode;
 	int ret = 0;
 
-	buf = alloc_pages_exact(size, GFP_KERNEL);
-	if (!buf)
+	struct map_benchmark_ops *mb_ops = dma_map_benchmark_ops[map_mode];
+	void *mparam = mb_ops->prepare(map);
+
+	if (!mparam)
 		return -ENOMEM;
 
 	while (!kthread_should_stop())  {
@@ -49,23 +253,12 @@ static int map_benchmark_thread(void *data)
 		ktime_t map_stime, map_etime, unmap_stime, unmap_etime;
 		ktime_t map_delta, unmap_delta;
 
-		/*
-		 * for a non-coherent device, if we don't stain them in the
-		 * cache, this will give an underestimate of the real-world
-		 * overhead of BIDIRECTIONAL or TO_DEVICE mappings;
-		 * 66 means evertything goes well! 66 is lucky.
-		 */
-		if (map->dir != DMA_FROM_DEVICE)
-			memset(buf, 0x66, size);
-
+		mb_ops->initialize_data(mparam);
 		map_stime = ktime_get();
-		dma_addr = dma_map_single(map->dev, buf, size, map->dir);
-		if (unlikely(dma_mapping_error(map->dev, dma_addr))) {
-			pr_err("dma_map_single failed on %s\n",
-				dev_name(map->dev));
-			ret = -ENOMEM;
+		ret = mb_ops->do_map(mparam);
+		if (ret)
 			goto out;
-		}
+
 		map_etime = ktime_get();
 		map_delta = ktime_sub(map_etime, map_stime);
 
@@ -73,7 +266,8 @@ static int map_benchmark_thread(void *data)
 		ndelay(map->bparam.dma_trans_ns);
 
 		unmap_stime = ktime_get();
-		dma_unmap_single(map->dev, dma_addr, size, map->dir);
+		mb_ops->do_unmap(mparam);
+
 		unmap_etime = ktime_get();
 		unmap_delta = ktime_sub(unmap_etime, unmap_stime);
 
@@ -108,7 +302,7 @@ static int map_benchmark_thread(void *data)
 	}
 
 out:
-	free_pages_exact(buf, size);
+	mb_ops->unprepare(mparam);
 	return ret;
 }
 
@@ -209,6 +403,12 @@ static long map_benchmark_ioctl(struct file *file, unsigned int cmd,
 
 	switch (cmd) {
 	case DMA_MAP_BENCHMARK:
+		if (map->bparam.map_mode < 0 ||
+		    map->bparam.map_mode >= DMA_MAP_BENCH_MODE_MAX) {
+			pr_err("invalid map mode\n");
+			return -EINVAL;
+		}
+
 		if (map->bparam.threads == 0 ||
 		    map->bparam.threads > DMA_MAP_MAX_THREADS) {
 			pr_err("invalid thread number\n");
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 3928a509c44c2..df3eccc7d4cac 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -164,9 +164,12 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	if (WARN_ON_ONCE(!dev->dma_mask))
 		return DMA_MAPPING_ERROR;
 
+	if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
+		return DMA_MAPPING_ERROR;
+
 	if (dma_map_direct(dev, ops) ||
 	    (!is_mmio && arch_dma_map_phys_direct(dev, phys + size)))
-		addr = dma_direct_map_phys(dev, phys, size, dir, attrs);
+		addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true);
 	else if (use_dma_iommu(dev))
 		addr = iommu_dma_map_phys(dev, phys, size, dir, attrs);
 	else if (ops->map_phys)
@@ -207,7 +210,7 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,
 	BUG_ON(!valid_dma_direction(dir));
 	if (dma_map_direct(dev, ops) ||
 	    (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size)))
-		dma_direct_unmap_phys(dev, addr, size, dir, attrs);
+		dma_direct_unmap_phys(dev, addr, size, dir, attrs, true);
 	else if (use_dma_iommu(dev))
 		iommu_dma_unmap_phys(dev, addr, size, dir, attrs);
 	else if (ops->unmap_phys)
@@ -235,6 +238,9 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 
 	BUG_ON(!valid_dma_direction(dir));
 
+	if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
+		return -EOPNOTSUPP;
+
 	if (WARN_ON_ONCE(!dev->dma_mask))
 		return 0;
 
@@ -373,7 +379,7 @@ void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 
 	BUG_ON(!valid_dma_direction(dir));
 	if (dma_map_direct(dev, ops))
-		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
+		dma_direct_sync_single_for_cpu(dev, addr, size, dir, true);
 	else if (use_dma_iommu(dev))
 		iommu_dma_sync_single_for_cpu(dev, addr, size, dir);
 	else if (ops->sync_single_for_cpu)
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index d8e6f1d889d55..487dcf73b88d1 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -30,6 +30,7 @@
 #include <linux/gfp.h>
 #include <linux/highmem.h>
 #include <linux/io.h>
+#include <linux/kmsan-checks.h>
 #include <linux/iommu-helper.h>
 #include <linux/init.h>
 #include <linux/memblock.h>
@@ -867,6 +868,9 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
 	if (orig_addr == INVALID_PHYS_ADDR)
 		return;
 
+	if (dir == DMA_FROM_DEVICE && !dev_is_dma_coherent(dev))
+		arch_sync_dma_flush();
+
 	/*
 	 * It's valid for tlb_offset to be negative. This can happen when the
 	 * "offset" returned by swiotlb_align_offset() is non-zero, and the
@@ -901,10 +905,19 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
 
 			local_irq_save(flags);
 			page = pfn_to_page(pfn);
-			if (dir == DMA_TO_DEVICE)
+			if (dir == DMA_TO_DEVICE) {
+				/*
+				 * Ideally, kmsan_check_highmem_page()
+				 * could be used here to detect infoleaks,
+				 * but callers may map uninitialized buffers
+				 * that will be written by the device,
+				 * causing false positives.
+				 */
 				memcpy_from_page(vaddr, page, offset, sz);
-			else
+			} else {
+				kmsan_unpoison_memory(vaddr, sz);
 				memcpy_to_page(page, offset, vaddr, sz);
+			}
 			local_irq_restore(flags);
 
 			size -= sz;
@@ -913,8 +926,15 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
 			offset = 0;
 		}
 	} else if (dir == DMA_TO_DEVICE) {
+		/*
+		 * Ideally, kmsan_check_memory() could be used here to detect
+		 * infoleaks (uninitialized data being sent to device), but
+		 * callers may map uninitialized buffers that will be written
+		 * by the device, causing false positives.
+		 */
 		memcpy(vaddr, phys_to_virt(orig_addr), size);
 	} else {
+		kmsan_unpoison_memory(vaddr, size);
 		memcpy(phys_to_virt(orig_addr), vaddr, size);
 	}
 }
@@ -1595,8 +1615,10 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
 		return DMA_MAPPING_ERROR;
 	}
 
-	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
 		arch_sync_dma_for_device(swiotlb_addr, size, dir);
+		arch_sync_dma_flush();
+	}
 	return dma_addr;
 }
 
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 9ef63e4147913..9e1a6afb07f22 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -50,7 +50,7 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re
 		local_irq_enable_exit_to_user(ti_work);
 
 		if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
-			if (!rseq_grant_slice_extension(ti_work & TIF_SLICE_EXT_DENY))
+			if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY))
 				schedule();
 		}
 
@@ -225,6 +225,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 		 */
 		if (state.exit_rcu) {
 			instrumentation_begin();
+			hrtimer_rearm_deferred();
 			/* Tell the tracer that IRET will enable interrupts */
 			trace_hardirqs_on_prepare();
 			lockdep_hardirqs_on_prepare();
@@ -238,6 +239,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 		if (IS_ENABLED(CONFIG_PREEMPTION))
 			irqentry_exit_cond_resched();
 
+		hrtimer_rearm_deferred();
 		/* Covers both tracing and lockdep */
 		trace_hardirqs_on();
 		instrumentation_end();
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 1f5699b339ec8..6b83c44075c8c 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -4813,7 +4813,7 @@ static void __perf_event_read(void *info)
 	struct perf_event *sub, *event = data->event;
 	struct perf_event_context *ctx = event->ctx;
 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
-	struct pmu *pmu = event->pmu;
+	struct pmu *pmu;
 
 	/*
 	 * If this is a task context, we need to check whether it is
@@ -4825,7 +4825,7 @@ static void __perf_event_read(void *info)
 	if (ctx->task && cpuctx->task_ctx != ctx)
 		return;
 
-	raw_spin_lock(&ctx->lock);
+	guard(raw_spinlock)(&ctx->lock);
 	ctx_time_update_event(ctx, event);
 
 	perf_event_update_time(event);
@@ -4833,25 +4833,22 @@ static void __perf_event_read(void *info)
 		perf_event_update_sibling_time(event);
 
 	if (event->state != PERF_EVENT_STATE_ACTIVE)
-		goto unlock;
+		return;
 
 	if (!data->group) {
-		pmu->read(event);
+		perf_pmu_read(event);
 		data->ret = 0;
-		goto unlock;
+		return;
 	}
 
+	pmu = event->pmu_ctx->pmu;
 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
 
-	pmu->read(event);
-
+	perf_pmu_read(event);
 	for_each_sibling_event(sub, event)
 		perf_pmu_read(sub);
 
 	data->ret = pmu->commit_txn(pmu);
-
-unlock:
-	raw_spin_unlock(&ctx->lock);
 }
 
 static inline u64 perf_event_count(struct perf_event *event, bool self)
@@ -5371,15 +5368,15 @@ static void unaccount_freq_event(void)
 
 
 static struct perf_ctx_data *
-alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global)
+alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global, gfp_t gfp_flags)
 {
 	struct perf_ctx_data *cd;
 
-	cd = kzalloc_obj(*cd);
+	cd = kzalloc_obj(*cd, gfp_flags);
 	if (!cd)
 		return NULL;
 
-	cd->data = kmem_cache_zalloc(ctx_cache, GFP_KERNEL);
+	cd->data = kmem_cache_zalloc(ctx_cache, gfp_flags);
 	if (!cd->data) {
 		kfree(cd);
 		return NULL;
@@ -5413,11 +5410,11 @@ static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd)
 
 static int
 attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
-		     bool global)
+		     bool global, gfp_t gfp_flags)
 {
 	struct perf_ctx_data *cd, *old = NULL;
 
-	cd = alloc_perf_ctx_data(ctx_cache, global);
+	cd = alloc_perf_ctx_data(ctx_cache, global, gfp_flags);
 	if (!cd)
 		return -ENOMEM;
 
@@ -5490,6 +5487,12 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache)
 					cd = NULL;
 			}
 			if (!cd) {
+				/*
+				 * Try to allocate context quickly before
+				 * traversing the whole thread list again.
+				 */
+				if (!attach_task_ctx_data(p, ctx_cache, true, GFP_NOWAIT))
+					continue;
 				get_task_struct(p);
 				goto alloc;
 			}
@@ -5500,7 +5503,7 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache)
 
 	return 0;
 alloc:
-	ret = attach_task_ctx_data(p, ctx_cache, true);
+	ret = attach_task_ctx_data(p, ctx_cache, true, GFP_KERNEL);
 	put_task_struct(p);
 	if (ret) {
 		__detach_global_ctx_data();
@@ -5520,7 +5523,7 @@ attach_perf_ctx_data(struct perf_event *event)
 		return -ENOMEM;
 
 	if (task)
-		return attach_task_ctx_data(task, ctx_cache, false);
+		return attach_task_ctx_data(task, ctx_cache, false, GFP_KERNEL);
 
 	ret = attach_global_ctx_data(ctx_cache);
 	if (ret)
@@ -5555,22 +5558,15 @@ static void __detach_global_ctx_data(void)
 	struct task_struct *g, *p;
 	struct perf_ctx_data *cd;
 
-again:
 	scoped_guard (rcu) {
 		for_each_process_thread(g, p) {
 			cd = rcu_dereference(p->perf_ctx_data);
-			if (!cd || !cd->global)
-				continue;
-			cd->global = 0;
-			get_task_struct(p);
-			goto detach;
+			if (cd && cd->global) {
+				cd->global = 0;
+				detach_task_ctx_data(p);
+			}
 		}
 	}
-	return;
-detach:
-	detach_task_ctx_data(p);
-	put_task_struct(p);
-	goto again;
 }
 
 static void detach_global_ctx_data(void)
@@ -7216,7 +7212,7 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma)
 #ifdef CONFIG_MMU
 	/* Clear any partial mappings on error. */
 	if (err)
-		zap_page_range_single(vma, vma->vm_start, nr_pages * PAGE_SIZE, NULL);
+		zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE);
 #endif
 
 	return err;
@@ -9241,7 +9237,7 @@ perf_event_alloc_task_data(struct task_struct *child,
 
 	return;
 attach:
-	attach_task_ctx_data(child, ctx_cache, true);
+	attach_task_ctx_data(child, ctx_cache, true, GFP_KERNEL);
 }
 
 void perf_event_fork(struct task_struct *task)
@@ -14744,7 +14740,7 @@ inherit_event(struct perf_event *parent_event,
 	get_ctx(child_ctx);
 	child_event->ctx = child_ctx;
 
-	pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event);
+	pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event);
 	if (IS_ERR(pmu_ctx)) {
 		free_event(child_event);
 		return ERR_CAST(pmu_ctx);
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 923b24b321cc0..4084e926e2844 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -344,7 +344,7 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
 static void update_ref_ctr_warn(struct uprobe *uprobe,
 				struct mm_struct *mm, short d)
 {
-	pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
+	pr_warn("ref_ctr %s failed for inode: 0x%llx offset: "
 		"0x%llx ref_ctr_offset: 0x%llx of mm: 0x%p\n",
 		d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
 		(unsigned long long) uprobe->offset,
@@ -982,7 +982,7 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe)
 static void
 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
 {
-	pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
+	pr_warn("ref_ctr_offset mismatch. inode: 0x%llx offset: 0x%llx "
 		"ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
 		uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
 		(unsigned long long) cur_uprobe->ref_ctr_offset,
diff --git a/kernel/exit.c b/kernel/exit.c
index ede3117fa7d41..1f32023d0dbe2 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -748,14 +748,12 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
 	tsk->exit_state = EXIT_ZOMBIE;
 
 	if (unlikely(tsk->ptrace)) {
-		int sig = thread_group_leader(tsk) &&
-				thread_group_empty(tsk) &&
-				!ptrace_reparented(tsk) ?
-			tsk->exit_signal : SIGCHLD;
+		int sig = thread_group_empty(tsk) && !ptrace_reparented(tsk)
+			  ? tsk->exit_signal : SIGCHLD;
 		autoreap = do_notify_parent(tsk, sig);
 	} else if (thread_group_leader(tsk)) {
 		autoreap = thread_group_empty(tsk) &&
-			do_notify_parent(tsk, tsk->exit_signal);
+			   do_notify_parent(tsk, tsk->exit_signal);
 	} else {
 		autoreap = true;
 		/* untraced sub-thread */
diff --git a/kernel/fork.c b/kernel/fork.c
index bc2bf58b93b65..3640457fd7887 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -134,6 +134,11 @@
  */
 #define MAX_THREADS FUTEX_TID_MASK
 
+/*
+ * Batch size of rss stat approximation
+ */
+#define RSS_STAT_BATCH_SIZE	32
+
 /*
  * Protected counters by write_lock_irq(&tasklist_lock)
  */
@@ -345,7 +350,7 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node)
 		stack = kasan_reset_tag(vm_area->addr);
 
 		/* Clear stale pointers from reused stack. */
-		memset(stack, 0, THREAD_SIZE);
+		clear_pages(vm_area->addr, vm_area->nr_pages);
 
 		tsk->stack_vm_area = vm_area;
 		tsk->stack = stack;
@@ -627,14 +632,12 @@ static void check_mm(struct mm_struct *mm)
 			 "Please make sure 'struct resident_page_types[]' is updated as well");
 
 	for (i = 0; i < NR_MM_COUNTERS; i++) {
-		long x = percpu_counter_sum(&mm->rss_stat[i]);
-
-		if (unlikely(x)) {
+		if (unlikely(percpu_counter_tree_precise_compare_value(&mm->rss_stat[i], 0) != 0))
 			pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
-				 mm, resident_page_types[i], x,
+				 mm, resident_page_types[i],
+				 percpu_counter_tree_precise_sum(&mm->rss_stat[i]),
 				 current->comm,
 				 task_pid_nr(current));
-		}
 	}
 
 	if (mm_pgtables_bytes(mm))
@@ -732,7 +735,7 @@ void __mmdrop(struct mm_struct *mm)
 	put_user_ns(mm->user_ns);
 	mm_pasid_drop(mm);
 	mm_destroy_cid(mm);
-	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+	percpu_counter_tree_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
 
 	free_mm(mm);
 }
@@ -1014,13 +1017,14 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 
 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
 
-static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
+static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT;
 
 static int __init coredump_filter_setup(char *s)
 {
-	default_dump_filter =
-		(simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
-		MMF_DUMP_FILTER_MASK;
+	if (kstrtoul(s, 0, &coredump_filter))
+		return 0;
+	coredump_filter <<= MMF_DUMP_FILTER_SHIFT;
+	coredump_filter &= MMF_DUMP_FILTER_MASK;
 	return 1;
 }
 
@@ -1105,8 +1109,16 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 
 		__mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags));
 		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
+
+		if (mm_flags_test(MMF_USER_HWCAP, current->mm)) {
+			spin_lock(&current->mm->arg_lock);
+			mm_flags_set(MMF_USER_HWCAP, mm);
+			memcpy(mm->saved_auxv, current->mm->saved_auxv,
+			       sizeof(mm->saved_auxv));
+			spin_unlock(&current->mm->arg_lock);
+		}
 	} else {
-		__mm_flags_overwrite_word(mm, default_dump_filter);
+		__mm_flags_overwrite_word(mm, coredump_filter);
 		mm->def_flags = 0;
 	}
 
@@ -1125,8 +1137,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	if (mm_alloc_cid(mm, p))
 		goto fail_cid;
 
-	if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
-				     NR_MM_COUNTERS))
+	if (percpu_counter_tree_init_many(mm->rss_stat, get_rss_stat_items(mm),
+					  NR_MM_COUNTERS, RSS_STAT_BATCH_SIZE,
+					  GFP_KERNEL_ACCOUNT))
 		goto fail_pcpu;
 
 	mm->user_ns = get_user_ns(user_ns);
@@ -2028,6 +2041,41 @@ __latent_entropy struct task_struct *copy_process(
 			return ERR_PTR(-EINVAL);
 	}
 
+	if (clone_flags & CLONE_AUTOREAP) {
+		if (clone_flags & CLONE_THREAD)
+			return ERR_PTR(-EINVAL);
+		if (clone_flags & CLONE_PARENT)
+			return ERR_PTR(-EINVAL);
+		if (args->exit_signal)
+			return ERR_PTR(-EINVAL);
+	}
+
+	if ((clone_flags & CLONE_PARENT) && current->signal->autoreap)
+		return ERR_PTR(-EINVAL);
+
+	if (clone_flags & CLONE_NNP) {
+		if (clone_flags & CLONE_THREAD)
+			return ERR_PTR(-EINVAL);
+	}
+
+	if (clone_flags & CLONE_PIDFD_AUTOKILL) {
+		if (!(clone_flags & CLONE_PIDFD))
+			return ERR_PTR(-EINVAL);
+		if (!(clone_flags & CLONE_AUTOREAP))
+			return ERR_PTR(-EINVAL);
+		if (clone_flags & CLONE_THREAD)
+			return ERR_PTR(-EINVAL);
+		/*
+		 * Without CLONE_NNP the child could escalate privileges
+		 * after being spawned, so require CAP_SYS_ADMIN.
+		 * With CLONE_NNP the child can't gain new privileges,
+		 * so allow unprivileged usage.
+		 */
+		if (!(clone_flags & CLONE_NNP) &&
+		    !ns_capable(current_user_ns(), CAP_SYS_ADMIN))
+			return ERR_PTR(-EPERM);
+	}
+
 	/*
 	 * Force any signals received before this point to be delivered
 	 * before the fork happens.  Collect up signals sent to multiple
@@ -2250,13 +2298,18 @@ __latent_entropy struct task_struct *copy_process(
 	 * if the fd table isn't shared).
 	 */
 	if (clone_flags & CLONE_PIDFD) {
-		int flags = (clone_flags & CLONE_THREAD) ? PIDFD_THREAD : 0;
+		unsigned flags = PIDFD_STALE;
+
+		if (clone_flags & CLONE_THREAD)
+			flags |= PIDFD_THREAD;
+		if (clone_flags & CLONE_PIDFD_AUTOKILL)
+			flags |= PIDFD_AUTOKILL;
 
 		/*
 		 * Note that no task has been attached to @pid yet indicate
 		 * that via CLONE_PIDFD.
 		 */
-		retval = pidfd_prepare(pid, flags | PIDFD_STALE, &pidfile);
+		retval = pidfd_prepare(pid, flags, &pidfile);
 		if (retval < 0)
 			goto bad_fork_free_pid;
 		pidfd = retval;
@@ -2392,7 +2445,11 @@ __latent_entropy struct task_struct *copy_process(
 
 	rseq_fork(p, clone_flags);
 
-	/* Don't start children in a dying pid namespace */
+	/*
+	 * If zap_pid_ns_processes() was called after alloc_pid(), the new
+	 * child missed SIGKILL.  If current is not in the same namespace,
+	 * we can't rely on fatal_signal_pending() below.
+	 */
 	if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
 		retval = -ENOMEM;
 		goto bad_fork_core_free;
@@ -2412,6 +2469,9 @@ __latent_entropy struct task_struct *copy_process(
 	 */
 	copy_seccomp(p);
 
+	if (clone_flags & CLONE_NNP)
+		task_set_no_new_privs(p);
+
 	init_task_pid_links(p);
 	if (likely(p->pid)) {
 		ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
@@ -2435,6 +2495,8 @@ __latent_entropy struct task_struct *copy_process(
 			 */
 			p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
 							 p->real_parent->signal->is_child_subreaper;
+			if (clone_flags & CLONE_AUTOREAP)
+				p->signal->autoreap = 1;
 			list_add_tail(&p->sibling, &p->real_parent->children);
 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
 			attach_pid(p, PIDTYPE_TGID);
@@ -2463,8 +2525,12 @@ __latent_entropy struct task_struct *copy_process(
 		fd_install(pidfd, pidfile);
 
 	proc_fork_connector(p);
-	sched_post_fork(p);
+	/*
+	 * sched_ext needs @p to be associated with its cgroup in its post_fork
+	 * hook. cgroup_post_fork() should come before sched_post_fork().
+	 */
 	cgroup_post_fork(p, args);
+	sched_post_fork(p);
 	perf_event_fork(p);
 
 	trace_task_newtask(p, clone_flags);
@@ -2606,8 +2672,6 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
  *
  * It copies the process, and if successful kick-starts
  * it and waits for it to finish using the VM if required.
- *
- * args->exit_signal is expected to be checked for sanity by the caller.
  */
 pid_t kernel_clone(struct kernel_clone_args *args)
 {
@@ -2632,6 +2696,9 @@ pid_t kernel_clone(struct kernel_clone_args *args)
 	    (args->pidfd == args->parent_tid))
 		return -EINVAL;
 
+	if (!valid_signal(args->exit_signal))
+		return -EINVAL;
+
 	/*
 	 * Determine whether and which event to report to ptracer.  When
 	 * called from kernel_thread or CLONE_UNTRACED is explicitly
@@ -2830,11 +2897,9 @@ static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
 		return -EINVAL;
 
 	/*
-	 * Verify that higher 32bits of exit_signal are unset and that
-	 * it is a valid signal
+	 * Verify that higher 32bits of exit_signal are unset
 	 */
-	if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
-		     !valid_signal(args.exit_signal)))
+	if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
 		return -EINVAL;
 
 	if ((args.flags & CLONE_INTO_CGROUP) &&
@@ -2896,7 +2961,8 @@ static bool clone3_args_valid(struct kernel_clone_args *kargs)
 {
 	/* Verify that no unknown flags are passed along. */
 	if (kargs->flags &
-	    ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP))
+	    ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP |
+	      CLONE_AUTOREAP | CLONE_NNP | CLONE_PIDFD_AUTOKILL))
 		return false;
 
 	/*
@@ -3008,7 +3074,7 @@ void __init mm_cache_init(void)
 	 * dynamically sized based on the maximum CPU number this system
 	 * can have, taking hotplug into account (nr_cpu_ids).
 	 */
-	mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();
+	mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size() + get_rss_stat_items_size();
 
 	mm_cachep = kmem_cache_create_usercopy("mm_struct",
 			mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
@@ -3174,11 +3240,10 @@ int ksys_unshare(unsigned long unshare_flags)
 					 new_cred, new_fs);
 	if (err)
 		goto bad_unshare_cleanup_cred;
-
 	if (new_cred) {
 		err = set_cred_ucounts(new_cred);
 		if (err)
-			goto bad_unshare_cleanup_cred;
+			goto bad_unshare_cleanup_nsproxy;
 	}
 
 	if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
@@ -3194,8 +3259,10 @@ int ksys_unshare(unsigned long unshare_flags)
 			shm_init_task(current);
 		}
 
-		if (new_nsproxy)
+		if (new_nsproxy) {
 			switch_task_namespaces(current, new_nsproxy);
+			new_nsproxy = NULL;
+		}
 
 		task_lock(current);
 
@@ -3224,13 +3291,15 @@ int ksys_unshare(unsigned long unshare_flags)
 
 	perf_event_namespaces(current);
 
+bad_unshare_cleanup_nsproxy:
+	if (new_nsproxy)
+		put_nsproxy(new_nsproxy);
 bad_unshare_cleanup_cred:
 	if (new_cred)
 		put_cred(new_cred);
 bad_unshare_cleanup_fd:
 	if (new_fd)
 		put_files_struct(new_fd);
-
 bad_unshare_cleanup_fs:
 	if (new_fs)
 		free_fs_struct(new_fs);
diff --git a/kernel/futex/Makefile b/kernel/futex/Makefile
index b77188d1fa07f..dce70f8a322ba 100644
--- a/kernel/futex/Makefile
+++ b/kernel/futex/Makefile
@@ -1,3 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
+CONTEXT_ANALYSIS := y
+
 obj-y += core.o syscalls.o pi.o requeue.o waitwake.o
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index cf7e610eac429..4bacf5565368c 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -864,7 +864,6 @@ void __futex_unqueue(struct futex_q *q)
 
 /* The key must be already stored in q->key. */
 void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb)
-	__acquires(&hb->lock)
 {
 	/*
 	 * Increment the counter before taking the lock so that
@@ -879,10 +878,10 @@ void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb)
 	q->lock_ptr = &hb->lock;
 
 	spin_lock(&hb->lock);
+	__acquire(q->lock_ptr);
 }
 
 void futex_q_unlock(struct futex_hash_bucket *hb)
-	__releases(&hb->lock)
 {
 	futex_hb_waiters_dec(hb);
 	spin_unlock(&hb->lock);
@@ -1443,12 +1442,15 @@ static void futex_cleanup(struct task_struct *tsk)
 void futex_exit_recursive(struct task_struct *tsk)
 {
 	/* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */
-	if (tsk->futex_state == FUTEX_STATE_EXITING)
+	if (tsk->futex_state == FUTEX_STATE_EXITING) {
+		__assume_ctx_lock(&tsk->futex_exit_mutex);
 		mutex_unlock(&tsk->futex_exit_mutex);
+	}
 	tsk->futex_state = FUTEX_STATE_DEAD;
 }
 
 static void futex_cleanup_begin(struct task_struct *tsk)
+	__acquires(&tsk->futex_exit_mutex)
 {
 	/*
 	 * Prevent various race issues against a concurrent incoming waiter
@@ -1475,6 +1477,7 @@ static void futex_cleanup_begin(struct task_struct *tsk)
 }
 
 static void futex_cleanup_end(struct task_struct *tsk, int state)
+	__releases(&tsk->futex_exit_mutex)
 {
 	/*
 	 * Lockless store. The only side effect is that an observer might
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 30c2afa038890..9f6bf6f585fca 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -217,7 +217,7 @@ enum futex_access {
 
 extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
 			 enum futex_access rw);
-extern void futex_q_lockptr_lock(struct futex_q *q);
+extern void futex_q_lockptr_lock(struct futex_q *q) __acquires(q->lock_ptr);
 extern struct hrtimer_sleeper *
 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
 		  int flags, u64 range_ns);
@@ -311,9 +311,11 @@ extern int futex_unqueue(struct futex_q *q);
 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
 			       struct task_struct *task)
 	__releases(&hb->lock)
+	__releases(q->lock_ptr)
 {
 	__futex_queue(q, hb, task);
 	spin_unlock(&hb->lock);
+	__release(q->lock_ptr);
 }
 
 extern void futex_unqueue_pi(struct futex_q *q);
@@ -358,9 +360,12 @@ static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
 #endif
 }
 
-extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb);
-extern void futex_q_unlock(struct futex_hash_bucket *hb);
+extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb)
+	__acquires(&hb->lock)
+	__acquires(q->lock_ptr);
 
+extern void futex_q_unlock(struct futex_hash_bucket *hb)
+	__releases(&hb->lock);
 
 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
 				union futex_key *key,
@@ -379,6 +384,9 @@ extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
  */
 static inline void
 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
+	__acquires(&hb1->lock)
+	__acquires(&hb2->lock)
+	__no_context_analysis
 {
 	if (hb1 > hb2)
 		swap(hb1, hb2);
@@ -390,6 +398,9 @@ double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
 
 static inline void
 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
+	__releases(&hb1->lock)
+	__releases(&hb2->lock)
+	__no_context_analysis
 {
 	spin_unlock(&hb1->lock);
 	if (hb1 != hb2)
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index bc1f7e83a37e1..49ab5f40e86c7 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -389,6 +389,7 @@ static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key,
 	 * Initialize the pi_mutex in locked state and make @p
 	 * the owner of it:
 	 */
+	__assume_ctx_lock(&pi_state->pi_mutex.wait_lock);
 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
 
 	/* Store the key for possible exit cleanups: */
@@ -614,6 +615,8 @@ int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
 static int wake_futex_pi(u32 __user *uaddr, u32 uval,
 			 struct futex_pi_state *pi_state,
 			 struct rt_mutex_waiter *top_waiter)
+	__must_hold(&pi_state->pi_mutex.wait_lock)
+	__releases(&pi_state->pi_mutex.wait_lock)
 {
 	struct task_struct *new_owner;
 	bool postunlock = false;
@@ -670,6 +673,8 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval,
 
 static int __fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
 				  struct task_struct *argowner)
+	__must_hold(&q->pi_state->pi_mutex.wait_lock)
+	__must_hold(q->lock_ptr)
 {
 	struct futex_pi_state *pi_state = q->pi_state;
 	struct task_struct *oldowner, *newowner;
@@ -966,6 +971,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
 				 * - EAGAIN: The user space value changed.
 				 */
 				futex_q_unlock(hb);
+				__release(q.lock_ptr);
 				/*
 				 * Handle the case where the owner is in the middle of
 				 * exiting. Wait for the exit to complete otherwise
@@ -1090,6 +1096,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
 		if (res)
 			ret = (res < 0) ? res : 0;
 
+		__release(&hb->lock);
 		futex_unqueue_pi(&q);
 		spin_unlock(q.lock_ptr);
 		if (q.drop_hb_ref) {
@@ -1101,10 +1108,12 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
 
 out_unlock_put_key:
 		futex_q_unlock(hb);
+		__release(q.lock_ptr);
 		goto out;
 
 uaddr_faulted:
 		futex_q_unlock(hb);
+		__release(q.lock_ptr);
 
 		ret = fault_in_user_writeable(uaddr);
 		if (ret)
diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c
index 1c2dd03f11ec4..ceed9d8790592 100644
--- a/kernel/futex/waitwake.c
+++ b/kernel/futex/waitwake.c
@@ -462,6 +462,7 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
 			}
 
 			futex_q_unlock(hb);
+			__release(q->lock_ptr);
 		}
 		__set_current_state(TASK_RUNNING);
 
@@ -628,6 +629,7 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
 
 		if (ret) {
 			futex_q_unlock(hb);
+			__release(q->lock_ptr);
 
 			ret = get_user(uval, uaddr);
 			if (ret)
@@ -641,11 +643,13 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
 
 		if (uval != val) {
 			futex_q_unlock(hb);
+			__release(q->lock_ptr);
 			return -EWOULDBLOCK;
 		}
 
 		if (key2 && futex_match(&q->key, key2)) {
 			futex_q_unlock(hb);
+			__release(q->lock_ptr);
 			return -EINVAL;
 		}
 
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index d2254c91450b2..6fcc94ce4ca9d 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -36,7 +36,7 @@ static int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT;
 /*
  * Total number of tasks detected as hung since boot:
  */
-static unsigned long __read_mostly sysctl_hung_task_detect_count;
+static atomic_long_t sysctl_hung_task_detect_count = ATOMIC_LONG_INIT(0);
 
 /*
  * Limit number of tasks checked in a batch.
@@ -223,37 +223,36 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
 }
 #endif
 
-static void check_hung_task(struct task_struct *t, unsigned long timeout,
-		unsigned long prev_detect_count)
+/**
+ * hung_task_info - Print diagnostic details for a hung task
+ * @t: Pointer to the detected hung task.
+ * @timeout: Timeout threshold for detecting hung tasks
+ * @this_round_count: Count of hung tasks detected in the current iteration
+ *
+ * Print structured information about the specified hung task, if warnings
+ * are enabled or if the panic batch threshold is exceeded.
+ */
+static void hung_task_info(struct task_struct *t, unsigned long timeout,
+			   unsigned long this_round_count)
 {
-	unsigned long total_hung_task;
-
-	if (!task_is_hung(t, timeout))
-		return;
-
-	/*
-	 * This counter tracks the total number of tasks detected as hung
-	 * since boot.
-	 */
-	sysctl_hung_task_detect_count++;
-
-	total_hung_task = sysctl_hung_task_detect_count - prev_detect_count;
 	trace_sched_process_hang(t);
 
-	if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) {
+	if (sysctl_hung_task_panic && this_round_count >= sysctl_hung_task_panic) {
 		console_verbose();
 		hung_task_call_panic = true;
 	}
 
 	/*
-	 * Ok, the task did not get scheduled for more than 2 minutes,
-	 * complain:
+	 * The given task did not get scheduled for more than
+	 * CONFIG_DEFAULT_HUNG_TASK_TIMEOUT. Therefore, complain
+	 * accordingly
 	 */
 	if (sysctl_hung_task_warnings || hung_task_call_panic) {
 		if (sysctl_hung_task_warnings > 0)
 			sysctl_hung_task_warnings--;
-		pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
-		       t->comm, t->pid, (jiffies - t->last_switch_time) / HZ);
+		pr_err("INFO: task %s:%d blocked%s for more than %ld seconds.\n",
+		       t->comm, t->pid, t->in_iowait ? " in I/O wait" : "",
+		       (jiffies - t->last_switch_time) / HZ);
 		pr_err("      %s %s %.*s\n",
 			print_tainted(), init_utsname()->release,
 			(int)strcspn(init_utsname()->version, " "),
@@ -297,15 +296,14 @@ static bool rcu_lock_break(struct task_struct *g, struct task_struct *t)
 
 /*
  * Check whether a TASK_UNINTERRUPTIBLE does not get woken up for
- * a really long time (120 seconds). If that happens, print out
- * a warning.
+ * a really long time. If that happens, print out a warning.
  */
 static void check_hung_uninterruptible_tasks(unsigned long timeout)
 {
 	int max_count = sysctl_hung_task_check_count;
 	unsigned long last_break = jiffies;
 	struct task_struct *g, *t;
-	unsigned long prev_detect_count = sysctl_hung_task_detect_count;
+	unsigned long this_round_count;
 	int need_warning = sysctl_hung_task_warnings;
 	unsigned long si_mask = hung_task_si_mask;
 
@@ -316,10 +314,9 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	if (test_taint(TAINT_DIE) || did_panic)
 		return;
 
-
+	this_round_count = 0;
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
-
 		if (!max_count--)
 			goto unlock;
 		if (time_after(jiffies, last_break + HUNG_TASK_LOCK_BREAK)) {
@@ -328,12 +325,22 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 			last_break = jiffies;
 		}
 
-		check_hung_task(t, timeout, prev_detect_count);
+		if (task_is_hung(t, timeout)) {
+			/*
+			 * Increment the global counter so that userspace could
+			 * start migrating tasks ASAP. But count the current
+			 * round separately because userspace could reset
+			 * the global counter at any time.
+			 */
+			atomic_long_inc(&sysctl_hung_task_detect_count);
+			this_round_count++;
+			hung_task_info(t, timeout, this_round_count);
+		}
 	}
  unlock:
 	rcu_read_unlock();
 
-	if (!(sysctl_hung_task_detect_count - prev_detect_count))
+	if (!this_round_count)
 		return;
 
 	if (need_warning || hung_task_call_panic) {
@@ -358,6 +365,46 @@ static long hung_timeout_jiffies(unsigned long last_checked,
 }
 
 #ifdef CONFIG_SYSCTL
+
+/**
+ * proc_dohung_task_detect_count - proc handler for hung_task_detect_count
+ * @table: Pointer to the struct ctl_table definition for this proc entry
+ * @dir: Flag indicating the operation
+ * @buffer: User space buffer for data transfer
+ * @lenp: Pointer to the length of the data being transferred
+ * @ppos: Pointer to the current file offset
+ *
+ * This handler is used for reading the current hung task detection count
+ * and for resetting it to zero when a write operation is performed using a
+ * zero value only.
+ * Return: 0 on success, or a negative error code on failure.
+ */
+static int proc_dohung_task_detect_count(const struct ctl_table *table, int dir,
+					 void *buffer, size_t *lenp, loff_t *ppos)
+{
+	unsigned long detect_count;
+	struct ctl_table proxy_table;
+	int err;
+
+	proxy_table = *table;
+	proxy_table.data = &detect_count;
+
+	if (SYSCTL_KERN_TO_USER(dir))
+		detect_count = atomic_long_read(&sysctl_hung_task_detect_count);
+
+	err = proc_doulongvec_minmax(&proxy_table, dir, buffer, lenp, ppos);
+	if (err < 0)
+		return err;
+
+	if (SYSCTL_USER_TO_KERN(dir)) {
+		if (detect_count)
+			return -EINVAL;
+		atomic_long_set(&sysctl_hung_task_detect_count, 0);
+	}
+
+	return 0;
+}
+
 /*
  * Process updating of timeout sysctl
  */
@@ -438,10 +485,9 @@ static const struct ctl_table hung_task_sysctls[] = {
 	},
 	{
 		.procname	= "hung_task_detect_count",
-		.data		= &sysctl_hung_task_detect_count,
 		.maxlen		= sizeof(unsigned long),
-		.mode		= 0444,
-		.proc_handler	= proc_doulongvec_minmax,
+		.mode		= 0644,
+		.proc_handler	= proc_dohung_task_detect_count,
 	},
 	{
 		.procname	= "hung_task_sys_info",
diff --git a/kernel/irq/matrix.c b/kernel/irq/matrix.c
index 0f79a4abea05e..faafb43a4e611 100644
--- a/kernel/irq/matrix.c
+++ b/kernel/irq/matrix.c
@@ -39,7 +39,7 @@ struct irq_matrix {
 
 /**
  * irq_alloc_matrix - Allocate a irq_matrix structure and initialize it
- * @matrix_bits:	Number of matrix bits must be <= IRQ_MATRIX_BITS
+ * @matrix_bits:	Number of matrix bits
  * @alloc_start:	From which bit the allocation search starts
  * @alloc_end:		At which bit the allocation search ends, i.e first
  *			invalid bit
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 7cb19e6014266..e851e4b37d0ef 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -529,15 +529,6 @@ void __init jump_label_init(void)
 	struct static_key *key = NULL;
 	struct jump_entry *iter;
 
-	/*
-	 * Since we are initializing the static_key.enabled field with
-	 * with the 'raw' int values (to avoid pulling in atomic.h) in
-	 * jump_label.h, let's make sure that is safe. There are only two
-	 * cases to check since we initialize to 0 or 1.
-	 */
-	BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
-	BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
-
 	if (static_key_initialized)
 		return;
 
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index aec2f06858afd..e6f796d43dd70 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -467,6 +467,40 @@ static int append_buildid(char *buffer,   const char *modname,
 
 #endif /* CONFIG_STACKTRACE_BUILD_ID */
 
+#include <linux/mod_lineinfo.h>
+
+bool kallsyms_lookup_lineinfo(unsigned long addr,
+			      const char **file, unsigned int *line)
+{
+	struct lineinfo_table tbl;
+	unsigned long long raw_offset;
+
+	if (!IS_ENABLED(CONFIG_KALLSYMS_LINEINFO) ||
+	    !lineinfo_num_entries || !lineinfo_num_blocks)
+		return false;
+
+	/* Compute offset from _text */
+	if (addr < (unsigned long)_text)
+		return false;
+
+	raw_offset = addr - (unsigned long)_text;
+	if (raw_offset > UINT_MAX)
+		return false;
+
+	tbl.blk_addrs	= lineinfo_block_addrs;
+	tbl.blk_offsets	= lineinfo_block_offsets;
+	tbl.data	= lineinfo_data;
+	tbl.data_size	= lineinfo_data_size;
+	tbl.file_offsets = lineinfo_file_offsets;
+	tbl.filenames	= lineinfo_filenames;
+	tbl.num_entries	= lineinfo_num_entries;
+	tbl.num_blocks	= lineinfo_num_blocks;
+	tbl.num_files	= lineinfo_num_files;
+	tbl.filenames_size = lineinfo_filenames_size;
+
+	return lineinfo_search(&tbl, (unsigned int)raw_offset, file, line);
+}
+
 /* Look up a kernel symbol and return it in a text buffer. */
 static int __sprint_symbol(char *buffer, unsigned long address,
 			   int symbol_offset, int add_offset, int add_buildid)
@@ -497,6 +531,28 @@ static int __sprint_symbol(char *buffer, unsigned long address,
 		len += sprintf(buffer + len, "]");
 	}
 
+	if (IS_ENABLED(CONFIG_KALLSYMS_LINEINFO)) {
+		const char *li_file;
+		unsigned int li_line;
+		bool found = false;
+
+		if (!modname)
+			found = kallsyms_lookup_lineinfo(address,
+							 &li_file, &li_line);
+		else if (IS_ENABLED(CONFIG_KALLSYMS_LINEINFO_MODULES)) {
+			struct module *mod = __module_address(address);
+
+			if (mod)
+				found = module_lookup_lineinfo(mod, address,
+							      &li_file,
+							      &li_line);
+		}
+
+		if (found)
+			len += snprintf(buffer + len, KSYM_SYMBOL_LEN - len,
+					" (%s:%u)", li_file, li_line);
+	}
+
 	return len;
 }
 
@@ -569,6 +625,7 @@ int sprint_backtrace(char *buffer, unsigned long address)
 {
 	return __sprint_symbol(buffer, address, -1, 1, 0);
 }
+EXPORT_SYMBOL_GPL(sprint_backtrace);
 
 /**
  * sprint_backtrace_build_id - Look up a backtrace symbol and return it in a text buffer
@@ -589,6 +646,7 @@ int sprint_backtrace_build_id(char *buffer, unsigned long address)
 {
 	return __sprint_symbol(buffer, address, -1, 1, 1);
 }
+EXPORT_SYMBOL_GPL(sprint_backtrace_build_id);
 
 /* To avoid using get_symbol_offset for every symbol, we carry prefix along. */
 struct kallsym_iter {
diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h
index 81a867dbe57d4..ffe4c658067ec 100644
--- a/kernel/kallsyms_internal.h
+++ b/kernel/kallsyms_internal.h
@@ -15,4 +15,15 @@ extern const u16 kallsyms_token_index[];
 extern const unsigned int kallsyms_markers[];
 extern const u8 kallsyms_seqs_of_names[];
 
+extern const u32 lineinfo_num_entries;
+extern const u32 lineinfo_num_files;
+extern const u32 lineinfo_num_blocks;
+extern const u32 lineinfo_block_addrs[];
+extern const u32 lineinfo_block_offsets[];
+extern const u32 lineinfo_data_size;
+extern const u8  lineinfo_data[];
+extern const u32 lineinfo_file_offsets[];
+extern const u32 lineinfo_filenames_size;
+extern const char lineinfo_filenames[];
+
 #endif // LINUX_KALLSYMS_INTERNAL_H_
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index 2fea396d29b97..a43d2da0fe3e7 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -47,7 +47,6 @@
 #include <asm/page.h>
 #include <asm/sections.h>
 
-#include <crypto/hash.h>
 #include "kexec_internal.h"
 
 atomic_t __kexec_lock = ATOMIC_INIT(0);
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index cc68a3692905d..532f455c5d4f4 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2025 Microsoft Corporation, Mike Rapoport <rppt@kernel.org>
  * Copyright (C) 2025 Google LLC, Changyuan Lyu <changyuanl@google.com>
  * Copyright (C) 2025 Pasha Tatashin <pasha.tatashin@soleen.com>
+ * Copyright (C) 2026 Google LLC, Jason Miu <jasonmiu@google.com>
  */
 
 #define pr_fmt(fmt) "KHO: " fmt
@@ -13,8 +14,10 @@
 #include <linux/cma.h>
 #include <linux/kmemleak.h>
 #include <linux/count_zeros.h>
+#include <linux/kasan.h>
 #include <linux/kexec.h>
 #include <linux/kexec_handover.h>
+#include <linux/kho_radix_tree.h>
 #include <linux/kho/abi/kexec_handover.h>
 #include <linux/libfdt.h>
 #include <linux/list.h>
@@ -64,163 +67,316 @@ static int __init kho_parse_enable(char *p)
 }
 early_param("kho", kho_parse_enable);
 
-/*
- * Keep track of memory that is to be preserved across KHO.
- *
- * The serializing side uses two levels of xarrays to manage chunks of per-order
- * PAGE_SIZE byte bitmaps. For instance if PAGE_SIZE = 4096, the entire 1G order
- * of a 8TB system would fit inside a single 4096 byte bitmap. For order 0
- * allocations each bitmap will cover 128M of address space. Thus, for 16G of
- * memory at most 512K of bitmap memory will be needed for order 0.
- *
- * This approach is fully incremental, as the serialization progresses folios
- * can continue be aggregated to the tracker. The final step, immediately prior
- * to kexec would serialize the xarray information into a linked list for the
- * successor kernel to parse.
- */
-
-#define PRESERVE_BITS (PAGE_SIZE * 8)
-
-struct kho_mem_phys_bits {
-	DECLARE_BITMAP(preserve, PRESERVE_BITS);
-};
-
-static_assert(sizeof(struct kho_mem_phys_bits) == PAGE_SIZE);
-
-struct kho_mem_phys {
-	/*
-	 * Points to kho_mem_phys_bits, a sparse bitmap array. Each bit is sized
-	 * to order.
-	 */
-	struct xarray phys_bits;
-};
-
-struct kho_mem_track {
-	/* Points to kho_mem_phys, each order gets its own bitmap tree */
-	struct xarray orders;
-};
-
-struct khoser_mem_chunk;
-
 struct kho_out {
 	void *fdt;
-	bool finalized;
-	struct mutex lock; /* protects KHO FDT finalization */
+	struct mutex lock; /* protects KHO FDT */
 
-	struct kho_mem_track track;
+	struct kho_radix_tree radix_tree;
 	struct kho_debugfs dbg;
 };
 
 static struct kho_out kho_out = {
 	.lock = __MUTEX_INITIALIZER(kho_out.lock),
-	.track = {
-		.orders = XARRAY_INIT(kho_out.track.orders, 0),
+	.radix_tree = {
+		.lock = __MUTEX_INITIALIZER(kho_out.radix_tree.lock),
 	},
-	.finalized = false,
 };
 
-static void *xa_load_or_alloc(struct xarray *xa, unsigned long index)
+/**
+ * kho_radix_encode_key - Encodes a physical address and order into a radix key.
+ * @phys: The physical address of the page.
+ * @order: The order of the page.
+ *
+ * This function combines a page's physical address and its order into a
+ * single unsigned long, which is used as a key for all radix tree
+ * operations.
+ *
+ * Return: The encoded unsigned long radix key.
+ */
+static unsigned long kho_radix_encode_key(phys_addr_t phys, unsigned int order)
 {
-	void *res = xa_load(xa, index);
+	/* Order bits part */
+	unsigned long h = 1UL << (KHO_ORDER_0_LOG2 - order);
+	/* Shifted physical address part */
+	unsigned long l = phys >> (PAGE_SHIFT + order);
 
-	if (res)
-		return res;
+	return h | l;
+}
 
-	void *elm __free(free_page) = (void *)get_zeroed_page(GFP_KERNEL);
+/**
+ * kho_radix_decode_key - Decodes a radix key back into a physical address and order.
+ * @key: The unsigned long key to decode.
+ * @order: An output parameter, a pointer to an unsigned int where the decoded
+ *         page order will be stored.
+ *
+ * This function reverses the encoding performed by kho_radix_encode_key(),
+ * extracting the original physical address and page order from a given key.
+ *
+ * Return: The decoded physical address.
+ */
+static phys_addr_t kho_radix_decode_key(unsigned long key, unsigned int *order)
+{
+	unsigned int order_bit = fls64(key);
+	phys_addr_t phys;
 
-	if (!elm)
-		return ERR_PTR(-ENOMEM);
+	/* order_bit is numbered starting at 1 from fls64 */
+	*order = KHO_ORDER_0_LOG2 - order_bit + 1;
+	/* The order is discarded by the shift */
+	phys = key << (PAGE_SHIFT + *order);
 
-	if (WARN_ON(kho_scratch_overlap(virt_to_phys(elm), PAGE_SIZE)))
-		return ERR_PTR(-EINVAL);
+	return phys;
+}
 
-	res = xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);
-	if (xa_is_err(res))
-		return ERR_PTR(xa_err(res));
-	else if (res)
-		return res;
+static unsigned long kho_radix_get_bitmap_index(unsigned long key)
+{
+	return key % (1 << KHO_BITMAP_SIZE_LOG2);
+}
+
+static unsigned long kho_radix_get_table_index(unsigned long key,
+					       unsigned int level)
+{
+	int s;
 
-	return no_free_ptr(elm);
+	s = ((level - 1) * KHO_TABLE_SIZE_LOG2) + KHO_BITMAP_SIZE_LOG2;
+	return (key >> s) % (1 << KHO_TABLE_SIZE_LOG2);
 }
 
-static void __kho_unpreserve_order(struct kho_mem_track *track, unsigned long pfn,
-				   unsigned int order)
+/**
+ * kho_radix_add_page - Marks a page as preserved in the radix tree.
+ * @tree: The KHO radix tree.
+ * @pfn: The page frame number of the page to preserve.
+ * @order: The order of the page.
+ *
+ * This function traverses the radix tree based on the key derived from @pfn
+ * and @order. It sets the corresponding bit in the leaf bitmap to mark the
+ * page for preservation. If intermediate nodes do not exist along the path,
+ * they are allocated and added to the tree.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+int kho_radix_add_page(struct kho_radix_tree *tree,
+		       unsigned long pfn, unsigned int order)
 {
-	struct kho_mem_phys_bits *bits;
-	struct kho_mem_phys *physxa;
-	const unsigned long pfn_high = pfn >> order;
+	/* Newly allocated nodes for error cleanup */
+	struct kho_radix_node *intermediate_nodes[KHO_TREE_MAX_DEPTH] = { 0 };
+	unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order);
+	struct kho_radix_node *anchor_node = NULL;
+	struct kho_radix_node *node = tree->root;
+	struct kho_radix_node *new_node;
+	unsigned int i, idx, anchor_idx;
+	struct kho_radix_leaf *leaf;
+	int err = 0;
 
-	physxa = xa_load(&track->orders, order);
-	if (WARN_ON_ONCE(!physxa))
-		return;
+	if (WARN_ON_ONCE(!tree->root))
+		return -EINVAL;
 
-	bits = xa_load(&physxa->phys_bits, pfn_high / PRESERVE_BITS);
-	if (WARN_ON_ONCE(!bits))
-		return;
+	might_sleep();
+
+	guard(mutex)(&tree->lock);
+
+	/* Go from high levels to low levels */
+	for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
+		idx = kho_radix_get_table_index(key, i);
+
+		if (node->table[idx]) {
+			node = phys_to_virt(node->table[idx]);
+			continue;
+		}
+
+		/* Next node is empty, create a new node for it */
+		new_node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL);
+		if (!new_node) {
+			err = -ENOMEM;
+			goto err_free_nodes;
+		}
+
+		node->table[idx] = virt_to_phys(new_node);
+
+		/*
+		 * Capture the node where the new branch starts for cleanup
+		 * if allocation fails.
+		 */
+		if (!anchor_node) {
+			anchor_node = node;
+			anchor_idx = idx;
+		}
+		intermediate_nodes[i] = new_node;
+
+		node = new_node;
+	}
+
+	/* Handle the leaf level bitmap (level 0) */
+	idx = kho_radix_get_bitmap_index(key);
+	leaf = (struct kho_radix_leaf *)node;
+	__set_bit(idx, leaf->bitmap);
 
-	clear_bit(pfn_high % PRESERVE_BITS, bits->preserve);
+	return 0;
+
+err_free_nodes:
+	for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
+		if (intermediate_nodes[i])
+			free_page((unsigned long)intermediate_nodes[i]);
+	}
+	if (anchor_node)
+		anchor_node->table[anchor_idx] = 0;
+
+	return err;
 }
+EXPORT_SYMBOL_GPL(kho_radix_add_page);
 
-static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn,
-			     unsigned long end_pfn)
+/**
+ * kho_radix_del_page - Removes a page's preservation status from the radix tree.
+ * @tree: The KHO radix tree.
+ * @pfn: The page frame number of the page to unpreserve.
+ * @order: The order of the page.
+ *
+ * This function traverses the radix tree and clears the bit corresponding to
+ * the page, effectively removing its "preserved" status. It does not free
+ * the tree's intermediate nodes, even if they become empty.
+ */
+void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
+			unsigned int order)
 {
-	unsigned int order;
+	unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order);
+	struct kho_radix_node *node = tree->root;
+	struct kho_radix_leaf *leaf;
+	unsigned int i, idx;
 
-	while (pfn < end_pfn) {
-		order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
+	if (WARN_ON_ONCE(!tree->root))
+		return;
 
-		__kho_unpreserve_order(track, pfn, order);
+	might_sleep();
 
-		pfn += 1 << order;
+	guard(mutex)(&tree->lock);
+
+	/* Go from high levels to low levels */
+	for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
+		idx = kho_radix_get_table_index(key, i);
+
+		/*
+		 * Attempting to delete a page that has not been preserved,
+		 * return with a warning.
+		 */
+		if (WARN_ON(!node->table[idx]))
+			return;
+
+		node = phys_to_virt(node->table[idx]);
 	}
+
+	/* Handle the leaf level bitmap (level 0) */
+	leaf = (struct kho_radix_leaf *)node;
+	idx = kho_radix_get_bitmap_index(key);
+	__clear_bit(idx, leaf->bitmap);
 }
+EXPORT_SYMBOL_GPL(kho_radix_del_page);
 
-static int __kho_preserve_order(struct kho_mem_track *track, unsigned long pfn,
-				unsigned int order)
+static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf,
+			       unsigned long key,
+			       kho_radix_tree_walk_callback_t cb)
 {
-	struct kho_mem_phys_bits *bits;
-	struct kho_mem_phys *physxa, *new_physxa;
-	const unsigned long pfn_high = pfn >> order;
+	unsigned long *bitmap = (unsigned long *)leaf;
+	unsigned int order;
+	phys_addr_t phys;
+	unsigned int i;
+	int err;
 
-	might_sleep();
-	physxa = xa_load(&track->orders, order);
-	if (!physxa) {
-		int err;
+	for_each_set_bit(i, bitmap, PAGE_SIZE * BITS_PER_BYTE) {
+		phys = kho_radix_decode_key(key | i, &order);
+		err = cb(phys, order);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
 
-		new_physxa = kzalloc_obj(*physxa);
-		if (!new_physxa)
-			return -ENOMEM;
+static int __kho_radix_walk_tree(struct kho_radix_node *root,
+				 unsigned int level, unsigned long start,
+				 kho_radix_tree_walk_callback_t cb)
+{
+	struct kho_radix_node *node;
+	struct kho_radix_leaf *leaf;
+	unsigned long key, i;
+	unsigned int shift;
+	int err;
+
+	for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) {
+		if (!root->table[i])
+			continue;
 
-		xa_init(&new_physxa->phys_bits);
-		physxa = xa_cmpxchg(&track->orders, order, NULL, new_physxa,
-				    GFP_KERNEL);
+		shift = ((level - 1) * KHO_TABLE_SIZE_LOG2) +
+			KHO_BITMAP_SIZE_LOG2;
+		key = start | (i << shift);
 
-		err = xa_err(physxa);
-		if (err || physxa) {
-			xa_destroy(&new_physxa->phys_bits);
-			kfree(new_physxa);
+		node = phys_to_virt(root->table[i]);
 
-			if (err)
-				return err;
+		if (level == 1) {
+			/*
+			 * we are at level 1,
+			 * node is pointing to the level 0 bitmap.
+			 */
+			leaf = (struct kho_radix_leaf *)node;
+			err = kho_radix_walk_leaf(leaf, key, cb);
 		} else {
-			physxa = new_physxa;
+			err  = __kho_radix_walk_tree(node, level - 1,
+						     key, cb);
 		}
+
+		if (err)
+			return err;
 	}
 
-	bits = xa_load_or_alloc(&physxa->phys_bits, pfn_high / PRESERVE_BITS);
-	if (IS_ERR(bits))
-		return PTR_ERR(bits);
+	return 0;
+}
+
+/**
+ * kho_radix_walk_tree - Traverses the radix tree and calls a callback for each preserved page.
+ * @tree: A pointer to the KHO radix tree to walk.
+ * @cb: A callback function of type kho_radix_tree_walk_callback_t that will be
+ *      invoked for each preserved page found in the tree. The callback receives
+ *      the physical address and order of the preserved page.
+ *
+ * This function walks the radix tree, searching from the specified top level
+ * down to the lowest level (level 0). For each preserved page found, it invokes
+ * the provided callback, passing the page's physical address and order.
+ *
+ * Return: 0 if the walk completed the specified tree, or the non-zero return
+ *         value from the callback that stopped the walk.
+ */
+int kho_radix_walk_tree(struct kho_radix_tree *tree,
+			kho_radix_tree_walk_callback_t cb)
+{
+	if (WARN_ON_ONCE(!tree->root))
+		return -EINVAL;
+
+	guard(mutex)(&tree->lock);
 
-	set_bit(pfn_high % PRESERVE_BITS, bits->preserve);
+	return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb);
+}
+EXPORT_SYMBOL_GPL(kho_radix_walk_tree);
 
-	return 0;
+static void __kho_unpreserve(struct kho_radix_tree *tree,
+			     unsigned long pfn, unsigned long end_pfn)
+{
+	unsigned int order;
+
+	while (pfn < end_pfn) {
+		order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
+
+		kho_radix_del_page(tree, pfn, order);
+
+		pfn += 1 << order;
+	}
 }
 
 /* For physically contiguous 0-order pages. */
 static void kho_init_pages(struct page *page, unsigned long nr_pages)
 {
-	for (unsigned long i = 0; i < nr_pages; i++)
+	for (unsigned long i = 0; i < nr_pages; i++) {
 		set_page_count(page + i, 1);
+		/* Clear each page's codetag to avoid accounting mismatch. */
+		clear_page_tag_ref(page + i);
+	}
 }
 
 static void kho_init_folio(struct page *page, unsigned int order)
@@ -229,6 +385,8 @@ static void kho_init_folio(struct page *page, unsigned int order)
 
 	/* Head page gets refcount of 1. */
 	set_page_count(page, 1);
+	/* Clear head page's codetag to avoid accounting mismatch. */
+	clear_page_tag_ref(page);
 
 	/* For higher order folios, tail pages get a page count of zero. */
 	for (unsigned long i = 1; i < nr_pages; i++)
@@ -253,7 +411,7 @@ static struct page *kho_restore_page(phys_addr_t phys, bool is_folio)
 	 * check also implicitly makes sure phys is order-aligned since for
 	 * non-order-aligned phys addresses, magic will never be set.
 	 */
-	if (WARN_ON_ONCE(info.magic != KHO_PAGE_MAGIC || info.order > MAX_PAGE_ORDER))
+	if (WARN_ON_ONCE(info.magic != KHO_PAGE_MAGIC))
 		return NULL;
 	nr_pages = (1 << info.order);
 
@@ -265,14 +423,6 @@ static struct page *kho_restore_page(phys_addr_t phys, bool is_folio)
 	else
 		kho_init_pages(page, nr_pages);
 
-	/* Always mark headpage's codetag as empty to avoid accounting mismatch */
-	clear_page_tag_ref(page);
-	if (!is_folio) {
-		/* Also do that for the non-compound tail pages */
-		for (unsigned int i = 1; i < nr_pages; i++)
-			clear_page_tag_ref(page + i);
-	}
-
 	adjust_managed_page_count(page, nr_pages);
 	return page;
 }
@@ -321,161 +471,24 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages)
 }
 EXPORT_SYMBOL_GPL(kho_restore_pages);
 
-/* Serialize and deserialize struct kho_mem_phys across kexec
- *
- * Record all the bitmaps in a linked list of pages for the next kernel to
- * process. Each chunk holds bitmaps of the same order and each block of bitmaps
- * starts at a given physical address. This allows the bitmaps to be sparse. The
- * xarray is used to store them in a tree while building up the data structure,
- * but the KHO successor kernel only needs to process them once in order.
- *
- * All of this memory is normal kmalloc() memory and is not marked for
- * preservation. The successor kernel will remain isolated to the scratch space
- * until it completes processing this list. Once processed all the memory
- * storing these ranges will be marked as free.
- */
-
-struct khoser_mem_bitmap_ptr {
-	phys_addr_t phys_start;
-	DECLARE_KHOSER_PTR(bitmap, struct kho_mem_phys_bits *);
-};
-
-struct khoser_mem_chunk_hdr {
-	DECLARE_KHOSER_PTR(next, struct khoser_mem_chunk *);
-	unsigned int order;
-	unsigned int num_elms;
-};
-
-#define KHOSER_BITMAP_SIZE                                   \
-	((PAGE_SIZE - sizeof(struct khoser_mem_chunk_hdr)) / \
-	 sizeof(struct khoser_mem_bitmap_ptr))
-
-struct khoser_mem_chunk {
-	struct khoser_mem_chunk_hdr hdr;
-	struct khoser_mem_bitmap_ptr bitmaps[KHOSER_BITMAP_SIZE];
-};
-
-static_assert(sizeof(struct khoser_mem_chunk) == PAGE_SIZE);
-
-static struct khoser_mem_chunk *new_chunk(struct khoser_mem_chunk *cur_chunk,
-					  unsigned long order)
-{
-	struct khoser_mem_chunk *chunk __free(free_page) = NULL;
-
-	chunk = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!chunk)
-		return ERR_PTR(-ENOMEM);
-
-	if (WARN_ON(kho_scratch_overlap(virt_to_phys(chunk), PAGE_SIZE)))
-		return ERR_PTR(-EINVAL);
-
-	chunk->hdr.order = order;
-	if (cur_chunk)
-		KHOSER_STORE_PTR(cur_chunk->hdr.next, chunk);
-	return no_free_ptr(chunk);
-}
-
-static void kho_mem_ser_free(struct khoser_mem_chunk *first_chunk)
+static int __init kho_preserved_memory_reserve(phys_addr_t phys,
+					       unsigned int order)
 {
-	struct khoser_mem_chunk *chunk = first_chunk;
-
-	while (chunk) {
-		struct khoser_mem_chunk *tmp = chunk;
-
-		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
-		free_page((unsigned long)tmp);
-	}
-}
-
-/*
- *  Update memory map property, if old one is found discard it via
- *  kho_mem_ser_free().
- */
-static void kho_update_memory_map(struct khoser_mem_chunk *first_chunk)
-{
-	void *ptr;
-	u64 phys;
-
-	ptr = fdt_getprop_w(kho_out.fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, NULL);
-
-	/* Check and discard previous memory map */
-	phys = get_unaligned((u64 *)ptr);
-	if (phys)
-		kho_mem_ser_free((struct khoser_mem_chunk *)phys_to_virt(phys));
-
-	/* Update with the new value */
-	phys = first_chunk ? (u64)virt_to_phys(first_chunk) : 0;
-	put_unaligned(phys, (u64 *)ptr);
-}
-
-static int kho_mem_serialize(struct kho_out *kho_out)
-{
-	struct khoser_mem_chunk *first_chunk = NULL;
-	struct khoser_mem_chunk *chunk = NULL;
-	struct kho_mem_phys *physxa;
-	unsigned long order;
-	int err = -ENOMEM;
-
-	xa_for_each(&kho_out->track.orders, order, physxa) {
-		struct kho_mem_phys_bits *bits;
-		unsigned long phys;
-
-		chunk = new_chunk(chunk, order);
-		if (IS_ERR(chunk)) {
-			err = PTR_ERR(chunk);
-			goto err_free;
-		}
-
-		if (!first_chunk)
-			first_chunk = chunk;
-
-		xa_for_each(&physxa->phys_bits, phys, bits) {
-			struct khoser_mem_bitmap_ptr *elm;
+	union kho_page_info info;
+	struct page *page;
+	u64 sz;
 
-			if (chunk->hdr.num_elms == ARRAY_SIZE(chunk->bitmaps)) {
-				chunk = new_chunk(chunk, order);
-				if (IS_ERR(chunk)) {
-					err = PTR_ERR(chunk);
-					goto err_free;
-				}
-			}
+	sz = 1 << (order + PAGE_SHIFT);
+	page = phys_to_page(phys);
 
-			elm = &chunk->bitmaps[chunk->hdr.num_elms];
-			chunk->hdr.num_elms++;
-			elm->phys_start = (phys * PRESERVE_BITS)
-					  << (order + PAGE_SHIFT);
-			KHOSER_STORE_PTR(elm->bitmap, bits);
-		}
-	}
-
-	kho_update_memory_map(first_chunk);
+	/* Reserve the memory preserved in KHO in memblock */
+	memblock_reserve(phys, sz);
+	memblock_reserved_mark_noinit(phys, sz);
+	info.magic = KHO_PAGE_MAGIC;
+	info.order = order;
+	page->private = info.page_private;
 
 	return 0;
-
-err_free:
-	kho_mem_ser_free(first_chunk);
-	return err;
-}
-
-static void __init deserialize_bitmap(unsigned int order,
-				      struct khoser_mem_bitmap_ptr *elm)
-{
-	struct kho_mem_phys_bits *bitmap = KHOSER_LOAD_PTR(elm->bitmap);
-	unsigned long bit;
-
-	for_each_set_bit(bit, bitmap->preserve, PRESERVE_BITS) {
-		int sz = 1 << (order + PAGE_SHIFT);
-		phys_addr_t phys =
-			elm->phys_start + (bit << (order + PAGE_SHIFT));
-		struct page *page = phys_to_page(phys);
-		union kho_page_info info;
-
-		memblock_reserve(phys, sz);
-		memblock_reserved_mark_noinit(phys, sz);
-		info.magic = KHO_PAGE_MAGIC;
-		info.order = order;
-		page->private = info.page_private;
-	}
 }
 
 /* Returns physical address of the preserved memory map from FDT */
@@ -486,25 +499,13 @@ static phys_addr_t __init kho_get_mem_map_phys(const void *fdt)
 
 	mem_ptr = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len);
 	if (!mem_ptr || len != sizeof(u64)) {
-		pr_err("failed to get preserved memory bitmaps\n");
+		pr_err("failed to get preserved memory map\n");
 		return 0;
 	}
 
 	return get_unaligned((const u64 *)mem_ptr);
 }
 
-static void __init kho_mem_deserialize(struct khoser_mem_chunk *chunk)
-{
-	while (chunk) {
-		unsigned int i;
-
-		for (i = 0; i != chunk->hdr.num_elms; i++)
-			deserialize_bitmap(chunk->hdr.order,
-					   &chunk->bitmaps[i]);
-		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
-	}
-}
-
 /*
  * With KHO enabled, memory can become fragmented because KHO regions may
  * be anywhere in physical address space. The scratch regions give us a
@@ -815,14 +816,14 @@ EXPORT_SYMBOL_GPL(kho_remove_subtree);
  */
 int kho_preserve_folio(struct folio *folio)
 {
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	const unsigned long pfn = folio_pfn(folio);
 	const unsigned int order = folio_order(folio);
-	struct kho_mem_track *track = &kho_out.track;
 
 	if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
 		return -EINVAL;
 
-	return __kho_preserve_order(track, pfn, order);
+	return kho_radix_add_page(tree, pfn, order);
 }
 EXPORT_SYMBOL_GPL(kho_preserve_folio);
 
@@ -836,11 +837,11 @@ EXPORT_SYMBOL_GPL(kho_preserve_folio);
  */
 void kho_unpreserve_folio(struct folio *folio)
 {
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	const unsigned long pfn = folio_pfn(folio);
 	const unsigned int order = folio_order(folio);
-	struct kho_mem_track *track = &kho_out.track;
 
-	__kho_unpreserve_order(track, pfn, order);
+	kho_radix_del_page(tree, pfn, order);
 }
 EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
 
@@ -856,7 +857,7 @@ EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
  */
 int kho_preserve_pages(struct page *page, unsigned long nr_pages)
 {
-	struct kho_mem_track *track = &kho_out.track;
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	const unsigned long start_pfn = page_to_pfn(page);
 	const unsigned long end_pfn = start_pfn + nr_pages;
 	unsigned long pfn = start_pfn;
@@ -869,10 +870,18 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
 	}
 
 	while (pfn < end_pfn) {
-		const unsigned int order =
+		unsigned int order =
 			min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
 
-		err = __kho_preserve_order(track, pfn, order);
+		/*
+		 * Make sure all the pages in a single preservation are in the
+		 * same NUMA node. The restore machinery can not cope with a
+		 * preservation spanning multiple NUMA nodes.
+		 */
+		while (pfn_to_nid(pfn) != pfn_to_nid(pfn + (1UL << order) - 1))
+			order--;
+
+		err = kho_radix_add_page(tree, pfn, order);
 		if (err) {
 			failed_pfn = pfn;
 			break;
@@ -882,7 +891,7 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
 	}
 
 	if (err)
-		__kho_unpreserve(track, start_pfn, failed_pfn);
+		__kho_unpreserve(tree, start_pfn, failed_pfn);
 
 	return err;
 }
@@ -900,11 +909,11 @@ EXPORT_SYMBOL_GPL(kho_preserve_pages);
  */
 void kho_unpreserve_pages(struct page *page, unsigned long nr_pages)
 {
-	struct kho_mem_track *track = &kho_out.track;
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	const unsigned long start_pfn = page_to_pfn(page);
 	const unsigned long end_pfn = start_pfn + nr_pages;
 
-	__kho_unpreserve(track, start_pfn, end_pfn);
+	__kho_unpreserve(tree, start_pfn, end_pfn);
 }
 EXPORT_SYMBOL_GPL(kho_unpreserve_pages);
 
@@ -963,14 +972,14 @@ static struct kho_vmalloc_chunk *new_vmalloc_chunk(struct kho_vmalloc_chunk *cur
 static void kho_vmalloc_unpreserve_chunk(struct kho_vmalloc_chunk *chunk,
 					 unsigned short order)
 {
-	struct kho_mem_track *track = &kho_out.track;
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	unsigned long pfn = PHYS_PFN(virt_to_phys(chunk));
 
-	__kho_unpreserve(track, pfn, pfn + 1);
+	__kho_unpreserve(tree, pfn, pfn + 1);
 
 	for (int i = 0; i < ARRAY_SIZE(chunk->phys) && chunk->phys[i]; i++) {
 		pfn = PHYS_PFN(chunk->phys[i]);
-		__kho_unpreserve(track, pfn, pfn + (1 << order));
+		__kho_unpreserve(tree, pfn, pfn + (1 << order));
 	}
 }
 
@@ -1077,6 +1086,7 @@ EXPORT_SYMBOL_GPL(kho_unpreserve_vmalloc);
 void *kho_restore_vmalloc(const struct kho_vmalloc *preservation)
 {
 	struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first);
+	kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_PROT_NORMAL;
 	unsigned int align, order, shift, vm_flags;
 	unsigned long total_pages, contig_pages;
 	unsigned long addr, size;
@@ -1128,7 +1138,8 @@ void *kho_restore_vmalloc(const struct kho_vmalloc *preservation)
 		goto err_free_pages_array;
 
 	area = __get_vm_area_node(total_pages * PAGE_SIZE, align, shift,
-				  vm_flags, VMALLOC_START, VMALLOC_END,
+				  vm_flags | VM_UNINITIALIZED,
+				  VMALLOC_START, VMALLOC_END,
 				  NUMA_NO_NODE, GFP_KERNEL,
 				  __builtin_return_address(0));
 	if (!area)
@@ -1143,6 +1154,13 @@ void *kho_restore_vmalloc(const struct kho_vmalloc *preservation)
 	area->nr_pages = total_pages;
 	area->pages = pages;
 
+	if (vm_flags & VM_ALLOC)
+		kasan_flags |= KASAN_VMALLOC_VM_ALLOC;
+
+	area->addr = kasan_unpoison_vmalloc(area->addr, total_pages * PAGE_SIZE,
+					    kasan_flags);
+	clear_vm_uninitialized_flag(area);
+
 	return area->addr;
 
 err_free_vm_area:
@@ -1239,33 +1257,9 @@ void kho_restore_free(void *mem)
 }
 EXPORT_SYMBOL_GPL(kho_restore_free);
 
-int kho_finalize(void)
-{
-	int ret;
-
-	if (!kho_enable)
-		return -EOPNOTSUPP;
-
-	guard(mutex)(&kho_out.lock);
-	ret = kho_mem_serialize(&kho_out);
-	if (ret)
-		return ret;
-
-	kho_out.finalized = true;
-
-	return 0;
-}
-
-bool kho_finalized(void)
-{
-	guard(mutex)(&kho_out.lock);
-	return kho_out.finalized;
-}
-
 struct kho_in {
 	phys_addr_t fdt_phys;
 	phys_addr_t scratch_phys;
-	phys_addr_t mem_map_phys;
 	struct kho_debugfs dbg;
 };
 
@@ -1333,18 +1327,46 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys)
 }
 EXPORT_SYMBOL_GPL(kho_retrieve_subtree);
 
+static int __init kho_mem_retrieve(const void *fdt)
+{
+	struct kho_radix_tree tree;
+	const phys_addr_t *mem;
+	int len;
+
+	/* Retrieve the KHO radix tree from passed-in FDT. */
+	mem = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len);
+
+	if (!mem || len != sizeof(*mem)) {
+		pr_err("failed to get preserved KHO memory tree\n");
+		return -ENOENT;
+	}
+
+	if (!*mem)
+		return -EINVAL;
+
+	tree.root = phys_to_virt(*mem);
+	mutex_init(&tree.lock);
+	return kho_radix_walk_tree(&tree, kho_preserved_memory_reserve);
+}
+
 static __init int kho_out_fdt_setup(void)
 {
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	void *root = kho_out.fdt;
-	u64 empty_mem_map = 0;
+	u64 preserved_mem_tree_pa;
 	int err;
 
 	err = fdt_create(root, PAGE_SIZE);
 	err |= fdt_finish_reservemap(root);
 	err |= fdt_begin_node(root, "");
 	err |= fdt_property_string(root, "compatible", KHO_FDT_COMPATIBLE);
-	err |= fdt_property(root, KHO_FDT_MEMORY_MAP_PROP_NAME, &empty_mem_map,
-			    sizeof(empty_mem_map));
+
+	preserved_mem_tree_pa = virt_to_phys(tree->root);
+
+	err |= fdt_property(root, KHO_FDT_MEMORY_MAP_PROP_NAME,
+			    &preserved_mem_tree_pa,
+			    sizeof(preserved_mem_tree_pa));
+
 	err |= fdt_end_node(root);
 	err |= fdt_finish(root);
 
@@ -1353,16 +1375,23 @@ static __init int kho_out_fdt_setup(void)
 
 static __init int kho_init(void)
 {
+	struct kho_radix_tree *tree = &kho_out.radix_tree;
 	const void *fdt = kho_get_fdt();
 	int err = 0;
 
 	if (!kho_enable)
 		return 0;
 
+	tree->root = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!tree->root) {
+		err = -ENOMEM;
+		goto err_free_scratch;
+	}
+
 	kho_out.fdt = kho_alloc_preserve(PAGE_SIZE);
 	if (IS_ERR(kho_out.fdt)) {
 		err = PTR_ERR(kho_out.fdt);
-		goto err_free_scratch;
+		goto err_free_kho_radix_tree_root;
 	}
 
 	err = kho_debugfs_init();
@@ -1408,6 +1437,9 @@ static __init int kho_init(void)
 
 err_free_fdt:
 	kho_unpreserve_free(kho_out.fdt);
+err_free_kho_radix_tree_root:
+	kfree(tree->root);
+	tree->root = NULL;
 err_free_scratch:
 	kho_out.fdt = NULL;
 	for (int i = 0; i < kho_scratch_cnt; i++) {
@@ -1447,10 +1479,12 @@ static void __init kho_release_scratch(void)
 
 void __init kho_memory_init(void)
 {
-	if (kho_in.mem_map_phys) {
+	if (kho_in.scratch_phys) {
 		kho_scratch = phys_to_virt(kho_in.scratch_phys);
 		kho_release_scratch();
-		kho_mem_deserialize(phys_to_virt(kho_in.mem_map_phys));
+
+		if (kho_mem_retrieve(kho_get_fdt()))
+			kho_in.fdt_phys = 0;
 	} else {
 		kho_reserve_scratch();
 	}
@@ -1528,7 +1562,6 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
 
 	kho_in.fdt_phys = fdt_phys;
 	kho_in.scratch_phys = scratch_phys;
-	kho_in.mem_map_phys = mem_map_phys;
 	kho_scratch_cnt = scratch_cnt;
 
 	populated = true;
diff --git a/kernel/liveupdate/kexec_handover_debugfs.c b/kernel/liveupdate/kexec_handover_debugfs.c
index 2f93939168ab7..acf3682226824 100644
--- a/kernel/liveupdate/kexec_handover_debugfs.c
+++ b/kernel/liveupdate/kexec_handover_debugfs.c
@@ -13,6 +13,7 @@
 #include <linux/io.h>
 #include <linux/libfdt.h>
 #include <linux/mm.h>
+#include <linux/kho/abi/kexec_handover.h>
 #include "kexec_handover_internal.h"
 
 static struct dentry *debugfs_root;
@@ -75,24 +76,6 @@ void kho_debugfs_fdt_remove(struct kho_debugfs *dbg, void *fdt)
 	}
 }
 
-static int kho_out_finalize_get(void *data, u64 *val)
-{
-	*val = kho_finalized();
-
-	return 0;
-}
-
-static int kho_out_finalize_set(void *data, u64 val)
-{
-	if (val)
-		return kho_finalize();
-	else
-		return -EINVAL;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(kho_out_finalize_fops, kho_out_finalize_get,
-			 kho_out_finalize_set, "%llu\n");
-
 static int scratch_phys_show(struct seq_file *m, void *v)
 {
 	for (int i = 0; i < kho_scratch_cnt; i++)
@@ -139,7 +122,7 @@ __init void kho_in_debugfs_init(struct kho_debugfs *dbg, const void *fdt)
 		const char *name = fdt_get_name(fdt, child, NULL);
 		const u64 *fdt_phys;
 
-		fdt_phys = fdt_getprop(fdt, child, "fdt", &len);
+		fdt_phys = fdt_getprop(fdt, child, KHO_FDT_SUB_TREE_PROP_NAME, &len);
 		if (!fdt_phys)
 			continue;
 		if (len != sizeof(*fdt_phys)) {
@@ -198,11 +181,6 @@ __init int kho_out_debugfs_init(struct kho_debugfs *dbg)
 	if (IS_ERR(f))
 		goto err_rmdir;
 
-	f = debugfs_create_file("finalize", 0600, dir, NULL,
-				&kho_out_finalize_fops);
-	if (IS_ERR(f))
-		goto err_rmdir;
-
 	dbg->dir = dir;
 	dbg->sub_fdt_dir = sub_fdt_dir;
 	return 0;
diff --git a/kernel/liveupdate/kexec_handover_internal.h b/kernel/liveupdate/kexec_handover_internal.h
index 0202c85ad14f9..9a832a35254cf 100644
--- a/kernel/liveupdate/kexec_handover_internal.h
+++ b/kernel/liveupdate/kexec_handover_internal.h
@@ -22,9 +22,6 @@ struct kho_debugfs {};
 extern struct kho_scratch *kho_scratch;
 extern unsigned int kho_scratch_cnt;
 
-bool kho_finalized(void);
-int kho_finalize(void);
-
 #ifdef CONFIG_KEXEC_HANDOVER_DEBUGFS
 int kho_debugfs_init(void);
 void kho_in_debugfs_init(struct kho_debugfs *dbg, const void *fdt);
diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c
index dda7bb57d421c..84ac728d63baa 100644
--- a/kernel/liveupdate/luo_core.c
+++ b/kernel/liveupdate/luo_core.c
@@ -230,17 +230,7 @@ int liveupdate_reboot(void)
 
 	luo_flb_serialize();
 
-	err = kho_finalize();
-	if (err) {
-		pr_err("kho_finalize failed %d\n", err);
-		/*
-		 * kho_finalize() may return libfdt errors, to aboid passing to
-		 * userspace unknown errors, change this to EAGAIN.
-		 */
-		err = -EAGAIN;
-	}
-
-	return err;
+	return 0;
 }
 
 /**
diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c
index 5acee4174bf00..a38ea49758249 100644
--- a/kernel/liveupdate/luo_file.c
+++ b/kernel/liveupdate/luo_file.c
@@ -112,6 +112,7 @@
 #include <linux/string.h>
 #include "luo_internal.h"
 
+static DECLARE_RWSEM(luo_file_handler_lock);
 static LIST_HEAD(luo_file_handler_list);
 
 /* 2 4K pages, give space for 128 files per file_set */
@@ -277,10 +278,12 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
 		goto  err_fput;
 
 	err = -ENOENT;
-	list_private_for_each_entry(fh, &luo_file_handler_list, list) {
-		if (fh->ops->can_preserve(fh, file)) {
-			err = 0;
-			break;
+	scoped_guard(rwsem_read, &luo_file_handler_lock) {
+		list_private_for_each_entry(fh, &luo_file_handler_list, list) {
+			if (fh->ops->can_preserve(fh, file)) {
+				err = 0;
+				break;
+			}
 		}
 	}
 
@@ -777,10 +780,12 @@ int luo_file_deserialize(struct luo_file_set *file_set,
 		bool handler_found = false;
 		struct luo_file *luo_file;
 
-		list_private_for_each_entry(fh, &luo_file_handler_list, list) {
-			if (!strcmp(fh->compatible, file_ser[i].compatible)) {
-				handler_found = true;
-				break;
+		scoped_guard(rwsem_read, &luo_file_handler_lock) {
+			list_private_for_each_entry(fh, &luo_file_handler_list, list) {
+				if (!strcmp(fh->compatible, file_ser[i].compatible)) {
+					handler_found = true;
+					break;
+				}
 			}
 		}
 
@@ -831,7 +836,6 @@ void luo_file_set_destroy(struct luo_file_set *file_set)
 int liveupdate_register_file_handler(struct liveupdate_file_handler *fh)
 {
 	struct liveupdate_file_handler *fh_iter;
-	int err;
 
 	if (!liveupdate_enabled())
 		return -EOPNOTSUPP;
@@ -842,42 +846,25 @@ int liveupdate_register_file_handler(struct liveupdate_file_handler *fh)
 		return -EINVAL;
 	}
 
-	/*
-	 * Ensure the system is quiescent (no active sessions).
-	 * This prevents registering new handlers while sessions are active or
-	 * while deserialization is in progress.
-	 */
-	if (!luo_session_quiesce())
-		return -EBUSY;
-
-	/* Check for duplicate compatible strings */
-	list_private_for_each_entry(fh_iter, &luo_file_handler_list, list) {
-		if (!strcmp(fh_iter->compatible, fh->compatible)) {
-			pr_err("File handler registration failed: Compatible string '%s' already registered.\n",
-			       fh->compatible);
-			err = -EEXIST;
-			goto err_resume;
+	scoped_guard(rwsem_write, &luo_file_handler_lock) {
+		/* Check for duplicate compatible strings */
+		list_private_for_each_entry(fh_iter, &luo_file_handler_list, list) {
+			if (!strcmp(fh_iter->compatible, fh->compatible)) {
+				pr_err("File handler registration failed: Compatible string '%s' already registered.\n",
+				       fh->compatible);
+				return -EEXIST;
+			}
 		}
-	}
 
-	/* Pin the module implementing the handler */
-	if (!try_module_get(fh->ops->owner)) {
-		err = -EAGAIN;
-		goto err_resume;
+		INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, flb_list));
+		init_rwsem(&ACCESS_PRIVATE(fh, flb_lock));
+		INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, list));
+		list_add_tail(&ACCESS_PRIVATE(fh, list), &luo_file_handler_list);
 	}
 
-	INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, flb_list));
-	INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, list));
-	list_add_tail(&ACCESS_PRIVATE(fh, list), &luo_file_handler_list);
-	luo_session_resume();
-
 	liveupdate_test_register(fh);
 
 	return 0;
-
-err_resume:
-	luo_session_resume();
-	return err;
 }
 
 /**
@@ -886,41 +873,14 @@ int liveupdate_register_file_handler(struct liveupdate_file_handler *fh)
  *
  * Unregisters the file handler from the liveupdate core. This function
  * reverses the operations of liveupdate_register_file_handler().
- *
- * It ensures safe removal by checking that:
- * No live update session is currently in progress.
- * No FLB registered with this file handler.
- *
- * If the unregistration fails, the internal test state is reverted.
- *
- * Return: 0 Success. -EOPNOTSUPP when live update is not enabled. -EBUSY A live
- * update is in progress, can't quiesce live update or FLB is registred with
- * this file handler.
  */
-int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
+void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
 {
-	int err = -EBUSY;
-
 	if (!liveupdate_enabled())
-		return -EOPNOTSUPP;
-
-	liveupdate_test_unregister(fh);
-
-	if (!luo_session_quiesce())
-		goto err_register;
-
-	if (!list_empty(&ACCESS_PRIVATE(fh, flb_list)))
-		goto err_resume;
-
-	list_del(&ACCESS_PRIVATE(fh, list));
-	module_put(fh->ops->owner);
-	luo_session_resume();
-
-	return 0;
+		return;
 
-err_resume:
-	luo_session_resume();
-err_register:
-	liveupdate_test_register(fh);
-	return err;
+	scoped_guard(rwsem_write, &luo_file_handler_lock) {
+		luo_flb_unregister_all(fh);
+		list_del(&ACCESS_PRIVATE(fh, list));
+	}
 }
diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c
index f52e8114837e3..874830f8e44d8 100644
--- a/kernel/liveupdate/luo_flb.c
+++ b/kernel/liveupdate/luo_flb.c
@@ -49,6 +49,7 @@
 #include <linux/liveupdate.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/rwsem.h>
 #include <linux/slab.h>
 #include <linux/unaligned.h>
 #include "luo_internal.h"
@@ -70,6 +71,7 @@ struct luo_flb_global {
 	long count;
 };
 
+static DECLARE_RWSEM(luo_flb_lock);
 static struct luo_flb_global luo_flb_global = {
 	.list = LIST_HEAD_INIT(luo_flb_global.list),
 };
@@ -110,10 +112,15 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
 			struct liveupdate_flb_op_args args = {0};
 			int err;
 
+			if (!try_module_get(flb->ops->owner))
+				return -ENODEV;
+
 			args.flb = flb;
 			err = flb->ops->preserve(&args);
-			if (err)
+			if (err) {
+				module_put(flb->ops->owner);
 				return err;
+			}
 			private->outgoing.data = args.data;
 			private->outgoing.obj = args.obj;
 		}
@@ -141,6 +148,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb)
 
 			private->outgoing.data = 0;
 			private->outgoing.obj = NULL;
+			module_put(flb->ops->owner);
 		}
 	}
 }
@@ -176,12 +184,17 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
 	if (!found)
 		return -ENOENT;
 
+	if (!try_module_get(flb->ops->owner))
+		return -ENODEV;
+
 	args.flb = flb;
 	args.data = private->incoming.data;
 
 	err = flb->ops->retrieve(&args);
-	if (err)
+	if (err) {
+		module_put(flb->ops->owner);
 		return err;
+	}
 
 	private->incoming.obj = args.obj;
 	private->incoming.retrieved = true;
@@ -215,6 +228,7 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
 			private->incoming.data = 0;
 			private->incoming.obj = NULL;
 			private->incoming.finished = true;
+			module_put(flb->ops->owner);
 		}
 	}
 }
@@ -240,6 +254,8 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh)
 	struct luo_flb_link *iter;
 	int err = 0;
 
+	guard(rwsem_read)(&ACCESS_PRIVATE(fh, flb_lock));
+
 	list_for_each_entry(iter, flb_list, list) {
 		err = luo_flb_file_preserve_one(iter->flb);
 		if (err)
@@ -272,6 +288,8 @@ void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh)
 	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
 	struct luo_flb_link *iter;
 
+	guard(rwsem_read)(&ACCESS_PRIVATE(fh, flb_lock));
+
 	list_for_each_entry_reverse(iter, flb_list, list)
 		luo_flb_file_unpreserve_one(iter->flb);
 }
@@ -292,10 +310,70 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh)
 	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
 	struct luo_flb_link *iter;
 
+	guard(rwsem_read)(&ACCESS_PRIVATE(fh, flb_lock));
+
 	list_for_each_entry_reverse(iter, flb_list, list)
 		luo_flb_file_finish_one(iter->flb);
 }
 
+static void luo_flb_unregister_one(struct liveupdate_file_handler *fh,
+				   struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter;
+	bool found = false;
+
+	/* Find and remove the link from the file handler's list */
+	list_for_each_entry(iter, flb_list, list) {
+		if (iter->flb == flb) {
+			list_del(&iter->list);
+			kfree(iter);
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		pr_warn("Failed to unregister FLB '%s': not found in file handler '%s'\n",
+			flb->compatible, fh->compatible);
+		return;
+	}
+
+	private->users--;
+
+	/*
+	 * If this is the last file-handler with which we are registred, remove
+	 * from the global list.
+	 */
+	if (!private->users) {
+		list_del_init(&private->list);
+		luo_flb_global.count--;
+	}
+}
+
+/**
+ * luo_flb_unregister_all - Unregister all FLBs associated with a file handler.
+ * @fh: The file handler whose FLBs should be unregistered.
+ *
+ * This function iterates through the list of FLBs associated with the given
+ * file handler and unregisters them all one by one.
+ */
+void luo_flb_unregister_all(struct liveupdate_file_handler *fh)
+{
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter, *tmp;
+
+	if (!liveupdate_enabled())
+		return;
+
+	guard(rwsem_write)(&luo_flb_lock);
+	guard(rwsem_write)(&ACCESS_PRIVATE(fh, flb_lock));
+
+	list_for_each_entry_safe(iter, tmp, flb_list, list)
+		luo_flb_unregister_one(fh, iter->flb);
+}
+
 /**
  * liveupdate_register_flb - Associate an FLB with a file handler and register it globally.
  * @fh:   The file handler that will now depend on the FLB.
@@ -326,7 +404,6 @@ int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 	struct luo_flb_link *link __free(kfree) = NULL;
 	struct liveupdate_flb *gflb;
 	struct luo_flb_link *iter;
-	int err;
 
 	if (!liveupdate_enabled())
 		return -EOPNOTSUPP;
@@ -347,19 +424,13 @@ int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 	if (!link)
 		return -ENOMEM;
 
-	/*
-	 * Ensure the system is quiescent (no active sessions).
-	 * This acts as a global lock for registration: no other thread can
-	 * be in this section, and no sessions can be creating/using FDs.
-	 */
-	if (!luo_session_quiesce())
-		return -EBUSY;
+	guard(rwsem_write)(&luo_flb_lock);
+	guard(rwsem_write)(&ACCESS_PRIVATE(fh, flb_lock));
 
 	/* Check that this FLB is not already linked to this file handler */
-	err = -EEXIST;
 	list_for_each_entry(iter, flb_list, list) {
 		if (iter->flb == flb)
-			goto err_resume;
+			return -EEXIST;
 	}
 
 	/*
@@ -367,25 +438,16 @@ int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 	 * is registered
 	 */
 	if (!private->users) {
-		if (WARN_ON(!list_empty(&private->list))) {
-			err = -EINVAL;
-			goto err_resume;
-		}
+		if (WARN_ON(!list_empty(&private->list)))
+			return -EINVAL;
 
-		if (luo_flb_global.count == LUO_FLB_MAX) {
-			err = -ENOSPC;
-			goto err_resume;
-		}
+		if (luo_flb_global.count == LUO_FLB_MAX)
+			return -ENOSPC;
 
 		/* Check that compatible string is unique in global list */
 		list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
 			if (!strcmp(gflb->compatible, flb->compatible))
-				goto err_resume;
-		}
-
-		if (!try_module_get(flb->ops->owner)) {
-			err = -EAGAIN;
-			goto err_resume;
+				return -EEXIST;
 		}
 
 		list_add_tail(&private->list, &luo_flb_global.list);
@@ -396,13 +458,8 @@ int liveupdate_register_flb(struct liveupdate_file_handler *fh,
 	private->users++;
 	link->flb = flb;
 	list_add_tail(&no_free_ptr(link)->list, flb_list);
-	luo_session_resume();
 
 	return 0;
-
-err_resume:
-	luo_session_resume();
-	return err;
 }
 
 /**
@@ -418,63 +475,17 @@ int liveupdate_register_flb(struct liveupdate_file_handler *fh,
  * the FLB is removed from the global registry and the reference to its
  * owner module (acquired during registration) is released.
  *
- * Context: This function ensures the session is quiesced (no active FDs
- *          being created) during the update. It is typically called from a
- *          subsystem's module exit function.
- * Return: 0 on success.
- *         -EOPNOTSUPP if live update is disabled.
- *         -EBUSY if the live update session is active and cannot be quiesced.
- *         -ENOENT if the FLB was not found in the file handler's list.
  */
-int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
-			      struct liveupdate_flb *flb)
+void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+			       struct liveupdate_flb *flb)
 {
-	struct luo_flb_private *private = luo_flb_get_private(flb);
-	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
-	struct luo_flb_link *iter;
-	int err = -ENOENT;
-
 	if (!liveupdate_enabled())
-		return -EOPNOTSUPP;
+		return;
 
-	/*
-	 * Ensure the system is quiescent (no active sessions).
-	 * This acts as a global lock for unregistration.
-	 */
-	if (!luo_session_quiesce())
-		return -EBUSY;
+	guard(rwsem_write)(&luo_flb_lock);
+	guard(rwsem_write)(&ACCESS_PRIVATE(fh, flb_lock));
 
-	/* Find and remove the link from the file handler's list */
-	list_for_each_entry(iter, flb_list, list) {
-		if (iter->flb == flb) {
-			list_del(&iter->list);
-			kfree(iter);
-			err = 0;
-			break;
-		}
-	}
-
-	if (err)
-		goto err_resume;
-
-	private->users--;
-	/*
-	 * If this is the last file-handler with which we are registred, remove
-	 * from the global list, and relese module reference.
-	 */
-	if (!private->users) {
-		list_del_init(&private->list);
-		luo_flb_global.count--;
-		module_put(flb->ops->owner);
-	}
-
-	luo_session_resume();
-
-	return 0;
-
-err_resume:
-	luo_session_resume();
-	return err;
+	luo_flb_unregister_one(fh, flb);
 }
 
 /**
@@ -492,7 +503,8 @@ int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
  *
  * Return: 0 on success, or a negative errno on failure. -ENODATA means no
  * incoming FLB data, -ENOENT means specific flb not found in the incoming
- * data, and -EOPNOTSUPP when live update is disabled or not configured.
+ * data, -ENODEV if the FLB's module is unloading, and -EOPNOTSUPP when
+ * live update is disabled or not configured.
  */
 int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
 {
@@ -638,6 +650,8 @@ void luo_flb_serialize(void)
 	struct liveupdate_flb *gflb;
 	int i = 0;
 
+	guard(rwsem_read)(&luo_flb_lock);
+
 	list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
 		struct luo_flb_private *private = luo_flb_get_private(gflb);
 
diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h
index 8083d8739b093..137ae317188a7 100644
--- a/kernel/liveupdate/luo_internal.h
+++ b/kernel/liveupdate/luo_internal.h
@@ -83,8 +83,6 @@ int __init luo_session_setup_outgoing(void *fdt);
 int __init luo_session_setup_incoming(void *fdt);
 int luo_session_serialize(void);
 int luo_session_deserialize(void);
-bool luo_session_quiesce(void);
-void luo_session_resume(void);
 
 int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd);
 void luo_file_unpreserve_files(struct luo_file_set *file_set);
@@ -103,16 +101,15 @@ void luo_file_set_destroy(struct luo_file_set *file_set);
 int luo_flb_file_preserve(struct liveupdate_file_handler *fh);
 void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh);
 void luo_flb_file_finish(struct liveupdate_file_handler *fh);
+void luo_flb_unregister_all(struct liveupdate_file_handler *fh);
 int __init luo_flb_setup_outgoing(void *fdt);
 int __init luo_flb_setup_incoming(void *fdt);
 void luo_flb_serialize(void);
 
 #ifdef CONFIG_LIVEUPDATE_TEST
 void liveupdate_test_register(struct liveupdate_file_handler *fh);
-void liveupdate_test_unregister(struct liveupdate_file_handler *fh);
 #else
 static inline void liveupdate_test_register(struct liveupdate_file_handler *fh) { }
-static inline void liveupdate_test_unregister(struct liveupdate_file_handler *fh) { }
 #endif
 
 #endif /* _LINUX_LUO_INTERNAL_H */
diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c
index 7836772956406..067ffb54c36a3 100644
--- a/kernel/liveupdate/luo_session.c
+++ b/kernel/liveupdate/luo_session.c
@@ -600,47 +600,3 @@ int luo_session_serialize(void)
 
 	return err;
 }
-
-/**
- * luo_session_quiesce - Ensure no active sessions exist and lock session lists.
- *
- * Acquires exclusive write locks on both incoming and outgoing session lists.
- * It then validates no sessions exist in either list.
- *
- * This mechanism is used during file handler un/registration to ensure that no
- * sessions are currently using the handler, and no new sessions can be created
- * while un/registration is in progress.
- *
- * This prevents registering new handlers while sessions are active or
- * while deserialization is in progress.
- *
- * Return:
- * true  - System is quiescent (0 sessions) and locked.
- * false - Active sessions exist. The locks are released internally.
- */
-bool luo_session_quiesce(void)
-{
-	down_write(&luo_session_global.incoming.rwsem);
-	down_write(&luo_session_global.outgoing.rwsem);
-
-	if (luo_session_global.incoming.count ||
-	    luo_session_global.outgoing.count) {
-		up_write(&luo_session_global.outgoing.rwsem);
-		up_write(&luo_session_global.incoming.rwsem);
-		return false;
-	}
-
-	return true;
-}
-
-/**
- * luo_session_resume - Unlock session lists and resume normal activity.
- *
- * Releases the exclusive locks acquired by a successful call to
- * luo_session_quiesce().
- */
-void luo_session_resume(void)
-{
-	up_write(&luo_session_global.outgoing.rwsem);
-	up_write(&luo_session_global.incoming.rwsem);
-}
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index a114949eeed5e..cee1901d4cff8 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -3,6 +3,11 @@
 # and is generally not a function of system call inputs.
 KCOV_INSTRUMENT		:= n
 
+CONTEXT_ANALYSIS_mutex.o := y
+CONTEXT_ANALYSIS_rtmutex_api.o := y
+CONTEXT_ANALYSIS_ww_rt_mutex.o := y
+CONTEXT_ANALYSIS_rwsem.o := y
+
 obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
 
 # Avoid recursion lockdep -> sanitizer -> ... -> lockdep & improve performance.
diff --git a/kernel/locking/mutex-debug.c b/kernel/locking/mutex-debug.c
index 2c6b02d4699be..94930d506bcfd 100644
--- a/kernel/locking/mutex-debug.c
+++ b/kernel/locking/mutex-debug.c
@@ -37,9 +37,8 @@ void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
 void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter)
 {
 	lockdep_assert_held(&lock->wait_lock);
-	DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list));
+	DEBUG_LOCKS_WARN_ON(!lock->first_waiter);
 	DEBUG_LOCKS_WARN_ON(waiter->magic != waiter);
-	DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
 }
 
 void debug_mutex_free_waiter(struct mutex_waiter *waiter)
@@ -62,7 +61,6 @@ void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
 {
 	struct mutex *blocked_on = __get_task_blocked_on(task);
 
-	DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
 	DEBUG_LOCKS_WARN_ON(waiter->task != task);
 	DEBUG_LOCKS_WARN_ON(blocked_on && blocked_on != lock);
 
@@ -74,7 +72,6 @@ void debug_mutex_unlock(struct mutex *lock)
 {
 	if (likely(debug_locks)) {
 		DEBUG_LOCKS_WARN_ON(lock->magic != lock);
-		DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
 	}
 }
 
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 2a1d165b3167e..427187ff02db3 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -46,8 +46,9 @@
 static void __mutex_init_generic(struct mutex *lock)
 {
 	atomic_long_set(&lock->owner, 0);
-	raw_spin_lock_init(&lock->wait_lock);
-	INIT_LIST_HEAD(&lock->wait_list);
+	scoped_guard (raw_spinlock_init, &lock->wait_lock) {
+		lock->first_waiter = NULL;
+	}
 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
 	osq_lock_init(&lock->osq);
 #endif
@@ -150,6 +151,7 @@ EXPORT_SYMBOL(mutex_init_generic);
  * follow with a __mutex_trylock() before failing.
  */
 static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
+	__cond_acquires(true, lock)
 {
 	unsigned long curr = (unsigned long)current;
 	unsigned long zero = 0UL;
@@ -163,6 +165,7 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
 }
 
 static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
+	__cond_releases(true, lock)
 {
 	unsigned long curr = (unsigned long)current;
 
@@ -171,7 +174,7 @@ static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
 
 #else /* !CONFIG_DEBUG_LOCK_ALLOC */
 
-void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key)
+void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key)
 {
 	__mutex_init_generic(lock);
 
@@ -181,7 +184,7 @@ void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_k
 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);
 }
-EXPORT_SYMBOL(mutex_init_lockep);
+EXPORT_SYMBOL(mutex_init_lockdep);
 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */
 
 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
@@ -194,33 +197,44 @@ static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
 	atomic_long_andnot(flag, &lock->owner);
 }
 
-static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
-{
-	return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
-}
-
 /*
  * Add @waiter to a given location in the lock wait_list and set the
  * FLAG_WAITERS flag if it's the first waiter.
  */
 static void
 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
-		   struct list_head *list)
+		   struct mutex_waiter *first)
+	__must_hold(&lock->wait_lock)
 {
 	hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX);
 	debug_mutex_add_waiter(lock, waiter, current);
 
-	list_add_tail(&waiter->list, list);
-	if (__mutex_waiter_is_first(lock, waiter))
+	if (!first)
+		first = lock->first_waiter;
+
+	if (first) {
+		list_add_tail(&waiter->list, &first->list);
+	} else {
+		INIT_LIST_HEAD(&waiter->list);
+		lock->first_waiter = waiter;
 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
+	}
 }
 
 static void
 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
-	list_del(&waiter->list);
-	if (likely(list_empty(&lock->wait_list)))
+	if (list_empty(&waiter->list)) {
 		__mutex_clear_flag(lock, MUTEX_FLAGS);
+		lock->first_waiter = NULL;
+	} else {
+		if (lock->first_waiter == waiter) {
+			lock->first_waiter = list_first_entry(&waiter->list,
+							      struct mutex_waiter, list);
+		}
+		list_del(&waiter->list);
+	}
 
 	debug_mutex_remove_waiter(lock, waiter, current);
 	hung_task_clear_blocker();
@@ -259,7 +273,8 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
  * We also put the fastpath first in the kernel image, to make sure the
  * branch is predicted by the CPU as default-untaken.
  */
-static void __sched __mutex_lock_slowpath(struct mutex *lock);
+static void __sched __mutex_lock_slowpath(struct mutex *lock)
+	__acquires(lock);
 
 /**
  * mutex_lock - acquire the mutex
@@ -340,7 +355,7 @@ bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
 	 * Similarly, stop spinning if we are no longer the
 	 * first waiter.
 	 */
-	if (waiter && !__mutex_waiter_is_first(lock, waiter))
+	if (waiter && data_race(lock->first_waiter != waiter))
 		return false;
 
 	return true;
@@ -525,7 +540,8 @@ mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
 }
 #endif
 
-static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
+static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
+	__releases(lock);
 
 /**
  * mutex_unlock - release the mutex
@@ -565,6 +581,7 @@ EXPORT_SYMBOL(mutex_unlock);
  * of a unlocked mutex is not allowed.
  */
 void __sched ww_mutex_unlock(struct ww_mutex *lock)
+	__no_context_analysis
 {
 	__ww_mutex_unlock(lock);
 	mutex_unlock(&lock->base);
@@ -578,6 +595,7 @@ static __always_inline int __sched
 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
 		    struct lockdep_map *nest_lock, unsigned long ip,
 		    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
+	__cond_acquires(0, lock)
 {
 	DEFINE_WAKE_Q(wake_q);
 	struct mutex_waiter waiter;
@@ -645,7 +663,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
 
 	if (!use_ww_ctx) {
 		/* add waiting tasks to the end of the waitqueue (FIFO): */
-		__mutex_add_waiter(lock, &waiter, &lock->wait_list);
+		__mutex_add_waiter(lock, &waiter, NULL);
 	} else {
 		/*
 		 * Add in stamp order, waking up waiters that must kill
@@ -691,7 +709,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
 
 		schedule_preempt_disabled();
 
-		first = __mutex_waiter_is_first(lock, &waiter);
+		first = lock->first_waiter == &waiter;
 
 		/*
 		 * As we likely have been woken up by task
@@ -734,8 +752,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
 		 * Wound-Wait; we stole the lock (!first_waiter), check the
 		 * waiters as anyone might want to wound us.
 		 */
-		if (!ww_ctx->is_wait_die &&
-		    !__mutex_waiter_is_first(lock, &waiter))
+		if (!ww_ctx->is_wait_die && lock->first_waiter != &waiter)
 			__ww_mutex_check_waiters(lock, ww_ctx, &wake_q);
 	}
 
@@ -772,6 +789,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
 static int __sched
 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
 	     struct lockdep_map *nest_lock, unsigned long ip)
+	__cond_acquires(0, lock)
 {
 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
 }
@@ -779,6 +797,7 @@ __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
 static int __sched
 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
 		unsigned long ip, struct ww_acquire_ctx *ww_ctx)
+	__cond_acquires(0, lock)
 {
 	return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);
 }
@@ -826,6 +845,7 @@ void __sched
 mutex_lock_nested(struct mutex *lock, unsigned int subclass)
 {
 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
+	__acquire(lock);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_nested);
@@ -834,6 +854,7 @@ void __sched
 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
 {
 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
+	__acquire(lock);
 }
 EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
 
@@ -862,12 +883,14 @@ mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
 	token = io_schedule_prepare();
 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
 			    subclass, NULL, _RET_IP_, NULL, 0);
+	__acquire(lock);
 	io_schedule_finish(token);
 }
 EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
 
 static inline int
 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+	__cond_releases(nonzero, lock)
 {
 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
 	unsigned tmp;
@@ -929,13 +952,16 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
  * Release the lock, slowpath:
  */
 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
+	__releases(lock)
 {
 	struct task_struct *next = NULL;
+	struct mutex_waiter *waiter;
 	DEFINE_WAKE_Q(wake_q);
 	unsigned long owner;
 	unsigned long flags;
 
 	mutex_release(&lock->dep_map, ip);
+	__release(lock);
 
 	/*
 	 * Release the lock before (potentially) taking the spinlock such that
@@ -962,12 +988,8 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
 
 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	debug_mutex_unlock(lock);
-	if (!list_empty(&lock->wait_list)) {
-		/* get the first entry from the wait-list: */
-		struct mutex_waiter *waiter =
-			list_first_entry(&lock->wait_list,
-					 struct mutex_waiter, list);
-
+	waiter = lock->first_waiter;
+	if (waiter) {
 		next = waiter->task;
 
 		debug_mutex_wake_waiter(lock, waiter);
@@ -1061,24 +1083,29 @@ EXPORT_SYMBOL_GPL(mutex_lock_io);
 
 static noinline void __sched
 __mutex_lock_slowpath(struct mutex *lock)
+	__acquires(lock)
 {
 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
+	__acquire(lock);
 }
 
 static noinline int __sched
 __mutex_lock_killable_slowpath(struct mutex *lock)
+	__cond_acquires(0, lock)
 {
 	return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
 }
 
 static noinline int __sched
 __mutex_lock_interruptible_slowpath(struct mutex *lock)
+	__cond_acquires(0, lock)
 {
 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
 }
 
 static noinline int __sched
 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+	__cond_acquires(0, lock)
 {
 	return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,
 			       _RET_IP_, ctx);
@@ -1087,6 +1114,7 @@ __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 static noinline int __sched
 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
 					    struct ww_acquire_ctx *ctx)
+	__cond_acquires(0, lock)
 {
 	return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,
 			       _RET_IP_, ctx);
diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 9ad4da8cea004..b94ef40c1f485 100644
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -7,6 +7,7 @@
  *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  */
 #ifndef CONFIG_PREEMPT_RT
+#include <linux/mutex.h>
 /*
  * This is the control structure for tasks blocked on mutex, which resides
  * on the blocked task's kernel stack:
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index c80902eacd797..ccaba6148b619 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -94,6 +94,7 @@ static inline int __ww_mutex_check_kill(struct rt_mutex *lock,
 
 static __always_inline struct task_struct *
 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
+	__must_hold(&lock->wait_lock)
 {
 	unsigned long val = (unsigned long)owner;
 
@@ -105,6 +106,7 @@ rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
 
 static __always_inline void
 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
+	__must_hold(&lock->wait_lock)
 {
 	/*
 	 * lock->wait_lock is held but explicit acquire semantics are needed
@@ -114,12 +116,14 @@ rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
 }
 
 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	/* lock->wait_lock is held so the unlock provides release semantics. */
 	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));
 }
 
 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	lock->owner = (struct task_struct *)
 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
@@ -127,6 +131,7 @@ static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
 
 static __always_inline void
 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)
+	__must_hold(&lock->wait_lock)
 {
 	unsigned long owner, *p = (unsigned long *) &lock->owner;
 
@@ -328,6 +333,7 @@ static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
 }
 
 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	lock->owner = (struct task_struct *)
 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
@@ -1206,6 +1212,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
 					   struct ww_acquire_ctx *ww_ctx,
 					   enum rtmutex_chainwalk chwalk,
 					   struct wake_q_head *wake_q)
+	__must_hold(&lock->wait_lock)
 {
 	struct task_struct *owner = rt_mutex_owner(lock);
 	struct rt_mutex_waiter *top_waiter = waiter;
@@ -1249,6 +1256,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
 
 		/* Check whether the waiter should back out immediately */
 		rtm = container_of(lock, struct rt_mutex, rtmutex);
+		__assume_ctx_lock(&rtm->rtmutex.wait_lock);
 		res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx, wake_q);
 		if (res) {
 			raw_spin_lock(&task->pi_lock);
@@ -1356,6 +1364,7 @@ static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
 }
 
 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	int ret = try_to_take_rt_mutex(lock, current, NULL);
 
@@ -1505,7 +1514,7 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
 		 *  - the VCPU on which owner runs is preempted
 		 */
 		if (!owner_on_cpu(owner) || need_resched() ||
-		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
+		    !data_race(rt_mutex_waiter_is_top_waiter(lock, waiter))) {
 			res = false;
 			break;
 		}
@@ -1538,6 +1547,7 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
  */
 static void __sched remove_waiter(struct rt_mutex_base *lock,
 				  struct rt_mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
 	struct task_struct *owner = rt_mutex_owner(lock);
@@ -1613,6 +1623,8 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 	struct task_struct *owner;
 	int ret = 0;
 
+	__assume_ctx_lock(&rtm->rtmutex.wait_lock);
+
 	lockevent_inc(rtmutex_slow_block);
 	for (;;) {
 		/* Try to acquire the lock: */
@@ -1658,6 +1670,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
 					     struct rt_mutex_base *lock,
 					     struct rt_mutex_waiter *w)
+	__must_hold(&lock->wait_lock)
 {
 	/*
 	 * If the result is not -EDEADLOCK or the caller requested
@@ -1694,11 +1707,13 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
 				       enum rtmutex_chainwalk chwalk,
 				       struct rt_mutex_waiter *waiter,
 				       struct wake_q_head *wake_q)
+	__must_hold(&lock->wait_lock)
 {
 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
 	struct ww_mutex *ww = ww_container_of(rtm);
 	int ret;
 
+	__assume_ctx_lock(&rtm->rtmutex.wait_lock);
 	lockdep_assert_held(&lock->wait_lock);
 	lockevent_inc(rtmutex_slowlock);
 
@@ -1750,6 +1765,7 @@ static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
 					     struct ww_acquire_ctx *ww_ctx,
 					     unsigned int state,
 					     struct wake_q_head *wake_q)
+	__must_hold(&lock->wait_lock)
 {
 	struct rt_mutex_waiter waiter;
 	int ret;
diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index 59dbd29cb219b..124219aea46e4 100644
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -526,6 +526,7 @@ static __always_inline int __mutex_lock_common(struct mutex *lock,
 					       unsigned int subclass,
 					       struct lockdep_map *nest_lock,
 					       unsigned long ip)
+	__acquires(lock) __no_context_analysis
 {
 	int ret;
 
@@ -647,6 +648,7 @@ EXPORT_SYMBOL(mutex_trylock);
 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */
 
 void __sched mutex_unlock(struct mutex *lock)
+	__releases(lock) __no_context_analysis
 {
 	mutex_release(&lock->dep_map, _RET_IP_);
 	__rt_mutex_unlock(&lock->rtmutex);
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index cf6ddd1b23a25..c38b7bdea7b35 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -79,12 +79,18 @@ struct rt_wake_q_head {
  * PI-futex support (proxy locking functions, etc.):
  */
 extern void rt_mutex_init_proxy_locked(struct rt_mutex_base *lock,
-				       struct task_struct *proxy_owner);
-extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock);
+				       struct task_struct *proxy_owner)
+	__must_hold(&lock->wait_lock);
+
+extern void rt_mutex_proxy_unlock(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock);
+
 extern int __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
 				     struct rt_mutex_waiter *waiter,
 				     struct task_struct *task,
-				     struct wake_q_head *);
+				     struct wake_q_head *)
+	__must_hold(&lock->wait_lock);
+
 extern int rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
 				     struct rt_mutex_waiter *waiter,
 				     struct task_struct *task);
@@ -94,8 +100,9 @@ extern int rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,
 extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock,
 				 struct rt_mutex_waiter *waiter);
 
-extern int rt_mutex_futex_trylock(struct rt_mutex_base *l);
-extern int __rt_mutex_futex_trylock(struct rt_mutex_base *l);
+extern int rt_mutex_futex_trylock(struct rt_mutex_base *lock);
+extern int __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock);
 
 extern void rt_mutex_futex_unlock(struct rt_mutex_base *lock);
 extern bool __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
@@ -109,6 +116,7 @@ extern void rt_mutex_postunlock(struct rt_wake_q_head *wqh);
  */
 #ifdef CONFIG_RT_MUTEXES
 static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	return !RB_EMPTY_ROOT(&lock->waiters.rb_root);
 }
@@ -120,6 +128,7 @@ static inline int rt_mutex_has_waiters(struct rt_mutex_base *lock)
  */
 static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock,
 						 struct rt_mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
 	struct rb_node *leftmost = rb_first_cached(&lock->waiters);
 
@@ -127,6 +136,7 @@ static inline bool rt_mutex_waiter_is_top_waiter(struct rt_mutex_base *lock,
 }
 
 static inline struct rt_mutex_waiter *rt_mutex_top_waiter(struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	struct rb_node *leftmost = rb_first_cached(&lock->waiters);
 	struct rt_mutex_waiter *w = NULL;
@@ -170,9 +180,10 @@ enum rtmutex_chainwalk {
 
 static inline void __rt_mutex_base_init(struct rt_mutex_base *lock)
 {
-	raw_spin_lock_init(&lock->wait_lock);
-	lock->waiters = RB_ROOT_CACHED;
-	lock->owner = NULL;
+	scoped_guard (raw_spinlock_init, &lock->wait_lock) {
+		lock->waiters = RB_ROOT_CACHED;
+		lock->owner = NULL;
+	}
 }
 
 /* Debug functions */
diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c
index 9f4322c074862..82e078c0665ac 100644
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -186,6 +186,7 @@ static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
 
 static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
 					 unsigned long flags)
+	__releases(&rwb->rtmutex.wait_lock)
 {
 	struct rt_mutex_base *rtm = &rwb->rtmutex;
 
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 24df4d98f7d20..bf647097369cb 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -72,7 +72,7 @@
 		#c, atomic_long_read(&(sem)->count),		\
 		(unsigned long) sem->magic,			\
 		atomic_long_read(&(sem)->owner), (long)current,	\
-		list_empty(&(sem)->wait_list) ? "" : "not "))	\
+		rwsem_is_contended(sem) ? "" : "not "))		\
 			debug_locks_off();			\
 	} while (0)
 #else
@@ -320,9 +320,10 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
 	sem->magic = sem;
 #endif
 	atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
-	raw_spin_lock_init(&sem->wait_lock);
-	INIT_LIST_HEAD(&sem->wait_list);
 	atomic_long_set(&sem->owner, 0L);
+	scoped_guard (raw_spinlock_init, &sem->wait_lock) {
+		sem->first_waiter = NULL;
+	}
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 	osq_lock_init(&sem->osq);
 #endif
@@ -341,8 +342,6 @@ struct rwsem_waiter {
 	unsigned long timeout;
 	bool handoff_set;
 };
-#define rwsem_first_waiter(sem) \
-	list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
 
 enum rwsem_wake_type {
 	RWSEM_WAKE_ANY,		/* Wake whatever's at head of wait list */
@@ -365,12 +364,22 @@ enum rwsem_wake_type {
  */
 #define MAX_READERS_WAKEUP	0x100
 
-static inline void
-rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+static inline
+bool __rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+	__must_hold(&sem->wait_lock)
 {
-	lockdep_assert_held(&sem->wait_lock);
-	list_add_tail(&waiter->list, &sem->wait_list);
-	/* caller will set RWSEM_FLAG_WAITERS */
+	if (list_empty(&waiter->list)) {
+		sem->first_waiter = NULL;
+		return false;
+	}
+
+	if (sem->first_waiter == waiter) {
+		sem->first_waiter = list_first_entry(&waiter->list,
+						     struct rwsem_waiter, list);
+	}
+	list_del(&waiter->list);
+
+	return true;
 }
 
 /*
@@ -385,14 +394,24 @@ static inline bool
 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
 {
 	lockdep_assert_held(&sem->wait_lock);
-	list_del(&waiter->list);
-	if (likely(!list_empty(&sem->wait_list)))
+	if (__rwsem_del_waiter(sem, waiter))
 		return true;
-
 	atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
 	return false;
 }
 
+static inline
+struct rwsem_waiter *next_waiter(const struct rw_semaphore *sem,
+				 const struct rwsem_waiter *waiter)
+	__must_hold(&sem->wait_lock)
+{
+	struct rwsem_waiter *next = list_first_entry(&waiter->list,
+						     struct rwsem_waiter, list);
+	if (next == sem->first_waiter)
+		return NULL;
+	return next;
+}
+
 /*
  * handle the lock release when processes blocked on it that can now run
  * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must
@@ -411,7 +430,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
 			    enum rwsem_wake_type wake_type,
 			    struct wake_q_head *wake_q)
 {
-	struct rwsem_waiter *waiter, *tmp;
+	struct rwsem_waiter *waiter, *next;
 	long oldcount, woken = 0, adjustment = 0;
 	struct list_head wlist;
 
@@ -421,7 +440,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
 	 * Take a peek at the queue head waiter such that we can determine
 	 * the wakeup(s) to perform.
 	 */
-	waiter = rwsem_first_waiter(sem);
+	waiter = sem->first_waiter;
 
 	if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
 		if (wake_type == RWSEM_WAKE_ANY) {
@@ -506,25 +525,28 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
 	 *    put them into wake_q to be woken up later.
 	 */
 	INIT_LIST_HEAD(&wlist);
-	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {
+	do {
+		next = next_waiter(sem, waiter);
 		if (waiter->type == RWSEM_WAITING_FOR_WRITE)
 			continue;
 
 		woken++;
 		list_move_tail(&waiter->list, &wlist);
+		if (sem->first_waiter == waiter)
+			sem->first_waiter = next;
 
 		/*
 		 * Limit # of readers that can be woken up per wakeup call.
 		 */
 		if (unlikely(woken >= MAX_READERS_WAKEUP))
 			break;
-	}
+	} while ((waiter = next) != NULL);
 
 	adjustment = woken * RWSEM_READER_BIAS - adjustment;
 	lockevent_cond_inc(rwsem_wake_reader, woken);
 
 	oldcount = atomic_long_read(&sem->count);
-	if (list_empty(&sem->wait_list)) {
+	if (!sem->first_waiter) {
 		/*
 		 * Combined with list_move_tail() above, this implies
 		 * rwsem_del_waiter().
@@ -545,7 +567,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
 		atomic_long_add(adjustment, &sem->count);
 
 	/* 2nd pass */
-	list_for_each_entry_safe(waiter, tmp, &wlist, list) {
+	list_for_each_entry_safe(waiter, next, &wlist, list) {
 		struct task_struct *tsk;
 
 		tsk = waiter->task;
@@ -577,7 +599,7 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
 		      struct wake_q_head *wake_q)
 		      __releases(&sem->wait_lock)
 {
-	bool first = rwsem_first_waiter(sem) == waiter;
+	bool first = sem->first_waiter == waiter;
 
 	wake_q_init(wake_q);
 
@@ -602,8 +624,9 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
  */
 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
 					struct rwsem_waiter *waiter)
+	__must_hold(&sem->wait_lock)
 {
-	struct rwsem_waiter *first = rwsem_first_waiter(sem);
+	struct rwsem_waiter *first = sem->first_waiter;
 	long count, new;
 
 	lockdep_assert_held(&sem->wait_lock);
@@ -639,7 +662,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
 			new |= RWSEM_WRITER_LOCKED;
 			new &= ~RWSEM_FLAG_HANDOFF;
 
-			if (list_is_singular(&sem->wait_list))
+			if (list_empty(&first->list))
 				new &= ~RWSEM_FLAG_WAITERS;
 		}
 	} while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new));
@@ -659,7 +682,8 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
 	 * Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on
 	 * success.
 	 */
-	list_del(&waiter->list);
+	__rwsem_del_waiter(sem, waiter);
+
 	rwsem_set_owner(sem);
 	return true;
 }
@@ -994,7 +1018,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
 {
 	long adjustment = -RWSEM_READER_BIAS;
 	long rcnt = (count >> RWSEM_READER_SHIFT);
-	struct rwsem_waiter waiter;
+	struct rwsem_waiter waiter, *first;
 	DEFINE_WAKE_Q(wake_q);
 
 	/*
@@ -1019,7 +1043,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
 		 */
 		if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) {
 			raw_spin_lock_irq(&sem->wait_lock);
-			if (!list_empty(&sem->wait_list))
+			if (sem->first_waiter)
 				rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,
 						&wake_q);
 			raw_spin_unlock_irq(&sem->wait_lock);
@@ -1035,7 +1059,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
 	waiter.handoff_set = false;
 
 	raw_spin_lock_irq(&sem->wait_lock);
-	if (list_empty(&sem->wait_list)) {
+	first = sem->first_waiter;
+	if (!first) {
 		/*
 		 * In case the wait queue is empty and the lock isn't owned
 		 * by a writer, this reader can exit the slowpath and return
@@ -1051,8 +1076,11 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
 			return sem;
 		}
 		adjustment += RWSEM_FLAG_WAITERS;
+		INIT_LIST_HEAD(&waiter.list);
+		sem->first_waiter = &waiter;
+	} else {
+		list_add_tail(&waiter.list, &first->list);
 	}
-	rwsem_add_waiter(sem, &waiter);
 
 	/* we're now waiting on the lock, but no longer actively locking */
 	count = atomic_long_add_return(adjustment, &sem->count);
@@ -1110,7 +1138,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
 static struct rw_semaphore __sched *
 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
 {
-	struct rwsem_waiter waiter;
+	struct rwsem_waiter waiter, *first;
 	DEFINE_WAKE_Q(wake_q);
 
 	/* do optimistic spinning and steal lock if possible */
@@ -1129,10 +1157,10 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
 	waiter.handoff_set = false;
 
 	raw_spin_lock_irq(&sem->wait_lock);
-	rwsem_add_waiter(sem, &waiter);
 
-	/* we're now waiting on the lock */
-	if (rwsem_first_waiter(sem) != &waiter) {
+	first = sem->first_waiter;
+	if (first) {
+		list_add_tail(&waiter.list, &first->list);
 		rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count),
 				       &wake_q);
 		if (!wake_q_empty(&wake_q)) {
@@ -1145,6 +1173,8 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
 			raw_spin_lock_irq(&sem->wait_lock);
 		}
 	} else {
+		INIT_LIST_HEAD(&waiter.list);
+		sem->first_waiter = &waiter;
 		atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);
 	}
 
@@ -1218,7 +1248,7 @@ static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
 
 	raw_spin_lock_irqsave(&sem->wait_lock, flags);
 
-	if (!list_empty(&sem->wait_list))
+	if (sem->first_waiter)
 		rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
 
 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
@@ -1239,7 +1269,7 @@ static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
 
 	raw_spin_lock_irqsave(&sem->wait_lock, flags);
 
-	if (!list_empty(&sem->wait_list))
+	if (sem->first_waiter)
 		rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q);
 
 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
@@ -1532,6 +1562,7 @@ static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)
  * lock for reading
  */
 void __sched down_read(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1541,6 +1572,7 @@ void __sched down_read(struct rw_semaphore *sem)
 EXPORT_SYMBOL(down_read);
 
 int __sched down_read_interruptible(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1555,6 +1587,7 @@ int __sched down_read_interruptible(struct rw_semaphore *sem)
 EXPORT_SYMBOL(down_read_interruptible);
 
 int __sched down_read_killable(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1572,6 +1605,7 @@ EXPORT_SYMBOL(down_read_killable);
  * trylock for reading -- returns 1 if successful, 0 if contention
  */
 int down_read_trylock(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	int ret = __down_read_trylock(sem);
 
@@ -1585,6 +1619,7 @@ EXPORT_SYMBOL(down_read_trylock);
  * lock for writing
  */
 void __sched down_write(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1596,6 +1631,7 @@ EXPORT_SYMBOL(down_write);
  * lock for writing
  */
 int __sched down_write_killable(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1614,6 +1650,7 @@ EXPORT_SYMBOL(down_write_killable);
  * trylock for writing -- returns 1 if successful, 0 if contention
  */
 int down_write_trylock(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	int ret = __down_write_trylock(sem);
 
@@ -1628,6 +1665,7 @@ EXPORT_SYMBOL(down_write_trylock);
  * release a read lock
  */
 void up_read(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	rwsem_release(&sem->dep_map, _RET_IP_);
 	__up_read(sem);
@@ -1638,6 +1676,7 @@ EXPORT_SYMBOL(up_read);
  * release a write lock
  */
 void up_write(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	rwsem_release(&sem->dep_map, _RET_IP_);
 	__up_write(sem);
@@ -1648,6 +1687,7 @@ EXPORT_SYMBOL(up_write);
  * downgrade write lock to read lock
  */
 void downgrade_write(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	lock_downgrade(&sem->dep_map, _RET_IP_);
 	__downgrade_write(sem);
@@ -1657,6 +1697,7 @@ EXPORT_SYMBOL(downgrade_write);
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
 void down_read_nested(struct rw_semaphore *sem, int subclass)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1665,6 +1706,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
 EXPORT_SYMBOL(down_read_nested);
 
 int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1679,6 +1721,7 @@ int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
 EXPORT_SYMBOL(down_read_killable_nested);
 
 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
@@ -1687,6 +1730,7 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
 EXPORT_SYMBOL(_down_write_nest_lock);
 
 void down_read_non_owner(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	might_sleep();
 	__down_read(sem);
@@ -1701,6 +1745,7 @@ void down_read_non_owner(struct rw_semaphore *sem)
 EXPORT_SYMBOL(down_read_non_owner);
 
 void down_write_nested(struct rw_semaphore *sem, int subclass)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1709,6 +1754,7 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)
 EXPORT_SYMBOL(down_write_nested);
 
 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
+	__no_context_analysis
 {
 	might_sleep();
 	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1724,6 +1770,7 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
 EXPORT_SYMBOL(down_write_killable_nested);
 
 void up_read_non_owner(struct rw_semaphore *sem)
+	__no_context_analysis
 {
 	DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
 	__up_read(sem);
diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c
index 3ef032e22f7ef..74d41433ba13c 100644
--- a/kernel/locking/semaphore.c
+++ b/kernel/locking/semaphore.c
@@ -21,7 +21,7 @@
  * too.
  *
  * The ->count variable represents how many more tasks can acquire this
- * semaphore.  If it's zero, there may be tasks waiting on the wait_list.
+ * semaphore.  If it's zero, there may be waiters.
  */
 
 #include <linux/compiler.h>
@@ -226,7 +226,7 @@ void __sched up(struct semaphore *sem)
 
 	hung_task_sem_clear_if_holder(sem);
 
-	if (likely(list_empty(&sem->wait_list)))
+	if (likely(!sem->first_waiter))
 		sem->count++;
 	else
 		__up(sem, &wake_q);
@@ -244,6 +244,21 @@ struct semaphore_waiter {
 	bool up;
 };
 
+static inline
+void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter)
+{
+	if (list_empty(&waiter->list)) {
+		sem->first_waiter = NULL;
+		return;
+	}
+
+	if (sem->first_waiter == waiter) {
+		sem->first_waiter = list_first_entry(&waiter->list,
+						     struct semaphore_waiter, list);
+	}
+	list_del(&waiter->list);
+}
+
 /*
  * Because this function is inlined, the 'state' parameter will be
  * constant, and thus optimised away by the compiler.  Likewise the
@@ -252,9 +267,15 @@ struct semaphore_waiter {
 static inline int __sched ___down_common(struct semaphore *sem, long state,
 								long timeout)
 {
-	struct semaphore_waiter waiter;
-
-	list_add_tail(&waiter.list, &sem->wait_list);
+	struct semaphore_waiter waiter, *first;
+
+	first = sem->first_waiter;
+	if (first) {
+		list_add_tail(&waiter.list, &first->list);
+	} else {
+		INIT_LIST_HEAD(&waiter.list);
+		sem->first_waiter = &waiter;
+	}
 	waiter.task = current;
 	waiter.up = false;
 
@@ -274,11 +295,11 @@ static inline int __sched ___down_common(struct semaphore *sem, long state,
 	}
 
  timed_out:
-	list_del(&waiter.list);
+	sem_del_waiter(sem, &waiter);
 	return -ETIME;
 
  interrupted:
-	list_del(&waiter.list);
+	sem_del_waiter(sem, &waiter);
 	return -EINTR;
 }
 
@@ -321,9 +342,9 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long timeout)
 static noinline void __sched __up(struct semaphore *sem,
 				  struct wake_q_head *wake_q)
 {
-	struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
-						struct semaphore_waiter, list);
-	list_del(&waiter->list);
+	struct semaphore_waiter *waiter = sem->first_waiter;
+
+	sem_del_waiter(sem, waiter);
 	waiter->up = true;
 	wake_q_add(wake_q, waiter->task);
 }
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index 7685defd7c526..b42d293da38b0 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -64,8 +64,9 @@ EXPORT_PER_CPU_SYMBOL(__mmiowb_state);
  * time (making _this_ CPU preemptible if possible), and we also signal
  * towards that other CPU that it should break the lock ASAP.
  */
-#define BUILD_LOCK_OPS(op, locktype)					\
+#define BUILD_LOCK_OPS(op, locktype, lock_ctx_op)			\
 static void __lockfunc __raw_##op##_lock(locktype##_t *lock)		\
+	lock_ctx_op(lock)						\
 {									\
 	for (;;) {							\
 		preempt_disable();					\
@@ -78,6 +79,7 @@ static void __lockfunc __raw_##op##_lock(locktype##_t *lock)		\
 }									\
 									\
 static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \
+	lock_ctx_op(lock)						\
 {									\
 	unsigned long flags;						\
 									\
@@ -96,11 +98,13 @@ static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \
 }									\
 									\
 static void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock)	\
+	lock_ctx_op(lock)						\
 {									\
 	_raw_##op##_lock_irqsave(lock);					\
 }									\
 									\
 static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock)		\
+	lock_ctx_op(lock)						\
 {									\
 	unsigned long flags;						\
 									\
@@ -123,11 +127,11 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock)		\
  *         __[spin|read|write]_lock_irqsave()
  *         __[spin|read|write]_lock_bh()
  */
-BUILD_LOCK_OPS(spin, raw_spinlock);
+BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
 
 #ifndef CONFIG_PREEMPT_RT
-BUILD_LOCK_OPS(read, rwlock);
-BUILD_LOCK_OPS(write, rwlock);
+BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
+BUILD_LOCK_OPS(write, rwlock, __acquires);
 #endif
 
 #endif
diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h
index 31a785afee6c0..b1834ab7e7824 100644
--- a/kernel/locking/ww_mutex.h
+++ b/kernel/locking/ww_mutex.h
@@ -4,24 +4,21 @@
 
 #define MUTEX		mutex
 #define MUTEX_WAITER	mutex_waiter
+#define WAIT_LOCK	wait_lock
 
 static inline struct mutex_waiter *
 __ww_waiter_first(struct mutex *lock)
+	__must_hold(&lock->wait_lock)
 {
-	struct mutex_waiter *w;
-
-	w = list_first_entry(&lock->wait_list, struct mutex_waiter, list);
-	if (list_entry_is_head(w, &lock->wait_list, list))
-		return NULL;
-
-	return w;
+	return lock->first_waiter;
 }
 
 static inline struct mutex_waiter *
 __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w)
+	__must_hold(&lock->wait_lock)
 {
 	w = list_next_entry(w, list);
-	if (list_entry_is_head(w, &lock->wait_list, list))
+	if (lock->first_waiter == w)
 		return NULL;
 
 	return w;
@@ -29,9 +26,10 @@ __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w)
 
 static inline struct mutex_waiter *
 __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w)
+	__must_hold(&lock->wait_lock)
 {
 	w = list_prev_entry(w, list);
-	if (list_entry_is_head(w, &lock->wait_list, list))
+	if (lock->first_waiter == w)
 		return NULL;
 
 	return w;
@@ -39,23 +37,20 @@ __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w)
 
 static inline struct mutex_waiter *
 __ww_waiter_last(struct mutex *lock)
+	__must_hold(&lock->wait_lock)
 {
-	struct mutex_waiter *w;
-
-	w = list_last_entry(&lock->wait_list, struct mutex_waiter, list);
-	if (list_entry_is_head(w, &lock->wait_list, list))
-		return NULL;
+	struct mutex_waiter *w = lock->first_waiter;
 
+	if (w)
+		w = list_prev_entry(w, list);
 	return w;
 }
 
 static inline void
 __ww_waiter_add(struct mutex *lock, struct mutex_waiter *waiter, struct mutex_waiter *pos)
+	__must_hold(&lock->wait_lock)
 {
-	struct list_head *p = &lock->wait_list;
-	if (pos)
-		p = &pos->list;
-	__mutex_add_waiter(lock, waiter, p);
+	__mutex_add_waiter(lock, waiter, pos);
 }
 
 static inline struct task_struct *
@@ -71,16 +66,19 @@ __ww_mutex_has_waiters(struct mutex *lock)
 }
 
 static inline void lock_wait_lock(struct mutex *lock, unsigned long *flags)
+	__acquires(&lock->wait_lock)
 {
 	raw_spin_lock_irqsave(&lock->wait_lock, *flags);
 }
 
 static inline void unlock_wait_lock(struct mutex *lock, unsigned long *flags)
+	__releases(&lock->wait_lock)
 {
 	raw_spin_unlock_irqrestore(&lock->wait_lock, *flags);
 }
 
 static inline void lockdep_assert_wait_lock_held(struct mutex *lock)
+	__must_hold(&lock->wait_lock)
 {
 	lockdep_assert_held(&lock->wait_lock);
 }
@@ -89,9 +87,11 @@ static inline void lockdep_assert_wait_lock_held(struct mutex *lock)
 
 #define MUTEX		rt_mutex
 #define MUTEX_WAITER	rt_mutex_waiter
+#define WAIT_LOCK	rtmutex.wait_lock
 
 static inline struct rt_mutex_waiter *
 __ww_waiter_first(struct rt_mutex *lock)
+	__must_hold(&lock->rtmutex.wait_lock)
 {
 	struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root);
 	if (!n)
@@ -119,6 +119,7 @@ __ww_waiter_prev(struct rt_mutex *lock, struct rt_mutex_waiter *w)
 
 static inline struct rt_mutex_waiter *
 __ww_waiter_last(struct rt_mutex *lock)
+	__must_hold(&lock->rtmutex.wait_lock)
 {
 	struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root);
 	if (!n)
@@ -140,21 +141,25 @@ __ww_mutex_owner(struct rt_mutex *lock)
 
 static inline bool
 __ww_mutex_has_waiters(struct rt_mutex *lock)
+	__must_hold(&lock->rtmutex.wait_lock)
 {
 	return rt_mutex_has_waiters(&lock->rtmutex);
 }
 
 static inline void lock_wait_lock(struct rt_mutex *lock, unsigned long *flags)
+	__acquires(&lock->rtmutex.wait_lock)
 {
 	raw_spin_lock_irqsave(&lock->rtmutex.wait_lock, *flags);
 }
 
 static inline void unlock_wait_lock(struct rt_mutex *lock, unsigned long *flags)
+	__releases(&lock->rtmutex.wait_lock)
 {
 	raw_spin_unlock_irqrestore(&lock->rtmutex.wait_lock, *flags);
 }
 
 static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock)
+	__must_hold(&lock->rtmutex.wait_lock)
 {
 	lockdep_assert_held(&lock->rtmutex.wait_lock);
 }
@@ -307,6 +312,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
 			     struct ww_acquire_ctx *ww_ctx,
 			     struct ww_acquire_ctx *hold_ctx,
 			     struct wake_q_head *wake_q)
+	__must_hold(&lock->WAIT_LOCK)
 {
 	struct task_struct *owner = __ww_mutex_owner(lock);
 
@@ -371,6 +377,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
 static void
 __ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx,
 			 struct wake_q_head *wake_q)
+	__must_hold(&lock->WAIT_LOCK)
 {
 	struct MUTEX_WAITER *cur;
 
@@ -397,6 +404,7 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 {
 	DEFINE_WAKE_Q(wake_q);
 	unsigned long flags;
+	bool has_waiters;
 
 	ww_mutex_lock_acquired(lock, ctx);
 
@@ -418,7 +426,8 @@ ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 	 * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx
 	 * and/or !empty list.
 	 */
-	if (likely(!__ww_mutex_has_waiters(&lock->base)))
+	has_waiters = data_race(__ww_mutex_has_waiters(&lock->base));
+	if (likely(!has_waiters))
 		return;
 
 	/*
@@ -464,6 +473,7 @@ __ww_mutex_kill(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx)
 static inline int
 __ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter,
 		      struct ww_acquire_ctx *ctx)
+	__must_hold(&lock->WAIT_LOCK)
 {
 	struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
 	struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
@@ -514,6 +524,7 @@ __ww_mutex_add_waiter(struct MUTEX_WAITER *waiter,
 		      struct MUTEX *lock,
 		      struct ww_acquire_ctx *ww_ctx,
 		      struct wake_q_head *wake_q)
+	__must_hold(&lock->WAIT_LOCK)
 {
 	struct MUTEX_WAITER *cur, *pos = NULL;
 	bool is_wait_die;
diff --git a/kernel/locking/ww_rt_mutex.c b/kernel/locking/ww_rt_mutex.c
index c7196de838edc..e07fb3b96bc33 100644
--- a/kernel/locking/ww_rt_mutex.c
+++ b/kernel/locking/ww_rt_mutex.c
@@ -90,6 +90,7 @@ ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 EXPORT_SYMBOL(ww_mutex_lock_interruptible);
 
 void __sched ww_mutex_unlock(struct ww_mutex *lock)
+	__no_context_analysis
 {
 	struct rt_mutex *rtm = &lock->base;
 
diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c
index 0fc11e45df9b9..743cf7033c37f 100644
--- a/kernel/module/kallsyms.c
+++ b/kernel/module/kallsyms.c
@@ -494,3 +494,79 @@ int module_kallsyms_on_each_symbol(const char *modname,
 	mutex_unlock(&module_mutex);
 	return ret;
 }
+
+#include <linux/mod_lineinfo.h>
+
+/*
+ * Look up source file:line for an address within a loaded module.
+ * Uses the .mod_lineinfo section embedded in the .ko at build time.
+ *
+ * Safe in NMI/panic context: no locks, no allocations.
+ * Caller must hold RCU read lock (or be in a context where the module
+ * cannot be unloaded).
+ */
+bool module_lookup_lineinfo(struct module *mod, unsigned long addr,
+			    const char **file, unsigned int *line)
+{
+	const struct mod_lineinfo_header *hdr;
+	struct lineinfo_table tbl;
+	const void *base;
+	u32 section_size;
+	unsigned long text_base;
+	unsigned long long raw_offset;
+
+	if (!IS_ENABLED(CONFIG_KALLSYMS_LINEINFO_MODULES))
+		return false;
+
+	base = mod->lineinfo_data;
+	if (!base)
+		return false;
+
+	section_size = mod->lineinfo_data_size;
+	if (section_size < sizeof(*hdr))
+		return false;
+
+	hdr = base;
+
+	if (hdr->num_entries == 0 || hdr->num_blocks == 0)
+		return false;
+
+	/* Validate each sub-array fits within the section */
+	if (hdr->blocks_offset + hdr->blocks_size > section_size)
+		return false;
+	if (hdr->data_offset + hdr->data_size > section_size)
+		return false;
+	if (hdr->files_offset + hdr->files_size > section_size)
+		return false;
+	if (hdr->filenames_offset + hdr->filenames_size > section_size)
+		return false;
+
+	/* Validate array sizes match declared counts */
+	if (hdr->blocks_size < hdr->num_blocks * 2 * sizeof(u32))
+		return false;
+	if (hdr->files_size < hdr->num_files * sizeof(u32))
+		return false;
+
+	/* Compute offset from module .text base */
+	text_base = (unsigned long)mod->mem[MOD_TEXT].base;
+	if (addr < text_base)
+		return false;
+
+	raw_offset = addr - text_base;
+	if (raw_offset > UINT_MAX)
+		return false;
+
+	tbl.blk_addrs	= base + hdr->blocks_offset;
+	tbl.blk_offsets	= base + hdr->blocks_offset +
+			  hdr->num_blocks * sizeof(u32);
+	tbl.data	= base + hdr->data_offset;
+	tbl.data_size	= hdr->data_size;
+	tbl.file_offsets = base + hdr->files_offset;
+	tbl.filenames	= base + hdr->filenames_offset;
+	tbl.num_entries	= hdr->num_entries;
+	tbl.num_blocks	= hdr->num_blocks;
+	tbl.num_files	= hdr->num_files;
+	tbl.filenames_size = hdr->filenames_size;
+
+	return lineinfo_search(&tbl, (unsigned int)raw_offset, file, line);
+}
diff --git a/kernel/module/main.c b/kernel/module/main.c
index c3ce106c70af1..d659821bad257 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -607,6 +607,36 @@ static const struct module_attribute modinfo_##field = {              \
 MODINFO_ATTR(version);
 MODINFO_ATTR(srcversion);
 
+static void setup_modinfo_import_ns(struct module *mod, const char *s)
+{
+	mod->imported_namespaces = NULL;
+}
+
+static ssize_t show_modinfo_import_ns(const struct module_attribute *mattr,
+				      struct module_kobject *mk, char *buffer)
+{
+	return sysfs_emit(buffer, "%s\n", mk->mod->imported_namespaces);
+}
+
+static int modinfo_import_ns_exists(struct module *mod)
+{
+	return mod->imported_namespaces != NULL;
+}
+
+static void free_modinfo_import_ns(struct module *mod)
+{
+	kfree(mod->imported_namespaces);
+	mod->imported_namespaces = NULL;
+}
+
+static const struct module_attribute modinfo_import_ns = {
+	.attr = { .name = "import_ns", .mode = 0444 },
+	.show = show_modinfo_import_ns,
+	.setup = setup_modinfo_import_ns,
+	.test = modinfo_import_ns_exists,
+	.free = free_modinfo_import_ns,
+};
+
 static struct {
 	char name[MODULE_NAME_LEN];
 	char taints[MODULE_FLAGS_BUF_SIZE];
@@ -1058,6 +1088,7 @@ const struct module_attribute *const modinfo_attrs[] = {
 	&module_uevent,
 	&modinfo_version,
 	&modinfo_srcversion,
+	&modinfo_import_ns,
 	&modinfo_initstate,
 	&modinfo_coresize,
 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
@@ -1408,7 +1439,7 @@ static void free_module(struct module *mod)
 	module_unload_free(mod);
 
 	/* Free any allocated parameters. */
-	destroy_params(mod->kp, mod->num_kp);
+	module_destroy_params(mod->kp, mod->num_kp);
 
 	if (is_livepatch_module(mod))
 		free_module_elf(mod);
@@ -1760,11 +1791,43 @@ static void module_license_taint_check(struct module *mod, const char *license)
 	}
 }
 
+static int copy_modinfo_import_ns(struct module *mod, struct load_info *info)
+{
+	char *ns;
+	size_t len, total_len = 0;
+	char *buf, *p;
+
+	for_each_modinfo_entry(ns, info, "import_ns")
+		total_len += strlen(ns) + 1;
+
+	if (!total_len) {
+		mod->imported_namespaces = NULL;
+		return 0;
+	}
+
+	buf = kmalloc(total_len, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	p = buf;
+	for_each_modinfo_entry(ns, info, "import_ns") {
+		len = strlen(ns);
+		memcpy(p, ns, len);
+		p += len;
+		*p++ = '\n';
+	}
+	/* Replace trailing newline with null terminator. */
+	*(p - 1) = '\0';
+
+	mod->imported_namespaces = buf;
+	return 0;
+}
+
 static int setup_modinfo(struct module *mod, struct load_info *info)
 {
 	const struct module_attribute *attr;
 	char *imported_namespace;
-	int i;
+	int i, err;
 
 	for (i = 0; (attr = modinfo_attrs[i]); i++) {
 		if (attr->setup)
@@ -1783,6 +1846,10 @@ static int setup_modinfo(struct module *mod, struct load_info *info)
 		}
 	}
 
+	err = copy_modinfo_import_ns(mod, info);
+	if (err)
+		return err;
+
 	return 0;
 }
 
@@ -2655,6 +2722,9 @@ static int find_module_sections(struct module *mod, struct load_info *info)
 	mod->btf_base_data = any_section_objs(info, ".BTF.base", 1,
 					      &mod->btf_base_data_size);
 #endif
+	if (IS_ENABLED(CONFIG_KALLSYMS_LINEINFO_MODULES))
+		mod->lineinfo_data = any_section_objs(info, ".mod_lineinfo", 1,
+						      &mod->lineinfo_data_size);
 #ifdef CONFIG_JUMP_LABEL
 	mod->jump_entries = section_objs(info, "__jump_table",
 					sizeof(*mod->jump_entries),
@@ -3519,7 +3589,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
 	mod_sysfs_teardown(mod);
  coming_cleanup:
 	mod->state = MODULE_STATE_GOING;
-	destroy_params(mod->kp, mod->num_kp);
+	module_destroy_params(mod->kp, mod->num_kp);
 	blocking_notifier_call_chain(&module_notify_list,
 				     MODULE_STATE_GOING, mod);
 	klp_module_going(mod);
diff --git a/kernel/padata.c b/kernel/padata.c
index 8657e6e0c224a..9e7cfa5ed55bc 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -732,32 +732,22 @@ EXPORT_SYMBOL(padata_set_cpumask);
 
 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
 {
-	int err = 0;
-
-	if (cpumask_test_cpu(cpu, cpu_online_mask)) {
-		err = padata_replace(pinst);
+	int err = padata_replace(pinst);
 
-		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
-		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
-			__padata_start(pinst);
-	}
+	if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
+	    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
+		__padata_start(pinst);
 
 	return err;
 }
 
 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
 {
-	int err = 0;
-
-	if (!cpumask_test_cpu(cpu, cpu_online_mask)) {
-		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
-		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
-			__padata_stop(pinst);
-
-		err = padata_replace(pinst);
-	}
+	if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
+	    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
+		__padata_stop(pinst);
 
-	return err;
+	return padata_replace(pinst);
 }
 
 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
diff --git a/kernel/panic.c b/kernel/panic.c
index c78600212b6c1..20feada5319d4 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -801,6 +801,8 @@ EXPORT_SYMBOL(panic);
  * Documentation/admin-guide/tainted-kernels.rst, including its
  * small shell script that prints the TAINT_FLAGS_COUNT bits of
  * /proc/sys/kernel/tainted.
+ *
+ * Also, update INIT_TAINT_BUF_MAX below.
  */
 const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = {
 	TAINT_FLAG(PROPRIETARY_MODULE,		'P', 'G'),
@@ -854,15 +856,54 @@ static void print_tainted_seq(struct seq_buf *s, bool verbose)
 	}
 }
 
+/* The initial buffer can accommodate all taint flags in verbose
+ * mode, with some headroom. Once the allocator is available, the
+ * exact size is allocated dynamically; the initial buffer remains
+ * as a fallback if allocation fails.
+ *
+ * The verbose taint string currently requires up to 327 characters.
+ */
+#define INIT_TAINT_BUF_MAX 350
+
+static char init_taint_buf[INIT_TAINT_BUF_MAX] __initdata;
+static char *taint_buf __refdata = init_taint_buf;
+static size_t taint_buf_size = INIT_TAINT_BUF_MAX;
+
+static __init int alloc_taint_buf(void)
+{
+	int i;
+	char *buf;
+	size_t size = 0;
+
+	size += sizeof("Tainted: ") - 1;
+	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
+		size += 2; /* For ", " */
+		size += 4; /* For "[%c]=" */
+		size += strlen(taint_flags[i].desc);
+	}
+
+	size += 1; /* For NULL terminator */
+
+	buf = kmalloc(size, GFP_KERNEL);
+
+	if (!buf) {
+		panic("Failed to allocate taint string buffer");
+	}
+
+	taint_buf = buf;
+	taint_buf_size = size;
+
+	return 0;
+}
+postcore_initcall(alloc_taint_buf);
+
 static const char *_print_tainted(bool verbose)
 {
-	/* FIXME: what should the size be? */
-	static char buf[sizeof(taint_flags)];
 	struct seq_buf s;
 
 	BUILD_BUG_ON(ARRAY_SIZE(taint_flags) != TAINT_FLAGS_COUNT);
 
-	seq_buf_init(&s, buf, sizeof(buf));
+	seq_buf_init(&s, taint_buf, taint_buf_size);
 
 	print_tainted_seq(&s, verbose);
 
diff --git a/kernel/params.c b/kernel/params.c
index 7188a12dbe864..74d620bc25217 100644
--- a/kernel/params.c
+++ b/kernel/params.c
@@ -161,7 +161,7 @@ static int parse_one(char *param,
 char *parse_args(const char *doing,
 		 char *args,
 		 const struct kernel_param *params,
-		 unsigned num,
+		 unsigned int num,
 		 s16 min_level,
 		 s16 max_level,
 		 void *arg, parse_unknown_fn unknown)
@@ -745,15 +745,6 @@ void module_param_sysfs_remove(struct module *mod)
 }
 #endif
 
-void destroy_params(const struct kernel_param *params, unsigned num)
-{
-	unsigned int i;
-
-	for (i = 0; i < num; i++)
-		if (params[i].ops->free)
-			params[i].ops->free(params[i].arg);
-}
-
 struct module_kobject * __init_or_module
 lookup_or_create_module_kobject(const char *name)
 {
@@ -985,3 +976,21 @@ static int __init param_sysfs_builtin_init(void)
 late_initcall(param_sysfs_builtin_init);
 
 #endif /* CONFIG_SYSFS */
+
+#ifdef CONFIG_MODULES
+
+/*
+ * module_destroy_params - free all parameters for one module
+ * @params: module parameters (array)
+ * @num: number of module parameters
+ */
+void module_destroy_params(const struct kernel_param *params, unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++)
+		if (params[i].ops->free)
+			params[i].ops->free(params[i].arg);
+}
+
+#endif /* CONFIG_MODULES */
diff --git a/kernel/pid.c b/kernel/pid.c
index 3b96571d0fe65..2f1dbcbc2349a 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -131,9 +131,8 @@ void free_pid(struct pid *pid)
 			wake_up_process(ns->child_reaper);
 			break;
 		case PIDNS_ADDING:
-			/* Handle a fork failure of the first process */
-			WARN_ON(ns->child_reaper);
-			ns->pid_allocated = 0;
+			/* Only possible if the 1st fork fails */
+			WARN_ON(READ_ONCE(ns->child_reaper));
 			break;
 		}
 
@@ -236,6 +235,10 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *arg_set_tid,
 	retried_preload = false;
 	idr_preload(GFP_KERNEL);
 	spin_lock(&pidmap_lock);
+	/* For the case when the previous attempt to create init failed */
+	if (ns->pid_allocated == PIDNS_ADDING)
+		idr_set_cursor(&ns->idr, 0);
+
 	for (tmp = ns, i = ns->level; i >= 0;) {
 		int tid = set_tid[ns->level - i];
 
@@ -311,6 +314,11 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *arg_set_tid,
 	 *
 	 * This can't be done earlier because we need to preserve other
 	 * error conditions.
+	 *
+	 * We need this even if copy_process() does the same check. If two
+	 * or more tasks from parent namespace try to inject a child into a
+	 * dead namespace, one of free_pid() calls from the copy_process()
+	 * error path may try to wakeup the possibly freed ns->child_reaper.
 	 */
 	retval = -ENOMEM;
 	if (unlikely(!(ns->pid_allocated & PIDNS_ADDING)))
@@ -338,10 +346,6 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *arg_set_tid,
 		idr_remove(&upid->ns->idr, upid->nr);
 	}
 
-	/* On failure to allocate the first pid, reset the state */
-	if (ns->pid_allocated == PIDNS_ADDING)
-		idr_set_cursor(&ns->idr, 0);
-
 	spin_unlock(&pidmap_lock);
 	idr_preload_end();
 
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 6e1321837c668..7dcccf378cc2f 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -13,6 +13,7 @@
 #include <linux/version.h>
 #include <linux/module.h>
 #include <linux/mm.h>
+#include <linux/mmzone_lock.h>
 #include <linux/suspend.h>
 #include <linux/delay.h>
 #include <linux/bitops.h>
@@ -1251,7 +1252,7 @@ static void mark_free_pages(struct zone *zone)
 	if (zone_is_empty(zone))
 		return;
 
-	spin_lock_irqsave(&zone->lock, flags);
+	zone_lock_irqsave(zone, flags);
 
 	max_zone_pfn = zone_end_pfn(zone);
 	for_each_valid_pfn(pfn, zone->zone_start_pfn, max_zone_pfn) {
@@ -1284,7 +1285,7 @@ static void mark_free_pages(struct zone *zone)
 			}
 		}
 	}
-	spin_unlock_irqrestore(&zone->lock, flags);
+	zone_unlock_irqrestore(zone, flags);
 }
 
 #ifdef CONFIG_HIGHMEM
diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h
index 1651b53ece34f..7824e6936e36a 100644
--- a/kernel/printk/printk_ringbuffer.h
+++ b/kernel/printk/printk_ringbuffer.h
@@ -127,7 +127,7 @@ enum desc_state {
 };
 
 #define _DATA_SIZE(sz_bits)	(1UL << (sz_bits))
-#define _DESCS_COUNT(ct_bits)	(1U << (ct_bits))
+#define _DESCS_COUNT(ct_bits)	(1UL << (ct_bits))
 #define DESC_SV_BITS		BITS_PER_LONG
 #define DESC_FLAGS_SHIFT	(DESC_SV_BITS - 2)
 #define DESC_FLAGS_MASK		(3UL << DESC_FLAGS_SHIFT)
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 392ec2f75f013..68c17daef8d40 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -549,7 +549,8 @@ static bool __ptrace_detach(struct task_struct *tracer, struct task_struct *p)
 	if (!dead && thread_group_empty(p)) {
 		if (!same_thread_group(p->real_parent, tracer))
 			dead = do_notify_parent(p, p->exit_signal);
-		else if (ignoring_children(tracer->sighand)) {
+		else if (ignoring_children(tracer->sighand) ||
+			 p->signal->autoreap) {
 			__wake_up_parent(p, tracer);
 			dead = true;
 		}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 496dff740dcaf..a66ea2fba6316 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -687,11 +687,6 @@ bool raw_spin_rq_trylock(struct rq *rq)
 	}
 }
 
-void raw_spin_rq_unlock(struct rq *rq)
-{
-	raw_spin_unlock(rq_lockp(rq));
-}
-
 /*
  * double_rq_lock - safely lock two runqueues
  */
@@ -872,7 +867,14 @@ void update_rq_clock(struct rq *rq)
  * Use HR-timers to deliver accurate preemption points.
  */
 
-static void hrtick_clear(struct rq *rq)
+enum {
+	HRTICK_SCHED_NONE		= 0,
+	HRTICK_SCHED_DEFER		= BIT(1),
+	HRTICK_SCHED_START		= BIT(2),
+	HRTICK_SCHED_REARM_HRTIMER	= BIT(3)
+};
+
+static void __used hrtick_clear(struct rq *rq)
 {
 	if (hrtimer_active(&rq->hrtick_timer))
 		hrtimer_cancel(&rq->hrtick_timer);
@@ -897,12 +899,24 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
 	return HRTIMER_NORESTART;
 }
 
-static void __hrtick_restart(struct rq *rq)
+static inline bool hrtick_needs_rearm(struct hrtimer *timer, ktime_t expires)
+{
+	/*
+	 * Queued is false when the timer is not started or currently
+	 * running the callback. In both cases, restart. If queued check
+	 * whether the expiry time actually changes substantially.
+	 */
+	return !hrtimer_is_queued(timer) ||
+		abs(expires - hrtimer_get_expires(timer)) > 5000;
+}
+
+static void hrtick_cond_restart(struct rq *rq)
 {
 	struct hrtimer *timer = &rq->hrtick_timer;
 	ktime_t time = rq->hrtick_time;
 
-	hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
+	if (hrtick_needs_rearm(timer, time))
+		hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
 }
 
 /*
@@ -914,7 +928,7 @@ static void __hrtick_start(void *arg)
 	struct rq_flags rf;
 
 	rq_lock(rq, &rf);
-	__hrtick_restart(rq);
+	hrtick_cond_restart(rq);
 	rq_unlock(rq, &rf);
 }
 
@@ -925,7 +939,6 @@ static void __hrtick_start(void *arg)
  */
 void hrtick_start(struct rq *rq, u64 delay)
 {
-	struct hrtimer *timer = &rq->hrtick_timer;
 	s64 delta;
 
 	/*
@@ -933,27 +946,67 @@ void hrtick_start(struct rq *rq, u64 delay)
 	 * doesn't make sense and can cause timer DoS.
 	 */
 	delta = max_t(s64, delay, 10000LL);
-	rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta);
+
+	/*
+	 * If this is in the middle of schedule() only note the delay
+	 * and let hrtick_schedule_exit() deal with it.
+	 */
+	if (rq->hrtick_sched) {
+		rq->hrtick_sched |= HRTICK_SCHED_START;
+		rq->hrtick_delay = delta;
+		return;
+	}
+
+	rq->hrtick_time = ktime_add_ns(ktime_get(), delta);
+	if (!hrtick_needs_rearm(&rq->hrtick_timer, rq->hrtick_time))
+		return;
 
 	if (rq == this_rq())
-		__hrtick_restart(rq);
+		hrtimer_start(&rq->hrtick_timer, rq->hrtick_time, HRTIMER_MODE_ABS_PINNED_HARD);
 	else
 		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
 }
 
-static void hrtick_rq_init(struct rq *rq)
+static inline void hrtick_schedule_enter(struct rq *rq)
 {
-	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
-	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+	rq->hrtick_sched = HRTICK_SCHED_DEFER;
+	if (hrtimer_test_and_clear_rearm_deferred())
+		rq->hrtick_sched |= HRTICK_SCHED_REARM_HRTIMER;
 }
-#else /* !CONFIG_SCHED_HRTICK: */
-static inline void hrtick_clear(struct rq *rq)
+
+static inline void hrtick_schedule_exit(struct rq *rq)
 {
+	if (rq->hrtick_sched & HRTICK_SCHED_START) {
+		rq->hrtick_time = ktime_add_ns(ktime_get(), rq->hrtick_delay);
+		hrtick_cond_restart(rq);
+	} else if (idle_rq(rq)) {
+		/*
+		 * No need for using hrtimer_is_active(). The timer is CPU local
+		 * and interrupts are disabled, so the callback cannot be
+		 * running and the queued state is valid.
+		 */
+		if (hrtimer_is_queued(&rq->hrtick_timer))
+			hrtimer_cancel(&rq->hrtick_timer);
+	}
+
+	if (rq->hrtick_sched & HRTICK_SCHED_REARM_HRTIMER)
+		__hrtimer_rearm_deferred();
+
+	rq->hrtick_sched = HRTICK_SCHED_NONE;
 }
 
-static inline void hrtick_rq_init(struct rq *rq)
+static void hrtick_rq_init(struct rq *rq)
 {
+	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
+	rq->hrtick_sched = HRTICK_SCHED_NONE;
+	hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
 }
+#else /* !CONFIG_SCHED_HRTICK: */
+static inline void hrtick_clear(struct rq *rq) { }
+static inline void hrtick_rq_init(struct rq *rq) { }
+static inline void hrtick_schedule_enter(struct rq *rq) { }
+static inline void hrtick_schedule_exit(struct rq *rq) { }
 #endif /* !CONFIG_SCHED_HRTICK */
 
 /*
@@ -4721,7 +4774,7 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
 		p->sched_class->task_fork(p);
 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
-	return scx_fork(p);
+	return scx_fork(p, kargs);
 }
 
 void sched_cancel_fork(struct task_struct *p)
@@ -5032,6 +5085,7 @@ static inline void finish_lock_switch(struct rq *rq)
 	 */
 	spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_);
 	__balance_callbacks(rq, NULL);
+	hrtick_schedule_exit(rq);
 	raw_spin_rq_unlock_irq(rq);
 }
 
@@ -5681,7 +5735,7 @@ static void sched_tick_remote(struct work_struct *work)
 	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
 	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
 	if (os == TICK_SCHED_REMOTE_RUNNING)
-		queue_delayed_work(system_unbound_wq, dwork, HZ);
+		queue_delayed_work(system_dfl_wq, dwork, HZ);
 }
 
 static void sched_tick_start(int cpu)
@@ -5700,7 +5754,7 @@ static void sched_tick_start(int cpu)
 	if (os == TICK_SCHED_REMOTE_OFFLINE) {
 		twork->cpu = cpu;
 		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
-		queue_delayed_work(system_unbound_wq, &twork->work, HZ);
+		queue_delayed_work(system_dfl_wq, &twork->work, HZ);
 	}
 }
 
@@ -6785,9 +6839,6 @@ static void __sched notrace __schedule(int sched_mode)
 
 	schedule_debug(prev, preempt);
 
-	if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
-		hrtick_clear(rq);
-
 	klp_sched_try_switch(prev);
 
 	local_irq_disable();
@@ -6814,6 +6865,8 @@ static void __sched notrace __schedule(int sched_mode)
 	rq_lock(rq, &rf);
 	smp_mb__after_spinlock();
 
+	hrtick_schedule_enter(rq);
+
 	/* Promote REQ to ACT */
 	rq->clock_update_flags <<= 1;
 	update_rq_clock(rq);
@@ -6916,6 +6969,7 @@ static void __sched notrace __schedule(int sched_mode)
 
 		rq_unpin_lock(rq, &rf);
 		__balance_callbacks(rq, NULL);
+		hrtick_schedule_exit(rq);
 		raw_spin_rq_unlock_irq(rq);
 	}
 	trace_sched_exit_tp(is_switch);
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index d08b004293234..9869025941a0a 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se)
 		act = ns_to_ktime(dl_next_period(dl_se));
 	}
 
-	now = hrtimer_cb_get_time(timer);
+	now = ktime_get();
 	delta = ktime_to_ns(now) - rq_clock(rq);
 	act = ktime_add_ns(act, delta);
 
@@ -2142,10 +2142,14 @@ update_stats_dequeue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,
 			int flags)
 {
 	struct task_struct *p = dl_task_of(dl_se);
+	struct rq *rq = rq_of_dl_rq(dl_rq);
 
 	if (!schedstat_enabled())
 		return;
 
+	if (p != rq->curr)
+		update_stats_wait_end_dl(dl_rq, dl_se);
+
 	if ((flags & DEQUEUE_SLEEP)) {
 		unsigned int state;
 
@@ -3613,13 +3617,26 @@ void __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
 	dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);
 }
 
-void __getparam_dl(struct task_struct *p, struct sched_attr *attr)
+void __getparam_dl(struct task_struct *p, struct sched_attr *attr, unsigned int flags)
 {
 	struct sched_dl_entity *dl_se = &p->dl;
+	struct rq *rq = task_rq(p);
+	u64 adj_deadline;
 
 	attr->sched_priority = p->rt_priority;
-	attr->sched_runtime = dl_se->dl_runtime;
-	attr->sched_deadline = dl_se->dl_deadline;
+	if (flags & SCHED_GETATTR_FLAG_DL_DYNAMIC) {
+		guard(raw_spinlock_irq)(&rq->__lock);
+		update_rq_clock(rq);
+		if (task_current(rq, p))
+			update_curr_dl(rq);
+
+		attr->sched_runtime = dl_se->runtime;
+		adj_deadline = dl_se->deadline - rq_clock(rq) + ktime_get_ns();
+		attr->sched_deadline = adj_deadline;
+	} else {
+		attr->sched_runtime = dl_se->dl_runtime;
+		attr->sched_deadline = dl_se->dl_deadline;
+	}
 	attr->sched_period = dl_se->dl_period;
 	attr->sched_flags &= ~SCHED_DL_FLAGS;
 	attr->sched_flags |= dl_se->flags;
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index b24f40f05019a..6246008c431e6 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -8,6 +8,7 @@
  */
 #include <linux/debugfs.h>
 #include <linux/nmi.h>
+#include <linux/log2.h>
 #include "sched.h"
 
 /*
@@ -901,10 +902,13 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
 
 void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 {
-	s64 left_vruntime = -1, zero_vruntime, right_vruntime = -1, left_deadline = -1, spread;
+	s64 left_vruntime = -1, right_vruntime = -1, left_deadline = -1, spread;
+	s64 zero_vruntime = -1, sum_w_vruntime = -1;
 	struct sched_entity *last, *first, *root;
 	struct rq *rq = cpu_rq(cpu);
+	unsigned int sum_shift;
 	unsigned long flags;
+	u64 sum_weight;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	SEQ_printf(m, "\n");
@@ -925,6 +929,9 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 	if (last)
 		right_vruntime = last->vruntime;
 	zero_vruntime = cfs_rq->zero_vruntime;
+	sum_w_vruntime = cfs_rq->sum_w_vruntime;
+	sum_weight = cfs_rq->sum_weight;
+	sum_shift = cfs_rq->sum_shift;
 	raw_spin_rq_unlock_irqrestore(rq, flags);
 
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "left_deadline",
@@ -933,6 +940,11 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 			SPLIT_NS(left_vruntime));
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "zero_vruntime",
 			SPLIT_NS(zero_vruntime));
+	SEQ_printf(m, "  .%-30s: %Ld (%d bits)\n", "sum_w_vruntime",
+		   sum_w_vruntime, ilog2(abs(sum_w_vruntime)));
+	SEQ_printf(m, "  .%-30s: %Lu\n", "sum_weight",
+		   sum_weight);
+	SEQ_printf(m, "  .%-30s: %u\n", "sum_shift", sum_shift);
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "avg_vruntime",
 			SPLIT_NS(avg_vruntime(cfs_rq)));
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "right_vruntime",
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 26a6ac2f88267..94548ee9ad858 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -9,6 +9,8 @@
 #include <linux/btf_ids.h>
 #include "ext_idle.h"
 
+static DEFINE_RAW_SPINLOCK(scx_sched_lock);
+
 /*
  * NOTE: sched_ext is in the process of growing multiple scheduler support and
  * scx_root usage is in a transitional state. Naked dereferences are safe if the
@@ -17,7 +19,23 @@
  * are used as temporary markers to indicate that the dereferences need to be
  * updated to point to the associated scheduler instances rather than scx_root.
  */
-static struct scx_sched __rcu *scx_root;
+struct scx_sched __rcu *scx_root;
+
+/*
+ * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock.
+ * Readers can hold either or rcu_read_lock().
+ */
+static LIST_HEAD(scx_sched_all);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+static const struct rhashtable_params scx_sched_hash_params = {
+	.key_len		= sizeof_field(struct scx_sched, ops.sub_cgroup_id),
+	.key_offset		= offsetof(struct scx_sched, ops.sub_cgroup_id),
+	.head_offset		= offsetof(struct scx_sched, hash_node),
+};
+
+static struct rhashtable scx_sched_hash;
+#endif
 
 /*
  * During exit, a task may schedule after losing its PIDs. When disabling the
@@ -33,37 +51,39 @@ static DEFINE_MUTEX(scx_enable_mutex);
 DEFINE_STATIC_KEY_FALSE(__scx_enabled);
 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
-static int scx_bypass_depth;
+static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
 static cpumask_var_t scx_bypass_lb_donee_cpumask;
 static cpumask_var_t scx_bypass_lb_resched_cpumask;
-static bool scx_aborting;
 static bool scx_init_task_enabled;
 static bool scx_switching_all;
 DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
 
-/*
- * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass
- * depth on enable failure. Will be removed when bypass depth is moved into the
- * sched instance.
- */
-static bool scx_bypassed_for_enable;
-
 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
 
+#ifdef CONFIG_EXT_SUB_SCHED
+/*
+ * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
+ * tasks for the sub-sched being enabled. Use a global variable instead of a
+ * per-task field as all enables are serialized.
+ */
+static struct scx_sched *scx_enabling_sub_sched;
+#else
+#define scx_enabling_sub_sched	(struct scx_sched *)NULL
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
 /*
- * A monotically increasing sequence number that is incremented every time a
- * scheduler is enabled. This can be used by to check if any custom sched_ext
+ * A monotonically increasing sequence number that is incremented every time a
+ * scheduler is enabled. This can be used to check if any custom sched_ext
  * scheduler has ever been used in the system.
  */
 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0);
 
 /*
- * The maximum amount of time in jiffies that a task may be runnable without
- * being scheduled on a CPU. If this timeout is exceeded, it will trigger
- * scx_error().
+ * Watchdog interval. All scx_sched's share a single watchdog timer and the
+ * interval is half of the shortest sch->watchdog_timeout.
  */
-static unsigned long scx_watchdog_timeout;
+static unsigned long scx_watchdog_interval;
 
 /*
  * The last time the delayed work was run. This delayed work relies on
@@ -106,25 +126,6 @@ static const struct rhashtable_params dsq_hash_params = {
 
 static LLIST_HEAD(dsqs_to_free);
 
-/* dispatch buf */
-struct scx_dsp_buf_ent {
-	struct task_struct	*task;
-	unsigned long		qseq;
-	u64			dsq_id;
-	u64			enq_flags;
-};
-
-static u32 scx_dsp_max_batch;
-
-struct scx_dsp_ctx {
-	struct rq		*rq;
-	u32			cursor;
-	u32			nr_tasks;
-	struct scx_dsp_buf_ent	buf[];
-};
-
-static struct scx_dsp_ctx __percpu *scx_dsp_ctx;
-
 /* string formatting from BPF */
 struct scx_bstr_buf {
 	u64			data[MAX_BPRINTF_VARARGS];
@@ -135,6 +136,8 @@ static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
 static struct scx_bstr_buf scx_exit_bstr_buf;
 
 /* ops debug dump */
+static DEFINE_RAW_SPINLOCK(scx_dump_lock);
+
 struct scx_dump_data {
 	s32			cpu;
 	bool			first;
@@ -156,7 +159,6 @@ static struct kset *scx_kset;
  * There usually is no reason to modify these as normal scheduler operation
  * shouldn't be affected by them. The knobs are primarily for debugging.
  */
-static u64 scx_slice_dfl = SCX_SLICE_DFL;
 static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;
 static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US;
 
@@ -193,10 +195,10 @@ MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microsecond
 #define CREATE_TRACE_POINTS
 #include <trace/events/sched_ext.h>
 
-static void process_ddsp_deferred_locals(struct rq *rq);
+static void run_deferred(struct rq *rq);
 static bool task_dead_and_done(struct task_struct *p);
-static u32 reenq_local(struct rq *rq);
 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
+static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
 static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
 		      s64 exit_code, const char *fmt, va_list args);
 
@@ -245,10 +247,104 @@ static bool u32_before(u32 a, u32 b)
 	return (s32)(a - b) < 0;
 }
 
-static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch,
-					      struct task_struct *p)
+#ifdef CONFIG_EXT_SUB_SCHED
+/**
+ * scx_parent - Find the parent sched
+ * @sch: sched to find the parent of
+ *
+ * Returns the parent scheduler or %NULL if @sch is root.
+ */
+static struct scx_sched *scx_parent(struct scx_sched *sch)
+{
+	if (sch->level)
+		return sch->ancestors[sch->level - 1];
+	else
+		return NULL;
+}
+
+/**
+ * scx_next_descendant_pre - find the next descendant for pre-order walk
+ * @pos: the current position (%NULL to initiate traversal)
+ * @root: sched whose descendants to walk
+ *
+ * To be used by scx_for_each_descendant_pre(). Find the next descendant to
+ * visit for pre-order traversal of @root's descendants. @root is included in
+ * the iteration and the first node to be visited.
+ */
+static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos,
+						 struct scx_sched *root)
+{
+	struct scx_sched *next;
+
+	lockdep_assert(lockdep_is_held(&scx_enable_mutex) ||
+		       lockdep_is_held(&scx_sched_lock));
+
+	/* if first iteration, visit @root */
+	if (!pos)
+		return root;
+
+	/* visit the first child if exists */
+	next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling);
+	if (next)
+		return next;
+
+	/* no child, visit my or the closest ancestor's next sibling */
+	while (pos != root) {
+		if (!list_is_last(&pos->sibling, &scx_parent(pos)->children))
+			return list_next_entry(pos, sibling);
+		pos = scx_parent(pos);
+	}
+
+	return NULL;
+}
+
+static struct scx_sched *scx_find_sub_sched(u64 cgroup_id)
+{
+	return rhashtable_lookup(&scx_sched_hash, &cgroup_id,
+				 scx_sched_hash_params);
+}
+
+static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)
+{
+	rcu_assign_pointer(p->scx.sched, sch);
+}
+#else	/* CONFIG_EXT_SUB_SCHED */
+static struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
+static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
+static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return NULL; }
+static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+/**
+ * scx_is_descendant - Test whether sched is a descendant
+ * @sch: sched to test
+ * @ancestor: ancestor sched to test against
+ *
+ * Test whether @sch is a descendant of @ancestor.
+ */
+static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor)
+{
+	if (sch->level < ancestor->level)
+		return false;
+	return sch->ancestors[ancestor->level] == ancestor;
+}
+
+/**
+ * scx_for_each_descendant_pre - pre-order walk of a sched's descendants
+ * @pos: iteration cursor
+ * @root: sched to walk the descendants of
+ *
+ * Walk @root's descendants. @root is included in the iteration and the first
+ * node to be visited. Must be called with either scx_enable_mutex or
+ * scx_sched_lock held.
+ */
+#define scx_for_each_descendant_pre(pos, root)					\
+	for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos);		\
+	     (pos) = scx_next_descendant_pre((pos), (root)))
+
+static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu)
 {
-	return sch->global_dsqs[cpu_to_node(task_cpu(p))];
+	return &sch->pnode[cpu_to_node(cpu)]->global_dsq;
 }
 
 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)
@@ -264,6 +360,108 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p)
 	return __setscheduler_class(p->policy, p->prio);
 }
 
+static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu)
+{
+	return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;
+}
+
+static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu)
+{
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * If @sch is a sub-sched which is bypassing, its tasks should go into
+	 * the bypass DSQs of the nearest ancestor which is not bypassing. The
+	 * not-bypassing ancestor is responsible for scheduling all tasks from
+	 * bypassing sub-trees. If all ancestors including root are bypassing,
+	 * all tasks should go to the root's bypass DSQs.
+	 *
+	 * Whenever a sched starts bypassing, all runnable tasks in its subtree
+	 * are re-enqueued after scx_bypassing() is turned on, guaranteeing that
+	 * all tasks are transferred to the right DSQs.
+	 */
+	while (scx_parent(sch) && scx_bypassing(sch, cpu))
+		sch = scx_parent(sch);
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+	return bypass_dsq(sch, cpu);
+}
+
+/**
+ * bypass_dsp_enabled - Check if bypass dispatch path is enabled
+ * @sch: scheduler to check
+ *
+ * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled
+ * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors
+ * are bypassing. In the former case, the ancestor is not itself bypassing but
+ * its bypass DSQs will be populated with bypassed tasks from descendants. Thus,
+ * the ancestor's bypass dispatch path must be active even though its own
+ * bypass_depth remains zero.
+ *
+ * This function checks bypass_dsp_enable_depth which is managed separately from
+ * bypass_depth to enable this decoupling. See enable_bypass_dsp() and
+ * disable_bypass_dsp().
+ */
+static bool bypass_dsp_enabled(struct scx_sched *sch)
+{
+	return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));
+}
+
+/**
+ * rq_is_open - Is the rq available for immediate execution of an SCX task?
+ * @rq: rq to test
+ * @enq_flags: optional %SCX_ENQ_* of the task being enqueued
+ *
+ * Returns %true if @rq is currently open for executing an SCX task. After a
+ * %false return, @rq is guaranteed to invoke SCX dispatch path at least once
+ * before going to idle and not inserting a task into @rq's local DSQ after a
+ * %false return doesn't cause @rq to stall.
+ */
+static bool rq_is_open(struct rq *rq, u64 enq_flags)
+{
+	lockdep_assert_rq_held(rq);
+
+	/*
+	 * A higher-priority class task is either running or in the process of
+	 * waking up on @rq.
+	 */
+	if (sched_class_above(rq->next_class, &ext_sched_class))
+		return false;
+
+	/*
+	 * @rq is either in transition to or in idle and there is no
+	 * higher-priority class task waking up on it.
+	 */
+	if (sched_class_above(&ext_sched_class, rq->next_class))
+		return true;
+
+	/*
+	 * @rq is either picking, in transition to, or running an SCX task.
+	 */
+
+	/*
+	 * If we're in the dispatch path holding rq lock, $curr may or may not
+	 * be ready depending on whether the on-going dispatch decides to extend
+	 * $curr's slice. We say yes here and resolve it at the end of dispatch.
+	 * See balance_one().
+	 */
+	if (rq->scx.flags & SCX_RQ_IN_BALANCE)
+		return true;
+
+	/*
+	 * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch,
+	 * so allow it to avoid spuriously triggering reenq on a combined
+	 * PREEMPT|IMMED insertion.
+	 */
+	if (enq_flags & SCX_ENQ_PREEMPT)
+		return true;
+
+	/*
+	 * @rq is either in transition to or running an SCX task and can't go
+	 * idle without another SCX dispatch cycle.
+	 */
+	return false;
+}
+
 /*
  * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX
  * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate
@@ -430,9 +628,22 @@ static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch,
 	return true;
 }
 
+enum scx_dsq_iter_flags {
+	/* iterate in the reverse dispatch order */
+	SCX_DSQ_ITER_REV		= 1U << 16,
+
+	__SCX_DSQ_ITER_HAS_SLICE	= 1U << 30,
+	__SCX_DSQ_ITER_HAS_VTIME	= 1U << 31,
+
+	__SCX_DSQ_ITER_USER_FLAGS	= SCX_DSQ_ITER_REV,
+	__SCX_DSQ_ITER_ALL_FLAGS	= __SCX_DSQ_ITER_USER_FLAGS |
+					  __SCX_DSQ_ITER_HAS_SLICE |
+					  __SCX_DSQ_ITER_HAS_VTIME,
+};
+
 /**
  * nldsq_next_task - Iterate to the next task in a non-local DSQ
- * @dsq: user dsq being iterated
+ * @dsq: non-local dsq being iterated
  * @cur: current position, %NULL to start iteration
  * @rev: walk backwards
  *
@@ -472,6 +683,85 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
 	for ((p) = nldsq_next_task((dsq), NULL, false); (p);			\
 	     (p) = nldsq_next_task((dsq), (p), false))
 
+/**
+ * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ
+ * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
+ * @dsq: non-local dsq being iterated
+ *
+ * Find the next task in a cursor based iteration. The caller must have
+ * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock
+ * between the iteration steps.
+ *
+ * Only tasks which were queued before @cursor was initialized are visible. This
+ * bounds the iteration and guarantees that vtime never jumps in the other
+ * direction while iterating.
+ */
+static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor,
+						  struct scx_dispatch_q *dsq)
+{
+	bool rev = cursor->flags & SCX_DSQ_ITER_REV;
+	struct task_struct *p;
+
+	lockdep_assert_held(&dsq->lock);
+	BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR));
+
+	if (list_empty(&cursor->node))
+		p = NULL;
+	else
+		p = container_of(cursor, struct task_struct, scx.dsq_list);
+
+	/* skip cursors and tasks that were queued after @cursor init */
+	do {
+		p = nldsq_next_task(dsq, p, rev);
+	} while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq)));
+
+	if (p) {
+		if (rev)
+			list_move_tail(&cursor->node, &p->scx.dsq_list.node);
+		else
+			list_move(&cursor->node, &p->scx.dsq_list.node);
+	} else {
+		list_del_init(&cursor->node);
+	}
+
+	return p;
+}
+
+/**
+ * nldsq_cursor_lost_task - Test whether someone else took the task since iteration
+ * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
+ * @rq: rq @p was on
+ * @dsq: dsq @p was on
+ * @p: target task
+ *
+ * @p is a task returned by nldsq_cursor_next_task(). The locks may have been
+ * dropped and re-acquired inbetween. Verify that no one else took or is in the
+ * process of taking @p from @dsq.
+ *
+ * On %false return, the caller can assume full ownership of @p.
+ */
+static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor,
+				   struct rq *rq, struct scx_dispatch_q *dsq,
+				   struct task_struct *p)
+{
+	lockdep_assert_rq_held(rq);
+	lockdep_assert_held(&dsq->lock);
+
+	/*
+	 * @p could have already left $src_dsq, got re-enqueud, or be in the
+	 * process of being consumed by someone else.
+	 */
+	if (unlikely(p->scx.dsq != dsq ||
+		     u32_before(cursor->priv, p->scx.dsq_seq) ||
+		     p->scx.holding_cpu >= 0))
+		return true;
+
+	/* if @p has stayed on @dsq, its rq couldn't have changed */
+	if (WARN_ON_ONCE(rq != task_rq(p)))
+		return true;
+
+	return false;
+}
 
 /*
  * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse]
@@ -479,19 +769,6 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
  * changes without breaking backward compatibility. Can be used with
  * bpf_for_each(). See bpf_iter_scx_dsq_*().
  */
-enum scx_dsq_iter_flags {
-	/* iterate in the reverse dispatch order */
-	SCX_DSQ_ITER_REV		= 1U << 16,
-
-	__SCX_DSQ_ITER_HAS_SLICE	= 1U << 30,
-	__SCX_DSQ_ITER_HAS_VTIME	= 1U << 31,
-
-	__SCX_DSQ_ITER_USER_FLAGS	= SCX_DSQ_ITER_REV,
-	__SCX_DSQ_ITER_ALL_FLAGS	= __SCX_DSQ_ITER_USER_FLAGS |
-					  __SCX_DSQ_ITER_HAS_SLICE |
-					  __SCX_DSQ_ITER_HAS_VTIME,
-};
-
 struct bpf_iter_scx_dsq_kern {
 	struct scx_dsq_list_node	cursor;
 	struct scx_dispatch_q		*dsq;
@@ -514,14 +791,31 @@ struct scx_task_iter {
 	struct rq_flags			rf;
 	u32				cnt;
 	bool				list_locked;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct cgroup			*cgrp;
+	struct cgroup_subsys_state	*css_pos;
+	struct css_task_iter		css_iter;
+#endif
 };
 
 /**
  * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration
  * @iter: iterator to init
+ * @cgrp: Optional root of cgroup subhierarchy to iterate
  *
- * Initialize @iter and return with scx_tasks_lock held. Once initialized, @iter
- * must eventually be stopped with scx_task_iter_stop().
+ * Initialize @iter. Once initialized, @iter must eventually be stopped with
+ * scx_task_iter_stop().
+ *
+ * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns
+ * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks.
+ *
+ * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using
+ * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup
+ * task migrations.
+ *
+ * The two modes of iterations are largely independent and it's likely that
+ * scx_tasks can be removed in favor of always using cgroup iteration if
+ * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS.
  *
  * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock()
  * between this and the first next() call or between any two next() calls. If
@@ -532,10 +826,19 @@ struct scx_task_iter {
  * All tasks which existed when the iteration started are guaranteed to be
  * visited as long as they are not dead.
  */
-static void scx_task_iter_start(struct scx_task_iter *iter)
+static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp)
 {
 	memset(iter, 0, sizeof(*iter));
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (cgrp) {
+		lockdep_assert_held(&cgroup_mutex);
+		iter->cgrp = cgrp;
+		iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self);
+		css_task_iter_start(iter->css_pos, 0, &iter->css_iter);
+		return;
+	}
+#endif
 	raw_spin_lock_irq(&scx_tasks_lock);
 
 	iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
@@ -588,6 +891,14 @@ static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter)
  */
 static void scx_task_iter_stop(struct scx_task_iter *iter)
 {
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (iter->cgrp) {
+		if (iter->css_pos)
+			css_task_iter_end(&iter->css_iter);
+		__scx_task_iter_rq_unlock(iter);
+		return;
+	}
+#endif
 	__scx_task_iter_maybe_relock(iter);
 	list_del_init(&iter->cursor.tasks_node);
 	scx_task_iter_unlock(iter);
@@ -611,6 +922,24 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
 		cond_resched();
 	}
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (iter->cgrp) {
+		while (iter->css_pos) {
+			struct task_struct *p;
+
+			p = css_task_iter_next(&iter->css_iter);
+			if (p)
+				return p;
+
+			css_task_iter_end(&iter->css_iter);
+			iter->css_pos = css_next_descendant_pre(iter->css_pos,
+								&iter->cgrp->self);
+			if (iter->css_pos)
+				css_task_iter_start(iter->css_pos, 0, &iter->css_iter);
+		}
+		return NULL;
+	}
+#endif
 	__scx_task_iter_maybe_relock(iter);
 
 	list_for_each_entry(pos, cursor, tasks_node) {
@@ -810,16 +1139,6 @@ static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err
 	return -EPROTO;
 }
 
-static void run_deferred(struct rq *rq)
-{
-	process_ddsp_deferred_locals(rq);
-
-	if (local_read(&rq->scx.reenq_local_deferred)) {
-		local_set(&rq->scx.reenq_local_deferred, 0);
-		reenq_local(rq);
-	}
-}
-
 static void deferred_bal_cb_workfn(struct rq *rq)
 {
 	run_deferred(rq);
@@ -898,6 +1217,81 @@ static void schedule_deferred_locked(struct rq *rq)
 	schedule_deferred(rq);
 }
 
+static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
+			       u64 reenq_flags, struct rq *locked_rq)
+{
+	struct rq *rq;
+
+	/*
+	 * Allowing reenqueues doesn't make sense while bypassing. This also
+	 * blocks from new reenqueues to be scheduled on dead scheds.
+	 */
+	if (unlikely(READ_ONCE(sch->bypass_depth)))
+		return;
+
+	if (dsq->id == SCX_DSQ_LOCAL) {
+		rq = container_of(dsq, struct rq, scx.local_dsq);
+
+		struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq));
+		struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local;
+
+		/*
+		 * Pairs with smp_mb() in process_deferred_reenq_locals() and
+		 * guarantees that there is a reenq_local() afterwards.
+		 */
+		smp_mb();
+
+		if (list_empty(&drl->node) ||
+		    (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) {
+
+			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
+
+			if (list_empty(&drl->node))
+				list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals);
+			WRITE_ONCE(drl->flags, drl->flags | reenq_flags);
+		}
+	} else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) {
+		rq = this_rq();
+
+		struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
+		struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
+
+		/*
+		 * Pairs with smp_mb() in process_deferred_reenq_users() and
+		 * guarantees that there is a reenq_user() afterwards.
+		 */
+		smp_mb();
+
+		if (list_empty(&dru->node) ||
+		    (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) {
+
+			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
+
+			if (list_empty(&dru->node))
+				list_move_tail(&dru->node, &rq->scx.deferred_reenq_users);
+			WRITE_ONCE(dru->flags, dru->flags | reenq_flags);
+		}
+	} else {
+		scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id);
+		return;
+	}
+
+	if (rq == locked_rq)
+		schedule_deferred_locked(rq);
+	else
+		schedule_deferred(rq);
+}
+
+static void schedule_reenq_local(struct rq *rq, u64 reenq_flags)
+{
+	struct scx_sched *root = rcu_dereference_sched(scx_root);
+
+	if (WARN_ON_ONCE(!root))
+		return;
+
+	schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);
+}
+
 /**
  * touch_core_sched - Update timestamp used for core-sched task ordering
  * @rq: rq to read clock from, must be locked
@@ -974,28 +1368,105 @@ static bool scx_dsq_priq_less(struct rb_node *node_a,
 	return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);
 }
 
-static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta)
+static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags)
 {
+	/* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */
+	WRITE_ONCE(dsq->nr, dsq->nr + 1);
+
 	/*
-	 * scx_bpf_dsq_nr_queued() reads ->nr without locking. Use READ_ONCE()
-	 * on the read side and WRITE_ONCE() on the write side to properly
-	 * annotate the concurrent lockless access and avoid KCSAN warnings.
+	 * Once @p reaches a local DSQ, it can only leave it by being dispatched
+	 * to the CPU or dequeued. In both cases, the only way @p can go back to
+	 * the BPF sched is through enqueueing. If being inserted into a local
+	 * DSQ with IMMED, persist the state until the next enqueueing event in
+	 * do_enqueue_task() so that we can maintain IMMED protection through
+	 * e.g. SAVE/RESTORE cycles and slice extensions.
 	 */
-	WRITE_ONCE(dsq->nr, READ_ONCE(dsq->nr) + delta);
+	if (enq_flags & SCX_ENQ_IMMED) {
+		if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
+			WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
+			return;
+		}
+		p->scx.flags |= SCX_TASK_IMMED;
+	}
+
+	if (p->scx.flags & SCX_TASK_IMMED) {
+		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
+
+		if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
+			return;
+
+		rq->scx.nr_immed++;
+
+		/*
+		 * If @rq already had other tasks or the current task is not
+		 * done yet, @p can't go on the CPU immediately. Re-enqueue.
+		 */
+		if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags)))
+			schedule_reenq_local(rq, 0);
+	}
+}
+
+static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p)
+{
+	/* see dsq_inc_nr() */
+	WRITE_ONCE(dsq->nr, dsq->nr - 1);
+
+	if (p->scx.flags & SCX_TASK_IMMED) {
+		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
+
+		if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) ||
+		    WARN_ON_ONCE(rq->scx.nr_immed <= 0))
+			return;
+
+		rq->scx.nr_immed--;
+	}
 }
 
 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
 {
-	p->scx.slice = READ_ONCE(scx_slice_dfl);
+	p->scx.slice = READ_ONCE(sch->slice_dfl);
 	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
 }
 
-static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p,
+/*
+ * Return true if @p is moving due to an internal SCX migration, false
+ * otherwise.
+ */
+static inline bool task_scx_migrating(struct task_struct *p)
+{
+	/*
+	 * We only need to check sticky_cpu: it is set to the destination
+	 * CPU in move_remote_task_to_local_dsq() before deactivate_task()
+	 * and cleared when the task is enqueued on the destination, so it
+	 * is only non-negative during an internal SCX migration.
+	 */
+	return p->scx.sticky_cpu >= 0;
+}
+
+/*
+ * Call ops.dequeue() if the task is in BPF custody and not migrating.
+ * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked.
+ */
+static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
+			      struct task_struct *p, u64 deq_flags)
+{
+	if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p))
+		return;
+
+	if (SCX_HAS_OP(sch, dequeue))
+		SCX_CALL_OP_TASK(sch, SCX_KF_REST, dequeue, rq, p, deq_flags);
+
+	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
+}
+
+static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p,
 			       u64 enq_flags)
 {
 	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
 	bool preempt = false;
 
+	call_task_dequeue(scx_root, rq, p, 0);
+
 	/*
 	 * If @rq is in balance, the CPU is already vacant and looking for the
 	 * next task to run. No need to preempt or trigger resched after moving
@@ -1014,8 +1485,9 @@ static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p
 		resched_curr(rq);
 }
 
-static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
-			     struct task_struct *p, u64 enq_flags)
+static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
+			     struct scx_dispatch_q *dsq, struct task_struct *p,
+			     u64 enq_flags)
 {
 	bool is_local = dsq->id == SCX_DSQ_LOCAL;
 
@@ -1031,7 +1503,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 			scx_error(sch, "attempting to dispatch to a destroyed dsq");
 			/* fall back to the global dsq */
 			raw_spin_unlock(&dsq->lock);
-			dsq = find_global_dsq(sch, p);
+			dsq = find_global_dsq(sch, task_cpu(p));
 			raw_spin_lock(&dsq->lock);
 		}
 	}
@@ -1106,7 +1578,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 	WRITE_ONCE(dsq->seq, dsq->seq + 1);
 	p->scx.dsq_seq = dsq->seq;
 
-	dsq_mod_nr(dsq, 1);
+	dsq_inc_nr(dsq, p, enq_flags);
 	p->scx.dsq = dsq;
 
 	/*
@@ -1118,17 +1590,34 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 	p->scx.ddsp_dsq_id = SCX_DSQ_INVALID;
 	p->scx.ddsp_enq_flags = 0;
 
+	/*
+	 * Update custody and call ops.dequeue() before clearing ops_state:
+	 * once ops_state is cleared, waiters in ops_dequeue() can proceed
+	 * and dequeue_task_scx() will RMW p->scx.flags. If we clear
+	 * ops_state first, both sides would modify p->scx.flags
+	 * concurrently in a non-atomic way.
+	 */
+	if (is_local) {
+		local_dsq_post_enq(dsq, p, enq_flags);
+	} else {
+		/*
+		 * Task on global/bypass DSQ: leave custody, task on
+		 * non-terminal DSQ: enter custody.
+		 */
+		if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS)
+			call_task_dequeue(sch, rq, p, 0);
+		else
+			p->scx.flags |= SCX_TASK_IN_CUSTODY;
+
+		raw_spin_unlock(&dsq->lock);
+	}
+
 	/*
 	 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
 	 * match waiters' load_acquire.
 	 */
 	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
 		atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
-
-	if (is_local)
-		local_dsq_post_enq(dsq, p, enq_flags);
-	else
-		raw_spin_unlock(&dsq->lock);
 }
 
 static void task_unlink_from_dsq(struct task_struct *p,
@@ -1143,7 +1632,7 @@ static void task_unlink_from_dsq(struct task_struct *p,
 	}
 
 	list_del_init(&p->scx.dsq_list.node);
-	dsq_mod_nr(dsq, -1);
+	dsq_dec_nr(dsq, p);
 
 	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) {
 		struct task_struct *first_task;
@@ -1222,7 +1711,7 @@ static void dispatch_dequeue_locked(struct task_struct *p,
 
 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
 						    struct rq *rq, u64 dsq_id,
-						    struct task_struct *p)
+						    s32 tcpu)
 {
 	struct scx_dispatch_q *dsq;
 
@@ -1233,20 +1722,19 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
 		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
 
 		if (!ops_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
-			return find_global_dsq(sch, p);
+			return find_global_dsq(sch, tcpu);
 
 		return &cpu_rq(cpu)->scx.local_dsq;
 	}
 
 	if (dsq_id == SCX_DSQ_GLOBAL)
-		dsq = find_global_dsq(sch, p);
+		dsq = find_global_dsq(sch, tcpu);
 	else
 		dsq = find_user_dsq(sch, dsq_id);
 
 	if (unlikely(!dsq)) {
-		scx_error(sch, "non-existent DSQ 0x%llx for %s[%d]",
-			  dsq_id, p->comm, p->pid);
-		return find_global_dsq(sch, p);
+		scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
+		return find_global_dsq(sch, tcpu);
 	}
 
 	return dsq;
@@ -1288,7 +1776,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
 {
 	struct rq *rq = task_rq(p);
 	struct scx_dispatch_q *dsq =
-		find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p);
+		find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
 
 	touch_core_sched_dispatch(rq, p);
 
@@ -1329,7 +1817,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
 		return;
 	}
 
-	dispatch_enqueue(sch, dsq, p,
+	dispatch_enqueue(sch, rq, dsq, p,
 			 p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
 }
 
@@ -1348,17 +1836,25 @@ static bool scx_rq_online(struct rq *rq)
 static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 			    int sticky_cpu)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 	struct task_struct **ddsp_taskp;
 	struct scx_dispatch_q *dsq;
 	unsigned long qseq;
 
 	WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
 
-	/* rq migration */
+	/* internal movements - rq migration / RESTORE */
 	if (sticky_cpu == cpu_of(rq))
 		goto local_norefill;
 
+	/*
+	 * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr().
+	 * Note that exiting and migration-disabled tasks that skip
+	 * ops.enqueue() below will lose IMMED protection unless
+	 * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set.
+	 */
+	p->scx.flags &= ~SCX_TASK_IMMED;
+
 	/*
 	 * If !scx_rq_online(), we already told the BPF scheduler that the CPU
 	 * is offline and are just running the hotplug path. Don't bother the
@@ -1367,7 +1863,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	if (!scx_rq_online(rq))
 		goto local;
 
-	if (scx_rq_bypassing(rq)) {
+	if (scx_bypassing(sch, cpu_of(rq))) {
 		__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
 		goto bypass;
 	}
@@ -1408,6 +1904,12 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
 		goto direct;
 
+	/*
+	 * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY
+	 * so ops.dequeue() is called when it leaves custody.
+	 */
+	p->scx.flags |= SCX_TASK_IN_CUSTODY;
+
 	/*
 	 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or
 	 * dequeue may be waiting. The store_release matches their load_acquire.
@@ -1419,16 +1921,16 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	direct_dispatch(sch, p, enq_flags);
 	return;
 local_norefill:
-	dispatch_enqueue(sch, &rq->scx.local_dsq, p, enq_flags);
+	dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags);
 	return;
 local:
 	dsq = &rq->scx.local_dsq;
 	goto enqueue;
 global:
-	dsq = find_global_dsq(sch, p);
+	dsq = find_global_dsq(sch, task_cpu(p));
 	goto enqueue;
 bypass:
-	dsq = &task_rq(p)->scx.bypass_dsq;
+	dsq = bypass_enq_target_dsq(sch, task_cpu(p));
 	goto enqueue;
 
 enqueue:
@@ -1439,7 +1941,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	 */
 	touch_core_sched(rq, p);
 	refill_task_slice_dfl(sch, p);
-	dispatch_enqueue(sch, dsq, p, enq_flags);
+	dispatch_enqueue(sch, rq, dsq, p, enq_flags);
 }
 
 static bool task_runnable(const struct task_struct *p)
@@ -1472,16 +1974,13 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
 
 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 	int sticky_cpu = p->scx.sticky_cpu;
 	u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags;
 
 	if (enq_flags & ENQUEUE_WAKEUP)
 		rq->scx.flags |= SCX_RQ_IN_WAKEUP;
 
-	if (sticky_cpu >= 0)
-		p->scx.sticky_cpu = -1;
-
 	/*
 	 * Restoring a running task will be immediately followed by
 	 * set_next_task_scx() which expects the task to not be on the BPF
@@ -1512,6 +2011,9 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 		dl_server_start(&rq->ext_server);
 
 	do_enqueue_task(rq, p, enq_flags, sticky_cpu);
+
+	if (sticky_cpu >= 0)
+		p->scx.sticky_cpu = -1;
 out:
 	rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
 
@@ -1522,7 +2024,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 
 static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 	unsigned long opss;
 
 	/* dequeue is always temporary, don't reset runnable_at */
@@ -1541,10 +2043,8 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
 		 */
 		BUG();
 	case SCX_OPSS_QUEUED:
-		if (SCX_HAS_OP(sch, dequeue))
-			SCX_CALL_OP_TASK(sch, SCX_KF_REST, dequeue, rq,
-					 p, deq_flags);
-
+		/* A queued task must always be in BPF scheduler's custody */
+		WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY));
 		if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
 					    SCX_OPSS_NONE))
 			break;
@@ -1567,11 +2067,35 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
 		BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
 		break;
 	}
+
+	/*
+	 * Call ops.dequeue() if the task is still in BPF custody.
+	 *
+	 * The code that clears ops_state to %SCX_OPSS_NONE does not always
+	 * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when
+	 * we're moving a task that was in %SCX_OPSS_DISPATCHING to a
+	 * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE
+	 * so that a concurrent dequeue can proceed, but we clear
+	 * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the
+	 * task. So we can see NONE + IN_CUSTODY here and we must handle
+	 * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see
+	 * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until
+	 * it is enqueued on the destination.
+	 */
+	call_task_dequeue(sch, rq, p, deq_flags);
 }
 
-static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags)
+static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
+	u64 deq_flags = core_deq_flags;
+
+	/*
+	 * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property
+	 * change (not sleep or core-sched pick).
+	 */
+	if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC)))
+		deq_flags |= SCX_DEQ_SCHED_CHANGE;
 
 	if (!(p->scx.flags & SCX_TASK_QUEUED)) {
 		WARN_ON_ONCE(task_runnable(p));
@@ -1615,8 +2139,8 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags
 
 static void yield_task_scx(struct rq *rq)
 {
-	struct scx_sched *sch = scx_root;
 	struct task_struct *p = rq->donor;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	if (SCX_HAS_OP(sch, yield))
 		SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq, p, NULL);
@@ -1626,16 +2150,40 @@ static void yield_task_scx(struct rq *rq)
 
 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
 {
-	struct scx_sched *sch = scx_root;
 	struct task_struct *from = rq->donor;
+	struct scx_sched *sch = scx_task_sched(from);
 
-	if (SCX_HAS_OP(sch, yield))
+	if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to))
 		return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, yield, rq,
 					      from, to);
 	else
 		return false;
 }
 
+static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags)
+{
+	/*
+	 * Preemption between SCX tasks is implemented by resetting the victim
+	 * task's slice to 0 and triggering reschedule on the target CPU.
+	 * Nothing to do.
+	 */
+	if (p->sched_class == &ext_sched_class)
+		return;
+
+	/*
+	 * Getting preempted by a higher-priority class. Reenqueue IMMED tasks.
+	 * This captures all preemption cases including:
+	 *
+	 * - A SCX task is currently running.
+	 *
+	 * - @rq is waking from idle due to a SCX task waking to it.
+	 *
+	 * - A higher-priority wakes up while SCX dispatch is in progress.
+	 */
+	if (rq->scx.nr_immed)
+		schedule_reenq_local(rq, 0);
+}
+
 static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
 					 struct scx_dispatch_q *src_dsq,
 					 struct rq *dst_rq)
@@ -1653,7 +2201,7 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
 	else
 		list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
 
-	dsq_mod_nr(dst_dsq, 1);
+	dsq_inc_nr(dst_dsq, p, enq_flags);
 	p->scx.dsq = dst_dsq;
 
 	local_dsq_post_enq(dst_dsq, p, enq_flags);
@@ -1673,10 +2221,13 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
 {
 	lockdep_assert_rq_held(src_rq);
 
-	/* the following marks @p MIGRATING which excludes dequeue */
+	/*
+	 * Set sticky_cpu before deactivate_task() to properly mark the
+	 * beginning of an SCX-internal migration.
+	 */
+	p->scx.sticky_cpu = cpu_of(dst_rq);
 	deactivate_task(src_rq, p, 0);
 	set_task_cpu(p, cpu_of(dst_rq));
-	p->scx.sticky_cpu = cpu_of(dst_rq);
 
 	raw_spin_rq_unlock(src_rq);
 	raw_spin_rq_lock(dst_rq);
@@ -1716,7 +2267,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 				      struct task_struct *p, struct rq *rq,
 				      bool enforce)
 {
-	int cpu = cpu_of(rq);
+	s32 cpu = cpu_of(rq);
 
 	WARN_ON_ONCE(task_cpu(p) == cpu);
 
@@ -1810,13 +2361,14 @@ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
 		!WARN_ON_ONCE(src_rq != task_rq(p));
 }
 
-static bool consume_remote_task(struct rq *this_rq, struct task_struct *p,
+static bool consume_remote_task(struct rq *this_rq,
+				struct task_struct *p, u64 enq_flags,
 				struct scx_dispatch_q *dsq, struct rq *src_rq)
 {
 	raw_spin_rq_unlock(this_rq);
 
 	if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) {
-		move_remote_task_to_local_dsq(p, 0, src_rq, this_rq);
+		move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq);
 		return true;
 	} else {
 		raw_spin_rq_unlock(src_rq);
@@ -1856,8 +2408,9 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 		dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
 		if (src_rq != dst_rq &&
 		    unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
-			dst_dsq = find_global_dsq(sch, p);
+			dst_dsq = find_global_dsq(sch, task_cpu(p));
 			dst_rq = src_rq;
+			enq_flags |= SCX_ENQ_GDSQ_FALLBACK;
 		}
 	} else {
 		/* no need to migrate if destination is a non-local DSQ */
@@ -1888,14 +2441,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 		dispatch_dequeue_locked(p, src_dsq);
 		raw_spin_unlock(&src_dsq->lock);
 
-		dispatch_enqueue(sch, dst_dsq, p, enq_flags);
+		dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags);
 	}
 
 	return dst_rq;
 }
 
 static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
-			       struct scx_dispatch_q *dsq)
+			       struct scx_dispatch_q *dsq, u64 enq_flags)
 {
 	struct task_struct *p;
 retry:
@@ -1920,18 +2473,18 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
 		 * the system into the bypass mode. This can easily live-lock the
 		 * machine. If aborting, exit from all non-bypass DSQs.
 		 */
-		if (unlikely(READ_ONCE(scx_aborting)) && dsq->id != SCX_DSQ_BYPASS)
+		if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS)
 			break;
 
 		if (rq == task_rq) {
 			task_unlink_from_dsq(p, dsq);
-			move_local_task_to_local_dsq(p, 0, dsq, rq);
+			move_local_task_to_local_dsq(p, enq_flags, dsq, rq);
 			raw_spin_unlock(&dsq->lock);
 			return true;
 		}
 
 		if (task_can_run_on_remote_rq(sch, p, rq, false)) {
-			if (likely(consume_remote_task(rq, p, dsq, task_rq)))
+			if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq)))
 				return true;
 			goto retry;
 		}
@@ -1945,7 +2498,7 @@ static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq)
 {
 	int node = cpu_to_node(cpu_of(rq));
 
-	return consume_dispatch_q(sch, rq, sch->global_dsqs[node]);
+	return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
 }
 
 /**
@@ -1978,15 +2531,15 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 	 * If dispatching to @rq that @p is already on, no lock dancing needed.
 	 */
 	if (rq == src_rq && rq == dst_rq) {
-		dispatch_enqueue(sch, dst_dsq, p,
+		dispatch_enqueue(sch, rq, dst_dsq, p,
 				 enq_flags | SCX_ENQ_CLEAR_OPSS);
 		return;
 	}
 
 	if (src_rq != dst_rq &&
 	    unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
-		dispatch_enqueue(sch, find_global_dsq(sch, p), p,
-				 enq_flags | SCX_ENQ_CLEAR_OPSS);
+		dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p,
+				 enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK);
 		return;
 	}
 
@@ -2023,7 +2576,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 		 */
 		if (src_rq == dst_rq) {
 			p->scx.holding_cpu = -1;
-			dispatch_enqueue(sch, &dst_rq->scx.local_dsq, p,
+			dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
 					 enq_flags);
 		} else {
 			move_remote_task_to_local_dsq(p, enq_flags,
@@ -2093,6 +2646,12 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
 		if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
 			return;
 
+		/* see SCX_EV_INSERT_NOT_OWNED definition */
+		if (unlikely(!scx_task_on_sched(sch, p))) {
+			__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
+			return;
+		}
+
 		/*
 		 * While we know @p is accessible, we don't yet have a claim on
 		 * it - the BPF scheduler is allowed to dispatch tasks
@@ -2117,17 +2676,17 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
 
 	BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
 
-	dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, p);
+	dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p));
 
 	if (dsq->id == SCX_DSQ_LOCAL)
 		dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags);
 	else
-		dispatch_enqueue(sch, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
+		dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
 }
 
 static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
 {
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
+	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
 	u32 u;
 
 	for (u = 0; u < dspc->cursor; u++) {
@@ -2154,67 +2713,54 @@ static inline void maybe_queue_balance_callback(struct rq *rq)
 	rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
 }
 
-static int balance_one(struct rq *rq, struct task_struct *prev)
+/*
+ * One user of this function is scx_bpf_dispatch() which can be called
+ * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
+ * from the call frame.
+ */
+static __always_inline bool
+scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
+		   struct task_struct *prev, bool nested)
 {
-	struct scx_sched *sch = scx_root;
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
-	bool prev_on_scx = prev->sched_class == &ext_sched_class;
-	bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED;
+	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
 	int nr_loops = SCX_DSP_MAX_LOOPS;
+	s32 cpu = cpu_of(rq);
+	bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
+		scx_task_on_sched(sch, prev);
 
-	lockdep_assert_rq_held(rq);
-	rq->scx.flags |= SCX_RQ_IN_BALANCE;
-	rq->scx.flags &= ~SCX_RQ_BAL_KEEP;
-
-	if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
-	    unlikely(rq->scx.cpu_released)) {
-		/*
-		 * If the previous sched_class for the current CPU was not SCX,
-		 * notify the BPF scheduler that it again has control of the
-		 * core. This callback complements ->cpu_release(), which is
-		 * emitted in switch_class().
-		 */
-		if (SCX_HAS_OP(sch, cpu_acquire))
-			SCX_CALL_OP(sch, SCX_KF_REST, cpu_acquire, rq,
-				    cpu_of(rq), NULL);
-		rq->scx.cpu_released = false;
-	}
+	if (consume_global_dsq(sch, rq))
+		return true;
 
-	if (prev_on_scx) {
-		update_curr_scx(rq);
+	if (bypass_dsp_enabled(sch)) {
+		/* if @sch is bypassing, only the bypass DSQs are active */
+		if (scx_bypassing(sch, cpu))
+			return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
 
+#ifdef CONFIG_EXT_SUB_SCHED
 		/*
-		 * If @prev is runnable & has slice left, it has priority and
-		 * fetching more just increases latency for the fetched tasks.
-		 * Tell pick_task_scx() to keep running @prev. If the BPF
-		 * scheduler wants to handle this explicitly, it should
-		 * implement ->cpu_release().
+		 * If @sch isn't bypassing but its children are, @sch is
+		 * responsible for making forward progress for both its own
+		 * tasks that aren't bypassing and the bypassing descendants'
+		 * tasks. The following implements a simple built-in behavior -
+		 * let each CPU try to run the bypass DSQ every Nth time.
 		 *
-		 * See scx_disable_workfn() for the explanation on the bypassing
-		 * test.
+		 * Later, if necessary, we can add an ops flag to suppress the
+		 * auto-consumption and a kfunc to consume the bypass DSQ and,
+		 * so that the BPF scheduler can fully control scheduling of
+		 * bypassed tasks.
 		 */
-		if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) {
-			rq->scx.flags |= SCX_RQ_BAL_KEEP;
-			goto has_tasks;
-		}
-	}
+		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 
-	/* if there already are tasks to run, nothing to do */
-	if (rq->scx.local_dsq.nr)
-		goto has_tasks;
-
-	if (consume_global_dsq(sch, rq))
-		goto has_tasks;
-
-	if (scx_rq_bypassing(rq)) {
-		if (consume_dispatch_q(sch, rq, &rq->scx.bypass_dsq))
-			goto has_tasks;
-		else
-			goto no_tasks;
+		if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
+		    consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) {
+			__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
+			return true;
+		}
+#endif	/* CONFIG_EXT_SUB_SCHED */
 	}
 
 	if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
-		goto no_tasks;
+		return false;
 
 	dspc->rq = rq;
 
@@ -2228,19 +2774,34 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 	do {
 		dspc->nr_tasks = 0;
 
-		SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq,
-			    cpu_of(rq), prev_on_scx ? prev : NULL);
+		if (nested) {
+			/*
+			 * If nested, don't update kf_mask as the originating
+			 * invocation would already have set it up.
+			 */
+			SCX_CALL_OP(sch, 0, dispatch, rq, cpu,
+				    prev_on_sch ? prev : NULL);
+		} else {
+			/*
+			 * If not nested, stash @prev so that nested invocations
+			 * can access it.
+			 */
+			rq->scx.sub_dispatch_prev = prev;
+			SCX_CALL_OP(sch, SCX_KF_DISPATCH, dispatch, rq, cpu,
+				    prev_on_sch ? prev : NULL);
+			rq->scx.sub_dispatch_prev = NULL;
+		}
 
 		flush_dispatch_buf(sch, rq);
 
-		if (prev_on_rq && prev->scx.slice) {
+		if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
 			rq->scx.flags |= SCX_RQ_BAL_KEEP;
-			goto has_tasks;
+			return true;
 		}
 		if (rq->scx.local_dsq.nr)
-			goto has_tasks;
+			return true;
 		if (consume_global_dsq(sch, rq))
-			goto has_tasks;
+			return true;
 
 		/*
 		 * ops.dispatch() can trap us in this loop by repeatedly
@@ -2249,21 +2810,80 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 		 * balance(), we want to complete this scheduling cycle and then
 		 * start a new one. IOW, we want to call resched_curr() on the
 		 * next, most likely idle, task, not the current one. Use
-		 * scx_kick_cpu() for deferred kicking.
+		 * __scx_bpf_kick_cpu() for deferred kicking.
 		 */
 		if (unlikely(!--nr_loops)) {
-			scx_kick_cpu(sch, cpu_of(rq), 0);
+			scx_kick_cpu(sch, cpu, 0);
 			break;
 		}
 	} while (dspc->nr_tasks);
 
-no_tasks:
+	/*
+	 * Prevent the CPU from going idle while bypassed descendants have tasks
+	 * queued. Without this fallback, bypassed tasks could stall if the host
+	 * scheduler's ops.dispatch() doesn't yield any tasks.
+	 */
+	if (bypass_dsp_enabled(sch))
+		return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
+
+	return false;
+}
+
+static int balance_one(struct rq *rq, struct task_struct *prev)
+{
+	struct scx_sched *sch = scx_root;
+	s32 cpu = cpu_of(rq);
+
+	lockdep_assert_rq_held(rq);
+	rq->scx.flags |= SCX_RQ_IN_BALANCE;
+	rq->scx.flags &= ~SCX_RQ_BAL_KEEP;
+
+	if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
+	    unlikely(rq->scx.cpu_released)) {
+		/*
+		 * If the previous sched_class for the current CPU was not SCX,
+		 * notify the BPF scheduler that it again has control of the
+		 * core. This callback complements ->cpu_release(), which is
+		 * emitted in switch_class().
+		 */
+		if (SCX_HAS_OP(sch, cpu_acquire))
+			SCX_CALL_OP(sch, SCX_KF_REST, cpu_acquire, rq, cpu, NULL);
+		rq->scx.cpu_released = false;
+	}
+
+	if (prev->sched_class == &ext_sched_class) {
+		update_curr_scx(rq);
+
+		/*
+		 * If @prev is runnable & has slice left, it has priority and
+		 * fetching more just increases latency for the fetched tasks.
+		 * Tell pick_task_scx() to keep running @prev. If the BPF
+		 * scheduler wants to handle this explicitly, it should
+		 * implement ->cpu_release().
+		 *
+		 * See scx_disable_workfn() for the explanation on the bypassing
+		 * test.
+		 */
+		if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
+		    !scx_bypassing(sch, cpu)) {
+			rq->scx.flags |= SCX_RQ_BAL_KEEP;
+			goto has_tasks;
+		}
+	}
+
+	/* if there already are tasks to run, nothing to do */
+	if (rq->scx.local_dsq.nr)
+		goto has_tasks;
+
+	if (scx_dispatch_sched(sch, rq, prev, false))
+		goto has_tasks;
+
 	/*
 	 * Didn't find another task to run. Keep running @prev unless
 	 * %SCX_OPS_ENQ_LAST is in effect.
 	 */
-	if (prev_on_rq &&
-	    (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_rq_bypassing(rq))) {
+	if ((prev->scx.flags & SCX_TASK_QUEUED) &&
+	    (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) {
 		rq->scx.flags |= SCX_RQ_BAL_KEEP;
 		__scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
 		goto has_tasks;
@@ -2272,40 +2892,26 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 	return false;
 
 has_tasks:
-	rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
-	return true;
-}
-
-static void process_ddsp_deferred_locals(struct rq *rq)
-{
-	struct task_struct *p;
-
-	lockdep_assert_rq_held(rq);
-
 	/*
-	 * Now that @rq can be unlocked, execute the deferred enqueueing of
-	 * tasks directly dispatched to the local DSQs of other CPUs. See
-	 * direct_dispatch(). Keep popping from the head instead of using
-	 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
-	 * temporarily.
+	 * @rq may have extra IMMED tasks without reenq scheduled:
+	 *
+	 * - rq_is_open() can't reliably tell when and how slice is going to be
+	 *   modified for $curr and allows IMMED tasks to be queued while
+	 *   dispatch is in progress.
+	 *
+	 * - A non-IMMED HEAD task can get queued in front of an IMMED task
+	 *   between the IMMED queueing and the subsequent scheduling event.
 	 */
-	while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
-				struct task_struct, scx.dsq_list.node))) {
-		struct scx_sched *sch = scx_root;
-		struct scx_dispatch_q *dsq;
-
-		list_del_init(&p->scx.dsq_list.node);
+	if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed))
+		schedule_reenq_local(rq, 0);
 
-		dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, p);
-		if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
-			dispatch_to_local_dsq(sch, rq, dsq, p,
-					      p->scx.ddsp_enq_flags);
-	}
+	rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
+	return true;
 }
 
 static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	if (p->scx.flags & SCX_TASK_QUEUED) {
 		/*
@@ -2402,7 +3008,7 @@ static void switch_class(struct rq *rq, struct task_struct *next)
 static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 			      struct task_struct *next)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	/* see kick_cpus_irq_workfn() */
 	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
@@ -2420,11 +3026,17 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 		 * If @p has slice left and is being put, @p is getting
 		 * preempted by a higher priority scheduler class or core-sched
 		 * forcing a different task. Leave it at the head of the local
-		 * DSQ.
+		 * DSQ unless it was an IMMED task. IMMED tasks should not
+		 * linger on a busy CPU, reenqueue them to the BPF scheduler.
 		 */
-		if (p->scx.slice && !scx_rq_bypassing(rq)) {
-			dispatch_enqueue(sch, &rq->scx.local_dsq, p,
-					 SCX_ENQ_HEAD);
+		if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) {
+			if (p->scx.flags & SCX_TASK_IMMED) {
+				p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
+				do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+				p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+			} else {
+				dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD);
+			}
 			goto switch_class;
 		}
 
@@ -2496,16 +3108,17 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
 	if (keep_prev) {
 		p = prev;
 		if (!p->scx.slice)
-			refill_task_slice_dfl(rcu_dereference_sched(scx_root), p);
+			refill_task_slice_dfl(scx_task_sched(p), p);
 	} else {
 		p = first_local_task(rq);
 		if (!p)
 			return NULL;
 
 		if (unlikely(!p->scx.slice)) {
-			struct scx_sched *sch = rcu_dereference_sched(scx_root);
+			struct scx_sched *sch = scx_task_sched(p);
 
-			if (!scx_rq_bypassing(rq) && !sch->warned_zero_slice) {
+			if (!scx_bypassing(sch, cpu_of(rq)) &&
+			    !sch->warned_zero_slice) {
 				printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n",
 						p->comm, p->pid, __func__);
 				sch->warned_zero_slice = true;
@@ -2571,16 +3184,17 @@ void ext_server_init(struct rq *rq)
 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
 		   bool in_fi)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch_a = scx_task_sched(a);
+	struct scx_sched *sch_b = scx_task_sched(b);
 
 	/*
 	 * The const qualifiers are dropped from task_struct pointers when
 	 * calling ops.core_sched_before(). Accesses are controlled by the
 	 * verifier.
 	 */
-	if (SCX_HAS_OP(sch, core_sched_before) &&
-	    !scx_rq_bypassing(task_rq(a)))
-		return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, core_sched_before,
+	if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) &&
+	    !scx_bypassing(sch_a, task_cpu(a)))
+		return SCX_CALL_OP_2TASKS_RET(sch_a, SCX_KF_REST, core_sched_before,
 					      NULL,
 					      (struct task_struct *)a,
 					      (struct task_struct *)b);
@@ -2591,8 +3205,8 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
 
 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
 {
-	struct scx_sched *sch = scx_root;
-	bool rq_bypass;
+	struct scx_sched *sch = scx_task_sched(p);
+	bool bypassing;
 
 	/*
 	 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it
@@ -2607,8 +3221,8 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 	if (unlikely(wake_flags & WF_EXEC))
 		return prev_cpu;
 
-	rq_bypass = scx_rq_bypassing(task_rq(p));
-	if (likely(SCX_HAS_OP(sch, select_cpu)) && !rq_bypass) {
+	bypassing = scx_bypassing(sch, task_cpu(p));
+	if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) {
 		s32 cpu;
 		struct task_struct **ddsp_taskp;
 
@@ -2638,7 +3252,7 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 		}
 		p->scx.selected_cpu = cpu;
 
-		if (rq_bypass)
+		if (bypassing)
 			__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
 		return cpu;
 	}
@@ -2652,7 +3266,7 @@ static void task_woken_scx(struct rq *rq, struct task_struct *p)
 static void set_cpus_allowed_scx(struct task_struct *p,
 				 struct affinity_context *ac)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	set_cpus_allowed_common(p, ac);
 
@@ -2675,7 +3289,7 @@ static void set_cpus_allowed_scx(struct task_struct *p,
 static void handle_hotplug(struct rq *rq, bool online)
 {
 	struct scx_sched *sch = scx_root;
-	int cpu = cpu_of(rq);
+	s32 cpu = cpu_of(rq);
 
 	atomic_long_inc(&scx_hotplug_seq);
 
@@ -2721,7 +3335,6 @@ static void rq_offline_scx(struct rq *rq)
 	rq->scx.flags &= ~SCX_RQ_ONLINE;
 }
 
-
 static bool check_rq_for_timeouts(struct rq *rq)
 {
 	struct scx_sched *sch;
@@ -2735,10 +3348,11 @@ static bool check_rq_for_timeouts(struct rq *rq)
 		goto out_unlock;
 
 	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
+		struct scx_sched *sch = scx_task_sched(p);
 		unsigned long last_runnable = p->scx.runnable_at;
 
 		if (unlikely(time_after(jiffies,
-					last_runnable + READ_ONCE(scx_watchdog_timeout)))) {
+					last_runnable + READ_ONCE(sch->watchdog_timeout)))) {
 			u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
 
 			scx_exit(sch, SCX_EXIT_ERROR_STALL, 0,
@@ -2755,6 +3369,7 @@ static bool check_rq_for_timeouts(struct rq *rq)
 
 static void scx_watchdog_workfn(struct work_struct *work)
 {
+	unsigned long intv;
 	int cpu;
 
 	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
@@ -2765,28 +3380,30 @@ static void scx_watchdog_workfn(struct work_struct *work)
 
 		cond_resched();
 	}
-	queue_delayed_work(system_unbound_wq, to_delayed_work(work),
-			   READ_ONCE(scx_watchdog_timeout) / 2);
+
+	intv = READ_ONCE(scx_watchdog_interval);
+	if (intv < ULONG_MAX)
+		queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv);
 }
 
 void scx_tick(struct rq *rq)
 {
-	struct scx_sched *sch;
+	struct scx_sched *root;
 	unsigned long last_check;
 
 	if (!scx_enabled())
 		return;
 
-	sch = rcu_dereference_bh(scx_root);
-	if (unlikely(!sch))
+	root = rcu_dereference_bh(scx_root);
+	if (unlikely(!root))
 		return;
 
 	last_check = READ_ONCE(scx_watchdog_timestamp);
 	if (unlikely(time_after(jiffies,
-				last_check + READ_ONCE(scx_watchdog_timeout)))) {
+				last_check + READ_ONCE(root->watchdog_timeout)))) {
 		u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
 
-		scx_exit(sch, SCX_EXIT_ERROR_STALL, 0,
+		scx_exit(root, SCX_EXIT_ERROR_STALL, 0,
 			 "watchdog failed to check in for %u.%03us",
 			 dur_ms / 1000, dur_ms % 1000);
 	}
@@ -2796,7 +3413,7 @@ void scx_tick(struct rq *rq)
 
 static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(curr);
 
 	update_curr_scx(rq);
 
@@ -2804,7 +3421,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
 	 * While disabling, always resched and refresh core-sched timestamp as
 	 * we can't trust the slice management or ops.core_sched_before().
 	 */
-	if (scx_rq_bypassing(rq)) {
+	if (scx_bypassing(sch, cpu_of(rq))) {
 		curr->scx.slice = 0;
 		touch_core_sched(rq, curr);
 	} else if (SCX_HAS_OP(sch, tick)) {
@@ -2837,18 +3454,16 @@ static struct cgroup *tg_cgrp(struct task_group *tg)
 
 #endif	/* CONFIG_EXT_GROUP_SCHED */
 
-static enum scx_task_state scx_get_task_state(const struct task_struct *p)
+static u32 scx_get_task_state(const struct task_struct *p)
 {
-	return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT;
+	return p->scx.flags & SCX_TASK_STATE_MASK;
 }
 
-static void scx_set_task_state(struct task_struct *p, enum scx_task_state state)
+static void scx_set_task_state(struct task_struct *p, u32 state)
 {
-	enum scx_task_state prev_state = scx_get_task_state(p);
+	u32 prev_state = scx_get_task_state(p);
 	bool warn = false;
 
-	BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS));
-
 	switch (state) {
 	case SCX_TASK_NONE:
 		break;
@@ -2866,23 +3481,22 @@ static void scx_set_task_state(struct task_struct *p, enum scx_task_state state)
 		return;
 	}
 
-	WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]",
+	WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
 		  prev_state, state, p->comm, p->pid);
 
 	p->scx.flags &= ~SCX_TASK_STATE_MASK;
-	p->scx.flags |= state << SCX_TASK_STATE_SHIFT;
+	p->scx.flags |= state;
 }
 
-static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork)
+static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
 {
-	struct scx_sched *sch = scx_root;
 	int ret;
 
 	p->scx.disallow = false;
 
 	if (SCX_HAS_OP(sch, init_task)) {
 		struct scx_init_task_args args = {
-			SCX_INIT_TASK_ARGS_CGROUP(tg)
+			SCX_INIT_TASK_ARGS_CGROUP(task_group(p))
 			.fork = fork,
 		};
 
@@ -2894,10 +3508,14 @@ static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork
 		}
 	}
 
-	scx_set_task_state(p, SCX_TASK_INIT);
-
 	if (p->scx.disallow) {
-		if (!fork) {
+		if (unlikely(scx_parent(sch))) {
+			scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]",
+				  p->comm, p->pid);
+		} else if (unlikely(fork)) {
+			scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork",
+				  p->comm, p->pid);
+		} else {
 			struct rq *rq;
 			struct rq_flags rf;
 
@@ -2916,24 +3534,42 @@ static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork
 			}
 
 			task_rq_unlock(rq, p, &rf);
-		} else if (p->policy == SCHED_EXT) {
-			scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork",
-				  p->comm, p->pid);
 		}
 	}
 
-	p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
 	return 0;
 }
 
-static void scx_enable_task(struct task_struct *p)
+static int scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
+{
+	int ret;
+
+	ret = __scx_init_task(sch, p, fork);
+	if (!ret) {
+		/*
+		 * While @p's rq is not locked. @p is not visible to the rest of
+		 * SCX yet and it's safe to update the flags and state.
+		 */
+		p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
+		scx_set_task_state(p, SCX_TASK_INIT);
+	}
+	return ret;
+}
+
+static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
 {
-	struct scx_sched *sch = scx_root;
 	struct rq *rq = task_rq(p);
 	u32 weight;
 
 	lockdep_assert_rq_held(rq);
 
+	/*
+	 * Verify the task is not in BPF scheduler's custody. If flag
+	 * transitions are consistent, the flag should always be clear
+	 * here.
+	 */
+	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
+
 	/*
 	 * Set the weight before calling ops.enable() so that the scheduler
 	 * doesn't see a stale value if they inspect the task struct.
@@ -2947,16 +3583,20 @@ static void scx_enable_task(struct task_struct *p)
 
 	if (SCX_HAS_OP(sch, enable))
 		SCX_CALL_OP_TASK(sch, SCX_KF_REST, enable, rq, p);
-	scx_set_task_state(p, SCX_TASK_ENABLED);
 
 	if (SCX_HAS_OP(sch, set_weight))
 		SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_weight, rq,
 				 p, p->scx.weight);
 }
 
-static void scx_disable_task(struct task_struct *p)
+static void scx_enable_task(struct scx_sched *sch, struct task_struct *p)
+{
+	__scx_enable_task(sch, p);
+	scx_set_task_state(p, SCX_TASK_ENABLED);
+}
+
+static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
 {
-	struct scx_sched *sch = scx_root;
 	struct rq *rq = task_rq(p);
 
 	lockdep_assert_rq_held(rq);
@@ -2965,15 +3605,23 @@ static void scx_disable_task(struct task_struct *p)
 	if (SCX_HAS_OP(sch, disable))
 		SCX_CALL_OP_TASK(sch, SCX_KF_REST, disable, rq, p);
 	scx_set_task_state(p, SCX_TASK_READY);
+
+	/*
+	 * Verify the task is not in BPF scheduler's custody. If flag
+	 * transitions are consistent, the flag should always be clear
+	 * here.
+	 */
+	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
 }
 
-static void scx_exit_task(struct task_struct *p)
+static void __scx_disable_and_exit_task(struct scx_sched *sch,
+					struct task_struct *p)
 {
-	struct scx_sched *sch = scx_root;
 	struct scx_exit_task_args args = {
 		.cancelled = false,
 	};
 
+	lockdep_assert_held(&p->pi_lock);
 	lockdep_assert_rq_held(task_rq(p));
 
 	switch (scx_get_task_state(p)) {
@@ -2985,7 +3633,7 @@ static void scx_exit_task(struct task_struct *p)
 	case SCX_TASK_READY:
 		break;
 	case SCX_TASK_ENABLED:
-		scx_disable_task(p);
+		scx_disable_task(sch, p);
 		break;
 	default:
 		WARN_ON_ONCE(true);
@@ -2995,6 +3643,25 @@ static void scx_exit_task(struct task_struct *p)
 	if (SCX_HAS_OP(sch, exit_task))
 		SCX_CALL_OP_TASK(sch, SCX_KF_REST, exit_task, task_rq(p),
 				 p, &args);
+}
+
+static void scx_disable_and_exit_task(struct scx_sched *sch,
+				      struct task_struct *p)
+{
+	__scx_disable_and_exit_task(sch, p);
+
+	/*
+	 * If set, @p exited between __scx_init_task() and scx_enable_task() in
+	 * scx_sub_enable() and is initialized for both the associated sched and
+	 * its parent. Disable and exit for the child too.
+	 */
+	if ((p->scx.flags & SCX_TASK_SUB_INIT) &&
+	    !WARN_ON_ONCE(!scx_enabling_sub_sched)) {
+		__scx_disable_and_exit_task(scx_enabling_sub_sched, p);
+		p->scx.flags &= ~SCX_TASK_SUB_INIT;
+	}
+
+	scx_set_task_sched(p, NULL);
 	scx_set_task_state(p, SCX_TASK_NONE);
 }
 
@@ -3008,7 +3675,7 @@ void init_scx_entity(struct sched_ext_entity *scx)
 	INIT_LIST_HEAD(&scx->runnable_node);
 	scx->runnable_at = jiffies;
 	scx->ddsp_dsq_id = SCX_DSQ_INVALID;
-	scx->slice = READ_ONCE(scx_slice_dfl);
+	scx->slice = SCX_SLICE_DFL;
 }
 
 void scx_pre_fork(struct task_struct *p)
@@ -3022,14 +3689,25 @@ void scx_pre_fork(struct task_struct *p)
 	percpu_down_read(&scx_fork_rwsem);
 }
 
-int scx_fork(struct task_struct *p)
+int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs)
 {
+	s32 ret;
+
 	percpu_rwsem_assert_held(&scx_fork_rwsem);
 
-	if (scx_init_task_enabled)
-		return scx_init_task(p, task_group(p), true);
-	else
-		return 0;
+	if (scx_init_task_enabled) {
+#ifdef CONFIG_EXT_SUB_SCHED
+		struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched;
+#else
+		struct scx_sched *sch = scx_root;
+#endif
+		ret = scx_init_task(sch, p, true);
+		if (!ret)
+			scx_set_task_sched(p, sch);
+		return ret;
+	}
+
+	return 0;
 }
 
 void scx_post_fork(struct task_struct *p)
@@ -3047,7 +3725,7 @@ void scx_post_fork(struct task_struct *p)
 			struct rq *rq;
 
 			rq = task_rq_lock(p, &rf);
-			scx_enable_task(p);
+			scx_enable_task(scx_task_sched(p), p);
 			task_rq_unlock(rq, p, &rf);
 		}
 	}
@@ -3067,7 +3745,7 @@ void scx_cancel_fork(struct task_struct *p)
 
 		rq = task_rq_lock(p, &rf);
 		WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
-		scx_exit_task(p);
+		scx_disable_and_exit_task(scx_task_sched(p), p);
 		task_rq_unlock(rq, p, &rf);
 	}
 
@@ -3118,15 +3796,15 @@ void sched_ext_dead(struct task_struct *p)
 	raw_spin_unlock_irqrestore(&scx_tasks_lock, flags);
 
 	/*
-	 * @p is off scx_tasks and wholly ours. scx_enable()'s READY -> ENABLED
-	 * transitions can't race us. Disable ops for @p.
+	 * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
+	 * ENABLED transitions can't race us. Disable ops for @p.
 	 */
 	if (scx_get_task_state(p) != SCX_TASK_NONE) {
 		struct rq_flags rf;
 		struct rq *rq;
 
 		rq = task_rq_lock(p, &rf);
-		scx_exit_task(p);
+		scx_disable_and_exit_task(scx_task_sched(p), p);
 		task_rq_unlock(rq, p, &rf);
 	}
 }
@@ -3134,7 +3812,7 @@ void sched_ext_dead(struct task_struct *p)
 static void reweight_task_scx(struct rq *rq, struct task_struct *p,
 			      const struct load_weight *lw)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	lockdep_assert_rq_held(task_rq(p));
 
@@ -3153,12 +3831,12 @@ static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio)
 
 static void switching_to_scx(struct rq *rq, struct task_struct *p)
 {
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	if (task_dead_and_done(p))
 		return;
 
-	scx_enable_task(p);
+	scx_enable_task(sch, p);
 
 	/*
 	 * set_cpus_allowed_scx() is not called while @p is associated with a
@@ -3174,11 +3852,9 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p)
 	if (task_dead_and_done(p))
 		return;
 
-	scx_disable_task(p);
+	scx_disable_task(scx_task_sched(p), p);
 }
 
-static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) {}
-
 static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
 
 int scx_check_setscheduler(struct task_struct *p, int policy)
@@ -3193,17 +3869,325 @@ int scx_check_setscheduler(struct task_struct *p, int policy)
 	return 0;
 }
 
+static void process_ddsp_deferred_locals(struct rq *rq)
+{
+	struct task_struct *p;
+
+	lockdep_assert_rq_held(rq);
+
+	/*
+	 * Now that @rq can be unlocked, execute the deferred enqueueing of
+	 * tasks directly dispatched to the local DSQs of other CPUs. See
+	 * direct_dispatch(). Keep popping from the head instead of using
+	 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
+	 * temporarily.
+	 */
+	while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
+				struct task_struct, scx.dsq_list.node))) {
+		struct scx_sched *sch = scx_task_sched(p);
+		struct scx_dispatch_q *dsq;
+
+		list_del_init(&p->scx.dsq_list.node);
+
+		dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
+		if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
+			dispatch_to_local_dsq(sch, rq, dsq, p,
+					      p->scx.ddsp_enq_flags);
+	}
+}
+
+/*
+ * Determine whether @p should be reenqueued from a local DSQ.
+ *
+ * @reenq_flags is mutable and accumulates state across the DSQ walk:
+ *
+ * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First"
+ *   tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at
+ *   the head consumes the first slot.
+ *
+ * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if
+ *   rq_is_open() is true.
+ *
+ * An IMMED task is kept (returns %false) only if it's the first task in the DSQ
+ * AND the current task is done — i.e. it will execute immediately. All other
+ * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head,
+ * every IMMED task behind it gets reenqueued.
+ *
+ * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ |
+ * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local
+ * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers
+ * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT
+ * in process_deferred_reenq_locals().
+ */
+static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason)
+{
+	bool first;
+
+	first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST);
+	*reenq_flags |= SCX_REENQ_TSR_NOT_FIRST;
+
+	*reason = SCX_TASK_REENQ_KFUNC;
+
+	if ((p->scx.flags & SCX_TASK_IMMED) &&
+	    (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) {
+		__scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1);
+		*reason = SCX_TASK_REENQ_IMMED;
+		return true;
+	}
+
+	return *reenq_flags & SCX_REENQ_ANY;
+}
+
+static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
+{
+	LIST_HEAD(tasks);
+	u32 nr_enqueued = 0;
+	struct task_struct *p, *n;
+
+	lockdep_assert_rq_held(rq);
+
+	if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK))
+		reenq_flags &= ~__SCX_REENQ_TSR_MASK;
+	if (rq_is_open(rq, 0))
+		reenq_flags |= SCX_REENQ_TSR_RQ_OPEN;
+
+	/*
+	 * The BPF scheduler may choose to dispatch tasks back to
+	 * @rq->scx.local_dsq. Move all candidate tasks off to a private list
+	 * first to avoid processing the same tasks repeatedly.
+	 */
+	list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
+				 scx.dsq_list.node) {
+		struct scx_sched *task_sch = scx_task_sched(p);
+		u32 reason;
+
+		/*
+		 * If @p is being migrated, @p's current CPU may not agree with
+		 * its allowed CPUs and the migration_cpu_stop is about to
+		 * deactivate and re-activate @p anyway. Skip re-enqueueing.
+		 *
+		 * While racing sched property changes may also dequeue and
+		 * re-enqueue a migrating task while its current CPU and allowed
+		 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
+		 * the current local DSQ for running tasks and thus are not
+		 * visible to the BPF scheduler.
+		 */
+		if (p->migration_pending)
+			continue;
+
+		if (!scx_is_descendant(task_sch, sch))
+			continue;
+
+		if (!local_task_should_reenq(p, &reenq_flags, &reason))
+			continue;
+
+		dispatch_dequeue(rq, p);
+
+		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		p->scx.flags |= reason;
+
+		list_add_tail(&p->scx.dsq_list.node, &tasks);
+	}
+
+	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
+		list_del_init(&p->scx.dsq_list.node);
+
+		do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+
+		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		nr_enqueued++;
+	}
+
+	return nr_enqueued;
+}
+
+static void process_deferred_reenq_locals(struct rq *rq)
+{
+	u64 seq = ++rq->scx.deferred_reenq_locals_seq;
+
+	lockdep_assert_rq_held(rq);
+
+	while (true) {
+		struct scx_sched *sch;
+		u64 reenq_flags;
+		bool skip = false;
+
+		scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
+			struct scx_deferred_reenq_local *drl =
+				list_first_entry_or_null(&rq->scx.deferred_reenq_locals,
+							 struct scx_deferred_reenq_local,
+							 node);
+			struct scx_sched_pcpu *sch_pcpu;
+
+			if (!drl)
+				return;
+
+			sch_pcpu = container_of(drl, struct scx_sched_pcpu,
+						deferred_reenq_local);
+			sch = sch_pcpu->sch;
+
+			reenq_flags = drl->flags;
+			WRITE_ONCE(drl->flags, 0);
+			list_del_init(&drl->node);
+
+			if (likely(drl->seq != seq)) {
+				drl->seq = seq;
+				drl->cnt = 0;
+			} else {
+				if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) {
+					scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times",
+						  drl->cnt);
+					skip = true;
+				}
+
+				__scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1);
+			}
+		}
+
+		if (!skip) {
+			/* see schedule_dsq_reenq() */
+			smp_mb();
+
+			reenq_local(sch, rq, reenq_flags);
+		}
+	}
+}
+
+static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason)
+{
+	*reason = SCX_TASK_REENQ_KFUNC;
+	return reenq_flags & SCX_REENQ_ANY;
+}
+
+static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags)
+{
+	struct rq *locked_rq = rq;
+	struct scx_sched *sch = dsq->sched;
+	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0);
+	struct task_struct *p;
+	s32 nr_enqueued = 0;
+
+	lockdep_assert_rq_held(rq);
+
+	raw_spin_lock(&dsq->lock);
+
+	while (likely(!READ_ONCE(sch->bypass_depth))) {
+		struct rq *task_rq;
+		u32 reason;
+
+		p = nldsq_cursor_next_task(&cursor, dsq);
+		if (!p)
+			break;
+
+		if (!user_task_should_reenq(p, reenq_flags, &reason))
+			continue;
+
+		task_rq = task_rq(p);
+
+		if (locked_rq != task_rq) {
+			if (locked_rq)
+				raw_spin_rq_unlock(locked_rq);
+			if (unlikely(!raw_spin_rq_trylock(task_rq))) {
+				raw_spin_unlock(&dsq->lock);
+				raw_spin_rq_lock(task_rq);
+				raw_spin_lock(&dsq->lock);
+			}
+			locked_rq = task_rq;
+
+			/* did we lose @p while switching locks? */
+			if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p))
+				continue;
+		}
+
+		/* @p is on @dsq, its rq and @dsq are locked */
+		dispatch_dequeue_locked(p, dsq);
+		raw_spin_unlock(&dsq->lock);
+
+		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		p->scx.flags |= reason;
+
+		do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
+
+		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+
+		if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
+			raw_spin_rq_unlock(locked_rq);
+			locked_rq = NULL;
+			cpu_relax();
+		}
+
+		raw_spin_lock(&dsq->lock);
+	}
+
+	list_del_init(&cursor.node);
+	raw_spin_unlock(&dsq->lock);
+
+	if (locked_rq != rq) {
+		if (locked_rq)
+			raw_spin_rq_unlock(locked_rq);
+		raw_spin_rq_lock(rq);
+	}
+}
+
+static void process_deferred_reenq_users(struct rq *rq)
+{
+	lockdep_assert_rq_held(rq);
+
+	while (true) {
+		struct scx_dispatch_q *dsq;
+		u64 reenq_flags;
+
+		scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
+			struct scx_deferred_reenq_user *dru =
+				list_first_entry_or_null(&rq->scx.deferred_reenq_users,
+							 struct scx_deferred_reenq_user,
+							 node);
+			struct scx_dsq_pcpu *dsq_pcpu;
+
+			if (!dru)
+				return;
+
+			dsq_pcpu = container_of(dru, struct scx_dsq_pcpu,
+						deferred_reenq_user);
+			dsq = dsq_pcpu->dsq;
+			reenq_flags = dru->flags;
+			WRITE_ONCE(dru->flags, 0);
+			list_del_init(&dru->node);
+		}
+
+		/* see schedule_dsq_reenq() */
+		smp_mb();
+
+		BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN);
+		reenq_user(rq, dsq, reenq_flags);
+	}
+}
+
+static void run_deferred(struct rq *rq)
+{
+	process_ddsp_deferred_locals(rq);
+
+	if (!list_empty(&rq->scx.deferred_reenq_locals))
+		process_deferred_reenq_locals(rq);
+
+	if (!list_empty(&rq->scx.deferred_reenq_users))
+		process_deferred_reenq_users(rq);
+}
+
 #ifdef CONFIG_NO_HZ_FULL
 bool scx_can_stop_tick(struct rq *rq)
 {
 	struct task_struct *p = rq->curr;
-
-	if (scx_rq_bypassing(rq))
-		return false;
+	struct scx_sched *sch = scx_task_sched(p);
 
 	if (p->sched_class != &ext_sched_class)
 		return true;
 
+	if (scx_bypassing(sch, cpu_of(rq)))
+		return false;
+
 	/*
 	 * @rq can dispatch from different DSQs, so we can't tell whether it
 	 * needs the tick or not by looking at nr_running. Allow stopping ticks
@@ -3406,33 +4390,55 @@ void scx_group_set_bandwidth(struct task_group *tg,
 
 	percpu_up_read(&scx_cgroup_ops_rwsem);
 }
+#endif	/* CONFIG_EXT_GROUP_SCHED */
+
+#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
+static struct cgroup *root_cgroup(void)
+{
+	return &cgrp_dfl_root.cgrp;
+}
+
+static struct cgroup *sch_cgroup(struct scx_sched *sch)
+{
+	return sch->cgrp;
+}
+
+/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
+static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
+{
+	struct cgroup *pos;
+	struct cgroup_subsys_state *css;
+
+	cgroup_for_each_live_descendant_pre(pos, css, cgrp)
+		rcu_assign_pointer(pos->scx_sched, sch);
+}
 
 static void scx_cgroup_lock(void)
 {
+#ifdef CONFIG_EXT_GROUP_SCHED
 	percpu_down_write(&scx_cgroup_ops_rwsem);
+#endif
 	cgroup_lock();
 }
 
 static void scx_cgroup_unlock(void)
 {
 	cgroup_unlock();
+#ifdef CONFIG_EXT_GROUP_SCHED
 	percpu_up_write(&scx_cgroup_ops_rwsem);
+#endif
 }
-
-#else	/* CONFIG_EXT_GROUP_SCHED */
-
+#else	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
+static struct cgroup *root_cgroup(void) { return NULL; }
+static struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
+static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
 static void scx_cgroup_lock(void) {}
 static void scx_cgroup_unlock(void) {}
-
-#endif	/* CONFIG_EXT_GROUP_SCHED */
+#endif	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
 
 /*
  * Omitted operations:
  *
- * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task
- *   isn't tied to the CPU at that point. Preemption is implemented by resetting
- *   the victim task's slice to 0 and triggering reschedule on the target CPU.
- *
  * - migrate_task_rq: Unnecessary as task to cpu mapping is transient.
  *
  * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
@@ -3473,13 +4479,60 @@ DEFINE_SCHED_CLASS(ext) = {
 #endif
 };
 
-static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id)
+static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id,
+		    struct scx_sched *sch)
 {
+	s32 cpu;
+
 	memset(dsq, 0, sizeof(*dsq));
 
 	raw_spin_lock_init(&dsq->lock);
 	INIT_LIST_HEAD(&dsq->list);
 	dsq->id = dsq_id;
+	dsq->sched = sch;
+
+	dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu);
+	if (!dsq->pcpu)
+		return -ENOMEM;
+
+	for_each_possible_cpu(cpu) {
+		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+
+		pcpu->dsq = dsq;
+		INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
+	}
+
+	return 0;
+}
+
+static void exit_dsq(struct scx_dispatch_q *dsq)
+{
+	s32 cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+		struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
+		struct rq *rq = cpu_rq(cpu);
+
+		/*
+		 * There must have been a RCU grace period since the last
+		 * insertion and @dsq should be off the deferred list by now.
+		 */
+		if (WARN_ON_ONCE(!list_empty(&dru->node))) {
+			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
+			list_del_init(&dru->node);
+		}
+	}
+
+	free_percpu(dsq->pcpu);
+}
+
+static void free_dsq_rcufn(struct rcu_head *rcu)
+{
+	struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu);
+
+	exit_dsq(dsq);
+	kfree(dsq);
 }
 
 static void free_dsq_irq_workfn(struct irq_work *irq_work)
@@ -3488,7 +4541,7 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work)
 	struct scx_dispatch_q *dsq, *tmp_dsq;
 
 	llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node)
-		kfree_rcu(dsq, rcu);
+		call_rcu(&dsq->rcu, free_dsq_rcufn);
 }
 
 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
@@ -3664,6 +4717,7 @@ static const struct attribute_group scx_global_attr_group = {
 	.attrs = scx_global_attrs,
 };
 
+static void free_pnode(struct scx_sched_pnode *pnode);
 static void free_exit_info(struct scx_exit_info *ei);
 
 static void scx_sched_free_rcu_work(struct work_struct *work)
@@ -3672,22 +4726,42 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
 	struct rhashtable_iter rht_iter;
 	struct scx_dispatch_q *dsq;
-	int node;
+	int cpu, node;
 
-	irq_work_sync(&sch->error_irq_work);
+	irq_work_sync(&sch->disable_irq_work);
 	kthread_destroy_worker(sch->helper);
+	timer_shutdown_sync(&sch->bypass_lb_timer);
 
-	free_percpu(sch->pcpu);
-
-	for_each_node_state(node, N_POSSIBLE)
-		kfree(sch->global_dsqs[node]);
-	kfree(sch->global_dsqs);
+#ifdef CONFIG_EXT_SUB_SCHED
+	kfree(sch->cgrp_path);
+	if (sch_cgroup(sch))
+		cgroup_put(sch_cgroup(sch));
+#endif	/* CONFIG_EXT_SUB_SCHED */
 
-	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
-	do {
-		rhashtable_walk_start(&rht_iter);
+	for_each_possible_cpu(cpu) {
+		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 
-		while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq))
+		/*
+		 * $sch would have entered bypass mode before the RCU grace
+		 * period. As that blocks new deferrals, all
+		 * deferred_reenq_local_node's must be off-list by now.
+		 */
+		WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
+
+		exit_dsq(bypass_dsq(sch, cpu));
+	}
+
+	free_percpu(sch->pcpu);
+
+	for_each_node_state(node, N_POSSIBLE)
+		free_pnode(sch->pnode[node]);
+	kfree(sch->pnode);
+
+	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
+	do {
+		rhashtable_walk_start(&rht_iter);
+
+		while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter))))
 			destroy_dsq(sch, dsq->id);
 
 		rhashtable_walk_stop(&rht_iter);
@@ -3704,7 +4778,7 @@ static void scx_kobj_release(struct kobject *kobj)
 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
 
 	INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work);
-	queue_rcu_work(system_unbound_wq, &sch->rcu_work);
+	queue_rcu_work(system_dfl_wq, &sch->rcu_work);
 }
 
 static ssize_t scx_attr_ops_show(struct kobject *kobj,
@@ -3733,10 +4807,14 @@ static ssize_t scx_attr_events_show(struct kobject *kobj,
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
+	at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED);
+	at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH);
 	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE);
+	at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED);
+	at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH);
 	return at;
 }
 SCX_ATTR(events);
@@ -3785,7 +4863,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p)
 	if (!scx_enabled())
 		return true;
 
-	sch = rcu_dereference_sched(scx_root);
+	sch = scx_task_sched(p);
 	if (unlikely(!sch))
 		return true;
 
@@ -3878,7 +4956,7 @@ void scx_softlockup(u32 dur_s)
  * a good state before taking more drastic actions.
  *
  * Returns %true if sched_ext is enabled and abort was initiated, which may
- * resolve the reported hardlockdup. %false if sched_ext is not enabled or
+ * resolve the reported hardlockup. %false if sched_ext is not enabled or
  * someone else already initiated abort.
  */
 bool scx_hardlockup(int cpu)
@@ -3891,13 +4969,14 @@ bool scx_hardlockup(int cpu)
 	return true;
 }
 
-static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
+static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
 			 struct cpumask *donee_mask, struct cpumask *resched_mask,
 			 u32 nr_donor_target, u32 nr_donee_target)
 {
-	struct scx_dispatch_q *donor_dsq = &rq->scx.bypass_dsq;
+	struct rq *donor_rq = cpu_rq(donor);
+	struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor);
 	struct task_struct *p, *n;
-	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, 0, 0);
+	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0);
 	s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target;
 	u32 nr_balanced = 0, min_delta_us;
 
@@ -3911,7 +4990,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 	if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us)))
 		return 0;
 
-	raw_spin_rq_lock_irq(rq);
+	raw_spin_rq_lock_irq(donor_rq);
 	raw_spin_lock(&donor_dsq->lock);
 	list_add(&cursor.node, &donor_dsq->list);
 resume:
@@ -3919,7 +4998,6 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 	n = nldsq_next_task(donor_dsq, n, false);
 
 	while ((p = n)) {
-		struct rq *donee_rq;
 		struct scx_dispatch_q *donee_dsq;
 		int donee;
 
@@ -3935,14 +5013,13 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 		if (donee >= nr_cpu_ids)
 			continue;
 
-		donee_rq = cpu_rq(donee);
-		donee_dsq = &donee_rq->scx.bypass_dsq;
+		donee_dsq = bypass_dsq(sch, donee);
 
 		/*
 		 * $p's rq is not locked but $p's DSQ lock protects its
 		 * scheduling properties making this test safe.
 		 */
-		if (!task_can_run_on_remote_rq(sch, p, donee_rq, false))
+		if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false))
 			continue;
 
 		/*
@@ -3957,7 +5034,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 		 * between bypass DSQs.
 		 */
 		dispatch_dequeue_locked(p, donor_dsq);
-		dispatch_enqueue(sch, donee_dsq, p, SCX_ENQ_NESTED);
+		dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);
 
 		/*
 		 * $donee might have been idle and need to be woken up. No need
@@ -3972,9 +5049,9 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 		if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) {
 			list_move_tail(&cursor.node, &n->scx.dsq_list.node);
 			raw_spin_unlock(&donor_dsq->lock);
-			raw_spin_rq_unlock_irq(rq);
+			raw_spin_rq_unlock_irq(donor_rq);
 			cpu_relax();
-			raw_spin_rq_lock_irq(rq);
+			raw_spin_rq_lock_irq(donor_rq);
 			raw_spin_lock(&donor_dsq->lock);
 			goto resume;
 		}
@@ -3982,7 +5059,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq,
 
 	list_del_init(&cursor.node);
 	raw_spin_unlock(&donor_dsq->lock);
-	raw_spin_rq_unlock_irq(rq);
+	raw_spin_rq_unlock_irq(donor_rq);
 
 	return nr_balanced;
 }
@@ -4000,7 +5077,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 
 	/* count the target tasks and CPUs */
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		u32 nr = READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr);
+		u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
 
 		nr_tasks += nr;
 		nr_cpus++;
@@ -4022,24 +5099,21 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 
 	cpumask_clear(donee_mask);
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		if (READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr) < nr_target)
+		if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target)
 			cpumask_set_cpu(cpu, donee_mask);
 	}
 
 	/* iterate !donee CPUs and see if they should be offloaded */
 	cpumask_clear(resched_mask);
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		struct rq *rq = cpu_rq(cpu);
-		struct scx_dispatch_q *donor_dsq = &rq->scx.bypass_dsq;
-
 		if (cpumask_empty(donee_mask))
 			break;
 		if (cpumask_test_cpu(cpu, donee_mask))
 			continue;
-		if (READ_ONCE(donor_dsq->nr) <= nr_donor_target)
+		if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target)
 			continue;
 
-		nr_balanced += bypass_lb_cpu(sch, rq, donee_mask, resched_mask,
+		nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask,
 					     nr_donor_target, nr_target);
 	}
 
@@ -4047,7 +5121,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 		resched_cpu(cpu);
 
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		u32 nr = READ_ONCE(cpu_rq(cpu)->scx.bypass_dsq.nr);
+		u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
 
 		after_min = min(nr, after_min);
 		after_max = max(nr, after_max);
@@ -4069,12 +5143,11 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
  */
 static void scx_bypass_lb_timerfn(struct timer_list *timer)
 {
-	struct scx_sched *sch;
+	struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer);
 	int node;
 	u32 intv_us;
 
-	sch = rcu_dereference_all(scx_root);
-	if (unlikely(!sch) || !READ_ONCE(scx_bypass_depth))
+	if (!bypass_dsp_enabled(sch))
 		return;
 
 	for_each_node_with_cpus(node)
@@ -4085,10 +5158,102 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer)
 		mod_timer(timer, jiffies + usecs_to_jiffies(intv_us));
 }
 
-static DEFINE_TIMER(scx_bypass_lb_timer, scx_bypass_lb_timerfn);
+static bool inc_bypass_depth(struct scx_sched *sch)
+{
+	lockdep_assert_held(&scx_bypass_lock);
+
+	WARN_ON_ONCE(sch->bypass_depth < 0);
+	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1);
+	if (sch->bypass_depth != 1)
+		return false;
+
+	WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC);
+	sch->bypass_timestamp = ktime_get_ns();
+	scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1);
+	return true;
+}
+
+static bool dec_bypass_depth(struct scx_sched *sch)
+{
+	lockdep_assert_held(&scx_bypass_lock);
+
+	WARN_ON_ONCE(sch->bypass_depth < 1);
+	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1);
+	if (sch->bypass_depth != 0)
+		return false;
+
+	WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL);
+	scx_add_event(sch, SCX_EV_BYPASS_DURATION,
+		      ktime_get_ns() - sch->bypass_timestamp);
+	return true;
+}
+
+static void enable_bypass_dsp(struct scx_sched *sch)
+{
+	struct scx_sched *host = scx_parent(sch) ?: sch;
+	u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us);
+	s32 ret;
+
+	/*
+	 * @sch->bypass_depth transitioning from 0 to 1 triggers enabling.
+	 * Shouldn't stagger.
+	 */
+	if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim)))
+		return;
+
+	/*
+	 * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of
+	 * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is
+	 * called iff @sch is not already bypassed due to an ancestor bypassing,
+	 * we can assume that the parent is not bypassing and thus will be the
+	 * host of the bypass DSQs.
+	 *
+	 * While the situation may change in the future, the following
+	 * guarantees that the nearest non-bypassing ancestor or root has bypass
+	 * dispatch enabled while a descendant is bypassing, which is all that's
+	 * required.
+	 *
+	 * bypass_dsp_enabled() test is used to determine whether to enter the
+	 * bypass dispatch handling path from both bypassing and hosting scheds.
+	 * Bump enable depth on both @sch and bypass dispatch host.
+	 */
+	ret = atomic_inc_return(&sch->bypass_dsp_enable_depth);
+	WARN_ON_ONCE(ret <= 0);
+
+	if (host != sch) {
+		ret = atomic_inc_return(&host->bypass_dsp_enable_depth);
+		WARN_ON_ONCE(ret <= 0);
+	}
+
+	/*
+	 * The LB timer will stop running if bypass dispatch is disabled. Start
+	 * after enabling bypass dispatch.
+	 */
+	if (intv_us && !timer_pending(&host->bypass_lb_timer))
+		mod_timer(&host->bypass_lb_timer,
+			  jiffies + usecs_to_jiffies(intv_us));
+}
+
+/* may be called without holding scx_bypass_lock */
+static void disable_bypass_dsp(struct scx_sched *sch)
+{
+	s32 ret;
+
+	if (!test_and_clear_bit(0, &sch->bypass_dsp_claim))
+		return;
+
+	ret = atomic_dec_return(&sch->bypass_dsp_enable_depth);
+	WARN_ON_ONCE(ret < 0);
+
+	if (scx_parent(sch)) {
+		ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth);
+		WARN_ON_ONCE(ret < 0);
+	}
+}
 
 /**
  * scx_bypass - [Un]bypass scx_ops and guarantee forward progress
+ * @sch: sched to bypass
  * @bypass: true for bypass, false for unbypass
  *
  * Bypassing guarantees that all runnable tasks make forward progress without
@@ -4118,49 +5283,42 @@ static DEFINE_TIMER(scx_bypass_lb_timer, scx_bypass_lb_timerfn);
  *
  * - scx_prio_less() reverts to the default core_sched_at order.
  */
-static void scx_bypass(bool bypass)
+static void scx_bypass(struct scx_sched *sch, bool bypass)
 {
-	static DEFINE_RAW_SPINLOCK(bypass_lock);
-	static unsigned long bypass_timestamp;
-	struct scx_sched *sch;
+	struct scx_sched *pos;
 	unsigned long flags;
 	int cpu;
 
-	raw_spin_lock_irqsave(&bypass_lock, flags);
-	sch = rcu_dereference_bh(scx_root);
+	raw_spin_lock_irqsave(&scx_bypass_lock, flags);
 
 	if (bypass) {
-		u32 intv_us;
-
-		WRITE_ONCE(scx_bypass_depth, scx_bypass_depth + 1);
-		WARN_ON_ONCE(scx_bypass_depth <= 0);
-		if (scx_bypass_depth != 1)
+		if (!inc_bypass_depth(sch))
 			goto unlock;
-		WRITE_ONCE(scx_slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC);
-		bypass_timestamp = ktime_get_ns();
-		if (sch)
-			scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1);
-
-		intv_us = READ_ONCE(scx_bypass_lb_intv_us);
-		if (intv_us && !timer_pending(&scx_bypass_lb_timer)) {
-			scx_bypass_lb_timer.expires =
-				jiffies + usecs_to_jiffies(intv_us);
-			add_timer_global(&scx_bypass_lb_timer);
-		}
+
+		enable_bypass_dsp(sch);
 	} else {
-		WRITE_ONCE(scx_bypass_depth, scx_bypass_depth - 1);
-		WARN_ON_ONCE(scx_bypass_depth < 0);
-		if (scx_bypass_depth != 0)
+		if (!dec_bypass_depth(sch))
 			goto unlock;
-		WRITE_ONCE(scx_slice_dfl, SCX_SLICE_DFL);
-		if (sch)
-			scx_add_event(sch, SCX_EV_BYPASS_DURATION,
-				      ktime_get_ns() - bypass_timestamp);
 	}
 
+	/*
+	 * Bypass state is propagated to all descendants - an scx_sched bypasses
+	 * if itself or any of its ancestors are in bypass mode.
+	 */
+	raw_spin_lock(&scx_sched_lock);
+	scx_for_each_descendant_pre(pos, sch) {
+		if (pos == sch)
+			continue;
+		if (bypass)
+			inc_bypass_depth(pos);
+		else
+			dec_bypass_depth(pos);
+	}
+	raw_spin_unlock(&scx_sched_lock);
+
 	/*
 	 * No task property is changing. We just need to make sure all currently
-	 * queued tasks are re-queued according to the new scx_rq_bypassing()
+	 * queued tasks are re-queued according to the new scx_bypassing()
 	 * state. As an optimization, walk each rq's runnable_list instead of
 	 * the scx_tasks list.
 	 *
@@ -4172,19 +5330,23 @@ static void scx_bypass(bool bypass)
 		struct task_struct *p, *n;
 
 		raw_spin_rq_lock(rq);
+		raw_spin_lock(&scx_sched_lock);
 
-		if (bypass) {
-			WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING);
-			rq->scx.flags |= SCX_RQ_BYPASSING;
-		} else {
-			WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING));
-			rq->scx.flags &= ~SCX_RQ_BYPASSING;
+		scx_for_each_descendant_pre(pos, sch) {
+			struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
+
+			if (pos->bypass_depth)
+				pcpu->flags |= SCX_SCHED_PCPU_BYPASSING;
+			else
+				pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
 		}
 
+		raw_spin_unlock(&scx_sched_lock);
+
 		/*
 		 * We need to guarantee that no tasks are on the BPF scheduler
 		 * while bypassing. Either we see enabled or the enable path
-		 * sees scx_rq_bypassing() before moving tasks to SCX.
+		 * sees scx_bypassing() before moving tasks to SCX.
 		 */
 		if (!scx_enabled()) {
 			raw_spin_rq_unlock(rq);
@@ -4200,6 +5362,9 @@ static void scx_bypass(bool bypass)
 		 */
 		list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
 						 scx.runnable_node) {
+			if (!scx_is_descendant(scx_task_sched(p), sch))
+				continue;
+
 			/* cycling deq/enq is enough, see the function comment */
 			scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
 				/* nothing */ ;
@@ -4213,8 +5378,11 @@ static void scx_bypass(bool bypass)
 		raw_spin_rq_unlock(rq);
 	}
 
+	/* disarming must come after moving all tasks out of the bypass DSQs */
+	if (!bypass)
+		disable_bypass_dsp(sch);
 unlock:
-	raw_spin_unlock_irqrestore(&bypass_lock, flags);
+	raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
 }
 
 static void free_exit_info(struct scx_exit_info *ei)
@@ -4256,6 +5424,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind)
 		return "unregistered from the main kernel";
 	case SCX_EXIT_SYSRQ:
 		return "disabled by sysrq-S";
+	case SCX_EXIT_PARENT:
+		return "parent exiting";
 	case SCX_EXIT_ERROR:
 		return "runtime error";
 	case SCX_EXIT_ERROR_BPF:
@@ -4281,28 +5451,279 @@ static void free_kick_syncs(void)
 	}
 }
 
-static void scx_disable_workfn(struct kthread_work *work)
+static void refresh_watchdog(void)
 {
-	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
-	struct scx_exit_info *ei = sch->exit_info;
+	struct scx_sched *sch;
+	unsigned long intv = ULONG_MAX;
+
+	/* take the shortest timeout and use its half for watchdog interval */
+	rcu_read_lock();
+	list_for_each_entry_rcu(sch, &scx_sched_all, all)
+		intv = max(min(intv, sch->watchdog_timeout / 2), 1);
+	rcu_read_unlock();
+
+	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
+	WRITE_ONCE(scx_watchdog_interval, intv);
+
+	if (intv < ULONG_MAX)
+		mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv);
+	else
+		cancel_delayed_work_sync(&scx_watchdog_work);
+}
+
+static s32 scx_link_sched(struct scx_sched *sch)
+{
+	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
+#ifdef CONFIG_EXT_SUB_SCHED
+		struct scx_sched *parent = scx_parent(sch);
+		s32 ret;
+
+		if (parent) {
+			/*
+			 * scx_claim_exit() propagates exit_kind transition to
+			 * its sub-scheds while holding scx_sched_lock - either
+			 * we can see the parent's non-NONE exit_kind or the
+			 * parent can shoot us down.
+			 */
+			if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) {
+				scx_error(sch, "parent disabled");
+				return -ENOENT;
+			}
+
+			ret = rhashtable_lookup_insert_fast(&scx_sched_hash,
+					&sch->hash_node, scx_sched_hash_params);
+			if (ret) {
+				scx_error(sch, "failed to insert into scx_sched_hash (%d)", ret);
+				return ret;
+			}
+
+			list_add_tail(&sch->sibling, &parent->children);
+		}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+		list_add_tail_rcu(&sch->all, &scx_sched_all);
+	}
+
+	refresh_watchdog();
+	return 0;
+}
+
+static void scx_unlink_sched(struct scx_sched *sch)
+{
+	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
+#ifdef CONFIG_EXT_SUB_SCHED
+		if (scx_parent(sch)) {
+			rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
+					       scx_sched_hash_params);
+			list_del_init(&sch->sibling);
+		}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+		list_del_rcu(&sch->all);
+	}
+
+	refresh_watchdog();
+}
+
+/*
+ * Called to disable future dumps and wait for in-progress one while disabling
+ * @sch. Once @sch becomes empty during disable, there's no point in dumping it.
+ * This prevents calling dump ops on a dead sch.
+ */
+static void scx_disable_dump(struct scx_sched *sch)
+{
+	guard(raw_spinlock_irqsave)(&scx_dump_lock);
+	sch->dump_disabled = true;
+}
+
+#ifdef CONFIG_EXT_SUB_SCHED
+static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
+
+static void drain_descendants(struct scx_sched *sch)
+{
+	/*
+	 * Child scheds that finished the critical part of disabling will take
+	 * themselves off @sch->children. Wait for it to drain. As propagation
+	 * is recursive, empty @sch->children means that all proper descendant
+	 * scheds reached unlinking stage.
+	 */
+	wait_event(scx_unlink_waitq, list_empty(&sch->children));
+}
+
+static void scx_fail_parent(struct scx_sched *sch,
+			    struct task_struct *failed, s32 fail_code)
+{
+	struct scx_sched *parent = scx_parent(sch);
 	struct scx_task_iter sti;
 	struct task_struct *p;
-	int kind, cpu;
 
-	kind = atomic_read(&sch->exit_kind);
-	while (true) {
-		if (kind == SCX_EXIT_DONE)	/* already disabled? */
-			return;
-		WARN_ON_ONCE(kind == SCX_EXIT_NONE);
-		if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE))
+	scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler",
+		  fail_code, failed->comm, failed->pid);
+
+	/*
+	 * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into
+	 * it. This may cause downstream failures on the BPF side but $parent is
+	 * dying anyway.
+	 */
+	scx_bypass(parent, true);
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		if (scx_task_on_sched(parent, p))
+			continue;
+
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			scx_disable_and_exit_task(sch, p);
+			rcu_assign_pointer(p->scx.sched, parent);
+		}
+	}
+	scx_task_iter_stop(&sti);
+}
+
+static void scx_sub_disable(struct scx_sched *sch)
+{
+	struct scx_sched *parent = scx_parent(sch);
+	struct scx_task_iter sti;
+	struct task_struct *p;
+	int ret;
+
+	/*
+	 * Guarantee forward progress and wait for descendants to be disabled.
+	 * To limit disruptions, $parent is not bypassed. Tasks are fully
+	 * prepped and then inserted back into $parent.
+	 */
+	scx_bypass(sch, true);
+	drain_descendants(sch);
+
+	/*
+	 * Here, every runnable task is guaranteed to make forward progress and
+	 * we can safely use blocking synchronization constructs. Actually
+	 * disable ops.
+	 */
+	mutex_lock(&scx_enable_mutex);
+	percpu_down_write(&scx_fork_rwsem);
+	scx_cgroup_lock();
+
+	set_cgroup_sched(sch_cgroup(sch), parent);
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		struct rq *rq;
+		struct rq_flags rf;
+
+		/* filter out duplicate visits */
+		if (scx_task_on_sched(parent, p))
+			continue;
+
+		/*
+		 * By the time control reaches here, all descendant schedulers
+		 * should already have been disabled.
+		 */
+		WARN_ON_ONCE(!scx_task_on_sched(sch, p));
+
+		/*
+		 * If $p is about to be freed, nothing prevents $sch from
+		 * unloading before $p reaches sched_ext_free(). Disable and
+		 * exit $p right away.
+		 */
+		if (!tryget_task_struct(p)) {
+			scx_disable_and_exit_task(sch, p);
+			continue;
+		}
+
+		scx_task_iter_unlock(&sti);
+
+		/*
+		 * $p is READY or ENABLED on @sch. Initialize for $parent,
+		 * disable and exit from @sch, and then switch over to $parent.
+		 *
+		 * If a task fails to initialize for $parent, the only available
+		 * action is disabling $parent too. While this allows disabling
+		 * of a child sched to cause the parent scheduler to fail, the
+		 * failure can only originate from ops.init_task() of the
+		 * parent. A child can't directly affect the parent through its
+		 * own failures.
+		 */
+		ret = __scx_init_task(parent, p, false);
+		if (ret) {
+			scx_fail_parent(sch, p, ret);
+			put_task_struct(p);
 			break;
+		}
+
+		rq = task_rq_lock(p, &rf);
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			/*
+			 * $p is initialized for $parent and still attached to
+			 * @sch. Disable and exit for @sch, switch over to
+			 * $parent, override the state to READY to account for
+			 * $p having already been initialized, and then enable.
+			 */
+			scx_disable_and_exit_task(sch, p);
+			scx_set_task_state(p, SCX_TASK_INIT);
+			rcu_assign_pointer(p->scx.sched, parent);
+			scx_set_task_state(p, SCX_TASK_READY);
+			scx_enable_task(parent, p);
+		}
+		task_rq_unlock(rq, p, &rf);
+
+		put_task_struct(p);
 	}
-	ei->kind = kind;
-	ei->reason = scx_exit_reason(ei->kind);
+	scx_task_iter_stop(&sti);
+
+	scx_disable_dump(sch);
+
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+
+	/*
+	 * All tasks are moved off of @sch but there may still be on-going
+	 * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use
+	 * the expedited version as ancestors may be waiting in bypass mode.
+	 * Also, tell the parent that there is no need to keep running bypass
+	 * DSQs for us.
+	 */
+	synchronize_rcu_expedited();
+	disable_bypass_dsp(sch);
 
-	/* guarantee forward progress by bypassing scx_ops */
-	scx_bypass(true);
-	WRITE_ONCE(scx_aborting, false);
+	scx_unlink_sched(sch);
+
+	mutex_unlock(&scx_enable_mutex);
+
+	/*
+	 * @sch is now unlinked from the parent's children list. Notify and call
+	 * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called
+	 * after unlinking and releasing all locks. See scx_claim_exit().
+	 */
+	wake_up_all(&scx_unlink_waitq);
+
+	if (parent->ops.sub_detach && sch->sub_attached) {
+		struct scx_sub_detach_args sub_detach_args = {
+			.ops = &sch->ops,
+			.cgroup_path = sch->cgrp_path,
+		};
+		SCX_CALL_OP(parent, SCX_KF_UNLOCKED, sub_detach, NULL,
+			    &sub_detach_args);
+	}
+
+	if (sch->ops.exit)
+		SCX_CALL_OP(sch, SCX_KF_UNLOCKED, exit, NULL, sch->exit_info);
+	kobject_del(&sch->kobj);
+}
+#else	/* CONFIG_EXT_SUB_SCHED */
+static void drain_descendants(struct scx_sched *sch) { }
+static void scx_sub_disable(struct scx_sched *sch) { }
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+static void scx_root_disable(struct scx_sched *sch)
+{
+	struct scx_exit_info *ei = sch->exit_info;
+	struct scx_task_iter sti;
+	struct task_struct *p;
+	int cpu;
+
+	/* guarantee forward progress and wait for descendants to be disabled */
+	scx_bypass(sch, true);
+	drain_descendants(sch);
 
 	switch (scx_set_enable_state(SCX_DISABLING)) {
 	case SCX_DISABLING:
@@ -4329,7 +5750,7 @@ static void scx_disable_workfn(struct kthread_work *work)
 
 	/*
 	 * Shut down cgroup support before tasks so that the cgroup attach path
-	 * doesn't race against scx_exit_task().
+	 * doesn't race against scx_disable_and_exit_task().
 	 */
 	scx_cgroup_lock();
 	scx_cgroup_exit(sch);
@@ -4343,7 +5764,7 @@ static void scx_disable_workfn(struct kthread_work *work)
 
 	scx_init_task_enabled = false;
 
-	scx_task_iter_start(&sti);
+	scx_task_iter_start(&sti, NULL);
 	while ((p = scx_task_iter_next_locked(&sti))) {
 		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
 		const struct sched_class *old_class = p->sched_class;
@@ -4358,9 +5779,16 @@ static void scx_disable_workfn(struct kthread_work *work)
 			p->sched_class = new_class;
 		}
 
-		scx_exit_task(p);
+		scx_disable_and_exit_task(scx_task_sched(p), p);
 	}
 	scx_task_iter_stop(&sti);
+
+	scx_disable_dump(sch);
+
+	scx_cgroup_lock();
+	set_cgroup_sched(sch_cgroup(sch), NULL);
+	scx_cgroup_unlock();
+
 	percpu_up_write(&scx_fork_rwsem);
 
 	/*
@@ -4395,7 +5823,7 @@ static void scx_disable_workfn(struct kthread_work *work)
 	if (sch->ops.exit)
 		SCX_CALL_OP(sch, SCX_KF_UNLOCKED, exit, NULL, ei);
 
-	cancel_delayed_work_sync(&scx_watchdog_work);
+	scx_unlink_sched(sch);
 
 	/*
 	 * scx_root clearing must be inside cpus_read_lock(). See
@@ -4412,21 +5840,13 @@ static void scx_disable_workfn(struct kthread_work *work)
 	 */
 	kobject_del(&sch->kobj);
 
-	free_percpu(scx_dsp_ctx);
-	scx_dsp_ctx = NULL;
-	scx_dsp_max_batch = 0;
 	free_kick_syncs();
 
-	if (scx_bypassed_for_enable) {
-		scx_bypassed_for_enable = false;
-		scx_bypass(false);
-	}
-
 	mutex_unlock(&scx_enable_mutex);
 
 	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
 done:
-	scx_bypass(false);
+	scx_bypass(sch, false);
 }
 
 /*
@@ -4442,6 +5862,9 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
 
 	lockdep_assert_preemption_disabled();
 
+	if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE))
+		kind = SCX_EXIT_ERROR;
+
 	if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind))
 		return false;
 
@@ -4450,25 +5873,61 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
 	 * flag to break potential live-lock scenarios, ensuring we can
 	 * successfully reach scx_bypass().
 	 */
-	WRITE_ONCE(scx_aborting, true);
+	WRITE_ONCE(sch->aborting, true);
+
+	/*
+	 * Propagate exits to descendants immediately. Each has a dedicated
+	 * helper kthread and can run in parallel. While most of disabling is
+	 * serialized, running them in separate threads allows parallelizing
+	 * ops.exit(), which can take arbitrarily long prolonging bypass mode.
+	 *
+	 * To guarantee forward progress, this propagation must be in-line so
+	 * that ->aborting is synchronously asserted for all sub-scheds. The
+	 * propagation is also the interlocking point against sub-sched
+	 * attachment. See scx_link_sched().
+	 *
+	 * This doesn't cause recursions as propagation only takes place for
+	 * non-propagation exits.
+	 */
+	if (kind != SCX_EXIT_PARENT) {
+		scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) {
+			struct scx_sched *pos;
+			scx_for_each_descendant_pre(pos, sch)
+				scx_disable(pos, SCX_EXIT_PARENT);
+		}
+	}
+
 	return true;
 }
 
-static void scx_disable(enum scx_exit_kind kind)
+static void scx_disable_workfn(struct kthread_work *work)
 {
-	struct scx_sched *sch;
-
-	if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE))
-		kind = SCX_EXIT_ERROR;
+	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
+	struct scx_exit_info *ei = sch->exit_info;
+	int kind;
 
-	rcu_read_lock();
-	sch = rcu_dereference(scx_root);
-	if (sch) {
-		guard(preempt)();
-		scx_claim_exit(sch, kind);
-		kthread_queue_work(sch->helper, &sch->disable_work);
+	kind = atomic_read(&sch->exit_kind);
+	while (true) {
+		if (kind == SCX_EXIT_DONE)	/* already disabled? */
+			return;
+		WARN_ON_ONCE(kind == SCX_EXIT_NONE);
+		if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE))
+			break;
 	}
-	rcu_read_unlock();
+	ei->kind = kind;
+	ei->reason = scx_exit_reason(ei->kind);
+
+	if (scx_parent(sch))
+		scx_sub_disable(sch);
+	else
+		scx_root_disable(sch);
+}
+
+static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)
+{
+	guard(preempt)();
+	if (scx_claim_exit(sch, kind))
+		irq_work_queue(&sch->disable_irq_work);
 }
 
 static void dump_newline(struct seq_buf *s)
@@ -4486,7 +5945,7 @@ static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...)
 
 #ifdef CONFIG_TRACEPOINTS
 	if (trace_sched_ext_dump_enabled()) {
-		/* protected by scx_dump_state()::dump_lock */
+		/* protected by scx_dump_lock */
 		static char line_buf[SCX_EXIT_MSG_LEN];
 
 		va_start(args, fmt);
@@ -4582,25 +6041,38 @@ static void ops_dump_exit(void)
 	scx_dump_data.cpu = -1;
 }
 
-static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx,
+static void scx_dump_task(struct scx_sched *sch,
+			  struct seq_buf *s, struct scx_dump_ctx *dctx,
 			  struct task_struct *p, char marker)
 {
 	static unsigned long bt[SCX_EXIT_BT_LEN];
-	struct scx_sched *sch = scx_root;
+	struct scx_sched *task_sch = scx_task_sched(p);
+	const char *own_marker;
+	char sch_id_buf[32];
 	char dsq_id_buf[19] = "(n/a)";
 	unsigned long ops_state = atomic_long_read(&p->scx.ops_state);
 	unsigned int bt_len = 0;
 
+	own_marker = task_sch == sch ? "*" : "";
+
+	if (task_sch->level == 0)
+		scnprintf(sch_id_buf, sizeof(sch_id_buf), "root");
+	else
+		scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu",
+			  task_sch->level, task_sch->ops.sub_cgroup_id);
+
 	if (p->scx.dsq)
 		scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx",
 			  (unsigned long long)p->scx.dsq->id);
 
 	dump_newline(s);
-	dump_line(s, " %c%c %s[%d] %+ldms",
+	dump_line(s, " %c%c %s[%d] %s%s %+ldms",
 		  marker, task_state_to_char(p), p->comm, p->pid,
+		  own_marker, sch_id_buf,
 		  jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
 	dump_line(s, "      scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
-		  scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK,
+		  scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT,
+		  p->scx.flags & ~SCX_TASK_STATE_MASK,
 		  p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK,
 		  ops_state >> SCX_OPSS_QSEQ_SHIFT);
 	dump_line(s, "      sticky/holding_cpu=%d/%d dsq_id=%s",
@@ -4625,11 +6097,17 @@ static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx,
 	}
 }
 
-static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
+/*
+ * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless
+ * of which scheduler they belong to. If false, only dump tasks owned by @sch.
+ * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped
+ * separately. For error dumps, @dump_all_tasks=true since only the failing
+ * scheduler is dumped.
+ */
+static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
+			   size_t dump_len, bool dump_all_tasks)
 {
-	static DEFINE_SPINLOCK(dump_lock);
 	static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n";
-	struct scx_sched *sch = scx_root;
 	struct scx_dump_ctx dctx = {
 		.kind = ei->kind,
 		.exit_code = ei->exit_code,
@@ -4639,14 +6117,24 @@ static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
 	};
 	struct seq_buf s;
 	struct scx_event_stats events;
-	unsigned long flags;
 	char *buf;
 	int cpu;
 
-	spin_lock_irqsave(&dump_lock, flags);
+	guard(raw_spinlock_irqsave)(&scx_dump_lock);
+
+	if (sch->dump_disabled)
+		return;
 
 	seq_buf_init(&s, ei->dump, dump_len);
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (sch->level == 0)
+		dump_line(&s, "%s: root", sch->ops.name);
+	else
+		dump_line(&s, "%s: sub%d-%llu %s",
+			  sch->ops.name, sch->level, sch->ops.sub_cgroup_id,
+			  sch->cgrp_path);
+#endif
 	if (ei->kind == SCX_EXIT_NONE) {
 		dump_line(&s, "Debug dump triggered by %s", ei->reason);
 	} else {
@@ -4739,11 +6227,13 @@ static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
 				seq_buf_set_overflow(&s);
 		}
 
-		if (rq->curr->sched_class == &ext_sched_class)
-			scx_dump_task(&s, &dctx, rq->curr, '*');
+		if (rq->curr->sched_class == &ext_sched_class &&
+		    (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
+			scx_dump_task(sch, &s, &dctx, rq->curr, '*');
 
 		list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
-			scx_dump_task(&s, &dctx, p, ' ');
+			if (dump_all_tasks || scx_task_on_sched(sch, p))
+				scx_dump_task(sch, &s, &dctx, p, ' ');
 	next:
 		rq_unlock_irqrestore(rq, &rf);
 	}
@@ -4758,25 +6248,27 @@ static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
 	scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST);
 	scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING);
 	scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
+	scx_dump_event(s, &events, SCX_EV_REENQ_IMMED);
+	scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT);
 	scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL);
 	scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION);
 	scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH);
 	scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE);
+	scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED);
+	scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH);
 
 	if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
 		memcpy(ei->dump + dump_len - sizeof(trunc_marker),
 		       trunc_marker, sizeof(trunc_marker));
-
-	spin_unlock_irqrestore(&dump_lock, flags);
 }
 
-static void scx_error_irq_workfn(struct irq_work *irq_work)
+static void scx_disable_irq_workfn(struct irq_work *irq_work)
 {
-	struct scx_sched *sch = container_of(irq_work, struct scx_sched, error_irq_work);
+	struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work);
 	struct scx_exit_info *ei = sch->exit_info;
 
 	if (ei->kind >= SCX_EXIT_ERROR)
-		scx_dump_state(ei, sch->ops.exit_dump_len);
+		scx_dump_state(sch, ei, sch->ops.exit_dump_len, true);
 
 	kthread_queue_work(sch->helper, &sch->disable_work);
 }
@@ -4806,7 +6298,7 @@ static bool scx_vexit(struct scx_sched *sch,
 	ei->kind = kind;
 	ei->reason = scx_exit_reason(ei->kind);
 
-	irq_work_queue(&sch->error_irq_work);
+	irq_work_queue(&sch->disable_irq_work);
 	return true;
 }
 
@@ -4837,14 +6329,47 @@ static int alloc_kick_syncs(void)
 	return 0;
 }
 
-static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
+static void free_pnode(struct scx_sched_pnode *pnode)
+{
+	if (!pnode)
+		return;
+	exit_dsq(&pnode->global_dsq);
+	kfree(pnode);
+}
+
+static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)
+{
+	struct scx_sched_pnode *pnode;
+
+	pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node);
+	if (!pnode)
+		return NULL;
+
+	if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) {
+		kfree(pnode);
+		return NULL;
+	}
+
+	return pnode;
+}
+
+/*
+ * Allocate and initialize a new scx_sched. @cgrp's reference is always
+ * consumed whether the function succeeds or fails.
+ */
+static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
+						 struct cgroup *cgrp,
+						 struct scx_sched *parent)
 {
 	struct scx_sched *sch;
-	int node, ret;
+	s32 level = parent ? parent->level + 1 : 0;
+	s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids;
 
-	sch = kzalloc_obj(*sch);
-	if (!sch)
-		return ERR_PTR(-ENOMEM);
+	sch = kzalloc_flex(*sch, ancestors, level + 1);
+	if (!sch) {
+		ret = -ENOMEM;
+		goto err_put_cgrp;
+	}
 
 	sch->exit_info = alloc_exit_info(ops->exit_dump_len);
 	if (!sch->exit_info) {
@@ -4856,29 +6381,42 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
 	if (ret < 0)
 		goto err_free_ei;
 
-	sch->global_dsqs = kzalloc_objs(sch->global_dsqs[0], nr_node_ids);
-	if (!sch->global_dsqs) {
+	sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids);
+	if (!sch->pnode) {
 		ret = -ENOMEM;
 		goto err_free_hash;
 	}
 
 	for_each_node_state(node, N_POSSIBLE) {
-		struct scx_dispatch_q *dsq;
-
-		dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node);
-		if (!dsq) {
+		sch->pnode[node] = alloc_pnode(sch, node);
+		if (!sch->pnode[node]) {
 			ret = -ENOMEM;
-			goto err_free_gdsqs;
+			goto err_free_pnode;
 		}
-
-		init_dsq(dsq, SCX_DSQ_GLOBAL);
-		sch->global_dsqs[node] = dsq;
 	}
 
-	sch->pcpu = alloc_percpu(struct scx_sched_pcpu);
+	sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
+	sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu,
+						 dsp_ctx.buf, sch->dsp_max_batch),
+				   __alignof__(struct scx_sched_pcpu));
 	if (!sch->pcpu) {
 		ret = -ENOMEM;
-		goto err_free_gdsqs;
+		goto err_free_pnode;
+	}
+
+	for_each_possible_cpu(cpu) {
+		ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
+		if (ret) {
+			bypass_fail_cpu = cpu;
+			goto err_free_pcpu;
+		}
+	}
+
+	for_each_possible_cpu(cpu) {
+		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+		pcpu->sch = sch;
+		INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node);
 	}
 
 	sch->helper = kthread_run_worker(0, "sched_ext_helper");
@@ -4889,33 +6427,94 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
 
 	sched_set_fifo(sch->helper->task);
 
+	if (parent)
+		memcpy(sch->ancestors, parent->ancestors,
+		       level * sizeof(parent->ancestors[0]));
+	sch->ancestors[level] = sch;
+	sch->level = level;
+
+	if (ops->timeout_ms)
+		sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms);
+	else
+		sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT;
+
+	sch->slice_dfl = SCX_SLICE_DFL;
 	atomic_set(&sch->exit_kind, SCX_EXIT_NONE);
-	init_irq_work(&sch->error_irq_work, scx_error_irq_workfn);
+	init_irq_work(&sch->disable_irq_work, scx_disable_irq_workfn);
 	kthread_init_work(&sch->disable_work, scx_disable_workfn);
+	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
 	sch->ops = *ops;
-	ops->priv = sch;
+	rcu_assign_pointer(ops->priv, sch);
+
+	sch->kobj.kset = scx_kset;
+
+#ifdef CONFIG_EXT_SUB_SCHED
+	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto err_stop_helper;
+	}
+	cgroup_path(cgrp, buf, PATH_MAX);
+	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
+	kfree(buf);
+	if (!sch->cgrp_path) {
+		ret = -ENOMEM;
+		goto err_stop_helper;
+	}
+
+	sch->cgrp = cgrp;
+	INIT_LIST_HEAD(&sch->children);
+	INIT_LIST_HEAD(&sch->sibling);
+
+	if (parent)
+		ret = kobject_init_and_add(&sch->kobj, &scx_ktype,
+					   &parent->sub_kset->kobj,
+					   "sub-%llu", cgroup_id(cgrp));
+	else
+		ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
+
+	if (ret < 0) {
+		kobject_put(&sch->kobj);
+		return ERR_PTR(ret);
+	}
 
-	sch->kobj.kset = scx_kset;
+	if (ops->sub_attach) {
+		sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
+		if (!sch->sub_kset) {
+			kobject_put(&sch->kobj);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
+#else	/* CONFIG_EXT_SUB_SCHED */
 	ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
-	if (ret < 0)
-		goto err_stop_helper;
-
+	if (ret < 0) {
+		kobject_put(&sch->kobj);
+		return ERR_PTR(ret);
+	}
+#endif	/* CONFIG_EXT_SUB_SCHED */
 	return sch;
 
 err_stop_helper:
 	kthread_destroy_worker(sch->helper);
 err_free_pcpu:
+	for_each_possible_cpu(cpu) {
+		if (cpu == bypass_fail_cpu)
+			break;
+		exit_dsq(bypass_dsq(sch, cpu));
+	}
 	free_percpu(sch->pcpu);
-err_free_gdsqs:
+err_free_pnode:
 	for_each_node_state(node, N_POSSIBLE)
-		kfree(sch->global_dsqs[node]);
-	kfree(sch->global_dsqs);
+		free_pnode(sch->pnode[node]);
+	kfree(sch->pnode);
 err_free_hash:
 	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
 err_free_ei:
 	free_exit_info(sch->exit_info);
 err_free_sch:
 	kfree(sch);
+err_put_cgrp:
+	cgroup_put(cgrp);
 	return ERR_PTR(ret);
 }
 
@@ -4964,9 +6563,6 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 		return -EINVAL;
 	}
 
-	if (ops->flags & SCX_OPS_HAS_CGROUP_WEIGHT)
-		pr_warn("SCX_OPS_HAS_CGROUP_WEIGHT is deprecated and a noop\n");
-
 	if (ops->cpu_acquire || ops->cpu_release)
 		pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
 
@@ -4986,15 +6582,14 @@ struct scx_enable_cmd {
 	int			ret;
 };
 
-static void scx_enable_workfn(struct kthread_work *work)
+static void scx_root_enable_workfn(struct kthread_work *work)
 {
-	struct scx_enable_cmd *cmd =
-		container_of(work, struct scx_enable_cmd, work);
+	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
 	struct sched_ext_ops *ops = cmd->ops;
+	struct cgroup *cgrp = root_cgroup();
 	struct scx_sched *sch;
 	struct scx_task_iter sti;
 	struct task_struct *p;
-	unsigned long timeout;
 	int i, cpu, ret;
 
 	mutex_lock(&scx_enable_mutex);
@@ -5008,7 +6603,8 @@ static void scx_enable_workfn(struct kthread_work *work)
 	if (ret)
 		goto err_unlock;
 
-	sch = scx_alloc_and_add_sched(ops);
+	cgroup_get(cgrp);
+	sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
 	if (IS_ERR(sch)) {
 		ret = PTR_ERR(sch);
 		goto err_free_ksyncs;
@@ -5020,13 +6616,15 @@ static void scx_enable_workfn(struct kthread_work *work)
 	 */
 	WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
 	WARN_ON_ONCE(scx_root);
-	if (WARN_ON_ONCE(READ_ONCE(scx_aborting)))
-		WRITE_ONCE(scx_aborting, false);
 
 	atomic_long_set(&scx_nr_rejected, 0);
 
-	for_each_possible_cpu(cpu)
-		cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE;
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		rq->scx.local_dsq.sched = sch;
+		rq->scx.cpuperf_target = SCX_CPUPERF_ONE;
+	}
 
 	/*
 	 * Keep CPUs stable during enable so that the BPF scheduler can track
@@ -5040,6 +6638,10 @@ static void scx_enable_workfn(struct kthread_work *work)
 	 */
 	rcu_assign_pointer(scx_root, sch);
 
+	ret = scx_link_sched(sch);
+	if (ret)
+		goto err_disable;
+
 	scx_idle_enable(ops);
 
 	if (sch->ops.init) {
@@ -5070,34 +6672,13 @@ static void scx_enable_workfn(struct kthread_work *work)
 	if (ret)
 		goto err_disable;
 
-	WARN_ON_ONCE(scx_dsp_ctx);
-	scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
-	scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf,
-						   scx_dsp_max_batch),
-				     __alignof__(struct scx_dsp_ctx));
-	if (!scx_dsp_ctx) {
-		ret = -ENOMEM;
-		goto err_disable;
-	}
-
-	if (ops->timeout_ms)
-		timeout = msecs_to_jiffies(ops->timeout_ms);
-	else
-		timeout = SCX_WATCHDOG_MAX_TIMEOUT;
-
-	WRITE_ONCE(scx_watchdog_timeout, timeout);
-	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
-	queue_delayed_work(system_unbound_wq, &scx_watchdog_work,
-			   READ_ONCE(scx_watchdog_timeout) / 2);
-
 	/*
 	 * Once __scx_enabled is set, %current can be switched to SCX anytime.
 	 * This can lead to stalls as some BPF schedulers (e.g. userspace
 	 * scheduling) may not function correctly before all tasks are switched.
 	 * Init in bypass mode to guarantee forward progress.
 	 */
-	scx_bypass(true);
-	scx_bypassed_for_enable = true;
+	scx_bypass(sch, true);
 
 	for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
 		if (((void (**)(void))ops)[i])
@@ -5129,11 +6710,12 @@ static void scx_enable_workfn(struct kthread_work *work)
 	 * never sees uninitialized tasks.
 	 */
 	scx_cgroup_lock();
+	set_cgroup_sched(sch_cgroup(sch), sch);
 	ret = scx_cgroup_init(sch);
 	if (ret)
 		goto err_disable_unlock_all;
 
-	scx_task_iter_start(&sti);
+	scx_task_iter_start(&sti, NULL);
 	while ((p = scx_task_iter_next_locked(&sti))) {
 		/*
 		 * @p may already be dead, have lost all its usages counts and
@@ -5145,7 +6727,7 @@ static void scx_enable_workfn(struct kthread_work *work)
 
 		scx_task_iter_unlock(&sti);
 
-		ret = scx_init_task(p, task_group(p), false);
+		ret = scx_init_task(sch, p, false);
 		if (ret) {
 			put_task_struct(p);
 			scx_task_iter_stop(&sti);
@@ -5154,6 +6736,7 @@ static void scx_enable_workfn(struct kthread_work *work)
 			goto err_disable_unlock_all;
 		}
 
+		scx_set_task_sched(p, sch);
 		scx_set_task_state(p, SCX_TASK_READY);
 
 		put_task_struct(p);
@@ -5175,7 +6758,7 @@ static void scx_enable_workfn(struct kthread_work *work)
 	 * scx_tasks_lock.
 	 */
 	percpu_down_write(&scx_fork_rwsem);
-	scx_task_iter_start(&sti);
+	scx_task_iter_start(&sti, NULL);
 	while ((p = scx_task_iter_next_locked(&sti))) {
 		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
 		const struct sched_class *old_class = p->sched_class;
@@ -5188,15 +6771,14 @@ static void scx_enable_workfn(struct kthread_work *work)
 			queue_flags |= DEQUEUE_CLASS;
 
 		scoped_guard (sched_change, p, queue_flags) {
-			p->scx.slice = READ_ONCE(scx_slice_dfl);
+			p->scx.slice = READ_ONCE(sch->slice_dfl);
 			p->sched_class = new_class;
 		}
 	}
 	scx_task_iter_stop(&sti);
 	percpu_up_write(&scx_fork_rwsem);
 
-	scx_bypassed_for_enable = false;
-	scx_bypass(false);
+	scx_bypass(sch, false);
 
 	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
 		WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE);
@@ -5238,12 +6820,318 @@ static void scx_enable_workfn(struct kthread_work *work)
 	 * Flush scx_disable_work to ensure that error is reported before init
 	 * completion. sch's base reference will be put by bpf_scx_unreg().
 	 */
-	scx_error(sch, "scx_enable() failed (%d)", ret);
+	scx_error(sch, "scx_root_enable() failed (%d)", ret);
+	kthread_flush_work(&sch->disable_work);
+	cmd->ret = 0;
+}
+
+#ifdef CONFIG_EXT_SUB_SCHED
+/* verify that a scheduler can be attached to @cgrp and return the parent */
+static struct scx_sched *find_parent_sched(struct cgroup *cgrp)
+{
+	struct scx_sched *parent = cgrp->scx_sched;
+	struct scx_sched *pos;
+
+	lockdep_assert_held(&scx_sched_lock);
+
+	/* can't attach twice to the same cgroup */
+	if (parent->cgrp == cgrp)
+		return ERR_PTR(-EBUSY);
+
+	/* does $parent allow sub-scheds? */
+	if (!parent->ops.sub_attach)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	/* can't insert between $parent and its exiting children */
+	list_for_each_entry(pos, &parent->children, sibling)
+		if (cgroup_is_descendant(pos->cgrp, cgrp))
+			return ERR_PTR(-EBUSY);
+
+	return parent;
+}
+
+static bool assert_task_ready_or_enabled(struct task_struct *p)
+{
+	u32 state = scx_get_task_state(p);
+
+	switch (state) {
+	case SCX_TASK_READY:
+	case SCX_TASK_ENABLED:
+		return true;
+	default:
+		WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched",
+			  state, p->comm, p->pid);
+		return false;
+	}
+}
+
+static void scx_sub_enable_workfn(struct kthread_work *work)
+{
+	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
+	struct sched_ext_ops *ops = cmd->ops;
+	struct cgroup *cgrp;
+	struct scx_sched *parent, *sch;
+	struct scx_task_iter sti;
+	struct task_struct *p;
+	s32 i, ret;
+
+	mutex_lock(&scx_enable_mutex);
+
+	if (!scx_enabled()) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	cgrp = cgroup_get_from_id(ops->sub_cgroup_id);
+	if (IS_ERR(cgrp)) {
+		ret = PTR_ERR(cgrp);
+		goto out_unlock;
+	}
+
+	raw_spin_lock_irq(&scx_sched_lock);
+	parent = find_parent_sched(cgrp);
+	if (IS_ERR(parent)) {
+		raw_spin_unlock_irq(&scx_sched_lock);
+		ret = PTR_ERR(parent);
+		goto out_put_cgrp;
+	}
+	kobject_get(&parent->kobj);
+	raw_spin_unlock_irq(&scx_sched_lock);
+
+	/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
+	sch = scx_alloc_and_add_sched(ops, cgrp, parent);
+	kobject_put(&parent->kobj);
+	if (IS_ERR(sch)) {
+		ret = PTR_ERR(sch);
+		goto out_unlock;
+	}
+
+	ret = scx_link_sched(sch);
+	if (ret)
+		goto err_disable;
+
+	if (sch->level >= SCX_SUB_MAX_DEPTH) {
+		scx_error(sch, "max nesting depth %d violated",
+			  SCX_SUB_MAX_DEPTH);
+		goto err_disable;
+	}
+
+	if (sch->ops.init) {
+		ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init, NULL);
+		if (ret) {
+			ret = ops_sanitize_err(sch, "init", ret);
+			scx_error(sch, "ops.init() failed (%d)", ret);
+			goto err_disable;
+		}
+		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
+	}
+
+	if (validate_ops(sch, ops))
+		goto err_disable;
+
+	struct scx_sub_attach_args sub_attach_args = {
+		.ops = &sch->ops,
+		.cgroup_path = sch->cgrp_path,
+	};
+
+	ret = SCX_CALL_OP_RET(parent, SCX_KF_UNLOCKED, sub_attach, NULL,
+			      &sub_attach_args);
+	if (ret) {
+		ret = ops_sanitize_err(sch, "sub_attach", ret);
+		scx_error(sch, "parent rejected (%d)", ret);
+		goto err_disable;
+	}
+	sch->sub_attached = true;
+
+	scx_bypass(sch, true);
+
+	for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
+		if (((void (**)(void))ops)[i])
+			set_bit(i, sch->has_op);
+
+	percpu_down_write(&scx_fork_rwsem);
+	scx_cgroup_lock();
+
+	/*
+	 * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see
+	 * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down.
+	 */
+	set_cgroup_sched(sch_cgroup(sch), sch);
+	if (!(cgrp->self.flags & CSS_ONLINE)) {
+		scx_error(sch, "cgroup is not online");
+		goto err_unlock_and_disable;
+	}
+
+	/*
+	 * Initialize tasks for the new child $sch without exiting them for
+	 * $parent so that the tasks can always be reverted back to $parent
+	 * sched on child init failure.
+	 */
+	WARN_ON_ONCE(scx_enabling_sub_sched);
+	scx_enabling_sub_sched = sch;
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		struct rq *rq;
+		struct rq_flags rf;
+
+		/*
+		 * Task iteration may visit the same task twice when racing
+		 * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which
+		 * finished __scx_init_task() and skip if set.
+		 *
+		 * A task may exit and get freed between __scx_init_task()
+		 * completion and scx_enable_task(). In such cases,
+		 * scx_disable_and_exit_task() must exit the task for both the
+		 * parent and child scheds.
+		 */
+		if (p->scx.flags & SCX_TASK_SUB_INIT)
+			continue;
+
+		/* see scx_root_enable() */
+		if (!tryget_task_struct(p))
+			continue;
+
+		if (!assert_task_ready_or_enabled(p)) {
+			ret = -EINVAL;
+			goto abort;
+		}
+
+		scx_task_iter_unlock(&sti);
+
+		/*
+		 * As $p is still on $parent, it can't be transitioned to INIT.
+		 * Let's worry about task state later. Use __scx_init_task().
+		 */
+		ret = __scx_init_task(sch, p, false);
+		if (ret)
+			goto abort;
+
+		rq = task_rq_lock(p, &rf);
+		p->scx.flags |= SCX_TASK_SUB_INIT;
+		task_rq_unlock(rq, p, &rf);
+
+		put_task_struct(p);
+	}
+	scx_task_iter_stop(&sti);
+
+	/*
+	 * All tasks are prepped. Disable/exit tasks for $parent and enable for
+	 * the new @sch.
+	 */
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		/*
+		 * Use clearing of %SCX_TASK_SUB_INIT to detect and skip
+		 * duplicate iterations.
+		 */
+		if (!(p->scx.flags & SCX_TASK_SUB_INIT))
+			continue;
+
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			/*
+			 * $p must be either READY or ENABLED. If ENABLED,
+			 * __scx_disabled_and_exit_task() first disables and
+			 * makes it READY. However, after exiting $p, it will
+			 * leave $p as READY.
+			 */
+			assert_task_ready_or_enabled(p);
+			__scx_disable_and_exit_task(parent, p);
+
+			/*
+			 * $p is now only initialized for @sch and READY, which
+			 * is what we want. Assign it to @sch and enable.
+			 */
+			rcu_assign_pointer(p->scx.sched, sch);
+			scx_enable_task(sch, p);
+
+			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+		}
+	}
+	scx_task_iter_stop(&sti);
+
+	scx_enabling_sub_sched = NULL;
+
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+
+	scx_bypass(sch, false);
+
+	pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name);
+	kobject_uevent(&sch->kobj, KOBJ_ADD);
+	ret = 0;
+	goto out_unlock;
+
+out_put_cgrp:
+	cgroup_put(cgrp);
+out_unlock:
+	mutex_unlock(&scx_enable_mutex);
+	cmd->ret = ret;
+	return;
+
+abort:
+	put_task_struct(p);
+	scx_task_iter_stop(&sti);
+	scx_enabling_sub_sched = NULL;
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		if (p->scx.flags & SCX_TASK_SUB_INIT) {
+			__scx_disable_and_exit_task(sch, p);
+			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+		}
+	}
+	scx_task_iter_stop(&sti);
+err_unlock_and_disable:
+	/* we'll soon enter disable path, keep bypass on */
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+err_disable:
+	mutex_unlock(&scx_enable_mutex);
 	kthread_flush_work(&sch->disable_work);
 	cmd->ret = 0;
 }
 
-static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
+static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb,
+				      unsigned long action, void *data)
+{
+	struct cgroup *cgrp = data;
+	struct cgroup *parent = cgroup_parent(cgrp);
+
+	if (!cgroup_on_dfl(cgrp))
+		return NOTIFY_OK;
+
+	switch (action) {
+	case CGROUP_LIFETIME_ONLINE:
+		/* inherit ->scx_sched from $parent */
+		if (parent)
+			rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched);
+		break;
+	case CGROUP_LIFETIME_OFFLINE:
+		/* if there is a sched attached, shoot it down */
+		if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp)
+			scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN,
+				 SCX_ECODE_RSN_CGROUP_OFFLINE,
+				 "cgroup %llu going offline", cgroup_id(cgrp));
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block scx_cgroup_lifetime_nb = {
+	.notifier_call = scx_cgroup_lifetime_notify,
+};
+
+static s32 __init scx_cgroup_lifetime_notifier_init(void)
+{
+	return blocking_notifier_chain_register(&cgroup_lifetime_notifier,
+						&scx_cgroup_lifetime_nb);
+}
+core_initcall(scx_cgroup_lifetime_notifier_init);
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 {
 	static struct kthread_worker *helper;
 	static DEFINE_MUTEX(helper_mutex);
@@ -5270,7 +7158,12 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 		mutex_unlock(&helper_mutex);
 	}
 
-	kthread_init_work(&cmd.work, scx_enable_workfn);
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (ops->sub_cgroup_id > 1)
+		kthread_init_work(&cmd.work, scx_sub_enable_workfn);
+	else
+#endif	/* CONFIG_EXT_SUB_SCHED */
+		kthread_init_work(&cmd.work, scx_root_enable_workfn);
 	cmd.ops = ops;
 
 	kthread_queue_work(READ_ONCE(helper), &cmd.work);
@@ -5311,12 +7204,17 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
 
 	t = btf_type_by_id(reg->btf, reg->btf_id);
 	if (t == task_struct_type) {
-		if (off >= offsetof(struct task_struct, scx.slice) &&
-		    off + size <= offsetofend(struct task_struct, scx.slice))
-			return SCALAR_VALUE;
-		if (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
-		    off + size <= offsetofend(struct task_struct, scx.dsq_vtime))
+		/*
+		 * COMPAT: Will be removed in v6.23.
+		 */
+		if ((off >= offsetof(struct task_struct, scx.slice) &&
+		     off + size <= offsetofend(struct task_struct, scx.slice)) ||
+		    (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
+		     off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) {
+			pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()");
 			return SCALAR_VALUE;
+		}
+
 		if (off >= offsetof(struct task_struct, scx.disallow) &&
 		    off + size <= offsetofend(struct task_struct, scx.disallow))
 			return SCALAR_VALUE;
@@ -5372,11 +7270,30 @@ static int bpf_scx_init_member(const struct btf_type *t,
 	case offsetof(struct sched_ext_ops, hotplug_seq):
 		ops->hotplug_seq = *(u64 *)(udata + moff);
 		return 1;
+#ifdef CONFIG_EXT_SUB_SCHED
+	case offsetof(struct sched_ext_ops, sub_cgroup_id):
+		ops->sub_cgroup_id = *(u64 *)(udata + moff);
+		return 1;
+#endif	/* CONFIG_EXT_SUB_SCHED */
 	}
 
 	return 0;
 }
 
+#ifdef CONFIG_EXT_SUB_SCHED
+static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog)
+{
+	struct scx_sched *sch;
+
+	guard(rcu)();
+	sch = scx_prog_sched(prog->aux);
+	if (unlikely(!sch))
+		return;
+
+	scx_error(sch, "dispatch recursion detected");
+}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
 static int bpf_scx_check_member(const struct btf_type *t,
 				const struct btf_member *member,
 				const struct bpf_prog *prog)
@@ -5394,12 +7311,30 @@ static int bpf_scx_check_member(const struct btf_type *t,
 	case offsetof(struct sched_ext_ops, cpu_offline):
 	case offsetof(struct sched_ext_ops, init):
 	case offsetof(struct sched_ext_ops, exit):
+	case offsetof(struct sched_ext_ops, sub_attach):
+	case offsetof(struct sched_ext_ops, sub_detach):
 		break;
 	default:
 		if (prog->sleepable)
 			return -EINVAL;
 	}
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * Enable private stack for operations that can nest along the
+	 * hierarchy.
+	 *
+	 * XXX - Ideally, we should only do this for scheds that allow
+	 * sub-scheds and sub-scheds themselves but I don't know how to access
+	 * struct_ops from here.
+	 */
+	switch (moff) {
+	case offsetof(struct sched_ext_ops, dispatch):
+		prog->aux->priv_stack_requested = true;
+		prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch;
+	}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
 	return 0;
 }
 
@@ -5411,10 +7346,11 @@ static int bpf_scx_reg(void *kdata, struct bpf_link *link)
 static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
 {
 	struct sched_ext_ops *ops = kdata;
-	struct scx_sched *sch = ops->priv;
+	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
 
-	scx_disable(SCX_EXIT_UNREG);
+	scx_disable(sch, SCX_EXIT_UNREG);
 	kthread_flush_work(&sch->disable_work);
+	RCU_INIT_POINTER(ops->priv, NULL);
 	kobject_put(&sch->kobj);
 }
 
@@ -5471,7 +7407,9 @@ static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgro
 static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {}
 static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {}
 static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {}
-#endif
+#endif	/* CONFIG_EXT_GROUP_SCHED */
+static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; }
+static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {}
 static void sched_ext_ops__cpu_online(s32 cpu) {}
 static void sched_ext_ops__cpu_offline(s32 cpu) {}
 static s32 sched_ext_ops__init(void) { return -EINVAL; }
@@ -5511,6 +7449,8 @@ static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
 	.cgroup_set_bandwidth	= sched_ext_ops__cgroup_set_bandwidth,
 	.cgroup_set_idle	= sched_ext_ops__cgroup_set_idle,
 #endif
+	.sub_attach		= sched_ext_ops__sub_attach,
+	.sub_detach		= sched_ext_ops__sub_detach,
 	.cpu_online		= sched_ext_ops__cpu_online,
 	.cpu_offline		= sched_ext_ops__cpu_offline,
 	.init			= sched_ext_ops__init,
@@ -5541,7 +7481,15 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
 
 static void sysrq_handle_sched_ext_reset(u8 key)
 {
-	scx_disable(SCX_EXIT_SYSRQ);
+	struct scx_sched *sch;
+
+	rcu_read_lock();
+	sch = rcu_dereference(scx_root);
+	if (likely(sch))
+		scx_disable(sch, SCX_EXIT_SYSRQ);
+	else
+		pr_info("sched_ext: BPF schedulers not loaded\n");
+	rcu_read_unlock();
 }
 
 static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
@@ -5554,9 +7502,10 @@ static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
 static void sysrq_handle_sched_ext_dump(u8 key)
 {
 	struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" };
+	struct scx_sched *sch;
 
-	if (scx_enabled())
-		scx_dump_state(&ei, 0);
+	list_for_each_entry_rcu(sch, &scx_sched_all, all)
+		scx_dump_state(sch, &ei, 0, false);
 }
 
 static const struct sysrq_key_op sysrq_sched_ext_dump_op = {
@@ -5741,6 +7690,14 @@ void print_scx_info(const char *log_lvl, struct task_struct *p)
 
 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr)
 {
+	struct scx_sched *sch;
+
+	guard(rcu)();
+
+	sch = rcu_dereference(scx_root);
+	if (!sch)
+		return NOTIFY_OK;
+
 	/*
 	 * SCX schedulers often have userspace components which are sometimes
 	 * involved in critial scheduling paths. PM operations involve freezing
@@ -5751,12 +7708,12 @@ static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *
 	case PM_HIBERNATION_PREPARE:
 	case PM_SUSPEND_PREPARE:
 	case PM_RESTORE_PREPARE:
-		scx_bypass(true);
+		scx_bypass(sch, true);
 		break;
 	case PM_POST_HIBERNATION:
 	case PM_POST_SUSPEND:
 	case PM_POST_RESTORE:
-		scx_bypass(false);
+		scx_bypass(sch, false);
 		break;
 	}
 
@@ -5785,8 +7742,9 @@ void __init init_sched_ext_class(void)
 		struct rq *rq = cpu_rq(cpu);
 		int  n = cpu_to_node(cpu);
 
-		init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL);
-		init_dsq(&rq->scx.bypass_dsq, SCX_DSQ_BYPASS);
+		/* local_dsq's sch will be set during scx_root_enable() */
+		BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
+
 		INIT_LIST_HEAD(&rq->scx.runnable_list);
 		INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
 
@@ -5794,6 +7752,9 @@ void __init init_sched_ext_class(void)
 		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n));
 		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n));
 		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n));
+		raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
+		INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
+		INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
 		rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
 		rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
 
@@ -5804,14 +7765,35 @@ void __init init_sched_ext_class(void)
 	register_sysrq_key('S', &sysrq_sched_ext_reset_op);
 	register_sysrq_key('D', &sysrq_sched_ext_dump_op);
 	INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+	BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params));
+#endif	/* CONFIG_EXT_SUB_SCHED */
 }
 
 
 /********************************************************************************
  * Helpers that can be called from the BPF scheduler.
  */
+static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
+{
+	bool is_local = dsq_id == SCX_DSQ_LOCAL ||
+		(dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
+
+	if (*enq_flags & SCX_ENQ_IMMED) {
+		if (unlikely(!is_local)) {
+			scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);
+			return false;
+		}
+	} else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) {
+		*enq_flags |= SCX_ENQ_IMMED;
+	}
+
+	return true;
+}
+
 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p,
-				    u64 enq_flags)
+				    u64 dsq_id, u64 *enq_flags)
 {
 	if (!scx_kf_allowed(sch, SCX_KF_ENQUEUE | SCX_KF_DISPATCH))
 		return false;
@@ -5823,18 +7805,27 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p
 		return false;
 	}
 
-	if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
-		scx_error(sch, "invalid enq_flags 0x%llx", enq_flags);
+	if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
+		scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
 		return false;
 	}
 
+	/* see SCX_EV_INSERT_NOT_OWNED definition */
+	if (unlikely(!scx_task_on_sched(sch, p))) {
+		__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
+		return false;
+	}
+
+	if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))
+		return false;
+
 	return true;
 }
 
 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
 				  u64 dsq_id, u64 enq_flags)
 {
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
+	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
 	struct task_struct *ddsp_task;
 
 	ddsp_task = __this_cpu_read(direct_dispatch_task);
@@ -5843,7 +7834,7 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
 		return;
 	}
 
-	if (unlikely(dspc->cursor >= scx_dsp_max_batch)) {
+	if (unlikely(dspc->cursor >= sch->dsp_max_batch)) {
 		scx_error(sch, "dispatch buffer overflow");
 		return;
 	}
@@ -5864,6 +7855,7 @@ __bpf_kfunc_start_defs();
  * @dsq_id: DSQ to insert into
  * @slice: duration @p can run for in nsecs, 0 to keep the current value
  * @enq_flags: SCX_ENQ_*
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to
  * call this function spuriously. Can be called from ops.enqueue(),
@@ -5898,16 +7890,17 @@ __bpf_kfunc_start_defs();
  * to check the return value.
  */
 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
-					 u64 slice, u64 enq_flags)
+					 u64 slice, u64 enq_flags,
+					 const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return false;
 
-	if (!scx_dsq_insert_preamble(sch, p, enq_flags))
+	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
 		return false;
 
 	if (slice)
@@ -5924,15 +7917,16 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
  * COMPAT: Will be removed in v6.23 along with the ___v2 suffix.
  */
 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id,
-					     u64 slice, u64 enq_flags)
+				    u64 slice, u64 enq_flags,
+				    const struct bpf_prog_aux *aux)
 {
-	scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags);
+	scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
 }
 
 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
 				 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
 {
-	if (!scx_dsq_insert_preamble(sch, p, enq_flags))
+	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
 		return false;
 
 	if (slice)
@@ -5963,6 +7957,7 @@ struct scx_bpf_dsq_insert_vtime_args {
  *       @args->slice: duration @p can run for in nsecs, 0 to keep the current value
  *       @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
  *       @args->enq_flags: SCX_ENQ_*
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument
  * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided
@@ -5987,13 +7982,14 @@ struct scx_bpf_dsq_insert_vtime_args {
  */
 __bpf_kfunc bool
 __scx_bpf_dsq_insert_vtime(struct task_struct *p,
-			   struct scx_bpf_dsq_insert_vtime_args *args)
+			   struct scx_bpf_dsq_insert_vtime_args *args,
+			   const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return false;
 
@@ -6015,15 +8011,26 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
 	if (unlikely(!sch))
 		return;
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * Disallow if any sub-scheds are attached. There is no way to tell
+	 * which scheduler called us, just error out @p's scheduler.
+	 */
+	if (unlikely(!list_empty(&sch->children))) {
+		scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used");
+		return;
+	}
+#endif
+
 	scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
 }
 
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)
-BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_RCU)
-BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)
 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
 
@@ -6035,8 +8042,8 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 			 struct task_struct *p, u64 dsq_id, u64 enq_flags)
 {
-	struct scx_sched *sch = scx_root;
 	struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
+	struct scx_sched *sch = src_dsq->sched;
 	struct rq *this_rq, *src_rq, *locked_rq;
 	bool dispatched = false;
 	bool in_balance;
@@ -6046,13 +8053,21 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 	    !scx_kf_allowed(sch, SCX_KF_DISPATCH))
 		return false;
 
+	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
+		return false;
+
 	/*
 	 * If the BPF scheduler keeps calling this function repeatedly, it can
 	 * cause similar live-lock conditions as consume_dispatch_q().
 	 */
-	if (unlikely(READ_ONCE(scx_aborting)))
+	if (unlikely(READ_ONCE(sch->aborting)))
 		return false;
 
+	if (unlikely(!scx_task_on_sched(sch, p))) {
+		scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler",
+			  p->comm, p->pid);
+	}
+
 	/*
 	 * Can be called from either ops.dispatch() locking this_rq() or any
 	 * context where no rq lock is held. If latter, lock @p's task_rq which
@@ -6076,20 +8091,14 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 	locked_rq = src_rq;
 	raw_spin_lock(&src_dsq->lock);
 
-	/*
-	 * Did someone else get to it? @p could have already left $src_dsq, got
-	 * re-enqueud, or be in the process of being consumed by someone else.
-	 */
-	if (unlikely(p->scx.dsq != src_dsq ||
-		     u32_before(kit->cursor.priv, p->scx.dsq_seq) ||
-		     p->scx.holding_cpu >= 0) ||
-	    WARN_ON_ONCE(src_rq != task_rq(p))) {
+	/* did someone else get to it while we dropped the locks? */
+	if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) {
 		raw_spin_unlock(&src_dsq->lock);
 		goto out;
 	}
 
 	/* @p is still on $src_dsq and stable, determine the destination */
-	dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, p);
+	dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p));
 
 	/*
 	 * Apply vtime and slice updates before moving so that the new time is
@@ -6123,45 +8132,49 @@ __bpf_kfunc_start_defs();
 
 /**
  * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Can only be called from ops.dispatch().
  */
-__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void)
+__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return 0;
 
 	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))
 		return 0;
 
-	return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor);
+	return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);
 }
 
 /**
  * scx_bpf_dispatch_cancel - Cancel the latest dispatch
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Cancel the latest dispatch. Can be called multiple times to cancel further
  * dispatches. Can only be called from ops.dispatch().
  */
-__bpf_kfunc void scx_bpf_dispatch_cancel(void)
+__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux)
 {
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
 	struct scx_sched *sch;
+	struct scx_dsp_ctx *dspc;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return;
 
 	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))
 		return;
 
+	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
+
 	if (dspc->cursor > 0)
 		dspc->cursor--;
 	else
@@ -6171,9 +8184,12 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(void)
 /**
  * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ
  * @dsq_id: DSQ to move task from
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ * @enq_flags: %SCX_ENQ_*
  *
  * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's
- * local DSQ for execution. Can only be called from ops.dispatch().
+ * local DSQ for execution with @enq_flags applied. Can only be called from
+ * ops.dispatch().
  *
  * This function flushes the in-flight dispatches from scx_bpf_dsq_insert()
  * before trying to move from the specified DSQ. It may also grab rq locks and
@@ -6182,21 +8198,27 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(void)
  * Returns %true if a task has been moved, %false if there isn't any task to
  * move.
  */
-__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id)
+__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
+						const struct bpf_prog_aux *aux)
 {
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
 	struct scx_dispatch_q *dsq;
 	struct scx_sched *sch;
+	struct scx_dsp_ctx *dspc;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return false;
 
 	if (!scx_kf_allowed(sch, SCX_KF_DISPATCH))
 		return false;
 
+	if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags))
+		return false;
+
+	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
+
 	flush_dispatch_buf(sch, dspc->rq);
 
 	dsq = find_user_dsq(sch, dsq_id);
@@ -6205,7 +8227,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id)
 		return false;
 	}
 
-	if (consume_dispatch_q(sch, dspc->rq, dsq)) {
+	if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
 		/*
 		 * A successfully consumed task can be dequeued before it starts
 		 * running while the CPU is trying to migrate other dispatched
@@ -6219,6 +8241,14 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id)
 	}
 }
 
+/*
+ * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future.
+ */
+__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)
+{
+	return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);
+}
+
 /**
  * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs
  * @it__iter: DSQ iterator in progress
@@ -6314,16 +8344,62 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
 			    p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
 }
 
+#ifdef CONFIG_EXT_SUB_SCHED
+/**
+ * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler
+ * @cgroup_id: cgroup ID of the child scheduler to dispatch
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Allows a parent scheduler to trigger dispatching on one of its direct
+ * child schedulers. The child scheduler runs its dispatch operation to
+ * move tasks from dispatch queues to the local runqueue.
+ *
+ * Returns: true on success, false if cgroup_id is invalid, not a direct
+ * child, or caller lacks dispatch permission.
+ */
+__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux)
+{
+	struct rq *this_rq = this_rq();
+	struct scx_sched *parent, *child;
+
+	guard(rcu)();
+	parent = scx_prog_sched(aux);
+	if (unlikely(!parent))
+		return false;
+
+	if (!scx_kf_allowed(parent, SCX_KF_DISPATCH))
+		return false;
+
+	child = scx_find_sub_sched(cgroup_id);
+
+	if (unlikely(!child))
+		return false;
+
+	if (unlikely(scx_parent(child) != parent)) {
+		scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu",
+			  cgroup_id);
+		return false;
+	}
+
+	return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev,
+				  true);
+}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
-BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots)
-BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel)
-BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local)
+BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
+#ifdef CONFIG_EXT_SUB_SCHED
+BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS)
+#endif
 BTF_KFUNCS_END(scx_kfunc_ids_dispatch)
 
 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
@@ -6331,67 +8407,23 @@ static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
 	.set			= &scx_kfunc_ids_dispatch,
 };
 
-static u32 reenq_local(struct rq *rq)
-{
-	LIST_HEAD(tasks);
-	u32 nr_enqueued = 0;
-	struct task_struct *p, *n;
-
-	lockdep_assert_rq_held(rq);
-
-	/*
-	 * The BPF scheduler may choose to dispatch tasks back to
-	 * @rq->scx.local_dsq. Move all candidate tasks off to a private list
-	 * first to avoid processing the same tasks repeatedly.
-	 */
-	list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
-				 scx.dsq_list.node) {
-		/*
-		 * If @p is being migrated, @p's current CPU may not agree with
-		 * its allowed CPUs and the migration_cpu_stop is about to
-		 * deactivate and re-activate @p anyway. Skip re-enqueueing.
-		 *
-		 * While racing sched property changes may also dequeue and
-		 * re-enqueue a migrating task while its current CPU and allowed
-		 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
-		 * the current local DSQ for running tasks and thus are not
-		 * visible to the BPF scheduler.
-		 */
-		if (p->migration_pending)
-			continue;
-
-		dispatch_dequeue(rq, p);
-		list_add_tail(&p->scx.dsq_list.node, &tasks);
-	}
-
-	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
-		list_del_init(&p->scx.dsq_list.node);
-		do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
-		nr_enqueued++;
-	}
-
-	return nr_enqueued;
-}
-
 __bpf_kfunc_start_defs();
 
 /**
  * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Iterate over all of the tasks currently enqueued on the local DSQ of the
  * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
  * processed tasks. Can only be called from ops.cpu_release().
- *
- * COMPAT: Will be removed in v6.23 along with the ___v2 suffix on the void
- * returning variant that can be called from anywhere.
  */
-__bpf_kfunc u32 scx_bpf_reenqueue_local(void)
+__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	struct rq *rq;
 
 	guard(rcu)();
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return 0;
 
@@ -6401,13 +8433,13 @@ __bpf_kfunc u32 scx_bpf_reenqueue_local(void)
 	rq = cpu_rq(smp_processor_id());
 	lockdep_assert_rq_held(rq);
 
-	return reenq_local(rq);
+	return reenq_local(sch, rq, SCX_REENQ_ANY);
 }
 
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)
-BTF_ID_FLAGS(func, scx_bpf_reenqueue_local)
+BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS)
 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
 
 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
@@ -6421,11 +8453,12 @@ __bpf_kfunc_start_defs();
  * scx_bpf_create_dsq - Create a custom DSQ
  * @dsq_id: DSQ to create
  * @node: NUMA node to allocate from
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable
  * scx callback, and any BPF_PROG_TYPE_SYSCALL prog.
  */
-__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node)
+__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux)
 {
 	struct scx_dispatch_q *dsq;
 	struct scx_sched *sch;
@@ -6442,27 +8475,39 @@ __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node)
 	if (!dsq)
 		return -ENOMEM;
 
-	init_dsq(dsq, dsq_id);
+	/*
+	 * init_dsq() must be called in GFP_KERNEL context. Init it with NULL
+	 * @sch and update afterwards.
+	 */
+	ret = init_dsq(dsq, dsq_id, NULL);
+	if (ret) {
+		kfree(dsq);
+		return ret;
+	}
 
 	rcu_read_lock();
 
-	sch = rcu_dereference(scx_root);
-	if (sch)
+	sch = scx_prog_sched(aux);
+	if (sch) {
+		dsq->sched = sch;
 		ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node,
 						    dsq_hash_params);
-	else
+	} else {
 		ret = -ENODEV;
+	}
 
 	rcu_read_unlock();
-	if (ret)
+	if (ret) {
+		exit_dsq(dsq);
 		kfree(dsq);
+	}
 	return ret;
 }
 
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_unlocked)
-BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE)
+BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
@@ -6480,12 +8525,21 @@ __bpf_kfunc_start_defs();
  * scx_bpf_task_set_slice - Set task's time slice
  * @p: task of interest
  * @slice: time slice to set in nsecs
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Set @p's time slice to @slice. Returns %true on success, %false if the
  * calling scheduler doesn't have authority over @p.
  */
-__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice)
+__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
+					const struct bpf_prog_aux *aux)
 {
+	struct scx_sched *sch;
+
+	guard(rcu)();
+	sch = scx_prog_sched(aux);
+	if (unlikely(!scx_task_on_sched(sch, p)))
+		return false;
+
 	p->scx.slice = slice;
 	return true;
 }
@@ -6494,12 +8548,21 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice)
  * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering
  * @p: task of interest
  * @vtime: virtual time to set
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Set @p's virtual time to @vtime. Returns %true on success, %false if the
  * calling scheduler doesn't have authority over @p.
  */
-__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime)
+__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
+					    const struct bpf_prog_aux *aux)
 {
+	struct scx_sched *sch;
+
+	guard(rcu)();
+	sch = scx_prog_sched(aux);
+	if (unlikely(!scx_task_on_sched(sch, p)))
+		return false;
+
 	p->scx.dsq_vtime = vtime;
 	return true;
 }
@@ -6521,7 +8584,7 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 	 * lead to irq_work_queue() malfunction such as infinite busy wait for
 	 * IRQ status update. Suppress kicking.
 	 */
-	if (scx_rq_bypassing(this_rq))
+	if (scx_bypassing(sch, cpu_of(this_rq)))
 		goto out;
 
 	/*
@@ -6561,18 +8624,19 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
  * scx_bpf_kick_cpu - Trigger reschedule on a CPU
  * @cpu: cpu to kick
  * @flags: %SCX_KICK_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
  * trigger rescheduling on a busy CPU. This can be called from any online
  * scx_ops operation and the actual kicking is performed asynchronously through
  * an irq work.
  */
-__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags)
+__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (likely(sch))
 		scx_kick_cpu(sch, cpu, flags);
 }
@@ -6646,13 +8710,14 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)
  * @it: iterator to initialize
  * @dsq_id: DSQ to iterate
  * @flags: %SCX_DSQ_ITER_*
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Initialize BPF iterator @it which can be used with bpf_for_each() to walk
  * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
  * tasks which are already queued when this function is invoked.
  */
 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
-				     u64 flags)
+				     u64 flags, const struct bpf_prog_aux *aux)
 {
 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;
 	struct scx_sched *sch;
@@ -6670,7 +8735,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
 	 */
 	kit->dsq = NULL;
 
-	sch = rcu_dereference_check(scx_root, rcu_read_lock_bh_held());
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -6681,8 +8746,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
 	if (!kit->dsq)
 		return -ENOENT;
 
-	kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, flags,
-					   READ_ONCE(kit->dsq->seq));
+	kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags);
 
 	return 0;
 }
@@ -6696,41 +8760,13 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
 {
 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;
-	bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;
-	struct task_struct *p;
-	unsigned long flags;
 
 	if (!kit->dsq)
 		return NULL;
 
-	raw_spin_lock_irqsave(&kit->dsq->lock, flags);
-
-	if (list_empty(&kit->cursor.node))
-		p = NULL;
-	else
-		p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);
-
-	/*
-	 * Only tasks which were queued before the iteration started are
-	 * visible. This bounds BPF iterations and guarantees that vtime never
-	 * jumps in the other direction while iterating.
-	 */
-	do {
-		p = nldsq_next_task(kit->dsq, p, rev);
-	} while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));
-
-	if (p) {
-		if (rev)
-			list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);
-		else
-			list_move(&kit->cursor.node, &p->scx.dsq_list.node);
-	} else {
-		list_del_init(&kit->cursor.node);
-	}
-
-	raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
+	guard(raw_spinlock_irqsave)(&kit->dsq->lock);
 
-	return p;
+	return nldsq_cursor_next_task(&kit->cursor, kit->dsq);
 }
 
 /**
@@ -6759,6 +8795,7 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
 /**
  * scx_bpf_dsq_peek - Lockless peek at the first element.
  * @dsq_id: DSQ to examine.
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Read the first element in the DSQ. This is semantically equivalent to using
  * the DSQ iterator, but is lockfree. Of course, like any lockless operation,
@@ -6767,12 +8804,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
  *
  * Returns the pointer, or NULL indicates an empty queue OR internal error.
  */
-__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id)
+__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
+						 const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	struct scx_dispatch_q *dsq;
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return NULL;
 
@@ -6790,6 +8828,62 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id)
 	return rcu_dereference(dsq->first_task);
 }
 
+/**
+ * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ
+ * @dsq_id: DSQ to re-enqueue
+ * @reenq_flags: %SCX_RENQ_*
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Iterate over all of the tasks currently enqueued on the DSQ identified by
+ * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are
+ * supported:
+ *
+ * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu)
+ * - User DSQs
+ *
+ * Re-enqueues are performed asynchronously. Can be called from anywhere.
+ */
+__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
+				   const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	struct scx_dispatch_q *dsq;
+
+	guard(preempt)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return;
+
+	if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) {
+		scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags);
+		return;
+	}
+
+	/* not specifying any filter bits is the same as %SCX_REENQ_ANY */
+	if (!(reenq_flags & __SCX_REENQ_FILTER_MASK))
+		reenq_flags |= SCX_REENQ_ANY;
+
+	dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id());
+	schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq());
+}
+
+/**
+ * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Iterate over all of the tasks currently enqueued on the local DSQ of the
+ * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from
+ * anywhere.
+ *
+ * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the
+ * future.
+ */
+__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux)
+{
+	scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux);
+}
+
 __bpf_kfunc_end_defs();
 
 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
@@ -6844,18 +8938,20 @@ __bpf_kfunc_start_defs();
  * @fmt: error message format string
  * @data: format string parameters packaged using ___bpf_fill() macro
  * @data__sz: @data len, must end in '__sz' for the verifier
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
  * disabling.
  */
 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
-				   unsigned long long *data, u32 data__sz)
+				   unsigned long long *data, u32 data__sz,
+				   const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
-	sch = rcu_dereference_bh(scx_root);
+	sch = scx_prog_sched(aux);
 	if (likely(sch) &&
 	    bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
 		scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line);
@@ -6867,18 +8963,19 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
  * @fmt: error message format string
  * @data: format string parameters packaged using ___bpf_fill() macro
  * @data__sz: @data len, must end in '__sz' for the verifier
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Indicate that the BPF scheduler encountered a fatal error and initiate ops
  * disabling.
  */
 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
-				    u32 data__sz)
+				    u32 data__sz, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
-	sch = rcu_dereference_bh(scx_root);
+	sch = scx_prog_sched(aux);
 	if (likely(sch) &&
 	    bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
 		scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line);
@@ -6890,6 +8987,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
  * @fmt: format string
  * @data: format string parameters packaged using ___bpf_fill() macro
  * @data__sz: @data len, must end in '__sz' for the verifier
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and
  * dump_task() to generate extra debug dump specific to the BPF scheduler.
@@ -6898,7 +8996,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
  * multiple calls. The last line is automatically terminated.
  */
 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
-				   u32 data__sz)
+				   u32 data__sz, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	struct scx_dump_data *dd = &scx_dump_data;
@@ -6907,7 +9005,7 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return;
 
@@ -6943,39 +9041,22 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
 		ops_dump_flush();
 }
 
-/**
- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
- *
- * Iterate over all of the tasks currently enqueued on the local DSQ of the
- * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from
- * anywhere.
- */
-__bpf_kfunc void scx_bpf_reenqueue_local___v2(void)
-{
-	struct rq *rq;
-
-	guard(preempt)();
-
-	rq = this_rq();
-	local_set(&rq->scx.reenq_local_deferred, 1);
-	schedule_deferred(rq);
-}
-
 /**
  * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU
  * @cpu: CPU of interest
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Return the maximum relative capacity of @cpu in relation to the most
  * performant CPU in the system. The return value is in the range [1,
  * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur().
  */
-__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)
+__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))
 		return arch_scale_cpu_capacity(cpu);
 	else
@@ -6985,6 +9066,7 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)
 /**
  * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
  * @cpu: CPU of interest
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Return the current relative performance of @cpu in relation to its maximum.
  * The return value is in the range [1, %SCX_CPUPERF_ONE].
@@ -6996,13 +9078,13 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)
  *
  * The result is in the range [1, %SCX_CPUPERF_ONE].
  */
-__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)
+__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))
 		return arch_scale_freq_capacity(cpu);
 	else
@@ -7013,6 +9095,7 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)
  * scx_bpf_cpuperf_set - Set the relative performance target of a CPU
  * @cpu: CPU of interest
  * @perf: target performance level [0, %SCX_CPUPERF_ONE]
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Set the target performance level of @cpu to @perf. @perf is in linear
  * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the
@@ -7023,13 +9106,13 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)
  * use. Consult hardware and cpufreq documentation for more information. The
  * current performance level can be monitored using scx_bpf_cpuperf_cur().
  */
-__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf)
+__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return;
 
@@ -7139,14 +9222,15 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
 /**
  * scx_bpf_cpu_rq - Fetch the rq of a CPU
  * @cpu: CPU of the rq
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  */
-__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu)
+__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return NULL;
 
@@ -7165,18 +9249,19 @@ __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu)
 
 /**
  * scx_bpf_locked_rq - Return the rq currently locked by SCX
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns the rq if a rq lock is currently held by SCX.
  * Otherwise emits an error and returns NULL.
  */
-__bpf_kfunc struct rq *scx_bpf_locked_rq(void)
+__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	struct rq *rq;
 
 	guard(preempt)();
 
-	sch = rcu_dereference_sched(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return NULL;
 
@@ -7192,16 +9277,17 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(void)
 /**
  * scx_bpf_cpu_curr - Return remote CPU's curr task
  * @cpu: CPU of interest
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Callers must hold RCU read lock (KF_RCU).
  */
-__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu)
+__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return NULL;
 
@@ -7211,41 +9297,6 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu)
 	return rcu_dereference(cpu_rq(cpu)->curr);
 }
 
-/**
- * scx_bpf_task_cgroup - Return the sched cgroup of a task
- * @p: task of interest
- *
- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
- * from the scheduler's POV. SCX operations should use this function to
- * determine @p's current cgroup as, unlike following @p->cgroups,
- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all
- * rq-locked operations. Can be called on the parameter tasks of rq-locked
- * operations. The restriction guarantees that @p's rq is locked by the caller.
- */
-#ifdef CONFIG_CGROUP_SCHED
-__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p)
-{
-	struct task_group *tg = p->sched_task_group;
-	struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
-	struct scx_sched *sch;
-
-	guard(rcu)();
-
-	sch = rcu_dereference(scx_root);
-	if (unlikely(!sch))
-		goto out;
-
-	if (!scx_kf_allowed_on_arg_tasks(sch, __SCX_KF_RQ_LOCKED, p))
-		goto out;
-
-	cgrp = tg_cgrp(tg);
-
-out:
-	cgroup_get(cgrp);
-	return cgrp;
-}
-#endif
-
 /**
  * scx_bpf_now - Returns a high-performance monotonically non-decreasing
  * clock for the current CPU. The clock returned is in nanoseconds.
@@ -7322,10 +9373,13 @@ static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *event
 		scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST);
 		scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING);
 		scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
+		scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED);
+		scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT);
 		scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL);
 		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION);
 		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH);
 		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE);
+		scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED);
 	}
 }
 
@@ -7359,25 +9413,63 @@ __bpf_kfunc void scx_bpf_events(struct scx_event_stats *events,
 	memcpy(events, &e_sys, events__sz);
 }
 
+#ifdef CONFIG_CGROUP_SCHED
+/**
+ * scx_bpf_task_cgroup - Return the sched cgroup of a task
+ * @p: task of interest
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
+ * from the scheduler's POV. SCX operations should use this function to
+ * determine @p's current cgroup as, unlike following @p->cgroups,
+ * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all
+ * rq-locked operations. Can be called on the parameter tasks of rq-locked
+ * operations. The restriction guarantees that @p's rq is locked by the caller.
+ */
+__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p,
+					       const struct bpf_prog_aux *aux)
+{
+	struct task_group *tg = p->sched_task_group;
+	struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
+	struct scx_sched *sch;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		goto out;
+
+	if (!scx_kf_allowed_on_arg_tasks(sch, __SCX_KF_RQ_LOCKED, p))
+		goto out;
+
+	cgrp = tg_cgrp(tg);
+
+out:
+	cgroup_get(cgrp);
+	return cgrp;
+}
+#endif	/* CONFIG_CGROUP_SCHED */
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_any)
-BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_RCU);
-BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_RCU);
-BTF_ID_FLAGS(func, scx_bpf_kick_cpu)
+BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
+BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
+BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
-BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_RCU_PROTECTED | KF_RET_NULL)
-BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED)
+BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL)
+BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED)
 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
-BTF_ID_FLAGS(func, scx_bpf_exit_bstr)
-BTF_ID_FLAGS(func, scx_bpf_error_bstr)
-BTF_ID_FLAGS(func, scx_bpf_dump_bstr)
-BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2)
-BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap)
-BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur)
-BTF_ID_FLAGS(func, scx_bpf_cpuperf_set)
+BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
@@ -7385,14 +9477,14 @@ BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_cpu_rq)
-BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_RET_NULL)
-BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_RET_NULL | KF_RCU_PROTECTED)
-#ifdef CONFIG_CGROUP_SCHED
-BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE)
-#endif
+BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)
+BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
 BTF_ID_FLAGS(func, scx_bpf_now)
 BTF_ID_FLAGS(func, scx_bpf_events)
+#ifdef CONFIG_CGROUP_SCHED
+BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE)
+#endif
 BTF_KFUNCS_END(scx_kfunc_ids_any)
 
 static const struct btf_kfunc_id_set scx_kfunc_set_any = {
diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h
index 43429b33e52c1..0b7fc46aee08c 100644
--- a/kernel/sched/ext.h
+++ b/kernel/sched/ext.h
@@ -11,7 +11,7 @@
 void scx_tick(struct rq *rq);
 void init_scx_entity(struct sched_ext_entity *scx);
 void scx_pre_fork(struct task_struct *p);
-int scx_fork(struct task_struct *p);
+int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs);
 void scx_post_fork(struct task_struct *p);
 void scx_cancel_fork(struct task_struct *p);
 bool scx_can_stop_tick(struct rq *rq);
@@ -44,7 +44,7 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
 
 static inline void scx_tick(struct rq *rq) {}
 static inline void scx_pre_fork(struct task_struct *p) {}
-static inline int scx_fork(struct task_struct *p) { return 0; }
+static inline int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) { return 0; }
 static inline void scx_post_fork(struct task_struct *p) {}
 static inline void scx_cancel_fork(struct task_struct *p) {}
 static inline u32 scx_cpuperf_target(s32 cpu) { return 0; }
diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c
index ba298ac3ce6cc..c7e4052626979 100644
--- a/kernel/sched/ext_idle.c
+++ b/kernel/sched/ext_idle.c
@@ -368,7 +368,7 @@ void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops)
 
 	/*
 	 * Enable NUMA optimization only when there are multiple NUMA domains
-	 * among the online CPUs and the NUMA domains don't perfectly overlaps
+	 * among the online CPUs and the NUMA domains don't perfectly overlap
 	 * with the LLC domains.
 	 *
 	 * If all CPUs belong to the same NUMA node and the same LLC domain,
@@ -767,7 +767,8 @@ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
 	 * either enqueue() sees the idle bit or update_idle() sees the task
 	 * that enqueue() queued.
 	 */
-	if (SCX_HAS_OP(sch, update_idle) && do_notify && !scx_rq_bypassing(rq))
+	if (SCX_HAS_OP(sch, update_idle) && do_notify &&
+	    !scx_bypassing(sch, cpu_of(rq)))
 		SCX_CALL_OP(sch, SCX_KF_REST, update_idle, rq, cpu_of(rq), idle);
 }
 
@@ -945,14 +946,15 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
  * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or
  *		      trigger an error if @cpu is invalid
  * @cpu: target CPU
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  */
-__bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
+__bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch) || !ops_cpu_valid(sch, cpu, NULL))
 		return NUMA_NO_NODE;
 	return cpu_to_node(cpu);
@@ -964,6 +966,7 @@ __bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
  * @prev_cpu: CPU @p was on previously
  * @wake_flags: %SCX_WAKE_* flags
  * @is_idle: out parameter indicating whether the returned CPU is idle
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked
  * context such as a BPF test_run() call, as long as built-in CPU selection
@@ -974,14 +977,15 @@ __bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
  * currently idle and thus a good candidate for direct dispatching.
  */
 __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
-				       u64 wake_flags, bool *is_idle)
+				       u64 wake_flags, bool *is_idle,
+				       const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	s32 cpu;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1009,6 +1013,7 @@ struct scx_bpf_select_cpu_and_args {
  *       @args->prev_cpu: CPU @p was on previously
  *       @args->wake_flags: %SCX_WAKE_* flags
  *       @args->flags: %SCX_PICK_IDLE* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument
  * limit. BPF programs should use scx_bpf_select_cpu_and() which is provided
@@ -1027,13 +1032,14 @@ struct scx_bpf_select_cpu_and_args {
  */
 __bpf_kfunc s32
 __scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed,
-			 struct scx_bpf_select_cpu_and_args *args)
+			 struct scx_bpf_select_cpu_and_args *args,
+			 const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1055,6 +1061,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64
 	if (unlikely(!sch))
 		return -ENODEV;
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * Disallow if any sub-scheds are attached. There is no way to tell
+	 * which scheduler called us, just error out @p's scheduler.
+	 */
+	if (unlikely(!list_empty(&sch->children))) {
+		scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used");
+		return -EINVAL;
+	}
+#endif
+
 	return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags,
 				     cpus_allowed, flags);
 }
@@ -1063,18 +1080,20 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64
  * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the
  * idle-tracking per-CPU cpumask of a target NUMA node.
  * @node: target NUMA node
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns an empty cpumask if idle tracking is not enabled, if @node is
  * not valid, or running on a UP kernel. In this case the actual error will
  * be reported to the BPF scheduler via scx_error().
  */
-__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask_node(int node)
+__bpf_kfunc const struct cpumask *
+scx_bpf_get_idle_cpumask_node(s32 node, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return cpu_none_mask;
 
@@ -1088,17 +1107,18 @@ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask_node(int node)
 /**
  * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking
  * per-CPU cpumask.
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns an empty mask if idle tracking is not enabled, or running on a
  * UP kernel.
  */
-__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void)
+__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return cpu_none_mask;
 
@@ -1118,18 +1138,20 @@ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void)
  * idle-tracking, per-physical-core cpumask of a target NUMA node. Can be
  * used to determine if an entire physical core is free.
  * @node: target NUMA node
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns an empty cpumask if idle tracking is not enabled, if @node is
  * not valid, or running on a UP kernel. In this case the actual error will
  * be reported to the BPF scheduler via scx_error().
  */
-__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask_node(int node)
+__bpf_kfunc const struct cpumask *
+scx_bpf_get_idle_smtmask_node(s32 node, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return cpu_none_mask;
 
@@ -1147,17 +1169,18 @@ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask_node(int node)
  * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking,
  * per-physical-core cpumask. Can be used to determine if an entire physical
  * core is free.
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns an empty mask if idle tracking is not enabled, or running on a
  * UP kernel.
  */
-__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void)
+__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return cpu_none_mask;
 
@@ -1193,6 +1216,7 @@ __bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask)
 /**
  * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state
  * @cpu: cpu to test and clear idle for
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Returns %true if @cpu was idle and its idle state was successfully cleared.
  * %false otherwise.
@@ -1200,13 +1224,13 @@ __bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask)
  * Unavailable if ops.update_idle() is implemented and
  * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
  */
-__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
+__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return false;
 
@@ -1224,6 +1248,7 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
  * @cpus_allowed: Allowed cpumask
  * @node: target NUMA node
  * @flags: %SCX_PICK_IDLE_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node.
  *
@@ -1239,13 +1264,14 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
  * %SCX_OPS_BUILTIN_IDLE_PER_NODE is not set.
  */
 __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed,
-					   int node, u64 flags)
+					   s32 node, u64 flags,
+					   const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1260,6 +1286,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed,
  * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
  * @cpus_allowed: Allowed cpumask
  * @flags: %SCX_PICK_IDLE_CPU_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu
  * number on success. -%EBUSY if no matching cpu was found.
@@ -1279,13 +1306,13 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed,
  * scx_bpf_pick_idle_cpu_node() instead.
  */
 __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
-				      u64 flags)
+				      u64 flags, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1306,6 +1333,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
  * @cpus_allowed: Allowed cpumask
  * @node: target NUMA node
  * @flags: %SCX_PICK_IDLE_CPU_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
  * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
@@ -1322,14 +1350,15 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
  * CPU.
  */
 __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed,
-					  int node, u64 flags)
+					  s32 node, u64 flags,
+					  const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	s32 cpu;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1355,6 +1384,7 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed,
  * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU
  * @cpus_allowed: Allowed cpumask
  * @flags: %SCX_PICK_IDLE_CPU_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
  * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
  * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
@@ -1369,14 +1399,14 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed,
  * scx_bpf_pick_any_cpu_node() instead.
  */
 __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed,
-				     u64 flags)
+				     u64 flags, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	s32 cpu;
 
 	guard(rcu)();
 
-	sch = rcu_dereference(scx_root);
+	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
 
@@ -1401,20 +1431,20 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed,
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_idle)
-BTF_ID_FLAGS(func, scx_bpf_cpu_node)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_ACQUIRE)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_ACQUIRE)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
 BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
-BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle)
-BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_RCU)
-BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)
 BTF_KFUNCS_END(scx_kfunc_ids_idle)
 
 static const struct btf_kfunc_id_set scx_kfunc_set_idle = {
diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h
index 00b450597f3e0..b4f36d8b9c1dd 100644
--- a/kernel/sched/ext_internal.h
+++ b/kernel/sched/ext_internal.h
@@ -24,10 +24,16 @@ enum scx_consts {
 	 */
 	SCX_TASK_ITER_BATCH		= 32,
 
+	SCX_BYPASS_HOST_NTH		= 2,
+
 	SCX_BYPASS_LB_DFL_INTV_US	= 500 * USEC_PER_MSEC,
 	SCX_BYPASS_LB_DONOR_PCT		= 125,
 	SCX_BYPASS_LB_MIN_DELTA_DIV	= 4,
 	SCX_BYPASS_LB_BATCH		= 256,
+
+	SCX_REENQ_LOCAL_MAX_REPEAT	= 256,
+
+	SCX_SUB_MAX_DEPTH		= 4,
 };
 
 enum scx_exit_kind {
@@ -38,6 +44,7 @@ enum scx_exit_kind {
 	SCX_EXIT_UNREG_BPF,	/* BPF-initiated unregistration */
 	SCX_EXIT_UNREG_KERN,	/* kernel-initiated unregistration */
 	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */
+	SCX_EXIT_PARENT,	/* parent exiting */
 
 	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */
 	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */
@@ -62,6 +69,7 @@ enum scx_exit_kind {
 enum scx_exit_code {
 	/* Reasons */
 	SCX_ECODE_RSN_HOTPLUG	= 1LLU << 32,
+	SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32,
 
 	/* Actions */
 	SCX_ECODE_ACT_RESTART	= 1LLU << 48,
@@ -175,9 +183,10 @@ enum scx_ops_flags {
 	SCX_OPS_BUILTIN_IDLE_PER_NODE	= 1LLU << 6,
 
 	/*
-	 * CPU cgroup support flags
+	 * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ
+	 * enqueues.
 	 */
-	SCX_OPS_HAS_CGROUP_WEIGHT	= 1LLU << 16,	/* DEPRECATED, will be removed on 6.18 */
+	SCX_OPS_ALWAYS_ENQ_IMMED	= 1LLU << 7,
 
 	SCX_OPS_ALL_FLAGS		= SCX_OPS_KEEP_BUILTIN_IDLE |
 					  SCX_OPS_ENQ_LAST |
@@ -186,7 +195,7 @@ enum scx_ops_flags {
 					  SCX_OPS_ALLOW_QUEUED_WAKEUP |
 					  SCX_OPS_SWITCH_PARTIAL |
 					  SCX_OPS_BUILTIN_IDLE_PER_NODE |
-					  SCX_OPS_HAS_CGROUP_WEIGHT,
+					  SCX_OPS_ALWAYS_ENQ_IMMED,
 
 	/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
 	__SCX_OPS_INTERNAL_MASK		= 0xffLLU << 56,
@@ -213,7 +222,7 @@ struct scx_exit_task_args {
 	bool cancelled;
 };
 
-/* argument container for ops->cgroup_init() */
+/* argument container for ops.cgroup_init() */
 struct scx_cgroup_init_args {
 	/* the weight of the cgroup [1..10000] */
 	u32			weight;
@@ -236,12 +245,12 @@ enum scx_cpu_preempt_reason {
 };
 
 /*
- * Argument container for ops->cpu_acquire(). Currently empty, but may be
+ * Argument container for ops.cpu_acquire(). Currently empty, but may be
  * expanded in the future.
  */
 struct scx_cpu_acquire_args {};
 
-/* argument container for ops->cpu_release() */
+/* argument container for ops.cpu_release() */
 struct scx_cpu_release_args {
 	/* the reason the CPU was preempted */
 	enum scx_cpu_preempt_reason reason;
@@ -250,9 +259,7 @@ struct scx_cpu_release_args {
 	struct task_struct	*task;
 };
 
-/*
- * Informational context provided to dump operations.
- */
+/* informational context provided to dump operations */
 struct scx_dump_ctx {
 	enum scx_exit_kind	kind;
 	s64			exit_code;
@@ -261,6 +268,18 @@ struct scx_dump_ctx {
 	u64			at_jiffies;
 };
 
+/* argument container for ops.sub_attach() */
+struct scx_sub_attach_args {
+	struct sched_ext_ops	*ops;
+	char			*cgroup_path;
+};
+
+/* argument container for ops.sub_detach() */
+struct scx_sub_detach_args {
+	struct sched_ext_ops	*ops;
+	char			*cgroup_path;
+};
+
 /**
  * struct sched_ext_ops - Operation table for BPF scheduler implementation
  *
@@ -721,6 +740,20 @@ struct sched_ext_ops {
 
 #endif	/* CONFIG_EXT_GROUP_SCHED */
 
+	/**
+	 * @sub_attach: Attach a sub-scheduler
+	 * @args: argument container, see the struct definition
+	 *
+	 * Return 0 to accept the sub-scheduler. -errno to reject.
+	 */
+	s32 (*sub_attach)(struct scx_sub_attach_args *args);
+
+	/**
+	 * @sub_detach: Detach a sub-scheduler
+	 * @args: argument container, see the struct definition
+	 */
+	void (*sub_detach)(struct scx_sub_detach_args *args);
+
 	/*
 	 * All online ops must come before ops.cpu_online().
 	 */
@@ -762,6 +795,10 @@ struct sched_ext_ops {
 	 */
 	void (*exit)(struct scx_exit_info *info);
 
+	/*
+	 * Data fields must comes after all ops fields.
+	 */
+
 	/**
 	 * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch
 	 */
@@ -796,6 +833,12 @@ struct sched_ext_ops {
 	 */
 	u64 hotplug_seq;
 
+	/**
+	 * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the
+	 * specified cgroup.
+	 */
+	u64 sub_cgroup_id;
+
 	/**
 	 * @name: BPF scheduler's name
 	 *
@@ -806,7 +849,7 @@ struct sched_ext_ops {
 	char name[SCX_OPS_NAME_LEN];
 
 	/* internal use only, must be NULL */
-	void *priv;
+	void __rcu *priv;
 };
 
 enum scx_opi {
@@ -853,6 +896,24 @@ struct scx_event_stats {
 	 */
 	s64		SCX_EV_ENQ_SKIP_MIGRATION_DISABLED;
 
+	/*
+	 * The number of times a task, enqueued on a local DSQ with
+	 * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for
+	 * immediate execution.
+	 */
+	s64		SCX_EV_REENQ_IMMED;
+
+	/*
+	 * The number of times a reenq of local DSQ caused another reenq of
+	 * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher
+	 * priority class task even if the BPF scheduler always satisfies the
+	 * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However,
+	 * that scenario is very unlikely and this count going up regularly
+	 * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ
+	 * incorrectly causing recursive reenqueues.
+	 */
+	s64		SCX_EV_REENQ_LOCAL_REPEAT;
+
 	/*
 	 * Total number of times a task's time slice was refilled with the
 	 * default value (SCX_SLICE_DFL).
@@ -873,15 +934,77 @@ struct scx_event_stats {
 	 * The number of times the bypassing mode has been activated.
 	 */
 	s64		SCX_EV_BYPASS_ACTIVATE;
+
+	/*
+	 * The number of times the scheduler attempted to insert a task that it
+	 * doesn't own into a DSQ. Such attempts are ignored.
+	 *
+	 * As BPF schedulers are allowed to ignore dequeues, it's difficult to
+	 * tell whether such an attempt is from a scheduler malfunction or an
+	 * ignored dequeue around sub-sched enabling. If this count keeps going
+	 * up regardless of sub-sched enabling, it likely indicates a bug in the
+	 * scheduler.
+	 */
+	s64		SCX_EV_INSERT_NOT_OWNED;
+
+	/*
+	 * The number of times tasks from bypassing descendants are scheduled
+	 * from sub_bypass_dsq's.
+	 */
+	s64		SCX_EV_SUB_BYPASS_DISPATCH;
+};
+
+struct scx_sched;
+
+enum scx_sched_pcpu_flags {
+	SCX_SCHED_PCPU_BYPASSING	= 1LLU << 0,
+};
+
+/* dispatch buf */
+struct scx_dsp_buf_ent {
+	struct task_struct	*task;
+	unsigned long		qseq;
+	u64			dsq_id;
+	u64			enq_flags;
+};
+
+struct scx_dsp_ctx {
+	struct rq		*rq;
+	u32			cursor;
+	u32			nr_tasks;
+	struct scx_dsp_buf_ent	buf[];
+};
+
+struct scx_deferred_reenq_local {
+	struct list_head	node;
+	u64			flags;
+	u64			seq;
+	u32			cnt;
 };
 
 struct scx_sched_pcpu {
+	struct scx_sched	*sch;
+	u64			flags;	/* protected by rq lock */
+
 	/*
 	 * The event counters are in a per-CPU variable to minimize the
 	 * accounting overhead. A system-wide view on the event counter is
 	 * constructed when requested by scx_bpf_events().
 	 */
 	struct scx_event_stats	event_stats;
+
+	struct scx_deferred_reenq_local deferred_reenq_local;
+	struct scx_dispatch_q	bypass_dsq;
+#ifdef CONFIG_EXT_SUB_SCHED
+	u32			bypass_host_seq;
+#endif
+
+	/* must be the last entry - contains flex array */
+	struct scx_dsp_ctx	dsp_ctx;
+};
+
+struct scx_sched_pnode {
+	struct scx_dispatch_q	global_dsq;
 };
 
 struct scx_sched {
@@ -897,15 +1020,50 @@ struct scx_sched {
 	 * per-node split isn't sufficient, it can be further split.
 	 */
 	struct rhashtable	dsq_hash;
-	struct scx_dispatch_q	**global_dsqs;
+	struct scx_sched_pnode	**pnode;
 	struct scx_sched_pcpu __percpu *pcpu;
 
+	u64			slice_dfl;
+	u64			bypass_timestamp;
+	s32			bypass_depth;
+
+	/* bypass dispatch path enable state, see bypass_dsp_enabled() */
+	unsigned long		bypass_dsp_claim;
+	atomic_t		bypass_dsp_enable_depth;
+
+	bool			aborting;
+	bool			dump_disabled;	/* protected by scx_dump_lock */
+	u32			dsp_max_batch;
+	s32			level;
+
 	/*
 	 * Updates to the following warned bitfields can race causing RMW issues
 	 * but it doesn't really matter.
 	 */
 	bool			warned_zero_slice:1;
 	bool			warned_deprecated_rq:1;
+	bool			warned_unassoc_progs:1;
+
+	struct list_head	all;
+
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct rhash_head	hash_node;
+
+	struct list_head	children;
+	struct list_head	sibling;
+	struct cgroup		*cgrp;
+	char			*cgrp_path;
+	struct kset		*sub_kset;
+
+	bool			sub_attached;
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+	/*
+	 * The maximum amount of time in jiffies that a task may be runnable
+	 * without being scheduled on a CPU. If this timeout is exceeded, it
+	 * will trigger scx_error().
+	 */
+	unsigned long		watchdog_timeout;
 
 	atomic_t		exit_kind;
 	struct scx_exit_info	*exit_info;
@@ -913,9 +1071,13 @@ struct scx_sched {
 	struct kobject		kobj;
 
 	struct kthread_worker	*helper;
-	struct irq_work		error_irq_work;
+	struct irq_work		disable_irq_work;
 	struct kthread_work	disable_work;
+	struct timer_list	bypass_lb_timer;
 	struct rcu_work		rcu_work;
+
+	/* all ancestors including self */
+	struct scx_sched	*ancestors[];
 };
 
 enum scx_wake_flags {
@@ -942,13 +1104,27 @@ enum scx_enq_flags {
 	SCX_ENQ_PREEMPT		= 1LLU << 32,
 
 	/*
-	 * The task being enqueued was previously enqueued on the current CPU's
-	 * %SCX_DSQ_LOCAL, but was removed from it in a call to the
-	 * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was
-	 * invoked in a ->cpu_release() callback, and the task is again
-	 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
-	 * task will not be scheduled on the CPU until at least the next invocation
-	 * of the ->cpu_acquire() callback.
+	 * Only allowed on local DSQs. Guarantees that the task either gets
+	 * on the CPU immediately and stays on it, or gets reenqueued back
+	 * to the BPF scheduler. It will never linger on a local DSQ or be
+	 * silently put back after preemption.
+	 *
+	 * The protection persists until the next fresh enqueue - it
+	 * survives SAVE/RESTORE cycles, slice extensions and preemption.
+	 * If the task can't stay on the CPU for any reason, it gets
+	 * reenqueued back to the BPF scheduler.
+	 *
+	 * Exiting and migration-disabled tasks bypass ops.enqueue() and
+	 * are placed directly on a local DSQ without IMMED protection
+	 * unless %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED
+	 * are set respectively.
+	 */
+	SCX_ENQ_IMMED		= 1LLU << 33,
+
+	/*
+	 * The task being enqueued was previously enqueued on a DSQ, but was
+	 * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find
+	 * out why a given task is being reenqueued.
 	 */
 	SCX_ENQ_REENQ		= 1LLU << 40,
 
@@ -969,6 +1145,7 @@ enum scx_enq_flags {
 	SCX_ENQ_CLEAR_OPSS	= 1LLU << 56,
 	SCX_ENQ_DSQ_PRIQ	= 1LLU << 57,
 	SCX_ENQ_NESTED		= 1LLU << 58,
+	SCX_ENQ_GDSQ_FALLBACK	= 1LLU << 59,	/* fell back to global DSQ */
 };
 
 enum scx_deq_flags {
@@ -982,6 +1159,28 @@ enum scx_deq_flags {
 	 * it hasn't been dispatched yet. Dequeue from the BPF side.
 	 */
 	SCX_DEQ_CORE_SCHED_EXEC	= 1LLU << 32,
+
+	/*
+	 * The task is being dequeued due to a property change (e.g.,
+	 * sched_setaffinity(), sched_setscheduler(), set_user_nice(),
+	 * etc.).
+	 */
+	SCX_DEQ_SCHED_CHANGE	= 1LLU << 33,
+};
+
+enum scx_reenq_flags {
+	/* low 16bits determine which tasks should be reenqueued */
+	SCX_REENQ_ANY		= 1LLU << 0,	/* all tasks */
+
+	__SCX_REENQ_FILTER_MASK	= 0xffffLLU,
+
+	__SCX_REENQ_USER_MASK	= SCX_REENQ_ANY,
+
+	/* bits 32-35 used by task_should_reenq() */
+	SCX_REENQ_TSR_RQ_OPEN	= 1LLU << 32,
+	SCX_REENQ_TSR_NOT_FIRST	= 1LLU << 33,
+
+	__SCX_REENQ_TSR_MASK	= 0xfLLU << 32,
 };
 
 enum scx_pick_idle_cpu_flags {
@@ -1161,6 +1360,7 @@ enum scx_ops_state {
 #define SCX_OPSS_STATE_MASK	((1LU << SCX_OPSS_QSEQ_SHIFT) - 1)
 #define SCX_OPSS_QSEQ_MASK	(~SCX_OPSS_STATE_MASK)
 
+extern struct scx_sched __rcu *scx_root;
 DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
 
 /*
@@ -1177,7 +1377,107 @@ static inline bool scx_kf_allowed_if_unlocked(void)
 	return !current->scx.kf_mask;
 }
 
-static inline bool scx_rq_bypassing(struct rq *rq)
+static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)
+{
+	return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags &
+			SCX_SCHED_PCPU_BYPASSING);
+}
+
+#ifdef CONFIG_EXT_SUB_SCHED
+/**
+ * scx_task_sched - Find scx_sched scheduling a task
+ * @p: task of interest
+ *
+ * Return @p's scheduler instance. Must be called with @p's pi_lock or rq lock
+ * held.
+ */
+static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
+{
+	return rcu_dereference_protected(p->scx.sched,
+					 lockdep_is_held(&p->pi_lock) ||
+					 lockdep_is_held(__rq_lockp(task_rq(p))));
+}
+
+/**
+ * scx_task_sched_rcu - Find scx_sched scheduling a task
+ * @p: task of interest
+ *
+ * Return @p's scheduler instance. The returned scx_sched is RCU protected.
+ */
+static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p)
+{
+	return rcu_dereference_all(p->scx.sched);
+}
+
+/**
+ * scx_task_on_sched - Is a task on the specified sched?
+ * @sch: sched to test against
+ * @p: task of interest
+ *
+ * Returns %true if @p is on @sch, %false otherwise.
+ */
+static inline bool scx_task_on_sched(struct scx_sched *sch,
+				     const struct task_struct *p)
+{
+	return rcu_access_pointer(p->scx.sched) == sch;
+}
+
+/**
+ * scx_prog_sched - Find scx_sched associated with a BPF prog
+ * @aux: aux passed in from BPF to a kfunc
+ *
+ * To be called from kfuncs. Return the scheduler instance associated with the
+ * BPF program given the implicit kfunc argument aux. The returned scx_sched is
+ * RCU protected.
+ */
+static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
+{
+	struct sched_ext_ops *ops;
+	struct scx_sched *root;
+
+	ops = bpf_prog_get_assoc_struct_ops(aux);
+	if (likely(ops))
+		return rcu_dereference_all(ops->priv);
+
+	root = rcu_dereference_all(scx_root);
+	if (root) {
+		/*
+		 * COMPAT-v6.19: Schedulers built before sub-sched support was
+		 * introduced may have unassociated non-struct_ops programs.
+		 */
+		if (!root->ops.sub_attach)
+			return root;
+
+		if (!root->warned_unassoc_progs) {
+			printk_deferred(KERN_WARNING "sched_ext: Unassociated program %s (id %d)\n",
+					aux->name, aux->id);
+			root->warned_unassoc_progs = true;
+		}
+	}
+
+	return NULL;
+}
+#else	/* CONFIG_EXT_SUB_SCHED */
+static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
+{
+	return rcu_dereference_protected(scx_root,
+					 lockdep_is_held(&p->pi_lock) ||
+					 lockdep_is_held(__rq_lockp(task_rq(p))));
+}
+
+static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p)
+{
+	return rcu_dereference_all(scx_root);
+}
+
+static inline bool scx_task_on_sched(struct scx_sched *sch,
+				     const struct task_struct *p)
+{
+	return true;
+}
+
+static struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
 {
-	return unlikely(rq->scx.flags & SCX_RQ_BYPASSING);
+	return rcu_dereference_all(scx_root);
 }
+#endif	/* CONFIG_EXT_SUB_SCHED */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index bf948db905ed1..1b62cb68faae4 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -225,6 +225,7 @@ void __init sched_init_granularity(void)
 	update_sysctl();
 }
 
+#ifndef CONFIG_64BIT
 #define WMULT_CONST	(~0U)
 #define WMULT_SHIFT	32
 
@@ -283,6 +284,12 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight
 
 	return mul_u64_u32_shr(delta_exec, fact, shift);
 }
+#else
+static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
+{
+	return (delta_exec * weight) / lw->weight;
+}
+#endif
 
 /*
  * delta /= w
@@ -665,25 +672,83 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Since zero_vruntime closely tracks the per-task service, these
  * deltas: (v_i - v0), will be in the order of the maximal (virtual) lag
  * induced in the system due to quantisation.
- *
- * Also, we use scale_load_down() to reduce the size.
- *
- * As measured, the max (key * weight) value was ~44 bits for a kernel build.
  */
+static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned long w)
+{
+#ifdef CONFIG_64BIT
+	if (cfs_rq->sum_shift)
+		w = max(2UL, w >> cfs_rq->sum_shift);
+#endif
+	return w;
+}
+
+static inline void
+__sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+	unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+	s64 w_vruntime, key = entity_key(cfs_rq, se);
+
+	w_vruntime = key * weight;
+	WARN_ON_ONCE((w_vruntime >> 63) != (w_vruntime >> 62));
+
+	cfs_rq->sum_w_vruntime += w_vruntime;
+	cfs_rq->sum_weight += weight;
+}
+
 static void
-sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
+sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	unsigned long weight = scale_load_down(se->load.weight);
-	s64 key = entity_key(cfs_rq, se);
+	unsigned long weight;
+	s64 key, tmp;
+
+again:
+	weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+	key = entity_key(cfs_rq, se);
+
+	if (check_mul_overflow(key, weight, &key))
+		goto overflow;
+
+	if (check_add_overflow(cfs_rq->sum_w_vruntime, key, &tmp))
+		goto overflow;
 
-	cfs_rq->sum_w_vruntime += key * weight;
+	cfs_rq->sum_w_vruntime = tmp;
 	cfs_rq->sum_weight += weight;
+	return;
+
+overflow:
+	/*
+	 * There's gotta be a limit -- if we're still failing at this point
+	 * there's really nothing much to be done about things.
+	 */
+	BUG_ON(cfs_rq->sum_shift >= 10);
+	cfs_rq->sum_shift++;
+
+	/*
+	 * Note: \Sum (k_i * (w_i >> 1)) != (\Sum (k_i * w_i)) >> 1
+	 */
+	cfs_rq->sum_w_vruntime = 0;
+	cfs_rq->sum_weight = 0;
+
+	for (struct rb_node *node = cfs_rq->tasks_timeline.rb_leftmost;
+	     node; node = rb_next(node))
+		__sum_w_vruntime_add(cfs_rq, __node_2_se(node));
+
+	goto again;
+}
+
+static void
+sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+	if (sched_feat(PARANOID_AVG))
+		return sum_w_vruntime_add_paranoid(cfs_rq, se);
+
+	__sum_w_vruntime_add(cfs_rq, se);
 }
 
 static void
 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	unsigned long weight = scale_load_down(se->load.weight);
+	unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight);
 	s64 key = entity_key(cfs_rq, se);
 
 	cfs_rq->sum_w_vruntime -= key * weight;
@@ -725,7 +790,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq)
 		s64 runtime = cfs_rq->sum_w_vruntime;
 
 		if (curr) {
-			unsigned long w = scale_load_down(curr->load.weight);
+			unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight);
 
 			runtime += entity_key(cfs_rq, curr) * w;
 			weight += w;
@@ -735,7 +800,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq)
 		if (runtime < 0)
 			runtime -= (weight - 1);
 
-		delta = div_s64(runtime, weight);
+		delta = div64_long(runtime, weight);
 	} else if (curr) {
 		/*
 		 * When there is but one element, it is the average.
@@ -764,17 +829,22 @@ static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq);
  *
  *   -r_max < lag < max(r_max, q)
  */
-static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
+static s64 entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 avruntime)
 {
 	u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC;
 	s64 vlag, limit;
 
-	WARN_ON_ONCE(!se->on_rq);
-
-	vlag = avg_vruntime(cfs_rq) - se->vruntime;
+	vlag = avruntime - se->vruntime;
 	limit = calc_delta_fair(max_slice, se);
 
-	se->vlag = clamp(vlag, -limit, limit);
+	return clamp(vlag, -limit, limit);
+}
+
+static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+	WARN_ON_ONCE(!se->on_rq);
+
+	se->vlag = entity_lag(cfs_rq, se, avg_vruntime(cfs_rq));
 }
 
 /*
@@ -801,7 +871,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
 	long load = cfs_rq->sum_weight;
 
 	if (curr && curr->on_rq) {
-		unsigned long weight = scale_load_down(curr->load.weight);
+		unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight);
 
 		avg += entity_key(cfs_rq, curr) * weight;
 		load += weight;
@@ -3840,23 +3910,125 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 		    se_weight(se) * -se->avg.load_sum);
 }
 
-static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags);
+static void
+rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot)
+{
+	unsigned long old_weight = se->load.weight;
+
+	/*
+	 * VRUNTIME
+	 * --------
+	 *
+	 * COROLLARY #1: The virtual runtime of the entity needs to be
+	 * adjusted if re-weight at !0-lag point.
+	 *
+	 * Proof: For contradiction assume this is not true, so we can
+	 * re-weight without changing vruntime at !0-lag point.
+	 *
+	 *             Weight	VRuntime   Avg-VRuntime
+	 *     before    w          v            V
+	 *      after    w'         v'           V'
+	 *
+	 * Since lag needs to be preserved through re-weight:
+	 *
+	 *	lag = (V - v)*w = (V'- v')*w', where v = v'
+	 *	==>	V' = (V - v)*w/w' + v		(1)
+	 *
+	 * Let W be the total weight of the entities before reweight,
+	 * since V' is the new weighted average of entities:
+	 *
+	 *	V' = (WV + w'v - wv) / (W + w' - w)	(2)
+	 *
+	 * by using (1) & (2) we obtain:
+	 *
+	 *	(WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v
+	 *	==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v
+	 *	==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v
+	 *	==>	(V - v)*W/(W + w' - w) = (V - v)*w/w' (3)
+	 *
+	 * Since we are doing at !0-lag point which means V != v, we
+	 * can simplify (3):
+	 *
+	 *	==>	W / (W + w' - w) = w / w'
+	 *	==>	Ww' = Ww + ww' - ww
+	 *	==>	W * (w' - w) = w * (w' - w)
+	 *	==>	W = w	(re-weight indicates w' != w)
+	 *
+	 * So the cfs_rq contains only one entity, hence vruntime of
+	 * the entity @v should always equal to the cfs_rq's weighted
+	 * average vruntime @V, which means we will always re-weight
+	 * at 0-lag point, thus breach assumption. Proof completed.
+	 *
+	 *
+	 * COROLLARY #2: Re-weight does NOT affect weighted average
+	 * vruntime of all the entities.
+	 *
+	 * Proof: According to corollary #1, Eq. (1) should be:
+	 *
+	 *	(V - v)*w = (V' - v')*w'
+	 *	==>    v' = V' - (V - v)*w/w'		(4)
+	 *
+	 * According to the weighted average formula, we have:
+	 *
+	 *	V' = (WV - wv + w'v') / (W - w + w')
+	 *	   = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w')
+	 *	   = (WV - wv + w'V' - Vw + wv) / (W - w + w')
+	 *	   = (WV + w'V' - Vw) / (W - w + w')
+	 *
+	 *	==>  V'*(W - w + w') = WV + w'V' - Vw
+	 *	==>	V' * (W - w) = (W - w) * V	(5)
+	 *
+	 * If the entity is the only one in the cfs_rq, then reweight
+	 * always occurs at 0-lag point, so V won't change. Or else
+	 * there are other entities, hence W != w, then Eq. (5) turns
+	 * into V' = V. So V won't change in either case, proof done.
+	 *
+	 *
+	 * So according to corollary #1 & #2, the effect of re-weight
+	 * on vruntime should be:
+	 *
+	 *	v' = V' - (V - v) * w / w'		(4)
+	 *	   = V  - (V - v) * w / w'
+	 *	   = V  - vl * w / w'
+	 *	   = V  - vl'
+	 */
+	se->vlag = div64_long(se->vlag * old_weight, weight);
+
+	/*
+	 * DEADLINE
+	 * --------
+	 *
+	 * When the weight changes, the virtual time slope changes and
+	 * we should adjust the relative virtual deadline accordingly.
+	 *
+	 *	d' = v' + (d - v)*w/w'
+	 *	   = V' - (V - v)*w/w' + (d - v)*w/w'
+	 *	   = V  - (V - v)*w/w' + (d - v)*w/w'
+	 *	   = V  + (d - V)*w/w'
+	 */
+	if (se->rel_deadline)
+		se->deadline = div64_long(se->deadline * old_weight, weight);
+
+	if (rel_vprot)
+		se->vprot = div64_long(se->vprot * old_weight, weight);
+}
 
 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
 			    unsigned long weight)
 {
 	bool curr = cfs_rq->curr == se;
 	bool rel_vprot = false;
-	u64 vprot;
+	u64 avruntime = 0;
 
 	if (se->on_rq) {
 		/* commit outstanding execution time */
 		update_curr(cfs_rq);
-		update_entity_lag(cfs_rq, se);
-		se->deadline -= se->vruntime;
+		avruntime = avg_vruntime(cfs_rq);
+		se->vlag = entity_lag(cfs_rq, se, avruntime);
+		se->deadline -= avruntime;
 		se->rel_deadline = 1;
 		if (curr && protect_slice(se)) {
-			vprot = se->vprot - se->vruntime;
+			se->vprot -= avruntime;
 			rel_vprot = true;
 		}
 
@@ -3867,30 +4039,23 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
 	}
 	dequeue_load_avg(cfs_rq, se);
 
-	/*
-	 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i),
-	 * we need to scale se->vlag when w_i changes.
-	 */
-	se->vlag = div_s64(se->vlag * se->load.weight, weight);
-	if (se->rel_deadline)
-		se->deadline = div_s64(se->deadline * se->load.weight, weight);
-
-	if (rel_vprot)
-		vprot = div_s64(vprot * se->load.weight, weight);
+	rescale_entity(se, weight, rel_vprot);
 
 	update_load_set(&se->load, weight);
 
 	do {
 		u32 divider = get_pelt_divider(&se->avg);
-
 		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
 	} while (0);
 
 	enqueue_load_avg(cfs_rq, se);
 	if (se->on_rq) {
-		place_entity(cfs_rq, se, 0);
 		if (rel_vprot)
-			se->vprot = se->vruntime + vprot;
+			se->vprot += avruntime;
+		se->deadline += avruntime;
+		se->rel_deadline = 0;
+		se->vruntime = avruntime - se->vlag;
+
 		update_load_add(&cfs_rq->load, se->load.weight);
 		if (!curr)
 			__enqueue_entity(cfs_rq, se);
@@ -5180,7 +5345,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 */
 	if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) {
 		struct sched_entity *curr = cfs_rq->curr;
-		unsigned long load;
+		long load;
 
 		lag = se->vlag;
 
@@ -5238,17 +5403,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		 */
 		load = cfs_rq->sum_weight;
 		if (curr && curr->on_rq)
-			load += scale_load_down(curr->load.weight);
+			load += avg_vruntime_weight(cfs_rq, curr->load.weight);
 
-		lag *= load + scale_load_down(se->load.weight);
+		lag *= load + avg_vruntime_weight(cfs_rq, se->load.weight);
 		if (WARN_ON_ONCE(!load))
 			load = 1;
-		lag = div_s64(lag, load);
+		lag = div64_long(lag, load);
 	}
 
 	se->vruntime = vruntime - lag;
 
-	if (se->rel_deadline) {
+	if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) {
 		se->deadline += se->vruntime;
 		se->rel_deadline = 0;
 		return;
@@ -5604,7 +5769,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
 	 * validating it and just reschedule.
 	 */
 	if (queued) {
-		resched_curr_lazy(rq_of(cfs_rq));
+		resched_curr(rq_of(cfs_rq));
 		return;
 	}
 #endif
@@ -6809,27 +6974,41 @@ static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct
 static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
 {
 	struct sched_entity *se = &p->se;
+	unsigned long scale = 1024;
+	unsigned long util = 0;
+	u64 vdelta;
+	u64 delta;
 
 	WARN_ON_ONCE(task_rq(p) != rq);
 
-	if (rq->cfs.h_nr_queued > 1) {
-		u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
-		u64 slice = se->slice;
-		s64 delta = slice - ran;
+	if (rq->cfs.h_nr_queued <= 1)
+		return;
 
-		if (delta < 0) {
-			if (task_current_donor(rq, p))
-				resched_curr(rq);
-			return;
-		}
-		hrtick_start(rq, delta);
+	/*
+	 * Compute time until virtual deadline
+	 */
+	vdelta = se->deadline - se->vruntime;
+	if ((s64)vdelta < 0) {
+		if (task_current_donor(rq, p))
+			resched_curr(rq);
+		return;
 	}
+	delta = (se->load.weight * vdelta) / NICE_0_LOAD;
+
+	/*
+	 * Correct for instantaneous load of other classes.
+	 */
+	util += cpu_util_irq(rq);
+	if (util && util < 1024) {
+		scale *= 1024;
+		scale /= (1024 - util);
+	}
+
+	hrtick_start(rq, (scale * delta) / 1024);
 }
 
 /*
- * called from enqueue/dequeue and updates the hrtick when the
- * current task is from our class and nr_running is low enough
- * to matter.
+ * Called on enqueue to start the hrtick when h_nr_queued becomes more than 1.
  */
 static void hrtick_update(struct rq *rq)
 {
@@ -6838,6 +7017,9 @@ static void hrtick_update(struct rq *rq)
 	if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class)
 		return;
 
+	if (hrtick_active(rq))
+		return;
+
 	hrtick_start_fair(rq, donor);
 }
 #else /* !CONFIG_SCHED_HRTICK: */
@@ -6853,16 +7035,15 @@ static inline void hrtick_update(struct rq *rq)
 
 static inline bool cpu_overutilized(int cpu)
 {
-	unsigned long  rq_util_min, rq_util_max;
+	unsigned long rq_util_max;
 
 	if (!sched_energy_enabled())
 		return false;
 
-	rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
 	rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
 
 	/* Return true only if the utilization doesn't fit CPU's capacity */
-	return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu);
+	return !util_fits_cpu(cpu_util_cfs(cpu), 0, rq_util_max, cpu);
 }
 
 /*
@@ -6900,9 +7081,15 @@ static int sched_idle_rq(struct rq *rq)
 			rq->nr_running);
 }
 
-static int sched_idle_cpu(int cpu)
+static int choose_sched_idle_rq(struct rq *rq, struct task_struct *p)
+{
+	return sched_idle_rq(rq) && !task_has_idle_policy(p);
+}
+
+static int choose_idle_cpu(int cpu, struct task_struct *p)
 {
-	return sched_idle_rq(cpu_rq(cpu));
+	return available_idle_cpu(cpu) ||
+	       choose_sched_idle_rq(cpu_rq(cpu), p);
 }
 
 static void
@@ -7160,9 +7347,6 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
 		WARN_ON_ONCE(!task_sleep);
 		WARN_ON_ONCE(p->on_rq != 1);
 
-		/* Fix-up what dequeue_task_fair() skipped */
-		hrtick_update(rq);
-
 		/*
 		 * Fix-up what block_task() skipped.
 		 *
@@ -7196,8 +7380,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	/*
 	 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED).
 	 */
-
-	hrtick_update(rq);
 	return true;
 }
 
@@ -7467,7 +7649,7 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
 		if (!sched_core_cookie_match(rq, p))
 			continue;
 
-		if (sched_idle_cpu(i))
+		if (choose_sched_idle_rq(rq, p))
 			return i;
 
 		if (available_idle_cpu(i)) {
@@ -7558,8 +7740,7 @@ static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct tas
 
 static inline int __select_idle_cpu(int cpu, struct task_struct *p)
 {
-	if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
-	    sched_cpu_cookie_match(cpu_rq(cpu), p))
+	if (choose_idle_cpu(cpu, p) && sched_cpu_cookie_match(cpu_rq(cpu), p))
 		return cpu;
 
 	return -1;
@@ -7632,7 +7813,8 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu
 		if (!available_idle_cpu(cpu)) {
 			idle = false;
 			if (*idle_cpu == -1) {
-				if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) {
+				if (choose_sched_idle_rq(cpu_rq(cpu), p) &&
+				    cpumask_test_cpu(cpu, cpus)) {
 					*idle_cpu = cpu;
 					break;
 				}
@@ -7667,7 +7849,7 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t
 		 */
 		if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
 			continue;
-		if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
+		if (choose_idle_cpu(cpu, p))
 			return cpu;
 	}
 
@@ -7706,21 +7888,26 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 {
 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
 	int i, cpu, idle_cpu = -1, nr = INT_MAX;
-	struct sched_domain_shared *sd_share;
-
-	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
 
 	if (sched_feat(SIS_UTIL)) {
-		sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, target));
-		if (sd_share) {
-			/* because !--nr is the condition to stop scan */
-			nr = READ_ONCE(sd_share->nr_idle_scan) + 1;
-			/* overloaded LLC is unlikely to have idle cpu/core */
-			if (nr == 1)
-				return -1;
-		}
+		/*
+		 * Increment because !--nr is the condition to stop scan.
+		 *
+		 * Since "sd" is "sd_llc" for target CPU dereferenced in the
+		 * caller, it is safe to directly dereference "sd->shared".
+		 * Topology bits always ensure it assigned for "sd_llc" abd it
+		 * cannot disappear as long as we have a RCU protected
+		 * reference to one the associated "sd" here.
+		 */
+		nr = READ_ONCE(sd->shared->nr_idle_scan) + 1;
+		/* overloaded LLC is unlikely to have idle cpu/core */
+		if (nr == 1)
+			return -1;
 	}
 
+	if (!cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr))
+		return -1;
+
 	if (static_branch_unlikely(&sched_cluster_active)) {
 		struct sched_group *sg = sd->groups;
 
@@ -7789,7 +7976,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 	for_each_cpu_wrap(cpu, cpus, target) {
 		unsigned long cpu_cap = capacity_of(cpu);
 
-		if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu))
+		if (!choose_idle_cpu(cpu, p))
 			continue;
 
 		fits = util_fits_cpu(task_util, util_min, util_max, cpu);
@@ -7860,7 +8047,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 	 */
 	lockdep_assert_irqs_disabled();
 
-	if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&
+	if (choose_idle_cpu(target, p) &&
 	    asym_fits_cpu(task_util, util_min, util_max, target))
 		return target;
 
@@ -7868,7 +8055,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 	 * If the previous CPU is cache affine and idle, don't be stupid:
 	 */
 	if (prev != target && cpus_share_cache(prev, target) &&
-	    (available_idle_cpu(prev) || sched_idle_cpu(prev)) &&
+	    choose_idle_cpu(prev, p) &&
 	    asym_fits_cpu(task_util, util_min, util_max, prev)) {
 
 		if (!static_branch_unlikely(&sched_cluster_active) ||
@@ -7900,7 +8087,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 	if (recent_used_cpu != prev &&
 	    recent_used_cpu != target &&
 	    cpus_share_cache(recent_used_cpu, target) &&
-	    (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
+	    choose_idle_cpu(recent_used_cpu, p) &&
 	    cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) &&
 	    asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) {
 
@@ -8400,10 +8587,9 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 	struct perf_domain *pd;
 	struct energy_env eenv;
 
-	rcu_read_lock();
 	pd = rcu_dereference_all(rd->pd);
 	if (!pd)
-		goto unlock;
+		return target;
 
 	/*
 	 * Energy-aware wake-up happens on the lowest sched_domain starting
@@ -8413,13 +8599,13 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 	while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
 		sd = sd->parent;
 	if (!sd)
-		goto unlock;
+		return target;
 
 	target = prev_cpu;
 
 	sync_entity_load_avg(&p->se);
 	if (!task_util_est(p) && p_util_min == 0)
-		goto unlock;
+		return target;
 
 	eenv_task_busy_time(&eenv, p, prev_cpu);
 
@@ -8514,7 +8700,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 						    prev_cpu);
 			/* CPU utilization has changed */
 			if (prev_delta < base_energy)
-				goto unlock;
+				return target;
 			prev_delta -= base_energy;
 			prev_actual_cap = cpu_actual_cap;
 			best_delta = min(best_delta, prev_delta);
@@ -8538,7 +8724,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 						   max_spare_cap_cpu);
 			/* CPU utilization has changed */
 			if (cur_delta < base_energy)
-				goto unlock;
+				return target;
 			cur_delta -= base_energy;
 
 			/*
@@ -8555,7 +8741,6 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 			best_actual_cap = cpu_actual_cap;
 		}
 	}
-	rcu_read_unlock();
 
 	if ((best_fits > prev_fits) ||
 	    ((best_fits > 0) && (best_delta < prev_delta)) ||
@@ -8563,11 +8748,6 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 		target = best_energy_cpu;
 
 	return target;
-
-unlock:
-	rcu_read_unlock();
-
-	return target;
 }
 
 /*
@@ -8612,7 +8792,6 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
 		want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
 	}
 
-	rcu_read_lock();
 	for_each_domain(cpu, tmp) {
 		/*
 		 * If both 'cpu' and 'prev_cpu' are part of this domain,
@@ -8638,14 +8817,13 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
 			break;
 	}
 
-	if (unlikely(sd)) {
-		/* Slow path */
-		new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
-	} else if (wake_flags & WF_TTWU) { /* XXX always ? */
-		/* Fast path */
-		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
-	}
-	rcu_read_unlock();
+	/* Slow path */
+	if (unlikely(sd))
+		return sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
+
+	/* Fast path */
+	if (wake_flags & WF_TTWU)
+		return select_idle_sibling(p, prev_cpu, new_cpu);
 
 	return new_cpu;
 }
@@ -10047,6 +10225,7 @@ struct sg_lb_stats {
 	unsigned int group_asym_packing;	/* Tasks should be moved to preferred CPU */
 	unsigned int group_smt_balance;		/* Task on busy SMT be moved */
 	unsigned long group_misfit_task_load;	/* A CPU has a task too big for its capacity */
+	unsigned int group_overutilized;	/* At least one CPU is overutilized in the group */
 #ifdef CONFIG_NUMA_BALANCING
 	unsigned int nr_numa_running;
 	unsigned int nr_preferred_running;
@@ -10279,6 +10458,13 @@ group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
 static inline bool
 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
 {
+	/*
+	 * With EAS and uclamp, 1 CPU in the group must be overutilized to
+	 * consider the group overloaded.
+	 */
+	if (sched_energy_enabled() && !sgs->group_overutilized)
+		return false;
+
 	if (sgs->sum_nr_running <= sgs->group_weight)
 		return false;
 
@@ -10462,14 +10648,12 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)
  * @group: sched_group whose statistics are to be updated.
  * @sgs: variable to hold the statistics for this group.
  * @sg_overloaded: sched_group is overloaded
- * @sg_overutilized: sched_group is overutilized
  */
 static inline void update_sg_lb_stats(struct lb_env *env,
 				      struct sd_lb_stats *sds,
 				      struct sched_group *group,
 				      struct sg_lb_stats *sgs,
-				      bool *sg_overloaded,
-				      bool *sg_overutilized)
+				      bool *sg_overloaded)
 {
 	int i, nr_running, local_group, sd_flags = env->sd->flags;
 	bool balancing_at_rd = !env->sd->parent;
@@ -10491,7 +10675,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 		sgs->sum_nr_running += nr_running;
 
 		if (cpu_overutilized(i))
-			*sg_overutilized = 1;
+			sgs->group_overutilized = 1;
 
 		/*
 		 * No need to call idle_cpu() if nr_running is not 0
@@ -11067,6 +11251,7 @@ static void update_idle_cpu_scan(struct lb_env *env,
 				 unsigned long sum_util)
 {
 	struct sched_domain_shared *sd_share;
+	struct sched_domain *sd = env->sd;
 	int llc_weight, pct;
 	u64 x, y, tmp;
 	/*
@@ -11080,11 +11265,7 @@ static void update_idle_cpu_scan(struct lb_env *env,
 	if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)
 		return;
 
-	llc_weight = per_cpu(sd_llc_size, env->dst_cpu);
-	if (env->sd->span_weight != llc_weight)
-		return;
-
-	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, env->dst_cpu));
+	sd_share = sd->shared;
 	if (!sd_share)
 		return;
 
@@ -11118,10 +11299,11 @@ static void update_idle_cpu_scan(struct lb_env *env,
 	 */
 	/* equation [3] */
 	x = sum_util;
+	llc_weight = sd->span_weight;
 	do_div(x, llc_weight);
 
 	/* equation [4] */
-	pct = env->sd->imbalance_pct;
+	pct = sd->imbalance_pct;
 	tmp = x * x * pct * pct;
 	do_div(tmp, 10000 * SCHED_CAPACITY_SCALE);
 	tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE);
@@ -11162,13 +11344,15 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
 				update_group_capacity(env->sd, env->dst_cpu);
 		}
 
-		update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized);
+		update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded);
 
 		if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
 			sds->busiest = sg;
 			sds->busiest_stat = *sgs;
 		}
 
+		sg_overutilized |= sgs->group_overutilized;
+
 		/* Now, start updating sd_lb_stats */
 		sds->total_load += sgs->group_load;
 		sds->total_capacity += sgs->group_capacity;
@@ -12289,7 +12473,30 @@ static inline void update_newidle_stats(struct sched_domain *sd, unsigned int su
 	sd->newidle_success += success;
 
 	if (sd->newidle_call >= 1024) {
-		sd->newidle_ratio = sd->newidle_success;
+		u64 now = sched_clock();
+		s64 delta = now - sd->newidle_stamp;
+		sd->newidle_stamp = now;
+		int ratio = 0;
+
+		if (delta < 0)
+			delta = 0;
+
+		if (sched_feat(NI_RATE)) {
+			/*
+			 * ratio  delta   freq
+			 *
+			 * 1024 -  4  s -  128 Hz
+			 *  512 -  2  s -  256 Hz
+			 *  256 -  1  s -  512 Hz
+			 *  128 - .5  s - 1024 Hz
+			 *   64 - .25 s - 2048 Hz
+			 */
+			ratio = delta >> 22;
+		}
+
+		ratio += sd->newidle_success;
+
+		sd->newidle_ratio = min(1024, ratio);
 		sd->newidle_call /= 2;
 		sd->newidle_success /= 2;
 	}
@@ -12336,7 +12543,7 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)
 {
 	int continue_balancing = 1;
 	int cpu = rq->cpu;
-	int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
+	int busy = idle != CPU_IDLE && !sched_idle_rq(rq);
 	unsigned long interval;
 	struct sched_domain *sd;
 	/* Earliest time when we have to do rebalance again */
@@ -12374,7 +12581,7 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)
 				 * state even if we migrated tasks. Update it.
 				 */
 				idle = idle_cpu(cpu);
-				busy = !idle && !sched_idle_cpu(cpu);
+				busy = !idle && !sched_idle_rq(rq);
 			}
 			sd->last_balance = jiffies;
 			interval = get_sd_balance_interval(sd, busy);
@@ -12996,7 +13203,7 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
 		if (sd->flags & SD_BALANCE_NEWIDLE) {
 			unsigned int weight = 1;
 
-			if (sched_feat(NI_RANDOM)) {
+			if (sched_feat(NI_RANDOM) && sd->newidle_ratio < 1024) {
 				/*
 				 * Throw a 1k sided dice; and only run
 				 * newidle_balance according to the success
@@ -13439,11 +13646,8 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 		entity_tick(cfs_rq, se, queued);
 	}
 
-	if (queued) {
-		if (!need_resched())
-			hrtick_start_fair(rq, curr);
+	if (queued)
 		return;
-	}
 
 	if (static_branch_unlikely(&sched_numa_balancing))
 		task_tick_numa(rq, curr);
@@ -14025,7 +14229,7 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m)
 			tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];
 		}
 		if (ng) {
-			gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],
+			gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
 			gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
 		}
 		print_numa_stats(m, node, tsf, tpf, gsf, gpf);
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 136a6584be797..84c4fe3abd743 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -58,13 +58,20 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true)
 SCHED_FEAT(DELAY_DEQUEUE, true)
 SCHED_FEAT(DELAY_ZERO, true)
 
+SCHED_FEAT(PARANOID_AVG, false)
+
 /*
  * Allow wakeup-time preemption of the current task:
  */
 SCHED_FEAT(WAKEUP_PREEMPTION, true)
 
+#ifdef CONFIG_HRTIMER_REARM_DEFERRED
+SCHED_FEAT(HRTICK, true)
+SCHED_FEAT(HRTICK_DL, true)
+#else
 SCHED_FEAT(HRTICK, false)
 SCHED_FEAT(HRTICK_DL, false)
+#endif
 
 /*
  * Decrement CPU capacity based on time not spent running tasks
@@ -126,3 +133,4 @@ SCHED_FEAT(LATENCY_WARN, false)
  * Do newidle balancing proportional to its success rate using randomization.
  */
 SCHED_FEAT(NI_RANDOM, true)
+SCHED_FEAT(NI_RATE, true)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index b954491651227..a83be0c834ddb 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -161,6 +161,14 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 	return cpuidle_enter(drv, dev, next_state);
 }
 
+static void idle_call_stop_or_retain_tick(bool stop_tick)
+{
+	if (stop_tick || tick_nohz_tick_stopped())
+		tick_nohz_idle_stop_tick();
+	else
+		tick_nohz_idle_retain_tick();
+}
+
 /**
  * cpuidle_idle_call - the main idle function
  *
@@ -170,7 +178,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev,
  * set, and it returns with polling set.  If it ever stops polling, it
  * must clear the polling bit.
  */
-static void cpuidle_idle_call(void)
+static void cpuidle_idle_call(bool stop_tick)
 {
 	struct cpuidle_device *dev = cpuidle_get_device();
 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
@@ -186,7 +194,7 @@ static void cpuidle_idle_call(void)
 	}
 
 	if (cpuidle_not_available(drv, dev)) {
-		tick_nohz_idle_stop_tick();
+		idle_call_stop_or_retain_tick(stop_tick);
 
 		default_idle_call();
 		goto exit_idle;
@@ -222,17 +230,19 @@ static void cpuidle_idle_call(void)
 		next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns);
 		call_cpuidle(drv, dev, next_state);
 	} else if (drv->state_count > 1) {
-		bool stop_tick = true;
+		/*
+		 * stop_tick is expected to be true by default by cpuidle
+		 * governors, which allows them to select idle states with
+		 * target residency above the tick period length.
+		 */
+		stop_tick = true;
 
 		/*
 		 * Ask the cpuidle framework to choose a convenient idle state.
 		 */
 		next_state = cpuidle_select(drv, dev, &stop_tick);
 
-		if (stop_tick || tick_nohz_tick_stopped())
-			tick_nohz_idle_stop_tick();
-		else
-			tick_nohz_idle_retain_tick();
+		idle_call_stop_or_retain_tick(stop_tick);
 
 		entered_state = call_cpuidle(drv, dev, next_state);
 		/*
@@ -240,7 +250,7 @@ static void cpuidle_idle_call(void)
 		 */
 		cpuidle_reflect(dev, entered_state);
 	} else {
-		tick_nohz_idle_retain_tick();
+		idle_call_stop_or_retain_tick(stop_tick);
 
 		/*
 		 * If there is only a single idle state (or none), there is
@@ -268,6 +278,7 @@ static void cpuidle_idle_call(void)
 static void do_idle(void)
 {
 	int cpu = smp_processor_id();
+	bool got_tick = false;
 
 	/*
 	 * Check if we need to update blocked load
@@ -338,8 +349,9 @@ static void do_idle(void)
 			tick_nohz_idle_restart_tick();
 			cpu_idle_poll();
 		} else {
-			cpuidle_idle_call();
+			cpuidle_idle_call(got_tick);
 		}
+		got_tick = tick_nohz_idle_got_tick();
 		arch_cpu_idle_exit();
 	}
 
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index f69e1f16d9238..3d823f5ffe2c8 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1302,13 +1302,18 @@ update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
 			int flags)
 {
 	struct task_struct *p = NULL;
+	struct rq *rq = rq_of_rt_rq(rt_rq);
 
 	if (!schedstat_enabled())
 		return;
 
-	if (rt_entity_is_task(rt_se))
+	if (rt_entity_is_task(rt_se)) {
 		p = rt_task_of(rt_se);
 
+		if (p != rq->curr)
+			update_stats_wait_end_rt(rt_rq, rt_se);
+	}
+
 	if ((flags & DEQUEUE_SLEEP) && p) {
 		unsigned int state;
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 43bbf0693cca4..a890d8b8b050b 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -356,7 +356,7 @@ extern int  sched_dl_global_validate(void);
 extern void sched_dl_do_global(void);
 extern int  sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr);
 extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);
-extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
+extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr, unsigned int flags);
 extern bool __checkparam_dl(const struct sched_attr *attr);
 extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
 extern int  dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
@@ -684,8 +684,9 @@ struct cfs_rq {
 
 	s64			sum_w_vruntime;
 	u64			sum_weight;
-
 	u64			zero_vruntime;
+	unsigned int		sum_shift;
+
 #ifdef CONFIG_SCHED_CORE
 	unsigned int		forceidle_seq;
 	u64			zero_vruntime_fi;
@@ -782,7 +783,6 @@ enum scx_rq_flags {
 	SCX_RQ_ONLINE		= 1 << 0,
 	SCX_RQ_CAN_STOP_TICK	= 1 << 1,
 	SCX_RQ_BAL_KEEP		= 1 << 3, /* balance decided to keep current */
-	SCX_RQ_BYPASSING	= 1 << 4,
 	SCX_RQ_CLK_VALID	= 1 << 5, /* RQ clock is fresh and valid */
 	SCX_RQ_BAL_CB_PENDING	= 1 << 6, /* must queue a cb after dispatching */
 
@@ -800,17 +800,23 @@ struct scx_rq {
 	u32			cpuperf_target;		/* [0, SCHED_CAPACITY_SCALE] */
 	bool			cpu_released;
 	u32			flags;
+	u32			nr_immed;		/* ENQ_IMMED tasks on local_dsq */
 	u64			clock;			/* current per-rq clock -- see scx_bpf_now() */
 	cpumask_var_t		cpus_to_kick;
 	cpumask_var_t		cpus_to_kick_if_idle;
 	cpumask_var_t		cpus_to_preempt;
 	cpumask_var_t		cpus_to_wait;
 	unsigned long		kick_sync;
-	local_t			reenq_local_deferred;
+
+	struct task_struct	*sub_dispatch_prev;
+
+	raw_spinlock_t		deferred_reenq_lock;
+	u64			deferred_reenq_locals_seq;
+	struct list_head	deferred_reenq_locals;	/* scheds requesting reenq of local DSQ */
+	struct list_head	deferred_reenq_users;	/* user DSQs requesting reenq */
 	struct balance_callback	deferred_bal_cb;
 	struct irq_work		deferred_irq_work;
 	struct irq_work		kick_cpus_irq_work;
-	struct scx_dispatch_q	bypass_dsq;
 };
 #endif /* CONFIG_SCHED_CLASS_EXT */
 
@@ -1285,6 +1291,8 @@ struct rq {
 	call_single_data_t	hrtick_csd;
 	struct hrtimer		hrtick_timer;
 	ktime_t			hrtick_time;
+	ktime_t			hrtick_delay;
+	unsigned int		hrtick_sched;
 #endif
 
 #ifdef CONFIG_SCHEDSTATS
@@ -1606,15 +1614,18 @@ extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
 extern bool raw_spin_rq_trylock(struct rq *rq)
 	__cond_acquires(true, __rq_lockp(rq));
 
-extern void raw_spin_rq_unlock(struct rq *rq)
-	__releases(__rq_lockp(rq));
-
 static inline void raw_spin_rq_lock(struct rq *rq)
 	__acquires(__rq_lockp(rq))
 {
 	raw_spin_rq_lock_nested(rq, 0);
 }
 
+static inline void raw_spin_rq_unlock(struct rq *rq)
+	__releases(__rq_lockp(rq))
+{
+	raw_spin_unlock(rq_lockp(rq));
+}
+
 static inline void raw_spin_rq_lock_irq(struct rq *rq)
 	__acquires(__rq_lockp(rq))
 {
@@ -2849,7 +2860,7 @@ static inline void idle_set_state(struct rq *rq,
 
 static inline struct cpuidle_state *idle_get_state(struct rq *rq)
 {
-	WARN_ON_ONCE(!rcu_read_lock_held());
+	lockdep_assert(rcu_read_lock_any_held());
 
 	return rq->idle_state;
 }
@@ -3030,46 +3041,31 @@ extern unsigned int sysctl_numa_balancing_hot_threshold;
  *  - enabled by features
  *  - hrtimer is actually high res
  */
-static inline int hrtick_enabled(struct rq *rq)
+static inline bool hrtick_enabled(struct rq *rq)
 {
-	if (!cpu_active(cpu_of(rq)))
-		return 0;
-	return hrtimer_is_hres_active(&rq->hrtick_timer);
+	return cpu_active(cpu_of(rq)) && hrtimer_highres_enabled();
 }
 
-static inline int hrtick_enabled_fair(struct rq *rq)
+static inline bool hrtick_enabled_fair(struct rq *rq)
 {
-	if (!sched_feat(HRTICK))
-		return 0;
-	return hrtick_enabled(rq);
+	return sched_feat(HRTICK) && hrtick_enabled(rq);
 }
 
-static inline int hrtick_enabled_dl(struct rq *rq)
+static inline bool hrtick_enabled_dl(struct rq *rq)
 {
-	if (!sched_feat(HRTICK_DL))
-		return 0;
-	return hrtick_enabled(rq);
+	return sched_feat(HRTICK_DL) && hrtick_enabled(rq);
 }
 
 extern void hrtick_start(struct rq *rq, u64 delay);
-
-#else /* !CONFIG_SCHED_HRTICK: */
-
-static inline int hrtick_enabled_fair(struct rq *rq)
-{
-	return 0;
-}
-
-static inline int hrtick_enabled_dl(struct rq *rq)
+static inline bool hrtick_active(struct rq *rq)
 {
-	return 0;
-}
-
-static inline int hrtick_enabled(struct rq *rq)
-{
-	return 0;
+	return hrtimer_active(&rq->hrtick_timer);
 }
 
+#else /* !CONFIG_SCHED_HRTICK: */
+static inline bool hrtick_enabled_fair(struct rq *rq) { return false; }
+static inline bool hrtick_enabled_dl(struct rq *rq) { return false; }
+static inline bool hrtick_enabled(struct rq *rq) { return false; }
 #endif /* !CONFIG_SCHED_HRTICK */
 
 #ifndef arch_scale_freq_tick
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index cadb0e9fe19b9..b215b0ead9a60 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -911,10 +911,10 @@ static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *a
 	return -E2BIG;
 }
 
-static void get_params(struct task_struct *p, struct sched_attr *attr)
+static void get_params(struct task_struct *p, struct sched_attr *attr, unsigned int flags)
 {
 	if (task_has_dl_policy(p)) {
-		__getparam_dl(p, attr);
+		__getparam_dl(p, attr, flags);
 	} else if (task_has_rt_policy(p)) {
 		attr->sched_priority = p->rt_priority;
 	} else {
@@ -980,7 +980,7 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
 		return -ESRCH;
 
 	if (attr.sched_flags & SCHED_FLAG_KEEP_PARAMS)
-		get_params(p, &attr);
+		get_params(p, &attr, 0);
 
 	return sched_setattr(p, &attr);
 }
@@ -1065,7 +1065,7 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
 	int retval;
 
 	if (unlikely(!uattr || pid < 0 || usize > PAGE_SIZE ||
-		      usize < SCHED_ATTR_SIZE_VER0 || flags))
+		     usize < SCHED_ATTR_SIZE_VER0))
 		return -EINVAL;
 
 	scoped_guard (rcu) {
@@ -1073,6 +1073,12 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
 		if (!p)
 			return -ESRCH;
 
+		if (flags) {
+			if (!task_has_dl_policy(p) ||
+			    flags != SCHED_GETATTR_FLAG_DL_DYNAMIC)
+				return -EINVAL;
+		}
+
 		retval = security_task_getscheduler(p);
 		if (retval)
 			return retval;
@@ -1080,7 +1086,7 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
 		kattr.sched_policy = p->policy;
 		if (p->sched_reset_on_fork)
 			kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
-		get_params(p, &kattr);
+		get_params(p, &kattr, flags);
 		kattr.sched_flags &= SCHED_FLAG_ALL;
 
 #ifdef CONFIG_UCLAMP_TASK
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 32dcddaead82d..43150591914bd 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/sched/isolation.h>
+#include <linux/sched/clock.h>
 #include <linux/bsearch.h>
 #include "sched.h"
 
@@ -684,6 +685,9 @@ static void update_top_cache_domain(int cpu)
 	if (sd) {
 		id = cpumask_first(sched_domain_span(sd));
 		size = cpumask_weight(sched_domain_span(sd));
+
+		/* If sd_llc exists, sd_llc_shared should exist too. */
+		WARN_ON_ONCE(!sd->shared);
 		sds = sd->shared;
 	}
 
@@ -732,6 +736,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 		if (sd_parent_degenerate(tmp, parent)) {
 			tmp->parent = parent->parent;
 
+			/* Pick reference to parent->shared. */
+			if (parent->shared) {
+				WARN_ON_ONCE(tmp->shared);
+				tmp->shared = parent->shared;
+				parent->shared = NULL;
+			}
+
 			if (parent->parent) {
 				parent->parent->child = tmp;
 				parent->parent->groups->flags = tmp->flags;
@@ -781,6 +792,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 }
 
 struct s_data {
+	struct sched_domain_shared * __percpu *sds;
 	struct sched_domain * __percpu *sd;
 	struct root_domain	*rd;
 };
@@ -788,6 +800,7 @@ struct s_data {
 enum s_alloc {
 	sa_rootdomain,
 	sa_sd,
+	sa_sd_shared,
 	sa_sd_storage,
 	sa_none,
 };
@@ -1534,6 +1547,9 @@ static void set_domain_attribute(struct sched_domain *sd,
 static void __sdt_free(const struct cpumask *cpu_map);
 static int __sdt_alloc(const struct cpumask *cpu_map);
 
+static void __sds_free(struct s_data *d, const struct cpumask *cpu_map);
+static int __sds_alloc(struct s_data *d, const struct cpumask *cpu_map);
+
 static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
 				 const struct cpumask *cpu_map)
 {
@@ -1545,6 +1561,9 @@ static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
 	case sa_sd:
 		free_percpu(d->sd);
 		fallthrough;
+	case sa_sd_shared:
+		__sds_free(d, cpu_map);
+		fallthrough;
 	case sa_sd_storage:
 		__sdt_free(cpu_map);
 		fallthrough;
@@ -1560,9 +1579,11 @@ __visit_domain_allocation_hell(struct s_data *d, const struct cpumask *cpu_map)
 
 	if (__sdt_alloc(cpu_map))
 		return sa_sd_storage;
+	if (__sds_alloc(d, cpu_map))
+		return sa_sd_shared;
 	d->sd = alloc_percpu(struct sched_domain *);
 	if (!d->sd)
-		return sa_sd_storage;
+		return sa_sd_shared;
 	d->rd = alloc_rootdomain();
 	if (!d->rd)
 		return sa_sd;
@@ -1575,21 +1596,25 @@ __visit_domain_allocation_hell(struct s_data *d, const struct cpumask *cpu_map)
  * sched_group structure so that the subsequent __free_domain_allocs()
  * will not free the data we're using.
  */
-static void claim_allocations(int cpu, struct sched_domain *sd)
+static void claim_allocations(int cpu, struct s_data *d)
 {
-	struct sd_data *sdd = sd->private;
+	struct sched_domain *sd;
+
+	if (atomic_read(&(*per_cpu_ptr(d->sds, cpu))->ref))
+		*per_cpu_ptr(d->sds, cpu) = NULL;
 
-	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
-	*per_cpu_ptr(sdd->sd, cpu) = NULL;
+	for (sd = *per_cpu_ptr(d->sd, cpu); sd; sd = sd->parent) {
+		struct sd_data *sdd = sd->private;
 
-	if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
-		*per_cpu_ptr(sdd->sds, cpu) = NULL;
+		WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
+		*per_cpu_ptr(sdd->sd, cpu) = NULL;
 
-	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
-		*per_cpu_ptr(sdd->sg, cpu) = NULL;
+		if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
+			*per_cpu_ptr(sdd->sg, cpu) = NULL;
 
-	if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
-		*per_cpu_ptr(sdd->sgc, cpu) = NULL;
+		if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
+			*per_cpu_ptr(sdd->sgc, cpu) = NULL;
+	}
 }
 
 #ifdef CONFIG_NUMA
@@ -1642,14 +1667,19 @@ sd_init(struct sched_domain_topology_level *tl,
 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
 	int sd_id, sd_weight, sd_flags = 0;
 	struct cpumask *sd_span;
+	u64 now = sched_clock();
 
-	sd_weight = cpumask_weight(tl->mask(tl, cpu));
+	sd_span = sched_domain_span(sd);
+	cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu));
+	sd_weight = cpumask_weight(sd_span);
+	sd_id = cpumask_first(sd_span);
 
 	if (tl->sd_flags)
 		sd_flags = (*tl->sd_flags)();
 	if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
-			"wrong sd_flags in topology description\n"))
+		      "wrong sd_flags in topology description\n"))
 		sd_flags &= TOPOLOGY_SD_FLAGS;
+	sd_flags |= asym_cpu_capacity_classify(sd_span, cpu_map);
 
 	*sd = (struct sched_domain){
 		.min_interval		= sd_weight,
@@ -1679,6 +1709,7 @@ sd_init(struct sched_domain_topology_level *tl,
 		.newidle_call		= 512,
 		.newidle_success	= 256,
 		.newidle_ratio		= 512,
+		.newidle_stamp		= now,
 
 		.max_newidle_lb_cost	= 0,
 		.last_decay_max_lb_cost	= jiffies,
@@ -1686,12 +1717,6 @@ sd_init(struct sched_domain_topology_level *tl,
 		.name			= tl->name,
 	};
 
-	sd_span = sched_domain_span(sd);
-	cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu));
-	sd_id = cpumask_first(sd_span);
-
-	sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map);
-
 	WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) ==
 		  (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY),
 		  "CPU capacity asymmetry not supported on SMT\n");
@@ -1727,16 +1752,6 @@ sd_init(struct sched_domain_topology_level *tl,
 		sd->cache_nice_tries = 1;
 	}
 
-	/*
-	 * For all levels sharing cache; connect a sched_domain_shared
-	 * instance.
-	 */
-	if (sd->flags & SD_SHARE_LLC) {
-		sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
-		atomic_inc(&sd->shared->ref);
-		atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
-	}
-
 	sd->private = sdd;
 
 	return sd;
@@ -2372,10 +2387,6 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
 		if (!sdd->sd)
 			return -ENOMEM;
 
-		sdd->sds = alloc_percpu(struct sched_domain_shared *);
-		if (!sdd->sds)
-			return -ENOMEM;
-
 		sdd->sg = alloc_percpu(struct sched_group *);
 		if (!sdd->sg)
 			return -ENOMEM;
@@ -2386,7 +2397,6 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
 
 		for_each_cpu(j, cpu_map) {
 			struct sched_domain *sd;
-			struct sched_domain_shared *sds;
 			struct sched_group *sg;
 			struct sched_group_capacity *sgc;
 
@@ -2397,13 +2407,6 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
 
 			*per_cpu_ptr(sdd->sd, j) = sd;
 
-			sds = kzalloc_node(sizeof(struct sched_domain_shared),
-					GFP_KERNEL, cpu_to_node(j));
-			if (!sds)
-				return -ENOMEM;
-
-			*per_cpu_ptr(sdd->sds, j) = sds;
-
 			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
 					GFP_KERNEL, cpu_to_node(j));
 			if (!sg)
@@ -2445,8 +2448,6 @@ static void __sdt_free(const struct cpumask *cpu_map)
 				kfree(*per_cpu_ptr(sdd->sd, j));
 			}
 
-			if (sdd->sds)
-				kfree(*per_cpu_ptr(sdd->sds, j));
 			if (sdd->sg)
 				kfree(*per_cpu_ptr(sdd->sg, j));
 			if (sdd->sgc)
@@ -2454,8 +2455,6 @@ static void __sdt_free(const struct cpumask *cpu_map)
 		}
 		free_percpu(sdd->sd);
 		sdd->sd = NULL;
-		free_percpu(sdd->sds);
-		sdd->sds = NULL;
 		free_percpu(sdd->sg);
 		sdd->sg = NULL;
 		free_percpu(sdd->sgc);
@@ -2463,6 +2462,42 @@ static void __sdt_free(const struct cpumask *cpu_map)
 	}
 }
 
+static int __sds_alloc(struct s_data *d, const struct cpumask *cpu_map)
+{
+	int j;
+
+	d->sds = alloc_percpu(struct sched_domain_shared *);
+	if (!d->sds)
+		return -ENOMEM;
+
+	for_each_cpu(j, cpu_map) {
+		struct sched_domain_shared *sds;
+
+		sds = kzalloc_node(sizeof(struct sched_domain_shared),
+				GFP_KERNEL, cpu_to_node(j));
+		if (!sds)
+			return -ENOMEM;
+
+		*per_cpu_ptr(d->sds, j) = sds;
+	}
+
+	return 0;
+}
+
+static void __sds_free(struct s_data *d, const struct cpumask *cpu_map)
+{
+	int j;
+
+	if (!d->sds)
+		return;
+
+	for_each_cpu(j, cpu_map)
+		kfree(*per_cpu_ptr(d->sds, j));
+
+	free_percpu(d->sds);
+	d->sds = NULL;
+}
+
 static struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
 		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
 		struct sched_domain *child, int cpu)
@@ -2548,6 +2583,74 @@ static bool topology_span_sane(const struct cpumask *cpu_map)
 	return true;
 }
 
+/*
+ * Calculate an allowed NUMA imbalance such that LLCs do not get
+ * imbalanced.
+ */
+static void adjust_numa_imbalance(struct sched_domain *sd_llc)
+{
+	struct sched_domain *parent;
+	unsigned int imb_span = 1;
+	unsigned int imb = 0;
+	unsigned int nr_llcs;
+
+	WARN_ON(!(sd_llc->flags & SD_SHARE_LLC));
+	WARN_ON(!sd_llc->parent);
+
+	/*
+	 * For a single LLC per node, allow an
+	 * imbalance up to 12.5% of the node. This is
+	 * arbitrary cutoff based two factors -- SMT and
+	 * memory channels. For SMT-2, the intent is to
+	 * avoid premature sharing of HT resources but
+	 * SMT-4 or SMT-8 *may* benefit from a different
+	 * cutoff. For memory channels, this is a very
+	 * rough estimate of how many channels may be
+	 * active and is based on recent CPUs with
+	 * many cores.
+	 *
+	 * For multiple LLCs, allow an imbalance
+	 * until multiple tasks would share an LLC
+	 * on one node while LLCs on another node
+	 * remain idle. This assumes that there are
+	 * enough logical CPUs per LLC to avoid SMT
+	 * factors and that there is a correlation
+	 * between LLCs and memory channels.
+	 */
+	nr_llcs = sd_llc->parent->span_weight / sd_llc->span_weight;
+	if (nr_llcs == 1)
+		imb = sd_llc->parent->span_weight >> 3;
+	else
+		imb = nr_llcs;
+
+	imb = max(1U, imb);
+	sd_llc->parent->imb_numa_nr = imb;
+
+	/*
+	 * Set span based on the first NUMA domain.
+	 *
+	 * NUMA systems always add a NODE domain before
+	 * iterating the NUMA domains. Since this is before
+	 * degeneration, start from sd_llc's parent's
+	 * parent which is the lowest an SD_NUMA domain can
+	 * be relative to sd_llc.
+	 */
+	parent = sd_llc->parent->parent;
+	while (parent && !(parent->flags & SD_NUMA))
+		parent = parent->parent;
+
+	imb_span = parent ? parent->span_weight : sd_llc->parent->span_weight;
+
+	/* Update the upper remainder of the topology */
+	parent = sd_llc->parent;
+	while (parent) {
+		int factor = max(1U, (parent->span_weight / imb_span));
+
+		parent->imb_numa_nr = imb * factor;
+		parent = parent->parent;
+	}
+}
+
 /*
  * Build sched domains for a given set of CPUs and attach the sched domains
  * to the individual CPUs
@@ -2605,61 +2708,28 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 		}
 	}
 
-	/*
-	 * Calculate an allowed NUMA imbalance such that LLCs do not get
-	 * imbalanced.
-	 */
 	for_each_cpu(i, cpu_map) {
-		unsigned int imb = 0;
-		unsigned int imb_span = 1;
+		sd = *per_cpu_ptr(d.sd, i);
+		if (!sd)
+			continue;
 
-		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
-			struct sched_domain *child = sd->child;
-
-			if (!(sd->flags & SD_SHARE_LLC) && child &&
-			    (child->flags & SD_SHARE_LLC)) {
-				struct sched_domain __rcu *top_p;
-				unsigned int nr_llcs;
-
-				/*
-				 * For a single LLC per node, allow an
-				 * imbalance up to 12.5% of the node. This is
-				 * arbitrary cutoff based two factors -- SMT and
-				 * memory channels. For SMT-2, the intent is to
-				 * avoid premature sharing of HT resources but
-				 * SMT-4 or SMT-8 *may* benefit from a different
-				 * cutoff. For memory channels, this is a very
-				 * rough estimate of how many channels may be
-				 * active and is based on recent CPUs with
-				 * many cores.
-				 *
-				 * For multiple LLCs, allow an imbalance
-				 * until multiple tasks would share an LLC
-				 * on one node while LLCs on another node
-				 * remain idle. This assumes that there are
-				 * enough logical CPUs per LLC to avoid SMT
-				 * factors and that there is a correlation
-				 * between LLCs and memory channels.
-				 */
-				nr_llcs = sd->span_weight / child->span_weight;
-				if (nr_llcs == 1)
-					imb = sd->span_weight >> 3;
-				else
-					imb = nr_llcs;
-				imb = max(1U, imb);
-				sd->imb_numa_nr = imb;
-
-				/* Set span based on the first NUMA domain. */
-				top_p = sd->parent;
-				while (top_p && !(top_p->flags & SD_NUMA)) {
-					top_p = top_p->parent;
-				}
-				imb_span = top_p ? top_p->span_weight : sd->span_weight;
-			} else {
-				int factor = max(1U, (sd->span_weight / imb_span));
+		/* First, find the topmost SD_SHARE_LLC domain */
+		while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
+			sd = sd->parent;
 
-				sd->imb_numa_nr = imb * factor;
-			}
+		if (sd->flags & SD_SHARE_LLC) {
+			int sd_id = cpumask_first(sched_domain_span(sd));
+
+			sd->shared = *per_cpu_ptr(d.sds, sd_id);
+			atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
+			atomic_inc(&sd->shared->ref);
+
+			/*
+			 * In presence of higher domains, adjust the
+			 * NUMA imbalance stats for the hierarchy.
+			 */
+			if (IS_ENABLED(CONFIG_NUMA) && sd->parent)
+				adjust_numa_imbalance(sd);
 		}
 	}
 
@@ -2668,10 +2738,10 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 		if (!cpumask_test_cpu(i, cpu_map))
 			continue;
 
-		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
-			claim_allocations(i, sd);
+		claim_allocations(i, &d);
+
+		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent)
 			init_sched_groups_capacity(i, sd);
-		}
 	}
 
 	/* Attach the domains */
diff --git a/kernel/signal.c b/kernel/signal.c
index d65d0fe24bfb8..32100bcc0a50c 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -1000,9 +1000,7 @@ static void complete_signal(int sig, struct task_struct *p, enum pid_type type)
 	 * Found a killable thread.  If the signal will be fatal,
 	 * then start taking the whole group down immediately.
 	 */
-	if (sig_fatal(p, sig) &&
-	    (signal->core_state || !(signal->flags & SIGNAL_GROUP_EXIT)) &&
-	    !sigismember(&t->real_blocked, sig) &&
+	if (sig_fatal(p, sig) && !sigismember(&t->real_blocked, sig) &&
 	    (sig == SIGKILL || !p->ptrace)) {
 		/*
 		 * This signal will be fatal to the whole group.
@@ -2173,13 +2171,13 @@ bool do_notify_parent(struct task_struct *tsk, int sig)
 	bool autoreap = false;
 	u64 utime, stime;
 
-	WARN_ON_ONCE(sig == -1);
+	if (WARN_ON_ONCE(!valid_signal(sig)))
+		return false;
 
 	/* do_notify_parent_cldstop should have been called instead.  */
 	WARN_ON_ONCE(task_is_stopped_or_traced(tsk));
 
-	WARN_ON_ONCE(!tsk->ptrace &&
-	       (tsk->group_leader != tsk || !thread_group_empty(tsk)));
+	WARN_ON_ONCE(!tsk->ptrace && !thread_group_empty(tsk));
 
 	/* ptraced, or group-leader without sub-threads */
 	do_notify_pidfd(tsk);
@@ -2251,11 +2249,15 @@ bool do_notify_parent(struct task_struct *tsk, int sig)
 		if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN)
 			sig = 0;
 	}
+	if (!tsk->ptrace && tsk->signal->autoreap) {
+		autoreap = true;
+		sig = 0;
+	}
 	/*
 	 * Send with __send_signal as si_pid and si_uid are in the
 	 * parent's namespaces.
 	 */
-	if (valid_signal(sig) && sig)
+	if (sig)
 		__send_signal_locked(sig, &info, tsk->parent, PIDTYPE_TGID, false);
 	__wake_up_parent(tsk, tsk->parent);
 	spin_unlock_irqrestore(&psig->siglock, flags);
diff --git a/kernel/smp.c b/kernel/smp.c
index f349960f79cad..09521cb38a55c 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -377,6 +377,20 @@ static __always_inline void csd_unlock(call_single_data_t *csd)
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
 
+#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
+static call_single_data_t *get_single_csd_data(int cpu)
+{
+	if (static_branch_unlikely(&csdlock_debug_enabled))
+		return per_cpu_ptr(&csd_data, cpu);
+	return this_cpu_ptr(&csd_data);
+}
+#else
+static call_single_data_t *get_single_csd_data(int cpu)
+{
+	return this_cpu_ptr(&csd_data);
+}
+#endif
+
 void __smp_call_single_queue(int cpu, struct llist_node *node)
 {
 	/*
@@ -670,7 +684,7 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
 
 	csd = &csd_stack;
 	if (!wait) {
-		csd = this_cpu_ptr(&csd_data);
+		csd = get_single_csd_data(cpu);
 		csd_lock(csd);
 	}
 
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 77198911b8dd4..a3bd54de82595 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -576,6 +576,10 @@ static inline bool lockdep_softirq_start(void) { return false; }
 static inline void lockdep_softirq_end(bool in_hardirq) { }
 #endif
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void handle_softirqs(bool ksirqd);
+#endif
+
 static void handle_softirqs(bool ksirqd)
 {
 	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
@@ -663,6 +667,13 @@ void irq_enter_rcu(void)
 {
 	__irq_enter_raw();
 
+	/*
+	 * If this is a nested interrupt that hits the exit_to_user_mode_loop
+	 * where it has enabled interrupts but before it has hit schedule() we
+	 * could have hrtimers in an undefined state. Fix it up here.
+	 */
+	hrtimer_rearm_deferred();
+
 	if (tick_nohz_full_cpu(smp_processor_id()) ||
 	    (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))
 		tick_irq_enter();
@@ -719,8 +730,14 @@ static inline void __irq_exit_rcu(void)
 #endif
 	account_hardirq_exit(current);
 	preempt_count_sub(HARDIRQ_OFFSET);
-	if (!in_interrupt() && local_softirq_pending())
+	if (!in_interrupt() && local_softirq_pending()) {
+		/*
+		 * If we left hrtimers unarmed, make sure to arm them now,
+		 * before enabling interrupts to run SoftIRQ.
+		 */
+		hrtimer_rearm_deferred();
 		invoke_softirq();
+	}
 
 	if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() &&
 	    local_timers_pending_force_th() && !(in_nmi() | in_hardirq()))
diff --git a/kernel/sys.c b/kernel/sys.c
index c86eba9aa7e9d..3ed35372aae9d 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -2147,18 +2147,11 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data
 	mm->arg_end	= prctl_map.arg_end;
 	mm->env_start	= prctl_map.env_start;
 	mm->env_end	= prctl_map.env_end;
-	spin_unlock(&mm->arg_lock);
-
-	/*
-	 * Note this update of @saved_auxv is lockless thus
-	 * if someone reads this member in procfs while we're
-	 * updating -- it may get partly updated results. It's
-	 * known and acceptable trade off: we leave it as is to
-	 * not introduce additional locks here making the kernel
-	 * more complex.
-	 */
-	if (prctl_map.auxv_size)
+	if (prctl_map.auxv_size) {
+		mm_flags_set(MMF_USER_HWCAP, mm);
 		memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv));
+	}
+	spin_unlock(&mm->arg_lock);
 
 	mmap_read_unlock(mm);
 	return 0;
@@ -2188,9 +2181,10 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr,
 
 	BUILD_BUG_ON(sizeof(user_auxv) != sizeof(mm->saved_auxv));
 
-	task_lock(current);
+	spin_lock(&mm->arg_lock);
+	mm_flags_set(MMF_USER_HWCAP, mm);
 	memcpy(mm->saved_auxv, user_auxv, len);
-	task_unlock(current);
+	spin_unlock(&mm->arg_lock);
 
 	return 0;
 }
@@ -2478,9 +2472,17 @@ static inline int prctl_get_mdwe(unsigned long arg2, unsigned long arg3,
 static int prctl_get_auxv(void __user *addr, unsigned long len)
 {
 	struct mm_struct *mm = current->mm;
+	unsigned long auxv[AT_VECTOR_SIZE];
 	unsigned long size = min_t(unsigned long, sizeof(mm->saved_auxv), len);
 
-	if (size && copy_to_user(addr, mm->saved_auxv, size))
+	if (!size)
+		return sizeof(mm->saved_auxv);
+
+	spin_lock(&mm->arg_lock);
+	memcpy(auxv, mm->saved_auxv, size);
+	spin_unlock(&mm->arg_lock);
+
+	if (copy_to_user(addr, auxv, size))
 		return -EFAULT;
 	return sizeof(mm->saved_auxv);
 }
diff --git a/kernel/time/.kunitconfig b/kernel/time/.kunitconfig
new file mode 100644
index 0000000000000..d60a611b28537
--- /dev/null
+++ b/kernel/time/.kunitconfig
@@ -0,0 +1,2 @@
+CONFIG_KUNIT=y
+CONFIG_TIME_KUNIT_TEST=y
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 7c6a52f7836ce..f9461586065c7 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -9,14 +9,13 @@
 config CLOCKSOURCE_WATCHDOG
 	bool
 
-# Architecture has extra clocksource data
-config ARCH_CLOCKSOURCE_DATA
-	bool
-
 # Architecture has extra clocksource init called from registration
 config ARCH_CLOCKSOURCE_INIT
 	bool
 
+config ARCH_WANTS_CLOCKSOURCE_READ_INLINE
+	bool
+
 # Timekeeping vsyscall support
 config GENERIC_TIME_VSYSCALL
 	bool
@@ -44,10 +43,23 @@ config GENERIC_CLOCKEVENTS_BROADCAST_IDLE
 config GENERIC_CLOCKEVENTS_MIN_ADJUST
 	bool
 
+config GENERIC_CLOCKEVENTS_COUPLED
+	bool
+
+config GENERIC_CLOCKEVENTS_COUPLED_INLINE
+	select GENERIC_CLOCKEVENTS_COUPLED
+	bool
+
 # Generic update of CMOS clock
 config GENERIC_CMOS_UPDATE
 	bool
 
+# Deferred rearming of the hrtimer interrupt
+config HRTIMER_REARM_DEFERRED
+       def_bool y
+       depends on GENERIC_ENTRY && HAVE_GENERIC_TIF_BITS
+       depends on HIGH_RES_TIMERS && SCHED_HRTICK
+
 # Select to handle posix CPU timers from task_work
 # and not from the timer interrupt context
 config HAVE_POSIX_CPU_TIMERS_TASK_WORK
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index eaae1ce9f0600..83712aa1d385c 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -292,6 +292,38 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
 
 #endif /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */
 
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_COUPLED
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_COUPLED_INLINE
+#include <asm/clock_inlined.h>
+#else
+static __always_inline void
+arch_inlined_clockevent_set_next_coupled(u64 u64 cycles, struct clock_event_device *dev) { }
+#endif
+
+static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires)
+{
+	u64 cycles;
+
+	if (unlikely(!(dev->features & CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED)))
+		return false;
+
+	if (unlikely(!ktime_expiry_to_cycles(dev->cs_id, expires, &cycles)))
+		return false;
+
+	if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED_INLINE))
+		arch_inlined_clockevent_set_next_coupled(cycles, dev);
+	else
+		dev->set_next_coupled(cycles, dev);
+	return true;
+}
+
+#else
+static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires)
+{
+	return false;
+}
+#endif
+
 /**
  * clockevents_program_event - Reprogram the clock event device.
  * @dev:	device to program
@@ -300,11 +332,10 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
  *
  * Returns 0 on success, -ETIME when the event is in the past.
  */
-int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
-			      bool force)
+int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, bool force)
 {
-	unsigned long long clc;
 	int64_t delta;
+	u64 cycles;
 	int rc;
 
 	if (WARN_ON_ONCE(expires < 0))
@@ -319,10 +350,13 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
 	WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n",
 		  clockevent_get_state(dev));
 
-	/* Shortcut for clockevent devices that can deal with ktime. */
-	if (dev->features & CLOCK_EVT_FEAT_KTIME)
+	/* ktime_t based reprogramming for the broadcast hrtimer device */
+	if (unlikely(dev->features & CLOCK_EVT_FEAT_HRTIMER))
 		return dev->set_next_ktime(expires, dev);
 
+	if (likely(clockevent_set_next_coupled(dev, expires)))
+		return 0;
+
 	delta = ktime_to_ns(ktime_sub(expires, ktime_get()));
 	if (delta <= 0)
 		return force ? clockevents_program_min_delta(dev) : -ETIME;
@@ -330,8 +364,8 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
 	delta = min(delta, (int64_t) dev->max_delta_ns);
 	delta = max(delta, (int64_t) dev->min_delta_ns);
 
-	clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
-	rc = dev->set_next_event((unsigned long) clc, dev);
+	cycles = ((u64)delta * dev->mult) >> dev->shift;
+	rc = dev->set_next_event((unsigned long) cycles, dev);
 
 	return (rc && force) ? clockevents_program_min_delta(dev) : rc;
 }
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index df71949616584..e753a0632ac80 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -651,6 +651,13 @@ static void clocksource_select_watchdog(bool fallback)
 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
 			continue;
 
+		/*
+		 * If it's not continuous, don't put the fox in charge of
+		 * the henhouse.
+		 */
+		if (!(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS))
+			continue;
+
 		/* Skip current if we were requested for a fallback. */
 		if (fallback && cs == old_wd)
 			continue;
@@ -1169,6 +1176,9 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq
 
 		clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,
 				       NSEC_PER_SEC / scale, sec * scale);
+
+		/* Update cs::freq_khz */
+		cs->freq_khz = div_u64((u64)freq * scale, 1000);
 	}
 
 	/*
@@ -1241,6 +1251,10 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
 
 	if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))
 		cs->id = CSID_GENERIC;
+
+	if (WARN_ON_ONCE(!freq && cs->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT))
+		cs->flags &= ~CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT;
+
 	if (cs->vdso_clock_mode < 0 ||
 	    cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) {
 		pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n",
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 860af7a584289..2db171bae05ce 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -49,6 +49,28 @@
 
 #include "tick-internal.h"
 
+/*
+ * Constants to set the queued state of the timer (INACTIVE, ENQUEUED)
+ *
+ * The callback state is kept separate in the CPU base because having it in
+ * the timer would required touching the timer after the callback, which
+ * makes it impossible to free the timer from the callback function.
+ *
+ * Therefore we track the callback state in:
+ *
+ *	timer->base->cpu_base->running == timer
+ *
+ * On SMP it is possible to have a "callback function running and enqueued"
+ * status. It happens for example when a posix timer expired and the callback
+ * queued a signal. Between dropping the lock which protects the posix timer
+ * and reacquiring the base lock of the hrtimer, another CPU can deliver the
+ * signal and rearm the timer.
+ *
+ * All state transitions are protected by cpu_base->lock.
+ */
+#define HRTIMER_STATE_INACTIVE	false
+#define HRTIMER_STATE_ENQUEUED	true
+
 /*
  * The resolution of the clocks. The resolution value is returned in
  * the clock_getres() system call to give application programmers an
@@ -77,43 +99,22 @@ static ktime_t __hrtimer_cb_get_time(clockid_t clock_id);
  * to reach a base using a clockid, hrtimer_clockid_to_base()
  * is used to convert from clockid to the proper hrtimer_base_type.
  */
+
+#define BASE_INIT(idx, cid)			\
+	[idx] = { .index = idx, .clockid = cid }
+
 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
 {
 	.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
-	.clock_base =
-	{
-		{
-			.index = HRTIMER_BASE_MONOTONIC,
-			.clockid = CLOCK_MONOTONIC,
-		},
-		{
-			.index = HRTIMER_BASE_REALTIME,
-			.clockid = CLOCK_REALTIME,
-		},
-		{
-			.index = HRTIMER_BASE_BOOTTIME,
-			.clockid = CLOCK_BOOTTIME,
-		},
-		{
-			.index = HRTIMER_BASE_TAI,
-			.clockid = CLOCK_TAI,
-		},
-		{
-			.index = HRTIMER_BASE_MONOTONIC_SOFT,
-			.clockid = CLOCK_MONOTONIC,
-		},
-		{
-			.index = HRTIMER_BASE_REALTIME_SOFT,
-			.clockid = CLOCK_REALTIME,
-		},
-		{
-			.index = HRTIMER_BASE_BOOTTIME_SOFT,
-			.clockid = CLOCK_BOOTTIME,
-		},
-		{
-			.index = HRTIMER_BASE_TAI_SOFT,
-			.clockid = CLOCK_TAI,
-		},
+	.clock_base = {
+		BASE_INIT(HRTIMER_BASE_MONOTONIC,	CLOCK_MONOTONIC),
+		BASE_INIT(HRTIMER_BASE_REALTIME,	CLOCK_REALTIME),
+		BASE_INIT(HRTIMER_BASE_BOOTTIME,	CLOCK_BOOTTIME),
+		BASE_INIT(HRTIMER_BASE_TAI,		CLOCK_TAI),
+		BASE_INIT(HRTIMER_BASE_MONOTONIC_SOFT,	CLOCK_MONOTONIC),
+		BASE_INIT(HRTIMER_BASE_REALTIME_SOFT,	CLOCK_REALTIME),
+		BASE_INIT(HRTIMER_BASE_BOOTTIME_SOFT,	CLOCK_BOOTTIME),
+		BASE_INIT(HRTIMER_BASE_TAI_SOFT,	CLOCK_TAI),
 	},
 	.csd = CSD_INIT(retrigger_next_event, NULL)
 };
@@ -126,23 +127,43 @@ static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base)
 		return likely(base->online);
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+DEFINE_STATIC_KEY_FALSE(hrtimer_highres_enabled_key);
+
+static void hrtimer_hres_workfn(struct work_struct *work)
+{
+	static_branch_enable(&hrtimer_highres_enabled_key);
+}
+
+static DECLARE_WORK(hrtimer_hres_work, hrtimer_hres_workfn);
+
+static inline void hrtimer_schedule_hres_work(void)
+{
+	if (!hrtimer_highres_enabled())
+		schedule_work(&hrtimer_hres_work);
+}
+#else
+static inline void hrtimer_schedule_hres_work(void) { }
+#endif
+
 /*
  * Functions and macros which are different for UP/SMP systems are kept in a
  * single place
  */
 #ifdef CONFIG_SMP
-
 /*
  * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
  * such that hrtimer_callback_running() can unconditionally dereference
  * timer->base->cpu_base
  */
 static struct hrtimer_cpu_base migration_cpu_base = {
-	.clock_base = { {
-		.cpu_base = &migration_cpu_base,
-		.seq      = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
-						     &migration_cpu_base.lock),
-	}, },
+	.clock_base = {
+		[0] = {
+			.cpu_base = &migration_cpu_base,
+			.seq      = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
+							     &migration_cpu_base.lock),
+		},
+	},
 };
 
 #define migration_base	migration_cpu_base.clock_base[0]
@@ -159,15 +180,13 @@ static struct hrtimer_cpu_base migration_cpu_base = {
  * possible to set timer->base = &migration_base and drop the lock: the timer
  * remains locked.
  */
-static
-struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
-					     unsigned long *flags)
+static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
+						    unsigned long *flags)
 	__acquires(&timer->base->lock)
 {
-	struct hrtimer_clock_base *base;
-
 	for (;;) {
-		base = READ_ONCE(timer->base);
+		struct hrtimer_clock_base *base = READ_ONCE(timer->base);
+
 		if (likely(base != &migration_base)) {
 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
 			if (likely(base == timer->base))
@@ -220,7 +239,7 @@ static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_
 	return expires >= new_base->cpu_base->expires_next;
 }
 
-static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, int pinned)
+static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, bool pinned)
 {
 	if (!hrtimer_base_is_online(base)) {
 		int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));
@@ -248,8 +267,7 @@ static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *
  * the timer callback is currently running.
  */
 static inline struct hrtimer_clock_base *
-switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
-		    int pinned)
+switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, bool pinned)
 {
 	struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
 	struct hrtimer_clock_base *new_base;
@@ -262,13 +280,12 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
 
 	if (base != new_base) {
 		/*
-		 * We are trying to move timer to new_base.
-		 * However we can't change timer's base while it is running,
-		 * so we keep it on the same CPU. No hassle vs. reprogramming
-		 * the event source in the high resolution case. The softirq
-		 * code will take care of this when the timer function has
-		 * completed. There is no conflict as we hold the lock until
-		 * the timer is enqueued.
+		 * We are trying to move timer to new_base. However we can't
+		 * change timer's base while it is running, so we keep it on
+		 * the same CPU. No hassle vs. reprogramming the event source
+		 * in the high resolution case. The remote CPU will take care
+		 * of this when the timer function has completed. There is no
+		 * conflict as we hold the lock until the timer is enqueued.
 		 */
 		if (unlikely(hrtimer_callback_running(timer)))
 			return base;
@@ -278,8 +295,7 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
 		raw_spin_unlock(&base->cpu_base->lock);
 		raw_spin_lock(&new_base->cpu_base->lock);
 
-		if (!hrtimer_suitable_target(timer, new_base, new_cpu_base,
-					     this_cpu_base)) {
+		if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) {
 			raw_spin_unlock(&new_base->cpu_base->lock);
 			raw_spin_lock(&base->cpu_base->lock);
 			new_cpu_base = this_cpu_base;
@@ -298,14 +314,13 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
 
 #else /* CONFIG_SMP */
 
-static inline struct hrtimer_clock_base *
-lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
+static inline struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
+							   unsigned long *flags)
 	__acquires(&timer->base->cpu_base->lock)
 {
 	struct hrtimer_clock_base *base = timer->base;
 
 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
-
 	return base;
 }
 
@@ -422,12 +437,37 @@ static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
 	}
 }
 
+/* Stub timer callback for improperly used timers. */
+static enum hrtimer_restart stub_timer(struct hrtimer *unused)
+{
+	WARN_ON_ONCE(1);
+	return HRTIMER_NORESTART;
+}
+
+/*
+ * hrtimer_fixup_assert_init is called when:
+ * - an untracked/uninit-ed object is found
+ */
+static bool hrtimer_fixup_assert_init(void *addr, enum debug_obj_state state)
+{
+	struct hrtimer *timer = addr;
+
+	switch (state) {
+	case ODEBUG_STATE_NOTAVAILABLE:
+		hrtimer_setup(timer, stub_timer, CLOCK_MONOTONIC, 0);
+		return true;
+	default:
+		return false;
+	}
+}
+
 static const struct debug_obj_descr hrtimer_debug_descr = {
-	.name		= "hrtimer",
-	.debug_hint	= hrtimer_debug_hint,
-	.fixup_init	= hrtimer_fixup_init,
-	.fixup_activate	= hrtimer_fixup_activate,
-	.fixup_free	= hrtimer_fixup_free,
+	.name			= "hrtimer",
+	.debug_hint		= hrtimer_debug_hint,
+	.fixup_init		= hrtimer_fixup_init,
+	.fixup_activate		= hrtimer_fixup_activate,
+	.fixup_free		= hrtimer_fixup_free,
+	.fixup_assert_init	= hrtimer_fixup_assert_init,
 };
 
 static inline void debug_hrtimer_init(struct hrtimer *timer)
@@ -440,8 +480,7 @@ static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)
 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);
 }
 
-static inline void debug_hrtimer_activate(struct hrtimer *timer,
-					  enum hrtimer_mode mode)
+static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode)
 {
 	debug_object_activate(timer, &hrtimer_debug_descr);
 }
@@ -451,6 +490,11 @@ static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
 	debug_object_deactivate(timer, &hrtimer_debug_descr);
 }
 
+static inline void debug_hrtimer_assert_init(struct hrtimer *timer)
+{
+	debug_object_assert_init(timer, &hrtimer_debug_descr);
+}
+
 void destroy_hrtimer_on_stack(struct hrtimer *timer)
 {
 	debug_object_free(timer, &hrtimer_debug_descr);
@@ -461,9 +505,9 @@ EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
 
 static inline void debug_hrtimer_init(struct hrtimer *timer) { }
 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }
-static inline void debug_hrtimer_activate(struct hrtimer *timer,
-					  enum hrtimer_mode mode) { }
+static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) { }
 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
+static inline void debug_hrtimer_assert_init(struct hrtimer *timer) { }
 #endif
 
 static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode)
@@ -479,80 +523,80 @@ static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid
 	trace_hrtimer_setup(timer, clockid, mode);
 }
 
-static inline void debug_activate(struct hrtimer *timer,
-				  enum hrtimer_mode mode)
+static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed)
 {
 	debug_hrtimer_activate(timer, mode);
-	trace_hrtimer_start(timer, mode);
+	trace_hrtimer_start(timer, mode, was_armed);
 }
 
-static inline void debug_deactivate(struct hrtimer *timer)
-{
-	debug_hrtimer_deactivate(timer);
-	trace_hrtimer_cancel(timer);
-}
+#define for_each_active_base(base, cpu_base, active)					\
+	for (unsigned int idx = ffs(active); idx--; idx = ffs((active)))		\
+		for (bool done = false; !done; active &= ~(1U << idx))			\
+			for (base = &cpu_base->clock_base[idx]; !done; done = true)
 
-static struct hrtimer_clock_base *
-__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
+#define hrtimer_from_timerqueue_node(_n) container_of_const(_n, struct hrtimer, node)
+
+#if defined(CONFIG_NO_HZ_COMMON)
+/*
+ * Same as hrtimer_bases_next_event() below, but skips the excluded timer and
+ * does not update cpu_base->next_timer/expires.
+ */
+static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base,
+						const struct hrtimer *exclude,
+						unsigned int active, ktime_t expires_next)
 {
-	unsigned int idx;
+	struct hrtimer_clock_base *base;
+	ktime_t expires;
 
-	if (!*active)
-		return NULL;
+	lockdep_assert_held(&cpu_base->lock);
 
-	idx = __ffs(*active);
-	*active &= ~(1U << idx);
+	for_each_active_base(base, cpu_base, active) {
+		expires = ktime_sub(base->expires_next, base->offset);
+		if (expires >= expires_next)
+			continue;
+
+		/*
+		 * If the excluded timer is the first on this base evaluate the
+		 * next timer.
+		 */
+		struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active);
 
-	return &cpu_base->clock_base[idx];
+		if (unlikely(&exclude->node == node)) {
+			node = timerqueue_linked_next(node);
+			if (!node)
+				continue;
+			expires = ktime_sub(node->expires, base->offset);
+			if (expires >= expires_next)
+				continue;
+		}
+		expires_next = expires;
+	}
+	/* If base->offset changed, the result might be negative */
+	return max(expires_next, 0);
 }
+#endif
 
-#define for_each_active_base(base, cpu_base, active)	\
-	while ((base = __next_base((cpu_base), &(active))))
+static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base)
+{
+	struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
 
-static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
-					 const struct hrtimer *exclude,
-					 unsigned int active,
-					 ktime_t expires_next)
+	return hrtimer_from_timerqueue_node(next);
+}
+
+/* Find the base with the earliest expiry */
+static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active,
+				ktime_t *expires_next, struct hrtimer **next_timer)
 {
 	struct hrtimer_clock_base *base;
 	ktime_t expires;
 
 	for_each_active_base(base, cpu_base, active) {
-		struct timerqueue_node *next;
-		struct hrtimer *timer;
-
-		next = timerqueue_getnext(&base->active);
-		timer = container_of(next, struct hrtimer, node);
-		if (timer == exclude) {
-			/* Get to the next timer in the queue. */
-			next = timerqueue_iterate_next(next);
-			if (!next)
-				continue;
-
-			timer = container_of(next, struct hrtimer, node);
-		}
-		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
-		if (expires < expires_next) {
-			expires_next = expires;
-
-			/* Skip cpu_base update if a timer is being excluded. */
-			if (exclude)
-				continue;
-
-			if (timer->is_soft)
-				cpu_base->softirq_next_timer = timer;
-			else
-				cpu_base->next_timer = timer;
+		expires = ktime_sub(base->expires_next, base->offset);
+		if (expires < *expires_next) {
+			*expires_next = expires;
+			*next_timer = clock_base_next_timer(base);
 		}
 	}
-	/*
-	 * clock_was_set() might have changed base->offset of any of
-	 * the clock bases so the result might be negative. Fix it up
-	 * to prevent a false positive in clockevents_program_event().
-	 */
-	if (expires_next < 0)
-		expires_next = 0;
-	return expires_next;
 }
 
 /*
@@ -575,30 +619,28 @@ static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
  *  - HRTIMER_ACTIVE_SOFT, or
  *  - HRTIMER_ACTIVE_HARD.
  */
-static ktime_t
-__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
+static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
 {
-	unsigned int active;
 	struct hrtimer *next_timer = NULL;
 	ktime_t expires_next = KTIME_MAX;
+	unsigned int active;
+
+	lockdep_assert_held(&cpu_base->lock);
 
 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
-		cpu_base->softirq_next_timer = NULL;
-		expires_next = __hrtimer_next_event_base(cpu_base, NULL,
-							 active, KTIME_MAX);
-
-		next_timer = cpu_base->softirq_next_timer;
+		if (active)
+			hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
+		cpu_base->softirq_next_timer = next_timer;
 	}
 
 	if (active_mask & HRTIMER_ACTIVE_HARD) {
 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
+		if (active)
+			hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
 		cpu_base->next_timer = next_timer;
-		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
-							 expires_next);
 	}
-
-	return expires_next;
+	return max(expires_next, 0);
 }
 
 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
@@ -638,8 +680,8 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
 	ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
 	ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
 
-	ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
-					    offs_real, offs_boot, offs_tai);
+	ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, offs_real,
+						   offs_boot, offs_tai);
 
 	base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
 	base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
@@ -649,7 +691,9 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
 }
 
 /*
- * Is the high resolution mode active ?
+ * Is the high resolution mode active in the CPU base. This cannot use the
+ * static key as the CPUs are switched to high resolution mode
+ * asynchronously.
  */
 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
 {
@@ -657,8 +701,13 @@ static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
 		cpu_base->hres_active : 0;
 }
 
-static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
-				struct hrtimer *next_timer,
+static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
+{
+	trace_hrtimer_rearm(expires_next, deferred);
+	tick_program_event(expires_next, 1);
+}
+
+static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
 				ktime_t expires_next)
 {
 	cpu_base->expires_next = expires_next;
@@ -683,20 +732,13 @@ static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
 	if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
 		return;
 
-	tick_program_event(expires_next, 1);
+	hrtimer_rearm_event(expires_next, false);
 }
 
-/*
- * Reprogram the event source with checking both queues for the
- * next event
- * Called with interrupts disabled and base->lock held
- */
-static void
-hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
+/* Reprogram the event source with a evaluation of all clock bases */
+static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal)
 {
-	ktime_t expires_next;
-
-	expires_next = hrtimer_update_next_event(cpu_base);
+	ktime_t expires_next = hrtimer_update_next_event(cpu_base);
 
 	if (skip_equal && expires_next == cpu_base->expires_next)
 		return;
@@ -707,57 +749,49 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
 /* High resolution timer related functions */
 #ifdef CONFIG_HIGH_RES_TIMERS
 
-/*
- * High resolution timer enabled ?
- */
+/* High resolution timer enabled ? */
 static bool hrtimer_hres_enabled __read_mostly  = true;
 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
 EXPORT_SYMBOL_GPL(hrtimer_resolution);
 
-/*
- * Enable / Disable high resolution mode
- */
+/* Enable / Disable high resolution mode */
 static int __init setup_hrtimer_hres(char *str)
 {
 	return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
 }
-
 __setup("highres=", setup_hrtimer_hres);
 
-/*
- * hrtimer_high_res_enabled - query, if the highres mode is enabled
- */
-static inline int hrtimer_is_hres_enabled(void)
+/* hrtimer_high_res_enabled - query, if the highres mode is enabled */
+static inline bool hrtimer_is_hres_enabled(void)
 {
 	return hrtimer_hres_enabled;
 }
 
-/*
- * Switch to high resolution mode
- */
+/* Switch to high resolution mode */
 static void hrtimer_switch_to_hres(void)
 {
 	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
 
 	if (tick_init_highres()) {
-		pr_warn("Could not switch to high resolution mode on CPU %u\n",
-			base->cpu);
+		pr_warn("Could not switch to high resolution mode on CPU %u\n",	base->cpu);
 		return;
 	}
-	base->hres_active = 1;
+	base->hres_active = true;
 	hrtimer_resolution = HIGH_RES_NSEC;
 
 	tick_setup_sched_timer(true);
 	/* "Retrigger" the interrupt to get things going */
 	retrigger_next_event(NULL);
+	hrtimer_schedule_hres_work();
 }
 
 #else
 
-static inline int hrtimer_is_hres_enabled(void) { return 0; }
+static inline bool hrtimer_is_hres_enabled(void) { return 0; }
 static inline void hrtimer_switch_to_hres(void) { }
 
 #endif /* CONFIG_HIGH_RES_TIMERS */
+
 /*
  * Retrigger next event is called after clock was set with interrupts
  * disabled through an SMP function call or directly from low level
@@ -792,13 +826,12 @@ static void retrigger_next_event(void *arg)
 	 * In periodic low resolution mode, the next softirq expiration
 	 * must also be updated.
 	 */
-	raw_spin_lock(&base->lock);
+	guard(raw_spinlock)(&base->lock);
 	hrtimer_update_base(base);
 	if (hrtimer_hres_active(base))
-		hrtimer_force_reprogram(base, 0);
+		hrtimer_force_reprogram(base, /* skip_equal */ false);
 	else
 		hrtimer_update_next_event(base);
-	raw_spin_unlock(&base->lock);
 }
 
 /*
@@ -812,10 +845,11 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
 	struct hrtimer_clock_base *base = timer->base;
-	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
+	ktime_t expires = hrtimer_get_expires(timer);
 
-	WARN_ON_ONCE(hrtimer_get_expires(timer) < 0);
+	WARN_ON_ONCE(expires < 0);
 
+	expires = ktime_sub(expires, base->offset);
 	/*
 	 * CLOCK_REALTIME timer might be requested with an absolute
 	 * expiry time which is less than base->offset. Set it to 0.
@@ -842,8 +876,7 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
 		timer_cpu_base->softirq_next_timer = timer;
 		timer_cpu_base->softirq_expires_next = expires;
 
-		if (!ktime_before(expires, timer_cpu_base->expires_next) ||
-		    !reprogram)
+		if (!ktime_before(expires, timer_cpu_base->expires_next) || !reprogram)
 			return;
 	}
 
@@ -857,11 +890,8 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
 	if (expires >= cpu_base->expires_next)
 		return;
 
-	/*
-	 * If the hrtimer interrupt is running, then it will reevaluate the
-	 * clock bases and reprogram the clock event device.
-	 */
-	if (cpu_base->in_hrtirq)
+	/* If a deferred rearm is pending skip reprogramming the device */
+	if (cpu_base->deferred_rearm)
 		return;
 
 	cpu_base->next_timer = timer;
@@ -869,8 +899,7 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
 	__hrtimer_reprogram(cpu_base, timer, expires);
 }
 
-static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
-			     unsigned int active)
+static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
 {
 	struct hrtimer_clock_base *base;
 	unsigned int seq;
@@ -896,13 +925,11 @@ static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
 	if (seq == cpu_base->clock_was_set_seq)
 		return false;
 
-	/*
-	 * If the remote CPU is currently handling an hrtimer interrupt, it
-	 * will reevaluate the first expiring timer of all clock bases
-	 * before reprogramming. Nothing to do here.
-	 */
-	if (cpu_base->in_hrtirq)
+	/* If a deferred rearm is pending the remote CPU will take care of it */
+	if (cpu_base->deferred_rearm) {
+		cpu_base->deferred_needs_update = true;
 		return false;
+	}
 
 	/*
 	 * Walk the affected clock bases and check whether the first expiring
@@ -913,9 +940,9 @@ static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
 	active &= cpu_base->active_bases;
 
 	for_each_active_base(base, cpu_base, active) {
-		struct timerqueue_node *next;
+		struct timerqueue_linked_node *next;
 
-		next = timerqueue_getnext(&base->active);
+		next = timerqueue_linked_first(&base->active);
 		expires = ktime_sub(next->expires, base->offset);
 		if (expires < cpu_base->expires_next)
 			return true;
@@ -947,11 +974,9 @@ static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
  */
 void clock_was_set(unsigned int bases)
 {
-	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);
 	cpumask_var_t mask;
-	int cpu;
 
-	if (!hrtimer_hres_active(cpu_base) && !tick_nohz_is_active())
+	if (!hrtimer_highres_enabled() && !tick_nohz_is_active())
 		goto out_timerfd;
 
 	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
@@ -960,23 +985,19 @@ void clock_was_set(unsigned int bases)
 	}
 
 	/* Avoid interrupting CPUs if possible */
-	cpus_read_lock();
-	for_each_online_cpu(cpu) {
-		unsigned long flags;
-
-		cpu_base = &per_cpu(hrtimer_bases, cpu);
-		raw_spin_lock_irqsave(&cpu_base->lock, flags);
+	scoped_guard(cpus_read_lock) {
+		int cpu;
 
-		if (update_needs_ipi(cpu_base, bases))
-			cpumask_set_cpu(cpu, mask);
+		for_each_online_cpu(cpu) {
+			struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
 
-		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+			guard(raw_spinlock_irqsave)(&cpu_base->lock);
+			if (update_needs_ipi(cpu_base, bases))
+				cpumask_set_cpu(cpu, mask);
+		}
+		scoped_guard(preempt)
+			smp_call_function_many(mask, retrigger_next_event, NULL, 1);
 	}
-
-	preempt_disable();
-	smp_call_function_many(mask, retrigger_next_event, NULL, 1);
-	preempt_enable();
-	cpus_read_unlock();
 	free_cpumask_var(mask);
 
 out_timerfd:
@@ -1011,11 +1032,8 @@ void hrtimers_resume_local(void)
 	retrigger_next_event(NULL);
 }
 
-/*
- * Counterpart to lock_hrtimer_base above:
- */
-static inline
-void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
+/* Counterpart to lock_hrtimer_base above */
+static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
 	__releases(&timer->base->cpu_base->lock)
 {
 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
@@ -1032,7 +1050,7 @@ void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  * .. note::
  *  This only updates the timer expiry value and does not requeue the timer.
  *
- * There is also a variant of the function hrtimer_forward_now().
+ * There is also a variant of this function: hrtimer_forward_now().
  *
  * Context: Can be safely called from the callback function of @timer. If called
  *          from other contexts @timer must neither be enqueued nor running the
@@ -1042,15 +1060,15 @@ void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  */
 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
 {
-	u64 orun = 1;
 	ktime_t delta;
+	u64 orun = 1;
 
 	delta = ktime_sub(now, hrtimer_get_expires(timer));
 
 	if (delta < 0)
 		return 0;
 
-	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
+	if (WARN_ON(timer->is_queued))
 		return 0;
 
 	if (interval < hrtimer_resolution)
@@ -1079,73 +1097,98 @@ EXPORT_SYMBOL_GPL(hrtimer_forward);
  * enqueue_hrtimer - internal function to (re)start a timer
  *
  * The timer is inserted in expiry order. Insertion into the
- * red black tree is O(log(n)). Must hold the base lock.
+ * red black tree is O(log(n)).
  *
  * Returns true when the new timer is the leftmost timer in the tree.
  */
 static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
-			    enum hrtimer_mode mode)
+			    enum hrtimer_mode mode, bool was_armed)
 {
-	debug_activate(timer, mode);
+	lockdep_assert_held(&base->cpu_base->lock);
+
+	debug_activate(timer, mode, was_armed);
 	WARN_ON_ONCE(!base->cpu_base->online);
 
 	base->cpu_base->active_bases |= 1 << base->index;
 
 	/* Pairs with the lockless read in hrtimer_is_queued() */
-	WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
+	WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
+
+	if (!timerqueue_linked_add(&base->active, &timer->node))
+		return false;
+
+	base->expires_next = hrtimer_get_expires(timer);
+	return true;
+}
 
-	return timerqueue_add(&base->active, &timer->node);
+static inline void base_update_next_timer(struct hrtimer_clock_base *base)
+{
+	struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
+
+	base->expires_next = next ? next->expires : KTIME_MAX;
 }
 
 /*
  * __remove_hrtimer - internal function to remove a timer
  *
- * Caller must hold the base lock.
- *
  * High resolution timer mode reprograms the clock event device when the
  * timer is the one which expires next. The caller can disable this by setting
  * reprogram to zero. This is useful, when the context does a reprogramming
  * anyway (e.g. timer interrupt)
  */
-static void __remove_hrtimer(struct hrtimer *timer,
-			     struct hrtimer_clock_base *base,
-			     u8 newstate, int reprogram)
+static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
+			     bool newstate, bool reprogram)
 {
 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
-	u8 state = timer->state;
+	bool was_first;
 
-	/* Pairs with the lockless read in hrtimer_is_queued() */
-	WRITE_ONCE(timer->state, newstate);
-	if (!(state & HRTIMER_STATE_ENQUEUED))
+	lockdep_assert_held(&cpu_base->lock);
+
+	if (!timer->is_queued)
 		return;
 
-	if (!timerqueue_del(&base->active, &timer->node))
+	/* Pairs with the lockless read in hrtimer_is_queued() */
+	WRITE_ONCE(timer->is_queued, newstate);
+
+	was_first = !timerqueue_linked_prev(&timer->node);
+
+	if (!timerqueue_linked_del(&base->active, &timer->node))
 		cpu_base->active_bases &= ~(1 << base->index);
 
+	/* Nothing to update if this was not the first timer in the base */
+	if (!was_first)
+		return;
+
+	base_update_next_timer(base);
+
 	/*
-	 * Note: If reprogram is false we do not update
-	 * cpu_base->next_timer. This happens when we remove the first
-	 * timer on a remote cpu. No harm as we never dereference
-	 * cpu_base->next_timer. So the worst thing what can happen is
-	 * an superfluous call to hrtimer_force_reprogram() on the
-	 * remote cpu later on if the same timer gets enqueued again.
+	 * If reprogram is false don't update cpu_base->next_timer and do not
+	 * touch the clock event device.
+	 *
+	 * This happens when removing the first timer on a remote CPU, which
+	 * will be handled by the remote CPU's interrupt. It also happens when
+	 * a local timer is removed to be immediately restarted. That's handled
+	 * at the call site.
 	 */
-	if (reprogram && timer == cpu_base->next_timer)
-		hrtimer_force_reprogram(cpu_base, 1);
+	if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy)
+		return;
+
+	if (cpu_base->deferred_rearm)
+		cpu_base->deferred_needs_update = true;
+	else
+		hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
 }
 
-/*
- * remove hrtimer, called with base lock held
- */
-static inline int
-remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
-	       bool restart, bool keep_local)
+static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
+				  bool newstate)
 {
-	u8 state = timer->state;
+	lockdep_assert_held(&base->cpu_base->lock);
 
-	if (state & HRTIMER_STATE_ENQUEUED) {
+	if (timer->is_queued) {
 		bool reprogram;
 
+		debug_hrtimer_deactivate(timer);
+
 		/*
 		 * Remove the timer and force reprogramming when high
 		 * resolution mode is active and the timer is on the current
@@ -1154,24 +1197,81 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
 		 * reprogramming happens in the interrupt handler. This is a
 		 * rare case and less expensive than a smp call.
 		 */
-		debug_deactivate(timer);
 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
 
-		/*
-		 * If the timer is not restarted then reprogramming is
-		 * required if the timer is local. If it is local and about
-		 * to be restarted, avoid programming it twice (on removal
-		 * and a moment later when it's requeued).
-		 */
-		if (!restart)
-			state = HRTIMER_STATE_INACTIVE;
-		else
-			reprogram &= !keep_local;
+		__remove_hrtimer(timer, base, newstate, reprogram);
+		return true;
+	}
+	return false;
+}
+
+/*
+ * Update in place has to retrieve the expiry times of the neighbour nodes
+ * if they exist. That is cache line neutral because the dequeue/enqueue
+ * operation is going to need the same cache lines. But there is a big win
+ * when the dequeue/enqueue can be avoided because the RB tree does not
+ * have to be rebalanced twice.
+ */
+static inline bool
+hrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires)
+{
+	struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node);
+	struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node);
+
+	/* If the new expiry goes behind the next timer, requeue is required */
+	if (next && expires > next->expires)
+		return false;
+
+	/* If this is the first timer, update in place */
+	if (!prev)
+		return true;
+
+	/* Update in place when it does not go ahead of the previous one */
+	return expires >= prev->expires;
+}
+
+static inline bool
+remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
+			     const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
+{
+	bool was_first = false;
+
+	/* Remove it from the timer queue if active */
+	if (timer->is_queued) {
+		was_first = !timerqueue_linked_prev(&timer->node);
+
+		/* Try to update in place to avoid the de/enqueue dance */
+		if (hrtimer_can_update_in_place(timer, base, expires)) {
+			hrtimer_set_expires_range_ns(timer, expires, delta_ns);
+			trace_hrtimer_start(timer, mode, true);
+			if (was_first)
+				base->expires_next = expires;
+			return was_first;
+		}
 
-		__remove_hrtimer(timer, base, state, reprogram);
-		return 1;
+		debug_hrtimer_deactivate(timer);
+		timerqueue_linked_del(&base->active, &timer->node);
 	}
-	return 0;
+
+	/* Set the new expiry time */
+	hrtimer_set_expires_range_ns(timer, expires, delta_ns);
+
+	debug_activate(timer, mode, timer->is_queued);
+	base->cpu_base->active_bases |= 1 << base->index;
+
+	/* Pairs with the lockless read in hrtimer_is_queued() */
+	WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
+
+	/* If it's the first expiring timer now or again, update base */
+	if (timerqueue_linked_add(&base->active, &timer->node)) {
+		base->expires_next = expires;
+		return true;
+	}
+
+	if (was_first)
+		base_update_next_timer(base);
+
+	return false;
 }
 
 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
@@ -1190,55 +1290,93 @@ static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
 	return tim;
 }
 
-static void
-hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
+static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
 {
-	ktime_t expires;
-
-	/*
-	 * Find the next SOFT expiration.
-	 */
-	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
+	ktime_t expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
 
 	/*
-	 * reprogramming needs to be triggered, even if the next soft
-	 * hrtimer expires at the same time than the next hard
+	 * Reprogramming needs to be triggered, even if the next soft
+	 * hrtimer expires at the same time as the next hard
 	 * hrtimer. cpu_base->softirq_expires_next needs to be updated!
 	 */
 	if (expires == KTIME_MAX)
 		return;
 
 	/*
-	 * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
-	 * cpu_base->*expires_next is only set by hrtimer_reprogram()
+	 * cpu_base->next_timer is recomputed by __hrtimer_get_next_event()
+	 * cpu_base->expires_next is only set by hrtimer_reprogram()
 	 */
 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
 }
 
-static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
-				    u64 delta_ns, const enum hrtimer_mode mode,
-				    struct hrtimer_clock_base *base)
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
+{
+	if (static_branch_likely(&timers_migration_enabled)) {
+		/*
+		 * If it is local and the first expiring timer keep it on the local
+		 * CPU to optimize reprogramming of the clockevent device. Also
+		 * avoid switch_hrtimer_base() overhead when local and pinned.
+		 */
+		if (!is_local)
+			return false;
+		if (is_first || is_pinned)
+			return true;
+
+		/* Honour the NOHZ full restrictions */
+		if (!housekeeping_cpu(smp_processor_id(), HK_TYPE_KERNEL_NOISE))
+			return false;
+
+		/*
+		 * If the tick is not stopped or need_resched() is set, then
+		 * there is no point in moving the timer somewhere else.
+		 */
+		return !tick_nohz_tick_stopped() || need_resched();
+	}
+	return is_local;
+}
+#else
+static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
+{
+	return is_local;
+}
+#endif
+
+static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first,
+				     bool is_pinned)
+{
+	/* If the timer is running the callback it has to stay on its CPU base. */
+	if (unlikely(timer->base->running == timer))
+		return true;
+
+	return hrtimer_prefer_local(is_local, is_first, is_pinned);
+}
+
+static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
+				     const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
 {
 	struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
-	struct hrtimer_clock_base *new_base;
-	bool force_local, first;
+	bool is_pinned, first, was_first, keep_base = false;
+	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
 
-	/*
-	 * If the timer is on the local cpu base and is the first expiring
-	 * timer then this might end up reprogramming the hardware twice
-	 * (on removal and on enqueue). To avoid that by prevent the
-	 * reprogram on removal, keep the timer local to the current CPU
-	 * and enforce reprogramming after it is queued no matter whether
-	 * it is the new first expiring timer again or not.
-	 */
-	force_local = base->cpu_base == this_cpu_base;
-	force_local &= base->cpu_base->next_timer == timer;
+	was_first = cpu_base->next_timer == timer;
+	is_pinned = !!(mode & HRTIMER_MODE_PINNED);
 
 	/*
-	 * Don't force local queuing if this enqueue happens on a unplugged
-	 * CPU after hrtimer_cpu_dying() has been invoked.
+	 * Don't keep it local if this enqueue happens on a unplugged CPU
+	 * after hrtimer_cpu_dying() has been invoked.
 	 */
-	force_local &= this_cpu_base->online;
+	if (likely(this_cpu_base->online)) {
+		bool is_local = cpu_base == this_cpu_base;
+
+		keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
+	}
+
+	/* Calculate absolute expiry time for relative timers */
+	if (mode & HRTIMER_MODE_REL)
+		tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
+	/* Compensate for low resolution granularity */
+	tim = hrtimer_update_lowres(timer, tim, mode);
 
 	/*
 	 * Remove an active timer from the queue. In case it is not queued
@@ -1250,32 +1388,41 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
 	 * reprogramming later if it was the first expiring timer.  This
 	 * avoids programming the underlying clock event twice (once at
 	 * removal and once after enqueue).
+	 *
+	 * @keep_base is also true if the timer callback is running on a
+	 * remote CPU and for local pinned timers.
 	 */
-	remove_hrtimer(timer, base, true, force_local);
+	if (likely(keep_base)) {
+		first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns);
+	} else {
+		/* Keep the ENQUEUED state in case it is queued */
+		bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);
 
-	if (mode & HRTIMER_MODE_REL)
-		tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
+		hrtimer_set_expires_range_ns(timer, tim, delta_ns);
 
-	tim = hrtimer_update_lowres(timer, tim, mode);
+		/* Switch the timer base, if necessary: */
+		base = switch_hrtimer_base(timer, base, is_pinned);
+		cpu_base = base->cpu_base;
 
-	hrtimer_set_expires_range_ns(timer, tim, delta_ns);
+		first = enqueue_hrtimer(timer, base, mode, was_armed);
+	}
 
-	/* Switch the timer base, if necessary: */
-	if (!force_local) {
-		new_base = switch_hrtimer_base(timer, base,
-					       mode & HRTIMER_MODE_PINNED);
-	} else {
-		new_base = base;
+	/* If a deferred rearm is pending skip reprogramming the device */
+	if (cpu_base->deferred_rearm) {
+		cpu_base->deferred_needs_update = true;
+		return false;
 	}
 
-	first = enqueue_hrtimer(timer, new_base, mode);
-	if (!force_local) {
+	if (!was_first || cpu_base != this_cpu_base) {
 		/*
-		 * If the current CPU base is online, then the timer is
-		 * never queued on a remote CPU if it would be the first
-		 * expiring timer there.
+		 * If the current CPU base is online, then the timer is never
+		 * queued on a remote CPU if it would be the first expiring
+		 * timer there unless the timer callback is currently executed
+		 * on the remote CPU. In the latter case the remote CPU will
+		 * re-evaluate the first expiring timer after completing the
+		 * callbacks.
 		 */
-		if (hrtimer_base_is_online(this_cpu_base))
+		if (likely(hrtimer_base_is_online(this_cpu_base)))
 			return first;
 
 		/*
@@ -1283,21 +1430,33 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
 		 * already offline. If the timer is the first to expire,
 		 * kick the remote CPU to reprogram the clock event.
 		 */
-		if (first) {
-			struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base;
+		if (first)
+			smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
+		return false;
+	}
 
-			smp_call_function_single_async(new_cpu_base->cpu, &new_cpu_base->csd);
-		}
-		return 0;
+	/*
+	 * Special case for the HRTICK timer. It is frequently rearmed and most
+	 * of the time moves the expiry into the future. That's expensive in
+	 * virtual machines and it's better to take the pointless already armed
+	 * interrupt than reprogramming the hardware on every context switch.
+	 *
+	 * If the new expiry is before the armed time, then reprogramming is
+	 * required.
+	 */
+	if (timer->is_lazy) {
+		if (cpu_base->expires_next <= hrtimer_get_expires(timer))
+			return false;
 	}
 
 	/*
-	 * Timer was forced to stay on the current CPU to avoid
-	 * reprogramming on removal and enqueue. Force reprogram the
-	 * hardware by evaluating the new first expiring timer.
+	 * Timer was the first expiring timer and forced to stay on the
+	 * current CPU to avoid reprogramming on removal and enqueue. Force
+	 * reprogram the hardware by evaluating the new first expiring
+	 * timer.
 	 */
-	hrtimer_force_reprogram(new_base->cpu_base, 1);
-	return 0;
+	hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
+	return false;
 }
 
 /**
@@ -1309,12 +1468,14 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
  *		relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
  *		softirq based mode is considered for debug purpose only!
  */
-void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
-			    u64 delta_ns, const enum hrtimer_mode mode)
+void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
+			    const enum hrtimer_mode mode)
 {
 	struct hrtimer_clock_base *base;
 	unsigned long flags;
 
+	debug_hrtimer_assert_init(timer);
+
 	/*
 	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
 	 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
@@ -1362,8 +1523,11 @@ int hrtimer_try_to_cancel(struct hrtimer *timer)
 
 	base = lock_hrtimer_base(timer, &flags);
 
-	if (!hrtimer_callback_running(timer))
-		ret = remove_hrtimer(timer, base, false, false);
+	if (!hrtimer_callback_running(timer)) {
+		ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE);
+		if (ret)
+			trace_hrtimer_cancel(timer);
+	}
 
 	unlock_hrtimer_base(timer, &flags);
 
@@ -1397,8 +1561,7 @@ static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
  * the timer callback to finish. Drop expiry_lock and reacquire it. That
  * allows the waiter to acquire the lock and make progress.
  */
-static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
-				      unsigned long flags)
+static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, unsigned long flags)
 {
 	if (atomic_read(&cpu_base->timer_waiters)) {
 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
@@ -1463,14 +1626,10 @@ void hrtimer_cancel_wait_running(const struct hrtimer *timer)
 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
 }
 #else
-static inline void
-hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
-static inline void
-hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
-static inline void
-hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
-static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
-					     unsigned long flags) { }
+static inline void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
+static inline void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
+static inline void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
+static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, unsigned long fl) { }
 #endif
 
 /**
@@ -1526,15 +1685,11 @@ u64 hrtimer_get_next_event(void)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
 	u64 expires = KTIME_MAX;
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&cpu_base->lock, flags);
 
+	guard(raw_spinlock_irqsave)(&cpu_base->lock);
 	if (!hrtimer_hres_active(cpu_base))
 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
 
-	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
-
 	return expires;
 }
 
@@ -1549,26 +1704,20 @@ u64 hrtimer_next_event_without(const struct hrtimer *exclude)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
 	u64 expires = KTIME_MAX;
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&cpu_base->lock, flags);
-
-	if (hrtimer_hres_active(cpu_base)) {
-		unsigned int active;
+	unsigned int active;
 
-		if (!cpu_base->softirq_activated) {
-			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
-			expires = __hrtimer_next_event_base(cpu_base, exclude,
-							    active, KTIME_MAX);
-		}
-		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
-		expires = __hrtimer_next_event_base(cpu_base, exclude, active,
-						    expires);
-	}
+	guard(raw_spinlock_irqsave)(&cpu_base->lock);
+	if (!hrtimer_hres_active(cpu_base))
+		return expires;
 
-	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+	active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
+	if (active && !cpu_base->softirq_activated)
+		expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);
 
-	return expires;
+	active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
+	if (!active)
+		return expires;
+	return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);
 }
 #endif
 
@@ -1612,8 +1761,7 @@ ktime_t hrtimer_cb_get_time(const struct hrtimer *timer)
 }
 EXPORT_SYMBOL_GPL(hrtimer_cb_get_time);
 
-static void __hrtimer_setup(struct hrtimer *timer,
-			    enum hrtimer_restart (*function)(struct hrtimer *),
+static void __hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*fn)(struct hrtimer *),
 			    clockid_t clock_id, enum hrtimer_mode mode)
 {
 	bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
@@ -1645,13 +1793,14 @@ static void __hrtimer_setup(struct hrtimer *timer,
 	base += hrtimer_clockid_to_base(clock_id);
 	timer->is_soft = softtimer;
 	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
+	timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM);
 	timer->base = &cpu_base->clock_base[base];
-	timerqueue_init(&timer->node);
+	timerqueue_linked_init(&timer->node);
 
-	if (WARN_ON_ONCE(!function))
+	if (WARN_ON_ONCE(!fn))
 		ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout;
 	else
-		ACCESS_PRIVATE(timer, function) = function;
+		ACCESS_PRIVATE(timer, function) = fn;
 }
 
 /**
@@ -1710,12 +1859,10 @@ bool hrtimer_active(const struct hrtimer *timer)
 		base = READ_ONCE(timer->base);
 		seq = raw_read_seqcount_begin(&base->seq);
 
-		if (timer->state != HRTIMER_STATE_INACTIVE ||
-		    base->running == timer)
+		if (timer->is_queued || base->running == timer)
 			return true;
 
-	} while (read_seqcount_retry(&base->seq, seq) ||
-		 base != READ_ONCE(timer->base));
+	} while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));
 
 	return false;
 }
@@ -1729,7 +1876,7 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
  *  - callback:	the timer is being ran
  *  - post:	the timer is inactive or (re)queued
  *
- * On the read side we ensure we observe timer->state and cpu_base->running
+ * On the read side we ensure we observe timer->is_queued and cpu_base->running
  * from the same section, if anything changed while we looked at it, we retry.
  * This includes timer->base changing because sequence numbers alone are
  * insufficient for that.
@@ -1738,11 +1885,9 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
  * a false negative if the read side got smeared over multiple consecutive
  * __run_hrtimer() invocations.
  */
-
-static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
-			  struct hrtimer_clock_base *base,
-			  struct hrtimer *timer, ktime_t *now,
-			  unsigned long flags) __must_hold(&cpu_base->lock)
+static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_clock_base *base,
+			  struct hrtimer *timer, ktime_t now, unsigned long flags)
+	__must_hold(&cpu_base->lock)
 {
 	enum hrtimer_restart (*fn)(struct hrtimer *);
 	bool expires_in_hardirq;
@@ -1754,15 +1899,15 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
 	base->running = timer;
 
 	/*
-	 * Separate the ->running assignment from the ->state assignment.
+	 * Separate the ->running assignment from the ->is_queued assignment.
 	 *
 	 * As with a regular write barrier, this ensures the read side in
 	 * hrtimer_active() cannot observe base->running == NULL &&
-	 * timer->state == INACTIVE.
+	 * timer->is_queued == INACTIVE.
 	 */
 	raw_write_seqcount_barrier(&base->seq);
 
-	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
+	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, false);
 	fn = ACCESS_PRIVATE(timer, function);
 
 	/*
@@ -1797,16 +1942,15 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
 	 * hrtimer_start_range_ns() can have popped in and enqueued the timer
 	 * for us already.
 	 */
-	if (restart != HRTIMER_NORESTART &&
-	    !(timer->state & HRTIMER_STATE_ENQUEUED))
-		enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
+	if (restart == HRTIMER_RESTART && !timer->is_queued)
+		enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false);
 
 	/*
-	 * Separate the ->running assignment from the ->state assignment.
+	 * Separate the ->running assignment from the ->is_queued assignment.
 	 *
 	 * As with a regular write barrier, this ensures the read side in
 	 * hrtimer_active() cannot observe base->running.timer == NULL &&
-	 * timer->state == INACTIVE.
+	 * timer->is_queued == INACTIVE.
 	 */
 	raw_write_seqcount_barrier(&base->seq);
 
@@ -1814,23 +1958,24 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
 	base->running = NULL;
 }
 
+static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
+{
+	struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
+
+	return next ? hrtimer_from_timerqueue_node(next) : NULL;
+}
+
 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
 				 unsigned long flags, unsigned int active_mask)
 {
-	struct hrtimer_clock_base *base;
 	unsigned int active = cpu_base->active_bases & active_mask;
+	struct hrtimer_clock_base *base;
 
 	for_each_active_base(base, cpu_base, active) {
-		struct timerqueue_node *node;
-		ktime_t basenow;
-
-		basenow = ktime_add(now, base->offset);
-
-		while ((node = timerqueue_getnext(&base->active))) {
-			struct hrtimer *timer;
-
-			timer = container_of(node, struct hrtimer, node);
+		ktime_t basenow = ktime_add(now, base->offset);
+		struct hrtimer *timer;
 
+		while ((timer = clock_base_next_timer(base))) {
 			/*
 			 * The immediate goal for using the softexpires is
 			 * minimizing wakeups, not running timers at the
@@ -1846,7 +1991,7 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
 			if (basenow < hrtimer_get_softexpires(timer))
 				break;
 
-			__run_hrtimer(cpu_base, base, timer, &basenow, flags);
+			__run_hrtimer(cpu_base, base, timer, basenow, flags);
 			if (active_mask == HRTIMER_ACTIVE_SOFT)
 				hrtimer_sync_wait_running(cpu_base, flags);
 		}
@@ -1865,7 +2010,7 @@ static __latent_entropy void hrtimer_run_softirq(void)
 	now = hrtimer_update_base(cpu_base);
 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
 
-	cpu_base->softirq_activated = 0;
+	cpu_base->softirq_activated = false;
 	hrtimer_update_softirq_timer(cpu_base, true);
 
 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
@@ -1874,6 +2019,63 @@ static __latent_entropy void hrtimer_run_softirq(void)
 
 #ifdef CONFIG_HIGH_RES_TIMERS
 
+/*
+ * Very similar to hrtimer_force_reprogram(), except it deals with
+ * deferred_rearm and hang_detected.
+ */
+static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred)
+{
+	cpu_base->expires_next = expires_next;
+	cpu_base->deferred_rearm = false;
+
+	if (unlikely(cpu_base->hang_detected)) {
+		/*
+		 * Give the system a chance to do something else than looping
+		 * on hrtimer interrupts.
+		 */
+		expires_next = ktime_add_ns(ktime_get(),
+					    min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time));
+	}
+	hrtimer_rearm_event(expires_next, deferred);
+}
+
+#ifdef CONFIG_HRTIMER_REARM_DEFERRED
+void __hrtimer_rearm_deferred(void)
+{
+	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
+	ktime_t expires_next;
+
+	if (!cpu_base->deferred_rearm)
+		return;
+
+	guard(raw_spinlock)(&cpu_base->lock);
+	if (cpu_base->deferred_needs_update) {
+		hrtimer_update_base(cpu_base);
+		expires_next = hrtimer_update_next_event(cpu_base);
+	} else {
+		/* No timer added/removed. Use the cached value */
+		expires_next = cpu_base->deferred_expires_next;
+	}
+	hrtimer_rearm(cpu_base, expires_next, true);
+}
+
+static __always_inline void
+hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
+{
+	/* hrtimer_interrupt() just re-evaluated the first expiring timer */
+	cpu_base->deferred_needs_update = false;
+	/* Cache the expiry time */
+	cpu_base->deferred_expires_next = expires_next;
+	set_thread_flag(TIF_HRTIMER_REARM);
+}
+#else  /* CONFIG_HRTIMER_REARM_DEFERRED */
+static __always_inline void
+hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
+{
+	hrtimer_rearm(cpu_base, expires_next, false);
+}
+#endif  /* !CONFIG_HRTIMER_REARM_DEFERRED */
+
 /*
  * High resolution timer interrupt
  * Called with interrupts disabled
@@ -1892,82 +2094,50 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
 	entry_time = now = hrtimer_update_base(cpu_base);
 retry:
-	cpu_base->in_hrtirq = 1;
+	cpu_base->deferred_rearm = true;
 	/*
-	 * We set expires_next to KTIME_MAX here with cpu_base->lock
-	 * held to prevent that a timer is enqueued in our queue via
-	 * the migration code. This does not affect enqueueing of
-	 * timers which run their callback and need to be requeued on
-	 * this CPU.
+	 * Set expires_next to KTIME_MAX, which prevents that remote CPUs queue
+	 * timers while __hrtimer_run_queues() is expiring the clock bases.
+	 * Timers which are re/enqueued on the local CPU are not affected by
+	 * this.
 	 */
 	cpu_base->expires_next = KTIME_MAX;
 
 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
 		cpu_base->softirq_expires_next = KTIME_MAX;
-		cpu_base->softirq_activated = 1;
+		cpu_base->softirq_activated = true;
 		raise_timer_softirq(HRTIMER_SOFTIRQ);
 	}
 
 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
 
-	/* Reevaluate the clock bases for the [soft] next expiry */
-	expires_next = hrtimer_update_next_event(cpu_base);
-	/*
-	 * Store the new expiry value so the migration code can verify
-	 * against it.
-	 */
-	cpu_base->expires_next = expires_next;
-	cpu_base->in_hrtirq = 0;
-	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
-
-	/* Reprogramming necessary ? */
-	if (!tick_program_event(expires_next, 0)) {
-		cpu_base->hang_detected = 0;
-		return;
-	}
-
 	/*
 	 * The next timer was already expired due to:
 	 * - tracing
 	 * - long lasting callbacks
 	 * - being scheduled away when running in a VM
 	 *
-	 * We need to prevent that we loop forever in the hrtimer
-	 * interrupt routine. We give it 3 attempts to avoid
-	 * overreacting on some spurious event.
-	 *
-	 * Acquire base lock for updating the offsets and retrieving
-	 * the current time.
+	 * We need to prevent that we loop forever in the hrtiner interrupt
+	 * routine. We give it 3 attempts to avoid overreacting on some
+	 * spurious event.
 	 */
-	raw_spin_lock_irqsave(&cpu_base->lock, flags);
 	now = hrtimer_update_base(cpu_base);
-	cpu_base->nr_retries++;
-	if (++retries < 3)
-		goto retry;
-	/*
-	 * Give the system a chance to do something else than looping
-	 * here. We stored the entry time, so we know exactly how long
-	 * we spent here. We schedule the next event this amount of
-	 * time away.
-	 */
-	cpu_base->nr_hangs++;
-	cpu_base->hang_detected = 1;
-	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+	expires_next = hrtimer_update_next_event(cpu_base);
+	cpu_base->hang_detected = false;
+	if (expires_next < now) {
+		if (++retries < 3)
+			goto retry;
+
+		delta = ktime_sub(now, entry_time);
+		cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
+		cpu_base->nr_hangs++;
+		cpu_base->hang_detected = true;
+	}
 
-	delta = ktime_sub(now, entry_time);
-	if ((unsigned int)delta > cpu_base->max_hang_time)
-		cpu_base->max_hang_time = (unsigned int) delta;
-	/*
-	 * Limit it to a sensible value as we enforce a longer
-	 * delay. Give the CPU at least 100ms to catch up.
-	 */
-	if (delta > 100 * NSEC_PER_MSEC)
-		expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
-	else
-		expires_next = ktime_add(now, delta);
-	tick_program_event(expires_next, 1);
-	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
+	hrtimer_interrupt_rearm(cpu_base, expires_next);
+	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
 }
+
 #endif /* !CONFIG_HIGH_RES_TIMERS */
 
 /*
@@ -1999,7 +2169,7 @@ void hrtimer_run_queues(void)
 
 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
 		cpu_base->softirq_expires_next = KTIME_MAX;
-		cpu_base->softirq_activated = 1;
+		cpu_base->softirq_activated = true;
 		raise_timer_softirq(HRTIMER_SOFTIRQ);
 	}
 
@@ -2012,8 +2182,7 @@ void hrtimer_run_queues(void)
  */
 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
 {
-	struct hrtimer_sleeper *t =
-		container_of(timer, struct hrtimer_sleeper, timer);
+	struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer);
 	struct task_struct *task = t->task;
 
 	t->task = NULL;
@@ -2031,8 +2200,7 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
  * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
  * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
  */
-void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
-				   enum hrtimer_mode mode)
+void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode mode)
 {
 	/*
 	 * Make the enqueue delivery mode check work on RT. If the sleeper
@@ -2048,8 +2216,8 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
 }
 EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
 
-static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl,
-				    clockid_t clock_id, enum hrtimer_mode mode)
+static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
+				    enum hrtimer_mode mode)
 {
 	/*
 	 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
@@ -2085,8 +2253,8 @@ static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl,
  * @clock_id:	the clock to be used
  * @mode:	timer mode abs/rel
  */
-void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl,
-				    clockid_t clock_id, enum hrtimer_mode mode)
+void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
+				    enum hrtimer_mode mode)
 {
 	debug_setup_on_stack(&sl->timer, clock_id, mode);
 	__hrtimer_setup_sleeper(sl, clock_id, mode);
@@ -2159,12 +2327,11 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
 	return ret;
 }
 
-long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
-		       const clockid_t clockid)
+long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, const clockid_t clockid)
 {
 	struct restart_block *restart;
 	struct hrtimer_sleeper t;
-	int ret = 0;
+	int ret;
 
 	hrtimer_setup_sleeper_on_stack(&t, clockid, mode);
 	hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);
@@ -2203,8 +2370,7 @@ SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
 	current->restart_block.fn = do_no_restart_syscall;
 	current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
 	current->restart_block.nanosleep.rmtp = rmtp;
-	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
-				 CLOCK_MONOTONIC);
+	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
 }
 
 #endif
@@ -2212,7 +2378,7 @@ SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
 #ifdef CONFIG_COMPAT_32BIT_TIME
 
 SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
-		       struct old_timespec32 __user *, rmtp)
+		struct old_timespec32 __user *, rmtp)
 {
 	struct timespec64 tu;
 
@@ -2225,8 +2391,7 @@ SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
 	current->restart_block.fn = do_no_restart_syscall;
 	current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
 	current->restart_block.nanosleep.compat_rmtp = rmtp;
-	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
-				 CLOCK_MONOTONIC);
+	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
 }
 #endif
 
@@ -2236,14 +2401,13 @@ SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
 int hrtimers_prepare_cpu(unsigned int cpu)
 {
 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
-	int i;
 
-	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
+	for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
 
 		clock_b->cpu_base = cpu_base;
 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
-		timerqueue_init_head(&clock_b->active);
+		timerqueue_linked_init_head(&clock_b->active);
 	}
 
 	cpu_base->cpu = cpu;
@@ -2257,13 +2421,14 @@ int hrtimers_cpu_starting(unsigned int cpu)
 
 	/* Clear out any left over state from a CPU down operation */
 	cpu_base->active_bases = 0;
-	cpu_base->hres_active = 0;
-	cpu_base->hang_detected = 0;
+	cpu_base->hres_active = false;
+	cpu_base->hang_detected = false;
 	cpu_base->next_timer = NULL;
 	cpu_base->softirq_next_timer = NULL;
 	cpu_base->expires_next = KTIME_MAX;
 	cpu_base->softirq_expires_next = KTIME_MAX;
-	cpu_base->online = 1;
+	cpu_base->softirq_activated = false;
+	cpu_base->online = true;
 	return 0;
 }
 
@@ -2272,20 +2437,20 @@ int hrtimers_cpu_starting(unsigned int cpu)
 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
 				struct hrtimer_clock_base *new_base)
 {
+	struct timerqueue_linked_node *node;
 	struct hrtimer *timer;
-	struct timerqueue_node *node;
 
-	while ((node = timerqueue_getnext(&old_base->active))) {
-		timer = container_of(node, struct hrtimer, node);
+	while ((node = timerqueue_linked_first(&old_base->active))) {
+		timer = hrtimer_from_timerqueue_node(node);
 		BUG_ON(hrtimer_callback_running(timer));
-		debug_deactivate(timer);
+		debug_hrtimer_deactivate(timer);
 
 		/*
 		 * Mark it as ENQUEUED not INACTIVE otherwise the
 		 * timer could be seen as !active and just vanish away
 		 * under us on another CPU
 		 */
-		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
+		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, false);
 		timer->base = new_base;
 		/*
 		 * Enqueue the timers on the new cpu. This does not
@@ -2295,13 +2460,13 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
 		 * sort out already expired timers and reprogram the
 		 * event device.
 		 */
-		enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
+		enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true);
 	}
 }
 
 int hrtimers_cpu_dying(unsigned int dying_cpu)
 {
-	int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
+	int ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
 	struct hrtimer_cpu_base *old_base, *new_base;
 
 	old_base = this_cpu_ptr(&hrtimer_bases);
@@ -2314,16 +2479,14 @@ int hrtimers_cpu_dying(unsigned int dying_cpu)
 	raw_spin_lock(&old_base->lock);
 	raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
 
-	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
-		migrate_hrtimer_list(&old_base->clock_base[i],
-				     &new_base->clock_base[i]);
-	}
+	for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
+		migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]);
 
 	/* Tell the other CPU to retrigger the next event */
 	smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
 
 	raw_spin_unlock(&new_base->lock);
-	old_base->online = 0;
+	old_base->online = false;
 	raw_spin_unlock(&old_base->lock);
 
 	return 0;
diff --git a/kernel/time/tick-broadcast-hrtimer.c b/kernel/time/tick-broadcast-hrtimer.c
index a88b72b0f35ee..51f6a1032c83e 100644
--- a/kernel/time/tick-broadcast-hrtimer.c
+++ b/kernel/time/tick-broadcast-hrtimer.c
@@ -78,7 +78,6 @@ static struct clock_event_device ce_broadcast_hrtimer = {
 	.set_state_shutdown	= bc_shutdown,
 	.set_next_ktime		= bc_set_next,
 	.features		= CLOCK_EVT_FEAT_ONESHOT |
-				  CLOCK_EVT_FEAT_KTIME |
 				  CLOCK_EVT_FEAT_HRTIMER,
 	.rating			= 0,
 	.bound_on		= -1,
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f7907fadd63f2..9e5264458414a 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -864,19 +864,32 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
+/* Simplified variant of hrtimer_forward_now() */
+static ktime_t tick_forward_now(ktime_t expires, ktime_t now)
+{
+	ktime_t delta = now - expires;
+
+	if (likely(delta < TICK_NSEC))
+		return expires + TICK_NSEC;
+
+	expires += TICK_NSEC * ktime_divns(delta, TICK_NSEC);
+	if (expires > now)
+		return expires;
+	return expires + TICK_NSEC;
+}
+
 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 {
-	hrtimer_cancel(&ts->sched_timer);
-	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
+	ktime_t expires = ts->last_tick;
 
-	/* Forward the time to expire in the future */
-	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
+	if (now >= expires)
+		expires = tick_forward_now(expires, now);
 
 	if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
-		hrtimer_start_expires(&ts->sched_timer,
-				      HRTIMER_MODE_ABS_PINNED_HARD);
+		hrtimer_start(&ts->sched_timer,	expires, HRTIMER_MODE_ABS_PINNED_HARD);
 	} else {
-		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
+		hrtimer_set_expires(&ts->sched_timer, expires);
+		tick_program_event(expires, 1);
 	}
 
 	/*
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index c07e562ee4c1a..c493a40103055 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -3,34 +3,30 @@
  *  Kernel timekeeping code and accessor functions. Based on code from
  *  timer.c, moved in commit 8524070b7982.
  */
-#include <linux/timekeeper_internal.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
+#include <linux/audit.h>
+#include <linux/clocksource.h>
+#include <linux/compiler.h>
+#include <linux/jiffies.h>
 #include <linux/kobject.h>
-#include <linux/percpu.h>
-#include <linux/init.h>
-#include <linux/mm.h>
+#include <linux/module.h>
 #include <linux/nmi.h>
-#include <linux/sched.h>
-#include <linux/sched/loadavg.h>
+#include <linux/pvclock_gtod.h>
+#include <linux/random.h>
 #include <linux/sched/clock.h>
+#include <linux/sched/loadavg.h>
+#include <linux/static_key.h>
+#include <linux/stop_machine.h>
 #include <linux/syscore_ops.h>
-#include <linux/clocksource.h>
-#include <linux/jiffies.h>
+#include <linux/tick.h>
 #include <linux/time.h>
 #include <linux/timex.h>
-#include <linux/tick.h>
-#include <linux/stop_machine.h>
-#include <linux/pvclock_gtod.h>
-#include <linux/compiler.h>
-#include <linux/audit.h>
-#include <linux/random.h>
+#include <linux/timekeeper_internal.h>
 
 #include <vdso/auxclock.h>
 
 #include "tick-internal.h"
-#include "ntp_internal.h"
 #include "timekeeping_internal.h"
+#include "ntp_internal.h"
 
 #define TK_CLEAR_NTP		(1 << 0)
 #define TK_CLOCK_WAS_SET	(1 << 1)
@@ -275,6 +271,11 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
 	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
 }
 
+#ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE
+#include <asm/clock_inlined.h>
+
+static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined);
+
 /*
  * tk_clock_read - atomic clocksource read() helper
  *
@@ -288,12 +289,35 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
  * a read of the fast-timekeeper tkrs (which is protected by its own locking
  * and update logic).
  */
-static inline u64 tk_clock_read(const struct tk_read_base *tkr)
+static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
+{
+	struct clocksource *clock = READ_ONCE(tkr->clock);
+
+	if (static_branch_likely(&clocksource_read_inlined))
+		return arch_inlined_clocksource_read(clock);
+
+	return clock->read(clock);
+}
+
+static inline void clocksource_disable_inline_read(void)
+{
+	static_branch_disable(&clocksource_read_inlined);
+}
+
+static inline void clocksource_enable_inline_read(void)
+{
+	static_branch_enable(&clocksource_read_inlined);
+}
+#else
+static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
 {
 	struct clocksource *clock = READ_ONCE(tkr->clock);
 
 	return clock->read(clock);
 }
+static inline void clocksource_disable_inline_read(void) { }
+static inline void clocksource_enable_inline_read(void) { }
+#endif
 
 /**
  * tk_setup_internals - Set up internals to use clocksource clock.
@@ -367,6 +391,27 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 	tk->tkr_raw.mult = clock->mult;
 	tk->ntp_err_mult = 0;
 	tk->skip_second_overflow = 0;
+
+	tk->cs_id = clock->id;
+
+	/* Coupled clockevent data */
+	if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) &&
+	    clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT) {
+		/*
+		 * Aim for an one hour maximum delta and use KHz to handle
+		 * clocksources with a frequency above 4GHz correctly as
+		 * the frequency argument of clocks_calc_mult_shift() is u32.
+		 */
+		clocks_calc_mult_shift(&tk->cs_ns_to_cyc_mult, &tk->cs_ns_to_cyc_shift,
+				       NSEC_PER_MSEC, clock->freq_khz, 3600 * 1000);
+		/*
+		 * Initialize the conversion limit as the previous clocksource
+		 * might have the same shift/mult pair so the quick check in
+		 * tk_update_ns_to_cyc() fails to update it after a clocksource
+		 * change leaving it effectivly zero.
+		 */
+		tk->cs_ns_to_cyc_maxns = div_u64(clock->mask, tk->cs_ns_to_cyc_mult);
+	}
 }
 
 /* Timekeeper helper functions. */
@@ -375,7 +420,7 @@ static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
 	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
 }
 
-static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
+static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
 {
 	/* Calculate the delta since the last update_wall_time() */
 	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
@@ -696,6 +741,36 @@ static inline void tk_update_ktime_data(struct timekeeper *tk)
 	tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
 }
 
+static inline void tk_update_ns_to_cyc(struct timekeeper *tks, struct timekeeper *tkc)
+{
+	struct tk_read_base *tkrs = &tks->tkr_mono;
+	struct tk_read_base *tkrc = &tkc->tkr_mono;
+	unsigned int shift;
+
+	if (!IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) ||
+	    !(tkrs->clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT))
+		return;
+
+	if (tkrs->mult == tkrc->mult && tkrs->shift == tkrc->shift)
+		return;
+	/*
+	 * The conversion math is simple:
+	 *
+	 *      CS::MULT       (1 << NS_TO_CYC_SHIFT)
+	 *   --------------- = ----------------------
+	 *   (1 << CS:SHIFT)       NS_TO_CYC_MULT
+	 *
+	 * Ergo:
+	 *
+	 *   NS_TO_CYC_MULT = (1 << (CS::SHIFT + NS_TO_CYC_SHIFT)) / CS::MULT
+	 *
+	 * NS_TO_CYC_SHIFT has been set up in tk_setup_internals()
+	 */
+	shift = tkrs->shift + tks->cs_ns_to_cyc_shift;
+	tks->cs_ns_to_cyc_mult = (u32)div_u64(1ULL << shift, tkrs->mult);
+	tks->cs_ns_to_cyc_maxns = div_u64(tkrs->clock->mask, tks->cs_ns_to_cyc_mult);
+}
+
 /*
  * Restore the shadow timekeeper from the real timekeeper.
  */
@@ -730,6 +805,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
 	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
 
 	if (tk->id == TIMEKEEPER_CORE) {
+		tk_update_ns_to_cyc(tk, &tkd->timekeeper);
 		update_vsyscall(tk);
 		update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
 
@@ -784,6 +860,71 @@ static void timekeeping_forward_now(struct timekeeper *tk)
 	tk_update_coarse_nsecs(tk);
 }
 
+/*
+ * ktime_expiry_to_cycles - Convert a expiry time to clocksource cycles
+ * @id:		Clocksource ID which is required for validity
+ * @expires_ns:	Absolute CLOCK_MONOTONIC expiry time (nsecs) to be converted
+ * @cycles:	Pointer to storage for corresponding absolute cycles value
+ *
+ * Convert a CLOCK_MONOTONIC based absolute expiry time to a cycles value
+ * based on the correlated clocksource of the clockevent device by using
+ * the base nanoseconds and cycles values of the last timekeeper update and
+ * converting the delta between @expires_ns and base nanoseconds to cycles.
+ *
+ * This only works for clockevent devices which are using a less than or
+ * equal comparator against the clocksource.
+ *
+ * Utilizing this avoids two clocksource reads for such devices, the
+ * ktime_get() in clockevents_program_event() to calculate the delta expiry
+ * value and the readout in the device::set_next_event() callback to
+ * convert the delta back to a absolute comparator value.
+ *
+ * Returns: True if @id matches the current clocksource ID, false otherwise
+ */
+bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles)
+{
+	struct timekeeper *tk = &tk_core.timekeeper;
+	struct tk_read_base *tkrm = &tk->tkr_mono;
+	ktime_t base_ns, delta_ns, max_ns;
+	u64 base_cycles, delta_cycles;
+	unsigned int seq;
+	u32 mult, shift;
+
+	/*
+	 * Racy check to avoid the seqcount overhead when ID does not match. If
+	 * the relevant clocksource is installed concurrently, then this will
+	 * just delay the switch over to this mechanism until the next event is
+	 * programmed. If the ID is not matching the clock events code will use
+	 * the regular relative set_next_event() callback as before.
+	 */
+	if (data_race(tk->cs_id) != id)
+		return false;
+
+	do {
+		seq = read_seqcount_begin(&tk_core.seq);
+
+		if (tk->cs_id != id)
+			return false;
+
+		base_cycles = tkrm->cycle_last;
+		base_ns = tkrm->base + (tkrm->xtime_nsec >> tkrm->shift);
+
+		mult = tk->cs_ns_to_cyc_mult;
+		shift = tk->cs_ns_to_cyc_shift;
+		max_ns = tk->cs_ns_to_cyc_maxns;
+
+	} while (read_seqcount_retry(&tk_core.seq, seq));
+
+	/* Prevent negative deltas and multiplication overflows */
+	delta_ns = min(expires_ns - base_ns, max_ns);
+	delta_ns = max(delta_ns, 0);
+
+	/* Convert to cycles */
+	delta_cycles = ((u64)delta_ns * mult) >> shift;
+	*cycles = base_cycles + delta_cycles;
+	return true;
+}
+
 /**
  * ktime_get_real_ts64 - Returns the time of day in a timespec64.
  * @ts:		pointer to the timespec to be set
@@ -848,7 +989,7 @@ u32 ktime_get_resolution_ns(void)
 }
 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
 
-static ktime_t *offsets[TK_OFFS_MAX] = {
+static const ktime_t *const offsets[TK_OFFS_MAX] = {
 	[TK_OFFS_REAL]	= &tk_core.timekeeper.offs_real,
 	[TK_OFFS_BOOT]	= &tk_core.timekeeper.offs_boot,
 	[TK_OFFS_TAI]	= &tk_core.timekeeper.offs_tai,
@@ -857,8 +998,9 @@ static ktime_t *offsets[TK_OFFS_MAX] = {
 ktime_t ktime_get_with_offset(enum tk_offsets offs)
 {
 	struct timekeeper *tk = &tk_core.timekeeper;
+	const ktime_t *offset = offsets[offs];
 	unsigned int seq;
-	ktime_t base, *offset = offsets[offs];
+	ktime_t base;
 	u64 nsecs;
 
 	WARN_ON(timekeeping_suspended);
@@ -878,8 +1020,9 @@ EXPORT_SYMBOL_GPL(ktime_get_with_offset);
 ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
 {
 	struct timekeeper *tk = &tk_core.timekeeper;
-	ktime_t base, *offset = offsets[offs];
+	const ktime_t *offset = offsets[offs];
 	unsigned int seq;
+	ktime_t base;
 	u64 nsecs;
 
 	WARN_ON(timekeeping_suspended);
@@ -902,7 +1045,7 @@ EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
  */
 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
 {
-	ktime_t *offset = offsets[offs];
+	const ktime_t *offset = offsets[offs];
 	unsigned int seq;
 	ktime_t tconv;
 
@@ -1631,7 +1774,19 @@ int timekeeping_notify(struct clocksource *clock)
 
 	if (tk->tkr_mono.clock == clock)
 		return 0;
+
+	/* Disable inlined reads accross the clocksource switch */
+	clocksource_disable_inline_read();
+
 	stop_machine(change_clocksource, clock, NULL);
+
+	/*
+	 * If the clocksource has been selected and supports inlined reads
+	 * enable the branch.
+	 */
+	if (tk->tkr_mono.clock == clock && clock->flags & CLOCK_SOURCE_CAN_INLINE_READ)
+		clocksource_enable_inline_read();
+
 	tick_clock_notify();
 	return tk->tkr_mono.clock == clock ? 0 : -1;
 }
@@ -2834,7 +2989,7 @@ static void tk_aux_update_clocksource(void)
 			continue;
 
 		timekeeping_forward_now(tks);
-		tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock);
+		tk_setup_internals(tks, tk_core.timekeeper.tkr_raw.clock);
 		timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
 	}
 }
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
index 543beba096c75..198d0608db742 100644
--- a/kernel/time/timekeeping.h
+++ b/kernel/time/timekeeping.h
@@ -9,6 +9,8 @@ extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq,
 					    ktime_t *offs_boot,
 					    ktime_t *offs_tai);
 
+bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles);
+
 extern int timekeeping_valid_for_hres(void);
 extern u64 timekeeping_max_deferment(void);
 extern void timekeeping_warp_clock(void);
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 488e47e96e93f..427d7ddea3afb 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -47,7 +47,7 @@ print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
 	    int idx, u64 now)
 {
 	SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, ACCESS_PRIVATE(timer, function));
-	SEQ_printf(m, ", S:%02x", timer->state);
+	SEQ_printf(m, ", S:%02x", timer->is_queued);
 	SEQ_printf(m, "\n");
 	SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n",
 		(unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)),
@@ -56,13 +56,11 @@ print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
 		(long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now));
 }
 
-static void
-print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base,
-		    u64 now)
+static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
 {
+	struct timerqueue_linked_node *curr;
 	struct hrtimer *timer, tmp;
 	unsigned long next = 0, i;
-	struct timerqueue_node *curr;
 	unsigned long flags;
 
 next_one:
@@ -72,13 +70,13 @@ print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base,
 
 	raw_spin_lock_irqsave(&base->cpu_base->lock, flags);
 
-	curr = timerqueue_getnext(&base->active);
+	curr = timerqueue_linked_first(&base->active);
 	/*
 	 * Crude but we have to do this O(N*N) thing, because
 	 * we have to unlock the base when printing:
 	 */
 	while (curr && i < next) {
-		curr = timerqueue_iterate_next(curr);
+		curr = timerqueue_linked_next(curr);
 		i++;
 	}
 
@@ -103,8 +101,8 @@ print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
 
 	SEQ_printf(m, "  .resolution: %u nsecs\n", hrtimer_resolution);
 #ifdef CONFIG_HIGH_RES_TIMERS
-	SEQ_printf(m, "  .offset:     %Lu nsecs\n",
-		   (unsigned long long) ktime_to_ns(base->offset));
+	SEQ_printf(m, "  .offset:     %Ld nsecs\n",
+		   (long long) base->offset);
 #endif
 	SEQ_printf(m,   "active timers:\n");
 	print_active_timers(m, base, now + ktime_to_ns(base->offset));
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 49de13cae4288..e130da35808f4 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -1281,4 +1281,18 @@ config HIST_TRIGGERS_DEBUG
 
 source "kernel/trace/rv/Kconfig"
 
+config TRACE_REMOTE
+	bool
+
+config SIMPLE_RING_BUFFER
+	bool
+
+config TRACE_REMOTE_TEST
+	tristate "Test module for remote tracing"
+	select TRACE_REMOTE
+	select SIMPLE_RING_BUFFER
+	help
+	  This trace remote includes a ring-buffer writer implementation using
+	  "simple_ring_buffer". This is solely intending for testing.
+
 endif # FTRACE
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 04096c21d06b7..c5e14ffd36ee2 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -128,4 +128,57 @@ obj-$(CONFIG_FPROBE_EVENTS) += trace_fprobe.o
 obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o
 obj-$(CONFIG_RV) += rv/
 
+obj-$(CONFIG_TRACE_REMOTE) += trace_remote.o
+obj-$(CONFIG_SIMPLE_RING_BUFFER) += simple_ring_buffer.o
+obj-$(CONFIG_TRACE_REMOTE_TEST) += remote_test.o
+
+#
+# simple_ring_buffer is used by the pKVM hypervisor which does not have access
+# to all kernel symbols. Fail the build if forbidden symbols are found.
+#
+# undefsyms_base generates a set of compiler and tooling-generated symbols that can
+# safely be ignored for simple_ring_buffer.
+#
+filechk_undefsyms_base = \
+	echo '$(pound)include <linux/atomic.h>'; \
+	echo '$(pound)include <linux/string.h>'; \
+	echo '$(pound)include <asm/page.h>'; \
+	echo 'static char page[PAGE_SIZE] __aligned(PAGE_SIZE);'; \
+	echo 'void undefsyms_base(void *p, int n);'; \
+	echo 'void undefsyms_base(void *p, int n) {'; \
+	echo '	char buffer[256] = { 0 };'; \
+	echo '	u32 u = 0;'; \
+	echo '	memset((char * volatile)page, 8, PAGE_SIZE);'; \
+	echo '	memset((char * volatile)buffer, 8, sizeof(buffer));'; \
+	echo '	memcpy((void * volatile)p, buffer, sizeof(buffer));'; \
+	echo '	cmpxchg((u32 * volatile)&u, 0, 8);'; \
+	echo '	WARN_ON(n == 0xdeadbeef);'; \
+	echo '}'
+
+$(obj)/undefsyms_base.c: FORCE
+	$(call filechk,undefsyms_base)
+
+clean-files += undefsyms_base.c
+
+$(obj)/undefsyms_base.o: $(obj)/undefsyms_base.c
+
+targets += undefsyms_base.o
+
+# Ensure KASAN is enabled to avoid logic that may disable FORTIFY_SOURCE when
+# KASAN is not enabled. undefsyms_base.o does not automatically get KASAN flags
+# because it is not linked into vmlinux.
+KASAN_SANITIZE_undefsyms_base.o := y
+
+UNDEFINED_ALLOWLIST = __asan __gcov __kasan __kcsan __hwasan __sancov __sanitizer __tsan __ubsan __x86_indirect_thunk \
+		      __msan simple_ring_buffer \
+		      $(shell $(NM) -u $(obj)/undefsyms_base.o 2>/dev/null | awk '{print $$2}')
+
+quiet_cmd_check_undefined = NM      $<
+      cmd_check_undefined = test -z "`$(NM) -u $< | grep -v $(addprefix -e , $(UNDEFINED_ALLOWLIST))`"
+
+$(obj)/%.o.checked: $(obj)/%.o $(obj)/undefsyms_base.o FORCE
+	$(call if_changed,check_undefined)
+
+always-$(CONFIG_SIMPLE_RING_BUFFER) += simple_ring_buffer.o.checked
+
 libftrace-y := ftrace.o
diff --git a/kernel/trace/remote_test.c b/kernel/trace/remote_test.c
new file mode 100644
index 0000000000000..6c1b7701ddae8
--- /dev/null
+++ b/kernel/trace/remote_test.c
@@ -0,0 +1,261 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 - Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <linux/module.h>
+#include <linux/simple_ring_buffer.h>
+#include <linux/trace_remote.h>
+#include <linux/tracefs.h>
+#include <linux/types.h>
+
+#define REMOTE_EVENT_INCLUDE_FILE kernel/trace/remote_test_events.h
+#include <trace/define_remote_events.h>
+
+static DEFINE_PER_CPU(struct simple_rb_per_cpu *, simple_rbs);
+static struct trace_buffer_desc *remote_test_buffer_desc;
+
+/*
+ * The trace_remote lock already serializes accesses from the trace_remote_callbacks.
+ * However write_event can still race with load/unload.
+ */
+static DEFINE_MUTEX(simple_rbs_lock);
+
+static int remote_test_load_simple_rb(int cpu, struct ring_buffer_desc *rb_desc)
+{
+	struct simple_rb_per_cpu *cpu_buffer;
+	struct simple_buffer_page *bpages;
+	int ret = -ENOMEM;
+
+	cpu_buffer = kmalloc_obj(*cpu_buffer);
+	if (!cpu_buffer)
+		return ret;
+
+	bpages = kmalloc_objs(*bpages, rb_desc->nr_page_va);
+	if (!bpages)
+		goto err_free_cpu_buffer;
+
+	ret = simple_ring_buffer_init(cpu_buffer, bpages, rb_desc);
+	if (ret)
+		goto err_free_bpages;
+
+	scoped_guard(mutex, &simple_rbs_lock) {
+		WARN_ON(*per_cpu_ptr(&simple_rbs, cpu));
+		*per_cpu_ptr(&simple_rbs, cpu) = cpu_buffer;
+	}
+
+	return 0;
+
+err_free_bpages:
+	kfree(bpages);
+
+err_free_cpu_buffer:
+	kfree(cpu_buffer);
+
+	return ret;
+}
+
+static void remote_test_unload_simple_rb(int cpu)
+{
+	struct simple_rb_per_cpu *cpu_buffer = *per_cpu_ptr(&simple_rbs, cpu);
+	struct simple_buffer_page *bpages;
+
+	if (!cpu_buffer)
+		return;
+
+	guard(mutex)(&simple_rbs_lock);
+
+	bpages = cpu_buffer->bpages;
+	simple_ring_buffer_unload(cpu_buffer);
+	kfree(bpages);
+	kfree(cpu_buffer);
+	*per_cpu_ptr(&simple_rbs, cpu) = NULL;
+}
+
+static struct trace_buffer_desc *remote_test_load(unsigned long size, void *unused)
+{
+	struct ring_buffer_desc *rb_desc;
+	struct trace_buffer_desc *desc;
+	size_t desc_size;
+	int cpu, ret;
+
+	if (WARN_ON(remote_test_buffer_desc))
+		return ERR_PTR(-EINVAL);
+
+	desc_size = trace_buffer_desc_size(size, num_possible_cpus());
+	if (desc_size == SIZE_MAX) {
+		ret = -E2BIG;
+		goto err;
+	}
+
+	desc = kmalloc(desc_size, GFP_KERNEL);
+	if (!desc) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = trace_remote_alloc_buffer(desc, desc_size, size, cpu_possible_mask);
+	if (ret)
+		goto err_free_desc;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, desc) {
+		ret = remote_test_load_simple_rb(rb_desc->cpu, rb_desc);
+		if (ret)
+			goto err_unload;
+	}
+
+	remote_test_buffer_desc = desc;
+
+	return remote_test_buffer_desc;
+
+err_unload:
+	for_each_ring_buffer_desc(rb_desc, cpu, remote_test_buffer_desc)
+		remote_test_unload_simple_rb(rb_desc->cpu);
+	trace_remote_free_buffer(remote_test_buffer_desc);
+
+err_free_desc:
+	kfree(desc);
+
+err:
+	return ERR_PTR(ret);
+}
+
+static void remote_test_unload(struct trace_buffer_desc *desc, void *unused)
+{
+	struct ring_buffer_desc *rb_desc;
+	int cpu;
+
+	if (WARN_ON(desc != remote_test_buffer_desc))
+		return;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, desc)
+		remote_test_unload_simple_rb(rb_desc->cpu);
+
+	remote_test_buffer_desc = NULL;
+	trace_remote_free_buffer(desc);
+	kfree(desc);
+}
+
+static int remote_test_enable_tracing(bool enable, void *unused)
+{
+	struct ring_buffer_desc *rb_desc;
+	int cpu;
+
+	if (!remote_test_buffer_desc)
+		return -ENODEV;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, remote_test_buffer_desc)
+		WARN_ON(simple_ring_buffer_enable_tracing(*per_cpu_ptr(&simple_rbs, rb_desc->cpu),
+							  enable));
+	return 0;
+}
+
+static int remote_test_swap_reader_page(unsigned int cpu, void *unused)
+{
+	struct simple_rb_per_cpu *cpu_buffer;
+
+	if (cpu >= NR_CPUS)
+		return -EINVAL;
+
+	cpu_buffer = *per_cpu_ptr(&simple_rbs, cpu);
+	if (!cpu_buffer)
+		return -EINVAL;
+
+	return simple_ring_buffer_swap_reader_page(cpu_buffer);
+}
+
+static int remote_test_reset(unsigned int cpu, void *unused)
+{
+	struct simple_rb_per_cpu *cpu_buffer;
+
+	if (cpu >= NR_CPUS)
+		return -EINVAL;
+
+	cpu_buffer = *per_cpu_ptr(&simple_rbs, cpu);
+	if (!cpu_buffer)
+		return -EINVAL;
+
+	return simple_ring_buffer_reset(cpu_buffer);
+}
+
+static int remote_test_enable_event(unsigned short id, bool enable, void *unused)
+{
+	if (id != REMOTE_TEST_EVENT_ID)
+		return -EINVAL;
+
+	/*
+	 * Let's just use the struct remote_event enabled field that is turned on and off by
+	 * trace_remote. This is a bit racy but good enough for a simple test module.
+	 */
+	return 0;
+}
+
+static ssize_t
+write_event_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *pos)
+{
+	struct remote_event_format_selftest *evt_test;
+	struct simple_rb_per_cpu *cpu_buffer;
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&simple_rbs_lock);
+
+	if (!remote_event_selftest.enabled)
+		return -ENODEV;
+
+	guard(preempt)();
+
+	cpu_buffer = *this_cpu_ptr(&simple_rbs);
+	if (!cpu_buffer)
+		return -ENODEV;
+
+	evt_test = simple_ring_buffer_reserve(cpu_buffer,
+					      sizeof(struct remote_event_format_selftest),
+					      trace_clock_global());
+	if (!evt_test)
+		return -ENODEV;
+
+	evt_test->hdr.id = REMOTE_TEST_EVENT_ID;
+	evt_test->id = val;
+
+	simple_ring_buffer_commit(cpu_buffer);
+
+	return cnt;
+}
+
+static const struct file_operations write_event_fops = {
+	.write	= write_event_write,
+};
+
+static int remote_test_init_tracefs(struct dentry *d, void *unused)
+{
+	return tracefs_create_file("write_event", 0200, d, NULL, &write_event_fops) ?
+		0 : -ENOMEM;
+}
+
+static struct trace_remote_callbacks trace_remote_callbacks = {
+	.init			= remote_test_init_tracefs,
+	.load_trace_buffer	= remote_test_load,
+	.unload_trace_buffer	= remote_test_unload,
+	.enable_tracing		= remote_test_enable_tracing,
+	.swap_reader_page	= remote_test_swap_reader_page,
+	.reset			= remote_test_reset,
+	.enable_event		= remote_test_enable_event,
+};
+
+static int __init remote_test_init(void)
+{
+	return trace_remote_register("test", &trace_remote_callbacks, NULL,
+				     &remote_event_selftest, 1);
+}
+
+module_init(remote_test_init);
+
+MODULE_DESCRIPTION("Test module for the trace remote interface");
+MODULE_AUTHOR("Vincent Donnefort");
+MODULE_LICENSE("GPL");
diff --git a/kernel/trace/remote_test_events.h b/kernel/trace/remote_test_events.h
new file mode 100644
index 0000000000000..26b93b3406fc3
--- /dev/null
+++ b/kernel/trace/remote_test_events.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#define REMOTE_TEST_EVENT_ID 1
+
+REMOTE_EVENT(selftest, REMOTE_TEST_EVENT_ID,
+	RE_STRUCT(
+		re_field(u64, id)
+	),
+	RE_PRINTK("id=%llu", __entry->id)
+);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 17d0ea0cc3e6f..96e0d80d492b6 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -4,6 +4,7 @@
  *
  * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
  */
+#include <linux/ring_buffer_types.h>
 #include <linux/sched/isolation.h>
 #include <linux/trace_recursion.h>
 #include <linux/trace_events.h>
@@ -157,23 +158,6 @@ int ring_buffer_print_entry_header(struct trace_seq *s)
 /* Used for individual buffers (after the counter) */
 #define RB_BUFFER_OFF		(1 << 20)
 
-#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
-
-#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
-#define RB_ALIGNMENT		4U
-#define RB_MAX_SMALL_DATA	(RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
-#define RB_EVNT_MIN_SIZE	8U	/* two 32bit words */
-
-#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS
-# define RB_FORCE_8BYTE_ALIGNMENT	0
-# define RB_ARCH_ALIGNMENT		RB_ALIGNMENT
-#else
-# define RB_FORCE_8BYTE_ALIGNMENT	1
-# define RB_ARCH_ALIGNMENT		8U
-#endif
-
-#define RB_ALIGN_DATA		__aligned(RB_ARCH_ALIGNMENT)
-
 /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
 #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
 
@@ -316,10 +300,6 @@ EXPORT_SYMBOL_GPL(ring_buffer_event_data);
 #define for_each_online_buffer_cpu(buffer, cpu)		\
 	for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
 
-#define TS_SHIFT	27
-#define TS_MASK		((1ULL << TS_SHIFT) - 1)
-#define TS_DELTA_TEST	(~TS_MASK)
-
 static u64 rb_event_time_stamp(struct ring_buffer_event *event)
 {
 	u64 ts;
@@ -338,12 +318,6 @@ static u64 rb_event_time_stamp(struct ring_buffer_event *event)
 
 #define RB_MISSED_MASK		(3 << 30)
 
-struct buffer_data_page {
-	u64		 time_stamp;	/* page time stamp */
-	local_t		 commit;	/* write committed index */
-	unsigned char	 data[] RB_ALIGN_DATA;	/* data of buffer page */
-};
-
 struct buffer_data_read_page {
 	unsigned		order;	/* order of the page */
 	struct buffer_data_page	*data;	/* actual data, stored in this page */
@@ -437,14 +411,6 @@ static struct buffer_data_page *alloc_cpu_data(int cpu, int order)
 	return dpage;
 }
 
-/*
- * We need to fit the time_stamp delta into 27 bits.
- */
-static inline bool test_time_stamp(u64 delta)
-{
-	return !!(delta & TS_DELTA_TEST);
-}
-
 struct rb_irq_work {
 	struct irq_work			work;
 	wait_queue_head_t		waiters;
@@ -555,10 +521,12 @@ struct ring_buffer_per_cpu {
 	unsigned int			mapped;
 	unsigned int			user_mapped;	/* user space mapping */
 	struct mutex			mapping_lock;
-	unsigned long			*subbuf_ids;	/* ID to subbuf VA */
+	struct buffer_page		**subbuf_ids;	/* ID to subbuf VA */
 	struct trace_buffer_meta	*meta_page;
 	struct ring_buffer_cpu_meta	*ring_meta;
 
+	struct ring_buffer_remote	*remote;
+
 	/* ring buffer pages to update, > 0 to add, < 0 to remove */
 	long				nr_pages_to_update;
 	struct list_head		new_pages; /* new pages to add */
@@ -581,6 +549,8 @@ struct trace_buffer {
 
 	struct ring_buffer_per_cpu	**buffers;
 
+	struct ring_buffer_remote	*remote;
+
 	struct hlist_node		node;
 	u64				(*clock)(void);
 
@@ -636,7 +606,8 @@ int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq
 	trace_seq_printf(s, "\tfield: char data;\t"
 			 "offset:%u;\tsize:%u;\tsigned:%u;\n",
 			 (unsigned int)offsetof(typeof(field), data),
-			 (unsigned int)buffer->subbuf_size,
+			 (unsigned int)(buffer ? buffer->subbuf_size :
+						 PAGE_SIZE - BUF_PAGE_HDR_SIZE),
 			 (unsigned int)is_signed_type(char));
 
 	return !trace_seq_has_overflowed(s);
@@ -2238,6 +2209,40 @@ static void rb_meta_buffer_update(struct ring_buffer_per_cpu *cpu_buffer,
 	}
 }
 
+static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace_desc, int cpu)
+{
+	struct ring_buffer_desc *desc, *end;
+	size_t len;
+	int i;
+
+	if (!trace_desc)
+		return NULL;
+
+	if (cpu >= trace_desc->nr_cpus)
+		return NULL;
+
+	end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len);
+	desc = __first_ring_buffer_desc(trace_desc);
+	len = struct_size(desc, page_va, desc->nr_page_va);
+	desc = (struct ring_buffer_desc *)((void *)desc + (len * cpu));
+
+	if (desc < end && desc->cpu == cpu)
+		return desc;
+
+	/* Missing CPUs, need to linear search */
+	for_each_ring_buffer_desc(desc, i, trace_desc) {
+		if (desc->cpu == cpu)
+			return desc;
+	}
+
+	return NULL;
+}
+
+static void *ring_buffer_desc_page(struct ring_buffer_desc *desc, int page_id)
+{
+	return page_id > desc->nr_page_va ? NULL : (void *)desc->page_va[page_id];
+}
+
 static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
 		long nr_pages, struct list_head *pages)
 {
@@ -2245,6 +2250,7 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
 	struct ring_buffer_cpu_meta *meta = NULL;
 	struct buffer_page *bpage, *tmp;
 	bool user_thread = current->mm != NULL;
+	struct ring_buffer_desc *desc = NULL;
 	long i;
 
 	/*
@@ -2273,6 +2279,12 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
 	if (buffer->range_addr_start)
 		meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu);
 
+	if (buffer->remote) {
+		desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu);
+		if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1)))
+			return -EINVAL;
+	}
+
 	for (i = 0; i < nr_pages; i++) {
 
 		bpage = alloc_cpu_page(cpu_buffer->cpu);
@@ -2297,6 +2309,16 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
 				rb_meta_buffer_update(cpu_buffer, bpage);
 			bpage->range = 1;
 			bpage->id = i + 1;
+		} else if (desc) {
+			void *p = ring_buffer_desc_page(desc, i + 1);
+
+			if (WARN_ON(!p))
+				goto free_pages;
+
+			bpage->page = p;
+			bpage->range = 1; /* bpage->page can't be freed */
+			bpage->id = i + 1;
+			cpu_buffer->subbuf_ids[i + 1] = bpage;
 		} else {
 			int order = cpu_buffer->buffer->subbuf_order;
 			bpage->page = alloc_cpu_data(cpu_buffer->cpu, order);
@@ -2394,6 +2416,30 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
 		if (cpu_buffer->ring_meta->head_buffer)
 			rb_meta_buffer_update(cpu_buffer, bpage);
 		bpage->range = 1;
+	} else if (buffer->remote) {
+		struct ring_buffer_desc *desc = ring_buffer_desc(buffer->remote->desc, cpu);
+
+		if (!desc)
+			goto fail_free_reader;
+
+		cpu_buffer->remote = buffer->remote;
+		cpu_buffer->meta_page = (struct trace_buffer_meta *)(void *)desc->meta_va;
+		cpu_buffer->nr_pages = nr_pages;
+		cpu_buffer->subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1,
+						 sizeof(*cpu_buffer->subbuf_ids), GFP_KERNEL);
+		if (!cpu_buffer->subbuf_ids)
+			goto fail_free_reader;
+
+		/* Remote buffers are read-only and immutable */
+		atomic_inc(&cpu_buffer->record_disabled);
+		atomic_inc(&cpu_buffer->resize_disabled);
+
+		bpage->page = ring_buffer_desc_page(desc, cpu_buffer->meta_page->reader.id);
+		if (!bpage->page)
+			goto fail_free_reader;
+
+		bpage->range = 1;
+		cpu_buffer->subbuf_ids[0] = bpage;
 	} else {
 		int order = cpu_buffer->buffer->subbuf_order;
 		bpage->page = alloc_cpu_data(cpu, order);
@@ -2453,6 +2499,9 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
 
 	irq_work_sync(&cpu_buffer->irq_work.work);
 
+	if (cpu_buffer->remote)
+		kfree(cpu_buffer->subbuf_ids);
+
 	free_buffer_page(cpu_buffer->reader_page);
 
 	if (head) {
@@ -2475,7 +2524,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
 					 int order, unsigned long start,
 					 unsigned long end,
 					 unsigned long scratch_size,
-					 struct lock_class_key *key)
+					 struct lock_class_key *key,
+					 struct ring_buffer_remote *remote)
 {
 	struct trace_buffer *buffer __free(kfree) = NULL;
 	long nr_pages;
@@ -2515,6 +2565,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
 	if (!buffer->buffers)
 		goto fail_free_cpumask;
 
+	cpu = raw_smp_processor_id();
+
 	/* If start/end are specified, then that overrides size */
 	if (start && end) {
 		unsigned long buffers_start;
@@ -2570,6 +2622,15 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
 		buffer->range_addr_end = end;
 
 		rb_range_meta_init(buffer, nr_pages, scratch_size);
+	} else if (remote) {
+		struct ring_buffer_desc *desc = ring_buffer_desc(remote->desc, cpu);
+
+		buffer->remote = remote;
+		/* The writer is remote. This ring-buffer is read-only */
+		atomic_inc(&buffer->record_disabled);
+		nr_pages = desc->nr_page_va - 1;
+		if (nr_pages < 2)
+			goto fail_free_buffers;
 	} else {
 
 		/* need at least two pages */
@@ -2578,7 +2639,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
 			nr_pages = 2;
 	}
 
-	cpu = raw_smp_processor_id();
 	cpumask_set_cpu(cpu, buffer->cpumask);
 	buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
 	if (!buffer->buffers[cpu])
@@ -2620,7 +2680,7 @@ struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
 					struct lock_class_key *key)
 {
 	/* Default buffer page size - one system page */
-	return alloc_buffer(size, flags, 0, 0, 0, 0, key);
+	return alloc_buffer(size, flags, 0, 0, 0, 0, key, NULL);
 
 }
 EXPORT_SYMBOL_GPL(__ring_buffer_alloc);
@@ -2647,7 +2707,18 @@ struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flag
 					       struct lock_class_key *key)
 {
 	return alloc_buffer(size, flags, order, start, start + range_size,
-			    scratch_size, key);
+			    scratch_size, key, NULL);
+}
+
+/**
+ * __ring_buffer_alloc_remote - allocate a new ring_buffer from a remote
+ * @remote: Contains a description of the ring-buffer pages and remote callbacks.
+ * @key: ring buffer reader_lock_key.
+ */
+struct trace_buffer *__ring_buffer_alloc_remote(struct ring_buffer_remote *remote,
+						struct lock_class_key *key)
+{
+	return alloc_buffer(0, 0, 0, 0, 0, 0, key, remote);
 }
 
 void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size)
@@ -5274,10 +5345,61 @@ unsigned long ring_buffer_overruns(struct trace_buffer *buffer)
 }
 EXPORT_SYMBOL_GPL(ring_buffer_overruns);
 
+static bool rb_read_remote_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
+{
+	local_set(&cpu_buffer->entries, READ_ONCE(cpu_buffer->meta_page->entries));
+	local_set(&cpu_buffer->overrun, READ_ONCE(cpu_buffer->meta_page->overrun));
+	local_set(&cpu_buffer->pages_touched, READ_ONCE(cpu_buffer->meta_page->pages_touched));
+	local_set(&cpu_buffer->pages_lost, READ_ONCE(cpu_buffer->meta_page->pages_lost));
+
+	return rb_num_of_entries(cpu_buffer);
+}
+
+static void rb_update_remote_head(struct ring_buffer_per_cpu *cpu_buffer)
+{
+	struct buffer_page *next, *orig;
+	int retry = 3;
+
+	orig = next = cpu_buffer->head_page;
+	rb_inc_page(&next);
+
+	/* Run after the writer */
+	while (cpu_buffer->head_page->page->time_stamp > next->page->time_stamp) {
+		rb_inc_page(&next);
+
+		rb_list_head_clear(cpu_buffer->head_page->list.prev);
+		rb_inc_page(&cpu_buffer->head_page);
+		rb_set_list_to_head(cpu_buffer->head_page->list.prev);
+
+		if (cpu_buffer->head_page == orig) {
+			if (WARN_ON_ONCE(!(--retry)))
+				return;
+		}
+	}
+
+	orig = cpu_buffer->commit_page = cpu_buffer->head_page;
+	retry = 3;
+
+	while (cpu_buffer->commit_page->page->time_stamp < next->page->time_stamp) {
+		rb_inc_page(&next);
+		rb_inc_page(&cpu_buffer->commit_page);
+
+		if (cpu_buffer->commit_page == orig) {
+			if (WARN_ON_ONCE(!(--retry)))
+				return;
+		}
+	}
+}
+
 static void rb_iter_reset(struct ring_buffer_iter *iter)
 {
 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
 
+	if (cpu_buffer->remote) {
+		rb_read_remote_meta_page(cpu_buffer);
+		rb_update_remote_head(cpu_buffer);
+	}
+
 	/* Iterator usage is expected to have record disabled */
 	iter->head_page = cpu_buffer->reader_page;
 	iter->head = cpu_buffer->reader_page->read;
@@ -5428,7 +5550,65 @@ rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
 }
 
 static struct buffer_page *
-rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
+__rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
+{
+	struct buffer_page *new_reader, *prev_reader, *prev_head, *new_head, *last;
+
+	if (!rb_read_remote_meta_page(cpu_buffer))
+		return NULL;
+
+	/* More to read on the reader page */
+	if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page)) {
+		if (!cpu_buffer->reader_page->read)
+			cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
+		return cpu_buffer->reader_page;
+	}
+
+	prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];
+
+	WARN_ON_ONCE(cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu,
+							  cpu_buffer->remote->priv));
+	/* nr_pages doesn't include the reader page */
+	if (WARN_ON_ONCE(cpu_buffer->meta_page->reader.id > cpu_buffer->nr_pages))
+		return NULL;
+
+	new_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];
+
+	WARN_ON_ONCE(prev_reader == new_reader);
+
+	prev_head = new_reader;  /* New reader was also the previous head */
+	new_head = prev_head;
+	rb_inc_page(&new_head);
+	last = prev_head;
+	rb_dec_page(&last);
+
+	/* Clear the old HEAD flag */
+	rb_list_head_clear(cpu_buffer->head_page->list.prev);
+
+	prev_reader->list.next = prev_head->list.next;
+	prev_reader->list.prev = prev_head->list.prev;
+
+	/* Swap prev_reader with new_reader */
+	last->list.next = &prev_reader->list;
+	new_head->list.prev = &prev_reader->list;
+
+	new_reader->list.prev = &new_reader->list;
+	new_reader->list.next = &new_head->list;
+
+	/* Reactivate the HEAD flag */
+	rb_set_list_to_head(&last->list);
+
+	cpu_buffer->head_page = new_head;
+	cpu_buffer->reader_page = new_reader;
+	cpu_buffer->pages = &new_head->list;
+	cpu_buffer->read_stamp = new_reader->page->time_stamp;
+	cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events;
+
+	return rb_page_size(cpu_buffer->reader_page) ? cpu_buffer->reader_page : NULL;
+}
+
+static struct buffer_page *
+__rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	struct buffer_page *reader = NULL;
 	unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);
@@ -5598,6 +5778,13 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
 	return reader;
 }
 
+static struct buffer_page *
+rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
+{
+	return cpu_buffer->remote ? __rb_get_reader_page_from_remote(cpu_buffer) :
+				    __rb_get_reader_page(cpu_buffer);
+}
+
 static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	struct ring_buffer_event *event;
@@ -6154,6 +6341,8 @@ static void rb_update_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
 	meta->entries = local_read(&cpu_buffer->entries);
 	meta->overrun = local_read(&cpu_buffer->overrun);
 	meta->read = cpu_buffer->read;
+	meta->pages_lost = local_read(&cpu_buffer->pages_lost);
+	meta->pages_touched = local_read(&cpu_buffer->pages_touched);
 
 	/* Some archs do not have data cache coherency between kernel and user-space */
 	flush_kernel_vmap_range(cpu_buffer->meta_page, PAGE_SIZE);
@@ -6164,6 +6353,23 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	struct buffer_page *page;
 
+	if (cpu_buffer->remote) {
+		if (!cpu_buffer->remote->reset)
+			return;
+
+		cpu_buffer->remote->reset(cpu_buffer->cpu, cpu_buffer->remote->priv);
+		rb_read_remote_meta_page(cpu_buffer);
+
+		/* Read related values, not covered by the meta-page */
+		local_set(&cpu_buffer->pages_read, 0);
+		cpu_buffer->read = 0;
+		cpu_buffer->read_bytes = 0;
+		cpu_buffer->last_overrun = 0;
+		cpu_buffer->reader_page->read = 0;
+
+		return;
+	}
+
 	rb_head_page_deactivate(cpu_buffer);
 
 	cpu_buffer->head_page
@@ -6394,6 +6600,46 @@ bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)
 }
 EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
 
+int ring_buffer_poll_remote(struct trace_buffer *buffer, int cpu)
+{
+	struct ring_buffer_per_cpu *cpu_buffer;
+
+	if (cpu != RING_BUFFER_ALL_CPUS) {
+		if (!cpumask_test_cpu(cpu, buffer->cpumask))
+			return -EINVAL;
+
+		cpu_buffer = buffer->buffers[cpu];
+
+		guard(raw_spinlock)(&cpu_buffer->reader_lock);
+		if (rb_read_remote_meta_page(cpu_buffer))
+			rb_wakeups(buffer, cpu_buffer);
+
+		return 0;
+	}
+
+	guard(cpus_read_lock)();
+
+	/*
+	 * Make sure all the ring buffers are up to date before we start reading
+	 * them.
+	 */
+	for_each_buffer_cpu(buffer, cpu) {
+		cpu_buffer = buffer->buffers[cpu];
+
+		guard(raw_spinlock)(&cpu_buffer->reader_lock);
+		rb_read_remote_meta_page(cpu_buffer);
+	}
+
+	for_each_buffer_cpu(buffer, cpu) {
+		cpu_buffer = buffer->buffers[cpu];
+
+		if (rb_num_of_entries(cpu_buffer))
+			rb_wakeups(buffer, cpu_buffer);
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
 /**
  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
@@ -6632,6 +6878,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer,
 	unsigned int commit;
 	unsigned int read;
 	u64 save_timestamp;
+	bool force_memcpy;
 
 	if (!cpumask_test_cpu(cpu, buffer->cpumask))
 		return -1;
@@ -6669,6 +6916,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer,
 	/* Check if any events were dropped */
 	missed_events = cpu_buffer->lost_events;
 
+	force_memcpy = cpu_buffer->mapped || cpu_buffer->remote;
+
 	/*
 	 * If this page has been partially read or
 	 * if len is not big enough to read the rest of the page or
@@ -6678,7 +6927,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer,
 	 */
 	if (read || (len < (commit - read)) ||
 	    cpu_buffer->reader_page == cpu_buffer->commit_page ||
-	    cpu_buffer->mapped) {
+	    force_memcpy) {
 		struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
 		unsigned int rpos = read;
 		unsigned int pos = 0;
@@ -7034,7 +7283,7 @@ static void rb_free_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
 }
 
 static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
-				   unsigned long *subbuf_ids)
+				   struct buffer_page **subbuf_ids)
 {
 	struct trace_buffer_meta *meta = cpu_buffer->meta_page;
 	unsigned int nr_subbufs = cpu_buffer->nr_pages + 1;
@@ -7043,7 +7292,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
 	int id = 0;
 
 	id = rb_page_id(cpu_buffer, cpu_buffer->reader_page, id);
-	subbuf_ids[id++] = (unsigned long)cpu_buffer->reader_page->page;
+	subbuf_ids[id++] = cpu_buffer->reader_page;
 	cnt++;
 
 	first_subbuf = subbuf = rb_set_head_page(cpu_buffer);
@@ -7053,7 +7302,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
 		if (WARN_ON(id >= nr_subbufs))
 			break;
 
-		subbuf_ids[id] = (unsigned long)subbuf->page;
+		subbuf_ids[id] = subbuf;
 
 		rb_inc_page(&subbuf);
 		id++;
@@ -7062,7 +7311,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
 
 	WARN_ON(cnt != nr_subbufs);
 
-	/* install subbuf ID to kern VA translation */
+	/* install subbuf ID to bpage translation */
 	cpu_buffer->subbuf_ids = subbuf_ids;
 
 	meta->meta_struct_len = sizeof(*meta);
@@ -7218,13 +7467,15 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
 	}
 
 	while (p < nr_pages) {
+		struct buffer_page *subbuf;
 		struct page *page;
 		int off = 0;
 
 		if (WARN_ON_ONCE(s >= nr_subbufs))
 			return -EINVAL;
 
-		page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]);
+		subbuf = cpu_buffer->subbuf_ids[s];
+		page = virt_to_page((void *)subbuf->page);
 
 		for (; off < (1 << (subbuf_order)); off++, page++) {
 			if (p >= nr_pages)
@@ -7251,10 +7502,11 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu,
 		    struct vm_area_struct *vma)
 {
 	struct ring_buffer_per_cpu *cpu_buffer;
-	unsigned long flags, *subbuf_ids;
+	struct buffer_page **subbuf_ids;
+	unsigned long flags;
 	int err;
 
-	if (!cpumask_test_cpu(cpu, buffer->cpumask))
+	if (!cpumask_test_cpu(cpu, buffer->cpumask) || buffer->remote)
 		return -EINVAL;
 
 	cpu_buffer = buffer->buffers[cpu];
@@ -7275,7 +7527,7 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu,
 	if (err)
 		return err;
 
-	/* subbuf_ids include the reader while nr_pages does not */
+	/* subbuf_ids includes the reader while nr_pages does not */
 	subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL);
 	if (!subbuf_ids) {
 		rb_free_meta_page(cpu_buffer);
diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c
new file mode 100644
index 0000000000000..02af2297ae5a9
--- /dev/null
+++ b/kernel/trace/simple_ring_buffer.c
@@ -0,0 +1,517 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 - Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <linux/atomic.h>
+#include <linux/simple_ring_buffer.h>
+
+#include <asm/barrier.h>
+#include <asm/local.h>
+
+enum simple_rb_link_type {
+	SIMPLE_RB_LINK_NORMAL		= 0,
+	SIMPLE_RB_LINK_HEAD		= 1,
+	SIMPLE_RB_LINK_HEAD_MOVING
+};
+
+#define SIMPLE_RB_LINK_MASK ~(SIMPLE_RB_LINK_HEAD | SIMPLE_RB_LINK_HEAD_MOVING)
+
+static void simple_bpage_set_head_link(struct simple_buffer_page *bpage)
+{
+	unsigned long link = (unsigned long)bpage->link.next;
+
+	link &= SIMPLE_RB_LINK_MASK;
+	link |= SIMPLE_RB_LINK_HEAD;
+
+	/*
+	 * Paired with simple_rb_find_head() to order access between the head
+	 * link and overrun. It ensures we always report an up-to-date value
+	 * after swapping the reader page.
+	 */
+	smp_store_release(&bpage->link.next, (struct list_head *)link);
+}
+
+static bool simple_bpage_unset_head_link(struct simple_buffer_page *bpage,
+					 struct simple_buffer_page *dst,
+					 enum simple_rb_link_type new_type)
+{
+	unsigned long *link = (unsigned long *)(&bpage->link.next);
+	unsigned long old = (*link & SIMPLE_RB_LINK_MASK) | SIMPLE_RB_LINK_HEAD;
+	unsigned long new = (unsigned long)(&dst->link) | new_type;
+
+	return try_cmpxchg(link, &old, new);
+}
+
+static void simple_bpage_set_normal_link(struct simple_buffer_page *bpage)
+{
+	unsigned long link = (unsigned long)bpage->link.next;
+
+	WRITE_ONCE(bpage->link.next, (struct list_head *)(link & SIMPLE_RB_LINK_MASK));
+}
+
+static struct simple_buffer_page *simple_bpage_from_link(struct list_head *link)
+{
+	unsigned long ptr = (unsigned long)link & SIMPLE_RB_LINK_MASK;
+
+	return container_of((struct list_head *)ptr, struct simple_buffer_page, link);
+}
+
+static struct simple_buffer_page *simple_bpage_next_page(struct simple_buffer_page *bpage)
+{
+	return simple_bpage_from_link(bpage->link.next);
+}
+
+static void simple_bpage_reset(struct simple_buffer_page *bpage)
+{
+	bpage->write = 0;
+	bpage->entries = 0;
+
+	local_set(&bpage->page->commit, 0);
+}
+
+static void simple_bpage_init(struct simple_buffer_page *bpage, void *page)
+{
+	INIT_LIST_HEAD(&bpage->link);
+	bpage->page = (struct buffer_data_page *)page;
+
+	simple_bpage_reset(bpage);
+}
+
+#define simple_rb_meta_inc(__meta, __inc)		\
+	WRITE_ONCE((__meta), (__meta + __inc))
+
+static bool simple_rb_loaded(struct simple_rb_per_cpu *cpu_buffer)
+{
+	return !!cpu_buffer->bpages;
+}
+
+static int simple_rb_find_head(struct simple_rb_per_cpu *cpu_buffer)
+{
+	int retry = cpu_buffer->nr_pages * 2;
+	struct simple_buffer_page *head;
+
+	head = cpu_buffer->head_page;
+
+	while (retry--) {
+		unsigned long link;
+
+spin:
+		/* See smp_store_release in simple_bpage_set_head_link() */
+		link = (unsigned long)smp_load_acquire(&head->link.prev->next);
+
+		switch (link & ~SIMPLE_RB_LINK_MASK) {
+		/* Found the head */
+		case SIMPLE_RB_LINK_HEAD:
+			cpu_buffer->head_page = head;
+			return 0;
+		/* The writer caught the head, we can spin, that won't be long */
+		case SIMPLE_RB_LINK_HEAD_MOVING:
+			goto spin;
+		}
+
+		head = simple_bpage_next_page(head);
+	}
+
+	return -EBUSY;
+}
+
+/**
+ * simple_ring_buffer_swap_reader_page - Swap ring-buffer head with the reader
+ * @cpu_buffer: A simple_rb_per_cpu
+ *
+ * This function enables consuming reading. It ensures the current head page will not be overwritten
+ * and can be safely read.
+ *
+ * Returns 0 on success, -ENODEV if @cpu_buffer was unloaded or -EBUSY if we failed to catch the
+ * head page.
+ */
+int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer)
+{
+	struct simple_buffer_page *last, *head, *reader;
+	unsigned long overrun;
+	int retry = 8;
+	int ret;
+
+	if (!simple_rb_loaded(cpu_buffer))
+		return -ENODEV;
+
+	reader = cpu_buffer->reader_page;
+
+	do {
+		/* Run after the writer to find the head */
+		ret = simple_rb_find_head(cpu_buffer);
+		if (ret)
+			return ret;
+
+		head = cpu_buffer->head_page;
+
+		/* Connect the reader page around the header page */
+		reader->link.next = head->link.next;
+		reader->link.prev = head->link.prev;
+
+		/* The last page before the head */
+		last = simple_bpage_from_link(head->link.prev);
+
+		/* The reader page points to the new header page */
+		simple_bpage_set_head_link(reader);
+
+		overrun = cpu_buffer->meta->overrun;
+	} while (!simple_bpage_unset_head_link(last, reader, SIMPLE_RB_LINK_NORMAL) && retry--);
+
+	if (!retry)
+		return -EINVAL;
+
+	cpu_buffer->head_page = simple_bpage_from_link(reader->link.next);
+	cpu_buffer->head_page->link.prev = &reader->link;
+	cpu_buffer->reader_page = head;
+	cpu_buffer->meta->reader.lost_events = overrun - cpu_buffer->last_overrun;
+	cpu_buffer->meta->reader.id = cpu_buffer->reader_page->id;
+	cpu_buffer->last_overrun = overrun;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_swap_reader_page);
+
+static struct simple_buffer_page *simple_rb_move_tail(struct simple_rb_per_cpu *cpu_buffer)
+{
+	struct simple_buffer_page *tail, *new_tail;
+
+	tail = cpu_buffer->tail_page;
+	new_tail = simple_bpage_next_page(tail);
+
+	if (simple_bpage_unset_head_link(tail, new_tail, SIMPLE_RB_LINK_HEAD_MOVING)) {
+		/*
+		 * Oh no! we've caught the head. There is none anymore and
+		 * swap_reader will spin until we set the new one. Overrun must
+		 * be written first, to make sure we report the correct number
+		 * of lost events.
+		 */
+		simple_rb_meta_inc(cpu_buffer->meta->overrun, new_tail->entries);
+		simple_rb_meta_inc(cpu_buffer->meta->pages_lost, 1);
+
+		simple_bpage_set_head_link(new_tail);
+		simple_bpage_set_normal_link(tail);
+	}
+
+	simple_bpage_reset(new_tail);
+	cpu_buffer->tail_page = new_tail;
+
+	simple_rb_meta_inc(cpu_buffer->meta->pages_touched, 1);
+
+	return new_tail;
+}
+
+static unsigned long rb_event_size(unsigned long length)
+{
+	struct ring_buffer_event *event;
+
+	return length + RB_EVNT_HDR_SIZE + sizeof(event->array[0]);
+}
+
+static struct ring_buffer_event *
+rb_event_add_ts_extend(struct ring_buffer_event *event, u64 delta)
+{
+	event->type_len = RINGBUF_TYPE_TIME_EXTEND;
+	event->time_delta = delta & TS_MASK;
+	event->array[0] = delta >> TS_SHIFT;
+
+	return (struct ring_buffer_event *)((unsigned long)event + 8);
+}
+
+static struct ring_buffer_event *
+simple_rb_reserve_next(struct simple_rb_per_cpu *cpu_buffer, unsigned long length, u64 timestamp)
+{
+	unsigned long ts_ext_size = 0, event_size = rb_event_size(length);
+	struct simple_buffer_page *tail = cpu_buffer->tail_page;
+	struct ring_buffer_event *event;
+	u32 write, prev_write;
+	u64 time_delta;
+
+	time_delta = timestamp - cpu_buffer->write_stamp;
+
+	if (test_time_stamp(time_delta))
+		ts_ext_size = 8;
+
+	prev_write = tail->write;
+	write = prev_write + event_size + ts_ext_size;
+
+	if (unlikely(write > (PAGE_SIZE - BUF_PAGE_HDR_SIZE)))
+		tail = simple_rb_move_tail(cpu_buffer);
+
+	if (!tail->entries) {
+		tail->page->time_stamp = timestamp;
+		time_delta = 0;
+		ts_ext_size = 0;
+		write = event_size;
+		prev_write = 0;
+	}
+
+	tail->write = write;
+	tail->entries++;
+
+	cpu_buffer->write_stamp = timestamp;
+
+	event = (struct ring_buffer_event *)(tail->page->data + prev_write);
+	if (ts_ext_size) {
+		event = rb_event_add_ts_extend(event, time_delta);
+		time_delta = 0;
+	}
+
+	event->type_len = 0;
+	event->time_delta = time_delta;
+	event->array[0] = event_size - RB_EVNT_HDR_SIZE;
+
+	return event;
+}
+
+/**
+ * simple_ring_buffer_reserve - Reserve an entry in @cpu_buffer
+ * @cpu_buffer:	A simple_rb_per_cpu
+ * @length:	Size of the entry in bytes
+ * @timestamp:	Timestamp of the entry
+ *
+ * Returns the address of the entry where to write data or NULL
+ */
+void *simple_ring_buffer_reserve(struct simple_rb_per_cpu *cpu_buffer, unsigned long length,
+				 u64 timestamp)
+{
+	struct ring_buffer_event *rb_event;
+
+	if (cmpxchg(&cpu_buffer->status, SIMPLE_RB_READY, SIMPLE_RB_WRITING) != SIMPLE_RB_READY)
+		return NULL;
+
+	rb_event = simple_rb_reserve_next(cpu_buffer, length, timestamp);
+
+	return &rb_event->array[1];
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_reserve);
+
+/**
+ * simple_ring_buffer_commit - Commit the entry reserved with simple_ring_buffer_reserve()
+ * @cpu_buffer:	The simple_rb_per_cpu where the entry has been reserved
+ */
+void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer)
+{
+	local_set(&cpu_buffer->tail_page->page->commit,
+		  cpu_buffer->tail_page->write);
+	simple_rb_meta_inc(cpu_buffer->meta->entries, 1);
+
+	/*
+	 * Paired with simple_rb_enable_tracing() to ensure data is
+	 * written to the ring-buffer before teardown.
+	 */
+	smp_store_release(&cpu_buffer->status, SIMPLE_RB_READY);
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_commit);
+
+static u32 simple_rb_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable)
+{
+	u32 prev_status;
+
+	if (enable)
+		return cmpxchg(&cpu_buffer->status, SIMPLE_RB_UNAVAILABLE, SIMPLE_RB_READY);
+
+	/* Wait for the buffer to be released */
+	do {
+		prev_status = cmpxchg_acquire(&cpu_buffer->status,
+					      SIMPLE_RB_READY,
+					      SIMPLE_RB_UNAVAILABLE);
+	} while (prev_status == SIMPLE_RB_WRITING);
+
+	return prev_status;
+}
+
+/**
+ * simple_ring_buffer_reset - Reset @cpu_buffer
+ * @cpu_buffer: A simple_rb_per_cpu
+ *
+ * This will not clear the content of the data, only reset counters and pointers
+ *
+ * Returns 0 on success or -ENODEV if @cpu_buffer was unloaded.
+ */
+int simple_ring_buffer_reset(struct simple_rb_per_cpu *cpu_buffer)
+{
+	struct simple_buffer_page *bpage;
+	u32 prev_status;
+	int ret;
+
+	if (!simple_rb_loaded(cpu_buffer))
+		return -ENODEV;
+
+	prev_status = simple_rb_enable_tracing(cpu_buffer, false);
+
+	ret = simple_rb_find_head(cpu_buffer);
+	if (ret)
+		return ret;
+
+	bpage = cpu_buffer->tail_page = cpu_buffer->head_page;
+	do {
+		simple_bpage_reset(bpage);
+		bpage = simple_bpage_next_page(bpage);
+	} while (bpage != cpu_buffer->head_page);
+
+	simple_bpage_reset(cpu_buffer->reader_page);
+
+	cpu_buffer->last_overrun = 0;
+	cpu_buffer->write_stamp = 0;
+
+	cpu_buffer->meta->reader.read = 0;
+	cpu_buffer->meta->reader.lost_events = 0;
+	cpu_buffer->meta->entries = 0;
+	cpu_buffer->meta->overrun = 0;
+	cpu_buffer->meta->read = 0;
+	cpu_buffer->meta->pages_lost = 0;
+	cpu_buffer->meta->pages_touched = 0;
+
+	if (prev_status == SIMPLE_RB_READY)
+		simple_rb_enable_tracing(cpu_buffer, true);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_reset);
+
+int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer,
+			       struct simple_buffer_page *bpages,
+			       const struct ring_buffer_desc *desc,
+			       void *(*load_page)(unsigned long va),
+			       void (*unload_page)(void *va))
+{
+	struct simple_buffer_page *bpage = bpages;
+	int ret = 0;
+	void *page;
+	int i;
+
+	/* At least 1 reader page and two pages in the ring-buffer */
+	if (desc->nr_page_va < 3)
+		return -EINVAL;
+
+	memset(cpu_buffer, 0, sizeof(*cpu_buffer));
+
+	cpu_buffer->meta = load_page(desc->meta_va);
+	if (!cpu_buffer->meta)
+		return -EINVAL;
+
+	memset(cpu_buffer->meta, 0, sizeof(*cpu_buffer->meta));
+	cpu_buffer->meta->meta_page_size = PAGE_SIZE;
+	cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages;
+
+	/* The reader page is not part of the ring initially */
+	page = load_page(desc->page_va[0]);
+	if (!page) {
+		unload_page(cpu_buffer->meta);
+		return -EINVAL;
+	}
+
+	simple_bpage_init(bpage, page);
+	bpage->id = 0;
+
+	cpu_buffer->nr_pages = 1;
+
+	cpu_buffer->reader_page = bpage;
+	cpu_buffer->tail_page = bpage + 1;
+	cpu_buffer->head_page = bpage + 1;
+
+	for (i = 1; i < desc->nr_page_va; i++) {
+		page = load_page(desc->page_va[i]);
+		if (!page) {
+			ret = -EINVAL;
+			break;
+		}
+
+		simple_bpage_init(++bpage, page);
+
+		bpage->link.next = &(bpage + 1)->link;
+		bpage->link.prev = &(bpage - 1)->link;
+		bpage->id = i;
+
+		cpu_buffer->nr_pages = i + 1;
+	}
+
+	if (ret) {
+		for (i--; i >= 0; i--)
+			unload_page((void *)desc->page_va[i]);
+		unload_page(cpu_buffer->meta);
+
+		return ret;
+	}
+
+	/* Close the ring */
+	bpage->link.next = &cpu_buffer->tail_page->link;
+	cpu_buffer->tail_page->link.prev = &bpage->link;
+
+	/* The last init'ed page points to the head page */
+	simple_bpage_set_head_link(bpage);
+
+	cpu_buffer->bpages = bpages;
+
+	return 0;
+}
+
+static void *__load_page(unsigned long page)
+{
+	return (void *)page;
+}
+
+static void __unload_page(void *page) { }
+
+/**
+ * simple_ring_buffer_init - Init @cpu_buffer based on @desc
+ * @cpu_buffer:	A simple_rb_per_cpu buffer to init, allocated by the caller.
+ * @bpages:	Array of simple_buffer_pages, with as many elements as @desc->nr_page_va
+ * @desc:	A ring_buffer_desc
+ *
+ * Returns 0 on success or -EINVAL if the content of @desc is invalid
+ */
+int simple_ring_buffer_init(struct simple_rb_per_cpu *cpu_buffer, struct simple_buffer_page *bpages,
+			    const struct ring_buffer_desc *desc)
+{
+	return simple_ring_buffer_init_mm(cpu_buffer, bpages, desc, __load_page, __unload_page);
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_init);
+
+void simple_ring_buffer_unload_mm(struct simple_rb_per_cpu *cpu_buffer,
+				  void (*unload_page)(void *))
+{
+	int p;
+
+	if (!simple_rb_loaded(cpu_buffer))
+		return;
+
+	simple_rb_enable_tracing(cpu_buffer, false);
+
+	unload_page(cpu_buffer->meta);
+	for (p = 0; p < cpu_buffer->nr_pages; p++)
+		unload_page(cpu_buffer->bpages[p].page);
+
+	cpu_buffer->bpages = NULL;
+}
+
+/**
+ * simple_ring_buffer_unload - Prepare @cpu_buffer for deletion
+ * @cpu_buffer:	A simple_rb_per_cpu that will be deleted.
+ */
+void simple_ring_buffer_unload(struct simple_rb_per_cpu *cpu_buffer)
+{
+	return simple_ring_buffer_unload_mm(cpu_buffer, __unload_page);
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_unload);
+
+/**
+ * simple_ring_buffer_enable_tracing - Enable or disable writing to @cpu_buffer
+ * @cpu_buffer: A simple_rb_per_cpu
+ * @enable:	True to enable tracing, False to disable it
+ *
+ * Returns 0 on success or -ENODEV if @cpu_buffer was unloaded
+ */
+int simple_ring_buffer_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable)
+{
+	if (!simple_rb_loaded(cpu_buffer))
+		return -ENODEV;
+
+	simple_rb_enable_tracing(cpu_buffer, enable);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(simple_ring_buffer_enable_tracing);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index ebd996f8710e6..5e1129b011cb6 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -3856,7 +3856,7 @@ static int s_show(struct seq_file *m, void *v)
  * Should be used after trace_array_get(), trace_types_lock
  * ensures that i_cdev was already initialized.
  */
-static inline int tracing_get_cpu(struct inode *inode)
+int tracing_get_cpu(struct inode *inode)
 {
 	if (inode->i_cdev) /* See trace_create_cpu_file() */
 		return (long)inode->i_cdev - 1;
@@ -8589,7 +8589,7 @@ static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
 	return tr->percpu_dir;
 }
 
-static struct dentry *
+struct dentry *
 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
 		      void *data, long cpu, const struct file_operations *fops)
 {
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index b8f3804586a06..fbbd0b2ee76f8 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -689,6 +689,13 @@ struct dentry *trace_create_file(const char *name,
 				 struct dentry *parent,
 				 void *data,
 				 const struct file_operations *fops);
+struct dentry *trace_create_cpu_file(const char *name,
+				     umode_t mode,
+				     struct dentry *parent,
+				     void *data,
+				     long cpu,
+				     const struct file_operations *fops);
+int tracing_get_cpu(struct inode *inode);
 
 
 /**
diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c
index 8bb95b2a6fcfe..39ac4eba07024 100644
--- a/kernel/trace/trace_events_synth.c
+++ b/kernel/trace/trace_events_synth.c
@@ -395,7 +395,7 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter,
 			n_u64++;
 		} else {
 			struct trace_print_flags __flags[] = {
-			    __def_gfpflag_names, {-1, NULL} };
+			    __def_gfpflag_names };
 			char *space = (i == se->n_fields - 1 ? "" : " ");
 
 			print_synth_event_num_val(s, print_fmt,
@@ -408,7 +408,7 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter,
 				trace_seq_puts(s, " (");
 				trace_print_flags_seq(s, "|",
 						      entry->fields[n_u64].as_u64,
-						      __flags);
+						      __flags, ARRAY_SIZE(__flags));
 				trace_seq_putc(s, ')');
 			}
 			n_u64++;
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 1996d7aba0383..96e2d22b43642 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -69,14 +69,15 @@ enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter)
 const char *
 trace_print_flags_seq(struct trace_seq *p, const char *delim,
 		      unsigned long flags,
-		      const struct trace_print_flags *flag_array)
+		      const struct trace_print_flags *flag_array,
+		      size_t flag_array_size)
 {
 	unsigned long mask;
 	const char *str;
 	const char *ret = trace_seq_buffer_ptr(p);
 	int i, first = 1;
 
-	for (i = 0;  flag_array[i].name && flags; i++) {
+	for (i = 0; i < flag_array_size && flags; i++) {
 
 		mask = flag_array[i].mask;
 		if ((flags & mask) != mask)
@@ -106,12 +107,13 @@ EXPORT_SYMBOL(trace_print_flags_seq);
 
 const char *
 trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
-			const struct trace_print_flags *symbol_array)
+			const struct trace_print_flags *symbol_array,
+			size_t symbol_array_size)
 {
 	int i;
 	const char *ret = trace_seq_buffer_ptr(p);
 
-	for (i = 0;  symbol_array[i].name; i++) {
+	for (i = 0; i < symbol_array_size; i++) {
 
 		if (val != symbol_array[i].mask)
 			continue;
@@ -133,14 +135,15 @@ EXPORT_SYMBOL(trace_print_symbols_seq);
 const char *
 trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
 		      unsigned long long flags,
-		      const struct trace_print_flags_u64 *flag_array)
+		      const struct trace_print_flags_u64 *flag_array,
+		      size_t flag_array_size)
 {
 	unsigned long long mask;
 	const char *str;
 	const char *ret = trace_seq_buffer_ptr(p);
 	int i, first = 1;
 
-	for (i = 0;  flag_array[i].name && flags; i++) {
+	for (i = 0; i < flag_array_size && flags; i++) {
 
 		mask = flag_array[i].mask;
 		if ((flags & mask) != mask)
@@ -170,12 +173,13 @@ EXPORT_SYMBOL(trace_print_flags_seq_u64);
 
 const char *
 trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val,
-			 const struct trace_print_flags_u64 *symbol_array)
+			    const struct trace_print_flags_u64 *symbol_array,
+			    size_t symbol_array_size)
 {
 	int i;
 	const char *ret = trace_seq_buffer_ptr(p);
 
-	for (i = 0;  symbol_array[i].name; i++) {
+	for (i = 0; i < symbol_array_size; i++) {
 
 		if (val != symbol_array[i].mask)
 			continue;
diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c
new file mode 100644
index 0000000000000..0d78e5f5fe98d
--- /dev/null
+++ b/kernel/trace/trace_remote.c
@@ -0,0 +1,1368 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 - Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <linux/kstrtox.h>
+#include <linux/lockdep.h>
+#include <linux/mutex.h>
+#include <linux/tracefs.h>
+#include <linux/trace_remote.h>
+#include <linux/trace_seq.h>
+#include <linux/types.h>
+
+#include "trace.h"
+
+#define TRACEFS_DIR		"remotes"
+#define TRACEFS_MODE_WRITE	0640
+#define TRACEFS_MODE_READ	0440
+
+enum tri_type {
+	TRI_CONSUMING,
+	TRI_NONCONSUMING,
+};
+
+struct trace_remote_iterator {
+	struct trace_remote		*remote;
+	struct trace_seq		seq;
+	struct delayed_work		poll_work;
+	unsigned long			lost_events;
+	u64				ts;
+	struct ring_buffer_iter		*rb_iter;
+	struct ring_buffer_iter		**rb_iters;
+	struct remote_event_hdr		*evt;
+	int				cpu;
+	int				evt_cpu;
+	loff_t				pos;
+	enum tri_type			type;
+};
+
+struct trace_remote {
+	struct trace_remote_callbacks	*cbs;
+	void				*priv;
+	struct trace_buffer		*trace_buffer;
+	struct trace_buffer_desc	*trace_buffer_desc;
+	struct dentry			*dentry;
+	struct eventfs_inode		*eventfs;
+	struct remote_event		*events;
+	unsigned long			nr_events;
+	unsigned long			trace_buffer_size;
+	struct ring_buffer_remote	rb_remote;
+	struct mutex			lock;
+	struct rw_semaphore		reader_lock;
+	struct rw_semaphore		*pcpu_reader_locks;
+	unsigned int			nr_readers;
+	unsigned int			poll_ms;
+	bool				tracing_on;
+};
+
+static bool trace_remote_loaded(struct trace_remote *remote)
+{
+	return !!remote->trace_buffer;
+}
+
+static int trace_remote_load(struct trace_remote *remote)
+{
+	struct ring_buffer_remote *rb_remote = &remote->rb_remote;
+	struct trace_buffer_desc *desc;
+
+	lockdep_assert_held(&remote->lock);
+
+	if (trace_remote_loaded(remote))
+		return 0;
+
+	desc = remote->cbs->load_trace_buffer(remote->trace_buffer_size, remote->priv);
+	if (IS_ERR(desc))
+		return PTR_ERR(desc);
+
+	rb_remote->desc = desc;
+	rb_remote->swap_reader_page = remote->cbs->swap_reader_page;
+	rb_remote->priv = remote->priv;
+	rb_remote->reset = remote->cbs->reset;
+	remote->trace_buffer = ring_buffer_alloc_remote(rb_remote);
+	if (!remote->trace_buffer) {
+		remote->cbs->unload_trace_buffer(desc, remote->priv);
+		return -ENOMEM;
+	}
+
+	remote->trace_buffer_desc = desc;
+
+	return 0;
+}
+
+static void trace_remote_try_unload(struct trace_remote *remote)
+{
+	lockdep_assert_held(&remote->lock);
+
+	if (!trace_remote_loaded(remote))
+		return;
+
+	/* The buffer is being read or writable */
+	if (remote->nr_readers || remote->tracing_on)
+		return;
+
+	/* The buffer has readable data */
+	if (!ring_buffer_empty(remote->trace_buffer))
+		return;
+
+	ring_buffer_free(remote->trace_buffer);
+	remote->trace_buffer = NULL;
+	remote->cbs->unload_trace_buffer(remote->trace_buffer_desc, remote->priv);
+}
+
+static int trace_remote_enable_tracing(struct trace_remote *remote)
+{
+	int ret;
+
+	lockdep_assert_held(&remote->lock);
+
+	if (remote->tracing_on)
+		return 0;
+
+	ret = trace_remote_load(remote);
+	if (ret)
+		return ret;
+
+	ret = remote->cbs->enable_tracing(true, remote->priv);
+	if (ret) {
+		trace_remote_try_unload(remote);
+		return ret;
+	}
+
+	remote->tracing_on = true;
+
+	return 0;
+}
+
+static int trace_remote_disable_tracing(struct trace_remote *remote)
+{
+	int ret;
+
+	lockdep_assert_held(&remote->lock);
+
+	if (!remote->tracing_on)
+		return 0;
+
+	ret = remote->cbs->enable_tracing(false, remote->priv);
+	if (ret)
+		return ret;
+
+	ring_buffer_poll_remote(remote->trace_buffer, RING_BUFFER_ALL_CPUS);
+	remote->tracing_on = false;
+	trace_remote_try_unload(remote);
+
+	return 0;
+}
+
+static void trace_remote_reset(struct trace_remote *remote, int cpu)
+{
+	lockdep_assert_held(&remote->lock);
+
+	if (!trace_remote_loaded(remote))
+		return;
+
+	if (cpu == RING_BUFFER_ALL_CPUS)
+		ring_buffer_reset(remote->trace_buffer);
+	else
+		ring_buffer_reset_cpu(remote->trace_buffer, cpu);
+
+	trace_remote_try_unload(remote);
+}
+
+static ssize_t
+tracing_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+	struct seq_file *seq = filp->private_data;
+	struct trace_remote *remote = seq->private;
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&remote->lock);
+
+	ret = val ? trace_remote_enable_tracing(remote) : trace_remote_disable_tracing(remote);
+	if (ret)
+		return ret;
+
+	return cnt;
+}
+static int tracing_on_show(struct seq_file *s, void *unused)
+{
+	struct trace_remote *remote = s->private;
+
+	seq_printf(s, "%d\n", remote->tracing_on);
+
+	return 0;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(tracing_on);
+
+static ssize_t buffer_size_kb_write(struct file *filp, const char __user *ubuf, size_t cnt,
+				    loff_t *ppos)
+{
+	struct seq_file *seq = filp->private_data;
+	struct trace_remote *remote = seq->private;
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+	if (ret)
+		return ret;
+
+	/* KiB to Bytes */
+	if (!val || check_shl_overflow(val, 10, &val))
+		return -EINVAL;
+
+	guard(mutex)(&remote->lock);
+
+	if (trace_remote_loaded(remote))
+		return -EBUSY;
+
+	remote->trace_buffer_size = val;
+
+	return cnt;
+}
+
+static int buffer_size_kb_show(struct seq_file *s, void *unused)
+{
+	struct trace_remote *remote = s->private;
+
+	seq_printf(s, "%lu (%s)\n", remote->trace_buffer_size >> 10,
+		   trace_remote_loaded(remote) ? "loaded" : "unloaded");
+
+	return 0;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(buffer_size_kb);
+
+static int trace_remote_get(struct trace_remote *remote, int cpu)
+{
+	int ret;
+
+	if (remote->nr_readers == UINT_MAX)
+		return -EBUSY;
+
+	ret = trace_remote_load(remote);
+	if (ret)
+		return ret;
+
+	if (cpu != RING_BUFFER_ALL_CPUS && !remote->pcpu_reader_locks) {
+		int lock_cpu;
+
+		remote->pcpu_reader_locks = kcalloc(nr_cpu_ids, sizeof(*remote->pcpu_reader_locks),
+						    GFP_KERNEL);
+		if (!remote->pcpu_reader_locks) {
+			trace_remote_try_unload(remote);
+			return -ENOMEM;
+		}
+
+		for_each_possible_cpu(lock_cpu)
+			init_rwsem(&remote->pcpu_reader_locks[lock_cpu]);
+	}
+
+	remote->nr_readers++;
+
+	return 0;
+}
+
+static void trace_remote_put(struct trace_remote *remote)
+{
+	if (WARN_ON(!remote->nr_readers))
+		return;
+
+	remote->nr_readers--;
+	if (remote->nr_readers)
+		return;
+
+	kfree(remote->pcpu_reader_locks);
+	remote->pcpu_reader_locks = NULL;
+
+	trace_remote_try_unload(remote);
+}
+
+static void __poll_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct trace_remote_iterator *iter;
+
+	iter = container_of(dwork, struct trace_remote_iterator, poll_work);
+	ring_buffer_poll_remote(iter->remote->trace_buffer, iter->cpu);
+	schedule_delayed_work((struct delayed_work *)work,
+			      msecs_to_jiffies(iter->remote->poll_ms));
+}
+
+static void __free_ring_buffer_iter(struct trace_remote_iterator *iter, int cpu)
+{
+	if (cpu != RING_BUFFER_ALL_CPUS) {
+		ring_buffer_read_finish(iter->rb_iter);
+		return;
+	}
+
+	for_each_possible_cpu(cpu) {
+		if (iter->rb_iters[cpu])
+			ring_buffer_read_finish(iter->rb_iters[cpu]);
+	}
+
+	kfree(iter->rb_iters);
+}
+
+static int __alloc_ring_buffer_iter(struct trace_remote_iterator *iter, int cpu)
+{
+	if (cpu != RING_BUFFER_ALL_CPUS) {
+		iter->rb_iter = ring_buffer_read_start(iter->remote->trace_buffer, cpu, GFP_KERNEL);
+
+		return iter->rb_iter ? 0 : -ENOMEM;
+	}
+
+	iter->rb_iters = kcalloc(nr_cpu_ids, sizeof(*iter->rb_iters), GFP_KERNEL);
+	if (!iter->rb_iters)
+		return -ENOMEM;
+
+	for_each_possible_cpu(cpu) {
+		iter->rb_iters[cpu] = ring_buffer_read_start(iter->remote->trace_buffer, cpu,
+							     GFP_KERNEL);
+		if (!iter->rb_iters[cpu]) {
+			__free_ring_buffer_iter(iter, RING_BUFFER_ALL_CPUS);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+static struct trace_remote_iterator
+*trace_remote_iter(struct trace_remote *remote, int cpu, enum tri_type type)
+{
+	struct trace_remote_iterator *iter = NULL;
+	int ret;
+
+	lockdep_assert_held(&remote->lock);
+
+	if (type == TRI_NONCONSUMING && !trace_remote_loaded(remote))
+		return NULL;
+
+	ret = trace_remote_get(remote, cpu);
+	if (ret)
+		return ERR_PTR(ret);
+
+	/* Test the CPU */
+	ret = ring_buffer_poll_remote(remote->trace_buffer, cpu);
+	if (ret)
+		goto err;
+
+	iter = kzalloc_obj(*iter);
+	if (iter) {
+		iter->remote = remote;
+		iter->cpu = cpu;
+		iter->type = type;
+		trace_seq_init(&iter->seq);
+
+		switch (type) {
+		case TRI_CONSUMING:
+			INIT_DELAYED_WORK(&iter->poll_work, __poll_remote);
+			schedule_delayed_work(&iter->poll_work, msecs_to_jiffies(remote->poll_ms));
+			break;
+		case TRI_NONCONSUMING:
+			ret = __alloc_ring_buffer_iter(iter, cpu);
+			break;
+		}
+
+		if (ret)
+			goto err;
+
+		return iter;
+	}
+	ret = -ENOMEM;
+
+err:
+	kfree(iter);
+	trace_remote_put(remote);
+
+	return ERR_PTR(ret);
+}
+
+static void trace_remote_iter_free(struct trace_remote_iterator *iter)
+{
+	struct trace_remote *remote;
+
+	if (!iter)
+		return;
+
+	remote = iter->remote;
+
+	lockdep_assert_held(&remote->lock);
+
+	switch (iter->type) {
+	case TRI_CONSUMING:
+		cancel_delayed_work_sync(&iter->poll_work);
+		break;
+	case TRI_NONCONSUMING:
+		__free_ring_buffer_iter(iter, iter->cpu);
+		break;
+	}
+
+	kfree(iter);
+	trace_remote_put(remote);
+}
+
+static void trace_remote_iter_read_start(struct trace_remote_iterator *iter)
+{
+	struct trace_remote *remote = iter->remote;
+	int cpu = iter->cpu;
+
+	/* Acquire global reader lock */
+	if (cpu == RING_BUFFER_ALL_CPUS && iter->type == TRI_CONSUMING)
+		down_write(&remote->reader_lock);
+	else
+		down_read(&remote->reader_lock);
+
+	if (cpu == RING_BUFFER_ALL_CPUS)
+		return;
+
+	/*
+	 * No need for the remote lock here, iter holds a reference on
+	 * remote->nr_readers
+	 */
+
+	/* Get the per-CPU one */
+	if (WARN_ON_ONCE(!remote->pcpu_reader_locks))
+		return;
+
+	if (iter->type == TRI_CONSUMING)
+		down_write(&remote->pcpu_reader_locks[cpu]);
+	else
+		down_read(&remote->pcpu_reader_locks[cpu]);
+}
+
+static void trace_remote_iter_read_finished(struct trace_remote_iterator *iter)
+{
+	struct trace_remote *remote = iter->remote;
+	int cpu = iter->cpu;
+
+	/* Release per-CPU reader lock */
+	if (cpu != RING_BUFFER_ALL_CPUS) {
+		/*
+		 * No need for the remote lock here, iter holds a reference on
+		 * remote->nr_readers
+		 */
+		if (iter->type == TRI_CONSUMING)
+			up_write(&remote->pcpu_reader_locks[cpu]);
+		else
+			up_read(&remote->pcpu_reader_locks[cpu]);
+	}
+
+	/* Release global reader lock */
+	if (cpu == RING_BUFFER_ALL_CPUS && iter->type == TRI_CONSUMING)
+		up_write(&remote->reader_lock);
+	else
+		up_read(&remote->reader_lock);
+}
+
+static struct ring_buffer_iter *__get_rb_iter(struct trace_remote_iterator *iter, int cpu)
+{
+	return iter->cpu != RING_BUFFER_ALL_CPUS ? iter->rb_iter : iter->rb_iters[cpu];
+}
+
+static struct ring_buffer_event *
+__peek_event(struct trace_remote_iterator *iter, int cpu, u64 *ts, unsigned long *lost_events)
+{
+	struct ring_buffer_event *rb_evt;
+	struct ring_buffer_iter *rb_iter;
+
+	switch (iter->type) {
+	case TRI_CONSUMING:
+		return ring_buffer_peek(iter->remote->trace_buffer, cpu, ts, lost_events);
+	case TRI_NONCONSUMING:
+		rb_iter = __get_rb_iter(iter, cpu);
+		rb_evt = ring_buffer_iter_peek(rb_iter, ts);
+		if (!rb_evt)
+			return NULL;
+
+		*lost_events = ring_buffer_iter_dropped(rb_iter);
+
+		return rb_evt;
+	}
+
+	return NULL;
+}
+
+static bool trace_remote_iter_read_event(struct trace_remote_iterator *iter)
+{
+	struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
+	struct ring_buffer_event *rb_evt;
+	int cpu = iter->cpu;
+
+	if (cpu != RING_BUFFER_ALL_CPUS) {
+		if (ring_buffer_empty_cpu(trace_buffer, cpu))
+			return false;
+
+		rb_evt = __peek_event(iter, cpu, &iter->ts, &iter->lost_events);
+		if (!rb_evt)
+			return false;
+
+		iter->evt_cpu = cpu;
+		iter->evt = ring_buffer_event_data(rb_evt);
+		return true;
+	}
+
+	iter->ts = U64_MAX;
+	for_each_possible_cpu(cpu) {
+		unsigned long lost_events;
+		u64 ts;
+
+		if (ring_buffer_empty_cpu(trace_buffer, cpu))
+			continue;
+
+		rb_evt = __peek_event(iter, cpu, &ts, &lost_events);
+		if (!rb_evt)
+			continue;
+
+		if (ts >= iter->ts)
+			continue;
+
+		iter->ts = ts;
+		iter->evt_cpu = cpu;
+		iter->evt = ring_buffer_event_data(rb_evt);
+		iter->lost_events = lost_events;
+	}
+
+	return iter->ts != U64_MAX;
+}
+
+static void trace_remote_iter_move(struct trace_remote_iterator *iter)
+{
+	struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
+
+	switch (iter->type) {
+	case TRI_CONSUMING:
+		ring_buffer_consume(trace_buffer, iter->evt_cpu, NULL, NULL);
+		break;
+	case TRI_NONCONSUMING:
+		ring_buffer_iter_advance(__get_rb_iter(iter, iter->evt_cpu));
+		break;
+	}
+}
+
+static struct remote_event *trace_remote_find_event(struct trace_remote *remote, unsigned short id);
+
+static int trace_remote_iter_print_event(struct trace_remote_iterator *iter)
+{
+	struct remote_event *evt;
+	unsigned long usecs_rem;
+	u64 ts = iter->ts;
+
+	if (iter->lost_events)
+		trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
+				 iter->evt_cpu, iter->lost_events);
+
+	do_div(ts, 1000);
+	usecs_rem = do_div(ts, USEC_PER_SEC);
+
+	trace_seq_printf(&iter->seq, "[%03d]\t%5llu.%06lu: ", iter->evt_cpu,
+			 ts, usecs_rem);
+
+	evt = trace_remote_find_event(iter->remote, iter->evt->id);
+	if (!evt)
+		trace_seq_printf(&iter->seq, "UNKNOWN id=%d\n", iter->evt->id);
+	else
+		evt->print(iter->evt, &iter->seq);
+
+	return trace_seq_has_overflowed(&iter->seq) ? -EOVERFLOW : 0;
+}
+
+static int trace_pipe_open(struct inode *inode, struct file *filp)
+{
+	struct trace_remote *remote = inode->i_private;
+	struct trace_remote_iterator *iter;
+	int cpu = tracing_get_cpu(inode);
+
+	guard(mutex)(&remote->lock);
+
+	iter = trace_remote_iter(remote, cpu, TRI_CONSUMING);
+	if (IS_ERR(iter))
+		return PTR_ERR(iter);
+
+	filp->private_data = iter;
+
+	return IS_ERR(iter) ? PTR_ERR(iter) : 0;
+}
+
+static int trace_pipe_release(struct inode *inode, struct file *filp)
+{
+	struct trace_remote_iterator *iter = filp->private_data;
+	struct trace_remote *remote = iter->remote;
+
+	guard(mutex)(&remote->lock);
+
+	trace_remote_iter_free(iter);
+
+	return 0;
+}
+
+static ssize_t trace_pipe_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+	struct trace_remote_iterator *iter = filp->private_data;
+	struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
+	int ret;
+
+copy_to_user:
+	ret = trace_seq_to_user(&iter->seq, ubuf, cnt);
+	if (ret != -EBUSY)
+		return ret;
+
+	trace_seq_init(&iter->seq);
+
+	ret = ring_buffer_wait(trace_buffer, iter->cpu, 0, NULL, NULL);
+	if (ret < 0)
+		return ret;
+
+	trace_remote_iter_read_start(iter);
+
+	while (trace_remote_iter_read_event(iter)) {
+		int prev_len = iter->seq.seq.len;
+
+		if (trace_remote_iter_print_event(iter)) {
+			iter->seq.seq.len = prev_len;
+			break;
+		}
+
+		trace_remote_iter_move(iter);
+	}
+
+	trace_remote_iter_read_finished(iter);
+
+	goto copy_to_user;
+}
+
+static const struct file_operations trace_pipe_fops = {
+	.open		= trace_pipe_open,
+	.read		= trace_pipe_read,
+	.release	= trace_pipe_release,
+};
+
+static void *trace_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	struct trace_remote_iterator *iter = m->private;
+
+	++*pos;
+
+	if (!iter || !trace_remote_iter_read_event(iter))
+		return NULL;
+
+	trace_remote_iter_move(iter);
+	iter->pos++;
+
+	return iter;
+}
+
+static void *trace_start(struct seq_file *m, loff_t *pos)
+{
+	struct trace_remote_iterator *iter = m->private;
+	loff_t i;
+
+	if (!iter)
+		return NULL;
+
+	trace_remote_iter_read_start(iter);
+
+	if (!*pos) {
+		iter->pos = -1;
+		return trace_next(m, NULL, &i);
+	}
+
+	i = iter->pos;
+	while (i < *pos) {
+		iter = trace_next(m, NULL, &i);
+		if (!iter)
+			return NULL;
+	}
+
+	return iter;
+}
+
+static int trace_show(struct seq_file *m, void *v)
+{
+	struct trace_remote_iterator *iter = v;
+
+	trace_seq_init(&iter->seq);
+
+	if (trace_remote_iter_print_event(iter)) {
+		seq_printf(m, "[EVENT %d PRINT TOO BIG]\n", iter->evt->id);
+		return 0;
+	}
+
+	return trace_print_seq(m, &iter->seq);
+}
+
+static void trace_stop(struct seq_file *m, void *v)
+{
+	struct trace_remote_iterator *iter = m->private;
+
+	if (iter)
+		trace_remote_iter_read_finished(iter);
+}
+
+static const struct seq_operations trace_sops = {
+	.start		= trace_start,
+	.next		= trace_next,
+	.show		= trace_show,
+	.stop		= trace_stop,
+};
+
+static int trace_open(struct inode *inode, struct file *filp)
+{
+	struct trace_remote *remote = inode->i_private;
+	struct trace_remote_iterator *iter = NULL;
+	int cpu = tracing_get_cpu(inode);
+	int ret;
+
+	if (!(filp->f_mode & FMODE_READ))
+		return 0;
+
+	guard(mutex)(&remote->lock);
+
+	iter = trace_remote_iter(remote, cpu, TRI_NONCONSUMING);
+	if (IS_ERR(iter))
+		return PTR_ERR(iter);
+
+	ret = seq_open(filp, &trace_sops);
+	if (ret) {
+		trace_remote_iter_free(iter);
+		return ret;
+	}
+
+	((struct seq_file *)filp->private_data)->private = (void *)iter;
+
+	return 0;
+}
+
+static int trace_release(struct inode *inode, struct file *filp)
+{
+	struct trace_remote_iterator *iter;
+
+	if (!(filp->f_mode & FMODE_READ))
+		return 0;
+
+	iter = ((struct seq_file *)filp->private_data)->private;
+	seq_release(inode, filp);
+
+	if (!iter)
+		return 0;
+
+	guard(mutex)(&iter->remote->lock);
+
+	trace_remote_iter_free(iter);
+
+	return 0;
+}
+
+static ssize_t trace_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+	struct inode *inode = file_inode(filp);
+	struct trace_remote *remote = inode->i_private;
+	int cpu = tracing_get_cpu(inode);
+
+	guard(mutex)(&remote->lock);
+
+	trace_remote_reset(remote, cpu);
+
+	return cnt;
+}
+
+static const struct file_operations trace_fops = {
+	.open		= trace_open,
+	.write		= trace_write,
+	.read		= seq_read,
+	.read_iter	= seq_read_iter,
+	.release	= trace_release,
+};
+
+static int trace_remote_init_tracefs(const char *name, struct trace_remote *remote)
+{
+	struct dentry *remote_d, *percpu_d, *d;
+	static struct dentry *root;
+	static DEFINE_MUTEX(lock);
+	bool root_inited = false;
+	int cpu;
+
+	guard(mutex)(&lock);
+
+	if (!root) {
+		root = tracefs_create_dir(TRACEFS_DIR, NULL);
+		if (!root) {
+			pr_err("Failed to create tracefs dir "TRACEFS_DIR"\n");
+			return -ENOMEM;
+		}
+		root_inited = true;
+	}
+
+	remote_d = tracefs_create_dir(name, root);
+	if (!remote_d) {
+		pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/\n", name);
+		goto err;
+	}
+
+	d = trace_create_file("tracing_on", TRACEFS_MODE_WRITE, remote_d, remote, &tracing_on_fops);
+	if (!d)
+		goto err;
+
+	d = trace_create_file("buffer_size_kb", TRACEFS_MODE_WRITE, remote_d, remote,
+			      &buffer_size_kb_fops);
+	if (!d)
+		goto err;
+
+	d = trace_create_file("trace_pipe", TRACEFS_MODE_READ, remote_d, remote, &trace_pipe_fops);
+	if (!d)
+		goto err;
+
+	d = trace_create_file("trace", TRACEFS_MODE_WRITE, remote_d, remote, &trace_fops);
+	if (!d)
+		goto err;
+
+	percpu_d = tracefs_create_dir("per_cpu", remote_d);
+	if (!percpu_d) {
+		pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/per_cpu/\n", name);
+		goto err;
+	}
+
+	for_each_possible_cpu(cpu) {
+		struct dentry *cpu_d;
+		char cpu_name[16];
+
+		snprintf(cpu_name, sizeof(cpu_name), "cpu%d", cpu);
+		cpu_d = tracefs_create_dir(cpu_name, percpu_d);
+		if (!cpu_d) {
+			pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/percpu/cpu%d\n",
+			       name, cpu);
+			goto err;
+		}
+
+		d = trace_create_cpu_file("trace_pipe", TRACEFS_MODE_READ, cpu_d, remote, cpu,
+					  &trace_pipe_fops);
+		if (!d)
+			goto err;
+
+		d = trace_create_cpu_file("trace", TRACEFS_MODE_WRITE, cpu_d, remote, cpu,
+					  &trace_fops);
+		if (!d)
+			goto err;
+	}
+
+	remote->dentry = remote_d;
+
+	return 0;
+
+err:
+	if (root_inited) {
+		tracefs_remove(root);
+		root = NULL;
+	} else {
+		tracefs_remove(remote_d);
+	}
+
+	return -ENOMEM;
+}
+
+static int trace_remote_register_events(const char *remote_name, struct trace_remote *remote,
+					struct remote_event *events, size_t nr_events);
+
+/**
+ * trace_remote_register() - Register a Tracefs remote
+ * @name:	Name of the remote, used for the Tracefs remotes/ directory.
+ * @cbs:	Set of callbacks used to control the remote.
+ * @priv:	Private data, passed to each callback from @cbs.
+ * @events:	Array of events. &remote_event.name and &remote_event.id must be
+ *		filled by the caller.
+ * @nr_events:	Number of events in the @events array.
+ *
+ * A trace remote is an entity, outside of the kernel (most likely firmware or
+ * hypervisor) capable of writing events into a Tracefs compatible ring-buffer.
+ * The kernel would then act as a reader.
+ *
+ * The registered remote will be found under the Tracefs directory
+ * remotes/<name>.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int trace_remote_register(const char *name, struct trace_remote_callbacks *cbs, void *priv,
+			  struct remote_event *events, size_t nr_events)
+{
+	struct trace_remote *remote;
+	int ret;
+
+	remote = kzalloc_obj(*remote);
+	if (!remote)
+		return -ENOMEM;
+
+	remote->cbs = cbs;
+	remote->priv = priv;
+	remote->trace_buffer_size = 7 << 10;
+	remote->poll_ms = 100;
+	mutex_init(&remote->lock);
+	init_rwsem(&remote->reader_lock);
+
+	if (trace_remote_init_tracefs(name, remote)) {
+		kfree(remote);
+		return -ENOMEM;
+	}
+
+	ret = trace_remote_register_events(name, remote, events, nr_events);
+	if (ret) {
+		pr_err("Failed to register events for trace remote '%s' (%d)\n",
+		       name, ret);
+		return ret;
+	}
+
+	ret = cbs->init ? cbs->init(remote->dentry, priv) : 0;
+	if (ret)
+		pr_err("Init failed for trace remote '%s' (%d)\n", name, ret);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(trace_remote_register);
+
+/**
+ * trace_remote_free_buffer() - Free trace buffer allocated with trace_remote_alloc_buffer()
+ * @desc:	Descriptor of the per-CPU ring-buffers, originally filled by
+ *		trace_remote_alloc_buffer()
+ *
+ * Most likely called from &trace_remote_callbacks.unload_trace_buffer.
+ */
+void trace_remote_free_buffer(struct trace_buffer_desc *desc)
+{
+	struct ring_buffer_desc *rb_desc;
+	int cpu;
+
+	for_each_ring_buffer_desc(rb_desc, cpu, desc) {
+		unsigned int id;
+
+		free_page(rb_desc->meta_va);
+
+		for (id = 0; id < rb_desc->nr_page_va; id++)
+			free_page(rb_desc->page_va[id]);
+	}
+}
+EXPORT_SYMBOL_GPL(trace_remote_free_buffer);
+
+/**
+ * trace_remote_alloc_buffer() - Dynamically allocate a trace buffer
+ * @desc:		Uninitialized trace_buffer_desc
+ * @desc_size:		Size of the trace_buffer_desc. Must be at least equal to
+ *			trace_buffer_desc_size()
+ * @buffer_size:	Size in bytes of each per-CPU ring-buffer
+ * @cpumask:		CPUs to allocate a ring-buffer for
+ *
+ * Helper to dynamically allocate a set of pages (enough to cover @buffer_size)
+ * for each CPU from @cpumask and fill @desc. Most likely called from
+ * &trace_remote_callbacks.load_trace_buffer.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size,
+			      const struct cpumask *cpumask)
+{
+	unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1;
+	void *desc_end = desc + desc_size;
+	struct ring_buffer_desc *rb_desc;
+	int cpu, ret = -ENOMEM;
+
+	if (desc_size < struct_size(desc, __data, 0))
+		return -EINVAL;
+
+	desc->nr_cpus = 0;
+	desc->struct_len = struct_size(desc, __data, 0);
+
+	rb_desc = (struct ring_buffer_desc *)&desc->__data[0];
+
+	for_each_cpu(cpu, cpumask) {
+		unsigned int id;
+
+		if ((void *)rb_desc + struct_size(rb_desc, page_va, nr_pages) > desc_end) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		rb_desc->cpu = cpu;
+		rb_desc->nr_page_va = 0;
+		rb_desc->meta_va = (unsigned long)__get_free_page(GFP_KERNEL);
+		if (!rb_desc->meta_va)
+			goto err;
+
+		for (id = 0; id < nr_pages; id++) {
+			rb_desc->page_va[id] = (unsigned long)__get_free_page(GFP_KERNEL);
+			if (!rb_desc->page_va[id])
+				goto err;
+
+			rb_desc->nr_page_va++;
+		}
+		desc->nr_cpus++;
+		desc->struct_len += offsetof(struct ring_buffer_desc, page_va);
+		desc->struct_len += struct_size(rb_desc, page_va, rb_desc->nr_page_va);
+		rb_desc = __next_ring_buffer_desc(rb_desc);
+	}
+
+	return 0;
+
+err:
+	trace_remote_free_buffer(desc);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(trace_remote_alloc_buffer);
+
+static int
+trace_remote_enable_event(struct trace_remote *remote, struct remote_event *evt, bool enable)
+{
+	int ret;
+
+	lockdep_assert_held(&remote->lock);
+
+	if (evt->enabled == enable)
+		return 0;
+
+	ret = remote->cbs->enable_event(evt->id, enable, remote->priv);
+	if (ret)
+		return ret;
+
+	evt->enabled = enable;
+
+	return 0;
+}
+
+static int remote_event_enable_show(struct seq_file *s, void *unused)
+{
+	struct remote_event *evt = s->private;
+
+	seq_printf(s, "%d\n", evt->enabled);
+
+	return 0;
+}
+
+static ssize_t remote_event_enable_write(struct file *filp, const char __user *ubuf,
+					 size_t count, loff_t *ppos)
+{
+	struct seq_file *seq = filp->private_data;
+	struct remote_event *evt = seq->private;
+	struct trace_remote *remote = evt->remote;
+	u8 enable;
+	int ret;
+
+	ret = kstrtou8_from_user(ubuf, count, 10, &enable);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&remote->lock);
+
+	ret = trace_remote_enable_event(remote, evt, enable);
+	if (ret)
+		return ret;
+
+	return count;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(remote_event_enable);
+
+static int remote_event_id_show(struct seq_file *s, void *unused)
+{
+	struct remote_event *evt = s->private;
+
+	seq_printf(s, "%d\n", evt->id);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(remote_event_id);
+
+static int remote_event_format_show(struct seq_file *s, void *unused)
+{
+	size_t offset = sizeof(struct remote_event_hdr);
+	struct remote_event *evt = s->private;
+	struct trace_event_fields *field;
+
+	seq_printf(s, "name: %s\n", evt->name);
+	seq_printf(s, "ID: %d\n", evt->id);
+	seq_puts(s,
+		 "format:\n\tfield:unsigned short common_type;\toffset:0;\tsize:2;\tsigned:0;\n\n");
+
+	field = &evt->fields[0];
+	while (field->name) {
+		seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%u;\tsigned:%d;\n",
+			   field->type, field->name, offset, field->size,
+			   field->is_signed);
+		offset += field->size;
+		field++;
+	}
+
+	if (field != &evt->fields[0])
+		seq_puts(s, "\n");
+
+	seq_printf(s, "print fmt: %s\n", evt->print_fmt);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(remote_event_format);
+
+static int remote_event_callback(const char *name, umode_t *mode, void **data,
+				 const struct file_operations **fops)
+{
+	if (!strcmp(name, "enable")) {
+		*mode = TRACEFS_MODE_WRITE;
+		*fops = &remote_event_enable_fops;
+		return 1;
+	}
+
+	if (!strcmp(name, "id")) {
+		*mode = TRACEFS_MODE_READ;
+		*fops = &remote_event_id_fops;
+		return 1;
+	}
+
+	if (!strcmp(name, "format")) {
+		*mode = TRACEFS_MODE_READ;
+		*fops = &remote_event_format_fops;
+		return 1;
+	}
+
+	return 0;
+}
+
+static ssize_t remote_events_dir_enable_write(struct file *filp, const char __user *ubuf,
+					      size_t count, loff_t *ppos)
+{
+	struct trace_remote *remote = file_inode(filp)->i_private;
+	int i, ret;
+	u8 enable;
+
+	ret = kstrtou8_from_user(ubuf, count, 10, &enable);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&remote->lock);
+
+	for (i = 0; i < remote->nr_events; i++) {
+		struct remote_event *evt = &remote->events[i];
+
+		trace_remote_enable_event(remote, evt, enable);
+	}
+
+	return count;
+}
+
+static ssize_t remote_events_dir_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
+					     loff_t *ppos)
+{
+	struct trace_remote *remote = file_inode(filp)->i_private;
+	const char enabled_char[] = {'0', '1', 'X'};
+	char enabled_str[] = " \n";
+	int i, enabled = -1;
+
+	guard(mutex)(&remote->lock);
+
+	for (i = 0; i < remote->nr_events; i++) {
+		struct remote_event *evt = &remote->events[i];
+
+		if (enabled == -1) {
+			enabled = evt->enabled;
+		} else if (enabled != evt->enabled) {
+			enabled = 2;
+			break;
+		}
+	}
+
+	enabled_str[0] = enabled_char[enabled == -1 ? 0 : enabled];
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, enabled_str, 2);
+}
+
+static const struct file_operations remote_events_dir_enable_fops = {
+	.write = remote_events_dir_enable_write,
+	.read = remote_events_dir_enable_read,
+};
+
+static ssize_t
+remote_events_dir_header_page_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+	struct trace_seq *s;
+	int ret;
+
+	s = kmalloc(sizeof(*s), GFP_KERNEL);
+	if (!s)
+		return -ENOMEM;
+
+	trace_seq_init(s);
+
+	ring_buffer_print_page_header(NULL, s);
+	ret = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, trace_seq_used(s));
+	kfree(s);
+
+	return ret;
+}
+
+static const struct file_operations remote_events_dir_header_page_fops = {
+	.read = remote_events_dir_header_page_read,
+};
+
+static ssize_t
+remote_events_dir_header_event_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+{
+	struct trace_seq *s;
+	int ret;
+
+	s = kmalloc(sizeof(*s), GFP_KERNEL);
+	if (!s)
+		return -ENOMEM;
+
+	trace_seq_init(s);
+
+	ring_buffer_print_entry_header(s);
+	ret = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, trace_seq_used(s));
+	kfree(s);
+
+	return ret;
+}
+
+static const struct file_operations remote_events_dir_header_event_fops = {
+	.read = remote_events_dir_header_event_read,
+};
+
+static int remote_events_dir_callback(const char *name, umode_t *mode, void **data,
+				      const struct file_operations **fops)
+{
+	if (!strcmp(name, "enable")) {
+		*mode = TRACEFS_MODE_WRITE;
+		*fops = &remote_events_dir_enable_fops;
+		return 1;
+	}
+
+	if (!strcmp(name, "header_page")) {
+		*mode = TRACEFS_MODE_READ;
+		*fops = &remote_events_dir_header_page_fops;
+		return 1;
+	}
+
+	if (!strcmp(name, "header_event")) {
+		*mode = TRACEFS_MODE_READ;
+		*fops = &remote_events_dir_header_event_fops;
+		return 1;
+	}
+
+	return 0;
+}
+
+static int trace_remote_init_eventfs(const char *remote_name, struct trace_remote *remote,
+				     struct remote_event *evt)
+{
+	struct eventfs_inode *eventfs = remote->eventfs;
+	static struct eventfs_entry dir_entries[] = {
+		{
+			.name		= "enable",
+			.callback	= remote_events_dir_callback,
+		}, {
+			.name		= "header_page",
+			.callback	= remote_events_dir_callback,
+		}, {
+			.name		= "header_event",
+			.callback	= remote_events_dir_callback,
+		}
+	};
+	static struct eventfs_entry entries[] = {
+		{
+			.name		= "enable",
+			.callback	= remote_event_callback,
+		}, {
+			.name		= "id",
+			.callback	= remote_event_callback,
+		}, {
+			.name		= "format",
+			.callback	= remote_event_callback,
+		}
+	};
+	bool eventfs_create = false;
+
+	if (!eventfs) {
+		eventfs = eventfs_create_events_dir("events", remote->dentry, dir_entries,
+						    ARRAY_SIZE(dir_entries), remote);
+		if (IS_ERR(eventfs))
+			return PTR_ERR(eventfs);
+
+		/*
+		 * Create similar hierarchy as local events even if a single system is supported at
+		 * the moment
+		 */
+		eventfs = eventfs_create_dir(remote_name, eventfs, NULL, 0, NULL);
+		if (IS_ERR(eventfs))
+			return PTR_ERR(eventfs);
+
+		remote->eventfs = eventfs;
+		eventfs_create = true;
+	}
+
+	eventfs = eventfs_create_dir(evt->name, eventfs, entries, ARRAY_SIZE(entries), evt);
+	if (IS_ERR(eventfs)) {
+		if (eventfs_create) {
+			eventfs_remove_events_dir(remote->eventfs);
+			remote->eventfs = NULL;
+		}
+		return PTR_ERR(eventfs);
+	}
+
+	return 0;
+}
+
+static int trace_remote_attach_events(struct trace_remote *remote, struct remote_event *events,
+				      size_t nr_events)
+{
+	int i;
+
+	for (i = 0; i < nr_events; i++) {
+		struct remote_event *evt = &events[i];
+
+		if (evt->remote)
+			return -EEXIST;
+
+		evt->remote = remote;
+
+		/* We need events to be sorted for efficient lookup */
+		if (i && evt->id <= events[i - 1].id)
+			return -EINVAL;
+	}
+
+	remote->events = events;
+	remote->nr_events = nr_events;
+
+	return 0;
+}
+
+static int trace_remote_register_events(const char *remote_name, struct trace_remote *remote,
+					struct remote_event *events, size_t nr_events)
+{
+	int i, ret;
+
+	ret = trace_remote_attach_events(remote, events, nr_events);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < nr_events; i++) {
+		struct remote_event *evt = &events[i];
+
+		ret = trace_remote_init_eventfs(remote_name, remote, evt);
+		if (ret)
+			pr_warn("Failed to init eventfs for event '%s' (%d)",
+				evt->name, ret);
+	}
+
+	return 0;
+}
+
+static int __cmp_events(const void *key, const void *data)
+{
+	const struct remote_event *evt = data;
+	int id = (int)((long)key);
+
+	return id - (int)evt->id;
+}
+
+static struct remote_event *trace_remote_find_event(struct trace_remote *remote, unsigned short id)
+{
+	return bsearch((const void *)(unsigned long)id, remote->events, remote->nr_events,
+		       sizeof(*remote->events), __cmp_events);
+}
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 37317b81fcdab..8ad72e17d8eb9 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -174,7 +174,6 @@ sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadat
 			{ O_NOFOLLOW, "O_NOFOLLOW" },
 			{ O_NOATIME, "O_NOATIME" },
 			{ O_CLOEXEC, "O_CLOEXEC" },
-			{ -1, NULL }
 		};
 
 	trace_seq_printf(s, "%s(", entry->name);
@@ -205,7 +204,7 @@ sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadat
 				trace_seq_puts(s, "O_RDONLY|");
 			}
 
-			trace_print_flags_seq(s, "|", bits, __flags);
+			trace_print_flags_seq(s, "|", bits, __flags, ARRAY_SIZE(__flags));
 			/*
 			 * trace_print_flags_seq() adds a '\0' to the
 			 * buffer, but this needs to append more to the seq.
diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
index 8d82913223a13..8614430ca212a 100644
--- a/kernel/vmcore_info.c
+++ b/kernel/vmcore_info.c
@@ -18,8 +18,6 @@
 #include <asm/page.h>
 #include <asm/sections.h>
 
-#include <crypto/sha1.h>
-
 #include "kallsyms_internal.h"
 #include "kexec_internal.h"
 
@@ -198,7 +196,7 @@ static int __init crash_save_vmcoreinfo_init(void)
 	VMCOREINFO_OFFSET(page, lru);
 	VMCOREINFO_OFFSET(page, _mapcount);
 	VMCOREINFO_OFFSET(page, private);
-	VMCOREINFO_OFFSET(page, compound_head);
+	VMCOREINFO_OFFSET(page, compound_info);
 	VMCOREINFO_OFFSET(pglist_data, node_zones);
 	VMCOREINFO_OFFSET(pglist_data, nr_zones);
 #ifdef CONFIG_FLATMEM
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 7d675781bc917..87dd5e0f6968d 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -60,6 +60,13 @@ unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
 int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
 # endif /* CONFIG_SMP */
 
+/*
+ * Number of consecutive missed interrupts before declaring a lockup.
+ * Default to 1 (immediate) for NMI/Perf. Buddy will overwrite this to 3.
+ */
+int __read_mostly watchdog_hardlockup_miss_thresh = 1;
+EXPORT_SYMBOL_GPL(watchdog_hardlockup_miss_thresh);
+
 /*
  * Should we panic when a soft-lockup or hard-lockup occurs:
  */
@@ -137,6 +144,7 @@ __setup("nmi_watchdog=", hardlockup_panic_setup);
 
 static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
 static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
+static DEFINE_PER_CPU(int, hrtimer_interrupts_missed);
 static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
 static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
 static unsigned long hard_lockup_nmi_warn;
@@ -159,21 +167,33 @@ void watchdog_hardlockup_touch_cpu(unsigned int cpu)
 	per_cpu(watchdog_hardlockup_touched, cpu) = true;
 }
 
-static bool is_hardlockup(unsigned int cpu)
+static void watchdog_hardlockup_update_reset(unsigned int cpu)
 {
 	int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));
 
-	if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
-		return true;
-
 	/*
 	 * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
 	 * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
 	 * written/read by a single CPU.
 	 */
 	per_cpu(hrtimer_interrupts_saved, cpu) = hrint;
+	per_cpu(hrtimer_interrupts_missed, cpu) = 0;
+}
+
+static bool is_hardlockup(unsigned int cpu)
+{
+	int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));
 
-	return false;
+	if (per_cpu(hrtimer_interrupts_saved, cpu) != hrint) {
+		watchdog_hardlockup_update_reset(cpu);
+		return false;
+	}
+
+	per_cpu(hrtimer_interrupts_missed, cpu)++;
+	if (per_cpu(hrtimer_interrupts_missed, cpu) % watchdog_hardlockup_miss_thresh)
+		return false;
+
+	return true;
 }
 
 static void watchdog_hardlockup_kick(void)
@@ -187,8 +207,11 @@ static void watchdog_hardlockup_kick(void)
 void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 {
 	int hardlockup_all_cpu_backtrace;
+	unsigned int this_cpu;
+	unsigned long flags;
 
 	if (per_cpu(watchdog_hardlockup_touched, cpu)) {
+		watchdog_hardlockup_update_reset(cpu);
 		per_cpu(watchdog_hardlockup_touched, cpu) = false;
 		return;
 	}
@@ -201,74 +224,73 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
 	 * fired multiple times before we overflow'd. If it hasn't
 	 * then this is a good indication the cpu is stuck
 	 */
-	if (is_hardlockup(cpu)) {
-		unsigned int this_cpu = smp_processor_id();
-		unsigned long flags;
+	if (!is_hardlockup(cpu)) {
+		per_cpu(watchdog_hardlockup_warned, cpu) = false;
+		return;
+	}
 
 #ifdef CONFIG_SYSFS
-		++hardlockup_count;
+	++hardlockup_count;
 #endif
-		/*
-		 * A poorly behaving BPF scheduler can trigger hard lockup by
-		 * e.g. putting numerous affinitized tasks in a single queue and
-		 * directing all CPUs at it. The following call can return true
-		 * only once when sched_ext is enabled and will immediately
-		 * abort the BPF scheduler and print out a warning message.
-		 */
-		if (scx_hardlockup(cpu))
-			return;
+	/*
+	 * A poorly behaving BPF scheduler can trigger hard lockup by
+	 * e.g. putting numerous affinitized tasks in a single queue and
+	 * directing all CPUs at it. The following call can return true
+	 * only once when sched_ext is enabled and will immediately
+	 * abort the BPF scheduler and print out a warning message.
+	 */
+	if (scx_hardlockup(cpu))
+		return;
 
-		/* Only print hardlockups once. */
-		if (per_cpu(watchdog_hardlockup_warned, cpu))
-			return;
+	/* Only print hardlockups once. */
+	if (per_cpu(watchdog_hardlockup_warned, cpu))
+		return;
 
-		/*
-		 * Prevent multiple hard-lockup reports if one cpu is already
-		 * engaged in dumping all cpu back traces.
-		 */
-		if (hardlockup_all_cpu_backtrace) {
-			if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
-				return;
-		}
+	/*
+	 * Prevent multiple hard-lockup reports if one cpu is already
+	 * engaged in dumping all cpu back traces.
+	 */
+	if (hardlockup_all_cpu_backtrace) {
+		if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
+			return;
+	}
 
-		/*
-		 * NOTE: we call printk_cpu_sync_get_irqsave() after printing
-		 * the lockup message. While it would be nice to serialize
-		 * that printout, we really want to make sure that if some
-		 * other CPU somehow locked up while holding the lock associated
-		 * with printk_cpu_sync_get_irqsave() that we can still at least
-		 * get the message about the lockup out.
-		 */
-		pr_emerg("CPU%u: Watchdog detected hard LOCKUP on cpu %u\n", this_cpu, cpu);
-		printk_cpu_sync_get_irqsave(flags);
+	/*
+	 * NOTE: we call printk_cpu_sync_get_irqsave() after printing
+	 * the lockup message. While it would be nice to serialize
+	 * that printout, we really want to make sure that if some
+	 * other CPU somehow locked up while holding the lock associated
+	 * with printk_cpu_sync_get_irqsave() that we can still at least
+	 * get the message about the lockup out.
+	 */
+	this_cpu = smp_processor_id();
+	pr_emerg("CPU%u: Watchdog detected hard LOCKUP on cpu %u\n", this_cpu, cpu);
+	printk_cpu_sync_get_irqsave(flags);
 
-		print_modules();
-		print_irqtrace_events(current);
-		if (cpu == this_cpu) {
-			if (regs)
-				show_regs(regs);
-			else
-				dump_stack();
-			printk_cpu_sync_put_irqrestore(flags);
-		} else {
-			printk_cpu_sync_put_irqrestore(flags);
-			trigger_single_cpu_backtrace(cpu);
-		}
+	print_modules();
+	print_irqtrace_events(current);
+	if (cpu == this_cpu) {
+		if (regs)
+			show_regs(regs);
+		else
+			dump_stack();
+		printk_cpu_sync_put_irqrestore(flags);
+	} else {
+		printk_cpu_sync_put_irqrestore(flags);
+		trigger_single_cpu_backtrace(cpu);
+	}
 
-		if (hardlockup_all_cpu_backtrace) {
-			trigger_allbutcpu_cpu_backtrace(cpu);
-			if (!hardlockup_panic)
-				clear_bit_unlock(0, &hard_lockup_nmi_warn);
-		}
+	if (hardlockup_all_cpu_backtrace) {
+		trigger_allbutcpu_cpu_backtrace(cpu);
+		if (!hardlockup_panic)
+			clear_bit_unlock(0, &hard_lockup_nmi_warn);
+	}
 
-		sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
-		if (hardlockup_panic)
-			nmi_panic(regs, "Hard LOCKUP");
+	sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
+	if (hardlockup_panic)
+		nmi_panic(regs, "Hard LOCKUP");
 
-		per_cpu(watchdog_hardlockup_warned, cpu) = true;
-	} else {
-		per_cpu(watchdog_hardlockup_warned, cpu) = false;
-	}
+	per_cpu(watchdog_hardlockup_warned, cpu) = true;
 }
 
 #else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */
diff --git a/kernel/watchdog_buddy.c b/kernel/watchdog_buddy.c
index ee754d767c213..3a1e57080c1c6 100644
--- a/kernel/watchdog_buddy.c
+++ b/kernel/watchdog_buddy.c
@@ -21,6 +21,7 @@ static unsigned int watchdog_next_cpu(unsigned int cpu)
 
 int __init watchdog_hardlockup_probe(void)
 {
+	watchdog_hardlockup_miss_thresh = 3;
 	return 0;
 }
 
@@ -86,14 +87,6 @@ void watchdog_buddy_check_hardlockup(int hrtimer_interrupts)
 {
 	unsigned int next_cpu;
 
-	/*
-	 * Test for hardlockups every 3 samples. The sample period is
-	 *  watchdog_thresh * 2 / 5, so 3 samples gets us back to slightly over
-	 *  watchdog_thresh (over by 20%).
-	 */
-	if (hrtimer_interrupts % 3 != 0)
-		return;
-
 	/* check for a hardlockup on the next CPU */
 	next_cpu = watchdog_next_cpu(smp_processor_id());
 	if (next_cpu >= nr_cpu_ids)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index b77119d71641a..fffe90cd65ccf 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -41,6 +41,7 @@
 #include <linux/mempolicy.h>
 #include <linux/freezer.h>
 #include <linux/debug_locks.h>
+#include <linux/device/devres.h>
 #include <linux/lockdep.h>
 #include <linux/idr.h>
 #include <linux/jhash.h>
@@ -404,7 +405,7 @@ struct work_offq_data {
 	u32			flags;
 };
 
-static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
+static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = {
 	[WQ_AFFN_DFL]		= "default",
 	[WQ_AFFN_CPU]		= "cpu",
 	[WQ_AFFN_SMT]		= "smt",
@@ -530,6 +531,8 @@ struct workqueue_struct *system_bh_wq;
 EXPORT_SYMBOL_GPL(system_bh_wq);
 struct workqueue_struct *system_bh_highpri_wq;
 EXPORT_SYMBOL_GPL(system_bh_highpri_wq);
+struct workqueue_struct *system_dfl_long_wq __ro_after_init;
+EXPORT_SYMBOL_GPL(system_dfl_long_wq);
 
 static int worker_thread(void *__worker);
 static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
@@ -2507,7 +2510,6 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
 	struct timer_list *timer = &dwork->timer;
 	struct work_struct *work = &dwork->work;
 
-	WARN_ON_ONCE(!wq);
 	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
 	WARN_ON_ONCE(timer_pending(timer));
 	WARN_ON_ONCE(!list_empty(&work->entry));
@@ -3157,6 +3159,10 @@ static bool manage_workers(struct worker *worker)
 	return true;
 }
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void process_one_work(struct worker *worker, struct work_struct *work);
+#endif
+
 /**
  * process_one_work - process single work
  * @worker: self
@@ -3180,6 +3186,9 @@ __acquires(&pool->lock)
 	unsigned long work_data;
 	int lockdep_start_depth, rcu_start_depth;
 	bool bh_draining = pool->flags & POOL_BH_DRAINING;
+#ifdef CONFIG_KCOV
+	unsigned int old_kcov_mode, new_kcov_mode;
+#endif
 #ifdef CONFIG_LOCKDEP
 	/*
 	 * It is permissible to free the struct work_struct from
@@ -3273,7 +3282,13 @@ __acquires(&pool->lock)
 	 */
 	lockdep_invariant_state(true);
 	trace_workqueue_execute_start(work);
+#ifdef CONFIG_KCOV
+	old_kcov_mode = READ_ONCE(current->kcov_mode);
+#endif
 	worker->current_func(work);
+#ifdef CONFIG_KCOV
+	new_kcov_mode = READ_ONCE(current->kcov_mode);
+#endif
 	/*
 	 * While we must be careful to not use "work" after this, the trace
 	 * point will only record its address.
@@ -3296,6 +3311,11 @@ __acquires(&pool->lock)
 		debug_show_held_locks(current);
 		dump_stack();
 	}
+#ifdef CONFIG_KCOV
+	if (unlikely((old_kcov_mode & ~(1 << 30)) != (new_kcov_mode & ~(1 << 30))))
+		pr_err("BUG: workqueue function %ps changed kcov_mode from %u to %u\n",
+		       worker->current_func, old_kcov_mode, new_kcov_mode);
+#endif
 
 	/*
 	 * The following prevents a kworker from hogging CPU on !PREEMPTION
@@ -3387,6 +3407,11 @@ static int worker_thread(void *__worker)
 	struct worker *worker = __worker;
 	struct worker_pool *pool = worker->pool;
 
+#ifdef CONFIG_KCOV
+	if (unlikely(current->kcov_mode & ~(1 << 30)))
+		pr_err("BUG: %s started with kcov_mode=%u\n", __func__, current->kcov_mode);
+#endif
+
 	/* tell the scheduler that this is a workqueue worker */
 	set_pf_worker(true);
 woke_up:
@@ -3539,6 +3564,11 @@ static int rescuer_thread(void *__rescuer)
 	struct workqueue_struct *wq = rescuer->rescue_wq;
 	bool should_stop;
 
+#ifdef CONFIG_KCOV
+	if (unlikely(current->kcov_mode & ~(1 << 30)))
+		pr_err("BUG: %s started with kcov_mode=%u\n", __func__, current->kcov_mode);
+#endif
+
 	set_user_nice(current, RESCUER_NICE_LEVEL);
 
 	/*
@@ -5892,6 +5922,33 @@ struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,
 }
 EXPORT_SYMBOL_GPL(alloc_workqueue_noprof);
 
+static void devm_workqueue_release(void *res)
+{
+	destroy_workqueue(res);
+}
+
+__printf(2, 5) struct workqueue_struct *
+devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags,
+		     int max_active, ...)
+{
+	struct workqueue_struct *wq;
+	va_list args;
+	int ret;
+
+	va_start(args, max_active);
+	wq = alloc_workqueue(fmt, flags, max_active, args);
+	va_end(args);
+	if (!wq)
+		return NULL;
+
+	ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq);
+	if (ret)
+		return NULL;
+
+	return wq;
+}
+EXPORT_SYMBOL_GPL(devm_alloc_workqueue);
+
 #ifdef CONFIG_LOCKDEP
 __printf(1, 5)
 struct workqueue_struct *
@@ -7066,13 +7123,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk)
 
 static int parse_affn_scope(const char *val)
 {
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
-		if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
-			return i;
-	}
-	return -EINVAL;
+	return sysfs_match_string(wq_affn_names, val);
 }
 
 static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp)
@@ -7179,7 +7230,26 @@ static struct attribute *wq_sysfs_attrs[] = {
 	&dev_attr_max_active.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(wq_sysfs);
+
+static umode_t wq_sysfs_is_visible(struct kobject *kobj, struct attribute *a, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct workqueue_struct *wq = dev_to_wq(dev);
+
+	/*
+	 * Adjusting max_active breaks ordering guarantee. Changing it has no
+	 * effect on BH worker. Limit max_active to RO in such case.
+	 */
+	if (wq->flags & (WQ_BH | __WQ_ORDERED))
+		return 0444;
+	return a->mode;
+}
+
+static const struct attribute_group wq_sysfs_group = {
+	.is_visible = wq_sysfs_is_visible,
+	.attrs = wq_sysfs_attrs,
+};
+__ATTRIBUTE_GROUPS(wq_sysfs);
 
 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
 			    char *buf)
@@ -7482,13 +7552,6 @@ int workqueue_sysfs_register(struct workqueue_struct *wq)
 	struct wq_device *wq_dev;
 	int ret;
 
-	/*
-	 * Adjusting max_active breaks ordering guarantee.  Disallow exposing
-	 * ordered workqueues.
-	 */
-	if (WARN_ON(wq->flags & __WQ_ORDERED))
-		return -EINVAL;
-
 	wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev);
 	if (!wq_dev)
 		return -ENOMEM;
@@ -7943,11 +8006,12 @@ void __init workqueue_init_early(void)
 	system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0);
 	system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
 					       WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0);
+	system_dfl_long_wq = alloc_workqueue("events_dfl_long", WQ_UNBOUND, WQ_MAX_ACTIVE);
 	BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq ||
 	       !system_unbound_wq || !system_freezable_wq || !system_dfl_wq ||
 	       !system_power_efficient_wq ||
 	       !system_freezable_power_efficient_wq ||
-	       !system_bh_wq || !system_bh_highpri_wq);
+	       !system_bh_wq || !system_bh_highpri_wq || !system_dfl_long_wq);
 }
 
 static void __init wq_cpu_intensive_thresh_init(void)
diff --git a/lib/Kconfig b/lib/Kconfig
index 0f2fb96106476..b614c0dcd52ae 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -52,6 +52,18 @@ config PACKING_KUNIT_TEST
 
 	  When in doubt, say N.
 
+config PERCPU_COUNTER_TREE_TEST
+	tristate "Hierarchical Per-CPU counter test" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds Kunit tests for the hierarchical per-cpu counters.
+
+	  For more information on KUnit and unit tests in general,
+	  please refer to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+	  When in doubt, say N.
+
 config BITREVERSE
 	tristate
 
@@ -138,6 +150,7 @@ config TRACE_MMIO_ACCESS
 
 source "lib/crc/Kconfig"
 source "lib/crypto/Kconfig"
+source "lib/raid/Kconfig"
 
 config XXHASH
 	tristate
@@ -625,9 +638,6 @@ config PLDMFW
 config ASN1_ENCODER
        tristate
 
-config POLYNOMIAL
-       tristate
-
 config FIRMWARE_TABLE
 	bool
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 93f356d2b3d95..6ef26c5a700d8 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -812,6 +812,38 @@ config DEBUG_OBJECTS_PERCPU_COUNTER
 	  percpu counter routines to track the life time of percpu counter
 	  objects and validate the percpu counter operations.
 
+config DEBUG_OBJECTS_POOL_SIZE_SHIFT
+	int "Metadata objects pool size"
+	depends on DEBUG_OBJECTS
+	range 0 21
+	default 6
+	help
+	  To debug object operations a certain amount of metadata has to be
+	  kept per object that is tracked. During boot a static amount of pre-
+	  allocated objects is set aside to be used for this task until such a
+	  time a dynamic allocator can be used. Once a dynamic allocator can be
+	  used an initial amount of objects is pre-allocated to be used when
+	  needed.
+
+	  This option sets the amount of both: the amount of initially
+	  statically allocated objects; and later the amount of dynamically
+	  pre-allocated objects. It is used as exponent to the power of 2,
+	  multiplied by the batch size used to set how many objects are move
+	  between pools at once.
+
+	  For example, when left at the default of 6:
+	      N_OBJECTS = 2^DEBUG_OBJECTS_POOL_SIZE_SHIFT * N_BATCH
+	      N_OBJECTS = 2^6 * 16
+	      N_OBJECTS = 1024
+	  By increasing the option by 1, you double the amount of objects.
+
+	  An indication that you need to increase this option is that during
+	  boot you see messages like this:
+	      ODEBUG: Out of memory. ODEBUG disabled
+
+	  If in doubt, leave the default.
+
+
 config DEBUG_OBJECTS_ENABLE_DEFAULT
 	int "debug_objects bootup default value (0-1)"
 	range 0 1
@@ -1618,14 +1650,22 @@ config LOCKDEP_STACK_TRACE_BITS
 	int "Size for MAX_STACK_TRACE_ENTRIES (as Nth power of 2)"
 	depends on LOCKDEP && !LOCKDEP_SMALL
 	range 10 26
+	default 21 if KASAN
 	default 19
 	help
 	  Try increasing this value if you hit "BUG: MAX_STACK_TRACE_ENTRIES too low!" message.
 
+	  KASAN significantly increases stack trace consumption because its
+	  slab tracking interacts with lockdep's dependency validation under
+	  PREEMPT_FULL, creating a feedback loop.  The higher default when
+	  KASAN is enabled costs ~12MB extra, which is negligible compared to
+	  KASAN's own shadow memory overhead.
+
 config LOCKDEP_STACK_TRACE_HASH_BITS
 	int "Size for STACK_TRACE_HASH_SIZE (as Nth power of 2)"
 	depends on LOCKDEP && !LOCKDEP_SMALL
 	range 10 26
+	default 16 if KASAN
 	default 14
 	help
 	  Try increasing this value if you need large STACK_TRACE_HASH_SIZE.
@@ -3048,6 +3088,16 @@ config LONGEST_SYM_KUNIT_TEST
 
 	  If unsure, say N.
 
+config LINEINFO_KUNIT_TEST
+	tristate "KUnit tests for kallsyms lineinfo" if !KUNIT_ALL_TESTS
+	depends on KUNIT && KALLSYMS_LINEINFO
+	default KUNIT_ALL_TESTS
+	help
+	  KUnit tests for the kallsyms source line info feature.
+	  Verifies that stack traces include correct (file.c:line) annotations.
+
+	  If unsure, say N.
+
 config HW_BREAKPOINT_KUNIT_TEST
 	bool "Test hw_breakpoint constraints accounting" if !KUNIT_ALL_TESTS
 	depends on HAVE_HW_BREAKPOINT
diff --git a/lib/Makefile b/lib/Makefile
index 1b9ee167517f3..3e1eeefd9832f 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -120,7 +120,7 @@ endif
 obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
 CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
 
-obj-y += math/ crc/ crypto/ tests/ vdso/
+obj-y += math/ crc/ crypto/ tests/ vdso/ raid/
 
 obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
 obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
@@ -181,6 +181,7 @@ obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
 obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
 obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
 obj-$(CONFIG_SMP) += percpu_counter.o
+obj-$(CONFIG_SMP) += percpu_counter_tree.o
 obj-$(CONFIG_AUDIT_GENERIC) += audit.o
 obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
 
@@ -243,8 +244,6 @@ obj-$(CONFIG_MEMREGION) += memregion.o
 obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
 obj-$(CONFIG_IRQ_POLL) += irq_poll.o
 
-obj-$(CONFIG_POLYNOMIAL) += polynomial.o
-
 # stackdepot.c should not be instrumented or call instrumented functions.
 # Prevent the compiler from calling builtins like memcmp() or bcmp() from this
 # file.
@@ -307,6 +306,7 @@ obj-$(CONFIG_UBSAN) += ubsan.o
 UBSAN_SANITIZE_ubsan.o := n
 KASAN_SANITIZE_ubsan.o := n
 KCSAN_SANITIZE_ubsan.o := n
+KMSAN_SANITIZE_ubsan.o := n
 CFLAGS_ubsan.o := -fno-stack-protector $(DISABLE_KSTACK_ERASE)
 
 obj-$(CONFIG_SBITMAP) += sbitmap.o
diff --git a/lib/audit.c b/lib/audit.c
index 738bda22dd39b..bc07fbd3a6983 100644
--- a/lib/audit.c
+++ b/lib/audit.c
@@ -42,7 +42,7 @@ int audit_classify_syscall(int abi, unsigned syscall)
 	if (audit_is_compat(abi))
 		return audit_classify_compat_syscall(abi, syscall);
 
-	switch(syscall) {
+	switch (syscall) {
 #ifdef __NR_open
 	case __NR_open:
 		return AUDITSC_OPEN;
diff --git a/lib/bch.c b/lib/bch.c
index 9561c08288025..c991c71c4cbdf 100644
--- a/lib/bch.c
+++ b/lib/bch.c
@@ -392,7 +392,7 @@ static void compute_syndromes(struct bch_control *bch, uint32_t *ecc,
 			for (j = 0; j < 2*t; j += 2)
 				syn[j] ^= a_pow(bch, (j+1)*(i+s));
 
-			poly ^= (1 << i);
+			poly ^= (1u << i);
 		}
 	} while (s > 0);
 
@@ -612,7 +612,7 @@ static int find_poly_deg2_roots(struct bch_control *bch, struct gf_poly *poly,
 		while (v) {
 			i = deg(v);
 			r ^= bch->xi_tab[i];
-			v ^= (1 << i);
+			v ^= (1u << i);
 		}
 		/* verify root */
 		if ((gf_sqr(bch, r)^r) == u) {
@@ -1116,7 +1116,7 @@ static void build_mod8_tables(struct bch_control *bch, const uint32_t *g)
 		for (b = 0; b < 4; b++) {
 			/* we want to compute (p(X).X^(8*b+deg(g))) mod g(X) */
 			tab = bch->mod8_tab + (b*256+i)*l;
-			data = i << (8*b);
+			data = (unsigned int)i << (8*b);
 			while (data) {
 				d = deg(data);
 				/* subtract X^d.g(X) from p(X).X^(8*b+deg(g)) */
diff --git a/lib/bug.c b/lib/bug.c
index 623c467a8b76c..bbc301097749b 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -254,7 +254,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 	if (file)
 		pr_crit("kernel BUG at %s:%u!\n", file, line);
 	else
-		pr_crit("Kernel BUG at %pB [verbose debug info unavailable]\n",
+		pr_crit("kernel BUG at %pB [verbose debug info unavailable]\n",
 			(void *)bugaddr);
 
 	return BUG_TRAP_TYPE_BUG;
@@ -263,7 +263,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs)
 {
 	enum bug_trap_type ret;
-	bool rcu = false;
+	bool rcu;
 
 	rcu = warn_rcu_enter();
 	ret = __report_bug(bug, bug_addr(bug), regs);
@@ -275,7 +275,7 @@ enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs)
 enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)
 {
 	enum bug_trap_type ret;
-	bool rcu = false;
+	bool rcu;
 
 	rcu = warn_rcu_enter();
 	ret = __report_bug(NULL, bugaddr, regs);
diff --git a/lib/cmdline.c b/lib/cmdline.c
index 90ed997d95701..16cce6621cec1 100644
--- a/lib/cmdline.c
+++ b/lib/cmdline.c
@@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n)
  *	When @pint is NULL the function can be used as a validator of
  *	the current option in the string.
  *
- *	Return values:
+ *	Return:
  *	0 - no int in string
  *	1 - int found, no subsequent comma
  *	2 - int found including a subsequent comma
@@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options);
  *
  *	Parses a string into a number.  The number stored at @ptr is
  *	potentially suffixed with K, M, G, T, P, E.
+ *
+ *	Return: The value as recognized by simple_strtoull() multiplied
+ *	by the value as specified by suffix, if any.
  */
 
 unsigned long long memparse(const char *ptr, char **retptr)
 {
 	char *endptr;	/* local pointer to end of parsed string */
-
 	unsigned long long ret = simple_strtoull(ptr, &endptr, 0);
+	unsigned int shl = 0;
 
+	/* Consume valid suffix even in case of overflow. */
 	switch (*endptr) {
 	case 'E':
 	case 'e':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'P':
 	case 'p':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'T':
 	case 't':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'G':
 	case 'g':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'M':
 	case 'm':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'K':
 	case 'k':
-		ret <<= 10;
-		endptr++;
+		shl += 10;
 		fallthrough;
 	default:
 		break;
 	}
 
+	if (shl && likely(ptr != endptr)) {
+		/* Have valid suffix with preceding number. */
+		if (unlikely(check_shl_overflow(ret, shl, &ret)))
+			ret = ULLONG_MAX;
+		endptr++;
+	}
+
 	if (retptr)
 		*retptr = endptr;
 
@@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse);
  *	This function parses a string containing a comma-separated list of
  *	strings like a=b,c.
  *
- *	Return true if there's such option in the string, or return false.
+ *	Return: True if there's such option in the string or false otherwise.
  */
 bool parse_option_str(const char *str, const char *option)
 {
diff --git a/lib/crc/.kunitconfig b/lib/crc/.kunitconfig
new file mode 100644
index 0000000000000..0a3671ba573f0
--- /dev/null
+++ b/lib/crc/.kunitconfig
@@ -0,0 +1,3 @@
+CONFIG_KUNIT=y
+CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y
+CONFIG_CRC_KUNIT_TEST=y
diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig
index 70e7a6016de32..52e216f397468 100644
--- a/lib/crc/Kconfig
+++ b/lib/crc/Kconfig
@@ -48,7 +48,7 @@ config CRC_T10DIF_ARCH
 	bool
 	depends on CRC_T10DIF && CRC_OPTIMIZATIONS
 	default y if ARM && KERNEL_MODE_NEON
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if PPC64 && ALTIVEC
 	default y if RISCV && RISCV_ISA_ZBC
 	default y if X86
@@ -99,18 +99,27 @@ config CRC_OPTIMIZATIONS
 
 config CRC_KUNIT_TEST
 	tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
-	depends on KUNIT
+	depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64)
 	default KUNIT_ALL_TESTS
+	help
+	  Unit tests for the CRC library functions.
+
+	  This is intended to help people writing architecture-specific
+	  optimized versions.  If unsure, say N.
+
+config CRC_ENABLE_ALL_FOR_KUNIT
+	tristate "Enable all CRC functions for KUnit test"
+	depends on KUNIT
 	select CRC7
 	select CRC16
 	select CRC_T10DIF
 	select CRC32
 	select CRC64
 	help
-	  Unit tests for the CRC library functions.
+	  Enable all CRC functions that have test code in CRC_KUNIT_TEST.
 
-	  This is intended to help people writing architecture-specific
-	  optimized versions.  If unsure, say N.
+	  Enable this only if you'd like the CRC KUnit test suite to test all
+	  the CRC variants, even ones that wouldn't otherwise need to be built.
 
 config CRC_BENCHMARK
 	bool "Benchmark for the CRC functions"
diff --git a/lib/crc/tests/crc_kunit.c b/lib/crc/tests/crc_kunit.c
index 9a450e25ac811..9428cd9136253 100644
--- a/lib/crc/tests/crc_kunit.c
+++ b/lib/crc/tests/crc_kunit.c
@@ -268,8 +268,7 @@ crc_benchmark(struct kunit *test,
 	}
 }
 
-/* crc7_be */
-
+#if IS_REACHABLE(CONFIG_CRC7)
 static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len)
 {
 	/*
@@ -294,9 +293,9 @@ static void crc7_be_benchmark(struct kunit *test)
 {
 	crc_benchmark(test, crc7_be_wrapper);
 }
+#endif /* CONFIG_CRC7 */
 
-/* crc16 */
-
+#if IS_REACHABLE(CONFIG_CRC16)
 static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len)
 {
 	return crc16(crc, p, len);
@@ -318,9 +317,9 @@ static void crc16_benchmark(struct kunit *test)
 {
 	crc_benchmark(test, crc16_wrapper);
 }
+#endif /* CONFIG_CRC16 */
 
-/* crc_t10dif */
-
+#if IS_REACHABLE(CONFIG_CRC_T10DIF)
 static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len)
 {
 	return crc_t10dif_update(crc, p, len);
@@ -342,6 +341,9 @@ static void crc_t10dif_benchmark(struct kunit *test)
 {
 	crc_benchmark(test, crc_t10dif_wrapper);
 }
+#endif /* CONFIG_CRC_T10DIF */
+
+#if IS_REACHABLE(CONFIG_CRC32)
 
 /* crc32_le */
 
@@ -414,6 +416,9 @@ static void crc32c_benchmark(struct kunit *test)
 {
 	crc_benchmark(test, crc32c_wrapper);
 }
+#endif /* CONFIG_CRC32 */
+
+#if IS_REACHABLE(CONFIG_CRC64)
 
 /* crc64_be */
 
@@ -463,24 +468,35 @@ static void crc64_nvme_benchmark(struct kunit *test)
 {
 	crc_benchmark(test, crc64_nvme_wrapper);
 }
+#endif /* CONFIG_CRC64 */
 
 static struct kunit_case crc_test_cases[] = {
+#if IS_REACHABLE(CONFIG_CRC7)
 	KUNIT_CASE(crc7_be_test),
 	KUNIT_CASE(crc7_be_benchmark),
+#endif
+#if IS_REACHABLE(CONFIG_CRC16)
 	KUNIT_CASE(crc16_test),
 	KUNIT_CASE(crc16_benchmark),
+#endif
+#if IS_REACHABLE(CONFIG_CRC_T10DIF)
 	KUNIT_CASE(crc_t10dif_test),
 	KUNIT_CASE(crc_t10dif_benchmark),
+#endif
+#if IS_REACHABLE(CONFIG_CRC32)
 	KUNIT_CASE(crc32_le_test),
 	KUNIT_CASE(crc32_le_benchmark),
 	KUNIT_CASE(crc32_be_test),
 	KUNIT_CASE(crc32_be_benchmark),
 	KUNIT_CASE(crc32c_test),
 	KUNIT_CASE(crc32c_benchmark),
+#endif
+#if IS_REACHABLE(CONFIG_CRC64)
 	KUNIT_CASE(crc64_be_test),
 	KUNIT_CASE(crc64_be_benchmark),
 	KUNIT_CASE(crc64_nvme_test),
 	KUNIT_CASE(crc64_nvme_benchmark),
+#endif
 	{},
 };
 
diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig
index 6b2ce28ae5091..63a592731d1dc 100644
--- a/lib/crypto/.kunitconfig
+++ b/lib/crypto/.kunitconfig
@@ -1,25 +1,8 @@
 CONFIG_KUNIT=y
 
-# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a
-# corresponding KUnit test.  Those symbols cannot be directly enabled here,
-# since they are hidden symbols.
-CONFIG_CRYPTO=y
-CONFIG_CRYPTO_ADIANTUM=y
-CONFIG_CRYPTO_BLAKE2B=y
-CONFIG_CRYPTO_CHACHA20POLY1305=y
-CONFIG_CRYPTO_HCTR2=y
-CONFIG_CRYPTO_MD5=y
-CONFIG_CRYPTO_MLDSA=y
-CONFIG_CRYPTO_SHA1=y
-CONFIG_CRYPTO_SHA256=y
-CONFIG_CRYPTO_SHA512=y
-CONFIG_CRYPTO_SHA3=y
-CONFIG_INET=y
-CONFIG_IPV6=y
-CONFIG_NET=y
-CONFIG_NETDEVICES=y
-CONFIG_WIREGUARD=y
+CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y
 
+CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 032f9755f999b..4910fe20e42ad 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -10,6 +10,8 @@ config CRYPTO_LIB_UTILS
 
 config CRYPTO_LIB_AES
 	tristate
+	# Select dependencies of modes that are part of libaes.
+	select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS
 
 config CRYPTO_LIB_AES_ARCH
 	bool
@@ -28,6 +30,14 @@ config CRYPTO_LIB_AESCFB
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_UTILS
 
+config CRYPTO_LIB_AES_CBC_MACS
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions.  Select
+	  this if your module uses any of the functions from
+	  <crypto/aes-cbc-macs.h>.
+
 config CRYPTO_LIB_AESGCM
 	tristate
 	select CRYPTO_LIB_AES
@@ -70,7 +80,7 @@ config CRYPTO_LIB_CHACHA_ARCH
 	bool
 	depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS && CPU_MIPS32_R2
 	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX
 	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
@@ -130,7 +140,7 @@ config CRYPTO_LIB_NH_ARCH
 	bool
 	depends on CRYPTO_LIB_NH && !UML && !KMSAN
 	default y if ARM && KERNEL_MODE_NEON
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if X86_64
 
 config CRYPTO_LIB_POLY1305
@@ -143,7 +153,7 @@ config CRYPTO_LIB_POLY1305_ARCH
 	bool
 	depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS
 	# The PPC64 code needs to be fixed to work in softirq context.
 	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN
@@ -177,7 +187,7 @@ config CRYPTO_LIB_POLYVAL
 config CRYPTO_LIB_POLYVAL_ARCH
 	bool
 	depends on CRYPTO_LIB_POLYVAL && !UML
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if X86_64
 
 config CRYPTO_LIB_CHACHA20POLY1305
@@ -196,7 +206,7 @@ config CRYPTO_LIB_SHA1_ARCH
 	bool
 	depends on CRYPTO_LIB_SHA1 && !UML
 	default y if ARM
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if MIPS && CPU_CAVIUM_OCTEON
 	default y if PPC
 	default y if S390
@@ -252,7 +262,7 @@ config CRYPTO_LIB_SHA3
 config CRYPTO_LIB_SHA3_ARCH
 	bool
 	depends on CRYPTO_LIB_SHA3 && !UML
-	default y if ARM64 && KERNEL_MODE_NEON
+	default y if ARM64
 	default y if S390
 
 config CRYPTO_LIB_SM3
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 725eef05b7588..b74f1e1231fff 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -23,11 +23,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)
 CFLAGS_aes.o += -I$(src)/$(SRCARCH)
 
 libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o
-
-ifeq ($(CONFIG_ARM64),y)
-libaes-y += arm64/aes-cipher-core.o
-libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o
-endif
+libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \
+			  arm64/aes-ce-core.o \
+			  arm64/aes-ce.o \
+			  arm64/aes-neon.o
 
 ifeq ($(CONFIG_PPC),y)
 ifeq ($(CONFIG_SPE),y)
@@ -55,6 +54,9 @@ libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
 libaes-$(CONFIG_X86) += x86/aes-aesni.o
 endif # CONFIG_CRYPTO_LIB_AES_ARCH
 
+# clean-files must be defined unconditionally
+clean-files += powerpc/aesp8-ppc.S
+
 ################################################################################
 
 obj-$(CONFIG_CRYPTO_LIB_AESCFB)			+= libaescfb.o
@@ -297,10 +299,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y)
 endif
 
 ifeq ($(CONFIG_ARM64),y)
-libsha256-y += arm64/sha256-core.o
+libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o
 $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl
 	$(call cmd,perlasm_with_args)
-libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o
 endif
 
 libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o
@@ -327,10 +328,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y)
 endif
 
 ifeq ($(CONFIG_ARM64),y)
-libsha512-y += arm64/sha512-core.o
+libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o
 $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl
 	$(call cmd,perlasm_with_args)
-libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
 endif
 
 libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index b73e19f1bb950..ca733f15b2a8f 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -4,12 +4,15 @@
  * Copyright 2026 Google LLC
  */
 
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/aes.h>
+#include <crypto/utils.h>
 #include <linux/cache.h>
 #include <linux/crypto.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
+#include "fips.h"
 
 static const u8 ____cacheline_aligned aes_sbox[] = {
 	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
@@ -512,10 +515,235 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
 }
 EXPORT_SYMBOL(aes_decrypt);
 
-#ifdef aes_mod_init_arch
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+
+#ifndef aes_cbcmac_blocks_arch
+static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
+				   const struct aes_enckey *key, const u8 *data,
+				   size_t nblocks, bool enc_before,
+				   bool enc_after)
+{
+	return false;
+}
+#endif
+
+/* This assumes nblocks >= 1. */
+static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE],
+			      const struct aes_enckey *key, const u8 *data,
+			      size_t nblocks, bool enc_before, bool enc_after)
+{
+	if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before,
+				   enc_after))
+		return;
+
+	if (enc_before)
+		aes_encrypt(key, h, h);
+	for (; nblocks > 1; nblocks--) {
+		crypto_xor(h, data, AES_BLOCK_SIZE);
+		data += AES_BLOCK_SIZE;
+		aes_encrypt(key, h, h);
+	}
+	crypto_xor(h, data, AES_BLOCK_SIZE);
+	if (enc_after)
+		aes_encrypt(key, h, h);
+}
+
+int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key,
+			size_t key_len)
+{
+	u64 hi, lo, mask;
+	int err;
+
+	/* Prepare the AES key. */
+	err = aes_prepareenckey(&key->aes, in_key, key_len);
+	if (err)
+		return err;
+
+	/*
+	 * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block,
+	 * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128).
+	 * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation".
+	 */
+	memset(key->k_final[0].b, 0, AES_BLOCK_SIZE);
+	aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b);
+	hi = be64_to_cpu(key->k_final[0].w[0]);
+	lo = be64_to_cpu(key->k_final[0].w[1]);
+	for (int i = 0; i < 2; i++) {
+		mask = ((s64)hi >> 63) & 0x87;
+		hi = (hi << 1) ^ (lo >> 63);
+		lo = (lo << 1) ^ mask;
+		key->k_final[i].w[0] = cpu_to_be64(hi);
+		key->k_final[i].w[1] = cpu_to_be64(lo);
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(aes_cmac_preparekey);
+
+void aes_xcbcmac_preparekey(struct aes_cmac_key *key,
+			    const u8 in_key[AES_KEYSIZE_128])
+{
+	static const u8 constants[3][AES_BLOCK_SIZE] = {
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
+		{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
+	};
+	u8 new_aes_key[AES_BLOCK_SIZE];
+
+	static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128);
+	aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE);
+	aes_encrypt(&key->aes, new_aes_key, constants[0]);
+	aes_encrypt(&key->aes, key->k_final[0].b, constants[1]);
+	aes_encrypt(&key->aes, key->k_final[1].b, constants[2]);
+	aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE);
+	memzero_explicit(new_aes_key, AES_BLOCK_SIZE);
+}
+EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey);
+
+void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len)
+{
+	bool enc_before = false;
+	size_t nblocks;
+
+	if (ctx->partial_len) {
+		/* XOR data into a pending block. */
+		size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len);
+
+		crypto_xor(&ctx->h[ctx->partial_len], data, l);
+		data += l;
+		data_len -= l;
+		ctx->partial_len += l;
+		if (data_len == 0) {
+			/*
+			 * Either the pending block hasn't been filled yet, or
+			 * no more data was given so it's not yet known whether
+			 * the block is the final block.
+			 */
+			return;
+		}
+		/* Pending block has been filled and isn't the final block. */
+		enc_before = true;
+	}
+
+	nblocks = data_len / AES_BLOCK_SIZE;
+	data_len %= AES_BLOCK_SIZE;
+	if (nblocks == 0) {
+		/* 0 additional full blocks, then optionally a partial block */
+		if (enc_before)
+			aes_encrypt(&ctx->key->aes, ctx->h, ctx->h);
+		crypto_xor(ctx->h, data, data_len);
+		ctx->partial_len = data_len;
+	} else if (data_len != 0) {
+		/* 1 or more additional full blocks, then a partial block */
+		aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks,
+				  enc_before, /* enc_after= */ true);
+		data += nblocks * AES_BLOCK_SIZE;
+		crypto_xor(ctx->h, data, data_len);
+		ctx->partial_len = data_len;
+	} else {
+		/*
+		 * 1 or more additional full blocks only.  Encryption of the
+		 * last block is delayed until it's known whether it's the final
+		 * block in the message or not.
+		 */
+		aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks,
+				  enc_before, /* enc_after= */ false);
+		ctx->partial_len = AES_BLOCK_SIZE;
+	}
+}
+EXPORT_SYMBOL_GPL(aes_cmac_update);
+
+void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE])
+{
+	if (ctx->partial_len == AES_BLOCK_SIZE) {
+		/* Final block is a full block.  Use k_final[0]. */
+		crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE);
+	} else {
+		/* Final block is a partial block.  Pad, and use k_final[1]. */
+		ctx->h[ctx->partial_len] ^= 0x80;
+		crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE);
+	}
+	aes_encrypt(&ctx->key->aes, out, ctx->h);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_GPL(aes_cmac_final);
+
+void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data,
+		       size_t data_len)
+{
+	bool enc_before = false;
+	size_t nblocks;
+
+	if (ctx->partial_len) {
+		size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len);
+
+		crypto_xor(&ctx->h[ctx->partial_len], data, l);
+		data += l;
+		data_len -= l;
+		ctx->partial_len += l;
+		if (ctx->partial_len < AES_BLOCK_SIZE)
+			return;
+		enc_before = true;
+	}
+
+	nblocks = data_len / AES_BLOCK_SIZE;
+	data_len %= AES_BLOCK_SIZE;
+	if (nblocks == 0) {
+		if (enc_before)
+			aes_encrypt(ctx->key, ctx->h, ctx->h);
+	} else {
+		aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before,
+				  /* enc_after= */ true);
+		data += nblocks * AES_BLOCK_SIZE;
+	}
+	crypto_xor(ctx->h, data, data_len);
+	ctx->partial_len = data_len;
+}
+EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL");
+
+void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE])
+{
+	if (ctx->partial_len)
+		aes_encrypt(ctx->key, out, ctx->h);
+	else
+		memcpy(out, ctx->h, AES_BLOCK_SIZE);
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL");
+
+/*
+ * FIPS cryptographic algorithm self-test for AES-CMAC.  As per the FIPS 140-3
+ * Implementation Guidance, a cryptographic algorithm self-test for at least one
+ * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes
+ * is implemented.  This fulfills that requirement via AES-CMAC.
+ *
+ * This is just for FIPS.  The full tests are in the KUnit test suite.
+ */
+static void __init aes_cmac_fips_test(void)
+{
+	struct aes_cmac_key key;
+	u8 mac[AES_BLOCK_SIZE];
+
+	if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) !=
+	    0)
+		panic("aes: CMAC FIPS self-test failed (preparekey)\n");
+	aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac);
+	if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0)
+		panic("aes: CMAC FIPS self-test failed (wrong MAC)\n");
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+static inline void aes_cmac_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+
 static int __init aes_mod_init(void)
 {
+#ifdef aes_mod_init_arch
 	aes_mod_init_arch();
+#endif
+	if (fips_enabled)
+		aes_cmac_fips_test();
 	return 0;
 }
 subsys_initcall(aes_mod_init);
@@ -524,7 +752,6 @@ static void __exit aes_mod_exit(void)
 {
 }
 module_exit(aes_mod_exit);
-#endif
 
 MODULE_DESCRIPTION("AES block cipher");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
diff --git a/lib/crypto/arm64/aes-ce.S b/lib/crypto/arm64/aes-ce.S
new file mode 100644
index 0000000000000..b853e02f7b1e3
--- /dev/null
+++ b/lib/crypto/arm64/aes-ce.S
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES cipher for ARMv8 with Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+#define AES_FUNC_START(func)		SYM_FUNC_START(ce_ ## func)
+#define AES_FUNC_END(func)		SYM_FUNC_END(ce_ ## func)
+
+	.arch		armv8-a+crypto
+
+	xtsmask		.req	v16
+	cbciv		.req	v16
+	vctr		.req	v16
+
+	.macro		xts_reload_mask, tmp
+	.endm
+
+	.macro		xts_cts_skip_tw, reg, lbl
+	.endm
+
+	/* preload all round keys */
+	.macro		load_round_keys, rk, nr, tmp
+	add		\tmp, \rk, \nr, sxtw #4
+	sub		\tmp, \tmp, #160
+	ld1		{v17.4s-v20.4s}, [\rk]
+	ld1		{v21.4s-v24.4s}, [\tmp], #64
+	ld1		{v25.4s-v28.4s}, [\tmp], #64
+	ld1		{v29.4s-v31.4s}, [\tmp]
+	.endm
+
+	/* prepare for encryption with key in rk[] */
+	.macro		enc_prepare, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	/* prepare for encryption (again) but with new key in rk[] */
+	.macro		enc_switch_key, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	/* prepare for decryption with key in rk[] */
+	.macro		dec_prepare, rounds, rk, temp
+	load_round_keys	\rk, \rounds, \temp
+	.endm
+
+	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
+	aes\de		\i0\().16b, \k\().16b
+	aes\mc		\i0\().16b, \i0\().16b
+	.ifnb		\i1
+	aes\de		\i1\().16b, \k\().16b
+	aes\mc		\i1\().16b, \i1\().16b
+	.ifnb		\i3
+	aes\de		\i2\().16b, \k\().16b
+	aes\mc		\i2\().16b, \i2\().16b
+	aes\de		\i3\().16b, \k\().16b
+	aes\mc		\i3\().16b, \i3\().16b
+	.ifnb		\i4
+	aes\de		\i4\().16b, \k\().16b
+	aes\mc		\i4\().16b, \i4\().16b
+	.endif
+	.endif
+	.endif
+	.endm
+
+	/* up to 5 interleaved encryption rounds with the same round key */
+	.macro		round_Nx, enc, k, i0, i1, i2, i3, i4
+	.ifc		\enc, e
+	do_enc_Nx	e, mc, \k, \i0, \i1, \i2, \i3, \i4
+	.else
+	do_enc_Nx	d, imc, \k, \i0, \i1, \i2, \i3, \i4
+	.endif
+	.endm
+
+	/* up to 5 interleaved final rounds */
+	.macro		fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
+	aes\de		\i0\().16b, \k\().16b
+	.ifnb		\i1
+	aes\de		\i1\().16b, \k\().16b
+	.ifnb		\i3
+	aes\de		\i2\().16b, \k\().16b
+	aes\de		\i3\().16b, \k\().16b
+	.ifnb		\i4
+	aes\de		\i4\().16b, \k\().16b
+	.endif
+	.endif
+	.endif
+	eor		\i0\().16b, \i0\().16b, \k2\().16b
+	.ifnb		\i1
+	eor		\i1\().16b, \i1\().16b, \k2\().16b
+	.ifnb		\i3
+	eor		\i2\().16b, \i2\().16b, \k2\().16b
+	eor		\i3\().16b, \i3\().16b, \k2\().16b
+	.ifnb		\i4
+	eor		\i4\().16b, \i4\().16b, \k2\().16b
+	.endif
+	.endif
+	.endif
+	.endm
+
+	/* up to 5 interleaved blocks */
+	.macro		do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
+	tbz		\rounds, #2, .L\@	/* 128 bits */
+	round_Nx	\enc, v17, \i0, \i1, \i2, \i3, \i4
+	round_Nx	\enc, v18, \i0, \i1, \i2, \i3, \i4
+	tbz		\rounds, #1, .L\@	/* 192 bits */
+	round_Nx	\enc, v19, \i0, \i1, \i2, \i3, \i4
+	round_Nx	\enc, v20, \i0, \i1, \i2, \i3, \i4
+.L\@:	.irp		key, v21, v22, v23, v24, v25, v26, v27, v28, v29
+	round_Nx	\enc, \key, \i0, \i1, \i2, \i3, \i4
+	.endr
+	fin_round_Nx	\enc, v30, v31, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+	.macro		encrypt_block, in, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \in
+	.endm
+
+	.macro		encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3
+	.endm
+
+	.macro		encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+	.macro		decrypt_block, in, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \in
+	.endm
+
+	.macro		decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3
+	.endm
+
+	.macro		decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3, \i4
+	.endm
+
+#define MAX_STRIDE	5
+
+#include "aes-modes.S"
diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S
new file mode 100644
index 0000000000000..fc89cd02b642c
--- /dev/null
+++ b/lib/crypto/arm64/aes-modes.S
@@ -0,0 +1,869 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Chaining mode wrappers for AES
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+/* included by aes-ce.S and aes-neon.S */
+
+	.text
+	.align		4
+
+#ifndef MAX_STRIDE
+#define MAX_STRIDE	4
+#endif
+
+#if MAX_STRIDE == 4
+#define ST4(x...) x
+#define ST5(x...)
+#else
+#define ST4(x...)
+#define ST5(x...) x
+#endif
+
+SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
+	encrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_encrypt_block4x)
+
+SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
+	decrypt_block4x	v0, v1, v2, v3, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_decrypt_block4x)
+
+#if MAX_STRIDE == 5
+SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
+	encrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_encrypt_block5x)
+
+SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
+	decrypt_block5x	v0, v1, v2, v3, v4, w3, x2, x8, w7
+	ret
+SYM_FUNC_END(aes_decrypt_block5x)
+#endif
+
+	/*
+	 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks)
+	 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks)
+	 */
+
+AES_FUNC_START(aes_ecb_encrypt)
+	frame_push	0
+
+	enc_prepare	w3, x2, x5
+
+.LecbencloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lecbenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	ld1		{v4.16b}, [x1], #16		)
+ST5(	bl		aes_encrypt_block5x		)
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LecbencloopNx
+.Lecbenc1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lecbencout
+.Lecbencloop:
+	ld1		{v0.16b}, [x1], #16		/* get next pt block */
+	encrypt_block	v0, w3, x2, x5, w6
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lecbencloop
+.Lecbencout:
+	frame_pop
+	ret
+AES_FUNC_END(aes_ecb_encrypt)
+
+
+AES_FUNC_START(aes_ecb_decrypt)
+	frame_push	0
+
+	dec_prepare	w3, x2, x5
+
+.LecbdecloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lecbdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+ST4(	bl		aes_decrypt_block4x		)
+ST5(	ld1		{v4.16b}, [x1], #16		)
+ST5(	bl		aes_decrypt_block5x		)
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LecbdecloopNx
+.Lecbdec1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lecbdecout
+.Lecbdecloop:
+	ld1		{v0.16b}, [x1], #16		/* get next ct block */
+	decrypt_block	v0, w3, x2, x5, w6
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lecbdecloop
+.Lecbdecout:
+	frame_pop
+	ret
+AES_FUNC_END(aes_ecb_decrypt)
+
+
+	/*
+	 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks, u8 iv[])
+	 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int blocks, u8 iv[])
+	 * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
+	 *			 int rounds, int blocks, u8 iv[],
+	 *			 u32 const rk2[]);
+	 * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
+	 *			 int rounds, int blocks, u8 iv[],
+	 *			 u32 const rk2[]);
+	 */
+
+AES_FUNC_START(aes_essiv_cbc_encrypt)
+	ld1		{v4.16b}, [x5]			/* get iv */
+
+	mov		w8, #14				/* AES-256: 14 rounds */
+	enc_prepare	w8, x6, x7
+	encrypt_block	v4, w8, x6, x7, w9
+	enc_switch_key	w3, x2, x6
+	b		.Lcbcencloop4x
+
+AES_FUNC_START(aes_cbc_encrypt)
+	ld1		{v4.16b}, [x5]			/* get iv */
+	enc_prepare	w3, x2, x6
+
+.Lcbcencloop4x:
+	subs		w4, w4, #4
+	bmi		.Lcbcenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	eor		v0.16b, v0.16b, v4.16b		/* ..and xor with iv */
+	encrypt_block	v0, w3, x2, x6, w7
+	eor		v1.16b, v1.16b, v0.16b
+	encrypt_block	v1, w3, x2, x6, w7
+	eor		v2.16b, v2.16b, v1.16b
+	encrypt_block	v2, w3, x2, x6, w7
+	eor		v3.16b, v3.16b, v2.16b
+	encrypt_block	v3, w3, x2, x6, w7
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v3.16b
+	b		.Lcbcencloop4x
+.Lcbcenc1x:
+	adds		w4, w4, #4
+	beq		.Lcbcencout
+.Lcbcencloop:
+	ld1		{v0.16b}, [x1], #16		/* get next pt block */
+	eor		v4.16b, v4.16b, v0.16b		/* ..and xor with iv */
+	encrypt_block	v4, w3, x2, x6, w7
+	st1		{v4.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lcbcencloop
+.Lcbcencout:
+	st1		{v4.16b}, [x5]			/* return iv */
+	ret
+AES_FUNC_END(aes_cbc_encrypt)
+AES_FUNC_END(aes_essiv_cbc_encrypt)
+
+AES_FUNC_START(aes_essiv_cbc_decrypt)
+	ld1		{cbciv.16b}, [x5]		/* get iv */
+
+	mov		w8, #14				/* AES-256: 14 rounds */
+	enc_prepare	w8, x6, x7
+	encrypt_block	cbciv, w8, x6, x7, w9
+	b		.Lessivcbcdecstart
+
+AES_FUNC_START(aes_cbc_decrypt)
+	ld1		{cbciv.16b}, [x5]		/* get iv */
+.Lessivcbcdecstart:
+	frame_push	0
+	dec_prepare	w3, x2, x6
+
+.LcbcdecloopNx:
+	subs		w4, w4, #MAX_STRIDE
+	bmi		.Lcbcdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+#if MAX_STRIDE == 5
+	ld1		{v4.16b}, [x1], #16		/* get 1 ct block */
+	mov		v5.16b, v0.16b
+	mov		v6.16b, v1.16b
+	mov		v7.16b, v2.16b
+	bl		aes_decrypt_block5x
+	sub		x1, x1, #32
+	eor		v0.16b, v0.16b, cbciv.16b
+	eor		v1.16b, v1.16b, v5.16b
+	ld1		{v5.16b}, [x1], #16		/* reload 1 ct block */
+	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
+	eor		v2.16b, v2.16b, v6.16b
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v4.16b, v4.16b, v5.16b
+#else
+	mov		v4.16b, v0.16b
+	mov		v5.16b, v1.16b
+	mov		v6.16b, v2.16b
+	bl		aes_decrypt_block4x
+	sub		x1, x1, #16
+	eor		v0.16b, v0.16b, cbciv.16b
+	eor		v1.16b, v1.16b, v4.16b
+	ld1		{cbciv.16b}, [x1], #16		/* reload 1 ct block */
+	eor		v2.16b, v2.16b, v5.16b
+	eor		v3.16b, v3.16b, v6.16b
+#endif
+	st1		{v0.16b-v3.16b}, [x0], #64
+ST5(	st1		{v4.16b}, [x0], #16		)
+	b		.LcbcdecloopNx
+.Lcbcdec1x:
+	adds		w4, w4, #MAX_STRIDE
+	beq		.Lcbcdecout
+.Lcbcdecloop:
+	ld1		{v1.16b}, [x1], #16		/* get next ct block */
+	mov		v0.16b, v1.16b			/* ...and copy to v0 */
+	decrypt_block	v0, w3, x2, x6, w7
+	eor		v0.16b, v0.16b, cbciv.16b	/* xor with iv => pt */
+	mov		cbciv.16b, v1.16b		/* ct is next iv */
+	st1		{v0.16b}, [x0], #16
+	subs		w4, w4, #1
+	bne		.Lcbcdecloop
+.Lcbcdecout:
+	st1		{cbciv.16b}, [x5]		/* return iv */
+	frame_pop
+	ret
+AES_FUNC_END(aes_cbc_decrypt)
+AES_FUNC_END(aes_essiv_cbc_decrypt)
+
+
+	/*
+	 * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
+	 *		       int rounds, int bytes, u8 const iv[])
+	 * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
+	 *		       int rounds, int bytes, u8 const iv[])
+	 */
+
+AES_FUNC_START(aes_cbc_cts_encrypt)
+	adr_l		x8, .Lcts_permute_table
+	sub		x4, x4, #16
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	ld1		{v3.16b}, [x8]
+	ld1		{v4.16b}, [x9]
+
+	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
+	ld1		{v1.16b}, [x1]
+
+	ld1		{v5.16b}, [x5]			/* get iv */
+	enc_prepare	w3, x2, x6
+
+	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
+	tbl		v1.16b, {v1.16b}, v4.16b
+	encrypt_block	v0, w3, x2, x6, w7
+
+	eor		v1.16b, v1.16b, v0.16b
+	tbl		v0.16b, {v0.16b}, v3.16b
+	encrypt_block	v1, w3, x2, x6, w7
+
+	add		x4, x0, x4
+	st1		{v0.16b}, [x4]			/* overlapping stores */
+	st1		{v1.16b}, [x0]
+	ret
+AES_FUNC_END(aes_cbc_cts_encrypt)
+
+AES_FUNC_START(aes_cbc_cts_decrypt)
+	adr_l		x8, .Lcts_permute_table
+	sub		x4, x4, #16
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	ld1		{v3.16b}, [x8]
+	ld1		{v4.16b}, [x9]
+
+	ld1		{v0.16b}, [x1], x4		/* overlapping loads */
+	ld1		{v1.16b}, [x1]
+
+	ld1		{v5.16b}, [x5]			/* get iv */
+	dec_prepare	w3, x2, x6
+
+	decrypt_block	v0, w3, x2, x6, w7
+	tbl		v2.16b, {v0.16b}, v3.16b
+	eor		v2.16b, v2.16b, v1.16b
+
+	tbx		v0.16b, {v1.16b}, v4.16b
+	decrypt_block	v0, w3, x2, x6, w7
+	eor		v0.16b, v0.16b, v5.16b		/* xor with iv */
+
+	add		x4, x0, x4
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	st1		{v0.16b}, [x0]
+	ret
+AES_FUNC_END(aes_cbc_cts_decrypt)
+
+	.section	".rodata", "a"
+	.align		6
+.Lcts_permute_table:
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
+	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.byte		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+	.previous
+
+	/*
+	 * This macro generates the code for CTR and XCTR mode.
+	 */
+.macro ctr_encrypt xctr
+	// Arguments
+	OUT		.req x0
+	IN		.req x1
+	KEY		.req x2
+	ROUNDS_W	.req w3
+	BYTES_W		.req w4
+	IV		.req x5
+	BYTE_CTR_W 	.req w6		// XCTR only
+	// Intermediate values
+	CTR_W		.req w11	// XCTR only
+	CTR		.req x11	// XCTR only
+	IV_PART		.req x12
+	BLOCKS		.req x13
+	BLOCKS_W	.req w13
+
+	frame_push	0
+
+	enc_prepare	ROUNDS_W, KEY, IV_PART
+	ld1		{vctr.16b}, [IV]
+
+	/*
+	 * Keep 64 bits of the IV in a register.  For CTR mode this lets us
+	 * easily increment the IV.  For XCTR mode this lets us efficiently XOR
+	 * the 64-bit counter with the IV.
+	 */
+	.if \xctr
+		umov		IV_PART, vctr.d[0]
+		lsr		CTR_W, BYTE_CTR_W, #4
+	.else
+		umov		IV_PART, vctr.d[1]
+		rev		IV_PART, IV_PART
+	.endif
+
+.LctrloopNx\xctr:
+	add		BLOCKS_W, BYTES_W, #15
+	sub		BYTES_W, BYTES_W, #MAX_STRIDE << 4
+	lsr		BLOCKS_W, BLOCKS_W, #4
+	mov		w8, #MAX_STRIDE
+	cmp		BLOCKS_W, w8
+	csel		BLOCKS_W, BLOCKS_W, w8, lt
+
+	/*
+	 * Set up the counter values in v0-v{MAX_STRIDE-1}.
+	 *
+	 * If we are encrypting less than MAX_STRIDE blocks, the tail block
+	 * handling code expects the last keystream block to be in
+	 * v{MAX_STRIDE-1}.  For example: if encrypting two blocks with
+	 * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
+	 */
+	.if \xctr
+		add		CTR, CTR, BLOCKS
+	.else
+		adds		IV_PART, IV_PART, BLOCKS
+	.endif
+	mov		v0.16b, vctr.16b
+	mov		v1.16b, vctr.16b
+	mov		v2.16b, vctr.16b
+	mov		v3.16b, vctr.16b
+ST5(	mov		v4.16b, vctr.16b		)
+	.if \xctr
+		sub		x6, CTR, #MAX_STRIDE - 1
+		sub		x7, CTR, #MAX_STRIDE - 2
+		sub		x8, CTR, #MAX_STRIDE - 3
+		sub		x9, CTR, #MAX_STRIDE - 4
+ST5(		sub		x10, CTR, #MAX_STRIDE - 5	)
+		eor		x6, x6, IV_PART
+		eor		x7, x7, IV_PART
+		eor		x8, x8, IV_PART
+		eor		x9, x9, IV_PART
+ST5(		eor		x10, x10, IV_PART		)
+		mov		v0.d[0], x6
+		mov		v1.d[0], x7
+		mov		v2.d[0], x8
+		mov		v3.d[0], x9
+ST5(		mov		v4.d[0], x10			)
+	.else
+		bcs		0f
+		.subsection	1
+		/*
+		 * This subsection handles carries.
+		 *
+		 * Conditional branching here is allowed with respect to time
+		 * invariance since the branches are dependent on the IV instead
+		 * of the plaintext or key.  This code is rarely executed in
+		 * practice anyway.
+		 */
+
+		/* Apply carry to outgoing counter. */
+0:		umov		x8, vctr.d[0]
+		rev		x8, x8
+		add		x8, x8, #1
+		rev		x8, x8
+		ins		vctr.d[0], x8
+
+		/*
+		 * Apply carry to counter blocks if needed.
+		 *
+		 * Since the carry flag was set, we know 0 <= IV_PART <
+		 * MAX_STRIDE.  Using the value of IV_PART we can determine how
+		 * many counter blocks need to be updated.
+		 */
+		cbz		IV_PART, 2f
+		adr		x16, 1f
+		sub		x16, x16, IV_PART, lsl #3
+		br		x16
+		bti		c
+		mov		v0.d[0], vctr.d[0]
+		bti		c
+		mov		v1.d[0], vctr.d[0]
+		bti		c
+		mov		v2.d[0], vctr.d[0]
+		bti		c
+		mov		v3.d[0], vctr.d[0]
+ST5(		bti		c				)
+ST5(		mov		v4.d[0], vctr.d[0]		)
+1:		b		2f
+		.previous
+
+2:		rev		x7, IV_PART
+		ins		vctr.d[1], x7
+		sub		x7, IV_PART, #MAX_STRIDE - 1
+		sub		x8, IV_PART, #MAX_STRIDE - 2
+		sub		x9, IV_PART, #MAX_STRIDE - 3
+		rev		x7, x7
+		rev		x8, x8
+		mov		v1.d[1], x7
+		rev		x9, x9
+ST5(		sub		x10, IV_PART, #MAX_STRIDE - 4	)
+		mov		v2.d[1], x8
+ST5(		rev		x10, x10			)
+		mov		v3.d[1], x9
+ST5(		mov		v4.d[1], x10			)
+	.endif
+
+	/*
+	 * If there are at least MAX_STRIDE blocks left, XOR the data with
+	 * keystream and store.  Otherwise jump to tail handling.
+	 */
+	tbnz		BYTES_W, #31, .Lctrtail\xctr
+	ld1		{v5.16b-v7.16b}, [IN], #48
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	bl		aes_encrypt_block5x		)
+	eor		v0.16b, v5.16b, v0.16b
+ST4(	ld1		{v5.16b}, [IN], #16		)
+	eor		v1.16b, v6.16b, v1.16b
+ST5(	ld1		{v5.16b-v6.16b}, [IN], #32	)
+	eor		v2.16b, v7.16b, v2.16b
+	eor		v3.16b, v5.16b, v3.16b
+ST5(	eor		v4.16b, v6.16b, v4.16b		)
+	st1		{v0.16b-v3.16b}, [OUT], #64
+ST5(	st1		{v4.16b}, [OUT], #16		)
+	cbz		BYTES_W, .Lctrout\xctr
+	b		.LctrloopNx\xctr
+
+.Lctrout\xctr:
+	.if !\xctr
+		st1		{vctr.16b}, [IV] /* return next CTR value */
+	.endif
+	frame_pop
+	ret
+
+.Lctrtail\xctr:
+	/*
+	 * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
+	 *
+	 * This code expects the last keystream block to be in v{MAX_STRIDE-1}.
+	 * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
+	 * v4 should have the next two counter blocks.
+	 *
+	 * This allows us to store the ciphertext by writing to overlapping
+	 * regions of memory.  Any invalid ciphertext blocks get overwritten by
+	 * correctly computed blocks.  This approach greatly simplifies the
+	 * logic for storing the ciphertext.
+	 */
+	mov		x16, #16
+	ands		w7, BYTES_W, #0xf
+	csel		x13, x7, x16, ne
+
+ST5(	cmp		BYTES_W, #64 - (MAX_STRIDE << 4))
+ST5(	csel		x14, x16, xzr, gt		)
+	cmp		BYTES_W, #48 - (MAX_STRIDE << 4)
+	csel		x15, x16, xzr, gt
+	cmp		BYTES_W, #32 - (MAX_STRIDE << 4)
+	csel		x16, x16, xzr, gt
+	cmp		BYTES_W, #16 - (MAX_STRIDE << 4)
+
+	adr_l		x9, .Lcts_permute_table
+	add		x9, x9, x13
+	ble		.Lctrtail1x\xctr
+
+ST5(	ld1		{v5.16b}, [IN], x14		)
+	ld1		{v6.16b}, [IN], x15
+	ld1		{v7.16b}, [IN], x16
+
+ST4(	bl		aes_encrypt_block4x		)
+ST5(	bl		aes_encrypt_block5x		)
+
+	ld1		{v8.16b}, [IN], x13
+	ld1		{v9.16b}, [IN]
+	ld1		{v10.16b}, [x9]
+
+ST4(	eor		v6.16b, v6.16b, v0.16b		)
+ST4(	eor		v7.16b, v7.16b, v1.16b		)
+ST4(	tbl		v3.16b, {v3.16b}, v10.16b	)
+ST4(	eor		v8.16b, v8.16b, v2.16b		)
+ST4(	eor		v9.16b, v9.16b, v3.16b		)
+
+ST5(	eor		v5.16b, v5.16b, v0.16b		)
+ST5(	eor		v6.16b, v6.16b, v1.16b		)
+ST5(	tbl		v4.16b, {v4.16b}, v10.16b	)
+ST5(	eor		v7.16b, v7.16b, v2.16b		)
+ST5(	eor		v8.16b, v8.16b, v3.16b		)
+ST5(	eor		v9.16b, v9.16b, v4.16b		)
+
+ST5(	st1		{v5.16b}, [OUT], x14		)
+	st1		{v6.16b}, [OUT], x15
+	st1		{v7.16b}, [OUT], x16
+	add		x13, x13, OUT
+	st1		{v9.16b}, [x13]		// overlapping stores
+	st1		{v8.16b}, [OUT]
+	b		.Lctrout\xctr
+
+.Lctrtail1x\xctr:
+	/*
+	 * Handle <= 16 bytes of plaintext
+	 *
+	 * This code always reads and writes 16 bytes.  To avoid out of bounds
+	 * accesses, XCTR and CTR modes must use a temporary buffer when
+	 * encrypting/decrypting less than 16 bytes.
+	 *
+	 * This code is unusual in that it loads the input and stores the output
+	 * relative to the end of the buffers rather than relative to the start.
+	 * This causes unusual behaviour when encrypting/decrypting less than 16
+	 * bytes; the end of the data is expected to be at the end of the
+	 * temporary buffer rather than the start of the data being at the start
+	 * of the temporary buffer.
+	 */
+	sub		x8, x7, #16
+	csel		x7, x7, x8, eq
+	add		IN, IN, x7
+	add		OUT, OUT, x7
+	ld1		{v5.16b}, [IN]
+	ld1		{v6.16b}, [OUT]
+ST5(	mov		v3.16b, v4.16b			)
+	encrypt_block	v3, ROUNDS_W, KEY, x8, w7
+	ld1		{v10.16b-v11.16b}, [x9]
+	tbl		v3.16b, {v3.16b}, v10.16b
+	sshr		v11.16b, v11.16b, #7
+	eor		v5.16b, v5.16b, v3.16b
+	bif		v5.16b, v6.16b, v11.16b
+	st1		{v5.16b}, [OUT]
+	b		.Lctrout\xctr
+
+	// Arguments
+	.unreq OUT
+	.unreq IN
+	.unreq KEY
+	.unreq ROUNDS_W
+	.unreq BYTES_W
+	.unreq IV
+	.unreq BYTE_CTR_W	// XCTR only
+	// Intermediate values
+	.unreq CTR_W		// XCTR only
+	.unreq CTR		// XCTR only
+	.unreq IV_PART
+	.unreq BLOCKS
+	.unreq BLOCKS_W
+.endm
+
+	/*
+	 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int bytes, u8 ctr[])
+	 *
+	 * The input and output buffers must always be at least 16 bytes even if
+	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
+	 * accesses will occur.  The data to be encrypted/decrypted is expected
+	 * to be at the end of this 16-byte temporary buffer rather than the
+	 * start.
+	 */
+
+AES_FUNC_START(aes_ctr_encrypt)
+	ctr_encrypt 0
+AES_FUNC_END(aes_ctr_encrypt)
+
+	/*
+	 * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+	 *		   int bytes, u8 const iv[], int byte_ctr)
+	 *
+	 * The input and output buffers must always be at least 16 bytes even if
+	 * encrypting/decrypting less than 16 bytes.  Otherwise out of bounds
+	 * accesses will occur.  The data to be encrypted/decrypted is expected
+	 * to be at the end of this 16-byte temporary buffer rather than the
+	 * start.
+	 */
+
+AES_FUNC_START(aes_xctr_encrypt)
+	ctr_encrypt 1
+AES_FUNC_END(aes_xctr_encrypt)
+
+
+	/*
+	 * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
+	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
+	 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
+	 *		   int bytes, u8 const rk2[], u8 iv[], int first)
+	 */
+
+	.macro		next_tweak, out, in, tmp
+	sshr		\tmp\().2d,  \in\().2d,   #63
+	and		\tmp\().16b, \tmp\().16b, xtsmask.16b
+	add		\out\().2d,  \in\().2d,   \in\().2d
+	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
+	eor		\out\().16b, \out\().16b, \tmp\().16b
+	.endm
+
+	.macro		xts_load_mask, tmp
+	movi		xtsmask.2s, #0x1
+	movi		\tmp\().2s, #0x87
+	uzp1		xtsmask.4s, xtsmask.4s, \tmp\().4s
+	.endm
+
+AES_FUNC_START(aes_xts_encrypt)
+	frame_push	0
+
+	ld1		{v4.16b}, [x6]
+	xts_load_mask	v8
+	cbz		w7, .Lxtsencnotfirst
+
+	enc_prepare	w3, x5, x8
+	xts_cts_skip_tw	w7, .LxtsencNx
+	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
+	enc_switch_key	w3, x2, x8
+	b		.LxtsencNx
+
+.Lxtsencnotfirst:
+	enc_prepare	w3, x2, x8
+.LxtsencloopNx:
+	next_tweak	v4, v4, v8
+.LxtsencNx:
+	subs		w4, w4, #64
+	bmi		.Lxtsenc1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 pt blocks */
+	next_tweak	v5, v4, v8
+	eor		v0.16b, v0.16b, v4.16b
+	next_tweak	v6, v5, v8
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	next_tweak	v7, v6, v8
+	eor		v3.16b, v3.16b, v7.16b
+	bl		aes_encrypt_block4x
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v0.16b, v0.16b, v4.16b
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v7.16b
+	cbz		w4, .Lxtsencret
+	xts_reload_mask	v8
+	b		.LxtsencloopNx
+.Lxtsenc1x:
+	adds		w4, w4, #64
+	beq		.Lxtsencout
+	subs		w4, w4, #16
+	bmi		.LxtsencctsNx
+.Lxtsencloop:
+	ld1		{v0.16b}, [x1], #16
+.Lxtsencctsout:
+	eor		v0.16b, v0.16b, v4.16b
+	encrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v4.16b
+	cbz		w4, .Lxtsencout
+	subs		w4, w4, #16
+	next_tweak	v4, v4, v8
+	bmi		.Lxtsenccts
+	st1		{v0.16b}, [x0], #16
+	b		.Lxtsencloop
+.Lxtsencout:
+	st1		{v0.16b}, [x0]
+.Lxtsencret:
+	st1		{v4.16b}, [x6]
+	frame_pop
+	ret
+
+.LxtsencctsNx:
+	mov		v0.16b, v3.16b
+	sub		x0, x0, #16
+.Lxtsenccts:
+	adr_l		x8, .Lcts_permute_table
+
+	add		x1, x1, w4, sxtw	/* rewind input pointer */
+	add		w4, w4, #16		/* # bytes in final block */
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	add		x4, x0, x4		/* output address of final block */
+
+	ld1		{v1.16b}, [x1]		/* load final block */
+	ld1		{v2.16b}, [x8]
+	ld1		{v3.16b}, [x9]
+
+	tbl		v2.16b, {v0.16b}, v2.16b
+	tbx		v0.16b, {v1.16b}, v3.16b
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	mov		w4, wzr
+	b		.Lxtsencctsout
+AES_FUNC_END(aes_xts_encrypt)
+
+AES_FUNC_START(aes_xts_decrypt)
+	frame_push	0
+
+	/* subtract 16 bytes if we are doing CTS */
+	sub		w8, w4, #0x10
+	tst		w4, #0xf
+	csel		w4, w4, w8, eq
+
+	ld1		{v4.16b}, [x6]
+	xts_load_mask	v8
+	xts_cts_skip_tw	w7, .Lxtsdecskiptw
+	cbz		w7, .Lxtsdecnotfirst
+
+	enc_prepare	w3, x5, x8
+	encrypt_block	v4, w3, x5, x8, w7		/* first tweak */
+.Lxtsdecskiptw:
+	dec_prepare	w3, x2, x8
+	b		.LxtsdecNx
+
+.Lxtsdecnotfirst:
+	dec_prepare	w3, x2, x8
+.LxtsdecloopNx:
+	next_tweak	v4, v4, v8
+.LxtsdecNx:
+	subs		w4, w4, #64
+	bmi		.Lxtsdec1x
+	ld1		{v0.16b-v3.16b}, [x1], #64	/* get 4 ct blocks */
+	next_tweak	v5, v4, v8
+	eor		v0.16b, v0.16b, v4.16b
+	next_tweak	v6, v5, v8
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	next_tweak	v7, v6, v8
+	eor		v3.16b, v3.16b, v7.16b
+	bl		aes_decrypt_block4x
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v0.16b, v0.16b, v4.16b
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	st1		{v0.16b-v3.16b}, [x0], #64
+	mov		v4.16b, v7.16b
+	cbz		w4, .Lxtsdecout
+	xts_reload_mask	v8
+	b		.LxtsdecloopNx
+.Lxtsdec1x:
+	adds		w4, w4, #64
+	beq		.Lxtsdecout
+	subs		w4, w4, #16
+.Lxtsdecloop:
+	ld1		{v0.16b}, [x1], #16
+	bmi		.Lxtsdeccts
+.Lxtsdecctsout:
+	eor		v0.16b, v0.16b, v4.16b
+	decrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v4.16b
+	st1		{v0.16b}, [x0], #16
+	cbz		w4, .Lxtsdecout
+	subs		w4, w4, #16
+	next_tweak	v4, v4, v8
+	b		.Lxtsdecloop
+.Lxtsdecout:
+	st1		{v4.16b}, [x6]
+	frame_pop
+	ret
+
+.Lxtsdeccts:
+	adr_l		x8, .Lcts_permute_table
+
+	add		x1, x1, w4, sxtw	/* rewind input pointer */
+	add		w4, w4, #16		/* # bytes in final block */
+	add		x9, x8, #32
+	add		x8, x8, x4
+	sub		x9, x9, x4
+	add		x4, x0, x4		/* output address of final block */
+
+	next_tweak	v5, v4, v8
+
+	ld1		{v1.16b}, [x1]		/* load final block */
+	ld1		{v2.16b}, [x8]
+	ld1		{v3.16b}, [x9]
+
+	eor		v0.16b, v0.16b, v5.16b
+	decrypt_block	v0, w3, x2, x8, w7
+	eor		v0.16b, v0.16b, v5.16b
+
+	tbl		v2.16b, {v0.16b}, v2.16b
+	tbx		v0.16b, {v1.16b}, v3.16b
+
+	st1		{v2.16b}, [x4]			/* overlapping stores */
+	mov		w4, wzr
+	b		.Lxtsdecctsout
+AES_FUNC_END(aes_xts_decrypt)
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+	/*
+	 * size_t aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+	 *			 size_t blocks, u8 dg[], int enc_before,
+	 *			 int enc_after);
+	 */
+AES_FUNC_START(aes_mac_update)
+	ld1		{v0.16b}, [x4]			/* get dg */
+	enc_prepare	w2, x1, x7
+	cbz		w5, .Lmacloop4x
+
+	encrypt_block	v0, w2, x1, x7, w8
+
+.Lmacloop4x:
+	subs		x3, x3, #4
+	bmi		.Lmac1x
+	ld1		{v1.16b-v4.16b}, [x0], #64	/* get next pt block */
+	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v2.16b
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v3.16b
+	encrypt_block	v0, w2, x1, x7, w8
+	eor		v0.16b, v0.16b, v4.16b
+	cmp		x3, xzr
+	csinv		w5, w6, wzr, eq
+	cbz		w5, .Lmacout
+	encrypt_block	v0, w2, x1, x7, w8
+	st1		{v0.16b}, [x4]			/* return dg */
+	cond_yield	.Lmacout, x7, x8
+	b		.Lmacloop4x
+.Lmac1x:
+	add		x3, x3, #4
+.Lmacloop:
+	cbz		x3, .Lmacout
+	ld1		{v1.16b}, [x0], #16		/* get next pt block */
+	eor		v0.16b, v0.16b, v1.16b		/* ..and xor with dg */
+
+	subs		x3, x3, #1
+	csinv		w5, w6, wzr, eq
+	cbz		w5, .Lmacout
+
+.Lmacenc:
+	encrypt_block	v0, w2, x1, x7, w8
+	b		.Lmacloop
+
+.Lmacout:
+	st1		{v0.16b}, [x4]			/* return dg */
+	mov		x0, x3
+	ret
+AES_FUNC_END(aes_mac_update)
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
diff --git a/lib/crypto/arm64/aes-neon.S b/lib/crypto/arm64/aes-neon.S
new file mode 100644
index 0000000000000..f37b1dbd887ff
--- /dev/null
+++ b/lib/crypto/arm64/aes-neon.S
@@ -0,0 +1,250 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * AES cipher for ARMv8 NEON
+ *
+ * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+#define AES_FUNC_START(func)		SYM_FUNC_START(neon_ ## func)
+#define AES_FUNC_END(func)		SYM_FUNC_END(neon_ ## func)
+
+	xtsmask		.req	v7
+	cbciv		.req	v7
+	vctr		.req	v4
+
+	.macro		xts_reload_mask, tmp
+	xts_load_mask	\tmp
+	.endm
+
+	/* special case for the neon-bs driver calling into this one for CTS */
+	.macro		xts_cts_skip_tw, reg, lbl
+	tbnz		\reg, #1, \lbl
+	.endm
+
+	/* multiply by polynomial 'x' in GF(2^8) */
+	.macro		mul_by_x, out, in, temp, const
+	sshr		\temp, \in, #7
+	shl		\out, \in, #1
+	and		\temp, \temp, \const
+	eor		\out, \out, \temp
+	.endm
+
+	/* multiply by polynomial 'x^2' in GF(2^8) */
+	.macro		mul_by_x2, out, in, temp, const
+	ushr		\temp, \in, #6
+	shl		\out, \in, #2
+	pmul		\temp, \temp, \const
+	eor		\out, \out, \temp
+	.endm
+
+	/* preload the entire Sbox */
+	.macro		prepare, sbox, shiftrows, temp
+	movi		v12.16b, #0x1b
+	ldr_l		q13, \shiftrows, \temp
+	ldr_l		q14, .Lror32by8, \temp
+	adr_l		\temp, \sbox
+	ld1		{v16.16b-v19.16b}, [\temp], #64
+	ld1		{v20.16b-v23.16b}, [\temp], #64
+	ld1		{v24.16b-v27.16b}, [\temp], #64
+	ld1		{v28.16b-v31.16b}, [\temp]
+	.endm
+
+	/* do preload for encryption */
+	.macro		enc_prepare, ignore0, ignore1, temp
+	prepare		crypto_aes_sbox, .LForward_ShiftRows, \temp
+	.endm
+
+	.macro		enc_switch_key, ignore0, ignore1, temp
+	/* do nothing */
+	.endm
+
+	/* do preload for decryption */
+	.macro		dec_prepare, ignore0, ignore1, temp
+	prepare		crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
+	.endm
+
+	/* apply SubBytes transformation using the preloaded Sbox */
+	.macro		sub_bytes, in
+	sub		v9.16b, \in\().16b, v15.16b
+	tbl		\in\().16b, {v16.16b-v19.16b}, \in\().16b
+	sub		v10.16b, v9.16b, v15.16b
+	tbx		\in\().16b, {v20.16b-v23.16b}, v9.16b
+	sub		v11.16b, v10.16b, v15.16b
+	tbx		\in\().16b, {v24.16b-v27.16b}, v10.16b
+	tbx		\in\().16b, {v28.16b-v31.16b}, v11.16b
+	.endm
+
+	/* apply MixColumns transformation */
+	.macro		mix_columns, in, enc
+	.if		\enc == 0
+	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
+	mul_by_x2	v8.16b, \in\().16b, v9.16b, v12.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	rev32		v8.8h, v8.8h
+	eor		\in\().16b, \in\().16b, v8.16b
+	.endif
+
+	mul_by_x	v9.16b, \in\().16b, v8.16b, v12.16b
+	rev32		v8.8h, \in\().8h
+	eor		v8.16b, v8.16b, v9.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	tbl		\in\().16b, {\in\().16b}, v14.16b
+	eor		\in\().16b, \in\().16b, v8.16b
+	.endm
+
+	.macro		do_block, enc, in, rounds, rk, rkp, i
+	ld1		{v15.4s}, [\rk]
+	add		\rkp, \rk, #16
+	mov		\i, \rounds
+.La\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
+	movi		v15.16b, #0x40
+	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
+	sub_bytes	\in
+	sub		\i, \i, #1
+	ld1		{v15.4s}, [\rkp], #16
+	cbz		\i, .Lb\@
+	mix_columns	\in, \enc
+	b		.La\@
+.Lb\@:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
+	.endm
+
+	.macro		encrypt_block, in, rounds, rk, rkp, i
+	do_block	1, \in, \rounds, \rk, \rkp, \i
+	.endm
+
+	.macro		decrypt_block, in, rounds, rk, rkp, i
+	do_block	0, \in, \rounds, \rk, \rkp, \i
+	.endm
+
+	/*
+	 * Interleaved versions: functionally equivalent to the
+	 * ones above, but applied to AES states in parallel.
+	 */
+
+	.macro		sub_bytes_4x, in0, in1, in2, in3
+	sub		v8.16b, \in0\().16b, v15.16b
+	tbl		\in0\().16b, {v16.16b-v19.16b}, \in0\().16b
+	sub		v9.16b, \in1\().16b, v15.16b
+	tbl		\in1\().16b, {v16.16b-v19.16b}, \in1\().16b
+	sub		v10.16b, \in2\().16b, v15.16b
+	tbl		\in2\().16b, {v16.16b-v19.16b}, \in2\().16b
+	sub		v11.16b, \in3\().16b, v15.16b
+	tbl		\in3\().16b, {v16.16b-v19.16b}, \in3\().16b
+	tbx		\in0\().16b, {v20.16b-v23.16b}, v8.16b
+	tbx		\in1\().16b, {v20.16b-v23.16b}, v9.16b
+	sub		v8.16b, v8.16b, v15.16b
+	tbx		\in2\().16b, {v20.16b-v23.16b}, v10.16b
+	sub		v9.16b, v9.16b, v15.16b
+	tbx		\in3\().16b, {v20.16b-v23.16b}, v11.16b
+	sub		v10.16b, v10.16b, v15.16b
+	tbx		\in0\().16b, {v24.16b-v27.16b}, v8.16b
+	sub		v11.16b, v11.16b, v15.16b
+	tbx		\in1\().16b, {v24.16b-v27.16b}, v9.16b
+	sub		v8.16b, v8.16b, v15.16b
+	tbx		\in2\().16b, {v24.16b-v27.16b}, v10.16b
+	sub		v9.16b, v9.16b, v15.16b
+	tbx		\in3\().16b, {v24.16b-v27.16b}, v11.16b
+	sub		v10.16b, v10.16b, v15.16b
+	tbx		\in0\().16b, {v28.16b-v31.16b}, v8.16b
+	sub		v11.16b, v11.16b, v15.16b
+	tbx		\in1\().16b, {v28.16b-v31.16b}, v9.16b
+	tbx		\in2\().16b, {v28.16b-v31.16b}, v10.16b
+	tbx		\in3\().16b, {v28.16b-v31.16b}, v11.16b
+	.endm
+
+	.macro		mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
+	sshr		\tmp0\().16b, \in0\().16b, #7
+	shl		\out0\().16b, \in0\().16b, #1
+	sshr		\tmp1\().16b, \in1\().16b, #7
+	and		\tmp0\().16b, \tmp0\().16b, \const\().16b
+	shl		\out1\().16b, \in1\().16b, #1
+	and		\tmp1\().16b, \tmp1\().16b, \const\().16b
+	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
+	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
+	.endm
+
+	.macro		mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
+	ushr		\tmp0\().16b, \in0\().16b, #6
+	shl		\out0\().16b, \in0\().16b, #2
+	ushr		\tmp1\().16b, \in1\().16b, #6
+	pmul		\tmp0\().16b, \tmp0\().16b, \const\().16b
+	shl		\out1\().16b, \in1\().16b, #2
+	pmul		\tmp1\().16b, \tmp1\().16b, \const\().16b
+	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
+	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
+	.endm
+
+	.macro		mix_columns_2x, in0, in1, enc
+	.if		\enc == 0
+	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
+	mul_by_x2_2x	v8, v9, \in0, \in1, v10, v11, v12
+	eor		\in0\().16b, \in0\().16b, v8.16b
+	rev32		v8.8h, v8.8h
+	eor		\in1\().16b, \in1\().16b, v9.16b
+	rev32		v9.8h, v9.8h
+	eor		\in0\().16b, \in0\().16b, v8.16b
+	eor		\in1\().16b, \in1\().16b, v9.16b
+	.endif
+
+	mul_by_x_2x	v8, v9, \in0, \in1, v10, v11, v12
+	rev32		v10.8h, \in0\().8h
+	rev32		v11.8h, \in1\().8h
+	eor		v10.16b, v10.16b, v8.16b
+	eor		v11.16b, v11.16b, v9.16b
+	eor		\in0\().16b, \in0\().16b, v10.16b
+	eor		\in1\().16b, \in1\().16b, v11.16b
+	tbl		\in0\().16b, {\in0\().16b}, v14.16b
+	tbl		\in1\().16b, {\in1\().16b}, v14.16b
+	eor		\in0\().16b, \in0\().16b, v10.16b
+	eor		\in1\().16b, \in1\().16b, v11.16b
+	.endm
+
+	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
+	ld1		{v15.4s}, [\rk]
+	add		\rkp, \rk, #16
+	mov		\i, \rounds
+.La\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
+	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
+	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
+	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
+	movi		v15.16b, #0x40
+	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
+	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
+	sub_bytes_4x	\in0, \in1, \in2, \in3
+	sub		\i, \i, #1
+	ld1		{v15.4s}, [\rkp], #16
+	cbz		\i, .Lb\@
+	mix_columns_2x	\in0, \in1, \enc
+	mix_columns_2x	\in2, \in3, \enc
+	b		.La\@
+.Lb\@:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
+	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
+	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
+	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
+	.endm
+
+	.macro		encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
+	do_block_4x	1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
+	.endm
+
+	.macro		decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
+	do_block_4x	0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
+	.endm
+
+#include "aes-modes.S"
+
+	.section	".rodata", "a"
+	.align		4
+.LForward_ShiftRows:
+	.octa		0x0b06010c07020d08030e09040f0a0500
+
+.LReverse_ShiftRows:
+	.octa		0x0306090c0f0205080b0e0104070a0d00
+
+.Lror32by8:
+	.octa		0x0c0f0e0d080b0a090407060500030201
diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h
index 63eea6271ef91..135d3324a30a6 100644
--- a/lib/crypto/arm64/aes.h
+++ b/lib/crypto/arm64/aes.h
@@ -11,6 +11,7 @@
 #include <linux/unaligned.h>
 #include <linux/cpufeature.h>
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
 
 struct aes_block {
@@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
 asmlinkage u32 __aes_ce_sub(u32 l);
 asmlinkage void __aes_ce_invert(struct aes_block *out,
 				const struct aes_block *in);
+asmlinkage size_t neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
+				      size_t blocks, u8 dg[], int enc_before,
+				      int enc_after);
 
 /*
  * Expand an AES key using the crypto extensions if supported and usable or
@@ -48,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
 	struct aes_block *key_enc, *key_dec;
 	int i, j;
 
-	if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) ||
-	    !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
+	if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
 		aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
 		return;
 	}
@@ -126,12 +129,40 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 }
 EXPORT_SYMBOL(ce_aes_expandkey);
 
+EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
+
+EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
+EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
+#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM)
+EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL");
+#endif
+
 static void aes_encrypt_arch(const struct aes_enckey *key,
 			     u8 out[AES_BLOCK_SIZE],
 			     const u8 in[AES_BLOCK_SIZE])
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
 		scoped_ksimd()
 			__aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
 	} else {
@@ -143,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key,
 			     u8 out[AES_BLOCK_SIZE],
 			     const u8 in[AES_BLOCK_SIZE])
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_aes) && likely(may_use_simd())) {
+	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
 		scoped_ksimd()
 			__aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
 					 key->nrounds);
@@ -154,11 +184,45 @@ static void aes_decrypt_arch(const struct aes_key *key,
 	}
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
+#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch
+static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
+				   const struct aes_enckey *key, const u8 *data,
+				   size_t nblocks, bool enc_before,
+				   bool enc_after)
+{
+	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
+		do {
+			size_t rem;
+
+			scoped_ksimd() {
+				if (static_branch_likely(&have_aes))
+					rem = ce_aes_mac_update(
+						data, key->k.rndkeys,
+						key->nrounds, nblocks, h,
+						enc_before, enc_after);
+				else
+					rem = neon_aes_mac_update(
+						data, key->k.rndkeys,
+						key->nrounds, nblocks, h,
+						enc_before, enc_after);
+			}
+			data += (nblocks - rem) * AES_BLOCK_SIZE;
+			nblocks = rem;
+			enc_before = false;
+		} while (nblocks);
+		return true;
+	}
+	return false;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
+
 #define aes_mod_init_arch aes_mod_init_arch
 static void aes_mod_init_arch(void)
 {
-	if (cpu_have_named_feature(AES))
-		static_branch_enable(&have_aes);
+	if (cpu_have_named_feature(ASIMD)) {
+		static_branch_enable(&have_neon);
+		if (cpu_have_named_feature(AES))
+			static_branch_enable(&have_aes);
+	}
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h
index 568dff0f276af..1fad3d7baa9a2 100644
--- a/lib/crypto/arm64/sha256.h
+++ b/lib/crypto/arm64/sha256.h
@@ -20,8 +20,7 @@ asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state,
 static void sha256_blocks(struct sha256_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_neon) && likely(may_use_simd())) {
+	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
 		if (static_branch_likely(&have_ce)) {
 			do {
 				size_t rem;
@@ -61,8 +60,7 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
 	 * Further limit len to 65536 to avoid spending too long with preemption
 	 * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
 	 */
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
+	if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
 	    len <= 65536 && likely(may_use_simd())) {
 		scoped_ksimd()
 			sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
@@ -78,7 +76,6 @@ static bool sha256_finup_2x_is_optimized_arch(void)
 	return static_key_enabled(&have_ce);
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
 #define sha256_mod_init_arch sha256_mod_init_arch
 static void sha256_mod_init_arch(void)
 {
@@ -88,4 +85,3 @@ static void sha256_mod_init_arch(void)
 			static_branch_enable(&have_ce);
 	}
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h
index 7eb7ef04d2687..d978c4d07e905 100644
--- a/lib/crypto/arm64/sha512.h
+++ b/lib/crypto/arm64/sha512.h
@@ -18,8 +18,7 @@ asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state,
 static void sha512_blocks(struct sha512_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
-	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
-	    static_branch_likely(&have_sha512_insns) &&
+	if (static_branch_likely(&have_sha512_insns) &&
 	    likely(may_use_simd())) {
 		do {
 			size_t rem;
@@ -35,11 +34,9 @@ static void sha512_blocks(struct sha512_block_state *state,
 	}
 }
 
-#ifdef CONFIG_KERNEL_MODE_NEON
 #define sha512_mod_init_arch sha512_mod_init_arch
 static void sha512_mod_init_arch(void)
 {
 	if (cpu_have_named_feature(SHA512))
 		static_branch_enable(&have_sha512_insns);
 }
-#endif /* CONFIG_KERNEL_MODE_NEON */
diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h
index 023410c2e0dbc..9fc49747db645 100644
--- a/lib/crypto/fips.h
+++ b/lib/crypto/fips.h
@@ -43,3 +43,8 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = {
 	0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00,
 	0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82,
 };
+
+static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = {
+	0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6,
+	0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18,
+};
diff --git a/lib/crypto/mpi/longlong.h b/lib/crypto/mpi/longlong.h
index b6fa1d08fb55b..6d31c3a729f12 100644
--- a/lib/crypto/mpi/longlong.h
+++ b/lib/crypto/mpi/longlong.h
@@ -66,12 +66,12 @@
  * denominator).  Like udiv_qrnnd but the numbers are signed.  The quotient
  * is rounded towards 0.
  *
- * 5) count_leading_zeros(count, x) counts the number of zero-bits from the
+ * 5) count_leading_zeros(x) counts the number of zero-bits from the
  * msb to the first non-zero bit in the UWtype X.  This is the number of
- * steps X needs to be shifted left to set the msb.  Undefined for X == 0,
- * unless the symbol COUNT_LEADING_ZEROS_0 is defined to some value.
+ * steps X needs to be shifted left to set the msb.
+ * count_leading_zeros(0) == BITS_PER_LONG
  *
- * 6) count_trailing_zeros(count, x) like count_leading_zeros, but counts
+ * 6) count_trailing_zeros() like count_leading_zeros(), but counts
  * from the least significant end.
  *
  * 7) add_ssaaaa(high_sum, low_sum, high_addend_1, low_addend_1,
diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
index 0de289b429a95..42e1770e1883b 100644
--- a/lib/crypto/tests/Kconfig
+++ b/lib/crypto/tests/Kconfig
@@ -1,9 +1,18 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST
+	tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message
+	  authentication codes.
+
 config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_BLAKE2B
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the BLAKE2b cryptographic hash function.
@@ -11,7 +20,7 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
 config CRYPTO_LIB_BLAKE2S_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS
 	depends on KUNIT
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	# No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't
 	# exist; the BLAKE2s code is always built-in for the /dev/random driver.
@@ -21,7 +30,7 @@ config CRYPTO_LIB_BLAKE2S_KUNIT_TEST
 config CRYPTO_LIB_CURVE25519_KUNIT_TEST
 	tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_CURVE25519
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the Curve25519 Diffie-Hellman function.
@@ -29,7 +38,7 @@ config CRYPTO_LIB_CURVE25519_KUNIT_TEST
 config CRYPTO_LIB_MD5_KUNIT_TEST
 	tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_MD5
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the MD5 cryptographic hash function and its
@@ -38,7 +47,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST
 config CRYPTO_LIB_MLDSA_KUNIT_TEST
 	tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_MLDSA
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the ML-DSA digital signature algorithm.
@@ -46,14 +55,14 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST
 config CRYPTO_LIB_NH_KUNIT_TEST
 	tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_NH
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	help
 	  KUnit tests for the NH almost-universal hash function.
 
 config CRYPTO_LIB_POLY1305_KUNIT_TEST
 	tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_POLY1305
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the Poly1305 library functions.
@@ -61,7 +70,7 @@ config CRYPTO_LIB_POLY1305_KUNIT_TEST
 config CRYPTO_LIB_POLYVAL_KUNIT_TEST
 	tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_POLYVAL
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the POLYVAL library functions.
@@ -69,7 +78,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST
 config CRYPTO_LIB_SHA1_KUNIT_TEST
 	tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA1
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-1 cryptographic hash function and its
@@ -80,7 +89,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST
 config CRYPTO_LIB_SHA256_KUNIT_TEST
 	tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA256
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions
@@ -91,7 +100,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST
 config CRYPTO_LIB_SHA512_KUNIT_TEST
 	tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA512
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions
@@ -100,13 +109,37 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST
 config CRYPTO_LIB_SHA3_KUNIT_TEST
 	tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_SHA3
-	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	default KUNIT_ALL_TESTS
 	select CRYPTO_LIB_BENCHMARK_VISIBLE
 	help
 	  KUnit tests for the SHA3 cryptographic hash and XOF functions,
 	  including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and
 	  SHAKE256.
 
+config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT
+	tristate "Enable all crypto library code for KUnit tests"
+	depends on KUNIT
+	select CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_BLAKE2B
+	select CRYPTO_LIB_CURVE25519
+	select CRYPTO_LIB_MD5
+	select CRYPTO_LIB_MLDSA
+	select CRYPTO_LIB_NH
+	select CRYPTO_LIB_POLY1305
+	select CRYPTO_LIB_POLYVAL
+	select CRYPTO_LIB_SHA1
+	select CRYPTO_LIB_SHA256
+	select CRYPTO_LIB_SHA512
+	select CRYPTO_LIB_SHA3
+	help
+	  Enable all the crypto library code that has KUnit tests.
+
+	  Enable this only if you'd like to test all the crypto library code,
+	  even code that wouldn't otherwise need to be built.
+
+	  You'll still need to enable the tests themselves, either individually
+	  or using KUNIT_ALL_TESTS.
+
 config CRYPTO_LIB_BENCHMARK_VISIBLE
 	bool
 
diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile
index f4262379f56cf..f864e0ffbee4d 100644
--- a/lib/crypto/tests/Makefile
+++ b/lib/crypto/tests/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o
diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h
new file mode 100644
index 0000000000000..8177862ba5a3b
--- /dev/null
+++ b/lib/crypto/tests/aes-cmac-testvecs.h
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */
+
+static const struct {
+	size_t data_len;
+	u8 digest[AES_BLOCK_SIZE];
+} hash_testvecs[] = {
+	{
+		.data_len = 0,
+		.digest = {
+			0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49,
+			0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b,
+		},
+	},
+	{
+		.data_len = 1,
+		.digest = {
+			0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f,
+			0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea,
+		},
+	},
+	{
+		.data_len = 2,
+		.digest = {
+			0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a,
+			0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19,
+		},
+	},
+	{
+		.data_len = 3,
+		.digest = {
+			0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6,
+			0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4,
+		},
+	},
+	{
+		.data_len = 16,
+		.digest = {
+			0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2,
+			0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1,
+		},
+	},
+	{
+		.data_len = 32,
+		.digest = {
+			0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0,
+			0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99,
+		},
+	},
+	{
+		.data_len = 48,
+		.digest = {
+			0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a,
+			0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87,
+		},
+	},
+	{
+		.data_len = 49,
+		.digest = {
+			0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e,
+			0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58,
+		},
+	},
+	{
+		.data_len = 63,
+		.digest = {
+			0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f,
+			0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf,
+		},
+	},
+	{
+		.data_len = 64,
+		.digest = {
+			0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca,
+			0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb,
+		},
+	},
+	{
+		.data_len = 65,
+		.digest = {
+			0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe,
+			0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04,
+		},
+	},
+	{
+		.data_len = 127,
+		.digest = {
+			0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef,
+			0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71,
+		},
+	},
+	{
+		.data_len = 128,
+		.digest = {
+			0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e,
+			0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74,
+		},
+	},
+	{
+		.data_len = 129,
+		.digest = {
+			0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87,
+			0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20,
+		},
+	},
+	{
+		.data_len = 256,
+		.digest = {
+			0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a,
+			0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf,
+		},
+	},
+	{
+		.data_len = 511,
+		.digest = {
+			0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f,
+			0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f,
+		},
+	},
+	{
+		.data_len = 513,
+		.digest = {
+			0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e,
+			0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a,
+		},
+	},
+	{
+		.data_len = 1000,
+		.digest = {
+			0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b,
+			0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a,
+		},
+	},
+	{
+		.data_len = 3333,
+		.digest = {
+			0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d,
+			0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce,
+		},
+	},
+	{
+		.data_len = 4096,
+		.digest = {
+			0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09,
+			0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec,
+		},
+	},
+	{
+		.data_len = 4128,
+		.digest = {
+			0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5,
+			0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a,
+		},
+	},
+	{
+		.data_len = 4160,
+		.digest = {
+			0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07,
+			0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69,
+		},
+	},
+	{
+		.data_len = 4224,
+		.digest = {
+			0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95,
+			0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15,
+		},
+	},
+	{
+		.data_len = 16384,
+		.digest = {
+			0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf,
+			0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6,
+		},
+	},
+};
+
+static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = {
+	0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c,
+	0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07,
+};
diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c
new file mode 100644
index 0000000000000..ae3745212f03f
--- /dev/null
+++ b/lib/crypto/tests/aes_cbc_macs_kunit.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2026 Google LLC
+ */
+#include <crypto/aes-cbc-macs.h>
+#include "aes-cmac-testvecs.h"
+
+/*
+ * A fixed key used when presenting AES-CMAC as an unkeyed hash function in
+ * order to reuse hash-test-template.h.  At the beginning of the test suite,
+ * this is initialized to a key prepared from bytes generated from a fixed seed.
+ */
+static struct aes_cmac_key test_key;
+
+static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx)
+{
+	aes_cmac_init(ctx, &test_key);
+}
+
+static void aes_cmac_withtestkey(const u8 *data, size_t data_len,
+				 u8 out[AES_BLOCK_SIZE])
+{
+	aes_cmac(&test_key, data, data_len, out);
+}
+
+#define HASH aes_cmac_withtestkey
+#define HASH_CTX aes_cmac_ctx
+#define HASH_SIZE AES_BLOCK_SIZE
+#define HASH_INIT aes_cmac_init_withtestkey
+#define HASH_UPDATE aes_cmac_update
+#define HASH_FINAL aes_cmac_final
+#include "hash-test-template.h"
+
+static int aes_cbc_macs_suite_init(struct kunit_suite *suite)
+{
+	u8 raw_key[AES_KEYSIZE_256];
+	int err;
+
+	rand_bytes_seeded_from_len(raw_key, sizeof(raw_key));
+	err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key));
+	if (err)
+		return err;
+	return hash_suite_init(suite);
+}
+
+static void aes_cbc_macs_suite_exit(struct kunit_suite *suite)
+{
+	hash_suite_exit(suite);
+}
+
+/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */
+static void test_aes_cmac_rfc4493(struct kunit *test)
+{
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6,
+		0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
+	};
+	static const struct {
+		size_t data_len;
+		const u8 data[40];
+		const u8 mac[AES_BLOCK_SIZE];
+	} testvecs[] = {
+		{
+			/* Example 1 from RFC 4493 */
+			.data_len = 0,
+			.mac = {
+				0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28,
+				0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46,
+			},
+
+		},
+		{
+			/* Example 2 from RFC 4493 */
+			.data = {
+				0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
+				0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
+			},
+			.data_len = 16,
+			.mac = {
+				0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44,
+				0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c,
+			},
+		},
+		{
+			/* Example 3 from RFC 4493 */
+			.data = {
+				0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
+				0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
+				0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
+				0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
+				0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
+			},
+			.data_len = 40,
+			.mac = {
+				0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30,
+				0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27,
+			},
+		},
+	};
+	struct aes_cmac_key key;
+	int err;
+
+	err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key));
+	KUNIT_ASSERT_EQ(test, err, 0);
+
+	for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) {
+		u8 mac[AES_BLOCK_SIZE];
+
+		aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac);
+		KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE);
+	}
+}
+
+/*
+ * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566.
+ *
+ * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC
+ * implementation is well covered by the AES-CMAC tests already.  Only the key
+ * preparation function differs; the rest of the code is shared.
+ */
+static void test_aes_xcbcmac_rfc3566(struct kunit *test)
+{
+	struct aes_cmac_key key;
+	/* AES-XCBC-MAC Test Case #4 from RFC 3566 */
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	};
+	static const u8 message[20] = {
+		0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+		0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+	};
+	static const u8 expected_mac[AES_BLOCK_SIZE] = {
+		0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15,
+		0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08,
+	};
+	u8 actual_mac[AES_BLOCK_SIZE];
+
+	aes_xcbcmac_preparekey(&key, raw_key);
+	aes_cmac(&key, message, sizeof(message), actual_mac);
+	KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE);
+}
+
+static void test_aes_cbcmac_rfc3610(struct kunit *test)
+{
+	/*
+	 * The following AES-CBC-MAC test vector is extracted from RFC 3610
+	 * Packet Vector #11.  It required some rearrangement to get the actual
+	 * input to AES-CBC-MAC from the values given.
+	 */
+	static const u8 raw_key[AES_KEYSIZE_128] = {
+		0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+		0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+	};
+	const size_t unpadded_data_len = 52;
+	static const u8 data[64] = {
+		/* clang-format off */
+		/* CCM header */
+		0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a,
+		0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14,
+		/* CCM additional authentication blocks */
+		0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+		0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00,
+		/* CCM message blocks */
+		0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+		0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
+		0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		/* clang-format on */
+	};
+	static const u8 expected_mac[AES_BLOCK_SIZE] = {
+		0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad,
+		0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc,
+	};
+	struct aes_enckey key;
+	struct aes_cbcmac_ctx ctx;
+	u8 actual_mac[AES_BLOCK_SIZE];
+	int err;
+
+	err = aes_prepareenckey(&key, raw_key, sizeof(raw_key));
+	KUNIT_ASSERT_EQ(test, err, 0);
+
+	/*
+	 * Trailing zeroes should not affect the CBC-MAC value, up to the next
+	 * AES block boundary.
+	 */
+	for (size_t data_len = unpadded_data_len; data_len <= sizeof(data);
+	     data_len++) {
+		aes_cbcmac_init(&ctx, &key);
+		aes_cbcmac_update(&ctx, data, data_len);
+		aes_cbcmac_final(&ctx, actual_mac);
+		KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac,
+				   AES_BLOCK_SIZE);
+
+		/* Incremental computations should produce the same result. */
+		for (size_t part1_len = 0; part1_len <= data_len; part1_len++) {
+			aes_cbcmac_init(&ctx, &key);
+			aes_cbcmac_update(&ctx, data, part1_len);
+			aes_cbcmac_update(&ctx, &data[part1_len],
+					  data_len - part1_len);
+			aes_cbcmac_final(&ctx, actual_mac);
+			KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac,
+					   AES_BLOCK_SIZE);
+		}
+	}
+}
+
+static struct kunit_case aes_cbc_macs_test_cases[] = {
+	HASH_KUNIT_CASES,
+	KUNIT_CASE(test_aes_cmac_rfc4493),
+	KUNIT_CASE(test_aes_xcbcmac_rfc3566),
+	KUNIT_CASE(test_aes_cbcmac_rfc3610),
+	KUNIT_CASE(benchmark_hash),
+	{},
+};
+
+static struct kunit_suite aes_cbc_macs_test_suite = {
+	.name = "aes_cbc_macs",
+	.test_cases = aes_cbc_macs_test_cases,
+	.suite_init = aes_cbc_macs_suite_init,
+	.suite_exit = aes_cbc_macs_suite_exit,
+};
+kunit_test_suite(aes_cbc_macs_test_suite);
+
+MODULE_DESCRIPTION(
+	"KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC");
+MODULE_IMPORT_NS("CRYPTO_INTERNAL");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h
index 38e33b22a0927..0ee69d8e39fe8 100644
--- a/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx);
 DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx);
 DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform);
 
+#define PHE_ALIGNMENT 16
+static void sha256_blocks_phe(struct sha256_block_state *state,
+			      const u8 *data, size_t nblocks)
+{
+	/*
+	 * On Zhaoxin processors, XSHA256 requires the %rdi register
+	 * in 64-bit mode (or %edi in 32-bit mode) to point to
+	 * a 32-byte, 16-byte-aligned buffer.
+	 */
+	u8 buf[32 + PHE_ALIGNMENT - 1];
+	u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT);
+	size_t padding = -1;
+
+	memcpy(dst, state, SHA256_DIGEST_SIZE);
+	asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */
+		     : "+a"(padding), "+c"(nblocks), "+S"(data)
+		     : "D"(dst)
+		     : "memory");
+	memcpy(state, dst, SHA256_DIGEST_SIZE);
+}
+
 static void sha256_blocks(struct sha256_block_state *state,
 			  const u8 *data, size_t nblocks)
 {
@@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void)
 	if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
 		static_call_update(sha256_blocks_x86, sha256_blocks_ni);
 		static_branch_enable(&have_sha_ni);
+	} else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) &&
+		   boot_cpu_has(X86_FEATURE_PHE_EN) &&
+		   boot_cpu_data.x86 >= 0x07) {
+		static_call_update(sha256_blocks_x86, sha256_blocks_phe);
 	} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 				     NULL) &&
 		   boot_cpu_has(X86_FEATURE_AVX)) {
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 12f50de85b621..040064c5a01f2 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -22,11 +22,18 @@
 #define ODEBUG_HASH_BITS	14
 #define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)
 
-/* Must be power of two */
+/*
+ * Must be power of two.
+ * Please change Kconfig help text of DEBUG_OBJECTS_POOL_SIZE_SHIFT when
+ * changed.
+ */
 #define ODEBUG_BATCH_SIZE	16
 
+#define ODEBUG_POOL_SHIFT	CONFIG_DEBUG_OBJECTS_POOL_SIZE_SHIFT
+static_assert(ODEBUG_POOL_SHIFT >= 0);
+
 /* Initial values. Must all be a multiple of batch size */
-#define ODEBUG_POOL_SIZE	(64 * ODEBUG_BATCH_SIZE)
+#define ODEBUG_POOL_SIZE	((1 << ODEBUG_POOL_SHIFT) * ODEBUG_BATCH_SIZE)
 #define ODEBUG_POOL_MIN_LEVEL	(ODEBUG_POOL_SIZE / 4)
 
 #define ODEBUG_POOL_PERCPU_SIZE	(8 * ODEBUG_BATCH_SIZE)
@@ -586,6 +593,10 @@ static void debug_objects_oom(void)
 	struct debug_bucket *db = obj_hash;
 	HLIST_HEAD(freelist);
 
+	/*
+	 * Please change Kconfig help text of DEBUG_OBJECTS_POOL_SIZE_SHIFT
+	 * when changed.
+	 */
 	pr_warn("Out of memory. ODEBUG disabled\n");
 
 	for (int i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c
index ca736166f1000..1288f146661f1 100644
--- a/lib/decompress_bunzip2.c
+++ b/lib/decompress_bunzip2.c
@@ -135,7 +135,7 @@ static unsigned int INIT get_bits(struct bunzip_data *bd, char bits_wanted)
 		}
 		/* Avoid 32-bit overflow (dump bit buffer to top of output) */
 		if (bd->inbufBitCount >= 24) {
-			bits = bd->inbufBits&((1 << bd->inbufBitCount)-1);
+			bits = bd->inbufBits & ((1ULL << bd->inbufBitCount) - 1);
 			bits_wanted -= bd->inbufBitCount;
 			bits <<= bits_wanted;
 			bd->inbufBitCount = 0;
@@ -146,7 +146,7 @@ static unsigned int INIT get_bits(struct bunzip_data *bd, char bits_wanted)
 	}
 	/* Calculate result */
 	bd->inbufBitCount -= bits_wanted;
-	bits |= (bd->inbufBits >> bd->inbufBitCount)&((1 << bits_wanted)-1);
+	bits |= (bd->inbufBits >> bd->inbufBitCount) & ((1ULL << bits_wanted) - 1);
 
 	return bits;
 }
diff --git a/lib/find_bit.c b/lib/find_bit.c
index 5a0066c26d9a6..5ac52dfce7300 100644
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -172,13 +172,6 @@ unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long
 }
 EXPORT_SYMBOL(__find_nth_and_bit);
 
-unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
-				 unsigned long size, unsigned long n)
-{
-	return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n);
-}
-EXPORT_SYMBOL(__find_nth_andnot_bit);
-
 unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1,
 					const unsigned long *addr2,
 					const unsigned long *addr3,
diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c
index 402e160e71863..00d9dc61cd464 100644
--- a/lib/find_bit_benchmark.c
+++ b/lib/find_bit_benchmark.c
@@ -30,18 +30,20 @@ static DECLARE_BITMAP(bitmap, BITMAP_LEN) __initdata;
 static DECLARE_BITMAP(bitmap2, BITMAP_LEN) __initdata;
 
 /*
- * This is Schlemiel the Painter's algorithm. It should be called after
- * all other tests for the same bitmap because it sets all bits of bitmap to 1.
+ * This is Schlemiel the Painter's algorithm.
  */
-static int __init test_find_first_bit(void *bitmap, unsigned long len)
+static int __init test_find_first_bit(const void *bitmap, unsigned long len)
 {
+	static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata;
 	unsigned long i, cnt;
 	ktime_t time;
 
+	bitmap_copy(cp, bitmap, BITMAP_LEN);
+
 	time = ktime_get();
 	for (cnt = i = 0; i < len; cnt++) {
-		i = find_first_bit(bitmap, len);
-		__clear_bit(i, bitmap);
+		i = find_first_bit(cp, len);
+		__clear_bit(i, cp);
 	}
 	time = ktime_get() - time;
 	pr_err("find_first_bit:     %18llu ns, %6ld iterations\n", time, cnt);
@@ -49,7 +51,8 @@ static int __init test_find_first_bit(void *bitmap, unsigned long len)
 	return 0;
 }
 
-static int __init test_find_first_and_bit(void *bitmap, const void *bitmap2, unsigned long len)
+static int __init test_find_first_and_bit(const void *bitmap, const void *bitmap2,
+		unsigned long len)
 {
 	static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata;
 	unsigned long i, cnt;
diff --git a/lib/fonts/font.h b/lib/fonts/font.h
new file mode 100644
index 0000000000000..4f1adf0b6b549
--- /dev/null
+++ b/lib/fonts/font.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _LIB_FONTS_FONT_H
+#define _LIB_FONTS_FONT_H
+
+#include <linux/font.h>
+
+/*
+ * Font data
+ */
+
+#define FONT_EXTRA_WORDS 4
+
+struct font_data {
+	unsigned int extra[FONT_EXTRA_WORDS];
+	unsigned char data[];
+} __packed;
+
+/*
+ * Built-in fonts
+ */
+
+#define VGA8x8_IDX	0
+#define VGA8x16_IDX	1
+#define PEARL8x8_IDX	2
+#define VGA6x11_IDX	3
+#define FONT7x14_IDX	4
+#define	FONT10x18_IDX	5
+#define SUN8x16_IDX	6
+#define SUN12x22_IDX	7
+#define ACORN8x8_IDX	8
+#define	MINI4x6_IDX	9
+#define FONT6x10_IDX	10
+#define TER16x32_IDX	11
+#define FONT6x8_IDX	12
+#define TER10x18_IDX	13
+
+#endif
diff --git a/lib/fonts/font_10x18.c b/lib/fonts/font_10x18.c
index 5d940db626e71..10edebc4bb74a 100644
--- a/lib/fonts/font_10x18.c
+++ b/lib/fonts/font_10x18.c
@@ -4,7 +4,7 @@
  * by Jurriaan Kalkman 06-2005  *
  ********************************/
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 9216
 
diff --git a/lib/fonts/font_6x10.c b/lib/fonts/font_6x10.c
index e65df019e0d29..660d3a371b309 100644
--- a/lib/fonts/font_6x10.c
+++ b/lib/fonts/font_6x10.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
+#include "font.h"
 
 #define FONTDATAMAX 2560
 
diff --git a/lib/fonts/font_6x11.c b/lib/fonts/font_6x11.c
index bd76b3f6b6350..671487ccc1724 100644
--- a/lib/fonts/font_6x11.c
+++ b/lib/fonts/font_6x11.c
@@ -5,7 +5,7 @@
 /*                                            */
 /**********************************************/
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX (11*256)
 
diff --git a/lib/fonts/font_6x8.c b/lib/fonts/font_6x8.c
index 06ace77925214..5811ee07f4d8c 100644
--- a/lib/fonts/font_6x8.c
+++ b/lib/fonts/font_6x8.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
+#include "font.h"
 
 #define FONTDATAMAX 2048
 
diff --git a/lib/fonts/font_7x14.c b/lib/fonts/font_7x14.c
index a2f561c9fa040..0c7475d643c82 100644
--- a/lib/fonts/font_7x14.c
+++ b/lib/fonts/font_7x14.c
@@ -4,7 +4,7 @@
 /* by Jurriaan Kalkman 05-2005        */
 /**************************************/
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 3584
 
diff --git a/lib/fonts/font_8x16.c b/lib/fonts/font_8x16.c
index 06ae140885141..523e95c75569e 100644
--- a/lib/fonts/font_8x16.c
+++ b/lib/fonts/font_8x16.c
@@ -5,9 +5,10 @@
 /*                                            */
 /**********************************************/
 
-#include <linux/font.h>
 #include <linux/module.h>
 
+#include "font.h"
+
 #define FONTDATAMAX 4096
 
 static const struct font_data fontdata_8x16 = {
diff --git a/lib/fonts/font_8x8.c b/lib/fonts/font_8x8.c
index 69570b8c31af2..e5b697fc96750 100644
--- a/lib/fonts/font_8x8.c
+++ b/lib/fonts/font_8x8.c
@@ -5,7 +5,7 @@
 /*                                            */
 /**********************************************/
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 2048
 
diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c
index 18755c33d2491..36c51016769d4 100644
--- a/lib/fonts/font_acorn_8x8.c
+++ b/lib/fonts/font_acorn_8x8.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Acorn-like font definition, with PC graphics characters */
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 2048
 
@@ -68,7 +68,7 @@ static const struct font_data acorndata_8x8 = {
 /* 3A */  0x00, 0x00, 0x18, 0x18, 0x00, 0x18, 0x18, 0x00, /* : */
 /* 3B */  0x00, 0x00, 0x18, 0x18, 0x00, 0x18, 0x18, 0x30, /* ; */
 /* 3C */  0x0C, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0C, 0x00, /* < */
-/* 3D */  0x00, 0x00, 0x7E, 0x00, 0x7E, 0x00, 0x00, 0x00, /* = */ 
+/* 3D */  0x00, 0x00, 0x7E, 0x00, 0x7E, 0x00, 0x00, 0x00, /* = */
 /* 3E */  0x30, 0x18, 0x0C, 0x06, 0x0C, 0x18, 0x30, 0x00, /* > */
 /* 3F */  0x3C, 0x66, 0x0C, 0x18, 0x18, 0x00, 0x18, 0x00, /* ? */
 /* 40 */  0x3C, 0x66, 0x6E, 0x6A, 0x6E, 0x60, 0x3C, 0x00, /* @ */
diff --git a/lib/fonts/font_mini_4x6.c b/lib/fonts/font_mini_4x6.c
index 8d39fd4479525..dc919c160ddea 100644
--- a/lib/fonts/font_mini_4x6.c
+++ b/lib/fonts/font_mini_4x6.c
@@ -18,15 +18,15 @@
 s{((0x)?[0-9a-fA-F]+)(.*\[([\*\ ]{4})\])}{
 
 	($num,$pat,$bits) = ($1,$3,$4);
-	
+
 	$bits =~ s/([^\s0])|(.)/ defined($1) + 0 /ge;
-	
+
 	$num = ord(pack("B8", $bits));
 	$num |= $num >> 4;
 	$num = sprintf("0x%.2x", $num);
-	
+
 	#print "$num,$pat,$bits\n";
-	
+
 	$num . $pat;
 }ge;
 
@@ -39,7 +39,7 @@ __END__;
    MSBit to LSBit = left to right.
  */
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 1536
 
diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c
index ae98ca17982e6..2438b374aceaf 100644
--- a/lib/fonts/font_pearl_8x8.c
+++ b/lib/fonts/font_pearl_8x8.c
@@ -10,7 +10,7 @@
 /*                                            */
 /**********************************************/
 
-#include <linux/font.h>
+#include "font.h"
 
 #define FONTDATAMAX 2048
 
diff --git a/lib/fonts/font_sun12x22.c b/lib/fonts/font_sun12x22.c
index 91daf5ab8b6ba..2afbc144bea81 100644
--- a/lib/fonts/font_sun12x22.c
+++ b/lib/fonts/font_sun12x22.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
+#include "font.h"
 
 #define FONTDATAMAX 11264
 
diff --git a/lib/fonts/font_sun8x16.c b/lib/fonts/font_sun8x16.c
index 81bb4eeae04e6..2b7b2d8e548ac 100644
--- a/lib/fonts/font_sun8x16.c
+++ b/lib/fonts/font_sun8x16.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
+#include "font.h"
 
 #define FONTDATAMAX 4096
 
diff --git a/lib/fonts/font_ter10x18.c b/lib/fonts/font_ter10x18.c
index 80356e9d56c73..3f30b4a211ab4 100644
--- a/lib/fonts/font_ter10x18.c
+++ b/lib/fonts/font_ter10x18.c
@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
 #include <linux/module.h>
 
+#include "font.h"
+
 #define FONTDATAMAX 9216
 
 static const struct font_data fontdata_ter10x18 = {
diff --git a/lib/fonts/font_ter16x32.c b/lib/fonts/font_ter16x32.c
index 5baedc573dd6b..93616cffe6421 100644
--- a/lib/fonts/font_ter16x32.c
+++ b/lib/fonts/font_ter16x32.c
@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/font.h>
+
 #include <linux/module.h>
 
+#include "font.h"
+
 #define FONTDATAMAX 16384
 
 static const struct font_data fontdata_ter16x32 = {
diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c
index a7f118b301719..5938f542906b0 100644
--- a/lib/fonts/fonts.c
+++ b/lib/fonts/fonts.c
@@ -12,13 +12,241 @@
  * for more details.
  */
 
+#include <linux/container_of.h>
+#include <linux/kd.h>
 #include <linux/module.h>
-#include <linux/types.h>
+#include <linux/overflow.h>
+#include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/types.h>
+
 #if defined(__mc68000__)
 #include <asm/setup.h>
 #endif
-#include <linux/font.h>
+
+#include "font.h"
+
+#define console_font_pitch(font) DIV_ROUND_UP((font)->width, 8)
+
+/*
+ * Helpers for font_data_t
+ */
+
+/* Extra word getters */
+#define REFCOUNT(fd)	(((int *)(fd))[-1])
+#define FNTSIZE(fd)	(((int *)(fd))[-2])
+#define FNTSUM(fd)	(((int *)(fd))[-4])
+
+static struct font_data *to_font_data_struct(font_data_t *fd)
+{
+	return container_of(fd, struct font_data, data[0]);
+}
+
+static bool font_data_is_internal(font_data_t *fd)
+{
+	return !REFCOUNT(fd); /* internal fonts have no reference counting */
+}
+
+static void font_data_free(font_data_t *fd)
+{
+	kfree(to_font_data_struct(fd));
+}
+
+/**
+ * font_data_import - Allocates and initializes font data from user space
+ * @font: A font from user space
+ * @vpitch: The size of a single glyph in @font in bytes
+ * @calc_csum: An optional helper to calculate a chechsum
+ *
+ * Font data from user space must be translated to the kernel's format. The
+ * font's glyph geometry and data is provided in @font. The parameter @vpitch
+ * gives the number of bytes per glyph, including trailing bytes.
+ *
+ * The parameter @calc_csum is optional. Fbcon passes crc32() to calculate the
+ * font data's checksum.
+ *
+ * Returns:
+ * Newly initialized font data on success, or a pointer-encoded errno value otherwise.
+ */
+font_data_t *font_data_import(const struct console_font *font, unsigned int vpitch,
+			      u32 (*calc_csum)(u32, const void *, size_t))
+{
+	unsigned int pitch = console_font_pitch(font);
+	unsigned int h = font->height;
+	unsigned int charcount = font->charcount;
+	const unsigned char *data = font->data;
+	u32 csum = 0;
+	struct font_data *font_data;
+	int size, alloc_size;
+	unsigned int i;
+	font_data_t *fd;
+
+	/* Check for integer overflow in font-size calculation */
+	if (check_mul_overflow(h, pitch, &size) ||
+	    check_mul_overflow(size, charcount, &size))
+		return ERR_PTR(-EINVAL);
+
+	/* Check for overflow in allocation size calculation */
+	if (check_add_overflow(sizeof(*font_data), size, &alloc_size))
+		return ERR_PTR(-EINVAL);
+
+	font_data = kmalloc(alloc_size, GFP_USER);
+	if (!font_data)
+		return ERR_PTR(-ENOMEM);
+	memset(font_data->extra, 0, sizeof(font_data->extra));
+
+	for (i = 0; i < charcount; ++i)
+		memcpy(font_data->data + i * h * pitch, data + i * vpitch * pitch, h * pitch);
+
+	if (calc_csum)
+		csum = calc_csum(0, font_data->data, size);
+
+	fd = font_data->data;
+	REFCOUNT(fd) = 1; /* start with reference acquired */
+	FNTSIZE(fd) = size;
+	FNTSUM(fd) = csum;
+
+	return fd;
+}
+EXPORT_SYMBOL_GPL(font_data_import);
+
+/**
+ * font_data_get - Acquires a reference on font data
+ * @fd: Font data
+ *
+ * Font data from user space is reference counted. The helper
+ * font_data_get() increases the reference counter by one. Invoke
+ * font_data_put() to release the reference.
+ *
+ * Internal font data is located in read-only memory. In this case
+ * the helper returns success without modifying the counter field.
+ * It is still required to call font_data_put() on internal font data.
+ */
+void font_data_get(font_data_t *fd)
+{
+	if (font_data_is_internal(fd))
+		return; /* never ref static data */
+
+	if (WARN_ON(!REFCOUNT(fd)))
+		return; /* should never be 0 */
+	++REFCOUNT(fd);
+}
+EXPORT_SYMBOL_GPL(font_data_get);
+
+/**
+ * font_data_put - Release a reference on font data
+ * @fd: Font data
+ *
+ * Font data from user space is reference counted. The helper
+ * font_data_put() decreases the reference counter by one. If this was
+ * the final reference, it frees the allocated memory.
+ *
+ * Internal font data is located in read-only memory. In this case
+ * the helper returns success without modifying the counter field.
+ *
+ * Returns:
+ * True if the font data's memory buffer has been freed, false otherwise.
+ */
+bool font_data_put(font_data_t *fd)
+{
+	unsigned int count;
+
+	if (font_data_is_internal(fd))
+		return false; /* never unref static data */
+
+	if (WARN_ON(!REFCOUNT(fd)))
+		return false; /* should never be 0 */
+
+	count = --REFCOUNT(fd);
+	if (!count)
+		font_data_free(fd);
+
+	return !count;
+}
+EXPORT_SYMBOL_GPL(font_data_put);
+
+/**
+ * font_data_size - Return size of the font data in bytes
+ * @fd: Font data
+ *
+ * Returns:
+ * The number of bytes in the given font data.
+ */
+unsigned int font_data_size(font_data_t *fd)
+{
+	return FNTSIZE(fd);
+}
+EXPORT_SYMBOL_GPL(font_data_size);
+
+/**
+ * font_data_is_equal - Compares font data for equality
+ * @lhs: Left-hand side font data
+ * @rhs: Right-hand-size font data
+ *
+ * Font data is equal if is constain the same sequence of values. The
+ * helper also use the checksum, if both arguments contain it. Font data
+ * coming from different origins, internal or from user space, is never
+ * equal. Allowing this would break reference counting.
+ *
+ * Returns:
+ * True if the given font data is equal, false otherwise.
+ */
+bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs)
+{
+	if (font_data_is_internal(lhs) != font_data_is_internal(rhs))
+		return false;
+	if (font_data_size(lhs) != font_data_size(rhs))
+		return false;
+	if (FNTSUM(lhs) && FNTSUM(rhs) && FNTSUM(lhs) != FNTSUM(rhs))
+		return false;
+
+	return !memcmp(lhs, rhs, FNTSIZE(lhs));
+}
+EXPORT_SYMBOL_GPL(font_data_is_equal);
+
+/**
+ * font_data_export - Stores font data for user space
+ * @fd: Font data
+ * @font: A font for user space
+ * @vpitch: The size of a single glyph in @font in bytes
+ *
+ * Store the font data given in @fd to the font in @font. Values and
+ * pointers in @font are pre-initialized. This helper mostly checks some
+ * corner cases and translates glyph sizes according to the value given
+ * @vpitch.
+ *
+ * Returns:
+ * 0 on success, or a negative errno code otherwise.
+ */
+int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch)
+{
+	const unsigned char *font_data = font_data_buf(fd);
+	unsigned char *data = font->data;
+	unsigned int pitch = console_font_pitch(font);
+	unsigned int glyphsize, i;
+
+	if (!font->width || !font->height || !font->charcount || !font->data)
+		return 0;
+
+	glyphsize = font->height * pitch;
+
+	if (font->charcount * glyphsize > font_data_size(fd))
+		return -EINVAL;
+
+	for (i = 0; i < font->charcount; i++) {
+		memcpy(data, font_data, glyphsize);
+		memset(data + glyphsize, 0, pitch * vpitch - glyphsize);
+		data += pitch * vpitch;
+		font_data += glyphsize;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(font_data_export);
+
+/*
+ * Font lookup
+ */
 
 static const struct font_desc *fonts[] = {
 #ifdef CONFIG_FONT_8x8
diff --git a/lib/glob.c b/lib/glob.c
index aa57900d2062c..7aca76c25bcb2 100644
--- a/lib/glob.c
+++ b/lib/glob.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
 #include <linux/module.h>
 #include <linux/glob.h>
+#include <linux/export.h>
 
 /*
  * The only reason this code can be compiled as a module is because the
@@ -20,7 +22,7 @@ MODULE_LICENSE("Dual MIT/GPL");
  * Pattern metacharacters are ?, *, [ and \.
  * (And, inside character classes, !, - and ].)
  *
- * This is small and simple implementation intended for device blacklists
+ * This is a small and simple implementation intended for device denylists
  * where a string is matched against a number of patterns.  Thus, it
  * does not preprocess the patterns.  It is non-recursive, and run-time
  * is at most quadratic: strlen(@str)*strlen(@pat).
@@ -45,7 +47,7 @@ bool __pure glob_match(char const *pat, char const *str)
 	 * (no exception for /), it can be easily proved that there's
 	 * never a need to backtrack multiple levels.
 	 */
-	char const *back_pat = NULL, *back_str;
+	char const *back_pat = NULL, *back_str = NULL;
 
 	/*
 	 * Loop over each token (character or class) in pat, matching
@@ -71,7 +73,7 @@ bool __pure glob_match(char const *pat, char const *str)
 			if (c == '\0')	/* No possible match */
 				return false;
 			bool match = false, inverted = (*pat == '!');
-			char const *class = pat + inverted;
+			char const *class = inverted ? pat + 1 : pat;
 			unsigned char a = *class++;
 
 			/*
@@ -94,7 +96,8 @@ bool __pure glob_match(char const *pat, char const *str)
 					class += 2;
 					/* Any special action if a > b? */
 				}
-				match |= (a <= c && c <= b);
+				if (a <= c && c <= b)
+					match = true;
 			} while ((a = *class++) != ']');
 
 			if (match == inverted)
diff --git a/lib/idr.c b/lib/idr.c
index 69bee5369670c..1649f41016e7c 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -495,10 +495,9 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	unsigned long index = min / IDA_BITMAP_BITS;
 	unsigned int offset = min % IDA_BITMAP_BITS;
 	unsigned long *addr, size, bit;
-	unsigned long tmp = 0;
+	unsigned long tmp;
 	unsigned long flags;
 	void *entry;
-	int ret;
 
 	if ((int)min < 0)
 		return -EINVAL;
@@ -508,40 +507,34 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	xa_lock_irqsave(&ida->xa, flags);
 
 	entry = xa_find(&ida->xa, &index, max / IDA_BITMAP_BITS, XA_PRESENT);
-	if (!entry) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (index > min / IDA_BITMAP_BITS)
-		offset = 0;
-	if (index * IDA_BITMAP_BITS + offset > max) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (xa_is_value(entry)) {
-		tmp = xa_to_value(entry);
-		addr = &tmp;
-		size = BITS_PER_XA_VALUE;
-	} else {
-		addr = ((struct ida_bitmap *)entry)->bitmap;
-		size = IDA_BITMAP_BITS;
-	}
-
-	bit = find_next_bit(addr, size, offset);
+	while (entry) {
+		if (index > min / IDA_BITMAP_BITS)
+			offset = 0;
+		if (index * IDA_BITMAP_BITS + offset > max)
+			break;
 
-	xa_unlock_irqrestore(&ida->xa, flags);
+		if (xa_is_value(entry)) {
+			tmp = xa_to_value(entry);
+			addr = &tmp;
+			size = BITS_PER_XA_VALUE;
+		} else {
+			addr = ((struct ida_bitmap *)entry)->bitmap;
+			size = IDA_BITMAP_BITS;
+		}
 
-	if (bit == size ||
-	    index * IDA_BITMAP_BITS + bit > max)
-		return -ENOENT;
+		bit = find_next_bit(addr, size, offset);
+		if (bit < size &&
+		    index * IDA_BITMAP_BITS + bit <= max) {
+			xa_unlock_irqrestore(&ida->xa, flags);
+			return index * IDA_BITMAP_BITS + bit;
+		}
 
-	return index * IDA_BITMAP_BITS + bit;
+		entry = xa_find_after(&ida->xa, &index,
+				      max / IDA_BITMAP_BITS, XA_PRESENT);
+	}
 
-err_unlock:
 	xa_unlock_irqrestore(&ida->xa, flags);
-	return ret;
+	return -ENOENT;
 }
 EXPORT_SYMBOL(ida_find_first_range);
 
diff --git a/lib/inflate.c b/lib/inflate.c
index eab886baa1b48..44a7da582baa0 100644
--- a/lib/inflate.c
+++ b/lib/inflate.c
@@ -9,7 +9,7 @@
  * based on gzip-1.0.3 
  *
  * Nicolas Pitre <nico@fluxnic.net>, 1999/04/14 :
- *   Little mods for all variable to reside either into rodata or bss segments
+ *   Little mods for all variables to reside either into rodata or bss segments
  *   by marking constant variables with 'const' and initializing all the others
  *   at run-time only.  This allows for the kernel uncompressor to run
  *   directly from Flash or ROM memory on embedded systems.
@@ -286,7 +286,7 @@ static void free(void *where)
    the longer codes.  The time it costs to decode the longer codes is
    then traded against the time it takes to make longer tables.
 
-   This results of this trade are in the variables lbits and dbits
+   The results of this trade are in the variables lbits and dbits
    below.  lbits is the number of bits the first level table for literal/
    length codes can decode in one step, and dbits is the same thing for
    the distance codes.  Subsequent tables are also less than or equal to
@@ -811,6 +811,8 @@ DEBG("<fix");
 
   /* decompress until an end-of-block code */
   if (inflate_codes(tl, td, bl, bd)) {
+    huft_free(tl);
+    huft_free(td);
     free(l);
     return 1;
   }
@@ -1007,10 +1009,10 @@ DEBG("dyn5d ");
 DEBG("dyn6 ");
 
   /* decompress until an end-of-block code */
-  if (inflate_codes(tl, td, bl, bd)) {
+  if (inflate_codes(tl, td, bl, bd))
     ret = 1;
-    goto out;
-  }
+  else
+    ret = 0;
 
 DEBG("dyn7 ");
 
@@ -1019,7 +1021,6 @@ DEBG("dyn7 ");
   huft_free(td);
 
   DEBG(">");
-  ret = 0;
 out:
   free(ll);
   return ret;
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
index 97be2a39f5371..edc4eb7c1bca1 100644
--- a/lib/kstrtox.c
+++ b/lib/kstrtox.c
@@ -39,25 +39,30 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base)
 	return s;
 }
 
-/*
- * Convert non-negative integer string representation in explicitly given radix
- * to an integer. A maximum of max_chars characters will be converted.
+/**
+ * _parse_integer_limit - Convert integer string representation to an integer
+ * @s: Integer string representation
+ * @base: Radix
+ * @p: Where to store result
+ * @max_chars: Maximum amount of characters to convert
+ *
+ * Convert non-negative integer string representation in explicitly given
+ * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX.
  *
- * Return number of characters consumed maybe or-ed with overflow bit.
- * If overflow occurs, result integer (incorrect) is still returned.
+ * This function is the workhorse of other string conversion functions and it
+ * is discouraged to use it explicitly. Consider kstrto*() family instead.
  *
- * Don't you dare use this function.
+ * Return: Number of characters consumed, maybe ORed with overflow bit
  */
 noinline
 unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p,
 				  size_t max_chars)
 {
+	unsigned int rv, overflow = 0;
 	unsigned long long res;
-	unsigned int rv;
 
 	res = 0;
-	rv = 0;
-	while (max_chars--) {
+	for (rv = 0; rv < max_chars; rv++, s++) {
 		unsigned int c = *s;
 		unsigned int lc = _tolower(c);
 		unsigned int val;
@@ -76,15 +81,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon
 		 * it in the max base we support (16)
 		 */
 		if (unlikely(res & (~0ull << 60))) {
-			if (res > div_u64(ULLONG_MAX - val, base))
-				rv |= KSTRTOX_OVERFLOW;
+			if (check_mul_overflow(res, base, &res) ||
+			    check_add_overflow(res, val, &res)) {
+				res = ULLONG_MAX;
+				overflow = KSTRTOX_OVERFLOW;
+			}
+		} else {
+			res = res * base + val;
 		}
-		res = res * base + val;
-		rv++;
-		s++;
 	}
 	*p = res;
-	return rv;
+	return rv | overflow;
 }
 
 noinline
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 5aa4c95000188..739918e859e57 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -101,6 +101,7 @@ static const unsigned long mt_max[] = {
 	[maple_leaf_64]		= ULONG_MAX,
 	[maple_range_64]	= ULONG_MAX,
 	[maple_arange_64]	= ULONG_MAX,
+	[maple_copy]		= ULONG_MAX,
 };
 #define mt_node_max(x) mt_max[mte_node_type(x)]
 #endif
@@ -110,6 +111,7 @@ static const unsigned char mt_slots[] = {
 	[maple_leaf_64]		= MAPLE_RANGE64_SLOTS,
 	[maple_range_64]	= MAPLE_RANGE64_SLOTS,
 	[maple_arange_64]	= MAPLE_ARANGE64_SLOTS,
+	[maple_copy]		= 3,
 };
 #define mt_slot_count(x) mt_slots[mte_node_type(x)]
 
@@ -118,6 +120,7 @@ static const unsigned char mt_pivots[] = {
 	[maple_leaf_64]		= MAPLE_RANGE64_SLOTS - 1,
 	[maple_range_64]	= MAPLE_RANGE64_SLOTS - 1,
 	[maple_arange_64]	= MAPLE_ARANGE64_SLOTS - 1,
+	[maple_copy]		= 3,
 };
 #define mt_pivot_count(x) mt_pivots[mte_node_type(x)]
 
@@ -126,48 +129,10 @@ static const unsigned char mt_min_slots[] = {
 	[maple_leaf_64]		= (MAPLE_RANGE64_SLOTS / 2) - 2,
 	[maple_range_64]	= (MAPLE_RANGE64_SLOTS / 2) - 2,
 	[maple_arange_64]	= (MAPLE_ARANGE64_SLOTS / 2) - 1,
+	[maple_copy]		= 1, /* Should never be used */
 };
 #define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)]
 
-#define MAPLE_BIG_NODE_SLOTS	(MAPLE_RANGE64_SLOTS * 2 + 2)
-#define MAPLE_BIG_NODE_GAPS	(MAPLE_ARANGE64_SLOTS * 2 + 1)
-
-struct maple_big_node {
-	unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1];
-	union {
-		struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS];
-		struct {
-			unsigned long padding[MAPLE_BIG_NODE_GAPS];
-			unsigned long gap[MAPLE_BIG_NODE_GAPS];
-		};
-	};
-	unsigned char b_end;
-	enum maple_type type;
-};
-
-/*
- * The maple_subtree_state is used to build a tree to replace a segment of an
- * existing tree in a more atomic way.  Any walkers of the older tree will hit a
- * dead node and restart on updates.
- */
-struct maple_subtree_state {
-	struct ma_state *orig_l;	/* Original left side of subtree */
-	struct ma_state *orig_r;	/* Original right side of subtree */
-	struct ma_state *l;		/* New left side of subtree */
-	struct ma_state *m;		/* New middle of subtree (rare) */
-	struct ma_state *r;		/* New right side of subtree */
-	struct ma_topiary *free;	/* nodes to be freed */
-	struct ma_topiary *destroy;	/* Nodes to be destroyed (walked and freed) */
-	struct maple_big_node *bn;
-};
-
-#ifdef CONFIG_KASAN_STACK
-/* Prevent mas_wr_bnode() from exceeding the stack frame limit */
-#define noinline_for_kasan noinline_for_stack
-#else
-#define noinline_for_kasan inline
-#endif
-
 /* Functions */
 static inline struct maple_node *mt_alloc_one(gfp_t gfp)
 {
@@ -349,6 +314,13 @@ static inline struct maple_enode *mt_mk_node(const struct maple_node *node,
 			(type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL);
 }
 
+static inline void ma_init_slot(void __rcu **slot, const struct maple_node *mn,
+				const enum maple_type mt)
+{
+	/* WARNING: this is unsafe if the slot is exposed to readers. */
+	RCU_INIT_POINTER(*slot, (void *)mt_mk_node(mn, mt));
+}
+
 static inline void *mte_mk_root(const struct maple_enode *node)
 {
 	return (void *)((unsigned long)node | MAPLE_ROOT_NODE);
@@ -605,6 +577,8 @@ static inline unsigned long *ma_pivots(struct maple_node *node,
 	case maple_range_64:
 	case maple_leaf_64:
 		return node->mr64.pivot;
+	case maple_copy:
+		return node->cp.pivot;
 	case maple_dense:
 		return NULL;
 	}
@@ -624,6 +598,8 @@ static inline unsigned long *ma_gaps(struct maple_node *node,
 	switch (type) {
 	case maple_arange_64:
 		return node->ma64.gap;
+	case maple_copy:
+		return node->cp.gap;
 	case maple_range_64:
 	case maple_leaf_64:
 	case maple_dense:
@@ -690,6 +666,7 @@ static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv,
 	case maple_arange_64:
 		node->ma64.pivot[piv] = val;
 		break;
+	case maple_copy:
 	case maple_dense:
 		break;
 	}
@@ -711,6 +688,8 @@ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt)
 	case maple_range_64:
 	case maple_leaf_64:
 		return mn->mr64.slot;
+	case maple_copy:
+		return mn->cp.slot;
 	case maple_dense:
 		return mn->slot;
 	}
@@ -1298,25 +1277,40 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
 	return mt_pivots[type];
 }
 
-/*
- * mas_leaf_max_gap() - Returns the largest gap in a leaf node
- * @mas: the maple state
- *
- * Return: The maximum gap in the leaf.
- */
-static unsigned long mas_leaf_max_gap(struct ma_state *mas)
+static inline
+void wr_mas_setup(struct ma_wr_state *wr_mas, struct ma_state *mas)
+{
+	wr_mas->node = mas_mn(mas);
+	wr_mas->type = mte_node_type(mas->node);
+	wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type);
+	wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type);
+	wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, mas->offset);
+	wr_mas->r_max = mas_safe_pivot(mas, wr_mas->pivots, mas->offset,
+				       wr_mas->type);
+}
+
+static inline
+void wr_mas_ascend(struct ma_wr_state *wr_mas)
+{
+	struct ma_state *mas = wr_mas->mas;
+
+	mas_ascend(mas);
+	wr_mas_setup(wr_mas, mas);
+	mas->end = ma_data_end(wr_mas->node, wr_mas->type, wr_mas->pivots,
+			       mas->max);
+	/* Careful, this may be wrong.. */
+	wr_mas->end_piv = wr_mas->r_max;
+	wr_mas->offset_end = mas->offset;
+}
+
+static inline unsigned long ma_leaf_max_gap(struct maple_node *mn,
+		enum maple_type mt, unsigned long min, unsigned long max,
+		unsigned long *pivots, void __rcu **slots)
 {
-	enum maple_type mt;
 	unsigned long pstart, gap, max_gap;
-	struct maple_node *mn;
-	unsigned long *pivots;
-	void __rcu **slots;
 	unsigned char i;
 	unsigned char max_piv;
 
-	mt = mte_node_type(mas->node);
-	mn = mas_mn(mas);
-	slots = ma_slots(mn, mt);
 	max_gap = 0;
 	if (unlikely(ma_is_dense(mt))) {
 		gap = 0;
@@ -1338,26 +1332,25 @@ static unsigned long mas_leaf_max_gap(struct ma_state *mas)
 	 * Check the first implied pivot optimizes the loop below and slot 1 may
 	 * be skipped if there is a gap in slot 0.
 	 */
-	pivots = ma_pivots(mn, mt);
 	if (likely(!slots[0])) {
-		max_gap = pivots[0] - mas->min + 1;
+		max_gap = pivots[0] - min + 1;
 		i = 2;
 	} else {
 		i = 1;
 	}
 
 	/* reduce max_piv as the special case is checked before the loop */
-	max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1;
+	max_piv = ma_data_end(mn, mt, pivots, max) - 1;
 	/*
 	 * Check end implied pivot which can only be a gap on the right most
 	 * node.
 	 */
-	if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) {
+	if (unlikely(max == ULONG_MAX) && !slots[max_piv + 1]) {
 		gap = ULONG_MAX - pivots[max_piv];
 		if (gap > max_gap)
 			max_gap = gap;
 
-		if (max_gap > pivots[max_piv] - mas->min)
+		if (max_gap > pivots[max_piv] - min)
 			return max_gap;
 	}
 
@@ -1377,6 +1370,27 @@ static unsigned long mas_leaf_max_gap(struct ma_state *mas)
 	return max_gap;
 }
 
+/*
+ * mas_leaf_max_gap() - Returns the largest gap in a leaf node
+ * @mas: the maple state
+ *
+ * Return: The maximum gap in the leaf.
+ */
+static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
+{
+	enum maple_type mt;
+	struct maple_node *mn;
+	unsigned long *pivots;
+	void __rcu **slots;
+
+	mn = mas_mn(mas);
+	mt = mte_node_type(mas->node);
+	slots = ma_slots(mn, mt);
+	pivots = ma_pivots(mn, mt);
+
+	return ma_leaf_max_gap(mn, mt, mas->min, mas->max, pivots, slots);
+}
+
 /*
  * ma_max_gap() - Get the maximum gap in a maple node (non-leaf)
  * @node: The maple node
@@ -1616,169 +1630,6 @@ static inline bool mas_find_child(struct ma_state *mas, struct ma_state *child)
 	return false;
 }
 
-/*
- * mab_shift_right() - Shift the data in mab right. Note, does not clean out the
- * old data or set b_node->b_end.
- * @b_node: the maple_big_node
- * @shift: the shift count
- */
-static inline void mab_shift_right(struct maple_big_node *b_node,
-				 unsigned char shift)
-{
-	unsigned long size = b_node->b_end * sizeof(unsigned long);
-
-	memmove(b_node->pivot + shift, b_node->pivot, size);
-	memmove(b_node->slot + shift, b_node->slot, size);
-	if (b_node->type == maple_arange_64)
-		memmove(b_node->gap + shift, b_node->gap, size);
-}
-
-/*
- * mab_middle_node() - Check if a middle node is needed (unlikely)
- * @b_node: the maple_big_node that contains the data.
- * @split: the potential split location
- * @slot_count: the size that can be stored in a single node being considered.
- *
- * Return: true if a middle node is required.
- */
-static inline bool mab_middle_node(struct maple_big_node *b_node, int split,
-				   unsigned char slot_count)
-{
-	unsigned char size = b_node->b_end;
-
-	if (size >= 2 * slot_count)
-		return true;
-
-	if (!b_node->slot[split] && (size >= 2 * slot_count - 1))
-		return true;
-
-	return false;
-}
-
-/*
- * mab_no_null_split() - ensure the split doesn't fall on a NULL
- * @b_node: the maple_big_node with the data
- * @split: the suggested split location
- * @slot_count: the number of slots in the node being considered.
- *
- * Return: the split location.
- */
-static inline int mab_no_null_split(struct maple_big_node *b_node,
-				    unsigned char split, unsigned char slot_count)
-{
-	if (!b_node->slot[split]) {
-		/*
-		 * If the split is less than the max slot && the right side will
-		 * still be sufficient, then increment the split on NULL.
-		 */
-		if ((split < slot_count - 1) &&
-		    (b_node->b_end - split) > (mt_min_slots[b_node->type]))
-			split++;
-		else
-			split--;
-	}
-	return split;
-}
-
-/*
- * mab_calc_split() - Calculate the split location and if there needs to be two
- * splits.
- * @mas: The maple state
- * @bn: The maple_big_node with the data
- * @mid_split: The second split, if required.  0 otherwise.
- *
- * Return: The first split location.  The middle split is set in @mid_split.
- */
-static inline int mab_calc_split(struct ma_state *mas,
-	 struct maple_big_node *bn, unsigned char *mid_split)
-{
-	unsigned char b_end = bn->b_end;
-	int split = b_end / 2; /* Assume equal split. */
-	unsigned char slot_count = mt_slots[bn->type];
-
-	/*
-	 * To support gap tracking, all NULL entries are kept together and a node cannot
-	 * end on a NULL entry, with the exception of the left-most leaf.  The
-	 * limitation means that the split of a node must be checked for this condition
-	 * and be able to put more data in one direction or the other.
-	 *
-	 * Although extremely rare, it is possible to enter what is known as the 3-way
-	 * split scenario.  The 3-way split comes about by means of a store of a range
-	 * that overwrites the end and beginning of two full nodes.  The result is a set
-	 * of entries that cannot be stored in 2 nodes.  Sometimes, these two nodes can
-	 * also be located in different parent nodes which are also full.  This can
-	 * carry upwards all the way to the root in the worst case.
-	 */
-	if (unlikely(mab_middle_node(bn, split, slot_count))) {
-		split = b_end / 3;
-		*mid_split = split * 2;
-	} else {
-		*mid_split = 0;
-	}
-
-	/* Avoid ending a node on a NULL entry */
-	split = mab_no_null_split(bn, split, slot_count);
-
-	if (unlikely(*mid_split))
-		*mid_split = mab_no_null_split(bn, *mid_split, slot_count);
-
-	return split;
-}
-
-/*
- * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node
- * and set @b_node->b_end to the next free slot.
- * @mas: The maple state
- * @mas_start: The starting slot to copy
- * @mas_end: The end slot to copy (inclusively)
- * @b_node: The maple_big_node to place the data
- * @mab_start: The starting location in maple_big_node to store the data.
- */
-static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start,
-			unsigned char mas_end, struct maple_big_node *b_node,
-			unsigned char mab_start)
-{
-	enum maple_type mt;
-	struct maple_node *node;
-	void __rcu **slots;
-	unsigned long *pivots, *gaps;
-	int i = mas_start, j = mab_start;
-	unsigned char piv_end;
-
-	node = mas_mn(mas);
-	mt = mte_node_type(mas->node);
-	pivots = ma_pivots(node, mt);
-	if (!i) {
-		b_node->pivot[j] = pivots[i++];
-		if (unlikely(i > mas_end))
-			goto complete;
-		j++;
-	}
-
-	piv_end = min(mas_end, mt_pivots[mt]);
-	for (; i < piv_end; i++, j++) {
-		b_node->pivot[j] = pivots[i];
-		if (unlikely(!b_node->pivot[j]))
-			goto complete;
-
-		if (unlikely(mas->max == b_node->pivot[j]))
-			goto complete;
-	}
-
-	b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt);
-
-complete:
-	b_node->b_end = ++j;
-	j -= mab_start;
-	slots = ma_slots(node, mt);
-	memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j);
-	if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) {
-		gaps = ma_gaps(node, mt);
-		memcpy(b_node->gap + mab_start, gaps + mas_start,
-		       sizeof(unsigned long) * j);
-	}
-}
-
 /*
  * mas_leaf_set_meta() - Set the metadata of a leaf if possible.
  * @node: The maple node
@@ -1792,134 +1643,6 @@ static inline void mas_leaf_set_meta(struct maple_node *node,
 		ma_set_meta(node, mt, 0, end);
 }
 
-/*
- * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node.
- * @b_node: the maple_big_node that has the data
- * @mab_start: the start location in @b_node.
- * @mab_end: The end location in @b_node (inclusively)
- * @mas: The maple state with the maple encoded node.
- */
-static inline void mab_mas_cp(struct maple_big_node *b_node,
-			      unsigned char mab_start, unsigned char mab_end,
-			      struct ma_state *mas, bool new_max)
-{
-	int i, j = 0;
-	enum maple_type mt = mte_node_type(mas->node);
-	struct maple_node *node = mte_to_node(mas->node);
-	void __rcu **slots = ma_slots(node, mt);
-	unsigned long *pivots = ma_pivots(node, mt);
-	unsigned long *gaps = NULL;
-	unsigned char end;
-
-	if (mab_end - mab_start > mt_pivots[mt])
-		mab_end--;
-
-	if (!pivots[mt_pivots[mt] - 1])
-		slots[mt_pivots[mt]] = NULL;
-
-	i = mab_start;
-	do {
-		pivots[j++] = b_node->pivot[i++];
-	} while (i <= mab_end && likely(b_node->pivot[i]));
-
-	memcpy(slots, b_node->slot + mab_start,
-	       sizeof(void *) * (i - mab_start));
-
-	if (new_max)
-		mas->max = b_node->pivot[i - 1];
-
-	end = j - 1;
-	if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) {
-		unsigned long max_gap = 0;
-		unsigned char offset = 0;
-
-		gaps = ma_gaps(node, mt);
-		do {
-			gaps[--j] = b_node->gap[--i];
-			if (gaps[j] > max_gap) {
-				offset = j;
-				max_gap = gaps[j];
-			}
-		} while (j);
-
-		ma_set_meta(node, mt, offset, end);
-	} else {
-		mas_leaf_set_meta(node, mt, end);
-	}
-}
-
-/*
- * mas_store_b_node() - Store an @entry into the b_node while also copying the
- * data from a maple encoded node.
- * @wr_mas: the maple write state
- * @b_node: the maple_big_node to fill with data
- * @offset_end: the offset to end copying
- *
- * Return: The actual end of the data stored in @b_node
- */
-static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas,
-		struct maple_big_node *b_node, unsigned char offset_end)
-{
-	unsigned char slot;
-	unsigned char b_end;
-	/* Possible underflow of piv will wrap back to 0 before use. */
-	unsigned long piv;
-	struct ma_state *mas = wr_mas->mas;
-
-	b_node->type = wr_mas->type;
-	b_end = 0;
-	slot = mas->offset;
-	if (slot) {
-		/* Copy start data up to insert. */
-		mas_mab_cp(mas, 0, slot - 1, b_node, 0);
-		b_end = b_node->b_end;
-		piv = b_node->pivot[b_end - 1];
-	} else
-		piv = mas->min - 1;
-
-	if (piv + 1 < mas->index) {
-		/* Handle range starting after old range */
-		b_node->slot[b_end] = wr_mas->content;
-		if (!wr_mas->content)
-			b_node->gap[b_end] = mas->index - 1 - piv;
-		b_node->pivot[b_end++] = mas->index - 1;
-	}
-
-	/* Store the new entry. */
-	mas->offset = b_end;
-	b_node->slot[b_end] = wr_mas->entry;
-	b_node->pivot[b_end] = mas->last;
-
-	/* Appended. */
-	if (mas->last >= mas->max)
-		goto b_end;
-
-	/* Handle new range ending before old range ends */
-	piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type);
-	if (piv > mas->last) {
-		if (offset_end != slot)
-			wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
-							  offset_end);
-
-		b_node->slot[++b_end] = wr_mas->content;
-		if (!wr_mas->content)
-			b_node->gap[b_end] = piv - mas->last + 1;
-		b_node->pivot[b_end] = piv;
-	}
-
-	slot = offset_end + 1;
-	if (slot > mas->end)
-		goto b_end;
-
-	/* Copy end data to the end of the node. */
-	mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end);
-	b_node->b_end--;
-	return;
-
-b_end:
-	b_node->b_end = b_end;
-}
-
 /*
  * mas_prev_sibling() - Find the previous node with the same parent.
  * @mas: the maple state
@@ -1964,25 +1687,6 @@ static inline bool mas_next_sibling(struct ma_state *mas)
 	return true;
 }
 
-/*
- * mas_node_or_none() - Set the enode and state.
- * @mas: the maple state
- * @enode: The encoded maple node.
- *
- * Set the node to the enode and the status.
- */
-static inline void mas_node_or_none(struct ma_state *mas,
-		struct maple_enode *enode)
-{
-	if (enode) {
-		mas->node = enode;
-		mas->status = ma_active;
-	} else {
-		mas->node = NULL;
-		mas->status = ma_none;
-	}
-}
-
 /*
  * mas_wr_node_walk() - Find the correct offset for the index in the @mas.
  *                      If @mas->index cannot be found within the containing
@@ -2016,277 +1720,55 @@ static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas)
 	wr_mas->offset_end = mas->offset = offset;
 }
 
-/*
- * mast_rebalance_next() - Rebalance against the next node
- * @mast: The maple subtree state
- */
-static inline void mast_rebalance_next(struct maple_subtree_state *mast)
-{
-	unsigned char b_end = mast->bn->b_end;
-
-	mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node),
-		   mast->bn, b_end);
-	mast->orig_r->last = mast->orig_r->max;
-}
-
-/*
- * mast_rebalance_prev() - Rebalance against the previous node
- * @mast: The maple subtree state
- */
-static inline void mast_rebalance_prev(struct maple_subtree_state *mast)
+static inline void rebalance_sib(struct ma_state *parent, struct ma_state *sib)
 {
-	unsigned char end = mas_data_end(mast->orig_l) + 1;
-	unsigned char b_end = mast->bn->b_end;
+	*sib = *parent;
+	/* Prioritize move right to pull data left */
+	if (sib->offset < sib->end)
+		sib->offset++;
+	else
+		sib->offset--;
 
-	mab_shift_right(mast->bn, end);
-	mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0);
-	mast->l->min = mast->orig_l->min;
-	mast->orig_l->index = mast->orig_l->min;
-	mast->bn->b_end = end + b_end;
-	mast->l->offset += end;
+	mas_descend(sib);
+	sib->end = mas_data_end(sib);
 }
 
-/*
- * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring
- * the node to the right.  Checking the nodes to the right then the left at each
- * level upwards until root is reached.
- * Data is copied into the @mast->bn.
- * @mast: The maple_subtree_state.
- */
 static inline
-bool mast_spanning_rebalance(struct maple_subtree_state *mast)
+void spanning_sib(struct ma_wr_state *l_wr_mas,
+		struct ma_wr_state *r_wr_mas, struct ma_state *nneighbour)
 {
-	struct ma_state r_tmp = *mast->orig_r;
-	struct ma_state l_tmp = *mast->orig_l;
+	struct ma_state l_tmp = *l_wr_mas->mas;
+	struct ma_state r_tmp = *r_wr_mas->mas;
 	unsigned char depth = 0;
 
 	do {
-		mas_ascend(mast->orig_r);
-		mas_ascend(mast->orig_l);
+		mas_ascend(&r_tmp);
+		mas_ascend(&l_tmp);
 		depth++;
-		if (mast->orig_r->offset < mas_data_end(mast->orig_r)) {
-			mast->orig_r->offset++;
-			do {
-				mas_descend(mast->orig_r);
-				mast->orig_r->offset = 0;
-			} while (--depth);
-
-			mast_rebalance_next(mast);
-			*mast->orig_l = l_tmp;
-			return true;
-		} else if (mast->orig_l->offset != 0) {
-			mast->orig_l->offset--;
+		if (r_tmp.offset < mas_data_end(&r_tmp)) {
+			r_tmp.offset++;
+			mas_descend(&r_tmp);
+			r_tmp.offset = 0;
+			while (--depth)
+				mas_descend(&r_tmp);
+
+			r_tmp.end = mas_data_end(&r_tmp);
+			*nneighbour = r_tmp;
+			return;
+		} else if (l_tmp.offset) {
+			l_tmp.offset--;
 			do {
-				mas_descend(mast->orig_l);
-				mast->orig_l->offset =
-					mas_data_end(mast->orig_l);
+				mas_descend(&l_tmp);
+				l_tmp.offset = mas_data_end(&l_tmp);
 			} while (--depth);
 
-			mast_rebalance_prev(mast);
-			*mast->orig_r = r_tmp;
-			return true;
+			l_tmp.end = l_tmp.offset;
+			*nneighbour = l_tmp;
+			return;
 		}
-	} while (!mte_is_root(mast->orig_r->node));
+	} while (!mte_is_root(r_tmp.node));
 
-	*mast->orig_r = r_tmp;
-	*mast->orig_l = l_tmp;
-	return false;
-}
-
-/*
- * mast_ascend() - Ascend the original left and right maple states.
- * @mast: the maple subtree state.
- *
- * Ascend the original left and right sides.  Set the offsets to point to the
- * data already in the new tree (@mast->l and @mast->r).
- */
-static inline void mast_ascend(struct maple_subtree_state *mast)
-{
-	MA_WR_STATE(wr_mas, mast->orig_r,  NULL);
-	mas_ascend(mast->orig_l);
-	mas_ascend(mast->orig_r);
-
-	mast->orig_r->offset = 0;
-	mast->orig_r->index = mast->r->max;
-	/* last should be larger than or equal to index */
-	if (mast->orig_r->last < mast->orig_r->index)
-		mast->orig_r->last = mast->orig_r->index;
-
-	wr_mas.type = mte_node_type(mast->orig_r->node);
-	mas_wr_node_walk(&wr_mas);
-	/* Set up the left side of things */
-	mast->orig_l->offset = 0;
-	mast->orig_l->index = mast->l->min;
-	wr_mas.mas = mast->orig_l;
-	wr_mas.type = mte_node_type(mast->orig_l->node);
-	mas_wr_node_walk(&wr_mas);
-
-	mast->bn->type = wr_mas.type;
-}
-
-/*
- * mas_new_ma_node() - Create and return a new maple node.  Helper function.
- * @mas: the maple state with the allocations.
- * @b_node: the maple_big_node with the type encoding.
- *
- * Use the node type from the maple_big_node to allocate a new node from the
- * ma_state.  This function exists mainly for code readability.
- *
- * Return: A new maple encoded node
- */
-static inline struct maple_enode
-*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node)
-{
-	return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type);
-}
-
-/*
- * mas_mab_to_node() - Set up right and middle nodes
- *
- * @mas: the maple state that contains the allocations.
- * @b_node: the node which contains the data.
- * @left: The pointer which will have the left node
- * @right: The pointer which may have the right node
- * @middle: the pointer which may have the middle node (rare)
- * @mid_split: the split location for the middle node
- *
- * Return: the split of left.
- */
-static inline unsigned char mas_mab_to_node(struct ma_state *mas,
-	struct maple_big_node *b_node, struct maple_enode **left,
-	struct maple_enode **right, struct maple_enode **middle,
-	unsigned char *mid_split)
-{
-	unsigned char split = 0;
-	unsigned char slot_count = mt_slots[b_node->type];
-
-	*left = mas_new_ma_node(mas, b_node);
-	*right = NULL;
-	*middle = NULL;
-	*mid_split = 0;
-
-	if (b_node->b_end < slot_count) {
-		split = b_node->b_end;
-	} else {
-		split = mab_calc_split(mas, b_node, mid_split);
-		*right = mas_new_ma_node(mas, b_node);
-	}
-
-	if (*mid_split)
-		*middle = mas_new_ma_node(mas, b_node);
-
-	return split;
-
-}
-
-/*
- * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end
- * pointer.
- * @b_node: the big node to add the entry
- * @mas: the maple state to get the pivot (mas->max)
- * @entry: the entry to add, if NULL nothing happens.
- */
-static inline void mab_set_b_end(struct maple_big_node *b_node,
-				 struct ma_state *mas,
-				 void *entry)
-{
-	if (!entry)
-		return;
-
-	b_node->slot[b_node->b_end] = entry;
-	if (mt_is_alloc(mas->tree))
-		b_node->gap[b_node->b_end] = mas_max_gap(mas);
-	b_node->pivot[b_node->b_end++] = mas->max;
-}
-
-/*
- * mas_set_split_parent() - combine_then_separate helper function.  Sets the parent
- * of @mas->node to either @left or @right, depending on @slot and @split
- *
- * @mas: the maple state with the node that needs a parent
- * @left: possible parent 1
- * @right: possible parent 2
- * @slot: the slot the mas->node was placed
- * @split: the split location between @left and @right
- */
-static inline void mas_set_split_parent(struct ma_state *mas,
-					struct maple_enode *left,
-					struct maple_enode *right,
-					unsigned char *slot, unsigned char split)
-{
-	if (mas_is_none(mas))
-		return;
-
-	if ((*slot) <= split)
-		mas_set_parent(mas, mas->node, left, *slot);
-	else if (right)
-		mas_set_parent(mas, mas->node, right, (*slot) - split - 1);
-
-	(*slot)++;
-}
-
-/*
- * mte_mid_split_check() - Check if the next node passes the mid-split
- * @l: Pointer to left encoded maple node.
- * @m: Pointer to middle encoded maple node.
- * @r: Pointer to right encoded maple node.
- * @slot: The offset
- * @split: The split location.
- * @mid_split: The middle split.
- */
-static inline void mte_mid_split_check(struct maple_enode **l,
-				       struct maple_enode **r,
-				       struct maple_enode *right,
-				       unsigned char slot,
-				       unsigned char *split,
-				       unsigned char mid_split)
-{
-	if (*r == right)
-		return;
-
-	if (slot < mid_split)
-		return;
-
-	*l = *r;
-	*r = right;
-	*split = mid_split;
-}
-
-/*
- * mast_set_split_parents() - Helper function to set three nodes parents.  Slot
- * is taken from @mast->l.
- * @mast: the maple subtree state
- * @left: the left node
- * @right: the right node
- * @split: the split location.
- */
-static inline void mast_set_split_parents(struct maple_subtree_state *mast,
-					  struct maple_enode *left,
-					  struct maple_enode *middle,
-					  struct maple_enode *right,
-					  unsigned char split,
-					  unsigned char mid_split)
-{
-	unsigned char slot;
-	struct maple_enode *l = left;
-	struct maple_enode *r = right;
-
-	if (mas_is_none(mast->l))
-		return;
-
-	if (middle)
-		r = middle;
-
-	slot = mast->l->offset;
-
-	mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-	mas_set_split_parent(mast->l, l, r, &slot, split);
-
-	mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-	mas_set_split_parent(mast->m, l, r, &slot, split);
-
-	mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-	mas_set_split_parent(mast->r, l, r, &slot, split);
+	WARN_ON_ONCE(1);
 }
 
 /*
@@ -2419,120 +1901,109 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 }
 
 /*
- * mas_wmb_replace() - Write memory barrier and replace
- * @mas: The maple state
- * @old_enode: The old maple encoded node that is being replaced.
- * @new_height: The new height of the tree as a result of the operation
+ * node_copy() - Copy from one node to another.
  *
- * Updates gap as necessary.
+ * @mas: The maple state
+ * @src: The source node
+ * @start: The offset into the src to start copying
+ * @size: The size to copy (non-zero)
+ * @s_max: The source node max
+ * @s_mt: The source maple node type
+ * @dst: The destination
+ * @d_start: The start location in the destination node
+ * @d_mt: The destination maple node type
  */
-static inline void mas_wmb_replace(struct ma_state *mas,
-		struct maple_enode *old_enode, unsigned char new_height)
+static inline
+unsigned long node_copy(struct ma_state *mas, struct maple_node *src,
+	unsigned char start, unsigned char size, unsigned long s_max,
+	enum maple_type s_mt, struct maple_node *dst, unsigned char d_start,
+	enum maple_type d_mt)
 {
-	/* Insert the new data in the tree */
-	mas_topiary_replace(mas, old_enode, new_height);
-
-	if (mte_is_leaf(mas->node))
-		return;
-
-	mas_update_gap(mas);
-}
+	unsigned long *s_pivots, *d_pivots;
+	void __rcu **s_slots, **d_slots;
+	unsigned long *s_gaps, *d_gaps;
+	unsigned long d_max;
 
-/*
- * mast_cp_to_nodes() - Copy data out to nodes.
- * @mast: The maple subtree state
- * @left: The left encoded maple node
- * @middle: The middle encoded maple node
- * @right: The right encoded maple node
- * @split: The location to split between left and (middle ? middle : right)
- * @mid_split: The location to split between middle and right.
- */
-static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
-	struct maple_enode *left, struct maple_enode *middle,
-	struct maple_enode *right, unsigned char split, unsigned char mid_split)
-{
-	bool new_lmax = true;
+	d_slots = ma_slots(dst, d_mt) + d_start;
+	d_pivots = ma_pivots(dst, d_mt) + d_start;
+	s_slots = ma_slots(src, s_mt) + start;
+	s_pivots = ma_pivots(src, s_mt) + start;
+	memcpy(d_slots, s_slots, size * sizeof(void __rcu *));
+	if (!ma_is_leaf(d_mt) && s_mt == maple_copy) {
+		struct maple_enode *edst = mt_mk_node(dst, d_mt);
 
-	mas_node_or_none(mast->l, left);
-	mas_node_or_none(mast->m, middle);
-	mas_node_or_none(mast->r, right);
 
-	mast->l->min = mast->orig_l->min;
-	if (split == mast->bn->b_end) {
-		mast->l->max = mast->orig_r->max;
-		new_lmax = false;
+		for (int i = 0; i < size; i++)
+			mas_set_parent(mas,
+				       mt_slot_locked(mas->tree, d_slots, i),
+				       edst, d_start + i);
 	}
 
-	mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax);
-
-	if (middle) {
-		mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true);
-		mast->m->min = mast->bn->pivot[split] + 1;
-		split = mid_split;
+	d_gaps = ma_gaps(dst, d_mt);
+	if (d_gaps) {
+		s_gaps = ma_gaps(src, s_mt) + start;
+		d_gaps += d_start;
+		memcpy(d_gaps, s_gaps, size * sizeof(unsigned long));
 	}
 
-	mast->r->max = mast->orig_r->max;
-	if (right) {
-		mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false);
-		mast->r->min = mast->bn->pivot[split] + 1;
-	}
-}
+	if (start + size - 1 < mt_pivots[s_mt])
+		d_max = s_pivots[size - 1];
+	else
+		d_max = s_max;
 
-/*
- * mast_combine_cp_left - Copy in the original left side of the tree into the
- * combined data set in the maple subtree state big node.
- * @mast: The maple subtree state
- */
-static inline void mast_combine_cp_left(struct maple_subtree_state *mast)
-{
-	unsigned char l_slot = mast->orig_l->offset;
+	if (d_start + size <= mt_pivots[d_mt])
+		d_pivots[size - 1] = d_max;
 
-	if (!l_slot)
-		return;
+	size--;
+	if (size)
+		memcpy(d_pivots, s_pivots, size * sizeof(unsigned long));
 
-	mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0);
+	return d_max;
 }
 
 /*
- * mast_combine_cp_right: Copy in the original right side of the tree into the
- * combined data set in the maple subtree state big node.
- * @mast: The maple subtree state
+ * node_finalise() - Zero out unused area and populate metadata
+ * @node: The maple node
+ * @mt: The maple node type
+ * @end: The end of the used area
  */
-static inline void mast_combine_cp_right(struct maple_subtree_state *mast)
+static inline
+void node_finalise(struct maple_node *node, enum maple_type mt,
+		   unsigned char end)
 {
-	if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max)
-		return;
+	unsigned char max_end = mt_slots[mt];
+	unsigned char size;
+	unsigned long *gaps;
+	unsigned char gap_slot;
 
-	mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1,
-		   mt_slot_count(mast->orig_r->node), mast->bn,
-		   mast->bn->b_end);
-	mast->orig_r->last = mast->orig_r->max;
-}
+	gaps = ma_gaps(node, mt);
+	if (end < max_end - 1) {
+		size = max_end - end;
+		memset(ma_slots(node, mt) + end, 0, size * sizeof(void *));
 
-/*
- * mast_sufficient: Check if the maple subtree state has enough data in the big
- * node to create at least one sufficient node
- * @mast: the maple subtree state
- */
-static inline bool mast_sufficient(struct maple_subtree_state *mast)
-{
-	if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node))
-		return true;
+		if (gaps)
+			memset(gaps + end, 0, size * sizeof(unsigned long));
 
-	return false;
-}
+		if (--size)
+			memset(ma_pivots(node, mt) + end, 0, size * sizeof(unsigned long));
+	}
 
-/*
- * mast_overflow: Check if there is too much data in the subtree state for a
- * single node.
- * @mast: The maple subtree state
- */
-static inline bool mast_overflow(struct maple_subtree_state *mast)
-{
-	if (mast->bn->b_end > mt_slot_count(mast->orig_l->node))
-		return true;
+	gap_slot = 0;
+	if (gaps && !ma_is_leaf(mt)) {
+		unsigned long max_gap;
 
-	return false;
+		max_gap = 0;
+		for (int i = 0; i <= end; i++)
+			if (gaps[i] > max_gap) {
+				gap_slot = i;
+				max_gap = gaps[i];
+			}
+	}
+
+	if (mt == maple_arange_64)
+		ma_set_meta(node, mt, gap_slot, end - 1);
+	else if (end <= max_end - 1)
+		ma_set_meta(node, mt, gap_slot, end - 1);
 }
 
 static inline void *mtree_range_walk(struct ma_state *mas)
@@ -2596,480 +2067,662 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 }
 
 /*
- * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers.
- * @mas: The starting maple state
- * @mast: The maple_subtree_state, keeps track of 4 maple states.
- * @count: The estimated count of iterations needed.
- *
- * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root
- * is hit.  First @b_node is split into two entries which are inserted into the
- * next iteration of the loop.  @b_node is returned populated with the final
- * iteration. @mas is used to obtain allocations.  orig_l_mas keeps track of the
- * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last
- * to account of what has been copied into the new sub-tree.  The update of
- * orig_l_mas->last is used in mas_consume to find the slots that will need to
- * be either freed or destroyed.  orig_l_mas->depth keeps track of the height of
- * the new sub-tree in case the sub-tree becomes the full tree.
- */
-static void mas_spanning_rebalance(struct ma_state *mas,
-		struct maple_subtree_state *mast, unsigned char count)
-{
-	unsigned char split, mid_split;
-	unsigned char slot = 0;
-	unsigned char new_height = 0; /* used if node is a new root */
-	struct maple_enode *left = NULL, *middle = NULL, *right = NULL;
-	struct maple_enode *old_enode;
+ * mas_wmb_replace() - Write memory barrier and replace
+ * @mas: The maple state
+ * @cp: The maple copy node
+ *
+ * Updates gap as necessary.
+ */
+static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp)
+{
+	struct maple_enode *old_enode;
+
+	old_enode = mas->node;
+	mas->node = mt_slot_locked(mas->tree, cp->slot, 0);
+	/* Insert the new data in the tree */
+	mas_topiary_replace(mas, old_enode, cp->height);
+	if (!mte_is_leaf(mas->node))
+		mas_update_gap(mas);
+
+	mtree_range_walk(mas);
+}
+
+
+/*
+ * cp_leaf_init() - Initialize a maple_copy node for the leaf level of a
+ * spanning store
+ * @cp: The maple copy node
+ * @mas: The maple state
+ * @l_wr_mas: The left write state of the spanning store
+ * @r_wr_mas: The right write state of the spanning store
+ */
+static inline void cp_leaf_init(struct maple_copy *cp,
+		struct ma_state *mas, struct ma_wr_state *l_wr_mas,
+		struct ma_wr_state *r_wr_mas)
+{
+	unsigned char end = 0;
+
+	/*
+	 * WARNING: The use of RCU_INIT_POINTER() makes it extremely important
+	 * to not expose the maple_copy node to any readers.  Exposure may
+	 * result in buggy code when a compiler reorders the instructions.
+	 */
+
+	cp->height = 1;
+	/* Create entries to insert including split entries to left and right */
+	if (l_wr_mas->r_min < mas->index) {
+		end++;
+		RCU_INIT_POINTER(cp->slot[0], l_wr_mas->content);
+		cp->pivot[0] = mas->index - 1;
+	}
+	RCU_INIT_POINTER(cp->slot[end], l_wr_mas->entry);
+	cp->pivot[end] = mas->last;
+
+	if (r_wr_mas->end_piv > mas->last) {
+		end++;
+		RCU_INIT_POINTER(cp->slot[end],
+				 r_wr_mas->slots[r_wr_mas->offset_end]);
+		cp->pivot[end] = r_wr_mas->end_piv;
+	}
+
+	cp->min = l_wr_mas->r_min;
+	cp->max = cp->pivot[end];
+	cp->end = end;
+}
+
+/*
+ * cp_data_calc() - Calculate the size of the data (1 indexed).
+ * @cp: The maple copy struct with the new data populated.
+ * @l_wr_mas: The maple write state containing the data to the left of the write
+ * @r_wr_mas: The maple write state containing the data to the right of the
+ * write
+ *
+ * cp->data is a size (not indexed by 0).
+ */
+static inline void cp_data_calc(struct maple_copy *cp,
+		struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas)
+{
+
+	/* Add 1 every time for the 0th element */
+	cp->data = l_wr_mas->mas->offset;
+	/* Add the new data and any partial overwrites */
+	cp->data += cp->end + 1;
+	/* Data from right (offset + 1 to end), +1 for zero */
+	cp->data += r_wr_mas->mas->end - r_wr_mas->offset_end;
+}
+
+static bool data_fits(struct ma_state *sib, struct ma_state *mas,
+		struct maple_copy *cp)
+{
+	unsigned char new_data;
+	enum maple_type type;
+	unsigned char space;
+	unsigned char end;
+
+	type = mte_node_type(mas->node);
+	space = 2 * mt_slots[type];
+	end = sib->end;
+
+	new_data = end + 1 + cp->data;
+	if (new_data > space)
+		return false;
+
+	/*
+	 * This is off by one by design.  The extra space is left to reduce
+	 * jitter in operations that add then remove two entries.
+	 *
+	 * end is an index while new space and data are both sizes.  Adding one
+	 * to end to convert the index to a size means that the below
+	 * calculation should be <=, but we want to keep an extra space in nodes
+	 * to reduce jitter.
+	 *
+	 * Note that it is still possible to get a full node on the left by the
+	 * NULL landing exactly on the split.  The NULL ending of a node happens
+	 * in the dst_setup() function, where we will either increase the split
+	 * by one or decrease it by one, if possible.  In the case of split
+	 * (this case), it is always possible to shift the spilt by one - again
+	 * because there is at least one slot free by the below checking.
+	 */
+	if (new_data < space)
+		return true;
+
+	return false;
+}
+
+static inline void push_data_sib(struct maple_copy *cp, struct ma_state *mas,
+		struct ma_state *sib, struct ma_state *parent)
+{
+
+	if (mte_is_root(mas->node))
+		goto no_push;
+
+
+	*sib = *parent;
+	if (sib->offset) {
+		sib->offset--;
+		mas_descend(sib);
+		sib->end = mas_data_end(sib);
+		if (data_fits(sib, mas, cp))	/* Push left */
+			return;
+
+		*sib = *parent;
+	}
+
+	if (sib->offset >= sib->end)
+		goto no_push;
+
+	sib->offset++;
+	mas_descend(sib);
+	sib->end = mas_data_end(sib);
+	if (data_fits(sib, mas, cp))		/* Push right*/
+		return;
+
+no_push:
+	sib->end = 0;
+}
+
+/*
+ * rebalance_data() - Calculate the @cp data, populate @sib if insufficient or
+ * if the data can be pushed into a sibling.
+ * @cp: The maple copy node
+ * @wr_mas: The left write maple state
+ * @sib: The maple state of the sibling.
+ *
+ * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to
+ * indicate it will not be used.
+ *
+ */
+static inline void rebalance_data(struct maple_copy *cp,
+		struct ma_wr_state *wr_mas, struct ma_state *sib,
+		struct ma_state *parent)
+{
+	cp_data_calc(cp, wr_mas, wr_mas);
+	sib->end = 0;
+	if (cp->data > mt_slots[wr_mas->type]) {
+		push_data_sib(cp, wr_mas->mas, sib, parent);
+		if (sib->end)
+			goto use_sib;
+	} else if (cp->data <= mt_min_slots[wr_mas->type]) {
+		if ((wr_mas->mas->min != 0) ||
+		    (wr_mas->mas->max != ULONG_MAX)) {
+			rebalance_sib(parent, sib);
+			goto use_sib;
+		}
+	}
+
+	return;
+
+use_sib:
+
+	cp->data += sib->end + 1;
+}
+
+/*
+ * spanning_data() - Calculate the @cp data and populate @sib if insufficient
+ * @cp: The maple copy node
+ * @l_wr_mas: The left write maple state
+ * @r_wr_mas: The right write maple state
+ * @sib: The maple state of the sibling.
+ *
+ * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to
+ * indicate it will not be used.
+ */
+static inline void spanning_data(struct maple_copy *cp,
+		struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas,
+		struct ma_state *sib)
+{
+	cp_data_calc(cp, l_wr_mas, r_wr_mas);
+	if (((l_wr_mas->mas->min != 0) || (r_wr_mas->mas->max != ULONG_MAX)) &&
+	    (cp->data <= mt_min_slots[l_wr_mas->type])) {
+		spanning_sib(l_wr_mas, r_wr_mas, sib);
+		cp->data += sib->end + 1;
+	} else {
+		sib->end = 0;
+	}
+}
+
+/*
+ * dst_setup() - Set up one or more destinations for the new data.
+ * @cp: The maple copy node
+ * @mas: The maple state
+ * @mt: The source node type
+ */
+static inline
+void dst_setup(struct maple_copy *cp, struct ma_state *mas, enum maple_type mt)
+{
+	/* Data is 1 indexed, every src has +1 added.  */
 
-	MA_STATE(l_mas, mas->tree, mas->index, mas->index);
-	MA_STATE(r_mas, mas->tree, mas->index, mas->last);
-	MA_STATE(m_mas, mas->tree, mas->index, mas->index);
+	if (cp->data <= mt_slots[mt]) {
+		cp->split = cp->data - 1;
+		cp->d_count = 1;
+		goto node_setup;
+	}
 
-	/*
-	 * The tree needs to be rebalanced and leaves need to be kept at the same level.
-	 * Rebalancing is done by use of the ``struct maple_topiary``.
-	 */
-	mast->l = &l_mas;
-	mast->m = &m_mas;
-	mast->r = &r_mas;
-	l_mas.status = r_mas.status = m_mas.status = ma_none;
+	cp->split = (cp->data - 1) / 2;
+	cp->d_count = 2;
+	if (cp->data < mt_slots[mt] * 2)
+		goto node_setup;
 
-	/* Check if this is not root and has sufficient data.  */
-	if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) &&
-	    unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type]))
-		mast_spanning_rebalance(mast);
+	if (cp->data == mt_slots[mt] * 2) {
+		unsigned char off;
+		unsigned char s;
 
-	/*
-	 * Each level of the tree is examined and balanced, pushing data to the left or
-	 * right, or rebalancing against left or right nodes is employed to avoid
-	 * rippling up the tree to limit the amount of churn.  Once a new sub-section of
-	 * the tree is created, there may be a mix of new and old nodes.  The old nodes
-	 * will have the incorrect parent pointers and currently be in two trees: the
-	 * original tree and the partially new tree.  To remedy the parent pointers in
-	 * the old tree, the new data is swapped into the active tree and a walk down
-	 * the tree is performed and the parent pointers are updated.
-	 * See mas_topiary_replace() for more information.
-	 */
-	while (count--) {
-		mast->bn->b_end--;
-		mast->bn->type = mte_node_type(mast->orig_l->node);
-		split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle,
-					&mid_split);
-		mast_set_split_parents(mast, left, middle, right, split,
-				       mid_split);
-		mast_cp_to_nodes(mast, left, middle, right, split, mid_split);
-		new_height++;
+		if (!ma_is_leaf(mt))
+			goto node_setup;
 
 		/*
-		 * Copy data from next level in the tree to mast->bn from next
-		 * iteration
+		 * Leaf nodes are a bit tricky because we cannot assume the data
+		 * can fit due to the NULL limitation on node ends.
 		 */
-		memset(mast->bn, 0, sizeof(struct maple_big_node));
-		mast->bn->type = mte_node_type(left);
-
-		/* Root already stored in l->node. */
-		if (mas_is_root_limits(mast->l))
-			goto new_root;
-
-		mast_ascend(mast);
-		mast_combine_cp_left(mast);
-		l_mas.offset = mast->bn->b_end;
-		mab_set_b_end(mast->bn, &l_mas, left);
-		mab_set_b_end(mast->bn, &m_mas, middle);
-		mab_set_b_end(mast->bn, &r_mas, right);
-
-		/* Copy anything necessary out of the right node. */
-		mast_combine_cp_right(mast);
-		mast->orig_l->last = mast->orig_l->max;
-
-		if (mast_sufficient(mast)) {
-			if (mast_overflow(mast))
-				continue;
+		off = cp->split;
+		for (s = 0; s < cp->s_count; s++) {
+			unsigned char s_off;
 
-			if (mast->orig_l->node == mast->orig_r->node) {
-			       /*
-				* The data in b_node should be stored in one
-				* node and in the tree
-				*/
-				slot = mast->l->offset;
+			s_off = cp->src[s].end - cp->src[s].start;
+			if (s_off >= off)
 				break;
-			}
 
-			continue;
+			s_off++;
+			off -= s_off;
 		}
 
-		/* May be a new root stored in mast->bn */
-		if (mas_is_root_limits(mast->orig_l))
-			break;
-
-		mast_spanning_rebalance(mast);
+		off += cp->src[s].start;
+		if (ma_slots(cp->src[s].node, cp->src[s].mt)[off])
+			goto node_setup;
 
-		/* rebalancing from other nodes may require another loop. */
-		if (!count)
-			count++;
-	}
+		cp->split++;
+		if (cp->split < mt_slots[mt])
+			goto node_setup;
 
-	l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)),
-				mte_node_type(mast->orig_l->node));
+		cp->split -= 2;
+		if (cp->data - 2 - cp->split < mt_slots[mt])
+			goto node_setup;
 
-	mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true);
-	new_height++;
-	mas_set_parent(mas, left, l_mas.node, slot);
-	if (middle)
-		mas_set_parent(mas, middle, l_mas.node, ++slot);
+	}
 
-	if (right)
-		mas_set_parent(mas, right, l_mas.node, ++slot);
+	/* No other choice but to 3-way split the data */
+	cp->split = (cp->data + 2) / 3;
+	cp->d_count = 3;
 
-	if (mas_is_root_limits(mast->l)) {
-new_root:
-		mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas));
-		while (!mte_is_root(mast->orig_l->node))
-			mast_ascend(mast);
-	} else {
-		mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent;
+node_setup:
+	for (int i = 0; i < cp->d_count; i++) {
+		cp->dst[i].mt = mt;
+		cp->dst[i].node = ma_mnode_ptr(mas_pop_node(mas));
 	}
-
-	old_enode = mast->orig_l->node;
-	mas->depth = l_mas.depth;
-	mas->node = l_mas.node;
-	mas->min = l_mas.min;
-	mas->max = l_mas.max;
-	mas->offset = l_mas.offset;
-	mas_wmb_replace(mas, old_enode, new_height);
-	mtree_range_walk(mas);
-	return;
 }
 
-/*
- * mas_rebalance() - Rebalance a given node.
- * @mas: The maple state
- * @b_node: The big maple node.
- *
- * Rebalance two nodes into a single node or two new nodes that are sufficient.
- * Continue upwards until tree is sufficient.
- */
-static inline void mas_rebalance(struct ma_state *mas,
-				struct maple_big_node *b_node)
+static inline void append_mas_cp(struct maple_copy *cp,
+	struct ma_state *mas, unsigned char start, unsigned char end)
 {
-	char empty_count = mas_mt_height(mas);
-	struct maple_subtree_state mast;
-	unsigned char shift, b_end = ++b_node->b_end;
-
-	MA_STATE(l_mas, mas->tree, mas->index, mas->last);
-	MA_STATE(r_mas, mas->tree, mas->index, mas->last);
+	struct maple_node *node;
+	enum maple_type mt;
+	unsigned char count;
 
-	trace_ma_op(TP_FCT, mas);
+	count = cp->s_count;
+	node = mas_mn(mas);
+	mt = mte_node_type(mas->node);
+	cp->src[count].node = node;
+	cp->src[count].mt = mt;
+	if (mas->end <= end)
+		cp->src[count].max = mas->max;
+	else
+		cp->src[count].max = ma_pivots(node, mt)[end];
 
-	/*
-	 * Rebalancing occurs if a node is insufficient.  Data is rebalanced
-	 * against the node to the right if it exists, otherwise the node to the
-	 * left of this node is rebalanced against this node.  If rebalancing
-	 * causes just one node to be produced instead of two, then the parent
-	 * is also examined and rebalanced if it is insufficient.  Every level
-	 * tries to combine the data in the same way.  If one node contains the
-	 * entire range of the tree, then that node is used as a new root node.
-	 */
+	cp->src[count].start = start;
+	cp->src[count].end = end;
+	cp->s_count++;
+}
 
-	mast.orig_l = &l_mas;
-	mast.orig_r = &r_mas;
-	mast.bn = b_node;
-	mast.bn->type = mte_node_type(mas->node);
+static inline void append_wr_mas_cp(struct maple_copy *cp,
+	struct ma_wr_state *wr_mas, unsigned char start, unsigned char end)
+{
+	unsigned char count;
 
-	l_mas = r_mas = *mas;
+	count = cp->s_count;
+	cp->src[count].node = wr_mas->node;
+	cp->src[count].mt = wr_mas->type;
+	if (wr_mas->mas->end <= end)
+		cp->src[count].max = wr_mas->mas->max;
+	else
+		cp->src[count].max = wr_mas->pivots[end];
 
-	if (mas_next_sibling(&r_mas)) {
-		mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end);
-		r_mas.last = r_mas.index = r_mas.max;
-	} else {
-		mas_prev_sibling(&l_mas);
-		shift = mas_data_end(&l_mas) + 1;
-		mab_shift_right(b_node, shift);
-		mas->offset += shift;
-		mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0);
-		b_node->b_end = shift + b_end;
-		l_mas.index = l_mas.last = l_mas.min;
-	}
+	cp->src[count].start = start;
+	cp->src[count].end = end;
+	cp->s_count++;
+}
 
-	return mas_spanning_rebalance(mas, &mast, empty_count);
+static inline void init_cp_src(struct maple_copy *cp)
+{
+	cp->src[cp->s_count].node = ma_mnode_ptr(cp);
+	cp->src[cp->s_count].mt = maple_copy;
+	cp->src[cp->s_count].max = cp->max;
+	cp->src[cp->s_count].start = 0;
+	cp->src[cp->s_count].end = cp->end;
+	cp->s_count++;
 }
 
 /*
- * mas_split_final_node() - Split the final node in a subtree operation.
- * @mast: the maple subtree state
- * @mas: The maple state
+ * multi_src_setup() - Set the @cp node up with multiple sources to copy from.
+ * @cp: The maple copy node
+ * @l_wr_mas: The left write maple state
+ * @r_wr_mas: The right write maple state
+ * @sib: The sibling maple state
+ *
+ * Note: @sib->end == 0 indicates no sibling will be used.
  */
-static inline void mas_split_final_node(struct maple_subtree_state *mast,
-					struct ma_state *mas)
+static inline
+void multi_src_setup(struct maple_copy *cp, struct ma_wr_state *l_wr_mas,
+		struct ma_wr_state *r_wr_mas, struct ma_state *sib)
 {
-	struct maple_enode *ancestor;
+	cp->s_count = 0;
+	if (sib->end && sib->max < l_wr_mas->mas->min)
+		append_mas_cp(cp, sib, 0, sib->end);
 
-	if (mte_is_root(mas->node)) {
-		if (mt_is_alloc(mas->tree))
-			mast->bn->type = maple_arange_64;
-		else
-			mast->bn->type = maple_range_64;
+	/* Copy left 0 - offset */
+	if (l_wr_mas->mas->offset) {
+		unsigned char off = l_wr_mas->mas->offset - 1;
+
+		append_wr_mas_cp(cp, l_wr_mas, 0, off);
+		cp->src[cp->s_count - 1].max = cp->min - 1;
 	}
-	/*
-	 * Only a single node is used here, could be root.
-	 * The Big_node data should just fit in a single node.
-	 */
-	ancestor = mas_new_ma_node(mas, mast->bn);
-	mas_set_parent(mas, mast->l->node, ancestor, mast->l->offset);
-	mas_set_parent(mas, mast->r->node, ancestor, mast->r->offset);
-	mte_to_node(ancestor)->parent = mas_mn(mas)->parent;
 
-	mast->l->node = ancestor;
-	mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true);
-	mas->offset = mast->bn->b_end - 1;
+	init_cp_src(cp);
+
+	/* Copy right either from offset or offset + 1 pending on r_max */
+	if (r_wr_mas->mas->end != r_wr_mas->offset_end)
+		append_wr_mas_cp(cp, r_wr_mas, r_wr_mas->offset_end + 1,
+			       r_wr_mas->mas->end);
+
+	if (sib->end && sib->min > r_wr_mas->mas->max)
+		append_mas_cp(cp, sib, 0, sib->end);
 }
 
-/*
- * mast_fill_bnode() - Copy data into the big node in the subtree state
- * @mast: The maple subtree state
- * @mas: the maple state
- * @skip: The number of entries to skip for new nodes insertion.
- */
-static inline void mast_fill_bnode(struct maple_subtree_state *mast,
-					 struct ma_state *mas,
-					 unsigned char skip)
+static inline
+void cp_data_write(struct maple_copy *cp, struct ma_state *mas)
 {
-	bool cp = true;
+	struct maple_node *dst, *src;
+	unsigned char s, d;
+	unsigned char dst_offset;
+	unsigned char data_offset;
+	unsigned char src_end, s_offset;
 	unsigned char split;
+	unsigned long s_max, d_max;
+	unsigned char dst_size;
+	enum maple_type s_mt, d_mt;
+
+	data_offset = 0;
+	s = d = 0;
+	/* Readability help */
+	src = cp->src[s].node;
+	dst = cp->dst[d].node;
+	s_offset = cp->src[s].start;
+	src_end = cp->src[s].end;
+	split = cp->split;
+	s_max = cp->src[s].max;
+	s_mt = cp->src[s].mt;
+	d_mt = cp->dst[d].mt;
+	do {
+		dst_offset = 0;
+		d_max = 0;
+		dst = cp->dst[d].node;
+		d_mt = cp->dst[d].mt;
+		dst_size = split + 1;
 
-	memset(mast->bn, 0, sizeof(struct maple_big_node));
+		while (dst_size) {
+			unsigned char size;
 
-	if (mte_is_root(mas->node)) {
-		cp = false;
-	} else {
-		mas_ascend(mas);
-		mas->offset = mte_parent_slot(mas->node);
-	}
+			if (src_end - s_offset + 1 < dst_size)
+				size = src_end - s_offset + 1;
+			else
+				size = dst_size;
+
+			d_max = node_copy(mas, src, s_offset, size, s_max, s_mt,
+					  dst, dst_offset, d_mt);
+
+			dst_offset += size;
+			s_offset += size;
+			if (s_offset > src_end) {
+				/* This source is exhausted */
+				s++;
+				if (s >= cp->s_count) {
+					cp->dst[d].max = d_max;
+					node_finalise(dst, d_mt, dst_offset);
+					return;
+				}
+				/* Reset local src */
+				src = cp->src[s].node;
+				s_offset = cp->src[s].start;
+				src_end = cp->src[s].end;
+				s_max = cp->src[s].max;
+				s_mt = cp->src[s].mt;
+			}
+
+			dst_size -= size;
+			data_offset += size;
+		}
 
-	if (cp && mast->l->offset)
-		mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0);
+		split = cp->split;
+		cp->dst[d].max = d_max;
+		/* Handle null entries */
+		if (cp->dst[d].max != ULONG_MAX &&
+		    !ma_slots(dst, d_mt)[dst_offset - 1]) {
+			if (s_offset == cp->src[s].start) {
+				s--;
+				src = cp->src[s].node;
+				src_end = cp->src[s].end;
+				s_max = cp->src[s].max;
+				s_mt = cp->src[s].mt;
+				s_offset = src_end;
+			} else {
+				s_offset--;
+			}
+			/* Set dst max and clear pivot */
+			split++;
+			data_offset--;
+			dst_offset--;
+			cp->dst[d].max = ma_pivots(dst, d_mt)[dst_offset - 1];
+		}
 
-	split = mast->bn->b_end;
-	mab_set_b_end(mast->bn, mast->l, mast->l->node);
-	mast->r->offset = mast->bn->b_end;
-	mab_set_b_end(mast->bn, mast->r, mast->r->node);
-	if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max)
-		cp = false;
+		node_finalise(dst, d_mt, dst_offset);
+		++d; /* Next destination */
+		if (d == cp->d_count - 1)
+			split = cp->data - data_offset;
 
-	if (cp)
-		mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
-			   mast->bn, mast->bn->b_end);
+		if (d >= cp->d_count) {
+			WARN_ON(data_offset < cp->data);
+			return;
+		}
 
-	mast->bn->b_end--;
-	mast->bn->type = mte_node_type(mas->node);
+	} while (data_offset <= cp->data);
 }
 
 /*
- * mast_split_data() - Split the data in the subtree state big node into regular
- * nodes.
- * @mast: The maple subtree state
+ * cp_dst_to_slots() - Migrate the maple copy destination to the maple copy
+ * slots
+ * @cp: The maple copy node
+ * @min: The minimal value represented
+ * @max: The maximum value represented
  * @mas: The maple state
- * @split: The location to split the big node
  */
-static inline void mast_split_data(struct maple_subtree_state *mast,
-	   struct ma_state *mas, unsigned char split)
+static inline void cp_dst_to_slots(struct maple_copy *cp, unsigned long min,
+		unsigned long max, struct ma_state *mas)
 {
-	unsigned char p_slot;
+	unsigned char d;
+	unsigned long slot_min = min;
 
-	mab_mas_cp(mast->bn, 0, split, mast->l, true);
-	mte_set_pivot(mast->r->node, 0, mast->r->max);
-	mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false);
-	mast->l->offset = mte_parent_slot(mas->node);
-	mast->l->max = mast->bn->pivot[split];
-	mast->r->min = mast->l->max + 1;
-	if (mte_is_leaf(mas->node))
-		return;
+	for (d = 0; d < cp->d_count; d++) {
+		struct maple_node *mn = cp->dst[d].node;
+		enum maple_type mt = cp->dst[d].mt;
+		unsigned long slot_max = cp->dst[d].max;
 
-	p_slot = mast->orig_l->offset;
-	mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
-			     &p_slot, split);
-	mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
-			     &p_slot, split);
-}
+		/*
+		 * Warning, see cp_leaf_init() comment and rcu_assign_pointer()
+		 * documentation.  Since these are new nodes, there are no
+		 * read-side operations that can view them until they are
+		 * inserted into the tree after an rcu_assign_pointer() call.
+		 */
+		ma_init_slot(&cp->slot[d], mn, mt);
+		cp->pivot[d] = slot_max;
+		if (mt_is_alloc(mas->tree)) {
+			if (ma_is_leaf(mt)) {
+				cp->gap[d] = ma_leaf_max_gap(mn, mt, slot_min,
+						 slot_max, ma_pivots(mn, mt),
+						 ma_slots(mn, mt));
+			} else {
+				unsigned long *gaps = ma_gaps(mn, mt);
 
-/*
- * mas_push_data() - Instead of splitting a node, it is beneficial to push the
- * data to the right or left node if there is room.
- * @mas: The maple state
- * @mast: The maple subtree state
- * @left: Push left or not.
- *
- * Keeping the height of the tree low means faster lookups.
- *
- * Return: True if pushed, false otherwise.
- */
-static inline bool mas_push_data(struct ma_state *mas,
-				struct maple_subtree_state *mast, bool left)
-{
-	unsigned char slot_total = mast->bn->b_end;
-	unsigned char end, space, split;
+				if (gaps) {
+					unsigned char gap_slot;
+
+					gap_slot = ma_meta_gap(mn);
+					cp->gap[d] = gaps[gap_slot];
+				}
+			}
+		}
+		slot_min = slot_max + 1;
+	}
 
-	MA_STATE(tmp_mas, mas->tree, mas->index, mas->last);
-	tmp_mas = *mas;
-	tmp_mas.depth = mast->l->depth;
+	cp->end = cp->d_count - 1;
+	cp->min = min;
+	cp->max = max;
+}
 
-	if (left && !mas_prev_sibling(&tmp_mas))
-		return false;
-	else if (!left && !mas_next_sibling(&tmp_mas))
+static inline bool cp_is_new_root(struct maple_copy *cp, struct ma_state *mas)
+{
+	if (cp->min || cp->max != ULONG_MAX)
 		return false;
 
-	end = mas_data_end(&tmp_mas);
-	slot_total += end;
-	space = 2 * mt_slot_count(mas->node) - 2;
-	/* -2 instead of -1 to ensure there isn't a triple split */
-	if (ma_is_leaf(mast->bn->type))
-		space--;
+	if (cp->d_count != 1) {
+		enum maple_type mt = maple_arange_64;
 
-	if (mas->max == ULONG_MAX)
-		space--;
+		if (!mt_is_alloc(mas->tree))
+			mt = maple_range_64;
 
-	if (slot_total >= space)
-		return false;
+		cp->data = cp->d_count;
+		cp->s_count = 0;
+		dst_setup(cp, mas, mt);
+		init_cp_src(cp);
+		node_copy(mas, cp->src[0].node, 0, cp->data, cp->max, maple_copy,
+			  cp->dst[0].node, 0, mt);
+		node_finalise(cp->dst[0].node, mt, cp->end + 1);
+		/*
+		 * Warning, see cp_leaf_init() comment and rcu_assign_pointer()
+		 * documentation.  Since this is a new root, there are no
+		 * read-side operations that can view it until it is insert into
+		 * the tree after an rcu_assign_pointer() call.
+		 */
+		ma_init_slot(&cp->slot[0], cp->dst[0].node, mt);
+		cp->height++;
+	}
+	WARN_ON_ONCE(cp->dst[0].node != mte_to_node(
+				mt_slot_locked(mas->tree, cp->slot, 0)));
+	cp->dst[0].node->parent = ma_parent_ptr(mas_tree_parent(mas));
+	mas->min = 0;
+	mas->max = ULONG_MAX;
+	mas->depth = 0;
+	mas->node = mas_root_locked(mas);
+	return true;
+}
 
-	/* Get the data; Fill mast->bn */
-	mast->bn->b_end++;
-	if (left) {
-		mab_shift_right(mast->bn, end + 1);
-		mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
-		mast->bn->b_end = slot_total + 1;
-	} else {
-		mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
-	}
+static inline bool cp_converged(struct maple_copy *cp, struct ma_state *mas,
+				struct ma_state *sib)
+{
+	if (cp->d_count != 1 || sib->end)
+		return false;
 
-	/* Configure mast for splitting of mast->bn */
-	split = mt_slots[mast->bn->type] - 2;
-	if (left) {
-		/*  Switch mas to prev node  */
-		*mas = tmp_mas;
-		/* Start using mast->l for the left side. */
-		tmp_mas.node = mast->l->node;
-		*mast->l = tmp_mas;
-	} else {
-		tmp_mas.node = mast->r->node;
-		*mast->r = tmp_mas;
-		split = slot_total - split;
-	}
-	split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]);
-	/* Update parent slot for split calculation. */
-	if (left)
-		mast->orig_l->offset += end + 1;
-
-	mast_split_data(mast, mas, split);
-	mast_fill_bnode(mast, mas, 2);
-	mas_split_final_node(mast, mas);
+	cp->dst[0].node->parent = ma_parent_ptr(mas_mn(mas)->parent);
 	return true;
 }
 
 /*
- * mas_split() - Split data that is too big for one node into two.
- * @mas: The maple state
- * @b_node: The maple big node
+ * spanning_ascend() - See if a spanning store operation has to keep walking up
+ * the tree
+ * @cp: The maple_copy node
+ * @l_wr_mas: The left maple write state
+ * @r_wr_mas: The right maple write state
+ * @sib: the maple state of the sibling
+ *
+ * Returns: True if another iteration is necessary.
  */
-static void mas_split(struct ma_state *mas, struct maple_big_node *b_node)
+static bool spanning_ascend(struct maple_copy *cp, struct ma_state *mas,
+			    struct ma_wr_state *l_wr_mas, struct ma_wr_state *r_wr_mas,
+			    struct ma_state *sib)
 {
-	struct maple_subtree_state mast;
-	int height = 0;
-	unsigned int orig_height = mas_mt_height(mas);
-	unsigned char mid_split, split = 0;
-	struct maple_enode *old;
-
-	/*
-	 * Splitting is handled differently from any other B-tree; the Maple
-	 * Tree splits upwards.  Splitting up means that the split operation
-	 * occurs when the walk of the tree hits the leaves and not on the way
-	 * down.  The reason for splitting up is that it is impossible to know
-	 * how much space will be needed until the leaf is (or leaves are)
-	 * reached.  Since overwriting data is allowed and a range could
-	 * overwrite more than one range or result in changing one entry into 3
-	 * entries, it is impossible to know if a split is required until the
-	 * data is examined.
-	 *
-	 * Splitting is a balancing act between keeping allocations to a minimum
-	 * and avoiding a 'jitter' event where a tree is expanded to make room
-	 * for an entry followed by a contraction when the entry is removed.  To
-	 * accomplish the balance, there are empty slots remaining in both left
-	 * and right nodes after a split.
-	 */
-	MA_STATE(l_mas, mas->tree, mas->index, mas->last);
-	MA_STATE(r_mas, mas->tree, mas->index, mas->last);
-	MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last);
-	MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last);
-
-	trace_ma_op(TP_FCT, mas);
-
-	mast.l = &l_mas;
-	mast.r = &r_mas;
-	mast.orig_l = &prev_l_mas;
-	mast.orig_r = &prev_r_mas;
-	mast.bn = b_node;
-
-	while (height++ <= orig_height) {
-		if (mt_slots[b_node->type] > b_node->b_end) {
-			mas_split_final_node(&mast, mas);
-			break;
-		}
+	if (sib->end) {
+		if (sib->max < l_wr_mas->mas->min)
+			*l_wr_mas->mas = *sib;
+		else
+			*r_wr_mas->mas = *sib;
+	}
 
-		l_mas = r_mas = *mas;
-		l_mas.node = mas_new_ma_node(mas, b_node);
-		r_mas.node = mas_new_ma_node(mas, b_node);
-		/*
-		 * Another way that 'jitter' is avoided is to terminate a split up early if the
-		 * left or right node has space to spare.  This is referred to as "pushing left"
-		 * or "pushing right" and is similar to the B* tree, except the nodes left or
-		 * right can rarely be reused due to RCU, but the ripple upwards is halted which
-		 * is a significant savings.
-		 */
-		/* Try to push left. */
-		if (mas_push_data(mas, &mast, true)) {
-			height++;
-			break;
-		}
-		/* Try to push right. */
-		if (mas_push_data(mas, &mast, false)) {
-			height++;
-			break;
-		}
+	cp_dst_to_slots(cp, l_wr_mas->mas->min, r_wr_mas->mas->max, mas);
+	if (cp_is_new_root(cp, mas))
+		return false;
 
-		split = mab_calc_split(mas, b_node, &mid_split);
-		mast_split_data(&mast, mas, split);
-		/*
-		 * Usually correct, mab_mas_cp in the above call overwrites
-		 * r->max.
-		 */
-		mast.r->max = mas->max;
-		mast_fill_bnode(&mast, mas, 1);
-		prev_l_mas = *mast.l;
-		prev_r_mas = *mast.r;
+	/* Converged and has a single destination */
+	if ((cp->d_count == 1) &&
+	    (l_wr_mas->mas->node == r_wr_mas->mas->node)) {
+		cp->dst[0].node->parent = ma_parent_ptr(mas_mn(mas)->parent);
+		return false;
 	}
 
-	/* Set the original node as dead */
-	old = mas->node;
-	mas->node = l_mas.node;
-	mas_wmb_replace(mas, old, height);
-	mtree_range_walk(mas);
-	return;
+	cp->height++;
+	wr_mas_ascend(l_wr_mas);
+	wr_mas_ascend(r_wr_mas);
+	return true;
+}
+
+static inline
+void copy_tree_location(const struct ma_state *src, struct ma_state *dst)
+{
+	dst->node = src->node;
+	dst->offset = src->offset;
+	dst->min = src->min;
+	dst->max = src->max;
+	dst->end = src->end;
+	dst->depth = src->depth;
 }
 
 /*
- * mas_commit_b_node() - Commit the big node into the tree.
- * @wr_mas: The maple write state
- * @b_node: The maple big node
+ * rebalance_ascend() - Ascend the tree and set up for the next loop - if
+ * necessary
+ *
+ * Return: True if there another rebalancing operation on the next level is
+ * needed, false otherwise.
  */
-static noinline_for_kasan void mas_commit_b_node(struct ma_wr_state *wr_mas,
-			    struct maple_big_node *b_node)
+static inline bool rebalance_ascend(struct maple_copy *cp,
+		struct ma_wr_state *wr_mas, struct ma_state *sib,
+		struct ma_state *parent)
 {
-	enum store_type type = wr_mas->mas->store_type;
+	struct ma_state *mas;
+	unsigned long min, max;
+
+	mas = wr_mas->mas;
+	if (!sib->end) {
+		min = mas->min;
+		max = mas->max;
+	} else if (sib->min > mas->max) { /* Move right succeeded */
+		min = mas->min;
+		max = sib->max;
+		wr_mas->offset_end = parent->offset + 1;
+	} else {
+		min = sib->min;
+		max = mas->max;
+		wr_mas->offset_end = parent->offset;
+		parent->offset--;
+	}
 
-	WARN_ON_ONCE(type != wr_rebalance && type != wr_split_store);
+	cp_dst_to_slots(cp, min, max, mas);
+	if (cp_is_new_root(cp, mas))
+		return false;
 
-	if (type == wr_rebalance)
-		return mas_rebalance(wr_mas->mas, b_node);
+	if (cp_converged(cp, mas, sib))
+		return false;
 
-	return mas_split(wr_mas->mas, b_node);
+	cp->height++;
+	copy_tree_location(parent, mas);
+	wr_mas_setup(wr_mas, mas);
+	return true;
 }
 
 /*
@@ -3112,7 +2765,6 @@ static inline void mas_root_expand(struct ma_state *mas, void *entry)
 	ma_set_meta(node, maple_leaf_64, 0, slot);
 	/* swap the new root into the tree */
 	rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
-	return;
 }
 
 /*
@@ -3269,6 +2921,7 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
 			l_mas->index = l_mas->min;
 
 		l_mas->offset = l_slot - 1;
+		l_wr_mas->r_min = l_mas->index;
 	}
 
 	if (!r_wr_mas->content) {
@@ -3279,8 +2932,9 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
 	    (r_mas->last < r_mas->max) &&
 	    !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
 		r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
-					     r_wr_mas->type, r_mas->offset + 1);
+					     r_mas->offset + 1, r_wr_mas->type);
 		r_mas->offset++;
+		r_wr_mas->r_max = r_mas->last;
 	}
 }
 
@@ -3382,8 +3036,6 @@ static inline void mas_new_root(struct ma_state *mas, void *entry)
 done:
 	if (xa_is_node(root))
 		mte_destroy_walk(root, mas->tree);
-
-	return;
 }
 /*
  * mas_wr_spanning_store() - Create a subtree with the store operation completed
@@ -3392,18 +3044,15 @@ static inline void mas_new_root(struct ma_state *mas, void *entry)
  * span.
  * @wr_mas: The maple write state
  */
-static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
+static void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 {
-	struct maple_subtree_state mast;
-	struct maple_big_node b_node;
+	struct maple_copy cp;
 	struct ma_state *mas;
-	unsigned char height;
+	struct ma_state sib;
 
 	/* Left and Right side of spanning store */
-	MA_STATE(l_mas, NULL, 0, 0);
 	MA_STATE(r_mas, NULL, 0, 0);
 	MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry);
-	MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry);
 
 	/*
 	 * A store operation that spans multiple nodes is called a spanning
@@ -3426,7 +3075,6 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 	 * Node rebalancing may occur due to this store, so there may be three new
 	 * entries per level plus a new root.
 	 */
-	height = mas_mt_height(mas);
 
 	/*
 	 * Set up right side.  Need to get to the next offset after the spanning
@@ -3441,42 +3089,31 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 	r_mas.index = r_mas.last;
 	mas_wr_walk_index(&r_wr_mas);
 	r_mas.last = r_mas.index = mas->last;
+	r_wr_mas.end_piv = r_wr_mas.r_max;
 
 	/* Set up left side. */
-	l_mas = *mas;
-	mas_wr_walk_index(&l_wr_mas);
+	mas_wr_walk_index(wr_mas);
 
 	if (!wr_mas->entry) {
-		mas_extend_spanning_null(&l_wr_mas, &r_wr_mas);
-		mas->offset = l_mas.offset;
-		mas->index = l_mas.index;
-		mas->last = l_mas.last = r_mas.last;
+		mas_extend_spanning_null(wr_mas, &r_wr_mas);
+		mas->last = r_mas.last;
 	}
 
 	/* expanding NULLs may make this cover the entire range */
-	if (!l_mas.index && r_mas.last == ULONG_MAX) {
+	if (!mas->index && r_mas.last == ULONG_MAX) {
 		mas_set_range(mas, 0, ULONG_MAX);
 		return mas_new_root(mas, wr_mas->entry);
 	}
 
-	memset(&b_node, 0, sizeof(struct maple_big_node));
-	/* Copy l_mas and store the value in b_node. */
-	mas_store_b_node(&l_wr_mas, &b_node, l_mas.end);
-	/* Copy r_mas into b_node if there is anything to copy. */
-	if (r_mas.max > r_mas.last)
-		mas_mab_cp(&r_mas, r_mas.offset, r_mas.end,
-			   &b_node, b_node.b_end + 1);
-	else
-		b_node.b_end++;
-
-	/* Stop spanning searches by searching for just index. */
-	l_mas.index = l_mas.last = mas->index;
+	cp_leaf_init(&cp, mas, wr_mas, &r_wr_mas);
+	do {
+		spanning_data(&cp, wr_mas, &r_wr_mas, &sib);
+		multi_src_setup(&cp, wr_mas, &r_wr_mas, &sib);
+		dst_setup(&cp, mas, wr_mas->type);
+		cp_data_write(&cp, mas);
+	} while (spanning_ascend(&cp, mas, wr_mas, &r_wr_mas, &sib));
 
-	mast.bn = &b_node;
-	mast.orig_l = &l_mas;
-	mast.orig_r = &r_mas;
-	/* Combine l_mas and r_mas and split them up evenly again. */
-	return mas_spanning_rebalance(mas, &mast, height + 1);
+	mas_wmb_replace(mas, &cp);
 }
 
 /*
@@ -3485,20 +3122,28 @@ static noinline void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
  *
  * Attempts to reuse the node, but may allocate.
  */
-static inline void mas_wr_node_store(struct ma_wr_state *wr_mas,
-				     unsigned char new_end)
+static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 {
-	struct ma_state *mas = wr_mas->mas;
-	void __rcu **dst_slots;
-	unsigned long *dst_pivots;
-	unsigned char dst_offset, offset_end = wr_mas->offset_end;
+	unsigned char dst_offset, offset_end;
+	unsigned char copy_size, node_pivots;
 	struct maple_node reuse, *newnode;
-	unsigned char copy_size, node_pivots = mt_pivots[wr_mas->type];
-	bool in_rcu = mt_in_rcu(mas->tree);
-	unsigned char height = mas_mt_height(mas);
+	unsigned long *dst_pivots;
+	void __rcu **dst_slots;
+	unsigned char new_end;
+	struct ma_state *mas;
+	bool in_rcu;
 
-	if (mas->last == wr_mas->end_piv)
+	mas = wr_mas->mas;
+	trace_ma_op(TP_FCT, mas);
+	in_rcu = mt_in_rcu(mas->tree);
+	offset_end = wr_mas->offset_end;
+	node_pivots = mt_pivots[wr_mas->type];
+	/* Assume last adds an entry */
+	new_end = mas->end + 1 - offset_end + mas->offset;
+	if (mas->last == wr_mas->end_piv) {
 		offset_end++; /* don't copy this offset */
+		new_end--;
+	}
 
 	/* set up node. */
 	if (in_rcu) {
@@ -3512,13 +3157,16 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas,
 	dst_pivots = ma_pivots(newnode, wr_mas->type);
 	dst_slots = ma_slots(newnode, wr_mas->type);
 	/* Copy from start to insert point */
-	memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset);
-	memcpy(dst_slots, wr_mas->slots, sizeof(void *) * mas->offset);
+	if (mas->offset) {
+		memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset);
+		memcpy(dst_slots, wr_mas->slots, sizeof(void __rcu *) * mas->offset);
+	}
 
 	/* Handle insert of new range starting after old range */
 	if (wr_mas->r_min < mas->index) {
 		rcu_assign_pointer(dst_slots[mas->offset], wr_mas->content);
 		dst_pivots[mas->offset++] = mas->index - 1;
+		new_end++;
 	}
 
 	/* Store the new entry and range end. */
@@ -3537,7 +3185,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas,
 	/* Copy to the end of node if necessary. */
 	copy_size = mas->end - offset_end + 1;
 	memcpy(dst_slots + dst_offset, wr_mas->slots + offset_end,
-	       sizeof(void *) * copy_size);
+	       sizeof(void __rcu *) * copy_size);
 	memcpy(dst_pivots + dst_offset, wr_mas->pivots + offset_end,
 	       sizeof(unsigned long) * (copy_size - 1));
 
@@ -3550,14 +3198,13 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas,
 		struct maple_enode *old_enode = mas->node;
 
 		mas->node = mt_mk_node(newnode, wr_mas->type);
-		mas_replace_node(mas, old_enode, height);
+		mas_replace_node(mas, old_enode, mas_mt_height(mas));
 	} else {
 		memcpy(wr_mas->node, newnode, sizeof(struct maple_node));
 	}
 	trace_ma_write(TP_FCT, mas, 0, wr_mas->entry);
 	mas_update_gap(mas);
 	mas->end = new_end;
-	return;
 }
 
 /*
@@ -3605,8 +3252,6 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas)
 	 */
 	if (!wr_mas->entry || gap)
 		mas_update_gap(mas);
-
-	return;
 }
 
 static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas)
@@ -3675,18 +3320,17 @@ static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas)
 /*
  * mas_wr_append: Attempt to append
  * @wr_mas: the maple write state
- * @new_end: The end of the node after the modification
  *
  * This is currently unsafe in rcu mode since the end of the node may be cached
  * by readers while the node contents may be updated which could result in
  * inaccurate information.
  */
-static inline void mas_wr_append(struct ma_wr_state *wr_mas,
-		unsigned char new_end)
+static inline void mas_wr_append(struct ma_wr_state *wr_mas)
 {
 	struct ma_state *mas = wr_mas->mas;
 	void __rcu **slots;
 	unsigned char end = mas->end;
+	unsigned char new_end = mas_wr_new_end(wr_mas);
 
 	if (new_end < mt_pivots[wr_mas->type]) {
 		wr_mas->pivots[new_end] = wr_mas->pivots[end];
@@ -3720,23 +3364,147 @@ static inline void mas_wr_append(struct ma_wr_state *wr_mas,
 
 	mas->end = new_end;
 	trace_ma_write(TP_FCT, mas, new_end, wr_mas->entry);
-	return;
 }
 
 /*
- * mas_wr_bnode() - Slow path for a modification.
- * @wr_mas: The write maple state
+ * split_ascend() - See if a split operation has to keep walking up the tree
+ * @cp: The maple_copy node
+ * @wr_mas: The maple write state
+ * @sib: the maple state of the sibling
+ *
+ * Return: true if another split operation on the next level is needed, false
+ * otherwise
+ */
+static inline bool split_ascend(struct maple_copy *cp,
+		struct ma_wr_state *wr_mas, struct ma_state *sib,
+		struct ma_state *parent)
+{
+	struct ma_state *mas;
+	unsigned long min, max;
+
+	mas = wr_mas->mas;
+	min = mas->min; /* push right, or normal split */
+	max = mas->max;
+	wr_mas->offset_end = parent->offset;
+	if (sib->end) {
+		if (sib->max < mas->min) {
+			min = sib->min; /* push left */
+			parent->offset--;
+		} else {
+			max = sib->max; /* push right */
+			wr_mas->offset_end++;
+		}
+	}
+
+	cp_dst_to_slots(cp, min, max, mas);
+	if (cp_is_new_root(cp, mas))
+		return false;
+
+	if (cp_converged(cp, mas, sib))
+		return false;
+
+	cp->height++;
+	copy_tree_location(parent, mas);
+	wr_mas_setup(wr_mas, mas);
+	return true;
+}
+
+/*
+ * split_data() - Calculate the @cp data, populate @sib if the data can be
+ * pushed into a sibling.
+ * @cp: The maple copy node
+ * @wr_mas: The left write maple state
+ * @sib: The maple state of the sibling.
+ *
+ * Note: @cp->data is a size and not indexed by 0. @sib->end may be set to 0 to
+ * indicate it will not be used.
  *
- * This is where split, rebalance end up.
  */
-static void mas_wr_bnode(struct ma_wr_state *wr_mas)
+static inline void split_data(struct maple_copy *cp,
+		struct ma_wr_state *wr_mas, struct ma_state *sib,
+		struct ma_state *parent)
+{
+	cp_data_calc(cp, wr_mas, wr_mas);
+	if (cp->data <= mt_slots[wr_mas->type]) {
+		sib->end = 0;
+		return;
+	}
+
+	push_data_sib(cp, wr_mas->mas, sib, parent);
+	if (sib->end)
+		cp->data += sib->end + 1;
+}
+
+/*
+ * mas_wr_split() - Expand one node into two
+ * @wr_mas: The write maple state
+ */
+static void mas_wr_split(struct ma_wr_state *wr_mas)
 {
-	struct maple_big_node b_node;
+	struct ma_state parent;
+	struct ma_state *mas;
+	struct maple_copy cp;
+	struct ma_state sib;
 
+	mas = wr_mas->mas;
 	trace_ma_write(TP_FCT, wr_mas->mas, 0, wr_mas->entry);
-	memset(&b_node, 0, sizeof(struct maple_big_node));
-	mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end);
-	mas_commit_b_node(wr_mas, &b_node);
+	parent = *mas;
+	cp_leaf_init(&cp, mas, wr_mas, wr_mas);
+	do {
+		if (!mte_is_root(parent.node)) {
+			mas_ascend(&parent);
+			parent.end = mas_data_end(&parent);
+		}
+		split_data(&cp, wr_mas, &sib, &parent);
+		multi_src_setup(&cp, wr_mas, wr_mas, &sib);
+		dst_setup(&cp, mas, wr_mas->type);
+		cp_data_write(&cp, mas);
+	} while (split_ascend(&cp, wr_mas, &sib, &parent));
+
+	mas_wmb_replace(mas, &cp);
+}
+
+/*
+ * mas_wr_rebalance() - Insufficient data in one node needs to either get data
+ * from a sibling or absorb a sibling all together.
+ * @wr_mas: The write maple state
+ *
+ * Rebalance is different than a spanning store in that the write state is
+ * already at the leaf node that's being altered.
+ */
+static void mas_wr_rebalance(struct ma_wr_state *wr_mas)
+{
+	struct ma_state parent;
+	struct ma_state *mas;
+	struct maple_copy cp;
+	struct ma_state sib;
+
+	/*
+	 * Rebalancing occurs if a node is insufficient.  Data is rebalanced
+	 * against the node to the right if it exists, otherwise the node to the
+	 * left of this node is rebalanced against this node.  If rebalancing
+	 * causes just one node to be produced instead of two, then the parent
+	 * is also examined and rebalanced if it is insufficient.  Every level
+	 * tries to combine the data in the same way.  If one node contains the
+	 * entire range of the tree, then that node is used as a new root node.
+	 */
+
+	mas = wr_mas->mas;
+	trace_ma_op(TP_FCT, mas);
+	parent = *mas;
+	cp_leaf_init(&cp, mas, wr_mas, wr_mas);
+	do {
+		if (!mte_is_root(parent.node)) {
+			mas_ascend(&parent);
+			parent.end = mas_data_end(&parent);
+		}
+		rebalance_data(&cp, wr_mas, &sib, &parent);
+		multi_src_setup(&cp, wr_mas, wr_mas, &sib);
+		dst_setup(&cp, mas, wr_mas->type);
+		cp_data_write(&cp, mas);
+	} while (rebalance_ascend(&cp, wr_mas, &sib, &parent));
+
+	mas_wmb_replace(mas, &cp);
 }
 
 /*
@@ -3746,7 +3514,6 @@ static void mas_wr_bnode(struct ma_wr_state *wr_mas)
 static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas)
 {
 	struct ma_state *mas = wr_mas->mas;
-	unsigned char new_end = mas_wr_new_end(wr_mas);
 
 	switch (mas->store_type) {
 	case wr_exact_fit:
@@ -3755,20 +3522,22 @@ static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas)
 			mas_update_gap(mas);
 		break;
 	case wr_append:
-		mas_wr_append(wr_mas, new_end);
+		mas_wr_append(wr_mas);
 		break;
 	case wr_slot_store:
 		mas_wr_slot_store(wr_mas);
 		break;
 	case wr_node_store:
-		mas_wr_node_store(wr_mas, new_end);
+		mas_wr_node_store(wr_mas);
 		break;
 	case wr_spanning_store:
 		mas_wr_spanning_store(wr_mas);
 		break;
 	case wr_split_store:
+		mas_wr_split(wr_mas);
+		break;
 	case wr_rebalance:
-		mas_wr_bnode(wr_mas);
+		mas_wr_rebalance(wr_mas);
 		break;
 	case wr_new_root:
 		mas_new_root(mas, wr_mas->entry);
@@ -3779,8 +3548,6 @@ static inline void mas_wr_store_entry(struct ma_wr_state *wr_mas)
 	case wr_invalid:
 		MT_BUG_ON(mas->tree, 1);
 	}
-
-	return;
 }
 
 static inline void mas_wr_prealloc_setup(struct ma_wr_state *wr_mas)
@@ -6260,8 +6027,15 @@ static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas,
 	for (i = 0; i < count; i++) {
 		val = (unsigned long)mt_slot_locked(mas->tree, slots, i);
 		val &= MAPLE_NODE_MASK;
-		new_slots[i] = ma_mnode_ptr((unsigned long)mas_pop_node(mas) |
-					    val);
+		/*
+		 * Warning, see rcu_assign_pointer() documentation.  Since this
+		 * is a duplication of a tree, there are no readers walking the
+		 * tree until after the rcu_assign_pointer() call in
+		 * mas_dup_build().
+		 */
+		RCU_INIT_POINTER(new_slots[i],
+				 ma_mnode_ptr((unsigned long)mas_pop_node(mas) |
+					      val));
 	}
 }
 
diff --git a/lib/math/Kconfig b/lib/math/Kconfig
index 0634b428d0cb7..0e6d9cffc5d66 100644
--- a/lib/math/Kconfig
+++ b/lib/math/Kconfig
@@ -5,6 +5,9 @@ config CORDIC
 	  This option provides an implementation of the CORDIC algorithm;
 	  calculations are in fixed point. Module will be called cordic.
 
+config POLYNOMIAL
+       tristate
+
 config PRIME_NUMBERS
 	tristate "Simple prime number generator for testing"
 	help
diff --git a/lib/math/Makefile b/lib/math/Makefile
index d1caba23baa0b..9a3850d55b79f 100644
--- a/lib/math/Makefile
+++ b/lib/math/Makefile
@@ -2,6 +2,7 @@
 obj-y += div64.o gcd.o lcm.o int_log.o int_pow.o int_sqrt.o reciprocal_div.o
 
 obj-$(CONFIG_CORDIC)		+= cordic.o
+obj-$(CONFIG_POLYNOMIAL)	+= polynomial.o
 obj-$(CONFIG_PRIME_NUMBERS)	+= prime_numbers.o
 obj-$(CONFIG_RATIONAL)		+= rational.o
 
diff --git a/lib/math/polynomial.c b/lib/math/polynomial.c
new file mode 100644
index 0000000000000..f26677cfeefff
--- /dev/null
+++ b/lib/math/polynomial.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generic polynomial calculation using integer coefficients.
+ *
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru>
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *
+ */
+
+#include <linux/export.h>
+#include <linux/math.h>
+#include <linux/module.h>
+#include <linux/polynomial.h>
+
+/*
+ * The following conversion is an example:
+ *
+ * The original translation formulae of the temperature (in degrees of Celsius)
+ * to PVT data and vice-versa are following:
+ *
+ * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) + 1.7204e2
+ * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) + 3.1020e-1*(N^1) - 4.838e1
+ *
+ * where T = [-48.380, 147.438]C and N = [0, 1023].
+ *
+ * They must be accordingly altered to be suitable for the integer arithmetics.
+ * The technique is called 'factor redistribution', which just makes sure the
+ * multiplications and divisions are made so to have a result of the operations
+ * within the integer numbers limit. In addition we need to translate the
+ * formulae to accept millidegrees of Celsius. Here what they look like after
+ * the alterations:
+ *
+ * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T + 17204e2) / 1e4
+ * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D - 48380
+ *
+ * where T = [-48380, 147438] mC and N = [0, 1023].
+ *
+ * static const struct polynomial poly_temp_to_N = {
+ *         .total_divider = 10000,
+ *         .terms = {
+ *                 {4, 18322, 10000, 10000},
+ *                 {3, 2343, 10000, 10},
+ *                 {2, 87018, 10000, 10},
+ *                 {1, 39269, 1000, 1},
+ *                 {0, 1720400, 1, 1}
+ *         }
+ * };
+ *
+ * static const struct polynomial poly_N_to_temp = {
+ *         .total_divider = 1,
+ *         .terms = {
+ *                 {4, -16743, 1000, 1},
+ *                 {3, 81542, 1000, 1},
+ *                 {2, -182010, 1000, 1},
+ *                 {1, 310200, 1000, 1},
+ *                 {0, -48380, 1, 1}
+ *         }
+ * };
+ */
+
+/**
+ * polynomial_calc - calculate a polynomial using integer arithmetic
+ *
+ * @poly: pointer to the descriptor of the polynomial
+ * @data: input value of the polynomial
+ *
+ * Calculate the result of a polynomial using only integer arithmetic. For
+ * this to work without too much loss of precision the coefficients has to
+ * be altered. This is called factor redistribution.
+ *
+ * Return: the result of the polynomial calculation.
+ */
+long polynomial_calc(const struct polynomial *poly, long data)
+{
+	const struct polynomial_term *term = poly->terms;
+	long total_divider = poly->total_divider ?: 1;
+	long tmp, ret = 0;
+	int deg;
+
+	/*
+	 * Here is the polynomial calculation function, which performs the
+	 * redistributed terms calculations. It's pretty straightforward.
+	 * We walk over each degree term up to the free one, and perform
+	 * the redistributed multiplication of the term coefficient, its
+	 * divider (as for the rationale fraction representation), data
+	 * power and the rational fraction divider leftover. Then all of
+	 * this is collected in a total sum variable, which value is
+	 * normalized by the total divider before being returned.
+	 */
+	do {
+		tmp = term->coef;
+		for (deg = 0; deg < term->deg; ++deg)
+			tmp = mult_frac(tmp, data, term->divider);
+		ret += tmp / term->divider_leftover;
+	} while ((term++)->deg);
+
+	return ret / total_divider;
+}
+EXPORT_SYMBOL_GPL(polynomial_calc);
+
+MODULE_DESCRIPTION("Generic polynomial calculations");
+MODULE_LICENSE("GPL");
diff --git a/lib/percpu_counter_tree.c b/lib/percpu_counter_tree.c
new file mode 100644
index 0000000000000..beb1144e6450f
--- /dev/null
+++ b/lib/percpu_counter_tree.c
@@ -0,0 +1,702 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+// SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+/*
+ * Split Counters With Tree Approximation Propagation
+ *
+ * * Propagation diagram when reaching batch size thresholds (± batch size):
+ *
+ * Example diagram for 8 CPUs:
+ *
+ * log2(8) = 3 levels
+ *
+ * At each level, each pair propagates its values to the next level when
+ * reaching the batch size thresholds.
+ *
+ * Counters at levels 0, 1, 2 can be kept on a single byte ([-128 .. +127] range),
+ * although it may be relevant to keep them on "long" counters for
+ * simplicity. (complexity vs memory footprint tradeoff)
+ *
+ * Counter at level 3 can be kept on a "long" counter.
+ *
+ * Level 0:  0    1    2    3    4    5    6    7
+ *           |   /     |   /     |   /     |   /
+ *           |  /      |  /      |  /      |  /
+ *           | /       | /       | /       | /
+ * Level 1:  0         1         2         3
+ *           |       /           |       /
+ *           |    /              |    /
+ *           | /                 | /
+ * Level 2:  0                   1
+ *           |               /
+ *           |         /
+ *           |   /
+ * Level 3:  0
+ *
+ * * Approximation accuracy:
+ *
+ * BATCH(level N): Level N batch size.
+ *
+ * Example for BATCH(level 0) = 32.
+ *
+ * BATCH(level 0) =  32
+ * BATCH(level 1) =  64
+ * BATCH(level 2) = 128
+ * BATCH(level N) = BATCH(level 0) * 2^N
+ *
+ *            per-counter     global
+ *            accuracy        accuracy
+ * Level 0:   [ -32 ..  +31]  ±256  (8 * 32)
+ * Level 1:   [ -64 ..  +63]  ±256  (4 * 64)
+ * Level 2:   [-128 .. +127]  ±256  (2 * 128)
+ * Total:      ------         ±768  (log2(nr_cpu_ids) * BATCH(level 0) * nr_cpu_ids)
+ *
+ * Note that the global accuracy can be calculated more precisely
+ * by taking into account that the positive accuracy range is
+ * 31 rather than 32.
+ *
+ * -----
+ *
+ * Approximate Sum Carry Propagation
+ *
+ * Let's define a number of counter bits for each level, e.g.:
+ *
+ * log2(BATCH(level 0)) = log2(32) = 5
+ * Let's assume, for this example, a 32-bit architecture (sizeof(long) == 4).
+ *
+ *               nr_bit        value_mask                      range
+ * Level 0:      5 bits        v                             0 ..  +31
+ * Level 1:      1 bit        (v & ~((1UL << 5) - 1))        0 ..  +63
+ * Level 2:      1 bit        (v & ~((1UL << 6) - 1))        0 .. +127
+ * Level 3:     25 bits       (v & ~((1UL << 7) - 1))        0 .. 2^32-1
+ *
+ * Note: Use a "long" per-cpu counter at level 0 to allow precise sum.
+ *
+ * Note: Use cacheline aligned counters at levels above 0 to prevent false sharing.
+ *       If memory footprint is an issue, a specialized allocator could be used
+ *       to eliminate padding.
+ *
+ * Example with expanded values:
+ *
+ * counter_add(counter, inc):
+ *
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = percpu_add_return(counter @ Level 0, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b00011111);  // Clear used bits
+ *                 // xor bit 5: underflow
+ *                 if ((inc ^ orig ^ res) & 0b00100000)
+ *                         inc -= 0b00100000;
+ *         } else {
+ *                 inc &= ~0b00011111;           // Clear used bits
+ *                 // xor bit 5: overflow
+ *                 if ((inc ^ orig ^ res) & 0b00100000)
+ *                         inc += 0b00100000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = atomic_long_add_return(counter @ Level 1, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b00111111);  // Clear used bits
+ *                 // xor bit 6: underflow
+ *                 if ((inc ^ orig ^ res) & 0b01000000)
+ *                         inc -= 0b01000000;
+ *         } else {
+ *                 inc &= ~0b00111111;           // Clear used bits
+ *                 // xor bit 6: overflow
+ *                 if ((inc ^ orig ^ res) & 0b01000000)
+ *                         inc += 0b01000000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = atomic_long_add_return(counter @ Level 2, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b01111111);  // Clear used bits
+ *                 // xor bit 7: underflow
+ *                 if ((inc ^ orig ^ res) & 0b10000000)
+ *                         inc -= 0b10000000;
+ *         } else {
+ *                 inc &= ~0b01111111;           // Clear used bits
+ *                 // xor bit 7: overflow
+ *                 if ((inc ^ orig ^ res) & 0b10000000)
+ *                         inc += 0b10000000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         atomic_long_add(counter @ Level 3, inc);
+ */
+
+#include <linux/percpu_counter_tree.h>
+#include <linux/cpumask.h>
+#include <linux/atomic.h>
+#include <linux/export.h>
+#include <linux/percpu.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/math.h>
+
+#define MAX_NR_LEVELS 5
+
+/*
+ * The counter configuration is selected at boot time based on the
+ * hardware topology.
+ */
+struct counter_config {
+	unsigned int nr_items;				/*
+							 * nr_items is the number of items in the tree for levels 1
+							 * up to and including the final level (approximate sum).
+							 * It excludes the level 0 per-CPU counters.
+							 */
+	unsigned char nr_levels;			/*
+							 * nr_levels is the number of hierarchical counter tree levels.
+							 * It excludes the final level (approximate sum).
+							 */
+	unsigned char n_arity_order[MAX_NR_LEVELS];	/*
+							 * n-arity of tree nodes for each level from
+							 * 0 to (nr_levels - 1).
+							 */
+};
+
+static const struct counter_config per_nr_cpu_order_config[] = {
+	[0] =	{ .nr_items = 0,	.nr_levels = 0,		.n_arity_order = { 0 } },
+	[1] =	{ .nr_items = 1,	.nr_levels = 1,		.n_arity_order = { 1 } },
+	[2] =	{ .nr_items = 3,	.nr_levels = 2,		.n_arity_order = { 1, 1 } },
+	[3] =	{ .nr_items = 7,	.nr_levels = 3,		.n_arity_order = { 1, 1, 1 } },
+	[4] =	{ .nr_items = 7,	.nr_levels = 3,		.n_arity_order = { 2, 1, 1 } },
+	[5] =	{ .nr_items = 11,	.nr_levels = 3,		.n_arity_order = { 2, 2, 1 } },
+	[6] =	{ .nr_items = 21,	.nr_levels = 3,		.n_arity_order = { 2, 2, 2 } },
+	[7] =	{ .nr_items = 21,	.nr_levels = 3,		.n_arity_order = { 3, 2, 2 } },
+	[8] =	{ .nr_items = 37,	.nr_levels = 3,		.n_arity_order = { 3, 3, 2 } },
+	[9] =	{ .nr_items = 73,	.nr_levels = 3,		.n_arity_order = { 3, 3, 3 } },
+	[10] =	{ .nr_items = 149,	.nr_levels = 4,		.n_arity_order = { 3, 3, 2, 2 } },
+	[11] =	{ .nr_items = 293,	.nr_levels = 4,		.n_arity_order = { 3, 3, 3, 2 } },
+	[12] =	{ .nr_items = 585,	.nr_levels = 4,		.n_arity_order = { 3, 3, 3, 3 } },
+	[13] =	{ .nr_items = 1173,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 2, 2 } },
+	[14] =	{ .nr_items = 2341,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 3, 2 } },
+	[15] =	{ .nr_items = 4681,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 3, 3 } },
+	[16] =	{ .nr_items = 4681,	.nr_levels = 5,		.n_arity_order = { 4, 3, 3, 3, 3 } },
+	[17] =	{ .nr_items = 8777,	.nr_levels = 5,		.n_arity_order = { 4, 4, 3, 3, 3 } },
+	[18] =	{ .nr_items = 17481,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 3, 3 } },
+	[19] =	{ .nr_items = 34953,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 4, 3 } },
+	[20] =	{ .nr_items = 69905,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 4, 4 } },
+};
+
+static const struct counter_config *counter_config;	/* Hierarchical counter configuration for the hardware topology. */
+static unsigned int nr_cpus_order;			/* Order of nr_cpu_ids. */
+static unsigned long accuracy_multiplier;		/* Calculate accuracy for a given batch size (multiplication factor). */
+
+static
+int __percpu_counter_tree_init(struct percpu_counter_tree *counter,
+			       unsigned long batch_size, gfp_t gfp_flags,
+			       unsigned long __percpu *level0,
+			       struct percpu_counter_tree_level_item *items)
+{
+	/* Batch size must be greater than 1, and a power of 2. */
+	if (WARN_ON(batch_size <= 1 || (batch_size & (batch_size - 1))))
+		return -EINVAL;
+	counter->batch_size = batch_size;
+	counter->bias = 0;
+	counter->level0 = level0;
+	counter->items = items;
+	if (!nr_cpus_order) {
+		counter->approx_sum.i = per_cpu_ptr(counter->level0, 0);
+		counter->level0_bit_mask = 0;
+	} else {
+		counter->approx_sum.a = &counter->items[counter_config->nr_items - 1].count;
+		counter->level0_bit_mask = 1UL << get_count_order(batch_size);
+	}
+	/*
+	 * Each tree item signed integer has a negative range which is
+	 * one unit greater than the positive range.
+	 */
+	counter->approx_accuracy_range.under = batch_size * accuracy_multiplier;
+	counter->approx_accuracy_range.over = (batch_size - 1) * accuracy_multiplier;
+	return 0;
+}
+
+/**
+ * percpu_counter_tree_init_many() - Initialize many per-CPU counter trees.
+ * @counters: An array of @nr_counters counters to initialize.
+ *	      Their memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ *	   This memory is provided by the caller.
+ *	   Its size needs to be at least @nr_counters * percpu_counter_tree_items_size().
+ * @nr_counters: The number of counter trees to initialize
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * 		carry propagation.
+ *		The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize many per-CPU counter trees using a single per-CPU
+ * allocator invocation for @nr_counters counters.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL:		- Invalid @batch_size argument
+ * * %-ENOMEM:		- Out of memory
+ */
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned long batch_size, gfp_t gfp_flags)
+{
+	void __percpu *level0, *level0_iter;
+	size_t counter_size = sizeof(*counters->level0),
+	       items_size = percpu_counter_tree_items_size();
+	void *items_iter;
+	unsigned int i;
+	int ret;
+
+	memset(items, 0, items_size * nr_counters);
+	level0 = __alloc_percpu_gfp(nr_counters * counter_size,
+				    __alignof__(*counters->level0), gfp_flags);
+	if (!level0)
+		return -ENOMEM;
+	level0_iter = level0;
+	items_iter = items;
+	for (i = 0; i < nr_counters; i++) {
+		ret = __percpu_counter_tree_init(&counters[i], batch_size, gfp_flags, level0_iter, items_iter);
+		if (ret)
+			goto free_level0;
+		level0_iter += counter_size;
+		items_iter += items_size;
+	}
+	return 0;
+
+free_level0:
+	free_percpu(level0);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_init_many);
+
+/**
+ * percpu_counter_tree_init() - Initialize one per-CPU counter tree.
+ * @counter: Counter to initialize.
+ *	     Its memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ *	   This memory is provided by the caller.
+ *	   Its size needs to be at least percpu_counter_tree_items_size().
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * 		carry propagation.
+ *		The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize one per-CPU counter tree.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL:		- Invalid @batch_size argument
+ * * %-ENOMEM:		- Out of memory
+ */
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned long batch_size, gfp_t gfp_flags)
+{
+	return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_init);
+
+/**
+ * percpu_counter_tree_destroy_many() - Destroy many per-CPU counter trees.
+ * @counters: Array of counters trees to destroy.
+ * @nr_counters: The number of counter trees to destroy.
+ *
+ * Release internal resources allocated for @nr_counters per-CPU counter trees.
+ */
+
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counters, unsigned int nr_counters)
+{
+	free_percpu(counters->level0);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_destroy_many);
+
+/**
+ * percpu_counter_tree_destroy() - Destroy one per-CPU counter tree.
+ * @counter: Counter to destroy.
+ *
+ * Release internal resources allocated for one per-CPU counter tree.
+ */
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_destroy_many(counter, 1);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_destroy);
+
+static
+long percpu_counter_tree_carry(long orig, long res, long inc, unsigned long bit_mask)
+{
+	if (inc < 0) {
+		inc = -(-inc & ~(bit_mask - 1));
+		/*
+		 * xor bit_mask: underflow.
+		 *
+		 * If inc has bit set, decrement an additional bit if
+		 * there is _no_ bit transition between orig and res.
+		 * Else, inc has bit cleared, decrement an additional
+		 * bit if there is a bit transition between orig and
+		 * res.
+		 */
+		if ((inc ^ orig ^ res) & bit_mask)
+			inc -= bit_mask;
+	} else {
+		inc &= ~(bit_mask - 1);
+		/*
+		 * xor bit_mask: overflow.
+		 *
+		 * If inc has bit set, increment an additional bit if
+		 * there is _no_ bit transition between orig and res.
+		 * Else, inc has bit cleared, increment an additional
+		 * bit if there is a bit transition between orig and
+		 * res.
+		 */
+		if ((inc ^ orig ^ res) & bit_mask)
+			inc += bit_mask;
+	}
+	return inc;
+}
+
+/*
+ * It does not matter through which path the carry propagates up the
+ * tree, therefore there is no need to disable preemption because the
+ * cpu number is only used to favor cache locality.
+ */
+static
+void percpu_counter_tree_add_slowpath(struct percpu_counter_tree *counter, long inc)
+{
+	unsigned int level_items, nr_levels = counter_config->nr_levels,
+		     level, n_arity_order;
+	unsigned long bit_mask;
+	struct percpu_counter_tree_level_item *item = counter->items;
+	unsigned int cpu = raw_smp_processor_id();
+
+	WARN_ON_ONCE(!nr_cpus_order);	/* Should never be called for 1 cpu. */
+
+	n_arity_order = counter_config->n_arity_order[0];
+	bit_mask = counter->level0_bit_mask << n_arity_order;
+	level_items = 1U << (nr_cpus_order - n_arity_order);
+
+	for (level = 1; level < nr_levels; level++) {
+		/*
+		 * For the purpose of carry propagation, the
+		 * intermediate level counters only need to keep track
+		 * of the bits relevant for carry propagation. We
+		 * therefore don't care about higher order bits.
+		 * Note that this optimization is unwanted if the
+		 * intended use is to track counters within intermediate
+		 * levels of the topology.
+		 */
+		if (abs(inc) & (bit_mask - 1)) {
+			atomic_long_t *count = &item[cpu & (level_items - 1)].count;
+			unsigned long orig, res;
+
+			res = atomic_long_add_return_relaxed(inc, count);
+			orig = res - inc;
+			inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+			if (likely(!inc))
+				return;
+		}
+		item += level_items;
+		n_arity_order = counter_config->n_arity_order[level];
+		level_items >>= n_arity_order;
+		bit_mask <<= n_arity_order;
+	}
+	atomic_long_add(inc, counter->approx_sum.a);
+}
+
+/**
+ * percpu_counter_tree_add() - Add to a per-CPU counter tree.
+ * @counter: Counter added to.
+ * @inc: Increment value (either positive or negative).
+ *
+ * Add @inc to a per-CPU counter tree. This is a fast-path which will
+ * typically increment per-CPU counters as long as there is no carry
+ * greater or equal to the counter tree batch size.
+ */
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, long inc)
+{
+	unsigned long bit_mask = counter->level0_bit_mask, orig, res;
+
+	res = this_cpu_add_return(*counter->level0, inc);
+	orig = res - inc;
+	inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+	if (likely(!inc))
+		return;
+	percpu_counter_tree_add_slowpath(counter, inc);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_add);
+
+static
+long percpu_counter_tree_precise_sum_unbiased(struct percpu_counter_tree *counter)
+{
+	unsigned long sum = 0;
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		sum += *per_cpu_ptr(counter->level0, cpu);
+	return (long) sum;
+}
+
+/**
+ * percpu_counter_tree_precise_sum() - Return precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the precise sum is relatively expensive because it needs to
+ * iterate over all CPUs.
+ * This is meant to be used when accuracy is preferred over speed.
+ *
+ * Return: The current precise counter sum.
+ */
+long percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_precise_sum_unbiased(counter) + READ_ONCE(counter->bias);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_precise_sum);
+
+static
+int compare_delta(long delta, unsigned long accuracy_pos, unsigned long accuracy_neg)
+{
+	if (delta >= 0) {
+		if (delta <= accuracy_pos)
+			return 0;
+		else
+			return 1;
+	} else {
+		if (-delta <= accuracy_neg)
+			return 0;
+		else
+			return -1;
+	}
+}
+
+/**
+ * percpu_counter_tree_approximate_compare - Approximated comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate an approximate comparison of two counter trees.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counters are found to be either less than or greater
+ * than the other. However, if the approximated comparison returns
+ * 0, the counters respective sums are found to be within the two
+ * counters accuracy range.
+ *
+ * Return:
+ * * %0		- Counters @a and @b do not differ by more than the sum of their respective
+ *                accuracy ranges.
+ * * %-1	- Counter @a less than counter @b.
+ * * %1		- Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	return compare_delta(percpu_counter_tree_approximate_sum(a) - percpu_counter_tree_approximate_sum(b),
+			     a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+			     a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_approximate_compare);
+
+/**
+ * percpu_counter_tree_approximate_compare_value - Approximated comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate an approximate comparison of a counter tree against a given value.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counter is found to be either less than or greater
+ * than the value. However, if the approximated comparison returns
+ * 0, the value is within the counter accuracy range.
+ *
+ * Return:
+ * * %0		- The value @v is within the accuracy range of the counter.
+ * * %-1	- The value @v is less than the counter.
+ * * %1		- The value @v is greater than the counter.
+ */
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, long v)
+{
+	return compare_delta(v - percpu_counter_tree_approximate_sum(counter),
+			     counter->approx_accuracy_range.under,
+			     counter->approx_accuracy_range.over);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_approximate_compare_value);
+
+/**
+ * percpu_counter_tree_precise_compare - Precise comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate a precise comparison of two counter trees.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly compare counters which are far apart. Only cases where
+ * counter sums are within the accuracy range require precise counter
+ * sums.
+ *
+ * Return:
+ * * %0		- Counters are equal.
+ * * %-1	- Counter @a less than counter @b.
+ * * %1		- Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	long count_a = percpu_counter_tree_approximate_sum(a),
+	     count_b = percpu_counter_tree_approximate_sum(b);
+	unsigned long accuracy_a, accuracy_b;
+	long delta = count_a - count_b;
+	int res;
+
+	res = compare_delta(delta,
+			    a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+			    a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+	/* The values are distanced enough for an accurate approximated comparison. */
+	if (res)
+		return res;
+
+	/*
+	 * The approximated comparison is within the accuracy range, therefore at least one
+	 * precise sum is needed. Sum the counter which has the largest accuracy first.
+	 */
+	if (delta >= 0) {
+		accuracy_a = a->approx_accuracy_range.under;
+		accuracy_b = b->approx_accuracy_range.over;
+	} else {
+		accuracy_a = a->approx_accuracy_range.over;
+		accuracy_b = b->approx_accuracy_range.under;
+	}
+	if (accuracy_b < accuracy_a) {
+		count_a = percpu_counter_tree_precise_sum(a);
+		res = compare_delta(count_a - count_b,
+				    b->approx_accuracy_range.under,
+				    b->approx_accuracy_range.over);
+		if (res)
+			return res;
+		/* Precise sum of second counter is required. */
+		count_b = percpu_counter_tree_precise_sum(b);
+	} else {
+		count_b = percpu_counter_tree_precise_sum(b);
+		res = compare_delta(count_a - count_b,
+				    a->approx_accuracy_range.over,
+				    a->approx_accuracy_range.under);
+		if (res)
+			return res;
+		/* Precise sum of second counter is required. */
+		count_a = percpu_counter_tree_precise_sum(a);
+	}
+	if (count_a - count_b < 0)
+		return -1;
+	if (count_a - count_b > 0)
+		return 1;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_precise_compare);
+
+/**
+ * percpu_counter_tree_precise_compare_value - Precise comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate a precise comparison of a counter tree against a given value.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly identify whether the counter and value are far apart.
+ * Only cases where the value is within the counter accuracy range
+ * require a precise counter sum.
+ *
+ * Return:
+ * * %0		- The value @v is equal to the counter.
+ * * %-1	- The value @v is less than the counter.
+ * * %1		- The value @v is greater than the counter.
+ */
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, long v)
+{
+	long count = percpu_counter_tree_approximate_sum(counter);
+	int res;
+
+	res = compare_delta(v - count,
+			    counter->approx_accuracy_range.under,
+			    counter->approx_accuracy_range.over);
+	/* The values are distanced enough for an accurate approximated comparison. */
+	if (res)
+		return res;
+
+	/* Precise sum is required. */
+	count = percpu_counter_tree_precise_sum(counter);
+	if (v - count < 0)
+		return -1;
+	if (v - count > 0)
+		return 1;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_precise_compare_value);
+
+static
+void percpu_counter_tree_set_bias(struct percpu_counter_tree *counter, long bias)
+{
+	WRITE_ONCE(counter->bias, bias);
+}
+
+/**
+ * percpu_counter_tree_set - Set the counter tree sum to a given value.
+ * @counter: Counter to set.
+ * @v: Value to set.
+ *
+ * Set the counter sum to a given value. It can be useful for instance
+ * to reset the counter sum to 0. Note that even after setting the
+ * counter sum to a given value, the counter sum approximation can
+ * return any value within the accuracy range around that value.
+ */
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, long v)
+{
+	percpu_counter_tree_set_bias(counter,
+				     v - percpu_counter_tree_precise_sum_unbiased(counter));
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_set);
+
+/*
+ * percpu_counter_tree_items_size - Query the size required for counter tree items.
+ *
+ * Query the size of the memory area required to hold the counter tree
+ * items. This depends on the hardware topology and is invariant after
+ * boot.
+ *
+ * Return: Size required to hold tree items.
+ */
+size_t percpu_counter_tree_items_size(void)
+{
+	if (!nr_cpus_order)
+		return 0;
+	return counter_config->nr_items * sizeof(struct percpu_counter_tree_level_item);
+}
+EXPORT_SYMBOL_GPL(percpu_counter_tree_items_size);
+
+static void __init calculate_accuracy_topology(void)
+{
+	unsigned int nr_levels = counter_config->nr_levels, level;
+	unsigned int level_items = 1U << nr_cpus_order;
+	unsigned long batch_size = 1;
+
+	for (level = 0; level < nr_levels; level++) {
+		unsigned int n_arity_order = counter_config->n_arity_order[level];
+
+		/*
+		 * The accuracy multiplier is derived from a batch size of 1
+		 * to speed up calculating the accuracy at tree initialization.
+		 */
+		accuracy_multiplier += batch_size * level_items;
+		batch_size <<= n_arity_order;
+		level_items >>= n_arity_order;
+	}
+}
+
+int __init percpu_counter_tree_subsystem_init(void)
+{
+	nr_cpus_order = get_count_order(nr_cpu_ids);
+	if (WARN_ON_ONCE(nr_cpus_order >= ARRAY_SIZE(per_nr_cpu_order_config))) {
+		printk(KERN_ERR "Unsupported number of CPUs (%u)\n", nr_cpu_ids);
+		return -1;
+	}
+	counter_config = &per_nr_cpu_order_config[nr_cpus_order];
+	calculate_accuracy_topology();
+	return 0;
+}
diff --git a/lib/polynomial.c b/lib/polynomial.c
deleted file mode 100644
index 66d383445fec9..0000000000000
--- a/lib/polynomial.c
+++ /dev/null
@@ -1,108 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Generic polynomial calculation using integer coefficients.
- *
- * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
- *
- * Authors:
- *   Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru>
- *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/polynomial.h>
-
-/*
- * Originally this was part of drivers/hwmon/bt1-pvt.c.
- * There the following conversion is used and should serve as an example here:
- *
- * The original translation formulae of the temperature (in degrees of Celsius)
- * to PVT data and vice-versa are following:
- *
- * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) +
- *     1.7204e2
- * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) +
- *     3.1020e-1*(N^1) - 4.838e1
- *
- * where T = [-48.380, 147.438]C and N = [0, 1023].
- *
- * They must be accordingly altered to be suitable for the integer arithmetics.
- * The technique is called 'factor redistribution', which just makes sure the
- * multiplications and divisions are made so to have a result of the operations
- * within the integer numbers limit. In addition we need to translate the
- * formulae to accept millidegrees of Celsius. Here what they look like after
- * the alterations:
- *
- * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T +
- *     17204e2) / 1e4
- * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D -
- *     48380
- * where T = [-48380, 147438] mC and N = [0, 1023].
- *
- * static const struct polynomial poly_temp_to_N = {
- *         .total_divider = 10000,
- *         .terms = {
- *                 {4, 18322, 10000, 10000},
- *                 {3, 2343, 10000, 10},
- *                 {2, 87018, 10000, 10},
- *                 {1, 39269, 1000, 1},
- *                 {0, 1720400, 1, 1}
- *         }
- * };
- *
- * static const struct polynomial poly_N_to_temp = {
- *         .total_divider = 1,
- *         .terms = {
- *                 {4, -16743, 1000, 1},
- *                 {3, 81542, 1000, 1},
- *                 {2, -182010, 1000, 1},
- *                 {1, 310200, 1000, 1},
- *                 {0, -48380, 1, 1}
- *         }
- * };
- */
-
-/**
- * polynomial_calc - calculate a polynomial using integer arithmetic
- *
- * @poly: pointer to the descriptor of the polynomial
- * @data: input value of the polynimal
- *
- * Calculate the result of a polynomial using only integer arithmetic. For
- * this to work without too much loss of precision the coefficients has to
- * be altered. This is called factor redistribution.
- *
- * Returns the result of the polynomial calculation.
- */
-long polynomial_calc(const struct polynomial *poly, long data)
-{
-	const struct polynomial_term *term = poly->terms;
-	long total_divider = poly->total_divider ?: 1;
-	long tmp, ret = 0;
-	int deg;
-
-	/*
-	 * Here is the polynomial calculation function, which performs the
-	 * redistributed terms calculations. It's pretty straightforward.
-	 * We walk over each degree term up to the free one, and perform
-	 * the redistributed multiplication of the term coefficient, its
-	 * divider (as for the rationale fraction representation), data
-	 * power and the rational fraction divider leftover. Then all of
-	 * this is collected in a total sum variable, which value is
-	 * normalized by the total divider before being returned.
-	 */
-	do {
-		tmp = term->coef;
-		for (deg = 0; deg < term->deg; ++deg)
-			tmp = mult_frac(tmp, data, term->divider);
-		ret += tmp / term->divider_leftover;
-	} while ((term++)->deg);
-
-	return ret / total_divider;
-}
-EXPORT_SYMBOL_GPL(polynomial_calc);
-
-MODULE_DESCRIPTION("Generic polynomial calculations");
-MODULE_LICENSE("GPL");
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
new file mode 100644
index 0000000000000..97c1238064664
--- /dev/null
+++ b/lib/raid/Kconfig
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config XOR_BLOCKS
+	tristate
+
+# selected by architectures that provide an optimized XOR implementation
+config XOR_BLOCKS_ARCH
+	bool
+
+config XOR_KUNIT_TEST
+	tristate "KUnit tests for xor_gen" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	select XOR_BLOCKS
+	help
+	  Unit tests for the XOR library functions.
+
+	  This is intended to help people writing architecture-specific
+	  optimized versions.  If unsure, say N.
diff --git a/lib/raid/Makefile b/lib/raid/Makefile
new file mode 100644
index 0000000000000..3540fe846dc42
--- /dev/null
+++ b/lib/raid/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y				+= xor/
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
new file mode 100644
index 0000000000000..e19998b3c633f
--- /dev/null
+++ b/lib/raid/xor/Makefile
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-y			+= -I $(src)
+
+obj-$(CONFIG_XOR_BLOCKS)	+= xor.o
+
+xor-y				+= xor-core.o
+xor-y				+= xor-8regs.o
+xor-y				+= xor-32regs.o
+xor-y				+= xor-8regs-prefetch.o
+xor-y				+= xor-32regs-prefetch.o
+
+ifeq ($(CONFIG_XOR_BLOCKS_ARCH),y)
+CFLAGS_xor-core.o		+= -I$(src)/$(SRCARCH)
+endif
+
+xor-$(CONFIG_ALPHA)		+= alpha/xor.o
+xor-$(CONFIG_ARM)		+= arm/xor.o
+ifeq ($(CONFIG_ARM),y)
+xor-$(CONFIG_KERNEL_MODE_NEON)	+= arm/xor-neon.o arm/xor-neon-glue.o
+else
+xor-$(CONFIG_KERNEL_MODE_NEON)	+= arm64/xor-neon.o arm64/xor-neon-glue.o
+endif
+xor-$(CONFIG_CPU_HAS_LSX)	+= loongarch/xor_simd.o
+xor-$(CONFIG_CPU_HAS_LSX)	+= loongarch/xor_simd_glue.o
+xor-$(CONFIG_ALTIVEC)		+= powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
+xor-$(CONFIG_RISCV_ISA_V)	+= riscv/xor.o riscv/xor-glue.o
+xor-$(CONFIG_SPARC32)		+= sparc/xor-sparc32.o
+xor-$(CONFIG_SPARC64)		+= sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
+xor-$(CONFIG_S390)		+= s390/xor.o
+xor-$(CONFIG_X86_32)		+= x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o
+xor-$(CONFIG_X86_64)		+= x86/xor-avx.o x86/xor-sse.o
+obj-y				+= tests/
+
+CFLAGS_arm/xor-neon.o		+= $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm/xor-neon.o	+= $(CC_FLAGS_NO_FPU)
+
+CFLAGS_arm64/xor-neon.o		+= $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm64/xor-neon.o	+= $(CC_FLAGS_NO_FPU)
+
+CFLAGS_powerpc/xor_vmx.o	+= -mhard-float -maltivec \
+				   $(call cc-option,-mabi=altivec) \
+				   -isystem $(shell $(CC) -print-file-name=include)
diff --git a/lib/raid/xor/alpha/xor.c b/lib/raid/xor/alpha/xor.c
new file mode 100644
index 0000000000000..a8f72f2dd3a5e
--- /dev/null
+++ b/lib/raid/xor/alpha/xor.c
@@ -0,0 +1,848 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Optimized XOR parity functions for alpha EV5 and EV6
+ */
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+extern void
+xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
+	    const unsigned long * __restrict p2);
+extern void
+xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
+	    const unsigned long * __restrict p2,
+	    const unsigned long * __restrict p3);
+extern void
+xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
+	    const unsigned long * __restrict p2,
+	    const unsigned long * __restrict p3,
+	    const unsigned long * __restrict p4);
+extern void
+xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
+	    const unsigned long * __restrict p2,
+	    const unsigned long * __restrict p3,
+	    const unsigned long * __restrict p4,
+	    const unsigned long * __restrict p5);
+
+extern void
+xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2);
+extern void
+xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3);
+extern void
+xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3,
+		     const unsigned long * __restrict p4);
+extern void
+xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3,
+		     const unsigned long * __restrict p4,
+		     const unsigned long * __restrict p5);
+
+asm("								\n\
+	.text							\n\
+	.align 3						\n\
+	.ent xor_alpha_2					\n\
+xor_alpha_2:							\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+	.align 4						\n\
+2:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,8($17)						\n\
+	ldq $3,8($18)						\n\
+								\n\
+	ldq $4,16($17)						\n\
+	ldq $5,16($18)						\n\
+	ldq $6,24($17)						\n\
+	ldq $7,24($18)						\n\
+								\n\
+	ldq $19,32($17)						\n\
+	ldq $20,32($18)						\n\
+	ldq $21,40($17)						\n\
+	ldq $22,40($18)						\n\
+								\n\
+	ldq $23,48($17)						\n\
+	ldq $24,48($18)						\n\
+	ldq $25,56($17)						\n\
+	xor $0,$1,$0		# 7 cycles from $1 load		\n\
+								\n\
+	ldq $27,56($18)						\n\
+	xor $2,$3,$2						\n\
+	stq $0,0($17)						\n\
+	xor $4,$5,$4						\n\
+								\n\
+	stq $2,8($17)						\n\
+	xor $6,$7,$6						\n\
+	stq $4,16($17)						\n\
+	xor $19,$20,$19						\n\
+								\n\
+	stq $6,24($17)						\n\
+	xor $21,$22,$21						\n\
+	stq $19,32($17)						\n\
+	xor $23,$24,$23						\n\
+								\n\
+	stq $21,40($17)						\n\
+	xor $25,$27,$25						\n\
+	stq $23,48($17)						\n\
+	subq $16,1,$16						\n\
+								\n\
+	stq $25,56($17)						\n\
+	addq $17,64,$17						\n\
+	addq $18,64,$18						\n\
+	bgt $16,2b						\n\
+								\n\
+	ret							\n\
+	.end xor_alpha_2					\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_3					\n\
+xor_alpha_3:							\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+	.align 4						\n\
+3:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,8($17)						\n\
+								\n\
+	ldq $4,8($18)						\n\
+	ldq $6,16($17)						\n\
+	ldq $7,16($18)						\n\
+	ldq $21,24($17)						\n\
+								\n\
+	ldq $22,24($18)						\n\
+	ldq $24,32($17)						\n\
+	ldq $25,32($18)						\n\
+	ldq $5,8($19)						\n\
+								\n\
+	ldq $20,16($19)						\n\
+	ldq $23,24($19)						\n\
+	ldq $27,32($19)						\n\
+	nop							\n\
+								\n\
+	xor $0,$1,$1		# 8 cycles from $0 load		\n\
+	xor $3,$4,$4		# 6 cycles from $4 load		\n\
+	xor $6,$7,$7		# 6 cycles from $7 load		\n\
+	xor $21,$22,$22		# 5 cycles from $22 load	\n\
+								\n\
+	xor $1,$2,$2		# 9 cycles from $2 load		\n\
+	xor $24,$25,$25		# 5 cycles from $25 load	\n\
+	stq $2,0($17)						\n\
+	xor $4,$5,$5		# 6 cycles from $5 load		\n\
+								\n\
+	stq $5,8($17)						\n\
+	xor $7,$20,$20		# 7 cycles from $20 load	\n\
+	stq $20,16($17)						\n\
+	xor $22,$23,$23		# 7 cycles from $23 load	\n\
+								\n\
+	stq $23,24($17)						\n\
+	xor $25,$27,$27		# 7 cycles from $27 load	\n\
+	stq $27,32($17)						\n\
+	nop							\n\
+								\n\
+	ldq $0,40($17)						\n\
+	ldq $1,40($18)						\n\
+	ldq $3,48($17)						\n\
+	ldq $4,48($18)						\n\
+								\n\
+	ldq $6,56($17)						\n\
+	ldq $7,56($18)						\n\
+	ldq $2,40($19)						\n\
+	ldq $5,48($19)						\n\
+								\n\
+	ldq $20,56($19)						\n\
+	xor $0,$1,$1		# 4 cycles from $1 load		\n\
+	xor $3,$4,$4		# 5 cycles from $4 load		\n\
+	xor $6,$7,$7		# 5 cycles from $7 load		\n\
+								\n\
+	xor $1,$2,$2		# 4 cycles from $2 load		\n\
+	xor $4,$5,$5		# 5 cycles from $5 load		\n\
+	stq $2,40($17)						\n\
+	xor $7,$20,$20		# 4 cycles from $20 load	\n\
+								\n\
+	stq $5,48($17)						\n\
+	subq $16,1,$16						\n\
+	stq $20,56($17)						\n\
+	addq $19,64,$19						\n\
+								\n\
+	addq $18,64,$18						\n\
+	addq $17,64,$17						\n\
+	bgt $16,3b						\n\
+	ret							\n\
+	.end xor_alpha_3					\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_4					\n\
+xor_alpha_4:							\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+	.align 4						\n\
+4:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,0($20)						\n\
+								\n\
+	ldq $4,8($17)						\n\
+	ldq $5,8($18)						\n\
+	ldq $6,8($19)						\n\
+	ldq $7,8($20)						\n\
+								\n\
+	ldq $21,16($17)						\n\
+	ldq $22,16($18)						\n\
+	ldq $23,16($19)						\n\
+	ldq $24,16($20)						\n\
+								\n\
+	ldq $25,24($17)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+	ldq $27,24($18)						\n\
+	xor $2,$3,$3		# 6 cycles from $3 load		\n\
+								\n\
+	ldq $0,24($19)						\n\
+	xor $1,$3,$3						\n\
+	ldq $1,24($20)						\n\
+	xor $4,$5,$5		# 7 cycles from $5 load		\n\
+								\n\
+	stq $3,0($17)						\n\
+	xor $6,$7,$7						\n\
+	xor $21,$22,$22		# 7 cycles from $22 load	\n\
+	xor $5,$7,$7						\n\
+								\n\
+	stq $7,8($17)						\n\
+	xor $23,$24,$24		# 7 cycles from $24 load	\n\
+	ldq $2,32($17)						\n\
+	xor $22,$24,$24						\n\
+								\n\
+	ldq $3,32($18)						\n\
+	ldq $4,32($19)						\n\
+	ldq $5,32($20)						\n\
+	xor $25,$27,$27		# 8 cycles from $27 load	\n\
+								\n\
+	ldq $6,40($17)						\n\
+	ldq $7,40($18)						\n\
+	ldq $21,40($19)						\n\
+	ldq $22,40($20)						\n\
+								\n\
+	stq $24,16($17)						\n\
+	xor $0,$1,$1		# 9 cycles from $1 load		\n\
+	xor $2,$3,$3		# 5 cycles from $3 load		\n\
+	xor $27,$1,$1						\n\
+								\n\
+	stq $1,24($17)						\n\
+	xor $4,$5,$5		# 5 cycles from $5 load		\n\
+	ldq $23,48($17)						\n\
+	ldq $24,48($18)						\n\
+								\n\
+	ldq $25,48($19)						\n\
+	xor $3,$5,$5						\n\
+	ldq $27,48($20)						\n\
+	ldq $0,56($17)						\n\
+								\n\
+	ldq $1,56($18)						\n\
+	ldq $2,56($19)						\n\
+	xor $6,$7,$7		# 8 cycles from $6 load		\n\
+	ldq $3,56($20)						\n\
+								\n\
+	stq $5,32($17)						\n\
+	xor $21,$22,$22		# 8 cycles from $22 load	\n\
+	xor $7,$22,$22						\n\
+	xor $23,$24,$24		# 5 cycles from $24 load	\n\
+								\n\
+	stq $22,40($17)						\n\
+	xor $25,$27,$27		# 5 cycles from $27 load	\n\
+	xor $24,$27,$27						\n\
+	xor $0,$1,$1		# 5 cycles from $1 load		\n\
+								\n\
+	stq $27,48($17)						\n\
+	xor $2,$3,$3		# 4 cycles from $3 load		\n\
+	xor $1,$3,$3						\n\
+	subq $16,1,$16						\n\
+								\n\
+	stq $3,56($17)						\n\
+	addq $20,64,$20						\n\
+	addq $19,64,$19						\n\
+	addq $18,64,$18						\n\
+								\n\
+	addq $17,64,$17						\n\
+	bgt $16,4b						\n\
+	ret							\n\
+	.end xor_alpha_4					\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_5					\n\
+xor_alpha_5:							\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+	.align 4						\n\
+5:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,0($20)						\n\
+								\n\
+	ldq $4,0($21)						\n\
+	ldq $5,8($17)						\n\
+	ldq $6,8($18)						\n\
+	ldq $7,8($19)						\n\
+								\n\
+	ldq $22,8($20)						\n\
+	ldq $23,8($21)						\n\
+	ldq $24,16($17)						\n\
+	ldq $25,16($18)						\n\
+								\n\
+	ldq $27,16($19)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+	ldq $28,16($20)						\n\
+	xor $2,$3,$3		# 6 cycles from $3 load		\n\
+								\n\
+	ldq $0,16($21)						\n\
+	xor $1,$3,$3						\n\
+	ldq $1,24($17)						\n\
+	xor $3,$4,$4		# 7 cycles from $4 load		\n\
+								\n\
+	stq $4,0($17)						\n\
+	xor $5,$6,$6		# 7 cycles from $6 load		\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	xor $6,$23,$23		# 7 cycles from $23 load	\n\
+								\n\
+	ldq $2,24($18)						\n\
+	xor $22,$23,$23						\n\
+	ldq $3,24($19)						\n\
+	xor $24,$25,$25		# 8 cycles from $25 load	\n\
+								\n\
+	stq $23,8($17)						\n\
+	xor $25,$27,$27		# 8 cycles from $27 load	\n\
+	ldq $4,24($20)						\n\
+	xor $28,$0,$0		# 7 cycles from $0 load		\n\
+								\n\
+	ldq $5,24($21)						\n\
+	xor $27,$0,$0						\n\
+	ldq $6,32($17)						\n\
+	ldq $7,32($18)						\n\
+								\n\
+	stq $0,16($17)						\n\
+	xor $1,$2,$2		# 6 cycles from $2 load		\n\
+	ldq $22,32($19)						\n\
+	xor $3,$4,$4		# 4 cycles from $4 load		\n\
+								\n\
+	ldq $23,32($20)						\n\
+	xor $2,$4,$4						\n\
+	ldq $24,32($21)						\n\
+	ldq $25,40($17)						\n\
+								\n\
+	ldq $27,40($18)						\n\
+	ldq $28,40($19)						\n\
+	ldq $0,40($20)						\n\
+	xor $4,$5,$5		# 7 cycles from $5 load		\n\
+								\n\
+	stq $5,24($17)						\n\
+	xor $6,$7,$7		# 7 cycles from $7 load		\n\
+	ldq $1,40($21)						\n\
+	ldq $2,48($17)						\n\
+								\n\
+	ldq $3,48($18)						\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	ldq $4,48($19)						\n\
+	xor $23,$24,$24		# 6 cycles from $24 load	\n\
+								\n\
+	ldq $5,48($20)						\n\
+	xor $22,$24,$24						\n\
+	ldq $6,48($21)						\n\
+	xor $25,$27,$27		# 7 cycles from $27 load	\n\
+								\n\
+	stq $24,32($17)						\n\
+	xor $27,$28,$28		# 8 cycles from $28 load	\n\
+	ldq $7,56($17)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+								\n\
+	ldq $22,56($18)						\n\
+	ldq $23,56($19)						\n\
+	ldq $24,56($20)						\n\
+	ldq $25,56($21)						\n\
+								\n\
+	xor $28,$1,$1						\n\
+	xor $2,$3,$3		# 9 cycles from $3 load		\n\
+	xor $3,$4,$4		# 9 cycles from $4 load		\n\
+	xor $5,$6,$6		# 8 cycles from $6 load		\n\
+								\n\
+	stq $1,40($17)						\n\
+	xor $4,$6,$6						\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	xor $23,$24,$24		# 6 cycles from $24 load	\n\
+								\n\
+	stq $6,48($17)						\n\
+	xor $22,$24,$24						\n\
+	subq $16,1,$16						\n\
+	xor $24,$25,$25		# 8 cycles from $25 load	\n\
+								\n\
+	stq $25,56($17)						\n\
+	addq $21,64,$21						\n\
+	addq $20,64,$20						\n\
+	addq $19,64,$19						\n\
+								\n\
+	addq $18,64,$18						\n\
+	addq $17,64,$17						\n\
+	bgt $16,5b						\n\
+	ret							\n\
+	.end xor_alpha_5					\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_prefetch_2				\n\
+xor_alpha_prefetch_2:						\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+								\n\
+	ldq $31, 0($17)						\n\
+	ldq $31, 0($18)						\n\
+								\n\
+	ldq $31, 64($17)					\n\
+	ldq $31, 64($18)					\n\
+								\n\
+	ldq $31, 128($17)					\n\
+	ldq $31, 128($18)					\n\
+								\n\
+	ldq $31, 192($17)					\n\
+	ldq $31, 192($18)					\n\
+	.align 4						\n\
+2:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,8($17)						\n\
+	ldq $3,8($18)						\n\
+								\n\
+	ldq $4,16($17)						\n\
+	ldq $5,16($18)						\n\
+	ldq $6,24($17)						\n\
+	ldq $7,24($18)						\n\
+								\n\
+	ldq $19,32($17)						\n\
+	ldq $20,32($18)						\n\
+	ldq $21,40($17)						\n\
+	ldq $22,40($18)						\n\
+								\n\
+	ldq $23,48($17)						\n\
+	ldq $24,48($18)						\n\
+	ldq $25,56($17)						\n\
+	ldq $27,56($18)						\n\
+								\n\
+	ldq $31,256($17)					\n\
+	xor $0,$1,$0		# 8 cycles from $1 load		\n\
+	ldq $31,256($18)					\n\
+	xor $2,$3,$2						\n\
+								\n\
+	stq $0,0($17)						\n\
+	xor $4,$5,$4						\n\
+	stq $2,8($17)						\n\
+	xor $6,$7,$6						\n\
+								\n\
+	stq $4,16($17)						\n\
+	xor $19,$20,$19						\n\
+	stq $6,24($17)						\n\
+	xor $21,$22,$21						\n\
+								\n\
+	stq $19,32($17)						\n\
+	xor $23,$24,$23						\n\
+	stq $21,40($17)						\n\
+	xor $25,$27,$25						\n\
+								\n\
+	stq $23,48($17)						\n\
+	subq $16,1,$16						\n\
+	stq $25,56($17)						\n\
+	addq $17,64,$17						\n\
+								\n\
+	addq $18,64,$18						\n\
+	bgt $16,2b						\n\
+	ret							\n\
+	.end xor_alpha_prefetch_2				\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_prefetch_3				\n\
+xor_alpha_prefetch_3:						\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+								\n\
+	ldq $31, 0($17)						\n\
+	ldq $31, 0($18)						\n\
+	ldq $31, 0($19)						\n\
+								\n\
+	ldq $31, 64($17)					\n\
+	ldq $31, 64($18)					\n\
+	ldq $31, 64($19)					\n\
+								\n\
+	ldq $31, 128($17)					\n\
+	ldq $31, 128($18)					\n\
+	ldq $31, 128($19)					\n\
+								\n\
+	ldq $31, 192($17)					\n\
+	ldq $31, 192($18)					\n\
+	ldq $31, 192($19)					\n\
+	.align 4						\n\
+3:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,8($17)						\n\
+								\n\
+	ldq $4,8($18)						\n\
+	ldq $6,16($17)						\n\
+	ldq $7,16($18)						\n\
+	ldq $21,24($17)						\n\
+								\n\
+	ldq $22,24($18)						\n\
+	ldq $24,32($17)						\n\
+	ldq $25,32($18)						\n\
+	ldq $5,8($19)						\n\
+								\n\
+	ldq $20,16($19)						\n\
+	ldq $23,24($19)						\n\
+	ldq $27,32($19)						\n\
+	nop							\n\
+								\n\
+	xor $0,$1,$1		# 8 cycles from $0 load		\n\
+	xor $3,$4,$4		# 7 cycles from $4 load		\n\
+	xor $6,$7,$7		# 6 cycles from $7 load		\n\
+	xor $21,$22,$22		# 5 cycles from $22 load	\n\
+								\n\
+	xor $1,$2,$2		# 9 cycles from $2 load		\n\
+	xor $24,$25,$25		# 5 cycles from $25 load	\n\
+	stq $2,0($17)						\n\
+	xor $4,$5,$5		# 6 cycles from $5 load		\n\
+								\n\
+	stq $5,8($17)						\n\
+	xor $7,$20,$20		# 7 cycles from $20 load	\n\
+	stq $20,16($17)						\n\
+	xor $22,$23,$23		# 7 cycles from $23 load	\n\
+								\n\
+	stq $23,24($17)						\n\
+	xor $25,$27,$27		# 7 cycles from $27 load	\n\
+	stq $27,32($17)						\n\
+	nop							\n\
+								\n\
+	ldq $0,40($17)						\n\
+	ldq $1,40($18)						\n\
+	ldq $3,48($17)						\n\
+	ldq $4,48($18)						\n\
+								\n\
+	ldq $6,56($17)						\n\
+	ldq $7,56($18)						\n\
+	ldq $2,40($19)						\n\
+	ldq $5,48($19)						\n\
+								\n\
+	ldq $20,56($19)						\n\
+	ldq $31,256($17)					\n\
+	ldq $31,256($18)					\n\
+	ldq $31,256($19)					\n\
+								\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+	xor $3,$4,$4		# 5 cycles from $4 load		\n\
+	xor $6,$7,$7		# 5 cycles from $7 load		\n\
+	xor $1,$2,$2		# 4 cycles from $2 load		\n\
+								\n\
+	xor $4,$5,$5		# 5 cycles from $5 load		\n\
+	xor $7,$20,$20		# 4 cycles from $20 load	\n\
+	stq $2,40($17)						\n\
+	subq $16,1,$16						\n\
+								\n\
+	stq $5,48($17)						\n\
+	addq $19,64,$19						\n\
+	stq $20,56($17)						\n\
+	addq $18,64,$18						\n\
+								\n\
+	addq $17,64,$17						\n\
+	bgt $16,3b						\n\
+	ret							\n\
+	.end xor_alpha_prefetch_3				\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_prefetch_4				\n\
+xor_alpha_prefetch_4:						\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+								\n\
+	ldq $31, 0($17)						\n\
+	ldq $31, 0($18)						\n\
+	ldq $31, 0($19)						\n\
+	ldq $31, 0($20)						\n\
+								\n\
+	ldq $31, 64($17)					\n\
+	ldq $31, 64($18)					\n\
+	ldq $31, 64($19)					\n\
+	ldq $31, 64($20)					\n\
+								\n\
+	ldq $31, 128($17)					\n\
+	ldq $31, 128($18)					\n\
+	ldq $31, 128($19)					\n\
+	ldq $31, 128($20)					\n\
+								\n\
+	ldq $31, 192($17)					\n\
+	ldq $31, 192($18)					\n\
+	ldq $31, 192($19)					\n\
+	ldq $31, 192($20)					\n\
+	.align 4						\n\
+4:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,0($20)						\n\
+								\n\
+	ldq $4,8($17)						\n\
+	ldq $5,8($18)						\n\
+	ldq $6,8($19)						\n\
+	ldq $7,8($20)						\n\
+								\n\
+	ldq $21,16($17)						\n\
+	ldq $22,16($18)						\n\
+	ldq $23,16($19)						\n\
+	ldq $24,16($20)						\n\
+								\n\
+	ldq $25,24($17)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+	ldq $27,24($18)						\n\
+	xor $2,$3,$3		# 6 cycles from $3 load		\n\
+								\n\
+	ldq $0,24($19)						\n\
+	xor $1,$3,$3						\n\
+	ldq $1,24($20)						\n\
+	xor $4,$5,$5		# 7 cycles from $5 load		\n\
+								\n\
+	stq $3,0($17)						\n\
+	xor $6,$7,$7						\n\
+	xor $21,$22,$22		# 7 cycles from $22 load	\n\
+	xor $5,$7,$7						\n\
+								\n\
+	stq $7,8($17)						\n\
+	xor $23,$24,$24		# 7 cycles from $24 load	\n\
+	ldq $2,32($17)						\n\
+	xor $22,$24,$24						\n\
+								\n\
+	ldq $3,32($18)						\n\
+	ldq $4,32($19)						\n\
+	ldq $5,32($20)						\n\
+	xor $25,$27,$27		# 8 cycles from $27 load	\n\
+								\n\
+	ldq $6,40($17)						\n\
+	ldq $7,40($18)						\n\
+	ldq $21,40($19)						\n\
+	ldq $22,40($20)						\n\
+								\n\
+	stq $24,16($17)						\n\
+	xor $0,$1,$1		# 9 cycles from $1 load		\n\
+	xor $2,$3,$3		# 5 cycles from $3 load		\n\
+	xor $27,$1,$1						\n\
+								\n\
+	stq $1,24($17)						\n\
+	xor $4,$5,$5		# 5 cycles from $5 load		\n\
+	ldq $23,48($17)						\n\
+	xor $3,$5,$5						\n\
+								\n\
+	ldq $24,48($18)						\n\
+	ldq $25,48($19)						\n\
+	ldq $27,48($20)						\n\
+	ldq $0,56($17)						\n\
+								\n\
+	ldq $1,56($18)						\n\
+	ldq $2,56($19)						\n\
+	ldq $3,56($20)						\n\
+	xor $6,$7,$7		# 8 cycles from $6 load		\n\
+								\n\
+	ldq $31,256($17)					\n\
+	xor $21,$22,$22		# 8 cycles from $22 load	\n\
+	ldq $31,256($18)					\n\
+	xor $7,$22,$22						\n\
+								\n\
+	ldq $31,256($19)					\n\
+	xor $23,$24,$24		# 6 cycles from $24 load	\n\
+	ldq $31,256($20)					\n\
+	xor $25,$27,$27		# 6 cycles from $27 load	\n\
+								\n\
+	stq $5,32($17)						\n\
+	xor $24,$27,$27						\n\
+	xor $0,$1,$1		# 7 cycles from $1 load		\n\
+	xor $2,$3,$3		# 6 cycles from $3 load		\n\
+								\n\
+	stq $22,40($17)						\n\
+	xor $1,$3,$3						\n\
+	stq $27,48($17)						\n\
+	subq $16,1,$16						\n\
+								\n\
+	stq $3,56($17)						\n\
+	addq $20,64,$20						\n\
+	addq $19,64,$19						\n\
+	addq $18,64,$18						\n\
+								\n\
+	addq $17,64,$17						\n\
+	bgt $16,4b						\n\
+	ret							\n\
+	.end xor_alpha_prefetch_4				\n\
+								\n\
+	.align 3						\n\
+	.ent xor_alpha_prefetch_5				\n\
+xor_alpha_prefetch_5:						\n\
+	.prologue 0						\n\
+	srl $16, 6, $16						\n\
+								\n\
+	ldq $31, 0($17)						\n\
+	ldq $31, 0($18)						\n\
+	ldq $31, 0($19)						\n\
+	ldq $31, 0($20)						\n\
+	ldq $31, 0($21)						\n\
+								\n\
+	ldq $31, 64($17)					\n\
+	ldq $31, 64($18)					\n\
+	ldq $31, 64($19)					\n\
+	ldq $31, 64($20)					\n\
+	ldq $31, 64($21)					\n\
+								\n\
+	ldq $31, 128($17)					\n\
+	ldq $31, 128($18)					\n\
+	ldq $31, 128($19)					\n\
+	ldq $31, 128($20)					\n\
+	ldq $31, 128($21)					\n\
+								\n\
+	ldq $31, 192($17)					\n\
+	ldq $31, 192($18)					\n\
+	ldq $31, 192($19)					\n\
+	ldq $31, 192($20)					\n\
+	ldq $31, 192($21)					\n\
+	.align 4						\n\
+5:								\n\
+	ldq $0,0($17)						\n\
+	ldq $1,0($18)						\n\
+	ldq $2,0($19)						\n\
+	ldq $3,0($20)						\n\
+								\n\
+	ldq $4,0($21)						\n\
+	ldq $5,8($17)						\n\
+	ldq $6,8($18)						\n\
+	ldq $7,8($19)						\n\
+								\n\
+	ldq $22,8($20)						\n\
+	ldq $23,8($21)						\n\
+	ldq $24,16($17)						\n\
+	ldq $25,16($18)						\n\
+								\n\
+	ldq $27,16($19)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+	ldq $28,16($20)						\n\
+	xor $2,$3,$3		# 6 cycles from $3 load		\n\
+								\n\
+	ldq $0,16($21)						\n\
+	xor $1,$3,$3						\n\
+	ldq $1,24($17)						\n\
+	xor $3,$4,$4		# 7 cycles from $4 load		\n\
+								\n\
+	stq $4,0($17)						\n\
+	xor $5,$6,$6		# 7 cycles from $6 load		\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	xor $6,$23,$23		# 7 cycles from $23 load	\n\
+								\n\
+	ldq $2,24($18)						\n\
+	xor $22,$23,$23						\n\
+	ldq $3,24($19)						\n\
+	xor $24,$25,$25		# 8 cycles from $25 load	\n\
+								\n\
+	stq $23,8($17)						\n\
+	xor $25,$27,$27		# 8 cycles from $27 load	\n\
+	ldq $4,24($20)						\n\
+	xor $28,$0,$0		# 7 cycles from $0 load		\n\
+								\n\
+	ldq $5,24($21)						\n\
+	xor $27,$0,$0						\n\
+	ldq $6,32($17)						\n\
+	ldq $7,32($18)						\n\
+								\n\
+	stq $0,16($17)						\n\
+	xor $1,$2,$2		# 6 cycles from $2 load		\n\
+	ldq $22,32($19)						\n\
+	xor $3,$4,$4		# 4 cycles from $4 load		\n\
+								\n\
+	ldq $23,32($20)						\n\
+	xor $2,$4,$4						\n\
+	ldq $24,32($21)						\n\
+	ldq $25,40($17)						\n\
+								\n\
+	ldq $27,40($18)						\n\
+	ldq $28,40($19)						\n\
+	ldq $0,40($20)						\n\
+	xor $4,$5,$5		# 7 cycles from $5 load		\n\
+								\n\
+	stq $5,24($17)						\n\
+	xor $6,$7,$7		# 7 cycles from $7 load		\n\
+	ldq $1,40($21)						\n\
+	ldq $2,48($17)						\n\
+								\n\
+	ldq $3,48($18)						\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	ldq $4,48($19)						\n\
+	xor $23,$24,$24		# 6 cycles from $24 load	\n\
+								\n\
+	ldq $5,48($20)						\n\
+	xor $22,$24,$24						\n\
+	ldq $6,48($21)						\n\
+	xor $25,$27,$27		# 7 cycles from $27 load	\n\
+								\n\
+	stq $24,32($17)						\n\
+	xor $27,$28,$28		# 8 cycles from $28 load	\n\
+	ldq $7,56($17)						\n\
+	xor $0,$1,$1		# 6 cycles from $1 load		\n\
+								\n\
+	ldq $22,56($18)						\n\
+	ldq $23,56($19)						\n\
+	ldq $24,56($20)						\n\
+	ldq $25,56($21)						\n\
+								\n\
+	ldq $31,256($17)					\n\
+	xor $28,$1,$1						\n\
+	ldq $31,256($18)					\n\
+	xor $2,$3,$3		# 9 cycles from $3 load		\n\
+								\n\
+	ldq $31,256($19)					\n\
+	xor $3,$4,$4		# 9 cycles from $4 load		\n\
+	ldq $31,256($20)					\n\
+	xor $5,$6,$6		# 8 cycles from $6 load		\n\
+								\n\
+	stq $1,40($17)						\n\
+	xor $4,$6,$6						\n\
+	xor $7,$22,$22		# 7 cycles from $22 load	\n\
+	xor $23,$24,$24		# 6 cycles from $24 load	\n\
+								\n\
+	stq $6,48($17)						\n\
+	xor $22,$24,$24						\n\
+	ldq $31,256($21)					\n\
+	xor $24,$25,$25		# 8 cycles from $25 load	\n\
+								\n\
+	stq $25,56($17)						\n\
+	subq $16,1,$16						\n\
+	addq $21,64,$21						\n\
+	addq $20,64,$20						\n\
+								\n\
+	addq $19,64,$19						\n\
+	addq $18,64,$18						\n\
+	addq $17,64,$17						\n\
+	bgt $16,5b						\n\
+								\n\
+	ret							\n\
+	.end xor_alpha_prefetch_5				\n\
+");
+
+DO_XOR_BLOCKS(alpha, xor_alpha_2, xor_alpha_3, xor_alpha_4, xor_alpha_5);
+
+struct xor_block_template xor_block_alpha = {
+	.name		= "alpha",
+	.xor_gen	= xor_gen_alpha,
+};
+
+DO_XOR_BLOCKS(alpha_prefetch, xor_alpha_prefetch_2, xor_alpha_prefetch_3,
+		xor_alpha_prefetch_4, xor_alpha_prefetch_5);
+
+struct xor_block_template xor_block_alpha_prefetch = {
+	.name		= "alpha prefetch",
+	.xor_gen	= xor_gen_alpha_prefetch,
+};
diff --git a/lib/raid/xor/alpha/xor_arch.h b/lib/raid/xor/alpha/xor_arch.h
new file mode 100644
index 0000000000000..0dcfea578a488
--- /dev/null
+++ b/lib/raid/xor/alpha/xor_arch.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/special_insns.h>
+
+extern struct xor_block_template xor_block_alpha;
+extern struct xor_block_template xor_block_alpha_prefetch;
+
+/*
+ * Force the use of alpha_prefetch if EV6, as it is significantly faster in the
+ * cold cache case.
+ */
+static __always_inline void __init arch_xor_init(void)
+{
+	if (implver() == IMPLVER_EV6) {
+		xor_force(&xor_block_alpha_prefetch);
+	} else {
+		xor_register(&xor_block_8regs);
+		xor_register(&xor_block_32regs);
+		xor_register(&xor_block_alpha);
+		xor_register(&xor_block_alpha_prefetch);
+	}
+}
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
new file mode 100644
index 0000000000000..cea39e0199048
--- /dev/null
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Copyright (C) 2001 Russell King
+ */
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	kernel_neon_begin();
+	xor_gen_neon_inner(dest, srcs, src_cnt, bytes);
+	kernel_neon_end();
+}
+
+struct xor_block_template xor_block_neon = {
+	.name		= "neon",
+	.xor_gen	= xor_gen_neon,
+};
diff --git a/lib/raid/xor/arm/xor-neon.c b/lib/raid/xor/arm/xor-neon.c
new file mode 100644
index 0000000000000..23147e3a79044
--- /dev/null
+++ b/lib/raid/xor/arm/xor-neon.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+#ifndef __ARM_NEON__
+#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
+#endif
+
+/*
+ * Pull in the reference implementations while instructing GCC (through
+ * -ftree-vectorize) to attempt to exploit implicit parallelism and emit
+ * NEON instructions. Clang does this by default at O2 so no pragma is
+ * needed.
+ */
+#ifdef CONFIG_CC_IS_GCC
+#pragma GCC optimize "tree-vectorize"
+#endif
+
+#define NO_TEMPLATE
+#include "../xor-8regs.c"
+
+__DO_XOR_BLOCKS(neon_inner, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5);
diff --git a/lib/raid/xor/arm/xor.c b/lib/raid/xor/arm/xor.c
new file mode 100644
index 0000000000000..45139b6c55eaa
--- /dev/null
+++ b/lib/raid/xor/arm/xor.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Copyright (C) 2001 Russell King
+ */
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+#define __XOR(a1, a2) a1 ^= a2
+
+#define GET_BLOCK_2(dst) \
+	__asm__("ldmia	%0, {%1, %2}" \
+		: "=r" (dst), "=r" (a1), "=r" (a2) \
+		: "0" (dst))
+
+#define GET_BLOCK_4(dst) \
+	__asm__("ldmia	%0, {%1, %2, %3, %4}" \
+		: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
+		: "0" (dst))
+
+#define XOR_BLOCK_2(src) \
+	__asm__("ldmia	%0!, {%1, %2}" \
+		: "=r" (src), "=r" (b1), "=r" (b2) \
+		: "0" (src)); \
+	__XOR(a1, b1); __XOR(a2, b2);
+
+#define XOR_BLOCK_4(src) \
+	__asm__("ldmia	%0!, {%1, %2, %3, %4}" \
+		: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
+		: "0" (src)); \
+	__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
+
+#define PUT_BLOCK_2(dst) \
+	__asm__ __volatile__("stmia	%0!, {%2, %3}" \
+		: "=r" (dst) \
+		: "0" (dst), "r" (a1), "r" (a2))
+
+#define PUT_BLOCK_4(dst) \
+	__asm__ __volatile__("stmia	%0!, {%2, %3, %4, %5}" \
+		: "=r" (dst) \
+		: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
+
+static void
+xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2)
+{
+	unsigned int lines = bytes / sizeof(unsigned long) / 4;
+	register unsigned int a1 __asm__("r4");
+	register unsigned int a2 __asm__("r5");
+	register unsigned int a3 __asm__("r6");
+	register unsigned int a4 __asm__("r10");
+	register unsigned int b1 __asm__("r8");
+	register unsigned int b2 __asm__("r9");
+	register unsigned int b3 __asm__("ip");
+	register unsigned int b4 __asm__("lr");
+
+	do {
+		GET_BLOCK_4(p1);
+		XOR_BLOCK_4(p2);
+		PUT_BLOCK_4(p1);
+	} while (--lines);
+}
+
+static void
+xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3)
+{
+	unsigned int lines = bytes / sizeof(unsigned long) / 4;
+	register unsigned int a1 __asm__("r4");
+	register unsigned int a2 __asm__("r5");
+	register unsigned int a3 __asm__("r6");
+	register unsigned int a4 __asm__("r10");
+	register unsigned int b1 __asm__("r8");
+	register unsigned int b2 __asm__("r9");
+	register unsigned int b3 __asm__("ip");
+	register unsigned int b4 __asm__("lr");
+
+	do {
+		GET_BLOCK_4(p1);
+		XOR_BLOCK_4(p2);
+		XOR_BLOCK_4(p3);
+		PUT_BLOCK_4(p1);
+	} while (--lines);
+}
+
+static void
+xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4)
+{
+	unsigned int lines = bytes / sizeof(unsigned long) / 2;
+	register unsigned int a1 __asm__("r8");
+	register unsigned int a2 __asm__("r9");
+	register unsigned int b1 __asm__("ip");
+	register unsigned int b2 __asm__("lr");
+
+	do {
+		GET_BLOCK_2(p1);
+		XOR_BLOCK_2(p2);
+		XOR_BLOCK_2(p3);
+		XOR_BLOCK_2(p4);
+		PUT_BLOCK_2(p1);
+	} while (--lines);
+}
+
+static void
+xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4,
+	       const unsigned long * __restrict p5)
+{
+	unsigned int lines = bytes / sizeof(unsigned long) / 2;
+	register unsigned int a1 __asm__("r8");
+	register unsigned int a2 __asm__("r9");
+	register unsigned int b1 __asm__("ip");
+	register unsigned int b2 __asm__("lr");
+
+	do {
+		GET_BLOCK_2(p1);
+		XOR_BLOCK_2(p2);
+		XOR_BLOCK_2(p3);
+		XOR_BLOCK_2(p4);
+		XOR_BLOCK_2(p5);
+		PUT_BLOCK_2(p1);
+	} while (--lines);
+}
+
+DO_XOR_BLOCKS(arm4regs, xor_arm4regs_2, xor_arm4regs_3, xor_arm4regs_4,
+		xor_arm4regs_5);
+
+struct xor_block_template xor_block_arm4regs = {
+	.name		= "arm4regs",
+	.xor_gen	= xor_gen_arm4regs,
+};
diff --git a/lib/raid/xor/arm/xor_arch.h b/lib/raid/xor/arm/xor_arch.h
new file mode 100644
index 0000000000000..775ff835df656
--- /dev/null
+++ b/lib/raid/xor/arm/xor_arch.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ *  Copyright (C) 2001 Russell King
+ */
+#include <asm/neon.h>
+
+extern struct xor_block_template xor_block_arm4regs;
+extern struct xor_block_template xor_block_neon;
+
+void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes);
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_arm4regs);
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_32regs);
+#ifdef CONFIG_KERNEL_MODE_NEON
+	if (cpu_has_neon())
+		xor_register(&xor_block_neon);
+#endif
+}
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
new file mode 100644
index 0000000000000..f0284f86feb4c
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Authors: Jackie Liu <liuyun01@kylinos.cn>
+ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
+ */
+
+#include <asm/simd.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+#include "xor-neon.h"
+
+#define XOR_TEMPLATE(_name)						\
+static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
+		unsigned int bytes)					\
+{									\
+	scoped_ksimd()							\
+		xor_gen_##_name##_inner(dest, srcs, src_cnt, bytes);	\
+}									\
+									\
+struct xor_block_template xor_block_##_name = {				\
+	.name   	= __stringify(_name),				\
+	.xor_gen	= xor_gen_##_name,				\
+};
+
+XOR_TEMPLATE(neon);
+XOR_TEMPLATE(eor3);
diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
new file mode 100644
index 0000000000000..97ef3cb924968
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon.c
@@ -0,0 +1,312 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Authors: Jackie Liu <liuyun01@kylinos.cn>
+ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
+ */
+
+#include <linux/cache.h>
+#include <asm/neon-intrinsics.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+#include "xor-neon.h"
+
+static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 */
+		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
+		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
+		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
+		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
+
+		/* store */
+		vst1q_u64(dp1 +  0, v0);
+		vst1q_u64(dp1 +  2, v1);
+		vst1q_u64(dp1 +  4, v2);
+		vst1q_u64(dp1 +  6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+	} while (--lines > 0);
+}
+
+static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 */
+		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
+		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
+		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
+		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
+
+		/* p1 ^= p3 */
+		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
+
+		/* store */
+		vst1q_u64(dp1 +  0, v0);
+		vst1q_u64(dp1 +  2, v1);
+		vst1q_u64(dp1 +  4, v2);
+		vst1q_u64(dp1 +  6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+	} while (--lines > 0);
+}
+
+static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3,
+		const unsigned long * __restrict p4)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+	uint64_t *dp4 = (uint64_t *)p4;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 */
+		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
+		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
+		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
+		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
+
+		/* p1 ^= p3 */
+		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
+
+		/* p1 ^= p4 */
+		v0 = veorq_u64(v0, vld1q_u64(dp4 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp4 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp4 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp4 +  6));
+
+		/* store */
+		vst1q_u64(dp1 +  0, v0);
+		vst1q_u64(dp1 +  2, v1);
+		vst1q_u64(dp1 +  4, v2);
+		vst1q_u64(dp1 +  6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+		dp4 += 8;
+	} while (--lines > 0);
+}
+
+static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3,
+		const unsigned long * __restrict p4,
+		const unsigned long * __restrict p5)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+	uint64_t *dp4 = (uint64_t *)p4;
+	uint64_t *dp5 = (uint64_t *)p5;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 */
+		v0 = veorq_u64(vld1q_u64(dp1 +  0), vld1q_u64(dp2 +  0));
+		v1 = veorq_u64(vld1q_u64(dp1 +  2), vld1q_u64(dp2 +  2));
+		v2 = veorq_u64(vld1q_u64(dp1 +  4), vld1q_u64(dp2 +  4));
+		v3 = veorq_u64(vld1q_u64(dp1 +  6), vld1q_u64(dp2 +  6));
+
+		/* p1 ^= p3 */
+		v0 = veorq_u64(v0, vld1q_u64(dp3 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp3 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp3 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp3 +  6));
+
+		/* p1 ^= p4 */
+		v0 = veorq_u64(v0, vld1q_u64(dp4 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp4 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp4 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp4 +  6));
+
+		/* p1 ^= p5 */
+		v0 = veorq_u64(v0, vld1q_u64(dp5 +  0));
+		v1 = veorq_u64(v1, vld1q_u64(dp5 +  2));
+		v2 = veorq_u64(v2, vld1q_u64(dp5 +  4));
+		v3 = veorq_u64(v3, vld1q_u64(dp5 +  6));
+
+		/* store */
+		vst1q_u64(dp1 +  0, v0);
+		vst1q_u64(dp1 +  2, v1);
+		vst1q_u64(dp1 +  4, v2);
+		vst1q_u64(dp1 +  6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+		dp4 += 8;
+		dp5 += 8;
+	} while (--lines > 0);
+}
+
+__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+		__xor_neon_5);
+
+static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
+{
+	uint64x2_t res;
+
+	asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n"
+	    "eor3 %0.16b, %1.16b, %2.16b, %3.16b"
+	    : "=w"(res) : "w"(p), "w"(q), "w"(r));
+	return res;
+}
+
+static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 ^ p3 */
+		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
+			  vld1q_u64(dp3 + 0));
+		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
+			  vld1q_u64(dp3 + 2));
+		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
+			  vld1q_u64(dp3 + 4));
+		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
+			  vld1q_u64(dp3 + 6));
+
+		/* store */
+		vst1q_u64(dp1 + 0, v0);
+		vst1q_u64(dp1 + 2, v1);
+		vst1q_u64(dp1 + 4, v2);
+		vst1q_u64(dp1 + 6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+	} while (--lines > 0);
+}
+
+static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3,
+		const unsigned long * __restrict p4)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+	uint64_t *dp4 = (uint64_t *)p4;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 ^ p3 */
+		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
+			  vld1q_u64(dp3 + 0));
+		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
+			  vld1q_u64(dp3 + 2));
+		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
+			  vld1q_u64(dp3 + 4));
+		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
+			  vld1q_u64(dp3 + 6));
+
+		/* p1 ^= p4 */
+		v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
+		v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
+		v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
+		v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
+
+		/* store */
+		vst1q_u64(dp1 + 0, v0);
+		vst1q_u64(dp1 + 2, v1);
+		vst1q_u64(dp1 + 4, v2);
+		vst1q_u64(dp1 + 6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+		dp4 += 8;
+	} while (--lines > 0);
+}
+
+static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+		const unsigned long * __restrict p2,
+		const unsigned long * __restrict p3,
+		const unsigned long * __restrict p4,
+		const unsigned long * __restrict p5)
+{
+	uint64_t *dp1 = (uint64_t *)p1;
+	uint64_t *dp2 = (uint64_t *)p2;
+	uint64_t *dp3 = (uint64_t *)p3;
+	uint64_t *dp4 = (uint64_t *)p4;
+	uint64_t *dp5 = (uint64_t *)p5;
+
+	register uint64x2_t v0, v1, v2, v3;
+	long lines = bytes / (sizeof(uint64x2_t) * 4);
+
+	do {
+		/* p1 ^= p2 ^ p3 */
+		v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
+			  vld1q_u64(dp3 + 0));
+		v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
+			  vld1q_u64(dp3 + 2));
+		v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
+			  vld1q_u64(dp3 + 4));
+		v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
+			  vld1q_u64(dp3 + 6));
+
+		/* p1 ^= p4 ^ p5 */
+		v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0));
+		v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2));
+		v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4));
+		v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6));
+
+		/* store */
+		vst1q_u64(dp1 + 0, v0);
+		vst1q_u64(dp1 + 2, v1);
+		vst1q_u64(dp1 + 4, v2);
+		vst1q_u64(dp1 + 6, v3);
+
+		dp1 += 8;
+		dp2 += 8;
+		dp3 += 8;
+		dp4 += 8;
+		dp5 += 8;
+	} while (--lines > 0);
+}
+
+__DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4,
+		__xor_eor3_5);
diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h
new file mode 100644
index 0000000000000..514699ba8f5f8
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon.h
@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes);
+void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes);
diff --git a/lib/raid/xor/arm64/xor_arch.h b/lib/raid/xor/arm64/xor_arch.h
new file mode 100644
index 0000000000000..5dbb40319501b
--- /dev/null
+++ b/lib/raid/xor/arm64/xor_arch.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Authors: Jackie Liu <liuyun01@kylinos.cn>
+ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
+ */
+#include <asm/simd.h>
+
+extern struct xor_block_template xor_block_neon;
+extern struct xor_block_template xor_block_eor3;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_32regs);
+	if (cpu_has_neon()) {
+		if (cpu_have_named_feature(SHA3))
+			xor_register(&xor_block_eor3);
+		else
+			xor_register(&xor_block_neon);
+	}
+}
diff --git a/lib/raid/xor/loongarch/xor_arch.h b/lib/raid/xor/loongarch/xor_arch.h
new file mode 100644
index 0000000000000..fe5e8244fd0eb
--- /dev/null
+++ b/lib/raid/xor/loongarch/xor_arch.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
+ */
+#include <asm/cpu-features.h>
+
+/*
+ * For grins, also test the generic routines.
+ *
+ * More importantly: it cannot be ruled out at this point of time, that some
+ * future (maybe reduced) models could run the vector algorithms slower than
+ * the scalar ones, maybe for errata or micro-op reasons. It may be
+ * appropriate to revisit this after one or two more uarch generations.
+ */
+
+extern struct xor_block_template xor_block_lsx;
+extern struct xor_block_template xor_block_lasx;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_8regs_p);
+	xor_register(&xor_block_32regs);
+	xor_register(&xor_block_32regs_p);
+#ifdef CONFIG_CPU_HAS_LSX
+	if (cpu_has_lsx)
+		xor_register(&xor_block_lsx);
+#endif
+#ifdef CONFIG_CPU_HAS_LASX
+	if (cpu_has_lasx)
+		xor_register(&xor_block_lasx);
+#endif
+}
diff --git a/lib/raid/xor/loongarch/xor_simd.c b/lib/raid/xor/loongarch/xor_simd.c
new file mode 100644
index 0000000000000..84cd24b728c47
--- /dev/null
+++ b/lib/raid/xor/loongarch/xor_simd.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * LoongArch SIMD XOR operations
+ *
+ * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
+ */
+
+#include "xor_simd.h"
+
+/*
+ * Process one cache line (64 bytes) per loop. This is assuming all future
+ * popular LoongArch cores are similar performance-characteristics-wise to the
+ * current models.
+ */
+#define LINE_WIDTH 64
+
+#ifdef CONFIG_CPU_HAS_LSX
+
+#define LD(reg, base, offset)	\
+	"vld $vr" #reg ", %[" #base "], " #offset "\n\t"
+#define ST(reg, base, offset)	\
+	"vst $vr" #reg ", %[" #base "], " #offset "\n\t"
+#define XOR(dj, k)	"vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t"
+
+#define LD_INOUT_LINE(base)	\
+	LD(0, base, 0)		\
+	LD(1, base, 16)		\
+	LD(2, base, 32)		\
+	LD(3, base, 48)
+
+#define LD_AND_XOR_LINE(base)	\
+	LD(4, base, 0)		\
+	LD(5, base, 16)		\
+	LD(6, base, 32)		\
+	LD(7, base, 48)		\
+	XOR(0, 4)		\
+	XOR(1, 5)		\
+	XOR(2, 6)		\
+	XOR(3, 7)
+
+#define ST_LINE(base)		\
+	ST(0, base, 0)		\
+	ST(1, base, 16)		\
+	ST(2, base, 32)		\
+	ST(3, base, 48)
+
+#define XOR_FUNC_NAME(nr) __xor_lsx_##nr
+#include "xor_template.c"
+
+#undef LD
+#undef ST
+#undef XOR
+#undef LD_INOUT_LINE
+#undef LD_AND_XOR_LINE
+#undef ST_LINE
+#undef XOR_FUNC_NAME
+
+#endif /* CONFIG_CPU_HAS_LSX */
+
+#ifdef CONFIG_CPU_HAS_LASX
+
+#define LD(reg, base, offset)	\
+	"xvld $xr" #reg ", %[" #base "], " #offset "\n\t"
+#define ST(reg, base, offset)	\
+	"xvst $xr" #reg ", %[" #base "], " #offset "\n\t"
+#define XOR(dj, k)	"xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t"
+
+#define LD_INOUT_LINE(base)	\
+	LD(0, base, 0)		\
+	LD(1, base, 32)
+
+#define LD_AND_XOR_LINE(base)	\
+	LD(2, base, 0)		\
+	LD(3, base, 32)		\
+	XOR(0, 2)		\
+	XOR(1, 3)
+
+#define ST_LINE(base)		\
+	ST(0, base, 0)		\
+	ST(1, base, 32)
+
+#define XOR_FUNC_NAME(nr) __xor_lasx_##nr
+#include "xor_template.c"
+
+#undef LD
+#undef ST
+#undef XOR
+#undef LD_INOUT_LINE
+#undef LD_AND_XOR_LINE
+#undef ST_LINE
+#undef XOR_FUNC_NAME
+
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/xor/loongarch/xor_simd.h b/lib/raid/xor/loongarch/xor_simd.h
new file mode 100644
index 0000000000000..f50f32514d804
--- /dev/null
+++ b/lib/raid/xor/loongarch/xor_simd.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Simple interface to link xor_simd.c and xor_simd_glue.c
+ *
+ * Separating these files ensures that no SIMD instructions are run outside of
+ * the kfpu critical section.
+ */
+
+#ifndef __LOONGARCH_LIB_XOR_SIMD_H
+#define __LOONGARCH_LIB_XOR_SIMD_H
+
+#ifdef CONFIG_CPU_HAS_LSX
+void __xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
+		 const unsigned long * __restrict p2);
+void __xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
+		 const unsigned long * __restrict p2, const unsigned long * __restrict p3);
+void __xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
+		 const unsigned long * __restrict p2, const unsigned long * __restrict p3,
+		 const unsigned long * __restrict p4);
+void __xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
+		 const unsigned long * __restrict p2, const unsigned long * __restrict p3,
+		 const unsigned long * __restrict p4, const unsigned long * __restrict p5);
+#endif /* CONFIG_CPU_HAS_LSX */
+
+#ifdef CONFIG_CPU_HAS_LASX
+void __xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
+		  const unsigned long * __restrict p2);
+void __xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
+		  const unsigned long * __restrict p2, const unsigned long * __restrict p3);
+void __xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
+		  const unsigned long * __restrict p2, const unsigned long * __restrict p3,
+		  const unsigned long * __restrict p4);
+void __xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
+		  const unsigned long * __restrict p2, const unsigned long * __restrict p3,
+		  const unsigned long * __restrict p4, const unsigned long * __restrict p5);
+#endif /* CONFIG_CPU_HAS_LASX */
+
+#endif /* __LOONGARCH_LIB_XOR_SIMD_H */
diff --git a/lib/raid/xor/loongarch/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c
new file mode 100644
index 0000000000000..7f324d924f879
--- /dev/null
+++ b/lib/raid/xor/loongarch/xor_simd_glue.c
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * LoongArch SIMD XOR operations
+ *
+ * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
+ */
+
+#include <linux/sched.h>
+#include <asm/fpu.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+#include "xor_simd.h"
+
+#define MAKE_XOR_GLUES(flavor)							\
+DO_XOR_BLOCKS(flavor##_inner, __xor_##flavor##_2, __xor_##flavor##_3,		\
+		__xor_##flavor##_4, __xor_##flavor##_5);			\
+										\
+static void xor_gen_##flavor(void *dest, void **srcs, unsigned int src_cnt,	\
+		unsigned int bytes)						\
+{										\
+	kernel_fpu_begin();							\
+	xor_gen_##flavor##_inner(dest, srcs, src_cnt, bytes);			\
+	kernel_fpu_end();							\
+}										\
+										\
+struct xor_block_template xor_block_##flavor = {				\
+	.name		= __stringify(flavor),					\
+	.xor_gen	= xor_gen_##flavor					\
+}
+
+#ifdef CONFIG_CPU_HAS_LSX
+MAKE_XOR_GLUES(lsx);
+#endif /* CONFIG_CPU_HAS_LSX */
+
+#ifdef CONFIG_CPU_HAS_LASX
+MAKE_XOR_GLUES(lasx);
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/xor/loongarch/xor_template.c b/lib/raid/xor/loongarch/xor_template.c
new file mode 100644
index 0000000000000..0358ced7fe333
--- /dev/null
+++ b/lib/raid/xor/loongarch/xor_template.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
+ *
+ * Template for XOR operations, instantiated in xor_simd.c.
+ *
+ * Expected preprocessor definitions:
+ *
+ * - LINE_WIDTH
+ * - XOR_FUNC_NAME(nr)
+ * - LD_INOUT_LINE(buf)
+ * - LD_AND_XOR_LINE(buf)
+ * - ST_LINE(buf)
+ */
+
+void XOR_FUNC_NAME(2)(unsigned long bytes,
+		      unsigned long * __restrict v1,
+		      const unsigned long * __restrict v2)
+{
+	unsigned long lines = bytes / LINE_WIDTH;
+
+	do {
+		__asm__ __volatile__ (
+			LD_INOUT_LINE(v1)
+			LD_AND_XOR_LINE(v2)
+			ST_LINE(v1)
+		: : [v1] "r"(v1), [v2] "r"(v2) : "memory"
+		);
+
+		v1 += LINE_WIDTH / sizeof(unsigned long);
+		v2 += LINE_WIDTH / sizeof(unsigned long);
+	} while (--lines > 0);
+}
+
+void XOR_FUNC_NAME(3)(unsigned long bytes,
+		      unsigned long * __restrict v1,
+		      const unsigned long * __restrict v2,
+		      const unsigned long * __restrict v3)
+{
+	unsigned long lines = bytes / LINE_WIDTH;
+
+	do {
+		__asm__ __volatile__ (
+			LD_INOUT_LINE(v1)
+			LD_AND_XOR_LINE(v2)
+			LD_AND_XOR_LINE(v3)
+			ST_LINE(v1)
+		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory"
+		);
+
+		v1 += LINE_WIDTH / sizeof(unsigned long);
+		v2 += LINE_WIDTH / sizeof(unsigned long);
+		v3 += LINE_WIDTH / sizeof(unsigned long);
+	} while (--lines > 0);
+}
+
+void XOR_FUNC_NAME(4)(unsigned long bytes,
+		      unsigned long * __restrict v1,
+		      const unsigned long * __restrict v2,
+		      const unsigned long * __restrict v3,
+		      const unsigned long * __restrict v4)
+{
+	unsigned long lines = bytes / LINE_WIDTH;
+
+	do {
+		__asm__ __volatile__ (
+			LD_INOUT_LINE(v1)
+			LD_AND_XOR_LINE(v2)
+			LD_AND_XOR_LINE(v3)
+			LD_AND_XOR_LINE(v4)
+			ST_LINE(v1)
+		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4)
+		: "memory"
+		);
+
+		v1 += LINE_WIDTH / sizeof(unsigned long);
+		v2 += LINE_WIDTH / sizeof(unsigned long);
+		v3 += LINE_WIDTH / sizeof(unsigned long);
+		v4 += LINE_WIDTH / sizeof(unsigned long);
+	} while (--lines > 0);
+}
+
+void XOR_FUNC_NAME(5)(unsigned long bytes,
+		      unsigned long * __restrict v1,
+		      const unsigned long * __restrict v2,
+		      const unsigned long * __restrict v3,
+		      const unsigned long * __restrict v4,
+		      const unsigned long * __restrict v5)
+{
+	unsigned long lines = bytes / LINE_WIDTH;
+
+	do {
+		__asm__ __volatile__ (
+			LD_INOUT_LINE(v1)
+			LD_AND_XOR_LINE(v2)
+			LD_AND_XOR_LINE(v3)
+			LD_AND_XOR_LINE(v4)
+			LD_AND_XOR_LINE(v5)
+			ST_LINE(v1)
+		: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4),
+		    [v5] "r"(v5) : "memory"
+		);
+
+		v1 += LINE_WIDTH / sizeof(unsigned long);
+		v2 += LINE_WIDTH / sizeof(unsigned long);
+		v3 += LINE_WIDTH / sizeof(unsigned long);
+		v4 += LINE_WIDTH / sizeof(unsigned long);
+		v5 += LINE_WIDTH / sizeof(unsigned long);
+	} while (--lines > 0);
+}
diff --git a/lib/raid/xor/powerpc/xor_arch.h b/lib/raid/xor/powerpc/xor_arch.h
new file mode 100644
index 0000000000000..3b00a4a2fd67c
--- /dev/null
+++ b/lib/raid/xor/powerpc/xor_arch.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Author: Anton Blanchard <anton@au.ibm.com>
+ */
+#include <asm/cpu_has_feature.h>
+
+extern struct xor_block_template xor_block_altivec;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_8regs_p);
+	xor_register(&xor_block_32regs);
+	xor_register(&xor_block_32regs_p);
+#ifdef CONFIG_ALTIVEC
+	if (cpu_has_feature(CPU_FTR_ALTIVEC))
+		xor_register(&xor_block_altivec);
+#endif
+}
diff --git a/lib/raid/xor/powerpc/xor_vmx.c b/lib/raid/xor/powerpc/xor_vmx.c
new file mode 100644
index 0000000000000..09bed98c1bc72
--- /dev/null
+++ b/lib/raid/xor/powerpc/xor_vmx.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Author: Anton Blanchard <anton@au.ibm.com>
+ */
+
+/*
+ * Sparse (as at v0.5.0) gets very, very confused by this file.
+ * Make it a bit simpler for it.
+ */
+#include "xor_impl.h"
+#if !defined(__CHECKER__)
+#include <altivec.h>
+#else
+#define vec_xor(a, b) a ^ b
+#define vector __attribute__((vector_size(16)))
+#endif
+
+#include "xor_vmx.h"
+
+typedef vector signed char unative_t;
+
+#define DEFINE(V)				\
+	unative_t *V = (unative_t *)V##_in;	\
+	unative_t V##_0, V##_1, V##_2, V##_3
+
+#define LOAD(V)			\
+	do {			\
+		V##_0 = V[0];	\
+		V##_1 = V[1];	\
+		V##_2 = V[2];	\
+		V##_3 = V[3];	\
+	} while (0)
+
+#define STORE(V)		\
+	do {			\
+		V[0] = V##_0;	\
+		V[1] = V##_1;	\
+		V[2] = V##_2;	\
+		V[3] = V##_3;	\
+	} while (0)
+
+#define XOR(V1, V2)					\
+	do {						\
+		V1##_0 = vec_xor(V1##_0, V2##_0);	\
+		V1##_1 = vec_xor(V1##_1, V2##_1);	\
+		V1##_2 = vec_xor(V1##_2, V2##_2);	\
+		V1##_3 = vec_xor(V1##_3, V2##_3);	\
+	} while (0)
+
+static void __xor_altivec_2(unsigned long bytes,
+		unsigned long * __restrict v1_in,
+		const unsigned long * __restrict v2_in)
+{
+	DEFINE(v1);
+	DEFINE(v2);
+	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
+
+	do {
+		LOAD(v1);
+		LOAD(v2);
+		XOR(v1, v2);
+		STORE(v1);
+
+		v1 += 4;
+		v2 += 4;
+	} while (--lines > 0);
+}
+
+static void __xor_altivec_3(unsigned long bytes,
+		unsigned long * __restrict v1_in,
+		const unsigned long * __restrict v2_in,
+		const unsigned long * __restrict v3_in)
+{
+	DEFINE(v1);
+	DEFINE(v2);
+	DEFINE(v3);
+	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
+
+	do {
+		LOAD(v1);
+		LOAD(v2);
+		LOAD(v3);
+		XOR(v1, v2);
+		XOR(v1, v3);
+		STORE(v1);
+
+		v1 += 4;
+		v2 += 4;
+		v3 += 4;
+	} while (--lines > 0);
+}
+
+static void __xor_altivec_4(unsigned long bytes,
+		unsigned long * __restrict v1_in,
+		const unsigned long * __restrict v2_in,
+		const unsigned long * __restrict v3_in,
+		const unsigned long * __restrict v4_in)
+{
+	DEFINE(v1);
+	DEFINE(v2);
+	DEFINE(v3);
+	DEFINE(v4);
+	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
+
+	do {
+		LOAD(v1);
+		LOAD(v2);
+		LOAD(v3);
+		LOAD(v4);
+		XOR(v1, v2);
+		XOR(v3, v4);
+		XOR(v1, v3);
+		STORE(v1);
+
+		v1 += 4;
+		v2 += 4;
+		v3 += 4;
+		v4 += 4;
+	} while (--lines > 0);
+}
+
+static void __xor_altivec_5(unsigned long bytes,
+		unsigned long * __restrict v1_in,
+		const unsigned long * __restrict v2_in,
+		const unsigned long * __restrict v3_in,
+		const unsigned long * __restrict v4_in,
+		const unsigned long * __restrict v5_in)
+{
+	DEFINE(v1);
+	DEFINE(v2);
+	DEFINE(v3);
+	DEFINE(v4);
+	DEFINE(v5);
+	unsigned long lines = bytes / (sizeof(unative_t)) / 4;
+
+	do {
+		LOAD(v1);
+		LOAD(v2);
+		LOAD(v3);
+		LOAD(v4);
+		LOAD(v5);
+		XOR(v1, v2);
+		XOR(v3, v4);
+		XOR(v1, v5);
+		XOR(v1, v3);
+		STORE(v1);
+
+		v1 += 4;
+		v2 += 4;
+		v3 += 4;
+		v4 += 4;
+		v5 += 4;
+	} while (--lines > 0);
+}
+
+__DO_XOR_BLOCKS(altivec_inner, __xor_altivec_2, __xor_altivec_3,
+		__xor_altivec_4, __xor_altivec_5);
diff --git a/lib/raid/xor/powerpc/xor_vmx.h b/lib/raid/xor/powerpc/xor_vmx.h
new file mode 100644
index 0000000000000..1d26c1133a868
--- /dev/null
+++ b/lib/raid/xor/powerpc/xor_vmx.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Simple interface to link xor_vmx.c and xor_vmx_glue.c
+ *
+ * Separating these file ensures that no altivec instructions are run
+ * outside of the enable/disable altivec block.
+ */
+
+void xor_gen_altivec_inner(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes);
diff --git a/lib/raid/xor/powerpc/xor_vmx_glue.c b/lib/raid/xor/powerpc/xor_vmx_glue.c
new file mode 100644
index 0000000000000..dbfbb5cadc36a
--- /dev/null
+++ b/lib/raid/xor/powerpc/xor_vmx_glue.c
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Altivec XOR operations
+ *
+ * Copyright 2017 IBM Corp.
+ */
+
+#include <linux/preempt.h>
+#include <linux/sched.h>
+#include <asm/switch_to.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+#include "xor_vmx.h"
+
+static void xor_gen_altivec(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	preempt_disable();
+	enable_kernel_altivec();
+	xor_gen_altivec_inner(dest, srcs, src_cnt, bytes);
+	disable_kernel_altivec();
+	preempt_enable();
+}
+
+struct xor_block_template xor_block_altivec = {
+	.name		= "altivec",
+	.xor_gen	= xor_gen_altivec,
+};
diff --git a/lib/raid/xor/riscv/xor-glue.c b/lib/raid/xor/riscv/xor-glue.c
new file mode 100644
index 0000000000000..2e4c1b05d998f
--- /dev/null
+++ b/lib/raid/xor/riscv/xor-glue.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2021 SiFive
+ */
+
+#include <asm/vector.h>
+#include <asm/switch_to.h>
+#include <asm/asm-prototypes.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+DO_XOR_BLOCKS(vector_inner, xor_regs_2_, xor_regs_3_, xor_regs_4_, xor_regs_5_);
+
+static void xor_gen_vector(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	kernel_vector_begin();
+	xor_gen_vector_inner(dest, srcs, src_cnt, bytes);
+	kernel_vector_end();
+}
+
+struct xor_block_template xor_block_rvv = {
+	.name		= "rvv",
+	.xor_gen	= xor_gen_vector,
+};
diff --git a/lib/raid/xor/riscv/xor.S b/lib/raid/xor/riscv/xor.S
new file mode 100644
index 0000000000000..56fb7fc1e2cd8
--- /dev/null
+++ b/lib/raid/xor/riscv/xor.S
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2021 SiFive
+ */
+#include <linux/linkage.h>
+#include <linux/export.h>
+#include <asm/asm.h>
+
+SYM_FUNC_START(xor_regs_2_)
+	vsetvli a3, a0, e8, m8, ta, ma
+	vle8.v v0, (a1)
+	vle8.v v8, (a2)
+	sub a0, a0, a3
+	vxor.vv v16, v0, v8
+	add a2, a2, a3
+	vse8.v v16, (a1)
+	add a1, a1, a3
+	bnez a0, xor_regs_2_
+	ret
+SYM_FUNC_END(xor_regs_2_)
+
+SYM_FUNC_START(xor_regs_3_)
+	vsetvli a4, a0, e8, m8, ta, ma
+	vle8.v v0, (a1)
+	vle8.v v8, (a2)
+	sub a0, a0, a4
+	vxor.vv v0, v0, v8
+	vle8.v v16, (a3)
+	add a2, a2, a4
+	vxor.vv v16, v0, v16
+	add a3, a3, a4
+	vse8.v v16, (a1)
+	add a1, a1, a4
+	bnez a0, xor_regs_3_
+	ret
+SYM_FUNC_END(xor_regs_3_)
+
+SYM_FUNC_START(xor_regs_4_)
+	vsetvli a5, a0, e8, m8, ta, ma
+	vle8.v v0, (a1)
+	vle8.v v8, (a2)
+	sub a0, a0, a5
+	vxor.vv v0, v0, v8
+	vle8.v v16, (a3)
+	add a2, a2, a5
+	vxor.vv v0, v0, v16
+	vle8.v v24, (a4)
+	add a3, a3, a5
+	vxor.vv v16, v0, v24
+	add a4, a4, a5
+	vse8.v v16, (a1)
+	add a1, a1, a5
+	bnez a0, xor_regs_4_
+	ret
+SYM_FUNC_END(xor_regs_4_)
+
+SYM_FUNC_START(xor_regs_5_)
+	vsetvli a6, a0, e8, m8, ta, ma
+	vle8.v v0, (a1)
+	vle8.v v8, (a2)
+	sub a0, a0, a6
+	vxor.vv v0, v0, v8
+	vle8.v v16, (a3)
+	add a2, a2, a6
+	vxor.vv v0, v0, v16
+	vle8.v v24, (a4)
+	add a3, a3, a6
+	vxor.vv v0, v0, v24
+	vle8.v v8, (a5)
+	add a4, a4, a6
+	vxor.vv v16, v0, v8
+	add a5, a5, a6
+	vse8.v v16, (a1)
+	add a1, a1, a6
+	bnez a0, xor_regs_5_
+	ret
+SYM_FUNC_END(xor_regs_5_)
diff --git a/lib/raid/xor/riscv/xor_arch.h b/lib/raid/xor/riscv/xor_arch.h
new file mode 100644
index 0000000000000..9240857d760b2
--- /dev/null
+++ b/lib/raid/xor/riscv/xor_arch.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2021 SiFive
+ */
+#include <asm/vector.h>
+
+extern struct xor_block_template xor_block_rvv;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_32regs);
+#ifdef CONFIG_RISCV_ISA_V
+	if (has_vector())
+		xor_register(&xor_block_rvv);
+#endif
+}
diff --git a/lib/raid/xor/s390/xor.c b/lib/raid/xor/s390/xor.c
new file mode 100644
index 0000000000000..0c478678a1291
--- /dev/null
+++ b/lib/raid/xor/s390/xor.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Optimized xor_block operation for RAID4/5
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+
+#include <linux/types.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2)
+{
+	asm volatile(
+		"	aghi	%0,-1\n"
+		"	jm	3f\n"
+		"	srlg	0,%0,8\n"
+		"	ltgr	0,0\n"
+		"	jz	1f\n"
+		"0:	xc	0(256,%1),0(%2)\n"
+		"	la	%1,256(%1)\n"
+		"	la	%2,256(%2)\n"
+		"	brctg	0,0b\n"
+		"1:	exrl	%0,2f\n"
+		"	j	3f\n"
+		"2:	xc	0(1,%1),0(%2)\n"
+		"3:"
+		: "+a" (bytes), "+a" (p1), "+a" (p2)
+		: : "0", "cc", "memory");
+}
+
+static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3)
+{
+	asm volatile(
+		"	aghi	%0,-1\n"
+		"	jm	4f\n"
+		"	srlg	0,%0,8\n"
+		"	ltgr	0,0\n"
+		"	jz	1f\n"
+		"0:	xc	0(256,%1),0(%2)\n"
+		"	xc	0(256,%1),0(%3)\n"
+		"	la	%1,256(%1)\n"
+		"	la	%2,256(%2)\n"
+		"	la	%3,256(%3)\n"
+		"	brctg	0,0b\n"
+		"1:	exrl	%0,2f\n"
+		"	exrl	%0,3f\n"
+		"	j	4f\n"
+		"2:	xc	0(1,%1),0(%2)\n"
+		"3:	xc	0(1,%1),0(%3)\n"
+		"4:"
+		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
+		: : "0", "cc", "memory");
+}
+
+static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3,
+		     const unsigned long * __restrict p4)
+{
+	asm volatile(
+		"	aghi	%0,-1\n"
+		"	jm	5f\n"
+		"	srlg	0,%0,8\n"
+		"	ltgr	0,0\n"
+		"	jz	1f\n"
+		"0:	xc	0(256,%1),0(%2)\n"
+		"	xc	0(256,%1),0(%3)\n"
+		"	xc	0(256,%1),0(%4)\n"
+		"	la	%1,256(%1)\n"
+		"	la	%2,256(%2)\n"
+		"	la	%3,256(%3)\n"
+		"	la	%4,256(%4)\n"
+		"	brctg	0,0b\n"
+		"1:	exrl	%0,2f\n"
+		"	exrl	%0,3f\n"
+		"	exrl	%0,4f\n"
+		"	j	5f\n"
+		"2:	xc	0(1,%1),0(%2)\n"
+		"3:	xc	0(1,%1),0(%3)\n"
+		"4:	xc	0(1,%1),0(%4)\n"
+		"5:"
+		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
+		: : "0", "cc", "memory");
+}
+
+static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
+		     const unsigned long * __restrict p2,
+		     const unsigned long * __restrict p3,
+		     const unsigned long * __restrict p4,
+		     const unsigned long * __restrict p5)
+{
+	asm volatile(
+		"	aghi	%0,-1\n"
+		"	jm	6f\n"
+		"	srlg	0,%0,8\n"
+		"	ltgr	0,0\n"
+		"	jz	1f\n"
+		"0:	xc	0(256,%1),0(%2)\n"
+		"	xc	0(256,%1),0(%3)\n"
+		"	xc	0(256,%1),0(%4)\n"
+		"	xc	0(256,%1),0(%5)\n"
+		"	la	%1,256(%1)\n"
+		"	la	%2,256(%2)\n"
+		"	la	%3,256(%3)\n"
+		"	la	%4,256(%4)\n"
+		"	la	%5,256(%5)\n"
+		"	brctg	0,0b\n"
+		"1:	exrl	%0,2f\n"
+		"	exrl	%0,3f\n"
+		"	exrl	%0,4f\n"
+		"	exrl	%0,5f\n"
+		"	j	6f\n"
+		"2:	xc	0(1,%1),0(%2)\n"
+		"3:	xc	0(1,%1),0(%3)\n"
+		"4:	xc	0(1,%1),0(%4)\n"
+		"5:	xc	0(1,%1),0(%5)\n"
+		"6:"
+		: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
+		  "+a" (p5)
+		: : "0", "cc", "memory");
+}
+
+DO_XOR_BLOCKS(xc, xor_xc_2, xor_xc_3, xor_xc_4, xor_xc_5);
+
+struct xor_block_template xor_block_xc = {
+	.name		= "xc",
+	.xor_gen	= xor_gen_xc,
+};
diff --git a/lib/raid/xor/s390/xor_arch.h b/lib/raid/xor/s390/xor_arch.h
new file mode 100644
index 0000000000000..4a233ed2b97a6
--- /dev/null
+++ b/lib/raid/xor/s390/xor_arch.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Optimited xor routines
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+extern struct xor_block_template xor_block_xc;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_force(&xor_block_xc);
+}
diff --git a/lib/raid/xor/sparc/xor-sparc32.c b/lib/raid/xor/sparc/xor-sparc32.c
new file mode 100644
index 0000000000000..fb37631e90e69
--- /dev/null
+++ b/lib/raid/xor/sparc/xor-sparc32.c
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * High speed xor_block operation for RAID4/5 utilizing the
+ * ldd/std SPARC instructions.
+ *
+ * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+static void
+sparc_2(unsigned long bytes, unsigned long * __restrict p1,
+	const unsigned long * __restrict p2)
+{
+	int lines = bytes / (sizeof (long)) / 8;
+
+	do {
+		__asm__ __volatile__(
+		  "ldd [%0 + 0x00], %%g2\n\t"
+		  "ldd [%0 + 0x08], %%g4\n\t"
+		  "ldd [%0 + 0x10], %%o0\n\t"
+		  "ldd [%0 + 0x18], %%o2\n\t"
+		  "ldd [%1 + 0x00], %%o4\n\t"
+		  "ldd [%1 + 0x08], %%l0\n\t"
+		  "ldd [%1 + 0x10], %%l2\n\t"
+		  "ldd [%1 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "std %%g2, [%0 + 0x00]\n\t"
+		  "std %%g4, [%0 + 0x08]\n\t"
+		  "std %%o0, [%0 + 0x10]\n\t"
+		  "std %%o2, [%0 + 0x18]\n"
+		:
+		: "r" (p1), "r" (p2)
+		: "g2", "g3", "g4", "g5",
+		  "o0", "o1", "o2", "o3", "o4", "o5",
+		  "l0", "l1", "l2", "l3", "l4", "l5");
+		p1 += 8;
+		p2 += 8;
+	} while (--lines > 0);
+}
+
+static void
+sparc_3(unsigned long bytes, unsigned long * __restrict p1,
+	const unsigned long * __restrict p2,
+	const unsigned long * __restrict p3)
+{
+	int lines = bytes / (sizeof (long)) / 8;
+
+	do {
+		__asm__ __volatile__(
+		  "ldd [%0 + 0x00], %%g2\n\t"
+		  "ldd [%0 + 0x08], %%g4\n\t"
+		  "ldd [%0 + 0x10], %%o0\n\t"
+		  "ldd [%0 + 0x18], %%o2\n\t"
+		  "ldd [%1 + 0x00], %%o4\n\t"
+		  "ldd [%1 + 0x08], %%l0\n\t"
+		  "ldd [%1 + 0x10], %%l2\n\t"
+		  "ldd [%1 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%2 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%2 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%2 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%2 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "std %%g2, [%0 + 0x00]\n\t"
+		  "std %%g4, [%0 + 0x08]\n\t"
+		  "std %%o0, [%0 + 0x10]\n\t"
+		  "std %%o2, [%0 + 0x18]\n"
+		:
+		: "r" (p1), "r" (p2), "r" (p3)
+		: "g2", "g3", "g4", "g5",
+		  "o0", "o1", "o2", "o3", "o4", "o5",
+		  "l0", "l1", "l2", "l3", "l4", "l5");
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+	} while (--lines > 0);
+}
+
+static void
+sparc_4(unsigned long bytes, unsigned long * __restrict p1,
+	const unsigned long * __restrict p2,
+	const unsigned long * __restrict p3,
+	const unsigned long * __restrict p4)
+{
+	int lines = bytes / (sizeof (long)) / 8;
+
+	do {
+		__asm__ __volatile__(
+		  "ldd [%0 + 0x00], %%g2\n\t"
+		  "ldd [%0 + 0x08], %%g4\n\t"
+		  "ldd [%0 + 0x10], %%o0\n\t"
+		  "ldd [%0 + 0x18], %%o2\n\t"
+		  "ldd [%1 + 0x00], %%o4\n\t"
+		  "ldd [%1 + 0x08], %%l0\n\t"
+		  "ldd [%1 + 0x10], %%l2\n\t"
+		  "ldd [%1 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%2 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%2 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%2 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%2 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%3 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%3 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%3 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%3 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "std %%g2, [%0 + 0x00]\n\t"
+		  "std %%g4, [%0 + 0x08]\n\t"
+		  "std %%o0, [%0 + 0x10]\n\t"
+		  "std %%o2, [%0 + 0x18]\n"
+		:
+		: "r" (p1), "r" (p2), "r" (p3), "r" (p4)
+		: "g2", "g3", "g4", "g5",
+		  "o0", "o1", "o2", "o3", "o4", "o5",
+		  "l0", "l1", "l2", "l3", "l4", "l5");
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+		p4 += 8;
+	} while (--lines > 0);
+}
+
+static void
+sparc_5(unsigned long bytes, unsigned long * __restrict p1,
+	const unsigned long * __restrict p2,
+	const unsigned long * __restrict p3,
+	const unsigned long * __restrict p4,
+	const unsigned long * __restrict p5)
+{
+	int lines = bytes / (sizeof (long)) / 8;
+
+	do {
+		__asm__ __volatile__(
+		  "ldd [%0 + 0x00], %%g2\n\t"
+		  "ldd [%0 + 0x08], %%g4\n\t"
+		  "ldd [%0 + 0x10], %%o0\n\t"
+		  "ldd [%0 + 0x18], %%o2\n\t"
+		  "ldd [%1 + 0x00], %%o4\n\t"
+		  "ldd [%1 + 0x08], %%l0\n\t"
+		  "ldd [%1 + 0x10], %%l2\n\t"
+		  "ldd [%1 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%2 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%2 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%2 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%2 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%3 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%3 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%3 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%3 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "ldd [%4 + 0x00], %%o4\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "ldd [%4 + 0x08], %%l0\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "ldd [%4 + 0x10], %%l2\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "ldd [%4 + 0x18], %%l4\n\t"
+		  "xor %%g2, %%o4, %%g2\n\t"
+		  "xor %%g3, %%o5, %%g3\n\t"
+		  "xor %%g4, %%l0, %%g4\n\t"
+		  "xor %%g5, %%l1, %%g5\n\t"
+		  "xor %%o0, %%l2, %%o0\n\t"
+		  "xor %%o1, %%l3, %%o1\n\t"
+		  "xor %%o2, %%l4, %%o2\n\t"
+		  "xor %%o3, %%l5, %%o3\n\t"
+		  "std %%g2, [%0 + 0x00]\n\t"
+		  "std %%g4, [%0 + 0x08]\n\t"
+		  "std %%o0, [%0 + 0x10]\n\t"
+		  "std %%o2, [%0 + 0x18]\n"
+		:
+		: "r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
+		: "g2", "g3", "g4", "g5",
+		  "o0", "o1", "o2", "o3", "o4", "o5",
+		  "l0", "l1", "l2", "l3", "l4", "l5");
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+		p4 += 8;
+		p5 += 8;
+	} while (--lines > 0);
+}
+
+DO_XOR_BLOCKS(sparc32, sparc_2, sparc_3, sparc_4, sparc_5);
+
+struct xor_block_template xor_block_SPARC = {
+	.name		= "SPARC",
+	.xor_gen	= xor_gen_sparc32,
+};
diff --git a/lib/raid/xor/sparc/xor-sparc64-glue.c b/lib/raid/xor/sparc/xor-sparc64-glue.c
new file mode 100644
index 0000000000000..a8a686e0d2583
--- /dev/null
+++ b/lib/raid/xor/sparc/xor-sparc64-glue.c
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * High speed xor_block operation for RAID4/5 utilizing the
+ * UltraSparc Visual Instruction Set and Niagara block-init
+ * twin-load instructions.
+ *
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
+ */
+
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2);
+void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3);
+void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4);
+void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4,
+	       const unsigned long * __restrict p5);
+
+/* XXX Ugh, write cheetah versions... -DaveM */
+
+DO_XOR_BLOCKS(vis, xor_vis_2, xor_vis_3, xor_vis_4, xor_vis_5);
+
+struct xor_block_template xor_block_VIS = {
+        .name		= "VIS",
+	.xor_gen	= xor_gen_vis,
+};
+
+void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1,
+		   const unsigned long * __restrict p2);
+void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1,
+		   const unsigned long * __restrict p2,
+		   const unsigned long * __restrict p3);
+void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1,
+		   const unsigned long * __restrict p2,
+		   const unsigned long * __restrict p3,
+		   const unsigned long * __restrict p4);
+void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
+		   const unsigned long * __restrict p2,
+		   const unsigned long * __restrict p3,
+		   const unsigned long * __restrict p4,
+		   const unsigned long * __restrict p5);
+
+DO_XOR_BLOCKS(niagara, xor_niagara_2, xor_niagara_3, xor_niagara_4,
+		xor_niagara_5);
+
+struct xor_block_template xor_block_niagara = {
+        .name		= "Niagara",
+	.xor_gen	= xor_gen_niagara,
+};
diff --git a/lib/raid/xor/sparc/xor-sparc64.S b/lib/raid/xor/sparc/xor-sparc64.S
new file mode 100644
index 0000000000000..a7b74d473bd47
--- /dev/null
+++ b/lib/raid/xor/sparc/xor-sparc64.S
@@ -0,0 +1,636 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * High speed xor_block operation for RAID4/5 utilizing the
+ * UltraSparc Visual Instruction Set and Niagara store-init/twin-load.
+ *
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
+ */
+
+#include <linux/export.h>
+#include <linux/linkage.h>
+#include <asm/visasm.h>
+#include <asm/asi.h>
+#include <asm/dcu.h>
+#include <asm/spitfire.h>
+
+/*
+ *	Requirements:
+ *	!(((long)dest | (long)sourceN) & (64 - 1)) &&
+ *	!(len & 127) && len >= 256
+ */
+	.text
+
+	/* VIS versions. */
+ENTRY(xor_vis_2)
+	rd	%fprs, %o5
+	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
+	be,pt	%icc, 0f
+	 sethi	%hi(VISenter), %g1
+	jmpl	%g1 + %lo(VISenter), %g7
+	 add	%g7, 8, %g7
+0:	wr	%g0, FPRS_FEF, %fprs
+	rd	%asi, %g1
+	wr	%g0, ASI_BLK_P, %asi
+	membar	#LoadStore|#StoreLoad|#StoreStore
+	sub	%o0, 128, %o0
+	ldda	[%o1] %asi, %f0
+	ldda	[%o2] %asi, %f16
+
+2:	ldda	[%o1 + 64] %asi, %f32
+	fxor	%f0, %f16, %f16
+	fxor	%f2, %f18, %f18
+	fxor	%f4, %f20, %f20
+	fxor	%f6, %f22, %f22
+	fxor	%f8, %f24, %f24
+	fxor	%f10, %f26, %f26
+	fxor	%f12, %f28, %f28
+	fxor	%f14, %f30, %f30
+	stda	%f16, [%o1] %asi
+	ldda	[%o2 + 64] %asi, %f48
+	ldda	[%o1 + 128] %asi, %f0
+	fxor	%f32, %f48, %f48
+	fxor	%f34, %f50, %f50
+	add	%o1, 128, %o1
+	fxor	%f36, %f52, %f52
+	add	%o2, 128, %o2
+	fxor	%f38, %f54, %f54
+	subcc	%o0, 128, %o0
+	fxor	%f40, %f56, %f56
+	fxor	%f42, %f58, %f58
+	fxor	%f44, %f60, %f60
+	fxor	%f46, %f62, %f62
+	stda	%f48, [%o1 - 64] %asi
+	bne,pt	%xcc, 2b
+	 ldda	[%o2] %asi, %f16
+
+	ldda	[%o1 + 64] %asi, %f32
+	fxor	%f0, %f16, %f16
+	fxor	%f2, %f18, %f18
+	fxor	%f4, %f20, %f20
+	fxor	%f6, %f22, %f22
+	fxor	%f8, %f24, %f24
+	fxor	%f10, %f26, %f26
+	fxor	%f12, %f28, %f28
+	fxor	%f14, %f30, %f30
+	stda	%f16, [%o1] %asi
+	ldda	[%o2 + 64] %asi, %f48
+	membar	#Sync
+	fxor	%f32, %f48, %f48
+	fxor	%f34, %f50, %f50
+	fxor	%f36, %f52, %f52
+	fxor	%f38, %f54, %f54
+	fxor	%f40, %f56, %f56
+	fxor	%f42, %f58, %f58
+	fxor	%f44, %f60, %f60
+	fxor	%f46, %f62, %f62
+	stda	%f48, [%o1 + 64] %asi
+	membar	#Sync|#StoreStore|#StoreLoad
+	wr	%g1, %g0, %asi
+	retl
+	  wr	%g0, 0, %fprs
+ENDPROC(xor_vis_2)
+
+ENTRY(xor_vis_3)
+	rd	%fprs, %o5
+	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
+	be,pt	%icc, 0f
+	 sethi	%hi(VISenter), %g1
+	jmpl	%g1 + %lo(VISenter), %g7
+	 add	%g7, 8, %g7
+0:	wr	%g0, FPRS_FEF, %fprs
+	rd	%asi, %g1
+	wr	%g0, ASI_BLK_P, %asi
+	membar	#LoadStore|#StoreLoad|#StoreStore
+	sub	%o0, 64, %o0
+	ldda	[%o1] %asi, %f0
+	ldda	[%o2] %asi, %f16
+
+3:	ldda	[%o3] %asi, %f32
+	fxor	%f0, %f16, %f48
+	fxor	%f2, %f18, %f50
+	add	%o1, 64, %o1
+	fxor	%f4, %f20, %f52
+	fxor	%f6, %f22, %f54
+	add	%o2, 64, %o2
+	fxor	%f8, %f24, %f56
+	fxor	%f10, %f26, %f58
+	fxor	%f12, %f28, %f60
+	fxor	%f14, %f30, %f62
+	ldda	[%o1] %asi, %f0
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	add	%o3, 64, %o3
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	subcc	%o0, 64, %o0
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	stda	%f48, [%o1 - 64] %asi
+	bne,pt	%xcc, 3b
+	 ldda	[%o2] %asi, %f16
+
+	ldda	[%o3] %asi, %f32
+	fxor	%f0, %f16, %f48
+	fxor	%f2, %f18, %f50
+	fxor	%f4, %f20, %f52
+	fxor	%f6, %f22, %f54
+	fxor	%f8, %f24, %f56
+	fxor	%f10, %f26, %f58
+	fxor	%f12, %f28, %f60
+	fxor	%f14, %f30, %f62
+	membar	#Sync
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	stda	%f48, [%o1] %asi
+	membar	#Sync|#StoreStore|#StoreLoad
+	wr	%g1, %g0, %asi
+	retl
+	 wr	%g0, 0, %fprs
+ENDPROC(xor_vis_3)
+
+ENTRY(xor_vis_4)
+	rd	%fprs, %o5
+	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
+	be,pt	%icc, 0f
+	 sethi	%hi(VISenter), %g1
+	jmpl	%g1 + %lo(VISenter), %g7
+	 add	%g7, 8, %g7
+0:	wr	%g0, FPRS_FEF, %fprs
+	rd	%asi, %g1
+	wr	%g0, ASI_BLK_P, %asi
+	membar	#LoadStore|#StoreLoad|#StoreStore
+	sub	%o0, 64, %o0
+	ldda	[%o1] %asi, %f0
+	ldda	[%o2] %asi, %f16
+
+4:	ldda	[%o3] %asi, %f32
+	fxor	%f0, %f16, %f16
+	fxor	%f2, %f18, %f18
+	add	%o1, 64, %o1
+	fxor	%f4, %f20, %f20
+	fxor	%f6, %f22, %f22
+	add	%o2, 64, %o2
+	fxor	%f8, %f24, %f24
+	fxor	%f10, %f26, %f26
+	fxor	%f12, %f28, %f28
+	fxor	%f14, %f30, %f30
+	ldda	[%o4] %asi, %f48
+	fxor	%f16, %f32, %f32
+	fxor	%f18, %f34, %f34
+	fxor	%f20, %f36, %f36
+	fxor	%f22, %f38, %f38
+	add	%o3, 64, %o3
+	fxor	%f24, %f40, %f40
+	fxor	%f26, %f42, %f42
+	fxor	%f28, %f44, %f44
+	fxor	%f30, %f46, %f46
+	ldda	[%o1] %asi, %f0
+	fxor	%f32, %f48, %f48
+	fxor	%f34, %f50, %f50
+	fxor	%f36, %f52, %f52
+	add	%o4, 64, %o4
+	fxor	%f38, %f54, %f54
+	fxor	%f40, %f56, %f56
+	fxor	%f42, %f58, %f58
+	subcc	%o0, 64, %o0
+	fxor	%f44, %f60, %f60
+	fxor	%f46, %f62, %f62
+	stda	%f48, [%o1 - 64] %asi
+	bne,pt	%xcc, 4b
+	 ldda	[%o2] %asi, %f16
+
+	ldda	[%o3] %asi, %f32
+	fxor	%f0, %f16, %f16
+	fxor	%f2, %f18, %f18
+	fxor	%f4, %f20, %f20
+	fxor	%f6, %f22, %f22
+	fxor	%f8, %f24, %f24
+	fxor	%f10, %f26, %f26
+	fxor	%f12, %f28, %f28
+	fxor	%f14, %f30, %f30
+	ldda	[%o4] %asi, %f48
+	fxor	%f16, %f32, %f32
+	fxor	%f18, %f34, %f34
+	fxor	%f20, %f36, %f36
+	fxor	%f22, %f38, %f38
+	fxor	%f24, %f40, %f40
+	fxor	%f26, %f42, %f42
+	fxor	%f28, %f44, %f44
+	fxor	%f30, %f46, %f46
+	membar	#Sync
+	fxor	%f32, %f48, %f48
+	fxor	%f34, %f50, %f50
+	fxor	%f36, %f52, %f52
+	fxor	%f38, %f54, %f54
+	fxor	%f40, %f56, %f56
+	fxor	%f42, %f58, %f58
+	fxor	%f44, %f60, %f60
+	fxor	%f46, %f62, %f62
+	stda	%f48, [%o1] %asi
+	membar	#Sync|#StoreStore|#StoreLoad
+	wr	%g1, %g0, %asi
+	retl
+	 wr	%g0, 0, %fprs
+ENDPROC(xor_vis_4)
+
+ENTRY(xor_vis_5)
+	save	%sp, -192, %sp
+	rd	%fprs, %o5
+	andcc	%o5, FPRS_FEF|FPRS_DU, %g0
+	be,pt	%icc, 0f
+	 sethi	%hi(VISenter), %g1
+	jmpl	%g1 + %lo(VISenter), %g7
+	 add	%g7, 8, %g7
+0:	wr	%g0, FPRS_FEF, %fprs
+	rd	%asi, %g1
+	wr	%g0, ASI_BLK_P, %asi
+	membar	#LoadStore|#StoreLoad|#StoreStore
+	sub	%i0, 64, %i0
+	ldda	[%i1] %asi, %f0
+	ldda	[%i2] %asi, %f16
+
+5:	ldda	[%i3] %asi, %f32
+	fxor	%f0, %f16, %f48
+	fxor	%f2, %f18, %f50
+	add	%i1, 64, %i1
+	fxor	%f4, %f20, %f52
+	fxor	%f6, %f22, %f54
+	add	%i2, 64, %i2
+	fxor	%f8, %f24, %f56
+	fxor	%f10, %f26, %f58
+	fxor	%f12, %f28, %f60
+	fxor	%f14, %f30, %f62
+	ldda	[%i4] %asi, %f16
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	add	%i3, 64, %i3
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	ldda	[%i5] %asi, %f32
+	fxor	%f48, %f16, %f48
+	fxor	%f50, %f18, %f50
+	add	%i4, 64, %i4
+	fxor	%f52, %f20, %f52
+	fxor	%f54, %f22, %f54
+	add	%i5, 64, %i5
+	fxor	%f56, %f24, %f56
+	fxor	%f58, %f26, %f58
+	fxor	%f60, %f28, %f60
+	fxor	%f62, %f30, %f62
+	ldda	[%i1] %asi, %f0
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	subcc	%i0, 64, %i0
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	stda	%f48, [%i1 - 64] %asi
+	bne,pt	%xcc, 5b
+	 ldda	[%i2] %asi, %f16
+
+	ldda	[%i3] %asi, %f32
+	fxor	%f0, %f16, %f48
+	fxor	%f2, %f18, %f50
+	fxor	%f4, %f20, %f52
+	fxor	%f6, %f22, %f54
+	fxor	%f8, %f24, %f56
+	fxor	%f10, %f26, %f58
+	fxor	%f12, %f28, %f60
+	fxor	%f14, %f30, %f62
+	ldda	[%i4] %asi, %f16
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	ldda	[%i5] %asi, %f32
+	fxor	%f48, %f16, %f48
+	fxor	%f50, %f18, %f50
+	fxor	%f52, %f20, %f52
+	fxor	%f54, %f22, %f54
+	fxor	%f56, %f24, %f56
+	fxor	%f58, %f26, %f58
+	fxor	%f60, %f28, %f60
+	fxor	%f62, %f30, %f62
+	membar	#Sync
+	fxor	%f48, %f32, %f48
+	fxor	%f50, %f34, %f50
+	fxor	%f52, %f36, %f52
+	fxor	%f54, %f38, %f54
+	fxor	%f56, %f40, %f56
+	fxor	%f58, %f42, %f58
+	fxor	%f60, %f44, %f60
+	fxor	%f62, %f46, %f62
+	stda	%f48, [%i1] %asi
+	membar	#Sync|#StoreStore|#StoreLoad
+	wr	%g1, %g0, %asi
+	wr	%g0, 0, %fprs
+	ret
+	 restore
+ENDPROC(xor_vis_5)
+
+	/* Niagara versions. */
+ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
+	save		%sp, -192, %sp
+	prefetch	[%i1], #n_writes
+	prefetch	[%i2], #one_read
+	rd		%asi, %g7
+	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
+	srlx		%i0, 6, %g1
+	mov		%i1, %i0
+	mov		%i2, %i1
+1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src  + 0x00 */
+	ldda		[%i1 + 0x10] %asi, %i4	/* %i4/%i5 = src  + 0x10 */
+	ldda		[%i1 + 0x20] %asi, %g2	/* %g2/%g3 = src  + 0x20 */
+	ldda		[%i1 + 0x30] %asi, %l0	/* %l0/%l1 = src  + 0x30 */
+	prefetch	[%i1 + 0x40], #one_read
+	ldda		[%i0 + 0x00] %asi, %o0  /* %o0/%o1 = dest + 0x00 */
+	ldda		[%i0 + 0x10] %asi, %o2  /* %o2/%o3 = dest + 0x10 */
+	ldda		[%i0 + 0x20] %asi, %o4  /* %o4/%o5 = dest + 0x20 */
+	ldda		[%i0 + 0x30] %asi, %l2  /* %l2/%l3 = dest + 0x30 */
+	prefetch	[%i0 + 0x40], #n_writes
+	xor		%o0, %i2, %o0
+	xor		%o1, %i3, %o1
+	stxa		%o0, [%i0 + 0x00] %asi
+	stxa		%o1, [%i0 + 0x08] %asi
+	xor		%o2, %i4, %o2
+	xor		%o3, %i5, %o3
+	stxa		%o2, [%i0 + 0x10] %asi
+	stxa		%o3, [%i0 + 0x18] %asi
+	xor		%o4, %g2, %o4
+	xor		%o5, %g3, %o5
+	stxa		%o4, [%i0 + 0x20] %asi
+	stxa		%o5, [%i0 + 0x28] %asi
+	xor		%l2, %l0, %l2
+	xor		%l3, %l1, %l3
+	stxa		%l2, [%i0 + 0x30] %asi
+	stxa		%l3, [%i0 + 0x38] %asi
+	add		%i0, 0x40, %i0
+	subcc		%g1, 1, %g1
+	bne,pt		%xcc, 1b
+	 add		%i1, 0x40, %i1
+	membar		#Sync
+	wr		%g7, 0x0, %asi
+	ret
+	 restore
+ENDPROC(xor_niagara_2)
+
+ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
+	save		%sp, -192, %sp
+	prefetch	[%i1], #n_writes
+	prefetch	[%i2], #one_read
+	prefetch	[%i3], #one_read
+	rd		%asi, %g7
+	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
+	srlx		%i0, 6, %g1
+	mov		%i1, %i0
+	mov		%i2, %i1
+	mov		%i3, %l7
+1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
+	ldda		[%i1 + 0x10] %asi, %i4	/* %i4/%i5 = src1 + 0x10 */
+	ldda		[%l7 + 0x00] %asi, %g2	/* %g2/%g3 = src2 + 0x00 */
+	ldda		[%l7 + 0x10] %asi, %l0	/* %l0/%l1 = src2 + 0x10 */
+	ldda		[%i0 + 0x00] %asi, %o0  /* %o0/%o1 = dest + 0x00 */
+	ldda		[%i0 + 0x10] %asi, %o2  /* %o2/%o3 = dest + 0x10 */
+	xor		%g2, %i2, %g2
+	xor		%g3, %i3, %g3
+	xor		%o0, %g2, %o0
+	xor		%o1, %g3, %o1
+	stxa		%o0, [%i0 + 0x00] %asi
+	stxa		%o1, [%i0 + 0x08] %asi
+	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
+	ldda		[%l7 + 0x20] %asi, %g2	/* %g2/%g3 = src2 + 0x20 */
+	ldda		[%i0 + 0x20] %asi, %o0	/* %o0/%o1 = dest + 0x20 */
+	xor		%l0, %i4, %l0
+	xor		%l1, %i5, %l1
+	xor		%o2, %l0, %o2
+	xor		%o3, %l1, %o3
+	stxa		%o2, [%i0 + 0x10] %asi
+	stxa		%o3, [%i0 + 0x18] %asi
+	ldda		[%i1 + 0x30] %asi, %i4	/* %i4/%i5 = src1 + 0x30 */
+	ldda		[%l7 + 0x30] %asi, %l0	/* %l0/%l1 = src2 + 0x30 */
+	ldda		[%i0 + 0x30] %asi, %o2	/* %o2/%o3 = dest + 0x30 */
+	prefetch	[%i1 + 0x40], #one_read
+	prefetch	[%l7 + 0x40], #one_read
+	prefetch	[%i0 + 0x40], #n_writes
+	xor		%g2, %i2, %g2
+	xor		%g3, %i3, %g3
+	xor		%o0, %g2, %o0
+	xor		%o1, %g3, %o1
+	stxa		%o0, [%i0 + 0x20] %asi
+	stxa		%o1, [%i0 + 0x28] %asi
+	xor		%l0, %i4, %l0
+	xor		%l1, %i5, %l1
+	xor		%o2, %l0, %o2
+	xor		%o3, %l1, %o3
+	stxa		%o2, [%i0 + 0x30] %asi
+	stxa		%o3, [%i0 + 0x38] %asi
+	add		%i0, 0x40, %i0
+	add		%i1, 0x40, %i1
+	subcc		%g1, 1, %g1
+	bne,pt		%xcc, 1b
+	 add		%l7, 0x40, %l7
+	membar		#Sync
+	wr		%g7, 0x0, %asi
+	ret
+	 restore
+ENDPROC(xor_niagara_3)
+
+ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
+	save		%sp, -192, %sp
+	prefetch	[%i1], #n_writes
+	prefetch	[%i2], #one_read
+	prefetch	[%i3], #one_read
+	prefetch	[%i4], #one_read
+	rd		%asi, %g7
+	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
+	srlx		%i0, 6, %g1
+	mov		%i1, %i0
+	mov		%i2, %i1
+	mov		%i3, %l7
+	mov		%i4, %l6
+1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
+	ldda		[%l7 + 0x00] %asi, %i4	/* %i4/%i5 = src2 + 0x00 */
+	ldda		[%l6 + 0x00] %asi, %g2	/* %g2/%g3 = src3 + 0x00 */
+	ldda		[%i0 + 0x00] %asi, %l0	/* %l0/%l1 = dest + 0x00 */
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x10] %asi, %i2	/* %i2/%i3 = src1 + 0x10 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x10] %asi, %i4	/* %i4/%i5 = src2 + 0x10 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	stxa		%l0, [%i0 + 0x00] %asi
+	stxa		%l1, [%i0 + 0x08] %asi
+	ldda		[%l6 + 0x10] %asi, %g2	/* %g2/%g3 = src3 + 0x10 */
+	ldda		[%i0 + 0x10] %asi, %l0	/* %l0/%l1 = dest + 0x10 */
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x20] %asi, %i4	/* %i4/%i5 = src2 + 0x20 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	stxa		%l0, [%i0 + 0x10] %asi
+	stxa		%l1, [%i0 + 0x18] %asi
+	ldda		[%l6 + 0x20] %asi, %g2	/* %g2/%g3 = src3 + 0x20 */
+	ldda		[%i0 + 0x20] %asi, %l0	/* %l0/%l1 = dest + 0x20 */
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x30] %asi, %i2	/* %i2/%i3 = src1 + 0x30 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x30] %asi, %i4	/* %i4/%i5 = src2 + 0x30 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	stxa		%l0, [%i0 + 0x20] %asi
+	stxa		%l1, [%i0 + 0x28] %asi
+	ldda		[%l6 + 0x30] %asi, %g2	/* %g2/%g3 = src3 + 0x30 */
+	ldda		[%i0 + 0x30] %asi, %l0	/* %l0/%l1 = dest + 0x30 */
+
+	prefetch	[%i1 + 0x40], #one_read
+	prefetch	[%l7 + 0x40], #one_read
+	prefetch	[%l6 + 0x40], #one_read
+	prefetch	[%i0 + 0x40], #n_writes
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	stxa		%l0, [%i0 + 0x30] %asi
+	stxa		%l1, [%i0 + 0x38] %asi
+
+	add		%i0, 0x40, %i0
+	add		%i1, 0x40, %i1
+	add		%l7, 0x40, %l7
+	subcc		%g1, 1, %g1
+	bne,pt		%xcc, 1b
+	 add		%l6, 0x40, %l6
+	membar		#Sync
+	wr		%g7, 0x0, %asi
+	ret
+	 restore
+ENDPROC(xor_niagara_4)
+
+ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */
+	save		%sp, -192, %sp
+	prefetch	[%i1], #n_writes
+	prefetch	[%i2], #one_read
+	prefetch	[%i3], #one_read
+	prefetch	[%i4], #one_read
+	prefetch	[%i5], #one_read
+	rd		%asi, %g7
+	wr		%g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
+	srlx		%i0, 6, %g1
+	mov		%i1, %i0
+	mov		%i2, %i1
+	mov		%i3, %l7
+	mov		%i4, %l6
+	mov		%i5, %l5
+1:	ldda		[%i1 + 0x00] %asi, %i2	/* %i2/%i3 = src1 + 0x00 */
+	ldda		[%l7 + 0x00] %asi, %i4	/* %i4/%i5 = src2 + 0x00 */
+	ldda		[%l6 + 0x00] %asi, %g2	/* %g2/%g3 = src3 + 0x00 */
+	ldda		[%l5 + 0x00] %asi, %l0	/* %l0/%l1 = src4 + 0x00 */
+	ldda		[%i0 + 0x00] %asi, %l2	/* %l2/%l3 = dest + 0x00 */
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x10] %asi, %i2	/* %i2/%i3 = src1 + 0x10 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x10] %asi, %i4	/* %i4/%i5 = src2 + 0x10 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	ldda		[%l6 + 0x10] %asi, %g2	/* %g2/%g3 = src3 + 0x10 */
+	xor		%l2, %l0, %l2
+	xor		%l3, %l1, %l3
+	stxa		%l2, [%i0 + 0x00] %asi
+	stxa		%l3, [%i0 + 0x08] %asi
+	ldda		[%l5 + 0x10] %asi, %l0	/* %l0/%l1 = src4 + 0x10 */
+	ldda		[%i0 + 0x10] %asi, %l2	/* %l2/%l3 = dest + 0x10 */
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x20] %asi, %i2	/* %i2/%i3 = src1 + 0x20 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x20] %asi, %i4	/* %i4/%i5 = src2 + 0x20 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	ldda		[%l6 + 0x20] %asi, %g2	/* %g2/%g3 = src3 + 0x20 */
+	xor		%l2, %l0, %l2
+	xor		%l3, %l1, %l3
+	stxa		%l2, [%i0 + 0x10] %asi
+	stxa		%l3, [%i0 + 0x18] %asi
+	ldda		[%l5 + 0x20] %asi, %l0	/* %l0/%l1 = src4 + 0x20 */
+	ldda		[%i0 + 0x20] %asi, %l2	/* %l2/%l3 = dest + 0x20 */
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	ldda		[%i1 + 0x30] %asi, %i2	/* %i2/%i3 = src1 + 0x30 */
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	ldda		[%l7 + 0x30] %asi, %i4	/* %i4/%i5 = src2 + 0x30 */
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	ldda		[%l6 + 0x30] %asi, %g2	/* %g2/%g3 = src3 + 0x30 */
+	xor		%l2, %l0, %l2
+	xor		%l3, %l1, %l3
+	stxa		%l2, [%i0 + 0x20] %asi
+	stxa		%l3, [%i0 + 0x28] %asi
+	ldda		[%l5 + 0x30] %asi, %l0	/* %l0/%l1 = src4 + 0x30 */
+	ldda		[%i0 + 0x30] %asi, %l2	/* %l2/%l3 = dest + 0x30 */
+
+	prefetch	[%i1 + 0x40], #one_read
+	prefetch	[%l7 + 0x40], #one_read
+	prefetch	[%l6 + 0x40], #one_read
+	prefetch	[%l5 + 0x40], #one_read
+	prefetch	[%i0 + 0x40], #n_writes
+
+	xor		%i4, %i2, %i4
+	xor		%i5, %i3, %i5
+	xor		%g2, %i4, %g2
+	xor		%g3, %i5, %g3
+	xor		%l0, %g2, %l0
+	xor		%l1, %g3, %l1
+	xor		%l2, %l0, %l2
+	xor		%l3, %l1, %l3
+	stxa		%l2, [%i0 + 0x30] %asi
+	stxa		%l3, [%i0 + 0x38] %asi
+
+	add		%i0, 0x40, %i0
+	add		%i1, 0x40, %i1
+	add		%l7, 0x40, %l7
+	add		%l6, 0x40, %l6
+	subcc		%g1, 1, %g1
+	bne,pt		%xcc, 1b
+	 add		%l5, 0x40, %l5
+	membar		#Sync
+	wr		%g7, 0x0, %asi
+	ret
+	 restore
+ENDPROC(xor_niagara_5)
diff --git a/lib/raid/xor/sparc/xor_arch.h b/lib/raid/xor/sparc/xor_arch.h
new file mode 100644
index 0000000000000..af288abe4e917
--- /dev/null
+++ b/lib/raid/xor/sparc/xor_arch.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
+ */
+#if defined(__sparc__) && defined(__arch64__)
+#include <asm/spitfire.h>
+
+extern struct xor_block_template xor_block_VIS;
+extern struct xor_block_template xor_block_niagara;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	/* Force VIS for everything except Niagara.  */
+	if (tlb_type == hypervisor &&
+	    (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
+	     sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
+	     sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
+	     sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
+	     sun4v_chip_type == SUN4V_CHIP_NIAGARA5))
+		xor_force(&xor_block_niagara);
+	else
+		xor_force(&xor_block_VIS);
+}
+#else /* sparc64 */
+
+extern struct xor_block_template xor_block_SPARC;
+
+static __always_inline void __init arch_xor_init(void)
+{
+	xor_register(&xor_block_8regs);
+	xor_register(&xor_block_32regs);
+	xor_register(&xor_block_SPARC);
+}
+#endif /* !sparc64 */
diff --git a/lib/raid/xor/tests/Makefile b/lib/raid/xor/tests/Makefile
new file mode 100644
index 0000000000000..661e8f6ffd1f3
--- /dev/null
+++ b/lib/raid/xor/tests/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o
diff --git a/lib/raid/xor/tests/xor_kunit.c b/lib/raid/xor/tests/xor_kunit.c
new file mode 100644
index 0000000000000..23ee415e914ca
--- /dev/null
+++ b/lib/raid/xor/tests/xor_kunit.c
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Unit test the XOR library functions.
+ *
+ * Copyright 2024 Google LLC
+ * Copyright 2026 Christoph Hellwig
+ *
+ * Based on the CRC tests by Eric Biggers <ebiggers@google.com>.
+ */
+#include <kunit/test.h>
+#include <linux/prandom.h>
+#include <linux/string_choices.h>
+#include <linux/vmalloc.h>
+#include <linux/raid/xor.h>
+
+#define XOR_KUNIT_SEED			42
+#define XOR_KUNIT_MAX_BYTES		16384
+#define XOR_KUNIT_MAX_BUFFERS		64
+#define XOR_KUNIT_NUM_TEST_ITERS	1000
+
+static struct rnd_state rng;
+static void *test_buffers[XOR_KUNIT_MAX_BUFFERS];
+static void *test_dest;
+static void *test_ref;
+static size_t test_buflen;
+
+static u32 rand32(void)
+{
+	return prandom_u32_state(&rng);
+}
+
+static u32 rand32_below(u32 ceil)
+{
+	return __limit_random_u32_below(ceil, prandom_u32_state(&rng));
+}
+
+/* Reference implementation using dumb byte-wise XOR */
+static void xor_ref(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	unsigned int off, idx;
+	u8 *d = dest;
+
+	for (off = 0; off < bytes; off++) {
+		for (idx = 0; idx < src_cnt; idx++) {
+			u8 *src = srcs[idx];
+
+			d[off] ^= src[off];
+		}
+	}
+}
+
+/* Generate a random length that is a multiple of 512. */
+static unsigned int generate_random_length(unsigned int max_length)
+{
+	return (rand32_below(max_length / 512) + 1) * 512;
+}
+
+/* Generate a random alignment that is a multiple of 32. */
+static unsigned int generate_random_alignment(unsigned int max_alignment)
+{
+	return (rand32_below((max_alignment + 1) / 32)) * 32;
+}
+
+static void xor_generate_random_data(void)
+{
+	int i;
+
+	prandom_bytes_state(&rng, test_dest, test_buflen);
+	memcpy(test_ref, test_dest, test_buflen);
+	for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
+		prandom_bytes_state(&rng, test_buffers[i], test_buflen);
+}
+
+/* Test that xor_gen gives the same result as a reference implementation. */
+static void xor_test(struct kunit *test)
+{
+	void *aligned_buffers[XOR_KUNIT_MAX_BUFFERS];
+	size_t i;
+
+	for (i = 0; i < XOR_KUNIT_NUM_TEST_ITERS; i++) {
+		unsigned int nr_buffers =
+			rand32_below(XOR_KUNIT_MAX_BUFFERS) + 1;
+		unsigned int len = generate_random_length(XOR_KUNIT_MAX_BYTES);
+		unsigned int max_alignment, align = 0;
+		void *buffers;
+
+		if (rand32() % 8 == 0)
+			/* Refresh the data occasionally. */
+			xor_generate_random_data();
+
+		/*
+		 * If we're not using the entire buffer size, inject randomize
+		 * alignment into the buffer.
+		 */
+		max_alignment = XOR_KUNIT_MAX_BYTES - len;
+		if (max_alignment) {
+			int j;
+
+			align = generate_random_alignment(max_alignment);
+			for (j = 0; j < nr_buffers; j++)
+				aligned_buffers[j] = test_buffers[j] +
+					generate_random_alignment(max_alignment);
+			buffers = aligned_buffers;
+		} else {
+			buffers = test_buffers;
+		}
+
+		/*
+		 * Compute the XOR, and verify that it equals the XOR computed
+		 * by a simple byte-at-a-time reference implementation.
+		 */
+		xor_ref(test_ref + align, buffers, nr_buffers, len);
+		xor_gen(test_dest + align, buffers, nr_buffers, len);
+		KUNIT_EXPECT_MEMEQ_MSG(test, test_ref, test_dest, len,
+				"Wrong result with buffers=%u, len=%u, align=%s",
+				nr_buffers, len, str_yes_no(max_alignment));
+	}
+}
+
+static struct kunit_case xor_test_cases[] = {
+	KUNIT_CASE(xor_test),
+	{},
+};
+
+static int xor_suite_init(struct kunit_suite *suite)
+{
+	int i;
+
+	/*
+	 * Allocate the test buffer using vmalloc() with a page-aligned length
+	 * so that it is immediately followed by a guard page.  This allows
+	 * buffer overreads to be detected, even in assembly code.
+	 */
+	test_buflen = round_up(XOR_KUNIT_MAX_BYTES, PAGE_SIZE);
+	test_ref = vmalloc(test_buflen);
+	if (!test_ref)
+		return -ENOMEM;
+	test_dest = vmalloc(test_buflen);
+	if (!test_dest)
+		goto out_free_ref;
+	for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) {
+		test_buffers[i] = vmalloc(test_buflen);
+		if (!test_buffers[i])
+			goto out_free_buffers;
+	}
+
+	prandom_seed_state(&rng, XOR_KUNIT_SEED);
+	xor_generate_random_data();
+	return 0;
+
+out_free_buffers:
+	while (--i >= 0)
+		vfree(test_buffers[i]);
+	vfree(test_dest);
+out_free_ref:
+	vfree(test_ref);
+	return -ENOMEM;
+}
+
+static void xor_suite_exit(struct kunit_suite *suite)
+{
+	int i;
+
+	vfree(test_ref);
+	vfree(test_dest);
+	for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
+		vfree(test_buffers[i]);
+}
+
+static struct kunit_suite xor_test_suite = {
+	.name		= "xor",
+	.test_cases	= xor_test_cases,
+	.suite_init	= xor_suite_init,
+	.suite_exit	= xor_suite_exit,
+};
+kunit_test_suite(xor_test_suite);
+
+MODULE_DESCRIPTION("Unit tests and benchmarks for the XOR library functions");
+MODULE_LICENSE("GPL");
diff --git a/lib/raid/xor/um/xor_arch.h b/lib/raid/xor/um/xor_arch.h
new file mode 100644
index 0000000000000..a33e57a26c5ed
--- /dev/null
+++ b/lib/raid/xor/um/xor_arch.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <../x86/xor_arch.h>
diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c
new file mode 100644
index 0000000000000..f7777d7aa269b
--- /dev/null
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Optimized XOR parity functions for AVX
+ *
+ * Copyright (C) 2012 Intel Corporation
+ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
+ *
+ * Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
+ */
+#include <linux/compiler.h>
+#include <asm/fpu/api.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+#define BLOCK4(i) \
+		BLOCK(32 * i, 0) \
+		BLOCK(32 * (i + 1), 1) \
+		BLOCK(32 * (i + 2), 2) \
+		BLOCK(32 * (i + 3), 3)
+
+#define BLOCK16() \
+		BLOCK4(0) \
+		BLOCK4(4) \
+		BLOCK4(8) \
+		BLOCK4(12)
+
+static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0,
+		      const unsigned long * __restrict p1)
+{
+	unsigned long lines = bytes >> 9;
+
+	while (lines--) {
+#undef BLOCK
+#define BLOCK(i, reg) \
+do { \
+	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p1[i / sizeof(*p1)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm"  #reg : : \
+		"m" (p0[i / sizeof(*p0)])); \
+	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
+		"=m" (p0[i / sizeof(*p0)])); \
+} while (0);
+
+		BLOCK16()
+
+		p0 = (unsigned long *)((uintptr_t)p0 + 512);
+		p1 = (unsigned long *)((uintptr_t)p1 + 512);
+	}
+}
+
+static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
+		      const unsigned long * __restrict p1,
+		      const unsigned long * __restrict p2)
+{
+	unsigned long lines = bytes >> 9;
+
+	while (lines--) {
+#undef BLOCK
+#define BLOCK(i, reg) \
+do { \
+	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p2[i / sizeof(*p2)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p1[i / sizeof(*p1)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p0[i / sizeof(*p0)])); \
+	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
+		"=m" (p0[i / sizeof(*p0)])); \
+} while (0);
+
+		BLOCK16()
+
+		p0 = (unsigned long *)((uintptr_t)p0 + 512);
+		p1 = (unsigned long *)((uintptr_t)p1 + 512);
+		p2 = (unsigned long *)((uintptr_t)p2 + 512);
+	}
+}
+
+static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
+		      const unsigned long * __restrict p1,
+		      const unsigned long * __restrict p2,
+		      const unsigned long * __restrict p3)
+{
+	unsigned long lines = bytes >> 9;
+
+	while (lines--) {
+#undef BLOCK
+#define BLOCK(i, reg) \
+do { \
+	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p3[i / sizeof(*p3)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p2[i / sizeof(*p2)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p1[i / sizeof(*p1)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p0[i / sizeof(*p0)])); \
+	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
+		"=m" (p0[i / sizeof(*p0)])); \
+} while (0);
+
+		BLOCK16();
+
+		p0 = (unsigned long *)((uintptr_t)p0 + 512);
+		p1 = (unsigned long *)((uintptr_t)p1 + 512);
+		p2 = (unsigned long *)((uintptr_t)p2 + 512);
+		p3 = (unsigned long *)((uintptr_t)p3 + 512);
+	}
+}
+
+static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
+	     const unsigned long * __restrict p1,
+	     const unsigned long * __restrict p2,
+	     const unsigned long * __restrict p3,
+	     const unsigned long * __restrict p4)
+{
+	unsigned long lines = bytes >> 9;
+
+	while (lines--) {
+#undef BLOCK
+#define BLOCK(i, reg) \
+do { \
+	asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p4[i / sizeof(*p4)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p3[i / sizeof(*p3)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p2[i / sizeof(*p2)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p1[i / sizeof(*p1)])); \
+	asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
+		"m" (p0[i / sizeof(*p0)])); \
+	asm volatile("vmovdqa %%ymm" #reg ", %0" : \
+		"=m" (p0[i / sizeof(*p0)])); \
+} while (0);
+
+		BLOCK16()
+
+		p0 = (unsigned long *)((uintptr_t)p0 + 512);
+		p1 = (unsigned long *)((uintptr_t)p1 + 512);
+		p2 = (unsigned long *)((uintptr_t)p2 + 512);
+		p3 = (unsigned long *)((uintptr_t)p3 + 512);
+		p4 = (unsigned long *)((uintptr_t)p4 + 512);
+	}
+}
+
+DO_XOR_BLOCKS(avx_inner, xor_avx_2, xor_avx_3, xor_avx_4, xor_avx_5);
+
+static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt,
+			unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_avx_inner(dest, srcs, src_cnt, bytes);
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_avx = {
+	.name		= "avx",
+	.xor_gen	= xor_gen_avx,
+};
diff --git a/lib/raid/xor/x86/xor-mmx.c b/lib/raid/xor/x86/xor-mmx.c
new file mode 100644
index 0000000000000..63a8b0444fcef
--- /dev/null
+++ b/lib/raid/xor/x86/xor-mmx.c
@@ -0,0 +1,515 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Optimized XOR parity functions for MMX.
+ *
+ * Copyright (C) 1998 Ingo Molnar.
+ */
+#include <asm/fpu/api.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+#define LD(x, y)	"       movq   8*("#x")(%1), %%mm"#y"   ;\n"
+#define ST(x, y)	"       movq %%mm"#y",   8*("#x")(%1)   ;\n"
+#define XO1(x, y)	"       pxor   8*("#x")(%2), %%mm"#y"   ;\n"
+#define XO2(x, y)	"       pxor   8*("#x")(%3), %%mm"#y"   ;\n"
+#define XO3(x, y)	"       pxor   8*("#x")(%4), %%mm"#y"   ;\n"
+#define XO4(x, y)	"       pxor   8*("#x")(%5), %%mm"#y"   ;\n"
+
+static void
+xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
+	      const unsigned long * __restrict p2)
+{
+	unsigned long lines = bytes >> 7;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)				\
+	LD(i, 0)				\
+		LD(i + 1, 1)			\
+			LD(i + 2, 2)		\
+				LD(i + 3, 3)	\
+	XO1(i, 0)				\
+	ST(i, 0)				\
+		XO1(i+1, 1)			\
+		ST(i+1, 1)			\
+			XO1(i + 2, 2)		\
+			ST(i + 2, 2)		\
+				XO1(i + 3, 3)	\
+				ST(i + 3, 3)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+	BLOCK(0)
+	BLOCK(4)
+	BLOCK(8)
+	BLOCK(12)
+
+	"       addl $128, %1         ;\n"
+	"       addl $128, %2         ;\n"
+	"       decl %0               ;\n"
+	"       jnz 1b                ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2)
+	:
+	: "memory");
+}
+
+static void
+xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
+	      const unsigned long * __restrict p2,
+	      const unsigned long * __restrict p3)
+{
+	unsigned long lines = bytes >> 7;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)				\
+	LD(i, 0)				\
+		LD(i + 1, 1)			\
+			LD(i + 2, 2)		\
+				LD(i + 3, 3)	\
+	XO1(i, 0)				\
+		XO1(i + 1, 1)			\
+			XO1(i + 2, 2)		\
+				XO1(i + 3, 3)	\
+	XO2(i, 0)				\
+	ST(i, 0)				\
+		XO2(i + 1, 1)			\
+		ST(i + 1, 1)			\
+			XO2(i + 2, 2)		\
+			ST(i + 2, 2)		\
+				XO2(i + 3, 3)	\
+				ST(i + 3, 3)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+	BLOCK(0)
+	BLOCK(4)
+	BLOCK(8)
+	BLOCK(12)
+
+	"       addl $128, %1         ;\n"
+	"       addl $128, %2         ;\n"
+	"       addl $128, %3         ;\n"
+	"       decl %0               ;\n"
+	"       jnz 1b                ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3)
+	:
+	: "memory");
+}
+
+static void
+xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
+	      const unsigned long * __restrict p2,
+	      const unsigned long * __restrict p3,
+	      const unsigned long * __restrict p4)
+{
+	unsigned long lines = bytes >> 7;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)				\
+	LD(i, 0)				\
+		LD(i + 1, 1)			\
+			LD(i + 2, 2)		\
+				LD(i + 3, 3)	\
+	XO1(i, 0)				\
+		XO1(i + 1, 1)			\
+			XO1(i + 2, 2)		\
+				XO1(i + 3, 3)	\
+	XO2(i, 0)				\
+		XO2(i + 1, 1)			\
+			XO2(i + 2, 2)		\
+				XO2(i + 3, 3)	\
+	XO3(i, 0)				\
+	ST(i, 0)				\
+		XO3(i + 1, 1)			\
+		ST(i + 1, 1)			\
+			XO3(i + 2, 2)		\
+			ST(i + 2, 2)		\
+				XO3(i + 3, 3)	\
+				ST(i + 3, 3)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+	BLOCK(0)
+	BLOCK(4)
+	BLOCK(8)
+	BLOCK(12)
+
+	"       addl $128, %1         ;\n"
+	"       addl $128, %2         ;\n"
+	"       addl $128, %3         ;\n"
+	"       addl $128, %4         ;\n"
+	"       decl %0               ;\n"
+	"       jnz 1b                ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
+	:
+	: "memory");
+}
+
+
+static void
+xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
+	      const unsigned long * __restrict p2,
+	      const unsigned long * __restrict p3,
+	      const unsigned long * __restrict p4,
+	      const unsigned long * __restrict p5)
+{
+	unsigned long lines = bytes >> 7;
+
+	/* Make sure GCC forgets anything it knows about p4 or p5,
+	   such that it won't pass to the asm volatile below a
+	   register that is shared with any other variable.  That's
+	   because we modify p4 and p5 there, but we can't mark them
+	   as read/write, otherwise we'd overflow the 10-asm-operands
+	   limit of GCC < 3.1.  */
+	asm("" : "+r" (p4), "+r" (p5));
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)				\
+	LD(i, 0)				\
+		LD(i + 1, 1)			\
+			LD(i + 2, 2)		\
+				LD(i + 3, 3)	\
+	XO1(i, 0)				\
+		XO1(i + 1, 1)			\
+			XO1(i + 2, 2)		\
+				XO1(i + 3, 3)	\
+	XO2(i, 0)				\
+		XO2(i + 1, 1)			\
+			XO2(i + 2, 2)		\
+				XO2(i + 3, 3)	\
+	XO3(i, 0)				\
+		XO3(i + 1, 1)			\
+			XO3(i + 2, 2)		\
+				XO3(i + 3, 3)	\
+	XO4(i, 0)				\
+	ST(i, 0)				\
+		XO4(i + 1, 1)			\
+		ST(i + 1, 1)			\
+			XO4(i + 2, 2)		\
+			ST(i + 2, 2)		\
+				XO4(i + 3, 3)	\
+				ST(i + 3, 3)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+	BLOCK(0)
+	BLOCK(4)
+	BLOCK(8)
+	BLOCK(12)
+
+	"       addl $128, %1         ;\n"
+	"       addl $128, %2         ;\n"
+	"       addl $128, %3         ;\n"
+	"       addl $128, %4         ;\n"
+	"       addl $128, %5         ;\n"
+	"       decl %0               ;\n"
+	"       jnz 1b                ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3)
+	: "r" (p4), "r" (p5)
+	: "memory");
+
+	/* p4 and p5 were modified, and now the variables are dead.
+	   Clobber them just to be sure nobody does something stupid
+	   like assuming they have some legal value.  */
+	asm("" : "=r" (p4), "=r" (p5));
+}
+
+#undef LD
+#undef XO1
+#undef XO2
+#undef XO3
+#undef XO4
+#undef ST
+#undef BLOCK
+
+static void
+xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
+	     const unsigned long * __restrict p2)
+{
+	unsigned long lines = bytes >> 6;
+
+	asm volatile(
+	" .align 32	             ;\n"
+	" 1:                         ;\n"
+	"       movq   (%1), %%mm0   ;\n"
+	"       movq  8(%1), %%mm1   ;\n"
+	"       pxor   (%2), %%mm0   ;\n"
+	"       movq 16(%1), %%mm2   ;\n"
+	"       movq %%mm0,   (%1)   ;\n"
+	"       pxor  8(%2), %%mm1   ;\n"
+	"       movq 24(%1), %%mm3   ;\n"
+	"       movq %%mm1,  8(%1)   ;\n"
+	"       pxor 16(%2), %%mm2   ;\n"
+	"       movq 32(%1), %%mm4   ;\n"
+	"       movq %%mm2, 16(%1)   ;\n"
+	"       pxor 24(%2), %%mm3   ;\n"
+	"       movq 40(%1), %%mm5   ;\n"
+	"       movq %%mm3, 24(%1)   ;\n"
+	"       pxor 32(%2), %%mm4   ;\n"
+	"       movq 48(%1), %%mm6   ;\n"
+	"       movq %%mm4, 32(%1)   ;\n"
+	"       pxor 40(%2), %%mm5   ;\n"
+	"       movq 56(%1), %%mm7   ;\n"
+	"       movq %%mm5, 40(%1)   ;\n"
+	"       pxor 48(%2), %%mm6   ;\n"
+	"       pxor 56(%2), %%mm7   ;\n"
+	"       movq %%mm6, 48(%1)   ;\n"
+	"       movq %%mm7, 56(%1)   ;\n"
+
+	"       addl $64, %1         ;\n"
+	"       addl $64, %2         ;\n"
+	"       decl %0              ;\n"
+	"       jnz 1b               ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2)
+	:
+	: "memory");
+}
+
+static void
+xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
+	     const unsigned long * __restrict p2,
+	     const unsigned long * __restrict p3)
+{
+	unsigned long lines = bytes >> 6;
+
+	asm volatile(
+	" .align 32,0x90             ;\n"
+	" 1:                         ;\n"
+	"       movq   (%1), %%mm0   ;\n"
+	"       movq  8(%1), %%mm1   ;\n"
+	"       pxor   (%2), %%mm0   ;\n"
+	"       movq 16(%1), %%mm2   ;\n"
+	"       pxor  8(%2), %%mm1   ;\n"
+	"       pxor   (%3), %%mm0   ;\n"
+	"       pxor 16(%2), %%mm2   ;\n"
+	"       movq %%mm0,   (%1)   ;\n"
+	"       pxor  8(%3), %%mm1   ;\n"
+	"       pxor 16(%3), %%mm2   ;\n"
+	"       movq 24(%1), %%mm3   ;\n"
+	"       movq %%mm1,  8(%1)   ;\n"
+	"       movq 32(%1), %%mm4   ;\n"
+	"       movq 40(%1), %%mm5   ;\n"
+	"       pxor 24(%2), %%mm3   ;\n"
+	"       movq %%mm2, 16(%1)   ;\n"
+	"       pxor 32(%2), %%mm4   ;\n"
+	"       pxor 24(%3), %%mm3   ;\n"
+	"       pxor 40(%2), %%mm5   ;\n"
+	"       movq %%mm3, 24(%1)   ;\n"
+	"       pxor 32(%3), %%mm4   ;\n"
+	"       pxor 40(%3), %%mm5   ;\n"
+	"       movq 48(%1), %%mm6   ;\n"
+	"       movq %%mm4, 32(%1)   ;\n"
+	"       movq 56(%1), %%mm7   ;\n"
+	"       pxor 48(%2), %%mm6   ;\n"
+	"       movq %%mm5, 40(%1)   ;\n"
+	"       pxor 56(%2), %%mm7   ;\n"
+	"       pxor 48(%3), %%mm6   ;\n"
+	"       pxor 56(%3), %%mm7   ;\n"
+	"       movq %%mm6, 48(%1)   ;\n"
+	"       movq %%mm7, 56(%1)   ;\n"
+
+	"       addl $64, %1         ;\n"
+	"       addl $64, %2         ;\n"
+	"       addl $64, %3         ;\n"
+	"       decl %0              ;\n"
+	"       jnz 1b               ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3)
+	:
+	: "memory" );
+}
+
+static void
+xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
+	     const unsigned long * __restrict p2,
+	     const unsigned long * __restrict p3,
+	     const unsigned long * __restrict p4)
+{
+	unsigned long lines = bytes >> 6;
+
+	asm volatile(
+	" .align 32,0x90             ;\n"
+	" 1:                         ;\n"
+	"       movq   (%1), %%mm0   ;\n"
+	"       movq  8(%1), %%mm1   ;\n"
+	"       pxor   (%2), %%mm0   ;\n"
+	"       movq 16(%1), %%mm2   ;\n"
+	"       pxor  8(%2), %%mm1   ;\n"
+	"       pxor   (%3), %%mm0   ;\n"
+	"       pxor 16(%2), %%mm2   ;\n"
+	"       pxor  8(%3), %%mm1   ;\n"
+	"       pxor   (%4), %%mm0   ;\n"
+	"       movq 24(%1), %%mm3   ;\n"
+	"       pxor 16(%3), %%mm2   ;\n"
+	"       pxor  8(%4), %%mm1   ;\n"
+	"       movq %%mm0,   (%1)   ;\n"
+	"       movq 32(%1), %%mm4   ;\n"
+	"       pxor 24(%2), %%mm3   ;\n"
+	"       pxor 16(%4), %%mm2   ;\n"
+	"       movq %%mm1,  8(%1)   ;\n"
+	"       movq 40(%1), %%mm5   ;\n"
+	"       pxor 32(%2), %%mm4   ;\n"
+	"       pxor 24(%3), %%mm3   ;\n"
+	"       movq %%mm2, 16(%1)   ;\n"
+	"       pxor 40(%2), %%mm5   ;\n"
+	"       pxor 32(%3), %%mm4   ;\n"
+	"       pxor 24(%4), %%mm3   ;\n"
+	"       movq %%mm3, 24(%1)   ;\n"
+	"       movq 56(%1), %%mm7   ;\n"
+	"       movq 48(%1), %%mm6   ;\n"
+	"       pxor 40(%3), %%mm5   ;\n"
+	"       pxor 32(%4), %%mm4   ;\n"
+	"       pxor 48(%2), %%mm6   ;\n"
+	"       movq %%mm4, 32(%1)   ;\n"
+	"       pxor 56(%2), %%mm7   ;\n"
+	"       pxor 40(%4), %%mm5   ;\n"
+	"       pxor 48(%3), %%mm6   ;\n"
+	"       pxor 56(%3), %%mm7   ;\n"
+	"       movq %%mm5, 40(%1)   ;\n"
+	"       pxor 48(%4), %%mm6   ;\n"
+	"       pxor 56(%4), %%mm7   ;\n"
+	"       movq %%mm6, 48(%1)   ;\n"
+	"       movq %%mm7, 56(%1)   ;\n"
+
+	"       addl $64, %1         ;\n"
+	"       addl $64, %2         ;\n"
+	"       addl $64, %3         ;\n"
+	"       addl $64, %4         ;\n"
+	"       decl %0              ;\n"
+	"       jnz 1b               ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
+	:
+	: "memory");
+}
+
+static void
+xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
+	     const unsigned long * __restrict p2,
+	     const unsigned long * __restrict p3,
+	     const unsigned long * __restrict p4,
+	     const unsigned long * __restrict p5)
+{
+	unsigned long lines = bytes >> 6;
+
+	/* Make sure GCC forgets anything it knows about p4 or p5,
+	   such that it won't pass to the asm volatile below a
+	   register that is shared with any other variable.  That's
+	   because we modify p4 and p5 there, but we can't mark them
+	   as read/write, otherwise we'd overflow the 10-asm-operands
+	   limit of GCC < 3.1.  */
+	asm("" : "+r" (p4), "+r" (p5));
+
+	asm volatile(
+	" .align 32,0x90             ;\n"
+	" 1:                         ;\n"
+	"       movq   (%1), %%mm0   ;\n"
+	"       movq  8(%1), %%mm1   ;\n"
+	"       pxor   (%2), %%mm0   ;\n"
+	"       pxor  8(%2), %%mm1   ;\n"
+	"       movq 16(%1), %%mm2   ;\n"
+	"       pxor   (%3), %%mm0   ;\n"
+	"       pxor  8(%3), %%mm1   ;\n"
+	"       pxor 16(%2), %%mm2   ;\n"
+	"       pxor   (%4), %%mm0   ;\n"
+	"       pxor  8(%4), %%mm1   ;\n"
+	"       pxor 16(%3), %%mm2   ;\n"
+	"       movq 24(%1), %%mm3   ;\n"
+	"       pxor   (%5), %%mm0   ;\n"
+	"       pxor  8(%5), %%mm1   ;\n"
+	"       movq %%mm0,   (%1)   ;\n"
+	"       pxor 16(%4), %%mm2   ;\n"
+	"       pxor 24(%2), %%mm3   ;\n"
+	"       movq %%mm1,  8(%1)   ;\n"
+	"       pxor 16(%5), %%mm2   ;\n"
+	"       pxor 24(%3), %%mm3   ;\n"
+	"       movq 32(%1), %%mm4   ;\n"
+	"       movq %%mm2, 16(%1)   ;\n"
+	"       pxor 24(%4), %%mm3   ;\n"
+	"       pxor 32(%2), %%mm4   ;\n"
+	"       movq 40(%1), %%mm5   ;\n"
+	"       pxor 24(%5), %%mm3   ;\n"
+	"       pxor 32(%3), %%mm4   ;\n"
+	"       pxor 40(%2), %%mm5   ;\n"
+	"       movq %%mm3, 24(%1)   ;\n"
+	"       pxor 32(%4), %%mm4   ;\n"
+	"       pxor 40(%3), %%mm5   ;\n"
+	"       movq 48(%1), %%mm6   ;\n"
+	"       movq 56(%1), %%mm7   ;\n"
+	"       pxor 32(%5), %%mm4   ;\n"
+	"       pxor 40(%4), %%mm5   ;\n"
+	"       pxor 48(%2), %%mm6   ;\n"
+	"       pxor 56(%2), %%mm7   ;\n"
+	"       movq %%mm4, 32(%1)   ;\n"
+	"       pxor 48(%3), %%mm6   ;\n"
+	"       pxor 56(%3), %%mm7   ;\n"
+	"       pxor 40(%5), %%mm5   ;\n"
+	"       pxor 48(%4), %%mm6   ;\n"
+	"       pxor 56(%4), %%mm7   ;\n"
+	"       movq %%mm5, 40(%1)   ;\n"
+	"       pxor 48(%5), %%mm6   ;\n"
+	"       pxor 56(%5), %%mm7   ;\n"
+	"       movq %%mm6, 48(%1)   ;\n"
+	"       movq %%mm7, 56(%1)   ;\n"
+
+	"       addl $64, %1         ;\n"
+	"       addl $64, %2         ;\n"
+	"       addl $64, %3         ;\n"
+	"       addl $64, %4         ;\n"
+	"       addl $64, %5         ;\n"
+	"       decl %0              ;\n"
+	"       jnz 1b               ;\n"
+	: "+r" (lines),
+	  "+r" (p1), "+r" (p2), "+r" (p3)
+	: "r" (p4), "r" (p5)
+	: "memory");
+
+	/* p4 and p5 were modified, and now the variables are dead.
+	   Clobber them just to be sure nobody does something stupid
+	   like assuming they have some legal value.  */
+	asm("" : "=r" (p4), "=r" (p5));
+}
+
+DO_XOR_BLOCKS(pII_mmx_inner, xor_pII_mmx_2, xor_pII_mmx_3, xor_pII_mmx_4,
+		xor_pII_mmx_5);
+
+static void xor_gen_pII_mmx(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_pII_mmx_inner(dest, srcs, src_cnt, bytes);
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_pII_mmx = {
+	.name		= "pII_mmx",
+	.xor_gen	= xor_gen_pII_mmx,
+};
+
+DO_XOR_BLOCKS(p5_mmx_inner, xor_p5_mmx_2, xor_p5_mmx_3, xor_p5_mmx_4,
+		xor_p5_mmx_5);
+
+static void xor_gen_p5_mmx(void *dest, void **srcs, unsigned int src_cnt,
+		unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_p5_mmx_inner(dest, srcs, src_cnt, bytes);
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_p5_mmx = {
+	.name		= "p5_mmx",
+	.xor_gen	= xor_gen_p5_mmx,
+};
diff --git a/lib/raid/xor/x86/xor-sse.c b/lib/raid/xor/x86/xor-sse.c
new file mode 100644
index 0000000000000..c6626ecae6ba5
--- /dev/null
+++ b/lib/raid/xor/x86/xor-sse.c
@@ -0,0 +1,459 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Optimized XOR parity functions for SSE.
+ *
+ * Cache avoiding checksumming functions utilizing KNI instructions
+ * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
+ *
+ * Based on
+ * High-speed RAID5 checksumming functions utilizing SSE instructions.
+ * Copyright (C) 1998 Ingo Molnar.
+ *
+ * x86-64 changes / gcc fixes from Andi Kleen.
+ * Copyright 2002 Andi Kleen, SuSE Labs.
+ */
+#include <asm/fpu/api.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+#ifdef CONFIG_X86_32
+/* reduce register pressure */
+# define XOR_CONSTANT_CONSTRAINT "i"
+#else
+# define XOR_CONSTANT_CONSTRAINT "re"
+#endif
+
+#define OFFS(x)		"16*("#x")"
+#define PF_OFFS(x)	"256+16*("#x")"
+#define PF0(x)		"	prefetchnta "PF_OFFS(x)"(%[p1])		;\n"
+#define LD(x, y)	"	movaps "OFFS(x)"(%[p1]), %%xmm"#y"	;\n"
+#define ST(x, y)	"	movaps %%xmm"#y", "OFFS(x)"(%[p1])	;\n"
+#define PF1(x)		"	prefetchnta "PF_OFFS(x)"(%[p2])		;\n"
+#define PF2(x)		"	prefetchnta "PF_OFFS(x)"(%[p3])		;\n"
+#define PF3(x)		"	prefetchnta "PF_OFFS(x)"(%[p4])		;\n"
+#define PF4(x)		"	prefetchnta "PF_OFFS(x)"(%[p5])		;\n"
+#define XO1(x, y)	"	xorps "OFFS(x)"(%[p2]), %%xmm"#y"	;\n"
+#define XO2(x, y)	"	xorps "OFFS(x)"(%[p3]), %%xmm"#y"	;\n"
+#define XO3(x, y)	"	xorps "OFFS(x)"(%[p4]), %%xmm"#y"	;\n"
+#define XO4(x, y)	"	xorps "OFFS(x)"(%[p5]), %%xmm"#y"	;\n"
+#define NOP(x)
+
+#define BLK64(pf, op, i)				\
+		pf(i)					\
+		op(i, 0)				\
+			op(i + 1, 1)			\
+				op(i + 2, 2)		\
+					op(i + 3, 3)
+
+static void
+xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
+	  const unsigned long * __restrict p2)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)					\
+		LD(i, 0)				\
+			LD(i + 1, 1)			\
+		PF1(i)					\
+				PF1(i + 2)		\
+				LD(i + 2, 2)		\
+					LD(i + 3, 3)	\
+		PF0(i + 4)				\
+				PF0(i + 6)		\
+		XO1(i, 0)				\
+			XO1(i + 1, 1)			\
+				XO1(i + 2, 2)		\
+					XO1(i + 3, 3)	\
+		ST(i, 0)				\
+			ST(i + 1, 1)			\
+				ST(i + 2, 2)		\
+					ST(i + 3, 3)	\
+
+
+		PF0(0)
+				PF0(2)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines),
+	  [p1] "+r" (p1), [p2] "+r" (p2)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)			\
+		BLK64(PF0, LD, i)	\
+		BLK64(PF1, XO1, i)	\
+		BLK64(NOP, ST, i)	\
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines),
+	  [p1] "+r" (p1), [p2] "+r" (p2)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
+	  const unsigned long * __restrict p2,
+	  const unsigned long * __restrict p3)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+		PF1(i)					\
+				PF1(i + 2)		\
+		LD(i, 0)				\
+			LD(i + 1, 1)			\
+				LD(i + 2, 2)		\
+					LD(i + 3, 3)	\
+		PF2(i)					\
+				PF2(i + 2)		\
+		PF0(i + 4)				\
+				PF0(i + 6)		\
+		XO1(i, 0)				\
+			XO1(i + 1, 1)			\
+				XO1(i + 2, 2)		\
+					XO1(i + 3, 3)	\
+		XO2(i, 0)				\
+			XO2(i + 1, 1)			\
+				XO2(i + 2, 2)		\
+					XO2(i + 3, 3)	\
+		ST(i, 0)				\
+			ST(i + 1, 1)			\
+				ST(i + 2, 2)		\
+					ST(i + 3, 3)	\
+
+
+		PF0(0)
+				PF0(2)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines),
+	  [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)			\
+		BLK64(PF0, LD, i)	\
+		BLK64(PF1, XO1, i)	\
+		BLK64(PF2, XO2, i)	\
+		BLK64(NOP, ST, i)	\
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines),
+	  [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
+	  const unsigned long * __restrict p2,
+	  const unsigned long * __restrict p3,
+	  const unsigned long * __restrict p4)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+		PF1(i)					\
+				PF1(i + 2)		\
+		LD(i, 0)				\
+			LD(i + 1, 1)			\
+				LD(i + 2, 2)		\
+					LD(i + 3, 3)	\
+		PF2(i)					\
+				PF2(i + 2)		\
+		XO1(i, 0)				\
+			XO1(i + 1, 1)			\
+				XO1(i + 2, 2)		\
+					XO1(i + 3, 3)	\
+		PF3(i)					\
+				PF3(i + 2)		\
+		PF0(i + 4)				\
+				PF0(i + 6)		\
+		XO2(i, 0)				\
+			XO2(i + 1, 1)			\
+				XO2(i + 2, 2)		\
+					XO2(i + 3, 3)	\
+		XO3(i, 0)				\
+			XO3(i + 1, 1)			\
+				XO3(i + 2, 2)		\
+					XO3(i + 3, 3)	\
+		ST(i, 0)				\
+			ST(i + 1, 1)			\
+				ST(i + 2, 2)		\
+					ST(i + 3, 3)	\
+
+
+		PF0(0)
+				PF0(2)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       add %[inc], %[p4]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines), [p1] "+r" (p1),
+	  [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)			\
+		BLK64(PF0, LD, i)	\
+		BLK64(PF1, XO1, i)	\
+		BLK64(PF2, XO2, i)	\
+		BLK64(PF3, XO3, i)	\
+		BLK64(NOP, ST, i)	\
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       add %[inc], %[p4]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines), [p1] "+r" (p1),
+	  [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
+	  const unsigned long * __restrict p2,
+	  const unsigned long * __restrict p3,
+	  const unsigned long * __restrict p4,
+	  const unsigned long * __restrict p5)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+		PF1(i)					\
+				PF1(i + 2)		\
+		LD(i, 0)				\
+			LD(i + 1, 1)			\
+				LD(i + 2, 2)		\
+					LD(i + 3, 3)	\
+		PF2(i)					\
+				PF2(i + 2)		\
+		XO1(i, 0)				\
+			XO1(i + 1, 1)			\
+				XO1(i + 2, 2)		\
+					XO1(i + 3, 3)	\
+		PF3(i)					\
+				PF3(i + 2)		\
+		XO2(i, 0)				\
+			XO2(i + 1, 1)			\
+				XO2(i + 2, 2)		\
+					XO2(i + 3, 3)	\
+		PF4(i)					\
+				PF4(i + 2)		\
+		PF0(i + 4)				\
+				PF0(i + 6)		\
+		XO3(i, 0)				\
+			XO3(i + 1, 1)			\
+				XO3(i + 2, 2)		\
+					XO3(i + 3, 3)	\
+		XO4(i, 0)				\
+			XO4(i + 1, 1)			\
+				XO4(i + 2, 2)		\
+					XO4(i + 3, 3)	\
+		ST(i, 0)				\
+			ST(i + 1, 1)			\
+				ST(i + 2, 2)		\
+					ST(i + 3, 3)	\
+
+
+		PF0(0)
+				PF0(2)
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       add %[inc], %[p4]       ;\n"
+	"       add %[inc], %[p5]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+	  [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+static void
+xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4,
+	       const unsigned long * __restrict p5)
+{
+	unsigned long lines = bytes >> 8;
+
+	asm volatile(
+#undef BLOCK
+#define BLOCK(i)			\
+		BLK64(PF0, LD, i)	\
+		BLK64(PF1, XO1, i)	\
+		BLK64(PF2, XO2, i)	\
+		BLK64(PF3, XO3, i)	\
+		BLK64(PF4, XO4, i)	\
+		BLK64(NOP, ST, i)	\
+
+	" .align 32			;\n"
+	" 1:                            ;\n"
+
+		BLOCK(0)
+		BLOCK(4)
+		BLOCK(8)
+		BLOCK(12)
+
+	"       add %[inc], %[p1]       ;\n"
+	"       add %[inc], %[p2]       ;\n"
+	"       add %[inc], %[p3]       ;\n"
+	"       add %[inc], %[p4]       ;\n"
+	"       add %[inc], %[p5]       ;\n"
+	"       dec %[cnt]              ;\n"
+	"       jnz 1b                  ;\n"
+	: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+	  [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+	: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+	: "memory");
+}
+
+DO_XOR_BLOCKS(sse_inner, xor_sse_2, xor_sse_3, xor_sse_4, xor_sse_5);
+
+static void xor_gen_sse(void *dest, void **srcs, unsigned int src_cnt,
+			unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_sse_inner(dest, srcs, src_cnt, bytes);
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_sse = {
+	.name		= "sse",
+	.xor_gen	= xor_gen_sse,
+};
+
+DO_XOR_BLOCKS(sse_pf64_inner, xor_sse_2_pf64, xor_sse_3_pf64, xor_sse_4_pf64,
+		xor_sse_5_pf64);
+
+static void xor_gen_sse_pf64(void *dest, void **srcs, unsigned int src_cnt,
+			unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_sse_pf64_inner(dest, srcs, src_cnt, bytes);
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_sse_pf64 = {
+	.name		= "prefetch64-sse",
+	.xor_gen	= xor_gen_sse_pf64,
+};
diff --git a/lib/raid/xor/x86/xor_arch.h b/lib/raid/xor/x86/xor_arch.h
new file mode 100644
index 0000000000000..99fe85a213c66
--- /dev/null
+++ b/lib/raid/xor/x86/xor_arch.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#include <asm/cpufeature.h>
+
+extern struct xor_block_template xor_block_pII_mmx;
+extern struct xor_block_template xor_block_p5_mmx;
+extern struct xor_block_template xor_block_sse;
+extern struct xor_block_template xor_block_sse_pf64;
+extern struct xor_block_template xor_block_avx;
+
+/*
+ * When SSE is available, use it as it can write around L2.  We may also be able
+ * to load into the L1 only depending on how the cpu deals with a load to a line
+ * that is being prefetched.
+ *
+ * When AVX2 is available, force using it as it is better by all measures.
+ *
+ * 32-bit without MMX can fall back to the generic routines.
+ */
+static __always_inline void __init arch_xor_init(void)
+{
+	if (boot_cpu_has(X86_FEATURE_AVX) &&
+	    boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+		xor_force(&xor_block_avx);
+	} else if (IS_ENABLED(CONFIG_X86_64) || boot_cpu_has(X86_FEATURE_XMM)) {
+		xor_register(&xor_block_sse);
+		xor_register(&xor_block_sse_pf64);
+	} else if (boot_cpu_has(X86_FEATURE_MMX)) {
+		xor_register(&xor_block_pII_mmx);
+		xor_register(&xor_block_p5_mmx);
+	} else {
+		xor_register(&xor_block_8regs);
+		xor_register(&xor_block_8regs_p);
+		xor_register(&xor_block_32regs);
+		xor_register(&xor_block_32regs_p);
+	}
+}
diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c
new file mode 100644
index 0000000000000..ade2a7d8cbe2a
--- /dev/null
+++ b/lib/raid/xor/xor-32regs-prefetch.c
@@ -0,0 +1,267 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/prefetch.h>
+#include "xor_impl.h"
+
+static void
+xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2)
+{
+	long lines = bytes / (sizeof (long)) / 8 - 1;
+
+	prefetchw(p1);
+	prefetch(p2);
+
+	do {
+		register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+		prefetchw(p1+8);
+		prefetch(p2+8);
+ once_more:
+		d0 = p1[0];	/* Pull the stuff into registers	*/
+		d1 = p1[1];	/*  ... in bursts, if possible.		*/
+		d2 = p1[2];
+		d3 = p1[3];
+		d4 = p1[4];
+		d5 = p1[5];
+		d6 = p1[6];
+		d7 = p1[7];
+		d0 ^= p2[0];
+		d1 ^= p2[1];
+		d2 ^= p2[2];
+		d3 ^= p2[3];
+		d4 ^= p2[4];
+		d5 ^= p2[5];
+		d6 ^= p2[6];
+		d7 ^= p2[7];
+		p1[0] = d0;	/* Store the result (in bursts)		*/
+		p1[1] = d1;
+		p1[2] = d2;
+		p1[3] = d3;
+		p1[4] = d4;
+		p1[5] = d5;
+		p1[6] = d6;
+		p1[7] = d7;
+		p1 += 8;
+		p2 += 8;
+	} while (--lines > 0);
+	if (lines == 0)
+		goto once_more;
+}
+
+static void
+xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3)
+{
+	long lines = bytes / (sizeof (long)) / 8 - 1;
+
+	prefetchw(p1);
+	prefetch(p2);
+	prefetch(p3);
+
+	do {
+		register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+		prefetchw(p1+8);
+		prefetch(p2+8);
+		prefetch(p3+8);
+ once_more:
+		d0 = p1[0];	/* Pull the stuff into registers	*/
+		d1 = p1[1];	/*  ... in bursts, if possible.		*/
+		d2 = p1[2];
+		d3 = p1[3];
+		d4 = p1[4];
+		d5 = p1[5];
+		d6 = p1[6];
+		d7 = p1[7];
+		d0 ^= p2[0];
+		d1 ^= p2[1];
+		d2 ^= p2[2];
+		d3 ^= p2[3];
+		d4 ^= p2[4];
+		d5 ^= p2[5];
+		d6 ^= p2[6];
+		d7 ^= p2[7];
+		d0 ^= p3[0];
+		d1 ^= p3[1];
+		d2 ^= p3[2];
+		d3 ^= p3[3];
+		d4 ^= p3[4];
+		d5 ^= p3[5];
+		d6 ^= p3[6];
+		d7 ^= p3[7];
+		p1[0] = d0;	/* Store the result (in bursts)		*/
+		p1[1] = d1;
+		p1[2] = d2;
+		p1[3] = d3;
+		p1[4] = d4;
+		p1[5] = d5;
+		p1[6] = d6;
+		p1[7] = d7;
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+	} while (--lines > 0);
+	if (lines == 0)
+		goto once_more;
+}
+
+static void
+xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4)
+{
+	long lines = bytes / (sizeof (long)) / 8 - 1;
+
+	prefetchw(p1);
+	prefetch(p2);
+	prefetch(p3);
+	prefetch(p4);
+
+	do {
+		register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+		prefetchw(p1+8);
+		prefetch(p2+8);
+		prefetch(p3+8);
+		prefetch(p4+8);
+ once_more:
+		d0 = p1[0];	/* Pull the stuff into registers	*/
+		d1 = p1[1];	/*  ... in bursts, if possible.		*/
+		d2 = p1[2];
+		d3 = p1[3];
+		d4 = p1[4];
+		d5 = p1[5];
+		d6 = p1[6];
+		d7 = p1[7];
+		d0 ^= p2[0];
+		d1 ^= p2[1];
+		d2 ^= p2[2];
+		d3 ^= p2[3];
+		d4 ^= p2[4];
+		d5 ^= p2[5];
+		d6 ^= p2[6];
+		d7 ^= p2[7];
+		d0 ^= p3[0];
+		d1 ^= p3[1];
+		d2 ^= p3[2];
+		d3 ^= p3[3];
+		d4 ^= p3[4];
+		d5 ^= p3[5];
+		d6 ^= p3[6];
+		d7 ^= p3[7];
+		d0 ^= p4[0];
+		d1 ^= p4[1];
+		d2 ^= p4[2];
+		d3 ^= p4[3];
+		d4 ^= p4[4];
+		d5 ^= p4[5];
+		d6 ^= p4[6];
+		d7 ^= p4[7];
+		p1[0] = d0;	/* Store the result (in bursts)		*/
+		p1[1] = d1;
+		p1[2] = d2;
+		p1[3] = d3;
+		p1[4] = d4;
+		p1[5] = d5;
+		p1[6] = d6;
+		p1[7] = d7;
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+		p4 += 8;
+	} while (--lines > 0);
+	if (lines == 0)
+		goto once_more;
+}
+
+static void
+xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
+	       const unsigned long * __restrict p2,
+	       const unsigned long * __restrict p3,
+	       const unsigned long * __restrict p4,
+	       const unsigned long * __restrict p5)
+{
+	long lines = bytes / (sizeof (long)) / 8 - 1;
+
+	prefetchw(p1);
+	prefetch(p2);
+	prefetch(p3);
+	prefetch(p4);
+	prefetch(p5);
+
+	do {
+		register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+		prefetchw(p1+8);
+		prefetch(p2+8);
+		prefetch(p3+8);
+		prefetch(p4+8);
+		prefetch(p5+8);
+ once_more:
+		d0 = p1[0];	/* Pull the stuff into registers	*/
+		d1 = p1[1];	/*  ... in bursts, if possible.		*/
+		d2 = p1[2];
+		d3 = p1[3];
+		d4 = p1[4];
+		d5 = p1[5];
+		d6 = p1[6];
+		d7 = p1[7];
+		d0 ^= p2[0];
+		d1 ^= p2[1];
+		d2 ^= p2[2];
+		d3 ^= p2[3];
+		d4 ^= p2[4];
+		d5 ^= p2[5];
+		d6 ^= p2[6];
+		d7 ^= p2[7];
+		d0 ^= p3[0];
+		d1 ^= p3[1];
+		d2 ^= p3[2];
+		d3 ^= p3[3];
+		d4 ^= p3[4];
+		d5 ^= p3[5];
+		d6 ^= p3[6];
+		d7 ^= p3[7];
+		d0 ^= p4[0];
+		d1 ^= p4[1];
+		d2 ^= p4[2];
+		d3 ^= p4[3];
+		d4 ^= p4[4];
+		d5 ^= p4[5];
+		d6 ^= p4[6];
+		d7 ^= p4[7];
+		d0 ^= p5[0];
+		d1 ^= p5[1];
+		d2 ^= p5[2];
+		d3 ^= p5[3];
+		d4 ^= p5[4];
+		d5 ^= p5[5];
+		d6 ^= p5[6];
+		d7 ^= p5[7];
+		p1[0] = d0;	/* Store the result (in bursts)		*/
+		p1[1] = d1;
+		p1[2] = d2;
+		p1[3] = d3;
+		p1[4] = d4;
+		p1[5] = d5;
+		p1[6] = d6;
+		p1[7] = d7;
+		p1 += 8;
+		p2 += 8;
+		p3 += 8;
+		p4 += 8;
+		p5 += 8;
+	} while (--lines > 0);
+	if (lines == 0)
+		goto once_more;
+}
+
+DO_XOR_BLOCKS(32regs_p, xor_32regs_p_2, xor_32regs_p_3, xor_32regs_p_4,
+		xor_32regs_p_5);
+
+struct xor_block_template xor_block_32regs_p = {
+	.name		= "32regs_prefetch",
+	.xor_gen	= xor_gen_32regs_p,
+};
diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c
new file mode 100644
index 0000000000000..759a31f764148
--- /dev/null
+++ b/lib/raid/xor/xor-32regs.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-or-l